| 20260505 03:30:20.577 | INFO | /opt/pacbio/bin/smrt-basecaller --process-log-interval=5000 --process-log=NONE --config=traceSaver.roi=[[2048,768,1,64],[2048,2304,1,64],[2048,3840,1,64],[2048,5376,1,64]]@bazWriter.samplingParams.filenames[0]=/data/nrta/2/m84031_260504_231011_s1/pb_internal/baz/m84031_260504_231011_s1_basic_preview.baz@bazWriter.samplingParams.filenames[1]=/data/nrta/2/m84031_260504_231011_s1/pb_internal/baz/m84031_260504_231011_s1_full_preview.baz@bazWriter.samplingParams.presets[0]=BasicPreview@bazWriter.samplingParams.presets[1]=FullPreview@realTimeMetrics.jsonOutputFile=/data/nrta/2/m84031_260504_231011_s1/pb_internal/m84031_260504_231011_s1.rtmetrics.json@/data/icc/m84031_260504_231011_s1/pb_internal/m84031_260504_231011_s1.basecaller.config.json@source.WXIPCDataSourceConfig.sraIndex=0@maskFraction=1.0000000@bazIO.MaxMiBToWrite=12874602@dataSource.darkCalFileName=/data/icc/m84031_260504_231011_s1/pb_internal/m84031_260504_231011_s1.darkcal.h5@dataSource.imagePsfKernel=[[0.0008,0.003,0.0166,0.0035,0.0011],[0.0027,0.013,0.0727,0.014,0.0032],[0.0118,0.0545,0.5944,0.0653,0.0124],[0.0032,0.0131,0.0727,0.0141,0.0031],[0.001,0.0029,0.0168,0.0034,0.0009]]@system.analyzerHardware=A100 --statusfd=2 --outputbazfile=/data/nrta/2/m84031_260504_231011_s1/pb_internal/baz/m84031_260504_231011_s1.bazfilelist.txt --outputtrcfile=/data/icc/m84031_260504_231011_s1/pb_internal/m84031_260504_231011_s1.trc.h5 --numWorkerThreads=0 --nop=0 --maxFrames=8640000 --log-level=INFO --log-file=/data/icc/m84031_260504_231011_s1/metadata/m84031_260504_231011_s1.basecaller.log,/var/log/pacbio/basecaller/m84031_260504_231011_s1.basecaller --log-directory-max-size=1073741820 | 20260505 03:30:20.577 | INFO | Version 26.1.0 | 20260505 03:30:20.577 | INFO | git branch: release/kestrel-13.5 | 20260505 03:30:20.577 | INFO | git commit hash: 2dc54f5a66f39b94662202d5514157d553e25f7d | 20260505 03:30:20.577 | INFO | git commit date: 2026-01-13 11:49:20 +0100 | 20260505 03:30:20.578 | INFO | { | 20260505 03:30:20.578 | INFO | "algorithm" : | 20260505 03:30:20.578 | INFO | { | 20260505 03:30:20.578 | INFO | "Metrics" : | 20260505 03:30:20.578 | INFO | { | 20260505 03:30:20.578 | INFO | "Method" : "Gpu", | 20260505 03:30:20.578 | INFO | "framesPerHFMetricBlock" : 4096, | 20260505 03:30:20.578 | INFO | "generateFeedbackMetrics" : true, | 20260505 03:30:20.578 | INFO | "realtimeActivityLabels" : true, | 20260505 03:30:20.578 | INFO | "sandwichTolerance" : 0 | 20260505 03:30:20.578 | INFO | }, | 20260505 03:30:20.578 | INFO | "baselineStatsAggregatorConfig" : | 20260505 03:30:20.578 | INFO | { | 20260505 03:30:20.578 | INFO | "Method" : "Gpu" | 20260505 03:30:20.578 | INFO | }, | 20260505 03:30:20.578 | INFO | "baselinerConfig" : | 20260505 03:30:20.578 | INFO | { | 20260505 03:30:20.578 | INFO | "Filter" : "MultiScaleLarge", | 20260505 03:30:20.578 | INFO | "JumpSuppression" : 0, | 20260505 03:30:20.578 | INFO | "MeanBiasAdjust" : 0, | 20260505 03:30:20.578 | INFO | "MeanEmaScaleStrides" : 0, | 20260505 03:30:20.578 | INFO | "Method" : "DeviceMultiScale", | 20260505 03:30:20.578 | INFO | "SigmaBiasAdjust" : 0, | 20260505 03:30:20.578 | INFO | "SigmaEmaScaleStrides" : 512 | 20260505 03:30:20.578 | INFO | }, | 20260505 03:30:20.578 | INFO | "dmeConfig" : | 20260505 03:30:20.578 | INFO | { | 20260505 03:30:20.578 | INFO | "AddPrelimEstimateConfidence" : true, | 20260505 03:30:20.578 | INFO | "AmplitudeFeedbackDampingFactor" : 0.89999997615814209, | 20260505 03:30:20.578 | INFO | "AmplitudeFeedbackInitSampleCount" : 2000, | 20260505 03:30:20.578 | INFO | "AnalogMixFractionThreshold" : | 20260505 03:30:20.578 | INFO | [ | 20260505 03:30:20.578 | INFO | 0.013000000268220901, | 20260505 03:30:20.578 | INFO | 0.039000000804662704 | 20260505 03:30:20.578 | INFO | ], | 20260505 03:30:20.578 | INFO | "ConfidenceHalfLife" : | 20260505 03:30:20.578 | INFO | [ | 20260505 03:30:20.578 | INFO | 4096, | 20260505 03:30:20.578 | INFO | 12000 | 20260505 03:30:20.578 | INFO | ], | 20260505 03:30:20.578 | INFO | "ConfidenceHalfLifePauseEnhance" : 0, | 20260505 03:30:20.578 | INFO | "ConvergeCoeff" : 2.9999999242136255e-05, | 20260505 03:30:20.579 | INFO | "EmIterationLimit" : 20, | 20260505 03:30:20.579 | INFO | "EnableAmplitudeRatioFeedback" : true, | 20260505 03:30:20.579 | INFO | "EnableFeedbackMultiloadAdjust" : false, | 20260505 03:30:20.579 | INFO | "EnablePulseVarianceFeedback" : true, | 20260505 03:30:20.579 | INFO | "FeedbackExcessNoiseCoeffDampFactor" : 19, | 20260505 03:30:20.579 | INFO | "FirstAttemptsIterMultiple" : 2, | 20260505 03:30:20.579 | INFO | "GTestStatFactor" : -1, | 20260505 03:30:20.579 | INFO | "GofLogChiSqrThresh1" : 111, | 20260505 03:30:20.579 | INFO | "GofLogChiSqrThresh2" : 8, | 20260505 03:30:20.579 | INFO | "HybridAtol" : 8, | 20260505 03:30:20.579 | INFO | "HybridRtol" : 48, | 20260505 03:30:20.579 | INFO | "IterateToLimit" : false, | 20260505 03:30:20.579 | INFO | "LaserPowerChangeReduceConfidence" : -1, | 20260505 03:30:20.579 | INFO | "MaxFeedbackVarRatio" : 2.5, | 20260505 03:30:20.579 | INFO | "Method" : "EmDevice", | 20260505 03:30:20.579 | INFO | "MinAnalogSnrThresh0" : 2, | 20260505 03:30:20.579 | INFO | "MinAnalogSnrThresh1" : 4, | 20260505 03:30:20.579 | INFO | "MinFeedbackBasesOverThreshold" : 4, | 20260505 03:30:20.579 | INFO | "MinFramesForEstimate" : 4000, | 20260505 03:30:20.579 | INFO | "ModelUpdateWeightMax" : 0.5, | 20260505 03:30:20.579 | INFO | "NumBinsMin" : 500, | 20260505 03:30:20.579 | INFO | "NumFramesPreAccumStats" : 1000, | 20260505 03:30:20.579 | INFO | "PulseAmpRegularization" : 0, | 20260505 03:30:20.579 | INFO | "PureUpdate" : false, | 20260505 03:30:20.579 | INFO | "RtalFeedbackLabelHistorySize" : 5, | 20260505 03:30:20.579 | INFO | "ScaleSnrConfTol" : 1, | 20260505 03:30:20.579 | INFO | "SimModel" : | 20260505 03:30:20.579 | INFO | { | 20260505 03:30:20.579 | INFO | "baselineMean" : 200, | 20260505 03:30:20.579 | INFO | "baselineVar" : 33, | 20260505 03:30:20.579 | INFO | "useSimulatedBaselineParams" : false | 20260505 03:30:20.579 | INFO | }, | 20260505 03:30:20.579 | INFO | "SnrDropThresh" : 1, | 20260505 03:30:20.579 | INFO | "SuccessConfidenceThresh" : 0.10000000149011612, | 20260505 03:30:20.579 | INFO | "ThreshAnalogSNR" : 2.1489999294281006, | 20260505 03:30:20.579 | INFO | "ThreshFractionFeedbackPulses" : 0.12999999523162842, | 20260505 03:30:20.579 | INFO | "XsnArrayCVRatios" : | 20260505 03:30:20.579 | INFO | [ | 20260505 03:30:20.579 | INFO | 0, | 20260505 03:30:20.579 | INFO | 0, | 20260505 03:30:20.579 | INFO | 0, | 20260505 03:30:20.579 | INFO | 0 | 20260505 03:30:20.579 | INFO | ], | 20260505 03:30:20.579 | INFO | "XsnCoeffLstSqWeightPower" : 2, | 20260505 03:30:20.579 | INFO | "XsnCoeffRatioCeil" : 10, | 20260505 03:30:20.579 | INFO | "XsnCoeffRatioFloor" : 0.0099999997764825821 | 20260505 03:30:20.579 | INFO | }, | 20260505 03:30:20.579 | INFO | "frameLabelerConfig" : | 20260505 03:30:20.579 | INFO | { | 20260505 03:30:20.579 | INFO | "Method" : "Device", | 20260505 03:30:20.579 | INFO | "roi" : | 20260505 03:30:20.579 | INFO | { | 20260505 03:30:20.579 | INFO | "filterType" : "Default", | 20260505 03:30:20.579 | INFO | "lowerThreshold" : 2, | 20260505 03:30:20.579 | INFO | "upperThreshold" : 6 | 20260505 03:30:20.579 | INFO | }, | 20260505 03:30:20.579 | INFO | "viterbi" : | 20260505 03:30:20.579 | INFO | { | 20260505 03:30:20.579 | INFO | "alpha" : 1, | 20260505 03:30:20.579 | INFO | "beta" : 1, | 20260505 03:30:20.579 | INFO | "gamma" : 1 | 20260505 03:30:20.579 | INFO | } | 20260505 03:30:20.579 | INFO | }, | 20260505 03:30:20.579 | INFO | "modelEstimationMode" : "DynamicEstimations", | 20260505 03:30:20.579 | INFO | "pulseAccumConfig" : | 20260505 03:30:20.579 | INFO | { | 20260505 03:30:20.579 | INFO | "Method" : "GpuPulses", | 20260505 03:30:20.579 | INFO | "XspAmpThresh" : 0.69999999999999996, | 20260505 03:30:20.579 | INFO | "XspWidthThresh" : 3.5, | 20260505 03:30:20.579 | INFO | "maxPulsePerSec" : 20, | 20260505 03:30:20.579 | INFO | "simConfig" : | 20260505 03:30:20.579 | INFO | { | 20260505 03:30:20.579 | INFO | "basecalls" : | 20260505 03:30:20.579 | INFO | [ | 20260505 03:30:20.579 | INFO | "A", | 20260505 03:30:20.579 | INFO | "C", | 20260505 03:30:20.579 | INFO | "G", | 20260505 03:30:20.579 | INFO | "T" | 20260505 03:30:20.579 | INFO | ], | 20260505 03:30:20.579 | INFO | "ipdVars" : [], | 20260505 03:30:20.579 | INFO | "ipds" : | 20260505 03:30:20.580 | INFO | [ | 20260505 03:30:20.580 | INFO | 6 | 20260505 03:30:20.580 | INFO | ], | 20260505 03:30:20.580 | INFO | "maxSignals" : | 20260505 03:30:20.580 | INFO | [ | 20260505 03:30:20.580 | INFO | 21, | 20260505 03:30:20.580 | INFO | 11, | 20260505 03:30:20.580 | INFO | 17, | 20260505 03:30:20.580 | INFO | 9 | 20260505 03:30:20.580 | INFO | ], | 20260505 03:30:20.580 | INFO | "maxSignalsVars" : [], | 20260505 03:30:20.580 | INFO | "meanSignals" : | 20260505 03:30:20.580 | INFO | [ | 20260505 03:30:20.580 | INFO | 20, | 20260505 03:30:20.580 | INFO | 10, | 20260505 03:30:20.580 | INFO | 16, | 20260505 03:30:20.580 | INFO | 8 | 20260505 03:30:20.580 | INFO | ], | 20260505 03:30:20.580 | INFO | "meanSignalsVars" : [], | 20260505 03:30:20.580 | INFO | "midSignals" : | 20260505 03:30:20.580 | INFO | [ | 20260505 03:30:20.580 | INFO | 21, | 20260505 03:30:20.580 | INFO | 11, | 20260505 03:30:20.580 | INFO | 17, | 20260505 03:30:20.580 | INFO | 9 | 20260505 03:30:20.580 | INFO | ], | 20260505 03:30:20.580 | INFO | "midSignalsVars" : [], | 20260505 03:30:20.580 | INFO | "pwVars" : [], | 20260505 03:30:20.580 | INFO | "pws" : | 20260505 03:30:20.580 | INFO | [ | 20260505 03:30:20.580 | INFO | 5 | 20260505 03:30:20.580 | INFO | ], | 20260505 03:30:20.580 | INFO | "seed" : 0 | 20260505 03:30:20.580 | INFO | } | 20260505 03:30:20.580 | INFO | }, | 20260505 03:30:20.580 | INFO | "pulseSerlzConfig" : | 20260505 03:30:20.580 | INFO | { | 20260505 03:30:20.580 | INFO | "Method" : "GpuPulses", | 20260505 03:30:20.580 | INFO | "PulseCountCoeff" : 2 | 20260505 03:30:20.580 | INFO | }, | 20260505 03:30:20.580 | INFO | "simulatedFaults" : | 20260505 03:30:20.580 | INFO | { | 20260505 03:30:20.580 | INFO | "negativeDyeSpectrumCounter" : 0 | 20260505 03:30:20.580 | INFO | }, | 20260505 03:30:20.580 | INFO | "staticDetModelConfig" : | 20260505 03:30:20.580 | INFO | { | 20260505 03:30:20.580 | INFO | "baselineMean" : 0, | 20260505 03:30:20.580 | INFO | "baselineVariance" : 33 | 20260505 03:30:20.580 | INFO | }, | 20260505 03:30:20.580 | INFO | "traceHistogramConfig" : | 20260505 03:30:20.580 | INFO | { | 20260505 03:30:20.580 | INFO | "BaselineStatMinFrameCount" : 50, | 20260505 03:30:20.580 | INFO | "BinSizeCoeff" : 0.20000000298023224, | 20260505 03:30:20.580 | INFO | "BinSizeLowBoundCoeff" : 1, | 20260505 03:30:20.580 | INFO | "FallBackBaselineSigma" : 10, | 20260505 03:30:20.580 | INFO | "Method" : "Gpu" | 20260505 03:30:20.580 | INFO | } | 20260505 03:30:20.580 | INFO | }, | 20260505 03:30:20.580 | INFO | "bazIO" : | 20260505 03:30:20.580 | INFO | { | 20260505 03:30:20.580 | INFO | "BufferSizeMiBytes" : 4, | 20260505 03:30:20.580 | INFO | "MaxMiBPerSecondWrite" : 450, | 20260505 03:30:20.580 | INFO | "MaxMiBToWrite" : 12874602, | 20260505 03:30:20.580 | INFO | "SyncSizeMiBytes" : 32, | 20260505 03:30:20.580 | INFO | "compression" : | 20260505 03:30:20.580 | INFO | { | 20260505 03:30:20.580 | INFO | "blockSize" : 131072, | 20260505 03:30:20.580 | INFO | "enabled" : true, | 20260505 03:30:20.580 | INFO | "level" : 1 | 20260505 03:30:20.580 | INFO | } | 20260505 03:30:20.580 | INFO | }, | 20260505 03:30:20.580 | INFO | "bazWriter" : | 20260505 03:30:20.580 | INFO | { | 20260505 03:30:20.580 | INFO | "maxChunksQueued" : 2, | 20260505 03:30:20.580 | INFO | "samplingParams" : | 20260505 03:30:20.580 | INFO | { | 20260505 03:30:20.580 | INFO | "filenames" : | 20260505 03:30:20.580 | INFO | [ | 20260505 03:30:20.580 | INFO | "/data/nrta/2/m84031_260504_231011_s1/pb_internal/baz/m84031_260504_231011_s1_basic_preview.baz", | 20260505 03:30:20.580 | INFO | "/data/nrta/2/m84031_260504_231011_s1/pb_internal/baz/m84031_260504_231011_s1_full_preview.baz" | 20260505 03:30:20.580 | INFO | ], | 20260505 03:30:20.580 | INFO | "presets" : | 20260505 03:30:20.581 | INFO | [ | 20260505 03:30:20.581 | INFO | "BasicPreview", | 20260505 03:30:20.581 | INFO | "FullPreview" | 20260505 03:30:20.581 | INFO | ], | 20260505 03:30:20.581 | INFO | "roiList" : | 20260505 03:30:20.581 | INFO | [ | 20260505 03:30:20.581 | INFO | [ | 20260505 03:30:20.581 | INFO | [ | 20260505 03:30:20.581 | INFO | 128, | 20260505 03:30:20.581 | INFO | 128, | 20260505 03:30:20.581 | INFO | 1, | 20260505 03:30:20.581 | INFO | 64 | 20260505 03:30:20.581 | INFO | ], | 20260505 03:30:20.581 | INFO | [ | 20260505 03:30:20.581 | INFO | 384, | 20260505 03:30:20.581 | INFO | 128, | 20260505 03:30:20.581 | INFO | 1, | 20260505 03:30:20.581 | INFO | 64 | 20260505 03:30:20.581 | INFO | ], | 20260505 03:30:20.581 | INFO | [ | 20260505 03:30:20.581 | INFO | 640, | 20260505 03:30:20.581 | INFO | 128, | 20260505 03:30:20.581 | INFO | 1, | 20260505 03:30:20.581 | INFO | 64 | 20260505 03:30:20.581 | INFO | ], | 20260505 03:30:20.581 | INFO | [ | 20260505 03:30:20.581 | INFO | 896, | 20260505 03:30:20.581 | INFO | 128, | 20260505 03:30:20.581 | INFO | 1, | 20260505 03:30:20.581 | INFO | 64 | 20260505 03:30:20.581 | INFO | ], | 20260505 03:30:20.581 | INFO | [ | 20260505 03:30:20.581 | INFO | 1152, | 20260505 03:30:20.581 | INFO | 128, | 20260505 03:30:20.581 | INFO | 1, | 20260505 03:30:20.581 | INFO | 64 | 20260505 03:30:20.581 | INFO | ], | 20260505 03:30:20.581 | INFO | [ | 20260505 03:30:20.581 | INFO | 1408, | 20260505 03:30:20.581 | INFO | 128, | 20260505 03:30:20.581 | INFO | 1, | 20260505 03:30:20.581 | INFO | 64 | 20260505 03:30:20.581 | INFO | ], | 20260505 03:30:20.581 | INFO | [ | 20260505 03:30:20.581 | INFO | 1664, | 20260505 03:30:20.581 | INFO | 128, | 20260505 03:30:20.581 | INFO | 1, | 20260505 03:30:20.581 | INFO | 64 | 20260505 03:30:20.581 | INFO | ], | 20260505 03:30:20.581 | INFO | [ | 20260505 03:30:20.581 | INFO | 1920, | 20260505 03:30:20.581 | INFO | 128, | 20260505 03:30:20.581 | INFO | 1, | 20260505 03:30:20.581 | INFO | 64 | 20260505 03:30:20.581 | INFO | ], | 20260505 03:30:20.581 | INFO | [ | 20260505 03:30:20.581 | INFO | 2176, | 20260505 03:30:20.581 | INFO | 128, | 20260505 03:30:20.581 | INFO | 1, | 20260505 03:30:20.581 | INFO | 64 | 20260505 03:30:20.581 | INFO | ], | 20260505 03:30:20.581 | INFO | [ | 20260505 03:30:20.581 | INFO | 2432, | 20260505 03:30:20.581 | INFO | 128, | 20260505 03:30:20.581 | INFO | 1, | 20260505 03:30:20.581 | INFO | 64 | 20260505 03:30:20.581 | INFO | ], | 20260505 03:30:20.581 | INFO | [ | 20260505 03:30:20.581 | INFO | 2688, | 20260505 03:30:20.581 | INFO | 128, | 20260505 03:30:20.581 | INFO | 1, | 20260505 03:30:20.581 | INFO | 64 | 20260505 03:30:20.581 | INFO | ], | 20260505 03:30:20.581 | INFO | [ | 20260505 03:30:20.581 | INFO | 2944, | 20260505 03:30:20.581 | INFO | 128, | 20260505 03:30:20.581 | INFO | 1, | 20260505 03:30:20.581 | INFO | 64 | 20260505 03:30:20.581 | INFO | ], | 20260505 03:30:20.581 | INFO | [ | 20260505 03:30:20.581 | INFO | 3200, | 20260505 03:30:20.581 | INFO | 128, | 20260505 03:30:20.582 | INFO | 1, | 20260505 03:30:20.582 | INFO | 64 | 20260505 03:30:20.582 | INFO | ], | 20260505 03:30:20.582 | INFO | [ | 20260505 03:30:20.582 | INFO | 3456, | 20260505 03:30:20.582 | INFO | 128, | 20260505 03:30:20.582 | INFO | 1, | 20260505 03:30:20.582 | INFO | 64 | 20260505 03:30:20.582 | INFO | ], | 20260505 03:30:20.582 | INFO | [ | 20260505 03:30:20.582 | INFO | 3712, | 20260505 03:30:20.582 | INFO | 128, | 20260505 03:30:20.582 | INFO | 1, | 20260505 03:30:20.582 | INFO | 64 | 20260505 03:30:20.582 | INFO | ], | 20260505 03:30:20.582 | INFO | [ | 20260505 03:30:20.582 | INFO | 3968, | 20260505 03:30:20.582 | INFO | 128, | 20260505 03:30:20.582 | INFO | 1, | 20260505 03:30:20.582 | INFO | 64 | 20260505 03:30:20.582 | INFO | ], | 20260505 03:30:20.582 | INFO | [ | 20260505 03:30:20.582 | INFO | 128, | 20260505 03:30:20.582 | INFO | 384, | 20260505 03:30:20.582 | INFO | 1, | 20260505 03:30:20.582 | INFO | 64 | 20260505 03:30:20.582 | INFO | ], | 20260505 03:30:20.582 | INFO | [ | 20260505 03:30:20.582 | INFO | 384, | 20260505 03:30:20.582 | INFO | 384, | 20260505 03:30:20.582 | INFO | 1, | 20260505 03:30:20.582 | INFO | 64 | 20260505 03:30:20.582 | INFO | ], | 20260505 03:30:20.582 | INFO | [ | 20260505 03:30:20.582 | INFO | 640, | 20260505 03:30:20.582 | INFO | 384, | 20260505 03:30:20.582 | INFO | 1, | 20260505 03:30:20.582 | INFO | 64 | 20260505 03:30:20.582 | INFO | ], | 20260505 03:30:20.582 | INFO | [ | 20260505 03:30:20.582 | INFO | 896, | 20260505 03:30:20.582 | INFO | 384, | 20260505 03:30:20.582 | INFO | 1, | 20260505 03:30:20.582 | INFO | 64 | 20260505 03:30:20.582 | INFO | ], | 20260505 03:30:20.582 | INFO | [ | 20260505 03:30:20.582 | INFO | 1152, | 20260505 03:30:20.582 | INFO | 384, | 20260505 03:30:20.582 | INFO | 1, | 20260505 03:30:20.582 | INFO | 64 | 20260505 03:30:20.582 | INFO | ], | 20260505 03:30:20.582 | INFO | [ | 20260505 03:30:20.582 | INFO | 1408, | 20260505 03:30:20.582 | INFO | 384, | 20260505 03:30:20.582 | INFO | 1, | 20260505 03:30:20.582 | INFO | 64 | 20260505 03:30:20.582 | INFO | ], | 20260505 03:30:20.582 | INFO | [ | 20260505 03:30:20.582 | INFO | 1664, | 20260505 03:30:20.582 | INFO | 384, | 20260505 03:30:20.582 | INFO | 1, | 20260505 03:30:20.582 | INFO | 64 | 20260505 03:30:20.582 | INFO | ], | 20260505 03:30:20.582 | INFO | [ | 20260505 03:30:20.582 | INFO | 1920, | 20260505 03:30:20.582 | INFO | 384, | 20260505 03:30:20.582 | INFO | 1, | 20260505 03:30:20.582 | INFO | 64 | 20260505 03:30:20.582 | INFO | ], | 20260505 03:30:20.582 | INFO | [ | 20260505 03:30:20.582 | INFO | 2176, | 20260505 03:30:20.582 | INFO | 384, | 20260505 03:30:20.582 | INFO | 1, | 20260505 03:30:20.582 | INFO | 64 | 20260505 03:30:20.582 | INFO | ], | 20260505 03:30:20.582 | INFO | [ | 20260505 03:30:20.582 | INFO | 2432, | 20260505 03:30:20.582 | INFO | 384, | 20260505 03:30:20.582 | INFO | 1, | 20260505 03:30:20.582 | INFO | 64 | 20260505 03:30:20.582 | INFO | ], | 20260505 03:30:20.582 | INFO | [ | 20260505 03:30:20.583 | INFO | 2688, | 20260505 03:30:20.583 | INFO | 384, | 20260505 03:30:20.583 | INFO | 1, | 20260505 03:30:20.583 | INFO | 64 | 20260505 03:30:20.583 | INFO | ], | 20260505 03:30:20.583 | INFO | [ | 20260505 03:30:20.583 | INFO | 2944, | 20260505 03:30:20.583 | INFO | 384, | 20260505 03:30:20.583 | INFO | 1, | 20260505 03:30:20.583 | INFO | 64 | 20260505 03:30:20.583 | INFO | ], | 20260505 03:30:20.583 | INFO | [ | 20260505 03:30:20.583 | INFO | 3200, | 20260505 03:30:20.583 | INFO | 384, | 20260505 03:30:20.583 | INFO | 1, | 20260505 03:30:20.583 | INFO | 64 | 20260505 03:30:20.583 | INFO | ], | 20260505 03:30:20.583 | INFO | [ | 20260505 03:30:20.583 | INFO | 3456, | 20260505 03:30:20.583 | INFO | 384, | 20260505 03:30:20.583 | INFO | 1, | 20260505 03:30:20.583 | INFO | 64 | 20260505 03:30:20.583 | INFO | ], | 20260505 03:30:20.583 | INFO | [ | 20260505 03:30:20.583 | INFO | 3712, | 20260505 03:30:20.583 | INFO | 384, | 20260505 03:30:20.583 | INFO | 1, | 20260505 03:30:20.583 | INFO | 64 | 20260505 03:30:20.583 | INFO | ], | 20260505 03:30:20.583 | INFO | [ | 20260505 03:30:20.583 | INFO | 3968, | 20260505 03:30:20.583 | INFO | 384, | 20260505 03:30:20.583 | INFO | 1, | 20260505 03:30:20.583 | INFO | 64 | 20260505 03:30:20.583 | INFO | ], | 20260505 03:30:20.583 | INFO | [ | 20260505 03:30:20.583 | INFO | 128, | 20260505 03:30:20.583 | INFO | 640, | 20260505 03:30:20.583 | INFO | 1, | 20260505 03:30:20.583 | INFO | 64 | 20260505 03:30:20.583 | INFO | ], | 20260505 03:30:20.583 | INFO | [ | 20260505 03:30:20.583 | INFO | 384, | 20260505 03:30:20.583 | INFO | 640, | 20260505 03:30:20.583 | INFO | 1, | 20260505 03:30:20.583 | INFO | 64 | 20260505 03:30:20.583 | INFO | ], | 20260505 03:30:20.583 | INFO | [ | 20260505 03:30:20.583 | INFO | 640, | 20260505 03:30:20.583 | INFO | 640, | 20260505 03:30:20.583 | INFO | 1, | 20260505 03:30:20.583 | INFO | 64 | 20260505 03:30:20.583 | INFO | ], | 20260505 03:30:20.583 | INFO | [ | 20260505 03:30:20.583 | INFO | 896, | 20260505 03:30:20.583 | INFO | 640, | 20260505 03:30:20.583 | INFO | 1, | 20260505 03:30:20.583 | INFO | 64 | 20260505 03:30:20.583 | INFO | ], | 20260505 03:30:20.583 | INFO | [ | 20260505 03:30:20.583 | INFO | 1152, | 20260505 03:30:20.583 | INFO | 640, | 20260505 03:30:20.583 | INFO | 1, | 20260505 03:30:20.583 | INFO | 64 | 20260505 03:30:20.583 | INFO | ], | 20260505 03:30:20.583 | INFO | [ | 20260505 03:30:20.583 | INFO | 1408, | 20260505 03:30:20.583 | INFO | 640, | 20260505 03:30:20.583 | INFO | 1, | 20260505 03:30:20.583 | INFO | 64 | 20260505 03:30:20.583 | INFO | ], | 20260505 03:30:20.583 | INFO | [ | 20260505 03:30:20.583 | INFO | 1664, | 20260505 03:30:20.583 | INFO | 640, | 20260505 03:30:20.583 | INFO | 1, | 20260505 03:30:20.583 | INFO | 64 | 20260505 03:30:20.583 | INFO | ], | 20260505 03:30:20.584 | INFO | [ | 20260505 03:30:20.584 | INFO | 1920, | 20260505 03:30:20.584 | INFO | 640, | 20260505 03:30:20.584 | INFO | 1, | 20260505 03:30:20.584 | INFO | 64 | 20260505 03:30:20.584 | INFO | ], | 20260505 03:30:20.584 | INFO | [ | 20260505 03:30:20.584 | INFO | 2176, | 20260505 03:30:20.584 | INFO | 640, | 20260505 03:30:20.584 | INFO | 1, | 20260505 03:30:20.584 | INFO | 64 | 20260505 03:30:20.584 | INFO | ], | 20260505 03:30:20.584 | INFO | [ | 20260505 03:30:20.584 | INFO | 2432, | 20260505 03:30:20.584 | INFO | 640, | 20260505 03:30:20.584 | INFO | 1, | 20260505 03:30:20.584 | INFO | 64 | 20260505 03:30:20.584 | INFO | ], | 20260505 03:30:20.584 | INFO | [ | 20260505 03:30:20.584 | INFO | 2688, | 20260505 03:30:20.584 | INFO | 640, | 20260505 03:30:20.584 | INFO | 1, | 20260505 03:30:20.584 | INFO | 64 | 20260505 03:30:20.584 | INFO | ], | 20260505 03:30:20.584 | INFO | [ | 20260505 03:30:20.584 | INFO | 2944, | 20260505 03:30:20.584 | INFO | 640, | 20260505 03:30:20.584 | INFO | 1, | 20260505 03:30:20.584 | INFO | 64 | 20260505 03:30:20.584 | INFO | ], | 20260505 03:30:20.584 | INFO | [ | 20260505 03:30:20.584 | INFO | 3200, | 20260505 03:30:20.584 | INFO | 640, | 20260505 03:30:20.584 | INFO | 1, | 20260505 03:30:20.584 | INFO | 64 | 20260505 03:30:20.584 | INFO | ], | 20260505 03:30:20.584 | INFO | [ | 20260505 03:30:20.584 | INFO | 3456, | 20260505 03:30:20.584 | INFO | 640, | 20260505 03:30:20.584 | INFO | 1, | 20260505 03:30:20.584 | INFO | 64 | 20260505 03:30:20.584 | INFO | ], | 20260505 03:30:20.584 | INFO | [ | 20260505 03:30:20.584 | INFO | 3712, | 20260505 03:30:20.584 | INFO | 640, | 20260505 03:30:20.584 | INFO | 1, | 20260505 03:30:20.584 | INFO | 64 | 20260505 03:30:20.584 | INFO | ], | 20260505 03:30:20.584 | INFO | [ | 20260505 03:30:20.584 | INFO | 3968, | 20260505 03:30:20.585 | INFO | 640, | 20260505 03:30:20.585 | INFO | 1, | 20260505 03:30:20.585 | INFO | 64 | 20260505 03:30:20.585 | INFO | ], | 20260505 03:30:20.585 | INFO | [ | 20260505 03:30:20.585 | INFO | 128, | 20260505 03:30:20.585 | INFO | 896, | 20260505 03:30:20.585 | INFO | 1, | 20260505 03:30:20.585 | INFO | 64 | 20260505 03:30:20.585 | INFO | ], | 20260505 03:30:20.585 | INFO | [ | 20260505 03:30:20.585 | INFO | 384, | 20260505 03:30:20.585 | INFO | 896, | 20260505 03:30:20.585 | INFO | 1, | 20260505 03:30:20.585 | INFO | 64 | 20260505 03:30:20.585 | INFO | ], | 20260505 03:30:20.585 | INFO | [ | 20260505 03:30:20.585 | INFO | 640, | 20260505 03:30:20.585 | INFO | 896, | 20260505 03:30:20.585 | INFO | 1, | 20260505 03:30:20.585 | INFO | 64 | 20260505 03:30:20.585 | INFO | ], | 20260505 03:30:20.585 | INFO | [ | 20260505 03:30:20.585 | INFO | 896, | 20260505 03:30:20.585 | INFO | 896, | 20260505 03:30:20.585 | INFO | 1, | 20260505 03:30:20.585 | INFO | 64 | 20260505 03:30:20.585 | INFO | ], | 20260505 03:30:20.585 | INFO | [ | 20260505 03:30:20.585 | INFO | 1152, | 20260505 03:30:20.585 | INFO | 896, | 20260505 03:30:20.585 | INFO | 1, | 20260505 03:30:20.585 | INFO | 64 | 20260505 03:30:20.585 | INFO | ], | 20260505 03:30:20.585 | INFO | [ | 20260505 03:30:20.585 | INFO | 1408, | 20260505 03:30:20.585 | INFO | 896, | 20260505 03:30:20.585 | INFO | 1, | 20260505 03:30:20.585 | INFO | 64 | 20260505 03:30:20.585 | INFO | ], | 20260505 03:30:20.585 | INFO | [ | 20260505 03:30:20.585 | INFO | 1664, | 20260505 03:30:20.585 | INFO | 896, | 20260505 03:30:20.585 | INFO | 1, | 20260505 03:30:20.585 | INFO | 64 | 20260505 03:30:20.585 | INFO | ], | 20260505 03:30:20.585 | INFO | [ | 20260505 03:30:20.585 | INFO | 1920, | 20260505 03:30:20.585 | INFO | 896, | 20260505 03:30:20.585 | INFO | 1, | 20260505 03:30:20.585 | INFO | 64 | 20260505 03:30:20.585 | INFO | ], | 20260505 03:30:20.585 | INFO | [ | 20260505 03:30:20.585 | INFO | 2176, | 20260505 03:30:20.585 | INFO | 896, | 20260505 03:30:20.585 | INFO | 1, | 20260505 03:30:20.585 | INFO | 64 | 20260505 03:30:20.585 | INFO | ], | 20260505 03:30:20.585 | INFO | [ | 20260505 03:30:20.585 | INFO | 2432, | 20260505 03:30:20.585 | INFO | 896, | 20260505 03:30:20.585 | INFO | 1, | 20260505 03:30:20.585 | INFO | 64 | 20260505 03:30:20.585 | INFO | ], | 20260505 03:30:20.585 | INFO | [ | 20260505 03:30:20.585 | INFO | 2688, | 20260505 03:30:20.585 | INFO | 896, | 20260505 03:30:20.585 | INFO | 1, | 20260505 03:30:20.585 | INFO | 64 | 20260505 03:30:20.585 | INFO | ], | 20260505 03:30:20.585 | INFO | [ | 20260505 03:30:20.585 | INFO | 2944, | 20260505 03:30:20.585 | INFO | 896, | 20260505 03:30:20.585 | INFO | 1, | 20260505 03:30:20.585 | INFO | 64 | 20260505 03:30:20.585 | INFO | ], | 20260505 03:30:20.585 | INFO | [ | 20260505 03:30:20.585 | INFO | 3200, | 20260505 03:30:20.585 | INFO | 896, | 20260505 03:30:20.585 | INFO | 1, | 20260505 03:30:20.585 | INFO | 64 | 20260505 03:30:20.585 | INFO | ], | 20260505 03:30:20.585 | INFO | [ | 20260505 03:30:20.585 | INFO | 3456, | 20260505 03:30:20.585 | INFO | 896, | 20260505 03:30:20.585 | INFO | 1, | 20260505 03:30:20.586 | INFO | 64 | 20260505 03:30:20.586 | INFO | ], | 20260505 03:30:20.586 | INFO | [ | 20260505 03:30:20.586 | INFO | 3712, | 20260505 03:30:20.586 | INFO | 896, | 20260505 03:30:20.586 | INFO | 1, | 20260505 03:30:20.586 | INFO | 64 | 20260505 03:30:20.586 | INFO | ], | 20260505 03:30:20.586 | INFO | [ | 20260505 03:30:20.586 | INFO | 3968, | 20260505 03:30:20.586 | INFO | 896, | 20260505 03:30:20.586 | INFO | 1, | 20260505 03:30:20.586 | INFO | 64 | 20260505 03:30:20.586 | INFO | ], | 20260505 03:30:20.586 | INFO | [ | 20260505 03:30:20.586 | INFO | 128, | 20260505 03:30:20.586 | INFO | 1152, | 20260505 03:30:20.586 | INFO | 1, | 20260505 03:30:20.586 | INFO | 64 | 20260505 03:30:20.586 | INFO | ], | 20260505 03:30:20.586 | INFO | [ | 20260505 03:30:20.586 | INFO | 384, | 20260505 03:30:20.586 | INFO | 1152, | 20260505 03:30:20.586 | INFO | 1, | 20260505 03:30:20.586 | INFO | 64 | 20260505 03:30:20.586 | INFO | ], | 20260505 03:30:20.586 | INFO | [ | 20260505 03:30:20.586 | INFO | 640, | 20260505 03:30:20.586 | INFO | 1152, | 20260505 03:30:20.586 | INFO | 1, | 20260505 03:30:20.586 | INFO | 64 | 20260505 03:30:20.586 | INFO | ], | 20260505 03:30:20.586 | INFO | [ | 20260505 03:30:20.586 | INFO | 896, | 20260505 03:30:20.586 | INFO | 1152, | 20260505 03:30:20.586 | INFO | 1, | 20260505 03:30:20.586 | INFO | 64 | 20260505 03:30:20.586 | INFO | ], | 20260505 03:30:20.586 | INFO | [ | 20260505 03:30:20.586 | INFO | 1152, | 20260505 03:30:20.586 | INFO | 1152, | 20260505 03:30:20.586 | INFO | 1, | 20260505 03:30:20.586 | INFO | 64 | 20260505 03:30:20.586 | INFO | ], | 20260505 03:30:20.586 | INFO | [ | 20260505 03:30:20.586 | INFO | 1408, | 20260505 03:30:20.586 | INFO | 1152, | 20260505 03:30:20.586 | INFO | 1, | 20260505 03:30:20.586 | INFO | 64 | 20260505 03:30:20.586 | INFO | ], | 20260505 03:30:20.586 | INFO | [ | 20260505 03:30:20.586 | INFO | 1664, | 20260505 03:30:20.586 | INFO | 1152, | 20260505 03:30:20.586 | INFO | 1, | 20260505 03:30:20.586 | INFO | 64 | 20260505 03:30:20.586 | INFO | ], | 20260505 03:30:20.586 | INFO | [ | 20260505 03:30:20.586 | INFO | 1920, | 20260505 03:30:20.586 | INFO | 1152, | 20260505 03:30:20.586 | INFO | 1, | 20260505 03:30:20.586 | INFO | 64 | 20260505 03:30:20.586 | INFO | ], | 20260505 03:30:20.586 | INFO | [ | 20260505 03:30:20.586 | INFO | 2176, | 20260505 03:30:20.586 | INFO | 1152, | 20260505 03:30:20.586 | INFO | 1, | 20260505 03:30:20.586 | INFO | 64 | 20260505 03:30:20.586 | INFO | ], | 20260505 03:30:20.586 | INFO | [ | 20260505 03:30:20.586 | INFO | 2432, | 20260505 03:30:20.586 | INFO | 1152, | 20260505 03:30:20.586 | INFO | 1, | 20260505 03:30:20.586 | INFO | 64 | 20260505 03:30:20.586 | INFO | ], | 20260505 03:30:20.586 | INFO | [ | 20260505 03:30:20.586 | INFO | 2688, | 20260505 03:30:20.586 | INFO | 1152, | 20260505 03:30:20.586 | INFO | 1, | 20260505 03:30:20.587 | INFO | 64 | 20260505 03:30:20.587 | INFO | ], | 20260505 03:30:20.587 | INFO | [ | 20260505 03:30:20.587 | INFO | 2944, | 20260505 03:30:20.587 | INFO | 1152, | 20260505 03:30:20.587 | INFO | 1, | 20260505 03:30:20.587 | INFO | 64 | 20260505 03:30:20.587 | INFO | ], | 20260505 03:30:20.587 | INFO | [ | 20260505 03:30:20.587 | INFO | 3200, | 20260505 03:30:20.587 | INFO | 1152, | 20260505 03:30:20.587 | INFO | 1, | 20260505 03:30:20.587 | INFO | 64 | 20260505 03:30:20.587 | INFO | ], | 20260505 03:30:20.587 | INFO | [ | 20260505 03:30:20.587 | INFO | 3456, | 20260505 03:30:20.587 | INFO | 1152, | 20260505 03:30:20.587 | INFO | 1, | 20260505 03:30:20.587 | INFO | 64 | 20260505 03:30:20.587 | INFO | ], | 20260505 03:30:20.587 | INFO | [ | 20260505 03:30:20.587 | INFO | 3712, | 20260505 03:30:20.587 | INFO | 1152, | 20260505 03:30:20.587 | INFO | 1, | 20260505 03:30:20.587 | INFO | 64 | 20260505 03:30:20.587 | INFO | ], | 20260505 03:30:20.587 | INFO | [ | 20260505 03:30:20.587 | INFO | 3968, | 20260505 03:30:20.587 | INFO | 1152, | 20260505 03:30:20.587 | INFO | 1, | 20260505 03:30:20.587 | INFO | 64 | 20260505 03:30:20.587 | INFO | ], | 20260505 03:30:20.587 | INFO | [ | 20260505 03:30:20.587 | INFO | 128, | 20260505 03:30:20.587 | INFO | 1408, | 20260505 03:30:20.587 | INFO | 1, | 20260505 03:30:20.587 | INFO | 64 | 20260505 03:30:20.587 | INFO | ], | 20260505 03:30:20.587 | INFO | [ | 20260505 03:30:20.587 | INFO | 384, | 20260505 03:30:20.587 | INFO | 1408, | 20260505 03:30:20.587 | INFO | 1, | 20260505 03:30:20.587 | INFO | 64 | 20260505 03:30:20.587 | INFO | ], | 20260505 03:30:20.587 | INFO | [ | 20260505 03:30:20.587 | INFO | 640, | 20260505 03:30:20.587 | INFO | 1408, | 20260505 03:30:20.587 | INFO | 1, | 20260505 03:30:20.587 | INFO | 64 | 20260505 03:30:20.587 | INFO | ], | 20260505 03:30:20.587 | INFO | [ | 20260505 03:30:20.587 | INFO | 896, | 20260505 03:30:20.587 | INFO | 1408, | 20260505 03:30:20.587 | INFO | 1, | 20260505 03:30:20.587 | INFO | 64 | 20260505 03:30:20.587 | INFO | ], | 20260505 03:30:20.587 | INFO | [ | 20260505 03:30:20.587 | INFO | 1152, | 20260505 03:30:20.587 | INFO | 1408, | 20260505 03:30:20.587 | INFO | 1, | 20260505 03:30:20.587 | INFO | 64 | 20260505 03:30:20.587 | INFO | ], | 20260505 03:30:20.587 | INFO | [ | 20260505 03:30:20.587 | INFO | 1408, | 20260505 03:30:20.587 | INFO | 1408, | 20260505 03:30:20.587 | INFO | 1, | 20260505 03:30:20.587 | INFO | 64 | 20260505 03:30:20.587 | INFO | ], | 20260505 03:30:20.587 | INFO | [ | 20260505 03:30:20.587 | INFO | 1664, | 20260505 03:30:20.587 | INFO | 1408, | 20260505 03:30:20.587 | INFO | 1, | 20260505 03:30:20.587 | INFO | 64 | 20260505 03:30:20.587 | INFO | ], | 20260505 03:30:20.587 | INFO | [ | 20260505 03:30:20.587 | INFO | 1920, | 20260505 03:30:20.587 | INFO | 1408, | 20260505 03:30:20.587 | INFO | 1, | 20260505 03:30:20.587 | INFO | 64 | 20260505 03:30:20.587 | INFO | ], | 20260505 03:30:20.587 | INFO | [ | 20260505 03:30:20.587 | INFO | 2176, | 20260505 03:30:20.587 | INFO | 1408, | 20260505 03:30:20.587 | INFO | 1, | 20260505 03:30:20.587 | INFO | 64 | 20260505 03:30:20.587 | INFO | ], | 20260505 03:30:20.588 | INFO | [ | 20260505 03:30:20.588 | INFO | 2432, | 20260505 03:30:20.588 | INFO | 1408, | 20260505 03:30:20.588 | INFO | 1, | 20260505 03:30:20.588 | INFO | 64 | 20260505 03:30:20.588 | INFO | ], | 20260505 03:30:20.588 | INFO | [ | 20260505 03:30:20.588 | INFO | 2688, | 20260505 03:30:20.588 | INFO | 1408, | 20260505 03:30:20.588 | INFO | 1, | 20260505 03:30:20.588 | INFO | 64 | 20260505 03:30:20.588 | INFO | ], | 20260505 03:30:20.588 | INFO | [ | 20260505 03:30:20.588 | INFO | 2944, | 20260505 03:30:20.588 | INFO | 1408, | 20260505 03:30:20.588 | INFO | 1, | 20260505 03:30:20.588 | INFO | 64 | 20260505 03:30:20.588 | INFO | ], | 20260505 03:30:20.588 | INFO | [ | 20260505 03:30:20.588 | INFO | 3200, | 20260505 03:30:20.588 | INFO | 1408, | 20260505 03:30:20.588 | INFO | 1, | 20260505 03:30:20.588 | INFO | 64 | 20260505 03:30:20.588 | INFO | ], | 20260505 03:30:20.588 | INFO | [ | 20260505 03:30:20.588 | INFO | 3456, | 20260505 03:30:20.588 | INFO | 1408, | 20260505 03:30:20.588 | INFO | 1, | 20260505 03:30:20.588 | INFO | 64 | 20260505 03:30:20.588 | INFO | ], | 20260505 03:30:20.588 | INFO | [ | 20260505 03:30:20.588 | INFO | 3712, | 20260505 03:30:20.588 | INFO | 1408, | 20260505 03:30:20.588 | INFO | 1, | 20260505 03:30:20.588 | INFO | 64 | 20260505 03:30:20.588 | INFO | ], | 20260505 03:30:20.588 | INFO | [ | 20260505 03:30:20.588 | INFO | 3968, | 20260505 03:30:20.588 | INFO | 1408, | 20260505 03:30:20.588 | INFO | 1, | 20260505 03:30:20.588 | INFO | 64 | 20260505 03:30:20.588 | INFO | ], | 20260505 03:30:20.588 | INFO | [ | 20260505 03:30:20.588 | INFO | 128, | 20260505 03:30:20.588 | INFO | 1664, | 20260505 03:30:20.588 | INFO | 1, | 20260505 03:30:20.588 | INFO | 64 | 20260505 03:30:20.588 | INFO | ], | 20260505 03:30:20.588 | INFO | [ | 20260505 03:30:20.588 | INFO | 384, | 20260505 03:30:20.588 | INFO | 1664, | 20260505 03:30:20.588 | INFO | 1, | 20260505 03:30:20.588 | INFO | 64 | 20260505 03:30:20.588 | INFO | ], | 20260505 03:30:20.588 | INFO | [ | 20260505 03:30:20.588 | INFO | 640, | 20260505 03:30:20.588 | INFO | 1664, | 20260505 03:30:20.588 | INFO | 1, | 20260505 03:30:20.588 | INFO | 64 | 20260505 03:30:20.588 | INFO | ], | 20260505 03:30:20.588 | INFO | [ | 20260505 03:30:20.588 | INFO | 896, | 20260505 03:30:20.588 | INFO | 1664, | 20260505 03:30:20.588 | INFO | 1, | 20260505 03:30:20.588 | INFO | 64 | 20260505 03:30:20.588 | INFO | ], | 20260505 03:30:20.588 | INFO | [ | 20260505 03:30:20.588 | INFO | 1152, | 20260505 03:30:20.588 | INFO | 1664, | 20260505 03:30:20.588 | INFO | 1, | 20260505 03:30:20.588 | INFO | 64 | 20260505 03:30:20.588 | INFO | ], | 20260505 03:30:20.588 | INFO | [ | 20260505 03:30:20.588 | INFO | 1408, | 20260505 03:30:20.588 | INFO | 1664, | 20260505 03:30:20.588 | INFO | 1, | 20260505 03:30:20.588 | INFO | 64 | 20260505 03:30:20.588 | INFO | ], | 20260505 03:30:20.588 | INFO | [ | 20260505 03:30:20.588 | INFO | 1664, | 20260505 03:30:20.588 | INFO | 1664, | 20260505 03:30:20.588 | INFO | 1, | 20260505 03:30:20.588 | INFO | 64 | 20260505 03:30:20.588 | INFO | ], | 20260505 03:30:20.589 | INFO | [ | 20260505 03:30:20.589 | INFO | 1920, | 20260505 03:30:20.589 | INFO | 1664, | 20260505 03:30:20.589 | INFO | 1, | 20260505 03:30:20.589 | INFO | 64 | 20260505 03:30:20.589 | INFO | ], | 20260505 03:30:20.589 | INFO | [ | 20260505 03:30:20.589 | INFO | 2176, | 20260505 03:30:20.589 | INFO | 1664, | 20260505 03:30:20.589 | INFO | 1, | 20260505 03:30:20.589 | INFO | 64 | 20260505 03:30:20.589 | INFO | ], | 20260505 03:30:20.589 | INFO | [ | 20260505 03:30:20.589 | INFO | 2432, | 20260505 03:30:20.589 | INFO | 1664, | 20260505 03:30:20.589 | INFO | 1, | 20260505 03:30:20.589 | INFO | 64 | 20260505 03:30:20.589 | INFO | ], | 20260505 03:30:20.589 | INFO | [ | 20260505 03:30:20.589 | INFO | 2688, | 20260505 03:30:20.589 | INFO | 1664, | 20260505 03:30:20.589 | INFO | 1, | 20260505 03:30:20.589 | INFO | 64 | 20260505 03:30:20.589 | INFO | ], | 20260505 03:30:20.589 | INFO | [ | 20260505 03:30:20.589 | INFO | 2944, | 20260505 03:30:20.589 | INFO | 1664, | 20260505 03:30:20.589 | INFO | 1, | 20260505 03:30:20.589 | INFO | 64 | 20260505 03:30:20.589 | INFO | ], | 20260505 03:30:20.589 | INFO | [ | 20260505 03:30:20.589 | INFO | 3200, | 20260505 03:30:20.589 | INFO | 1664, | 20260505 03:30:20.589 | INFO | 1, | 20260505 03:30:20.589 | INFO | 64 | 20260505 03:30:20.589 | INFO | ], | 20260505 03:30:20.589 | INFO | [ | 20260505 03:30:20.589 | INFO | 3456, | 20260505 03:30:20.589 | INFO | 1664, | 20260505 03:30:20.589 | INFO | 1, | 20260505 03:30:20.589 | INFO | 64 | 20260505 03:30:20.589 | INFO | ], | 20260505 03:30:20.589 | INFO | [ | 20260505 03:30:20.589 | INFO | 3712, | 20260505 03:30:20.589 | INFO | 1664, | 20260505 03:30:20.589 | INFO | 1, | 20260505 03:30:20.589 | INFO | 64 | 20260505 03:30:20.589 | INFO | ], | 20260505 03:30:20.589 | INFO | [ | 20260505 03:30:20.589 | INFO | 3968, | 20260505 03:30:20.589 | INFO | 1664, | 20260505 03:30:20.589 | INFO | 1, | 20260505 03:30:20.589 | INFO | 64 | 20260505 03:30:20.589 | INFO | ], | 20260505 03:30:20.589 | INFO | [ | 20260505 03:30:20.589 | INFO | 128, | 20260505 03:30:20.589 | INFO | 1920, | 20260505 03:30:20.589 | INFO | 1, | 20260505 03:30:20.589 | INFO | 64 | 20260505 03:30:20.589 | INFO | ], | 20260505 03:30:20.589 | INFO | [ | 20260505 03:30:20.589 | INFO | 384, | 20260505 03:30:20.589 | INFO | 1920, | 20260505 03:30:20.589 | INFO | 1, | 20260505 03:30:20.589 | INFO | 64 | 20260505 03:30:20.589 | INFO | ], | 20260505 03:30:20.589 | INFO | [ | 20260505 03:30:20.589 | INFO | 640, | 20260505 03:30:20.589 | INFO | 1920, | 20260505 03:30:20.589 | INFO | 1, | 20260505 03:30:20.589 | INFO | 64 | 20260505 03:30:20.589 | INFO | ], | 20260505 03:30:20.589 | INFO | [ | 20260505 03:30:20.589 | INFO | 896, | 20260505 03:30:20.589 | INFO | 1920, | 20260505 03:30:20.589 | INFO | 1, | 20260505 03:30:20.589 | INFO | 64 | 20260505 03:30:20.589 | INFO | ], | 20260505 03:30:20.589 | INFO | [ | 20260505 03:30:20.589 | INFO | 1152, | 20260505 03:30:20.590 | INFO | 1920, | 20260505 03:30:20.590 | INFO | 1, | 20260505 03:30:20.590 | INFO | 64 | 20260505 03:30:20.590 | INFO | ], | 20260505 03:30:20.590 | INFO | [ | 20260505 03:30:20.590 | INFO | 1408, | 20260505 03:30:20.590 | INFO | 1920, | 20260505 03:30:20.590 | INFO | 1, | 20260505 03:30:20.590 | INFO | 64 | 20260505 03:30:20.590 | INFO | ], | 20260505 03:30:20.590 | INFO | [ | 20260505 03:30:20.590 | INFO | 1664, | 20260505 03:30:20.590 | INFO | 1920, | 20260505 03:30:20.590 | INFO | 1, | 20260505 03:30:20.590 | INFO | 64 | 20260505 03:30:20.590 | INFO | ], | 20260505 03:30:20.590 | INFO | [ | 20260505 03:30:20.590 | INFO | 1920, | 20260505 03:30:20.590 | INFO | 1920, | 20260505 03:30:20.590 | INFO | 1, | 20260505 03:30:20.590 | INFO | 64 | 20260505 03:30:20.590 | INFO | ], | 20260505 03:30:20.590 | INFO | [ | 20260505 03:30:20.590 | INFO | 2176, | 20260505 03:30:20.590 | INFO | 1920, | 20260505 03:30:20.590 | INFO | 1, | 20260505 03:30:20.590 | INFO | 64 | 20260505 03:30:20.590 | INFO | ], | 20260505 03:30:20.590 | INFO | [ | 20260505 03:30:20.590 | INFO | 2432, | 20260505 03:30:20.590 | INFO | 1920, | 20260505 03:30:20.590 | INFO | 1, | 20260505 03:30:20.590 | INFO | 64 | 20260505 03:30:20.590 | INFO | ], | 20260505 03:30:20.590 | INFO | [ | 20260505 03:30:20.590 | INFO | 2688, | 20260505 03:30:20.590 | INFO | 1920, | 20260505 03:30:20.590 | INFO | 1, | 20260505 03:30:20.590 | INFO | 64 | 20260505 03:30:20.590 | INFO | ], | 20260505 03:30:20.590 | INFO | [ | 20260505 03:30:20.590 | INFO | 2944, | 20260505 03:30:20.590 | INFO | 1920, | 20260505 03:30:20.590 | INFO | 1, | 20260505 03:30:20.590 | INFO | 64 | 20260505 03:30:20.590 | INFO | ], | 20260505 03:30:20.590 | INFO | [ | 20260505 03:30:20.590 | INFO | 3200, | 20260505 03:30:20.590 | INFO | 1920, | 20260505 03:30:20.590 | INFO | 1, | 20260505 03:30:20.590 | INFO | 64 | 20260505 03:30:20.590 | INFO | ], | 20260505 03:30:20.590 | INFO | [ | 20260505 03:30:20.590 | INFO | 3456, | 20260505 03:30:20.590 | INFO | 1920, | 20260505 03:30:20.590 | INFO | 1, | 20260505 03:30:20.590 | INFO | 64 | 20260505 03:30:20.590 | INFO | ], | 20260505 03:30:20.590 | INFO | [ | 20260505 03:30:20.590 | INFO | 3712, | 20260505 03:30:20.590 | INFO | 1920, | 20260505 03:30:20.590 | INFO | 1, | 20260505 03:30:20.590 | INFO | 64 | 20260505 03:30:20.590 | INFO | ], | 20260505 03:30:20.590 | INFO | [ | 20260505 03:30:20.590 | INFO | 3968, | 20260505 03:30:20.590 | INFO | 1920, | 20260505 03:30:20.590 | INFO | 1, | 20260505 03:30:20.590 | INFO | 64 | 20260505 03:30:20.590 | INFO | ], | 20260505 03:30:20.590 | INFO | [ | 20260505 03:30:20.590 | INFO | 128, | 20260505 03:30:20.590 | INFO | 2176, | 20260505 03:30:20.590 | INFO | 1, | 20260505 03:30:20.590 | INFO | 64 | 20260505 03:30:20.590 | INFO | ], | 20260505 03:30:20.590 | INFO | [ | 20260505 03:30:20.590 | INFO | 384, | 20260505 03:30:20.590 | INFO | 2176, | 20260505 03:30:20.590 | INFO | 1, | 20260505 03:30:20.590 | INFO | 64 | 20260505 03:30:20.590 | INFO | ], | 20260505 03:30:20.590 | INFO | [ | 20260505 03:30:20.590 | INFO | 640, | 20260505 03:30:20.591 | INFO | 2176, | 20260505 03:30:20.591 | INFO | 1, | 20260505 03:30:20.591 | INFO | 64 | 20260505 03:30:20.591 | INFO | ], | 20260505 03:30:20.591 | INFO | [ | 20260505 03:30:20.591 | INFO | 896, | 20260505 03:30:20.591 | INFO | 2176, | 20260505 03:30:20.591 | INFO | 1, | 20260505 03:30:20.591 | INFO | 64 | 20260505 03:30:20.591 | INFO | ], | 20260505 03:30:20.591 | INFO | [ | 20260505 03:30:20.591 | INFO | 1152, | 20260505 03:30:20.591 | INFO | 2176, | 20260505 03:30:20.591 | INFO | 1, | 20260505 03:30:20.591 | INFO | 64 | 20260505 03:30:20.591 | INFO | ], | 20260505 03:30:20.591 | INFO | [ | 20260505 03:30:20.591 | INFO | 1408, | 20260505 03:30:20.591 | INFO | 2176, | 20260505 03:30:20.591 | INFO | 1, | 20260505 03:30:20.591 | INFO | 64 | 20260505 03:30:20.591 | INFO | ], | 20260505 03:30:20.591 | INFO | [ | 20260505 03:30:20.591 | INFO | 1664, | 20260505 03:30:20.591 | INFO | 2176, | 20260505 03:30:20.591 | INFO | 1, | 20260505 03:30:20.591 | INFO | 64 | 20260505 03:30:20.591 | INFO | ], | 20260505 03:30:20.591 | INFO | [ | 20260505 03:30:20.591 | INFO | 1920, | 20260505 03:30:20.591 | INFO | 2176, | 20260505 03:30:20.591 | INFO | 1, | 20260505 03:30:20.591 | INFO | 64 | 20260505 03:30:20.591 | INFO | ], | 20260505 03:30:20.591 | INFO | [ | 20260505 03:30:20.591 | INFO | 2176, | 20260505 03:30:20.591 | INFO | 2176, | 20260505 03:30:20.591 | INFO | 1, | 20260505 03:30:20.591 | INFO | 64 | 20260505 03:30:20.591 | INFO | ], | 20260505 03:30:20.591 | INFO | [ | 20260505 03:30:20.591 | INFO | 2432, | 20260505 03:30:20.591 | INFO | 2176, | 20260505 03:30:20.591 | INFO | 1, | 20260505 03:30:20.591 | INFO | 64 | 20260505 03:30:20.591 | INFO | ], | 20260505 03:30:20.591 | INFO | [ | 20260505 03:30:20.591 | INFO | 2688, | 20260505 03:30:20.591 | INFO | 2176, | 20260505 03:30:20.591 | INFO | 1, | 20260505 03:30:20.591 | INFO | 64 | 20260505 03:30:20.591 | INFO | ], | 20260505 03:30:20.591 | INFO | [ | 20260505 03:30:20.591 | INFO | 2944, | 20260505 03:30:20.591 | INFO | 2176, | 20260505 03:30:20.591 | INFO | 1, | 20260505 03:30:20.591 | INFO | 64 | 20260505 03:30:20.591 | INFO | ], | 20260505 03:30:20.591 | INFO | [ | 20260505 03:30:20.591 | INFO | 3200, | 20260505 03:30:20.591 | INFO | 2176, | 20260505 03:30:20.591 | INFO | 1, | 20260505 03:30:20.591 | INFO | 64 | 20260505 03:30:20.591 | INFO | ], | 20260505 03:30:20.591 | INFO | [ | 20260505 03:30:20.591 | INFO | 3456, | 20260505 03:30:20.591 | INFO | 2176, | 20260505 03:30:20.591 | INFO | 1, | 20260505 03:30:20.591 | INFO | 64 | 20260505 03:30:20.591 | INFO | ], | 20260505 03:30:20.591 | INFO | [ | 20260505 03:30:20.591 | INFO | 3712, | 20260505 03:30:20.591 | INFO | 2176, | 20260505 03:30:20.591 | INFO | 1, | 20260505 03:30:20.591 | INFO | 64 | 20260505 03:30:20.591 | INFO | ], | 20260505 03:30:20.591 | INFO | [ | 20260505 03:30:20.591 | INFO | 3968, | 20260505 03:30:20.591 | INFO | 2176, | 20260505 03:30:20.591 | INFO | 1, | 20260505 03:30:20.591 | INFO | 64 | 20260505 03:30:20.591 | INFO | ], | 20260505 03:30:20.591 | INFO | [ | 20260505 03:30:20.591 | INFO | 128, | 20260505 03:30:20.591 | INFO | 2432, | 20260505 03:30:20.592 | INFO | 1, | 20260505 03:30:20.592 | INFO | 64 | 20260505 03:30:20.592 | INFO | ], | 20260505 03:30:20.592 | INFO | [ | 20260505 03:30:20.592 | INFO | 384, | 20260505 03:30:20.592 | INFO | 2432, | 20260505 03:30:20.592 | INFO | 1, | 20260505 03:30:20.592 | INFO | 64 | 20260505 03:30:20.592 | INFO | ], | 20260505 03:30:20.592 | INFO | [ | 20260505 03:30:20.592 | INFO | 640, | 20260505 03:30:20.592 | INFO | 2432, | 20260505 03:30:20.592 | INFO | 1, | 20260505 03:30:20.592 | INFO | 64 | 20260505 03:30:20.592 | INFO | ], | 20260505 03:30:20.592 | INFO | [ | 20260505 03:30:20.592 | INFO | 896, | 20260505 03:30:20.592 | INFO | 2432, | 20260505 03:30:20.592 | INFO | 1, | 20260505 03:30:20.592 | INFO | 64 | 20260505 03:30:20.592 | INFO | ], | 20260505 03:30:20.592 | INFO | [ | 20260505 03:30:20.592 | INFO | 1152, | 20260505 03:30:20.592 | INFO | 2432, | 20260505 03:30:20.592 | INFO | 1, | 20260505 03:30:20.592 | INFO | 64 | 20260505 03:30:20.592 | INFO | ], | 20260505 03:30:20.592 | INFO | [ | 20260505 03:30:20.592 | INFO | 1408, | 20260505 03:30:20.592 | INFO | 2432, | 20260505 03:30:20.592 | INFO | 1, | 20260505 03:30:20.592 | INFO | 64 | 20260505 03:30:20.592 | INFO | ], | 20260505 03:30:20.592 | INFO | [ | 20260505 03:30:20.592 | INFO | 1664, | 20260505 03:30:20.592 | INFO | 2432, | 20260505 03:30:20.592 | INFO | 1, | 20260505 03:30:20.592 | INFO | 64 | 20260505 03:30:20.592 | INFO | ], | 20260505 03:30:20.592 | INFO | [ | 20260505 03:30:20.592 | INFO | 1920, | 20260505 03:30:20.592 | INFO | 2432, | 20260505 03:30:20.592 | INFO | 1, | 20260505 03:30:20.592 | INFO | 64 | 20260505 03:30:20.592 | INFO | ], | 20260505 03:30:20.592 | INFO | [ | 20260505 03:30:20.592 | INFO | 2176, | 20260505 03:30:20.592 | INFO | 2432, | 20260505 03:30:20.592 | INFO | 1, | 20260505 03:30:20.592 | INFO | 64 | 20260505 03:30:20.592 | INFO | ], | 20260505 03:30:20.592 | INFO | [ | 20260505 03:30:20.592 | INFO | 2432, | 20260505 03:30:20.592 | INFO | 2432, | 20260505 03:30:20.592 | INFO | 1, | 20260505 03:30:20.592 | INFO | 64 | 20260505 03:30:20.592 | INFO | ], | 20260505 03:30:20.592 | INFO | [ | 20260505 03:30:20.592 | INFO | 2688, | 20260505 03:30:20.592 | INFO | 2432, | 20260505 03:30:20.592 | INFO | 1, | 20260505 03:30:20.592 | INFO | 64 | 20260505 03:30:20.592 | INFO | ], | 20260505 03:30:20.592 | INFO | [ | 20260505 03:30:20.592 | INFO | 2944, | 20260505 03:30:20.592 | INFO | 2432, | 20260505 03:30:20.592 | INFO | 1, | 20260505 03:30:20.592 | INFO | 64 | 20260505 03:30:20.592 | INFO | ], | 20260505 03:30:20.592 | INFO | [ | 20260505 03:30:20.592 | INFO | 3200, | 20260505 03:30:20.592 | INFO | 2432, | 20260505 03:30:20.592 | INFO | 1, | 20260505 03:30:20.592 | INFO | 64 | 20260505 03:30:20.592 | INFO | ], | 20260505 03:30:20.592 | INFO | [ | 20260505 03:30:20.592 | INFO | 3456, | 20260505 03:30:20.592 | INFO | 2432, | 20260505 03:30:20.592 | INFO | 1, | 20260505 03:30:20.592 | INFO | 64 | 20260505 03:30:20.592 | INFO | ], | 20260505 03:30:20.592 | INFO | [ | 20260505 03:30:20.592 | INFO | 3712, | 20260505 03:30:20.592 | INFO | 2432, | 20260505 03:30:20.593 | INFO | 1, | 20260505 03:30:20.593 | INFO | 64 | 20260505 03:30:20.593 | INFO | ], | 20260505 03:30:20.593 | INFO | [ | 20260505 03:30:20.593 | INFO | 3968, | 20260505 03:30:20.593 | INFO | 2432, | 20260505 03:30:20.593 | INFO | 1, | 20260505 03:30:20.593 | INFO | 64 | 20260505 03:30:20.593 | INFO | ], | 20260505 03:30:20.593 | INFO | [ | 20260505 03:30:20.593 | INFO | 128, | 20260505 03:30:20.593 | INFO | 2688, | 20260505 03:30:20.593 | INFO | 1, | 20260505 03:30:20.593 | INFO | 64 | 20260505 03:30:20.593 | INFO | ], | 20260505 03:30:20.593 | INFO | [ | 20260505 03:30:20.593 | INFO | 384, | 20260505 03:30:20.593 | INFO | 2688, | 20260505 03:30:20.593 | INFO | 1, | 20260505 03:30:20.593 | INFO | 64 | 20260505 03:30:20.593 | INFO | ], | 20260505 03:30:20.593 | INFO | [ | 20260505 03:30:20.593 | INFO | 640, | 20260505 03:30:20.593 | INFO | 2688, | 20260505 03:30:20.593 | INFO | 1, | 20260505 03:30:20.593 | INFO | 64 | 20260505 03:30:20.593 | INFO | ], | 20260505 03:30:20.593 | INFO | [ | 20260505 03:30:20.593 | INFO | 896, | 20260505 03:30:20.593 | INFO | 2688, | 20260505 03:30:20.593 | INFO | 1, | 20260505 03:30:20.593 | INFO | 64 | 20260505 03:30:20.593 | INFO | ], | 20260505 03:30:20.593 | INFO | [ | 20260505 03:30:20.593 | INFO | 1152, | 20260505 03:30:20.593 | INFO | 2688, | 20260505 03:30:20.593 | INFO | 1, | 20260505 03:30:20.593 | INFO | 64 | 20260505 03:30:20.593 | INFO | ], | 20260505 03:30:20.593 | INFO | [ | 20260505 03:30:20.593 | INFO | 1408, | 20260505 03:30:20.593 | INFO | 2688, | 20260505 03:30:20.593 | INFO | 1, | 20260505 03:30:20.593 | INFO | 64 | 20260505 03:30:20.593 | INFO | ], | 20260505 03:30:20.593 | INFO | [ | 20260505 03:30:20.593 | INFO | 1664, | 20260505 03:30:20.593 | INFO | 2688, | 20260505 03:30:20.593 | INFO | 1, | 20260505 03:30:20.593 | INFO | 64 | 20260505 03:30:20.593 | INFO | ], | 20260505 03:30:20.593 | INFO | [ | 20260505 03:30:20.593 | INFO | 1920, | 20260505 03:30:20.593 | INFO | 2688, | 20260505 03:30:20.593 | INFO | 1, | 20260505 03:30:20.593 | INFO | 64 | 20260505 03:30:20.593 | INFO | ], | 20260505 03:30:20.593 | INFO | [ | 20260505 03:30:20.593 | INFO | 2176, | 20260505 03:30:20.593 | INFO | 2688, | 20260505 03:30:20.593 | INFO | 1, | 20260505 03:30:20.593 | INFO | 64 | 20260505 03:30:20.593 | INFO | ], | 20260505 03:30:20.593 | INFO | [ | 20260505 03:30:20.593 | INFO | 2432, | 20260505 03:30:20.593 | INFO | 2688, | 20260505 03:30:20.593 | INFO | 1, | 20260505 03:30:20.593 | INFO | 64 | 20260505 03:30:20.593 | INFO | ], | 20260505 03:30:20.593 | INFO | [ | 20260505 03:30:20.593 | INFO | 2688, | 20260505 03:30:20.593 | INFO | 2688, | 20260505 03:30:20.593 | INFO | 1, | 20260505 03:30:20.593 | INFO | 64 | 20260505 03:30:20.593 | INFO | ], | 20260505 03:30:20.593 | INFO | [ | 20260505 03:30:20.593 | INFO | 2944, | 20260505 03:30:20.593 | INFO | 2688, | 20260505 03:30:20.593 | INFO | 1, | 20260505 03:30:20.593 | INFO | 64 | 20260505 03:30:20.593 | INFO | ], | 20260505 03:30:20.593 | INFO | [ | 20260505 03:30:20.593 | INFO | 3200, | 20260505 03:30:20.594 | INFO | 2688, | 20260505 03:30:20.594 | INFO | 1, | 20260505 03:30:20.594 | INFO | 64 | 20260505 03:30:20.594 | INFO | ], | 20260505 03:30:20.594 | INFO | [ | 20260505 03:30:20.594 | INFO | 3456, | 20260505 03:30:20.594 | INFO | 2688, | 20260505 03:30:20.594 | INFO | 1, | 20260505 03:30:20.594 | INFO | 64 | 20260505 03:30:20.594 | INFO | ], | 20260505 03:30:20.594 | INFO | [ | 20260505 03:30:20.594 | INFO | 3712, | 20260505 03:30:20.594 | INFO | 2688, | 20260505 03:30:20.594 | INFO | 1, | 20260505 03:30:20.594 | INFO | 64 | 20260505 03:30:20.594 | INFO | ], | 20260505 03:30:20.594 | INFO | [ | 20260505 03:30:20.594 | INFO | 3968, | 20260505 03:30:20.594 | INFO | 2688, | 20260505 03:30:20.594 | INFO | 1, | 20260505 03:30:20.594 | INFO | 64 | 20260505 03:30:20.594 | INFO | ], | 20260505 03:30:20.594 | INFO | [ | 20260505 03:30:20.594 | INFO | 128, | 20260505 03:30:20.594 | INFO | 2944, | 20260505 03:30:20.594 | INFO | 1, | 20260505 03:30:20.594 | INFO | 64 | 20260505 03:30:20.594 | INFO | ], | 20260505 03:30:20.594 | INFO | [ | 20260505 03:30:20.594 | INFO | 384, | 20260505 03:30:20.594 | INFO | 2944, | 20260505 03:30:20.594 | INFO | 1, | 20260505 03:30:20.594 | INFO | 64 | 20260505 03:30:20.594 | INFO | ], | 20260505 03:30:20.594 | INFO | [ | 20260505 03:30:20.594 | INFO | 640, | 20260505 03:30:20.594 | INFO | 2944, | 20260505 03:30:20.594 | INFO | 1, | 20260505 03:30:20.594 | INFO | 64 | 20260505 03:30:20.594 | INFO | ], | 20260505 03:30:20.594 | INFO | [ | 20260505 03:30:20.594 | INFO | 896, | 20260505 03:30:20.594 | INFO | 2944, | 20260505 03:30:20.594 | INFO | 1, | 20260505 03:30:20.594 | INFO | 64 | 20260505 03:30:20.594 | INFO | ], | 20260505 03:30:20.594 | INFO | [ | 20260505 03:30:20.594 | INFO | 1152, | 20260505 03:30:20.594 | INFO | 2944, | 20260505 03:30:20.594 | INFO | 1, | 20260505 03:30:20.594 | INFO | 64 | 20260505 03:30:20.594 | INFO | ], | 20260505 03:30:20.594 | INFO | [ | 20260505 03:30:20.594 | INFO | 1408, | 20260505 03:30:20.594 | INFO | 2944, | 20260505 03:30:20.594 | INFO | 1, | 20260505 03:30:20.594 | INFO | 64 | 20260505 03:30:20.594 | INFO | ], | 20260505 03:30:20.594 | INFO | [ | 20260505 03:30:20.594 | INFO | 1664, | 20260505 03:30:20.594 | INFO | 2944, | 20260505 03:30:20.594 | INFO | 1, | 20260505 03:30:20.594 | INFO | 64 | 20260505 03:30:20.594 | INFO | ], | 20260505 03:30:20.594 | INFO | [ | 20260505 03:30:20.594 | INFO | 1920, | 20260505 03:30:20.594 | INFO | 2944, | 20260505 03:30:20.594 | INFO | 1, | 20260505 03:30:20.594 | INFO | 64 | 20260505 03:30:20.594 | INFO | ], | 20260505 03:30:20.594 | INFO | [ | 20260505 03:30:20.594 | INFO | 2176, | 20260505 03:30:20.594 | INFO | 2944, | 20260505 03:30:20.594 | INFO | 1, | 20260505 03:30:20.594 | INFO | 64 | 20260505 03:30:20.594 | INFO | ], | 20260505 03:30:20.594 | INFO | [ | 20260505 03:30:20.594 | INFO | 2432, | 20260505 03:30:20.594 | INFO | 2944, | 20260505 03:30:20.594 | INFO | 1, | 20260505 03:30:20.594 | INFO | 64 | 20260505 03:30:20.594 | INFO | ], | 20260505 03:30:20.594 | INFO | [ | 20260505 03:30:20.594 | INFO | 2688, | 20260505 03:30:20.595 | INFO | 2944, | 20260505 03:30:20.595 | INFO | 1, | 20260505 03:30:20.595 | INFO | 64 | 20260505 03:30:20.595 | INFO | ], | 20260505 03:30:20.595 | INFO | [ | 20260505 03:30:20.595 | INFO | 2944, | 20260505 03:30:20.595 | INFO | 2944, | 20260505 03:30:20.595 | INFO | 1, | 20260505 03:30:20.595 | INFO | 64 | 20260505 03:30:20.595 | INFO | ], | 20260505 03:30:20.595 | INFO | [ | 20260505 03:30:20.595 | INFO | 3200, | 20260505 03:30:20.595 | INFO | 2944, | 20260505 03:30:20.595 | INFO | 1, | 20260505 03:30:20.595 | INFO | 64 | 20260505 03:30:20.595 | INFO | ], | 20260505 03:30:20.595 | INFO | [ | 20260505 03:30:20.595 | INFO | 3456, | 20260505 03:30:20.595 | INFO | 2944, | 20260505 03:30:20.595 | INFO | 1, | 20260505 03:30:20.595 | INFO | 64 | 20260505 03:30:20.595 | INFO | ], | 20260505 03:30:20.595 | INFO | [ | 20260505 03:30:20.595 | INFO | 3712, | 20260505 03:30:20.595 | INFO | 2944, | 20260505 03:30:20.595 | INFO | 1, | 20260505 03:30:20.595 | INFO | 64 | 20260505 03:30:20.595 | INFO | ], | 20260505 03:30:20.595 | INFO | [ | 20260505 03:30:20.595 | INFO | 3968, | 20260505 03:30:20.595 | INFO | 2944, | 20260505 03:30:20.595 | INFO | 1, | 20260505 03:30:20.595 | INFO | 64 | 20260505 03:30:20.595 | INFO | ], | 20260505 03:30:20.595 | INFO | [ | 20260505 03:30:20.595 | INFO | 128, | 20260505 03:30:20.595 | INFO | 3200, | 20260505 03:30:20.595 | INFO | 1, | 20260505 03:30:20.595 | INFO | 64 | 20260505 03:30:20.595 | INFO | ], | 20260505 03:30:20.595 | INFO | [ | 20260505 03:30:20.595 | INFO | 384, | 20260505 03:30:20.595 | INFO | 3200, | 20260505 03:30:20.595 | INFO | 1, | 20260505 03:30:20.595 | INFO | 64 | 20260505 03:30:20.595 | INFO | ], | 20260505 03:30:20.595 | INFO | [ | 20260505 03:30:20.595 | INFO | 640, | 20260505 03:30:20.595 | INFO | 3200, | 20260505 03:30:20.595 | INFO | 1, | 20260505 03:30:20.595 | INFO | 64 | 20260505 03:30:20.595 | INFO | ], | 20260505 03:30:20.595 | INFO | [ | 20260505 03:30:20.595 | INFO | 896, | 20260505 03:30:20.595 | INFO | 3200, | 20260505 03:30:20.595 | INFO | 1, | 20260505 03:30:20.595 | INFO | 64 | 20260505 03:30:20.595 | INFO | ], | 20260505 03:30:20.595 | INFO | [ | 20260505 03:30:20.595 | INFO | 1152, | 20260505 03:30:20.595 | INFO | 3200, | 20260505 03:30:20.595 | INFO | 1, | 20260505 03:30:20.595 | INFO | 64 | 20260505 03:30:20.595 | INFO | ], | 20260505 03:30:20.595 | INFO | [ | 20260505 03:30:20.595 | INFO | 1408, | 20260505 03:30:20.595 | INFO | 3200, | 20260505 03:30:20.595 | INFO | 1, | 20260505 03:30:20.595 | INFO | 64 | 20260505 03:30:20.595 | INFO | ], | 20260505 03:30:20.595 | INFO | [ | 20260505 03:30:20.595 | INFO | 1664, | 20260505 03:30:20.595 | INFO | 3200, | 20260505 03:30:20.595 | INFO | 1, | 20260505 03:30:20.595 | INFO | 64 | 20260505 03:30:20.595 | INFO | ], | 20260505 03:30:20.595 | INFO | [ | 20260505 03:30:20.595 | INFO | 1920, | 20260505 03:30:20.595 | INFO | 3200, | 20260505 03:30:20.595 | INFO | 1, | 20260505 03:30:20.595 | INFO | 64 | 20260505 03:30:20.595 | INFO | ], | 20260505 03:30:20.595 | INFO | [ | 20260505 03:30:20.595 | INFO | 2176, | 20260505 03:30:20.595 | INFO | 3200, | 20260505 03:30:20.596 | INFO | 1, | 20260505 03:30:20.596 | INFO | 64 | 20260505 03:30:20.596 | INFO | ], | 20260505 03:30:20.596 | INFO | [ | 20260505 03:30:20.596 | INFO | 2432, | 20260505 03:30:20.596 | INFO | 3200, | 20260505 03:30:20.596 | INFO | 1, | 20260505 03:30:20.596 | INFO | 64 | 20260505 03:30:20.596 | INFO | ], | 20260505 03:30:20.596 | INFO | [ | 20260505 03:30:20.596 | INFO | 2688, | 20260505 03:30:20.596 | INFO | 3200, | 20260505 03:30:20.596 | INFO | 1, | 20260505 03:30:20.596 | INFO | 64 | 20260505 03:30:20.596 | INFO | ], | 20260505 03:30:20.596 | INFO | [ | 20260505 03:30:20.596 | INFO | 2944, | 20260505 03:30:20.596 | INFO | 3200, | 20260505 03:30:20.596 | INFO | 1, | 20260505 03:30:20.596 | INFO | 64 | 20260505 03:30:20.596 | INFO | ], | 20260505 03:30:20.596 | INFO | [ | 20260505 03:30:20.596 | INFO | 3200, | 20260505 03:30:20.596 | INFO | 3200, | 20260505 03:30:20.596 | INFO | 1, | 20260505 03:30:20.596 | INFO | 64 | 20260505 03:30:20.596 | INFO | ], | 20260505 03:30:20.596 | INFO | [ | 20260505 03:30:20.596 | INFO | 3456, | 20260505 03:30:20.596 | INFO | 3200, | 20260505 03:30:20.596 | INFO | 1, | 20260505 03:30:20.596 | INFO | 64 | 20260505 03:30:20.596 | INFO | ], | 20260505 03:30:20.596 | INFO | [ | 20260505 03:30:20.596 | INFO | 3712, | 20260505 03:30:20.596 | INFO | 3200, | 20260505 03:30:20.596 | INFO | 1, | 20260505 03:30:20.596 | INFO | 64 | 20260505 03:30:20.596 | INFO | ], | 20260505 03:30:20.596 | INFO | [ | 20260505 03:30:20.596 | INFO | 3968, | 20260505 03:30:20.596 | INFO | 3200, | 20260505 03:30:20.596 | INFO | 1, | 20260505 03:30:20.596 | INFO | 64 | 20260505 03:30:20.596 | INFO | ], | 20260505 03:30:20.596 | INFO | [ | 20260505 03:30:20.596 | INFO | 128, | 20260505 03:30:20.596 | INFO | 3456, | 20260505 03:30:20.596 | INFO | 1, | 20260505 03:30:20.596 | INFO | 64 | 20260505 03:30:20.596 | INFO | ], | 20260505 03:30:20.596 | INFO | [ | 20260505 03:30:20.596 | INFO | 384, | 20260505 03:30:20.596 | INFO | 3456, | 20260505 03:30:20.596 | INFO | 1, | 20260505 03:30:20.596 | INFO | 64 | 20260505 03:30:20.596 | INFO | ], | 20260505 03:30:20.596 | INFO | [ | 20260505 03:30:20.596 | INFO | 640, | 20260505 03:30:20.596 | INFO | 3456, | 20260505 03:30:20.596 | INFO | 1, | 20260505 03:30:20.596 | INFO | 64 | 20260505 03:30:20.596 | INFO | ], | 20260505 03:30:20.596 | INFO | [ | 20260505 03:30:20.596 | INFO | 896, | 20260505 03:30:20.596 | INFO | 3456, | 20260505 03:30:20.596 | INFO | 1, | 20260505 03:30:20.596 | INFO | 64 | 20260505 03:30:20.596 | INFO | ], | 20260505 03:30:20.596 | INFO | [ | 20260505 03:30:20.596 | INFO | 1152, | 20260505 03:30:20.596 | INFO | 3456, | 20260505 03:30:20.596 | INFO | 1, | 20260505 03:30:20.596 | INFO | 64 | 20260505 03:30:20.596 | INFO | ], | 20260505 03:30:20.596 | INFO | [ | 20260505 03:30:20.596 | INFO | 1408, | 20260505 03:30:20.596 | INFO | 3456, | 20260505 03:30:20.596 | INFO | 1, | 20260505 03:30:20.596 | INFO | 64 | 20260505 03:30:20.596 | INFO | ], | 20260505 03:30:20.596 | INFO | [ | 20260505 03:30:20.596 | INFO | 1664, | 20260505 03:30:20.597 | INFO | 3456, | 20260505 03:30:20.597 | INFO | 1, | 20260505 03:30:20.597 | INFO | 64 | 20260505 03:30:20.597 | INFO | ], | 20260505 03:30:20.597 | INFO | [ | 20260505 03:30:20.597 | INFO | 1920, | 20260505 03:30:20.597 | INFO | 3456, | 20260505 03:30:20.597 | INFO | 1, | 20260505 03:30:20.597 | INFO | 64 | 20260505 03:30:20.597 | INFO | ], | 20260505 03:30:20.597 | INFO | [ | 20260505 03:30:20.597 | INFO | 2176, | 20260505 03:30:20.597 | INFO | 3456, | 20260505 03:30:20.597 | INFO | 1, | 20260505 03:30:20.597 | INFO | 64 | 20260505 03:30:20.597 | INFO | ], | 20260505 03:30:20.597 | INFO | [ | 20260505 03:30:20.597 | INFO | 2432, | 20260505 03:30:20.597 | INFO | 3456, | 20260505 03:30:20.597 | INFO | 1, | 20260505 03:30:20.597 | INFO | 64 | 20260505 03:30:20.597 | INFO | ], | 20260505 03:30:20.597 | INFO | [ | 20260505 03:30:20.597 | INFO | 2688, | 20260505 03:30:20.597 | INFO | 3456, | 20260505 03:30:20.597 | INFO | 1, | 20260505 03:30:20.597 | INFO | 64 | 20260505 03:30:20.597 | INFO | ], | 20260505 03:30:20.597 | INFO | [ | 20260505 03:30:20.597 | INFO | 2944, | 20260505 03:30:20.597 | INFO | 3456, | 20260505 03:30:20.597 | INFO | 1, | 20260505 03:30:20.597 | INFO | 64 | 20260505 03:30:20.597 | INFO | ], | 20260505 03:30:20.597 | INFO | [ | 20260505 03:30:20.597 | INFO | 3200, | 20260505 03:30:20.597 | INFO | 3456, | 20260505 03:30:20.597 | INFO | 1, | 20260505 03:30:20.597 | INFO | 64 | 20260505 03:30:20.597 | INFO | ], | 20260505 03:30:20.597 | INFO | [ | 20260505 03:30:20.597 | INFO | 3456, | 20260505 03:30:20.597 | INFO | 3456, | 20260505 03:30:20.597 | INFO | 1, | 20260505 03:30:20.597 | INFO | 64 | 20260505 03:30:20.597 | INFO | ], | 20260505 03:30:20.597 | INFO | [ | 20260505 03:30:20.597 | INFO | 3712, | 20260505 03:30:20.597 | INFO | 3456, | 20260505 03:30:20.597 | INFO | 1, | 20260505 03:30:20.597 | INFO | 64 | 20260505 03:30:20.597 | INFO | ], | 20260505 03:30:20.597 | INFO | [ | 20260505 03:30:20.597 | INFO | 3968, | 20260505 03:30:20.597 | INFO | 3456, | 20260505 03:30:20.597 | INFO | 1, | 20260505 03:30:20.597 | INFO | 64 | 20260505 03:30:20.597 | INFO | ], | 20260505 03:30:20.597 | INFO | [ | 20260505 03:30:20.597 | INFO | 128, | 20260505 03:30:20.597 | INFO | 3712, | 20260505 03:30:20.597 | INFO | 1, | 20260505 03:30:20.597 | INFO | 64 | 20260505 03:30:20.597 | INFO | ], | 20260505 03:30:20.597 | INFO | [ | 20260505 03:30:20.597 | INFO | 384, | 20260505 03:30:20.597 | INFO | 3712, | 20260505 03:30:20.597 | INFO | 1, | 20260505 03:30:20.597 | INFO | 64 | 20260505 03:30:20.597 | INFO | ], | 20260505 03:30:20.597 | INFO | [ | 20260505 03:30:20.597 | INFO | 640, | 20260505 03:30:20.598 | INFO | 3712, | 20260505 03:30:20.598 | INFO | 1, | 20260505 03:30:20.598 | INFO | 64 | 20260505 03:30:20.598 | INFO | ], | 20260505 03:30:20.598 | INFO | [ | 20260505 03:30:20.598 | INFO | 896, | 20260505 03:30:20.598 | INFO | 3712, | 20260505 03:30:20.598 | INFO | 1, | 20260505 03:30:20.598 | INFO | 64 | 20260505 03:30:20.598 | INFO | ], | 20260505 03:30:20.598 | INFO | [ | 20260505 03:30:20.598 | INFO | 1152, | 20260505 03:30:20.598 | INFO | 3712, | 20260505 03:30:20.598 | INFO | 1, | 20260505 03:30:20.598 | INFO | 64 | 20260505 03:30:20.598 | INFO | ], | 20260505 03:30:20.598 | INFO | [ | 20260505 03:30:20.598 | INFO | 1408, | 20260505 03:30:20.598 | INFO | 3712, | 20260505 03:30:20.598 | INFO | 1, | 20260505 03:30:20.598 | INFO | 64 | 20260505 03:30:20.598 | INFO | ], | 20260505 03:30:20.598 | INFO | [ | 20260505 03:30:20.598 | INFO | 1664, | 20260505 03:30:20.598 | INFO | 3712, | 20260505 03:30:20.598 | INFO | 1, | 20260505 03:30:20.598 | INFO | 64 | 20260505 03:30:20.598 | INFO | ], | 20260505 03:30:20.598 | INFO | [ | 20260505 03:30:20.598 | INFO | 1920, | 20260505 03:30:20.598 | INFO | 3712, | 20260505 03:30:20.598 | INFO | 1, | 20260505 03:30:20.598 | INFO | 64 | 20260505 03:30:20.598 | INFO | ], | 20260505 03:30:20.598 | INFO | [ | 20260505 03:30:20.598 | INFO | 2176, | 20260505 03:30:20.598 | INFO | 3712, | 20260505 03:30:20.598 | INFO | 1, | 20260505 03:30:20.598 | INFO | 64 | 20260505 03:30:20.598 | INFO | ], | 20260505 03:30:20.598 | INFO | [ | 20260505 03:30:20.598 | INFO | 2432, | 20260505 03:30:20.598 | INFO | 3712, | 20260505 03:30:20.598 | INFO | 1, | 20260505 03:30:20.598 | INFO | 64 | 20260505 03:30:20.598 | INFO | ], | 20260505 03:30:20.598 | INFO | [ | 20260505 03:30:20.598 | INFO | 2688, | 20260505 03:30:20.598 | INFO | 3712, | 20260505 03:30:20.598 | INFO | 1, | 20260505 03:30:20.598 | INFO | 64 | 20260505 03:30:20.598 | INFO | ], | 20260505 03:30:20.598 | INFO | [ | 20260505 03:30:20.598 | INFO | 2944, | 20260505 03:30:20.598 | INFO | 3712, | 20260505 03:30:20.598 | INFO | 1, | 20260505 03:30:20.598 | INFO | 64 | 20260505 03:30:20.598 | INFO | ], | 20260505 03:30:20.598 | INFO | [ | 20260505 03:30:20.598 | INFO | 3200, | 20260505 03:30:20.598 | INFO | 3712, | 20260505 03:30:20.598 | INFO | 1, | 20260505 03:30:20.598 | INFO | 64 | 20260505 03:30:20.598 | INFO | ], | 20260505 03:30:20.598 | INFO | [ | 20260505 03:30:20.598 | INFO | 3456, | 20260505 03:30:20.598 | INFO | 3712, | 20260505 03:30:20.598 | INFO | 1, | 20260505 03:30:20.599 | INFO | 64 | 20260505 03:30:20.599 | INFO | ], | 20260505 03:30:20.599 | INFO | [ | 20260505 03:30:20.599 | INFO | 3712, | 20260505 03:30:20.599 | INFO | 3712, | 20260505 03:30:20.599 | INFO | 1, | 20260505 03:30:20.599 | INFO | 64 | 20260505 03:30:20.599 | INFO | ], | 20260505 03:30:20.599 | INFO | [ | 20260505 03:30:20.599 | INFO | 3968, | 20260505 03:30:20.599 | INFO | 3712, | 20260505 03:30:20.599 | INFO | 1, | 20260505 03:30:20.599 | INFO | 64 | 20260505 03:30:20.599 | INFO | ], | 20260505 03:30:20.599 | INFO | [ | 20260505 03:30:20.599 | INFO | 128, | 20260505 03:30:20.599 | INFO | 3968, | 20260505 03:30:20.599 | INFO | 1, | 20260505 03:30:20.599 | INFO | 64 | 20260505 03:30:20.599 | INFO | ], | 20260505 03:30:20.599 | INFO | [ | 20260505 03:30:20.599 | INFO | 384, | 20260505 03:30:20.599 | INFO | 3968, | 20260505 03:30:20.599 | INFO | 1, | 20260505 03:30:20.599 | INFO | 64 | 20260505 03:30:20.599 | INFO | ], | 20260505 03:30:20.599 | INFO | [ | 20260505 03:30:20.599 | INFO | 640, | 20260505 03:30:20.599 | INFO | 3968, | 20260505 03:30:20.599 | INFO | 1, | 20260505 03:30:20.599 | INFO | 64 | 20260505 03:30:20.599 | INFO | ], | 20260505 03:30:20.599 | INFO | [ | 20260505 03:30:20.599 | INFO | 896, | 20260505 03:30:20.599 | INFO | 3968, | 20260505 03:30:20.599 | INFO | 1, | 20260505 03:30:20.599 | INFO | 64 | 20260505 03:30:20.599 | INFO | ], | 20260505 03:30:20.599 | INFO | [ | 20260505 03:30:20.599 | INFO | 1152, | 20260505 03:30:20.599 | INFO | 3968, | 20260505 03:30:20.599 | INFO | 1, | 20260505 03:30:20.599 | INFO | 64 | 20260505 03:30:20.599 | INFO | ], | 20260505 03:30:20.599 | INFO | [ | 20260505 03:30:20.599 | INFO | 1408, | 20260505 03:30:20.599 | INFO | 3968, | 20260505 03:30:20.599 | INFO | 1, | 20260505 03:30:20.599 | INFO | 64 | 20260505 03:30:20.599 | INFO | ], | 20260505 03:30:20.599 | INFO | [ | 20260505 03:30:20.599 | INFO | 1664, | 20260505 03:30:20.599 | INFO | 3968, | 20260505 03:30:20.599 | INFO | 1, | 20260505 03:30:20.599 | INFO | 64 | 20260505 03:30:20.599 | INFO | ], | 20260505 03:30:20.599 | INFO | [ | 20260505 03:30:20.599 | INFO | 1920, | 20260505 03:30:20.599 | INFO | 3968, | 20260505 03:30:20.599 | INFO | 1, | 20260505 03:30:20.599 | INFO | 64 | 20260505 03:30:20.599 | INFO | ], | 20260505 03:30:20.599 | INFO | [ | 20260505 03:30:20.599 | INFO | 2176, | 20260505 03:30:20.599 | INFO | 3968, | 20260505 03:30:20.599 | INFO | 1, | 20260505 03:30:20.599 | INFO | 64 | 20260505 03:30:20.599 | INFO | ], | 20260505 03:30:20.599 | INFO | [ | 20260505 03:30:20.599 | INFO | 2432, | 20260505 03:30:20.599 | INFO | 3968, | 20260505 03:30:20.599 | INFO | 1, | 20260505 03:30:20.599 | INFO | 64 | 20260505 03:30:20.599 | INFO | ], | 20260505 03:30:20.599 | INFO | [ | 20260505 03:30:20.599 | INFO | 2688, | 20260505 03:30:20.599 | INFO | 3968, | 20260505 03:30:20.599 | INFO | 1, | 20260505 03:30:20.599 | INFO | 64 | 20260505 03:30:20.599 | INFO | ], | 20260505 03:30:20.599 | INFO | [ | 20260505 03:30:20.599 | INFO | 2944, | 20260505 03:30:20.599 | INFO | 3968, | 20260505 03:30:20.600 | INFO | 1, | 20260505 03:30:20.600 | INFO | 64 | 20260505 03:30:20.600 | INFO | ], | 20260505 03:30:20.600 | INFO | [ | 20260505 03:30:20.600 | INFO | 3200, | 20260505 03:30:20.600 | INFO | 3968, | 20260505 03:30:20.600 | INFO | 1, | 20260505 03:30:20.600 | INFO | 64 | 20260505 03:30:20.600 | INFO | ], | 20260505 03:30:20.600 | INFO | [ | 20260505 03:30:20.600 | INFO | 3456, | 20260505 03:30:20.600 | INFO | 3968, | 20260505 03:30:20.600 | INFO | 1, | 20260505 03:30:20.600 | INFO | 64 | 20260505 03:30:20.600 | INFO | ], | 20260505 03:30:20.600 | INFO | [ | 20260505 03:30:20.600 | INFO | 3712, | 20260505 03:30:20.600 | INFO | 3968, | 20260505 03:30:20.600 | INFO | 1, | 20260505 03:30:20.600 | INFO | 64 | 20260505 03:30:20.600 | INFO | ], | 20260505 03:30:20.600 | INFO | [ | 20260505 03:30:20.600 | INFO | 3968, | 20260505 03:30:20.600 | INFO | 3968, | 20260505 03:30:20.600 | INFO | 1, | 20260505 03:30:20.600 | INFO | 64 | 20260505 03:30:20.600 | INFO | ], | 20260505 03:30:20.600 | INFO | [ | 20260505 03:30:20.600 | INFO | 128, | 20260505 03:30:20.600 | INFO | 4224, | 20260505 03:30:20.600 | INFO | 1, | 20260505 03:30:20.600 | INFO | 64 | 20260505 03:30:20.600 | INFO | ], | 20260505 03:30:20.600 | INFO | [ | 20260505 03:30:20.600 | INFO | 384, | 20260505 03:30:20.600 | INFO | 4224, | 20260505 03:30:20.600 | INFO | 1, | 20260505 03:30:20.600 | INFO | 64 | 20260505 03:30:20.600 | INFO | ], | 20260505 03:30:20.600 | INFO | [ | 20260505 03:30:20.600 | INFO | 640, | 20260505 03:30:20.600 | INFO | 4224, | 20260505 03:30:20.600 | INFO | 1, | 20260505 03:30:20.600 | INFO | 64 | 20260505 03:30:20.600 | INFO | ], | 20260505 03:30:20.600 | INFO | [ | 20260505 03:30:20.600 | INFO | 896, | 20260505 03:30:20.600 | INFO | 4224, | 20260505 03:30:20.600 | INFO | 1, | 20260505 03:30:20.600 | INFO | 64 | 20260505 03:30:20.600 | INFO | ], | 20260505 03:30:20.600 | INFO | [ | 20260505 03:30:20.600 | INFO | 1152, | 20260505 03:30:20.600 | INFO | 4224, | 20260505 03:30:20.600 | INFO | 1, | 20260505 03:30:20.600 | INFO | 64 | 20260505 03:30:20.600 | INFO | ], | 20260505 03:30:20.600 | INFO | [ | 20260505 03:30:20.600 | INFO | 1408, | 20260505 03:30:20.600 | INFO | 4224, | 20260505 03:30:20.600 | INFO | 1, | 20260505 03:30:20.600 | INFO | 64 | 20260505 03:30:20.600 | INFO | ], | 20260505 03:30:20.600 | INFO | [ | 20260505 03:30:20.600 | INFO | 1664, | 20260505 03:30:20.600 | INFO | 4224, | 20260505 03:30:20.600 | INFO | 1, | 20260505 03:30:20.600 | INFO | 64 | 20260505 03:30:20.600 | INFO | ], | 20260505 03:30:20.600 | INFO | [ | 20260505 03:30:20.600 | INFO | 1920, | 20260505 03:30:20.600 | INFO | 4224, | 20260505 03:30:20.600 | INFO | 1, | 20260505 03:30:20.600 | INFO | 64 | 20260505 03:30:20.600 | INFO | ], | 20260505 03:30:20.600 | INFO | [ | 20260505 03:30:20.600 | INFO | 2176, | 20260505 03:30:20.600 | INFO | 4224, | 20260505 03:30:20.600 | INFO | 1, | 20260505 03:30:20.600 | INFO | 64 | 20260505 03:30:20.600 | INFO | ], | 20260505 03:30:20.600 | INFO | [ | 20260505 03:30:20.600 | INFO | 2432, | 20260505 03:30:20.600 | INFO | 4224, | 20260505 03:30:20.601 | INFO | 1, | 20260505 03:30:20.601 | INFO | 64 | 20260505 03:30:20.601 | INFO | ], | 20260505 03:30:20.601 | INFO | [ | 20260505 03:30:20.601 | INFO | 2688, | 20260505 03:30:20.601 | INFO | 4224, | 20260505 03:30:20.601 | INFO | 1, | 20260505 03:30:20.601 | INFO | 64 | 20260505 03:30:20.601 | INFO | ], | 20260505 03:30:20.601 | INFO | [ | 20260505 03:30:20.601 | INFO | 2944, | 20260505 03:30:20.601 | INFO | 4224, | 20260505 03:30:20.601 | INFO | 1, | 20260505 03:30:20.601 | INFO | 64 | 20260505 03:30:20.601 | INFO | ], | 20260505 03:30:20.601 | INFO | [ | 20260505 03:30:20.601 | INFO | 3200, | 20260505 03:30:20.601 | INFO | 4224, | 20260505 03:30:20.601 | INFO | 1, | 20260505 03:30:20.601 | INFO | 64 | 20260505 03:30:20.601 | INFO | ], | 20260505 03:30:20.601 | INFO | [ | 20260505 03:30:20.601 | INFO | 3456, | 20260505 03:30:20.601 | INFO | 4224, | 20260505 03:30:20.601 | INFO | 1, | 20260505 03:30:20.601 | INFO | 64 | 20260505 03:30:20.601 | INFO | ], | 20260505 03:30:20.601 | INFO | [ | 20260505 03:30:20.601 | INFO | 3712, | 20260505 03:30:20.601 | INFO | 4224, | 20260505 03:30:20.601 | INFO | 1, | 20260505 03:30:20.601 | INFO | 64 | 20260505 03:30:20.601 | INFO | ], | 20260505 03:30:20.601 | INFO | [ | 20260505 03:30:20.601 | INFO | 3968, | 20260505 03:30:20.601 | INFO | 4224, | 20260505 03:30:20.601 | INFO | 1, | 20260505 03:30:20.601 | INFO | 64 | 20260505 03:30:20.601 | INFO | ], | 20260505 03:30:20.601 | INFO | [ | 20260505 03:30:20.601 | INFO | 128, | 20260505 03:30:20.601 | INFO | 4480, | 20260505 03:30:20.601 | INFO | 1, | 20260505 03:30:20.601 | INFO | 64 | 20260505 03:30:20.601 | INFO | ], | 20260505 03:30:20.601 | INFO | [ | 20260505 03:30:20.601 | INFO | 384, | 20260505 03:30:20.601 | INFO | 4480, | 20260505 03:30:20.601 | INFO | 1, | 20260505 03:30:20.601 | INFO | 64 | 20260505 03:30:20.601 | INFO | ], | 20260505 03:30:20.601 | INFO | [ | 20260505 03:30:20.601 | INFO | 640, | 20260505 03:30:20.601 | INFO | 4480, | 20260505 03:30:20.601 | INFO | 1, | 20260505 03:30:20.601 | INFO | 64 | 20260505 03:30:20.601 | INFO | ], | 20260505 03:30:20.601 | INFO | [ | 20260505 03:30:20.601 | INFO | 896, | 20260505 03:30:20.601 | INFO | 4480, | 20260505 03:30:20.601 | INFO | 1, | 20260505 03:30:20.601 | INFO | 64 | 20260505 03:30:20.601 | INFO | ], | 20260505 03:30:20.601 | INFO | [ | 20260505 03:30:20.601 | INFO | 1152, | 20260505 03:30:20.601 | INFO | 4480, | 20260505 03:30:20.601 | INFO | 1, | 20260505 03:30:20.601 | INFO | 64 | 20260505 03:30:20.601 | INFO | ], | 20260505 03:30:20.601 | INFO | [ | 20260505 03:30:20.601 | INFO | 1408, | 20260505 03:30:20.601 | INFO | 4480, | 20260505 03:30:20.601 | INFO | 1, | 20260505 03:30:20.601 | INFO | 64 | 20260505 03:30:20.601 | INFO | ], | 20260505 03:30:20.601 | INFO | [ | 20260505 03:30:20.601 | INFO | 1664, | 20260505 03:30:20.601 | INFO | 4480, | 20260505 03:30:20.601 | INFO | 1, | 20260505 03:30:20.601 | INFO | 64 | 20260505 03:30:20.601 | INFO | ], | 20260505 03:30:20.602 | INFO | [ | 20260505 03:30:20.602 | INFO | 1920, | 20260505 03:30:20.602 | INFO | 4480, | 20260505 03:30:20.602 | INFO | 1, | 20260505 03:30:20.602 | INFO | 64 | 20260505 03:30:20.602 | INFO | ], | 20260505 03:30:20.602 | INFO | [ | 20260505 03:30:20.602 | INFO | 2176, | 20260505 03:30:20.602 | INFO | 4480, | 20260505 03:30:20.602 | INFO | 1, | 20260505 03:30:20.602 | INFO | 64 | 20260505 03:30:20.602 | INFO | ], | 20260505 03:30:20.602 | INFO | [ | 20260505 03:30:20.602 | INFO | 2432, | 20260505 03:30:20.602 | INFO | 4480, | 20260505 03:30:20.602 | INFO | 1, | 20260505 03:30:20.602 | INFO | 64 | 20260505 03:30:20.602 | INFO | ], | 20260505 03:30:20.602 | INFO | [ | 20260505 03:30:20.602 | INFO | 2688, | 20260505 03:30:20.602 | INFO | 4480, | 20260505 03:30:20.602 | INFO | 1, | 20260505 03:30:20.602 | INFO | 64 | 20260505 03:30:20.602 | INFO | ], | 20260505 03:30:20.602 | INFO | [ | 20260505 03:30:20.602 | INFO | 2944, | 20260505 03:30:20.602 | INFO | 4480, | 20260505 03:30:20.602 | INFO | 1, | 20260505 03:30:20.602 | INFO | 64 | 20260505 03:30:20.602 | INFO | ], | 20260505 03:30:20.602 | INFO | [ | 20260505 03:30:20.602 | INFO | 3200, | 20260505 03:30:20.602 | INFO | 4480, | 20260505 03:30:20.602 | INFO | 1, | 20260505 03:30:20.602 | INFO | 64 | 20260505 03:30:20.602 | INFO | ], | 20260505 03:30:20.602 | INFO | [ | 20260505 03:30:20.602 | INFO | 3456, | 20260505 03:30:20.602 | INFO | 4480, | 20260505 03:30:20.602 | INFO | 1, | 20260505 03:30:20.602 | INFO | 64 | 20260505 03:30:20.602 | INFO | ], | 20260505 03:30:20.602 | INFO | [ | 20260505 03:30:20.602 | INFO | 3712, | 20260505 03:30:20.602 | INFO | 4480, | 20260505 03:30:20.602 | INFO | 1, | 20260505 03:30:20.602 | INFO | 64 | 20260505 03:30:20.602 | INFO | ], | 20260505 03:30:20.602 | INFO | [ | 20260505 03:30:20.602 | INFO | 3968, | 20260505 03:30:20.602 | INFO | 4480, | 20260505 03:30:20.602 | INFO | 1, | 20260505 03:30:20.602 | INFO | 64 | 20260505 03:30:20.602 | INFO | ], | 20260505 03:30:20.602 | INFO | [ | 20260505 03:30:20.602 | INFO | 128, | 20260505 03:30:20.602 | INFO | 4736, | 20260505 03:30:20.602 | INFO | 1, | 20260505 03:30:20.602 | INFO | 64 | 20260505 03:30:20.602 | INFO | ], | 20260505 03:30:20.602 | INFO | [ | 20260505 03:30:20.602 | INFO | 384, | 20260505 03:30:20.602 | INFO | 4736, | 20260505 03:30:20.602 | INFO | 1, | 20260505 03:30:20.602 | INFO | 64 | 20260505 03:30:20.602 | INFO | ], | 20260505 03:30:20.602 | INFO | [ | 20260505 03:30:20.602 | INFO | 640, | 20260505 03:30:20.602 | INFO | 4736, | 20260505 03:30:20.602 | INFO | 1, | 20260505 03:30:20.602 | INFO | 64 | 20260505 03:30:20.602 | INFO | ], | 20260505 03:30:20.602 | INFO | [ | 20260505 03:30:20.602 | INFO | 896, | 20260505 03:30:20.602 | INFO | 4736, | 20260505 03:30:20.602 | INFO | 1, | 20260505 03:30:20.602 | INFO | 64 | 20260505 03:30:20.602 | INFO | ], | 20260505 03:30:20.602 | INFO | [ | 20260505 03:30:20.602 | INFO | 1152, | 20260505 03:30:20.602 | INFO | 4736, | 20260505 03:30:20.602 | INFO | 1, | 20260505 03:30:20.602 | INFO | 64 | 20260505 03:30:20.603 | INFO | ], | 20260505 03:30:20.603 | INFO | [ | 20260505 03:30:20.603 | INFO | 1408, | 20260505 03:30:20.603 | INFO | 4736, | 20260505 03:30:20.603 | INFO | 1, | 20260505 03:30:20.603 | INFO | 64 | 20260505 03:30:20.603 | INFO | ], | 20260505 03:30:20.603 | INFO | [ | 20260505 03:30:20.603 | INFO | 1664, | 20260505 03:30:20.603 | INFO | 4736, | 20260505 03:30:20.603 | INFO | 1, | 20260505 03:30:20.603 | INFO | 64 | 20260505 03:30:20.603 | INFO | ], | 20260505 03:30:20.603 | INFO | [ | 20260505 03:30:20.603 | INFO | 1920, | 20260505 03:30:20.603 | INFO | 4736, | 20260505 03:30:20.603 | INFO | 1, | 20260505 03:30:20.603 | INFO | 64 | 20260505 03:30:20.603 | INFO | ], | 20260505 03:30:20.603 | INFO | [ | 20260505 03:30:20.603 | INFO | 2176, | 20260505 03:30:20.603 | INFO | 4736, | 20260505 03:30:20.603 | INFO | 1, | 20260505 03:30:20.603 | INFO | 64 | 20260505 03:30:20.603 | INFO | ], | 20260505 03:30:20.603 | INFO | [ | 20260505 03:30:20.603 | INFO | 2432, | 20260505 03:30:20.603 | INFO | 4736, | 20260505 03:30:20.603 | INFO | 1, | 20260505 03:30:20.603 | INFO | 64 | 20260505 03:30:20.603 | INFO | ], | 20260505 03:30:20.603 | INFO | [ | 20260505 03:30:20.603 | INFO | 2688, | 20260505 03:30:20.603 | INFO | 4736, | 20260505 03:30:20.603 | INFO | 1, | 20260505 03:30:20.603 | INFO | 64 | 20260505 03:30:20.603 | INFO | ], | 20260505 03:30:20.603 | INFO | [ | 20260505 03:30:20.603 | INFO | 2944, | 20260505 03:30:20.603 | INFO | 4736, | 20260505 03:30:20.603 | INFO | 1, | 20260505 03:30:20.603 | INFO | 64 | 20260505 03:30:20.603 | INFO | ], | 20260505 03:30:20.603 | INFO | [ | 20260505 03:30:20.603 | INFO | 3200, | 20260505 03:30:20.603 | INFO | 4736, | 20260505 03:30:20.603 | INFO | 1, | 20260505 03:30:20.603 | INFO | 64 | 20260505 03:30:20.603 | INFO | ], | 20260505 03:30:20.603 | INFO | [ | 20260505 03:30:20.603 | INFO | 3456, | 20260505 03:30:20.603 | INFO | 4736, | 20260505 03:30:20.603 | INFO | 1, | 20260505 03:30:20.603 | INFO | 64 | 20260505 03:30:20.603 | INFO | ], | 20260505 03:30:20.603 | INFO | [ | 20260505 03:30:20.603 | INFO | 3712, | 20260505 03:30:20.603 | INFO | 4736, | 20260505 03:30:20.603 | INFO | 1, | 20260505 03:30:20.603 | INFO | 64 | 20260505 03:30:20.603 | INFO | ], | 20260505 03:30:20.603 | INFO | [ | 20260505 03:30:20.603 | INFO | 3968, | 20260505 03:30:20.603 | INFO | 4736, | 20260505 03:30:20.603 | INFO | 1, | 20260505 03:30:20.603 | INFO | 64 | 20260505 03:30:20.603 | INFO | ], | 20260505 03:30:20.603 | INFO | [ | 20260505 03:30:20.603 | INFO | 128, | 20260505 03:30:20.603 | INFO | 4992, | 20260505 03:30:20.603 | INFO | 1, | 20260505 03:30:20.603 | INFO | 64 | 20260505 03:30:20.603 | INFO | ], | 20260505 03:30:20.603 | INFO | [ | 20260505 03:30:20.603 | INFO | 384, | 20260505 03:30:20.603 | INFO | 4992, | 20260505 03:30:20.603 | INFO | 1, | 20260505 03:30:20.603 | INFO | 64 | 20260505 03:30:20.603 | INFO | ], | 20260505 03:30:20.603 | INFO | [ | 20260505 03:30:20.603 | INFO | 640, | 20260505 03:30:20.603 | INFO | 4992, | 20260505 03:30:20.604 | INFO | 1, | 20260505 03:30:20.604 | INFO | 64 | 20260505 03:30:20.604 | INFO | ], | 20260505 03:30:20.604 | INFO | [ | 20260505 03:30:20.604 | INFO | 896, | 20260505 03:30:20.604 | INFO | 4992, | 20260505 03:30:20.604 | INFO | 1, | 20260505 03:30:20.604 | INFO | 64 | 20260505 03:30:20.604 | INFO | ], | 20260505 03:30:20.604 | INFO | [ | 20260505 03:30:20.604 | INFO | 1152, | 20260505 03:30:20.604 | INFO | 4992, | 20260505 03:30:20.604 | INFO | 1, | 20260505 03:30:20.604 | INFO | 64 | 20260505 03:30:20.604 | INFO | ], | 20260505 03:30:20.604 | INFO | [ | 20260505 03:30:20.604 | INFO | 1408, | 20260505 03:30:20.604 | INFO | 4992, | 20260505 03:30:20.604 | INFO | 1, | 20260505 03:30:20.604 | INFO | 64 | 20260505 03:30:20.604 | INFO | ], | 20260505 03:30:20.604 | INFO | [ | 20260505 03:30:20.604 | INFO | 1664, | 20260505 03:30:20.604 | INFO | 4992, | 20260505 03:30:20.604 | INFO | 1, | 20260505 03:30:20.604 | INFO | 64 | 20260505 03:30:20.604 | INFO | ], | 20260505 03:30:20.604 | INFO | [ | 20260505 03:30:20.604 | INFO | 1920, | 20260505 03:30:20.604 | INFO | 4992, | 20260505 03:30:20.604 | INFO | 1, | 20260505 03:30:20.604 | INFO | 64 | 20260505 03:30:20.604 | INFO | ], | 20260505 03:30:20.604 | INFO | [ | 20260505 03:30:20.604 | INFO | 2176, | 20260505 03:30:20.604 | INFO | 4992, | 20260505 03:30:20.604 | INFO | 1, | 20260505 03:30:20.604 | INFO | 64 | 20260505 03:30:20.604 | INFO | ], | 20260505 03:30:20.604 | INFO | [ | 20260505 03:30:20.604 | INFO | 2432, | 20260505 03:30:20.604 | INFO | 4992, | 20260505 03:30:20.604 | INFO | 1, | 20260505 03:30:20.604 | INFO | 64 | 20260505 03:30:20.604 | INFO | ], | 20260505 03:30:20.604 | INFO | [ | 20260505 03:30:20.604 | INFO | 2688, | 20260505 03:30:20.604 | INFO | 4992, | 20260505 03:30:20.604 | INFO | 1, | 20260505 03:30:20.604 | INFO | 64 | 20260505 03:30:20.604 | INFO | ], | 20260505 03:30:20.604 | INFO | [ | 20260505 03:30:20.604 | INFO | 2944, | 20260505 03:30:20.604 | INFO | 4992, | 20260505 03:30:20.604 | INFO | 1, | 20260505 03:30:20.604 | INFO | 64 | 20260505 03:30:20.604 | INFO | ], | 20260505 03:30:20.604 | INFO | [ | 20260505 03:30:20.604 | INFO | 3200, | 20260505 03:30:20.604 | INFO | 4992, | 20260505 03:30:20.604 | INFO | 1, | 20260505 03:30:20.604 | INFO | 64 | 20260505 03:30:20.604 | INFO | ], | 20260505 03:30:20.604 | INFO | [ | 20260505 03:30:20.604 | INFO | 3456, | 20260505 03:30:20.604 | INFO | 4992, | 20260505 03:30:20.604 | INFO | 1, | 20260505 03:30:20.604 | INFO | 64 | 20260505 03:30:20.604 | INFO | ], | 20260505 03:30:20.604 | INFO | [ | 20260505 03:30:20.604 | INFO | 3712, | 20260505 03:30:20.604 | INFO | 4992, | 20260505 03:30:20.604 | INFO | 1, | 20260505 03:30:20.604 | INFO | 64 | 20260505 03:30:20.604 | INFO | ], | 20260505 03:30:20.604 | INFO | [ | 20260505 03:30:20.604 | INFO | 3968, | 20260505 03:30:20.604 | INFO | 4992, | 20260505 03:30:20.604 | INFO | 1, | 20260505 03:30:20.604 | INFO | 64 | 20260505 03:30:20.604 | INFO | ], | 20260505 03:30:20.604 | INFO | [ | 20260505 03:30:20.605 | INFO | 128, | 20260505 03:30:20.605 | INFO | 5248, | 20260505 03:30:20.605 | INFO | 1, | 20260505 03:30:20.605 | INFO | 64 | 20260505 03:30:20.605 | INFO | ], | 20260505 03:30:20.605 | INFO | [ | 20260505 03:30:20.605 | INFO | 384, | 20260505 03:30:20.605 | INFO | 5248, | 20260505 03:30:20.605 | INFO | 1, | 20260505 03:30:20.605 | INFO | 64 | 20260505 03:30:20.605 | INFO | ], | 20260505 03:30:20.605 | INFO | [ | 20260505 03:30:20.605 | INFO | 640, | 20260505 03:30:20.605 | INFO | 5248, | 20260505 03:30:20.605 | INFO | 1, | 20260505 03:30:20.605 | INFO | 64 | 20260505 03:30:20.605 | INFO | ], | 20260505 03:30:20.605 | INFO | [ | 20260505 03:30:20.605 | INFO | 896, | 20260505 03:30:20.605 | INFO | 5248, | 20260505 03:30:20.605 | INFO | 1, | 20260505 03:30:20.605 | INFO | 64 | 20260505 03:30:20.605 | INFO | ], | 20260505 03:30:20.605 | INFO | [ | 20260505 03:30:20.605 | INFO | 1152, | 20260505 03:30:20.605 | INFO | 5248, | 20260505 03:30:20.605 | INFO | 1, | 20260505 03:30:20.605 | INFO | 64 | 20260505 03:30:20.605 | INFO | ], | 20260505 03:30:20.605 | INFO | [ | 20260505 03:30:20.605 | INFO | 1408, | 20260505 03:30:20.605 | INFO | 5248, | 20260505 03:30:20.605 | INFO | 1, | 20260505 03:30:20.605 | INFO | 64 | 20260505 03:30:20.605 | INFO | ], | 20260505 03:30:20.605 | INFO | [ | 20260505 03:30:20.605 | INFO | 1664, | 20260505 03:30:20.605 | INFO | 5248, | 20260505 03:30:20.605 | INFO | 1, | 20260505 03:30:20.605 | INFO | 64 | 20260505 03:30:20.605 | INFO | ], | 20260505 03:30:20.605 | INFO | [ | 20260505 03:30:20.605 | INFO | 1920, | 20260505 03:30:20.605 | INFO | 5248, | 20260505 03:30:20.605 | INFO | 1, | 20260505 03:30:20.605 | INFO | 64 | 20260505 03:30:20.605 | INFO | ], | 20260505 03:30:20.605 | INFO | [ | 20260505 03:30:20.605 | INFO | 2176, | 20260505 03:30:20.605 | INFO | 5248, | 20260505 03:30:20.605 | INFO | 1, | 20260505 03:30:20.605 | INFO | 64 | 20260505 03:30:20.605 | INFO | ], | 20260505 03:30:20.605 | INFO | [ | 20260505 03:30:20.605 | INFO | 2432, | 20260505 03:30:20.605 | INFO | 5248, | 20260505 03:30:20.605 | INFO | 1, | 20260505 03:30:20.605 | INFO | 64 | 20260505 03:30:20.605 | INFO | ], | 20260505 03:30:20.605 | INFO | [ | 20260505 03:30:20.605 | INFO | 2688, | 20260505 03:30:20.605 | INFO | 5248, | 20260505 03:30:20.605 | INFO | 1, | 20260505 03:30:20.605 | INFO | 64 | 20260505 03:30:20.605 | INFO | ], | 20260505 03:30:20.605 | INFO | [ | 20260505 03:30:20.605 | INFO | 2944, | 20260505 03:30:20.605 | INFO | 5248, | 20260505 03:30:20.605 | INFO | 1, | 20260505 03:30:20.605 | INFO | 64 | 20260505 03:30:20.605 | INFO | ], | 20260505 03:30:20.605 | INFO | [ | 20260505 03:30:20.605 | INFO | 3200, | 20260505 03:30:20.605 | INFO | 5248, | 20260505 03:30:20.605 | INFO | 1, | 20260505 03:30:20.605 | INFO | 64 | 20260505 03:30:20.605 | INFO | ], | 20260505 03:30:20.605 | INFO | [ | 20260505 03:30:20.605 | INFO | 3456, | 20260505 03:30:20.605 | INFO | 5248, | 20260505 03:30:20.605 | INFO | 1, | 20260505 03:30:20.605 | INFO | 64 | 20260505 03:30:20.605 | INFO | ], | 20260505 03:30:20.605 | INFO | [ | 20260505 03:30:20.605 | INFO | 3712, | 20260505 03:30:20.606 | INFO | 5248, | 20260505 03:30:20.606 | INFO | 1, | 20260505 03:30:20.606 | INFO | 64 | 20260505 03:30:20.606 | INFO | ], | 20260505 03:30:20.606 | INFO | [ | 20260505 03:30:20.606 | INFO | 3968, | 20260505 03:30:20.606 | INFO | 5248, | 20260505 03:30:20.606 | INFO | 1, | 20260505 03:30:20.606 | INFO | 64 | 20260505 03:30:20.606 | INFO | ], | 20260505 03:30:20.606 | INFO | [ | 20260505 03:30:20.606 | INFO | 128, | 20260505 03:30:20.606 | INFO | 5504, | 20260505 03:30:20.606 | INFO | 1, | 20260505 03:30:20.606 | INFO | 64 | 20260505 03:30:20.606 | INFO | ], | 20260505 03:30:20.606 | INFO | [ | 20260505 03:30:20.606 | INFO | 384, | 20260505 03:30:20.606 | INFO | 5504, | 20260505 03:30:20.606 | INFO | 1, | 20260505 03:30:20.606 | INFO | 64 | 20260505 03:30:20.606 | INFO | ], | 20260505 03:30:20.606 | INFO | [ | 20260505 03:30:20.606 | INFO | 640, | 20260505 03:30:20.606 | INFO | 5504, | 20260505 03:30:20.606 | INFO | 1, | 20260505 03:30:20.606 | INFO | 64 | 20260505 03:30:20.606 | INFO | ], | 20260505 03:30:20.606 | INFO | [ | 20260505 03:30:20.606 | INFO | 896, | 20260505 03:30:20.606 | INFO | 5504, | 20260505 03:30:20.606 | INFO | 1, | 20260505 03:30:20.606 | INFO | 64 | 20260505 03:30:20.606 | INFO | ], | 20260505 03:30:20.606 | INFO | [ | 20260505 03:30:20.606 | INFO | 1152, | 20260505 03:30:20.606 | INFO | 5504, | 20260505 03:30:20.606 | INFO | 1, | 20260505 03:30:20.606 | INFO | 64 | 20260505 03:30:20.606 | INFO | ], | 20260505 03:30:20.606 | INFO | [ | 20260505 03:30:20.606 | INFO | 1408, | 20260505 03:30:20.606 | INFO | 5504, | 20260505 03:30:20.606 | INFO | 1, | 20260505 03:30:20.606 | INFO | 64 | 20260505 03:30:20.606 | INFO | ], | 20260505 03:30:20.606 | INFO | [ | 20260505 03:30:20.606 | INFO | 1664, | 20260505 03:30:20.606 | INFO | 5504, | 20260505 03:30:20.606 | INFO | 1, | 20260505 03:30:20.606 | INFO | 64 | 20260505 03:30:20.606 | INFO | ], | 20260505 03:30:20.606 | INFO | [ | 20260505 03:30:20.606 | INFO | 1920, | 20260505 03:30:20.606 | INFO | 5504, | 20260505 03:30:20.606 | INFO | 1, | 20260505 03:30:20.606 | INFO | 64 | 20260505 03:30:20.606 | INFO | ], | 20260505 03:30:20.606 | INFO | [ | 20260505 03:30:20.606 | INFO | 2176, | 20260505 03:30:20.606 | INFO | 5504, | 20260505 03:30:20.606 | INFO | 1, | 20260505 03:30:20.606 | INFO | 64 | 20260505 03:30:20.606 | INFO | ], | 20260505 03:30:20.606 | INFO | [ | 20260505 03:30:20.606 | INFO | 2432, | 20260505 03:30:20.606 | INFO | 5504, | 20260505 03:30:20.606 | INFO | 1, | 20260505 03:30:20.606 | INFO | 64 | 20260505 03:30:20.606 | INFO | ], | 20260505 03:30:20.606 | INFO | [ | 20260505 03:30:20.606 | INFO | 2688, | 20260505 03:30:20.606 | INFO | 5504, | 20260505 03:30:20.606 | INFO | 1, | 20260505 03:30:20.606 | INFO | 64 | 20260505 03:30:20.606 | INFO | ], | 20260505 03:30:20.606 | INFO | [ | 20260505 03:30:20.606 | INFO | 2944, | 20260505 03:30:20.606 | INFO | 5504, | 20260505 03:30:20.606 | INFO | 1, | 20260505 03:30:20.606 | INFO | 64 | 20260505 03:30:20.606 | INFO | ], | 20260505 03:30:20.607 | INFO | [ | 20260505 03:30:20.607 | INFO | 3200, | 20260505 03:30:20.607 | INFO | 5504, | 20260505 03:30:20.607 | INFO | 1, | 20260505 03:30:20.607 | INFO | 64 | 20260505 03:30:20.607 | INFO | ], | 20260505 03:30:20.607 | INFO | [ | 20260505 03:30:20.607 | INFO | 3456, | 20260505 03:30:20.607 | INFO | 5504, | 20260505 03:30:20.607 | INFO | 1, | 20260505 03:30:20.607 | INFO | 64 | 20260505 03:30:20.607 | INFO | ], | 20260505 03:30:20.607 | INFO | [ | 20260505 03:30:20.607 | INFO | 3712, | 20260505 03:30:20.607 | INFO | 5504, | 20260505 03:30:20.607 | INFO | 1, | 20260505 03:30:20.607 | INFO | 64 | 20260505 03:30:20.607 | INFO | ], | 20260505 03:30:20.607 | INFO | [ | 20260505 03:30:20.607 | INFO | 3968, | 20260505 03:30:20.607 | INFO | 5504, | 20260505 03:30:20.607 | INFO | 1, | 20260505 03:30:20.607 | INFO | 64 | 20260505 03:30:20.607 | INFO | ], | 20260505 03:30:20.607 | INFO | [ | 20260505 03:30:20.607 | INFO | 128, | 20260505 03:30:20.607 | INFO | 5760, | 20260505 03:30:20.607 | INFO | 1, | 20260505 03:30:20.607 | INFO | 64 | 20260505 03:30:20.607 | INFO | ], | 20260505 03:30:20.607 | INFO | [ | 20260505 03:30:20.607 | INFO | 384, | 20260505 03:30:20.607 | INFO | 5760, | 20260505 03:30:20.607 | INFO | 1, | 20260505 03:30:20.607 | INFO | 64 | 20260505 03:30:20.607 | INFO | ], | 20260505 03:30:20.607 | INFO | [ | 20260505 03:30:20.607 | INFO | 640, | 20260505 03:30:20.607 | INFO | 5760, | 20260505 03:30:20.607 | INFO | 1, | 20260505 03:30:20.607 | INFO | 64 | 20260505 03:30:20.607 | INFO | ], | 20260505 03:30:20.607 | INFO | [ | 20260505 03:30:20.607 | INFO | 896, | 20260505 03:30:20.607 | INFO | 5760, | 20260505 03:30:20.607 | INFO | 1, | 20260505 03:30:20.607 | INFO | 64 | 20260505 03:30:20.607 | INFO | ], | 20260505 03:30:20.607 | INFO | [ | 20260505 03:30:20.607 | INFO | 1152, | 20260505 03:30:20.607 | INFO | 5760, | 20260505 03:30:20.607 | INFO | 1, | 20260505 03:30:20.607 | INFO | 64 | 20260505 03:30:20.607 | INFO | ], | 20260505 03:30:20.607 | INFO | [ | 20260505 03:30:20.607 | INFO | 1408, | 20260505 03:30:20.607 | INFO | 5760, | 20260505 03:30:20.607 | INFO | 1, | 20260505 03:30:20.607 | INFO | 64 | 20260505 03:30:20.607 | INFO | ], | 20260505 03:30:20.607 | INFO | [ | 20260505 03:30:20.607 | INFO | 1664, | 20260505 03:30:20.607 | INFO | 5760, | 20260505 03:30:20.607 | INFO | 1, | 20260505 03:30:20.607 | INFO | 64 | 20260505 03:30:20.607 | INFO | ], | 20260505 03:30:20.607 | INFO | [ | 20260505 03:30:20.607 | INFO | 1920, | 20260505 03:30:20.607 | INFO | 5760, | 20260505 03:30:20.607 | INFO | 1, | 20260505 03:30:20.607 | INFO | 64 | 20260505 03:30:20.607 | INFO | ], | 20260505 03:30:20.607 | INFO | [ | 20260505 03:30:20.607 | INFO | 2176, | 20260505 03:30:20.607 | INFO | 5760, | 20260505 03:30:20.607 | INFO | 1, | 20260505 03:30:20.607 | INFO | 64 | 20260505 03:30:20.607 | INFO | ], | 20260505 03:30:20.607 | INFO | [ | 20260505 03:30:20.607 | INFO | 2432, | 20260505 03:30:20.607 | INFO | 5760, | 20260505 03:30:20.608 | INFO | 1, | 20260505 03:30:20.608 | INFO | 64 | 20260505 03:30:20.608 | INFO | ], | 20260505 03:30:20.608 | INFO | [ | 20260505 03:30:20.608 | INFO | 2688, | 20260505 03:30:20.608 | INFO | 5760, | 20260505 03:30:20.608 | INFO | 1, | 20260505 03:30:20.608 | INFO | 64 | 20260505 03:30:20.608 | INFO | ], | 20260505 03:30:20.608 | INFO | [ | 20260505 03:30:20.608 | INFO | 2944, | 20260505 03:30:20.608 | INFO | 5760, | 20260505 03:30:20.608 | INFO | 1, | 20260505 03:30:20.608 | INFO | 64 | 20260505 03:30:20.608 | INFO | ], | 20260505 03:30:20.608 | INFO | [ | 20260505 03:30:20.608 | INFO | 3200, | 20260505 03:30:20.608 | INFO | 5760, | 20260505 03:30:20.608 | INFO | 1, | 20260505 03:30:20.608 | INFO | 64 | 20260505 03:30:20.608 | INFO | ], | 20260505 03:30:20.608 | INFO | [ | 20260505 03:30:20.608 | INFO | 3456, | 20260505 03:30:20.608 | INFO | 5760, | 20260505 03:30:20.608 | INFO | 1, | 20260505 03:30:20.608 | INFO | 64 | 20260505 03:30:20.608 | INFO | ], | 20260505 03:30:20.608 | INFO | [ | 20260505 03:30:20.608 | INFO | 3712, | 20260505 03:30:20.608 | INFO | 5760, | 20260505 03:30:20.608 | INFO | 1, | 20260505 03:30:20.608 | INFO | 64 | 20260505 03:30:20.608 | INFO | ], | 20260505 03:30:20.608 | INFO | [ | 20260505 03:30:20.608 | INFO | 3968, | 20260505 03:30:20.608 | INFO | 5760, | 20260505 03:30:20.608 | INFO | 1, | 20260505 03:30:20.608 | INFO | 64 | 20260505 03:30:20.608 | INFO | ], | 20260505 03:30:20.608 | INFO | [ | 20260505 03:30:20.608 | INFO | 128, | 20260505 03:30:20.608 | INFO | 6016, | 20260505 03:30:20.608 | INFO | 1, | 20260505 03:30:20.608 | INFO | 64 | 20260505 03:30:20.608 | INFO | ], | 20260505 03:30:20.608 | INFO | [ | 20260505 03:30:20.608 | INFO | 384, | 20260505 03:30:20.608 | INFO | 6016, | 20260505 03:30:20.608 | INFO | 1, | 20260505 03:30:20.608 | INFO | 64 | 20260505 03:30:20.608 | INFO | ], | 20260505 03:30:20.608 | INFO | [ | 20260505 03:30:20.608 | INFO | 640, | 20260505 03:30:20.608 | INFO | 6016, | 20260505 03:30:20.608 | INFO | 1, | 20260505 03:30:20.608 | INFO | 64 | 20260505 03:30:20.608 | INFO | ], | 20260505 03:30:20.608 | INFO | [ | 20260505 03:30:20.608 | INFO | 896, | 20260505 03:30:20.608 | INFO | 6016, | 20260505 03:30:20.608 | INFO | 1, | 20260505 03:30:20.608 | INFO | 64 | 20260505 03:30:20.608 | INFO | ], | 20260505 03:30:20.608 | INFO | [ | 20260505 03:30:20.608 | INFO | 1152, | 20260505 03:30:20.608 | INFO | 6016, | 20260505 03:30:20.608 | INFO | 1, | 20260505 03:30:20.608 | INFO | 64 | 20260505 03:30:20.608 | INFO | ], | 20260505 03:30:20.608 | INFO | [ | 20260505 03:30:20.608 | INFO | 1408, | 20260505 03:30:20.608 | INFO | 6016, | 20260505 03:30:20.608 | INFO | 1, | 20260505 03:30:20.608 | INFO | 64 | 20260505 03:30:20.608 | INFO | ], | 20260505 03:30:20.608 | INFO | [ | 20260505 03:30:20.608 | INFO | 1664, | 20260505 03:30:20.608 | INFO | 6016, | 20260505 03:30:20.608 | INFO | 1, | 20260505 03:30:20.608 | INFO | 64 | 20260505 03:30:20.608 | INFO | ], | 20260505 03:30:20.609 | INFO | [ | 20260505 03:30:20.609 | INFO | 1920, | 20260505 03:30:20.609 | INFO | 6016, | 20260505 03:30:20.609 | INFO | 1, | 20260505 03:30:20.609 | INFO | 64 | 20260505 03:30:20.609 | INFO | ], | 20260505 03:30:20.609 | INFO | [ | 20260505 03:30:20.609 | INFO | 2176, | 20260505 03:30:20.609 | INFO | 6016, | 20260505 03:30:20.609 | INFO | 1, | 20260505 03:30:20.609 | INFO | 64 | 20260505 03:30:20.609 | INFO | ], | 20260505 03:30:20.609 | INFO | [ | 20260505 03:30:20.609 | INFO | 2432, | 20260505 03:30:20.609 | INFO | 6016, | 20260505 03:30:20.609 | INFO | 1, | 20260505 03:30:20.609 | INFO | 64 | 20260505 03:30:20.609 | INFO | ], | 20260505 03:30:20.609 | INFO | [ | 20260505 03:30:20.609 | INFO | 2688, | 20260505 03:30:20.609 | INFO | 6016, | 20260505 03:30:20.609 | INFO | 1, | 20260505 03:30:20.609 | INFO | 64 | 20260505 03:30:20.609 | INFO | ], | 20260505 03:30:20.609 | INFO | [ | 20260505 03:30:20.609 | INFO | 2944, | 20260505 03:30:20.609 | INFO | 6016, | 20260505 03:30:20.609 | INFO | 1, | 20260505 03:30:20.609 | INFO | 64 | 20260505 03:30:20.609 | INFO | ], | 20260505 03:30:20.609 | INFO | [ | 20260505 03:30:20.609 | INFO | 3200, | 20260505 03:30:20.609 | INFO | 6016, | 20260505 03:30:20.609 | INFO | 1, | 20260505 03:30:20.609 | INFO | 64 | 20260505 03:30:20.609 | INFO | ], | 20260505 03:30:20.609 | INFO | [ | 20260505 03:30:20.609 | INFO | 3456, | 20260505 03:30:20.609 | INFO | 6016, | 20260505 03:30:20.609 | INFO | 1, | 20260505 03:30:20.609 | INFO | 64 | 20260505 03:30:20.609 | INFO | ], | 20260505 03:30:20.609 | INFO | [ | 20260505 03:30:20.609 | INFO | 3712, | 20260505 03:30:20.609 | INFO | 6016, | 20260505 03:30:20.609 | INFO | 1, | 20260505 03:30:20.609 | INFO | 64 | 20260505 03:30:20.609 | INFO | ], | 20260505 03:30:20.609 | INFO | [ | 20260505 03:30:20.609 | INFO | 3968, | 20260505 03:30:20.609 | INFO | 6016, | 20260505 03:30:20.609 | INFO | 1, | 20260505 03:30:20.609 | INFO | 64 | 20260505 03:30:20.609 | INFO | ] | 20260505 03:30:20.609 | INFO | ], | 20260505 03:30:20.609 | INFO | [ | 20260505 03:30:20.609 | INFO | [ | 20260505 03:30:20.609 | INFO | 409, | 20260505 03:30:20.609 | INFO | 384, | 20260505 03:30:20.609 | INFO | 1, | 20260505 03:30:20.609 | INFO | 64 | 20260505 03:30:20.609 | INFO | ], | 20260505 03:30:20.609 | INFO | [ | 20260505 03:30:20.609 | INFO | 1228, | 20260505 03:30:20.609 | INFO | 384, | 20260505 03:30:20.609 | INFO | 1, | 20260505 03:30:20.609 | INFO | 64 | 20260505 03:30:20.609 | INFO | ], | 20260505 03:30:20.609 | INFO | [ | 20260505 03:30:20.609 | INFO | 2047, | 20260505 03:30:20.609 | INFO | 384, | 20260505 03:30:20.609 | INFO | 1, | 20260505 03:30:20.609 | INFO | 64 | 20260505 03:30:20.609 | INFO | ], | 20260505 03:30:20.609 | INFO | [ | 20260505 03:30:20.609 | INFO | 2866, | 20260505 03:30:20.609 | INFO | 384, | 20260505 03:30:20.609 | INFO | 1, | 20260505 03:30:20.609 | INFO | 64 | 20260505 03:30:20.609 | INFO | ], | 20260505 03:30:20.609 | INFO | [ | 20260505 03:30:20.609 | INFO | 3685, | 20260505 03:30:20.609 | INFO | 384, | 20260505 03:30:20.609 | INFO | 1, | 20260505 03:30:20.610 | INFO | 64 | 20260505 03:30:20.610 | INFO | ], | 20260505 03:30:20.610 | INFO | [ | 20260505 03:30:20.610 | INFO | 409, | 20260505 03:30:20.610 | INFO | 1152, | 20260505 03:30:20.610 | INFO | 1, | 20260505 03:30:20.610 | INFO | 64 | 20260505 03:30:20.610 | INFO | ], | 20260505 03:30:20.610 | INFO | [ | 20260505 03:30:20.610 | INFO | 1228, | 20260505 03:30:20.610 | INFO | 1152, | 20260505 03:30:20.610 | INFO | 1, | 20260505 03:30:20.610 | INFO | 64 | 20260505 03:30:20.610 | INFO | ], | 20260505 03:30:20.610 | INFO | [ | 20260505 03:30:20.610 | INFO | 2047, | 20260505 03:30:20.610 | INFO | 1152, | 20260505 03:30:20.610 | INFO | 1, | 20260505 03:30:20.610 | INFO | 64 | 20260505 03:30:20.610 | INFO | ], | 20260505 03:30:20.610 | INFO | [ | 20260505 03:30:20.610 | INFO | 2866, | 20260505 03:30:20.610 | INFO | 1152, | 20260505 03:30:20.610 | INFO | 1, | 20260505 03:30:20.610 | INFO | 64 | 20260505 03:30:20.610 | INFO | ], | 20260505 03:30:20.610 | INFO | [ | 20260505 03:30:20.610 | INFO | 3685, | 20260505 03:30:20.610 | INFO | 1152, | 20260505 03:30:20.610 | INFO | 1, | 20260505 03:30:20.610 | INFO | 64 | 20260505 03:30:20.610 | INFO | ], | 20260505 03:30:20.610 | INFO | [ | 20260505 03:30:20.610 | INFO | 409, | 20260505 03:30:20.610 | INFO | 1920, | 20260505 03:30:20.610 | INFO | 1, | 20260505 03:30:20.610 | INFO | 64 | 20260505 03:30:20.610 | INFO | ], | 20260505 03:30:20.610 | INFO | [ | 20260505 03:30:20.610 | INFO | 1228, | 20260505 03:30:20.610 | INFO | 1920, | 20260505 03:30:20.610 | INFO | 1, | 20260505 03:30:20.610 | INFO | 64 | 20260505 03:30:20.610 | INFO | ], | 20260505 03:30:20.610 | INFO | [ | 20260505 03:30:20.610 | INFO | 2047, | 20260505 03:30:20.610 | INFO | 1920, | 20260505 03:30:20.610 | INFO | 1, | 20260505 03:30:20.610 | INFO | 64 | 20260505 03:30:20.610 | INFO | ], | 20260505 03:30:20.610 | INFO | [ | 20260505 03:30:20.610 | INFO | 2866, | 20260505 03:30:20.610 | INFO | 1920, | 20260505 03:30:20.610 | INFO | 1, | 20260505 03:30:20.610 | INFO | 64 | 20260505 03:30:20.610 | INFO | ], | 20260505 03:30:20.610 | INFO | [ | 20260505 03:30:20.610 | INFO | 3685, | 20260505 03:30:20.610 | INFO | 1920, | 20260505 03:30:20.610 | INFO | 1, | 20260505 03:30:20.610 | INFO | 64 | 20260505 03:30:20.610 | INFO | ], | 20260505 03:30:20.610 | INFO | [ | 20260505 03:30:20.610 | INFO | 409, | 20260505 03:30:20.610 | INFO | 2688, | 20260505 03:30:20.610 | INFO | 1, | 20260505 03:30:20.610 | INFO | 64 | 20260505 03:30:20.610 | INFO | ], | 20260505 03:30:20.610 | INFO | [ | 20260505 03:30:20.610 | INFO | 1228, | 20260505 03:30:20.610 | INFO | 2688, | 20260505 03:30:20.610 | INFO | 1, | 20260505 03:30:20.610 | INFO | 64 | 20260505 03:30:20.610 | INFO | ], | 20260505 03:30:20.610 | INFO | [ | 20260505 03:30:20.610 | INFO | 2047, | 20260505 03:30:20.610 | INFO | 2688, | 20260505 03:30:20.610 | INFO | 1, | 20260505 03:30:20.610 | INFO | 64 | 20260505 03:30:20.610 | INFO | ], | 20260505 03:30:20.610 | INFO | [ | 20260505 03:30:20.610 | INFO | 2866, | 20260505 03:30:20.610 | INFO | 2688, | 20260505 03:30:20.610 | INFO | 1, | 20260505 03:30:20.611 | INFO | 64 | 20260505 03:30:20.611 | INFO | ], | 20260505 03:30:20.611 | INFO | [ | 20260505 03:30:20.611 | INFO | 3685, | 20260505 03:30:20.611 | INFO | 2688, | 20260505 03:30:20.611 | INFO | 1, | 20260505 03:30:20.611 | INFO | 64 | 20260505 03:30:20.611 | INFO | ], | 20260505 03:30:20.611 | INFO | [ | 20260505 03:30:20.611 | INFO | 409, | 20260505 03:30:20.611 | INFO | 3456, | 20260505 03:30:20.611 | INFO | 1, | 20260505 03:30:20.611 | INFO | 64 | 20260505 03:30:20.611 | INFO | ], | 20260505 03:30:20.611 | INFO | [ | 20260505 03:30:20.611 | INFO | 1228, | 20260505 03:30:20.611 | INFO | 3456, | 20260505 03:30:20.611 | INFO | 1, | 20260505 03:30:20.611 | INFO | 64 | 20260505 03:30:20.611 | INFO | ], | 20260505 03:30:20.611 | INFO | [ | 20260505 03:30:20.611 | INFO | 2047, | 20260505 03:30:20.611 | INFO | 3456, | 20260505 03:30:20.611 | INFO | 1, | 20260505 03:30:20.611 | INFO | 64 | 20260505 03:30:20.611 | INFO | ], | 20260505 03:30:20.611 | INFO | [ | 20260505 03:30:20.611 | INFO | 2866, | 20260505 03:30:20.611 | INFO | 3456, | 20260505 03:30:20.611 | INFO | 1, | 20260505 03:30:20.611 | INFO | 64 | 20260505 03:30:20.611 | INFO | ], | 20260505 03:30:20.611 | INFO | [ | 20260505 03:30:20.611 | INFO | 3685, | 20260505 03:30:20.611 | INFO | 3456, | 20260505 03:30:20.611 | INFO | 1, | 20260505 03:30:20.611 | INFO | 64 | 20260505 03:30:20.611 | INFO | ], | 20260505 03:30:20.611 | INFO | [ | 20260505 03:30:20.611 | INFO | 409, | 20260505 03:30:20.611 | INFO | 4224, | 20260505 03:30:20.611 | INFO | 1, | 20260505 03:30:20.611 | INFO | 64 | 20260505 03:30:20.611 | INFO | ], | 20260505 03:30:20.611 | INFO | [ | 20260505 03:30:20.611 | INFO | 1228, | 20260505 03:30:20.611 | INFO | 4224, | 20260505 03:30:20.611 | INFO | 1, | 20260505 03:30:20.611 | INFO | 64 | 20260505 03:30:20.611 | INFO | ], | 20260505 03:30:20.611 | INFO | [ | 20260505 03:30:20.611 | INFO | 2047, | 20260505 03:30:20.611 | INFO | 4224, | 20260505 03:30:20.611 | INFO | 1, | 20260505 03:30:20.611 | INFO | 64 | 20260505 03:30:20.611 | INFO | ], | 20260505 03:30:20.611 | INFO | [ | 20260505 03:30:20.611 | INFO | 2866, | 20260505 03:30:20.611 | INFO | 4224, | 20260505 03:30:20.611 | INFO | 1, | 20260505 03:30:20.611 | INFO | 64 | 20260505 03:30:20.611 | INFO | ], | 20260505 03:30:20.611 | INFO | [ | 20260505 03:30:20.611 | INFO | 3685, | 20260505 03:30:20.611 | INFO | 4224, | 20260505 03:30:20.611 | INFO | 1, | 20260505 03:30:20.611 | INFO | 64 | 20260505 03:30:20.611 | INFO | ], | 20260505 03:30:20.611 | INFO | [ | 20260505 03:30:20.611 | INFO | 409, | 20260505 03:30:20.611 | INFO | 4992, | 20260505 03:30:20.611 | INFO | 1, | 20260505 03:30:20.611 | INFO | 64 | 20260505 03:30:20.611 | INFO | ], | 20260505 03:30:20.611 | INFO | [ | 20260505 03:30:20.611 | INFO | 1228, | 20260505 03:30:20.611 | INFO | 4992, | 20260505 03:30:20.611 | INFO | 1, | 20260505 03:30:20.611 | INFO | 64 | 20260505 03:30:20.611 | INFO | ], | 20260505 03:30:20.611 | INFO | [ | 20260505 03:30:20.611 | INFO | 2047, | 20260505 03:30:20.611 | INFO | 4992, | 20260505 03:30:20.611 | INFO | 1, | 20260505 03:30:20.612 | INFO | 64 | 20260505 03:30:20.612 | INFO | ], | 20260505 03:30:20.612 | INFO | [ | 20260505 03:30:20.612 | INFO | 2866, | 20260505 03:30:20.612 | INFO | 4992, | 20260505 03:30:20.612 | INFO | 1, | 20260505 03:30:20.612 | INFO | 64 | 20260505 03:30:20.612 | INFO | ], | 20260505 03:30:20.612 | INFO | [ | 20260505 03:30:20.612 | INFO | 3685, | 20260505 03:30:20.612 | INFO | 4992, | 20260505 03:30:20.612 | INFO | 1, | 20260505 03:30:20.612 | INFO | 64 | 20260505 03:30:20.612 | INFO | ], | 20260505 03:30:20.612 | INFO | [ | 20260505 03:30:20.612 | INFO | 409, | 20260505 03:30:20.612 | INFO | 5760, | 20260505 03:30:20.612 | INFO | 1, | 20260505 03:30:20.612 | INFO | 64 | 20260505 03:30:20.612 | INFO | ], | 20260505 03:30:20.612 | INFO | [ | 20260505 03:30:20.612 | INFO | 1228, | 20260505 03:30:20.612 | INFO | 5760, | 20260505 03:30:20.612 | INFO | 1, | 20260505 03:30:20.612 | INFO | 64 | 20260505 03:30:20.612 | INFO | ], | 20260505 03:30:20.612 | INFO | [ | 20260505 03:30:20.612 | INFO | 2047, | 20260505 03:30:20.612 | INFO | 5760, | 20260505 03:30:20.612 | INFO | 1, | 20260505 03:30:20.612 | INFO | 64 | 20260505 03:30:20.612 | INFO | ], | 20260505 03:30:20.612 | INFO | [ | 20260505 03:30:20.612 | INFO | 2866, | 20260505 03:30:20.612 | INFO | 5760, | 20260505 03:30:20.612 | INFO | 1, | 20260505 03:30:20.612 | INFO | 64 | 20260505 03:30:20.612 | INFO | ], | 20260505 03:30:20.612 | INFO | [ | 20260505 03:30:20.612 | INFO | 3685, | 20260505 03:30:20.612 | INFO | 5760, | 20260505 03:30:20.612 | INFO | 1, | 20260505 03:30:20.612 | INFO | 64 | 20260505 03:30:20.612 | INFO | ] | 20260505 03:30:20.612 | INFO | ] | 20260505 03:30:20.612 | INFO | ] | 20260505 03:30:20.612 | INFO | } | 20260505 03:30:20.612 | INFO | }, | 20260505 03:30:20.612 | INFO | "dataSource" : | 20260505 03:30:20.612 | INFO | { | 20260505 03:30:20.612 | INFO | "crosstalkFilterKernel" : [], | 20260505 03:30:20.612 | INFO | "darkCalFileName" : "/data/icc/m84031_260504_231011_s1/pb_internal/m84031_260504_231011_s1.darkcal.h5", | 20260505 03:30:20.612 | INFO | "imagePsfKernel" : | 20260505 03:30:20.612 | INFO | [ | 20260505 03:30:20.612 | INFO | [ | 20260505 03:30:20.612 | INFO | 0.00080000000000000004, | 20260505 03:30:20.612 | INFO | 0.0030000000000000001, | 20260505 03:30:20.612 | INFO | 0.0166, | 20260505 03:30:20.612 | INFO | 0.0035000000000000001, | 20260505 03:30:20.612 | INFO | 0.0011000000000000001 | 20260505 03:30:20.612 | INFO | ], | 20260505 03:30:20.612 | INFO | [ | 20260505 03:30:20.612 | INFO | 0.0027000000000000001, | 20260505 03:30:20.612 | INFO | 0.012999999999999999, | 20260505 03:30:20.612 | INFO | 0.072700000000000001, | 20260505 03:30:20.612 | INFO | 0.014, | 20260505 03:30:20.612 | INFO | 0.0032000000000000002 | 20260505 03:30:20.612 | INFO | ], | 20260505 03:30:20.612 | INFO | [ | 20260505 03:30:20.612 | INFO | 0.0118, | 20260505 03:30:20.612 | INFO | 0.0545, | 20260505 03:30:20.612 | INFO | 0.59440000000000004, | 20260505 03:30:20.612 | INFO | 0.065299999999999997, | 20260505 03:30:20.612 | INFO | 0.0124 | 20260505 03:30:20.612 | INFO | ], | 20260505 03:30:20.612 | INFO | [ | 20260505 03:30:20.612 | INFO | 0.0032000000000000002, | 20260505 03:30:20.612 | INFO | 0.013100000000000001, | 20260505 03:30:20.612 | INFO | 0.072700000000000001, | 20260505 03:30:20.612 | INFO | 0.0141, | 20260505 03:30:20.612 | INFO | 0.0030999999999999999 | 20260505 03:30:20.612 | INFO | ], | 20260505 03:30:20.613 | INFO | [ | 20260505 03:30:20.613 | INFO | 0.001, | 20260505 03:30:20.613 | INFO | 0.0028999999999999998, | 20260505 03:30:20.613 | INFO | 0.016799999999999999, | 20260505 03:30:20.613 | INFO | 0.0033999999999999998, | 20260505 03:30:20.613 | INFO | 0.00089999999999999998 | 20260505 03:30:20.613 | INFO | ] | 20260505 03:30:20.613 | INFO | ] | 20260505 03:30:20.613 | INFO | }, | 20260505 03:30:20.613 | INFO | "internalMode" : false, | 20260505 03:30:20.613 | INFO | "layout" : | 20260505 03:30:20.613 | INFO | { | 20260505 03:30:20.613 | INFO | "framesPerChunk" : 512, | 20260505 03:30:20.613 | INFO | "lanesPerPool" : 4096, | 20260505 03:30:20.613 | INFO | "zmwsPerLane" : 64 | 20260505 03:30:20.613 | INFO | }, | 20260505 03:30:20.613 | INFO | "maskFraction" : 1, | 20260505 03:30:20.613 | INFO | "monitoringReportInterval" : 8192, | 20260505 03:30:20.613 | INFO | "multipleBazFiles" : true, | 20260505 03:30:20.613 | INFO | "prelimHQ" : | 20260505 03:30:20.613 | INFO | { | 20260505 03:30:20.613 | INFO | "enablePreHQ" : true, | 20260505 03:30:20.613 | INFO | "expectedFrameRate" : 100, | 20260505 03:30:20.613 | INFO | "expectedPulsesPerZmw" : 164, | 20260505 03:30:20.613 | INFO | "fracMultiBlocks" : 0.20000000298023224, | 20260505 03:30:20.613 | INFO | "fracSingleBlocks" : 0.5, | 20260505 03:30:20.613 | INFO | "hqBufferGrowthFactor" : 2, | 20260505 03:30:20.613 | INFO | "hqThrottleFraction" : 1, | 20260505 03:30:20.613 | INFO | "maxPreHQBuffers" : 1, | 20260505 03:30:20.613 | INFO | "maxSustainedPulseRate" : 4, | 20260505 03:30:20.613 | INFO | "preHQBufferGrowthFactor" : 1, | 20260505 03:30:20.613 | INFO | "requiredActivityLabels" : 10, | 20260505 03:30:20.613 | INFO | "zmwOutputStride" : 1 | 20260505 03:30:20.613 | INFO | }, | 20260505 03:30:20.613 | INFO | "realTimeMetrics" : | 20260505 03:30:20.613 | INFO | { | 20260505 03:30:20.613 | INFO | "csvOutputFile" : "", | 20260505 03:30:20.613 | INFO | "diagOutputFile" : "", | 20260505 03:30:20.613 | INFO | "jsonOutputFile" : "/data/nrta/2/m84031_260504_231011_s1/pb_internal/m84031_260504_231011_s1.rtmetrics.json", | 20260505 03:30:20.613 | INFO | "minConfidenceScore" : 0, | 20260505 03:30:20.613 | INFO | "regions" : | 20260505 03:30:20.613 | INFO | [ | 20260505 03:30:20.613 | INFO | { | 20260505 03:30:20.613 | INFO | "featuresForFilter" : | 20260505 03:30:20.613 | INFO | [ | 20260505 03:30:20.613 | INFO | "PorSequencing" | 20260505 03:30:20.613 | INFO | ], | 20260505 03:30:20.613 | INFO | "medianIntraLaneStride" : 8, | 20260505 03:30:20.613 | INFO | "metrics" : | 20260505 03:30:20.613 | INFO | [ | 20260505 03:30:20.613 | INFO | "PulseRate", | 20260505 03:30:20.613 | INFO | "PulseWidth", | 20260505 03:30:20.613 | INFO | "BaseRate", | 20260505 03:30:20.613 | INFO | "BaseWidth", | 20260505 03:30:20.613 | INFO | "Baseline", | 20260505 03:30:20.613 | INFO | "BaselineStd", | 20260505 03:30:20.613 | INFO | "Pkmid", | 20260505 03:30:20.613 | INFO | "Snr" | 20260505 03:30:20.613 | INFO | ], | 20260505 03:30:20.613 | INFO | "minSampleSize" : 1000, | 20260505 03:30:20.613 | INFO | "name" : "FullChip", | 20260505 03:30:20.613 | INFO | "roi" : | 20260505 03:30:20.613 | INFO | [ | 20260505 03:30:20.613 | INFO | [ | 20260505 03:30:20.613 | INFO | 0, | 20260505 03:30:20.613 | INFO | 0, | 20260505 03:30:20.613 | INFO | 4096, | 20260505 03:30:20.613 | INFO | 6144 | 20260505 03:30:20.613 | INFO | ] | 20260505 03:30:20.613 | INFO | ], | 20260505 03:30:20.613 | INFO | "useSingleActivityLabels" : true | 20260505 03:30:20.613 | INFO | }, | 20260505 03:30:20.613 | INFO | { | 20260505 03:30:20.613 | INFO | "featuresForFilter" : | 20260505 03:30:20.613 | INFO | [ | 20260505 03:30:20.613 | INFO | "PorSequencing" | 20260505 03:30:20.613 | INFO | ], | 20260505 03:30:20.613 | INFO | "medianIntraLaneStride" : 1, | 20260505 03:30:20.613 | INFO | "metrics" : | 20260505 03:30:20.613 | INFO | [ | 20260505 03:30:20.613 | INFO | "Baseline", | 20260505 03:30:20.613 | INFO | "BaselineStd", | 20260505 03:30:20.613 | INFO | "Pkmid", | 20260505 03:30:20.613 | INFO | "Snr" | 20260505 03:30:20.614 | INFO | ], | 20260505 03:30:20.614 | INFO | "minSampleSize" : 1000, | 20260505 03:30:20.614 | INFO | "name" : "TopStrip", | 20260505 03:30:20.614 | INFO | "roi" : | 20260505 03:30:20.614 | INFO | [ | 20260505 03:30:20.614 | INFO | [ | 20260505 03:30:20.614 | INFO | 200, | 20260505 03:30:20.614 | INFO | 64, | 20260505 03:30:20.614 | INFO | 64, | 20260505 03:30:20.614 | INFO | 6016 | 20260505 03:30:20.614 | INFO | ] | 20260505 03:30:20.614 | INFO | ], | 20260505 03:30:20.614 | INFO | "useSingleActivityLabels" : true | 20260505 03:30:20.614 | INFO | }, | 20260505 03:30:20.614 | INFO | { | 20260505 03:30:20.614 | INFO | "featuresForFilter" : | 20260505 03:30:20.614 | INFO | [ | 20260505 03:30:20.614 | INFO | "PorSequencing" | 20260505 03:30:20.614 | INFO | ], | 20260505 03:30:20.614 | INFO | "medianIntraLaneStride" : 1, | 20260505 03:30:20.614 | INFO | "metrics" : | 20260505 03:30:20.614 | INFO | [ | 20260505 03:30:20.614 | INFO | "Baseline", | 20260505 03:30:20.614 | INFO | "BaselineStd", | 20260505 03:30:20.614 | INFO | "Pkmid", | 20260505 03:30:20.614 | INFO | "Snr" | 20260505 03:30:20.614 | INFO | ], | 20260505 03:30:20.614 | INFO | "minSampleSize" : 1000, | 20260505 03:30:20.614 | INFO | "name" : "MidStrip", | 20260505 03:30:20.614 | INFO | "roi" : | 20260505 03:30:20.614 | INFO | [ | 20260505 03:30:20.614 | INFO | [ | 20260505 03:30:20.614 | INFO | 2016, | 20260505 03:30:20.614 | INFO | 64, | 20260505 03:30:20.614 | INFO | 64, | 20260505 03:30:20.614 | INFO | 6016 | 20260505 03:30:20.614 | INFO | ] | 20260505 03:30:20.614 | INFO | ], | 20260505 03:30:20.614 | INFO | "useSingleActivityLabels" : true | 20260505 03:30:20.614 | INFO | }, | 20260505 03:30:20.614 | INFO | { | 20260505 03:30:20.614 | INFO | "featuresForFilter" : | 20260505 03:30:20.614 | INFO | [ | 20260505 03:30:20.614 | INFO | "PorSequencing" | 20260505 03:30:20.614 | INFO | ], | 20260505 03:30:20.614 | INFO | "medianIntraLaneStride" : 1, | 20260505 03:30:20.614 | INFO | "metrics" : | 20260505 03:30:20.614 | INFO | [ | 20260505 03:30:20.614 | INFO | "Baseline", | 20260505 03:30:20.614 | INFO | "BaselineStd", | 20260505 03:30:20.614 | INFO | "Pkmid", | 20260505 03:30:20.614 | INFO | "Snr" | 20260505 03:30:20.614 | INFO | ], | 20260505 03:30:20.614 | INFO | "minSampleSize" : 1000, | 20260505 03:30:20.614 | INFO | "name" : "BotStrip", | 20260505 03:30:20.614 | INFO | "roi" : | 20260505 03:30:20.614 | INFO | [ | 20260505 03:30:20.614 | INFO | [ | 20260505 03:30:20.614 | INFO | 3832, | 20260505 03:30:20.614 | INFO | 64, | 20260505 03:30:20.614 | INFO | 64, | 20260505 03:30:20.614 | INFO | 6016 | 20260505 03:30:20.614 | INFO | ] | 20260505 03:30:20.614 | INFO | ], | 20260505 03:30:20.614 | INFO | "useSingleActivityLabels" : true | 20260505 03:30:20.614 | INFO | }, | 20260505 03:30:20.614 | INFO | { | 20260505 03:30:20.614 | INFO | "featuresForFilter" : | 20260505 03:30:20.614 | INFO | [ | 20260505 03:30:20.614 | INFO | "LaserPower2p0x" | 20260505 03:30:20.614 | INFO | ], | 20260505 03:30:20.614 | INFO | "medianIntraLaneStride" : 1, | 20260505 03:30:20.614 | INFO | "metrics" : | 20260505 03:30:20.614 | INFO | [ | 20260505 03:30:20.614 | INFO | "Baseline", | 20260505 03:30:20.614 | INFO | "BaselineStd", | 20260505 03:30:20.614 | INFO | "Pkmid", | 20260505 03:30:20.614 | INFO | "Snr" | 20260505 03:30:20.614 | INFO | ], | 20260505 03:30:20.614 | INFO | "minSampleSize" : 1000, | 20260505 03:30:20.614 | INFO | "name" : "FullChip_LP2P0X", | 20260505 03:30:20.614 | INFO | "roi" : | 20260505 03:30:20.614 | INFO | [ | 20260505 03:30:20.614 | INFO | [ | 20260505 03:30:20.615 | INFO | 0, | 20260505 03:30:20.615 | INFO | 0, | 20260505 03:30:20.615 | INFO | 4096, | 20260505 03:30:20.615 | INFO | 6144 | 20260505 03:30:20.615 | INFO | ] | 20260505 03:30:20.615 | INFO | ], | 20260505 03:30:20.615 | INFO | "useSingleActivityLabels" : true | 20260505 03:30:20.615 | INFO | }, | 20260505 03:30:20.615 | INFO | { | 20260505 03:30:20.615 | INFO | "featuresForFilter" : | 20260505 03:30:20.615 | INFO | [ | 20260505 03:30:20.615 | INFO | "LaserPower1p5x" | 20260505 03:30:20.615 | INFO | ], | 20260505 03:30:20.615 | INFO | "medianIntraLaneStride" : 1, | 20260505 03:30:20.615 | INFO | "metrics" : | 20260505 03:30:20.615 | INFO | [ | 20260505 03:30:20.615 | INFO | "Baseline", | 20260505 03:30:20.615 | INFO | "BaselineStd", | 20260505 03:30:20.615 | INFO | "Pkmid", | 20260505 03:30:20.615 | INFO | "Snr" | 20260505 03:30:20.615 | INFO | ], | 20260505 03:30:20.615 | INFO | "minSampleSize" : 1000, | 20260505 03:30:20.615 | INFO | "name" : "FullChip_LP1P5X", | 20260505 03:30:20.615 | INFO | "roi" : | 20260505 03:30:20.615 | INFO | [ | 20260505 03:30:20.615 | INFO | [ | 20260505 03:30:20.615 | INFO | 0, | 20260505 03:30:20.615 | INFO | 0, | 20260505 03:30:20.615 | INFO | 4096, | 20260505 03:30:20.615 | INFO | 6144 | 20260505 03:30:20.615 | INFO | ] | 20260505 03:30:20.615 | INFO | ], | 20260505 03:30:20.615 | INFO | "useSingleActivityLabels" : true | 20260505 03:30:20.615 | INFO | }, | 20260505 03:30:20.615 | INFO | { | 20260505 03:30:20.615 | INFO | "featuresForFilter" : | 20260505 03:30:20.615 | INFO | [ | 20260505 03:30:20.615 | INFO | "LaserPower0p5x" | 20260505 03:30:20.615 | INFO | ], | 20260505 03:30:20.615 | INFO | "medianIntraLaneStride" : 1, | 20260505 03:30:20.615 | INFO | "metrics" : | 20260505 03:30:20.615 | INFO | [ | 20260505 03:30:20.615 | INFO | "Baseline", | 20260505 03:30:20.615 | INFO | "BaselineStd", | 20260505 03:30:20.615 | INFO | "Pkmid", | 20260505 03:30:20.615 | INFO | "Snr" | 20260505 03:30:20.615 | INFO | ], | 20260505 03:30:20.615 | INFO | "minSampleSize" : 1000, | 20260505 03:30:20.615 | INFO | "name" : "FullChip_LP0P5X", | 20260505 03:30:20.615 | INFO | "roi" : | 20260505 03:30:20.615 | INFO | [ | 20260505 03:30:20.615 | INFO | [ | 20260505 03:30:20.615 | INFO | 0, | 20260505 03:30:20.615 | INFO | 0, | 20260505 03:30:20.615 | INFO | 4096, | 20260505 03:30:20.615 | INFO | 6144 | 20260505 03:30:20.615 | INFO | ] | 20260505 03:30:20.615 | INFO | ], | 20260505 03:30:20.615 | INFO | "useSingleActivityLabels" : true | 20260505 03:30:20.615 | INFO | }, | 20260505 03:30:20.615 | INFO | { | 20260505 03:30:20.615 | INFO | "featuresForFilter" : | 20260505 03:30:20.615 | INFO | [ | 20260505 03:30:20.615 | INFO | "LaserPower0p0x" | 20260505 03:30:20.615 | INFO | ], | 20260505 03:30:20.615 | INFO | "medianIntraLaneStride" : 1, | 20260505 03:30:20.615 | INFO | "metrics" : | 20260505 03:30:20.615 | INFO | [ | 20260505 03:30:20.615 | INFO | "Baseline", | 20260505 03:30:20.615 | INFO | "BaselineStd", | 20260505 03:30:20.615 | INFO | "Pkmid", | 20260505 03:30:20.615 | INFO | "Snr" | 20260505 03:30:20.615 | INFO | ], | 20260505 03:30:20.615 | INFO | "minSampleSize" : 1000, | 20260505 03:30:20.615 | INFO | "name" : "FullChip_LP0P0X", | 20260505 03:30:20.615 | INFO | "roi" : | 20260505 03:30:20.615 | INFO | [ | 20260505 03:30:20.615 | INFO | [ | 20260505 03:30:20.615 | INFO | 0, | 20260505 03:30:20.615 | INFO | 0, | 20260505 03:30:20.615 | INFO | 4096, | 20260505 03:30:20.615 | INFO | 6144 | 20260505 03:30:20.616 | INFO | ] | 20260505 03:30:20.616 | INFO | ], | 20260505 03:30:20.616 | INFO | "useSingleActivityLabels" : true | 20260505 03:30:20.616 | INFO | }, | 20260505 03:30:20.616 | INFO | { | 20260505 03:30:20.616 | INFO | "featuresForFilter" : | 20260505 03:30:20.616 | INFO | [ | 20260505 03:30:20.616 | INFO | "LaserScatter" | 20260505 03:30:20.616 | INFO | ], | 20260505 03:30:20.616 | INFO | "medianIntraLaneStride" : 1, | 20260505 03:30:20.616 | INFO | "metrics" : | 20260505 03:30:20.616 | INFO | [ | 20260505 03:30:20.616 | INFO | "Baseline", | 20260505 03:30:20.616 | INFO | "BaselineStd" | 20260505 03:30:20.616 | INFO | ], | 20260505 03:30:20.616 | INFO | "minSampleSize" : 1000, | 20260505 03:30:20.616 | INFO | "name" : "FullChip_LS", | 20260505 03:30:20.616 | INFO | "roi" : | 20260505 03:30:20.616 | INFO | [ | 20260505 03:30:20.616 | INFO | [ | 20260505 03:30:20.616 | INFO | 0, | 20260505 03:30:20.616 | INFO | 0, | 20260505 03:30:20.616 | INFO | 4096, | 20260505 03:30:20.616 | INFO | 6144 | 20260505 03:30:20.616 | INFO | ] | 20260505 03:30:20.616 | INFO | ], | 20260505 03:30:20.616 | INFO | "useSingleActivityLabels" : false | 20260505 03:30:20.616 | INFO | } | 20260505 03:30:20.616 | INFO | ] | 20260505 03:30:20.616 | INFO | }, | 20260505 03:30:20.616 | INFO | "reportChecking" : | 20260505 03:30:20.616 | INFO | { | 20260505 03:30:20.616 | INFO | "activeFractionThreshold" : 0.97999999999999998, | 20260505 03:30:20.616 | INFO | "growingQueueSizeThreshold" : 2, | 20260505 03:30:20.616 | INFO | "queueSizeThreshold" : 20, | 20260505 03:30:20.616 | INFO | "totalTimeMsThreshold" : 1000000 | 20260505 03:30:20.616 | INFO | }, | 20260505 03:30:20.616 | INFO | "smrtMask" : | 20260505 03:30:20.616 | INFO | { | 20260505 03:30:20.616 | INFO | "smrtMaskScoreFilePath" : "" | 20260505 03:30:20.616 | INFO | }, | 20260505 03:30:20.616 | INFO | "source" : | 20260505 03:30:20.616 | INFO | { | 20260505 03:30:20.616 | INFO | "WXIPCDataSourceConfig" : | 20260505 03:30:20.616 | INFO | { | 20260505 03:30:20.616 | INFO | "acqConfig" : | 20260505 03:30:20.616 | INFO | { | 20260505 03:30:20.616 | INFO | "A" : | 20260505 03:30:20.616 | INFO | { | 20260505 03:30:20.616 | INFO | "baseLabel" : "A", | 20260505 03:30:20.616 | INFO | "excessNoiseCV" : 0.10000000149011612, | 20260505 03:30:20.616 | INFO | "interPulseDistance" : 0.079999998211860657, | 20260505 03:30:20.616 | INFO | "ipd2SlowStepRatio" : 0, | 20260505 03:30:20.616 | INFO | "pulseWidth" : 0.16599999368190765, | 20260505 03:30:20.616 | INFO | "pw2SlowStepRatio" : 3.2000000476837158, | 20260505 03:30:20.616 | INFO | "relAmplitude" : 0.67000001668930054 | 20260505 03:30:20.616 | INFO | }, | 20260505 03:30:20.616 | INFO | "C" : | 20260505 03:30:20.616 | INFO | { | 20260505 03:30:20.616 | INFO | "baseLabel" : "C", | 20260505 03:30:20.616 | INFO | "excessNoiseCV" : 0.10000000149011612, | 20260505 03:30:20.616 | INFO | "interPulseDistance" : 0.070000000298023224, | 20260505 03:30:20.616 | INFO | "ipd2SlowStepRatio" : 0, | 20260505 03:30:20.616 | INFO | "pulseWidth" : 0.20900000631809235, | 20260505 03:30:20.616 | INFO | "pw2SlowStepRatio" : 3.2000000476837158, | 20260505 03:30:20.616 | INFO | "relAmplitude" : 1 | 20260505 03:30:20.616 | INFO | }, | 20260505 03:30:20.616 | INFO | "G" : | 20260505 03:30:20.616 | INFO | { | 20260505 03:30:20.616 | INFO | "baseLabel" : "G", | 20260505 03:30:20.616 | INFO | "excessNoiseCV" : 0.10000000149011612, | 20260505 03:30:20.616 | INFO | "interPulseDistance" : 0.070000000298023224, | 20260505 03:30:20.616 | INFO | "ipd2SlowStepRatio" : 0, | 20260505 03:30:20.616 | INFO | "pulseWidth" : 0.19300000369548798, | 20260505 03:30:20.616 | INFO | "pw2SlowStepRatio" : 3.2000000476837158, | 20260505 03:30:20.616 | INFO | "relAmplitude" : 0.25999999046325684 | 20260505 03:30:20.617 | INFO | }, | 20260505 03:30:20.617 | INFO | "T" : | 20260505 03:30:20.617 | INFO | { | 20260505 03:30:20.617 | INFO | "baseLabel" : "T", | 20260505 03:30:20.617 | INFO | "excessNoiseCV" : 0.10000000149011612, | 20260505 03:30:20.617 | INFO | "interPulseDistance" : 0.079999998211860657, | 20260505 03:30:20.617 | INFO | "ipd2SlowStepRatio" : 0, | 20260505 03:30:20.617 | INFO | "pulseWidth" : 0.16300000250339508, | 20260505 03:30:20.617 | INFO | "pw2SlowStepRatio" : 3.2000000476837158, | 20260505 03:30:20.617 | INFO | "relAmplitude" : 0.44499999284744263 | 20260505 03:30:20.617 | INFO | }, | 20260505 03:30:20.617 | INFO | "chipLayoutName" : "KestrelRTO2", | 20260505 03:30:20.617 | INFO | "photoelectronSensitivity" : 6.25, | 20260505 03:30:20.617 | INFO | "refSnr" : 16.700000762939453 | 20260505 03:30:20.617 | INFO | }, | 20260505 03:30:20.617 | INFO | "autoConnect" : true, | 20260505 03:30:20.617 | INFO | "canonicalFrameRate" : 100, | 20260505 03:30:20.617 | INFO | "chunkValidationLevel" : 0, | 20260505 03:30:20.617 | INFO | "daemonUrl" : "http://localhost:23602", | 20260505 03:30:20.617 | INFO | "dataMode" : "FRAME", | 20260505 03:30:20.617 | INFO | "dataPath" : "Normal", | 20260505 03:30:20.617 | INFO | "loopback" : false, | 20260505 03:30:20.617 | INFO | "maskCardinalityFraction" : 1, | 20260505 03:30:20.617 | INFO | "maskEncoding" : "COMPRESSEDBASE64_V2", | 20260505 03:30:20.617 | INFO | "maxPopLoops" : 10, | 20260505 03:30:20.617 | INFO | "pedestal" : -32768, | 20260505 03:30:20.617 | INFO | "pointerReplacementSafetyFactor" : 0.80000000000000004, | 20260505 03:30:20.617 | INFO | "primerScaler" : 1, | 20260505 03:30:20.617 | INFO | "simConfig" : | 20260505 03:30:20.617 | INFO | { | 20260505 03:30:20.617 | INFO | "numCols" : 0, | 20260505 03:30:20.617 | INFO | "numRows" : 0, | 20260505 03:30:20.617 | INFO | "numSubSensors" : 2, | 20260505 03:30:20.617 | INFO | "platformOverride" : "DONT_CARE", | 20260505 03:30:20.617 | INFO | "regionNumSlotsPerTrack" : 128, | 20260505 03:30:20.617 | INFO | "regionNumTracks" : 8, | 20260505 03:30:20.617 | INFO | "regionSize" : 1024 | 20260505 03:30:20.617 | INFO | }, | 20260505 03:30:20.617 | INFO | "simulatedFrameRate" : 100, | 20260505 03:30:20.617 | INFO | "simulatedInputFile" : "CONSTANT/123", | 20260505 03:30:20.617 | INFO | "simulationCondensePixels" : true, | 20260505 03:30:20.617 | INFO | "simulationNumRowsToReplicate" : 20, | 20260505 03:30:20.617 | INFO | "simulationRoi" : [], | 20260505 03:30:20.617 | INFO | "sleepDebug" : 0, | 20260505 03:30:20.617 | INFO | "smrtMaskMode" : "DEFAULT", | 20260505 03:30:20.617 | INFO | "sraIndex" : 0, | 20260505 03:30:20.617 | INFO | "tilePoolFactor" : 3, | 20260505 03:30:20.617 | INFO | "timeouts" : | 20260505 03:30:20.617 | INFO | { | 20260505 03:30:20.617 | INFO | "armedTimeoutSeconds" : 400, | 20260505 03:30:20.617 | INFO | "doneTimeoutSeconds" : 10, | 20260505 03:30:20.617 | INFO | "firstChunkTimeoutSeconds" : 400, | 20260505 03:30:20.617 | INFO | "httpTimeoutSeconds" : 10, | 20260505 03:30:20.617 | INFO | "notifiedTimeoutSeconds" : 15, | 20260505 03:30:20.617 | INFO | "resetTimeoutSeconds" : 15, | 20260505 03:30:20.617 | INFO | "subsequentChunkTimeoutSeconds" : -1, | 20260505 03:30:20.617 | INFO | "wxDaemonTimeoutSeconds" : 60 | 20260505 03:30:20.617 | INFO | }, | 20260505 03:30:20.617 | INFO | "verbosity" : 0 | 20260505 03:30:20.617 | INFO | } | 20260505 03:30:20.617 | INFO | }, | 20260505 03:30:20.617 | INFO | "system" : | 20260505 03:30:20.617 | INFO | { | 20260505 03:30:20.617 | INFO | "analyzerHardware" : "A100", | 20260505 03:30:20.617 | INFO | "basecallerBatchConcurrency" : 1, | 20260505 03:30:20.617 | INFO | "basecallerConcurrency" : 3, | 20260505 03:30:20.617 | INFO | "basecallerDeviceConcurrency" : 3, | 20260505 03:30:20.617 | INFO | "basecallerGpuDevices" : | 20260505 03:30:20.617 | INFO | [ | 20260505 03:30:20.617 | INFO | 0 | 20260505 03:30:20.617 | INFO | ], | 20260505 03:30:20.617 | INFO | "ioConcurrency" : 5, | 20260505 03:30:20.617 | INFO | "maxPermGpuDataMB" : 10000, | 20260505 03:30:20.617 | INFO | "numWorkerThreads" : 12, | 20260505 03:30:20.617 | INFO | "nvidiaSmiRunTimeSeconds" : 30, | 20260505 03:30:20.617 | INFO | "serialGpuCompute" : true | 20260505 03:30:20.617 | INFO | }, | 20260505 03:30:20.617 | INFO | "traceSaver" : | 20260505 03:30:20.617 | INFO | { | 20260505 03:30:20.617 | INFO | "bufferedChunksAllowed" : 5, | 20260505 03:30:20.618 | INFO | "frameChunking" : 512, | 20260505 03:30:20.618 | INFO | "outFormat" : "Natural", | 20260505 03:30:20.618 | INFO | "roi" : | 20260505 03:30:20.618 | INFO | [ | 20260505 03:30:20.618 | INFO | [ | 20260505 03:30:20.618 | INFO | 2048, | 20260505 03:30:20.618 | INFO | 768, | 20260505 03:30:20.618 | INFO | 1, | 20260505 03:30:20.618 | INFO | 64 | 20260505 03:30:20.618 | INFO | ], | 20260505 03:30:20.618 | INFO | [ | 20260505 03:30:20.618 | INFO | 2048, | 20260505 03:30:20.618 | INFO | 2304, | 20260505 03:30:20.618 | INFO | 1, | 20260505 03:30:20.618 | INFO | 64 | 20260505 03:30:20.618 | INFO | ], | 20260505 03:30:20.618 | INFO | [ | 20260505 03:30:20.618 | INFO | 2048, | 20260505 03:30:20.618 | INFO | 3840, | 20260505 03:30:20.618 | INFO | 1, | 20260505 03:30:20.618 | INFO | 64 | 20260505 03:30:20.618 | INFO | ], | 20260505 03:30:20.618 | INFO | [ | 20260505 03:30:20.618 | INFO | 2048, | 20260505 03:30:20.618 | INFO | 5376, | 20260505 03:30:20.618 | INFO | 1, | 20260505 03:30:20.618 | INFO | 64 | 20260505 03:30:20.618 | INFO | ] | 20260505 03:30:20.618 | INFO | ], | 20260505 03:30:20.618 | INFO | "zmwChunking" : 2048 | 20260505 03:30:20.618 | INFO | } | 20260505 03:30:20.618 | INFO | } | 20260505 03:30:20.618 | INFO | Output Target: /data/nrta/2/m84031_260504_231011_s1/pb_internal/baz/m84031_260504_231011_s1.bazfilelist.txt | 20260505 03:30:20.667 | INFO | Found 1 CUDA devices | 20260505 03:30:20.667 | INFO | CUDA GPU device: 0 | 20260505 03:30:20.667 | INFO | Device Name:NVIDIA A100 80GB PCIe | 20260505 03:30:20.667 | INFO | UUID:GPU-ce26be3f-889e-2795-9302-85bf090c57ae | 20260505 03:30:20.667 | INFO | Global memory:85095874560 | 20260505 03:30:20.667 | INFO | sharedMemPerBlock:49152 | 20260505 03:30:20.667 | INFO | major/minor:8/0 | 20260505 03:30:20.668 | INFO | No message | 20260505 03:30:20.668 | INFO | CpuSchedulerCanary: starting | 20260505 03:30:20.668 | INFO | Starting CPU scheduling canary | 20260505 03:30:20.670 | INFO | Waiting for wx-daemon. Current boot state:"Daemon ready" | 20260505 03:30:20.671 | INFO | Connecting to shared memory key: 0x50420051 at 0x00500000000 | 20260505 03:30:24.813 | WARN | Precalculated crosstalk filter will not be used, one of the dimensions is zero | 20260505 03:30:24.813 | INFO | PSF will be passed to the DataSource for processing. | 20260505 03:30:24.814 | INFO | WXIPCDataSource expectedMaskedZmws:25165824 | 20260505 03:30:24.814 | INFO | WXIPCDataSource expectedSensorZmws:25165824 | 20260505 03:30:24.814 | WARN | PSF:[ [ 0.00079984 0.0029994 0.0165967 0.0034993 0.00109978] [ 0.00269946 0.0129974 0.0726855 0.0139972 0.00319936] [ 0.0117976 0.0544891 0.594281 0.0652869 0.0123975] [ 0.00319936 0.0130974 0.0726855 0.0140972 0.00309938] [ 0.0009998 0.00289942 0.0167966 0.00339932 0.00089982] ] | 20260505 03:30:24.814 | WARN | bNorm:0.811438 | 20260505 03:30:24.814 | WARN | PSF is not that close to the identity PSF. Norm of deviation = 0.811438 | 20260505 03:30:24.815 | INFO | Based on platform 6 the smrtMaskMode default is set to DISABLED | 20260505 03:30:24.815 | INFO | Connecting to wx-daemon http://localhost:23602 | 20260505 03:30:24.815 | INFO | platform:Kestrel | 20260505 03:30:24.816 | INFO | wxShortVersion: 26.1.0 wxLongVersion: 26.1.0.a8206dc561f8e1cdf6683a2ad782923bb7106e43 wxIpcVersion: 3 wxPersonality: 1.8.2-0 dataSourceShortVersion: 26.1.0 dataSourceLongVersion: 26.1.0.0 dataSourceIpcVersion: 3 | 20260505 03:30:24.817 | INFO | Waiting for wx-daemon. Current boot state:"Daemon ready" | 20260505 03:30:24.918 | INFO | Connecting IPC | 20260505 03:30:24.920 | INFO | Reconfiguring:{ "cameraConfig" : 0, "cardinalityFraction" : 1, "crosstalkKernel" : [ [ 0.00014065283436721698, 0.00024733493649707156, -0.02646965601969033, 1.2938141044737791e-05, -0.000209726572126589 ], [ 2.8209306317858476e-05, -0.011227287975635325, -0.16249877620543987, -0.009973250094704407, -0.00028787427166992058 ], [ -0.019849015808807709, -0.11953626080922616, 1.7245260129463982, -0.14535734185743521, -0.018293397259077079 ], [ -0.0011299207645029945, -0.011351514333903183, -0.16237503050695248, -0.010276039796743052, -7.8888800056879383e-05 ], [ -0.00019585354582084264, 0.00063817726959822143, -0.026998977274235791, 0.00028844750081735671, 0.00022703896098751053 ] ], "darkCalFileName" : "/data/icc/m84031_260504_231011_s1/pb_internal/m84031_260504_231011_s1.darkcal.h5", "encoding" : "UINT8", "expectedZmws" : 0, "layoutDims" : [ 4096, 512, 64 ], "layoutType" : "BLOCK_LAYOUT_DENSE", "memoryPoolIndex" : -1, "mode" : "AUTO", "mosi_socket_address" : "tcp://127.0.0.1:34475", "numFrames" : 8640000, "pedestal" : -32768, "requireAuroraUp" : true, "smrtMaskType" : "" } | 20260505 03:30:24.920 | INFO | Reconfigure returned:OK "ok" | 20260505 03:30:25.021 | INFO | Connecting to SRA:0 using tcp://127.0.0.1:40063 | 20260505 03:30:25.022 | INFO | wx-daemon requests 192 pointers | 20260505 03:30:25.022 | INFO | wxipc is sending requests 192 pointers | 20260505 03:30:25.024 | INFO | Pushing 192 free packets, with packetSize 134217728 | 20260505 03:30:25.024 | INFO | Done, Pushed 192 free packets | 20260505 03:30:25.024 | INFO | Sent first free packet to tcp://127.0.0.1:40063 | 20260505 03:30:25.024 | INFO | bursted 192 free packets | 20260505 03:30:25.025 | INFO | Reconfigure_continue returned:OK "ok" | 20260505 03:30:31.464 | INFO | Polled wx-daemon for pedestal and cached 20 | 20260505 03:30:31.464 | INFO | Chip layout: KestrelRTO2 | 20260505 03:30:31.464 | INFO | Frame rate: 100 | 20260505 03:30:31.464 | INFO | Photo-electron sensitivity: 6.25 | 20260505 03:30:31.464 | INFO | Reference SNR: 16.7 | 20260505 03:30:31.464 | INFO | Analog properties: [C, RA 1, exnCV 0.1, ipd 0.07, pw 0.209, pw2ssr 3.2, ipd2ssr 0] | 20260505 03:30:31.464 | INFO | Analog properties: [A, RA 0.67, exnCV 0.1, ipd 0.08, pw 0.166, pw2ssr 3.2, ipd2ssr 0] | 20260505 03:30:31.464 | INFO | Analog properties: [T, RA 0.445, exnCV 0.1, ipd 0.08, pw 0.163, pw2ssr 3.2, ipd2ssr 0] | 20260505 03:30:31.464 | INFO | Analog properties: [G, RA 0.26, exnCV 0.1, ipd 0.07, pw 0.193, pw2ssr 3.2, ipd2ssr 0] | 20260505 03:30:31.464 | INFO | Number of analysis zmwLanes (after smrt masking) = 393216 | 20260505 03:30:31.464 | INFO | Number of analysis chunks = 16875 | 20260505 03:30:31.464 | INFO | Instantiating TrivialRepacker | 20260505 03:30:31.464 | WARN | WXIPCDataSource returning dummy instrument name WXIPCDataSource | 20260505 03:30:31.465 | INFO | OOM score went from -1 to 666 at CreateTracePrepper checkpoint 1 | 20260505 03:30:31.465 | INFO | memory usage change: vm:26471823 RSS:25915 shared:22513 text:4978 data:23695 at CreateTracePrepper checkpoint 1 | 20260505 03:30:31.465 | INFO | Added 2 lanes to the ROI | 20260505 03:30:31.465 | INFO | Added 2 lanes to the ROI | 20260505 03:30:31.466 | INFO | Added 2 lanes to the ROI | 20260505 03:30:31.466 | INFO | Added 2 lanes to the ROI | 20260505 03:30:32.122 | INFO | Creating cached UnitCellProperties for dataSource 0x4f34280 | 20260505 03:30:32.605 | INFO | Done creating cached UnitCellProperties for dataSource 0x4f34280 | 20260505 03:30:32.605 | INFO | memory usage change: vm:26618843 RSS:173489 shared:22561 text:4978 data:170715 at SmrtBasecaller::CreateTraceSaver checkpoint 2 | 20260505 03:30:32.608 | INFO | TraceSaverBody created | 20260505 03:30:32.627 | INFO | memory usage change: vm:26570370 RSS:124828 shared:22655 text:4978 data:122242 at SmrtBasecaller::RunAnalyzer checkpoint 1 | 20260505 03:30:32.627 | INFO | Resource: 0, Device: 0, Assigning 3 CudaStreams | 20260505 03:30:32.627 | INFO | TraceHistogramAccumulator: BinSizeCoeff = 0.2. | 20260505 03:30:32.627 | INFO | TraceHistogramAccumulator: BaselineStatMinFrameCount = 50. | 20260505 03:30:32.627 | INFO | TraceHistogramAccumulator: FallBackBaselineSigma = 10. | 20260505 03:30:32.627 | INFO | DetectionModelEstimator: NumFramesPreAccumStats = 1000. | 20260505 03:30:32.627 | INFO | DetectionModelEstimator: MinFramesForEstimate= 4000. | 20260505 03:30:32.627 | INFO | Pulse signal variance shot noise coefficient = 1.77377. | 20260505 03:30:33.180 | INFO | OOM score went from 666 to 667 at BasecallerBody checkpoint 3 | 20260505 03:30:33.180 | INFO | memory usage change: vm:68517105 RSS:339032 shared:232959 text:4978 data:125937 at BasecallerBody checkpoint 3 | 20260505 03:30:50.461 | INFO | OOM score went from 667 to 684 at SmrtBasecaller::RunAnalyzer checkpoint 2 | 20260505 03:30:50.461 | INFO | memory usage change: vm:75587989 RSS:6877454 shared:6770180 text:4978 data:127125 at SmrtBasecaller::RunAnalyzer checkpoint 2 | 20260505 03:30:50.462 | INFO | CreateRealTimeMetrics: 9 regions | 20260505 03:30:50.462 | INFO | SmrtBasecaller::CreateRealTimeMetrics line 1109 region.roi:0,0,4096,6144 | 20260505 03:30:50.496 | INFO | Added 196608 lanes to the ROI | 20260505 03:30:50.556 | INFO | Added 196608 lanes to the ROI | 20260505 03:30:50.685 | INFO | memory usage change: vm:75593763 RSS:6883196 shared:6770208 text:4978 data:132899 at SmrtBasecaller::CreateRealTimeMetrics checkpoint 2 | 20260505 03:30:50.727 | INFO | memory usage change: vm:75618340 RSS:6907814 shared:6770208 text:4978 data:157476 at SmrtBasecaller::CreateRealTimeMetrics checkpoint 4 | 20260505 03:30:50.799 | INFO | memory usage change: vm:75642917 RSS:6932366 shared:6770208 text:4978 data:182053 at SmrtBasecaller::CreateRealTimeMetrics checkpoint 7 | 20260505 03:30:50.809 | INFO | SmrtBasecaller::CreateRealTimeMetrics line 1109 region.roi:200,64,64,6016 | 20260505 03:30:50.809 | INFO | memory usage change: vm:75618340 RSS:6907821 shared:6770208 text:4978 data:157476 at SmrtBasecaller::SelectedBasecallerLanesWithinROI | 20260505 03:30:50.810 | INFO | Added 3008 lanes to the ROI | 20260505 03:30:50.810 | INFO | Added 3008 lanes to the ROI | 20260505 03:30:50.812 | INFO | SmrtBasecaller::CreateRealTimeMetrics line 1109 region.roi:2016,64,64,6016 | 20260505 03:30:50.812 | INFO | Added 3008 lanes to the ROI | 20260505 03:30:50.813 | INFO | Added 3008 lanes to the ROI | 20260505 03:30:50.814 | INFO | SmrtBasecaller::CreateRealTimeMetrics line 1109 region.roi:3832,64,64,6016 | 20260505 03:30:50.815 | INFO | Added 3008 lanes to the ROI | 20260505 03:30:50.815 | INFO | Added 3008 lanes to the ROI | 20260505 03:30:50.817 | INFO | SmrtBasecaller::CreateRealTimeMetrics line 1109 region.roi:0,0,4096,6144 | 20260505 03:30:50.843 | INFO | Added 196608 lanes to the ROI | 20260505 03:30:50.894 | INFO | Added 196608 lanes to the ROI | 20260505 03:30:51.012 | INFO | memory usage change: vm:75619873 RSS:6909338 shared:6770208 text:4978 data:159009 at SmrtBasecaller::CreateRealTimeMetrics checkpoint 2 | 20260505 03:30:51.052 | INFO | memory usage change: vm:75644450 RSS:6933890 shared:6770208 text:4978 data:183586 at SmrtBasecaller::CreateRealTimeMetrics checkpoint 4 | 20260505 03:30:51.125 | INFO | memory usage change: vm:75669027 RSS:6958442 shared:6770208 text:4978 data:208163 at SmrtBasecaller::CreateRealTimeMetrics checkpoint 7 | 20260505 03:30:51.134 | INFO | SmrtBasecaller::CreateRealTimeMetrics line 1109 region.roi:0,0,4096,6144 | 20260505 03:30:51.134 | INFO | memory usage change: vm:75644450 RSS:6933931 shared:6770208 text:4978 data:183586 at SmrtBasecaller::SelectedBasecallerLanesWithinROI | 20260505 03:30:51.159 | INFO | Added 196608 lanes to the ROI | 20260505 03:30:51.205 | INFO | Added 196608 lanes to the ROI | 20260505 03:30:51.313 | INFO | memory usage change: vm:75645218 RSS:6934656 shared:6770208 text:4978 data:184354 at SmrtBasecaller::CreateRealTimeMetrics checkpoint 2 | 20260505 03:30:51.353 | INFO | memory usage change: vm:75669795 RSS:6959274 shared:6770208 text:4978 data:208931 at SmrtBasecaller::CreateRealTimeMetrics checkpoint 4 | 20260505 03:30:51.426 | INFO | memory usage change: vm:75694372 RSS:6983826 shared:6770208 text:4978 data:233508 at SmrtBasecaller::CreateRealTimeMetrics checkpoint 7 | 20260505 03:30:51.435 | INFO | SmrtBasecaller::CreateRealTimeMetrics line 1109 region.roi:0,0,4096,6144 | 20260505 03:30:51.435 | INFO | memory usage change: vm:75669795 RSS:6959276 shared:6770208 text:4978 data:208931 at SmrtBasecaller::SelectedBasecallerLanesWithinROI | 20260505 03:30:51.458 | INFO | Added 196608 lanes to the ROI | 20260505 03:30:51.504 | INFO | Added 196608 lanes to the ROI | 20260505 03:30:51.613 | INFO | memory usage change: vm:75670179 RSS:6959605 shared:6770208 text:4978 data:209315 at SmrtBasecaller::CreateRealTimeMetrics checkpoint 2 | 20260505 03:30:51.654 | INFO | memory usage change: vm:75694756 RSS:6984223 shared:6770208 text:4978 data:233892 at SmrtBasecaller::CreateRealTimeMetrics checkpoint 4 | 20260505 03:30:51.727 | INFO | memory usage change: vm:75719333 RSS:7008775 shared:6770208 text:4978 data:258469 at SmrtBasecaller::CreateRealTimeMetrics checkpoint 7 | 20260505 03:30:51.736 | INFO | SmrtBasecaller::CreateRealTimeMetrics line 1109 region.roi:0,0,4096,6144 | 20260505 03:30:51.736 | INFO | memory usage change: vm:75694756 RSS:6984237 shared:6770208 text:4978 data:233892 at SmrtBasecaller::SelectedBasecallerLanesWithinROI | 20260505 03:30:51.760 | INFO | Added 196608 lanes to the ROI | 20260505 03:30:51.802 | INFO | Added 196608 lanes to the ROI | 20260505 03:30:51.902 | INFO | memory usage change: vm:75695140 RSS:6984566 shared:6770208 text:4978 data:234276 at SmrtBasecaller::CreateRealTimeMetrics checkpoint 2 | 20260505 03:30:51.943 | INFO | memory usage change: vm:75719717 RSS:7009184 shared:6770208 text:4978 data:258853 at SmrtBasecaller::CreateRealTimeMetrics checkpoint 4 | 20260505 03:30:52.015 | INFO | memory usage change: vm:75744294 RSS:7033736 shared:6770208 text:4978 data:283430 at SmrtBasecaller::CreateRealTimeMetrics checkpoint 7 | 20260505 03:30:52.025 | INFO | SmrtBasecaller::CreateRealTimeMetrics line 1109 region.roi:0,0,4096,6144 | 20260505 03:30:52.025 | INFO | memory usage change: vm:75719717 RSS:7009198 shared:6770208 text:4978 data:258853 at SmrtBasecaller::SelectedBasecallerLanesWithinROI | 20260505 03:30:52.051 | INFO | Added 196608 lanes to the ROI | 20260505 03:30:52.096 | INFO | Added 196608 lanes to the ROI | 20260505 03:30:52.200 | INFO | memory usage change: vm:75720101 RSS:7009527 shared:6770208 text:4978 data:259237 at SmrtBasecaller::CreateRealTimeMetrics checkpoint 2 | 20260505 03:30:52.240 | INFO | memory usage change: vm:75744678 RSS:7034145 shared:6770208 text:4978 data:283814 at SmrtBasecaller::CreateRealTimeMetrics checkpoint 4 | 20260505 03:30:52.313 | INFO | memory usage change: vm:75769255 RSS:7058697 shared:6770208 text:4978 data:308391 at SmrtBasecaller::CreateRealTimeMetrics checkpoint 7 | 20260505 03:30:52.323 | INFO | SmrtBasecaller::CreateRealTimeMetrics line 1139: properties size:9 | 20260505 03:30:52.323 | INFO | memory usage change: vm:75744678 RSS:7034159 shared:6770208 text:4978 data:283814 at SmrtBasecaller::CreateRealTimeMetrics checkpoint 8 | 20260505 03:30:52.512 | INFO | memory usage change: vm:75799976 RSS:7089400 shared:6770208 text:4978 data:339112 at SmrtBasecaller::CreateRealTimeMetrics checkpoint 9 | 20260505 03:30:53.492 | INFO | memory usage change: vm:75641921 RSS:6931466 shared:6770272 text:4978 data:181057 at SmrtBasecaller::RunAnalyzer checkpoint 4 | 20260505 03:30:56.602 | INFO | OOM score went from 684 to 687 at SmrtBasecaller::RunAnalyzer checkpoint 6 | 20260505 03:30:56.602 | INFO | memory usage change: vm:80072214 RSS:8386054 shared:6770288 text:4978 data:4611350 at SmrtBasecaller::RunAnalyzer checkpoint 6 | 20260505 03:30:56.602 | INFO | AddNode CreateBazSaver started | 20260505 03:30:57.015 | INFO | OOM score went from 687 to 688 at SmrtBasecaller::CreateBazSaver checkpoint 3 | 20260505 03:30:57.015 | INFO | memory usage change: vm:80154303 RSS:8449644 shared:6770288 text:4978 data:4693439 at SmrtBasecaller::CreateBazSaver checkpoint 3 | 20260505 03:30:59.758 | INFO | memory usage change: vm:80350993 RSS:8812879 shared:6770288 text:4978 data:4890129 at SmrtBasecaller::CreateBazSaver checkpoint 4 | 20260505 03:30:59.758 | INFO | Enabling real-time activity labels in BAZ file. | 20260505 03:30:59.758 | INFO | Opening 96 BAZ files for writing with filename prefix: /data/nrta/2/m84031_260504_231011_s1/pb_internal/baz/m84031_260504_231011_s1 zmws: 25165824 | 20260505 03:30:59.758 | INFO | Generating BAZ file list to file: /data/nrta/2/m84031_260504_231011_s1/pb_internal/baz/m84031_260504_231011_s1.bazfilelist.txt | 20260505 03:31:02.945 | INFO | Opened 1 baz files so far | 20260505 03:31:05.073 | INFO | Opened 2 baz files so far | 20260505 03:31:05.109 | INFO | Opened 3 baz files so far | 20260505 03:31:05.159 | INFO | Opened 4 baz files so far | 20260505 03:31:05.354 | INFO | Opened 5 baz files so far | 20260505 03:31:05.392 | INFO | Opened 6 baz files so far | 20260505 03:31:05.404 | INFO | Opened 7 baz files so far | 20260505 03:31:05.405 | INFO | Opened 8 baz files so far | 20260505 03:31:05.408 | INFO | Opened 9 baz files so far | 20260505 03:31:05.415 | INFO | Opened 10 baz files so far | 20260505 03:31:05.416 | INFO | Opened 11 baz files so far | 20260505 03:31:05.417 | INFO | Opened 12 baz files so far | 20260505 03:31:05.462 | INFO | Opened 13 baz files so far | 20260505 03:31:07.502 | INFO | Opened 14 baz files so far | 20260505 03:31:07.569 | INFO | Opened 15 baz files so far | 20260505 03:31:07.619 | INFO | Opened 16 baz files so far | 20260505 03:31:07.801 | INFO | Opened 17 baz files so far | 20260505 03:31:07.914 | INFO | Opened 18 baz files so far | 20260505 03:31:07.922 | INFO | Opened 20 baz files so far | 20260505 03:31:07.925 | INFO | Opened 21 baz files so far | 20260505 03:31:07.925 | INFO | Opened 22 baz files so far | 20260505 03:31:07.919 | INFO | Opened 19 baz files so far | 20260505 03:31:07.926 | INFO | Opened 23 baz files so far | 20260505 03:31:07.927 | INFO | Opened 24 baz files so far | 20260505 03:31:07.928 | INFO | Opened 25 baz files so far | 20260505 03:31:09.852 | INFO | Opened 26 baz files so far | 20260505 03:31:09.915 | INFO | Opened 27 baz files so far | 20260505 03:31:09.982 | INFO | Opened 28 baz files so far | 20260505 03:31:10.148 | INFO | Opened 29 baz files so far | 20260505 03:31:10.315 | INFO | Opened 30 baz files so far | 20260505 03:31:10.398 | INFO | Opened 31 baz files so far | 20260505 03:31:10.403 | INFO | Opened 32 baz files so far | 20260505 03:31:10.409 | INFO | Opened 33 baz files so far | 20260505 03:31:10.409 | INFO | Opened 34 baz files so far | 20260505 03:31:10.413 | INFO | Opened 35 baz files so far | 20260505 03:31:10.415 | INFO | Opened 36 baz files so far | 20260505 03:31:10.422 | INFO | Opened 37 baz files so far | 20260505 03:31:12.205 | INFO | Opened 38 baz files so far | 20260505 03:31:12.284 | INFO | Opened 39 baz files so far | 20260505 03:31:12.352 | INFO | Opened 40 baz files so far | 20260505 03:31:12.649 | INFO | Opened 41 baz files so far | 20260505 03:31:12.663 | INFO | Opened 42 baz files so far | 20260505 03:31:12.714 | INFO | Opened 43 baz files so far | 20260505 03:31:12.871 | INFO | Opened 44 baz files so far | 20260505 03:31:12.875 | INFO | Opened 45 baz files so far | 20260505 03:31:12.879 | INFO | Opened 46 baz files so far | 20260505 03:31:12.884 | INFO | Opened 47 baz files so far | 20260505 03:31:12.884 | INFO | Opened 48 baz files so far | 20260505 03:31:12.888 | INFO | Opened 49 baz files so far | 20260505 03:31:14.567 | INFO | Opened 50 baz files so far | 20260505 03:31:14.639 | INFO | Opened 51 baz files so far | 20260505 03:31:14.704 | INFO | Opened 52 baz files so far | 20260505 03:31:15.053 | INFO | Opened 53 baz files so far | 20260505 03:31:15.124 | INFO | Opened 54 baz files so far | 20260505 03:31:15.134 | INFO | Opened 55 baz files so far | 20260505 03:31:15.292 | INFO | Opened 56 baz files so far | 20260505 03:31:15.316 | INFO | Opened 57 baz files so far | 20260505 03:31:15.325 | INFO | Opened 58 baz files so far | 20260505 03:31:15.326 | INFO | Opened 59 baz files so far | 20260505 03:31:15.329 | INFO | Opened 60 baz files so far | 20260505 03:31:15.340 | INFO | Opened 61 baz files so far | 20260505 03:31:16.907 | INFO | Opened 62 baz files so far | 20260505 03:31:16.953 | INFO | Opened 63 baz files so far | 20260505 03:31:17.084 | INFO | Opened 64 baz files so far | 20260505 03:31:17.350 | INFO | Opened 65 baz files so far | 20260505 03:31:17.660 | INFO | Opened 66 baz files so far | 20260505 03:31:17.668 | INFO | Opened 67 baz files so far | 20260505 03:31:17.714 | INFO | Opened 68 baz files so far | 20260505 03:31:17.762 | INFO | Opened 69 baz files so far | 20260505 03:31:17.762 | INFO | Opened 70 baz files so far | 20260505 03:31:17.771 | INFO | Opened 71 baz files so far | 20260505 03:31:17.774 | INFO | Opened 72 baz files so far | 20260505 03:31:17.786 | INFO | Opened 73 baz files so far | 20260505 03:31:19.251 | INFO | Opened 74 baz files so far | 20260505 03:31:19.276 | INFO | Opened 75 baz files so far | 20260505 03:31:19.455 | INFO | Opened 76 baz files so far | 20260505 03:31:19.647 | INFO | Opened 77 baz files so far | 20260505 03:31:20.035 | INFO | Opened 78 baz files so far | 20260505 03:31:20.056 | INFO | Opened 79 baz files so far | 20260505 03:31:20.091 | INFO | Opened 80 baz files so far | 20260505 03:31:20.201 | INFO | Opened 81 baz files so far | 20260505 03:31:20.202 | INFO | Opened 82 baz files so far | 20260505 03:31:20.202 | INFO | Opened 83 baz files so far | 20260505 03:31:20.203 | INFO | Opened 84 baz files so far | 20260505 03:31:20.218 | INFO | Opened 85 baz files so far | 20260505 03:31:21.610 | INFO | Opened 86 baz files so far | 20260505 03:31:21.637 | INFO | Opened 87 baz files so far | 20260505 03:31:21.829 | INFO | Opened 88 baz files so far | 20260505 03:31:21.869 | INFO | Opened 89 baz files so far | 20260505 03:31:22.530 | INFO | Opened 90 baz files so far | 20260505 03:31:22.534 | INFO | Opened 91 baz files so far | 20260505 03:31:22.554 | INFO | Opened 92 baz files so far | 20260505 03:31:22.643 | INFO | Opened 93 baz files so far | 20260505 03:31:22.649 | INFO | Opened 94 baz files so far | 20260505 03:31:22.652 | INFO | Opened 95 baz files so far | 20260505 03:31:22.670 | INFO | Opened 96 baz files so far | 20260505 03:31:22.679 | INFO | Finished opening a total of 96 baz files | 20260505 03:31:22.679 | INFO | OOM score went from 688 to 692 at BazWriterBody checkpoint 3 | 20260505 03:31:22.679 | INFO | memory usage change: vm:81604256 RSS:10055134 shared:6770400 text:4978 data:6143381 at BazWriterBody checkpoint 3 | 20260505 03:31:23.359 | INFO | AddNode CreateBazSaver done | 20260505 03:31:23.359 | INFO | OOM score went from 692 to 691 at SmrtBasecaller::RunAnalyzer checkpoint 7 | 20260505 03:31:23.359 | INFO | memory usage change: vm:81359194 RSS:9828504 shared:6770400 text:4978 data:5898319 at SmrtBasecaller::RunAnalyzer checkpoint 7 | 20260505 03:31:23.360 | INFO | Added 192 lanes to the ROI | 20260505 03:31:23.360 | INFO | Added 192 lanes to the ROI | 20260505 03:31:23.361 | INFO | Added 20 lanes to the ROI | 20260505 03:31:23.361 | INFO | Added 20 lanes to the ROI | 20260505 03:31:26.921 | INFO | memory usage change: vm:81359194 RSS:9828552 shared:6770416 text:4978 data:5898319 at AddNode CreateSampledBazSaver done | 20260505 03:31:26.958 | INFO | OOM score went from 691 to 690 at SmrtBasecaller::RunAnalyzer checkpoint 9 | 20260505 03:31:26.958 | INFO | memory usage change: vm:81260889 RSS:9730247 shared:6770416 text:4978 data:5800014 at SmrtBasecaller::RunAnalyzer checkpoint 9 | 20260505 03:31:26.959 | INFO | Entered sourceRunning loop, awaiting data from datasource ... | 20260505 03:31:32.959 | WARN | No incoming data for 6.00007 seconds, receive wait 6.00006 seconds, last deliver duration 0 seconds | 20260505 03:31:33.310 | INFO | Base Frame index set to 199244 | 20260505 03:31:33.310 | INFO | Movie started at frame:0, expectedEnd Frame set to:8640000 | 20260505 03:31:33.310 | INFO | Analyzing chunk frames = [0,512) | 20260505 03:31:33.310 | INFO | MemInfo: { | 20260505 03:31:33.310 | INFO | "gpu" : " 11 GB", | 20260505 03:31:33.310 | INFO | "huge" : "", | 20260505 03:31:33.310 | INFO | "huge_shared" : " 25 GB", | 20260505 03:31:33.310 | INFO | "malloc" : " 15 GB", | 20260505 03:31:33.310 | INFO | "pinned" : " 27 GB" | 20260505 03:31:33.310 | INFO | } | 20260505 03:31:33.310 | INFO | Perf report for the last 61.8449 seconds: | 20260505 03:31:33.310 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:31:33.310 | INFO | REPACKER : 0.0%, 0.0, 0.0ms, 0 | 20260505 03:31:33.310 | INFO | PREP_TRACE : 0.0%, 0.0, 0.0ms, 0 | 20260505 03:31:33.310 | INFO | SAVE_TRACE : 0.0%, 0.0, 0.0ms, 0 | 20260505 03:31:33.310 | INFO | ANALYSIS : 0.0%, 0.0, 0.0ms, 0 | 20260505 03:31:33.310 | INFO | PREP_RT_METRICS : 0.0%, 0.0, 0.0ms, 0 | 20260505 03:31:33.310 | INFO | RT_METRICS : 0.0%, 0.0, 0.0ms, 0 | 20260505 03:31:33.310 | INFO | PRE_HQ : 0.0%, 0.0, 0.0ms, 0 | 20260505 03:31:33.310 | INFO | BAZWRITER : 0.0%, 0.0, 0.0ms, 0 | 20260505 03:31:33.310 | INFO | BAZ_SAMPLING : 0.0%, 0.0, 0.0ms, 0 | 20260505 03:31:38.449 | INFO | Analyzing chunk frames = [512,1024) | 20260505 03:31:38.449 | INFO | MemInfo: { | 20260505 03:31:38.449 | INFO | "gpu" : " 13 GB", | 20260505 03:31:38.449 | INFO | "huge" : "", | 20260505 03:31:38.449 | INFO | "huge_shared" : " 27 GB", | 20260505 03:31:38.449 | INFO | "malloc" : " 15 GB", | 20260505 03:31:38.449 | INFO | "pinned" : " 28 GB" | 20260505 03:31:38.449 | INFO | } | 20260505 03:31:38.449 | INFO | Perf report for the last 5.13917 seconds: | 20260505 03:31:38.449 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:31:38.449 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:31:38.449 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:31:38.449 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:31:38.449 | INFO | ANALYSIS : 90.0%, 3.0, 140.2ms, 0 | 20260505 03:31:38.449 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:31:38.449 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:31:38.449 | INFO | PRE_HQ : 5.7%, 1.0, 2.9ms, 0 | 20260505 03:31:38.449 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:31:38.449 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:31:38.450 | INFO | Uploaded 36GiB to GPU in 1.70669s (21.0934GiB/s) | 20260505 03:31:38.450 | INFO | Downloaded 24GiB to GPU in 1.07554s (22.3144GiB/s) | 20260505 03:31:43.588 | INFO | Analyzing chunk frames = [1024,1536) | 20260505 03:31:43.588 | INFO | MemInfo: { | 20260505 03:31:43.588 | INFO | "gpu" : " 13 GB", | 20260505 03:31:43.588 | INFO | "huge" : "", | 20260505 03:31:43.588 | INFO | "huge_shared" : " 27 GB", | 20260505 03:31:43.588 | INFO | "malloc" : " 15 GB", | 20260505 03:31:43.588 | INFO | "pinned" : " 28 GB" | 20260505 03:31:43.588 | INFO | } | 20260505 03:31:43.588 | INFO | Perf report for the last 5.13919 seconds: | 20260505 03:31:43.588 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:31:43.588 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:31:43.588 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:31:43.589 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:31:43.589 | INFO | ANALYSIS : 42.4%, 3.0, 65.9ms, 0 | 20260505 03:31:43.589 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:31:43.589 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:31:43.589 | INFO | PRE_HQ : 5.6%, 1.0, 2.9ms, 0 | 20260505 03:31:43.589 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:31:43.589 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:31:43.589 | INFO | Uploaded 36GiB to GPU in 1.59026s (22.6377GiB/s) | 20260505 03:31:43.589 | INFO | Downloaded 24GiB to GPU in 1.11108s (21.6007GiB/s) | 20260505 03:31:48.728 | INFO | Analyzing chunk frames = [1536,2048) | 20260505 03:31:48.728 | INFO | MemInfo: { | 20260505 03:31:48.728 | INFO | "gpu" : " 13 GB", | 20260505 03:31:48.728 | INFO | "huge" : "", | 20260505 03:31:48.728 | INFO | "huge_shared" : " 27 GB", | 20260505 03:31:48.728 | INFO | "malloc" : " 15 GB", | 20260505 03:31:48.728 | INFO | "pinned" : " 28 GB" | 20260505 03:31:48.728 | INFO | } | 20260505 03:31:48.728 | INFO | Perf report for the last 5.1394 seconds: | 20260505 03:31:48.728 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:31:48.728 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:31:48.728 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:31:48.728 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:31:48.728 | INFO | ANALYSIS : 42.6%, 3.0, 66.3ms, 0 | 20260505 03:31:48.728 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:31:48.728 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:31:48.728 | INFO | PRE_HQ : 5.6%, 1.0, 2.9ms, 0 | 20260505 03:31:48.728 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:31:48.728 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:31:48.728 | INFO | Uploaded 36GiB to GPU in 1.58955s (22.648GiB/s) | 20260505 03:31:48.729 | INFO | Downloaded 24GiB to GPU in 1.11357s (21.5523GiB/s) | 20260505 03:31:53.867 | INFO | Analyzing chunk frames = [2048,2560) | 20260505 03:31:53.867 | INFO | MemInfo: { | 20260505 03:31:53.867 | INFO | "gpu" : " 13 GB", | 20260505 03:31:53.867 | INFO | "huge" : "", | 20260505 03:31:53.867 | INFO | "huge_shared" : " 27 GB", | 20260505 03:31:53.867 | INFO | "malloc" : " 15 GB", | 20260505 03:31:53.867 | INFO | "pinned" : " 28 GB" | 20260505 03:31:53.867 | INFO | } | 20260505 03:31:53.867 | INFO | Perf report for the last 5.13934 seconds: | 20260505 03:31:53.867 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:31:53.867 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:31:53.867 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:31:53.867 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:31:53.867 | INFO | ANALYSIS : 42.5%, 3.0, 66.2ms, 0 | 20260505 03:31:53.867 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:31:53.867 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:31:53.867 | INFO | PRE_HQ : 5.6%, 1.0, 2.9ms, 0 | 20260505 03:31:53.867 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:31:53.867 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:31:53.868 | INFO | Uploaded 36GiB to GPU in 1.58919s (22.653GiB/s) | 20260505 03:31:53.868 | INFO | Downloaded 24GiB to GPU in 1.11776s (21.4714GiB/s) | 20260505 03:31:59.007 | INFO | Analyzing chunk frames = [2560,3072) | 20260505 03:31:59.007 | INFO | MemInfo: { | 20260505 03:31:59.007 | INFO | "gpu" : " 13 GB", | 20260505 03:31:59.007 | INFO | "huge" : "", | 20260505 03:31:59.007 | INFO | "huge_shared" : " 27 GB", | 20260505 03:31:59.007 | INFO | "malloc" : " 15 GB", | 20260505 03:31:59.007 | INFO | "pinned" : " 28 GB" | 20260505 03:31:59.007 | INFO | } | 20260505 03:31:59.007 | INFO | Perf report for the last 5.13956 seconds: | 20260505 03:31:59.007 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:31:59.007 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:31:59.007 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:31:59.007 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:31:59.007 | INFO | ANALYSIS : 42.2%, 3.0, 65.7ms, 0 | 20260505 03:31:59.007 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:31:59.007 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:31:59.007 | INFO | PRE_HQ : 5.6%, 1.0, 2.9ms, 0 | 20260505 03:31:59.007 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:31:59.007 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:31:59.007 | INFO | Uploaded 36GiB to GPU in 1.59217s (22.6106GiB/s) | 20260505 03:31:59.007 | INFO | Downloaded 24GiB to GPU in 1.11901s (21.4475GiB/s) | 20260505 03:32:04.146 | INFO | Analyzing chunk frames = [3072,3584) | 20260505 03:32:04.146 | INFO | MemInfo: { | 20260505 03:32:04.146 | INFO | "gpu" : " 13 GB", | 20260505 03:32:04.146 | INFO | "huge" : "", | 20260505 03:32:04.146 | INFO | "huge_shared" : " 27 GB", | 20260505 03:32:04.146 | INFO | "malloc" : " 15 GB", | 20260505 03:32:04.146 | INFO | "pinned" : " 28 GB" | 20260505 03:32:04.146 | INFO | } | 20260505 03:32:04.146 | INFO | Perf report for the last 5.13899 seconds: | 20260505 03:32:04.146 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:32:04.146 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:04.146 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:04.146 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:04.146 | INFO | ANALYSIS : 42.5%, 3.0, 66.2ms, 0 | 20260505 03:32:04.146 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:04.146 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:04.146 | INFO | PRE_HQ : 5.6%, 1.0, 2.9ms, 0 | 20260505 03:32:04.146 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:04.146 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:04.146 | INFO | Uploaded 36GiB to GPU in 1.58951s (22.6485GiB/s) | 20260505 03:32:04.146 | INFO | Downloaded 24GiB to GPU in 1.11711s (21.484GiB/s) | 20260505 03:32:09.285 | INFO | Analyzing chunk frames = [3584,4096) | 20260505 03:32:09.285 | INFO | MemInfo: { | 20260505 03:32:09.285 | INFO | "gpu" : " 13 GB", | 20260505 03:32:09.285 | INFO | "huge" : "", | 20260505 03:32:09.285 | INFO | "huge_shared" : " 27 GB", | 20260505 03:32:09.285 | INFO | "malloc" : " 15 GB", | 20260505 03:32:09.285 | INFO | "pinned" : " 28 GB" | 20260505 03:32:09.285 | INFO | } | 20260505 03:32:09.285 | INFO | Perf report for the last 5.13937 seconds: | 20260505 03:32:09.285 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:32:09.285 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:09.285 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:09.285 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:09.285 | INFO | ANALYSIS : 42.3%, 3.0, 65.9ms, 0 | 20260505 03:32:09.285 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:09.285 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:09.285 | INFO | PRE_HQ : 5.6%, 1.0, 2.9ms, 0 | 20260505 03:32:09.285 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:09.285 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:09.285 | INFO | Uploaded 36GiB to GPU in 1.59105s (22.6266GiB/s) | 20260505 03:32:09.285 | INFO | Downloaded 24GiB to GPU in 1.11776s (21.4716GiB/s) | 20260505 03:32:09.355 | INFO | Starting a round of Baz IO | 20260505 03:32:11.487 | INFO | Finished a round of Baz IO: Wrote 3.469 KiB in 2.131 seconds | 20260505 03:32:11.487 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 100.00% / 0.00% / 0.00% | 20260505 03:32:11.487 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0% / 0% / 0% | 20260505 03:32:11.487 | INFO | Baz IO overall: | 20260505 03:32:11.487 | INFO | BytesWritten : 2571416390 | 20260505 03:32:11.487 | INFO | NumEvents : 0 | 20260505 03:32:14.425 | INFO | Analyzing chunk frames = [4096,4608) | 20260505 03:32:14.425 | INFO | MemInfo: { | 20260505 03:32:14.425 | INFO | "gpu" : " 13 GB", | 20260505 03:32:14.425 | INFO | "huge" : "", | 20260505 03:32:14.425 | INFO | "huge_shared" : " 27 GB", | 20260505 03:32:14.425 | INFO | "malloc" : " 15 GB", | 20260505 03:32:14.426 | INFO | "pinned" : " 28 GB" | 20260505 03:32:14.426 | INFO | } | 20260505 03:32:14.426 | INFO | Perf report for the last 5.14106 seconds: | 20260505 03:32:14.426 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:32:14.426 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:14.426 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:14.426 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:14.426 | INFO | ANALYSIS : 42.2%, 3.0, 65.7ms, 0 | 20260505 03:32:14.426 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:14.426 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:14.426 | INFO | PRE_HQ : 39.7%, 1.1, 23.7ms, 0 | 20260505 03:32:14.426 | INFO | BAZWRITER : 10.2%, 1.0, 5.3ms, 0 | 20260505 03:32:14.426 | INFO | BAZ_SAMPLING : 0.1%, 1.0, 0.0ms, 0 | 20260505 03:32:14.426 | INFO | Uploaded 36GiB to GPU in 1.59813s (22.5264GiB/s) | 20260505 03:32:14.426 | INFO | Downloaded 24GiB to GPU in 1.12542s (21.3253GiB/s) | 20260505 03:32:19.564 | INFO | Analyzing chunk frames = [4608,5120) | 20260505 03:32:19.564 | INFO | MemInfo: { | 20260505 03:32:19.564 | INFO | "gpu" : " 13 GB", | 20260505 03:32:19.564 | INFO | "huge" : "", | 20260505 03:32:19.564 | INFO | "huge_shared" : " 27 GB", | 20260505 03:32:19.564 | INFO | "malloc" : " 15 GB", | 20260505 03:32:19.564 | INFO | "pinned" : " 28 GB" | 20260505 03:32:19.564 | INFO | } | 20260505 03:32:19.564 | INFO | Perf report for the last 5.13801 seconds: | 20260505 03:32:19.564 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:32:19.564 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:19.564 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:19.564 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:19.564 | INFO | ANALYSIS : 42.4%, 3.0, 66.0ms, 0 | 20260505 03:32:19.564 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:19.564 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:19.564 | INFO | PRE_HQ : 5.6%, 1.0, 2.9ms, 0 | 20260505 03:32:19.564 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:19.564 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:19.564 | INFO | Uploaded 36GiB to GPU in 1.59156s (22.6193GiB/s) | 20260505 03:32:19.564 | INFO | Downloaded 24GiB to GPU in 1.11967s (21.4349GiB/s) | 20260505 03:32:24.703 | INFO | Analyzing chunk frames = [5120,5632) | 20260505 03:32:24.703 | INFO | MemInfo: { | 20260505 03:32:24.703 | INFO | "gpu" : " 13 GB", | 20260505 03:32:24.703 | INFO | "huge" : "", | 20260505 03:32:24.703 | INFO | "huge_shared" : " 27 GB", | 20260505 03:32:24.703 | INFO | "malloc" : " 15 GB", | 20260505 03:32:24.703 | INFO | "pinned" : " 28 GB" | 20260505 03:32:24.703 | INFO | } | 20260505 03:32:24.703 | INFO | Perf report for the last 5.1392 seconds: | 20260505 03:32:24.703 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:32:24.703 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:24.703 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:24.703 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:24.703 | INFO | ANALYSIS : 42.5%, 3.0, 66.2ms, 0 | 20260505 03:32:24.703 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:24.703 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:24.703 | INFO | PRE_HQ : 5.6%, 1.0, 2.9ms, 0 | 20260505 03:32:24.703 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:24.703 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:24.704 | INFO | Uploaded 36GiB to GPU in 1.59016s (22.6393GiB/s) | 20260505 03:32:24.704 | INFO | Downloaded 24GiB to GPU in 1.11832s (21.4607GiB/s) | 20260505 03:32:29.843 | INFO | Analyzing chunk frames = [5632,6144) | 20260505 03:32:29.843 | INFO | MemInfo: { | 20260505 03:32:29.843 | INFO | "gpu" : " 13 GB", | 20260505 03:32:29.843 | INFO | "huge" : "", | 20260505 03:32:29.843 | INFO | "huge_shared" : " 27 GB", | 20260505 03:32:29.843 | INFO | "malloc" : " 15 GB", | 20260505 03:32:29.843 | INFO | "pinned" : " 28 GB" | 20260505 03:32:29.843 | INFO | } | 20260505 03:32:29.843 | INFO | Perf report for the last 5.13942 seconds: | 20260505 03:32:29.843 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:32:29.843 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:29.843 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:29.843 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:29.843 | INFO | ANALYSIS : 42.5%, 3.0, 66.1ms, 0 | 20260505 03:32:29.843 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:29.843 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:29.843 | INFO | PRE_HQ : 5.7%, 1.0, 2.9ms, 0 | 20260505 03:32:29.843 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:29.843 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:29.843 | INFO | Uploaded 36GiB to GPU in 1.58989s (22.643GiB/s) | 20260505 03:32:29.843 | INFO | Downloaded 24GiB to GPU in 1.11529s (21.5191GiB/s) | 20260505 03:32:34.982 | INFO | Analyzing chunk frames = [6144,6656) | 20260505 03:32:34.982 | INFO | MemInfo: { | 20260505 03:32:34.982 | INFO | "gpu" : " 13 GB", | 20260505 03:32:34.982 | INFO | "huge" : "", | 20260505 03:32:34.982 | INFO | "huge_shared" : " 27 GB", | 20260505 03:32:34.982 | INFO | "malloc" : " 15 GB", | 20260505 03:32:34.982 | INFO | "pinned" : " 28 GB" | 20260505 03:32:34.982 | INFO | } | 20260505 03:32:34.982 | INFO | Perf report for the last 5.13925 seconds: | 20260505 03:32:34.982 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:32:34.982 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:34.982 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:34.982 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:34.982 | INFO | ANALYSIS : 42.5%, 3.0, 66.1ms, 0 | 20260505 03:32:34.982 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:34.982 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:34.982 | INFO | PRE_HQ : 5.7%, 1.0, 2.9ms, 0 | 20260505 03:32:34.982 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:34.982 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:34.983 | INFO | Uploaded 36GiB to GPU in 1.59057s (22.6335GiB/s) | 20260505 03:32:34.983 | INFO | Downloaded 24GiB to GPU in 1.11591s (21.5071GiB/s) | 20260505 03:32:40.121 | INFO | Analyzing chunk frames = [6656,7168) | 20260505 03:32:40.121 | INFO | MemInfo: { | 20260505 03:32:40.121 | INFO | "gpu" : " 13 GB", | 20260505 03:32:40.121 | INFO | "huge" : "", | 20260505 03:32:40.121 | INFO | "huge_shared" : " 27 GB", | 20260505 03:32:40.121 | INFO | "malloc" : " 15 GB", | 20260505 03:32:40.121 | INFO | "pinned" : " 28 GB" | 20260505 03:32:40.121 | INFO | } | 20260505 03:32:40.121 | INFO | Perf report for the last 5.1393 seconds: | 20260505 03:32:40.121 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:32:40.121 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:40.121 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:40.121 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:40.121 | INFO | ANALYSIS : 42.5%, 3.0, 66.2ms, 0 | 20260505 03:32:40.121 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:40.121 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:40.121 | INFO | PRE_HQ : 5.6%, 1.0, 2.9ms, 0 | 20260505 03:32:40.122 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:40.122 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:40.122 | INFO | Uploaded 36GiB to GPU in 1.59055s (22.6337GiB/s) | 20260505 03:32:40.122 | INFO | Downloaded 24GiB to GPU in 1.1169s (21.488GiB/s) | 20260505 03:32:45.260 | INFO | Analyzing chunk frames = [7168,7680) | 20260505 03:32:45.260 | INFO | MemInfo: { | 20260505 03:32:45.260 | INFO | "gpu" : " 13 GB", | 20260505 03:32:45.260 | INFO | "huge" : "", | 20260505 03:32:45.261 | INFO | "huge_shared" : " 27 GB", | 20260505 03:32:45.261 | INFO | "malloc" : " 15 GB", | 20260505 03:32:45.261 | INFO | "pinned" : " 28 GB" | 20260505 03:32:45.261 | INFO | } | 20260505 03:32:45.261 | INFO | Perf report for the last 5.13934 seconds: | 20260505 03:32:45.261 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:32:45.261 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:45.261 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:45.261 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:45.261 | INFO | ANALYSIS : 42.5%, 3.0, 66.1ms, 0 | 20260505 03:32:45.261 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:45.261 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:45.261 | INFO | PRE_HQ : 5.7%, 1.0, 3.0ms, 0 | 20260505 03:32:45.261 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:45.261 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:45.261 | INFO | Uploaded 36GiB to GPU in 1.59068s (22.6318GiB/s) | 20260505 03:32:45.261 | INFO | Downloaded 24GiB to GPU in 1.11764s (21.4739GiB/s) | 20260505 03:32:50.400 | INFO | Analyzing chunk frames = [7680,8192) | 20260505 03:32:50.400 | INFO | MemInfo: { | 20260505 03:32:50.400 | INFO | "gpu" : " 13 GB", | 20260505 03:32:50.400 | INFO | "huge" : "", | 20260505 03:32:50.400 | INFO | "huge_shared" : " 27 GB", | 20260505 03:32:50.400 | INFO | "malloc" : " 15 GB", | 20260505 03:32:50.400 | INFO | "pinned" : " 28 GB" | 20260505 03:32:50.400 | INFO | } | 20260505 03:32:50.400 | INFO | Perf report for the last 5.13915 seconds: | 20260505 03:32:50.400 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:32:50.400 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:50.400 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:50.400 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:50.400 | INFO | ANALYSIS : 42.6%, 3.0, 66.4ms, 0 | 20260505 03:32:50.400 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:50.400 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:50.400 | INFO | PRE_HQ : 5.6%, 1.0, 2.9ms, 0 | 20260505 03:32:50.400 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:50.400 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:50.400 | INFO | Uploaded 36GiB to GPU in 1.58975s (22.645GiB/s) | 20260505 03:32:50.400 | INFO | Downloaded 24GiB to GPU in 1.11849s (21.4575GiB/s) | 20260505 03:32:50.400 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 03:32:50.400 | INFO | Total mem usage: 14.6 GiB | 20260505 03:32:50.400 | INFO | All Time High Water Marks sum: 14.6 GiB (Savings: 0.0527%) | 20260505 03:32:50.400 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: 0.0527%) | 20260505 03:32:50.400 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 03:32:50.400 | INFO | High Water Breakdown: | 20260505 03:32:50.400 | INFO | 54.6% : 7.96 GiB ( 100% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 03:32:50.400 | INFO | 45.3% : 6.6 GiB ( 100% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 03:32:50.400 | INFO | 0.128% : 19.1 MiB ( 100% Of Peak) : SampledBazData | 20260505 03:32:50.400 | INFO | ------------------------------------------ | 20260505 03:32:50.400 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:32:50.401 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 03:32:50.401 | INFO | Total mem usage: 26.5 GiB | 20260505 03:32:50.401 | INFO | All Time High Water Marks sum: 26.5 GiB (Savings: 0%) | 20260505 03:32:50.401 | INFO | Recent High Water Marks Sum: 26.5 GiB (Savings: 0%) | 20260505 03:32:50.401 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 03:32:50.401 | INFO | High Water Breakdown: | 20260505 03:32:50.401 | INFO | 54.1% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 03:32:50.401 | INFO | 35% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 03:32:50.401 | INFO | 3.54% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 03:32:50.401 | INFO | 3.01% : 816 MiB ( 100% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 03:32:50.401 | INFO | 1.9% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 03:32:50.401 | INFO | 1.41% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 03:32:50.401 | INFO | 0.987% : 268 MiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 03:32:50.401 | INFO | 0.0147% : 4 MiB ( 100% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 03:32:50.401 | INFO | 0.0147% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 03:32:50.401 | INFO | 0.00147% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 03:32:50.401 | INFO | 9.03e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 03:32:50.401 | INFO | ------------------------------------------ | 20260505 03:32:50.401 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:32:50.401 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 03:32:50.401 | INFO | Total mem usage: 25.6 GiB | 20260505 03:32:50.401 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 03:32:50.401 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: 0%) | 20260505 03:32:50.401 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 03:32:50.401 | INFO | High Water Breakdown: | 20260505 03:32:50.401 | INFO | 100% : 25.6 GiB ( 100% Of Peak) : WXIPCDataSourceConfig | 20260505 03:32:50.401 | INFO | ------------------------------------------ | 20260505 03:32:50.401 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:32:50.401 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 03:32:50.401 | INFO | Total mem usage: 12.5 GiB | 20260505 03:32:50.401 | INFO | All Time High Water Marks sum: 12.5 GiB (Savings: 0%) | 20260505 03:32:50.401 | INFO | Recent High Water Marks Sum: 12.5 GiB (Savings: 0%) | 20260505 03:32:50.401 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 03:32:50.401 | INFO | High Water Breakdown: | 20260505 03:32:50.401 | INFO | 43.3% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 03:32:50.401 | INFO | 17.7% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 03:32:50.401 | INFO | 11.3% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 03:32:50.401 | INFO | 9.03% : 1.12 GiB ( 100% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 03:32:50.401 | INFO | 4.51% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 03:32:50.401 | INFO | 3.6% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 03:32:50.401 | INFO | 3.01% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 03:32:50.401 | INFO | 2.33% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 03:32:50.401 | INFO | 2.19% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 03:32:50.401 | INFO | 1% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 03:32:50.401 | INFO | 1% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 03:32:50.401 | INFO | 0.752% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 03:32:50.401 | INFO | 0.235% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 03:32:50.401 | INFO | 0.0941% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 03:32:50.401 | INFO | ------------------------------------------ | 20260505 03:32:50.401 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:32:50.401 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 03:32:50.401 | INFO | PerfReport: Gathered 1440 samples. | 20260505 03:32:50.401 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 03:32:50.401 | INFO | PerfReport: Full Execution -- 7.25e+01 / -nan, 2.86e+01 / 0.00e+00 | 20260505 03:32:50.401 | INFO | PerfReport: 22.97%, Upload -- 1.67e+01 / 0.00e+00, 4.34e+00 / 0.00e+00 | 20260505 03:32:50.401 | INFO | PerfReport: 16.02%, Download -- 1.16e+01 / 0.00e+00, 1.85e-01 / 0.00e+00 | 20260505 03:32:50.401 | INFO | PerfReport: 33.28%, Compute -- 2.41e+01 / 0.00e+00, 1.45e+01 / 0.00e+00 | 20260505 03:32:50.401 | INFO | PerfReport: 0.71%, LockUpload -- 5.17e-01 / 0.00e+00, 3.76e+00 / 0.00e+00 | 20260505 03:32:50.401 | INFO | PerfReport: 0.00%, LockDownload -- 4.91e-04 / 0.00e+00, 1.59e-04 / 0.00e+00 | 20260505 03:32:50.401 | INFO | PerfReport: 27.00%, LockCompute -- 1.96e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260505 03:32:50.401 | INFO | PerfReport: 0.01%, Misc -- 9.33e-03 / 0.00e+00, 7.59e-03 / 0.00e+00 | 20260505 03:32:50.478 | INFO | Starting a round of Baz IO | 20260505 03:32:52.610 | INFO | Finished a round of Baz IO: Wrote 3.469 KiB in 2.132 seconds | 20260505 03:32:52.610 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 100.00% / 0.00% / 0.00% | 20260505 03:32:52.610 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0% / 0% / 0% | 20260505 03:32:52.610 | INFO | Baz IO overall: | 20260505 03:32:52.610 | INFO | BytesWritten : 2571419942 | 20260505 03:32:52.610 | INFO | NumEvents : 0 | 20260505 03:32:55.539 | INFO | Analyzing chunk frames = [8192,8704) | 20260505 03:32:55.539 | INFO | MemInfo: { | 20260505 03:32:55.539 | INFO | "gpu" : " 13 GB", | 20260505 03:32:55.539 | INFO | "huge" : "", | 20260505 03:32:55.539 | INFO | "huge_shared" : " 27 GB", | 20260505 03:32:55.539 | INFO | "malloc" : " 15 GB", | 20260505 03:32:55.539 | INFO | "pinned" : " 28 GB" | 20260505 03:32:55.539 | INFO | } | 20260505 03:32:55.539 | INFO | Perf report for the last 5.13959 seconds: | 20260505 03:32:55.539 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:32:55.539 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:55.539 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:55.540 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:55.540 | INFO | ANALYSIS : 42.4%, 3.0, 65.9ms, 0 | 20260505 03:32:55.540 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:55.540 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:32:55.540 | INFO | PRE_HQ : 39.0%, 1.1, 21.9ms, 0 | 20260505 03:32:55.540 | INFO | BAZWRITER : 9.7%, 1.0, 5.0ms, 0 | 20260505 03:32:55.540 | INFO | BAZ_SAMPLING : 0.1%, 1.0, 0.0ms, 0 | 20260505 03:32:55.540 | INFO | Uploaded 36GiB to GPU in 1.59785s (22.5302GiB/s) | 20260505 03:32:55.540 | INFO | Downloaded 24GiB to GPU in 1.12557s (21.3225GiB/s) | 20260505 03:33:00.678 | INFO | Analyzing chunk frames = [8704,9216) | 20260505 03:33:00.678 | INFO | MemInfo: { | 20260505 03:33:00.678 | INFO | "gpu" : " 13 GB", | 20260505 03:33:00.678 | INFO | "huge" : "", | 20260505 03:33:00.679 | INFO | "huge_shared" : " 27 GB", | 20260505 03:33:00.679 | INFO | "malloc" : " 15 GB", | 20260505 03:33:00.679 | INFO | "pinned" : " 28 GB" | 20260505 03:33:00.679 | INFO | } | 20260505 03:33:00.679 | INFO | Perf report for the last 5.13919 seconds: | 20260505 03:33:00.679 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:33:00.679 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:00.679 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:00.679 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:00.679 | INFO | ANALYSIS : 42.5%, 3.0, 66.1ms, 0 | 20260505 03:33:00.679 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:00.679 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:00.679 | INFO | PRE_HQ : 5.7%, 1.0, 2.9ms, 0 | 20260505 03:33:00.679 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:00.679 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:00.679 | INFO | Uploaded 36GiB to GPU in 1.59035s (22.6365GiB/s) | 20260505 03:33:00.679 | INFO | Downloaded 24GiB to GPU in 1.11548s (21.5153GiB/s) | 20260505 03:33:05.818 | INFO | Analyzing chunk frames = [9216,9728) | 20260505 03:33:05.818 | INFO | MemInfo: { | 20260505 03:33:05.818 | INFO | "gpu" : " 13 GB", | 20260505 03:33:05.818 | INFO | "huge" : "", | 20260505 03:33:05.818 | INFO | "huge_shared" : " 27 GB", | 20260505 03:33:05.818 | INFO | "malloc" : " 15 GB", | 20260505 03:33:05.818 | INFO | "pinned" : " 28 GB" | 20260505 03:33:05.818 | INFO | } | 20260505 03:33:05.818 | INFO | Perf report for the last 5.13926 seconds: | 20260505 03:33:05.818 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:33:05.818 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:05.818 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:05.818 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:05.818 | INFO | ANALYSIS : 42.4%, 3.0, 66.0ms, 0 | 20260505 03:33:05.818 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:05.818 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:05.818 | INFO | PRE_HQ : 5.6%, 1.0, 2.9ms, 0 | 20260505 03:33:05.818 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:05.818 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:05.819 | INFO | Uploaded 36GiB to GPU in 1.59103s (22.6268GiB/s) | 20260505 03:33:05.819 | INFO | Downloaded 24GiB to GPU in 1.11663s (21.4933GiB/s) | 20260505 03:33:10.957 | INFO | Analyzing chunk frames = [9728,10240) | 20260505 03:33:10.957 | INFO | MemInfo: { | 20260505 03:33:10.957 | INFO | "gpu" : " 13 GB", | 20260505 03:33:10.957 | INFO | "huge" : "", | 20260505 03:33:10.957 | INFO | "huge_shared" : " 27 GB", | 20260505 03:33:10.957 | INFO | "malloc" : " 15 GB", | 20260505 03:33:10.957 | INFO | "pinned" : " 28 GB" | 20260505 03:33:10.957 | INFO | } | 20260505 03:33:10.957 | INFO | Perf report for the last 5.13937 seconds: | 20260505 03:33:10.957 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:33:10.957 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:10.957 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:10.957 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:10.957 | INFO | ANALYSIS : 42.6%, 3.0, 66.4ms, 0 | 20260505 03:33:10.957 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:10.957 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:10.957 | INFO | PRE_HQ : 5.6%, 1.0, 2.9ms, 0 | 20260505 03:33:10.957 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:10.957 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:10.958 | INFO | Uploaded 36GiB to GPU in 1.58999s (22.6417GiB/s) | 20260505 03:33:10.958 | INFO | Downloaded 24GiB to GPU in 1.11354s (21.5529GiB/s) | 20260505 03:33:16.097 | INFO | Analyzing chunk frames = [10240,10752) | 20260505 03:33:16.097 | INFO | MemInfo: { | 20260505 03:33:16.097 | INFO | "gpu" : " 13 GB", | 20260505 03:33:16.097 | INFO | "huge" : "", | 20260505 03:33:16.097 | INFO | "huge_shared" : " 27 GB", | 20260505 03:33:16.097 | INFO | "malloc" : " 15 GB", | 20260505 03:33:16.097 | INFO | "pinned" : " 28 GB" | 20260505 03:33:16.097 | INFO | } | 20260505 03:33:16.097 | INFO | Perf report for the last 5.13945 seconds: | 20260505 03:33:16.097 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:33:16.097 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:16.097 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:16.097 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:16.097 | INFO | ANALYSIS : 42.8%, 3.0, 66.7ms, 0 | 20260505 03:33:16.097 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:16.097 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:16.097 | INFO | PRE_HQ : 5.7%, 1.0, 3.0ms, 0 | 20260505 03:33:16.097 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:16.097 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:16.097 | INFO | Uploaded 36GiB to GPU in 1.58797s (22.6704GiB/s) | 20260505 03:33:16.097 | INFO | Downloaded 24GiB to GPU in 1.11514s (21.5219GiB/s) | 20260505 03:33:21.236 | INFO | Analyzing chunk frames = [10752,11264) | 20260505 03:33:21.236 | INFO | MemInfo: { | 20260505 03:33:21.236 | INFO | "gpu" : " 13 GB", | 20260505 03:33:21.236 | INFO | "huge" : "", | 20260505 03:33:21.236 | INFO | "huge_shared" : " 27 GB", | 20260505 03:33:21.236 | INFO | "malloc" : " 15 GB", | 20260505 03:33:21.236 | INFO | "pinned" : " 28 GB" | 20260505 03:33:21.236 | INFO | } | 20260505 03:33:21.236 | INFO | Perf report for the last 5.13946 seconds: | 20260505 03:33:21.236 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:33:21.236 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:21.236 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:21.236 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:21.236 | INFO | ANALYSIS : 43.3%, 3.0, 67.4ms, 0 | 20260505 03:33:21.236 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:21.236 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:21.236 | INFO | PRE_HQ : 5.7%, 1.0, 2.9ms, 0 | 20260505 03:33:21.236 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:21.236 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:21.237 | INFO | Uploaded 36GiB to GPU in 1.58447s (22.7205GiB/s) | 20260505 03:33:21.237 | INFO | Downloaded 24GiB to GPU in 1.10949s (21.6317GiB/s) | 20260505 03:33:26.375 | INFO | Analyzing chunk frames = [11264,11776) | 20260505 03:33:26.375 | INFO | MemInfo: { | 20260505 03:33:26.375 | INFO | "gpu" : " 13 GB", | 20260505 03:33:26.375 | INFO | "huge" : "", | 20260505 03:33:26.375 | INFO | "huge_shared" : " 27 GB", | 20260505 03:33:26.375 | INFO | "malloc" : " 15 GB", | 20260505 03:33:26.376 | INFO | "pinned" : " 28 GB" | 20260505 03:33:26.376 | INFO | } | 20260505 03:33:26.376 | INFO | Perf report for the last 5.13929 seconds: | 20260505 03:33:26.376 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:33:26.376 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:26.376 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:26.376 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:26.376 | INFO | ANALYSIS : 43.4%, 3.0, 67.5ms, 0 | 20260505 03:33:26.376 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:26.376 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:26.376 | INFO | PRE_HQ : 5.6%, 1.0, 2.9ms, 0 | 20260505 03:33:26.376 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:26.376 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:26.376 | INFO | Uploaded 36GiB to GPU in 1.58389s (22.7289GiB/s) | 20260505 03:33:26.376 | INFO | Downloaded 24GiB to GPU in 1.10731s (21.6742GiB/s) | 20260505 03:33:31.515 | INFO | Analyzing chunk frames = [11776,12288) | 20260505 03:33:31.515 | INFO | MemInfo: { | 20260505 03:33:31.515 | INFO | "gpu" : " 13 GB", | 20260505 03:33:31.515 | INFO | "huge" : "", | 20260505 03:33:31.515 | INFO | "huge_shared" : " 27 GB", | 20260505 03:33:31.515 | INFO | "malloc" : " 15 GB", | 20260505 03:33:31.515 | INFO | "pinned" : " 28 GB" | 20260505 03:33:31.515 | INFO | } | 20260505 03:33:31.515 | INFO | Perf report for the last 5.13923 seconds: | 20260505 03:33:31.515 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:33:31.515 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:31.515 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:31.515 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:31.515 | INFO | ANALYSIS : 43.7%, 3.0, 68.1ms, 0 | 20260505 03:33:31.515 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:31.515 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:31.515 | INFO | PRE_HQ : 5.6%, 1.0, 2.9ms, 0 | 20260505 03:33:31.515 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:31.515 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:31.515 | INFO | Uploaded 36GiB to GPU in 1.58107s (22.7695GiB/s) | 20260505 03:33:31.515 | INFO | Downloaded 24GiB to GPU in 1.10541s (21.7113GiB/s) | 20260505 03:33:31.588 | INFO | Starting a round of Baz IO | 20260505 03:33:33.783 | INFO | Finished a round of Baz IO: Wrote 3.469 KiB in 2.195 seconds | 20260505 03:33:33.783 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 100.00% / 0.00% / 0.00% | 20260505 03:33:33.783 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0% / 0% / 0% | 20260505 03:33:33.783 | INFO | Baz IO overall: | 20260505 03:33:33.783 | INFO | BytesWritten : 2571423494 | 20260505 03:33:33.783 | INFO | NumEvents : 0 | 20260505 03:33:36.654 | INFO | Analyzing chunk frames = [12288,12800) | 20260505 03:33:36.654 | INFO | MemInfo: { | 20260505 03:33:36.654 | INFO | "gpu" : " 14 GB", | 20260505 03:33:36.654 | INFO | "huge" : "", | 20260505 03:33:36.654 | INFO | "huge_shared" : " 27 GB", | 20260505 03:33:36.654 | INFO | "malloc" : " 15 GB", | 20260505 03:33:36.654 | INFO | "pinned" : " 28 GB" | 20260505 03:33:36.654 | INFO | } | 20260505 03:33:36.654 | INFO | Perf report for the last 5.1394 seconds: | 20260505 03:33:36.654 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:33:36.654 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:36.654 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:36.654 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:36.654 | INFO | ANALYSIS : 43.7%, 3.0, 68.0ms, 0 | 20260505 03:33:36.654 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:36.654 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:36.654 | INFO | PRE_HQ : 39.4%, 1.0, 21.3ms, 0 | 20260505 03:33:36.654 | INFO | BAZWRITER : 9.9%, 1.0, 5.1ms, 0 | 20260505 03:33:36.654 | INFO | BAZ_SAMPLING : 0.1%, 1.0, 0.0ms, 0 | 20260505 03:33:36.654 | INFO | Uploaded 36GiB to GPU in 1.58691s (22.6856GiB/s) | 20260505 03:33:36.655 | INFO | Downloaded 24GiB to GPU in 1.11216s (21.5796GiB/s) | 20260505 03:33:41.793 | INFO | Analyzing chunk frames = [12800,13312) | 20260505 03:33:41.793 | INFO | MemInfo: { | 20260505 03:33:41.793 | INFO | "gpu" : " 14 GB", | 20260505 03:33:41.793 | INFO | "huge" : "", | 20260505 03:33:41.793 | INFO | "huge_shared" : " 27 GB", | 20260505 03:33:41.793 | INFO | "malloc" : " 15 GB", | 20260505 03:33:41.793 | INFO | "pinned" : " 28 GB" | 20260505 03:33:41.793 | INFO | } | 20260505 03:33:41.794 | INFO | Perf report for the last 5.13929 seconds: | 20260505 03:33:41.794 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:33:41.794 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:41.794 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:41.794 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:41.794 | INFO | ANALYSIS : 44.3%, 3.0, 69.0ms, 0 | 20260505 03:33:41.794 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:41.794 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:41.794 | INFO | PRE_HQ : 5.6%, 1.0, 2.9ms, 0 | 20260505 03:33:41.794 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:41.794 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:41.794 | INFO | Uploaded 36GiB to GPU in 1.57647s (22.8358GiB/s) | 20260505 03:33:41.794 | INFO | Downloaded 24GiB to GPU in 1.10352s (21.7486GiB/s) | 20260505 03:33:46.933 | INFO | Analyzing chunk frames = [13312,13824) | 20260505 03:33:46.933 | INFO | MemInfo: { | 20260505 03:33:46.933 | INFO | "gpu" : " 14 GB", | 20260505 03:33:46.933 | INFO | "huge" : "", | 20260505 03:33:46.933 | INFO | "huge_shared" : " 27 GB", | 20260505 03:33:46.933 | INFO | "malloc" : " 15 GB", | 20260505 03:33:46.933 | INFO | "pinned" : " 28 GB" | 20260505 03:33:46.933 | INFO | } | 20260505 03:33:46.933 | INFO | Perf report for the last 5.13923 seconds: | 20260505 03:33:46.933 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:33:46.933 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:46.933 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:46.933 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:46.933 | INFO | ANALYSIS : 44.5%, 3.0, 69.2ms, 0 | 20260505 03:33:46.933 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:46.933 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:46.933 | INFO | PRE_HQ : 5.6%, 1.0, 2.9ms, 0 | 20260505 03:33:46.933 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:46.933 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:46.933 | INFO | Uploaded 36GiB to GPU in 1.57494s (22.8581GiB/s) | 20260505 03:33:46.933 | INFO | Downloaded 24GiB to GPU in 1.09913s (21.8355GiB/s) | 20260505 03:33:52.072 | INFO | Analyzing chunk frames = [13824,14336) | 20260505 03:33:52.072 | INFO | MemInfo: { | 20260505 03:33:52.072 | INFO | "gpu" : " 14 GB", | 20260505 03:33:52.072 | INFO | "huge" : "", | 20260505 03:33:52.072 | INFO | "huge_shared" : " 27 GB", | 20260505 03:33:52.072 | INFO | "malloc" : " 15 GB", | 20260505 03:33:52.072 | INFO | "pinned" : " 28 GB" | 20260505 03:33:52.072 | INFO | } | 20260505 03:33:52.072 | INFO | Perf report for the last 5.13945 seconds: | 20260505 03:33:52.072 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:33:52.072 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:52.072 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:52.072 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:52.072 | INFO | ANALYSIS : 44.6%, 3.0, 69.5ms, 0 | 20260505 03:33:52.072 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:52.072 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:52.072 | INFO | PRE_HQ : 5.7%, 1.0, 3.0ms, 0 | 20260505 03:33:52.072 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:52.072 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:52.073 | INFO | Uploaded 36GiB to GPU in 1.57272s (22.8902GiB/s) | 20260505 03:33:52.073 | INFO | Downloaded 24GiB to GPU in 1.09642s (21.8894GiB/s) | 20260505 03:33:57.211 | INFO | Analyzing chunk frames = [14336,14848) | 20260505 03:33:57.211 | INFO | MemInfo: { | 20260505 03:33:57.211 | INFO | "gpu" : " 16 GB", | 20260505 03:33:57.211 | INFO | "huge" : "", | 20260505 03:33:57.211 | INFO | "huge_shared" : " 27 GB", | 20260505 03:33:57.211 | INFO | "malloc" : " 15 GB", | 20260505 03:33:57.212 | INFO | "pinned" : " 28 GB" | 20260505 03:33:57.212 | INFO | } | 20260505 03:33:57.212 | INFO | Perf report for the last 5.13933 seconds: | 20260505 03:33:57.212 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:33:57.212 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:57.212 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:57.212 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:57.212 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260505 03:33:57.212 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:57.212 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:57.212 | INFO | PRE_HQ : 11.4%, 1.0, 6.2ms, 0 | 20260505 03:33:57.212 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:57.212 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:33:57.212 | INFO | Uploaded 36GiB to GPU in 1.5736s (22.8774GiB/s) | 20260505 03:33:57.212 | INFO | Downloaded 26GiB to GPU in 1.21463s (21.4057GiB/s) | 20260505 03:34:02.351 | INFO | Analyzing chunk frames = [14848,15360) | 20260505 03:34:02.351 | INFO | MemInfo: { | 20260505 03:34:02.351 | INFO | "gpu" : " 16 GB", | 20260505 03:34:02.351 | INFO | "huge" : "", | 20260505 03:34:02.351 | INFO | "huge_shared" : " 27 GB", | 20260505 03:34:02.351 | INFO | "malloc" : " 15 GB", | 20260505 03:34:02.351 | INFO | "pinned" : " 28 GB" | 20260505 03:34:02.351 | INFO | } | 20260505 03:34:02.351 | INFO | Perf report for the last 5.13918 seconds: | 20260505 03:34:02.351 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:34:02.351 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:02.351 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:02.351 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:02.351 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260505 03:34:02.351 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:02.351 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:02.351 | INFO | PRE_HQ : 15.4%, 1.0, 8.3ms, 0 | 20260505 03:34:02.351 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:02.351 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:02.351 | INFO | Uploaded 36GiB to GPU in 1.57536s (22.8519GiB/s) | 20260505 03:34:02.351 | INFO | Downloaded 27GiB to GPU in 1.2014s (22.4737GiB/s) | 20260505 03:34:07.490 | INFO | Analyzing chunk frames = [15360,15872) | 20260505 03:34:07.490 | INFO | MemInfo: { | 20260505 03:34:07.490 | INFO | "gpu" : " 16 GB", | 20260505 03:34:07.490 | INFO | "huge" : "", | 20260505 03:34:07.490 | INFO | "huge_shared" : " 27 GB", | 20260505 03:34:07.490 | INFO | "malloc" : " 15 GB", | 20260505 03:34:07.490 | INFO | "pinned" : " 28 GB" | 20260505 03:34:07.490 | INFO | } | 20260505 03:34:07.490 | INFO | Perf report for the last 5.13928 seconds: | 20260505 03:34:07.490 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:34:07.490 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:07.490 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:07.490 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:07.490 | INFO | ANALYSIS : 64.7%, 3.0, 101.0ms, 0 | 20260505 03:34:07.490 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:07.490 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:07.490 | INFO | PRE_HQ : 18.5%, 1.0, 9.9ms, 0 | 20260505 03:34:07.490 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:07.490 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:07.491 | INFO | Uploaded 36GiB to GPU in 1.64633s (21.8668GiB/s) | 20260505 03:34:07.491 | INFO | Downloaded 28GiB to GPU in 1.32082s (21.199GiB/s) | 20260505 03:34:12.629 | INFO | Analyzing chunk frames = [15872,16384) | 20260505 03:34:12.629 | INFO | MemInfo: { | 20260505 03:34:12.629 | INFO | "gpu" : " 16 GB", | 20260505 03:34:12.629 | INFO | "huge" : "", | 20260505 03:34:12.629 | INFO | "huge_shared" : " 27 GB", | 20260505 03:34:12.629 | INFO | "malloc" : " 15 GB", | 20260505 03:34:12.629 | INFO | "pinned" : " 28 GB" | 20260505 03:34:12.630 | INFO | } | 20260505 03:34:12.630 | INFO | Perf report for the last 5.13945 seconds: | 20260505 03:34:12.630 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:34:12.630 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:12.630 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:12.630 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:12.630 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 03:34:12.630 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:12.630 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:12.630 | INFO | PRE_HQ : 21.5%, 1.0, 11.5ms, 0 | 20260505 03:34:12.630 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:12.630 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:12.630 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 03:34:12.630 | INFO | Total mem usage: 14.6 GiB | 20260505 03:34:12.630 | INFO | All Time High Water Marks sum: 14.6 GiB (Savings: 0.0527%) | 20260505 03:34:12.630 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: 0.0527%) | 20260505 03:34:12.630 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 03:34:12.630 | INFO | High Water Breakdown: | 20260505 03:34:12.630 | INFO | 54.6% : 7.96 GiB ( 100% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 03:34:12.630 | INFO | 45.3% : 6.6 GiB ( 100% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 03:34:12.630 | INFO | 0.128% : 19.1 MiB ( 100% Of Peak) : SampledBazData | 20260505 03:34:12.630 | INFO | ------------------------------------------ | 20260505 03:34:12.630 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:34:12.630 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 03:34:12.630 | INFO | Total mem usage: 26.5 GiB | 20260505 03:34:12.630 | INFO | All Time High Water Marks sum: 26.7 GiB (Savings: 0.768%) | 20260505 03:34:12.630 | INFO | Recent High Water Marks Sum: 26.7 GiB (Savings: 0.768%) | 20260505 03:34:12.630 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 03:34:12.630 | INFO | High Water Breakdown: | 20260505 03:34:12.630 | INFO | 53.7% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 03:34:12.630 | INFO | 34.7% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 03:34:12.630 | INFO | 3.51% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 03:34:12.630 | INFO | 2.98% : 816 MiB ( 100% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 03:34:12.630 | INFO | 1.89% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 03:34:12.630 | INFO | 1.4% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 03:34:12.630 | INFO | 0.98% : 268 MiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 03:34:12.630 | INFO | 0.768% : 210 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 03:34:12.630 | INFO | 0.0146% : 4 MiB ( 100% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 03:34:12.630 | INFO | 0.0146% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 03:34:12.630 | INFO | 0.00146% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 03:34:12.630 | INFO | 8.96e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 03:34:12.630 | INFO | ------------------------------------------ | 20260505 03:34:12.630 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:34:12.630 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 03:34:12.630 | INFO | Total mem usage: 25.6 GiB | 20260505 03:34:12.630 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 03:34:12.630 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 03:34:12.630 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 03:34:12.630 | INFO | High Water Breakdown: | 20260505 03:34:12.630 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 03:34:12.630 | INFO | ------------------------------------------ | 20260505 03:34:12.630 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:34:12.630 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 03:34:12.630 | INFO | Total mem usage: 14.7 GiB | 20260505 03:34:12.630 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 8.25%) | 20260505 03:34:12.630 | INFO | Recent High Water Marks Sum: 16 GiB (Savings: 8.25%) | 20260505 03:34:12.630 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 03:34:12.630 | INFO | High Water Breakdown: | 20260505 03:34:12.630 | INFO | 33.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 03:34:12.630 | INFO | 13.8% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 03:34:12.630 | INFO | 8.8% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 03:34:12.630 | INFO | 7.04% : 1.12 GiB ( 100% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 03:34:12.630 | INFO | 6.45% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 03:34:12.630 | INFO | 5.08% : 832 MiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 03:34:12.630 | INFO | 5.04% : 824 MiB ( 100% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 03:34:12.630 | INFO | 3.52% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 03:34:12.630 | INFO | 3.16% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 03:34:12.630 | INFO | 2.8% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 03:34:12.630 | INFO | 2.35% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 03:34:12.630 | INFO | 1.81% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 03:34:12.630 | INFO | 1.71% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 03:34:12.630 | INFO | 1.63% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 03:34:12.630 | INFO | 0.782% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 03:34:12.630 | INFO | 0.782% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 03:34:12.630 | INFO | 0.642% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 03:34:12.630 | INFO | 0.587% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 03:34:12.630 | INFO | 0.183% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 03:34:12.630 | INFO | 0.0733% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 03:34:12.630 | INFO | 0.0244% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 03:34:12.630 | INFO | 0.00611% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 03:34:12.630 | INFO | ------------------------------------------ | 20260505 03:34:12.630 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:34:12.630 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 03:34:12.630 | INFO | PerfReport: Gathered 1536 samples. | 20260505 03:34:12.630 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 03:34:12.630 | INFO | PerfReport: Full Execution -- 7.67e+01 / -nan, 2.31e+01 / 0.00e+00 | 20260505 03:34:12.630 | INFO | PerfReport: 21.56%, Upload -- 1.65e+01 / 0.00e+00, 1.69e+00 / 0.00e+00 | 20260505 03:34:12.630 | INFO | PerfReport: 15.59%, Download -- 1.20e+01 / 0.00e+00, 2.72e+00 / 0.00e+00 | 20260505 03:34:12.630 | INFO | PerfReport: 33.27%, Compute -- 2.55e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 03:34:12.630 | INFO | PerfReport: 0.66%, LockUpload -- 5.03e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260505 03:34:12.630 | INFO | PerfReport: 0.00%, LockDownload -- 4.89e-04 / 0.00e+00, 1.43e-04 / 0.00e+00 | 20260505 03:34:12.630 | INFO | PerfReport: 28.92%, LockCompute -- 2.22e+01 / 0.00e+00, 1.76e+01 / 0.00e+00 | 20260505 03:34:12.630 | INFO | PerfReport: 0.01%, Misc -- 1.01e-02 / 0.00e+00, 8.92e-03 / 0.00e+00 | 20260505 03:34:12.630 | INFO | Uploaded 36GiB to GPU in 1.58339s (22.736GiB/s) | 20260505 03:34:12.631 | INFO | Downloaded 29GiB to GPU in 1.30531s (22.2169GiB/s) | 20260505 03:34:12.753 | INFO | Starting a round of Baz IO | 20260505 03:34:16.109 | INFO | Finished a round of Baz IO: Wrote 3.469 KiB in 3.356 seconds | 20260505 03:34:16.109 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 100.00% / 0.00% / 0.00% | 20260505 03:34:16.109 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0239213% / 0% / 0% | 20260505 03:34:16.109 | INFO | Baz IO overall: | 20260505 03:34:16.109 | INFO | BytesWritten : 2571427046 | 20260505 03:34:16.109 | INFO | NumEvents : 0 | 20260505 03:34:17.769 | INFO | Analyzing chunk frames = [16384,16896) | 20260505 03:34:17.769 | INFO | MemInfo: { | 20260505 03:34:17.769 | INFO | "gpu" : " 17 GB", | 20260505 03:34:17.769 | INFO | "huge" : "", | 20260505 03:34:17.769 | INFO | "huge_shared" : " 27 GB", | 20260505 03:34:17.769 | INFO | "malloc" : " 15 GB", | 20260505 03:34:17.769 | INFO | "pinned" : " 29 GB" | 20260505 03:34:17.769 | INFO | } | 20260505 03:34:17.769 | INFO | Perf report for the last 5.1394 seconds: | 20260505 03:34:17.769 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:34:17.769 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:17.769 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:17.769 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:17.769 | INFO | ANALYSIS : 67.1%, 3.0, 104.9ms, 0 | 20260505 03:34:17.769 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:17.769 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:17.769 | INFO | PRE_HQ : 53.8%, 1.6, 45.1ms, 0 | 20260505 03:34:17.769 | INFO | BAZWRITER : 9.3%, 1.0, 5.0ms, 0 | 20260505 03:34:17.769 | INFO | BAZ_SAMPLING : 0.1%, 1.0, 0.0ms, 0 | 20260505 03:34:17.769 | INFO | Uploaded 36GiB to GPU in 1.78675s (20.1483GiB/s) | 20260505 03:34:17.769 | INFO | Downloaded 33GiB to GPU in 1.66889s (19.7737GiB/s) | 20260505 03:34:22.908 | INFO | Analyzing chunk frames = [16896,17408) | 20260505 03:34:22.908 | INFO | MemInfo: { | 20260505 03:34:22.908 | INFO | "gpu" : " 17 GB", | 20260505 03:34:22.908 | INFO | "huge" : "", | 20260505 03:34:22.908 | INFO | "huge_shared" : " 27 GB", | 20260505 03:34:22.908 | INFO | "malloc" : " 15 GB", | 20260505 03:34:22.908 | INFO | "pinned" : " 29 GB" | 20260505 03:34:22.908 | INFO | } | 20260505 03:34:22.908 | INFO | Perf report for the last 5.13934 seconds: | 20260505 03:34:22.908 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:34:22.908 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:22.908 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:22.908 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:22.908 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 03:34:22.908 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:22.908 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:22.908 | INFO | PRE_HQ : 35.8%, 1.1, 20.4ms, 0 | 20260505 03:34:22.908 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:22.908 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:22.909 | INFO | Uploaded 36GiB to GPU in 1.59155s (22.6195GiB/s) | 20260505 03:34:22.909 | INFO | Downloaded 32GiB to GPU in 1.41052s (22.6867GiB/s) | 20260505 03:34:28.047 | INFO | Analyzing chunk frames = [17408,17920) | 20260505 03:34:28.047 | INFO | MemInfo: { | 20260505 03:34:28.047 | INFO | "gpu" : " 17 GB", | 20260505 03:34:28.047 | INFO | "huge" : "", | 20260505 03:34:28.047 | INFO | "huge_shared" : " 27 GB", | 20260505 03:34:28.048 | INFO | "malloc" : " 15 GB", | 20260505 03:34:28.048 | INFO | "pinned" : " 29 GB" | 20260505 03:34:28.048 | INFO | } | 20260505 03:34:28.048 | INFO | Perf report for the last 5.13922 seconds: | 20260505 03:34:28.048 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:34:28.048 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:28.048 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:28.048 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:28.048 | INFO | ANALYSIS : 63.6%, 3.0, 98.8ms, 0 | 20260505 03:34:28.048 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:28.048 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:28.048 | INFO | PRE_HQ : 31.6%, 1.0, 16.6ms, 0 | 20260505 03:34:28.048 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:28.048 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:28.048 | INFO | Uploaded 36GiB to GPU in 1.60011s (22.4984GiB/s) | 20260505 03:34:28.048 | INFO | Downloaded 33GiB to GPU in 1.46945s (22.4574GiB/s) | 20260505 03:34:33.187 | INFO | Analyzing chunk frames = [17920,18432) | 20260505 03:34:33.187 | INFO | MemInfo: { | 20260505 03:34:33.187 | INFO | "gpu" : " 17 GB", | 20260505 03:34:33.187 | INFO | "huge" : "", | 20260505 03:34:33.187 | INFO | "huge_shared" : " 27 GB", | 20260505 03:34:33.187 | INFO | "malloc" : " 15 GB", | 20260505 03:34:33.187 | INFO | "pinned" : " 29 GB" | 20260505 03:34:33.187 | INFO | } | 20260505 03:34:33.187 | INFO | Perf report for the last 5.13941 seconds: | 20260505 03:34:33.187 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:34:33.187 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:33.187 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:33.187 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:33.187 | INFO | ANALYSIS : 63.8%, 3.0, 98.8ms, 0 | 20260505 03:34:33.187 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:33.187 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:33.187 | INFO | PRE_HQ : 33.9%, 1.0, 17.9ms, 0 | 20260505 03:34:33.187 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:33.187 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:33.187 | INFO | Uploaded 36GiB to GPU in 1.60335s (22.4529GiB/s) | 20260505 03:34:33.187 | INFO | Downloaded 34GiB to GPU in 1.52165s (22.3441GiB/s) | 20260505 03:34:38.326 | INFO | Analyzing chunk frames = [18432,18944) | 20260505 03:34:38.326 | INFO | MemInfo: { | 20260505 03:34:38.326 | INFO | "gpu" : " 17 GB", | 20260505 03:34:38.326 | INFO | "huge" : "", | 20260505 03:34:38.326 | INFO | "huge_shared" : " 27 GB", | 20260505 03:34:38.326 | INFO | "malloc" : " 15 GB", | 20260505 03:34:38.326 | INFO | "pinned" : " 29 GB" | 20260505 03:34:38.326 | INFO | } | 20260505 03:34:38.326 | INFO | Perf report for the last 5.13939 seconds: | 20260505 03:34:38.326 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:34:38.326 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:38.326 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:38.326 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:38.326 | INFO | ANALYSIS : 61.4%, 3.0, 95.9ms, 0 | 20260505 03:34:38.326 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:38.326 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:38.327 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 03:34:38.327 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:38.327 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:38.327 | INFO | Uploaded 36GiB to GPU in 1.60015s (22.4979GiB/s) | 20260505 03:34:38.327 | INFO | Downloaded 34GiB to GPU in 1.51839s (22.3922GiB/s) | 20260505 03:34:43.465 | INFO | Analyzing chunk frames = [18944,19456) | 20260505 03:34:43.465 | INFO | MemInfo: { | 20260505 03:34:43.465 | INFO | "gpu" : " 17 GB", | 20260505 03:34:43.465 | INFO | "huge" : "", | 20260505 03:34:43.465 | INFO | "huge_shared" : " 27 GB", | 20260505 03:34:43.466 | INFO | "malloc" : " 15 GB", | 20260505 03:34:43.466 | INFO | "pinned" : " 29 GB" | 20260505 03:34:43.466 | INFO | } | 20260505 03:34:43.466 | INFO | Perf report for the last 5.13921 seconds: | 20260505 03:34:43.466 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:34:43.466 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:43.466 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:43.466 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:43.466 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 03:34:43.466 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:43.466 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:43.466 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 03:34:43.466 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:43.466 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:43.466 | INFO | Uploaded 36GiB to GPU in 1.60116s (22.4837GiB/s) | 20260505 03:34:43.466 | INFO | Downloaded 34GiB to GPU in 1.51845s (22.3913GiB/s) | 20260505 03:34:48.605 | INFO | Analyzing chunk frames = [19456,19968) | 20260505 03:34:48.605 | INFO | MemInfo: { | 20260505 03:34:48.605 | INFO | "gpu" : " 17 GB", | 20260505 03:34:48.605 | INFO | "huge" : "", | 20260505 03:34:48.605 | INFO | "huge_shared" : " 27 GB", | 20260505 03:34:48.605 | INFO | "malloc" : " 15 GB", | 20260505 03:34:48.605 | INFO | "pinned" : " 29 GB" | 20260505 03:34:48.605 | INFO | } | 20260505 03:34:48.605 | INFO | Perf report for the last 5.13942 seconds: | 20260505 03:34:48.605 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:34:48.605 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:48.605 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:48.605 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:48.605 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260505 03:34:48.605 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:48.605 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:48.605 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 03:34:48.605 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:48.605 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:48.605 | INFO | Uploaded 36GiB to GPU in 1.60196s (22.4725GiB/s) | 20260505 03:34:48.605 | INFO | Downloaded 34GiB to GPU in 1.51968s (22.3731GiB/s) | 20260505 03:34:53.744 | INFO | Analyzing chunk frames = [19968,20480) | 20260505 03:34:53.744 | INFO | MemInfo: { | 20260505 03:34:53.744 | INFO | "gpu" : " 17 GB", | 20260505 03:34:53.744 | INFO | "huge" : "", | 20260505 03:34:53.744 | INFO | "huge_shared" : " 27 GB", | 20260505 03:34:53.744 | INFO | "malloc" : " 15 GB", | 20260505 03:34:53.744 | INFO | "pinned" : " 29 GB" | 20260505 03:34:53.744 | INFO | } | 20260505 03:34:53.744 | INFO | Perf report for the last 5.1392 seconds: | 20260505 03:34:53.744 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:34:53.744 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:53.744 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:53.744 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:53.744 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 03:34:53.744 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:53.744 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:53.744 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 03:34:53.744 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:53.744 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:53.745 | INFO | Uploaded 36GiB to GPU in 1.60111s (22.4844GiB/s) | 20260505 03:34:53.745 | INFO | Downloaded 34GiB to GPU in 1.51934s (22.3781GiB/s) | 20260505 03:34:53.838 | INFO | Starting a round of Baz IO | 20260505 03:34:57.057 | INFO | Finished a round of Baz IO: Wrote 3.469 KiB in 3.219 seconds | 20260505 03:34:57.057 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 100.00% / 0.00% / 0.00% | 20260505 03:34:57.057 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.67426% / 0% / 0% | 20260505 03:34:57.057 | INFO | Baz IO overall: | 20260505 03:34:57.057 | INFO | BytesWritten : 2571430598 | 20260505 03:34:57.057 | INFO | NumEvents : 0 | 20260505 03:34:58.883 | INFO | Analyzing chunk frames = [20480,20992) | 20260505 03:34:58.883 | INFO | MemInfo: { | 20260505 03:34:58.883 | INFO | "gpu" : " 17 GB", | 20260505 03:34:58.883 | INFO | "huge" : "", | 20260505 03:34:58.884 | INFO | "huge_shared" : " 27 GB", | 20260505 03:34:58.884 | INFO | "malloc" : " 15 GB", | 20260505 03:34:58.884 | INFO | "pinned" : " 29 GB" | 20260505 03:34:58.884 | INFO | } | 20260505 03:34:58.884 | INFO | Perf report for the last 5.13932 seconds: | 20260505 03:34:58.884 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:34:58.884 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:58.884 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:58.884 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:58.884 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 03:34:58.884 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:34:58.884 | INFO | RT_METRICS : 31.3%, 1.0, 21.8ms, 0 | 20260505 03:34:58.884 | INFO | PRE_HQ : 56.5%, 1.9, 56.5ms, 0 | 20260505 03:34:58.884 | INFO | BAZWRITER : 9.8%, 1.0, 5.1ms, 0 | 20260505 03:34:58.884 | INFO | BAZ_SAMPLING : 0.1%, 1.0, 0.0ms, 0 | 20260505 03:34:58.884 | INFO | Uploaded 36GiB to GPU in 1.66122s (21.6709GiB/s) | 20260505 03:34:58.884 | INFO | Downloaded 38GiB to GPU in 1.72271s (22.0583GiB/s) | 20260505 03:35:04.023 | INFO | Analyzing chunk frames = [20992,21504) | 20260505 03:35:04.023 | INFO | MemInfo: { | 20260505 03:35:04.023 | INFO | "gpu" : " 17 GB", | 20260505 03:35:04.023 | INFO | "huge" : "", | 20260505 03:35:04.023 | INFO | "huge_shared" : " 27 GB", | 20260505 03:35:04.023 | INFO | "malloc" : " 15 GB", | 20260505 03:35:04.023 | INFO | "pinned" : " 29 GB" | 20260505 03:35:04.023 | INFO | } | 20260505 03:35:04.023 | INFO | Perf report for the last 5.13939 seconds: | 20260505 03:35:04.023 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:35:04.023 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:04.023 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:04.023 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:04.023 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 03:35:04.023 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:04.023 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:04.023 | INFO | PRE_HQ : 39.7%, 1.1, 21.9ms, 0 | 20260505 03:35:04.023 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:04.023 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:04.024 | INFO | Uploaded 36GiB to GPU in 1.59863s (22.5193GiB/s) | 20260505 03:35:04.024 | INFO | Downloaded 34GiB to GPU in 1.51558s (22.4336GiB/s) | 20260505 03:35:09.162 | INFO | Analyzing chunk frames = [21504,22016) | 20260505 03:35:09.162 | INFO | MemInfo: { | 20260505 03:35:09.162 | INFO | "gpu" : " 17 GB", | 20260505 03:35:09.162 | INFO | "huge" : "", | 20260505 03:35:09.162 | INFO | "huge_shared" : " 27 GB", | 20260505 03:35:09.162 | INFO | "malloc" : " 15 GB", | 20260505 03:35:09.162 | INFO | "pinned" : " 29 GB" | 20260505 03:35:09.162 | INFO | } | 20260505 03:35:09.162 | INFO | Perf report for the last 5.13941 seconds: | 20260505 03:35:09.162 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:35:09.162 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:09.162 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:09.162 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:09.162 | INFO | ANALYSIS : 62.4%, 3.0, 96.9ms, 0 | 20260505 03:35:09.162 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:09.163 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:09.163 | INFO | PRE_HQ : 33.0%, 1.0, 17.1ms, 0 | 20260505 03:35:09.163 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:09.163 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:09.163 | INFO | Uploaded 36GiB to GPU in 1.60253s (22.4644GiB/s) | 20260505 03:35:09.163 | INFO | Downloaded 34GiB to GPU in 1.52018s (22.3657GiB/s) | 20260505 03:35:14.302 | INFO | Analyzing chunk frames = [22016,22528) | 20260505 03:35:14.302 | INFO | MemInfo: { | 20260505 03:35:14.302 | INFO | "gpu" : " 17 GB", | 20260505 03:35:14.302 | INFO | "huge" : "", | 20260505 03:35:14.302 | INFO | "huge_shared" : " 27 GB", | 20260505 03:35:14.302 | INFO | "malloc" : " 15 GB", | 20260505 03:35:14.302 | INFO | "pinned" : " 29 GB" | 20260505 03:35:14.302 | INFO | } | 20260505 03:35:14.302 | INFO | Perf report for the last 5.13953 seconds: | 20260505 03:35:14.302 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:35:14.302 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:14.302 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:14.302 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:14.302 | INFO | ANALYSIS : 61.8%, 3.0, 95.8ms, 0 | 20260505 03:35:14.302 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:14.302 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:14.302 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260505 03:35:14.302 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:14.302 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:14.303 | INFO | Uploaded 36GiB to GPU in 1.60127s (22.4821GiB/s) | 20260505 03:35:14.303 | INFO | Downloaded 34GiB to GPU in 1.52061s (22.3595GiB/s) | 20260505 03:35:19.441 | INFO | Analyzing chunk frames = [22528,23040) | 20260505 03:35:19.441 | INFO | MemInfo: { | 20260505 03:35:19.441 | INFO | "gpu" : " 17 GB", | 20260505 03:35:19.441 | INFO | "huge" : "", | 20260505 03:35:19.441 | INFO | "huge_shared" : " 27 GB", | 20260505 03:35:19.441 | INFO | "malloc" : " 15 GB", | 20260505 03:35:19.441 | INFO | "pinned" : " 29 GB" | 20260505 03:35:19.441 | INFO | } | 20260505 03:35:19.441 | INFO | Perf report for the last 5.13916 seconds: | 20260505 03:35:19.441 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:35:19.441 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:19.441 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:19.441 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:19.441 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260505 03:35:19.441 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:19.441 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:19.441 | INFO | PRE_HQ : 32.1%, 1.0, 16.6ms, 0 | 20260505 03:35:19.441 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:19.441 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:19.441 | INFO | Uploaded 36GiB to GPU in 1.60018s (22.4974GiB/s) | 20260505 03:35:19.442 | INFO | Downloaded 34GiB to GPU in 1.51664s (22.418GiB/s) | 20260505 03:35:24.580 | INFO | Analyzing chunk frames = [23040,23552) | 20260505 03:35:24.580 | INFO | MemInfo: { | 20260505 03:35:24.580 | INFO | "gpu" : " 17 GB", | 20260505 03:35:24.580 | INFO | "huge" : "", | 20260505 03:35:24.580 | INFO | "huge_shared" : " 27 GB", | 20260505 03:35:24.580 | INFO | "malloc" : " 15 GB", | 20260505 03:35:24.580 | INFO | "pinned" : " 29 GB" | 20260505 03:35:24.580 | INFO | } | 20260505 03:35:24.580 | INFO | Perf report for the last 5.13912 seconds: | 20260505 03:35:24.580 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:35:24.580 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:24.580 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:24.580 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:24.580 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260505 03:35:24.580 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:24.580 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:24.580 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 03:35:24.580 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:24.580 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:24.581 | INFO | Uploaded 36GiB to GPU in 1.60089s (22.4875GiB/s) | 20260505 03:35:24.581 | INFO | Downloaded 34GiB to GPU in 1.51804s (22.3973GiB/s) | 20260505 03:35:29.719 | INFO | Analyzing chunk frames = [23552,24064) | 20260505 03:35:29.719 | INFO | MemInfo: { | 20260505 03:35:29.719 | INFO | "gpu" : " 17 GB", | 20260505 03:35:29.719 | INFO | "huge" : "", | 20260505 03:35:29.719 | INFO | "huge_shared" : " 27 GB", | 20260505 03:35:29.719 | INFO | "malloc" : " 15 GB", | 20260505 03:35:29.719 | INFO | "pinned" : " 29 GB" | 20260505 03:35:29.720 | INFO | } | 20260505 03:35:29.720 | INFO | Perf report for the last 5.13938 seconds: | 20260505 03:35:29.720 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:35:29.720 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:29.720 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:29.720 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:29.720 | INFO | ANALYSIS : 57.6%, 3.0, 89.8ms, 0 | 20260505 03:35:29.720 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:29.720 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:29.720 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 03:35:29.720 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:29.720 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:29.720 | INFO | Uploaded 36GiB to GPU in 1.60038s (22.4947GiB/s) | 20260505 03:35:29.720 | INFO | Downloaded 34GiB to GPU in 1.51806s (22.3969GiB/s) | 20260505 03:35:34.859 | INFO | Analyzing chunk frames = [24064,24576) | 20260505 03:35:34.859 | INFO | MemInfo: { | 20260505 03:35:34.859 | INFO | "gpu" : " 17 GB", | 20260505 03:35:34.859 | INFO | "huge" : "", | 20260505 03:35:34.859 | INFO | "huge_shared" : " 27 GB", | 20260505 03:35:34.859 | INFO | "malloc" : " 15 GB", | 20260505 03:35:34.859 | INFO | "pinned" : " 29 GB" | 20260505 03:35:34.859 | INFO | } | 20260505 03:35:34.859 | INFO | Perf report for the last 5.13934 seconds: | 20260505 03:35:34.859 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:35:34.859 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:34.859 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:34.859 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:34.859 | INFO | ANALYSIS : 57.6%, 3.0, 89.8ms, 0 | 20260505 03:35:34.859 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:34.859 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:34.859 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 03:35:34.859 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:34.859 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:34.859 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 03:35:34.860 | INFO | Total mem usage: 14.7 GiB | 20260505 03:35:34.860 | INFO | All Time High Water Marks sum: 14.7 GiB (Savings: 0.0524%) | 20260505 03:35:34.860 | INFO | Recent High Water Marks Sum: 14.7 GiB (Savings: 0.0524%) | 20260505 03:35:34.860 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 03:35:34.860 | INFO | High Water Breakdown: | 20260505 03:35:34.860 | INFO | 54.8% : 8.04 GiB ( 100% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 03:35:34.860 | INFO | 45.1% : 6.61 GiB ( 100% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 03:35:34.860 | INFO | 0.127% : 19.1 MiB ( 100% Of Peak) : SampledBazData | 20260505 03:35:34.860 | INFO | ------------------------------------------ | 20260505 03:35:34.860 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:35:34.860 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 03:35:34.860 | INFO | Total mem usage: 26.5 GiB | 20260505 03:35:34.860 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260505 03:35:34.860 | INFO | Recent High Water Marks Sum: 26.3 GiB (Savings: -0.93%) | 20260505 03:35:34.860 | INFO | Unused Cache: 1.03 GiB (Waste: 3.9%) | 20260505 03:35:34.860 | INFO | High Water Breakdown: | 20260505 03:35:34.860 | INFO | 54.6% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 03:35:34.860 | INFO | 35.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 03:35:34.860 | INFO | 3.57% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 03:35:34.860 | INFO | 1.92% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 03:35:34.860 | INFO | 1.56% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 03:35:34.860 | INFO | 1.43% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 03:35:34.860 | INFO | 0.996% : 268 MiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 03:35:34.860 | INFO | Uploaded 36GiB to GPU in 1.60091s (22.4872GiB/s) | 20260505 03:35:34.860 | INFO | 0.543% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 03:35:34.860 | INFO | Downloaded 34GiB to GPU in 1.51804s (22.3973GiB/s) | 20260505 03:35:34.860 | INFO | 0.0149% : 4 MiB ( 100% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 03:35:34.860 | INFO | 0.0149% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 03:35:34.860 | INFO | 0.00148% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 03:35:34.860 | INFO | 9.11e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 03:35:34.860 | INFO | 8.51e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 03:35:34.860 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 03:35:34.860 | INFO | ------------------------------------------ | 20260505 03:35:34.860 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:35:34.860 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 03:35:34.860 | INFO | Total mem usage: 25.6 GiB | 20260505 03:35:34.860 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 03:35:34.860 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 03:35:34.860 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 03:35:34.860 | INFO | High Water Breakdown: | 20260505 03:35:34.860 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 03:35:34.860 | INFO | ------------------------------------------ | 20260505 03:35:34.860 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:35:34.860 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 03:35:34.860 | INFO | Total mem usage: 15.7 GiB | 20260505 03:35:34.860 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260505 03:35:34.860 | INFO | Recent High Water Marks Sum: 17 GiB (Savings: 7.99%) | 20260505 03:35:34.860 | INFO | Unused Cache: 786 MiB (Waste: 4.89%) | 20260505 03:35:34.860 | INFO | High Water Breakdown: | 20260505 03:35:34.860 | INFO | 31.6% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 03:35:34.860 | INFO | 12.9% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 03:35:34.860 | INFO | 8.25% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 03:35:34.860 | INFO | 8.14% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 03:35:34.860 | INFO | 6.05% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 03:35:34.860 | INFO | 5.92% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 03:35:34.860 | INFO | 4.73% : 824 MiB ( 100% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 03:35:34.860 | INFO | 3.3% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 03:35:34.860 | INFO | 3.3% : 576 MiB ( 50% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 03:35:34.860 | INFO | 2.96% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 03:35:34.860 | INFO | 2.63% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 03:35:34.860 | INFO | 2.2% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 03:35:34.860 | INFO | 1.7% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 03:35:34.860 | INFO | 1.6% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 03:35:34.860 | INFO | 1.53% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 03:35:34.860 | INFO | 0.734% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 03:35:34.861 | INFO | 0.734% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 03:35:34.861 | INFO | 0.602% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 03:35:34.861 | INFO | 0.55% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 03:35:34.861 | INFO | 0.209% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 03:35:34.861 | INFO | 0.172% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 03:35:34.861 | INFO | 0.0688% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 03:35:34.861 | INFO | 0.0229% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 03:35:34.861 | INFO | 0.00573% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 03:35:34.861 | INFO | 1.31e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 03:35:34.861 | INFO | ------------------------------------------ | 20260505 03:35:34.861 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:35:34.861 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 03:35:34.861 | INFO | PerfReport: Gathered 288 samples. | 20260505 03:35:34.861 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 03:35:34.861 | INFO | PerfReport: Full Execution -- 3.04e+01 / -nan, 1.17e+01 / 0.00e+00 | 20260505 03:35:34.861 | INFO | PerfReport: 0.28%, AggregateStats -- 8.62e-02 / 0.00e+00, 3.19e-03 / 0.00e+00 | 20260505 03:35:34.861 | INFO | PerfReport: 10.82%, Baseline -- 3.29e+00 / 0.00e+00, 1.07e-02 / 0.00e+00 | 20260505 03:35:34.861 | INFO | PerfReport: 2.99%, Histogram -- 9.10e-01 / 0.00e+00, 4.62e-02 / 0.00e+00 | 20260505 03:35:34.861 | INFO | PerfReport: 16.36%, DME -- 3.99e+01 / 0.00e+00, 1.30e+00 / 0.00e+00 | 20260505 03:35:34.861 | INFO | PerfReport: 56.58%, FrameLabeling -- 1.72e+01 / 0.00e+00, 4.14e-02 / 0.00e+00 | 20260505 03:35:34.861 | INFO | PerfReport: 7.59%, PulseAccumulating -- 2.31e+00 / 0.00e+00, 1.48e-01 / 0.00e+00 | 20260505 03:35:34.861 | INFO | PerfReport: 3.69%, Metrics -- 1.12e+00 / 0.00e+00, 1.60e-01 / 0.00e+00 | 20260505 03:35:34.861 | INFO | PerfReport: 1.68%, PulseSerializing -- 5.10e-01 / 0.00e+00, 8.19e-02 / 0.00e+00 | 20260505 03:35:34.861 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 03:35:34.861 | INFO | PerfReport: Gathered 1536 samples. | 20260505 03:35:34.861 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 03:35:34.861 | INFO | PerfReport: Full Execution -- 9.82e+01 / -nan, 2.87e+01 / 0.00e+00 | 20260505 03:35:34.861 | INFO | PerfReport: 17.14%, Upload -- 1.68e+01 / 0.00e+00, 2.47e+00 / 0.00e+00 | 20260505 03:35:34.861 | INFO | PerfReport: 16.24%, Download -- 1.59e+01 / 0.00e+00, 2.95e+00 / 0.00e+00 | 20260505 03:35:34.861 | INFO | PerfReport: 33.31%, Compute -- 3.27e+01 / 0.00e+00, 1.98e+01 / 0.00e+00 | 20260505 03:35:34.861 | INFO | PerfReport: 0.51%, LockUpload -- 5.03e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260505 03:35:34.861 | INFO | PerfReport: 0.02%, LockDownload -- 1.82e-02 / 0.00e+00, 4.01e-01 / 0.00e+00 | 20260505 03:35:34.861 | INFO | PerfReport: 32.77%, LockCompute -- 3.22e+01 / 0.00e+00, 2.55e+01 / 0.00e+00 | 20260505 03:35:34.861 | INFO | PerfReport: 0.02%, Misc -- 1.50e-02 / 0.00e+00, 5.03e-02 / 0.00e+00 | 20260505 03:35:34.952 | INFO | Starting a round of Baz IO | 20260505 03:35:37.899 | INFO | Finished a round of Baz IO: Wrote 3.469 KiB in 2.946 seconds | 20260505 03:35:37.899 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 100.00% / 0.00% / 0.00% | 20260505 03:35:37.899 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 11.007% / 0% / 0% | 20260505 03:35:37.899 | INFO | Baz IO overall: | 20260505 03:35:37.899 | INFO | BytesWritten : 2571434150 | 20260505 03:35:37.899 | INFO | NumEvents : 0 | 20260505 03:35:37.900 | INFO | Populating RTMetrics report | 20260505 03:35:37.900 | INFO | RTMetrics report populated | 20260505 03:35:37.900 | INFO | Opening tmp RTMetrics json file | 20260505 03:35:37.901 | INFO | Serializing RTMetrics to json | 20260505 03:35:37.902 | INFO | Serialization done, swapping tmp file now... | 20260505 03:35:37.903 | INFO | RTMetrics done | 20260505 03:35:39.998 | INFO | Analyzing chunk frames = [24576,25088) | 20260505 03:35:39.998 | INFO | MemInfo: { | 20260505 03:35:39.998 | INFO | "gpu" : " 17 GB", | 20260505 03:35:39.998 | INFO | "huge" : "", | 20260505 03:35:39.998 | INFO | "huge_shared" : " 27 GB", | 20260505 03:35:39.998 | INFO | "malloc" : " 15 GB", | 20260505 03:35:39.998 | INFO | "pinned" : " 29 GB" | 20260505 03:35:39.998 | INFO | } | 20260505 03:35:39.998 | INFO | Perf report for the last 5.13934 seconds: | 20260505 03:35:39.998 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:35:39.998 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:39.998 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:39.998 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:39.998 | INFO | ANALYSIS : 58.3%, 3.0, 90.9ms, 0 | 20260505 03:35:39.998 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:39.998 | INFO | RT_METRICS : 41.1%, 1.0, 21.6ms, 0 | 20260505 03:35:39.998 | INFO | PRE_HQ : 55.7%, 1.9, 56.1ms, 0 | 20260505 03:35:39.998 | INFO | BAZWRITER : 9.9%, 1.0, 5.1ms, 0 | 20260505 03:35:39.998 | INFO | BAZ_SAMPLING : 0.1%, 1.0, 0.0ms, 0 | 20260505 03:35:39.999 | INFO | Uploaded 36GiB to GPU in 1.64795s (21.8453GiB/s) | 20260505 03:35:39.999 | INFO | Downloaded 38GiB to GPU in 1.70711s (22.2598GiB/s) | 20260505 03:35:45.137 | INFO | Analyzing chunk frames = [25088,25600) | 20260505 03:35:45.137 | INFO | MemInfo: { | 20260505 03:35:45.137 | INFO | "gpu" : " 17 GB", | 20260505 03:35:45.137 | INFO | "huge" : "", | 20260505 03:35:45.138 | INFO | "huge_shared" : " 27 GB", | 20260505 03:35:45.138 | INFO | "malloc" : " 15 GB", | 20260505 03:35:45.138 | INFO | "pinned" : " 29 GB" | 20260505 03:35:45.138 | INFO | } | 20260505 03:35:45.138 | INFO | Perf report for the last 5.13936 seconds: | 20260505 03:35:45.138 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:35:45.138 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:45.138 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:45.138 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:45.138 | INFO | ANALYSIS : 58.0%, 3.0, 90.5ms, 0 | 20260505 03:35:45.138 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:45.138 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:45.138 | INFO | PRE_HQ : 36.3%, 1.0, 19.1ms, 0 | 20260505 03:35:45.138 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:45.138 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:45.138 | INFO | Uploaded 36GiB to GPU in 1.599s (22.514GiB/s) | 20260505 03:35:45.138 | INFO | Downloaded 34GiB to GPU in 1.51517s (22.4398GiB/s) | 20260505 03:35:50.277 | INFO | Analyzing chunk frames = [25600,26112) | 20260505 03:35:50.277 | INFO | MemInfo: { | 20260505 03:35:50.277 | INFO | "gpu" : " 17 GB", | 20260505 03:35:50.277 | INFO | "huge" : "", | 20260505 03:35:50.277 | INFO | "huge_shared" : " 27 GB", | 20260505 03:35:50.277 | INFO | "malloc" : " 15 GB", | 20260505 03:35:50.277 | INFO | "pinned" : " 29 GB" | 20260505 03:35:50.277 | INFO | } | 20260505 03:35:50.277 | INFO | Perf report for the last 5.13946 seconds: | 20260505 03:35:50.277 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:35:50.277 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:50.277 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:50.277 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:50.277 | INFO | ANALYSIS : 57.5%, 3.0, 89.4ms, 0 | 20260505 03:35:50.277 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:50.277 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:50.277 | INFO | PRE_HQ : 32.7%, 1.0, 17.0ms, 0 | 20260505 03:35:50.277 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:50.277 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:50.277 | INFO | Uploaded 36GiB to GPU in 1.60201s (22.4717GiB/s) | 20260505 03:35:50.278 | INFO | Downloaded 34GiB to GPU in 1.51922s (22.3799GiB/s) | 20260505 03:35:55.416 | INFO | Analyzing chunk frames = [26112,26624) | 20260505 03:35:55.416 | INFO | MemInfo: { | 20260505 03:35:55.416 | INFO | "gpu" : " 17 GB", | 20260505 03:35:55.416 | INFO | "huge" : "", | 20260505 03:35:55.416 | INFO | "huge_shared" : " 27 GB", | 20260505 03:35:55.416 | INFO | "malloc" : " 15 GB", | 20260505 03:35:55.416 | INFO | "pinned" : " 29 GB" | 20260505 03:35:55.416 | INFO | } | 20260505 03:35:55.416 | INFO | Perf report for the last 5.13911 seconds: | 20260505 03:35:55.416 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:35:55.416 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:55.416 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:55.416 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:55.416 | INFO | ANALYSIS : 57.2%, 3.0, 88.7ms, 0 | 20260505 03:35:55.416 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:55.416 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:55.416 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260505 03:35:55.416 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:55.416 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:35:55.417 | INFO | Uploaded 36GiB to GPU in 1.60158s (22.4778GiB/s) | 20260505 03:35:55.417 | INFO | Downloaded 34GiB to GPU in 1.51825s (22.3942GiB/s) | 20260505 03:36:00.556 | INFO | Analyzing chunk frames = [26624,27136) | 20260505 03:36:00.556 | INFO | MemInfo: { | 20260505 03:36:00.556 | INFO | "gpu" : " 17 GB", | 20260505 03:36:00.556 | INFO | "huge" : "", | 20260505 03:36:00.556 | INFO | "huge_shared" : " 27 GB", | 20260505 03:36:00.556 | INFO | "malloc" : " 15 GB", | 20260505 03:36:00.556 | INFO | "pinned" : " 29 GB" | 20260505 03:36:00.556 | INFO | } | 20260505 03:36:00.556 | INFO | Perf report for the last 5.13941 seconds: | 20260505 03:36:00.556 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:36:00.556 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:00.556 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:00.556 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:00.556 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260505 03:36:00.556 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:00.556 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:00.556 | INFO | PRE_HQ : 31.9%, 1.0, 16.5ms, 0 | 20260505 03:36:00.556 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:00.556 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:00.556 | INFO | Uploaded 36GiB to GPU in 1.59929s (22.51GiB/s) | 20260505 03:36:00.556 | INFO | Downloaded 34GiB to GPU in 1.51568s (22.4322GiB/s) | 20260505 03:36:05.695 | INFO | Analyzing chunk frames = [27136,27648) | 20260505 03:36:05.695 | INFO | MemInfo: { | 20260505 03:36:05.695 | INFO | "gpu" : " 17 GB", | 20260505 03:36:05.695 | INFO | "huge" : "", | 20260505 03:36:05.695 | INFO | "huge_shared" : " 27 GB", | 20260505 03:36:05.695 | INFO | "malloc" : " 15 GB", | 20260505 03:36:05.695 | INFO | "pinned" : " 29 GB" | 20260505 03:36:05.695 | INFO | } | 20260505 03:36:05.695 | INFO | Perf report for the last 5.13942 seconds: | 20260505 03:36:05.695 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:36:05.695 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:05.695 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:05.695 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:05.695 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260505 03:36:05.695 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:05.695 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:05.695 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 03:36:05.695 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:05.695 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:05.695 | INFO | Uploaded 36GiB to GPU in 1.59966s (22.5048GiB/s) | 20260505 03:36:05.695 | INFO | Downloaded 34GiB to GPU in 1.51582s (22.4301GiB/s) | 20260505 03:36:10.834 | INFO | Analyzing chunk frames = [27648,28160) | 20260505 03:36:10.834 | INFO | MemInfo: { | 20260505 03:36:10.834 | INFO | "gpu" : " 17 GB", | 20260505 03:36:10.834 | INFO | "huge" : "", | 20260505 03:36:10.834 | INFO | "huge_shared" : " 27 GB", | 20260505 03:36:10.834 | INFO | "malloc" : " 15 GB", | 20260505 03:36:10.834 | INFO | "pinned" : " 29 GB" | 20260505 03:36:10.834 | INFO | } | 20260505 03:36:10.835 | INFO | Perf report for the last 5.13934 seconds: | 20260505 03:36:10.835 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:36:10.835 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:10.835 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:10.835 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:10.835 | INFO | ANALYSIS : 57.7%, 3.0, 89.9ms, 0 | 20260505 03:36:10.835 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:10.835 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:10.835 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 03:36:10.835 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:10.835 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:10.835 | INFO | Uploaded 36GiB to GPU in 1.59987s (22.5018GiB/s) | 20260505 03:36:10.835 | INFO | Downloaded 34GiB to GPU in 1.51667s (22.4175GiB/s) | 20260505 03:36:15.974 | INFO | Analyzing chunk frames = [28160,28672) | 20260505 03:36:15.974 | INFO | MemInfo: { | 20260505 03:36:15.974 | INFO | "gpu" : " 17 GB", | 20260505 03:36:15.974 | INFO | "huge" : "", | 20260505 03:36:15.974 | INFO | "huge_shared" : " 27 GB", | 20260505 03:36:15.974 | INFO | "malloc" : " 15 GB", | 20260505 03:36:15.974 | INFO | "pinned" : " 29 GB" | 20260505 03:36:15.974 | INFO | } | 20260505 03:36:15.974 | INFO | Perf report for the last 5.13921 seconds: | 20260505 03:36:15.974 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:36:15.974 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:15.974 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:15.974 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:15.974 | INFO | ANALYSIS : 57.7%, 3.0, 90.0ms, 0 | 20260505 03:36:15.974 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:15.974 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:15.974 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 03:36:15.974 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:15.974 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:15.974 | INFO | Uploaded 36GiB to GPU in 1.6007s (22.4902GiB/s) | 20260505 03:36:15.974 | INFO | Downloaded 34GiB to GPU in 1.51743s (22.4062GiB/s) | 20260505 03:36:16.066 | INFO | Starting a round of Baz IO | 20260505 03:36:19.018 | INFO | Finished a round of Baz IO: Wrote 3.469 KiB in 2.951 seconds | 20260505 03:36:19.018 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 100.00% / 0.00% / 0.00% | 20260505 03:36:19.018 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 11.7469% / 0% / 0% | 20260505 03:36:19.018 | INFO | Baz IO overall: | 20260505 03:36:19.018 | INFO | BytesWritten : 2571437702 | 20260505 03:36:19.018 | INFO | NumEvents : 0 | 20260505 03:36:19.021 | INFO | Populating RTMetrics report | 20260505 03:36:19.021 | INFO | RTMetrics report populated | 20260505 03:36:19.021 | INFO | Opening tmp RTMetrics json file | 20260505 03:36:19.022 | INFO | Serializing RTMetrics to json | 20260505 03:36:19.023 | INFO | Serialization done, swapping tmp file now... | 20260505 03:36:19.023 | INFO | RTMetrics done | 20260505 03:36:21.113 | INFO | Analyzing chunk frames = [28672,29184) | 20260505 03:36:21.113 | INFO | MemInfo: { | 20260505 03:36:21.113 | INFO | "gpu" : " 17 GB", | 20260505 03:36:21.113 | INFO | "huge" : "", | 20260505 03:36:21.113 | INFO | "huge_shared" : " 27 GB", | 20260505 03:36:21.113 | INFO | "malloc" : " 15 GB", | 20260505 03:36:21.113 | INFO | "pinned" : " 29 GB" | 20260505 03:36:21.113 | INFO | } | 20260505 03:36:21.113 | INFO | Perf report for the last 5.13928 seconds: | 20260505 03:36:21.113 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:36:21.113 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:21.113 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:21.113 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:21.113 | INFO | ANALYSIS : 58.4%, 3.0, 91.1ms, 0 | 20260505 03:36:21.113 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:21.113 | INFO | RT_METRICS : 41.1%, 1.0, 21.6ms, 0 | 20260505 03:36:21.113 | INFO | PRE_HQ : 55.7%, 1.9, 55.7ms, 0 | 20260505 03:36:21.113 | INFO | BAZWRITER : 9.7%, 1.0, 5.0ms, 0 | 20260505 03:36:21.113 | INFO | BAZ_SAMPLING : 0.1%, 1.0, 0.0ms, 0 | 20260505 03:36:21.113 | INFO | Uploaded 36GiB to GPU in 1.64572s (21.875GiB/s) | 20260505 03:36:21.113 | INFO | Downloaded 38GiB to GPU in 1.70521s (22.2847GiB/s) | 20260505 03:36:26.252 | INFO | Analyzing chunk frames = [29184,29696) | 20260505 03:36:26.252 | INFO | MemInfo: { | 20260505 03:36:26.252 | INFO | "gpu" : " 17 GB", | 20260505 03:36:26.252 | INFO | "huge" : "", | 20260505 03:36:26.252 | INFO | "huge_shared" : " 27 GB", | 20260505 03:36:26.252 | INFO | "malloc" : " 15 GB", | 20260505 03:36:26.252 | INFO | "pinned" : " 29 GB" | 20260505 03:36:26.252 | INFO | } | 20260505 03:36:26.253 | INFO | Perf report for the last 5.13944 seconds: | 20260505 03:36:26.253 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:36:26.253 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:26.253 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:26.253 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:26.253 | INFO | ANALYSIS : 58.2%, 3.0, 90.7ms, 0 | 20260505 03:36:26.253 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:26.253 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:26.253 | INFO | PRE_HQ : 36.1%, 1.0, 19.2ms, 0 | 20260505 03:36:26.253 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:26.253 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:26.253 | INFO | Uploaded 36GiB to GPU in 1.59834s (22.5234GiB/s) | 20260505 03:36:26.253 | INFO | Downloaded 34GiB to GPU in 1.51387s (22.4589GiB/s) | 20260505 03:36:31.392 | INFO | Analyzing chunk frames = [29696,30208) | 20260505 03:36:31.392 | INFO | MemInfo: { | 20260505 03:36:31.392 | INFO | "gpu" : " 17 GB", | 20260505 03:36:31.392 | INFO | "huge" : "", | 20260505 03:36:31.392 | INFO | "huge_shared" : " 27 GB", | 20260505 03:36:31.392 | INFO | "malloc" : " 15 GB", | 20260505 03:36:31.392 | INFO | "pinned" : " 29 GB" | 20260505 03:36:31.392 | INFO | } | 20260505 03:36:31.392 | INFO | Perf report for the last 5.13923 seconds: | 20260505 03:36:31.392 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:36:31.392 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:31.392 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:31.392 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:31.392 | INFO | ANALYSIS : 57.7%, 3.0, 89.6ms, 0 | 20260505 03:36:31.392 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:31.392 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:31.392 | INFO | PRE_HQ : 32.9%, 1.0, 17.1ms, 0 | 20260505 03:36:31.392 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:31.392 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:31.392 | INFO | Uploaded 36GiB to GPU in 1.60115s (22.4838GiB/s) | 20260505 03:36:31.392 | INFO | Downloaded 34GiB to GPU in 1.51938s (22.3775GiB/s) | 20260505 03:36:36.531 | INFO | Analyzing chunk frames = [30208,30720) | 20260505 03:36:36.531 | INFO | MemInfo: { | 20260505 03:36:36.531 | INFO | "gpu" : " 17 GB", | 20260505 03:36:36.531 | INFO | "huge" : "", | 20260505 03:36:36.531 | INFO | "huge_shared" : " 27 GB", | 20260505 03:36:36.531 | INFO | "malloc" : " 15 GB", | 20260505 03:36:36.531 | INFO | "pinned" : " 29 GB" | 20260505 03:36:36.531 | INFO | } | 20260505 03:36:36.531 | INFO | Perf report for the last 5.13955 seconds: | 20260505 03:36:36.531 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:36:36.531 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:36.531 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:36.531 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:36.531 | INFO | ANALYSIS : 57.3%, 3.0, 88.9ms, 0 | 20260505 03:36:36.531 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:36.531 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:36.531 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260505 03:36:36.531 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:36.531 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:36.532 | INFO | Uploaded 36GiB to GPU in 1.59998s (22.5003GiB/s) | 20260505 03:36:36.532 | INFO | Downloaded 34GiB to GPU in 1.51787s (22.3999GiB/s) | 20260505 03:36:41.670 | INFO | Analyzing chunk frames = [30720,31232) | 20260505 03:36:41.670 | INFO | MemInfo: { | 20260505 03:36:41.670 | INFO | "gpu" : " 17 GB", | 20260505 03:36:41.670 | INFO | "huge" : "", | 20260505 03:36:41.670 | INFO | "huge_shared" : " 27 GB", | 20260505 03:36:41.670 | INFO | "malloc" : " 15 GB", | 20260505 03:36:41.670 | INFO | "pinned" : " 29 GB" | 20260505 03:36:41.670 | INFO | } | 20260505 03:36:41.670 | INFO | Perf report for the last 5.13915 seconds: | 20260505 03:36:41.670 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:36:41.670 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:41.671 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:41.671 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:41.671 | INFO | ANALYSIS : 57.3%, 3.0, 89.3ms, 0 | 20260505 03:36:41.671 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:41.671 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:41.671 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260505 03:36:41.671 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:41.671 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:41.671 | INFO | Uploaded 36GiB to GPU in 1.59863s (22.5192GiB/s) | 20260505 03:36:41.671 | INFO | Downloaded 34GiB to GPU in 1.51436s (22.4517GiB/s) | 20260505 03:36:46.810 | INFO | Analyzing chunk frames = [31232,31744) | 20260505 03:36:46.810 | INFO | MemInfo: { | 20260505 03:36:46.810 | INFO | "gpu" : " 17 GB", | 20260505 03:36:46.810 | INFO | "huge" : "", | 20260505 03:36:46.810 | INFO | "huge_shared" : " 27 GB", | 20260505 03:36:46.810 | INFO | "malloc" : " 15 GB", | 20260505 03:36:46.810 | INFO | "pinned" : " 29 GB" | 20260505 03:36:46.810 | INFO | } | 20260505 03:36:46.810 | INFO | Perf report for the last 5.13933 seconds: | 20260505 03:36:46.810 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:36:46.810 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:46.810 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:46.810 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:46.810 | INFO | ANALYSIS : 57.5%, 3.0, 89.8ms, 0 | 20260505 03:36:46.810 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:46.810 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:46.810 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 03:36:46.810 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:46.810 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:46.810 | INFO | Uploaded 36GiB to GPU in 1.60006s (22.4992GiB/s) | 20260505 03:36:46.810 | INFO | Downloaded 34GiB to GPU in 1.51652s (22.4198GiB/s) | 20260505 03:36:51.949 | INFO | Analyzing chunk frames = [31744,32256) | 20260505 03:36:51.949 | INFO | MemInfo: { | 20260505 03:36:51.949 | INFO | "gpu" : " 17 GB", | 20260505 03:36:51.949 | INFO | "huge" : "", | 20260505 03:36:51.949 | INFO | "huge_shared" : " 27 GB", | 20260505 03:36:51.949 | INFO | "malloc" : " 15 GB", | 20260505 03:36:51.949 | INFO | "pinned" : " 29 GB" | 20260505 03:36:51.949 | INFO | } | 20260505 03:36:51.949 | INFO | Perf report for the last 5.13944 seconds: | 20260505 03:36:51.949 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:36:51.949 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:51.949 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:51.949 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:51.949 | INFO | ANALYSIS : 57.8%, 3.0, 90.1ms, 0 | 20260505 03:36:51.949 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:51.949 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:51.949 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 03:36:51.949 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:51.949 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:51.950 | INFO | Uploaded 36GiB to GPU in 1.59896s (22.5146GiB/s) | 20260505 03:36:51.950 | INFO | Downloaded 34GiB to GPU in 1.51655s (22.4193GiB/s) | 20260505 03:36:57.088 | INFO | Analyzing chunk frames = [32256,32768) | 20260505 03:36:57.088 | INFO | MemInfo: { | 20260505 03:36:57.088 | INFO | "gpu" : " 17 GB", | 20260505 03:36:57.088 | INFO | "huge" : "", | 20260505 03:36:57.088 | INFO | "huge_shared" : " 27 GB", | 20260505 03:36:57.089 | INFO | "malloc" : " 15 GB", | 20260505 03:36:57.089 | INFO | "pinned" : " 29 GB" | 20260505 03:36:57.089 | INFO | } | 20260505 03:36:57.089 | INFO | Perf report for the last 5.13935 seconds: | 20260505 03:36:57.089 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:36:57.089 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:57.089 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:57.089 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:57.089 | INFO | ANALYSIS : 57.9%, 3.0, 90.3ms, 0 | 20260505 03:36:57.089 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:57.089 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:57.089 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260505 03:36:57.089 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:57.089 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:36:57.089 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 03:36:57.089 | INFO | Total mem usage: 14.7 GiB | 20260505 03:36:57.089 | INFO | All Time High Water Marks sum: 14.7 GiB (Savings: 0.0524%) | 20260505 03:36:57.089 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -0.57%) | 20260505 03:36:57.089 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 03:36:57.089 | INFO | High Water Breakdown: | 20260505 03:36:57.089 | INFO | 54.6% : 7.96 GiB ( 99% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 03:36:57.089 | INFO | 45.3% : 6.6 GiB ( 99.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 03:36:57.089 | INFO | Uploaded 36GiB to GPU in 1.59814s (22.5261GiB/s) | 20260505 03:36:57.089 | INFO | 0.128% : 19.1 MiB ( 100% Of Peak) : SampledBazData | 20260505 03:36:57.089 | INFO | Downloaded 34GiB to GPU in 1.51586s (22.4294GiB/s) | 20260505 03:36:57.089 | INFO | ------------------------------------------ | 20260505 03:36:57.089 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:36:57.090 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 03:36:57.090 | INFO | Total mem usage: 26.5 GiB | 20260505 03:36:57.090 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260505 03:36:57.090 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 03:36:57.090 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260505 03:36:57.090 | INFO | High Water Breakdown: | 20260505 03:36:57.090 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 03:36:57.090 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 03:36:57.090 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 03:36:57.090 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 03:36:57.090 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 03:36:57.090 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 03:36:57.090 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 03:36:57.090 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 03:36:57.090 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 03:36:57.090 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 03:36:57.090 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 03:36:57.090 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 03:36:57.090 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 03:36:57.090 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 03:36:57.090 | INFO | ------------------------------------------ | 20260505 03:36:57.090 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:36:57.090 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 03:36:57.090 | INFO | Total mem usage: 25.6 GiB | 20260505 03:36:57.090 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 03:36:57.090 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 03:36:57.090 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 03:36:57.090 | INFO | High Water Breakdown: | 20260505 03:36:57.090 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 03:36:57.090 | INFO | ------------------------------------------ | 20260505 03:36:57.090 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:36:57.090 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 03:36:57.090 | INFO | Total mem usage: 15.7 GiB | 20260505 03:36:57.090 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260505 03:36:57.090 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 03:36:57.090 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260505 03:36:57.090 | INFO | High Water Breakdown: | 20260505 03:36:57.090 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 03:36:57.090 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 03:36:57.090 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 03:36:57.090 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 03:36:57.090 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 03:36:57.090 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 03:36:57.090 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 03:36:57.090 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 03:36:57.090 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 03:36:57.090 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 03:36:57.090 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 03:36:57.090 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 03:36:57.090 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 03:36:57.090 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 03:36:57.090 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 03:36:57.090 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 03:36:57.090 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 03:36:57.090 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 03:36:57.090 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 03:36:57.090 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 03:36:57.090 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 03:36:57.090 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 03:36:57.090 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 03:36:57.090 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 03:36:57.090 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 03:36:57.090 | INFO | ------------------------------------------ | 20260505 03:36:57.090 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:36:57.090 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 03:36:57.090 | INFO | PerfReport: Gathered 1536 samples. | 20260505 03:36:57.090 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 03:36:57.090 | INFO | PerfReport: Full Execution -- 3.05e+01 / -nan, 1.28e+01 / 0.00e+00 | 20260505 03:36:57.090 | INFO | PerfReport: 0.28%, AggregateStats -- 8.59e-02 / 0.00e+00, 3.64e-03 / 0.00e+00 | 20260505 03:36:57.090 | INFO | PerfReport: 10.80%, Baseline -- 3.30e+00 / 0.00e+00, 1.61e-02 / 0.00e+00 | 20260505 03:36:57.090 | INFO | PerfReport: 2.98%, Histogram -- 9.11e-01 / 0.00e+00, 5.09e-02 / 0.00e+00 | 20260505 03:36:57.090 | INFO | PerfReport: 16.18%, DME -- 3.95e+01 / 0.00e+00, 2.90e+00 / 0.00e+00 | 20260505 03:36:57.090 | INFO | PerfReport: 56.42%, FrameLabeling -- 1.72e+01 / 0.00e+00, 4.57e-02 / 0.00e+00 | 20260505 03:36:57.090 | INFO | PerfReport: 7.64%, PulseAccumulating -- 2.33e+00 / 0.00e+00, 1.54e-01 / 0.00e+00 | 20260505 03:36:57.090 | INFO | PerfReport: 3.99%, Metrics -- 1.22e+00 / 0.00e+00, 2.04e-01 / 0.00e+00 | 20260505 03:36:57.090 | INFO | PerfReport: 1.70%, PulseSerializing -- 5.20e-01 / 0.00e+00, 8.61e-02 / 0.00e+00 | 20260505 03:36:57.090 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 03:36:57.090 | INFO | PerfReport: Gathered 1536 samples. | 20260505 03:36:57.090 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 03:36:57.090 | INFO | PerfReport: Full Execution -- 9.17e+01 / -nan, 1.93e+01 / 0.00e+00 | 20260505 03:36:57.091 | INFO | PerfReport: 18.23%, Upload -- 1.67e+01 / 0.00e+00, 3.86e-01 / 0.00e+00 | 20260505 03:36:57.091 | INFO | PerfReport: 17.49%, Download -- 1.60e+01 / 0.00e+00, 7.14e-01 / 0.00e+00 | 20260505 03:36:57.091 | INFO | PerfReport: 33.29%, Compute -- 3.05e+01 / 0.00e+00, 1.28e+01 / 0.00e+00 | 20260505 03:36:57.091 | INFO | PerfReport: 0.55%, LockUpload -- 5.04e-01 / 0.00e+00, 3.60e+00 / 0.00e+00 | 20260505 03:36:57.091 | INFO | PerfReport: 0.00%, LockDownload -- 3.85e-04 / 0.00e+00, 1.39e-04 / 0.00e+00 | 20260505 03:36:57.091 | INFO | PerfReport: 30.42%, LockCompute -- 2.79e+01 / 0.00e+00, 1.74e+01 / 0.00e+00 | 20260505 03:36:57.091 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.66e-02 / 0.00e+00 | 20260505 03:36:57.182 | INFO | Starting a round of Baz IO | 20260505 03:37:00.139 | INFO | Finished a round of Baz IO: Wrote 3.469 KiB in 2.957 seconds | 20260505 03:37:00.139 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 100.00% / 0.00% / 0.00% | 20260505 03:37:00.139 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 12.7452% / 0% / 0% | 20260505 03:37:00.139 | INFO | Baz IO overall: | 20260505 03:37:00.139 | INFO | BytesWritten : 2571441254 | 20260505 03:37:00.139 | INFO | NumEvents : 0 | 20260505 03:37:00.141 | INFO | Populating RTMetrics report | 20260505 03:37:00.141 | INFO | RTMetrics report populated | 20260505 03:37:00.141 | INFO | Opening tmp RTMetrics json file | 20260505 03:37:00.142 | INFO | Serializing RTMetrics to json | 20260505 03:37:00.143 | INFO | Serialization done, swapping tmp file now... | 20260505 03:37:00.144 | INFO | RTMetrics done | 20260505 03:37:02.228 | INFO | Analyzing chunk frames = [32768,33280) | 20260505 03:37:02.228 | INFO | MemInfo: { | 20260505 03:37:02.228 | INFO | "gpu" : " 17 GB", | 20260505 03:37:02.228 | INFO | "huge" : "", | 20260505 03:37:02.228 | INFO | "huge_shared" : " 27 GB", | 20260505 03:37:02.228 | INFO | "malloc" : " 15 GB", | 20260505 03:37:02.228 | INFO | "pinned" : " 29 GB" | 20260505 03:37:02.228 | INFO | } | 20260505 03:37:02.228 | INFO | Perf report for the last 5.13907 seconds: | 20260505 03:37:02.228 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:37:02.228 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:02.228 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:02.228 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:02.228 | INFO | ANALYSIS : 58.5%, 3.0, 91.3ms, 0 | 20260505 03:37:02.228 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:02.228 | INFO | RT_METRICS : 41.3%, 1.0, 21.7ms, 0 | 20260505 03:37:02.228 | INFO | PRE_HQ : 55.7%, 1.9, 55.4ms, 0 | 20260505 03:37:02.228 | INFO | BAZWRITER : 9.9%, 1.0, 5.1ms, 0 | 20260505 03:37:02.228 | INFO | BAZ_SAMPLING : 0.1%, 1.0, 0.0ms, 0 | 20260505 03:37:02.228 | INFO | Uploaded 36GiB to GPU in 1.64422s (21.8949GiB/s) | 20260505 03:37:02.228 | INFO | Downloaded 38GiB to GPU in 1.70387s (22.3021GiB/s) | 20260505 03:37:07.367 | INFO | Analyzing chunk frames = [33280,33792) | 20260505 03:37:07.367 | INFO | MemInfo: { | 20260505 03:37:07.367 | INFO | "gpu" : " 17 GB", | 20260505 03:37:07.367 | INFO | "huge" : "", | 20260505 03:37:07.367 | INFO | "huge_shared" : " 27 GB", | 20260505 03:37:07.367 | INFO | "malloc" : " 15 GB", | 20260505 03:37:07.367 | INFO | "pinned" : " 29 GB" | 20260505 03:37:07.367 | INFO | } | 20260505 03:37:07.367 | INFO | Perf report for the last 5.13963 seconds: | 20260505 03:37:07.367 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:37:07.367 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:07.367 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:07.367 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:07.367 | INFO | ANALYSIS : 58.3%, 3.0, 91.0ms, 0 | 20260505 03:37:07.367 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:07.367 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:07.367 | INFO | PRE_HQ : 35.1%, 1.0, 18.3ms, 0 | 20260505 03:37:07.368 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:07.368 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:07.368 | INFO | Uploaded 36GiB to GPU in 1.59678s (22.5453GiB/s) | 20260505 03:37:07.368 | INFO | Downloaded 34GiB to GPU in 1.5132s (22.469GiB/s) | 20260505 03:37:12.506 | INFO | Analyzing chunk frames = [33792,34304) | 20260505 03:37:12.506 | INFO | MemInfo: { | 20260505 03:37:12.506 | INFO | "gpu" : " 17 GB", | 20260505 03:37:12.506 | INFO | "huge" : "", | 20260505 03:37:12.506 | INFO | "huge_shared" : " 27 GB", | 20260505 03:37:12.506 | INFO | "malloc" : " 15 GB", | 20260505 03:37:12.506 | INFO | "pinned" : " 29 GB" | 20260505 03:37:12.506 | INFO | } | 20260505 03:37:12.507 | INFO | Perf report for the last 5.13914 seconds: | 20260505 03:37:12.507 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:37:12.507 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:12.507 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:12.507 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:12.507 | INFO | ANALYSIS : 57.8%, 3.0, 89.7ms, 0 | 20260505 03:37:12.507 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:12.507 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:12.507 | INFO | PRE_HQ : 32.9%, 1.0, 17.1ms, 0 | 20260505 03:37:12.507 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:12.507 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:12.507 | INFO | Uploaded 36GiB to GPU in 1.6009s (22.4874GiB/s) | 20260505 03:37:12.507 | INFO | Downloaded 34GiB to GPU in 1.51843s (22.3916GiB/s) | 20260505 03:37:17.646 | INFO | Analyzing chunk frames = [34304,34816) | 20260505 03:37:17.646 | INFO | MemInfo: { | 20260505 03:37:17.646 | INFO | "gpu" : " 17 GB", | 20260505 03:37:17.646 | INFO | "huge" : "", | 20260505 03:37:17.646 | INFO | "huge_shared" : " 27 GB", | 20260505 03:37:17.646 | INFO | "malloc" : " 15 GB", | 20260505 03:37:17.646 | INFO | "pinned" : " 29 GB" | 20260505 03:37:17.646 | INFO | } | 20260505 03:37:17.646 | INFO | Perf report for the last 5.13961 seconds: | 20260505 03:37:17.646 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:37:17.646 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:17.646 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:17.646 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:17.646 | INFO | ANALYSIS : 57.5%, 3.0, 89.1ms, 0 | 20260505 03:37:17.646 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:17.646 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:17.646 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260505 03:37:17.646 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:17.646 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:17.647 | INFO | Uploaded 36GiB to GPU in 1.60066s (22.4907GiB/s) | 20260505 03:37:17.647 | INFO | Downloaded 34GiB to GPU in 1.51707s (22.4116GiB/s) | 20260505 03:37:22.785 | INFO | Analyzing chunk frames = [34816,35328) | 20260505 03:37:22.785 | INFO | MemInfo: { | 20260505 03:37:22.785 | INFO | "gpu" : " 17 GB", | 20260505 03:37:22.785 | INFO | "huge" : "", | 20260505 03:37:22.785 | INFO | "huge_shared" : " 27 GB", | 20260505 03:37:22.785 | INFO | "malloc" : " 15 GB", | 20260505 03:37:22.785 | INFO | "pinned" : " 29 GB" | 20260505 03:37:22.785 | INFO | } | 20260505 03:37:22.785 | INFO | Perf report for the last 5.13919 seconds: | 20260505 03:37:22.785 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:37:22.785 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:22.785 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:22.785 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:22.785 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260505 03:37:22.785 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:22.785 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:22.785 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 03:37:22.785 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:22.785 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:22.786 | INFO | Uploaded 36GiB to GPU in 1.59805s (22.5274GiB/s) | 20260505 03:37:22.786 | INFO | Downloaded 34GiB to GPU in 1.514s (22.4571GiB/s) | 20260505 03:37:27.924 | INFO | Analyzing chunk frames = [35328,35840) | 20260505 03:37:27.924 | INFO | MemInfo: { | 20260505 03:37:27.924 | INFO | "gpu" : " 17 GB", | 20260505 03:37:27.924 | INFO | "huge" : "", | 20260505 03:37:27.924 | INFO | "huge_shared" : " 27 GB", | 20260505 03:37:27.924 | INFO | "malloc" : " 15 GB", | 20260505 03:37:27.924 | INFO | "pinned" : " 29 GB" | 20260505 03:37:27.925 | INFO | } | 20260505 03:37:27.925 | INFO | Perf report for the last 5.13922 seconds: | 20260505 03:37:27.925 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:37:27.925 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:27.925 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:27.925 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:27.925 | INFO | ANALYSIS : 57.7%, 3.0, 90.1ms, 0 | 20260505 03:37:27.925 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:27.925 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:27.925 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260505 03:37:27.925 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:27.925 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:27.925 | INFO | Uploaded 36GiB to GPU in 1.5981s (22.5267GiB/s) | 20260505 03:37:27.925 | INFO | Downloaded 34GiB to GPU in 1.51467s (22.4471GiB/s) | 20260505 03:37:33.064 | INFO | Analyzing chunk frames = [35840,36352) | 20260505 03:37:33.064 | INFO | MemInfo: { | 20260505 03:37:33.064 | INFO | "gpu" : " 17 GB", | 20260505 03:37:33.064 | INFO | "huge" : "", | 20260505 03:37:33.064 | INFO | "huge_shared" : " 27 GB", | 20260505 03:37:33.064 | INFO | "malloc" : " 15 GB", | 20260505 03:37:33.064 | INFO | "pinned" : " 29 GB" | 20260505 03:37:33.064 | INFO | } | 20260505 03:37:33.064 | INFO | Perf report for the last 5.13934 seconds: | 20260505 03:37:33.064 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:37:33.064 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:33.064 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:33.064 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:33.064 | INFO | ANALYSIS : 57.9%, 3.0, 90.3ms, 0 | 20260505 03:37:33.064 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:33.064 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:33.064 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260505 03:37:33.064 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:33.064 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:33.064 | INFO | Uploaded 36GiB to GPU in 1.59946s (22.5077GiB/s) | 20260505 03:37:33.064 | INFO | Downloaded 34GiB to GPU in 1.51645s (22.4208GiB/s) | 20260505 03:37:38.203 | INFO | Analyzing chunk frames = [36352,36864) | 20260505 03:37:38.203 | INFO | MemInfo: { | 20260505 03:37:38.203 | INFO | "gpu" : " 17 GB", | 20260505 03:37:38.203 | INFO | "huge" : "", | 20260505 03:37:38.203 | INFO | "huge_shared" : " 27 GB", | 20260505 03:37:38.203 | INFO | "malloc" : " 15 GB", | 20260505 03:37:38.203 | INFO | "pinned" : " 29 GB" | 20260505 03:37:38.203 | INFO | } | 20260505 03:37:38.203 | INFO | Perf report for the last 5.13942 seconds: | 20260505 03:37:38.203 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:37:38.203 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:38.203 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:38.203 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:38.203 | INFO | ANALYSIS : 58.0%, 3.0, 90.5ms, 0 | 20260505 03:37:38.203 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:38.203 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:38.204 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 03:37:38.204 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:38.204 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:38.204 | INFO | Uploaded 36GiB to GPU in 1.59838s (22.5227GiB/s) | 20260505 03:37:38.204 | INFO | Downloaded 34GiB to GPU in 1.51588s (22.4292GiB/s) | 20260505 03:37:38.298 | INFO | Starting a round of Baz IO | 20260505 03:37:41.257 | INFO | Finished a round of Baz IO: Wrote 3.469 KiB in 2.959 seconds | 20260505 03:37:41.257 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 100.00% / 0.00% / 0.00% | 20260505 03:37:41.257 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 13.5208% / 0% / 0% | 20260505 03:37:41.257 | INFO | Baz IO overall: | 20260505 03:37:41.257 | INFO | BytesWritten : 2571444806 | 20260505 03:37:41.257 | INFO | NumEvents : 0 | 20260505 03:37:41.260 | INFO | Populating RTMetrics report | 20260505 03:37:41.260 | INFO | RTMetrics report populated | 20260505 03:37:41.260 | INFO | Opening tmp RTMetrics json file | 20260505 03:37:41.261 | INFO | Serializing RTMetrics to json | 20260505 03:37:41.262 | INFO | Serialization done, swapping tmp file now... | 20260505 03:37:41.262 | INFO | RTMetrics done | 20260505 03:37:43.342 | INFO | Analyzing chunk frames = [36864,37376) | 20260505 03:37:43.342 | INFO | MemInfo: { | 20260505 03:37:43.342 | INFO | "gpu" : " 17 GB", | 20260505 03:37:43.342 | INFO | "huge" : "", | 20260505 03:37:43.343 | INFO | "huge_shared" : " 27 GB", | 20260505 03:37:43.343 | INFO | "malloc" : " 15 GB", | 20260505 03:37:43.343 | INFO | "pinned" : " 29 GB" | 20260505 03:37:43.343 | INFO | } | 20260505 03:37:43.343 | INFO | Perf report for the last 5.13928 seconds: | 20260505 03:37:43.343 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:37:43.343 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:43.343 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:43.343 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:43.343 | INFO | ANALYSIS : 58.6%, 3.0, 91.4ms, 0 | 20260505 03:37:43.343 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:43.343 | INFO | RT_METRICS : 41.3%, 1.0, 21.6ms, 0 | 20260505 03:37:43.343 | INFO | PRE_HQ : 55.8%, 1.9, 55.2ms, 0 | 20260505 03:37:43.343 | INFO | BAZWRITER : 9.9%, 1.0, 5.1ms, 0 | 20260505 03:37:43.343 | INFO | BAZ_SAMPLING : 0.1%, 1.0, 0.0ms, 0 | 20260505 03:37:43.343 | INFO | Uploaded 36GiB to GPU in 1.63944s (21.9588GiB/s) | 20260505 03:37:43.343 | INFO | Downloaded 38GiB to GPU in 1.70013s (22.3512GiB/s) | 20260505 03:37:48.482 | INFO | Analyzing chunk frames = [37376,37888) | 20260505 03:37:48.482 | INFO | MemInfo: { | 20260505 03:37:48.482 | INFO | "gpu" : " 17 GB", | 20260505 03:37:48.482 | INFO | "huge" : "", | 20260505 03:37:48.482 | INFO | "huge_shared" : " 27 GB", | 20260505 03:37:48.482 | INFO | "malloc" : " 15 GB", | 20260505 03:37:48.482 | INFO | "pinned" : " 29 GB" | 20260505 03:37:48.482 | INFO | } | 20260505 03:37:48.482 | INFO | Perf report for the last 5.13941 seconds: | 20260505 03:37:48.482 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:37:48.482 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:48.482 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:48.482 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:48.482 | INFO | ANALYSIS : 58.4%, 3.0, 91.1ms, 0 | 20260505 03:37:48.482 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:48.482 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:48.482 | INFO | PRE_HQ : 34.6%, 1.0, 18.0ms, 0 | 20260505 03:37:48.482 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:48.482 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:48.482 | INFO | Uploaded 36GiB to GPU in 1.59635s (22.5514GiB/s) | 20260505 03:37:48.483 | INFO | Downloaded 34GiB to GPU in 1.51368s (22.4618GiB/s) | 20260505 03:37:53.621 | INFO | Analyzing chunk frames = [37888,38400) | 20260505 03:37:53.621 | INFO | MemInfo: { | 20260505 03:37:53.621 | INFO | "gpu" : " 17 GB", | 20260505 03:37:53.621 | INFO | "huge" : "", | 20260505 03:37:53.621 | INFO | "huge_shared" : " 27 GB", | 20260505 03:37:53.621 | INFO | "malloc" : " 15 GB", | 20260505 03:37:53.621 | INFO | "pinned" : " 29 GB" | 20260505 03:37:53.621 | INFO | } | 20260505 03:37:53.621 | INFO | Perf report for the last 5.13926 seconds: | 20260505 03:37:53.621 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:37:53.621 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:53.621 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:53.621 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:53.621 | INFO | ANALYSIS : 57.8%, 3.0, 89.8ms, 0 | 20260505 03:37:53.621 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:53.621 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:53.621 | INFO | PRE_HQ : 32.8%, 1.0, 17.0ms, 0 | 20260505 03:37:53.621 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:53.621 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:53.622 | INFO | Uploaded 36GiB to GPU in 1.59933s (22.5095GiB/s) | 20260505 03:37:53.622 | INFO | Downloaded 34GiB to GPU in 1.51707s (22.4116GiB/s) | 20260505 03:37:58.760 | INFO | Analyzing chunk frames = [38400,38912) | 20260505 03:37:58.760 | INFO | MemInfo: { | 20260505 03:37:58.760 | INFO | "gpu" : " 17 GB", | 20260505 03:37:58.760 | INFO | "huge" : "", | 20260505 03:37:58.760 | INFO | "huge_shared" : " 27 GB", | 20260505 03:37:58.761 | INFO | "malloc" : " 15 GB", | 20260505 03:37:58.761 | INFO | "pinned" : " 29 GB" | 20260505 03:37:58.761 | INFO | } | 20260505 03:37:58.761 | INFO | Perf report for the last 5.13929 seconds: | 20260505 03:37:58.761 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:37:58.761 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:58.761 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:58.761 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:58.761 | INFO | ANALYSIS : 57.5%, 3.0, 89.2ms, 0 | 20260505 03:37:58.761 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:58.761 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:58.761 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260505 03:37:58.761 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:58.761 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:37:58.761 | INFO | Uploaded 36GiB to GPU in 1.59936s (22.509GiB/s) | 20260505 03:37:58.761 | INFO | Downloaded 34GiB to GPU in 1.51585s (22.4296GiB/s) | 20260505 03:38:03.900 | INFO | Analyzing chunk frames = [38912,39424) | 20260505 03:38:03.900 | INFO | MemInfo: { | 20260505 03:38:03.900 | INFO | "gpu" : " 17 GB", | 20260505 03:38:03.900 | INFO | "huge" : "", | 20260505 03:38:03.900 | INFO | "huge_shared" : " 27 GB", | 20260505 03:38:03.900 | INFO | "malloc" : " 15 GB", | 20260505 03:38:03.900 | INFO | "pinned" : " 29 GB" | 20260505 03:38:03.900 | INFO | } | 20260505 03:38:03.900 | INFO | Perf report for the last 5.13915 seconds: | 20260505 03:38:03.900 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:38:03.900 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:03.900 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:03.900 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:03.900 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260505 03:38:03.900 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:03.900 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:03.900 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260505 03:38:03.900 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:03.900 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:03.900 | INFO | Uploaded 36GiB to GPU in 1.59786s (22.5301GiB/s) | 20260505 03:38:03.900 | INFO | Downloaded 34GiB to GPU in 1.516s (22.4274GiB/s) | 20260505 03:38:09.039 | INFO | Analyzing chunk frames = [39424,39936) | 20260505 03:38:09.039 | INFO | MemInfo: { | 20260505 03:38:09.039 | INFO | "gpu" : " 17 GB", | 20260505 03:38:09.039 | INFO | "huge" : "", | 20260505 03:38:09.039 | INFO | "huge_shared" : " 27 GB", | 20260505 03:38:09.039 | INFO | "malloc" : " 15 GB", | 20260505 03:38:09.039 | INFO | "pinned" : " 29 GB" | 20260505 03:38:09.039 | INFO | } | 20260505 03:38:09.039 | INFO | Perf report for the last 5.13945 seconds: | 20260505 03:38:09.039 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:38:09.039 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:09.039 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:09.039 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:09.039 | INFO | ANALYSIS : 57.7%, 3.0, 90.1ms, 0 | 20260505 03:38:09.039 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:09.039 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:09.039 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260505 03:38:09.039 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:09.039 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:09.040 | INFO | Uploaded 36GiB to GPU in 1.59829s (22.5241GiB/s) | 20260505 03:38:09.040 | INFO | Downloaded 34GiB to GPU in 1.5177s (22.4023GiB/s) | 20260505 03:38:14.179 | INFO | Analyzing chunk frames = [39936,40448) | 20260505 03:38:14.179 | INFO | MemInfo: { | 20260505 03:38:14.179 | INFO | "gpu" : " 17 GB", | 20260505 03:38:14.179 | INFO | "huge" : "", | 20260505 03:38:14.179 | INFO | "huge_shared" : " 27 GB", | 20260505 03:38:14.179 | INFO | "malloc" : " 15 GB", | 20260505 03:38:14.179 | INFO | "pinned" : " 29 GB" | 20260505 03:38:14.179 | INFO | } | 20260505 03:38:14.179 | INFO | Perf report for the last 5.13975 seconds: | 20260505 03:38:14.179 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:38:14.179 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:14.179 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:14.179 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:14.179 | INFO | ANALYSIS : 57.9%, 3.0, 90.3ms, 0 | 20260505 03:38:14.179 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:14.179 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:14.179 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 03:38:14.179 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:14.179 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:14.180 | INFO | Uploaded 36GiB to GPU in 1.59796s (22.5287GiB/s) | 20260505 03:38:14.180 | INFO | Downloaded 34GiB to GPU in 1.51492s (22.4434GiB/s) | 20260505 03:38:19.318 | INFO | Analyzing chunk frames = [40448,40960) | 20260505 03:38:19.318 | INFO | MemInfo: { | 20260505 03:38:19.318 | INFO | "gpu" : " 17 GB", | 20260505 03:38:19.318 | INFO | "huge" : "", | 20260505 03:38:19.318 | INFO | "huge_shared" : " 27 GB", | 20260505 03:38:19.318 | INFO | "malloc" : " 15 GB", | 20260505 03:38:19.318 | INFO | "pinned" : " 29 GB" | 20260505 03:38:19.318 | INFO | } | 20260505 03:38:19.318 | INFO | Perf report for the last 5.13905 seconds: | 20260505 03:38:19.318 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:38:19.318 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:19.318 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:19.318 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:19.318 | INFO | ANALYSIS : 58.0%, 3.0, 90.5ms, 0 | 20260505 03:38:19.318 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:19.318 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:19.318 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260505 03:38:19.318 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:19.318 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:19.318 | INFO | Uploaded 36GiB to GPU in 1.5982s (22.5254GiB/s) | 20260505 03:38:19.318 | INFO | Downloaded 34GiB to GPU in 1.51552s (22.4346GiB/s) | 20260505 03:38:19.319 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 03:38:19.319 | INFO | Total mem usage: 14.7 GiB | 20260505 03:38:19.319 | INFO | All Time High Water Marks sum: 14.7 GiB (Savings: 0.0524%) | 20260505 03:38:19.319 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -0.577%) | 20260505 03:38:19.319 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 03:38:19.319 | INFO | High Water Breakdown: | 20260505 03:38:19.319 | INFO | 54.6% : 7.96 GiB ( 99% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 03:38:19.319 | INFO | 45.3% : 6.6 GiB ( 99.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 03:38:19.319 | INFO | 0.128% : 19.1 MiB ( 100% Of Peak) : SampledBazData | 20260505 03:38:19.319 | INFO | ------------------------------------------ | 20260505 03:38:19.319 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:38:19.319 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 03:38:19.319 | INFO | Total mem usage: 26.5 GiB | 20260505 03:38:19.319 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260505 03:38:19.319 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 03:38:19.319 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260505 03:38:19.319 | INFO | High Water Breakdown: | 20260505 03:38:19.319 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 03:38:19.319 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 03:38:19.319 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 03:38:19.319 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 03:38:19.319 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 03:38:19.319 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 03:38:19.319 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 03:38:19.319 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 03:38:19.319 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 03:38:19.319 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 03:38:19.319 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 03:38:19.319 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 03:38:19.319 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 03:38:19.319 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 03:38:19.319 | INFO | ------------------------------------------ | 20260505 03:38:19.319 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:38:19.319 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 03:38:19.319 | INFO | Total mem usage: 25.6 GiB | 20260505 03:38:19.319 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 03:38:19.319 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 03:38:19.319 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 03:38:19.319 | INFO | High Water Breakdown: | 20260505 03:38:19.319 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 03:38:19.319 | INFO | ------------------------------------------ | 20260505 03:38:19.319 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:38:19.319 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 03:38:19.319 | INFO | Total mem usage: 15.7 GiB | 20260505 03:38:19.319 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260505 03:38:19.319 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 03:38:19.319 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260505 03:38:19.319 | INFO | High Water Breakdown: | 20260505 03:38:19.319 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 03:38:19.319 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 03:38:19.319 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 03:38:19.319 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 03:38:19.320 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 03:38:19.320 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 03:38:19.320 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 03:38:19.320 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 03:38:19.320 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 03:38:19.320 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 03:38:19.320 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 03:38:19.320 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 03:38:19.320 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 03:38:19.320 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 03:38:19.320 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 03:38:19.320 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 03:38:19.320 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 03:38:19.320 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 03:38:19.320 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 03:38:19.320 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 03:38:19.320 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 03:38:19.320 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 03:38:19.320 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 03:38:19.320 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 03:38:19.320 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 03:38:19.320 | INFO | ------------------------------------------ | 20260505 03:38:19.320 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:38:19.320 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 03:38:19.320 | INFO | PerfReport: Gathered 1536 samples. | 20260505 03:38:19.320 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 03:38:19.320 | INFO | PerfReport: Full Execution -- 3.07e+01 / -nan, 1.29e+01 / 0.00e+00 | 20260505 03:38:19.320 | INFO | PerfReport: 0.28%, AggregateStats -- 8.63e-02 / 0.00e+00, 4.09e-03 / 0.00e+00 | 20260505 03:38:19.320 | INFO | PerfReport: 10.76%, Baseline -- 3.30e+00 / 0.00e+00, 1.65e-02 / 0.00e+00 | 20260505 03:38:19.320 | INFO | PerfReport: 2.98%, Histogram -- 9.14e-01 / 0.00e+00, 5.16e-02 / 0.00e+00 | 20260505 03:38:19.320 | INFO | PerfReport: 16.17%, DME -- 3.97e+01 / 0.00e+00, 2.89e+00 / 0.00e+00 | 20260505 03:38:19.320 | INFO | PerfReport: 56.24%, FrameLabeling -- 1.72e+01 / 0.00e+00, 4.62e-02 / 0.00e+00 | 20260505 03:38:19.320 | INFO | PerfReport: 7.71%, PulseAccumulating -- 2.36e+00 / 0.00e+00, 1.58e-01 / 0.00e+00 | 20260505 03:38:19.320 | INFO | PerfReport: 4.11%, Metrics -- 1.26e+00 / 0.00e+00, 2.11e-01 / 0.00e+00 | 20260505 03:38:19.320 | INFO | PerfReport: 1.74%, PulseSerializing -- 5.34e-01 / 0.00e+00, 8.94e-02 / 0.00e+00 | 20260505 03:38:19.320 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 03:38:19.320 | INFO | PerfReport: Gathered 1536 samples. | 20260505 03:38:19.320 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 03:38:19.320 | INFO | PerfReport: Full Execution -- 9.21e+01 / -nan, 1.95e+01 / 0.00e+00 | 20260505 03:38:19.320 | INFO | PerfReport: 18.14%, Upload -- 1.67e+01 / 0.00e+00, 3.75e-01 / 0.00e+00 | 20260505 03:38:19.320 | INFO | PerfReport: 17.41%, Download -- 1.60e+01 / 0.00e+00, 7.03e-01 / 0.00e+00 | 20260505 03:38:19.320 | INFO | PerfReport: 33.29%, Compute -- 3.07e+01 / 0.00e+00, 1.29e+01 / 0.00e+00 | 20260505 03:38:19.320 | INFO | PerfReport: 0.55%, LockUpload -- 5.04e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260505 03:38:19.320 | INFO | PerfReport: 0.00%, LockDownload -- 3.88e-04 / 0.00e+00, 9.96e-05 / 0.00e+00 | 20260505 03:38:19.320 | INFO | PerfReport: 30.59%, LockCompute -- 2.82e+01 / 0.00e+00, 1.75e+01 / 0.00e+00 | 20260505 03:38:19.320 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.40e-02 / 0.00e+00 | 20260505 03:38:19.413 | INFO | Starting a round of Baz IO | 20260505 03:38:22.374 | INFO | Finished a round of Baz IO: Wrote 3.469 KiB in 2.961 seconds | 20260505 03:38:22.374 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 100.00% / 0.00% / 0.00% | 20260505 03:38:22.374 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 13.8951% / 0% / 0% | 20260505 03:38:22.374 | INFO | Baz IO overall: | 20260505 03:38:22.374 | INFO | BytesWritten : 2571448358 | 20260505 03:38:22.374 | INFO | NumEvents : 0 | 20260505 03:38:22.376 | INFO | Populating RTMetrics report | 20260505 03:38:22.376 | INFO | RTMetrics report populated | 20260505 03:38:22.376 | INFO | Opening tmp RTMetrics json file | 20260505 03:38:22.377 | INFO | Serializing RTMetrics to json | 20260505 03:38:22.378 | INFO | Serialization done, swapping tmp file now... | 20260505 03:38:22.379 | INFO | RTMetrics done | 20260505 03:38:24.457 | INFO | Analyzing chunk frames = [40960,41472) | 20260505 03:38:24.457 | INFO | MemInfo: { | 20260505 03:38:24.457 | INFO | "gpu" : " 17 GB", | 20260505 03:38:24.457 | INFO | "huge" : "", | 20260505 03:38:24.457 | INFO | "huge_shared" : " 27 GB", | 20260505 03:38:24.457 | INFO | "malloc" : " 15 GB", | 20260505 03:38:24.457 | INFO | "pinned" : " 29 GB" | 20260505 03:38:24.457 | INFO | } | 20260505 03:38:24.457 | INFO | Perf report for the last 5.13928 seconds: | 20260505 03:38:24.457 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:38:24.457 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:24.457 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:24.457 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:24.457 | INFO | ANALYSIS : 58.6%, 3.0, 91.4ms, 0 | 20260505 03:38:24.457 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:24.458 | INFO | RT_METRICS : 41.3%, 1.0, 21.7ms, 0 | 20260505 03:38:24.458 | INFO | PRE_HQ : 56.1%, 1.9, 56.5ms, 0 | 20260505 03:38:24.458 | INFO | BAZWRITER : 9.8%, 1.0, 5.0ms, 0 | 20260505 03:38:24.458 | INFO | BAZ_SAMPLING : 0.1%, 1.0, 0.0ms, 0 | 20260505 03:38:24.458 | INFO | Uploaded 36GiB to GPU in 1.64282s (21.9136GiB/s) | 20260505 03:38:24.458 | INFO | Downloaded 38GiB to GPU in 1.7019s (22.328GiB/s) | 20260505 03:38:29.597 | INFO | Analyzing chunk frames = [41472,41984) | 20260505 03:38:29.597 | INFO | MemInfo: { | 20260505 03:38:29.597 | INFO | "gpu" : " 17 GB", | 20260505 03:38:29.597 | INFO | "huge" : "", | 20260505 03:38:29.597 | INFO | "huge_shared" : " 27 GB", | 20260505 03:38:29.597 | INFO | "malloc" : " 15 GB", | 20260505 03:38:29.597 | INFO | "pinned" : " 29 GB" | 20260505 03:38:29.597 | INFO | } | 20260505 03:38:29.597 | INFO | Perf report for the last 5.13962 seconds: | 20260505 03:38:29.597 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:38:29.597 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:29.597 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:29.597 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:29.597 | INFO | ANALYSIS : 58.5%, 3.0, 91.2ms, 0 | 20260505 03:38:29.597 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:29.597 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:29.597 | INFO | PRE_HQ : 34.4%, 1.0, 17.9ms, 0 | 20260505 03:38:29.597 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:29.597 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:29.598 | INFO | Uploaded 36GiB to GPU in 1.5959s (22.5578GiB/s) | 20260505 03:38:29.598 | INFO | Downloaded 34GiB to GPU in 1.5124s (22.4808GiB/s) | 20260505 03:38:34.736 | INFO | Analyzing chunk frames = [41984,42496) | 20260505 03:38:34.736 | INFO | MemInfo: { | 20260505 03:38:34.736 | INFO | "gpu" : " 17 GB", | 20260505 03:38:34.736 | INFO | "huge" : "", | 20260505 03:38:34.736 | INFO | "huge_shared" : " 27 GB", | 20260505 03:38:34.736 | INFO | "malloc" : " 15 GB", | 20260505 03:38:34.736 | INFO | "pinned" : " 29 GB" | 20260505 03:38:34.736 | INFO | } | 20260505 03:38:34.736 | INFO | Perf report for the last 5.13898 seconds: | 20260505 03:38:34.736 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:38:34.736 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:34.736 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:34.736 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:34.736 | INFO | ANALYSIS : 57.9%, 3.0, 89.9ms, 0 | 20260505 03:38:34.736 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:34.736 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:34.736 | INFO | PRE_HQ : 32.7%, 1.0, 16.9ms, 0 | 20260505 03:38:34.736 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:34.736 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:34.736 | INFO | Uploaded 36GiB to GPU in 1.59954s (22.5064GiB/s) | 20260505 03:38:34.736 | INFO | Downloaded 34GiB to GPU in 1.51627s (22.4234GiB/s) | 20260505 03:38:39.875 | INFO | Analyzing chunk frames = [42496,43008) | 20260505 03:38:39.875 | INFO | MemInfo: { | 20260505 03:38:39.875 | INFO | "gpu" : " 17 GB", | 20260505 03:38:39.875 | INFO | "huge" : "", | 20260505 03:38:39.875 | INFO | "huge_shared" : " 27 GB", | 20260505 03:38:39.875 | INFO | "malloc" : " 15 GB", | 20260505 03:38:39.875 | INFO | "pinned" : " 29 GB" | 20260505 03:38:39.875 | INFO | } | 20260505 03:38:39.875 | INFO | Perf report for the last 5.13919 seconds: | 20260505 03:38:39.875 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:38:39.875 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:39.875 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:39.875 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:39.875 | INFO | ANALYSIS : 57.5%, 3.0, 89.2ms, 0 | 20260505 03:38:39.875 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:39.875 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:39.875 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 03:38:39.876 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:39.876 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:39.876 | INFO | Uploaded 36GiB to GPU in 1.59869s (22.5185GiB/s) | 20260505 03:38:39.876 | INFO | Downloaded 34GiB to GPU in 1.51544s (22.4357GiB/s) | 20260505 03:38:45.015 | INFO | Analyzing chunk frames = [43008,43520) | 20260505 03:38:45.015 | INFO | MemInfo: { | 20260505 03:38:45.015 | INFO | "gpu" : " 17 GB", | 20260505 03:38:45.015 | INFO | "huge" : "", | 20260505 03:38:45.015 | INFO | "huge_shared" : " 27 GB", | 20260505 03:38:45.015 | INFO | "malloc" : " 15 GB", | 20260505 03:38:45.015 | INFO | "pinned" : " 29 GB" | 20260505 03:38:45.015 | INFO | } | 20260505 03:38:45.015 | INFO | Perf report for the last 5.13962 seconds: | 20260505 03:38:45.015 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:38:45.015 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:45.015 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:45.015 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:45.015 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260505 03:38:45.015 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:45.015 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:45.015 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 03:38:45.015 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:45.015 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:45.015 | INFO | Uploaded 36GiB to GPU in 1.59747s (22.5357GiB/s) | 20260505 03:38:45.015 | INFO | Downloaded 34GiB to GPU in 1.51373s (22.4611GiB/s) | 20260505 03:38:50.154 | INFO | Analyzing chunk frames = [43520,44032) | 20260505 03:38:50.154 | INFO | MemInfo: { | 20260505 03:38:50.154 | INFO | "gpu" : " 17 GB", | 20260505 03:38:50.154 | INFO | "huge" : "", | 20260505 03:38:50.154 | INFO | "huge_shared" : " 27 GB", | 20260505 03:38:50.154 | INFO | "malloc" : " 15 GB", | 20260505 03:38:50.154 | INFO | "pinned" : " 29 GB" | 20260505 03:38:50.154 | INFO | } | 20260505 03:38:50.154 | INFO | Perf report for the last 5.13935 seconds: | 20260505 03:38:50.154 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:38:50.154 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:50.154 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:50.154 | INFO | SAVE_TRACE : 0.1%, 1.0, 0.1ms, 0 | 20260505 03:38:50.154 | INFO | ANALYSIS : 57.8%, 3.0, 90.1ms, 0 | 20260505 03:38:50.154 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:50.154 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:50.154 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 03:38:50.154 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:50.154 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:50.155 | INFO | Uploaded 36GiB to GPU in 1.59836s (22.5231GiB/s) | 20260505 03:38:50.155 | INFO | Downloaded 34GiB to GPU in 1.51511s (22.4406GiB/s) | 20260505 03:38:55.293 | INFO | Analyzing chunk frames = [44032,44544) | 20260505 03:38:55.293 | INFO | MemInfo: { | 20260505 03:38:55.293 | INFO | "gpu" : " 17 GB", | 20260505 03:38:55.293 | INFO | "huge" : "", | 20260505 03:38:55.293 | INFO | "huge_shared" : " 27 GB", | 20260505 03:38:55.293 | INFO | "malloc" : " 15 GB", | 20260505 03:38:55.293 | INFO | "pinned" : " 29 GB" | 20260505 03:38:55.293 | INFO | } | 20260505 03:38:55.293 | INFO | Perf report for the last 5.13914 seconds: | 20260505 03:38:55.294 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:38:55.294 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:55.294 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:55.294 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:55.294 | INFO | ANALYSIS : 58.0%, 3.0, 90.5ms, 0 | 20260505 03:38:55.294 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:55.294 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:55.294 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 03:38:55.294 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:55.294 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:38:55.294 | INFO | Uploaded 36GiB to GPU in 1.5976s (22.5338GiB/s) | 20260505 03:38:55.294 | INFO | Downloaded 34GiB to GPU in 1.51534s (22.4372GiB/s) | 20260505 03:39:00.432 | INFO | Analyzing chunk frames = [44544,45056) | 20260505 03:39:00.432 | INFO | MemInfo: { | 20260505 03:39:00.432 | INFO | "gpu" : " 17 GB", | 20260505 03:39:00.433 | INFO | "huge" : "", | 20260505 03:39:00.433 | INFO | "huge_shared" : " 27 GB", | 20260505 03:39:00.433 | INFO | "malloc" : " 15 GB", | 20260505 03:39:00.433 | INFO | "pinned" : " 29 GB" | 20260505 03:39:00.433 | INFO | } | 20260505 03:39:00.433 | INFO | Perf report for the last 5.13938 seconds: | 20260505 03:39:00.433 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:39:00.433 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:00.433 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:00.433 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:00.433 | INFO | ANALYSIS : 58.0%, 3.0, 90.5ms, 0 | 20260505 03:39:00.433 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:00.433 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:00.433 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 03:39:00.433 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:00.433 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:00.433 | INFO | Uploaded 36GiB to GPU in 1.59843s (22.5221GiB/s) | 20260505 03:39:00.433 | INFO | Downloaded 34GiB to GPU in 1.51526s (22.4384GiB/s) | 20260505 03:39:00.527 | INFO | Starting a round of Baz IO | 20260505 03:39:03.494 | INFO | Finished a round of Baz IO: Wrote 3.469 KiB in 2.967 seconds | 20260505 03:39:03.494 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 100.00% / 0.00% / 0.00% | 20260505 03:39:03.494 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 14.2978% / 0% / 0% | 20260505 03:39:03.494 | INFO | Baz IO overall: | 20260505 03:39:03.494 | INFO | BytesWritten : 2571451910 | 20260505 03:39:03.494 | INFO | NumEvents : 0 | 20260505 03:39:03.495 | INFO | Populating RTMetrics report | 20260505 03:39:03.496 | INFO | RTMetrics report populated | 20260505 03:39:03.496 | INFO | Opening tmp RTMetrics json file | 20260505 03:39:03.496 | INFO | Serializing RTMetrics to json | 20260505 03:39:03.497 | INFO | Serialization done, swapping tmp file now... | 20260505 03:39:03.498 | INFO | RTMetrics done | 20260505 03:39:05.572 | INFO | Analyzing chunk frames = [45056,45568) | 20260505 03:39:05.572 | INFO | MemInfo: { | 20260505 03:39:05.572 | INFO | "gpu" : " 17 GB", | 20260505 03:39:05.572 | INFO | "huge" : "", | 20260505 03:39:05.572 | INFO | "huge_shared" : " 27 GB", | 20260505 03:39:05.572 | INFO | "malloc" : " 15 GB", | 20260505 03:39:05.572 | INFO | "pinned" : " 29 GB" | 20260505 03:39:05.572 | INFO | } | 20260505 03:39:05.572 | INFO | Perf report for the last 5.13926 seconds: | 20260505 03:39:05.572 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:39:05.572 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:05.572 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:05.572 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:05.572 | INFO | ANALYSIS : 58.7%, 3.0, 91.6ms, 0 | 20260505 03:39:05.572 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:05.572 | INFO | RT_METRICS : 41.3%, 1.0, 21.7ms, 0 | 20260505 03:39:05.572 | INFO | PRE_HQ : 56.2%, 1.9, 56.2ms, 0 | 20260505 03:39:05.572 | INFO | BAZWRITER : 9.9%, 1.0, 5.1ms, 0 | 20260505 03:39:05.572 | INFO | BAZ_SAMPLING : 0.1%, 1.0, 0.0ms, 0 | 20260505 03:39:05.572 | INFO | Uploaded 36GiB to GPU in 1.63889s (21.9661GiB/s) | 20260505 03:39:05.572 | INFO | Downloaded 38GiB to GPU in 1.69922s (22.3632GiB/s) | 20260505 03:39:10.711 | INFO | Analyzing chunk frames = [45568,46080) | 20260505 03:39:10.711 | INFO | MemInfo: { | 20260505 03:39:10.711 | INFO | "gpu" : " 17 GB", | 20260505 03:39:10.711 | INFO | "huge" : "", | 20260505 03:39:10.711 | INFO | "huge_shared" : " 27 GB", | 20260505 03:39:10.711 | INFO | "malloc" : " 15 GB", | 20260505 03:39:10.711 | INFO | "pinned" : " 29 GB" | 20260505 03:39:10.711 | INFO | } | 20260505 03:39:10.711 | INFO | Perf report for the last 5.13941 seconds: | 20260505 03:39:10.711 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:39:10.711 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:10.711 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:10.711 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:10.712 | INFO | ANALYSIS : 58.5%, 3.0, 91.2ms, 0 | 20260505 03:39:10.712 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:10.712 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:10.712 | INFO | PRE_HQ : 34.0%, 1.0, 17.8ms, 0 | 20260505 03:39:10.712 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:10.712 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:10.712 | INFO | Uploaded 36GiB to GPU in 1.5955s (22.5635GiB/s) | 20260505 03:39:10.712 | INFO | Downloaded 34GiB to GPU in 1.51219s (22.4839GiB/s) | 20260505 03:39:15.851 | INFO | Analyzing chunk frames = [46080,46592) | 20260505 03:39:15.851 | INFO | MemInfo: { | 20260505 03:39:15.851 | INFO | "gpu" : " 17 GB", | 20260505 03:39:15.851 | INFO | "huge" : "", | 20260505 03:39:15.851 | INFO | "huge_shared" : " 27 GB", | 20260505 03:39:15.851 | INFO | "malloc" : " 15 GB", | 20260505 03:39:15.851 | INFO | "pinned" : " 29 GB" | 20260505 03:39:15.851 | INFO | } | 20260505 03:39:15.851 | INFO | Perf report for the last 5.13942 seconds: | 20260505 03:39:15.851 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:39:15.851 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:15.851 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:15.851 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:15.851 | INFO | ANALYSIS : 57.9%, 3.0, 90.0ms, 0 | 20260505 03:39:15.851 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:15.851 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:15.851 | INFO | PRE_HQ : 32.8%, 1.0, 17.0ms, 0 | 20260505 03:39:15.851 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:15.851 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:15.851 | INFO | Uploaded 36GiB to GPU in 1.59935s (22.5092GiB/s) | 20260505 03:39:15.851 | INFO | Downloaded 34GiB to GPU in 1.51584s (22.4298GiB/s) | 20260505 03:39:20.990 | INFO | Analyzing chunk frames = [46592,47104) | 20260505 03:39:20.990 | INFO | MemInfo: { | 20260505 03:39:20.990 | INFO | "gpu" : " 17 GB", | 20260505 03:39:20.990 | INFO | "huge" : "", | 20260505 03:39:20.990 | INFO | "huge_shared" : " 27 GB", | 20260505 03:39:20.990 | INFO | "malloc" : " 15 GB", | 20260505 03:39:20.990 | INFO | "pinned" : " 29 GB" | 20260505 03:39:20.990 | INFO | } | 20260505 03:39:20.990 | INFO | Perf report for the last 5.13934 seconds: | 20260505 03:39:20.990 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:39:20.990 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:20.990 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:20.990 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:20.990 | INFO | ANALYSIS : 57.6%, 3.0, 89.2ms, 0 | 20260505 03:39:20.990 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:20.990 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:20.990 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260505 03:39:20.990 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:20.990 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:20.991 | INFO | Uploaded 36GiB to GPU in 1.59872s (22.518GiB/s) | 20260505 03:39:20.991 | INFO | Downloaded 34GiB to GPU in 1.51578s (22.4308GiB/s) | 20260505 03:39:26.129 | INFO | Analyzing chunk frames = [47104,47616) | 20260505 03:39:26.129 | INFO | MemInfo: { | 20260505 03:39:26.129 | INFO | "gpu" : " 17 GB", | 20260505 03:39:26.129 | INFO | "huge" : "", | 20260505 03:39:26.129 | INFO | "huge_shared" : " 27 GB", | 20260505 03:39:26.129 | INFO | "malloc" : " 15 GB", | 20260505 03:39:26.129 | INFO | "pinned" : " 29 GB" | 20260505 03:39:26.130 | INFO | } | 20260505 03:39:26.130 | INFO | Perf report for the last 5.13936 seconds: | 20260505 03:39:26.130 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:39:26.130 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:26.130 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:26.130 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:26.130 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260505 03:39:26.130 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:26.130 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:26.130 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260505 03:39:26.130 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:26.130 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:26.130 | INFO | Uploaded 36GiB to GPU in 1.59651s (22.5492GiB/s) | 20260505 03:39:26.130 | INFO | Downloaded 34GiB to GPU in 1.51254s (22.4787GiB/s) | 20260505 03:39:31.269 | INFO | Analyzing chunk frames = [47616,48128) | 20260505 03:39:31.269 | INFO | MemInfo: { | 20260505 03:39:31.269 | INFO | "gpu" : " 17 GB", | 20260505 03:39:31.269 | INFO | "huge" : "", | 20260505 03:39:31.269 | INFO | "huge_shared" : " 27 GB", | 20260505 03:39:31.269 | INFO | "malloc" : " 15 GB", | 20260505 03:39:31.269 | INFO | "pinned" : " 29 GB" | 20260505 03:39:31.269 | INFO | } | 20260505 03:39:31.269 | INFO | Perf report for the last 5.1393 seconds: | 20260505 03:39:31.269 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:39:31.269 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:31.269 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:31.269 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:31.269 | INFO | ANALYSIS : 57.8%, 3.0, 90.1ms, 0 | 20260505 03:39:31.269 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:31.269 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:31.269 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 03:39:31.269 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:31.269 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:31.269 | INFO | Uploaded 36GiB to GPU in 1.598s (22.5281GiB/s) | 20260505 03:39:31.269 | INFO | Downloaded 34GiB to GPU in 1.51451s (22.4495GiB/s) | 20260505 03:39:36.408 | INFO | Analyzing chunk frames = [48128,48640) | 20260505 03:39:36.408 | INFO | MemInfo: { | 20260505 03:39:36.408 | INFO | "gpu" : " 17 GB", | 20260505 03:39:36.408 | INFO | "huge" : "", | 20260505 03:39:36.408 | INFO | "huge_shared" : " 27 GB", | 20260505 03:39:36.408 | INFO | "malloc" : " 15 GB", | 20260505 03:39:36.408 | INFO | "pinned" : " 29 GB" | 20260505 03:39:36.408 | INFO | } | 20260505 03:39:36.408 | INFO | Perf report for the last 5.13913 seconds: | 20260505 03:39:36.408 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:39:36.408 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:36.408 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:36.408 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:36.408 | INFO | ANALYSIS : 58.0%, 3.0, 90.5ms, 0 | 20260505 03:39:36.408 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:36.408 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:36.408 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 03:39:36.408 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:36.408 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:36.409 | INFO | Uploaded 36GiB to GPU in 1.59656s (22.5485GiB/s) | 20260505 03:39:36.409 | INFO | Downloaded 34GiB to GPU in 1.5143s (22.4525GiB/s) | 20260505 03:39:41.547 | INFO | Analyzing chunk frames = [48640,49152) | 20260505 03:39:41.547 | INFO | MemInfo: { | 20260505 03:39:41.547 | INFO | "gpu" : " 17 GB", | 20260505 03:39:41.547 | INFO | "huge" : "", | 20260505 03:39:41.547 | INFO | "huge_shared" : " 27 GB", | 20260505 03:39:41.547 | INFO | "malloc" : " 15 GB", | 20260505 03:39:41.547 | INFO | "pinned" : " 29 GB" | 20260505 03:39:41.547 | INFO | } | 20260505 03:39:41.548 | INFO | Perf report for the last 5.1395 seconds: | 20260505 03:39:41.548 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:39:41.548 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:41.548 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:41.548 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:41.548 | INFO | ANALYSIS : 58.1%, 3.0, 90.6ms, 0 | 20260505 03:39:41.548 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:41.548 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:41.548 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 03:39:41.548 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:41.548 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:41.548 | INFO | Uploaded 36GiB to GPU in 1.59785s (22.5303GiB/s) | 20260505 03:39:41.548 | INFO | Downloaded 34GiB to GPU in 1.51432s (22.4523GiB/s) | 20260505 03:39:41.548 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 03:39:41.548 | INFO | Total mem usage: 14.7 GiB | 20260505 03:39:41.548 | INFO | All Time High Water Marks sum: 14.7 GiB (Savings: 0.0524%) | 20260505 03:39:41.548 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -0.685%) | 20260505 03:39:41.548 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 03:39:41.548 | INFO | High Water Breakdown: | 20260505 03:39:41.548 | INFO | 54.6% : 7.96 GiB ( 99% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 03:39:41.548 | INFO | 45.2% : 6.59 GiB ( 99.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 03:39:41.548 | INFO | 0.128% : 19.1 MiB ( 100% Of Peak) : SampledBazData | 20260505 03:39:41.548 | INFO | ------------------------------------------ | 20260505 03:39:41.548 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:39:41.548 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 03:39:41.548 | INFO | Total mem usage: 26.5 GiB | 20260505 03:39:41.548 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260505 03:39:41.548 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260505 03:39:41.548 | INFO | Unused Cache: 1.6 GiB (Waste: 6.05%) | 20260505 03:39:41.548 | INFO | High Water Breakdown: | 20260505 03:39:41.548 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 03:39:41.548 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 03:39:41.548 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 03:39:41.548 | INFO | 1.6% : 418 MiB ( 99.5% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 03:39:41.548 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 03:39:41.548 | INFO | 0.42% : 110 MiB ( 75% Of Peak) : /BasecallingMetrics.h:66 | 20260505 03:39:41.548 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 03:39:41.548 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 03:39:41.548 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 03:39:41.548 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 03:39:41.548 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 03:39:41.548 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 03:39:41.548 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 03:39:41.548 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 03:39:41.548 | INFO | ------------------------------------------ | 20260505 03:39:41.548 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:39:41.548 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 03:39:41.548 | INFO | Total mem usage: 25.6 GiB | 20260505 03:39:41.548 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 03:39:41.548 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 03:39:41.548 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 03:39:41.548 | INFO | High Water Breakdown: | 20260505 03:39:41.548 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 03:39:41.548 | INFO | ------------------------------------------ | 20260505 03:39:41.548 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:39:41.548 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 03:39:41.548 | INFO | Total mem usage: 15.7 GiB | 20260505 03:39:41.548 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260505 03:39:41.548 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 03:39:41.548 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260505 03:39:41.548 | INFO | High Water Breakdown: | 20260505 03:39:41.548 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 03:39:41.548 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 03:39:41.548 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 03:39:41.548 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 03:39:41.548 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 03:39:41.548 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 03:39:41.548 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 03:39:41.548 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 03:39:41.548 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 03:39:41.548 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 03:39:41.548 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 03:39:41.548 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 03:39:41.548 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 03:39:41.548 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 03:39:41.548 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 03:39:41.548 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 03:39:41.548 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 03:39:41.548 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 03:39:41.548 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 03:39:41.548 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 03:39:41.548 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 03:39:41.548 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 03:39:41.548 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 03:39:41.548 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 03:39:41.548 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 03:39:41.548 | INFO | ------------------------------------------ | 20260505 03:39:41.548 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:39:41.548 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 03:39:41.548 | INFO | PerfReport: Gathered 1536 samples. | 20260505 03:39:41.548 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 03:39:41.548 | INFO | PerfReport: Full Execution -- 3.07e+01 / -nan, 1.29e+01 / 0.00e+00 | 20260505 03:39:41.548 | INFO | PerfReport: 0.28%, AggregateStats -- 8.61e-02 / 0.00e+00, 4.47e-03 / 0.00e+00 | 20260505 03:39:41.548 | INFO | PerfReport: 10.74%, Baseline -- 3.30e+00 / 0.00e+00, 1.66e-02 / 0.00e+00 | 20260505 03:39:41.548 | INFO | PerfReport: 2.98%, Histogram -- 9.16e-01 / 0.00e+00, 5.16e-02 / 0.00e+00 | 20260505 03:39:41.548 | INFO | PerfReport: 16.15%, DME -- 3.97e+01 / 0.00e+00, 2.99e+00 / 0.00e+00 | 20260505 03:39:41.548 | INFO | PerfReport: 56.15%, FrameLabeling -- 1.72e+01 / 0.00e+00, 4.66e-02 / 0.00e+00 | 20260505 03:39:41.548 | INFO | PerfReport: 7.75%, PulseAccumulating -- 2.38e+00 / 0.00e+00, 1.62e-01 / 0.00e+00 | 20260505 03:39:41.548 | INFO | PerfReport: 4.17%, Metrics -- 1.28e+00 / 0.00e+00, 2.13e-01 / 0.00e+00 | 20260505 03:39:41.548 | INFO | PerfReport: 1.76%, PulseSerializing -- 5.42e-01 / 0.00e+00, 8.97e-02 / 0.00e+00 | 20260505 03:39:41.548 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 03:39:41.549 | INFO | PerfReport: Gathered 1536 samples. | 20260505 03:39:41.549 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 03:39:41.549 | INFO | PerfReport: Full Execution -- 9.22e+01 / -nan, 1.94e+01 / 0.00e+00 | 20260505 03:39:41.549 | INFO | PerfReport: 18.11%, Upload -- 1.67e+01 / 0.00e+00, 3.71e-01 / 0.00e+00 | 20260505 03:39:41.549 | INFO | PerfReport: 17.37%, Download -- 1.60e+01 / 0.00e+00, 6.98e-01 / 0.00e+00 | 20260505 03:39:41.549 | INFO | PerfReport: 33.29%, Compute -- 3.07e+01 / 0.00e+00, 1.29e+01 / 0.00e+00 | 20260505 03:39:41.549 | INFO | PerfReport: 0.54%, LockUpload -- 5.02e-01 / 0.00e+00, 3.60e+00 / 0.00e+00 | 20260505 03:39:41.549 | INFO | PerfReport: 0.00%, LockDownload -- 3.99e-04 / 0.00e+00, 1.00e-04 / 0.00e+00 | 20260505 03:39:41.549 | INFO | PerfReport: 30.67%, LockCompute -- 2.83e+01 / 0.00e+00, 1.73e+01 / 0.00e+00 | 20260505 03:39:41.549 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.68e-02 / 0.00e+00 | 20260505 03:39:41.642 | INFO | Starting a round of Baz IO | 20260505 03:39:44.614 | INFO | Finished a round of Baz IO: Wrote 3.469 KiB in 2.972 seconds | 20260505 03:39:44.614 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 100.00% / 0.00% / 0.00% | 20260505 03:39:44.614 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 14.6434% / 0% / 0% | 20260505 03:39:44.614 | INFO | Baz IO overall: | 20260505 03:39:44.614 | INFO | BytesWritten : 2571455462 | 20260505 03:39:44.614 | INFO | NumEvents : 0 | 20260505 03:39:44.614 | INFO | Populating RTMetrics report | 20260505 03:39:44.615 | INFO | RTMetrics report populated | 20260505 03:39:44.615 | INFO | Opening tmp RTMetrics json file | 20260505 03:39:44.615 | INFO | Serializing RTMetrics to json | 20260505 03:39:44.617 | INFO | Serialization done, swapping tmp file now... | 20260505 03:39:44.617 | INFO | RTMetrics done | 20260505 03:39:46.687 | INFO | Analyzing chunk frames = [49152,49664) | 20260505 03:39:46.687 | INFO | MemInfo: { | 20260505 03:39:46.687 | INFO | "gpu" : " 17 GB", | 20260505 03:39:46.687 | INFO | "huge" : "", | 20260505 03:39:46.687 | INFO | "huge_shared" : " 27 GB", | 20260505 03:39:46.687 | INFO | "malloc" : " 15 GB", | 20260505 03:39:46.687 | INFO | "pinned" : " 29 GB" | 20260505 03:39:46.687 | INFO | } | 20260505 03:39:46.687 | INFO | Perf report for the last 5.1393 seconds: | 20260505 03:39:46.687 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:39:46.687 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:46.687 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:46.687 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:46.687 | INFO | ANALYSIS : 58.8%, 3.0, 91.7ms, 0 | 20260505 03:39:46.687 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:46.687 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260505 03:39:46.687 | INFO | PRE_HQ : 56.4%, 1.9, 56.5ms, 0 | 20260505 03:39:46.687 | INFO | BAZWRITER : 10.0%, 1.0, 5.1ms, 0 | 20260505 03:39:46.687 | INFO | BAZ_SAMPLING : 0.1%, 1.0, 0.0ms, 0 | 20260505 03:39:46.687 | INFO | Uploaded 36GiB to GPU in 1.63915s (21.9626GiB/s) | 20260505 03:39:46.687 | INFO | Downloaded 38GiB to GPU in 1.69881s (22.3687GiB/s) | 20260505 03:39:51.826 | INFO | Analyzing chunk frames = [49664,50176) | 20260505 03:39:51.826 | INFO | MemInfo: { | 20260505 03:39:51.826 | INFO | "gpu" : " 17 GB", | 20260505 03:39:51.826 | INFO | "huge" : "", | 20260505 03:39:51.826 | INFO | "huge_shared" : " 27 GB", | 20260505 03:39:51.826 | INFO | "malloc" : " 15 GB", | 20260505 03:39:51.826 | INFO | "pinned" : " 29 GB" | 20260505 03:39:51.827 | INFO | } | 20260505 03:39:51.827 | INFO | Perf report for the last 5.13977 seconds: | 20260505 03:39:51.827 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:39:51.827 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:51.827 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:51.827 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:51.827 | INFO | ANALYSIS : 58.5%, 3.0, 91.3ms, 0 | 20260505 03:39:51.827 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:51.827 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:51.827 | INFO | PRE_HQ : 33.6%, 1.0, 17.5ms, 0 | 20260505 03:39:51.827 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:51.827 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:51.827 | INFO | Uploaded 36GiB to GPU in 1.59577s (22.5597GiB/s) | 20260505 03:39:51.827 | INFO | Downloaded 34GiB to GPU in 1.5119s (22.4883GiB/s) | 20260505 03:39:56.965 | INFO | Analyzing chunk frames = [50176,50688) | 20260505 03:39:56.965 | INFO | MemInfo: { | 20260505 03:39:56.965 | INFO | "gpu" : " 17 GB", | 20260505 03:39:56.965 | INFO | "huge" : "", | 20260505 03:39:56.965 | INFO | "huge_shared" : " 27 GB", | 20260505 03:39:56.965 | INFO | "malloc" : " 15 GB", | 20260505 03:39:56.966 | INFO | "pinned" : " 29 GB" | 20260505 03:39:56.966 | INFO | } | 20260505 03:39:56.966 | INFO | Perf report for the last 5.13902 seconds: | 20260505 03:39:56.966 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:39:56.966 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:56.966 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:56.966 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:56.966 | INFO | ANALYSIS : 58.0%, 3.0, 90.1ms, 0 | 20260505 03:39:56.966 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:56.966 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:56.966 | INFO | PRE_HQ : 32.7%, 1.0, 17.0ms, 0 | 20260505 03:39:56.966 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:56.966 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:39:56.966 | INFO | Uploaded 36GiB to GPU in 1.59848s (22.5215GiB/s) | 20260505 03:39:56.966 | INFO | Downloaded 34GiB to GPU in 1.51504s (22.4416GiB/s) | 20260505 03:40:02.105 | INFO | Analyzing chunk frames = [50688,51200) | 20260505 03:40:02.105 | INFO | MemInfo: { | 20260505 03:40:02.105 | INFO | "gpu" : " 17 GB", | 20260505 03:40:02.105 | INFO | "huge" : "", | 20260505 03:40:02.105 | INFO | "huge_shared" : " 27 GB", | 20260505 03:40:02.105 | INFO | "malloc" : " 15 GB", | 20260505 03:40:02.105 | INFO | "pinned" : " 29 GB" | 20260505 03:40:02.105 | INFO | } | 20260505 03:40:02.105 | INFO | Perf report for the last 5.13925 seconds: | 20260505 03:40:02.105 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:40:02.105 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:02.105 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:02.105 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:02.105 | INFO | ANALYSIS : 57.6%, 3.0, 89.4ms, 0 | 20260505 03:40:02.105 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:02.105 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:02.105 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260505 03:40:02.105 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:02.105 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:02.105 | INFO | Uploaded 36GiB to GPU in 1.59894s (22.5149GiB/s) | 20260505 03:40:02.105 | INFO | Downloaded 34GiB to GPU in 1.51618s (22.4248GiB/s) | 20260505 03:40:07.244 | INFO | Analyzing chunk frames = [51200,51712) | 20260505 03:40:07.244 | INFO | MemInfo: { | 20260505 03:40:07.244 | INFO | "gpu" : " 17 GB", | 20260505 03:40:07.244 | INFO | "huge" : "", | 20260505 03:40:07.244 | INFO | "huge_shared" : " 27 GB", | 20260505 03:40:07.244 | INFO | "malloc" : " 15 GB", | 20260505 03:40:07.244 | INFO | "pinned" : " 29 GB" | 20260505 03:40:07.244 | INFO | } | 20260505 03:40:07.244 | INFO | Perf report for the last 5.13942 seconds: | 20260505 03:40:07.244 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:40:07.244 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:07.244 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:07.244 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:07.244 | INFO | ANALYSIS : 57.6%, 3.0, 89.8ms, 0 | 20260505 03:40:07.244 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:07.245 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:07.245 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260505 03:40:07.245 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:07.245 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:07.245 | INFO | Uploaded 36GiB to GPU in 1.59639s (22.5509GiB/s) | 20260505 03:40:07.245 | INFO | Downloaded 34GiB to GPU in 1.51244s (22.4803GiB/s) | 20260505 03:40:12.383 | INFO | Analyzing chunk frames = [51712,52224) | 20260505 03:40:12.383 | INFO | MemInfo: { | 20260505 03:40:12.383 | INFO | "gpu" : " 17 GB", | 20260505 03:40:12.383 | INFO | "huge" : "", | 20260505 03:40:12.383 | INFO | "huge_shared" : " 27 GB", | 20260505 03:40:12.384 | INFO | "malloc" : " 15 GB", | 20260505 03:40:12.384 | INFO | "pinned" : " 29 GB" | 20260505 03:40:12.384 | INFO | } | 20260505 03:40:12.384 | INFO | Perf report for the last 5.13926 seconds: | 20260505 03:40:12.384 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:40:12.384 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:12.384 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:12.384 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:12.384 | INFO | ANALYSIS : 57.8%, 3.0, 90.2ms, 0 | 20260505 03:40:12.384 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:12.384 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:12.384 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260505 03:40:12.384 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:12.384 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:12.384 | INFO | Uploaded 36GiB to GPU in 1.59784s (22.5304GiB/s) | 20260505 03:40:12.384 | INFO | Downloaded 34GiB to GPU in 1.51376s (22.4606GiB/s) | 20260505 03:40:17.523 | INFO | Analyzing chunk frames = [52224,52736) | 20260505 03:40:17.523 | INFO | MemInfo: { | 20260505 03:40:17.523 | INFO | "gpu" : " 17 GB", | 20260505 03:40:17.523 | INFO | "huge" : "", | 20260505 03:40:17.523 | INFO | "huge_shared" : " 27 GB", | 20260505 03:40:17.523 | INFO | "malloc" : " 15 GB", | 20260505 03:40:17.523 | INFO | "pinned" : " 29 GB" | 20260505 03:40:17.523 | INFO | } | 20260505 03:40:17.523 | INFO | Perf report for the last 5.1395 seconds: | 20260505 03:40:17.523 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:40:17.523 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:17.523 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:17.523 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:17.523 | INFO | ANALYSIS : 58.1%, 3.0, 90.6ms, 0 | 20260505 03:40:17.523 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:17.523 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:17.523 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 03:40:17.523 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:17.523 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:17.524 | INFO | Uploaded 36GiB to GPU in 1.59759s (22.534GiB/s) | 20260505 03:40:17.524 | INFO | Downloaded 34GiB to GPU in 1.51498s (22.4425GiB/s) | 20260505 03:40:22.662 | INFO | Analyzing chunk frames = [52736,53248) | 20260505 03:40:22.662 | INFO | MemInfo: { | 20260505 03:40:22.662 | INFO | "gpu" : " 17 GB", | 20260505 03:40:22.662 | INFO | "huge" : "", | 20260505 03:40:22.662 | INFO | "huge_shared" : " 27 GB", | 20260505 03:40:22.662 | INFO | "malloc" : " 15 GB", | 20260505 03:40:22.662 | INFO | "pinned" : " 29 GB" | 20260505 03:40:22.662 | INFO | } | 20260505 03:40:22.662 | INFO | Perf report for the last 5.13913 seconds: | 20260505 03:40:22.662 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:40:22.662 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:22.662 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:22.662 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:22.662 | INFO | ANALYSIS : 58.2%, 3.0, 90.7ms, 0 | 20260505 03:40:22.662 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:22.662 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:22.662 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 03:40:22.662 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:22.663 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:22.663 | INFO | Uploaded 36GiB to GPU in 1.59731s (22.5379GiB/s) | 20260505 03:40:22.663 | INFO | Downloaded 34GiB to GPU in 1.51466s (22.4472GiB/s) | 20260505 03:40:22.762 | INFO | Starting a round of Baz IO | 20260505 03:40:25.728 | INFO | Populating RTMetrics report | 20260505 03:40:25.729 | INFO | RTMetrics report populated | 20260505 03:40:25.729 | INFO | Opening tmp RTMetrics json file | 20260505 03:40:25.729 | INFO | Serializing RTMetrics to json | 20260505 03:40:25.731 | INFO | Serialization done, swapping tmp file now... | 20260505 03:40:25.731 | INFO | RTMetrics done | 20260505 03:40:25.805 | INFO | Finished a round of Baz IO: Wrote 906.6 MiB in 3.042 seconds | 20260505 03:40:25.805 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 84.13% / 15.87% / 0.00% | 20260505 03:40:25.805 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 14.8967% / 0.100458% / 0% | 20260505 03:40:25.805 | INFO | Baz IO overall: | 20260505 03:40:25.805 | INFO | BytesWritten : 3522100340 | 20260505 03:40:25.805 | INFO | NumEvents : 348782540 | 20260505 03:40:27.802 | INFO | Analyzing chunk frames = [53248,53760) | 20260505 03:40:27.802 | INFO | MemInfo: { | 20260505 03:40:27.802 | INFO | "gpu" : " 17 GB", | 20260505 03:40:27.802 | INFO | "huge" : "", | 20260505 03:40:27.802 | INFO | "huge_shared" : " 27 GB", | 20260505 03:40:27.802 | INFO | "malloc" : " 15 GB", | 20260505 03:40:27.802 | INFO | "pinned" : " 29 GB" | 20260505 03:40:27.802 | INFO | } | 20260505 03:40:27.802 | INFO | Perf report for the last 5.13943 seconds: | 20260505 03:40:27.802 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:40:27.802 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:27.802 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:27.802 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:27.802 | INFO | ANALYSIS : 58.7%, 3.0, 91.7ms, 0 | 20260505 03:40:27.802 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:27.802 | INFO | RT_METRICS : 41.5%, 1.0, 21.8ms, 0 | 20260505 03:40:27.802 | INFO | PRE_HQ : 58.4%, 2.1, 62.6ms, 0 | 20260505 03:40:27.802 | INFO | BAZWRITER : 58.8%, 2.4, 73.9ms, 0 | 20260505 03:40:27.802 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260505 03:40:27.802 | INFO | Uploaded 36GiB to GPU in 1.6341s (22.0305GiB/s) | 20260505 03:40:27.802 | INFO | Downloaded 38GiB to GPU in 1.69528s (22.4152GiB/s) | 20260505 03:40:32.941 | INFO | Analyzing chunk frames = [53760,54272) | 20260505 03:40:32.941 | INFO | MemInfo: { | 20260505 03:40:32.941 | INFO | "gpu" : " 17 GB", | 20260505 03:40:32.941 | INFO | "huge" : "", | 20260505 03:40:32.941 | INFO | "huge_shared" : " 27 GB", | 20260505 03:40:32.941 | INFO | "malloc" : " 15 GB", | 20260505 03:40:32.941 | INFO | "pinned" : " 29 GB" | 20260505 03:40:32.941 | INFO | } | 20260505 03:40:32.941 | INFO | Perf report for the last 5.1393 seconds: | 20260505 03:40:32.941 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:40:32.941 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:32.941 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:32.941 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:32.941 | INFO | ANALYSIS : 58.6%, 3.0, 91.4ms, 0 | 20260505 03:40:32.941 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:32.941 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:32.941 | INFO | PRE_HQ : 33.6%, 1.0, 17.4ms, 0 | 20260505 03:40:32.941 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:32.941 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:32.942 | INFO | Uploaded 36GiB to GPU in 1.59562s (22.5618GiB/s) | 20260505 03:40:32.942 | INFO | Downloaded 34GiB to GPU in 1.51242s (22.4806GiB/s) | 20260505 03:40:38.080 | INFO | Analyzing chunk frames = [54272,54784) | 20260505 03:40:38.080 | INFO | MemInfo: { | 20260505 03:40:38.080 | INFO | "gpu" : " 17 GB", | 20260505 03:40:38.080 | INFO | "huge" : "", | 20260505 03:40:38.080 | INFO | "huge_shared" : " 27 GB", | 20260505 03:40:38.080 | INFO | "malloc" : " 15 GB", | 20260505 03:40:38.080 | INFO | "pinned" : " 29 GB" | 20260505 03:40:38.081 | INFO | } | 20260505 03:40:38.081 | INFO | Perf report for the last 5.13946 seconds: | 20260505 03:40:38.081 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:40:38.081 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:38.081 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:38.081 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:38.081 | INFO | ANALYSIS : 58.0%, 3.0, 90.1ms, 0 | 20260505 03:40:38.081 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:38.081 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:38.081 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260505 03:40:38.081 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:38.081 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:38.081 | INFO | Uploaded 36GiB to GPU in 1.59887s (22.5159GiB/s) | 20260505 03:40:38.081 | INFO | Downloaded 34GiB to GPU in 1.51562s (22.433GiB/s) | 20260505 03:40:43.220 | INFO | Analyzing chunk frames = [54784,55296) | 20260505 03:40:43.220 | INFO | MemInfo: { | 20260505 03:40:43.220 | INFO | "gpu" : " 17 GB", | 20260505 03:40:43.220 | INFO | "huge" : "", | 20260505 03:40:43.220 | INFO | "huge_shared" : " 27 GB", | 20260505 03:40:43.220 | INFO | "malloc" : " 15 GB", | 20260505 03:40:43.220 | INFO | "pinned" : " 29 GB" | 20260505 03:40:43.220 | INFO | } | 20260505 03:40:43.220 | INFO | Perf report for the last 5.13921 seconds: | 20260505 03:40:43.220 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:40:43.220 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:43.220 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:43.220 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:43.220 | INFO | ANALYSIS : 57.7%, 3.0, 89.5ms, 0 | 20260505 03:40:43.220 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:43.220 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:43.220 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260505 03:40:43.220 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:43.220 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:43.220 | INFO | Uploaded 36GiB to GPU in 1.59824s (22.5247GiB/s) | 20260505 03:40:43.220 | INFO | Downloaded 34GiB to GPU in 1.51449s (22.4498GiB/s) | 20260505 03:40:48.359 | INFO | Analyzing chunk frames = [55296,55808) | 20260505 03:40:48.359 | INFO | MemInfo: { | 20260505 03:40:48.359 | INFO | "gpu" : " 17 GB", | 20260505 03:40:48.359 | INFO | "huge" : "", | 20260505 03:40:48.359 | INFO | "huge_shared" : " 27 GB", | 20260505 03:40:48.359 | INFO | "malloc" : " 15 GB", | 20260505 03:40:48.359 | INFO | "pinned" : " 29 GB" | 20260505 03:40:48.359 | INFO | } | 20260505 03:40:48.359 | INFO | Perf report for the last 5.13941 seconds: | 20260505 03:40:48.359 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:40:48.359 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:48.359 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:48.359 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:48.359 | INFO | ANALYSIS : 57.6%, 3.0, 89.8ms, 0 | 20260505 03:40:48.359 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:48.359 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:48.359 | INFO | PRE_HQ : 32.6%, 1.0, 16.9ms, 0 | 20260505 03:40:48.359 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:48.359 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:48.360 | INFO | Uploaded 36GiB to GPU in 1.59711s (22.5407GiB/s) | 20260505 03:40:48.360 | INFO | Downloaded 34GiB to GPU in 1.51306s (22.4711GiB/s) | 20260505 03:40:53.498 | INFO | Analyzing chunk frames = [55808,56320) | 20260505 03:40:53.498 | INFO | MemInfo: { | 20260505 03:40:53.498 | INFO | "gpu" : " 17 GB", | 20260505 03:40:53.498 | INFO | "huge" : "", | 20260505 03:40:53.498 | INFO | "huge_shared" : " 27 GB", | 20260505 03:40:53.498 | INFO | "malloc" : " 15 GB", | 20260505 03:40:53.498 | INFO | "pinned" : " 29 GB" | 20260505 03:40:53.498 | INFO | } | 20260505 03:40:53.498 | INFO | Perf report for the last 5.13927 seconds: | 20260505 03:40:53.499 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:40:53.499 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:53.499 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:53.499 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:53.499 | INFO | ANALYSIS : 57.8%, 3.0, 90.2ms, 0 | 20260505 03:40:53.499 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:53.499 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:53.499 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260505 03:40:53.499 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:53.499 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:53.499 | INFO | Uploaded 36GiB to GPU in 1.59669s (22.5466GiB/s) | 20260505 03:40:53.499 | INFO | Downloaded 34GiB to GPU in 1.51279s (22.475GiB/s) | 20260505 03:40:58.638 | INFO | Analyzing chunk frames = [56320,56832) | 20260505 03:40:58.638 | INFO | MemInfo: { | 20260505 03:40:58.638 | INFO | "gpu" : " 17 GB", | 20260505 03:40:58.638 | INFO | "huge" : "", | 20260505 03:40:58.638 | INFO | "huge_shared" : " 27 GB", | 20260505 03:40:58.638 | INFO | "malloc" : " 15 GB", | 20260505 03:40:58.638 | INFO | "pinned" : " 29 GB" | 20260505 03:40:58.638 | INFO | } | 20260505 03:40:58.638 | INFO | Perf report for the last 5.13944 seconds: | 20260505 03:40:58.638 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:40:58.638 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:58.638 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:58.638 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:58.638 | INFO | ANALYSIS : 58.0%, 3.0, 90.5ms, 0 | 20260505 03:40:58.638 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:58.638 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:58.638 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 03:40:58.638 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:58.638 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:40:58.639 | INFO | Uploaded 36GiB to GPU in 1.59699s (22.5424GiB/s) | 20260505 03:40:58.639 | INFO | Downloaded 34GiB to GPU in 1.51496s (22.4429GiB/s) | 20260505 03:41:03.777 | INFO | Analyzing chunk frames = [56832,57344) | 20260505 03:41:03.777 | INFO | MemInfo: { | 20260505 03:41:03.777 | INFO | "gpu" : " 17 GB", | 20260505 03:41:03.777 | INFO | "huge" : "", | 20260505 03:41:03.777 | INFO | "huge_shared" : " 27 GB", | 20260505 03:41:03.777 | INFO | "malloc" : " 15 GB", | 20260505 03:41:03.777 | INFO | "pinned" : " 29 GB" | 20260505 03:41:03.777 | INFO | } | 20260505 03:41:03.777 | INFO | Perf report for the last 5.1392 seconds: | 20260505 03:41:03.777 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:41:03.777 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:03.777 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:03.777 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:03.777 | INFO | ANALYSIS : 58.2%, 3.0, 90.7ms, 0 | 20260505 03:41:03.777 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:03.777 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:03.777 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 03:41:03.777 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:03.777 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:03.777 | INFO | Uploaded 36GiB to GPU in 1.59791s (22.5294GiB/s) | 20260505 03:41:03.777 | INFO | Downloaded 34GiB to GPU in 1.51529s (22.438GiB/s) | 20260505 03:41:03.778 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 03:41:03.778 | INFO | Total mem usage: 14.9 GiB | 20260505 03:41:03.778 | INFO | All Time High Water Marks sum: 15 GiB (Savings: 0.845%) | 20260505 03:41:03.778 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: 0.845%) | 20260505 03:41:03.778 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 03:41:03.778 | INFO | High Water Breakdown: | 20260505 03:41:03.778 | INFO | 55.6% : 8.34 GiB ( 100% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 03:41:03.778 | INFO | 44.3% : 6.65 GiB ( 100% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 03:41:03.778 | INFO | 0.124% : 19.1 MiB ( 100% Of Peak) : SampledBazData | 20260505 03:41:03.778 | INFO | ------------------------------------------ | 20260505 03:41:03.778 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:41:03.778 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 03:41:03.778 | INFO | Total mem usage: 26.5 GiB | 20260505 03:41:03.778 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260505 03:41:03.778 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 03:41:03.778 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260505 03:41:03.778 | INFO | High Water Breakdown: | 20260505 03:41:03.778 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 03:41:03.778 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 03:41:03.778 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 03:41:03.778 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 03:41:03.778 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 03:41:03.778 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 03:41:03.778 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 03:41:03.778 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 03:41:03.778 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 03:41:03.778 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 03:41:03.778 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 03:41:03.778 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 03:41:03.778 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 03:41:03.778 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 03:41:03.778 | INFO | ------------------------------------------ | 20260505 03:41:03.778 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:41:03.778 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 03:41:03.778 | INFO | Total mem usage: 25.6 GiB | 20260505 03:41:03.778 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 03:41:03.778 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 03:41:03.778 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 03:41:03.778 | INFO | High Water Breakdown: | 20260505 03:41:03.778 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 03:41:03.778 | INFO | ------------------------------------------ | 20260505 03:41:03.778 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:41:03.778 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 03:41:03.778 | INFO | Total mem usage: 15.7 GiB | 20260505 03:41:03.778 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260505 03:41:03.778 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 03:41:03.778 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260505 03:41:03.778 | INFO | High Water Breakdown: | 20260505 03:41:03.778 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 03:41:03.778 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 03:41:03.778 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 03:41:03.778 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 03:41:03.778 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 03:41:03.778 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 03:41:03.778 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 03:41:03.778 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 03:41:03.778 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 03:41:03.778 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 03:41:03.778 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 03:41:03.778 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 03:41:03.778 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 03:41:03.778 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 03:41:03.778 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 03:41:03.778 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 03:41:03.778 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 03:41:03.778 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 03:41:03.778 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 03:41:03.778 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 03:41:03.778 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 03:41:03.778 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 03:41:03.778 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 03:41:03.779 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 03:41:03.779 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 03:41:03.779 | INFO | ------------------------------------------ | 20260505 03:41:03.779 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:41:03.779 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 03:41:03.779 | INFO | PerfReport: Gathered 1536 samples. | 20260505 03:41:03.779 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 03:41:03.779 | INFO | PerfReport: Full Execution -- 3.07e+01 / -nan, 1.30e+01 / 0.00e+00 | 20260505 03:41:03.779 | INFO | PerfReport: 0.28%, AggregateStats -- 8.63e-02 / 0.00e+00, 5.76e-03 / 0.00e+00 | 20260505 03:41:03.779 | INFO | PerfReport: 10.73%, Baseline -- 3.30e+00 / 0.00e+00, 1.66e-02 / 0.00e+00 | 20260505 03:41:03.779 | INFO | PerfReport: 2.99%, Histogram -- 9.18e-01 / 0.00e+00, 5.20e-02 / 0.00e+00 | 20260505 03:41:03.779 | INFO | PerfReport: 16.11%, DME -- 3.96e+01 / 0.00e+00, 3.07e+00 / 0.00e+00 | 20260505 03:41:03.779 | INFO | PerfReport: 56.07%, FrameLabeling -- 1.72e+01 / 0.00e+00, 4.61e-02 / 0.00e+00 | 20260505 03:41:03.779 | INFO | PerfReport: 7.80%, PulseAccumulating -- 2.40e+00 / 0.00e+00, 1.64e-01 / 0.00e+00 | 20260505 03:41:03.779 | INFO | PerfReport: 4.23%, Metrics -- 1.30e+00 / 0.00e+00, 2.16e-01 / 0.00e+00 | 20260505 03:41:03.779 | INFO | PerfReport: 1.79%, PulseSerializing -- 5.49e-01 / 0.00e+00, 9.15e-02 / 0.00e+00 | 20260505 03:41:03.779 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 03:41:03.779 | INFO | PerfReport: Gathered 1536 samples. | 20260505 03:41:03.779 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 03:41:03.779 | INFO | PerfReport: Full Execution -- 9.24e+01 / -nan, 1.94e+01 / 0.00e+00 | 20260505 03:41:03.779 | INFO | PerfReport: 18.07%, Upload -- 1.67e+01 / 0.00e+00, 3.63e-01 / 0.00e+00 | 20260505 03:41:03.779 | INFO | PerfReport: 17.34%, Download -- 1.60e+01 / 0.00e+00, 6.84e-01 / 0.00e+00 | 20260505 03:41:03.779 | INFO | PerfReport: 33.29%, Compute -- 3.07e+01 / 0.00e+00, 1.30e+01 / 0.00e+00 | 20260505 03:41:03.779 | INFO | PerfReport: 0.54%, LockUpload -- 5.03e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260505 03:41:03.779 | INFO | PerfReport: 0.00%, LockDownload -- 4.06e-04 / 0.00e+00, 1.33e-04 / 0.00e+00 | 20260505 03:41:03.779 | INFO | PerfReport: 30.74%, LockCompute -- 2.84e+01 / 0.00e+00, 1.73e+01 / 0.00e+00 | 20260505 03:41:03.779 | INFO | PerfReport: 0.01%, Misc -- 1.26e-02 / 0.00e+00, 9.56e-03 / 0.00e+00 | 20260505 03:41:03.875 | INFO | Starting a round of Baz IO | 20260505 03:41:06.844 | INFO | Populating RTMetrics report | 20260505 03:41:06.844 | INFO | RTMetrics report populated | 20260505 03:41:06.844 | INFO | Opening tmp RTMetrics json file | 20260505 03:41:06.856 | INFO | Serializing RTMetrics to json | 20260505 03:41:06.857 | INFO | Serialization done, swapping tmp file now... | 20260505 03:41:06.857 | INFO | RTMetrics done | 20260505 03:41:06.949 | INFO | Finished a round of Baz IO: Wrote 1.235 GiB in 3.074 seconds | 20260505 03:41:06.949 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 74.02% / 10.10% / 15.87% | 20260505 03:41:06.949 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 15.1647% / 0.0617464% / 7.94729e-06% | 20260505 03:41:06.949 | INFO | Baz IO overall: | 20260505 03:41:06.949 | INFO | BytesWritten : 4848078761 | 20260505 03:41:06.949 | INFO | NumEvents : 918544341 | 20260505 03:41:08.916 | INFO | Analyzing chunk frames = [57344,57856) | 20260505 03:41:08.916 | INFO | MemInfo: { | 20260505 03:41:08.916 | INFO | "gpu" : " 17 GB", | 20260505 03:41:08.916 | INFO | "huge" : "", | 20260505 03:41:08.916 | INFO | "huge_shared" : " 27 GB", | 20260505 03:41:08.916 | INFO | "malloc" : " 15 GB", | 20260505 03:41:08.916 | INFO | "pinned" : " 29 GB" | 20260505 03:41:08.916 | INFO | } | 20260505 03:41:08.916 | INFO | Perf report for the last 5.13915 seconds: | 20260505 03:41:08.916 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:41:08.916 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:08.916 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:08.916 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:08.916 | INFO | ANALYSIS : 58.7%, 3.0, 91.7ms, 0 | 20260505 03:41:08.916 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:08.916 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260505 03:41:08.916 | INFO | PRE_HQ : 58.3%, 2.2, 65.6ms, 0 | 20260505 03:41:08.916 | INFO | BAZWRITER : 59.0%, 2.9, 88.0ms, 0 | 20260505 03:41:08.916 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 03:41:08.917 | INFO | Uploaded 36GiB to GPU in 1.6334s (22.04GiB/s) | 20260505 03:41:08.917 | INFO | Downloaded 38GiB to GPU in 1.69585s (22.4076GiB/s) | 20260505 03:41:14.056 | INFO | Analyzing chunk frames = [57856,58368) | 20260505 03:41:14.056 | INFO | MemInfo: { | 20260505 03:41:14.056 | INFO | "gpu" : " 17 GB", | 20260505 03:41:14.056 | INFO | "huge" : "", | 20260505 03:41:14.056 | INFO | "huge_shared" : " 27 GB", | 20260505 03:41:14.056 | INFO | "malloc" : " 15 GB", | 20260505 03:41:14.056 | INFO | "pinned" : " 29 GB" | 20260505 03:41:14.056 | INFO | } | 20260505 03:41:14.056 | INFO | Perf report for the last 5.1398 seconds: | 20260505 03:41:14.056 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:41:14.056 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:14.056 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:14.056 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:14.056 | INFO | ANALYSIS : 58.6%, 3.0, 91.4ms, 0 | 20260505 03:41:14.056 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:14.056 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:14.056 | INFO | PRE_HQ : 33.0%, 1.0, 17.4ms, 0 | 20260505 03:41:14.056 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:14.056 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:14.056 | INFO | Uploaded 36GiB to GPU in 1.59508s (22.5693GiB/s) | 20260505 03:41:14.056 | INFO | Downloaded 34GiB to GPU in 1.5113s (22.4972GiB/s) | 20260505 03:41:19.195 | INFO | Analyzing chunk frames = [58368,58880) | 20260505 03:41:19.195 | INFO | MemInfo: { | 20260505 03:41:19.195 | INFO | "gpu" : " 17 GB", | 20260505 03:41:19.195 | INFO | "huge" : "", | 20260505 03:41:19.195 | INFO | "huge_shared" : " 27 GB", | 20260505 03:41:19.195 | INFO | "malloc" : " 15 GB", | 20260505 03:41:19.195 | INFO | "pinned" : " 29 GB" | 20260505 03:41:19.195 | INFO | } | 20260505 03:41:19.195 | INFO | Perf report for the last 5.13896 seconds: | 20260505 03:41:19.195 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:41:19.195 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:19.195 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:19.195 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:19.195 | INFO | ANALYSIS : 58.0%, 3.0, 90.1ms, 0 | 20260505 03:41:19.195 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:19.195 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:19.195 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 03:41:19.195 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:19.195 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:19.195 | INFO | Uploaded 36GiB to GPU in 1.59895s (22.5147GiB/s) | 20260505 03:41:19.196 | INFO | Downloaded 34GiB to GPU in 1.51573s (22.4314GiB/s) | 20260505 03:41:24.334 | INFO | Analyzing chunk frames = [58880,59392) | 20260505 03:41:24.334 | INFO | MemInfo: { | 20260505 03:41:24.334 | INFO | "gpu" : " 17 GB", | 20260505 03:41:24.334 | INFO | "huge" : "", | 20260505 03:41:24.334 | INFO | "huge_shared" : " 27 GB", | 20260505 03:41:24.334 | INFO | "malloc" : " 15 GB", | 20260505 03:41:24.334 | INFO | "pinned" : " 29 GB" | 20260505 03:41:24.335 | INFO | } | 20260505 03:41:24.335 | INFO | Perf report for the last 5.13944 seconds: | 20260505 03:41:24.335 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:41:24.335 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:24.335 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:24.335 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:24.335 | INFO | ANALYSIS : 57.7%, 3.0, 89.4ms, 0 | 20260505 03:41:24.335 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:24.335 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:24.335 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 03:41:24.335 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:24.335 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:24.335 | INFO | Uploaded 36GiB to GPU in 1.59812s (22.5265GiB/s) | 20260505 03:41:24.335 | INFO | Downloaded 34GiB to GPU in 1.51476s (22.4459GiB/s) | 20260505 03:41:29.474 | INFO | Analyzing chunk frames = [59392,59904) | 20260505 03:41:29.474 | INFO | MemInfo: { | 20260505 03:41:29.474 | INFO | "gpu" : " 17 GB", | 20260505 03:41:29.474 | INFO | "huge" : "", | 20260505 03:41:29.474 | INFO | "huge_shared" : " 27 GB", | 20260505 03:41:29.474 | INFO | "malloc" : " 15 GB", | 20260505 03:41:29.474 | INFO | "pinned" : " 29 GB" | 20260505 03:41:29.474 | INFO | } | 20260505 03:41:29.474 | INFO | Perf report for the last 5.13946 seconds: | 20260505 03:41:29.474 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:41:29.474 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:29.474 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:29.474 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:29.474 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260505 03:41:29.474 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:29.474 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:29.474 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 03:41:29.474 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:29.474 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:29.475 | INFO | Uploaded 36GiB to GPU in 1.59674s (22.5459GiB/s) | 20260505 03:41:29.475 | INFO | Downloaded 34GiB to GPU in 1.51314s (22.4698GiB/s) | 20260505 03:41:34.613 | INFO | Analyzing chunk frames = [59904,60416) | 20260505 03:41:34.613 | INFO | MemInfo: { | 20260505 03:41:34.613 | INFO | "gpu" : " 17 GB", | 20260505 03:41:34.613 | INFO | "huge" : "", | 20260505 03:41:34.613 | INFO | "huge_shared" : " 27 GB", | 20260505 03:41:34.613 | INFO | "malloc" : " 15 GB", | 20260505 03:41:34.613 | INFO | "pinned" : " 29 GB" | 20260505 03:41:34.613 | INFO | } | 20260505 03:41:34.613 | INFO | Perf report for the last 5.13915 seconds: | 20260505 03:41:34.613 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:41:34.613 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:34.613 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:34.613 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:34.613 | INFO | ANALYSIS : 57.9%, 3.0, 90.3ms, 0 | 20260505 03:41:34.613 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:34.613 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:34.613 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 03:41:34.613 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:34.613 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:34.614 | INFO | Uploaded 36GiB to GPU in 1.59737s (22.5371GiB/s) | 20260505 03:41:34.614 | INFO | Downloaded 34GiB to GPU in 1.51309s (22.4706GiB/s) | 20260505 03:41:39.752 | INFO | Analyzing chunk frames = [60416,60928) | 20260505 03:41:39.752 | INFO | MemInfo: { | 20260505 03:41:39.752 | INFO | "gpu" : " 17 GB", | 20260505 03:41:39.752 | INFO | "huge" : "", | 20260505 03:41:39.752 | INFO | "huge_shared" : " 27 GB", | 20260505 03:41:39.752 | INFO | "malloc" : " 15 GB", | 20260505 03:41:39.752 | INFO | "pinned" : " 29 GB" | 20260505 03:41:39.752 | INFO | } | 20260505 03:41:39.752 | INFO | Perf report for the last 5.13924 seconds: | 20260505 03:41:39.752 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:41:39.752 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:39.753 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:39.753 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:39.753 | INFO | ANALYSIS : 58.0%, 3.0, 90.5ms, 0 | 20260505 03:41:39.753 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:39.753 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:39.753 | INFO | PRE_HQ : 32.8%, 1.0, 17.0ms, 0 | 20260505 03:41:39.753 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:39.753 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:39.753 | INFO | Uploaded 36GiB to GPU in 1.59779s (22.5311GiB/s) | 20260505 03:41:39.753 | INFO | Downloaded 34GiB to GPU in 1.51429s (22.4528GiB/s) | 20260505 03:41:44.892 | INFO | Analyzing chunk frames = [60928,61440) | 20260505 03:41:44.892 | INFO | MemInfo: { | 20260505 03:41:44.892 | INFO | "gpu" : " 17 GB", | 20260505 03:41:44.892 | INFO | "huge" : "", | 20260505 03:41:44.892 | INFO | "huge_shared" : " 27 GB", | 20260505 03:41:44.892 | INFO | "malloc" : " 15 GB", | 20260505 03:41:44.892 | INFO | "pinned" : " 29 GB" | 20260505 03:41:44.892 | INFO | } | 20260505 03:41:44.892 | INFO | Perf report for the last 5.13949 seconds: | 20260505 03:41:44.892 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:41:44.892 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:44.892 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:44.892 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:44.892 | INFO | ANALYSIS : 58.2%, 3.0, 90.7ms, 0 | 20260505 03:41:44.892 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:44.892 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:44.892 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 03:41:44.892 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:44.892 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:44.892 | INFO | Uploaded 36GiB to GPU in 1.59739s (22.5368GiB/s) | 20260505 03:41:44.892 | INFO | Downloaded 34GiB to GPU in 1.51405s (22.4564GiB/s) | 20260505 03:41:44.989 | INFO | Starting a round of Baz IO | 20260505 03:41:47.960 | INFO | Populating RTMetrics report | 20260505 03:41:47.960 | INFO | RTMetrics report populated | 20260505 03:41:47.960 | INFO | Opening tmp RTMetrics json file | 20260505 03:41:47.968 | INFO | Serializing RTMetrics to json | 20260505 03:41:47.969 | INFO | Serialization done, swapping tmp file now... | 20260505 03:41:47.970 | INFO | RTMetrics done | 20260505 03:41:48.013 | INFO | Finished a round of Baz IO: Wrote 1.157 GiB in 3.024 seconds | 20260505 03:41:48.013 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 73.03% / 0.99% / 25.98% | 20260505 03:41:48.013 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 15.3551% / 0.00495513% / 3.97364e-06% | 20260505 03:41:48.013 | INFO | Baz IO overall: | 20260505 03:41:48.013 | INFO | BytesWritten : 6090385930 | 20260505 03:41:48.013 | INFO | NumEvents : 1508114632 | 20260505 03:41:50.031 | INFO | Analyzing chunk frames = [61440,61952) | 20260505 03:41:50.031 | INFO | MemInfo: { | 20260505 03:41:50.031 | INFO | "gpu" : " 17 GB", | 20260505 03:41:50.031 | INFO | "huge" : "", | 20260505 03:41:50.031 | INFO | "huge_shared" : " 27 GB", | 20260505 03:41:50.031 | INFO | "malloc" : " 15 GB", | 20260505 03:41:50.031 | INFO | "pinned" : " 29 GB" | 20260505 03:41:50.031 | INFO | } | 20260505 03:41:50.031 | INFO | Perf report for the last 5.13927 seconds: | 20260505 03:41:50.031 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:41:50.031 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:50.031 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:50.031 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:50.031 | INFO | ANALYSIS : 58.8%, 3.0, 91.7ms, 0 | 20260505 03:41:50.031 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:50.031 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260505 03:41:50.031 | INFO | PRE_HQ : 58.3%, 2.1, 65.0ms, 0 | 20260505 03:41:50.031 | INFO | BAZWRITER : 58.2%, 2.3, 68.7ms, 0 | 20260505 03:41:50.031 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 03:41:50.032 | INFO | Uploaded 36GiB to GPU in 1.6418s (21.9271GiB/s) | 20260505 03:41:50.032 | INFO | Downloaded 38GiB to GPU in 1.70316s (22.3114GiB/s) | 20260505 03:41:55.170 | INFO | Analyzing chunk frames = [61952,62464) | 20260505 03:41:55.170 | INFO | MemInfo: { | 20260505 03:41:55.170 | INFO | "gpu" : " 17 GB", | 20260505 03:41:55.170 | INFO | "huge" : "", | 20260505 03:41:55.170 | INFO | "huge_shared" : " 27 GB", | 20260505 03:41:55.170 | INFO | "malloc" : " 15 GB", | 20260505 03:41:55.170 | INFO | "pinned" : " 29 GB" | 20260505 03:41:55.170 | INFO | } | 20260505 03:41:55.171 | INFO | Perf report for the last 5.13931 seconds: | 20260505 03:41:55.171 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:41:55.171 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:55.171 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:55.171 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:55.171 | INFO | ANALYSIS : 58.6%, 3.0, 91.4ms, 0 | 20260505 03:41:55.171 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:55.171 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:55.171 | INFO | PRE_HQ : 32.8%, 1.0, 17.2ms, 0 | 20260505 03:41:55.171 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:55.171 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:41:55.171 | INFO | Uploaded 36GiB to GPU in 1.59471s (22.5746GiB/s) | 20260505 03:41:55.171 | INFO | Downloaded 34GiB to GPU in 1.51102s (22.5014GiB/s) | 20260505 03:42:00.310 | INFO | Analyzing chunk frames = [62464,62976) | 20260505 03:42:00.310 | INFO | MemInfo: { | 20260505 03:42:00.310 | INFO | "gpu" : " 17 GB", | 20260505 03:42:00.310 | INFO | "huge" : "", | 20260505 03:42:00.310 | INFO | "huge_shared" : " 27 GB", | 20260505 03:42:00.310 | INFO | "malloc" : " 15 GB", | 20260505 03:42:00.310 | INFO | "pinned" : " 29 GB" | 20260505 03:42:00.310 | INFO | } | 20260505 03:42:00.310 | INFO | Perf report for the last 5.13937 seconds: | 20260505 03:42:00.310 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:42:00.310 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:00.310 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:00.310 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:00.310 | INFO | ANALYSIS : 58.0%, 3.0, 90.1ms, 0 | 20260505 03:42:00.310 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:00.310 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:00.310 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260505 03:42:00.310 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:00.310 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:00.311 | INFO | Uploaded 36GiB to GPU in 1.5987s (22.5184GiB/s) | 20260505 03:42:00.311 | INFO | Downloaded 34GiB to GPU in 1.51544s (22.4358GiB/s) | 20260505 03:42:05.449 | INFO | Analyzing chunk frames = [62976,63488) | 20260505 03:42:05.449 | INFO | MemInfo: { | 20260505 03:42:05.449 | INFO | "gpu" : " 17 GB", | 20260505 03:42:05.449 | INFO | "huge" : "", | 20260505 03:42:05.449 | INFO | "huge_shared" : " 27 GB", | 20260505 03:42:05.449 | INFO | "malloc" : " 15 GB", | 20260505 03:42:05.449 | INFO | "pinned" : " 29 GB" | 20260505 03:42:05.449 | INFO | } | 20260505 03:42:05.449 | INFO | Perf report for the last 5.13929 seconds: | 20260505 03:42:05.449 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:42:05.449 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:05.449 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:05.449 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:05.449 | INFO | ANALYSIS : 57.7%, 3.0, 89.5ms, 0 | 20260505 03:42:05.449 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:05.449 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:05.449 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260505 03:42:05.449 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:05.449 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:05.450 | INFO | Uploaded 36GiB to GPU in 1.59793s (22.5292GiB/s) | 20260505 03:42:05.450 | INFO | Downloaded 34GiB to GPU in 1.51426s (22.4532GiB/s) | 20260505 03:42:10.588 | INFO | Analyzing chunk frames = [63488,64000) | 20260505 03:42:10.588 | INFO | MemInfo: { | 20260505 03:42:10.588 | INFO | "gpu" : " 17 GB", | 20260505 03:42:10.588 | INFO | "huge" : "", | 20260505 03:42:10.588 | INFO | "huge_shared" : " 27 GB", | 20260505 03:42:10.588 | INFO | "malloc" : " 15 GB", | 20260505 03:42:10.588 | INFO | "pinned" : " 29 GB" | 20260505 03:42:10.588 | INFO | } | 20260505 03:42:10.589 | INFO | Perf report for the last 5.13927 seconds: | 20260505 03:42:10.589 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:42:10.589 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:10.589 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:10.589 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:10.589 | INFO | ANALYSIS : 57.5%, 3.0, 89.8ms, 0 | 20260505 03:42:10.589 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:10.589 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:10.589 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 03:42:10.589 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:10.589 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:10.589 | INFO | Uploaded 36GiB to GPU in 1.5963s (22.5522GiB/s) | 20260505 03:42:10.589 | INFO | Downloaded 34GiB to GPU in 1.51175s (22.4905GiB/s) | 20260505 03:42:15.728 | INFO | Analyzing chunk frames = [64000,64512) | 20260505 03:42:15.728 | INFO | MemInfo: { | 20260505 03:42:15.728 | INFO | "gpu" : " 17 GB", | 20260505 03:42:15.728 | INFO | "huge" : "", | 20260505 03:42:15.728 | INFO | "huge_shared" : " 27 GB", | 20260505 03:42:15.728 | INFO | "malloc" : " 15 GB", | 20260505 03:42:15.728 | INFO | "pinned" : " 29 GB" | 20260505 03:42:15.728 | INFO | } | 20260505 03:42:15.728 | INFO | Perf report for the last 5.13949 seconds: | 20260505 03:42:15.728 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:42:15.728 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:15.728 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:15.728 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:15.728 | INFO | ANALYSIS : 57.8%, 3.0, 90.2ms, 0 | 20260505 03:42:15.728 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:15.728 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:15.728 | INFO | PRE_HQ : 32.7%, 1.0, 17.0ms, 0 | 20260505 03:42:15.728 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:15.728 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:15.728 | INFO | Uploaded 36GiB to GPU in 1.59737s (22.5371GiB/s) | 20260505 03:42:15.728 | INFO | Downloaded 34GiB to GPU in 1.51352s (22.4642GiB/s) | 20260505 03:42:20.867 | INFO | Analyzing chunk frames = [64512,65024) | 20260505 03:42:20.867 | INFO | MemInfo: { | 20260505 03:42:20.867 | INFO | "gpu" : " 17 GB", | 20260505 03:42:20.867 | INFO | "huge" : "", | 20260505 03:42:20.867 | INFO | "huge_shared" : " 27 GB", | 20260505 03:42:20.867 | INFO | "malloc" : " 15 GB", | 20260505 03:42:20.867 | INFO | "pinned" : " 29 GB" | 20260505 03:42:20.867 | INFO | } | 20260505 03:42:20.867 | INFO | Perf report for the last 5.13926 seconds: | 20260505 03:42:20.867 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:42:20.867 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:20.867 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:20.867 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:20.867 | INFO | ANALYSIS : 58.1%, 3.0, 90.6ms, 0 | 20260505 03:42:20.867 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:20.867 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:20.867 | INFO | PRE_HQ : 32.7%, 1.0, 17.0ms, 0 | 20260505 03:42:20.867 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:20.867 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:20.868 | INFO | Uploaded 36GiB to GPU in 1.59713s (22.5405GiB/s) | 20260505 03:42:20.868 | INFO | Downloaded 34GiB to GPU in 1.51347s (22.4649GiB/s) | 20260505 03:42:26.006 | INFO | Analyzing chunk frames = [65024,65536) | 20260505 03:42:26.006 | INFO | MemInfo: { | 20260505 03:42:26.006 | INFO | "gpu" : " 17 GB", | 20260505 03:42:26.006 | INFO | "huge" : "", | 20260505 03:42:26.006 | INFO | "huge_shared" : " 27 GB", | 20260505 03:42:26.007 | INFO | "malloc" : " 15 GB", | 20260505 03:42:26.007 | INFO | "pinned" : " 29 GB" | 20260505 03:42:26.007 | INFO | } | 20260505 03:42:26.007 | INFO | Perf report for the last 5.13931 seconds: | 20260505 03:42:26.007 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:42:26.007 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:26.007 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:26.007 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:26.007 | INFO | ANALYSIS : 58.1%, 3.0, 90.7ms, 0 | 20260505 03:42:26.007 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:26.007 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:26.007 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 03:42:26.007 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:26.007 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:26.007 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 03:42:26.007 | INFO | Total mem usage: 14.9 GiB | 20260505 03:42:26.007 | INFO | All Time High Water Marks sum: 15.2 GiB (Savings: 1.52%) | 20260505 03:42:26.007 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: 0.265%) | 20260505 03:42:26.007 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 03:42:26.007 | INFO | High Water Breakdown: | 20260505 03:42:26.007 | INFO | 56.7% : 8.49 GiB ( 100% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 03:42:26.007 | INFO | 43.1% : 6.45 GiB ( 97.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 03:42:26.007 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 03:42:26.007 | INFO | Uploaded 36GiB to GPU in 1.59776s (22.5315GiB/s) | 20260505 03:42:26.007 | INFO | Downloaded 34GiB to GPU in 1.51406s (22.4561GiB/s) | 20260505 03:42:26.007 | INFO | ------------------------------------------ | 20260505 03:42:26.007 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:42:26.007 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 03:42:26.007 | INFO | Total mem usage: 26.5 GiB | 20260505 03:42:26.008 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260505 03:42:26.008 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 03:42:26.008 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260505 03:42:26.008 | INFO | High Water Breakdown: | 20260505 03:42:26.008 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 03:42:26.008 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 03:42:26.008 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 03:42:26.008 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 03:42:26.008 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 03:42:26.008 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 03:42:26.008 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 03:42:26.008 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 03:42:26.008 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 03:42:26.008 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 03:42:26.008 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 03:42:26.008 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 03:42:26.008 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 03:42:26.008 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 03:42:26.008 | INFO | ------------------------------------------ | 20260505 03:42:26.008 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:42:26.008 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 03:42:26.008 | INFO | Total mem usage: 25.6 GiB | 20260505 03:42:26.008 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 03:42:26.008 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 03:42:26.008 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 03:42:26.008 | INFO | High Water Breakdown: | 20260505 03:42:26.008 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 03:42:26.008 | INFO | ------------------------------------------ | 20260505 03:42:26.008 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:42:26.008 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 03:42:26.008 | INFO | Total mem usage: 15.7 GiB | 20260505 03:42:26.008 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260505 03:42:26.008 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 03:42:26.008 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260505 03:42:26.008 | INFO | High Water Breakdown: | 20260505 03:42:26.008 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 03:42:26.008 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 03:42:26.008 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 03:42:26.008 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 03:42:26.008 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 03:42:26.008 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 03:42:26.008 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 03:42:26.008 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 03:42:26.008 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 03:42:26.008 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 03:42:26.008 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 03:42:26.008 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 03:42:26.008 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 03:42:26.008 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 03:42:26.008 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 03:42:26.008 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 03:42:26.008 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 03:42:26.008 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 03:42:26.008 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 03:42:26.008 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 03:42:26.008 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 03:42:26.008 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 03:42:26.008 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 03:42:26.008 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 03:42:26.008 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 03:42:26.008 | INFO | ------------------------------------------ | 20260505 03:42:26.008 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:42:26.008 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 03:42:26.008 | INFO | PerfReport: Gathered 1536 samples. | 20260505 03:42:26.008 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 03:42:26.008 | INFO | PerfReport: Full Execution -- 3.07e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260505 03:42:26.008 | INFO | PerfReport: 0.28%, AggregateStats -- 8.62e-02 / 0.00e+00, 3.96e-03 / 0.00e+00 | 20260505 03:42:26.008 | INFO | PerfReport: 10.72%, Baseline -- 3.30e+00 / 0.00e+00, 1.44e-02 / 0.00e+00 | 20260505 03:42:26.008 | INFO | PerfReport: 2.99%, Histogram -- 9.20e-01 / 0.00e+00, 5.08e-02 / 0.00e+00 | 20260505 03:42:26.008 | INFO | PerfReport: 16.07%, DME -- 3.95e+01 / 0.00e+00, 3.21e+00 / 0.00e+00 | 20260505 03:42:26.008 | INFO | PerfReport: 56.03%, FrameLabeling -- 1.72e+01 / 0.00e+00, 4.63e-02 / 0.00e+00 | 20260505 03:42:26.008 | INFO | PerfReport: 7.83%, PulseAccumulating -- 2.41e+00 / 0.00e+00, 1.65e-01 / 0.00e+00 | 20260505 03:42:26.008 | INFO | PerfReport: 4.28%, Metrics -- 1.32e+00 / 0.00e+00, 2.17e-01 / 0.00e+00 | 20260505 03:42:26.008 | INFO | PerfReport: 1.80%, PulseSerializing -- 5.54e-01 / 0.00e+00, 9.11e-02 / 0.00e+00 | 20260505 03:42:26.008 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 03:42:26.008 | INFO | PerfReport: Gathered 1536 samples. | 20260505 03:42:26.008 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 03:42:26.008 | INFO | PerfReport: Full Execution -- 9.24e+01 / -nan, 1.94e+01 / 0.00e+00 | 20260505 03:42:26.009 | INFO | PerfReport: 18.07%, Upload -- 1.67e+01 / 0.00e+00, 3.65e-01 / 0.00e+00 | 20260505 03:42:26.009 | INFO | PerfReport: 17.33%, Download -- 1.60e+01 / 0.00e+00, 6.91e-01 / 0.00e+00 | 20260505 03:42:26.009 | INFO | PerfReport: 33.29%, Compute -- 3.08e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260505 03:42:26.009 | INFO | PerfReport: 0.54%, LockUpload -- 5.03e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260505 03:42:26.009 | INFO | PerfReport: 0.00%, LockDownload -- 4.01e-04 / 0.00e+00, 1.98e-04 / 0.00e+00 | 20260505 03:42:26.009 | INFO | PerfReport: 30.75%, LockCompute -- 2.84e+01 / 0.00e+00, 1.73e+01 / 0.00e+00 | 20260505 03:42:26.009 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 2.09e-02 / 0.00e+00 | 20260505 03:42:26.104 | INFO | Starting a round of Baz IO | 20260505 03:42:29.079 | INFO | Populating RTMetrics report | 20260505 03:42:29.080 | INFO | RTMetrics report populated | 20260505 03:42:29.080 | INFO | Opening tmp RTMetrics json file | 20260505 03:42:29.093 | INFO | Serializing RTMetrics to json | 20260505 03:42:29.094 | INFO | Serialization done, swapping tmp file now... | 20260505 03:42:29.095 | INFO | RTMetrics done | 20260505 03:42:29.133 | INFO | Finished a round of Baz IO: Wrote 1.198 GiB in 3.028 seconds | 20260505 03:42:29.133 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 72.10% / 0.93% / 26.97% | 20260505 03:42:29.133 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 15.5319% / 0.00463724% / 0% | 20260505 03:42:29.133 | INFO | Baz IO overall: | 20260505 03:42:29.133 | INFO | BytesWritten : 7377004329 | 20260505 03:42:29.133 | INFO | NumEvents : 2116348936 | 20260505 03:42:31.146 | INFO | Analyzing chunk frames = [65536,66048) | 20260505 03:42:31.146 | INFO | MemInfo: { | 20260505 03:42:31.146 | INFO | "gpu" : " 17 GB", | 20260505 03:42:31.146 | INFO | "huge" : "", | 20260505 03:42:31.146 | INFO | "huge_shared" : " 27 GB", | 20260505 03:42:31.146 | INFO | "malloc" : " 15 GB", | 20260505 03:42:31.146 | INFO | "pinned" : " 29 GB" | 20260505 03:42:31.146 | INFO | } | 20260505 03:42:31.146 | INFO | Perf report for the last 5.1394 seconds: | 20260505 03:42:31.146 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:42:31.146 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:31.146 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:31.146 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:31.146 | INFO | ANALYSIS : 58.8%, 3.0, 91.8ms, 0 | 20260505 03:42:31.146 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:31.146 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260505 03:42:31.146 | INFO | PRE_HQ : 58.3%, 2.2, 65.3ms, 0 | 20260505 03:42:31.146 | INFO | BAZWRITER : 58.4%, 2.4, 71.0ms, 0 | 20260505 03:42:31.146 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 03:42:31.146 | INFO | Uploaded 36GiB to GPU in 1.63412s (22.0302GiB/s) | 20260505 03:42:31.146 | INFO | Downloaded 38GiB to GPU in 1.69639s (22.4006GiB/s) | 20260505 03:42:36.285 | INFO | Analyzing chunk frames = [66048,66560) | 20260505 03:42:36.285 | INFO | MemInfo: { | 20260505 03:42:36.285 | INFO | "gpu" : " 17 GB", | 20260505 03:42:36.285 | INFO | "huge" : "", | 20260505 03:42:36.285 | INFO | "huge_shared" : " 27 GB", | 20260505 03:42:36.285 | INFO | "malloc" : " 15 GB", | 20260505 03:42:36.285 | INFO | "pinned" : " 29 GB" | 20260505 03:42:36.285 | INFO | } | 20260505 03:42:36.285 | INFO | Perf report for the last 5.13928 seconds: | 20260505 03:42:36.285 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:42:36.285 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:36.285 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:36.285 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:36.285 | INFO | ANALYSIS : 58.6%, 3.0, 91.4ms, 0 | 20260505 03:42:36.285 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:36.285 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:36.285 | INFO | PRE_HQ : 32.8%, 1.0, 17.2ms, 0 | 20260505 03:42:36.285 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:36.285 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:36.286 | INFO | Uploaded 36GiB to GPU in 1.59469s (22.5749GiB/s) | 20260505 03:42:36.286 | INFO | Downloaded 34GiB to GPU in 1.51046s (22.5097GiB/s) | 20260505 03:42:41.425 | INFO | Analyzing chunk frames = [66560,67072) | 20260505 03:42:41.425 | INFO | MemInfo: { | 20260505 03:42:41.425 | INFO | "gpu" : " 17 GB", | 20260505 03:42:41.425 | INFO | "huge" : "", | 20260505 03:42:41.425 | INFO | "huge_shared" : " 27 GB", | 20260505 03:42:41.425 | INFO | "malloc" : " 15 GB", | 20260505 03:42:41.425 | INFO | "pinned" : " 29 GB" | 20260505 03:42:41.425 | INFO | } | 20260505 03:42:41.425 | INFO | Perf report for the last 5.13936 seconds: | 20260505 03:42:41.425 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:42:41.425 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:41.425 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:41.425 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:41.425 | INFO | ANALYSIS : 58.1%, 3.0, 90.2ms, 0 | 20260505 03:42:41.425 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:41.425 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:41.425 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260505 03:42:41.425 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:41.425 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:41.425 | INFO | Uploaded 36GiB to GPU in 1.59797s (22.5286GiB/s) | 20260505 03:42:41.425 | INFO | Downloaded 34GiB to GPU in 1.51467s (22.4471GiB/s) | 20260505 03:42:46.564 | INFO | Analyzing chunk frames = [67072,67584) | 20260505 03:42:46.564 | INFO | MemInfo: { | 20260505 03:42:46.564 | INFO | "gpu" : " 17 GB", | 20260505 03:42:46.564 | INFO | "huge" : "", | 20260505 03:42:46.564 | INFO | "huge_shared" : " 27 GB", | 20260505 03:42:46.564 | INFO | "malloc" : " 15 GB", | 20260505 03:42:46.564 | INFO | "pinned" : " 29 GB" | 20260505 03:42:46.564 | INFO | } | 20260505 03:42:46.564 | INFO | Perf report for the last 5.13925 seconds: | 20260505 03:42:46.564 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:42:46.564 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:46.564 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:46.564 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:46.564 | INFO | ANALYSIS : 57.7%, 3.0, 89.4ms, 0 | 20260505 03:42:46.564 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:46.564 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:46.564 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260505 03:42:46.564 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:46.564 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:46.564 | INFO | Uploaded 36GiB to GPU in 1.5985s (22.5212GiB/s) | 20260505 03:42:46.564 | INFO | Downloaded 34GiB to GPU in 1.51458s (22.4485GiB/s) | 20260505 03:42:51.703 | INFO | Analyzing chunk frames = [67584,68096) | 20260505 03:42:51.703 | INFO | MemInfo: { | 20260505 03:42:51.703 | INFO | "gpu" : " 17 GB", | 20260505 03:42:51.703 | INFO | "huge" : "", | 20260505 03:42:51.703 | INFO | "huge_shared" : " 27 GB", | 20260505 03:42:51.703 | INFO | "malloc" : " 15 GB", | 20260505 03:42:51.703 | INFO | "pinned" : " 29 GB" | 20260505 03:42:51.703 | INFO | } | 20260505 03:42:51.703 | INFO | Perf report for the last 5.13943 seconds: | 20260505 03:42:51.703 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:42:51.703 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:51.703 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:51.703 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:51.704 | INFO | ANALYSIS : 57.6%, 3.0, 89.8ms, 0 | 20260505 03:42:51.704 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:51.704 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:51.704 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260505 03:42:51.704 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:51.704 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:51.704 | INFO | Uploaded 36GiB to GPU in 1.59611s (22.5549GiB/s) | 20260505 03:42:51.704 | INFO | Downloaded 34GiB to GPU in 1.5117s (22.4912GiB/s) | 20260505 03:42:56.842 | INFO | Analyzing chunk frames = [68096,68608) | 20260505 03:42:56.842 | INFO | MemInfo: { | 20260505 03:42:56.842 | INFO | "gpu" : " 17 GB", | 20260505 03:42:56.842 | INFO | "huge" : "", | 20260505 03:42:56.842 | INFO | "huge_shared" : " 27 GB", | 20260505 03:42:56.842 | INFO | "malloc" : " 15 GB", | 20260505 03:42:56.842 | INFO | "pinned" : " 29 GB" | 20260505 03:42:56.842 | INFO | } | 20260505 03:42:56.843 | INFO | Perf report for the last 5.13916 seconds: | 20260505 03:42:56.843 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:42:56.843 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:56.843 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:56.843 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:56.843 | INFO | ANALYSIS : 57.9%, 3.0, 90.3ms, 0 | 20260505 03:42:56.843 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:56.843 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:56.843 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260505 03:42:56.843 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:56.843 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:42:56.843 | INFO | Uploaded 36GiB to GPU in 1.59666s (22.547GiB/s) | 20260505 03:42:56.843 | INFO | Downloaded 34GiB to GPU in 1.5129s (22.4734GiB/s) | 20260505 03:43:01.982 | INFO | Analyzing chunk frames = [68608,69120) | 20260505 03:43:01.982 | INFO | MemInfo: { | 20260505 03:43:01.982 | INFO | "gpu" : " 17 GB", | 20260505 03:43:01.982 | INFO | "huge" : "", | 20260505 03:43:01.982 | INFO | "huge_shared" : " 27 GB", | 20260505 03:43:01.982 | INFO | "malloc" : " 15 GB", | 20260505 03:43:01.982 | INFO | "pinned" : " 29 GB" | 20260505 03:43:01.982 | INFO | } | 20260505 03:43:01.982 | INFO | Perf report for the last 5.13942 seconds: | 20260505 03:43:01.982 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:43:01.982 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:01.982 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:01.982 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:01.982 | INFO | ANALYSIS : 58.1%, 3.0, 90.7ms, 0 | 20260505 03:43:01.982 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:01.982 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:01.982 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260505 03:43:01.982 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:01.982 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:01.983 | INFO | Uploaded 36GiB to GPU in 1.5971s (22.5409GiB/s) | 20260505 03:43:01.983 | INFO | Downloaded 34GiB to GPU in 1.51423s (22.4536GiB/s) | 20260505 03:43:07.121 | INFO | Analyzing chunk frames = [69120,69632) | 20260505 03:43:07.121 | INFO | MemInfo: { | 20260505 03:43:07.121 | INFO | "gpu" : " 17 GB", | 20260505 03:43:07.121 | INFO | "huge" : "", | 20260505 03:43:07.121 | INFO | "huge_shared" : " 27 GB", | 20260505 03:43:07.121 | INFO | "malloc" : " 15 GB", | 20260505 03:43:07.121 | INFO | "pinned" : " 29 GB" | 20260505 03:43:07.121 | INFO | } | 20260505 03:43:07.121 | INFO | Perf report for the last 5.13939 seconds: | 20260505 03:43:07.121 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:43:07.121 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:07.121 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:07.121 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:07.122 | INFO | ANALYSIS : 58.2%, 3.0, 90.8ms, 0 | 20260505 03:43:07.122 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:07.122 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:07.122 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 03:43:07.122 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:07.122 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:07.122 | INFO | Uploaded 36GiB to GPU in 1.59679s (22.5453GiB/s) | 20260505 03:43:07.122 | INFO | Downloaded 34GiB to GPU in 1.51458s (22.4484GiB/s) | 20260505 03:43:07.219 | INFO | Starting a round of Baz IO | 20260505 03:43:10.197 | INFO | Populating RTMetrics report | 20260505 03:43:10.197 | INFO | RTMetrics report populated | 20260505 03:43:10.197 | INFO | Opening tmp RTMetrics json file | 20260505 03:43:10.198 | INFO | Serializing RTMetrics to json | 20260505 03:43:10.211 | INFO | Serialization done, swapping tmp file now... | 20260505 03:43:10.212 | INFO | RTMetrics done | 20260505 03:43:10.254 | INFO | Finished a round of Baz IO: Wrote 1.239 GiB in 3.034 seconds | 20260505 03:43:10.254 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 71.23% / 0.87% / 27.90% | 20260505 03:43:10.254 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 15.7569% / 0.00428359% / 3.97364e-06% | 20260505 03:43:10.254 | INFO | Baz IO overall: | 20260505 03:43:10.254 | INFO | BytesWritten : 8707109879 | 20260505 03:43:10.254 | INFO | NumEvents : 2743629799 | 20260505 03:43:12.261 | INFO | Analyzing chunk frames = [69632,70144) | 20260505 03:43:12.261 | INFO | MemInfo: { | 20260505 03:43:12.261 | INFO | "gpu" : " 17 GB", | 20260505 03:43:12.261 | INFO | "huge" : "", | 20260505 03:43:12.261 | INFO | "huge_shared" : " 27 GB", | 20260505 03:43:12.261 | INFO | "malloc" : " 15 GB", | 20260505 03:43:12.261 | INFO | "pinned" : " 29 GB" | 20260505 03:43:12.261 | INFO | } | 20260505 03:43:12.261 | INFO | Perf report for the last 5.13941 seconds: | 20260505 03:43:12.261 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:43:12.261 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:12.261 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:12.261 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:12.261 | INFO | ANALYSIS : 58.9%, 3.0, 91.9ms, 0 | 20260505 03:43:12.261 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:12.261 | INFO | RT_METRICS : 42.2%, 1.0, 22.1ms, 0 | 20260505 03:43:12.261 | INFO | PRE_HQ : 58.5%, 2.2, 65.4ms, 0 | 20260505 03:43:12.261 | INFO | BAZWRITER : 58.6%, 2.4, 73.6ms, 0 | 20260505 03:43:12.261 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 03:43:12.261 | INFO | Uploaded 36GiB to GPU in 1.63327s (22.0416GiB/s) | 20260505 03:43:12.262 | INFO | Downloaded 38GiB to GPU in 1.69499s (22.419GiB/s) | 20260505 03:43:17.400 | INFO | Analyzing chunk frames = [70144,70656) | 20260505 03:43:17.400 | INFO | MemInfo: { | 20260505 03:43:17.400 | INFO | "gpu" : " 17 GB", | 20260505 03:43:17.400 | INFO | "huge" : "", | 20260505 03:43:17.400 | INFO | "huge_shared" : " 27 GB", | 20260505 03:43:17.400 | INFO | "malloc" : " 15 GB", | 20260505 03:43:17.400 | INFO | "pinned" : " 29 GB" | 20260505 03:43:17.400 | INFO | } | 20260505 03:43:17.400 | INFO | Perf report for the last 5.13957 seconds: | 20260505 03:43:17.400 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:43:17.400 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:17.400 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:17.400 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:17.401 | INFO | ANALYSIS : 58.7%, 3.0, 91.5ms, 0 | 20260505 03:43:17.401 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:17.401 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:17.401 | INFO | PRE_HQ : 32.8%, 1.0, 17.1ms, 0 | 20260505 03:43:17.401 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:17.401 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:17.401 | INFO | Uploaded 36GiB to GPU in 1.59559s (22.5622GiB/s) | 20260505 03:43:17.401 | INFO | Downloaded 34GiB to GPU in 1.51128s (22.4974GiB/s) | 20260505 03:43:22.539 | INFO | Analyzing chunk frames = [70656,71168) | 20260505 03:43:22.539 | INFO | MemInfo: { | 20260505 03:43:22.539 | INFO | "gpu" : " 17 GB", | 20260505 03:43:22.539 | INFO | "huge" : "", | 20260505 03:43:22.539 | INFO | "huge_shared" : " 27 GB", | 20260505 03:43:22.540 | INFO | "malloc" : " 15 GB", | 20260505 03:43:22.540 | INFO | "pinned" : " 29 GB" | 20260505 03:43:22.540 | INFO | } | 20260505 03:43:22.540 | INFO | Perf report for the last 5.13926 seconds: | 20260505 03:43:22.540 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:43:22.540 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:22.540 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:22.540 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:22.540 | INFO | ANALYSIS : 58.1%, 3.0, 90.3ms, 0 | 20260505 03:43:22.540 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:22.540 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:22.540 | INFO | PRE_HQ : 32.6%, 1.0, 16.9ms, 0 | 20260505 03:43:22.540 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:22.540 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:22.540 | INFO | Uploaded 36GiB to GPU in 1.59791s (22.5295GiB/s) | 20260505 03:43:22.540 | INFO | Downloaded 34GiB to GPU in 1.51495s (22.443GiB/s) | 20260505 03:43:27.679 | INFO | Analyzing chunk frames = [71168,71680) | 20260505 03:43:27.679 | INFO | MemInfo: { | 20260505 03:43:27.679 | INFO | "gpu" : " 17 GB", | 20260505 03:43:27.679 | INFO | "huge" : "", | 20260505 03:43:27.679 | INFO | "huge_shared" : " 27 GB", | 20260505 03:43:27.679 | INFO | "malloc" : " 15 GB", | 20260505 03:43:27.679 | INFO | "pinned" : " 29 GB" | 20260505 03:43:27.679 | INFO | } | 20260505 03:43:27.679 | INFO | Perf report for the last 5.1393 seconds: | 20260505 03:43:27.679 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:43:27.679 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:27.679 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:27.679 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:27.679 | INFO | ANALYSIS : 57.7%, 3.0, 89.5ms, 0 | 20260505 03:43:27.679 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:27.679 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:27.679 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260505 03:43:27.679 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:27.679 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:27.680 | INFO | Uploaded 36GiB to GPU in 1.59752s (22.535GiB/s) | 20260505 03:43:27.680 | INFO | Downloaded 34GiB to GPU in 1.51458s (22.4485GiB/s) | 20260505 03:43:32.818 | INFO | Analyzing chunk frames = [71680,72192) | 20260505 03:43:32.818 | INFO | MemInfo: { | 20260505 03:43:32.818 | INFO | "gpu" : " 17 GB", | 20260505 03:43:32.818 | INFO | "huge" : "", | 20260505 03:43:32.818 | INFO | "huge_shared" : " 27 GB", | 20260505 03:43:32.818 | INFO | "malloc" : " 15 GB", | 20260505 03:43:32.818 | INFO | "pinned" : " 29 GB" | 20260505 03:43:32.818 | INFO | } | 20260505 03:43:32.818 | INFO | Perf report for the last 5.13933 seconds: | 20260505 03:43:32.818 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:43:32.818 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:32.818 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:32.818 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:32.818 | INFO | ANALYSIS : 57.6%, 3.0, 89.9ms, 0 | 20260505 03:43:32.818 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:32.818 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:32.818 | INFO | PRE_HQ : 32.8%, 1.0, 17.0ms, 0 | 20260505 03:43:32.819 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:32.819 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:32.819 | INFO | Uploaded 36GiB to GPU in 1.59623s (22.5531GiB/s) | 20260505 03:43:32.819 | INFO | Downloaded 34GiB to GPU in 1.51267s (22.4768GiB/s) | 20260505 03:43:37.957 | INFO | Analyzing chunk frames = [72192,72704) | 20260505 03:43:37.957 | INFO | MemInfo: { | 20260505 03:43:37.957 | INFO | "gpu" : " 17 GB", | 20260505 03:43:37.957 | INFO | "huge" : "", | 20260505 03:43:37.957 | INFO | "huge_shared" : " 27 GB", | 20260505 03:43:37.958 | INFO | "malloc" : " 15 GB", | 20260505 03:43:37.958 | INFO | "pinned" : " 29 GB" | 20260505 03:43:37.958 | INFO | } | 20260505 03:43:37.958 | INFO | Perf report for the last 5.13935 seconds: | 20260505 03:43:37.958 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:43:37.958 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:37.958 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:37.958 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:37.958 | INFO | ANALYSIS : 57.9%, 3.0, 90.3ms, 0 | 20260505 03:43:37.958 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:37.958 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:37.958 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 03:43:37.958 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:37.958 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:37.958 | INFO | Uploaded 36GiB to GPU in 1.59653s (22.549GiB/s) | 20260505 03:43:37.958 | INFO | Downloaded 34GiB to GPU in 1.51258s (22.4781GiB/s) | 20260505 03:43:43.097 | INFO | Analyzing chunk frames = [72704,73216) | 20260505 03:43:43.097 | INFO | MemInfo: { | 20260505 03:43:43.097 | INFO | "gpu" : " 17 GB", | 20260505 03:43:43.097 | INFO | "huge" : "", | 20260505 03:43:43.097 | INFO | "huge_shared" : " 27 GB", | 20260505 03:43:43.097 | INFO | "malloc" : " 15 GB", | 20260505 03:43:43.097 | INFO | "pinned" : " 29 GB" | 20260505 03:43:43.097 | INFO | } | 20260505 03:43:43.097 | INFO | Perf report for the last 5.13927 seconds: | 20260505 03:43:43.097 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:43:43.097 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:43.097 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:43.097 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:43.097 | INFO | ANALYSIS : 58.2%, 3.0, 90.8ms, 0 | 20260505 03:43:43.097 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:43.097 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:43.097 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260505 03:43:43.097 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:43.097 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:43.097 | INFO | Uploaded 36GiB to GPU in 1.59625s (22.5528GiB/s) | 20260505 03:43:43.097 | INFO | Downloaded 34GiB to GPU in 1.51267s (22.4768GiB/s) | 20260505 03:43:48.236 | INFO | Analyzing chunk frames = [73216,73728) | 20260505 03:43:48.236 | INFO | MemInfo: { | 20260505 03:43:48.236 | INFO | "gpu" : " 17 GB", | 20260505 03:43:48.236 | INFO | "huge" : "", | 20260505 03:43:48.236 | INFO | "huge_shared" : " 27 GB", | 20260505 03:43:48.236 | INFO | "malloc" : " 15 GB", | 20260505 03:43:48.236 | INFO | "pinned" : " 29 GB" | 20260505 03:43:48.236 | INFO | } | 20260505 03:43:48.236 | INFO | Perf report for the last 5.13925 seconds: | 20260505 03:43:48.236 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:43:48.236 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:48.236 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:48.236 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:48.236 | INFO | ANALYSIS : 58.3%, 3.0, 90.9ms, 0 | 20260505 03:43:48.236 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:48.236 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:48.236 | INFO | PRE_HQ : 31.9%, 1.0, 16.5ms, 0 | 20260505 03:43:48.236 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:48.236 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:48.237 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 03:43:48.237 | INFO | Total mem usage: 14.9 GiB | 20260505 03:43:48.237 | INFO | All Time High Water Marks sum: 15.2 GiB (Savings: 1.52%) | 20260505 03:43:48.237 | INFO | Recent High Water Marks Sum: 14.7 GiB (Savings: -1.49%) | 20260505 03:43:48.237 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 03:43:48.237 | INFO | High Water Breakdown: | 20260505 03:43:48.237 | INFO | 57.3% : 8.42 GiB ( 99.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 03:43:48.237 | INFO | 42.6% : 6.26 GiB ( 94.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 03:43:48.237 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 03:43:48.237 | INFO | ------------------------------------------ | 20260505 03:43:48.237 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:43:48.237 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 03:43:48.237 | INFO | Total mem usage: 26.5 GiB | 20260505 03:43:48.237 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260505 03:43:48.237 | INFO | Uploaded 36GiB to GPU in 1.59763s (22.5334GiB/s) | 20260505 03:43:48.237 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 03:43:48.237 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260505 03:43:48.237 | INFO | High Water Breakdown: | 20260505 03:43:48.237 | INFO | Downloaded 34GiB to GPU in 1.51435s (22.4519GiB/s) | 20260505 03:43:48.237 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 03:43:48.237 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 03:43:48.237 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 03:43:48.237 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 03:43:48.237 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 03:43:48.237 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 03:43:48.237 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 03:43:48.237 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 03:43:48.237 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 03:43:48.237 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 03:43:48.237 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 03:43:48.237 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 03:43:48.237 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 03:43:48.237 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 03:43:48.237 | INFO | ------------------------------------------ | 20260505 03:43:48.237 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:43:48.237 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 03:43:48.237 | INFO | Total mem usage: 25.6 GiB | 20260505 03:43:48.237 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 03:43:48.237 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 03:43:48.237 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 03:43:48.237 | INFO | High Water Breakdown: | 20260505 03:43:48.237 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 03:43:48.237 | INFO | ------------------------------------------ | 20260505 03:43:48.237 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:43:48.237 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 03:43:48.237 | INFO | Total mem usage: 15.7 GiB | 20260505 03:43:48.237 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260505 03:43:48.237 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 03:43:48.237 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260505 03:43:48.237 | INFO | High Water Breakdown: | 20260505 03:43:48.237 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 03:43:48.237 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 03:43:48.237 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 03:43:48.238 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 03:43:48.238 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 03:43:48.238 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 03:43:48.238 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 03:43:48.238 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 03:43:48.238 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 03:43:48.238 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 03:43:48.238 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 03:43:48.238 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 03:43:48.238 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 03:43:48.238 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 03:43:48.238 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 03:43:48.238 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 03:43:48.238 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 03:43:48.238 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 03:43:48.238 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 03:43:48.238 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 03:43:48.238 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 03:43:48.238 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 03:43:48.238 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 03:43:48.238 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 03:43:48.238 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 03:43:48.238 | INFO | ------------------------------------------ | 20260505 03:43:48.238 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:43:48.238 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 03:43:48.238 | INFO | PerfReport: Gathered 1536 samples. | 20260505 03:43:48.238 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 03:43:48.238 | INFO | PerfReport: Full Execution -- 3.08e+01 / -nan, 1.29e+01 / 0.00e+00 | 20260505 03:43:48.238 | INFO | PerfReport: 0.28%, AggregateStats -- 8.65e-02 / 0.00e+00, 4.54e-03 / 0.00e+00 | 20260505 03:43:48.238 | INFO | PerfReport: 10.71%, Baseline -- 3.30e+00 / 0.00e+00, 1.71e-02 / 0.00e+00 | 20260505 03:43:48.238 | INFO | PerfReport: 3.00%, Histogram -- 9.23e-01 / 0.00e+00, 5.25e-02 / 0.00e+00 | 20260505 03:43:48.238 | INFO | PerfReport: 16.04%, DME -- 3.95e+01 / 0.00e+00, 3.35e+00 / 0.00e+00 | 20260505 03:43:48.238 | INFO | PerfReport: 55.96%, FrameLabeling -- 1.72e+01 / 0.00e+00, 4.64e-02 / 0.00e+00 | 20260505 03:43:48.238 | INFO | PerfReport: 7.86%, PulseAccumulating -- 2.42e+00 / 0.00e+00, 1.65e-01 / 0.00e+00 | 20260505 03:43:48.238 | INFO | PerfReport: 4.33%, Metrics -- 1.33e+00 / 0.00e+00, 2.18e-01 / 0.00e+00 | 20260505 03:43:48.238 | INFO | PerfReport: 1.82%, PulseSerializing -- 5.61e-01 / 0.00e+00, 8.99e-02 / 0.00e+00 | 20260505 03:43:48.238 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 03:43:48.238 | INFO | PerfReport: Gathered 1536 samples. | 20260505 03:43:48.238 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 03:43:48.238 | INFO | PerfReport: Full Execution -- 9.25e+01 / -nan, 1.92e+01 / 0.00e+00 | 20260505 03:43:48.238 | INFO | PerfReport: 18.04%, Upload -- 1.67e+01 / 0.00e+00, 3.58e-01 / 0.00e+00 | 20260505 03:43:48.238 | INFO | PerfReport: 17.30%, Download -- 1.60e+01 / 0.00e+00, 6.82e-01 / 0.00e+00 | 20260505 03:43:48.238 | INFO | PerfReport: 33.29%, Compute -- 3.08e+01 / 0.00e+00, 1.29e+01 / 0.00e+00 | 20260505 03:43:48.238 | INFO | PerfReport: 0.55%, LockUpload -- 5.04e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260505 03:43:48.238 | INFO | PerfReport: 0.00%, LockDownload -- 4.00e-04 / 0.00e+00, 1.33e-04 / 0.00e+00 | 20260505 03:43:48.238 | INFO | PerfReport: 30.81%, LockCompute -- 2.85e+01 / 0.00e+00, 1.73e+01 / 0.00e+00 | 20260505 03:43:48.238 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.61e-02 / 0.00e+00 | 20260505 03:43:48.334 | INFO | Starting a round of Baz IO | 20260505 03:43:51.312 | INFO | Populating RTMetrics report | 20260505 03:43:51.312 | INFO | RTMetrics report populated | 20260505 03:43:51.312 | INFO | Opening tmp RTMetrics json file | 20260505 03:43:51.313 | INFO | Serializing RTMetrics to json | 20260505 03:43:51.326 | INFO | Serialization done, swapping tmp file now... | 20260505 03:43:51.329 | INFO | RTMetrics done | 20260505 03:43:51.369 | INFO | Finished a round of Baz IO: Wrote 1.277 GiB in 3.034 seconds | 20260505 03:43:51.369 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 70.41% / 0.82% / 28.77% | 20260505 03:43:51.369 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 15.9421% / 0.00415246% / 2.78155e-05% | 20260505 03:43:51.369 | INFO | Baz IO overall: | 20260505 03:43:51.369 | INFO | BytesWritten : 10077972856 | 20260505 03:43:51.369 | INFO | NumEvents : 3388806630 | 20260505 03:43:53.375 | INFO | Analyzing chunk frames = [73728,74240) | 20260505 03:43:53.375 | INFO | MemInfo: { | 20260505 03:43:53.375 | INFO | "gpu" : " 17 GB", | 20260505 03:43:53.375 | INFO | "huge" : "", | 20260505 03:43:53.375 | INFO | "huge_shared" : " 27 GB", | 20260505 03:43:53.375 | INFO | "malloc" : " 15 GB", | 20260505 03:43:53.375 | INFO | "pinned" : " 29 GB" | 20260505 03:43:53.375 | INFO | } | 20260505 03:43:53.375 | INFO | Perf report for the last 5.13919 seconds: | 20260505 03:43:53.376 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:43:53.376 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:53.376 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:53.376 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:53.376 | INFO | ANALYSIS : 58.9%, 3.0, 91.9ms, 0 | 20260505 03:43:53.376 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:53.376 | INFO | RT_METRICS : 42.2%, 1.0, 22.2ms, 0 | 20260505 03:43:53.376 | INFO | PRE_HQ : 58.5%, 2.2, 65.5ms, 0 | 20260505 03:43:53.376 | INFO | BAZWRITER : 58.6%, 2.5, 76.2ms, 0 | 20260505 03:43:53.376 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 03:43:53.376 | INFO | Uploaded 36GiB to GPU in 1.63085s (22.0744GiB/s) | 20260505 03:43:53.376 | INFO | Downloaded 38GiB to GPU in 1.69321s (22.4426GiB/s) | 20260505 03:43:58.515 | INFO | Analyzing chunk frames = [74240,74752) | 20260505 03:43:58.515 | INFO | MemInfo: { | 20260505 03:43:58.515 | INFO | "gpu" : " 17 GB", | 20260505 03:43:58.515 | INFO | "huge" : "", | 20260505 03:43:58.515 | INFO | "huge_shared" : " 27 GB", | 20260505 03:43:58.515 | INFO | "malloc" : " 15 GB", | 20260505 03:43:58.515 | INFO | "pinned" : " 29 GB" | 20260505 03:43:58.515 | INFO | } | 20260505 03:43:58.515 | INFO | Perf report for the last 5.13948 seconds: | 20260505 03:43:58.515 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:43:58.515 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:58.515 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:58.515 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:58.515 | INFO | ANALYSIS : 58.7%, 3.0, 91.6ms, 0 | 20260505 03:43:58.515 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:58.515 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:58.515 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260505 03:43:58.515 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:58.515 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:43:58.516 | INFO | Uploaded 36GiB to GPU in 1.5939s (22.5861GiB/s) | 20260505 03:43:58.516 | INFO | Downloaded 34GiB to GPU in 1.50991s (22.5178GiB/s) | 20260505 03:44:03.654 | INFO | Analyzing chunk frames = [74752,75264) | 20260505 03:44:03.654 | INFO | MemInfo: { | 20260505 03:44:03.654 | INFO | "gpu" : " 17 GB", | 20260505 03:44:03.654 | INFO | "huge" : "", | 20260505 03:44:03.654 | INFO | "huge_shared" : " 27 GB", | 20260505 03:44:03.654 | INFO | "malloc" : " 15 GB", | 20260505 03:44:03.654 | INFO | "pinned" : " 29 GB" | 20260505 03:44:03.654 | INFO | } | 20260505 03:44:03.654 | INFO | Perf report for the last 5.1395 seconds: | 20260505 03:44:03.654 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:44:03.654 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:03.655 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:03.655 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:03.655 | INFO | ANALYSIS : 58.1%, 3.0, 90.2ms, 0 | 20260505 03:44:03.655 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:03.655 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:03.655 | INFO | PRE_HQ : 32.6%, 1.0, 16.9ms, 0 | 20260505 03:44:03.655 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:03.655 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:03.655 | INFO | Uploaded 36GiB to GPU in 1.59923s (22.5108GiB/s) | 20260505 03:44:03.655 | INFO | Downloaded 34GiB to GPU in 1.51568s (22.4322GiB/s) | 20260505 03:44:08.793 | INFO | Analyzing chunk frames = [75264,75776) | 20260505 03:44:08.793 | INFO | MemInfo: { | 20260505 03:44:08.794 | INFO | "gpu" : " 17 GB", | 20260505 03:44:08.794 | INFO | "huge" : "", | 20260505 03:44:08.794 | INFO | "huge_shared" : " 27 GB", | 20260505 03:44:08.794 | INFO | "malloc" : " 15 GB", | 20260505 03:44:08.794 | INFO | "pinned" : " 29 GB" | 20260505 03:44:08.794 | INFO | } | 20260505 03:44:08.794 | INFO | Perf report for the last 5.13931 seconds: | 20260505 03:44:08.794 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:44:08.794 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:08.794 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:08.794 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:08.794 | INFO | ANALYSIS : 57.8%, 3.0, 89.6ms, 0 | 20260505 03:44:08.794 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:08.794 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:08.794 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260505 03:44:08.794 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:08.794 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:08.794 | INFO | Uploaded 36GiB to GPU in 1.59708s (22.5411GiB/s) | 20260505 03:44:08.794 | INFO | Downloaded 34GiB to GPU in 1.51348s (22.4648GiB/s) | 20260505 03:44:13.933 | INFO | Analyzing chunk frames = [75776,76288) | 20260505 03:44:13.933 | INFO | MemInfo: { | 20260505 03:44:13.933 | INFO | "gpu" : " 17 GB", | 20260505 03:44:13.933 | INFO | "huge" : "", | 20260505 03:44:13.933 | INFO | "huge_shared" : " 27 GB", | 20260505 03:44:13.933 | INFO | "malloc" : " 15 GB", | 20260505 03:44:13.933 | INFO | "pinned" : " 29 GB" | 20260505 03:44:13.933 | INFO | } | 20260505 03:44:13.933 | INFO | Perf report for the last 5.13947 seconds: | 20260505 03:44:13.933 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:44:13.933 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:13.933 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:13.933 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:13.933 | INFO | ANALYSIS : 57.6%, 3.0, 89.8ms, 0 | 20260505 03:44:13.933 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:13.933 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:13.933 | INFO | PRE_HQ : 32.6%, 1.0, 16.9ms, 0 | 20260505 03:44:13.933 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:13.933 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:13.933 | INFO | Uploaded 36GiB to GPU in 1.59614s (22.5544GiB/s) | 20260505 03:44:13.933 | INFO | Downloaded 34GiB to GPU in 1.51217s (22.4843GiB/s) | 20260505 03:44:19.072 | INFO | Analyzing chunk frames = [76288,76800) | 20260505 03:44:19.072 | INFO | MemInfo: { | 20260505 03:44:19.072 | INFO | "gpu" : " 17 GB", | 20260505 03:44:19.072 | INFO | "huge" : "", | 20260505 03:44:19.072 | INFO | "huge_shared" : " 27 GB", | 20260505 03:44:19.072 | INFO | "malloc" : " 15 GB", | 20260505 03:44:19.072 | INFO | "pinned" : " 29 GB" | 20260505 03:44:19.072 | INFO | } | 20260505 03:44:19.072 | INFO | Perf report for the last 5.13912 seconds: | 20260505 03:44:19.072 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:44:19.072 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:19.072 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:19.072 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:19.072 | INFO | ANALYSIS : 58.0%, 3.0, 90.4ms, 0 | 20260505 03:44:19.073 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:19.073 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:19.073 | INFO | PRE_HQ : 33.0%, 1.0, 17.1ms, 0 | 20260505 03:44:19.073 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:19.073 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:19.073 | INFO | Uploaded 36GiB to GPU in 1.59608s (22.5552GiB/s) | 20260505 03:44:19.073 | INFO | Downloaded 34GiB to GPU in 1.51233s (22.4819GiB/s) | 20260505 03:44:24.211 | INFO | Analyzing chunk frames = [76800,77312) | 20260505 03:44:24.211 | INFO | MemInfo: { | 20260505 03:44:24.212 | INFO | "gpu" : " 17 GB", | 20260505 03:44:24.212 | INFO | "huge" : "", | 20260505 03:44:24.212 | INFO | "huge_shared" : " 27 GB", | 20260505 03:44:24.212 | INFO | "malloc" : " 15 GB", | 20260505 03:44:24.212 | INFO | "pinned" : " 29 GB" | 20260505 03:44:24.212 | INFO | } | 20260505 03:44:24.212 | INFO | Perf report for the last 5.13933 seconds: | 20260505 03:44:24.212 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:44:24.212 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:24.212 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:24.212 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:24.212 | INFO | ANALYSIS : 58.1%, 3.0, 90.7ms, 0 | 20260505 03:44:24.212 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:24.212 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:24.212 | INFO | PRE_HQ : 32.8%, 1.0, 17.1ms, 0 | 20260505 03:44:24.212 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:24.212 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:24.212 | INFO | Uploaded 36GiB to GPU in 1.59646s (22.5499GiB/s) | 20260505 03:44:24.212 | INFO | Downloaded 34GiB to GPU in 1.51337s (22.4664GiB/s) | 20260505 03:44:29.351 | INFO | Analyzing chunk frames = [77312,77824) | 20260505 03:44:29.351 | INFO | MemInfo: { | 20260505 03:44:29.351 | INFO | "gpu" : " 17 GB", | 20260505 03:44:29.351 | INFO | "huge" : "", | 20260505 03:44:29.351 | INFO | "huge_shared" : " 27 GB", | 20260505 03:44:29.351 | INFO | "malloc" : " 15 GB", | 20260505 03:44:29.351 | INFO | "pinned" : " 29 GB" | 20260505 03:44:29.351 | INFO | } | 20260505 03:44:29.351 | INFO | Perf report for the last 5.13966 seconds: | 20260505 03:44:29.351 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:44:29.351 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:29.351 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:29.351 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:29.352 | INFO | ANALYSIS : 58.3%, 3.0, 90.9ms, 0 | 20260505 03:44:29.352 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:29.352 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:29.352 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 03:44:29.352 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:29.352 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:29.352 | INFO | Uploaded 36GiB to GPU in 1.59666s (22.5471GiB/s) | 20260505 03:44:29.352 | INFO | Downloaded 34GiB to GPU in 1.51302s (22.4716GiB/s) | 20260505 03:44:29.449 | INFO | Starting a round of Baz IO | 20260505 03:44:32.426 | INFO | Populating RTMetrics report | 20260505 03:44:32.427 | INFO | RTMetrics report populated | 20260505 03:44:32.427 | INFO | Opening tmp RTMetrics json file | 20260505 03:44:32.427 | INFO | Serializing RTMetrics to json | 20260505 03:44:32.429 | INFO | Serialization done, swapping tmp file now... | 20260505 03:44:32.429 | INFO | RTMetrics done | 20260505 03:44:32.484 | INFO | Finished a round of Baz IO: Wrote 1.307 GiB in 3.035 seconds | 20260505 03:44:32.485 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 69.60% / 0.80% / 29.59% | 20260505 03:44:32.485 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 15.9812% / 0.00383854% / 1.98682e-05% | 20260505 03:44:32.485 | INFO | Baz IO overall: | 20260505 03:44:32.485 | INFO | BytesWritten : 11480990670 | 20260505 03:44:32.485 | INFO | NumEvents : 4048541977 | 20260505 03:44:34.490 | INFO | Analyzing chunk frames = [77824,78336) | 20260505 03:44:34.490 | INFO | MemInfo: { | 20260505 03:44:34.490 | INFO | "gpu" : " 17 GB", | 20260505 03:44:34.490 | INFO | "huge" : "", | 20260505 03:44:34.490 | INFO | "huge_shared" : " 27 GB", | 20260505 03:44:34.490 | INFO | "malloc" : " 15 GB", | 20260505 03:44:34.490 | INFO | "pinned" : " 29 GB" | 20260505 03:44:34.490 | INFO | } | 20260505 03:44:34.490 | INFO | Perf report for the last 5.13904 seconds: | 20260505 03:44:34.490 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:44:34.490 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:34.490 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:34.491 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:34.491 | INFO | ANALYSIS : 58.9%, 3.0, 91.9ms, 0 | 20260505 03:44:34.491 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:34.491 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260505 03:44:34.491 | INFO | PRE_HQ : 58.4%, 2.2, 65.5ms, 0 | 20260505 03:44:34.491 | INFO | BAZWRITER : 58.6%, 2.6, 76.9ms, 0 | 20260505 03:44:34.491 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 03:44:34.491 | INFO | Uploaded 36GiB to GPU in 1.63247s (22.0525GiB/s) | 20260505 03:44:34.491 | INFO | Downloaded 38GiB to GPU in 1.69461s (22.424GiB/s) | 20260505 03:44:39.630 | INFO | Analyzing chunk frames = [78336,78848) | 20260505 03:44:39.630 | INFO | MemInfo: { | 20260505 03:44:39.630 | INFO | "gpu" : " 17 GB", | 20260505 03:44:39.630 | INFO | "huge" : "", | 20260505 03:44:39.630 | INFO | "huge_shared" : " 27 GB", | 20260505 03:44:39.630 | INFO | "malloc" : " 15 GB", | 20260505 03:44:39.630 | INFO | "pinned" : " 29 GB" | 20260505 03:44:39.630 | INFO | } | 20260505 03:44:39.630 | INFO | Perf report for the last 5.13934 seconds: | 20260505 03:44:39.630 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:44:39.630 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:39.630 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:39.630 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:39.630 | INFO | ANALYSIS : 58.8%, 3.0, 91.7ms, 0 | 20260505 03:44:39.630 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:39.630 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:39.630 | INFO | PRE_HQ : 33.3%, 1.0, 17.8ms, 0 | 20260505 03:44:39.630 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:39.630 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:39.630 | INFO | Uploaded 36GiB to GPU in 1.59387s (22.5865GiB/s) | 20260505 03:44:39.630 | INFO | Downloaded 34GiB to GPU in 1.50966s (22.5217GiB/s) | 20260505 03:44:44.769 | INFO | Analyzing chunk frames = [78848,79360) | 20260505 03:44:44.769 | INFO | MemInfo: { | 20260505 03:44:44.769 | INFO | "gpu" : " 17 GB", | 20260505 03:44:44.769 | INFO | "huge" : "", | 20260505 03:44:44.769 | INFO | "huge_shared" : " 27 GB", | 20260505 03:44:44.769 | INFO | "malloc" : " 15 GB", | 20260505 03:44:44.769 | INFO | "pinned" : " 29 GB" | 20260505 03:44:44.769 | INFO | } | 20260505 03:44:44.769 | INFO | Perf report for the last 5.13941 seconds: | 20260505 03:44:44.769 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:44:44.769 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:44.769 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:44.769 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:44.769 | INFO | ANALYSIS : 58.1%, 3.0, 90.2ms, 0 | 20260505 03:44:44.769 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:44.769 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:44.769 | INFO | PRE_HQ : 32.6%, 1.0, 16.9ms, 0 | 20260505 03:44:44.769 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:44.769 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:44.769 | INFO | Uploaded 36GiB to GPU in 1.5981s (22.5268GiB/s) | 20260505 03:44:44.770 | INFO | Downloaded 34GiB to GPU in 1.51467s (22.4472GiB/s) | 20260505 03:44:49.908 | INFO | Analyzing chunk frames = [79360,79872) | 20260505 03:44:49.908 | INFO | MemInfo: { | 20260505 03:44:49.908 | INFO | "gpu" : " 17 GB", | 20260505 03:44:49.908 | INFO | "huge" : "", | 20260505 03:44:49.908 | INFO | "huge_shared" : " 27 GB", | 20260505 03:44:49.908 | INFO | "malloc" : " 15 GB", | 20260505 03:44:49.908 | INFO | "pinned" : " 29 GB" | 20260505 03:44:49.908 | INFO | } | 20260505 03:44:49.909 | INFO | Perf report for the last 5.13926 seconds: | 20260505 03:44:49.909 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:44:49.909 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:49.909 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:49.909 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:49.909 | INFO | ANALYSIS : 57.8%, 3.0, 89.6ms, 0 | 20260505 03:44:49.909 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:49.909 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:49.909 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 03:44:49.909 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:49.909 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:49.909 | INFO | Uploaded 36GiB to GPU in 1.59757s (22.5342GiB/s) | 20260505 03:44:49.909 | INFO | Downloaded 34GiB to GPU in 1.51386s (22.4591GiB/s) | 20260505 03:44:55.048 | INFO | Analyzing chunk frames = [79872,80384) | 20260505 03:44:55.048 | INFO | MemInfo: { | 20260505 03:44:55.048 | INFO | "gpu" : " 17 GB", | 20260505 03:44:55.048 | INFO | "huge" : "", | 20260505 03:44:55.048 | INFO | "huge_shared" : " 27 GB", | 20260505 03:44:55.048 | INFO | "malloc" : " 15 GB", | 20260505 03:44:55.048 | INFO | "pinned" : " 29 GB" | 20260505 03:44:55.048 | INFO | } | 20260505 03:44:55.048 | INFO | Perf report for the last 5.13934 seconds: | 20260505 03:44:55.048 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:44:55.048 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:55.048 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:55.048 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:55.048 | INFO | ANALYSIS : 57.6%, 3.0, 89.8ms, 0 | 20260505 03:44:55.048 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:55.048 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:55.048 | INFO | PRE_HQ : 32.9%, 1.0, 17.1ms, 0 | 20260505 03:44:55.048 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:55.048 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:44:55.049 | INFO | Uploaded 36GiB to GPU in 1.59634s (22.5516GiB/s) | 20260505 03:44:55.049 | INFO | Downloaded 34GiB to GPU in 1.51201s (22.4866GiB/s) | 20260505 03:45:00.187 | INFO | Analyzing chunk frames = [80384,80896) | 20260505 03:45:00.187 | INFO | MemInfo: { | 20260505 03:45:00.187 | INFO | "gpu" : " 17 GB", | 20260505 03:45:00.187 | INFO | "huge" : "", | 20260505 03:45:00.187 | INFO | "huge_shared" : " 27 GB", | 20260505 03:45:00.187 | INFO | "malloc" : " 15 GB", | 20260505 03:45:00.187 | INFO | "pinned" : " 29 GB" | 20260505 03:45:00.187 | INFO | } | 20260505 03:45:00.187 | INFO | Perf report for the last 5.13929 seconds: | 20260505 03:45:00.187 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:45:00.187 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:00.187 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:00.187 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:00.187 | INFO | ANALYSIS : 57.9%, 3.0, 90.4ms, 0 | 20260505 03:45:00.187 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:00.187 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:00.187 | INFO | PRE_HQ : 32.9%, 1.0, 17.1ms, 0 | 20260505 03:45:00.187 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:00.187 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:00.188 | INFO | Uploaded 36GiB to GPU in 1.59564s (22.5615GiB/s) | 20260505 03:45:00.188 | INFO | Downloaded 34GiB to GPU in 1.51174s (22.4906GiB/s) | 20260505 03:45:05.326 | INFO | Analyzing chunk frames = [80896,81408) | 20260505 03:45:05.326 | INFO | MemInfo: { | 20260505 03:45:05.326 | INFO | "gpu" : " 17 GB", | 20260505 03:45:05.326 | INFO | "huge" : "", | 20260505 03:45:05.326 | INFO | "huge_shared" : " 27 GB", | 20260505 03:45:05.327 | INFO | "malloc" : " 15 GB", | 20260505 03:45:05.327 | INFO | "pinned" : " 29 GB" | 20260505 03:45:05.327 | INFO | } | 20260505 03:45:05.327 | INFO | Perf report for the last 5.13944 seconds: | 20260505 03:45:05.327 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:45:05.327 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:05.327 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:05.327 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:05.327 | INFO | ANALYSIS : 58.2%, 3.0, 90.7ms, 0 | 20260505 03:45:05.327 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:05.327 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:05.327 | INFO | PRE_HQ : 33.0%, 1.0, 17.1ms, 0 | 20260505 03:45:05.327 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:05.327 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:05.327 | INFO | Uploaded 36GiB to GPU in 1.59657s (22.5484GiB/s) | 20260505 03:45:05.327 | INFO | Downloaded 34GiB to GPU in 1.5131s (22.4705GiB/s) | 20260505 03:45:10.466 | INFO | Analyzing chunk frames = [81408,81920) | 20260505 03:45:10.466 | INFO | MemInfo: { | 20260505 03:45:10.466 | INFO | "gpu" : " 17 GB", | 20260505 03:45:10.466 | INFO | "huge" : "", | 20260505 03:45:10.466 | INFO | "huge_shared" : " 27 GB", | 20260505 03:45:10.466 | INFO | "malloc" : " 15 GB", | 20260505 03:45:10.466 | INFO | "pinned" : " 29 GB" | 20260505 03:45:10.466 | INFO | } | 20260505 03:45:10.466 | INFO | Perf report for the last 5.13926 seconds: | 20260505 03:45:10.466 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:45:10.466 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:10.466 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:10.466 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:10.466 | INFO | ANALYSIS : 58.3%, 3.0, 91.0ms, 0 | 20260505 03:45:10.466 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:10.466 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:10.466 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 03:45:10.466 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:10.466 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:10.466 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 03:45:10.466 | INFO | Total mem usage: 14.9 GiB | 20260505 03:45:10.466 | INFO | All Time High Water Marks sum: 15.2 GiB (Savings: 1.52%) | 20260505 03:45:10.466 | INFO | Recent High Water Marks Sum: 14.7 GiB (Savings: -1.66%) | 20260505 03:45:10.466 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 03:45:10.466 | INFO | High Water Breakdown: | 20260505 03:45:10.466 | INFO | 57.4% : 8.43 GiB ( 99.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 03:45:10.466 | INFO | 42.5% : 6.24 GiB ( 93.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 03:45:10.467 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 03:45:10.467 | INFO | ------------------------------------------ | 20260505 03:45:10.467 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:45:10.467 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 03:45:10.467 | INFO | Total mem usage: 26.5 GiB | 20260505 03:45:10.467 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260505 03:45:10.467 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 03:45:10.467 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260505 03:45:10.467 | INFO | High Water Breakdown: | 20260505 03:45:10.467 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 03:45:10.467 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 03:45:10.467 | INFO | Uploaded 36GiB to GPU in 1.59639s (22.5509GiB/s) | 20260505 03:45:10.467 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 03:45:10.467 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 03:45:10.467 | INFO | Downloaded 34GiB to GPU in 1.51382s (22.4598GiB/s) | 20260505 03:45:10.467 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 03:45:10.467 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 03:45:10.467 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 03:45:10.467 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 03:45:10.467 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 03:45:10.467 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 03:45:10.467 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 03:45:10.467 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 03:45:10.467 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 03:45:10.467 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 03:45:10.467 | INFO | ------------------------------------------ | 20260505 03:45:10.467 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:45:10.467 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 03:45:10.467 | INFO | Total mem usage: 25.6 GiB | 20260505 03:45:10.467 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 03:45:10.467 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 03:45:10.467 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 03:45:10.467 | INFO | High Water Breakdown: | 20260505 03:45:10.467 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 03:45:10.467 | INFO | ------------------------------------------ | 20260505 03:45:10.467 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:45:10.467 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 03:45:10.467 | INFO | Total mem usage: 15.7 GiB | 20260505 03:45:10.467 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260505 03:45:10.467 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 03:45:10.467 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260505 03:45:10.467 | INFO | High Water Breakdown: | 20260505 03:45:10.467 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 03:45:10.467 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 03:45:10.467 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 03:45:10.467 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 03:45:10.467 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 03:45:10.467 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 03:45:10.467 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 03:45:10.467 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 03:45:10.467 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 03:45:10.467 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 03:45:10.467 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 03:45:10.467 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 03:45:10.467 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 03:45:10.467 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 03:45:10.467 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 03:45:10.467 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 03:45:10.467 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 03:45:10.467 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 03:45:10.467 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 03:45:10.467 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 03:45:10.467 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 03:45:10.467 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 03:45:10.467 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 03:45:10.467 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 03:45:10.467 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 03:45:10.467 | INFO | ------------------------------------------ | 20260505 03:45:10.467 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:45:10.467 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 03:45:10.467 | INFO | PerfReport: Gathered 1536 samples. | 20260505 03:45:10.467 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 03:45:10.467 | INFO | PerfReport: Full Execution -- 3.08e+01 / -nan, 1.31e+01 / 0.00e+00 | 20260505 03:45:10.467 | INFO | PerfReport: 0.28%, AggregateStats -- 8.64e-02 / 0.00e+00, 4.78e-03 / 0.00e+00 | 20260505 03:45:10.467 | INFO | PerfReport: 10.70%, Baseline -- 3.30e+00 / 0.00e+00, 1.75e-02 / 0.00e+00 | 20260505 03:45:10.467 | INFO | PerfReport: 3.00%, Histogram -- 9.26e-01 / 0.00e+00, 5.27e-02 / 0.00e+00 | 20260505 03:45:10.467 | INFO | PerfReport: 16.02%, DME -- 3.95e+01 / 0.00e+00, 3.41e+00 / 0.00e+00 | 20260505 03:45:10.467 | INFO | PerfReport: 55.93%, FrameLabeling -- 1.72e+01 / 0.00e+00, 4.53e-02 / 0.00e+00 | 20260505 03:45:10.468 | INFO | PerfReport: 7.88%, PulseAccumulating -- 2.43e+00 / 0.00e+00, 1.66e-01 / 0.00e+00 | 20260505 03:45:10.468 | INFO | PerfReport: 4.35%, Metrics -- 1.34e+00 / 0.00e+00, 2.18e-01 / 0.00e+00 | 20260505 03:45:10.468 | INFO | PerfReport: 1.83%, PulseSerializing -- 5.63e-01 / 0.00e+00, 8.96e-02 / 0.00e+00 | 20260505 03:45:10.468 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 03:45:10.468 | INFO | PerfReport: Gathered 1536 samples. | 20260505 03:45:10.468 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 03:45:10.468 | INFO | PerfReport: Full Execution -- 9.25e+01 / -nan, 1.93e+01 / 0.00e+00 | 20260505 03:45:10.468 | INFO | PerfReport: 18.02%, Upload -- 1.67e+01 / 0.00e+00, 3.52e-01 / 0.00e+00 | 20260505 03:45:10.468 | INFO | PerfReport: 17.28%, Download -- 1.60e+01 / 0.00e+00, 6.77e-01 / 0.00e+00 | 20260505 03:45:10.468 | INFO | PerfReport: 33.29%, Compute -- 3.08e+01 / 0.00e+00, 1.31e+01 / 0.00e+00 | 20260505 03:45:10.468 | INFO | PerfReport: 0.55%, LockUpload -- 5.05e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260505 03:45:10.468 | INFO | PerfReport: 0.00%, LockDownload -- 3.96e-04 / 0.00e+00, 1.41e-04 / 0.00e+00 | 20260505 03:45:10.468 | INFO | PerfReport: 30.84%, LockCompute -- 2.85e+01 / 0.00e+00, 1.74e+01 / 0.00e+00 | 20260505 03:45:10.468 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.42e-02 / 0.00e+00 | 20260505 03:45:10.564 | INFO | Starting a round of Baz IO | 20260505 03:45:13.541 | INFO | Populating RTMetrics report | 20260505 03:45:13.542 | INFO | RTMetrics report populated | 20260505 03:45:13.542 | INFO | Opening tmp RTMetrics json file | 20260505 03:45:13.543 | INFO | Serializing RTMetrics to json | 20260505 03:45:13.544 | INFO | Serialization done, swapping tmp file now... | 20260505 03:45:13.545 | INFO | RTMetrics done | 20260505 03:45:13.601 | INFO | Finished a round of Baz IO: Wrote 1.332 GiB in 3.036 seconds | 20260505 03:45:13.601 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 68.82% / 0.78% / 30.40% | 20260505 03:45:13.601 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 15.9443% / 0.00369549% / 7.94729e-06% | 20260505 03:45:13.601 | INFO | Baz IO overall: | 20260505 03:45:13.601 | INFO | BytesWritten : 12911162149 | 20260505 03:45:13.601 | INFO | NumEvents : 4721158208 | 20260505 03:45:15.605 | INFO | Analyzing chunk frames = [81920,82432) | 20260505 03:45:15.605 | INFO | MemInfo: { | 20260505 03:45:15.605 | INFO | "gpu" : " 17 GB", | 20260505 03:45:15.605 | INFO | "huge" : "", | 20260505 03:45:15.605 | INFO | "huge_shared" : " 27 GB", | 20260505 03:45:15.605 | INFO | "malloc" : " 15 GB", | 20260505 03:45:15.605 | INFO | "pinned" : " 29 GB" | 20260505 03:45:15.605 | INFO | } | 20260505 03:45:15.605 | INFO | Perf report for the last 5.13921 seconds: | 20260505 03:45:15.605 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:45:15.605 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:15.605 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:15.605 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:15.605 | INFO | ANALYSIS : 58.9%, 3.0, 91.9ms, 0 | 20260505 03:45:15.605 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:15.605 | INFO | RT_METRICS : 42.2%, 1.0, 22.1ms, 0 | 20260505 03:45:15.605 | INFO | PRE_HQ : 58.4%, 2.2, 65.4ms, 0 | 20260505 03:45:15.605 | INFO | BAZWRITER : 58.4%, 2.6, 77.3ms, 0 | 20260505 03:45:15.605 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 03:45:15.606 | INFO | Uploaded 36GiB to GPU in 1.62558s (22.146GiB/s) | 20260505 03:45:15.606 | INFO | Downloaded 38GiB to GPU in 1.689s (22.4985GiB/s) | 20260505 03:45:20.744 | INFO | Analyzing chunk frames = [82432,82944) | 20260505 03:45:20.744 | INFO | MemInfo: { | 20260505 03:45:20.744 | INFO | "gpu" : " 17 GB", | 20260505 03:45:20.744 | INFO | "huge" : "", | 20260505 03:45:20.744 | INFO | "huge_shared" : " 27 GB", | 20260505 03:45:20.745 | INFO | "malloc" : " 15 GB", | 20260505 03:45:20.745 | INFO | "pinned" : " 29 GB" | 20260505 03:45:20.745 | INFO | } | 20260505 03:45:20.745 | INFO | Perf report for the last 5.13951 seconds: | 20260505 03:45:20.745 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:45:20.745 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:20.745 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:20.745 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:20.745 | INFO | ANALYSIS : 58.7%, 3.0, 91.6ms, 0 | 20260505 03:45:20.745 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:20.745 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:20.745 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260505 03:45:20.745 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:20.745 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:20.745 | INFO | Uploaded 36GiB to GPU in 1.59374s (22.5884GiB/s) | 20260505 03:45:20.745 | INFO | Downloaded 34GiB to GPU in 1.51031s (22.512GiB/s) | 20260505 03:45:25.884 | INFO | Analyzing chunk frames = [82944,83456) | 20260505 03:45:25.884 | INFO | MemInfo: { | 20260505 03:45:25.884 | INFO | "gpu" : " 17 GB", | 20260505 03:45:25.884 | INFO | "huge" : "", | 20260505 03:45:25.884 | INFO | "huge_shared" : " 27 GB", | 20260505 03:45:25.884 | INFO | "malloc" : " 15 GB", | 20260505 03:45:25.884 | INFO | "pinned" : " 29 GB" | 20260505 03:45:25.884 | INFO | } | 20260505 03:45:25.884 | INFO | Perf report for the last 5.13932 seconds: | 20260505 03:45:25.884 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:45:25.884 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:25.884 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:25.884 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:25.884 | INFO | ANALYSIS : 58.1%, 3.0, 90.2ms, 0 | 20260505 03:45:25.884 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:25.884 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:25.884 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 03:45:25.884 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:25.884 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:25.884 | INFO | Uploaded 36GiB to GPU in 1.59771s (22.5322GiB/s) | 20260505 03:45:25.884 | INFO | Downloaded 34GiB to GPU in 1.51447s (22.4501GiB/s) | 20260505 03:45:31.023 | INFO | Analyzing chunk frames = [83456,83968) | 20260505 03:45:31.023 | INFO | MemInfo: { | 20260505 03:45:31.023 | INFO | "gpu" : " 17 GB", | 20260505 03:45:31.023 | INFO | "huge" : "", | 20260505 03:45:31.023 | INFO | "huge_shared" : " 27 GB", | 20260505 03:45:31.023 | INFO | "malloc" : " 15 GB", | 20260505 03:45:31.023 | INFO | "pinned" : " 29 GB" | 20260505 03:45:31.023 | INFO | } | 20260505 03:45:31.023 | INFO | Perf report for the last 5.1394 seconds: | 20260505 03:45:31.023 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:45:31.023 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:31.023 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:31.023 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:31.023 | INFO | ANALYSIS : 57.7%, 3.0, 89.6ms, 0 | 20260505 03:45:31.024 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:31.024 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:31.024 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260505 03:45:31.024 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:31.024 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:31.024 | INFO | Uploaded 36GiB to GPU in 1.59728s (22.5383GiB/s) | 20260505 03:45:31.024 | INFO | Downloaded 34GiB to GPU in 1.51308s (22.4708GiB/s) | 20260505 03:45:36.162 | INFO | Analyzing chunk frames = [83968,84480) | 20260505 03:45:36.162 | INFO | MemInfo: { | 20260505 03:45:36.162 | INFO | "gpu" : " 17 GB", | 20260505 03:45:36.162 | INFO | "huge" : "", | 20260505 03:45:36.162 | INFO | "huge_shared" : " 27 GB", | 20260505 03:45:36.162 | INFO | "malloc" : " 15 GB", | 20260505 03:45:36.162 | INFO | "pinned" : " 29 GB" | 20260505 03:45:36.163 | INFO | } | 20260505 03:45:36.163 | INFO | Perf report for the last 5.13932 seconds: | 20260505 03:45:36.163 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:45:36.163 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:36.163 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:36.163 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:36.163 | INFO | ANALYSIS : 57.7%, 3.0, 89.9ms, 0 | 20260505 03:45:36.163 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:36.163 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:36.163 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 03:45:36.163 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:36.163 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:36.163 | INFO | Uploaded 36GiB to GPU in 1.5956s (22.562GiB/s) | 20260505 03:45:36.163 | INFO | Downloaded 34GiB to GPU in 1.51145s (22.4949GiB/s) | 20260505 03:45:41.302 | INFO | Analyzing chunk frames = [84480,84992) | 20260505 03:45:41.302 | INFO | MemInfo: { | 20260505 03:45:41.302 | INFO | "gpu" : " 17 GB", | 20260505 03:45:41.302 | INFO | "huge" : "", | 20260505 03:45:41.302 | INFO | "huge_shared" : " 27 GB", | 20260505 03:45:41.302 | INFO | "malloc" : " 15 GB", | 20260505 03:45:41.302 | INFO | "pinned" : " 29 GB" | 20260505 03:45:41.302 | INFO | } | 20260505 03:45:41.302 | INFO | Perf report for the last 5.13932 seconds: | 20260505 03:45:41.302 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:45:41.302 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:41.302 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:41.302 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:41.302 | INFO | ANALYSIS : 57.9%, 3.0, 90.4ms, 0 | 20260505 03:45:41.302 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:41.302 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:41.302 | INFO | PRE_HQ : 32.8%, 1.0, 17.1ms, 0 | 20260505 03:45:41.302 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:41.302 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:41.303 | INFO | Uploaded 36GiB to GPU in 1.59628s (22.5525GiB/s) | 20260505 03:45:41.303 | INFO | Downloaded 34GiB to GPU in 1.51282s (22.4746GiB/s) | 20260505 03:45:46.441 | INFO | Analyzing chunk frames = [84992,85504) | 20260505 03:45:46.441 | INFO | MemInfo: { | 20260505 03:45:46.441 | INFO | "gpu" : " 17 GB", | 20260505 03:45:46.441 | INFO | "huge" : "", | 20260505 03:45:46.441 | INFO | "huge_shared" : " 27 GB", | 20260505 03:45:46.441 | INFO | "malloc" : " 15 GB", | 20260505 03:45:46.441 | INFO | "pinned" : " 29 GB" | 20260505 03:45:46.441 | INFO | } | 20260505 03:45:46.441 | INFO | Perf report for the last 5.13929 seconds: | 20260505 03:45:46.441 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:45:46.441 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:46.441 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:46.441 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:46.441 | INFO | ANALYSIS : 58.2%, 3.0, 90.9ms, 0 | 20260505 03:45:46.442 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:46.442 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:46.442 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260505 03:45:46.442 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:46.442 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:46.442 | INFO | Uploaded 36GiB to GPU in 1.59652s (22.549GiB/s) | 20260505 03:45:46.442 | INFO | Downloaded 34GiB to GPU in 1.51313s (22.4699GiB/s) | 20260505 03:45:51.580 | INFO | Analyzing chunk frames = [85504,86016) | 20260505 03:45:51.580 | INFO | MemInfo: { | 20260505 03:45:51.580 | INFO | "gpu" : " 17 GB", | 20260505 03:45:51.580 | INFO | "huge" : "", | 20260505 03:45:51.580 | INFO | "huge_shared" : " 27 GB", | 20260505 03:45:51.580 | INFO | "malloc" : " 15 GB", | 20260505 03:45:51.581 | INFO | "pinned" : " 29 GB" | 20260505 03:45:51.581 | INFO | } | 20260505 03:45:51.581 | INFO | Perf report for the last 5.13934 seconds: | 20260505 03:45:51.581 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:45:51.581 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:51.581 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:51.581 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:51.581 | INFO | ANALYSIS : 58.3%, 3.0, 90.9ms, 0 | 20260505 03:45:51.581 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:51.581 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:51.581 | INFO | PRE_HQ : 31.7%, 1.0, 16.4ms, 0 | 20260505 03:45:51.581 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:51.581 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:51.581 | INFO | Uploaded 36GiB to GPU in 1.59663s (22.5475GiB/s) | 20260505 03:45:51.581 | INFO | Downloaded 34GiB to GPU in 1.51386s (22.4592GiB/s) | 20260505 03:45:51.678 | INFO | Starting a round of Baz IO | 20260505 03:45:54.663 | INFO | Populating RTMetrics report | 20260505 03:45:54.663 | INFO | RTMetrics report populated | 20260505 03:45:54.663 | INFO | Opening tmp RTMetrics json file | 20260505 03:45:54.664 | INFO | Serializing RTMetrics to json | 20260505 03:45:54.665 | INFO | Serialization done, swapping tmp file now... | 20260505 03:45:54.667 | INFO | RTMetrics done | 20260505 03:45:54.724 | INFO | Finished a round of Baz IO: Wrote 1.362 GiB in 3.045 seconds | 20260505 03:45:54.724 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 68.07% / 0.75% / 31.18% | 20260505 03:45:54.724 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 15.975% / 0.00362396% / 1.19209e-05% | 20260505 03:45:54.724 | INFO | Baz IO overall: | 20260505 03:45:54.724 | INFO | BytesWritten : 14373586302 | 20260505 03:45:54.724 | INFO | NumEvents : 5408352543 | 20260505 03:45:56.720 | INFO | Analyzing chunk frames = [86016,86528) | 20260505 03:45:56.720 | INFO | MemInfo: { | 20260505 03:45:56.720 | INFO | "gpu" : " 17 GB", | 20260505 03:45:56.720 | INFO | "huge" : "", | 20260505 03:45:56.720 | INFO | "huge_shared" : " 27 GB", | 20260505 03:45:56.720 | INFO | "malloc" : " 15 GB", | 20260505 03:45:56.720 | INFO | "pinned" : " 29 GB" | 20260505 03:45:56.720 | INFO | } | 20260505 03:45:56.720 | INFO | Perf report for the last 5.13938 seconds: | 20260505 03:45:56.720 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:45:56.720 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:56.720 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:56.720 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:56.720 | INFO | ANALYSIS : 59.0%, 3.0, 92.1ms, 0 | 20260505 03:45:56.720 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:45:56.720 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260505 03:45:56.720 | INFO | PRE_HQ : 58.5%, 2.2, 65.3ms, 0 | 20260505 03:45:56.720 | INFO | BAZWRITER : 58.8%, 2.6, 79.4ms, 0 | 20260505 03:45:56.720 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 03:45:56.720 | INFO | Uploaded 36GiB to GPU in 1.62947s (22.0931GiB/s) | 20260505 03:45:56.721 | INFO | Downloaded 38GiB to GPU in 1.69196s (22.4592GiB/s) | 20260505 03:46:01.859 | INFO | Analyzing chunk frames = [86528,87040) | 20260505 03:46:01.859 | INFO | MemInfo: { | 20260505 03:46:01.859 | INFO | "gpu" : " 17 GB", | 20260505 03:46:01.859 | INFO | "huge" : "", | 20260505 03:46:01.859 | INFO | "huge_shared" : " 27 GB", | 20260505 03:46:01.859 | INFO | "malloc" : " 15 GB", | 20260505 03:46:01.859 | INFO | "pinned" : " 29 GB" | 20260505 03:46:01.859 | INFO | } | 20260505 03:46:01.859 | INFO | Perf report for the last 5.13912 seconds: | 20260505 03:46:01.859 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:46:01.859 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:01.859 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:01.859 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:01.859 | INFO | ANALYSIS : 58.7%, 3.0, 91.5ms, 0 | 20260505 03:46:01.860 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:01.860 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:01.860 | INFO | PRE_HQ : 32.7%, 1.0, 17.0ms, 0 | 20260505 03:46:01.860 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:01.860 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:01.860 | INFO | Uploaded 36GiB to GPU in 1.59504s (22.5699GiB/s) | 20260505 03:46:01.860 | INFO | Downloaded 34GiB to GPU in 1.51068s (22.5064GiB/s) | 20260505 03:46:06.998 | INFO | Analyzing chunk frames = [87040,87552) | 20260505 03:46:06.998 | INFO | MemInfo: { | 20260505 03:46:06.998 | INFO | "gpu" : " 17 GB", | 20260505 03:46:06.998 | INFO | "huge" : "", | 20260505 03:46:06.998 | INFO | "huge_shared" : " 27 GB", | 20260505 03:46:06.998 | INFO | "malloc" : " 15 GB", | 20260505 03:46:06.999 | INFO | "pinned" : " 29 GB" | 20260505 03:46:06.999 | INFO | } | 20260505 03:46:06.999 | INFO | Perf report for the last 5.13935 seconds: | 20260505 03:46:06.999 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:46:06.999 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:06.999 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:06.999 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:06.999 | INFO | ANALYSIS : 58.1%, 3.0, 90.3ms, 0 | 20260505 03:46:06.999 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:06.999 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:06.999 | INFO | PRE_HQ : 32.7%, 1.0, 17.0ms, 0 | 20260505 03:46:06.999 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:06.999 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:06.999 | INFO | Uploaded 36GiB to GPU in 1.59676s (22.5456GiB/s) | 20260505 03:46:06.999 | INFO | Downloaded 34GiB to GPU in 1.51436s (22.4517GiB/s) | 20260505 03:46:12.138 | INFO | Analyzing chunk frames = [87552,88064) | 20260505 03:46:12.138 | INFO | MemInfo: { | 20260505 03:46:12.138 | INFO | "gpu" : " 17 GB", | 20260505 03:46:12.138 | INFO | "huge" : "", | 20260505 03:46:12.138 | INFO | "huge_shared" : " 27 GB", | 20260505 03:46:12.138 | INFO | "malloc" : " 15 GB", | 20260505 03:46:12.138 | INFO | "pinned" : " 29 GB" | 20260505 03:46:12.138 | INFO | } | 20260505 03:46:12.138 | INFO | Perf report for the last 5.13923 seconds: | 20260505 03:46:12.138 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:46:12.138 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:12.138 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:12.138 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:12.138 | INFO | ANALYSIS : 57.7%, 3.0, 89.5ms, 0 | 20260505 03:46:12.138 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:12.138 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:12.138 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 03:46:12.138 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:12.138 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:12.138 | INFO | Uploaded 36GiB to GPU in 1.5965s (22.5494GiB/s) | 20260505 03:46:12.138 | INFO | Downloaded 34GiB to GPU in 1.51355s (22.4638GiB/s) | 20260505 03:46:17.277 | INFO | Analyzing chunk frames = [88064,88576) | 20260505 03:46:17.277 | INFO | MemInfo: { | 20260505 03:46:17.277 | INFO | "gpu" : " 17 GB", | 20260505 03:46:17.277 | INFO | "huge" : "", | 20260505 03:46:17.277 | INFO | "huge_shared" : " 27 GB", | 20260505 03:46:17.277 | INFO | "malloc" : " 15 GB", | 20260505 03:46:17.277 | INFO | "pinned" : " 29 GB" | 20260505 03:46:17.277 | INFO | } | 20260505 03:46:17.277 | INFO | Perf report for the last 5.13949 seconds: | 20260505 03:46:17.277 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:46:17.277 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:17.277 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:17.277 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:17.277 | INFO | ANALYSIS : 57.6%, 3.0, 89.9ms, 0 | 20260505 03:46:17.277 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:17.277 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:17.277 | INFO | PRE_HQ : 32.8%, 1.0, 17.0ms, 0 | 20260505 03:46:17.278 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:17.278 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:17.278 | INFO | Uploaded 36GiB to GPU in 1.59561s (22.562GiB/s) | 20260505 03:46:17.278 | INFO | Downloaded 34GiB to GPU in 1.51232s (22.482GiB/s) | 20260505 03:46:22.417 | INFO | Analyzing chunk frames = [88576,89088) | 20260505 03:46:22.417 | INFO | MemInfo: { | 20260505 03:46:22.417 | INFO | "gpu" : " 17 GB", | 20260505 03:46:22.417 | INFO | "huge" : "", | 20260505 03:46:22.417 | INFO | "huge_shared" : " 27 GB", | 20260505 03:46:22.417 | INFO | "malloc" : " 15 GB", | 20260505 03:46:22.417 | INFO | "pinned" : " 29 GB" | 20260505 03:46:22.417 | INFO | } | 20260505 03:46:22.417 | INFO | Perf report for the last 5.13941 seconds: | 20260505 03:46:22.417 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:46:22.417 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:22.417 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:22.417 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:22.417 | INFO | ANALYSIS : 57.9%, 3.0, 90.3ms, 0 | 20260505 03:46:22.417 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:22.417 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:22.417 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260505 03:46:22.417 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:22.417 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:22.417 | INFO | Uploaded 36GiB to GPU in 1.59618s (22.5538GiB/s) | 20260505 03:46:22.417 | INFO | Downloaded 34GiB to GPU in 1.5126s (22.4779GiB/s) | 20260505 03:46:27.556 | INFO | Analyzing chunk frames = [89088,89600) | 20260505 03:46:27.556 | INFO | MemInfo: { | 20260505 03:46:27.556 | INFO | "gpu" : " 17 GB", | 20260505 03:46:27.556 | INFO | "huge" : "", | 20260505 03:46:27.556 | INFO | "huge_shared" : " 27 GB", | 20260505 03:46:27.556 | INFO | "malloc" : " 15 GB", | 20260505 03:46:27.556 | INFO | "pinned" : " 29 GB" | 20260505 03:46:27.556 | INFO | } | 20260505 03:46:27.556 | INFO | Perf report for the last 5.1394 seconds: | 20260505 03:46:27.556 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:46:27.556 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:27.556 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:27.556 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:27.556 | INFO | ANALYSIS : 58.1%, 3.0, 90.7ms, 0 | 20260505 03:46:27.556 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:27.556 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:27.556 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260505 03:46:27.556 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:27.556 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:27.556 | INFO | Uploaded 36GiB to GPU in 1.59627s (22.5526GiB/s) | 20260505 03:46:27.557 | INFO | Downloaded 34GiB to GPU in 1.51308s (22.4707GiB/s) | 20260505 03:46:32.695 | INFO | Analyzing chunk frames = [89600,90112) | 20260505 03:46:32.695 | INFO | MemInfo: { | 20260505 03:46:32.695 | INFO | "gpu" : " 17 GB", | 20260505 03:46:32.695 | INFO | "huge" : "", | 20260505 03:46:32.695 | INFO | "huge_shared" : " 27 GB", | 20260505 03:46:32.695 | INFO | "malloc" : " 15 GB", | 20260505 03:46:32.695 | INFO | "pinned" : " 29 GB" | 20260505 03:46:32.695 | INFO | } | 20260505 03:46:32.695 | INFO | Perf report for the last 5.13907 seconds: | 20260505 03:46:32.695 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:46:32.695 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:32.695 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:32.695 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:32.696 | INFO | ANALYSIS : 58.2%, 3.0, 90.8ms, 0 | 20260505 03:46:32.696 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:32.696 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:32.696 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260505 03:46:32.696 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:32.696 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:32.696 | INFO | Uploaded 36GiB to GPU in 1.59753s (22.5348GiB/s) | 20260505 03:46:32.696 | INFO | Downloaded 34GiB to GPU in 1.51393s (22.4581GiB/s) | 20260505 03:46:32.696 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 03:46:32.696 | INFO | Total mem usage: 14.9 GiB | 20260505 03:46:32.696 | INFO | All Time High Water Marks sum: 15.2 GiB (Savings: 1.52%) | 20260505 03:46:32.696 | INFO | Recent High Water Marks Sum: 14.7 GiB (Savings: -1.85%) | 20260505 03:46:32.696 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 03:46:32.696 | INFO | High Water Breakdown: | 20260505 03:46:32.696 | INFO | 57.5% : 8.42 GiB ( 99.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 03:46:32.696 | INFO | 42.4% : 6.21 GiB ( 93.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 03:46:32.696 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 03:46:32.696 | INFO | ------------------------------------------ | 20260505 03:46:32.696 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:46:32.696 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 03:46:32.696 | INFO | Total mem usage: 26.5 GiB | 20260505 03:46:32.696 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260505 03:46:32.696 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 03:46:32.696 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260505 03:46:32.696 | INFO | High Water Breakdown: | 20260505 03:46:32.696 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 03:46:32.696 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 03:46:32.696 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 03:46:32.696 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 03:46:32.696 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 03:46:32.696 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 03:46:32.696 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 03:46:32.696 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 03:46:32.696 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 03:46:32.696 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 03:46:32.696 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 03:46:32.696 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 03:46:32.696 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 03:46:32.696 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 03:46:32.696 | INFO | ------------------------------------------ | 20260505 03:46:32.696 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:46:32.696 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 03:46:32.697 | INFO | Total mem usage: 25.6 GiB | 20260505 03:46:32.697 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 03:46:32.697 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 03:46:32.697 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 03:46:32.697 | INFO | High Water Breakdown: | 20260505 03:46:32.697 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 03:46:32.697 | INFO | ------------------------------------------ | 20260505 03:46:32.697 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:46:32.697 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 03:46:32.697 | INFO | Total mem usage: 15.7 GiB | 20260505 03:46:32.697 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260505 03:46:32.697 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 03:46:32.697 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260505 03:46:32.697 | INFO | High Water Breakdown: | 20260505 03:46:32.697 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 03:46:32.697 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 03:46:32.697 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 03:46:32.697 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 03:46:32.697 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 03:46:32.697 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 03:46:32.697 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 03:46:32.697 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 03:46:32.697 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 03:46:32.697 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 03:46:32.697 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 03:46:32.697 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 03:46:32.697 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 03:46:32.697 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 03:46:32.697 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 03:46:32.697 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 03:46:32.697 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 03:46:32.697 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 03:46:32.697 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 03:46:32.697 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 03:46:32.697 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 03:46:32.697 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 03:46:32.697 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 03:46:32.697 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 03:46:32.697 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 03:46:32.697 | INFO | ------------------------------------------ | 20260505 03:46:32.697 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:46:32.697 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 03:46:32.697 | INFO | PerfReport: Gathered 1536 samples. | 20260505 03:46:32.697 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 03:46:32.697 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.07e+01, 1.30e+01 / 0.00e+00 | 20260505 03:46:32.697 | INFO | PerfReport: 0.28%, AggregateStats -- 8.62e-02 / 8.71e-02, 3.97e-03 / 0.00e+00 | 20260505 03:46:32.697 | INFO | PerfReport: 10.70%, Baseline -- 3.30e+00 / 3.32e+00, 1.63e-02 / 0.00e+00 | 20260505 03:46:32.697 | INFO | PerfReport: 3.01%, Histogram -- 9.28e-01 / 1.07e+00, 5.27e-02 / 0.00e+00 | 20260505 03:46:32.697 | INFO | PerfReport: 15.99%, DME -- 3.94e+01 / 4.05e+01, 3.52e+00 / 0.00e+00 | 20260505 03:46:32.697 | INFO | PerfReport: 55.94%, FrameLabeling -- 1.72e+01 / 2.10e+01, 1.05e-01 / 0.00e+00 | 20260505 03:46:32.697 | INFO | PerfReport: 7.89%, PulseAccumulating -- 2.43e+00 / 2.50e+00, 1.67e-01 / 0.00e+00 | 20260505 03:46:32.697 | INFO | PerfReport: 4.36%, Metrics -- 1.34e+00 / 1.62e+00, 2.20e-01 / 0.00e+00 | 20260505 03:46:32.697 | INFO | PerfReport: 1.83%, PulseSerializing -- 5.63e-01 / 6.29e-01, 9.04e-02 / 0.00e+00 | 20260505 03:46:32.697 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 03:46:32.697 | INFO | PerfReport: Gathered 1536 samples. | 20260505 03:46:32.697 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 03:46:32.697 | INFO | PerfReport: Full Execution -- 9.25e+01 / -nan, 1.94e+01 / 0.00e+00 | 20260505 03:46:32.697 | INFO | PerfReport: 18.01%, Upload -- 1.67e+01 / 0.00e+00, 3.45e-01 / 0.00e+00 | 20260505 03:46:32.697 | INFO | PerfReport: 17.28%, Download -- 1.60e+01 / 0.00e+00, 6.64e-01 / 0.00e+00 | 20260505 03:46:32.697 | INFO | PerfReport: 33.29%, Compute -- 3.08e+01 / 0.00e+00, 1.30e+01 / 0.00e+00 | 20260505 03:46:32.697 | INFO | PerfReport: 0.55%, LockUpload -- 5.05e-01 / 0.00e+00, 3.63e+00 / 0.00e+00 | 20260505 03:46:32.697 | INFO | PerfReport: 0.00%, LockDownload -- 3.97e-04 / 0.00e+00, 1.61e-04 / 0.00e+00 | 20260505 03:46:32.697 | INFO | PerfReport: 30.86%, LockCompute -- 2.86e+01 / 0.00e+00, 1.74e+01 / 0.00e+00 | 20260505 03:46:32.697 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.42e-02 / 0.00e+00 | 20260505 03:46:32.793 | INFO | Starting a round of Baz IO | 20260505 03:46:35.773 | INFO | Populating RTMetrics report | 20260505 03:46:35.773 | INFO | RTMetrics report populated | 20260505 03:46:35.773 | INFO | Opening tmp RTMetrics json file | 20260505 03:46:35.774 | INFO | Serializing RTMetrics to json | 20260505 03:46:35.779 | INFO | Serialization done, swapping tmp file now... | 20260505 03:46:35.792 | INFO | RTMetrics done | 20260505 03:46:35.833 | INFO | Finished a round of Baz IO: Wrote 1.38 GiB in 3.04 seconds | 20260505 03:46:35.833 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 67.34% / 0.73% / 31.93% | 20260505 03:46:35.833 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 15.8567% / 0.00315507% / 1.98682e-05% | 20260505 03:46:35.833 | INFO | Baz IO overall: | 20260505 03:46:35.833 | INFO | BytesWritten : 15855374676 | 20260505 03:46:35.833 | INFO | NumEvents : 6105055729 | 20260505 03:46:37.835 | INFO | Analyzing chunk frames = [90112,90624) | 20260505 03:46:37.835 | INFO | MemInfo: { | 20260505 03:46:37.835 | INFO | "gpu" : " 17 GB", | 20260505 03:46:37.835 | INFO | "huge" : "", | 20260505 03:46:37.835 | INFO | "huge_shared" : " 27 GB", | 20260505 03:46:37.835 | INFO | "malloc" : " 15 GB", | 20260505 03:46:37.835 | INFO | "pinned" : " 29 GB" | 20260505 03:46:37.835 | INFO | } | 20260505 03:46:37.835 | INFO | Perf report for the last 5.13934 seconds: | 20260505 03:46:37.835 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:46:37.835 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:37.835 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:37.835 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:37.835 | INFO | ANALYSIS : 58.9%, 3.0, 91.9ms, 0 | 20260505 03:46:37.835 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:37.835 | INFO | RT_METRICS : 43.0%, 1.0, 22.6ms, 0 | 20260505 03:46:37.835 | INFO | PRE_HQ : 58.4%, 2.1, 65.2ms, 0 | 20260505 03:46:37.835 | INFO | BAZWRITER : 58.7%, 2.7, 80.4ms, 0 | 20260505 03:46:37.835 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 03:46:37.835 | INFO | Uploaded 36GiB to GPU in 1.63097s (22.0728GiB/s) | 20260505 03:46:37.835 | INFO | Downloaded 38GiB to GPU in 1.69314s (22.4435GiB/s) | 20260505 03:46:42.974 | INFO | Analyzing chunk frames = [90624,91136) | 20260505 03:46:42.974 | INFO | MemInfo: { | 20260505 03:46:42.974 | INFO | "gpu" : " 17 GB", | 20260505 03:46:42.974 | INFO | "huge" : "", | 20260505 03:46:42.974 | INFO | "huge_shared" : " 27 GB", | 20260505 03:46:42.974 | INFO | "malloc" : " 15 GB", | 20260505 03:46:42.974 | INFO | "pinned" : " 29 GB" | 20260505 03:46:42.974 | INFO | } | 20260505 03:46:42.974 | INFO | Perf report for the last 5.13954 seconds: | 20260505 03:46:42.974 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:46:42.974 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:42.974 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:42.974 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:42.974 | INFO | ANALYSIS : 58.7%, 3.0, 91.5ms, 0 | 20260505 03:46:42.974 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:42.974 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:42.974 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260505 03:46:42.974 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:42.974 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:42.975 | INFO | Uploaded 36GiB to GPU in 1.5945s (22.5776GiB/s) | 20260505 03:46:42.975 | INFO | Downloaded 34GiB to GPU in 1.51066s (22.5067GiB/s) | 20260505 03:46:48.113 | INFO | Analyzing chunk frames = [91136,91648) | 20260505 03:46:48.113 | INFO | MemInfo: { | 20260505 03:46:48.113 | INFO | "gpu" : " 17 GB", | 20260505 03:46:48.113 | INFO | "huge" : "", | 20260505 03:46:48.113 | INFO | "huge_shared" : " 27 GB", | 20260505 03:46:48.113 | INFO | "malloc" : " 15 GB", | 20260505 03:46:48.113 | INFO | "pinned" : " 29 GB" | 20260505 03:46:48.113 | INFO | } | 20260505 03:46:48.114 | INFO | Perf report for the last 5.13932 seconds: | 20260505 03:46:48.114 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:46:48.114 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:48.114 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:48.114 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:48.114 | INFO | ANALYSIS : 58.0%, 3.0, 90.2ms, 0 | 20260505 03:46:48.114 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:48.114 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:48.114 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 03:46:48.114 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:48.114 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:48.114 | INFO | Uploaded 36GiB to GPU in 1.59811s (22.5266GiB/s) | 20260505 03:46:48.114 | INFO | Downloaded 34GiB to GPU in 1.51521s (22.4392GiB/s) | 20260505 03:46:53.252 | INFO | Analyzing chunk frames = [91648,92160) | 20260505 03:46:53.253 | INFO | MemInfo: { | 20260505 03:46:53.253 | INFO | "gpu" : " 17 GB", | 20260505 03:46:53.253 | INFO | "huge" : "", | 20260505 03:46:53.253 | INFO | "huge_shared" : " 27 GB", | 20260505 03:46:53.253 | INFO | "malloc" : " 15 GB", | 20260505 03:46:53.253 | INFO | "pinned" : " 29 GB" | 20260505 03:46:53.253 | INFO | } | 20260505 03:46:53.253 | INFO | Perf report for the last 5.13927 seconds: | 20260505 03:46:53.253 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:46:53.253 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:53.253 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:53.253 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:53.253 | INFO | ANALYSIS : 57.7%, 3.0, 89.5ms, 0 | 20260505 03:46:53.253 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:53.253 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:53.253 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 03:46:53.253 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:53.253 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:53.253 | INFO | Uploaded 36GiB to GPU in 1.5965s (22.5493GiB/s) | 20260505 03:46:53.253 | INFO | Downloaded 34GiB to GPU in 1.51384s (22.4594GiB/s) | 20260505 03:46:58.392 | INFO | Analyzing chunk frames = [92160,92672) | 20260505 03:46:58.392 | INFO | MemInfo: { | 20260505 03:46:58.392 | INFO | "gpu" : " 17 GB", | 20260505 03:46:58.392 | INFO | "huge" : "", | 20260505 03:46:58.392 | INFO | "huge_shared" : " 27 GB", | 20260505 03:46:58.392 | INFO | "malloc" : " 15 GB", | 20260505 03:46:58.392 | INFO | "pinned" : " 29 GB" | 20260505 03:46:58.392 | INFO | } | 20260505 03:46:58.392 | INFO | Perf report for the last 5.13931 seconds: | 20260505 03:46:58.392 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:46:58.392 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:58.392 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:58.392 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:58.392 | INFO | ANALYSIS : 57.6%, 3.0, 89.8ms, 0 | 20260505 03:46:58.392 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:58.392 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:58.392 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 03:46:58.392 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:58.392 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:46:58.393 | INFO | Uploaded 36GiB to GPU in 1.59573s (22.5602GiB/s) | 20260505 03:46:58.393 | INFO | Downloaded 34GiB to GPU in 1.51194s (22.4877GiB/s) | 20260505 03:47:03.531 | INFO | Analyzing chunk frames = [92672,93184) | 20260505 03:47:03.531 | INFO | MemInfo: { | 20260505 03:47:03.531 | INFO | "gpu" : " 17 GB", | 20260505 03:47:03.531 | INFO | "huge" : "", | 20260505 03:47:03.531 | INFO | "huge_shared" : " 27 GB", | 20260505 03:47:03.531 | INFO | "malloc" : " 15 GB", | 20260505 03:47:03.531 | INFO | "pinned" : " 29 GB" | 20260505 03:47:03.531 | INFO | } | 20260505 03:47:03.531 | INFO | Perf report for the last 5.13908 seconds: | 20260505 03:47:03.531 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:47:03.531 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:03.531 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:03.531 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:03.531 | INFO | ANALYSIS : 57.9%, 3.0, 90.3ms, 0 | 20260505 03:47:03.531 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:03.531 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:03.532 | INFO | PRE_HQ : 32.9%, 1.0, 17.1ms, 0 | 20260505 03:47:03.532 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:03.532 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:03.532 | INFO | Uploaded 36GiB to GPU in 1.59666s (22.547GiB/s) | 20260505 03:47:03.532 | INFO | Downloaded 34GiB to GPU in 1.51378s (22.4603GiB/s) | 20260505 03:47:08.671 | INFO | Analyzing chunk frames = [93184,93696) | 20260505 03:47:08.671 | INFO | MemInfo: { | 20260505 03:47:08.671 | INFO | "gpu" : " 17 GB", | 20260505 03:47:08.671 | INFO | "huge" : "", | 20260505 03:47:08.671 | INFO | "huge_shared" : " 27 GB", | 20260505 03:47:08.671 | INFO | "malloc" : " 15 GB", | 20260505 03:47:08.671 | INFO | "pinned" : " 29 GB" | 20260505 03:47:08.671 | INFO | } | 20260505 03:47:08.671 | INFO | Perf report for the last 5.13965 seconds: | 20260505 03:47:08.671 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:47:08.671 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:08.671 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:08.671 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:08.671 | INFO | ANALYSIS : 58.1%, 3.0, 90.7ms, 0 | 20260505 03:47:08.671 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:08.671 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:08.671 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260505 03:47:08.671 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:08.671 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:08.671 | INFO | Uploaded 36GiB to GPU in 1.59659s (22.5481GiB/s) | 20260505 03:47:08.671 | INFO | Downloaded 34GiB to GPU in 1.51307s (22.4709GiB/s) | 20260505 03:47:13.810 | INFO | Analyzing chunk frames = [93696,94208) | 20260505 03:47:13.810 | INFO | MemInfo: { | 20260505 03:47:13.810 | INFO | "gpu" : " 17 GB", | 20260505 03:47:13.810 | INFO | "huge" : "", | 20260505 03:47:13.810 | INFO | "huge_shared" : " 27 GB", | 20260505 03:47:13.810 | INFO | "malloc" : " 15 GB", | 20260505 03:47:13.810 | INFO | "pinned" : " 29 GB" | 20260505 03:47:13.810 | INFO | } | 20260505 03:47:13.810 | INFO | Perf report for the last 5.13921 seconds: | 20260505 03:47:13.810 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:47:13.810 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:13.810 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:13.810 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:13.810 | INFO | ANALYSIS : 58.2%, 3.0, 90.7ms, 0 | 20260505 03:47:13.810 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:13.810 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:13.810 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 03:47:13.810 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:13.810 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:13.811 | INFO | Uploaded 36GiB to GPU in 1.59773s (22.532GiB/s) | 20260505 03:47:13.811 | INFO | Downloaded 34GiB to GPU in 1.51391s (22.4584GiB/s) | 20260505 03:47:13.909 | INFO | Starting a round of Baz IO | 20260505 03:47:16.887 | INFO | Populating RTMetrics report | 20260505 03:47:16.887 | INFO | RTMetrics report populated | 20260505 03:47:16.887 | INFO | Opening tmp RTMetrics json file | 20260505 03:47:16.889 | INFO | Serializing RTMetrics to json | 20260505 03:47:16.890 | INFO | Serialization done, swapping tmp file now... | 20260505 03:47:16.891 | INFO | RTMetrics done | 20260505 03:47:16.948 | INFO | Finished a round of Baz IO: Wrote 1.396 GiB in 3.038 seconds | 20260505 03:47:16.948 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 66.61% / 0.73% / 32.66% | 20260505 03:47:16.948 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 15.7166% / 0.00320276% / 1.98682e-05% | 20260505 03:47:16.948 | INFO | Baz IO overall: | 20260505 03:47:16.948 | INFO | BytesWritten : 17354172728 | 20260505 03:47:16.948 | INFO | NumEvents : 6811600773 | 20260505 03:47:18.949 | INFO | Analyzing chunk frames = [94208,94720) | 20260505 03:47:18.949 | INFO | MemInfo: { | 20260505 03:47:18.949 | INFO | "gpu" : " 17 GB", | 20260505 03:47:18.949 | INFO | "huge" : "", | 20260505 03:47:18.949 | INFO | "huge_shared" : " 27 GB", | 20260505 03:47:18.949 | INFO | "malloc" : " 15 GB", | 20260505 03:47:18.949 | INFO | "pinned" : " 29 GB" | 20260505 03:47:18.949 | INFO | } | 20260505 03:47:18.949 | INFO | Perf report for the last 5.13932 seconds: | 20260505 03:47:18.949 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:47:18.949 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:18.949 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:18.949 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:18.949 | INFO | ANALYSIS : 58.9%, 3.0, 91.9ms, 0 | 20260505 03:47:18.949 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:18.949 | INFO | RT_METRICS : 43.0%, 1.0, 22.6ms, 0 | 20260505 03:47:18.949 | INFO | PRE_HQ : 58.4%, 2.2, 65.4ms, 0 | 20260505 03:47:18.950 | INFO | BAZWRITER : 58.7%, 2.7, 81.5ms, 0 | 20260505 03:47:18.950 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 03:47:18.950 | INFO | Uploaded 36GiB to GPU in 1.63215s (22.0569GiB/s) | 20260505 03:47:18.950 | INFO | Downloaded 38GiB to GPU in 1.69467s (22.4232GiB/s) | 20260505 03:47:24.089 | INFO | Analyzing chunk frames = [94720,95232) | 20260505 03:47:24.089 | INFO | MemInfo: { | 20260505 03:47:24.089 | INFO | "gpu" : " 17 GB", | 20260505 03:47:24.089 | INFO | "huge" : "", | 20260505 03:47:24.089 | INFO | "huge_shared" : " 27 GB", | 20260505 03:47:24.089 | INFO | "malloc" : " 15 GB", | 20260505 03:47:24.089 | INFO | "pinned" : " 29 GB" | 20260505 03:47:24.089 | INFO | } | 20260505 03:47:24.089 | INFO | Perf report for the last 5.13937 seconds: | 20260505 03:47:24.089 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:47:24.089 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:24.089 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:24.089 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:24.089 | INFO | ANALYSIS : 58.7%, 3.0, 91.5ms, 0 | 20260505 03:47:24.089 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:24.089 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:24.089 | INFO | PRE_HQ : 32.6%, 1.0, 16.9ms, 0 | 20260505 03:47:24.089 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:24.089 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:24.089 | INFO | Uploaded 36GiB to GPU in 1.59476s (22.5739GiB/s) | 20260505 03:47:24.089 | INFO | Downloaded 34GiB to GPU in 1.51042s (22.5103GiB/s) | 20260505 03:47:29.228 | INFO | Analyzing chunk frames = [95232,95744) | 20260505 03:47:29.228 | INFO | MemInfo: { | 20260505 03:47:29.228 | INFO | "gpu" : " 17 GB", | 20260505 03:47:29.228 | INFO | "huge" : "", | 20260505 03:47:29.228 | INFO | "huge_shared" : " 27 GB", | 20260505 03:47:29.228 | INFO | "malloc" : " 15 GB", | 20260505 03:47:29.228 | INFO | "pinned" : " 29 GB" | 20260505 03:47:29.228 | INFO | } | 20260505 03:47:29.228 | INFO | Perf report for the last 5.13955 seconds: | 20260505 03:47:29.228 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:47:29.228 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:29.228 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:29.228 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:29.228 | INFO | ANALYSIS : 58.0%, 3.0, 90.1ms, 0 | 20260505 03:47:29.228 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:29.228 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:29.229 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260505 03:47:29.229 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:29.229 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:29.229 | INFO | Uploaded 36GiB to GPU in 1.59845s (22.5218GiB/s) | 20260505 03:47:29.229 | INFO | Downloaded 34GiB to GPU in 1.51478s (22.4455GiB/s) | 20260505 03:47:34.367 | INFO | Analyzing chunk frames = [95744,96256) | 20260505 03:47:34.367 | INFO | MemInfo: { | 20260505 03:47:34.367 | INFO | "gpu" : " 17 GB", | 20260505 03:47:34.367 | INFO | "huge" : "", | 20260505 03:47:34.367 | INFO | "huge_shared" : " 27 GB", | 20260505 03:47:34.367 | INFO | "malloc" : " 15 GB", | 20260505 03:47:34.367 | INFO | "pinned" : " 29 GB" | 20260505 03:47:34.367 | INFO | } | 20260505 03:47:34.367 | INFO | Perf report for the last 5.13886 seconds: | 20260505 03:47:34.367 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:47:34.367 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:34.367 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:34.367 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:34.368 | INFO | ANALYSIS : 57.7%, 3.0, 89.4ms, 0 | 20260505 03:47:34.368 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:34.368 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:34.368 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 03:47:34.368 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:34.368 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:34.368 | INFO | Uploaded 36GiB to GPU in 1.59784s (22.5305GiB/s) | 20260505 03:47:34.368 | INFO | Downloaded 34GiB to GPU in 1.51395s (22.4578GiB/s) | 20260505 03:47:39.507 | INFO | Analyzing chunk frames = [96256,96768) | 20260505 03:47:39.507 | INFO | MemInfo: { | 20260505 03:47:39.507 | INFO | "gpu" : " 17 GB", | 20260505 03:47:39.507 | INFO | "huge" : "", | 20260505 03:47:39.507 | INFO | "huge_shared" : " 27 GB", | 20260505 03:47:39.507 | INFO | "malloc" : " 15 GB", | 20260505 03:47:39.507 | INFO | "pinned" : " 29 GB" | 20260505 03:47:39.507 | INFO | } | 20260505 03:47:39.507 | INFO | Perf report for the last 5.13966 seconds: | 20260505 03:47:39.507 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:47:39.507 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:39.507 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:39.507 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:39.507 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260505 03:47:39.507 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:39.507 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:39.507 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260505 03:47:39.507 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:39.507 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:39.507 | INFO | Uploaded 36GiB to GPU in 1.59633s (22.5517GiB/s) | 20260505 03:47:39.507 | INFO | Downloaded 34GiB to GPU in 1.51211s (22.4851GiB/s) | 20260505 03:47:44.646 | INFO | Analyzing chunk frames = [96768,97280) | 20260505 03:47:44.646 | INFO | MemInfo: { | 20260505 03:47:44.646 | INFO | "gpu" : " 17 GB", | 20260505 03:47:44.646 | INFO | "huge" : "", | 20260505 03:47:44.646 | INFO | "huge_shared" : " 27 GB", | 20260505 03:47:44.646 | INFO | "malloc" : " 15 GB", | 20260505 03:47:44.646 | INFO | "pinned" : " 29 GB" | 20260505 03:47:44.646 | INFO | } | 20260505 03:47:44.646 | INFO | Perf report for the last 5.13943 seconds: | 20260505 03:47:44.646 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:47:44.646 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:44.646 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:44.646 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:44.646 | INFO | ANALYSIS : 57.9%, 3.0, 90.3ms, 0 | 20260505 03:47:44.646 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:44.646 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:44.647 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260505 03:47:44.647 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:44.647 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:44.647 | INFO | Uploaded 36GiB to GPU in 1.59645s (22.55GiB/s) | 20260505 03:47:44.647 | INFO | Downloaded 34GiB to GPU in 1.51198s (22.4871GiB/s) | 20260505 03:47:49.785 | INFO | Analyzing chunk frames = [97280,97792) | 20260505 03:47:49.785 | INFO | MemInfo: { | 20260505 03:47:49.785 | INFO | "gpu" : " 17 GB", | 20260505 03:47:49.785 | INFO | "huge" : "", | 20260505 03:47:49.785 | INFO | "huge_shared" : " 27 GB", | 20260505 03:47:49.785 | INFO | "malloc" : " 15 GB", | 20260505 03:47:49.785 | INFO | "pinned" : " 29 GB" | 20260505 03:47:49.785 | INFO | } | 20260505 03:47:49.786 | INFO | Perf report for the last 5.13916 seconds: | 20260505 03:47:49.786 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:47:49.786 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:49.786 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:49.786 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:49.786 | INFO | ANALYSIS : 58.1%, 3.0, 90.6ms, 0 | 20260505 03:47:49.786 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:49.786 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:49.786 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260505 03:47:49.786 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:49.786 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:49.786 | INFO | Uploaded 36GiB to GPU in 1.59678s (22.5454GiB/s) | 20260505 03:47:49.786 | INFO | Downloaded 34GiB to GPU in 1.51372s (22.4612GiB/s) | 20260505 03:47:54.925 | INFO | Analyzing chunk frames = [97792,98304) | 20260505 03:47:54.925 | INFO | MemInfo: { | 20260505 03:47:54.925 | INFO | "gpu" : " 17 GB", | 20260505 03:47:54.925 | INFO | "huge" : "", | 20260505 03:47:54.925 | INFO | "huge_shared" : " 27 GB", | 20260505 03:47:54.925 | INFO | "malloc" : " 15 GB", | 20260505 03:47:54.925 | INFO | "pinned" : " 29 GB" | 20260505 03:47:54.925 | INFO | } | 20260505 03:47:54.925 | INFO | Perf report for the last 5.13927 seconds: | 20260505 03:47:54.925 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:47:54.925 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:54.925 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:54.925 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:54.925 | INFO | ANALYSIS : 58.2%, 3.0, 90.8ms, 0 | 20260505 03:47:54.925 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:54.925 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:54.925 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260505 03:47:54.925 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:54.925 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:47:54.925 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 03:47:54.925 | INFO | Total mem usage: 14.9 GiB | 20260505 03:47:54.925 | INFO | All Time High Water Marks sum: 15.2 GiB (Savings: 1.52%) | 20260505 03:47:54.925 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -1.99%) | 20260505 03:47:54.925 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 03:47:54.925 | INFO | High Water Breakdown: | 20260505 03:47:54.925 | INFO | Uploaded 36GiB to GPU in 1.59713s (22.5405GiB/s) | 20260505 03:47:54.926 | INFO | 57.6% : 8.43 GiB ( 99.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 03:47:54.926 | INFO | 42.3% : 6.19 GiB ( 93.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 03:47:54.926 | INFO | Downloaded 34GiB to GPU in 1.51383s (22.4596GiB/s) | 20260505 03:47:54.926 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 03:47:54.926 | INFO | ------------------------------------------ | 20260505 03:47:54.926 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:47:54.926 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 03:47:54.926 | INFO | Total mem usage: 26.5 GiB | 20260505 03:47:54.926 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260505 03:47:54.926 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 03:47:54.926 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260505 03:47:54.926 | INFO | High Water Breakdown: | 20260505 03:47:54.926 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 03:47:54.926 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 03:47:54.926 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 03:47:54.926 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 03:47:54.926 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 03:47:54.926 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 03:47:54.926 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 03:47:54.926 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 03:47:54.926 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 03:47:54.926 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 03:47:54.926 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 03:47:54.926 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 03:47:54.926 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 03:47:54.926 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 03:47:54.926 | INFO | ------------------------------------------ | 20260505 03:47:54.926 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:47:54.926 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 03:47:54.926 | INFO | Total mem usage: 25.6 GiB | 20260505 03:47:54.926 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 03:47:54.926 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 03:47:54.926 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 03:47:54.926 | INFO | High Water Breakdown: | 20260505 03:47:54.926 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 03:47:54.926 | INFO | ------------------------------------------ | 20260505 03:47:54.926 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:47:54.926 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 03:47:54.926 | INFO | Total mem usage: 15.7 GiB | 20260505 03:47:54.926 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260505 03:47:54.926 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 03:47:54.926 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260505 03:47:54.926 | INFO | High Water Breakdown: | 20260505 03:47:54.926 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 03:47:54.926 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 03:47:54.926 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 03:47:54.926 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 03:47:54.926 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 03:47:54.926 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 03:47:54.926 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 03:47:54.926 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 03:47:54.926 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 03:47:54.926 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 03:47:54.926 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 03:47:54.926 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 03:47:54.926 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 03:47:54.926 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 03:47:54.926 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 03:47:54.926 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 03:47:54.926 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 03:47:54.926 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 03:47:54.926 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 03:47:54.926 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 03:47:54.926 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 03:47:54.926 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 03:47:54.926 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 03:47:54.926 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 03:47:54.926 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 03:47:54.926 | INFO | ------------------------------------------ | 20260505 03:47:54.926 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:47:54.926 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 03:47:54.927 | INFO | PerfReport: Gathered 1536 samples. | 20260505 03:47:54.927 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 03:47:54.927 | INFO | PerfReport: Full Execution -- 3.08e+01 / -nan, 1.28e+01 / 0.00e+00 | 20260505 03:47:54.927 | INFO | PerfReport: 0.28%, AggregateStats -- 8.65e-02 / -nan, 4.29e-03 / 0.00e+00 | 20260505 03:47:54.927 | INFO | PerfReport: 10.71%, Baseline -- 3.30e+00 / -nan, 1.75e-02 / 0.00e+00 | 20260505 03:47:54.927 | INFO | PerfReport: 3.01%, Histogram -- 9.27e-01 / -nan, 5.20e-02 / 0.00e+00 | 20260505 03:47:54.927 | INFO | PerfReport: 15.98%, DME -- 3.93e+01 / -nan, 3.49e+00 / 0.00e+00 | 20260505 03:47:54.927 | INFO | PerfReport: 55.98%, FrameLabeling -- 1.72e+01 / -nan, 4.49e-02 / 0.00e+00 | 20260505 03:47:54.927 | INFO | PerfReport: 7.88%, PulseAccumulating -- 2.42e+00 / -nan, 1.66e-01 / 0.00e+00 | 20260505 03:47:54.927 | INFO | PerfReport: 4.34%, Metrics -- 1.33e+00 / -nan, 2.18e-01 / 0.00e+00 | 20260505 03:47:54.927 | INFO | PerfReport: 1.82%, PulseSerializing -- 5.59e-01 / -nan, 8.92e-02 / 0.00e+00 | 20260505 03:47:54.927 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 03:47:54.927 | INFO | PerfReport: Gathered 1536 samples. | 20260505 03:47:54.927 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 03:47:54.927 | INFO | PerfReport: Full Execution -- 9.24e+01 / -nan, 1.93e+01 / 0.00e+00 | 20260505 03:47:54.927 | INFO | PerfReport: 18.04%, Upload -- 1.67e+01 / 0.00e+00, 3.49e-01 / 0.00e+00 | 20260505 03:47:54.927 | INFO | PerfReport: 17.31%, Download -- 1.60e+01 / 0.00e+00, 6.77e-01 / 0.00e+00 | 20260505 03:47:54.927 | INFO | PerfReport: 33.29%, Compute -- 3.08e+01 / 0.00e+00, 1.28e+01 / 0.00e+00 | 20260505 03:47:54.927 | INFO | PerfReport: 0.55%, LockUpload -- 5.04e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260505 03:47:54.927 | INFO | PerfReport: 0.00%, LockDownload -- 4.02e-04 / 0.00e+00, 2.20e-04 / 0.00e+00 | 20260505 03:47:54.927 | INFO | PerfReport: 30.80%, LockCompute -- 2.85e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260505 03:47:54.927 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.60e-02 / 0.00e+00 | 20260505 03:47:55.022 | INFO | Starting a round of Baz IO | 20260505 03:47:57.998 | INFO | Populating RTMetrics report | 20260505 03:47:57.998 | INFO | RTMetrics report populated | 20260505 03:47:57.998 | INFO | Opening tmp RTMetrics json file | 20260505 03:47:58.015 | INFO | Serializing RTMetrics to json | 20260505 03:47:58.025 | INFO | Serialization done, swapping tmp file now... | 20260505 03:47:58.045 | INFO | RTMetrics done | 20260505 03:47:58.057 | INFO | Finished a round of Baz IO: Wrote 1.412 GiB in 3.034 seconds | 20260505 03:47:58.057 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 65.92% / 0.68% / 33.39% | 20260505 03:47:58.057 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 15.5658% / 0.00267426% / 1.19209e-05% | 20260505 03:47:58.057 | INFO | Baz IO overall: | 20260505 03:47:58.057 | INFO | BytesWritten : 18869943717 | 20260505 03:47:58.057 | INFO | NumEvents : 7527204752 | 20260505 03:48:00.064 | INFO | Analyzing chunk frames = [98304,98816) | 20260505 03:48:00.064 | INFO | MemInfo: { | 20260505 03:48:00.064 | INFO | "gpu" : " 17 GB", | 20260505 03:48:00.064 | INFO | "huge" : "", | 20260505 03:48:00.064 | INFO | "huge_shared" : " 27 GB", | 20260505 03:48:00.064 | INFO | "malloc" : " 15 GB", | 20260505 03:48:00.064 | INFO | "pinned" : " 29 GB" | 20260505 03:48:00.064 | INFO | } | 20260505 03:48:00.064 | INFO | Perf report for the last 5.13927 seconds: | 20260505 03:48:00.064 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:48:00.064 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:00.064 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:00.064 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:00.064 | INFO | ANALYSIS : 58.8%, 3.0, 91.7ms, 0 | 20260505 03:48:00.064 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:00.064 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260505 03:48:00.064 | INFO | PRE_HQ : 58.3%, 2.1, 65.1ms, 0 | 20260505 03:48:00.064 | INFO | BAZWRITER : 58.6%, 2.7, 81.8ms, 0 | 20260505 03:48:00.064 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 03:48:00.065 | INFO | Uploaded 36GiB to GPU in 1.63574s (22.0085GiB/s) | 20260505 03:48:00.065 | INFO | Downloaded 38GiB to GPU in 1.6984s (22.374GiB/s) | 20260505 03:48:05.203 | INFO | Analyzing chunk frames = [98816,99328) | 20260505 03:48:05.203 | INFO | MemInfo: { | 20260505 03:48:05.203 | INFO | "gpu" : " 17 GB", | 20260505 03:48:05.203 | INFO | "huge" : "", | 20260505 03:48:05.203 | INFO | "huge_shared" : " 27 GB", | 20260505 03:48:05.203 | INFO | "malloc" : " 15 GB", | 20260505 03:48:05.203 | INFO | "pinned" : " 29 GB" | 20260505 03:48:05.203 | INFO | } | 20260505 03:48:05.204 | INFO | Perf report for the last 5.13939 seconds: | 20260505 03:48:05.204 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:48:05.204 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:05.204 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:05.204 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:05.204 | INFO | ANALYSIS : 58.6%, 3.0, 91.5ms, 0 | 20260505 03:48:05.204 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:05.204 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:05.204 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260505 03:48:05.204 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:05.204 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:05.204 | INFO | Uploaded 36GiB to GPU in 1.59456s (22.5768GiB/s) | 20260505 03:48:05.204 | INFO | Downloaded 34GiB to GPU in 1.51016s (22.5142GiB/s) | 20260505 03:48:10.343 | INFO | Analyzing chunk frames = [99328,99840) | 20260505 03:48:10.343 | INFO | MemInfo: { | 20260505 03:48:10.343 | INFO | "gpu" : " 17 GB", | 20260505 03:48:10.343 | INFO | "huge" : "", | 20260505 03:48:10.343 | INFO | "huge_shared" : " 27 GB", | 20260505 03:48:10.343 | INFO | "malloc" : " 15 GB", | 20260505 03:48:10.343 | INFO | "pinned" : " 29 GB" | 20260505 03:48:10.343 | INFO | } | 20260505 03:48:10.343 | INFO | Perf report for the last 5.13926 seconds: | 20260505 03:48:10.343 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:48:10.343 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:10.343 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:10.343 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:10.343 | INFO | ANALYSIS : 58.0%, 3.0, 90.1ms, 0 | 20260505 03:48:10.343 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:10.343 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:10.343 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260505 03:48:10.343 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:10.343 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:10.344 | INFO | Uploaded 36GiB to GPU in 1.59849s (22.5212GiB/s) | 20260505 03:48:10.344 | INFO | Downloaded 34GiB to GPU in 1.51555s (22.4342GiB/s) | 20260505 03:48:15.482 | INFO | Analyzing chunk frames = [99840,100352) | 20260505 03:48:15.482 | INFO | MemInfo: { | 20260505 03:48:15.482 | INFO | "gpu" : " 17 GB", | 20260505 03:48:15.482 | INFO | "huge" : "", | 20260505 03:48:15.482 | INFO | "huge_shared" : " 27 GB", | 20260505 03:48:15.482 | INFO | "malloc" : " 15 GB", | 20260505 03:48:15.482 | INFO | "pinned" : " 29 GB" | 20260505 03:48:15.482 | INFO | } | 20260505 03:48:15.482 | INFO | Perf report for the last 5.13952 seconds: | 20260505 03:48:15.482 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:48:15.482 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:15.482 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:15.482 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:15.482 | INFO | ANALYSIS : 57.6%, 3.0, 89.4ms, 0 | 20260505 03:48:15.483 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:15.483 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:15.483 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260505 03:48:15.483 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:15.483 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:15.483 | INFO | Uploaded 36GiB to GPU in 1.59782s (22.5307GiB/s) | 20260505 03:48:15.483 | INFO | Downloaded 34GiB to GPU in 1.51445s (22.4504GiB/s) | 20260505 03:48:20.621 | INFO | Analyzing chunk frames = [100352,100864) | 20260505 03:48:20.621 | INFO | MemInfo: { | 20260505 03:48:20.621 | INFO | "gpu" : " 17 GB", | 20260505 03:48:20.621 | INFO | "huge" : "", | 20260505 03:48:20.621 | INFO | "huge_shared" : " 27 GB", | 20260505 03:48:20.621 | INFO | "malloc" : " 15 GB", | 20260505 03:48:20.621 | INFO | "pinned" : " 29 GB" | 20260505 03:48:20.621 | INFO | } | 20260505 03:48:20.621 | INFO | Perf report for the last 5.13911 seconds: | 20260505 03:48:20.622 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:48:20.622 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:20.622 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:20.622 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:20.622 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260505 03:48:20.622 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:20.622 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:20.622 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 03:48:20.622 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:20.622 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:20.622 | INFO | Uploaded 36GiB to GPU in 1.5966s (22.5479GiB/s) | 20260505 03:48:20.622 | INFO | Downloaded 34GiB to GPU in 1.51264s (22.4773GiB/s) | 20260505 03:48:25.761 | INFO | Analyzing chunk frames = [100864,101376) | 20260505 03:48:25.761 | INFO | MemInfo: { | 20260505 03:48:25.761 | INFO | "gpu" : " 17 GB", | 20260505 03:48:25.761 | INFO | "huge" : "", | 20260505 03:48:25.761 | INFO | "huge_shared" : " 27 GB", | 20260505 03:48:25.761 | INFO | "malloc" : " 15 GB", | 20260505 03:48:25.761 | INFO | "pinned" : " 29 GB" | 20260505 03:48:25.761 | INFO | } | 20260505 03:48:25.761 | INFO | Perf report for the last 5.13933 seconds: | 20260505 03:48:25.761 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:48:25.761 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:25.761 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:25.761 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:25.761 | INFO | ANALYSIS : 57.9%, 3.0, 90.3ms, 0 | 20260505 03:48:25.761 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:25.761 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:25.761 | INFO | PRE_HQ : 32.3%, 1.0, 16.7ms, 0 | 20260505 03:48:25.761 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:25.761 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:25.761 | INFO | Uploaded 36GiB to GPU in 1.59664s (22.5474GiB/s) | 20260505 03:48:25.761 | INFO | Downloaded 34GiB to GPU in 1.51196s (22.4874GiB/s) | 20260505 03:48:30.900 | INFO | Analyzing chunk frames = [101376,101888) | 20260505 03:48:30.900 | INFO | MemInfo: { | 20260505 03:48:30.900 | INFO | "gpu" : " 17 GB", | 20260505 03:48:30.900 | INFO | "huge" : "", | 20260505 03:48:30.900 | INFO | "huge_shared" : " 27 GB", | 20260505 03:48:30.900 | INFO | "malloc" : " 15 GB", | 20260505 03:48:30.900 | INFO | "pinned" : " 29 GB" | 20260505 03:48:30.900 | INFO | } | 20260505 03:48:30.900 | INFO | Perf report for the last 5.13958 seconds: | 20260505 03:48:30.900 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:48:30.900 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:30.900 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:30.900 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:30.900 | INFO | ANALYSIS : 58.1%, 3.0, 90.6ms, 0 | 20260505 03:48:30.900 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:30.900 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:30.901 | INFO | PRE_HQ : 32.7%, 1.0, 17.0ms, 0 | 20260505 03:48:30.901 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:30.901 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:30.901 | INFO | Uploaded 36GiB to GPU in 1.59751s (22.535GiB/s) | 20260505 03:48:30.901 | INFO | Downloaded 34GiB to GPU in 1.51385s (22.4593GiB/s) | 20260505 03:48:36.039 | INFO | Analyzing chunk frames = [101888,102400) | 20260505 03:48:36.039 | INFO | MemInfo: { | 20260505 03:48:36.039 | INFO | "gpu" : " 17 GB", | 20260505 03:48:36.039 | INFO | "huge" : "", | 20260505 03:48:36.039 | INFO | "huge_shared" : " 27 GB", | 20260505 03:48:36.039 | INFO | "malloc" : " 15 GB", | 20260505 03:48:36.040 | INFO | "pinned" : " 29 GB" | 20260505 03:48:36.040 | INFO | } | 20260505 03:48:36.040 | INFO | Perf report for the last 5.13927 seconds: | 20260505 03:48:36.040 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:48:36.040 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:36.040 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:36.040 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:36.040 | INFO | ANALYSIS : 58.2%, 3.0, 90.8ms, 0 | 20260505 03:48:36.040 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:36.040 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:36.040 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 03:48:36.040 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:36.040 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:36.040 | INFO | Uploaded 36GiB to GPU in 1.59741s (22.5365GiB/s) | 20260505 03:48:36.040 | INFO | Downloaded 34GiB to GPU in 1.51432s (22.4523GiB/s) | 20260505 03:48:36.137 | INFO | Starting a round of Baz IO | 20260505 03:48:39.111 | INFO | Populating RTMetrics report | 20260505 03:48:39.111 | INFO | RTMetrics report populated | 20260505 03:48:39.111 | INFO | Opening tmp RTMetrics json file | 20260505 03:48:39.112 | INFO | Serializing RTMetrics to json | 20260505 03:48:39.115 | INFO | Serialization done, swapping tmp file now... | 20260505 03:48:39.130 | INFO | RTMetrics done | 20260505 03:48:39.172 | INFO | Finished a round of Baz IO: Wrote 1.429 GiB in 3.035 seconds | 20260505 03:48:39.172 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 65.28% / 0.64% / 34.08% | 20260505 03:48:39.172 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 15.4435% / 0.00235637% / 1.98682e-05% | 20260505 03:48:39.172 | INFO | Baz IO overall: | 20260505 03:48:39.172 | INFO | BytesWritten : 20404460491 | 20260505 03:48:39.172 | INFO | NumEvents : 8252073031 | 20260505 03:48:41.179 | INFO | Analyzing chunk frames = [102400,102912) | 20260505 03:48:41.179 | INFO | MemInfo: { | 20260505 03:48:41.179 | INFO | "gpu" : " 17 GB", | 20260505 03:48:41.179 | INFO | "huge" : "", | 20260505 03:48:41.179 | INFO | "huge_shared" : " 27 GB", | 20260505 03:48:41.179 | INFO | "malloc" : " 15 GB", | 20260505 03:48:41.179 | INFO | "pinned" : " 29 GB" | 20260505 03:48:41.179 | INFO | } | 20260505 03:48:41.179 | INFO | Perf report for the last 5.13934 seconds: | 20260505 03:48:41.179 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:48:41.179 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:41.179 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:41.179 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:41.179 | INFO | ANALYSIS : 58.8%, 3.0, 91.7ms, 0 | 20260505 03:48:41.179 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:41.179 | INFO | RT_METRICS : 43.4%, 1.0, 22.7ms, 0 | 20260505 03:48:41.179 | INFO | PRE_HQ : 58.3%, 2.1, 65.0ms, 0 | 20260505 03:48:41.179 | INFO | BAZWRITER : 58.5%, 2.7, 82.5ms, 0 | 20260505 03:48:41.179 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 03:48:41.179 | INFO | Uploaded 36GiB to GPU in 1.63383s (22.0342GiB/s) | 20260505 03:48:41.179 | INFO | Downloaded 38GiB to GPU in 1.6942s (22.4294GiB/s) | 20260505 03:48:46.318 | INFO | Analyzing chunk frames = [102912,103424) | 20260505 03:48:46.318 | INFO | MemInfo: { | 20260505 03:48:46.318 | INFO | "gpu" : " 17 GB", | 20260505 03:48:46.318 | INFO | "huge" : "", | 20260505 03:48:46.318 | INFO | "huge_shared" : " 27 GB", | 20260505 03:48:46.318 | INFO | "malloc" : " 15 GB", | 20260505 03:48:46.318 | INFO | "pinned" : " 29 GB" | 20260505 03:48:46.318 | INFO | } | 20260505 03:48:46.318 | INFO | Perf report for the last 5.13923 seconds: | 20260505 03:48:46.318 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:48:46.318 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:46.318 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:46.318 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:46.318 | INFO | ANALYSIS : 58.6%, 3.0, 91.4ms, 0 | 20260505 03:48:46.318 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:46.318 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:46.318 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 03:48:46.318 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:46.318 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:46.319 | INFO | Uploaded 36GiB to GPU in 1.59532s (22.566GiB/s) | 20260505 03:48:46.319 | INFO | Downloaded 34GiB to GPU in 1.51147s (22.4946GiB/s) | 20260505 03:48:51.457 | INFO | Analyzing chunk frames = [103424,103936) | 20260505 03:48:51.457 | INFO | MemInfo: { | 20260505 03:48:51.457 | INFO | "gpu" : " 17 GB", | 20260505 03:48:51.457 | INFO | "huge" : "", | 20260505 03:48:51.457 | INFO | "huge_shared" : " 27 GB", | 20260505 03:48:51.457 | INFO | "malloc" : " 15 GB", | 20260505 03:48:51.457 | INFO | "pinned" : " 29 GB" | 20260505 03:48:51.457 | INFO | } | 20260505 03:48:51.458 | INFO | Perf report for the last 5.1394 seconds: | 20260505 03:48:51.458 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:48:51.458 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:51.458 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:51.458 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:51.458 | INFO | ANALYSIS : 57.9%, 3.0, 90.0ms, 0 | 20260505 03:48:51.458 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:51.458 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:51.458 | INFO | PRE_HQ : 32.9%, 1.0, 17.1ms, 0 | 20260505 03:48:51.458 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:51.458 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:51.458 | INFO | Uploaded 36GiB to GPU in 1.59889s (22.5156GiB/s) | 20260505 03:48:51.458 | INFO | Downloaded 34GiB to GPU in 1.51596s (22.4281GiB/s) | 20260505 03:48:56.597 | INFO | Analyzing chunk frames = [103936,104448) | 20260505 03:48:56.597 | INFO | MemInfo: { | 20260505 03:48:56.597 | INFO | "gpu" : " 17 GB", | 20260505 03:48:56.597 | INFO | "huge" : "", | 20260505 03:48:56.597 | INFO | "huge_shared" : " 27 GB", | 20260505 03:48:56.597 | INFO | "malloc" : " 15 GB", | 20260505 03:48:56.597 | INFO | "pinned" : " 29 GB" | 20260505 03:48:56.597 | INFO | } | 20260505 03:48:56.597 | INFO | Perf report for the last 5.13928 seconds: | 20260505 03:48:56.597 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:48:56.597 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:56.597 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:56.597 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:56.597 | INFO | ANALYSIS : 57.6%, 3.0, 89.4ms, 0 | 20260505 03:48:56.597 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:56.597 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:56.597 | INFO | PRE_HQ : 32.8%, 1.0, 17.0ms, 0 | 20260505 03:48:56.597 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:56.597 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:48:56.597 | INFO | Uploaded 36GiB to GPU in 1.59815s (22.5261GiB/s) | 20260505 03:48:56.597 | INFO | Downloaded 34GiB to GPU in 1.51452s (22.4494GiB/s) | 20260505 03:49:01.736 | INFO | Analyzing chunk frames = [104448,104960) | 20260505 03:49:01.736 | INFO | MemInfo: { | 20260505 03:49:01.736 | INFO | "gpu" : " 17 GB", | 20260505 03:49:01.736 | INFO | "huge" : "", | 20260505 03:49:01.736 | INFO | "huge_shared" : " 27 GB", | 20260505 03:49:01.736 | INFO | "malloc" : " 15 GB", | 20260505 03:49:01.736 | INFO | "pinned" : " 29 GB" | 20260505 03:49:01.736 | INFO | } | 20260505 03:49:01.736 | INFO | Perf report for the last 5.13946 seconds: | 20260505 03:49:01.736 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:49:01.736 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:01.736 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:01.737 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:01.737 | INFO | ANALYSIS : 57.5%, 3.0, 89.8ms, 0 | 20260505 03:49:01.737 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:01.737 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:01.737 | INFO | PRE_HQ : 33.0%, 1.0, 17.1ms, 0 | 20260505 03:49:01.737 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:01.737 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:01.737 | INFO | Uploaded 36GiB to GPU in 1.59689s (22.5438GiB/s) | 20260505 03:49:01.737 | INFO | Downloaded 34GiB to GPU in 1.51295s (22.4727GiB/s) | 20260505 03:49:06.875 | INFO | Analyzing chunk frames = [104960,105472) | 20260505 03:49:06.875 | INFO | MemInfo: { | 20260505 03:49:06.875 | INFO | "gpu" : " 17 GB", | 20260505 03:49:06.875 | INFO | "huge" : "", | 20260505 03:49:06.875 | INFO | "huge_shared" : " 27 GB", | 20260505 03:49:06.875 | INFO | "malloc" : " 15 GB", | 20260505 03:49:06.875 | INFO | "pinned" : " 29 GB" | 20260505 03:49:06.875 | INFO | } | 20260505 03:49:06.876 | INFO | Perf report for the last 5.13921 seconds: | 20260505 03:49:06.876 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:49:06.876 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:06.876 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:06.876 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:06.876 | INFO | ANALYSIS : 57.8%, 3.0, 90.2ms, 0 | 20260505 03:49:06.876 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:06.876 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:06.876 | INFO | PRE_HQ : 32.6%, 1.0, 16.9ms, 0 | 20260505 03:49:06.876 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:06.876 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:06.876 | INFO | Uploaded 36GiB to GPU in 1.59681s (22.545GiB/s) | 20260505 03:49:06.876 | INFO | Downloaded 34GiB to GPU in 1.51323s (22.4685GiB/s) | 20260505 03:49:12.015 | INFO | Analyzing chunk frames = [105472,105984) | 20260505 03:49:12.015 | INFO | MemInfo: { | 20260505 03:49:12.015 | INFO | "gpu" : " 17 GB", | 20260505 03:49:12.015 | INFO | "huge" : "", | 20260505 03:49:12.015 | INFO | "huge_shared" : " 27 GB", | 20260505 03:49:12.015 | INFO | "malloc" : " 15 GB", | 20260505 03:49:12.015 | INFO | "pinned" : " 29 GB" | 20260505 03:49:12.015 | INFO | } | 20260505 03:49:12.015 | INFO | Perf report for the last 5.13947 seconds: | 20260505 03:49:12.015 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:49:12.015 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:12.015 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:12.015 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:12.015 | INFO | ANALYSIS : 58.1%, 3.0, 90.6ms, 0 | 20260505 03:49:12.015 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:12.015 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:12.015 | INFO | PRE_HQ : 33.2%, 1.0, 17.2ms, 0 | 20260505 03:49:12.015 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:12.015 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:12.016 | INFO | Uploaded 36GiB to GPU in 1.59682s (22.5449GiB/s) | 20260505 03:49:12.016 | INFO | Downloaded 34GiB to GPU in 1.51329s (22.4676GiB/s) | 20260505 03:49:17.154 | INFO | Analyzing chunk frames = [105984,106496) | 20260505 03:49:17.154 | INFO | MemInfo: { | 20260505 03:49:17.154 | INFO | "gpu" : " 17 GB", | 20260505 03:49:17.154 | INFO | "huge" : "", | 20260505 03:49:17.154 | INFO | "huge_shared" : " 27 GB", | 20260505 03:49:17.154 | INFO | "malloc" : " 15 GB", | 20260505 03:49:17.154 | INFO | "pinned" : " 29 GB" | 20260505 03:49:17.154 | INFO | } | 20260505 03:49:17.154 | INFO | Perf report for the last 5.13927 seconds: | 20260505 03:49:17.154 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:49:17.154 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:17.154 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:17.154 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:17.155 | INFO | ANALYSIS : 58.1%, 3.0, 90.7ms, 0 | 20260505 03:49:17.155 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:17.155 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:17.155 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260505 03:49:17.155 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:17.155 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:17.155 | INFO | Uploaded 36GiB to GPU in 1.59868s (22.5186GiB/s) | 20260505 03:49:17.155 | INFO | Downloaded 34GiB to GPU in 1.51512s (22.4404GiB/s) | 20260505 03:49:17.155 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 03:49:17.155 | INFO | Total mem usage: 14.9 GiB | 20260505 03:49:17.155 | INFO | All Time High Water Marks sum: 15.2 GiB (Savings: 1.52%) | 20260505 03:49:17.155 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -2.15%) | 20260505 03:49:17.155 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 03:49:17.155 | INFO | High Water Breakdown: | 20260505 03:49:17.155 | INFO | 57.7% : 8.43 GiB ( 99.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 03:49:17.155 | INFO | 42.2% : 6.16 GiB ( 92.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 03:49:17.155 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 03:49:17.155 | INFO | ------------------------------------------ | 20260505 03:49:17.155 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:49:17.155 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 03:49:17.155 | INFO | Total mem usage: 26.5 GiB | 20260505 03:49:17.155 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260505 03:49:17.155 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 03:49:17.155 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260505 03:49:17.155 | INFO | High Water Breakdown: | 20260505 03:49:17.155 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 03:49:17.155 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 03:49:17.155 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 03:49:17.155 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 03:49:17.155 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 03:49:17.155 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 03:49:17.155 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 03:49:17.155 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 03:49:17.155 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 03:49:17.155 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 03:49:17.155 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 03:49:17.155 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 03:49:17.155 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 03:49:17.155 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 03:49:17.155 | INFO | ------------------------------------------ | 20260505 03:49:17.155 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:49:17.155 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 03:49:17.156 | INFO | Total mem usage: 25.6 GiB | 20260505 03:49:17.156 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 03:49:17.156 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 03:49:17.156 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 03:49:17.156 | INFO | High Water Breakdown: | 20260505 03:49:17.156 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 03:49:17.156 | INFO | ------------------------------------------ | 20260505 03:49:17.156 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:49:17.156 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 03:49:17.156 | INFO | Total mem usage: 15.7 GiB | 20260505 03:49:17.156 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260505 03:49:17.156 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 03:49:17.156 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260505 03:49:17.156 | INFO | High Water Breakdown: | 20260505 03:49:17.156 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 03:49:17.156 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 03:49:17.156 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 03:49:17.156 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 03:49:17.156 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 03:49:17.156 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 03:49:17.156 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 03:49:17.156 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 03:49:17.156 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 03:49:17.156 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 03:49:17.156 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 03:49:17.156 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 03:49:17.156 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 03:49:17.156 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 03:49:17.156 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 03:49:17.156 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 03:49:17.156 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 03:49:17.156 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 03:49:17.156 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 03:49:17.156 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 03:49:17.156 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 03:49:17.156 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 03:49:17.156 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 03:49:17.156 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 03:49:17.156 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 03:49:17.156 | INFO | ------------------------------------------ | 20260505 03:49:17.156 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:49:17.156 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 03:49:17.156 | INFO | PerfReport: Gathered 1536 samples. | 20260505 03:49:17.156 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 03:49:17.156 | INFO | PerfReport: Full Execution -- 3.07e+01 / -nan, 1.28e+01 / 0.00e+00 | 20260505 03:49:17.156 | INFO | PerfReport: 0.28%, AggregateStats -- 8.64e-02 / -nan, 3.89e-03 / 0.00e+00 | 20260505 03:49:17.156 | INFO | PerfReport: 10.72%, Baseline -- 3.30e+00 / -nan, 1.59e-02 / 0.00e+00 | 20260505 03:49:17.156 | INFO | PerfReport: 3.01%, Histogram -- 9.27e-01 / -nan, 5.20e-02 / 0.00e+00 | 20260505 03:49:17.156 | INFO | PerfReport: 15.98%, DME -- 3.93e+01 / -nan, 3.45e+00 / 0.00e+00 | 20260505 03:49:17.156 | INFO | PerfReport: 56.02%, FrameLabeling -- 1.72e+01 / -nan, 4.43e-02 / 0.00e+00 | 20260505 03:49:17.156 | INFO | PerfReport: 7.86%, PulseAccumulating -- 2.42e+00 / -nan, 1.66e-01 / 0.00e+00 | 20260505 03:49:17.156 | INFO | PerfReport: 4.31%, Metrics -- 1.33e+00 / -nan, 2.16e-01 / 0.00e+00 | 20260505 03:49:17.156 | INFO | PerfReport: 1.80%, PulseSerializing -- 5.54e-01 / -nan, 8.88e-02 / 0.00e+00 | 20260505 03:49:17.156 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 03:49:17.156 | INFO | PerfReport: Gathered 1536 samples. | 20260505 03:49:17.156 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 03:49:17.156 | INFO | PerfReport: Full Execution -- 9.24e+01 / -nan, 1.93e+01 / 0.00e+00 | 20260505 03:49:17.156 | INFO | PerfReport: 18.07%, Upload -- 1.67e+01 / 0.00e+00, 3.51e-01 / 0.00e+00 | 20260505 03:49:17.156 | INFO | PerfReport: 17.33%, Download -- 1.60e+01 / 0.00e+00, 6.83e-01 / 0.00e+00 | 20260505 03:49:17.156 | INFO | PerfReport: 33.29%, Compute -- 3.07e+01 / 0.00e+00, 1.28e+01 / 0.00e+00 | 20260505 03:49:17.156 | INFO | PerfReport: 0.55%, LockUpload -- 5.04e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260505 03:49:17.156 | INFO | PerfReport: 0.00%, LockDownload -- 4.07e-04 / 0.00e+00, 9.97e-05 / 0.00e+00 | 20260505 03:49:17.156 | INFO | PerfReport: 30.75%, LockCompute -- 2.84e+01 / 0.00e+00, 1.72e+01 / 0.00e+00 | 20260505 03:49:17.156 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.58e-02 / 0.00e+00 | 20260505 03:49:17.252 | INFO | Starting a round of Baz IO | 20260505 03:49:20.228 | INFO | Populating RTMetrics report | 20260505 03:49:20.228 | INFO | RTMetrics report populated | 20260505 03:49:20.228 | INFO | Opening tmp RTMetrics json file | 20260505 03:49:20.230 | INFO | Serializing RTMetrics to json | 20260505 03:49:20.243 | INFO | Serialization done, swapping tmp file now... | 20260505 03:49:20.251 | INFO | RTMetrics done | 20260505 03:49:20.292 | INFO | Finished a round of Baz IO: Wrote 1.447 GiB in 3.039 seconds | 20260505 03:49:20.292 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 64.65% / 0.63% / 34.72% | 20260505 03:49:20.292 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 15.2893% / 0.00216564% / 1.98682e-05% | 20260505 03:49:20.292 | INFO | Baz IO overall: | 20260505 03:49:20.292 | INFO | BytesWritten : 21957748217 | 20260505 03:49:20.292 | INFO | NumEvents : 8986236668 | 20260505 03:49:22.293 | INFO | Analyzing chunk frames = [106496,107008) | 20260505 03:49:22.294 | INFO | MemInfo: { | 20260505 03:49:22.294 | INFO | "gpu" : " 17 GB", | 20260505 03:49:22.294 | INFO | "huge" : "", | 20260505 03:49:22.294 | INFO | "huge_shared" : " 27 GB", | 20260505 03:49:22.294 | INFO | "malloc" : " 15 GB", | 20260505 03:49:22.294 | INFO | "pinned" : " 29 GB" | 20260505 03:49:22.294 | INFO | } | 20260505 03:49:22.294 | INFO | Perf report for the last 5.13912 seconds: | 20260505 03:49:22.294 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:49:22.294 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:22.294 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:22.294 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:22.294 | INFO | ANALYSIS : 58.8%, 3.0, 91.8ms, 0 | 20260505 03:49:22.294 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:22.294 | INFO | RT_METRICS : 43.6%, 1.0, 22.8ms, 0 | 20260505 03:49:22.294 | INFO | PRE_HQ : 58.3%, 2.1, 65.0ms, 0 | 20260505 03:49:22.294 | INFO | BAZWRITER : 58.7%, 2.8, 83.7ms, 0 | 20260505 03:49:22.294 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 03:49:22.294 | INFO | Uploaded 36GiB to GPU in 1.63576s (22.0082GiB/s) | 20260505 03:49:22.294 | INFO | Downloaded 38GiB to GPU in 1.6973s (22.3885GiB/s) | 20260505 03:49:27.433 | INFO | Analyzing chunk frames = [107008,107520) | 20260505 03:49:27.433 | INFO | MemInfo: { | 20260505 03:49:27.433 | INFO | "gpu" : " 17 GB", | 20260505 03:49:27.433 | INFO | "huge" : "", | 20260505 03:49:27.433 | INFO | "huge_shared" : " 27 GB", | 20260505 03:49:27.433 | INFO | "malloc" : " 15 GB", | 20260505 03:49:27.433 | INFO | "pinned" : " 29 GB" | 20260505 03:49:27.433 | INFO | } | 20260505 03:49:27.433 | INFO | Perf report for the last 5.13952 seconds: | 20260505 03:49:27.433 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:49:27.433 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:27.433 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:27.433 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:27.433 | INFO | ANALYSIS : 58.5%, 3.0, 91.3ms, 0 | 20260505 03:49:27.433 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:27.433 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:27.433 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260505 03:49:27.433 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:27.433 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:27.433 | INFO | Uploaded 36GiB to GPU in 1.59522s (22.5674GiB/s) | 20260505 03:49:27.433 | INFO | Downloaded 34GiB to GPU in 1.51129s (22.4974GiB/s) | 20260505 03:49:32.572 | INFO | Analyzing chunk frames = [107520,108032) | 20260505 03:49:32.572 | INFO | MemInfo: { | 20260505 03:49:32.572 | INFO | "gpu" : " 17 GB", | 20260505 03:49:32.572 | INFO | "huge" : "", | 20260505 03:49:32.572 | INFO | "huge_shared" : " 27 GB", | 20260505 03:49:32.572 | INFO | "malloc" : " 15 GB", | 20260505 03:49:32.572 | INFO | "pinned" : " 29 GB" | 20260505 03:49:32.572 | INFO | } | 20260505 03:49:32.572 | INFO | Perf report for the last 5.13924 seconds: | 20260505 03:49:32.572 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:49:32.572 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:32.572 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:32.572 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:32.572 | INFO | ANALYSIS : 58.0%, 3.0, 90.1ms, 0 | 20260505 03:49:32.572 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:32.572 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:32.572 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260505 03:49:32.573 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:32.573 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:32.573 | INFO | Uploaded 36GiB to GPU in 1.59977s (22.5033GiB/s) | 20260505 03:49:32.573 | INFO | Downloaded 34GiB to GPU in 1.51649s (22.4202GiB/s) | 20260505 03:49:37.711 | INFO | Analyzing chunk frames = [108032,108544) | 20260505 03:49:37.711 | INFO | MemInfo: { | 20260505 03:49:37.711 | INFO | "gpu" : " 17 GB", | 20260505 03:49:37.711 | INFO | "huge" : "", | 20260505 03:49:37.712 | INFO | "huge_shared" : " 27 GB", | 20260505 03:49:37.712 | INFO | "malloc" : " 15 GB", | 20260505 03:49:37.712 | INFO | "pinned" : " 29 GB" | 20260505 03:49:37.712 | INFO | } | 20260505 03:49:37.712 | INFO | Perf report for the last 5.13937 seconds: | 20260505 03:49:37.712 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:49:37.712 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:37.712 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:37.712 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:37.712 | INFO | ANALYSIS : 57.5%, 3.0, 89.3ms, 0 | 20260505 03:49:37.712 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:37.712 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:37.712 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 03:49:37.712 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:37.712 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:37.712 | INFO | Uploaded 36GiB to GPU in 1.59849s (22.5212GiB/s) | 20260505 03:49:37.712 | INFO | Downloaded 34GiB to GPU in 1.51506s (22.4413GiB/s) | 20260505 03:49:42.851 | INFO | Analyzing chunk frames = [108544,109056) | 20260505 03:49:42.851 | INFO | MemInfo: { | 20260505 03:49:42.851 | INFO | "gpu" : " 17 GB", | 20260505 03:49:42.851 | INFO | "huge" : "", | 20260505 03:49:42.851 | INFO | "huge_shared" : " 27 GB", | 20260505 03:49:42.851 | INFO | "malloc" : " 15 GB", | 20260505 03:49:42.851 | INFO | "pinned" : " 29 GB" | 20260505 03:49:42.851 | INFO | } | 20260505 03:49:42.851 | INFO | Perf report for the last 5.13941 seconds: | 20260505 03:49:42.851 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:49:42.851 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:42.851 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:42.851 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:42.851 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260505 03:49:42.851 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:42.851 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:42.851 | INFO | PRE_HQ : 32.7%, 1.0, 17.0ms, 0 | 20260505 03:49:42.851 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:42.851 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:42.852 | INFO | Uploaded 36GiB to GPU in 1.59648s (22.5497GiB/s) | 20260505 03:49:42.852 | INFO | Downloaded 34GiB to GPU in 1.51257s (22.4783GiB/s) | 20260505 03:49:47.990 | INFO | Analyzing chunk frames = [109056,109568) | 20260505 03:49:47.990 | INFO | MemInfo: { | 20260505 03:49:47.990 | INFO | "gpu" : " 17 GB", | 20260505 03:49:47.990 | INFO | "huge" : "", | 20260505 03:49:47.990 | INFO | "huge_shared" : " 27 GB", | 20260505 03:49:47.990 | INFO | "malloc" : " 15 GB", | 20260505 03:49:47.990 | INFO | "pinned" : " 29 GB" | 20260505 03:49:47.990 | INFO | } | 20260505 03:49:47.990 | INFO | Perf report for the last 5.13928 seconds: | 20260505 03:49:47.990 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:49:47.990 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:47.990 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:47.991 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:47.991 | INFO | ANALYSIS : 57.8%, 3.0, 90.1ms, 0 | 20260505 03:49:47.991 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:47.991 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:47.991 | INFO | PRE_HQ : 32.7%, 1.0, 17.0ms, 0 | 20260505 03:49:47.991 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:47.991 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:47.991 | INFO | Uploaded 36GiB to GPU in 1.5982s (22.5253GiB/s) | 20260505 03:49:47.991 | INFO | Downloaded 34GiB to GPU in 1.51479s (22.4454GiB/s) | 20260505 03:49:53.130 | INFO | Analyzing chunk frames = [109568,110080) | 20260505 03:49:53.130 | INFO | MemInfo: { | 20260505 03:49:53.130 | INFO | "gpu" : " 17 GB", | 20260505 03:49:53.130 | INFO | "huge" : "", | 20260505 03:49:53.130 | INFO | "huge_shared" : " 27 GB", | 20260505 03:49:53.130 | INFO | "malloc" : " 15 GB", | 20260505 03:49:53.130 | INFO | "pinned" : " 29 GB" | 20260505 03:49:53.130 | INFO | } | 20260505 03:49:53.130 | INFO | Perf report for the last 5.13935 seconds: | 20260505 03:49:53.130 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:49:53.130 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:53.130 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:53.130 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:53.130 | INFO | ANALYSIS : 58.1%, 3.0, 90.6ms, 0 | 20260505 03:49:53.130 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:53.130 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:53.130 | INFO | PRE_HQ : 32.8%, 1.0, 17.0ms, 0 | 20260505 03:49:53.130 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:53.130 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:53.130 | INFO | Uploaded 36GiB to GPU in 1.59712s (22.5405GiB/s) | 20260505 03:49:53.130 | INFO | Downloaded 34GiB to GPU in 1.51382s (22.4597GiB/s) | 20260505 03:49:58.269 | INFO | Analyzing chunk frames = [110080,110592) | 20260505 03:49:58.269 | INFO | MemInfo: { | 20260505 03:49:58.269 | INFO | "gpu" : " 17 GB", | 20260505 03:49:58.269 | INFO | "huge" : "", | 20260505 03:49:58.269 | INFO | "huge_shared" : " 27 GB", | 20260505 03:49:58.269 | INFO | "malloc" : " 15 GB", | 20260505 03:49:58.269 | INFO | "pinned" : " 29 GB" | 20260505 03:49:58.269 | INFO | } | 20260505 03:49:58.269 | INFO | Perf report for the last 5.13939 seconds: | 20260505 03:49:58.269 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:49:58.269 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:58.269 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:58.269 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:58.269 | INFO | ANALYSIS : 58.1%, 3.0, 90.7ms, 0 | 20260505 03:49:58.269 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:58.269 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:58.269 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 03:49:58.269 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:58.269 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:49:58.269 | INFO | Uploaded 36GiB to GPU in 1.59806s (22.5274GiB/s) | 20260505 03:49:58.270 | INFO | Downloaded 34GiB to GPU in 1.5146s (22.4481GiB/s) | 20260505 03:49:58.367 | INFO | Starting a round of Baz IO | 20260505 03:50:01.345 | INFO | Populating RTMetrics report | 20260505 03:50:01.345 | INFO | RTMetrics report populated | 20260505 03:50:01.346 | INFO | Opening tmp RTMetrics json file | 20260505 03:50:01.349 | INFO | Serializing RTMetrics to json | 20260505 03:50:01.365 | INFO | Serialization done, swapping tmp file now... | 20260505 03:50:01.365 | INFO | RTMetrics done | 20260505 03:50:01.404 | INFO | Finished a round of Baz IO: Wrote 1.461 GiB in 3.036 seconds | 20260505 03:50:01.404 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 64.03% / 0.62% / 35.35% | 20260505 03:50:01.404 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 15.0667% / 0.0018239% / 1.98682e-05% | 20260505 03:50:01.404 | INFO | Baz IO overall: | 20260505 03:50:01.404 | INFO | BytesWritten : 23526335081 | 20260505 03:50:01.404 | INFO | NumEvents : 9726779698 | 20260505 03:50:03.408 | INFO | Analyzing chunk frames = [110592,111104) | 20260505 03:50:03.408 | INFO | MemInfo: { | 20260505 03:50:03.408 | INFO | "gpu" : " 17 GB", | 20260505 03:50:03.408 | INFO | "huge" : "", | 20260505 03:50:03.408 | INFO | "huge_shared" : " 27 GB", | 20260505 03:50:03.408 | INFO | "malloc" : " 15 GB", | 20260505 03:50:03.408 | INFO | "pinned" : " 29 GB" | 20260505 03:50:03.408 | INFO | } | 20260505 03:50:03.409 | INFO | Perf report for the last 5.13927 seconds: | 20260505 03:50:03.409 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:50:03.409 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:03.409 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:03.409 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:03.409 | INFO | ANALYSIS : 58.8%, 3.0, 91.7ms, 0 | 20260505 03:50:03.409 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:03.409 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260505 03:50:03.409 | INFO | PRE_HQ : 58.3%, 2.2, 65.7ms, 0 | 20260505 03:50:03.409 | INFO | BAZWRITER : 58.6%, 2.8, 84.5ms, 0 | 20260505 03:50:03.409 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 03:50:03.409 | INFO | Uploaded 36GiB to GPU in 1.63849s (21.9715GiB/s) | 20260505 03:50:03.409 | INFO | Downloaded 38GiB to GPU in 1.70058s (22.3454GiB/s) | 20260505 03:50:08.548 | INFO | Analyzing chunk frames = [111104,111616) | 20260505 03:50:08.548 | INFO | MemInfo: { | 20260505 03:50:08.548 | INFO | "gpu" : " 17 GB", | 20260505 03:50:08.548 | INFO | "huge" : "", | 20260505 03:50:08.548 | INFO | "huge_shared" : " 27 GB", | 20260505 03:50:08.548 | INFO | "malloc" : " 15 GB", | 20260505 03:50:08.548 | INFO | "pinned" : " 29 GB" | 20260505 03:50:08.548 | INFO | } | 20260505 03:50:08.548 | INFO | Perf report for the last 5.13944 seconds: | 20260505 03:50:08.548 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:50:08.548 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:08.548 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:08.548 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:08.548 | INFO | ANALYSIS : 58.5%, 3.0, 91.3ms, 0 | 20260505 03:50:08.548 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:08.548 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:08.548 | INFO | PRE_HQ : 32.6%, 1.0, 16.9ms, 0 | 20260505 03:50:08.548 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:08.548 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:08.549 | INFO | Uploaded 36GiB to GPU in 1.59544s (22.5643GiB/s) | 20260505 03:50:08.549 | INFO | Downloaded 34GiB to GPU in 1.5111s (22.5002GiB/s) | 20260505 03:50:13.687 | INFO | Analyzing chunk frames = [111616,112128) | 20260505 03:50:13.687 | INFO | MemInfo: { | 20260505 03:50:13.687 | INFO | "gpu" : " 17 GB", | 20260505 03:50:13.687 | INFO | "huge" : "", | 20260505 03:50:13.687 | INFO | "huge_shared" : " 27 GB", | 20260505 03:50:13.687 | INFO | "malloc" : " 15 GB", | 20260505 03:50:13.687 | INFO | "pinned" : " 29 GB" | 20260505 03:50:13.687 | INFO | } | 20260505 03:50:13.687 | INFO | Perf report for the last 5.13927 seconds: | 20260505 03:50:13.687 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:50:13.687 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:13.687 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:13.687 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:13.687 | INFO | ANALYSIS : 57.9%, 3.0, 90.0ms, 0 | 20260505 03:50:13.687 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:13.687 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:13.687 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260505 03:50:13.687 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:13.687 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:13.688 | INFO | Uploaded 36GiB to GPU in 1.59808s (22.527GiB/s) | 20260505 03:50:13.688 | INFO | Downloaded 34GiB to GPU in 1.51564s (22.4327GiB/s) | 20260505 03:50:18.826 | INFO | Analyzing chunk frames = [112128,112640) | 20260505 03:50:18.826 | INFO | MemInfo: { | 20260505 03:50:18.826 | INFO | "gpu" : " 17 GB", | 20260505 03:50:18.826 | INFO | "huge" : "", | 20260505 03:50:18.826 | INFO | "huge_shared" : " 27 GB", | 20260505 03:50:18.826 | INFO | "malloc" : " 15 GB", | 20260505 03:50:18.826 | INFO | "pinned" : " 29 GB" | 20260505 03:50:18.826 | INFO | } | 20260505 03:50:18.827 | INFO | Perf report for the last 5.13927 seconds: | 20260505 03:50:18.827 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:50:18.827 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:18.827 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:18.827 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:18.827 | INFO | ANALYSIS : 57.5%, 3.0, 89.3ms, 0 | 20260505 03:50:18.827 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:18.827 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:18.827 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260505 03:50:18.827 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:18.827 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:18.827 | INFO | Uploaded 36GiB to GPU in 1.59915s (22.5119GiB/s) | 20260505 03:50:18.827 | INFO | Downloaded 34GiB to GPU in 1.51523s (22.4389GiB/s) | 20260505 03:50:23.966 | INFO | Analyzing chunk frames = [112640,113152) | 20260505 03:50:23.966 | INFO | MemInfo: { | 20260505 03:50:23.966 | INFO | "gpu" : " 17 GB", | 20260505 03:50:23.966 | INFO | "huge" : "", | 20260505 03:50:23.966 | INFO | "huge_shared" : " 27 GB", | 20260505 03:50:23.966 | INFO | "malloc" : " 15 GB", | 20260505 03:50:23.966 | INFO | "pinned" : " 29 GB" | 20260505 03:50:23.966 | INFO | } | 20260505 03:50:23.966 | INFO | Perf report for the last 5.13944 seconds: | 20260505 03:50:23.966 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:50:23.966 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:23.966 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:23.966 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:23.966 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260505 03:50:23.966 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:23.966 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:23.966 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260505 03:50:23.966 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:23.966 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:23.967 | INFO | Uploaded 36GiB to GPU in 1.59651s (22.5491GiB/s) | 20260505 03:50:23.967 | INFO | Downloaded 34GiB to GPU in 1.51235s (22.4816GiB/s) | 20260505 03:50:29.105 | INFO | Analyzing chunk frames = [113152,113664) | 20260505 03:50:29.105 | INFO | MemInfo: { | 20260505 03:50:29.105 | INFO | "gpu" : " 17 GB", | 20260505 03:50:29.105 | INFO | "huge" : "", | 20260505 03:50:29.105 | INFO | "huge_shared" : " 27 GB", | 20260505 03:50:29.105 | INFO | "malloc" : " 15 GB", | 20260505 03:50:29.105 | INFO | "pinned" : " 29 GB" | 20260505 03:50:29.105 | INFO | } | 20260505 03:50:29.105 | INFO | Perf report for the last 5.1395 seconds: | 20260505 03:50:29.106 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:50:29.106 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:29.106 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:29.106 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:29.106 | INFO | ANALYSIS : 57.8%, 3.0, 90.1ms, 0 | 20260505 03:50:29.106 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:29.106 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:29.106 | INFO | PRE_HQ : 33.0%, 1.0, 17.1ms, 0 | 20260505 03:50:29.106 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:29.106 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:29.106 | INFO | Uploaded 36GiB to GPU in 1.59737s (22.537GiB/s) | 20260505 03:50:29.106 | INFO | Downloaded 34GiB to GPU in 1.51357s (22.4634GiB/s) | 20260505 03:50:34.244 | INFO | Analyzing chunk frames = [113664,114176) | 20260505 03:50:34.244 | INFO | MemInfo: { | 20260505 03:50:34.244 | INFO | "gpu" : " 17 GB", | 20260505 03:50:34.244 | INFO | "huge" : "", | 20260505 03:50:34.244 | INFO | "huge_shared" : " 27 GB", | 20260505 03:50:34.244 | INFO | "malloc" : " 15 GB", | 20260505 03:50:34.244 | INFO | "pinned" : " 29 GB" | 20260505 03:50:34.245 | INFO | } | 20260505 03:50:34.245 | INFO | Perf report for the last 5.13907 seconds: | 20260505 03:50:34.245 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:50:34.245 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:34.245 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:34.245 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:34.245 | INFO | ANALYSIS : 58.0%, 3.0, 90.5ms, 0 | 20260505 03:50:34.245 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:34.245 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:34.245 | INFO | PRE_HQ : 32.7%, 1.0, 17.0ms, 0 | 20260505 03:50:34.245 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:34.245 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:34.245 | INFO | Uploaded 36GiB to GPU in 1.59727s (22.5384GiB/s) | 20260505 03:50:34.245 | INFO | Downloaded 34GiB to GPU in 1.51432s (22.4523GiB/s) | 20260505 03:50:39.384 | INFO | Analyzing chunk frames = [114176,114688) | 20260505 03:50:39.384 | INFO | MemInfo: { | 20260505 03:50:39.384 | INFO | "gpu" : " 17 GB", | 20260505 03:50:39.384 | INFO | "huge" : "", | 20260505 03:50:39.384 | INFO | "huge_shared" : " 27 GB", | 20260505 03:50:39.384 | INFO | "malloc" : " 15 GB", | 20260505 03:50:39.384 | INFO | "pinned" : " 29 GB" | 20260505 03:50:39.384 | INFO | } | 20260505 03:50:39.384 | INFO | Perf report for the last 5.1394 seconds: | 20260505 03:50:39.384 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:50:39.384 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:39.384 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:39.384 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:39.384 | INFO | ANALYSIS : 58.1%, 3.0, 90.7ms, 0 | 20260505 03:50:39.384 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:39.384 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:39.384 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 03:50:39.384 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:39.384 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:39.385 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 03:50:39.385 | INFO | Total mem usage: 14.9 GiB | 20260505 03:50:39.385 | INFO | All Time High Water Marks sum: 15.2 GiB (Savings: 1.52%) | 20260505 03:50:39.385 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -2.2%) | 20260505 03:50:39.385 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 03:50:39.385 | INFO | High Water Breakdown: | 20260505 03:50:39.385 | INFO | 57.8% : 8.44 GiB ( 99.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 03:50:39.385 | INFO | 42.1% : 6.14 GiB ( 92.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 03:50:39.385 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 03:50:39.385 | INFO | ------------------------------------------ | 20260505 03:50:39.385 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:50:39.385 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 03:50:39.385 | INFO | Total mem usage: 26.5 GiB | 20260505 03:50:39.385 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260505 03:50:39.385 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 03:50:39.385 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260505 03:50:39.385 | INFO | High Water Breakdown: | 20260505 03:50:39.385 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 03:50:39.385 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 03:50:39.385 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 03:50:39.385 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 03:50:39.385 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 03:50:39.385 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 03:50:39.385 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 03:50:39.385 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 03:50:39.385 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 03:50:39.385 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 03:50:39.385 | INFO | Uploaded 36GiB to GPU in 1.59818s (22.5256GiB/s) | 20260505 03:50:39.385 | INFO | Downloaded 34GiB to GPU in 1.51496s (22.4428GiB/s) | 20260505 03:50:39.385 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 03:50:39.385 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 03:50:39.385 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 03:50:39.385 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 03:50:39.385 | INFO | ------------------------------------------ | 20260505 03:50:39.385 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:50:39.385 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 03:50:39.385 | INFO | Total mem usage: 25.6 GiB | 20260505 03:50:39.385 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 03:50:39.385 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 03:50:39.385 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 03:50:39.385 | INFO | High Water Breakdown: | 20260505 03:50:39.385 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 03:50:39.385 | INFO | ------------------------------------------ | 20260505 03:50:39.385 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:50:39.385 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 03:50:39.385 | INFO | Total mem usage: 15.7 GiB | 20260505 03:50:39.385 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260505 03:50:39.385 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 03:50:39.385 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260505 03:50:39.385 | INFO | High Water Breakdown: | 20260505 03:50:39.385 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 03:50:39.385 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 03:50:39.385 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 03:50:39.385 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 03:50:39.385 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 03:50:39.385 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 03:50:39.385 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 03:50:39.385 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 03:50:39.385 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 03:50:39.385 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 03:50:39.385 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 03:50:39.385 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 03:50:39.385 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 03:50:39.385 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 03:50:39.385 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 03:50:39.385 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 03:50:39.385 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 03:50:39.385 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 03:50:39.385 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 03:50:39.385 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 03:50:39.385 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 03:50:39.385 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 03:50:39.385 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 03:50:39.385 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 03:50:39.385 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 03:50:39.385 | INFO | ------------------------------------------ | 20260505 03:50:39.385 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:50:39.385 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 03:50:39.385 | INFO | PerfReport: Gathered 1536 samples. | 20260505 03:50:39.385 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 03:50:39.385 | INFO | PerfReport: Full Execution -- 3.07e+01 / -nan, 1.29e+01 / 0.00e+00 | 20260505 03:50:39.385 | INFO | PerfReport: 0.28%, AggregateStats -- 8.66e-02 / -nan, 6.51e-03 / 0.00e+00 | 20260505 03:50:39.385 | INFO | PerfReport: 10.73%, Baseline -- 3.30e+00 / -nan, 1.75e-02 / 0.00e+00 | 20260505 03:50:39.385 | INFO | PerfReport: 3.01%, Histogram -- 9.26e-01 / -nan, 5.23e-02 / 0.00e+00 | 20260505 03:50:39.385 | INFO | PerfReport: 16.01%, DME -- 3.93e+01 / -nan, 3.42e+00 / 0.00e+00 | 20260505 03:50:39.385 | INFO | PerfReport: 56.05%, FrameLabeling -- 1.72e+01 / -nan, 4.54e-02 / 0.00e+00 | 20260505 03:50:39.385 | INFO | PerfReport: 7.84%, PulseAccumulating -- 2.41e+00 / -nan, 1.64e-01 / 0.00e+00 | 20260505 03:50:39.386 | INFO | PerfReport: 4.28%, Metrics -- 1.32e+00 / -nan, 2.15e-01 / 0.00e+00 | 20260505 03:50:39.386 | INFO | PerfReport: 1.79%, PulseSerializing -- 5.50e-01 / -nan, 8.81e-02 / 0.00e+00 | 20260505 03:50:39.386 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 03:50:39.386 | INFO | PerfReport: Gathered 1536 samples. | 20260505 03:50:39.386 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 03:50:39.386 | INFO | PerfReport: Full Execution -- 9.23e+01 / -nan, 1.93e+01 / 0.00e+00 | 20260505 03:50:39.386 | INFO | PerfReport: 18.08%, Upload -- 1.67e+01 / 0.00e+00, 3.62e-01 / 0.00e+00 | 20260505 03:50:39.386 | INFO | PerfReport: 17.35%, Download -- 1.60e+01 / 0.00e+00, 6.90e-01 / 0.00e+00 | 20260505 03:50:39.386 | INFO | PerfReport: 33.29%, Compute -- 3.07e+01 / 0.00e+00, 1.29e+01 / 0.00e+00 | 20260505 03:50:39.386 | INFO | PerfReport: 0.55%, LockUpload -- 5.04e-01 / 0.00e+00, 3.60e+00 / 0.00e+00 | 20260505 03:50:39.386 | INFO | PerfReport: 0.00%, LockDownload -- 4.03e-04 / 0.00e+00, 9.49e-05 / 0.00e+00 | 20260505 03:50:39.386 | INFO | PerfReport: 30.72%, LockCompute -- 2.84e+01 / 0.00e+00, 1.73e+01 / 0.00e+00 | 20260505 03:50:39.386 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.13e-02 / 0.00e+00 | 20260505 03:50:39.482 | INFO | Starting a round of Baz IO | 20260505 03:50:42.456 | INFO | Populating RTMetrics report | 20260505 03:50:42.456 | INFO | RTMetrics report populated | 20260505 03:50:42.456 | INFO | Opening tmp RTMetrics json file | 20260505 03:50:42.458 | INFO | Serializing RTMetrics to json | 20260505 03:50:42.468 | INFO | Serialization done, swapping tmp file now... | 20260505 03:50:42.479 | INFO | RTMetrics done | 20260505 03:50:42.519 | INFO | Finished a round of Baz IO: Wrote 1.48 GiB in 3.036 seconds | 20260505 03:50:42.519 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 63.41% / 0.61% / 35.97% | 20260505 03:50:42.519 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 14.9509% / 0.00194311% / 3.17891e-05% | 20260505 03:50:42.519 | INFO | Baz IO overall: | 20260505 03:50:42.519 | INFO | BytesWritten : 25115160564 | 20260505 03:50:42.519 | INFO | NumEvents : 10477152148 | 20260505 03:50:44.523 | INFO | Analyzing chunk frames = [114688,115200) | 20260505 03:50:44.523 | INFO | MemInfo: { | 20260505 03:50:44.523 | INFO | "gpu" : " 17 GB", | 20260505 03:50:44.523 | INFO | "huge" : "", | 20260505 03:50:44.523 | INFO | "huge_shared" : " 27 GB", | 20260505 03:50:44.523 | INFO | "malloc" : " 15 GB", | 20260505 03:50:44.523 | INFO | "pinned" : " 29 GB" | 20260505 03:50:44.523 | INFO | } | 20260505 03:50:44.523 | INFO | Perf report for the last 5.13946 seconds: | 20260505 03:50:44.524 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:50:44.524 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:44.524 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:44.524 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:44.524 | INFO | ANALYSIS : 58.8%, 3.0, 91.7ms, 0 | 20260505 03:50:44.524 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:44.524 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260505 03:50:44.524 | INFO | PRE_HQ : 58.3%, 2.2, 65.4ms, 0 | 20260505 03:50:44.524 | INFO | BAZWRITER : 58.6%, 2.8, 85.0ms, 0 | 20260505 03:50:44.524 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 03:50:44.524 | INFO | Uploaded 36GiB to GPU in 1.63442s (22.0262GiB/s) | 20260505 03:50:44.524 | INFO | Downloaded 38GiB to GPU in 1.69542s (22.4134GiB/s) | 20260505 03:50:49.662 | INFO | Analyzing chunk frames = [115200,115712) | 20260505 03:50:49.662 | INFO | MemInfo: { | 20260505 03:50:49.662 | INFO | "gpu" : " 17 GB", | 20260505 03:50:49.662 | INFO | "huge" : "", | 20260505 03:50:49.662 | INFO | "huge_shared" : " 27 GB", | 20260505 03:50:49.662 | INFO | "malloc" : " 15 GB", | 20260505 03:50:49.662 | INFO | "pinned" : " 29 GB" | 20260505 03:50:49.663 | INFO | } | 20260505 03:50:49.663 | INFO | Perf report for the last 5.13916 seconds: | 20260505 03:50:49.663 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:50:49.663 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:49.663 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:49.663 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:49.663 | INFO | ANALYSIS : 58.6%, 3.0, 91.4ms, 0 | 20260505 03:50:49.663 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:49.663 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:49.663 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260505 03:50:49.663 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:49.663 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:49.663 | INFO | Uploaded 36GiB to GPU in 1.59523s (22.5673GiB/s) | 20260505 03:50:49.663 | INFO | Downloaded 34GiB to GPU in 1.51115s (22.4995GiB/s) | 20260505 03:50:54.802 | INFO | Analyzing chunk frames = [115712,116224) | 20260505 03:50:54.802 | INFO | MemInfo: { | 20260505 03:50:54.802 | INFO | "gpu" : " 17 GB", | 20260505 03:50:54.802 | INFO | "huge" : "", | 20260505 03:50:54.802 | INFO | "huge_shared" : " 27 GB", | 20260505 03:50:54.802 | INFO | "malloc" : " 15 GB", | 20260505 03:50:54.802 | INFO | "pinned" : " 29 GB" | 20260505 03:50:54.802 | INFO | } | 20260505 03:50:54.802 | INFO | Perf report for the last 5.13935 seconds: | 20260505 03:50:54.802 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:50:54.802 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:54.802 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:54.802 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:54.802 | INFO | ANALYSIS : 57.9%, 3.0, 90.0ms, 0 | 20260505 03:50:54.802 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:54.802 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:54.802 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260505 03:50:54.802 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:54.802 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:54.802 | INFO | Uploaded 36GiB to GPU in 1.59958s (22.5059GiB/s) | 20260505 03:50:54.802 | INFO | Downloaded 34GiB to GPU in 1.51676s (22.4162GiB/s) | 20260505 03:50:59.941 | INFO | Analyzing chunk frames = [116224,116736) | 20260505 03:50:59.941 | INFO | MemInfo: { | 20260505 03:50:59.941 | INFO | "gpu" : " 17 GB", | 20260505 03:50:59.941 | INFO | "huge" : "", | 20260505 03:50:59.941 | INFO | "huge_shared" : " 27 GB", | 20260505 03:50:59.941 | INFO | "malloc" : " 15 GB", | 20260505 03:50:59.941 | INFO | "pinned" : " 29 GB" | 20260505 03:50:59.941 | INFO | } | 20260505 03:50:59.941 | INFO | Perf report for the last 5.13922 seconds: | 20260505 03:50:59.941 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:50:59.941 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:59.941 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:59.941 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:59.941 | INFO | ANALYSIS : 57.6%, 3.0, 89.3ms, 0 | 20260505 03:50:59.941 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:59.941 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:59.941 | INFO | PRE_HQ : 32.7%, 1.0, 17.0ms, 0 | 20260505 03:50:59.941 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:59.941 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:50:59.942 | INFO | Uploaded 36GiB to GPU in 1.59881s (22.5167GiB/s) | 20260505 03:50:59.942 | INFO | Downloaded 34GiB to GPU in 1.51538s (22.4366GiB/s) | 20260505 03:51:05.080 | INFO | Analyzing chunk frames = [116736,117248) | 20260505 03:51:05.080 | INFO | MemInfo: { | 20260505 03:51:05.080 | INFO | "gpu" : " 17 GB", | 20260505 03:51:05.080 | INFO | "huge" : "", | 20260505 03:51:05.080 | INFO | "huge_shared" : " 27 GB", | 20260505 03:51:05.080 | INFO | "malloc" : " 15 GB", | 20260505 03:51:05.080 | INFO | "pinned" : " 29 GB" | 20260505 03:51:05.081 | INFO | } | 20260505 03:51:05.081 | INFO | Perf report for the last 5.13934 seconds: | 20260505 03:51:05.081 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:51:05.081 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:05.081 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:05.081 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:05.081 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260505 03:51:05.081 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:05.081 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:05.081 | INFO | PRE_HQ : 32.6%, 1.0, 16.9ms, 0 | 20260505 03:51:05.081 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:05.081 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:05.081 | INFO | Uploaded 36GiB to GPU in 1.59699s (22.5424GiB/s) | 20260505 03:51:05.081 | INFO | Downloaded 34GiB to GPU in 1.51337s (22.4664GiB/s) | 20260505 03:51:10.220 | INFO | Analyzing chunk frames = [117248,117760) | 20260505 03:51:10.220 | INFO | MemInfo: { | 20260505 03:51:10.220 | INFO | "gpu" : " 17 GB", | 20260505 03:51:10.220 | INFO | "huge" : "", | 20260505 03:51:10.220 | INFO | "huge_shared" : " 27 GB", | 20260505 03:51:10.220 | INFO | "malloc" : " 15 GB", | 20260505 03:51:10.220 | INFO | "pinned" : " 29 GB" | 20260505 03:51:10.220 | INFO | } | 20260505 03:51:10.220 | INFO | Perf report for the last 5.13927 seconds: | 20260505 03:51:10.220 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:51:10.220 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:10.220 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:10.220 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:10.220 | INFO | ANALYSIS : 57.8%, 3.0, 90.1ms, 0 | 20260505 03:51:10.220 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:10.220 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:10.220 | INFO | PRE_HQ : 32.6%, 1.0, 16.9ms, 0 | 20260505 03:51:10.220 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:10.220 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:10.220 | INFO | Uploaded 36GiB to GPU in 1.59701s (22.5421GiB/s) | 20260505 03:51:10.220 | INFO | Downloaded 34GiB to GPU in 1.51276s (22.4754GiB/s) | 20260505 03:51:15.359 | INFO | Analyzing chunk frames = [117760,118272) | 20260505 03:51:15.359 | INFO | MemInfo: { | 20260505 03:51:15.359 | INFO | "gpu" : " 17 GB", | 20260505 03:51:15.359 | INFO | "huge" : "", | 20260505 03:51:15.359 | INFO | "huge_shared" : " 27 GB", | 20260505 03:51:15.359 | INFO | "malloc" : " 15 GB", | 20260505 03:51:15.359 | INFO | "pinned" : " 29 GB" | 20260505 03:51:15.359 | INFO | } | 20260505 03:51:15.359 | INFO | Perf report for the last 5.1395 seconds: | 20260505 03:51:15.359 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:51:15.359 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:15.359 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:15.359 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:15.359 | INFO | ANALYSIS : 58.0%, 3.0, 90.4ms, 0 | 20260505 03:51:15.359 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:15.359 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:15.359 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 03:51:15.360 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:15.360 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:15.360 | INFO | Uploaded 36GiB to GPU in 1.59756s (22.5344GiB/s) | 20260505 03:51:15.360 | INFO | Downloaded 34GiB to GPU in 1.51408s (22.4558GiB/s) | 20260505 03:51:20.498 | INFO | Analyzing chunk frames = [118272,118784) | 20260505 03:51:20.498 | INFO | MemInfo: { | 20260505 03:51:20.499 | INFO | "gpu" : " 17 GB", | 20260505 03:51:20.499 | INFO | "huge" : "", | 20260505 03:51:20.499 | INFO | "huge_shared" : " 27 GB", | 20260505 03:51:20.499 | INFO | "malloc" : " 15 GB", | 20260505 03:51:20.499 | INFO | "pinned" : " 29 GB" | 20260505 03:51:20.499 | INFO | } | 20260505 03:51:20.499 | INFO | Perf report for the last 5.13931 seconds: | 20260505 03:51:20.499 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:51:20.499 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:20.499 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:20.499 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:20.499 | INFO | ANALYSIS : 58.1%, 3.0, 90.7ms, 0 | 20260505 03:51:20.499 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:20.499 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:20.499 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260505 03:51:20.499 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:20.499 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:20.499 | INFO | Uploaded 36GiB to GPU in 1.59749s (22.5353GiB/s) | 20260505 03:51:20.499 | INFO | Downloaded 34GiB to GPU in 1.51423s (22.4537GiB/s) | 20260505 03:51:20.596 | INFO | Starting a round of Baz IO | 20260505 03:51:23.571 | INFO | Populating RTMetrics report | 20260505 03:51:23.571 | INFO | RTMetrics report populated | 20260505 03:51:23.571 | INFO | Opening tmp RTMetrics json file | 20260505 03:51:23.572 | INFO | Serializing RTMetrics to json | 20260505 03:51:23.583 | INFO | Serialization done, swapping tmp file now... | 20260505 03:51:23.594 | INFO | RTMetrics done | 20260505 03:51:23.632 | INFO | Finished a round of Baz IO: Wrote 1.496 GiB in 3.035 seconds | 20260505 03:51:23.632 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 62.82% / 0.59% / 36.59% | 20260505 03:51:23.632 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 14.7887% / 0.00169675% / 1.19209e-05% | 20260505 03:51:23.632 | INFO | Baz IO overall: | 20260505 03:51:23.632 | INFO | BytesWritten : 26721351780 | 20260505 03:51:23.632 | INFO | NumEvents : 11235099780 | 20260505 03:51:25.638 | INFO | Analyzing chunk frames = [118784,119296) | 20260505 03:51:25.638 | INFO | MemInfo: { | 20260505 03:51:25.638 | INFO | "gpu" : " 17 GB", | 20260505 03:51:25.638 | INFO | "huge" : "", | 20260505 03:51:25.638 | INFO | "huge_shared" : " 27 GB", | 20260505 03:51:25.638 | INFO | "malloc" : " 15 GB", | 20260505 03:51:25.638 | INFO | "pinned" : " 29 GB" | 20260505 03:51:25.638 | INFO | } | 20260505 03:51:25.638 | INFO | Perf report for the last 5.13935 seconds: | 20260505 03:51:25.638 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:51:25.638 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:25.638 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:25.638 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:25.638 | INFO | ANALYSIS : 58.7%, 3.0, 91.6ms, 0 | 20260505 03:51:25.638 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:25.638 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260505 03:51:25.638 | INFO | PRE_HQ : 58.2%, 2.1, 64.9ms, 0 | 20260505 03:51:25.638 | INFO | BAZWRITER : 58.6%, 2.9, 86.0ms, 0 | 20260505 03:51:25.638 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 03:51:25.638 | INFO | Uploaded 36GiB to GPU in 1.63429s (22.028GiB/s) | 20260505 03:51:25.638 | INFO | Downloaded 38GiB to GPU in 1.69737s (22.3875GiB/s) | 20260505 03:51:30.777 | INFO | Analyzing chunk frames = [119296,119808) | 20260505 03:51:30.777 | INFO | MemInfo: { | 20260505 03:51:30.777 | INFO | "gpu" : " 17 GB", | 20260505 03:51:30.777 | INFO | "huge" : "", | 20260505 03:51:30.777 | INFO | "huge_shared" : " 27 GB", | 20260505 03:51:30.777 | INFO | "malloc" : " 15 GB", | 20260505 03:51:30.777 | INFO | "pinned" : " 29 GB" | 20260505 03:51:30.777 | INFO | } | 20260505 03:51:30.777 | INFO | Perf report for the last 5.13937 seconds: | 20260505 03:51:30.777 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:51:30.777 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:30.777 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:30.777 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:30.777 | INFO | ANALYSIS : 58.5%, 3.0, 91.3ms, 0 | 20260505 03:51:30.778 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:30.778 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:30.778 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 03:51:30.778 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:30.778 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:30.778 | INFO | Uploaded 36GiB to GPU in 1.59496s (22.5711GiB/s) | 20260505 03:51:30.778 | INFO | Downloaded 34GiB to GPU in 1.51113s (22.4998GiB/s) | 20260505 03:51:35.917 | INFO | Analyzing chunk frames = [119808,120320) | 20260505 03:51:35.917 | INFO | MemInfo: { | 20260505 03:51:35.917 | INFO | "gpu" : " 17 GB", | 20260505 03:51:35.917 | INFO | "huge" : "", | 20260505 03:51:35.917 | INFO | "huge_shared" : " 27 GB", | 20260505 03:51:35.917 | INFO | "malloc" : " 15 GB", | 20260505 03:51:35.917 | INFO | "pinned" : " 29 GB" | 20260505 03:51:35.917 | INFO | } | 20260505 03:51:35.917 | INFO | Perf report for the last 5.13935 seconds: | 20260505 03:51:35.917 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:51:35.917 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:35.917 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:35.917 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:35.917 | INFO | ANALYSIS : 57.9%, 3.0, 89.9ms, 0 | 20260505 03:51:35.917 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:35.917 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:35.917 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260505 03:51:35.917 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:35.917 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:35.917 | INFO | Uploaded 36GiB to GPU in 1.59956s (22.5062GiB/s) | 20260505 03:51:35.917 | INFO | Downloaded 34GiB to GPU in 1.51676s (22.4163GiB/s) | 20260505 03:51:41.056 | INFO | Analyzing chunk frames = [120320,120832) | 20260505 03:51:41.056 | INFO | MemInfo: { | 20260505 03:51:41.056 | INFO | "gpu" : " 17 GB", | 20260505 03:51:41.056 | INFO | "huge" : "", | 20260505 03:51:41.056 | INFO | "huge_shared" : " 27 GB", | 20260505 03:51:41.056 | INFO | "malloc" : " 15 GB", | 20260505 03:51:41.056 | INFO | "pinned" : " 29 GB" | 20260505 03:51:41.056 | INFO | } | 20260505 03:51:41.056 | INFO | Perf report for the last 5.1393 seconds: | 20260505 03:51:41.056 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:51:41.056 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:41.056 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:41.056 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:41.056 | INFO | ANALYSIS : 57.6%, 3.0, 89.3ms, 0 | 20260505 03:51:41.056 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:41.056 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:41.056 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260505 03:51:41.056 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:41.056 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:41.056 | INFO | Uploaded 36GiB to GPU in 1.59857s (22.5202GiB/s) | 20260505 03:51:41.056 | INFO | Downloaded 34GiB to GPU in 1.51551s (22.4348GiB/s) | 20260505 03:51:46.195 | INFO | Analyzing chunk frames = [120832,121344) | 20260505 03:51:46.195 | INFO | MemInfo: { | 20260505 03:51:46.195 | INFO | "gpu" : " 17 GB", | 20260505 03:51:46.195 | INFO | "huge" : "", | 20260505 03:51:46.195 | INFO | "huge_shared" : " 27 GB", | 20260505 03:51:46.195 | INFO | "malloc" : " 15 GB", | 20260505 03:51:46.195 | INFO | "pinned" : " 29 GB" | 20260505 03:51:46.195 | INFO | } | 20260505 03:51:46.195 | INFO | Perf report for the last 5.13937 seconds: | 20260505 03:51:46.195 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:51:46.195 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:46.196 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:46.196 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:46.196 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260505 03:51:46.196 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:46.196 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:46.196 | INFO | PRE_HQ : 32.6%, 1.0, 16.9ms, 0 | 20260505 03:51:46.196 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:46.196 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:46.196 | INFO | Uploaded 36GiB to GPU in 1.5975s (22.5352GiB/s) | 20260505 03:51:46.196 | INFO | Downloaded 34GiB to GPU in 1.51357s (22.4634GiB/s) | 20260505 03:51:51.335 | INFO | Analyzing chunk frames = [121344,121856) | 20260505 03:51:51.335 | INFO | MemInfo: { | 20260505 03:51:51.335 | INFO | "gpu" : " 17 GB", | 20260505 03:51:51.335 | INFO | "huge" : "", | 20260505 03:51:51.335 | INFO | "huge_shared" : " 27 GB", | 20260505 03:51:51.335 | INFO | "malloc" : " 15 GB", | 20260505 03:51:51.335 | INFO | "pinned" : " 29 GB" | 20260505 03:51:51.335 | INFO | } | 20260505 03:51:51.335 | INFO | Perf report for the last 5.1393 seconds: | 20260505 03:51:51.335 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:51:51.335 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:51.335 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:51.335 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:51.335 | INFO | ANALYSIS : 57.8%, 3.0, 90.1ms, 0 | 20260505 03:51:51.335 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:51.335 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:51.335 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260505 03:51:51.335 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:51.335 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:51.335 | INFO | Uploaded 36GiB to GPU in 1.59684s (22.5445GiB/s) | 20260505 03:51:51.335 | INFO | Downloaded 34GiB to GPU in 1.51319s (22.4692GiB/s) | 20260505 03:51:56.474 | INFO | Analyzing chunk frames = [121856,122368) | 20260505 03:51:56.474 | INFO | MemInfo: { | 20260505 03:51:56.474 | INFO | "gpu" : " 17 GB", | 20260505 03:51:56.474 | INFO | "huge" : "", | 20260505 03:51:56.474 | INFO | "huge_shared" : " 27 GB", | 20260505 03:51:56.474 | INFO | "malloc" : " 15 GB", | 20260505 03:51:56.474 | INFO | "pinned" : " 29 GB" | 20260505 03:51:56.474 | INFO | } | 20260505 03:51:56.474 | INFO | Perf report for the last 5.13938 seconds: | 20260505 03:51:56.474 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:51:56.474 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:56.474 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:56.474 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:56.474 | INFO | ANALYSIS : 57.9%, 3.0, 90.4ms, 0 | 20260505 03:51:56.474 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:56.474 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:56.474 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260505 03:51:56.474 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:56.474 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:51:56.475 | INFO | Uploaded 36GiB to GPU in 1.59775s (22.5317GiB/s) | 20260505 03:51:56.475 | INFO | Downloaded 34GiB to GPU in 1.51458s (22.4484GiB/s) | 20260505 03:52:01.613 | INFO | Analyzing chunk frames = [122368,122880) | 20260505 03:52:01.613 | INFO | MemInfo: { | 20260505 03:52:01.613 | INFO | "gpu" : " 17 GB", | 20260505 03:52:01.613 | INFO | "huge" : "", | 20260505 03:52:01.613 | INFO | "huge_shared" : " 27 GB", | 20260505 03:52:01.613 | INFO | "malloc" : " 15 GB", | 20260505 03:52:01.613 | INFO | "pinned" : " 29 GB" | 20260505 03:52:01.613 | INFO | } | 20260505 03:52:01.613 | INFO | Perf report for the last 5.13915 seconds: | 20260505 03:52:01.613 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:52:01.613 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:01.613 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:01.613 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:01.613 | INFO | ANALYSIS : 58.1%, 3.0, 90.7ms, 0 | 20260505 03:52:01.613 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:01.613 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:01.613 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 03:52:01.614 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:01.614 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:01.614 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 03:52:01.614 | INFO | Total mem usage: 14.9 GiB | 20260505 03:52:01.614 | INFO | All Time High Water Marks sum: 15.2 GiB (Savings: 1.52%) | 20260505 03:52:01.614 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -2.42%) | 20260505 03:52:01.614 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 03:52:01.614 | INFO | High Water Breakdown: | 20260505 03:52:01.614 | INFO | 57.8% : 8.43 GiB ( 99.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 03:52:01.614 | INFO | 42% : 6.12 GiB ( 92.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 03:52:01.614 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 03:52:01.614 | INFO | ------------------------------------------ | 20260505 03:52:01.614 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:52:01.614 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 03:52:01.614 | INFO | Total mem usage: 26.5 GiB | 20260505 03:52:01.614 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260505 03:52:01.614 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 03:52:01.614 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260505 03:52:01.614 | INFO | High Water Breakdown: | 20260505 03:52:01.614 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 03:52:01.614 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 03:52:01.614 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 03:52:01.614 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 03:52:01.614 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 03:52:01.614 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 03:52:01.614 | INFO | Uploaded 36GiB to GPU in 1.59782s (22.5307GiB/s) | 20260505 03:52:01.614 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 03:52:01.614 | INFO | Downloaded 34GiB to GPU in 1.51451s (22.4495GiB/s) | 20260505 03:52:01.614 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 03:52:01.614 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 03:52:01.614 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 03:52:01.614 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 03:52:01.614 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 03:52:01.614 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 03:52:01.614 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 03:52:01.614 | INFO | ------------------------------------------ | 20260505 03:52:01.614 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:52:01.614 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 03:52:01.615 | INFO | Total mem usage: 25.6 GiB | 20260505 03:52:01.615 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 03:52:01.615 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 03:52:01.615 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 03:52:01.615 | INFO | High Water Breakdown: | 20260505 03:52:01.615 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 03:52:01.615 | INFO | ------------------------------------------ | 20260505 03:52:01.615 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:52:01.615 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 03:52:01.615 | INFO | Total mem usage: 15.7 GiB | 20260505 03:52:01.615 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260505 03:52:01.615 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 03:52:01.615 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260505 03:52:01.615 | INFO | High Water Breakdown: | 20260505 03:52:01.615 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 03:52:01.615 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 03:52:01.615 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 03:52:01.615 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 03:52:01.615 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 03:52:01.615 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 03:52:01.615 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 03:52:01.615 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 03:52:01.615 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 03:52:01.615 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 03:52:01.615 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 03:52:01.615 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 03:52:01.615 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 03:52:01.615 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 03:52:01.615 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 03:52:01.615 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 03:52:01.615 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 03:52:01.615 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 03:52:01.615 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 03:52:01.615 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 03:52:01.615 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 03:52:01.615 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 03:52:01.615 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 03:52:01.615 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 03:52:01.615 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 03:52:01.615 | INFO | ------------------------------------------ | 20260505 03:52:01.615 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:52:01.615 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 03:52:01.615 | INFO | PerfReport: Gathered 1536 samples. | 20260505 03:52:01.615 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 03:52:01.615 | INFO | PerfReport: Full Execution -- 3.07e+01 / -nan, 1.29e+01 / 0.00e+00 | 20260505 03:52:01.615 | INFO | PerfReport: 0.28%, AggregateStats -- 8.63e-02 / -nan, 4.17e-03 / 0.00e+00 | 20260505 03:52:01.615 | INFO | PerfReport: 10.74%, Baseline -- 3.30e+00 / -nan, 1.78e-02 / 0.00e+00 | 20260505 03:52:01.615 | INFO | PerfReport: 3.01%, Histogram -- 9.25e-01 / -nan, 5.20e-02 / 0.00e+00 | 20260505 03:52:01.615 | INFO | PerfReport: 16.01%, DME -- 3.93e+01 / -nan, 3.34e+00 / 0.00e+00 | 20260505 03:52:01.615 | INFO | PerfReport: 56.08%, FrameLabeling -- 1.72e+01 / -nan, 4.58e-02 / 0.00e+00 | 20260505 03:52:01.615 | INFO | PerfReport: 7.83%, PulseAccumulating -- 2.40e+00 / -nan, 1.63e-01 / 0.00e+00 | 20260505 03:52:01.615 | INFO | PerfReport: 4.26%, Metrics -- 1.31e+00 / -nan, 2.12e-01 / 0.00e+00 | 20260505 03:52:01.615 | INFO | PerfReport: 1.78%, PulseSerializing -- 5.47e-01 / -nan, 8.71e-02 / 0.00e+00 | 20260505 03:52:01.615 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 03:52:01.615 | INFO | PerfReport: Gathered 1536 samples. | 20260505 03:52:01.615 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 03:52:01.615 | INFO | PerfReport: Full Execution -- 9.22e+01 / -nan, 1.91e+01 / 0.00e+00 | 20260505 03:52:01.615 | INFO | PerfReport: 18.09%, Upload -- 1.67e+01 / 0.00e+00, 3.55e-01 / 0.00e+00 | 20260505 03:52:01.615 | INFO | PerfReport: 17.36%, Download -- 1.60e+01 / 0.00e+00, 6.79e-01 / 0.00e+00 | 20260505 03:52:01.615 | INFO | PerfReport: 33.29%, Compute -- 3.07e+01 / 0.00e+00, 1.29e+01 / 0.00e+00 | 20260505 03:52:01.615 | INFO | PerfReport: 0.55%, LockUpload -- 5.04e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260505 03:52:01.615 | INFO | PerfReport: 0.00%, LockDownload -- 3.95e-04 / 0.00e+00, 9.85e-05 / 0.00e+00 | 20260505 03:52:01.615 | INFO | PerfReport: 30.70%, LockCompute -- 2.83e+01 / 0.00e+00, 1.72e+01 / 0.00e+00 | 20260505 03:52:01.615 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.59e-02 / 0.00e+00 | 20260505 03:52:01.711 | INFO | Starting a round of Baz IO | 20260505 03:52:04.681 | INFO | Populating RTMetrics report | 20260505 03:52:04.681 | INFO | RTMetrics report populated | 20260505 03:52:04.681 | INFO | Opening tmp RTMetrics json file | 20260505 03:52:04.683 | INFO | Serializing RTMetrics to json | 20260505 03:52:04.684 | INFO | Serialization done, swapping tmp file now... | 20260505 03:52:04.685 | INFO | RTMetrics done | 20260505 03:52:04.746 | INFO | Finished a round of Baz IO: Wrote 1.514 GiB in 3.034 seconds | 20260505 03:52:04.746 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 62.25% / 0.57% / 37.18% | 20260505 03:52:04.746 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 14.7032% / 0.00179211% / 2.38419e-05% | 20260505 03:52:04.746 | INFO | Baz IO overall: | 20260505 03:52:04.746 | INFO | BytesWritten : 28346829642 | 20260505 03:52:04.746 | INFO | NumEvents : 12002502244 | 20260505 03:52:06.753 | INFO | Analyzing chunk frames = [122880,123392) | 20260505 03:52:06.753 | INFO | MemInfo: { | 20260505 03:52:06.753 | INFO | "gpu" : " 17 GB", | 20260505 03:52:06.753 | INFO | "huge" : "", | 20260505 03:52:06.753 | INFO | "huge_shared" : " 27 GB", | 20260505 03:52:06.753 | INFO | "malloc" : " 15 GB", | 20260505 03:52:06.753 | INFO | "pinned" : " 29 GB" | 20260505 03:52:06.753 | INFO | } | 20260505 03:52:06.753 | INFO | Perf report for the last 5.13935 seconds: | 20260505 03:52:06.753 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:52:06.753 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:06.753 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:06.753 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:06.753 | INFO | ANALYSIS : 58.7%, 3.0, 91.5ms, 0 | 20260505 03:52:06.753 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:06.753 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260505 03:52:06.753 | INFO | PRE_HQ : 58.2%, 2.2, 65.2ms, 0 | 20260505 03:52:06.753 | INFO | BAZWRITER : 58.6%, 2.9, 87.4ms, 0 | 20260505 03:52:06.753 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 03:52:06.753 | INFO | Uploaded 36GiB to GPU in 1.62941s (22.094GiB/s) | 20260505 03:52:06.754 | INFO | Downloaded 38GiB to GPU in 1.69156s (22.4645GiB/s) | 20260505 03:52:11.892 | INFO | Analyzing chunk frames = [123392,123904) | 20260505 03:52:11.892 | INFO | MemInfo: { | 20260505 03:52:11.892 | INFO | "gpu" : " 17 GB", | 20260505 03:52:11.892 | INFO | "huge" : "", | 20260505 03:52:11.892 | INFO | "huge_shared" : " 27 GB", | 20260505 03:52:11.892 | INFO | "malloc" : " 15 GB", | 20260505 03:52:11.892 | INFO | "pinned" : " 29 GB" | 20260505 03:52:11.892 | INFO | } | 20260505 03:52:11.892 | INFO | Perf report for the last 5.13941 seconds: | 20260505 03:52:11.892 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:52:11.892 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:11.892 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:11.892 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:11.892 | INFO | ANALYSIS : 58.5%, 3.0, 91.3ms, 0 | 20260505 03:52:11.892 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:11.892 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:11.892 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260505 03:52:11.892 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:11.892 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:11.893 | INFO | Uploaded 36GiB to GPU in 1.59508s (22.5694GiB/s) | 20260505 03:52:11.893 | INFO | Downloaded 34GiB to GPU in 1.51068s (22.5064GiB/s) | 20260505 03:52:17.031 | INFO | Analyzing chunk frames = [123904,124416) | 20260505 03:52:17.031 | INFO | MemInfo: { | 20260505 03:52:17.031 | INFO | "gpu" : " 17 GB", | 20260505 03:52:17.031 | INFO | "huge" : "", | 20260505 03:52:17.031 | INFO | "huge_shared" : " 27 GB", | 20260505 03:52:17.031 | INFO | "malloc" : " 15 GB", | 20260505 03:52:17.031 | INFO | "pinned" : " 29 GB" | 20260505 03:52:17.031 | INFO | } | 20260505 03:52:17.032 | INFO | Perf report for the last 5.13934 seconds: | 20260505 03:52:17.032 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:52:17.032 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:17.032 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:17.032 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:17.032 | INFO | ANALYSIS : 57.8%, 3.0, 89.9ms, 0 | 20260505 03:52:17.032 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:17.032 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:17.032 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 03:52:17.032 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:17.032 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:17.032 | INFO | Uploaded 36GiB to GPU in 1.60052s (22.4927GiB/s) | 20260505 03:52:17.032 | INFO | Downloaded 34GiB to GPU in 1.51757s (22.4042GiB/s) | 20260505 03:52:22.171 | INFO | Analyzing chunk frames = [124416,124928) | 20260505 03:52:22.171 | INFO | MemInfo: { | 20260505 03:52:22.171 | INFO | "gpu" : " 17 GB", | 20260505 03:52:22.171 | INFO | "huge" : "", | 20260505 03:52:22.171 | INFO | "huge_shared" : " 27 GB", | 20260505 03:52:22.171 | INFO | "malloc" : " 15 GB", | 20260505 03:52:22.171 | INFO | "pinned" : " 29 GB" | 20260505 03:52:22.171 | INFO | } | 20260505 03:52:22.171 | INFO | Perf report for the last 5.13927 seconds: | 20260505 03:52:22.171 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:52:22.171 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:22.171 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:22.171 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:22.171 | INFO | ANALYSIS : 57.5%, 3.0, 89.2ms, 0 | 20260505 03:52:22.171 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:22.171 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:22.171 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260505 03:52:22.171 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:22.171 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:22.171 | INFO | Uploaded 36GiB to GPU in 1.59868s (22.5185GiB/s) | 20260505 03:52:22.171 | INFO | Downloaded 34GiB to GPU in 1.51464s (22.4476GiB/s) | 20260505 03:52:27.310 | INFO | Analyzing chunk frames = [124928,125440) | 20260505 03:52:27.310 | INFO | MemInfo: { | 20260505 03:52:27.310 | INFO | "gpu" : " 17 GB", | 20260505 03:52:27.310 | INFO | "huge" : "", | 20260505 03:52:27.310 | INFO | "huge_shared" : " 27 GB", | 20260505 03:52:27.310 | INFO | "malloc" : " 15 GB", | 20260505 03:52:27.310 | INFO | "pinned" : " 29 GB" | 20260505 03:52:27.310 | INFO | } | 20260505 03:52:27.310 | INFO | Perf report for the last 5.13948 seconds: | 20260505 03:52:27.310 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:52:27.310 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:27.310 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:27.310 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:27.310 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260505 03:52:27.310 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:27.310 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:27.310 | INFO | PRE_HQ : 32.8%, 1.0, 17.0ms, 0 | 20260505 03:52:27.310 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:27.310 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:27.311 | INFO | Uploaded 36GiB to GPU in 1.59726s (22.5386GiB/s) | 20260505 03:52:27.311 | INFO | Downloaded 34GiB to GPU in 1.51354s (22.4638GiB/s) | 20260505 03:52:32.449 | INFO | Analyzing chunk frames = [125440,125952) | 20260505 03:52:32.449 | INFO | MemInfo: { | 20260505 03:52:32.449 | INFO | "gpu" : " 17 GB", | 20260505 03:52:32.449 | INFO | "huge" : "", | 20260505 03:52:32.449 | INFO | "huge_shared" : " 27 GB", | 20260505 03:52:32.449 | INFO | "malloc" : " 15 GB", | 20260505 03:52:32.449 | INFO | "pinned" : " 29 GB" | 20260505 03:52:32.449 | INFO | } | 20260505 03:52:32.449 | INFO | Perf report for the last 5.13917 seconds: | 20260505 03:52:32.449 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:52:32.449 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:32.449 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:32.450 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:32.450 | INFO | ANALYSIS : 57.7%, 3.0, 90.1ms, 0 | 20260505 03:52:32.450 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:32.450 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:32.450 | INFO | PRE_HQ : 32.9%, 1.0, 17.1ms, 0 | 20260505 03:52:32.450 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:32.450 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:32.450 | INFO | Uploaded 36GiB to GPU in 1.59813s (22.5263GiB/s) | 20260505 03:52:32.450 | INFO | Downloaded 34GiB to GPU in 1.51437s (22.4515GiB/s) | 20260505 03:52:37.589 | INFO | Analyzing chunk frames = [125952,126464) | 20260505 03:52:37.589 | INFO | MemInfo: { | 20260505 03:52:37.589 | INFO | "gpu" : " 17 GB", | 20260505 03:52:37.589 | INFO | "huge" : "", | 20260505 03:52:37.589 | INFO | "huge_shared" : " 27 GB", | 20260505 03:52:37.589 | INFO | "malloc" : " 15 GB", | 20260505 03:52:37.589 | INFO | "pinned" : " 29 GB" | 20260505 03:52:37.589 | INFO | } | 20260505 03:52:37.589 | INFO | Perf report for the last 5.13919 seconds: | 20260505 03:52:37.589 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:52:37.589 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:37.589 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:37.589 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:37.589 | INFO | ANALYSIS : 58.0%, 3.0, 90.4ms, 0 | 20260505 03:52:37.589 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:37.589 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:37.589 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260505 03:52:37.589 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:37.589 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:37.589 | INFO | Uploaded 36GiB to GPU in 1.59736s (22.5372GiB/s) | 20260505 03:52:37.589 | INFO | Downloaded 34GiB to GPU in 1.51435s (22.4519GiB/s) | 20260505 03:52:42.728 | INFO | Analyzing chunk frames = [126464,126976) | 20260505 03:52:42.728 | INFO | MemInfo: { | 20260505 03:52:42.728 | INFO | "gpu" : " 17 GB", | 20260505 03:52:42.728 | INFO | "huge" : "", | 20260505 03:52:42.728 | INFO | "huge_shared" : " 27 GB", | 20260505 03:52:42.728 | INFO | "malloc" : " 15 GB", | 20260505 03:52:42.728 | INFO | "pinned" : " 29 GB" | 20260505 03:52:42.728 | INFO | } | 20260505 03:52:42.728 | INFO | Perf report for the last 5.13932 seconds: | 20260505 03:52:42.728 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:52:42.728 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:42.728 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:42.728 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:42.728 | INFO | ANALYSIS : 58.0%, 3.0, 90.5ms, 0 | 20260505 03:52:42.728 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:42.728 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:42.728 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260505 03:52:42.728 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:42.728 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:42.728 | INFO | Uploaded 36GiB to GPU in 1.59802s (22.5279GiB/s) | 20260505 03:52:42.728 | INFO | Downloaded 34GiB to GPU in 1.51535s (22.4371GiB/s) | 20260505 03:52:42.825 | INFO | Starting a round of Baz IO | 20260505 03:52:45.796 | INFO | Populating RTMetrics report | 20260505 03:52:45.797 | INFO | RTMetrics report populated | 20260505 03:52:45.797 | INFO | Opening tmp RTMetrics json file | 20260505 03:52:45.798 | INFO | Serializing RTMetrics to json | 20260505 03:52:45.799 | INFO | Serialization done, swapping tmp file now... | 20260505 03:52:45.799 | INFO | RTMetrics done | 20260505 03:52:45.862 | INFO | Finished a round of Baz IO: Wrote 1.53 GiB in 3.036 seconds | 20260505 03:52:45.862 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 61.69% / 0.56% / 37.75% | 20260505 03:52:45.862 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 14.5357% / 0.00164509% / 2.38419e-05% | 20260505 03:52:45.862 | INFO | Baz IO overall: | 20260505 03:52:45.862 | INFO | BytesWritten : 29989716653 | 20260505 03:52:45.862 | INFO | NumEvents : 12777252492 | 20260505 03:52:47.867 | INFO | Analyzing chunk frames = [126976,127488) | 20260505 03:52:47.867 | INFO | MemInfo: { | 20260505 03:52:47.867 | INFO | "gpu" : " 17 GB", | 20260505 03:52:47.867 | INFO | "huge" : "", | 20260505 03:52:47.867 | INFO | "huge_shared" : " 27 GB", | 20260505 03:52:47.868 | INFO | "malloc" : " 15 GB", | 20260505 03:52:47.868 | INFO | "pinned" : " 29 GB" | 20260505 03:52:47.868 | INFO | } | 20260505 03:52:47.868 | INFO | Perf report for the last 5.13964 seconds: | 20260505 03:52:47.868 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:52:47.868 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:47.868 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:47.868 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:47.868 | INFO | ANALYSIS : 58.7%, 3.0, 91.6ms, 0 | 20260505 03:52:47.868 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:47.868 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260505 03:52:47.868 | INFO | PRE_HQ : 58.2%, 2.2, 65.4ms, 0 | 20260505 03:52:47.868 | INFO | BAZWRITER : 58.6%, 3.0, 89.7ms, 0 | 20260505 03:52:47.868 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 03:52:47.868 | INFO | Uploaded 36GiB to GPU in 1.63757s (21.9838GiB/s) | 20260505 03:52:47.868 | INFO | Downloaded 38GiB to GPU in 1.70035s (22.3484GiB/s) | 20260505 03:52:53.007 | INFO | Analyzing chunk frames = [127488,128000) | 20260505 03:52:53.007 | INFO | MemInfo: { | 20260505 03:52:53.007 | INFO | "gpu" : " 17 GB", | 20260505 03:52:53.007 | INFO | "huge" : "", | 20260505 03:52:53.007 | INFO | "huge_shared" : " 27 GB", | 20260505 03:52:53.007 | INFO | "malloc" : " 15 GB", | 20260505 03:52:53.007 | INFO | "pinned" : " 29 GB" | 20260505 03:52:53.007 | INFO | } | 20260505 03:52:53.007 | INFO | Perf report for the last 5.13945 seconds: | 20260505 03:52:53.007 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:52:53.007 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:53.007 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:53.007 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:53.007 | INFO | ANALYSIS : 58.4%, 3.0, 91.1ms, 0 | 20260505 03:52:53.007 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:53.007 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:53.007 | INFO | PRE_HQ : 32.7%, 1.0, 17.1ms, 0 | 20260505 03:52:53.007 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:53.007 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:53.008 | INFO | Uploaded 36GiB to GPU in 1.59588s (22.5581GiB/s) | 20260505 03:52:53.008 | INFO | Downloaded 34GiB to GPU in 1.51195s (22.4875GiB/s) | 20260505 03:52:58.146 | INFO | Analyzing chunk frames = [128000,128512) | 20260505 03:52:58.146 | INFO | MemInfo: { | 20260505 03:52:58.146 | INFO | "gpu" : " 17 GB", | 20260505 03:52:58.146 | INFO | "huge" : "", | 20260505 03:52:58.146 | INFO | "huge_shared" : " 27 GB", | 20260505 03:52:58.146 | INFO | "malloc" : " 15 GB", | 20260505 03:52:58.146 | INFO | "pinned" : " 29 GB" | 20260505 03:52:58.146 | INFO | } | 20260505 03:52:58.146 | INFO | Perf report for the last 5.13922 seconds: | 20260505 03:52:58.146 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:52:58.146 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:58.146 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:58.146 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:58.146 | INFO | ANALYSIS : 57.9%, 3.0, 89.9ms, 0 | 20260505 03:52:58.146 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:58.147 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:58.147 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260505 03:52:58.147 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:58.147 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:52:58.147 | INFO | Uploaded 36GiB to GPU in 1.59926s (22.5104GiB/s) | 20260505 03:52:58.147 | INFO | Downloaded 34GiB to GPU in 1.51598s (22.4278GiB/s) | 20260505 03:53:03.285 | INFO | Analyzing chunk frames = [128512,129024) | 20260505 03:53:03.285 | INFO | MemInfo: { | 20260505 03:53:03.285 | INFO | "gpu" : " 17 GB", | 20260505 03:53:03.285 | INFO | "huge" : "", | 20260505 03:53:03.285 | INFO | "huge_shared" : " 27 GB", | 20260505 03:53:03.285 | INFO | "malloc" : " 15 GB", | 20260505 03:53:03.285 | INFO | "pinned" : " 29 GB" | 20260505 03:53:03.285 | INFO | } | 20260505 03:53:03.286 | INFO | Perf report for the last 5.13915 seconds: | 20260505 03:53:03.286 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:53:03.286 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:03.286 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:03.286 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:03.286 | INFO | ANALYSIS : 57.4%, 3.0, 89.1ms, 0 | 20260505 03:53:03.286 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:03.286 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:03.286 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 03:53:03.286 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:03.286 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:03.286 | INFO | Uploaded 36GiB to GPU in 1.59978s (22.503GiB/s) | 20260505 03:53:03.286 | INFO | Downloaded 34GiB to GPU in 1.51588s (22.4293GiB/s) | 20260505 03:53:08.425 | INFO | Analyzing chunk frames = [129024,129536) | 20260505 03:53:08.425 | INFO | MemInfo: { | 20260505 03:53:08.425 | INFO | "gpu" : " 17 GB", | 20260505 03:53:08.425 | INFO | "huge" : "", | 20260505 03:53:08.425 | INFO | "huge_shared" : " 27 GB", | 20260505 03:53:08.425 | INFO | "malloc" : " 15 GB", | 20260505 03:53:08.425 | INFO | "pinned" : " 29 GB" | 20260505 03:53:08.425 | INFO | } | 20260505 03:53:08.425 | INFO | Perf report for the last 5.13935 seconds: | 20260505 03:53:08.425 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:53:08.425 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:08.425 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:08.425 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:08.425 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260505 03:53:08.425 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:08.425 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:08.425 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260505 03:53:08.425 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:08.425 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:08.425 | INFO | Uploaded 36GiB to GPU in 1.59711s (22.5408GiB/s) | 20260505 03:53:08.426 | INFO | Downloaded 34GiB to GPU in 1.51351s (22.4643GiB/s) | 20260505 03:53:13.564 | INFO | Analyzing chunk frames = [129536,130048) | 20260505 03:53:13.564 | INFO | MemInfo: { | 20260505 03:53:13.564 | INFO | "gpu" : " 17 GB", | 20260505 03:53:13.564 | INFO | "huge" : "", | 20260505 03:53:13.564 | INFO | "huge_shared" : " 27 GB", | 20260505 03:53:13.564 | INFO | "malloc" : " 15 GB", | 20260505 03:53:13.564 | INFO | "pinned" : " 29 GB" | 20260505 03:53:13.564 | INFO | } | 20260505 03:53:13.564 | INFO | Perf report for the last 5.13938 seconds: | 20260505 03:53:13.564 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:53:13.564 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:13.564 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:13.564 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:13.564 | INFO | ANALYSIS : 57.7%, 3.0, 90.0ms, 0 | 20260505 03:53:13.564 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:13.564 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:13.564 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260505 03:53:13.564 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:13.564 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:13.565 | INFO | Uploaded 36GiB to GPU in 1.59791s (22.5294GiB/s) | 20260505 03:53:13.565 | INFO | Downloaded 34GiB to GPU in 1.51396s (22.4577GiB/s) | 20260505 03:53:18.704 | INFO | Analyzing chunk frames = [130048,130560) | 20260505 03:53:18.704 | INFO | MemInfo: { | 20260505 03:53:18.704 | INFO | "gpu" : " 17 GB", | 20260505 03:53:18.704 | INFO | "huge" : "", | 20260505 03:53:18.704 | INFO | "huge_shared" : " 27 GB", | 20260505 03:53:18.704 | INFO | "malloc" : " 15 GB", | 20260505 03:53:18.704 | INFO | "pinned" : " 29 GB" | 20260505 03:53:18.704 | INFO | } | 20260505 03:53:18.704 | INFO | Perf report for the last 5.13955 seconds: | 20260505 03:53:18.704 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:53:18.704 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:18.704 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:18.704 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:18.704 | INFO | ANALYSIS : 58.0%, 3.0, 90.4ms, 0 | 20260505 03:53:18.704 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:18.704 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:18.704 | INFO | PRE_HQ : 33.0%, 1.0, 17.2ms, 0 | 20260505 03:53:18.704 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:18.704 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:18.704 | INFO | Uploaded 36GiB to GPU in 1.59813s (22.5263GiB/s) | 20260505 03:53:18.704 | INFO | Downloaded 34GiB to GPU in 1.51558s (22.4336GiB/s) | 20260505 03:53:23.843 | INFO | Analyzing chunk frames = [130560,131072) | 20260505 03:53:23.843 | INFO | MemInfo: { | 20260505 03:53:23.843 | INFO | "gpu" : " 17 GB", | 20260505 03:53:23.843 | INFO | "huge" : "", | 20260505 03:53:23.843 | INFO | "huge_shared" : " 27 GB", | 20260505 03:53:23.843 | INFO | "malloc" : " 15 GB", | 20260505 03:53:23.843 | INFO | "pinned" : " 29 GB" | 20260505 03:53:23.843 | INFO | } | 20260505 03:53:23.843 | INFO | Perf report for the last 5.139 seconds: | 20260505 03:53:23.843 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:53:23.843 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:23.843 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:23.843 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:23.843 | INFO | ANALYSIS : 58.0%, 3.0, 90.5ms, 0 | 20260505 03:53:23.843 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:23.843 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:23.843 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260505 03:53:23.843 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:23.843 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:23.843 | INFO | Uploaded 36GiB to GPU in 1.59779s (22.5311GiB/s) | 20260505 03:53:23.843 | INFO | Downloaded 34GiB to GPU in 1.51493s (22.4433GiB/s) | 20260505 03:53:23.844 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 03:53:23.844 | INFO | Total mem usage: 14.9 GiB | 20260505 03:53:23.844 | INFO | All Time High Water Marks sum: 15.2 GiB (Savings: 1.52%) | 20260505 03:53:23.844 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -2.45%) | 20260505 03:53:23.844 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 03:53:23.844 | INFO | High Water Breakdown: | 20260505 03:53:23.844 | INFO | 57.9% : 8.44 GiB ( 99.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 03:53:23.844 | INFO | 41.9% : 6.11 GiB ( 91.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 03:53:23.844 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 03:53:23.844 | INFO | ------------------------------------------ | 20260505 03:53:23.844 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:53:23.844 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 03:53:23.844 | INFO | Total mem usage: 26.5 GiB | 20260505 03:53:23.844 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260505 03:53:23.844 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 03:53:23.844 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260505 03:53:23.844 | INFO | High Water Breakdown: | 20260505 03:53:23.844 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 03:53:23.844 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 03:53:23.844 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 03:53:23.844 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 03:53:23.844 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 03:53:23.844 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 03:53:23.844 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 03:53:23.844 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 03:53:23.844 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 03:53:23.844 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 03:53:23.844 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 03:53:23.844 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 03:53:23.844 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 03:53:23.844 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 03:53:23.844 | INFO | ------------------------------------------ | 20260505 03:53:23.844 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:53:23.844 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 03:53:23.844 | INFO | Total mem usage: 25.6 GiB | 20260505 03:53:23.844 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 03:53:23.844 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 03:53:23.844 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 03:53:23.844 | INFO | High Water Breakdown: | 20260505 03:53:23.844 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 03:53:23.844 | INFO | ------------------------------------------ | 20260505 03:53:23.844 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:53:23.844 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 03:53:23.844 | INFO | Total mem usage: 15.7 GiB | 20260505 03:53:23.844 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260505 03:53:23.844 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 03:53:23.844 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260505 03:53:23.844 | INFO | High Water Breakdown: | 20260505 03:53:23.844 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 03:53:23.844 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 03:53:23.844 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 03:53:23.844 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 03:53:23.844 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 03:53:23.844 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 03:53:23.844 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 03:53:23.844 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 03:53:23.844 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 03:53:23.844 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 03:53:23.844 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 03:53:23.844 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 03:53:23.844 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 03:53:23.844 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 03:53:23.844 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 03:53:23.844 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 03:53:23.844 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 03:53:23.844 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 03:53:23.844 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 03:53:23.844 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 03:53:23.844 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 03:53:23.844 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 03:53:23.844 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 03:53:23.844 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 03:53:23.844 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 03:53:23.844 | INFO | ------------------------------------------ | 20260505 03:53:23.844 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:53:23.844 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 03:53:23.844 | INFO | PerfReport: Gathered 1536 samples. | 20260505 03:53:23.844 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 03:53:23.844 | INFO | PerfReport: Full Execution -- 3.07e+01 / -nan, 1.29e+01 / 0.00e+00 | 20260505 03:53:23.844 | INFO | PerfReport: 0.28%, AggregateStats -- 8.62e-02 / -nan, 3.77e-03 / 0.00e+00 | 20260505 03:53:23.844 | INFO | PerfReport: 10.74%, Baseline -- 3.30e+00 / -nan, 1.77e-02 / 0.00e+00 | 20260505 03:53:23.844 | INFO | PerfReport: 3.01%, Histogram -- 9.24e-01 / -nan, 5.22e-02 / 0.00e+00 | 20260505 03:53:23.845 | INFO | PerfReport: 16.01%, DME -- 3.93e+01 / -nan, 3.35e+00 / 0.00e+00 | 20260505 03:53:23.845 | INFO | PerfReport: 56.13%, FrameLabeling -- 1.72e+01 / -nan, 4.48e-02 / 0.00e+00 | 20260505 03:53:23.845 | INFO | PerfReport: 7.82%, PulseAccumulating -- 2.40e+00 / -nan, 1.64e-01 / 0.00e+00 | 20260505 03:53:23.845 | INFO | PerfReport: 4.24%, Metrics -- 1.30e+00 / -nan, 2.12e-01 / 0.00e+00 | 20260505 03:53:23.845 | INFO | PerfReport: 1.77%, PulseSerializing -- 5.44e-01 / -nan, 8.66e-02 / 0.00e+00 | 20260505 03:53:23.845 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 03:53:23.845 | INFO | PerfReport: Gathered 1536 samples. | 20260505 03:53:23.845 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 03:53:23.845 | INFO | PerfReport: Full Execution -- 9.22e+01 / -nan, 1.92e+01 / 0.00e+00 | 20260505 03:53:23.845 | INFO | PerfReport: 18.11%, Upload -- 1.67e+01 / 0.00e+00, 3.59e-01 / 0.00e+00 | 20260505 03:53:23.845 | INFO | PerfReport: 17.37%, Download -- 1.60e+01 / 0.00e+00, 6.82e-01 / 0.00e+00 | 20260505 03:53:23.845 | INFO | PerfReport: 33.29%, Compute -- 3.07e+01 / 0.00e+00, 1.29e+01 / 0.00e+00 | 20260505 03:53:23.845 | INFO | PerfReport: 0.55%, LockUpload -- 5.04e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260505 03:53:23.845 | INFO | PerfReport: 0.00%, LockDownload -- 4.00e-04 / 0.00e+00, 1.02e-04 / 0.00e+00 | 20260505 03:53:23.845 | INFO | PerfReport: 30.66%, LockCompute -- 2.83e+01 / 0.00e+00, 1.72e+01 / 0.00e+00 | 20260505 03:53:23.845 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.87e-02 / 0.00e+00 | 20260505 03:53:23.942 | INFO | Starting a round of Baz IO | 20260505 03:53:26.911 | INFO | Populating RTMetrics report | 20260505 03:53:26.911 | INFO | RTMetrics report populated | 20260505 03:53:26.911 | INFO | Opening tmp RTMetrics json file | 20260505 03:53:26.912 | INFO | Serializing RTMetrics to json | 20260505 03:53:26.914 | INFO | Serialization done, swapping tmp file now... | 20260505 03:53:26.915 | INFO | RTMetrics done | 20260505 03:53:26.976 | INFO | Finished a round of Baz IO: Wrote 1.544 GiB in 3.033 seconds | 20260505 03:53:26.976 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 61.18% / 0.52% / 38.31% | 20260505 03:53:26.976 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 14.4081% / 0.0016729% / 2.38419e-05% | 20260505 03:53:26.976 | INFO | Baz IO overall: | 20260505 03:53:26.976 | INFO | BytesWritten : 31647301460 | 20260505 03:53:26.976 | INFO | NumEvents : 13559097129 | 20260505 03:53:28.982 | INFO | Analyzing chunk frames = [131072,131584) | 20260505 03:53:28.982 | INFO | MemInfo: { | 20260505 03:53:28.982 | INFO | "gpu" : " 17 GB", | 20260505 03:53:28.982 | INFO | "huge" : "", | 20260505 03:53:28.982 | INFO | "huge_shared" : " 27 GB", | 20260505 03:53:28.982 | INFO | "malloc" : " 14 GB", | 20260505 03:53:28.982 | INFO | "pinned" : " 29 GB" | 20260505 03:53:28.982 | INFO | } | 20260505 03:53:28.982 | INFO | Perf report for the last 5.13959 seconds: | 20260505 03:53:28.983 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:53:28.983 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:28.983 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:28.983 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:28.983 | INFO | ANALYSIS : 58.7%, 3.0, 91.5ms, 0 | 20260505 03:53:28.983 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:28.983 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260505 03:53:28.983 | INFO | PRE_HQ : 58.1%, 2.2, 65.1ms, 0 | 20260505 03:53:28.983 | INFO | BAZWRITER : 58.6%, 3.0, 90.4ms, 0 | 20260505 03:53:28.983 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 03:53:28.983 | INFO | Uploaded 36GiB to GPU in 1.63111s (22.0708GiB/s) | 20260505 03:53:28.983 | INFO | Downloaded 38GiB to GPU in 1.69455s (22.4249GiB/s) | 20260505 03:53:34.122 | INFO | Analyzing chunk frames = [131584,132096) | 20260505 03:53:34.122 | INFO | MemInfo: { | 20260505 03:53:34.122 | INFO | "gpu" : " 17 GB", | 20260505 03:53:34.122 | INFO | "huge" : "", | 20260505 03:53:34.122 | INFO | "huge_shared" : " 27 GB", | 20260505 03:53:34.122 | INFO | "malloc" : " 14 GB", | 20260505 03:53:34.122 | INFO | "pinned" : " 29 GB" | 20260505 03:53:34.122 | INFO | } | 20260505 03:53:34.122 | INFO | Perf report for the last 5.13936 seconds: | 20260505 03:53:34.122 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:53:34.122 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:34.122 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:34.122 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:34.122 | INFO | ANALYSIS : 58.4%, 3.0, 91.1ms, 0 | 20260505 03:53:34.122 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:34.122 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:34.122 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260505 03:53:34.122 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:34.122 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:34.122 | INFO | Uploaded 36GiB to GPU in 1.59617s (22.554GiB/s) | 20260505 03:53:34.122 | INFO | Downloaded 34GiB to GPU in 1.51312s (22.4702GiB/s) | 20260505 03:53:39.261 | INFO | Analyzing chunk frames = [132096,132608) | 20260505 03:53:39.261 | INFO | MemInfo: { | 20260505 03:53:39.261 | INFO | "gpu" : " 17 GB", | 20260505 03:53:39.261 | INFO | "huge" : "", | 20260505 03:53:39.261 | INFO | "huge_shared" : " 27 GB", | 20260505 03:53:39.261 | INFO | "malloc" : " 14 GB", | 20260505 03:53:39.261 | INFO | "pinned" : " 29 GB" | 20260505 03:53:39.261 | INFO | } | 20260505 03:53:39.261 | INFO | Perf report for the last 5.13932 seconds: | 20260505 03:53:39.261 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:53:39.261 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:39.261 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:39.261 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:39.261 | INFO | ANALYSIS : 57.8%, 3.0, 89.8ms, 0 | 20260505 03:53:39.261 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:39.261 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:39.261 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260505 03:53:39.261 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:39.261 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:39.261 | INFO | Uploaded 36GiB to GPU in 1.59941s (22.5083GiB/s) | 20260505 03:53:39.262 | INFO | Downloaded 34GiB to GPU in 1.51595s (22.4282GiB/s) | 20260505 03:53:44.401 | INFO | Analyzing chunk frames = [132608,133120) | 20260505 03:53:44.401 | INFO | MemInfo: { | 20260505 03:53:44.401 | INFO | "gpu" : " 17 GB", | 20260505 03:53:44.401 | INFO | "huge" : "", | 20260505 03:53:44.401 | INFO | "huge_shared" : " 27 GB", | 20260505 03:53:44.401 | INFO | "malloc" : " 14 GB", | 20260505 03:53:44.401 | INFO | "pinned" : " 29 GB" | 20260505 03:53:44.401 | INFO | } | 20260505 03:53:44.401 | INFO | Perf report for the last 5.13965 seconds: | 20260505 03:53:44.401 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:53:44.401 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:44.401 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:44.401 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:44.401 | INFO | ANALYSIS : 57.4%, 3.0, 89.1ms, 0 | 20260505 03:53:44.401 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:44.401 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:44.401 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260505 03:53:44.401 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:44.401 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:44.401 | INFO | Uploaded 36GiB to GPU in 1.59949s (22.5071GiB/s) | 20260505 03:53:44.401 | INFO | Downloaded 34GiB to GPU in 1.51622s (22.4242GiB/s) | 20260505 03:53:49.540 | INFO | Analyzing chunk frames = [133120,133632) | 20260505 03:53:49.540 | INFO | MemInfo: { | 20260505 03:53:49.540 | INFO | "gpu" : " 17 GB", | 20260505 03:53:49.540 | INFO | "huge" : "", | 20260505 03:53:49.540 | INFO | "huge_shared" : " 27 GB", | 20260505 03:53:49.540 | INFO | "malloc" : " 14 GB", | 20260505 03:53:49.540 | INFO | "pinned" : " 29 GB" | 20260505 03:53:49.540 | INFO | } | 20260505 03:53:49.540 | INFO | Perf report for the last 5.13899 seconds: | 20260505 03:53:49.540 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:53:49.540 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:49.540 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:49.540 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:49.540 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260505 03:53:49.540 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:49.540 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:49.540 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260505 03:53:49.540 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:49.540 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:49.540 | INFO | Uploaded 36GiB to GPU in 1.59738s (22.5369GiB/s) | 20260505 03:53:49.540 | INFO | Downloaded 34GiB to GPU in 1.51346s (22.4651GiB/s) | 20260505 03:53:54.679 | INFO | Analyzing chunk frames = [133632,134144) | 20260505 03:53:54.679 | INFO | MemInfo: { | 20260505 03:53:54.679 | INFO | "gpu" : " 17 GB", | 20260505 03:53:54.679 | INFO | "huge" : "", | 20260505 03:53:54.679 | INFO | "huge_shared" : " 27 GB", | 20260505 03:53:54.679 | INFO | "malloc" : " 14 GB", | 20260505 03:53:54.679 | INFO | "pinned" : " 29 GB" | 20260505 03:53:54.679 | INFO | } | 20260505 03:53:54.679 | INFO | Perf report for the last 5.13923 seconds: | 20260505 03:53:54.679 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:53:54.679 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:54.679 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:54.679 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:54.679 | INFO | ANALYSIS : 57.7%, 3.0, 90.0ms, 0 | 20260505 03:53:54.679 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:54.679 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:54.679 | INFO | PRE_HQ : 32.8%, 1.0, 17.0ms, 0 | 20260505 03:53:54.679 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:54.679 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:54.680 | INFO | Uploaded 36GiB to GPU in 1.598s (22.5281GiB/s) | 20260505 03:53:54.680 | INFO | Downloaded 34GiB to GPU in 1.5139s (22.4586GiB/s) | 20260505 03:53:59.818 | INFO | Analyzing chunk frames = [134144,134656) | 20260505 03:53:59.818 | INFO | MemInfo: { | 20260505 03:53:59.818 | INFO | "gpu" : " 17 GB", | 20260505 03:53:59.818 | INFO | "huge" : "", | 20260505 03:53:59.818 | INFO | "huge_shared" : " 27 GB", | 20260505 03:53:59.818 | INFO | "malloc" : " 14 GB", | 20260505 03:53:59.818 | INFO | "pinned" : " 29 GB" | 20260505 03:53:59.819 | INFO | } | 20260505 03:53:59.819 | INFO | Perf report for the last 5.13947 seconds: | 20260505 03:53:59.819 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:53:59.819 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:59.819 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:59.819 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:59.819 | INFO | ANALYSIS : 57.9%, 3.0, 90.4ms, 0 | 20260505 03:53:59.819 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:59.819 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:59.819 | INFO | PRE_HQ : 32.8%, 1.0, 17.0ms, 0 | 20260505 03:53:59.819 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:59.819 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:53:59.819 | INFO | Uploaded 36GiB to GPU in 1.59778s (22.5313GiB/s) | 20260505 03:53:59.819 | INFO | Downloaded 34GiB to GPU in 1.51457s (22.4486GiB/s) | 20260505 03:54:04.958 | INFO | Analyzing chunk frames = [134656,135168) | 20260505 03:54:04.958 | INFO | MemInfo: { | 20260505 03:54:04.958 | INFO | "gpu" : " 17 GB", | 20260505 03:54:04.958 | INFO | "huge" : "", | 20260505 03:54:04.958 | INFO | "huge_shared" : " 27 GB", | 20260505 03:54:04.958 | INFO | "malloc" : " 15 GB", | 20260505 03:54:04.958 | INFO | "pinned" : " 29 GB" | 20260505 03:54:04.958 | INFO | } | 20260505 03:54:04.958 | INFO | Perf report for the last 5.13938 seconds: | 20260505 03:54:04.958 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:54:04.958 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:04.958 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:04.958 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:04.958 | INFO | ANALYSIS : 58.0%, 3.0, 90.5ms, 0 | 20260505 03:54:04.958 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:04.958 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:04.958 | INFO | PRE_HQ : 32.1%, 1.0, 16.6ms, 0 | 20260505 03:54:04.958 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:04.958 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:04.958 | INFO | Uploaded 36GiB to GPU in 1.59895s (22.5148GiB/s) | 20260505 03:54:04.958 | INFO | Downloaded 34GiB to GPU in 1.51583s (22.43GiB/s) | 20260505 03:54:05.055 | INFO | Starting a round of Baz IO | 20260505 03:54:08.026 | INFO | Populating RTMetrics report | 20260505 03:54:08.027 | INFO | RTMetrics report populated | 20260505 03:54:08.027 | INFO | Opening tmp RTMetrics json file | 20260505 03:54:08.028 | INFO | Serializing RTMetrics to json | 20260505 03:54:08.029 | INFO | Serialization done, swapping tmp file now... | 20260505 03:54:08.030 | INFO | RTMetrics done | 20260505 03:54:08.098 | INFO | Finished a round of Baz IO: Wrote 1.558 GiB in 3.042 seconds | 20260505 03:54:08.098 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 60.69% / 0.48% / 38.82% | 20260505 03:54:08.098 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 14.2984% / 0.00147422% / 2.78155e-05% | 20260505 03:54:08.098 | INFO | Baz IO overall: | 20260505 03:54:08.098 | INFO | BytesWritten : 33319845151 | 20260505 03:54:08.098 | INFO | NumEvents : 14348012886 | 20260505 03:54:10.097 | INFO | Analyzing chunk frames = [135168,135680) | 20260505 03:54:10.097 | INFO | MemInfo: { | 20260505 03:54:10.097 | INFO | "gpu" : " 17 GB", | 20260505 03:54:10.097 | INFO | "huge" : "", | 20260505 03:54:10.097 | INFO | "huge_shared" : " 27 GB", | 20260505 03:54:10.097 | INFO | "malloc" : " 14 GB", | 20260505 03:54:10.097 | INFO | "pinned" : " 29 GB" | 20260505 03:54:10.097 | INFO | } | 20260505 03:54:10.097 | INFO | Perf report for the last 5.13918 seconds: | 20260505 03:54:10.097 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:54:10.097 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:10.097 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:10.097 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:10.097 | INFO | ANALYSIS : 58.7%, 3.0, 91.6ms, 0 | 20260505 03:54:10.097 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:10.097 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260505 03:54:10.097 | INFO | PRE_HQ : 58.2%, 2.2, 65.1ms, 0 | 20260505 03:54:10.097 | INFO | BAZWRITER : 58.8%, 3.0, 91.4ms, 0 | 20260505 03:54:10.097 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 03:54:10.098 | INFO | Uploaded 36GiB to GPU in 1.63503s (22.0179GiB/s) | 20260505 03:54:10.098 | INFO | Downloaded 38GiB to GPU in 1.69671s (22.3963GiB/s) | 20260505 03:54:15.236 | INFO | Analyzing chunk frames = [135680,136192) | 20260505 03:54:15.236 | INFO | MemInfo: { | 20260505 03:54:15.236 | INFO | "gpu" : " 17 GB", | 20260505 03:54:15.236 | INFO | "huge" : "", | 20260505 03:54:15.236 | INFO | "huge_shared" : " 27 GB", | 20260505 03:54:15.236 | INFO | "malloc" : " 14 GB", | 20260505 03:54:15.236 | INFO | "pinned" : " 29 GB" | 20260505 03:54:15.236 | INFO | } | 20260505 03:54:15.237 | INFO | Perf report for the last 5.13936 seconds: | 20260505 03:54:15.237 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:54:15.237 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:15.237 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:15.237 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:15.237 | INFO | ANALYSIS : 58.5%, 3.0, 91.2ms, 0 | 20260505 03:54:15.237 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:15.237 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:15.237 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260505 03:54:15.237 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:15.237 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:15.237 | INFO | Uploaded 36GiB to GPU in 1.59576s (22.5598GiB/s) | 20260505 03:54:15.237 | INFO | Downloaded 34GiB to GPU in 1.51244s (22.4802GiB/s) | 20260505 03:54:20.376 | INFO | Analyzing chunk frames = [136192,136704) | 20260505 03:54:20.376 | INFO | MemInfo: { | 20260505 03:54:20.376 | INFO | "gpu" : " 17 GB", | 20260505 03:54:20.376 | INFO | "huge" : "", | 20260505 03:54:20.376 | INFO | "huge_shared" : " 27 GB", | 20260505 03:54:20.376 | INFO | "malloc" : " 14 GB", | 20260505 03:54:20.376 | INFO | "pinned" : " 29 GB" | 20260505 03:54:20.376 | INFO | } | 20260505 03:54:20.376 | INFO | Perf report for the last 5.13947 seconds: | 20260505 03:54:20.376 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:54:20.376 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:20.376 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:20.376 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:20.376 | INFO | ANALYSIS : 57.8%, 3.0, 89.8ms, 0 | 20260505 03:54:20.376 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:20.376 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:20.376 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260505 03:54:20.376 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:20.376 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:20.376 | INFO | Uploaded 36GiB to GPU in 1.59946s (22.5076GiB/s) | 20260505 03:54:20.376 | INFO | Downloaded 34GiB to GPU in 1.51675s (22.4163GiB/s) | 20260505 03:54:25.515 | INFO | Analyzing chunk frames = [136704,137216) | 20260505 03:54:25.515 | INFO | MemInfo: { | 20260505 03:54:25.515 | INFO | "gpu" : " 17 GB", | 20260505 03:54:25.515 | INFO | "huge" : "", | 20260505 03:54:25.515 | INFO | "huge_shared" : " 27 GB", | 20260505 03:54:25.515 | INFO | "malloc" : " 14 GB", | 20260505 03:54:25.515 | INFO | "pinned" : " 29 GB" | 20260505 03:54:25.515 | INFO | } | 20260505 03:54:25.515 | INFO | Perf report for the last 5.13926 seconds: | 20260505 03:54:25.515 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:54:25.515 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:25.515 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:25.515 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:25.515 | INFO | ANALYSIS : 57.5%, 3.0, 89.1ms, 0 | 20260505 03:54:25.515 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:25.515 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:25.515 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 03:54:25.515 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:25.515 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:25.516 | INFO | Uploaded 36GiB to GPU in 1.59895s (22.5147GiB/s) | 20260505 03:54:25.516 | INFO | Downloaded 34GiB to GPU in 1.5156s (22.4333GiB/s) | 20260505 03:54:30.654 | INFO | Analyzing chunk frames = [137216,137728) | 20260505 03:54:30.654 | INFO | MemInfo: { | 20260505 03:54:30.654 | INFO | "gpu" : " 17 GB", | 20260505 03:54:30.654 | INFO | "huge" : "", | 20260505 03:54:30.654 | INFO | "huge_shared" : " 27 GB", | 20260505 03:54:30.654 | INFO | "malloc" : " 14 GB", | 20260505 03:54:30.654 | INFO | "pinned" : " 29 GB" | 20260505 03:54:30.655 | INFO | } | 20260505 03:54:30.655 | INFO | Perf report for the last 5.13933 seconds: | 20260505 03:54:30.655 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:54:30.655 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:30.655 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:30.655 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:30.655 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 03:54:30.655 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:30.655 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:30.655 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260505 03:54:30.655 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:30.655 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:30.655 | INFO | Uploaded 36GiB to GPU in 1.59771s (22.5323GiB/s) | 20260505 03:54:30.655 | INFO | Downloaded 34GiB to GPU in 1.51371s (22.4613GiB/s) | 20260505 03:54:35.794 | INFO | Analyzing chunk frames = [137728,138240) | 20260505 03:54:35.794 | INFO | MemInfo: { | 20260505 03:54:35.794 | INFO | "gpu" : " 17 GB", | 20260505 03:54:35.794 | INFO | "huge" : "", | 20260505 03:54:35.794 | INFO | "huge_shared" : " 27 GB", | 20260505 03:54:35.794 | INFO | "malloc" : " 14 GB", | 20260505 03:54:35.794 | INFO | "pinned" : " 29 GB" | 20260505 03:54:35.794 | INFO | } | 20260505 03:54:35.794 | INFO | Perf report for the last 5.13938 seconds: | 20260505 03:54:35.794 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:54:35.794 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:35.794 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:35.794 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:35.794 | INFO | ANALYSIS : 57.7%, 3.0, 90.0ms, 0 | 20260505 03:54:35.794 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:35.794 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:35.794 | INFO | PRE_HQ : 32.7%, 1.0, 17.0ms, 0 | 20260505 03:54:35.794 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:35.794 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:35.794 | INFO | Uploaded 36GiB to GPU in 1.59743s (22.5362GiB/s) | 20260505 03:54:35.795 | INFO | Downloaded 34GiB to GPU in 1.51339s (22.4661GiB/s) | 20260505 03:54:40.933 | INFO | Analyzing chunk frames = [138240,138752) | 20260505 03:54:40.933 | INFO | MemInfo: { | 20260505 03:54:40.933 | INFO | "gpu" : " 17 GB", | 20260505 03:54:40.933 | INFO | "huge" : "", | 20260505 03:54:40.933 | INFO | "huge_shared" : " 27 GB", | 20260505 03:54:40.933 | INFO | "malloc" : " 14 GB", | 20260505 03:54:40.933 | INFO | "pinned" : " 29 GB" | 20260505 03:54:40.933 | INFO | } | 20260505 03:54:40.933 | INFO | Perf report for the last 5.13928 seconds: | 20260505 03:54:40.933 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:54:40.933 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:40.933 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:40.933 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:40.933 | INFO | ANALYSIS : 57.9%, 3.0, 90.3ms, 0 | 20260505 03:54:40.933 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:40.933 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:40.933 | INFO | PRE_HQ : 32.9%, 1.0, 17.1ms, 0 | 20260505 03:54:40.933 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:40.933 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:40.934 | INFO | Uploaded 36GiB to GPU in 1.59816s (22.5258GiB/s) | 20260505 03:54:40.934 | INFO | Downloaded 34GiB to GPU in 1.51521s (22.4392GiB/s) | 20260505 03:54:46.072 | INFO | Analyzing chunk frames = [138752,139264) | 20260505 03:54:46.072 | INFO | MemInfo: { | 20260505 03:54:46.072 | INFO | "gpu" : " 17 GB", | 20260505 03:54:46.072 | INFO | "huge" : "", | 20260505 03:54:46.072 | INFO | "huge_shared" : " 27 GB", | 20260505 03:54:46.072 | INFO | "malloc" : " 15 GB", | 20260505 03:54:46.072 | INFO | "pinned" : " 29 GB" | 20260505 03:54:46.072 | INFO | } | 20260505 03:54:46.073 | INFO | Perf report for the last 5.13935 seconds: | 20260505 03:54:46.073 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:54:46.073 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:46.073 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:46.073 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:46.073 | INFO | ANALYSIS : 58.0%, 3.0, 90.5ms, 0 | 20260505 03:54:46.073 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:46.073 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:46.073 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 03:54:46.073 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:46.073 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:46.073 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 03:54:46.073 | INFO | Total mem usage: 14.9 GiB | 20260505 03:54:46.073 | INFO | All Time High Water Marks sum: 15.2 GiB (Savings: 1.52%) | 20260505 03:54:46.073 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -2.59%) | 20260505 03:54:46.073 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 03:54:46.073 | INFO | High Water Breakdown: | 20260505 03:54:46.073 | INFO | 58% : 8.44 GiB ( 99.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 03:54:46.073 | INFO | 41.8% : 6.09 GiB ( 91.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 03:54:46.073 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 03:54:46.073 | INFO | ------------------------------------------ | 20260505 03:54:46.073 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:54:46.073 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 03:54:46.073 | INFO | Total mem usage: 26.5 GiB | 20260505 03:54:46.073 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260505 03:54:46.073 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 03:54:46.073 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260505 03:54:46.073 | INFO | High Water Breakdown: | 20260505 03:54:46.073 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 03:54:46.073 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 03:54:46.073 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 03:54:46.073 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 03:54:46.073 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 03:54:46.073 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 03:54:46.073 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 03:54:46.073 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 03:54:46.073 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 03:54:46.073 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 03:54:46.073 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 03:54:46.073 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 03:54:46.073 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 03:54:46.073 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 03:54:46.073 | INFO | ------------------------------------------ | 20260505 03:54:46.073 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:54:46.073 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 03:54:46.073 | INFO | Total mem usage: 25.6 GiB | 20260505 03:54:46.073 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 03:54:46.073 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 03:54:46.073 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 03:54:46.073 | INFO | High Water Breakdown: | 20260505 03:54:46.073 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 03:54:46.073 | INFO | ------------------------------------------ | 20260505 03:54:46.073 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:54:46.073 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 03:54:46.073 | INFO | Total mem usage: 15.7 GiB | 20260505 03:54:46.073 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260505 03:54:46.073 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 03:54:46.073 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260505 03:54:46.073 | INFO | High Water Breakdown: | 20260505 03:54:46.073 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 03:54:46.073 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 03:54:46.073 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 03:54:46.073 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 03:54:46.073 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 03:54:46.073 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 03:54:46.073 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 03:54:46.073 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 03:54:46.073 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 03:54:46.073 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 03:54:46.073 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 03:54:46.073 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 03:54:46.073 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 03:54:46.073 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 03:54:46.073 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 03:54:46.073 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 03:54:46.073 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 03:54:46.073 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 03:54:46.073 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 03:54:46.073 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 03:54:46.073 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 03:54:46.073 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 03:54:46.073 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 03:54:46.073 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 03:54:46.073 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 03:54:46.073 | INFO | ------------------------------------------ | 20260505 03:54:46.073 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:54:46.073 | INFO | Uploaded 36GiB to GPU in 1.59934s (22.5093GiB/s) | 20260505 03:54:46.073 | INFO | Downloaded 34GiB to GPU in 1.51612s (22.4256GiB/s) | 20260505 03:54:46.073 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 03:54:46.074 | INFO | PerfReport: Gathered 1536 samples. | 20260505 03:54:46.074 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 03:54:46.074 | INFO | PerfReport: Full Execution -- 3.07e+01 / -nan, 1.28e+01 / 0.00e+00 | 20260505 03:54:46.074 | INFO | PerfReport: 0.28%, AggregateStats -- 8.65e-02 / -nan, 4.57e-03 / 0.00e+00 | 20260505 03:54:46.074 | INFO | PerfReport: 10.75%, Baseline -- 3.30e+00 / -nan, 1.67e-02 / 0.00e+00 | 20260505 03:54:46.074 | INFO | PerfReport: 3.01%, Histogram -- 9.24e-01 / -nan, 5.19e-02 / 0.00e+00 | 20260505 03:54:46.074 | INFO | PerfReport: 16.02%, DME -- 3.93e+01 / -nan, 3.35e+00 / 0.00e+00 | 20260505 03:54:46.075 | INFO | PerfReport: 56.15%, FrameLabeling -- 1.72e+01 / -nan, 4.42e-02 / 0.00e+00 | 20260505 03:54:46.075 | INFO | PerfReport: 7.80%, PulseAccumulating -- 2.39e+00 / -nan, 1.61e-01 / 0.00e+00 | 20260505 03:54:46.075 | INFO | PerfReport: 4.22%, Metrics -- 1.29e+00 / -nan, 2.11e-01 / 0.00e+00 | 20260505 03:54:46.075 | INFO | PerfReport: 1.76%, PulseSerializing -- 5.41e-01 / -nan, 8.62e-02 / 0.00e+00 | 20260505 03:54:46.075 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 03:54:46.075 | INFO | PerfReport: Gathered 1536 samples. | 20260505 03:54:46.075 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 03:54:46.075 | INFO | PerfReport: Full Execution -- 9.21e+01 / -nan, 1.92e+01 / 0.00e+00 | 20260505 03:54:46.075 | INFO | PerfReport: 18.12%, Upload -- 1.67e+01 / 0.00e+00, 3.56e-01 / 0.00e+00 | 20260505 03:54:46.075 | INFO | PerfReport: 17.38%, Download -- 1.60e+01 / 0.00e+00, 6.79e-01 / 0.00e+00 | 20260505 03:54:46.075 | INFO | PerfReport: 33.29%, Compute -- 3.07e+01 / 0.00e+00, 1.28e+01 / 0.00e+00 | 20260505 03:54:46.075 | INFO | PerfReport: 0.55%, LockUpload -- 5.04e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260505 03:54:46.075 | INFO | PerfReport: 0.00%, LockDownload -- 3.98e-04 / 0.00e+00, 9.51e-05 / 0.00e+00 | 20260505 03:54:46.075 | INFO | PerfReport: 30.64%, LockCompute -- 2.82e+01 / 0.00e+00, 1.74e+01 / 0.00e+00 | 20260505 03:54:46.075 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.11e-02 / 0.00e+00 | 20260505 03:54:46.171 | INFO | Starting a round of Baz IO | 20260505 03:54:49.139 | INFO | Populating RTMetrics report | 20260505 03:54:49.139 | INFO | RTMetrics report populated | 20260505 03:54:49.139 | INFO | Opening tmp RTMetrics json file | 20260505 03:54:49.141 | INFO | Serializing RTMetrics to json | 20260505 03:54:49.142 | INFO | Serialization done, swapping tmp file now... | 20260505 03:54:49.142 | INFO | RTMetrics done | 20260505 03:54:49.209 | INFO | Finished a round of Baz IO: Wrote 1.57 GiB in 3.037 seconds | 20260505 03:54:49.209 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 60.23% / 0.46% / 39.31% | 20260505 03:54:49.209 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 14.1659% / 0.00145435% / 2.38419e-05% | 20260505 03:54:49.209 | INFO | Baz IO overall: | 20260505 03:54:49.209 | INFO | BytesWritten : 35005529802 | 20260505 03:54:49.209 | INFO | NumEvents : 15142377266 | 20260505 03:54:51.212 | INFO | Analyzing chunk frames = [139264,139776) | 20260505 03:54:51.212 | INFO | MemInfo: { | 20260505 03:54:51.212 | INFO | "gpu" : " 17 GB", | 20260505 03:54:51.212 | INFO | "huge" : "", | 20260505 03:54:51.212 | INFO | "huge_shared" : " 27 GB", | 20260505 03:54:51.212 | INFO | "malloc" : " 14 GB", | 20260505 03:54:51.212 | INFO | "pinned" : " 29 GB" | 20260505 03:54:51.212 | INFO | } | 20260505 03:54:51.212 | INFO | Perf report for the last 5.1393 seconds: | 20260505 03:54:51.212 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:54:51.212 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:51.212 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:51.212 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:51.212 | INFO | ANALYSIS : 58.6%, 3.0, 91.5ms, 0 | 20260505 03:54:51.212 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:51.212 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260505 03:54:51.212 | INFO | PRE_HQ : 58.1%, 2.2, 64.8ms, 0 | 20260505 03:54:51.212 | INFO | BAZWRITER : 58.7%, 3.0, 91.7ms, 0 | 20260505 03:54:51.212 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 03:54:51.212 | INFO | Uploaded 36GiB to GPU in 1.63796s (21.9785GiB/s) | 20260505 03:54:51.212 | INFO | Downloaded 38GiB to GPU in 1.70003s (22.3525GiB/s) | 20260505 03:54:56.351 | INFO | Analyzing chunk frames = [139776,140288) | 20260505 03:54:56.351 | INFO | MemInfo: { | 20260505 03:54:56.351 | INFO | "gpu" : " 17 GB", | 20260505 03:54:56.351 | INFO | "huge" : "", | 20260505 03:54:56.351 | INFO | "huge_shared" : " 27 GB", | 20260505 03:54:56.351 | INFO | "malloc" : " 14 GB", | 20260505 03:54:56.351 | INFO | "pinned" : " 29 GB" | 20260505 03:54:56.351 | INFO | } | 20260505 03:54:56.351 | INFO | Perf report for the last 5.1394 seconds: | 20260505 03:54:56.351 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:54:56.351 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:56.351 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:56.351 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:56.351 | INFO | ANALYSIS : 58.5%, 3.0, 91.2ms, 0 | 20260505 03:54:56.352 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:56.352 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:56.352 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260505 03:54:56.352 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:56.352 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:54:56.352 | INFO | Uploaded 36GiB to GPU in 1.59609s (22.5552GiB/s) | 20260505 03:54:56.352 | INFO | Downloaded 34GiB to GPU in 1.51183s (22.4894GiB/s) | 20260505 03:55:01.490 | INFO | Analyzing chunk frames = [140288,140800) | 20260505 03:55:01.490 | INFO | MemInfo: { | 20260505 03:55:01.490 | INFO | "gpu" : " 17 GB", | 20260505 03:55:01.490 | INFO | "huge" : "", | 20260505 03:55:01.490 | INFO | "huge_shared" : " 27 GB", | 20260505 03:55:01.490 | INFO | "malloc" : " 14 GB", | 20260505 03:55:01.490 | INFO | "pinned" : " 29 GB" | 20260505 03:55:01.491 | INFO | } | 20260505 03:55:01.491 | INFO | Perf report for the last 5.13931 seconds: | 20260505 03:55:01.491 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:55:01.491 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:01.491 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:01.491 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:01.491 | INFO | ANALYSIS : 57.8%, 3.0, 89.8ms, 0 | 20260505 03:55:01.491 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:01.491 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:01.491 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 03:55:01.491 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:01.491 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:01.491 | INFO | Uploaded 36GiB to GPU in 1.6006s (22.4916GiB/s) | 20260505 03:55:01.491 | INFO | Downloaded 34GiB to GPU in 1.51778s (22.4012GiB/s) | 20260505 03:55:06.630 | INFO | Analyzing chunk frames = [140800,141312) | 20260505 03:55:06.630 | INFO | MemInfo: { | 20260505 03:55:06.630 | INFO | "gpu" : " 17 GB", | 20260505 03:55:06.630 | INFO | "huge" : "", | 20260505 03:55:06.630 | INFO | "huge_shared" : " 27 GB", | 20260505 03:55:06.630 | INFO | "malloc" : " 14 GB", | 20260505 03:55:06.630 | INFO | "pinned" : " 29 GB" | 20260505 03:55:06.630 | INFO | } | 20260505 03:55:06.630 | INFO | Perf report for the last 5.13934 seconds: | 20260505 03:55:06.630 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:55:06.630 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:06.630 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:06.630 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:06.630 | INFO | ANALYSIS : 57.5%, 3.0, 89.1ms, 0 | 20260505 03:55:06.630 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:06.630 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:06.630 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 03:55:06.630 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:06.630 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:06.631 | INFO | Uploaded 36GiB to GPU in 1.59907s (22.5131GiB/s) | 20260505 03:55:06.631 | INFO | Downloaded 34GiB to GPU in 1.51519s (22.4395GiB/s) | 20260505 03:55:11.769 | INFO | Analyzing chunk frames = [141312,141824) | 20260505 03:55:11.769 | INFO | MemInfo: { | 20260505 03:55:11.769 | INFO | "gpu" : " 17 GB", | 20260505 03:55:11.769 | INFO | "huge" : "", | 20260505 03:55:11.769 | INFO | "huge_shared" : " 27 GB", | 20260505 03:55:11.769 | INFO | "malloc" : " 14 GB", | 20260505 03:55:11.769 | INFO | "pinned" : " 29 GB" | 20260505 03:55:11.769 | INFO | } | 20260505 03:55:11.769 | INFO | Perf report for the last 5.13921 seconds: | 20260505 03:55:11.769 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:55:11.769 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:11.769 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:11.769 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:11.769 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 03:55:11.769 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:11.769 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:11.769 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260505 03:55:11.769 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:11.769 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:11.770 | INFO | Uploaded 36GiB to GPU in 1.59673s (22.546GiB/s) | 20260505 03:55:11.770 | INFO | Downloaded 34GiB to GPU in 1.51414s (22.455GiB/s) | 20260505 03:55:16.909 | INFO | Analyzing chunk frames = [141824,142336) | 20260505 03:55:16.909 | INFO | MemInfo: { | 20260505 03:55:16.909 | INFO | "gpu" : " 17 GB", | 20260505 03:55:16.909 | INFO | "huge" : "", | 20260505 03:55:16.909 | INFO | "huge_shared" : " 27 GB", | 20260505 03:55:16.909 | INFO | "malloc" : " 14 GB", | 20260505 03:55:16.909 | INFO | "pinned" : " 29 GB" | 20260505 03:55:16.909 | INFO | } | 20260505 03:55:16.909 | INFO | Perf report for the last 5.1394 seconds: | 20260505 03:55:16.909 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:55:16.909 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:16.909 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:16.909 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:16.909 | INFO | ANALYSIS : 57.7%, 3.0, 90.0ms, 0 | 20260505 03:55:16.909 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:16.909 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:16.909 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260505 03:55:16.909 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:16.909 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:16.909 | INFO | Uploaded 36GiB to GPU in 1.59723s (22.539GiB/s) | 20260505 03:55:16.909 | INFO | Downloaded 34GiB to GPU in 1.5141s (22.4556GiB/s) | 20260505 03:55:22.048 | INFO | Analyzing chunk frames = [142336,142848) | 20260505 03:55:22.048 | INFO | MemInfo: { | 20260505 03:55:22.048 | INFO | "gpu" : " 17 GB", | 20260505 03:55:22.048 | INFO | "huge" : "", | 20260505 03:55:22.048 | INFO | "huge_shared" : " 27 GB", | 20260505 03:55:22.048 | INFO | "malloc" : " 14 GB", | 20260505 03:55:22.048 | INFO | "pinned" : " 29 GB" | 20260505 03:55:22.048 | INFO | } | 20260505 03:55:22.048 | INFO | Perf report for the last 5.13931 seconds: | 20260505 03:55:22.048 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:55:22.048 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:22.048 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:22.048 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:22.048 | INFO | ANALYSIS : 57.9%, 3.0, 90.3ms, 0 | 20260505 03:55:22.048 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:22.048 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:22.048 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260505 03:55:22.048 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:22.048 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:22.049 | INFO | Uploaded 36GiB to GPU in 1.59829s (22.5241GiB/s) | 20260505 03:55:22.049 | INFO | Downloaded 34GiB to GPU in 1.51525s (22.4385GiB/s) | 20260505 03:55:27.187 | INFO | Analyzing chunk frames = [142848,143360) | 20260505 03:55:27.187 | INFO | MemInfo: { | 20260505 03:55:27.187 | INFO | "gpu" : " 17 GB", | 20260505 03:55:27.187 | INFO | "huge" : "", | 20260505 03:55:27.187 | INFO | "huge_shared" : " 27 GB", | 20260505 03:55:27.187 | INFO | "malloc" : " 15 GB", | 20260505 03:55:27.187 | INFO | "pinned" : " 29 GB" | 20260505 03:55:27.187 | INFO | } | 20260505 03:55:27.187 | INFO | Perf report for the last 5.13936 seconds: | 20260505 03:55:27.187 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:55:27.187 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:27.187 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:27.187 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:27.187 | INFO | ANALYSIS : 58.0%, 3.0, 90.5ms, 0 | 20260505 03:55:27.187 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:27.187 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:27.188 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260505 03:55:27.188 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:27.188 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:27.188 | INFO | Uploaded 36GiB to GPU in 1.59681s (22.545GiB/s) | 20260505 03:55:27.188 | INFO | Downloaded 34GiB to GPU in 1.5155s (22.4349GiB/s) | 20260505 03:55:27.285 | INFO | Starting a round of Baz IO | 20260505 03:55:30.252 | INFO | Populating RTMetrics report | 20260505 03:55:30.253 | INFO | RTMetrics report populated | 20260505 03:55:30.253 | INFO | Opening tmp RTMetrics json file | 20260505 03:55:30.254 | INFO | Serializing RTMetrics to json | 20260505 03:55:30.255 | INFO | Serialization done, swapping tmp file now... | 20260505 03:55:30.255 | INFO | RTMetrics done | 20260505 03:55:30.320 | INFO | Finished a round of Baz IO: Wrote 1.584 GiB in 3.034 seconds | 20260505 03:55:30.320 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 59.79% / 0.44% / 39.77% | 20260505 03:55:30.320 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 14.096% / 0.00133117% / 1.98682e-05% | 20260505 03:55:30.320 | INFO | Baz IO overall: | 20260505 03:55:30.320 | INFO | BytesWritten : 36706563317 | 20260505 03:55:30.320 | INFO | NumEvents : 15944541233 | 20260505 03:55:32.326 | INFO | Analyzing chunk frames = [143360,143872) | 20260505 03:55:32.326 | INFO | MemInfo: { | 20260505 03:55:32.327 | INFO | "gpu" : " 17 GB", | 20260505 03:55:32.327 | INFO | "huge" : "", | 20260505 03:55:32.327 | INFO | "huge_shared" : " 27 GB", | 20260505 03:55:32.327 | INFO | "malloc" : " 14 GB", | 20260505 03:55:32.327 | INFO | "pinned" : " 29 GB" | 20260505 03:55:32.327 | INFO | } | 20260505 03:55:32.327 | INFO | Perf report for the last 5.13927 seconds: | 20260505 03:55:32.327 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:55:32.327 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:32.327 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:32.327 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:32.327 | INFO | ANALYSIS : 58.6%, 3.0, 91.4ms, 0 | 20260505 03:55:32.327 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:32.327 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260505 03:55:32.327 | INFO | PRE_HQ : 58.2%, 2.1, 64.7ms, 0 | 20260505 03:55:32.327 | INFO | BAZWRITER : 58.6%, 3.1, 92.7ms, 0 | 20260505 03:55:32.327 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 03:55:32.327 | INFO | Uploaded 36GiB to GPU in 1.63867s (21.9691GiB/s) | 20260505 03:55:32.327 | INFO | Downloaded 38GiB to GPU in 1.69968s (22.3572GiB/s) | 20260505 03:55:37.466 | INFO | Analyzing chunk frames = [143872,144384) | 20260505 03:55:37.466 | INFO | MemInfo: { | 20260505 03:55:37.466 | INFO | "gpu" : " 17 GB", | 20260505 03:55:37.466 | INFO | "huge" : "", | 20260505 03:55:37.466 | INFO | "huge_shared" : " 27 GB", | 20260505 03:55:37.466 | INFO | "malloc" : " 14 GB", | 20260505 03:55:37.466 | INFO | "pinned" : " 29 GB" | 20260505 03:55:37.466 | INFO | } | 20260505 03:55:37.466 | INFO | Perf report for the last 5.13936 seconds: | 20260505 03:55:37.466 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:55:37.466 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:37.466 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:37.466 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:37.466 | INFO | ANALYSIS : 58.4%, 3.0, 91.1ms, 0 | 20260505 03:55:37.466 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:37.466 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:37.467 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260505 03:55:37.467 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:37.467 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:37.467 | INFO | Uploaded 36GiB to GPU in 1.59601s (22.5563GiB/s) | 20260505 03:55:37.467 | INFO | Downloaded 34GiB to GPU in 1.512s (22.4868GiB/s) | 20260505 03:55:42.606 | INFO | Analyzing chunk frames = [144384,144896) | 20260505 03:55:42.606 | INFO | MemInfo: { | 20260505 03:55:42.606 | INFO | "gpu" : " 17 GB", | 20260505 03:55:42.606 | INFO | "huge" : "", | 20260505 03:55:42.606 | INFO | "huge_shared" : " 27 GB", | 20260505 03:55:42.606 | INFO | "malloc" : " 14 GB", | 20260505 03:55:42.606 | INFO | "pinned" : " 29 GB" | 20260505 03:55:42.606 | INFO | } | 20260505 03:55:42.607 | INFO | Perf report for the last 5.14041 seconds: | 20260505 03:55:42.607 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:55:42.607 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:42.607 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:42.607 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:42.607 | INFO | ANALYSIS : 57.7%, 3.0, 89.7ms, 0 | 20260505 03:55:42.607 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:42.607 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:42.607 | INFO | PRE_HQ : 31.9%, 1.0, 16.5ms, 0 | 20260505 03:55:42.607 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:42.607 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:42.607 | INFO | Uploaded 36GiB to GPU in 1.59978s (22.503GiB/s) | 20260505 03:55:42.607 | INFO | Downloaded 34GiB to GPU in 1.5166s (22.4185GiB/s) | 20260505 03:55:47.745 | INFO | Analyzing chunk frames = [144896,145408) | 20260505 03:55:47.745 | INFO | MemInfo: { | 20260505 03:55:47.745 | INFO | "gpu" : " 17 GB", | 20260505 03:55:47.745 | INFO | "huge" : "", | 20260505 03:55:47.745 | INFO | "huge_shared" : " 27 GB", | 20260505 03:55:47.745 | INFO | "malloc" : " 14 GB", | 20260505 03:55:47.745 | INFO | "pinned" : " 29 GB" | 20260505 03:55:47.745 | INFO | } | 20260505 03:55:47.745 | INFO | Perf report for the last 5.13848 seconds: | 20260505 03:55:47.745 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:55:47.745 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:47.745 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:47.745 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:47.745 | INFO | ANALYSIS : 57.5%, 3.0, 89.1ms, 0 | 20260505 03:55:47.745 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:47.745 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:47.745 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260505 03:55:47.745 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:47.745 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:47.745 | INFO | Uploaded 36GiB to GPU in 1.59928s (22.5102GiB/s) | 20260505 03:55:47.745 | INFO | Downloaded 34GiB to GPU in 1.51597s (22.4279GiB/s) | 20260505 03:55:52.884 | INFO | Analyzing chunk frames = [145408,145920) | 20260505 03:55:52.884 | INFO | MemInfo: { | 20260505 03:55:52.884 | INFO | "gpu" : " 17 GB", | 20260505 03:55:52.884 | INFO | "huge" : "", | 20260505 03:55:52.884 | INFO | "huge_shared" : " 27 GB", | 20260505 03:55:52.887 | INFO | "malloc" : " 14 GB", | 20260505 03:55:52.887 | INFO | "pinned" : " 29 GB" | 20260505 03:55:52.887 | INFO | } | 20260505 03:55:52.887 | INFO | Perf report for the last 5.14244 seconds: | 20260505 03:55:52.887 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:55:52.887 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:52.887 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:52.887 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:52.888 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 03:55:52.888 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:52.888 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:52.888 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 03:55:52.888 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:52.888 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:52.888 | INFO | Uploaded 36GiB to GPU in 1.59784s (22.5304GiB/s) | 20260505 03:55:52.888 | INFO | Downloaded 34GiB to GPU in 1.51341s (22.4658GiB/s) | 20260505 03:55:58.023 | INFO | Analyzing chunk frames = [145920,146432) | 20260505 03:55:58.023 | INFO | MemInfo: { | 20260505 03:55:58.023 | INFO | "gpu" : " 17 GB", | 20260505 03:55:58.023 | INFO | "huge" : "", | 20260505 03:55:58.023 | INFO | "huge_shared" : " 27 GB", | 20260505 03:55:58.023 | INFO | "malloc" : " 14 GB", | 20260505 03:55:58.023 | INFO | "pinned" : " 29 GB" | 20260505 03:55:58.023 | INFO | } | 20260505 03:55:58.024 | INFO | Perf report for the last 5.13604 seconds: | 20260505 03:55:58.024 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:55:58.024 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:58.024 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:58.024 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:58.024 | INFO | ANALYSIS : 57.7%, 3.0, 90.0ms, 0 | 20260505 03:55:58.024 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:58.024 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:58.024 | INFO | PRE_HQ : 32.1%, 1.0, 16.6ms, 0 | 20260505 03:55:58.024 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:58.024 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:55:58.024 | INFO | Uploaded 36GiB to GPU in 1.59804s (22.5275GiB/s) | 20260505 03:55:58.024 | INFO | Downloaded 34GiB to GPU in 1.51387s (22.459GiB/s) | 20260505 03:56:03.163 | INFO | Analyzing chunk frames = [146432,146944) | 20260505 03:56:03.163 | INFO | MemInfo: { | 20260505 03:56:03.163 | INFO | "gpu" : " 17 GB", | 20260505 03:56:03.163 | INFO | "huge" : "", | 20260505 03:56:03.163 | INFO | "huge_shared" : " 27 GB", | 20260505 03:56:03.163 | INFO | "malloc" : " 14 GB", | 20260505 03:56:03.163 | INFO | "pinned" : " 29 GB" | 20260505 03:56:03.163 | INFO | } | 20260505 03:56:03.163 | INFO | Perf report for the last 5.1393 seconds: | 20260505 03:56:03.163 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:56:03.163 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:03.163 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:03.163 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:03.163 | INFO | ANALYSIS : 57.9%, 3.0, 90.3ms, 0 | 20260505 03:56:03.163 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:03.163 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:03.163 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260505 03:56:03.163 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:03.163 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:03.163 | INFO | Uploaded 36GiB to GPU in 1.59859s (22.5199GiB/s) | 20260505 03:56:03.163 | INFO | Downloaded 34GiB to GPU in 1.51604s (22.4268GiB/s) | 20260505 03:56:08.303 | INFO | Analyzing chunk frames = [146944,147456) | 20260505 03:56:08.303 | INFO | MemInfo: { | 20260505 03:56:08.303 | INFO | "gpu" : " 17 GB", | 20260505 03:56:08.303 | INFO | "huge" : "", | 20260505 03:56:08.303 | INFO | "huge_shared" : " 27 GB", | 20260505 03:56:08.303 | INFO | "malloc" : " 14 GB", | 20260505 03:56:08.303 | INFO | "pinned" : " 29 GB" | 20260505 03:56:08.303 | INFO | } | 20260505 03:56:08.303 | INFO | Perf report for the last 5.14016 seconds: | 20260505 03:56:08.303 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:56:08.303 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:08.303 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:08.303 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:08.303 | INFO | ANALYSIS : 58.0%, 3.0, 90.4ms, 0 | 20260505 03:56:08.303 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:08.303 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:08.303 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 03:56:08.303 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:08.303 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:08.303 | INFO | Uploaded 36GiB to GPU in 1.59831s (22.5238GiB/s) | 20260505 03:56:08.303 | INFO | Downloaded 34GiB to GPU in 1.51635s (22.4223GiB/s) | 20260505 03:56:08.304 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 03:56:08.304 | INFO | Total mem usage: 14.9 GiB | 20260505 03:56:08.304 | INFO | All Time High Water Marks sum: 15.2 GiB (Savings: 1.52%) | 20260505 03:56:08.304 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -2.65%) | 20260505 03:56:08.304 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 03:56:08.304 | INFO | High Water Breakdown: | 20260505 03:56:08.304 | INFO | 58.1% : 8.44 GiB ( 99.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 03:56:08.304 | INFO | 41.8% : 6.08 GiB ( 91.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 03:56:08.304 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 03:56:08.304 | INFO | ------------------------------------------ | 20260505 03:56:08.304 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:56:08.304 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 03:56:08.304 | INFO | Total mem usage: 26.5 GiB | 20260505 03:56:08.304 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260505 03:56:08.304 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 03:56:08.304 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260505 03:56:08.304 | INFO | High Water Breakdown: | 20260505 03:56:08.304 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 03:56:08.304 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 03:56:08.304 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 03:56:08.304 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 03:56:08.304 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 03:56:08.304 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 03:56:08.304 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 03:56:08.304 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 03:56:08.304 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 03:56:08.304 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 03:56:08.304 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 03:56:08.304 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 03:56:08.304 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 03:56:08.304 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 03:56:08.304 | INFO | ------------------------------------------ | 20260505 03:56:08.304 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:56:08.304 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 03:56:08.304 | INFO | Total mem usage: 25.6 GiB | 20260505 03:56:08.304 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 03:56:08.304 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 03:56:08.304 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 03:56:08.304 | INFO | High Water Breakdown: | 20260505 03:56:08.304 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 03:56:08.304 | INFO | ------------------------------------------ | 20260505 03:56:08.304 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:56:08.304 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 03:56:08.304 | INFO | Total mem usage: 15.7 GiB | 20260505 03:56:08.304 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260505 03:56:08.304 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 03:56:08.304 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260505 03:56:08.304 | INFO | High Water Breakdown: | 20260505 03:56:08.304 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 03:56:08.304 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 03:56:08.304 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 03:56:08.304 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 03:56:08.304 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 03:56:08.304 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 03:56:08.304 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 03:56:08.304 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 03:56:08.304 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 03:56:08.304 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 03:56:08.304 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 03:56:08.304 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 03:56:08.304 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 03:56:08.304 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 03:56:08.304 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 03:56:08.304 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 03:56:08.304 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 03:56:08.304 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 03:56:08.304 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 03:56:08.304 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 03:56:08.304 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 03:56:08.304 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 03:56:08.304 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 03:56:08.305 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 03:56:08.305 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 03:56:08.305 | INFO | ------------------------------------------ | 20260505 03:56:08.305 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:56:08.305 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 03:56:08.305 | INFO | PerfReport: Gathered 1536 samples. | 20260505 03:56:08.305 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 03:56:08.305 | INFO | PerfReport: Full Execution -- 3.06e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260505 03:56:08.305 | INFO | PerfReport: 0.28%, AggregateStats -- 8.63e-02 / -nan, 5.02e-03 / 0.00e+00 | 20260505 03:56:08.305 | INFO | PerfReport: 10.75%, Baseline -- 3.29e+00 / -nan, 1.73e-02 / 0.00e+00 | 20260505 03:56:08.305 | INFO | PerfReport: 3.01%, Histogram -- 9.24e-01 / -nan, 5.18e-02 / 0.00e+00 | 20260505 03:56:08.305 | INFO | PerfReport: 16.01%, DME -- 3.93e+01 / -nan, 3.33e+00 / 0.00e+00 | 20260505 03:56:08.305 | INFO | PerfReport: 56.18%, FrameLabeling -- 1.72e+01 / -nan, 4.46e-02 / 0.00e+00 | 20260505 03:56:08.305 | INFO | PerfReport: 7.79%, PulseAccumulating -- 2.39e+00 / -nan, 1.61e-01 / 0.00e+00 | 20260505 03:56:08.305 | INFO | PerfReport: 4.21%, Metrics -- 1.29e+00 / -nan, 2.10e-01 / 0.00e+00 | 20260505 03:56:08.305 | INFO | PerfReport: 1.76%, PulseSerializing -- 5.39e-01 / -nan, 8.64e-02 / 0.00e+00 | 20260505 03:56:08.305 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 03:56:08.305 | INFO | PerfReport: Gathered 1536 samples. | 20260505 03:56:08.305 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 03:56:08.305 | INFO | PerfReport: Full Execution -- 9.21e+01 / -nan, 1.93e+01 / 0.00e+00 | 20260505 03:56:08.305 | INFO | PerfReport: 18.13%, Upload -- 1.67e+01 / 0.00e+00, 3.65e-01 / 0.00e+00 | 20260505 03:56:08.305 | INFO | PerfReport: 17.40%, Download -- 1.60e+01 / 0.00e+00, 6.94e-01 / 0.00e+00 | 20260505 03:56:08.305 | INFO | PerfReport: 33.29%, Compute -- 3.07e+01 / 0.00e+00, 1.28e+01 / 0.00e+00 | 20260505 03:56:08.305 | INFO | PerfReport: 0.55%, LockUpload -- 5.05e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260505 03:56:08.305 | INFO | PerfReport: 0.00%, LockDownload -- 3.95e-04 / 0.00e+00, 1.90e-04 / 0.00e+00 | 20260505 03:56:08.305 | INFO | PerfReport: 30.61%, LockCompute -- 2.82e+01 / 0.00e+00, 1.74e+01 / 0.00e+00 | 20260505 03:56:08.305 | INFO | PerfReport: 0.02%, Misc -- 1.45e-02 / 0.00e+00, 2.83e-02 / 0.00e+00 | 20260505 03:56:08.400 | INFO | Starting a round of Baz IO | 20260505 03:56:11.369 | INFO | Populating RTMetrics report | 20260505 03:56:11.370 | INFO | RTMetrics report populated | 20260505 03:56:11.370 | INFO | Opening tmp RTMetrics json file | 20260505 03:56:11.371 | INFO | Serializing RTMetrics to json | 20260505 03:56:11.372 | INFO | Serialization done, swapping tmp file now... | 20260505 03:56:11.373 | INFO | RTMetrics done | 20260505 03:56:11.438 | INFO | Finished a round of Baz IO: Wrote 1.598 GiB in 3.037 seconds | 20260505 03:56:11.438 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 59.38% / 0.42% / 40.21% | 20260505 03:56:11.438 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 14.0499% / 0.0014027% / 2.78155e-05% | 20260505 03:56:11.438 | INFO | Baz IO overall: | 20260505 03:56:11.438 | INFO | BytesWritten : 38422333799 | 20260505 03:56:11.438 | INFO | NumEvents : 16754013476 | 20260505 03:56:13.441 | INFO | Analyzing chunk frames = [147456,147968) | 20260505 03:56:13.441 | INFO | MemInfo: { | 20260505 03:56:13.441 | INFO | "gpu" : " 17 GB", | 20260505 03:56:13.441 | INFO | "huge" : "", | 20260505 03:56:13.442 | INFO | "huge_shared" : " 27 GB", | 20260505 03:56:13.442 | INFO | "malloc" : " 14 GB", | 20260505 03:56:13.442 | INFO | "pinned" : " 29 GB" | 20260505 03:56:13.442 | INFO | } | 20260505 03:56:13.442 | INFO | Perf report for the last 5.13858 seconds: | 20260505 03:56:13.442 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:56:13.442 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:13.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:13.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:13.442 | INFO | ANALYSIS : 58.6%, 3.0, 91.5ms, 0 | 20260505 03:56:13.442 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:13.442 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260505 03:56:13.442 | INFO | PRE_HQ : 58.1%, 2.1, 64.7ms, 0 | 20260505 03:56:13.442 | INFO | BAZWRITER : 58.7%, 3.1, 93.0ms, 0 | 20260505 03:56:13.442 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 03:56:13.442 | INFO | Uploaded 36GiB to GPU in 1.63736s (21.9867GiB/s) | 20260505 03:56:13.442 | INFO | Downloaded 38GiB to GPU in 1.69896s (22.3666GiB/s) | 20260505 03:56:18.581 | INFO | Analyzing chunk frames = [147968,148480) | 20260505 03:56:18.581 | INFO | MemInfo: { | 20260505 03:56:18.581 | INFO | "gpu" : " 17 GB", | 20260505 03:56:18.581 | INFO | "huge" : "", | 20260505 03:56:18.581 | INFO | "huge_shared" : " 27 GB", | 20260505 03:56:18.581 | INFO | "malloc" : " 14 GB", | 20260505 03:56:18.581 | INFO | "pinned" : " 29 GB" | 20260505 03:56:18.581 | INFO | } | 20260505 03:56:18.581 | INFO | Perf report for the last 5.13909 seconds: | 20260505 03:56:18.584 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:56:18.584 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:18.584 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:18.584 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:18.584 | INFO | ANALYSIS : 58.4%, 3.0, 91.1ms, 0 | 20260505 03:56:18.584 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:18.584 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:18.584 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 03:56:18.584 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:18.584 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:18.584 | INFO | Uploaded 36GiB to GPU in 1.59638s (22.551GiB/s) | 20260505 03:56:18.584 | INFO | Downloaded 34GiB to GPU in 1.51242s (22.4805GiB/s) | 20260505 03:56:23.720 | INFO | Analyzing chunk frames = [148480,148992) | 20260505 03:56:23.720 | INFO | MemInfo: { | 20260505 03:56:23.720 | INFO | "gpu" : " 17 GB", | 20260505 03:56:23.720 | INFO | "huge" : "", | 20260505 03:56:23.720 | INFO | "huge_shared" : " 27 GB", | 20260505 03:56:23.720 | INFO | "malloc" : " 14 GB", | 20260505 03:56:23.720 | INFO | "pinned" : " 29 GB" | 20260505 03:56:23.720 | INFO | } | 20260505 03:56:23.720 | INFO | Perf report for the last 5.13959 seconds: | 20260505 03:56:23.720 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:56:23.720 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:23.720 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:23.720 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:23.720 | INFO | ANALYSIS : 57.8%, 3.0, 89.8ms, 0 | 20260505 03:56:23.720 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:23.720 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:23.720 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260505 03:56:23.720 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:23.720 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:23.721 | INFO | Uploaded 36GiB to GPU in 1.59953s (22.5066GiB/s) | 20260505 03:56:23.721 | INFO | Downloaded 34GiB to GPU in 1.51697s (22.4132GiB/s) | 20260505 03:56:28.860 | INFO | Analyzing chunk frames = [148992,149504) | 20260505 03:56:28.860 | INFO | MemInfo: { | 20260505 03:56:28.860 | INFO | "gpu" : " 17 GB", | 20260505 03:56:28.860 | INFO | "huge" : "", | 20260505 03:56:28.860 | INFO | "huge_shared" : " 27 GB", | 20260505 03:56:28.860 | INFO | "malloc" : " 14 GB", | 20260505 03:56:28.860 | INFO | "pinned" : " 29 GB" | 20260505 03:56:28.860 | INFO | } | 20260505 03:56:28.860 | INFO | Perf report for the last 5.13934 seconds: | 20260505 03:56:28.860 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:56:28.860 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:28.860 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:28.860 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:28.860 | INFO | ANALYSIS : 57.4%, 3.0, 89.0ms, 0 | 20260505 03:56:28.860 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:28.860 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:28.860 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260505 03:56:28.860 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:28.860 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:28.860 | INFO | Uploaded 36GiB to GPU in 1.59947s (22.5074GiB/s) | 20260505 03:56:28.860 | INFO | Downloaded 34GiB to GPU in 1.51636s (22.4222GiB/s) | 20260505 03:56:33.999 | INFO | Analyzing chunk frames = [149504,150016) | 20260505 03:56:33.999 | INFO | MemInfo: { | 20260505 03:56:33.999 | INFO | "gpu" : " 17 GB", | 20260505 03:56:33.999 | INFO | "huge" : "", | 20260505 03:56:33.999 | INFO | "huge_shared" : " 27 GB", | 20260505 03:56:33.999 | INFO | "malloc" : " 14 GB", | 20260505 03:56:33.999 | INFO | "pinned" : " 29 GB" | 20260505 03:56:33.999 | INFO | } | 20260505 03:56:33.999 | INFO | Perf report for the last 5.13944 seconds: | 20260505 03:56:33.999 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:56:33.999 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:33.999 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:33.999 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:33.999 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260505 03:56:33.999 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:33.999 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:33.999 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 03:56:33.999 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:33.999 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:33.999 | INFO | Uploaded 36GiB to GPU in 1.59659s (22.548GiB/s) | 20260505 03:56:33.999 | INFO | Downloaded 34GiB to GPU in 1.51332s (22.4671GiB/s) | 20260505 03:56:39.138 | INFO | Analyzing chunk frames = [150016,150528) | 20260505 03:56:39.138 | INFO | MemInfo: { | 20260505 03:56:39.138 | INFO | "gpu" : " 17 GB", | 20260505 03:56:39.138 | INFO | "huge" : "", | 20260505 03:56:39.138 | INFO | "huge_shared" : " 27 GB", | 20260505 03:56:39.138 | INFO | "malloc" : " 14 GB", | 20260505 03:56:39.138 | INFO | "pinned" : " 29 GB" | 20260505 03:56:39.138 | INFO | } | 20260505 03:56:39.138 | INFO | Perf report for the last 5.13922 seconds: | 20260505 03:56:39.138 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:56:39.138 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:39.138 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:39.138 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:39.138 | INFO | ANALYSIS : 57.6%, 3.0, 89.9ms, 0 | 20260505 03:56:39.138 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:39.138 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:39.138 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260505 03:56:39.138 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:39.139 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:39.139 | INFO | Uploaded 36GiB to GPU in 1.59795s (22.5289GiB/s) | 20260505 03:56:39.139 | INFO | Downloaded 34GiB to GPU in 1.51469s (22.4469GiB/s) | 20260505 03:56:44.278 | INFO | Analyzing chunk frames = [150528,151040) | 20260505 03:56:44.278 | INFO | MemInfo: { | 20260505 03:56:44.278 | INFO | "gpu" : " 17 GB", | 20260505 03:56:44.278 | INFO | "huge" : "", | 20260505 03:56:44.278 | INFO | "huge_shared" : " 27 GB", | 20260505 03:56:44.278 | INFO | "malloc" : " 14 GB", | 20260505 03:56:44.278 | INFO | "pinned" : " 29 GB" | 20260505 03:56:44.278 | INFO | } | 20260505 03:56:44.278 | INFO | Perf report for the last 5.13958 seconds: | 20260505 03:56:44.278 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:56:44.278 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:44.278 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:44.278 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:44.278 | INFO | ANALYSIS : 57.9%, 3.0, 90.3ms, 0 | 20260505 03:56:44.278 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:44.278 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:44.278 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260505 03:56:44.278 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:44.278 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:44.278 | INFO | Uploaded 36GiB to GPU in 1.59791s (22.5294GiB/s) | 20260505 03:56:44.281 | INFO | Downloaded 34GiB to GPU in 1.51531s (22.4377GiB/s) | 20260505 03:56:49.417 | INFO | Analyzing chunk frames = [151040,151552) | 20260505 03:56:49.417 | INFO | MemInfo: { | 20260505 03:56:49.417 | INFO | "gpu" : " 17 GB", | 20260505 03:56:49.417 | INFO | "huge" : "", | 20260505 03:56:49.417 | INFO | "huge_shared" : " 27 GB", | 20260505 03:56:49.418 | INFO | "malloc" : " 14 GB", | 20260505 03:56:49.418 | INFO | "pinned" : " 29 GB" | 20260505 03:56:49.418 | INFO | } | 20260505 03:56:49.418 | INFO | Perf report for the last 5.13967 seconds: | 20260505 03:56:49.418 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:56:49.418 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:49.418 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:49.418 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:49.418 | INFO | ANALYSIS : 58.0%, 3.0, 90.5ms, 0 | 20260505 03:56:49.418 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:49.418 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:49.418 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 03:56:49.418 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:49.418 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:49.418 | INFO | Uploaded 36GiB to GPU in 1.5981s (22.5267GiB/s) | 20260505 03:56:49.418 | INFO | Downloaded 34GiB to GPU in 1.51602s (22.4271GiB/s) | 20260505 03:56:49.515 | INFO | Starting a round of Baz IO | 20260505 03:56:52.482 | INFO | Populating RTMetrics report | 20260505 03:56:52.483 | INFO | RTMetrics report populated | 20260505 03:56:52.483 | INFO | Opening tmp RTMetrics json file | 20260505 03:56:52.484 | INFO | Serializing RTMetrics to json | 20260505 03:56:52.485 | INFO | Serialization done, swapping tmp file now... | 20260505 03:56:52.486 | INFO | RTMetrics done | 20260505 03:56:52.552 | INFO | Finished a round of Baz IO: Wrote 1.609 GiB in 3.037 seconds | 20260505 03:56:52.552 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 58.97% / 0.40% / 40.62% | 20260505 03:56:52.552 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 13.9412% / 0.00128349% / 4.37101e-05% | 20260505 03:56:52.552 | INFO | Baz IO overall: | 20260505 03:56:52.552 | INFO | BytesWritten : 40149813915 | 20260505 03:56:52.552 | INFO | NumEvents : 17568785636 | 20260505 03:56:54.556 | INFO | Analyzing chunk frames = [151552,152064) | 20260505 03:56:54.556 | INFO | MemInfo: { | 20260505 03:56:54.556 | INFO | "gpu" : " 17 GB", | 20260505 03:56:54.556 | INFO | "huge" : "", | 20260505 03:56:54.556 | INFO | "huge_shared" : " 27 GB", | 20260505 03:56:54.556 | INFO | "malloc" : " 14 GB", | 20260505 03:56:54.556 | INFO | "pinned" : " 29 GB" | 20260505 03:56:54.556 | INFO | } | 20260505 03:56:54.556 | INFO | Perf report for the last 5.13876 seconds: | 20260505 03:56:54.556 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:56:54.556 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:54.556 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:54.556 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:54.556 | INFO | ANALYSIS : 58.6%, 3.0, 91.4ms, 0 | 20260505 03:56:54.556 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:54.556 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260505 03:56:54.557 | INFO | PRE_HQ : 58.0%, 2.1, 64.7ms, 0 | 20260505 03:56:54.557 | INFO | BAZWRITER : 58.7%, 3.1, 93.6ms, 0 | 20260505 03:56:54.557 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 03:56:54.557 | INFO | Uploaded 36GiB to GPU in 1.63876s (21.9679GiB/s) | 20260505 03:56:54.557 | INFO | Downloaded 38GiB to GPU in 1.70151s (22.3331GiB/s) | 20260505 03:56:59.696 | INFO | Analyzing chunk frames = [152064,152576) | 20260505 03:56:59.696 | INFO | MemInfo: { | 20260505 03:56:59.696 | INFO | "gpu" : " 17 GB", | 20260505 03:56:59.696 | INFO | "huge" : "", | 20260505 03:56:59.696 | INFO | "huge_shared" : " 27 GB", | 20260505 03:56:59.696 | INFO | "malloc" : " 14 GB", | 20260505 03:56:59.696 | INFO | "pinned" : " 29 GB" | 20260505 03:56:59.696 | INFO | } | 20260505 03:56:59.696 | INFO | Perf report for the last 5.13964 seconds: | 20260505 03:56:59.696 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:56:59.696 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:59.696 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:59.696 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:59.696 | INFO | ANALYSIS : 58.3%, 3.0, 91.0ms, 0 | 20260505 03:56:59.696 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:59.696 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:59.696 | INFO | PRE_HQ : 32.1%, 1.0, 16.6ms, 0 | 20260505 03:56:59.696 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:59.696 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:56:59.697 | INFO | Uploaded 36GiB to GPU in 1.5966s (22.548GiB/s) | 20260505 03:56:59.697 | INFO | Downloaded 34GiB to GPU in 1.51285s (22.4741GiB/s) | 20260505 03:57:04.835 | INFO | Analyzing chunk frames = [152576,153088) | 20260505 03:57:04.835 | INFO | MemInfo: { | 20260505 03:57:04.835 | INFO | "gpu" : " 17 GB", | 20260505 03:57:04.835 | INFO | "huge" : "", | 20260505 03:57:04.835 | INFO | "huge_shared" : " 27 GB", | 20260505 03:57:04.835 | INFO | "malloc" : " 14 GB", | 20260505 03:57:04.835 | INFO | "pinned" : " 29 GB" | 20260505 03:57:04.835 | INFO | } | 20260505 03:57:04.835 | INFO | Perf report for the last 5.13905 seconds: | 20260505 03:57:04.835 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:57:04.835 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:04.835 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:04.835 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:04.835 | INFO | ANALYSIS : 57.8%, 3.0, 89.8ms, 0 | 20260505 03:57:04.835 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:04.835 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:04.835 | INFO | PRE_HQ : 32.6%, 1.0, 16.9ms, 0 | 20260505 03:57:04.835 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:04.835 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:04.835 | INFO | Uploaded 36GiB to GPU in 1.59987s (22.5018GiB/s) | 20260505 03:57:04.835 | INFO | Downloaded 34GiB to GPU in 1.51735s (22.4075GiB/s) | 20260505 03:57:09.974 | INFO | Analyzing chunk frames = [153088,153600) | 20260505 03:57:09.974 | INFO | MemInfo: { | 20260505 03:57:09.974 | INFO | "gpu" : " 17 GB", | 20260505 03:57:09.974 | INFO | "huge" : "", | 20260505 03:57:09.974 | INFO | "huge_shared" : " 27 GB", | 20260505 03:57:09.974 | INFO | "malloc" : " 14 GB", | 20260505 03:57:09.974 | INFO | "pinned" : " 29 GB" | 20260505 03:57:09.974 | INFO | } | 20260505 03:57:09.975 | INFO | Perf report for the last 5.13941 seconds: | 20260505 03:57:09.975 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:57:09.975 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:09.975 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:09.975 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:09.975 | INFO | ANALYSIS : 57.4%, 3.0, 89.0ms, 0 | 20260505 03:57:09.975 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:09.975 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:09.975 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260505 03:57:09.975 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:09.975 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:09.975 | INFO | Uploaded 36GiB to GPU in 1.59976s (22.5033GiB/s) | 20260505 03:57:09.975 | INFO | Downloaded 34GiB to GPU in 1.5163s (22.423GiB/s) | 20260505 03:57:15.114 | INFO | Analyzing chunk frames = [153600,154112) | 20260505 03:57:15.114 | INFO | MemInfo: { | 20260505 03:57:15.114 | INFO | "gpu" : " 17 GB", | 20260505 03:57:15.114 | INFO | "huge" : "", | 20260505 03:57:15.114 | INFO | "huge_shared" : " 27 GB", | 20260505 03:57:15.114 | INFO | "malloc" : " 14 GB", | 20260505 03:57:15.114 | INFO | "pinned" : " 29 GB" | 20260505 03:57:15.114 | INFO | } | 20260505 03:57:15.114 | INFO | Perf report for the last 5.13911 seconds: | 20260505 03:57:15.114 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:57:15.114 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:15.114 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:15.114 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:15.114 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260505 03:57:15.114 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:15.114 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:15.114 | INFO | PRE_HQ : 32.6%, 1.0, 16.9ms, 0 | 20260505 03:57:15.114 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:15.114 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:15.114 | INFO | Uploaded 36GiB to GPU in 1.59749s (22.5354GiB/s) | 20260505 03:57:15.114 | INFO | Downloaded 34GiB to GPU in 1.51409s (22.4558GiB/s) | 20260505 03:57:20.253 | INFO | Analyzing chunk frames = [154112,154624) | 20260505 03:57:20.253 | INFO | MemInfo: { | 20260505 03:57:20.253 | INFO | "gpu" : " 17 GB", | 20260505 03:57:20.253 | INFO | "huge" : "", | 20260505 03:57:20.253 | INFO | "huge_shared" : " 27 GB", | 20260505 03:57:20.253 | INFO | "malloc" : " 14 GB", | 20260505 03:57:20.253 | INFO | "pinned" : " 29 GB" | 20260505 03:57:20.253 | INFO | } | 20260505 03:57:20.253 | INFO | Perf report for the last 5.13939 seconds: | 20260505 03:57:20.253 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:57:20.253 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:20.253 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:20.253 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:20.253 | INFO | ANALYSIS : 57.6%, 3.0, 89.9ms, 0 | 20260505 03:57:20.253 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:20.253 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:20.253 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260505 03:57:20.253 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:20.253 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:20.253 | INFO | Uploaded 36GiB to GPU in 1.59838s (22.5229GiB/s) | 20260505 03:57:20.254 | INFO | Downloaded 34GiB to GPU in 1.51466s (22.4473GiB/s) | 20260505 03:57:25.392 | INFO | Analyzing chunk frames = [154624,155136) | 20260505 03:57:25.392 | INFO | MemInfo: { | 20260505 03:57:25.392 | INFO | "gpu" : " 17 GB", | 20260505 03:57:25.392 | INFO | "huge" : "", | 20260505 03:57:25.392 | INFO | "huge_shared" : " 27 GB", | 20260505 03:57:25.392 | INFO | "malloc" : " 14 GB", | 20260505 03:57:25.392 | INFO | "pinned" : " 29 GB" | 20260505 03:57:25.392 | INFO | } | 20260505 03:57:25.392 | INFO | Perf report for the last 5.13924 seconds: | 20260505 03:57:25.395 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:57:25.395 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:25.395 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:25.395 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:25.395 | INFO | ANALYSIS : 57.9%, 3.0, 90.3ms, 0 | 20260505 03:57:25.395 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:25.395 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:25.395 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 03:57:25.395 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:25.395 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:25.396 | INFO | Uploaded 36GiB to GPU in 1.59851s (22.5209GiB/s) | 20260505 03:57:25.396 | INFO | Downloaded 34GiB to GPU in 1.51524s (22.4386GiB/s) | 20260505 03:57:30.532 | INFO | Analyzing chunk frames = [155136,155648) | 20260505 03:57:30.532 | INFO | MemInfo: { | 20260505 03:57:30.532 | INFO | "gpu" : " 17 GB", | 20260505 03:57:30.532 | INFO | "huge" : "", | 20260505 03:57:30.532 | INFO | "huge_shared" : " 27 GB", | 20260505 03:57:30.532 | INFO | "malloc" : " 14 GB", | 20260505 03:57:30.532 | INFO | "pinned" : " 29 GB" | 20260505 03:57:30.532 | INFO | } | 20260505 03:57:30.532 | INFO | Perf report for the last 5.14013 seconds: | 20260505 03:57:30.533 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:57:30.533 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:30.533 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:30.533 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:30.533 | INFO | ANALYSIS : 57.9%, 3.0, 90.4ms, 0 | 20260505 03:57:30.533 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:30.533 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:30.533 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 03:57:30.533 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:30.533 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:30.533 | INFO | Uploaded 36GiB to GPU in 1.59867s (22.5187GiB/s) | 20260505 03:57:30.533 | INFO | Downloaded 34GiB to GPU in 1.51631s (22.4228GiB/s) | 20260505 03:57:30.533 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 03:57:30.533 | INFO | Total mem usage: 14.9 GiB | 20260505 03:57:30.533 | INFO | All Time High Water Marks sum: 15.2 GiB (Savings: 1.52%) | 20260505 03:57:30.533 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -2.8%) | 20260505 03:57:30.533 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 03:57:30.533 | INFO | High Water Breakdown: | 20260505 03:57:30.533 | INFO | 58.1% : 8.44 GiB ( 99.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 03:57:30.533 | INFO | 41.7% : 6.06 GiB ( 91.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 03:57:30.533 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 03:57:30.533 | INFO | ------------------------------------------ | 20260505 03:57:30.533 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:57:30.533 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 03:57:30.533 | INFO | Total mem usage: 26.5 GiB | 20260505 03:57:30.533 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260505 03:57:30.533 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 03:57:30.533 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260505 03:57:30.533 | INFO | High Water Breakdown: | 20260505 03:57:30.533 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 03:57:30.533 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 03:57:30.533 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 03:57:30.533 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 03:57:30.533 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 03:57:30.533 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 03:57:30.533 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 03:57:30.533 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 03:57:30.533 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 03:57:30.533 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 03:57:30.533 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 03:57:30.533 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 03:57:30.533 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 03:57:30.533 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 03:57:30.534 | INFO | ------------------------------------------ | 20260505 03:57:30.534 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:57:30.534 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 03:57:30.534 | INFO | Total mem usage: 25.6 GiB | 20260505 03:57:30.534 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 03:57:30.534 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 03:57:30.534 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 03:57:30.534 | INFO | High Water Breakdown: | 20260505 03:57:30.534 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 03:57:30.534 | INFO | ------------------------------------------ | 20260505 03:57:30.534 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:57:30.534 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 03:57:30.534 | INFO | Total mem usage: 15.7 GiB | 20260505 03:57:30.534 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260505 03:57:30.534 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 03:57:30.534 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260505 03:57:30.534 | INFO | High Water Breakdown: | 20260505 03:57:30.534 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 03:57:30.534 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 03:57:30.534 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 03:57:30.534 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 03:57:30.534 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 03:57:30.534 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 03:57:30.534 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 03:57:30.534 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 03:57:30.534 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 03:57:30.534 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 03:57:30.534 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 03:57:30.534 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 03:57:30.534 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 03:57:30.534 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 03:57:30.534 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 03:57:30.534 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 03:57:30.534 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 03:57:30.534 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 03:57:30.534 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 03:57:30.534 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 03:57:30.534 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 03:57:30.534 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 03:57:30.534 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 03:57:30.534 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 03:57:30.534 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 03:57:30.534 | INFO | ------------------------------------------ | 20260505 03:57:30.534 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:57:30.534 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 03:57:30.534 | INFO | PerfReport: Gathered 1536 samples. | 20260505 03:57:30.534 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 03:57:30.534 | INFO | PerfReport: Full Execution -- 3.06e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260505 03:57:30.534 | INFO | PerfReport: 0.28%, AggregateStats -- 8.62e-02 / -nan, 4.14e-03 / 0.00e+00 | 20260505 03:57:30.534 | INFO | PerfReport: 10.76%, Baseline -- 3.30e+00 / -nan, 1.94e-02 / 0.00e+00 | 20260505 03:57:30.534 | INFO | PerfReport: 3.01%, Histogram -- 9.24e-01 / -nan, 5.11e-02 / 0.00e+00 | 20260505 03:57:30.534 | INFO | PerfReport: 16.02%, DME -- 3.93e+01 / -nan, 3.25e+00 / 0.00e+00 | 20260505 03:57:30.534 | INFO | PerfReport: 56.20%, FrameLabeling -- 1.72e+01 / -nan, 4.49e-02 / 0.00e+00 | 20260505 03:57:30.534 | INFO | PerfReport: 7.78%, PulseAccumulating -- 2.38e+00 / -nan, 1.62e-01 / 0.00e+00 | 20260505 03:57:30.534 | INFO | PerfReport: 4.19%, Metrics -- 1.28e+00 / -nan, 2.09e-01 / 0.00e+00 | 20260505 03:57:30.534 | INFO | PerfReport: 1.75%, PulseSerializing -- 5.37e-01 / -nan, 8.63e-02 / 0.00e+00 | 20260505 03:57:30.534 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 03:57:30.534 | INFO | PerfReport: Gathered 1536 samples. | 20260505 03:57:30.534 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 03:57:30.534 | INFO | PerfReport: Full Execution -- 9.21e+01 / -nan, 1.92e+01 / 0.00e+00 | 20260505 03:57:30.534 | INFO | PerfReport: 18.14%, Upload -- 1.67e+01 / 0.00e+00, 3.63e-01 / 0.00e+00 | 20260505 03:57:30.534 | INFO | PerfReport: 17.41%, Download -- 1.60e+01 / 0.00e+00, 6.93e-01 / 0.00e+00 | 20260505 03:57:30.534 | INFO | PerfReport: 33.29%, Compute -- 3.06e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260505 03:57:30.534 | INFO | PerfReport: 0.55%, LockUpload -- 5.05e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260505 03:57:30.534 | INFO | PerfReport: 0.00%, LockDownload -- 3.95e-04 / 0.00e+00, 9.51e-05 / 0.00e+00 | 20260505 03:57:30.534 | INFO | PerfReport: 30.59%, LockCompute -- 2.82e+01 / 0.00e+00, 1.74e+01 / 0.00e+00 | 20260505 03:57:30.534 | INFO | PerfReport: 0.02%, Misc -- 1.95e-02 / 0.00e+00, 1.37e-01 / 0.00e+00 | 20260505 03:57:30.630 | INFO | Starting a round of Baz IO | 20260505 03:57:33.596 | INFO | Populating RTMetrics report | 20260505 03:57:33.596 | INFO | RTMetrics report populated | 20260505 03:57:33.596 | INFO | Opening tmp RTMetrics json file | 20260505 03:57:33.597 | INFO | Serializing RTMetrics to json | 20260505 03:57:33.598 | INFO | Serialization done, swapping tmp file now... | 20260505 03:57:33.599 | INFO | RTMetrics done | 20260505 03:57:33.664 | INFO | Finished a round of Baz IO: Wrote 1.619 GiB in 3.034 seconds | 20260505 03:57:33.664 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 58.59% / 0.38% / 41.03% | 20260505 03:57:33.664 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 13.815% / 0.00114441% / 3.97364e-05% | 20260505 03:57:33.664 | INFO | Baz IO overall: | 20260505 03:57:33.664 | INFO | BytesWritten : 41888591090 | 20260505 03:57:33.664 | INFO | NumEvents : 18388437931 | 20260505 03:57:35.671 | INFO | Analyzing chunk frames = [155648,156160) | 20260505 03:57:35.671 | INFO | MemInfo: { | 20260505 03:57:35.671 | INFO | "gpu" : " 17 GB", | 20260505 03:57:35.671 | INFO | "huge" : "", | 20260505 03:57:35.671 | INFO | "huge_shared" : " 27 GB", | 20260505 03:57:35.671 | INFO | "malloc" : " 14 GB", | 20260505 03:57:35.671 | INFO | "pinned" : " 29 GB" | 20260505 03:57:35.671 | INFO | } | 20260505 03:57:35.671 | INFO | Perf report for the last 5.13891 seconds: | 20260505 03:57:35.671 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:57:35.671 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:35.671 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:35.671 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:35.671 | INFO | ANALYSIS : 58.6%, 3.0, 91.4ms, 0 | 20260505 03:57:35.671 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:35.671 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260505 03:57:35.671 | INFO | PRE_HQ : 58.0%, 2.1, 64.7ms, 0 | 20260505 03:57:35.671 | INFO | BAZWRITER : 58.6%, 3.1, 94.8ms, 0 | 20260505 03:57:35.672 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 03:57:35.672 | INFO | Uploaded 36GiB to GPU in 1.6372s (21.9887GiB/s) | 20260505 03:57:35.672 | INFO | Downloaded 38GiB to GPU in 1.69875s (22.3694GiB/s) | 20260505 03:57:40.810 | INFO | Analyzing chunk frames = [156160,156672) | 20260505 03:57:40.810 | INFO | MemInfo: { | 20260505 03:57:40.810 | INFO | "gpu" : " 17 GB", | 20260505 03:57:40.810 | INFO | "huge" : "", | 20260505 03:57:40.810 | INFO | "huge_shared" : " 27 GB", | 20260505 03:57:40.810 | INFO | "malloc" : " 14 GB", | 20260505 03:57:40.810 | INFO | "pinned" : " 29 GB" | 20260505 03:57:40.810 | INFO | } | 20260505 03:57:40.811 | INFO | Perf report for the last 5.13915 seconds: | 20260505 03:57:40.811 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:57:40.811 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:40.811 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:40.811 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:40.811 | INFO | ANALYSIS : 58.3%, 3.0, 91.0ms, 0 | 20260505 03:57:40.811 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:40.811 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:40.811 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 03:57:40.811 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:40.811 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:40.811 | INFO | Uploaded 36GiB to GPU in 1.59645s (22.55GiB/s) | 20260505 03:57:40.811 | INFO | Downloaded 34GiB to GPU in 1.51318s (22.4692GiB/s) | 20260505 03:57:45.950 | INFO | Analyzing chunk frames = [156672,157184) | 20260505 03:57:45.950 | INFO | MemInfo: { | 20260505 03:57:45.950 | INFO | "gpu" : " 17 GB", | 20260505 03:57:45.950 | INFO | "huge" : "", | 20260505 03:57:45.950 | INFO | "huge_shared" : " 27 GB", | 20260505 03:57:45.950 | INFO | "malloc" : " 14 GB", | 20260505 03:57:45.950 | INFO | "pinned" : " 29 GB" | 20260505 03:57:45.950 | INFO | } | 20260505 03:57:45.950 | INFO | Perf report for the last 5.13914 seconds: | 20260505 03:57:45.950 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:57:45.950 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:45.950 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:45.950 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:45.950 | INFO | ANALYSIS : 57.7%, 3.0, 89.7ms, 0 | 20260505 03:57:45.950 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:45.950 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:45.950 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260505 03:57:45.950 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:45.950 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:45.950 | INFO | Uploaded 36GiB to GPU in 1.59971s (22.504GiB/s) | 20260505 03:57:45.950 | INFO | Downloaded 34GiB to GPU in 1.51672s (22.4168GiB/s) | 20260505 03:57:51.089 | INFO | Analyzing chunk frames = [157184,157696) | 20260505 03:57:51.089 | INFO | MemInfo: { | 20260505 03:57:51.089 | INFO | "gpu" : " 17 GB", | 20260505 03:57:51.089 | INFO | "huge" : "", | 20260505 03:57:51.089 | INFO | "huge_shared" : " 27 GB", | 20260505 03:57:51.089 | INFO | "malloc" : " 14 GB", | 20260505 03:57:51.089 | INFO | "pinned" : " 29 GB" | 20260505 03:57:51.089 | INFO | } | 20260505 03:57:51.089 | INFO | Perf report for the last 5.13942 seconds: | 20260505 03:57:51.089 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:57:51.089 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:51.089 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:51.089 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:51.089 | INFO | ANALYSIS : 57.4%, 3.0, 89.1ms, 0 | 20260505 03:57:51.089 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:51.089 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:51.089 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260505 03:57:51.089 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:51.089 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:51.089 | INFO | Uploaded 36GiB to GPU in 1.59884s (22.5164GiB/s) | 20260505 03:57:51.090 | INFO | Downloaded 34GiB to GPU in 1.51524s (22.4387GiB/s) | 20260505 03:57:56.228 | INFO | Analyzing chunk frames = [157696,158208) | 20260505 03:57:56.228 | INFO | MemInfo: { | 20260505 03:57:56.228 | INFO | "gpu" : " 17 GB", | 20260505 03:57:56.228 | INFO | "huge" : "", | 20260505 03:57:56.228 | INFO | "huge_shared" : " 27 GB", | 20260505 03:57:56.228 | INFO | "malloc" : " 14 GB", | 20260505 03:57:56.228 | INFO | "pinned" : " 29 GB" | 20260505 03:57:56.228 | INFO | } | 20260505 03:57:56.228 | INFO | Perf report for the last 5.1393 seconds: | 20260505 03:57:56.229 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:57:56.229 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:56.229 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:56.229 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:56.229 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 03:57:56.229 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:56.229 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:56.229 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 03:57:56.229 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:56.229 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:57:56.229 | INFO | Uploaded 36GiB to GPU in 1.59776s (22.5315GiB/s) | 20260505 03:57:56.229 | INFO | Downloaded 34GiB to GPU in 1.51441s (22.4511GiB/s) | 20260505 03:58:01.368 | INFO | Analyzing chunk frames = [158208,158720) | 20260505 03:58:01.368 | INFO | MemInfo: { | 20260505 03:58:01.368 | INFO | "gpu" : " 17 GB", | 20260505 03:58:01.368 | INFO | "huge" : "", | 20260505 03:58:01.368 | INFO | "huge_shared" : " 27 GB", | 20260505 03:58:01.368 | INFO | "malloc" : " 14 GB", | 20260505 03:58:01.368 | INFO | "pinned" : " 29 GB" | 20260505 03:58:01.368 | INFO | } | 20260505 03:58:01.368 | INFO | Perf report for the last 5.13936 seconds: | 20260505 03:58:01.368 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:58:01.368 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:01.368 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:01.368 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:01.368 | INFO | ANALYSIS : 57.7%, 3.0, 90.0ms, 0 | 20260505 03:58:01.368 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:01.368 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:01.368 | INFO | PRE_HQ : 32.7%, 1.0, 17.0ms, 0 | 20260505 03:58:01.368 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:01.368 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:01.368 | INFO | Uploaded 36GiB to GPU in 1.59939s (22.5086GiB/s) | 20260505 03:58:01.369 | INFO | Downloaded 34GiB to GPU in 1.51554s (22.4343GiB/s) | 20260505 03:58:06.508 | INFO | Analyzing chunk frames = [158720,159232) | 20260505 03:58:06.508 | INFO | MemInfo: { | 20260505 03:58:06.508 | INFO | "gpu" : " 17 GB", | 20260505 03:58:06.508 | INFO | "huge" : "", | 20260505 03:58:06.508 | INFO | "huge_shared" : " 27 GB", | 20260505 03:58:06.508 | INFO | "malloc" : " 14 GB", | 20260505 03:58:06.508 | INFO | "pinned" : " 29 GB" | 20260505 03:58:06.508 | INFO | } | 20260505 03:58:06.508 | INFO | Perf report for the last 5.14009 seconds: | 20260505 03:58:06.508 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:58:06.508 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:06.508 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:06.508 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:06.508 | INFO | ANALYSIS : 57.8%, 3.0, 90.3ms, 0 | 20260505 03:58:06.508 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:06.508 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:06.508 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 03:58:06.508 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:06.508 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:06.508 | INFO | Uploaded 36GiB to GPU in 1.59667s (22.5469GiB/s) | 20260505 03:58:06.508 | INFO | Downloaded 34GiB to GPU in 1.51586s (22.4295GiB/s) | 20260505 03:58:11.646 | INFO | Analyzing chunk frames = [159232,159744) | 20260505 03:58:11.646 | INFO | MemInfo: { | 20260505 03:58:11.646 | INFO | "gpu" : " 17 GB", | 20260505 03:58:11.646 | INFO | "huge" : "", | 20260505 03:58:11.646 | INFO | "huge_shared" : " 27 GB", | 20260505 03:58:11.646 | INFO | "malloc" : " 14 GB", | 20260505 03:58:11.646 | INFO | "pinned" : " 29 GB" | 20260505 03:58:11.647 | INFO | } | 20260505 03:58:11.647 | INFO | Perf report for the last 5.13878 seconds: | 20260505 03:58:11.647 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:58:11.647 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:11.647 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:11.647 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:11.647 | INFO | ANALYSIS : 57.9%, 3.0, 90.4ms, 0 | 20260505 03:58:11.647 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:11.647 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:11.647 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 03:58:11.647 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:11.647 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:11.647 | INFO | Uploaded 36GiB to GPU in 1.59735s (22.5374GiB/s) | 20260505 03:58:11.647 | INFO | Downloaded 34GiB to GPU in 1.51581s (22.4303GiB/s) | 20260505 03:58:11.744 | INFO | Starting a round of Baz IO | 20260505 03:58:14.709 | INFO | Populating RTMetrics report | 20260505 03:58:14.710 | INFO | RTMetrics report populated | 20260505 03:58:14.710 | INFO | Opening tmp RTMetrics json file | 20260505 03:58:14.711 | INFO | Serializing RTMetrics to json | 20260505 03:58:14.712 | INFO | Serialization done, swapping tmp file now... | 20260505 03:58:14.713 | INFO | RTMetrics done | 20260505 03:58:14.785 | INFO | Finished a round of Baz IO: Wrote 1.632 GiB in 3.041 seconds | 20260505 03:58:14.785 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 58.22% / 0.37% / 41.41% | 20260505 03:58:14.785 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 13.7377% / 0.00114441% / 2.78155e-05% | 20260505 03:58:14.785 | INFO | Baz IO overall: | 20260505 03:58:14.785 | INFO | BytesWritten : 43640547739 | 20260505 03:58:14.785 | INFO | NumEvents : 19214293560 | 20260505 03:58:16.786 | INFO | Analyzing chunk frames = [159744,160256) | 20260505 03:58:16.786 | INFO | MemInfo: { | 20260505 03:58:16.786 | INFO | "gpu" : " 17 GB", | 20260505 03:58:16.786 | INFO | "huge" : "", | 20260505 03:58:16.786 | INFO | "huge_shared" : " 27 GB", | 20260505 03:58:16.786 | INFO | "malloc" : " 14 GB", | 20260505 03:58:16.786 | INFO | "pinned" : " 29 GB" | 20260505 03:58:16.786 | INFO | } | 20260505 03:58:16.786 | INFO | Perf report for the last 5.13938 seconds: | 20260505 03:58:16.786 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:58:16.786 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:16.786 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:16.786 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:16.786 | INFO | ANALYSIS : 58.6%, 3.0, 91.3ms, 0 | 20260505 03:58:16.786 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:16.786 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260505 03:58:16.786 | INFO | PRE_HQ : 58.1%, 2.2, 65.2ms, 0 | 20260505 03:58:16.786 | INFO | BAZWRITER : 58.7%, 3.2, 96.5ms, 0 | 20260505 03:58:16.786 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 03:58:16.787 | INFO | Uploaded 36GiB to GPU in 1.63481s (22.0209GiB/s) | 20260505 03:58:16.787 | INFO | Downloaded 38GiB to GPU in 1.69679s (22.3952GiB/s) | 20260505 03:58:21.925 | INFO | Analyzing chunk frames = [160256,160768) | 20260505 03:58:21.925 | INFO | MemInfo: { | 20260505 03:58:21.925 | INFO | "gpu" : " 17 GB", | 20260505 03:58:21.925 | INFO | "huge" : "", | 20260505 03:58:21.925 | INFO | "huge_shared" : " 27 GB", | 20260505 03:58:21.925 | INFO | "malloc" : " 14 GB", | 20260505 03:58:21.925 | INFO | "pinned" : " 29 GB" | 20260505 03:58:21.925 | INFO | } | 20260505 03:58:21.925 | INFO | Perf report for the last 5.13937 seconds: | 20260505 03:58:21.925 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:58:21.925 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:21.925 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:21.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:21.926 | INFO | ANALYSIS : 58.4%, 3.0, 91.1ms, 0 | 20260505 03:58:21.926 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:21.926 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:21.926 | INFO | PRE_HQ : 32.2%, 1.0, 16.8ms, 0 | 20260505 03:58:21.926 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:21.926 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:21.926 | INFO | Uploaded 36GiB to GPU in 1.59562s (22.5618GiB/s) | 20260505 03:58:21.926 | INFO | Downloaded 34GiB to GPU in 1.51199s (22.487GiB/s) | 20260505 03:58:27.064 | INFO | Analyzing chunk frames = [160768,161280) | 20260505 03:58:27.064 | INFO | MemInfo: { | 20260505 03:58:27.064 | INFO | "gpu" : " 17 GB", | 20260505 03:58:27.064 | INFO | "huge" : "", | 20260505 03:58:27.064 | INFO | "huge_shared" : " 27 GB", | 20260505 03:58:27.065 | INFO | "malloc" : " 14 GB", | 20260505 03:58:27.065 | INFO | "pinned" : " 29 GB" | 20260505 03:58:27.065 | INFO | } | 20260505 03:58:27.065 | INFO | Perf report for the last 5.13932 seconds: | 20260505 03:58:27.065 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:58:27.065 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:27.065 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:27.065 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:27.065 | INFO | ANALYSIS : 57.7%, 3.0, 89.6ms, 0 | 20260505 03:58:27.065 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:27.065 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:27.065 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260505 03:58:27.065 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:27.065 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:27.065 | INFO | Uploaded 36GiB to GPU in 1.60051s (22.4929GiB/s) | 20260505 03:58:27.065 | INFO | Downloaded 34GiB to GPU in 1.51762s (22.4035GiB/s) | 20260505 03:58:32.204 | INFO | Analyzing chunk frames = [161280,161792) | 20260505 03:58:32.204 | INFO | MemInfo: { | 20260505 03:58:32.204 | INFO | "gpu" : " 17 GB", | 20260505 03:58:32.204 | INFO | "huge" : "", | 20260505 03:58:32.204 | INFO | "huge_shared" : " 27 GB", | 20260505 03:58:32.207 | INFO | "malloc" : " 14 GB", | 20260505 03:58:32.207 | INFO | "pinned" : " 29 GB" | 20260505 03:58:32.207 | INFO | } | 20260505 03:58:32.207 | INFO | Perf report for the last 5.1422 seconds: | 20260505 03:58:32.207 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:58:32.207 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:32.207 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:32.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:32.207 | INFO | ANALYSIS : 57.4%, 3.0, 89.0ms, 0 | 20260505 03:58:32.207 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:32.207 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:32.207 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260505 03:58:32.207 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:32.207 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:32.207 | INFO | Uploaded 36GiB to GPU in 1.5995s (22.507GiB/s) | 20260505 03:58:32.207 | INFO | Downloaded 34GiB to GPU in 1.51733s (22.4077GiB/s) | 20260505 03:58:37.343 | INFO | Analyzing chunk frames = [161792,162304) | 20260505 03:58:37.343 | INFO | MemInfo: { | 20260505 03:58:37.343 | INFO | "gpu" : " 17 GB", | 20260505 03:58:37.343 | INFO | "huge" : "", | 20260505 03:58:37.343 | INFO | "huge_shared" : " 27 GB", | 20260505 03:58:37.343 | INFO | "malloc" : " 14 GB", | 20260505 03:58:37.343 | INFO | "pinned" : " 29 GB" | 20260505 03:58:37.343 | INFO | } | 20260505 03:58:37.343 | INFO | Perf report for the last 5.1362 seconds: | 20260505 03:58:37.343 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:58:37.343 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:37.343 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:37.343 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:37.343 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 03:58:37.343 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:37.343 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:37.343 | INFO | PRE_HQ : 32.3%, 1.0, 16.7ms, 0 | 20260505 03:58:37.343 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:37.343 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:37.344 | INFO | Uploaded 36GiB to GPU in 1.59765s (22.5331GiB/s) | 20260505 03:58:37.347 | INFO | Downloaded 34GiB to GPU in 1.51428s (22.4529GiB/s) | 20260505 03:58:42.482 | INFO | Analyzing chunk frames = [162304,162816) | 20260505 03:58:42.482 | INFO | MemInfo: { | 20260505 03:58:42.482 | INFO | "gpu" : " 17 GB", | 20260505 03:58:42.483 | INFO | "huge" : "", | 20260505 03:58:42.483 | INFO | "huge_shared" : " 27 GB", | 20260505 03:58:42.483 | INFO | "malloc" : " 14 GB", | 20260505 03:58:42.483 | INFO | "pinned" : " 29 GB" | 20260505 03:58:42.483 | INFO | } | 20260505 03:58:42.483 | INFO | Perf report for the last 5.13951 seconds: | 20260505 03:58:42.483 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:58:42.483 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:42.483 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:42.483 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:42.483 | INFO | ANALYSIS : 57.7%, 3.0, 90.0ms, 0 | 20260505 03:58:42.483 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:42.483 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:42.483 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 03:58:42.483 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:42.483 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:42.483 | INFO | Uploaded 36GiB to GPU in 1.59756s (22.5344GiB/s) | 20260505 03:58:42.483 | INFO | Downloaded 34GiB to GPU in 1.51441s (22.451GiB/s) | 20260505 03:58:47.622 | INFO | Analyzing chunk frames = [162816,163328) | 20260505 03:58:47.622 | INFO | MemInfo: { | 20260505 03:58:47.622 | INFO | "gpu" : " 17 GB", | 20260505 03:58:47.622 | INFO | "huge" : "", | 20260505 03:58:47.622 | INFO | "huge_shared" : " 27 GB", | 20260505 03:58:47.622 | INFO | "malloc" : " 14 GB", | 20260505 03:58:47.622 | INFO | "pinned" : " 29 GB" | 20260505 03:58:47.622 | INFO | } | 20260505 03:58:47.622 | INFO | Perf report for the last 5.13913 seconds: | 20260505 03:58:47.622 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:58:47.622 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:47.622 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:47.622 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:47.622 | INFO | ANALYSIS : 57.8%, 3.0, 90.2ms, 0 | 20260505 03:58:47.622 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:47.622 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:47.622 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 03:58:47.622 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:47.622 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:47.622 | INFO | Uploaded 36GiB to GPU in 1.59801s (22.5281GiB/s) | 20260505 03:58:47.622 | INFO | Downloaded 34GiB to GPU in 1.51577s (22.4309GiB/s) | 20260505 03:58:52.761 | INFO | Analyzing chunk frames = [163328,163840) | 20260505 03:58:52.761 | INFO | MemInfo: { | 20260505 03:58:52.761 | INFO | "gpu" : " 17 GB", | 20260505 03:58:52.761 | INFO | "huge" : "", | 20260505 03:58:52.761 | INFO | "huge_shared" : " 27 GB", | 20260505 03:58:52.761 | INFO | "malloc" : " 14 GB", | 20260505 03:58:52.761 | INFO | "pinned" : " 29 GB" | 20260505 03:58:52.761 | INFO | } | 20260505 03:58:52.761 | INFO | Perf report for the last 5.13931 seconds: | 20260505 03:58:52.761 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:58:52.761 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:52.761 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:52.761 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:52.761 | INFO | ANALYSIS : 58.0%, 3.0, 90.5ms, 0 | 20260505 03:58:52.761 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:52.761 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:52.761 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 03:58:52.761 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:52.761 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:52.762 | INFO | Uploaded 36GiB to GPU in 1.60328s (22.454GiB/s) | 20260505 03:58:52.762 | INFO | Downloaded 34GiB to GPU in 1.51979s (22.3715GiB/s) | 20260505 03:58:52.762 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 03:58:52.762 | INFO | Total mem usage: 14.9 GiB | 20260505 03:58:52.762 | INFO | All Time High Water Marks sum: 15.2 GiB (Savings: 1.52%) | 20260505 03:58:52.762 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -2.77%) | 20260505 03:58:52.762 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 03:58:52.762 | INFO | High Water Breakdown: | 20260505 03:58:52.762 | INFO | 58.2% : 8.45 GiB ( 99.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 03:58:52.762 | INFO | 41.7% : 6.05 GiB ( 91% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 03:58:52.762 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 03:58:52.762 | INFO | ------------------------------------------ | 20260505 03:58:52.762 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:58:52.762 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 03:58:52.762 | INFO | Total mem usage: 26.5 GiB | 20260505 03:58:52.762 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260505 03:58:52.762 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 03:58:52.762 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260505 03:58:52.762 | INFO | High Water Breakdown: | 20260505 03:58:52.762 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 03:58:52.762 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 03:58:52.762 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 03:58:52.762 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 03:58:52.762 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 03:58:52.762 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 03:58:52.762 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 03:58:52.762 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 03:58:52.762 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 03:58:52.762 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 03:58:52.762 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 03:58:52.762 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 03:58:52.762 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 03:58:52.762 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 03:58:52.762 | INFO | ------------------------------------------ | 20260505 03:58:52.762 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:58:52.762 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 03:58:52.762 | INFO | Total mem usage: 25.6 GiB | 20260505 03:58:52.762 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 03:58:52.762 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 03:58:52.762 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 03:58:52.762 | INFO | High Water Breakdown: | 20260505 03:58:52.762 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 03:58:52.762 | INFO | ------------------------------------------ | 20260505 03:58:52.762 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:58:52.762 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 03:58:52.762 | INFO | Total mem usage: 15.7 GiB | 20260505 03:58:52.762 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260505 03:58:52.762 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 03:58:52.762 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260505 03:58:52.762 | INFO | High Water Breakdown: | 20260505 03:58:52.762 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 03:58:52.762 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 03:58:52.762 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 03:58:52.762 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 03:58:52.762 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 03:58:52.762 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 03:58:52.762 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 03:58:52.762 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 03:58:52.762 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 03:58:52.762 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 03:58:52.762 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 03:58:52.762 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 03:58:52.762 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 03:58:52.762 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 03:58:52.763 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 03:58:52.763 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 03:58:52.763 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 03:58:52.763 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 03:58:52.763 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 03:58:52.763 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 03:58:52.763 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 03:58:52.763 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 03:58:52.763 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 03:58:52.763 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 03:58:52.763 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 03:58:52.763 | INFO | ------------------------------------------ | 20260505 03:58:52.763 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 03:58:52.763 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 03:58:52.763 | INFO | PerfReport: Gathered 1536 samples. | 20260505 03:58:52.763 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 03:58:52.763 | INFO | PerfReport: Full Execution -- 3.06e+01 / 7.08e+01, 1.30e+01 / 0.00e+00 | 20260505 03:58:52.763 | INFO | PerfReport: 0.28%, AggregateStats -- 8.63e-02 / 8.51e-02, 6.05e-03 / 0.00e+00 | 20260505 03:58:52.763 | INFO | PerfReport: 10.76%, Baseline -- 3.30e+00 / 3.31e+00, 1.74e-02 / 0.00e+00 | 20260505 03:58:52.763 | INFO | PerfReport: 3.01%, Histogram -- 9.23e-01 / 1.06e+00, 5.28e-02 / 0.00e+00 | 20260505 03:58:52.763 | INFO | PerfReport: 16.02%, DME -- 3.93e+01 / 4.49e+01, 3.31e+00 / 0.00e+00 | 20260505 03:58:52.763 | INFO | PerfReport: 56.21%, FrameLabeling -- 1.72e+01 / 1.72e+01, 4.52e-02 / 0.00e+00 | 20260505 03:58:52.763 | INFO | PerfReport: 7.78%, PulseAccumulating -- 2.38e+00 / 2.44e+00, 1.61e-01 / 0.00e+00 | 20260505 03:58:52.763 | INFO | PerfReport: 4.18%, Metrics -- 1.28e+00 / 1.26e+00, 2.08e-01 / 0.00e+00 | 20260505 03:58:52.763 | INFO | PerfReport: 1.75%, PulseSerializing -- 5.35e-01 / 5.76e-01, 8.57e-02 / 0.00e+00 | 20260505 03:58:52.763 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 03:58:52.763 | INFO | PerfReport: Gathered 1536 samples. | 20260505 03:58:52.763 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 03:58:52.763 | INFO | PerfReport: Full Execution -- 9.20e+01 / -nan, 1.92e+01 / 0.00e+00 | 20260505 03:58:52.763 | INFO | PerfReport: 18.15%, Upload -- 1.67e+01 / 0.00e+00, 3.84e-01 / 0.00e+00 | 20260505 03:58:52.763 | INFO | PerfReport: 17.42%, Download -- 1.60e+01 / 0.00e+00, 6.96e-01 / 0.00e+00 | 20260505 03:58:52.763 | INFO | PerfReport: 33.29%, Compute -- 3.06e+01 / 0.00e+00, 1.30e+01 / 0.00e+00 | 20260505 03:58:52.763 | INFO | PerfReport: 0.55%, LockUpload -- 5.05e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260505 03:58:52.763 | INFO | PerfReport: 0.00%, LockDownload -- 3.99e-04 / 0.00e+00, 1.40e-04 / 0.00e+00 | 20260505 03:58:52.763 | INFO | PerfReport: 30.58%, LockCompute -- 2.81e+01 / 0.00e+00, 1.73e+01 / 0.00e+00 | 20260505 03:58:52.763 | INFO | PerfReport: 0.02%, Misc -- 1.87e-02 / 0.00e+00, 1.28e-01 / 0.00e+00 | 20260505 03:58:52.858 | INFO | Starting a round of Baz IO | 20260505 03:58:55.823 | INFO | Populating RTMetrics report | 20260505 03:58:55.824 | INFO | RTMetrics report populated | 20260505 03:58:55.824 | INFO | Opening tmp RTMetrics json file | 20260505 03:58:55.825 | INFO | Serializing RTMetrics to json | 20260505 03:58:55.827 | INFO | Serialization done, swapping tmp file now... | 20260505 03:58:55.827 | INFO | RTMetrics done | 20260505 03:58:55.893 | INFO | Finished a round of Baz IO: Wrote 1.643 GiB in 3.034 seconds | 20260505 03:58:55.893 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 57.86% / 0.36% / 41.78% | 20260505 03:58:55.893 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 13.7061% / 0.0011762% / 3.57628e-05% | 20260505 03:58:55.893 | INFO | Baz IO overall: | 20260505 03:58:55.893 | INFO | BytesWritten : 45404542146 | 20260505 03:58:55.893 | INFO | NumEvents : 20045985521 | 20260505 03:58:57.901 | INFO | Analyzing chunk frames = [163840,164352) | 20260505 03:58:57.901 | INFO | MemInfo: { | 20260505 03:58:57.901 | INFO | "gpu" : " 17 GB", | 20260505 03:58:57.901 | INFO | "huge" : "", | 20260505 03:58:57.901 | INFO | "huge_shared" : " 27 GB", | 20260505 03:58:57.901 | INFO | "malloc" : " 14 GB", | 20260505 03:58:57.901 | INFO | "pinned" : " 29 GB" | 20260505 03:58:57.901 | INFO | } | 20260505 03:58:57.901 | INFO | Perf report for the last 5.13945 seconds: | 20260505 03:58:57.901 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:58:57.901 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:57.901 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:57.901 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:57.901 | INFO | ANALYSIS : 58.5%, 3.0, 91.3ms, 0 | 20260505 03:58:57.901 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:58:57.901 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260505 03:58:57.901 | INFO | PRE_HQ : 58.0%, 2.1, 64.7ms, 0 | 20260505 03:58:57.901 | INFO | BAZWRITER : 58.6%, 3.2, 96.3ms, 0 | 20260505 03:58:57.901 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 03:58:57.901 | INFO | Uploaded 36GiB to GPU in 1.63779s (21.9809GiB/s) | 20260505 03:58:57.901 | INFO | Downloaded 38GiB to GPU in 1.69922s (22.3631GiB/s) | 20260505 03:59:03.040 | INFO | Analyzing chunk frames = [164352,164864) | 20260505 03:59:03.040 | INFO | MemInfo: { | 20260505 03:59:03.040 | INFO | "gpu" : " 17 GB", | 20260505 03:59:03.040 | INFO | "huge" : "", | 20260505 03:59:03.040 | INFO | "huge_shared" : " 27 GB", | 20260505 03:59:03.040 | INFO | "malloc" : " 14 GB", | 20260505 03:59:03.040 | INFO | "pinned" : " 29 GB" | 20260505 03:59:03.040 | INFO | } | 20260505 03:59:03.040 | INFO | Perf report for the last 5.13956 seconds: | 20260505 03:59:03.040 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:59:03.040 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:03.040 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:03.040 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:03.040 | INFO | ANALYSIS : 58.4%, 3.0, 91.1ms, 0 | 20260505 03:59:03.040 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:03.040 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:03.040 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260505 03:59:03.040 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:03.040 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:03.041 | INFO | Uploaded 36GiB to GPU in 1.59679s (22.5452GiB/s) | 20260505 03:59:03.041 | INFO | Downloaded 34GiB to GPU in 1.51328s (22.4677GiB/s) | 20260505 03:59:08.179 | INFO | Analyzing chunk frames = [164864,165376) | 20260505 03:59:08.179 | INFO | MemInfo: { | 20260505 03:59:08.179 | INFO | "gpu" : " 17 GB", | 20260505 03:59:08.179 | INFO | "huge" : "", | 20260505 03:59:08.179 | INFO | "huge_shared" : " 27 GB", | 20260505 03:59:08.179 | INFO | "malloc" : " 14 GB", | 20260505 03:59:08.179 | INFO | "pinned" : " 29 GB" | 20260505 03:59:08.179 | INFO | } | 20260505 03:59:08.179 | INFO | Perf report for the last 5.13905 seconds: | 20260505 03:59:08.183 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:59:08.183 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:08.183 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:08.183 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:08.183 | INFO | ANALYSIS : 57.7%, 3.0, 89.6ms, 0 | 20260505 03:59:08.183 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:08.183 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:08.183 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260505 03:59:08.183 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:08.183 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:08.183 | INFO | Uploaded 36GiB to GPU in 1.60044s (22.4939GiB/s) | 20260505 03:59:08.183 | INFO | Downloaded 34GiB to GPU in 1.51727s (22.4087GiB/s) | 20260505 03:59:13.319 | INFO | Analyzing chunk frames = [165376,165888) | 20260505 03:59:13.319 | INFO | MemInfo: { | 20260505 03:59:13.319 | INFO | "gpu" : " 17 GB", | 20260505 03:59:13.319 | INFO | "huge" : "", | 20260505 03:59:13.319 | INFO | "huge_shared" : " 27 GB", | 20260505 03:59:13.319 | INFO | "malloc" : " 14 GB", | 20260505 03:59:13.319 | INFO | "pinned" : " 29 GB" | 20260505 03:59:13.319 | INFO | } | 20260505 03:59:13.319 | INFO | Perf report for the last 5.13932 seconds: | 20260505 03:59:13.319 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:59:13.319 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:13.319 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:13.319 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:13.319 | INFO | ANALYSIS : 57.4%, 3.0, 89.0ms, 0 | 20260505 03:59:13.319 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:13.319 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:13.319 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260505 03:59:13.319 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:13.319 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:13.319 | INFO | Uploaded 36GiB to GPU in 1.59962s (22.5053GiB/s) | 20260505 03:59:13.319 | INFO | Downloaded 34GiB to GPU in 1.51627s (22.4235GiB/s) | 20260505 03:59:18.458 | INFO | Analyzing chunk frames = [165888,166400) | 20260505 03:59:18.458 | INFO | MemInfo: { | 20260505 03:59:18.458 | INFO | "gpu" : " 17 GB", | 20260505 03:59:18.458 | INFO | "huge" : "", | 20260505 03:59:18.458 | INFO | "huge_shared" : " 27 GB", | 20260505 03:59:18.458 | INFO | "malloc" : " 14 GB", | 20260505 03:59:18.458 | INFO | "pinned" : " 29 GB" | 20260505 03:59:18.458 | INFO | } | 20260505 03:59:18.458 | INFO | Perf report for the last 5.13937 seconds: | 20260505 03:59:18.461 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:59:18.461 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:18.461 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:18.461 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:18.461 | INFO | ANALYSIS : 57.3%, 3.0, 89.3ms, 0 | 20260505 03:59:18.461 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:18.461 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:18.461 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260505 03:59:18.461 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:18.461 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:18.462 | INFO | Uploaded 36GiB to GPU in 1.59863s (22.5193GiB/s) | 20260505 03:59:18.462 | INFO | Downloaded 34GiB to GPU in 1.51432s (22.4524GiB/s) | 20260505 03:59:23.599 | INFO | Analyzing chunk frames = [166400,166912) | 20260505 03:59:23.599 | INFO | MemInfo: { | 20260505 03:59:23.599 | INFO | "gpu" : " 17 GB", | 20260505 03:59:23.599 | INFO | "huge" : "", | 20260505 03:59:23.599 | INFO | "huge_shared" : " 27 GB", | 20260505 03:59:23.599 | INFO | "malloc" : " 14 GB", | 20260505 03:59:23.599 | INFO | "pinned" : " 29 GB" | 20260505 03:59:23.599 | INFO | } | 20260505 03:59:23.599 | INFO | Perf report for the last 5.14101 seconds: | 20260505 03:59:23.599 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:59:23.599 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:23.599 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:23.599 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:23.599 | INFO | ANALYSIS : 57.6%, 3.0, 89.9ms, 0 | 20260505 03:59:23.599 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:23.599 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:23.599 | INFO | PRE_HQ : 32.7%, 1.0, 17.0ms, 0 | 20260505 03:59:23.599 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:23.599 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:23.600 | INFO | Uploaded 36GiB to GPU in 1.59843s (22.522GiB/s) | 20260505 03:59:23.600 | INFO | Downloaded 34GiB to GPU in 1.51477s (22.4456GiB/s) | 20260505 03:59:28.737 | INFO | Analyzing chunk frames = [166912,167424) | 20260505 03:59:28.737 | INFO | MemInfo: { | 20260505 03:59:28.737 | INFO | "gpu" : " 17 GB", | 20260505 03:59:28.737 | INFO | "huge" : "", | 20260505 03:59:28.737 | INFO | "huge_shared" : " 27 GB", | 20260505 03:59:28.737 | INFO | "malloc" : " 14 GB", | 20260505 03:59:28.737 | INFO | "pinned" : " 29 GB" | 20260505 03:59:28.737 | INFO | } | 20260505 03:59:28.737 | INFO | Perf report for the last 5.13785 seconds: | 20260505 03:59:28.737 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:59:28.737 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:28.737 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:28.737 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:28.737 | INFO | ANALYSIS : 57.9%, 3.0, 90.2ms, 0 | 20260505 03:59:28.737 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:28.737 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:28.737 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260505 03:59:28.737 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:28.737 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:28.737 | INFO | Uploaded 36GiB to GPU in 1.59848s (22.5215GiB/s) | 20260505 03:59:28.737 | INFO | Downloaded 34GiB to GPU in 1.51517s (22.4397GiB/s) | 20260505 03:59:33.876 | INFO | Analyzing chunk frames = [167424,167936) | 20260505 03:59:33.876 | INFO | MemInfo: { | 20260505 03:59:33.876 | INFO | "gpu" : " 17 GB", | 20260505 03:59:33.876 | INFO | "huge" : "", | 20260505 03:59:33.876 | INFO | "huge_shared" : " 27 GB", | 20260505 03:59:33.876 | INFO | "malloc" : " 14 GB", | 20260505 03:59:33.876 | INFO | "pinned" : " 29 GB" | 20260505 03:59:33.876 | INFO | } | 20260505 03:59:33.876 | INFO | Perf report for the last 5.13937 seconds: | 20260505 03:59:33.876 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:59:33.876 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:33.876 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:33.876 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:33.876 | INFO | ANALYSIS : 57.9%, 3.0, 90.3ms, 0 | 20260505 03:59:33.876 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:33.876 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:33.876 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 03:59:33.876 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:33.876 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:33.877 | INFO | Uploaded 36GiB to GPU in 1.59952s (22.5068GiB/s) | 20260505 03:59:33.877 | INFO | Downloaded 34GiB to GPU in 1.51676s (22.4163GiB/s) | 20260505 03:59:33.974 | INFO | Starting a round of Baz IO | 20260505 03:59:36.938 | INFO | Populating RTMetrics report | 20260505 03:59:36.939 | INFO | RTMetrics report populated | 20260505 03:59:36.939 | INFO | Opening tmp RTMetrics json file | 20260505 03:59:36.939 | INFO | Serializing RTMetrics to json | 20260505 03:59:36.941 | INFO | Serialization done, swapping tmp file now... | 20260505 03:59:36.941 | INFO | RTMetrics done | 20260505 03:59:37.012 | INFO | Finished a round of Baz IO: Wrote 1.653 GiB in 3.038 seconds | 20260505 03:59:37.012 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 57.52% / 0.35% / 42.14% | 20260505 03:59:37.012 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 13.6549% / 0.0011007% / 3.17891e-05% | 20260505 03:59:37.012 | INFO | Baz IO overall: | 20260505 03:59:37.012 | INFO | BytesWritten : 47179607107 | 20260505 03:59:37.012 | INFO | NumEvents : 20882669428 | 20260505 03:59:39.015 | INFO | Analyzing chunk frames = [167936,168448) | 20260505 03:59:39.015 | INFO | MemInfo: { | 20260505 03:59:39.015 | INFO | "gpu" : " 17 GB", | 20260505 03:59:39.015 | INFO | "huge" : "", | 20260505 03:59:39.015 | INFO | "huge_shared" : " 27 GB", | 20260505 03:59:39.015 | INFO | "malloc" : " 14 GB", | 20260505 03:59:39.015 | INFO | "pinned" : " 29 GB" | 20260505 03:59:39.015 | INFO | } | 20260505 03:59:39.015 | INFO | Perf report for the last 5.139 seconds: | 20260505 03:59:39.015 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:59:39.015 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:39.015 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:39.015 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:39.015 | INFO | ANALYSIS : 58.5%, 3.0, 91.3ms, 0 | 20260505 03:59:39.015 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:39.015 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260505 03:59:39.015 | INFO | PRE_HQ : 58.0%, 2.2, 64.9ms, 0 | 20260505 03:59:39.016 | INFO | BAZWRITER : 58.7%, 3.2, 97.5ms, 0 | 20260505 03:59:39.016 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 03:59:39.016 | INFO | Uploaded 36GiB to GPU in 1.63605s (22.0043GiB/s) | 20260505 03:59:39.016 | INFO | Downloaded 38GiB to GPU in 1.69733s (22.3882GiB/s) | 20260505 03:59:44.155 | INFO | Analyzing chunk frames = [168448,168960) | 20260505 03:59:44.155 | INFO | MemInfo: { | 20260505 03:59:44.155 | INFO | "gpu" : " 17 GB", | 20260505 03:59:44.155 | INFO | "huge" : "", | 20260505 03:59:44.155 | INFO | "huge_shared" : " 27 GB", | 20260505 03:59:44.155 | INFO | "malloc" : " 14 GB", | 20260505 03:59:44.155 | INFO | "pinned" : " 29 GB" | 20260505 03:59:44.155 | INFO | } | 20260505 03:59:44.155 | INFO | Perf report for the last 5.1397 seconds: | 20260505 03:59:44.155 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:59:44.155 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:44.155 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:44.155 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:44.155 | INFO | ANALYSIS : 58.3%, 3.0, 91.0ms, 0 | 20260505 03:59:44.155 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:44.155 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:44.155 | INFO | PRE_HQ : 32.3%, 1.0, 16.7ms, 0 | 20260505 03:59:44.155 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:44.155 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:44.156 | INFO | Uploaded 36GiB to GPU in 1.5968s (22.5451GiB/s) | 20260505 03:59:44.156 | INFO | Downloaded 34GiB to GPU in 1.51331s (22.4674GiB/s) | 20260505 03:59:49.294 | INFO | Analyzing chunk frames = [168960,169472) | 20260505 03:59:49.294 | INFO | MemInfo: { | 20260505 03:59:49.294 | INFO | "gpu" : " 17 GB", | 20260505 03:59:49.294 | INFO | "huge" : "", | 20260505 03:59:49.294 | INFO | "huge_shared" : " 27 GB", | 20260505 03:59:49.294 | INFO | "malloc" : " 14 GB", | 20260505 03:59:49.294 | INFO | "pinned" : " 29 GB" | 20260505 03:59:49.294 | INFO | } | 20260505 03:59:49.294 | INFO | Perf report for the last 5.1391 seconds: | 20260505 03:59:49.295 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:59:49.295 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:49.295 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:49.295 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:49.295 | INFO | ANALYSIS : 57.7%, 3.0, 89.7ms, 0 | 20260505 03:59:49.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:49.296 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:49.296 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260505 03:59:49.296 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:49.296 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:49.296 | INFO | Uploaded 36GiB to GPU in 1.6004s (22.4944GiB/s) | 20260505 03:59:49.296 | INFO | Downloaded 34GiB to GPU in 1.51738s (22.4071GiB/s) | 20260505 03:59:54.433 | INFO | Analyzing chunk frames = [169472,169984) | 20260505 03:59:54.433 | INFO | MemInfo: { | 20260505 03:59:54.433 | INFO | "gpu" : " 17 GB", | 20260505 03:59:54.433 | INFO | "huge" : "", | 20260505 03:59:54.433 | INFO | "huge_shared" : " 27 GB", | 20260505 03:59:54.433 | INFO | "malloc" : " 14 GB", | 20260505 03:59:54.433 | INFO | "pinned" : " 29 GB" | 20260505 03:59:54.433 | INFO | } | 20260505 03:59:54.433 | INFO | Perf report for the last 5.13921 seconds: | 20260505 03:59:54.434 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:59:54.434 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:54.434 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:54.434 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:54.434 | INFO | ANALYSIS : 57.4%, 3.0, 89.0ms, 0 | 20260505 03:59:54.434 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:54.434 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:54.434 | INFO | PRE_HQ : 32.3%, 1.0, 16.7ms, 0 | 20260505 03:59:54.434 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:54.434 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:54.434 | INFO | Uploaded 36GiB to GPU in 1.59977s (22.5032GiB/s) | 20260505 03:59:54.434 | INFO | Downloaded 34GiB to GPU in 1.51662s (22.4183GiB/s) | 20260505 03:59:59.573 | INFO | Analyzing chunk frames = [169984,170496) | 20260505 03:59:59.573 | INFO | MemInfo: { | 20260505 03:59:59.573 | INFO | "gpu" : " 17 GB", | 20260505 03:59:59.573 | INFO | "huge" : "", | 20260505 03:59:59.573 | INFO | "huge_shared" : " 27 GB", | 20260505 03:59:59.573 | INFO | "malloc" : " 14 GB", | 20260505 03:59:59.573 | INFO | "pinned" : " 29 GB" | 20260505 03:59:59.573 | INFO | } | 20260505 03:59:59.573 | INFO | Perf report for the last 5.13971 seconds: | 20260505 03:59:59.574 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 03:59:59.574 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:59.574 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:59.574 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:59.574 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 03:59:59.574 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:59.574 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:59.574 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260505 03:59:59.574 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:59.574 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 03:59:59.574 | INFO | Uploaded 36GiB to GPU in 1.59824s (22.5248GiB/s) | 20260505 03:59:59.574 | INFO | Downloaded 34GiB to GPU in 1.5142s (22.4541GiB/s) | 20260505 04:00:04.712 | INFO | Analyzing chunk frames = [170496,171008) | 20260505 04:00:04.712 | INFO | MemInfo: { | 20260505 04:00:04.712 | INFO | "gpu" : " 17 GB", | 20260505 04:00:04.712 | INFO | "huge" : "", | 20260505 04:00:04.712 | INFO | "huge_shared" : " 27 GB", | 20260505 04:00:04.712 | INFO | "malloc" : " 14 GB", | 20260505 04:00:04.712 | INFO | "pinned" : " 29 GB" | 20260505 04:00:04.712 | INFO | } | 20260505 04:00:04.712 | INFO | Perf report for the last 5.139 seconds: | 20260505 04:00:04.712 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:00:04.712 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:04.712 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:04.712 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:04.712 | INFO | ANALYSIS : 57.6%, 3.0, 89.8ms, 0 | 20260505 04:00:04.712 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:04.712 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:04.712 | INFO | PRE_HQ : 32.8%, 1.0, 17.0ms, 0 | 20260505 04:00:04.712 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:04.712 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:04.713 | INFO | Uploaded 36GiB to GPU in 1.59852s (22.5208GiB/s) | 20260505 04:00:04.713 | INFO | Downloaded 34GiB to GPU in 1.51531s (22.4377GiB/s) | 20260505 04:00:09.851 | INFO | Analyzing chunk frames = [171008,171520) | 20260505 04:00:09.851 | INFO | MemInfo: { | 20260505 04:00:09.851 | INFO | "gpu" : " 17 GB", | 20260505 04:00:09.851 | INFO | "huge" : "", | 20260505 04:00:09.851 | INFO | "huge_shared" : " 27 GB", | 20260505 04:00:09.851 | INFO | "malloc" : " 14 GB", | 20260505 04:00:09.851 | INFO | "pinned" : " 29 GB" | 20260505 04:00:09.851 | INFO | } | 20260505 04:00:09.851 | INFO | Perf report for the last 5.13936 seconds: | 20260505 04:00:09.852 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:00:09.852 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:09.852 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:09.852 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:09.852 | INFO | ANALYSIS : 57.9%, 3.0, 90.3ms, 0 | 20260505 04:00:09.852 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:09.852 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:09.852 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260505 04:00:09.852 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:09.852 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:09.852 | INFO | Uploaded 36GiB to GPU in 1.59834s (22.5234GiB/s) | 20260505 04:00:09.852 | INFO | Downloaded 34GiB to GPU in 1.51541s (22.4361GiB/s) | 20260505 04:00:14.991 | INFO | Analyzing chunk frames = [171520,172032) | 20260505 04:00:14.991 | INFO | MemInfo: { | 20260505 04:00:14.991 | INFO | "gpu" : " 17 GB", | 20260505 04:00:14.991 | INFO | "huge" : "", | 20260505 04:00:14.991 | INFO | "huge_shared" : " 27 GB", | 20260505 04:00:14.991 | INFO | "malloc" : " 14 GB", | 20260505 04:00:14.991 | INFO | "pinned" : " 29 GB" | 20260505 04:00:14.991 | INFO | } | 20260505 04:00:14.991 | INFO | Perf report for the last 5.13931 seconds: | 20260505 04:00:14.991 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:00:14.991 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:14.991 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:14.991 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:14.991 | INFO | ANALYSIS : 57.9%, 3.0, 90.3ms, 0 | 20260505 04:00:14.991 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:14.991 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:14.991 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260505 04:00:14.991 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:14.991 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:14.991 | INFO | Uploaded 36GiB to GPU in 1.59933s (22.5095GiB/s) | 20260505 04:00:14.991 | INFO | Downloaded 34GiB to GPU in 1.51606s (22.4265GiB/s) | 20260505 04:00:14.991 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 04:00:14.992 | INFO | Total mem usage: 14.9 GiB | 20260505 04:00:14.992 | INFO | All Time High Water Marks sum: 15.2 GiB (Savings: 1.52%) | 20260505 04:00:14.992 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -2.96%) | 20260505 04:00:14.992 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 04:00:14.992 | INFO | High Water Breakdown: | 20260505 04:00:14.992 | INFO | 58.2% : 8.44 GiB ( 99.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 04:00:14.992 | INFO | 41.6% : 6.03 GiB ( 90.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 04:00:14.992 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 04:00:14.992 | INFO | ------------------------------------------ | 20260505 04:00:14.992 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:00:14.992 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 04:00:14.992 | INFO | Total mem usage: 26.5 GiB | 20260505 04:00:14.992 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260505 04:00:14.992 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 04:00:14.992 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260505 04:00:14.992 | INFO | High Water Breakdown: | 20260505 04:00:14.992 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:00:14.992 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:00:14.992 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:00:14.992 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:00:14.992 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:00:14.992 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:00:14.992 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 04:00:14.992 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 04:00:14.992 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:00:14.992 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 04:00:14.992 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 04:00:14.992 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 04:00:14.992 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 04:00:14.992 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 04:00:14.992 | INFO | ------------------------------------------ | 20260505 04:00:14.992 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:00:14.992 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 04:00:14.992 | INFO | Total mem usage: 25.6 GiB | 20260505 04:00:14.992 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 04:00:14.992 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 04:00:14.992 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 04:00:14.992 | INFO | High Water Breakdown: | 20260505 04:00:14.992 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 04:00:14.992 | INFO | ------------------------------------------ | 20260505 04:00:14.992 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:00:14.992 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 04:00:14.992 | INFO | Total mem usage: 15.7 GiB | 20260505 04:00:14.992 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260505 04:00:14.992 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 04:00:14.992 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260505 04:00:14.992 | INFO | High Water Breakdown: | 20260505 04:00:14.992 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 04:00:14.992 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 04:00:14.992 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 04:00:14.992 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 04:00:14.992 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 04:00:14.992 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 04:00:14.992 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 04:00:14.992 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 04:00:14.992 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 04:00:14.992 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:00:14.992 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 04:00:14.992 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:00:14.992 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 04:00:14.992 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 04:00:14.992 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 04:00:14.992 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 04:00:14.992 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:00:14.992 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 04:00:14.992 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:00:14.992 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:00:14.992 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:00:14.992 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 04:00:14.992 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 04:00:14.992 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:00:14.992 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 04:00:14.993 | INFO | ------------------------------------------ | 20260505 04:00:14.993 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:00:14.993 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 04:00:14.993 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:00:14.993 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:00:14.993 | INFO | PerfReport: Full Execution -- 3.06e+01 / -nan, 1.28e+01 / 0.00e+00 | 20260505 04:00:14.993 | INFO | PerfReport: 0.28%, AggregateStats -- 8.63e-02 / -nan, 4.38e-03 / 0.00e+00 | 20260505 04:00:14.993 | INFO | PerfReport: 10.76%, Baseline -- 3.30e+00 / -nan, 1.71e-02 / 0.00e+00 | 20260505 04:00:14.993 | INFO | PerfReport: 3.02%, Histogram -- 9.23e-01 / -nan, 5.20e-02 / 0.00e+00 | 20260505 04:00:14.993 | INFO | PerfReport: 16.01%, DME -- 3.92e+01 / -nan, 3.24e+00 / 0.00e+00 | 20260505 04:00:14.993 | INFO | PerfReport: 56.22%, FrameLabeling -- 1.72e+01 / -nan, 4.40e-02 / 0.00e+00 | 20260505 04:00:14.993 | INFO | PerfReport: 7.77%, PulseAccumulating -- 2.38e+00 / -nan, 1.60e-01 / 0.00e+00 | 20260505 04:00:14.993 | INFO | PerfReport: 4.18%, Metrics -- 1.28e+00 / -nan, 2.09e-01 / 0.00e+00 | 20260505 04:00:14.993 | INFO | PerfReport: 1.75%, PulseSerializing -- 5.35e-01 / -nan, 8.60e-02 / 0.00e+00 | 20260505 04:00:14.993 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 04:00:14.993 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:00:14.993 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:00:14.993 | INFO | PerfReport: Full Execution -- 9.20e+01 / -nan, 1.91e+01 / 0.00e+00 | 20260505 04:00:14.993 | INFO | PerfReport: 18.16%, Upload -- 1.67e+01 / 0.00e+00, 3.64e-01 / 0.00e+00 | 20260505 04:00:14.993 | INFO | PerfReport: 17.42%, Download -- 1.60e+01 / 0.00e+00, 6.89e-01 / 0.00e+00 | 20260505 04:00:14.993 | INFO | PerfReport: 33.29%, Compute -- 3.06e+01 / 0.00e+00, 1.28e+01 / 0.00e+00 | 20260505 04:00:14.993 | INFO | PerfReport: 0.55%, LockUpload -- 5.05e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260505 04:00:14.993 | INFO | PerfReport: 0.00%, LockDownload -- 3.98e-04 / 0.00e+00, 1.64e-04 / 0.00e+00 | 20260505 04:00:14.993 | INFO | PerfReport: 30.56%, LockCompute -- 2.81e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260505 04:00:14.993 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.15e-02 / 0.00e+00 | 20260505 04:00:15.088 | INFO | Starting a round of Baz IO | 20260505 04:00:18.055 | INFO | Populating RTMetrics report | 20260505 04:00:18.056 | INFO | RTMetrics report populated | 20260505 04:00:18.056 | INFO | Opening tmp RTMetrics json file | 20260505 04:00:18.057 | INFO | Serializing RTMetrics to json | 20260505 04:00:18.058 | INFO | Serialization done, swapping tmp file now... | 20260505 04:00:18.059 | INFO | RTMetrics done | 20260505 04:00:18.125 | INFO | Finished a round of Baz IO: Wrote 1.666 GiB in 3.037 seconds | 20260505 04:00:18.125 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 57.19% / 0.33% / 42.48% | 20260505 04:00:18.125 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 13.6342% / 0.00103315% / 3.97364e-05% | 20260505 04:00:18.125 | INFO | Baz IO overall: | 20260505 04:00:18.125 | INFO | BytesWritten : 48968416419 | 20260505 04:00:18.125 | INFO | NumEvents : 21725833324 | 20260505 04:00:20.130 | INFO | Analyzing chunk frames = [172032,172544) | 20260505 04:00:20.130 | INFO | MemInfo: { | 20260505 04:00:20.130 | INFO | "gpu" : " 17 GB", | 20260505 04:00:20.130 | INFO | "huge" : "", | 20260505 04:00:20.130 | INFO | "huge_shared" : " 27 GB", | 20260505 04:00:20.130 | INFO | "malloc" : " 14 GB", | 20260505 04:00:20.130 | INFO | "pinned" : " 29 GB" | 20260505 04:00:20.130 | INFO | } | 20260505 04:00:20.130 | INFO | Perf report for the last 5.13952 seconds: | 20260505 04:00:20.130 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:00:20.130 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:20.130 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:20.130 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:20.130 | INFO | ANALYSIS : 58.6%, 3.0, 91.4ms, 0 | 20260505 04:00:20.130 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:20.130 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260505 04:00:20.130 | INFO | PRE_HQ : 58.0%, 2.2, 64.8ms, 0 | 20260505 04:00:20.130 | INFO | BAZWRITER : 58.7%, 3.2, 97.4ms, 0 | 20260505 04:00:20.130 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:00:20.131 | INFO | Uploaded 36GiB to GPU in 1.63632s (22.0006GiB/s) | 20260505 04:00:20.131 | INFO | Downloaded 38GiB to GPU in 1.69703s (22.392GiB/s) | 20260505 04:00:25.270 | INFO | Analyzing chunk frames = [172544,173056) | 20260505 04:00:25.271 | INFO | MemInfo: { | 20260505 04:00:25.271 | INFO | "gpu" : " 17 GB", | 20260505 04:00:25.271 | INFO | "huge" : "", | 20260505 04:00:25.271 | INFO | "huge_shared" : " 27 GB", | 20260505 04:00:25.271 | INFO | "malloc" : " 14 GB", | 20260505 04:00:25.271 | INFO | "pinned" : " 29 GB" | 20260505 04:00:25.271 | INFO | } | 20260505 04:00:25.271 | INFO | Perf report for the last 5.14051 seconds: | 20260505 04:00:25.271 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:00:25.271 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:25.271 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:25.271 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:25.271 | INFO | ANALYSIS : 58.3%, 3.0, 91.0ms, 0 | 20260505 04:00:25.271 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:25.271 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:25.271 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 04:00:25.271 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:25.271 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:25.271 | INFO | Uploaded 36GiB to GPU in 1.59665s (22.5472GiB/s) | 20260505 04:00:25.271 | INFO | Downloaded 34GiB to GPU in 1.51297s (22.4724GiB/s) | 20260505 04:00:30.409 | INFO | Analyzing chunk frames = [173056,173568) | 20260505 04:00:30.409 | INFO | MemInfo: { | 20260505 04:00:30.409 | INFO | "gpu" : " 17 GB", | 20260505 04:00:30.409 | INFO | "huge" : "", | 20260505 04:00:30.409 | INFO | "huge_shared" : " 27 GB", | 20260505 04:00:30.409 | INFO | "malloc" : " 14 GB", | 20260505 04:00:30.409 | INFO | "pinned" : " 29 GB" | 20260505 04:00:30.409 | INFO | } | 20260505 04:00:30.409 | INFO | Perf report for the last 5.13802 seconds: | 20260505 04:00:30.409 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:00:30.409 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:30.409 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:30.409 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:30.409 | INFO | ANALYSIS : 57.8%, 3.0, 89.8ms, 0 | 20260505 04:00:30.409 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:30.409 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:30.409 | INFO | PRE_HQ : 31.9%, 1.0, 16.5ms, 0 | 20260505 04:00:30.409 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:30.409 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:30.409 | INFO | Uploaded 36GiB to GPU in 1.5998s (22.5028GiB/s) | 20260505 04:00:30.409 | INFO | Downloaded 34GiB to GPU in 1.51605s (22.4267GiB/s) | 20260505 04:00:35.548 | INFO | Analyzing chunk frames = [173568,174080) | 20260505 04:00:35.548 | INFO | MemInfo: { | 20260505 04:00:35.548 | INFO | "gpu" : " 17 GB", | 20260505 04:00:35.548 | INFO | "huge" : "", | 20260505 04:00:35.548 | INFO | "huge_shared" : " 27 GB", | 20260505 04:00:35.548 | INFO | "malloc" : " 14 GB", | 20260505 04:00:35.548 | INFO | "pinned" : " 29 GB" | 20260505 04:00:35.548 | INFO | } | 20260505 04:00:35.548 | INFO | Perf report for the last 5.13931 seconds: | 20260505 04:00:35.548 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:00:35.548 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:35.548 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:35.548 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:35.548 | INFO | ANALYSIS : 57.4%, 3.0, 89.0ms, 0 | 20260505 04:00:35.548 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:35.548 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:35.548 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260505 04:00:35.548 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:35.548 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:35.549 | INFO | Uploaded 36GiB to GPU in 1.60025s (22.4965GiB/s) | 20260505 04:00:35.549 | INFO | Downloaded 34GiB to GPU in 1.51725s (22.409GiB/s) | 20260505 04:00:40.687 | INFO | Analyzing chunk frames = [174080,174592) | 20260505 04:00:40.687 | INFO | MemInfo: { | 20260505 04:00:40.687 | INFO | "gpu" : " 17 GB", | 20260505 04:00:40.687 | INFO | "huge" : "", | 20260505 04:00:40.687 | INFO | "huge_shared" : " 27 GB", | 20260505 04:00:40.687 | INFO | "malloc" : " 14 GB", | 20260505 04:00:40.687 | INFO | "pinned" : " 29 GB" | 20260505 04:00:40.687 | INFO | } | 20260505 04:00:40.687 | INFO | Perf report for the last 5.13921 seconds: | 20260505 04:00:40.688 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:00:40.688 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:40.688 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:40.688 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:40.688 | INFO | ANALYSIS : 57.3%, 3.0, 89.5ms, 0 | 20260505 04:00:40.688 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:40.688 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:40.688 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260505 04:00:40.688 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:40.688 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:40.688 | INFO | Uploaded 36GiB to GPU in 1.59796s (22.5288GiB/s) | 20260505 04:00:40.688 | INFO | Downloaded 34GiB to GPU in 1.51396s (22.4577GiB/s) | 20260505 04:00:45.827 | INFO | Analyzing chunk frames = [174592,175104) | 20260505 04:00:45.827 | INFO | MemInfo: { | 20260505 04:00:45.827 | INFO | "gpu" : " 17 GB", | 20260505 04:00:45.827 | INFO | "huge" : "", | 20260505 04:00:45.827 | INFO | "huge_shared" : " 27 GB", | 20260505 04:00:45.827 | INFO | "malloc" : " 14 GB", | 20260505 04:00:45.827 | INFO | "pinned" : " 29 GB" | 20260505 04:00:45.827 | INFO | } | 20260505 04:00:45.827 | INFO | Perf report for the last 5.13932 seconds: | 20260505 04:00:45.827 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:00:45.827 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:45.827 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:45.827 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:45.827 | INFO | ANALYSIS : 57.6%, 3.0, 89.8ms, 0 | 20260505 04:00:45.827 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:45.827 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:45.827 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260505 04:00:45.827 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:45.827 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:45.828 | INFO | Uploaded 36GiB to GPU in 1.59797s (22.5286GiB/s) | 20260505 04:00:45.828 | INFO | Downloaded 34GiB to GPU in 1.51434s (22.4521GiB/s) | 20260505 04:00:50.966 | INFO | Analyzing chunk frames = [175104,175616) | 20260505 04:00:50.966 | INFO | MemInfo: { | 20260505 04:00:50.966 | INFO | "gpu" : " 17 GB", | 20260505 04:00:50.966 | INFO | "huge" : "", | 20260505 04:00:50.966 | INFO | "huge_shared" : " 27 GB", | 20260505 04:00:50.966 | INFO | "malloc" : " 14 GB", | 20260505 04:00:50.966 | INFO | "pinned" : " 29 GB" | 20260505 04:00:50.966 | INFO | } | 20260505 04:00:50.966 | INFO | Perf report for the last 5.13942 seconds: | 20260505 04:00:50.966 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:00:50.966 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:50.966 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:50.966 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:50.966 | INFO | ANALYSIS : 57.9%, 3.0, 90.3ms, 0 | 20260505 04:00:50.966 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:50.966 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:50.966 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 04:00:50.966 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:50.966 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:50.967 | INFO | Uploaded 36GiB to GPU in 1.59831s (22.5238GiB/s) | 20260505 04:00:50.967 | INFO | Downloaded 34GiB to GPU in 1.51521s (22.4391GiB/s) | 20260505 04:00:56.105 | INFO | Analyzing chunk frames = [175616,176128) | 20260505 04:00:56.105 | INFO | MemInfo: { | 20260505 04:00:56.106 | INFO | "gpu" : " 17 GB", | 20260505 04:00:56.106 | INFO | "huge" : "", | 20260505 04:00:56.106 | INFO | "huge_shared" : " 27 GB", | 20260505 04:00:56.106 | INFO | "malloc" : " 14 GB", | 20260505 04:00:56.106 | INFO | "pinned" : " 29 GB" | 20260505 04:00:56.106 | INFO | } | 20260505 04:00:56.106 | INFO | Perf report for the last 5.13958 seconds: | 20260505 04:00:56.106 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:00:56.106 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:56.106 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:56.106 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:56.106 | INFO | ANALYSIS : 57.9%, 3.0, 90.4ms, 0 | 20260505 04:00:56.106 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:56.106 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:56.106 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260505 04:00:56.106 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:56.106 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:00:56.106 | INFO | Uploaded 36GiB to GPU in 1.5997s (22.5042GiB/s) | 20260505 04:00:56.106 | INFO | Downloaded 34GiB to GPU in 1.51615s (22.4252GiB/s) | 20260505 04:00:56.203 | INFO | Starting a round of Baz IO | 20260505 04:00:59.171 | INFO | Populating RTMetrics report | 20260505 04:00:59.171 | INFO | RTMetrics report populated | 20260505 04:00:59.171 | INFO | Opening tmp RTMetrics json file | 20260505 04:00:59.172 | INFO | Serializing RTMetrics to json | 20260505 04:00:59.174 | INFO | Serialization done, swapping tmp file now... | 20260505 04:00:59.174 | INFO | RTMetrics done | 20260505 04:00:59.242 | INFO | Finished a round of Baz IO: Wrote 1.679 GiB in 3.039 seconds | 20260505 04:00:59.242 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 56.86% / 0.32% / 42.81% | 20260505 04:00:59.242 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 13.6548% / 0.000961622% / 3.57628e-05% | 20260505 04:00:59.242 | INFO | Baz IO overall: | 20260505 04:00:59.242 | INFO | BytesWritten : 50771642141 | 20260505 04:00:59.242 | INFO | NumEvents : 22576685021 | 20260505 04:01:01.245 | INFO | Analyzing chunk frames = [176128,176640) | 20260505 04:01:01.245 | INFO | MemInfo: { | 20260505 04:01:01.245 | INFO | "gpu" : " 17 GB", | 20260505 04:01:01.245 | INFO | "huge" : "", | 20260505 04:01:01.245 | INFO | "huge_shared" : " 27 GB", | 20260505 04:01:01.245 | INFO | "malloc" : " 14 GB", | 20260505 04:01:01.245 | INFO | "pinned" : " 29 GB" | 20260505 04:01:01.245 | INFO | } | 20260505 04:01:01.245 | INFO | Perf report for the last 5.13923 seconds: | 20260505 04:01:01.245 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:01:01.245 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:01.245 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:01.245 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:01.245 | INFO | ANALYSIS : 58.6%, 3.0, 91.4ms, 0 | 20260505 04:01:01.245 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:01.245 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260505 04:01:01.245 | INFO | PRE_HQ : 58.1%, 2.2, 65.0ms, 0 | 20260505 04:01:01.245 | INFO | BAZWRITER : 58.7%, 3.3, 98.9ms, 0 | 20260505 04:01:01.245 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:01:01.245 | INFO | Uploaded 36GiB to GPU in 1.63866s (21.9691GiB/s) | 20260505 04:01:01.245 | INFO | Downloaded 38GiB to GPU in 1.6987s (22.3701GiB/s) | 20260505 04:01:06.384 | INFO | Analyzing chunk frames = [176640,177152) | 20260505 04:01:06.384 | INFO | MemInfo: { | 20260505 04:01:06.384 | INFO | "gpu" : " 17 GB", | 20260505 04:01:06.385 | INFO | "huge" : "", | 20260505 04:01:06.385 | INFO | "huge_shared" : " 27 GB", | 20260505 04:01:06.385 | INFO | "malloc" : " 14 GB", | 20260505 04:01:06.385 | INFO | "pinned" : " 29 GB" | 20260505 04:01:06.385 | INFO | } | 20260505 04:01:06.385 | INFO | Perf report for the last 5.1397 seconds: | 20260505 04:01:06.385 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:01:06.385 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:06.385 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:06.385 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:06.385 | INFO | ANALYSIS : 58.3%, 3.0, 91.0ms, 0 | 20260505 04:01:06.385 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:06.385 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:06.385 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 04:01:06.385 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:06.385 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:06.385 | INFO | Uploaded 36GiB to GPU in 1.59644s (22.5502GiB/s) | 20260505 04:01:06.385 | INFO | Downloaded 34GiB to GPU in 1.51298s (22.4723GiB/s) | 20260505 04:01:11.524 | INFO | Analyzing chunk frames = [177152,177664) | 20260505 04:01:11.524 | INFO | MemInfo: { | 20260505 04:01:11.524 | INFO | "gpu" : " 17 GB", | 20260505 04:01:11.524 | INFO | "huge" : "", | 20260505 04:01:11.524 | INFO | "huge_shared" : " 27 GB", | 20260505 04:01:11.524 | INFO | "malloc" : " 14 GB", | 20260505 04:01:11.524 | INFO | "pinned" : " 29 GB" | 20260505 04:01:11.524 | INFO | } | 20260505 04:01:11.524 | INFO | Perf report for the last 5.13889 seconds: | 20260505 04:01:11.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:01:11.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:11.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:11.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:11.527 | INFO | ANALYSIS : 57.7%, 3.0, 89.7ms, 0 | 20260505 04:01:11.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:11.527 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:11.527 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260505 04:01:11.527 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:11.527 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:11.527 | INFO | Uploaded 36GiB to GPU in 1.60023s (22.4968GiB/s) | 20260505 04:01:11.527 | INFO | Downloaded 34GiB to GPU in 1.51693s (22.4137GiB/s) | 20260505 04:01:16.663 | INFO | Analyzing chunk frames = [177664,178176) | 20260505 04:01:16.663 | INFO | MemInfo: { | 20260505 04:01:16.663 | INFO | "gpu" : " 17 GB", | 20260505 04:01:16.663 | INFO | "huge" : "", | 20260505 04:01:16.663 | INFO | "huge_shared" : " 27 GB", | 20260505 04:01:16.663 | INFO | "malloc" : " 14 GB", | 20260505 04:01:16.663 | INFO | "pinned" : " 29 GB" | 20260505 04:01:16.663 | INFO | } | 20260505 04:01:16.663 | INFO | Perf report for the last 5.1395 seconds: | 20260505 04:01:16.663 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:01:16.663 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:16.663 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:16.663 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:16.663 | INFO | ANALYSIS : 57.4%, 3.0, 89.0ms, 0 | 20260505 04:01:16.663 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:16.663 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:16.663 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 04:01:16.663 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:16.663 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:16.663 | INFO | Uploaded 36GiB to GPU in 1.59993s (22.501GiB/s) | 20260505 04:01:16.664 | INFO | Downloaded 34GiB to GPU in 1.51685s (22.4148GiB/s) | 20260505 04:01:21.802 | INFO | Analyzing chunk frames = [178176,178688) | 20260505 04:01:21.802 | INFO | MemInfo: { | 20260505 04:01:21.802 | INFO | "gpu" : " 17 GB", | 20260505 04:01:21.802 | INFO | "huge" : "", | 20260505 04:01:21.802 | INFO | "huge_shared" : " 27 GB", | 20260505 04:01:21.802 | INFO | "malloc" : " 14 GB", | 20260505 04:01:21.802 | INFO | "pinned" : " 29 GB" | 20260505 04:01:21.802 | INFO | } | 20260505 04:01:21.802 | INFO | Perf report for the last 5.13921 seconds: | 20260505 04:01:21.802 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:01:21.802 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:21.802 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:21.802 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:21.802 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 04:01:21.802 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:21.802 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:21.803 | INFO | PRE_HQ : 32.7%, 1.0, 17.0ms, 0 | 20260505 04:01:21.803 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:21.803 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:21.803 | INFO | Uploaded 36GiB to GPU in 1.59826s (22.5244GiB/s) | 20260505 04:01:21.803 | INFO | Downloaded 34GiB to GPU in 1.5141s (22.4556GiB/s) | 20260505 04:01:26.941 | INFO | Analyzing chunk frames = [178688,179200) | 20260505 04:01:26.941 | INFO | MemInfo: { | 20260505 04:01:26.941 | INFO | "gpu" : " 17 GB", | 20260505 04:01:26.941 | INFO | "huge" : "", | 20260505 04:01:26.941 | INFO | "huge_shared" : " 27 GB", | 20260505 04:01:26.941 | INFO | "malloc" : " 14 GB", | 20260505 04:01:26.941 | INFO | "pinned" : " 29 GB" | 20260505 04:01:26.941 | INFO | } | 20260505 04:01:26.942 | INFO | Perf report for the last 5.13929 seconds: | 20260505 04:01:26.942 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:01:26.942 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:26.942 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:26.942 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:26.942 | INFO | ANALYSIS : 57.6%, 3.0, 89.9ms, 0 | 20260505 04:01:26.942 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:26.942 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:26.942 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260505 04:01:26.942 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:26.942 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:26.942 | INFO | Uploaded 36GiB to GPU in 1.59748s (22.5355GiB/s) | 20260505 04:01:26.942 | INFO | Downloaded 34GiB to GPU in 1.51479s (22.4454GiB/s) | 20260505 04:01:32.081 | INFO | Analyzing chunk frames = [179200,179712) | 20260505 04:01:32.081 | INFO | MemInfo: { | 20260505 04:01:32.081 | INFO | "gpu" : " 17 GB", | 20260505 04:01:32.081 | INFO | "huge" : "", | 20260505 04:01:32.081 | INFO | "huge_shared" : " 27 GB", | 20260505 04:01:32.081 | INFO | "malloc" : " 14 GB", | 20260505 04:01:32.081 | INFO | "pinned" : " 29 GB" | 20260505 04:01:32.081 | INFO | } | 20260505 04:01:32.081 | INFO | Perf report for the last 5.13933 seconds: | 20260505 04:01:32.081 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:01:32.081 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:32.081 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:32.081 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:32.081 | INFO | ANALYSIS : 57.8%, 3.0, 90.1ms, 0 | 20260505 04:01:32.081 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:32.081 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:32.081 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260505 04:01:32.081 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:32.081 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:32.082 | INFO | Uploaded 36GiB to GPU in 1.59886s (22.516GiB/s) | 20260505 04:01:32.082 | INFO | Downloaded 34GiB to GPU in 1.51612s (22.4256GiB/s) | 20260505 04:01:37.220 | INFO | Analyzing chunk frames = [179712,180224) | 20260505 04:01:37.220 | INFO | MemInfo: { | 20260505 04:01:37.220 | INFO | "gpu" : " 17 GB", | 20260505 04:01:37.220 | INFO | "huge" : "", | 20260505 04:01:37.220 | INFO | "huge_shared" : " 27 GB", | 20260505 04:01:37.220 | INFO | "malloc" : " 14 GB", | 20260505 04:01:37.220 | INFO | "pinned" : " 29 GB" | 20260505 04:01:37.220 | INFO | } | 20260505 04:01:37.220 | INFO | Perf report for the last 5.13913 seconds: | 20260505 04:01:37.221 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:01:37.221 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:37.221 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:37.221 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:37.221 | INFO | ANALYSIS : 57.9%, 3.0, 90.4ms, 0 | 20260505 04:01:37.221 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:37.221 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:37.221 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260505 04:01:37.221 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:37.221 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:37.221 | INFO | Uploaded 36GiB to GPU in 1.59871s (22.5181GiB/s) | 20260505 04:01:37.221 | INFO | Downloaded 34GiB to GPU in 1.5157s (22.4319GiB/s) | 20260505 04:01:37.221 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 04:01:37.221 | INFO | Total mem usage: 14.9 GiB | 20260505 04:01:37.221 | INFO | All Time High Water Marks sum: 15.2 GiB (Savings: 1.52%) | 20260505 04:01:37.221 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -2.95%) | 20260505 04:01:37.221 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 04:01:37.221 | INFO | High Water Breakdown: | 20260505 04:01:37.221 | INFO | 58.3% : 8.45 GiB ( 99.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 04:01:37.221 | INFO | 41.6% : 6.02 GiB ( 90.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 04:01:37.221 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 04:01:37.221 | INFO | ------------------------------------------ | 20260505 04:01:37.221 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:01:37.221 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 04:01:37.221 | INFO | Total mem usage: 26.5 GiB | 20260505 04:01:37.221 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260505 04:01:37.221 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 04:01:37.221 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260505 04:01:37.221 | INFO | High Water Breakdown: | 20260505 04:01:37.221 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:01:37.221 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:01:37.221 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:01:37.221 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:01:37.221 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:01:37.221 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:01:37.221 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 04:01:37.221 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 04:01:37.221 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:01:37.221 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 04:01:37.221 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 04:01:37.221 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 04:01:37.221 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 04:01:37.221 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 04:01:37.221 | INFO | ------------------------------------------ | 20260505 04:01:37.221 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:01:37.221 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 04:01:37.221 | INFO | Total mem usage: 25.6 GiB | 20260505 04:01:37.221 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 04:01:37.221 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 04:01:37.221 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 04:01:37.221 | INFO | High Water Breakdown: | 20260505 04:01:37.221 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 04:01:37.221 | INFO | ------------------------------------------ | 20260505 04:01:37.221 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:01:37.221 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 04:01:37.221 | INFO | Total mem usage: 15.7 GiB | 20260505 04:01:37.221 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260505 04:01:37.221 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 04:01:37.221 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260505 04:01:37.221 | INFO | High Water Breakdown: | 20260505 04:01:37.221 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 04:01:37.221 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 04:01:37.221 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 04:01:37.221 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 04:01:37.221 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 04:01:37.221 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 04:01:37.221 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 04:01:37.221 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 04:01:37.221 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 04:01:37.221 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:01:37.221 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 04:01:37.221 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:01:37.221 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 04:01:37.221 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 04:01:37.221 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 04:01:37.221 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 04:01:37.221 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:01:37.221 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 04:01:37.221 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:01:37.221 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:01:37.221 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:01:37.221 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 04:01:37.221 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 04:01:37.221 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:01:37.221 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 04:01:37.221 | INFO | ------------------------------------------ | 20260505 04:01:37.221 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:01:37.221 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 04:01:37.221 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:01:37.222 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:01:37.222 | INFO | PerfReport: Full Execution -- 3.06e+01 / -nan, 1.28e+01 / 0.00e+00 | 20260505 04:01:37.222 | INFO | PerfReport: 0.28%, AggregateStats -- 8.63e-02 / -nan, 3.83e-03 / 0.00e+00 | 20260505 04:01:37.222 | INFO | PerfReport: 10.76%, Baseline -- 3.30e+00 / -nan, 1.72e-02 / 0.00e+00 | 20260505 04:01:37.222 | INFO | PerfReport: 3.02%, Histogram -- 9.24e-01 / -nan, 5.21e-02 / 0.00e+00 | 20260505 04:01:37.222 | INFO | PerfReport: 16.02%, DME -- 3.92e+01 / -nan, 3.19e+00 / 0.00e+00 | 20260505 04:01:37.222 | INFO | PerfReport: 56.22%, FrameLabeling -- 1.72e+01 / -nan, 4.45e-02 / 0.00e+00 | 20260505 04:01:37.222 | INFO | PerfReport: 7.77%, PulseAccumulating -- 2.38e+00 / -nan, 1.60e-01 / 0.00e+00 | 20260505 04:01:37.222 | INFO | PerfReport: 4.18%, Metrics -- 1.28e+00 / -nan, 2.08e-01 / 0.00e+00 | 20260505 04:01:37.222 | INFO | PerfReport: 1.74%, PulseSerializing -- 5.34e-01 / -nan, 8.56e-02 / 0.00e+00 | 20260505 04:01:37.222 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 04:01:37.222 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:01:37.222 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:01:37.222 | INFO | PerfReport: Full Execution -- 9.20e+01 / -nan, 1.92e+01 / 0.00e+00 | 20260505 04:01:37.222 | INFO | PerfReport: 18.16%, Upload -- 1.67e+01 / 0.00e+00, 3.67e-01 / 0.00e+00 | 20260505 04:01:37.222 | INFO | PerfReport: 17.42%, Download -- 1.60e+01 / 0.00e+00, 6.89e-01 / 0.00e+00 | 20260505 04:01:37.222 | INFO | PerfReport: 33.29%, Compute -- 3.06e+01 / 0.00e+00, 1.28e+01 / 0.00e+00 | 20260505 04:01:37.222 | INFO | PerfReport: 0.55%, LockUpload -- 5.03e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260505 04:01:37.222 | INFO | PerfReport: 0.00%, LockDownload -- 4.03e-04 / 0.00e+00, 1.71e-04 / 0.00e+00 | 20260505 04:01:37.222 | INFO | PerfReport: 30.57%, LockCompute -- 2.81e+01 / 0.00e+00, 1.72e+01 / 0.00e+00 | 20260505 04:01:37.222 | INFO | PerfReport: 0.02%, Misc -- 1.41e-02 / 0.00e+00, 2.27e-02 / 0.00e+00 | 20260505 04:01:37.318 | INFO | Starting a round of Baz IO | 20260505 04:01:40.283 | INFO | Populating RTMetrics report | 20260505 04:01:40.283 | INFO | RTMetrics report populated | 20260505 04:01:40.283 | INFO | Opening tmp RTMetrics json file | 20260505 04:01:40.284 | INFO | Serializing RTMetrics to json | 20260505 04:01:40.286 | INFO | Serialization done, swapping tmp file now... | 20260505 04:01:40.286 | INFO | RTMetrics done | 20260505 04:01:40.359 | INFO | Finished a round of Baz IO: Wrote 1.686 GiB in 3.04 seconds | 20260505 04:01:40.359 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 56.55% / 0.32% / 43.14% | 20260505 04:01:40.359 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 13.5166% / 0.00100931% / 2.38419e-05% | 20260505 04:01:40.359 | INFO | Baz IO overall: | 20260505 04:01:40.359 | INFO | BytesWritten : 52581563714 | 20260505 04:01:40.359 | INFO | NumEvents : 23430019905 | 20260505 04:01:42.363 | INFO | Analyzing chunk frames = [180224,180736) | 20260505 04:01:42.363 | INFO | MemInfo: { | 20260505 04:01:42.364 | INFO | "gpu" : " 17 GB", | 20260505 04:01:42.364 | INFO | "huge" : "", | 20260505 04:01:42.364 | INFO | "huge_shared" : " 27 GB", | 20260505 04:01:42.364 | INFO | "malloc" : " 14 GB", | 20260505 04:01:42.364 | INFO | "pinned" : " 29 GB" | 20260505 04:01:42.364 | INFO | } | 20260505 04:01:42.364 | INFO | Perf report for the last 5.14341 seconds: | 20260505 04:01:42.364 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:01:42.364 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:42.364 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:42.364 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:42.364 | INFO | ANALYSIS : 58.5%, 3.0, 91.3ms, 0 | 20260505 04:01:42.364 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:42.364 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260505 04:01:42.364 | INFO | PRE_HQ : 58.0%, 2.2, 65.1ms, 0 | 20260505 04:01:42.364 | INFO | BAZWRITER : 58.7%, 3.3, 99.2ms, 0 | 20260505 04:01:42.364 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:01:42.364 | INFO | Uploaded 36GiB to GPU in 1.63669s (21.9956GiB/s) | 20260505 04:01:42.364 | INFO | Downloaded 38GiB to GPU in 1.6963s (22.4017GiB/s) | 20260505 04:01:47.499 | INFO | Analyzing chunk frames = [180736,181248) | 20260505 04:01:47.499 | INFO | MemInfo: { | 20260505 04:01:47.499 | INFO | "gpu" : " 17 GB", | 20260505 04:01:47.499 | INFO | "huge" : "", | 20260505 04:01:47.499 | INFO | "huge_shared" : " 27 GB", | 20260505 04:01:47.499 | INFO | "malloc" : " 14 GB", | 20260505 04:01:47.499 | INFO | "pinned" : " 29 GB" | 20260505 04:01:47.499 | INFO | } | 20260505 04:01:47.500 | INFO | Perf report for the last 5.13661 seconds: | 20260505 04:01:47.500 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:01:47.500 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:47.500 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:47.500 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:47.500 | INFO | ANALYSIS : 58.4%, 3.0, 91.1ms, 0 | 20260505 04:01:47.500 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:47.500 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:47.500 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260505 04:01:47.500 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:47.500 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:47.500 | INFO | Uploaded 36GiB to GPU in 1.59693s (22.5432GiB/s) | 20260505 04:01:47.500 | INFO | Downloaded 34GiB to GPU in 1.51382s (22.4598GiB/s) | 20260505 04:01:52.638 | INFO | Analyzing chunk frames = [181248,181760) | 20260505 04:01:52.638 | INFO | MemInfo: { | 20260505 04:01:52.638 | INFO | "gpu" : " 17 GB", | 20260505 04:01:52.638 | INFO | "huge" : "", | 20260505 04:01:52.638 | INFO | "huge_shared" : " 27 GB", | 20260505 04:01:52.638 | INFO | "malloc" : " 14 GB", | 20260505 04:01:52.638 | INFO | "pinned" : " 29 GB" | 20260505 04:01:52.638 | INFO | } | 20260505 04:01:52.638 | INFO | Perf report for the last 5.13823 seconds: | 20260505 04:01:52.638 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:01:52.638 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:52.638 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:52.638 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:52.639 | INFO | ANALYSIS : 57.7%, 3.0, 89.6ms, 0 | 20260505 04:01:52.639 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:52.639 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:52.639 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 04:01:52.639 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:52.639 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:52.639 | INFO | Uploaded 36GiB to GPU in 1.5994s (22.5084GiB/s) | 20260505 04:01:52.639 | INFO | Downloaded 34GiB to GPU in 1.51728s (22.4086GiB/s) | 20260505 04:01:57.778 | INFO | Analyzing chunk frames = [181760,182272) | 20260505 04:01:57.778 | INFO | MemInfo: { | 20260505 04:01:57.778 | INFO | "gpu" : " 17 GB", | 20260505 04:01:57.778 | INFO | "huge" : "", | 20260505 04:01:57.778 | INFO | "huge_shared" : " 27 GB", | 20260505 04:01:57.778 | INFO | "malloc" : " 14 GB", | 20260505 04:01:57.778 | INFO | "pinned" : " 29 GB" | 20260505 04:01:57.778 | INFO | } | 20260505 04:01:57.778 | INFO | Perf report for the last 5.13929 seconds: | 20260505 04:01:57.778 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:01:57.778 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:57.778 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:57.778 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:57.778 | INFO | ANALYSIS : 57.4%, 3.0, 89.0ms, 0 | 20260505 04:01:57.778 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:57.778 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:57.778 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260505 04:01:57.778 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:57.778 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:01:57.778 | INFO | Uploaded 36GiB to GPU in 1.59932s (22.5096GiB/s) | 20260505 04:01:57.778 | INFO | Downloaded 34GiB to GPU in 1.51544s (22.4357GiB/s) | 20260505 04:02:02.917 | INFO | Analyzing chunk frames = [182272,182784) | 20260505 04:02:02.917 | INFO | MemInfo: { | 20260505 04:02:02.917 | INFO | "gpu" : " 17 GB", | 20260505 04:02:02.917 | INFO | "huge" : "", | 20260505 04:02:02.917 | INFO | "huge_shared" : " 27 GB", | 20260505 04:02:02.917 | INFO | "malloc" : " 14 GB", | 20260505 04:02:02.917 | INFO | "pinned" : " 29 GB" | 20260505 04:02:02.917 | INFO | } | 20260505 04:02:02.917 | INFO | Perf report for the last 5.13943 seconds: | 20260505 04:02:02.917 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:02:02.917 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:02.917 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:02.917 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:02.917 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 04:02:02.917 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:02.917 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:02.917 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260505 04:02:02.917 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:02.917 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:02.918 | INFO | Uploaded 36GiB to GPU in 1.5986s (22.5197GiB/s) | 20260505 04:02:02.918 | INFO | Downloaded 34GiB to GPU in 1.51519s (22.4395GiB/s) | 20260505 04:02:08.056 | INFO | Analyzing chunk frames = [182784,183296) | 20260505 04:02:08.056 | INFO | MemInfo: { | 20260505 04:02:08.056 | INFO | "gpu" : " 17 GB", | 20260505 04:02:08.056 | INFO | "huge" : "", | 20260505 04:02:08.056 | INFO | "huge_shared" : " 27 GB", | 20260505 04:02:08.056 | INFO | "malloc" : " 14 GB", | 20260505 04:02:08.056 | INFO | "pinned" : " 29 GB" | 20260505 04:02:08.056 | INFO | } | 20260505 04:02:08.056 | INFO | Perf report for the last 5.13926 seconds: | 20260505 04:02:08.056 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:02:08.056 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:08.056 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:08.056 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:08.056 | INFO | ANALYSIS : 57.6%, 3.0, 89.9ms, 0 | 20260505 04:02:08.057 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:08.057 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:08.057 | INFO | PRE_HQ : 32.6%, 1.0, 16.9ms, 0 | 20260505 04:02:08.057 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:08.057 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:08.057 | INFO | Uploaded 36GiB to GPU in 1.59808s (22.527GiB/s) | 20260505 04:02:08.057 | INFO | Downloaded 34GiB to GPU in 1.51452s (22.4494GiB/s) | 20260505 04:02:13.196 | INFO | Analyzing chunk frames = [183296,183808) | 20260505 04:02:13.196 | INFO | MemInfo: { | 20260505 04:02:13.196 | INFO | "gpu" : " 17 GB", | 20260505 04:02:13.196 | INFO | "huge" : "", | 20260505 04:02:13.196 | INFO | "huge_shared" : " 27 GB", | 20260505 04:02:13.196 | INFO | "malloc" : " 14 GB", | 20260505 04:02:13.196 | INFO | "pinned" : " 29 GB" | 20260505 04:02:13.196 | INFO | } | 20260505 04:02:13.196 | INFO | Perf report for the last 5.13925 seconds: | 20260505 04:02:13.196 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:02:13.196 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:13.196 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:13.196 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:13.196 | INFO | ANALYSIS : 57.8%, 3.0, 90.1ms, 0 | 20260505 04:02:13.196 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:13.196 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:13.196 | INFO | PRE_HQ : 32.8%, 1.0, 17.0ms, 0 | 20260505 04:02:13.196 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:13.196 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:13.196 | INFO | Uploaded 36GiB to GPU in 1.59866s (22.5189GiB/s) | 20260505 04:02:13.197 | INFO | Downloaded 34GiB to GPU in 1.51565s (22.4326GiB/s) | 20260505 04:02:18.336 | INFO | Analyzing chunk frames = [183808,184320) | 20260505 04:02:18.336 | INFO | MemInfo: { | 20260505 04:02:18.336 | INFO | "gpu" : " 17 GB", | 20260505 04:02:18.336 | INFO | "huge" : "", | 20260505 04:02:18.336 | INFO | "huge_shared" : " 27 GB", | 20260505 04:02:18.336 | INFO | "malloc" : " 14 GB", | 20260505 04:02:18.336 | INFO | "pinned" : " 29 GB" | 20260505 04:02:18.336 | INFO | } | 20260505 04:02:18.336 | INFO | Perf report for the last 5.14049 seconds: | 20260505 04:02:18.336 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:02:18.336 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:18.336 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:18.336 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:18.336 | INFO | ANALYSIS : 57.9%, 3.0, 90.4ms, 0 | 20260505 04:02:18.336 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:18.336 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:18.336 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260505 04:02:18.336 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:18.336 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:18.337 | INFO | Uploaded 36GiB to GPU in 1.59891s (22.5153GiB/s) | 20260505 04:02:18.337 | INFO | Downloaded 34GiB to GPU in 1.51693s (22.4137GiB/s) | 20260505 04:02:18.435 | INFO | Starting a round of Baz IO | 20260505 04:02:21.398 | INFO | Populating RTMetrics report | 20260505 04:02:21.399 | INFO | RTMetrics report populated | 20260505 04:02:21.399 | INFO | Opening tmp RTMetrics json file | 20260505 04:02:21.399 | INFO | Serializing RTMetrics to json | 20260505 04:02:21.401 | INFO | Serialization done, swapping tmp file now... | 20260505 04:02:21.402 | INFO | RTMetrics done | 20260505 04:02:21.472 | INFO | Finished a round of Baz IO: Wrote 1.696 GiB in 3.037 seconds | 20260505 04:02:21.472 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 56.24% / 0.30% / 43.45% | 20260505 04:02:21.472 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 13.4658% / 0.000961622% / 1.58946e-05% | 20260505 04:02:21.472 | INFO | Baz IO overall: | 20260505 04:02:21.472 | INFO | BytesWritten : 54402857074 | 20260505 04:02:21.472 | INFO | NumEvents : 24288743965 | 20260505 04:02:23.475 | INFO | Analyzing chunk frames = [184320,184832) | 20260505 04:02:23.475 | INFO | MemInfo: { | 20260505 04:02:23.475 | INFO | "gpu" : " 17 GB", | 20260505 04:02:23.475 | INFO | "huge" : "", | 20260505 04:02:23.475 | INFO | "huge_shared" : " 27 GB", | 20260505 04:02:23.475 | INFO | "malloc" : " 14 GB", | 20260505 04:02:23.475 | INFO | "pinned" : " 29 GB" | 20260505 04:02:23.475 | INFO | } | 20260505 04:02:23.475 | INFO | Perf report for the last 5.13902 seconds: | 20260505 04:02:23.475 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:02:23.475 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:23.475 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:23.475 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:23.475 | INFO | ANALYSIS : 58.5%, 3.0, 91.3ms, 0 | 20260505 04:02:23.475 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:23.476 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260505 04:02:23.476 | INFO | PRE_HQ : 58.0%, 2.2, 65.1ms, 0 | 20260505 04:02:23.476 | INFO | BAZWRITER : 58.5%, 3.3, 99.8ms, 0 | 20260505 04:02:23.476 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:02:23.476 | INFO | Uploaded 36GiB to GPU in 1.63917s (21.9624GiB/s) | 20260505 04:02:23.476 | INFO | Downloaded 38GiB to GPU in 1.69968s (22.3572GiB/s) | 20260505 04:02:28.614 | INFO | Analyzing chunk frames = [184832,185344) | 20260505 04:02:28.614 | INFO | MemInfo: { | 20260505 04:02:28.614 | INFO | "gpu" : " 17 GB", | 20260505 04:02:28.614 | INFO | "huge" : "", | 20260505 04:02:28.614 | INFO | "huge_shared" : " 27 GB", | 20260505 04:02:28.614 | INFO | "malloc" : " 14 GB", | 20260505 04:02:28.614 | INFO | "pinned" : " 29 GB" | 20260505 04:02:28.614 | INFO | } | 20260505 04:02:28.614 | INFO | Perf report for the last 5.13871 seconds: | 20260505 04:02:28.614 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:02:28.614 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:28.614 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:28.614 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:28.614 | INFO | ANALYSIS : 58.3%, 3.0, 91.0ms, 0 | 20260505 04:02:28.614 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:28.614 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:28.614 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 04:02:28.614 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:28.614 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:28.614 | INFO | Uploaded 36GiB to GPU in 1.59651s (22.5492GiB/s) | 20260505 04:02:28.614 | INFO | Downloaded 34GiB to GPU in 1.51308s (22.4708GiB/s) | 20260505 04:02:33.753 | INFO | Analyzing chunk frames = [185344,185856) | 20260505 04:02:33.753 | INFO | MemInfo: { | 20260505 04:02:33.753 | INFO | "gpu" : " 17 GB", | 20260505 04:02:33.753 | INFO | "huge" : "", | 20260505 04:02:33.753 | INFO | "huge_shared" : " 27 GB", | 20260505 04:02:33.753 | INFO | "malloc" : " 14 GB", | 20260505 04:02:33.753 | INFO | "pinned" : " 29 GB" | 20260505 04:02:33.753 | INFO | } | 20260505 04:02:33.753 | INFO | Perf report for the last 5.13926 seconds: | 20260505 04:02:33.753 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:02:33.753 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:33.753 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:33.753 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:33.753 | INFO | ANALYSIS : 57.7%, 3.0, 89.6ms, 0 | 20260505 04:02:33.754 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:33.754 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:33.754 | INFO | PRE_HQ : 32.1%, 1.0, 16.6ms, 0 | 20260505 04:02:33.754 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:33.754 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:33.754 | INFO | Uploaded 36GiB to GPU in 1.60039s (22.4945GiB/s) | 20260505 04:02:33.754 | INFO | Downloaded 34GiB to GPU in 1.5183s (22.3934GiB/s) | 20260505 04:02:38.892 | INFO | Analyzing chunk frames = [185856,186368) | 20260505 04:02:38.892 | INFO | MemInfo: { | 20260505 04:02:38.892 | INFO | "gpu" : " 17 GB", | 20260505 04:02:38.892 | INFO | "huge" : "", | 20260505 04:02:38.892 | INFO | "huge_shared" : " 27 GB", | 20260505 04:02:38.892 | INFO | "malloc" : " 14 GB", | 20260505 04:02:38.892 | INFO | "pinned" : " 29 GB" | 20260505 04:02:38.892 | INFO | } | 20260505 04:02:38.892 | INFO | Perf report for the last 5.13905 seconds: | 20260505 04:02:38.892 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:02:38.892 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:38.893 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:38.893 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:38.893 | INFO | ANALYSIS : 57.4%, 3.0, 89.0ms, 0 | 20260505 04:02:38.893 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:38.893 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:38.893 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260505 04:02:38.893 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:38.893 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:38.893 | INFO | Uploaded 36GiB to GPU in 1.59943s (22.508GiB/s) | 20260505 04:02:38.893 | INFO | Downloaded 34GiB to GPU in 1.51674s (22.4165GiB/s) | 20260505 04:02:44.032 | INFO | Analyzing chunk frames = [186368,186880) | 20260505 04:02:44.032 | INFO | MemInfo: { | 20260505 04:02:44.032 | INFO | "gpu" : " 17 GB", | 20260505 04:02:44.032 | INFO | "huge" : "", | 20260505 04:02:44.032 | INFO | "huge_shared" : " 27 GB", | 20260505 04:02:44.032 | INFO | "malloc" : " 14 GB", | 20260505 04:02:44.032 | INFO | "pinned" : " 29 GB" | 20260505 04:02:44.032 | INFO | } | 20260505 04:02:44.032 | INFO | Perf report for the last 5.13979 seconds: | 20260505 04:02:44.032 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:02:44.032 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:44.032 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:44.032 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:44.032 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 04:02:44.032 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:44.032 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:44.032 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260505 04:02:44.032 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:44.032 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:44.033 | INFO | Uploaded 36GiB to GPU in 1.59789s (22.5298GiB/s) | 20260505 04:02:44.033 | INFO | Downloaded 34GiB to GPU in 1.51373s (22.461GiB/s) | 20260505 04:02:49.171 | INFO | Analyzing chunk frames = [186880,187392) | 20260505 04:02:49.171 | INFO | MemInfo: { | 20260505 04:02:49.171 | INFO | "gpu" : " 17 GB", | 20260505 04:02:49.171 | INFO | "huge" : "", | 20260505 04:02:49.171 | INFO | "huge_shared" : " 27 GB", | 20260505 04:02:49.171 | INFO | "malloc" : " 14 GB", | 20260505 04:02:49.171 | INFO | "pinned" : " 29 GB" | 20260505 04:02:49.171 | INFO | } | 20260505 04:02:49.171 | INFO | Perf report for the last 5.13916 seconds: | 20260505 04:02:49.171 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:02:49.171 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:49.171 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:49.171 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:49.171 | INFO | ANALYSIS : 57.6%, 3.0, 89.9ms, 0 | 20260505 04:02:49.171 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:49.171 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:49.171 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260505 04:02:49.171 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:49.171 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:49.172 | INFO | Uploaded 36GiB to GPU in 1.59808s (22.527GiB/s) | 20260505 04:02:49.172 | INFO | Downloaded 34GiB to GPU in 1.51478s (22.4455GiB/s) | 20260505 04:02:54.310 | INFO | Analyzing chunk frames = [187392,187904) | 20260505 04:02:54.310 | INFO | MemInfo: { | 20260505 04:02:54.310 | INFO | "gpu" : " 17 GB", | 20260505 04:02:54.311 | INFO | "huge" : "", | 20260505 04:02:54.311 | INFO | "huge_shared" : " 27 GB", | 20260505 04:02:54.311 | INFO | "malloc" : " 14 GB", | 20260505 04:02:54.311 | INFO | "pinned" : " 29 GB" | 20260505 04:02:54.311 | INFO | } | 20260505 04:02:54.311 | INFO | Perf report for the last 5.13924 seconds: | 20260505 04:02:54.311 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:02:54.311 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:54.311 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:54.311 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:54.311 | INFO | ANALYSIS : 57.8%, 3.0, 90.1ms, 0 | 20260505 04:02:54.311 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:54.311 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:54.311 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260505 04:02:54.311 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:54.311 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:54.311 | INFO | Uploaded 36GiB to GPU in 1.59891s (22.5153GiB/s) | 20260505 04:02:54.311 | INFO | Downloaded 34GiB to GPU in 1.51611s (22.4258GiB/s) | 20260505 04:02:59.450 | INFO | Analyzing chunk frames = [187904,188416) | 20260505 04:02:59.450 | INFO | MemInfo: { | 20260505 04:02:59.450 | INFO | "gpu" : " 17 GB", | 20260505 04:02:59.450 | INFO | "huge" : "", | 20260505 04:02:59.450 | INFO | "huge_shared" : " 27 GB", | 20260505 04:02:59.450 | INFO | "malloc" : " 14 GB", | 20260505 04:02:59.450 | INFO | "pinned" : " 29 GB" | 20260505 04:02:59.450 | INFO | } | 20260505 04:02:59.450 | INFO | Perf report for the last 5.13985 seconds: | 20260505 04:02:59.450 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:02:59.450 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:59.450 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:59.450 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:59.450 | INFO | ANALYSIS : 57.9%, 3.0, 90.4ms, 0 | 20260505 04:02:59.450 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:59.451 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:59.451 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 04:02:59.451 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:59.451 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:02:59.451 | INFO | Uploaded 36GiB to GPU in 1.59827s (22.5244GiB/s) | 20260505 04:02:59.451 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 04:02:59.451 | INFO | Total mem usage: 14.9 GiB | 20260505 04:02:59.451 | INFO | All Time High Water Marks sum: 15.2 GiB (Savings: 1.52%) | 20260505 04:02:59.451 | INFO | Downloaded 34GiB to GPU in 1.51566s (22.4324GiB/s) | 20260505 04:02:59.451 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -3.05%) | 20260505 04:02:59.451 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 04:02:59.451 | INFO | High Water Breakdown: | 20260505 04:02:59.451 | INFO | 58.3% : 8.45 GiB ( 99.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 04:02:59.451 | INFO | 41.5% : 6.01 GiB ( 90.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 04:02:59.451 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 04:02:59.451 | INFO | ------------------------------------------ | 20260505 04:02:59.451 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:02:59.451 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 04:02:59.451 | INFO | Total mem usage: 26.5 GiB | 20260505 04:02:59.451 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260505 04:02:59.451 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 04:02:59.451 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260505 04:02:59.451 | INFO | High Water Breakdown: | 20260505 04:02:59.451 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:02:59.451 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:02:59.451 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:02:59.451 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:02:59.451 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:02:59.451 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:02:59.451 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 04:02:59.451 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 04:02:59.451 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:02:59.451 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 04:02:59.451 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 04:02:59.451 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 04:02:59.451 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 04:02:59.451 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 04:02:59.451 | INFO | ------------------------------------------ | 20260505 04:02:59.452 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:02:59.452 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 04:02:59.452 | INFO | Total mem usage: 25.6 GiB | 20260505 04:02:59.452 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 04:02:59.452 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 04:02:59.452 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 04:02:59.452 | INFO | High Water Breakdown: | 20260505 04:02:59.452 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 04:02:59.452 | INFO | ------------------------------------------ | 20260505 04:02:59.452 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:02:59.452 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 04:02:59.452 | INFO | Total mem usage: 15.7 GiB | 20260505 04:02:59.452 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260505 04:02:59.452 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 04:02:59.452 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260505 04:02:59.452 | INFO | High Water Breakdown: | 20260505 04:02:59.452 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 04:02:59.452 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 04:02:59.452 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 04:02:59.452 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 04:02:59.452 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 04:02:59.452 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 04:02:59.452 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 04:02:59.452 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 04:02:59.452 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 04:02:59.452 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:02:59.452 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 04:02:59.452 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:02:59.452 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 04:02:59.452 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 04:02:59.452 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 04:02:59.452 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 04:02:59.452 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:02:59.452 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 04:02:59.452 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:02:59.452 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:02:59.452 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:02:59.452 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 04:02:59.452 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 04:02:59.452 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:02:59.452 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 04:02:59.452 | INFO | ------------------------------------------ | 20260505 04:02:59.452 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:02:59.452 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 04:02:59.452 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:02:59.452 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:02:59.452 | INFO | PerfReport: Full Execution -- 3.06e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260505 04:02:59.452 | INFO | PerfReport: 0.28%, AggregateStats -- 8.64e-02 / -nan, 5.71e-03 / 0.00e+00 | 20260505 04:02:59.452 | INFO | PerfReport: 10.76%, Baseline -- 3.29e+00 / -nan, 1.69e-02 / 0.00e+00 | 20260505 04:02:59.452 | INFO | PerfReport: 3.02%, Histogram -- 9.24e-01 / -nan, 5.12e-02 / 0.00e+00 | 20260505 04:02:59.452 | INFO | PerfReport: 16.01%, DME -- 3.92e+01 / -nan, 3.26e+00 / 0.00e+00 | 20260505 04:02:59.452 | INFO | PerfReport: 56.23%, FrameLabeling -- 1.72e+01 / -nan, 4.43e-02 / 0.00e+00 | 20260505 04:02:59.452 | INFO | PerfReport: 7.77%, PulseAccumulating -- 2.38e+00 / -nan, 1.61e-01 / 0.00e+00 | 20260505 04:02:59.452 | INFO | PerfReport: 4.17%, Metrics -- 1.28e+00 / -nan, 2.08e-01 / 0.00e+00 | 20260505 04:02:59.452 | INFO | PerfReport: 1.74%, PulseSerializing -- 5.33e-01 / -nan, 8.56e-02 / 0.00e+00 | 20260505 04:02:59.452 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 04:02:59.452 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:02:59.452 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:02:59.452 | INFO | PerfReport: Full Execution -- 9.20e+01 / -nan, 1.91e+01 / 0.00e+00 | 20260505 04:02:59.452 | INFO | PerfReport: 18.16%, Upload -- 1.67e+01 / 0.00e+00, 3.64e-01 / 0.00e+00 | 20260505 04:02:59.452 | INFO | PerfReport: 17.42%, Download -- 1.60e+01 / 0.00e+00, 6.88e-01 / 0.00e+00 | 20260505 04:02:59.452 | INFO | PerfReport: 33.29%, Compute -- 3.06e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260505 04:02:59.452 | INFO | PerfReport: 0.55%, LockUpload -- 5.04e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260505 04:02:59.452 | INFO | PerfReport: 0.00%, LockDownload -- 4.00e-04 / 0.00e+00, 2.65e-04 / 0.00e+00 | 20260505 04:02:59.452 | INFO | PerfReport: 30.56%, LockCompute -- 2.81e+01 / 0.00e+00, 1.72e+01 / 0.00e+00 | 20260505 04:02:59.452 | INFO | PerfReport: 0.02%, Misc -- 1.51e-02 / 0.00e+00, 4.47e-02 / 0.00e+00 | 20260505 04:02:59.547 | INFO | Starting a round of Baz IO | 20260505 04:03:02.515 | INFO | Populating RTMetrics report | 20260505 04:03:02.515 | INFO | RTMetrics report populated | 20260505 04:03:02.516 | INFO | Opening tmp RTMetrics json file | 20260505 04:03:02.516 | INFO | Serializing RTMetrics to json | 20260505 04:03:02.518 | INFO | Serialization done, swapping tmp file now... | 20260505 04:03:02.518 | INFO | RTMetrics done | 20260505 04:03:02.589 | INFO | Finished a round of Baz IO: Wrote 1.706 GiB in 3.041 seconds | 20260505 04:03:02.589 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 55.95% / 0.30% / 43.76% | 20260505 04:03:02.589 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 13.4062% / 0.000870228% / 1.98682e-05% | 20260505 04:03:02.589 | INFO | Baz IO overall: | 20260505 04:03:02.589 | INFO | BytesWritten : 56234406880 | 20260505 04:03:02.589 | INFO | NumEvents : 25152406078 | 20260505 04:03:04.589 | INFO | Analyzing chunk frames = [188416,188928) | 20260505 04:03:04.589 | INFO | MemInfo: { | 20260505 04:03:04.589 | INFO | "gpu" : " 17 GB", | 20260505 04:03:04.589 | INFO | "huge" : "", | 20260505 04:03:04.589 | INFO | "huge_shared" : " 27 GB", | 20260505 04:03:04.589 | INFO | "malloc" : " 14 GB", | 20260505 04:03:04.589 | INFO | "pinned" : " 29 GB" | 20260505 04:03:04.589 | INFO | } | 20260505 04:03:04.589 | INFO | Perf report for the last 5.13877 seconds: | 20260505 04:03:04.589 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:03:04.589 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:04.589 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:04.589 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:04.589 | INFO | ANALYSIS : 58.6%, 3.0, 91.4ms, 0 | 20260505 04:03:04.589 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:04.589 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260505 04:03:04.589 | INFO | PRE_HQ : 58.1%, 2.2, 64.8ms, 0 | 20260505 04:03:04.589 | INFO | BAZWRITER : 58.8%, 3.3, 100.6ms, 0 | 20260505 04:03:04.589 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:03:04.590 | INFO | Uploaded 36GiB to GPU in 1.63451s (22.025GiB/s) | 20260505 04:03:04.591 | INFO | Downloaded 38GiB to GPU in 1.69254s (22.4515GiB/s) | 20260505 04:03:09.728 | INFO | Analyzing chunk frames = [188928,189440) | 20260505 04:03:09.728 | INFO | MemInfo: { | 20260505 04:03:09.728 | INFO | "gpu" : " 17 GB", | 20260505 04:03:09.729 | INFO | "huge" : "", | 20260505 04:03:09.729 | INFO | "huge_shared" : " 27 GB", | 20260505 04:03:09.729 | INFO | "malloc" : " 14 GB", | 20260505 04:03:09.729 | INFO | "pinned" : " 29 GB" | 20260505 04:03:09.729 | INFO | } | 20260505 04:03:09.729 | INFO | Perf report for the last 5.13949 seconds: | 20260505 04:03:09.729 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:03:09.729 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:09.729 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:09.729 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:09.729 | INFO | ANALYSIS : 58.3%, 3.0, 91.0ms, 0 | 20260505 04:03:09.729 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:09.729 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:09.729 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 04:03:09.729 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:09.729 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:09.729 | INFO | Uploaded 36GiB to GPU in 1.59695s (22.5429GiB/s) | 20260505 04:03:09.729 | INFO | Downloaded 34GiB to GPU in 1.51296s (22.4725GiB/s) | 20260505 04:03:14.868 | INFO | Analyzing chunk frames = [189440,189952) | 20260505 04:03:14.868 | INFO | MemInfo: { | 20260505 04:03:14.868 | INFO | "gpu" : " 17 GB", | 20260505 04:03:14.868 | INFO | "huge" : "", | 20260505 04:03:14.868 | INFO | "huge_shared" : " 27 GB", | 20260505 04:03:14.868 | INFO | "malloc" : " 14 GB", | 20260505 04:03:14.868 | INFO | "pinned" : " 29 GB" | 20260505 04:03:14.868 | INFO | } | 20260505 04:03:14.868 | INFO | Perf report for the last 5.13912 seconds: | 20260505 04:03:14.868 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:03:14.868 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:14.868 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:14.868 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:14.868 | INFO | ANALYSIS : 57.7%, 3.0, 89.7ms, 0 | 20260505 04:03:14.868 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:14.868 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:14.868 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260505 04:03:14.868 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:14.868 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:14.868 | INFO | Uploaded 36GiB to GPU in 1.60027s (22.4962GiB/s) | 20260505 04:03:14.868 | INFO | Downloaded 34GiB to GPU in 1.51725s (22.4089GiB/s) | 20260505 04:03:20.007 | INFO | Analyzing chunk frames = [189952,190464) | 20260505 04:03:20.007 | INFO | MemInfo: { | 20260505 04:03:20.007 | INFO | "gpu" : " 17 GB", | 20260505 04:03:20.007 | INFO | "huge" : "", | 20260505 04:03:20.007 | INFO | "huge_shared" : " 27 GB", | 20260505 04:03:20.007 | INFO | "malloc" : " 14 GB", | 20260505 04:03:20.007 | INFO | "pinned" : " 29 GB" | 20260505 04:03:20.007 | INFO | } | 20260505 04:03:20.007 | INFO | Perf report for the last 5.13938 seconds: | 20260505 04:03:20.007 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:03:20.007 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:20.007 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:20.007 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:20.007 | INFO | ANALYSIS : 57.3%, 3.0, 88.9ms, 0 | 20260505 04:03:20.007 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:20.007 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:20.007 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 04:03:20.008 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:20.008 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:20.008 | INFO | Uploaded 36GiB to GPU in 1.59987s (22.5019GiB/s) | 20260505 04:03:20.008 | INFO | Downloaded 34GiB to GPU in 1.51683s (22.4152GiB/s) | 20260505 04:03:25.146 | INFO | Analyzing chunk frames = [190464,190976) | 20260505 04:03:25.146 | INFO | MemInfo: { | 20260505 04:03:25.146 | INFO | "gpu" : " 17 GB", | 20260505 04:03:25.146 | INFO | "huge" : "", | 20260505 04:03:25.146 | INFO | "huge_shared" : " 27 GB", | 20260505 04:03:25.146 | INFO | "malloc" : " 14 GB", | 20260505 04:03:25.146 | INFO | "pinned" : " 29 GB" | 20260505 04:03:25.146 | INFO | } | 20260505 04:03:25.147 | INFO | Perf report for the last 5.13945 seconds: | 20260505 04:03:25.147 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:03:25.147 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:25.147 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:25.147 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:25.147 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 04:03:25.147 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:25.147 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:25.147 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260505 04:03:25.147 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:25.147 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:25.147 | INFO | Uploaded 36GiB to GPU in 1.59785s (22.5302GiB/s) | 20260505 04:03:25.147 | INFO | Downloaded 34GiB to GPU in 1.51441s (22.451GiB/s) | 20260505 04:03:30.286 | INFO | Analyzing chunk frames = [190976,191488) | 20260505 04:03:30.286 | INFO | MemInfo: { | 20260505 04:03:30.286 | INFO | "gpu" : " 17 GB", | 20260505 04:03:30.286 | INFO | "huge" : "", | 20260505 04:03:30.286 | INFO | "huge_shared" : " 27 GB", | 20260505 04:03:30.286 | INFO | "malloc" : " 14 GB", | 20260505 04:03:30.286 | INFO | "pinned" : " 29 GB" | 20260505 04:03:30.286 | INFO | } | 20260505 04:03:30.286 | INFO | Perf report for the last 5.1393 seconds: | 20260505 04:03:30.286 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:03:30.286 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:30.286 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:30.286 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:30.286 | INFO | ANALYSIS : 57.6%, 3.0, 89.8ms, 0 | 20260505 04:03:30.286 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:30.286 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:30.286 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260505 04:03:30.286 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:30.286 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:30.287 | INFO | Uploaded 36GiB to GPU in 1.59826s (22.5245GiB/s) | 20260505 04:03:30.287 | INFO | Downloaded 34GiB to GPU in 1.51509s (22.4409GiB/s) | 20260505 04:03:35.425 | INFO | Analyzing chunk frames = [191488,192000) | 20260505 04:03:35.425 | INFO | MemInfo: { | 20260505 04:03:35.425 | INFO | "gpu" : " 17 GB", | 20260505 04:03:35.425 | INFO | "huge" : "", | 20260505 04:03:35.425 | INFO | "huge_shared" : " 27 GB", | 20260505 04:03:35.425 | INFO | "malloc" : " 14 GB", | 20260505 04:03:35.425 | INFO | "pinned" : " 29 GB" | 20260505 04:03:35.425 | INFO | } | 20260505 04:03:35.425 | INFO | Perf report for the last 5.13932 seconds: | 20260505 04:03:35.425 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:03:35.426 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:35.426 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:35.426 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:35.426 | INFO | ANALYSIS : 57.8%, 3.0, 90.2ms, 0 | 20260505 04:03:35.426 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:35.426 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:35.426 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 04:03:35.426 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:35.426 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:35.426 | INFO | Uploaded 36GiB to GPU in 1.59867s (22.5188GiB/s) | 20260505 04:03:35.426 | INFO | Downloaded 34GiB to GPU in 1.51602s (22.4271GiB/s) | 20260505 04:03:40.565 | INFO | Analyzing chunk frames = [192000,192512) | 20260505 04:03:40.565 | INFO | MemInfo: { | 20260505 04:03:40.565 | INFO | "gpu" : " 17 GB", | 20260505 04:03:40.565 | INFO | "huge" : "", | 20260505 04:03:40.565 | INFO | "huge_shared" : " 27 GB", | 20260505 04:03:40.565 | INFO | "malloc" : " 14 GB", | 20260505 04:03:40.565 | INFO | "pinned" : " 29 GB" | 20260505 04:03:40.565 | INFO | } | 20260505 04:03:40.565 | INFO | Perf report for the last 5.13942 seconds: | 20260505 04:03:40.565 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:03:40.565 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:40.565 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:40.565 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:40.565 | INFO | ANALYSIS : 57.9%, 3.0, 90.3ms, 0 | 20260505 04:03:40.565 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:40.565 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:40.565 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 04:03:40.565 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:40.565 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:40.565 | INFO | Uploaded 36GiB to GPU in 1.59906s (22.5132GiB/s) | 20260505 04:03:40.565 | INFO | Downloaded 34GiB to GPU in 1.51642s (22.4212GiB/s) | 20260505 04:03:40.662 | INFO | Starting a round of Baz IO | 20260505 04:03:43.628 | INFO | Populating RTMetrics report | 20260505 04:03:43.629 | INFO | RTMetrics report populated | 20260505 04:03:43.629 | INFO | Opening tmp RTMetrics json file | 20260505 04:03:43.629 | INFO | Serializing RTMetrics to json | 20260505 04:03:43.631 | INFO | Serialization done, swapping tmp file now... | 20260505 04:03:43.631 | INFO | RTMetrics done | 20260505 04:03:43.703 | INFO | Finished a round of Baz IO: Wrote 1.715 GiB in 3.041 seconds | 20260505 04:03:43.703 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 55.66% / 0.29% / 44.05% | 20260505 04:03:43.703 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 13.3565% / 0.000782808% / 3.57628e-05% | 20260505 04:03:43.703 | INFO | Baz IO overall: | 20260505 04:03:43.703 | INFO | BytesWritten : 58076151506 | 20260505 04:03:43.703 | INFO | NumEvents : 26021200798 | 20260505 04:03:45.704 | INFO | Analyzing chunk frames = [192512,193024) | 20260505 04:03:45.704 | INFO | MemInfo: { | 20260505 04:03:45.704 | INFO | "gpu" : " 17 GB", | 20260505 04:03:45.704 | INFO | "huge" : "", | 20260505 04:03:45.704 | INFO | "huge_shared" : " 27 GB", | 20260505 04:03:45.704 | INFO | "malloc" : " 14 GB", | 20260505 04:03:45.704 | INFO | "pinned" : " 29 GB" | 20260505 04:03:45.704 | INFO | } | 20260505 04:03:45.704 | INFO | Perf report for the last 5.13923 seconds: | 20260505 04:03:45.704 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:03:45.704 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:45.704 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:45.704 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:45.704 | INFO | ANALYSIS : 58.5%, 3.0, 91.3ms, 0 | 20260505 04:03:45.704 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:45.704 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260505 04:03:45.704 | INFO | PRE_HQ : 57.9%, 2.2, 64.7ms, 0 | 20260505 04:03:45.704 | INFO | BAZWRITER : 58.7%, 3.3, 100.4ms, 0 | 20260505 04:03:45.704 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:03:45.704 | INFO | Uploaded 36GiB to GPU in 1.63818s (21.9756GiB/s) | 20260505 04:03:45.705 | INFO | Downloaded 38GiB to GPU in 1.69796s (22.3798GiB/s) | 20260505 04:03:50.843 | INFO | Analyzing chunk frames = [193024,193536) | 20260505 04:03:50.843 | INFO | MemInfo: { | 20260505 04:03:50.843 | INFO | "gpu" : " 17 GB", | 20260505 04:03:50.843 | INFO | "huge" : "", | 20260505 04:03:50.843 | INFO | "huge_shared" : " 27 GB", | 20260505 04:03:50.843 | INFO | "malloc" : " 14 GB", | 20260505 04:03:50.843 | INFO | "pinned" : " 29 GB" | 20260505 04:03:50.843 | INFO | } | 20260505 04:03:50.843 | INFO | Perf report for the last 5.13927 seconds: | 20260505 04:03:50.843 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:03:50.843 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:50.843 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:50.843 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:50.843 | INFO | ANALYSIS : 58.3%, 3.0, 90.9ms, 0 | 20260505 04:03:50.844 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:50.844 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:50.844 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260505 04:03:50.844 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:50.844 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:50.844 | INFO | Uploaded 36GiB to GPU in 1.59668s (22.5468GiB/s) | 20260505 04:03:50.844 | INFO | Downloaded 34GiB to GPU in 1.51292s (22.4731GiB/s) | 20260505 04:03:55.983 | INFO | Analyzing chunk frames = [193536,194048) | 20260505 04:03:55.983 | INFO | MemInfo: { | 20260505 04:03:55.983 | INFO | "gpu" : " 17 GB", | 20260505 04:03:55.983 | INFO | "huge" : "", | 20260505 04:03:55.983 | INFO | "huge_shared" : " 27 GB", | 20260505 04:03:55.983 | INFO | "malloc" : " 14 GB", | 20260505 04:03:55.983 | INFO | "pinned" : " 29 GB" | 20260505 04:03:55.983 | INFO | } | 20260505 04:03:55.983 | INFO | Perf report for the last 5.13941 seconds: | 20260505 04:03:55.983 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:03:55.983 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:55.983 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:55.983 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:55.983 | INFO | ANALYSIS : 57.7%, 3.0, 89.7ms, 0 | 20260505 04:03:55.983 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:55.983 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:55.983 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260505 04:03:55.983 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:55.983 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:03:55.983 | INFO | Uploaded 36GiB to GPU in 1.60033s (22.4954GiB/s) | 20260505 04:03:55.983 | INFO | Downloaded 34GiB to GPU in 1.5168s (22.4156GiB/s) | 20260505 04:04:01.122 | INFO | Analyzing chunk frames = [194048,194560) | 20260505 04:04:01.122 | INFO | MemInfo: { | 20260505 04:04:01.122 | INFO | "gpu" : " 17 GB", | 20260505 04:04:01.122 | INFO | "huge" : "", | 20260505 04:04:01.122 | INFO | "huge_shared" : " 27 GB", | 20260505 04:04:01.122 | INFO | "malloc" : " 14 GB", | 20260505 04:04:01.122 | INFO | "pinned" : " 29 GB" | 20260505 04:04:01.122 | INFO | } | 20260505 04:04:01.122 | INFO | Perf report for the last 5.13951 seconds: | 20260505 04:04:01.122 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:04:01.122 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:01.122 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:01.122 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:01.122 | INFO | ANALYSIS : 57.3%, 3.0, 89.0ms, 0 | 20260505 04:04:01.122 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:01.122 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:01.122 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260505 04:04:01.122 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:01.122 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:01.123 | INFO | Uploaded 36GiB to GPU in 1.60038s (22.4947GiB/s) | 20260505 04:04:01.123 | INFO | Downloaded 34GiB to GPU in 1.51749s (22.4054GiB/s) | 20260505 04:04:06.261 | INFO | Analyzing chunk frames = [194560,195072) | 20260505 04:04:06.261 | INFO | MemInfo: { | 20260505 04:04:06.261 | INFO | "gpu" : " 17 GB", | 20260505 04:04:06.261 | INFO | "huge" : "", | 20260505 04:04:06.261 | INFO | "huge_shared" : " 27 GB", | 20260505 04:04:06.261 | INFO | "malloc" : " 14 GB", | 20260505 04:04:06.261 | INFO | "pinned" : " 29 GB" | 20260505 04:04:06.261 | INFO | } | 20260505 04:04:06.261 | INFO | Perf report for the last 5.13914 seconds: | 20260505 04:04:06.262 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:04:06.262 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:06.262 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:06.262 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:06.262 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 04:04:06.262 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:06.262 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:06.262 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260505 04:04:06.262 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:06.262 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:06.262 | INFO | Uploaded 36GiB to GPU in 1.59738s (22.5369GiB/s) | 20260505 04:04:06.262 | INFO | Downloaded 34GiB to GPU in 1.51391s (22.4584GiB/s) | 20260505 04:04:11.401 | INFO | Analyzing chunk frames = [195072,195584) | 20260505 04:04:11.401 | INFO | MemInfo: { | 20260505 04:04:11.401 | INFO | "gpu" : " 17 GB", | 20260505 04:04:11.401 | INFO | "huge" : "", | 20260505 04:04:11.401 | INFO | "huge_shared" : " 27 GB", | 20260505 04:04:11.401 | INFO | "malloc" : " 14 GB", | 20260505 04:04:11.401 | INFO | "pinned" : " 29 GB" | 20260505 04:04:11.401 | INFO | } | 20260505 04:04:11.401 | INFO | Perf report for the last 5.13916 seconds: | 20260505 04:04:11.401 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:04:11.401 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:11.401 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:11.401 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:11.401 | INFO | ANALYSIS : 57.6%, 3.0, 89.8ms, 0 | 20260505 04:04:11.401 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:11.401 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:11.401 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260505 04:04:11.401 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:11.401 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:11.401 | INFO | Uploaded 36GiB to GPU in 1.59809s (22.5269GiB/s) | 20260505 04:04:11.401 | INFO | Downloaded 34GiB to GPU in 1.51454s (22.449GiB/s) | 20260505 04:04:16.540 | INFO | Analyzing chunk frames = [195584,196096) | 20260505 04:04:16.540 | INFO | MemInfo: { | 20260505 04:04:16.540 | INFO | "gpu" : " 17 GB", | 20260505 04:04:16.540 | INFO | "huge" : "", | 20260505 04:04:16.540 | INFO | "huge_shared" : " 27 GB", | 20260505 04:04:16.540 | INFO | "malloc" : " 14 GB", | 20260505 04:04:16.540 | INFO | "pinned" : " 29 GB" | 20260505 04:04:16.540 | INFO | } | 20260505 04:04:16.540 | INFO | Perf report for the last 5.13947 seconds: | 20260505 04:04:16.540 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:04:16.540 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:16.540 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:16.540 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:16.540 | INFO | ANALYSIS : 57.8%, 3.0, 90.2ms, 0 | 20260505 04:04:16.540 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:16.540 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:16.540 | INFO | PRE_HQ : 32.6%, 1.0, 16.9ms, 0 | 20260505 04:04:16.540 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:16.540 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:16.541 | INFO | Uploaded 36GiB to GPU in 1.59873s (22.5178GiB/s) | 20260505 04:04:16.541 | INFO | Downloaded 34GiB to GPU in 1.51562s (22.4331GiB/s) | 20260505 04:04:21.679 | INFO | Analyzing chunk frames = [196096,196608) | 20260505 04:04:21.679 | INFO | MemInfo: { | 20260505 04:04:21.679 | INFO | "gpu" : " 17 GB", | 20260505 04:04:21.679 | INFO | "huge" : "", | 20260505 04:04:21.679 | INFO | "huge_shared" : " 27 GB", | 20260505 04:04:21.679 | INFO | "malloc" : " 14 GB", | 20260505 04:04:21.679 | INFO | "pinned" : " 29 GB" | 20260505 04:04:21.679 | INFO | } | 20260505 04:04:21.679 | INFO | Perf report for the last 5.13933 seconds: | 20260505 04:04:21.680 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:04:21.680 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:21.680 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:21.680 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:21.680 | INFO | ANALYSIS : 57.9%, 3.0, 90.3ms, 0 | 20260505 04:04:21.680 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:21.680 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:21.680 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 04:04:21.680 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:21.680 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:21.680 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 04:04:21.680 | INFO | Total mem usage: 14.9 GiB | 20260505 04:04:21.680 | INFO | All Time High Water Marks sum: 15.2 GiB (Savings: 1.65%) | 20260505 04:04:21.680 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -2.53%) | 20260505 04:04:21.680 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 04:04:21.680 | INFO | High Water Breakdown: | 20260505 04:04:21.680 | INFO | 58.5% : 8.51 GiB ( 100% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 04:04:21.680 | INFO | 41.4% : 6.03 GiB ( 90.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 04:04:21.680 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 04:04:21.680 | INFO | ------------------------------------------ | 20260505 04:04:21.680 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:04:21.680 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 04:04:21.680 | INFO | Total mem usage: 26.5 GiB | 20260505 04:04:21.680 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260505 04:04:21.680 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 04:04:21.680 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260505 04:04:21.680 | INFO | High Water Breakdown: | 20260505 04:04:21.680 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:04:21.680 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:04:21.680 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:04:21.680 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:04:21.680 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:04:21.680 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:04:21.680 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 04:04:21.680 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 04:04:21.680 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:04:21.680 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 04:04:21.680 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 04:04:21.680 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 04:04:21.680 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 04:04:21.680 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 04:04:21.680 | INFO | Uploaded 36GiB to GPU in 1.59892s (22.5152GiB/s) | 20260505 04:04:21.681 | INFO | ------------------------------------------ | 20260505 04:04:21.681 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:04:21.681 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 04:04:21.681 | INFO | Total mem usage: 25.6 GiB | 20260505 04:04:21.681 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 04:04:21.681 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 04:04:21.681 | INFO | Downloaded 34GiB to GPU in 1.51551s (22.4347GiB/s) | 20260505 04:04:21.681 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 04:04:21.681 | INFO | High Water Breakdown: | 20260505 04:04:21.681 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 04:04:21.681 | INFO | ------------------------------------------ | 20260505 04:04:21.681 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:04:21.681 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 04:04:21.681 | INFO | Total mem usage: 15.7 GiB | 20260505 04:04:21.681 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260505 04:04:21.681 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 04:04:21.681 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260505 04:04:21.681 | INFO | High Water Breakdown: | 20260505 04:04:21.681 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 04:04:21.681 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 04:04:21.681 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 04:04:21.681 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 04:04:21.681 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 04:04:21.681 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 04:04:21.681 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 04:04:21.681 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 04:04:21.681 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 04:04:21.681 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:04:21.681 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 04:04:21.681 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:04:21.681 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 04:04:21.681 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 04:04:21.681 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 04:04:21.681 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 04:04:21.681 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:04:21.681 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 04:04:21.681 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:04:21.681 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:04:21.681 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:04:21.681 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 04:04:21.681 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 04:04:21.681 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:04:21.681 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 04:04:21.681 | INFO | ------------------------------------------ | 20260505 04:04:21.681 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:04:21.681 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 04:04:21.681 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:04:21.681 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:04:21.681 | INFO | PerfReport: Full Execution -- 3.06e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260505 04:04:21.681 | INFO | PerfReport: 0.28%, AggregateStats -- 8.61e-02 / -nan, 3.83e-03 / 0.00e+00 | 20260505 04:04:21.681 | INFO | PerfReport: 10.77%, Baseline -- 3.30e+00 / -nan, 1.73e-02 / 0.00e+00 | 20260505 04:04:21.681 | INFO | PerfReport: 3.02%, Histogram -- 9.24e-01 / -nan, 5.13e-02 / 0.00e+00 | 20260505 04:04:21.681 | INFO | PerfReport: 16.01%, DME -- 3.92e+01 / -nan, 3.24e+00 / 0.00e+00 | 20260505 04:04:21.681 | INFO | PerfReport: 56.25%, FrameLabeling -- 1.72e+01 / -nan, 4.45e-02 / 0.00e+00 | 20260505 04:04:21.681 | INFO | PerfReport: 7.76%, PulseAccumulating -- 2.38e+00 / -nan, 1.61e-01 / 0.00e+00 | 20260505 04:04:21.681 | INFO | PerfReport: 4.17%, Metrics -- 1.28e+00 / -nan, 2.08e-01 / 0.00e+00 | 20260505 04:04:21.681 | INFO | PerfReport: 1.74%, PulseSerializing -- 5.32e-01 / -nan, 8.50e-02 / 0.00e+00 | 20260505 04:04:21.681 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 04:04:21.681 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:04:21.681 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:04:21.681 | INFO | PerfReport: Full Execution -- 9.20e+01 / -nan, 1.92e+01 / 0.00e+00 | 20260505 04:04:21.681 | INFO | PerfReport: 18.16%, Upload -- 1.67e+01 / 0.00e+00, 3.64e-01 / 0.00e+00 | 20260505 04:04:21.681 | INFO | PerfReport: 17.42%, Download -- 1.60e+01 / 0.00e+00, 6.79e-01 / 0.00e+00 | 20260505 04:04:21.681 | INFO | PerfReport: 33.29%, Compute -- 3.06e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260505 04:04:21.681 | INFO | PerfReport: 0.55%, LockUpload -- 5.04e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260505 04:04:21.681 | INFO | PerfReport: 0.00%, LockDownload -- 3.96e-04 / 0.00e+00, 1.33e-04 / 0.00e+00 | 20260505 04:04:21.681 | INFO | PerfReport: 30.56%, LockCompute -- 2.81e+01 / 0.00e+00, 1.73e+01 / 0.00e+00 | 20260505 04:04:21.681 | INFO | PerfReport: 0.02%, Misc -- 1.46e-02 / 0.00e+00, 4.25e-02 / 0.00e+00 | 20260505 04:04:21.778 | INFO | Starting a round of Baz IO | 20260505 04:04:24.745 | INFO | Populating RTMetrics report | 20260505 04:04:24.745 | INFO | RTMetrics report populated | 20260505 04:04:24.746 | INFO | Opening tmp RTMetrics json file | 20260505 04:04:24.747 | INFO | Serializing RTMetrics to json | 20260505 04:04:24.748 | INFO | Serialization done, swapping tmp file now... | 20260505 04:04:24.748 | INFO | RTMetrics done | 20260505 04:04:24.822 | INFO | Finished a round of Baz IO: Wrote 1.726 GiB in 3.043 seconds | 20260505 04:04:24.822 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 55.38% / 0.28% / 44.34% | 20260505 04:04:24.822 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 13.3071% / 0.000902017% / 2.38419e-05% | 20260505 04:04:24.822 | INFO | Baz IO overall: | 20260505 04:04:24.822 | INFO | BytesWritten : 59929301430 | 20260505 04:04:24.822 | INFO | NumEvents : 26895284940 | 20260505 04:04:26.819 | INFO | Analyzing chunk frames = [196608,197120) | 20260505 04:04:26.819 | INFO | MemInfo: { | 20260505 04:04:26.819 | INFO | "gpu" : " 17 GB", | 20260505 04:04:26.819 | INFO | "huge" : "", | 20260505 04:04:26.819 | INFO | "huge_shared" : " 27 GB", | 20260505 04:04:26.819 | INFO | "malloc" : " 14 GB", | 20260505 04:04:26.819 | INFO | "pinned" : " 29 GB" | 20260505 04:04:26.819 | INFO | } | 20260505 04:04:26.819 | INFO | Perf report for the last 5.13942 seconds: | 20260505 04:04:26.819 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:04:26.819 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:26.819 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:26.819 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:26.819 | INFO | ANALYSIS : 58.6%, 3.0, 91.4ms, 0 | 20260505 04:04:26.819 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:26.819 | INFO | RT_METRICS : 44.3%, 1.0, 23.3ms, 0 | 20260505 04:04:26.819 | INFO | PRE_HQ : 58.0%, 2.2, 64.8ms, 0 | 20260505 04:04:26.819 | INFO | BAZWRITER : 58.8%, 3.4, 102.4ms, 0 | 20260505 04:04:26.819 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:04:26.819 | INFO | Uploaded 36GiB to GPU in 1.63709s (21.9902GiB/s) | 20260505 04:04:26.819 | INFO | Downloaded 38GiB to GPU in 1.69808s (22.3782GiB/s) | 20260505 04:04:31.958 | INFO | Analyzing chunk frames = [197120,197632) | 20260505 04:04:31.958 | INFO | MemInfo: { | 20260505 04:04:31.958 | INFO | "gpu" : " 17 GB", | 20260505 04:04:31.958 | INFO | "huge" : "", | 20260505 04:04:31.958 | INFO | "huge_shared" : " 27 GB", | 20260505 04:04:31.958 | INFO | "malloc" : " 14 GB", | 20260505 04:04:31.958 | INFO | "pinned" : " 29 GB" | 20260505 04:04:31.958 | INFO | } | 20260505 04:04:31.958 | INFO | Perf report for the last 5.13934 seconds: | 20260505 04:04:31.958 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:04:31.958 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:31.958 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:31.958 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:31.958 | INFO | ANALYSIS : 58.3%, 3.0, 90.9ms, 0 | 20260505 04:04:31.958 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:31.958 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:31.958 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 04:04:31.958 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:31.958 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:31.959 | INFO | Uploaded 36GiB to GPU in 1.5979s (22.5296GiB/s) | 20260505 04:04:31.959 | INFO | Downloaded 34GiB to GPU in 1.514s (22.4571GiB/s) | 20260505 04:04:37.097 | INFO | Analyzing chunk frames = [197632,198144) | 20260505 04:04:37.097 | INFO | MemInfo: { | 20260505 04:04:37.097 | INFO | "gpu" : " 17 GB", | 20260505 04:04:37.097 | INFO | "huge" : "", | 20260505 04:04:37.097 | INFO | "huge_shared" : " 27 GB", | 20260505 04:04:37.097 | INFO | "malloc" : " 14 GB", | 20260505 04:04:37.097 | INFO | "pinned" : " 29 GB" | 20260505 04:04:37.097 | INFO | } | 20260505 04:04:37.098 | INFO | Perf report for the last 5.13932 seconds: | 20260505 04:04:37.098 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:04:37.098 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:37.098 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:37.098 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:37.098 | INFO | ANALYSIS : 57.7%, 3.0, 89.7ms, 0 | 20260505 04:04:37.098 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:37.098 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:37.098 | INFO | PRE_HQ : 31.9%, 1.0, 16.5ms, 0 | 20260505 04:04:37.098 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:37.098 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:37.098 | INFO | Uploaded 36GiB to GPU in 1.60028s (22.4961GiB/s) | 20260505 04:04:37.098 | INFO | Downloaded 34GiB to GPU in 1.51719s (22.4098GiB/s) | 20260505 04:04:42.237 | INFO | Analyzing chunk frames = [198144,198656) | 20260505 04:04:42.237 | INFO | MemInfo: { | 20260505 04:04:42.237 | INFO | "gpu" : " 17 GB", | 20260505 04:04:42.237 | INFO | "huge" : "", | 20260505 04:04:42.237 | INFO | "huge_shared" : " 27 GB", | 20260505 04:04:42.237 | INFO | "malloc" : " 14 GB", | 20260505 04:04:42.237 | INFO | "pinned" : " 29 GB" | 20260505 04:04:42.237 | INFO | } | 20260505 04:04:42.237 | INFO | Perf report for the last 5.13919 seconds: | 20260505 04:04:42.237 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:04:42.237 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:42.237 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:42.237 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:42.237 | INFO | ANALYSIS : 57.4%, 3.0, 89.0ms, 0 | 20260505 04:04:42.237 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:42.237 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:42.237 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 04:04:42.237 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:42.237 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:42.237 | INFO | Uploaded 36GiB to GPU in 1.59939s (22.5086GiB/s) | 20260505 04:04:42.237 | INFO | Downloaded 34GiB to GPU in 1.51588s (22.4291GiB/s) | 20260505 04:04:47.376 | INFO | Analyzing chunk frames = [198656,199168) | 20260505 04:04:47.376 | INFO | MemInfo: { | 20260505 04:04:47.376 | INFO | "gpu" : " 17 GB", | 20260505 04:04:47.376 | INFO | "huge" : "", | 20260505 04:04:47.376 | INFO | "huge_shared" : " 27 GB", | 20260505 04:04:47.376 | INFO | "malloc" : " 14 GB", | 20260505 04:04:47.376 | INFO | "pinned" : " 29 GB" | 20260505 04:04:47.376 | INFO | } | 20260505 04:04:47.376 | INFO | Perf report for the last 5.13932 seconds: | 20260505 04:04:47.376 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:04:47.376 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:47.376 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:47.376 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:47.376 | INFO | ANALYSIS : 57.2%, 3.0, 89.3ms, 0 | 20260505 04:04:47.376 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:47.376 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:47.376 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260505 04:04:47.376 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:47.376 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:47.377 | INFO | Uploaded 36GiB to GPU in 1.59917s (22.5117GiB/s) | 20260505 04:04:47.377 | INFO | Downloaded 34GiB to GPU in 1.51526s (22.4383GiB/s) | 20260505 04:04:52.515 | INFO | Analyzing chunk frames = [199168,199680) | 20260505 04:04:52.515 | INFO | MemInfo: { | 20260505 04:04:52.515 | INFO | "gpu" : " 17 GB", | 20260505 04:04:52.515 | INFO | "huge" : "", | 20260505 04:04:52.515 | INFO | "huge_shared" : " 27 GB", | 20260505 04:04:52.515 | INFO | "malloc" : " 14 GB", | 20260505 04:04:52.515 | INFO | "pinned" : " 29 GB" | 20260505 04:04:52.515 | INFO | } | 20260505 04:04:52.515 | INFO | Perf report for the last 5.13938 seconds: | 20260505 04:04:52.516 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:04:52.516 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:52.516 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:52.516 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:52.516 | INFO | ANALYSIS : 57.6%, 3.0, 89.8ms, 0 | 20260505 04:04:52.516 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:52.516 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:52.516 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 04:04:52.516 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:52.516 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:52.516 | INFO | Uploaded 36GiB to GPU in 1.59799s (22.5283GiB/s) | 20260505 04:04:52.516 | INFO | Downloaded 34GiB to GPU in 1.51535s (22.4371GiB/s) | 20260505 04:04:57.655 | INFO | Analyzing chunk frames = [199680,200192) | 20260505 04:04:57.655 | INFO | MemInfo: { | 20260505 04:04:57.655 | INFO | "gpu" : " 17 GB", | 20260505 04:04:57.655 | INFO | "huge" : "", | 20260505 04:04:57.655 | INFO | "huge_shared" : " 27 GB", | 20260505 04:04:57.655 | INFO | "malloc" : " 14 GB", | 20260505 04:04:57.655 | INFO | "pinned" : " 29 GB" | 20260505 04:04:57.655 | INFO | } | 20260505 04:04:57.655 | INFO | Perf report for the last 5.1393 seconds: | 20260505 04:04:57.655 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:04:57.655 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:57.655 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:57.655 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:57.655 | INFO | ANALYSIS : 57.7%, 3.0, 90.1ms, 0 | 20260505 04:04:57.655 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:57.655 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:57.655 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 04:04:57.655 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:57.655 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:04:57.655 | INFO | Uploaded 36GiB to GPU in 1.59902s (22.5138GiB/s) | 20260505 04:04:57.655 | INFO | Downloaded 34GiB to GPU in 1.51573s (22.4314GiB/s) | 20260505 04:05:02.794 | INFO | Analyzing chunk frames = [200192,200704) | 20260505 04:05:02.794 | INFO | MemInfo: { | 20260505 04:05:02.794 | INFO | "gpu" : " 17 GB", | 20260505 04:05:02.794 | INFO | "huge" : "", | 20260505 04:05:02.794 | INFO | "huge_shared" : " 27 GB", | 20260505 04:05:02.794 | INFO | "malloc" : " 14 GB", | 20260505 04:05:02.794 | INFO | "pinned" : " 29 GB" | 20260505 04:05:02.794 | INFO | } | 20260505 04:05:02.794 | INFO | Perf report for the last 5.1394 seconds: | 20260505 04:05:02.794 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:05:02.794 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:02.794 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:02.794 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:02.794 | INFO | ANALYSIS : 57.9%, 3.0, 90.3ms, 0 | 20260505 04:05:02.794 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:02.794 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:02.794 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260505 04:05:02.794 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:02.794 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:02.794 | INFO | Uploaded 36GiB to GPU in 1.59933s (22.5094GiB/s) | 20260505 04:05:02.795 | INFO | Downloaded 34GiB to GPU in 1.51665s (22.4179GiB/s) | 20260505 04:05:02.891 | INFO | Starting a round of Baz IO | 20260505 04:05:05.855 | INFO | Populating RTMetrics report | 20260505 04:05:05.856 | INFO | RTMetrics report populated | 20260505 04:05:05.856 | INFO | Opening tmp RTMetrics json file | 20260505 04:05:05.857 | INFO | Serializing RTMetrics to json | 20260505 04:05:05.858 | INFO | Serialization done, swapping tmp file now... | 20260505 04:05:05.858 | INFO | RTMetrics done | 20260505 04:05:05.932 | INFO | Finished a round of Baz IO: Wrote 1.733 GiB in 3.041 seconds | 20260505 04:05:05.932 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 55.11% / 0.27% / 44.62% | 20260505 04:05:05.932 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 13.2084% / 0.000858307% / 6.35783e-05% | 20260505 04:05:05.932 | INFO | Baz IO overall: | 20260505 04:05:05.932 | INFO | BytesWritten : 61789696802 | 20260505 04:05:05.932 | INFO | NumEvents : 27772410351 | 20260505 04:05:07.933 | INFO | Analyzing chunk frames = [200704,201216) | 20260505 04:05:07.933 | INFO | MemInfo: { | 20260505 04:05:07.933 | INFO | "gpu" : " 17 GB", | 20260505 04:05:07.933 | INFO | "huge" : "", | 20260505 04:05:07.933 | INFO | "huge_shared" : " 27 GB", | 20260505 04:05:07.933 | INFO | "malloc" : " 14 GB", | 20260505 04:05:07.933 | INFO | "pinned" : " 29 GB" | 20260505 04:05:07.933 | INFO | } | 20260505 04:05:07.933 | INFO | Perf report for the last 5.13926 seconds: | 20260505 04:05:07.934 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:05:07.934 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:07.934 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:07.934 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:07.934 | INFO | ANALYSIS : 58.5%, 3.0, 91.3ms, 0 | 20260505 04:05:07.934 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:07.934 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260505 04:05:07.934 | INFO | PRE_HQ : 58.0%, 2.2, 65.0ms, 0 | 20260505 04:05:07.934 | INFO | BAZWRITER : 58.7%, 3.4, 103.0ms, 0 | 20260505 04:05:07.934 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:05:07.934 | INFO | Uploaded 36GiB to GPU in 1.63235s (22.0541GiB/s) | 20260505 04:05:07.934 | INFO | Downloaded 38GiB to GPU in 1.69478s (22.4217GiB/s) | 20260505 04:05:13.073 | INFO | Analyzing chunk frames = [201216,201728) | 20260505 04:05:13.073 | INFO | MemInfo: { | 20260505 04:05:13.073 | INFO | "gpu" : " 17 GB", | 20260505 04:05:13.073 | INFO | "huge" : "", | 20260505 04:05:13.073 | INFO | "huge_shared" : " 27 GB", | 20260505 04:05:13.073 | INFO | "malloc" : " 14 GB", | 20260505 04:05:13.073 | INFO | "pinned" : " 29 GB" | 20260505 04:05:13.073 | INFO | } | 20260505 04:05:13.073 | INFO | Perf report for the last 5.13919 seconds: | 20260505 04:05:13.073 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:05:13.073 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:13.073 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:13.073 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:13.073 | INFO | ANALYSIS : 58.3%, 3.0, 91.0ms, 0 | 20260505 04:05:13.073 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:13.073 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:13.073 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260505 04:05:13.073 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:13.073 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:13.073 | INFO | Uploaded 36GiB to GPU in 1.59682s (22.5448GiB/s) | 20260505 04:05:13.073 | INFO | Downloaded 34GiB to GPU in 1.51363s (22.4625GiB/s) | 20260505 04:05:18.212 | INFO | Analyzing chunk frames = [201728,202240) | 20260505 04:05:18.212 | INFO | MemInfo: { | 20260505 04:05:18.212 | INFO | "gpu" : " 17 GB", | 20260505 04:05:18.212 | INFO | "huge" : "", | 20260505 04:05:18.212 | INFO | "huge_shared" : " 27 GB", | 20260505 04:05:18.212 | INFO | "malloc" : " 14 GB", | 20260505 04:05:18.212 | INFO | "pinned" : " 29 GB" | 20260505 04:05:18.212 | INFO | } | 20260505 04:05:18.212 | INFO | Perf report for the last 5.13975 seconds: | 20260505 04:05:18.212 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:05:18.212 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:18.212 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:18.213 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:18.213 | INFO | ANALYSIS : 57.7%, 3.0, 89.6ms, 0 | 20260505 04:05:18.213 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:18.213 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:18.213 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 04:05:18.213 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:18.213 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:18.213 | INFO | Uploaded 36GiB to GPU in 1.60161s (22.4774GiB/s) | 20260505 04:05:18.213 | INFO | Downloaded 34GiB to GPU in 1.51836s (22.3926GiB/s) | 20260505 04:05:23.352 | INFO | Analyzing chunk frames = [202240,202752) | 20260505 04:05:23.352 | INFO | MemInfo: { | 20260505 04:05:23.352 | INFO | "gpu" : " 17 GB", | 20260505 04:05:23.352 | INFO | "huge" : "", | 20260505 04:05:23.352 | INFO | "huge_shared" : " 27 GB", | 20260505 04:05:23.352 | INFO | "malloc" : " 14 GB", | 20260505 04:05:23.352 | INFO | "pinned" : " 29 GB" | 20260505 04:05:23.352 | INFO | } | 20260505 04:05:23.352 | INFO | Perf report for the last 5.13937 seconds: | 20260505 04:05:23.352 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:05:23.352 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:23.352 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:23.352 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:23.352 | INFO | ANALYSIS : 57.4%, 3.0, 89.0ms, 0 | 20260505 04:05:23.352 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:23.352 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:23.352 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260505 04:05:23.352 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:23.352 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:23.352 | INFO | Uploaded 36GiB to GPU in 1.60005s (22.4993GiB/s) | 20260505 04:05:23.352 | INFO | Downloaded 34GiB to GPU in 1.5165s (22.42GiB/s) | 20260505 04:05:28.491 | INFO | Analyzing chunk frames = [202752,203264) | 20260505 04:05:28.491 | INFO | MemInfo: { | 20260505 04:05:28.491 | INFO | "gpu" : " 17 GB", | 20260505 04:05:28.491 | INFO | "huge" : "", | 20260505 04:05:28.491 | INFO | "huge_shared" : " 27 GB", | 20260505 04:05:28.491 | INFO | "malloc" : " 14 GB", | 20260505 04:05:28.491 | INFO | "pinned" : " 29 GB" | 20260505 04:05:28.491 | INFO | } | 20260505 04:05:28.491 | INFO | Perf report for the last 5.139 seconds: | 20260505 04:05:28.491 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:05:28.491 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:28.491 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:28.491 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:28.491 | INFO | ANALYSIS : 57.3%, 3.0, 89.3ms, 0 | 20260505 04:05:28.491 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:28.491 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:28.491 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 04:05:28.491 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:28.491 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:28.491 | INFO | Uploaded 36GiB to GPU in 1.5981s (22.5268GiB/s) | 20260505 04:05:28.491 | INFO | Downloaded 34GiB to GPU in 1.51449s (22.4498GiB/s) | 20260505 04:05:33.630 | INFO | Analyzing chunk frames = [203264,203776) | 20260505 04:05:33.630 | INFO | MemInfo: { | 20260505 04:05:33.630 | INFO | "gpu" : " 17 GB", | 20260505 04:05:33.630 | INFO | "huge" : "", | 20260505 04:05:33.630 | INFO | "huge_shared" : " 27 GB", | 20260505 04:05:33.630 | INFO | "malloc" : " 14 GB", | 20260505 04:05:33.630 | INFO | "pinned" : " 29 GB" | 20260505 04:05:33.630 | INFO | } | 20260505 04:05:33.630 | INFO | Perf report for the last 5.13932 seconds: | 20260505 04:05:33.630 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:05:33.630 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:33.630 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:33.630 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:33.630 | INFO | ANALYSIS : 57.6%, 3.0, 89.8ms, 0 | 20260505 04:05:33.630 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:33.630 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:33.630 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260505 04:05:33.630 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:33.630 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:33.631 | INFO | Uploaded 36GiB to GPU in 1.59823s (22.525GiB/s) | 20260505 04:05:33.631 | INFO | Downloaded 34GiB to GPU in 1.51535s (22.437GiB/s) | 20260505 04:05:38.769 | INFO | Analyzing chunk frames = [203776,204288) | 20260505 04:05:38.769 | INFO | MemInfo: { | 20260505 04:05:38.769 | INFO | "gpu" : " 17 GB", | 20260505 04:05:38.769 | INFO | "huge" : "", | 20260505 04:05:38.769 | INFO | "huge_shared" : " 27 GB", | 20260505 04:05:38.769 | INFO | "malloc" : " 14 GB", | 20260505 04:05:38.769 | INFO | "pinned" : " 29 GB" | 20260505 04:05:38.769 | INFO | } | 20260505 04:05:38.769 | INFO | Perf report for the last 5.13932 seconds: | 20260505 04:05:38.770 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:05:38.770 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:38.770 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:38.770 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:38.770 | INFO | ANALYSIS : 57.8%, 3.0, 90.1ms, 0 | 20260505 04:05:38.770 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:38.770 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:38.770 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260505 04:05:38.770 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:38.770 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:38.770 | INFO | Uploaded 36GiB to GPU in 1.59914s (22.5121GiB/s) | 20260505 04:05:38.770 | INFO | Downloaded 34GiB to GPU in 1.51655s (22.4193GiB/s) | 20260505 04:05:43.909 | INFO | Analyzing chunk frames = [204288,204800) | 20260505 04:05:43.909 | INFO | MemInfo: { | 20260505 04:05:43.909 | INFO | "gpu" : " 17 GB", | 20260505 04:05:43.909 | INFO | "huge" : "", | 20260505 04:05:43.909 | INFO | "huge_shared" : " 27 GB", | 20260505 04:05:43.909 | INFO | "malloc" : " 14 GB", | 20260505 04:05:43.909 | INFO | "pinned" : " 29 GB" | 20260505 04:05:43.909 | INFO | } | 20260505 04:05:43.909 | INFO | Perf report for the last 5.13967 seconds: | 20260505 04:05:43.909 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:05:43.909 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:43.909 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:43.909 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:43.909 | INFO | ANALYSIS : 57.9%, 3.0, 90.3ms, 0 | 20260505 04:05:43.909 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:43.909 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:43.909 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260505 04:05:43.909 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:43.909 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:43.909 | INFO | Uploaded 36GiB to GPU in 1.59918s (22.5116GiB/s) | 20260505 04:05:43.909 | INFO | Downloaded 34GiB to GPU in 1.5161s (22.426GiB/s) | 20260505 04:05:43.910 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 04:05:43.910 | INFO | Total mem usage: 14.9 GiB | 20260505 04:05:43.910 | INFO | All Time High Water Marks sum: 15.2 GiB (Savings: 1.66%) | 20260505 04:05:43.910 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -2.61%) | 20260505 04:05:43.910 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 04:05:43.910 | INFO | High Water Breakdown: | 20260505 04:05:43.910 | INFO | 58.5% : 8.51 GiB ( 100% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 04:05:43.910 | INFO | 41.3% : 6.01 GiB ( 90.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 04:05:43.910 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 04:05:43.910 | INFO | ------------------------------------------ | 20260505 04:05:43.910 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:05:43.910 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 04:05:43.910 | INFO | Total mem usage: 26.5 GiB | 20260505 04:05:43.910 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260505 04:05:43.910 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 04:05:43.910 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260505 04:05:43.910 | INFO | High Water Breakdown: | 20260505 04:05:43.910 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:05:43.910 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:05:43.910 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:05:43.910 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:05:43.910 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:05:43.910 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:05:43.910 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 04:05:43.910 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 04:05:43.910 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:05:43.910 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 04:05:43.910 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 04:05:43.910 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 04:05:43.910 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 04:05:43.910 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 04:05:43.910 | INFO | ------------------------------------------ | 20260505 04:05:43.910 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:05:43.910 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 04:05:43.910 | INFO | Total mem usage: 25.6 GiB | 20260505 04:05:43.910 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 04:05:43.910 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 04:05:43.910 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 04:05:43.910 | INFO | High Water Breakdown: | 20260505 04:05:43.910 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 04:05:43.910 | INFO | ------------------------------------------ | 20260505 04:05:43.910 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:05:43.910 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 04:05:43.910 | INFO | Total mem usage: 15.7 GiB | 20260505 04:05:43.910 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260505 04:05:43.910 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 04:05:43.910 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260505 04:05:43.910 | INFO | High Water Breakdown: | 20260505 04:05:43.910 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 04:05:43.910 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 04:05:43.910 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 04:05:43.910 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 04:05:43.910 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 04:05:43.910 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 04:05:43.910 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 04:05:43.910 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 04:05:43.910 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 04:05:43.910 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:05:43.910 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 04:05:43.910 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:05:43.910 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 04:05:43.910 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 04:05:43.910 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 04:05:43.910 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 04:05:43.910 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:05:43.910 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 04:05:43.910 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:05:43.910 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:05:43.910 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:05:43.910 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 04:05:43.910 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 04:05:43.910 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:05:43.910 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 04:05:43.910 | INFO | ------------------------------------------ | 20260505 04:05:43.910 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:05:43.910 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 04:05:43.910 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:05:43.910 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:05:43.910 | INFO | PerfReport: Full Execution -- 3.06e+01 / -nan, 1.29e+01 / 0.00e+00 | 20260505 04:05:43.910 | INFO | PerfReport: 0.28%, AggregateStats -- 8.64e-02 / -nan, 4.43e-03 / 0.00e+00 | 20260505 04:05:43.910 | INFO | PerfReport: 10.78%, Baseline -- 3.30e+00 / -nan, 1.92e-02 / 0.00e+00 | 20260505 04:05:43.910 | INFO | PerfReport: 3.02%, Histogram -- 9.24e-01 / -nan, 5.22e-02 / 0.00e+00 | 20260505 04:05:43.910 | INFO | PerfReport: 16.01%, DME -- 3.92e+01 / -nan, 3.26e+00 / 0.00e+00 | 20260505 04:05:43.910 | INFO | PerfReport: 56.26%, FrameLabeling -- 1.72e+01 / -nan, 4.48e-02 / 0.00e+00 | 20260505 04:05:43.911 | INFO | PerfReport: 7.76%, PulseAccumulating -- 2.37e+00 / -nan, 1.62e-01 / 0.00e+00 | 20260505 04:05:43.911 | INFO | PerfReport: 4.16%, Metrics -- 1.27e+00 / -nan, 2.07e-01 / 0.00e+00 | 20260505 04:05:43.911 | INFO | PerfReport: 1.73%, PulseSerializing -- 5.31e-01 / -nan, 8.51e-02 / 0.00e+00 | 20260505 04:05:43.911 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 04:05:43.911 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:05:43.911 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:05:43.911 | INFO | PerfReport: Full Execution -- 9.19e+01 / -nan, 1.92e+01 / 0.00e+00 | 20260505 04:05:43.911 | INFO | PerfReport: 18.17%, Upload -- 1.67e+01 / 0.00e+00, 3.61e-01 / 0.00e+00 | 20260505 04:05:43.911 | INFO | PerfReport: 17.43%, Download -- 1.60e+01 / 0.00e+00, 6.84e-01 / 0.00e+00 | 20260505 04:05:43.911 | INFO | PerfReport: 33.29%, Compute -- 3.06e+01 / 0.00e+00, 1.29e+01 / 0.00e+00 | 20260505 04:05:43.911 | INFO | PerfReport: 0.55%, LockUpload -- 5.04e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260505 04:05:43.911 | INFO | PerfReport: 0.00%, LockDownload -- 4.01e-04 / 0.00e+00, 9.45e-05 / 0.00e+00 | 20260505 04:05:43.911 | INFO | PerfReport: 30.54%, LockCompute -- 2.81e+01 / 0.00e+00, 1.73e+01 / 0.00e+00 | 20260505 04:05:43.911 | INFO | PerfReport: 0.02%, Misc -- 1.49e-02 / 0.00e+00, 2.96e-02 / 0.00e+00 | 20260505 04:05:44.006 | INFO | Starting a round of Baz IO | 20260505 04:05:46.970 | INFO | Populating RTMetrics report | 20260505 04:05:46.970 | INFO | RTMetrics report populated | 20260505 04:05:46.970 | INFO | Opening tmp RTMetrics json file | 20260505 04:05:46.971 | INFO | Serializing RTMetrics to json | 20260505 04:05:46.972 | INFO | Serialization done, swapping tmp file now... | 20260505 04:05:46.973 | INFO | RTMetrics done | 20260505 04:05:47.046 | INFO | Finished a round of Baz IO: Wrote 1.74 GiB in 3.04 seconds | 20260505 04:05:47.046 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 54.84% / 0.27% / 44.89% | 20260505 04:05:47.046 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 13.1464% / 0.000743071% / 2.38419e-05% | 20260505 04:05:47.046 | INFO | Baz IO overall: | 20260505 04:05:47.047 | INFO | BytesWritten : 63658442967 | 20260505 04:05:47.047 | INFO | NumEvents : 28654107344 | 20260505 04:05:49.048 | INFO | Analyzing chunk frames = [204800,205312) | 20260505 04:05:49.048 | INFO | MemInfo: { | 20260505 04:05:49.048 | INFO | "gpu" : " 17 GB", | 20260505 04:05:49.048 | INFO | "huge" : "", | 20260505 04:05:49.048 | INFO | "huge_shared" : " 27 GB", | 20260505 04:05:49.048 | INFO | "malloc" : " 14 GB", | 20260505 04:05:49.048 | INFO | "pinned" : " 29 GB" | 20260505 04:05:49.048 | INFO | } | 20260505 04:05:49.048 | INFO | Perf report for the last 5.139 seconds: | 20260505 04:05:49.048 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:05:49.048 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:49.048 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:49.048 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:49.048 | INFO | ANALYSIS : 58.5%, 3.0, 91.3ms, 0 | 20260505 04:05:49.048 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:49.048 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260505 04:05:49.048 | INFO | PRE_HQ : 58.0%, 2.2, 64.9ms, 0 | 20260505 04:05:49.048 | INFO | BAZWRITER : 58.7%, 3.4, 102.8ms, 0 | 20260505 04:05:49.048 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:05:49.049 | INFO | Uploaded 36GiB to GPU in 1.64131s (21.9337GiB/s) | 20260505 04:05:49.049 | INFO | Downloaded 38GiB to GPU in 1.70084s (22.3419GiB/s) | 20260505 04:05:54.188 | INFO | Analyzing chunk frames = [205312,205824) | 20260505 04:05:54.188 | INFO | MemInfo: { | 20260505 04:05:54.188 | INFO | "gpu" : " 17 GB", | 20260505 04:05:54.188 | INFO | "huge" : "", | 20260505 04:05:54.188 | INFO | "huge_shared" : " 27 GB", | 20260505 04:05:54.188 | INFO | "malloc" : " 14 GB", | 20260505 04:05:54.188 | INFO | "pinned" : " 29 GB" | 20260505 04:05:54.188 | INFO | } | 20260505 04:05:54.188 | INFO | Perf report for the last 5.13943 seconds: | 20260505 04:05:54.188 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:05:54.188 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:54.188 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:54.188 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:54.188 | INFO | ANALYSIS : 58.3%, 3.0, 91.0ms, 0 | 20260505 04:05:54.188 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:54.188 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:54.188 | INFO | PRE_HQ : 31.9%, 1.0, 16.5ms, 0 | 20260505 04:05:54.188 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:54.188 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:54.188 | INFO | Uploaded 36GiB to GPU in 1.59666s (22.5471GiB/s) | 20260505 04:05:54.188 | INFO | Downloaded 34GiB to GPU in 1.51312s (22.4702GiB/s) | 20260505 04:05:59.327 | INFO | Analyzing chunk frames = [205824,206336) | 20260505 04:05:59.327 | INFO | MemInfo: { | 20260505 04:05:59.327 | INFO | "gpu" : " 17 GB", | 20260505 04:05:59.327 | INFO | "huge" : "", | 20260505 04:05:59.327 | INFO | "huge_shared" : " 27 GB", | 20260505 04:05:59.327 | INFO | "malloc" : " 14 GB", | 20260505 04:05:59.327 | INFO | "pinned" : " 29 GB" | 20260505 04:05:59.327 | INFO | } | 20260505 04:05:59.327 | INFO | Perf report for the last 5.13985 seconds: | 20260505 04:05:59.328 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:05:59.328 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:59.328 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:59.328 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:59.328 | INFO | ANALYSIS : 57.7%, 3.0, 89.6ms, 0 | 20260505 04:05:59.328 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:59.328 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:59.328 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260505 04:05:59.328 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:59.328 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:05:59.328 | INFO | Uploaded 36GiB to GPU in 1.60089s (22.4874GiB/s) | 20260505 04:05:59.328 | INFO | Downloaded 34GiB to GPU in 1.51788s (22.3997GiB/s) | 20260505 04:06:04.470 | INFO | Analyzing chunk frames = [206336,206848) | 20260505 04:06:04.470 | INFO | MemInfo: { | 20260505 04:06:04.470 | INFO | "gpu" : " 17 GB", | 20260505 04:06:04.470 | INFO | "huge" : "", | 20260505 04:06:04.470 | INFO | "huge_shared" : " 27 GB", | 20260505 04:06:04.470 | INFO | "malloc" : " 14 GB", | 20260505 04:06:04.470 | INFO | "pinned" : " 29 GB" | 20260505 04:06:04.470 | INFO | } | 20260505 04:06:04.470 | INFO | Perf report for the last 5.1427 seconds: | 20260505 04:06:04.470 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:06:04.470 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:04.470 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:04.470 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:04.470 | INFO | ANALYSIS : 57.4%, 3.0, 89.0ms, 0 | 20260505 04:06:04.470 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:04.470 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:04.470 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 04:06:04.471 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:04.471 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:04.471 | INFO | Uploaded 36GiB to GPU in 1.59986s (22.5019GiB/s) | 20260505 04:06:04.471 | INFO | Downloaded 34GiB to GPU in 1.5165s (22.42GiB/s) | 20260505 04:06:09.629 | INFO | Analyzing chunk frames = [206848,207360) | 20260505 04:06:09.629 | INFO | MemInfo: { | 20260505 04:06:09.629 | INFO | "gpu" : " 17 GB", | 20260505 04:06:09.629 | INFO | "huge" : "", | 20260505 04:06:09.629 | INFO | "huge_shared" : " 27 GB", | 20260505 04:06:09.629 | INFO | "malloc" : " 14 GB", | 20260505 04:06:09.629 | INFO | "pinned" : " 29 GB" | 20260505 04:06:09.629 | INFO | } | 20260505 04:06:09.629 | INFO | Perf report for the last 5.15903 seconds: | 20260505 04:06:09.629 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:06:09.629 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:09.629 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:09.629 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:09.629 | INFO | ANALYSIS : 57.1%, 3.0, 89.3ms, 0 | 20260505 04:06:09.629 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:09.630 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:09.630 | INFO | PRE_HQ : 32.0%, 1.0, 16.7ms, 0 | 20260505 04:06:09.630 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:09.630 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:09.630 | INFO | Uploaded 36GiB to GPU in 1.59814s (22.5262GiB/s) | 20260505 04:06:09.630 | INFO | Downloaded 34GiB to GPU in 1.51414s (22.455GiB/s) | 20260505 04:06:14.750 | INFO | Analyzing chunk frames = [207360,207872) | 20260505 04:06:14.750 | INFO | MemInfo: { | 20260505 04:06:14.750 | INFO | "gpu" : " 17 GB", | 20260505 04:06:14.750 | INFO | "huge" : "", | 20260505 04:06:14.750 | INFO | "huge_shared" : " 27 GB", | 20260505 04:06:14.750 | INFO | "malloc" : " 14 GB", | 20260505 04:06:14.750 | INFO | "pinned" : " 29 GB" | 20260505 04:06:14.750 | INFO | } | 20260505 04:06:14.751 | INFO | Perf report for the last 5.12135 seconds: | 20260505 04:06:14.751 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:06:14.751 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:14.751 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:14.751 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:14.751 | INFO | ANALYSIS : 57.8%, 3.0, 89.8ms, 0 | 20260505 04:06:14.751 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:14.751 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:14.751 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260505 04:06:14.751 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:14.751 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:14.751 | INFO | Uploaded 36GiB to GPU in 1.59851s (22.521GiB/s) | 20260505 04:06:14.751 | INFO | Downloaded 34GiB to GPU in 1.51483s (22.4448GiB/s) | 20260505 04:06:19.884 | INFO | Analyzing chunk frames = [207872,208384) | 20260505 04:06:19.884 | INFO | MemInfo: { | 20260505 04:06:19.884 | INFO | "gpu" : " 17 GB", | 20260505 04:06:19.884 | INFO | "huge" : "", | 20260505 04:06:19.884 | INFO | "huge_shared" : " 27 GB", | 20260505 04:06:19.884 | INFO | "malloc" : " 14 GB", | 20260505 04:06:19.884 | INFO | "pinned" : " 29 GB" | 20260505 04:06:19.884 | INFO | } | 20260505 04:06:19.884 | INFO | Perf report for the last 5.13369 seconds: | 20260505 04:06:19.885 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:06:19.885 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:19.885 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:19.885 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:19.885 | INFO | ANALYSIS : 57.9%, 3.0, 90.2ms, 0 | 20260505 04:06:19.885 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:19.885 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:19.885 | INFO | PRE_HQ : 34.6%, 1.0, 17.9ms, 0 | 20260505 04:06:19.885 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:19.885 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:19.885 | INFO | Uploaded 36GiB to GPU in 1.60368s (22.4484GiB/s) | 20260505 04:06:19.885 | INFO | Downloaded 34GiB to GPU in 1.52019s (22.3657GiB/s) | 20260505 04:06:25.024 | INFO | Analyzing chunk frames = [208384,208896) | 20260505 04:06:25.024 | INFO | MemInfo: { | 20260505 04:06:25.024 | INFO | "gpu" : " 17 GB", | 20260505 04:06:25.024 | INFO | "huge" : "", | 20260505 04:06:25.024 | INFO | "huge_shared" : " 27 GB", | 20260505 04:06:25.024 | INFO | "malloc" : " 14 GB", | 20260505 04:06:25.024 | INFO | "pinned" : " 29 GB" | 20260505 04:06:25.024 | INFO | } | 20260505 04:06:25.024 | INFO | Perf report for the last 5.1394 seconds: | 20260505 04:06:25.024 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:06:25.024 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:25.024 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:25.024 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:25.024 | INFO | ANALYSIS : 57.9%, 3.0, 90.3ms, 0 | 20260505 04:06:25.024 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:25.024 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:25.024 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 04:06:25.024 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:25.024 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:25.024 | INFO | Uploaded 36GiB to GPU in 1.59795s (22.5289GiB/s) | 20260505 04:06:25.024 | INFO | Downloaded 34GiB to GPU in 1.51546s (22.4355GiB/s) | 20260505 04:06:25.120 | INFO | Starting a round of Baz IO | 20260505 04:06:28.088 | INFO | Populating RTMetrics report | 20260505 04:06:28.089 | INFO | RTMetrics report populated | 20260505 04:06:28.089 | INFO | Opening tmp RTMetrics json file | 20260505 04:06:28.089 | INFO | Serializing RTMetrics to json | 20260505 04:06:28.091 | INFO | Serialization done, swapping tmp file now... | 20260505 04:06:28.091 | INFO | RTMetrics done | 20260505 04:06:28.162 | INFO | Finished a round of Baz IO: Wrote 1.75 GiB in 3.041 seconds | 20260505 04:06:28.162 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 54.58% / 0.26% / 45.16% | 20260505 04:06:28.162 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 13.082% / 0.000890096% / 5.16574e-05% | 20260505 04:06:28.162 | INFO | Baz IO overall: | 20260505 04:06:28.162 | INFO | BytesWritten : 65537202151 | 20260505 04:06:28.162 | INFO | NumEvents : 29540345919 | 20260505 04:06:30.163 | INFO | Analyzing chunk frames = [208896,209408) | 20260505 04:06:30.163 | INFO | MemInfo: { | 20260505 04:06:30.163 | INFO | "gpu" : " 17 GB", | 20260505 04:06:30.163 | INFO | "huge" : "", | 20260505 04:06:30.163 | INFO | "huge_shared" : " 27 GB", | 20260505 04:06:30.163 | INFO | "malloc" : " 14 GB", | 20260505 04:06:30.163 | INFO | "pinned" : " 29 GB" | 20260505 04:06:30.163 | INFO | } | 20260505 04:06:30.163 | INFO | Perf report for the last 5.1395 seconds: | 20260505 04:06:30.163 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:06:30.163 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:30.163 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:30.163 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:30.163 | INFO | ANALYSIS : 58.6%, 3.0, 91.4ms, 0 | 20260505 04:06:30.163 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:30.163 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260505 04:06:30.163 | INFO | PRE_HQ : 58.1%, 2.2, 64.9ms, 0 | 20260505 04:06:30.163 | INFO | BAZWRITER : 58.7%, 3.4, 103.1ms, 0 | 20260505 04:06:30.163 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:06:30.164 | INFO | Uploaded 36GiB to GPU in 1.63687s (21.9932GiB/s) | 20260505 04:06:30.164 | INFO | Downloaded 38GiB to GPU in 1.69731s (22.3883GiB/s) | 20260505 04:06:35.302 | INFO | Analyzing chunk frames = [209408,209920) | 20260505 04:06:35.302 | INFO | MemInfo: { | 20260505 04:06:35.302 | INFO | "gpu" : " 17 GB", | 20260505 04:06:35.302 | INFO | "huge" : "", | 20260505 04:06:35.302 | INFO | "huge_shared" : " 27 GB", | 20260505 04:06:35.302 | INFO | "malloc" : " 14 GB", | 20260505 04:06:35.302 | INFO | "pinned" : " 29 GB" | 20260505 04:06:35.302 | INFO | } | 20260505 04:06:35.303 | INFO | Perf report for the last 5.13931 seconds: | 20260505 04:06:35.303 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:06:35.303 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:35.303 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:35.303 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:35.303 | INFO | ANALYSIS : 58.3%, 3.0, 91.0ms, 0 | 20260505 04:06:35.303 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:35.303 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:35.303 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 04:06:35.303 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:35.303 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:35.303 | INFO | Uploaded 36GiB to GPU in 1.59693s (22.5432GiB/s) | 20260505 04:06:35.303 | INFO | Downloaded 34GiB to GPU in 1.51385s (22.4592GiB/s) | 20260505 04:06:40.451 | INFO | Analyzing chunk frames = [209920,210432) | 20260505 04:06:40.451 | INFO | MemInfo: { | 20260505 04:06:40.451 | INFO | "gpu" : " 17 GB", | 20260505 04:06:40.451 | INFO | "huge" : "", | 20260505 04:06:40.451 | INFO | "huge_shared" : " 27 GB", | 20260505 04:06:40.451 | INFO | "malloc" : " 14 GB", | 20260505 04:06:40.451 | INFO | "pinned" : " 29 GB" | 20260505 04:06:40.451 | INFO | } | 20260505 04:06:40.451 | INFO | Perf report for the last 5.14863 seconds: | 20260505 04:06:40.451 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:06:40.451 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:40.451 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:40.451 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:40.451 | INFO | ANALYSIS : 57.6%, 3.0, 89.6ms, 0 | 20260505 04:06:40.451 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:40.451 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:40.451 | INFO | PRE_HQ : 31.8%, 1.0, 16.6ms, 0 | 20260505 04:06:40.451 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:40.451 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:40.452 | INFO | Uploaded 36GiB to GPU in 1.59999s (22.5001GiB/s) | 20260505 04:06:40.452 | INFO | Downloaded 34GiB to GPU in 1.51682s (22.4154GiB/s) | 20260505 04:06:45.581 | INFO | Analyzing chunk frames = [210432,210944) | 20260505 04:06:45.581 | INFO | MemInfo: { | 20260505 04:06:45.581 | INFO | "gpu" : " 17 GB", | 20260505 04:06:45.581 | INFO | "huge" : "", | 20260505 04:06:45.581 | INFO | "huge_shared" : " 27 GB", | 20260505 04:06:45.581 | INFO | "malloc" : " 14 GB", | 20260505 04:06:45.581 | INFO | "pinned" : " 29 GB" | 20260505 04:06:45.581 | INFO | } | 20260505 04:06:45.581 | INFO | Perf report for the last 5.13019 seconds: | 20260505 04:06:45.581 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:06:45.581 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:45.581 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:45.581 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:45.582 | INFO | ANALYSIS : 57.4%, 3.0, 88.9ms, 0 | 20260505 04:06:45.582 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:45.582 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:45.582 | INFO | PRE_HQ : 32.6%, 1.0, 16.9ms, 0 | 20260505 04:06:45.582 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:45.582 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:45.582 | INFO | Uploaded 36GiB to GPU in 1.6005s (22.493GiB/s) | 20260505 04:06:45.582 | INFO | Downloaded 34GiB to GPU in 1.51764s (22.4033GiB/s) | 20260505 04:06:50.720 | INFO | Analyzing chunk frames = [210944,211456) | 20260505 04:06:50.720 | INFO | MemInfo: { | 20260505 04:06:50.720 | INFO | "gpu" : " 17 GB", | 20260505 04:06:50.720 | INFO | "huge" : "", | 20260505 04:06:50.720 | INFO | "huge_shared" : " 27 GB", | 20260505 04:06:50.720 | INFO | "malloc" : " 14 GB", | 20260505 04:06:50.720 | INFO | "pinned" : " 29 GB" | 20260505 04:06:50.720 | INFO | } | 20260505 04:06:50.721 | INFO | Perf report for the last 5.13918 seconds: | 20260505 04:06:50.721 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:06:50.721 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:50.721 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:50.721 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:50.721 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 04:06:50.721 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:50.721 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:50.721 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 04:06:50.721 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:50.721 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:50.721 | INFO | Uploaded 36GiB to GPU in 1.59788s (22.5299GiB/s) | 20260505 04:06:50.721 | INFO | Downloaded 34GiB to GPU in 1.51482s (22.4449GiB/s) | 20260505 04:06:55.860 | INFO | Analyzing chunk frames = [211456,211968) | 20260505 04:06:55.860 | INFO | MemInfo: { | 20260505 04:06:55.860 | INFO | "gpu" : " 17 GB", | 20260505 04:06:55.860 | INFO | "huge" : "", | 20260505 04:06:55.860 | INFO | "huge_shared" : " 27 GB", | 20260505 04:06:55.860 | INFO | "malloc" : " 14 GB", | 20260505 04:06:55.860 | INFO | "pinned" : " 29 GB" | 20260505 04:06:55.860 | INFO | } | 20260505 04:06:55.860 | INFO | Perf report for the last 5.13935 seconds: | 20260505 04:06:55.860 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:06:55.860 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:55.860 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:55.860 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:55.860 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260505 04:06:55.860 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:55.860 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:55.860 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260505 04:06:55.860 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:55.860 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:06:55.861 | INFO | Uploaded 36GiB to GPU in 1.59874s (22.5177GiB/s) | 20260505 04:06:55.861 | INFO | Downloaded 34GiB to GPU in 1.51523s (22.4388GiB/s) | 20260505 04:07:00.999 | INFO | Analyzing chunk frames = [211968,212480) | 20260505 04:07:00.999 | INFO | MemInfo: { | 20260505 04:07:00.999 | INFO | "gpu" : " 17 GB", | 20260505 04:07:00.999 | INFO | "huge" : "", | 20260505 04:07:00.999 | INFO | "huge_shared" : " 27 GB", | 20260505 04:07:00.999 | INFO | "malloc" : " 14 GB", | 20260505 04:07:00.999 | INFO | "pinned" : " 29 GB" | 20260505 04:07:00.999 | INFO | } | 20260505 04:07:00.999 | INFO | Perf report for the last 5.13944 seconds: | 20260505 04:07:00.999 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:07:00.999 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:00.999 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:00.999 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:01.000 | INFO | ANALYSIS : 57.8%, 3.0, 90.2ms, 0 | 20260505 04:07:01.000 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:01.000 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:01.000 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260505 04:07:01.000 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:01.000 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:01.000 | INFO | Uploaded 36GiB to GPU in 1.59848s (22.5214GiB/s) | 20260505 04:07:01.000 | INFO | Downloaded 34GiB to GPU in 1.5161s (22.4259GiB/s) | 20260505 04:07:06.138 | INFO | Analyzing chunk frames = [212480,212992) | 20260505 04:07:06.138 | INFO | MemInfo: { | 20260505 04:07:06.138 | INFO | "gpu" : " 17 GB", | 20260505 04:07:06.138 | INFO | "huge" : "", | 20260505 04:07:06.138 | INFO | "huge_shared" : " 27 GB", | 20260505 04:07:06.138 | INFO | "malloc" : " 14 GB", | 20260505 04:07:06.138 | INFO | "pinned" : " 29 GB" | 20260505 04:07:06.138 | INFO | } | 20260505 04:07:06.139 | INFO | Perf report for the last 5.13922 seconds: | 20260505 04:07:06.139 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:07:06.139 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:06.139 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:06.139 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:06.139 | INFO | ANALYSIS : 57.8%, 3.0, 90.2ms, 0 | 20260505 04:07:06.139 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:06.139 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:06.139 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260505 04:07:06.139 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:06.139 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:06.139 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 04:07:06.139 | INFO | Total mem usage: 14.9 GiB | 20260505 04:07:06.139 | INFO | All Time High Water Marks sum: 15.2 GiB (Savings: 1.66%) | 20260505 04:07:06.139 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -3.16%) | 20260505 04:07:06.139 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 04:07:06.139 | INFO | High Water Breakdown: | 20260505 04:07:06.139 | INFO | 58.5% : 8.46 GiB ( 99.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 04:07:06.139 | INFO | 41.4% : 5.99 GiB ( 90.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 04:07:06.139 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 04:07:06.139 | INFO | ------------------------------------------ | 20260505 04:07:06.139 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:07:06.139 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 04:07:06.139 | INFO | Total mem usage: 26.5 GiB | 20260505 04:07:06.139 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260505 04:07:06.139 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 04:07:06.139 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260505 04:07:06.139 | INFO | High Water Breakdown: | 20260505 04:07:06.139 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:07:06.139 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:07:06.139 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:07:06.139 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:07:06.139 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:07:06.139 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:07:06.139 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 04:07:06.139 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 04:07:06.139 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:07:06.139 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 04:07:06.139 | INFO | Uploaded 36GiB to GPU in 1.59954s (22.5064GiB/s) | 20260505 04:07:06.139 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 04:07:06.139 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 04:07:06.139 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 04:07:06.139 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 04:07:06.139 | INFO | Downloaded 34GiB to GPU in 1.51731s (22.408GiB/s) | 20260505 04:07:06.139 | INFO | ------------------------------------------ | 20260505 04:07:06.140 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:07:06.140 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 04:07:06.140 | INFO | Total mem usage: 25.6 GiB | 20260505 04:07:06.140 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 04:07:06.140 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 04:07:06.140 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 04:07:06.140 | INFO | High Water Breakdown: | 20260505 04:07:06.140 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 04:07:06.140 | INFO | ------------------------------------------ | 20260505 04:07:06.140 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:07:06.140 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 04:07:06.140 | INFO | Total mem usage: 15.7 GiB | 20260505 04:07:06.140 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260505 04:07:06.140 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 04:07:06.140 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260505 04:07:06.140 | INFO | High Water Breakdown: | 20260505 04:07:06.140 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 04:07:06.140 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 04:07:06.140 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 04:07:06.140 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 04:07:06.140 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 04:07:06.140 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 04:07:06.140 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 04:07:06.140 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 04:07:06.140 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 04:07:06.140 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:07:06.140 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 04:07:06.140 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:07:06.140 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 04:07:06.140 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 04:07:06.140 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 04:07:06.140 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 04:07:06.140 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:07:06.140 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 04:07:06.140 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:07:06.140 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:07:06.140 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:07:06.140 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 04:07:06.140 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 04:07:06.140 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:07:06.140 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 04:07:06.140 | INFO | ------------------------------------------ | 20260505 04:07:06.140 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:07:06.140 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 04:07:06.140 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:07:06.140 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:07:06.140 | INFO | PerfReport: Full Execution -- 3.06e+01 / -nan, 1.28e+01 / 0.00e+00 | 20260505 04:07:06.140 | INFO | PerfReport: 0.28%, AggregateStats -- 8.63e-02 / -nan, 4.60e-03 / 0.00e+00 | 20260505 04:07:06.140 | INFO | PerfReport: 10.77%, Baseline -- 3.30e+00 / -nan, 1.93e-02 / 0.00e+00 | 20260505 04:07:06.140 | INFO | PerfReport: 3.02%, Histogram -- 9.24e-01 / -nan, 5.20e-02 / 0.00e+00 | 20260505 04:07:06.140 | INFO | PerfReport: 16.02%, DME -- 3.92e+01 / -nan, 3.19e+00 / 0.00e+00 | 20260505 04:07:06.140 | INFO | PerfReport: 56.27%, FrameLabeling -- 1.72e+01 / -nan, 4.53e-02 / 0.00e+00 | 20260505 04:07:06.140 | INFO | PerfReport: 7.75%, PulseAccumulating -- 2.37e+00 / -nan, 1.60e-01 / 0.00e+00 | 20260505 04:07:06.140 | INFO | PerfReport: 4.15%, Metrics -- 1.27e+00 / -nan, 2.07e-01 / 0.00e+00 | 20260505 04:07:06.140 | INFO | PerfReport: 1.73%, PulseSerializing -- 5.30e-01 / -nan, 8.49e-02 / 0.00e+00 | 20260505 04:07:06.140 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 04:07:06.140 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:07:06.140 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:07:06.140 | INFO | PerfReport: Full Execution -- 9.19e+01 / -nan, 1.91e+01 / 0.00e+00 | 20260505 04:07:06.140 | INFO | PerfReport: 18.18%, Upload -- 1.67e+01 / 0.00e+00, 3.66e-01 / 0.00e+00 | 20260505 04:07:06.140 | INFO | PerfReport: 17.44%, Download -- 1.60e+01 / 0.00e+00, 6.87e-01 / 0.00e+00 | 20260505 04:07:06.140 | INFO | PerfReport: 33.29%, Compute -- 3.06e+01 / 0.00e+00, 1.28e+01 / 0.00e+00 | 20260505 04:07:06.140 | INFO | PerfReport: 0.55%, LockUpload -- 5.04e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260505 04:07:06.140 | INFO | PerfReport: 0.00%, LockDownload -- 3.99e-04 / 0.00e+00, 1.03e-04 / 0.00e+00 | 20260505 04:07:06.140 | INFO | PerfReport: 30.53%, LockCompute -- 2.81e+01 / 0.00e+00, 1.73e+01 / 0.00e+00 | 20260505 04:07:06.140 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.26e-02 / 0.00e+00 | 20260505 04:07:06.237 | INFO | Starting a round of Baz IO | 20260505 04:07:09.202 | INFO | Populating RTMetrics report | 20260505 04:07:09.202 | INFO | RTMetrics report populated | 20260505 04:07:09.202 | INFO | Opening tmp RTMetrics json file | 20260505 04:07:09.203 | INFO | Serializing RTMetrics to json | 20260505 04:07:09.204 | INFO | Serialization done, swapping tmp file now... | 20260505 04:07:09.205 | INFO | RTMetrics done | 20260505 04:07:09.281 | INFO | Finished a round of Baz IO: Wrote 1.756 GiB in 3.044 seconds | 20260505 04:07:09.281 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 54.33% / 0.26% / 45.42% | 20260505 04:07:09.281 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 12.9849% / 0.000790755% / 2.38419e-05% | 20260505 04:07:09.281 | INFO | Baz IO overall: | 20260505 04:07:09.281 | INFO | BytesWritten : 67422522045 | 20260505 04:07:09.281 | INFO | NumEvents : 30429526164 | 20260505 04:07:11.278 | INFO | Analyzing chunk frames = [212992,213504) | 20260505 04:07:11.278 | INFO | MemInfo: { | 20260505 04:07:11.278 | INFO | "gpu" : " 17 GB", | 20260505 04:07:11.278 | INFO | "huge" : "", | 20260505 04:07:11.278 | INFO | "huge_shared" : " 27 GB", | 20260505 04:07:11.278 | INFO | "malloc" : " 14 GB", | 20260505 04:07:11.278 | INFO | "pinned" : " 29 GB" | 20260505 04:07:11.278 | INFO | } | 20260505 04:07:11.278 | INFO | Perf report for the last 5.13922 seconds: | 20260505 04:07:11.278 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:07:11.278 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:11.278 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:11.278 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:11.278 | INFO | ANALYSIS : 58.5%, 3.0, 91.3ms, 0 | 20260505 04:07:11.278 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:11.278 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260505 04:07:11.278 | INFO | PRE_HQ : 58.0%, 2.2, 64.9ms, 0 | 20260505 04:07:11.278 | INFO | BAZWRITER : 58.8%, 3.5, 104.3ms, 0 | 20260505 04:07:11.278 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:07:11.278 | INFO | Uploaded 36GiB to GPU in 1.63549s (22.0117GiB/s) | 20260505 04:07:11.278 | INFO | Downloaded 38GiB to GPU in 1.6966s (22.3977GiB/s) | 20260505 04:07:16.417 | INFO | Analyzing chunk frames = [213504,214016) | 20260505 04:07:16.417 | INFO | MemInfo: { | 20260505 04:07:16.417 | INFO | "gpu" : " 17 GB", | 20260505 04:07:16.417 | INFO | "huge" : "", | 20260505 04:07:16.417 | INFO | "huge_shared" : " 27 GB", | 20260505 04:07:16.417 | INFO | "malloc" : " 14 GB", | 20260505 04:07:16.417 | INFO | "pinned" : " 29 GB" | 20260505 04:07:16.417 | INFO | } | 20260505 04:07:16.417 | INFO | Perf report for the last 5.13958 seconds: | 20260505 04:07:16.418 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:07:16.418 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:16.418 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:16.418 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:16.418 | INFO | ANALYSIS : 58.3%, 3.0, 90.9ms, 0 | 20260505 04:07:16.418 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:16.418 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:16.418 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 04:07:16.418 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:16.418 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:16.418 | INFO | Uploaded 36GiB to GPU in 1.59737s (22.537GiB/s) | 20260505 04:07:16.418 | INFO | Downloaded 34GiB to GPU in 1.51391s (22.4584GiB/s) | 20260505 04:07:21.556 | INFO | Analyzing chunk frames = [214016,214528) | 20260505 04:07:21.557 | INFO | MemInfo: { | 20260505 04:07:21.557 | INFO | "gpu" : " 17 GB", | 20260505 04:07:21.557 | INFO | "huge" : "", | 20260505 04:07:21.557 | INFO | "huge_shared" : " 27 GB", | 20260505 04:07:21.557 | INFO | "malloc" : " 14 GB", | 20260505 04:07:21.557 | INFO | "pinned" : " 29 GB" | 20260505 04:07:21.557 | INFO | } | 20260505 04:07:21.557 | INFO | Perf report for the last 5.13933 seconds: | 20260505 04:07:21.557 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:07:21.557 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:21.557 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:21.557 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:21.557 | INFO | ANALYSIS : 57.7%, 3.0, 89.6ms, 0 | 20260505 04:07:21.557 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:21.557 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:21.557 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 04:07:21.557 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:21.557 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:21.557 | INFO | Uploaded 36GiB to GPU in 1.60035s (22.495GiB/s) | 20260505 04:07:21.557 | INFO | Downloaded 34GiB to GPU in 1.51741s (22.4066GiB/s) | 20260505 04:07:26.696 | INFO | Analyzing chunk frames = [214528,215040) | 20260505 04:07:26.696 | INFO | MemInfo: { | 20260505 04:07:26.696 | INFO | "gpu" : " 17 GB", | 20260505 04:07:26.696 | INFO | "huge" : "", | 20260505 04:07:26.696 | INFO | "huge_shared" : " 27 GB", | 20260505 04:07:26.696 | INFO | "malloc" : " 14 GB", | 20260505 04:07:26.696 | INFO | "pinned" : " 29 GB" | 20260505 04:07:26.696 | INFO | } | 20260505 04:07:26.696 | INFO | Perf report for the last 5.13924 seconds: | 20260505 04:07:26.696 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:07:26.696 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:26.696 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:26.696 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:26.696 | INFO | ANALYSIS : 57.3%, 3.0, 88.9ms, 0 | 20260505 04:07:26.696 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:26.696 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:26.696 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 04:07:26.696 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:26.696 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:26.697 | INFO | Uploaded 36GiB to GPU in 1.59991s (22.5012GiB/s) | 20260505 04:07:26.697 | INFO | Downloaded 34GiB to GPU in 1.51706s (22.4118GiB/s) | 20260505 04:07:31.835 | INFO | Analyzing chunk frames = [215040,215552) | 20260505 04:07:31.835 | INFO | MemInfo: { | 20260505 04:07:31.835 | INFO | "gpu" : " 17 GB", | 20260505 04:07:31.835 | INFO | "huge" : "", | 20260505 04:07:31.835 | INFO | "huge_shared" : " 27 GB", | 20260505 04:07:31.835 | INFO | "malloc" : " 14 GB", | 20260505 04:07:31.835 | INFO | "pinned" : " 29 GB" | 20260505 04:07:31.835 | INFO | } | 20260505 04:07:31.835 | INFO | Perf report for the last 5.13915 seconds: | 20260505 04:07:31.835 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:07:31.835 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:31.835 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:31.835 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:31.835 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 04:07:31.836 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:31.836 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:31.836 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260505 04:07:31.836 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:31.836 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:31.836 | INFO | Uploaded 36GiB to GPU in 1.59874s (22.5178GiB/s) | 20260505 04:07:31.836 | INFO | Downloaded 34GiB to GPU in 1.51564s (22.4328GiB/s) | 20260505 04:07:36.974 | INFO | Analyzing chunk frames = [215552,216064) | 20260505 04:07:36.974 | INFO | MemInfo: { | 20260505 04:07:36.974 | INFO | "gpu" : " 17 GB", | 20260505 04:07:36.974 | INFO | "huge" : "", | 20260505 04:07:36.974 | INFO | "huge_shared" : " 27 GB", | 20260505 04:07:36.974 | INFO | "malloc" : " 14 GB", | 20260505 04:07:36.974 | INFO | "pinned" : " 29 GB" | 20260505 04:07:36.974 | INFO | } | 20260505 04:07:36.975 | INFO | Perf report for the last 5.13948 seconds: | 20260505 04:07:36.975 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:07:36.975 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:36.975 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:36.975 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:36.975 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260505 04:07:36.975 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:36.975 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:36.975 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260505 04:07:36.975 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:36.975 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:36.975 | INFO | Uploaded 36GiB to GPU in 1.59839s (22.5227GiB/s) | 20260505 04:07:36.975 | INFO | Downloaded 34GiB to GPU in 1.51533s (22.4373GiB/s) | 20260505 04:07:42.114 | INFO | Analyzing chunk frames = [216064,216576) | 20260505 04:07:42.114 | INFO | MemInfo: { | 20260505 04:07:42.114 | INFO | "gpu" : " 17 GB", | 20260505 04:07:42.114 | INFO | "huge" : "", | 20260505 04:07:42.114 | INFO | "huge_shared" : " 27 GB", | 20260505 04:07:42.114 | INFO | "malloc" : " 14 GB", | 20260505 04:07:42.114 | INFO | "pinned" : " 29 GB" | 20260505 04:07:42.114 | INFO | } | 20260505 04:07:42.114 | INFO | Perf report for the last 5.13931 seconds: | 20260505 04:07:42.114 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:07:42.114 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:42.114 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:42.114 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:42.114 | INFO | ANALYSIS : 57.8%, 3.0, 90.2ms, 0 | 20260505 04:07:42.114 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:42.114 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:42.114 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260505 04:07:42.114 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:42.114 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:42.114 | INFO | Uploaded 36GiB to GPU in 1.59683s (22.5447GiB/s) | 20260505 04:07:42.114 | INFO | Downloaded 34GiB to GPU in 1.51603s (22.4271GiB/s) | 20260505 04:07:47.253 | INFO | Analyzing chunk frames = [216576,217088) | 20260505 04:07:47.253 | INFO | MemInfo: { | 20260505 04:07:47.253 | INFO | "gpu" : " 17 GB", | 20260505 04:07:47.253 | INFO | "huge" : "", | 20260505 04:07:47.253 | INFO | "huge_shared" : " 27 GB", | 20260505 04:07:47.253 | INFO | "malloc" : " 14 GB", | 20260505 04:07:47.253 | INFO | "pinned" : " 29 GB" | 20260505 04:07:47.253 | INFO | } | 20260505 04:07:47.254 | INFO | Perf report for the last 5.13945 seconds: | 20260505 04:07:47.254 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:07:47.254 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:47.254 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:47.254 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:47.254 | INFO | ANALYSIS : 57.8%, 3.0, 90.2ms, 0 | 20260505 04:07:47.254 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:47.254 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:47.254 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 04:07:47.254 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:47.254 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:47.254 | INFO | Uploaded 36GiB to GPU in 1.59945s (22.5078GiB/s) | 20260505 04:07:47.254 | INFO | Downloaded 34GiB to GPU in 1.51653s (22.4196GiB/s) | 20260505 04:07:47.350 | INFO | Starting a round of Baz IO | 20260505 04:07:50.316 | INFO | Populating RTMetrics report | 20260505 04:07:50.317 | INFO | RTMetrics report populated | 20260505 04:07:50.317 | INFO | Opening tmp RTMetrics json file | 20260505 04:07:50.317 | INFO | Serializing RTMetrics to json | 20260505 04:07:50.319 | INFO | Serialization done, swapping tmp file now... | 20260505 04:07:50.319 | INFO | RTMetrics done | 20260505 04:07:50.392 | INFO | Finished a round of Baz IO: Wrote 1.764 GiB in 3.041 seconds | 20260505 04:07:50.392 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 54.08% / 0.25% / 45.67% | 20260505 04:07:50.392 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 12.9115% / 0.000727177% / 3.17891e-05% | 20260505 04:07:50.392 | INFO | Baz IO overall: | 20260505 04:07:50.392 | INFO | BytesWritten : 69316142718 | 20260505 04:07:50.392 | INFO | NumEvents : 31322742291 | 20260505 04:07:52.393 | INFO | Analyzing chunk frames = [217088,217600) | 20260505 04:07:52.393 | INFO | MemInfo: { | 20260505 04:07:52.393 | INFO | "gpu" : " 17 GB", | 20260505 04:07:52.393 | INFO | "huge" : "", | 20260505 04:07:52.393 | INFO | "huge_shared" : " 27 GB", | 20260505 04:07:52.393 | INFO | "malloc" : " 14 GB", | 20260505 04:07:52.393 | INFO | "pinned" : " 29 GB" | 20260505 04:07:52.393 | INFO | } | 20260505 04:07:52.393 | INFO | Perf report for the last 5.1393 seconds: | 20260505 04:07:52.393 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:07:52.393 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:52.393 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:52.393 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:52.393 | INFO | ANALYSIS : 58.5%, 3.0, 91.3ms, 0 | 20260505 04:07:52.393 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:52.393 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260505 04:07:52.393 | INFO | PRE_HQ : 58.1%, 2.2, 65.0ms, 0 | 20260505 04:07:52.393 | INFO | BAZWRITER : 58.7%, 3.5, 104.4ms, 0 | 20260505 04:07:52.393 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:07:52.393 | INFO | Uploaded 36GiB to GPU in 1.63258s (22.0509GiB/s) | 20260505 04:07:52.393 | INFO | Downloaded 38GiB to GPU in 1.69421s (22.4293GiB/s) | 20260505 04:07:57.532 | INFO | Analyzing chunk frames = [217600,218112) | 20260505 04:07:57.532 | INFO | MemInfo: { | 20260505 04:07:57.532 | INFO | "gpu" : " 17 GB", | 20260505 04:07:57.532 | INFO | "huge" : "", | 20260505 04:07:57.532 | INFO | "huge_shared" : " 27 GB", | 20260505 04:07:57.532 | INFO | "malloc" : " 14 GB", | 20260505 04:07:57.532 | INFO | "pinned" : " 29 GB" | 20260505 04:07:57.532 | INFO | } | 20260505 04:07:57.532 | INFO | Perf report for the last 5.13936 seconds: | 20260505 04:07:57.532 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:07:57.532 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:57.532 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:57.532 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:57.532 | INFO | ANALYSIS : 58.2%, 3.0, 90.9ms, 0 | 20260505 04:07:57.532 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:57.532 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:57.532 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260505 04:07:57.532 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:57.532 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:07:57.533 | INFO | Uploaded 36GiB to GPU in 1.597s (22.5423GiB/s) | 20260505 04:07:57.533 | INFO | Downloaded 34GiB to GPU in 1.51354s (22.464GiB/s) | 20260505 04:08:02.671 | INFO | Analyzing chunk frames = [218112,218624) | 20260505 04:08:02.671 | INFO | MemInfo: { | 20260505 04:08:02.671 | INFO | "gpu" : " 17 GB", | 20260505 04:08:02.671 | INFO | "huge" : "", | 20260505 04:08:02.671 | INFO | "huge_shared" : " 27 GB", | 20260505 04:08:02.671 | INFO | "malloc" : " 14 GB", | 20260505 04:08:02.671 | INFO | "pinned" : " 29 GB" | 20260505 04:08:02.671 | INFO | } | 20260505 04:08:02.672 | INFO | Perf report for the last 5.13929 seconds: | 20260505 04:08:02.672 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:08:02.672 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:02.672 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:02.672 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:02.672 | INFO | ANALYSIS : 57.7%, 3.0, 89.7ms, 0 | 20260505 04:08:02.672 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:02.672 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:02.672 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 04:08:02.672 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:02.672 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:02.672 | INFO | Uploaded 36GiB to GPU in 1.60088s (22.4876GiB/s) | 20260505 04:08:02.672 | INFO | Downloaded 34GiB to GPU in 1.51764s (22.4032GiB/s) | 20260505 04:08:07.811 | INFO | Analyzing chunk frames = [218624,219136) | 20260505 04:08:07.811 | INFO | MemInfo: { | 20260505 04:08:07.811 | INFO | "gpu" : " 17 GB", | 20260505 04:08:07.811 | INFO | "huge" : "", | 20260505 04:08:07.811 | INFO | "huge_shared" : " 27 GB", | 20260505 04:08:07.811 | INFO | "malloc" : " 14 GB", | 20260505 04:08:07.811 | INFO | "pinned" : " 29 GB" | 20260505 04:08:07.811 | INFO | } | 20260505 04:08:07.811 | INFO | Perf report for the last 5.13942 seconds: | 20260505 04:08:07.811 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:08:07.811 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:07.811 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:07.811 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:07.811 | INFO | ANALYSIS : 57.3%, 3.0, 88.9ms, 0 | 20260505 04:08:07.811 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:07.811 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:07.811 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260505 04:08:07.811 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:07.811 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:07.812 | INFO | Uploaded 36GiB to GPU in 1.60021s (22.497GiB/s) | 20260505 04:08:07.812 | INFO | Downloaded 34GiB to GPU in 1.51735s (22.4075GiB/s) | 20260505 04:08:12.950 | INFO | Analyzing chunk frames = [219136,219648) | 20260505 04:08:12.950 | INFO | MemInfo: { | 20260505 04:08:12.950 | INFO | "gpu" : " 17 GB", | 20260505 04:08:12.950 | INFO | "huge" : "", | 20260505 04:08:12.950 | INFO | "huge_shared" : " 27 GB", | 20260505 04:08:12.950 | INFO | "malloc" : " 14 GB", | 20260505 04:08:12.950 | INFO | "pinned" : " 29 GB" | 20260505 04:08:12.950 | INFO | } | 20260505 04:08:12.950 | INFO | Perf report for the last 5.13919 seconds: | 20260505 04:08:12.950 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:08:12.950 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:12.950 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:12.950 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:12.950 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 04:08:12.950 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:12.950 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:12.950 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260505 04:08:12.950 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:12.950 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:12.950 | INFO | Uploaded 36GiB to GPU in 1.59805s (22.5274GiB/s) | 20260505 04:08:12.951 | INFO | Downloaded 34GiB to GPU in 1.51419s (22.4542GiB/s) | 20260505 04:08:18.089 | INFO | Analyzing chunk frames = [219648,220160) | 20260505 04:08:18.089 | INFO | MemInfo: { | 20260505 04:08:18.089 | INFO | "gpu" : " 17 GB", | 20260505 04:08:18.089 | INFO | "huge" : "", | 20260505 04:08:18.089 | INFO | "huge_shared" : " 27 GB", | 20260505 04:08:18.089 | INFO | "malloc" : " 14 GB", | 20260505 04:08:18.089 | INFO | "pinned" : " 29 GB" | 20260505 04:08:18.089 | INFO | } | 20260505 04:08:18.090 | INFO | Perf report for the last 5.13935 seconds: | 20260505 04:08:18.090 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:08:18.090 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:18.090 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:18.090 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:18.090 | INFO | ANALYSIS : 57.6%, 3.0, 89.8ms, 0 | 20260505 04:08:18.090 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:18.090 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:18.090 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260505 04:08:18.090 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:18.090 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:18.090 | INFO | Uploaded 36GiB to GPU in 1.59919s (22.5114GiB/s) | 20260505 04:08:18.090 | INFO | Downloaded 34GiB to GPU in 1.51612s (22.4256GiB/s) | 20260505 04:08:23.229 | INFO | Analyzing chunk frames = [220160,220672) | 20260505 04:08:23.229 | INFO | MemInfo: { | 20260505 04:08:23.229 | INFO | "gpu" : " 17 GB", | 20260505 04:08:23.229 | INFO | "huge" : "", | 20260505 04:08:23.229 | INFO | "huge_shared" : " 27 GB", | 20260505 04:08:23.229 | INFO | "malloc" : " 14 GB", | 20260505 04:08:23.229 | INFO | "pinned" : " 29 GB" | 20260505 04:08:23.229 | INFO | } | 20260505 04:08:23.229 | INFO | Perf report for the last 5.13936 seconds: | 20260505 04:08:23.229 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:08:23.229 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:23.229 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:23.229 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:23.229 | INFO | ANALYSIS : 57.7%, 3.0, 90.1ms, 0 | 20260505 04:08:23.229 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:23.229 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:23.229 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 04:08:23.229 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:23.229 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:23.229 | INFO | Uploaded 36GiB to GPU in 1.59894s (22.515GiB/s) | 20260505 04:08:23.229 | INFO | Downloaded 34GiB to GPU in 1.51614s (22.4254GiB/s) | 20260505 04:08:28.368 | INFO | Analyzing chunk frames = [220672,221184) | 20260505 04:08:28.368 | INFO | MemInfo: { | 20260505 04:08:28.368 | INFO | "gpu" : " 17 GB", | 20260505 04:08:28.368 | INFO | "huge" : "", | 20260505 04:08:28.368 | INFO | "huge_shared" : " 27 GB", | 20260505 04:08:28.368 | INFO | "malloc" : " 14 GB", | 20260505 04:08:28.368 | INFO | "pinned" : " 29 GB" | 20260505 04:08:28.368 | INFO | } | 20260505 04:08:28.368 | INFO | Perf report for the last 5.13946 seconds: | 20260505 04:08:28.368 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:08:28.368 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:28.368 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:28.368 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:28.368 | INFO | ANALYSIS : 57.9%, 3.0, 90.3ms, 0 | 20260505 04:08:28.368 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:28.368 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:28.369 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260505 04:08:28.369 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:28.369 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:28.369 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 04:08:28.369 | INFO | Total mem usage: 14.9 GiB | 20260505 04:08:28.369 | INFO | All Time High Water Marks sum: 15.2 GiB (Savings: 1.66%) | 20260505 04:08:28.369 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -3.09%) | 20260505 04:08:28.369 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 04:08:28.369 | INFO | High Water Breakdown: | 20260505 04:08:28.369 | INFO | 58.5% : 8.47 GiB ( 99.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 04:08:28.369 | INFO | 41.3% : 5.99 GiB ( 90.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 04:08:28.369 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 04:08:28.369 | INFO | ------------------------------------------ | 20260505 04:08:28.369 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:08:28.369 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 04:08:28.369 | INFO | Total mem usage: 26.5 GiB | 20260505 04:08:28.369 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260505 04:08:28.369 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 04:08:28.369 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260505 04:08:28.369 | INFO | High Water Breakdown: | 20260505 04:08:28.369 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:08:28.369 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:08:28.369 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:08:28.369 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:08:28.369 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:08:28.369 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:08:28.369 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 04:08:28.369 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 04:08:28.369 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:08:28.369 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 04:08:28.369 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 04:08:28.369 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 04:08:28.369 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 04:08:28.369 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 04:08:28.369 | INFO | ------------------------------------------ | 20260505 04:08:28.369 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:08:28.369 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 04:08:28.369 | INFO | Total mem usage: 25.6 GiB | 20260505 04:08:28.369 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 04:08:28.369 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 04:08:28.369 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 04:08:28.369 | INFO | High Water Breakdown: | 20260505 04:08:28.369 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 04:08:28.369 | INFO | ------------------------------------------ | 20260505 04:08:28.369 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:08:28.369 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 04:08:28.369 | INFO | Total mem usage: 15.7 GiB | 20260505 04:08:28.369 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260505 04:08:28.369 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 04:08:28.369 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260505 04:08:28.369 | INFO | High Water Breakdown: | 20260505 04:08:28.369 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 04:08:28.369 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 04:08:28.369 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 04:08:28.369 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 04:08:28.369 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 04:08:28.369 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 04:08:28.369 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 04:08:28.369 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 04:08:28.369 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 04:08:28.369 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:08:28.369 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 04:08:28.369 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:08:28.369 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 04:08:28.369 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 04:08:28.369 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 04:08:28.369 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 04:08:28.369 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:08:28.369 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 04:08:28.369 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:08:28.369 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:08:28.369 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:08:28.369 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 04:08:28.369 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 04:08:28.369 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:08:28.369 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 04:08:28.369 | INFO | ------------------------------------------ | 20260505 04:08:28.369 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:08:28.369 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 04:08:28.369 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:08:28.369 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:08:28.369 | INFO | PerfReport: Full Execution -- 3.06e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260505 04:08:28.369 | INFO | PerfReport: 0.28%, AggregateStats -- 8.63e-02 / -nan, 4.71e-03 / 0.00e+00 | 20260505 04:08:28.369 | INFO | PerfReport: 10.78%, Baseline -- 3.30e+00 / -nan, 2.00e-02 / 0.00e+00 | 20260505 04:08:28.369 | INFO | PerfReport: 3.02%, Histogram -- 9.24e-01 / -nan, 5.18e-02 / 0.00e+00 | 20260505 04:08:28.369 | INFO | PerfReport: 16.00%, DME -- 3.92e+01 / -nan, 3.26e+00 / 0.00e+00 | 20260505 04:08:28.369 | INFO | PerfReport: 56.29%, FrameLabeling -- 1.72e+01 / -nan, 4.60e-02 / 0.00e+00 | 20260505 04:08:28.369 | INFO | PerfReport: 7.75%, PulseAccumulating -- 2.37e+00 / -nan, 1.60e-01 / 0.00e+00 | 20260505 04:08:28.369 | INFO | PerfReport: 4.15%, Metrics -- 1.27e+00 / -nan, 2.06e-01 / 0.00e+00 | 20260505 04:08:28.369 | INFO | PerfReport: 1.73%, PulseSerializing -- 5.28e-01 / -nan, 8.47e-02 / 0.00e+00 | 20260505 04:08:28.369 | INFO | Uploaded 36GiB to GPU in 1.59939s (22.5086GiB/s) | 20260505 04:08:28.369 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 04:08:28.369 | INFO | Downloaded 34GiB to GPU in 1.51611s (22.4258GiB/s) | 20260505 04:08:28.369 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:08:28.369 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:08:28.369 | INFO | PerfReport: Full Execution -- 9.19e+01 / -nan, 1.91e+01 / 0.00e+00 | 20260505 04:08:28.369 | INFO | PerfReport: 18.17%, Upload -- 1.67e+01 / 0.00e+00, 3.57e-01 / 0.00e+00 | 20260505 04:08:28.369 | INFO | PerfReport: 17.44%, Download -- 1.60e+01 / 0.00e+00, 6.75e-01 / 0.00e+00 | 20260505 04:08:28.369 | INFO | PerfReport: 33.29%, Compute -- 3.06e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260505 04:08:28.369 | INFO | PerfReport: 0.55%, LockUpload -- 5.05e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260505 04:08:28.369 | INFO | PerfReport: 0.00%, LockDownload -- 3.95e-04 / 0.00e+00, 1.57e-04 / 0.00e+00 | 20260505 04:08:28.369 | INFO | PerfReport: 30.53%, LockCompute -- 2.81e+01 / 0.00e+00, 1.73e+01 / 0.00e+00 | 20260505 04:08:28.369 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.39e-02 / 0.00e+00 | 20260505 04:08:28.466 | INFO | Starting a round of Baz IO | 20260505 04:08:31.429 | INFO | Populating RTMetrics report | 20260505 04:08:31.430 | INFO | RTMetrics report populated | 20260505 04:08:31.430 | INFO | Opening tmp RTMetrics json file | 20260505 04:08:31.430 | INFO | Serializing RTMetrics to json | 20260505 04:08:31.431 | INFO | Serialization done, swapping tmp file now... | 20260505 04:08:31.432 | INFO | RTMetrics done | 20260505 04:08:31.506 | INFO | Finished a round of Baz IO: Wrote 1.773 GiB in 3.04 seconds | 20260505 04:08:31.507 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 53.83% / 0.24% / 45.92% | 20260505 04:08:31.507 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 12.8831% / 0.000735124% / 5.16574e-05% | 20260505 04:08:31.507 | INFO | Baz IO overall: | 20260505 04:08:31.507 | INFO | BytesWritten : 71219491310 | 20260505 04:08:31.507 | INFO | NumEvents : 32221099218 | 20260505 04:08:33.507 | INFO | Analyzing chunk frames = [221184,221696) | 20260505 04:08:33.507 | INFO | MemInfo: { | 20260505 04:08:33.507 | INFO | "gpu" : " 17 GB", | 20260505 04:08:33.507 | INFO | "huge" : "", | 20260505 04:08:33.507 | INFO | "huge_shared" : " 27 GB", | 20260505 04:08:33.507 | INFO | "malloc" : " 14 GB", | 20260505 04:08:33.507 | INFO | "pinned" : " 29 GB" | 20260505 04:08:33.507 | INFO | } | 20260505 04:08:33.508 | INFO | Perf report for the last 5.13914 seconds: | 20260505 04:08:33.508 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:08:33.508 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:33.508 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:33.508 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:33.508 | INFO | ANALYSIS : 58.5%, 3.0, 91.2ms, 0 | 20260505 04:08:33.508 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:33.508 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260505 04:08:33.508 | INFO | PRE_HQ : 57.9%, 2.2, 64.8ms, 0 | 20260505 04:08:33.508 | INFO | BAZWRITER : 58.7%, 3.5, 104.3ms, 0 | 20260505 04:08:33.508 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:08:33.508 | INFO | Uploaded 36GiB to GPU in 1.64004s (21.9507GiB/s) | 20260505 04:08:33.508 | INFO | Downloaded 38GiB to GPU in 1.70114s (22.338GiB/s) | 20260505 04:08:38.647 | INFO | Analyzing chunk frames = [221696,222208) | 20260505 04:08:38.647 | INFO | MemInfo: { | 20260505 04:08:38.647 | INFO | "gpu" : " 17 GB", | 20260505 04:08:38.647 | INFO | "huge" : "", | 20260505 04:08:38.647 | INFO | "huge_shared" : " 27 GB", | 20260505 04:08:38.647 | INFO | "malloc" : " 14 GB", | 20260505 04:08:38.647 | INFO | "pinned" : " 29 GB" | 20260505 04:08:38.647 | INFO | } | 20260505 04:08:38.647 | INFO | Perf report for the last 5.13945 seconds: | 20260505 04:08:38.647 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:08:38.647 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:38.647 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:38.647 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:38.647 | INFO | ANALYSIS : 58.3%, 3.0, 90.9ms, 0 | 20260505 04:08:38.647 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:38.647 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:38.647 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260505 04:08:38.647 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:38.647 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:38.647 | INFO | Uploaded 36GiB to GPU in 1.59654s (22.5488GiB/s) | 20260505 04:08:38.647 | INFO | Downloaded 34GiB to GPU in 1.51337s (22.4664GiB/s) | 20260505 04:08:43.786 | INFO | Analyzing chunk frames = [222208,222720) | 20260505 04:08:43.786 | INFO | MemInfo: { | 20260505 04:08:43.786 | INFO | "gpu" : " 17 GB", | 20260505 04:08:43.786 | INFO | "huge" : "", | 20260505 04:08:43.786 | INFO | "huge_shared" : " 27 GB", | 20260505 04:08:43.786 | INFO | "malloc" : " 14 GB", | 20260505 04:08:43.786 | INFO | "pinned" : " 29 GB" | 20260505 04:08:43.786 | INFO | } | 20260505 04:08:43.786 | INFO | Perf report for the last 5.13906 seconds: | 20260505 04:08:43.786 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:08:43.786 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:43.786 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:43.786 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:43.786 | INFO | ANALYSIS : 57.6%, 3.0, 89.6ms, 0 | 20260505 04:08:43.786 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:43.786 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:43.786 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 04:08:43.786 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:43.786 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:43.787 | INFO | Uploaded 36GiB to GPU in 1.60073s (22.4897GiB/s) | 20260505 04:08:43.787 | INFO | Downloaded 34GiB to GPU in 1.518s (22.3979GiB/s) | 20260505 04:08:48.925 | INFO | Analyzing chunk frames = [222720,223232) | 20260505 04:08:48.925 | INFO | MemInfo: { | 20260505 04:08:48.925 | INFO | "gpu" : " 17 GB", | 20260505 04:08:48.925 | INFO | "huge" : "", | 20260505 04:08:48.925 | INFO | "huge_shared" : " 27 GB", | 20260505 04:08:48.925 | INFO | "malloc" : " 14 GB", | 20260505 04:08:48.925 | INFO | "pinned" : " 29 GB" | 20260505 04:08:48.926 | INFO | } | 20260505 04:08:48.926 | INFO | Perf report for the last 5.13959 seconds: | 20260505 04:08:48.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:08:48.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:48.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:48.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:48.926 | INFO | ANALYSIS : 57.3%, 3.0, 88.9ms, 0 | 20260505 04:08:48.926 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:48.926 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:48.926 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 04:08:48.926 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:48.926 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:48.926 | INFO | Uploaded 36GiB to GPU in 1.60039s (22.4945GiB/s) | 20260505 04:08:48.926 | INFO | Downloaded 34GiB to GPU in 1.5169s (22.4141GiB/s) | 20260505 04:08:54.065 | INFO | Analyzing chunk frames = [223232,223744) | 20260505 04:08:54.065 | INFO | MemInfo: { | 20260505 04:08:54.065 | INFO | "gpu" : " 17 GB", | 20260505 04:08:54.065 | INFO | "huge" : "", | 20260505 04:08:54.065 | INFO | "huge_shared" : " 27 GB", | 20260505 04:08:54.065 | INFO | "malloc" : " 14 GB", | 20260505 04:08:54.065 | INFO | "pinned" : " 29 GB" | 20260505 04:08:54.065 | INFO | } | 20260505 04:08:54.065 | INFO | Perf report for the last 5.13929 seconds: | 20260505 04:08:54.065 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:08:54.065 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:54.065 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:54.065 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:54.065 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260505 04:08:54.065 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:54.065 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:54.065 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 04:08:54.065 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:54.065 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:54.066 | INFO | Uploaded 36GiB to GPU in 1.5982s (22.5254GiB/s) | 20260505 04:08:54.066 | INFO | Downloaded 34GiB to GPU in 1.51411s (22.4554GiB/s) | 20260505 04:08:59.204 | INFO | Analyzing chunk frames = [223744,224256) | 20260505 04:08:59.204 | INFO | MemInfo: { | 20260505 04:08:59.204 | INFO | "gpu" : " 17 GB", | 20260505 04:08:59.204 | INFO | "huge" : "", | 20260505 04:08:59.204 | INFO | "huge_shared" : " 27 GB", | 20260505 04:08:59.204 | INFO | "malloc" : " 14 GB", | 20260505 04:08:59.204 | INFO | "pinned" : " 29 GB" | 20260505 04:08:59.204 | INFO | } | 20260505 04:08:59.204 | INFO | Perf report for the last 5.13942 seconds: | 20260505 04:08:59.205 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:08:59.205 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:59.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:59.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:59.205 | INFO | ANALYSIS : 57.6%, 3.0, 89.9ms, 0 | 20260505 04:08:59.205 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:59.205 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:59.205 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260505 04:08:59.205 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:59.205 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:08:59.205 | INFO | Uploaded 36GiB to GPU in 1.59821s (22.5251GiB/s) | 20260505 04:08:59.205 | INFO | Downloaded 34GiB to GPU in 1.51771s (22.4022GiB/s) | 20260505 04:09:04.343 | INFO | Analyzing chunk frames = [224256,224768) | 20260505 04:09:04.343 | INFO | MemInfo: { | 20260505 04:09:04.343 | INFO | "gpu" : " 17 GB", | 20260505 04:09:04.343 | INFO | "huge" : "", | 20260505 04:09:04.343 | INFO | "huge_shared" : " 27 GB", | 20260505 04:09:04.343 | INFO | "malloc" : " 14 GB", | 20260505 04:09:04.343 | INFO | "pinned" : " 29 GB" | 20260505 04:09:04.343 | INFO | } | 20260505 04:09:04.344 | INFO | Perf report for the last 5.13933 seconds: | 20260505 04:09:04.344 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:09:04.344 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:04.344 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:04.344 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:04.344 | INFO | ANALYSIS : 57.7%, 3.0, 90.1ms, 0 | 20260505 04:09:04.344 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:04.344 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:04.344 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260505 04:09:04.344 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:04.344 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:04.344 | INFO | Uploaded 36GiB to GPU in 1.59907s (22.5131GiB/s) | 20260505 04:09:04.344 | INFO | Downloaded 34GiB to GPU in 1.51606s (22.4266GiB/s) | 20260505 04:09:09.483 | INFO | Analyzing chunk frames = [224768,225280) | 20260505 04:09:09.483 | INFO | MemInfo: { | 20260505 04:09:09.483 | INFO | "gpu" : " 17 GB", | 20260505 04:09:09.483 | INFO | "huge" : "", | 20260505 04:09:09.483 | INFO | "huge_shared" : " 27 GB", | 20260505 04:09:09.483 | INFO | "malloc" : " 14 GB", | 20260505 04:09:09.483 | INFO | "pinned" : " 29 GB" | 20260505 04:09:09.483 | INFO | } | 20260505 04:09:09.483 | INFO | Perf report for the last 5.13906 seconds: | 20260505 04:09:09.483 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:09:09.483 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:09.483 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:09.483 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:09.483 | INFO | ANALYSIS : 57.9%, 3.0, 90.3ms, 0 | 20260505 04:09:09.483 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:09.483 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:09.483 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260505 04:09:09.483 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:09.483 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:09.484 | INFO | Uploaded 36GiB to GPU in 1.59917s (22.5116GiB/s) | 20260505 04:09:09.484 | INFO | Downloaded 34GiB to GPU in 1.51631s (22.4229GiB/s) | 20260505 04:09:09.580 | INFO | Starting a round of Baz IO | 20260505 04:09:12.551 | INFO | Populating RTMetrics report | 20260505 04:09:12.551 | INFO | RTMetrics report populated | 20260505 04:09:12.551 | INFO | Opening tmp RTMetrics json file | 20260505 04:09:12.551 | INFO | Serializing RTMetrics to json | 20260505 04:09:12.553 | INFO | Serialization done, swapping tmp file now... | 20260505 04:09:12.553 | INFO | RTMetrics done | 20260505 04:09:12.629 | INFO | Finished a round of Baz IO: Wrote 1.781 GiB in 3.049 seconds | 20260505 04:09:12.630 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 53.60% / 0.24% / 46.17% | 20260505 04:09:12.630 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 12.8436% / 0.000754992% / 3.57628e-05% | 20260505 04:09:12.630 | INFO | Baz IO overall: | 20260505 04:09:12.630 | INFO | BytesWritten : 73131597972 | 20260505 04:09:12.630 | INFO | NumEvents : 33123776853 | 20260505 04:09:14.622 | INFO | Analyzing chunk frames = [225280,225792) | 20260505 04:09:14.622 | INFO | MemInfo: { | 20260505 04:09:14.622 | INFO | "gpu" : " 17 GB", | 20260505 04:09:14.622 | INFO | "huge" : "", | 20260505 04:09:14.622 | INFO | "huge_shared" : " 27 GB", | 20260505 04:09:14.622 | INFO | "malloc" : " 14 GB", | 20260505 04:09:14.622 | INFO | "pinned" : " 29 GB" | 20260505 04:09:14.622 | INFO | } | 20260505 04:09:14.622 | INFO | Perf report for the last 5.13959 seconds: | 20260505 04:09:14.622 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:09:14.622 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:14.622 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:14.622 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:14.623 | INFO | ANALYSIS : 58.6%, 3.0, 91.4ms, 0 | 20260505 04:09:14.623 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:14.623 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260505 04:09:14.623 | INFO | PRE_HQ : 58.1%, 2.1, 64.8ms, 0 | 20260505 04:09:14.623 | INFO | BAZWRITER : 58.9%, 3.5, 105.1ms, 0 | 20260505 04:09:14.623 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:09:14.623 | INFO | Uploaded 36GiB to GPU in 1.63837s (21.973GiB/s) | 20260505 04:09:14.623 | INFO | Downloaded 38GiB to GPU in 1.69844s (22.3735GiB/s) | 20260505 04:09:19.761 | INFO | Analyzing chunk frames = [225792,226304) | 20260505 04:09:19.761 | INFO | MemInfo: { | 20260505 04:09:19.761 | INFO | "gpu" : " 17 GB", | 20260505 04:09:19.761 | INFO | "huge" : "", | 20260505 04:09:19.761 | INFO | "huge_shared" : " 27 GB", | 20260505 04:09:19.761 | INFO | "malloc" : " 14 GB", | 20260505 04:09:19.761 | INFO | "pinned" : " 29 GB" | 20260505 04:09:19.761 | INFO | } | 20260505 04:09:19.762 | INFO | Perf report for the last 5.13928 seconds: | 20260505 04:09:19.762 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:09:19.762 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:19.762 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:19.762 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:19.762 | INFO | ANALYSIS : 58.3%, 3.0, 90.9ms, 0 | 20260505 04:09:19.762 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:19.762 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:19.762 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 04:09:19.762 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:19.762 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:19.762 | INFO | Uploaded 36GiB to GPU in 1.59665s (22.5473GiB/s) | 20260505 04:09:19.762 | INFO | Downloaded 34GiB to GPU in 1.51299s (22.472GiB/s) | 20260505 04:09:24.901 | INFO | Analyzing chunk frames = [226304,226816) | 20260505 04:09:24.901 | INFO | MemInfo: { | 20260505 04:09:24.901 | INFO | "gpu" : " 17 GB", | 20260505 04:09:24.901 | INFO | "huge" : "", | 20260505 04:09:24.901 | INFO | "huge_shared" : " 27 GB", | 20260505 04:09:24.901 | INFO | "malloc" : " 14 GB", | 20260505 04:09:24.901 | INFO | "pinned" : " 29 GB" | 20260505 04:09:24.901 | INFO | } | 20260505 04:09:24.901 | INFO | Perf report for the last 5.13928 seconds: | 20260505 04:09:24.901 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:09:24.901 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:24.901 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:24.901 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:24.901 | INFO | ANALYSIS : 57.6%, 3.0, 89.5ms, 0 | 20260505 04:09:24.901 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:24.901 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:24.901 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260505 04:09:24.901 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:24.901 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:24.902 | INFO | Uploaded 36GiB to GPU in 1.6006s (22.4916GiB/s) | 20260505 04:09:24.902 | INFO | Downloaded 34GiB to GPU in 1.51721s (22.4096GiB/s) | 20260505 04:09:30.040 | INFO | Analyzing chunk frames = [226816,227328) | 20260505 04:09:30.040 | INFO | MemInfo: { | 20260505 04:09:30.040 | INFO | "gpu" : " 17 GB", | 20260505 04:09:30.040 | INFO | "huge" : "", | 20260505 04:09:30.040 | INFO | "huge_shared" : " 27 GB", | 20260505 04:09:30.040 | INFO | "malloc" : " 14 GB", | 20260505 04:09:30.040 | INFO | "pinned" : " 29 GB" | 20260505 04:09:30.040 | INFO | } | 20260505 04:09:30.040 | INFO | Perf report for the last 5.13936 seconds: | 20260505 04:09:30.040 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:09:30.040 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:30.040 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:30.041 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:30.041 | INFO | ANALYSIS : 57.4%, 3.0, 89.0ms, 0 | 20260505 04:09:30.041 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:30.041 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:30.041 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 04:09:30.041 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:30.041 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:30.041 | INFO | Uploaded 36GiB to GPU in 1.59987s (22.5019GiB/s) | 20260505 04:09:30.041 | INFO | Downloaded 34GiB to GPU in 1.51667s (22.4175GiB/s) | 20260505 04:09:35.180 | INFO | Analyzing chunk frames = [227328,227840) | 20260505 04:09:35.180 | INFO | MemInfo: { | 20260505 04:09:35.180 | INFO | "gpu" : " 17 GB", | 20260505 04:09:35.180 | INFO | "huge" : "", | 20260505 04:09:35.180 | INFO | "huge_shared" : " 27 GB", | 20260505 04:09:35.180 | INFO | "malloc" : " 14 GB", | 20260505 04:09:35.180 | INFO | "pinned" : " 29 GB" | 20260505 04:09:35.180 | INFO | } | 20260505 04:09:35.180 | INFO | Perf report for the last 5.13947 seconds: | 20260505 04:09:35.180 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:09:35.180 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:35.180 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:35.180 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:35.180 | INFO | ANALYSIS : 57.2%, 3.0, 89.3ms, 0 | 20260505 04:09:35.180 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:35.180 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:35.180 | INFO | PRE_HQ : 32.7%, 1.0, 17.0ms, 0 | 20260505 04:09:35.180 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:35.180 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:35.181 | INFO | Uploaded 36GiB to GPU in 1.59876s (22.5174GiB/s) | 20260505 04:09:35.181 | INFO | Downloaded 34GiB to GPU in 1.51482s (22.4449GiB/s) | 20260505 04:09:40.340 | INFO | Analyzing chunk frames = [227840,228352) | 20260505 04:09:40.340 | INFO | MemInfo: { | 20260505 04:09:40.340 | INFO | "gpu" : " 17 GB", | 20260505 04:09:40.341 | INFO | "huge" : "", | 20260505 04:09:40.341 | INFO | "huge_shared" : " 27 GB", | 20260505 04:09:40.341 | INFO | "malloc" : " 14 GB", | 20260505 04:09:40.341 | INFO | "pinned" : " 29 GB" | 20260505 04:09:40.341 | INFO | } | 20260505 04:09:40.341 | INFO | Perf report for the last 5.16075 seconds: | 20260505 04:09:40.341 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:09:40.341 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:40.341 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:40.341 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:40.341 | INFO | ANALYSIS : 57.3%, 3.0, 89.8ms, 0 | 20260505 04:09:40.341 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:40.341 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:40.341 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260505 04:09:40.341 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:40.341 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:40.341 | INFO | Uploaded 36GiB to GPU in 1.59876s (22.5175GiB/s) | 20260505 04:09:40.341 | INFO | Downloaded 34GiB to GPU in 1.51496s (22.4428GiB/s) | 20260505 04:09:45.459 | INFO | Analyzing chunk frames = [228352,228864) | 20260505 04:09:45.459 | INFO | MemInfo: { | 20260505 04:09:45.459 | INFO | "gpu" : " 17 GB", | 20260505 04:09:45.459 | INFO | "huge" : "", | 20260505 04:09:45.459 | INFO | "huge_shared" : " 27 GB", | 20260505 04:09:45.459 | INFO | "malloc" : " 14 GB", | 20260505 04:09:45.459 | INFO | "pinned" : " 29 GB" | 20260505 04:09:45.459 | INFO | } | 20260505 04:09:45.459 | INFO | Perf report for the last 5.11841 seconds: | 20260505 04:09:45.459 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:09:45.459 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:45.459 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:45.459 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:45.459 | INFO | ANALYSIS : 58.0%, 3.0, 90.0ms, 0 | 20260505 04:09:45.459 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:45.459 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:45.459 | INFO | PRE_HQ : 32.5%, 1.0, 16.8ms, 0 | 20260505 04:09:45.459 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:45.459 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:45.459 | INFO | Uploaded 36GiB to GPU in 1.59948s (22.5073GiB/s) | 20260505 04:09:45.459 | INFO | Downloaded 34GiB to GPU in 1.51763s (22.4033GiB/s) | 20260505 04:09:50.598 | INFO | Analyzing chunk frames = [228864,229376) | 20260505 04:09:50.598 | INFO | MemInfo: { | 20260505 04:09:50.598 | INFO | "gpu" : " 17 GB", | 20260505 04:09:50.598 | INFO | "huge" : "", | 20260505 04:09:50.598 | INFO | "huge_shared" : " 27 GB", | 20260505 04:09:50.598 | INFO | "malloc" : " 14 GB", | 20260505 04:09:50.598 | INFO | "pinned" : " 29 GB" | 20260505 04:09:50.598 | INFO | } | 20260505 04:09:50.598 | INFO | Perf report for the last 5.13881 seconds: | 20260505 04:09:50.598 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:09:50.598 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:50.598 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:50.598 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:50.598 | INFO | ANALYSIS : 57.9%, 3.0, 90.3ms, 0 | 20260505 04:09:50.598 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:50.598 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:50.598 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 04:09:50.598 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:50.598 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:50.598 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 04:09:50.598 | INFO | Uploaded 36GiB to GPU in 1.59947s (22.5075GiB/s) | 20260505 04:09:50.598 | INFO | Total mem usage: 14.9 GiB | 20260505 04:09:50.598 | INFO | Downloaded 34GiB to GPU in 1.51609s (22.4261GiB/s) | 20260505 04:09:50.598 | INFO | All Time High Water Marks sum: 15.2 GiB (Savings: 1.66%) | 20260505 04:09:50.598 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -3.08%) | 20260505 04:09:50.598 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 04:09:50.598 | INFO | High Water Breakdown: | 20260505 04:09:50.599 | INFO | 58.6% : 8.48 GiB ( 99.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 04:09:50.599 | INFO | 41.3% : 5.98 GiB ( 89.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 04:09:50.599 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 04:09:50.599 | INFO | ------------------------------------------ | 20260505 04:09:50.599 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:09:50.599 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 04:09:50.599 | INFO | Total mem usage: 26.5 GiB | 20260505 04:09:50.599 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260505 04:09:50.599 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 04:09:50.599 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260505 04:09:50.599 | INFO | High Water Breakdown: | 20260505 04:09:50.599 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:09:50.599 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:09:50.599 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:09:50.599 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:09:50.599 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:09:50.599 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:09:50.599 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 04:09:50.599 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 04:09:50.599 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:09:50.599 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 04:09:50.599 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 04:09:50.599 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 04:09:50.599 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 04:09:50.599 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 04:09:50.599 | INFO | ------------------------------------------ | 20260505 04:09:50.599 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:09:50.599 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 04:09:50.599 | INFO | Total mem usage: 25.6 GiB | 20260505 04:09:50.599 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 04:09:50.599 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 04:09:50.599 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 04:09:50.599 | INFO | High Water Breakdown: | 20260505 04:09:50.599 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 04:09:50.599 | INFO | ------------------------------------------ | 20260505 04:09:50.599 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:09:50.599 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 04:09:50.599 | INFO | Total mem usage: 15.7 GiB | 20260505 04:09:50.599 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260505 04:09:50.599 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 04:09:50.599 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260505 04:09:50.599 | INFO | High Water Breakdown: | 20260505 04:09:50.599 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 04:09:50.599 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 04:09:50.599 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 04:09:50.599 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 04:09:50.599 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 04:09:50.599 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 04:09:50.599 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 04:09:50.599 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 04:09:50.599 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 04:09:50.599 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:09:50.599 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 04:09:50.599 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:09:50.599 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 04:09:50.599 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 04:09:50.599 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 04:09:50.599 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 04:09:50.599 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:09:50.599 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 04:09:50.599 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:09:50.599 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:09:50.599 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:09:50.599 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 04:09:50.599 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 04:09:50.599 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:09:50.599 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 04:09:50.599 | INFO | ------------------------------------------ | 20260505 04:09:50.599 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:09:50.600 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 04:09:50.600 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:09:50.600 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:09:50.600 | INFO | PerfReport: Full Execution -- 3.06e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260505 04:09:50.600 | INFO | PerfReport: 0.28%, AggregateStats -- 8.64e-02 / -nan, 6.02e-03 / 0.00e+00 | 20260505 04:09:50.600 | INFO | PerfReport: 10.78%, Baseline -- 3.30e+00 / -nan, 6.14e-02 / 0.00e+00 | 20260505 04:09:50.600 | INFO | PerfReport: 3.02%, Histogram -- 9.24e-01 / -nan, 5.12e-02 / 0.00e+00 | 20260505 04:09:50.600 | INFO | PerfReport: 16.00%, DME -- 3.92e+01 / -nan, 3.25e+00 / 0.00e+00 | 20260505 04:09:50.600 | INFO | PerfReport: 56.29%, FrameLabeling -- 1.72e+01 / -nan, 5.58e-02 / 0.00e+00 | 20260505 04:09:50.600 | INFO | PerfReport: 7.75%, PulseAccumulating -- 2.37e+00 / -nan, 1.99e-01 / 0.00e+00 | 20260505 04:09:50.600 | INFO | PerfReport: 4.15%, Metrics -- 1.27e+00 / -nan, 2.20e-01 / 0.00e+00 | 20260505 04:09:50.600 | INFO | PerfReport: 1.72%, PulseSerializing -- 5.27e-01 / -nan, 8.47e-02 / 0.00e+00 | 20260505 04:09:50.600 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 04:09:50.600 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:09:50.600 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:09:50.600 | INFO | PerfReport: Full Execution -- 9.19e+01 / -nan, 1.92e+01 / 0.00e+00 | 20260505 04:09:50.600 | INFO | PerfReport: 18.18%, Upload -- 1.67e+01 / 0.00e+00, 3.70e-01 / 0.00e+00 | 20260505 04:09:50.600 | INFO | PerfReport: 17.44%, Download -- 1.60e+01 / 0.00e+00, 7.00e-01 / 0.00e+00 | 20260505 04:09:50.600 | INFO | PerfReport: 33.29%, Compute -- 3.06e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260505 04:09:50.600 | INFO | PerfReport: 0.55%, LockUpload -- 5.03e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260505 04:09:50.600 | INFO | PerfReport: 0.00%, LockDownload -- 3.82e-04 / 0.00e+00, 9.00e-05 / 0.00e+00 | 20260505 04:09:50.600 | INFO | PerfReport: 30.52%, LockCompute -- 2.80e+01 / 0.00e+00, 1.72e+01 / 0.00e+00 | 20260505 04:09:50.600 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.75e-02 / 0.00e+00 | 20260505 04:09:50.695 | INFO | Starting a round of Baz IO | 20260505 04:09:53.661 | INFO | Populating RTMetrics report | 20260505 04:09:53.661 | INFO | RTMetrics report populated | 20260505 04:09:53.661 | INFO | Opening tmp RTMetrics json file | 20260505 04:09:53.662 | INFO | Serializing RTMetrics to json | 20260505 04:09:53.663 | INFO | Serialization done, swapping tmp file now... | 20260505 04:09:53.664 | INFO | RTMetrics done | 20260505 04:09:53.741 | INFO | Finished a round of Baz IO: Wrote 1.786 GiB in 3.045 seconds | 20260505 04:09:53.741 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 53.36% / 0.23% / 46.40% | 20260505 04:09:53.741 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 12.7417% / 0.000651677% / 2.78155e-05% | 20260505 04:09:53.741 | INFO | Baz IO overall: | 20260505 04:09:53.741 | INFO | BytesWritten : 75049459322 | 20260505 04:09:53.741 | INFO | NumEvents : 34028753463 | 20260505 04:09:55.737 | INFO | Analyzing chunk frames = [229376,229888) | 20260505 04:09:55.737 | INFO | MemInfo: { | 20260505 04:09:55.737 | INFO | "gpu" : " 17 GB", | 20260505 04:09:55.737 | INFO | "huge" : "", | 20260505 04:09:55.737 | INFO | "huge_shared" : " 27 GB", | 20260505 04:09:55.737 | INFO | "malloc" : " 14 GB", | 20260505 04:09:55.737 | INFO | "pinned" : " 29 GB" | 20260505 04:09:55.737 | INFO | } | 20260505 04:09:55.737 | INFO | Perf report for the last 5.13916 seconds: | 20260505 04:09:55.737 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:09:55.737 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:55.737 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:55.737 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:55.737 | INFO | ANALYSIS : 58.5%, 3.0, 91.3ms, 0 | 20260505 04:09:55.737 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:09:55.737 | INFO | RT_METRICS : 44.7%, 1.0, 23.4ms, 0 | 20260505 04:09:55.737 | INFO | PRE_HQ : 57.9%, 2.2, 64.9ms, 0 | 20260505 04:09:55.737 | INFO | BAZWRITER : 58.8%, 3.5, 105.5ms, 0 | 20260505 04:09:55.737 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:09:55.737 | INFO | Uploaded 36GiB to GPU in 1.64148s (21.9314GiB/s) | 20260505 04:09:55.737 | INFO | Downloaded 38GiB to GPU in 1.70118s (22.3375GiB/s) | 20260505 04:10:00.876 | INFO | Analyzing chunk frames = [229888,230400) | 20260505 04:10:00.876 | INFO | MemInfo: { | 20260505 04:10:00.876 | INFO | "gpu" : " 17 GB", | 20260505 04:10:00.876 | INFO | "huge" : "", | 20260505 04:10:00.876 | INFO | "huge_shared" : " 27 GB", | 20260505 04:10:00.876 | INFO | "malloc" : " 14 GB", | 20260505 04:10:00.876 | INFO | "pinned" : " 29 GB" | 20260505 04:10:00.876 | INFO | } | 20260505 04:10:00.876 | INFO | Perf report for the last 5.13933 seconds: | 20260505 04:10:00.876 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:10:00.876 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:00.876 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:00.876 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:00.876 | INFO | ANALYSIS : 58.3%, 3.0, 90.9ms, 0 | 20260505 04:10:00.876 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:00.877 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:00.877 | INFO | PRE_HQ : 32.3%, 1.0, 17.0ms, 0 | 20260505 04:10:00.877 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:00.877 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:00.877 | INFO | Uploaded 36GiB to GPU in 1.59705s (22.5415GiB/s) | 20260505 04:10:00.877 | INFO | Downloaded 34GiB to GPU in 1.51317s (22.4694GiB/s) | 20260505 04:10:06.016 | INFO | Analyzing chunk frames = [230400,230912) | 20260505 04:10:06.016 | INFO | MemInfo: { | 20260505 04:10:06.016 | INFO | "gpu" : " 17 GB", | 20260505 04:10:06.016 | INFO | "huge" : "", | 20260505 04:10:06.016 | INFO | "huge_shared" : " 27 GB", | 20260505 04:10:06.016 | INFO | "malloc" : " 14 GB", | 20260505 04:10:06.016 | INFO | "pinned" : " 29 GB" | 20260505 04:10:06.016 | INFO | } | 20260505 04:10:06.016 | INFO | Perf report for the last 5.13941 seconds: | 20260505 04:10:06.016 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:10:06.016 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:06.016 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:06.016 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:06.016 | INFO | ANALYSIS : 57.7%, 3.0, 89.6ms, 0 | 20260505 04:10:06.016 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:06.016 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:06.016 | INFO | PRE_HQ : 31.9%, 1.0, 16.5ms, 0 | 20260505 04:10:06.016 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:06.016 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:06.016 | INFO | Uploaded 36GiB to GPU in 1.60084s (22.4883GiB/s) | 20260505 04:10:06.016 | INFO | Downloaded 34GiB to GPU in 1.51747s (22.4057GiB/s) | 20260505 04:10:11.155 | INFO | Analyzing chunk frames = [230912,231424) | 20260505 04:10:11.155 | INFO | MemInfo: { | 20260505 04:10:11.155 | INFO | "gpu" : " 17 GB", | 20260505 04:10:11.155 | INFO | "huge" : "", | 20260505 04:10:11.155 | INFO | "huge_shared" : " 27 GB", | 20260505 04:10:11.155 | INFO | "malloc" : " 14 GB", | 20260505 04:10:11.155 | INFO | "pinned" : " 29 GB" | 20260505 04:10:11.155 | INFO | } | 20260505 04:10:11.155 | INFO | Perf report for the last 5.13934 seconds: | 20260505 04:10:11.155 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:10:11.155 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:11.155 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:11.155 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:11.155 | INFO | ANALYSIS : 57.3%, 3.0, 88.9ms, 0 | 20260505 04:10:11.155 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:11.155 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:11.155 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260505 04:10:11.155 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:11.155 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:11.156 | INFO | Uploaded 36GiB to GPU in 1.60021s (22.4971GiB/s) | 20260505 04:10:11.156 | INFO | Downloaded 34GiB to GPU in 1.51647s (22.4205GiB/s) | 20260505 04:10:16.294 | INFO | Analyzing chunk frames = [231424,231936) | 20260505 04:10:16.294 | INFO | MemInfo: { | 20260505 04:10:16.294 | INFO | "gpu" : " 17 GB", | 20260505 04:10:16.294 | INFO | "huge" : "", | 20260505 04:10:16.294 | INFO | "huge_shared" : " 27 GB", | 20260505 04:10:16.294 | INFO | "malloc" : " 14 GB", | 20260505 04:10:16.294 | INFO | "pinned" : " 29 GB" | 20260505 04:10:16.294 | INFO | } | 20260505 04:10:16.295 | INFO | Perf report for the last 5.13932 seconds: | 20260505 04:10:16.295 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:10:16.295 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:16.295 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:16.295 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:16.295 | INFO | ANALYSIS : 57.3%, 3.0, 89.3ms, 0 | 20260505 04:10:16.295 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:16.295 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:16.295 | INFO | PRE_HQ : 32.6%, 1.0, 16.9ms, 0 | 20260505 04:10:16.295 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:16.295 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:16.295 | INFO | Uploaded 36GiB to GPU in 1.59883s (22.5164GiB/s) | 20260505 04:10:16.295 | INFO | Downloaded 34GiB to GPU in 1.51482s (22.4449GiB/s) | 20260505 04:10:21.434 | INFO | Analyzing chunk frames = [231936,232448) | 20260505 04:10:21.434 | INFO | MemInfo: { | 20260505 04:10:21.434 | INFO | "gpu" : " 17 GB", | 20260505 04:10:21.434 | INFO | "huge" : "", | 20260505 04:10:21.434 | INFO | "huge_shared" : " 27 GB", | 20260505 04:10:21.434 | INFO | "malloc" : " 14 GB", | 20260505 04:10:21.434 | INFO | "pinned" : " 29 GB" | 20260505 04:10:21.434 | INFO | } | 20260505 04:10:21.434 | INFO | Perf report for the last 5.13938 seconds: | 20260505 04:10:21.434 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:10:21.434 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:21.434 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:21.434 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:21.434 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260505 04:10:21.434 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:21.434 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:21.434 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260505 04:10:21.434 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:21.434 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:21.434 | INFO | Uploaded 36GiB to GPU in 1.59787s (22.53GiB/s) | 20260505 04:10:21.434 | INFO | Downloaded 34GiB to GPU in 1.51507s (22.4412GiB/s) | 20260505 04:10:26.573 | INFO | Analyzing chunk frames = [232448,232960) | 20260505 04:10:26.573 | INFO | MemInfo: { | 20260505 04:10:26.573 | INFO | "gpu" : " 17 GB", | 20260505 04:10:26.573 | INFO | "huge" : "", | 20260505 04:10:26.573 | INFO | "huge_shared" : " 27 GB", | 20260505 04:10:26.573 | INFO | "malloc" : " 14 GB", | 20260505 04:10:26.573 | INFO | "pinned" : " 29 GB" | 20260505 04:10:26.573 | INFO | } | 20260505 04:10:26.573 | INFO | Perf report for the last 5.13932 seconds: | 20260505 04:10:26.573 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:10:26.573 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:26.573 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:26.573 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:26.573 | INFO | ANALYSIS : 57.8%, 3.0, 90.1ms, 0 | 20260505 04:10:26.573 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:26.573 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:26.573 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 04:10:26.573 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:26.573 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:26.574 | INFO | Uploaded 36GiB to GPU in 1.5988s (22.5169GiB/s) | 20260505 04:10:26.574 | INFO | Downloaded 34GiB to GPU in 1.51584s (22.4299GiB/s) | 20260505 04:10:31.712 | INFO | Analyzing chunk frames = [232960,233472) | 20260505 04:10:31.712 | INFO | MemInfo: { | 20260505 04:10:31.712 | INFO | "gpu" : " 17 GB", | 20260505 04:10:31.712 | INFO | "huge" : "", | 20260505 04:10:31.712 | INFO | "huge_shared" : " 27 GB", | 20260505 04:10:31.712 | INFO | "malloc" : " 14 GB", | 20260505 04:10:31.712 | INFO | "pinned" : " 29 GB" | 20260505 04:10:31.712 | INFO | } | 20260505 04:10:31.712 | INFO | Perf report for the last 5.13906 seconds: | 20260505 04:10:31.712 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:10:31.713 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:31.713 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:31.713 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:31.713 | INFO | ANALYSIS : 57.8%, 3.0, 90.2ms, 0 | 20260505 04:10:31.713 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:31.713 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:31.713 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260505 04:10:31.713 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:31.713 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:31.713 | INFO | Uploaded 36GiB to GPU in 1.60037s (22.4948GiB/s) | 20260505 04:10:31.713 | INFO | Downloaded 34GiB to GPU in 1.51698s (22.413GiB/s) | 20260505 04:10:31.822 | INFO | Starting a round of Baz IO | 20260505 04:10:34.774 | INFO | Populating RTMetrics report | 20260505 04:10:34.775 | INFO | RTMetrics report populated | 20260505 04:10:34.775 | INFO | Opening tmp RTMetrics json file | 20260505 04:10:34.776 | INFO | Serializing RTMetrics to json | 20260505 04:10:34.777 | INFO | Serialization done, swapping tmp file now... | 20260505 04:10:34.777 | INFO | RTMetrics done | 20260505 04:10:34.859 | INFO | Finished a round of Baz IO: Wrote 1.795 GiB in 3.037 seconds | 20260505 04:10:34.859 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 53.13% / 0.23% / 46.64% | 20260505 04:10:34.859 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 12.6992% / 0.000790755% / 3.57628e-05% | 20260505 04:10:34.859 | INFO | Baz IO overall: | 20260505 04:10:34.859 | INFO | BytesWritten : 76977152494 | 20260505 04:10:34.859 | INFO | NumEvents : 34938723711 | 20260505 04:10:36.852 | INFO | Analyzing chunk frames = [233472,233984) | 20260505 04:10:36.852 | INFO | MemInfo: { | 20260505 04:10:36.852 | INFO | "gpu" : " 17 GB", | 20260505 04:10:36.852 | INFO | "huge" : "", | 20260505 04:10:36.852 | INFO | "huge_shared" : " 27 GB", | 20260505 04:10:36.852 | INFO | "malloc" : " 14 GB", | 20260505 04:10:36.852 | INFO | "pinned" : " 29 GB" | 20260505 04:10:36.852 | INFO | } | 20260505 04:10:36.852 | INFO | Perf report for the last 5.13952 seconds: | 20260505 04:10:36.852 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:10:36.852 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:36.852 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:36.852 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:36.852 | INFO | ANALYSIS : 58.5%, 3.0, 91.2ms, 0 | 20260505 04:10:36.852 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:36.852 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260505 04:10:36.852 | INFO | PRE_HQ : 57.9%, 2.2, 65.6ms, 0 | 20260505 04:10:36.852 | INFO | BAZWRITER : 59.1%, 3.5, 106.7ms, 0 | 20260505 04:10:36.852 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:10:36.852 | INFO | Uploaded 36GiB to GPU in 1.63818s (21.9756GiB/s) | 20260505 04:10:36.852 | INFO | Downloaded 38GiB to GPU in 1.70123s (22.3368GiB/s) | 20260505 04:10:41.991 | INFO | Analyzing chunk frames = [233984,234496) | 20260505 04:10:41.991 | INFO | MemInfo: { | 20260505 04:10:41.991 | INFO | "gpu" : " 17 GB", | 20260505 04:10:41.991 | INFO | "huge" : "", | 20260505 04:10:41.991 | INFO | "huge_shared" : " 27 GB", | 20260505 04:10:41.991 | INFO | "malloc" : " 14 GB", | 20260505 04:10:41.991 | INFO | "pinned" : " 29 GB" | 20260505 04:10:41.991 | INFO | } | 20260505 04:10:41.991 | INFO | Perf report for the last 5.13938 seconds: | 20260505 04:10:41.991 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:10:41.991 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:41.991 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:41.991 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:41.991 | INFO | ANALYSIS : 58.2%, 3.0, 90.9ms, 0 | 20260505 04:10:41.991 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:41.991 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:41.991 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 04:10:41.991 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:41.991 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:41.991 | INFO | Uploaded 36GiB to GPU in 1.59758s (22.5341GiB/s) | 20260505 04:10:41.992 | INFO | Downloaded 34GiB to GPU in 1.51368s (22.4619GiB/s) | 20260505 04:10:47.130 | INFO | Analyzing chunk frames = [234496,235008) | 20260505 04:10:47.130 | INFO | MemInfo: { | 20260505 04:10:47.130 | INFO | "gpu" : " 17 GB", | 20260505 04:10:47.130 | INFO | "huge" : "", | 20260505 04:10:47.130 | INFO | "huge_shared" : " 27 GB", | 20260505 04:10:47.130 | INFO | "malloc" : " 14 GB", | 20260505 04:10:47.130 | INFO | "pinned" : " 29 GB" | 20260505 04:10:47.130 | INFO | } | 20260505 04:10:47.130 | INFO | Perf report for the last 5.13922 seconds: | 20260505 04:10:47.130 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:10:47.130 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:47.130 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:47.130 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:47.130 | INFO | ANALYSIS : 57.6%, 3.0, 89.6ms, 0 | 20260505 04:10:47.131 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:47.131 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:47.131 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260505 04:10:47.131 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:47.131 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:47.131 | INFO | Uploaded 36GiB to GPU in 1.6007s (22.4901GiB/s) | 20260505 04:10:47.131 | INFO | Downloaded 34GiB to GPU in 1.51832s (22.3932GiB/s) | 20260505 04:10:52.270 | INFO | Analyzing chunk frames = [235008,235520) | 20260505 04:10:52.270 | INFO | MemInfo: { | 20260505 04:10:52.270 | INFO | "gpu" : " 17 GB", | 20260505 04:10:52.270 | INFO | "huge" : "", | 20260505 04:10:52.270 | INFO | "huge_shared" : " 27 GB", | 20260505 04:10:52.270 | INFO | "malloc" : " 14 GB", | 20260505 04:10:52.270 | INFO | "pinned" : " 29 GB" | 20260505 04:10:52.270 | INFO | } | 20260505 04:10:52.270 | INFO | Perf report for the last 5.13937 seconds: | 20260505 04:10:52.270 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:10:52.270 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:52.270 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:52.270 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:52.270 | INFO | ANALYSIS : 57.3%, 3.0, 88.9ms, 0 | 20260505 04:10:52.270 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:52.270 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:52.270 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260505 04:10:52.270 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:52.270 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:52.270 | INFO | Uploaded 36GiB to GPU in 1.6003s (22.4958GiB/s) | 20260505 04:10:52.270 | INFO | Downloaded 34GiB to GPU in 1.51764s (22.4032GiB/s) | 20260505 04:10:57.409 | INFO | Analyzing chunk frames = [235520,236032) | 20260505 04:10:57.409 | INFO | MemInfo: { | 20260505 04:10:57.409 | INFO | "gpu" : " 17 GB", | 20260505 04:10:57.409 | INFO | "huge" : "", | 20260505 04:10:57.409 | INFO | "huge_shared" : " 27 GB", | 20260505 04:10:57.409 | INFO | "malloc" : " 14 GB", | 20260505 04:10:57.409 | INFO | "pinned" : " 29 GB" | 20260505 04:10:57.409 | INFO | } | 20260505 04:10:57.409 | INFO | Perf report for the last 5.13938 seconds: | 20260505 04:10:57.409 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:10:57.409 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:57.409 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:57.409 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:57.409 | INFO | ANALYSIS : 57.3%, 3.0, 89.3ms, 0 | 20260505 04:10:57.409 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:57.409 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:57.409 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260505 04:10:57.409 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:57.409 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:10:57.410 | INFO | Uploaded 36GiB to GPU in 1.5976s (22.5337GiB/s) | 20260505 04:10:57.410 | INFO | Downloaded 34GiB to GPU in 1.51378s (22.4603GiB/s) | 20260505 04:11:02.548 | INFO | Analyzing chunk frames = [236032,236544) | 20260505 04:11:02.548 | INFO | MemInfo: { | 20260505 04:11:02.548 | INFO | "gpu" : " 17 GB", | 20260505 04:11:02.548 | INFO | "huge" : "", | 20260505 04:11:02.548 | INFO | "huge_shared" : " 27 GB", | 20260505 04:11:02.548 | INFO | "malloc" : " 14 GB", | 20260505 04:11:02.549 | INFO | "pinned" : " 29 GB" | 20260505 04:11:02.549 | INFO | } | 20260505 04:11:02.549 | INFO | Perf report for the last 5.13937 seconds: | 20260505 04:11:02.549 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:11:02.549 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:02.549 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:02.549 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:02.549 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260505 04:11:02.549 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:02.549 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:02.549 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260505 04:11:02.549 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:02.549 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:02.549 | INFO | Uploaded 36GiB to GPU in 1.59934s (22.5093GiB/s) | 20260505 04:11:02.549 | INFO | Downloaded 34GiB to GPU in 1.51599s (22.4275GiB/s) | 20260505 04:11:07.688 | INFO | Analyzing chunk frames = [236544,237056) | 20260505 04:11:07.688 | INFO | MemInfo: { | 20260505 04:11:07.688 | INFO | "gpu" : " 17 GB", | 20260505 04:11:07.688 | INFO | "huge" : "", | 20260505 04:11:07.688 | INFO | "huge_shared" : " 27 GB", | 20260505 04:11:07.688 | INFO | "malloc" : " 14 GB", | 20260505 04:11:07.688 | INFO | "pinned" : " 29 GB" | 20260505 04:11:07.688 | INFO | } | 20260505 04:11:07.688 | INFO | Perf report for the last 5.13918 seconds: | 20260505 04:11:07.688 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:11:07.688 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:07.688 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:07.688 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:07.688 | INFO | ANALYSIS : 57.8%, 3.0, 90.1ms, 0 | 20260505 04:11:07.688 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:07.688 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:07.688 | INFO | PRE_HQ : 32.7%, 1.0, 17.0ms, 0 | 20260505 04:11:07.688 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:07.688 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:07.688 | INFO | Uploaded 36GiB to GPU in 1.5976s (22.5338GiB/s) | 20260505 04:11:07.688 | INFO | Downloaded 34GiB to GPU in 1.51629s (22.4231GiB/s) | 20260505 04:11:12.827 | INFO | Analyzing chunk frames = [237056,237568) | 20260505 04:11:12.827 | INFO | MemInfo: { | 20260505 04:11:12.827 | INFO | "gpu" : " 17 GB", | 20260505 04:11:12.827 | INFO | "huge" : "", | 20260505 04:11:12.827 | INFO | "huge_shared" : " 27 GB", | 20260505 04:11:12.827 | INFO | "malloc" : " 14 GB", | 20260505 04:11:12.827 | INFO | "pinned" : " 29 GB" | 20260505 04:11:12.827 | INFO | } | 20260505 04:11:12.827 | INFO | Perf report for the last 5.13945 seconds: | 20260505 04:11:12.827 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:11:12.827 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:12.827 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:12.827 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:12.827 | INFO | ANALYSIS : 57.9%, 3.0, 90.3ms, 0 | 20260505 04:11:12.827 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:12.827 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:12.827 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260505 04:11:12.827 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:12.827 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:12.828 | INFO | Uploaded 36GiB to GPU in 1.60004s (22.4994GiB/s) | 20260505 04:11:12.828 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 04:11:12.828 | INFO | Total mem usage: 14.9 GiB | 20260505 04:11:12.828 | INFO | Downloaded 34GiB to GPU in 1.51733s (22.4078GiB/s) | 20260505 04:11:12.828 | INFO | All Time High Water Marks sum: 15.2 GiB (Savings: 1.89%) | 20260505 04:11:12.828 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -2.56%) | 20260505 04:11:12.828 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 04:11:12.828 | INFO | High Water Breakdown: | 20260505 04:11:12.828 | INFO | 58.7% : 8.55 GiB ( 100% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 04:11:12.828 | INFO | 41.1% : 5.99 GiB ( 90.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 04:11:12.828 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 04:11:12.828 | INFO | ------------------------------------------ | 20260505 04:11:12.828 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:11:12.828 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 04:11:12.828 | INFO | Total mem usage: 26.5 GiB | 20260505 04:11:12.828 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260505 04:11:12.828 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 04:11:12.828 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260505 04:11:12.828 | INFO | High Water Breakdown: | 20260505 04:11:12.828 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:11:12.828 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:11:12.828 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:11:12.828 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:11:12.828 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:11:12.828 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:11:12.828 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 04:11:12.828 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 04:11:12.828 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:11:12.828 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 04:11:12.828 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 04:11:12.828 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 04:11:12.828 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 04:11:12.828 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 04:11:12.828 | INFO | ------------------------------------------ | 20260505 04:11:12.828 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:11:12.828 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 04:11:12.828 | INFO | Total mem usage: 25.6 GiB | 20260505 04:11:12.828 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 04:11:12.828 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 04:11:12.828 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 04:11:12.828 | INFO | High Water Breakdown: | 20260505 04:11:12.828 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 04:11:12.828 | INFO | ------------------------------------------ | 20260505 04:11:12.828 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:11:12.828 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 04:11:12.828 | INFO | Total mem usage: 15.7 GiB | 20260505 04:11:12.828 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260505 04:11:12.828 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 04:11:12.828 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260505 04:11:12.828 | INFO | High Water Breakdown: | 20260505 04:11:12.829 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 04:11:12.829 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 04:11:12.829 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 04:11:12.829 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 04:11:12.829 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 04:11:12.829 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 04:11:12.829 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 04:11:12.829 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 04:11:12.829 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 04:11:12.829 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:11:12.829 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 04:11:12.829 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:11:12.829 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 04:11:12.829 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 04:11:12.829 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 04:11:12.829 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 04:11:12.829 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:11:12.829 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 04:11:12.829 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:11:12.829 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:11:12.829 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:11:12.829 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 04:11:12.829 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 04:11:12.829 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:11:12.829 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 04:11:12.829 | INFO | ------------------------------------------ | 20260505 04:11:12.829 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:11:12.829 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 04:11:12.829 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:11:12.829 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:11:12.829 | INFO | PerfReport: Full Execution -- 3.06e+01 / -nan, 1.29e+01 / 0.00e+00 | 20260505 04:11:12.829 | INFO | PerfReport: 0.28%, AggregateStats -- 8.63e-02 / -nan, 4.23e-03 / 0.00e+00 | 20260505 04:11:12.829 | INFO | PerfReport: 10.78%, Baseline -- 3.30e+00 / -nan, 1.94e-02 / 0.00e+00 | 20260505 04:11:12.829 | INFO | PerfReport: 3.02%, Histogram -- 9.24e-01 / -nan, 5.25e-02 / 0.00e+00 | 20260505 04:11:12.829 | INFO | PerfReport: 16.01%, DME -- 3.92e+01 / -nan, 3.27e+00 / 0.00e+00 | 20260505 04:11:12.829 | INFO | PerfReport: 56.31%, FrameLabeling -- 1.72e+01 / -nan, 4.62e-02 / 0.00e+00 | 20260505 04:11:12.829 | INFO | PerfReport: 7.74%, PulseAccumulating -- 2.37e+00 / -nan, 1.60e-01 / 0.00e+00 | 20260505 04:11:12.829 | INFO | PerfReport: 4.14%, Metrics -- 1.26e+00 / -nan, 2.06e-01 / 0.00e+00 | 20260505 04:11:12.829 | INFO | PerfReport: 1.72%, PulseSerializing -- 5.26e-01 / -nan, 8.45e-02 / 0.00e+00 | 20260505 04:11:12.829 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 04:11:12.829 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:11:12.829 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:11:12.829 | INFO | PerfReport: Full Execution -- 9.19e+01 / -nan, 1.91e+01 / 0.00e+00 | 20260505 04:11:12.829 | INFO | PerfReport: 18.19%, Upload -- 1.67e+01 / 0.00e+00, 3.69e-01 / 0.00e+00 | 20260505 04:11:12.829 | INFO | PerfReport: 17.45%, Download -- 1.60e+01 / 0.00e+00, 6.95e-01 / 0.00e+00 | 20260505 04:11:12.829 | INFO | PerfReport: 33.29%, Compute -- 3.06e+01 / 0.00e+00, 1.29e+01 / 0.00e+00 | 20260505 04:11:12.829 | INFO | PerfReport: 0.55%, LockUpload -- 5.04e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260505 04:11:12.829 | INFO | PerfReport: 0.00%, LockDownload -- 3.92e-04 / 0.00e+00, 1.06e-04 / 0.00e+00 | 20260505 04:11:12.829 | INFO | PerfReport: 30.50%, LockCompute -- 2.80e+01 / 0.00e+00, 1.73e+01 / 0.00e+00 | 20260505 04:11:12.829 | INFO | PerfReport: 0.02%, Misc -- 1.40e-02 / 0.00e+00, 2.75e-02 / 0.00e+00 | 20260505 04:11:12.924 | INFO | Starting a round of Baz IO | 20260505 04:11:15.891 | INFO | Populating RTMetrics report | 20260505 04:11:15.892 | INFO | RTMetrics report populated | 20260505 04:11:15.892 | INFO | Opening tmp RTMetrics json file | 20260505 04:11:15.893 | INFO | Serializing RTMetrics to json | 20260505 04:11:15.894 | INFO | Serialization done, swapping tmp file now... | 20260505 04:11:15.894 | INFO | RTMetrics done | 20260505 04:11:15.974 | INFO | Finished a round of Baz IO: Wrote 1.802 GiB in 3.05 seconds | 20260505 04:11:15.974 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 52.90% / 0.23% / 46.87% | 20260505 04:11:15.974 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 12.6328% / 0.000683467% / 4.76837e-05% | 20260505 04:11:15.974 | INFO | Baz IO overall: | 20260505 04:11:15.974 | INFO | BytesWritten : 78911978638 | 20260505 04:11:15.974 | INFO | NumEvents : 35852168683 | 20260505 04:11:17.966 | INFO | Analyzing chunk frames = [237568,238080) | 20260505 04:11:17.966 | INFO | MemInfo: { | 20260505 04:11:17.966 | INFO | "gpu" : " 17 GB", | 20260505 04:11:17.966 | INFO | "huge" : "", | 20260505 04:11:17.966 | INFO | "huge_shared" : " 27 GB", | 20260505 04:11:17.966 | INFO | "malloc" : " 14 GB", | 20260505 04:11:17.966 | INFO | "pinned" : " 29 GB" | 20260505 04:11:17.966 | INFO | } | 20260505 04:11:17.966 | INFO | Perf report for the last 5.13906 seconds: | 20260505 04:11:17.966 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:11:17.966 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:17.966 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:17.967 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:17.967 | INFO | ANALYSIS : 58.5%, 3.0, 91.2ms, 0 | 20260505 04:11:17.967 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:17.967 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260505 04:11:17.967 | INFO | PRE_HQ : 58.0%, 2.2, 65.8ms, 0 | 20260505 04:11:17.967 | INFO | BAZWRITER : 58.9%, 3.5, 107.3ms, 0 | 20260505 04:11:17.967 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:11:17.967 | INFO | Uploaded 36GiB to GPU in 1.63974s (21.9547GiB/s) | 20260505 04:11:17.967 | INFO | Downloaded 38GiB to GPU in 1.70235s (22.322GiB/s) | 20260505 04:11:23.106 | INFO | Analyzing chunk frames = [238080,238592) | 20260505 04:11:23.106 | INFO | MemInfo: { | 20260505 04:11:23.106 | INFO | "gpu" : " 17 GB", | 20260505 04:11:23.106 | INFO | "huge" : "", | 20260505 04:11:23.106 | INFO | "huge_shared" : " 27 GB", | 20260505 04:11:23.106 | INFO | "malloc" : " 14 GB", | 20260505 04:11:23.106 | INFO | "pinned" : " 29 GB" | 20260505 04:11:23.106 | INFO | } | 20260505 04:11:23.106 | INFO | Perf report for the last 5.13947 seconds: | 20260505 04:11:23.106 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:11:23.106 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:23.106 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:23.106 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:23.106 | INFO | ANALYSIS : 58.2%, 3.0, 90.8ms, 0 | 20260505 04:11:23.106 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:23.106 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:23.106 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260505 04:11:23.106 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:23.106 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:23.106 | INFO | Uploaded 36GiB to GPU in 1.59743s (22.5361GiB/s) | 20260505 04:11:23.106 | INFO | Downloaded 34GiB to GPU in 1.51396s (22.4576GiB/s) | 20260505 04:11:28.245 | INFO | Analyzing chunk frames = [238592,239104) | 20260505 04:11:28.245 | INFO | MemInfo: { | 20260505 04:11:28.245 | INFO | "gpu" : " 17 GB", | 20260505 04:11:28.245 | INFO | "huge" : "", | 20260505 04:11:28.245 | INFO | "huge_shared" : " 27 GB", | 20260505 04:11:28.245 | INFO | "malloc" : " 14 GB", | 20260505 04:11:28.245 | INFO | "pinned" : " 29 GB" | 20260505 04:11:28.245 | INFO | } | 20260505 04:11:28.245 | INFO | Perf report for the last 5.13936 seconds: | 20260505 04:11:28.245 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:11:28.245 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:28.245 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:28.245 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:28.245 | INFO | ANALYSIS : 57.7%, 3.0, 89.6ms, 0 | 20260505 04:11:28.245 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:28.245 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:28.245 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260505 04:11:28.245 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:28.245 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:28.246 | INFO | Uploaded 36GiB to GPU in 1.6009s (22.4873GiB/s) | 20260505 04:11:28.246 | INFO | Downloaded 34GiB to GPU in 1.5177s (22.4023GiB/s) | 20260505 04:11:33.384 | INFO | Analyzing chunk frames = [239104,239616) | 20260505 04:11:33.384 | INFO | MemInfo: { | 20260505 04:11:33.384 | INFO | "gpu" : " 17 GB", | 20260505 04:11:33.384 | INFO | "huge" : "", | 20260505 04:11:33.385 | INFO | "huge_shared" : " 27 GB", | 20260505 04:11:33.385 | INFO | "malloc" : " 14 GB", | 20260505 04:11:33.385 | INFO | "pinned" : " 29 GB" | 20260505 04:11:33.385 | INFO | } | 20260505 04:11:33.385 | INFO | Perf report for the last 5.13944 seconds: | 20260505 04:11:33.385 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:11:33.385 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:33.385 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:33.385 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:33.385 | INFO | ANALYSIS : 57.3%, 3.0, 88.9ms, 0 | 20260505 04:11:33.385 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:33.385 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:33.385 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 04:11:33.385 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:33.385 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:33.385 | INFO | Uploaded 36GiB to GPU in 1.6s (22.5GiB/s) | 20260505 04:11:33.385 | INFO | Downloaded 34GiB to GPU in 1.51752s (22.405GiB/s) | 20260505 04:11:38.524 | INFO | Analyzing chunk frames = [239616,240128) | 20260505 04:11:38.524 | INFO | MemInfo: { | 20260505 04:11:38.524 | INFO | "gpu" : " 17 GB", | 20260505 04:11:38.524 | INFO | "huge" : "", | 20260505 04:11:38.524 | INFO | "huge_shared" : " 27 GB", | 20260505 04:11:38.524 | INFO | "malloc" : " 14 GB", | 20260505 04:11:38.524 | INFO | "pinned" : " 29 GB" | 20260505 04:11:38.524 | INFO | } | 20260505 04:11:38.524 | INFO | Perf report for the last 5.13924 seconds: | 20260505 04:11:38.524 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:11:38.524 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:38.524 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:38.524 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:38.524 | INFO | ANALYSIS : 57.2%, 3.0, 89.3ms, 0 | 20260505 04:11:38.524 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:38.524 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:38.524 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 04:11:38.524 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:38.524 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:38.524 | INFO | Uploaded 36GiB to GPU in 1.59814s (22.5262GiB/s) | 20260505 04:11:38.524 | INFO | Downloaded 34GiB to GPU in 1.51511s (22.4406GiB/s) | 20260505 04:11:43.663 | INFO | Analyzing chunk frames = [240128,240640) | 20260505 04:11:43.663 | INFO | MemInfo: { | 20260505 04:11:43.663 | INFO | "gpu" : " 17 GB", | 20260505 04:11:43.663 | INFO | "huge" : "", | 20260505 04:11:43.663 | INFO | "huge_shared" : " 27 GB", | 20260505 04:11:43.663 | INFO | "malloc" : " 14 GB", | 20260505 04:11:43.663 | INFO | "pinned" : " 29 GB" | 20260505 04:11:43.663 | INFO | } | 20260505 04:11:43.663 | INFO | Perf report for the last 5.13933 seconds: | 20260505 04:11:43.663 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:11:43.663 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:43.663 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:43.663 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:43.663 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260505 04:11:43.663 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:43.663 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:43.663 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260505 04:11:43.663 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:43.663 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:43.664 | INFO | Uploaded 36GiB to GPU in 1.59789s (22.5298GiB/s) | 20260505 04:11:43.664 | INFO | Downloaded 34GiB to GPU in 1.51458s (22.4484GiB/s) | 20260505 04:11:48.802 | INFO | Analyzing chunk frames = [240640,241152) | 20260505 04:11:48.802 | INFO | MemInfo: { | 20260505 04:11:48.802 | INFO | "gpu" : " 17 GB", | 20260505 04:11:48.802 | INFO | "huge" : "", | 20260505 04:11:48.802 | INFO | "huge_shared" : " 27 GB", | 20260505 04:11:48.802 | INFO | "malloc" : " 14 GB", | 20260505 04:11:48.802 | INFO | "pinned" : " 29 GB" | 20260505 04:11:48.803 | INFO | } | 20260505 04:11:48.803 | INFO | Perf report for the last 5.13936 seconds: | 20260505 04:11:48.803 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:11:48.803 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:48.803 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:48.803 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:48.803 | INFO | ANALYSIS : 57.7%, 3.0, 90.0ms, 0 | 20260505 04:11:48.803 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:48.803 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:48.803 | INFO | PRE_HQ : 32.9%, 1.0, 17.1ms, 0 | 20260505 04:11:48.803 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:48.803 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:48.803 | INFO | Uploaded 36GiB to GPU in 1.59932s (22.5096GiB/s) | 20260505 04:11:48.803 | INFO | Downloaded 34GiB to GPU in 1.5183s (22.3934GiB/s) | 20260505 04:11:53.942 | INFO | Analyzing chunk frames = [241152,241664) | 20260505 04:11:53.942 | INFO | MemInfo: { | 20260505 04:11:53.942 | INFO | "gpu" : " 17 GB", | 20260505 04:11:53.942 | INFO | "huge" : "", | 20260505 04:11:53.942 | INFO | "huge_shared" : " 27 GB", | 20260505 04:11:53.942 | INFO | "malloc" : " 14 GB", | 20260505 04:11:53.942 | INFO | "pinned" : " 29 GB" | 20260505 04:11:53.942 | INFO | } | 20260505 04:11:53.942 | INFO | Perf report for the last 5.13934 seconds: | 20260505 04:11:53.942 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:11:53.942 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:53.942 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:53.942 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:53.942 | INFO | ANALYSIS : 57.8%, 3.0, 90.2ms, 0 | 20260505 04:11:53.942 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:53.942 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:53.942 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 04:11:53.942 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:53.942 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:53.943 | INFO | Uploaded 36GiB to GPU in 1.59968s (22.5045GiB/s) | 20260505 04:11:53.943 | INFO | Downloaded 34GiB to GPU in 1.51637s (22.422GiB/s) | 20260505 04:11:54.040 | INFO | Starting a round of Baz IO | 20260505 04:11:57.002 | INFO | Populating RTMetrics report | 20260505 04:11:57.002 | INFO | RTMetrics report populated | 20260505 04:11:57.002 | INFO | Opening tmp RTMetrics json file | 20260505 04:11:57.003 | INFO | Serializing RTMetrics to json | 20260505 04:11:57.004 | INFO | Serialization done, swapping tmp file now... | 20260505 04:11:57.005 | INFO | RTMetrics done | 20260505 04:11:57.081 | INFO | Finished a round of Baz IO: Wrote 1.809 GiB in 3.041 seconds | 20260505 04:11:57.081 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 52.68% / 0.22% / 47.10% | 20260505 04:11:57.081 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 12.5659% / 0.000683467% / 4.37101e-05% | 20260505 04:11:57.081 | INFO | Baz IO overall: | 20260505 04:11:57.081 | INFO | BytesWritten : 80854595745 | 20260505 04:11:57.081 | INFO | NumEvents : 36769455992 | 20260505 04:11:59.081 | INFO | Analyzing chunk frames = [241664,242176) | 20260505 04:11:59.081 | INFO | MemInfo: { | 20260505 04:11:59.081 | INFO | "gpu" : " 17 GB", | 20260505 04:11:59.081 | INFO | "huge" : "", | 20260505 04:11:59.081 | INFO | "huge_shared" : " 27 GB", | 20260505 04:11:59.081 | INFO | "malloc" : " 14 GB", | 20260505 04:11:59.082 | INFO | "pinned" : " 29 GB" | 20260505 04:11:59.082 | INFO | } | 20260505 04:11:59.082 | INFO | Perf report for the last 5.1396 seconds: | 20260505 04:11:59.082 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:11:59.082 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:59.082 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:59.082 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:59.082 | INFO | ANALYSIS : 58.4%, 3.0, 91.1ms, 0 | 20260505 04:11:59.082 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:11:59.082 | INFO | RT_METRICS : 44.7%, 1.0, 23.4ms, 0 | 20260505 04:11:59.082 | INFO | PRE_HQ : 58.0%, 2.2, 65.1ms, 0 | 20260505 04:11:59.082 | INFO | BAZWRITER : 58.7%, 3.5, 106.2ms, 0 | 20260505 04:11:59.082 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:11:59.082 | INFO | Uploaded 36GiB to GPU in 1.63957s (21.957GiB/s) | 20260505 04:11:59.082 | INFO | Downloaded 38GiB to GPU in 1.69623s (22.4026GiB/s) | 20260505 04:12:04.220 | INFO | Analyzing chunk frames = [242176,242688) | 20260505 04:12:04.220 | INFO | MemInfo: { | 20260505 04:12:04.220 | INFO | "gpu" : " 17 GB", | 20260505 04:12:04.220 | INFO | "huge" : "", | 20260505 04:12:04.220 | INFO | "huge_shared" : " 27 GB", | 20260505 04:12:04.220 | INFO | "malloc" : " 14 GB", | 20260505 04:12:04.220 | INFO | "pinned" : " 29 GB" | 20260505 04:12:04.221 | INFO | } | 20260505 04:12:04.221 | INFO | Perf report for the last 5.13895 seconds: | 20260505 04:12:04.221 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:12:04.221 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:04.221 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:04.221 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:04.221 | INFO | ANALYSIS : 58.3%, 3.0, 90.9ms, 0 | 20260505 04:12:04.221 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:04.221 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:04.221 | INFO | PRE_HQ : 32.1%, 1.0, 16.8ms, 0 | 20260505 04:12:04.221 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:04.221 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:04.221 | INFO | Uploaded 36GiB to GPU in 1.59725s (22.5387GiB/s) | 20260505 04:12:04.221 | INFO | Downloaded 34GiB to GPU in 1.51416s (22.4547GiB/s) | 20260505 04:12:09.360 | INFO | Analyzing chunk frames = [242688,243200) | 20260505 04:12:09.360 | INFO | MemInfo: { | 20260505 04:12:09.360 | INFO | "gpu" : " 17 GB", | 20260505 04:12:09.360 | INFO | "huge" : "", | 20260505 04:12:09.360 | INFO | "huge_shared" : " 27 GB", | 20260505 04:12:09.360 | INFO | "malloc" : " 14 GB", | 20260505 04:12:09.360 | INFO | "pinned" : " 29 GB" | 20260505 04:12:09.360 | INFO | } | 20260505 04:12:09.360 | INFO | Perf report for the last 5.13944 seconds: | 20260505 04:12:09.360 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:12:09.360 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:09.360 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:09.360 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:09.360 | INFO | ANALYSIS : 57.6%, 3.0, 89.5ms, 0 | 20260505 04:12:09.360 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:09.360 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:09.360 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260505 04:12:09.360 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:09.360 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:09.361 | INFO | Uploaded 36GiB to GPU in 1.60096s (22.4865GiB/s) | 20260505 04:12:09.361 | INFO | Downloaded 34GiB to GPU in 1.52096s (22.3544GiB/s) | 20260505 04:12:14.499 | INFO | Analyzing chunk frames = [243200,243712) | 20260505 04:12:14.499 | INFO | MemInfo: { | 20260505 04:12:14.499 | INFO | "gpu" : " 17 GB", | 20260505 04:12:14.499 | INFO | "huge" : "", | 20260505 04:12:14.499 | INFO | "huge_shared" : " 27 GB", | 20260505 04:12:14.499 | INFO | "malloc" : " 14 GB", | 20260505 04:12:14.500 | INFO | "pinned" : " 29 GB" | 20260505 04:12:14.500 | INFO | } | 20260505 04:12:14.500 | INFO | Perf report for the last 5.13957 seconds: | 20260505 04:12:14.500 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:12:14.500 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:14.500 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:14.500 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:14.500 | INFO | ANALYSIS : 57.3%, 3.0, 88.9ms, 0 | 20260505 04:12:14.500 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:14.500 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:14.500 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 04:12:14.500 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:14.500 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:14.500 | INFO | Uploaded 36GiB to GPU in 1.59959s (22.5058GiB/s) | 20260505 04:12:14.500 | INFO | Downloaded 34GiB to GPU in 1.51637s (22.4219GiB/s) | 20260505 04:12:19.639 | INFO | Analyzing chunk frames = [243712,244224) | 20260505 04:12:19.639 | INFO | MemInfo: { | 20260505 04:12:19.639 | INFO | "gpu" : " 17 GB", | 20260505 04:12:19.639 | INFO | "huge" : "", | 20260505 04:12:19.639 | INFO | "huge_shared" : " 27 GB", | 20260505 04:12:19.639 | INFO | "malloc" : " 14 GB", | 20260505 04:12:19.639 | INFO | "pinned" : " 29 GB" | 20260505 04:12:19.639 | INFO | } | 20260505 04:12:19.639 | INFO | Perf report for the last 5.1392 seconds: | 20260505 04:12:19.639 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:12:19.639 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:19.639 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:19.639 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:19.639 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260505 04:12:19.639 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:19.639 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:19.639 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260505 04:12:19.639 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:19.639 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:19.639 | INFO | Uploaded 36GiB to GPU in 1.59921s (22.5111GiB/s) | 20260505 04:12:19.639 | INFO | Downloaded 34GiB to GPU in 1.51554s (22.4343GiB/s) | 20260505 04:12:24.778 | INFO | Analyzing chunk frames = [244224,244736) | 20260505 04:12:24.778 | INFO | MemInfo: { | 20260505 04:12:24.778 | INFO | "gpu" : " 17 GB", | 20260505 04:12:24.778 | INFO | "huge" : "", | 20260505 04:12:24.778 | INFO | "huge_shared" : " 27 GB", | 20260505 04:12:24.778 | INFO | "malloc" : " 14 GB", | 20260505 04:12:24.778 | INFO | "pinned" : " 29 GB" | 20260505 04:12:24.778 | INFO | } | 20260505 04:12:24.778 | INFO | Perf report for the last 5.1393 seconds: | 20260505 04:12:24.778 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:12:24.778 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:24.778 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:24.778 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:24.778 | INFO | ANALYSIS : 57.5%, 3.0, 89.8ms, 0 | 20260505 04:12:24.778 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:24.778 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:24.778 | INFO | PRE_HQ : 32.8%, 1.0, 17.0ms, 0 | 20260505 04:12:24.778 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:24.778 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:24.778 | INFO | Uploaded 36GiB to GPU in 1.59898s (22.5144GiB/s) | 20260505 04:12:24.779 | INFO | Downloaded 34GiB to GPU in 1.51533s (22.4374GiB/s) | 20260505 04:12:29.917 | INFO | Analyzing chunk frames = [244736,245248) | 20260505 04:12:29.917 | INFO | MemInfo: { | 20260505 04:12:29.917 | INFO | "gpu" : " 17 GB", | 20260505 04:12:29.917 | INFO | "huge" : "", | 20260505 04:12:29.917 | INFO | "huge_shared" : " 27 GB", | 20260505 04:12:29.917 | INFO | "malloc" : " 14 GB", | 20260505 04:12:29.917 | INFO | "pinned" : " 29 GB" | 20260505 04:12:29.917 | INFO | } | 20260505 04:12:29.917 | INFO | Perf report for the last 5.13928 seconds: | 20260505 04:12:29.917 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:12:29.917 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:29.917 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:29.917 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:29.917 | INFO | ANALYSIS : 57.7%, 3.0, 90.0ms, 0 | 20260505 04:12:29.918 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:29.918 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:29.918 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260505 04:12:29.918 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:29.918 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:29.918 | INFO | Uploaded 36GiB to GPU in 1.59999s (22.5002GiB/s) | 20260505 04:12:29.918 | INFO | Downloaded 34GiB to GPU in 1.51675s (22.4164GiB/s) | 20260505 04:12:35.057 | INFO | Analyzing chunk frames = [245248,245760) | 20260505 04:12:35.057 | INFO | MemInfo: { | 20260505 04:12:35.057 | INFO | "gpu" : " 17 GB", | 20260505 04:12:35.057 | INFO | "huge" : "", | 20260505 04:12:35.057 | INFO | "huge_shared" : " 27 GB", | 20260505 04:12:35.057 | INFO | "malloc" : " 14 GB", | 20260505 04:12:35.057 | INFO | "pinned" : " 29 GB" | 20260505 04:12:35.057 | INFO | } | 20260505 04:12:35.057 | INFO | Perf report for the last 5.1394 seconds: | 20260505 04:12:35.057 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:12:35.057 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:35.057 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:35.057 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:35.057 | INFO | ANALYSIS : 57.8%, 3.0, 90.2ms, 0 | 20260505 04:12:35.057 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:35.057 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:35.057 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260505 04:12:35.057 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:35.057 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:35.057 | INFO | Uploaded 36GiB to GPU in 1.59978s (22.5031GiB/s) | 20260505 04:12:35.057 | INFO | Downloaded 34GiB to GPU in 1.51685s (22.4149GiB/s) | 20260505 04:12:35.057 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 04:12:35.057 | INFO | Total mem usage: 14.9 GiB | 20260505 04:12:35.057 | INFO | All Time High Water Marks sum: 15.2 GiB (Savings: 1.89%) | 20260505 04:12:35.057 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -2.77%) | 20260505 04:12:35.057 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 04:12:35.057 | INFO | High Water Breakdown: | 20260505 04:12:35.057 | INFO | 58.7% : 8.52 GiB ( 99.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 04:12:35.057 | INFO | 41.2% : 5.98 GiB ( 90% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 04:12:35.057 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 04:12:35.057 | INFO | ------------------------------------------ | 20260505 04:12:35.057 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:12:35.057 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 04:12:35.057 | INFO | Total mem usage: 26.5 GiB | 20260505 04:12:35.057 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260505 04:12:35.057 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 04:12:35.057 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260505 04:12:35.057 | INFO | High Water Breakdown: | 20260505 04:12:35.057 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:12:35.057 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:12:35.057 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:12:35.057 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:12:35.057 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:12:35.057 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:12:35.057 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 04:12:35.057 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 04:12:35.057 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:12:35.057 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 04:12:35.057 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 04:12:35.057 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 04:12:35.057 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 04:12:35.058 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 04:12:35.058 | INFO | ------------------------------------------ | 20260505 04:12:35.058 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:12:35.058 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 04:12:35.058 | INFO | Total mem usage: 25.6 GiB | 20260505 04:12:35.058 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 04:12:35.058 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 04:12:35.058 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 04:12:35.058 | INFO | High Water Breakdown: | 20260505 04:12:35.058 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 04:12:35.058 | INFO | ------------------------------------------ | 20260505 04:12:35.058 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:12:35.058 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 04:12:35.058 | INFO | Total mem usage: 15.7 GiB | 20260505 04:12:35.058 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260505 04:12:35.058 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 04:12:35.058 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260505 04:12:35.058 | INFO | High Water Breakdown: | 20260505 04:12:35.058 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 04:12:35.058 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 04:12:35.058 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 04:12:35.058 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 04:12:35.058 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 04:12:35.058 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 04:12:35.058 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 04:12:35.058 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 04:12:35.058 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 04:12:35.058 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:12:35.058 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 04:12:35.058 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:12:35.058 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 04:12:35.058 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 04:12:35.058 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 04:12:35.058 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 04:12:35.058 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:12:35.058 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 04:12:35.058 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:12:35.058 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:12:35.058 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:12:35.058 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 04:12:35.058 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 04:12:35.058 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:12:35.058 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 04:12:35.058 | INFO | ------------------------------------------ | 20260505 04:12:35.058 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:12:35.058 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 04:12:35.058 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:12:35.058 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:12:35.058 | INFO | PerfReport: Full Execution -- 3.06e+01 / -nan, 1.29e+01 / 0.00e+00 | 20260505 04:12:35.058 | INFO | PerfReport: 0.28%, AggregateStats -- 8.64e-02 / -nan, 4.88e-03 / 0.00e+00 | 20260505 04:12:35.058 | INFO | PerfReport: 10.79%, Baseline -- 3.30e+00 / -nan, 1.92e-02 / 0.00e+00 | 20260505 04:12:35.058 | INFO | PerfReport: 3.02%, Histogram -- 9.24e-01 / -nan, 5.23e-02 / 0.00e+00 | 20260505 04:12:35.058 | INFO | PerfReport: 16.01%, DME -- 3.92e+01 / -nan, 3.25e+00 / 0.00e+00 | 20260505 04:12:35.058 | INFO | PerfReport: 56.31%, FrameLabeling -- 1.72e+01 / -nan, 4.57e-02 / 0.00e+00 | 20260505 04:12:35.058 | INFO | PerfReport: 7.73%, PulseAccumulating -- 2.36e+00 / -nan, 1.59e-01 / 0.00e+00 | 20260505 04:12:35.058 | INFO | PerfReport: 4.13%, Metrics -- 1.26e+00 / -nan, 2.06e-01 / 0.00e+00 | 20260505 04:12:35.058 | INFO | PerfReport: 1.72%, PulseSerializing -- 5.25e-01 / -nan, 8.45e-02 / 0.00e+00 | 20260505 04:12:35.058 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 04:12:35.058 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:12:35.058 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:12:35.058 | INFO | PerfReport: Full Execution -- 9.18e+01 / -nan, 1.91e+01 / 0.00e+00 | 20260505 04:12:35.058 | INFO | PerfReport: 18.20%, Upload -- 1.67e+01 / 0.00e+00, 3.91e-01 / 0.00e+00 | 20260505 04:12:35.058 | INFO | PerfReport: 17.46%, Download -- 1.60e+01 / 0.00e+00, 6.87e-01 / 0.00e+00 | 20260505 04:12:35.058 | INFO | PerfReport: 33.29%, Compute -- 3.06e+01 / 0.00e+00, 1.29e+01 / 0.00e+00 | 20260505 04:12:35.058 | INFO | PerfReport: 0.55%, LockUpload -- 5.03e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260505 04:12:35.058 | INFO | PerfReport: 0.00%, LockDownload -- 3.95e-04 / 0.00e+00, 1.08e-04 / 0.00e+00 | 20260505 04:12:35.058 | INFO | PerfReport: 30.49%, LockCompute -- 2.80e+01 / 0.00e+00, 1.73e+01 / 0.00e+00 | 20260505 04:12:35.058 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.47e-02 / 0.00e+00 | 20260505 04:12:35.154 | INFO | Starting a round of Baz IO | 20260505 04:12:38.115 | INFO | Populating RTMetrics report | 20260505 04:12:38.116 | INFO | RTMetrics report populated | 20260505 04:12:38.116 | INFO | Opening tmp RTMetrics json file | 20260505 04:12:38.117 | INFO | Serializing RTMetrics to json | 20260505 04:12:38.118 | INFO | Serialization done, swapping tmp file now... | 20260505 04:12:38.118 | INFO | RTMetrics done | 20260505 04:12:38.197 | INFO | Finished a round of Baz IO: Wrote 1.816 GiB in 3.043 seconds | 20260505 04:12:38.198 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 52.46% / 0.22% / 47.32% | 20260505 04:12:38.198 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 12.5071% / 0.00068744% / 5.16574e-05% | 20260505 04:12:38.198 | INFO | Baz IO overall: | 20260505 04:12:38.198 | INFO | BytesWritten : 82804900247 | 20260505 04:12:38.198 | INFO | NumEvents : 37690138437 | 20260505 04:12:40.211 | INFO | Analyzing chunk frames = [245760,246272) | 20260505 04:12:40.211 | INFO | MemInfo: { | 20260505 04:12:40.211 | INFO | "gpu" : " 17 GB", | 20260505 04:12:40.211 | INFO | "huge" : "", | 20260505 04:12:40.211 | INFO | "huge_shared" : " 27 GB", | 20260505 04:12:40.211 | INFO | "malloc" : " 14 GB", | 20260505 04:12:40.211 | INFO | "pinned" : " 29 GB" | 20260505 04:12:40.211 | INFO | } | 20260505 04:12:40.211 | INFO | Perf report for the last 5.15424 seconds: | 20260505 04:12:40.211 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:12:40.211 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:40.211 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:40.211 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:40.211 | INFO | ANALYSIS : 58.3%, 3.0, 91.0ms, 0 | 20260505 04:12:40.211 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:40.211 | INFO | RT_METRICS : 44.6%, 1.0, 23.5ms, 0 | 20260505 04:12:40.211 | INFO | PRE_HQ : 57.8%, 2.2, 65.3ms, 0 | 20260505 04:12:40.211 | INFO | BAZWRITER : 58.6%, 3.6, 107.8ms, 0 | 20260505 04:12:40.211 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:12:40.212 | INFO | Uploaded 36GiB to GPU in 1.63727s (21.9878GiB/s) | 20260505 04:12:40.212 | INFO | Downloaded 38GiB to GPU in 1.69877s (22.3691GiB/s) | 20260505 04:12:45.335 | INFO | Analyzing chunk frames = [246272,246784) | 20260505 04:12:45.335 | INFO | MemInfo: { | 20260505 04:12:45.336 | INFO | "gpu" : " 17 GB", | 20260505 04:12:45.336 | INFO | "huge" : "", | 20260505 04:12:45.336 | INFO | "huge_shared" : " 27 GB", | 20260505 04:12:45.336 | INFO | "malloc" : " 14 GB", | 20260505 04:12:45.336 | INFO | "pinned" : " 29 GB" | 20260505 04:12:45.336 | INFO | } | 20260505 04:12:45.336 | INFO | Perf report for the last 5.1247 seconds: | 20260505 04:12:45.336 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:12:45.336 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:45.336 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:45.336 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:45.336 | INFO | ANALYSIS : 58.4%, 3.0, 90.9ms, 0 | 20260505 04:12:45.336 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:45.336 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:45.336 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 04:12:45.336 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:45.336 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:45.336 | INFO | Uploaded 36GiB to GPU in 1.59855s (22.5204GiB/s) | 20260505 04:12:45.336 | INFO | Downloaded 34GiB to GPU in 1.51475s (22.446GiB/s) | 20260505 04:12:50.475 | INFO | Analyzing chunk frames = [246784,247296) | 20260505 04:12:50.475 | INFO | MemInfo: { | 20260505 04:12:50.475 | INFO | "gpu" : " 17 GB", | 20260505 04:12:50.475 | INFO | "huge" : "", | 20260505 04:12:50.475 | INFO | "huge_shared" : " 27 GB", | 20260505 04:12:50.475 | INFO | "malloc" : " 14 GB", | 20260505 04:12:50.475 | INFO | "pinned" : " 29 GB" | 20260505 04:12:50.475 | INFO | } | 20260505 04:12:50.475 | INFO | Perf report for the last 5.13901 seconds: | 20260505 04:12:50.475 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:12:50.475 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:50.475 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:50.475 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:50.475 | INFO | ANALYSIS : 57.6%, 3.0, 89.5ms, 0 | 20260505 04:12:50.475 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:50.475 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:50.475 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260505 04:12:50.475 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:50.475 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:50.475 | INFO | Uploaded 36GiB to GPU in 1.60121s (22.483GiB/s) | 20260505 04:12:50.475 | INFO | Downloaded 34GiB to GPU in 1.51938s (22.3776GiB/s) | 20260505 04:12:55.614 | INFO | Analyzing chunk frames = [247296,247808) | 20260505 04:12:55.614 | INFO | MemInfo: { | 20260505 04:12:55.614 | INFO | "gpu" : " 17 GB", | 20260505 04:12:55.614 | INFO | "huge" : "", | 20260505 04:12:55.614 | INFO | "huge_shared" : " 27 GB", | 20260505 04:12:55.614 | INFO | "malloc" : " 14 GB", | 20260505 04:12:55.614 | INFO | "pinned" : " 29 GB" | 20260505 04:12:55.614 | INFO | } | 20260505 04:12:55.614 | INFO | Perf report for the last 5.1393 seconds: | 20260505 04:12:55.614 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:12:55.614 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:55.614 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:55.614 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:55.614 | INFO | ANALYSIS : 57.3%, 3.0, 88.9ms, 0 | 20260505 04:12:55.614 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:55.614 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:55.614 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260505 04:12:55.614 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:55.614 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:12:55.614 | INFO | Uploaded 36GiB to GPU in 1.60003s (22.4996GiB/s) | 20260505 04:12:55.614 | INFO | Downloaded 34GiB to GPU in 1.51622s (22.4242GiB/s) | 20260505 04:13:00.753 | INFO | Analyzing chunk frames = [247808,248320) | 20260505 04:13:00.753 | INFO | MemInfo: { | 20260505 04:13:00.753 | INFO | "gpu" : " 17 GB", | 20260505 04:13:00.753 | INFO | "huge" : "", | 20260505 04:13:00.753 | INFO | "huge_shared" : " 27 GB", | 20260505 04:13:00.753 | INFO | "malloc" : " 14 GB", | 20260505 04:13:00.753 | INFO | "pinned" : " 29 GB" | 20260505 04:13:00.753 | INFO | } | 20260505 04:13:00.753 | INFO | Perf report for the last 5.13933 seconds: | 20260505 04:13:00.753 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:13:00.753 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:00.753 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:00.753 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:00.754 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260505 04:13:00.754 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:00.754 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:00.754 | INFO | PRE_HQ : 32.1%, 1.0, 16.6ms, 0 | 20260505 04:13:00.754 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:00.754 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:00.754 | INFO | Uploaded 36GiB to GPU in 1.5988s (22.5168GiB/s) | 20260505 04:13:00.754 | INFO | Downloaded 34GiB to GPU in 1.51441s (22.451GiB/s) | 20260505 04:13:05.893 | INFO | Analyzing chunk frames = [248320,248832) | 20260505 04:13:05.893 | INFO | MemInfo: { | 20260505 04:13:05.893 | INFO | "gpu" : " 17 GB", | 20260505 04:13:05.893 | INFO | "huge" : "", | 20260505 04:13:05.893 | INFO | "huge_shared" : " 27 GB", | 20260505 04:13:05.893 | INFO | "malloc" : " 14 GB", | 20260505 04:13:05.893 | INFO | "pinned" : " 29 GB" | 20260505 04:13:05.893 | INFO | } | 20260505 04:13:05.893 | INFO | Perf report for the last 5.13939 seconds: | 20260505 04:13:05.893 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:13:05.893 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:05.893 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:05.893 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:05.893 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260505 04:13:05.893 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:05.893 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:05.893 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 04:13:05.893 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:05.893 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:05.893 | INFO | Uploaded 36GiB to GPU in 1.59888s (22.5158GiB/s) | 20260505 04:13:05.894 | INFO | Downloaded 34GiB to GPU in 1.51488s (22.444GiB/s) | 20260505 04:13:11.032 | INFO | Analyzing chunk frames = [248832,249344) | 20260505 04:13:11.032 | INFO | MemInfo: { | 20260505 04:13:11.032 | INFO | "gpu" : " 17 GB", | 20260505 04:13:11.032 | INFO | "huge" : "", | 20260505 04:13:11.032 | INFO | "huge_shared" : " 27 GB", | 20260505 04:13:11.032 | INFO | "malloc" : " 14 GB", | 20260505 04:13:11.032 | INFO | "pinned" : " 29 GB" | 20260505 04:13:11.032 | INFO | } | 20260505 04:13:11.032 | INFO | Perf report for the last 5.13937 seconds: | 20260505 04:13:11.032 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:13:11.032 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:11.032 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:11.032 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:11.032 | INFO | ANALYSIS : 57.7%, 3.0, 90.0ms, 0 | 20260505 04:13:11.032 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:11.032 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:11.032 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260505 04:13:11.032 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:11.032 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:11.033 | INFO | Uploaded 36GiB to GPU in 1.59944s (22.5078GiB/s) | 20260505 04:13:11.033 | INFO | Downloaded 34GiB to GPU in 1.51609s (22.4261GiB/s) | 20260505 04:13:16.171 | INFO | Analyzing chunk frames = [249344,249856) | 20260505 04:13:16.171 | INFO | MemInfo: { | 20260505 04:13:16.171 | INFO | "gpu" : " 17 GB", | 20260505 04:13:16.171 | INFO | "huge" : "", | 20260505 04:13:16.171 | INFO | "huge_shared" : " 27 GB", | 20260505 04:13:16.171 | INFO | "malloc" : " 14 GB", | 20260505 04:13:16.171 | INFO | "pinned" : " 29 GB" | 20260505 04:13:16.171 | INFO | } | 20260505 04:13:16.171 | INFO | Perf report for the last 5.13905 seconds: | 20260505 04:13:16.171 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:13:16.171 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:16.171 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:16.172 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:16.172 | INFO | ANALYSIS : 57.8%, 3.0, 90.2ms, 0 | 20260505 04:13:16.172 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:16.172 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:16.172 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260505 04:13:16.172 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:16.172 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:16.172 | INFO | Uploaded 36GiB to GPU in 1.59983s (22.5024GiB/s) | 20260505 04:13:16.172 | INFO | Downloaded 34GiB to GPU in 1.51682s (22.4153GiB/s) | 20260505 04:13:16.269 | INFO | Starting a round of Baz IO | 20260505 04:13:19.230 | INFO | Populating RTMetrics report | 20260505 04:13:19.231 | INFO | RTMetrics report populated | 20260505 04:13:19.231 | INFO | Opening tmp RTMetrics json file | 20260505 04:13:19.231 | INFO | Serializing RTMetrics to json | 20260505 04:13:19.233 | INFO | Serialization done, swapping tmp file now... | 20260505 04:13:19.233 | INFO | RTMetrics done | 20260505 04:13:19.315 | INFO | Finished a round of Baz IO: Wrote 1.823 GiB in 3.046 seconds | 20260505 04:13:19.315 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 52.25% / 0.21% / 47.54% | 20260505 04:13:19.315 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 12.4446% / 0.000707308% / 5.5631e-05% | 20260505 04:13:19.315 | INFO | Baz IO overall: | 20260505 04:13:19.315 | INFO | BytesWritten : 84762225779 | 20260505 04:13:19.315 | INFO | NumEvents : 38614165201 | 20260505 04:13:21.311 | INFO | Analyzing chunk frames = [249856,250368) | 20260505 04:13:21.311 | INFO | MemInfo: { | 20260505 04:13:21.311 | INFO | "gpu" : " 17 GB", | 20260505 04:13:21.311 | INFO | "huge" : "", | 20260505 04:13:21.311 | INFO | "huge_shared" : " 27 GB", | 20260505 04:13:21.311 | INFO | "malloc" : " 14 GB", | 20260505 04:13:21.311 | INFO | "pinned" : " 29 GB" | 20260505 04:13:21.311 | INFO | } | 20260505 04:13:21.311 | INFO | Perf report for the last 5.13941 seconds: | 20260505 04:13:21.311 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:13:21.311 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:21.311 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:21.311 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:21.311 | INFO | ANALYSIS : 58.4%, 3.0, 91.1ms, 0 | 20260505 04:13:21.311 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:21.311 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260505 04:13:21.311 | INFO | PRE_HQ : 58.0%, 2.2, 65.2ms, 0 | 20260505 04:13:21.311 | INFO | BAZWRITER : 58.8%, 3.6, 108.0ms, 0 | 20260505 04:13:21.311 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:13:21.311 | INFO | Uploaded 36GiB to GPU in 1.63412s (22.0302GiB/s) | 20260505 04:13:21.311 | INFO | Downloaded 38GiB to GPU in 1.69651s (22.3989GiB/s) | 20260505 04:13:26.450 | INFO | Analyzing chunk frames = [250368,250880) | 20260505 04:13:26.450 | INFO | MemInfo: { | 20260505 04:13:26.450 | INFO | "gpu" : " 17 GB", | 20260505 04:13:26.450 | INFO | "huge" : "", | 20260505 04:13:26.450 | INFO | "huge_shared" : " 27 GB", | 20260505 04:13:26.450 | INFO | "malloc" : " 14 GB", | 20260505 04:13:26.450 | INFO | "pinned" : " 29 GB" | 20260505 04:13:26.450 | INFO | } | 20260505 04:13:26.450 | INFO | Perf report for the last 5.13936 seconds: | 20260505 04:13:26.450 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:13:26.450 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:26.450 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:26.450 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:26.450 | INFO | ANALYSIS : 58.3%, 3.0, 90.9ms, 0 | 20260505 04:13:26.450 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:26.450 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:26.450 | INFO | PRE_HQ : 32.0%, 1.0, 16.8ms, 0 | 20260505 04:13:26.450 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:26.450 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:26.450 | INFO | Uploaded 36GiB to GPU in 1.59705s (22.5416GiB/s) | 20260505 04:13:26.451 | INFO | Downloaded 34GiB to GPU in 1.51273s (22.4759GiB/s) | 20260505 04:13:31.589 | INFO | Analyzing chunk frames = [250880,251392) | 20260505 04:13:31.589 | INFO | MemInfo: { | 20260505 04:13:31.589 | INFO | "gpu" : " 17 GB", | 20260505 04:13:31.589 | INFO | "huge" : "", | 20260505 04:13:31.589 | INFO | "huge_shared" : " 27 GB", | 20260505 04:13:31.589 | INFO | "malloc" : " 14 GB", | 20260505 04:13:31.589 | INFO | "pinned" : " 29 GB" | 20260505 04:13:31.589 | INFO | } | 20260505 04:13:31.589 | INFO | Perf report for the last 5.13936 seconds: | 20260505 04:13:31.589 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:13:31.589 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:31.589 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:31.590 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:31.590 | INFO | ANALYSIS : 57.6%, 3.0, 89.5ms, 0 | 20260505 04:13:31.590 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:31.590 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:31.590 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260505 04:13:31.590 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:31.590 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:31.590 | INFO | Uploaded 36GiB to GPU in 1.60109s (22.4847GiB/s) | 20260505 04:13:31.590 | INFO | Downloaded 34GiB to GPU in 1.51816s (22.3956GiB/s) | 20260505 04:13:36.729 | INFO | Analyzing chunk frames = [251392,251904) | 20260505 04:13:36.729 | INFO | MemInfo: { | 20260505 04:13:36.729 | INFO | "gpu" : " 17 GB", | 20260505 04:13:36.729 | INFO | "huge" : "", | 20260505 04:13:36.729 | INFO | "huge_shared" : " 27 GB", | 20260505 04:13:36.729 | INFO | "malloc" : " 14 GB", | 20260505 04:13:36.729 | INFO | "pinned" : " 29 GB" | 20260505 04:13:36.729 | INFO | } | 20260505 04:13:36.729 | INFO | Perf report for the last 5.13926 seconds: | 20260505 04:13:36.729 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:13:36.729 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:36.729 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:36.729 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:36.729 | INFO | ANALYSIS : 57.3%, 3.0, 88.8ms, 0 | 20260505 04:13:36.729 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:36.729 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:36.729 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 04:13:36.729 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:36.729 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:36.729 | INFO | Uploaded 36GiB to GPU in 1.60082s (22.4885GiB/s) | 20260505 04:13:36.729 | INFO | Downloaded 34GiB to GPU in 1.51744s (22.4061GiB/s) | 20260505 04:13:41.868 | INFO | Analyzing chunk frames = [251904,252416) | 20260505 04:13:41.868 | INFO | MemInfo: { | 20260505 04:13:41.868 | INFO | "gpu" : " 17 GB", | 20260505 04:13:41.868 | INFO | "huge" : "", | 20260505 04:13:41.868 | INFO | "huge_shared" : " 27 GB", | 20260505 04:13:41.868 | INFO | "malloc" : " 14 GB", | 20260505 04:13:41.868 | INFO | "pinned" : " 29 GB" | 20260505 04:13:41.868 | INFO | } | 20260505 04:13:41.868 | INFO | Perf report for the last 5.13935 seconds: | 20260505 04:13:41.868 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:13:41.868 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:41.868 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:41.868 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:41.868 | INFO | ANALYSIS : 57.3%, 3.0, 89.3ms, 0 | 20260505 04:13:41.868 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:41.868 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:41.868 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 04:13:41.868 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:41.868 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:41.869 | INFO | Uploaded 36GiB to GPU in 1.59866s (22.5189GiB/s) | 20260505 04:13:41.869 | INFO | Downloaded 34GiB to GPU in 1.51503s (22.4418GiB/s) | 20260505 04:13:47.007 | INFO | Analyzing chunk frames = [252416,252928) | 20260505 04:13:47.007 | INFO | MemInfo: { | 20260505 04:13:47.007 | INFO | "gpu" : " 17 GB", | 20260505 04:13:47.007 | INFO | "huge" : "", | 20260505 04:13:47.007 | INFO | "huge_shared" : " 27 GB", | 20260505 04:13:47.007 | INFO | "malloc" : " 14 GB", | 20260505 04:13:47.007 | INFO | "pinned" : " 29 GB" | 20260505 04:13:47.008 | INFO | } | 20260505 04:13:47.008 | INFO | Perf report for the last 5.13944 seconds: | 20260505 04:13:47.008 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:13:47.008 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:47.008 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:47.008 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:47.008 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260505 04:13:47.008 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:47.008 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:47.008 | INFO | PRE_HQ : 32.3%, 1.0, 16.7ms, 0 | 20260505 04:13:47.008 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:47.008 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:47.008 | INFO | Uploaded 36GiB to GPU in 1.59911s (22.5126GiB/s) | 20260505 04:13:47.008 | INFO | Downloaded 34GiB to GPU in 1.51639s (22.4217GiB/s) | 20260505 04:13:52.147 | INFO | Analyzing chunk frames = [252928,253440) | 20260505 04:13:52.147 | INFO | MemInfo: { | 20260505 04:13:52.147 | INFO | "gpu" : " 17 GB", | 20260505 04:13:52.147 | INFO | "huge" : "", | 20260505 04:13:52.147 | INFO | "huge_shared" : " 27 GB", | 20260505 04:13:52.147 | INFO | "malloc" : " 14 GB", | 20260505 04:13:52.147 | INFO | "pinned" : " 29 GB" | 20260505 04:13:52.147 | INFO | } | 20260505 04:13:52.147 | INFO | Perf report for the last 5.1392 seconds: | 20260505 04:13:52.147 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:13:52.147 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:52.147 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:52.147 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:52.147 | INFO | ANALYSIS : 57.8%, 3.0, 90.2ms, 0 | 20260505 04:13:52.147 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:52.147 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:52.147 | INFO | PRE_HQ : 32.7%, 1.0, 17.0ms, 0 | 20260505 04:13:52.147 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:52.147 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:52.147 | INFO | Uploaded 36GiB to GPU in 1.59977s (22.5032GiB/s) | 20260505 04:13:52.147 | INFO | Downloaded 34GiB to GPU in 1.51608s (22.4263GiB/s) | 20260505 04:13:57.286 | INFO | Analyzing chunk frames = [253440,253952) | 20260505 04:13:57.286 | INFO | MemInfo: { | 20260505 04:13:57.286 | INFO | "gpu" : " 17 GB", | 20260505 04:13:57.286 | INFO | "huge" : "", | 20260505 04:13:57.286 | INFO | "huge_shared" : " 27 GB", | 20260505 04:13:57.286 | INFO | "malloc" : " 14 GB", | 20260505 04:13:57.286 | INFO | "pinned" : " 29 GB" | 20260505 04:13:57.286 | INFO | } | 20260505 04:13:57.286 | INFO | Perf report for the last 5.13942 seconds: | 20260505 04:13:57.286 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:13:57.286 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:57.286 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:57.286 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:57.286 | INFO | ANALYSIS : 57.8%, 3.0, 90.2ms, 0 | 20260505 04:13:57.286 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:57.286 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:57.286 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 04:13:57.286 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:57.286 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:13:57.286 | INFO | Uploaded 36GiB to GPU in 1.59879s (22.5171GiB/s) | 20260505 04:13:57.286 | INFO | Downloaded 34GiB to GPU in 1.51716s (22.4103GiB/s) | 20260505 04:13:57.287 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 04:13:57.287 | INFO | Total mem usage: 14.9 GiB | 20260505 04:13:57.287 | INFO | All Time High Water Marks sum: 15.2 GiB (Savings: 1.89%) | 20260505 04:13:57.287 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -2.92%) | 20260505 04:13:57.287 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 04:13:57.287 | INFO | High Water Breakdown: | 20260505 04:13:57.287 | INFO | 58.7% : 8.51 GiB ( 99.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 04:13:57.287 | INFO | 41.2% : 5.97 GiB ( 89.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 04:13:57.287 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 04:13:57.287 | INFO | ------------------------------------------ | 20260505 04:13:57.287 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:13:57.287 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 04:13:57.287 | INFO | Total mem usage: 26.5 GiB | 20260505 04:13:57.287 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260505 04:13:57.287 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 04:13:57.287 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260505 04:13:57.287 | INFO | High Water Breakdown: | 20260505 04:13:57.287 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:13:57.287 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:13:57.287 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:13:57.287 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:13:57.287 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:13:57.287 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:13:57.287 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 04:13:57.287 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 04:13:57.287 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:13:57.287 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 04:13:57.287 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 04:13:57.287 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 04:13:57.287 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 04:13:57.287 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 04:13:57.287 | INFO | ------------------------------------------ | 20260505 04:13:57.287 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:13:57.287 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 04:13:57.287 | INFO | Total mem usage: 25.6 GiB | 20260505 04:13:57.287 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 04:13:57.287 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 04:13:57.287 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 04:13:57.287 | INFO | High Water Breakdown: | 20260505 04:13:57.287 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 04:13:57.287 | INFO | ------------------------------------------ | 20260505 04:13:57.287 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:13:57.287 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 04:13:57.287 | INFO | Total mem usage: 15.7 GiB | 20260505 04:13:57.287 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260505 04:13:57.287 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 04:13:57.287 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260505 04:13:57.287 | INFO | High Water Breakdown: | 20260505 04:13:57.287 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 04:13:57.287 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 04:13:57.287 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 04:13:57.287 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 04:13:57.287 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 04:13:57.287 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 04:13:57.288 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 04:13:57.288 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 04:13:57.288 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 04:13:57.288 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:13:57.288 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 04:13:57.288 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:13:57.288 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 04:13:57.288 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 04:13:57.288 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 04:13:57.288 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 04:13:57.288 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:13:57.288 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 04:13:57.288 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:13:57.288 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:13:57.288 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:13:57.288 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 04:13:57.288 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 04:13:57.288 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:13:57.288 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 04:13:57.288 | INFO | ------------------------------------------ | 20260505 04:13:57.288 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:13:57.288 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 04:13:57.288 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:13:57.288 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:13:57.288 | INFO | PerfReport: Full Execution -- 3.06e+01 / -nan, 1.28e+01 / 0.00e+00 | 20260505 04:13:57.288 | INFO | PerfReport: 0.28%, AggregateStats -- 8.64e-02 / -nan, 4.29e-03 / 0.00e+00 | 20260505 04:13:57.288 | INFO | PerfReport: 10.78%, Baseline -- 3.30e+00 / -nan, 1.77e-02 / 0.00e+00 | 20260505 04:13:57.288 | INFO | PerfReport: 3.02%, Histogram -- 9.25e-01 / -nan, 5.58e-02 / 0.00e+00 | 20260505 04:13:57.288 | INFO | PerfReport: 16.01%, DME -- 3.91e+01 / -nan, 3.24e+00 / 0.00e+00 | 20260505 04:13:57.288 | INFO | PerfReport: 56.33%, FrameLabeling -- 1.72e+01 / -nan, 4.58e-02 / 0.00e+00 | 20260505 04:13:57.288 | INFO | PerfReport: 7.73%, PulseAccumulating -- 2.36e+00 / -nan, 1.60e-01 / 0.00e+00 | 20260505 04:13:57.288 | INFO | PerfReport: 4.12%, Metrics -- 1.26e+00 / -nan, 2.06e-01 / 0.00e+00 | 20260505 04:13:57.288 | INFO | PerfReport: 1.71%, PulseSerializing -- 5.24e-01 / -nan, 8.43e-02 / 0.00e+00 | 20260505 04:13:57.288 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 04:13:57.288 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:13:57.288 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:13:57.288 | INFO | PerfReport: Full Execution -- 9.18e+01 / -nan, 1.91e+01 / 0.00e+00 | 20260505 04:13:57.288 | INFO | PerfReport: 18.20%, Upload -- 1.67e+01 / 0.00e+00, 3.58e-01 / 0.00e+00 | 20260505 04:13:57.288 | INFO | PerfReport: 17.46%, Download -- 1.60e+01 / 0.00e+00, 6.86e-01 / 0.00e+00 | 20260505 04:13:57.288 | INFO | PerfReport: 33.30%, Compute -- 3.06e+01 / 0.00e+00, 1.28e+01 / 0.00e+00 | 20260505 04:13:57.288 | INFO | PerfReport: 0.55%, LockUpload -- 5.04e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260505 04:13:57.288 | INFO | PerfReport: 0.00%, LockDownload -- 4.00e-04 / 0.00e+00, 1.00e-04 / 0.00e+00 | 20260505 04:13:57.288 | INFO | PerfReport: 30.48%, LockCompute -- 2.80e+01 / 0.00e+00, 1.72e+01 / 0.00e+00 | 20260505 04:13:57.288 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.82e-02 / 0.00e+00 | 20260505 04:13:57.383 | INFO | Starting a round of Baz IO | 20260505 04:14:00.347 | INFO | Populating RTMetrics report | 20260505 04:14:00.347 | INFO | RTMetrics report populated | 20260505 04:14:00.347 | INFO | Opening tmp RTMetrics json file | 20260505 04:14:00.348 | INFO | Serializing RTMetrics to json | 20260505 04:14:00.349 | INFO | Serialization done, swapping tmp file now... | 20260505 04:14:00.350 | INFO | RTMetrics done | 20260505 04:14:00.428 | INFO | Finished a round of Baz IO: Wrote 1.831 GiB in 3.045 seconds | 20260505 04:14:00.429 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 52.04% / 0.21% / 47.75% | 20260505 04:14:00.429 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 12.3948% / 0.000584126% / 3.57628e-05% | 20260505 04:14:00.429 | INFO | Baz IO overall: | 20260505 04:14:00.429 | INFO | BytesWritten : 86728184882 | 20260505 04:14:00.429 | INFO | NumEvents : 39542673021 | 20260505 04:14:02.425 | INFO | Analyzing chunk frames = [253952,254464) | 20260505 04:14:02.425 | INFO | MemInfo: { | 20260505 04:14:02.425 | INFO | "gpu" : " 17 GB", | 20260505 04:14:02.425 | INFO | "huge" : "", | 20260505 04:14:02.425 | INFO | "huge_shared" : " 27 GB", | 20260505 04:14:02.425 | INFO | "malloc" : " 14 GB", | 20260505 04:14:02.426 | INFO | "pinned" : " 29 GB" | 20260505 04:14:02.426 | INFO | } | 20260505 04:14:02.426 | INFO | Perf report for the last 5.13937 seconds: | 20260505 04:14:02.426 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:14:02.426 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:02.426 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:02.426 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:02.426 | INFO | ANALYSIS : 58.5%, 3.0, 91.1ms, 0 | 20260505 04:14:02.426 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:02.426 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260505 04:14:02.426 | INFO | PRE_HQ : 58.0%, 2.2, 65.4ms, 0 | 20260505 04:14:02.426 | INFO | BAZWRITER : 58.8%, 3.6, 108.2ms, 0 | 20260505 04:14:02.426 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:14:02.426 | INFO | Uploaded 36GiB to GPU in 1.63334s (22.0407GiB/s) | 20260505 04:14:02.426 | INFO | Downloaded 38GiB to GPU in 1.69557s (22.4114GiB/s) | 20260505 04:14:07.565 | INFO | Analyzing chunk frames = [254464,254976) | 20260505 04:14:07.565 | INFO | MemInfo: { | 20260505 04:14:07.565 | INFO | "gpu" : " 17 GB", | 20260505 04:14:07.565 | INFO | "huge" : "", | 20260505 04:14:07.565 | INFO | "huge_shared" : " 27 GB", | 20260505 04:14:07.565 | INFO | "malloc" : " 14 GB", | 20260505 04:14:07.565 | INFO | "pinned" : " 29 GB" | 20260505 04:14:07.565 | INFO | } | 20260505 04:14:07.565 | INFO | Perf report for the last 5.13932 seconds: | 20260505 04:14:07.565 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:14:07.565 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:07.565 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:07.565 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:07.565 | INFO | ANALYSIS : 58.2%, 3.0, 90.8ms, 0 | 20260505 04:14:07.565 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:07.565 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:07.565 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260505 04:14:07.565 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:07.565 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:07.565 | INFO | Uploaded 36GiB to GPU in 1.59793s (22.5291GiB/s) | 20260505 04:14:07.565 | INFO | Downloaded 34GiB to GPU in 1.51432s (22.4523GiB/s) | 20260505 04:14:12.704 | INFO | Analyzing chunk frames = [254976,255488) | 20260505 04:14:12.704 | INFO | MemInfo: { | 20260505 04:14:12.704 | INFO | "gpu" : " 17 GB", | 20260505 04:14:12.704 | INFO | "huge" : "", | 20260505 04:14:12.704 | INFO | "huge_shared" : " 27 GB", | 20260505 04:14:12.704 | INFO | "malloc" : " 14 GB", | 20260505 04:14:12.704 | INFO | "pinned" : " 29 GB" | 20260505 04:14:12.704 | INFO | } | 20260505 04:14:12.704 | INFO | Perf report for the last 5.13921 seconds: | 20260505 04:14:12.704 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:14:12.704 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:12.704 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:12.704 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:12.704 | INFO | ANALYSIS : 57.6%, 3.0, 89.6ms, 0 | 20260505 04:14:12.704 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:12.704 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:12.704 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 04:14:12.704 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:12.704 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:12.704 | INFO | Uploaded 36GiB to GPU in 1.60135s (22.481GiB/s) | 20260505 04:14:12.705 | INFO | Downloaded 34GiB to GPU in 1.51835s (22.3928GiB/s) | 20260505 04:14:17.843 | INFO | Analyzing chunk frames = [255488,256000) | 20260505 04:14:17.843 | INFO | MemInfo: { | 20260505 04:14:17.843 | INFO | "gpu" : " 17 GB", | 20260505 04:14:17.843 | INFO | "huge" : "", | 20260505 04:14:17.843 | INFO | "huge_shared" : " 27 GB", | 20260505 04:14:17.844 | INFO | "malloc" : " 14 GB", | 20260505 04:14:17.844 | INFO | "pinned" : " 29 GB" | 20260505 04:14:17.844 | INFO | } | 20260505 04:14:17.844 | INFO | Perf report for the last 5.13951 seconds: | 20260505 04:14:17.844 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:14:17.844 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:17.844 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:17.844 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:17.844 | INFO | ANALYSIS : 57.3%, 3.0, 88.8ms, 0 | 20260505 04:14:17.844 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:17.844 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:17.844 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260505 04:14:17.844 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:17.844 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:17.844 | INFO | Uploaded 36GiB to GPU in 1.60137s (22.4807GiB/s) | 20260505 04:14:17.844 | INFO | Downloaded 34GiB to GPU in 1.51806s (22.397GiB/s) | 20260505 04:14:22.983 | INFO | Analyzing chunk frames = [256000,256512) | 20260505 04:14:22.983 | INFO | MemInfo: { | 20260505 04:14:22.983 | INFO | "gpu" : " 17 GB", | 20260505 04:14:22.983 | INFO | "huge" : "", | 20260505 04:14:22.983 | INFO | "huge_shared" : " 27 GB", | 20260505 04:14:22.983 | INFO | "malloc" : " 14 GB", | 20260505 04:14:22.983 | INFO | "pinned" : " 29 GB" | 20260505 04:14:22.983 | INFO | } | 20260505 04:14:22.983 | INFO | Perf report for the last 5.13926 seconds: | 20260505 04:14:22.983 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:14:22.983 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:22.983 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:22.983 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:22.983 | INFO | ANALYSIS : 57.2%, 3.0, 89.3ms, 0 | 20260505 04:14:22.983 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:22.983 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:22.983 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260505 04:14:22.983 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:22.983 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:22.983 | INFO | Uploaded 36GiB to GPU in 1.599s (22.5141GiB/s) | 20260505 04:14:22.983 | INFO | Downloaded 34GiB to GPU in 1.51448s (22.4499GiB/s) | 20260505 04:14:28.122 | INFO | Analyzing chunk frames = [256512,257024) | 20260505 04:14:28.122 | INFO | MemInfo: { | 20260505 04:14:28.122 | INFO | "gpu" : " 17 GB", | 20260505 04:14:28.122 | INFO | "huge" : "", | 20260505 04:14:28.122 | INFO | "huge_shared" : " 27 GB", | 20260505 04:14:28.122 | INFO | "malloc" : " 14 GB", | 20260505 04:14:28.122 | INFO | "pinned" : " 29 GB" | 20260505 04:14:28.122 | INFO | } | 20260505 04:14:28.122 | INFO | Perf report for the last 5.1393 seconds: | 20260505 04:14:28.122 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:14:28.122 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:28.122 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:28.122 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:28.122 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260505 04:14:28.122 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:28.122 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:28.122 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 04:14:28.122 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:28.122 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:28.123 | INFO | Uploaded 36GiB to GPU in 1.59903s (22.5136GiB/s) | 20260505 04:14:28.123 | INFO | Downloaded 34GiB to GPU in 1.51552s (22.4345GiB/s) | 20260505 04:14:33.261 | INFO | Analyzing chunk frames = [257024,257536) | 20260505 04:14:33.261 | INFO | MemInfo: { | 20260505 04:14:33.261 | INFO | "gpu" : " 17 GB", | 20260505 04:14:33.261 | INFO | "huge" : "", | 20260505 04:14:33.261 | INFO | "huge_shared" : " 27 GB", | 20260505 04:14:33.262 | INFO | "malloc" : " 14 GB", | 20260505 04:14:33.262 | INFO | "pinned" : " 29 GB" | 20260505 04:14:33.262 | INFO | } | 20260505 04:14:33.262 | INFO | Perf report for the last 5.13932 seconds: | 20260505 04:14:33.262 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:14:33.262 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:33.262 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:33.262 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:33.262 | INFO | ANALYSIS : 57.7%, 3.0, 90.1ms, 0 | 20260505 04:14:33.262 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:33.262 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:33.262 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260505 04:14:33.262 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:33.262 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:33.262 | INFO | Uploaded 36GiB to GPU in 1.5993s (22.5098GiB/s) | 20260505 04:14:33.262 | INFO | Downloaded 34GiB to GPU in 1.51721s (22.4095GiB/s) | 20260505 04:14:38.401 | INFO | Analyzing chunk frames = [257536,258048) | 20260505 04:14:38.401 | INFO | MemInfo: { | 20260505 04:14:38.401 | INFO | "gpu" : " 17 GB", | 20260505 04:14:38.401 | INFO | "huge" : "", | 20260505 04:14:38.401 | INFO | "huge_shared" : " 27 GB", | 20260505 04:14:38.401 | INFO | "malloc" : " 14 GB", | 20260505 04:14:38.401 | INFO | "pinned" : " 29 GB" | 20260505 04:14:38.401 | INFO | } | 20260505 04:14:38.401 | INFO | Perf report for the last 5.13946 seconds: | 20260505 04:14:38.401 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:14:38.401 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:38.401 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:38.401 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:38.401 | INFO | ANALYSIS : 57.8%, 3.0, 90.2ms, 0 | 20260505 04:14:38.401 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:38.401 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:38.401 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 04:14:38.401 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:38.401 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:38.402 | INFO | Uploaded 36GiB to GPU in 1.60008s (22.4989GiB/s) | 20260505 04:14:38.402 | INFO | Downloaded 34GiB to GPU in 1.51694s (22.4135GiB/s) | 20260505 04:14:38.498 | INFO | Starting a round of Baz IO | 20260505 04:14:41.461 | INFO | Populating RTMetrics report | 20260505 04:14:41.462 | INFO | RTMetrics report populated | 20260505 04:14:41.462 | INFO | Opening tmp RTMetrics json file | 20260505 04:14:41.463 | INFO | Serializing RTMetrics to json | 20260505 04:14:41.464 | INFO | Serialization done, swapping tmp file now... | 20260505 04:14:41.464 | INFO | RTMetrics done | 20260505 04:14:41.544 | INFO | Finished a round of Baz IO: Wrote 1.836 GiB in 3.045 seconds | 20260505 04:14:41.544 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 51.83% / 0.21% / 47.96% | 20260505 04:14:41.544 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 12.2866% / 0.000532468% / 3.57628e-05% | 20260505 04:14:41.544 | INFO | Baz IO overall: | 20260505 04:14:41.544 | INFO | BytesWritten : 88699244050 | 20260505 04:14:41.544 | INFO | NumEvents : 40472913787 | 20260505 04:14:43.540 | INFO | Analyzing chunk frames = [258048,258560) | 20260505 04:14:43.540 | INFO | MemInfo: { | 20260505 04:14:43.540 | INFO | "gpu" : " 17 GB", | 20260505 04:14:43.540 | INFO | "huge" : "", | 20260505 04:14:43.540 | INFO | "huge_shared" : " 27 GB", | 20260505 04:14:43.540 | INFO | "malloc" : " 14 GB", | 20260505 04:14:43.540 | INFO | "pinned" : " 29 GB" | 20260505 04:14:43.540 | INFO | } | 20260505 04:14:43.540 | INFO | Perf report for the last 5.13927 seconds: | 20260505 04:14:43.540 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:14:43.540 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:43.540 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:43.540 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:43.540 | INFO | ANALYSIS : 58.5%, 3.0, 91.1ms, 0 | 20260505 04:14:43.541 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:43.541 | INFO | RT_METRICS : 44.7%, 1.0, 23.5ms, 0 | 20260505 04:14:43.541 | INFO | PRE_HQ : 57.9%, 2.2, 64.6ms, 0 | 20260505 04:14:43.541 | INFO | BAZWRITER : 58.8%, 3.6, 108.7ms, 0 | 20260505 04:14:43.541 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:14:43.541 | INFO | Uploaded 36GiB to GPU in 1.63963s (21.9562GiB/s) | 20260505 04:14:43.541 | INFO | Downloaded 38GiB to GPU in 1.69931s (22.362GiB/s) | 20260505 04:14:48.680 | INFO | Analyzing chunk frames = [258560,259072) | 20260505 04:14:48.680 | INFO | MemInfo: { | 20260505 04:14:48.680 | INFO | "gpu" : " 17 GB", | 20260505 04:14:48.680 | INFO | "huge" : "", | 20260505 04:14:48.680 | INFO | "huge_shared" : " 27 GB", | 20260505 04:14:48.680 | INFO | "malloc" : " 14 GB", | 20260505 04:14:48.680 | INFO | "pinned" : " 29 GB" | 20260505 04:14:48.680 | INFO | } | 20260505 04:14:48.680 | INFO | Perf report for the last 5.13935 seconds: | 20260505 04:14:48.680 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:14:48.680 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:48.680 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:48.680 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:48.680 | INFO | ANALYSIS : 58.2%, 3.0, 90.8ms, 0 | 20260505 04:14:48.680 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:48.680 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:48.680 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260505 04:14:48.680 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:48.680 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:48.680 | INFO | Uploaded 36GiB to GPU in 1.59789s (22.5297GiB/s) | 20260505 04:14:48.680 | INFO | Downloaded 34GiB to GPU in 1.51372s (22.4612GiB/s) | 20260505 04:14:53.819 | INFO | Analyzing chunk frames = [259072,259584) | 20260505 04:14:53.819 | INFO | MemInfo: { | 20260505 04:14:53.819 | INFO | "gpu" : " 17 GB", | 20260505 04:14:53.819 | INFO | "huge" : "", | 20260505 04:14:53.819 | INFO | "huge_shared" : " 27 GB", | 20260505 04:14:53.819 | INFO | "malloc" : " 14 GB", | 20260505 04:14:53.819 | INFO | "pinned" : " 29 GB" | 20260505 04:14:53.819 | INFO | } | 20260505 04:14:53.819 | INFO | Perf report for the last 5.13925 seconds: | 20260505 04:14:53.819 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:14:53.819 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:53.819 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:53.819 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:53.819 | INFO | ANALYSIS : 57.6%, 3.0, 89.5ms, 0 | 20260505 04:14:53.819 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:53.819 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:53.819 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 04:14:53.819 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:53.819 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:53.819 | INFO | Uploaded 36GiB to GPU in 1.60051s (22.4928GiB/s) | 20260505 04:14:53.819 | INFO | Downloaded 34GiB to GPU in 1.51819s (22.395GiB/s) | 20260505 04:14:58.959 | INFO | Analyzing chunk frames = [259584,260096) | 20260505 04:14:58.959 | INFO | MemInfo: { | 20260505 04:14:58.959 | INFO | "gpu" : " 17 GB", | 20260505 04:14:58.959 | INFO | "huge" : "", | 20260505 04:14:58.959 | INFO | "huge_shared" : " 27 GB", | 20260505 04:14:58.959 | INFO | "malloc" : " 14 GB", | 20260505 04:14:58.959 | INFO | "pinned" : " 29 GB" | 20260505 04:14:58.959 | INFO | } | 20260505 04:14:58.959 | INFO | Perf report for the last 5.13972 seconds: | 20260505 04:14:58.959 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:14:58.959 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:58.959 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:58.959 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:58.959 | INFO | ANALYSIS : 57.2%, 3.0, 88.8ms, 0 | 20260505 04:14:58.959 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:58.959 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:58.959 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 04:14:58.959 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:58.959 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:14:58.959 | INFO | Uploaded 36GiB to GPU in 1.60072s (22.4899GiB/s) | 20260505 04:14:58.959 | INFO | Downloaded 34GiB to GPU in 1.51694s (22.4135GiB/s) | 20260505 04:15:04.097 | INFO | Analyzing chunk frames = [260096,260608) | 20260505 04:15:04.097 | INFO | MemInfo: { | 20260505 04:15:04.097 | INFO | "gpu" : " 17 GB", | 20260505 04:15:04.098 | INFO | "huge" : "", | 20260505 04:15:04.098 | INFO | "huge_shared" : " 27 GB", | 20260505 04:15:04.098 | INFO | "malloc" : " 14 GB", | 20260505 04:15:04.098 | INFO | "pinned" : " 29 GB" | 20260505 04:15:04.098 | INFO | } | 20260505 04:15:04.098 | INFO | Perf report for the last 5.13897 seconds: | 20260505 04:15:04.098 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:15:04.098 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:04.098 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:04.098 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:04.098 | INFO | ANALYSIS : 57.2%, 3.0, 89.3ms, 0 | 20260505 04:15:04.098 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:04.098 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:04.098 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260505 04:15:04.098 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:04.098 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:04.098 | INFO | Uploaded 36GiB to GPU in 1.59846s (22.5216GiB/s) | 20260505 04:15:04.098 | INFO | Downloaded 34GiB to GPU in 1.51486s (22.4443GiB/s) | 20260505 04:15:09.237 | INFO | Analyzing chunk frames = [260608,261120) | 20260505 04:15:09.237 | INFO | MemInfo: { | 20260505 04:15:09.237 | INFO | "gpu" : " 17 GB", | 20260505 04:15:09.237 | INFO | "huge" : "", | 20260505 04:15:09.237 | INFO | "huge_shared" : " 27 GB", | 20260505 04:15:09.237 | INFO | "malloc" : " 14 GB", | 20260505 04:15:09.237 | INFO | "pinned" : " 29 GB" | 20260505 04:15:09.237 | INFO | } | 20260505 04:15:09.237 | INFO | Perf report for the last 5.1395 seconds: | 20260505 04:15:09.237 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:15:09.237 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:09.237 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:09.237 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:09.237 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260505 04:15:09.237 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:09.237 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:09.237 | INFO | PRE_HQ : 32.3%, 1.0, 16.7ms, 0 | 20260505 04:15:09.237 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:09.237 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:09.238 | INFO | Uploaded 36GiB to GPU in 1.59875s (22.5176GiB/s) | 20260505 04:15:09.238 | INFO | Downloaded 34GiB to GPU in 1.5153s (22.4378GiB/s) | 20260505 04:15:14.376 | INFO | Analyzing chunk frames = [261120,261632) | 20260505 04:15:14.376 | INFO | MemInfo: { | 20260505 04:15:14.376 | INFO | "gpu" : " 17 GB", | 20260505 04:15:14.377 | INFO | "huge" : "", | 20260505 04:15:14.377 | INFO | "huge_shared" : " 27 GB", | 20260505 04:15:14.377 | INFO | "malloc" : " 14 GB", | 20260505 04:15:14.377 | INFO | "pinned" : " 29 GB" | 20260505 04:15:14.377 | INFO | } | 20260505 04:15:14.377 | INFO | Perf report for the last 5.13953 seconds: | 20260505 04:15:14.377 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:15:14.377 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:14.377 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:14.377 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:14.377 | INFO | ANALYSIS : 57.7%, 3.0, 90.0ms, 0 | 20260505 04:15:14.377 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:14.377 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:14.377 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 04:15:14.377 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:14.377 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:14.377 | INFO | Uploaded 36GiB to GPU in 1.59981s (22.5027GiB/s) | 20260505 04:15:14.377 | INFO | Downloaded 34GiB to GPU in 1.51604s (22.4268GiB/s) | 20260505 04:15:19.516 | INFO | Analyzing chunk frames = [261632,262144) | 20260505 04:15:19.516 | INFO | MemInfo: { | 20260505 04:15:19.516 | INFO | "gpu" : " 17 GB", | 20260505 04:15:19.516 | INFO | "huge" : "", | 20260505 04:15:19.516 | INFO | "huge_shared" : " 27 GB", | 20260505 04:15:19.516 | INFO | "malloc" : " 14 GB", | 20260505 04:15:19.516 | INFO | "pinned" : " 29 GB" | 20260505 04:15:19.516 | INFO | } | 20260505 04:15:19.516 | INFO | Perf report for the last 5.13909 seconds: | 20260505 04:15:19.516 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:15:19.516 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:19.516 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:19.516 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:19.516 | INFO | ANALYSIS : 57.9%, 3.0, 90.3ms, 0 | 20260505 04:15:19.516 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:19.516 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:19.516 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260505 04:15:19.516 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:19.516 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:19.516 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 04:15:19.516 | INFO | Total mem usage: 14.9 GiB | 20260505 04:15:19.516 | INFO | All Time High Water Marks sum: 15.2 GiB (Savings: 1.92%) | 20260505 04:15:19.516 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -2.66%) | 20260505 04:15:19.516 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 04:15:19.516 | INFO | High Water Breakdown: | 20260505 04:15:19.516 | INFO | 58.8% : 8.55 GiB ( 100% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 04:15:19.516 | INFO | 41% : 5.97 GiB ( 89.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 04:15:19.516 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 04:15:19.516 | INFO | Uploaded 36GiB to GPU in 1.59965s (22.5049GiB/s) | 20260505 04:15:19.516 | INFO | ------------------------------------------ | 20260505 04:15:19.516 | INFO | Downloaded 34GiB to GPU in 1.51627s (22.4235GiB/s) | 20260505 04:15:19.517 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:15:19.517 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 04:15:19.517 | INFO | Total mem usage: 26.5 GiB | 20260505 04:15:19.517 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260505 04:15:19.517 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 04:15:19.517 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260505 04:15:19.517 | INFO | High Water Breakdown: | 20260505 04:15:19.517 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:15:19.517 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:15:19.517 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:15:19.517 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:15:19.517 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:15:19.517 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:15:19.517 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 04:15:19.517 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 04:15:19.517 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:15:19.517 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 04:15:19.517 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 04:15:19.517 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 04:15:19.517 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 04:15:19.517 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 04:15:19.517 | INFO | ------------------------------------------ | 20260505 04:15:19.517 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:15:19.517 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 04:15:19.517 | INFO | Total mem usage: 25.6 GiB | 20260505 04:15:19.517 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 04:15:19.517 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 04:15:19.517 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 04:15:19.517 | INFO | High Water Breakdown: | 20260505 04:15:19.517 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 04:15:19.517 | INFO | ------------------------------------------ | 20260505 04:15:19.517 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:15:19.517 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 04:15:19.517 | INFO | Total mem usage: 15.7 GiB | 20260505 04:15:19.517 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260505 04:15:19.517 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 04:15:19.517 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260505 04:15:19.517 | INFO | High Water Breakdown: | 20260505 04:15:19.517 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 04:15:19.517 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 04:15:19.517 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 04:15:19.517 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 04:15:19.517 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 04:15:19.517 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 04:15:19.517 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 04:15:19.517 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 04:15:19.517 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 04:15:19.517 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:15:19.517 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 04:15:19.517 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:15:19.517 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 04:15:19.517 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 04:15:19.517 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 04:15:19.517 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 04:15:19.517 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:15:19.517 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 04:15:19.517 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:15:19.517 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:15:19.517 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:15:19.517 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 04:15:19.517 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 04:15:19.517 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:15:19.517 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 04:15:19.517 | INFO | ------------------------------------------ | 20260505 04:15:19.517 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:15:19.517 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 04:15:19.517 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:15:19.517 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:15:19.517 | INFO | PerfReport: Full Execution -- 3.06e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260505 04:15:19.517 | INFO | PerfReport: 0.28%, AggregateStats -- 8.62e-02 / -nan, 3.96e-03 / 0.00e+00 | 20260505 04:15:19.517 | INFO | PerfReport: 10.79%, Baseline -- 3.30e+00 / -nan, 1.86e-02 / 0.00e+00 | 20260505 04:15:19.517 | INFO | PerfReport: 3.02%, Histogram -- 9.24e-01 / -nan, 5.16e-02 / 0.00e+00 | 20260505 04:15:19.517 | INFO | PerfReport: 16.01%, DME -- 3.91e+01 / -nan, 3.24e+00 / 0.00e+00 | 20260505 04:15:19.517 | INFO | PerfReport: 56.33%, FrameLabeling -- 1.72e+01 / -nan, 4.47e-02 / 0.00e+00 | 20260505 04:15:19.518 | INFO | PerfReport: 7.73%, PulseAccumulating -- 2.36e+00 / -nan, 1.59e-01 / 0.00e+00 | 20260505 04:15:19.518 | INFO | PerfReport: 4.11%, Metrics -- 1.26e+00 / -nan, 2.05e-01 / 0.00e+00 | 20260505 04:15:19.518 | INFO | PerfReport: 1.71%, PulseSerializing -- 5.23e-01 / -nan, 8.42e-02 / 0.00e+00 | 20260505 04:15:19.518 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 04:15:19.518 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:15:19.518 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:15:19.518 | INFO | PerfReport: Full Execution -- 9.18e+01 / -nan, 1.91e+01 / 0.00e+00 | 20260505 04:15:19.518 | INFO | PerfReport: 18.20%, Upload -- 1.67e+01 / 0.00e+00, 3.63e-01 / 0.00e+00 | 20260505 04:15:19.518 | INFO | PerfReport: 17.46%, Download -- 1.60e+01 / 0.00e+00, 6.84e-01 / 0.00e+00 | 20260505 04:15:19.518 | INFO | PerfReport: 33.30%, Compute -- 3.06e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260505 04:15:19.518 | INFO | PerfReport: 0.55%, LockUpload -- 5.05e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260505 04:15:19.518 | INFO | PerfReport: 0.00%, LockDownload -- 3.92e-04 / 0.00e+00, 1.03e-04 / 0.00e+00 | 20260505 04:15:19.518 | INFO | PerfReport: 30.47%, LockCompute -- 2.80e+01 / 0.00e+00, 1.72e+01 / 0.00e+00 | 20260505 04:15:19.518 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.67e-02 / 0.00e+00 | 20260505 04:15:19.613 | INFO | Starting a round of Baz IO | 20260505 04:15:22.574 | INFO | Populating RTMetrics report | 20260505 04:15:22.575 | INFO | RTMetrics report populated | 20260505 04:15:22.575 | INFO | Opening tmp RTMetrics json file | 20260505 04:15:22.577 | INFO | Serializing RTMetrics to json | 20260505 04:15:22.578 | INFO | Serialization done, swapping tmp file now... | 20260505 04:15:22.579 | INFO | RTMetrics done | 20260505 04:15:22.656 | INFO | Finished a round of Baz IO: Wrote 1.843 GiB in 3.042 seconds | 20260505 04:15:22.656 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 51.63% / 0.20% / 48.17% | 20260505 04:15:22.656 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 12.2705% / 0.000627836% / 5.16574e-05% | 20260505 04:15:22.656 | INFO | Baz IO overall: | 20260505 04:15:22.656 | INFO | BytesWritten : 90678520633 | 20260505 04:15:22.656 | INFO | NumEvents : 41407580539 | 20260505 04:15:24.655 | INFO | Analyzing chunk frames = [262144,262656) | 20260505 04:15:24.655 | INFO | MemInfo: { | 20260505 04:15:24.655 | INFO | "gpu" : " 17 GB", | 20260505 04:15:24.655 | INFO | "huge" : "", | 20260505 04:15:24.655 | INFO | "huge_shared" : " 27 GB", | 20260505 04:15:24.655 | INFO | "malloc" : " 14 GB", | 20260505 04:15:24.655 | INFO | "pinned" : " 29 GB" | 20260505 04:15:24.655 | INFO | } | 20260505 04:15:24.655 | INFO | Perf report for the last 5.13922 seconds: | 20260505 04:15:24.655 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:15:24.655 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:24.655 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:24.655 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:24.655 | INFO | ANALYSIS : 58.4%, 3.0, 91.1ms, 0 | 20260505 04:15:24.655 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:24.655 | INFO | RT_METRICS : 44.9%, 1.0, 23.5ms, 0 | 20260505 04:15:24.655 | INFO | PRE_HQ : 58.0%, 2.2, 65.2ms, 0 | 20260505 04:15:24.655 | INFO | BAZWRITER : 58.8%, 3.6, 108.6ms, 0 | 20260505 04:15:24.655 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:15:24.655 | INFO | Uploaded 36GiB to GPU in 1.63608s (22.0038GiB/s) | 20260505 04:15:24.655 | INFO | Downloaded 38GiB to GPU in 1.6981s (22.378GiB/s) | 20260505 04:15:29.794 | INFO | Analyzing chunk frames = [262656,263168) | 20260505 04:15:29.794 | INFO | MemInfo: { | 20260505 04:15:29.794 | INFO | "gpu" : " 17 GB", | 20260505 04:15:29.794 | INFO | "huge" : "", | 20260505 04:15:29.794 | INFO | "huge_shared" : " 27 GB", | 20260505 04:15:29.794 | INFO | "malloc" : " 14 GB", | 20260505 04:15:29.794 | INFO | "pinned" : " 29 GB" | 20260505 04:15:29.794 | INFO | } | 20260505 04:15:29.795 | INFO | Perf report for the last 5.13943 seconds: | 20260505 04:15:29.795 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:15:29.795 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:29.795 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:29.795 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:29.795 | INFO | ANALYSIS : 58.3%, 3.0, 90.9ms, 0 | 20260505 04:15:29.795 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:29.795 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:29.795 | INFO | PRE_HQ : 31.7%, 1.0, 16.4ms, 0 | 20260505 04:15:29.795 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:29.795 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:29.795 | INFO | Uploaded 36GiB to GPU in 1.59576s (22.5598GiB/s) | 20260505 04:15:29.795 | INFO | Downloaded 34GiB to GPU in 1.51329s (22.4676GiB/s) | 20260505 04:15:34.934 | INFO | Analyzing chunk frames = [263168,263680) | 20260505 04:15:34.934 | INFO | MemInfo: { | 20260505 04:15:34.934 | INFO | "gpu" : " 17 GB", | 20260505 04:15:34.934 | INFO | "huge" : "", | 20260505 04:15:34.934 | INFO | "huge_shared" : " 27 GB", | 20260505 04:15:34.934 | INFO | "malloc" : " 14 GB", | 20260505 04:15:34.934 | INFO | "pinned" : " 29 GB" | 20260505 04:15:34.934 | INFO | } | 20260505 04:15:34.934 | INFO | Perf report for the last 5.13941 seconds: | 20260505 04:15:34.934 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:15:34.934 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:34.934 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:34.934 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:34.934 | INFO | ANALYSIS : 57.6%, 3.0, 89.5ms, 0 | 20260505 04:15:34.934 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:34.934 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:34.934 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 04:15:34.934 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:34.934 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:34.934 | INFO | Uploaded 36GiB to GPU in 1.60054s (22.4923GiB/s) | 20260505 04:15:34.934 | INFO | Downloaded 34GiB to GPU in 1.51803s (22.3974GiB/s) | 20260505 04:15:40.088 | INFO | Analyzing chunk frames = [263680,264192) | 20260505 04:15:40.088 | INFO | MemInfo: { | 20260505 04:15:40.088 | INFO | "gpu" : " 17 GB", | 20260505 04:15:40.088 | INFO | "huge" : "", | 20260505 04:15:40.088 | INFO | "huge_shared" : " 27 GB", | 20260505 04:15:40.088 | INFO | "malloc" : " 14 GB", | 20260505 04:15:40.088 | INFO | "pinned" : " 29 GB" | 20260505 04:15:40.088 | INFO | } | 20260505 04:15:40.088 | INFO | Perf report for the last 5.15451 seconds: | 20260505 04:15:40.088 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:15:40.088 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:40.088 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:40.089 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:40.089 | INFO | ANALYSIS : 57.1%, 3.0, 88.9ms, 0 | 20260505 04:15:40.089 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:40.089 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:40.089 | INFO | PRE_HQ : 32.2%, 1.0, 16.8ms, 0 | 20260505 04:15:40.089 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:40.089 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:40.089 | INFO | Uploaded 36GiB to GPU in 1.59958s (22.5058GiB/s) | 20260505 04:15:40.089 | INFO | Downloaded 34GiB to GPU in 1.51741s (22.4066GiB/s) | 20260505 04:15:45.213 | INFO | Analyzing chunk frames = [264192,264704) | 20260505 04:15:45.213 | INFO | MemInfo: { | 20260505 04:15:45.213 | INFO | "gpu" : " 17 GB", | 20260505 04:15:45.213 | INFO | "huge" : "", | 20260505 04:15:45.213 | INFO | "huge_shared" : " 27 GB", | 20260505 04:15:45.213 | INFO | "malloc" : " 14 GB", | 20260505 04:15:45.213 | INFO | "pinned" : " 29 GB" | 20260505 04:15:45.213 | INFO | } | 20260505 04:15:45.213 | INFO | Perf report for the last 5.12438 seconds: | 20260505 04:15:45.213 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:15:45.213 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:45.213 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:45.213 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:45.213 | INFO | ANALYSIS : 57.4%, 3.0, 89.2ms, 0 | 20260505 04:15:45.213 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:45.213 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:45.213 | INFO | PRE_HQ : 32.5%, 1.0, 16.8ms, 0 | 20260505 04:15:45.213 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:45.213 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:45.213 | INFO | Uploaded 36GiB to GPU in 1.59939s (22.5086GiB/s) | 20260505 04:15:45.213 | INFO | Downloaded 34GiB to GPU in 1.51643s (22.4211GiB/s) | 20260505 04:15:50.352 | INFO | Analyzing chunk frames = [264704,265216) | 20260505 04:15:50.352 | INFO | MemInfo: { | 20260505 04:15:50.352 | INFO | "gpu" : " 17 GB", | 20260505 04:15:50.352 | INFO | "huge" : "", | 20260505 04:15:50.352 | INFO | "huge_shared" : " 27 GB", | 20260505 04:15:50.352 | INFO | "malloc" : " 14 GB", | 20260505 04:15:50.352 | INFO | "pinned" : " 29 GB" | 20260505 04:15:50.352 | INFO | } | 20260505 04:15:50.352 | INFO | Perf report for the last 5.13908 seconds: | 20260505 04:15:50.352 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:15:50.352 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:50.352 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:50.352 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:50.352 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260505 04:15:50.352 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:50.352 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:50.352 | INFO | PRE_HQ : 32.9%, 1.0, 17.1ms, 0 | 20260505 04:15:50.352 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:50.352 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:50.352 | INFO | Uploaded 36GiB to GPU in 1.59907s (22.5131GiB/s) | 20260505 04:15:50.353 | INFO | Downloaded 34GiB to GPU in 1.51568s (22.4322GiB/s) | 20260505 04:15:55.491 | INFO | Analyzing chunk frames = [265216,265728) | 20260505 04:15:55.491 | INFO | MemInfo: { | 20260505 04:15:55.491 | INFO | "gpu" : " 17 GB", | 20260505 04:15:55.491 | INFO | "huge" : "", | 20260505 04:15:55.491 | INFO | "huge_shared" : " 27 GB", | 20260505 04:15:55.491 | INFO | "malloc" : " 14 GB", | 20260505 04:15:55.491 | INFO | "pinned" : " 29 GB" | 20260505 04:15:55.491 | INFO | } | 20260505 04:15:55.491 | INFO | Perf report for the last 5.13912 seconds: | 20260505 04:15:55.491 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:15:55.491 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:55.491 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:55.491 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:55.491 | INFO | ANALYSIS : 57.7%, 3.0, 90.0ms, 0 | 20260505 04:15:55.491 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:55.491 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:55.491 | INFO | PRE_HQ : 32.5%, 1.0, 16.8ms, 0 | 20260505 04:15:55.491 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:55.491 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:15:55.491 | INFO | Uploaded 36GiB to GPU in 1.59931s (22.5097GiB/s) | 20260505 04:15:55.492 | INFO | Downloaded 34GiB to GPU in 1.51652s (22.4197GiB/s) | 20260505 04:16:00.630 | INFO | Analyzing chunk frames = [265728,266240) | 20260505 04:16:00.630 | INFO | MemInfo: { | 20260505 04:16:00.630 | INFO | "gpu" : " 17 GB", | 20260505 04:16:00.630 | INFO | "huge" : "", | 20260505 04:16:00.630 | INFO | "huge_shared" : " 27 GB", | 20260505 04:16:00.630 | INFO | "malloc" : " 14 GB", | 20260505 04:16:00.631 | INFO | "pinned" : " 29 GB" | 20260505 04:16:00.631 | INFO | } | 20260505 04:16:00.631 | INFO | Perf report for the last 5.13962 seconds: | 20260505 04:16:00.631 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:16:00.631 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:00.631 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:00.631 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:00.631 | INFO | ANALYSIS : 57.8%, 3.0, 90.2ms, 0 | 20260505 04:16:00.631 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:00.631 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:00.631 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 04:16:00.631 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:00.631 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:00.631 | INFO | Uploaded 36GiB to GPU in 1.59899s (22.5142GiB/s) | 20260505 04:16:00.631 | INFO | Downloaded 34GiB to GPU in 1.51673s (22.4167GiB/s) | 20260505 04:16:00.727 | INFO | Starting a round of Baz IO | 20260505 04:16:03.690 | INFO | Populating RTMetrics report | 20260505 04:16:03.690 | INFO | RTMetrics report populated | 20260505 04:16:03.690 | INFO | Opening tmp RTMetrics json file | 20260505 04:16:03.691 | INFO | Serializing RTMetrics to json | 20260505 04:16:03.692 | INFO | Serialization done, swapping tmp file now... | 20260505 04:16:03.692 | INFO | RTMetrics done | 20260505 04:16:03.775 | INFO | Finished a round of Baz IO: Wrote 1.851 GiB in 3.047 seconds | 20260505 04:16:03.775 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 51.43% / 0.20% / 48.37% | 20260505 04:16:03.775 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 12.2275% / 0.000596046% / 2.38419e-05% | 20260505 04:16:03.775 | INFO | Baz IO overall: | 20260505 04:16:03.775 | INFO | BytesWritten : 92665492708 | 20260505 04:16:03.775 | INFO | NumEvents : 42346545628 | 20260505 04:16:05.770 | INFO | Analyzing chunk frames = [266240,266752) | 20260505 04:16:05.770 | INFO | MemInfo: { | 20260505 04:16:05.770 | INFO | "gpu" : " 17 GB", | 20260505 04:16:05.770 | INFO | "huge" : "", | 20260505 04:16:05.770 | INFO | "huge_shared" : " 27 GB", | 20260505 04:16:05.770 | INFO | "malloc" : " 14 GB", | 20260505 04:16:05.770 | INFO | "pinned" : " 29 GB" | 20260505 04:16:05.770 | INFO | } | 20260505 04:16:05.770 | INFO | Perf report for the last 5.13921 seconds: | 20260505 04:16:05.770 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:16:05.770 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:05.770 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:05.770 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:05.770 | INFO | ANALYSIS : 58.4%, 3.0, 91.0ms, 0 | 20260505 04:16:05.770 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:05.770 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260505 04:16:05.770 | INFO | PRE_HQ : 57.9%, 2.2, 64.9ms, 0 | 20260505 04:16:05.770 | INFO | BAZWRITER : 58.8%, 3.6, 109.0ms, 0 | 20260505 04:16:05.770 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:16:05.770 | INFO | Uploaded 36GiB to GPU in 1.63879s (21.9675GiB/s) | 20260505 04:16:05.770 | INFO | Downloaded 38GiB to GPU in 1.69916s (22.364GiB/s) | 20260505 04:16:10.909 | INFO | Analyzing chunk frames = [266752,267264) | 20260505 04:16:10.909 | INFO | MemInfo: { | 20260505 04:16:10.909 | INFO | "gpu" : " 17 GB", | 20260505 04:16:10.909 | INFO | "huge" : "", | 20260505 04:16:10.909 | INFO | "huge_shared" : " 27 GB", | 20260505 04:16:10.909 | INFO | "malloc" : " 14 GB", | 20260505 04:16:10.909 | INFO | "pinned" : " 29 GB" | 20260505 04:16:10.909 | INFO | } | 20260505 04:16:10.909 | INFO | Perf report for the last 5.13943 seconds: | 20260505 04:16:10.909 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:16:10.909 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:10.909 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:10.910 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:10.910 | INFO | ANALYSIS : 58.2%, 3.0, 90.9ms, 0 | 20260505 04:16:10.910 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:10.910 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:10.910 | INFO | PRE_HQ : 32.4%, 1.0, 17.0ms, 0 | 20260505 04:16:10.910 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:10.910 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:10.910 | INFO | Uploaded 36GiB to GPU in 1.5971s (22.5408GiB/s) | 20260505 04:16:10.910 | INFO | Downloaded 34GiB to GPU in 1.51292s (22.4731GiB/s) | 20260505 04:16:16.048 | INFO | Analyzing chunk frames = [267264,267776) | 20260505 04:16:16.048 | INFO | MemInfo: { | 20260505 04:16:16.048 | INFO | "gpu" : " 17 GB", | 20260505 04:16:16.048 | INFO | "huge" : "", | 20260505 04:16:16.048 | INFO | "huge_shared" : " 27 GB", | 20260505 04:16:16.048 | INFO | "malloc" : " 14 GB", | 20260505 04:16:16.048 | INFO | "pinned" : " 29 GB" | 20260505 04:16:16.049 | INFO | } | 20260505 04:16:16.049 | INFO | Perf report for the last 5.13922 seconds: | 20260505 04:16:16.049 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:16:16.049 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:16.049 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:16.049 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:16.049 | INFO | ANALYSIS : 57.6%, 3.0, 89.5ms, 0 | 20260505 04:16:16.049 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:16.049 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:16.049 | INFO | PRE_HQ : 32.3%, 1.0, 16.7ms, 0 | 20260505 04:16:16.049 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:16.049 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:16.049 | INFO | Uploaded 36GiB to GPU in 1.60146s (22.4795GiB/s) | 20260505 04:16:16.049 | INFO | Downloaded 34GiB to GPU in 1.51901s (22.3829GiB/s) | 20260505 04:16:21.188 | INFO | Analyzing chunk frames = [267776,268288) | 20260505 04:16:21.188 | INFO | MemInfo: { | 20260505 04:16:21.188 | INFO | "gpu" : " 17 GB", | 20260505 04:16:21.188 | INFO | "huge" : "", | 20260505 04:16:21.188 | INFO | "huge_shared" : " 27 GB", | 20260505 04:16:21.188 | INFO | "malloc" : " 14 GB", | 20260505 04:16:21.188 | INFO | "pinned" : " 29 GB" | 20260505 04:16:21.188 | INFO | } | 20260505 04:16:21.188 | INFO | Perf report for the last 5.1394 seconds: | 20260505 04:16:21.188 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:16:21.188 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:21.188 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:21.188 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:21.188 | INFO | ANALYSIS : 57.3%, 3.0, 88.9ms, 0 | 20260505 04:16:21.188 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:21.188 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:21.188 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260505 04:16:21.188 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:21.188 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:21.188 | INFO | Uploaded 36GiB to GPU in 1.60052s (22.4927GiB/s) | 20260505 04:16:21.188 | INFO | Downloaded 34GiB to GPU in 1.51725s (22.409GiB/s) | 20260505 04:16:26.327 | INFO | Analyzing chunk frames = [268288,268800) | 20260505 04:16:26.327 | INFO | MemInfo: { | 20260505 04:16:26.327 | INFO | "gpu" : " 17 GB", | 20260505 04:16:26.327 | INFO | "huge" : "", | 20260505 04:16:26.327 | INFO | "huge_shared" : " 27 GB", | 20260505 04:16:26.327 | INFO | "malloc" : " 14 GB", | 20260505 04:16:26.327 | INFO | "pinned" : " 29 GB" | 20260505 04:16:26.327 | INFO | } | 20260505 04:16:26.327 | INFO | Perf report for the last 5.13933 seconds: | 20260505 04:16:26.327 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:16:26.327 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:26.327 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:26.327 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:26.327 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260505 04:16:26.327 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:26.327 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:26.327 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 04:16:26.328 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:26.328 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:26.328 | INFO | Uploaded 36GiB to GPU in 1.59876s (22.5175GiB/s) | 20260505 04:16:26.328 | INFO | Downloaded 34GiB to GPU in 1.51559s (22.4336GiB/s) | 20260505 04:16:31.466 | INFO | Analyzing chunk frames = [268800,269312) | 20260505 04:16:31.466 | INFO | MemInfo: { | 20260505 04:16:31.466 | INFO | "gpu" : " 17 GB", | 20260505 04:16:31.466 | INFO | "huge" : "", | 20260505 04:16:31.466 | INFO | "huge_shared" : " 27 GB", | 20260505 04:16:31.467 | INFO | "malloc" : " 14 GB", | 20260505 04:16:31.467 | INFO | "pinned" : " 29 GB" | 20260505 04:16:31.467 | INFO | } | 20260505 04:16:31.467 | INFO | Perf report for the last 5.13936 seconds: | 20260505 04:16:31.467 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:16:31.467 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:31.467 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:31.467 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:31.467 | INFO | ANALYSIS : 57.5%, 3.0, 89.8ms, 0 | 20260505 04:16:31.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:31.467 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:31.467 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260505 04:16:31.467 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:31.467 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:31.467 | INFO | Uploaded 36GiB to GPU in 1.59888s (22.5158GiB/s) | 20260505 04:16:31.467 | INFO | Downloaded 34GiB to GPU in 1.51546s (22.4354GiB/s) | 20260505 04:16:36.606 | INFO | Analyzing chunk frames = [269312,269824) | 20260505 04:16:36.606 | INFO | MemInfo: { | 20260505 04:16:36.606 | INFO | "gpu" : " 17 GB", | 20260505 04:16:36.606 | INFO | "huge" : "", | 20260505 04:16:36.606 | INFO | "huge_shared" : " 27 GB", | 20260505 04:16:36.606 | INFO | "malloc" : " 14 GB", | 20260505 04:16:36.606 | INFO | "pinned" : " 29 GB" | 20260505 04:16:36.606 | INFO | } | 20260505 04:16:36.606 | INFO | Perf report for the last 5.13929 seconds: | 20260505 04:16:36.606 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:16:36.606 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:36.606 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:36.606 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:36.606 | INFO | ANALYSIS : 57.7%, 3.0, 90.0ms, 0 | 20260505 04:16:36.606 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:36.606 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:36.606 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 04:16:36.606 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:36.606 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:36.606 | INFO | Uploaded 36GiB to GPU in 1.59931s (22.5098GiB/s) | 20260505 04:16:36.606 | INFO | Downloaded 34GiB to GPU in 1.51653s (22.4195GiB/s) | 20260505 04:16:41.745 | INFO | Analyzing chunk frames = [269824,270336) | 20260505 04:16:41.745 | INFO | MemInfo: { | 20260505 04:16:41.745 | INFO | "gpu" : " 17 GB", | 20260505 04:16:41.745 | INFO | "huge" : "", | 20260505 04:16:41.745 | INFO | "huge_shared" : " 27 GB", | 20260505 04:16:41.745 | INFO | "malloc" : " 14 GB", | 20260505 04:16:41.745 | INFO | "pinned" : " 29 GB" | 20260505 04:16:41.745 | INFO | } | 20260505 04:16:41.745 | INFO | Perf report for the last 5.13928 seconds: | 20260505 04:16:41.745 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:16:41.745 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:41.745 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:41.745 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:41.745 | INFO | ANALYSIS : 57.8%, 3.0, 90.2ms, 0 | 20260505 04:16:41.745 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:41.745 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:41.745 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260505 04:16:41.746 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:41.746 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:41.746 | INFO | Uploaded 36GiB to GPU in 1.59928s (22.5101GiB/s) | 20260505 04:16:41.746 | INFO | Downloaded 34GiB to GPU in 1.51633s (22.4226GiB/s) | 20260505 04:16:41.746 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 04:16:41.746 | INFO | Total mem usage: 14.9 GiB | 20260505 04:16:41.746 | INFO | All Time High Water Marks sum: 15.2 GiB (Savings: 1.92%) | 20260505 04:16:41.746 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -2.97%) | 20260505 04:16:41.746 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 04:16:41.746 | INFO | High Water Breakdown: | 20260505 04:16:41.746 | INFO | 58.8% : 8.52 GiB ( 99.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 04:16:41.746 | INFO | 41.1% : 5.95 GiB ( 89.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 04:16:41.746 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 04:16:41.746 | INFO | ------------------------------------------ | 20260505 04:16:41.746 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:16:41.746 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 04:16:41.746 | INFO | Total mem usage: 26.5 GiB | 20260505 04:16:41.746 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260505 04:16:41.746 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 04:16:41.746 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260505 04:16:41.746 | INFO | High Water Breakdown: | 20260505 04:16:41.746 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:16:41.746 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:16:41.746 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:16:41.746 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:16:41.746 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:16:41.746 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:16:41.746 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 04:16:41.746 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 04:16:41.746 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:16:41.746 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 04:16:41.746 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 04:16:41.746 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 04:16:41.746 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 04:16:41.746 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 04:16:41.746 | INFO | ------------------------------------------ | 20260505 04:16:41.746 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:16:41.746 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 04:16:41.746 | INFO | Total mem usage: 25.6 GiB | 20260505 04:16:41.746 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 04:16:41.746 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 04:16:41.746 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 04:16:41.746 | INFO | High Water Breakdown: | 20260505 04:16:41.746 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 04:16:41.746 | INFO | ------------------------------------------ | 20260505 04:16:41.746 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:16:41.746 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 04:16:41.746 | INFO | Total mem usage: 15.7 GiB | 20260505 04:16:41.746 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260505 04:16:41.746 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 04:16:41.746 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260505 04:16:41.746 | INFO | High Water Breakdown: | 20260505 04:16:41.746 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 04:16:41.746 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 04:16:41.746 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 04:16:41.746 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 04:16:41.746 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 04:16:41.746 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 04:16:41.746 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 04:16:41.746 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 04:16:41.746 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 04:16:41.746 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:16:41.746 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 04:16:41.746 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:16:41.746 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 04:16:41.746 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 04:16:41.746 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 04:16:41.746 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 04:16:41.746 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:16:41.746 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 04:16:41.746 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:16:41.746 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:16:41.746 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:16:41.746 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 04:16:41.746 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 04:16:41.746 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:16:41.746 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 04:16:41.746 | INFO | ------------------------------------------ | 20260505 04:16:41.746 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:16:41.746 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 04:16:41.746 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:16:41.746 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:16:41.746 | INFO | PerfReport: Full Execution -- 3.06e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260505 04:16:41.746 | INFO | PerfReport: 0.28%, AggregateStats -- 8.63e-02 / -nan, 4.44e-03 / 0.00e+00 | 20260505 04:16:41.746 | INFO | PerfReport: 10.79%, Baseline -- 3.30e+00 / -nan, 1.73e-02 / 0.00e+00 | 20260505 04:16:41.746 | INFO | PerfReport: 3.03%, Histogram -- 9.24e-01 / -nan, 5.23e-02 / 0.00e+00 | 20260505 04:16:41.746 | INFO | PerfReport: 16.02%, DME -- 3.91e+01 / -nan, 3.23e+00 / 0.00e+00 | 20260505 04:16:41.746 | INFO | PerfReport: 56.35%, FrameLabeling -- 1.72e+01 / -nan, 4.56e-02 / 0.00e+00 | 20260505 04:16:41.746 | INFO | PerfReport: 7.72%, PulseAccumulating -- 2.36e+00 / -nan, 1.60e-01 / 0.00e+00 | 20260505 04:16:41.746 | INFO | PerfReport: 4.11%, Metrics -- 1.26e+00 / -nan, 2.06e-01 / 0.00e+00 | 20260505 04:16:41.746 | INFO | PerfReport: 1.71%, PulseSerializing -- 5.22e-01 / -nan, 8.37e-02 / 0.00e+00 | 20260505 04:16:41.746 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 04:16:41.746 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:16:41.746 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:16:41.746 | INFO | PerfReport: Full Execution -- 9.18e+01 / -nan, 1.90e+01 / 0.00e+00 | 20260505 04:16:41.746 | INFO | PerfReport: 18.20%, Upload -- 1.67e+01 / 0.00e+00, 3.62e-01 / 0.00e+00 | 20260505 04:16:41.746 | INFO | PerfReport: 17.47%, Download -- 1.60e+01 / 0.00e+00, 6.84e-01 / 0.00e+00 | 20260505 04:16:41.746 | INFO | PerfReport: 33.29%, Compute -- 3.06e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260505 04:16:41.746 | INFO | PerfReport: 0.55%, LockUpload -- 5.02e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260505 04:16:41.746 | INFO | PerfReport: 0.00%, LockDownload -- 3.97e-04 / 0.00e+00, 1.77e-04 / 0.00e+00 | 20260505 04:16:41.746 | INFO | PerfReport: 30.47%, LockCompute -- 2.80e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260505 04:16:41.746 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 2.29e-02 / 0.00e+00 | 20260505 04:16:41.842 | INFO | Starting a round of Baz IO | 20260505 04:16:44.807 | INFO | Populating RTMetrics report | 20260505 04:16:44.807 | INFO | RTMetrics report populated | 20260505 04:16:44.807 | INFO | Opening tmp RTMetrics json file | 20260505 04:16:44.808 | INFO | Serializing RTMetrics to json | 20260505 04:16:44.809 | INFO | Serialization done, swapping tmp file now... | 20260505 04:16:44.810 | INFO | RTMetrics done | 20260505 04:16:44.884 | INFO | Finished a round of Baz IO: Wrote 1.854 GiB in 3.041 seconds | 20260505 04:16:44.884 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 51.23% / 0.20% / 48.57% | 20260505 04:16:44.884 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 12.1182% / 0.000504653% / 2.38419e-05% | 20260505 04:16:44.884 | INFO | Baz IO overall: | 20260505 04:16:44.884 | INFO | BytesWritten : 94656585346 | 20260505 04:16:44.884 | INFO | NumEvents : 43286787682 | 20260505 04:16:46.884 | INFO | Analyzing chunk frames = [270336,270848) | 20260505 04:16:46.884 | INFO | MemInfo: { | 20260505 04:16:46.884 | INFO | "gpu" : " 17 GB", | 20260505 04:16:46.884 | INFO | "huge" : "", | 20260505 04:16:46.884 | INFO | "huge_shared" : " 27 GB", | 20260505 04:16:46.884 | INFO | "malloc" : " 14 GB", | 20260505 04:16:46.884 | INFO | "pinned" : " 29 GB" | 20260505 04:16:46.885 | INFO | } | 20260505 04:16:46.885 | INFO | Perf report for the last 5.13924 seconds: | 20260505 04:16:46.885 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:16:46.885 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:46.885 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:46.885 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:46.885 | INFO | ANALYSIS : 58.4%, 3.0, 91.0ms, 0 | 20260505 04:16:46.885 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:46.885 | INFO | RT_METRICS : 44.9%, 1.0, 23.5ms, 0 | 20260505 04:16:46.885 | INFO | PRE_HQ : 57.9%, 2.2, 64.9ms, 0 | 20260505 04:16:46.885 | INFO | BAZWRITER : 58.7%, 3.7, 110.3ms, 0 | 20260505 04:16:46.885 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:16:46.885 | INFO | Uploaded 36GiB to GPU in 1.63527s (22.0147GiB/s) | 20260505 04:16:46.885 | INFO | Downloaded 38GiB to GPU in 1.69711s (22.391GiB/s) | 20260505 04:16:52.024 | INFO | Analyzing chunk frames = [270848,271360) | 20260505 04:16:52.024 | INFO | MemInfo: { | 20260505 04:16:52.024 | INFO | "gpu" : " 17 GB", | 20260505 04:16:52.024 | INFO | "huge" : "", | 20260505 04:16:52.024 | INFO | "huge_shared" : " 27 GB", | 20260505 04:16:52.024 | INFO | "malloc" : " 14 GB", | 20260505 04:16:52.024 | INFO | "pinned" : " 29 GB" | 20260505 04:16:52.024 | INFO | } | 20260505 04:16:52.024 | INFO | Perf report for the last 5.13949 seconds: | 20260505 04:16:52.024 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:16:52.024 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:52.024 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:52.024 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:52.024 | INFO | ANALYSIS : 58.2%, 3.0, 90.8ms, 0 | 20260505 04:16:52.024 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:52.024 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:52.024 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260505 04:16:52.024 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:52.024 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:52.024 | INFO | Uploaded 36GiB to GPU in 1.59698s (22.5425GiB/s) | 20260505 04:16:52.025 | INFO | Downloaded 34GiB to GPU in 1.51317s (22.4694GiB/s) | 20260505 04:16:57.163 | INFO | Analyzing chunk frames = [271360,271872) | 20260505 04:16:57.163 | INFO | MemInfo: { | 20260505 04:16:57.163 | INFO | "gpu" : " 17 GB", | 20260505 04:16:57.163 | INFO | "huge" : "", | 20260505 04:16:57.163 | INFO | "huge_shared" : " 27 GB", | 20260505 04:16:57.163 | INFO | "malloc" : " 14 GB", | 20260505 04:16:57.163 | INFO | "pinned" : " 29 GB" | 20260505 04:16:57.163 | INFO | } | 20260505 04:16:57.163 | INFO | Perf report for the last 5.13921 seconds: | 20260505 04:16:57.163 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:16:57.163 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:57.163 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:57.163 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:57.163 | INFO | ANALYSIS : 57.6%, 3.0, 89.5ms, 0 | 20260505 04:16:57.163 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:57.163 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:57.163 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260505 04:16:57.163 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:57.163 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:16:57.164 | INFO | Uploaded 36GiB to GPU in 1.60095s (22.4867GiB/s) | 20260505 04:16:57.164 | INFO | Downloaded 34GiB to GPU in 1.51779s (22.401GiB/s) | 20260505 04:17:02.302 | INFO | Analyzing chunk frames = [271872,272384) | 20260505 04:17:02.302 | INFO | MemInfo: { | 20260505 04:17:02.302 | INFO | "gpu" : " 17 GB", | 20260505 04:17:02.302 | INFO | "huge" : "", | 20260505 04:17:02.302 | INFO | "huge_shared" : " 27 GB", | 20260505 04:17:02.303 | INFO | "malloc" : " 14 GB", | 20260505 04:17:02.303 | INFO | "pinned" : " 29 GB" | 20260505 04:17:02.303 | INFO | } | 20260505 04:17:02.303 | INFO | Perf report for the last 5.13941 seconds: | 20260505 04:17:02.303 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:17:02.303 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:02.303 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:02.303 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:02.303 | INFO | ANALYSIS : 57.2%, 3.0, 88.8ms, 0 | 20260505 04:17:02.303 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:02.303 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:02.303 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260505 04:17:02.303 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:02.303 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:02.303 | INFO | Uploaded 36GiB to GPU in 1.60094s (22.4868GiB/s) | 20260505 04:17:02.303 | INFO | Downloaded 34GiB to GPU in 1.51733s (22.4078GiB/s) | 20260505 04:17:07.442 | INFO | Analyzing chunk frames = [272384,272896) | 20260505 04:17:07.442 | INFO | MemInfo: { | 20260505 04:17:07.442 | INFO | "gpu" : " 17 GB", | 20260505 04:17:07.442 | INFO | "huge" : "", | 20260505 04:17:07.442 | INFO | "huge_shared" : " 27 GB", | 20260505 04:17:07.442 | INFO | "malloc" : " 14 GB", | 20260505 04:17:07.442 | INFO | "pinned" : " 29 GB" | 20260505 04:17:07.442 | INFO | } | 20260505 04:17:07.442 | INFO | Perf report for the last 5.13938 seconds: | 20260505 04:17:07.442 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:17:07.442 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:07.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:07.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:07.442 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260505 04:17:07.442 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:07.442 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:07.442 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260505 04:17:07.442 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:07.442 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:07.442 | INFO | Uploaded 36GiB to GPU in 1.59847s (22.5215GiB/s) | 20260505 04:17:07.443 | INFO | Downloaded 34GiB to GPU in 1.51472s (22.4464GiB/s) | 20260505 04:17:12.581 | INFO | Analyzing chunk frames = [272896,273408) | 20260505 04:17:12.581 | INFO | MemInfo: { | 20260505 04:17:12.581 | INFO | "gpu" : " 17 GB", | 20260505 04:17:12.581 | INFO | "huge" : "", | 20260505 04:17:12.581 | INFO | "huge_shared" : " 27 GB", | 20260505 04:17:12.581 | INFO | "malloc" : " 14 GB", | 20260505 04:17:12.581 | INFO | "pinned" : " 29 GB" | 20260505 04:17:12.581 | INFO | } | 20260505 04:17:12.581 | INFO | Perf report for the last 5.13918 seconds: | 20260505 04:17:12.581 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:17:12.581 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:12.581 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:12.581 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:12.581 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260505 04:17:12.581 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:12.581 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:12.581 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260505 04:17:12.581 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:12.582 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:12.582 | INFO | Uploaded 36GiB to GPU in 1.59926s (22.5104GiB/s) | 20260505 04:17:12.582 | INFO | Downloaded 34GiB to GPU in 1.51511s (22.4406GiB/s) | 20260505 04:17:17.721 | INFO | Analyzing chunk frames = [273408,273920) | 20260505 04:17:17.721 | INFO | MemInfo: { | 20260505 04:17:17.721 | INFO | "gpu" : " 17 GB", | 20260505 04:17:17.721 | INFO | "huge" : "", | 20260505 04:17:17.721 | INFO | "huge_shared" : " 27 GB", | 20260505 04:17:17.721 | INFO | "malloc" : " 14 GB", | 20260505 04:17:17.721 | INFO | "pinned" : " 29 GB" | 20260505 04:17:17.721 | INFO | } | 20260505 04:17:17.721 | INFO | Perf report for the last 5.13945 seconds: | 20260505 04:17:17.721 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:17:17.721 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:17.721 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:17.721 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:17.721 | INFO | ANALYSIS : 57.7%, 3.0, 90.0ms, 0 | 20260505 04:17:17.721 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:17.721 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:17.721 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 04:17:17.721 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:17.721 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:17.721 | INFO | Uploaded 36GiB to GPU in 1.60037s (22.4948GiB/s) | 20260505 04:17:17.721 | INFO | Downloaded 34GiB to GPU in 1.51673s (22.4167GiB/s) | 20260505 04:17:22.860 | INFO | Analyzing chunk frames = [273920,274432) | 20260505 04:17:22.860 | INFO | MemInfo: { | 20260505 04:17:22.860 | INFO | "gpu" : " 17 GB", | 20260505 04:17:22.860 | INFO | "huge" : "", | 20260505 04:17:22.860 | INFO | "huge_shared" : " 27 GB", | 20260505 04:17:22.860 | INFO | "malloc" : " 14 GB", | 20260505 04:17:22.860 | INFO | "pinned" : " 29 GB" | 20260505 04:17:22.860 | INFO | } | 20260505 04:17:22.860 | INFO | Perf report for the last 5.13931 seconds: | 20260505 04:17:22.860 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:17:22.860 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:22.860 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:22.860 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:22.860 | INFO | ANALYSIS : 57.7%, 3.0, 90.1ms, 0 | 20260505 04:17:22.860 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:22.860 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:22.860 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260505 04:17:22.860 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:22.860 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:22.860 | INFO | Uploaded 36GiB to GPU in 1.6004s (22.4944GiB/s) | 20260505 04:17:22.860 | INFO | Downloaded 34GiB to GPU in 1.51748s (22.4056GiB/s) | 20260505 04:17:22.958 | INFO | Starting a round of Baz IO | 20260505 04:17:25.922 | INFO | Populating RTMetrics report | 20260505 04:17:25.922 | INFO | RTMetrics report populated | 20260505 04:17:25.922 | INFO | Opening tmp RTMetrics json file | 20260505 04:17:25.923 | INFO | Serializing RTMetrics to json | 20260505 04:17:25.924 | INFO | Serialization done, swapping tmp file now... | 20260505 04:17:25.925 | INFO | RTMetrics done | 20260505 04:17:26.007 | INFO | Finished a round of Baz IO: Wrote 1.859 GiB in 3.049 seconds | 20260505 04:17:26.007 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 51.04% / 0.19% / 48.77% | 20260505 04:17:26.007 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 12.035% / 0.000560284% / 2.38419e-05% | 20260505 04:17:26.007 | INFO | Baz IO overall: | 20260505 04:17:26.007 | INFO | BytesWritten : 96653108599 | 20260505 04:17:26.007 | INFO | NumEvents : 44229683179 | 20260505 04:17:27.999 | INFO | Analyzing chunk frames = [274432,274944) | 20260505 04:17:27.999 | INFO | MemInfo: { | 20260505 04:17:27.999 | INFO | "gpu" : " 17 GB", | 20260505 04:17:27.999 | INFO | "huge" : "", | 20260505 04:17:27.999 | INFO | "huge_shared" : " 27 GB", | 20260505 04:17:27.999 | INFO | "malloc" : " 14 GB", | 20260505 04:17:27.999 | INFO | "pinned" : " 29 GB" | 20260505 04:17:27.999 | INFO | } | 20260505 04:17:27.999 | INFO | Perf report for the last 5.13919 seconds: | 20260505 04:17:27.999 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:17:27.999 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:27.999 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:27.999 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:27.999 | INFO | ANALYSIS : 58.5%, 3.0, 91.2ms, 0 | 20260505 04:17:27.999 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:27.999 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260505 04:17:27.999 | INFO | PRE_HQ : 57.9%, 2.2, 64.8ms, 0 | 20260505 04:17:27.999 | INFO | BAZWRITER : 58.9%, 3.6, 110.3ms, 0 | 20260505 04:17:27.999 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:17:28.000 | INFO | Uploaded 36GiB to GPU in 1.63878s (21.9675GiB/s) | 20260505 04:17:28.000 | INFO | Downloaded 38GiB to GPU in 1.70003s (22.3525GiB/s) | 20260505 04:17:33.139 | INFO | Analyzing chunk frames = [274944,275456) | 20260505 04:17:33.139 | INFO | MemInfo: { | 20260505 04:17:33.139 | INFO | "gpu" : " 17 GB", | 20260505 04:17:33.139 | INFO | "huge" : "", | 20260505 04:17:33.139 | INFO | "huge_shared" : " 27 GB", | 20260505 04:17:33.139 | INFO | "malloc" : " 14 GB", | 20260505 04:17:33.139 | INFO | "pinned" : " 29 GB" | 20260505 04:17:33.139 | INFO | } | 20260505 04:17:33.139 | INFO | Perf report for the last 5.13953 seconds: | 20260505 04:17:33.139 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:17:33.139 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:33.139 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:33.139 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:33.139 | INFO | ANALYSIS : 58.2%, 3.0, 90.7ms, 0 | 20260505 04:17:33.139 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:33.139 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:33.139 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260505 04:17:33.139 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:33.139 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:33.139 | INFO | Uploaded 36GiB to GPU in 1.59834s (22.5234GiB/s) | 20260505 04:17:33.139 | INFO | Downloaded 34GiB to GPU in 1.51504s (22.4417GiB/s) | 20260505 04:17:38.278 | INFO | Analyzing chunk frames = [275456,275968) | 20260505 04:17:38.278 | INFO | MemInfo: { | 20260505 04:17:38.278 | INFO | "gpu" : " 17 GB", | 20260505 04:17:38.278 | INFO | "huge" : "", | 20260505 04:17:38.278 | INFO | "huge_shared" : " 27 GB", | 20260505 04:17:38.278 | INFO | "malloc" : " 14 GB", | 20260505 04:17:38.278 | INFO | "pinned" : " 29 GB" | 20260505 04:17:38.278 | INFO | } | 20260505 04:17:38.278 | INFO | Perf report for the last 5.13922 seconds: | 20260505 04:17:38.278 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:17:38.278 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:38.278 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:38.278 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:38.278 | INFO | ANALYSIS : 57.6%, 3.0, 89.5ms, 0 | 20260505 04:17:38.278 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:38.278 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:38.278 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260505 04:17:38.278 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:38.278 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:38.278 | INFO | Uploaded 36GiB to GPU in 1.60082s (22.4885GiB/s) | 20260505 04:17:38.278 | INFO | Downloaded 34GiB to GPU in 1.51754s (22.4046GiB/s) | 20260505 04:17:43.417 | INFO | Analyzing chunk frames = [275968,276480) | 20260505 04:17:43.417 | INFO | MemInfo: { | 20260505 04:17:43.417 | INFO | "gpu" : " 17 GB", | 20260505 04:17:43.417 | INFO | "huge" : "", | 20260505 04:17:43.417 | INFO | "huge_shared" : " 27 GB", | 20260505 04:17:43.417 | INFO | "malloc" : " 14 GB", | 20260505 04:17:43.417 | INFO | "pinned" : " 29 GB" | 20260505 04:17:43.417 | INFO | } | 20260505 04:17:43.418 | INFO | Perf report for the last 5.13939 seconds: | 20260505 04:17:43.418 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:17:43.418 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:43.418 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:43.418 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:43.418 | INFO | ANALYSIS : 57.2%, 3.0, 88.8ms, 0 | 20260505 04:17:43.418 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:43.418 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:43.418 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260505 04:17:43.418 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:43.418 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:43.418 | INFO | Uploaded 36GiB to GPU in 1.60061s (22.4915GiB/s) | 20260505 04:17:43.418 | INFO | Downloaded 34GiB to GPU in 1.51734s (22.4076GiB/s) | 20260505 04:17:48.557 | INFO | Analyzing chunk frames = [276480,276992) | 20260505 04:17:48.557 | INFO | MemInfo: { | 20260505 04:17:48.557 | INFO | "gpu" : " 17 GB", | 20260505 04:17:48.557 | INFO | "huge" : "", | 20260505 04:17:48.557 | INFO | "huge_shared" : " 27 GB", | 20260505 04:17:48.557 | INFO | "malloc" : " 14 GB", | 20260505 04:17:48.557 | INFO | "pinned" : " 29 GB" | 20260505 04:17:48.557 | INFO | } | 20260505 04:17:48.557 | INFO | Perf report for the last 5.13928 seconds: | 20260505 04:17:48.557 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:17:48.557 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:48.557 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:48.557 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:48.557 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260505 04:17:48.557 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:48.557 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:48.557 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260505 04:17:48.557 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:48.557 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:48.557 | INFO | Uploaded 36GiB to GPU in 1.59893s (22.515GiB/s) | 20260505 04:17:48.557 | INFO | Downloaded 34GiB to GPU in 1.51587s (22.4294GiB/s) | 20260505 04:17:53.696 | INFO | Analyzing chunk frames = [276992,277504) | 20260505 04:17:53.696 | INFO | MemInfo: { | 20260505 04:17:53.696 | INFO | "gpu" : " 17 GB", | 20260505 04:17:53.696 | INFO | "huge" : "", | 20260505 04:17:53.696 | INFO | "huge_shared" : " 27 GB", | 20260505 04:17:53.696 | INFO | "malloc" : " 14 GB", | 20260505 04:17:53.696 | INFO | "pinned" : " 29 GB" | 20260505 04:17:53.696 | INFO | } | 20260505 04:17:53.696 | INFO | Perf report for the last 5.13945 seconds: | 20260505 04:17:53.696 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:17:53.696 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:53.696 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:53.696 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:53.696 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260505 04:17:53.696 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:53.696 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:53.696 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260505 04:17:53.696 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:53.696 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:53.697 | INFO | Uploaded 36GiB to GPU in 1.59851s (22.5209GiB/s) | 20260505 04:17:53.697 | INFO | Downloaded 34GiB to GPU in 1.51581s (22.4303GiB/s) | 20260505 04:17:58.835 | INFO | Analyzing chunk frames = [277504,278016) | 20260505 04:17:58.835 | INFO | MemInfo: { | 20260505 04:17:58.835 | INFO | "gpu" : " 17 GB", | 20260505 04:17:58.835 | INFO | "huge" : "", | 20260505 04:17:58.835 | INFO | "huge_shared" : " 27 GB", | 20260505 04:17:58.835 | INFO | "malloc" : " 14 GB", | 20260505 04:17:58.836 | INFO | "pinned" : " 29 GB" | 20260505 04:17:58.836 | INFO | } | 20260505 04:17:58.836 | INFO | Perf report for the last 5.13938 seconds: | 20260505 04:17:58.836 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:17:58.836 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:58.836 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:58.836 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:58.836 | INFO | ANALYSIS : 57.7%, 3.0, 90.0ms, 0 | 20260505 04:17:58.836 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:58.836 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:58.836 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 04:17:58.836 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:58.836 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:17:58.836 | INFO | Uploaded 36GiB to GPU in 1.59883s (22.5165GiB/s) | 20260505 04:17:58.836 | INFO | Downloaded 34GiB to GPU in 1.51592s (22.4286GiB/s) | 20260505 04:18:03.975 | INFO | Analyzing chunk frames = [278016,278528) | 20260505 04:18:03.975 | INFO | MemInfo: { | 20260505 04:18:03.975 | INFO | "gpu" : " 17 GB", | 20260505 04:18:03.975 | INFO | "huge" : "", | 20260505 04:18:03.975 | INFO | "huge_shared" : " 27 GB", | 20260505 04:18:03.975 | INFO | "malloc" : " 14 GB", | 20260505 04:18:03.975 | INFO | "pinned" : " 29 GB" | 20260505 04:18:03.975 | INFO | } | 20260505 04:18:03.975 | INFO | Perf report for the last 5.13929 seconds: | 20260505 04:18:03.975 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:18:03.975 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:03.975 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:03.975 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:03.975 | INFO | ANALYSIS : 57.7%, 3.0, 90.1ms, 0 | 20260505 04:18:03.975 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:03.975 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:03.975 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 04:18:03.975 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:03.975 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:03.976 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 04:18:03.976 | INFO | Total mem usage: 14.9 GiB | 20260505 04:18:03.976 | INFO | All Time High Water Marks sum: 15.2 GiB (Savings: 1.92%) | 20260505 04:18:03.976 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -2.96%) | 20260505 04:18:03.976 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 04:18:03.976 | INFO | High Water Breakdown: | 20260505 04:18:03.976 | INFO | 58.8% : 8.53 GiB ( 99.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 04:18:03.976 | INFO | 41% : 5.95 GiB ( 89.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 04:18:03.976 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 04:18:03.976 | INFO | ------------------------------------------ | 20260505 04:18:03.976 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:18:03.976 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 04:18:03.976 | INFO | Total mem usage: 26.5 GiB | 20260505 04:18:03.976 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260505 04:18:03.976 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 04:18:03.976 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260505 04:18:03.976 | INFO | High Water Breakdown: | 20260505 04:18:03.976 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:18:03.976 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:18:03.976 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:18:03.976 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:18:03.976 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:18:03.976 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:18:03.976 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 04:18:03.976 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 04:18:03.976 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:18:03.976 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 04:18:03.976 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 04:18:03.976 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 04:18:03.976 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 04:18:03.976 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 04:18:03.976 | INFO | ------------------------------------------ | 20260505 04:18:03.976 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:18:03.976 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 04:18:03.976 | INFO | Total mem usage: 25.6 GiB | 20260505 04:18:03.976 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 04:18:03.976 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 04:18:03.976 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 04:18:03.976 | INFO | High Water Breakdown: | 20260505 04:18:03.976 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 04:18:03.976 | INFO | ------------------------------------------ | 20260505 04:18:03.976 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:18:03.976 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 04:18:03.976 | INFO | Total mem usage: 15.7 GiB | 20260505 04:18:03.976 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260505 04:18:03.976 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 04:18:03.976 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260505 04:18:03.976 | INFO | High Water Breakdown: | 20260505 04:18:03.976 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 04:18:03.976 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 04:18:03.976 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 04:18:03.976 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 04:18:03.976 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 04:18:03.976 | INFO | Uploaded 36GiB to GPU in 1.59904s (22.5135GiB/s) | 20260505 04:18:03.976 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 04:18:03.976 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 04:18:03.976 | INFO | Downloaded 34GiB to GPU in 1.51688s (22.4144GiB/s) | 20260505 04:18:03.976 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 04:18:03.976 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 04:18:03.976 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:18:03.976 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 04:18:03.976 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:18:03.976 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 04:18:03.976 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 04:18:03.976 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 04:18:03.976 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 04:18:03.976 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:18:03.976 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 04:18:03.976 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:18:03.976 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:18:03.976 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:18:03.976 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 04:18:03.976 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 04:18:03.976 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:18:03.976 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 04:18:03.976 | INFO | ------------------------------------------ | 20260505 04:18:03.976 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:18:03.976 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 04:18:03.976 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:18:03.976 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:18:03.976 | INFO | PerfReport: Full Execution -- 3.05e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260505 04:18:03.976 | INFO | PerfReport: 0.28%, AggregateStats -- 8.64e-02 / -nan, 6.26e-03 / 0.00e+00 | 20260505 04:18:03.976 | INFO | PerfReport: 10.79%, Baseline -- 3.30e+00 / -nan, 1.89e-02 / 0.00e+00 | 20260505 04:18:03.976 | INFO | PerfReport: 3.03%, Histogram -- 9.24e-01 / -nan, 5.20e-02 / 0.00e+00 | 20260505 04:18:03.976 | INFO | PerfReport: 16.00%, DME -- 3.91e+01 / -nan, 3.28e+00 / 0.00e+00 | 20260505 04:18:03.976 | INFO | PerfReport: 56.37%, FrameLabeling -- 1.72e+01 / -nan, 1.00e-01 / 0.00e+00 | 20260505 04:18:03.976 | INFO | PerfReport: 7.72%, PulseAccumulating -- 2.36e+00 / -nan, 1.59e-01 / 0.00e+00 | 20260505 04:18:03.976 | INFO | PerfReport: 4.10%, Metrics -- 1.25e+00 / -nan, 2.04e-01 / 0.00e+00 | 20260505 04:18:03.976 | INFO | PerfReport: 1.70%, PulseSerializing -- 5.21e-01 / -nan, 8.38e-02 / 0.00e+00 | 20260505 04:18:03.976 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 04:18:03.977 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:18:03.977 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:18:03.977 | INFO | PerfReport: Full Execution -- 9.17e+01 / -nan, 1.90e+01 / 0.00e+00 | 20260505 04:18:03.977 | INFO | PerfReport: 18.21%, Upload -- 1.67e+01 / 0.00e+00, 3.68e-01 / 0.00e+00 | 20260505 04:18:03.977 | INFO | PerfReport: 17.48%, Download -- 1.60e+01 / 0.00e+00, 6.93e-01 / 0.00e+00 | 20260505 04:18:03.977 | INFO | PerfReport: 33.29%, Compute -- 3.05e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260505 04:18:03.977 | INFO | PerfReport: 0.55%, LockUpload -- 5.03e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260505 04:18:03.977 | INFO | PerfReport: 0.00%, LockDownload -- 3.97e-04 / 0.00e+00, 1.37e-04 / 0.00e+00 | 20260505 04:18:03.977 | INFO | PerfReport: 30.45%, LockCompute -- 2.79e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260505 04:18:03.977 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 2.68e-02 / 0.00e+00 | 20260505 04:18:04.073 | INFO | Starting a round of Baz IO | 20260505 04:18:07.034 | INFO | Populating RTMetrics report | 20260505 04:18:07.035 | INFO | RTMetrics report populated | 20260505 04:18:07.035 | INFO | Opening tmp RTMetrics json file | 20260505 04:18:07.036 | INFO | Serializing RTMetrics to json | 20260505 04:18:07.037 | INFO | Serialization done, swapping tmp file now... | 20260505 04:18:07.038 | INFO | RTMetrics done | 20260505 04:18:07.116 | INFO | Finished a round of Baz IO: Wrote 1.866 GiB in 3.043 seconds | 20260505 04:18:07.116 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 50.85% / 0.19% / 48.96% | 20260505 04:18:07.116 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 12.0026% / 0.000615915% / 2.38419e-05% | 20260505 04:18:07.116 | INFO | Baz IO overall: | 20260505 04:18:07.116 | INFO | BytesWritten : 98656901341 | 20260505 04:18:07.116 | INFO | NumEvents : 45176637795 | 20260505 04:18:09.114 | INFO | Analyzing chunk frames = [278528,279040) | 20260505 04:18:09.114 | INFO | MemInfo: { | 20260505 04:18:09.114 | INFO | "gpu" : " 17 GB", | 20260505 04:18:09.114 | INFO | "huge" : "", | 20260505 04:18:09.114 | INFO | "huge_shared" : " 27 GB", | 20260505 04:18:09.114 | INFO | "malloc" : " 14 GB", | 20260505 04:18:09.114 | INFO | "pinned" : " 29 GB" | 20260505 04:18:09.114 | INFO | } | 20260505 04:18:09.114 | INFO | Perf report for the last 5.13915 seconds: | 20260505 04:18:09.114 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:18:09.114 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:09.114 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:09.114 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:09.114 | INFO | ANALYSIS : 58.4%, 3.0, 91.0ms, 0 | 20260505 04:18:09.114 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:09.114 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260505 04:18:09.114 | INFO | PRE_HQ : 57.8%, 2.2, 64.6ms, 0 | 20260505 04:18:09.114 | INFO | BAZWRITER : 58.8%, 3.7, 110.3ms, 0 | 20260505 04:18:09.114 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:18:09.115 | INFO | Uploaded 36GiB to GPU in 1.63917s (21.9624GiB/s) | 20260505 04:18:09.115 | INFO | Downloaded 38GiB to GPU in 1.70054s (22.3458GiB/s) | 20260505 04:18:14.254 | INFO | Analyzing chunk frames = [279040,279552) | 20260505 04:18:14.254 | INFO | MemInfo: { | 20260505 04:18:14.254 | INFO | "gpu" : " 17 GB", | 20260505 04:18:14.254 | INFO | "huge" : "", | 20260505 04:18:14.254 | INFO | "huge_shared" : " 27 GB", | 20260505 04:18:14.254 | INFO | "malloc" : " 14 GB", | 20260505 04:18:14.254 | INFO | "pinned" : " 29 GB" | 20260505 04:18:14.254 | INFO | } | 20260505 04:18:14.254 | INFO | Perf report for the last 5.1397 seconds: | 20260505 04:18:14.254 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:18:14.254 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:14.254 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:14.254 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:14.254 | INFO | ANALYSIS : 58.2%, 3.0, 90.8ms, 0 | 20260505 04:18:14.254 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:14.254 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:14.254 | INFO | PRE_HQ : 32.4%, 1.0, 17.5ms, 0 | 20260505 04:18:14.254 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:14.254 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:14.254 | INFO | Uploaded 36GiB to GPU in 1.59738s (22.5369GiB/s) | 20260505 04:18:14.255 | INFO | Downloaded 34GiB to GPU in 1.51375s (22.4608GiB/s) | 20260505 04:18:19.393 | INFO | Analyzing chunk frames = [279552,280064) | 20260505 04:18:19.393 | INFO | MemInfo: { | 20260505 04:18:19.393 | INFO | "gpu" : " 17 GB", | 20260505 04:18:19.393 | INFO | "huge" : "", | 20260505 04:18:19.393 | INFO | "huge_shared" : " 27 GB", | 20260505 04:18:19.393 | INFO | "malloc" : " 14 GB", | 20260505 04:18:19.393 | INFO | "pinned" : " 29 GB" | 20260505 04:18:19.393 | INFO | } | 20260505 04:18:19.393 | INFO | Perf report for the last 5.13899 seconds: | 20260505 04:18:19.393 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:18:19.393 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:19.393 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:19.393 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:19.393 | INFO | ANALYSIS : 57.6%, 3.0, 89.5ms, 0 | 20260505 04:18:19.393 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:19.393 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:19.393 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260505 04:18:19.393 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:19.393 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:19.393 | INFO | Uploaded 36GiB to GPU in 1.60113s (22.4841GiB/s) | 20260505 04:18:19.394 | INFO | Downloaded 34GiB to GPU in 1.51852s (22.3902GiB/s) | 20260505 04:18:24.532 | INFO | Analyzing chunk frames = [280064,280576) | 20260505 04:18:24.532 | INFO | MemInfo: { | 20260505 04:18:24.532 | INFO | "gpu" : " 17 GB", | 20260505 04:18:24.532 | INFO | "huge" : "", | 20260505 04:18:24.532 | INFO | "huge_shared" : " 27 GB", | 20260505 04:18:24.532 | INFO | "malloc" : " 14 GB", | 20260505 04:18:24.532 | INFO | "pinned" : " 29 GB" | 20260505 04:18:24.532 | INFO | } | 20260505 04:18:24.532 | INFO | Perf report for the last 5.13948 seconds: | 20260505 04:18:24.532 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:18:24.532 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:24.532 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:24.532 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:24.532 | INFO | ANALYSIS : 57.2%, 3.0, 88.8ms, 0 | 20260505 04:18:24.532 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:24.532 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:24.532 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260505 04:18:24.532 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:24.533 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:24.533 | INFO | Uploaded 36GiB to GPU in 1.60085s (22.4881GiB/s) | 20260505 04:18:24.533 | INFO | Downloaded 34GiB to GPU in 1.51752s (22.405GiB/s) | 20260505 04:18:29.672 | INFO | Analyzing chunk frames = [280576,281088) | 20260505 04:18:29.672 | INFO | MemInfo: { | 20260505 04:18:29.672 | INFO | "gpu" : " 17 GB", | 20260505 04:18:29.672 | INFO | "huge" : "", | 20260505 04:18:29.672 | INFO | "huge_shared" : " 27 GB", | 20260505 04:18:29.672 | INFO | "malloc" : " 14 GB", | 20260505 04:18:29.672 | INFO | "pinned" : " 29 GB" | 20260505 04:18:29.672 | INFO | } | 20260505 04:18:29.672 | INFO | Perf report for the last 5.1395 seconds: | 20260505 04:18:29.672 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:18:29.672 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:29.672 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:29.672 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:29.672 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260505 04:18:29.672 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:29.672 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:29.672 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 04:18:29.672 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:29.672 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:29.672 | INFO | Uploaded 36GiB to GPU in 1.59911s (22.5126GiB/s) | 20260505 04:18:29.672 | INFO | Downloaded 34GiB to GPU in 1.51499s (22.4424GiB/s) | 20260505 04:18:34.811 | INFO | Analyzing chunk frames = [281088,281600) | 20260505 04:18:34.811 | INFO | MemInfo: { | 20260505 04:18:34.811 | INFO | "gpu" : " 17 GB", | 20260505 04:18:34.811 | INFO | "huge" : "", | 20260505 04:18:34.811 | INFO | "huge_shared" : " 27 GB", | 20260505 04:18:34.811 | INFO | "malloc" : " 14 GB", | 20260505 04:18:34.811 | INFO | "pinned" : " 29 GB" | 20260505 04:18:34.811 | INFO | } | 20260505 04:18:34.811 | INFO | Perf report for the last 5.13935 seconds: | 20260505 04:18:34.811 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:18:34.811 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:34.811 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:34.811 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:34.811 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260505 04:18:34.811 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:34.811 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:34.811 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260505 04:18:34.811 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:34.811 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:34.812 | INFO | Uploaded 36GiB to GPU in 1.59953s (22.5066GiB/s) | 20260505 04:18:34.812 | INFO | Downloaded 34GiB to GPU in 1.51564s (22.4328GiB/s) | 20260505 04:18:39.972 | INFO | Analyzing chunk frames = [281600,282112) | 20260505 04:18:39.972 | INFO | MemInfo: { | 20260505 04:18:39.972 | INFO | "gpu" : " 17 GB", | 20260505 04:18:39.972 | INFO | "huge" : "", | 20260505 04:18:39.972 | INFO | "huge_shared" : " 27 GB", | 20260505 04:18:39.972 | INFO | "malloc" : " 14 GB", | 20260505 04:18:39.972 | INFO | "pinned" : " 29 GB" | 20260505 04:18:39.972 | INFO | } | 20260505 04:18:39.972 | INFO | Perf report for the last 5.16127 seconds: | 20260505 04:18:39.972 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:18:39.972 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:39.973 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:39.973 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:39.973 | INFO | ANALYSIS : 57.4%, 3.0, 90.0ms, 0 | 20260505 04:18:39.973 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:39.973 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:39.973 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260505 04:18:39.973 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:39.973 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:39.973 | INFO | Uploaded 36GiB to GPU in 1.59957s (22.506GiB/s) | 20260505 04:18:39.973 | INFO | Downloaded 34GiB to GPU in 1.51693s (22.4137GiB/s) | 20260505 04:18:45.090 | INFO | Analyzing chunk frames = [282112,282624) | 20260505 04:18:45.090 | INFO | MemInfo: { | 20260505 04:18:45.090 | INFO | "gpu" : " 17 GB", | 20260505 04:18:45.090 | INFO | "huge" : "", | 20260505 04:18:45.090 | INFO | "huge_shared" : " 27 GB", | 20260505 04:18:45.090 | INFO | "malloc" : " 14 GB", | 20260505 04:18:45.090 | INFO | "pinned" : " 29 GB" | 20260505 04:18:45.090 | INFO | } | 20260505 04:18:45.090 | INFO | Perf report for the last 5.11781 seconds: | 20260505 04:18:45.090 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:18:45.090 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:45.090 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:45.090 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:45.090 | INFO | ANALYSIS : 58.0%, 3.0, 90.1ms, 0 | 20260505 04:18:45.090 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:45.090 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:45.090 | INFO | PRE_HQ : 31.6%, 1.0, 16.3ms, 0 | 20260505 04:18:45.090 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:45.090 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:45.091 | INFO | Uploaded 36GiB to GPU in 1.60079s (22.4888GiB/s) | 20260505 04:18:45.091 | INFO | Downloaded 34GiB to GPU in 1.51719s (22.4099GiB/s) | 20260505 04:18:45.188 | INFO | Starting a round of Baz IO | 20260505 04:18:48.147 | INFO | Populating RTMetrics report | 20260505 04:18:48.147 | INFO | RTMetrics report populated | 20260505 04:18:48.147 | INFO | Opening tmp RTMetrics json file | 20260505 04:18:48.147 | INFO | Serializing RTMetrics to json | 20260505 04:18:48.148 | INFO | Serialization done, swapping tmp file now... | 20260505 04:18:48.149 | INFO | RTMetrics done | 20260505 04:18:48.228 | INFO | Finished a round of Baz IO: Wrote 1.873 GiB in 3.04 seconds | 20260505 04:18:48.228 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 50.66% / 0.19% / 49.15% | 20260505 04:18:48.228 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 11.9405% / 0.000496705% / 3.57628e-05% | 20260505 04:18:48.228 | INFO | Baz IO overall: | 20260505 04:18:48.228 | INFO | BytesWritten : 100668086756 | 20260505 04:18:48.228 | INFO | NumEvents : 46127107058 | 20260505 04:18:50.229 | INFO | Analyzing chunk frames = [282624,283136) | 20260505 04:18:50.229 | INFO | MemInfo: { | 20260505 04:18:50.229 | INFO | "gpu" : " 17 GB", | 20260505 04:18:50.229 | INFO | "huge" : "", | 20260505 04:18:50.229 | INFO | "huge_shared" : " 27 GB", | 20260505 04:18:50.229 | INFO | "malloc" : " 14 GB", | 20260505 04:18:50.229 | INFO | "pinned" : " 29 GB" | 20260505 04:18:50.229 | INFO | } | 20260505 04:18:50.229 | INFO | Perf report for the last 5.13863 seconds: | 20260505 04:18:50.229 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:18:50.229 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:50.229 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:50.229 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:50.229 | INFO | ANALYSIS : 58.4%, 3.0, 90.8ms, 0 | 20260505 04:18:50.229 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:50.229 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260505 04:18:50.229 | INFO | PRE_HQ : 57.9%, 2.2, 64.9ms, 0 | 20260505 04:18:50.229 | INFO | BAZWRITER : 58.5%, 3.7, 111.2ms, 0 | 20260505 04:18:50.229 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:18:50.229 | INFO | Uploaded 36GiB to GPU in 1.6368s (21.9941GiB/s) | 20260505 04:18:50.229 | INFO | Downloaded 38GiB to GPU in 1.69791s (22.3804GiB/s) | 20260505 04:18:55.368 | INFO | Analyzing chunk frames = [283136,283648) | 20260505 04:18:55.368 | INFO | MemInfo: { | 20260505 04:18:55.368 | INFO | "gpu" : " 17 GB", | 20260505 04:18:55.368 | INFO | "huge" : "", | 20260505 04:18:55.368 | INFO | "huge_shared" : " 27 GB", | 20260505 04:18:55.368 | INFO | "malloc" : " 14 GB", | 20260505 04:18:55.368 | INFO | "pinned" : " 29 GB" | 20260505 04:18:55.368 | INFO | } | 20260505 04:18:55.368 | INFO | Perf report for the last 5.13949 seconds: | 20260505 04:18:55.368 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:18:55.368 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:55.368 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:55.369 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:55.369 | INFO | ANALYSIS : 58.2%, 3.0, 90.8ms, 0 | 20260505 04:18:55.369 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:55.369 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:55.369 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 04:18:55.369 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:55.369 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:18:55.369 | INFO | Uploaded 36GiB to GPU in 1.59715s (22.5401GiB/s) | 20260505 04:18:55.369 | INFO | Downloaded 34GiB to GPU in 1.51329s (22.4675GiB/s) | 20260505 04:19:00.507 | INFO | Analyzing chunk frames = [283648,284160) | 20260505 04:19:00.507 | INFO | MemInfo: { | 20260505 04:19:00.507 | INFO | "gpu" : " 17 GB", | 20260505 04:19:00.507 | INFO | "huge" : "", | 20260505 04:19:00.507 | INFO | "huge_shared" : " 27 GB", | 20260505 04:19:00.507 | INFO | "malloc" : " 14 GB", | 20260505 04:19:00.507 | INFO | "pinned" : " 29 GB" | 20260505 04:19:00.508 | INFO | } | 20260505 04:19:00.508 | INFO | Perf report for the last 5.1392 seconds: | 20260505 04:19:00.508 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:19:00.508 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:00.508 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:00.508 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:00.508 | INFO | ANALYSIS : 57.6%, 3.0, 89.5ms, 0 | 20260505 04:19:00.508 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:00.508 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:00.508 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260505 04:19:00.508 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:00.508 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:00.508 | INFO | Uploaded 36GiB to GPU in 1.60118s (22.4835GiB/s) | 20260505 04:19:00.508 | INFO | Downloaded 34GiB to GPU in 1.51792s (22.399GiB/s) | 20260505 04:19:05.647 | INFO | Analyzing chunk frames = [284160,284672) | 20260505 04:19:05.647 | INFO | MemInfo: { | 20260505 04:19:05.647 | INFO | "gpu" : " 17 GB", | 20260505 04:19:05.647 | INFO | "huge" : "", | 20260505 04:19:05.647 | INFO | "huge_shared" : " 27 GB", | 20260505 04:19:05.647 | INFO | "malloc" : " 14 GB", | 20260505 04:19:05.647 | INFO | "pinned" : " 29 GB" | 20260505 04:19:05.647 | INFO | } | 20260505 04:19:05.647 | INFO | Perf report for the last 5.13922 seconds: | 20260505 04:19:05.647 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:19:05.647 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:05.647 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:05.647 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:05.647 | INFO | ANALYSIS : 57.3%, 3.0, 88.8ms, 0 | 20260505 04:19:05.647 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:05.647 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:05.647 | INFO | PRE_HQ : 31.7%, 1.0, 16.4ms, 0 | 20260505 04:19:05.647 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:05.647 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:05.647 | INFO | Uploaded 36GiB to GPU in 1.60058s (22.4919GiB/s) | 20260505 04:19:05.647 | INFO | Downloaded 34GiB to GPU in 1.51717s (22.4101GiB/s) | 20260505 04:19:10.786 | INFO | Analyzing chunk frames = [284672,285184) | 20260505 04:19:10.786 | INFO | MemInfo: { | 20260505 04:19:10.786 | INFO | "gpu" : " 17 GB", | 20260505 04:19:10.786 | INFO | "huge" : "", | 20260505 04:19:10.786 | INFO | "huge_shared" : " 27 GB", | 20260505 04:19:10.786 | INFO | "malloc" : " 14 GB", | 20260505 04:19:10.786 | INFO | "pinned" : " 29 GB" | 20260505 04:19:10.786 | INFO | } | 20260505 04:19:10.786 | INFO | Perf report for the last 5.13933 seconds: | 20260505 04:19:10.786 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:19:10.786 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:10.786 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:10.786 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:10.786 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260505 04:19:10.786 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:10.786 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:10.786 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 04:19:10.786 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:10.786 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:10.786 | INFO | Uploaded 36GiB to GPU in 1.59903s (22.5136GiB/s) | 20260505 04:19:10.787 | INFO | Downloaded 34GiB to GPU in 1.51529s (22.438GiB/s) | 20260505 04:19:15.925 | INFO | Analyzing chunk frames = [285184,285696) | 20260505 04:19:15.925 | INFO | MemInfo: { | 20260505 04:19:15.925 | INFO | "gpu" : " 17 GB", | 20260505 04:19:15.925 | INFO | "huge" : "", | 20260505 04:19:15.925 | INFO | "huge_shared" : " 27 GB", | 20260505 04:19:15.925 | INFO | "malloc" : " 14 GB", | 20260505 04:19:15.926 | INFO | "pinned" : " 29 GB" | 20260505 04:19:15.926 | INFO | } | 20260505 04:19:15.926 | INFO | Perf report for the last 5.13949 seconds: | 20260505 04:19:15.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:19:15.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:15.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:15.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:15.926 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260505 04:19:15.926 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:15.926 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:15.926 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260505 04:19:15.926 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:15.926 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:15.926 | INFO | Uploaded 36GiB to GPU in 1.59902s (22.5138GiB/s) | 20260505 04:19:15.926 | INFO | Downloaded 34GiB to GPU in 1.5156s (22.4334GiB/s) | 20260505 04:19:21.065 | INFO | Analyzing chunk frames = [285696,286208) | 20260505 04:19:21.065 | INFO | MemInfo: { | 20260505 04:19:21.065 | INFO | "gpu" : " 17 GB", | 20260505 04:19:21.065 | INFO | "huge" : "", | 20260505 04:19:21.065 | INFO | "huge_shared" : " 27 GB", | 20260505 04:19:21.065 | INFO | "malloc" : " 14 GB", | 20260505 04:19:21.065 | INFO | "pinned" : " 29 GB" | 20260505 04:19:21.065 | INFO | } | 20260505 04:19:21.065 | INFO | Perf report for the last 5.13938 seconds: | 20260505 04:19:21.065 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:19:21.065 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:21.065 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:21.065 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:21.065 | INFO | ANALYSIS : 57.7%, 3.0, 90.0ms, 0 | 20260505 04:19:21.065 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:21.065 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:21.065 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260505 04:19:21.065 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:21.065 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:21.065 | INFO | Uploaded 36GiB to GPU in 1.59945s (22.5077GiB/s) | 20260505 04:19:21.065 | INFO | Downloaded 34GiB to GPU in 1.51628s (22.4233GiB/s) | 20260505 04:19:26.204 | INFO | Analyzing chunk frames = [286208,286720) | 20260505 04:19:26.204 | INFO | MemInfo: { | 20260505 04:19:26.204 | INFO | "gpu" : " 17 GB", | 20260505 04:19:26.204 | INFO | "huge" : "", | 20260505 04:19:26.204 | INFO | "huge_shared" : " 27 GB", | 20260505 04:19:26.204 | INFO | "malloc" : " 14 GB", | 20260505 04:19:26.204 | INFO | "pinned" : " 29 GB" | 20260505 04:19:26.204 | INFO | } | 20260505 04:19:26.204 | INFO | Perf report for the last 5.13931 seconds: | 20260505 04:19:26.204 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:19:26.204 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:26.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:26.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:26.205 | INFO | ANALYSIS : 57.8%, 3.0, 90.2ms, 0 | 20260505 04:19:26.205 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:26.205 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:26.205 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 04:19:26.205 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:26.205 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:26.205 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 04:19:26.205 | INFO | Total mem usage: 14.9 GiB | 20260505 04:19:26.205 | INFO | All Time High Water Marks sum: 15.2 GiB (Savings: 1.97%) | 20260505 04:19:26.205 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -2.74%) | 20260505 04:19:26.205 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 04:19:26.205 | INFO | High Water Breakdown: | 20260505 04:19:26.205 | INFO | 58.9% : 8.56 GiB ( 100% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 04:19:26.205 | INFO | 40.9% : 5.95 GiB ( 89.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 04:19:26.205 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 04:19:26.205 | INFO | Uploaded 36GiB to GPU in 1.59962s (22.5053GiB/s) | 20260505 04:19:26.205 | INFO | ------------------------------------------ | 20260505 04:19:26.205 | INFO | Downloaded 34GiB to GPU in 1.51668s (22.4174GiB/s) | 20260505 04:19:26.205 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:19:26.205 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 04:19:26.205 | INFO | Total mem usage: 26.5 GiB | 20260505 04:19:26.205 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260505 04:19:26.205 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 04:19:26.205 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260505 04:19:26.205 | INFO | High Water Breakdown: | 20260505 04:19:26.205 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:19:26.205 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:19:26.205 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:19:26.205 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:19:26.205 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:19:26.205 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:19:26.205 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 04:19:26.205 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 04:19:26.205 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:19:26.206 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 04:19:26.206 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 04:19:26.206 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 04:19:26.206 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 04:19:26.206 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 04:19:26.206 | INFO | ------------------------------------------ | 20260505 04:19:26.206 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:19:26.206 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 04:19:26.206 | INFO | Total mem usage: 25.6 GiB | 20260505 04:19:26.206 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 04:19:26.206 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 04:19:26.206 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 04:19:26.206 | INFO | High Water Breakdown: | 20260505 04:19:26.206 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 04:19:26.206 | INFO | ------------------------------------------ | 20260505 04:19:26.206 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:19:26.206 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 04:19:26.206 | INFO | Total mem usage: 15.7 GiB | 20260505 04:19:26.206 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260505 04:19:26.206 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 04:19:26.206 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260505 04:19:26.206 | INFO | High Water Breakdown: | 20260505 04:19:26.206 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 04:19:26.206 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 04:19:26.206 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 04:19:26.206 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 04:19:26.206 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 04:19:26.206 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 04:19:26.206 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 04:19:26.206 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 04:19:26.206 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 04:19:26.206 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:19:26.206 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 04:19:26.206 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:19:26.206 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 04:19:26.206 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 04:19:26.206 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 04:19:26.206 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 04:19:26.206 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:19:26.206 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 04:19:26.206 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:19:26.206 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:19:26.206 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:19:26.206 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 04:19:26.206 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 04:19:26.206 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:19:26.206 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 04:19:26.206 | INFO | ------------------------------------------ | 20260505 04:19:26.206 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:19:26.206 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 04:19:26.206 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:19:26.206 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:19:26.206 | INFO | PerfReport: Full Execution -- 3.05e+01 / -nan, 1.28e+01 / 0.00e+00 | 20260505 04:19:26.206 | INFO | PerfReport: 0.28%, AggregateStats -- 8.63e-02 / -nan, 4.14e-03 / 0.00e+00 | 20260505 04:19:26.206 | INFO | PerfReport: 10.80%, Baseline -- 3.30e+00 / -nan, 1.77e-02 / 0.00e+00 | 20260505 04:19:26.206 | INFO | PerfReport: 3.03%, Histogram -- 9.24e-01 / -nan, 5.19e-02 / 0.00e+00 | 20260505 04:19:26.206 | INFO | PerfReport: 16.01%, DME -- 3.91e+01 / -nan, 3.22e+00 / 0.00e+00 | 20260505 04:19:26.206 | INFO | PerfReport: 56.37%, FrameLabeling -- 1.72e+01 / -nan, 4.44e-02 / 0.00e+00 | 20260505 04:19:26.206 | INFO | PerfReport: 7.71%, PulseAccumulating -- 2.36e+00 / -nan, 1.58e-01 / 0.00e+00 | 20260505 04:19:26.206 | INFO | PerfReport: 4.10%, Metrics -- 1.25e+00 / -nan, 2.04e-01 / 0.00e+00 | 20260505 04:19:26.206 | INFO | PerfReport: 1.70%, PulseSerializing -- 5.20e-01 / -nan, 8.36e-02 / 0.00e+00 | 20260505 04:19:26.206 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 04:19:26.206 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:19:26.206 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:19:26.206 | INFO | PerfReport: Full Execution -- 9.17e+01 / -nan, 1.92e+01 / 0.00e+00 | 20260505 04:19:26.206 | INFO | PerfReport: 18.22%, Upload -- 1.67e+01 / 0.00e+00, 3.66e-01 / 0.00e+00 | 20260505 04:19:26.207 | INFO | PerfReport: 17.48%, Download -- 1.60e+01 / 0.00e+00, 6.91e-01 / 0.00e+00 | 20260505 04:19:26.207 | INFO | PerfReport: 33.30%, Compute -- 3.05e+01 / 0.00e+00, 1.28e+01 / 0.00e+00 | 20260505 04:19:26.207 | INFO | PerfReport: 0.55%, LockUpload -- 5.03e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260505 04:19:26.207 | INFO | PerfReport: 0.00%, LockDownload -- 4.02e-04 / 0.00e+00, 1.33e-04 / 0.00e+00 | 20260505 04:19:26.207 | INFO | PerfReport: 30.44%, LockCompute -- 2.79e+01 / 0.00e+00, 1.73e+01 / 0.00e+00 | 20260505 04:19:26.207 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.26e-02 / 0.00e+00 | 20260505 04:19:26.301 | INFO | Starting a round of Baz IO | 20260505 04:19:29.263 | INFO | Populating RTMetrics report | 20260505 04:19:29.263 | INFO | RTMetrics report populated | 20260505 04:19:29.263 | INFO | Opening tmp RTMetrics json file | 20260505 04:19:29.264 | INFO | Serializing RTMetrics to json | 20260505 04:19:29.265 | INFO | Serialization done, swapping tmp file now... | 20260505 04:19:29.265 | INFO | RTMetrics done | 20260505 04:19:29.351 | INFO | Finished a round of Baz IO: Wrote 1.88 GiB in 3.049 seconds | 20260505 04:19:29.351 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 50.47% / 0.19% / 49.34% | 20260505 04:19:29.351 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 11.8896% / 0.000552336% / 3.57628e-05% | 20260505 04:19:29.351 | INFO | Baz IO overall: | 20260505 04:19:29.351 | INFO | BytesWritten : 102686954480 | 20260505 04:19:29.351 | INFO | NumEvents : 47081225874 | 20260505 04:19:31.343 | INFO | Analyzing chunk frames = [286720,287232) | 20260505 04:19:31.343 | INFO | MemInfo: { | 20260505 04:19:31.343 | INFO | "gpu" : " 17 GB", | 20260505 04:19:31.343 | INFO | "huge" : "", | 20260505 04:19:31.343 | INFO | "huge_shared" : " 27 GB", | 20260505 04:19:31.343 | INFO | "malloc" : " 14 GB", | 20260505 04:19:31.343 | INFO | "pinned" : " 29 GB" | 20260505 04:19:31.343 | INFO | } | 20260505 04:19:31.344 | INFO | Perf report for the last 5.13908 seconds: | 20260505 04:19:31.344 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:19:31.344 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:31.344 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:31.344 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:31.344 | INFO | ANALYSIS : 58.4%, 3.0, 91.0ms, 0 | 20260505 04:19:31.344 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:31.344 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260505 04:19:31.344 | INFO | PRE_HQ : 57.9%, 2.2, 64.8ms, 0 | 20260505 04:19:31.344 | INFO | BAZWRITER : 58.9%, 3.7, 111.7ms, 0 | 20260505 04:19:31.344 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:19:31.344 | INFO | Uploaded 36GiB to GPU in 1.63914s (21.9628GiB/s) | 20260505 04:19:31.344 | INFO | Downloaded 38GiB to GPU in 1.69994s (22.3538GiB/s) | 20260505 04:19:36.483 | INFO | Analyzing chunk frames = [287232,287744) | 20260505 04:19:36.483 | INFO | MemInfo: { | 20260505 04:19:36.483 | INFO | "gpu" : " 17 GB", | 20260505 04:19:36.483 | INFO | "huge" : "", | 20260505 04:19:36.483 | INFO | "huge_shared" : " 27 GB", | 20260505 04:19:36.483 | INFO | "malloc" : " 14 GB", | 20260505 04:19:36.483 | INFO | "pinned" : " 29 GB" | 20260505 04:19:36.483 | INFO | } | 20260505 04:19:36.483 | INFO | Perf report for the last 5.13957 seconds: | 20260505 04:19:36.483 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:19:36.483 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:36.483 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:36.483 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:36.483 | INFO | ANALYSIS : 58.2%, 3.0, 90.8ms, 0 | 20260505 04:19:36.483 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:36.483 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:36.483 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260505 04:19:36.483 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:36.483 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:36.484 | INFO | Uploaded 36GiB to GPU in 1.597s (22.5423GiB/s) | 20260505 04:19:36.484 | INFO | Downloaded 34GiB to GPU in 1.51366s (22.462GiB/s) | 20260505 04:19:41.622 | INFO | Analyzing chunk frames = [287744,288256) | 20260505 04:19:41.622 | INFO | MemInfo: { | 20260505 04:19:41.622 | INFO | "gpu" : " 17 GB", | 20260505 04:19:41.622 | INFO | "huge" : "", | 20260505 04:19:41.622 | INFO | "huge_shared" : " 27 GB", | 20260505 04:19:41.622 | INFO | "malloc" : " 14 GB", | 20260505 04:19:41.622 | INFO | "pinned" : " 29 GB" | 20260505 04:19:41.622 | INFO | } | 20260505 04:19:41.623 | INFO | Perf report for the last 5.13939 seconds: | 20260505 04:19:41.623 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:19:41.623 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:41.623 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:41.623 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:41.623 | INFO | ANALYSIS : 57.5%, 3.0, 89.4ms, 0 | 20260505 04:19:41.623 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:41.623 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:41.623 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 04:19:41.623 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:41.623 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:41.623 | INFO | Uploaded 36GiB to GPU in 1.60103s (22.4855GiB/s) | 20260505 04:19:41.623 | INFO | Downloaded 34GiB to GPU in 1.51843s (22.3916GiB/s) | 20260505 04:19:46.761 | INFO | Analyzing chunk frames = [288256,288768) | 20260505 04:19:46.761 | INFO | MemInfo: { | 20260505 04:19:46.761 | INFO | "gpu" : " 17 GB", | 20260505 04:19:46.761 | INFO | "huge" : "", | 20260505 04:19:46.761 | INFO | "huge_shared" : " 27 GB", | 20260505 04:19:46.761 | INFO | "malloc" : " 14 GB", | 20260505 04:19:46.761 | INFO | "pinned" : " 29 GB" | 20260505 04:19:46.761 | INFO | } | 20260505 04:19:46.762 | INFO | Perf report for the last 5.13914 seconds: | 20260505 04:19:46.762 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:19:46.762 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:46.762 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:46.762 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:46.762 | INFO | ANALYSIS : 57.3%, 3.0, 88.8ms, 0 | 20260505 04:19:46.762 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:46.762 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:46.762 | INFO | PRE_HQ : 32.1%, 1.0, 16.6ms, 0 | 20260505 04:19:46.762 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:46.762 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:46.762 | INFO | Uploaded 36GiB to GPU in 1.60038s (22.4946GiB/s) | 20260505 04:19:46.762 | INFO | Downloaded 34GiB to GPU in 1.51764s (22.4032GiB/s) | 20260505 04:19:51.901 | INFO | Analyzing chunk frames = [288768,289280) | 20260505 04:19:51.901 | INFO | MemInfo: { | 20260505 04:19:51.901 | INFO | "gpu" : " 17 GB", | 20260505 04:19:51.901 | INFO | "huge" : "", | 20260505 04:19:51.901 | INFO | "huge_shared" : " 27 GB", | 20260505 04:19:51.901 | INFO | "malloc" : " 14 GB", | 20260505 04:19:51.901 | INFO | "pinned" : " 29 GB" | 20260505 04:19:51.901 | INFO | } | 20260505 04:19:51.901 | INFO | Perf report for the last 5.13934 seconds: | 20260505 04:19:51.901 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:19:51.901 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:51.901 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:51.901 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:51.901 | INFO | ANALYSIS : 57.2%, 3.0, 89.1ms, 0 | 20260505 04:19:51.901 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:51.901 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:51.901 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260505 04:19:51.901 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:51.901 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:51.902 | INFO | Uploaded 36GiB to GPU in 1.59903s (22.5137GiB/s) | 20260505 04:19:51.902 | INFO | Downloaded 34GiB to GPU in 1.51512s (22.4405GiB/s) | 20260505 04:19:57.040 | INFO | Analyzing chunk frames = [289280,289792) | 20260505 04:19:57.040 | INFO | MemInfo: { | 20260505 04:19:57.040 | INFO | "gpu" : " 17 GB", | 20260505 04:19:57.040 | INFO | "huge" : "", | 20260505 04:19:57.040 | INFO | "huge_shared" : " 27 GB", | 20260505 04:19:57.040 | INFO | "malloc" : " 14 GB", | 20260505 04:19:57.040 | INFO | "pinned" : " 29 GB" | 20260505 04:19:57.040 | INFO | } | 20260505 04:19:57.041 | INFO | Perf report for the last 5.13946 seconds: | 20260505 04:19:57.041 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:19:57.041 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:57.041 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:57.041 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:57.041 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260505 04:19:57.041 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:57.041 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:57.041 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 04:19:57.041 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:57.041 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:19:57.041 | INFO | Uploaded 36GiB to GPU in 1.59888s (22.5158GiB/s) | 20260505 04:19:57.041 | INFO | Downloaded 34GiB to GPU in 1.51569s (22.432GiB/s) | 20260505 04:20:02.180 | INFO | Analyzing chunk frames = [289792,290304) | 20260505 04:20:02.180 | INFO | MemInfo: { | 20260505 04:20:02.180 | INFO | "gpu" : " 17 GB", | 20260505 04:20:02.180 | INFO | "huge" : "", | 20260505 04:20:02.180 | INFO | "huge_shared" : " 27 GB", | 20260505 04:20:02.180 | INFO | "malloc" : " 14 GB", | 20260505 04:20:02.180 | INFO | "pinned" : " 29 GB" | 20260505 04:20:02.180 | INFO | } | 20260505 04:20:02.180 | INFO | Perf report for the last 5.13924 seconds: | 20260505 04:20:02.180 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:20:02.180 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:02.180 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:02.180 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:02.180 | INFO | ANALYSIS : 57.6%, 3.0, 89.9ms, 0 | 20260505 04:20:02.180 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:02.180 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:02.180 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260505 04:20:02.180 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:02.180 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:02.180 | INFO | Uploaded 36GiB to GPU in 1.6s (22.5GiB/s) | 20260505 04:20:02.180 | INFO | Downloaded 34GiB to GPU in 1.51666s (22.4177GiB/s) | 20260505 04:20:07.319 | INFO | Analyzing chunk frames = [290304,290816) | 20260505 04:20:07.319 | INFO | MemInfo: { | 20260505 04:20:07.319 | INFO | "gpu" : " 17 GB", | 20260505 04:20:07.319 | INFO | "huge" : "", | 20260505 04:20:07.319 | INFO | "huge_shared" : " 27 GB", | 20260505 04:20:07.319 | INFO | "malloc" : " 14 GB", | 20260505 04:20:07.319 | INFO | "pinned" : " 29 GB" | 20260505 04:20:07.319 | INFO | } | 20260505 04:20:07.319 | INFO | Perf report for the last 5.13948 seconds: | 20260505 04:20:07.319 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:20:07.319 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:07.319 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:07.319 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:07.319 | INFO | ANALYSIS : 57.8%, 3.0, 90.1ms, 0 | 20260505 04:20:07.319 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:07.319 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:07.319 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 04:20:07.319 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:07.319 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:07.320 | INFO | Uploaded 36GiB to GPU in 1.59979s (22.503GiB/s) | 20260505 04:20:07.320 | INFO | Downloaded 34GiB to GPU in 1.51662s (22.4183GiB/s) | 20260505 04:20:07.416 | INFO | Starting a round of Baz IO | 20260505 04:20:10.375 | INFO | Populating RTMetrics report | 20260505 04:20:10.375 | INFO | RTMetrics report populated | 20260505 04:20:10.375 | INFO | Opening tmp RTMetrics json file | 20260505 04:20:10.375 | INFO | Serializing RTMetrics to json | 20260505 04:20:10.377 | INFO | Serialization done, swapping tmp file now... | 20260505 04:20:10.377 | INFO | RTMetrics done | 20260505 04:20:10.463 | INFO | Finished a round of Baz IO: Wrote 1.884 GiB in 3.046 seconds | 20260505 04:20:10.463 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 50.29% / 0.18% / 49.53% | 20260505 04:20:10.463 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 11.8208% / 0.0005126% / 3.57628e-05% | 20260505 04:20:10.463 | INFO | Baz IO overall: | 20260505 04:20:10.463 | INFO | BytesWritten : 104710400644 | 20260505 04:20:10.463 | INFO | NumEvents : 48037498920 | 20260505 04:20:12.458 | INFO | Analyzing chunk frames = [290816,291328) | 20260505 04:20:12.458 | INFO | MemInfo: { | 20260505 04:20:12.458 | INFO | "gpu" : " 17 GB", | 20260505 04:20:12.458 | INFO | "huge" : "", | 20260505 04:20:12.458 | INFO | "huge_shared" : " 27 GB", | 20260505 04:20:12.458 | INFO | "malloc" : " 14 GB", | 20260505 04:20:12.458 | INFO | "pinned" : " 29 GB" | 20260505 04:20:12.458 | INFO | } | 20260505 04:20:12.458 | INFO | Perf report for the last 5.13921 seconds: | 20260505 04:20:12.458 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:20:12.458 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:12.458 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:12.459 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:12.459 | INFO | ANALYSIS : 58.4%, 3.0, 90.9ms, 0 | 20260505 04:20:12.459 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:12.459 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260505 04:20:12.459 | INFO | PRE_HQ : 57.8%, 2.2, 64.9ms, 0 | 20260505 04:20:12.459 | INFO | BAZWRITER : 58.8%, 3.7, 111.9ms, 0 | 20260505 04:20:12.459 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:20:12.459 | INFO | Uploaded 36GiB to GPU in 1.646s (21.8712GiB/s) | 20260505 04:20:12.459 | INFO | Downloaded 38GiB to GPU in 1.70388s (22.302GiB/s) | 20260505 04:20:17.598 | INFO | Analyzing chunk frames = [291328,291840) | 20260505 04:20:17.598 | INFO | MemInfo: { | 20260505 04:20:17.598 | INFO | "gpu" : " 17 GB", | 20260505 04:20:17.598 | INFO | "huge" : "", | 20260505 04:20:17.598 | INFO | "huge_shared" : " 27 GB", | 20260505 04:20:17.598 | INFO | "malloc" : " 14 GB", | 20260505 04:20:17.598 | INFO | "pinned" : " 29 GB" | 20260505 04:20:17.598 | INFO | } | 20260505 04:20:17.598 | INFO | Perf report for the last 5.13928 seconds: | 20260505 04:20:17.598 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:20:17.598 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:17.598 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:17.598 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:17.598 | INFO | ANALYSIS : 58.2%, 3.0, 90.7ms, 0 | 20260505 04:20:17.598 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:17.598 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:17.598 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260505 04:20:17.598 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:17.598 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:17.598 | INFO | Uploaded 36GiB to GPU in 1.59761s (22.5336GiB/s) | 20260505 04:20:17.598 | INFO | Downloaded 34GiB to GPU in 1.51489s (22.4439GiB/s) | 20260505 04:20:22.737 | INFO | Analyzing chunk frames = [291840,292352) | 20260505 04:20:22.737 | INFO | MemInfo: { | 20260505 04:20:22.737 | INFO | "gpu" : " 17 GB", | 20260505 04:20:22.737 | INFO | "huge" : "", | 20260505 04:20:22.737 | INFO | "huge_shared" : " 27 GB", | 20260505 04:20:22.737 | INFO | "malloc" : " 14 GB", | 20260505 04:20:22.737 | INFO | "pinned" : " 29 GB" | 20260505 04:20:22.737 | INFO | } | 20260505 04:20:22.737 | INFO | Perf report for the last 5.13938 seconds: | 20260505 04:20:22.737 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:20:22.737 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:22.737 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:22.737 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:22.737 | INFO | ANALYSIS : 57.6%, 3.0, 89.5ms, 0 | 20260505 04:20:22.737 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:22.737 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:22.737 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 04:20:22.737 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:22.737 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:22.738 | INFO | Uploaded 36GiB to GPU in 1.60101s (22.4858GiB/s) | 20260505 04:20:22.738 | INFO | Downloaded 34GiB to GPU in 1.51844s (22.3914GiB/s) | 20260505 04:20:27.876 | INFO | Analyzing chunk frames = [292352,292864) | 20260505 04:20:27.876 | INFO | MemInfo: { | 20260505 04:20:27.876 | INFO | "gpu" : " 17 GB", | 20260505 04:20:27.876 | INFO | "huge" : "", | 20260505 04:20:27.876 | INFO | "huge_shared" : " 27 GB", | 20260505 04:20:27.876 | INFO | "malloc" : " 14 GB", | 20260505 04:20:27.876 | INFO | "pinned" : " 29 GB" | 20260505 04:20:27.876 | INFO | } | 20260505 04:20:27.876 | INFO | Perf report for the last 5.13926 seconds: | 20260505 04:20:27.876 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:20:27.876 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:27.876 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:27.876 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:27.876 | INFO | ANALYSIS : 57.2%, 3.0, 88.7ms, 0 | 20260505 04:20:27.876 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:27.877 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:27.877 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260505 04:20:27.877 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:27.877 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:27.877 | INFO | Uploaded 36GiB to GPU in 1.60088s (22.4876GiB/s) | 20260505 04:20:27.877 | INFO | Downloaded 34GiB to GPU in 1.51778s (22.4011GiB/s) | 20260505 04:20:33.016 | INFO | Analyzing chunk frames = [292864,293376) | 20260505 04:20:33.016 | INFO | MemInfo: { | 20260505 04:20:33.016 | INFO | "gpu" : " 17 GB", | 20260505 04:20:33.016 | INFO | "huge" : "", | 20260505 04:20:33.016 | INFO | "huge_shared" : " 27 GB", | 20260505 04:20:33.016 | INFO | "malloc" : " 14 GB", | 20260505 04:20:33.016 | INFO | "pinned" : " 29 GB" | 20260505 04:20:33.016 | INFO | } | 20260505 04:20:33.016 | INFO | Perf report for the last 5.13933 seconds: | 20260505 04:20:33.016 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:20:33.016 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:33.016 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:33.016 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:33.016 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260505 04:20:33.016 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:33.016 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:33.016 | INFO | PRE_HQ : 32.7%, 1.0, 17.0ms, 0 | 20260505 04:20:33.016 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:33.016 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:33.016 | INFO | Uploaded 36GiB to GPU in 1.59936s (22.509GiB/s) | 20260505 04:20:33.016 | INFO | Downloaded 34GiB to GPU in 1.51606s (22.4265GiB/s) | 20260505 04:20:38.155 | INFO | Analyzing chunk frames = [293376,293888) | 20260505 04:20:38.155 | INFO | MemInfo: { | 20260505 04:20:38.155 | INFO | "gpu" : " 17 GB", | 20260505 04:20:38.155 | INFO | "huge" : "", | 20260505 04:20:38.155 | INFO | "huge_shared" : " 27 GB", | 20260505 04:20:38.155 | INFO | "malloc" : " 14 GB", | 20260505 04:20:38.155 | INFO | "pinned" : " 29 GB" | 20260505 04:20:38.155 | INFO | } | 20260505 04:20:38.155 | INFO | Perf report for the last 5.13952 seconds: | 20260505 04:20:38.155 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:20:38.155 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:38.155 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:38.155 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:38.155 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260505 04:20:38.155 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:38.155 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:38.155 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260505 04:20:38.155 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:38.155 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:38.156 | INFO | Uploaded 36GiB to GPU in 1.59907s (22.5131GiB/s) | 20260505 04:20:38.156 | INFO | Downloaded 34GiB to GPU in 1.51645s (22.4208GiB/s) | 20260505 04:20:43.294 | INFO | Analyzing chunk frames = [293888,294400) | 20260505 04:20:43.294 | INFO | MemInfo: { | 20260505 04:20:43.294 | INFO | "gpu" : " 17 GB", | 20260505 04:20:43.294 | INFO | "huge" : "", | 20260505 04:20:43.294 | INFO | "huge_shared" : " 27 GB", | 20260505 04:20:43.294 | INFO | "malloc" : " 14 GB", | 20260505 04:20:43.294 | INFO | "pinned" : " 29 GB" | 20260505 04:20:43.294 | INFO | } | 20260505 04:20:43.295 | INFO | Perf report for the last 5.13918 seconds: | 20260505 04:20:43.295 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:20:43.295 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:43.295 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:43.295 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:43.295 | INFO | ANALYSIS : 57.7%, 3.0, 90.0ms, 0 | 20260505 04:20:43.295 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:43.295 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:43.295 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260505 04:20:43.295 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:43.295 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:43.295 | INFO | Uploaded 36GiB to GPU in 1.59823s (22.5249GiB/s) | 20260505 04:20:43.295 | INFO | Downloaded 34GiB to GPU in 1.51642s (22.4212GiB/s) | 20260505 04:20:48.434 | INFO | Analyzing chunk frames = [294400,294912) | 20260505 04:20:48.434 | INFO | MemInfo: { | 20260505 04:20:48.434 | INFO | "gpu" : " 17 GB", | 20260505 04:20:48.434 | INFO | "huge" : "", | 20260505 04:20:48.434 | INFO | "huge_shared" : " 27 GB", | 20260505 04:20:48.434 | INFO | "malloc" : " 14 GB", | 20260505 04:20:48.434 | INFO | "pinned" : " 29 GB" | 20260505 04:20:48.434 | INFO | } | 20260505 04:20:48.434 | INFO | Perf report for the last 5.13935 seconds: | 20260505 04:20:48.434 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:20:48.434 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:48.434 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:48.434 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:48.434 | INFO | ANALYSIS : 57.7%, 3.0, 90.1ms, 0 | 20260505 04:20:48.434 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:48.434 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:48.434 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 04:20:48.434 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:48.434 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:48.434 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 04:20:48.434 | INFO | Total mem usage: 14.9 GiB | 20260505 04:20:48.434 | INFO | All Time High Water Marks sum: 15.2 GiB (Savings: 1.98%) | 20260505 04:20:48.434 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -2.78%) | 20260505 04:20:48.434 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 04:20:48.435 | INFO | High Water Breakdown: | 20260505 04:20:48.435 | INFO | 59% : 8.56 GiB ( 100% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 04:20:48.435 | INFO | 40.9% : 5.94 GiB ( 89.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 04:20:48.435 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 04:20:48.435 | INFO | ------------------------------------------ | 20260505 04:20:48.435 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:20:48.435 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 04:20:48.435 | INFO | Total mem usage: 26.5 GiB | 20260505 04:20:48.435 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260505 04:20:48.435 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 04:20:48.435 | INFO | Uploaded 36GiB to GPU in 1.60031s (22.4956GiB/s) | 20260505 04:20:48.435 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260505 04:20:48.435 | INFO | Downloaded 34GiB to GPU in 1.51727s (22.4087GiB/s) | 20260505 04:20:48.435 | INFO | High Water Breakdown: | 20260505 04:20:48.435 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:20:48.435 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:20:48.435 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:20:48.435 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:20:48.435 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:20:48.435 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:20:48.435 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 04:20:48.435 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 04:20:48.435 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:20:48.435 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 04:20:48.435 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 04:20:48.435 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 04:20:48.435 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 04:20:48.435 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 04:20:48.435 | INFO | ------------------------------------------ | 20260505 04:20:48.435 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:20:48.435 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 04:20:48.435 | INFO | Total mem usage: 25.6 GiB | 20260505 04:20:48.435 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 04:20:48.435 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 04:20:48.435 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 04:20:48.435 | INFO | High Water Breakdown: | 20260505 04:20:48.435 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 04:20:48.435 | INFO | ------------------------------------------ | 20260505 04:20:48.435 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:20:48.435 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 04:20:48.435 | INFO | Total mem usage: 15.7 GiB | 20260505 04:20:48.435 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260505 04:20:48.435 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 04:20:48.435 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260505 04:20:48.435 | INFO | High Water Breakdown: | 20260505 04:20:48.435 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 04:20:48.435 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 04:20:48.435 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 04:20:48.435 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 04:20:48.435 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 04:20:48.435 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 04:20:48.435 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 04:20:48.435 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 04:20:48.435 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 04:20:48.435 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:20:48.435 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 04:20:48.435 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:20:48.435 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 04:20:48.435 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 04:20:48.435 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 04:20:48.435 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 04:20:48.435 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:20:48.435 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 04:20:48.435 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:20:48.435 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:20:48.435 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:20:48.435 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 04:20:48.435 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 04:20:48.435 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:20:48.435 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 04:20:48.435 | INFO | ------------------------------------------ | 20260505 04:20:48.435 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:20:48.435 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 04:20:48.435 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:20:48.435 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:20:48.435 | INFO | PerfReport: Full Execution -- 3.05e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260505 04:20:48.435 | INFO | PerfReport: 0.28%, AggregateStats -- 8.61e-02 / -nan, 4.44e-03 / 0.00e+00 | 20260505 04:20:48.435 | INFO | PerfReport: 10.79%, Baseline -- 3.29e+00 / -nan, 1.72e-02 / 0.00e+00 | 20260505 04:20:48.436 | INFO | PerfReport: 3.03%, Histogram -- 9.24e-01 / -nan, 5.12e-02 / 0.00e+00 | 20260505 04:20:48.436 | INFO | PerfReport: 16.02%, DME -- 3.91e+01 / -nan, 3.29e+00 / 0.00e+00 | 20260505 04:20:48.436 | INFO | PerfReport: 56.38%, FrameLabeling -- 1.72e+01 / -nan, 4.52e-02 / 0.00e+00 | 20260505 04:20:48.436 | INFO | PerfReport: 7.71%, PulseAccumulating -- 2.35e+00 / -nan, 1.59e-01 / 0.00e+00 | 20260505 04:20:48.436 | INFO | PerfReport: 4.09%, Metrics -- 1.25e+00 / -nan, 2.05e-01 / 0.00e+00 | 20260505 04:20:48.436 | INFO | PerfReport: 1.70%, PulseSerializing -- 5.19e-01 / -nan, 8.34e-02 / 0.00e+00 | 20260505 04:20:48.436 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 04:20:48.436 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:20:48.436 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:20:48.436 | INFO | PerfReport: Full Execution -- 9.17e+01 / -nan, 1.92e+01 / 0.00e+00 | 20260505 04:20:48.436 | INFO | PerfReport: 18.23%, Upload -- 1.67e+01 / 0.00e+00, 3.82e-01 / 0.00e+00 | 20260505 04:20:48.436 | INFO | PerfReport: 17.49%, Download -- 1.60e+01 / 0.00e+00, 6.98e-01 / 0.00e+00 | 20260505 04:20:48.436 | INFO | PerfReport: 33.30%, Compute -- 3.05e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260505 04:20:48.436 | INFO | PerfReport: 0.55%, LockUpload -- 5.05e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260505 04:20:48.436 | INFO | PerfReport: 0.00%, LockDownload -- 3.94e-04 / 0.00e+00, 1.07e-04 / 0.00e+00 | 20260505 04:20:48.436 | INFO | PerfReport: 30.42%, LockCompute -- 2.79e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260505 04:20:48.436 | INFO | PerfReport: 0.01%, Misc -- 1.26e-02 / 0.00e+00, 1.14e-02 / 0.00e+00 | 20260505 04:20:48.531 | INFO | Starting a round of Baz IO | 20260505 04:20:51.493 | INFO | Populating RTMetrics report | 20260505 04:20:51.494 | INFO | RTMetrics report populated | 20260505 04:20:51.494 | INFO | Opening tmp RTMetrics json file | 20260505 04:20:51.494 | INFO | Serializing RTMetrics to json | 20260505 04:20:51.496 | INFO | Serialization done, swapping tmp file now... | 20260505 04:20:51.496 | INFO | RTMetrics done | 20260505 04:20:51.578 | INFO | Finished a round of Baz IO: Wrote 1.891 GiB in 3.046 seconds | 20260505 04:20:51.578 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 50.11% / 0.18% / 49.71% | 20260505 04:20:51.578 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 11.7671% / 0.000568231% / 4.37101e-05% | 20260505 04:20:51.578 | INFO | Baz IO overall: | 20260505 04:20:51.578 | INFO | BytesWritten : 106740959608 | 20260505 04:20:51.578 | INFO | NumEvents : 48997602660 | 20260505 04:20:53.573 | INFO | Analyzing chunk frames = [294912,295424) | 20260505 04:20:53.573 | INFO | MemInfo: { | 20260505 04:20:53.573 | INFO | "gpu" : " 17 GB", | 20260505 04:20:53.573 | INFO | "huge" : "", | 20260505 04:20:53.573 | INFO | "huge_shared" : " 27 GB", | 20260505 04:20:53.573 | INFO | "malloc" : " 14 GB", | 20260505 04:20:53.573 | INFO | "pinned" : " 29 GB" | 20260505 04:20:53.573 | INFO | } | 20260505 04:20:53.573 | INFO | Perf report for the last 5.13925 seconds: | 20260505 04:20:53.573 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:20:53.573 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:53.573 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:53.573 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:53.573 | INFO | ANALYSIS : 58.4%, 3.0, 91.1ms, 0 | 20260505 04:20:53.573 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:53.573 | INFO | RT_METRICS : 44.9%, 1.0, 23.6ms, 0 | 20260505 04:20:53.573 | INFO | PRE_HQ : 57.9%, 2.2, 64.8ms, 0 | 20260505 04:20:53.573 | INFO | BAZWRITER : 58.8%, 3.7, 112.1ms, 0 | 20260505 04:20:53.573 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:20:53.574 | INFO | Uploaded 36GiB to GPU in 1.63899s (21.9647GiB/s) | 20260505 04:20:53.574 | INFO | Downloaded 38GiB to GPU in 1.70126s (22.3364GiB/s) | 20260505 04:20:58.712 | INFO | Analyzing chunk frames = [295424,295936) | 20260505 04:20:58.712 | INFO | MemInfo: { | 20260505 04:20:58.712 | INFO | "gpu" : " 17 GB", | 20260505 04:20:58.712 | INFO | "huge" : "", | 20260505 04:20:58.712 | INFO | "huge_shared" : " 27 GB", | 20260505 04:20:58.712 | INFO | "malloc" : " 14 GB", | 20260505 04:20:58.713 | INFO | "pinned" : " 29 GB" | 20260505 04:20:58.713 | INFO | } | 20260505 04:20:58.713 | INFO | Perf report for the last 5.13952 seconds: | 20260505 04:20:58.713 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:20:58.713 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:58.713 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:58.713 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:58.713 | INFO | ANALYSIS : 58.2%, 3.0, 90.7ms, 0 | 20260505 04:20:58.713 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:58.713 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:58.713 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260505 04:20:58.713 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:58.713 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:20:58.713 | INFO | Uploaded 36GiB to GPU in 1.59746s (22.5358GiB/s) | 20260505 04:20:58.713 | INFO | Downloaded 34GiB to GPU in 1.51441s (22.451GiB/s) | 20260505 04:21:03.852 | INFO | Analyzing chunk frames = [295936,296448) | 20260505 04:21:03.852 | INFO | MemInfo: { | 20260505 04:21:03.852 | INFO | "gpu" : " 17 GB", | 20260505 04:21:03.852 | INFO | "huge" : "", | 20260505 04:21:03.852 | INFO | "huge_shared" : " 27 GB", | 20260505 04:21:03.852 | INFO | "malloc" : " 14 GB", | 20260505 04:21:03.852 | INFO | "pinned" : " 29 GB" | 20260505 04:21:03.852 | INFO | } | 20260505 04:21:03.852 | INFO | Perf report for the last 5.13924 seconds: | 20260505 04:21:03.852 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:21:03.852 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:03.852 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:03.852 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:03.852 | INFO | ANALYSIS : 57.6%, 3.0, 89.4ms, 0 | 20260505 04:21:03.852 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:03.852 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:03.852 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 04:21:03.852 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:03.852 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:03.852 | INFO | Uploaded 36GiB to GPU in 1.60172s (22.4758GiB/s) | 20260505 04:21:03.852 | INFO | Downloaded 34GiB to GPU in 1.51888s (22.385GiB/s) | 20260505 04:21:08.991 | INFO | Analyzing chunk frames = [296448,296960) | 20260505 04:21:08.991 | INFO | MemInfo: { | 20260505 04:21:08.991 | INFO | "gpu" : " 17 GB", | 20260505 04:21:08.991 | INFO | "huge" : "", | 20260505 04:21:08.991 | INFO | "huge_shared" : " 27 GB", | 20260505 04:21:08.991 | INFO | "malloc" : " 14 GB", | 20260505 04:21:08.991 | INFO | "pinned" : " 29 GB" | 20260505 04:21:08.991 | INFO | } | 20260505 04:21:08.991 | INFO | Perf report for the last 5.13941 seconds: | 20260505 04:21:08.991 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:21:08.991 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:08.991 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:08.991 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:08.991 | INFO | ANALYSIS : 57.2%, 3.0, 88.7ms, 0 | 20260505 04:21:08.991 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:08.991 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:08.991 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260505 04:21:08.991 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:08.992 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:08.992 | INFO | Uploaded 36GiB to GPU in 1.60055s (22.4923GiB/s) | 20260505 04:21:08.992 | INFO | Downloaded 34GiB to GPU in 1.51754s (22.4047GiB/s) | 20260505 04:21:14.131 | INFO | Analyzing chunk frames = [296960,297472) | 20260505 04:21:14.131 | INFO | MemInfo: { | 20260505 04:21:14.131 | INFO | "gpu" : " 17 GB", | 20260505 04:21:14.131 | INFO | "huge" : "", | 20260505 04:21:14.131 | INFO | "huge_shared" : " 27 GB", | 20260505 04:21:14.131 | INFO | "malloc" : " 14 GB", | 20260505 04:21:14.131 | INFO | "pinned" : " 29 GB" | 20260505 04:21:14.131 | INFO | } | 20260505 04:21:14.131 | INFO | Perf report for the last 5.1394 seconds: | 20260505 04:21:14.131 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:21:14.131 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:14.131 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:14.131 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:14.131 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260505 04:21:14.131 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:14.131 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:14.131 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260505 04:21:14.131 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:14.131 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:14.131 | INFO | Uploaded 36GiB to GPU in 1.59869s (22.5184GiB/s) | 20260505 04:21:14.131 | INFO | Downloaded 34GiB to GPU in 1.51503s (22.4418GiB/s) | 20260505 04:21:19.270 | INFO | Analyzing chunk frames = [297472,297984) | 20260505 04:21:19.270 | INFO | MemInfo: { | 20260505 04:21:19.270 | INFO | "gpu" : " 17 GB", | 20260505 04:21:19.270 | INFO | "huge" : "", | 20260505 04:21:19.270 | INFO | "huge_shared" : " 27 GB", | 20260505 04:21:19.270 | INFO | "malloc" : " 14 GB", | 20260505 04:21:19.270 | INFO | "pinned" : " 29 GB" | 20260505 04:21:19.270 | INFO | } | 20260505 04:21:19.270 | INFO | Perf report for the last 5.13914 seconds: | 20260505 04:21:19.270 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:21:19.270 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:19.270 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:19.270 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:19.270 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260505 04:21:19.270 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:19.270 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:19.270 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 04:21:19.270 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:19.270 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:19.270 | INFO | Uploaded 36GiB to GPU in 1.59974s (22.5037GiB/s) | 20260505 04:21:19.270 | INFO | Downloaded 34GiB to GPU in 1.51699s (22.4128GiB/s) | 20260505 04:21:24.409 | INFO | Analyzing chunk frames = [297984,298496) | 20260505 04:21:24.409 | INFO | MemInfo: { | 20260505 04:21:24.409 | INFO | "gpu" : " 17 GB", | 20260505 04:21:24.409 | INFO | "huge" : "", | 20260505 04:21:24.409 | INFO | "huge_shared" : " 27 GB", | 20260505 04:21:24.409 | INFO | "malloc" : " 14 GB", | 20260505 04:21:24.409 | INFO | "pinned" : " 29 GB" | 20260505 04:21:24.409 | INFO | } | 20260505 04:21:24.409 | INFO | Perf report for the last 5.13947 seconds: | 20260505 04:21:24.409 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:21:24.409 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:24.409 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:24.409 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:24.409 | INFO | ANALYSIS : 57.7%, 3.0, 90.0ms, 0 | 20260505 04:21:24.409 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:24.410 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:24.410 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 04:21:24.410 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:24.410 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:24.410 | INFO | Uploaded 36GiB to GPU in 1.59958s (22.5059GiB/s) | 20260505 04:21:24.410 | INFO | Downloaded 34GiB to GPU in 1.51661s (22.4184GiB/s) | 20260505 04:21:29.549 | INFO | Analyzing chunk frames = [298496,299008) | 20260505 04:21:29.549 | INFO | MemInfo: { | 20260505 04:21:29.549 | INFO | "gpu" : " 17 GB", | 20260505 04:21:29.549 | INFO | "huge" : "", | 20260505 04:21:29.549 | INFO | "huge_shared" : " 27 GB", | 20260505 04:21:29.549 | INFO | "malloc" : " 14 GB", | 20260505 04:21:29.549 | INFO | "pinned" : " 29 GB" | 20260505 04:21:29.549 | INFO | } | 20260505 04:21:29.549 | INFO | Perf report for the last 5.1396 seconds: | 20260505 04:21:29.549 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:21:29.549 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:29.549 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:29.549 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:29.549 | INFO | ANALYSIS : 57.7%, 3.0, 90.0ms, 0 | 20260505 04:21:29.549 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:29.549 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:29.549 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 04:21:29.549 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:29.549 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:29.550 | INFO | Uploaded 36GiB to GPU in 1.60004s (22.4995GiB/s) | 20260505 04:21:29.550 | INFO | Downloaded 34GiB to GPU in 1.51727s (22.4087GiB/s) | 20260505 04:21:29.647 | INFO | Starting a round of Baz IO | 20260505 04:21:32.607 | INFO | Populating RTMetrics report | 20260505 04:21:32.607 | INFO | RTMetrics report populated | 20260505 04:21:32.607 | INFO | Opening tmp RTMetrics json file | 20260505 04:21:32.608 | INFO | Serializing RTMetrics to json | 20260505 04:21:32.609 | INFO | Serialization done, swapping tmp file now... | 20260505 04:21:32.610 | INFO | RTMetrics done | 20260505 04:21:32.691 | INFO | Finished a round of Baz IO: Wrote 1.898 GiB in 3.044 seconds | 20260505 04:21:32.691 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 49.93% / 0.18% / 49.89% | 20260505 04:21:32.691 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 11.7191% / 0.000480811% / 3.57628e-05% | 20260505 04:21:32.691 | INFO | Baz IO overall: | 20260505 04:21:32.691 | INFO | BytesWritten : 108778736755 | 20260505 04:21:32.692 | INFO | NumEvents : 49960941412 | 20260505 04:21:34.688 | INFO | Analyzing chunk frames = [299008,299520) | 20260505 04:21:34.688 | INFO | MemInfo: { | 20260505 04:21:34.688 | INFO | "gpu" : " 17 GB", | 20260505 04:21:34.688 | INFO | "huge" : "", | 20260505 04:21:34.688 | INFO | "huge_shared" : " 27 GB", | 20260505 04:21:34.688 | INFO | "malloc" : " 14 GB", | 20260505 04:21:34.688 | INFO | "pinned" : " 29 GB" | 20260505 04:21:34.688 | INFO | } | 20260505 04:21:34.688 | INFO | Perf report for the last 5.13898 seconds: | 20260505 04:21:34.688 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:21:34.688 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:34.688 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:34.688 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:34.688 | INFO | ANALYSIS : 58.4%, 3.0, 90.9ms, 0 | 20260505 04:21:34.688 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:34.688 | INFO | RT_METRICS : 44.9%, 1.0, 23.5ms, 0 | 20260505 04:21:34.688 | INFO | PRE_HQ : 57.8%, 2.2, 64.6ms, 0 | 20260505 04:21:34.688 | INFO | BAZWRITER : 58.8%, 3.7, 112.3ms, 0 | 20260505 04:21:34.688 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:21:34.688 | INFO | Uploaded 36GiB to GPU in 1.63891s (21.9659GiB/s) | 20260505 04:21:34.688 | INFO | Downloaded 38GiB to GPU in 1.70139s (22.3347GiB/s) | 20260505 04:21:39.852 | INFO | Analyzing chunk frames = [299520,300032) | 20260505 04:21:39.852 | INFO | MemInfo: { | 20260505 04:21:39.852 | INFO | "gpu" : " 17 GB", | 20260505 04:21:39.852 | INFO | "huge" : "", | 20260505 04:21:39.852 | INFO | "huge_shared" : " 27 GB", | 20260505 04:21:39.852 | INFO | "malloc" : " 14 GB", | 20260505 04:21:39.852 | INFO | "pinned" : " 29 GB" | 20260505 04:21:39.852 | INFO | } | 20260505 04:21:39.852 | INFO | Perf report for the last 5.16447 seconds: | 20260505 04:21:39.852 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:21:39.852 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:39.852 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:39.852 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:39.852 | INFO | ANALYSIS : 57.9%, 3.0, 90.7ms, 0 | 20260505 04:21:39.853 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:39.853 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:39.853 | INFO | PRE_HQ : 31.8%, 1.0, 16.7ms, 0 | 20260505 04:21:39.853 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:39.853 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:39.853 | INFO | Uploaded 36GiB to GPU in 1.59792s (22.5293GiB/s) | 20260505 04:21:39.853 | INFO | Downloaded 34GiB to GPU in 1.51411s (22.4555GiB/s) | 20260505 04:21:44.968 | INFO | Analyzing chunk frames = [300032,300544) | 20260505 04:21:44.968 | INFO | MemInfo: { | 20260505 04:21:44.968 | INFO | "gpu" : " 17 GB", | 20260505 04:21:44.968 | INFO | "huge" : "", | 20260505 04:21:44.968 | INFO | "huge_shared" : " 27 GB", | 20260505 04:21:44.968 | INFO | "malloc" : " 14 GB", | 20260505 04:21:44.968 | INFO | "pinned" : " 29 GB" | 20260505 04:21:44.968 | INFO | } | 20260505 04:21:44.968 | INFO | Perf report for the last 5.11533 seconds: | 20260505 04:21:44.968 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:21:44.968 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:44.968 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:44.968 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:44.968 | INFO | ANALYSIS : 57.8%, 3.0, 89.4ms, 0 | 20260505 04:21:44.968 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:44.968 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:44.968 | INFO | PRE_HQ : 32.2%, 1.0, 16.6ms, 0 | 20260505 04:21:44.968 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:44.968 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:44.968 | INFO | Uploaded 36GiB to GPU in 1.60187s (22.4737GiB/s) | 20260505 04:21:44.968 | INFO | Downloaded 34GiB to GPU in 1.51865s (22.3883GiB/s) | 20260505 04:21:50.106 | INFO | Analyzing chunk frames = [300544,301056) | 20260505 04:21:50.106 | INFO | MemInfo: { | 20260505 04:21:50.106 | INFO | "gpu" : " 17 GB", | 20260505 04:21:50.106 | INFO | "huge" : "", | 20260505 04:21:50.106 | INFO | "huge_shared" : " 27 GB", | 20260505 04:21:50.106 | INFO | "malloc" : " 14 GB", | 20260505 04:21:50.106 | INFO | "pinned" : " 29 GB" | 20260505 04:21:50.106 | INFO | } | 20260505 04:21:50.106 | INFO | Perf report for the last 5.13836 seconds: | 20260505 04:21:50.106 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:21:50.106 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:50.106 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:50.106 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:50.106 | INFO | ANALYSIS : 57.3%, 3.0, 88.8ms, 0 | 20260505 04:21:50.106 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:50.106 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:50.106 | INFO | PRE_HQ : 31.9%, 1.0, 16.5ms, 0 | 20260505 04:21:50.106 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:50.106 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:50.106 | INFO | Uploaded 36GiB to GPU in 1.60142s (22.48GiB/s) | 20260505 04:21:50.107 | INFO | Downloaded 34GiB to GPU in 1.51855s (22.3897GiB/s) | 20260505 04:21:55.245 | INFO | Analyzing chunk frames = [301056,301568) | 20260505 04:21:55.245 | INFO | MemInfo: { | 20260505 04:21:55.245 | INFO | "gpu" : " 17 GB", | 20260505 04:21:55.245 | INFO | "huge" : "", | 20260505 04:21:55.245 | INFO | "huge_shared" : " 27 GB", | 20260505 04:21:55.245 | INFO | "malloc" : " 14 GB", | 20260505 04:21:55.245 | INFO | "pinned" : " 29 GB" | 20260505 04:21:55.245 | INFO | } | 20260505 04:21:55.246 | INFO | Perf report for the last 5.13931 seconds: | 20260505 04:21:55.246 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:21:55.246 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:55.246 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:55.246 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:55.246 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260505 04:21:55.246 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:55.246 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:55.246 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260505 04:21:55.246 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:55.246 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:21:55.246 | INFO | Uploaded 36GiB to GPU in 1.59893s (22.5151GiB/s) | 20260505 04:21:55.246 | INFO | Downloaded 34GiB to GPU in 1.51503s (22.4418GiB/s) | 20260505 04:22:00.384 | INFO | Analyzing chunk frames = [301568,302080) | 20260505 04:22:00.384 | INFO | MemInfo: { | 20260505 04:22:00.384 | INFO | "gpu" : " 17 GB", | 20260505 04:22:00.385 | INFO | "huge" : "", | 20260505 04:22:00.385 | INFO | "huge_shared" : " 27 GB", | 20260505 04:22:00.385 | INFO | "malloc" : " 14 GB", | 20260505 04:22:00.385 | INFO | "pinned" : " 29 GB" | 20260505 04:22:00.385 | INFO | } | 20260505 04:22:00.385 | INFO | Perf report for the last 5.13916 seconds: | 20260505 04:22:00.385 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:22:00.385 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:00.385 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:00.385 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:00.385 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260505 04:22:00.385 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:00.385 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:00.385 | INFO | PRE_HQ : 31.9%, 1.0, 16.5ms, 0 | 20260505 04:22:00.385 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:00.385 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:00.385 | INFO | Uploaded 36GiB to GPU in 1.59953s (22.5065GiB/s) | 20260505 04:22:00.385 | INFO | Downloaded 34GiB to GPU in 1.51629s (22.4232GiB/s) | 20260505 04:22:05.524 | INFO | Analyzing chunk frames = [302080,302592) | 20260505 04:22:05.524 | INFO | MemInfo: { | 20260505 04:22:05.524 | INFO | "gpu" : " 17 GB", | 20260505 04:22:05.524 | INFO | "huge" : "", | 20260505 04:22:05.524 | INFO | "huge_shared" : " 27 GB", | 20260505 04:22:05.524 | INFO | "malloc" : " 14 GB", | 20260505 04:22:05.524 | INFO | "pinned" : " 29 GB" | 20260505 04:22:05.524 | INFO | } | 20260505 04:22:05.524 | INFO | Perf report for the last 5.13949 seconds: | 20260505 04:22:05.524 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:22:05.524 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:05.524 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:05.524 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:05.524 | INFO | ANALYSIS : 57.7%, 3.0, 90.0ms, 0 | 20260505 04:22:05.524 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:05.524 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:05.524 | INFO | PRE_HQ : 31.9%, 1.0, 16.5ms, 0 | 20260505 04:22:05.524 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:05.524 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:05.525 | INFO | Uploaded 36GiB to GPU in 1.59943s (22.508GiB/s) | 20260505 04:22:05.525 | INFO | Downloaded 34GiB to GPU in 1.51646s (22.4206GiB/s) | 20260505 04:22:10.663 | INFO | Analyzing chunk frames = [302592,303104) | 20260505 04:22:10.663 | INFO | MemInfo: { | 20260505 04:22:10.663 | INFO | "gpu" : " 17 GB", | 20260505 04:22:10.663 | INFO | "huge" : "", | 20260505 04:22:10.663 | INFO | "huge_shared" : " 27 GB", | 20260505 04:22:10.663 | INFO | "malloc" : " 14 GB", | 20260505 04:22:10.663 | INFO | "pinned" : " 29 GB" | 20260505 04:22:10.663 | INFO | } | 20260505 04:22:10.663 | INFO | Perf report for the last 5.13923 seconds: | 20260505 04:22:10.663 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:22:10.663 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:10.663 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:10.663 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:10.664 | INFO | ANALYSIS : 57.8%, 3.0, 90.1ms, 0 | 20260505 04:22:10.664 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:10.664 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:10.664 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 04:22:10.664 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:10.664 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:10.664 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 04:22:10.664 | INFO | Total mem usage: 14.9 GiB | 20260505 04:22:10.664 | INFO | All Time High Water Marks sum: 15.2 GiB (Savings: 1.98%) | 20260505 04:22:10.664 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -3.03%) | 20260505 04:22:10.664 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 04:22:10.664 | INFO | High Water Breakdown: | 20260505 04:22:10.664 | INFO | 58.9% : 8.54 GiB ( 99.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 04:22:10.664 | INFO | 41% : 5.93 GiB ( 89.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 04:22:10.664 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 04:22:10.664 | INFO | ------------------------------------------ | 20260505 04:22:10.664 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:22:10.664 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 04:22:10.664 | INFO | Total mem usage: 26.5 GiB | 20260505 04:22:10.664 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260505 04:22:10.664 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 04:22:10.664 | INFO | Uploaded 36GiB to GPU in 1.59959s (22.5058GiB/s) | 20260505 04:22:10.664 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260505 04:22:10.664 | INFO | High Water Breakdown: | 20260505 04:22:10.664 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:22:10.664 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:22:10.664 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:22:10.664 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:22:10.664 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:22:10.664 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:22:10.664 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 04:22:10.664 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 04:22:10.664 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:22:10.664 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 04:22:10.664 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 04:22:10.664 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 04:22:10.664 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 04:22:10.664 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 04:22:10.664 | INFO | Downloaded 34GiB to GPU in 1.51682s (22.4154GiB/s) | 20260505 04:22:10.665 | INFO | ------------------------------------------ | 20260505 04:22:10.665 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:22:10.665 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 04:22:10.665 | INFO | Total mem usage: 25.6 GiB | 20260505 04:22:10.665 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 04:22:10.665 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 04:22:10.665 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 04:22:10.665 | INFO | High Water Breakdown: | 20260505 04:22:10.665 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 04:22:10.665 | INFO | ------------------------------------------ | 20260505 04:22:10.665 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:22:10.665 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 04:22:10.665 | INFO | Total mem usage: 15.7 GiB | 20260505 04:22:10.665 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260505 04:22:10.665 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 04:22:10.665 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260505 04:22:10.665 | INFO | High Water Breakdown: | 20260505 04:22:10.665 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 04:22:10.665 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 04:22:10.665 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 04:22:10.665 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 04:22:10.665 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 04:22:10.665 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 04:22:10.665 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 04:22:10.665 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 04:22:10.665 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 04:22:10.665 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:22:10.665 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 04:22:10.665 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:22:10.665 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 04:22:10.665 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 04:22:10.665 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 04:22:10.665 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 04:22:10.665 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:22:10.665 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 04:22:10.665 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:22:10.665 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:22:10.665 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:22:10.665 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 04:22:10.665 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 04:22:10.665 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:22:10.665 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 04:22:10.665 | INFO | ------------------------------------------ | 20260505 04:22:10.665 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:22:10.665 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 04:22:10.665 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:22:10.665 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:22:10.665 | INFO | PerfReport: Full Execution -- 3.05e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260505 04:22:10.665 | INFO | PerfReport: 0.28%, AggregateStats -- 8.63e-02 / -nan, 5.94e-03 / 0.00e+00 | 20260505 04:22:10.665 | INFO | PerfReport: 10.80%, Baseline -- 3.30e+00 / -nan, 1.82e-02 / 0.00e+00 | 20260505 04:22:10.665 | INFO | PerfReport: 3.03%, Histogram -- 9.24e-01 / -nan, 5.15e-02 / 0.00e+00 | 20260505 04:22:10.665 | INFO | PerfReport: 16.02%, DME -- 3.91e+01 / -nan, 3.22e+00 / 0.00e+00 | 20260505 04:22:10.665 | INFO | PerfReport: 56.38%, FrameLabeling -- 1.72e+01 / -nan, 4.60e-02 / 0.00e+00 | 20260505 04:22:10.665 | INFO | PerfReport: 7.71%, PulseAccumulating -- 2.35e+00 / -nan, 1.57e-01 / 0.00e+00 | 20260505 04:22:10.665 | INFO | PerfReport: 4.09%, Metrics -- 1.25e+00 / -nan, 2.05e-01 / 0.00e+00 | 20260505 04:22:10.665 | INFO | PerfReport: 1.70%, PulseSerializing -- 5.18e-01 / -nan, 8.34e-02 / 0.00e+00 | 20260505 04:22:10.665 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 04:22:10.665 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:22:10.665 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:22:10.665 | INFO | PerfReport: Full Execution -- 9.17e+01 / -nan, 1.89e+01 / 0.00e+00 | 20260505 04:22:10.665 | INFO | PerfReport: 18.23%, Upload -- 1.67e+01 / 0.00e+00, 3.66e-01 / 0.00e+00 | 20260505 04:22:10.665 | INFO | PerfReport: 17.49%, Download -- 1.60e+01 / 0.00e+00, 6.92e-01 / 0.00e+00 | 20260505 04:22:10.665 | INFO | PerfReport: 33.30%, Compute -- 3.05e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260505 04:22:10.665 | INFO | PerfReport: 0.55%, LockUpload -- 5.05e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260505 04:22:10.665 | INFO | PerfReport: 0.00%, LockDownload -- 3.97e-04 / 0.00e+00, 1.61e-04 / 0.00e+00 | 20260505 04:22:10.665 | INFO | PerfReport: 30.42%, LockCompute -- 2.79e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260505 04:22:10.665 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.56e-02 / 0.00e+00 | 20260505 04:22:10.761 | INFO | Starting a round of Baz IO | 20260505 04:22:13.720 | INFO | Populating RTMetrics report | 20260505 04:22:13.720 | INFO | RTMetrics report populated | 20260505 04:22:13.720 | INFO | Opening tmp RTMetrics json file | 20260505 04:22:13.722 | INFO | Serializing RTMetrics to json | 20260505 04:22:13.723 | INFO | Serialization done, swapping tmp file now... | 20260505 04:22:13.723 | INFO | RTMetrics done | 20260505 04:22:13.804 | INFO | Finished a round of Baz IO: Wrote 1.904 GiB in 3.042 seconds | 20260505 04:22:13.804 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 49.75% / 0.18% / 50.07% | 20260505 04:22:13.804 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 11.6707% / 0.000472864% / 3.57628e-05% | 20260505 04:22:13.804 | INFO | Baz IO overall: | 20260505 04:22:13.804 | INFO | BytesWritten : 110823526589 | 20260505 04:22:13.804 | INFO | NumEvents : 50927476686 | 20260505 04:22:15.803 | INFO | Analyzing chunk frames = [303104,303616) | 20260505 04:22:15.803 | INFO | MemInfo: { | 20260505 04:22:15.803 | INFO | "gpu" : " 17 GB", | 20260505 04:22:15.803 | INFO | "huge" : "", | 20260505 04:22:15.803 | INFO | "huge_shared" : " 27 GB", | 20260505 04:22:15.803 | INFO | "malloc" : " 14 GB", | 20260505 04:22:15.803 | INFO | "pinned" : " 29 GB" | 20260505 04:22:15.803 | INFO | } | 20260505 04:22:15.803 | INFO | Perf report for the last 5.13918 seconds: | 20260505 04:22:15.803 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:22:15.803 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:15.803 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:15.803 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:15.803 | INFO | ANALYSIS : 58.4%, 3.0, 90.9ms, 0 | 20260505 04:22:15.803 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:15.803 | INFO | RT_METRICS : 44.9%, 1.0, 23.5ms, 0 | 20260505 04:22:15.803 | INFO | PRE_HQ : 57.8%, 2.1, 64.5ms, 0 | 20260505 04:22:15.803 | INFO | BAZWRITER : 58.8%, 3.7, 112.9ms, 0 | 20260505 04:22:15.803 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:22:15.803 | INFO | Uploaded 36GiB to GPU in 1.63812s (21.9764GiB/s) | 20260505 04:22:15.803 | INFO | Downloaded 38GiB to GPU in 1.70028s (22.3493GiB/s) | 20260505 04:22:20.942 | INFO | Analyzing chunk frames = [303616,304128) | 20260505 04:22:20.942 | INFO | MemInfo: { | 20260505 04:22:20.942 | INFO | "gpu" : " 17 GB", | 20260505 04:22:20.942 | INFO | "huge" : "", | 20260505 04:22:20.942 | INFO | "huge_shared" : " 27 GB", | 20260505 04:22:20.942 | INFO | "malloc" : " 14 GB", | 20260505 04:22:20.942 | INFO | "pinned" : " 29 GB" | 20260505 04:22:20.942 | INFO | } | 20260505 04:22:20.942 | INFO | Perf report for the last 5.13946 seconds: | 20260505 04:22:20.942 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:22:20.942 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:20.942 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:20.942 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:20.942 | INFO | ANALYSIS : 58.2%, 3.0, 90.8ms, 0 | 20260505 04:22:20.942 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:20.942 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:20.942 | INFO | PRE_HQ : 31.7%, 1.0, 16.4ms, 0 | 20260505 04:22:20.942 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:20.942 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:20.942 | INFO | Uploaded 36GiB to GPU in 1.59777s (22.5314GiB/s) | 20260505 04:22:20.943 | INFO | Downloaded 34GiB to GPU in 1.51439s (22.4513GiB/s) | 20260505 04:22:26.081 | INFO | Analyzing chunk frames = [304128,304640) | 20260505 04:22:26.081 | INFO | MemInfo: { | 20260505 04:22:26.081 | INFO | "gpu" : " 17 GB", | 20260505 04:22:26.081 | INFO | "huge" : "", | 20260505 04:22:26.081 | INFO | "huge_shared" : " 27 GB", | 20260505 04:22:26.081 | INFO | "malloc" : " 14 GB", | 20260505 04:22:26.081 | INFO | "pinned" : " 29 GB" | 20260505 04:22:26.081 | INFO | } | 20260505 04:22:26.082 | INFO | Perf report for the last 5.13939 seconds: | 20260505 04:22:26.082 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:22:26.082 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:26.082 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:26.082 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:26.082 | INFO | ANALYSIS : 57.5%, 3.0, 89.4ms, 0 | 20260505 04:22:26.082 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:26.082 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:26.082 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 04:22:26.082 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:26.082 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:26.082 | INFO | Uploaded 36GiB to GPU in 1.6016s (22.4775GiB/s) | 20260505 04:22:26.082 | INFO | Downloaded 34GiB to GPU in 1.51894s (22.384GiB/s) | 20260505 04:22:31.220 | INFO | Analyzing chunk frames = [304640,305152) | 20260505 04:22:31.220 | INFO | MemInfo: { | 20260505 04:22:31.220 | INFO | "gpu" : " 17 GB", | 20260505 04:22:31.220 | INFO | "huge" : "", | 20260505 04:22:31.220 | INFO | "huge_shared" : " 27 GB", | 20260505 04:22:31.220 | INFO | "malloc" : " 14 GB", | 20260505 04:22:31.220 | INFO | "pinned" : " 29 GB" | 20260505 04:22:31.221 | INFO | } | 20260505 04:22:31.221 | INFO | Perf report for the last 5.13918 seconds: | 20260505 04:22:31.221 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:22:31.221 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:31.221 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:31.221 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:31.221 | INFO | ANALYSIS : 57.3%, 3.0, 88.8ms, 0 | 20260505 04:22:31.221 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:31.221 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:31.221 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 04:22:31.221 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:31.221 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:31.221 | INFO | Uploaded 36GiB to GPU in 1.60153s (22.4785GiB/s) | 20260505 04:22:31.221 | INFO | Downloaded 34GiB to GPU in 1.51823s (22.3945GiB/s) | 20260505 04:22:36.360 | INFO | Analyzing chunk frames = [305152,305664) | 20260505 04:22:36.360 | INFO | MemInfo: { | 20260505 04:22:36.360 | INFO | "gpu" : " 17 GB", | 20260505 04:22:36.360 | INFO | "huge" : "", | 20260505 04:22:36.360 | INFO | "huge_shared" : " 27 GB", | 20260505 04:22:36.360 | INFO | "malloc" : " 14 GB", | 20260505 04:22:36.360 | INFO | "pinned" : " 29 GB" | 20260505 04:22:36.360 | INFO | } | 20260505 04:22:36.360 | INFO | Perf report for the last 5.13942 seconds: | 20260505 04:22:36.360 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:22:36.360 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:36.360 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:36.360 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:36.360 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260505 04:22:36.360 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:36.360 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:36.360 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260505 04:22:36.360 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:36.360 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:36.360 | INFO | Uploaded 36GiB to GPU in 1.59826s (22.5244GiB/s) | 20260505 04:22:36.361 | INFO | Downloaded 34GiB to GPU in 1.51602s (22.4272GiB/s) | 20260505 04:22:41.499 | INFO | Analyzing chunk frames = [305664,306176) | 20260505 04:22:41.499 | INFO | MemInfo: { | 20260505 04:22:41.499 | INFO | "gpu" : " 17 GB", | 20260505 04:22:41.499 | INFO | "huge" : "", | 20260505 04:22:41.499 | INFO | "huge_shared" : " 27 GB", | 20260505 04:22:41.499 | INFO | "malloc" : " 14 GB", | 20260505 04:22:41.499 | INFO | "pinned" : " 29 GB" | 20260505 04:22:41.499 | INFO | } | 20260505 04:22:41.499 | INFO | Perf report for the last 5.13921 seconds: | 20260505 04:22:41.499 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:22:41.499 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:41.499 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:41.499 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:41.499 | INFO | ANALYSIS : 57.5%, 3.0, 89.6ms, 0 | 20260505 04:22:41.499 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:41.499 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:41.500 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260505 04:22:41.500 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:41.500 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:41.500 | INFO | Uploaded 36GiB to GPU in 1.59889s (22.5157GiB/s) | 20260505 04:22:41.500 | INFO | Downloaded 34GiB to GPU in 1.51655s (22.4193GiB/s) | 20260505 04:22:46.639 | INFO | Analyzing chunk frames = [306176,306688) | 20260505 04:22:46.639 | INFO | MemInfo: { | 20260505 04:22:46.639 | INFO | "gpu" : " 17 GB", | 20260505 04:22:46.639 | INFO | "huge" : "", | 20260505 04:22:46.639 | INFO | "huge_shared" : " 27 GB", | 20260505 04:22:46.639 | INFO | "malloc" : " 14 GB", | 20260505 04:22:46.639 | INFO | "pinned" : " 29 GB" | 20260505 04:22:46.639 | INFO | } | 20260505 04:22:46.639 | INFO | Perf report for the last 5.1393 seconds: | 20260505 04:22:46.639 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:22:46.639 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:46.639 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:46.639 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:46.639 | INFO | ANALYSIS : 57.6%, 3.0, 89.8ms, 0 | 20260505 04:22:46.639 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:46.639 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:46.639 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260505 04:22:46.639 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:46.639 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:46.639 | INFO | Uploaded 36GiB to GPU in 1.60521s (22.4269GiB/s) | 20260505 04:22:46.639 | INFO | Downloaded 34GiB to GPU in 1.51857s (22.3894GiB/s) | 20260505 04:22:51.778 | INFO | Analyzing chunk frames = [306688,307200) | 20260505 04:22:51.778 | INFO | MemInfo: { | 20260505 04:22:51.778 | INFO | "gpu" : " 17 GB", | 20260505 04:22:51.778 | INFO | "huge" : "", | 20260505 04:22:51.778 | INFO | "huge_shared" : " 27 GB", | 20260505 04:22:51.778 | INFO | "malloc" : " 14 GB", | 20260505 04:22:51.778 | INFO | "pinned" : " 29 GB" | 20260505 04:22:51.778 | INFO | } | 20260505 04:22:51.778 | INFO | Perf report for the last 5.13953 seconds: | 20260505 04:22:51.778 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:22:51.778 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:51.778 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:51.778 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:51.778 | INFO | ANALYSIS : 57.7%, 3.0, 90.1ms, 0 | 20260505 04:22:51.778 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:51.778 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:51.778 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 04:22:51.778 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:51.778 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:51.779 | INFO | Uploaded 36GiB to GPU in 1.59934s (22.5093GiB/s) | 20260505 04:22:51.779 | INFO | Downloaded 34GiB to GPU in 1.5169s (22.4141GiB/s) | 20260505 04:22:51.876 | INFO | Starting a round of Baz IO | 20260505 04:22:54.833 | INFO | Populating RTMetrics report | 20260505 04:22:54.834 | INFO | RTMetrics report populated | 20260505 04:22:54.834 | INFO | Opening tmp RTMetrics json file | 20260505 04:22:54.834 | INFO | Serializing RTMetrics to json | 20260505 04:22:54.835 | INFO | Serialization done, swapping tmp file now... | 20260505 04:22:54.836 | INFO | RTMetrics done | 20260505 04:22:54.923 | INFO | Finished a round of Baz IO: Wrote 1.907 GiB in 3.047 seconds | 20260505 04:22:54.923 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 49.58% / 0.17% / 50.25% | 20260505 04:22:54.923 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 11.5694% / 0.000445048% / 2.78155e-05% | 20260505 04:22:54.923 | INFO | Baz IO overall: | 20260505 04:22:54.923 | INFO | BytesWritten : 112871106264 | 20260505 04:22:54.923 | INFO | NumEvents : 51894214856 | 20260505 04:22:56.917 | INFO | Analyzing chunk frames = [307200,307712) | 20260505 04:22:56.917 | INFO | MemInfo: { | 20260505 04:22:56.917 | INFO | "gpu" : " 17 GB", | 20260505 04:22:56.917 | INFO | "huge" : "", | 20260505 04:22:56.917 | INFO | "huge_shared" : " 27 GB", | 20260505 04:22:56.917 | INFO | "malloc" : " 14 GB", | 20260505 04:22:56.917 | INFO | "pinned" : " 29 GB" | 20260505 04:22:56.917 | INFO | } | 20260505 04:22:56.917 | INFO | Perf report for the last 5.13925 seconds: | 20260505 04:22:56.917 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:22:56.917 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:56.917 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:56.917 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:56.918 | INFO | ANALYSIS : 58.4%, 3.0, 90.8ms, 0 | 20260505 04:22:56.918 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:22:56.918 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260505 04:22:56.918 | INFO | PRE_HQ : 57.7%, 2.2, 65.2ms, 0 | 20260505 04:22:56.918 | INFO | BAZWRITER : 58.9%, 3.7, 112.8ms, 0 | 20260505 04:22:56.918 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:22:56.918 | INFO | Uploaded 36GiB to GPU in 1.6421s (21.9231GiB/s) | 20260505 04:22:56.918 | INFO | Downloaded 38GiB to GPU in 1.70388s (22.3021GiB/s) | 20260505 04:23:02.057 | INFO | Analyzing chunk frames = [307712,308224) | 20260505 04:23:02.057 | INFO | MemInfo: { | 20260505 04:23:02.057 | INFO | "gpu" : " 17 GB", | 20260505 04:23:02.057 | INFO | "huge" : "", | 20260505 04:23:02.057 | INFO | "huge_shared" : " 27 GB", | 20260505 04:23:02.057 | INFO | "malloc" : " 14 GB", | 20260505 04:23:02.057 | INFO | "pinned" : " 29 GB" | 20260505 04:23:02.057 | INFO | } | 20260505 04:23:02.057 | INFO | Perf report for the last 5.13951 seconds: | 20260505 04:23:02.057 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:23:02.057 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:02.057 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:02.057 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:02.057 | INFO | ANALYSIS : 58.2%, 3.0, 90.8ms, 0 | 20260505 04:23:02.057 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:02.057 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:02.057 | INFO | PRE_HQ : 32.4%, 1.0, 17.0ms, 0 | 20260505 04:23:02.057 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:02.057 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:02.057 | INFO | Uploaded 36GiB to GPU in 1.59781s (22.5309GiB/s) | 20260505 04:23:02.057 | INFO | Downloaded 34GiB to GPU in 1.51447s (22.4501GiB/s) | 20260505 04:23:07.196 | INFO | Analyzing chunk frames = [308224,308736) | 20260505 04:23:07.196 | INFO | MemInfo: { | 20260505 04:23:07.196 | INFO | "gpu" : " 17 GB", | 20260505 04:23:07.196 | INFO | "huge" : "", | 20260505 04:23:07.196 | INFO | "huge_shared" : " 27 GB", | 20260505 04:23:07.196 | INFO | "malloc" : " 14 GB", | 20260505 04:23:07.196 | INFO | "pinned" : " 29 GB" | 20260505 04:23:07.196 | INFO | } | 20260505 04:23:07.196 | INFO | Perf report for the last 5.13912 seconds: | 20260505 04:23:07.196 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:23:07.196 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:07.196 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:07.196 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:07.196 | INFO | ANALYSIS : 57.5%, 3.0, 89.3ms, 0 | 20260505 04:23:07.196 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:07.196 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:07.196 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260505 04:23:07.196 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:07.196 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:07.196 | INFO | Uploaded 36GiB to GPU in 1.60197s (22.4723GiB/s) | 20260505 04:23:07.197 | INFO | Downloaded 34GiB to GPU in 1.51908s (22.382GiB/s) | 20260505 04:23:12.335 | INFO | Analyzing chunk frames = [308736,309248) | 20260505 04:23:12.335 | INFO | MemInfo: { | 20260505 04:23:12.335 | INFO | "gpu" : " 17 GB", | 20260505 04:23:12.335 | INFO | "huge" : "", | 20260505 04:23:12.335 | INFO | "huge_shared" : " 27 GB", | 20260505 04:23:12.335 | INFO | "malloc" : " 14 GB", | 20260505 04:23:12.335 | INFO | "pinned" : " 29 GB" | 20260505 04:23:12.336 | INFO | } | 20260505 04:23:12.336 | INFO | Perf report for the last 5.13956 seconds: | 20260505 04:23:12.336 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:23:12.336 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:12.336 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:12.336 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:12.336 | INFO | ANALYSIS : 57.2%, 3.0, 88.8ms, 0 | 20260505 04:23:12.336 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:12.336 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:12.336 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 04:23:12.336 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:12.336 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:12.336 | INFO | Uploaded 36GiB to GPU in 1.60105s (22.4852GiB/s) | 20260505 04:23:12.336 | INFO | Downloaded 34GiB to GPU in 1.51804s (22.3973GiB/s) | 20260505 04:23:17.475 | INFO | Analyzing chunk frames = [309248,309760) | 20260505 04:23:17.475 | INFO | MemInfo: { | 20260505 04:23:17.475 | INFO | "gpu" : " 17 GB", | 20260505 04:23:17.475 | INFO | "huge" : "", | 20260505 04:23:17.475 | INFO | "huge_shared" : " 27 GB", | 20260505 04:23:17.475 | INFO | "malloc" : " 14 GB", | 20260505 04:23:17.475 | INFO | "pinned" : " 29 GB" | 20260505 04:23:17.475 | INFO | } | 20260505 04:23:17.475 | INFO | Perf report for the last 5.13921 seconds: | 20260505 04:23:17.475 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:23:17.475 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:17.475 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:17.475 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:17.475 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260505 04:23:17.475 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:17.475 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:17.475 | INFO | PRE_HQ : 32.7%, 1.0, 17.0ms, 0 | 20260505 04:23:17.475 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:17.475 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:17.475 | INFO | Uploaded 36GiB to GPU in 1.60051s (22.4928GiB/s) | 20260505 04:23:17.476 | INFO | Downloaded 34GiB to GPU in 1.51663s (22.4181GiB/s) | 20260505 04:23:22.614 | INFO | Analyzing chunk frames = [309760,310272) | 20260505 04:23:22.614 | INFO | MemInfo: { | 20260505 04:23:22.614 | INFO | "gpu" : " 17 GB", | 20260505 04:23:22.614 | INFO | "huge" : "", | 20260505 04:23:22.614 | INFO | "huge_shared" : " 27 GB", | 20260505 04:23:22.614 | INFO | "malloc" : " 14 GB", | 20260505 04:23:22.614 | INFO | "pinned" : " 29 GB" | 20260505 04:23:22.614 | INFO | } | 20260505 04:23:22.614 | INFO | Perf report for the last 5.13944 seconds: | 20260505 04:23:22.614 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:23:22.614 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:22.614 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:22.614 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:22.614 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260505 04:23:22.614 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:22.614 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:22.614 | INFO | PRE_HQ : 32.6%, 1.0, 16.9ms, 0 | 20260505 04:23:22.614 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:22.615 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:22.615 | INFO | Uploaded 36GiB to GPU in 1.60007s (22.499GiB/s) | 20260505 04:23:22.615 | INFO | Downloaded 34GiB to GPU in 1.51611s (22.4258GiB/s) | 20260505 04:23:27.753 | INFO | Analyzing chunk frames = [310272,310784) | 20260505 04:23:27.753 | INFO | MemInfo: { | 20260505 04:23:27.753 | INFO | "gpu" : " 17 GB", | 20260505 04:23:27.753 | INFO | "huge" : "", | 20260505 04:23:27.753 | INFO | "huge_shared" : " 27 GB", | 20260505 04:23:27.754 | INFO | "malloc" : " 14 GB", | 20260505 04:23:27.754 | INFO | "pinned" : " 29 GB" | 20260505 04:23:27.754 | INFO | } | 20260505 04:23:27.754 | INFO | Perf report for the last 5.13928 seconds: | 20260505 04:23:27.754 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:23:27.754 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:27.754 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:27.754 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:27.754 | INFO | ANALYSIS : 57.6%, 3.0, 89.9ms, 0 | 20260505 04:23:27.754 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:27.754 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:27.754 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 04:23:27.754 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:27.754 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:27.754 | INFO | Uploaded 36GiB to GPU in 1.59997s (22.5004GiB/s) | 20260505 04:23:27.754 | INFO | Downloaded 34GiB to GPU in 1.51692s (22.4139GiB/s) | 20260505 04:23:32.893 | INFO | Analyzing chunk frames = [310784,311296) | 20260505 04:23:32.893 | INFO | MemInfo: { | 20260505 04:23:32.893 | INFO | "gpu" : " 17 GB", | 20260505 04:23:32.893 | INFO | "huge" : "", | 20260505 04:23:32.893 | INFO | "huge_shared" : " 27 GB", | 20260505 04:23:32.893 | INFO | "malloc" : " 14 GB", | 20260505 04:23:32.893 | INFO | "pinned" : " 29 GB" | 20260505 04:23:32.893 | INFO | } | 20260505 04:23:32.893 | INFO | Perf report for the last 5.13955 seconds: | 20260505 04:23:32.893 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:23:32.893 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:32.893 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:32.893 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:32.893 | INFO | ANALYSIS : 57.7%, 3.0, 90.1ms, 0 | 20260505 04:23:32.893 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:32.893 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:32.893 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260505 04:23:32.893 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:32.893 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:32.894 | INFO | Uploaded 36GiB to GPU in 1.60055s (22.4922GiB/s) | 20260505 04:23:32.894 | INFO | Downloaded 34GiB to GPU in 1.51752s (22.405GiB/s) | 20260505 04:23:32.894 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 04:23:32.894 | INFO | Total mem usage: 14.9 GiB | 20260505 04:23:32.894 | INFO | All Time High Water Marks sum: 15.2 GiB (Savings: 1.98%) | 20260505 04:23:32.894 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -2.91%) | 20260505 04:23:32.894 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 04:23:32.894 | INFO | High Water Breakdown: | 20260505 04:23:32.894 | INFO | 59% : 8.56 GiB ( 99.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 04:23:32.894 | INFO | 40.9% : 5.93 GiB ( 89.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 04:23:32.894 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 04:23:32.894 | INFO | ------------------------------------------ | 20260505 04:23:32.894 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:23:32.894 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 04:23:32.894 | INFO | Total mem usage: 26.5 GiB | 20260505 04:23:32.894 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260505 04:23:32.894 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 04:23:32.894 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260505 04:23:32.894 | INFO | High Water Breakdown: | 20260505 04:23:32.894 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:23:32.894 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:23:32.894 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:23:32.894 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:23:32.894 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:23:32.894 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:23:32.894 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 04:23:32.894 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 04:23:32.894 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:23:32.894 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 04:23:32.894 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 04:23:32.894 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 04:23:32.894 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 04:23:32.894 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 04:23:32.894 | INFO | ------------------------------------------ | 20260505 04:23:32.894 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:23:32.894 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 04:23:32.894 | INFO | Total mem usage: 25.6 GiB | 20260505 04:23:32.894 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 04:23:32.894 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 04:23:32.894 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 04:23:32.894 | INFO | High Water Breakdown: | 20260505 04:23:32.894 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 04:23:32.894 | INFO | ------------------------------------------ | 20260505 04:23:32.894 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:23:32.894 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 04:23:32.894 | INFO | Total mem usage: 15.7 GiB | 20260505 04:23:32.894 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260505 04:23:32.894 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 04:23:32.894 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260505 04:23:32.894 | INFO | High Water Breakdown: | 20260505 04:23:32.895 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 04:23:32.895 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 04:23:32.895 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 04:23:32.895 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 04:23:32.895 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 04:23:32.895 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 04:23:32.895 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 04:23:32.895 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 04:23:32.895 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 04:23:32.895 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:23:32.895 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 04:23:32.895 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:23:32.895 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 04:23:32.895 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 04:23:32.895 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 04:23:32.895 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 04:23:32.895 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:23:32.895 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 04:23:32.895 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:23:32.895 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:23:32.895 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:23:32.895 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 04:23:32.895 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 04:23:32.895 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:23:32.895 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 04:23:32.895 | INFO | ------------------------------------------ | 20260505 04:23:32.895 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:23:32.895 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 04:23:32.895 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:23:32.895 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:23:32.895 | INFO | PerfReport: Full Execution -- 3.05e+01 / -nan, 1.29e+01 / 0.00e+00 | 20260505 04:23:32.895 | INFO | PerfReport: 0.28%, AggregateStats -- 8.63e-02 / -nan, 4.41e-03 / 0.00e+00 | 20260505 04:23:32.895 | INFO | PerfReport: 10.80%, Baseline -- 3.30e+00 / -nan, 1.63e-02 / 0.00e+00 | 20260505 04:23:32.895 | INFO | PerfReport: 3.03%, Histogram -- 9.24e-01 / -nan, 5.26e-02 / 0.00e+00 | 20260505 04:23:32.895 | INFO | PerfReport: 16.02%, DME -- 3.91e+01 / -nan, 3.20e+00 / 0.00e+00 | 20260505 04:23:32.895 | INFO | PerfReport: 56.40%, FrameLabeling -- 1.72e+01 / -nan, 4.59e-02 / 0.00e+00 | 20260505 04:23:32.895 | INFO | PerfReport: 7.70%, PulseAccumulating -- 2.35e+00 / -nan, 1.58e-01 / 0.00e+00 | 20260505 04:23:32.895 | INFO | PerfReport: 4.08%, Metrics -- 1.24e+00 / -nan, 2.05e-01 / 0.00e+00 | 20260505 04:23:32.895 | INFO | PerfReport: 1.69%, PulseSerializing -- 5.17e-01 / -nan, 8.26e-02 / 0.00e+00 | 20260505 04:23:32.895 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 04:23:32.895 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:23:32.895 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:23:32.895 | INFO | PerfReport: Full Execution -- 9.17e+01 / -nan, 1.91e+01 / 0.00e+00 | 20260505 04:23:32.895 | INFO | PerfReport: 18.24%, Upload -- 1.67e+01 / 0.00e+00, 3.85e-01 / 0.00e+00 | 20260505 04:23:32.895 | INFO | PerfReport: 17.50%, Download -- 1.60e+01 / 0.00e+00, 6.94e-01 / 0.00e+00 | 20260505 04:23:32.895 | INFO | PerfReport: 33.30%, Compute -- 3.05e+01 / 0.00e+00, 1.29e+01 / 0.00e+00 | 20260505 04:23:32.895 | INFO | PerfReport: 0.55%, LockUpload -- 5.03e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260505 04:23:32.895 | INFO | PerfReport: 0.00%, LockDownload -- 3.99e-04 / 0.00e+00, 1.80e-04 / 0.00e+00 | 20260505 04:23:32.895 | INFO | PerfReport: 30.39%, LockCompute -- 2.79e+01 / 0.00e+00, 1.73e+01 / 0.00e+00 | 20260505 04:23:32.895 | INFO | PerfReport: 0.02%, Misc -- 1.39e-02 / 0.00e+00, 2.73e-02 / 0.00e+00 | 20260505 04:23:32.991 | INFO | Starting a round of Baz IO | 20260505 04:23:35.948 | INFO | Populating RTMetrics report | 20260505 04:23:35.949 | INFO | RTMetrics report populated | 20260505 04:23:35.949 | INFO | Opening tmp RTMetrics json file | 20260505 04:23:35.949 | INFO | Serializing RTMetrics to json | 20260505 04:23:35.950 | INFO | Serialization done, swapping tmp file now... | 20260505 04:23:35.951 | INFO | RTMetrics done | 20260505 04:23:36.035 | INFO | Finished a round of Baz IO: Wrote 1.911 GiB in 3.043 seconds | 20260505 04:23:36.035 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 49.41% / 0.17% / 50.42% | 20260505 04:23:36.035 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 11.4922% / 0.000445048% / 2.78155e-05% | 20260505 04:23:36.035 | INFO | Baz IO overall: | 20260505 04:23:36.035 | INFO | BytesWritten : 114922689756 | 20260505 04:23:36.035 | INFO | NumEvents : 52862944823 | 20260505 04:23:38.032 | INFO | Analyzing chunk frames = [311296,311808) | 20260505 04:23:38.032 | INFO | MemInfo: { | 20260505 04:23:38.032 | INFO | "gpu" : " 17 GB", | 20260505 04:23:38.032 | INFO | "huge" : "", | 20260505 04:23:38.032 | INFO | "huge_shared" : " 27 GB", | 20260505 04:23:38.032 | INFO | "malloc" : " 14 GB", | 20260505 04:23:38.032 | INFO | "pinned" : " 29 GB" | 20260505 04:23:38.032 | INFO | } | 20260505 04:23:38.032 | INFO | Perf report for the last 5.13892 seconds: | 20260505 04:23:38.032 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:23:38.032 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:38.032 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:38.032 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:38.032 | INFO | ANALYSIS : 58.3%, 3.0, 90.8ms, 0 | 20260505 04:23:38.032 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:38.032 | INFO | RT_METRICS : 45.1%, 1.0, 23.6ms, 0 | 20260505 04:23:38.032 | INFO | PRE_HQ : 57.9%, 2.2, 65.3ms, 0 | 20260505 04:23:38.032 | INFO | BAZWRITER : 58.8%, 3.7, 113.0ms, 0 | 20260505 04:23:38.032 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:23:38.033 | INFO | Uploaded 36GiB to GPU in 1.63594s (22.0057GiB/s) | 20260505 04:23:38.033 | INFO | Downloaded 38GiB to GPU in 1.69898s (22.3664GiB/s) | 20260505 04:23:43.171 | INFO | Analyzing chunk frames = [311808,312320) | 20260505 04:23:43.171 | INFO | MemInfo: { | 20260505 04:23:43.171 | INFO | "gpu" : " 17 GB", | 20260505 04:23:43.171 | INFO | "huge" : "", | 20260505 04:23:43.171 | INFO | "huge_shared" : " 27 GB", | 20260505 04:23:43.172 | INFO | "malloc" : " 14 GB", | 20260505 04:23:43.172 | INFO | "pinned" : " 29 GB" | 20260505 04:23:43.172 | INFO | } | 20260505 04:23:43.172 | INFO | Perf report for the last 5.13951 seconds: | 20260505 04:23:43.172 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:23:43.172 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:43.172 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:43.172 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:43.172 | INFO | ANALYSIS : 58.2%, 3.0, 90.8ms, 0 | 20260505 04:23:43.172 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:43.172 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:43.172 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 04:23:43.172 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:43.172 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:43.172 | INFO | Uploaded 36GiB to GPU in 1.59754s (22.5346GiB/s) | 20260505 04:23:43.172 | INFO | Downloaded 34GiB to GPU in 1.51358s (22.4632GiB/s) | 20260505 04:23:48.311 | INFO | Analyzing chunk frames = [312320,312832) | 20260505 04:23:48.311 | INFO | MemInfo: { | 20260505 04:23:48.311 | INFO | "gpu" : " 17 GB", | 20260505 04:23:48.311 | INFO | "huge" : "", | 20260505 04:23:48.311 | INFO | "huge_shared" : " 27 GB", | 20260505 04:23:48.311 | INFO | "malloc" : " 14 GB", | 20260505 04:23:48.311 | INFO | "pinned" : " 29 GB" | 20260505 04:23:48.311 | INFO | } | 20260505 04:23:48.311 | INFO | Perf report for the last 5.13935 seconds: | 20260505 04:23:48.311 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:23:48.311 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:48.311 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:48.311 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:48.311 | INFO | ANALYSIS : 57.5%, 3.0, 89.3ms, 0 | 20260505 04:23:48.311 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:48.311 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:48.311 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260505 04:23:48.311 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:48.311 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:48.311 | INFO | Uploaded 36GiB to GPU in 1.60219s (22.4692GiB/s) | 20260505 04:23:48.311 | INFO | Downloaded 34GiB to GPU in 1.51906s (22.3822GiB/s) | 20260505 04:23:53.450 | INFO | Analyzing chunk frames = [312832,313344) | 20260505 04:23:53.450 | INFO | MemInfo: { | 20260505 04:23:53.450 | INFO | "gpu" : " 17 GB", | 20260505 04:23:53.450 | INFO | "huge" : "", | 20260505 04:23:53.450 | INFO | "huge_shared" : " 27 GB", | 20260505 04:23:53.450 | INFO | "malloc" : " 14 GB", | 20260505 04:23:53.450 | INFO | "pinned" : " 29 GB" | 20260505 04:23:53.450 | INFO | } | 20260505 04:23:53.450 | INFO | Perf report for the last 5.13935 seconds: | 20260505 04:23:53.450 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:23:53.450 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:53.450 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:53.450 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:53.450 | INFO | ANALYSIS : 57.2%, 3.0, 88.7ms, 0 | 20260505 04:23:53.450 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:53.450 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:53.450 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260505 04:23:53.451 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:53.451 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:53.451 | INFO | Uploaded 36GiB to GPU in 1.60063s (22.4912GiB/s) | 20260505 04:23:53.451 | INFO | Downloaded 34GiB to GPU in 1.51657s (22.419GiB/s) | 20260505 04:23:58.589 | INFO | Analyzing chunk frames = [313344,313856) | 20260505 04:23:58.589 | INFO | MemInfo: { | 20260505 04:23:58.589 | INFO | "gpu" : " 17 GB", | 20260505 04:23:58.589 | INFO | "huge" : "", | 20260505 04:23:58.590 | INFO | "huge_shared" : " 27 GB", | 20260505 04:23:58.590 | INFO | "malloc" : " 14 GB", | 20260505 04:23:58.590 | INFO | "pinned" : " 29 GB" | 20260505 04:23:58.590 | INFO | } | 20260505 04:23:58.590 | INFO | Perf report for the last 5.13932 seconds: | 20260505 04:23:58.590 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:23:58.590 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:58.590 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:58.590 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:58.590 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260505 04:23:58.590 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:58.590 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:58.590 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 04:23:58.590 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:58.590 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:23:58.590 | INFO | Uploaded 36GiB to GPU in 1.59921s (22.5111GiB/s) | 20260505 04:23:58.590 | INFO | Downloaded 34GiB to GPU in 1.51542s (22.4361GiB/s) | 20260505 04:24:03.729 | INFO | Analyzing chunk frames = [313856,314368) | 20260505 04:24:03.729 | INFO | MemInfo: { | 20260505 04:24:03.729 | INFO | "gpu" : " 17 GB", | 20260505 04:24:03.729 | INFO | "huge" : "", | 20260505 04:24:03.729 | INFO | "huge_shared" : " 27 GB", | 20260505 04:24:03.729 | INFO | "malloc" : " 14 GB", | 20260505 04:24:03.729 | INFO | "pinned" : " 29 GB" | 20260505 04:24:03.729 | INFO | } | 20260505 04:24:03.729 | INFO | Perf report for the last 5.13949 seconds: | 20260505 04:24:03.729 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:24:03.729 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:03.729 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:03.729 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:03.729 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260505 04:24:03.729 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:03.729 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:03.729 | INFO | PRE_HQ : 32.7%, 1.0, 17.0ms, 0 | 20260505 04:24:03.729 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:03.729 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:03.730 | INFO | Uploaded 36GiB to GPU in 1.60061s (22.4914GiB/s) | 20260505 04:24:03.730 | INFO | Downloaded 34GiB to GPU in 1.51657s (22.419GiB/s) | 20260505 04:24:08.868 | INFO | Analyzing chunk frames = [314368,314880) | 20260505 04:24:08.868 | INFO | MemInfo: { | 20260505 04:24:08.868 | INFO | "gpu" : " 17 GB", | 20260505 04:24:08.868 | INFO | "huge" : "", | 20260505 04:24:08.868 | INFO | "huge_shared" : " 27 GB", | 20260505 04:24:08.868 | INFO | "malloc" : " 14 GB", | 20260505 04:24:08.868 | INFO | "pinned" : " 29 GB" | 20260505 04:24:08.868 | INFO | } | 20260505 04:24:08.868 | INFO | Perf report for the last 5.13917 seconds: | 20260505 04:24:08.868 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:24:08.868 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:08.868 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:08.868 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:08.868 | INFO | ANALYSIS : 57.7%, 3.0, 89.9ms, 0 | 20260505 04:24:08.868 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:08.868 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:08.869 | INFO | PRE_HQ : 33.0%, 1.0, 17.1ms, 0 | 20260505 04:24:08.869 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:08.869 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:08.869 | INFO | Uploaded 36GiB to GPU in 1.60022s (22.497GiB/s) | 20260505 04:24:08.869 | INFO | Downloaded 34GiB to GPU in 1.51686s (22.4148GiB/s) | 20260505 04:24:14.008 | INFO | Analyzing chunk frames = [314880,315392) | 20260505 04:24:14.008 | INFO | MemInfo: { | 20260505 04:24:14.008 | INFO | "gpu" : " 17 GB", | 20260505 04:24:14.008 | INFO | "huge" : "", | 20260505 04:24:14.008 | INFO | "huge_shared" : " 27 GB", | 20260505 04:24:14.008 | INFO | "malloc" : " 14 GB", | 20260505 04:24:14.008 | INFO | "pinned" : " 29 GB" | 20260505 04:24:14.008 | INFO | } | 20260505 04:24:14.008 | INFO | Perf report for the last 5.13946 seconds: | 20260505 04:24:14.008 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:24:14.008 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:14.008 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:14.008 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:14.008 | INFO | ANALYSIS : 57.7%, 3.0, 90.0ms, 0 | 20260505 04:24:14.008 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:14.008 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:14.008 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 04:24:14.008 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:14.008 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:14.008 | INFO | Uploaded 36GiB to GPU in 1.60029s (22.4959GiB/s) | 20260505 04:24:14.008 | INFO | Downloaded 34GiB to GPU in 1.51707s (22.4116GiB/s) | 20260505 04:24:14.105 | INFO | Starting a round of Baz IO | 20260505 04:24:17.066 | INFO | Populating RTMetrics report | 20260505 04:24:17.066 | INFO | RTMetrics report populated | 20260505 04:24:17.066 | INFO | Opening tmp RTMetrics json file | 20260505 04:24:17.067 | INFO | Serializing RTMetrics to json | 20260505 04:24:17.068 | INFO | Serialization done, swapping tmp file now... | 20260505 04:24:17.068 | INFO | RTMetrics done | 20260505 04:24:17.149 | INFO | Finished a round of Baz IO: Wrote 1.917 GiB in 3.044 seconds | 20260505 04:24:17.149 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 49.24% / 0.17% / 50.59% | 20260505 04:24:17.149 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 11.4447% / 0.00046889% / 5.96046e-05% | 20260505 04:24:17.149 | INFO | Baz IO overall: | 20260505 04:24:17.149 | INFO | BytesWritten : 116981501027 | 20260505 04:24:17.149 | INFO | NumEvents : 53835519572 | 20260505 04:24:19.147 | INFO | Analyzing chunk frames = [315392,315904) | 20260505 04:24:19.147 | INFO | MemInfo: { | 20260505 04:24:19.147 | INFO | "gpu" : " 17 GB", | 20260505 04:24:19.147 | INFO | "huge" : "", | 20260505 04:24:19.147 | INFO | "huge_shared" : " 27 GB", | 20260505 04:24:19.147 | INFO | "malloc" : " 14 GB", | 20260505 04:24:19.147 | INFO | "pinned" : " 29 GB" | 20260505 04:24:19.147 | INFO | } | 20260505 04:24:19.147 | INFO | Perf report for the last 5.1392 seconds: | 20260505 04:24:19.147 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:24:19.147 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:19.147 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:19.147 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:19.147 | INFO | ANALYSIS : 58.4%, 3.0, 90.9ms, 0 | 20260505 04:24:19.147 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:19.147 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260505 04:24:19.147 | INFO | PRE_HQ : 57.9%, 2.2, 65.2ms, 0 | 20260505 04:24:19.147 | INFO | BAZWRITER : 58.8%, 3.8, 113.8ms, 0 | 20260505 04:24:19.147 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:24:19.148 | INFO | Uploaded 36GiB to GPU in 1.63632s (22.0006GiB/s) | 20260505 04:24:19.148 | INFO | Downloaded 38GiB to GPU in 1.69929s (22.3622GiB/s) | 20260505 04:24:24.286 | INFO | Analyzing chunk frames = [315904,316416) | 20260505 04:24:24.286 | INFO | MemInfo: { | 20260505 04:24:24.286 | INFO | "gpu" : " 17 GB", | 20260505 04:24:24.286 | INFO | "huge" : "", | 20260505 04:24:24.286 | INFO | "huge_shared" : " 27 GB", | 20260505 04:24:24.286 | INFO | "malloc" : " 14 GB", | 20260505 04:24:24.286 | INFO | "pinned" : " 29 GB" | 20260505 04:24:24.286 | INFO | } | 20260505 04:24:24.286 | INFO | Perf report for the last 5.13933 seconds: | 20260505 04:24:24.286 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:24:24.286 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:24.286 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:24.286 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:24.286 | INFO | ANALYSIS : 58.1%, 3.0, 90.7ms, 0 | 20260505 04:24:24.286 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:24.287 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:24.287 | INFO | PRE_HQ : 32.3%, 1.0, 16.9ms, 0 | 20260505 04:24:24.287 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:24.287 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:24.287 | INFO | Uploaded 36GiB to GPU in 1.59837s (22.5229GiB/s) | 20260505 04:24:24.287 | INFO | Downloaded 34GiB to GPU in 1.51455s (22.4489GiB/s) | 20260505 04:24:29.426 | INFO | Analyzing chunk frames = [316416,316928) | 20260505 04:24:29.426 | INFO | MemInfo: { | 20260505 04:24:29.426 | INFO | "gpu" : " 17 GB", | 20260505 04:24:29.426 | INFO | "huge" : "", | 20260505 04:24:29.426 | INFO | "huge_shared" : " 27 GB", | 20260505 04:24:29.426 | INFO | "malloc" : " 14 GB", | 20260505 04:24:29.426 | INFO | "pinned" : " 29 GB" | 20260505 04:24:29.426 | INFO | } | 20260505 04:24:29.426 | INFO | Perf report for the last 5.13932 seconds: | 20260505 04:24:29.426 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:24:29.426 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:29.426 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:29.426 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:29.426 | INFO | ANALYSIS : 57.5%, 3.0, 89.4ms, 0 | 20260505 04:24:29.426 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:29.426 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:29.426 | INFO | PRE_HQ : 31.9%, 1.0, 16.5ms, 0 | 20260505 04:24:29.426 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:29.426 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:29.426 | INFO | Uploaded 36GiB to GPU in 1.60143s (22.4799GiB/s) | 20260505 04:24:29.426 | INFO | Downloaded 34GiB to GPU in 1.51854s (22.39GiB/s) | 20260505 04:24:34.565 | INFO | Analyzing chunk frames = [316928,317440) | 20260505 04:24:34.565 | INFO | MemInfo: { | 20260505 04:24:34.565 | INFO | "gpu" : " 17 GB", | 20260505 04:24:34.565 | INFO | "huge" : "", | 20260505 04:24:34.565 | INFO | "huge_shared" : " 27 GB", | 20260505 04:24:34.565 | INFO | "malloc" : " 14 GB", | 20260505 04:24:34.565 | INFO | "pinned" : " 29 GB" | 20260505 04:24:34.565 | INFO | } | 20260505 04:24:34.565 | INFO | Perf report for the last 5.13931 seconds: | 20260505 04:24:34.565 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:24:34.565 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:34.565 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:34.565 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:34.565 | INFO | ANALYSIS : 57.2%, 3.0, 88.7ms, 0 | 20260505 04:24:34.565 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:34.565 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:34.565 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260505 04:24:34.565 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:34.565 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:34.565 | INFO | Uploaded 36GiB to GPU in 1.60106s (22.4852GiB/s) | 20260505 04:24:34.565 | INFO | Downloaded 34GiB to GPU in 1.51781s (22.4008GiB/s) | 20260505 04:24:39.725 | INFO | Analyzing chunk frames = [317440,317952) | 20260505 04:24:39.725 | INFO | MemInfo: { | 20260505 04:24:39.725 | INFO | "gpu" : " 17 GB", | 20260505 04:24:39.725 | INFO | "huge" : "", | 20260505 04:24:39.726 | INFO | "huge_shared" : " 27 GB", | 20260505 04:24:39.726 | INFO | "malloc" : " 14 GB", | 20260505 04:24:39.726 | INFO | "pinned" : " 29 GB" | 20260505 04:24:39.726 | INFO | } | 20260505 04:24:39.726 | INFO | Perf report for the last 5.16054 seconds: | 20260505 04:24:39.726 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:24:39.726 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:39.726 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:39.726 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:39.726 | INFO | ANALYSIS : 56.9%, 3.0, 89.2ms, 0 | 20260505 04:24:39.726 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:39.726 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:39.726 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260505 04:24:39.726 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:39.726 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:39.726 | INFO | Uploaded 36GiB to GPU in 1.59933s (22.5095GiB/s) | 20260505 04:24:39.726 | INFO | Downloaded 34GiB to GPU in 1.51547s (22.4353GiB/s) | 20260505 04:24:44.845 | INFO | Analyzing chunk frames = [317952,318464) | 20260505 04:24:44.845 | INFO | MemInfo: { | 20260505 04:24:44.845 | INFO | "gpu" : " 17 GB", | 20260505 04:24:44.845 | INFO | "huge" : "", | 20260505 04:24:44.845 | INFO | "huge_shared" : " 27 GB", | 20260505 04:24:44.845 | INFO | "malloc" : " 14 GB", | 20260505 04:24:44.845 | INFO | "pinned" : " 29 GB" | 20260505 04:24:44.845 | INFO | } | 20260505 04:24:44.845 | INFO | Perf report for the last 5.11927 seconds: | 20260505 04:24:44.845 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:24:44.845 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:44.845 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:44.845 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:44.845 | INFO | ANALYSIS : 57.6%, 3.0, 89.5ms, 0 | 20260505 04:24:44.845 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:44.845 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:44.845 | INFO | PRE_HQ : 32.5%, 1.0, 16.8ms, 0 | 20260505 04:24:44.845 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:44.845 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:44.845 | INFO | Uploaded 36GiB to GPU in 1.60052s (22.4927GiB/s) | 20260505 04:24:44.845 | INFO | Downloaded 34GiB to GPU in 1.517s (22.4126GiB/s) | 20260505 04:24:49.983 | INFO | Analyzing chunk frames = [318464,318976) | 20260505 04:24:49.983 | INFO | MemInfo: { | 20260505 04:24:49.983 | INFO | "gpu" : " 17 GB", | 20260505 04:24:49.983 | INFO | "huge" : "", | 20260505 04:24:49.983 | INFO | "huge_shared" : " 27 GB", | 20260505 04:24:49.983 | INFO | "malloc" : " 14 GB", | 20260505 04:24:49.983 | INFO | "pinned" : " 29 GB" | 20260505 04:24:49.983 | INFO | } | 20260505 04:24:49.983 | INFO | Perf report for the last 5.13818 seconds: | 20260505 04:24:49.983 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:24:49.983 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:49.983 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:49.983 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:49.983 | INFO | ANALYSIS : 57.7%, 3.0, 89.9ms, 0 | 20260505 04:24:49.983 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:49.983 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:49.983 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260505 04:24:49.983 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:49.983 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:49.983 | INFO | Uploaded 36GiB to GPU in 1.60015s (22.4978GiB/s) | 20260505 04:24:49.983 | INFO | Downloaded 34GiB to GPU in 1.51626s (22.4236GiB/s) | 20260505 04:24:55.122 | INFO | Analyzing chunk frames = [318976,319488) | 20260505 04:24:55.122 | INFO | MemInfo: { | 20260505 04:24:55.122 | INFO | "gpu" : " 17 GB", | 20260505 04:24:55.122 | INFO | "huge" : "", | 20260505 04:24:55.122 | INFO | "huge_shared" : " 27 GB", | 20260505 04:24:55.122 | INFO | "malloc" : " 14 GB", | 20260505 04:24:55.122 | INFO | "pinned" : " 29 GB" | 20260505 04:24:55.122 | INFO | } | 20260505 04:24:55.122 | INFO | Perf report for the last 5.13932 seconds: | 20260505 04:24:55.122 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:24:55.122 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:55.122 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:55.122 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:55.122 | INFO | ANALYSIS : 57.7%, 3.0, 90.0ms, 0 | 20260505 04:24:55.122 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:55.123 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:55.123 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260505 04:24:55.123 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:55.123 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:24:55.123 | INFO | Uploaded 36GiB to GPU in 1.60095s (22.4867GiB/s) | 20260505 04:24:55.123 | INFO | Downloaded 34GiB to GPU in 1.5179s (22.3994GiB/s) | 20260505 04:24:55.123 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 04:24:55.123 | INFO | Total mem usage: 14.9 GiB | 20260505 04:24:55.123 | INFO | All Time High Water Marks sum: 15.2 GiB (Savings: 1.99%) | 20260505 04:24:55.123 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -2.88%) | 20260505 04:24:55.123 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 04:24:55.123 | INFO | High Water Breakdown: | 20260505 04:24:55.123 | INFO | 59% : 8.56 GiB ( 100% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 04:24:55.123 | INFO | 40.8% : 5.92 GiB ( 89.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 04:24:55.123 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 04:24:55.123 | INFO | ------------------------------------------ | 20260505 04:24:55.123 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:24:55.123 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 04:24:55.123 | INFO | Total mem usage: 26.5 GiB | 20260505 04:24:55.123 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260505 04:24:55.123 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 04:24:55.123 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260505 04:24:55.123 | INFO | High Water Breakdown: | 20260505 04:24:55.123 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:24:55.123 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:24:55.123 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:24:55.123 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:24:55.123 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:24:55.123 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:24:55.123 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 04:24:55.123 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 04:24:55.123 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:24:55.124 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 04:24:55.124 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 04:24:55.124 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 04:24:55.124 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 04:24:55.124 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 04:24:55.124 | INFO | ------------------------------------------ | 20260505 04:24:55.124 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:24:55.124 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 04:24:55.124 | INFO | Total mem usage: 25.6 GiB | 20260505 04:24:55.124 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 04:24:55.124 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 04:24:55.124 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 04:24:55.124 | INFO | High Water Breakdown: | 20260505 04:24:55.124 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 04:24:55.124 | INFO | ------------------------------------------ | 20260505 04:24:55.124 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:24:55.124 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 04:24:55.124 | INFO | Total mem usage: 15.7 GiB | 20260505 04:24:55.124 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260505 04:24:55.124 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 04:24:55.124 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260505 04:24:55.124 | INFO | High Water Breakdown: | 20260505 04:24:55.124 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 04:24:55.124 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 04:24:55.124 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 04:24:55.124 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 04:24:55.124 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 04:24:55.124 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 04:24:55.124 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 04:24:55.124 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 04:24:55.124 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 04:24:55.124 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:24:55.124 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 04:24:55.124 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:24:55.124 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 04:24:55.124 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 04:24:55.124 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 04:24:55.124 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 04:24:55.124 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:24:55.124 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 04:24:55.124 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:24:55.124 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:24:55.124 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:24:55.124 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 04:24:55.124 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 04:24:55.124 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:24:55.124 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 04:24:55.124 | INFO | ------------------------------------------ | 20260505 04:24:55.124 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:24:55.124 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 04:24:55.124 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:24:55.124 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:24:55.124 | INFO | PerfReport: Full Execution -- 3.05e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260505 04:24:55.124 | INFO | PerfReport: 0.28%, AggregateStats -- 8.65e-02 / -nan, 5.27e-03 / 0.00e+00 | 20260505 04:24:55.124 | INFO | PerfReport: 10.80%, Baseline -- 3.30e+00 / -nan, 1.86e-02 / 0.00e+00 | 20260505 04:24:55.124 | INFO | PerfReport: 3.03%, Histogram -- 9.25e-01 / -nan, 5.22e-02 / 0.00e+00 | 20260505 04:24:55.124 | INFO | PerfReport: 16.02%, DME -- 3.91e+01 / -nan, 3.20e+00 / 0.00e+00 | 20260505 04:24:55.124 | INFO | PerfReport: 56.41%, FrameLabeling -- 1.72e+01 / -nan, 4.50e-02 / 0.00e+00 | 20260505 04:24:55.124 | INFO | PerfReport: 7.70%, PulseAccumulating -- 2.35e+00 / -nan, 1.58e-01 / 0.00e+00 | 20260505 04:24:55.124 | INFO | PerfReport: 4.07%, Metrics -- 1.24e+00 / -nan, 2.04e-01 / 0.00e+00 | 20260505 04:24:55.124 | INFO | PerfReport: 1.69%, PulseSerializing -- 5.16e-01 / -nan, 8.32e-02 / 0.00e+00 | 20260505 04:24:55.124 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 04:24:55.124 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:24:55.124 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:24:55.124 | INFO | PerfReport: Full Execution -- 9.16e+01 / -nan, 1.91e+01 / 0.00e+00 | 20260505 04:24:55.124 | INFO | PerfReport: 18.24%, Upload -- 1.67e+01 / 0.00e+00, 3.60e-01 / 0.00e+00 | 20260505 04:24:55.124 | INFO | PerfReport: 17.50%, Download -- 1.60e+01 / 0.00e+00, 6.89e-01 / 0.00e+00 | 20260505 04:24:55.124 | INFO | PerfReport: 33.30%, Compute -- 3.05e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260505 04:24:55.124 | INFO | PerfReport: 0.55%, LockUpload -- 5.02e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260505 04:24:55.124 | INFO | PerfReport: 0.00%, LockDownload -- 3.92e-04 / 0.00e+00, 9.44e-05 / 0.00e+00 | 20260505 04:24:55.124 | INFO | PerfReport: 30.40%, LockCompute -- 2.79e+01 / 0.00e+00, 1.73e+01 / 0.00e+00 | 20260505 04:24:55.124 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.06e-02 / 0.00e+00 | 20260505 04:24:55.219 | INFO | Starting a round of Baz IO | 20260505 04:24:58.179 | INFO | Populating RTMetrics report | 20260505 04:24:58.179 | INFO | RTMetrics report populated | 20260505 04:24:58.179 | INFO | Opening tmp RTMetrics json file | 20260505 04:24:58.180 | INFO | Serializing RTMetrics to json | 20260505 04:24:58.181 | INFO | Serialization done, swapping tmp file now... | 20260505 04:24:58.182 | INFO | RTMetrics done | 20260505 04:24:58.272 | INFO | Finished a round of Baz IO: Wrote 1.924 GiB in 3.052 seconds | 20260505 04:24:58.272 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 49.08% / 0.16% / 50.76% | 20260505 04:24:58.272 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 11.4066% / 0.000413259% / 4.76837e-05% | 20260505 04:24:58.272 | INFO | Baz IO overall: | 20260505 04:24:58.272 | INFO | BytesWritten : 119047235604 | 20260505 04:24:58.272 | INFO | NumEvents : 54811638492 | 20260505 04:25:00.262 | INFO | Analyzing chunk frames = [319488,320000) | 20260505 04:25:00.262 | INFO | MemInfo: { | 20260505 04:25:00.262 | INFO | "gpu" : " 17 GB", | 20260505 04:25:00.262 | INFO | "huge" : "", | 20260505 04:25:00.262 | INFO | "huge_shared" : " 27 GB", | 20260505 04:25:00.262 | INFO | "malloc" : " 14 GB", | 20260505 04:25:00.262 | INFO | "pinned" : " 29 GB" | 20260505 04:25:00.262 | INFO | } | 20260505 04:25:00.262 | INFO | Perf report for the last 5.13926 seconds: | 20260505 04:25:00.262 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:25:00.262 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:00.262 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:00.262 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:00.262 | INFO | ANALYSIS : 58.4%, 3.0, 90.8ms, 0 | 20260505 04:25:00.262 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:00.262 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260505 04:25:00.262 | INFO | PRE_HQ : 57.9%, 2.2, 64.9ms, 0 | 20260505 04:25:00.262 | INFO | BAZWRITER : 58.9%, 3.8, 114.1ms, 0 | 20260505 04:25:00.262 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:25:00.262 | INFO | Uploaded 36GiB to GPU in 1.63893s (21.9655GiB/s) | 20260505 04:25:00.262 | INFO | Downloaded 38GiB to GPU in 1.69952s (22.3592GiB/s) | 20260505 04:25:05.401 | INFO | Analyzing chunk frames = [320000,320512) | 20260505 04:25:05.401 | INFO | MemInfo: { | 20260505 04:25:05.401 | INFO | "gpu" : " 17 GB", | 20260505 04:25:05.401 | INFO | "huge" : "", | 20260505 04:25:05.401 | INFO | "huge_shared" : " 27 GB", | 20260505 04:25:05.401 | INFO | "malloc" : " 14 GB", | 20260505 04:25:05.401 | INFO | "pinned" : " 29 GB" | 20260505 04:25:05.401 | INFO | } | 20260505 04:25:05.401 | INFO | Perf report for the last 5.13953 seconds: | 20260505 04:25:05.401 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:25:05.401 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:05.401 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:05.401 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:05.401 | INFO | ANALYSIS : 58.1%, 3.0, 90.6ms, 0 | 20260505 04:25:05.401 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:05.401 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:05.401 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260505 04:25:05.401 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:05.401 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:05.402 | INFO | Uploaded 36GiB to GPU in 1.59816s (22.5259GiB/s) | 20260505 04:25:05.402 | INFO | Downloaded 34GiB to GPU in 1.51437s (22.4516GiB/s) | 20260505 04:25:10.540 | INFO | Analyzing chunk frames = [320512,321024) | 20260505 04:25:10.540 | INFO | MemInfo: { | 20260505 04:25:10.540 | INFO | "gpu" : " 17 GB", | 20260505 04:25:10.540 | INFO | "huge" : "", | 20260505 04:25:10.540 | INFO | "huge_shared" : " 27 GB", | 20260505 04:25:10.540 | INFO | "malloc" : " 14 GB", | 20260505 04:25:10.540 | INFO | "pinned" : " 29 GB" | 20260505 04:25:10.541 | INFO | } | 20260505 04:25:10.541 | INFO | Perf report for the last 5.13931 seconds: | 20260505 04:25:10.541 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:25:10.541 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:10.541 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:10.541 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:10.541 | INFO | ANALYSIS : 57.5%, 3.0, 89.4ms, 0 | 20260505 04:25:10.541 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:10.541 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:10.541 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260505 04:25:10.541 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:10.541 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:10.541 | INFO | Uploaded 36GiB to GPU in 1.60155s (22.4782GiB/s) | 20260505 04:25:10.541 | INFO | Downloaded 34GiB to GPU in 1.51865s (22.3883GiB/s) | 20260505 04:25:15.680 | INFO | Analyzing chunk frames = [321024,321536) | 20260505 04:25:15.680 | INFO | MemInfo: { | 20260505 04:25:15.680 | INFO | "gpu" : " 17 GB", | 20260505 04:25:15.680 | INFO | "huge" : "", | 20260505 04:25:15.680 | INFO | "huge_shared" : " 27 GB", | 20260505 04:25:15.680 | INFO | "malloc" : " 14 GB", | 20260505 04:25:15.680 | INFO | "pinned" : " 29 GB" | 20260505 04:25:15.680 | INFO | } | 20260505 04:25:15.680 | INFO | Perf report for the last 5.13925 seconds: | 20260505 04:25:15.680 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:25:15.680 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:15.680 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:15.680 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:15.680 | INFO | ANALYSIS : 57.2%, 3.0, 88.7ms, 0 | 20260505 04:25:15.680 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:15.680 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:15.680 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 04:25:15.680 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:15.680 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:15.681 | INFO | Uploaded 36GiB to GPU in 1.60136s (22.4809GiB/s) | 20260505 04:25:15.681 | INFO | Downloaded 34GiB to GPU in 1.51808s (22.3967GiB/s) | 20260505 04:25:20.819 | INFO | Analyzing chunk frames = [321536,322048) | 20260505 04:25:20.819 | INFO | MemInfo: { | 20260505 04:25:20.819 | INFO | "gpu" : " 17 GB", | 20260505 04:25:20.819 | INFO | "huge" : "", | 20260505 04:25:20.819 | INFO | "huge_shared" : " 27 GB", | 20260505 04:25:20.819 | INFO | "malloc" : " 14 GB", | 20260505 04:25:20.819 | INFO | "pinned" : " 29 GB" | 20260505 04:25:20.819 | INFO | } | 20260505 04:25:20.819 | INFO | Perf report for the last 5.1395 seconds: | 20260505 04:25:20.819 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:25:20.819 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:20.819 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:20.819 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:20.819 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260505 04:25:20.819 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:20.819 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:20.820 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260505 04:25:20.820 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:20.820 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:20.820 | INFO | Uploaded 36GiB to GPU in 1.59957s (22.5061GiB/s) | 20260505 04:25:20.820 | INFO | Downloaded 34GiB to GPU in 1.51576s (22.431GiB/s) | 20260505 04:25:25.958 | INFO | Analyzing chunk frames = [322048,322560) | 20260505 04:25:25.958 | INFO | MemInfo: { | 20260505 04:25:25.958 | INFO | "gpu" : " 17 GB", | 20260505 04:25:25.958 | INFO | "huge" : "", | 20260505 04:25:25.958 | INFO | "huge_shared" : " 27 GB", | 20260505 04:25:25.959 | INFO | "malloc" : " 14 GB", | 20260505 04:25:25.959 | INFO | "pinned" : " 29 GB" | 20260505 04:25:25.959 | INFO | } | 20260505 04:25:25.959 | INFO | Perf report for the last 5.13951 seconds: | 20260505 04:25:25.959 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:25:25.959 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:25.959 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:25.959 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:25.959 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260505 04:25:25.959 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:25.959 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:25.959 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 04:25:25.959 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:25.959 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:25.959 | INFO | Uploaded 36GiB to GPU in 1.59966s (22.5049GiB/s) | 20260505 04:25:25.960 | INFO | Downloaded 34GiB to GPU in 1.51622s (22.4242GiB/s) | 20260505 04:25:31.098 | INFO | Analyzing chunk frames = [322560,323072) | 20260505 04:25:31.098 | INFO | MemInfo: { | 20260505 04:25:31.098 | INFO | "gpu" : " 17 GB", | 20260505 04:25:31.098 | INFO | "huge" : "", | 20260505 04:25:31.098 | INFO | "huge_shared" : " 27 GB", | 20260505 04:25:31.098 | INFO | "malloc" : " 14 GB", | 20260505 04:25:31.098 | INFO | "pinned" : " 29 GB" | 20260505 04:25:31.098 | INFO | } | 20260505 04:25:31.098 | INFO | Perf report for the last 5.13906 seconds: | 20260505 04:25:31.098 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:25:31.098 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:31.098 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:31.098 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:31.098 | INFO | ANALYSIS : 57.7%, 3.0, 90.0ms, 0 | 20260505 04:25:31.098 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:31.098 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:31.098 | INFO | PRE_HQ : 32.6%, 1.0, 16.9ms, 0 | 20260505 04:25:31.098 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:31.098 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:31.098 | INFO | Uploaded 36GiB to GPU in 1.6002s (22.4972GiB/s) | 20260505 04:25:31.098 | INFO | Downloaded 34GiB to GPU in 1.51756s (22.4044GiB/s) | 20260505 04:25:36.237 | INFO | Analyzing chunk frames = [323072,323584) | 20260505 04:25:36.237 | INFO | MemInfo: { | 20260505 04:25:36.237 | INFO | "gpu" : " 17 GB", | 20260505 04:25:36.237 | INFO | "huge" : "", | 20260505 04:25:36.237 | INFO | "huge_shared" : " 27 GB", | 20260505 04:25:36.237 | INFO | "malloc" : " 14 GB", | 20260505 04:25:36.237 | INFO | "pinned" : " 29 GB" | 20260505 04:25:36.237 | INFO | } | 20260505 04:25:36.237 | INFO | Perf report for the last 5.13939 seconds: | 20260505 04:25:36.237 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:25:36.237 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:36.237 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:36.237 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:36.237 | INFO | ANALYSIS : 57.7%, 3.0, 90.1ms, 0 | 20260505 04:25:36.237 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:36.237 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:36.237 | INFO | PRE_HQ : 31.7%, 1.0, 16.4ms, 0 | 20260505 04:25:36.237 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:36.237 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:36.238 | INFO | Uploaded 36GiB to GPU in 1.59905s (22.5134GiB/s) | 20260505 04:25:36.238 | INFO | Downloaded 34GiB to GPU in 1.51657s (22.419GiB/s) | 20260505 04:25:36.334 | INFO | Starting a round of Baz IO | 20260505 04:25:39.291 | INFO | Populating RTMetrics report | 20260505 04:25:39.292 | INFO | RTMetrics report populated | 20260505 04:25:39.292 | INFO | Opening tmp RTMetrics json file | 20260505 04:25:39.292 | INFO | Serializing RTMetrics to json | 20260505 04:25:39.296 | INFO | Serialization done, swapping tmp file now... | 20260505 04:25:39.297 | INFO | RTMetrics done | 20260505 04:25:39.383 | INFO | Finished a round of Baz IO: Wrote 1.929 GiB in 3.048 seconds | 20260505 04:25:39.383 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.92% / 0.16% / 50.92% | 20260505 04:25:39.383 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 11.3713% / 0.000433127% / 5.16574e-05% | 20260505 04:25:39.383 | INFO | Baz IO overall: | 20260505 04:25:39.383 | INFO | BytesWritten : 121118539111 | 20260505 04:25:39.383 | INFO | NumEvents : 55791055912 | 20260505 04:25:41.376 | INFO | Analyzing chunk frames = [323584,324096) | 20260505 04:25:41.376 | INFO | MemInfo: { | 20260505 04:25:41.376 | INFO | "gpu" : " 17 GB", | 20260505 04:25:41.376 | INFO | "huge" : "", | 20260505 04:25:41.376 | INFO | "huge_shared" : " 27 GB", | 20260505 04:25:41.376 | INFO | "malloc" : " 14 GB", | 20260505 04:25:41.376 | INFO | "pinned" : " 29 GB" | 20260505 04:25:41.376 | INFO | } | 20260505 04:25:41.377 | INFO | Perf report for the last 5.13923 seconds: | 20260505 04:25:41.377 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:25:41.377 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:41.377 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:41.377 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:41.377 | INFO | ANALYSIS : 58.3%, 3.0, 90.8ms, 0 | 20260505 04:25:41.377 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:41.377 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260505 04:25:41.377 | INFO | PRE_HQ : 57.8%, 2.2, 64.7ms, 0 | 20260505 04:25:41.377 | INFO | BAZWRITER : 58.9%, 3.8, 113.7ms, 0 | 20260505 04:25:41.377 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:25:41.377 | INFO | Uploaded 36GiB to GPU in 1.63929s (21.9608GiB/s) | 20260505 04:25:41.377 | INFO | Downloaded 38GiB to GPU in 1.70133s (22.3355GiB/s) | 20260505 04:25:46.516 | INFO | Analyzing chunk frames = [324096,324608) | 20260505 04:25:46.516 | INFO | MemInfo: { | 20260505 04:25:46.516 | INFO | "gpu" : " 17 GB", | 20260505 04:25:46.516 | INFO | "huge" : "", | 20260505 04:25:46.516 | INFO | "huge_shared" : " 27 GB", | 20260505 04:25:46.516 | INFO | "malloc" : " 14 GB", | 20260505 04:25:46.516 | INFO | "pinned" : " 29 GB" | 20260505 04:25:46.516 | INFO | } | 20260505 04:25:46.516 | INFO | Perf report for the last 5.13957 seconds: | 20260505 04:25:46.516 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:25:46.516 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:46.516 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:46.516 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:46.516 | INFO | ANALYSIS : 58.2%, 3.0, 90.7ms, 0 | 20260505 04:25:46.516 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:46.516 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:46.516 | INFO | PRE_HQ : 32.0%, 1.0, 16.7ms, 0 | 20260505 04:25:46.516 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:46.516 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:46.516 | INFO | Uploaded 36GiB to GPU in 1.59823s (22.5249GiB/s) | 20260505 04:25:46.516 | INFO | Downloaded 34GiB to GPU in 1.51527s (22.4382GiB/s) | 20260505 04:25:51.655 | INFO | Analyzing chunk frames = [324608,325120) | 20260505 04:25:51.655 | INFO | MemInfo: { | 20260505 04:25:51.655 | INFO | "gpu" : " 17 GB", | 20260505 04:25:51.655 | INFO | "huge" : "", | 20260505 04:25:51.655 | INFO | "huge_shared" : " 27 GB", | 20260505 04:25:51.655 | INFO | "malloc" : " 14 GB", | 20260505 04:25:51.655 | INFO | "pinned" : " 29 GB" | 20260505 04:25:51.655 | INFO | } | 20260505 04:25:51.655 | INFO | Perf report for the last 5.13928 seconds: | 20260505 04:25:51.655 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:25:51.655 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:51.655 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:51.655 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:51.656 | INFO | ANALYSIS : 57.5%, 3.0, 89.3ms, 0 | 20260505 04:25:51.656 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:51.656 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:51.656 | INFO | PRE_HQ : 31.9%, 1.0, 16.5ms, 0 | 20260505 04:25:51.656 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:51.656 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:51.656 | INFO | Uploaded 36GiB to GPU in 1.60103s (22.4855GiB/s) | 20260505 04:25:51.656 | INFO | Downloaded 34GiB to GPU in 1.51909s (22.3818GiB/s) | 20260505 04:25:56.794 | INFO | Analyzing chunk frames = [325120,325632) | 20260505 04:25:56.794 | INFO | MemInfo: { | 20260505 04:25:56.795 | INFO | "gpu" : " 17 GB", | 20260505 04:25:56.795 | INFO | "huge" : "", | 20260505 04:25:56.795 | INFO | "huge_shared" : " 27 GB", | 20260505 04:25:56.795 | INFO | "malloc" : " 14 GB", | 20260505 04:25:56.795 | INFO | "pinned" : " 29 GB" | 20260505 04:25:56.795 | INFO | } | 20260505 04:25:56.795 | INFO | Perf report for the last 5.13925 seconds: | 20260505 04:25:56.795 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:25:56.795 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:56.795 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:56.795 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:56.795 | INFO | ANALYSIS : 57.2%, 3.0, 88.7ms, 0 | 20260505 04:25:56.795 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:56.795 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:56.795 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 04:25:56.795 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:56.795 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:25:56.795 | INFO | Uploaded 36GiB to GPU in 1.60127s (22.4821GiB/s) | 20260505 04:25:56.795 | INFO | Downloaded 34GiB to GPU in 1.51822s (22.3947GiB/s) | 20260505 04:26:01.934 | INFO | Analyzing chunk frames = [325632,326144) | 20260505 04:26:01.934 | INFO | MemInfo: { | 20260505 04:26:01.934 | INFO | "gpu" : " 17 GB", | 20260505 04:26:01.934 | INFO | "huge" : "", | 20260505 04:26:01.934 | INFO | "huge_shared" : " 27 GB", | 20260505 04:26:01.934 | INFO | "malloc" : " 14 GB", | 20260505 04:26:01.934 | INFO | "pinned" : " 29 GB" | 20260505 04:26:01.934 | INFO | } | 20260505 04:26:01.934 | INFO | Perf report for the last 5.13925 seconds: | 20260505 04:26:01.934 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:26:01.934 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:01.934 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:01.934 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:01.934 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260505 04:26:01.934 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:01.934 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:01.934 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260505 04:26:01.934 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:01.934 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:01.934 | INFO | Uploaded 36GiB to GPU in 1.60015s (22.4978GiB/s) | 20260505 04:26:01.934 | INFO | Downloaded 34GiB to GPU in 1.51629s (22.4231GiB/s) | 20260505 04:26:07.073 | INFO | Analyzing chunk frames = [326144,326656) | 20260505 04:26:07.073 | INFO | MemInfo: { | 20260505 04:26:07.073 | INFO | "gpu" : " 17 GB", | 20260505 04:26:07.073 | INFO | "huge" : "", | 20260505 04:26:07.073 | INFO | "huge_shared" : " 27 GB", | 20260505 04:26:07.073 | INFO | "malloc" : " 14 GB", | 20260505 04:26:07.073 | INFO | "pinned" : " 29 GB" | 20260505 04:26:07.073 | INFO | } | 20260505 04:26:07.073 | INFO | Perf report for the last 5.1393 seconds: | 20260505 04:26:07.073 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:26:07.073 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:07.073 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:07.073 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:07.073 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260505 04:26:07.073 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:07.073 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:07.073 | INFO | PRE_HQ : 31.9%, 1.0, 16.5ms, 0 | 20260505 04:26:07.073 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:07.073 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:07.074 | INFO | Uploaded 36GiB to GPU in 1.59921s (22.5111GiB/s) | 20260505 04:26:07.074 | INFO | Downloaded 34GiB to GPU in 1.51593s (22.4285GiB/s) | 20260505 04:26:12.212 | INFO | Analyzing chunk frames = [326656,327168) | 20260505 04:26:12.212 | INFO | MemInfo: { | 20260505 04:26:12.212 | INFO | "gpu" : " 17 GB", | 20260505 04:26:12.212 | INFO | "huge" : "", | 20260505 04:26:12.213 | INFO | "huge_shared" : " 27 GB", | 20260505 04:26:12.213 | INFO | "malloc" : " 14 GB", | 20260505 04:26:12.213 | INFO | "pinned" : " 29 GB" | 20260505 04:26:12.213 | INFO | } | 20260505 04:26:12.213 | INFO | Perf report for the last 5.1394 seconds: | 20260505 04:26:12.213 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:26:12.213 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:12.213 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:12.213 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:12.213 | INFO | ANALYSIS : 57.6%, 3.0, 89.8ms, 0 | 20260505 04:26:12.213 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:12.213 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:12.213 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260505 04:26:12.213 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:12.213 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:12.213 | INFO | Uploaded 36GiB to GPU in 1.59912s (22.5123GiB/s) | 20260505 04:26:12.213 | INFO | Downloaded 34GiB to GPU in 1.51775s (22.4016GiB/s) | 20260505 04:26:17.352 | INFO | Analyzing chunk frames = [327168,327680) | 20260505 04:26:17.352 | INFO | MemInfo: { | 20260505 04:26:17.352 | INFO | "gpu" : " 17 GB", | 20260505 04:26:17.352 | INFO | "huge" : "", | 20260505 04:26:17.352 | INFO | "huge_shared" : " 27 GB", | 20260505 04:26:17.352 | INFO | "malloc" : " 14 GB", | 20260505 04:26:17.352 | INFO | "pinned" : " 29 GB" | 20260505 04:26:17.352 | INFO | } | 20260505 04:26:17.352 | INFO | Perf report for the last 5.13929 seconds: | 20260505 04:26:17.352 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:26:17.352 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:17.352 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:17.352 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:17.352 | INFO | ANALYSIS : 57.7%, 3.0, 90.1ms, 0 | 20260505 04:26:17.352 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:17.352 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:17.352 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260505 04:26:17.352 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:17.352 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:17.352 | INFO | Uploaded 36GiB to GPU in 1.6008s (22.4888GiB/s) | 20260505 04:26:17.352 | INFO | Downloaded 34GiB to GPU in 1.51778s (22.4011GiB/s) | 20260505 04:26:17.352 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 04:26:17.353 | INFO | Total mem usage: 14.9 GiB | 20260505 04:26:17.353 | INFO | All Time High Water Marks sum: 15.2 GiB (Savings: 2.01%) | 20260505 04:26:17.353 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -2.9%) | 20260505 04:26:17.353 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 04:26:17.353 | INFO | High Water Breakdown: | 20260505 04:26:17.353 | INFO | 59.1% : 8.57 GiB ( 100% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 04:26:17.353 | INFO | 40.8% : 5.92 GiB ( 89.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 04:26:17.353 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 04:26:17.353 | INFO | ------------------------------------------ | 20260505 04:26:17.353 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:26:17.353 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 04:26:17.353 | INFO | Total mem usage: 26.5 GiB | 20260505 04:26:17.353 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260505 04:26:17.353 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 04:26:17.353 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260505 04:26:17.353 | INFO | High Water Breakdown: | 20260505 04:26:17.353 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:26:17.353 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:26:17.353 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:26:17.353 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:26:17.353 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:26:17.353 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:26:17.353 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 04:26:17.353 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 04:26:17.353 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:26:17.353 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 04:26:17.353 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 04:26:17.353 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 04:26:17.353 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 04:26:17.353 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 04:26:17.353 | INFO | ------------------------------------------ | 20260505 04:26:17.353 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:26:17.353 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 04:26:17.353 | INFO | Total mem usage: 25.6 GiB | 20260505 04:26:17.353 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 04:26:17.353 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 04:26:17.353 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 04:26:17.353 | INFO | High Water Breakdown: | 20260505 04:26:17.353 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 04:26:17.353 | INFO | ------------------------------------------ | 20260505 04:26:17.353 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:26:17.353 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 04:26:17.353 | INFO | Total mem usage: 15.7 GiB | 20260505 04:26:17.353 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260505 04:26:17.353 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 04:26:17.353 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260505 04:26:17.353 | INFO | High Water Breakdown: | 20260505 04:26:17.353 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 04:26:17.353 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 04:26:17.353 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 04:26:17.353 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 04:26:17.353 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 04:26:17.353 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 04:26:17.353 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 04:26:17.353 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 04:26:17.353 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 04:26:17.353 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:26:17.353 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 04:26:17.353 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:26:17.353 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 04:26:17.353 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 04:26:17.353 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 04:26:17.353 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 04:26:17.353 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:26:17.353 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 04:26:17.353 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:26:17.353 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:26:17.353 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:26:17.354 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 04:26:17.354 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 04:26:17.354 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:26:17.354 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 04:26:17.354 | INFO | ------------------------------------------ | 20260505 04:26:17.354 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:26:17.354 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 04:26:17.354 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:26:17.354 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:26:17.354 | INFO | PerfReport: Full Execution -- 3.05e+01 / -nan, 1.29e+01 / 0.00e+00 | 20260505 04:26:17.354 | INFO | PerfReport: 0.28%, AggregateStats -- 8.66e-02 / -nan, 6.77e-03 / 0.00e+00 | 20260505 04:26:17.354 | INFO | PerfReport: 10.80%, Baseline -- 3.30e+00 / -nan, 1.84e-02 / 0.00e+00 | 20260505 04:26:17.354 | INFO | PerfReport: 3.03%, Histogram -- 9.24e-01 / -nan, 5.25e-02 / 0.00e+00 | 20260505 04:26:17.354 | INFO | PerfReport: 16.02%, DME -- 3.91e+01 / -nan, 3.19e+00 / 0.00e+00 | 20260505 04:26:17.354 | INFO | PerfReport: 56.41%, FrameLabeling -- 1.72e+01 / -nan, 4.36e-02 / 0.00e+00 | 20260505 04:26:17.354 | INFO | PerfReport: 7.69%, PulseAccumulating -- 2.35e+00 / -nan, 1.59e-01 / 0.00e+00 | 20260505 04:26:17.354 | INFO | PerfReport: 4.07%, Metrics -- 1.24e+00 / -nan, 2.03e-01 / 0.00e+00 | 20260505 04:26:17.354 | INFO | PerfReport: 1.69%, PulseSerializing -- 5.15e-01 / -nan, 8.24e-02 / 0.00e+00 | 20260505 04:26:17.354 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 04:26:17.354 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:26:17.354 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:26:17.354 | INFO | PerfReport: Full Execution -- 9.16e+01 / -nan, 1.91e+01 / 0.00e+00 | 20260505 04:26:17.354 | INFO | PerfReport: 18.24%, Upload -- 1.67e+01 / 0.00e+00, 3.71e-01 / 0.00e+00 | 20260505 04:26:17.354 | INFO | PerfReport: 17.50%, Download -- 1.60e+01 / 0.00e+00, 6.92e-01 / 0.00e+00 | 20260505 04:26:17.354 | INFO | PerfReport: 33.30%, Compute -- 3.05e+01 / 0.00e+00, 1.29e+01 / 0.00e+00 | 20260505 04:26:17.354 | INFO | PerfReport: 0.55%, LockUpload -- 5.04e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260505 04:26:17.354 | INFO | PerfReport: 0.00%, LockDownload -- 3.93e-04 / 0.00e+00, 1.40e-04 / 0.00e+00 | 20260505 04:26:17.354 | INFO | PerfReport: 30.39%, LockCompute -- 2.79e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260505 04:26:17.354 | INFO | PerfReport: 0.02%, Misc -- 1.39e-02 / 0.00e+00, 2.28e-02 / 0.00e+00 | 20260505 04:26:17.450 | INFO | Starting a round of Baz IO | 20260505 04:26:20.406 | INFO | Populating RTMetrics report | 20260505 04:26:20.406 | INFO | RTMetrics report populated | 20260505 04:26:20.406 | INFO | Opening tmp RTMetrics json file | 20260505 04:26:20.407 | INFO | Serializing RTMetrics to json | 20260505 04:26:20.408 | INFO | Serialization done, swapping tmp file now... | 20260505 04:26:20.408 | INFO | RTMetrics done | 20260505 04:26:20.495 | INFO | Finished a round of Baz IO: Wrote 1.934 GiB in 3.045 seconds | 20260505 04:26:20.495 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.75% / 0.16% / 51.08% | 20260505 04:26:20.495 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 11.3289% / 0.000401338% / 4.37101e-05% | 20260505 04:26:20.495 | INFO | Baz IO overall: | 20260505 04:26:20.495 | INFO | BytesWritten : 123195216155 | 20260505 04:26:20.495 | INFO | NumEvents : 56773484886 | 20260505 04:26:22.491 | INFO | Analyzing chunk frames = [327680,328192) | 20260505 04:26:22.491 | INFO | MemInfo: { | 20260505 04:26:22.491 | INFO | "gpu" : " 17 GB", | 20260505 04:26:22.491 | INFO | "huge" : "", | 20260505 04:26:22.491 | INFO | "huge_shared" : " 27 GB", | 20260505 04:26:22.491 | INFO | "malloc" : " 14 GB", | 20260505 04:26:22.491 | INFO | "pinned" : " 29 GB" | 20260505 04:26:22.491 | INFO | } | 20260505 04:26:22.491 | INFO | Perf report for the last 5.13918 seconds: | 20260505 04:26:22.491 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:26:22.491 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:22.491 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:22.491 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:22.491 | INFO | ANALYSIS : 58.3%, 3.0, 90.8ms, 0 | 20260505 04:26:22.491 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:22.491 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260505 04:26:22.491 | INFO | PRE_HQ : 57.7%, 2.2, 64.7ms, 0 | 20260505 04:26:22.491 | INFO | BAZWRITER : 58.8%, 3.8, 113.9ms, 0 | 20260505 04:26:22.491 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:26:22.492 | INFO | Uploaded 36GiB to GPU in 1.63664s (21.9963GiB/s) | 20260505 04:26:22.492 | INFO | Downloaded 38GiB to GPU in 1.6982s (22.3766GiB/s) | 20260505 04:26:27.631 | INFO | Analyzing chunk frames = [328192,328704) | 20260505 04:26:27.631 | INFO | MemInfo: { | 20260505 04:26:27.631 | INFO | "gpu" : " 17 GB", | 20260505 04:26:27.631 | INFO | "huge" : "", | 20260505 04:26:27.631 | INFO | "huge_shared" : " 27 GB", | 20260505 04:26:27.631 | INFO | "malloc" : " 14 GB", | 20260505 04:26:27.631 | INFO | "pinned" : " 29 GB" | 20260505 04:26:27.631 | INFO | } | 20260505 04:26:27.631 | INFO | Perf report for the last 5.13959 seconds: | 20260505 04:26:27.631 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:26:27.631 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:27.631 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:27.631 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:27.631 | INFO | ANALYSIS : 58.1%, 3.0, 90.7ms, 0 | 20260505 04:26:27.631 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:27.631 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:27.631 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260505 04:26:27.631 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:27.631 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:27.631 | INFO | Uploaded 36GiB to GPU in 1.59789s (22.5297GiB/s) | 20260505 04:26:27.631 | INFO | Downloaded 34GiB to GPU in 1.51386s (22.4591GiB/s) | 20260505 04:26:32.770 | INFO | Analyzing chunk frames = [328704,329216) | 20260505 04:26:32.770 | INFO | MemInfo: { | 20260505 04:26:32.770 | INFO | "gpu" : " 17 GB", | 20260505 04:26:32.770 | INFO | "huge" : "", | 20260505 04:26:32.770 | INFO | "huge_shared" : " 27 GB", | 20260505 04:26:32.770 | INFO | "malloc" : " 14 GB", | 20260505 04:26:32.770 | INFO | "pinned" : " 29 GB" | 20260505 04:26:32.770 | INFO | } | 20260505 04:26:32.770 | INFO | Perf report for the last 5.13933 seconds: | 20260505 04:26:32.770 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:26:32.770 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:32.770 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:32.770 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:32.770 | INFO | ANALYSIS : 57.5%, 3.0, 89.4ms, 0 | 20260505 04:26:32.770 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:32.770 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:32.770 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260505 04:26:32.770 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:32.770 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:32.770 | INFO | Uploaded 36GiB to GPU in 1.60246s (22.4654GiB/s) | 20260505 04:26:32.771 | INFO | Downloaded 34GiB to GPU in 1.51911s (22.3815GiB/s) | 20260505 04:26:37.909 | INFO | Analyzing chunk frames = [329216,329728) | 20260505 04:26:37.909 | INFO | MemInfo: { | 20260505 04:26:37.909 | INFO | "gpu" : " 17 GB", | 20260505 04:26:37.909 | INFO | "huge" : "", | 20260505 04:26:37.909 | INFO | "huge_shared" : " 27 GB", | 20260505 04:26:37.909 | INFO | "malloc" : " 14 GB", | 20260505 04:26:37.909 | INFO | "pinned" : " 29 GB" | 20260505 04:26:37.909 | INFO | } | 20260505 04:26:37.909 | INFO | Perf report for the last 5.13916 seconds: | 20260505 04:26:37.909 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:26:37.909 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:37.909 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:37.909 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:37.909 | INFO | ANALYSIS : 57.2%, 3.0, 88.7ms, 0 | 20260505 04:26:37.909 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:37.909 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:37.910 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260505 04:26:37.910 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:37.910 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:37.910 | INFO | Uploaded 36GiB to GPU in 1.60017s (22.4976GiB/s) | 20260505 04:26:37.910 | INFO | Downloaded 34GiB to GPU in 1.51775s (22.4016GiB/s) | 20260505 04:26:43.049 | INFO | Analyzing chunk frames = [329728,330240) | 20260505 04:26:43.049 | INFO | MemInfo: { | 20260505 04:26:43.049 | INFO | "gpu" : " 17 GB", | 20260505 04:26:43.049 | INFO | "huge" : "", | 20260505 04:26:43.049 | INFO | "huge_shared" : " 27 GB", | 20260505 04:26:43.049 | INFO | "malloc" : " 14 GB", | 20260505 04:26:43.049 | INFO | "pinned" : " 29 GB" | 20260505 04:26:43.049 | INFO | } | 20260505 04:26:43.049 | INFO | Perf report for the last 5.13949 seconds: | 20260505 04:26:43.049 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:26:43.049 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:43.049 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:43.049 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:43.049 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260505 04:26:43.049 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:43.049 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:43.049 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260505 04:26:43.049 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:43.049 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:43.049 | INFO | Uploaded 36GiB to GPU in 1.59944s (22.5078GiB/s) | 20260505 04:26:43.049 | INFO | Downloaded 34GiB to GPU in 1.5156s (22.4334GiB/s) | 20260505 04:26:48.188 | INFO | Analyzing chunk frames = [330240,330752) | 20260505 04:26:48.188 | INFO | MemInfo: { | 20260505 04:26:48.188 | INFO | "gpu" : " 17 GB", | 20260505 04:26:48.188 | INFO | "huge" : "", | 20260505 04:26:48.188 | INFO | "huge_shared" : " 27 GB", | 20260505 04:26:48.188 | INFO | "malloc" : " 14 GB", | 20260505 04:26:48.188 | INFO | "pinned" : " 29 GB" | 20260505 04:26:48.188 | INFO | } | 20260505 04:26:48.188 | INFO | Perf report for the last 5.13938 seconds: | 20260505 04:26:48.188 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:26:48.188 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:48.188 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:48.188 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:48.188 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260505 04:26:48.188 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:48.188 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:48.188 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260505 04:26:48.188 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:48.188 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:48.189 | INFO | Uploaded 36GiB to GPU in 1.60003s (22.4995GiB/s) | 20260505 04:26:48.189 | INFO | Downloaded 34GiB to GPU in 1.51652s (22.4197GiB/s) | 20260505 04:26:53.327 | INFO | Analyzing chunk frames = [330752,331264) | 20260505 04:26:53.327 | INFO | MemInfo: { | 20260505 04:26:53.327 | INFO | "gpu" : " 17 GB", | 20260505 04:26:53.327 | INFO | "huge" : "", | 20260505 04:26:53.327 | INFO | "huge_shared" : " 27 GB", | 20260505 04:26:53.327 | INFO | "malloc" : " 14 GB", | 20260505 04:26:53.328 | INFO | "pinned" : " 29 GB" | 20260505 04:26:53.328 | INFO | } | 20260505 04:26:53.328 | INFO | Perf report for the last 5.13936 seconds: | 20260505 04:26:53.328 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:26:53.328 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:53.328 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:53.328 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:53.328 | INFO | ANALYSIS : 57.6%, 3.0, 89.8ms, 0 | 20260505 04:26:53.328 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:53.328 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:53.328 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260505 04:26:53.328 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:53.328 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:53.328 | INFO | Uploaded 36GiB to GPU in 1.59981s (22.5027GiB/s) | 20260505 04:26:53.328 | INFO | Downloaded 34GiB to GPU in 1.51716s (22.4103GiB/s) | 20260505 04:26:58.467 | INFO | Analyzing chunk frames = [331264,331776) | 20260505 04:26:58.467 | INFO | MemInfo: { | 20260505 04:26:58.467 | INFO | "gpu" : " 17 GB", | 20260505 04:26:58.467 | INFO | "huge" : "", | 20260505 04:26:58.467 | INFO | "huge_shared" : " 27 GB", | 20260505 04:26:58.467 | INFO | "malloc" : " 14 GB", | 20260505 04:26:58.467 | INFO | "pinned" : " 29 GB" | 20260505 04:26:58.467 | INFO | } | 20260505 04:26:58.467 | INFO | Perf report for the last 5.13924 seconds: | 20260505 04:26:58.467 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:26:58.467 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:58.467 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:58.467 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:58.467 | INFO | ANALYSIS : 57.7%, 3.0, 90.1ms, 0 | 20260505 04:26:58.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:58.467 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:58.467 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 04:26:58.467 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:58.467 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:26:58.467 | INFO | Uploaded 36GiB to GPU in 1.60013s (22.4982GiB/s) | 20260505 04:26:58.467 | INFO | Downloaded 34GiB to GPU in 1.51701s (22.4124GiB/s) | 20260505 04:26:58.563 | INFO | Starting a round of Baz IO | 20260505 04:27:01.522 | INFO | Populating RTMetrics report | 20260505 04:27:01.522 | INFO | RTMetrics report populated | 20260505 04:27:01.522 | INFO | Opening tmp RTMetrics json file | 20260505 04:27:01.523 | INFO | Serializing RTMetrics to json | 20260505 04:27:01.524 | INFO | Serialization done, swapping tmp file now... | 20260505 04:27:01.524 | INFO | RTMetrics done | 20260505 04:27:01.611 | INFO | Finished a round of Baz IO: Wrote 1.94 GiB in 3.048 seconds | 20260505 04:27:01.611 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.60% / 0.16% / 51.25% | 20260505 04:27:01.611 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 11.2656% / 0.000298023% / 1.98682e-05% | 20260505 04:27:01.611 | INFO | Baz IO overall: | 20260505 04:27:01.611 | INFO | BytesWritten : 125278711294 | 20260505 04:27:01.611 | INFO | NumEvents : 57758622794 | 20260505 04:27:03.606 | INFO | Analyzing chunk frames = [331776,332288) | 20260505 04:27:03.606 | INFO | MemInfo: { | 20260505 04:27:03.606 | INFO | "gpu" : " 17 GB", | 20260505 04:27:03.606 | INFO | "huge" : "", | 20260505 04:27:03.606 | INFO | "huge_shared" : " 27 GB", | 20260505 04:27:03.606 | INFO | "malloc" : " 14 GB", | 20260505 04:27:03.606 | INFO | "pinned" : " 29 GB" | 20260505 04:27:03.606 | INFO | } | 20260505 04:27:03.606 | INFO | Perf report for the last 5.13923 seconds: | 20260505 04:27:03.606 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:27:03.606 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:03.606 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:03.606 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:03.606 | INFO | ANALYSIS : 58.3%, 3.0, 90.7ms, 0 | 20260505 04:27:03.606 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:03.606 | INFO | RT_METRICS : 45.1%, 1.0, 23.7ms, 0 | 20260505 04:27:03.606 | INFO | PRE_HQ : 57.9%, 2.2, 65.3ms, 0 | 20260505 04:27:03.606 | INFO | BAZWRITER : 58.9%, 3.8, 115.1ms, 0 | 20260505 04:27:03.606 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:27:03.606 | INFO | Uploaded 36GiB to GPU in 1.63494s (22.0191GiB/s) | 20260505 04:27:03.606 | INFO | Downloaded 38GiB to GPU in 1.6982s (22.3767GiB/s) | 20260505 04:27:08.745 | INFO | Analyzing chunk frames = [332288,332800) | 20260505 04:27:08.745 | INFO | MemInfo: { | 20260505 04:27:08.745 | INFO | "gpu" : " 17 GB", | 20260505 04:27:08.745 | INFO | "huge" : "", | 20260505 04:27:08.745 | INFO | "huge_shared" : " 27 GB", | 20260505 04:27:08.745 | INFO | "malloc" : " 14 GB", | 20260505 04:27:08.745 | INFO | "pinned" : " 29 GB" | 20260505 04:27:08.745 | INFO | } | 20260505 04:27:08.745 | INFO | Perf report for the last 5.13936 seconds: | 20260505 04:27:08.745 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:27:08.745 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:08.745 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:08.746 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:08.746 | INFO | ANALYSIS : 58.2%, 3.0, 90.7ms, 0 | 20260505 04:27:08.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:08.746 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:08.746 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 04:27:08.746 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:08.746 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:08.746 | INFO | Uploaded 36GiB to GPU in 1.59757s (22.5343GiB/s) | 20260505 04:27:08.746 | INFO | Downloaded 34GiB to GPU in 1.51375s (22.4607GiB/s) | 20260505 04:27:13.885 | INFO | Analyzing chunk frames = [332800,333312) | 20260505 04:27:13.885 | INFO | MemInfo: { | 20260505 04:27:13.885 | INFO | "gpu" : " 17 GB", | 20260505 04:27:13.885 | INFO | "huge" : "", | 20260505 04:27:13.885 | INFO | "huge_shared" : " 27 GB", | 20260505 04:27:13.885 | INFO | "malloc" : " 14 GB", | 20260505 04:27:13.885 | INFO | "pinned" : " 29 GB" | 20260505 04:27:13.885 | INFO | } | 20260505 04:27:13.885 | INFO | Perf report for the last 5.13935 seconds: | 20260505 04:27:13.885 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:27:13.885 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:13.885 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:13.885 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:13.885 | INFO | ANALYSIS : 57.5%, 3.0, 89.3ms, 0 | 20260505 04:27:13.885 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:13.885 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:13.885 | INFO | PRE_HQ : 31.9%, 1.0, 16.5ms, 0 | 20260505 04:27:13.885 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:13.885 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:13.885 | INFO | Uploaded 36GiB to GPU in 1.60207s (22.471GiB/s) | 20260505 04:27:13.885 | INFO | Downloaded 34GiB to GPU in 1.51916s (22.3808GiB/s) | 20260505 04:27:19.024 | INFO | Analyzing chunk frames = [333312,333824) | 20260505 04:27:19.024 | INFO | MemInfo: { | 20260505 04:27:19.024 | INFO | "gpu" : " 17 GB", | 20260505 04:27:19.024 | INFO | "huge" : "", | 20260505 04:27:19.024 | INFO | "huge_shared" : " 27 GB", | 20260505 04:27:19.024 | INFO | "malloc" : " 14 GB", | 20260505 04:27:19.024 | INFO | "pinned" : " 29 GB" | 20260505 04:27:19.024 | INFO | } | 20260505 04:27:19.024 | INFO | Perf report for the last 5.13937 seconds: | 20260505 04:27:19.024 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:27:19.024 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:19.024 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:19.024 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:19.024 | INFO | ANALYSIS : 57.2%, 3.0, 88.8ms, 0 | 20260505 04:27:19.024 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:19.024 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:19.024 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260505 04:27:19.024 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:19.024 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:19.025 | INFO | Uploaded 36GiB to GPU in 1.60109s (22.4846GiB/s) | 20260505 04:27:19.025 | INFO | Downloaded 34GiB to GPU in 1.5177s (22.4023GiB/s) | 20260505 04:27:24.163 | INFO | Analyzing chunk frames = [333824,334336) | 20260505 04:27:24.163 | INFO | MemInfo: { | 20260505 04:27:24.163 | INFO | "gpu" : " 17 GB", | 20260505 04:27:24.163 | INFO | "huge" : "", | 20260505 04:27:24.163 | INFO | "huge_shared" : " 27 GB", | 20260505 04:27:24.163 | INFO | "malloc" : " 14 GB", | 20260505 04:27:24.163 | INFO | "pinned" : " 29 GB" | 20260505 04:27:24.163 | INFO | } | 20260505 04:27:24.163 | INFO | Perf report for the last 5.13928 seconds: | 20260505 04:27:24.163 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:27:24.163 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:24.164 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:24.164 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:24.164 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260505 04:27:24.164 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:24.164 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:24.164 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260505 04:27:24.164 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:24.164 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:24.164 | INFO | Uploaded 36GiB to GPU in 1.59955s (22.5063GiB/s) | 20260505 04:27:24.164 | INFO | Downloaded 34GiB to GPU in 1.51599s (22.4276GiB/s) | 20260505 04:27:29.303 | INFO | Analyzing chunk frames = [334336,334848) | 20260505 04:27:29.303 | INFO | MemInfo: { | 20260505 04:27:29.303 | INFO | "gpu" : " 17 GB", | 20260505 04:27:29.303 | INFO | "huge" : "", | 20260505 04:27:29.303 | INFO | "huge_shared" : " 27 GB", | 20260505 04:27:29.303 | INFO | "malloc" : " 14 GB", | 20260505 04:27:29.303 | INFO | "pinned" : " 29 GB" | 20260505 04:27:29.303 | INFO | } | 20260505 04:27:29.303 | INFO | Perf report for the last 5.13977 seconds: | 20260505 04:27:29.303 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:27:29.303 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:29.303 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:29.303 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:29.303 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260505 04:27:29.303 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:29.303 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:29.303 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 04:27:29.303 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:29.303 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:29.304 | INFO | Uploaded 36GiB to GPU in 1.60023s (22.4968GiB/s) | 20260505 04:27:29.304 | INFO | Downloaded 34GiB to GPU in 1.51625s (22.4237GiB/s) | 20260505 04:27:34.442 | INFO | Analyzing chunk frames = [334848,335360) | 20260505 04:27:34.442 | INFO | MemInfo: { | 20260505 04:27:34.442 | INFO | "gpu" : " 17 GB", | 20260505 04:27:34.442 | INFO | "huge" : "", | 20260505 04:27:34.442 | INFO | "huge_shared" : " 27 GB", | 20260505 04:27:34.442 | INFO | "malloc" : " 14 GB", | 20260505 04:27:34.442 | INFO | "pinned" : " 29 GB" | 20260505 04:27:34.442 | INFO | } | 20260505 04:27:34.442 | INFO | Perf report for the last 5.139 seconds: | 20260505 04:27:34.442 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:27:34.442 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:34.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:34.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:34.442 | INFO | ANALYSIS : 57.7%, 3.0, 89.9ms, 0 | 20260505 04:27:34.442 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:34.442 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:34.442 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260505 04:27:34.442 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:34.442 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:34.443 | INFO | Uploaded 36GiB to GPU in 1.59989s (22.5015GiB/s) | 20260505 04:27:34.443 | INFO | Downloaded 34GiB to GPU in 1.51724s (22.409GiB/s) | 20260505 04:27:39.600 | INFO | Analyzing chunk frames = [335360,335872) | 20260505 04:27:39.600 | INFO | MemInfo: { | 20260505 04:27:39.600 | INFO | "gpu" : " 17 GB", | 20260505 04:27:39.600 | INFO | "huge" : "", | 20260505 04:27:39.600 | INFO | "huge_shared" : " 27 GB", | 20260505 04:27:39.600 | INFO | "malloc" : " 14 GB", | 20260505 04:27:39.601 | INFO | "pinned" : " 29 GB" | 20260505 04:27:39.601 | INFO | } | 20260505 04:27:39.601 | INFO | Perf report for the last 5.15839 seconds: | 20260505 04:27:39.601 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:27:39.601 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:39.601 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:39.601 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:39.601 | INFO | ANALYSIS : 57.5%, 3.0, 90.0ms, 0 | 20260505 04:27:39.601 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:39.601 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:39.601 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260505 04:27:39.601 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:39.601 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:39.601 | INFO | Uploaded 36GiB to GPU in 1.60061s (22.4914GiB/s) | 20260505 04:27:39.601 | INFO | Downloaded 34GiB to GPU in 1.51746s (22.4059GiB/s) | 20260505 04:27:39.601 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 04:27:39.601 | INFO | Total mem usage: 14.9 GiB | 20260505 04:27:39.601 | INFO | All Time High Water Marks sum: 15.2 GiB (Savings: 2.01%) | 20260505 04:27:39.601 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -3.08%) | 20260505 04:27:39.601 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 04:27:39.601 | INFO | High Water Breakdown: | 20260505 04:27:39.601 | INFO | 59% : 8.55 GiB ( 99.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 04:27:39.601 | INFO | 40.8% : 5.91 GiB ( 89% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 04:27:39.601 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 04:27:39.601 | INFO | ------------------------------------------ | 20260505 04:27:39.601 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:27:39.601 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 04:27:39.601 | INFO | Total mem usage: 26.5 GiB | 20260505 04:27:39.601 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260505 04:27:39.601 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 04:27:39.601 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260505 04:27:39.601 | INFO | High Water Breakdown: | 20260505 04:27:39.601 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:27:39.601 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:27:39.601 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:27:39.601 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:27:39.601 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:27:39.601 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:27:39.601 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 04:27:39.601 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 04:27:39.601 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:27:39.601 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 04:27:39.601 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 04:27:39.601 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 04:27:39.601 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 04:27:39.601 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 04:27:39.601 | INFO | ------------------------------------------ | 20260505 04:27:39.601 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:27:39.601 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 04:27:39.601 | INFO | Total mem usage: 25.6 GiB | 20260505 04:27:39.601 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 04:27:39.601 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 04:27:39.601 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 04:27:39.601 | INFO | High Water Breakdown: | 20260505 04:27:39.601 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 04:27:39.601 | INFO | ------------------------------------------ | 20260505 04:27:39.601 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:27:39.601 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 04:27:39.601 | INFO | Total mem usage: 15.7 GiB | 20260505 04:27:39.601 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260505 04:27:39.601 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 04:27:39.601 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260505 04:27:39.601 | INFO | High Water Breakdown: | 20260505 04:27:39.601 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 04:27:39.601 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 04:27:39.601 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 04:27:39.601 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 04:27:39.601 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 04:27:39.601 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 04:27:39.601 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 04:27:39.601 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 04:27:39.601 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 04:27:39.601 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:27:39.601 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 04:27:39.601 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:27:39.601 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 04:27:39.601 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 04:27:39.601 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 04:27:39.601 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 04:27:39.601 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:27:39.601 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 04:27:39.601 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:27:39.601 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:27:39.601 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:27:39.601 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 04:27:39.601 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 04:27:39.601 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:27:39.601 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 04:27:39.601 | INFO | ------------------------------------------ | 20260505 04:27:39.601 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:27:39.601 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 04:27:39.601 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:27:39.602 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:27:39.602 | INFO | PerfReport: Full Execution -- 3.05e+01 / -nan, 1.28e+01 / 0.00e+00 | 20260505 04:27:39.602 | INFO | PerfReport: 0.28%, AggregateStats -- 8.64e-02 / -nan, 4.22e-03 / 0.00e+00 | 20260505 04:27:39.602 | INFO | PerfReport: 10.80%, Baseline -- 3.30e+00 / -nan, 1.70e-02 / 0.00e+00 | 20260505 04:27:39.602 | INFO | PerfReport: 3.03%, Histogram -- 9.24e-01 / -nan, 5.23e-02 / 0.00e+00 | 20260505 04:27:39.602 | INFO | PerfReport: 16.03%, DME -- 3.91e+01 / -nan, 3.24e+00 / 0.00e+00 | 20260505 04:27:39.602 | INFO | PerfReport: 56.42%, FrameLabeling -- 1.72e+01 / -nan, 4.46e-02 / 0.00e+00 | 20260505 04:27:39.602 | INFO | PerfReport: 7.69%, PulseAccumulating -- 2.35e+00 / -nan, 1.58e-01 / 0.00e+00 | 20260505 04:27:39.602 | INFO | PerfReport: 4.06%, Metrics -- 1.24e+00 / -nan, 2.02e-01 / 0.00e+00 | 20260505 04:27:39.602 | INFO | PerfReport: 1.69%, PulseSerializing -- 5.14e-01 / -nan, 8.29e-02 / 0.00e+00 | 20260505 04:27:39.602 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 04:27:39.602 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:27:39.602 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:27:39.602 | INFO | PerfReport: Full Execution -- 9.16e+01 / -nan, 1.91e+01 / 0.00e+00 | 20260505 04:27:39.602 | INFO | PerfReport: 18.24%, Upload -- 1.67e+01 / 0.00e+00, 3.62e-01 / 0.00e+00 | 20260505 04:27:39.602 | INFO | PerfReport: 17.50%, Download -- 1.60e+01 / 0.00e+00, 6.86e-01 / 0.00e+00 | 20260505 04:27:39.602 | INFO | PerfReport: 33.30%, Compute -- 3.05e+01 / 0.00e+00, 1.28e+01 / 0.00e+00 | 20260505 04:27:39.602 | INFO | PerfReport: 0.55%, LockUpload -- 5.04e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260505 04:27:39.602 | INFO | PerfReport: 0.00%, LockDownload -- 4.00e-04 / 0.00e+00, 1.71e-04 / 0.00e+00 | 20260505 04:27:39.602 | INFO | PerfReport: 30.39%, LockCompute -- 2.78e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260505 04:27:39.602 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.69e-02 / 0.00e+00 | 20260505 04:27:39.697 | INFO | Starting a round of Baz IO | 20260505 04:27:42.656 | INFO | Populating RTMetrics report | 20260505 04:27:42.657 | INFO | RTMetrics report populated | 20260505 04:27:42.657 | INFO | Opening tmp RTMetrics json file | 20260505 04:27:42.660 | INFO | Serializing RTMetrics to json | 20260505 04:27:42.662 | INFO | Serialization done, swapping tmp file now... | 20260505 04:27:42.662 | INFO | RTMetrics done | 20260505 04:27:42.755 | INFO | Finished a round of Baz IO: Wrote 1.944 GiB in 3.058 seconds | 20260505 04:27:42.756 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.44% / 0.16% / 51.40% | 20260505 04:27:42.756 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 11.1844% / 0.000397364% / 4.76837e-05% | 20260505 04:27:42.756 | INFO | Baz IO overall: | 20260505 04:27:42.756 | INFO | BytesWritten : 127365607363 | 20260505 04:27:42.756 | INFO | NumEvents : 58745021585 | 20260505 04:27:44.722 | INFO | Analyzing chunk frames = [335872,336384) | 20260505 04:27:44.722 | INFO | MemInfo: { | 20260505 04:27:44.722 | INFO | "gpu" : " 17 GB", | 20260505 04:27:44.722 | INFO | "huge" : "", | 20260505 04:27:44.722 | INFO | "huge_shared" : " 27 GB", | 20260505 04:27:44.722 | INFO | "malloc" : " 14 GB", | 20260505 04:27:44.722 | INFO | "pinned" : " 29 GB" | 20260505 04:27:44.722 | INFO | } | 20260505 04:27:44.722 | INFO | Perf report for the last 5.12093 seconds: | 20260505 04:27:44.722 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:27:44.722 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:44.722 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:44.722 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:44.722 | INFO | ANALYSIS : 58.6%, 3.0, 90.8ms, 0 | 20260505 04:27:44.722 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:44.722 | INFO | RT_METRICS : 45.3%, 1.0, 23.7ms, 0 | 20260505 04:27:44.722 | INFO | PRE_HQ : 58.0%, 2.2, 65.2ms, 0 | 20260505 04:27:44.722 | INFO | BAZWRITER : 59.3%, 3.8, 115.7ms, 0 | 20260505 04:27:44.722 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:27:44.722 | INFO | Uploaded 36GiB to GPU in 1.6374s (21.9861GiB/s) | 20260505 04:27:44.722 | INFO | Downloaded 38GiB to GPU in 1.69911s (22.3646GiB/s) | 20260505 04:27:49.860 | INFO | Analyzing chunk frames = [336384,336896) | 20260505 04:27:49.860 | INFO | MemInfo: { | 20260505 04:27:49.860 | INFO | "gpu" : " 17 GB", | 20260505 04:27:49.860 | INFO | "huge" : "", | 20260505 04:27:49.860 | INFO | "huge_shared" : " 27 GB", | 20260505 04:27:49.860 | INFO | "malloc" : " 14 GB", | 20260505 04:27:49.860 | INFO | "pinned" : " 29 GB" | 20260505 04:27:49.860 | INFO | } | 20260505 04:27:49.860 | INFO | Perf report for the last 5.1385 seconds: | 20260505 04:27:49.860 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:27:49.860 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:49.860 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:49.860 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:49.860 | INFO | ANALYSIS : 58.1%, 3.0, 90.6ms, 0 | 20260505 04:27:49.860 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:49.860 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:49.860 | INFO | PRE_HQ : 31.9%, 1.0, 16.5ms, 0 | 20260505 04:27:49.860 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:49.860 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:49.860 | INFO | Uploaded 36GiB to GPU in 1.59822s (22.5251GiB/s) | 20260505 04:27:49.860 | INFO | Downloaded 34GiB to GPU in 1.51485s (22.4445GiB/s) | 20260505 04:27:54.999 | INFO | Analyzing chunk frames = [336896,337408) | 20260505 04:27:54.999 | INFO | MemInfo: { | 20260505 04:27:54.999 | INFO | "gpu" : " 17 GB", | 20260505 04:27:54.999 | INFO | "huge" : "", | 20260505 04:27:54.999 | INFO | "huge_shared" : " 27 GB", | 20260505 04:27:54.999 | INFO | "malloc" : " 14 GB", | 20260505 04:27:54.999 | INFO | "pinned" : " 29 GB" | 20260505 04:27:55.000 | INFO | } | 20260505 04:27:55.000 | INFO | Perf report for the last 5.13941 seconds: | 20260505 04:27:55.000 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:27:55.000 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:55.000 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:55.000 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:55.000 | INFO | ANALYSIS : 57.5%, 3.0, 89.4ms, 0 | 20260505 04:27:55.000 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:55.000 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:55.000 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260505 04:27:55.000 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:55.000 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:27:55.000 | INFO | Uploaded 36GiB to GPU in 1.60167s (22.4766GiB/s) | 20260505 04:27:55.000 | INFO | Downloaded 34GiB to GPU in 1.51887s (22.3851GiB/s) | 20260505 04:28:00.139 | INFO | Analyzing chunk frames = [337408,337920) | 20260505 04:28:00.139 | INFO | MemInfo: { | 20260505 04:28:00.139 | INFO | "gpu" : " 17 GB", | 20260505 04:28:00.139 | INFO | "huge" : "", | 20260505 04:28:00.139 | INFO | "huge_shared" : " 27 GB", | 20260505 04:28:00.139 | INFO | "malloc" : " 14 GB", | 20260505 04:28:00.139 | INFO | "pinned" : " 29 GB" | 20260505 04:28:00.139 | INFO | } | 20260505 04:28:00.139 | INFO | Perf report for the last 5.13928 seconds: | 20260505 04:28:00.139 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:28:00.139 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:00.139 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:00.139 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:00.139 | INFO | ANALYSIS : 57.2%, 3.0, 88.7ms, 0 | 20260505 04:28:00.139 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:00.139 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:00.139 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 04:28:00.139 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:00.139 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:00.139 | INFO | Uploaded 36GiB to GPU in 1.60174s (22.4756GiB/s) | 20260505 04:28:00.139 | INFO | Downloaded 34GiB to GPU in 1.51834s (22.3929GiB/s) | 20260505 04:28:05.278 | INFO | Analyzing chunk frames = [337920,338432) | 20260505 04:28:05.278 | INFO | MemInfo: { | 20260505 04:28:05.278 | INFO | "gpu" : " 17 GB", | 20260505 04:28:05.278 | INFO | "huge" : "", | 20260505 04:28:05.278 | INFO | "huge_shared" : " 27 GB", | 20260505 04:28:05.278 | INFO | "malloc" : " 14 GB", | 20260505 04:28:05.278 | INFO | "pinned" : " 29 GB" | 20260505 04:28:05.278 | INFO | } | 20260505 04:28:05.278 | INFO | Perf report for the last 5.1395 seconds: | 20260505 04:28:05.278 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:28:05.278 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:05.278 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:05.278 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:05.278 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260505 04:28:05.278 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:05.278 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:05.278 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 04:28:05.279 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:05.279 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:05.279 | INFO | Uploaded 36GiB to GPU in 1.59953s (22.5066GiB/s) | 20260505 04:28:05.279 | INFO | Downloaded 34GiB to GPU in 1.51555s (22.4341GiB/s) | 20260505 04:28:10.417 | INFO | Analyzing chunk frames = [338432,338944) | 20260505 04:28:10.417 | INFO | MemInfo: { | 20260505 04:28:10.417 | INFO | "gpu" : " 17 GB", | 20260505 04:28:10.417 | INFO | "huge" : "", | 20260505 04:28:10.418 | INFO | "huge_shared" : " 27 GB", | 20260505 04:28:10.418 | INFO | "malloc" : " 14 GB", | 20260505 04:28:10.418 | INFO | "pinned" : " 29 GB" | 20260505 04:28:10.418 | INFO | } | 20260505 04:28:10.418 | INFO | Perf report for the last 5.13933 seconds: | 20260505 04:28:10.418 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:28:10.418 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:10.418 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:10.418 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:10.418 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260505 04:28:10.418 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:10.418 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:10.418 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260505 04:28:10.418 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:10.418 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:10.418 | INFO | Uploaded 36GiB to GPU in 1.60001s (22.4998GiB/s) | 20260505 04:28:10.418 | INFO | Downloaded 34GiB to GPU in 1.51631s (22.4229GiB/s) | 20260505 04:28:15.557 | INFO | Analyzing chunk frames = [338944,339456) | 20260505 04:28:15.557 | INFO | MemInfo: { | 20260505 04:28:15.557 | INFO | "gpu" : " 17 GB", | 20260505 04:28:15.557 | INFO | "huge" : "", | 20260505 04:28:15.557 | INFO | "huge_shared" : " 27 GB", | 20260505 04:28:15.557 | INFO | "malloc" : " 14 GB", | 20260505 04:28:15.557 | INFO | "pinned" : " 29 GB" | 20260505 04:28:15.557 | INFO | } | 20260505 04:28:15.557 | INFO | Perf report for the last 5.13925 seconds: | 20260505 04:28:15.557 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:28:15.557 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:15.557 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:15.557 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:15.557 | INFO | ANALYSIS : 57.6%, 3.0, 89.9ms, 0 | 20260505 04:28:15.557 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:15.557 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:15.557 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260505 04:28:15.557 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:15.557 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:15.557 | INFO | Uploaded 36GiB to GPU in 1.60013s (22.4982GiB/s) | 20260505 04:28:15.557 | INFO | Downloaded 34GiB to GPU in 1.51728s (22.4086GiB/s) | 20260505 04:28:20.696 | INFO | Analyzing chunk frames = [339456,339968) | 20260505 04:28:20.696 | INFO | MemInfo: { | 20260505 04:28:20.696 | INFO | "gpu" : " 17 GB", | 20260505 04:28:20.696 | INFO | "huge" : "", | 20260505 04:28:20.696 | INFO | "huge_shared" : " 27 GB", | 20260505 04:28:20.696 | INFO | "malloc" : " 14 GB", | 20260505 04:28:20.696 | INFO | "pinned" : " 29 GB" | 20260505 04:28:20.696 | INFO | } | 20260505 04:28:20.696 | INFO | Perf report for the last 5.13941 seconds: | 20260505 04:28:20.696 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:28:20.696 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:20.696 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:20.696 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:20.696 | INFO | ANALYSIS : 57.6%, 3.0, 89.9ms, 0 | 20260505 04:28:20.697 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:20.697 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:20.697 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 04:28:20.697 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:20.697 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:20.697 | INFO | Uploaded 36GiB to GPU in 1.60101s (22.4858GiB/s) | 20260505 04:28:20.697 | INFO | Downloaded 34GiB to GPU in 1.51796s (22.3985GiB/s) | 20260505 04:28:20.793 | INFO | Starting a round of Baz IO | 20260505 04:28:23.751 | INFO | Populating RTMetrics report | 20260505 04:28:23.752 | INFO | RTMetrics report populated | 20260505 04:28:23.752 | INFO | Opening tmp RTMetrics json file | 20260505 04:28:23.753 | INFO | Serializing RTMetrics to json | 20260505 04:28:23.754 | INFO | Serialization done, swapping tmp file now... | 20260505 04:28:23.755 | INFO | RTMetrics done | 20260505 04:28:23.840 | INFO | Finished a round of Baz IO: Wrote 1.949 GiB in 3.046 seconds | 20260505 04:28:23.840 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.29% / 0.16% / 51.56% | 20260505 04:28:23.840 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 11.1428% / 0.000421206% / 3.97364e-05% | 20260505 04:28:23.840 | INFO | Baz IO overall: | 20260505 04:28:23.840 | INFO | BytesWritten : 129457853481 | 20260505 04:28:23.840 | INFO | NumEvents : 59734922844 | 20260505 04:28:25.835 | INFO | Analyzing chunk frames = [339968,340480) | 20260505 04:28:25.835 | INFO | MemInfo: { | 20260505 04:28:25.835 | INFO | "gpu" : " 17 GB", | 20260505 04:28:25.835 | INFO | "huge" : "", | 20260505 04:28:25.835 | INFO | "huge_shared" : " 27 GB", | 20260505 04:28:25.835 | INFO | "malloc" : " 14 GB", | 20260505 04:28:25.835 | INFO | "pinned" : " 29 GB" | 20260505 04:28:25.835 | INFO | } | 20260505 04:28:25.835 | INFO | Perf report for the last 5.13905 seconds: | 20260505 04:28:25.835 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:28:25.835 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:25.835 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:25.836 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:25.836 | INFO | ANALYSIS : 58.3%, 3.0, 90.8ms, 0 | 20260505 04:28:25.836 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:25.836 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260505 04:28:25.836 | INFO | PRE_HQ : 57.9%, 2.2, 65.0ms, 0 | 20260505 04:28:25.836 | INFO | BAZWRITER : 58.8%, 3.8, 115.3ms, 0 | 20260505 04:28:25.836 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:28:25.836 | INFO | Uploaded 36GiB to GPU in 1.64022s (21.9483GiB/s) | 20260505 04:28:25.836 | INFO | Downloaded 38GiB to GPU in 1.70157s (22.3323GiB/s) | 20260505 04:28:30.975 | INFO | Analyzing chunk frames = [340480,340992) | 20260505 04:28:30.975 | INFO | MemInfo: { | 20260505 04:28:30.975 | INFO | "gpu" : " 17 GB", | 20260505 04:28:30.975 | INFO | "huge" : "", | 20260505 04:28:30.975 | INFO | "huge_shared" : " 27 GB", | 20260505 04:28:30.975 | INFO | "malloc" : " 14 GB", | 20260505 04:28:30.975 | INFO | "pinned" : " 29 GB" | 20260505 04:28:30.975 | INFO | } | 20260505 04:28:30.975 | INFO | Perf report for the last 5.13958 seconds: | 20260505 04:28:30.975 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:28:30.975 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:30.975 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:30.975 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:30.975 | INFO | ANALYSIS : 58.1%, 3.0, 90.6ms, 0 | 20260505 04:28:30.975 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:30.975 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:30.975 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260505 04:28:30.975 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:30.975 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:30.975 | INFO | Uploaded 36GiB to GPU in 1.59849s (22.5212GiB/s) | 20260505 04:28:30.975 | INFO | Downloaded 34GiB to GPU in 1.51515s (22.44GiB/s) | 20260505 04:28:36.114 | INFO | Analyzing chunk frames = [340992,341504) | 20260505 04:28:36.114 | INFO | MemInfo: { | 20260505 04:28:36.114 | INFO | "gpu" : " 17 GB", | 20260505 04:28:36.114 | INFO | "huge" : "", | 20260505 04:28:36.114 | INFO | "huge_shared" : " 27 GB", | 20260505 04:28:36.114 | INFO | "malloc" : " 14 GB", | 20260505 04:28:36.114 | INFO | "pinned" : " 29 GB" | 20260505 04:28:36.114 | INFO | } | 20260505 04:28:36.114 | INFO | Perf report for the last 5.13921 seconds: | 20260505 04:28:36.114 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:28:36.114 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:36.114 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:36.114 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:36.114 | INFO | ANALYSIS : 57.5%, 3.0, 89.3ms, 0 | 20260505 04:28:36.114 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:36.114 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:36.114 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260505 04:28:36.114 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:36.114 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:36.115 | INFO | Uploaded 36GiB to GPU in 1.60062s (22.4912GiB/s) | 20260505 04:28:36.115 | INFO | Downloaded 34GiB to GPU in 1.51861s (22.3889GiB/s) | 20260505 04:28:41.253 | INFO | Analyzing chunk frames = [341504,342016) | 20260505 04:28:41.253 | INFO | MemInfo: { | 20260505 04:28:41.253 | INFO | "gpu" : " 17 GB", | 20260505 04:28:41.253 | INFO | "huge" : "", | 20260505 04:28:41.253 | INFO | "huge_shared" : " 27 GB", | 20260505 04:28:41.254 | INFO | "malloc" : " 14 GB", | 20260505 04:28:41.254 | INFO | "pinned" : " 29 GB" | 20260505 04:28:41.254 | INFO | } | 20260505 04:28:41.254 | INFO | Perf report for the last 5.13936 seconds: | 20260505 04:28:41.254 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:28:41.254 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:41.254 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:41.254 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:41.254 | INFO | ANALYSIS : 57.1%, 3.0, 88.6ms, 0 | 20260505 04:28:41.254 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:41.254 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:41.254 | INFO | PRE_HQ : 31.7%, 1.0, 16.4ms, 0 | 20260505 04:28:41.254 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:41.254 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:41.254 | INFO | Uploaded 36GiB to GPU in 1.60042s (22.4942GiB/s) | 20260505 04:28:41.254 | INFO | Downloaded 34GiB to GPU in 1.51798s (22.3982GiB/s) | 20260505 04:28:46.393 | INFO | Analyzing chunk frames = [342016,342528) | 20260505 04:28:46.393 | INFO | MemInfo: { | 20260505 04:28:46.393 | INFO | "gpu" : " 17 GB", | 20260505 04:28:46.393 | INFO | "huge" : "", | 20260505 04:28:46.393 | INFO | "huge_shared" : " 27 GB", | 20260505 04:28:46.393 | INFO | "malloc" : " 14 GB", | 20260505 04:28:46.393 | INFO | "pinned" : " 29 GB" | 20260505 04:28:46.393 | INFO | } | 20260505 04:28:46.393 | INFO | Perf report for the last 5.1392 seconds: | 20260505 04:28:46.393 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:28:46.393 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:46.393 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:46.393 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:46.393 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260505 04:28:46.393 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:46.393 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:46.393 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260505 04:28:46.393 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:46.393 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:46.393 | INFO | Uploaded 36GiB to GPU in 1.60003s (22.4996GiB/s) | 20260505 04:28:46.393 | INFO | Downloaded 34GiB to GPU in 1.51658s (22.4189GiB/s) | 20260505 04:28:51.532 | INFO | Analyzing chunk frames = [342528,343040) | 20260505 04:28:51.532 | INFO | MemInfo: { | 20260505 04:28:51.532 | INFO | "gpu" : " 17 GB", | 20260505 04:28:51.532 | INFO | "huge" : "", | 20260505 04:28:51.532 | INFO | "huge_shared" : " 27 GB", | 20260505 04:28:51.532 | INFO | "malloc" : " 14 GB", | 20260505 04:28:51.532 | INFO | "pinned" : " 29 GB" | 20260505 04:28:51.532 | INFO | } | 20260505 04:28:51.532 | INFO | Perf report for the last 5.13938 seconds: | 20260505 04:28:51.532 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:28:51.532 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:51.532 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:51.532 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:51.532 | INFO | ANALYSIS : 57.3%, 3.0, 89.5ms, 0 | 20260505 04:28:51.532 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:51.532 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:51.532 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260505 04:28:51.532 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:51.532 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:51.533 | INFO | Uploaded 36GiB to GPU in 1.59981s (22.5027GiB/s) | 20260505 04:28:51.533 | INFO | Downloaded 34GiB to GPU in 1.51632s (22.4227GiB/s) | 20260505 04:28:56.671 | INFO | Analyzing chunk frames = [343040,343552) | 20260505 04:28:56.671 | INFO | MemInfo: { | 20260505 04:28:56.671 | INFO | "gpu" : " 17 GB", | 20260505 04:28:56.671 | INFO | "huge" : "", | 20260505 04:28:56.671 | INFO | "huge_shared" : " 27 GB", | 20260505 04:28:56.671 | INFO | "malloc" : " 14 GB", | 20260505 04:28:56.671 | INFO | "pinned" : " 29 GB" | 20260505 04:28:56.671 | INFO | } | 20260505 04:28:56.672 | INFO | Perf report for the last 5.13925 seconds: | 20260505 04:28:56.672 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:28:56.672 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:56.672 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:56.672 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:56.672 | INFO | ANALYSIS : 57.6%, 3.0, 89.9ms, 0 | 20260505 04:28:56.672 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:56.672 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:56.672 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 04:28:56.672 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:56.672 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:28:56.672 | INFO | Uploaded 36GiB to GPU in 1.59965s (22.5049GiB/s) | 20260505 04:28:56.672 | INFO | Downloaded 34GiB to GPU in 1.51745s (22.406GiB/s) | 20260505 04:29:01.811 | INFO | Analyzing chunk frames = [343552,344064) | 20260505 04:29:01.811 | INFO | MemInfo: { | 20260505 04:29:01.811 | INFO | "gpu" : " 17 GB", | 20260505 04:29:01.811 | INFO | "huge" : "", | 20260505 04:29:01.811 | INFO | "huge_shared" : " 27 GB", | 20260505 04:29:01.811 | INFO | "malloc" : " 14 GB", | 20260505 04:29:01.811 | INFO | "pinned" : " 29 GB" | 20260505 04:29:01.811 | INFO | } | 20260505 04:29:01.811 | INFO | Perf report for the last 5.13956 seconds: | 20260505 04:29:01.811 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:29:01.811 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:01.811 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:01.811 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:01.811 | INFO | ANALYSIS : 57.6%, 3.0, 89.9ms, 0 | 20260505 04:29:01.811 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:01.811 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:01.811 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260505 04:29:01.811 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:01.811 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:01.811 | INFO | Uploaded 36GiB to GPU in 1.60208s (22.4708GiB/s) | 20260505 04:29:01.811 | INFO | Downloaded 34GiB to GPU in 1.51988s (22.3703GiB/s) | 20260505 04:29:01.812 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 04:29:01.812 | INFO | Total mem usage: 14.9 GiB | 20260505 04:29:01.812 | INFO | All Time High Water Marks sum: 15.2 GiB (Savings: 2.01%) | 20260505 04:29:01.812 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -3.09%) | 20260505 04:29:01.812 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 04:29:01.812 | INFO | High Water Breakdown: | 20260505 04:29:01.812 | INFO | 59.1% : 8.55 GiB ( 99.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 04:29:01.812 | INFO | 40.8% : 5.91 GiB ( 88.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 04:29:01.812 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 04:29:01.812 | INFO | ------------------------------------------ | 20260505 04:29:01.812 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:29:01.812 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 04:29:01.812 | INFO | Total mem usage: 26.5 GiB | 20260505 04:29:01.812 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260505 04:29:01.812 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 04:29:01.812 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260505 04:29:01.812 | INFO | High Water Breakdown: | 20260505 04:29:01.812 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:29:01.812 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:29:01.812 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:29:01.812 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:29:01.812 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:29:01.812 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:29:01.812 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 04:29:01.812 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 04:29:01.812 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:29:01.812 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 04:29:01.812 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 04:29:01.812 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 04:29:01.812 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 04:29:01.812 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 04:29:01.812 | INFO | ------------------------------------------ | 20260505 04:29:01.812 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:29:01.812 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 04:29:01.812 | INFO | Total mem usage: 25.6 GiB | 20260505 04:29:01.812 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 04:29:01.812 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 04:29:01.812 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 04:29:01.812 | INFO | High Water Breakdown: | 20260505 04:29:01.812 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 04:29:01.812 | INFO | ------------------------------------------ | 20260505 04:29:01.812 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:29:01.812 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 04:29:01.812 | INFO | Total mem usage: 15.7 GiB | 20260505 04:29:01.812 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260505 04:29:01.812 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 04:29:01.812 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260505 04:29:01.812 | INFO | High Water Breakdown: | 20260505 04:29:01.812 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 04:29:01.812 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 04:29:01.812 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 04:29:01.812 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 04:29:01.812 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 04:29:01.812 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 04:29:01.812 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 04:29:01.812 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 04:29:01.812 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 04:29:01.812 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:29:01.812 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 04:29:01.812 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:29:01.812 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 04:29:01.812 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 04:29:01.812 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 04:29:01.812 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 04:29:01.812 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:29:01.812 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 04:29:01.812 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:29:01.812 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:29:01.812 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:29:01.812 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 04:29:01.812 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 04:29:01.812 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:29:01.812 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 04:29:01.812 | INFO | ------------------------------------------ | 20260505 04:29:01.812 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:29:01.812 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 04:29:01.812 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:29:01.812 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:29:01.813 | INFO | PerfReport: Full Execution -- 3.05e+01 / -nan, 1.28e+01 / 0.00e+00 | 20260505 04:29:01.813 | INFO | PerfReport: 0.28%, AggregateStats -- 8.64e-02 / -nan, 4.48e-03 / 0.00e+00 | 20260505 04:29:01.813 | INFO | PerfReport: 10.80%, Baseline -- 3.29e+00 / -nan, 1.82e-02 / 0.00e+00 | 20260505 04:29:01.813 | INFO | PerfReport: 3.03%, Histogram -- 9.24e-01 / -nan, 5.22e-02 / 0.00e+00 | 20260505 04:29:01.813 | INFO | PerfReport: 16.02%, DME -- 3.91e+01 / -nan, 3.26e+00 / 0.00e+00 | 20260505 04:29:01.813 | INFO | PerfReport: 56.44%, FrameLabeling -- 1.72e+01 / -nan, 4.46e-02 / 0.00e+00 | 20260505 04:29:01.813 | INFO | PerfReport: 7.68%, PulseAccumulating -- 2.34e+00 / -nan, 1.58e-01 / 0.00e+00 | 20260505 04:29:01.813 | INFO | PerfReport: 4.06%, Metrics -- 1.24e+00 / -nan, 2.03e-01 / 0.00e+00 | 20260505 04:29:01.813 | INFO | PerfReport: 1.68%, PulseSerializing -- 5.13e-01 / -nan, 8.27e-02 / 0.00e+00 | 20260505 04:29:01.813 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 04:29:01.813 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:29:01.813 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:29:01.813 | INFO | PerfReport: Full Execution -- 9.16e+01 / -nan, 1.90e+01 / 0.00e+00 | 20260505 04:29:01.813 | INFO | PerfReport: 18.25%, Upload -- 1.67e+01 / 0.00e+00, 3.70e-01 / 0.00e+00 | 20260505 04:29:01.813 | INFO | PerfReport: 17.52%, Download -- 1.60e+01 / 0.00e+00, 6.91e-01 / 0.00e+00 | 20260505 04:29:01.813 | INFO | PerfReport: 33.30%, Compute -- 3.05e+01 / 0.00e+00, 1.28e+01 / 0.00e+00 | 20260505 04:29:01.813 | INFO | PerfReport: 0.55%, LockUpload -- 5.04e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260505 04:29:01.813 | INFO | PerfReport: 0.00%, LockDownload -- 3.93e-04 / 0.00e+00, 9.38e-05 / 0.00e+00 | 20260505 04:29:01.813 | INFO | PerfReport: 30.37%, LockCompute -- 2.78e+01 / 0.00e+00, 1.72e+01 / 0.00e+00 | 20260505 04:29:01.813 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.45e-02 / 0.00e+00 | 20260505 04:29:01.908 | INFO | Starting a round of Baz IO | 20260505 04:29:04.865 | INFO | Populating RTMetrics report | 20260505 04:29:04.865 | INFO | RTMetrics report populated | 20260505 04:29:04.865 | INFO | Opening tmp RTMetrics json file | 20260505 04:29:04.866 | INFO | Serializing RTMetrics to json | 20260505 04:29:04.867 | INFO | Serialization done, swapping tmp file now... | 20260505 04:29:04.867 | INFO | RTMetrics done | 20260505 04:29:04.952 | INFO | Finished a round of Baz IO: Wrote 1.952 GiB in 3.043 seconds | 20260505 04:29:04.952 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.13% / 0.15% / 51.71% | 20260505 04:29:04.952 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 11.0658% / 0.000361602% / 5.96046e-05% | 20260505 04:29:04.952 | INFO | Baz IO overall: | 20260505 04:29:04.952 | INFO | BytesWritten : 131553765214 | 20260505 04:29:04.952 | INFO | NumEvents : 60726156985 | 20260505 04:29:06.950 | INFO | Analyzing chunk frames = [344064,344576) | 20260505 04:29:06.950 | INFO | MemInfo: { | 20260505 04:29:06.950 | INFO | "gpu" : " 17 GB", | 20260505 04:29:06.950 | INFO | "huge" : "", | 20260505 04:29:06.950 | INFO | "huge_shared" : " 27 GB", | 20260505 04:29:06.950 | INFO | "malloc" : " 14 GB", | 20260505 04:29:06.950 | INFO | "pinned" : " 29 GB" | 20260505 04:29:06.950 | INFO | } | 20260505 04:29:06.950 | INFO | Perf report for the last 5.13915 seconds: | 20260505 04:29:06.950 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:29:06.950 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:06.950 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:06.950 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:06.950 | INFO | ANALYSIS : 58.3%, 3.0, 90.6ms, 0 | 20260505 04:29:06.950 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:06.950 | INFO | RT_METRICS : 45.1%, 1.0, 23.6ms, 0 | 20260505 04:29:06.951 | INFO | PRE_HQ : 57.8%, 2.2, 64.9ms, 0 | 20260505 04:29:06.951 | INFO | BAZWRITER : 58.8%, 3.8, 116.1ms, 0 | 20260505 04:29:06.951 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:29:06.951 | INFO | Uploaded 36GiB to GPU in 1.64065s (21.9426GiB/s) | 20260505 04:29:06.951 | INFO | Downloaded 38GiB to GPU in 1.70233s (22.3223GiB/s) | 20260505 04:29:12.090 | INFO | Analyzing chunk frames = [344576,345088) | 20260505 04:29:12.090 | INFO | MemInfo: { | 20260505 04:29:12.090 | INFO | "gpu" : " 17 GB", | 20260505 04:29:12.090 | INFO | "huge" : "", | 20260505 04:29:12.090 | INFO | "huge_shared" : " 27 GB", | 20260505 04:29:12.090 | INFO | "malloc" : " 14 GB", | 20260505 04:29:12.090 | INFO | "pinned" : " 29 GB" | 20260505 04:29:12.090 | INFO | } | 20260505 04:29:12.090 | INFO | Perf report for the last 5.13958 seconds: | 20260505 04:29:12.090 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:29:12.090 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:12.090 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:12.090 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:12.090 | INFO | ANALYSIS : 58.1%, 3.0, 90.7ms, 0 | 20260505 04:29:12.090 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:12.090 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:12.090 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 04:29:12.090 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:12.090 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:12.090 | INFO | Uploaded 36GiB to GPU in 1.59809s (22.5268GiB/s) | 20260505 04:29:12.090 | INFO | Downloaded 34GiB to GPU in 1.5145s (22.4496GiB/s) | 20260505 04:29:17.229 | INFO | Analyzing chunk frames = [345088,345600) | 20260505 04:29:17.229 | INFO | MemInfo: { | 20260505 04:29:17.229 | INFO | "gpu" : " 17 GB", | 20260505 04:29:17.229 | INFO | "huge" : "", | 20260505 04:29:17.229 | INFO | "huge_shared" : " 27 GB", | 20260505 04:29:17.229 | INFO | "malloc" : " 14 GB", | 20260505 04:29:17.229 | INFO | "pinned" : " 29 GB" | 20260505 04:29:17.229 | INFO | } | 20260505 04:29:17.229 | INFO | Perf report for the last 5.13936 seconds: | 20260505 04:29:17.229 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:29:17.229 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:17.229 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:17.229 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:17.229 | INFO | ANALYSIS : 57.4%, 3.0, 89.3ms, 0 | 20260505 04:29:17.229 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:17.229 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:17.229 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260505 04:29:17.229 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:17.229 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:17.230 | INFO | Uploaded 36GiB to GPU in 1.60286s (22.4599GiB/s) | 20260505 04:29:17.230 | INFO | Downloaded 34GiB to GPU in 1.52029s (22.3641GiB/s) | 20260505 04:29:22.368 | INFO | Analyzing chunk frames = [345600,346112) | 20260505 04:29:22.368 | INFO | MemInfo: { | 20260505 04:29:22.368 | INFO | "gpu" : " 17 GB", | 20260505 04:29:22.368 | INFO | "huge" : "", | 20260505 04:29:22.368 | INFO | "huge_shared" : " 27 GB", | 20260505 04:29:22.368 | INFO | "malloc" : " 14 GB", | 20260505 04:29:22.368 | INFO | "pinned" : " 29 GB" | 20260505 04:29:22.368 | INFO | } | 20260505 04:29:22.368 | INFO | Perf report for the last 5.13918 seconds: | 20260505 04:29:22.368 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:29:22.368 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:22.368 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:22.369 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:22.369 | INFO | ANALYSIS : 57.2%, 3.0, 88.7ms, 0 | 20260505 04:29:22.369 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:22.369 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:22.369 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 04:29:22.369 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:22.369 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:22.369 | INFO | Uploaded 36GiB to GPU in 1.60093s (22.487GiB/s) | 20260505 04:29:22.369 | INFO | Downloaded 34GiB to GPU in 1.51751s (22.4051GiB/s) | 20260505 04:29:27.507 | INFO | Analyzing chunk frames = [346112,346624) | 20260505 04:29:27.508 | INFO | MemInfo: { | 20260505 04:29:27.508 | INFO | "gpu" : " 17 GB", | 20260505 04:29:27.508 | INFO | "huge" : "", | 20260505 04:29:27.508 | INFO | "huge_shared" : " 27 GB", | 20260505 04:29:27.508 | INFO | "malloc" : " 14 GB", | 20260505 04:29:27.508 | INFO | "pinned" : " 29 GB" | 20260505 04:29:27.508 | INFO | } | 20260505 04:29:27.508 | INFO | Perf report for the last 5.13929 seconds: | 20260505 04:29:27.508 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:29:27.508 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:27.508 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:27.508 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:27.508 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260505 04:29:27.508 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:27.508 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:27.508 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260505 04:29:27.508 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:27.508 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:27.508 | INFO | Uploaded 36GiB to GPU in 1.59997s (22.5004GiB/s) | 20260505 04:29:27.508 | INFO | Downloaded 34GiB to GPU in 1.51638s (22.4219GiB/s) | 20260505 04:29:32.647 | INFO | Analyzing chunk frames = [346624,347136) | 20260505 04:29:32.647 | INFO | MemInfo: { | 20260505 04:29:32.647 | INFO | "gpu" : " 17 GB", | 20260505 04:29:32.647 | INFO | "huge" : "", | 20260505 04:29:32.647 | INFO | "huge_shared" : " 27 GB", | 20260505 04:29:32.647 | INFO | "malloc" : " 14 GB", | 20260505 04:29:32.647 | INFO | "pinned" : " 29 GB" | 20260505 04:29:32.647 | INFO | } | 20260505 04:29:32.647 | INFO | Perf report for the last 5.1393 seconds: | 20260505 04:29:32.647 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:29:32.647 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:32.647 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:32.647 | INFO | SAVE_TRACE : 0.1%, 1.0, 0.0ms, 0 | 20260505 04:29:32.647 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260505 04:29:32.647 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:32.647 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:32.647 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260505 04:29:32.647 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:32.647 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:32.647 | INFO | Uploaded 36GiB to GPU in 1.60015s (22.4979GiB/s) | 20260505 04:29:32.647 | INFO | Downloaded 34GiB to GPU in 1.51639s (22.4217GiB/s) | 20260505 04:29:37.786 | INFO | Analyzing chunk frames = [347136,347648) | 20260505 04:29:37.786 | INFO | MemInfo: { | 20260505 04:29:37.786 | INFO | "gpu" : " 17 GB", | 20260505 04:29:37.786 | INFO | "huge" : "", | 20260505 04:29:37.786 | INFO | "huge_shared" : " 27 GB", | 20260505 04:29:37.786 | INFO | "malloc" : " 14 GB", | 20260505 04:29:37.786 | INFO | "pinned" : " 29 GB" | 20260505 04:29:37.786 | INFO | } | 20260505 04:29:37.786 | INFO | Perf report for the last 5.13944 seconds: | 20260505 04:29:37.786 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:29:37.786 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:37.786 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:37.787 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:37.787 | INFO | ANALYSIS : 57.6%, 3.0, 89.8ms, 0 | 20260505 04:29:37.787 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:37.787 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:37.787 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260505 04:29:37.787 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:37.787 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:37.787 | INFO | Uploaded 36GiB to GPU in 1.60015s (22.4979GiB/s) | 20260505 04:29:37.787 | INFO | Downloaded 34GiB to GPU in 1.51757s (22.4042GiB/s) | 20260505 04:29:42.926 | INFO | Analyzing chunk frames = [347648,348160) | 20260505 04:29:42.926 | INFO | MemInfo: { | 20260505 04:29:42.926 | INFO | "gpu" : " 17 GB", | 20260505 04:29:42.926 | INFO | "huge" : "", | 20260505 04:29:42.926 | INFO | "huge_shared" : " 27 GB", | 20260505 04:29:42.926 | INFO | "malloc" : " 14 GB", | 20260505 04:29:42.926 | INFO | "pinned" : " 29 GB" | 20260505 04:29:42.926 | INFO | } | 20260505 04:29:42.926 | INFO | Perf report for the last 5.13938 seconds: | 20260505 04:29:42.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:29:42.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:42.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:42.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:42.926 | INFO | ANALYSIS : 57.7%, 3.0, 90.1ms, 0 | 20260505 04:29:42.926 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:42.926 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:42.926 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 04:29:42.926 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:42.926 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:42.926 | INFO | Uploaded 36GiB to GPU in 1.60001s (22.4999GiB/s) | 20260505 04:29:42.926 | INFO | Downloaded 34GiB to GPU in 1.51795s (22.3987GiB/s) | 20260505 04:29:43.023 | INFO | Starting a round of Baz IO | 20260505 04:29:45.978 | INFO | Populating RTMetrics report | 20260505 04:29:45.978 | INFO | RTMetrics report populated | 20260505 04:29:45.978 | INFO | Opening tmp RTMetrics json file | 20260505 04:29:45.979 | INFO | Serializing RTMetrics to json | 20260505 04:29:45.980 | INFO | Serialization done, swapping tmp file now... | 20260505 04:29:45.981 | INFO | RTMetrics done | 20260505 04:29:46.064 | INFO | Finished a round of Baz IO: Wrote 1.958 GiB in 3.041 seconds | 20260505 04:29:46.064 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.98% / 0.15% / 51.87% | 20260505 04:29:46.064 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 11.0194% / 0.000417233% / 5.16574e-05% | 20260505 04:29:46.064 | INFO | Baz IO overall: | 20260505 04:29:46.064 | INFO | BytesWritten : 133655703293 | 20260505 04:29:46.064 | INFO | NumEvents : 61720655829 | 20260505 04:29:48.065 | INFO | Analyzing chunk frames = [348160,348672) | 20260505 04:29:48.065 | INFO | MemInfo: { | 20260505 04:29:48.065 | INFO | "gpu" : " 17 GB", | 20260505 04:29:48.065 | INFO | "huge" : "", | 20260505 04:29:48.065 | INFO | "huge_shared" : " 27 GB", | 20260505 04:29:48.065 | INFO | "malloc" : " 14 GB", | 20260505 04:29:48.065 | INFO | "pinned" : " 29 GB" | 20260505 04:29:48.065 | INFO | } | 20260505 04:29:48.065 | INFO | Perf report for the last 5.13932 seconds: | 20260505 04:29:48.065 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:29:48.065 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:48.065 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:48.065 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:48.065 | INFO | ANALYSIS : 58.3%, 3.0, 90.5ms, 0 | 20260505 04:29:48.065 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:48.065 | INFO | RT_METRICS : 45.1%, 1.0, 23.7ms, 0 | 20260505 04:29:48.065 | INFO | PRE_HQ : 57.7%, 2.2, 64.7ms, 0 | 20260505 04:29:48.065 | INFO | BAZWRITER : 58.7%, 3.9, 116.7ms, 0 | 20260505 04:29:48.065 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:29:48.066 | INFO | Uploaded 36GiB to GPU in 1.63192s (22.0599GiB/s) | 20260505 04:29:48.066 | INFO | Downloaded 38GiB to GPU in 1.69336s (22.4406GiB/s) | 20260505 04:29:53.204 | INFO | Analyzing chunk frames = [348672,349184) | 20260505 04:29:53.204 | INFO | MemInfo: { | 20260505 04:29:53.204 | INFO | "gpu" : " 17 GB", | 20260505 04:29:53.204 | INFO | "huge" : "", | 20260505 04:29:53.204 | INFO | "huge_shared" : " 27 GB", | 20260505 04:29:53.204 | INFO | "malloc" : " 14 GB", | 20260505 04:29:53.204 | INFO | "pinned" : " 29 GB" | 20260505 04:29:53.204 | INFO | } | 20260505 04:29:53.204 | INFO | Perf report for the last 5.13927 seconds: | 20260505 04:29:53.204 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:29:53.205 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:53.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:53.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:53.205 | INFO | ANALYSIS : 58.2%, 3.0, 90.7ms, 0 | 20260505 04:29:53.205 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:53.205 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:53.205 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260505 04:29:53.205 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:53.205 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:53.205 | INFO | Uploaded 36GiB to GPU in 1.59819s (22.5255GiB/s) | 20260505 04:29:53.205 | INFO | Downloaded 34GiB to GPU in 1.51458s (22.4485GiB/s) | 20260505 04:29:58.344 | INFO | Analyzing chunk frames = [349184,349696) | 20260505 04:29:58.344 | INFO | MemInfo: { | 20260505 04:29:58.344 | INFO | "gpu" : " 17 GB", | 20260505 04:29:58.344 | INFO | "huge" : "", | 20260505 04:29:58.344 | INFO | "huge_shared" : " 27 GB", | 20260505 04:29:58.344 | INFO | "malloc" : " 14 GB", | 20260505 04:29:58.344 | INFO | "pinned" : " 29 GB" | 20260505 04:29:58.344 | INFO | } | 20260505 04:29:58.344 | INFO | Perf report for the last 5.13934 seconds: | 20260505 04:29:58.344 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:29:58.344 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:58.344 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:58.344 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:58.344 | INFO | ANALYSIS : 57.5%, 3.0, 89.3ms, 0 | 20260505 04:29:58.344 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:58.344 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:58.344 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260505 04:29:58.344 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:58.344 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:29:58.344 | INFO | Uploaded 36GiB to GPU in 1.60178s (22.475GiB/s) | 20260505 04:29:58.344 | INFO | Downloaded 34GiB to GPU in 1.51899s (22.3833GiB/s) | 20260505 04:30:03.483 | INFO | Analyzing chunk frames = [349696,350208) | 20260505 04:30:03.483 | INFO | MemInfo: { | 20260505 04:30:03.483 | INFO | "gpu" : " 17 GB", | 20260505 04:30:03.483 | INFO | "huge" : "", | 20260505 04:30:03.483 | INFO | "huge_shared" : " 27 GB", | 20260505 04:30:03.483 | INFO | "malloc" : " 14 GB", | 20260505 04:30:03.483 | INFO | "pinned" : " 29 GB" | 20260505 04:30:03.483 | INFO | } | 20260505 04:30:03.483 | INFO | Perf report for the last 5.1394 seconds: | 20260505 04:30:03.483 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:30:03.483 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:03.483 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:03.483 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:03.483 | INFO | ANALYSIS : 57.2%, 3.0, 88.7ms, 0 | 20260505 04:30:03.483 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:03.483 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:03.483 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260505 04:30:03.483 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:03.483 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:03.484 | INFO | Uploaded 36GiB to GPU in 1.60159s (22.4776GiB/s) | 20260505 04:30:03.484 | INFO | Downloaded 34GiB to GPU in 1.51846s (22.3911GiB/s) | 20260505 04:30:08.622 | INFO | Analyzing chunk frames = [350208,350720) | 20260505 04:30:08.622 | INFO | MemInfo: { | 20260505 04:30:08.622 | INFO | "gpu" : " 17 GB", | 20260505 04:30:08.622 | INFO | "huge" : "", | 20260505 04:30:08.622 | INFO | "huge_shared" : " 27 GB", | 20260505 04:30:08.622 | INFO | "malloc" : " 14 GB", | 20260505 04:30:08.622 | INFO | "pinned" : " 29 GB" | 20260505 04:30:08.623 | INFO | } | 20260505 04:30:08.623 | INFO | Perf report for the last 5.13933 seconds: | 20260505 04:30:08.623 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:30:08.623 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:08.623 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:08.623 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:08.623 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260505 04:30:08.623 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:08.623 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:08.623 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260505 04:30:08.623 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:08.623 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:08.623 | INFO | Uploaded 36GiB to GPU in 1.5996s (22.5056GiB/s) | 20260505 04:30:08.623 | INFO | Downloaded 34GiB to GPU in 1.51597s (22.4279GiB/s) | 20260505 04:30:13.762 | INFO | Analyzing chunk frames = [350720,351232) | 20260505 04:30:13.762 | INFO | MemInfo: { | 20260505 04:30:13.762 | INFO | "gpu" : " 17 GB", | 20260505 04:30:13.762 | INFO | "huge" : "", | 20260505 04:30:13.762 | INFO | "huge_shared" : " 27 GB", | 20260505 04:30:13.762 | INFO | "malloc" : " 14 GB", | 20260505 04:30:13.762 | INFO | "pinned" : " 29 GB" | 20260505 04:30:13.762 | INFO | } | 20260505 04:30:13.762 | INFO | Perf report for the last 5.13914 seconds: | 20260505 04:30:13.762 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:30:13.762 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:13.762 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:13.762 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:13.762 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260505 04:30:13.762 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:13.762 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:13.762 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260505 04:30:13.762 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:13.762 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:13.762 | INFO | Uploaded 36GiB to GPU in 1.59948s (22.5073GiB/s) | 20260505 04:30:13.762 | INFO | Downloaded 34GiB to GPU in 1.5158s (22.4305GiB/s) | 20260505 04:30:18.901 | INFO | Analyzing chunk frames = [351232,351744) | 20260505 04:30:18.901 | INFO | MemInfo: { | 20260505 04:30:18.901 | INFO | "gpu" : " 17 GB", | 20260505 04:30:18.901 | INFO | "huge" : "", | 20260505 04:30:18.901 | INFO | "huge_shared" : " 27 GB", | 20260505 04:30:18.901 | INFO | "malloc" : " 14 GB", | 20260505 04:30:18.901 | INFO | "pinned" : " 29 GB" | 20260505 04:30:18.901 | INFO | } | 20260505 04:30:18.901 | INFO | Perf report for the last 5.13954 seconds: | 20260505 04:30:18.901 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:30:18.901 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:18.901 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:18.901 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:18.901 | INFO | ANALYSIS : 57.6%, 3.0, 89.8ms, 0 | 20260505 04:30:18.901 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:18.901 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:18.901 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 04:30:18.901 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:18.901 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:18.902 | INFO | Uploaded 36GiB to GPU in 1.60079s (22.4889GiB/s) | 20260505 04:30:18.902 | INFO | Downloaded 34GiB to GPU in 1.51792s (22.3991GiB/s) | 20260505 04:30:24.040 | INFO | Analyzing chunk frames = [351744,352256) | 20260505 04:30:24.040 | INFO | MemInfo: { | 20260505 04:30:24.040 | INFO | "gpu" : " 17 GB", | 20260505 04:30:24.040 | INFO | "huge" : "", | 20260505 04:30:24.040 | INFO | "huge_shared" : " 27 GB", | 20260505 04:30:24.040 | INFO | "malloc" : " 14 GB", | 20260505 04:30:24.040 | INFO | "pinned" : " 29 GB" | 20260505 04:30:24.040 | INFO | } | 20260505 04:30:24.040 | INFO | Perf report for the last 5.13915 seconds: | 20260505 04:30:24.040 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:30:24.040 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:24.040 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:24.041 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:24.041 | INFO | ANALYSIS : 57.7%, 3.0, 90.0ms, 0 | 20260505 04:30:24.041 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:24.041 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:24.041 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 04:30:24.041 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:24.041 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:24.041 | INFO | Uploaded 36GiB to GPU in 1.60025s (22.4965GiB/s) | 20260505 04:30:24.041 | INFO | Downloaded 34GiB to GPU in 1.51717s (22.4102GiB/s) | 20260505 04:30:24.041 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 04:30:24.041 | INFO | Total mem usage: 14.9 GiB | 20260505 04:30:24.041 | INFO | All Time High Water Marks sum: 15.2 GiB (Savings: 2.07%) | 20260505 04:30:24.041 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -2.94%) | 20260505 04:30:24.041 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 04:30:24.041 | INFO | High Water Breakdown: | 20260505 04:30:24.041 | INFO | 59.1% : 8.58 GiB ( 100% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 04:30:24.041 | INFO | 40.7% : 5.9 GiB ( 88.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 04:30:24.041 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 04:30:24.041 | INFO | ------------------------------------------ | 20260505 04:30:24.041 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:30:24.041 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 04:30:24.041 | INFO | Total mem usage: 26.5 GiB | 20260505 04:30:24.041 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260505 04:30:24.041 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 04:30:24.041 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260505 04:30:24.041 | INFO | High Water Breakdown: | 20260505 04:30:24.041 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:30:24.041 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:30:24.041 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:30:24.041 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:30:24.041 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:30:24.041 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:30:24.041 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 04:30:24.041 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 04:30:24.041 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:30:24.041 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 04:30:24.041 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 04:30:24.041 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 04:30:24.041 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 04:30:24.041 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 04:30:24.041 | INFO | ------------------------------------------ | 20260505 04:30:24.041 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:30:24.041 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 04:30:24.041 | INFO | Total mem usage: 25.6 GiB | 20260505 04:30:24.041 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 04:30:24.041 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 04:30:24.041 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 04:30:24.041 | INFO | High Water Breakdown: | 20260505 04:30:24.041 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 04:30:24.041 | INFO | ------------------------------------------ | 20260505 04:30:24.041 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:30:24.041 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 04:30:24.041 | INFO | Total mem usage: 15.7 GiB | 20260505 04:30:24.041 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260505 04:30:24.041 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 04:30:24.041 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260505 04:30:24.041 | INFO | High Water Breakdown: | 20260505 04:30:24.041 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 04:30:24.041 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 04:30:24.041 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 04:30:24.041 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 04:30:24.041 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 04:30:24.041 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 04:30:24.041 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 04:30:24.041 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 04:30:24.041 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 04:30:24.041 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:30:24.041 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 04:30:24.041 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:30:24.041 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 04:30:24.041 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 04:30:24.041 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 04:30:24.041 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 04:30:24.041 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:30:24.041 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 04:30:24.041 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:30:24.041 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:30:24.041 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:30:24.041 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 04:30:24.041 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 04:30:24.041 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:30:24.041 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 04:30:24.041 | INFO | ------------------------------------------ | 20260505 04:30:24.041 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:30:24.041 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 04:30:24.042 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:30:24.042 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:30:24.042 | INFO | PerfReport: Full Execution -- 3.05e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260505 04:30:24.042 | INFO | PerfReport: 0.28%, AggregateStats -- 8.65e-02 / -nan, 5.28e-03 / 0.00e+00 | 20260505 04:30:24.042 | INFO | PerfReport: 10.81%, Baseline -- 3.30e+00 / -nan, 1.70e-02 / 0.00e+00 | 20260505 04:30:24.042 | INFO | PerfReport: 3.03%, Histogram -- 9.25e-01 / -nan, 5.22e-02 / 0.00e+00 | 20260505 04:30:24.042 | INFO | PerfReport: 16.03%, DME -- 3.91e+01 / -nan, 3.26e+00 / 0.00e+00 | 20260505 04:30:24.042 | INFO | PerfReport: 56.44%, FrameLabeling -- 1.72e+01 / -nan, 4.53e-02 / 0.00e+00 | 20260505 04:30:24.042 | INFO | PerfReport: 7.68%, PulseAccumulating -- 2.34e+00 / -nan, 1.57e-01 / 0.00e+00 | 20260505 04:30:24.042 | INFO | PerfReport: 4.05%, Metrics -- 1.23e+00 / -nan, 2.04e-01 / 0.00e+00 | 20260505 04:30:24.042 | INFO | PerfReport: 1.68%, PulseSerializing -- 5.12e-01 / -nan, 8.23e-02 / 0.00e+00 | 20260505 04:30:24.042 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 04:30:24.042 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:30:24.042 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:30:24.042 | INFO | PerfReport: Full Execution -- 9.16e+01 / -nan, 1.91e+01 / 0.00e+00 | 20260505 04:30:24.042 | INFO | PerfReport: 18.25%, Upload -- 1.67e+01 / 0.00e+00, 3.61e-01 / 0.00e+00 | 20260505 04:30:24.042 | INFO | PerfReport: 17.51%, Download -- 1.60e+01 / 0.00e+00, 6.86e-01 / 0.00e+00 | 20260505 04:30:24.042 | INFO | PerfReport: 33.31%, Compute -- 3.05e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260505 04:30:24.042 | INFO | PerfReport: 0.55%, LockUpload -- 5.04e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260505 04:30:24.042 | INFO | PerfReport: 0.00%, LockDownload -- 3.99e-04 / 0.00e+00, 1.52e-04 / 0.00e+00 | 20260505 04:30:24.042 | INFO | PerfReport: 30.36%, LockCompute -- 2.78e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260505 04:30:24.042 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.15e-02 / 0.00e+00 | 20260505 04:30:24.137 | INFO | Starting a round of Baz IO | 20260505 04:30:27.093 | INFO | Populating RTMetrics report | 20260505 04:30:27.093 | INFO | RTMetrics report populated | 20260505 04:30:27.093 | INFO | Opening tmp RTMetrics json file | 20260505 04:30:27.094 | INFO | Serializing RTMetrics to json | 20260505 04:30:27.095 | INFO | Serialization done, swapping tmp file now... | 20260505 04:30:27.096 | INFO | RTMetrics done | 20260505 04:30:27.180 | INFO | Finished a round of Baz IO: Wrote 1.962 GiB in 3.043 seconds | 20260505 04:30:27.180 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.83% / 0.15% / 52.02% | 20260505 04:30:27.180 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 10.9354% / 0.00042518% / 7.54992e-05% | 20260505 04:30:27.180 | INFO | Baz IO overall: | 20260505 04:30:27.180 | INFO | BytesWritten : 135762666583 | 20260505 04:30:27.180 | INFO | NumEvents : 62716840711 | 20260505 04:30:29.180 | INFO | Analyzing chunk frames = [352256,352768) | 20260505 04:30:29.180 | INFO | MemInfo: { | 20260505 04:30:29.180 | INFO | "gpu" : " 17 GB", | 20260505 04:30:29.180 | INFO | "huge" : "", | 20260505 04:30:29.180 | INFO | "huge_shared" : " 27 GB", | 20260505 04:30:29.180 | INFO | "malloc" : " 14 GB", | 20260505 04:30:29.180 | INFO | "pinned" : " 29 GB" | 20260505 04:30:29.180 | INFO | } | 20260505 04:30:29.180 | INFO | Perf report for the last 5.13955 seconds: | 20260505 04:30:29.180 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:30:29.180 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:29.180 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:29.180 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:29.180 | INFO | ANALYSIS : 58.3%, 3.0, 90.7ms, 0 | 20260505 04:30:29.180 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:29.180 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260505 04:30:29.180 | INFO | PRE_HQ : 57.7%, 2.1, 64.4ms, 0 | 20260505 04:30:29.180 | INFO | BAZWRITER : 58.8%, 3.9, 116.5ms, 0 | 20260505 04:30:29.180 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:30:29.180 | INFO | Uploaded 36GiB to GPU in 1.63619s (22.0024GiB/s) | 20260505 04:30:29.180 | INFO | Downloaded 38GiB to GPU in 1.69923s (22.3631GiB/s) | 20260505 04:30:34.319 | INFO | Analyzing chunk frames = [352768,353280) | 20260505 04:30:34.319 | INFO | MemInfo: { | 20260505 04:30:34.319 | INFO | "gpu" : " 17 GB", | 20260505 04:30:34.319 | INFO | "huge" : "", | 20260505 04:30:34.319 | INFO | "huge_shared" : " 27 GB", | 20260505 04:30:34.319 | INFO | "malloc" : " 14 GB", | 20260505 04:30:34.319 | INFO | "pinned" : " 29 GB" | 20260505 04:30:34.319 | INFO | } | 20260505 04:30:34.319 | INFO | Perf report for the last 5.13914 seconds: | 20260505 04:30:34.319 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:30:34.319 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:34.319 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:34.319 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:34.319 | INFO | ANALYSIS : 58.1%, 3.0, 90.6ms, 0 | 20260505 04:30:34.319 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:34.319 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:34.319 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260505 04:30:34.319 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:34.319 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:34.320 | INFO | Uploaded 36GiB to GPU in 1.59806s (22.5273GiB/s) | 20260505 04:30:34.320 | INFO | Downloaded 34GiB to GPU in 1.51467s (22.4472GiB/s) | 20260505 04:30:39.473 | INFO | Analyzing chunk frames = [353280,353792) | 20260505 04:30:39.473 | INFO | MemInfo: { | 20260505 04:30:39.473 | INFO | "gpu" : " 17 GB", | 20260505 04:30:39.473 | INFO | "huge" : "", | 20260505 04:30:39.473 | INFO | "huge_shared" : " 27 GB", | 20260505 04:30:39.473 | INFO | "malloc" : " 14 GB", | 20260505 04:30:39.473 | INFO | "pinned" : " 29 GB" | 20260505 04:30:39.473 | INFO | } | 20260505 04:30:39.473 | INFO | Perf report for the last 5.15426 seconds: | 20260505 04:30:39.473 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:30:39.473 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:39.474 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:39.474 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:39.474 | INFO | ANALYSIS : 57.3%, 3.0, 89.3ms, 0 | 20260505 04:30:39.474 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:39.474 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:39.474 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260505 04:30:39.474 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:39.474 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:39.474 | INFO | Uploaded 36GiB to GPU in 1.60181s (22.4746GiB/s) | 20260505 04:30:39.474 | INFO | Downloaded 34GiB to GPU in 1.51898s (22.3834GiB/s) | 20260505 04:30:44.599 | INFO | Analyzing chunk frames = [353792,354304) | 20260505 04:30:44.599 | INFO | MemInfo: { | 20260505 04:30:44.599 | INFO | "gpu" : " 17 GB", | 20260505 04:30:44.599 | INFO | "huge" : "", | 20260505 04:30:44.599 | INFO | "huge_shared" : " 27 GB", | 20260505 04:30:44.599 | INFO | "malloc" : " 14 GB", | 20260505 04:30:44.599 | INFO | "pinned" : " 29 GB" | 20260505 04:30:44.599 | INFO | } | 20260505 04:30:44.599 | INFO | Perf report for the last 5.12549 seconds: | 20260505 04:30:44.599 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:30:44.599 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:44.599 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:44.599 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:44.599 | INFO | ANALYSIS : 57.3%, 3.0, 88.7ms, 0 | 20260505 04:30:44.599 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:44.599 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:44.599 | INFO | PRE_HQ : 31.7%, 1.0, 16.4ms, 0 | 20260505 04:30:44.599 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:44.599 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:44.599 | INFO | Uploaded 36GiB to GPU in 1.60165s (22.4768GiB/s) | 20260505 04:30:44.599 | INFO | Downloaded 34GiB to GPU in 1.51756s (22.4044GiB/s) | 20260505 04:30:49.737 | INFO | Analyzing chunk frames = [354304,354816) | 20260505 04:30:49.737 | INFO | MemInfo: { | 20260505 04:30:49.737 | INFO | "gpu" : " 17 GB", | 20260505 04:30:49.737 | INFO | "huge" : "", | 20260505 04:30:49.737 | INFO | "huge_shared" : " 27 GB", | 20260505 04:30:49.737 | INFO | "malloc" : " 14 GB", | 20260505 04:30:49.737 | INFO | "pinned" : " 29 GB" | 20260505 04:30:49.737 | INFO | } | 20260505 04:30:49.737 | INFO | Perf report for the last 5.13827 seconds: | 20260505 04:30:49.737 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:30:49.737 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:49.737 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:49.737 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:49.737 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260505 04:30:49.737 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:49.737 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:49.737 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 04:30:49.737 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:49.737 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:49.738 | INFO | Uploaded 36GiB to GPU in 1.59938s (22.5088GiB/s) | 20260505 04:30:49.738 | INFO | Downloaded 34GiB to GPU in 1.51574s (22.4313GiB/s) | 20260505 04:30:54.876 | INFO | Analyzing chunk frames = [354816,355328) | 20260505 04:30:54.876 | INFO | MemInfo: { | 20260505 04:30:54.876 | INFO | "gpu" : " 17 GB", | 20260505 04:30:54.876 | INFO | "huge" : "", | 20260505 04:30:54.876 | INFO | "huge_shared" : " 27 GB", | 20260505 04:30:54.876 | INFO | "malloc" : " 14 GB", | 20260505 04:30:54.876 | INFO | "pinned" : " 29 GB" | 20260505 04:30:54.876 | INFO | } | 20260505 04:30:54.876 | INFO | Perf report for the last 5.1392 seconds: | 20260505 04:30:54.876 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:30:54.876 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:54.876 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:54.876 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:54.877 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 04:30:54.877 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:54.877 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:54.877 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 04:30:54.877 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:54.877 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:30:54.877 | INFO | Uploaded 36GiB to GPU in 1.59965s (22.5049GiB/s) | 20260505 04:30:54.877 | INFO | Downloaded 34GiB to GPU in 1.51641s (22.4213GiB/s) | 20260505 04:31:00.016 | INFO | Analyzing chunk frames = [355328,355840) | 20260505 04:31:00.016 | INFO | MemInfo: { | 20260505 04:31:00.016 | INFO | "gpu" : " 17 GB", | 20260505 04:31:00.016 | INFO | "huge" : "", | 20260505 04:31:00.016 | INFO | "huge_shared" : " 27 GB", | 20260505 04:31:00.016 | INFO | "malloc" : " 14 GB", | 20260505 04:31:00.016 | INFO | "pinned" : " 29 GB" | 20260505 04:31:00.016 | INFO | } | 20260505 04:31:00.016 | INFO | Perf report for the last 5.13948 seconds: | 20260505 04:31:00.016 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:31:00.016 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:00.016 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:00.016 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:00.016 | INFO | ANALYSIS : 57.6%, 3.0, 89.9ms, 0 | 20260505 04:31:00.016 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:00.016 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:00.016 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260505 04:31:00.016 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:00.016 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:00.017 | INFO | Uploaded 36GiB to GPU in 1.60015s (22.4979GiB/s) | 20260505 04:31:00.017 | INFO | Downloaded 34GiB to GPU in 1.51715s (22.4104GiB/s) | 20260505 04:31:05.155 | INFO | Analyzing chunk frames = [355840,356352) | 20260505 04:31:05.155 | INFO | MemInfo: { | 20260505 04:31:05.155 | INFO | "gpu" : " 17 GB", | 20260505 04:31:05.155 | INFO | "huge" : "", | 20260505 04:31:05.155 | INFO | "huge_shared" : " 27 GB", | 20260505 04:31:05.155 | INFO | "malloc" : " 14 GB", | 20260505 04:31:05.155 | INFO | "pinned" : " 29 GB" | 20260505 04:31:05.155 | INFO | } | 20260505 04:31:05.155 | INFO | Perf report for the last 5.13941 seconds: | 20260505 04:31:05.155 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:31:05.155 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:05.155 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:05.155 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:05.155 | INFO | ANALYSIS : 57.7%, 3.0, 89.9ms, 0 | 20260505 04:31:05.155 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:05.156 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:05.156 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 04:31:05.156 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:05.156 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:05.156 | INFO | Uploaded 36GiB to GPU in 1.60079s (22.4889GiB/s) | 20260505 04:31:05.156 | INFO | Downloaded 34GiB to GPU in 1.51804s (22.3973GiB/s) | 20260505 04:31:05.252 | INFO | Starting a round of Baz IO | 20260505 04:31:08.210 | INFO | Populating RTMetrics report | 20260505 04:31:08.211 | INFO | RTMetrics report populated | 20260505 04:31:08.211 | INFO | Opening tmp RTMetrics json file | 20260505 04:31:08.212 | INFO | Serializing RTMetrics to json | 20260505 04:31:08.213 | INFO | Serialization done, swapping tmp file now... | 20260505 04:31:08.213 | INFO | RTMetrics done | 20260505 04:31:08.297 | INFO | Finished a round of Baz IO: Wrote 1.967 GiB in 3.044 seconds | 20260505 04:31:08.297 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.68% / 0.15% / 52.17% | 20260505 04:31:08.297 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 10.908% / 0.000349681% / 5.16574e-05% | 20260505 04:31:08.297 | INFO | Baz IO overall: | 20260505 04:31:08.297 | INFO | BytesWritten : 137874927839 | 20260505 04:31:08.297 | INFO | NumEvents : 63715840271 | 20260505 04:31:10.294 | INFO | Analyzing chunk frames = [356352,356864) | 20260505 04:31:10.294 | INFO | MemInfo: { | 20260505 04:31:10.294 | INFO | "gpu" : " 17 GB", | 20260505 04:31:10.294 | INFO | "huge" : "", | 20260505 04:31:10.294 | INFO | "huge_shared" : " 27 GB", | 20260505 04:31:10.294 | INFO | "malloc" : " 14 GB", | 20260505 04:31:10.294 | INFO | "pinned" : " 29 GB" | 20260505 04:31:10.294 | INFO | } | 20260505 04:31:10.294 | INFO | Perf report for the last 5.13905 seconds: | 20260505 04:31:10.295 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:31:10.295 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:10.295 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:10.295 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:10.295 | INFO | ANALYSIS : 58.3%, 3.0, 90.8ms, 0 | 20260505 04:31:10.295 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:10.295 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260505 04:31:10.295 | INFO | PRE_HQ : 57.7%, 2.1, 64.3ms, 0 | 20260505 04:31:10.295 | INFO | BAZWRITER : 58.8%, 3.8, 115.9ms, 0 | 20260505 04:31:10.295 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:31:10.295 | INFO | Uploaded 36GiB to GPU in 1.64049s (21.9447GiB/s) | 20260505 04:31:10.295 | INFO | Downloaded 38GiB to GPU in 1.70345s (22.3077GiB/s) | 20260505 04:31:15.434 | INFO | Analyzing chunk frames = [356864,357376) | 20260505 04:31:15.434 | INFO | MemInfo: { | 20260505 04:31:15.434 | INFO | "gpu" : " 17 GB", | 20260505 04:31:15.434 | INFO | "huge" : "", | 20260505 04:31:15.434 | INFO | "huge_shared" : " 27 GB", | 20260505 04:31:15.434 | INFO | "malloc" : " 14 GB", | 20260505 04:31:15.434 | INFO | "pinned" : " 29 GB" | 20260505 04:31:15.434 | INFO | } | 20260505 04:31:15.434 | INFO | Perf report for the last 5.13951 seconds: | 20260505 04:31:15.434 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:31:15.434 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:15.434 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:15.434 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:15.434 | INFO | ANALYSIS : 58.1%, 3.0, 90.6ms, 0 | 20260505 04:31:15.434 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:15.434 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:15.434 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 04:31:15.434 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:15.434 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:15.435 | INFO | Uploaded 36GiB to GPU in 1.59828s (22.5242GiB/s) | 20260505 04:31:15.435 | INFO | Downloaded 34GiB to GPU in 1.51456s (22.4488GiB/s) | 20260505 04:31:20.573 | INFO | Analyzing chunk frames = [357376,357888) | 20260505 04:31:20.573 | INFO | MemInfo: { | 20260505 04:31:20.573 | INFO | "gpu" : " 17 GB", | 20260505 04:31:20.573 | INFO | "huge" : "", | 20260505 04:31:20.573 | INFO | "huge_shared" : " 27 GB", | 20260505 04:31:20.573 | INFO | "malloc" : " 14 GB", | 20260505 04:31:20.573 | INFO | "pinned" : " 29 GB" | 20260505 04:31:20.573 | INFO | } | 20260505 04:31:20.573 | INFO | Perf report for the last 5.13939 seconds: | 20260505 04:31:20.573 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:31:20.573 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:20.573 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:20.573 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:20.573 | INFO | ANALYSIS : 57.5%, 3.0, 89.4ms, 0 | 20260505 04:31:20.573 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:20.574 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:20.574 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260505 04:31:20.574 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:20.574 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:20.574 | INFO | Uploaded 36GiB to GPU in 1.60146s (22.4796GiB/s) | 20260505 04:31:20.574 | INFO | Downloaded 34GiB to GPU in 1.51837s (22.3924GiB/s) | 20260505 04:31:25.712 | INFO | Analyzing chunk frames = [357888,358400) | 20260505 04:31:25.712 | INFO | MemInfo: { | 20260505 04:31:25.712 | INFO | "gpu" : " 17 GB", | 20260505 04:31:25.712 | INFO | "huge" : "", | 20260505 04:31:25.712 | INFO | "huge_shared" : " 27 GB", | 20260505 04:31:25.712 | INFO | "malloc" : " 14 GB", | 20260505 04:31:25.713 | INFO | "pinned" : " 29 GB" | 20260505 04:31:25.713 | INFO | } | 20260505 04:31:25.713 | INFO | Perf report for the last 5.13932 seconds: | 20260505 04:31:25.713 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:31:25.713 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:25.713 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:25.713 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:25.713 | INFO | ANALYSIS : 57.1%, 3.0, 88.6ms, 0 | 20260505 04:31:25.713 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:25.713 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:25.713 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 04:31:25.713 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:25.713 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:25.713 | INFO | Uploaded 36GiB to GPU in 1.60162s (22.4773GiB/s) | 20260505 04:31:25.713 | INFO | Downloaded 34GiB to GPU in 1.51801s (22.3977GiB/s) | 20260505 04:31:30.852 | INFO | Analyzing chunk frames = [358400,358912) | 20260505 04:31:30.852 | INFO | MemInfo: { | 20260505 04:31:30.852 | INFO | "gpu" : " 17 GB", | 20260505 04:31:30.852 | INFO | "huge" : "", | 20260505 04:31:30.852 | INFO | "huge_shared" : " 27 GB", | 20260505 04:31:30.852 | INFO | "malloc" : " 14 GB", | 20260505 04:31:30.852 | INFO | "pinned" : " 29 GB" | 20260505 04:31:30.852 | INFO | } | 20260505 04:31:30.852 | INFO | Perf report for the last 5.1393 seconds: | 20260505 04:31:30.852 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:31:30.852 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:30.852 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:30.852 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:30.852 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260505 04:31:30.852 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:30.852 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:30.852 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260505 04:31:30.852 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:30.852 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:30.853 | INFO | Uploaded 36GiB to GPU in 1.59937s (22.5088GiB/s) | 20260505 04:31:30.853 | INFO | Downloaded 34GiB to GPU in 1.5161s (22.426GiB/s) | 20260505 04:31:35.991 | INFO | Analyzing chunk frames = [358912,359424) | 20260505 04:31:35.991 | INFO | MemInfo: { | 20260505 04:31:35.991 | INFO | "gpu" : " 17 GB", | 20260505 04:31:35.991 | INFO | "huge" : "", | 20260505 04:31:35.991 | INFO | "huge_shared" : " 27 GB", | 20260505 04:31:35.991 | INFO | "malloc" : " 14 GB", | 20260505 04:31:35.991 | INFO | "pinned" : " 29 GB" | 20260505 04:31:35.991 | INFO | } | 20260505 04:31:35.991 | INFO | Perf report for the last 5.13931 seconds: | 20260505 04:31:35.991 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:31:35.991 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:35.991 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:35.991 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:35.991 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260505 04:31:35.991 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:35.991 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:35.991 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260505 04:31:35.992 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:35.992 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:35.992 | INFO | Uploaded 36GiB to GPU in 1.59979s (22.5029GiB/s) | 20260505 04:31:35.992 | INFO | Downloaded 34GiB to GPU in 1.51646s (22.4207GiB/s) | 20260505 04:31:41.130 | INFO | Analyzing chunk frames = [359424,359936) | 20260505 04:31:41.130 | INFO | MemInfo: { | 20260505 04:31:41.130 | INFO | "gpu" : " 17 GB", | 20260505 04:31:41.130 | INFO | "huge" : "", | 20260505 04:31:41.130 | INFO | "huge_shared" : " 27 GB", | 20260505 04:31:41.130 | INFO | "malloc" : " 14 GB", | 20260505 04:31:41.131 | INFO | "pinned" : " 29 GB" | 20260505 04:31:41.131 | INFO | } | 20260505 04:31:41.131 | INFO | Perf report for the last 5.1393 seconds: | 20260505 04:31:41.131 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:31:41.131 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:41.131 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:41.131 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:41.131 | INFO | ANALYSIS : 57.6%, 3.0, 89.9ms, 0 | 20260505 04:31:41.131 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:41.131 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:41.131 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260505 04:31:41.131 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:41.131 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:41.131 | INFO | Uploaded 36GiB to GPU in 1.6002s (22.4971GiB/s) | 20260505 04:31:41.131 | INFO | Downloaded 34GiB to GPU in 1.51747s (22.4057GiB/s) | 20260505 04:31:46.270 | INFO | Analyzing chunk frames = [359936,360448) | 20260505 04:31:46.270 | INFO | MemInfo: { | 20260505 04:31:46.270 | INFO | "gpu" : " 17 GB", | 20260505 04:31:46.270 | INFO | "huge" : "", | 20260505 04:31:46.270 | INFO | "huge_shared" : " 27 GB", | 20260505 04:31:46.270 | INFO | "malloc" : " 14 GB", | 20260505 04:31:46.270 | INFO | "pinned" : " 29 GB" | 20260505 04:31:46.270 | INFO | } | 20260505 04:31:46.270 | INFO | Perf report for the last 5.1395 seconds: | 20260505 04:31:46.270 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:31:46.270 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:46.270 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:46.270 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:46.270 | INFO | ANALYSIS : 57.7%, 3.0, 90.0ms, 0 | 20260505 04:31:46.270 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:46.270 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:46.270 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260505 04:31:46.270 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:46.270 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:46.271 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 04:31:46.271 | INFO | Total mem usage: 14.9 GiB | 20260505 04:31:46.271 | INFO | All Time High Water Marks sum: 15.2 GiB (Savings: 2.08%) | 20260505 04:31:46.271 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -2.96%) | 20260505 04:31:46.271 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 04:31:46.271 | INFO | High Water Breakdown: | 20260505 04:31:46.271 | INFO | 59.2% : 8.58 GiB ( 100% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 04:31:46.271 | INFO | 40.7% : 5.9 GiB ( 88.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 04:31:46.271 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 04:31:46.271 | INFO | ------------------------------------------ | 20260505 04:31:46.271 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:31:46.271 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 04:31:46.271 | INFO | Total mem usage: 26.5 GiB | 20260505 04:31:46.271 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260505 04:31:46.271 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 04:31:46.271 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260505 04:31:46.271 | INFO | High Water Breakdown: | 20260505 04:31:46.271 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:31:46.271 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:31:46.271 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:31:46.271 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:31:46.271 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:31:46.271 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:31:46.271 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 04:31:46.271 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 04:31:46.271 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:31:46.271 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 04:31:46.271 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 04:31:46.271 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 04:31:46.271 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 04:31:46.271 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 04:31:46.271 | INFO | ------------------------------------------ | 20260505 04:31:46.271 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:31:46.271 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 04:31:46.271 | INFO | Total mem usage: 25.6 GiB | 20260505 04:31:46.271 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 04:31:46.271 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 04:31:46.271 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 04:31:46.271 | INFO | High Water Breakdown: | 20260505 04:31:46.271 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 04:31:46.271 | INFO | ------------------------------------------ | 20260505 04:31:46.271 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:31:46.271 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 04:31:46.271 | INFO | Total mem usage: 15.7 GiB | 20260505 04:31:46.271 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260505 04:31:46.271 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 04:31:46.271 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260505 04:31:46.271 | INFO | High Water Breakdown: | 20260505 04:31:46.271 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 04:31:46.271 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 04:31:46.271 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 04:31:46.271 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 04:31:46.271 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 04:31:46.271 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 04:31:46.271 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 04:31:46.271 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 04:31:46.271 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 04:31:46.271 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:31:46.271 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 04:31:46.271 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:31:46.271 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 04:31:46.271 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 04:31:46.271 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 04:31:46.271 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 04:31:46.271 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:31:46.271 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 04:31:46.271 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:31:46.271 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:31:46.271 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:31:46.271 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 04:31:46.271 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 04:31:46.271 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:31:46.271 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 04:31:46.271 | INFO | ------------------------------------------ | 20260505 04:31:46.271 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:31:46.271 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 04:31:46.271 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:31:46.271 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:31:46.271 | INFO | PerfReport: Full Execution -- 3.05e+01 / -nan, 1.28e+01 / 0.00e+00 | 20260505 04:31:46.271 | INFO | PerfReport: 0.28%, AggregateStats -- 8.63e-02 / -nan, 3.89e-03 / 0.00e+00 | 20260505 04:31:46.271 | INFO | PerfReport: 10.81%, Baseline -- 3.30e+00 / -nan, 1.72e-02 / 0.00e+00 | 20260505 04:31:46.271 | INFO | PerfReport: 3.03%, Histogram -- 9.24e-01 / -nan, 5.23e-02 / 0.00e+00 | 20260505 04:31:46.271 | INFO | PerfReport: 16.03%, DME -- 3.91e+01 / -nan, 3.20e+00 / 0.00e+00 | 20260505 04:31:46.271 | INFO | PerfReport: 56.45%, FrameLabeling -- 1.72e+01 / -nan, 4.53e-02 / 0.00e+00 | 20260505 04:31:46.271 | INFO | PerfReport: 7.68%, PulseAccumulating -- 2.34e+00 / -nan, 1.57e-01 / 0.00e+00 | 20260505 04:31:46.271 | INFO | PerfReport: 4.04%, Metrics -- 1.23e+00 / -nan, 2.03e-01 / 0.00e+00 | 20260505 04:31:46.271 | INFO | PerfReport: 1.68%, PulseSerializing -- 5.11e-01 / -nan, 8.25e-02 / 0.00e+00 | 20260505 04:31:46.271 | INFO | Uploaded 36GiB to GPU in 1.601s (22.486GiB/s) | 20260505 04:31:46.271 | INFO | Downloaded 34GiB to GPU in 1.51812s (22.3962GiB/s) | 20260505 04:31:46.271 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 04:31:46.272 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:31:46.272 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:31:46.272 | INFO | PerfReport: Full Execution -- 9.16e+01 / -nan, 1.91e+01 / 0.00e+00 | 20260505 04:31:46.272 | INFO | PerfReport: 18.26%, Upload -- 1.67e+01 / 0.00e+00, 3.67e-01 / 0.00e+00 | 20260505 04:31:46.272 | INFO | PerfReport: 17.52%, Download -- 1.60e+01 / 0.00e+00, 6.96e-01 / 0.00e+00 | 20260505 04:31:46.272 | INFO | PerfReport: 33.30%, Compute -- 3.05e+01 / 0.00e+00, 1.28e+01 / 0.00e+00 | 20260505 04:31:46.272 | INFO | PerfReport: 0.55%, LockUpload -- 5.03e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260505 04:31:46.272 | INFO | PerfReport: 0.00%, LockDownload -- 3.89e-04 / 0.00e+00, 8.69e-05 / 0.00e+00 | 20260505 04:31:46.272 | INFO | PerfReport: 30.36%, LockCompute -- 2.78e+01 / 0.00e+00, 1.72e+01 / 0.00e+00 | 20260505 04:31:46.272 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.40e-02 / 0.00e+00 | 20260505 04:31:46.367 | INFO | Starting a round of Baz IO | 20260505 04:31:49.326 | INFO | Populating RTMetrics report | 20260505 04:31:49.326 | INFO | RTMetrics report populated | 20260505 04:31:49.326 | INFO | Opening tmp RTMetrics json file | 20260505 04:31:49.328 | INFO | Serializing RTMetrics to json | 20260505 04:31:49.329 | INFO | Serialization done, swapping tmp file now... | 20260505 04:31:49.329 | INFO | RTMetrics done | 20260505 04:31:49.413 | INFO | Finished a round of Baz IO: Wrote 1.973 GiB in 3.045 seconds | 20260505 04:31:49.413 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.53% / 0.15% / 52.32% | 20260505 04:31:49.413 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 10.879% / 0.000325839% / 4.76837e-05% | 20260505 04:31:49.413 | INFO | Baz IO overall: | 20260505 04:31:49.413 | INFO | BytesWritten : 139993934888 | 20260505 04:31:49.413 | INFO | NumEvents : 64718820284 | 20260505 04:31:51.409 | INFO | Analyzing chunk frames = [360448,360960) | 20260505 04:31:51.409 | INFO | MemInfo: { | 20260505 04:31:51.409 | INFO | "gpu" : " 17 GB", | 20260505 04:31:51.409 | INFO | "huge" : "", | 20260505 04:31:51.409 | INFO | "huge_shared" : " 27 GB", | 20260505 04:31:51.409 | INFO | "malloc" : " 14 GB", | 20260505 04:31:51.409 | INFO | "pinned" : " 29 GB" | 20260505 04:31:51.409 | INFO | } | 20260505 04:31:51.409 | INFO | Perf report for the last 5.13903 seconds: | 20260505 04:31:51.409 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:31:51.409 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:51.409 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:51.409 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:51.409 | INFO | ANALYSIS : 58.3%, 3.0, 90.8ms, 0 | 20260505 04:31:51.409 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:51.409 | INFO | RT_METRICS : 45.1%, 1.0, 23.6ms, 0 | 20260505 04:31:51.409 | INFO | PRE_HQ : 57.8%, 2.2, 64.6ms, 0 | 20260505 04:31:51.409 | INFO | BAZWRITER : 58.6%, 3.9, 116.8ms, 0 | 20260505 04:31:51.409 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 04:31:51.410 | INFO | Uploaded 36GiB to GPU in 1.63685s (21.9935GiB/s) | 20260505 04:31:51.410 | INFO | Downloaded 38GiB to GPU in 1.70037s (22.3481GiB/s) | 20260505 04:31:56.549 | INFO | Analyzing chunk frames = [360960,361472) | 20260505 04:31:56.549 | INFO | MemInfo: { | 20260505 04:31:56.549 | INFO | "gpu" : " 17 GB", | 20260505 04:31:56.549 | INFO | "huge" : "", | 20260505 04:31:56.549 | INFO | "huge_shared" : " 27 GB", | 20260505 04:31:56.549 | INFO | "malloc" : " 14 GB", | 20260505 04:31:56.549 | INFO | "pinned" : " 29 GB" | 20260505 04:31:56.549 | INFO | } | 20260505 04:31:56.549 | INFO | Perf report for the last 5.13962 seconds: | 20260505 04:31:56.549 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:31:56.549 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:56.549 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:56.549 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:56.549 | INFO | ANALYSIS : 58.1%, 3.0, 90.6ms, 0 | 20260505 04:31:56.549 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:56.549 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:56.549 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 04:31:56.549 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:56.549 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:31:56.549 | INFO | Uploaded 36GiB to GPU in 1.59816s (22.5259GiB/s) | 20260505 04:31:56.549 | INFO | Downloaded 34GiB to GPU in 1.51472s (22.4464GiB/s) | 20260505 04:32:01.688 | INFO | Analyzing chunk frames = [361472,361984) | 20260505 04:32:01.688 | INFO | MemInfo: { | 20260505 04:32:01.688 | INFO | "gpu" : " 17 GB", | 20260505 04:32:01.688 | INFO | "huge" : "", | 20260505 04:32:01.688 | INFO | "huge_shared" : " 27 GB", | 20260505 04:32:01.688 | INFO | "malloc" : " 14 GB", | 20260505 04:32:01.688 | INFO | "pinned" : " 29 GB" | 20260505 04:32:01.688 | INFO | } | 20260505 04:32:01.688 | INFO | Perf report for the last 5.13922 seconds: | 20260505 04:32:01.688 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:32:01.688 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:01.688 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:01.688 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:01.688 | INFO | ANALYSIS : 57.5%, 3.0, 89.4ms, 0 | 20260505 04:32:01.688 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:01.688 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:01.688 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 04:32:01.688 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:01.688 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:01.689 | INFO | Uploaded 36GiB to GPU in 1.60249s (22.465GiB/s) | 20260505 04:32:01.689 | INFO | Downloaded 34GiB to GPU in 1.52022s (22.3652GiB/s) | 20260505 04:32:06.827 | INFO | Analyzing chunk frames = [361984,362496) | 20260505 04:32:06.827 | INFO | MemInfo: { | 20260505 04:32:06.827 | INFO | "gpu" : " 17 GB", | 20260505 04:32:06.827 | INFO | "huge" : "", | 20260505 04:32:06.827 | INFO | "huge_shared" : " 27 GB", | 20260505 04:32:06.827 | INFO | "malloc" : " 14 GB", | 20260505 04:32:06.827 | INFO | "pinned" : " 29 GB" | 20260505 04:32:06.827 | INFO | } | 20260505 04:32:06.827 | INFO | Perf report for the last 5.13938 seconds: | 20260505 04:32:06.828 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:32:06.828 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:06.828 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:06.828 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:06.828 | INFO | ANALYSIS : 57.2%, 3.0, 88.6ms, 0 | 20260505 04:32:06.828 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:06.828 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:06.828 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 04:32:06.828 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:06.828 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:06.828 | INFO | Uploaded 36GiB to GPU in 1.6015s (22.4789GiB/s) | 20260505 04:32:06.828 | INFO | Downloaded 34GiB to GPU in 1.51853s (22.3901GiB/s) | 20260505 04:32:11.967 | INFO | Analyzing chunk frames = [362496,363008) | 20260505 04:32:11.967 | INFO | MemInfo: { | 20260505 04:32:11.967 | INFO | "gpu" : " 17 GB", | 20260505 04:32:11.967 | INFO | "huge" : "", | 20260505 04:32:11.967 | INFO | "huge_shared" : " 27 GB", | 20260505 04:32:11.967 | INFO | "malloc" : " 14 GB", | 20260505 04:32:11.967 | INFO | "pinned" : " 29 GB" | 20260505 04:32:11.967 | INFO | } | 20260505 04:32:11.967 | INFO | Perf report for the last 5.13934 seconds: | 20260505 04:32:11.967 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:32:11.967 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:11.967 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:11.967 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:11.967 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260505 04:32:11.967 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:11.967 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:11.967 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260505 04:32:11.967 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:11.967 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:11.967 | INFO | Uploaded 36GiB to GPU in 1.59847s (22.5215GiB/s) | 20260505 04:32:11.968 | INFO | Downloaded 34GiB to GPU in 1.51567s (22.4323GiB/s) | 20260505 04:32:17.106 | INFO | Analyzing chunk frames = [363008,363520) | 20260505 04:32:17.106 | INFO | MemInfo: { | 20260505 04:32:17.106 | INFO | "gpu" : " 17 GB", | 20260505 04:32:17.106 | INFO | "huge" : "", | 20260505 04:32:17.106 | INFO | "huge_shared" : " 27 GB", | 20260505 04:32:17.106 | INFO | "malloc" : " 14 GB", | 20260505 04:32:17.106 | INFO | "pinned" : " 29 GB" | 20260505 04:32:17.106 | INFO | } | 20260505 04:32:17.106 | INFO | Perf report for the last 5.13933 seconds: | 20260505 04:32:17.106 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:32:17.106 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:17.106 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:17.106 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:17.106 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260505 04:32:17.106 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:17.106 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:17.106 | INFO | PRE_HQ : 31.9%, 1.0, 16.5ms, 0 | 20260505 04:32:17.106 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:17.106 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:17.106 | INFO | Uploaded 36GiB to GPU in 1.60011s (22.4984GiB/s) | 20260505 04:32:17.106 | INFO | Downloaded 34GiB to GPU in 1.51764s (22.4031GiB/s) | 20260505 04:32:22.245 | INFO | Analyzing chunk frames = [363520,364032) | 20260505 04:32:22.245 | INFO | MemInfo: { | 20260505 04:32:22.245 | INFO | "gpu" : " 17 GB", | 20260505 04:32:22.245 | INFO | "huge" : "", | 20260505 04:32:22.245 | INFO | "huge_shared" : " 27 GB", | 20260505 04:32:22.245 | INFO | "malloc" : " 14 GB", | 20260505 04:32:22.245 | INFO | "pinned" : " 29 GB" | 20260505 04:32:22.245 | INFO | } | 20260505 04:32:22.245 | INFO | Perf report for the last 5.1393 seconds: | 20260505 04:32:22.246 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:32:22.246 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:22.246 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:22.246 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:22.246 | INFO | ANALYSIS : 57.6%, 3.0, 89.9ms, 0 | 20260505 04:32:22.246 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:22.246 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:22.246 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260505 04:32:22.246 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:22.246 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:22.246 | INFO | Uploaded 36GiB to GPU in 1.59999s (22.5002GiB/s) | 20260505 04:32:22.246 | INFO | Downloaded 34GiB to GPU in 1.51696s (22.4132GiB/s) | 20260505 04:32:27.385 | INFO | Analyzing chunk frames = [364032,364544) | 20260505 04:32:27.385 | INFO | MemInfo: { | 20260505 04:32:27.385 | INFO | "gpu" : " 17 GB", | 20260505 04:32:27.385 | INFO | "huge" : "", | 20260505 04:32:27.385 | INFO | "huge_shared" : " 27 GB", | 20260505 04:32:27.385 | INFO | "malloc" : " 14 GB", | 20260505 04:32:27.385 | INFO | "pinned" : " 29 GB" | 20260505 04:32:27.385 | INFO | } | 20260505 04:32:27.385 | INFO | Perf report for the last 5.13935 seconds: | 20260505 04:32:27.385 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:32:27.385 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:27.385 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:27.385 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:27.385 | INFO | ANALYSIS : 57.6%, 3.0, 89.9ms, 0 | 20260505 04:32:27.385 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:27.385 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:27.385 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 04:32:27.385 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:27.385 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:27.385 | INFO | Uploaded 36GiB to GPU in 1.60074s (22.4896GiB/s) | 20260505 04:32:27.386 | INFO | Downloaded 34GiB to GPU in 1.51838s (22.3923GiB/s) | 20260505 04:32:27.482 | INFO | Starting a round of Baz IO | 20260505 04:32:30.439 | INFO | Populating RTMetrics report | 20260505 04:32:30.440 | INFO | RTMetrics report populated | 20260505 04:32:30.440 | INFO | Opening tmp RTMetrics json file | 20260505 04:32:30.440 | INFO | Serializing RTMetrics to json | 20260505 04:32:30.441 | INFO | Serialization done, swapping tmp file now... | 20260505 04:32:30.442 | INFO | RTMetrics done | 20260505 04:32:30.526 | INFO | Finished a round of Baz IO: Wrote 1.976 GiB in 3.043 seconds | 20260505 04:32:30.526 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.38% / 0.15% / 52.47% | 20260505 04:32:30.526 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 10.8089% / 0.000369549% / 4.76837e-05% | 20260505 04:32:30.526 | INFO | Baz IO overall: | 20260505 04:32:30.526 | INFO | BytesWritten : 142115948244 | 20260505 04:32:30.526 | INFO | NumEvents : 65722823320 | 20260505 04:32:32.524 | INFO | Analyzing chunk frames = [364544,365056) | 20260505 04:32:32.524 | INFO | MemInfo: { | 20260505 04:32:32.524 | INFO | "gpu" : " 17 GB", | 20260505 04:32:32.524 | INFO | "huge" : "", | 20260505 04:32:32.524 | INFO | "huge_shared" : " 27 GB", | 20260505 04:32:32.524 | INFO | "malloc" : " 14 GB", | 20260505 04:32:32.524 | INFO | "pinned" : " 29 GB" | 20260505 04:32:32.524 | INFO | } | 20260505 04:32:32.524 | INFO | Perf report for the last 5.1391 seconds: | 20260505 04:32:32.524 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:32:32.524 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:32.524 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:32.524 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:32.524 | INFO | ANALYSIS : 58.3%, 3.0, 90.8ms, 0 | 20260505 04:32:32.524 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:32.524 | INFO | RT_METRICS : 45.3%, 1.0, 23.7ms, 0 | 20260505 04:32:32.524 | INFO | PRE_HQ : 57.7%, 2.2, 64.5ms, 0 | 20260505 04:32:32.524 | INFO | BAZWRITER : 58.8%, 3.9, 117.1ms, 0 | 20260505 04:32:32.524 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:32:32.525 | INFO | Uploaded 36GiB to GPU in 1.64234s (21.92GiB/s) | 20260505 04:32:32.525 | INFO | Downloaded 38GiB to GPU in 1.70363s (22.3053GiB/s) | 20260505 04:32:37.663 | INFO | Analyzing chunk frames = [365056,365568) | 20260505 04:32:37.663 | INFO | MemInfo: { | 20260505 04:32:37.663 | INFO | "gpu" : " 17 GB", | 20260505 04:32:37.663 | INFO | "huge" : "", | 20260505 04:32:37.663 | INFO | "huge_shared" : " 27 GB", | 20260505 04:32:37.663 | INFO | "malloc" : " 14 GB", | 20260505 04:32:37.663 | INFO | "pinned" : " 29 GB" | 20260505 04:32:37.664 | INFO | } | 20260505 04:32:37.664 | INFO | Perf report for the last 5.13958 seconds: | 20260505 04:32:37.664 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:32:37.664 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:37.664 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:37.664 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:37.664 | INFO | ANALYSIS : 58.1%, 3.0, 90.6ms, 0 | 20260505 04:32:37.664 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:37.664 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:37.664 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260505 04:32:37.664 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:37.664 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:37.664 | INFO | Uploaded 36GiB to GPU in 1.59783s (22.5306GiB/s) | 20260505 04:32:37.664 | INFO | Downloaded 34GiB to GPU in 1.51422s (22.4539GiB/s) | 20260505 04:32:42.803 | INFO | Analyzing chunk frames = [365568,366080) | 20260505 04:32:42.803 | INFO | MemInfo: { | 20260505 04:32:42.803 | INFO | "gpu" : " 17 GB", | 20260505 04:32:42.803 | INFO | "huge" : "", | 20260505 04:32:42.803 | INFO | "huge_shared" : " 27 GB", | 20260505 04:32:42.803 | INFO | "malloc" : " 14 GB", | 20260505 04:32:42.803 | INFO | "pinned" : " 29 GB" | 20260505 04:32:42.803 | INFO | } | 20260505 04:32:42.803 | INFO | Perf report for the last 5.13925 seconds: | 20260505 04:32:42.803 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:32:42.803 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:42.803 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:42.803 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:42.803 | INFO | ANALYSIS : 57.5%, 3.0, 89.3ms, 0 | 20260505 04:32:42.803 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:42.803 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:42.803 | INFO | PRE_HQ : 31.7%, 1.0, 16.4ms, 0 | 20260505 04:32:42.803 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:42.803 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:42.803 | INFO | Uploaded 36GiB to GPU in 1.60203s (22.4715GiB/s) | 20260505 04:32:42.803 | INFO | Downloaded 34GiB to GPU in 1.51914s (22.381GiB/s) | 20260505 04:32:47.942 | INFO | Analyzing chunk frames = [366080,366592) | 20260505 04:32:47.942 | INFO | MemInfo: { | 20260505 04:32:47.942 | INFO | "gpu" : " 17 GB", | 20260505 04:32:47.942 | INFO | "huge" : "", | 20260505 04:32:47.942 | INFO | "huge_shared" : " 27 GB", | 20260505 04:32:47.942 | INFO | "malloc" : " 14 GB", | 20260505 04:32:47.942 | INFO | "pinned" : " 29 GB" | 20260505 04:32:47.942 | INFO | } | 20260505 04:32:47.942 | INFO | Perf report for the last 5.1396 seconds: | 20260505 04:32:47.942 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:32:47.942 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:47.942 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:47.942 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:47.943 | INFO | ANALYSIS : 57.2%, 3.0, 88.7ms, 0 | 20260505 04:32:47.943 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:47.943 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:47.943 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260505 04:32:47.943 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:47.943 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:47.943 | INFO | Uploaded 36GiB to GPU in 1.60198s (22.4722GiB/s) | 20260505 04:32:47.943 | INFO | Downloaded 34GiB to GPU in 1.51907s (22.3821GiB/s) | 20260505 04:32:53.081 | INFO | Analyzing chunk frames = [366592,367104) | 20260505 04:32:53.081 | INFO | MemInfo: { | 20260505 04:32:53.081 | INFO | "gpu" : " 17 GB", | 20260505 04:32:53.081 | INFO | "huge" : "", | 20260505 04:32:53.081 | INFO | "huge_shared" : " 27 GB", | 20260505 04:32:53.081 | INFO | "malloc" : " 14 GB", | 20260505 04:32:53.081 | INFO | "pinned" : " 29 GB" | 20260505 04:32:53.081 | INFO | } | 20260505 04:32:53.081 | INFO | Perf report for the last 5.13904 seconds: | 20260505 04:32:53.081 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:32:53.082 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:53.082 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:53.082 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:53.082 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260505 04:32:53.082 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:53.082 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:53.082 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260505 04:32:53.082 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:53.082 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:53.082 | INFO | Uploaded 36GiB to GPU in 1.59992s (22.5012GiB/s) | 20260505 04:32:53.082 | INFO | Downloaded 34GiB to GPU in 1.51591s (22.4288GiB/s) | 20260505 04:32:58.221 | INFO | Analyzing chunk frames = [367104,367616) | 20260505 04:32:58.221 | INFO | MemInfo: { | 20260505 04:32:58.221 | INFO | "gpu" : " 17 GB", | 20260505 04:32:58.221 | INFO | "huge" : "", | 20260505 04:32:58.221 | INFO | "huge_shared" : " 27 GB", | 20260505 04:32:58.221 | INFO | "malloc" : " 14 GB", | 20260505 04:32:58.221 | INFO | "pinned" : " 29 GB" | 20260505 04:32:58.221 | INFO | } | 20260505 04:32:58.221 | INFO | Perf report for the last 5.13922 seconds: | 20260505 04:32:58.221 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:32:58.221 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:58.221 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:58.221 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:58.221 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260505 04:32:58.221 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:58.221 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:58.221 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260505 04:32:58.221 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:58.221 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:32:58.221 | INFO | Uploaded 36GiB to GPU in 1.59958s (22.5059GiB/s) | 20260505 04:32:58.221 | INFO | Downloaded 34GiB to GPU in 1.5165s (22.42GiB/s) | 20260505 04:33:03.360 | INFO | Analyzing chunk frames = [367616,368128) | 20260505 04:33:03.360 | INFO | MemInfo: { | 20260505 04:33:03.360 | INFO | "gpu" : " 17 GB", | 20260505 04:33:03.360 | INFO | "huge" : "", | 20260505 04:33:03.360 | INFO | "huge_shared" : " 27 GB", | 20260505 04:33:03.360 | INFO | "malloc" : " 14 GB", | 20260505 04:33:03.360 | INFO | "pinned" : " 29 GB" | 20260505 04:33:03.360 | INFO | } | 20260505 04:33:03.360 | INFO | Perf report for the last 5.13961 seconds: | 20260505 04:33:03.360 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:33:03.360 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:03.360 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:03.360 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:03.360 | INFO | ANALYSIS : 57.6%, 3.0, 89.8ms, 0 | 20260505 04:33:03.360 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:03.360 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:03.360 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260505 04:33:03.360 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:03.360 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:03.361 | INFO | Uploaded 36GiB to GPU in 1.60094s (22.4868GiB/s) | 20260505 04:33:03.361 | INFO | Downloaded 34GiB to GPU in 1.518s (22.3979GiB/s) | 20260505 04:33:08.499 | INFO | Analyzing chunk frames = [368128,368640) | 20260505 04:33:08.499 | INFO | MemInfo: { | 20260505 04:33:08.499 | INFO | "gpu" : " 17 GB", | 20260505 04:33:08.499 | INFO | "huge" : "", | 20260505 04:33:08.500 | INFO | "huge_shared" : " 27 GB", | 20260505 04:33:08.500 | INFO | "malloc" : " 14 GB", | 20260505 04:33:08.500 | INFO | "pinned" : " 29 GB" | 20260505 04:33:08.500 | INFO | } | 20260505 04:33:08.500 | INFO | Perf report for the last 5.13941 seconds: | 20260505 04:33:08.500 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:33:08.500 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:08.500 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:08.500 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:08.500 | INFO | ANALYSIS : 57.7%, 3.0, 90.0ms, 0 | 20260505 04:33:08.500 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:08.500 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:08.500 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 04:33:08.500 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:08.500 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:08.500 | INFO | Uploaded 36GiB to GPU in 1.60038s (22.4947GiB/s) | 20260505 04:33:08.500 | INFO | Downloaded 34GiB to GPU in 1.51761s (22.4037GiB/s) | 20260505 04:33:08.500 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 04:33:08.500 | INFO | Total mem usage: 14.9 GiB | 20260505 04:33:08.500 | INFO | All Time High Water Marks sum: 15.2 GiB (Savings: 2.08%) | 20260505 04:33:08.500 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -3.1%) | 20260505 04:33:08.500 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 04:33:08.500 | INFO | High Water Breakdown: | 20260505 04:33:08.500 | INFO | 59.2% : 8.56 GiB ( 99.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 04:33:08.500 | INFO | 40.7% : 5.89 GiB ( 88.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 04:33:08.500 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 04:33:08.500 | INFO | ------------------------------------------ | 20260505 04:33:08.500 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:33:08.500 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 04:33:08.500 | INFO | Total mem usage: 26.5 GiB | 20260505 04:33:08.500 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260505 04:33:08.501 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 04:33:08.501 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260505 04:33:08.501 | INFO | High Water Breakdown: | 20260505 04:33:08.501 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:33:08.501 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:33:08.501 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:33:08.501 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:33:08.501 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:33:08.501 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:33:08.501 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 04:33:08.501 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 04:33:08.501 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:33:08.501 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 04:33:08.501 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 04:33:08.501 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 04:33:08.501 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 04:33:08.501 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 04:33:08.501 | INFO | ------------------------------------------ | 20260505 04:33:08.501 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:33:08.501 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 04:33:08.501 | INFO | Total mem usage: 25.6 GiB | 20260505 04:33:08.501 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 04:33:08.501 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 04:33:08.501 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 04:33:08.501 | INFO | High Water Breakdown: | 20260505 04:33:08.501 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 04:33:08.501 | INFO | ------------------------------------------ | 20260505 04:33:08.501 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:33:08.501 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 04:33:08.501 | INFO | Total mem usage: 15.7 GiB | 20260505 04:33:08.501 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260505 04:33:08.501 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 04:33:08.501 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260505 04:33:08.501 | INFO | High Water Breakdown: | 20260505 04:33:08.501 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 04:33:08.501 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 04:33:08.501 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 04:33:08.501 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 04:33:08.501 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 04:33:08.501 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 04:33:08.501 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 04:33:08.501 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 04:33:08.501 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 04:33:08.501 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:33:08.501 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 04:33:08.501 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:33:08.501 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 04:33:08.501 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 04:33:08.501 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 04:33:08.501 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 04:33:08.501 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:33:08.501 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 04:33:08.501 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:33:08.501 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:33:08.501 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:33:08.501 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 04:33:08.501 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 04:33:08.501 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:33:08.501 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 04:33:08.501 | INFO | ------------------------------------------ | 20260505 04:33:08.501 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:33:08.501 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 04:33:08.501 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:33:08.501 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:33:08.501 | INFO | PerfReport: Full Execution -- 3.05e+01 / -nan, 1.28e+01 / 0.00e+00 | 20260505 04:33:08.501 | INFO | PerfReport: 0.28%, AggregateStats -- 8.66e-02 / -nan, 6.47e-03 / 0.00e+00 | 20260505 04:33:08.501 | INFO | PerfReport: 10.81%, Baseline -- 3.30e+00 / -nan, 1.90e-02 / 0.00e+00 | 20260505 04:33:08.501 | INFO | PerfReport: 3.03%, Histogram -- 9.24e-01 / -nan, 5.23e-02 / 0.00e+00 | 20260505 04:33:08.501 | INFO | PerfReport: 16.03%, DME -- 3.91e+01 / -nan, 3.17e+00 / 0.00e+00 | 20260505 04:33:08.501 | INFO | PerfReport: 56.45%, FrameLabeling -- 1.72e+01 / -nan, 4.42e-02 / 0.00e+00 | 20260505 04:33:08.501 | INFO | PerfReport: 7.67%, PulseAccumulating -- 2.34e+00 / -nan, 1.56e-01 / 0.00e+00 | 20260505 04:33:08.501 | INFO | PerfReport: 4.04%, Metrics -- 1.23e+00 / -nan, 2.02e-01 / 0.00e+00 | 20260505 04:33:08.501 | INFO | PerfReport: 1.67%, PulseSerializing -- 5.10e-01 / -nan, 8.20e-02 / 0.00e+00 | 20260505 04:33:08.501 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 04:33:08.501 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:33:08.501 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:33:08.501 | INFO | PerfReport: Full Execution -- 9.16e+01 / -nan, 1.89e+01 / 0.00e+00 | 20260505 04:33:08.501 | INFO | PerfReport: 18.26%, Upload -- 1.67e+01 / 0.00e+00, 3.70e-01 / 0.00e+00 | 20260505 04:33:08.501 | INFO | PerfReport: 17.52%, Download -- 1.60e+01 / 0.00e+00, 6.98e-01 / 0.00e+00 | 20260505 04:33:08.501 | INFO | PerfReport: 33.30%, Compute -- 3.05e+01 / 0.00e+00, 1.28e+01 / 0.00e+00 | 20260505 04:33:08.501 | INFO | PerfReport: 0.55%, LockUpload -- 5.04e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260505 04:33:08.501 | INFO | PerfReport: 0.00%, LockDownload -- 3.99e-04 / 0.00e+00, 1.44e-04 / 0.00e+00 | 20260505 04:33:08.501 | INFO | PerfReport: 30.36%, LockCompute -- 2.78e+01 / 0.00e+00, 1.72e+01 / 0.00e+00 | 20260505 04:33:08.501 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.57e-02 / 0.00e+00 | 20260505 04:33:08.596 | INFO | Starting a round of Baz IO | 20260505 04:33:11.553 | INFO | Populating RTMetrics report | 20260505 04:33:11.553 | INFO | RTMetrics report populated | 20260505 04:33:11.553 | INFO | Opening tmp RTMetrics json file | 20260505 04:33:11.554 | INFO | Serializing RTMetrics to json | 20260505 04:33:11.555 | INFO | Serialization done, swapping tmp file now... | 20260505 04:33:11.556 | INFO | RTMetrics done | 20260505 04:33:11.640 | INFO | Finished a round of Baz IO: Wrote 1.982 GiB in 3.043 seconds | 20260505 04:33:11.640 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.24% / 0.14% / 52.62% | 20260505 04:33:11.640 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 10.7433% / 0.000377496% / 5.16574e-05% | 20260505 04:33:11.640 | INFO | Baz IO overall: | 20260505 04:33:11.640 | INFO | BytesWritten : 144243822252 | 20260505 04:33:11.640 | INFO | NumEvents : 66729888847 | 20260505 04:33:13.639 | INFO | Analyzing chunk frames = [368640,369152) | 20260505 04:33:13.639 | INFO | MemInfo: { | 20260505 04:33:13.639 | INFO | "gpu" : " 17 GB", | 20260505 04:33:13.639 | INFO | "huge" : "", | 20260505 04:33:13.639 | INFO | "huge_shared" : " 27 GB", | 20260505 04:33:13.639 | INFO | "malloc" : " 14 GB", | 20260505 04:33:13.639 | INFO | "pinned" : " 29 GB" | 20260505 04:33:13.639 | INFO | } | 20260505 04:33:13.639 | INFO | Perf report for the last 5.139 seconds: | 20260505 04:33:13.639 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:33:13.639 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:13.639 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:13.639 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:13.639 | INFO | ANALYSIS : 58.3%, 3.0, 90.7ms, 0 | 20260505 04:33:13.639 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:13.639 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260505 04:33:13.639 | INFO | PRE_HQ : 57.6%, 2.2, 64.3ms, 0 | 20260505 04:33:13.639 | INFO | BAZWRITER : 58.8%, 3.9, 117.1ms, 0 | 20260505 04:33:13.639 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:33:13.639 | INFO | Uploaded 36GiB to GPU in 1.64139s (21.9327GiB/s) | 20260505 04:33:13.639 | INFO | Downloaded 38GiB to GPU in 1.70382s (22.3029GiB/s) | 20260505 04:33:18.778 | INFO | Analyzing chunk frames = [369152,369664) | 20260505 04:33:18.778 | INFO | MemInfo: { | 20260505 04:33:18.778 | INFO | "gpu" : " 17 GB", | 20260505 04:33:18.778 | INFO | "huge" : "", | 20260505 04:33:18.778 | INFO | "huge_shared" : " 27 GB", | 20260505 04:33:18.778 | INFO | "malloc" : " 14 GB", | 20260505 04:33:18.778 | INFO | "pinned" : " 29 GB" | 20260505 04:33:18.778 | INFO | } | 20260505 04:33:18.778 | INFO | Perf report for the last 5.13941 seconds: | 20260505 04:33:18.778 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:33:18.778 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:18.778 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:18.778 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:18.778 | INFO | ANALYSIS : 58.1%, 3.0, 90.6ms, 0 | 20260505 04:33:18.778 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:18.778 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:18.778 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260505 04:33:18.778 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:18.778 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:18.779 | INFO | Uploaded 36GiB to GPU in 1.59842s (22.5222GiB/s) | 20260505 04:33:18.779 | INFO | Downloaded 34GiB to GPU in 1.51479s (22.4454GiB/s) | 20260505 04:33:23.917 | INFO | Analyzing chunk frames = [369664,370176) | 20260505 04:33:23.917 | INFO | MemInfo: { | 20260505 04:33:23.917 | INFO | "gpu" : " 17 GB", | 20260505 04:33:23.917 | INFO | "huge" : "", | 20260505 04:33:23.917 | INFO | "huge_shared" : " 27 GB", | 20260505 04:33:23.917 | INFO | "malloc" : " 14 GB", | 20260505 04:33:23.917 | INFO | "pinned" : " 29 GB" | 20260505 04:33:23.917 | INFO | } | 20260505 04:33:23.918 | INFO | Perf report for the last 5.13934 seconds: | 20260505 04:33:23.918 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:33:23.918 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:23.918 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:23.918 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:23.918 | INFO | ANALYSIS : 57.4%, 3.0, 89.2ms, 0 | 20260505 04:33:23.918 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:23.918 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:23.918 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 04:33:23.918 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:23.918 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:23.918 | INFO | Uploaded 36GiB to GPU in 1.60208s (22.4709GiB/s) | 20260505 04:33:23.918 | INFO | Downloaded 34GiB to GPU in 1.5194s (22.3773GiB/s) | 20260505 04:33:29.057 | INFO | Analyzing chunk frames = [370176,370688) | 20260505 04:33:29.057 | INFO | MemInfo: { | 20260505 04:33:29.057 | INFO | "gpu" : " 17 GB", | 20260505 04:33:29.057 | INFO | "huge" : "", | 20260505 04:33:29.057 | INFO | "huge_shared" : " 27 GB", | 20260505 04:33:29.057 | INFO | "malloc" : " 14 GB", | 20260505 04:33:29.057 | INFO | "pinned" : " 29 GB" | 20260505 04:33:29.057 | INFO | } | 20260505 04:33:29.057 | INFO | Perf report for the last 5.13965 seconds: | 20260505 04:33:29.057 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:33:29.057 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:29.057 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:29.057 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:29.057 | INFO | ANALYSIS : 57.1%, 3.0, 88.6ms, 0 | 20260505 04:33:29.057 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:29.057 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:29.057 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260505 04:33:29.057 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:29.057 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:29.057 | INFO | Uploaded 36GiB to GPU in 1.60128s (22.4821GiB/s) | 20260505 04:33:29.058 | INFO | Downloaded 34GiB to GPU in 1.51792s (22.3991GiB/s) | 20260505 04:33:34.196 | INFO | Analyzing chunk frames = [370688,371200) | 20260505 04:33:34.196 | INFO | MemInfo: { | 20260505 04:33:34.196 | INFO | "gpu" : " 17 GB", | 20260505 04:33:34.196 | INFO | "huge" : "", | 20260505 04:33:34.196 | INFO | "huge_shared" : " 27 GB", | 20260505 04:33:34.196 | INFO | "malloc" : " 14 GB", | 20260505 04:33:34.196 | INFO | "pinned" : " 29 GB" | 20260505 04:33:34.196 | INFO | } | 20260505 04:33:34.196 | INFO | Perf report for the last 5.13911 seconds: | 20260505 04:33:34.196 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:33:34.196 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:34.196 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:34.196 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:34.196 | INFO | ANALYSIS : 57.0%, 3.0, 89.0ms, 0 | 20260505 04:33:34.196 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:34.196 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:34.196 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260505 04:33:34.196 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:34.197 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:34.197 | INFO | Uploaded 36GiB to GPU in 1.60016s (22.4978GiB/s) | 20260505 04:33:34.197 | INFO | Downloaded 34GiB to GPU in 1.51663s (22.4181GiB/s) | 20260505 04:33:39.345 | INFO | Analyzing chunk frames = [371200,371712) | 20260505 04:33:39.345 | INFO | MemInfo: { | 20260505 04:33:39.345 | INFO | "gpu" : " 17 GB", | 20260505 04:33:39.345 | INFO | "huge" : "", | 20260505 04:33:39.345 | INFO | "huge_shared" : " 27 GB", | 20260505 04:33:39.345 | INFO | "malloc" : " 14 GB", | 20260505 04:33:39.345 | INFO | "pinned" : " 29 GB" | 20260505 04:33:39.346 | INFO | } | 20260505 04:33:39.346 | INFO | Perf report for the last 5.14919 seconds: | 20260505 04:33:39.346 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:33:39.346 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:39.346 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:39.346 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:39.346 | INFO | ANALYSIS : 57.3%, 3.0, 89.5ms, 0 | 20260505 04:33:39.346 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:39.346 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:39.346 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 04:33:39.346 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:39.346 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:39.346 | INFO | Uploaded 36GiB to GPU in 1.59967s (22.5047GiB/s) | 20260505 04:33:39.346 | INFO | Downloaded 34GiB to GPU in 1.51664s (22.418GiB/s) | 20260505 04:33:44.476 | INFO | Analyzing chunk frames = [371712,372224) | 20260505 04:33:44.476 | INFO | MemInfo: { | 20260505 04:33:44.476 | INFO | "gpu" : " 17 GB", | 20260505 04:33:44.476 | INFO | "huge" : "", | 20260505 04:33:44.476 | INFO | "huge_shared" : " 27 GB", | 20260505 04:33:44.476 | INFO | "malloc" : " 14 GB", | 20260505 04:33:44.476 | INFO | "pinned" : " 29 GB" | 20260505 04:33:44.476 | INFO | } | 20260505 04:33:44.476 | INFO | Perf report for the last 5.13046 seconds: | 20260505 04:33:44.476 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:33:44.476 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:44.476 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:44.476 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:44.476 | INFO | ANALYSIS : 57.7%, 3.0, 89.8ms, 0 | 20260505 04:33:44.476 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:44.476 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:44.476 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260505 04:33:44.476 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:44.476 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:44.476 | INFO | Uploaded 36GiB to GPU in 1.60097s (22.4863GiB/s) | 20260505 04:33:44.476 | INFO | Downloaded 34GiB to GPU in 1.51798s (22.3982GiB/s) | 20260505 04:33:49.614 | INFO | Analyzing chunk frames = [372224,372736) | 20260505 04:33:49.614 | INFO | MemInfo: { | 20260505 04:33:49.614 | INFO | "gpu" : " 17 GB", | 20260505 04:33:49.614 | INFO | "huge" : "", | 20260505 04:33:49.614 | INFO | "huge_shared" : " 27 GB", | 20260505 04:33:49.614 | INFO | "malloc" : " 14 GB", | 20260505 04:33:49.614 | INFO | "pinned" : " 29 GB" | 20260505 04:33:49.614 | INFO | } | 20260505 04:33:49.614 | INFO | Perf report for the last 5.13833 seconds: | 20260505 04:33:49.614 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:33:49.614 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:49.614 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:49.614 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:49.614 | INFO | ANALYSIS : 57.7%, 3.0, 90.0ms, 0 | 20260505 04:33:49.614 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:49.615 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:49.615 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260505 04:33:49.615 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:49.615 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:49.615 | INFO | Uploaded 36GiB to GPU in 1.60027s (22.4963GiB/s) | 20260505 04:33:49.615 | INFO | Downloaded 34GiB to GPU in 1.51735s (22.4075GiB/s) | 20260505 04:33:49.711 | INFO | Starting a round of Baz IO | 20260505 04:33:52.666 | INFO | Populating RTMetrics report | 20260505 04:33:52.667 | INFO | RTMetrics report populated | 20260505 04:33:52.667 | INFO | Opening tmp RTMetrics json file | 20260505 04:33:52.667 | INFO | Serializing RTMetrics to json | 20260505 04:33:52.668 | INFO | Serialization done, swapping tmp file now... | 20260505 04:33:52.669 | INFO | RTMetrics done | 20260505 04:33:52.755 | INFO | Finished a round of Baz IO: Wrote 1.987 GiB in 3.043 seconds | 20260505 04:33:52.755 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.09% / 0.14% / 52.76% | 20260505 04:33:52.755 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 10.6802% / 0.000325839% / 5.5631e-05% | 20260505 04:33:52.755 | INFO | Baz IO overall: | 20260505 04:33:52.755 | INFO | BytesWritten : 146376885317 | 20260505 04:33:52.755 | INFO | NumEvents : 67739251348 | 20260505 04:33:54.753 | INFO | Analyzing chunk frames = [372736,373248) | 20260505 04:33:54.753 | INFO | MemInfo: { | 20260505 04:33:54.753 | INFO | "gpu" : " 17 GB", | 20260505 04:33:54.753 | INFO | "huge" : "", | 20260505 04:33:54.754 | INFO | "huge_shared" : " 27 GB", | 20260505 04:33:54.754 | INFO | "malloc" : " 14 GB", | 20260505 04:33:54.754 | INFO | "pinned" : " 29 GB" | 20260505 04:33:54.754 | INFO | } | 20260505 04:33:54.754 | INFO | Perf report for the last 5.13924 seconds: | 20260505 04:33:54.754 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:33:54.754 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:54.754 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:54.754 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:54.754 | INFO | ANALYSIS : 58.3%, 3.0, 90.7ms, 0 | 20260505 04:33:54.754 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:54.754 | INFO | RT_METRICS : 45.1%, 1.0, 23.6ms, 0 | 20260505 04:33:54.754 | INFO | PRE_HQ : 57.7%, 2.2, 64.7ms, 0 | 20260505 04:33:54.754 | INFO | BAZWRITER : 58.8%, 3.9, 117.5ms, 0 | 20260505 04:33:54.754 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:33:54.754 | INFO | Uploaded 36GiB to GPU in 1.63863s (21.9696GiB/s) | 20260505 04:33:54.754 | INFO | Downloaded 38GiB to GPU in 1.70085s (22.3418GiB/s) | 20260505 04:33:59.893 | INFO | Analyzing chunk frames = [373248,373760) | 20260505 04:33:59.893 | INFO | MemInfo: { | 20260505 04:33:59.893 | INFO | "gpu" : " 17 GB", | 20260505 04:33:59.893 | INFO | "huge" : "", | 20260505 04:33:59.893 | INFO | "huge_shared" : " 27 GB", | 20260505 04:33:59.893 | INFO | "malloc" : " 14 GB", | 20260505 04:33:59.893 | INFO | "pinned" : " 29 GB" | 20260505 04:33:59.893 | INFO | } | 20260505 04:33:59.893 | INFO | Perf report for the last 5.13922 seconds: | 20260505 04:33:59.893 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:33:59.893 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:59.893 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:59.893 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:59.893 | INFO | ANALYSIS : 58.1%, 3.0, 90.6ms, 0 | 20260505 04:33:59.893 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:59.893 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:59.893 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 04:33:59.893 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:59.893 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:33:59.893 | INFO | Uploaded 36GiB to GPU in 1.59792s (22.5293GiB/s) | 20260505 04:33:59.894 | INFO | Downloaded 34GiB to GPU in 1.51441s (22.4509GiB/s) | 20260505 04:34:05.032 | INFO | Analyzing chunk frames = [373760,374272) | 20260505 04:34:05.032 | INFO | MemInfo: { | 20260505 04:34:05.032 | INFO | "gpu" : " 17 GB", | 20260505 04:34:05.032 | INFO | "huge" : "", | 20260505 04:34:05.032 | INFO | "huge_shared" : " 27 GB", | 20260505 04:34:05.032 | INFO | "malloc" : " 14 GB", | 20260505 04:34:05.032 | INFO | "pinned" : " 29 GB" | 20260505 04:34:05.032 | INFO | } | 20260505 04:34:05.032 | INFO | Perf report for the last 5.13932 seconds: | 20260505 04:34:05.032 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:34:05.032 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:05.032 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:05.032 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:05.032 | INFO | ANALYSIS : 57.4%, 3.0, 89.2ms, 0 | 20260505 04:34:05.032 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:05.032 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:05.032 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 04:34:05.032 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:05.032 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:05.032 | INFO | Uploaded 36GiB to GPU in 1.60236s (22.4668GiB/s) | 20260505 04:34:05.033 | INFO | Downloaded 34GiB to GPU in 1.5199s (22.3698GiB/s) | 20260505 04:34:10.171 | INFO | Analyzing chunk frames = [374272,374784) | 20260505 04:34:10.171 | INFO | MemInfo: { | 20260505 04:34:10.171 | INFO | "gpu" : " 17 GB", | 20260505 04:34:10.172 | INFO | "huge" : "", | 20260505 04:34:10.172 | INFO | "huge_shared" : " 27 GB", | 20260505 04:34:10.172 | INFO | "malloc" : " 14 GB", | 20260505 04:34:10.172 | INFO | "pinned" : " 29 GB" | 20260505 04:34:10.172 | INFO | } | 20260505 04:34:10.172 | INFO | Perf report for the last 5.13955 seconds: | 20260505 04:34:10.172 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:34:10.172 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:10.172 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:10.172 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:10.172 | INFO | ANALYSIS : 57.1%, 3.0, 88.6ms, 0 | 20260505 04:34:10.172 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:10.172 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:10.172 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260505 04:34:10.172 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:10.172 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:10.172 | INFO | Uploaded 36GiB to GPU in 1.60144s (22.4798GiB/s) | 20260505 04:34:10.172 | INFO | Downloaded 34GiB to GPU in 1.51782s (22.4006GiB/s) | 20260505 04:34:15.311 | INFO | Analyzing chunk frames = [374784,375296) | 20260505 04:34:15.311 | INFO | MemInfo: { | 20260505 04:34:15.311 | INFO | "gpu" : " 17 GB", | 20260505 04:34:15.311 | INFO | "huge" : "", | 20260505 04:34:15.311 | INFO | "huge_shared" : " 27 GB", | 20260505 04:34:15.311 | INFO | "malloc" : " 14 GB", | 20260505 04:34:15.311 | INFO | "pinned" : " 29 GB" | 20260505 04:34:15.311 | INFO | } | 20260505 04:34:15.311 | INFO | Perf report for the last 5.13936 seconds: | 20260505 04:34:15.311 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:34:15.311 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:15.311 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:15.311 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:15.311 | INFO | ANALYSIS : 57.0%, 3.0, 89.0ms, 0 | 20260505 04:34:15.311 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:15.311 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:15.311 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 04:34:15.311 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:15.311 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:15.311 | INFO | Uploaded 36GiB to GPU in 1.60009s (22.4988GiB/s) | 20260505 04:34:15.311 | INFO | Downloaded 34GiB to GPU in 1.51655s (22.4193GiB/s) | 20260505 04:34:20.450 | INFO | Analyzing chunk frames = [375296,375808) | 20260505 04:34:20.450 | INFO | MemInfo: { | 20260505 04:34:20.450 | INFO | "gpu" : " 17 GB", | 20260505 04:34:20.450 | INFO | "huge" : "", | 20260505 04:34:20.450 | INFO | "huge_shared" : " 27 GB", | 20260505 04:34:20.450 | INFO | "malloc" : " 14 GB", | 20260505 04:34:20.450 | INFO | "pinned" : " 29 GB" | 20260505 04:34:20.450 | INFO | } | 20260505 04:34:20.450 | INFO | Perf report for the last 5.13926 seconds: | 20260505 04:34:20.450 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:34:20.450 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:20.450 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:20.450 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:20.450 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260505 04:34:20.450 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:20.451 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:20.451 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 04:34:20.451 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:20.451 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:20.451 | INFO | Uploaded 36GiB to GPU in 1.59983s (22.5024GiB/s) | 20260505 04:34:20.451 | INFO | Downloaded 34GiB to GPU in 1.51645s (22.4208GiB/s) | 20260505 04:34:25.590 | INFO | Analyzing chunk frames = [375808,376320) | 20260505 04:34:25.590 | INFO | MemInfo: { | 20260505 04:34:25.590 | INFO | "gpu" : " 17 GB", | 20260505 04:34:25.590 | INFO | "huge" : "", | 20260505 04:34:25.590 | INFO | "huge_shared" : " 27 GB", | 20260505 04:34:25.590 | INFO | "malloc" : " 14 GB", | 20260505 04:34:25.590 | INFO | "pinned" : " 29 GB" | 20260505 04:34:25.590 | INFO | } | 20260505 04:34:25.590 | INFO | Perf report for the last 5.13957 seconds: | 20260505 04:34:25.590 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:34:25.590 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:25.590 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:25.590 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:25.590 | INFO | ANALYSIS : 57.6%, 3.0, 89.9ms, 0 | 20260505 04:34:25.590 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:25.590 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:25.590 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260505 04:34:25.590 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:25.590 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:25.590 | INFO | Uploaded 36GiB to GPU in 1.6003s (22.4958GiB/s) | 20260505 04:34:25.590 | INFO | Downloaded 34GiB to GPU in 1.51691s (22.4139GiB/s) | 20260505 04:34:30.729 | INFO | Analyzing chunk frames = [376320,376832) | 20260505 04:34:30.729 | INFO | MemInfo: { | 20260505 04:34:30.729 | INFO | "gpu" : " 17 GB", | 20260505 04:34:30.729 | INFO | "huge" : "", | 20260505 04:34:30.729 | INFO | "huge_shared" : " 27 GB", | 20260505 04:34:30.729 | INFO | "malloc" : " 14 GB", | 20260505 04:34:30.729 | INFO | "pinned" : " 29 GB" | 20260505 04:34:30.729 | INFO | } | 20260505 04:34:30.729 | INFO | Perf report for the last 5.13906 seconds: | 20260505 04:34:30.729 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:34:30.729 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:30.729 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:30.729 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:30.729 | INFO | ANALYSIS : 57.7%, 3.0, 89.9ms, 0 | 20260505 04:34:30.729 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:30.729 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:30.729 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260505 04:34:30.729 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:30.729 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:30.729 | INFO | Uploaded 36GiB to GPU in 1.60052s (22.4926GiB/s) | 20260505 04:34:30.729 | INFO | Downloaded 34GiB to GPU in 1.51774s (22.4018GiB/s) | 20260505 04:34:30.729 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 04:34:30.729 | INFO | Total mem usage: 14.9 GiB | 20260505 04:34:30.729 | INFO | All Time High Water Marks sum: 15.2 GiB (Savings: 2.08%) | 20260505 04:34:30.729 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -3.19%) | 20260505 04:34:30.729 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 04:34:30.729 | INFO | High Water Breakdown: | 20260505 04:34:30.729 | INFO | 59.2% : 8.56 GiB ( 99.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 04:34:30.729 | INFO | 40.7% : 5.89 GiB ( 88.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 04:34:30.729 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 04:34:30.729 | INFO | ------------------------------------------ | 20260505 04:34:30.729 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:34:30.729 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 04:34:30.729 | INFO | Total mem usage: 26.5 GiB | 20260505 04:34:30.729 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260505 04:34:30.730 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 04:34:30.730 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260505 04:34:30.730 | INFO | High Water Breakdown: | 20260505 04:34:30.730 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:34:30.730 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:34:30.730 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:34:30.730 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:34:30.730 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:34:30.730 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:34:30.730 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 04:34:30.730 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 04:34:30.730 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:34:30.730 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 04:34:30.730 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 04:34:30.730 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 04:34:30.730 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 04:34:30.730 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 04:34:30.730 | INFO | ------------------------------------------ | 20260505 04:34:30.730 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:34:30.730 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 04:34:30.730 | INFO | Total mem usage: 25.6 GiB | 20260505 04:34:30.730 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 04:34:30.730 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 04:34:30.730 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 04:34:30.730 | INFO | High Water Breakdown: | 20260505 04:34:30.730 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 04:34:30.730 | INFO | ------------------------------------------ | 20260505 04:34:30.730 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:34:30.730 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 04:34:30.730 | INFO | Total mem usage: 15.7 GiB | 20260505 04:34:30.730 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260505 04:34:30.730 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 04:34:30.730 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260505 04:34:30.730 | INFO | High Water Breakdown: | 20260505 04:34:30.730 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 04:34:30.730 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 04:34:30.730 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 04:34:30.730 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 04:34:30.730 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 04:34:30.730 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 04:34:30.730 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 04:34:30.730 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 04:34:30.730 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 04:34:30.730 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:34:30.730 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 04:34:30.730 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:34:30.730 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 04:34:30.730 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 04:34:30.730 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 04:34:30.730 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 04:34:30.730 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:34:30.730 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 04:34:30.730 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:34:30.730 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:34:30.730 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:34:30.730 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 04:34:30.730 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 04:34:30.730 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:34:30.730 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 04:34:30.730 | INFO | ------------------------------------------ | 20260505 04:34:30.730 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:34:30.730 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 04:34:30.730 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:34:30.730 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:34:30.730 | INFO | PerfReport: Full Execution -- 3.05e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260505 04:34:30.730 | INFO | PerfReport: 0.28%, AggregateStats -- 8.64e-02 / -nan, 4.31e-03 / 0.00e+00 | 20260505 04:34:30.730 | INFO | PerfReport: 10.81%, Baseline -- 3.30e+00 / -nan, 1.64e-02 / 0.00e+00 | 20260505 04:34:30.730 | INFO | PerfReport: 3.03%, Histogram -- 9.24e-01 / -nan, 5.17e-02 / 0.00e+00 | 20260505 04:34:30.730 | INFO | PerfReport: 16.03%, DME -- 3.91e+01 / -nan, 3.26e+00 / 0.00e+00 | 20260505 04:34:30.730 | INFO | PerfReport: 56.47%, FrameLabeling -- 1.72e+01 / -nan, 4.47e-02 / 0.00e+00 | 20260505 04:34:30.730 | INFO | PerfReport: 7.67%, PulseAccumulating -- 2.34e+00 / -nan, 1.55e-01 / 0.00e+00 | 20260505 04:34:30.730 | INFO | PerfReport: 4.03%, Metrics -- 1.23e+00 / -nan, 2.03e-01 / 0.00e+00 | 20260505 04:34:30.730 | INFO | PerfReport: 1.67%, PulseSerializing -- 5.09e-01 / -nan, 8.23e-02 / 0.00e+00 | 20260505 04:34:30.730 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 04:34:30.730 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:34:30.730 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:34:30.730 | INFO | PerfReport: Full Execution -- 9.15e+01 / -nan, 1.89e+01 / 0.00e+00 | 20260505 04:34:30.731 | INFO | PerfReport: 18.27%, Upload -- 1.67e+01 / 0.00e+00, 3.69e-01 / 0.00e+00 | 20260505 04:34:30.731 | INFO | PerfReport: 17.53%, Download -- 1.60e+01 / 0.00e+00, 7.02e-01 / 0.00e+00 | 20260505 04:34:30.731 | INFO | PerfReport: 33.30%, Compute -- 3.05e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260505 04:34:30.731 | INFO | PerfReport: 0.55%, LockUpload -- 5.04e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260505 04:34:30.731 | INFO | PerfReport: 0.00%, LockDownload -- 3.98e-04 / 0.00e+00, 1.86e-04 / 0.00e+00 | 20260505 04:34:30.731 | INFO | PerfReport: 30.34%, LockCompute -- 2.78e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260505 04:34:30.731 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 9.91e-03 / 0.00e+00 | 20260505 04:34:30.826 | INFO | Starting a round of Baz IO | 20260505 04:34:33.783 | INFO | Populating RTMetrics report | 20260505 04:34:33.783 | INFO | RTMetrics report populated | 20260505 04:34:33.783 | INFO | Opening tmp RTMetrics json file | 20260505 04:34:33.784 | INFO | Serializing RTMetrics to json | 20260505 04:34:33.785 | INFO | Serialization done, swapping tmp file now... | 20260505 04:34:33.786 | INFO | RTMetrics done | 20260505 04:34:33.871 | INFO | Finished a round of Baz IO: Wrote 1.991 GiB in 3.044 seconds | 20260505 04:34:33.871 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.95% / 0.14% / 52.91% | 20260505 04:34:33.871 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 10.6474% / 0.000389417% / 5.16574e-05% | 20260505 04:34:33.871 | INFO | Baz IO overall: | 20260505 04:34:33.871 | INFO | BytesWritten : 148514361867 | 20260505 04:34:33.871 | INFO | NumEvents : 68750798991 | 20260505 04:34:35.868 | INFO | Analyzing chunk frames = [376832,377344) | 20260505 04:34:35.868 | INFO | MemInfo: { | 20260505 04:34:35.868 | INFO | "gpu" : " 17 GB", | 20260505 04:34:35.868 | INFO | "huge" : "", | 20260505 04:34:35.868 | INFO | "huge_shared" : " 27 GB", | 20260505 04:34:35.868 | INFO | "malloc" : " 14 GB", | 20260505 04:34:35.868 | INFO | "pinned" : " 29 GB" | 20260505 04:34:35.868 | INFO | } | 20260505 04:34:35.868 | INFO | Perf report for the last 5.13943 seconds: | 20260505 04:34:35.869 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:34:35.869 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:35.869 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:35.869 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:35.869 | INFO | ANALYSIS : 58.3%, 3.0, 90.8ms, 0 | 20260505 04:34:35.869 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:35.869 | INFO | RT_METRICS : 45.1%, 1.0, 23.7ms, 0 | 20260505 04:34:35.869 | INFO | PRE_HQ : 57.6%, 2.2, 64.5ms, 0 | 20260505 04:34:35.869 | INFO | BAZWRITER : 58.8%, 3.9, 117.2ms, 0 | 20260505 04:34:35.869 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:34:35.869 | INFO | Uploaded 36GiB to GPU in 1.63892s (21.9657GiB/s) | 20260505 04:34:35.869 | INFO | Downloaded 38GiB to GPU in 1.70184s (22.3288GiB/s) | 20260505 04:34:41.008 | INFO | Analyzing chunk frames = [377344,377856) | 20260505 04:34:41.008 | INFO | MemInfo: { | 20260505 04:34:41.008 | INFO | "gpu" : " 17 GB", | 20260505 04:34:41.008 | INFO | "huge" : "", | 20260505 04:34:41.008 | INFO | "huge_shared" : " 27 GB", | 20260505 04:34:41.008 | INFO | "malloc" : " 14 GB", | 20260505 04:34:41.008 | INFO | "pinned" : " 29 GB" | 20260505 04:34:41.008 | INFO | } | 20260505 04:34:41.008 | INFO | Perf report for the last 5.13934 seconds: | 20260505 04:34:41.008 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:34:41.008 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:41.008 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:41.008 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:41.008 | INFO | ANALYSIS : 58.1%, 3.0, 90.6ms, 0 | 20260505 04:34:41.008 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:41.008 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:41.008 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 04:34:41.008 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:41.008 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:41.008 | INFO | Uploaded 36GiB to GPU in 1.59847s (22.5215GiB/s) | 20260505 04:34:41.008 | INFO | Downloaded 34GiB to GPU in 1.51461s (22.448GiB/s) | 20260505 04:34:46.147 | INFO | Analyzing chunk frames = [377856,378368) | 20260505 04:34:46.147 | INFO | MemInfo: { | 20260505 04:34:46.147 | INFO | "gpu" : " 17 GB", | 20260505 04:34:46.147 | INFO | "huge" : "", | 20260505 04:34:46.147 | INFO | "huge_shared" : " 27 GB", | 20260505 04:34:46.147 | INFO | "malloc" : " 14 GB", | 20260505 04:34:46.147 | INFO | "pinned" : " 29 GB" | 20260505 04:34:46.147 | INFO | } | 20260505 04:34:46.147 | INFO | Perf report for the last 5.13932 seconds: | 20260505 04:34:46.147 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:34:46.147 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:46.147 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:46.147 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:46.147 | INFO | ANALYSIS : 57.5%, 3.0, 89.3ms, 0 | 20260505 04:34:46.147 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:46.147 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:46.147 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260505 04:34:46.147 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:46.147 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:46.147 | INFO | Uploaded 36GiB to GPU in 1.60227s (22.4682GiB/s) | 20260505 04:34:46.148 | INFO | Downloaded 34GiB to GPU in 1.51946s (22.3764GiB/s) | 20260505 04:34:51.286 | INFO | Analyzing chunk frames = [378368,378880) | 20260505 04:34:51.286 | INFO | MemInfo: { | 20260505 04:34:51.286 | INFO | "gpu" : " 17 GB", | 20260505 04:34:51.286 | INFO | "huge" : "", | 20260505 04:34:51.286 | INFO | "huge_shared" : " 27 GB", | 20260505 04:34:51.286 | INFO | "malloc" : " 14 GB", | 20260505 04:34:51.286 | INFO | "pinned" : " 29 GB" | 20260505 04:34:51.286 | INFO | } | 20260505 04:34:51.286 | INFO | Perf report for the last 5.13926 seconds: | 20260505 04:34:51.286 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:34:51.286 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:51.286 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:51.286 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:51.287 | INFO | ANALYSIS : 57.1%, 3.0, 88.6ms, 0 | 20260505 04:34:51.287 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:51.287 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:51.287 | INFO | PRE_HQ : 31.9%, 1.0, 16.5ms, 0 | 20260505 04:34:51.287 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:51.287 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:51.287 | INFO | Uploaded 36GiB to GPU in 1.6013s (22.4817GiB/s) | 20260505 04:34:51.287 | INFO | Downloaded 34GiB to GPU in 1.51831s (22.3933GiB/s) | 20260505 04:34:56.426 | INFO | Analyzing chunk frames = [378880,379392) | 20260505 04:34:56.426 | INFO | MemInfo: { | 20260505 04:34:56.426 | INFO | "gpu" : " 17 GB", | 20260505 04:34:56.426 | INFO | "huge" : "", | 20260505 04:34:56.426 | INFO | "huge_shared" : " 27 GB", | 20260505 04:34:56.426 | INFO | "malloc" : " 14 GB", | 20260505 04:34:56.426 | INFO | "pinned" : " 29 GB" | 20260505 04:34:56.426 | INFO | } | 20260505 04:34:56.426 | INFO | Perf report for the last 5.13936 seconds: | 20260505 04:34:56.426 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:34:56.426 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:56.426 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:56.426 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:56.426 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260505 04:34:56.426 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:56.426 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:56.426 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 04:34:56.426 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:56.426 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:34:56.426 | INFO | Uploaded 36GiB to GPU in 1.59856s (22.5203GiB/s) | 20260505 04:34:56.426 | INFO | Downloaded 34GiB to GPU in 1.51589s (22.4291GiB/s) | 20260505 04:35:01.565 | INFO | Analyzing chunk frames = [379392,379904) | 20260505 04:35:01.565 | INFO | MemInfo: { | 20260505 04:35:01.565 | INFO | "gpu" : " 17 GB", | 20260505 04:35:01.565 | INFO | "huge" : "", | 20260505 04:35:01.565 | INFO | "huge_shared" : " 27 GB", | 20260505 04:35:01.565 | INFO | "malloc" : " 14 GB", | 20260505 04:35:01.565 | INFO | "pinned" : " 29 GB" | 20260505 04:35:01.565 | INFO | } | 20260505 04:35:01.565 | INFO | Perf report for the last 5.1392 seconds: | 20260505 04:35:01.565 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:35:01.565 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:01.565 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:01.565 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:01.565 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 04:35:01.565 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:01.565 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:01.565 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260505 04:35:01.565 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:01.565 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:01.565 | INFO | Uploaded 36GiB to GPU in 1.601s (22.486GiB/s) | 20260505 04:35:01.566 | INFO | Downloaded 34GiB to GPU in 1.51775s (22.4015GiB/s) | 20260505 04:35:06.704 | INFO | Analyzing chunk frames = [379904,380416) | 20260505 04:35:06.704 | INFO | MemInfo: { | 20260505 04:35:06.704 | INFO | "gpu" : " 17 GB", | 20260505 04:35:06.705 | INFO | "huge" : "", | 20260505 04:35:06.705 | INFO | "huge_shared" : " 27 GB", | 20260505 04:35:06.705 | INFO | "malloc" : " 14 GB", | 20260505 04:35:06.705 | INFO | "pinned" : " 29 GB" | 20260505 04:35:06.705 | INFO | } | 20260505 04:35:06.705 | INFO | Perf report for the last 5.13961 seconds: | 20260505 04:35:06.705 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:35:06.705 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:06.705 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:06.705 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:06.705 | INFO | ANALYSIS : 57.6%, 3.0, 89.9ms, 0 | 20260505 04:35:06.705 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:06.705 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:06.705 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260505 04:35:06.705 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:06.705 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:06.705 | INFO | Uploaded 36GiB to GPU in 1.60019s (22.4974GiB/s) | 20260505 04:35:06.705 | INFO | Downloaded 34GiB to GPU in 1.51756s (22.4043GiB/s) | 20260505 04:35:11.844 | INFO | Analyzing chunk frames = [380416,380928) | 20260505 04:35:11.844 | INFO | MemInfo: { | 20260505 04:35:11.844 | INFO | "gpu" : " 17 GB", | 20260505 04:35:11.844 | INFO | "huge" : "", | 20260505 04:35:11.844 | INFO | "huge_shared" : " 27 GB", | 20260505 04:35:11.844 | INFO | "malloc" : " 14 GB", | 20260505 04:35:11.844 | INFO | "pinned" : " 29 GB" | 20260505 04:35:11.844 | INFO | } | 20260505 04:35:11.844 | INFO | Perf report for the last 5.13918 seconds: | 20260505 04:35:11.844 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:35:11.844 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:11.844 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:11.844 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:11.844 | INFO | ANALYSIS : 57.6%, 3.0, 89.9ms, 0 | 20260505 04:35:11.844 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:11.844 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:11.844 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 04:35:11.844 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:11.844 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:11.844 | INFO | Uploaded 36GiB to GPU in 1.60049s (22.4931GiB/s) | 20260505 04:35:11.844 | INFO | Downloaded 34GiB to GPU in 1.51753s (22.4048GiB/s) | 20260505 04:35:11.940 | INFO | Starting a round of Baz IO | 20260505 04:35:14.900 | INFO | Populating RTMetrics report | 20260505 04:35:14.900 | INFO | RTMetrics report populated | 20260505 04:35:14.900 | INFO | Opening tmp RTMetrics json file | 20260505 04:35:14.901 | INFO | Serializing RTMetrics to json | 20260505 04:35:14.902 | INFO | Serialization done, swapping tmp file now... | 20260505 04:35:14.902 | INFO | RTMetrics done | 20260505 04:35:14.986 | INFO | Finished a round of Baz IO: Wrote 1.996 GiB in 3.045 seconds | 20260505 04:35:14.986 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.81% / 0.14% / 53.05% | 20260505 04:35:14.986 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 10.6418% / 0.000286102% / 3.57628e-05% | 20260505 04:35:14.986 | INFO | Baz IO overall: | 20260505 04:35:14.986 | INFO | BytesWritten : 150658023864 | 20260505 04:35:14.986 | INFO | NumEvents : 69765098949 | 20260505 04:35:16.983 | INFO | Analyzing chunk frames = [380928,381440) | 20260505 04:35:16.983 | INFO | MemInfo: { | 20260505 04:35:16.983 | INFO | "gpu" : " 17 GB", | 20260505 04:35:16.983 | INFO | "huge" : "", | 20260505 04:35:16.983 | INFO | "huge_shared" : " 27 GB", | 20260505 04:35:16.983 | INFO | "malloc" : " 14 GB", | 20260505 04:35:16.983 | INFO | "pinned" : " 29 GB" | 20260505 04:35:16.983 | INFO | } | 20260505 04:35:16.983 | INFO | Perf report for the last 5.13936 seconds: | 20260505 04:35:16.983 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:35:16.983 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:16.983 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:16.983 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:16.983 | INFO | ANALYSIS : 58.3%, 3.0, 90.7ms, 0 | 20260505 04:35:16.983 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:16.983 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260505 04:35:16.983 | INFO | PRE_HQ : 57.7%, 2.2, 64.7ms, 0 | 20260505 04:35:16.983 | INFO | BAZWRITER : 58.8%, 3.9, 118.5ms, 0 | 20260505 04:35:16.983 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:35:16.984 | INFO | Uploaded 36GiB to GPU in 1.63683s (21.9937GiB/s) | 20260505 04:35:16.984 | INFO | Downloaded 38GiB to GPU in 1.70052s (22.3462GiB/s) | 20260505 04:35:22.122 | INFO | Analyzing chunk frames = [381440,381952) | 20260505 04:35:22.122 | INFO | MemInfo: { | 20260505 04:35:22.122 | INFO | "gpu" : " 17 GB", | 20260505 04:35:22.122 | INFO | "huge" : "", | 20260505 04:35:22.122 | INFO | "huge_shared" : " 27 GB", | 20260505 04:35:22.122 | INFO | "malloc" : " 14 GB", | 20260505 04:35:22.122 | INFO | "pinned" : " 29 GB" | 20260505 04:35:22.122 | INFO | } | 20260505 04:35:22.123 | INFO | Perf report for the last 5.13927 seconds: | 20260505 04:35:22.123 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:35:22.123 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:22.123 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:22.123 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:22.123 | INFO | ANALYSIS : 58.1%, 3.0, 90.6ms, 0 | 20260505 04:35:22.123 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:22.123 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:22.123 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 04:35:22.123 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:22.123 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:22.123 | INFO | Uploaded 36GiB to GPU in 1.59843s (22.5221GiB/s) | 20260505 04:35:22.123 | INFO | Downloaded 34GiB to GPU in 1.5151s (22.4408GiB/s) | 20260505 04:35:27.262 | INFO | Analyzing chunk frames = [381952,382464) | 20260505 04:35:27.262 | INFO | MemInfo: { | 20260505 04:35:27.262 | INFO | "gpu" : " 17 GB", | 20260505 04:35:27.262 | INFO | "huge" : "", | 20260505 04:35:27.262 | INFO | "huge_shared" : " 27 GB", | 20260505 04:35:27.262 | INFO | "malloc" : " 14 GB", | 20260505 04:35:27.262 | INFO | "pinned" : " 29 GB" | 20260505 04:35:27.262 | INFO | } | 20260505 04:35:27.262 | INFO | Perf report for the last 5.13936 seconds: | 20260505 04:35:27.262 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:35:27.262 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:27.262 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:27.262 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:27.262 | INFO | ANALYSIS : 57.5%, 3.0, 89.3ms, 0 | 20260505 04:35:27.262 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:27.262 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:27.262 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260505 04:35:27.262 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:27.262 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:27.262 | INFO | Uploaded 36GiB to GPU in 1.60208s (22.4707GiB/s) | 20260505 04:35:27.262 | INFO | Downloaded 34GiB to GPU in 1.51904s (22.3826GiB/s) | 20260505 04:35:32.401 | INFO | Analyzing chunk frames = [382464,382976) | 20260505 04:35:32.401 | INFO | MemInfo: { | 20260505 04:35:32.401 | INFO | "gpu" : " 17 GB", | 20260505 04:35:32.401 | INFO | "huge" : "", | 20260505 04:35:32.401 | INFO | "huge_shared" : " 27 GB", | 20260505 04:35:32.401 | INFO | "malloc" : " 14 GB", | 20260505 04:35:32.401 | INFO | "pinned" : " 29 GB" | 20260505 04:35:32.401 | INFO | } | 20260505 04:35:32.401 | INFO | Perf report for the last 5.13932 seconds: | 20260505 04:35:32.401 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:35:32.401 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:32.401 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:32.401 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:32.401 | INFO | ANALYSIS : 57.1%, 3.0, 88.6ms, 0 | 20260505 04:35:32.401 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:32.401 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:32.401 | INFO | PRE_HQ : 31.7%, 1.0, 16.4ms, 0 | 20260505 04:35:32.401 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:32.401 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:32.402 | INFO | Uploaded 36GiB to GPU in 1.60279s (22.4608GiB/s) | 20260505 04:35:32.402 | INFO | Downloaded 34GiB to GPU in 1.51949s (22.3759GiB/s) | 20260505 04:35:37.540 | INFO | Analyzing chunk frames = [382976,383488) | 20260505 04:35:37.540 | INFO | MemInfo: { | 20260505 04:35:37.540 | INFO | "gpu" : " 17 GB", | 20260505 04:35:37.540 | INFO | "huge" : "", | 20260505 04:35:37.540 | INFO | "huge_shared" : " 27 GB", | 20260505 04:35:37.540 | INFO | "malloc" : " 14 GB", | 20260505 04:35:37.541 | INFO | "pinned" : " 29 GB" | 20260505 04:35:37.541 | INFO | } | 20260505 04:35:37.541 | INFO | Perf report for the last 5.13936 seconds: | 20260505 04:35:37.541 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:35:37.541 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:37.541 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:37.541 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:37.541 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260505 04:35:37.541 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:37.541 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:37.541 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 04:35:37.541 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:37.541 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:37.541 | INFO | Uploaded 36GiB to GPU in 1.59948s (22.5073GiB/s) | 20260505 04:35:37.541 | INFO | Downloaded 34GiB to GPU in 1.51556s (22.4339GiB/s) | 20260505 04:35:42.680 | INFO | Analyzing chunk frames = [383488,384000) | 20260505 04:35:42.680 | INFO | MemInfo: { | 20260505 04:35:42.680 | INFO | "gpu" : " 17 GB", | 20260505 04:35:42.680 | INFO | "huge" : "", | 20260505 04:35:42.680 | INFO | "huge_shared" : " 27 GB", | 20260505 04:35:42.680 | INFO | "malloc" : " 14 GB", | 20260505 04:35:42.680 | INFO | "pinned" : " 29 GB" | 20260505 04:35:42.680 | INFO | } | 20260505 04:35:42.680 | INFO | Perf report for the last 5.13918 seconds: | 20260505 04:35:42.680 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:35:42.680 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:42.680 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:42.680 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:42.680 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 04:35:42.680 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:42.680 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:42.680 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260505 04:35:42.680 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:42.680 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:42.680 | INFO | Uploaded 36GiB to GPU in 1.6001s (22.4986GiB/s) | 20260505 04:35:42.680 | INFO | Downloaded 34GiB to GPU in 1.51653s (22.4196GiB/s) | 20260505 04:35:47.819 | INFO | Analyzing chunk frames = [384000,384512) | 20260505 04:35:47.819 | INFO | MemInfo: { | 20260505 04:35:47.819 | INFO | "gpu" : " 17 GB", | 20260505 04:35:47.819 | INFO | "huge" : "", | 20260505 04:35:47.819 | INFO | "huge_shared" : " 27 GB", | 20260505 04:35:47.819 | INFO | "malloc" : " 14 GB", | 20260505 04:35:47.819 | INFO | "pinned" : " 29 GB" | 20260505 04:35:47.819 | INFO | } | 20260505 04:35:47.819 | INFO | Perf report for the last 5.13961 seconds: | 20260505 04:35:47.819 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:35:47.819 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:47.819 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:47.819 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:47.819 | INFO | ANALYSIS : 57.6%, 3.0, 89.9ms, 0 | 20260505 04:35:47.819 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:47.819 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:47.820 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 04:35:47.820 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:47.820 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:47.820 | INFO | Uploaded 36GiB to GPU in 1.601s (22.4859GiB/s) | 20260505 04:35:47.820 | INFO | Downloaded 34GiB to GPU in 1.51827s (22.394GiB/s) | 20260505 04:35:52.958 | INFO | Analyzing chunk frames = [384512,385024) | 20260505 04:35:52.958 | INFO | MemInfo: { | 20260505 04:35:52.958 | INFO | "gpu" : " 17 GB", | 20260505 04:35:52.958 | INFO | "huge" : "", | 20260505 04:35:52.959 | INFO | "huge_shared" : " 27 GB", | 20260505 04:35:52.959 | INFO | "malloc" : " 14 GB", | 20260505 04:35:52.959 | INFO | "pinned" : " 29 GB" | 20260505 04:35:52.959 | INFO | } | 20260505 04:35:52.959 | INFO | Perf report for the last 5.13927 seconds: | 20260505 04:35:52.959 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:35:52.959 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:52.959 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:52.959 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:52.959 | INFO | ANALYSIS : 57.6%, 3.0, 89.9ms, 0 | 20260505 04:35:52.959 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:52.959 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:52.959 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 04:35:52.959 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:52.959 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:52.959 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 04:35:52.959 | INFO | Total mem usage: 14.9 GiB | 20260505 04:35:52.959 | INFO | All Time High Water Marks sum: 15.2 GiB (Savings: 2.08%) | 20260505 04:35:52.959 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -3.15%) | 20260505 04:35:52.959 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 04:35:52.959 | INFO | High Water Breakdown: | 20260505 04:35:52.959 | INFO | 59.2% : 8.57 GiB ( 99.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 04:35:52.959 | INFO | 40.7% : 5.88 GiB ( 88.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 04:35:52.959 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 04:35:52.959 | INFO | ------------------------------------------ | 20260505 04:35:52.959 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:35:52.959 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 04:35:52.959 | INFO | Total mem usage: 26.5 GiB | 20260505 04:35:52.959 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260505 04:35:52.959 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 04:35:52.959 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260505 04:35:52.959 | INFO | High Water Breakdown: | 20260505 04:35:52.959 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:35:52.959 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:35:52.959 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:35:52.959 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:35:52.959 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:35:52.959 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:35:52.959 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 04:35:52.959 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 04:35:52.959 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:35:52.959 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 04:35:52.959 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 04:35:52.959 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 04:35:52.959 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 04:35:52.959 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 04:35:52.959 | INFO | ------------------------------------------ | 20260505 04:35:52.959 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:35:52.959 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 04:35:52.959 | INFO | Total mem usage: 25.6 GiB | 20260505 04:35:52.959 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 04:35:52.959 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 04:35:52.959 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 04:35:52.959 | INFO | High Water Breakdown: | 20260505 04:35:52.959 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 04:35:52.959 | INFO | ------------------------------------------ | 20260505 04:35:52.959 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:35:52.959 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 04:35:52.959 | INFO | Total mem usage: 15.7 GiB | 20260505 04:35:52.959 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260505 04:35:52.959 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 04:35:52.959 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260505 04:35:52.959 | INFO | High Water Breakdown: | 20260505 04:35:52.959 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 04:35:52.959 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 04:35:52.959 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 04:35:52.959 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 04:35:52.959 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 04:35:52.959 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 04:35:52.959 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 04:35:52.959 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 04:35:52.959 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 04:35:52.959 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:35:52.959 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 04:35:52.959 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:35:52.959 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 04:35:52.959 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 04:35:52.959 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 04:35:52.959 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 04:35:52.959 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:35:52.959 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 04:35:52.959 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:35:52.959 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:35:52.959 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:35:52.959 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 04:35:52.959 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 04:35:52.959 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:35:52.959 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 04:35:52.959 | INFO | ------------------------------------------ | 20260505 04:35:52.959 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:35:52.959 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 04:35:52.959 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:35:52.959 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:35:52.959 | INFO | PerfReport: Full Execution -- 3.05e+01 / -nan, 1.28e+01 / 0.00e+00 | 20260505 04:35:52.959 | INFO | PerfReport: 0.28%, AggregateStats -- 8.62e-02 / -nan, 4.68e-03 / 0.00e+00 | 20260505 04:35:52.959 | INFO | PerfReport: 10.81%, Baseline -- 3.30e+00 / -nan, 1.73e-02 / 0.00e+00 | 20260505 04:35:52.959 | INFO | PerfReport: 3.03%, Histogram -- 9.25e-01 / -nan, 5.19e-02 / 0.00e+00 | 20260505 04:35:52.959 | INFO | PerfReport: 16.02%, DME -- 3.91e+01 / -nan, 3.26e+00 / 0.00e+00 | 20260505 04:35:52.959 | INFO | PerfReport: 56.47%, FrameLabeling -- 1.72e+01 / -nan, 4.55e-02 / 0.00e+00 | 20260505 04:35:52.959 | INFO | PerfReport: 7.67%, PulseAccumulating -- 2.34e+00 / -nan, 1.57e-01 / 0.00e+00 | 20260505 04:35:52.959 | INFO | PerfReport: 4.03%, Metrics -- 1.23e+00 / -nan, 2.02e-01 / 0.00e+00 | 20260505 04:35:52.959 | INFO | PerfReport: 1.67%, PulseSerializing -- 5.09e-01 / -nan, 8.18e-02 / 0.00e+00 | 20260505 04:35:52.959 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 04:35:52.959 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:35:52.959 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:35:52.959 | INFO | Uploaded 36GiB to GPU in 1.60081s (22.4885GiB/s) | 20260505 04:35:52.959 | INFO | PerfReport: Full Execution -- 9.15e+01 / -nan, 1.89e+01 / 0.00e+00 | 20260505 04:35:52.959 | INFO | PerfReport: 18.26%, Upload -- 1.67e+01 / 0.00e+00, 3.70e-01 / 0.00e+00 | 20260505 04:35:52.959 | INFO | Downloaded 34GiB to GPU in 1.51791s (22.3993GiB/s) | 20260505 04:35:52.959 | INFO | PerfReport: 17.53%, Download -- 1.60e+01 / 0.00e+00, 6.94e-01 / 0.00e+00 | 20260505 04:35:52.960 | INFO | PerfReport: 33.30%, Compute -- 3.05e+01 / 0.00e+00, 1.28e+01 / 0.00e+00 | 20260505 04:35:52.960 | INFO | PerfReport: 0.55%, LockUpload -- 5.03e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260505 04:35:52.960 | INFO | PerfReport: 0.00%, LockDownload -- 4.04e-04 / 0.00e+00, 1.95e-04 / 0.00e+00 | 20260505 04:35:52.960 | INFO | PerfReport: 30.34%, LockCompute -- 2.78e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260505 04:35:52.960 | INFO | PerfReport: 0.02%, Misc -- 1.44e-02 / 0.00e+00, 2.39e-02 / 0.00e+00 | 20260505 04:35:53.057 | INFO | Starting a round of Baz IO | 20260505 04:35:56.015 | INFO | Populating RTMetrics report | 20260505 04:35:56.016 | INFO | RTMetrics report populated | 20260505 04:35:56.016 | INFO | Opening tmp RTMetrics json file | 20260505 04:35:56.016 | INFO | Serializing RTMetrics to json | 20260505 04:35:56.018 | INFO | Serialization done, swapping tmp file now... | 20260505 04:35:56.018 | INFO | RTMetrics done | 20260505 04:35:56.111 | INFO | Finished a round of Baz IO: Wrote 2.001 GiB in 3.053 seconds | 20260505 04:35:56.111 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.67% / 0.14% / 53.19% | 20260505 04:35:56.111 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 10.5907% / 0.00029405% / 3.57628e-05% | 20260505 04:35:56.111 | INFO | Baz IO overall: | 20260505 04:35:56.111 | INFO | BytesWritten : 152806880566 | 20260505 04:35:56.111 | INFO | NumEvents : 70782367871 | 20260505 04:35:58.098 | INFO | Analyzing chunk frames = [385024,385536) | 20260505 04:35:58.098 | INFO | MemInfo: { | 20260505 04:35:58.098 | INFO | "gpu" : " 17 GB", | 20260505 04:35:58.098 | INFO | "huge" : "", | 20260505 04:35:58.098 | INFO | "huge_shared" : " 27 GB", | 20260505 04:35:58.098 | INFO | "malloc" : " 14 GB", | 20260505 04:35:58.098 | INFO | "pinned" : " 29 GB" | 20260505 04:35:58.098 | INFO | } | 20260505 04:35:58.098 | INFO | Perf report for the last 5.13918 seconds: | 20260505 04:35:58.098 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:35:58.098 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:58.098 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:58.098 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:58.098 | INFO | ANALYSIS : 58.4%, 3.0, 90.8ms, 0 | 20260505 04:35:58.098 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:35:58.098 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260505 04:35:58.098 | INFO | PRE_HQ : 57.7%, 2.2, 64.8ms, 0 | 20260505 04:35:58.098 | INFO | BAZWRITER : 59.0%, 3.9, 118.4ms, 0 | 20260505 04:35:58.098 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:35:58.098 | INFO | Uploaded 36GiB to GPU in 1.6373s (21.9875GiB/s) | 20260505 04:35:58.098 | INFO | Downloaded 38GiB to GPU in 1.6991s (22.3648GiB/s) | 20260505 04:36:03.237 | INFO | Analyzing chunk frames = [385536,386048) | 20260505 04:36:03.237 | INFO | MemInfo: { | 20260505 04:36:03.237 | INFO | "gpu" : " 17 GB", | 20260505 04:36:03.237 | INFO | "huge" : "", | 20260505 04:36:03.237 | INFO | "huge_shared" : " 27 GB", | 20260505 04:36:03.237 | INFO | "malloc" : " 14 GB", | 20260505 04:36:03.237 | INFO | "pinned" : " 29 GB" | 20260505 04:36:03.237 | INFO | } | 20260505 04:36:03.237 | INFO | Perf report for the last 5.13945 seconds: | 20260505 04:36:03.237 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:36:03.237 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:03.237 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:03.237 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:03.237 | INFO | ANALYSIS : 58.1%, 3.0, 90.6ms, 0 | 20260505 04:36:03.237 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:03.237 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:03.237 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260505 04:36:03.238 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:03.238 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:03.238 | INFO | Uploaded 36GiB to GPU in 1.59792s (22.5293GiB/s) | 20260505 04:36:03.238 | INFO | Downloaded 34GiB to GPU in 1.51654s (22.4194GiB/s) | 20260505 04:36:08.377 | INFO | Analyzing chunk frames = [386048,386560) | 20260505 04:36:08.377 | INFO | MemInfo: { | 20260505 04:36:08.377 | INFO | "gpu" : " 17 GB", | 20260505 04:36:08.377 | INFO | "huge" : "", | 20260505 04:36:08.377 | INFO | "huge_shared" : " 27 GB", | 20260505 04:36:08.377 | INFO | "malloc" : " 14 GB", | 20260505 04:36:08.377 | INFO | "pinned" : " 29 GB" | 20260505 04:36:08.377 | INFO | } | 20260505 04:36:08.377 | INFO | Perf report for the last 5.13947 seconds: | 20260505 04:36:08.377 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:36:08.377 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:08.377 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:08.377 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:08.377 | INFO | ANALYSIS : 57.4%, 3.0, 89.2ms, 0 | 20260505 04:36:08.377 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:08.377 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:08.377 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260505 04:36:08.377 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:08.377 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:08.377 | INFO | Uploaded 36GiB to GPU in 1.60245s (22.4657GiB/s) | 20260505 04:36:08.377 | INFO | Downloaded 34GiB to GPU in 1.5191s (22.3817GiB/s) | 20260505 04:36:13.516 | INFO | Analyzing chunk frames = [386560,387072) | 20260505 04:36:13.516 | INFO | MemInfo: { | 20260505 04:36:13.516 | INFO | "gpu" : " 17 GB", | 20260505 04:36:13.516 | INFO | "huge" : "", | 20260505 04:36:13.516 | INFO | "huge_shared" : " 27 GB", | 20260505 04:36:13.516 | INFO | "malloc" : " 14 GB", | 20260505 04:36:13.516 | INFO | "pinned" : " 29 GB" | 20260505 04:36:13.516 | INFO | } | 20260505 04:36:13.516 | INFO | Perf report for the last 5.139 seconds: | 20260505 04:36:13.516 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:36:13.516 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:13.516 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:13.516 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:13.516 | INFO | ANALYSIS : 57.1%, 3.0, 88.6ms, 0 | 20260505 04:36:13.516 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:13.516 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:13.516 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260505 04:36:13.516 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:13.516 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:13.516 | INFO | Uploaded 36GiB to GPU in 1.601s (22.486GiB/s) | 20260505 04:36:13.516 | INFO | Downloaded 34GiB to GPU in 1.51828s (22.3938GiB/s) | 20260505 04:36:18.655 | INFO | Analyzing chunk frames = [387072,387584) | 20260505 04:36:18.655 | INFO | MemInfo: { | 20260505 04:36:18.655 | INFO | "gpu" : " 17 GB", | 20260505 04:36:18.655 | INFO | "huge" : "", | 20260505 04:36:18.655 | INFO | "huge_shared" : " 27 GB", | 20260505 04:36:18.655 | INFO | "malloc" : " 14 GB", | 20260505 04:36:18.655 | INFO | "pinned" : " 29 GB" | 20260505 04:36:18.655 | INFO | } | 20260505 04:36:18.655 | INFO | Perf report for the last 5.13953 seconds: | 20260505 04:36:18.655 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:36:18.655 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:18.655 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:18.655 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:18.655 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260505 04:36:18.655 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:18.655 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:18.656 | INFO | PRE_HQ : 31.7%, 1.0, 16.4ms, 0 | 20260505 04:36:18.656 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:18.656 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:18.656 | INFO | Uploaded 36GiB to GPU in 1.60055s (22.4923GiB/s) | 20260505 04:36:18.656 | INFO | Downloaded 34GiB to GPU in 1.51771s (22.4022GiB/s) | 20260505 04:36:23.795 | INFO | Analyzing chunk frames = [387584,388096) | 20260505 04:36:23.795 | INFO | MemInfo: { | 20260505 04:36:23.795 | INFO | "gpu" : " 17 GB", | 20260505 04:36:23.795 | INFO | "huge" : "", | 20260505 04:36:23.795 | INFO | "huge_shared" : " 27 GB", | 20260505 04:36:23.795 | INFO | "malloc" : " 14 GB", | 20260505 04:36:23.795 | INFO | "pinned" : " 29 GB" | 20260505 04:36:23.795 | INFO | } | 20260505 04:36:23.795 | INFO | Perf report for the last 5.13946 seconds: | 20260505 04:36:23.795 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:36:23.795 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:23.795 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:23.795 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:23.795 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260505 04:36:23.795 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:23.795 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:23.795 | INFO | PRE_HQ : 31.7%, 1.0, 16.4ms, 0 | 20260505 04:36:23.795 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:23.795 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:23.796 | INFO | Uploaded 36GiB to GPU in 1.59999s (22.5001GiB/s) | 20260505 04:36:23.796 | INFO | Downloaded 34GiB to GPU in 1.51625s (22.4237GiB/s) | 20260505 04:36:28.934 | INFO | Analyzing chunk frames = [388096,388608) | 20260505 04:36:28.934 | INFO | MemInfo: { | 20260505 04:36:28.934 | INFO | "gpu" : " 17 GB", | 20260505 04:36:28.934 | INFO | "huge" : "", | 20260505 04:36:28.934 | INFO | "huge_shared" : " 27 GB", | 20260505 04:36:28.934 | INFO | "malloc" : " 14 GB", | 20260505 04:36:28.934 | INFO | "pinned" : " 29 GB" | 20260505 04:36:28.934 | INFO | } | 20260505 04:36:28.934 | INFO | Perf report for the last 5.13934 seconds: | 20260505 04:36:28.934 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:36:28.934 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:28.934 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:28.934 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:28.934 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260505 04:36:28.934 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:28.934 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:28.934 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 04:36:28.934 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:28.934 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:28.934 | INFO | Uploaded 36GiB to GPU in 1.6001s (22.4986GiB/s) | 20260505 04:36:28.934 | INFO | Downloaded 34GiB to GPU in 1.51755s (22.4045GiB/s) | 20260505 04:36:34.073 | INFO | Analyzing chunk frames = [388608,389120) | 20260505 04:36:34.073 | INFO | MemInfo: { | 20260505 04:36:34.073 | INFO | "gpu" : " 17 GB", | 20260505 04:36:34.073 | INFO | "huge" : "", | 20260505 04:36:34.073 | INFO | "huge_shared" : " 27 GB", | 20260505 04:36:34.073 | INFO | "malloc" : " 14 GB", | 20260505 04:36:34.073 | INFO | "pinned" : " 29 GB" | 20260505 04:36:34.073 | INFO | } | 20260505 04:36:34.073 | INFO | Perf report for the last 5.13908 seconds: | 20260505 04:36:34.073 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:36:34.073 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:34.073 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:34.073 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:34.073 | INFO | ANALYSIS : 57.7%, 3.0, 90.0ms, 0 | 20260505 04:36:34.073 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:34.073 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:34.073 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 04:36:34.073 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:34.073 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:34.074 | INFO | Uploaded 36GiB to GPU in 1.60044s (22.4938GiB/s) | 20260505 04:36:34.074 | INFO | Downloaded 34GiB to GPU in 1.51752s (22.4049GiB/s) | 20260505 04:36:34.171 | INFO | Starting a round of Baz IO | 20260505 04:36:37.125 | INFO | Populating RTMetrics report | 20260505 04:36:37.126 | INFO | RTMetrics report populated | 20260505 04:36:37.126 | INFO | Opening tmp RTMetrics json file | 20260505 04:36:37.126 | INFO | Serializing RTMetrics to json | 20260505 04:36:37.127 | INFO | Serialization done, swapping tmp file now... | 20260505 04:36:37.128 | INFO | RTMetrics done | 20260505 04:36:37.215 | INFO | Finished a round of Baz IO: Wrote 2.005 GiB in 3.044 seconds | 20260505 04:36:37.216 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.53% / 0.14% / 53.33% | 20260505 04:36:37.216 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 10.522% / 0.00033776% / 3.97364e-05% | 20260505 04:36:37.216 | INFO | Baz IO overall: | 20260505 04:36:37.216 | INFO | BytesWritten : 154959397571 | 20260505 04:36:37.216 | INFO | NumEvents : 71801275607 | 20260505 04:36:39.212 | INFO | Analyzing chunk frames = [389120,389632) | 20260505 04:36:39.212 | INFO | MemInfo: { | 20260505 04:36:39.212 | INFO | "gpu" : " 17 GB", | 20260505 04:36:39.213 | INFO | "huge" : "", | 20260505 04:36:39.213 | INFO | "huge_shared" : " 27 GB", | 20260505 04:36:39.213 | INFO | "malloc" : " 14 GB", | 20260505 04:36:39.213 | INFO | "pinned" : " 29 GB" | 20260505 04:36:39.213 | INFO | } | 20260505 04:36:39.213 | INFO | Perf report for the last 5.13934 seconds: | 20260505 04:36:39.213 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:36:39.213 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:39.213 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:39.213 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:39.213 | INFO | ANALYSIS : 58.3%, 3.0, 90.7ms, 0 | 20260505 04:36:39.213 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:39.213 | INFO | RT_METRICS : 45.1%, 1.0, 23.6ms, 0 | 20260505 04:36:39.213 | INFO | PRE_HQ : 57.6%, 2.1, 64.3ms, 0 | 20260505 04:36:39.213 | INFO | BAZWRITER : 58.8%, 3.9, 119.3ms, 0 | 20260505 04:36:39.213 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:36:39.213 | INFO | Uploaded 36GiB to GPU in 1.63949s (21.958GiB/s) | 20260505 04:36:39.213 | INFO | Downloaded 38GiB to GPU in 1.70237s (22.3218GiB/s) | 20260505 04:36:44.352 | INFO | Analyzing chunk frames = [389632,390144) | 20260505 04:36:44.352 | INFO | MemInfo: { | 20260505 04:36:44.352 | INFO | "gpu" : " 17 GB", | 20260505 04:36:44.352 | INFO | "huge" : "", | 20260505 04:36:44.352 | INFO | "huge_shared" : " 27 GB", | 20260505 04:36:44.352 | INFO | "malloc" : " 14 GB", | 20260505 04:36:44.352 | INFO | "pinned" : " 29 GB" | 20260505 04:36:44.352 | INFO | } | 20260505 04:36:44.352 | INFO | Perf report for the last 5.13974 seconds: | 20260505 04:36:44.352 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:36:44.352 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:44.352 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:44.352 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:44.352 | INFO | ANALYSIS : 58.1%, 3.0, 90.7ms, 0 | 20260505 04:36:44.352 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:44.353 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:44.353 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 04:36:44.353 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:44.353 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:44.353 | INFO | Uploaded 36GiB to GPU in 1.59857s (22.5202GiB/s) | 20260505 04:36:44.353 | INFO | Downloaded 34GiB to GPU in 1.51461s (22.448GiB/s) | 20260505 04:36:49.491 | INFO | Analyzing chunk frames = [390144,390656) | 20260505 04:36:49.491 | INFO | MemInfo: { | 20260505 04:36:49.491 | INFO | "gpu" : " 17 GB", | 20260505 04:36:49.491 | INFO | "huge" : "", | 20260505 04:36:49.491 | INFO | "huge_shared" : " 27 GB", | 20260505 04:36:49.491 | INFO | "malloc" : " 14 GB", | 20260505 04:36:49.491 | INFO | "pinned" : " 29 GB" | 20260505 04:36:49.491 | INFO | } | 20260505 04:36:49.491 | INFO | Perf report for the last 5.13894 seconds: | 20260505 04:36:49.491 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:36:49.491 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:49.491 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:49.491 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:49.491 | INFO | ANALYSIS : 57.4%, 3.0, 89.2ms, 0 | 20260505 04:36:49.491 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:49.491 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:49.491 | INFO | PRE_HQ : 31.7%, 1.0, 16.4ms, 0 | 20260505 04:36:49.491 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:49.492 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:49.492 | INFO | Uploaded 36GiB to GPU in 1.60258s (22.4638GiB/s) | 20260505 04:36:49.492 | INFO | Downloaded 34GiB to GPU in 1.51941s (22.3771GiB/s) | 20260505 04:36:54.631 | INFO | Analyzing chunk frames = [390656,391168) | 20260505 04:36:54.631 | INFO | MemInfo: { | 20260505 04:36:54.631 | INFO | "gpu" : " 17 GB", | 20260505 04:36:54.631 | INFO | "huge" : "", | 20260505 04:36:54.631 | INFO | "huge_shared" : " 27 GB", | 20260505 04:36:54.631 | INFO | "malloc" : " 14 GB", | 20260505 04:36:54.631 | INFO | "pinned" : " 29 GB" | 20260505 04:36:54.631 | INFO | } | 20260505 04:36:54.631 | INFO | Perf report for the last 5.13945 seconds: | 20260505 04:36:54.631 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:36:54.631 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:54.631 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:54.631 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:54.631 | INFO | ANALYSIS : 57.1%, 3.0, 88.6ms, 0 | 20260505 04:36:54.631 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:54.631 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:54.631 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 04:36:54.631 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:54.631 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:54.631 | INFO | Uploaded 36GiB to GPU in 1.60149s (22.479GiB/s) | 20260505 04:36:54.631 | INFO | Downloaded 34GiB to GPU in 1.51804s (22.3973GiB/s) | 20260505 04:36:59.770 | INFO | Analyzing chunk frames = [391168,391680) | 20260505 04:36:59.770 | INFO | MemInfo: { | 20260505 04:36:59.770 | INFO | "gpu" : " 17 GB", | 20260505 04:36:59.770 | INFO | "huge" : "", | 20260505 04:36:59.770 | INFO | "huge_shared" : " 27 GB", | 20260505 04:36:59.770 | INFO | "malloc" : " 14 GB", | 20260505 04:36:59.770 | INFO | "pinned" : " 29 GB" | 20260505 04:36:59.770 | INFO | } | 20260505 04:36:59.770 | INFO | Perf report for the last 5.13944 seconds: | 20260505 04:36:59.770 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:36:59.770 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:59.770 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:59.770 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:59.770 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260505 04:36:59.770 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:59.770 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:59.770 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260505 04:36:59.770 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:59.770 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:36:59.771 | INFO | Uploaded 36GiB to GPU in 1.60024s (22.4966GiB/s) | 20260505 04:36:59.771 | INFO | Downloaded 34GiB to GPU in 1.51613s (22.4255GiB/s) | 20260505 04:37:04.909 | INFO | Analyzing chunk frames = [391680,392192) | 20260505 04:37:04.909 | INFO | MemInfo: { | 20260505 04:37:04.909 | INFO | "gpu" : " 17 GB", | 20260505 04:37:04.909 | INFO | "huge" : "", | 20260505 04:37:04.910 | INFO | "huge_shared" : " 27 GB", | 20260505 04:37:04.910 | INFO | "malloc" : " 14 GB", | 20260505 04:37:04.910 | INFO | "pinned" : " 29 GB" | 20260505 04:37:04.910 | INFO | } | 20260505 04:37:04.910 | INFO | Perf report for the last 5.13934 seconds: | 20260505 04:37:04.910 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:37:04.910 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:04.910 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:04.910 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:04.910 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260505 04:37:04.910 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:04.910 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:04.910 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260505 04:37:04.910 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:04.910 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:04.910 | INFO | Uploaded 36GiB to GPU in 1.60007s (22.499GiB/s) | 20260505 04:37:04.910 | INFO | Downloaded 34GiB to GPU in 1.51636s (22.4221GiB/s) | 20260505 04:37:10.049 | INFO | Analyzing chunk frames = [392192,392704) | 20260505 04:37:10.049 | INFO | MemInfo: { | 20260505 04:37:10.049 | INFO | "gpu" : " 17 GB", | 20260505 04:37:10.049 | INFO | "huge" : "", | 20260505 04:37:10.049 | INFO | "huge_shared" : " 27 GB", | 20260505 04:37:10.049 | INFO | "malloc" : " 14 GB", | 20260505 04:37:10.049 | INFO | "pinned" : " 29 GB" | 20260505 04:37:10.049 | INFO | } | 20260505 04:37:10.049 | INFO | Perf report for the last 5.13929 seconds: | 20260505 04:37:10.049 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:37:10.049 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:10.049 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:10.049 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:10.049 | INFO | ANALYSIS : 57.6%, 3.0, 89.8ms, 0 | 20260505 04:37:10.049 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:10.049 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:10.049 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260505 04:37:10.049 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:10.049 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:10.049 | INFO | Uploaded 36GiB to GPU in 1.60107s (22.485GiB/s) | 20260505 04:37:10.049 | INFO | Downloaded 34GiB to GPU in 1.51718s (22.4099GiB/s) | 20260505 04:37:15.188 | INFO | Analyzing chunk frames = [392704,393216) | 20260505 04:37:15.188 | INFO | MemInfo: { | 20260505 04:37:15.188 | INFO | "gpu" : " 17 GB", | 20260505 04:37:15.188 | INFO | "huge" : "", | 20260505 04:37:15.188 | INFO | "huge_shared" : " 27 GB", | 20260505 04:37:15.188 | INFO | "malloc" : " 14 GB", | 20260505 04:37:15.188 | INFO | "pinned" : " 29 GB" | 20260505 04:37:15.188 | INFO | } | 20260505 04:37:15.188 | INFO | Perf report for the last 5.13939 seconds: | 20260505 04:37:15.188 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:37:15.188 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:15.188 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:15.188 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:15.188 | INFO | ANALYSIS : 57.7%, 3.0, 90.0ms, 0 | 20260505 04:37:15.188 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:15.188 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:15.188 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260505 04:37:15.188 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:15.188 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:15.189 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 04:37:15.189 | INFO | Total mem usage: 14.9 GiB | 20260505 04:37:15.189 | INFO | All Time High Water Marks sum: 15.3 GiB (Savings: 2.13%) | 20260505 04:37:15.189 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -3.04%) | 20260505 04:37:15.189 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 04:37:15.189 | INFO | High Water Breakdown: | 20260505 04:37:15.189 | INFO | 59.3% : 8.59 GiB ( 100% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 04:37:15.189 | INFO | 40.6% : 5.88 GiB ( 88.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 04:37:15.189 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 04:37:15.189 | INFO | ------------------------------------------ | 20260505 04:37:15.189 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:37:15.189 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 04:37:15.189 | INFO | Total mem usage: 26.5 GiB | 20260505 04:37:15.189 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260505 04:37:15.189 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 04:37:15.189 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260505 04:37:15.189 | INFO | High Water Breakdown: | 20260505 04:37:15.189 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:37:15.189 | INFO | Uploaded 36GiB to GPU in 1.60054s (22.4924GiB/s) | 20260505 04:37:15.189 | INFO | Downloaded 34GiB to GPU in 1.51788s (22.3997GiB/s) | 20260505 04:37:15.189 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:37:15.189 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:37:15.189 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:37:15.189 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:37:15.189 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:37:15.189 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 04:37:15.189 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 04:37:15.189 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:37:15.189 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 04:37:15.189 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 04:37:15.189 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 04:37:15.189 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 04:37:15.189 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 04:37:15.189 | INFO | ------------------------------------------ | 20260505 04:37:15.189 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:37:15.189 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 04:37:15.189 | INFO | Total mem usage: 25.6 GiB | 20260505 04:37:15.189 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 04:37:15.189 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 04:37:15.189 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 04:37:15.189 | INFO | High Water Breakdown: | 20260505 04:37:15.189 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 04:37:15.189 | INFO | ------------------------------------------ | 20260505 04:37:15.189 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:37:15.189 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 04:37:15.189 | INFO | Total mem usage: 15.7 GiB | 20260505 04:37:15.189 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260505 04:37:15.189 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 04:37:15.189 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260505 04:37:15.189 | INFO | High Water Breakdown: | 20260505 04:37:15.189 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 04:37:15.190 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 04:37:15.190 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 04:37:15.190 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 04:37:15.190 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 04:37:15.190 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 04:37:15.190 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 04:37:15.190 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 04:37:15.190 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 04:37:15.190 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:37:15.190 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 04:37:15.190 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:37:15.190 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 04:37:15.190 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 04:37:15.190 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 04:37:15.190 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 04:37:15.190 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:37:15.190 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 04:37:15.190 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:37:15.190 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:37:15.190 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:37:15.190 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 04:37:15.190 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 04:37:15.190 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:37:15.190 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 04:37:15.190 | INFO | ------------------------------------------ | 20260505 04:37:15.190 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:37:15.190 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 04:37:15.190 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:37:15.190 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:37:15.190 | INFO | PerfReport: Full Execution -- 3.05e+01 / -nan, 1.28e+01 / 0.00e+00 | 20260505 04:37:15.190 | INFO | PerfReport: 0.29%, AggregateStats -- 8.71e-02 / -nan, 2.37e-02 / 0.00e+00 | 20260505 04:37:15.190 | INFO | PerfReport: 10.82%, Baseline -- 3.30e+00 / -nan, 1.75e-02 / 0.00e+00 | 20260505 04:37:15.190 | INFO | PerfReport: 3.03%, Histogram -- 9.25e-01 / -nan, 5.23e-02 / 0.00e+00 | 20260505 04:37:15.190 | INFO | PerfReport: 16.02%, DME -- 3.91e+01 / -nan, 3.18e+00 / 0.00e+00 | 20260505 04:37:15.190 | INFO | PerfReport: 56.48%, FrameLabeling -- 1.72e+01 / -nan, 4.62e-02 / 0.00e+00 | 20260505 04:37:15.190 | INFO | PerfReport: 7.66%, PulseAccumulating -- 2.34e+00 / -nan, 1.56e-01 / 0.00e+00 | 20260505 04:37:15.190 | INFO | PerfReport: 4.03%, Metrics -- 1.23e+00 / -nan, 2.02e-01 / 0.00e+00 | 20260505 04:37:15.190 | INFO | PerfReport: 1.67%, PulseSerializing -- 5.08e-01 / -nan, 8.16e-02 / 0.00e+00 | 20260505 04:37:15.190 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 04:37:15.190 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:37:15.190 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:37:15.190 | INFO | PerfReport: Full Execution -- 9.15e+01 / -nan, 1.91e+01 / 0.00e+00 | 20260505 04:37:15.190 | INFO | PerfReport: 18.27%, Upload -- 1.67e+01 / 0.00e+00, 3.66e-01 / 0.00e+00 | 20260505 04:37:15.190 | INFO | PerfReport: 17.53%, Download -- 1.60e+01 / 0.00e+00, 6.92e-01 / 0.00e+00 | 20260505 04:37:15.190 | INFO | PerfReport: 33.30%, Compute -- 3.05e+01 / 0.00e+00, 1.28e+01 / 0.00e+00 | 20260505 04:37:15.190 | INFO | PerfReport: 0.55%, LockUpload -- 5.03e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260505 04:37:15.190 | INFO | PerfReport: 0.00%, LockDownload -- 3.98e-04 / 0.00e+00, 1.38e-04 / 0.00e+00 | 20260505 04:37:15.190 | INFO | PerfReport: 30.34%, LockCompute -- 2.78e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260505 04:37:15.190 | INFO | PerfReport: 0.02%, Misc -- 1.40e-02 / 0.00e+00, 2.87e-02 / 0.00e+00 | 20260505 04:37:15.286 | INFO | Starting a round of Baz IO | 20260505 04:37:18.240 | INFO | Populating RTMetrics report | 20260505 04:37:18.240 | INFO | RTMetrics report populated | 20260505 04:37:18.240 | INFO | Opening tmp RTMetrics json file | 20260505 04:37:18.242 | INFO | Serializing RTMetrics to json | 20260505 04:37:18.243 | INFO | Serialization done, swapping tmp file now... | 20260505 04:37:18.243 | INFO | RTMetrics done | 20260505 04:37:18.327 | INFO | Finished a round of Baz IO: Wrote 2.009 GiB in 3.041 seconds | 20260505 04:37:18.327 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.39% / 0.14% / 53.47% | 20260505 04:37:18.327 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 10.4574% / 0.000333786% / 4.76837e-05% | 20260505 04:37:18.327 | INFO | Baz IO overall: | 20260505 04:37:18.327 | INFO | BytesWritten : 157116275799 | 20260505 04:37:18.327 | INFO | NumEvents : 72822310908 | 20260505 04:37:20.328 | INFO | Analyzing chunk frames = [393216,393728) | 20260505 04:37:20.328 | INFO | MemInfo: { | 20260505 04:37:20.328 | INFO | "gpu" : " 17 GB", | 20260505 04:37:20.328 | INFO | "huge" : "", | 20260505 04:37:20.328 | INFO | "huge_shared" : " 27 GB", | 20260505 04:37:20.328 | INFO | "malloc" : " 14 GB", | 20260505 04:37:20.328 | INFO | "pinned" : " 29 GB" | 20260505 04:37:20.328 | INFO | } | 20260505 04:37:20.328 | INFO | Perf report for the last 5.13932 seconds: | 20260505 04:37:20.328 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:37:20.328 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:20.328 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:20.328 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:20.328 | INFO | ANALYSIS : 58.3%, 3.0, 90.7ms, 0 | 20260505 04:37:20.328 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:20.328 | INFO | RT_METRICS : 45.1%, 1.0, 23.7ms, 0 | 20260505 04:37:20.328 | INFO | PRE_HQ : 57.7%, 2.1, 64.3ms, 0 | 20260505 04:37:20.328 | INFO | BAZWRITER : 58.7%, 4.0, 119.2ms, 0 | 20260505 04:37:20.328 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:37:20.328 | INFO | Uploaded 36GiB to GPU in 1.64139s (21.9326GiB/s) | 20260505 04:37:20.328 | INFO | Downloaded 38GiB to GPU in 1.70323s (22.3106GiB/s) | 20260505 04:37:25.467 | INFO | Analyzing chunk frames = [393728,394240) | 20260505 04:37:25.467 | INFO | MemInfo: { | 20260505 04:37:25.467 | INFO | "gpu" : " 17 GB", | 20260505 04:37:25.467 | INFO | "huge" : "", | 20260505 04:37:25.467 | INFO | "huge_shared" : " 27 GB", | 20260505 04:37:25.467 | INFO | "malloc" : " 14 GB", | 20260505 04:37:25.467 | INFO | "pinned" : " 29 GB" | 20260505 04:37:25.467 | INFO | } | 20260505 04:37:25.467 | INFO | Perf report for the last 5.13932 seconds: | 20260505 04:37:25.467 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:37:25.467 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:25.467 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:25.467 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:25.467 | INFO | ANALYSIS : 58.1%, 3.0, 90.6ms, 0 | 20260505 04:37:25.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:25.467 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:25.467 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260505 04:37:25.467 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:25.467 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:25.467 | INFO | Uploaded 36GiB to GPU in 1.59835s (22.5233GiB/s) | 20260505 04:37:25.467 | INFO | Downloaded 34GiB to GPU in 1.51471s (22.4465GiB/s) | 20260505 04:37:30.606 | INFO | Analyzing chunk frames = [394240,394752) | 20260505 04:37:30.606 | INFO | MemInfo: { | 20260505 04:37:30.606 | INFO | "gpu" : " 17 GB", | 20260505 04:37:30.606 | INFO | "huge" : "", | 20260505 04:37:30.606 | INFO | "huge_shared" : " 27 GB", | 20260505 04:37:30.606 | INFO | "malloc" : " 14 GB", | 20260505 04:37:30.606 | INFO | "pinned" : " 29 GB" | 20260505 04:37:30.606 | INFO | } | 20260505 04:37:30.606 | INFO | Perf report for the last 5.13915 seconds: | 20260505 04:37:30.606 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:37:30.606 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:30.606 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:30.606 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:30.606 | INFO | ANALYSIS : 57.4%, 3.0, 89.2ms, 0 | 20260505 04:37:30.606 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:30.606 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:30.606 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 04:37:30.606 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:30.606 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:30.606 | INFO | Uploaded 36GiB to GPU in 1.60267s (22.4625GiB/s) | 20260505 04:37:30.606 | INFO | Downloaded 34GiB to GPU in 1.51978s (22.3717GiB/s) | 20260505 04:37:35.745 | INFO | Analyzing chunk frames = [394752,395264) | 20260505 04:37:35.745 | INFO | MemInfo: { | 20260505 04:37:35.745 | INFO | "gpu" : " 17 GB", | 20260505 04:37:35.745 | INFO | "huge" : "", | 20260505 04:37:35.746 | INFO | "huge_shared" : " 27 GB", | 20260505 04:37:35.746 | INFO | "malloc" : " 14 GB", | 20260505 04:37:35.746 | INFO | "pinned" : " 29 GB" | 20260505 04:37:35.746 | INFO | } | 20260505 04:37:35.746 | INFO | Perf report for the last 5.13947 seconds: | 20260505 04:37:35.746 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:37:35.746 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:35.746 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:35.746 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:35.746 | INFO | ANALYSIS : 57.2%, 3.0, 88.7ms, 0 | 20260505 04:37:35.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:35.746 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:35.746 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260505 04:37:35.746 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:35.746 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:35.746 | INFO | Uploaded 36GiB to GPU in 1.60161s (22.4773GiB/s) | 20260505 04:37:35.746 | INFO | Downloaded 34GiB to GPU in 1.51779s (22.4009GiB/s) | 20260505 04:37:40.885 | INFO | Analyzing chunk frames = [395264,395776) | 20260505 04:37:40.885 | INFO | MemInfo: { | 20260505 04:37:40.885 | INFO | "gpu" : " 17 GB", | 20260505 04:37:40.885 | INFO | "huge" : "", | 20260505 04:37:40.885 | INFO | "huge_shared" : " 27 GB", | 20260505 04:37:40.885 | INFO | "malloc" : " 14 GB", | 20260505 04:37:40.885 | INFO | "pinned" : " 29 GB" | 20260505 04:37:40.885 | INFO | } | 20260505 04:37:40.885 | INFO | Perf report for the last 5.13933 seconds: | 20260505 04:37:40.885 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:37:40.885 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:40.885 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:40.885 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:40.885 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260505 04:37:40.885 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:40.885 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:40.885 | INFO | PRE_HQ : 31.7%, 1.0, 16.4ms, 0 | 20260505 04:37:40.885 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:40.885 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:40.885 | INFO | Uploaded 36GiB to GPU in 1.60016s (22.4978GiB/s) | 20260505 04:37:40.885 | INFO | Downloaded 34GiB to GPU in 1.51652s (22.4198GiB/s) | 20260505 04:37:46.024 | INFO | Analyzing chunk frames = [395776,396288) | 20260505 04:37:46.024 | INFO | MemInfo: { | 20260505 04:37:46.024 | INFO | "gpu" : " 17 GB", | 20260505 04:37:46.024 | INFO | "huge" : "", | 20260505 04:37:46.024 | INFO | "huge_shared" : " 27 GB", | 20260505 04:37:46.024 | INFO | "malloc" : " 14 GB", | 20260505 04:37:46.024 | INFO | "pinned" : " 29 GB" | 20260505 04:37:46.024 | INFO | } | 20260505 04:37:46.024 | INFO | Perf report for the last 5.13924 seconds: | 20260505 04:37:46.024 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:37:46.024 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:46.024 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:46.024 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:46.024 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 04:37:46.024 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:46.024 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:46.024 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 04:37:46.024 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:46.024 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:46.025 | INFO | Uploaded 36GiB to GPU in 1.60017s (22.4976GiB/s) | 20260505 04:37:46.025 | INFO | Downloaded 34GiB to GPU in 1.5168s (22.4157GiB/s) | 20260505 04:37:51.163 | INFO | Analyzing chunk frames = [396288,396800) | 20260505 04:37:51.163 | INFO | MemInfo: { | 20260505 04:37:51.163 | INFO | "gpu" : " 17 GB", | 20260505 04:37:51.163 | INFO | "huge" : "", | 20260505 04:37:51.163 | INFO | "huge_shared" : " 27 GB", | 20260505 04:37:51.163 | INFO | "malloc" : " 14 GB", | 20260505 04:37:51.163 | INFO | "pinned" : " 29 GB" | 20260505 04:37:51.164 | INFO | } | 20260505 04:37:51.164 | INFO | Perf report for the last 5.1394 seconds: | 20260505 04:37:51.164 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:37:51.164 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:51.164 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:51.164 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:51.164 | INFO | ANALYSIS : 57.6%, 3.0, 89.8ms, 0 | 20260505 04:37:51.164 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:51.164 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:51.164 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260505 04:37:51.164 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:51.164 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:51.164 | INFO | Uploaded 36GiB to GPU in 1.60025s (22.4965GiB/s) | 20260505 04:37:51.164 | INFO | Downloaded 34GiB to GPU in 1.51701s (22.4125GiB/s) | 20260505 04:37:56.303 | INFO | Analyzing chunk frames = [396800,397312) | 20260505 04:37:56.303 | INFO | MemInfo: { | 20260505 04:37:56.303 | INFO | "gpu" : " 17 GB", | 20260505 04:37:56.303 | INFO | "huge" : "", | 20260505 04:37:56.303 | INFO | "huge_shared" : " 27 GB", | 20260505 04:37:56.303 | INFO | "malloc" : " 14 GB", | 20260505 04:37:56.303 | INFO | "pinned" : " 29 GB" | 20260505 04:37:56.303 | INFO | } | 20260505 04:37:56.303 | INFO | Perf report for the last 5.13929 seconds: | 20260505 04:37:56.303 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:37:56.303 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:56.303 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:56.303 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:56.303 | INFO | ANALYSIS : 57.6%, 3.0, 89.9ms, 0 | 20260505 04:37:56.303 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:56.303 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:56.303 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260505 04:37:56.303 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:56.303 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:37:56.304 | INFO | Uploaded 36GiB to GPU in 1.6008s (22.4888GiB/s) | 20260505 04:37:56.304 | INFO | Downloaded 34GiB to GPU in 1.51807s (22.3968GiB/s) | 20260505 04:37:56.400 | INFO | Starting a round of Baz IO | 20260505 04:37:59.356 | INFO | Populating RTMetrics report | 20260505 04:37:59.356 | INFO | RTMetrics report populated | 20260505 04:37:59.356 | INFO | Opening tmp RTMetrics json file | 20260505 04:37:59.357 | INFO | Serializing RTMetrics to json | 20260505 04:37:59.358 | INFO | Serialization done, swapping tmp file now... | 20260505 04:37:59.358 | INFO | RTMetrics done | 20260505 04:37:59.446 | INFO | Finished a round of Baz IO: Wrote 2.015 GiB in 3.045 seconds | 20260505 04:37:59.446 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.26% / 0.14% / 53.61% | 20260505 04:37:59.446 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 10.432% / 0.000353654% / 4.76837e-05% | 20260505 04:37:59.446 | INFO | Baz IO overall: | 20260505 04:37:59.446 | INFO | BytesWritten : 159279369788 | 20260505 04:37:59.446 | INFO | NumEvents : 73847007109 | 20260505 04:38:01.442 | INFO | Analyzing chunk frames = [397312,397824) | 20260505 04:38:01.442 | INFO | MemInfo: { | 20260505 04:38:01.442 | INFO | "gpu" : " 17 GB", | 20260505 04:38:01.442 | INFO | "huge" : "", | 20260505 04:38:01.442 | INFO | "huge_shared" : " 27 GB", | 20260505 04:38:01.442 | INFO | "malloc" : " 14 GB", | 20260505 04:38:01.442 | INFO | "pinned" : " 29 GB" | 20260505 04:38:01.442 | INFO | } | 20260505 04:38:01.442 | INFO | Perf report for the last 5.13948 seconds: | 20260505 04:38:01.442 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:38:01.442 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:01.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:01.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:01.443 | INFO | ANALYSIS : 58.3%, 3.0, 90.7ms, 0 | 20260505 04:38:01.443 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:01.443 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260505 04:38:01.443 | INFO | PRE_HQ : 57.6%, 2.2, 64.5ms, 0 | 20260505 04:38:01.443 | INFO | BAZWRITER : 58.8%, 3.9, 118.3ms, 0 | 20260505 04:38:01.443 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:38:01.443 | INFO | Uploaded 36GiB to GPU in 1.63943s (21.9588GiB/s) | 20260505 04:38:01.443 | INFO | Downloaded 38GiB to GPU in 1.70119s (22.3373GiB/s) | 20260505 04:38:06.582 | INFO | Analyzing chunk frames = [397824,398336) | 20260505 04:38:06.582 | INFO | MemInfo: { | 20260505 04:38:06.582 | INFO | "gpu" : " 17 GB", | 20260505 04:38:06.582 | INFO | "huge" : "", | 20260505 04:38:06.582 | INFO | "huge_shared" : " 27 GB", | 20260505 04:38:06.582 | INFO | "malloc" : " 14 GB", | 20260505 04:38:06.582 | INFO | "pinned" : " 29 GB" | 20260505 04:38:06.582 | INFO | } | 20260505 04:38:06.582 | INFO | Perf report for the last 5.13929 seconds: | 20260505 04:38:06.582 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:38:06.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:06.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:06.582 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:06.582 | INFO | ANALYSIS : 58.0%, 3.0, 90.5ms, 0 | 20260505 04:38:06.582 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:06.582 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:06.582 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 04:38:06.582 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:06.582 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:06.582 | INFO | Uploaded 36GiB to GPU in 1.59912s (22.5124GiB/s) | 20260505 04:38:06.582 | INFO | Downloaded 34GiB to GPU in 1.51554s (22.4342GiB/s) | 20260505 04:38:11.721 | INFO | Analyzing chunk frames = [398336,398848) | 20260505 04:38:11.721 | INFO | MemInfo: { | 20260505 04:38:11.721 | INFO | "gpu" : " 17 GB", | 20260505 04:38:11.721 | INFO | "huge" : "", | 20260505 04:38:11.721 | INFO | "huge_shared" : " 27 GB", | 20260505 04:38:11.721 | INFO | "malloc" : " 14 GB", | 20260505 04:38:11.721 | INFO | "pinned" : " 29 GB" | 20260505 04:38:11.721 | INFO | } | 20260505 04:38:11.721 | INFO | Perf report for the last 5.13957 seconds: | 20260505 04:38:11.721 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:38:11.721 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:11.721 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:11.721 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:11.721 | INFO | ANALYSIS : 57.5%, 3.0, 89.3ms, 0 | 20260505 04:38:11.721 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:11.721 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:11.721 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260505 04:38:11.721 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:11.721 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:11.722 | INFO | Uploaded 36GiB to GPU in 1.60234s (22.4672GiB/s) | 20260505 04:38:11.722 | INFO | Downloaded 34GiB to GPU in 1.5193s (22.3788GiB/s) | 20260505 04:38:16.860 | INFO | Analyzing chunk frames = [398848,399360) | 20260505 04:38:16.860 | INFO | MemInfo: { | 20260505 04:38:16.860 | INFO | "gpu" : " 17 GB", | 20260505 04:38:16.860 | INFO | "huge" : "", | 20260505 04:38:16.860 | INFO | "huge_shared" : " 27 GB", | 20260505 04:38:16.860 | INFO | "malloc" : " 14 GB", | 20260505 04:38:16.860 | INFO | "pinned" : " 29 GB" | 20260505 04:38:16.860 | INFO | } | 20260505 04:38:16.860 | INFO | Perf report for the last 5.13912 seconds: | 20260505 04:38:16.860 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:38:16.860 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:16.860 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:16.861 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:16.861 | INFO | ANALYSIS : 57.1%, 3.0, 88.6ms, 0 | 20260505 04:38:16.861 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:16.861 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:16.861 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 04:38:16.861 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:16.861 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:16.861 | INFO | Uploaded 36GiB to GPU in 1.60313s (22.4561GiB/s) | 20260505 04:38:16.861 | INFO | Downloaded 34GiB to GPU in 1.5201s (22.367GiB/s) | 20260505 04:38:22.000 | INFO | Analyzing chunk frames = [399360,399872) | 20260505 04:38:22.000 | INFO | MemInfo: { | 20260505 04:38:22.000 | INFO | "gpu" : " 17 GB", | 20260505 04:38:22.000 | INFO | "huge" : "", | 20260505 04:38:22.000 | INFO | "huge_shared" : " 27 GB", | 20260505 04:38:22.000 | INFO | "malloc" : " 14 GB", | 20260505 04:38:22.000 | INFO | "pinned" : " 29 GB" | 20260505 04:38:22.000 | INFO | } | 20260505 04:38:22.000 | INFO | Perf report for the last 5.13944 seconds: | 20260505 04:38:22.000 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:38:22.000 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:22.000 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:22.000 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:22.000 | INFO | ANALYSIS : 57.0%, 3.0, 89.0ms, 0 | 20260505 04:38:22.000 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:22.000 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:22.000 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260505 04:38:22.000 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:22.000 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:22.000 | INFO | Uploaded 36GiB to GPU in 1.59977s (22.5032GiB/s) | 20260505 04:38:22.000 | INFO | Downloaded 34GiB to GPU in 1.51547s (22.4353GiB/s) | 20260505 04:38:27.139 | INFO | Analyzing chunk frames = [399872,400384) | 20260505 04:38:27.139 | INFO | MemInfo: { | 20260505 04:38:27.139 | INFO | "gpu" : " 17 GB", | 20260505 04:38:27.139 | INFO | "huge" : "", | 20260505 04:38:27.139 | INFO | "huge_shared" : " 27 GB", | 20260505 04:38:27.139 | INFO | "malloc" : " 14 GB", | 20260505 04:38:27.139 | INFO | "pinned" : " 29 GB" | 20260505 04:38:27.139 | INFO | } | 20260505 04:38:27.139 | INFO | Perf report for the last 5.13919 seconds: | 20260505 04:38:27.139 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:38:27.139 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:27.139 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:27.139 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:27.140 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 04:38:27.140 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:27.140 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:27.140 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260505 04:38:27.140 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:27.140 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:27.140 | INFO | Uploaded 36GiB to GPU in 1.60067s (22.4906GiB/s) | 20260505 04:38:27.140 | INFO | Downloaded 34GiB to GPU in 1.51721s (22.4096GiB/s) | 20260505 04:38:32.278 | INFO | Analyzing chunk frames = [400384,400896) | 20260505 04:38:32.278 | INFO | MemInfo: { | 20260505 04:38:32.278 | INFO | "gpu" : " 17 GB", | 20260505 04:38:32.278 | INFO | "huge" : "", | 20260505 04:38:32.278 | INFO | "huge_shared" : " 27 GB", | 20260505 04:38:32.278 | INFO | "malloc" : " 14 GB", | 20260505 04:38:32.279 | INFO | "pinned" : " 29 GB" | 20260505 04:38:32.279 | INFO | } | 20260505 04:38:32.279 | INFO | Perf report for the last 5.13944 seconds: | 20260505 04:38:32.279 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:38:32.279 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:32.279 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:32.279 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:32.279 | INFO | ANALYSIS : 57.6%, 3.0, 89.8ms, 0 | 20260505 04:38:32.279 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:32.279 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:32.279 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260505 04:38:32.279 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:32.279 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:32.279 | INFO | Uploaded 36GiB to GPU in 1.60073s (22.4897GiB/s) | 20260505 04:38:32.279 | INFO | Downloaded 34GiB to GPU in 1.51804s (22.3973GiB/s) | 20260505 04:38:37.418 | INFO | Analyzing chunk frames = [400896,401408) | 20260505 04:38:37.418 | INFO | MemInfo: { | 20260505 04:38:37.418 | INFO | "gpu" : " 17 GB", | 20260505 04:38:37.418 | INFO | "huge" : "", | 20260505 04:38:37.418 | INFO | "huge_shared" : " 27 GB", | 20260505 04:38:37.418 | INFO | "malloc" : " 14 GB", | 20260505 04:38:37.418 | INFO | "pinned" : " 29 GB" | 20260505 04:38:37.418 | INFO | } | 20260505 04:38:37.418 | INFO | Perf report for the last 5.13929 seconds: | 20260505 04:38:37.418 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:38:37.418 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:37.418 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:37.418 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:37.418 | INFO | ANALYSIS : 57.6%, 3.0, 89.9ms, 0 | 20260505 04:38:37.418 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:37.418 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:37.418 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 04:38:37.418 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:37.418 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:37.418 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 04:38:37.418 | INFO | Total mem usage: 14.9 GiB | 20260505 04:38:37.418 | INFO | All Time High Water Marks sum: 15.3 GiB (Savings: 2.13%) | 20260505 04:38:37.418 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -3.07%) | 20260505 04:38:37.418 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 04:38:37.418 | INFO | High Water Breakdown: | 20260505 04:38:37.418 | INFO | 59.3% : 8.59 GiB ( 100% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 04:38:37.418 | INFO | 40.6% : 5.88 GiB ( 88.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 04:38:37.418 | INFO | Uploaded 36GiB to GPU in 1.60103s (22.4856GiB/s) | 20260505 04:38:37.419 | INFO | Downloaded 34GiB to GPU in 1.51801s (22.3977GiB/s) | 20260505 04:38:37.418 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 04:38:37.419 | INFO | ------------------------------------------ | 20260505 04:38:37.419 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:38:37.419 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 04:38:37.419 | INFO | Total mem usage: 26.5 GiB | 20260505 04:38:37.419 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260505 04:38:37.419 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 04:38:37.419 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260505 04:38:37.419 | INFO | High Water Breakdown: | 20260505 04:38:37.419 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:38:37.419 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:38:37.419 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:38:37.419 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:38:37.419 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:38:37.419 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:38:37.419 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 04:38:37.419 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 04:38:37.419 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:38:37.419 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 04:38:37.419 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 04:38:37.419 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 04:38:37.419 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 04:38:37.419 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 04:38:37.419 | INFO | ------------------------------------------ | 20260505 04:38:37.419 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:38:37.419 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 04:38:37.419 | INFO | Total mem usage: 25.6 GiB | 20260505 04:38:37.419 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 04:38:37.419 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 04:38:37.419 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 04:38:37.419 | INFO | High Water Breakdown: | 20260505 04:38:37.419 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 04:38:37.419 | INFO | ------------------------------------------ | 20260505 04:38:37.419 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:38:37.419 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 04:38:37.419 | INFO | Total mem usage: 15.7 GiB | 20260505 04:38:37.419 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260505 04:38:37.419 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 04:38:37.419 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260505 04:38:37.419 | INFO | High Water Breakdown: | 20260505 04:38:37.419 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 04:38:37.419 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 04:38:37.419 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 04:38:37.419 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 04:38:37.419 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 04:38:37.419 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 04:38:37.419 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 04:38:37.419 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 04:38:37.419 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 04:38:37.419 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:38:37.419 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 04:38:37.419 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:38:37.419 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 04:38:37.419 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 04:38:37.419 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 04:38:37.419 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 04:38:37.419 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:38:37.419 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 04:38:37.419 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:38:37.419 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:38:37.419 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:38:37.419 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 04:38:37.419 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 04:38:37.419 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:38:37.419 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 04:38:37.419 | INFO | ------------------------------------------ | 20260505 04:38:37.419 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:38:37.419 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 04:38:37.419 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:38:37.419 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:38:37.419 | INFO | PerfReport: Full Execution -- 3.05e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260505 04:38:37.419 | INFO | PerfReport: 0.28%, AggregateStats -- 8.63e-02 / -nan, 4.43e-03 / 0.00e+00 | 20260505 04:38:37.419 | INFO | PerfReport: 10.82%, Baseline -- 3.30e+00 / -nan, 1.72e-02 / 0.00e+00 | 20260505 04:38:37.419 | INFO | PerfReport: 3.04%, Histogram -- 9.25e-01 / -nan, 5.22e-02 / 0.00e+00 | 20260505 04:38:37.419 | INFO | PerfReport: 16.03%, DME -- 3.91e+01 / -nan, 3.23e+00 / 0.00e+00 | 20260505 04:38:37.419 | INFO | PerfReport: 56.49%, FrameLabeling -- 1.72e+01 / -nan, 4.57e-02 / 0.00e+00 | 20260505 04:38:37.419 | INFO | PerfReport: 7.66%, PulseAccumulating -- 2.33e+00 / -nan, 1.55e-01 / 0.00e+00 | 20260505 04:38:37.419 | INFO | PerfReport: 4.02%, Metrics -- 1.23e+00 / -nan, 2.01e-01 / 0.00e+00 | 20260505 04:38:37.419 | INFO | PerfReport: 1.66%, PulseSerializing -- 5.07e-01 / -nan, 8.14e-02 / 0.00e+00 | 20260505 04:38:37.419 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 04:38:37.419 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:38:37.419 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:38:37.420 | INFO | PerfReport: Full Execution -- 9.15e+01 / -nan, 1.89e+01 / 0.00e+00 | 20260505 04:38:37.420 | INFO | PerfReport: 18.28%, Upload -- 1.67e+01 / 0.00e+00, 3.72e-01 / 0.00e+00 | 20260505 04:38:37.420 | INFO | PerfReport: 17.54%, Download -- 1.60e+01 / 0.00e+00, 6.96e-01 / 0.00e+00 | 20260505 04:38:37.420 | INFO | PerfReport: 33.30%, Compute -- 3.05e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260505 04:38:37.420 | INFO | PerfReport: 0.55%, LockUpload -- 5.04e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260505 04:38:37.420 | INFO | PerfReport: 0.00%, LockDownload -- 3.98e-04 / 0.00e+00, 9.53e-05 / 0.00e+00 | 20260505 04:38:37.420 | INFO | PerfReport: 30.32%, LockCompute -- 2.77e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260505 04:38:37.420 | INFO | PerfReport: 0.02%, Misc -- 1.48e-02 / 0.00e+00, 2.52e-02 / 0.00e+00 | 20260505 04:38:37.515 | INFO | Starting a round of Baz IO | 20260505 04:38:40.471 | INFO | Populating RTMetrics report | 20260505 04:38:40.471 | INFO | RTMetrics report populated | 20260505 04:38:40.471 | INFO | Opening tmp RTMetrics json file | 20260505 04:38:40.472 | INFO | Serializing RTMetrics to json | 20260505 04:38:40.473 | INFO | Serialization done, swapping tmp file now... | 20260505 04:38:40.473 | INFO | RTMetrics done | 20260505 04:38:40.560 | INFO | Finished a round of Baz IO: Wrote 2.017 GiB in 3.045 seconds | 20260505 04:38:40.560 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.12% / 0.13% / 53.74% | 20260505 04:38:40.560 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 10.3556% / 0.000286102% / 5.5631e-05% | 20260505 04:38:40.560 | INFO | Baz IO overall: | 20260505 04:38:40.560 | INFO | BytesWritten : 161444621272 | 20260505 04:38:40.560 | INFO | NumEvents : 74872509572 | 20260505 04:38:42.557 | INFO | Analyzing chunk frames = [401408,401920) | 20260505 04:38:42.557 | INFO | MemInfo: { | 20260505 04:38:42.557 | INFO | "gpu" : " 17 GB", | 20260505 04:38:42.557 | INFO | "huge" : "", | 20260505 04:38:42.557 | INFO | "huge_shared" : " 27 GB", | 20260505 04:38:42.557 | INFO | "malloc" : " 14 GB", | 20260505 04:38:42.557 | INFO | "pinned" : " 29 GB" | 20260505 04:38:42.557 | INFO | } | 20260505 04:38:42.557 | INFO | Perf report for the last 5.1391 seconds: | 20260505 04:38:42.557 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:38:42.557 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:42.557 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:42.557 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:42.557 | INFO | ANALYSIS : 58.3%, 3.0, 90.7ms, 0 | 20260505 04:38:42.557 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:42.557 | INFO | RT_METRICS : 45.1%, 1.0, 23.7ms, 0 | 20260505 04:38:42.557 | INFO | PRE_HQ : 57.6%, 2.1, 64.1ms, 0 | 20260505 04:38:42.557 | INFO | BAZWRITER : 58.8%, 3.9, 118.9ms, 0 | 20260505 04:38:42.557 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:38:42.557 | INFO | Uploaded 36GiB to GPU in 1.63418s (22.0294GiB/s) | 20260505 04:38:42.557 | INFO | Downloaded 38GiB to GPU in 1.69573s (22.4092GiB/s) | 20260505 04:38:47.696 | INFO | Analyzing chunk frames = [401920,402432) | 20260505 04:38:47.696 | INFO | MemInfo: { | 20260505 04:38:47.696 | INFO | "gpu" : " 17 GB", | 20260505 04:38:47.696 | INFO | "huge" : "", | 20260505 04:38:47.696 | INFO | "huge_shared" : " 27 GB", | 20260505 04:38:47.697 | INFO | "malloc" : " 14 GB", | 20260505 04:38:47.697 | INFO | "pinned" : " 29 GB" | 20260505 04:38:47.697 | INFO | } | 20260505 04:38:47.697 | INFO | Perf report for the last 5.13969 seconds: | 20260505 04:38:47.697 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:38:47.697 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:47.697 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:47.697 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:47.697 | INFO | ANALYSIS : 58.0%, 3.0, 90.5ms, 0 | 20260505 04:38:47.697 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:47.697 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:47.697 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260505 04:38:47.697 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:47.697 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:47.697 | INFO | Uploaded 36GiB to GPU in 1.5999s (22.5014GiB/s) | 20260505 04:38:47.697 | INFO | Downloaded 34GiB to GPU in 1.51682s (22.4153GiB/s) | 20260505 04:38:52.836 | INFO | Analyzing chunk frames = [402432,402944) | 20260505 04:38:52.836 | INFO | MemInfo: { | 20260505 04:38:52.836 | INFO | "gpu" : " 17 GB", | 20260505 04:38:52.836 | INFO | "huge" : "", | 20260505 04:38:52.836 | INFO | "huge_shared" : " 27 GB", | 20260505 04:38:52.836 | INFO | "malloc" : " 14 GB", | 20260505 04:38:52.836 | INFO | "pinned" : " 29 GB" | 20260505 04:38:52.836 | INFO | } | 20260505 04:38:52.836 | INFO | Perf report for the last 5.13924 seconds: | 20260505 04:38:52.836 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:38:52.836 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:52.836 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:52.836 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:52.836 | INFO | ANALYSIS : 57.4%, 3.0, 89.3ms, 0 | 20260505 04:38:52.836 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:52.836 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:52.836 | INFO | PRE_HQ : 31.7%, 1.0, 16.4ms, 0 | 20260505 04:38:52.836 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:52.836 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:52.837 | INFO | Uploaded 36GiB to GPU in 1.60238s (22.4666GiB/s) | 20260505 04:38:52.837 | INFO | Downloaded 34GiB to GPU in 1.51904s (22.3826GiB/s) | 20260505 04:38:57.975 | INFO | Analyzing chunk frames = [402944,403456) | 20260505 04:38:57.975 | INFO | MemInfo: { | 20260505 04:38:57.975 | INFO | "gpu" : " 17 GB", | 20260505 04:38:57.975 | INFO | "huge" : "", | 20260505 04:38:57.975 | INFO | "huge_shared" : " 27 GB", | 20260505 04:38:57.975 | INFO | "malloc" : " 14 GB", | 20260505 04:38:57.975 | INFO | "pinned" : " 29 GB" | 20260505 04:38:57.975 | INFO | } | 20260505 04:38:57.975 | INFO | Perf report for the last 5.13926 seconds: | 20260505 04:38:57.975 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:38:57.975 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:57.975 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:57.975 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:57.975 | INFO | ANALYSIS : 57.1%, 3.0, 88.5ms, 0 | 20260505 04:38:57.975 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:57.975 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:57.975 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 04:38:57.975 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:57.975 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:38:57.976 | INFO | Uploaded 36GiB to GPU in 1.60173s (22.4757GiB/s) | 20260505 04:38:57.976 | INFO | Downloaded 34GiB to GPU in 1.51879s (22.3863GiB/s) | 20260505 04:39:03.115 | INFO | Analyzing chunk frames = [403456,403968) | 20260505 04:39:03.115 | INFO | MemInfo: { | 20260505 04:39:03.115 | INFO | "gpu" : " 17 GB", | 20260505 04:39:03.115 | INFO | "huge" : "", | 20260505 04:39:03.115 | INFO | "huge_shared" : " 27 GB", | 20260505 04:39:03.115 | INFO | "malloc" : " 14 GB", | 20260505 04:39:03.115 | INFO | "pinned" : " 29 GB" | 20260505 04:39:03.115 | INFO | } | 20260505 04:39:03.115 | INFO | Perf report for the last 5.1396 seconds: | 20260505 04:39:03.115 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:39:03.115 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:03.115 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:03.115 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:03.115 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260505 04:39:03.115 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:03.115 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:03.115 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260505 04:39:03.115 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:03.115 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:03.115 | INFO | Uploaded 36GiB to GPU in 1.60031s (22.4956GiB/s) | 20260505 04:39:03.115 | INFO | Downloaded 34GiB to GPU in 1.51671s (22.417GiB/s) | 20260505 04:39:08.254 | INFO | Analyzing chunk frames = [403968,404480) | 20260505 04:39:08.254 | INFO | MemInfo: { | 20260505 04:39:08.254 | INFO | "gpu" : " 17 GB", | 20260505 04:39:08.254 | INFO | "huge" : "", | 20260505 04:39:08.254 | INFO | "huge_shared" : " 27 GB", | 20260505 04:39:08.254 | INFO | "malloc" : " 14 GB", | 20260505 04:39:08.254 | INFO | "pinned" : " 29 GB" | 20260505 04:39:08.254 | INFO | } | 20260505 04:39:08.254 | INFO | Perf report for the last 5.13901 seconds: | 20260505 04:39:08.254 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:39:08.254 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:08.254 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:08.254 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:08.254 | INFO | ANALYSIS : 57.3%, 3.0, 89.3ms, 0 | 20260505 04:39:08.254 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:08.254 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:08.254 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 04:39:08.254 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:08.254 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:08.254 | INFO | Uploaded 36GiB to GPU in 1.59972s (22.5039GiB/s) | 20260505 04:39:08.254 | INFO | Downloaded 34GiB to GPU in 1.51615s (22.4253GiB/s) | 20260505 04:39:13.393 | INFO | Analyzing chunk frames = [404480,404992) | 20260505 04:39:13.393 | INFO | MemInfo: { | 20260505 04:39:13.393 | INFO | "gpu" : " 17 GB", | 20260505 04:39:13.393 | INFO | "huge" : "", | 20260505 04:39:13.393 | INFO | "huge_shared" : " 27 GB", | 20260505 04:39:13.393 | INFO | "malloc" : " 14 GB", | 20260505 04:39:13.393 | INFO | "pinned" : " 29 GB" | 20260505 04:39:13.393 | INFO | } | 20260505 04:39:13.393 | INFO | Perf report for the last 5.1393 seconds: | 20260505 04:39:13.393 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:39:13.393 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:13.393 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:13.393 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:13.393 | INFO | ANALYSIS : 57.6%, 3.0, 89.8ms, 0 | 20260505 04:39:13.393 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:13.393 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:13.393 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 04:39:13.393 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:13.393 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:13.393 | INFO | Uploaded 36GiB to GPU in 1.5994s (22.5084GiB/s) | 20260505 04:39:13.393 | INFO | Downloaded 34GiB to GPU in 1.51705s (22.4119GiB/s) | 20260505 04:39:18.532 | INFO | Analyzing chunk frames = [404992,405504) | 20260505 04:39:18.532 | INFO | MemInfo: { | 20260505 04:39:18.532 | INFO | "gpu" : " 17 GB", | 20260505 04:39:18.532 | INFO | "huge" : "", | 20260505 04:39:18.532 | INFO | "huge_shared" : " 27 GB", | 20260505 04:39:18.532 | INFO | "malloc" : " 14 GB", | 20260505 04:39:18.533 | INFO | "pinned" : " 29 GB" | 20260505 04:39:18.533 | INFO | } | 20260505 04:39:18.533 | INFO | Perf report for the last 5.13947 seconds: | 20260505 04:39:18.533 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:39:18.533 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:18.533 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:18.533 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:18.533 | INFO | ANALYSIS : 57.6%, 3.0, 89.8ms, 0 | 20260505 04:39:18.533 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:18.533 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:18.533 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 04:39:18.533 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:18.533 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:18.533 | INFO | Uploaded 36GiB to GPU in 1.60037s (22.4949GiB/s) | 20260505 04:39:18.533 | INFO | Downloaded 34GiB to GPU in 1.51807s (22.3968GiB/s) | 20260505 04:39:18.630 | INFO | Starting a round of Baz IO | 20260505 04:39:21.585 | INFO | Populating RTMetrics report | 20260505 04:39:21.585 | INFO | RTMetrics report populated | 20260505 04:39:21.585 | INFO | Opening tmp RTMetrics json file | 20260505 04:39:21.586 | INFO | Serializing RTMetrics to json | 20260505 04:39:21.587 | INFO | Serialization done, swapping tmp file now... | 20260505 04:39:21.587 | INFO | RTMetrics done | 20260505 04:39:21.675 | INFO | Finished a round of Baz IO: Wrote 2.019 GiB in 3.045 seconds | 20260505 04:39:21.675 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.99% / 0.13% / 53.88% | 20260505 04:39:21.675 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 10.2845% / 0.000266234% / 3.97364e-05% | 20260505 04:39:21.675 | INFO | Baz IO overall: | 20260505 04:39:21.675 | INFO | BytesWritten : 163612666059 | 20260505 04:39:21.675 | INFO | NumEvents : 75899930208 | 20260505 04:39:23.672 | INFO | Analyzing chunk frames = [405504,406016) | 20260505 04:39:23.672 | INFO | MemInfo: { | 20260505 04:39:23.672 | INFO | "gpu" : " 17 GB", | 20260505 04:39:23.672 | INFO | "huge" : "", | 20260505 04:39:23.672 | INFO | "huge_shared" : " 27 GB", | 20260505 04:39:23.672 | INFO | "malloc" : " 14 GB", | 20260505 04:39:23.672 | INFO | "pinned" : " 29 GB" | 20260505 04:39:23.672 | INFO | } | 20260505 04:39:23.672 | INFO | Perf report for the last 5.13945 seconds: | 20260505 04:39:23.672 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:39:23.672 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:23.672 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:23.672 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:23.672 | INFO | ANALYSIS : 58.3%, 3.0, 90.6ms, 0 | 20260505 04:39:23.672 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:23.672 | INFO | RT_METRICS : 45.3%, 1.0, 23.7ms, 0 | 20260505 04:39:23.672 | INFO | PRE_HQ : 57.7%, 2.2, 64.7ms, 0 | 20260505 04:39:23.672 | INFO | BAZWRITER : 58.8%, 4.0, 120.1ms, 0 | 20260505 04:39:23.672 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:39:23.672 | INFO | Uploaded 36GiB to GPU in 1.63833s (21.9736GiB/s) | 20260505 04:39:23.672 | INFO | Downloaded 38GiB to GPU in 1.70037s (22.3481GiB/s) | 20260505 04:39:28.811 | INFO | Analyzing chunk frames = [406016,406528) | 20260505 04:39:28.811 | INFO | MemInfo: { | 20260505 04:39:28.811 | INFO | "gpu" : " 17 GB", | 20260505 04:39:28.811 | INFO | "huge" : "", | 20260505 04:39:28.811 | INFO | "huge_shared" : " 27 GB", | 20260505 04:39:28.811 | INFO | "malloc" : " 14 GB", | 20260505 04:39:28.811 | INFO | "pinned" : " 29 GB" | 20260505 04:39:28.811 | INFO | } | 20260505 04:39:28.811 | INFO | Perf report for the last 5.1393 seconds: | 20260505 04:39:28.811 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:39:28.811 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:28.811 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:28.812 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:28.812 | INFO | ANALYSIS : 58.0%, 3.0, 90.5ms, 0 | 20260505 04:39:28.812 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:28.812 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:28.812 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 04:39:28.812 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:28.812 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:28.812 | INFO | Uploaded 36GiB to GPU in 1.59892s (22.5152GiB/s) | 20260505 04:39:28.812 | INFO | Downloaded 34GiB to GPU in 1.51525s (22.4386GiB/s) | 20260505 04:39:33.951 | INFO | Analyzing chunk frames = [406528,407040) | 20260505 04:39:33.951 | INFO | MemInfo: { | 20260505 04:39:33.951 | INFO | "gpu" : " 17 GB", | 20260505 04:39:33.951 | INFO | "huge" : "", | 20260505 04:39:33.951 | INFO | "huge_shared" : " 27 GB", | 20260505 04:39:33.951 | INFO | "malloc" : " 14 GB", | 20260505 04:39:33.951 | INFO | "pinned" : " 29 GB" | 20260505 04:39:33.951 | INFO | } | 20260505 04:39:33.951 | INFO | Perf report for the last 5.13937 seconds: | 20260505 04:39:33.951 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:39:33.951 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:33.951 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:33.951 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:33.951 | INFO | ANALYSIS : 57.4%, 3.0, 89.3ms, 0 | 20260505 04:39:33.951 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:33.951 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:33.951 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260505 04:39:33.951 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:33.951 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:33.951 | INFO | Uploaded 36GiB to GPU in 1.602s (22.4719GiB/s) | 20260505 04:39:33.952 | INFO | Downloaded 34GiB to GPU in 1.51888s (22.3849GiB/s) | 20260505 04:39:39.090 | INFO | Analyzing chunk frames = [407040,407552) | 20260505 04:39:39.090 | INFO | MemInfo: { | 20260505 04:39:39.090 | INFO | "gpu" : " 17 GB", | 20260505 04:39:39.090 | INFO | "huge" : "", | 20260505 04:39:39.090 | INFO | "huge_shared" : " 27 GB", | 20260505 04:39:39.090 | INFO | "malloc" : " 14 GB", | 20260505 04:39:39.090 | INFO | "pinned" : " 29 GB" | 20260505 04:39:39.090 | INFO | } | 20260505 04:39:39.090 | INFO | Perf report for the last 5.13918 seconds: | 20260505 04:39:39.090 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:39:39.090 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:39.090 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:39.090 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:39.090 | INFO | ANALYSIS : 57.1%, 3.0, 88.6ms, 0 | 20260505 04:39:39.090 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:39.090 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:39.090 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 04:39:39.090 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:39.090 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:39.090 | INFO | Uploaded 36GiB to GPU in 1.60072s (22.4899GiB/s) | 20260505 04:39:39.090 | INFO | Downloaded 34GiB to GPU in 1.51798s (22.3982GiB/s) | 20260505 04:39:44.229 | INFO | Analyzing chunk frames = [407552,408064) | 20260505 04:39:44.229 | INFO | MemInfo: { | 20260505 04:39:44.229 | INFO | "gpu" : " 17 GB", | 20260505 04:39:44.229 | INFO | "huge" : "", | 20260505 04:39:44.229 | INFO | "huge_shared" : " 27 GB", | 20260505 04:39:44.229 | INFO | "malloc" : " 14 GB", | 20260505 04:39:44.230 | INFO | "pinned" : " 29 GB" | 20260505 04:39:44.230 | INFO | } | 20260505 04:39:44.230 | INFO | Perf report for the last 5.13966 seconds: | 20260505 04:39:44.230 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:39:44.230 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:44.230 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:44.230 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:44.230 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260505 04:39:44.230 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:44.230 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:44.230 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 04:39:44.230 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:44.230 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:44.230 | INFO | Uploaded 36GiB to GPU in 1.60016s (22.4978GiB/s) | 20260505 04:39:44.230 | INFO | Downloaded 34GiB to GPU in 1.51652s (22.4197GiB/s) | 20260505 04:39:49.369 | INFO | Analyzing chunk frames = [408064,408576) | 20260505 04:39:49.369 | INFO | MemInfo: { | 20260505 04:39:49.369 | INFO | "gpu" : " 17 GB", | 20260505 04:39:49.369 | INFO | "huge" : "", | 20260505 04:39:49.369 | INFO | "huge_shared" : " 27 GB", | 20260505 04:39:49.369 | INFO | "malloc" : " 14 GB", | 20260505 04:39:49.369 | INFO | "pinned" : " 29 GB" | 20260505 04:39:49.369 | INFO | } | 20260505 04:39:49.369 | INFO | Perf report for the last 5.13907 seconds: | 20260505 04:39:49.369 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:39:49.369 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:49.369 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:49.369 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:49.369 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 04:39:49.369 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:49.369 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:49.369 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 04:39:49.369 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:49.369 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:49.369 | INFO | Uploaded 36GiB to GPU in 1.60059s (22.4917GiB/s) | 20260505 04:39:49.369 | INFO | Downloaded 34GiB to GPU in 1.51709s (22.4113GiB/s) | 20260505 04:39:54.508 | INFO | Analyzing chunk frames = [408576,409088) | 20260505 04:39:54.508 | INFO | MemInfo: { | 20260505 04:39:54.508 | INFO | "gpu" : " 17 GB", | 20260505 04:39:54.508 | INFO | "huge" : "", | 20260505 04:39:54.508 | INFO | "huge_shared" : " 27 GB", | 20260505 04:39:54.508 | INFO | "malloc" : " 14 GB", | 20260505 04:39:54.508 | INFO | "pinned" : " 29 GB" | 20260505 04:39:54.508 | INFO | } | 20260505 04:39:54.508 | INFO | Perf report for the last 5.13921 seconds: | 20260505 04:39:54.508 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:39:54.508 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:54.508 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:54.508 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:54.508 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260505 04:39:54.508 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:54.508 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:54.508 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260505 04:39:54.508 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:54.508 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:54.508 | INFO | Uploaded 36GiB to GPU in 1.60104s (22.4854GiB/s) | 20260505 04:39:54.508 | INFO | Downloaded 34GiB to GPU in 1.51769s (22.4024GiB/s) | 20260505 04:39:59.647 | INFO | Analyzing chunk frames = [409088,409600) | 20260505 04:39:59.647 | INFO | MemInfo: { | 20260505 04:39:59.647 | INFO | "gpu" : " 17 GB", | 20260505 04:39:59.647 | INFO | "huge" : "", | 20260505 04:39:59.647 | INFO | "huge_shared" : " 27 GB", | 20260505 04:39:59.648 | INFO | "malloc" : " 14 GB", | 20260505 04:39:59.648 | INFO | "pinned" : " 29 GB" | 20260505 04:39:59.648 | INFO | } | 20260505 04:39:59.648 | INFO | Perf report for the last 5.13962 seconds: | 20260505 04:39:59.648 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:39:59.648 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:59.648 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:59.648 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:59.648 | INFO | ANALYSIS : 57.6%, 3.0, 89.9ms, 0 | 20260505 04:39:59.648 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:59.648 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:59.648 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 04:39:59.648 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:59.648 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:39:59.648 | INFO | Uploaded 36GiB to GPU in 1.60062s (22.4912GiB/s) | 20260505 04:39:59.648 | INFO | Downloaded 34GiB to GPU in 1.51733s (22.4078GiB/s) | 20260505 04:39:59.648 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 04:39:59.648 | INFO | Total mem usage: 14.9 GiB | 20260505 04:39:59.648 | INFO | All Time High Water Marks sum: 15.3 GiB (Savings: 2.13%) | 20260505 04:39:59.648 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -3.24%) | 20260505 04:39:59.648 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 04:39:59.648 | INFO | High Water Breakdown: | 20260505 04:39:59.648 | INFO | 59.3% : 8.57 GiB ( 99.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 04:39:59.648 | INFO | 40.6% : 5.87 GiB ( 88.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 04:39:59.648 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 04:39:59.648 | INFO | ------------------------------------------ | 20260505 04:39:59.648 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:39:59.648 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 04:39:59.648 | INFO | Total mem usage: 26.5 GiB | 20260505 04:39:59.648 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260505 04:39:59.648 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 04:39:59.648 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260505 04:39:59.648 | INFO | High Water Breakdown: | 20260505 04:39:59.648 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:39:59.648 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:39:59.648 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:39:59.649 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:39:59.649 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:39:59.649 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:39:59.649 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 04:39:59.649 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 04:39:59.649 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:39:59.649 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 04:39:59.649 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 04:39:59.649 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 04:39:59.649 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 04:39:59.649 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 04:39:59.649 | INFO | ------------------------------------------ | 20260505 04:39:59.649 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:39:59.649 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 04:39:59.649 | INFO | Total mem usage: 25.6 GiB | 20260505 04:39:59.649 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 04:39:59.649 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 04:39:59.649 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 04:39:59.649 | INFO | High Water Breakdown: | 20260505 04:39:59.649 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 04:39:59.649 | INFO | ------------------------------------------ | 20260505 04:39:59.649 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:39:59.649 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 04:39:59.649 | INFO | Total mem usage: 15.7 GiB | 20260505 04:39:59.649 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260505 04:39:59.649 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 04:39:59.649 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260505 04:39:59.649 | INFO | High Water Breakdown: | 20260505 04:39:59.649 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 04:39:59.649 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 04:39:59.649 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 04:39:59.649 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 04:39:59.649 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 04:39:59.649 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 04:39:59.649 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 04:39:59.649 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 04:39:59.649 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 04:39:59.649 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:39:59.649 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 04:39:59.649 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:39:59.649 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 04:39:59.649 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 04:39:59.649 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 04:39:59.649 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 04:39:59.649 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:39:59.649 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 04:39:59.649 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:39:59.649 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:39:59.649 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:39:59.649 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 04:39:59.649 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 04:39:59.649 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:39:59.649 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 04:39:59.649 | INFO | ------------------------------------------ | 20260505 04:39:59.649 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:39:59.649 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 04:39:59.649 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:39:59.649 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:39:59.649 | INFO | PerfReport: Full Execution -- 3.05e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260505 04:39:59.649 | INFO | PerfReport: 0.28%, AggregateStats -- 8.63e-02 / -nan, 4.36e-03 / 0.00e+00 | 20260505 04:39:59.649 | INFO | PerfReport: 10.82%, Baseline -- 3.30e+00 / -nan, 1.81e-02 / 0.00e+00 | 20260505 04:39:59.649 | INFO | PerfReport: 3.03%, Histogram -- 9.24e-01 / -nan, 5.18e-02 / 0.00e+00 | 20260505 04:39:59.649 | INFO | PerfReport: 16.02%, DME -- 3.90e+01 / -nan, 3.23e+00 / 0.00e+00 | 20260505 04:39:59.649 | INFO | PerfReport: 56.51%, FrameLabeling -- 1.72e+01 / -nan, 4.55e-02 / 0.00e+00 | 20260505 04:39:59.649 | INFO | PerfReport: 7.65%, PulseAccumulating -- 2.33e+00 / -nan, 1.55e-01 / 0.00e+00 | 20260505 04:39:59.649 | INFO | PerfReport: 4.02%, Metrics -- 1.22e+00 / -nan, 2.03e-01 / 0.00e+00 | 20260505 04:39:59.649 | INFO | PerfReport: 1.66%, PulseSerializing -- 5.06e-01 / -nan, 8.11e-02 / 0.00e+00 | 20260505 04:39:59.649 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 04:39:59.649 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:39:59.649 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:39:59.649 | INFO | PerfReport: Full Execution -- 9.15e+01 / -nan, 1.91e+01 / 0.00e+00 | 20260505 04:39:59.649 | INFO | PerfReport: 18.28%, Upload -- 1.67e+01 / 0.00e+00, 3.63e-01 / 0.00e+00 | 20260505 04:39:59.649 | INFO | PerfReport: 17.54%, Download -- 1.60e+01 / 0.00e+00, 6.88e-01 / 0.00e+00 | 20260505 04:39:59.649 | INFO | PerfReport: 33.30%, Compute -- 3.05e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260505 04:39:59.649 | INFO | PerfReport: 0.55%, LockUpload -- 5.03e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260505 04:39:59.649 | INFO | PerfReport: 0.00%, LockDownload -- 3.96e-04 / 0.00e+00, 9.47e-05 / 0.00e+00 | 20260505 04:39:59.649 | INFO | PerfReport: 30.32%, LockCompute -- 2.77e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260505 04:39:59.649 | INFO | PerfReport: 0.02%, Misc -- 1.44e-02 / 0.00e+00, 2.57e-02 / 0.00e+00 | 20260505 04:39:59.745 | INFO | Starting a round of Baz IO | 20260505 04:40:02.696 | INFO | Populating RTMetrics report | 20260505 04:40:02.696 | INFO | RTMetrics report populated | 20260505 04:40:02.696 | INFO | Opening tmp RTMetrics json file | 20260505 04:40:02.700 | INFO | Serializing RTMetrics to json | 20260505 04:40:02.701 | INFO | Serialization done, swapping tmp file now... | 20260505 04:40:02.701 | INFO | RTMetrics done | 20260505 04:40:02.784 | INFO | Finished a round of Baz IO: Wrote 2.025 GiB in 3.039 seconds | 20260505 04:40:02.784 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.86% / 0.13% / 54.01% | 20260505 04:40:02.784 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 10.2542% / 0.000313918% / 6.35783e-05% | 20260505 04:40:02.784 | INFO | Baz IO overall: | 20260505 04:40:02.784 | INFO | BytesWritten : 165786518771 | 20260505 04:40:02.784 | INFO | NumEvents : 76930699054 | 20260505 04:40:04.787 | INFO | Analyzing chunk frames = [409600,410112) | 20260505 04:40:04.787 | INFO | MemInfo: { | 20260505 04:40:04.787 | INFO | "gpu" : " 17 GB", | 20260505 04:40:04.787 | INFO | "huge" : "", | 20260505 04:40:04.787 | INFO | "huge_shared" : " 27 GB", | 20260505 04:40:04.787 | INFO | "malloc" : " 14 GB", | 20260505 04:40:04.787 | INFO | "pinned" : " 29 GB" | 20260505 04:40:04.787 | INFO | } | 20260505 04:40:04.787 | INFO | Perf report for the last 5.13896 seconds: | 20260505 04:40:04.787 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:40:04.787 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:04.787 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:04.787 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:04.787 | INFO | ANALYSIS : 58.2%, 3.0, 90.4ms, 0 | 20260505 04:40:04.787 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:04.787 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260505 04:40:04.787 | INFO | PRE_HQ : 57.6%, 2.2, 64.7ms, 0 | 20260505 04:40:04.787 | INFO | BAZWRITER : 58.6%, 4.0, 119.3ms, 0 | 20260505 04:40:04.787 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 04:40:04.787 | INFO | Uploaded 36GiB to GPU in 1.63569s (22.009GiB/s) | 20260505 04:40:04.787 | INFO | Downloaded 38GiB to GPU in 1.69718s (22.3901GiB/s) | 20260505 04:40:09.926 | INFO | Analyzing chunk frames = [410112,410624) | 20260505 04:40:09.926 | INFO | MemInfo: { | 20260505 04:40:09.926 | INFO | "gpu" : " 17 GB", | 20260505 04:40:09.926 | INFO | "huge" : "", | 20260505 04:40:09.926 | INFO | "huge_shared" : " 27 GB", | 20260505 04:40:09.926 | INFO | "malloc" : " 14 GB", | 20260505 04:40:09.926 | INFO | "pinned" : " 29 GB" | 20260505 04:40:09.926 | INFO | } | 20260505 04:40:09.926 | INFO | Perf report for the last 5.13945 seconds: | 20260505 04:40:09.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:40:09.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:09.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:09.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:09.926 | INFO | ANALYSIS : 58.0%, 3.0, 90.5ms, 0 | 20260505 04:40:09.926 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:09.926 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:09.926 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 04:40:09.926 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:09.926 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:09.927 | INFO | Uploaded 36GiB to GPU in 1.59836s (22.5231GiB/s) | 20260505 04:40:09.927 | INFO | Downloaded 34GiB to GPU in 1.51501s (22.4421GiB/s) | 20260505 04:40:15.065 | INFO | Analyzing chunk frames = [410624,411136) | 20260505 04:40:15.065 | INFO | MemInfo: { | 20260505 04:40:15.065 | INFO | "gpu" : " 17 GB", | 20260505 04:40:15.065 | INFO | "huge" : "", | 20260505 04:40:15.065 | INFO | "huge_shared" : " 27 GB", | 20260505 04:40:15.065 | INFO | "malloc" : " 14 GB", | 20260505 04:40:15.065 | INFO | "pinned" : " 29 GB" | 20260505 04:40:15.065 | INFO | } | 20260505 04:40:15.065 | INFO | Perf report for the last 5.1392 seconds: | 20260505 04:40:15.065 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:40:15.065 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:15.065 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:15.065 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:15.065 | INFO | ANALYSIS : 57.4%, 3.0, 89.1ms, 0 | 20260505 04:40:15.065 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:15.065 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:15.065 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 04:40:15.065 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:15.065 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:15.066 | INFO | Uploaded 36GiB to GPU in 1.6024s (22.4663GiB/s) | 20260505 04:40:15.066 | INFO | Downloaded 34GiB to GPU in 1.51959s (22.3745GiB/s) | 20260505 04:40:20.205 | INFO | Analyzing chunk frames = [411136,411648) | 20260505 04:40:20.205 | INFO | MemInfo: { | 20260505 04:40:20.205 | INFO | "gpu" : " 17 GB", | 20260505 04:40:20.205 | INFO | "huge" : "", | 20260505 04:40:20.205 | INFO | "huge_shared" : " 27 GB", | 20260505 04:40:20.205 | INFO | "malloc" : " 14 GB", | 20260505 04:40:20.205 | INFO | "pinned" : " 29 GB" | 20260505 04:40:20.205 | INFO | } | 20260505 04:40:20.205 | INFO | Perf report for the last 5.13948 seconds: | 20260505 04:40:20.205 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:40:20.205 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:20.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:20.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:20.205 | INFO | ANALYSIS : 57.1%, 3.0, 88.6ms, 0 | 20260505 04:40:20.205 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:20.205 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:20.205 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260505 04:40:20.205 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:20.205 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:20.205 | INFO | Uploaded 36GiB to GPU in 1.60183s (22.4743GiB/s) | 20260505 04:40:20.205 | INFO | Downloaded 34GiB to GPU in 1.51865s (22.3882GiB/s) | 20260505 04:40:25.344 | INFO | Analyzing chunk frames = [411648,412160) | 20260505 04:40:25.344 | INFO | MemInfo: { | 20260505 04:40:25.344 | INFO | "gpu" : " 17 GB", | 20260505 04:40:25.344 | INFO | "huge" : "", | 20260505 04:40:25.344 | INFO | "huge_shared" : " 27 GB", | 20260505 04:40:25.344 | INFO | "malloc" : " 14 GB", | 20260505 04:40:25.344 | INFO | "pinned" : " 29 GB" | 20260505 04:40:25.344 | INFO | } | 20260505 04:40:25.344 | INFO | Perf report for the last 5.13941 seconds: | 20260505 04:40:25.344 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:40:25.344 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:25.344 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:25.344 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:25.344 | INFO | ANALYSIS : 57.0%, 3.0, 89.0ms, 0 | 20260505 04:40:25.344 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:25.344 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:25.344 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260505 04:40:25.344 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:25.344 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:25.345 | INFO | Uploaded 36GiB to GPU in 1.60025s (22.4965GiB/s) | 20260505 04:40:25.345 | INFO | Downloaded 34GiB to GPU in 1.51667s (22.4176GiB/s) | 20260505 04:40:30.483 | INFO | Analyzing chunk frames = [412160,412672) | 20260505 04:40:30.483 | INFO | MemInfo: { | 20260505 04:40:30.483 | INFO | "gpu" : " 17 GB", | 20260505 04:40:30.483 | INFO | "huge" : "", | 20260505 04:40:30.483 | INFO | "huge_shared" : " 27 GB", | 20260505 04:40:30.483 | INFO | "malloc" : " 14 GB", | 20260505 04:40:30.483 | INFO | "pinned" : " 29 GB" | 20260505 04:40:30.483 | INFO | } | 20260505 04:40:30.484 | INFO | Perf report for the last 5.13931 seconds: | 20260505 04:40:30.484 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:40:30.484 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:30.484 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:30.484 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:30.484 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 04:40:30.484 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:30.484 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:30.484 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 04:40:30.484 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:30.484 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:30.484 | INFO | Uploaded 36GiB to GPU in 1.60017s (22.4976GiB/s) | 20260505 04:40:30.484 | INFO | Downloaded 34GiB to GPU in 1.51634s (22.4225GiB/s) | 20260505 04:40:35.622 | INFO | Analyzing chunk frames = [412672,413184) | 20260505 04:40:35.622 | INFO | MemInfo: { | 20260505 04:40:35.622 | INFO | "gpu" : " 17 GB", | 20260505 04:40:35.622 | INFO | "huge" : "", | 20260505 04:40:35.622 | INFO | "huge_shared" : " 27 GB", | 20260505 04:40:35.622 | INFO | "malloc" : " 14 GB", | 20260505 04:40:35.623 | INFO | "pinned" : " 29 GB" | 20260505 04:40:35.623 | INFO | } | 20260505 04:40:35.623 | INFO | Perf report for the last 5.13919 seconds: | 20260505 04:40:35.623 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:40:35.623 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:35.623 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:35.623 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:35.623 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260505 04:40:35.623 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:35.623 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:35.623 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260505 04:40:35.623 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:35.623 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:35.623 | INFO | Uploaded 36GiB to GPU in 1.60079s (22.4889GiB/s) | 20260505 04:40:35.623 | INFO | Downloaded 34GiB to GPU in 1.51798s (22.3981GiB/s) | 20260505 04:40:40.762 | INFO | Analyzing chunk frames = [413184,413696) | 20260505 04:40:40.762 | INFO | MemInfo: { | 20260505 04:40:40.762 | INFO | "gpu" : " 17 GB", | 20260505 04:40:40.762 | INFO | "huge" : "", | 20260505 04:40:40.762 | INFO | "huge_shared" : " 27 GB", | 20260505 04:40:40.762 | INFO | "malloc" : " 14 GB", | 20260505 04:40:40.762 | INFO | "pinned" : " 29 GB" | 20260505 04:40:40.762 | INFO | } | 20260505 04:40:40.762 | INFO | Perf report for the last 5.13946 seconds: | 20260505 04:40:40.762 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:40:40.762 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:40.762 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:40.762 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:40.762 | INFO | ANALYSIS : 57.6%, 3.0, 89.9ms, 0 | 20260505 04:40:40.762 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:40.762 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:40.762 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 04:40:40.762 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:40.762 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:40.762 | INFO | Uploaded 36GiB to GPU in 1.60079s (22.4889GiB/s) | 20260505 04:40:40.762 | INFO | Downloaded 34GiB to GPU in 1.51778s (22.4012GiB/s) | 20260505 04:40:40.861 | INFO | Starting a round of Baz IO | 20260505 04:40:43.815 | INFO | Populating RTMetrics report | 20260505 04:40:43.815 | INFO | RTMetrics report populated | 20260505 04:40:43.815 | INFO | Opening tmp RTMetrics json file | 20260505 04:40:43.818 | INFO | Serializing RTMetrics to json | 20260505 04:40:43.820 | INFO | Serialization done, swapping tmp file now... | 20260505 04:40:43.820 | INFO | RTMetrics done | 20260505 04:40:43.909 | INFO | Finished a round of Baz IO: Wrote 2.026 GiB in 3.047 seconds | 20260505 04:40:43.909 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.73% / 0.13% / 54.14% | 20260505 04:40:43.909 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 10.1817% / 0.000313918% / 4.76837e-05% | 20260505 04:40:43.909 | INFO | Baz IO overall: | 20260505 04:40:43.909 | INFO | BytesWritten : 167962373154 | 20260505 04:40:43.909 | INFO | NumEvents : 77961821499 | 20260505 04:40:45.901 | INFO | Analyzing chunk frames = [413696,414208) | 20260505 04:40:45.901 | INFO | MemInfo: { | 20260505 04:40:45.901 | INFO | "gpu" : " 17 GB", | 20260505 04:40:45.901 | INFO | "huge" : "", | 20260505 04:40:45.901 | INFO | "huge_shared" : " 27 GB", | 20260505 04:40:45.901 | INFO | "malloc" : " 14 GB", | 20260505 04:40:45.901 | INFO | "pinned" : " 29 GB" | 20260505 04:40:45.901 | INFO | } | 20260505 04:40:45.901 | INFO | Perf report for the last 5.13924 seconds: | 20260505 04:40:45.901 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:40:45.901 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:45.901 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:45.902 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:45.902 | INFO | ANALYSIS : 58.3%, 3.0, 90.6ms, 0 | 20260505 04:40:45.902 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:45.902 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260505 04:40:45.902 | INFO | PRE_HQ : 57.6%, 2.2, 64.3ms, 0 | 20260505 04:40:45.902 | INFO | BAZWRITER : 58.9%, 4.0, 120.5ms, 0 | 20260505 04:40:45.902 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 04:40:45.902 | INFO | Uploaded 36GiB to GPU in 1.63634s (22.0003GiB/s) | 20260505 04:40:45.902 | INFO | Downloaded 38GiB to GPU in 1.69805s (22.3785GiB/s) | 20260505 04:40:51.041 | INFO | Analyzing chunk frames = [414208,414720) | 20260505 04:40:51.041 | INFO | MemInfo: { | 20260505 04:40:51.041 | INFO | "gpu" : " 17 GB", | 20260505 04:40:51.041 | INFO | "huge" : "", | 20260505 04:40:51.041 | INFO | "huge_shared" : " 27 GB", | 20260505 04:40:51.041 | INFO | "malloc" : " 14 GB", | 20260505 04:40:51.041 | INFO | "pinned" : " 29 GB" | 20260505 04:40:51.041 | INFO | } | 20260505 04:40:51.041 | INFO | Perf report for the last 5.13936 seconds: | 20260505 04:40:51.041 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:40:51.041 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:51.041 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:51.041 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:51.041 | INFO | ANALYSIS : 58.0%, 3.0, 90.5ms, 0 | 20260505 04:40:51.041 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:51.041 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:51.041 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260505 04:40:51.041 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:51.041 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:51.041 | INFO | Uploaded 36GiB to GPU in 1.59861s (22.5195GiB/s) | 20260505 04:40:51.041 | INFO | Downloaded 34GiB to GPU in 1.51517s (22.4397GiB/s) | 20260505 04:40:56.180 | INFO | Analyzing chunk frames = [414720,415232) | 20260505 04:40:56.180 | INFO | MemInfo: { | 20260505 04:40:56.180 | INFO | "gpu" : " 17 GB", | 20260505 04:40:56.180 | INFO | "huge" : "", | 20260505 04:40:56.180 | INFO | "huge_shared" : " 27 GB", | 20260505 04:40:56.180 | INFO | "malloc" : " 14 GB", | 20260505 04:40:56.180 | INFO | "pinned" : " 29 GB" | 20260505 04:40:56.180 | INFO | } | 20260505 04:40:56.180 | INFO | Perf report for the last 5.13939 seconds: | 20260505 04:40:56.180 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:40:56.180 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:56.180 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:56.180 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:56.180 | INFO | ANALYSIS : 57.4%, 3.0, 89.1ms, 0 | 20260505 04:40:56.180 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:56.180 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:56.180 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 04:40:56.180 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:56.180 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:40:56.181 | INFO | Uploaded 36GiB to GPU in 1.60268s (22.4624GiB/s) | 20260505 04:40:56.181 | INFO | Downloaded 34GiB to GPU in 1.51976s (22.3719GiB/s) | 20260505 04:41:01.319 | INFO | Analyzing chunk frames = [415232,415744) | 20260505 04:41:01.319 | INFO | MemInfo: { | 20260505 04:41:01.319 | INFO | "gpu" : " 17 GB", | 20260505 04:41:01.319 | INFO | "huge" : "", | 20260505 04:41:01.319 | INFO | "huge_shared" : " 27 GB", | 20260505 04:41:01.319 | INFO | "malloc" : " 14 GB", | 20260505 04:41:01.319 | INFO | "pinned" : " 29 GB" | 20260505 04:41:01.319 | INFO | } | 20260505 04:41:01.319 | INFO | Perf report for the last 5.13917 seconds: | 20260505 04:41:01.319 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:41:01.319 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:01.319 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:01.319 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:01.319 | INFO | ANALYSIS : 57.1%, 3.0, 88.6ms, 0 | 20260505 04:41:01.319 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:01.319 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:01.319 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260505 04:41:01.320 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:01.320 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:01.320 | INFO | Uploaded 36GiB to GPU in 1.60231s (22.4676GiB/s) | 20260505 04:41:01.320 | INFO | Downloaded 34GiB to GPU in 1.51798s (22.3982GiB/s) | 20260505 04:41:06.459 | INFO | Analyzing chunk frames = [415744,416256) | 20260505 04:41:06.459 | INFO | MemInfo: { | 20260505 04:41:06.459 | INFO | "gpu" : " 17 GB", | 20260505 04:41:06.459 | INFO | "huge" : "", | 20260505 04:41:06.459 | INFO | "huge_shared" : " 27 GB", | 20260505 04:41:06.459 | INFO | "malloc" : " 14 GB", | 20260505 04:41:06.459 | INFO | "pinned" : " 29 GB" | 20260505 04:41:06.459 | INFO | } | 20260505 04:41:06.459 | INFO | Perf report for the last 5.13941 seconds: | 20260505 04:41:06.459 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:41:06.459 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:06.459 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:06.459 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:06.459 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260505 04:41:06.459 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:06.459 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:06.459 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 04:41:06.459 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:06.459 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:06.459 | INFO | Uploaded 36GiB to GPU in 1.60067s (22.4906GiB/s) | 20260505 04:41:06.459 | INFO | Downloaded 34GiB to GPU in 1.51667s (22.4176GiB/s) | 20260505 04:41:11.598 | INFO | Analyzing chunk frames = [416256,416768) | 20260505 04:41:11.598 | INFO | MemInfo: { | 20260505 04:41:11.598 | INFO | "gpu" : " 17 GB", | 20260505 04:41:11.598 | INFO | "huge" : "", | 20260505 04:41:11.598 | INFO | "huge_shared" : " 27 GB", | 20260505 04:41:11.598 | INFO | "malloc" : " 14 GB", | 20260505 04:41:11.598 | INFO | "pinned" : " 29 GB" | 20260505 04:41:11.598 | INFO | } | 20260505 04:41:11.598 | INFO | Perf report for the last 5.13954 seconds: | 20260505 04:41:11.598 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:41:11.598 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:11.598 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:11.598 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:11.598 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 04:41:11.598 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:11.598 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:11.598 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 04:41:11.598 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:11.599 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:11.599 | INFO | Uploaded 36GiB to GPU in 1.60015s (22.4978GiB/s) | 20260505 04:41:11.599 | INFO | Downloaded 34GiB to GPU in 1.51668s (22.4173GiB/s) | 20260505 04:41:16.737 | INFO | Analyzing chunk frames = [416768,417280) | 20260505 04:41:16.737 | INFO | MemInfo: { | 20260505 04:41:16.737 | INFO | "gpu" : " 17 GB", | 20260505 04:41:16.737 | INFO | "huge" : "", | 20260505 04:41:16.737 | INFO | "huge_shared" : " 27 GB", | 20260505 04:41:16.738 | INFO | "malloc" : " 14 GB", | 20260505 04:41:16.738 | INFO | "pinned" : " 29 GB" | 20260505 04:41:16.738 | INFO | } | 20260505 04:41:16.738 | INFO | Perf report for the last 5.13926 seconds: | 20260505 04:41:16.738 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:41:16.738 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:16.738 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:16.738 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:16.738 | INFO | ANALYSIS : 57.6%, 3.0, 89.8ms, 0 | 20260505 04:41:16.738 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:16.738 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:16.738 | INFO | PRE_HQ : 31.7%, 1.0, 16.4ms, 0 | 20260505 04:41:16.738 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:16.738 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:16.738 | INFO | Uploaded 36GiB to GPU in 1.60115s (22.4839GiB/s) | 20260505 04:41:16.738 | INFO | Downloaded 34GiB to GPU in 1.51818s (22.3952GiB/s) | 20260505 04:41:21.877 | INFO | Analyzing chunk frames = [417280,417792) | 20260505 04:41:21.877 | INFO | MemInfo: { | 20260505 04:41:21.877 | INFO | "gpu" : " 17 GB", | 20260505 04:41:21.877 | INFO | "huge" : "", | 20260505 04:41:21.877 | INFO | "huge_shared" : " 27 GB", | 20260505 04:41:21.877 | INFO | "malloc" : " 14 GB", | 20260505 04:41:21.877 | INFO | "pinned" : " 29 GB" | 20260505 04:41:21.877 | INFO | } | 20260505 04:41:21.877 | INFO | Perf report for the last 5.13926 seconds: | 20260505 04:41:21.877 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:41:21.877 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:21.877 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:21.877 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:21.877 | INFO | ANALYSIS : 57.6%, 3.0, 89.8ms, 0 | 20260505 04:41:21.877 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:21.877 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:21.877 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 04:41:21.877 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:21.877 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:21.877 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 04:41:21.877 | INFO | Total mem usage: 14.9 GiB | 20260505 04:41:21.877 | INFO | All Time High Water Marks sum: 15.3 GiB (Savings: 2.13%) | 20260505 04:41:21.877 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -3.27%) | 20260505 04:41:21.877 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 04:41:21.877 | INFO | High Water Breakdown: | 20260505 04:41:21.877 | INFO | 59.3% : 8.57 GiB ( 99.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 04:41:21.877 | INFO | 40.6% : 5.86 GiB ( 88.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 04:41:21.877 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 04:41:21.877 | INFO | ------------------------------------------ | 20260505 04:41:21.877 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:41:21.877 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 04:41:21.877 | INFO | Total mem usage: 26.5 GiB | 20260505 04:41:21.877 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260505 04:41:21.877 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 04:41:21.877 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260505 04:41:21.877 | INFO | High Water Breakdown: | 20260505 04:41:21.877 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:41:21.877 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:41:21.878 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:41:21.878 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:41:21.878 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:41:21.878 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:41:21.878 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 04:41:21.878 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 04:41:21.878 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:41:21.878 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 04:41:21.878 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 04:41:21.878 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 04:41:21.878 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 04:41:21.878 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 04:41:21.878 | INFO | ------------------------------------------ | 20260505 04:41:21.878 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:41:21.878 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 04:41:21.878 | INFO | Total mem usage: 25.6 GiB | 20260505 04:41:21.878 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 04:41:21.878 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 04:41:21.878 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 04:41:21.878 | INFO | High Water Breakdown: | 20260505 04:41:21.878 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 04:41:21.878 | INFO | ------------------------------------------ | 20260505 04:41:21.878 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:41:21.878 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 04:41:21.878 | INFO | Total mem usage: 15.7 GiB | 20260505 04:41:21.878 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260505 04:41:21.878 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 04:41:21.878 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260505 04:41:21.878 | INFO | High Water Breakdown: | 20260505 04:41:21.878 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 04:41:21.878 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 04:41:21.878 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 04:41:21.878 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 04:41:21.878 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 04:41:21.878 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 04:41:21.878 | INFO | Uploaded 36GiB to GPU in 1.60117s (22.4835GiB/s) | 20260505 04:41:21.878 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 04:41:21.878 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 04:41:21.878 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 04:41:21.878 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:41:21.878 | INFO | Downloaded 34GiB to GPU in 1.51793s (22.3989GiB/s) | 20260505 04:41:21.878 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 04:41:21.878 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:41:21.878 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 04:41:21.878 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 04:41:21.878 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 04:41:21.878 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 04:41:21.878 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:41:21.878 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 04:41:21.878 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:41:21.878 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:41:21.878 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:41:21.878 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 04:41:21.878 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 04:41:21.878 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:41:21.878 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 04:41:21.878 | INFO | ------------------------------------------ | 20260505 04:41:21.878 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:41:21.878 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 04:41:21.878 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:41:21.878 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:41:21.878 | INFO | PerfReport: Full Execution -- 3.05e+01 / -nan, 1.28e+01 / 0.00e+00 | 20260505 04:41:21.878 | INFO | PerfReport: 0.28%, AggregateStats -- 8.62e-02 / -nan, 4.04e-03 / 0.00e+00 | 20260505 04:41:21.878 | INFO | PerfReport: 10.82%, Baseline -- 3.30e+00 / -nan, 1.62e-02 / 0.00e+00 | 20260505 04:41:21.878 | INFO | PerfReport: 3.03%, Histogram -- 9.24e-01 / -nan, 5.21e-02 / 0.00e+00 | 20260505 04:41:21.878 | INFO | PerfReport: 16.03%, DME -- 3.91e+01 / -nan, 3.20e+00 / 0.00e+00 | 20260505 04:41:21.878 | INFO | PerfReport: 56.50%, FrameLabeling -- 1.72e+01 / -nan, 4.53e-02 / 0.00e+00 | 20260505 04:41:21.878 | INFO | PerfReport: 7.65%, PulseAccumulating -- 2.33e+00 / -nan, 1.55e-01 / 0.00e+00 | 20260505 04:41:21.878 | INFO | PerfReport: 4.02%, Metrics -- 1.22e+00 / -nan, 2.31e-01 / 0.00e+00 | 20260505 04:41:21.878 | INFO | PerfReport: 1.66%, PulseSerializing -- 5.05e-01 / -nan, 8.10e-02 / 0.00e+00 | 20260505 04:41:21.878 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 04:41:21.878 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:41:21.878 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:41:21.878 | INFO | PerfReport: Full Execution -- 9.15e+01 / -nan, 1.90e+01 / 0.00e+00 | 20260505 04:41:21.878 | INFO | PerfReport: 18.28%, Upload -- 1.67e+01 / 0.00e+00, 3.65e-01 / 0.00e+00 | 20260505 04:41:21.878 | INFO | PerfReport: 17.54%, Download -- 1.60e+01 / 0.00e+00, 6.85e-01 / 0.00e+00 | 20260505 04:41:21.878 | INFO | PerfReport: 33.30%, Compute -- 3.05e+01 / 0.00e+00, 1.28e+01 / 0.00e+00 | 20260505 04:41:21.878 | INFO | PerfReport: 0.55%, LockUpload -- 5.03e-01 / 0.00e+00, 3.60e+00 / 0.00e+00 | 20260505 04:41:21.878 | INFO | PerfReport: 0.00%, LockDownload -- 3.91e-04 / 0.00e+00, 9.16e-05 / 0.00e+00 | 20260505 04:41:21.878 | INFO | PerfReport: 30.31%, LockCompute -- 2.77e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260505 04:41:21.878 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.87e-02 / 0.00e+00 | 20260505 04:41:21.974 | INFO | Starting a round of Baz IO | 20260505 04:41:24.929 | INFO | Populating RTMetrics report | 20260505 04:41:24.930 | INFO | RTMetrics report populated | 20260505 04:41:24.930 | INFO | Opening tmp RTMetrics json file | 20260505 04:41:24.932 | INFO | Serializing RTMetrics to json | 20260505 04:41:24.934 | INFO | Serialization done, swapping tmp file now... | 20260505 04:41:24.935 | INFO | RTMetrics done | 20260505 04:41:25.016 | INFO | Finished a round of Baz IO: Wrote 2.032 GiB in 3.041 seconds | 20260505 04:41:25.016 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.60% / 0.13% / 54.27% | 20260505 04:41:25.016 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 10.1528% / 0.000282129% / 5.5631e-05% | 20260505 04:41:25.016 | INFO | Baz IO overall: | 20260505 04:41:25.016 | INFO | BytesWritten : 170144357428 | 20260505 04:41:25.016 | INFO | NumEvents : 78996582908 | 20260505 04:41:27.016 | INFO | Analyzing chunk frames = [417792,418304) | 20260505 04:41:27.016 | INFO | MemInfo: { | 20260505 04:41:27.016 | INFO | "gpu" : " 17 GB", | 20260505 04:41:27.016 | INFO | "huge" : "", | 20260505 04:41:27.016 | INFO | "huge_shared" : " 27 GB", | 20260505 04:41:27.016 | INFO | "malloc" : " 14 GB", | 20260505 04:41:27.016 | INFO | "pinned" : " 29 GB" | 20260505 04:41:27.016 | INFO | } | 20260505 04:41:27.016 | INFO | Perf report for the last 5.13932 seconds: | 20260505 04:41:27.016 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:41:27.016 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:27.016 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:27.016 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:27.016 | INFO | ANALYSIS : 58.3%, 3.0, 90.5ms, 0 | 20260505 04:41:27.016 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:27.016 | INFO | RT_METRICS : 45.3%, 1.0, 23.7ms, 0 | 20260505 04:41:27.016 | INFO | PRE_HQ : 57.6%, 2.2, 64.3ms, 0 | 20260505 04:41:27.016 | INFO | BAZWRITER : 58.8%, 4.0, 119.9ms, 0 | 20260505 04:41:27.016 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 04:41:27.017 | INFO | Uploaded 36GiB to GPU in 1.63868s (21.9689GiB/s) | 20260505 04:41:27.017 | INFO | Downloaded 38GiB to GPU in 1.69935s (22.3614GiB/s) | 20260505 04:41:32.155 | INFO | Analyzing chunk frames = [418304,418816) | 20260505 04:41:32.155 | INFO | MemInfo: { | 20260505 04:41:32.155 | INFO | "gpu" : " 17 GB", | 20260505 04:41:32.155 | INFO | "huge" : "", | 20260505 04:41:32.156 | INFO | "huge_shared" : " 27 GB", | 20260505 04:41:32.156 | INFO | "malloc" : " 14 GB", | 20260505 04:41:32.156 | INFO | "pinned" : " 29 GB" | 20260505 04:41:32.156 | INFO | } | 20260505 04:41:32.156 | INFO | Perf report for the last 5.13948 seconds: | 20260505 04:41:32.156 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:41:32.156 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:32.156 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:32.156 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:32.156 | INFO | ANALYSIS : 58.0%, 3.0, 90.5ms, 0 | 20260505 04:41:32.156 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:32.156 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:32.156 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260505 04:41:32.156 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:32.156 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:32.156 | INFO | Uploaded 36GiB to GPU in 1.59941s (22.5083GiB/s) | 20260505 04:41:32.156 | INFO | Downloaded 34GiB to GPU in 1.51558s (22.4336GiB/s) | 20260505 04:41:37.295 | INFO | Analyzing chunk frames = [418816,419328) | 20260505 04:41:37.295 | INFO | MemInfo: { | 20260505 04:41:37.295 | INFO | "gpu" : " 17 GB", | 20260505 04:41:37.295 | INFO | "huge" : "", | 20260505 04:41:37.295 | INFO | "huge_shared" : " 27 GB", | 20260505 04:41:37.295 | INFO | "malloc" : " 14 GB", | 20260505 04:41:37.295 | INFO | "pinned" : " 29 GB" | 20260505 04:41:37.295 | INFO | } | 20260505 04:41:37.295 | INFO | Perf report for the last 5.13922 seconds: | 20260505 04:41:37.295 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:41:37.295 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:37.295 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:37.295 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:37.295 | INFO | ANALYSIS : 57.4%, 3.0, 89.3ms, 0 | 20260505 04:41:37.295 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:37.295 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:37.295 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260505 04:41:37.295 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:37.295 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:37.295 | INFO | Uploaded 36GiB to GPU in 1.60213s (22.4701GiB/s) | 20260505 04:41:37.295 | INFO | Downloaded 34GiB to GPU in 1.51943s (22.3768GiB/s) | 20260505 04:41:42.434 | INFO | Analyzing chunk frames = [419328,419840) | 20260505 04:41:42.434 | INFO | MemInfo: { | 20260505 04:41:42.434 | INFO | "gpu" : " 17 GB", | 20260505 04:41:42.434 | INFO | "huge" : "", | 20260505 04:41:42.434 | INFO | "huge_shared" : " 27 GB", | 20260505 04:41:42.434 | INFO | "malloc" : " 14 GB", | 20260505 04:41:42.434 | INFO | "pinned" : " 29 GB" | 20260505 04:41:42.434 | INFO | } | 20260505 04:41:42.434 | INFO | Perf report for the last 5.13936 seconds: | 20260505 04:41:42.434 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:41:42.434 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:42.434 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:42.434 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:42.434 | INFO | ANALYSIS : 57.1%, 3.0, 88.5ms, 0 | 20260505 04:41:42.434 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:42.434 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:42.434 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 04:41:42.434 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:42.434 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:42.435 | INFO | Uploaded 36GiB to GPU in 1.60203s (22.4716GiB/s) | 20260505 04:41:42.435 | INFO | Downloaded 34GiB to GPU in 1.51853s (22.3901GiB/s) | 20260505 04:41:47.573 | INFO | Analyzing chunk frames = [419840,420352) | 20260505 04:41:47.573 | INFO | MemInfo: { | 20260505 04:41:47.573 | INFO | "gpu" : " 17 GB", | 20260505 04:41:47.573 | INFO | "huge" : "", | 20260505 04:41:47.573 | INFO | "huge_shared" : " 27 GB", | 20260505 04:41:47.573 | INFO | "malloc" : " 14 GB", | 20260505 04:41:47.573 | INFO | "pinned" : " 29 GB" | 20260505 04:41:47.573 | INFO | } | 20260505 04:41:47.573 | INFO | Perf report for the last 5.13912 seconds: | 20260505 04:41:47.573 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:41:47.573 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:47.573 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:47.574 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:47.574 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260505 04:41:47.574 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:47.574 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:47.574 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260505 04:41:47.574 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:47.574 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:47.574 | INFO | Uploaded 36GiB to GPU in 1.60071s (22.49GiB/s) | 20260505 04:41:47.574 | INFO | Downloaded 34GiB to GPU in 1.51672s (22.4168GiB/s) | 20260505 04:41:52.713 | INFO | Analyzing chunk frames = [420352,420864) | 20260505 04:41:52.713 | INFO | MemInfo: { | 20260505 04:41:52.713 | INFO | "gpu" : " 17 GB", | 20260505 04:41:52.713 | INFO | "huge" : "", | 20260505 04:41:52.713 | INFO | "huge_shared" : " 27 GB", | 20260505 04:41:52.713 | INFO | "malloc" : " 14 GB", | 20260505 04:41:52.713 | INFO | "pinned" : " 29 GB" | 20260505 04:41:52.713 | INFO | } | 20260505 04:41:52.713 | INFO | Perf report for the last 5.13931 seconds: | 20260505 04:41:52.713 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:41:52.713 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:52.713 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:52.713 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:52.713 | INFO | ANALYSIS : 57.3%, 3.0, 89.3ms, 0 | 20260505 04:41:52.713 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:52.713 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:52.713 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260505 04:41:52.713 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:52.713 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:52.713 | INFO | Uploaded 36GiB to GPU in 1.60033s (22.4954GiB/s) | 20260505 04:41:52.713 | INFO | Downloaded 34GiB to GPU in 1.51677s (22.416GiB/s) | 20260505 04:41:57.852 | INFO | Analyzing chunk frames = [420864,421376) | 20260505 04:41:57.852 | INFO | MemInfo: { | 20260505 04:41:57.852 | INFO | "gpu" : " 17 GB", | 20260505 04:41:57.852 | INFO | "huge" : "", | 20260505 04:41:57.852 | INFO | "huge_shared" : " 27 GB", | 20260505 04:41:57.852 | INFO | "malloc" : " 14 GB", | 20260505 04:41:57.852 | INFO | "pinned" : " 29 GB" | 20260505 04:41:57.852 | INFO | } | 20260505 04:41:57.852 | INFO | Perf report for the last 5.13947 seconds: | 20260505 04:41:57.852 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:41:57.852 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:57.852 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:57.852 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:57.852 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260505 04:41:57.852 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:57.852 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:57.852 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 04:41:57.852 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:57.852 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:41:57.853 | INFO | Uploaded 36GiB to GPU in 1.60048s (22.4932GiB/s) | 20260505 04:41:57.853 | INFO | Downloaded 34GiB to GPU in 1.51721s (22.4095GiB/s) | 20260505 04:42:02.991 | INFO | Analyzing chunk frames = [421376,421888) | 20260505 04:42:02.991 | INFO | MemInfo: { | 20260505 04:42:02.991 | INFO | "gpu" : " 17 GB", | 20260505 04:42:02.991 | INFO | "huge" : "", | 20260505 04:42:02.991 | INFO | "huge_shared" : " 27 GB", | 20260505 04:42:02.991 | INFO | "malloc" : " 14 GB", | 20260505 04:42:02.991 | INFO | "pinned" : " 29 GB" | 20260505 04:42:02.992 | INFO | } | 20260505 04:42:02.992 | INFO | Perf report for the last 5.13924 seconds: | 20260505 04:42:02.992 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:42:02.992 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:02.992 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:02.992 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:02.992 | INFO | ANALYSIS : 57.6%, 3.0, 89.9ms, 0 | 20260505 04:42:02.992 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:02.992 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:02.992 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 04:42:02.992 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:02.992 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:02.992 | INFO | Uploaded 36GiB to GPU in 1.60173s (22.4756GiB/s) | 20260505 04:42:02.992 | INFO | Downloaded 34GiB to GPU in 1.51857s (22.3894GiB/s) | 20260505 04:42:03.089 | INFO | Starting a round of Baz IO | 20260505 04:42:06.043 | INFO | Populating RTMetrics report | 20260505 04:42:06.044 | INFO | RTMetrics report populated | 20260505 04:42:06.044 | INFO | Opening tmp RTMetrics json file | 20260505 04:42:06.046 | INFO | Serializing RTMetrics to json | 20260505 04:42:06.065 | INFO | Serialization done, swapping tmp file now... | 20260505 04:42:06.087 | INFO | RTMetrics done | 20260505 04:42:06.209 | INFO | Finished a round of Baz IO: Wrote 2.037 GiB in 3.119 seconds | 20260505 04:42:06.209 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.47% / 0.13% / 54.40% | 20260505 04:42:06.209 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 10.1106% / 0.000290076% / 5.16574e-05% | 20260505 04:42:06.209 | INFO | Baz IO overall: | 20260505 04:42:06.209 | INFO | BytesWritten : 172331409494 | 20260505 04:42:06.209 | INFO | NumEvents : 80033886157 | 20260505 04:42:08.131 | INFO | Analyzing chunk frames = [421888,422400) | 20260505 04:42:08.131 | INFO | MemInfo: { | 20260505 04:42:08.131 | INFO | "gpu" : " 17 GB", | 20260505 04:42:08.131 | INFO | "huge" : "", | 20260505 04:42:08.131 | INFO | "huge_shared" : " 27 GB", | 20260505 04:42:08.131 | INFO | "malloc" : " 14 GB", | 20260505 04:42:08.131 | INFO | "pinned" : " 29 GB" | 20260505 04:42:08.131 | INFO | } | 20260505 04:42:08.131 | INFO | Perf report for the last 5.13928 seconds: | 20260505 04:42:08.131 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:42:08.131 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:08.131 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:08.131 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:08.131 | INFO | ANALYSIS : 58.3%, 2.9, 88.9ms, 0 | 20260505 04:42:08.131 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:08.131 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 04:42:08.131 | INFO | PRE_HQ : 57.6%, 2.1, 63.8ms, 0 | 20260505 04:42:08.131 | INFO | BAZWRITER : 60.3%, 4.2, 130.7ms, 0 | 20260505 04:42:08.131 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 04:42:08.131 | INFO | Uploaded 36GiB to GPU in 1.64678s (21.8608GiB/s) | 20260505 04:42:08.131 | INFO | Downloaded 38GiB to GPU in 1.70902s (22.2349GiB/s) | 20260505 04:42:13.270 | INFO | Analyzing chunk frames = [422400,422912) | 20260505 04:42:13.270 | INFO | MemInfo: { | 20260505 04:42:13.270 | INFO | "gpu" : " 17 GB", | 20260505 04:42:13.270 | INFO | "huge" : "", | 20260505 04:42:13.270 | INFO | "huge_shared" : " 27 GB", | 20260505 04:42:13.270 | INFO | "malloc" : " 14 GB", | 20260505 04:42:13.270 | INFO | "pinned" : " 29 GB" | 20260505 04:42:13.270 | INFO | } | 20260505 04:42:13.270 | INFO | Perf report for the last 5.13953 seconds: | 20260505 04:42:13.270 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:42:13.270 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:13.270 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:13.270 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:13.270 | INFO | ANALYSIS : 58.0%, 3.0, 90.5ms, 0 | 20260505 04:42:13.270 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:13.270 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:13.270 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 04:42:13.271 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:13.271 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:13.271 | INFO | Uploaded 36GiB to GPU in 1.5992s (22.5113GiB/s) | 20260505 04:42:13.271 | INFO | Downloaded 34GiB to GPU in 1.51551s (22.4347GiB/s) | 20260505 04:42:18.409 | INFO | Analyzing chunk frames = [422912,423424) | 20260505 04:42:18.409 | INFO | MemInfo: { | 20260505 04:42:18.409 | INFO | "gpu" : " 17 GB", | 20260505 04:42:18.409 | INFO | "huge" : "", | 20260505 04:42:18.409 | INFO | "huge_shared" : " 27 GB", | 20260505 04:42:18.410 | INFO | "malloc" : " 14 GB", | 20260505 04:42:18.410 | INFO | "pinned" : " 29 GB" | 20260505 04:42:18.410 | INFO | } | 20260505 04:42:18.410 | INFO | Perf report for the last 5.13933 seconds: | 20260505 04:42:18.410 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:42:18.410 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:18.410 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:18.410 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:18.410 | INFO | ANALYSIS : 57.4%, 3.0, 89.2ms, 0 | 20260505 04:42:18.410 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:18.410 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:18.410 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 04:42:18.410 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:18.410 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:18.410 | INFO | Uploaded 36GiB to GPU in 1.60399s (22.444GiB/s) | 20260505 04:42:18.410 | INFO | Downloaded 34GiB to GPU in 1.51995s (22.3691GiB/s) | 20260505 04:42:23.549 | INFO | Analyzing chunk frames = [423424,423936) | 20260505 04:42:23.549 | INFO | MemInfo: { | 20260505 04:42:23.549 | INFO | "gpu" : " 17 GB", | 20260505 04:42:23.549 | INFO | "huge" : "", | 20260505 04:42:23.549 | INFO | "huge_shared" : " 27 GB", | 20260505 04:42:23.549 | INFO | "malloc" : " 14 GB", | 20260505 04:42:23.549 | INFO | "pinned" : " 29 GB" | 20260505 04:42:23.549 | INFO | } | 20260505 04:42:23.549 | INFO | Perf report for the last 5.13939 seconds: | 20260505 04:42:23.549 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:42:23.549 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:23.549 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:23.549 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:23.549 | INFO | ANALYSIS : 57.1%, 3.0, 88.5ms, 0 | 20260505 04:42:23.549 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:23.549 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:23.549 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260505 04:42:23.549 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:23.549 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:23.549 | INFO | Uploaded 36GiB to GPU in 1.60242s (22.4661GiB/s) | 20260505 04:42:23.549 | INFO | Downloaded 34GiB to GPU in 1.51854s (22.3899GiB/s) | 20260505 04:42:28.688 | INFO | Analyzing chunk frames = [423936,424448) | 20260505 04:42:28.688 | INFO | MemInfo: { | 20260505 04:42:28.688 | INFO | "gpu" : " 17 GB", | 20260505 04:42:28.688 | INFO | "huge" : "", | 20260505 04:42:28.688 | INFO | "huge_shared" : " 27 GB", | 20260505 04:42:28.688 | INFO | "malloc" : " 14 GB", | 20260505 04:42:28.688 | INFO | "pinned" : " 29 GB" | 20260505 04:42:28.688 | INFO | } | 20260505 04:42:28.688 | INFO | Perf report for the last 5.13908 seconds: | 20260505 04:42:28.688 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:42:28.688 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:28.688 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:28.688 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:28.688 | INFO | ANALYSIS : 57.0%, 3.0, 89.0ms, 0 | 20260505 04:42:28.688 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:28.688 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:28.688 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 04:42:28.688 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:28.688 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:28.689 | INFO | Uploaded 36GiB to GPU in 1.60057s (22.492GiB/s) | 20260505 04:42:28.689 | INFO | Downloaded 34GiB to GPU in 1.51638s (22.4219GiB/s) | 20260505 04:42:33.828 | INFO | Analyzing chunk frames = [424448,424960) | 20260505 04:42:33.828 | INFO | MemInfo: { | 20260505 04:42:33.828 | INFO | "gpu" : " 17 GB", | 20260505 04:42:33.828 | INFO | "huge" : "", | 20260505 04:42:33.828 | INFO | "huge_shared" : " 27 GB", | 20260505 04:42:33.828 | INFO | "malloc" : " 14 GB", | 20260505 04:42:33.828 | INFO | "pinned" : " 29 GB" | 20260505 04:42:33.828 | INFO | } | 20260505 04:42:33.828 | INFO | Perf report for the last 5.1396 seconds: | 20260505 04:42:33.828 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:42:33.828 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:33.828 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:33.828 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:33.828 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 04:42:33.828 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:33.828 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:33.828 | INFO | PRE_HQ : 31.7%, 1.0, 16.4ms, 0 | 20260505 04:42:33.828 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:33.828 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:33.828 | INFO | Uploaded 36GiB to GPU in 1.60161s (22.4774GiB/s) | 20260505 04:42:33.828 | INFO | Downloaded 34GiB to GPU in 1.51796s (22.3985GiB/s) | 20260505 04:42:38.967 | INFO | Analyzing chunk frames = [424960,425472) | 20260505 04:42:38.967 | INFO | MemInfo: { | 20260505 04:42:38.967 | INFO | "gpu" : " 17 GB", | 20260505 04:42:38.967 | INFO | "huge" : "", | 20260505 04:42:38.967 | INFO | "huge_shared" : " 27 GB", | 20260505 04:42:38.967 | INFO | "malloc" : " 14 GB", | 20260505 04:42:38.967 | INFO | "pinned" : " 29 GB" | 20260505 04:42:38.967 | INFO | } | 20260505 04:42:38.967 | INFO | Perf report for the last 5.13918 seconds: | 20260505 04:42:38.967 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:42:38.967 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:38.967 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:38.967 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:38.967 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260505 04:42:38.967 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:38.967 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:38.967 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 04:42:38.967 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:38.967 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:38.967 | INFO | Uploaded 36GiB to GPU in 1.6005s (22.493GiB/s) | 20260505 04:42:38.967 | INFO | Downloaded 34GiB to GPU in 1.51841s (22.3918GiB/s) | 20260505 04:42:44.106 | INFO | Analyzing chunk frames = [425472,425984) | 20260505 04:42:44.106 | INFO | MemInfo: { | 20260505 04:42:44.106 | INFO | "gpu" : " 17 GB", | 20260505 04:42:44.106 | INFO | "huge" : "", | 20260505 04:42:44.106 | INFO | "huge_shared" : " 27 GB", | 20260505 04:42:44.106 | INFO | "malloc" : " 14 GB", | 20260505 04:42:44.107 | INFO | "pinned" : " 29 GB" | 20260505 04:42:44.107 | INFO | } | 20260505 04:42:44.107 | INFO | Perf report for the last 5.13956 seconds: | 20260505 04:42:44.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:42:44.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:44.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:44.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:44.107 | INFO | ANALYSIS : 57.6%, 3.0, 89.8ms, 0 | 20260505 04:42:44.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:44.107 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:44.107 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 04:42:44.107 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:44.107 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:44.107 | INFO | Uploaded 36GiB to GPU in 1.60165s (22.4769GiB/s) | 20260505 04:42:44.107 | INFO | Downloaded 34GiB to GPU in 1.51897s (22.3837GiB/s) | 20260505 04:42:44.107 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 04:42:44.107 | INFO | Total mem usage: 15.1 GiB | 20260505 04:42:44.107 | INFO | All Time High Water Marks sum: 15.7 GiB (Savings: 4.09%) | 20260505 04:42:44.107 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: 0.051%) | 20260505 04:42:44.107 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 04:42:44.107 | INFO | High Water Breakdown: | 20260505 04:42:44.107 | INFO | 60% : 9.03 GiB ( 100% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 04:42:44.107 | INFO | 39.9% : 6.01 GiB ( 90.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 04:42:44.107 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 04:42:44.107 | INFO | ------------------------------------------ | 20260505 04:42:44.107 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:42:44.107 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 04:42:44.107 | INFO | Total mem usage: 26.5 GiB | 20260505 04:42:44.107 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260505 04:42:44.107 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 04:42:44.107 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260505 04:42:44.107 | INFO | High Water Breakdown: | 20260505 04:42:44.107 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:42:44.107 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:42:44.107 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:42:44.107 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:42:44.108 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:42:44.108 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:42:44.108 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 04:42:44.108 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 04:42:44.108 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:42:44.108 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 04:42:44.108 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 04:42:44.108 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 04:42:44.108 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 04:42:44.108 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 04:42:44.108 | INFO | ------------------------------------------ | 20260505 04:42:44.108 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:42:44.108 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 04:42:44.108 | INFO | Total mem usage: 25.6 GiB | 20260505 04:42:44.108 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 04:42:44.108 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 04:42:44.108 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 04:42:44.108 | INFO | High Water Breakdown: | 20260505 04:42:44.108 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 04:42:44.108 | INFO | ------------------------------------------ | 20260505 04:42:44.108 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:42:44.108 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 04:42:44.108 | INFO | Total mem usage: 15.7 GiB | 20260505 04:42:44.108 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260505 04:42:44.108 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 04:42:44.108 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260505 04:42:44.108 | INFO | High Water Breakdown: | 20260505 04:42:44.108 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 04:42:44.108 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 04:42:44.108 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 04:42:44.108 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 04:42:44.108 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 04:42:44.108 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 04:42:44.108 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 04:42:44.108 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 04:42:44.108 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 04:42:44.108 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:42:44.108 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 04:42:44.108 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:42:44.108 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 04:42:44.108 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 04:42:44.108 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 04:42:44.108 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 04:42:44.108 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:42:44.108 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 04:42:44.108 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:42:44.108 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:42:44.108 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:42:44.108 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 04:42:44.108 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 04:42:44.108 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:42:44.108 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 04:42:44.108 | INFO | ------------------------------------------ | 20260505 04:42:44.108 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:42:44.108 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 04:42:44.108 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:42:44.108 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:42:44.108 | INFO | PerfReport: Full Execution -- 3.05e+01 / -nan, 1.28e+01 / 0.00e+00 | 20260505 04:42:44.108 | INFO | PerfReport: 0.28%, AggregateStats -- 8.64e-02 / -nan, 4.09e-03 / 0.00e+00 | 20260505 04:42:44.108 | INFO | PerfReport: 10.82%, Baseline -- 3.30e+00 / -nan, 1.88e-02 / 0.00e+00 | 20260505 04:42:44.108 | INFO | PerfReport: 3.04%, Histogram -- 9.24e-01 / -nan, 5.24e-02 / 0.00e+00 | 20260505 04:42:44.108 | INFO | PerfReport: 16.04%, DME -- 3.91e+01 / -nan, 3.15e+00 / 0.00e+00 | 20260505 04:42:44.108 | INFO | PerfReport: 56.52%, FrameLabeling -- 1.72e+01 / -nan, 4.53e-02 / 0.00e+00 | 20260505 04:42:44.108 | INFO | PerfReport: 7.64%, PulseAccumulating -- 2.33e+00 / -nan, 1.55e-01 / 0.00e+00 | 20260505 04:42:44.108 | INFO | PerfReport: 4.00%, Metrics -- 1.22e+00 / -nan, 2.01e-01 / 0.00e+00 | 20260505 04:42:44.108 | INFO | PerfReport: 1.66%, PulseSerializing -- 5.04e-01 / -nan, 8.10e-02 / 0.00e+00 | 20260505 04:42:44.108 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 04:42:44.108 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:42:44.108 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:42:44.108 | INFO | PerfReport: Full Execution -- 9.13e+01 / -nan, 1.90e+01 / 0.00e+00 | 20260505 04:42:44.108 | INFO | PerfReport: 18.32%, Upload -- 1.67e+01 / 0.00e+00, 3.82e-01 / 0.00e+00 | 20260505 04:42:44.108 | INFO | PerfReport: 17.58%, Download -- 1.61e+01 / 0.00e+00, 7.10e-01 / 0.00e+00 | 20260505 04:42:44.108 | INFO | PerfReport: 33.35%, Compute -- 3.05e+01 / 0.00e+00, 1.28e+01 / 0.00e+00 | 20260505 04:42:44.108 | INFO | PerfReport: 0.55%, LockUpload -- 5.04e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260505 04:42:44.108 | INFO | PerfReport: 0.00%, LockDownload -- 3.91e-04 / 0.00e+00, 1.47e-04 / 0.00e+00 | 20260505 04:42:44.108 | INFO | PerfReport: 30.19%, LockCompute -- 2.76e+01 / 0.00e+00, 1.72e+01 / 0.00e+00 | 20260505 04:42:44.108 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.46e-02 / 0.00e+00 | 20260505 04:42:44.204 | INFO | Starting a round of Baz IO | 20260505 04:42:47.156 | INFO | Populating RTMetrics report | 20260505 04:42:47.157 | INFO | RTMetrics report populated | 20260505 04:42:47.157 | INFO | Opening tmp RTMetrics json file | 20260505 04:42:47.158 | INFO | Serializing RTMetrics to json | 20260505 04:42:47.159 | INFO | Serialization done, swapping tmp file now... | 20260505 04:42:47.160 | INFO | RTMetrics done | 20260505 04:42:47.247 | INFO | Finished a round of Baz IO: Wrote 2.04 GiB in 3.043 seconds | 20260505 04:42:47.247 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.34% / 0.13% / 54.53% | 20260505 04:42:47.247 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 10.0338% / 0.000266234% / 5.96046e-05% | 20260505 04:42:47.247 | INFO | Baz IO overall: | 20260505 04:42:47.247 | INFO | BytesWritten : 174521367767 | 20260505 04:42:47.247 | INFO | NumEvents : 81071534488 | 20260505 04:42:49.245 | INFO | Analyzing chunk frames = [425984,426496) | 20260505 04:42:49.245 | INFO | MemInfo: { | 20260505 04:42:49.245 | INFO | "gpu" : " 17 GB", | 20260505 04:42:49.245 | INFO | "huge" : "", | 20260505 04:42:49.245 | INFO | "huge_shared" : " 27 GB", | 20260505 04:42:49.245 | INFO | "malloc" : " 14 GB", | 20260505 04:42:49.245 | INFO | "pinned" : " 29 GB" | 20260505 04:42:49.245 | INFO | } | 20260505 04:42:49.246 | INFO | Perf report for the last 5.1389 seconds: | 20260505 04:42:49.246 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:42:49.246 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:49.246 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:49.246 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:49.246 | INFO | ANALYSIS : 58.2%, 3.0, 90.6ms, 0 | 20260505 04:42:49.246 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:49.246 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260505 04:42:49.246 | INFO | PRE_HQ : 57.6%, 2.1, 64.3ms, 0 | 20260505 04:42:49.246 | INFO | BAZWRITER : 58.8%, 4.0, 120.8ms, 0 | 20260505 04:42:49.246 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:42:49.246 | INFO | Uploaded 36GiB to GPU in 1.64158s (21.9301GiB/s) | 20260505 04:42:49.246 | INFO | Downloaded 38GiB to GPU in 1.70413s (22.2987GiB/s) | 20260505 04:42:54.385 | INFO | Analyzing chunk frames = [426496,427008) | 20260505 04:42:54.385 | INFO | MemInfo: { | 20260505 04:42:54.385 | INFO | "gpu" : " 17 GB", | 20260505 04:42:54.385 | INFO | "huge" : "", | 20260505 04:42:54.385 | INFO | "huge_shared" : " 27 GB", | 20260505 04:42:54.385 | INFO | "malloc" : " 14 GB", | 20260505 04:42:54.385 | INFO | "pinned" : " 29 GB" | 20260505 04:42:54.385 | INFO | } | 20260505 04:42:54.385 | INFO | Perf report for the last 5.13961 seconds: | 20260505 04:42:54.385 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:42:54.385 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:54.385 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:54.385 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:54.385 | INFO | ANALYSIS : 58.0%, 3.0, 90.4ms, 0 | 20260505 04:42:54.385 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:54.385 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:54.385 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 04:42:54.385 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:54.385 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:54.385 | INFO | Uploaded 36GiB to GPU in 1.59916s (22.5118GiB/s) | 20260505 04:42:54.385 | INFO | Downloaded 34GiB to GPU in 1.51581s (22.4303GiB/s) | 20260505 04:42:59.525 | INFO | Analyzing chunk frames = [427008,427520) | 20260505 04:42:59.525 | INFO | MemInfo: { | 20260505 04:42:59.525 | INFO | "gpu" : " 17 GB", | 20260505 04:42:59.525 | INFO | "huge" : "", | 20260505 04:42:59.525 | INFO | "huge_shared" : " 27 GB", | 20260505 04:42:59.525 | INFO | "malloc" : " 14 GB", | 20260505 04:42:59.525 | INFO | "pinned" : " 29 GB" | 20260505 04:42:59.525 | INFO | } | 20260505 04:42:59.525 | INFO | Perf report for the last 5.13962 seconds: | 20260505 04:42:59.525 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:42:59.525 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:59.525 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:59.525 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:59.525 | INFO | ANALYSIS : 57.4%, 3.0, 89.2ms, 0 | 20260505 04:42:59.525 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:59.525 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:59.525 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 04:42:59.525 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:59.525 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:42:59.525 | INFO | Uploaded 36GiB to GPU in 1.60238s (22.4666GiB/s) | 20260505 04:42:59.525 | INFO | Downloaded 34GiB to GPU in 1.51983s (22.3709GiB/s) | 20260505 04:43:04.664 | INFO | Analyzing chunk frames = [427520,428032) | 20260505 04:43:04.664 | INFO | MemInfo: { | 20260505 04:43:04.664 | INFO | "gpu" : " 17 GB", | 20260505 04:43:04.664 | INFO | "huge" : "", | 20260505 04:43:04.664 | INFO | "huge_shared" : " 27 GB", | 20260505 04:43:04.664 | INFO | "malloc" : " 14 GB", | 20260505 04:43:04.664 | INFO | "pinned" : " 29 GB" | 20260505 04:43:04.664 | INFO | } | 20260505 04:43:04.664 | INFO | Perf report for the last 5.13913 seconds: | 20260505 04:43:04.664 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:43:04.664 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:04.664 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:04.664 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:04.664 | INFO | ANALYSIS : 57.1%, 3.0, 88.6ms, 0 | 20260505 04:43:04.664 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:04.664 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:04.664 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 04:43:04.664 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:04.664 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:04.664 | INFO | Uploaded 36GiB to GPU in 1.60208s (22.4708GiB/s) | 20260505 04:43:04.664 | INFO | Downloaded 34GiB to GPU in 1.51851s (22.3903GiB/s) | 20260505 04:43:09.803 | INFO | Analyzing chunk frames = [428032,428544) | 20260505 04:43:09.803 | INFO | MemInfo: { | 20260505 04:43:09.803 | INFO | "gpu" : " 17 GB", | 20260505 04:43:09.803 | INFO | "huge" : "", | 20260505 04:43:09.803 | INFO | "huge_shared" : " 27 GB", | 20260505 04:43:09.803 | INFO | "malloc" : " 14 GB", | 20260505 04:43:09.803 | INFO | "pinned" : " 29 GB" | 20260505 04:43:09.803 | INFO | } | 20260505 04:43:09.803 | INFO | Perf report for the last 5.13921 seconds: | 20260505 04:43:09.803 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:43:09.803 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:09.803 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:09.803 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:09.803 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260505 04:43:09.803 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:09.803 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:09.803 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 04:43:09.803 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:09.803 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:09.803 | INFO | Uploaded 36GiB to GPU in 1.60059s (22.4916GiB/s) | 20260505 04:43:09.804 | INFO | Downloaded 34GiB to GPU in 1.51724s (22.4091GiB/s) | 20260505 04:43:14.942 | INFO | Analyzing chunk frames = [428544,429056) | 20260505 04:43:14.942 | INFO | MemInfo: { | 20260505 04:43:14.942 | INFO | "gpu" : " 17 GB", | 20260505 04:43:14.942 | INFO | "huge" : "", | 20260505 04:43:14.942 | INFO | "huge_shared" : " 27 GB", | 20260505 04:43:14.942 | INFO | "malloc" : " 14 GB", | 20260505 04:43:14.942 | INFO | "pinned" : " 29 GB" | 20260505 04:43:14.942 | INFO | } | 20260505 04:43:14.942 | INFO | Perf report for the last 5.13929 seconds: | 20260505 04:43:14.942 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:43:14.942 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:14.942 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:14.942 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:14.943 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 04:43:14.943 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:14.943 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:14.943 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 04:43:14.943 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:14.943 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:14.943 | INFO | Uploaded 36GiB to GPU in 1.60026s (22.4964GiB/s) | 20260505 04:43:14.943 | INFO | Downloaded 34GiB to GPU in 1.51645s (22.4208GiB/s) | 20260505 04:43:20.082 | INFO | Analyzing chunk frames = [429056,429568) | 20260505 04:43:20.082 | INFO | MemInfo: { | 20260505 04:43:20.082 | INFO | "gpu" : " 17 GB", | 20260505 04:43:20.082 | INFO | "huge" : "", | 20260505 04:43:20.082 | INFO | "huge_shared" : " 27 GB", | 20260505 04:43:20.082 | INFO | "malloc" : " 14 GB", | 20260505 04:43:20.082 | INFO | "pinned" : " 29 GB" | 20260505 04:43:20.082 | INFO | } | 20260505 04:43:20.082 | INFO | Perf report for the last 5.13942 seconds: | 20260505 04:43:20.082 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:43:20.082 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:20.082 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:20.082 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:20.082 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260505 04:43:20.082 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:20.082 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:20.082 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 04:43:20.082 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:20.082 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:20.082 | INFO | Uploaded 36GiB to GPU in 1.6008s (22.4887GiB/s) | 20260505 04:43:20.082 | INFO | Downloaded 34GiB to GPU in 1.51738s (22.407GiB/s) | 20260505 04:43:25.221 | INFO | Analyzing chunk frames = [429568,430080) | 20260505 04:43:25.221 | INFO | MemInfo: { | 20260505 04:43:25.221 | INFO | "gpu" : " 17 GB", | 20260505 04:43:25.221 | INFO | "huge" : "", | 20260505 04:43:25.221 | INFO | "huge_shared" : " 27 GB", | 20260505 04:43:25.221 | INFO | "malloc" : " 14 GB", | 20260505 04:43:25.221 | INFO | "pinned" : " 29 GB" | 20260505 04:43:25.221 | INFO | } | 20260505 04:43:25.221 | INFO | Perf report for the last 5.1393 seconds: | 20260505 04:43:25.221 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:43:25.221 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:25.221 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:25.221 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:25.221 | INFO | ANALYSIS : 57.6%, 3.0, 89.9ms, 0 | 20260505 04:43:25.221 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:25.221 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:25.221 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 04:43:25.221 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:25.221 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:25.222 | INFO | Uploaded 36GiB to GPU in 1.60095s (22.4866GiB/s) | 20260505 04:43:25.222 | INFO | Downloaded 34GiB to GPU in 1.51815s (22.3957GiB/s) | 20260505 04:43:25.319 | INFO | Starting a round of Baz IO | 20260505 04:43:28.270 | INFO | Populating RTMetrics report | 20260505 04:43:28.270 | INFO | RTMetrics report populated | 20260505 04:43:28.270 | INFO | Opening tmp RTMetrics json file | 20260505 04:43:28.271 | INFO | Serializing RTMetrics to json | 20260505 04:43:28.272 | INFO | Serialization done, swapping tmp file now... | 20260505 04:43:28.272 | INFO | RTMetrics done | 20260505 04:43:28.358 | INFO | Finished a round of Baz IO: Wrote 2.044 GiB in 3.039 seconds | 20260505 04:43:28.358 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.21% / 0.13% / 54.66% | 20260505 04:43:28.358 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 10.006% / 0.000301997% / 5.96046e-05% | 20260505 04:43:28.358 | INFO | Baz IO overall: | 20260505 04:43:28.358 | INFO | BytesWritten : 176716420319 | 20260505 04:43:28.358 | INFO | NumEvents : 82112460755 | 20260505 04:43:30.360 | INFO | Analyzing chunk frames = [430080,430592) | 20260505 04:43:30.360 | INFO | MemInfo: { | 20260505 04:43:30.360 | INFO | "gpu" : " 17 GB", | 20260505 04:43:30.360 | INFO | "huge" : "", | 20260505 04:43:30.360 | INFO | "huge_shared" : " 27 GB", | 20260505 04:43:30.360 | INFO | "malloc" : " 14 GB", | 20260505 04:43:30.360 | INFO | "pinned" : " 29 GB" | 20260505 04:43:30.360 | INFO | } | 20260505 04:43:30.361 | INFO | Perf report for the last 5.13929 seconds: | 20260505 04:43:30.361 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:43:30.361 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:30.361 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:30.361 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:30.361 | INFO | ANALYSIS : 58.2%, 3.0, 90.5ms, 0 | 20260505 04:43:30.361 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:30.361 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260505 04:43:30.361 | INFO | PRE_HQ : 57.5%, 2.1, 64.1ms, 0 | 20260505 04:43:30.361 | INFO | BAZWRITER : 58.7%, 4.0, 120.9ms, 0 | 20260505 04:43:30.361 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:43:30.361 | INFO | Uploaded 36GiB to GPU in 1.63901s (21.9644GiB/s) | 20260505 04:43:30.361 | INFO | Downloaded 38GiB to GPU in 1.701s (22.3398GiB/s) | 20260505 04:43:35.500 | INFO | Analyzing chunk frames = [430592,431104) | 20260505 04:43:35.500 | INFO | MemInfo: { | 20260505 04:43:35.500 | INFO | "gpu" : " 17 GB", | 20260505 04:43:35.500 | INFO | "huge" : "", | 20260505 04:43:35.500 | INFO | "huge_shared" : " 27 GB", | 20260505 04:43:35.500 | INFO | "malloc" : " 14 GB", | 20260505 04:43:35.500 | INFO | "pinned" : " 29 GB" | 20260505 04:43:35.500 | INFO | } | 20260505 04:43:35.500 | INFO | Perf report for the last 5.13931 seconds: | 20260505 04:43:35.500 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:43:35.500 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:35.500 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:35.500 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:35.500 | INFO | ANALYSIS : 58.1%, 3.0, 90.5ms, 0 | 20260505 04:43:35.500 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:35.500 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:35.500 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 04:43:35.500 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:35.500 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:35.500 | INFO | Uploaded 36GiB to GPU in 1.59888s (22.5158GiB/s) | 20260505 04:43:35.500 | INFO | Downloaded 34GiB to GPU in 1.51565s (22.4327GiB/s) | 20260505 04:43:40.639 | INFO | Analyzing chunk frames = [431104,431616) | 20260505 04:43:40.639 | INFO | MemInfo: { | 20260505 04:43:40.639 | INFO | "gpu" : " 17 GB", | 20260505 04:43:40.639 | INFO | "huge" : "", | 20260505 04:43:40.639 | INFO | "huge_shared" : " 27 GB", | 20260505 04:43:40.639 | INFO | "malloc" : " 14 GB", | 20260505 04:43:40.639 | INFO | "pinned" : " 29 GB" | 20260505 04:43:40.639 | INFO | } | 20260505 04:43:40.639 | INFO | Perf report for the last 5.13952 seconds: | 20260505 04:43:40.639 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:43:40.639 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:40.639 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:40.639 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:40.639 | INFO | ANALYSIS : 57.4%, 3.0, 89.2ms, 0 | 20260505 04:43:40.639 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:40.639 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:40.639 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260505 04:43:40.639 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:40.639 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:40.640 | INFO | Uploaded 36GiB to GPU in 1.60315s (22.4559GiB/s) | 20260505 04:43:40.640 | INFO | Downloaded 34GiB to GPU in 1.52029s (22.3641GiB/s) | 20260505 04:43:45.778 | INFO | Analyzing chunk frames = [431616,432128) | 20260505 04:43:45.778 | INFO | MemInfo: { | 20260505 04:43:45.778 | INFO | "gpu" : " 17 GB", | 20260505 04:43:45.778 | INFO | "huge" : "", | 20260505 04:43:45.778 | INFO | "huge_shared" : " 27 GB", | 20260505 04:43:45.778 | INFO | "malloc" : " 14 GB", | 20260505 04:43:45.778 | INFO | "pinned" : " 29 GB" | 20260505 04:43:45.778 | INFO | } | 20260505 04:43:45.779 | INFO | Perf report for the last 5.13918 seconds: | 20260505 04:43:45.779 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:43:45.779 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:45.779 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:45.779 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:45.779 | INFO | ANALYSIS : 57.1%, 3.0, 88.6ms, 0 | 20260505 04:43:45.779 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:45.779 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:45.779 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260505 04:43:45.779 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:45.779 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:45.779 | INFO | Uploaded 36GiB to GPU in 1.60224s (22.4685GiB/s) | 20260505 04:43:45.779 | INFO | Downloaded 34GiB to GPU in 1.51885s (22.3853GiB/s) | 20260505 04:43:50.918 | INFO | Analyzing chunk frames = [432128,432640) | 20260505 04:43:50.918 | INFO | MemInfo: { | 20260505 04:43:50.918 | INFO | "gpu" : " 17 GB", | 20260505 04:43:50.918 | INFO | "huge" : "", | 20260505 04:43:50.918 | INFO | "huge_shared" : " 27 GB", | 20260505 04:43:50.918 | INFO | "malloc" : " 14 GB", | 20260505 04:43:50.918 | INFO | "pinned" : " 29 GB" | 20260505 04:43:50.918 | INFO | } | 20260505 04:43:50.918 | INFO | Perf report for the last 5.13948 seconds: | 20260505 04:43:50.918 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:43:50.918 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:50.918 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:50.918 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:50.918 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260505 04:43:50.918 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:50.918 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:50.918 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260505 04:43:50.918 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:50.918 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:50.918 | INFO | Uploaded 36GiB to GPU in 1.60047s (22.4933GiB/s) | 20260505 04:43:50.918 | INFO | Downloaded 34GiB to GPU in 1.5167s (22.4171GiB/s) | 20260505 04:43:56.057 | INFO | Analyzing chunk frames = [432640,433152) | 20260505 04:43:56.057 | INFO | MemInfo: { | 20260505 04:43:56.057 | INFO | "gpu" : " 17 GB", | 20260505 04:43:56.057 | INFO | "huge" : "", | 20260505 04:43:56.057 | INFO | "huge_shared" : " 27 GB", | 20260505 04:43:56.057 | INFO | "malloc" : " 14 GB", | 20260505 04:43:56.057 | INFO | "pinned" : " 29 GB" | 20260505 04:43:56.057 | INFO | } | 20260505 04:43:56.057 | INFO | Perf report for the last 5.13931 seconds: | 20260505 04:43:56.057 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:43:56.057 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:56.057 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:56.057 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:56.057 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 04:43:56.057 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:56.057 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:56.057 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 04:43:56.057 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:56.058 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:43:56.058 | INFO | Uploaded 36GiB to GPU in 1.60042s (22.494GiB/s) | 20260505 04:43:56.058 | INFO | Downloaded 34GiB to GPU in 1.51708s (22.4114GiB/s) | 20260505 04:44:01.196 | INFO | Analyzing chunk frames = [433152,433664) | 20260505 04:44:01.196 | INFO | MemInfo: { | 20260505 04:44:01.196 | INFO | "gpu" : " 17 GB", | 20260505 04:44:01.196 | INFO | "huge" : "", | 20260505 04:44:01.196 | INFO | "huge_shared" : " 27 GB", | 20260505 04:44:01.197 | INFO | "malloc" : " 14 GB", | 20260505 04:44:01.197 | INFO | "pinned" : " 29 GB" | 20260505 04:44:01.197 | INFO | } | 20260505 04:44:01.197 | INFO | Perf report for the last 5.13936 seconds: | 20260505 04:44:01.197 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:44:01.197 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:01.197 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:01.197 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:01.197 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260505 04:44:01.197 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:01.197 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:01.197 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 04:44:01.197 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:01.197 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:01.197 | INFO | Uploaded 36GiB to GPU in 1.60188s (22.4736GiB/s) | 20260505 04:44:01.197 | INFO | Downloaded 34GiB to GPU in 1.51846s (22.3911GiB/s) | 20260505 04:44:06.336 | INFO | Analyzing chunk frames = [433664,434176) | 20260505 04:44:06.336 | INFO | MemInfo: { | 20260505 04:44:06.336 | INFO | "gpu" : " 17 GB", | 20260505 04:44:06.336 | INFO | "huge" : "", | 20260505 04:44:06.336 | INFO | "huge_shared" : " 27 GB", | 20260505 04:44:06.336 | INFO | "malloc" : " 14 GB", | 20260505 04:44:06.336 | INFO | "pinned" : " 29 GB" | 20260505 04:44:06.336 | INFO | } | 20260505 04:44:06.336 | INFO | Perf report for the last 5.13926 seconds: | 20260505 04:44:06.336 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:44:06.336 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:06.336 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:06.336 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:06.336 | INFO | ANALYSIS : 57.6%, 3.0, 89.9ms, 0 | 20260505 04:44:06.336 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:06.336 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:06.336 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 04:44:06.336 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:06.336 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:06.336 | INFO | Uploaded 36GiB to GPU in 1.60148s (22.4792GiB/s) | 20260505 04:44:06.336 | INFO | Downloaded 34GiB to GPU in 1.51818s (22.3953GiB/s) | 20260505 04:44:06.336 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 04:44:06.336 | INFO | Total mem usage: 15.1 GiB | 20260505 04:44:06.336 | INFO | All Time High Water Marks sum: 15.7 GiB (Savings: 4.09%) | 20260505 04:44:06.336 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -4.07%) | 20260505 04:44:06.336 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 04:44:06.337 | INFO | High Water Breakdown: | 20260505 04:44:06.337 | INFO | 59.4% : 8.59 GiB ( 95.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 04:44:06.337 | INFO | 40.5% : 5.86 GiB ( 88.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 04:44:06.337 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 04:44:06.337 | INFO | ------------------------------------------ | 20260505 04:44:06.337 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:44:06.337 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 04:44:06.337 | INFO | Total mem usage: 26.5 GiB | 20260505 04:44:06.337 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260505 04:44:06.337 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 04:44:06.337 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260505 04:44:06.337 | INFO | High Water Breakdown: | 20260505 04:44:06.337 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:44:06.337 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:44:06.337 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:44:06.337 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:44:06.337 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:44:06.337 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:44:06.337 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 04:44:06.337 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 04:44:06.337 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:44:06.337 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 04:44:06.337 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 04:44:06.337 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 04:44:06.337 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 04:44:06.337 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 04:44:06.337 | INFO | ------------------------------------------ | 20260505 04:44:06.337 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:44:06.337 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 04:44:06.337 | INFO | Total mem usage: 25.6 GiB | 20260505 04:44:06.337 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 04:44:06.337 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 04:44:06.337 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 04:44:06.337 | INFO | High Water Breakdown: | 20260505 04:44:06.337 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 04:44:06.337 | INFO | ------------------------------------------ | 20260505 04:44:06.337 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:44:06.337 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 04:44:06.337 | INFO | Total mem usage: 15.7 GiB | 20260505 04:44:06.337 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260505 04:44:06.337 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 04:44:06.337 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260505 04:44:06.337 | INFO | High Water Breakdown: | 20260505 04:44:06.337 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 04:44:06.337 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 04:44:06.337 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 04:44:06.337 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 04:44:06.337 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 04:44:06.337 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 04:44:06.337 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 04:44:06.337 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 04:44:06.337 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 04:44:06.337 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:44:06.337 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 04:44:06.337 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:44:06.337 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 04:44:06.337 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 04:44:06.337 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 04:44:06.337 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 04:44:06.337 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:44:06.337 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 04:44:06.337 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:44:06.338 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:44:06.338 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:44:06.338 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 04:44:06.338 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 04:44:06.338 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:44:06.338 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 04:44:06.338 | INFO | ------------------------------------------ | 20260505 04:44:06.338 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:44:06.338 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 04:44:06.338 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:44:06.338 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:44:06.338 | INFO | PerfReport: Full Execution -- 3.04e+01 / -nan, 1.28e+01 / 0.00e+00 | 20260505 04:44:06.338 | INFO | PerfReport: 0.28%, AggregateStats -- 8.65e-02 / -nan, 4.49e-03 / 0.00e+00 | 20260505 04:44:06.338 | INFO | PerfReport: 10.83%, Baseline -- 3.30e+00 / -nan, 1.64e-02 / 0.00e+00 | 20260505 04:44:06.338 | INFO | PerfReport: 3.04%, Histogram -- 9.24e-01 / -nan, 5.23e-02 / 0.00e+00 | 20260505 04:44:06.338 | INFO | PerfReport: 16.03%, DME -- 3.91e+01 / -nan, 3.21e+00 / 0.00e+00 | 20260505 04:44:06.338 | INFO | PerfReport: 56.52%, FrameLabeling -- 1.72e+01 / -nan, 4.46e-02 / 0.00e+00 | 20260505 04:44:06.338 | INFO | PerfReport: 7.64%, PulseAccumulating -- 2.33e+00 / -nan, 1.55e-01 / 0.00e+00 | 20260505 04:44:06.338 | INFO | PerfReport: 4.00%, Metrics -- 1.22e+00 / -nan, 2.01e-01 / 0.00e+00 | 20260505 04:44:06.338 | INFO | PerfReport: 1.65%, PulseSerializing -- 5.03e-01 / -nan, 8.16e-02 / 0.00e+00 | 20260505 04:44:06.338 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 04:44:06.338 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:44:06.338 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:44:06.338 | INFO | PerfReport: Full Execution -- 9.14e+01 / -nan, 1.89e+01 / 0.00e+00 | 20260505 04:44:06.338 | INFO | PerfReport: 18.29%, Upload -- 1.67e+01 / 0.00e+00, 3.72e-01 / 0.00e+00 | 20260505 04:44:06.338 | INFO | PerfReport: 17.55%, Download -- 1.61e+01 / 0.00e+00, 6.99e-01 / 0.00e+00 | 20260505 04:44:06.338 | INFO | PerfReport: 33.30%, Compute -- 3.05e+01 / 0.00e+00, 1.28e+01 / 0.00e+00 | 20260505 04:44:06.338 | INFO | PerfReport: 0.55%, LockUpload -- 5.03e-01 / 0.00e+00, 3.60e+00 / 0.00e+00 | 20260505 04:44:06.338 | INFO | PerfReport: 0.00%, LockDownload -- 3.96e-04 / 0.00e+00, 1.98e-04 / 0.00e+00 | 20260505 04:44:06.338 | INFO | PerfReport: 30.29%, LockCompute -- 2.77e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260505 04:44:06.338 | INFO | PerfReport: 0.01%, Misc -- 1.27e-02 / 0.00e+00, 9.83e-03 / 0.00e+00 | 20260505 04:44:06.433 | INFO | Starting a round of Baz IO | 20260505 04:44:09.386 | INFO | Populating RTMetrics report | 20260505 04:44:09.386 | INFO | RTMetrics report populated | 20260505 04:44:09.386 | INFO | Opening tmp RTMetrics json file | 20260505 04:44:09.387 | INFO | Serializing RTMetrics to json | 20260505 04:44:09.388 | INFO | Serialization done, swapping tmp file now... | 20260505 04:44:09.389 | INFO | RTMetrics done | 20260505 04:44:09.475 | INFO | Finished a round of Baz IO: Wrote 2.048 GiB in 3.042 seconds | 20260505 04:44:09.475 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.08% / 0.13% / 54.79% | 20260505 04:44:09.475 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.96987% / 0.000266234% / 4.37101e-05% | 20260505 04:44:09.475 | INFO | Baz IO overall: | 20260505 04:44:09.475 | INFO | BytesWritten : 178915541418 | 20260505 04:44:09.475 | INFO | NumEvents : 83155692218 | 20260505 04:44:11.475 | INFO | Analyzing chunk frames = [434176,434688) | 20260505 04:44:11.475 | INFO | MemInfo: { | 20260505 04:44:11.475 | INFO | "gpu" : " 17 GB", | 20260505 04:44:11.475 | INFO | "huge" : "", | 20260505 04:44:11.475 | INFO | "huge_shared" : " 27 GB", | 20260505 04:44:11.475 | INFO | "malloc" : " 14 GB", | 20260505 04:44:11.475 | INFO | "pinned" : " 29 GB" | 20260505 04:44:11.475 | INFO | } | 20260505 04:44:11.475 | INFO | Perf report for the last 5.13924 seconds: | 20260505 04:44:11.475 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:44:11.475 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:11.475 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:11.475 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:11.475 | INFO | ANALYSIS : 58.2%, 3.0, 90.6ms, 0 | 20260505 04:44:11.475 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:11.475 | INFO | RT_METRICS : 45.3%, 1.0, 23.7ms, 0 | 20260505 04:44:11.475 | INFO | PRE_HQ : 57.6%, 2.1, 64.1ms, 0 | 20260505 04:44:11.475 | INFO | BAZWRITER : 58.8%, 4.0, 120.5ms, 0 | 20260505 04:44:11.475 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:44:11.476 | INFO | Uploaded 36GiB to GPU in 1.63994s (21.952GiB/s) | 20260505 04:44:11.476 | INFO | Downloaded 38GiB to GPU in 1.70079s (22.3425GiB/s) | 20260505 04:44:16.614 | INFO | Analyzing chunk frames = [434688,435200) | 20260505 04:44:16.614 | INFO | MemInfo: { | 20260505 04:44:16.614 | INFO | "gpu" : " 17 GB", | 20260505 04:44:16.614 | INFO | "huge" : "", | 20260505 04:44:16.614 | INFO | "huge_shared" : " 27 GB", | 20260505 04:44:16.614 | INFO | "malloc" : " 14 GB", | 20260505 04:44:16.614 | INFO | "pinned" : " 29 GB" | 20260505 04:44:16.614 | INFO | } | 20260505 04:44:16.615 | INFO | Perf report for the last 5.13928 seconds: | 20260505 04:44:16.615 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:44:16.615 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:16.615 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:16.615 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:16.615 | INFO | ANALYSIS : 58.0%, 3.0, 90.5ms, 0 | 20260505 04:44:16.615 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:16.615 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:16.615 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 04:44:16.615 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:16.615 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:16.615 | INFO | Uploaded 36GiB to GPU in 1.59943s (22.508GiB/s) | 20260505 04:44:16.615 | INFO | Downloaded 34GiB to GPU in 1.51587s (22.4294GiB/s) | 20260505 04:44:21.754 | INFO | Analyzing chunk frames = [435200,435712) | 20260505 04:44:21.754 | INFO | MemInfo: { | 20260505 04:44:21.754 | INFO | "gpu" : " 17 GB", | 20260505 04:44:21.754 | INFO | "huge" : "", | 20260505 04:44:21.754 | INFO | "huge_shared" : " 27 GB", | 20260505 04:44:21.754 | INFO | "malloc" : " 14 GB", | 20260505 04:44:21.754 | INFO | "pinned" : " 29 GB" | 20260505 04:44:21.754 | INFO | } | 20260505 04:44:21.754 | INFO | Perf report for the last 5.13939 seconds: | 20260505 04:44:21.754 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:44:21.754 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:21.754 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:21.754 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:21.754 | INFO | ANALYSIS : 57.4%, 3.0, 89.1ms, 0 | 20260505 04:44:21.754 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:21.754 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:21.754 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 04:44:21.754 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:21.754 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:21.754 | INFO | Uploaded 36GiB to GPU in 1.60261s (22.4633GiB/s) | 20260505 04:44:21.754 | INFO | Downloaded 34GiB to GPU in 1.5199s (22.37GiB/s) | 20260505 04:44:26.893 | INFO | Analyzing chunk frames = [435712,436224) | 20260505 04:44:26.893 | INFO | MemInfo: { | 20260505 04:44:26.893 | INFO | "gpu" : " 17 GB", | 20260505 04:44:26.893 | INFO | "huge" : "", | 20260505 04:44:26.893 | INFO | "huge_shared" : " 27 GB", | 20260505 04:44:26.893 | INFO | "malloc" : " 14 GB", | 20260505 04:44:26.893 | INFO | "pinned" : " 29 GB" | 20260505 04:44:26.893 | INFO | } | 20260505 04:44:26.893 | INFO | Perf report for the last 5.1393 seconds: | 20260505 04:44:26.893 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:44:26.893 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:26.893 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:26.893 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:26.893 | INFO | ANALYSIS : 57.1%, 3.0, 88.5ms, 0 | 20260505 04:44:26.893 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:26.893 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:26.893 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 04:44:26.893 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:26.893 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:26.894 | INFO | Uploaded 36GiB to GPU in 1.60167s (22.4766GiB/s) | 20260505 04:44:26.894 | INFO | Downloaded 34GiB to GPU in 1.5181s (22.3965GiB/s) | 20260505 04:44:32.032 | INFO | Analyzing chunk frames = [436224,436736) | 20260505 04:44:32.032 | INFO | MemInfo: { | 20260505 04:44:32.032 | INFO | "gpu" : " 17 GB", | 20260505 04:44:32.032 | INFO | "huge" : "", | 20260505 04:44:32.032 | INFO | "huge_shared" : " 27 GB", | 20260505 04:44:32.033 | INFO | "malloc" : " 14 GB", | 20260505 04:44:32.033 | INFO | "pinned" : " 29 GB" | 20260505 04:44:32.033 | INFO | } | 20260505 04:44:32.033 | INFO | Perf report for the last 5.13939 seconds: | 20260505 04:44:32.033 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:44:32.033 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:32.033 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:32.033 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:32.033 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260505 04:44:32.033 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:32.033 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:32.033 | INFO | PRE_HQ : 31.7%, 1.0, 16.4ms, 0 | 20260505 04:44:32.033 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:32.033 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:32.033 | INFO | Uploaded 36GiB to GPU in 1.60152s (22.4786GiB/s) | 20260505 04:44:32.033 | INFO | Downloaded 34GiB to GPU in 1.51796s (22.3985GiB/s) | 20260505 04:44:37.172 | INFO | Analyzing chunk frames = [436736,437248) | 20260505 04:44:37.172 | INFO | MemInfo: { | 20260505 04:44:37.172 | INFO | "gpu" : " 17 GB", | 20260505 04:44:37.172 | INFO | "huge" : "", | 20260505 04:44:37.172 | INFO | "huge_shared" : " 27 GB", | 20260505 04:44:37.172 | INFO | "malloc" : " 14 GB", | 20260505 04:44:37.172 | INFO | "pinned" : " 29 GB" | 20260505 04:44:37.172 | INFO | } | 20260505 04:44:37.172 | INFO | Perf report for the last 5.1392 seconds: | 20260505 04:44:37.172 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:44:37.172 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:37.172 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:37.172 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:37.172 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 04:44:37.172 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:37.172 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:37.172 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 04:44:37.172 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:37.172 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:37.173 | INFO | Uploaded 36GiB to GPU in 1.60096s (22.4866GiB/s) | 20260505 04:44:37.173 | INFO | Downloaded 34GiB to GPU in 1.51762s (22.4035GiB/s) | 20260505 04:44:42.311 | INFO | Analyzing chunk frames = [437248,437760) | 20260505 04:44:42.311 | INFO | MemInfo: { | 20260505 04:44:42.311 | INFO | "gpu" : " 17 GB", | 20260505 04:44:42.311 | INFO | "huge" : "", | 20260505 04:44:42.311 | INFO | "huge_shared" : " 27 GB", | 20260505 04:44:42.311 | INFO | "malloc" : " 14 GB", | 20260505 04:44:42.311 | INFO | "pinned" : " 29 GB" | 20260505 04:44:42.311 | INFO | } | 20260505 04:44:42.311 | INFO | Perf report for the last 5.13951 seconds: | 20260505 04:44:42.311 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:44:42.311 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:42.311 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:42.311 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:42.311 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260505 04:44:42.312 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:42.312 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:42.312 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 04:44:42.312 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:42.312 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:42.312 | INFO | Uploaded 36GiB to GPU in 1.60106s (22.4851GiB/s) | 20260505 04:44:42.312 | INFO | Downloaded 34GiB to GPU in 1.51795s (22.3986GiB/s) | 20260505 04:44:47.450 | INFO | Analyzing chunk frames = [437760,438272) | 20260505 04:44:47.450 | INFO | MemInfo: { | 20260505 04:44:47.450 | INFO | "gpu" : " 17 GB", | 20260505 04:44:47.450 | INFO | "huge" : "", | 20260505 04:44:47.450 | INFO | "huge_shared" : " 27 GB", | 20260505 04:44:47.451 | INFO | "malloc" : " 14 GB", | 20260505 04:44:47.451 | INFO | "pinned" : " 29 GB" | 20260505 04:44:47.451 | INFO | } | 20260505 04:44:47.451 | INFO | Perf report for the last 5.13931 seconds: | 20260505 04:44:47.451 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:44:47.451 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:47.451 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:47.451 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:47.451 | INFO | ANALYSIS : 57.6%, 3.0, 89.9ms, 0 | 20260505 04:44:47.451 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:47.451 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:47.451 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 04:44:47.451 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:47.451 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:47.451 | INFO | Uploaded 36GiB to GPU in 1.60125s (22.4824GiB/s) | 20260505 04:44:47.451 | INFO | Downloaded 34GiB to GPU in 1.5184s (22.392GiB/s) | 20260505 04:44:47.548 | INFO | Starting a round of Baz IO | 20260505 04:44:50.501 | INFO | Populating RTMetrics report | 20260505 04:44:50.501 | INFO | RTMetrics report populated | 20260505 04:44:50.501 | INFO | Opening tmp RTMetrics json file | 20260505 04:44:50.502 | INFO | Serializing RTMetrics to json | 20260505 04:44:50.504 | INFO | Serialization done, swapping tmp file now... | 20260505 04:44:50.505 | INFO | RTMetrics done | 20260505 04:44:50.594 | INFO | Finished a round of Baz IO: Wrote 2.05 GiB in 3.046 seconds | 20260505 04:44:50.594 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.96% / 0.13% / 54.92% | 20260505 04:44:50.594 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.88887% / 0.000266234% / 4.76837e-05% | 20260505 04:44:50.594 | INFO | Baz IO overall: | 20260505 04:44:50.594 | INFO | BytesWritten : 181116869916 | 20260505 04:44:50.594 | INFO | NumEvents : 84199592086 | 20260505 04:44:52.590 | INFO | Analyzing chunk frames = [438272,438784) | 20260505 04:44:52.590 | INFO | MemInfo: { | 20260505 04:44:52.590 | INFO | "gpu" : " 17 GB", | 20260505 04:44:52.590 | INFO | "huge" : "", | 20260505 04:44:52.590 | INFO | "huge_shared" : " 27 GB", | 20260505 04:44:52.590 | INFO | "malloc" : " 14 GB", | 20260505 04:44:52.590 | INFO | "pinned" : " 29 GB" | 20260505 04:44:52.590 | INFO | } | 20260505 04:44:52.590 | INFO | Perf report for the last 5.13933 seconds: | 20260505 04:44:52.590 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:44:52.590 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:52.590 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:52.590 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:52.590 | INFO | ANALYSIS : 58.2%, 3.0, 90.5ms, 0 | 20260505 04:44:52.590 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:52.590 | INFO | RT_METRICS : 45.5%, 1.0, 23.9ms, 0 | 20260505 04:44:52.590 | INFO | PRE_HQ : 57.8%, 2.2, 65.0ms, 0 | 20260505 04:44:52.590 | INFO | BAZWRITER : 58.8%, 4.0, 120.3ms, 0 | 20260505 04:44:52.590 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 04:44:52.590 | INFO | Uploaded 36GiB to GPU in 1.63547s (22.0121GiB/s) | 20260505 04:44:52.590 | INFO | Downloaded 38GiB to GPU in 1.69854s (22.3721GiB/s) | 20260505 04:44:57.729 | INFO | Analyzing chunk frames = [438784,439296) | 20260505 04:44:57.729 | INFO | MemInfo: { | 20260505 04:44:57.729 | INFO | "gpu" : " 17 GB", | 20260505 04:44:57.729 | INFO | "huge" : "", | 20260505 04:44:57.729 | INFO | "huge_shared" : " 27 GB", | 20260505 04:44:57.729 | INFO | "malloc" : " 14 GB", | 20260505 04:44:57.729 | INFO | "pinned" : " 29 GB" | 20260505 04:44:57.729 | INFO | } | 20260505 04:44:57.729 | INFO | Perf report for the last 5.13941 seconds: | 20260505 04:44:57.729 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:44:57.729 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:57.729 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:57.730 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:57.730 | INFO | ANALYSIS : 58.0%, 3.0, 90.5ms, 0 | 20260505 04:44:57.730 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:57.730 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:57.730 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 04:44:57.730 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:57.730 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:44:57.730 | INFO | Uploaded 36GiB to GPU in 1.59914s (22.5121GiB/s) | 20260505 04:44:57.730 | INFO | Downloaded 34GiB to GPU in 1.51563s (22.4329GiB/s) | 20260505 04:45:02.868 | INFO | Analyzing chunk frames = [439296,439808) | 20260505 04:45:02.869 | INFO | MemInfo: { | 20260505 04:45:02.869 | INFO | "gpu" : " 17 GB", | 20260505 04:45:02.869 | INFO | "huge" : "", | 20260505 04:45:02.869 | INFO | "huge_shared" : " 27 GB", | 20260505 04:45:02.869 | INFO | "malloc" : " 14 GB", | 20260505 04:45:02.869 | INFO | "pinned" : " 29 GB" | 20260505 04:45:02.869 | INFO | } | 20260505 04:45:02.869 | INFO | Perf report for the last 5.13928 seconds: | 20260505 04:45:02.869 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:45:02.869 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:02.869 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:02.869 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:02.869 | INFO | ANALYSIS : 57.4%, 3.0, 89.2ms, 0 | 20260505 04:45:02.869 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:02.869 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:02.869 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260505 04:45:02.869 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:02.869 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:02.869 | INFO | Uploaded 36GiB to GPU in 1.60438s (22.4385GiB/s) | 20260505 04:45:02.869 | INFO | Downloaded 34GiB to GPU in 1.5208s (22.3567GiB/s) | 20260505 04:45:08.008 | INFO | Analyzing chunk frames = [439808,440320) | 20260505 04:45:08.008 | INFO | MemInfo: { | 20260505 04:45:08.008 | INFO | "gpu" : " 17 GB", | 20260505 04:45:08.008 | INFO | "huge" : "", | 20260505 04:45:08.008 | INFO | "huge_shared" : " 27 GB", | 20260505 04:45:08.008 | INFO | "malloc" : " 14 GB", | 20260505 04:45:08.008 | INFO | "pinned" : " 29 GB" | 20260505 04:45:08.008 | INFO | } | 20260505 04:45:08.008 | INFO | Perf report for the last 5.13947 seconds: | 20260505 04:45:08.008 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:45:08.008 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:08.008 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:08.008 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:08.008 | INFO | ANALYSIS : 57.1%, 3.0, 88.5ms, 0 | 20260505 04:45:08.008 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:08.008 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:08.008 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 04:45:08.008 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:08.008 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:08.009 | INFO | Uploaded 36GiB to GPU in 1.60241s (22.4662GiB/s) | 20260505 04:45:08.009 | INFO | Downloaded 34GiB to GPU in 1.51902s (22.3828GiB/s) | 20260505 04:45:13.147 | INFO | Analyzing chunk frames = [440320,440832) | 20260505 04:45:13.147 | INFO | MemInfo: { | 20260505 04:45:13.147 | INFO | "gpu" : " 17 GB", | 20260505 04:45:13.147 | INFO | "huge" : "", | 20260505 04:45:13.147 | INFO | "huge_shared" : " 27 GB", | 20260505 04:45:13.147 | INFO | "malloc" : " 14 GB", | 20260505 04:45:13.148 | INFO | "pinned" : " 29 GB" | 20260505 04:45:13.148 | INFO | } | 20260505 04:45:13.148 | INFO | Perf report for the last 5.1393 seconds: | 20260505 04:45:13.148 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:45:13.148 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:13.148 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:13.148 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:13.148 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260505 04:45:13.148 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:13.148 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:13.148 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260505 04:45:13.148 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:13.148 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:13.148 | INFO | Uploaded 36GiB to GPU in 1.60041s (22.4943GiB/s) | 20260505 04:45:13.148 | INFO | Downloaded 34GiB to GPU in 1.5164s (22.4215GiB/s) | 20260505 04:45:18.287 | INFO | Analyzing chunk frames = [440832,441344) | 20260505 04:45:18.287 | INFO | MemInfo: { | 20260505 04:45:18.287 | INFO | "gpu" : " 17 GB", | 20260505 04:45:18.287 | INFO | "huge" : "", | 20260505 04:45:18.287 | INFO | "huge_shared" : " 27 GB", | 20260505 04:45:18.287 | INFO | "malloc" : " 14 GB", | 20260505 04:45:18.287 | INFO | "pinned" : " 29 GB" | 20260505 04:45:18.287 | INFO | } | 20260505 04:45:18.287 | INFO | Perf report for the last 5.13921 seconds: | 20260505 04:45:18.287 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:45:18.287 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:18.287 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:18.287 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:18.287 | INFO | ANALYSIS : 57.3%, 3.0, 89.3ms, 0 | 20260505 04:45:18.287 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:18.287 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:18.287 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260505 04:45:18.287 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:18.287 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:18.287 | INFO | Uploaded 36GiB to GPU in 1.60365s (22.4487GiB/s) | 20260505 04:45:18.287 | INFO | Downloaded 34GiB to GPU in 1.51794s (22.3987GiB/s) | 20260505 04:45:23.426 | INFO | Analyzing chunk frames = [441344,441856) | 20260505 04:45:23.426 | INFO | MemInfo: { | 20260505 04:45:23.426 | INFO | "gpu" : " 17 GB", | 20260505 04:45:23.426 | INFO | "huge" : "", | 20260505 04:45:23.426 | INFO | "huge_shared" : " 27 GB", | 20260505 04:45:23.426 | INFO | "malloc" : " 14 GB", | 20260505 04:45:23.426 | INFO | "pinned" : " 29 GB" | 20260505 04:45:23.426 | INFO | } | 20260505 04:45:23.426 | INFO | Perf report for the last 5.13931 seconds: | 20260505 04:45:23.426 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:45:23.426 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:23.426 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:23.426 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:23.426 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260505 04:45:23.426 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:23.426 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:23.426 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 04:45:23.426 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:23.426 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:23.427 | INFO | Uploaded 36GiB to GPU in 1.60032s (22.4955GiB/s) | 20260505 04:45:23.427 | INFO | Downloaded 34GiB to GPU in 1.5171s (22.4112GiB/s) | 20260505 04:45:28.565 | INFO | Analyzing chunk frames = [441856,442368) | 20260505 04:45:28.565 | INFO | MemInfo: { | 20260505 04:45:28.565 | INFO | "gpu" : " 17 GB", | 20260505 04:45:28.565 | INFO | "huge" : "", | 20260505 04:45:28.565 | INFO | "huge_shared" : " 27 GB", | 20260505 04:45:28.565 | INFO | "malloc" : " 14 GB", | 20260505 04:45:28.565 | INFO | "pinned" : " 29 GB" | 20260505 04:45:28.565 | INFO | } | 20260505 04:45:28.566 | INFO | Perf report for the last 5.1394 seconds: | 20260505 04:45:28.566 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:45:28.566 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:28.566 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:28.566 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:28.566 | INFO | ANALYSIS : 57.6%, 3.0, 89.9ms, 0 | 20260505 04:45:28.566 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:28.566 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:28.566 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 04:45:28.566 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:28.566 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:28.566 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 04:45:28.566 | INFO | Total mem usage: 15.1 GiB | 20260505 04:45:28.566 | INFO | All Time High Water Marks sum: 15.7 GiB (Savings: 4.09%) | 20260505 04:45:28.566 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -4.09%) | 20260505 04:45:28.566 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 04:45:28.566 | INFO | High Water Breakdown: | 20260505 04:45:28.566 | INFO | 59.4% : 8.59 GiB ( 95.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 04:45:28.566 | INFO | 40.5% : 5.86 GiB ( 88.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 04:45:28.566 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 04:45:28.566 | INFO | ------------------------------------------ | 20260505 04:45:28.566 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:45:28.566 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 04:45:28.566 | INFO | Total mem usage: 26.5 GiB | 20260505 04:45:28.566 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260505 04:45:28.566 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 04:45:28.566 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260505 04:45:28.566 | INFO | High Water Breakdown: | 20260505 04:45:28.566 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:45:28.566 | INFO | Uploaded 36GiB to GPU in 1.60049s (22.4931GiB/s) | 20260505 04:45:28.566 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:45:28.566 | INFO | Downloaded 34GiB to GPU in 1.51795s (22.3986GiB/s) | 20260505 04:45:28.566 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:45:28.566 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:45:28.566 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:45:28.566 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:45:28.566 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 04:45:28.566 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 04:45:28.566 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:45:28.566 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 04:45:28.566 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 04:45:28.566 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 04:45:28.566 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 04:45:28.566 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 04:45:28.566 | INFO | ------------------------------------------ | 20260505 04:45:28.566 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:45:28.566 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 04:45:28.567 | INFO | Total mem usage: 25.6 GiB | 20260505 04:45:28.567 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 04:45:28.567 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 04:45:28.567 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 04:45:28.567 | INFO | High Water Breakdown: | 20260505 04:45:28.567 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 04:45:28.567 | INFO | ------------------------------------------ | 20260505 04:45:28.567 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:45:28.567 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 04:45:28.567 | INFO | Total mem usage: 15.7 GiB | 20260505 04:45:28.567 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260505 04:45:28.567 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 04:45:28.567 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260505 04:45:28.567 | INFO | High Water Breakdown: | 20260505 04:45:28.567 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 04:45:28.567 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 04:45:28.567 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 04:45:28.567 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 04:45:28.567 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 04:45:28.567 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 04:45:28.567 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 04:45:28.567 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 04:45:28.567 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 04:45:28.567 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:45:28.567 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 04:45:28.567 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:45:28.567 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 04:45:28.567 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 04:45:28.567 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 04:45:28.567 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 04:45:28.567 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:45:28.567 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 04:45:28.567 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:45:28.567 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:45:28.567 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:45:28.567 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 04:45:28.567 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 04:45:28.567 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:45:28.567 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 04:45:28.567 | INFO | ------------------------------------------ | 20260505 04:45:28.567 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:45:28.567 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 04:45:28.567 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:45:28.567 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:45:28.567 | INFO | PerfReport: Full Execution -- 3.04e+01 / -nan, 1.29e+01 / 0.00e+00 | 20260505 04:45:28.567 | INFO | PerfReport: 0.28%, AggregateStats -- 8.63e-02 / -nan, 4.40e-03 / 0.00e+00 | 20260505 04:45:28.567 | INFO | PerfReport: 10.83%, Baseline -- 3.30e+00 / -nan, 1.76e-02 / 0.00e+00 | 20260505 04:45:28.567 | INFO | PerfReport: 3.04%, Histogram -- 9.24e-01 / -nan, 5.25e-02 / 0.00e+00 | 20260505 04:45:28.567 | INFO | PerfReport: 16.03%, DME -- 3.91e+01 / -nan, 3.26e+00 / 0.00e+00 | 20260505 04:45:28.567 | INFO | PerfReport: 56.54%, FrameLabeling -- 1.72e+01 / -nan, 4.54e-02 / 0.00e+00 | 20260505 04:45:28.567 | INFO | PerfReport: 7.64%, PulseAccumulating -- 2.33e+00 / -nan, 1.54e-01 / 0.00e+00 | 20260505 04:45:28.567 | INFO | PerfReport: 3.99%, Metrics -- 1.22e+00 / -nan, 2.00e-01 / 0.00e+00 | 20260505 04:45:28.567 | INFO | PerfReport: 1.65%, PulseSerializing -- 5.03e-01 / -nan, 8.09e-02 / 0.00e+00 | 20260505 04:45:28.567 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 04:45:28.567 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:45:28.567 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:45:28.567 | INFO | PerfReport: Full Execution -- 9.14e+01 / -nan, 1.91e+01 / 0.00e+00 | 20260505 04:45:28.567 | INFO | PerfReport: 18.30%, Upload -- 1.67e+01 / 0.00e+00, 3.74e-01 / 0.00e+00 | 20260505 04:45:28.567 | INFO | PerfReport: 17.55%, Download -- 1.60e+01 / 0.00e+00, 6.90e-01 / 0.00e+00 | 20260505 04:45:28.567 | INFO | PerfReport: 33.30%, Compute -- 3.05e+01 / 0.00e+00, 1.29e+01 / 0.00e+00 | 20260505 04:45:28.567 | INFO | PerfReport: 0.55%, LockUpload -- 5.03e-01 / 0.00e+00, 3.60e+00 / 0.00e+00 | 20260505 04:45:28.567 | INFO | PerfReport: 0.00%, LockDownload -- 3.92e-04 / 0.00e+00, 1.89e-04 / 0.00e+00 | 20260505 04:45:28.567 | INFO | PerfReport: 30.28%, LockCompute -- 2.77e+01 / 0.00e+00, 1.73e+01 / 0.00e+00 | 20260505 04:45:28.567 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.26e-02 / 0.00e+00 | 20260505 04:45:28.663 | INFO | Starting a round of Baz IO | 20260505 04:45:31.619 | INFO | Populating RTMetrics report | 20260505 04:45:31.620 | INFO | RTMetrics report populated | 20260505 04:45:31.620 | INFO | Opening tmp RTMetrics json file | 20260505 04:45:31.622 | INFO | Serializing RTMetrics to json | 20260505 04:45:31.624 | INFO | Serialization done, swapping tmp file now... | 20260505 04:45:31.625 | INFO | RTMetrics done | 20260505 04:45:31.710 | INFO | Finished a round of Baz IO: Wrote 2.056 GiB in 3.046 seconds | 20260505 04:45:31.710 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.83% / 0.12% / 55.04% | 20260505 04:45:31.710 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.88127% / 0.000282129% / 3.97364e-05% | 20260505 04:45:31.710 | INFO | Baz IO overall: | 20260505 04:45:31.710 | INFO | BytesWritten : 183324733024 | 20260505 04:45:31.710 | INFO | NumEvents : 85247340578 | 20260505 04:45:33.705 | INFO | Analyzing chunk frames = [442368,442880) | 20260505 04:45:33.705 | INFO | MemInfo: { | 20260505 04:45:33.705 | INFO | "gpu" : " 17 GB", | 20260505 04:45:33.705 | INFO | "huge" : "", | 20260505 04:45:33.705 | INFO | "huge_shared" : " 27 GB", | 20260505 04:45:33.705 | INFO | "malloc" : " 14 GB", | 20260505 04:45:33.705 | INFO | "pinned" : " 29 GB" | 20260505 04:45:33.705 | INFO | } | 20260505 04:45:33.705 | INFO | Perf report for the last 5.13946 seconds: | 20260505 04:45:33.705 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:45:33.705 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:33.705 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:33.705 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:33.705 | INFO | ANALYSIS : 58.3%, 3.0, 90.6ms, 0 | 20260505 04:45:33.705 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:33.705 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260505 04:45:33.705 | INFO | PRE_HQ : 57.6%, 2.2, 64.8ms, 0 | 20260505 04:45:33.705 | INFO | BAZWRITER : 58.9%, 4.0, 121.5ms, 0 | 20260505 04:45:33.705 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 04:45:33.705 | INFO | Uploaded 36GiB to GPU in 1.6348s (22.021GiB/s) | 20260505 04:45:33.705 | INFO | Downloaded 38GiB to GPU in 1.69685s (22.3945GiB/s) | 20260505 04:45:38.844 | INFO | Analyzing chunk frames = [442880,443392) | 20260505 04:45:38.844 | INFO | MemInfo: { | 20260505 04:45:38.844 | INFO | "gpu" : " 17 GB", | 20260505 04:45:38.844 | INFO | "huge" : "", | 20260505 04:45:38.844 | INFO | "huge_shared" : " 27 GB", | 20260505 04:45:38.844 | INFO | "malloc" : " 14 GB", | 20260505 04:45:38.844 | INFO | "pinned" : " 29 GB" | 20260505 04:45:38.844 | INFO | } | 20260505 04:45:38.844 | INFO | Perf report for the last 5.13923 seconds: | 20260505 04:45:38.844 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:45:38.844 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:38.844 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:38.844 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:38.844 | INFO | ANALYSIS : 58.0%, 3.0, 90.4ms, 0 | 20260505 04:45:38.844 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:38.844 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:38.844 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260505 04:45:38.844 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:38.844 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:38.845 | INFO | Uploaded 36GiB to GPU in 1.5992s (22.5112GiB/s) | 20260505 04:45:38.845 | INFO | Downloaded 34GiB to GPU in 1.51566s (22.4325GiB/s) | 20260505 04:45:43.984 | INFO | Analyzing chunk frames = [443392,443904) | 20260505 04:45:43.984 | INFO | MemInfo: { | 20260505 04:45:43.984 | INFO | "gpu" : " 17 GB", | 20260505 04:45:43.984 | INFO | "huge" : "", | 20260505 04:45:43.984 | INFO | "huge_shared" : " 27 GB", | 20260505 04:45:43.984 | INFO | "malloc" : " 14 GB", | 20260505 04:45:43.984 | INFO | "pinned" : " 29 GB" | 20260505 04:45:43.984 | INFO | } | 20260505 04:45:43.984 | INFO | Perf report for the last 5.13969 seconds: | 20260505 04:45:43.984 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:45:43.984 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:43.984 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:43.984 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:43.984 | INFO | ANALYSIS : 57.4%, 3.0, 89.2ms, 0 | 20260505 04:45:43.984 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:43.984 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:43.984 | INFO | PRE_HQ : 31.7%, 1.0, 16.4ms, 0 | 20260505 04:45:43.984 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:43.984 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:43.984 | INFO | Uploaded 36GiB to GPU in 1.60259s (22.4636GiB/s) | 20260505 04:45:43.984 | INFO | Downloaded 34GiB to GPU in 1.51915s (22.3809GiB/s) | 20260505 04:45:49.123 | INFO | Analyzing chunk frames = [443904,444416) | 20260505 04:45:49.123 | INFO | MemInfo: { | 20260505 04:45:49.123 | INFO | "gpu" : " 17 GB", | 20260505 04:45:49.123 | INFO | "huge" : "", | 20260505 04:45:49.123 | INFO | "huge_shared" : " 27 GB", | 20260505 04:45:49.123 | INFO | "malloc" : " 14 GB", | 20260505 04:45:49.123 | INFO | "pinned" : " 29 GB" | 20260505 04:45:49.123 | INFO | } | 20260505 04:45:49.123 | INFO | Perf report for the last 5.13892 seconds: | 20260505 04:45:49.123 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:45:49.123 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:49.123 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:49.123 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:49.123 | INFO | ANALYSIS : 57.0%, 3.0, 88.5ms, 0 | 20260505 04:45:49.123 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:49.123 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:49.123 | INFO | PRE_HQ : 31.9%, 1.0, 16.5ms, 0 | 20260505 04:45:49.123 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:49.123 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:49.124 | INFO | Uploaded 36GiB to GPU in 1.60244s (22.4658GiB/s) | 20260505 04:45:49.124 | INFO | Downloaded 34GiB to GPU in 1.51933s (22.3783GiB/s) | 20260505 04:45:54.262 | INFO | Analyzing chunk frames = [444416,444928) | 20260505 04:45:54.262 | INFO | MemInfo: { | 20260505 04:45:54.262 | INFO | "gpu" : " 17 GB", | 20260505 04:45:54.262 | INFO | "huge" : "", | 20260505 04:45:54.262 | INFO | "huge_shared" : " 27 GB", | 20260505 04:45:54.262 | INFO | "malloc" : " 14 GB", | 20260505 04:45:54.262 | INFO | "pinned" : " 29 GB" | 20260505 04:45:54.262 | INFO | } | 20260505 04:45:54.262 | INFO | Perf report for the last 5.13935 seconds: | 20260505 04:45:54.262 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:45:54.262 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:54.262 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:54.262 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:54.262 | INFO | ANALYSIS : 57.0%, 3.0, 89.0ms, 0 | 20260505 04:45:54.262 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:54.262 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:54.262 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260505 04:45:54.262 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:54.262 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:54.262 | INFO | Uploaded 36GiB to GPU in 1.60023s (22.4967GiB/s) | 20260505 04:45:54.263 | INFO | Downloaded 34GiB to GPU in 1.51654s (22.4195GiB/s) | 20260505 04:45:59.402 | INFO | Analyzing chunk frames = [444928,445440) | 20260505 04:45:59.402 | INFO | MemInfo: { | 20260505 04:45:59.402 | INFO | "gpu" : " 17 GB", | 20260505 04:45:59.402 | INFO | "huge" : "", | 20260505 04:45:59.402 | INFO | "huge_shared" : " 27 GB", | 20260505 04:45:59.402 | INFO | "malloc" : " 14 GB", | 20260505 04:45:59.402 | INFO | "pinned" : " 29 GB" | 20260505 04:45:59.402 | INFO | } | 20260505 04:45:59.402 | INFO | Perf report for the last 5.1398 seconds: | 20260505 04:45:59.402 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:45:59.402 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:59.402 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:59.402 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:59.402 | INFO | ANALYSIS : 57.2%, 3.0, 89.3ms, 0 | 20260505 04:45:59.402 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:59.402 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:59.402 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260505 04:45:59.402 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:59.402 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:45:59.402 | INFO | Uploaded 36GiB to GPU in 1.60092s (22.4871GiB/s) | 20260505 04:45:59.402 | INFO | Downloaded 34GiB to GPU in 1.51766s (22.4029GiB/s) | 20260505 04:46:04.541 | INFO | Analyzing chunk frames = [445440,445952) | 20260505 04:46:04.541 | INFO | MemInfo: { | 20260505 04:46:04.541 | INFO | "gpu" : " 17 GB", | 20260505 04:46:04.541 | INFO | "huge" : "", | 20260505 04:46:04.541 | INFO | "huge_shared" : " 27 GB", | 20260505 04:46:04.541 | INFO | "malloc" : " 14 GB", | 20260505 04:46:04.541 | INFO | "pinned" : " 29 GB" | 20260505 04:46:04.541 | INFO | } | 20260505 04:46:04.541 | INFO | Perf report for the last 5.13894 seconds: | 20260505 04:46:04.541 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:46:04.541 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:04.541 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:04.541 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:04.541 | INFO | ANALYSIS : 57.5%, 3.0, 89.8ms, 0 | 20260505 04:46:04.541 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:04.541 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:04.541 | INFO | PRE_HQ : 32.6%, 1.0, 16.9ms, 0 | 20260505 04:46:04.541 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:04.541 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:04.541 | INFO | Uploaded 36GiB to GPU in 1.60113s (22.4841GiB/s) | 20260505 04:46:04.541 | INFO | Downloaded 34GiB to GPU in 1.51777s (22.4013GiB/s) | 20260505 04:46:09.680 | INFO | Analyzing chunk frames = [445952,446464) | 20260505 04:46:09.680 | INFO | MemInfo: { | 20260505 04:46:09.680 | INFO | "gpu" : " 17 GB", | 20260505 04:46:09.680 | INFO | "huge" : "", | 20260505 04:46:09.680 | INFO | "huge_shared" : " 27 GB", | 20260505 04:46:09.680 | INFO | "malloc" : " 14 GB", | 20260505 04:46:09.680 | INFO | "pinned" : " 29 GB" | 20260505 04:46:09.680 | INFO | } | 20260505 04:46:09.680 | INFO | Perf report for the last 5.13932 seconds: | 20260505 04:46:09.680 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:46:09.680 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:09.680 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:09.680 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:09.680 | INFO | ANALYSIS : 57.6%, 3.0, 89.8ms, 0 | 20260505 04:46:09.680 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:09.680 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:09.680 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 04:46:09.680 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:09.680 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:09.681 | INFO | Uploaded 36GiB to GPU in 1.60091s (22.4873GiB/s) | 20260505 04:46:09.681 | INFO | Downloaded 34GiB to GPU in 1.51917s (22.3806GiB/s) | 20260505 04:46:09.778 | INFO | Starting a round of Baz IO | 20260505 04:46:12.732 | INFO | Populating RTMetrics report | 20260505 04:46:12.733 | INFO | RTMetrics report populated | 20260505 04:46:12.733 | INFO | Opening tmp RTMetrics json file | 20260505 04:46:12.733 | INFO | Serializing RTMetrics to json | 20260505 04:46:12.734 | INFO | Serialization done, swapping tmp file now... | 20260505 04:46:12.735 | INFO | RTMetrics done | 20260505 04:46:12.823 | INFO | Finished a round of Baz IO: Wrote 2.061 GiB in 3.044 seconds | 20260505 04:46:12.823 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.71% / 0.12% / 55.17% | 20260505 04:46:12.823 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.80879% / 0.000301997% / 3.57628e-05% | 20260505 04:46:12.823 | INFO | Baz IO overall: | 20260505 04:46:12.823 | INFO | BytesWritten : 185538093932 | 20260505 04:46:12.823 | INFO | NumEvents : 86296926675 | 20260505 04:46:14.820 | INFO | Analyzing chunk frames = [446464,446976) | 20260505 04:46:14.820 | INFO | MemInfo: { | 20260505 04:46:14.820 | INFO | "gpu" : " 17 GB", | 20260505 04:46:14.820 | INFO | "huge" : "", | 20260505 04:46:14.820 | INFO | "huge_shared" : " 27 GB", | 20260505 04:46:14.820 | INFO | "malloc" : " 14 GB", | 20260505 04:46:14.820 | INFO | "pinned" : " 29 GB" | 20260505 04:46:14.820 | INFO | } | 20260505 04:46:14.820 | INFO | Perf report for the last 5.13942 seconds: | 20260505 04:46:14.820 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:46:14.820 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:14.820 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:14.820 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:14.820 | INFO | ANALYSIS : 58.3%, 3.0, 90.6ms, 0 | 20260505 04:46:14.820 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:14.820 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260505 04:46:14.820 | INFO | PRE_HQ : 57.7%, 2.2, 64.7ms, 0 | 20260505 04:46:14.820 | INFO | BAZWRITER : 58.8%, 4.0, 121.1ms, 0 | 20260505 04:46:14.820 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:46:14.820 | INFO | Uploaded 36GiB to GPU in 1.63753s (21.9843GiB/s) | 20260505 04:46:14.820 | INFO | Downloaded 38GiB to GPU in 1.69871s (22.37GiB/s) | 20260505 04:46:19.959 | INFO | Analyzing chunk frames = [446976,447488) | 20260505 04:46:19.959 | INFO | MemInfo: { | 20260505 04:46:19.959 | INFO | "gpu" : " 17 GB", | 20260505 04:46:19.959 | INFO | "huge" : "", | 20260505 04:46:19.959 | INFO | "huge_shared" : " 27 GB", | 20260505 04:46:19.959 | INFO | "malloc" : " 14 GB", | 20260505 04:46:19.959 | INFO | "pinned" : " 29 GB" | 20260505 04:46:19.959 | INFO | } | 20260505 04:46:19.959 | INFO | Perf report for the last 5.1393 seconds: | 20260505 04:46:19.959 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:46:19.959 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:19.959 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:19.959 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:19.959 | INFO | ANALYSIS : 58.0%, 3.0, 90.5ms, 0 | 20260505 04:46:19.959 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:19.959 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:19.959 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260505 04:46:19.959 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:19.959 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:19.959 | INFO | Uploaded 36GiB to GPU in 1.59924s (22.5107GiB/s) | 20260505 04:46:19.959 | INFO | Downloaded 34GiB to GPU in 1.51527s (22.4382GiB/s) | 20260505 04:46:25.098 | INFO | Analyzing chunk frames = [447488,448000) | 20260505 04:46:25.098 | INFO | MemInfo: { | 20260505 04:46:25.098 | INFO | "gpu" : " 17 GB", | 20260505 04:46:25.098 | INFO | "huge" : "", | 20260505 04:46:25.098 | INFO | "huge_shared" : " 27 GB", | 20260505 04:46:25.098 | INFO | "malloc" : " 14 GB", | 20260505 04:46:25.098 | INFO | "pinned" : " 29 GB" | 20260505 04:46:25.098 | INFO | } | 20260505 04:46:25.098 | INFO | Perf report for the last 5.13932 seconds: | 20260505 04:46:25.098 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:46:25.098 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:25.098 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:25.098 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:25.098 | INFO | ANALYSIS : 57.4%, 3.0, 89.2ms, 0 | 20260505 04:46:25.098 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:25.099 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:25.099 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260505 04:46:25.099 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:25.099 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:25.099 | INFO | Uploaded 36GiB to GPU in 1.6023s (22.4677GiB/s) | 20260505 04:46:25.099 | INFO | Downloaded 34GiB to GPU in 1.51945s (22.3766GiB/s) | 20260505 04:46:30.238 | INFO | Analyzing chunk frames = [448000,448512) | 20260505 04:46:30.238 | INFO | MemInfo: { | 20260505 04:46:30.238 | INFO | "gpu" : " 17 GB", | 20260505 04:46:30.238 | INFO | "huge" : "", | 20260505 04:46:30.238 | INFO | "huge_shared" : " 27 GB", | 20260505 04:46:30.238 | INFO | "malloc" : " 14 GB", | 20260505 04:46:30.238 | INFO | "pinned" : " 29 GB" | 20260505 04:46:30.238 | INFO | } | 20260505 04:46:30.238 | INFO | Perf report for the last 5.13934 seconds: | 20260505 04:46:30.238 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:46:30.238 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:30.238 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:30.238 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:30.238 | INFO | ANALYSIS : 57.1%, 3.0, 88.5ms, 0 | 20260505 04:46:30.238 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:30.238 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:30.238 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 04:46:30.238 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:30.238 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:30.238 | INFO | Uploaded 36GiB to GPU in 1.60194s (22.4727GiB/s) | 20260505 04:46:30.238 | INFO | Downloaded 34GiB to GPU in 1.51902s (22.3829GiB/s) | 20260505 04:46:35.377 | INFO | Analyzing chunk frames = [448512,449024) | 20260505 04:46:35.377 | INFO | MemInfo: { | 20260505 04:46:35.377 | INFO | "gpu" : " 17 GB", | 20260505 04:46:35.377 | INFO | "huge" : "", | 20260505 04:46:35.377 | INFO | "huge_shared" : " 27 GB", | 20260505 04:46:35.377 | INFO | "malloc" : " 14 GB", | 20260505 04:46:35.377 | INFO | "pinned" : " 29 GB" | 20260505 04:46:35.377 | INFO | } | 20260505 04:46:35.377 | INFO | Perf report for the last 5.13941 seconds: | 20260505 04:46:35.377 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:46:35.377 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:35.377 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:35.377 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:35.377 | INFO | ANALYSIS : 57.0%, 3.0, 88.8ms, 0 | 20260505 04:46:35.377 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:35.377 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:35.377 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260505 04:46:35.377 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:35.377 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:35.378 | INFO | Uploaded 36GiB to GPU in 1.60063s (22.4912GiB/s) | 20260505 04:46:35.378 | INFO | Downloaded 34GiB to GPU in 1.5166s (22.4186GiB/s) | 20260505 04:46:40.516 | INFO | Analyzing chunk frames = [449024,449536) | 20260505 04:46:40.516 | INFO | MemInfo: { | 20260505 04:46:40.516 | INFO | "gpu" : " 17 GB", | 20260505 04:46:40.516 | INFO | "huge" : "", | 20260505 04:46:40.516 | INFO | "huge_shared" : " 27 GB", | 20260505 04:46:40.516 | INFO | "malloc" : " 14 GB", | 20260505 04:46:40.516 | INFO | "pinned" : " 29 GB" | 20260505 04:46:40.516 | INFO | } | 20260505 04:46:40.516 | INFO | Perf report for the last 5.13915 seconds: | 20260505 04:46:40.516 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:46:40.516 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:40.516 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:40.516 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:40.516 | INFO | ANALYSIS : 57.3%, 3.0, 89.5ms, 0 | 20260505 04:46:40.516 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:40.516 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:40.516 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 04:46:40.516 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:40.516 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:40.517 | INFO | Uploaded 36GiB to GPU in 1.60083s (22.4883GiB/s) | 20260505 04:46:40.517 | INFO | Downloaded 34GiB to GPU in 1.51712s (22.4109GiB/s) | 20260505 04:46:45.656 | INFO | Analyzing chunk frames = [449536,450048) | 20260505 04:46:45.656 | INFO | MemInfo: { | 20260505 04:46:45.656 | INFO | "gpu" : " 17 GB", | 20260505 04:46:45.656 | INFO | "huge" : "", | 20260505 04:46:45.656 | INFO | "huge_shared" : " 27 GB", | 20260505 04:46:45.656 | INFO | "malloc" : " 14 GB", | 20260505 04:46:45.656 | INFO | "pinned" : " 29 GB" | 20260505 04:46:45.656 | INFO | } | 20260505 04:46:45.656 | INFO | Perf report for the last 5.1394 seconds: | 20260505 04:46:45.656 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:46:45.656 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:45.656 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:45.656 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:45.656 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260505 04:46:45.656 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:45.656 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:45.656 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 04:46:45.656 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:45.656 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:45.656 | INFO | Uploaded 36GiB to GPU in 1.60125s (22.4824GiB/s) | 20260505 04:46:45.656 | INFO | Downloaded 34GiB to GPU in 1.51805s (22.3971GiB/s) | 20260505 04:46:50.795 | INFO | Analyzing chunk frames = [450048,450560) | 20260505 04:46:50.795 | INFO | MemInfo: { | 20260505 04:46:50.795 | INFO | "gpu" : " 17 GB", | 20260505 04:46:50.795 | INFO | "huge" : "", | 20260505 04:46:50.795 | INFO | "huge_shared" : " 27 GB", | 20260505 04:46:50.795 | INFO | "malloc" : " 14 GB", | 20260505 04:46:50.795 | INFO | "pinned" : " 29 GB" | 20260505 04:46:50.795 | INFO | } | 20260505 04:46:50.795 | INFO | Perf report for the last 5.13945 seconds: | 20260505 04:46:50.795 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:46:50.795 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:50.795 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:50.795 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:50.795 | INFO | ANALYSIS : 57.6%, 3.0, 89.9ms, 0 | 20260505 04:46:50.795 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:50.795 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:50.795 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 04:46:50.795 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:50.795 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:50.796 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 04:46:50.796 | INFO | Total mem usage: 15.1 GiB | 20260505 04:46:50.796 | INFO | All Time High Water Marks sum: 15.7 GiB (Savings: 4.09%) | 20260505 04:46:50.796 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -4.14%) | 20260505 04:46:50.796 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 04:46:50.796 | INFO | High Water Breakdown: | 20260505 04:46:50.796 | INFO | 59.4% : 8.59 GiB ( 95.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 04:46:50.796 | INFO | 40.5% : 5.85 GiB ( 88.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 04:46:50.796 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 04:46:50.796 | INFO | Uploaded 36GiB to GPU in 1.60105s (22.4852GiB/s) | 20260505 04:46:50.796 | INFO | Downloaded 34GiB to GPU in 1.51829s (22.3936GiB/s) | 20260505 04:46:50.796 | INFO | ------------------------------------------ | 20260505 04:46:50.796 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:46:50.796 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 04:46:50.796 | INFO | Total mem usage: 26.5 GiB | 20260505 04:46:50.796 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260505 04:46:50.796 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 04:46:50.796 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260505 04:46:50.796 | INFO | High Water Breakdown: | 20260505 04:46:50.796 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:46:50.796 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:46:50.796 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:46:50.796 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:46:50.796 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:46:50.796 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:46:50.796 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 04:46:50.796 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 04:46:50.796 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:46:50.796 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 04:46:50.796 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 04:46:50.796 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 04:46:50.796 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 04:46:50.796 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 04:46:50.796 | INFO | ------------------------------------------ | 20260505 04:46:50.796 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:46:50.796 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 04:46:50.796 | INFO | Total mem usage: 25.6 GiB | 20260505 04:46:50.796 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 04:46:50.796 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 04:46:50.796 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 04:46:50.796 | INFO | High Water Breakdown: | 20260505 04:46:50.796 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 04:46:50.796 | INFO | ------------------------------------------ | 20260505 04:46:50.796 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:46:50.796 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 04:46:50.796 | INFO | Total mem usage: 15.7 GiB | 20260505 04:46:50.796 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260505 04:46:50.796 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 04:46:50.796 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260505 04:46:50.796 | INFO | High Water Breakdown: | 20260505 04:46:50.796 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 04:46:50.796 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 04:46:50.796 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 04:46:50.796 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 04:46:50.796 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 04:46:50.796 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 04:46:50.796 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 04:46:50.796 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 04:46:50.796 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 04:46:50.796 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:46:50.796 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 04:46:50.797 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:46:50.797 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 04:46:50.797 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 04:46:50.797 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 04:46:50.797 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 04:46:50.797 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:46:50.797 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 04:46:50.797 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:46:50.797 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:46:50.797 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:46:50.797 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 04:46:50.797 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 04:46:50.797 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:46:50.797 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 04:46:50.797 | INFO | ------------------------------------------ | 20260505 04:46:50.797 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:46:50.797 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 04:46:50.797 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:46:50.797 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:46:50.797 | INFO | PerfReport: Full Execution -- 3.04e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260505 04:46:50.797 | INFO | PerfReport: 0.28%, AggregateStats -- 8.66e-02 / -nan, 5.19e-03 / 0.00e+00 | 20260505 04:46:50.797 | INFO | PerfReport: 10.83%, Baseline -- 3.30e+00 / -nan, 1.80e-02 / 0.00e+00 | 20260505 04:46:50.797 | INFO | PerfReport: 3.04%, Histogram -- 9.24e-01 / -nan, 5.21e-02 / 0.00e+00 | 20260505 04:46:50.797 | INFO | PerfReport: 16.03%, DME -- 3.90e+01 / -nan, 3.24e+00 / 0.00e+00 | 20260505 04:46:50.797 | INFO | PerfReport: 56.54%, FrameLabeling -- 1.72e+01 / -nan, 4.58e-02 / 0.00e+00 | 20260505 04:46:50.797 | INFO | PerfReport: 7.64%, PulseAccumulating -- 2.33e+00 / -nan, 1.55e-01 / 0.00e+00 | 20260505 04:46:50.797 | INFO | PerfReport: 3.99%, Metrics -- 1.22e+00 / -nan, 2.00e-01 / 0.00e+00 | 20260505 04:46:50.797 | INFO | PerfReport: 1.65%, PulseSerializing -- 5.02e-01 / -nan, 8.09e-02 / 0.00e+00 | 20260505 04:46:50.797 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 04:46:50.797 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:46:50.797 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:46:50.797 | INFO | PerfReport: Full Execution -- 9.14e+01 / -nan, 1.91e+01 / 0.00e+00 | 20260505 04:46:50.797 | INFO | PerfReport: 18.29%, Upload -- 1.67e+01 / 0.00e+00, 3.65e-01 / 0.00e+00 | 20260505 04:46:50.797 | INFO | PerfReport: 17.55%, Download -- 1.60e+01 / 0.00e+00, 6.77e-01 / 0.00e+00 | 20260505 04:46:50.797 | INFO | PerfReport: 33.30%, Compute -- 3.05e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260505 04:46:50.797 | INFO | PerfReport: 0.55%, LockUpload -- 5.04e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260505 04:46:50.797 | INFO | PerfReport: 0.00%, LockDownload -- 3.96e-04 / 0.00e+00, 1.01e-04 / 0.00e+00 | 20260505 04:46:50.797 | INFO | PerfReport: 30.29%, LockCompute -- 2.77e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260505 04:46:50.797 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.70e-02 / 0.00e+00 | 20260505 04:46:50.893 | INFO | Starting a round of Baz IO | 20260505 04:46:53.843 | INFO | Populating RTMetrics report | 20260505 04:46:53.844 | INFO | RTMetrics report populated | 20260505 04:46:53.844 | INFO | Opening tmp RTMetrics json file | 20260505 04:46:53.847 | INFO | Serializing RTMetrics to json | 20260505 04:46:53.848 | INFO | Serialization done, swapping tmp file now... | 20260505 04:46:53.849 | INFO | RTMetrics done | 20260505 04:46:53.937 | INFO | Finished a round of Baz IO: Wrote 2.064 GiB in 3.043 seconds | 20260505 04:46:53.937 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.59% / 0.12% / 55.29% | 20260505 04:46:53.937 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.78126% / 0.00029405% / 5.96046e-05% | 20260505 04:46:53.937 | INFO | Baz IO overall: | 20260505 04:46:53.937 | INFO | BytesWritten : 187754427146 | 20260505 04:46:53.937 | INFO | NumEvents : 87348320507 | 20260505 04:46:55.934 | INFO | Analyzing chunk frames = [450560,451072) | 20260505 04:46:55.934 | INFO | MemInfo: { | 20260505 04:46:55.934 | INFO | "gpu" : " 17 GB", | 20260505 04:46:55.934 | INFO | "huge" : "", | 20260505 04:46:55.934 | INFO | "huge_shared" : " 27 GB", | 20260505 04:46:55.934 | INFO | "malloc" : " 14 GB", | 20260505 04:46:55.934 | INFO | "pinned" : " 29 GB" | 20260505 04:46:55.934 | INFO | } | 20260505 04:46:55.934 | INFO | Perf report for the last 5.13918 seconds: | 20260505 04:46:55.934 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:46:55.934 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:55.934 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:55.934 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:55.934 | INFO | ANALYSIS : 58.2%, 3.0, 90.5ms, 0 | 20260505 04:46:55.935 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:46:55.935 | INFO | RT_METRICS : 45.3%, 1.0, 23.8ms, 0 | 20260505 04:46:55.935 | INFO | PRE_HQ : 57.6%, 2.2, 64.4ms, 0 | 20260505 04:46:55.935 | INFO | BAZWRITER : 58.8%, 4.1, 122.4ms, 0 | 20260505 04:46:55.935 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 04:46:55.935 | INFO | Uploaded 36GiB to GPU in 1.63705s (21.9907GiB/s) | 20260505 04:46:55.935 | INFO | Downloaded 38GiB to GPU in 1.69913s (22.3644GiB/s) | 20260505 04:47:01.074 | INFO | Analyzing chunk frames = [451072,451584) | 20260505 04:47:01.074 | INFO | MemInfo: { | 20260505 04:47:01.074 | INFO | "gpu" : " 17 GB", | 20260505 04:47:01.074 | INFO | "huge" : "", | 20260505 04:47:01.074 | INFO | "huge_shared" : " 27 GB", | 20260505 04:47:01.074 | INFO | "malloc" : " 14 GB", | 20260505 04:47:01.074 | INFO | "pinned" : " 29 GB" | 20260505 04:47:01.074 | INFO | } | 20260505 04:47:01.074 | INFO | Perf report for the last 5.1395 seconds: | 20260505 04:47:01.074 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:47:01.074 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:01.074 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:01.074 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:01.074 | INFO | ANALYSIS : 58.0%, 3.0, 90.5ms, 0 | 20260505 04:47:01.074 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:01.074 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:01.074 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 04:47:01.074 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:01.074 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:01.074 | INFO | Uploaded 36GiB to GPU in 1.59905s (22.5134GiB/s) | 20260505 04:47:01.075 | INFO | Downloaded 34GiB to GPU in 1.51474s (22.4462GiB/s) | 20260505 04:47:06.213 | INFO | Analyzing chunk frames = [451584,452096) | 20260505 04:47:06.213 | INFO | MemInfo: { | 20260505 04:47:06.213 | INFO | "gpu" : " 17 GB", | 20260505 04:47:06.213 | INFO | "huge" : "", | 20260505 04:47:06.213 | INFO | "huge_shared" : " 27 GB", | 20260505 04:47:06.213 | INFO | "malloc" : " 14 GB", | 20260505 04:47:06.213 | INFO | "pinned" : " 29 GB" | 20260505 04:47:06.213 | INFO | } | 20260505 04:47:06.213 | INFO | Perf report for the last 5.13922 seconds: | 20260505 04:47:06.213 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:47:06.213 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:06.213 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:06.213 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:06.213 | INFO | ANALYSIS : 57.3%, 3.0, 89.1ms, 0 | 20260505 04:47:06.213 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:06.213 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:06.213 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260505 04:47:06.213 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:06.213 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:06.214 | INFO | Uploaded 36GiB to GPU in 1.60289s (22.4594GiB/s) | 20260505 04:47:06.214 | INFO | Downloaded 34GiB to GPU in 1.51964s (22.3737GiB/s) | 20260505 04:47:11.352 | INFO | Analyzing chunk frames = [452096,452608) | 20260505 04:47:11.352 | INFO | MemInfo: { | 20260505 04:47:11.352 | INFO | "gpu" : " 17 GB", | 20260505 04:47:11.352 | INFO | "huge" : "", | 20260505 04:47:11.352 | INFO | "huge_shared" : " 27 GB", | 20260505 04:47:11.352 | INFO | "malloc" : " 14 GB", | 20260505 04:47:11.352 | INFO | "pinned" : " 29 GB" | 20260505 04:47:11.352 | INFO | } | 20260505 04:47:11.352 | INFO | Perf report for the last 5.1393 seconds: | 20260505 04:47:11.352 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:47:11.352 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:11.352 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:11.353 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:11.353 | INFO | ANALYSIS : 57.1%, 3.0, 88.6ms, 0 | 20260505 04:47:11.353 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:11.353 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:11.353 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 04:47:11.353 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:11.353 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:11.353 | INFO | Uploaded 36GiB to GPU in 1.60138s (22.4806GiB/s) | 20260505 04:47:11.353 | INFO | Downloaded 34GiB to GPU in 1.51881s (22.386GiB/s) | 20260505 04:47:16.492 | INFO | Analyzing chunk frames = [452608,453120) | 20260505 04:47:16.492 | INFO | MemInfo: { | 20260505 04:47:16.492 | INFO | "gpu" : " 17 GB", | 20260505 04:47:16.492 | INFO | "huge" : "", | 20260505 04:47:16.492 | INFO | "huge_shared" : " 27 GB", | 20260505 04:47:16.492 | INFO | "malloc" : " 14 GB", | 20260505 04:47:16.492 | INFO | "pinned" : " 29 GB" | 20260505 04:47:16.492 | INFO | } | 20260505 04:47:16.492 | INFO | Perf report for the last 5.13942 seconds: | 20260505 04:47:16.492 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:47:16.492 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:16.492 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:16.492 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:16.492 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260505 04:47:16.492 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:16.492 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:16.492 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260505 04:47:16.492 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:16.492 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:16.492 | INFO | Uploaded 36GiB to GPU in 1.60066s (22.4907GiB/s) | 20260505 04:47:16.492 | INFO | Downloaded 34GiB to GPU in 1.51745s (22.406GiB/s) | 20260505 04:47:21.631 | INFO | Analyzing chunk frames = [453120,453632) | 20260505 04:47:21.631 | INFO | MemInfo: { | 20260505 04:47:21.631 | INFO | "gpu" : " 17 GB", | 20260505 04:47:21.631 | INFO | "huge" : "", | 20260505 04:47:21.631 | INFO | "huge_shared" : " 27 GB", | 20260505 04:47:21.631 | INFO | "malloc" : " 14 GB", | 20260505 04:47:21.631 | INFO | "pinned" : " 29 GB" | 20260505 04:47:21.631 | INFO | } | 20260505 04:47:21.631 | INFO | Perf report for the last 5.13928 seconds: | 20260505 04:47:21.631 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:47:21.631 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:21.631 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:21.631 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:21.631 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 04:47:21.631 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:21.631 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:21.631 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 04:47:21.631 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:21.631 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:21.632 | INFO | Uploaded 36GiB to GPU in 1.59932s (22.5095GiB/s) | 20260505 04:47:21.632 | INFO | Downloaded 34GiB to GPU in 1.51706s (22.4117GiB/s) | 20260505 04:47:26.770 | INFO | Analyzing chunk frames = [453632,454144) | 20260505 04:47:26.770 | INFO | MemInfo: { | 20260505 04:47:26.770 | INFO | "gpu" : " 17 GB", | 20260505 04:47:26.770 | INFO | "huge" : "", | 20260505 04:47:26.770 | INFO | "huge_shared" : " 27 GB", | 20260505 04:47:26.770 | INFO | "malloc" : " 14 GB", | 20260505 04:47:26.770 | INFO | "pinned" : " 29 GB" | 20260505 04:47:26.770 | INFO | } | 20260505 04:47:26.770 | INFO | Perf report for the last 5.1392 seconds: | 20260505 04:47:26.770 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:47:26.770 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:26.770 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:26.770 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:26.770 | INFO | ANALYSIS : 57.5%, 3.0, 89.6ms, 0 | 20260505 04:47:26.770 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:26.770 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:26.770 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 04:47:26.770 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:26.770 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:26.771 | INFO | Uploaded 36GiB to GPU in 1.60094s (22.4868GiB/s) | 20260505 04:47:26.771 | INFO | Downloaded 34GiB to GPU in 1.51854s (22.39GiB/s) | 20260505 04:47:31.910 | INFO | Analyzing chunk frames = [454144,454656) | 20260505 04:47:31.910 | INFO | MemInfo: { | 20260505 04:47:31.910 | INFO | "gpu" : " 17 GB", | 20260505 04:47:31.910 | INFO | "huge" : "", | 20260505 04:47:31.910 | INFO | "huge_shared" : " 27 GB", | 20260505 04:47:31.910 | INFO | "malloc" : " 14 GB", | 20260505 04:47:31.910 | INFO | "pinned" : " 29 GB" | 20260505 04:47:31.910 | INFO | } | 20260505 04:47:31.910 | INFO | Perf report for the last 5.13945 seconds: | 20260505 04:47:31.910 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:47:31.910 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:31.910 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:31.910 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:31.910 | INFO | ANALYSIS : 57.6%, 3.0, 89.9ms, 0 | 20260505 04:47:31.910 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:31.910 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:31.910 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 04:47:31.910 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:31.910 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:31.910 | INFO | Uploaded 36GiB to GPU in 1.60093s (22.4869GiB/s) | 20260505 04:47:31.910 | INFO | Downloaded 34GiB to GPU in 1.51801s (22.3978GiB/s) | 20260505 04:47:32.007 | INFO | Starting a round of Baz IO | 20260505 04:47:34.958 | INFO | Populating RTMetrics report | 20260505 04:47:34.959 | INFO | RTMetrics report populated | 20260505 04:47:34.959 | INFO | Opening tmp RTMetrics json file | 20260505 04:47:34.960 | INFO | Serializing RTMetrics to json | 20260505 04:47:34.962 | INFO | Serialization done, swapping tmp file now... | 20260505 04:47:34.965 | INFO | RTMetrics done | 20260505 04:47:35.056 | INFO | Finished a round of Baz IO: Wrote 2.068 GiB in 3.048 seconds | 20260505 04:47:35.056 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.47% / 0.12% / 55.41% | 20260505 04:47:35.056 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.75518% / 0.000274181% / 4.76837e-05% | 20260505 04:47:35.056 | INFO | Baz IO overall: | 20260505 04:47:35.056 | INFO | BytesWritten : 189974664265 | 20260505 04:47:35.056 | INFO | NumEvents : 88402610546 | 20260505 04:47:37.049 | INFO | Analyzing chunk frames = [454656,455168) | 20260505 04:47:37.049 | INFO | MemInfo: { | 20260505 04:47:37.049 | INFO | "gpu" : " 17 GB", | 20260505 04:47:37.049 | INFO | "huge" : "", | 20260505 04:47:37.049 | INFO | "huge_shared" : " 27 GB", | 20260505 04:47:37.049 | INFO | "malloc" : " 14 GB", | 20260505 04:47:37.049 | INFO | "pinned" : " 29 GB" | 20260505 04:47:37.049 | INFO | } | 20260505 04:47:37.049 | INFO | Perf report for the last 5.13929 seconds: | 20260505 04:47:37.049 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:47:37.049 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:37.049 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:37.049 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:37.049 | INFO | ANALYSIS : 58.2%, 3.0, 90.5ms, 0 | 20260505 04:47:37.049 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:37.049 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260505 04:47:37.049 | INFO | PRE_HQ : 57.5%, 2.2, 64.4ms, 0 | 20260505 04:47:37.049 | INFO | BAZWRITER : 58.9%, 4.0, 121.8ms, 0 | 20260505 04:47:37.049 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 04:47:37.050 | INFO | Uploaded 36GiB to GPU in 1.63567s (22.0093GiB/s) | 20260505 04:47:37.050 | INFO | Downloaded 38GiB to GPU in 1.69862s (22.3711GiB/s) | 20260505 04:47:42.188 | INFO | Analyzing chunk frames = [455168,455680) | 20260505 04:47:42.188 | INFO | MemInfo: { | 20260505 04:47:42.188 | INFO | "gpu" : " 17 GB", | 20260505 04:47:42.188 | INFO | "huge" : "", | 20260505 04:47:42.188 | INFO | "huge_shared" : " 27 GB", | 20260505 04:47:42.188 | INFO | "malloc" : " 14 GB", | 20260505 04:47:42.189 | INFO | "pinned" : " 29 GB" | 20260505 04:47:42.189 | INFO | } | 20260505 04:47:42.189 | INFO | Perf report for the last 5.13941 seconds: | 20260505 04:47:42.189 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:47:42.189 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:42.189 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:42.189 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:42.189 | INFO | ANALYSIS : 58.0%, 3.0, 90.5ms, 0 | 20260505 04:47:42.189 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:42.189 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:42.189 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260505 04:47:42.189 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:42.189 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:42.189 | INFO | Uploaded 36GiB to GPU in 1.59885s (22.5161GiB/s) | 20260505 04:47:42.189 | INFO | Downloaded 34GiB to GPU in 1.51524s (22.4386GiB/s) | 20260505 04:47:47.328 | INFO | Analyzing chunk frames = [455680,456192) | 20260505 04:47:47.328 | INFO | MemInfo: { | 20260505 04:47:47.328 | INFO | "gpu" : " 17 GB", | 20260505 04:47:47.328 | INFO | "huge" : "", | 20260505 04:47:47.328 | INFO | "huge_shared" : " 27 GB", | 20260505 04:47:47.328 | INFO | "malloc" : " 14 GB", | 20260505 04:47:47.328 | INFO | "pinned" : " 29 GB" | 20260505 04:47:47.328 | INFO | } | 20260505 04:47:47.328 | INFO | Perf report for the last 5.13939 seconds: | 20260505 04:47:47.328 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:47:47.328 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:47.328 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:47.328 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:47.328 | INFO | ANALYSIS : 57.3%, 3.0, 89.1ms, 0 | 20260505 04:47:47.328 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:47.328 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:47.328 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260505 04:47:47.328 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:47.328 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:47.328 | INFO | Uploaded 36GiB to GPU in 1.603s (22.4579GiB/s) | 20260505 04:47:47.328 | INFO | Downloaded 34GiB to GPU in 1.51978s (22.3717GiB/s) | 20260505 04:47:52.467 | INFO | Analyzing chunk frames = [456192,456704) | 20260505 04:47:52.467 | INFO | MemInfo: { | 20260505 04:47:52.467 | INFO | "gpu" : " 17 GB", | 20260505 04:47:52.467 | INFO | "huge" : "", | 20260505 04:47:52.467 | INFO | "huge_shared" : " 27 GB", | 20260505 04:47:52.467 | INFO | "malloc" : " 14 GB", | 20260505 04:47:52.467 | INFO | "pinned" : " 29 GB" | 20260505 04:47:52.467 | INFO | } | 20260505 04:47:52.467 | INFO | Perf report for the last 5.13928 seconds: | 20260505 04:47:52.467 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:47:52.467 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:52.467 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:52.467 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:52.467 | INFO | ANALYSIS : 57.1%, 3.0, 88.5ms, 0 | 20260505 04:47:52.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:52.467 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:52.467 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260505 04:47:52.467 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:52.467 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:52.468 | INFO | Uploaded 36GiB to GPU in 1.6022s (22.4691GiB/s) | 20260505 04:47:52.468 | INFO | Downloaded 34GiB to GPU in 1.51901s (22.383GiB/s) | 20260505 04:47:57.606 | INFO | Analyzing chunk frames = [456704,457216) | 20260505 04:47:57.606 | INFO | MemInfo: { | 20260505 04:47:57.606 | INFO | "gpu" : " 17 GB", | 20260505 04:47:57.606 | INFO | "huge" : "", | 20260505 04:47:57.606 | INFO | "huge_shared" : " 27 GB", | 20260505 04:47:57.606 | INFO | "malloc" : " 14 GB", | 20260505 04:47:57.607 | INFO | "pinned" : " 29 GB" | 20260505 04:47:57.607 | INFO | } | 20260505 04:47:57.607 | INFO | Perf report for the last 5.13928 seconds: | 20260505 04:47:57.607 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:47:57.607 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:57.607 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:57.607 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:57.607 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260505 04:47:57.607 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:57.607 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:57.607 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260505 04:47:57.607 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:57.607 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:47:57.607 | INFO | Uploaded 36GiB to GPU in 1.60063s (22.4912GiB/s) | 20260505 04:47:57.607 | INFO | Downloaded 34GiB to GPU in 1.51702s (22.4124GiB/s) | 20260505 04:48:02.746 | INFO | Analyzing chunk frames = [457216,457728) | 20260505 04:48:02.746 | INFO | MemInfo: { | 20260505 04:48:02.746 | INFO | "gpu" : " 17 GB", | 20260505 04:48:02.746 | INFO | "huge" : "", | 20260505 04:48:02.746 | INFO | "huge_shared" : " 27 GB", | 20260505 04:48:02.746 | INFO | "malloc" : " 14 GB", | 20260505 04:48:02.746 | INFO | "pinned" : " 29 GB" | 20260505 04:48:02.746 | INFO | } | 20260505 04:48:02.746 | INFO | Perf report for the last 5.1395 seconds: | 20260505 04:48:02.746 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:48:02.746 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:02.746 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:02.746 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:02.746 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 04:48:02.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:02.746 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:02.746 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260505 04:48:02.746 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:02.746 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:02.746 | INFO | Uploaded 36GiB to GPU in 1.60106s (22.485GiB/s) | 20260505 04:48:02.746 | INFO | Downloaded 34GiB to GPU in 1.51848s (22.3908GiB/s) | 20260505 04:48:07.885 | INFO | Analyzing chunk frames = [457728,458240) | 20260505 04:48:07.885 | INFO | MemInfo: { | 20260505 04:48:07.885 | INFO | "gpu" : " 17 GB", | 20260505 04:48:07.885 | INFO | "huge" : "", | 20260505 04:48:07.885 | INFO | "huge_shared" : " 27 GB", | 20260505 04:48:07.885 | INFO | "malloc" : " 14 GB", | 20260505 04:48:07.885 | INFO | "pinned" : " 29 GB" | 20260505 04:48:07.885 | INFO | } | 20260505 04:48:07.885 | INFO | Perf report for the last 5.13922 seconds: | 20260505 04:48:07.885 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:48:07.885 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:07.885 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:07.885 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:07.885 | INFO | ANALYSIS : 57.5%, 3.0, 89.6ms, 0 | 20260505 04:48:07.885 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:07.885 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:07.885 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 04:48:07.885 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:07.885 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:07.886 | INFO | Uploaded 36GiB to GPU in 1.60026s (22.4964GiB/s) | 20260505 04:48:07.886 | INFO | Downloaded 34GiB to GPU in 1.51878s (22.3865GiB/s) | 20260505 04:48:13.024 | INFO | Analyzing chunk frames = [458240,458752) | 20260505 04:48:13.024 | INFO | MemInfo: { | 20260505 04:48:13.025 | INFO | "gpu" : " 17 GB", | 20260505 04:48:13.025 | INFO | "huge" : "", | 20260505 04:48:13.025 | INFO | "huge_shared" : " 27 GB", | 20260505 04:48:13.025 | INFO | "malloc" : " 14 GB", | 20260505 04:48:13.025 | INFO | "pinned" : " 29 GB" | 20260505 04:48:13.025 | INFO | } | 20260505 04:48:13.025 | INFO | Perf report for the last 5.1394 seconds: | 20260505 04:48:13.025 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:48:13.025 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:13.025 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:13.025 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:13.025 | INFO | ANALYSIS : 57.6%, 3.0, 89.9ms, 0 | 20260505 04:48:13.025 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:13.025 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:13.025 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260505 04:48:13.025 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:13.025 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:13.025 | INFO | Uploaded 36GiB to GPU in 1.60055s (22.4923GiB/s) | 20260505 04:48:13.025 | INFO | Downloaded 34GiB to GPU in 1.51834s (22.3929GiB/s) | 20260505 04:48:13.025 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 04:48:13.025 | INFO | Total mem usage: 15.1 GiB | 20260505 04:48:13.025 | INFO | All Time High Water Marks sum: 15.7 GiB (Savings: 4.09%) | 20260505 04:48:13.025 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -4.34%) | 20260505 04:48:13.025 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 04:48:13.025 | INFO | High Water Breakdown: | 20260505 04:48:13.025 | INFO | 59.4% : 8.57 GiB ( 94.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 04:48:13.025 | INFO | 40.5% : 5.84 GiB ( 87.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 04:48:13.025 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 04:48:13.025 | INFO | ------------------------------------------ | 20260505 04:48:13.025 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:48:13.025 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 04:48:13.025 | INFO | Total mem usage: 26.5 GiB | 20260505 04:48:13.025 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260505 04:48:13.025 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 04:48:13.025 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260505 04:48:13.026 | INFO | High Water Breakdown: | 20260505 04:48:13.026 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:48:13.026 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:48:13.026 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:48:13.026 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:48:13.026 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:48:13.026 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:48:13.026 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 04:48:13.026 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 04:48:13.026 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:48:13.026 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 04:48:13.026 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 04:48:13.026 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 04:48:13.026 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 04:48:13.026 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 04:48:13.026 | INFO | ------------------------------------------ | 20260505 04:48:13.026 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:48:13.026 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 04:48:13.026 | INFO | Total mem usage: 25.6 GiB | 20260505 04:48:13.026 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 04:48:13.026 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 04:48:13.026 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 04:48:13.026 | INFO | High Water Breakdown: | 20260505 04:48:13.026 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 04:48:13.026 | INFO | ------------------------------------------ | 20260505 04:48:13.026 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:48:13.026 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 04:48:13.026 | INFO | Total mem usage: 15.7 GiB | 20260505 04:48:13.026 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260505 04:48:13.026 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 04:48:13.026 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260505 04:48:13.026 | INFO | High Water Breakdown: | 20260505 04:48:13.026 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 04:48:13.026 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 04:48:13.026 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 04:48:13.026 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 04:48:13.026 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 04:48:13.026 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 04:48:13.026 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 04:48:13.026 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 04:48:13.026 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 04:48:13.026 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:48:13.026 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 04:48:13.026 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:48:13.026 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 04:48:13.026 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 04:48:13.026 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 04:48:13.026 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 04:48:13.026 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:48:13.026 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 04:48:13.026 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:48:13.026 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:48:13.026 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:48:13.026 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 04:48:13.026 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 04:48:13.026 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:48:13.026 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 04:48:13.026 | INFO | ------------------------------------------ | 20260505 04:48:13.026 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:48:13.026 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 04:48:13.026 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:48:13.026 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:48:13.026 | INFO | PerfReport: Full Execution -- 3.04e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260505 04:48:13.026 | INFO | PerfReport: 0.28%, AggregateStats -- 8.62e-02 / -nan, 5.46e-03 / 0.00e+00 | 20260505 04:48:13.026 | INFO | PerfReport: 10.83%, Baseline -- 3.30e+00 / -nan, 1.69e-02 / 0.00e+00 | 20260505 04:48:13.026 | INFO | PerfReport: 3.04%, Histogram -- 9.24e-01 / -nan, 5.22e-02 / 0.00e+00 | 20260505 04:48:13.026 | INFO | PerfReport: 16.03%, DME -- 3.90e+01 / -nan, 3.22e+00 / 0.00e+00 | 20260505 04:48:13.026 | INFO | PerfReport: 56.55%, FrameLabeling -- 1.72e+01 / -nan, 4.52e-02 / 0.00e+00 | 20260505 04:48:13.026 | INFO | PerfReport: 7.63%, PulseAccumulating -- 2.32e+00 / -nan, 1.54e-01 / 0.00e+00 | 20260505 04:48:13.026 | INFO | PerfReport: 3.99%, Metrics -- 1.21e+00 / -nan, 1.99e-01 / 0.00e+00 | 20260505 04:48:13.026 | INFO | PerfReport: 1.65%, PulseSerializing -- 5.01e-01 / -nan, 8.04e-02 / 0.00e+00 | 20260505 04:48:13.026 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 04:48:13.026 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:48:13.026 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:48:13.026 | INFO | PerfReport: Full Execution -- 9.14e+01 / -nan, 1.90e+01 / 0.00e+00 | 20260505 04:48:13.026 | INFO | PerfReport: 18.29%, Upload -- 1.67e+01 / 0.00e+00, 3.59e-01 / 0.00e+00 | 20260505 04:48:13.027 | INFO | PerfReport: 17.56%, Download -- 1.60e+01 / 0.00e+00, 6.86e-01 / 0.00e+00 | 20260505 04:48:13.027 | INFO | PerfReport: 33.30%, Compute -- 3.04e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260505 04:48:13.027 | INFO | PerfReport: 0.55%, LockUpload -- 5.05e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260505 04:48:13.027 | INFO | PerfReport: 0.00%, LockDownload -- 3.88e-04 / 0.00e+00, 9.65e-05 / 0.00e+00 | 20260505 04:48:13.027 | INFO | PerfReport: 30.28%, LockCompute -- 2.77e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260505 04:48:13.027 | INFO | PerfReport: 0.02%, Misc -- 1.43e-02 / 0.00e+00, 2.58e-02 / 0.00e+00 | 20260505 04:48:13.122 | INFO | Starting a round of Baz IO | 20260505 04:48:16.076 | INFO | Populating RTMetrics report | 20260505 04:48:16.076 | INFO | RTMetrics report populated | 20260505 04:48:16.076 | INFO | Opening tmp RTMetrics json file | 20260505 04:48:16.077 | INFO | Serializing RTMetrics to json | 20260505 04:48:16.078 | INFO | Serialization done, swapping tmp file now... | 20260505 04:48:16.082 | INFO | RTMetrics done | 20260505 04:48:16.167 | INFO | Finished a round of Baz IO: Wrote 2.07 GiB in 3.044 seconds | 20260505 04:48:16.167 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.35% / 0.12% / 55.53% | 20260505 04:48:16.167 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.68487% / 0.000278155% / 5.16574e-05% | 20260505 04:48:16.167 | INFO | Baz IO overall: | 20260505 04:48:16.167 | INFO | BytesWritten : 192197377571 | 20260505 04:48:16.167 | INFO | NumEvents : 89457584155 | 20260505 04:48:18.164 | INFO | Analyzing chunk frames = [458752,459264) | 20260505 04:48:18.164 | INFO | MemInfo: { | 20260505 04:48:18.164 | INFO | "gpu" : " 17 GB", | 20260505 04:48:18.164 | INFO | "huge" : "", | 20260505 04:48:18.164 | INFO | "huge_shared" : " 27 GB", | 20260505 04:48:18.164 | INFO | "malloc" : " 14 GB", | 20260505 04:48:18.164 | INFO | "pinned" : " 29 GB" | 20260505 04:48:18.164 | INFO | } | 20260505 04:48:18.164 | INFO | Perf report for the last 5.13927 seconds: | 20260505 04:48:18.164 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:48:18.164 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:18.164 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:18.164 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:18.164 | INFO | ANALYSIS : 58.3%, 3.0, 90.6ms, 0 | 20260505 04:48:18.164 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:18.164 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260505 04:48:18.164 | INFO | PRE_HQ : 57.6%, 2.2, 64.6ms, 0 | 20260505 04:48:18.164 | INFO | BAZWRITER : 58.8%, 4.0, 122.1ms, 0 | 20260505 04:48:18.164 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 04:48:18.165 | INFO | Uploaded 36GiB to GPU in 1.6393s (21.9606GiB/s) | 20260505 04:48:18.165 | INFO | Downloaded 38GiB to GPU in 1.70131s (22.3357GiB/s) | 20260505 04:48:23.303 | INFO | Analyzing chunk frames = [459264,459776) | 20260505 04:48:23.303 | INFO | MemInfo: { | 20260505 04:48:23.303 | INFO | "gpu" : " 17 GB", | 20260505 04:48:23.303 | INFO | "huge" : "", | 20260505 04:48:23.303 | INFO | "huge_shared" : " 27 GB", | 20260505 04:48:23.303 | INFO | "malloc" : " 14 GB", | 20260505 04:48:23.303 | INFO | "pinned" : " 29 GB" | 20260505 04:48:23.303 | INFO | } | 20260505 04:48:23.303 | INFO | Perf report for the last 5.13928 seconds: | 20260505 04:48:23.303 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:48:23.303 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:23.303 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:23.303 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:23.303 | INFO | ANALYSIS : 58.0%, 3.0, 90.4ms, 0 | 20260505 04:48:23.303 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:23.303 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:23.303 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 04:48:23.303 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:23.303 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:23.304 | INFO | Uploaded 36GiB to GPU in 1.59871s (22.5182GiB/s) | 20260505 04:48:23.304 | INFO | Downloaded 34GiB to GPU in 1.51492s (22.4435GiB/s) | 20260505 04:48:28.443 | INFO | Analyzing chunk frames = [459776,460288) | 20260505 04:48:28.443 | INFO | MemInfo: { | 20260505 04:48:28.443 | INFO | "gpu" : " 17 GB", | 20260505 04:48:28.443 | INFO | "huge" : "", | 20260505 04:48:28.443 | INFO | "huge_shared" : " 27 GB", | 20260505 04:48:28.443 | INFO | "malloc" : " 14 GB", | 20260505 04:48:28.443 | INFO | "pinned" : " 29 GB" | 20260505 04:48:28.443 | INFO | } | 20260505 04:48:28.443 | INFO | Perf report for the last 5.13961 seconds: | 20260505 04:48:28.443 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:48:28.443 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:28.443 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:28.443 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:28.443 | INFO | ANALYSIS : 57.4%, 3.0, 89.2ms, 0 | 20260505 04:48:28.443 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:28.443 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:28.443 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260505 04:48:28.443 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:28.443 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:28.443 | INFO | Uploaded 36GiB to GPU in 1.60303s (22.4575GiB/s) | 20260505 04:48:28.443 | INFO | Downloaded 34GiB to GPU in 1.52042s (22.3623GiB/s) | 20260505 04:48:33.582 | INFO | Analyzing chunk frames = [460288,460800) | 20260505 04:48:33.582 | INFO | MemInfo: { | 20260505 04:48:33.582 | INFO | "gpu" : " 17 GB", | 20260505 04:48:33.582 | INFO | "huge" : "", | 20260505 04:48:33.582 | INFO | "huge_shared" : " 27 GB", | 20260505 04:48:33.582 | INFO | "malloc" : " 14 GB", | 20260505 04:48:33.582 | INFO | "pinned" : " 29 GB" | 20260505 04:48:33.582 | INFO | } | 20260505 04:48:33.582 | INFO | Perf report for the last 5.13921 seconds: | 20260505 04:48:33.582 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:48:33.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:33.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:33.582 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:33.582 | INFO | ANALYSIS : 57.0%, 3.0, 88.4ms, 0 | 20260505 04:48:33.582 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:33.582 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:33.582 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 04:48:33.582 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:33.582 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:33.583 | INFO | Uploaded 36GiB to GPU in 1.60291s (22.4591GiB/s) | 20260505 04:48:33.583 | INFO | Downloaded 34GiB to GPU in 1.51957s (22.3748GiB/s) | 20260505 04:48:38.721 | INFO | Analyzing chunk frames = [460800,461312) | 20260505 04:48:38.721 | INFO | MemInfo: { | 20260505 04:48:38.721 | INFO | "gpu" : " 17 GB", | 20260505 04:48:38.721 | INFO | "huge" : "", | 20260505 04:48:38.721 | INFO | "huge_shared" : " 27 GB", | 20260505 04:48:38.721 | INFO | "malloc" : " 14 GB", | 20260505 04:48:38.721 | INFO | "pinned" : " 29 GB" | 20260505 04:48:38.721 | INFO | } | 20260505 04:48:38.721 | INFO | Perf report for the last 5.13929 seconds: | 20260505 04:48:38.721 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:48:38.721 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:38.721 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:38.722 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:38.722 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260505 04:48:38.722 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:38.722 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:38.722 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 04:48:38.722 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:38.722 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:38.722 | INFO | Uploaded 36GiB to GPU in 1.60028s (22.4961GiB/s) | 20260505 04:48:38.722 | INFO | Downloaded 34GiB to GPU in 1.51702s (22.4123GiB/s) | 20260505 04:48:43.861 | INFO | Analyzing chunk frames = [461312,461824) | 20260505 04:48:43.861 | INFO | MemInfo: { | 20260505 04:48:43.861 | INFO | "gpu" : " 17 GB", | 20260505 04:48:43.861 | INFO | "huge" : "", | 20260505 04:48:43.861 | INFO | "huge_shared" : " 27 GB", | 20260505 04:48:43.861 | INFO | "malloc" : " 14 GB", | 20260505 04:48:43.861 | INFO | "pinned" : " 29 GB" | 20260505 04:48:43.861 | INFO | } | 20260505 04:48:43.861 | INFO | Perf report for the last 5.13926 seconds: | 20260505 04:48:43.861 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:48:43.861 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:43.861 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:43.861 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:43.861 | INFO | ANALYSIS : 57.2%, 3.0, 89.3ms, 0 | 20260505 04:48:43.861 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:43.861 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:43.861 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 04:48:43.861 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:43.861 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:43.861 | INFO | Uploaded 36GiB to GPU in 1.60142s (22.48GiB/s) | 20260505 04:48:43.861 | INFO | Downloaded 34GiB to GPU in 1.51775s (22.4016GiB/s) | 20260505 04:48:49.000 | INFO | Analyzing chunk frames = [461824,462336) | 20260505 04:48:49.000 | INFO | MemInfo: { | 20260505 04:48:49.000 | INFO | "gpu" : " 17 GB", | 20260505 04:48:49.000 | INFO | "huge" : "", | 20260505 04:48:49.000 | INFO | "huge_shared" : " 27 GB", | 20260505 04:48:49.000 | INFO | "malloc" : " 14 GB", | 20260505 04:48:49.000 | INFO | "pinned" : " 29 GB" | 20260505 04:48:49.000 | INFO | } | 20260505 04:48:49.001 | INFO | Perf report for the last 5.13976 seconds: | 20260505 04:48:49.001 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:48:49.001 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:49.001 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:49.001 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:49.001 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260505 04:48:49.001 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:49.001 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:49.001 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 04:48:49.001 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:49.001 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:49.001 | INFO | Uploaded 36GiB to GPU in 1.6006s (22.4915GiB/s) | 20260505 04:48:49.001 | INFO | Downloaded 34GiB to GPU in 1.51836s (22.3925GiB/s) | 20260505 04:48:54.139 | INFO | Analyzing chunk frames = [462336,462848) | 20260505 04:48:54.139 | INFO | MemInfo: { | 20260505 04:48:54.139 | INFO | "gpu" : " 17 GB", | 20260505 04:48:54.139 | INFO | "huge" : "", | 20260505 04:48:54.139 | INFO | "huge_shared" : " 27 GB", | 20260505 04:48:54.139 | INFO | "malloc" : " 14 GB", | 20260505 04:48:54.139 | INFO | "pinned" : " 29 GB" | 20260505 04:48:54.139 | INFO | } | 20260505 04:48:54.139 | INFO | Perf report for the last 5.13893 seconds: | 20260505 04:48:54.139 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:48:54.139 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:54.139 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:54.140 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:54.140 | INFO | ANALYSIS : 57.6%, 3.0, 89.8ms, 0 | 20260505 04:48:54.140 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:54.140 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:54.140 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 04:48:54.140 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:54.140 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:54.140 | INFO | Uploaded 36GiB to GPU in 1.60106s (22.4851GiB/s) | 20260505 04:48:54.140 | INFO | Downloaded 34GiB to GPU in 1.51789s (22.3996GiB/s) | 20260505 04:48:54.236 | INFO | Starting a round of Baz IO | 20260505 04:48:57.189 | INFO | Populating RTMetrics report | 20260505 04:48:57.189 | INFO | RTMetrics report populated | 20260505 04:48:57.189 | INFO | Opening tmp RTMetrics json file | 20260505 04:48:57.190 | INFO | Serializing RTMetrics to json | 20260505 04:48:57.191 | INFO | Serialization done, swapping tmp file now... | 20260505 04:48:57.192 | INFO | RTMetrics done | 20260505 04:48:57.279 | INFO | Finished a round of Baz IO: Wrote 2.074 GiB in 3.042 seconds | 20260505 04:48:57.279 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.23% / 0.12% / 55.65% | 20260505 04:48:57.279 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.63482% / 0.000234445% / 3.97364e-05% | 20260505 04:48:57.279 | INFO | Baz IO overall: | 20260505 04:48:57.279 | INFO | BytesWritten : 194424801789 | 20260505 04:48:57.279 | INFO | NumEvents : 90514907519 | 20260505 04:48:59.279 | INFO | Analyzing chunk frames = [462848,463360) | 20260505 04:48:59.279 | INFO | MemInfo: { | 20260505 04:48:59.279 | INFO | "gpu" : " 17 GB", | 20260505 04:48:59.279 | INFO | "huge" : "", | 20260505 04:48:59.279 | INFO | "huge_shared" : " 27 GB", | 20260505 04:48:59.279 | INFO | "malloc" : " 14 GB", | 20260505 04:48:59.279 | INFO | "pinned" : " 29 GB" | 20260505 04:48:59.279 | INFO | } | 20260505 04:48:59.279 | INFO | Perf report for the last 5.13966 seconds: | 20260505 04:48:59.279 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:48:59.279 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:59.279 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:59.279 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:59.279 | INFO | ANALYSIS : 58.2%, 3.0, 90.5ms, 0 | 20260505 04:48:59.279 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:48:59.279 | INFO | RT_METRICS : 45.3%, 1.0, 23.8ms, 0 | 20260505 04:48:59.279 | INFO | PRE_HQ : 57.5%, 2.2, 64.3ms, 0 | 20260505 04:48:59.279 | INFO | BAZWRITER : 58.7%, 4.0, 121.5ms, 0 | 20260505 04:48:59.279 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:48:59.279 | INFO | Uploaded 36GiB to GPU in 1.64044s (21.9453GiB/s) | 20260505 04:48:59.280 | INFO | Downloaded 38GiB to GPU in 1.70392s (22.3015GiB/s) | 20260505 04:49:04.418 | INFO | Analyzing chunk frames = [463360,463872) | 20260505 04:49:04.418 | INFO | MemInfo: { | 20260505 04:49:04.418 | INFO | "gpu" : " 17 GB", | 20260505 04:49:04.418 | INFO | "huge" : "", | 20260505 04:49:04.418 | INFO | "huge_shared" : " 27 GB", | 20260505 04:49:04.418 | INFO | "malloc" : " 14 GB", | 20260505 04:49:04.418 | INFO | "pinned" : " 29 GB" | 20260505 04:49:04.418 | INFO | } | 20260505 04:49:04.418 | INFO | Perf report for the last 5.13911 seconds: | 20260505 04:49:04.418 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:49:04.418 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:04.418 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:04.418 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:04.418 | INFO | ANALYSIS : 58.0%, 3.0, 90.4ms, 0 | 20260505 04:49:04.418 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:04.418 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:04.418 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260505 04:49:04.418 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:04.418 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:04.419 | INFO | Uploaded 36GiB to GPU in 1.5996s (22.5056GiB/s) | 20260505 04:49:04.419 | INFO | Downloaded 34GiB to GPU in 1.51607s (22.4264GiB/s) | 20260505 04:49:09.557 | INFO | Analyzing chunk frames = [463872,464384) | 20260505 04:49:09.557 | INFO | MemInfo: { | 20260505 04:49:09.557 | INFO | "gpu" : " 17 GB", | 20260505 04:49:09.557 | INFO | "huge" : "", | 20260505 04:49:09.557 | INFO | "huge_shared" : " 27 GB", | 20260505 04:49:09.557 | INFO | "malloc" : " 14 GB", | 20260505 04:49:09.558 | INFO | "pinned" : " 29 GB" | 20260505 04:49:09.558 | INFO | } | 20260505 04:49:09.558 | INFO | Perf report for the last 5.13933 seconds: | 20260505 04:49:09.558 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:49:09.558 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:09.558 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:09.558 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:09.558 | INFO | ANALYSIS : 57.4%, 3.0, 89.2ms, 0 | 20260505 04:49:09.558 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:09.558 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:09.558 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 04:49:09.558 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:09.558 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:09.558 | INFO | Uploaded 36GiB to GPU in 1.60323s (22.4546GiB/s) | 20260505 04:49:09.558 | INFO | Downloaded 34GiB to GPU in 1.52022s (22.3652GiB/s) | 20260505 04:49:14.697 | INFO | Analyzing chunk frames = [464384,464896) | 20260505 04:49:14.697 | INFO | MemInfo: { | 20260505 04:49:14.697 | INFO | "gpu" : " 17 GB", | 20260505 04:49:14.697 | INFO | "huge" : "", | 20260505 04:49:14.697 | INFO | "huge_shared" : " 27 GB", | 20260505 04:49:14.697 | INFO | "malloc" : " 14 GB", | 20260505 04:49:14.697 | INFO | "pinned" : " 29 GB" | 20260505 04:49:14.697 | INFO | } | 20260505 04:49:14.697 | INFO | Perf report for the last 5.13939 seconds: | 20260505 04:49:14.697 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:49:14.697 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:14.697 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:14.697 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:14.697 | INFO | ANALYSIS : 57.0%, 3.0, 88.4ms, 0 | 20260505 04:49:14.697 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:14.697 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:14.697 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 04:49:14.697 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:14.697 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:14.697 | INFO | Uploaded 36GiB to GPU in 1.60268s (22.4624GiB/s) | 20260505 04:49:14.697 | INFO | Downloaded 34GiB to GPU in 1.51893s (22.3841GiB/s) | 20260505 04:49:19.836 | INFO | Analyzing chunk frames = [464896,465408) | 20260505 04:49:19.836 | INFO | MemInfo: { | 20260505 04:49:19.836 | INFO | "gpu" : " 17 GB", | 20260505 04:49:19.836 | INFO | "huge" : "", | 20260505 04:49:19.836 | INFO | "huge_shared" : " 27 GB", | 20260505 04:49:19.836 | INFO | "malloc" : " 14 GB", | 20260505 04:49:19.836 | INFO | "pinned" : " 29 GB" | 20260505 04:49:19.836 | INFO | } | 20260505 04:49:19.836 | INFO | Perf report for the last 5.13935 seconds: | 20260505 04:49:19.836 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:49:19.836 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:19.836 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:19.836 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:19.836 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260505 04:49:19.836 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:19.836 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:19.837 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 04:49:19.837 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:19.837 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:19.837 | INFO | Uploaded 36GiB to GPU in 1.60066s (22.4907GiB/s) | 20260505 04:49:19.837 | INFO | Downloaded 34GiB to GPU in 1.5167s (22.4171GiB/s) | 20260505 04:49:24.975 | INFO | Analyzing chunk frames = [465408,465920) | 20260505 04:49:24.975 | INFO | MemInfo: { | 20260505 04:49:24.975 | INFO | "gpu" : " 17 GB", | 20260505 04:49:24.975 | INFO | "huge" : "", | 20260505 04:49:24.975 | INFO | "huge_shared" : " 27 GB", | 20260505 04:49:24.976 | INFO | "malloc" : " 14 GB", | 20260505 04:49:24.976 | INFO | "pinned" : " 29 GB" | 20260505 04:49:24.976 | INFO | } | 20260505 04:49:24.976 | INFO | Perf report for the last 5.1393 seconds: | 20260505 04:49:24.976 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:49:24.976 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:24.976 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:24.976 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:24.976 | INFO | ANALYSIS : 57.3%, 3.0, 89.3ms, 0 | 20260505 04:49:24.976 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:24.976 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:24.976 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 04:49:24.976 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:24.976 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:24.976 | INFO | Uploaded 36GiB to GPU in 1.60122s (22.4829GiB/s) | 20260505 04:49:24.976 | INFO | Downloaded 34GiB to GPU in 1.5181s (22.3964GiB/s) | 20260505 04:49:30.115 | INFO | Analyzing chunk frames = [465920,466432) | 20260505 04:49:30.115 | INFO | MemInfo: { | 20260505 04:49:30.115 | INFO | "gpu" : " 17 GB", | 20260505 04:49:30.115 | INFO | "huge" : "", | 20260505 04:49:30.115 | INFO | "huge_shared" : " 27 GB", | 20260505 04:49:30.115 | INFO | "malloc" : " 14 GB", | 20260505 04:49:30.115 | INFO | "pinned" : " 29 GB" | 20260505 04:49:30.115 | INFO | } | 20260505 04:49:30.115 | INFO | Perf report for the last 5.13928 seconds: | 20260505 04:49:30.115 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:49:30.115 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:30.115 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:30.115 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:30.115 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260505 04:49:30.115 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:30.115 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:30.115 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 04:49:30.115 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:30.115 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:30.115 | INFO | Uploaded 36GiB to GPU in 1.60104s (22.4854GiB/s) | 20260505 04:49:30.115 | INFO | Downloaded 34GiB to GPU in 1.51718s (22.4099GiB/s) | 20260505 04:49:35.254 | INFO | Analyzing chunk frames = [466432,466944) | 20260505 04:49:35.254 | INFO | MemInfo: { | 20260505 04:49:35.254 | INFO | "gpu" : " 17 GB", | 20260505 04:49:35.254 | INFO | "huge" : "", | 20260505 04:49:35.254 | INFO | "huge_shared" : " 27 GB", | 20260505 04:49:35.254 | INFO | "malloc" : " 14 GB", | 20260505 04:49:35.254 | INFO | "pinned" : " 29 GB" | 20260505 04:49:35.254 | INFO | } | 20260505 04:49:35.254 | INFO | Perf report for the last 5.13932 seconds: | 20260505 04:49:35.254 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:49:35.254 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:35.254 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:35.254 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:35.254 | INFO | ANALYSIS : 57.6%, 3.0, 89.8ms, 0 | 20260505 04:49:35.254 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:35.254 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:35.254 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 04:49:35.254 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:35.254 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:35.255 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 04:49:35.255 | INFO | Total mem usage: 15.1 GiB | 20260505 04:49:35.255 | INFO | All Time High Water Marks sum: 15.7 GiB (Savings: 4.09%) | 20260505 04:49:35.255 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -4.24%) | 20260505 04:49:35.255 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 04:49:35.255 | INFO | High Water Breakdown: | 20260505 04:49:35.255 | INFO | 59.4% : 8.58 GiB ( 95% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 04:49:35.255 | INFO | 40.4% : 5.84 GiB ( 87.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 04:49:35.255 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 04:49:35.255 | INFO | Uploaded 36GiB to GPU in 1.6013s (22.4817GiB/s) | 20260505 04:49:35.255 | INFO | ------------------------------------------ | 20260505 04:49:35.255 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:49:35.255 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 04:49:35.255 | INFO | Downloaded 34GiB to GPU in 1.51845s (22.3912GiB/s) | 20260505 04:49:35.255 | INFO | Total mem usage: 26.5 GiB | 20260505 04:49:35.255 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260505 04:49:35.255 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 04:49:35.255 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260505 04:49:35.255 | INFO | High Water Breakdown: | 20260505 04:49:35.255 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:49:35.255 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:49:35.255 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:49:35.255 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:49:35.255 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:49:35.255 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:49:35.255 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 04:49:35.255 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 04:49:35.255 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:49:35.255 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 04:49:35.255 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 04:49:35.255 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 04:49:35.255 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 04:49:35.255 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 04:49:35.255 | INFO | ------------------------------------------ | 20260505 04:49:35.255 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:49:35.255 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 04:49:35.255 | INFO | Total mem usage: 25.6 GiB | 20260505 04:49:35.255 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 04:49:35.255 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 04:49:35.255 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 04:49:35.255 | INFO | High Water Breakdown: | 20260505 04:49:35.255 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 04:49:35.255 | INFO | ------------------------------------------ | 20260505 04:49:35.255 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:49:35.255 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 04:49:35.255 | INFO | Total mem usage: 15.7 GiB | 20260505 04:49:35.255 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260505 04:49:35.255 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 04:49:35.255 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260505 04:49:35.255 | INFO | High Water Breakdown: | 20260505 04:49:35.255 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 04:49:35.255 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 04:49:35.255 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 04:49:35.255 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 04:49:35.255 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 04:49:35.255 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 04:49:35.255 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 04:49:35.255 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 04:49:35.255 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 04:49:35.255 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:49:35.255 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 04:49:35.255 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:49:35.255 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 04:49:35.255 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 04:49:35.255 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 04:49:35.255 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 04:49:35.255 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:49:35.255 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 04:49:35.255 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:49:35.255 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:49:35.256 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:49:35.256 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 04:49:35.256 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 04:49:35.256 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:49:35.256 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 04:49:35.256 | INFO | ------------------------------------------ | 20260505 04:49:35.256 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:49:35.256 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 04:49:35.256 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:49:35.256 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:49:35.256 | INFO | PerfReport: Full Execution -- 3.04e+01 / -nan, 1.28e+01 / 0.00e+00 | 20260505 04:49:35.256 | INFO | PerfReport: 0.28%, AggregateStats -- 8.62e-02 / -nan, 4.18e-03 / 0.00e+00 | 20260505 04:49:35.256 | INFO | PerfReport: 10.83%, Baseline -- 3.30e+00 / -nan, 2.02e-02 / 0.00e+00 | 20260505 04:49:35.256 | INFO | PerfReport: 3.04%, Histogram -- 9.25e-01 / -nan, 5.27e-02 / 0.00e+00 | 20260505 04:49:35.256 | INFO | PerfReport: 16.03%, DME -- 3.90e+01 / -nan, 3.24e+00 / 0.00e+00 | 20260505 04:49:35.256 | INFO | PerfReport: 56.57%, FrameLabeling -- 1.72e+01 / -nan, 4.55e-02 / 0.00e+00 | 20260505 04:49:35.256 | INFO | PerfReport: 7.62%, PulseAccumulating -- 2.32e+00 / -nan, 1.55e-01 / 0.00e+00 | 20260505 04:49:35.256 | INFO | PerfReport: 3.98%, Metrics -- 1.21e+00 / -nan, 1.98e-01 / 0.00e+00 | 20260505 04:49:35.256 | INFO | PerfReport: 1.64%, PulseSerializing -- 5.00e-01 / -nan, 8.07e-02 / 0.00e+00 | 20260505 04:49:35.256 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 04:49:35.256 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:49:35.256 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:49:35.256 | INFO | PerfReport: Full Execution -- 9.14e+01 / -nan, 1.90e+01 / 0.00e+00 | 20260505 04:49:35.256 | INFO | PerfReport: 18.30%, Upload -- 1.67e+01 / 0.00e+00, 3.74e-01 / 0.00e+00 | 20260505 04:49:35.256 | INFO | PerfReport: 17.56%, Download -- 1.61e+01 / 0.00e+00, 7.02e-01 / 0.00e+00 | 20260505 04:49:35.256 | INFO | PerfReport: 33.30%, Compute -- 3.04e+01 / 0.00e+00, 1.28e+01 / 0.00e+00 | 20260505 04:49:35.256 | INFO | PerfReport: 0.55%, LockUpload -- 5.03e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260505 04:49:35.256 | INFO | PerfReport: 0.00%, LockDownload -- 3.86e-04 / 0.00e+00, 9.05e-05 / 0.00e+00 | 20260505 04:49:35.256 | INFO | PerfReport: 30.26%, LockCompute -- 2.77e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260505 04:49:35.256 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.51e-02 / 0.00e+00 | 20260505 04:49:35.352 | INFO | Starting a round of Baz IO | 20260505 04:49:38.304 | INFO | Populating RTMetrics report | 20260505 04:49:38.305 | INFO | RTMetrics report populated | 20260505 04:49:38.305 | INFO | Opening tmp RTMetrics json file | 20260505 04:49:38.306 | INFO | Serializing RTMetrics to json | 20260505 04:49:38.308 | INFO | Serialization done, swapping tmp file now... | 20260505 04:49:38.310 | INFO | RTMetrics done | 20260505 04:49:38.402 | INFO | Finished a round of Baz IO: Wrote 2.079 GiB in 3.049 seconds | 20260505 04:49:38.402 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.11% / 0.12% / 55.77% | 20260505 04:49:38.402 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.57716% / 0.00026226% / 3.17891e-05% | 20260505 04:49:38.402 | INFO | Baz IO overall: | 20260505 04:49:38.402 | INFO | BytesWritten : 196656936153 | 20260505 04:49:38.402 | INFO | NumEvents : 91574210401 | 20260505 04:49:40.393 | INFO | Analyzing chunk frames = [466944,467456) | 20260505 04:49:40.393 | INFO | MemInfo: { | 20260505 04:49:40.393 | INFO | "gpu" : " 17 GB", | 20260505 04:49:40.393 | INFO | "huge" : "", | 20260505 04:49:40.393 | INFO | "huge_shared" : " 27 GB", | 20260505 04:49:40.393 | INFO | "malloc" : " 14 GB", | 20260505 04:49:40.393 | INFO | "pinned" : " 29 GB" | 20260505 04:49:40.393 | INFO | } | 20260505 04:49:40.393 | INFO | Perf report for the last 5.13914 seconds: | 20260505 04:49:40.393 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:49:40.394 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:40.394 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:40.394 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:40.394 | INFO | ANALYSIS : 58.2%, 3.0, 90.6ms, 0 | 20260505 04:49:40.394 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:40.394 | INFO | RT_METRICS : 45.3%, 1.0, 23.8ms, 0 | 20260505 04:49:40.394 | INFO | PRE_HQ : 57.5%, 2.2, 64.4ms, 0 | 20260505 04:49:40.394 | INFO | BAZWRITER : 58.9%, 4.1, 123.9ms, 0 | 20260505 04:49:40.394 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 04:49:40.394 | INFO | Uploaded 36GiB to GPU in 1.64173s (21.928GiB/s) | 20260505 04:49:40.394 | INFO | Downloaded 38GiB to GPU in 1.70287s (22.3153GiB/s) | 20260505 04:49:45.533 | INFO | Analyzing chunk frames = [467456,467968) | 20260505 04:49:45.533 | INFO | MemInfo: { | 20260505 04:49:45.533 | INFO | "gpu" : " 17 GB", | 20260505 04:49:45.533 | INFO | "huge" : "", | 20260505 04:49:45.533 | INFO | "huge_shared" : " 27 GB", | 20260505 04:49:45.533 | INFO | "malloc" : " 14 GB", | 20260505 04:49:45.533 | INFO | "pinned" : " 29 GB" | 20260505 04:49:45.533 | INFO | } | 20260505 04:49:45.533 | INFO | Perf report for the last 5.13964 seconds: | 20260505 04:49:45.533 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:49:45.533 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:45.533 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:45.533 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:45.533 | INFO | ANALYSIS : 58.0%, 3.0, 90.4ms, 0 | 20260505 04:49:45.533 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:45.533 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:45.533 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 04:49:45.533 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:45.533 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:45.534 | INFO | Uploaded 36GiB to GPU in 1.59958s (22.5058GiB/s) | 20260505 04:49:45.534 | INFO | Downloaded 34GiB to GPU in 1.5163s (22.423GiB/s) | 20260505 04:49:50.672 | INFO | Analyzing chunk frames = [467968,468480) | 20260505 04:49:50.672 | INFO | MemInfo: { | 20260505 04:49:50.672 | INFO | "gpu" : " 17 GB", | 20260505 04:49:50.672 | INFO | "huge" : "", | 20260505 04:49:50.672 | INFO | "huge_shared" : " 27 GB", | 20260505 04:49:50.672 | INFO | "malloc" : " 14 GB", | 20260505 04:49:50.672 | INFO | "pinned" : " 29 GB" | 20260505 04:49:50.672 | INFO | } | 20260505 04:49:50.672 | INFO | Perf report for the last 5.13921 seconds: | 20260505 04:49:50.672 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:49:50.672 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:50.672 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:50.672 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:50.672 | INFO | ANALYSIS : 57.4%, 3.0, 89.1ms, 0 | 20260505 04:49:50.672 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:50.672 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:50.672 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 04:49:50.672 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:50.672 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:50.673 | INFO | Uploaded 36GiB to GPU in 1.60267s (22.4625GiB/s) | 20260505 04:49:50.673 | INFO | Downloaded 34GiB to GPU in 1.51993s (22.3695GiB/s) | 20260505 04:49:55.811 | INFO | Analyzing chunk frames = [468480,468992) | 20260505 04:49:55.811 | INFO | MemInfo: { | 20260505 04:49:55.811 | INFO | "gpu" : " 17 GB", | 20260505 04:49:55.811 | INFO | "huge" : "", | 20260505 04:49:55.811 | INFO | "huge_shared" : " 27 GB", | 20260505 04:49:55.811 | INFO | "malloc" : " 14 GB", | 20260505 04:49:55.811 | INFO | "pinned" : " 29 GB" | 20260505 04:49:55.812 | INFO | } | 20260505 04:49:55.812 | INFO | Perf report for the last 5.13929 seconds: | 20260505 04:49:55.812 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:49:55.812 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:55.812 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:55.812 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:55.812 | INFO | ANALYSIS : 57.0%, 3.0, 88.5ms, 0 | 20260505 04:49:55.812 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:55.812 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:55.812 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 04:49:55.812 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:55.812 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:49:55.812 | INFO | Uploaded 36GiB to GPU in 1.60284s (22.4602GiB/s) | 20260505 04:49:55.812 | INFO | Downloaded 34GiB to GPU in 1.51942s (22.377GiB/s) | 20260505 04:50:00.951 | INFO | Analyzing chunk frames = [468992,469504) | 20260505 04:50:00.951 | INFO | MemInfo: { | 20260505 04:50:00.951 | INFO | "gpu" : " 17 GB", | 20260505 04:50:00.951 | INFO | "huge" : "", | 20260505 04:50:00.951 | INFO | "huge_shared" : " 27 GB", | 20260505 04:50:00.951 | INFO | "malloc" : " 14 GB", | 20260505 04:50:00.951 | INFO | "pinned" : " 29 GB" | 20260505 04:50:00.951 | INFO | } | 20260505 04:50:00.951 | INFO | Perf report for the last 5.13945 seconds: | 20260505 04:50:00.951 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:50:00.951 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:00.951 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:00.951 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:00.951 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260505 04:50:00.951 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:00.951 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:00.951 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 04:50:00.951 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:00.951 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:00.951 | INFO | Uploaded 36GiB to GPU in 1.60067s (22.4906GiB/s) | 20260505 04:50:00.951 | INFO | Downloaded 34GiB to GPU in 1.5165s (22.42GiB/s) | 20260505 04:50:06.090 | INFO | Analyzing chunk frames = [469504,470016) | 20260505 04:50:06.090 | INFO | MemInfo: { | 20260505 04:50:06.090 | INFO | "gpu" : " 17 GB", | 20260505 04:50:06.090 | INFO | "huge" : "", | 20260505 04:50:06.090 | INFO | "huge_shared" : " 27 GB", | 20260505 04:50:06.090 | INFO | "malloc" : " 14 GB", | 20260505 04:50:06.090 | INFO | "pinned" : " 29 GB" | 20260505 04:50:06.090 | INFO | } | 20260505 04:50:06.090 | INFO | Perf report for the last 5.13929 seconds: | 20260505 04:50:06.090 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:50:06.090 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:06.090 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:06.090 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:06.090 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 04:50:06.090 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:06.090 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:06.091 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 04:50:06.091 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:06.091 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:06.091 | INFO | Uploaded 36GiB to GPU in 1.60075s (22.4895GiB/s) | 20260505 04:50:06.091 | INFO | Downloaded 34GiB to GPU in 1.51736s (22.4073GiB/s) | 20260505 04:50:11.229 | INFO | Analyzing chunk frames = [470016,470528) | 20260505 04:50:11.230 | INFO | MemInfo: { | 20260505 04:50:11.230 | INFO | "gpu" : " 17 GB", | 20260505 04:50:11.230 | INFO | "huge" : "", | 20260505 04:50:11.230 | INFO | "huge_shared" : " 27 GB", | 20260505 04:50:11.230 | INFO | "malloc" : " 14 GB", | 20260505 04:50:11.230 | INFO | "pinned" : " 29 GB" | 20260505 04:50:11.230 | INFO | } | 20260505 04:50:11.230 | INFO | Perf report for the last 5.13938 seconds: | 20260505 04:50:11.230 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:50:11.230 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:11.230 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:11.230 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:11.230 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260505 04:50:11.230 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:11.230 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:11.230 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 04:50:11.230 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:11.230 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:11.230 | INFO | Uploaded 36GiB to GPU in 1.60148s (22.4792GiB/s) | 20260505 04:50:11.230 | INFO | Downloaded 34GiB to GPU in 1.51835s (22.3927GiB/s) | 20260505 04:50:16.369 | INFO | Analyzing chunk frames = [470528,471040) | 20260505 04:50:16.369 | INFO | MemInfo: { | 20260505 04:50:16.369 | INFO | "gpu" : " 17 GB", | 20260505 04:50:16.369 | INFO | "huge" : "", | 20260505 04:50:16.369 | INFO | "huge_shared" : " 27 GB", | 20260505 04:50:16.369 | INFO | "malloc" : " 14 GB", | 20260505 04:50:16.369 | INFO | "pinned" : " 29 GB" | 20260505 04:50:16.369 | INFO | } | 20260505 04:50:16.369 | INFO | Perf report for the last 5.13937 seconds: | 20260505 04:50:16.369 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:50:16.369 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:16.369 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:16.369 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:16.369 | INFO | ANALYSIS : 57.6%, 3.0, 89.8ms, 0 | 20260505 04:50:16.369 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:16.369 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:16.369 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 04:50:16.369 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:16.369 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:16.370 | INFO | Uploaded 36GiB to GPU in 1.60227s (22.4682GiB/s) | 20260505 04:50:16.370 | INFO | Downloaded 34GiB to GPU in 1.51913s (22.3812GiB/s) | 20260505 04:50:16.466 | INFO | Starting a round of Baz IO | 20260505 04:50:19.416 | INFO | Populating RTMetrics report | 20260505 04:50:19.416 | INFO | RTMetrics report populated | 20260505 04:50:19.416 | INFO | Opening tmp RTMetrics json file | 20260505 04:50:19.417 | INFO | Serializing RTMetrics to json | 20260505 04:50:19.418 | INFO | Serialization done, swapping tmp file now... | 20260505 04:50:19.418 | INFO | RTMetrics done | 20260505 04:50:19.508 | INFO | Finished a round of Baz IO: Wrote 2.08 GiB in 3.041 seconds | 20260505 04:50:19.508 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.99% / 0.12% / 55.89% | 20260505 04:50:19.508 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.52626% / 0.000270208% / 3.97364e-05% | 20260505 04:50:19.508 | INFO | Baz IO overall: | 20260505 04:50:19.508 | INFO | BytesWritten : 198890839361 | 20260505 04:50:19.508 | INFO | NumEvents : 92634130408 | 20260505 04:50:21.508 | INFO | Analyzing chunk frames = [471040,471552) | 20260505 04:50:21.508 | INFO | MemInfo: { | 20260505 04:50:21.508 | INFO | "gpu" : " 17 GB", | 20260505 04:50:21.508 | INFO | "huge" : "", | 20260505 04:50:21.508 | INFO | "huge_shared" : " 27 GB", | 20260505 04:50:21.508 | INFO | "malloc" : " 14 GB", | 20260505 04:50:21.508 | INFO | "pinned" : " 29 GB" | 20260505 04:50:21.508 | INFO | } | 20260505 04:50:21.508 | INFO | Perf report for the last 5.13921 seconds: | 20260505 04:50:21.508 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:50:21.508 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:21.508 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:21.508 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:21.508 | INFO | ANALYSIS : 58.2%, 3.0, 90.4ms, 0 | 20260505 04:50:21.509 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:21.509 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260505 04:50:21.509 | INFO | PRE_HQ : 57.5%, 2.2, 64.3ms, 0 | 20260505 04:50:21.509 | INFO | BAZWRITER : 58.5%, 4.1, 122.1ms, 0 | 20260505 04:50:21.509 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:50:21.509 | INFO | Uploaded 36GiB to GPU in 1.6413s (21.9339GiB/s) | 20260505 04:50:21.509 | INFO | Downloaded 38GiB to GPU in 1.70528s (22.2837GiB/s) | 20260505 04:50:26.647 | INFO | Analyzing chunk frames = [471552,472064) | 20260505 04:50:26.647 | INFO | MemInfo: { | 20260505 04:50:26.647 | INFO | "gpu" : " 17 GB", | 20260505 04:50:26.647 | INFO | "huge" : "", | 20260505 04:50:26.647 | INFO | "huge_shared" : " 27 GB", | 20260505 04:50:26.647 | INFO | "malloc" : " 14 GB", | 20260505 04:50:26.648 | INFO | "pinned" : " 29 GB" | 20260505 04:50:26.648 | INFO | } | 20260505 04:50:26.648 | INFO | Perf report for the last 5.13928 seconds: | 20260505 04:50:26.648 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:50:26.648 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:26.648 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:26.648 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:26.648 | INFO | ANALYSIS : 58.0%, 3.0, 90.5ms, 0 | 20260505 04:50:26.648 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:26.648 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:26.648 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 04:50:26.648 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:26.648 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:26.648 | INFO | Uploaded 36GiB to GPU in 1.59884s (22.5164GiB/s) | 20260505 04:50:26.648 | INFO | Downloaded 34GiB to GPU in 1.51579s (22.4306GiB/s) | 20260505 04:50:31.787 | INFO | Analyzing chunk frames = [472064,472576) | 20260505 04:50:31.787 | INFO | MemInfo: { | 20260505 04:50:31.787 | INFO | "gpu" : " 17 GB", | 20260505 04:50:31.787 | INFO | "huge" : "", | 20260505 04:50:31.787 | INFO | "huge_shared" : " 27 GB", | 20260505 04:50:31.787 | INFO | "malloc" : " 14 GB", | 20260505 04:50:31.787 | INFO | "pinned" : " 29 GB" | 20260505 04:50:31.787 | INFO | } | 20260505 04:50:31.787 | INFO | Perf report for the last 5.1395 seconds: | 20260505 04:50:31.787 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:50:31.787 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:31.787 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:31.787 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:31.787 | INFO | ANALYSIS : 57.3%, 3.0, 89.0ms, 0 | 20260505 04:50:31.787 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:31.787 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:31.787 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 04:50:31.787 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:31.787 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:31.787 | INFO | Uploaded 36GiB to GPU in 1.6035s (22.4509GiB/s) | 20260505 04:50:31.788 | INFO | Downloaded 34GiB to GPU in 1.52099s (22.3538GiB/s) | 20260505 04:50:36.926 | INFO | Analyzing chunk frames = [472576,473088) | 20260505 04:50:36.926 | INFO | MemInfo: { | 20260505 04:50:36.926 | INFO | "gpu" : " 17 GB", | 20260505 04:50:36.926 | INFO | "huge" : "", | 20260505 04:50:36.926 | INFO | "huge_shared" : " 27 GB", | 20260505 04:50:36.926 | INFO | "malloc" : " 14 GB", | 20260505 04:50:36.926 | INFO | "pinned" : " 29 GB" | 20260505 04:50:36.926 | INFO | } | 20260505 04:50:36.926 | INFO | Perf report for the last 5.13921 seconds: | 20260505 04:50:36.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:50:36.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:36.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:36.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:36.926 | INFO | ANALYSIS : 57.1%, 3.0, 88.5ms, 0 | 20260505 04:50:36.926 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:36.927 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:36.927 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260505 04:50:36.927 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:36.927 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:36.927 | INFO | Uploaded 36GiB to GPU in 1.6022s (22.4691GiB/s) | 20260505 04:50:36.927 | INFO | Downloaded 34GiB to GPU in 1.5188s (22.3861GiB/s) | 20260505 04:50:42.065 | INFO | Analyzing chunk frames = [473088,473600) | 20260505 04:50:42.065 | INFO | MemInfo: { | 20260505 04:50:42.065 | INFO | "gpu" : " 17 GB", | 20260505 04:50:42.065 | INFO | "huge" : "", | 20260505 04:50:42.065 | INFO | "huge_shared" : " 27 GB", | 20260505 04:50:42.066 | INFO | "malloc" : " 14 GB", | 20260505 04:50:42.066 | INFO | "pinned" : " 29 GB" | 20260505 04:50:42.066 | INFO | } | 20260505 04:50:42.066 | INFO | Perf report for the last 5.13923 seconds: | 20260505 04:50:42.066 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:50:42.066 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:42.066 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:42.066 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:42.066 | INFO | ANALYSIS : 57.0%, 3.0, 88.8ms, 0 | 20260505 04:50:42.066 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:42.066 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:42.066 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 04:50:42.066 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:42.066 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:42.066 | INFO | Uploaded 36GiB to GPU in 1.60105s (22.4853GiB/s) | 20260505 04:50:42.066 | INFO | Downloaded 34GiB to GPU in 1.51787s (22.3998GiB/s) | 20260505 04:50:47.205 | INFO | Analyzing chunk frames = [473600,474112) | 20260505 04:50:47.205 | INFO | MemInfo: { | 20260505 04:50:47.205 | INFO | "gpu" : " 17 GB", | 20260505 04:50:47.205 | INFO | "huge" : "", | 20260505 04:50:47.205 | INFO | "huge_shared" : " 27 GB", | 20260505 04:50:47.205 | INFO | "malloc" : " 14 GB", | 20260505 04:50:47.205 | INFO | "pinned" : " 29 GB" | 20260505 04:50:47.205 | INFO | } | 20260505 04:50:47.205 | INFO | Perf report for the last 5.13954 seconds: | 20260505 04:50:47.205 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:50:47.205 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:47.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:47.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:47.205 | INFO | ANALYSIS : 57.3%, 3.0, 89.3ms, 0 | 20260505 04:50:47.205 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:47.205 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:47.205 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260505 04:50:47.205 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:47.205 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:47.206 | INFO | Uploaded 36GiB to GPU in 1.60102s (22.4856GiB/s) | 20260505 04:50:47.206 | INFO | Downloaded 34GiB to GPU in 1.51774s (22.4017GiB/s) | 20260505 04:50:52.344 | INFO | Analyzing chunk frames = [474112,474624) | 20260505 04:50:52.344 | INFO | MemInfo: { | 20260505 04:50:52.344 | INFO | "gpu" : " 17 GB", | 20260505 04:50:52.344 | INFO | "huge" : "", | 20260505 04:50:52.344 | INFO | "huge_shared" : " 27 GB", | 20260505 04:50:52.344 | INFO | "malloc" : " 14 GB", | 20260505 04:50:52.344 | INFO | "pinned" : " 29 GB" | 20260505 04:50:52.344 | INFO | } | 20260505 04:50:52.344 | INFO | Perf report for the last 5.13926 seconds: | 20260505 04:50:52.344 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:50:52.344 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:52.344 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:52.345 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:52.345 | INFO | ANALYSIS : 57.5%, 3.0, 89.6ms, 0 | 20260505 04:50:52.345 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:52.345 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:52.345 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260505 04:50:52.345 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:52.345 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:52.345 | INFO | Uploaded 36GiB to GPU in 1.60134s (22.4812GiB/s) | 20260505 04:50:52.345 | INFO | Downloaded 34GiB to GPU in 1.51854s (22.3899GiB/s) | 20260505 04:50:57.484 | INFO | Analyzing chunk frames = [474624,475136) | 20260505 04:50:57.484 | INFO | MemInfo: { | 20260505 04:50:57.484 | INFO | "gpu" : " 17 GB", | 20260505 04:50:57.484 | INFO | "huge" : "", | 20260505 04:50:57.484 | INFO | "huge_shared" : " 27 GB", | 20260505 04:50:57.484 | INFO | "malloc" : " 14 GB", | 20260505 04:50:57.484 | INFO | "pinned" : " 29 GB" | 20260505 04:50:57.484 | INFO | } | 20260505 04:50:57.484 | INFO | Perf report for the last 5.1393 seconds: | 20260505 04:50:57.484 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:50:57.484 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:57.484 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:57.484 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:57.484 | INFO | ANALYSIS : 57.6%, 3.0, 89.8ms, 0 | 20260505 04:50:57.484 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:57.484 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:57.484 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 04:50:57.484 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:57.484 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:50:57.484 | INFO | Uploaded 36GiB to GPU in 1.60113s (22.4841GiB/s) | 20260505 04:50:57.484 | INFO | Downloaded 34GiB to GPU in 1.51817s (22.3954GiB/s) | 20260505 04:50:57.484 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 04:50:57.484 | INFO | Total mem usage: 15.1 GiB | 20260505 04:50:57.484 | INFO | All Time High Water Marks sum: 15.7 GiB (Savings: 4.09%) | 20260505 04:50:57.484 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -4.2%) | 20260505 04:50:57.484 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 04:50:57.484 | INFO | High Water Breakdown: | 20260505 04:50:57.484 | INFO | 59.5% : 8.59 GiB ( 95.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 04:50:57.484 | INFO | 40.4% : 5.84 GiB ( 87.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 04:50:57.484 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 04:50:57.484 | INFO | ------------------------------------------ | 20260505 04:50:57.484 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:50:57.484 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 04:50:57.484 | INFO | Total mem usage: 26.5 GiB | 20260505 04:50:57.484 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260505 04:50:57.484 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 04:50:57.485 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260505 04:50:57.485 | INFO | High Water Breakdown: | 20260505 04:50:57.485 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:50:57.485 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:50:57.485 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:50:57.485 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:50:57.485 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:50:57.485 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:50:57.485 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 04:50:57.485 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 04:50:57.485 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:50:57.485 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 04:50:57.485 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 04:50:57.485 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 04:50:57.485 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 04:50:57.485 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 04:50:57.485 | INFO | ------------------------------------------ | 20260505 04:50:57.485 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:50:57.485 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 04:50:57.485 | INFO | Total mem usage: 25.6 GiB | 20260505 04:50:57.485 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 04:50:57.485 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 04:50:57.485 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 04:50:57.485 | INFO | High Water Breakdown: | 20260505 04:50:57.485 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 04:50:57.485 | INFO | ------------------------------------------ | 20260505 04:50:57.485 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:50:57.485 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 04:50:57.485 | INFO | Total mem usage: 15.7 GiB | 20260505 04:50:57.485 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260505 04:50:57.485 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 04:50:57.485 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260505 04:50:57.485 | INFO | High Water Breakdown: | 20260505 04:50:57.485 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 04:50:57.485 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 04:50:57.485 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 04:50:57.485 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 04:50:57.485 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 04:50:57.485 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 04:50:57.485 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 04:50:57.485 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 04:50:57.485 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 04:50:57.485 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:50:57.485 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 04:50:57.485 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:50:57.485 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 04:50:57.485 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 04:50:57.485 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 04:50:57.485 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 04:50:57.485 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:50:57.485 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 04:50:57.485 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:50:57.485 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:50:57.485 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:50:57.485 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 04:50:57.485 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 04:50:57.485 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:50:57.485 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 04:50:57.485 | INFO | ------------------------------------------ | 20260505 04:50:57.485 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:50:57.485 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 04:50:57.485 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:50:57.485 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:50:57.485 | INFO | PerfReport: Full Execution -- 3.04e+01 / -nan, 1.28e+01 / 0.00e+00 | 20260505 04:50:57.485 | INFO | PerfReport: 0.28%, AggregateStats -- 8.63e-02 / -nan, 3.93e-03 / 0.00e+00 | 20260505 04:50:57.485 | INFO | PerfReport: 10.84%, Baseline -- 3.30e+00 / -nan, 1.73e-02 / 0.00e+00 | 20260505 04:50:57.485 | INFO | PerfReport: 3.04%, Histogram -- 9.24e-01 / -nan, 5.23e-02 / 0.00e+00 | 20260505 04:50:57.485 | INFO | PerfReport: 16.04%, DME -- 3.90e+01 / -nan, 3.24e+00 / 0.00e+00 | 20260505 04:50:57.485 | INFO | PerfReport: 56.57%, FrameLabeling -- 1.72e+01 / -nan, 4.53e-02 / 0.00e+00 | 20260505 04:50:57.485 | INFO | PerfReport: 7.62%, PulseAccumulating -- 2.32e+00 / -nan, 1.54e-01 / 0.00e+00 | 20260505 04:50:57.485 | INFO | PerfReport: 3.97%, Metrics -- 1.21e+00 / -nan, 1.99e-01 / 0.00e+00 | 20260505 04:50:57.485 | INFO | PerfReport: 1.64%, PulseSerializing -- 4.99e-01 / -nan, 8.05e-02 / 0.00e+00 | 20260505 04:50:57.485 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 04:50:57.485 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:50:57.485 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:50:57.485 | INFO | PerfReport: Full Execution -- 9.14e+01 / -nan, 1.90e+01 / 0.00e+00 | 20260505 04:50:57.485 | INFO | PerfReport: 18.31%, Upload -- 1.67e+01 / 0.00e+00, 3.75e-01 / 0.00e+00 | 20260505 04:50:57.485 | INFO | PerfReport: 17.57%, Download -- 1.61e+01 / 0.00e+00, 7.04e-01 / 0.00e+00 | 20260505 04:50:57.485 | INFO | PerfReport: 33.30%, Compute -- 3.04e+01 / 0.00e+00, 1.28e+01 / 0.00e+00 | 20260505 04:50:57.485 | INFO | PerfReport: 0.55%, LockUpload -- 5.03e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260505 04:50:57.485 | INFO | PerfReport: 0.00%, LockDownload -- 3.90e-04 / 0.00e+00, 9.36e-05 / 0.00e+00 | 20260505 04:50:57.485 | INFO | PerfReport: 30.24%, LockCompute -- 2.76e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260505 04:50:57.485 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.65e-02 / 0.00e+00 | 20260505 04:50:57.581 | INFO | Starting a round of Baz IO | 20260505 04:51:00.621 | INFO | Populating RTMetrics report | 20260505 04:51:00.621 | INFO | RTMetrics report populated | 20260505 04:51:00.621 | INFO | Opening tmp RTMetrics json file | 20260505 04:51:00.636 | INFO | Serializing RTMetrics to json | 20260505 04:51:00.637 | INFO | Serialization done, swapping tmp file now... | 20260505 04:51:00.638 | INFO | RTMetrics done | 20260505 04:51:00.719 | INFO | Finished a round of Baz IO: Wrote 2.085 GiB in 3.137 seconds | 20260505 04:51:00.719 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.88% / 0.12% / 56.01% | 20260505 04:51:00.719 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.50533% / 0.000242392% / 5.5631e-05% | 20260505 04:51:00.719 | INFO | Baz IO overall: | 20260505 04:51:00.719 | INFO | BytesWritten : 201129242319 | 20260505 04:51:00.719 | INFO | NumEvents : 93696576588 | 20260505 04:51:02.623 | INFO | Analyzing chunk frames = [475136,475648) | 20260505 04:51:02.623 | INFO | MemInfo: { | 20260505 04:51:02.623 | INFO | "gpu" : " 17 GB", | 20260505 04:51:02.623 | INFO | "huge" : "", | 20260505 04:51:02.623 | INFO | "huge_shared" : " 27 GB", | 20260505 04:51:02.623 | INFO | "malloc" : " 14 GB", | 20260505 04:51:02.623 | INFO | "pinned" : " 29 GB" | 20260505 04:51:02.623 | INFO | } | 20260505 04:51:02.623 | INFO | Perf report for the last 5.13918 seconds: | 20260505 04:51:02.623 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:51:02.623 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:02.623 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:02.623 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:02.623 | INFO | ANALYSIS : 59.9%, 2.9, 92.4ms, 0 | 20260505 04:51:02.623 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:02.623 | INFO | RT_METRICS : 47.6%, 1.0, 24.9ms, 0 | 20260505 04:51:02.623 | INFO | PRE_HQ : 59.3%, 2.2, 67.5ms, 0 | 20260505 04:51:02.623 | INFO | BAZWRITER : 60.7%, 4.0, 124.9ms, 0 | 20260505 04:51:02.623 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 04:51:02.623 | INFO | Uploaded 36GiB to GPU in 1.74081s (20.68GiB/s) | 20260505 04:51:02.623 | INFO | Downloaded 38GiB to GPU in 1.81006s (20.9938GiB/s) | 20260505 04:51:07.762 | INFO | Analyzing chunk frames = [475648,476160) | 20260505 04:51:07.762 | INFO | MemInfo: { | 20260505 04:51:07.762 | INFO | "gpu" : " 17 GB", | 20260505 04:51:07.762 | INFO | "huge" : "", | 20260505 04:51:07.762 | INFO | "huge_shared" : " 27 GB", | 20260505 04:51:07.762 | INFO | "malloc" : " 14 GB", | 20260505 04:51:07.763 | INFO | "pinned" : " 29 GB" | 20260505 04:51:07.763 | INFO | } | 20260505 04:51:07.763 | INFO | Perf report for the last 5.13959 seconds: | 20260505 04:51:07.763 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:51:07.763 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:07.763 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:07.763 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:07.763 | INFO | ANALYSIS : 58.0%, 3.0, 90.4ms, 0 | 20260505 04:51:07.763 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:07.763 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:07.763 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 04:51:07.763 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:07.763 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:07.763 | INFO | Uploaded 36GiB to GPU in 1.59645s (22.55GiB/s) | 20260505 04:51:07.763 | INFO | Downloaded 34GiB to GPU in 1.51648s (22.4204GiB/s) | 20260505 04:51:12.902 | INFO | Analyzing chunk frames = [476160,476672) | 20260505 04:51:12.902 | INFO | MemInfo: { | 20260505 04:51:12.902 | INFO | "gpu" : " 17 GB", | 20260505 04:51:12.902 | INFO | "huge" : "", | 20260505 04:51:12.902 | INFO | "huge_shared" : " 27 GB", | 20260505 04:51:12.902 | INFO | "malloc" : " 14 GB", | 20260505 04:51:12.902 | INFO | "pinned" : " 29 GB" | 20260505 04:51:12.902 | INFO | } | 20260505 04:51:12.902 | INFO | Perf report for the last 5.13942 seconds: | 20260505 04:51:12.902 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:51:12.902 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:12.902 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:12.902 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:12.902 | INFO | ANALYSIS : 57.3%, 3.0, 89.1ms, 0 | 20260505 04:51:12.902 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:12.902 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:12.902 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 04:51:12.902 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:12.902 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:12.903 | INFO | Uploaded 36GiB to GPU in 1.60023s (22.4968GiB/s) | 20260505 04:51:12.903 | INFO | Downloaded 34GiB to GPU in 1.52085s (22.356GiB/s) | 20260505 04:51:18.041 | INFO | Analyzing chunk frames = [476672,477184) | 20260505 04:51:18.041 | INFO | MemInfo: { | 20260505 04:51:18.041 | INFO | "gpu" : " 17 GB", | 20260505 04:51:18.041 | INFO | "huge" : "", | 20260505 04:51:18.041 | INFO | "huge_shared" : " 27 GB", | 20260505 04:51:18.041 | INFO | "malloc" : " 14 GB", | 20260505 04:51:18.041 | INFO | "pinned" : " 29 GB" | 20260505 04:51:18.041 | INFO | } | 20260505 04:51:18.041 | INFO | Perf report for the last 5.1391 seconds: | 20260505 04:51:18.041 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:51:18.041 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:18.041 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:18.041 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:18.041 | INFO | ANALYSIS : 57.0%, 3.0, 88.5ms, 0 | 20260505 04:51:18.041 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:18.041 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:18.041 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260505 04:51:18.041 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:18.041 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:18.041 | INFO | Uploaded 36GiB to GPU in 1.60004s (22.4994GiB/s) | 20260505 04:51:18.041 | INFO | Downloaded 34GiB to GPU in 1.52044s (22.3619GiB/s) | 20260505 04:51:23.180 | INFO | Analyzing chunk frames = [477184,477696) | 20260505 04:51:23.180 | INFO | MemInfo: { | 20260505 04:51:23.180 | INFO | "gpu" : " 17 GB", | 20260505 04:51:23.180 | INFO | "huge" : "", | 20260505 04:51:23.181 | INFO | "huge_shared" : " 27 GB", | 20260505 04:51:23.181 | INFO | "malloc" : " 14 GB", | 20260505 04:51:23.181 | INFO | "pinned" : " 29 GB" | 20260505 04:51:23.181 | INFO | } | 20260505 04:51:23.181 | INFO | Perf report for the last 5.13954 seconds: | 20260505 04:51:23.181 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:51:23.181 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:23.181 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:23.181 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:23.181 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260505 04:51:23.181 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:23.181 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:23.181 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 04:51:23.181 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:23.181 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:23.182 | INFO | Uploaded 36GiB to GPU in 1.59786s (22.5302GiB/s) | 20260505 04:51:23.182 | INFO | Downloaded 34GiB to GPU in 1.51796s (22.3985GiB/s) | 20260505 04:51:28.320 | INFO | Analyzing chunk frames = [477696,478208) | 20260505 04:51:28.320 | INFO | MemInfo: { | 20260505 04:51:28.320 | INFO | "gpu" : " 17 GB", | 20260505 04:51:28.320 | INFO | "huge" : "", | 20260505 04:51:28.320 | INFO | "huge_shared" : " 27 GB", | 20260505 04:51:28.320 | INFO | "malloc" : " 14 GB", | 20260505 04:51:28.320 | INFO | "pinned" : " 29 GB" | 20260505 04:51:28.320 | INFO | } | 20260505 04:51:28.320 | INFO | Perf report for the last 5.13923 seconds: | 20260505 04:51:28.320 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:51:28.320 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:28.320 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:28.320 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:28.320 | INFO | ANALYSIS : 57.3%, 3.0, 89.3ms, 0 | 20260505 04:51:28.320 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:28.320 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:28.320 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 04:51:28.320 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:28.320 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:28.320 | INFO | Uploaded 36GiB to GPU in 1.59735s (22.5373GiB/s) | 20260505 04:51:28.320 | INFO | Downloaded 34GiB to GPU in 1.51791s (22.3992GiB/s) | 20260505 04:51:33.459 | INFO | Analyzing chunk frames = [478208,478720) | 20260505 04:51:33.459 | INFO | MemInfo: { | 20260505 04:51:33.459 | INFO | "gpu" : " 17 GB", | 20260505 04:51:33.459 | INFO | "huge" : "", | 20260505 04:51:33.459 | INFO | "huge_shared" : " 27 GB", | 20260505 04:51:33.459 | INFO | "malloc" : " 14 GB", | 20260505 04:51:33.459 | INFO | "pinned" : " 29 GB" | 20260505 04:51:33.459 | INFO | } | 20260505 04:51:33.459 | INFO | Perf report for the last 5.13937 seconds: | 20260505 04:51:33.459 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:51:33.459 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:33.459 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:33.459 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:33.459 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260505 04:51:33.459 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:33.459 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:33.459 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260505 04:51:33.459 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:33.459 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:33.460 | INFO | Uploaded 36GiB to GPU in 1.59892s (22.5153GiB/s) | 20260505 04:51:33.460 | INFO | Downloaded 34GiB to GPU in 1.51964s (22.3737GiB/s) | 20260505 04:51:38.598 | INFO | Analyzing chunk frames = [478720,479232) | 20260505 04:51:38.598 | INFO | MemInfo: { | 20260505 04:51:38.598 | INFO | "gpu" : " 17 GB", | 20260505 04:51:38.598 | INFO | "huge" : "", | 20260505 04:51:38.598 | INFO | "huge_shared" : " 27 GB", | 20260505 04:51:38.598 | INFO | "malloc" : " 14 GB", | 20260505 04:51:38.598 | INFO | "pinned" : " 29 GB" | 20260505 04:51:38.598 | INFO | } | 20260505 04:51:38.599 | INFO | Perf report for the last 5.1392 seconds: | 20260505 04:51:38.599 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:51:38.599 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:38.599 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:38.599 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:38.599 | INFO | ANALYSIS : 57.6%, 3.0, 89.9ms, 0 | 20260505 04:51:38.599 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:38.599 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:38.599 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260505 04:51:38.599 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:38.599 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:38.599 | INFO | Uploaded 36GiB to GPU in 1.59864s (22.5191GiB/s) | 20260505 04:51:38.599 | INFO | Downloaded 34GiB to GPU in 1.5194s (22.3772GiB/s) | 20260505 04:51:38.696 | INFO | Starting a round of Baz IO | 20260505 04:51:41.648 | INFO | Populating RTMetrics report | 20260505 04:51:41.648 | INFO | RTMetrics report populated | 20260505 04:51:41.648 | INFO | Opening tmp RTMetrics json file | 20260505 04:51:41.649 | INFO | Serializing RTMetrics to json | 20260505 04:51:41.651 | INFO | Serialization done, swapping tmp file now... | 20260505 04:51:41.654 | INFO | RTMetrics done | 20260505 04:51:41.743 | INFO | Finished a round of Baz IO: Wrote 2.092 GiB in 3.046 seconds | 20260505 04:51:41.743 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.76% / 0.12% / 56.12% | 20260505 04:51:41.743 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.47982% / 0.000214577% / 5.96046e-05% | 20260505 04:51:41.743 | INFO | Baz IO overall: | 20260505 04:51:41.743 | INFO | BytesWritten : 203375298015 | 20260505 04:51:41.743 | INFO | NumEvents : 94763116900 | 20260505 04:51:43.738 | INFO | Analyzing chunk frames = [479232,479744) | 20260505 04:51:43.738 | INFO | MemInfo: { | 20260505 04:51:43.738 | INFO | "gpu" : " 17 GB", | 20260505 04:51:43.738 | INFO | "huge" : "", | 20260505 04:51:43.738 | INFO | "huge_shared" : " 27 GB", | 20260505 04:51:43.738 | INFO | "malloc" : " 14 GB", | 20260505 04:51:43.738 | INFO | "pinned" : " 29 GB" | 20260505 04:51:43.738 | INFO | } | 20260505 04:51:43.738 | INFO | Perf report for the last 5.1394 seconds: | 20260505 04:51:43.738 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:51:43.738 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:43.738 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:43.738 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:43.738 | INFO | ANALYSIS : 58.2%, 3.0, 90.4ms, 0 | 20260505 04:51:43.738 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:43.738 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260505 04:51:43.738 | INFO | PRE_HQ : 57.5%, 2.2, 64.4ms, 0 | 20260505 04:51:43.738 | INFO | BAZWRITER : 58.9%, 4.1, 122.7ms, 0 | 20260505 04:51:43.738 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 04:51:43.739 | INFO | Uploaded 36GiB to GPU in 1.63501s (22.0182GiB/s) | 20260505 04:51:43.739 | INFO | Downloaded 38GiB to GPU in 1.70003s (22.3525GiB/s) | 20260505 04:51:48.877 | INFO | Analyzing chunk frames = [479744,480256) | 20260505 04:51:48.877 | INFO | MemInfo: { | 20260505 04:51:48.877 | INFO | "gpu" : " 17 GB", | 20260505 04:51:48.877 | INFO | "huge" : "", | 20260505 04:51:48.877 | INFO | "huge_shared" : " 27 GB", | 20260505 04:51:48.877 | INFO | "malloc" : " 14 GB", | 20260505 04:51:48.877 | INFO | "pinned" : " 29 GB" | 20260505 04:51:48.877 | INFO | } | 20260505 04:51:48.877 | INFO | Perf report for the last 5.13944 seconds: | 20260505 04:51:48.877 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:51:48.877 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:48.877 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:48.877 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:48.877 | INFO | ANALYSIS : 57.9%, 3.0, 90.4ms, 0 | 20260505 04:51:48.877 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:48.878 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:48.878 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 04:51:48.878 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:48.878 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:48.878 | INFO | Uploaded 36GiB to GPU in 1.59988s (22.5017GiB/s) | 20260505 04:51:48.878 | INFO | Downloaded 34GiB to GPU in 1.51655s (22.4193GiB/s) | 20260505 04:51:54.016 | INFO | Analyzing chunk frames = [480256,480768) | 20260505 04:51:54.016 | INFO | MemInfo: { | 20260505 04:51:54.016 | INFO | "gpu" : " 17 GB", | 20260505 04:51:54.016 | INFO | "huge" : "", | 20260505 04:51:54.017 | INFO | "huge_shared" : " 27 GB", | 20260505 04:51:54.017 | INFO | "malloc" : " 14 GB", | 20260505 04:51:54.017 | INFO | "pinned" : " 29 GB" | 20260505 04:51:54.017 | INFO | } | 20260505 04:51:54.017 | INFO | Perf report for the last 5.13931 seconds: | 20260505 04:51:54.017 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:51:54.017 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:54.017 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:54.017 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:54.017 | INFO | ANALYSIS : 57.4%, 3.0, 89.1ms, 0 | 20260505 04:51:54.017 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:54.017 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:54.017 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260505 04:51:54.017 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:54.017 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:54.018 | INFO | Uploaded 36GiB to GPU in 1.60312s (22.4563GiB/s) | 20260505 04:51:54.018 | INFO | Downloaded 34GiB to GPU in 1.52075s (22.3574GiB/s) | 20260505 04:51:59.156 | INFO | Analyzing chunk frames = [480768,481280) | 20260505 04:51:59.156 | INFO | MemInfo: { | 20260505 04:51:59.156 | INFO | "gpu" : " 17 GB", | 20260505 04:51:59.156 | INFO | "huge" : "", | 20260505 04:51:59.156 | INFO | "huge_shared" : " 27 GB", | 20260505 04:51:59.156 | INFO | "malloc" : " 14 GB", | 20260505 04:51:59.156 | INFO | "pinned" : " 29 GB" | 20260505 04:51:59.156 | INFO | } | 20260505 04:51:59.156 | INFO | Perf report for the last 5.13967 seconds: | 20260505 04:51:59.156 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:51:59.156 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:59.156 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:59.156 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:59.156 | INFO | ANALYSIS : 57.0%, 3.0, 88.4ms, 0 | 20260505 04:51:59.156 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:59.156 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:59.157 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 04:51:59.157 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:59.157 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:51:59.157 | INFO | Uploaded 36GiB to GPU in 1.60221s (22.469GiB/s) | 20260505 04:51:59.157 | INFO | Downloaded 34GiB to GPU in 1.51908s (22.3819GiB/s) | 20260505 04:52:04.295 | INFO | Analyzing chunk frames = [481280,481792) | 20260505 04:52:04.295 | INFO | MemInfo: { | 20260505 04:52:04.295 | INFO | "gpu" : " 17 GB", | 20260505 04:52:04.295 | INFO | "huge" : "", | 20260505 04:52:04.295 | INFO | "huge_shared" : " 27 GB", | 20260505 04:52:04.295 | INFO | "malloc" : " 14 GB", | 20260505 04:52:04.295 | INFO | "pinned" : " 29 GB" | 20260505 04:52:04.295 | INFO | } | 20260505 04:52:04.295 | INFO | Perf report for the last 5.13905 seconds: | 20260505 04:52:04.295 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:52:04.295 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:04.296 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:04.296 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:04.296 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260505 04:52:04.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:04.296 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:04.296 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 04:52:04.296 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:04.296 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:04.296 | INFO | Uploaded 36GiB to GPU in 1.60107s (22.485GiB/s) | 20260505 04:52:04.296 | INFO | Downloaded 34GiB to GPU in 1.51762s (22.4035GiB/s) | 20260505 04:52:09.434 | INFO | Analyzing chunk frames = [481792,482304) | 20260505 04:52:09.435 | INFO | MemInfo: { | 20260505 04:52:09.435 | INFO | "gpu" : " 17 GB", | 20260505 04:52:09.435 | INFO | "huge" : "", | 20260505 04:52:09.435 | INFO | "huge_shared" : " 27 GB", | 20260505 04:52:09.435 | INFO | "malloc" : " 14 GB", | 20260505 04:52:09.435 | INFO | "pinned" : " 29 GB" | 20260505 04:52:09.435 | INFO | } | 20260505 04:52:09.435 | INFO | Perf report for the last 5.13934 seconds: | 20260505 04:52:09.435 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:52:09.435 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:09.435 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:09.435 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:09.435 | INFO | ANALYSIS : 57.2%, 3.0, 89.3ms, 0 | 20260505 04:52:09.435 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:09.435 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:09.435 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 04:52:09.435 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:09.435 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:09.435 | INFO | Uploaded 36GiB to GPU in 1.60109s (22.4847GiB/s) | 20260505 04:52:09.435 | INFO | Downloaded 34GiB to GPU in 1.51804s (22.3972GiB/s) | 20260505 04:52:14.574 | INFO | Analyzing chunk frames = [482304,482816) | 20260505 04:52:14.574 | INFO | MemInfo: { | 20260505 04:52:14.574 | INFO | "gpu" : " 17 GB", | 20260505 04:52:14.574 | INFO | "huge" : "", | 20260505 04:52:14.574 | INFO | "huge_shared" : " 27 GB", | 20260505 04:52:14.574 | INFO | "malloc" : " 14 GB", | 20260505 04:52:14.574 | INFO | "pinned" : " 29 GB" | 20260505 04:52:14.574 | INFO | } | 20260505 04:52:14.574 | INFO | Perf report for the last 5.13942 seconds: | 20260505 04:52:14.574 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:52:14.574 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:14.574 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:14.574 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:14.574 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260505 04:52:14.574 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:14.574 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:14.574 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260505 04:52:14.574 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:14.574 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:14.575 | INFO | Uploaded 36GiB to GPU in 1.60087s (22.4878GiB/s) | 20260505 04:52:14.575 | INFO | Downloaded 34GiB to GPU in 1.5185s (22.3905GiB/s) | 20260505 04:52:19.713 | INFO | Analyzing chunk frames = [482816,483328) | 20260505 04:52:19.713 | INFO | MemInfo: { | 20260505 04:52:19.713 | INFO | "gpu" : " 17 GB", | 20260505 04:52:19.713 | INFO | "huge" : "", | 20260505 04:52:19.713 | INFO | "huge_shared" : " 27 GB", | 20260505 04:52:19.713 | INFO | "malloc" : " 14 GB", | 20260505 04:52:19.713 | INFO | "pinned" : " 29 GB" | 20260505 04:52:19.713 | INFO | } | 20260505 04:52:19.713 | INFO | Perf report for the last 5.13898 seconds: | 20260505 04:52:19.713 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:52:19.713 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:19.713 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:19.713 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:19.713 | INFO | ANALYSIS : 57.6%, 3.0, 89.8ms, 0 | 20260505 04:52:19.713 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:19.713 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:19.713 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 04:52:19.713 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:19.713 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:19.714 | INFO | Uploaded 36GiB to GPU in 1.60178s (22.475GiB/s) | 20260505 04:52:19.714 | INFO | Downloaded 34GiB to GPU in 1.51931s (22.3786GiB/s) | 20260505 04:52:19.714 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 04:52:19.714 | INFO | Total mem usage: 15.1 GiB | 20260505 04:52:19.714 | INFO | All Time High Water Marks sum: 15.7 GiB (Savings: 4.09%) | 20260505 04:52:19.714 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -3.77%) | 20260505 04:52:19.714 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 04:52:19.714 | INFO | High Water Breakdown: | 20260505 04:52:19.714 | INFO | 59.5% : 8.63 GiB ( 95.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 04:52:19.714 | INFO | 40.4% : 5.86 GiB ( 88.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 04:52:19.714 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 04:52:19.714 | INFO | ------------------------------------------ | 20260505 04:52:19.714 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:52:19.714 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 04:52:19.714 | INFO | Total mem usage: 26.5 GiB | 20260505 04:52:19.714 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 04:52:19.714 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260505 04:52:19.714 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260505 04:52:19.714 | INFO | High Water Breakdown: | 20260505 04:52:19.714 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:52:19.714 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:52:19.714 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:52:19.714 | INFO | 2% : 525 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:52:19.714 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:52:19.714 | INFO | 0.695% : 182 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:52:19.714 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 04:52:19.714 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 04:52:19.714 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:52:19.714 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 04:52:19.714 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 04:52:19.714 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 04:52:19.714 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 04:52:19.714 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 04:52:19.714 | INFO | ------------------------------------------ | 20260505 04:52:19.714 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:52:19.714 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 04:52:19.714 | INFO | Total mem usage: 25.6 GiB | 20260505 04:52:19.714 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 04:52:19.714 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 04:52:19.714 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 04:52:19.714 | INFO | High Water Breakdown: | 20260505 04:52:19.714 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 04:52:19.714 | INFO | ------------------------------------------ | 20260505 04:52:19.714 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:52:19.714 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 04:52:19.714 | INFO | Total mem usage: 15.7 GiB | 20260505 04:52:19.715 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 04:52:19.715 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 5.01%) | 20260505 04:52:19.715 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260505 04:52:19.715 | INFO | High Water Breakdown: | 20260505 04:52:19.715 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 04:52:19.715 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 04:52:19.715 | INFO | 8.52% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 04:52:19.715 | INFO | 8.4% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 04:52:19.715 | INFO | 6.25% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 04:52:19.715 | INFO | 6.11% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 04:52:19.715 | INFO | 4.83% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 04:52:19.715 | INFO | 3.41% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 04:52:19.715 | INFO | 3.06% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 04:52:19.715 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:52:19.715 | INFO | 2.27% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 04:52:19.715 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:52:19.715 | INFO | 1.65% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 04:52:19.715 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 04:52:19.715 | INFO | 0.758% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 04:52:19.715 | INFO | 0.758% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 04:52:19.715 | INFO | 0.621% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:52:19.715 | INFO | 0.568% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 04:52:19.715 | INFO | 0.432% : 73 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:52:19.715 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:52:19.715 | INFO | 0.071% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:52:19.715 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 04:52:19.715 | INFO | 0.00592% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 04:52:19.715 | INFO | 1.35e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:52:19.715 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 04:52:19.715 | INFO | ------------------------------------------ | 20260505 04:52:19.715 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:52:19.715 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 04:52:19.715 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:52:19.715 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:52:19.715 | INFO | PerfReport: Full Execution -- 3.05e+01 / 9.30e+01, 1.28e+01 / 0.00e+00 | 20260505 04:52:19.715 | INFO | PerfReport: 0.28%, AggregateStats -- 8.65e-02 / 8.72e-02, 1.14e-02 / 0.00e+00 | 20260505 04:52:19.715 | INFO | PerfReport: 10.96%, Baseline -- 3.34e+00 / 7.04e+01, 1.70e+00 / 0.00e+00 | 20260505 04:52:19.715 | INFO | PerfReport: 3.03%, Histogram -- 9.24e-01 / 9.12e-01, 5.21e-02 / 0.00e+00 | 20260505 04:52:19.715 | INFO | PerfReport: 16.01%, DME -- 3.90e+01 / -nan, 3.21e+00 / 0.00e+00 | 20260505 04:52:19.715 | INFO | PerfReport: 56.47%, FrameLabeling -- 1.72e+01 / 1.72e+01, 4.52e-02 / 0.00e+00 | 20260505 04:52:19.715 | INFO | PerfReport: 7.63%, PulseAccumulating -- 2.33e+00 / 2.34e+00, 2.99e-01 / 0.00e+00 | 20260505 04:52:19.715 | INFO | PerfReport: 3.97%, Metrics -- 1.21e+00 / 1.47e+00, 2.01e-01 / 0.00e+00 | 20260505 04:52:19.715 | INFO | PerfReport: 1.64%, PulseSerializing -- 4.99e-01 / 5.18e-01, 8.17e-02 / 0.00e+00 | 20260505 04:52:19.715 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 04:52:19.715 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:52:19.715 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:52:19.715 | INFO | PerfReport: Full Execution -- 9.15e+01 / -nan, 1.91e+01 / 0.00e+00 | 20260505 04:52:19.715 | INFO | PerfReport: 18.34%, Upload -- 1.68e+01 / 0.00e+00, 1.83e+00 / 0.00e+00 | 20260505 04:52:19.715 | INFO | PerfReport: 17.63%, Download -- 1.61e+01 / 0.00e+00, 2.06e+00 / 0.00e+00 | 20260505 04:52:19.715 | INFO | PerfReport: 33.32%, Compute -- 3.05e+01 / 0.00e+00, 1.28e+01 / 0.00e+00 | 20260505 04:52:19.715 | INFO | PerfReport: 0.55%, LockUpload -- 5.05e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260505 04:52:19.715 | INFO | PerfReport: 0.01%, LockDownload -- 7.78e-03 / 0.00e+00, 2.88e-01 / 0.00e+00 | 20260505 04:52:19.715 | INFO | PerfReport: 30.13%, LockCompute -- 2.76e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260505 04:52:19.715 | INFO | PerfReport: 0.02%, Misc -- 1.43e-02 / 0.00e+00, 2.06e-02 / 0.00e+00 | 20260505 04:52:19.811 | INFO | Starting a round of Baz IO | 20260505 04:52:22.763 | INFO | Populating RTMetrics report | 20260505 04:52:22.763 | INFO | RTMetrics report populated | 20260505 04:52:22.763 | INFO | Opening tmp RTMetrics json file | 20260505 04:52:22.767 | INFO | Serializing RTMetrics to json | 20260505 04:52:22.768 | INFO | Serialization done, swapping tmp file now... | 20260505 04:52:22.768 | INFO | RTMetrics done | 20260505 04:52:22.854 | INFO | Finished a round of Baz IO: Wrote 2.093 GiB in 3.043 seconds | 20260505 04:52:22.854 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.65% / 0.11% / 56.24% | 20260505 04:52:22.854 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.40599% / 0.000234445% / 3.97364e-05% | 20260505 04:52:22.854 | INFO | Baz IO overall: | 20260505 04:52:22.855 | INFO | BytesWritten : 205622275965 | 20260505 04:52:22.855 | INFO | NumEvents : 95829482522 | 20260505 04:52:24.853 | INFO | Analyzing chunk frames = [483328,483840) | 20260505 04:52:24.853 | INFO | MemInfo: { | 20260505 04:52:24.853 | INFO | "gpu" : " 17 GB", | 20260505 04:52:24.853 | INFO | "huge" : "", | 20260505 04:52:24.853 | INFO | "huge_shared" : " 27 GB", | 20260505 04:52:24.853 | INFO | "malloc" : " 14 GB", | 20260505 04:52:24.853 | INFO | "pinned" : " 29 GB" | 20260505 04:52:24.853 | INFO | } | 20260505 04:52:24.853 | INFO | Perf report for the last 5.13938 seconds: | 20260505 04:52:24.853 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:52:24.853 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:24.853 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:24.853 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:24.853 | INFO | ANALYSIS : 58.2%, 3.0, 90.5ms, 0 | 20260505 04:52:24.853 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:24.853 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260505 04:52:24.853 | INFO | PRE_HQ : 57.5%, 2.2, 64.4ms, 0 | 20260505 04:52:24.853 | INFO | BAZWRITER : 58.8%, 4.1, 122.5ms, 0 | 20260505 04:52:24.853 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:52:24.853 | INFO | Uploaded 36GiB to GPU in 1.63923s (21.9616GiB/s) | 20260505 04:52:24.853 | INFO | Downloaded 38GiB to GPU in 1.70108s (22.3388GiB/s) | 20260505 04:52:29.992 | INFO | Analyzing chunk frames = [483840,484352) | 20260505 04:52:29.992 | INFO | MemInfo: { | 20260505 04:52:29.992 | INFO | "gpu" : " 17 GB", | 20260505 04:52:29.992 | INFO | "huge" : "", | 20260505 04:52:29.992 | INFO | "huge_shared" : " 27 GB", | 20260505 04:52:29.992 | INFO | "malloc" : " 14 GB", | 20260505 04:52:29.992 | INFO | "pinned" : " 29 GB" | 20260505 04:52:29.992 | INFO | } | 20260505 04:52:29.992 | INFO | Perf report for the last 5.1398 seconds: | 20260505 04:52:29.992 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:52:29.992 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:29.992 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:29.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:29.993 | INFO | ANALYSIS : 57.9%, 3.0, 90.4ms, 0 | 20260505 04:52:29.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:29.993 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:29.993 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 04:52:29.993 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:29.993 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:29.993 | INFO | Uploaded 36GiB to GPU in 1.59669s (22.5467GiB/s) | 20260505 04:52:29.993 | INFO | Downloaded 34GiB to GPU in 1.51611s (22.4259GiB/s) | 20260505 04:52:35.131 | INFO | Analyzing chunk frames = [484352,484864) | 20260505 04:52:35.131 | INFO | MemInfo: { | 20260505 04:52:35.131 | INFO | "gpu" : " 17 GB", | 20260505 04:52:35.131 | INFO | "huge" : "", | 20260505 04:52:35.131 | INFO | "huge_shared" : " 27 GB", | 20260505 04:52:35.131 | INFO | "malloc" : " 14 GB", | 20260505 04:52:35.131 | INFO | "pinned" : " 29 GB" | 20260505 04:52:35.131 | INFO | } | 20260505 04:52:35.132 | INFO | Perf report for the last 5.13903 seconds: | 20260505 04:52:35.132 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:52:35.132 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:35.132 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:35.132 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:35.132 | INFO | ANALYSIS : 57.4%, 3.0, 89.1ms, 0 | 20260505 04:52:35.132 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:35.132 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:35.132 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 04:52:35.132 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:35.132 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:35.132 | INFO | Uploaded 36GiB to GPU in 1.60025s (22.4965GiB/s) | 20260505 04:52:35.132 | INFO | Downloaded 34GiB to GPU in 1.51984s (22.3707GiB/s) | 20260505 04:52:40.271 | INFO | Analyzing chunk frames = [484864,485376) | 20260505 04:52:40.271 | INFO | MemInfo: { | 20260505 04:52:40.271 | INFO | "gpu" : " 17 GB", | 20260505 04:52:40.271 | INFO | "huge" : "", | 20260505 04:52:40.271 | INFO | "huge_shared" : " 27 GB", | 20260505 04:52:40.271 | INFO | "malloc" : " 14 GB", | 20260505 04:52:40.271 | INFO | "pinned" : " 29 GB" | 20260505 04:52:40.271 | INFO | } | 20260505 04:52:40.271 | INFO | Perf report for the last 5.13935 seconds: | 20260505 04:52:40.271 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:52:40.271 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:40.271 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:40.271 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:40.271 | INFO | ANALYSIS : 57.0%, 3.0, 88.4ms, 0 | 20260505 04:52:40.271 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:40.271 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:40.271 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 04:52:40.271 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:40.271 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:40.271 | INFO | Uploaded 36GiB to GPU in 1.60006s (22.4991GiB/s) | 20260505 04:52:40.271 | INFO | Downloaded 34GiB to GPU in 1.52004s (22.3678GiB/s) | 20260505 04:52:45.410 | INFO | Analyzing chunk frames = [485376,485888) | 20260505 04:52:45.410 | INFO | MemInfo: { | 20260505 04:52:45.410 | INFO | "gpu" : " 17 GB", | 20260505 04:52:45.410 | INFO | "huge" : "", | 20260505 04:52:45.410 | INFO | "huge_shared" : " 27 GB", | 20260505 04:52:45.410 | INFO | "malloc" : " 14 GB", | 20260505 04:52:45.410 | INFO | "pinned" : " 29 GB" | 20260505 04:52:45.410 | INFO | } | 20260505 04:52:45.410 | INFO | Perf report for the last 5.13926 seconds: | 20260505 04:52:45.410 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:52:45.410 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:45.410 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:45.410 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:45.410 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260505 04:52:45.410 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:45.410 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:45.410 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 04:52:45.410 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:45.410 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:45.410 | INFO | Uploaded 36GiB to GPU in 1.59803s (22.5278GiB/s) | 20260505 04:52:45.410 | INFO | Downloaded 34GiB to GPU in 1.51712s (22.4109GiB/s) | 20260505 04:52:50.550 | INFO | Analyzing chunk frames = [485888,486400) | 20260505 04:52:50.550 | INFO | MemInfo: { | 20260505 04:52:50.550 | INFO | "gpu" : " 17 GB", | 20260505 04:52:50.550 | INFO | "huge" : "", | 20260505 04:52:50.550 | INFO | "huge_shared" : " 27 GB", | 20260505 04:52:50.550 | INFO | "malloc" : " 14 GB", | 20260505 04:52:50.550 | INFO | "pinned" : " 29 GB" | 20260505 04:52:50.550 | INFO | } | 20260505 04:52:50.550 | INFO | Perf report for the last 5.13998 seconds: | 20260505 04:52:50.550 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:52:50.550 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:50.550 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:50.550 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:50.550 | INFO | ANALYSIS : 57.2%, 3.0, 89.3ms, 0 | 20260505 04:52:50.550 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:50.550 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:50.550 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 04:52:50.550 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:50.550 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:50.550 | INFO | Uploaded 36GiB to GPU in 1.59861s (22.5196GiB/s) | 20260505 04:52:50.550 | INFO | Downloaded 34GiB to GPU in 1.51865s (22.3883GiB/s) | 20260505 04:52:55.689 | INFO | Analyzing chunk frames = [486400,486912) | 20260505 04:52:55.689 | INFO | MemInfo: { | 20260505 04:52:55.689 | INFO | "gpu" : " 17 GB", | 20260505 04:52:55.689 | INFO | "huge" : "", | 20260505 04:52:55.689 | INFO | "huge_shared" : " 27 GB", | 20260505 04:52:55.689 | INFO | "malloc" : " 14 GB", | 20260505 04:52:55.689 | INFO | "pinned" : " 29 GB" | 20260505 04:52:55.689 | INFO | } | 20260505 04:52:55.689 | INFO | Perf report for the last 5.13886 seconds: | 20260505 04:52:55.689 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:52:55.689 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:55.689 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:55.689 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:55.689 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260505 04:52:55.689 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:55.689 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:55.689 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 04:52:55.689 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:55.689 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:52:55.689 | INFO | Uploaded 36GiB to GPU in 1.59862s (22.5195GiB/s) | 20260505 04:52:55.689 | INFO | Downloaded 34GiB to GPU in 1.51915s (22.3809GiB/s) | 20260505 04:53:00.828 | INFO | Analyzing chunk frames = [486912,487424) | 20260505 04:53:00.828 | INFO | MemInfo: { | 20260505 04:53:00.828 | INFO | "gpu" : " 17 GB", | 20260505 04:53:00.828 | INFO | "huge" : "", | 20260505 04:53:00.828 | INFO | "huge_shared" : " 27 GB", | 20260505 04:53:00.828 | INFO | "malloc" : " 14 GB", | 20260505 04:53:00.828 | INFO | "pinned" : " 29 GB" | 20260505 04:53:00.828 | INFO | } | 20260505 04:53:00.828 | INFO | Perf report for the last 5.13924 seconds: | 20260505 04:53:00.828 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:53:00.828 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:00.828 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:00.828 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:00.828 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260505 04:53:00.828 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:00.828 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:00.828 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 04:53:00.828 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:00.828 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:00.829 | INFO | Uploaded 36GiB to GPU in 1.59925s (22.5106GiB/s) | 20260505 04:53:00.829 | INFO | Downloaded 34GiB to GPU in 1.51955s (22.375GiB/s) | 20260505 04:53:00.925 | INFO | Starting a round of Baz IO | 20260505 04:53:03.876 | INFO | Populating RTMetrics report | 20260505 04:53:03.877 | INFO | RTMetrics report populated | 20260505 04:53:03.877 | INFO | Opening tmp RTMetrics json file | 20260505 04:53:03.879 | INFO | Serializing RTMetrics to json | 20260505 04:53:03.881 | INFO | Serialization done, swapping tmp file now... | 20260505 04:53:03.883 | INFO | RTMetrics done | 20260505 04:53:03.973 | INFO | Finished a round of Baz IO: Wrote 2.095 GiB in 3.047 seconds | 20260505 04:53:03.973 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.53% / 0.12% / 56.35% | 20260505 04:53:03.973 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.35653% / 0.000234445% / 7.54992e-05% | 20260505 04:53:03.973 | INFO | Baz IO overall: | 20260505 04:53:03.973 | INFO | BytesWritten : 207872102179 | 20260505 04:53:03.973 | INFO | NumEvents : 96896794490 | 20260505 04:53:05.967 | INFO | Analyzing chunk frames = [487424,487936) | 20260505 04:53:05.967 | INFO | MemInfo: { | 20260505 04:53:05.967 | INFO | "gpu" : " 17 GB", | 20260505 04:53:05.967 | INFO | "huge" : "", | 20260505 04:53:05.967 | INFO | "huge_shared" : " 27 GB", | 20260505 04:53:05.967 | INFO | "malloc" : " 14 GB", | 20260505 04:53:05.967 | INFO | "pinned" : " 29 GB" | 20260505 04:53:05.967 | INFO | } | 20260505 04:53:05.967 | INFO | Perf report for the last 5.1392 seconds: | 20260505 04:53:05.967 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:53:05.967 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:05.967 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:05.968 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:05.968 | INFO | ANALYSIS : 58.2%, 3.0, 90.5ms, 0 | 20260505 04:53:05.968 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:05.968 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260505 04:53:05.968 | INFO | PRE_HQ : 57.5%, 2.2, 64.4ms, 0 | 20260505 04:53:05.968 | INFO | BAZWRITER : 58.9%, 4.0, 122.3ms, 0 | 20260505 04:53:05.968 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 04:53:05.968 | INFO | Uploaded 36GiB to GPU in 1.63836s (21.9732GiB/s) | 20260505 04:53:05.968 | INFO | Downloaded 38GiB to GPU in 1.70281s (22.3161GiB/s) | 20260505 04:53:11.107 | INFO | Analyzing chunk frames = [487936,488448) | 20260505 04:53:11.107 | INFO | MemInfo: { | 20260505 04:53:11.107 | INFO | "gpu" : " 17 GB", | 20260505 04:53:11.107 | INFO | "huge" : "", | 20260505 04:53:11.107 | INFO | "huge_shared" : " 27 GB", | 20260505 04:53:11.107 | INFO | "malloc" : " 14 GB", | 20260505 04:53:11.107 | INFO | "pinned" : " 29 GB" | 20260505 04:53:11.107 | INFO | } | 20260505 04:53:11.107 | INFO | Perf report for the last 5.13931 seconds: | 20260505 04:53:11.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:53:11.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:11.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:11.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:11.107 | INFO | ANALYSIS : 58.0%, 3.0, 90.4ms, 0 | 20260505 04:53:11.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:11.107 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:11.107 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 04:53:11.107 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:11.107 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:11.107 | INFO | Uploaded 36GiB to GPU in 1.59951s (22.5069GiB/s) | 20260505 04:53:11.107 | INFO | Downloaded 34GiB to GPU in 1.51624s (22.4239GiB/s) | 20260505 04:53:16.246 | INFO | Analyzing chunk frames = [488448,488960) | 20260505 04:53:16.246 | INFO | MemInfo: { | 20260505 04:53:16.246 | INFO | "gpu" : " 17 GB", | 20260505 04:53:16.246 | INFO | "huge" : "", | 20260505 04:53:16.246 | INFO | "huge_shared" : " 27 GB", | 20260505 04:53:16.246 | INFO | "malloc" : " 14 GB", | 20260505 04:53:16.246 | INFO | "pinned" : " 29 GB" | 20260505 04:53:16.246 | INFO | } | 20260505 04:53:16.246 | INFO | Perf report for the last 5.13944 seconds: | 20260505 04:53:16.246 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:53:16.246 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:16.246 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:16.246 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:16.246 | INFO | ANALYSIS : 57.4%, 3.0, 89.1ms, 0 | 20260505 04:53:16.246 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:16.246 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:16.246 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 04:53:16.246 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:16.246 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:16.247 | INFO | Uploaded 36GiB to GPU in 1.60367s (22.4484GiB/s) | 20260505 04:53:16.247 | INFO | Downloaded 34GiB to GPU in 1.52025s (22.3647GiB/s) | 20260505 04:53:21.385 | INFO | Analyzing chunk frames = [488960,489472) | 20260505 04:53:21.385 | INFO | MemInfo: { | 20260505 04:53:21.385 | INFO | "gpu" : " 17 GB", | 20260505 04:53:21.385 | INFO | "huge" : "", | 20260505 04:53:21.385 | INFO | "huge_shared" : " 27 GB", | 20260505 04:53:21.386 | INFO | "malloc" : " 14 GB", | 20260505 04:53:21.386 | INFO | "pinned" : " 29 GB" | 20260505 04:53:21.386 | INFO | } | 20260505 04:53:21.386 | INFO | Perf report for the last 5.13938 seconds: | 20260505 04:53:21.386 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:53:21.386 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:21.386 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:21.386 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:21.386 | INFO | ANALYSIS : 57.0%, 3.0, 88.4ms, 0 | 20260505 04:53:21.386 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:21.386 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:21.386 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260505 04:53:21.386 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:21.386 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:21.386 | INFO | Uploaded 36GiB to GPU in 1.60133s (22.4813GiB/s) | 20260505 04:53:21.386 | INFO | Downloaded 34GiB to GPU in 1.52003s (22.368GiB/s) | 20260505 04:53:26.525 | INFO | Analyzing chunk frames = [489472,489984) | 20260505 04:53:26.525 | INFO | MemInfo: { | 20260505 04:53:26.525 | INFO | "gpu" : " 17 GB", | 20260505 04:53:26.525 | INFO | "huge" : "", | 20260505 04:53:26.525 | INFO | "huge_shared" : " 27 GB", | 20260505 04:53:26.525 | INFO | "malloc" : " 14 GB", | 20260505 04:53:26.525 | INFO | "pinned" : " 29 GB" | 20260505 04:53:26.525 | INFO | } | 20260505 04:53:26.525 | INFO | Perf report for the last 5.13929 seconds: | 20260505 04:53:26.525 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:53:26.525 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:26.525 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:26.525 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:26.525 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260505 04:53:26.525 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:26.525 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:26.525 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 04:53:26.525 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:26.525 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:26.526 | INFO | Uploaded 36GiB to GPU in 1.60033s (22.4953GiB/s) | 20260505 04:53:26.526 | INFO | Downloaded 34GiB to GPU in 1.51726s (22.4088GiB/s) | 20260505 04:53:31.664 | INFO | Analyzing chunk frames = [489984,490496) | 20260505 04:53:31.664 | INFO | MemInfo: { | 20260505 04:53:31.664 | INFO | "gpu" : " 17 GB", | 20260505 04:53:31.664 | INFO | "huge" : "", | 20260505 04:53:31.664 | INFO | "huge_shared" : " 27 GB", | 20260505 04:53:31.664 | INFO | "malloc" : " 14 GB", | 20260505 04:53:31.664 | INFO | "pinned" : " 29 GB" | 20260505 04:53:31.664 | INFO | } | 20260505 04:53:31.664 | INFO | Perf report for the last 5.13927 seconds: | 20260505 04:53:31.664 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:53:31.664 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:31.664 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:31.664 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:31.664 | INFO | ANALYSIS : 57.3%, 3.0, 89.3ms, 0 | 20260505 04:53:31.664 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:31.664 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:31.664 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260505 04:53:31.664 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:31.664 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:31.665 | INFO | Uploaded 36GiB to GPU in 1.60184s (22.4741GiB/s) | 20260505 04:53:31.665 | INFO | Downloaded 34GiB to GPU in 1.51858s (22.3894GiB/s) | 20260505 04:53:36.803 | INFO | Analyzing chunk frames = [490496,491008) | 20260505 04:53:36.803 | INFO | MemInfo: { | 20260505 04:53:36.803 | INFO | "gpu" : " 17 GB", | 20260505 04:53:36.803 | INFO | "huge" : "", | 20260505 04:53:36.803 | INFO | "huge_shared" : " 27 GB", | 20260505 04:53:36.803 | INFO | "malloc" : " 14 GB", | 20260505 04:53:36.803 | INFO | "pinned" : " 29 GB" | 20260505 04:53:36.803 | INFO | } | 20260505 04:53:36.804 | INFO | Perf report for the last 5.13931 seconds: | 20260505 04:53:36.804 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:53:36.804 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:36.804 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:36.804 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:36.804 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260505 04:53:36.804 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:36.804 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:36.804 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 04:53:36.804 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:36.804 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:36.804 | INFO | Uploaded 36GiB to GPU in 1.60132s (22.4814GiB/s) | 20260505 04:53:36.804 | INFO | Downloaded 34GiB to GPU in 1.51869s (22.3878GiB/s) | 20260505 04:53:41.943 | INFO | Analyzing chunk frames = [491008,491520) | 20260505 04:53:41.943 | INFO | MemInfo: { | 20260505 04:53:41.943 | INFO | "gpu" : " 17 GB", | 20260505 04:53:41.943 | INFO | "huge" : "", | 20260505 04:53:41.943 | INFO | "huge_shared" : " 27 GB", | 20260505 04:53:41.943 | INFO | "malloc" : " 14 GB", | 20260505 04:53:41.943 | INFO | "pinned" : " 29 GB" | 20260505 04:53:41.943 | INFO | } | 20260505 04:53:41.943 | INFO | Perf report for the last 5.13946 seconds: | 20260505 04:53:41.943 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:53:41.943 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:41.943 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:41.943 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:41.943 | INFO | ANALYSIS : 57.6%, 3.0, 89.8ms, 0 | 20260505 04:53:41.943 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:41.943 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:41.943 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 04:53:41.943 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:41.943 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:41.943 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 04:53:41.943 | INFO | Total mem usage: 15.1 GiB | 20260505 04:53:41.943 | INFO | All Time High Water Marks sum: 15.7 GiB (Savings: 4.09%) | 20260505 04:53:41.944 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -4.2%) | 20260505 04:53:41.944 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 04:53:41.944 | INFO | High Water Breakdown: | 20260505 04:53:41.944 | INFO | 59.5% : 8.6 GiB ( 95.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 04:53:41.944 | INFO | 40.4% : 5.83 GiB ( 87.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 04:53:41.944 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 04:53:41.944 | INFO | ------------------------------------------ | 20260505 04:53:41.944 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:53:41.944 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 04:53:41.944 | INFO | Total mem usage: 26.5 GiB | 20260505 04:53:41.944 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 04:53:41.944 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 04:53:41.944 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 04:53:41.944 | INFO | High Water Breakdown: | 20260505 04:53:41.944 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:53:41.944 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:53:41.944 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:53:41.944 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:53:41.944 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:53:41.944 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:53:41.944 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 04:53:41.944 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 04:53:41.944 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:53:41.944 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 04:53:41.944 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 04:53:41.944 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 04:53:41.944 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 04:53:41.944 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 04:53:41.944 | INFO | ------------------------------------------ | 20260505 04:53:41.944 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:53:41.944 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 04:53:41.944 | INFO | Total mem usage: 25.6 GiB | 20260505 04:53:41.944 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 04:53:41.944 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 04:53:41.944 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 04:53:41.944 | INFO | High Water Breakdown: | 20260505 04:53:41.944 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 04:53:41.944 | INFO | ------------------------------------------ | 20260505 04:53:41.944 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:53:41.944 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 04:53:41.944 | INFO | Total mem usage: 15.7 GiB | 20260505 04:53:41.944 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 04:53:41.944 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 04:53:41.944 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260505 04:53:41.944 | INFO | High Water Breakdown: | 20260505 04:53:41.944 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 04:53:41.944 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 04:53:41.944 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 04:53:41.944 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 04:53:41.944 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 04:53:41.944 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 04:53:41.944 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 04:53:41.944 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 04:53:41.944 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 04:53:41.944 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:53:41.944 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 04:53:41.944 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:53:41.944 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 04:53:41.944 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 04:53:41.944 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 04:53:41.944 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 04:53:41.944 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:53:41.944 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 04:53:41.944 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:53:41.944 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:53:41.944 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:53:41.944 | INFO | Uploaded 36GiB to GPU in 1.60137s (22.4807GiB/s) | 20260505 04:53:41.944 | INFO | Downloaded 34GiB to GPU in 1.51851s (22.3903GiB/s) | 20260505 04:53:41.944 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 04:53:41.944 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 04:53:41.944 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:53:41.944 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 04:53:41.944 | INFO | ------------------------------------------ | 20260505 04:53:41.944 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:53:41.944 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 04:53:41.944 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:53:41.944 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:53:41.944 | INFO | PerfReport: Full Execution -- 3.04e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260505 04:53:41.944 | INFO | PerfReport: 0.28%, AggregateStats -- 8.65e-02 / -nan, 5.20e-03 / 0.00e+00 | 20260505 04:53:41.944 | INFO | PerfReport: 10.84%, Baseline -- 3.30e+00 / -nan, 1.84e-02 / 0.00e+00 | 20260505 04:53:41.944 | INFO | PerfReport: 3.04%, Histogram -- 9.24e-01 / -nan, 5.15e-02 / 0.00e+00 | 20260505 04:53:41.944 | INFO | PerfReport: 16.04%, DME -- 3.90e+01 / -nan, 3.11e+00 / 0.00e+00 | 20260505 04:53:41.944 | INFO | PerfReport: 56.58%, FrameLabeling -- 1.72e+01 / -nan, 4.51e-02 / 0.00e+00 | 20260505 04:53:41.944 | INFO | PerfReport: 7.61%, PulseAccumulating -- 2.32e+00 / -nan, 1.54e-01 / 0.00e+00 | 20260505 04:53:41.944 | INFO | PerfReport: 3.96%, Metrics -- 1.21e+00 / -nan, 1.99e-01 / 0.00e+00 | 20260505 04:53:41.944 | INFO | PerfReport: 1.64%, PulseSerializing -- 4.98e-01 / -nan, 7.98e-02 / 0.00e+00 | 20260505 04:53:41.944 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 04:53:41.944 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:53:41.944 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:53:41.944 | INFO | PerfReport: Full Execution -- 9.14e+01 / -nan, 1.90e+01 / 0.00e+00 | 20260505 04:53:41.944 | INFO | PerfReport: 18.30%, Upload -- 1.67e+01 / 0.00e+00, 3.70e-01 / 0.00e+00 | 20260505 04:53:41.944 | INFO | PerfReport: 17.58%, Download -- 1.61e+01 / 0.00e+00, 6.95e-01 / 0.00e+00 | 20260505 04:53:41.944 | INFO | PerfReport: 33.30%, Compute -- 3.04e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260505 04:53:41.944 | INFO | PerfReport: 0.55%, LockUpload -- 5.03e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260505 04:53:41.944 | INFO | PerfReport: 0.00%, LockDownload -- 3.95e-04 / 0.00e+00, 9.52e-05 / 0.00e+00 | 20260505 04:53:41.944 | INFO | PerfReport: 30.26%, LockCompute -- 2.76e+01 / 0.00e+00, 1.72e+01 / 0.00e+00 | 20260505 04:53:41.944 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.52e-02 / 0.00e+00 | 20260505 04:53:42.040 | INFO | Starting a round of Baz IO | 20260505 04:53:44.989 | INFO | Populating RTMetrics report | 20260505 04:53:44.989 | INFO | RTMetrics report populated | 20260505 04:53:44.989 | INFO | Opening tmp RTMetrics json file | 20260505 04:53:44.990 | INFO | Serializing RTMetrics to json | 20260505 04:53:44.992 | INFO | Serialization done, swapping tmp file now... | 20260505 04:53:44.992 | INFO | RTMetrics done | 20260505 04:53:45.078 | INFO | Finished a round of Baz IO: Wrote 2.099 GiB in 3.037 seconds | 20260505 04:53:45.078 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.42% / 0.11% / 56.47% | 20260505 04:53:45.078 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.3062% / 0.000254313% / 5.5631e-05% | 20260505 04:53:45.078 | INFO | Baz IO overall: | 20260505 04:53:45.079 | INFO | BytesWritten : 210125540706 | 20260505 04:53:45.079 | INFO | NumEvents : 97965842209 | 20260505 04:53:47.082 | INFO | Analyzing chunk frames = [491520,492032) | 20260505 04:53:47.082 | INFO | MemInfo: { | 20260505 04:53:47.082 | INFO | "gpu" : " 17 GB", | 20260505 04:53:47.082 | INFO | "huge" : "", | 20260505 04:53:47.082 | INFO | "huge_shared" : " 27 GB", | 20260505 04:53:47.082 | INFO | "malloc" : " 14 GB", | 20260505 04:53:47.082 | INFO | "pinned" : " 29 GB" | 20260505 04:53:47.082 | INFO | } | 20260505 04:53:47.082 | INFO | Perf report for the last 5.13901 seconds: | 20260505 04:53:47.082 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:53:47.082 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:47.082 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:47.082 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:47.082 | INFO | ANALYSIS : 58.1%, 3.0, 90.4ms, 0 | 20260505 04:53:47.082 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:47.082 | INFO | RT_METRICS : 45.5%, 1.0, 23.8ms, 0 | 20260505 04:53:47.082 | INFO | PRE_HQ : 57.5%, 2.2, 64.8ms, 0 | 20260505 04:53:47.082 | INFO | BAZWRITER : 58.7%, 4.1, 123.8ms, 0 | 20260505 04:53:47.082 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:53:47.083 | INFO | Uploaded 36GiB to GPU in 1.64226s (21.921GiB/s) | 20260505 04:53:47.083 | INFO | Downloaded 38GiB to GPU in 1.7082s (22.2456GiB/s) | 20260505 04:53:52.221 | INFO | Analyzing chunk frames = [492032,492544) | 20260505 04:53:52.221 | INFO | MemInfo: { | 20260505 04:53:52.221 | INFO | "gpu" : " 17 GB", | 20260505 04:53:52.221 | INFO | "huge" : "", | 20260505 04:53:52.221 | INFO | "huge_shared" : " 27 GB", | 20260505 04:53:52.221 | INFO | "malloc" : " 14 GB", | 20260505 04:53:52.222 | INFO | "pinned" : " 29 GB" | 20260505 04:53:52.222 | INFO | } | 20260505 04:53:52.222 | INFO | Perf report for the last 5.13962 seconds: | 20260505 04:53:52.222 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:53:52.222 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:52.222 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:52.222 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:52.222 | INFO | ANALYSIS : 58.0%, 3.0, 90.5ms, 0 | 20260505 04:53:52.222 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:52.222 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:52.222 | INFO | PRE_HQ : 31.6%, 1.0, 16.6ms, 0 | 20260505 04:53:52.222 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:52.222 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:52.222 | INFO | Uploaded 36GiB to GPU in 1.59911s (22.5126GiB/s) | 20260505 04:53:52.222 | INFO | Downloaded 34GiB to GPU in 1.51558s (22.4337GiB/s) | 20260505 04:53:57.361 | INFO | Analyzing chunk frames = [492544,493056) | 20260505 04:53:57.361 | INFO | MemInfo: { | 20260505 04:53:57.361 | INFO | "gpu" : " 17 GB", | 20260505 04:53:57.361 | INFO | "huge" : "", | 20260505 04:53:57.361 | INFO | "huge_shared" : " 27 GB", | 20260505 04:53:57.361 | INFO | "malloc" : " 14 GB", | 20260505 04:53:57.361 | INFO | "pinned" : " 29 GB" | 20260505 04:53:57.361 | INFO | } | 20260505 04:53:57.361 | INFO | Perf report for the last 5.13937 seconds: | 20260505 04:53:57.361 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:53:57.361 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:57.361 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:57.361 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:57.361 | INFO | ANALYSIS : 57.3%, 3.0, 89.0ms, 0 | 20260505 04:53:57.361 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:57.361 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:57.361 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 04:53:57.361 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:57.361 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:53:57.362 | INFO | Uploaded 36GiB to GPU in 1.60335s (22.4529GiB/s) | 20260505 04:53:57.362 | INFO | Downloaded 34GiB to GPU in 1.52037s (22.363GiB/s) | 20260505 04:54:02.500 | INFO | Analyzing chunk frames = [493056,493568) | 20260505 04:54:02.500 | INFO | MemInfo: { | 20260505 04:54:02.500 | INFO | "gpu" : " 17 GB", | 20260505 04:54:02.500 | INFO | "huge" : "", | 20260505 04:54:02.500 | INFO | "huge_shared" : " 27 GB", | 20260505 04:54:02.500 | INFO | "malloc" : " 14 GB", | 20260505 04:54:02.500 | INFO | "pinned" : " 29 GB" | 20260505 04:54:02.500 | INFO | } | 20260505 04:54:02.500 | INFO | Perf report for the last 5.13933 seconds: | 20260505 04:54:02.500 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:54:02.500 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:02.500 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:02.500 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:02.500 | INFO | ANALYSIS : 57.1%, 3.0, 88.5ms, 0 | 20260505 04:54:02.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:02.501 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:02.501 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 04:54:02.501 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:02.501 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:02.501 | INFO | Uploaded 36GiB to GPU in 1.60299s (22.458GiB/s) | 20260505 04:54:02.501 | INFO | Downloaded 34GiB to GPU in 1.52027s (22.3644GiB/s) | 20260505 04:54:07.639 | INFO | Analyzing chunk frames = [493568,494080) | 20260505 04:54:07.639 | INFO | MemInfo: { | 20260505 04:54:07.639 | INFO | "gpu" : " 17 GB", | 20260505 04:54:07.639 | INFO | "huge" : "", | 20260505 04:54:07.639 | INFO | "huge_shared" : " 27 GB", | 20260505 04:54:07.639 | INFO | "malloc" : " 14 GB", | 20260505 04:54:07.640 | INFO | "pinned" : " 29 GB" | 20260505 04:54:07.640 | INFO | } | 20260505 04:54:07.640 | INFO | Perf report for the last 5.13924 seconds: | 20260505 04:54:07.640 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:54:07.640 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:07.640 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:07.640 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:07.640 | INFO | ANALYSIS : 57.0%, 3.0, 88.8ms, 0 | 20260505 04:54:07.640 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:07.640 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:07.640 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260505 04:54:07.640 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:07.640 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:07.640 | INFO | Uploaded 36GiB to GPU in 1.60125s (22.4825GiB/s) | 20260505 04:54:07.640 | INFO | Downloaded 34GiB to GPU in 1.51817s (22.3954GiB/s) | 20260505 04:54:12.779 | INFO | Analyzing chunk frames = [494080,494592) | 20260505 04:54:12.779 | INFO | MemInfo: { | 20260505 04:54:12.779 | INFO | "gpu" : " 17 GB", | 20260505 04:54:12.779 | INFO | "huge" : "", | 20260505 04:54:12.779 | INFO | "huge_shared" : " 27 GB", | 20260505 04:54:12.779 | INFO | "malloc" : " 14 GB", | 20260505 04:54:12.779 | INFO | "pinned" : " 29 GB" | 20260505 04:54:12.779 | INFO | } | 20260505 04:54:12.779 | INFO | Perf report for the last 5.1394 seconds: | 20260505 04:54:12.779 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:54:12.779 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:12.779 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:12.779 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:12.779 | INFO | ANALYSIS : 57.3%, 3.0, 89.3ms, 0 | 20260505 04:54:12.779 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:12.779 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:12.779 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 04:54:12.779 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:12.779 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:12.780 | INFO | Uploaded 36GiB to GPU in 1.60025s (22.4964GiB/s) | 20260505 04:54:12.780 | INFO | Downloaded 34GiB to GPU in 1.51769s (22.4024GiB/s) | 20260505 04:54:17.918 | INFO | Analyzing chunk frames = [494592,495104) | 20260505 04:54:17.918 | INFO | MemInfo: { | 20260505 04:54:17.918 | INFO | "gpu" : " 17 GB", | 20260505 04:54:17.918 | INFO | "huge" : "", | 20260505 04:54:17.918 | INFO | "huge_shared" : " 27 GB", | 20260505 04:54:17.918 | INFO | "malloc" : " 14 GB", | 20260505 04:54:17.918 | INFO | "pinned" : " 29 GB" | 20260505 04:54:17.918 | INFO | } | 20260505 04:54:17.918 | INFO | Perf report for the last 5.13937 seconds: | 20260505 04:54:17.918 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:54:17.918 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:17.918 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:17.918 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:17.918 | INFO | ANALYSIS : 57.5%, 3.0, 89.6ms, 0 | 20260505 04:54:17.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:17.919 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:17.919 | INFO | PRE_HQ : 31.9%, 1.0, 16.5ms, 0 | 20260505 04:54:17.919 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:17.919 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:17.919 | INFO | Uploaded 36GiB to GPU in 1.60186s (22.4738GiB/s) | 20260505 04:54:17.919 | INFO | Downloaded 34GiB to GPU in 1.51884s (22.3855GiB/s) | 20260505 04:54:23.058 | INFO | Analyzing chunk frames = [495104,495616) | 20260505 04:54:23.058 | INFO | MemInfo: { | 20260505 04:54:23.058 | INFO | "gpu" : " 17 GB", | 20260505 04:54:23.058 | INFO | "huge" : "", | 20260505 04:54:23.058 | INFO | "huge_shared" : " 27 GB", | 20260505 04:54:23.058 | INFO | "malloc" : " 14 GB", | 20260505 04:54:23.058 | INFO | "pinned" : " 29 GB" | 20260505 04:54:23.058 | INFO | } | 20260505 04:54:23.058 | INFO | Perf report for the last 5.13933 seconds: | 20260505 04:54:23.058 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:54:23.058 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:23.058 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:23.058 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:23.058 | INFO | ANALYSIS : 57.6%, 3.0, 89.8ms, 0 | 20260505 04:54:23.058 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:23.058 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:23.058 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260505 04:54:23.058 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:23.058 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:23.058 | INFO | Uploaded 36GiB to GPU in 1.60108s (22.4848GiB/s) | 20260505 04:54:23.058 | INFO | Downloaded 34GiB to GPU in 1.51833s (22.3931GiB/s) | 20260505 04:54:23.155 | INFO | Starting a round of Baz IO | 20260505 04:54:26.105 | INFO | Populating RTMetrics report | 20260505 04:54:26.105 | INFO | RTMetrics report populated | 20260505 04:54:26.105 | INFO | Opening tmp RTMetrics json file | 20260505 04:54:26.106 | INFO | Serializing RTMetrics to json | 20260505 04:54:26.107 | INFO | Serialization done, swapping tmp file now... | 20260505 04:54:26.107 | INFO | RTMetrics done | 20260505 04:54:26.203 | INFO | Finished a round of Baz IO: Wrote 2.103 GiB in 3.047 seconds | 20260505 04:54:26.203 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.31% / 0.11% / 56.58% | 20260505 04:54:26.203 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.27934% / 0.000182788% / 4.37101e-05% | 20260505 04:54:26.203 | INFO | Baz IO overall: | 20260505 04:54:26.203 | INFO | BytesWritten : 212383443686 | 20260505 04:54:26.203 | INFO | NumEvents : 99038080490 | 20260505 04:54:28.197 | INFO | Analyzing chunk frames = [495616,496128) | 20260505 04:54:28.197 | INFO | MemInfo: { | 20260505 04:54:28.197 | INFO | "gpu" : " 17 GB", | 20260505 04:54:28.197 | INFO | "huge" : "", | 20260505 04:54:28.197 | INFO | "huge_shared" : " 27 GB", | 20260505 04:54:28.197 | INFO | "malloc" : " 14 GB", | 20260505 04:54:28.197 | INFO | "pinned" : " 29 GB" | 20260505 04:54:28.197 | INFO | } | 20260505 04:54:28.197 | INFO | Perf report for the last 5.13921 seconds: | 20260505 04:54:28.197 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:54:28.197 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:28.197 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:28.197 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:28.197 | INFO | ANALYSIS : 58.2%, 3.0, 90.4ms, 0 | 20260505 04:54:28.197 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:28.197 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260505 04:54:28.197 | INFO | PRE_HQ : 57.7%, 2.2, 64.9ms, 0 | 20260505 04:54:28.197 | INFO | BAZWRITER : 58.9%, 4.1, 124.1ms, 0 | 20260505 04:54:28.197 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:54:28.198 | INFO | Uploaded 36GiB to GPU in 1.64211s (21.923GiB/s) | 20260505 04:54:28.198 | INFO | Downloaded 38GiB to GPU in 1.70673s (22.2648GiB/s) | 20260505 04:54:33.336 | INFO | Analyzing chunk frames = [496128,496640) | 20260505 04:54:33.336 | INFO | MemInfo: { | 20260505 04:54:33.336 | INFO | "gpu" : " 17 GB", | 20260505 04:54:33.336 | INFO | "huge" : "", | 20260505 04:54:33.336 | INFO | "huge_shared" : " 27 GB", | 20260505 04:54:33.336 | INFO | "malloc" : " 14 GB", | 20260505 04:54:33.336 | INFO | "pinned" : " 29 GB" | 20260505 04:54:33.336 | INFO | } | 20260505 04:54:33.336 | INFO | Perf report for the last 5.13912 seconds: | 20260505 04:54:33.336 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:54:33.336 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:33.336 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:33.336 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:33.336 | INFO | ANALYSIS : 58.0%, 3.0, 90.5ms, 0 | 20260505 04:54:33.336 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:33.336 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:33.337 | INFO | PRE_HQ : 31.4%, 1.0, 16.5ms, 0 | 20260505 04:54:33.337 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:33.337 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:33.337 | INFO | Uploaded 36GiB to GPU in 1.59715s (22.5401GiB/s) | 20260505 04:54:33.337 | INFO | Downloaded 34GiB to GPU in 1.51647s (22.4205GiB/s) | 20260505 04:54:38.475 | INFO | Analyzing chunk frames = [496640,497152) | 20260505 04:54:38.475 | INFO | MemInfo: { | 20260505 04:54:38.475 | INFO | "gpu" : " 17 GB", | 20260505 04:54:38.475 | INFO | "huge" : "", | 20260505 04:54:38.475 | INFO | "huge_shared" : " 27 GB", | 20260505 04:54:38.475 | INFO | "malloc" : " 14 GB", | 20260505 04:54:38.476 | INFO | "pinned" : " 29 GB" | 20260505 04:54:38.476 | INFO | } | 20260505 04:54:38.476 | INFO | Perf report for the last 5.13942 seconds: | 20260505 04:54:38.476 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:54:38.476 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:38.476 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:38.476 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:38.476 | INFO | ANALYSIS : 57.3%, 3.0, 89.1ms, 0 | 20260505 04:54:38.476 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:38.476 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:38.476 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260505 04:54:38.476 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:38.476 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:38.476 | INFO | Uploaded 36GiB to GPU in 1.60076s (22.4893GiB/s) | 20260505 04:54:38.476 | INFO | Downloaded 34GiB to GPU in 1.52174s (22.3429GiB/s) | 20260505 04:54:43.615 | INFO | Analyzing chunk frames = [497152,497664) | 20260505 04:54:43.615 | INFO | MemInfo: { | 20260505 04:54:43.615 | INFO | "gpu" : " 17 GB", | 20260505 04:54:43.615 | INFO | "huge" : "", | 20260505 04:54:43.615 | INFO | "huge_shared" : " 27 GB", | 20260505 04:54:43.615 | INFO | "malloc" : " 14 GB", | 20260505 04:54:43.615 | INFO | "pinned" : " 29 GB" | 20260505 04:54:43.615 | INFO | } | 20260505 04:54:43.615 | INFO | Perf report for the last 5.13923 seconds: | 20260505 04:54:43.615 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:54:43.615 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:43.615 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:43.615 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:43.615 | INFO | ANALYSIS : 57.0%, 3.0, 88.5ms, 0 | 20260505 04:54:43.615 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:43.615 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:43.615 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260505 04:54:43.615 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:43.615 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:43.616 | INFO | Uploaded 36GiB to GPU in 1.59956s (22.5062GiB/s) | 20260505 04:54:43.616 | INFO | Downloaded 34GiB to GPU in 1.51995s (22.3692GiB/s) | 20260505 04:54:48.754 | INFO | Analyzing chunk frames = [497664,498176) | 20260505 04:54:48.754 | INFO | MemInfo: { | 20260505 04:54:48.754 | INFO | "gpu" : " 17 GB", | 20260505 04:54:48.754 | INFO | "huge" : "", | 20260505 04:54:48.754 | INFO | "huge_shared" : " 27 GB", | 20260505 04:54:48.754 | INFO | "malloc" : " 14 GB", | 20260505 04:54:48.754 | INFO | "pinned" : " 29 GB" | 20260505 04:54:48.754 | INFO | } | 20260505 04:54:48.755 | INFO | Perf report for the last 5.13975 seconds: | 20260505 04:54:48.755 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:54:48.755 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:48.755 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:48.755 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:48.755 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260505 04:54:48.755 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:48.755 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:48.755 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 04:54:48.755 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:48.755 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:48.755 | INFO | Uploaded 36GiB to GPU in 1.59915s (22.512GiB/s) | 20260505 04:54:48.755 | INFO | Downloaded 34GiB to GPU in 1.51877s (22.3866GiB/s) | 20260505 04:54:53.894 | INFO | Analyzing chunk frames = [498176,498688) | 20260505 04:54:53.894 | INFO | MemInfo: { | 20260505 04:54:53.894 | INFO | "gpu" : " 17 GB", | 20260505 04:54:53.894 | INFO | "huge" : "", | 20260505 04:54:53.894 | INFO | "huge_shared" : " 27 GB", | 20260505 04:54:53.894 | INFO | "malloc" : " 14 GB", | 20260505 04:54:53.894 | INFO | "pinned" : " 29 GB" | 20260505 04:54:53.894 | INFO | } | 20260505 04:54:53.894 | INFO | Perf report for the last 5.13922 seconds: | 20260505 04:54:53.894 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:54:53.894 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:53.894 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:53.894 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:53.894 | INFO | ANALYSIS : 57.2%, 3.0, 89.3ms, 0 | 20260505 04:54:53.894 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:53.894 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:53.894 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260505 04:54:53.894 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:53.894 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:53.894 | INFO | Uploaded 36GiB to GPU in 1.59836s (22.5232GiB/s) | 20260505 04:54:53.894 | INFO | Downloaded 34GiB to GPU in 1.51853s (22.3901GiB/s) | 20260505 04:54:59.033 | INFO | Analyzing chunk frames = [498688,499200) | 20260505 04:54:59.033 | INFO | MemInfo: { | 20260505 04:54:59.033 | INFO | "gpu" : " 17 GB", | 20260505 04:54:59.033 | INFO | "huge" : "", | 20260505 04:54:59.033 | INFO | "huge_shared" : " 27 GB", | 20260505 04:54:59.033 | INFO | "malloc" : " 14 GB", | 20260505 04:54:59.033 | INFO | "pinned" : " 29 GB" | 20260505 04:54:59.033 | INFO | } | 20260505 04:54:59.033 | INFO | Perf report for the last 5.13958 seconds: | 20260505 04:54:59.033 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:54:59.033 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:59.033 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:59.033 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:59.034 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260505 04:54:59.034 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:59.034 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:59.034 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 04:54:59.034 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:59.034 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:54:59.034 | INFO | Uploaded 36GiB to GPU in 1.59777s (22.5315GiB/s) | 20260505 04:54:59.034 | INFO | Downloaded 34GiB to GPU in 1.51977s (22.3718GiB/s) | 20260505 04:55:04.172 | INFO | Analyzing chunk frames = [499200,499712) | 20260505 04:55:04.172 | INFO | MemInfo: { | 20260505 04:55:04.172 | INFO | "gpu" : " 17 GB", | 20260505 04:55:04.172 | INFO | "huge" : "", | 20260505 04:55:04.172 | INFO | "huge_shared" : " 27 GB", | 20260505 04:55:04.173 | INFO | "malloc" : " 14 GB", | 20260505 04:55:04.173 | INFO | "pinned" : " 29 GB" | 20260505 04:55:04.173 | INFO | } | 20260505 04:55:04.173 | INFO | Perf report for the last 5.1391 seconds: | 20260505 04:55:04.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:55:04.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:04.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:04.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:04.173 | INFO | ANALYSIS : 57.5%, 3.0, 89.8ms, 0 | 20260505 04:55:04.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:04.173 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:04.173 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 04:55:04.173 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:04.173 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:04.173 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 04:55:04.173 | INFO | Total mem usage: 15.1 GiB | 20260505 04:55:04.173 | INFO | All Time High Water Marks sum: 15.7 GiB (Savings: 4.09%) | 20260505 04:55:04.173 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -4.27%) | 20260505 04:55:04.173 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 04:55:04.173 | INFO | High Water Breakdown: | 20260505 04:55:04.173 | INFO | 59.5% : 8.59 GiB ( 95.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 04:55:04.173 | INFO | 40.4% : 5.83 GiB ( 87.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 04:55:04.173 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 04:55:04.173 | INFO | ------------------------------------------ | 20260505 04:55:04.173 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:55:04.173 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 04:55:04.173 | INFO | Total mem usage: 26.5 GiB | 20260505 04:55:04.173 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 04:55:04.173 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 04:55:04.173 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 04:55:04.173 | INFO | High Water Breakdown: | 20260505 04:55:04.173 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:55:04.173 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:55:04.173 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:55:04.173 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:55:04.173 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:55:04.173 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:55:04.173 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 04:55:04.173 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 04:55:04.173 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:55:04.173 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 04:55:04.173 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 04:55:04.173 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 04:55:04.173 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 04:55:04.173 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 04:55:04.173 | INFO | ------------------------------------------ | 20260505 04:55:04.173 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:55:04.173 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 04:55:04.173 | INFO | Total mem usage: 25.6 GiB | 20260505 04:55:04.173 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 04:55:04.173 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 04:55:04.173 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 04:55:04.173 | INFO | High Water Breakdown: | 20260505 04:55:04.173 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 04:55:04.173 | INFO | ------------------------------------------ | 20260505 04:55:04.173 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:55:04.173 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 04:55:04.173 | INFO | Total mem usage: 15.7 GiB | 20260505 04:55:04.173 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 04:55:04.173 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 04:55:04.173 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260505 04:55:04.173 | INFO | High Water Breakdown: | 20260505 04:55:04.173 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 04:55:04.173 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 04:55:04.173 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 04:55:04.173 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 04:55:04.173 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 04:55:04.173 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 04:55:04.173 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 04:55:04.173 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 04:55:04.173 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 04:55:04.173 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:55:04.173 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 04:55:04.173 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:55:04.173 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 04:55:04.173 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 04:55:04.173 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 04:55:04.173 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 04:55:04.173 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:55:04.173 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 04:55:04.173 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:55:04.173 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:55:04.173 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:55:04.173 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 04:55:04.173 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 04:55:04.173 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:55:04.173 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 04:55:04.173 | INFO | ------------------------------------------ | 20260505 04:55:04.173 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:55:04.173 | INFO | Uploaded 36GiB to GPU in 1.59888s (22.5158GiB/s) | 20260505 04:55:04.173 | INFO | Downloaded 34GiB to GPU in 1.51938s (22.3775GiB/s) | 20260505 04:55:04.173 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 04:55:04.173 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:55:04.173 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:55:04.173 | INFO | PerfReport: Full Execution -- 3.04e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260505 04:55:04.173 | INFO | PerfReport: 0.28%, AggregateStats -- 8.65e-02 / -nan, 6.45e-03 / 0.00e+00 | 20260505 04:55:04.173 | INFO | PerfReport: 10.84%, Baseline -- 3.30e+00 / -nan, 1.76e-02 / 0.00e+00 | 20260505 04:55:04.173 | INFO | PerfReport: 3.04%, Histogram -- 9.24e-01 / -nan, 5.20e-02 / 0.00e+00 | 20260505 04:55:04.173 | INFO | PerfReport: 16.02%, DME -- 3.90e+01 / -nan, 3.22e+00 / 0.00e+00 | 20260505 04:55:04.173 | INFO | PerfReport: 56.60%, FrameLabeling -- 1.72e+01 / -nan, 4.61e-02 / 0.00e+00 | 20260505 04:55:04.173 | INFO | PerfReport: 7.61%, PulseAccumulating -- 2.32e+00 / -nan, 1.54e-01 / 0.00e+00 | 20260505 04:55:04.173 | INFO | PerfReport: 3.96%, Metrics -- 1.21e+00 / -nan, 1.99e-01 / 0.00e+00 | 20260505 04:55:04.173 | INFO | PerfReport: 1.63%, PulseSerializing -- 4.97e-01 / -nan, 8.02e-02 / 0.00e+00 | 20260505 04:55:04.173 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 04:55:04.173 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:55:04.173 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:55:04.173 | INFO | PerfReport: Full Execution -- 9.13e+01 / -nan, 1.91e+01 / 0.00e+00 | 20260505 04:55:04.173 | INFO | PerfReport: 18.31%, Upload -- 1.67e+01 / 0.00e+00, 3.75e-01 / 0.00e+00 | 20260505 04:55:04.173 | INFO | PerfReport: 17.59%, Download -- 1.61e+01 / 0.00e+00, 7.14e-01 / 0.00e+00 | 20260505 04:55:04.173 | INFO | PerfReport: 33.30%, Compute -- 3.04e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260505 04:55:04.173 | INFO | PerfReport: 0.55%, LockUpload -- 5.05e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260505 04:55:04.173 | INFO | PerfReport: 0.00%, LockDownload -- 4.13e-04 / 0.00e+00, 2.21e-04 / 0.00e+00 | 20260505 04:55:04.173 | INFO | PerfReport: 30.23%, LockCompute -- 2.76e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260505 04:55:04.173 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.70e-02 / 0.00e+00 | 20260505 04:55:04.270 | INFO | Starting a round of Baz IO | 20260505 04:55:07.218 | INFO | Populating RTMetrics report | 20260505 04:55:07.218 | INFO | RTMetrics report populated | 20260505 04:55:07.218 | INFO | Opening tmp RTMetrics json file | 20260505 04:55:07.221 | INFO | Serializing RTMetrics to json | 20260505 04:55:07.225 | INFO | Serialization done, swapping tmp file now... | 20260505 04:55:07.226 | INFO | RTMetrics done | 20260505 04:55:07.313 | INFO | Finished a round of Baz IO: Wrote 2.105 GiB in 3.043 seconds | 20260505 04:55:07.313 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.20% / 0.11% / 56.69% | 20260505 04:55:07.313 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.21992% / 0.00025034% / 5.16574e-05% | 20260505 04:55:07.313 | INFO | Baz IO overall: | 20260505 04:55:07.313 | INFO | BytesWritten : 214643374890 | 20260505 04:55:07.313 | INFO | NumEvents : 100111187845 | 20260505 04:55:09.312 | INFO | Analyzing chunk frames = [499712,500224) | 20260505 04:55:09.312 | INFO | MemInfo: { | 20260505 04:55:09.312 | INFO | "gpu" : " 17 GB", | 20260505 04:55:09.312 | INFO | "huge" : "", | 20260505 04:55:09.312 | INFO | "huge_shared" : " 27 GB", | 20260505 04:55:09.312 | INFO | "malloc" : " 14 GB", | 20260505 04:55:09.312 | INFO | "pinned" : " 29 GB" | 20260505 04:55:09.312 | INFO | } | 20260505 04:55:09.312 | INFO | Perf report for the last 5.13911 seconds: | 20260505 04:55:09.312 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:55:09.312 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:09.312 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:09.312 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:09.312 | INFO | ANALYSIS : 58.1%, 3.0, 90.3ms, 0 | 20260505 04:55:09.312 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:09.312 | INFO | RT_METRICS : 45.7%, 1.0, 23.9ms, 0 | 20260505 04:55:09.312 | INFO | PRE_HQ : 57.7%, 2.2, 65.3ms, 0 | 20260505 04:55:09.312 | INFO | BAZWRITER : 58.8%, 4.1, 124.0ms, 0 | 20260505 04:55:09.312 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 04:55:09.312 | INFO | Uploaded 36GiB to GPU in 1.63822s (21.975GiB/s) | 20260505 04:55:09.313 | INFO | Downloaded 38GiB to GPU in 1.70159s (22.332GiB/s) | 20260505 04:55:14.451 | INFO | Analyzing chunk frames = [500224,500736) | 20260505 04:55:14.451 | INFO | MemInfo: { | 20260505 04:55:14.451 | INFO | "gpu" : " 17 GB", | 20260505 04:55:14.451 | INFO | "huge" : "", | 20260505 04:55:14.451 | INFO | "huge_shared" : " 27 GB", | 20260505 04:55:14.451 | INFO | "malloc" : " 14 GB", | 20260505 04:55:14.451 | INFO | "pinned" : " 29 GB" | 20260505 04:55:14.451 | INFO | } | 20260505 04:55:14.451 | INFO | Perf report for the last 5.13974 seconds: | 20260505 04:55:14.451 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:55:14.451 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:14.451 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:14.451 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:14.452 | INFO | ANALYSIS : 57.9%, 3.0, 90.4ms, 0 | 20260505 04:55:14.452 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:14.452 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:14.452 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 04:55:14.452 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:14.452 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:14.452 | INFO | Uploaded 36GiB to GPU in 1.59935s (22.5092GiB/s) | 20260505 04:55:14.452 | INFO | Downloaded 34GiB to GPU in 1.51551s (22.4346GiB/s) | 20260505 04:55:19.590 | INFO | Analyzing chunk frames = [500736,501248) | 20260505 04:55:19.590 | INFO | MemInfo: { | 20260505 04:55:19.590 | INFO | "gpu" : " 17 GB", | 20260505 04:55:19.590 | INFO | "huge" : "", | 20260505 04:55:19.590 | INFO | "huge_shared" : " 27 GB", | 20260505 04:55:19.590 | INFO | "malloc" : " 14 GB", | 20260505 04:55:19.590 | INFO | "pinned" : " 29 GB" | 20260505 04:55:19.590 | INFO | } | 20260505 04:55:19.590 | INFO | Perf report for the last 5.13906 seconds: | 20260505 04:55:19.591 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:55:19.591 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:19.591 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:19.591 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:19.591 | INFO | ANALYSIS : 57.3%, 3.0, 89.1ms, 0 | 20260505 04:55:19.591 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:19.591 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:19.591 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260505 04:55:19.591 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:19.591 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:19.591 | INFO | Uploaded 36GiB to GPU in 1.60333s (22.4533GiB/s) | 20260505 04:55:19.591 | INFO | Downloaded 34GiB to GPU in 1.5209s (22.3552GiB/s) | 20260505 04:55:24.730 | INFO | Analyzing chunk frames = [501248,501760) | 20260505 04:55:24.730 | INFO | MemInfo: { | 20260505 04:55:24.730 | INFO | "gpu" : " 17 GB", | 20260505 04:55:24.730 | INFO | "huge" : "", | 20260505 04:55:24.730 | INFO | "huge_shared" : " 27 GB", | 20260505 04:55:24.730 | INFO | "malloc" : " 14 GB", | 20260505 04:55:24.730 | INFO | "pinned" : " 29 GB" | 20260505 04:55:24.730 | INFO | } | 20260505 04:55:24.730 | INFO | Perf report for the last 5.13944 seconds: | 20260505 04:55:24.730 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:55:24.730 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:24.730 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:24.730 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:24.730 | INFO | ANALYSIS : 57.0%, 3.0, 88.4ms, 0 | 20260505 04:55:24.730 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:24.730 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:24.730 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260505 04:55:24.730 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:24.730 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:24.730 | INFO | Uploaded 36GiB to GPU in 1.60275s (22.4614GiB/s) | 20260505 04:55:24.730 | INFO | Downloaded 34GiB to GPU in 1.51985s (22.3706GiB/s) | 20260505 04:55:29.869 | INFO | Analyzing chunk frames = [501760,502272) | 20260505 04:55:29.869 | INFO | MemInfo: { | 20260505 04:55:29.869 | INFO | "gpu" : " 17 GB", | 20260505 04:55:29.869 | INFO | "huge" : "", | 20260505 04:55:29.869 | INFO | "huge_shared" : " 27 GB", | 20260505 04:55:29.869 | INFO | "malloc" : " 14 GB", | 20260505 04:55:29.869 | INFO | "pinned" : " 29 GB" | 20260505 04:55:29.869 | INFO | } | 20260505 04:55:29.869 | INFO | Perf report for the last 5.13934 seconds: | 20260505 04:55:29.869 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:55:29.869 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:29.869 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:29.869 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:29.869 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260505 04:55:29.869 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:29.869 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:29.869 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260505 04:55:29.869 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:29.869 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:29.870 | INFO | Uploaded 36GiB to GPU in 1.6007s (22.4902GiB/s) | 20260505 04:55:29.870 | INFO | Downloaded 34GiB to GPU in 1.51669s (22.4172GiB/s) | 20260505 04:55:35.008 | INFO | Analyzing chunk frames = [502272,502784) | 20260505 04:55:35.008 | INFO | MemInfo: { | 20260505 04:55:35.008 | INFO | "gpu" : " 17 GB", | 20260505 04:55:35.008 | INFO | "huge" : "", | 20260505 04:55:35.008 | INFO | "huge_shared" : " 27 GB", | 20260505 04:55:35.008 | INFO | "malloc" : " 14 GB", | 20260505 04:55:35.008 | INFO | "pinned" : " 29 GB" | 20260505 04:55:35.008 | INFO | } | 20260505 04:55:35.009 | INFO | Perf report for the last 5.13927 seconds: | 20260505 04:55:35.009 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:55:35.009 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:35.009 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:35.009 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:35.009 | INFO | ANALYSIS : 57.2%, 3.0, 89.3ms, 0 | 20260505 04:55:35.009 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:35.009 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:35.009 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260505 04:55:35.009 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:35.009 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:35.009 | INFO | Uploaded 36GiB to GPU in 1.60178s (22.4749GiB/s) | 20260505 04:55:35.009 | INFO | Downloaded 34GiB to GPU in 1.51833s (22.393GiB/s) | 20260505 04:55:40.148 | INFO | Analyzing chunk frames = [502784,503296) | 20260505 04:55:40.148 | INFO | MemInfo: { | 20260505 04:55:40.148 | INFO | "gpu" : " 17 GB", | 20260505 04:55:40.148 | INFO | "huge" : "", | 20260505 04:55:40.148 | INFO | "huge_shared" : " 27 GB", | 20260505 04:55:40.148 | INFO | "malloc" : " 14 GB", | 20260505 04:55:40.148 | INFO | "pinned" : " 29 GB" | 20260505 04:55:40.148 | INFO | } | 20260505 04:55:40.148 | INFO | Perf report for the last 5.13939 seconds: | 20260505 04:55:40.148 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:55:40.148 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:40.148 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:40.148 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:40.148 | INFO | ANALYSIS : 57.5%, 3.0, 89.6ms, 0 | 20260505 04:55:40.148 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:40.148 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:40.148 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260505 04:55:40.148 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:40.148 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:40.149 | INFO | Uploaded 36GiB to GPU in 1.60185s (22.474GiB/s) | 20260505 04:55:40.149 | INFO | Downloaded 34GiB to GPU in 1.51912s (22.3813GiB/s) | 20260505 04:55:45.287 | INFO | Analyzing chunk frames = [503296,503808) | 20260505 04:55:45.287 | INFO | MemInfo: { | 20260505 04:55:45.287 | INFO | "gpu" : " 17 GB", | 20260505 04:55:45.287 | INFO | "huge" : "", | 20260505 04:55:45.287 | INFO | "huge_shared" : " 27 GB", | 20260505 04:55:45.287 | INFO | "malloc" : " 14 GB", | 20260505 04:55:45.287 | INFO | "pinned" : " 29 GB" | 20260505 04:55:45.287 | INFO | } | 20260505 04:55:45.287 | INFO | Perf report for the last 5.13937 seconds: | 20260505 04:55:45.287 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:55:45.287 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:45.287 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:45.287 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:45.287 | INFO | ANALYSIS : 57.5%, 3.0, 89.8ms, 0 | 20260505 04:55:45.287 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:45.288 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:45.288 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260505 04:55:45.288 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:45.288 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:45.288 | INFO | Uploaded 36GiB to GPU in 1.60222s (22.4688GiB/s) | 20260505 04:55:45.288 | INFO | Downloaded 34GiB to GPU in 1.51918s (22.3805GiB/s) | 20260505 04:55:45.385 | INFO | Starting a round of Baz IO | 20260505 04:55:48.336 | INFO | Populating RTMetrics report | 20260505 04:55:48.336 | INFO | RTMetrics report populated | 20260505 04:55:48.336 | INFO | Opening tmp RTMetrics json file | 20260505 04:55:48.337 | INFO | Serializing RTMetrics to json | 20260505 04:55:48.338 | INFO | Serialization done, swapping tmp file now... | 20260505 04:55:48.340 | INFO | RTMetrics done | 20260505 04:55:48.431 | INFO | Finished a round of Baz IO: Wrote 2.111 GiB in 3.045 seconds | 20260505 04:55:48.431 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.09% / 0.11% / 56.80% | 20260505 04:55:48.431 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.18567% / 0.000298023% / 4.37101e-05% | 20260505 04:55:48.431 | INFO | Baz IO overall: | 20260505 04:55:48.431 | INFO | BytesWritten : 216909844916 | 20260505 04:55:48.431 | INFO | NumEvents : 101187793691 | 20260505 04:55:50.426 | INFO | Analyzing chunk frames = [503808,504320) | 20260505 04:55:50.426 | INFO | MemInfo: { | 20260505 04:55:50.426 | INFO | "gpu" : " 17 GB", | 20260505 04:55:50.426 | INFO | "huge" : "", | 20260505 04:55:50.426 | INFO | "huge_shared" : " 27 GB", | 20260505 04:55:50.426 | INFO | "malloc" : " 14 GB", | 20260505 04:55:50.426 | INFO | "pinned" : " 29 GB" | 20260505 04:55:50.426 | INFO | } | 20260505 04:55:50.427 | INFO | Perf report for the last 5.13913 seconds: | 20260505 04:55:50.427 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:55:50.427 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:50.427 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:50.427 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:50.427 | INFO | ANALYSIS : 58.2%, 3.0, 90.5ms, 0 | 20260505 04:55:50.427 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:50.427 | INFO | RT_METRICS : 45.5%, 1.0, 23.8ms, 0 | 20260505 04:55:50.427 | INFO | PRE_HQ : 57.6%, 2.2, 65.0ms, 0 | 20260505 04:55:50.427 | INFO | BAZWRITER : 58.8%, 4.1, 124.4ms, 0 | 20260505 04:55:50.427 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 04:55:50.427 | INFO | Uploaded 36GiB to GPU in 1.63866s (21.9692GiB/s) | 20260505 04:55:50.427 | INFO | Downloaded 38GiB to GPU in 1.70447s (22.2944GiB/s) | 20260505 04:55:55.566 | INFO | Analyzing chunk frames = [504320,504832) | 20260505 04:55:55.566 | INFO | MemInfo: { | 20260505 04:55:55.566 | INFO | "gpu" : " 17 GB", | 20260505 04:55:55.566 | INFO | "huge" : "", | 20260505 04:55:55.566 | INFO | "huge_shared" : " 27 GB", | 20260505 04:55:55.566 | INFO | "malloc" : " 14 GB", | 20260505 04:55:55.566 | INFO | "pinned" : " 29 GB" | 20260505 04:55:55.566 | INFO | } | 20260505 04:55:55.566 | INFO | Perf report for the last 5.13953 seconds: | 20260505 04:55:55.566 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:55:55.566 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:55.566 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:55.566 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:55.566 | INFO | ANALYSIS : 57.9%, 3.0, 90.3ms, 0 | 20260505 04:55:55.566 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:55.566 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:55.566 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 04:55:55.566 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:55.566 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:55:55.566 | INFO | Uploaded 36GiB to GPU in 1.59974s (22.5036GiB/s) | 20260505 04:55:55.566 | INFO | Downloaded 34GiB to GPU in 1.51699s (22.4128GiB/s) | 20260505 04:56:00.705 | INFO | Analyzing chunk frames = [504832,505344) | 20260505 04:56:00.705 | INFO | MemInfo: { | 20260505 04:56:00.705 | INFO | "gpu" : " 17 GB", | 20260505 04:56:00.705 | INFO | "huge" : "", | 20260505 04:56:00.705 | INFO | "huge_shared" : " 27 GB", | 20260505 04:56:00.705 | INFO | "malloc" : " 14 GB", | 20260505 04:56:00.705 | INFO | "pinned" : " 29 GB" | 20260505 04:56:00.705 | INFO | } | 20260505 04:56:00.705 | INFO | Perf report for the last 5.13926 seconds: | 20260505 04:56:00.705 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:56:00.705 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:00.705 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:00.705 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:00.705 | INFO | ANALYSIS : 57.3%, 3.0, 89.1ms, 0 | 20260505 04:56:00.705 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:00.705 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:00.705 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260505 04:56:00.705 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:00.706 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:00.706 | INFO | Uploaded 36GiB to GPU in 1.60337s (22.4527GiB/s) | 20260505 04:56:00.706 | INFO | Downloaded 34GiB to GPU in 1.521s (22.3537GiB/s) | 20260505 04:56:05.845 | INFO | Analyzing chunk frames = [505344,505856) | 20260505 04:56:05.845 | INFO | MemInfo: { | 20260505 04:56:05.845 | INFO | "gpu" : " 17 GB", | 20260505 04:56:05.845 | INFO | "huge" : "", | 20260505 04:56:05.845 | INFO | "huge_shared" : " 27 GB", | 20260505 04:56:05.845 | INFO | "malloc" : " 14 GB", | 20260505 04:56:05.845 | INFO | "pinned" : " 29 GB" | 20260505 04:56:05.845 | INFO | } | 20260505 04:56:05.845 | INFO | Perf report for the last 5.13942 seconds: | 20260505 04:56:05.845 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:56:05.845 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:05.845 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:05.845 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:05.845 | INFO | ANALYSIS : 57.0%, 3.0, 88.4ms, 0 | 20260505 04:56:05.845 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:05.845 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:05.845 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260505 04:56:05.845 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:05.845 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:05.845 | INFO | Uploaded 36GiB to GPU in 1.60264s (22.4629GiB/s) | 20260505 04:56:05.845 | INFO | Downloaded 34GiB to GPU in 1.51943s (22.3769GiB/s) | 20260505 04:56:10.984 | INFO | Analyzing chunk frames = [505856,506368) | 20260505 04:56:10.984 | INFO | MemInfo: { | 20260505 04:56:10.984 | INFO | "gpu" : " 17 GB", | 20260505 04:56:10.984 | INFO | "huge" : "", | 20260505 04:56:10.984 | INFO | "huge_shared" : " 27 GB", | 20260505 04:56:10.984 | INFO | "malloc" : " 14 GB", | 20260505 04:56:10.984 | INFO | "pinned" : " 29 GB" | 20260505 04:56:10.984 | INFO | } | 20260505 04:56:10.984 | INFO | Perf report for the last 5.13926 seconds: | 20260505 04:56:10.984 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:56:10.984 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:10.984 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:10.984 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:10.984 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260505 04:56:10.984 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:10.984 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:10.984 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260505 04:56:10.984 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:10.984 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:10.984 | INFO | Uploaded 36GiB to GPU in 1.60092s (22.4871GiB/s) | 20260505 04:56:10.984 | INFO | Downloaded 34GiB to GPU in 1.51715s (22.4105GiB/s) | 20260505 04:56:16.123 | INFO | Analyzing chunk frames = [506368,506880) | 20260505 04:56:16.123 | INFO | MemInfo: { | 20260505 04:56:16.123 | INFO | "gpu" : " 17 GB", | 20260505 04:56:16.123 | INFO | "huge" : "", | 20260505 04:56:16.123 | INFO | "huge_shared" : " 27 GB", | 20260505 04:56:16.123 | INFO | "malloc" : " 14 GB", | 20260505 04:56:16.123 | INFO | "pinned" : " 29 GB" | 20260505 04:56:16.123 | INFO | } | 20260505 04:56:16.123 | INFO | Perf report for the last 5.13925 seconds: | 20260505 04:56:16.123 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:56:16.123 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:16.123 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:16.123 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:16.123 | INFO | ANALYSIS : 57.2%, 3.0, 89.3ms, 0 | 20260505 04:56:16.123 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:16.123 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:16.123 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260505 04:56:16.123 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:16.123 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:16.124 | INFO | Uploaded 36GiB to GPU in 1.602s (22.472GiB/s) | 20260505 04:56:16.124 | INFO | Downloaded 34GiB to GPU in 1.51834s (22.3929GiB/s) | 20260505 04:56:21.262 | INFO | Analyzing chunk frames = [506880,507392) | 20260505 04:56:21.262 | INFO | MemInfo: { | 20260505 04:56:21.262 | INFO | "gpu" : " 17 GB", | 20260505 04:56:21.262 | INFO | "huge" : "", | 20260505 04:56:21.262 | INFO | "huge_shared" : " 27 GB", | 20260505 04:56:21.262 | INFO | "malloc" : " 14 GB", | 20260505 04:56:21.262 | INFO | "pinned" : " 29 GB" | 20260505 04:56:21.263 | INFO | } | 20260505 04:56:21.263 | INFO | Perf report for the last 5.13925 seconds: | 20260505 04:56:21.263 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:56:21.263 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:21.263 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:21.263 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:21.263 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260505 04:56:21.263 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:21.263 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:21.263 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260505 04:56:21.263 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:21.263 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:21.263 | INFO | Uploaded 36GiB to GPU in 1.60158s (22.4778GiB/s) | 20260505 04:56:21.263 | INFO | Downloaded 34GiB to GPU in 1.5187s (22.3875GiB/s) | 20260505 04:56:26.402 | INFO | Analyzing chunk frames = [507392,507904) | 20260505 04:56:26.402 | INFO | MemInfo: { | 20260505 04:56:26.402 | INFO | "gpu" : " 17 GB", | 20260505 04:56:26.402 | INFO | "huge" : "", | 20260505 04:56:26.402 | INFO | "huge_shared" : " 27 GB", | 20260505 04:56:26.402 | INFO | "malloc" : " 14 GB", | 20260505 04:56:26.402 | INFO | "pinned" : " 29 GB" | 20260505 04:56:26.402 | INFO | } | 20260505 04:56:26.402 | INFO | Perf report for the last 5.13937 seconds: | 20260505 04:56:26.402 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:56:26.402 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:26.402 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:26.402 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:26.402 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260505 04:56:26.402 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:26.402 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:26.402 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 04:56:26.402 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:26.402 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:26.403 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 04:56:26.403 | INFO | Total mem usage: 15.1 GiB | 20260505 04:56:26.403 | INFO | All Time High Water Marks sum: 15.7 GiB (Savings: 4.09%) | 20260505 04:56:26.403 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -4.37%) | 20260505 04:56:26.403 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 04:56:26.403 | INFO | High Water Breakdown: | 20260505 04:56:26.403 | INFO | 59.5% : 8.58 GiB ( 95% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 04:56:26.403 | INFO | 40.4% : 5.82 GiB ( 87.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 04:56:26.403 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 04:56:26.403 | INFO | ------------------------------------------ | 20260505 04:56:26.403 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:56:26.403 | INFO | Uploaded 36GiB to GPU in 1.60184s (22.4742GiB/s) | 20260505 04:56:26.403 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 04:56:26.403 | INFO | Total mem usage: 26.5 GiB | 20260505 04:56:26.403 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 04:56:26.403 | INFO | Downloaded 34GiB to GPU in 1.51928s (22.379GiB/s) | 20260505 04:56:26.403 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 04:56:26.403 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 04:56:26.403 | INFO | High Water Breakdown: | 20260505 04:56:26.403 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:56:26.403 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:56:26.403 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:56:26.403 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:56:26.403 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:56:26.403 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:56:26.403 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 04:56:26.403 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 04:56:26.403 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:56:26.403 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 04:56:26.403 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 04:56:26.403 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 04:56:26.403 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 04:56:26.403 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 04:56:26.403 | INFO | ------------------------------------------ | 20260505 04:56:26.403 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:56:26.403 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 04:56:26.403 | INFO | Total mem usage: 25.6 GiB | 20260505 04:56:26.403 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 04:56:26.403 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 04:56:26.403 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 04:56:26.403 | INFO | High Water Breakdown: | 20260505 04:56:26.403 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 04:56:26.403 | INFO | ------------------------------------------ | 20260505 04:56:26.403 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:56:26.403 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 04:56:26.403 | INFO | Total mem usage: 15.7 GiB | 20260505 04:56:26.403 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 04:56:26.403 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 04:56:26.403 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260505 04:56:26.403 | INFO | High Water Breakdown: | 20260505 04:56:26.403 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 04:56:26.403 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 04:56:26.403 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 04:56:26.403 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 04:56:26.403 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 04:56:26.403 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 04:56:26.403 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 04:56:26.403 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 04:56:26.403 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 04:56:26.403 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:56:26.403 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 04:56:26.403 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:56:26.403 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 04:56:26.403 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 04:56:26.403 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 04:56:26.403 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 04:56:26.403 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:56:26.404 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 04:56:26.404 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:56:26.404 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:56:26.404 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:56:26.404 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 04:56:26.404 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 04:56:26.404 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:56:26.404 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 04:56:26.404 | INFO | ------------------------------------------ | 20260505 04:56:26.404 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:56:26.404 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 04:56:26.404 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:56:26.404 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:56:26.404 | INFO | PerfReport: Full Execution -- 3.04e+01 / -nan, 1.29e+01 / 0.00e+00 | 20260505 04:56:26.404 | INFO | PerfReport: 0.28%, AggregateStats -- 8.66e-02 / -nan, 3.85e-03 / 0.00e+00 | 20260505 04:56:26.404 | INFO | PerfReport: 10.84%, Baseline -- 3.30e+00 / -nan, 1.78e-02 / 0.00e+00 | 20260505 04:56:26.404 | INFO | PerfReport: 3.04%, Histogram -- 9.25e-01 / -nan, 5.30e-02 / 0.00e+00 | 20260505 04:56:26.404 | INFO | PerfReport: 16.03%, DME -- 3.90e+01 / -nan, 3.22e+00 / 0.00e+00 | 20260505 04:56:26.404 | INFO | PerfReport: 56.60%, FrameLabeling -- 1.72e+01 / -nan, 4.49e-02 / 0.00e+00 | 20260505 04:56:26.404 | INFO | PerfReport: 7.61%, PulseAccumulating -- 2.31e+00 / -nan, 1.53e-01 / 0.00e+00 | 20260505 04:56:26.404 | INFO | PerfReport: 3.96%, Metrics -- 1.20e+00 / -nan, 1.97e-01 / 0.00e+00 | 20260505 04:56:26.404 | INFO | PerfReport: 1.63%, PulseSerializing -- 4.97e-01 / -nan, 8.04e-02 / 0.00e+00 | 20260505 04:56:26.404 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 04:56:26.404 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:56:26.404 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:56:26.404 | INFO | PerfReport: Full Execution -- 9.13e+01 / -nan, 1.91e+01 / 0.00e+00 | 20260505 04:56:26.404 | INFO | PerfReport: 18.32%, Upload -- 1.67e+01 / 0.00e+00, 3.70e-01 / 0.00e+00 | 20260505 04:56:26.404 | INFO | PerfReport: 17.58%, Download -- 1.61e+01 / 0.00e+00, 6.97e-01 / 0.00e+00 | 20260505 04:56:26.404 | INFO | PerfReport: 33.30%, Compute -- 3.04e+01 / 0.00e+00, 1.29e+01 / 0.00e+00 | 20260505 04:56:26.404 | INFO | PerfReport: 0.55%, LockUpload -- 5.03e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260505 04:56:26.404 | INFO | PerfReport: 0.00%, LockDownload -- 3.91e-04 / 0.00e+00, 9.46e-05 / 0.00e+00 | 20260505 04:56:26.404 | INFO | PerfReport: 30.22%, LockCompute -- 2.76e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260505 04:56:26.404 | INFO | PerfReport: 0.02%, Misc -- 1.38e-02 / 0.00e+00, 2.28e-02 / 0.00e+00 | 20260505 04:56:26.499 | INFO | Starting a round of Baz IO | 20260505 04:56:29.450 | INFO | Populating RTMetrics report | 20260505 04:56:29.451 | INFO | RTMetrics report populated | 20260505 04:56:29.451 | INFO | Opening tmp RTMetrics json file | 20260505 04:56:29.451 | INFO | Serializing RTMetrics to json | 20260505 04:56:29.453 | INFO | Serialization done, swapping tmp file now... | 20260505 04:56:29.453 | INFO | RTMetrics done | 20260505 04:56:29.544 | INFO | Finished a round of Baz IO: Wrote 2.113 GiB in 3.044 seconds | 20260505 04:56:29.544 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.98% / 0.11% / 56.91% | 20260505 04:56:29.544 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.1455% / 0.000202656% / 5.5631e-05% | 20260505 04:56:29.544 | INFO | Baz IO overall: | 20260505 04:56:29.544 | INFO | BytesWritten : 219179107130 | 20260505 04:56:29.544 | INFO | NumEvents : 102265550781 | 20260505 04:56:31.541 | INFO | Analyzing chunk frames = [507904,508416) | 20260505 04:56:31.541 | INFO | MemInfo: { | 20260505 04:56:31.541 | INFO | "gpu" : " 17 GB", | 20260505 04:56:31.541 | INFO | "huge" : "", | 20260505 04:56:31.541 | INFO | "huge_shared" : " 27 GB", | 20260505 04:56:31.541 | INFO | "malloc" : " 14 GB", | 20260505 04:56:31.541 | INFO | "pinned" : " 29 GB" | 20260505 04:56:31.541 | INFO | } | 20260505 04:56:31.541 | INFO | Perf report for the last 5.13924 seconds: | 20260505 04:56:31.541 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:56:31.541 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:31.541 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:31.541 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:31.541 | INFO | ANALYSIS : 58.2%, 3.0, 90.5ms, 0 | 20260505 04:56:31.541 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:31.541 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260505 04:56:31.541 | INFO | PRE_HQ : 57.5%, 2.2, 65.0ms, 0 | 20260505 04:56:31.541 | INFO | BAZWRITER : 58.8%, 4.1, 124.2ms, 0 | 20260505 04:56:31.541 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 04:56:31.542 | INFO | Uploaded 36GiB to GPU in 1.6396s (21.9566GiB/s) | 20260505 04:56:31.542 | INFO | Downloaded 38GiB to GPU in 1.70298s (22.3138GiB/s) | 20260505 04:56:36.680 | INFO | Analyzing chunk frames = [508416,508928) | 20260505 04:56:36.680 | INFO | MemInfo: { | 20260505 04:56:36.680 | INFO | "gpu" : " 17 GB", | 20260505 04:56:36.680 | INFO | "huge" : "", | 20260505 04:56:36.681 | INFO | "huge_shared" : " 27 GB", | 20260505 04:56:36.681 | INFO | "malloc" : " 14 GB", | 20260505 04:56:36.681 | INFO | "pinned" : " 29 GB" | 20260505 04:56:36.681 | INFO | } | 20260505 04:56:36.681 | INFO | Perf report for the last 5.13946 seconds: | 20260505 04:56:36.681 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:56:36.681 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:36.681 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:36.681 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:36.681 | INFO | ANALYSIS : 57.9%, 3.0, 90.3ms, 0 | 20260505 04:56:36.681 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:36.681 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:36.681 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 04:56:36.681 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:36.681 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:36.681 | INFO | Uploaded 36GiB to GPU in 1.59937s (22.5088GiB/s) | 20260505 04:56:36.681 | INFO | Downloaded 34GiB to GPU in 1.51596s (22.428GiB/s) | 20260505 04:56:41.820 | INFO | Analyzing chunk frames = [508928,509440) | 20260505 04:56:41.820 | INFO | MemInfo: { | 20260505 04:56:41.820 | INFO | "gpu" : " 17 GB", | 20260505 04:56:41.820 | INFO | "huge" : "", | 20260505 04:56:41.820 | INFO | "huge_shared" : " 27 GB", | 20260505 04:56:41.820 | INFO | "malloc" : " 14 GB", | 20260505 04:56:41.820 | INFO | "pinned" : " 29 GB" | 20260505 04:56:41.820 | INFO | } | 20260505 04:56:41.820 | INFO | Perf report for the last 5.13939 seconds: | 20260505 04:56:41.820 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:56:41.820 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:41.820 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:41.820 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:41.820 | INFO | ANALYSIS : 57.3%, 3.0, 89.1ms, 0 | 20260505 04:56:41.820 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:41.820 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:41.820 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 04:56:41.820 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:41.820 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:41.821 | INFO | Uploaded 36GiB to GPU in 1.60321s (22.4549GiB/s) | 20260505 04:56:41.821 | INFO | Downloaded 34GiB to GPU in 1.51997s (22.3689GiB/s) | 20260505 04:56:46.959 | INFO | Analyzing chunk frames = [509440,509952) | 20260505 04:56:46.959 | INFO | MemInfo: { | 20260505 04:56:46.959 | INFO | "gpu" : " 17 GB", | 20260505 04:56:46.959 | INFO | "huge" : "", | 20260505 04:56:46.959 | INFO | "huge_shared" : " 27 GB", | 20260505 04:56:46.959 | INFO | "malloc" : " 14 GB", | 20260505 04:56:46.959 | INFO | "pinned" : " 29 GB" | 20260505 04:56:46.959 | INFO | } | 20260505 04:56:46.960 | INFO | Perf report for the last 5.13941 seconds: | 20260505 04:56:46.960 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:56:46.960 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:46.960 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:46.960 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:46.960 | INFO | ANALYSIS : 57.0%, 3.0, 88.4ms, 0 | 20260505 04:56:46.960 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:46.960 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:46.960 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260505 04:56:46.960 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:46.960 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:46.960 | INFO | Uploaded 36GiB to GPU in 1.60386s (22.4459GiB/s) | 20260505 04:56:46.960 | INFO | Downloaded 34GiB to GPU in 1.52108s (22.3525GiB/s) | 20260505 04:56:52.098 | INFO | Analyzing chunk frames = [509952,510464) | 20260505 04:56:52.098 | INFO | MemInfo: { | 20260505 04:56:52.098 | INFO | "gpu" : " 17 GB", | 20260505 04:56:52.099 | INFO | "huge" : "", | 20260505 04:56:52.099 | INFO | "huge_shared" : " 27 GB", | 20260505 04:56:52.099 | INFO | "malloc" : " 14 GB", | 20260505 04:56:52.099 | INFO | "pinned" : " 29 GB" | 20260505 04:56:52.099 | INFO | } | 20260505 04:56:52.099 | INFO | Perf report for the last 5.13918 seconds: | 20260505 04:56:52.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:56:52.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:52.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:52.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:52.099 | INFO | ANALYSIS : 57.0%, 3.0, 88.8ms, 0 | 20260505 04:56:52.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:52.099 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:52.099 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 04:56:52.099 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:52.099 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:52.099 | INFO | Uploaded 36GiB to GPU in 1.60083s (22.4883GiB/s) | 20260505 04:56:52.099 | INFO | Downloaded 34GiB to GPU in 1.51706s (22.4118GiB/s) | 20260505 04:56:57.238 | INFO | Analyzing chunk frames = [510464,510976) | 20260505 04:56:57.238 | INFO | MemInfo: { | 20260505 04:56:57.238 | INFO | "gpu" : " 17 GB", | 20260505 04:56:57.238 | INFO | "huge" : "", | 20260505 04:56:57.238 | INFO | "huge_shared" : " 27 GB", | 20260505 04:56:57.238 | INFO | "malloc" : " 14 GB", | 20260505 04:56:57.238 | INFO | "pinned" : " 29 GB" | 20260505 04:56:57.238 | INFO | } | 20260505 04:56:57.238 | INFO | Perf report for the last 5.13939 seconds: | 20260505 04:56:57.238 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:56:57.238 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:57.238 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:57.238 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:57.238 | INFO | ANALYSIS : 57.2%, 3.0, 89.3ms, 0 | 20260505 04:56:57.238 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:57.238 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:57.238 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 04:56:57.238 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:57.238 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:56:57.239 | INFO | Uploaded 36GiB to GPU in 1.60089s (22.4875GiB/s) | 20260505 04:56:57.239 | INFO | Downloaded 34GiB to GPU in 1.51719s (22.4099GiB/s) | 20260505 04:57:02.377 | INFO | Analyzing chunk frames = [510976,511488) | 20260505 04:57:02.377 | INFO | MemInfo: { | 20260505 04:57:02.377 | INFO | "gpu" : " 17 GB", | 20260505 04:57:02.377 | INFO | "huge" : "", | 20260505 04:57:02.377 | INFO | "huge_shared" : " 27 GB", | 20260505 04:57:02.377 | INFO | "malloc" : " 14 GB", | 20260505 04:57:02.377 | INFO | "pinned" : " 29 GB" | 20260505 04:57:02.377 | INFO | } | 20260505 04:57:02.377 | INFO | Perf report for the last 5.13934 seconds: | 20260505 04:57:02.377 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:57:02.377 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:02.377 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:02.377 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:02.378 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260505 04:57:02.378 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:02.378 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:02.378 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 04:57:02.378 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:02.378 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:02.378 | INFO | Uploaded 36GiB to GPU in 1.60208s (22.4708GiB/s) | 20260505 04:57:02.378 | INFO | Downloaded 34GiB to GPU in 1.51885s (22.3854GiB/s) | 20260505 04:57:07.517 | INFO | Analyzing chunk frames = [511488,512000) | 20260505 04:57:07.517 | INFO | MemInfo: { | 20260505 04:57:07.517 | INFO | "gpu" : " 17 GB", | 20260505 04:57:07.517 | INFO | "huge" : "", | 20260505 04:57:07.517 | INFO | "huge_shared" : " 27 GB", | 20260505 04:57:07.517 | INFO | "malloc" : " 14 GB", | 20260505 04:57:07.517 | INFO | "pinned" : " 29 GB" | 20260505 04:57:07.517 | INFO | } | 20260505 04:57:07.517 | INFO | Perf report for the last 5.13949 seconds: | 20260505 04:57:07.517 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:57:07.517 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:07.517 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:07.517 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:07.517 | INFO | ANALYSIS : 57.6%, 3.0, 89.8ms, 0 | 20260505 04:57:07.517 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:07.517 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:07.517 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 04:57:07.517 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:07.517 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:07.518 | INFO | Uploaded 36GiB to GPU in 1.60094s (22.4868GiB/s) | 20260505 04:57:07.518 | INFO | Downloaded 34GiB to GPU in 1.51947s (22.3762GiB/s) | 20260505 04:57:07.614 | INFO | Starting a round of Baz IO | 20260505 04:57:10.562 | INFO | Populating RTMetrics report | 20260505 04:57:10.562 | INFO | RTMetrics report populated | 20260505 04:57:10.562 | INFO | Opening tmp RTMetrics json file | 20260505 04:57:10.563 | INFO | Serializing RTMetrics to json | 20260505 04:57:10.564 | INFO | Serialization done, swapping tmp file now... | 20260505 04:57:10.565 | INFO | RTMetrics done | 20260505 04:57:10.655 | INFO | Finished a round of Baz IO: Wrote 2.113 GiB in 3.041 seconds | 20260505 04:57:10.655 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.87% / 0.11% / 57.02% | 20260505 04:57:10.656 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.07096% / 0.000230471% / 5.16574e-05% | 20260505 04:57:10.656 | INFO | Baz IO overall: | 20260505 04:57:10.656 | INFO | BytesWritten : 221447703151 | 20260505 04:57:10.656 | INFO | NumEvents : 103342552701 | 20260505 04:57:12.656 | INFO | Analyzing chunk frames = [512000,512512) | 20260505 04:57:12.656 | INFO | MemInfo: { | 20260505 04:57:12.656 | INFO | "gpu" : " 17 GB", | 20260505 04:57:12.656 | INFO | "huge" : "", | 20260505 04:57:12.656 | INFO | "huge_shared" : " 27 GB", | 20260505 04:57:12.656 | INFO | "malloc" : " 14 GB", | 20260505 04:57:12.656 | INFO | "pinned" : " 29 GB" | 20260505 04:57:12.656 | INFO | } | 20260505 04:57:12.656 | INFO | Perf report for the last 5.13921 seconds: | 20260505 04:57:12.656 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:57:12.656 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:12.656 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:12.656 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:12.656 | INFO | ANALYSIS : 58.1%, 3.0, 90.3ms, 0 | 20260505 04:57:12.656 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:12.656 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260505 04:57:12.656 | INFO | PRE_HQ : 57.5%, 2.2, 64.2ms, 0 | 20260505 04:57:12.656 | INFO | BAZWRITER : 58.7%, 4.1, 124.4ms, 0 | 20260505 04:57:12.656 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:57:12.656 | INFO | Uploaded 36GiB to GPU in 1.6377s (21.9821GiB/s) | 20260505 04:57:12.657 | INFO | Downloaded 38GiB to GPU in 1.70352s (22.3068GiB/s) | 20260505 04:57:17.795 | INFO | Analyzing chunk frames = [512512,513024) | 20260505 04:57:17.795 | INFO | MemInfo: { | 20260505 04:57:17.795 | INFO | "gpu" : " 17 GB", | 20260505 04:57:17.795 | INFO | "huge" : "", | 20260505 04:57:17.795 | INFO | "huge_shared" : " 27 GB", | 20260505 04:57:17.795 | INFO | "malloc" : " 14 GB", | 20260505 04:57:17.795 | INFO | "pinned" : " 29 GB" | 20260505 04:57:17.795 | INFO | } | 20260505 04:57:17.795 | INFO | Perf report for the last 5.13925 seconds: | 20260505 04:57:17.795 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:57:17.795 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:17.795 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:17.796 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:17.796 | INFO | ANALYSIS : 58.0%, 3.0, 90.4ms, 0 | 20260505 04:57:17.796 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:17.796 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:17.796 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 04:57:17.796 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:17.796 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:17.796 | INFO | Uploaded 36GiB to GPU in 1.59954s (22.5064GiB/s) | 20260505 04:57:17.796 | INFO | Downloaded 34GiB to GPU in 1.51566s (22.4325GiB/s) | 20260505 04:57:22.934 | INFO | Analyzing chunk frames = [513024,513536) | 20260505 04:57:22.934 | INFO | MemInfo: { | 20260505 04:57:22.934 | INFO | "gpu" : " 17 GB", | 20260505 04:57:22.934 | INFO | "huge" : "", | 20260505 04:57:22.934 | INFO | "huge_shared" : " 27 GB", | 20260505 04:57:22.934 | INFO | "malloc" : " 14 GB", | 20260505 04:57:22.934 | INFO | "pinned" : " 29 GB" | 20260505 04:57:22.935 | INFO | } | 20260505 04:57:22.935 | INFO | Perf report for the last 5.13922 seconds: | 20260505 04:57:22.935 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:57:22.935 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:22.935 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:22.935 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:22.935 | INFO | ANALYSIS : 57.3%, 3.0, 89.0ms, 0 | 20260505 04:57:22.935 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:22.935 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:22.935 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260505 04:57:22.935 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:22.935 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:22.935 | INFO | Uploaded 36GiB to GPU in 1.60172s (22.4758GiB/s) | 20260505 04:57:22.935 | INFO | Downloaded 34GiB to GPU in 1.51994s (22.3693GiB/s) | 20260505 04:57:28.074 | INFO | Analyzing chunk frames = [513536,514048) | 20260505 04:57:28.074 | INFO | MemInfo: { | 20260505 04:57:28.074 | INFO | "gpu" : " 17 GB", | 20260505 04:57:28.074 | INFO | "huge" : "", | 20260505 04:57:28.074 | INFO | "huge_shared" : " 27 GB", | 20260505 04:57:28.074 | INFO | "malloc" : " 14 GB", | 20260505 04:57:28.074 | INFO | "pinned" : " 29 GB" | 20260505 04:57:28.074 | INFO | } | 20260505 04:57:28.074 | INFO | Perf report for the last 5.13942 seconds: | 20260505 04:57:28.074 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:57:28.074 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:28.074 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:28.074 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:28.074 | INFO | ANALYSIS : 57.0%, 3.0, 88.4ms, 0 | 20260505 04:57:28.074 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:28.074 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:28.074 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 04:57:28.074 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:28.074 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:28.075 | INFO | Uploaded 36GiB to GPU in 1.60153s (22.4785GiB/s) | 20260505 04:57:28.075 | INFO | Downloaded 34GiB to GPU in 1.51983s (22.3709GiB/s) | 20260505 04:57:33.213 | INFO | Analyzing chunk frames = [514048,514560) | 20260505 04:57:33.213 | INFO | MemInfo: { | 20260505 04:57:33.213 | INFO | "gpu" : " 17 GB", | 20260505 04:57:33.213 | INFO | "huge" : "", | 20260505 04:57:33.213 | INFO | "huge_shared" : " 27 GB", | 20260505 04:57:33.213 | INFO | "malloc" : " 14 GB", | 20260505 04:57:33.213 | INFO | "pinned" : " 29 GB" | 20260505 04:57:33.213 | INFO | } | 20260505 04:57:33.213 | INFO | Perf report for the last 5.13934 seconds: | 20260505 04:57:33.213 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:57:33.213 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:33.213 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:33.214 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:33.214 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260505 04:57:33.214 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:33.214 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:33.214 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 04:57:33.214 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:33.214 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:33.214 | INFO | Uploaded 36GiB to GPU in 1.60078s (22.489GiB/s) | 20260505 04:57:33.214 | INFO | Downloaded 34GiB to GPU in 1.51795s (22.3986GiB/s) | 20260505 04:57:38.352 | INFO | Analyzing chunk frames = [514560,515072) | 20260505 04:57:38.352 | INFO | MemInfo: { | 20260505 04:57:38.352 | INFO | "gpu" : " 17 GB", | 20260505 04:57:38.352 | INFO | "huge" : "", | 20260505 04:57:38.353 | INFO | "huge_shared" : " 27 GB", | 20260505 04:57:38.353 | INFO | "malloc" : " 14 GB", | 20260505 04:57:38.353 | INFO | "pinned" : " 29 GB" | 20260505 04:57:38.353 | INFO | } | 20260505 04:57:38.353 | INFO | Perf report for the last 5.13922 seconds: | 20260505 04:57:38.353 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:57:38.353 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:38.353 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:38.353 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:38.353 | INFO | ANALYSIS : 57.2%, 3.0, 89.3ms, 0 | 20260505 04:57:38.353 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:38.353 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:38.353 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 04:57:38.353 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:38.353 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:38.353 | INFO | Uploaded 36GiB to GPU in 1.60075s (22.4895GiB/s) | 20260505 04:57:38.353 | INFO | Downloaded 34GiB to GPU in 1.51745s (22.406GiB/s) | 20260505 04:57:43.492 | INFO | Analyzing chunk frames = [515072,515584) | 20260505 04:57:43.492 | INFO | MemInfo: { | 20260505 04:57:43.492 | INFO | "gpu" : " 17 GB", | 20260505 04:57:43.492 | INFO | "huge" : "", | 20260505 04:57:43.492 | INFO | "huge_shared" : " 27 GB", | 20260505 04:57:43.492 | INFO | "malloc" : " 14 GB", | 20260505 04:57:43.492 | INFO | "pinned" : " 29 GB" | 20260505 04:57:43.492 | INFO | } | 20260505 04:57:43.492 | INFO | Perf report for the last 5.13938 seconds: | 20260505 04:57:43.492 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:57:43.492 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:43.492 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:43.492 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:43.492 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260505 04:57:43.492 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:43.492 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:43.492 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260505 04:57:43.492 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:43.492 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:43.492 | INFO | Uploaded 36GiB to GPU in 1.60158s (22.4778GiB/s) | 20260505 04:57:43.493 | INFO | Downloaded 34GiB to GPU in 1.51878s (22.3864GiB/s) | 20260505 04:57:48.631 | INFO | Analyzing chunk frames = [515584,516096) | 20260505 04:57:48.631 | INFO | MemInfo: { | 20260505 04:57:48.631 | INFO | "gpu" : " 17 GB", | 20260505 04:57:48.631 | INFO | "huge" : "", | 20260505 04:57:48.631 | INFO | "huge_shared" : " 27 GB", | 20260505 04:57:48.631 | INFO | "malloc" : " 14 GB", | 20260505 04:57:48.631 | INFO | "pinned" : " 29 GB" | 20260505 04:57:48.631 | INFO | } | 20260505 04:57:48.631 | INFO | Perf report for the last 5.13927 seconds: | 20260505 04:57:48.631 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:57:48.631 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:48.631 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:48.631 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:48.631 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260505 04:57:48.631 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:48.632 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:48.632 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 04:57:48.632 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:48.632 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:48.632 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 04:57:48.632 | INFO | Total mem usage: 15.1 GiB | 20260505 04:57:48.632 | INFO | All Time High Water Marks sum: 15.7 GiB (Savings: 4.09%) | 20260505 04:57:48.632 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -4.3%) | 20260505 04:57:48.632 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 04:57:48.632 | INFO | High Water Breakdown: | 20260505 04:57:48.632 | INFO | 59.6% : 8.6 GiB ( 95.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 04:57:48.632 | INFO | 40.3% : 5.82 GiB ( 87.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 04:57:48.632 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 04:57:48.632 | INFO | ------------------------------------------ | 20260505 04:57:48.632 | INFO | Uploaded 36GiB to GPU in 1.6008s (22.4888GiB/s) | 20260505 04:57:48.632 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:57:48.632 | INFO | Downloaded 34GiB to GPU in 1.51942s (22.3769GiB/s) | 20260505 04:57:48.632 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 04:57:48.632 | INFO | Total mem usage: 26.5 GiB | 20260505 04:57:48.632 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 04:57:48.632 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 04:57:48.632 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 04:57:48.632 | INFO | High Water Breakdown: | 20260505 04:57:48.632 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:57:48.632 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:57:48.632 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:57:48.632 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:57:48.632 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:57:48.632 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:57:48.632 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 04:57:48.632 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 04:57:48.632 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:57:48.632 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 04:57:48.632 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 04:57:48.633 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 04:57:48.633 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 04:57:48.633 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 04:57:48.633 | INFO | ------------------------------------------ | 20260505 04:57:48.633 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:57:48.633 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 04:57:48.633 | INFO | Total mem usage: 25.6 GiB | 20260505 04:57:48.633 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 04:57:48.633 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 04:57:48.633 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 04:57:48.633 | INFO | High Water Breakdown: | 20260505 04:57:48.633 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 04:57:48.633 | INFO | ------------------------------------------ | 20260505 04:57:48.633 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:57:48.633 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 04:57:48.633 | INFO | Total mem usage: 15.7 GiB | 20260505 04:57:48.633 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 04:57:48.633 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 04:57:48.633 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260505 04:57:48.633 | INFO | High Water Breakdown: | 20260505 04:57:48.633 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 04:57:48.633 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 04:57:48.633 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 04:57:48.633 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 04:57:48.633 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 04:57:48.633 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 04:57:48.633 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 04:57:48.633 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 04:57:48.633 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 04:57:48.633 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:57:48.633 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 04:57:48.633 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:57:48.633 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 04:57:48.633 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 04:57:48.633 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 04:57:48.633 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 04:57:48.633 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:57:48.633 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 04:57:48.633 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:57:48.633 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:57:48.633 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:57:48.633 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 04:57:48.633 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 04:57:48.633 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:57:48.633 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 04:57:48.633 | INFO | ------------------------------------------ | 20260505 04:57:48.633 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:57:48.633 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 04:57:48.633 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:57:48.633 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:57:48.633 | INFO | PerfReport: Full Execution -- 3.04e+01 / -nan, 1.28e+01 / 0.00e+00 | 20260505 04:57:48.633 | INFO | PerfReport: 0.28%, AggregateStats -- 8.64e-02 / -nan, 6.81e-03 / 0.00e+00 | 20260505 04:57:48.633 | INFO | PerfReport: 10.84%, Baseline -- 3.30e+00 / -nan, 1.80e-02 / 0.00e+00 | 20260505 04:57:48.633 | INFO | PerfReport: 3.04%, Histogram -- 9.24e-01 / -nan, 5.23e-02 / 0.00e+00 | 20260505 04:57:48.633 | INFO | PerfReport: 16.04%, DME -- 3.90e+01 / -nan, 3.27e+00 / 0.00e+00 | 20260505 04:57:48.633 | INFO | PerfReport: 56.61%, FrameLabeling -- 1.72e+01 / -nan, 4.55e-02 / 0.00e+00 | 20260505 04:57:48.633 | INFO | PerfReport: 7.60%, PulseAccumulating -- 2.31e+00 / -nan, 1.52e-01 / 0.00e+00 | 20260505 04:57:48.633 | INFO | PerfReport: 3.95%, Metrics -- 1.20e+00 / -nan, 1.99e-01 / 0.00e+00 | 20260505 04:57:48.633 | INFO | PerfReport: 1.63%, PulseSerializing -- 4.95e-01 / -nan, 7.98e-02 / 0.00e+00 | 20260505 04:57:48.633 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 04:57:48.633 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:57:48.633 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:57:48.633 | INFO | PerfReport: Full Execution -- 9.13e+01 / -nan, 1.90e+01 / 0.00e+00 | 20260505 04:57:48.633 | INFO | PerfReport: 18.32%, Upload -- 1.67e+01 / 0.00e+00, 3.75e-01 / 0.00e+00 | 20260505 04:57:48.633 | INFO | PerfReport: 17.59%, Download -- 1.61e+01 / 0.00e+00, 7.03e-01 / 0.00e+00 | 20260505 04:57:48.633 | INFO | PerfReport: 33.31%, Compute -- 3.04e+01 / 0.00e+00, 1.28e+01 / 0.00e+00 | 20260505 04:57:48.633 | INFO | PerfReport: 0.55%, LockUpload -- 5.01e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260505 04:57:48.633 | INFO | PerfReport: 0.00%, LockDownload -- 3.92e-04 / 0.00e+00, 1.58e-04 / 0.00e+00 | 20260505 04:57:48.633 | INFO | PerfReport: 30.22%, LockCompute -- 2.76e+01 / 0.00e+00, 1.72e+01 / 0.00e+00 | 20260505 04:57:48.633 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.67e-02 / 0.00e+00 | 20260505 04:57:48.729 | INFO | Starting a round of Baz IO | 20260505 04:57:51.678 | INFO | Populating RTMetrics report | 20260505 04:57:51.678 | INFO | RTMetrics report populated | 20260505 04:57:51.678 | INFO | Opening tmp RTMetrics json file | 20260505 04:57:51.683 | INFO | Serializing RTMetrics to json | 20260505 04:57:51.685 | INFO | Serialization done, swapping tmp file now... | 20260505 04:57:51.686 | INFO | RTMetrics done | 20260505 04:57:51.773 | INFO | Finished a round of Baz IO: Wrote 2.117 GiB in 3.044 seconds | 20260505 04:57:51.773 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.77% / 0.11% / 57.13% | 20260505 04:57:51.773 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.03416% / 0.000234445% / 5.5631e-05% | 20260505 04:57:51.773 | INFO | Baz IO overall: | 20260505 04:57:51.773 | INFO | BytesWritten : 223721326902 | 20260505 04:57:51.773 | INFO | NumEvents : 104421683810 | 20260505 04:57:53.771 | INFO | Analyzing chunk frames = [516096,516608) | 20260505 04:57:53.771 | INFO | MemInfo: { | 20260505 04:57:53.771 | INFO | "gpu" : " 17 GB", | 20260505 04:57:53.771 | INFO | "huge" : "", | 20260505 04:57:53.771 | INFO | "huge_shared" : " 27 GB", | 20260505 04:57:53.771 | INFO | "malloc" : " 14 GB", | 20260505 04:57:53.771 | INFO | "pinned" : " 29 GB" | 20260505 04:57:53.771 | INFO | } | 20260505 04:57:53.771 | INFO | Perf report for the last 5.13951 seconds: | 20260505 04:57:53.771 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:57:53.771 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:53.771 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:53.771 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:53.771 | INFO | ANALYSIS : 58.1%, 3.0, 90.3ms, 0 | 20260505 04:57:53.771 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:53.771 | INFO | RT_METRICS : 45.5%, 1.0, 23.8ms, 0 | 20260505 04:57:53.771 | INFO | PRE_HQ : 57.5%, 2.2, 64.4ms, 0 | 20260505 04:57:53.771 | INFO | BAZWRITER : 58.8%, 4.1, 125.2ms, 0 | 20260505 04:57:53.771 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 04:57:53.772 | INFO | Uploaded 36GiB to GPU in 1.63684s (21.9936GiB/s) | 20260505 04:57:53.772 | INFO | Downloaded 38GiB to GPU in 1.70103s (22.3395GiB/s) | 20260505 04:57:58.910 | INFO | Analyzing chunk frames = [516608,517120) | 20260505 04:57:58.910 | INFO | MemInfo: { | 20260505 04:57:58.911 | INFO | "gpu" : " 17 GB", | 20260505 04:57:58.911 | INFO | "huge" : "", | 20260505 04:57:58.911 | INFO | "huge_shared" : " 27 GB", | 20260505 04:57:58.911 | INFO | "malloc" : " 14 GB", | 20260505 04:57:58.911 | INFO | "pinned" : " 29 GB" | 20260505 04:57:58.911 | INFO | } | 20260505 04:57:58.911 | INFO | Perf report for the last 5.14017 seconds: | 20260505 04:57:58.911 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:57:58.911 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:58.911 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:58.911 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:58.911 | INFO | ANALYSIS : 58.0%, 3.0, 90.5ms, 0 | 20260505 04:57:58.911 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:58.911 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:58.911 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 04:57:58.911 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:58.911 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:57:58.911 | INFO | Uploaded 36GiB to GPU in 1.59894s (22.515GiB/s) | 20260505 04:57:58.911 | INFO | Downloaded 34GiB to GPU in 1.51568s (22.4322GiB/s) | 20260505 04:58:04.049 | INFO | Analyzing chunk frames = [517120,517632) | 20260505 04:58:04.049 | INFO | MemInfo: { | 20260505 04:58:04.049 | INFO | "gpu" : " 17 GB", | 20260505 04:58:04.049 | INFO | "huge" : "", | 20260505 04:58:04.049 | INFO | "huge_shared" : " 27 GB", | 20260505 04:58:04.049 | INFO | "malloc" : " 14 GB", | 20260505 04:58:04.049 | INFO | "pinned" : " 29 GB" | 20260505 04:58:04.049 | INFO | } | 20260505 04:58:04.049 | INFO | Perf report for the last 5.13837 seconds: | 20260505 04:58:04.049 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:58:04.050 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:04.050 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:04.050 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:04.050 | INFO | ANALYSIS : 57.3%, 3.0, 89.0ms, 0 | 20260505 04:58:04.050 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:04.050 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:04.050 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260505 04:58:04.050 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:04.050 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:04.050 | INFO | Uploaded 36GiB to GPU in 1.6036s (22.4495GiB/s) | 20260505 04:58:04.050 | INFO | Downloaded 34GiB to GPU in 1.52068s (22.3584GiB/s) | 20260505 04:58:09.189 | INFO | Analyzing chunk frames = [517632,518144) | 20260505 04:58:09.189 | INFO | MemInfo: { | 20260505 04:58:09.189 | INFO | "gpu" : " 17 GB", | 20260505 04:58:09.189 | INFO | "huge" : "", | 20260505 04:58:09.189 | INFO | "huge_shared" : " 27 GB", | 20260505 04:58:09.189 | INFO | "malloc" : " 14 GB", | 20260505 04:58:09.189 | INFO | "pinned" : " 29 GB" | 20260505 04:58:09.189 | INFO | } | 20260505 04:58:09.189 | INFO | Perf report for the last 5.1392 seconds: | 20260505 04:58:09.189 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:58:09.189 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:09.189 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:09.189 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:09.189 | INFO | ANALYSIS : 57.0%, 3.0, 88.4ms, 0 | 20260505 04:58:09.189 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:09.189 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:09.189 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 04:58:09.189 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:09.189 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:09.189 | INFO | Uploaded 36GiB to GPU in 1.60252s (22.4646GiB/s) | 20260505 04:58:09.189 | INFO | Downloaded 34GiB to GPU in 1.51915s (22.3809GiB/s) | 20260505 04:58:14.328 | INFO | Analyzing chunk frames = [518144,518656) | 20260505 04:58:14.328 | INFO | MemInfo: { | 20260505 04:58:14.328 | INFO | "gpu" : " 17 GB", | 20260505 04:58:14.328 | INFO | "huge" : "", | 20260505 04:58:14.328 | INFO | "huge_shared" : " 27 GB", | 20260505 04:58:14.328 | INFO | "malloc" : " 14 GB", | 20260505 04:58:14.328 | INFO | "pinned" : " 29 GB" | 20260505 04:58:14.328 | INFO | } | 20260505 04:58:14.328 | INFO | Perf report for the last 5.13971 seconds: | 20260505 04:58:14.328 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:58:14.328 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:14.328 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:14.328 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:14.329 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260505 04:58:14.329 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:14.329 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:14.329 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260505 04:58:14.329 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:14.329 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:14.329 | INFO | Uploaded 36GiB to GPU in 1.6015s (22.4789GiB/s) | 20260505 04:58:14.329 | INFO | Downloaded 34GiB to GPU in 1.51814s (22.3958GiB/s) | 20260505 04:58:19.467 | INFO | Analyzing chunk frames = [518656,519168) | 20260505 04:58:19.467 | INFO | MemInfo: { | 20260505 04:58:19.467 | INFO | "gpu" : " 17 GB", | 20260505 04:58:19.467 | INFO | "huge" : "", | 20260505 04:58:19.467 | INFO | "huge_shared" : " 27 GB", | 20260505 04:58:19.467 | INFO | "malloc" : " 14 GB", | 20260505 04:58:19.467 | INFO | "pinned" : " 29 GB" | 20260505 04:58:19.467 | INFO | } | 20260505 04:58:19.467 | INFO | Perf report for the last 5.13897 seconds: | 20260505 04:58:19.467 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:58:19.467 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:19.467 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:19.467 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:19.467 | INFO | ANALYSIS : 57.2%, 3.0, 89.3ms, 0 | 20260505 04:58:19.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:19.467 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:19.467 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 04:58:19.467 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:19.467 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:19.468 | INFO | Uploaded 36GiB to GPU in 1.60077s (22.4892GiB/s) | 20260505 04:58:19.468 | INFO | Downloaded 34GiB to GPU in 1.51758s (22.4041GiB/s) | 20260505 04:58:24.607 | INFO | Analyzing chunk frames = [519168,519680) | 20260505 04:58:24.607 | INFO | MemInfo: { | 20260505 04:58:24.607 | INFO | "gpu" : " 17 GB", | 20260505 04:58:24.607 | INFO | "huge" : "", | 20260505 04:58:24.607 | INFO | "huge_shared" : " 27 GB", | 20260505 04:58:24.607 | INFO | "malloc" : " 14 GB", | 20260505 04:58:24.607 | INFO | "pinned" : " 29 GB" | 20260505 04:58:24.607 | INFO | } | 20260505 04:58:24.607 | INFO | Perf report for the last 5.13929 seconds: | 20260505 04:58:24.607 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:58:24.607 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:24.607 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:24.607 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:24.607 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260505 04:58:24.607 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:24.607 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:24.607 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 04:58:24.607 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:24.607 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:24.607 | INFO | Uploaded 36GiB to GPU in 1.60172s (22.4759GiB/s) | 20260505 04:58:24.607 | INFO | Downloaded 34GiB to GPU in 1.51897s (22.3835GiB/s) | 20260505 04:58:29.746 | INFO | Analyzing chunk frames = [519680,520192) | 20260505 04:58:29.746 | INFO | MemInfo: { | 20260505 04:58:29.746 | INFO | "gpu" : " 17 GB", | 20260505 04:58:29.746 | INFO | "huge" : "", | 20260505 04:58:29.746 | INFO | "huge_shared" : " 27 GB", | 20260505 04:58:29.746 | INFO | "malloc" : " 14 GB", | 20260505 04:58:29.746 | INFO | "pinned" : " 29 GB" | 20260505 04:58:29.746 | INFO | } | 20260505 04:58:29.746 | INFO | Perf report for the last 5.13931 seconds: | 20260505 04:58:29.746 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:58:29.746 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:29.746 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:29.746 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:29.746 | INFO | ANALYSIS : 57.5%, 3.0, 89.8ms, 0 | 20260505 04:58:29.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:29.746 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:29.746 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 04:58:29.746 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:29.746 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:29.746 | INFO | Uploaded 36GiB to GPU in 1.60134s (22.4811GiB/s) | 20260505 04:58:29.746 | INFO | Downloaded 34GiB to GPU in 1.51838s (22.3923GiB/s) | 20260505 04:58:29.843 | INFO | Starting a round of Baz IO | 20260505 04:58:32.791 | INFO | Populating RTMetrics report | 20260505 04:58:32.791 | INFO | RTMetrics report populated | 20260505 04:58:32.791 | INFO | Opening tmp RTMetrics json file | 20260505 04:58:32.795 | INFO | Serializing RTMetrics to json | 20260505 04:58:32.797 | INFO | Serialization done, swapping tmp file now... | 20260505 04:58:32.797 | INFO | RTMetrics done | 20260505 04:58:32.889 | INFO | Finished a round of Baz IO: Wrote 2.121 GiB in 3.046 seconds | 20260505 04:58:32.889 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.66% / 0.11% / 57.23% | 20260505 04:58:32.889 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.99755% / 0.000198682% / 3.57628e-05% | 20260505 04:58:32.889 | INFO | Baz IO overall: | 20260505 04:58:32.889 | INFO | BytesWritten : 225999097983 | 20260505 04:58:32.889 | INFO | NumEvents : 105503027319 | 20260505 04:58:34.885 | INFO | Analyzing chunk frames = [520192,520704) | 20260505 04:58:34.885 | INFO | MemInfo: { | 20260505 04:58:34.885 | INFO | "gpu" : " 17 GB", | 20260505 04:58:34.885 | INFO | "huge" : "", | 20260505 04:58:34.885 | INFO | "huge_shared" : " 27 GB", | 20260505 04:58:34.885 | INFO | "malloc" : " 14 GB", | 20260505 04:58:34.885 | INFO | "pinned" : " 29 GB" | 20260505 04:58:34.885 | INFO | } | 20260505 04:58:34.886 | INFO | Perf report for the last 5.13949 seconds: | 20260505 04:58:34.886 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:58:34.886 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:34.886 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:34.886 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:34.886 | INFO | ANALYSIS : 58.1%, 3.0, 90.3ms, 0 | 20260505 04:58:34.886 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:34.886 | INFO | RT_METRICS : 45.5%, 1.0, 23.8ms, 0 | 20260505 04:58:34.886 | INFO | PRE_HQ : 57.5%, 2.2, 64.2ms, 0 | 20260505 04:58:34.886 | INFO | BAZWRITER : 58.8%, 4.1, 125.1ms, 0 | 20260505 04:58:34.886 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 04:58:34.886 | INFO | Uploaded 36GiB to GPU in 1.6392s (21.9619GiB/s) | 20260505 04:58:34.886 | INFO | Downloaded 38GiB to GPU in 1.70231s (22.3226GiB/s) | 20260505 04:58:40.025 | INFO | Analyzing chunk frames = [520704,521216) | 20260505 04:58:40.025 | INFO | MemInfo: { | 20260505 04:58:40.025 | INFO | "gpu" : " 17 GB", | 20260505 04:58:40.025 | INFO | "huge" : "", | 20260505 04:58:40.025 | INFO | "huge_shared" : " 27 GB", | 20260505 04:58:40.025 | INFO | "malloc" : " 14 GB", | 20260505 04:58:40.025 | INFO | "pinned" : " 29 GB" | 20260505 04:58:40.025 | INFO | } | 20260505 04:58:40.025 | INFO | Perf report for the last 5.13913 seconds: | 20260505 04:58:40.025 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:58:40.025 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:40.025 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:40.025 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:40.025 | INFO | ANALYSIS : 58.0%, 3.0, 90.4ms, 0 | 20260505 04:58:40.025 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:40.025 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:40.025 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 04:58:40.025 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:40.025 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:40.025 | INFO | Uploaded 36GiB to GPU in 1.59937s (22.5089GiB/s) | 20260505 04:58:40.025 | INFO | Downloaded 34GiB to GPU in 1.51608s (22.4263GiB/s) | 20260505 04:58:45.164 | INFO | Analyzing chunk frames = [521216,521728) | 20260505 04:58:45.164 | INFO | MemInfo: { | 20260505 04:58:45.164 | INFO | "gpu" : " 17 GB", | 20260505 04:58:45.164 | INFO | "huge" : "", | 20260505 04:58:45.164 | INFO | "huge_shared" : " 27 GB", | 20260505 04:58:45.164 | INFO | "malloc" : " 14 GB", | 20260505 04:58:45.164 | INFO | "pinned" : " 29 GB" | 20260505 04:58:45.164 | INFO | } | 20260505 04:58:45.164 | INFO | Perf report for the last 5.13945 seconds: | 20260505 04:58:45.164 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:58:45.164 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:45.164 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:45.164 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:45.164 | INFO | ANALYSIS : 57.3%, 3.0, 89.1ms, 0 | 20260505 04:58:45.164 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:45.164 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:45.164 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 04:58:45.164 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:45.164 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:45.165 | INFO | Uploaded 36GiB to GPU in 1.60383s (22.4463GiB/s) | 20260505 04:58:45.165 | INFO | Downloaded 34GiB to GPU in 1.52052s (22.3607GiB/s) | 20260505 04:58:50.303 | INFO | Analyzing chunk frames = [521728,522240) | 20260505 04:58:50.303 | INFO | MemInfo: { | 20260505 04:58:50.303 | INFO | "gpu" : " 17 GB", | 20260505 04:58:50.303 | INFO | "huge" : "", | 20260505 04:58:50.303 | INFO | "huge_shared" : " 27 GB", | 20260505 04:58:50.303 | INFO | "malloc" : " 14 GB", | 20260505 04:58:50.303 | INFO | "pinned" : " 29 GB" | 20260505 04:58:50.303 | INFO | } | 20260505 04:58:50.303 | INFO | Perf report for the last 5.13927 seconds: | 20260505 04:58:50.303 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:58:50.303 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:50.303 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:50.303 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:50.303 | INFO | ANALYSIS : 57.0%, 3.0, 88.3ms, 0 | 20260505 04:58:50.304 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:50.304 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:50.304 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 04:58:50.304 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:50.304 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:50.304 | INFO | Uploaded 36GiB to GPU in 1.60309s (22.4566GiB/s) | 20260505 04:58:50.304 | INFO | Downloaded 34GiB to GPU in 1.51917s (22.3807GiB/s) | 20260505 04:58:55.443 | INFO | Analyzing chunk frames = [522240,522752) | 20260505 04:58:55.443 | INFO | MemInfo: { | 20260505 04:58:55.443 | INFO | "gpu" : " 17 GB", | 20260505 04:58:55.443 | INFO | "huge" : "", | 20260505 04:58:55.443 | INFO | "huge_shared" : " 27 GB", | 20260505 04:58:55.443 | INFO | "malloc" : " 14 GB", | 20260505 04:58:55.443 | INFO | "pinned" : " 29 GB" | 20260505 04:58:55.443 | INFO | } | 20260505 04:58:55.443 | INFO | Perf report for the last 5.13931 seconds: | 20260505 04:58:55.443 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:58:55.443 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:55.443 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:55.443 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:55.443 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260505 04:58:55.443 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:55.443 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:55.443 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 04:58:55.443 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:55.443 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:58:55.443 | INFO | Uploaded 36GiB to GPU in 1.59994s (22.5009GiB/s) | 20260505 04:58:55.443 | INFO | Downloaded 34GiB to GPU in 1.51727s (22.4086GiB/s) | 20260505 04:59:00.582 | INFO | Analyzing chunk frames = [522752,523264) | 20260505 04:59:00.582 | INFO | MemInfo: { | 20260505 04:59:00.582 | INFO | "gpu" : " 17 GB", | 20260505 04:59:00.582 | INFO | "huge" : "", | 20260505 04:59:00.582 | INFO | "huge_shared" : " 27 GB", | 20260505 04:59:00.582 | INFO | "malloc" : " 14 GB", | 20260505 04:59:00.582 | INFO | "pinned" : " 29 GB" | 20260505 04:59:00.582 | INFO | } | 20260505 04:59:00.582 | INFO | Perf report for the last 5.1394 seconds: | 20260505 04:59:00.582 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:59:00.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:00.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:00.582 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:00.582 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260505 04:59:00.582 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:00.582 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:00.582 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 04:59:00.582 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:00.582 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:00.583 | INFO | Uploaded 36GiB to GPU in 1.60147s (22.4793GiB/s) | 20260505 04:59:00.583 | INFO | Downloaded 34GiB to GPU in 1.5181s (22.3965GiB/s) | 20260505 04:59:05.721 | INFO | Analyzing chunk frames = [523264,523776) | 20260505 04:59:05.721 | INFO | MemInfo: { | 20260505 04:59:05.721 | INFO | "gpu" : " 17 GB", | 20260505 04:59:05.721 | INFO | "huge" : "", | 20260505 04:59:05.721 | INFO | "huge_shared" : " 27 GB", | 20260505 04:59:05.721 | INFO | "malloc" : " 14 GB", | 20260505 04:59:05.721 | INFO | "pinned" : " 29 GB" | 20260505 04:59:05.721 | INFO | } | 20260505 04:59:05.721 | INFO | Perf report for the last 5.13926 seconds: | 20260505 04:59:05.721 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:59:05.721 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:05.721 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:05.721 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:05.722 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260505 04:59:05.722 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:05.722 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:05.722 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260505 04:59:05.722 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:05.722 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:05.722 | INFO | Uploaded 36GiB to GPU in 1.60136s (22.4809GiB/s) | 20260505 04:59:05.722 | INFO | Downloaded 34GiB to GPU in 1.51918s (22.3805GiB/s) | 20260505 04:59:10.861 | INFO | Analyzing chunk frames = [523776,524288) | 20260505 04:59:10.861 | INFO | MemInfo: { | 20260505 04:59:10.861 | INFO | "gpu" : " 17 GB", | 20260505 04:59:10.861 | INFO | "huge" : "", | 20260505 04:59:10.861 | INFO | "huge_shared" : " 27 GB", | 20260505 04:59:10.861 | INFO | "malloc" : " 14 GB", | 20260505 04:59:10.861 | INFO | "pinned" : " 29 GB" | 20260505 04:59:10.861 | INFO | } | 20260505 04:59:10.861 | INFO | Perf report for the last 5.1394 seconds: | 20260505 04:59:10.861 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:59:10.861 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:10.861 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:10.861 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:10.861 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260505 04:59:10.861 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:10.861 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:10.861 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 04:59:10.861 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:10.861 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:10.861 | INFO | Uploaded 36GiB to GPU in 1.60189s (22.4735GiB/s) | 20260505 04:59:10.861 | INFO | Downloaded 34GiB to GPU in 1.51954s (22.3752GiB/s) | 20260505 04:59:10.861 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 04:59:10.861 | INFO | Total mem usage: 15.1 GiB | 20260505 04:59:10.861 | INFO | All Time High Water Marks sum: 15.7 GiB (Savings: 4.09%) | 20260505 04:59:10.861 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -4.3%) | 20260505 04:59:10.861 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 04:59:10.861 | INFO | High Water Breakdown: | 20260505 04:59:10.861 | INFO | 59.6% : 8.6 GiB ( 95.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 04:59:10.861 | INFO | 40.3% : 5.82 GiB ( 87.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 04:59:10.861 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 04:59:10.861 | INFO | ------------------------------------------ | 20260505 04:59:10.861 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:59:10.861 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 04:59:10.861 | INFO | Total mem usage: 26.5 GiB | 20260505 04:59:10.861 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 04:59:10.861 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 04:59:10.861 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 04:59:10.861 | INFO | High Water Breakdown: | 20260505 04:59:10.861 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:59:10.861 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:59:10.861 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:59:10.861 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:59:10.861 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:59:10.861 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:59:10.861 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 04:59:10.861 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 04:59:10.861 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:59:10.861 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 04:59:10.861 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 04:59:10.861 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 04:59:10.861 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 04:59:10.861 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 04:59:10.861 | INFO | ------------------------------------------ | 20260505 04:59:10.861 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:59:10.861 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 04:59:10.861 | INFO | Total mem usage: 25.6 GiB | 20260505 04:59:10.861 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 04:59:10.861 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 04:59:10.861 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 04:59:10.861 | INFO | High Water Breakdown: | 20260505 04:59:10.861 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 04:59:10.861 | INFO | ------------------------------------------ | 20260505 04:59:10.861 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:59:10.861 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 04:59:10.861 | INFO | Total mem usage: 15.7 GiB | 20260505 04:59:10.861 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 04:59:10.861 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 04:59:10.861 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260505 04:59:10.861 | INFO | High Water Breakdown: | 20260505 04:59:10.861 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 04:59:10.861 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 04:59:10.861 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 04:59:10.861 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 04:59:10.861 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 04:59:10.861 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 04:59:10.861 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 04:59:10.861 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 04:59:10.861 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 04:59:10.861 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 04:59:10.861 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 04:59:10.861 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 04:59:10.861 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 04:59:10.861 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 04:59:10.861 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 04:59:10.861 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 04:59:10.861 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 04:59:10.861 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 04:59:10.861 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 04:59:10.861 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 04:59:10.861 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 04:59:10.861 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 04:59:10.861 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 04:59:10.861 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 04:59:10.861 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 04:59:10.861 | INFO | ------------------------------------------ | 20260505 04:59:10.861 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 04:59:10.861 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 04:59:10.861 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:59:10.861 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:59:10.861 | INFO | PerfReport: Full Execution -- 3.04e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260505 04:59:10.862 | INFO | PerfReport: 0.28%, AggregateStats -- 8.64e-02 / -nan, 4.69e-03 / 0.00e+00 | 20260505 04:59:10.862 | INFO | PerfReport: 10.84%, Baseline -- 3.30e+00 / -nan, 1.67e-02 / 0.00e+00 | 20260505 04:59:10.862 | INFO | PerfReport: 3.04%, Histogram -- 9.24e-01 / -nan, 5.20e-02 / 0.00e+00 | 20260505 04:59:10.862 | INFO | PerfReport: 16.03%, DME -- 3.90e+01 / -nan, 3.22e+00 / 0.00e+00 | 20260505 04:59:10.862 | INFO | PerfReport: 56.62%, FrameLabeling -- 1.72e+01 / -nan, 4.50e-02 / 0.00e+00 | 20260505 04:59:10.862 | INFO | PerfReport: 7.60%, PulseAccumulating -- 2.31e+00 / -nan, 1.53e-01 / 0.00e+00 | 20260505 04:59:10.862 | INFO | PerfReport: 3.94%, Metrics -- 1.20e+00 / -nan, 1.98e-01 / 0.00e+00 | 20260505 04:59:10.862 | INFO | PerfReport: 1.63%, PulseSerializing -- 4.95e-01 / -nan, 7.98e-02 / 0.00e+00 | 20260505 04:59:10.862 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 04:59:10.862 | INFO | PerfReport: Gathered 1536 samples. | 20260505 04:59:10.862 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 04:59:10.862 | INFO | PerfReport: Full Execution -- 9.13e+01 / -nan, 1.90e+01 / 0.00e+00 | 20260505 04:59:10.862 | INFO | PerfReport: 18.33%, Upload -- 1.67e+01 / 0.00e+00, 3.66e-01 / 0.00e+00 | 20260505 04:59:10.862 | INFO | PerfReport: 17.59%, Download -- 1.61e+01 / 0.00e+00, 6.92e-01 / 0.00e+00 | 20260505 04:59:10.862 | INFO | PerfReport: 33.30%, Compute -- 3.04e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260505 04:59:10.862 | INFO | PerfReport: 0.55%, LockUpload -- 5.06e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260505 04:59:10.862 | INFO | PerfReport: 0.00%, LockDownload -- 3.94e-04 / 0.00e+00, 1.26e-04 / 0.00e+00 | 20260505 04:59:10.862 | INFO | PerfReport: 30.21%, LockCompute -- 2.76e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260505 04:59:10.862 | INFO | PerfReport: 0.02%, Misc -- 1.40e-02 / 0.00e+00, 2.02e-02 / 0.00e+00 | 20260505 04:59:10.958 | INFO | Starting a round of Baz IO | 20260505 04:59:13.909 | INFO | Populating RTMetrics report | 20260505 04:59:13.909 | INFO | RTMetrics report populated | 20260505 04:59:13.909 | INFO | Opening tmp RTMetrics json file | 20260505 04:59:13.910 | INFO | Serializing RTMetrics to json | 20260505 04:59:13.911 | INFO | Serialization done, swapping tmp file now... | 20260505 04:59:13.912 | INFO | RTMetrics done | 20260505 04:59:14.009 | INFO | Finished a round of Baz IO: Wrote 2.123 GiB in 3.05 seconds | 20260505 04:59:14.009 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.55% / 0.11% / 57.34% | 20260505 04:59:14.009 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.94361% / 0.000226498% / 3.97364e-05% | 20260505 04:59:14.009 | INFO | Baz IO overall: | 20260505 04:59:14.009 | INFO | BytesWritten : 228278662326 | 20260505 04:59:14.009 | INFO | NumEvents : 106585607477 | 20260505 04:59:16.000 | INFO | Analyzing chunk frames = [524288,524800) | 20260505 04:59:16.000 | INFO | MemInfo: { | 20260505 04:59:16.000 | INFO | "gpu" : " 17 GB", | 20260505 04:59:16.000 | INFO | "huge" : "", | 20260505 04:59:16.000 | INFO | "huge_shared" : " 27 GB", | 20260505 04:59:16.000 | INFO | "malloc" : " 14 GB", | 20260505 04:59:16.000 | INFO | "pinned" : " 29 GB" | 20260505 04:59:16.000 | INFO | } | 20260505 04:59:16.000 | INFO | Perf report for the last 5.13932 seconds: | 20260505 04:59:16.000 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:59:16.000 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:16.000 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:16.000 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:16.000 | INFO | ANALYSIS : 58.2%, 3.0, 90.4ms, 0 | 20260505 04:59:16.000 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:16.000 | INFO | RT_METRICS : 45.5%, 1.0, 23.9ms, 0 | 20260505 04:59:16.000 | INFO | PRE_HQ : 57.6%, 2.2, 64.7ms, 0 | 20260505 04:59:16.000 | INFO | BAZWRITER : 58.9%, 4.1, 124.6ms, 0 | 20260505 04:59:16.000 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:59:16.000 | INFO | Uploaded 36GiB to GPU in 1.63976s (21.9545GiB/s) | 20260505 04:59:16.001 | INFO | Downloaded 38GiB to GPU in 1.70421s (22.2977GiB/s) | 20260505 04:59:21.139 | INFO | Analyzing chunk frames = [524800,525312) | 20260505 04:59:21.139 | INFO | MemInfo: { | 20260505 04:59:21.139 | INFO | "gpu" : " 17 GB", | 20260505 04:59:21.139 | INFO | "huge" : "", | 20260505 04:59:21.139 | INFO | "huge_shared" : " 27 GB", | 20260505 04:59:21.139 | INFO | "malloc" : " 14 GB", | 20260505 04:59:21.139 | INFO | "pinned" : " 29 GB" | 20260505 04:59:21.139 | INFO | } | 20260505 04:59:21.140 | INFO | Perf report for the last 5.13931 seconds: | 20260505 04:59:21.140 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:59:21.140 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:21.140 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:21.140 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:21.140 | INFO | ANALYSIS : 57.9%, 3.0, 90.3ms, 0 | 20260505 04:59:21.140 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:21.140 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:21.140 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260505 04:59:21.140 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:21.140 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:21.140 | INFO | Uploaded 36GiB to GPU in 1.59992s (22.5012GiB/s) | 20260505 04:59:21.140 | INFO | Downloaded 34GiB to GPU in 1.51621s (22.4244GiB/s) | 20260505 04:59:26.279 | INFO | Analyzing chunk frames = [525312,525824) | 20260505 04:59:26.279 | INFO | MemInfo: { | 20260505 04:59:26.279 | INFO | "gpu" : " 17 GB", | 20260505 04:59:26.279 | INFO | "huge" : "", | 20260505 04:59:26.279 | INFO | "huge_shared" : " 27 GB", | 20260505 04:59:26.279 | INFO | "malloc" : " 14 GB", | 20260505 04:59:26.279 | INFO | "pinned" : " 29 GB" | 20260505 04:59:26.279 | INFO | } | 20260505 04:59:26.279 | INFO | Perf report for the last 5.13929 seconds: | 20260505 04:59:26.279 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:59:26.279 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:26.279 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:26.279 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:26.279 | INFO | ANALYSIS : 57.3%, 3.0, 89.1ms, 0 | 20260505 04:59:26.279 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:26.279 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:26.279 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 04:59:26.279 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:26.279 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:26.279 | INFO | Uploaded 36GiB to GPU in 1.60332s (22.4534GiB/s) | 20260505 04:59:26.279 | INFO | Downloaded 34GiB to GPU in 1.52104s (22.3531GiB/s) | 20260505 04:59:31.418 | INFO | Analyzing chunk frames = [525824,526336) | 20260505 04:59:31.418 | INFO | MemInfo: { | 20260505 04:59:31.418 | INFO | "gpu" : " 17 GB", | 20260505 04:59:31.418 | INFO | "huge" : "", | 20260505 04:59:31.418 | INFO | "huge_shared" : " 27 GB", | 20260505 04:59:31.418 | INFO | "malloc" : " 14 GB", | 20260505 04:59:31.418 | INFO | "pinned" : " 29 GB" | 20260505 04:59:31.418 | INFO | } | 20260505 04:59:31.418 | INFO | Perf report for the last 5.13934 seconds: | 20260505 04:59:31.418 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:59:31.418 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:31.418 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:31.418 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:31.418 | INFO | ANALYSIS : 57.0%, 3.0, 88.4ms, 0 | 20260505 04:59:31.418 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:31.418 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:31.418 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260505 04:59:31.418 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:31.418 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:31.419 | INFO | Uploaded 36GiB to GPU in 1.60343s (22.4518GiB/s) | 20260505 04:59:31.419 | INFO | Downloaded 34GiB to GPU in 1.52011s (22.3668GiB/s) | 20260505 04:59:36.557 | INFO | Analyzing chunk frames = [526336,526848) | 20260505 04:59:36.557 | INFO | MemInfo: { | 20260505 04:59:36.557 | INFO | "gpu" : " 17 GB", | 20260505 04:59:36.557 | INFO | "huge" : "", | 20260505 04:59:36.557 | INFO | "huge_shared" : " 27 GB", | 20260505 04:59:36.557 | INFO | "malloc" : " 14 GB", | 20260505 04:59:36.557 | INFO | "pinned" : " 29 GB" | 20260505 04:59:36.557 | INFO | } | 20260505 04:59:36.558 | INFO | Perf report for the last 5.13934 seconds: | 20260505 04:59:36.558 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:59:36.558 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:36.558 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:36.558 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:36.558 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260505 04:59:36.558 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:36.558 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:36.558 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 04:59:36.558 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:36.558 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:36.558 | INFO | Uploaded 36GiB to GPU in 1.59967s (22.5046GiB/s) | 20260505 04:59:36.558 | INFO | Downloaded 34GiB to GPU in 1.51759s (22.4039GiB/s) | 20260505 04:59:41.697 | INFO | Analyzing chunk frames = [526848,527360) | 20260505 04:59:41.697 | INFO | MemInfo: { | 20260505 04:59:41.697 | INFO | "gpu" : " 17 GB", | 20260505 04:59:41.697 | INFO | "huge" : "", | 20260505 04:59:41.697 | INFO | "huge_shared" : " 27 GB", | 20260505 04:59:41.697 | INFO | "malloc" : " 14 GB", | 20260505 04:59:41.697 | INFO | "pinned" : " 29 GB" | 20260505 04:59:41.697 | INFO | } | 20260505 04:59:41.697 | INFO | Perf report for the last 5.13937 seconds: | 20260505 04:59:41.697 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:59:41.697 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:41.697 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:41.697 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:41.697 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260505 04:59:41.697 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:41.697 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:41.697 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 04:59:41.697 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:41.697 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:41.697 | INFO | Uploaded 36GiB to GPU in 1.60112s (22.4843GiB/s) | 20260505 04:59:41.697 | INFO | Downloaded 34GiB to GPU in 1.51857s (22.3895GiB/s) | 20260505 04:59:46.836 | INFO | Analyzing chunk frames = [527360,527872) | 20260505 04:59:46.836 | INFO | MemInfo: { | 20260505 04:59:46.836 | INFO | "gpu" : " 17 GB", | 20260505 04:59:46.836 | INFO | "huge" : "", | 20260505 04:59:46.836 | INFO | "huge_shared" : " 27 GB", | 20260505 04:59:46.836 | INFO | "malloc" : " 14 GB", | 20260505 04:59:46.836 | INFO | "pinned" : " 29 GB" | 20260505 04:59:46.836 | INFO | } | 20260505 04:59:46.836 | INFO | Perf report for the last 5.13932 seconds: | 20260505 04:59:46.836 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:59:46.836 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:46.836 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:46.836 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:46.836 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260505 04:59:46.836 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:46.836 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:46.836 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 04:59:46.836 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:46.836 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:46.837 | INFO | Uploaded 36GiB to GPU in 1.60191s (22.4731GiB/s) | 20260505 04:59:46.837 | INFO | Downloaded 34GiB to GPU in 1.51963s (22.3738GiB/s) | 20260505 04:59:51.975 | INFO | Analyzing chunk frames = [527872,528384) | 20260505 04:59:51.975 | INFO | MemInfo: { | 20260505 04:59:51.975 | INFO | "gpu" : " 17 GB", | 20260505 04:59:51.976 | INFO | "huge" : "", | 20260505 04:59:51.976 | INFO | "huge_shared" : " 27 GB", | 20260505 04:59:51.976 | INFO | "malloc" : " 14 GB", | 20260505 04:59:51.976 | INFO | "pinned" : " 29 GB" | 20260505 04:59:51.976 | INFO | } | 20260505 04:59:51.976 | INFO | Perf report for the last 5.13937 seconds: | 20260505 04:59:51.976 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:59:51.976 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:51.976 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:51.976 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:51.976 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260505 04:59:51.976 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:51.976 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:51.976 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 04:59:51.976 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:51.976 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:51.976 | INFO | Uploaded 36GiB to GPU in 1.60156s (22.4781GiB/s) | 20260505 04:59:51.976 | INFO | Downloaded 34GiB to GPU in 1.51881s (22.3859GiB/s) | 20260505 04:59:52.073 | INFO | Starting a round of Baz IO | 20260505 04:59:55.022 | INFO | Populating RTMetrics report | 20260505 04:59:55.022 | INFO | RTMetrics report populated | 20260505 04:59:55.022 | INFO | Opening tmp RTMetrics json file | 20260505 04:59:55.023 | INFO | Serializing RTMetrics to json | 20260505 04:59:55.024 | INFO | Serialization done, swapping tmp file now... | 20260505 04:59:55.024 | INFO | RTMetrics done | 20260505 04:59:55.123 | INFO | Finished a round of Baz IO: Wrote 2.129 GiB in 3.05 seconds | 20260505 04:59:55.123 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.45% / 0.11% / 57.45% | 20260505 04:59:55.123 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.91918% / 0.000226498% / 4.76837e-05% | 20260505 04:59:55.123 | INFO | Baz IO overall: | 20260505 04:59:55.123 | INFO | BytesWritten : 230564455763 | 20260505 04:59:55.123 | INFO | NumEvents : 107671564865 | 20260505 04:59:57.115 | INFO | Analyzing chunk frames = [528384,528896) | 20260505 04:59:57.115 | INFO | MemInfo: { | 20260505 04:59:57.115 | INFO | "gpu" : " 17 GB", | 20260505 04:59:57.115 | INFO | "huge" : "", | 20260505 04:59:57.115 | INFO | "huge_shared" : " 27 GB", | 20260505 04:59:57.115 | INFO | "malloc" : " 14 GB", | 20260505 04:59:57.115 | INFO | "pinned" : " 29 GB" | 20260505 04:59:57.115 | INFO | } | 20260505 04:59:57.115 | INFO | Perf report for the last 5.13935 seconds: | 20260505 04:59:57.115 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 04:59:57.115 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:57.115 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:57.115 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:57.115 | INFO | ANALYSIS : 58.2%, 3.0, 90.3ms, 0 | 20260505 04:59:57.115 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 04:59:57.115 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260505 04:59:57.115 | INFO | PRE_HQ : 57.5%, 2.2, 64.3ms, 0 | 20260505 04:59:57.115 | INFO | BAZWRITER : 58.9%, 4.2, 126.3ms, 0 | 20260505 04:59:57.115 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 04:59:57.115 | INFO | Uploaded 36GiB to GPU in 1.6415s (21.9312GiB/s) | 20260505 04:59:57.116 | INFO | Downloaded 38GiB to GPU in 1.70629s (22.2705GiB/s) | 20260505 05:00:02.254 | INFO | Analyzing chunk frames = [528896,529408) | 20260505 05:00:02.254 | INFO | MemInfo: { | 20260505 05:00:02.254 | INFO | "gpu" : " 17 GB", | 20260505 05:00:02.254 | INFO | "huge" : "", | 20260505 05:00:02.254 | INFO | "huge_shared" : " 27 GB", | 20260505 05:00:02.254 | INFO | "malloc" : " 14 GB", | 20260505 05:00:02.254 | INFO | "pinned" : " 29 GB" | 20260505 05:00:02.254 | INFO | } | 20260505 05:00:02.254 | INFO | Perf report for the last 5.13934 seconds: | 20260505 05:00:02.254 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:00:02.254 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:02.254 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:02.254 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:02.254 | INFO | ANALYSIS : 57.9%, 3.0, 90.3ms, 0 | 20260505 05:00:02.254 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:02.254 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:02.254 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 05:00:02.254 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:02.254 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:02.255 | INFO | Uploaded 36GiB to GPU in 1.6006s (22.4916GiB/s) | 20260505 05:00:02.255 | INFO | Downloaded 34GiB to GPU in 1.51716s (22.4103GiB/s) | 20260505 05:00:07.393 | INFO | Analyzing chunk frames = [529408,529920) | 20260505 05:00:07.393 | INFO | MemInfo: { | 20260505 05:00:07.393 | INFO | "gpu" : " 17 GB", | 20260505 05:00:07.393 | INFO | "huge" : "", | 20260505 05:00:07.394 | INFO | "huge_shared" : " 27 GB", | 20260505 05:00:07.394 | INFO | "malloc" : " 14 GB", | 20260505 05:00:07.394 | INFO | "pinned" : " 29 GB" | 20260505 05:00:07.394 | INFO | } | 20260505 05:00:07.394 | INFO | Perf report for the last 5.13928 seconds: | 20260505 05:00:07.394 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:00:07.394 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:07.394 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:07.394 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:07.394 | INFO | ANALYSIS : 57.3%, 3.0, 89.1ms, 0 | 20260505 05:00:07.394 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:07.394 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:07.394 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260505 05:00:07.394 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:07.394 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:07.394 | INFO | Uploaded 36GiB to GPU in 1.60269s (22.4623GiB/s) | 20260505 05:00:07.394 | INFO | Downloaded 34GiB to GPU in 1.52006s (22.3676GiB/s) | 20260505 05:00:12.533 | INFO | Analyzing chunk frames = [529920,530432) | 20260505 05:00:12.533 | INFO | MemInfo: { | 20260505 05:00:12.533 | INFO | "gpu" : " 17 GB", | 20260505 05:00:12.533 | INFO | "huge" : "", | 20260505 05:00:12.533 | INFO | "huge_shared" : " 27 GB", | 20260505 05:00:12.533 | INFO | "malloc" : " 14 GB", | 20260505 05:00:12.533 | INFO | "pinned" : " 29 GB" | 20260505 05:00:12.533 | INFO | } | 20260505 05:00:12.533 | INFO | Perf report for the last 5.13928 seconds: | 20260505 05:00:12.533 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:00:12.533 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:12.533 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:12.533 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:12.533 | INFO | ANALYSIS : 57.0%, 3.0, 88.3ms, 0 | 20260505 05:00:12.533 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:12.533 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:12.533 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260505 05:00:12.533 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:12.533 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:12.533 | INFO | Uploaded 36GiB to GPU in 1.60283s (22.4603GiB/s) | 20260505 05:00:12.533 | INFO | Downloaded 34GiB to GPU in 1.51984s (22.3707GiB/s) | 20260505 05:00:17.672 | INFO | Analyzing chunk frames = [530432,530944) | 20260505 05:00:17.672 | INFO | MemInfo: { | 20260505 05:00:17.672 | INFO | "gpu" : " 17 GB", | 20260505 05:00:17.672 | INFO | "huge" : "", | 20260505 05:00:17.672 | INFO | "huge_shared" : " 27 GB", | 20260505 05:00:17.672 | INFO | "malloc" : " 14 GB", | 20260505 05:00:17.672 | INFO | "pinned" : " 29 GB" | 20260505 05:00:17.672 | INFO | } | 20260505 05:00:17.672 | INFO | Perf report for the last 5.13938 seconds: | 20260505 05:00:17.672 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:00:17.672 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:17.672 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:17.672 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:17.672 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260505 05:00:17.672 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:17.672 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:17.672 | INFO | PRE_HQ : 31.7%, 1.0, 16.4ms, 0 | 20260505 05:00:17.672 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:17.672 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:17.673 | INFO | Uploaded 36GiB to GPU in 1.6016s (22.4775GiB/s) | 20260505 05:00:17.673 | INFO | Downloaded 34GiB to GPU in 1.51831s (22.3934GiB/s) | 20260505 05:00:22.812 | INFO | Analyzing chunk frames = [530944,531456) | 20260505 05:00:22.812 | INFO | MemInfo: { | 20260505 05:00:22.812 | INFO | "gpu" : " 17 GB", | 20260505 05:00:22.812 | INFO | "huge" : "", | 20260505 05:00:22.812 | INFO | "huge_shared" : " 27 GB", | 20260505 05:00:22.812 | INFO | "malloc" : " 14 GB", | 20260505 05:00:22.812 | INFO | "pinned" : " 29 GB" | 20260505 05:00:22.812 | INFO | } | 20260505 05:00:22.812 | INFO | Perf report for the last 5.13928 seconds: | 20260505 05:00:22.812 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:00:22.812 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:22.812 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:22.812 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:22.812 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260505 05:00:22.812 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:22.812 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:22.812 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 05:00:22.812 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:22.812 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:22.812 | INFO | Uploaded 36GiB to GPU in 1.60135s (22.481GiB/s) | 20260505 05:00:22.812 | INFO | Downloaded 34GiB to GPU in 1.51845s (22.3912GiB/s) | 20260505 05:00:27.951 | INFO | Analyzing chunk frames = [531456,531968) | 20260505 05:00:27.951 | INFO | MemInfo: { | 20260505 05:00:27.951 | INFO | "gpu" : " 17 GB", | 20260505 05:00:27.951 | INFO | "huge" : "", | 20260505 05:00:27.951 | INFO | "huge_shared" : " 27 GB", | 20260505 05:00:27.951 | INFO | "malloc" : " 14 GB", | 20260505 05:00:27.951 | INFO | "pinned" : " 29 GB" | 20260505 05:00:27.951 | INFO | } | 20260505 05:00:27.951 | INFO | Perf report for the last 5.13947 seconds: | 20260505 05:00:27.951 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:00:27.951 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:27.951 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:27.951 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:27.951 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260505 05:00:27.951 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:27.951 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:27.951 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 05:00:27.951 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:27.951 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:27.951 | INFO | Uploaded 36GiB to GPU in 1.60163s (22.4771GiB/s) | 20260505 05:00:27.952 | INFO | Downloaded 34GiB to GPU in 1.51883s (22.3857GiB/s) | 20260505 05:00:33.090 | INFO | Analyzing chunk frames = [531968,532480) | 20260505 05:00:33.090 | INFO | MemInfo: { | 20260505 05:00:33.090 | INFO | "gpu" : " 17 GB", | 20260505 05:00:33.090 | INFO | "huge" : "", | 20260505 05:00:33.090 | INFO | "huge_shared" : " 27 GB", | 20260505 05:00:33.090 | INFO | "malloc" : " 14 GB", | 20260505 05:00:33.090 | INFO | "pinned" : " 29 GB" | 20260505 05:00:33.090 | INFO | } | 20260505 05:00:33.090 | INFO | Perf report for the last 5.1393 seconds: | 20260505 05:00:33.090 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:00:33.090 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:33.090 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:33.090 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:33.090 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260505 05:00:33.090 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:33.090 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:33.090 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260505 05:00:33.091 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:33.091 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:33.091 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 05:00:33.091 | INFO | Total mem usage: 15.1 GiB | 20260505 05:00:33.091 | INFO | All Time High Water Marks sum: 15.7 GiB (Savings: 4.09%) | 20260505 05:00:33.091 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -4.3%) | 20260505 05:00:33.091 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 05:00:33.091 | INFO | High Water Breakdown: | 20260505 05:00:33.091 | INFO | 59.6% : 8.6 GiB ( 95.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 05:00:33.091 | INFO | Uploaded 36GiB to GPU in 1.60159s (22.4777GiB/s) | 20260505 05:00:33.091 | INFO | 40.3% : 5.82 GiB ( 87.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 05:00:33.091 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 05:00:33.091 | INFO | ------------------------------------------ | 20260505 05:00:33.091 | INFO | Downloaded 34GiB to GPU in 1.51947s (22.3762GiB/s) | 20260505 05:00:33.091 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:00:33.091 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 05:00:33.091 | INFO | Total mem usage: 26.5 GiB | 20260505 05:00:33.091 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 05:00:33.091 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 05:00:33.091 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 05:00:33.091 | INFO | High Water Breakdown: | 20260505 05:00:33.091 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:00:33.091 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:00:33.091 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:00:33.091 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:00:33.091 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:00:33.091 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:00:33.091 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 05:00:33.091 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 05:00:33.091 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:00:33.091 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 05:00:33.091 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 05:00:33.091 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 05:00:33.091 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 05:00:33.091 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 05:00:33.091 | INFO | ------------------------------------------ | 20260505 05:00:33.091 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:00:33.091 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 05:00:33.091 | INFO | Total mem usage: 25.6 GiB | 20260505 05:00:33.091 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 05:00:33.091 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 05:00:33.091 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 05:00:33.091 | INFO | High Water Breakdown: | 20260505 05:00:33.091 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 05:00:33.091 | INFO | ------------------------------------------ | 20260505 05:00:33.091 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:00:33.091 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 05:00:33.092 | INFO | Total mem usage: 15.7 GiB | 20260505 05:00:33.092 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 05:00:33.092 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 05:00:33.092 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260505 05:00:33.092 | INFO | High Water Breakdown: | 20260505 05:00:33.092 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 05:00:33.092 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 05:00:33.092 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 05:00:33.092 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 05:00:33.092 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 05:00:33.092 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 05:00:33.092 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 05:00:33.092 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 05:00:33.092 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 05:00:33.092 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:00:33.092 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 05:00:33.092 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:00:33.092 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 05:00:33.092 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 05:00:33.092 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 05:00:33.092 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 05:00:33.092 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:00:33.092 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 05:00:33.092 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:00:33.092 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:00:33.092 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:00:33.092 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 05:00:33.092 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 05:00:33.092 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:00:33.092 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 05:00:33.092 | INFO | ------------------------------------------ | 20260505 05:00:33.092 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:00:33.092 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 05:00:33.092 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:00:33.092 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:00:33.092 | INFO | PerfReport: Full Execution -- 3.04e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260505 05:00:33.092 | INFO | PerfReport: 0.28%, AggregateStats -- 8.62e-02 / -nan, 5.04e-03 / 0.00e+00 | 20260505 05:00:33.092 | INFO | PerfReport: 10.85%, Baseline -- 3.30e+00 / -nan, 1.88e-02 / 0.00e+00 | 20260505 05:00:33.092 | INFO | PerfReport: 3.04%, Histogram -- 9.24e-01 / -nan, 5.17e-02 / 0.00e+00 | 20260505 05:00:33.092 | INFO | PerfReport: 16.03%, DME -- 3.90e+01 / -nan, 3.20e+00 / 0.00e+00 | 20260505 05:00:33.092 | INFO | PerfReport: 56.63%, FrameLabeling -- 1.72e+01 / -nan, 4.54e-02 / 0.00e+00 | 20260505 05:00:33.092 | INFO | PerfReport: 7.60%, PulseAccumulating -- 2.31e+00 / -nan, 1.53e-01 / 0.00e+00 | 20260505 05:00:33.092 | INFO | PerfReport: 3.94%, Metrics -- 1.20e+00 / -nan, 1.99e-01 / 0.00e+00 | 20260505 05:00:33.092 | INFO | PerfReport: 1.62%, PulseSerializing -- 4.94e-01 / -nan, 7.97e-02 / 0.00e+00 | 20260505 05:00:33.092 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 05:00:33.092 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:00:33.092 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:00:33.092 | INFO | PerfReport: Full Execution -- 9.13e+01 / -nan, 1.89e+01 / 0.00e+00 | 20260505 05:00:33.092 | INFO | PerfReport: 18.34%, Upload -- 1.67e+01 / 0.00e+00, 3.79e-01 / 0.00e+00 | 20260505 05:00:33.092 | INFO | PerfReport: 17.60%, Download -- 1.61e+01 / 0.00e+00, 7.09e-01 / 0.00e+00 | 20260505 05:00:33.092 | INFO | PerfReport: 33.31%, Compute -- 3.04e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260505 05:00:33.092 | INFO | PerfReport: 0.55%, LockUpload -- 5.04e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260505 05:00:33.092 | INFO | PerfReport: 0.00%, LockDownload -- 3.94e-04 / 0.00e+00, 9.80e-05 / 0.00e+00 | 20260505 05:00:33.092 | INFO | PerfReport: 30.19%, LockCompute -- 2.76e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260505 05:00:33.092 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.84e-02 / 0.00e+00 | 20260505 05:00:33.188 | INFO | Starting a round of Baz IO | 20260505 05:00:36.135 | INFO | Populating RTMetrics report | 20260505 05:00:36.136 | INFO | RTMetrics report populated | 20260505 05:00:36.136 | INFO | Opening tmp RTMetrics json file | 20260505 05:00:36.137 | INFO | Serializing RTMetrics to json | 20260505 05:00:36.139 | INFO | Serialization done, swapping tmp file now... | 20260505 05:00:36.139 | INFO | RTMetrics done | 20260505 05:00:36.235 | INFO | Finished a round of Baz IO: Wrote 2.131 GiB in 3.047 seconds | 20260505 05:00:36.235 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.35% / 0.10% / 57.55% | 20260505 05:00:36.235 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.86207% / 0.000222524% / 3.57628e-05% | 20260505 05:00:36.236 | INFO | Baz IO overall: | 20260505 05:00:36.236 | INFO | BytesWritten : 232852538714 | 20260505 05:00:36.236 | INFO | NumEvents : 108757955360 | 20260505 05:00:38.229 | INFO | Analyzing chunk frames = [532480,532992) | 20260505 05:00:38.229 | INFO | MemInfo: { | 20260505 05:00:38.229 | INFO | "gpu" : " 17 GB", | 20260505 05:00:38.229 | INFO | "huge" : "", | 20260505 05:00:38.229 | INFO | "huge_shared" : " 27 GB", | 20260505 05:00:38.229 | INFO | "malloc" : " 14 GB", | 20260505 05:00:38.229 | INFO | "pinned" : " 29 GB" | 20260505 05:00:38.229 | INFO | } | 20260505 05:00:38.230 | INFO | Perf report for the last 5.13912 seconds: | 20260505 05:00:38.230 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:00:38.230 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:38.230 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:38.230 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:38.230 | INFO | ANALYSIS : 58.1%, 3.0, 90.3ms, 0 | 20260505 05:00:38.230 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:38.230 | INFO | RT_METRICS : 45.5%, 1.0, 23.8ms, 0 | 20260505 05:00:38.230 | INFO | PRE_HQ : 57.5%, 2.2, 64.4ms, 0 | 20260505 05:00:38.230 | INFO | BAZWRITER : 58.9%, 4.2, 126.1ms, 0 | 20260505 05:00:38.230 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 05:00:38.230 | INFO | Uploaded 36GiB to GPU in 1.63291s (22.0465GiB/s) | 20260505 05:00:38.230 | INFO | Downloaded 38GiB to GPU in 1.69887s (22.3679GiB/s) | 20260505 05:00:43.369 | INFO | Analyzing chunk frames = [532992,533504) | 20260505 05:00:43.369 | INFO | MemInfo: { | 20260505 05:00:43.369 | INFO | "gpu" : " 17 GB", | 20260505 05:00:43.369 | INFO | "huge" : "", | 20260505 05:00:43.369 | INFO | "huge_shared" : " 27 GB", | 20260505 05:00:43.369 | INFO | "malloc" : " 14 GB", | 20260505 05:00:43.369 | INFO | "pinned" : " 29 GB" | 20260505 05:00:43.369 | INFO | } | 20260505 05:00:43.369 | INFO | Perf report for the last 5.13956 seconds: | 20260505 05:00:43.369 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:00:43.369 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:43.369 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:43.369 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:43.369 | INFO | ANALYSIS : 57.9%, 3.0, 90.4ms, 0 | 20260505 05:00:43.369 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:43.369 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:43.369 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 05:00:43.369 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:43.369 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:43.370 | INFO | Uploaded 36GiB to GPU in 1.59968s (22.5045GiB/s) | 20260505 05:00:43.370 | INFO | Downloaded 34GiB to GPU in 1.51591s (22.4287GiB/s) | 20260505 05:00:48.508 | INFO | Analyzing chunk frames = [533504,534016) | 20260505 05:00:48.508 | INFO | MemInfo: { | 20260505 05:00:48.508 | INFO | "gpu" : " 17 GB", | 20260505 05:00:48.508 | INFO | "huge" : "", | 20260505 05:00:48.508 | INFO | "huge_shared" : " 27 GB", | 20260505 05:00:48.508 | INFO | "malloc" : " 14 GB", | 20260505 05:00:48.508 | INFO | "pinned" : " 29 GB" | 20260505 05:00:48.508 | INFO | } | 20260505 05:00:48.508 | INFO | Perf report for the last 5.1392 seconds: | 20260505 05:00:48.508 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:00:48.508 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:48.508 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:48.508 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:48.508 | INFO | ANALYSIS : 57.3%, 3.0, 89.0ms, 0 | 20260505 05:00:48.508 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:48.508 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:48.508 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 05:00:48.508 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:48.508 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:48.509 | INFO | Uploaded 36GiB to GPU in 1.60338s (22.4526GiB/s) | 20260505 05:00:48.509 | INFO | Downloaded 34GiB to GPU in 1.52157s (22.3453GiB/s) | 20260505 05:00:53.648 | INFO | Analyzing chunk frames = [534016,534528) | 20260505 05:00:53.648 | INFO | MemInfo: { | 20260505 05:00:53.648 | INFO | "gpu" : " 17 GB", | 20260505 05:00:53.648 | INFO | "huge" : "", | 20260505 05:00:53.648 | INFO | "huge_shared" : " 27 GB", | 20260505 05:00:53.648 | INFO | "malloc" : " 14 GB", | 20260505 05:00:53.648 | INFO | "pinned" : " 29 GB" | 20260505 05:00:53.648 | INFO | } | 20260505 05:00:53.648 | INFO | Perf report for the last 5.13948 seconds: | 20260505 05:00:53.648 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:00:53.648 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:53.648 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:53.648 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:53.648 | INFO | ANALYSIS : 57.0%, 3.0, 88.4ms, 0 | 20260505 05:00:53.648 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:53.648 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:53.648 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260505 05:00:53.648 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:53.648 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:53.648 | INFO | Uploaded 36GiB to GPU in 1.60298s (22.4582GiB/s) | 20260505 05:00:53.648 | INFO | Downloaded 34GiB to GPU in 1.51998s (22.3687GiB/s) | 20260505 05:00:58.787 | INFO | Analyzing chunk frames = [534528,535040) | 20260505 05:00:58.787 | INFO | MemInfo: { | 20260505 05:00:58.787 | INFO | "gpu" : " 17 GB", | 20260505 05:00:58.787 | INFO | "huge" : "", | 20260505 05:00:58.787 | INFO | "huge_shared" : " 27 GB", | 20260505 05:00:58.787 | INFO | "malloc" : " 14 GB", | 20260505 05:00:58.787 | INFO | "pinned" : " 29 GB" | 20260505 05:00:58.787 | INFO | } | 20260505 05:00:58.787 | INFO | Perf report for the last 5.13929 seconds: | 20260505 05:00:58.787 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:00:58.787 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:58.787 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:58.787 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:58.787 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260505 05:00:58.787 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:58.787 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:58.787 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260505 05:00:58.787 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:58.787 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:00:58.788 | INFO | Uploaded 36GiB to GPU in 1.6016s (22.4775GiB/s) | 20260505 05:00:58.788 | INFO | Downloaded 34GiB to GPU in 1.51824s (22.3943GiB/s) | 20260505 05:01:03.926 | INFO | Analyzing chunk frames = [535040,535552) | 20260505 05:01:03.926 | INFO | MemInfo: { | 20260505 05:01:03.926 | INFO | "gpu" : " 17 GB", | 20260505 05:01:03.926 | INFO | "huge" : "", | 20260505 05:01:03.926 | INFO | "huge_shared" : " 27 GB", | 20260505 05:01:03.927 | INFO | "malloc" : " 14 GB", | 20260505 05:01:03.927 | INFO | "pinned" : " 29 GB" | 20260505 05:01:03.927 | INFO | } | 20260505 05:01:03.927 | INFO | Perf report for the last 5.13947 seconds: | 20260505 05:01:03.927 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:01:03.927 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:03.927 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:03.927 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:03.927 | INFO | ANALYSIS : 57.2%, 3.0, 89.3ms, 0 | 20260505 05:01:03.927 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:03.927 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:03.927 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 05:01:03.927 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:03.927 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:03.927 | INFO | Uploaded 36GiB to GPU in 1.60136s (22.4809GiB/s) | 20260505 05:01:03.927 | INFO | Downloaded 34GiB to GPU in 1.5191s (22.3817GiB/s) | 20260505 05:01:09.066 | INFO | Analyzing chunk frames = [535552,536064) | 20260505 05:01:09.066 | INFO | MemInfo: { | 20260505 05:01:09.066 | INFO | "gpu" : " 17 GB", | 20260505 05:01:09.066 | INFO | "huge" : "", | 20260505 05:01:09.066 | INFO | "huge_shared" : " 27 GB", | 20260505 05:01:09.066 | INFO | "malloc" : " 14 GB", | 20260505 05:01:09.066 | INFO | "pinned" : " 29 GB" | 20260505 05:01:09.066 | INFO | } | 20260505 05:01:09.066 | INFO | Perf report for the last 5.13929 seconds: | 20260505 05:01:09.066 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:01:09.066 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:09.066 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:09.066 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:09.066 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260505 05:01:09.066 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:09.066 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:09.066 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260505 05:01:09.066 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:09.066 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:09.066 | INFO | Uploaded 36GiB to GPU in 1.60123s (22.4827GiB/s) | 20260505 05:01:09.066 | INFO | Downloaded 34GiB to GPU in 1.51862s (22.3888GiB/s) | 20260505 05:01:14.205 | INFO | Analyzing chunk frames = [536064,536576) | 20260505 05:01:14.205 | INFO | MemInfo: { | 20260505 05:01:14.205 | INFO | "gpu" : " 17 GB", | 20260505 05:01:14.205 | INFO | "huge" : "", | 20260505 05:01:14.205 | INFO | "huge_shared" : " 27 GB", | 20260505 05:01:14.205 | INFO | "malloc" : " 14 GB", | 20260505 05:01:14.205 | INFO | "pinned" : " 29 GB" | 20260505 05:01:14.205 | INFO | } | 20260505 05:01:14.205 | INFO | Perf report for the last 5.13953 seconds: | 20260505 05:01:14.205 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:01:14.205 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:14.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:14.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:14.205 | INFO | ANALYSIS : 57.5%, 3.0, 89.8ms, 0 | 20260505 05:01:14.206 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:14.206 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:14.206 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:01:14.206 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:14.206 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:14.206 | INFO | Uploaded 36GiB to GPU in 1.6017s (22.4761GiB/s) | 20260505 05:01:14.206 | INFO | Downloaded 34GiB to GPU in 1.51854s (22.3899GiB/s) | 20260505 05:01:14.303 | INFO | Starting a round of Baz IO | 20260505 05:01:17.250 | INFO | Populating RTMetrics report | 20260505 05:01:17.250 | INFO | RTMetrics report populated | 20260505 05:01:17.250 | INFO | Opening tmp RTMetrics json file | 20260505 05:01:17.251 | INFO | Serializing RTMetrics to json | 20260505 05:01:17.252 | INFO | Serialization done, swapping tmp file now... | 20260505 05:01:17.253 | INFO | RTMetrics done | 20260505 05:01:17.352 | INFO | Finished a round of Baz IO: Wrote 2.135 GiB in 3.049 seconds | 20260505 05:01:17.352 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.24% / 0.10% / 57.65% | 20260505 05:01:17.352 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.83817% / 0.000182788% / 3.57628e-05% | 20260505 05:01:17.352 | INFO | Baz IO overall: | 20260505 05:01:17.352 | INFO | BytesWritten : 235145392844 | 20260505 05:01:17.352 | INFO | NumEvents : 109847206131 | 20260505 05:01:19.344 | INFO | Analyzing chunk frames = [536576,537088) | 20260505 05:01:19.344 | INFO | MemInfo: { | 20260505 05:01:19.344 | INFO | "gpu" : " 17 GB", | 20260505 05:01:19.344 | INFO | "huge" : "", | 20260505 05:01:19.345 | INFO | "huge_shared" : " 27 GB", | 20260505 05:01:19.345 | INFO | "malloc" : " 14 GB", | 20260505 05:01:19.345 | INFO | "pinned" : " 29 GB" | 20260505 05:01:19.345 | INFO | } | 20260505 05:01:19.345 | INFO | Perf report for the last 5.13915 seconds: | 20260505 05:01:19.345 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:01:19.345 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:19.345 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:19.345 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:19.345 | INFO | ANALYSIS : 58.1%, 3.0, 90.2ms, 0 | 20260505 05:01:19.345 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:19.345 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260505 05:01:19.345 | INFO | PRE_HQ : 57.5%, 2.2, 64.4ms, 0 | 20260505 05:01:19.345 | INFO | BAZWRITER : 58.9%, 4.2, 125.8ms, 0 | 20260505 05:01:19.345 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 05:01:19.345 | INFO | Uploaded 36GiB to GPU in 1.63731s (21.9872GiB/s) | 20260505 05:01:19.345 | INFO | Downloaded 38GiB to GPU in 1.70226s (22.3233GiB/s) | 20260505 05:01:24.484 | INFO | Analyzing chunk frames = [537088,537600) | 20260505 05:01:24.484 | INFO | MemInfo: { | 20260505 05:01:24.484 | INFO | "gpu" : " 17 GB", | 20260505 05:01:24.484 | INFO | "huge" : "", | 20260505 05:01:24.484 | INFO | "huge_shared" : " 27 GB", | 20260505 05:01:24.484 | INFO | "malloc" : " 14 GB", | 20260505 05:01:24.484 | INFO | "pinned" : " 29 GB" | 20260505 05:01:24.484 | INFO | } | 20260505 05:01:24.484 | INFO | Perf report for the last 5.13912 seconds: | 20260505 05:01:24.484 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:01:24.484 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:24.484 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:24.484 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:24.484 | INFO | ANALYSIS : 58.0%, 3.0, 90.4ms, 0 | 20260505 05:01:24.484 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:24.484 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:24.484 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 05:01:24.484 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:24.484 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:24.484 | INFO | Uploaded 36GiB to GPU in 1.59966s (22.5048GiB/s) | 20260505 05:01:24.484 | INFO | Downloaded 34GiB to GPU in 1.51623s (22.424GiB/s) | 20260505 05:01:29.623 | INFO | Analyzing chunk frames = [537600,538112) | 20260505 05:01:29.623 | INFO | MemInfo: { | 20260505 05:01:29.623 | INFO | "gpu" : " 17 GB", | 20260505 05:01:29.623 | INFO | "huge" : "", | 20260505 05:01:29.623 | INFO | "huge_shared" : " 27 GB", | 20260505 05:01:29.623 | INFO | "malloc" : " 14 GB", | 20260505 05:01:29.623 | INFO | "pinned" : " 29 GB" | 20260505 05:01:29.623 | INFO | } | 20260505 05:01:29.623 | INFO | Perf report for the last 5.13954 seconds: | 20260505 05:01:29.623 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:01:29.623 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:29.623 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:29.623 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:29.623 | INFO | ANALYSIS : 57.2%, 3.0, 89.0ms, 0 | 20260505 05:01:29.623 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:29.623 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:29.623 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 05:01:29.623 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:29.623 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:29.624 | INFO | Uploaded 36GiB to GPU in 1.60371s (22.4479GiB/s) | 20260505 05:01:29.624 | INFO | Downloaded 34GiB to GPU in 1.52062s (22.3593GiB/s) | 20260505 05:01:34.762 | INFO | Analyzing chunk frames = [538112,538624) | 20260505 05:01:34.762 | INFO | MemInfo: { | 20260505 05:01:34.762 | INFO | "gpu" : " 17 GB", | 20260505 05:01:34.762 | INFO | "huge" : "", | 20260505 05:01:34.762 | INFO | "huge_shared" : " 27 GB", | 20260505 05:01:34.762 | INFO | "malloc" : " 14 GB", | 20260505 05:01:34.762 | INFO | "pinned" : " 29 GB" | 20260505 05:01:34.762 | INFO | } | 20260505 05:01:34.762 | INFO | Perf report for the last 5.13906 seconds: | 20260505 05:01:34.762 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:01:34.762 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:34.762 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:34.762 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:34.762 | INFO | ANALYSIS : 57.0%, 3.0, 88.4ms, 0 | 20260505 05:01:34.762 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:34.762 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:34.762 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 05:01:34.762 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:34.762 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:34.763 | INFO | Uploaded 36GiB to GPU in 1.60281s (22.4606GiB/s) | 20260505 05:01:34.763 | INFO | Downloaded 34GiB to GPU in 1.5195s (22.3758GiB/s) | 20260505 05:01:39.902 | INFO | Analyzing chunk frames = [538624,539136) | 20260505 05:01:39.902 | INFO | MemInfo: { | 20260505 05:01:39.902 | INFO | "gpu" : " 17 GB", | 20260505 05:01:39.902 | INFO | "huge" : "", | 20260505 05:01:39.902 | INFO | "huge_shared" : " 27 GB", | 20260505 05:01:39.902 | INFO | "malloc" : " 14 GB", | 20260505 05:01:39.902 | INFO | "pinned" : " 29 GB" | 20260505 05:01:39.902 | INFO | } | 20260505 05:01:39.902 | INFO | Perf report for the last 5.13938 seconds: | 20260505 05:01:39.902 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:01:39.902 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:39.902 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:39.902 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:39.902 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260505 05:01:39.902 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:39.902 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:39.902 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260505 05:01:39.902 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:39.902 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:39.902 | INFO | Uploaded 36GiB to GPU in 1.60164s (22.477GiB/s) | 20260505 05:01:39.902 | INFO | Downloaded 34GiB to GPU in 1.51833s (22.3931GiB/s) | 20260505 05:01:45.041 | INFO | Analyzing chunk frames = [539136,539648) | 20260505 05:01:45.041 | INFO | MemInfo: { | 20260505 05:01:45.041 | INFO | "gpu" : " 17 GB", | 20260505 05:01:45.041 | INFO | "huge" : "", | 20260505 05:01:45.041 | INFO | "huge_shared" : " 27 GB", | 20260505 05:01:45.041 | INFO | "malloc" : " 14 GB", | 20260505 05:01:45.041 | INFO | "pinned" : " 29 GB" | 20260505 05:01:45.041 | INFO | } | 20260505 05:01:45.041 | INFO | Perf report for the last 5.13936 seconds: | 20260505 05:01:45.041 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:01:45.041 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:45.041 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:45.041 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:45.041 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260505 05:01:45.041 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:45.041 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:45.041 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 05:01:45.041 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:45.042 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:45.042 | INFO | Uploaded 36GiB to GPU in 1.60148s (22.4792GiB/s) | 20260505 05:01:45.042 | INFO | Downloaded 34GiB to GPU in 1.51845s (22.3912GiB/s) | 20260505 05:01:50.180 | INFO | Analyzing chunk frames = [539648,540160) | 20260505 05:01:50.180 | INFO | MemInfo: { | 20260505 05:01:50.180 | INFO | "gpu" : " 17 GB", | 20260505 05:01:50.180 | INFO | "huge" : "", | 20260505 05:01:50.180 | INFO | "huge_shared" : " 27 GB", | 20260505 05:01:50.180 | INFO | "malloc" : " 14 GB", | 20260505 05:01:50.181 | INFO | "pinned" : " 29 GB" | 20260505 05:01:50.181 | INFO | } | 20260505 05:01:50.181 | INFO | Perf report for the last 5.13948 seconds: | 20260505 05:01:50.181 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:01:50.181 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:50.181 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:50.181 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:50.181 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260505 05:01:50.181 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:50.181 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:50.181 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 05:01:50.181 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:50.181 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:50.181 | INFO | Uploaded 36GiB to GPU in 1.60222s (22.4688GiB/s) | 20260505 05:01:50.181 | INFO | Downloaded 34GiB to GPU in 1.51815s (22.3957GiB/s) | 20260505 05:01:55.320 | INFO | Analyzing chunk frames = [540160,540672) | 20260505 05:01:55.320 | INFO | MemInfo: { | 20260505 05:01:55.320 | INFO | "gpu" : " 17 GB", | 20260505 05:01:55.320 | INFO | "huge" : "", | 20260505 05:01:55.320 | INFO | "huge_shared" : " 27 GB", | 20260505 05:01:55.320 | INFO | "malloc" : " 14 GB", | 20260505 05:01:55.320 | INFO | "pinned" : " 29 GB" | 20260505 05:01:55.320 | INFO | } | 20260505 05:01:55.320 | INFO | Perf report for the last 5.13926 seconds: | 20260505 05:01:55.320 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:01:55.320 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:55.320 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:55.320 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:55.320 | INFO | ANALYSIS : 57.6%, 3.0, 89.8ms, 0 | 20260505 05:01:55.320 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:55.320 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:55.320 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 05:01:55.320 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:55.320 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:01:55.320 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 05:01:55.320 | INFO | Total mem usage: 15.1 GiB | 20260505 05:01:55.320 | INFO | All Time High Water Marks sum: 15.7 GiB (Savings: 4.09%) | 20260505 05:01:55.320 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -4.29%) | 20260505 05:01:55.320 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 05:01:55.320 | INFO | High Water Breakdown: | 20260505 05:01:55.320 | INFO | 59.6% : 8.6 GiB ( 95.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 05:01:55.320 | INFO | 40.3% : 5.82 GiB ( 87.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 05:01:55.320 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 05:01:55.320 | INFO | ------------------------------------------ | 20260505 05:01:55.320 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:01:55.320 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 05:01:55.320 | INFO | Total mem usage: 26.5 GiB | 20260505 05:01:55.320 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 05:01:55.320 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 05:01:55.320 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 05:01:55.320 | INFO | High Water Breakdown: | 20260505 05:01:55.320 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:01:55.320 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:01:55.320 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:01:55.320 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:01:55.320 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:01:55.320 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:01:55.320 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 05:01:55.320 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 05:01:55.320 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:01:55.320 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 05:01:55.320 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 05:01:55.320 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 05:01:55.320 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 05:01:55.320 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 05:01:55.320 | INFO | ------------------------------------------ | 20260505 05:01:55.320 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:01:55.320 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 05:01:55.320 | INFO | Total mem usage: 25.6 GiB | 20260505 05:01:55.320 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 05:01:55.320 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 05:01:55.320 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 05:01:55.320 | INFO | High Water Breakdown: | 20260505 05:01:55.320 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 05:01:55.320 | INFO | ------------------------------------------ | 20260505 05:01:55.320 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:01:55.320 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 05:01:55.320 | INFO | Total mem usage: 15.7 GiB | 20260505 05:01:55.320 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 05:01:55.320 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 05:01:55.320 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260505 05:01:55.320 | INFO | High Water Breakdown: | 20260505 05:01:55.320 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 05:01:55.320 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 05:01:55.320 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 05:01:55.320 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 05:01:55.320 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 05:01:55.320 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 05:01:55.320 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 05:01:55.320 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 05:01:55.320 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 05:01:55.320 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:01:55.320 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 05:01:55.320 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:01:55.320 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 05:01:55.320 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 05:01:55.320 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 05:01:55.320 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 05:01:55.320 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:01:55.320 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 05:01:55.320 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:01:55.320 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:01:55.320 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:01:55.320 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 05:01:55.320 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 05:01:55.320 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:01:55.320 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 05:01:55.320 | INFO | ------------------------------------------ | 20260505 05:01:55.320 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:01:55.320 | INFO | Uploaded 36GiB to GPU in 1.60208s (22.4708GiB/s) | 20260505 05:01:55.320 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 05:01:55.321 | INFO | Downloaded 34GiB to GPU in 1.51945s (22.3765GiB/s) | 20260505 05:01:55.321 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:01:55.321 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:01:55.321 | INFO | PerfReport: Full Execution -- 3.04e+01 / -nan, 1.24e+01 / 0.00e+00 | 20260505 05:01:55.321 | INFO | PerfReport: 0.28%, AggregateStats -- 8.65e-02 / -nan, 4.71e-03 / 0.00e+00 | 20260505 05:01:55.321 | INFO | PerfReport: 10.84%, Baseline -- 3.30e+00 / -nan, 1.79e-02 / 0.00e+00 | 20260505 05:01:55.321 | INFO | PerfReport: 3.04%, Histogram -- 9.24e-01 / -nan, 5.06e-02 / 0.00e+00 | 20260505 05:01:55.321 | INFO | PerfReport: 16.04%, DME -- 3.90e+01 / -nan, 3.17e+00 / 0.00e+00 | 20260505 05:01:55.321 | INFO | PerfReport: 56.63%, FrameLabeling -- 1.72e+01 / -nan, 4.68e-02 / 0.00e+00 | 20260505 05:01:55.321 | INFO | PerfReport: 7.60%, PulseAccumulating -- 2.31e+00 / -nan, 1.52e-01 / 0.00e+00 | 20260505 05:01:55.321 | INFO | PerfReport: 3.94%, Metrics -- 1.20e+00 / -nan, 2.00e-01 / 0.00e+00 | 20260505 05:01:55.321 | INFO | PerfReport: 1.62%, PulseSerializing -- 4.93e-01 / -nan, 7.95e-02 / 0.00e+00 | 20260505 05:01:55.321 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 05:01:55.321 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:01:55.321 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:01:55.321 | INFO | PerfReport: Full Execution -- 9.13e+01 / -nan, 1.90e+01 / 0.00e+00 | 20260505 05:01:55.321 | INFO | PerfReport: 18.33%, Upload -- 1.67e+01 / 0.00e+00, 3.62e-01 / 0.00e+00 | 20260505 05:01:55.321 | INFO | PerfReport: 17.59%, Download -- 1.61e+01 / 0.00e+00, 6.87e-01 / 0.00e+00 | 20260505 05:01:55.321 | INFO | PerfReport: 33.31%, Compute -- 3.04e+01 / 0.00e+00, 1.24e+01 / 0.00e+00 | 20260505 05:01:55.321 | INFO | PerfReport: 0.55%, LockUpload -- 5.05e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260505 05:01:55.321 | INFO | PerfReport: 0.00%, LockDownload -- 3.94e-04 / 0.00e+00, 1.77e-04 / 0.00e+00 | 20260505 05:01:55.321 | INFO | PerfReport: 30.20%, LockCompute -- 2.76e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260505 05:01:55.321 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.20e-02 / 0.00e+00 | 20260505 05:01:55.417 | INFO | Starting a round of Baz IO | 20260505 05:01:58.364 | INFO | Populating RTMetrics report | 20260505 05:01:58.364 | INFO | RTMetrics report populated | 20260505 05:01:58.364 | INFO | Opening tmp RTMetrics json file | 20260505 05:01:58.365 | INFO | Serializing RTMetrics to json | 20260505 05:01:58.366 | INFO | Serialization done, swapping tmp file now... | 20260505 05:01:58.366 | INFO | RTMetrics done | 20260505 05:01:58.457 | INFO | Finished a round of Baz IO: Wrote 2.137 GiB in 3.039 seconds | 20260505 05:01:58.457 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.14% / 0.10% / 57.76% | 20260505 05:01:58.457 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.78821% / 0.00026226% / 3.57628e-05% | 20260505 05:01:58.457 | INFO | Baz IO overall: | 20260505 05:01:58.457 | INFO | BytesWritten : 237439457502 | 20260505 05:01:58.457 | INFO | NumEvents : 110937503917 | 20260505 05:02:00.459 | INFO | Analyzing chunk frames = [540672,541184) | 20260505 05:02:00.459 | INFO | MemInfo: { | 20260505 05:02:00.459 | INFO | "gpu" : " 17 GB", | 20260505 05:02:00.459 | INFO | "huge" : "", | 20260505 05:02:00.459 | INFO | "huge_shared" : " 27 GB", | 20260505 05:02:00.459 | INFO | "malloc" : " 14 GB", | 20260505 05:02:00.459 | INFO | "pinned" : " 29 GB" | 20260505 05:02:00.459 | INFO | } | 20260505 05:02:00.459 | INFO | Perf report for the last 5.13922 seconds: | 20260505 05:02:00.459 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:02:00.459 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:00.459 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:00.459 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:00.459 | INFO | ANALYSIS : 58.1%, 3.0, 90.3ms, 0 | 20260505 05:02:00.459 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:00.459 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260505 05:02:00.459 | INFO | PRE_HQ : 57.4%, 2.2, 64.2ms, 0 | 20260505 05:02:00.459 | INFO | BAZWRITER : 58.7%, 4.2, 126.2ms, 0 | 20260505 05:02:00.459 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 05:02:00.460 | INFO | Uploaded 36GiB to GPU in 1.6453s (21.8805GiB/s) | 20260505 05:02:00.460 | INFO | Downloaded 38GiB to GPU in 1.70915s (22.2332GiB/s) | 20260505 05:02:05.598 | INFO | Analyzing chunk frames = [541184,541696) | 20260505 05:02:05.598 | INFO | MemInfo: { | 20260505 05:02:05.598 | INFO | "gpu" : " 17 GB", | 20260505 05:02:05.598 | INFO | "huge" : "", | 20260505 05:02:05.598 | INFO | "huge_shared" : " 27 GB", | 20260505 05:02:05.598 | INFO | "malloc" : " 14 GB", | 20260505 05:02:05.599 | INFO | "pinned" : " 29 GB" | 20260505 05:02:05.599 | INFO | } | 20260505 05:02:05.599 | INFO | Perf report for the last 5.13954 seconds: | 20260505 05:02:05.599 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:02:05.599 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:05.599 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:05.599 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:05.599 | INFO | ANALYSIS : 57.9%, 3.0, 90.4ms, 0 | 20260505 05:02:05.599 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:05.599 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:05.599 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 05:02:05.599 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:05.599 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:05.599 | INFO | Uploaded 36GiB to GPU in 1.59918s (22.5116GiB/s) | 20260505 05:02:05.599 | INFO | Downloaded 34GiB to GPU in 1.51586s (22.4296GiB/s) | 20260505 05:02:10.738 | INFO | Analyzing chunk frames = [541696,542208) | 20260505 05:02:10.738 | INFO | MemInfo: { | 20260505 05:02:10.738 | INFO | "gpu" : " 17 GB", | 20260505 05:02:10.738 | INFO | "huge" : "", | 20260505 05:02:10.738 | INFO | "huge_shared" : " 27 GB", | 20260505 05:02:10.738 | INFO | "malloc" : " 14 GB", | 20260505 05:02:10.738 | INFO | "pinned" : " 29 GB" | 20260505 05:02:10.738 | INFO | } | 20260505 05:02:10.738 | INFO | Perf report for the last 5.13917 seconds: | 20260505 05:02:10.738 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:02:10.738 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:10.738 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:10.738 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:10.738 | INFO | ANALYSIS : 57.3%, 3.0, 89.0ms, 0 | 20260505 05:02:10.738 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:10.738 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:10.738 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 05:02:10.738 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:10.738 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:10.738 | INFO | Uploaded 36GiB to GPU in 1.6033s (22.4537GiB/s) | 20260505 05:02:10.738 | INFO | Downloaded 34GiB to GPU in 1.52118s (22.3511GiB/s) | 20260505 05:02:15.877 | INFO | Analyzing chunk frames = [542208,542720) | 20260505 05:02:15.877 | INFO | MemInfo: { | 20260505 05:02:15.877 | INFO | "gpu" : " 17 GB", | 20260505 05:02:15.877 | INFO | "huge" : "", | 20260505 05:02:15.877 | INFO | "huge_shared" : " 27 GB", | 20260505 05:02:15.877 | INFO | "malloc" : " 14 GB", | 20260505 05:02:15.877 | INFO | "pinned" : " 29 GB" | 20260505 05:02:15.877 | INFO | } | 20260505 05:02:15.877 | INFO | Perf report for the last 5.13945 seconds: | 20260505 05:02:15.877 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:02:15.877 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:15.877 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:15.877 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:15.877 | INFO | ANALYSIS : 56.9%, 3.0, 88.3ms, 0 | 20260505 05:02:15.877 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:15.877 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:15.877 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 05:02:15.877 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:15.877 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:15.878 | INFO | Uploaded 36GiB to GPU in 1.60302s (22.4576GiB/s) | 20260505 05:02:15.878 | INFO | Downloaded 34GiB to GPU in 1.51955s (22.375GiB/s) | 20260505 05:02:21.016 | INFO | Analyzing chunk frames = [542720,543232) | 20260505 05:02:21.016 | INFO | MemInfo: { | 20260505 05:02:21.016 | INFO | "gpu" : " 17 GB", | 20260505 05:02:21.016 | INFO | "huge" : "", | 20260505 05:02:21.016 | INFO | "huge_shared" : " 27 GB", | 20260505 05:02:21.016 | INFO | "malloc" : " 14 GB", | 20260505 05:02:21.016 | INFO | "pinned" : " 29 GB" | 20260505 05:02:21.016 | INFO | } | 20260505 05:02:21.017 | INFO | Perf report for the last 5.13917 seconds: | 20260505 05:02:21.017 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:02:21.017 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:21.017 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:21.017 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:21.017 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260505 05:02:21.017 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:21.017 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:21.017 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 05:02:21.017 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:21.017 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:21.017 | INFO | Uploaded 36GiB to GPU in 1.60082s (22.4884GiB/s) | 20260505 05:02:21.017 | INFO | Downloaded 34GiB to GPU in 1.51758s (22.4041GiB/s) | 20260505 05:02:26.156 | INFO | Analyzing chunk frames = [543232,543744) | 20260505 05:02:26.156 | INFO | MemInfo: { | 20260505 05:02:26.156 | INFO | "gpu" : " 17 GB", | 20260505 05:02:26.156 | INFO | "huge" : "", | 20260505 05:02:26.156 | INFO | "huge_shared" : " 27 GB", | 20260505 05:02:26.156 | INFO | "malloc" : " 14 GB", | 20260505 05:02:26.156 | INFO | "pinned" : " 29 GB" | 20260505 05:02:26.156 | INFO | } | 20260505 05:02:26.156 | INFO | Perf report for the last 5.13927 seconds: | 20260505 05:02:26.156 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:02:26.156 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:26.156 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:26.156 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:26.156 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260505 05:02:26.156 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:26.156 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:26.156 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260505 05:02:26.156 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:26.156 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:26.156 | INFO | Uploaded 36GiB to GPU in 1.60182s (22.4744GiB/s) | 20260505 05:02:26.156 | INFO | Downloaded 34GiB to GPU in 1.51785s (22.4002GiB/s) | 20260505 05:02:31.295 | INFO | Analyzing chunk frames = [543744,544256) | 20260505 05:02:31.295 | INFO | MemInfo: { | 20260505 05:02:31.295 | INFO | "gpu" : " 17 GB", | 20260505 05:02:31.295 | INFO | "huge" : "", | 20260505 05:02:31.295 | INFO | "huge_shared" : " 27 GB", | 20260505 05:02:31.295 | INFO | "malloc" : " 14 GB", | 20260505 05:02:31.295 | INFO | "pinned" : " 29 GB" | 20260505 05:02:31.295 | INFO | } | 20260505 05:02:31.295 | INFO | Perf report for the last 5.13943 seconds: | 20260505 05:02:31.295 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:02:31.295 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:31.295 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:31.295 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:31.295 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260505 05:02:31.295 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:31.295 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:31.295 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260505 05:02:31.295 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:31.295 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:31.295 | INFO | Uploaded 36GiB to GPU in 1.60257s (22.464GiB/s) | 20260505 05:02:31.295 | INFO | Downloaded 34GiB to GPU in 1.51987s (22.3704GiB/s) | 20260505 05:02:36.434 | INFO | Analyzing chunk frames = [544256,544768) | 20260505 05:02:36.434 | INFO | MemInfo: { | 20260505 05:02:36.434 | INFO | "gpu" : " 17 GB", | 20260505 05:02:36.435 | INFO | "huge" : "", | 20260505 05:02:36.435 | INFO | "huge_shared" : " 27 GB", | 20260505 05:02:36.435 | INFO | "malloc" : " 14 GB", | 20260505 05:02:36.435 | INFO | "pinned" : " 29 GB" | 20260505 05:02:36.435 | INFO | } | 20260505 05:02:36.435 | INFO | Perf report for the last 5.13944 seconds: | 20260505 05:02:36.435 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:02:36.435 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:36.435 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:36.435 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:36.435 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260505 05:02:36.435 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:36.435 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:36.435 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 05:02:36.435 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:36.435 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:36.435 | INFO | Uploaded 36GiB to GPU in 1.60075s (22.4895GiB/s) | 20260505 05:02:36.435 | INFO | Downloaded 34GiB to GPU in 1.51985s (22.3707GiB/s) | 20260505 05:02:36.531 | INFO | Starting a round of Baz IO | 20260505 05:02:39.481 | INFO | Populating RTMetrics report | 20260505 05:02:39.481 | INFO | RTMetrics report populated | 20260505 05:02:39.481 | INFO | Opening tmp RTMetrics json file | 20260505 05:02:39.482 | INFO | Serializing RTMetrics to json | 20260505 05:02:39.483 | INFO | Serialization done, swapping tmp file now... | 20260505 05:02:39.484 | INFO | RTMetrics done | 20260505 05:02:39.574 | INFO | Finished a round of Baz IO: Wrote 2.138 GiB in 3.042 seconds | 20260505 05:02:39.574 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.04% / 0.10% / 57.86% | 20260505 05:02:39.574 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.73324% / 0.000206629% / 5.16574e-05% | 20260505 05:02:39.574 | INFO | Baz IO overall: | 20260505 05:02:39.574 | INFO | BytesWritten : 239734662852 | 20260505 05:02:39.574 | INFO | NumEvents : 112028499992 | 20260505 05:02:41.574 | INFO | Analyzing chunk frames = [544768,545280) | 20260505 05:02:41.574 | INFO | MemInfo: { | 20260505 05:02:41.574 | INFO | "gpu" : " 17 GB", | 20260505 05:02:41.574 | INFO | "huge" : "", | 20260505 05:02:41.574 | INFO | "huge_shared" : " 27 GB", | 20260505 05:02:41.574 | INFO | "malloc" : " 14 GB", | 20260505 05:02:41.574 | INFO | "pinned" : " 29 GB" | 20260505 05:02:41.574 | INFO | } | 20260505 05:02:41.574 | INFO | Perf report for the last 5.13934 seconds: | 20260505 05:02:41.574 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:02:41.574 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:41.574 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:41.574 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:41.574 | INFO | ANALYSIS : 58.1%, 3.0, 90.3ms, 0 | 20260505 05:02:41.574 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:41.574 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260505 05:02:41.574 | INFO | PRE_HQ : 57.5%, 2.2, 64.3ms, 0 | 20260505 05:02:41.574 | INFO | BAZWRITER : 58.8%, 4.2, 126.7ms, 0 | 20260505 05:02:41.574 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 05:02:41.574 | INFO | Uploaded 36GiB to GPU in 1.63825s (21.9746GiB/s) | 20260505 05:02:41.574 | INFO | Downloaded 38GiB to GPU in 1.70295s (22.3143GiB/s) | 20260505 05:02:46.713 | INFO | Analyzing chunk frames = [545280,545792) | 20260505 05:02:46.713 | INFO | MemInfo: { | 20260505 05:02:46.713 | INFO | "gpu" : " 17 GB", | 20260505 05:02:46.713 | INFO | "huge" : "", | 20260505 05:02:46.713 | INFO | "huge_shared" : " 27 GB", | 20260505 05:02:46.713 | INFO | "malloc" : " 14 GB", | 20260505 05:02:46.713 | INFO | "pinned" : " 29 GB" | 20260505 05:02:46.713 | INFO | } | 20260505 05:02:46.713 | INFO | Perf report for the last 5.1393 seconds: | 20260505 05:02:46.713 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:02:46.713 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:46.713 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:46.713 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:46.713 | INFO | ANALYSIS : 57.9%, 3.0, 90.3ms, 0 | 20260505 05:02:46.713 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:46.713 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:46.713 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 05:02:46.713 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:46.714 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:46.714 | INFO | Uploaded 36GiB to GPU in 1.60045s (22.4937GiB/s) | 20260505 05:02:46.714 | INFO | Downloaded 34GiB to GPU in 1.51702s (22.4123GiB/s) | 20260505 05:02:51.852 | INFO | Analyzing chunk frames = [545792,546304) | 20260505 05:02:51.852 | INFO | MemInfo: { | 20260505 05:02:51.852 | INFO | "gpu" : " 17 GB", | 20260505 05:02:51.852 | INFO | "huge" : "", | 20260505 05:02:51.852 | INFO | "huge_shared" : " 27 GB", | 20260505 05:02:51.853 | INFO | "malloc" : " 14 GB", | 20260505 05:02:51.853 | INFO | "pinned" : " 29 GB" | 20260505 05:02:51.853 | INFO | } | 20260505 05:02:51.853 | INFO | Perf report for the last 5.1392 seconds: | 20260505 05:02:51.853 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:02:51.853 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:51.853 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:51.853 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:51.853 | INFO | ANALYSIS : 57.3%, 3.0, 89.0ms, 0 | 20260505 05:02:51.853 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:51.853 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:51.853 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260505 05:02:51.853 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:51.853 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:51.853 | INFO | Uploaded 36GiB to GPU in 1.60324s (22.4546GiB/s) | 20260505 05:02:51.853 | INFO | Downloaded 34GiB to GPU in 1.52082s (22.3564GiB/s) | 20260505 05:02:56.992 | INFO | Analyzing chunk frames = [546304,546816) | 20260505 05:02:56.992 | INFO | MemInfo: { | 20260505 05:02:56.992 | INFO | "gpu" : " 17 GB", | 20260505 05:02:56.992 | INFO | "huge" : "", | 20260505 05:02:56.992 | INFO | "huge_shared" : " 27 GB", | 20260505 05:02:56.992 | INFO | "malloc" : " 14 GB", | 20260505 05:02:56.992 | INFO | "pinned" : " 29 GB" | 20260505 05:02:56.992 | INFO | } | 20260505 05:02:56.992 | INFO | Perf report for the last 5.13943 seconds: | 20260505 05:02:56.992 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:02:56.992 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:56.992 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:56.992 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:56.992 | INFO | ANALYSIS : 56.9%, 3.0, 88.3ms, 0 | 20260505 05:02:56.992 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:56.992 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:56.992 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 05:02:56.992 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:56.992 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:02:56.992 | INFO | Uploaded 36GiB to GPU in 1.60317s (22.4555GiB/s) | 20260505 05:02:56.992 | INFO | Downloaded 34GiB to GPU in 1.51999s (22.3686GiB/s) | 20260505 05:03:02.131 | INFO | Analyzing chunk frames = [546816,547328) | 20260505 05:03:02.131 | INFO | MemInfo: { | 20260505 05:03:02.131 | INFO | "gpu" : " 17 GB", | 20260505 05:03:02.131 | INFO | "huge" : "", | 20260505 05:03:02.131 | INFO | "huge_shared" : " 27 GB", | 20260505 05:03:02.131 | INFO | "malloc" : " 14 GB", | 20260505 05:03:02.131 | INFO | "pinned" : " 29 GB" | 20260505 05:03:02.131 | INFO | } | 20260505 05:03:02.131 | INFO | Perf report for the last 5.1393 seconds: | 20260505 05:03:02.131 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:03:02.131 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:02.131 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:02.131 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:02.131 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260505 05:03:02.131 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:02.131 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:02.131 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 05:03:02.131 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:02.131 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:02.132 | INFO | Uploaded 36GiB to GPU in 1.60194s (22.4728GiB/s) | 20260505 05:03:02.132 | INFO | Downloaded 34GiB to GPU in 1.51871s (22.3874GiB/s) | 20260505 05:03:07.271 | INFO | Analyzing chunk frames = [547328,547840) | 20260505 05:03:07.271 | INFO | MemInfo: { | 20260505 05:03:07.271 | INFO | "gpu" : " 17 GB", | 20260505 05:03:07.271 | INFO | "huge" : "", | 20260505 05:03:07.271 | INFO | "huge_shared" : " 27 GB", | 20260505 05:03:07.271 | INFO | "malloc" : " 14 GB", | 20260505 05:03:07.271 | INFO | "pinned" : " 29 GB" | 20260505 05:03:07.271 | INFO | } | 20260505 05:03:07.271 | INFO | Perf report for the last 5.13941 seconds: | 20260505 05:03:07.271 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:03:07.271 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:07.271 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:07.271 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:07.271 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260505 05:03:07.271 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:07.271 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:07.271 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260505 05:03:07.271 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:07.271 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:07.271 | INFO | Uploaded 36GiB to GPU in 1.60182s (22.4745GiB/s) | 20260505 05:03:07.271 | INFO | Downloaded 34GiB to GPU in 1.51824s (22.3944GiB/s) | 20260505 05:03:12.410 | INFO | Analyzing chunk frames = [547840,548352) | 20260505 05:03:12.410 | INFO | MemInfo: { | 20260505 05:03:12.410 | INFO | "gpu" : " 17 GB", | 20260505 05:03:12.410 | INFO | "huge" : "", | 20260505 05:03:12.410 | INFO | "huge_shared" : " 27 GB", | 20260505 05:03:12.410 | INFO | "malloc" : " 14 GB", | 20260505 05:03:12.410 | INFO | "pinned" : " 29 GB" | 20260505 05:03:12.410 | INFO | } | 20260505 05:03:12.410 | INFO | Perf report for the last 5.13913 seconds: | 20260505 05:03:12.410 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:03:12.410 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:12.410 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:12.410 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:12.410 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260505 05:03:12.410 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:12.410 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:12.410 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260505 05:03:12.410 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:12.410 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:12.410 | INFO | Uploaded 36GiB to GPU in 1.6014s (22.4804GiB/s) | 20260505 05:03:12.410 | INFO | Downloaded 34GiB to GPU in 1.51855s (22.3898GiB/s) | 20260505 05:03:17.549 | INFO | Analyzing chunk frames = [548352,548864) | 20260505 05:03:17.549 | INFO | MemInfo: { | 20260505 05:03:17.549 | INFO | "gpu" : " 17 GB", | 20260505 05:03:17.549 | INFO | "huge" : "", | 20260505 05:03:17.549 | INFO | "huge_shared" : " 27 GB", | 20260505 05:03:17.549 | INFO | "malloc" : " 14 GB", | 20260505 05:03:17.549 | INFO | "pinned" : " 29 GB" | 20260505 05:03:17.549 | INFO | } | 20260505 05:03:17.550 | INFO | Perf report for the last 5.13959 seconds: | 20260505 05:03:17.550 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:03:17.550 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:17.550 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:17.550 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:17.550 | INFO | ANALYSIS : 57.5%, 3.0, 89.6ms, 0 | 20260505 05:03:17.550 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:17.550 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:17.550 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260505 05:03:17.550 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:17.550 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:17.550 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 05:03:17.550 | INFO | Total mem usage: 15.1 GiB | 20260505 05:03:17.550 | INFO | All Time High Water Marks sum: 15.7 GiB (Savings: 4.09%) | 20260505 05:03:17.550 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -4.31%) | 20260505 05:03:17.550 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 05:03:17.550 | INFO | High Water Breakdown: | 20260505 05:03:17.550 | INFO | 59.6% : 8.6 GiB ( 95.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 05:03:17.550 | INFO | 40.3% : 5.81 GiB ( 87.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 05:03:17.550 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 05:03:17.550 | INFO | ------------------------------------------ | 20260505 05:03:17.550 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:03:17.550 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 05:03:17.550 | INFO | Total mem usage: 26.5 GiB | 20260505 05:03:17.550 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 05:03:17.550 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 05:03:17.550 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 05:03:17.550 | INFO | High Water Breakdown: | 20260505 05:03:17.550 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:03:17.550 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:03:17.550 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:03:17.550 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:03:17.550 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:03:17.550 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:03:17.550 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 05:03:17.550 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 05:03:17.550 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:03:17.550 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 05:03:17.550 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 05:03:17.550 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 05:03:17.550 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 05:03:17.550 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 05:03:17.550 | INFO | ------------------------------------------ | 20260505 05:03:17.550 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:03:17.550 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 05:03:17.550 | INFO | Total mem usage: 25.6 GiB | 20260505 05:03:17.550 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 05:03:17.550 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 05:03:17.550 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 05:03:17.550 | INFO | High Water Breakdown: | 20260505 05:03:17.550 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 05:03:17.550 | INFO | ------------------------------------------ | 20260505 05:03:17.550 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:03:17.550 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 05:03:17.550 | INFO | Total mem usage: 15.7 GiB | 20260505 05:03:17.550 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 05:03:17.550 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 05:03:17.550 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260505 05:03:17.550 | INFO | High Water Breakdown: | 20260505 05:03:17.550 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 05:03:17.550 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 05:03:17.550 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 05:03:17.550 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 05:03:17.550 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 05:03:17.550 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 05:03:17.550 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 05:03:17.550 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 05:03:17.550 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 05:03:17.550 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:03:17.550 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 05:03:17.550 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:03:17.550 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 05:03:17.550 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 05:03:17.550 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 05:03:17.550 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 05:03:17.550 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:03:17.550 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 05:03:17.550 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:03:17.550 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:03:17.550 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:03:17.550 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 05:03:17.550 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 05:03:17.550 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:03:17.550 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 05:03:17.550 | INFO | ------------------------------------------ | 20260505 05:03:17.550 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:03:17.550 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 05:03:17.550 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:03:17.550 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:03:17.550 | INFO | PerfReport: Full Execution -- 3.04e+01 / -nan, 1.28e+01 / 0.00e+00 | 20260505 05:03:17.550 | INFO | PerfReport: 0.28%, AggregateStats -- 8.63e-02 / -nan, 4.81e-03 / 0.00e+00 | 20260505 05:03:17.550 | INFO | PerfReport: 10.85%, Baseline -- 3.30e+00 / -nan, 1.71e-02 / 0.00e+00 | 20260505 05:03:17.550 | INFO | PerfReport: 3.04%, Histogram -- 9.24e-01 / -nan, 5.26e-02 / 0.00e+00 | 20260505 05:03:17.550 | INFO | PerfReport: 16.04%, DME -- 3.90e+01 / -nan, 3.26e+00 / 0.00e+00 | 20260505 05:03:17.550 | INFO | PerfReport: 56.64%, FrameLabeling -- 1.72e+01 / -nan, 4.49e-02 / 0.00e+00 | 20260505 05:03:17.550 | INFO | PerfReport: 7.59%, PulseAccumulating -- 2.31e+00 / -nan, 1.52e-01 / 0.00e+00 | 20260505 05:03:17.550 | INFO | PerfReport: 3.93%, Metrics -- 1.20e+00 / -nan, 1.98e-01 / 0.00e+00 | 20260505 05:03:17.550 | INFO | PerfReport: 1.62%, PulseSerializing -- 4.93e-01 / -nan, 7.93e-02 / 0.00e+00 | 20260505 05:03:17.550 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 05:03:17.550 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:03:17.550 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:03:17.550 | INFO | Uploaded 36GiB to GPU in 1.6027s (22.4621GiB/s) | 20260505 05:03:17.550 | INFO | Downloaded 34GiB to GPU in 1.52025s (22.3647GiB/s) | 20260505 05:03:17.550 | INFO | PerfReport: Full Execution -- 9.12e+01 / -nan, 1.88e+01 / 0.00e+00 | 20260505 05:03:17.551 | INFO | PerfReport: 18.35%, Upload -- 1.67e+01 / 0.00e+00, 3.77e-01 / 0.00e+00 | 20260505 05:03:17.551 | INFO | PerfReport: 17.61%, Download -- 1.61e+01 / 0.00e+00, 7.12e-01 / 0.00e+00 | 20260505 05:03:17.551 | INFO | PerfReport: 33.30%, Compute -- 3.04e+01 / 0.00e+00, 1.28e+01 / 0.00e+00 | 20260505 05:03:17.551 | INFO | PerfReport: 0.55%, LockUpload -- 5.06e-01 / 0.00e+00, 3.63e+00 / 0.00e+00 | 20260505 05:03:17.551 | INFO | PerfReport: 0.00%, LockDownload -- 3.84e-04 / 0.00e+00, 9.32e-05 / 0.00e+00 | 20260505 05:03:17.551 | INFO | PerfReport: 30.17%, LockCompute -- 2.75e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260505 05:03:17.551 | INFO | PerfReport: 0.02%, Misc -- 1.42e-02 / 0.00e+00, 2.69e-02 / 0.00e+00 | 20260505 05:03:17.648 | INFO | Starting a round of Baz IO | 20260505 05:03:20.597 | INFO | Populating RTMetrics report | 20260505 05:03:20.597 | INFO | RTMetrics report populated | 20260505 05:03:20.597 | INFO | Opening tmp RTMetrics json file | 20260505 05:03:20.598 | INFO | Serializing RTMetrics to json | 20260505 05:03:20.599 | INFO | Serialization done, swapping tmp file now... | 20260505 05:03:20.600 | INFO | RTMetrics done | 20260505 05:03:20.690 | INFO | Finished a round of Baz IO: Wrote 2.142 GiB in 3.042 seconds | 20260505 05:03:20.690 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.94% / 0.10% / 57.96% | 20260505 05:03:20.690 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.71702% / 0.000135104% / 5.16574e-05% | 20260505 05:03:20.690 | INFO | Baz IO overall: | 20260505 05:03:20.690 | INFO | BytesWritten : 242034199860 | 20260505 05:03:20.690 | INFO | NumEvents : 113122089500 | 20260505 05:03:22.689 | INFO | Analyzing chunk frames = [548864,549376) | 20260505 05:03:22.689 | INFO | MemInfo: { | 20260505 05:03:22.689 | INFO | "gpu" : " 17 GB", | 20260505 05:03:22.689 | INFO | "huge" : "", | 20260505 05:03:22.689 | INFO | "huge_shared" : " 27 GB", | 20260505 05:03:22.689 | INFO | "malloc" : " 14 GB", | 20260505 05:03:22.689 | INFO | "pinned" : " 29 GB" | 20260505 05:03:22.689 | INFO | } | 20260505 05:03:22.689 | INFO | Perf report for the last 5.13923 seconds: | 20260505 05:03:22.689 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:03:22.689 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:22.689 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:22.689 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:22.689 | INFO | ANALYSIS : 58.1%, 3.0, 90.3ms, 0 | 20260505 05:03:22.689 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:22.689 | INFO | RT_METRICS : 45.5%, 1.0, 23.9ms, 0 | 20260505 05:03:22.689 | INFO | PRE_HQ : 57.5%, 2.2, 64.2ms, 0 | 20260505 05:03:22.689 | INFO | BAZWRITER : 58.6%, 4.2, 125.7ms, 0 | 20260505 05:03:22.689 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 05:03:22.689 | INFO | Uploaded 36GiB to GPU in 1.63928s (21.9609GiB/s) | 20260505 05:03:22.689 | INFO | Downloaded 38GiB to GPU in 1.70302s (22.3133GiB/s) | 20260505 05:03:27.828 | INFO | Analyzing chunk frames = [549376,549888) | 20260505 05:03:27.828 | INFO | MemInfo: { | 20260505 05:03:27.828 | INFO | "gpu" : " 17 GB", | 20260505 05:03:27.828 | INFO | "huge" : "", | 20260505 05:03:27.828 | INFO | "huge_shared" : " 27 GB", | 20260505 05:03:27.828 | INFO | "malloc" : " 14 GB", | 20260505 05:03:27.828 | INFO | "pinned" : " 29 GB" | 20260505 05:03:27.828 | INFO | } | 20260505 05:03:27.828 | INFO | Perf report for the last 5.13924 seconds: | 20260505 05:03:27.828 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:03:27.828 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:27.828 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:27.828 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:27.828 | INFO | ANALYSIS : 57.9%, 3.0, 90.3ms, 0 | 20260505 05:03:27.828 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:27.828 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:27.828 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:03:27.828 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:27.828 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:27.828 | INFO | Uploaded 36GiB to GPU in 1.59993s (22.501GiB/s) | 20260505 05:03:27.828 | INFO | Downloaded 34GiB to GPU in 1.51667s (22.4175GiB/s) | 20260505 05:03:32.967 | INFO | Analyzing chunk frames = [549888,550400) | 20260505 05:03:32.967 | INFO | MemInfo: { | 20260505 05:03:32.967 | INFO | "gpu" : " 17 GB", | 20260505 05:03:32.967 | INFO | "huge" : "", | 20260505 05:03:32.967 | INFO | "huge_shared" : " 27 GB", | 20260505 05:03:32.967 | INFO | "malloc" : " 14 GB", | 20260505 05:03:32.967 | INFO | "pinned" : " 29 GB" | 20260505 05:03:32.967 | INFO | } | 20260505 05:03:32.968 | INFO | Perf report for the last 5.13956 seconds: | 20260505 05:03:32.968 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:03:32.968 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:32.968 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:32.968 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:32.968 | INFO | ANALYSIS : 57.3%, 3.0, 89.0ms, 0 | 20260505 05:03:32.968 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:32.968 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:32.968 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260505 05:03:32.968 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:32.968 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:32.968 | INFO | Uploaded 36GiB to GPU in 1.60439s (22.4384GiB/s) | 20260505 05:03:32.968 | INFO | Downloaded 34GiB to GPU in 1.52247s (22.3321GiB/s) | 20260505 05:03:38.107 | INFO | Analyzing chunk frames = [550400,550912) | 20260505 05:03:38.107 | INFO | MemInfo: { | 20260505 05:03:38.107 | INFO | "gpu" : " 17 GB", | 20260505 05:03:38.107 | INFO | "huge" : "", | 20260505 05:03:38.107 | INFO | "huge_shared" : " 27 GB", | 20260505 05:03:38.107 | INFO | "malloc" : " 14 GB", | 20260505 05:03:38.107 | INFO | "pinned" : " 29 GB" | 20260505 05:03:38.107 | INFO | } | 20260505 05:03:38.107 | INFO | Perf report for the last 5.13916 seconds: | 20260505 05:03:38.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:03:38.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:38.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:38.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:38.107 | INFO | ANALYSIS : 57.0%, 3.0, 88.3ms, 0 | 20260505 05:03:38.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:38.107 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:38.107 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 05:03:38.107 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:38.107 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:38.107 | INFO | Uploaded 36GiB to GPU in 1.60332s (22.4534GiB/s) | 20260505 05:03:38.107 | INFO | Downloaded 34GiB to GPU in 1.52078s (22.357GiB/s) | 20260505 05:03:43.246 | INFO | Analyzing chunk frames = [550912,551424) | 20260505 05:03:43.246 | INFO | MemInfo: { | 20260505 05:03:43.246 | INFO | "gpu" : " 17 GB", | 20260505 05:03:43.246 | INFO | "huge" : "", | 20260505 05:03:43.246 | INFO | "huge_shared" : " 27 GB", | 20260505 05:03:43.246 | INFO | "malloc" : " 14 GB", | 20260505 05:03:43.246 | INFO | "pinned" : " 29 GB" | 20260505 05:03:43.246 | INFO | } | 20260505 05:03:43.246 | INFO | Perf report for the last 5.13935 seconds: | 20260505 05:03:43.246 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:03:43.246 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:43.246 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:43.246 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:43.246 | INFO | ANALYSIS : 57.0%, 3.0, 88.8ms, 0 | 20260505 05:03:43.246 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:43.246 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:43.246 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260505 05:03:43.246 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:43.246 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:43.246 | INFO | Uploaded 36GiB to GPU in 1.60084s (22.4882GiB/s) | 20260505 05:03:43.246 | INFO | Downloaded 34GiB to GPU in 1.51699s (22.4128GiB/s) | 20260505 05:03:48.385 | INFO | Analyzing chunk frames = [551424,551936) | 20260505 05:03:48.385 | INFO | MemInfo: { | 20260505 05:03:48.385 | INFO | "gpu" : " 17 GB", | 20260505 05:03:48.385 | INFO | "huge" : "", | 20260505 05:03:48.385 | INFO | "huge_shared" : " 27 GB", | 20260505 05:03:48.385 | INFO | "malloc" : " 14 GB", | 20260505 05:03:48.385 | INFO | "pinned" : " 29 GB" | 20260505 05:03:48.385 | INFO | } | 20260505 05:03:48.385 | INFO | Perf report for the last 5.13937 seconds: | 20260505 05:03:48.385 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:03:48.385 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:48.385 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:48.386 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:48.386 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260505 05:03:48.386 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:48.386 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:48.386 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260505 05:03:48.386 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:48.386 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:48.386 | INFO | Uploaded 36GiB to GPU in 1.60221s (22.469GiB/s) | 20260505 05:03:48.386 | INFO | Downloaded 34GiB to GPU in 1.51872s (22.3872GiB/s) | 20260505 05:03:53.525 | INFO | Analyzing chunk frames = [551936,552448) | 20260505 05:03:53.525 | INFO | MemInfo: { | 20260505 05:03:53.525 | INFO | "gpu" : " 17 GB", | 20260505 05:03:53.525 | INFO | "huge" : "", | 20260505 05:03:53.525 | INFO | "huge_shared" : " 27 GB", | 20260505 05:03:53.525 | INFO | "malloc" : " 14 GB", | 20260505 05:03:53.525 | INFO | "pinned" : " 29 GB" | 20260505 05:03:53.525 | INFO | } | 20260505 05:03:53.525 | INFO | Perf report for the last 5.13941 seconds: | 20260505 05:03:53.525 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:03:53.525 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:53.525 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:53.525 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:53.525 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260505 05:03:53.525 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:53.525 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:53.525 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260505 05:03:53.525 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:53.525 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:53.525 | INFO | Uploaded 36GiB to GPU in 1.60178s (22.475GiB/s) | 20260505 05:03:53.525 | INFO | Downloaded 34GiB to GPU in 1.5186s (22.389GiB/s) | 20260505 05:03:58.664 | INFO | Analyzing chunk frames = [552448,552960) | 20260505 05:03:58.664 | INFO | MemInfo: { | 20260505 05:03:58.664 | INFO | "gpu" : " 17 GB", | 20260505 05:03:58.664 | INFO | "huge" : "", | 20260505 05:03:58.664 | INFO | "huge_shared" : " 27 GB", | 20260505 05:03:58.664 | INFO | "malloc" : " 14 GB", | 20260505 05:03:58.664 | INFO | "pinned" : " 29 GB" | 20260505 05:03:58.664 | INFO | } | 20260505 05:03:58.664 | INFO | Perf report for the last 5.13931 seconds: | 20260505 05:03:58.664 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:03:58.664 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:58.664 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:58.664 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:58.664 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260505 05:03:58.664 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:58.664 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:58.664 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 05:03:58.664 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:58.664 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:03:58.665 | INFO | Uploaded 36GiB to GPU in 1.60203s (22.4715GiB/s) | 20260505 05:03:58.665 | INFO | Downloaded 34GiB to GPU in 1.51983s (22.3709GiB/s) | 20260505 05:03:58.761 | INFO | Starting a round of Baz IO | 20260505 05:04:01.706 | INFO | Populating RTMetrics report | 20260505 05:04:01.706 | INFO | RTMetrics report populated | 20260505 05:04:01.706 | INFO | Opening tmp RTMetrics json file | 20260505 05:04:01.711 | INFO | Serializing RTMetrics to json | 20260505 05:04:01.714 | INFO | Serialization done, swapping tmp file now... | 20260505 05:04:01.715 | INFO | RTMetrics done | 20260505 05:04:01.804 | INFO | Finished a round of Baz IO: Wrote 2.146 GiB in 3.043 seconds | 20260505 05:04:01.804 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.84% / 0.10% / 58.06% | 20260505 05:04:01.804 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.68943% / 0.000170867% / 3.97364e-05% | 20260505 05:04:01.804 | INFO | Baz IO overall: | 20260505 05:04:01.804 | INFO | BytesWritten : 244338121616 | 20260505 05:04:01.804 | INFO | NumEvents : 114218093519 | 20260505 05:04:03.803 | INFO | Analyzing chunk frames = [552960,553472) | 20260505 05:04:03.803 | INFO | MemInfo: { | 20260505 05:04:03.803 | INFO | "gpu" : " 17 GB", | 20260505 05:04:03.803 | INFO | "huge" : "", | 20260505 05:04:03.804 | INFO | "huge_shared" : " 27 GB", | 20260505 05:04:03.804 | INFO | "malloc" : " 14 GB", | 20260505 05:04:03.804 | INFO | "pinned" : " 29 GB" | 20260505 05:04:03.804 | INFO | } | 20260505 05:04:03.804 | INFO | Perf report for the last 5.13939 seconds: | 20260505 05:04:03.804 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:04:03.804 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:03.804 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:03.804 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:03.804 | INFO | ANALYSIS : 58.1%, 3.0, 90.1ms, 0 | 20260505 05:04:03.804 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:03.804 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260505 05:04:03.804 | INFO | PRE_HQ : 57.5%, 2.2, 65.0ms, 0 | 20260505 05:04:03.804 | INFO | BAZWRITER : 58.8%, 4.2, 127.8ms, 0 | 20260505 05:04:03.804 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 05:04:03.804 | INFO | Uploaded 36GiB to GPU in 1.64114s (21.936GiB/s) | 20260505 05:04:03.804 | INFO | Downloaded 38GiB to GPU in 1.70561s (22.2794GiB/s) | 20260505 05:04:08.943 | INFO | Analyzing chunk frames = [553472,553984) | 20260505 05:04:08.943 | INFO | MemInfo: { | 20260505 05:04:08.943 | INFO | "gpu" : " 17 GB", | 20260505 05:04:08.943 | INFO | "huge" : "", | 20260505 05:04:08.943 | INFO | "huge_shared" : " 27 GB", | 20260505 05:04:08.943 | INFO | "malloc" : " 14 GB", | 20260505 05:04:08.943 | INFO | "pinned" : " 29 GB" | 20260505 05:04:08.943 | INFO | } | 20260505 05:04:08.943 | INFO | Perf report for the last 5.13922 seconds: | 20260505 05:04:08.943 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:04:08.943 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:08.943 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:08.943 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:08.943 | INFO | ANALYSIS : 57.9%, 3.0, 90.3ms, 0 | 20260505 05:04:08.943 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:08.943 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:08.943 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 05:04:08.943 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:08.943 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:08.943 | INFO | Uploaded 36GiB to GPU in 1.59939s (22.5085GiB/s) | 20260505 05:04:08.943 | INFO | Downloaded 34GiB to GPU in 1.51629s (22.4232GiB/s) | 20260505 05:04:14.082 | INFO | Analyzing chunk frames = [553984,554496) | 20260505 05:04:14.082 | INFO | MemInfo: { | 20260505 05:04:14.082 | INFO | "gpu" : " 17 GB", | 20260505 05:04:14.082 | INFO | "huge" : "", | 20260505 05:04:14.082 | INFO | "huge_shared" : " 27 GB", | 20260505 05:04:14.082 | INFO | "malloc" : " 14 GB", | 20260505 05:04:14.082 | INFO | "pinned" : " 29 GB" | 20260505 05:04:14.083 | INFO | } | 20260505 05:04:14.083 | INFO | Perf report for the last 5.13968 seconds: | 20260505 05:04:14.083 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:04:14.083 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:14.083 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:14.083 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:14.083 | INFO | ANALYSIS : 57.2%, 3.0, 88.9ms, 0 | 20260505 05:04:14.083 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:14.083 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:14.083 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 05:04:14.083 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:14.083 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:14.083 | INFO | Uploaded 36GiB to GPU in 1.60372s (22.4478GiB/s) | 20260505 05:04:14.083 | INFO | Downloaded 34GiB to GPU in 1.5207s (22.3582GiB/s) | 20260505 05:04:19.221 | INFO | Analyzing chunk frames = [554496,555008) | 20260505 05:04:19.221 | INFO | MemInfo: { | 20260505 05:04:19.221 | INFO | "gpu" : " 17 GB", | 20260505 05:04:19.222 | INFO | "huge" : "", | 20260505 05:04:19.222 | INFO | "huge_shared" : " 27 GB", | 20260505 05:04:19.222 | INFO | "malloc" : " 14 GB", | 20260505 05:04:19.222 | INFO | "pinned" : " 29 GB" | 20260505 05:04:19.222 | INFO | } | 20260505 05:04:19.222 | INFO | Perf report for the last 5.13903 seconds: | 20260505 05:04:19.222 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:04:19.222 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:19.222 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:19.222 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:19.222 | INFO | ANALYSIS : 57.0%, 3.0, 88.4ms, 0 | 20260505 05:04:19.222 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:19.222 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:19.222 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 05:04:19.222 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:19.222 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:19.222 | INFO | Uploaded 36GiB to GPU in 1.60288s (22.4596GiB/s) | 20260505 05:04:19.222 | INFO | Downloaded 34GiB to GPU in 1.51975s (22.3721GiB/s) | 20260505 05:04:24.361 | INFO | Analyzing chunk frames = [555008,555520) | 20260505 05:04:24.361 | INFO | MemInfo: { | 20260505 05:04:24.361 | INFO | "gpu" : " 17 GB", | 20260505 05:04:24.361 | INFO | "huge" : "", | 20260505 05:04:24.361 | INFO | "huge_shared" : " 27 GB", | 20260505 05:04:24.361 | INFO | "malloc" : " 14 GB", | 20260505 05:04:24.361 | INFO | "pinned" : " 29 GB" | 20260505 05:04:24.361 | INFO | } | 20260505 05:04:24.361 | INFO | Perf report for the last 5.13925 seconds: | 20260505 05:04:24.361 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:04:24.361 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:24.361 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:24.361 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:24.361 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260505 05:04:24.361 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:24.361 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:24.361 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260505 05:04:24.361 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:24.361 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:24.361 | INFO | Uploaded 36GiB to GPU in 1.60155s (22.4783GiB/s) | 20260505 05:04:24.361 | INFO | Downloaded 34GiB to GPU in 1.51769s (22.4025GiB/s) | 20260505 05:04:29.500 | INFO | Analyzing chunk frames = [555520,556032) | 20260505 05:04:29.500 | INFO | MemInfo: { | 20260505 05:04:29.500 | INFO | "gpu" : " 17 GB", | 20260505 05:04:29.500 | INFO | "huge" : "", | 20260505 05:04:29.501 | INFO | "huge_shared" : " 27 GB", | 20260505 05:04:29.501 | INFO | "malloc" : " 14 GB", | 20260505 05:04:29.501 | INFO | "pinned" : " 29 GB" | 20260505 05:04:29.501 | INFO | } | 20260505 05:04:29.501 | INFO | Perf report for the last 5.1397 seconds: | 20260505 05:04:29.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:04:29.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:29.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:29.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:29.501 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260505 05:04:29.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:29.501 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:29.501 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 05:04:29.501 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:29.501 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:29.501 | INFO | Uploaded 36GiB to GPU in 1.60156s (22.4781GiB/s) | 20260505 05:04:29.501 | INFO | Downloaded 34GiB to GPU in 1.51815s (22.3956GiB/s) | 20260505 05:04:34.639 | INFO | Analyzing chunk frames = [556032,556544) | 20260505 05:04:34.639 | INFO | MemInfo: { | 20260505 05:04:34.639 | INFO | "gpu" : " 17 GB", | 20260505 05:04:34.639 | INFO | "huge" : "", | 20260505 05:04:34.639 | INFO | "huge_shared" : " 27 GB", | 20260505 05:04:34.640 | INFO | "malloc" : " 14 GB", | 20260505 05:04:34.640 | INFO | "pinned" : " 29 GB" | 20260505 05:04:34.640 | INFO | } | 20260505 05:04:34.640 | INFO | Perf report for the last 5.13898 seconds: | 20260505 05:04:34.640 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:04:34.640 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:34.640 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:34.640 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:34.640 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260505 05:04:34.640 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:34.640 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:34.640 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 05:04:34.640 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:34.640 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:34.640 | INFO | Uploaded 36GiB to GPU in 1.60218s (22.4693GiB/s) | 20260505 05:04:34.640 | INFO | Downloaded 34GiB to GPU in 1.51927s (22.3792GiB/s) | 20260505 05:04:39.779 | INFO | Analyzing chunk frames = [556544,557056) | 20260505 05:04:39.779 | INFO | MemInfo: { | 20260505 05:04:39.779 | INFO | "gpu" : " 17 GB", | 20260505 05:04:39.779 | INFO | "huge" : "", | 20260505 05:04:39.779 | INFO | "huge_shared" : " 27 GB", | 20260505 05:04:39.779 | INFO | "malloc" : " 14 GB", | 20260505 05:04:39.779 | INFO | "pinned" : " 29 GB" | 20260505 05:04:39.779 | INFO | } | 20260505 05:04:39.779 | INFO | Perf report for the last 5.1393 seconds: | 20260505 05:04:39.779 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:04:39.779 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:39.779 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:39.779 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:39.779 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260505 05:04:39.779 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:39.779 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:39.779 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260505 05:04:39.779 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:39.779 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:39.779 | INFO | Uploaded 36GiB to GPU in 1.60202s (22.4716GiB/s) | 20260505 05:04:39.779 | INFO | Downloaded 34GiB to GPU in 1.51953s (22.3753GiB/s) | 20260505 05:04:39.779 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 05:04:39.780 | INFO | Total mem usage: 15.1 GiB | 20260505 05:04:39.780 | INFO | All Time High Water Marks sum: 15.7 GiB (Savings: 4.09%) | 20260505 05:04:39.780 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -3.96%) | 20260505 05:04:39.780 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 05:04:39.780 | INFO | High Water Breakdown: | 20260505 05:04:39.780 | INFO | 59.6% : 8.64 GiB ( 95.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 05:04:39.780 | INFO | 40.2% : 5.82 GiB ( 87.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 05:04:39.780 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 05:04:39.780 | INFO | ------------------------------------------ | 20260505 05:04:39.780 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:04:39.780 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 05:04:39.780 | INFO | Total mem usage: 26.5 GiB | 20260505 05:04:39.780 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 05:04:39.780 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 05:04:39.780 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 05:04:39.780 | INFO | High Water Breakdown: | 20260505 05:04:39.780 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:04:39.780 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:04:39.780 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:04:39.780 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:04:39.780 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:04:39.780 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:04:39.780 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 05:04:39.780 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 05:04:39.780 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:04:39.780 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 05:04:39.780 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 05:04:39.780 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 05:04:39.780 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 05:04:39.780 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 05:04:39.780 | INFO | ------------------------------------------ | 20260505 05:04:39.780 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:04:39.780 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 05:04:39.780 | INFO | Total mem usage: 25.6 GiB | 20260505 05:04:39.780 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 05:04:39.780 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 05:04:39.780 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 05:04:39.780 | INFO | High Water Breakdown: | 20260505 05:04:39.780 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 05:04:39.780 | INFO | ------------------------------------------ | 20260505 05:04:39.780 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:04:39.780 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 05:04:39.780 | INFO | Total mem usage: 15.7 GiB | 20260505 05:04:39.780 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 05:04:39.780 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 05:04:39.780 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260505 05:04:39.780 | INFO | High Water Breakdown: | 20260505 05:04:39.780 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 05:04:39.780 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 05:04:39.780 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 05:04:39.780 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 05:04:39.780 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 05:04:39.780 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 05:04:39.780 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 05:04:39.780 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 05:04:39.780 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 05:04:39.780 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:04:39.780 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 05:04:39.780 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:04:39.780 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 05:04:39.780 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 05:04:39.780 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 05:04:39.780 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 05:04:39.780 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:04:39.780 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 05:04:39.780 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:04:39.780 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:04:39.780 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:04:39.780 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 05:04:39.780 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 05:04:39.780 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:04:39.780 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 05:04:39.780 | INFO | ------------------------------------------ | 20260505 05:04:39.780 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:04:39.780 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 05:04:39.780 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:04:39.780 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:04:39.780 | INFO | PerfReport: Full Execution -- 3.04e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260505 05:04:39.780 | INFO | PerfReport: 0.28%, AggregateStats -- 8.64e-02 / -nan, 4.44e-03 / 0.00e+00 | 20260505 05:04:39.780 | INFO | PerfReport: 10.85%, Baseline -- 3.30e+00 / -nan, 1.65e-02 / 0.00e+00 | 20260505 05:04:39.780 | INFO | PerfReport: 3.04%, Histogram -- 9.24e-01 / -nan, 5.21e-02 / 0.00e+00 | 20260505 05:04:39.780 | INFO | PerfReport: 16.04%, DME -- 3.90e+01 / -nan, 3.17e+00 / 0.00e+00 | 20260505 05:04:39.780 | INFO | PerfReport: 56.64%, FrameLabeling -- 1.72e+01 / -nan, 4.48e-02 / 0.00e+00 | 20260505 05:04:39.781 | INFO | PerfReport: 7.59%, PulseAccumulating -- 2.31e+00 / -nan, 1.52e-01 / 0.00e+00 | 20260505 05:04:39.781 | INFO | PerfReport: 3.93%, Metrics -- 1.20e+00 / -nan, 1.97e-01 / 0.00e+00 | 20260505 05:04:39.781 | INFO | PerfReport: 1.62%, PulseSerializing -- 4.92e-01 / -nan, 7.96e-02 / 0.00e+00 | 20260505 05:04:39.781 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 05:04:39.781 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:04:39.781 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:04:39.781 | INFO | PerfReport: Full Execution -- 9.12e+01 / -nan, 1.91e+01 / 0.00e+00 | 20260505 05:04:39.781 | INFO | PerfReport: 18.35%, Upload -- 1.67e+01 / 0.00e+00, 3.75e-01 / 0.00e+00 | 20260505 05:04:39.781 | INFO | PerfReport: 17.61%, Download -- 1.61e+01 / 0.00e+00, 7.05e-01 / 0.00e+00 | 20260505 05:04:39.781 | INFO | PerfReport: 33.31%, Compute -- 3.04e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260505 05:04:39.781 | INFO | PerfReport: 0.55%, LockUpload -- 5.04e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260505 05:04:39.781 | INFO | PerfReport: 0.00%, LockDownload -- 3.94e-04 / 0.00e+00, 1.37e-04 / 0.00e+00 | 20260505 05:04:39.781 | INFO | PerfReport: 30.17%, LockCompute -- 2.75e+01 / 0.00e+00, 1.72e+01 / 0.00e+00 | 20260505 05:04:39.781 | INFO | PerfReport: 0.02%, Misc -- 1.43e-02 / 0.00e+00, 3.28e-02 / 0.00e+00 | 20260505 05:04:39.876 | INFO | Starting a round of Baz IO | 20260505 05:04:42.822 | INFO | Populating RTMetrics report | 20260505 05:04:42.822 | INFO | RTMetrics report populated | 20260505 05:04:42.822 | INFO | Opening tmp RTMetrics json file | 20260505 05:04:42.823 | INFO | Serializing RTMetrics to json | 20260505 05:04:42.824 | INFO | Serialization done, swapping tmp file now... | 20260505 05:04:42.824 | INFO | RTMetrics done | 20260505 05:04:42.923 | INFO | Finished a round of Baz IO: Wrote 2.15 GiB in 3.047 seconds | 20260505 05:04:42.923 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.74% / 0.10% / 58.16% | 20260505 05:04:42.923 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.64809% / 0.000258287% / 5.5631e-05% | 20260505 05:04:42.923 | INFO | Baz IO overall: | 20260505 05:04:42.923 | INFO | BytesWritten : 246646392119 | 20260505 05:04:42.923 | INFO | NumEvents : 115315884640 | 20260505 05:04:44.918 | INFO | Analyzing chunk frames = [557056,557568) | 20260505 05:04:44.918 | INFO | MemInfo: { | 20260505 05:04:44.918 | INFO | "gpu" : " 17 GB", | 20260505 05:04:44.918 | INFO | "huge" : "", | 20260505 05:04:44.918 | INFO | "huge_shared" : " 27 GB", | 20260505 05:04:44.918 | INFO | "malloc" : " 14 GB", | 20260505 05:04:44.918 | INFO | "pinned" : " 29 GB" | 20260505 05:04:44.918 | INFO | } | 20260505 05:04:44.918 | INFO | Perf report for the last 5.13942 seconds: | 20260505 05:04:44.918 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:04:44.918 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:44.918 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:44.918 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:44.918 | INFO | ANALYSIS : 58.1%, 3.0, 90.4ms, 0 | 20260505 05:04:44.918 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:44.918 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260505 05:04:44.918 | INFO | PRE_HQ : 57.4%, 2.2, 64.4ms, 0 | 20260505 05:04:44.918 | INFO | BAZWRITER : 58.8%, 4.2, 125.8ms, 0 | 20260505 05:04:44.918 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 05:04:44.919 | INFO | Uploaded 36GiB to GPU in 1.6425s (21.9178GiB/s) | 20260505 05:04:44.919 | INFO | Downloaded 38GiB to GPU in 1.7074s (22.2561GiB/s) | 20260505 05:04:50.057 | INFO | Analyzing chunk frames = [557568,558080) | 20260505 05:04:50.057 | INFO | MemInfo: { | 20260505 05:04:50.057 | INFO | "gpu" : " 17 GB", | 20260505 05:04:50.057 | INFO | "huge" : "", | 20260505 05:04:50.058 | INFO | "huge_shared" : " 27 GB", | 20260505 05:04:50.058 | INFO | "malloc" : " 14 GB", | 20260505 05:04:50.058 | INFO | "pinned" : " 29 GB" | 20260505 05:04:50.058 | INFO | } | 20260505 05:04:50.058 | INFO | Perf report for the last 5.13925 seconds: | 20260505 05:04:50.058 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:04:50.058 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:50.058 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:50.058 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:50.058 | INFO | ANALYSIS : 57.9%, 3.0, 90.3ms, 0 | 20260505 05:04:50.058 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:50.058 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:50.058 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 05:04:50.058 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:50.058 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:50.058 | INFO | Uploaded 36GiB to GPU in 1.59932s (22.5096GiB/s) | 20260505 05:04:50.058 | INFO | Downloaded 34GiB to GPU in 1.5163s (22.4229GiB/s) | 20260505 05:04:55.197 | INFO | Analyzing chunk frames = [558080,558592) | 20260505 05:04:55.197 | INFO | MemInfo: { | 20260505 05:04:55.197 | INFO | "gpu" : " 17 GB", | 20260505 05:04:55.197 | INFO | "huge" : "", | 20260505 05:04:55.197 | INFO | "huge_shared" : " 27 GB", | 20260505 05:04:55.197 | INFO | "malloc" : " 14 GB", | 20260505 05:04:55.197 | INFO | "pinned" : " 29 GB" | 20260505 05:04:55.197 | INFO | } | 20260505 05:04:55.197 | INFO | Perf report for the last 5.13937 seconds: | 20260505 05:04:55.197 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:04:55.197 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:55.197 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:55.197 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:55.197 | INFO | ANALYSIS : 57.2%, 3.0, 89.0ms, 0 | 20260505 05:04:55.197 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:55.197 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:55.197 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 05:04:55.197 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:55.197 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:04:55.197 | INFO | Uploaded 36GiB to GPU in 1.60415s (22.4418GiB/s) | 20260505 05:04:55.197 | INFO | Downloaded 34GiB to GPU in 1.52148s (22.3467GiB/s) | 20260505 05:05:00.336 | INFO | Analyzing chunk frames = [558592,559104) | 20260505 05:05:00.336 | INFO | MemInfo: { | 20260505 05:05:00.336 | INFO | "gpu" : " 17 GB", | 20260505 05:05:00.336 | INFO | "huge" : "", | 20260505 05:05:00.336 | INFO | "huge_shared" : " 27 GB", | 20260505 05:05:00.336 | INFO | "malloc" : " 14 GB", | 20260505 05:05:00.336 | INFO | "pinned" : " 29 GB" | 20260505 05:05:00.336 | INFO | } | 20260505 05:05:00.336 | INFO | Perf report for the last 5.13926 seconds: | 20260505 05:05:00.336 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:05:00.336 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:00.336 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:00.336 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:00.336 | INFO | ANALYSIS : 57.0%, 3.0, 88.4ms, 0 | 20260505 05:05:00.336 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:00.336 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:00.336 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260505 05:05:00.336 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:00.337 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:00.337 | INFO | Uploaded 36GiB to GPU in 1.60312s (22.4562GiB/s) | 20260505 05:05:00.337 | INFO | Downloaded 34GiB to GPU in 1.51971s (22.3727GiB/s) | 20260505 05:05:05.475 | INFO | Analyzing chunk frames = [559104,559616) | 20260505 05:05:05.475 | INFO | MemInfo: { | 20260505 05:05:05.475 | INFO | "gpu" : " 17 GB", | 20260505 05:05:05.476 | INFO | "huge" : "", | 20260505 05:05:05.476 | INFO | "huge_shared" : " 27 GB", | 20260505 05:05:05.476 | INFO | "malloc" : " 14 GB", | 20260505 05:05:05.476 | INFO | "pinned" : " 29 GB" | 20260505 05:05:05.476 | INFO | } | 20260505 05:05:05.476 | INFO | Perf report for the last 5.13934 seconds: | 20260505 05:05:05.476 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:05:05.476 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:05.476 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:05.476 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:05.476 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260505 05:05:05.476 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:05.476 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:05.476 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260505 05:05:05.476 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:05.476 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:05.476 | INFO | Uploaded 36GiB to GPU in 1.60118s (22.4834GiB/s) | 20260505 05:05:05.476 | INFO | Downloaded 34GiB to GPU in 1.51821s (22.3948GiB/s) | 20260505 05:05:10.615 | INFO | Analyzing chunk frames = [559616,560128) | 20260505 05:05:10.615 | INFO | MemInfo: { | 20260505 05:05:10.615 | INFO | "gpu" : " 17 GB", | 20260505 05:05:10.615 | INFO | "huge" : "", | 20260505 05:05:10.615 | INFO | "huge_shared" : " 27 GB", | 20260505 05:05:10.615 | INFO | "malloc" : " 14 GB", | 20260505 05:05:10.615 | INFO | "pinned" : " 29 GB" | 20260505 05:05:10.615 | INFO | } | 20260505 05:05:10.615 | INFO | Perf report for the last 5.13951 seconds: | 20260505 05:05:10.615 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:05:10.615 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:10.615 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:10.615 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:10.615 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260505 05:05:10.615 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:10.615 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:10.615 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 05:05:10.615 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:10.615 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:10.615 | INFO | Uploaded 36GiB to GPU in 1.60053s (22.4926GiB/s) | 20260505 05:05:10.615 | INFO | Downloaded 34GiB to GPU in 1.51785s (22.4GiB/s) | 20260505 05:05:15.754 | INFO | Analyzing chunk frames = [560128,560640) | 20260505 05:05:15.754 | INFO | MemInfo: { | 20260505 05:05:15.754 | INFO | "gpu" : " 17 GB", | 20260505 05:05:15.754 | INFO | "huge" : "", | 20260505 05:05:15.754 | INFO | "huge_shared" : " 27 GB", | 20260505 05:05:15.754 | INFO | "malloc" : " 14 GB", | 20260505 05:05:15.754 | INFO | "pinned" : " 29 GB" | 20260505 05:05:15.754 | INFO | } | 20260505 05:05:15.755 | INFO | Perf report for the last 5.13937 seconds: | 20260505 05:05:15.755 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:05:15.755 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:15.755 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:15.755 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:15.755 | INFO | ANALYSIS : 57.3%, 3.0, 89.5ms, 0 | 20260505 05:05:15.755 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:15.755 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:15.755 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 05:05:15.755 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:15.755 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:15.755 | INFO | Uploaded 36GiB to GPU in 1.60174s (22.4755GiB/s) | 20260505 05:05:15.755 | INFO | Downloaded 34GiB to GPU in 1.52031s (22.3639GiB/s) | 20260505 05:05:20.894 | INFO | Analyzing chunk frames = [560640,561152) | 20260505 05:05:20.894 | INFO | MemInfo: { | 20260505 05:05:20.894 | INFO | "gpu" : " 17 GB", | 20260505 05:05:20.894 | INFO | "huge" : "", | 20260505 05:05:20.894 | INFO | "huge_shared" : " 27 GB", | 20260505 05:05:20.894 | INFO | "malloc" : " 14 GB", | 20260505 05:05:20.894 | INFO | "pinned" : " 29 GB" | 20260505 05:05:20.894 | INFO | } | 20260505 05:05:20.894 | INFO | Perf report for the last 5.1393 seconds: | 20260505 05:05:20.894 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:05:20.894 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:20.894 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:20.894 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:20.894 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260505 05:05:20.894 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:20.894 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:20.894 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:05:20.894 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:20.894 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:20.894 | INFO | Uploaded 36GiB to GPU in 1.60175s (22.4754GiB/s) | 20260505 05:05:20.894 | INFO | Downloaded 34GiB to GPU in 1.51899s (22.3833GiB/s) | 20260505 05:05:20.991 | INFO | Starting a round of Baz IO | 20260505 05:05:23.936 | INFO | Populating RTMetrics report | 20260505 05:05:23.936 | INFO | RTMetrics report populated | 20260505 05:05:23.936 | INFO | Opening tmp RTMetrics json file | 20260505 05:05:23.937 | INFO | Serializing RTMetrics to json | 20260505 05:05:23.940 | INFO | Serialization done, swapping tmp file now... | 20260505 05:05:23.942 | INFO | RTMetrics done | 20260505 05:05:24.034 | INFO | Finished a round of Baz IO: Wrote 2.152 GiB in 3.043 seconds | 20260505 05:05:24.034 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.64% / 0.10% / 58.26% | 20260505 05:05:24.034 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.60397% / 0.000178814% / 4.37101e-05% | 20260505 05:05:24.034 | INFO | Baz IO overall: | 20260505 05:05:24.034 | INFO | BytesWritten : 248956824885 | 20260505 05:05:24.034 | INFO | NumEvents : 116414615020 | 20260505 05:05:26.033 | INFO | Analyzing chunk frames = [561152,561664) | 20260505 05:05:26.033 | INFO | MemInfo: { | 20260505 05:05:26.033 | INFO | "gpu" : " 17 GB", | 20260505 05:05:26.033 | INFO | "huge" : "", | 20260505 05:05:26.033 | INFO | "huge_shared" : " 27 GB", | 20260505 05:05:26.033 | INFO | "malloc" : " 14 GB", | 20260505 05:05:26.033 | INFO | "pinned" : " 29 GB" | 20260505 05:05:26.033 | INFO | } | 20260505 05:05:26.033 | INFO | Perf report for the last 5.13935 seconds: | 20260505 05:05:26.033 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:05:26.033 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:26.033 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:26.033 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:26.033 | INFO | ANALYSIS : 58.1%, 3.0, 90.3ms, 0 | 20260505 05:05:26.033 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:26.033 | INFO | RT_METRICS : 45.5%, 1.0, 23.9ms, 0 | 20260505 05:05:26.033 | INFO | PRE_HQ : 57.4%, 2.2, 64.4ms, 0 | 20260505 05:05:26.033 | INFO | BAZWRITER : 58.8%, 4.2, 126.0ms, 0 | 20260505 05:05:26.033 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 05:05:26.033 | INFO | Uploaded 36GiB to GPU in 1.64012s (21.9496GiB/s) | 20260505 05:05:26.034 | INFO | Downloaded 38GiB to GPU in 1.70491s (22.2885GiB/s) | 20260505 05:05:31.172 | INFO | Analyzing chunk frames = [561664,562176) | 20260505 05:05:31.172 | INFO | MemInfo: { | 20260505 05:05:31.172 | INFO | "gpu" : " 17 GB", | 20260505 05:05:31.172 | INFO | "huge" : "", | 20260505 05:05:31.172 | INFO | "huge_shared" : " 27 GB", | 20260505 05:05:31.172 | INFO | "malloc" : " 14 GB", | 20260505 05:05:31.172 | INFO | "pinned" : " 29 GB" | 20260505 05:05:31.172 | INFO | } | 20260505 05:05:31.172 | INFO | Perf report for the last 5.1392 seconds: | 20260505 05:05:31.172 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:05:31.172 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:31.172 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:31.172 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:31.172 | INFO | ANALYSIS : 57.9%, 3.0, 90.3ms, 0 | 20260505 05:05:31.172 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:31.172 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:31.172 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 05:05:31.172 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:31.173 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:31.173 | INFO | Uploaded 36GiB to GPU in 1.60022s (22.497GiB/s) | 20260505 05:05:31.173 | INFO | Downloaded 34GiB to GPU in 1.51627s (22.4235GiB/s) | 20260505 05:05:36.312 | INFO | Analyzing chunk frames = [562176,562688) | 20260505 05:05:36.312 | INFO | MemInfo: { | 20260505 05:05:36.312 | INFO | "gpu" : " 17 GB", | 20260505 05:05:36.312 | INFO | "huge" : "", | 20260505 05:05:36.312 | INFO | "huge_shared" : " 27 GB", | 20260505 05:05:36.312 | INFO | "malloc" : " 14 GB", | 20260505 05:05:36.312 | INFO | "pinned" : " 29 GB" | 20260505 05:05:36.312 | INFO | } | 20260505 05:05:36.312 | INFO | Perf report for the last 5.13933 seconds: | 20260505 05:05:36.312 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:05:36.312 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:36.312 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:36.312 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:36.312 | INFO | ANALYSIS : 57.2%, 3.0, 89.0ms, 0 | 20260505 05:05:36.312 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:36.312 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:36.312 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260505 05:05:36.312 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:36.312 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:36.312 | INFO | Uploaded 36GiB to GPU in 1.60363s (22.4491GiB/s) | 20260505 05:05:36.312 | INFO | Downloaded 34GiB to GPU in 1.5217s (22.3434GiB/s) | 20260505 05:05:41.451 | INFO | Analyzing chunk frames = [562688,563200) | 20260505 05:05:41.451 | INFO | MemInfo: { | 20260505 05:05:41.451 | INFO | "gpu" : " 17 GB", | 20260505 05:05:41.451 | INFO | "huge" : "", | 20260505 05:05:41.451 | INFO | "huge_shared" : " 27 GB", | 20260505 05:05:41.451 | INFO | "malloc" : " 14 GB", | 20260505 05:05:41.451 | INFO | "pinned" : " 29 GB" | 20260505 05:05:41.451 | INFO | } | 20260505 05:05:41.451 | INFO | Perf report for the last 5.13926 seconds: | 20260505 05:05:41.451 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:05:41.451 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:41.451 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:41.451 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:41.451 | INFO | ANALYSIS : 57.0%, 3.0, 88.4ms, 0 | 20260505 05:05:41.451 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:41.451 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:41.451 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260505 05:05:41.451 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:41.451 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:41.451 | INFO | Uploaded 36GiB to GPU in 1.60293s (22.4589GiB/s) | 20260505 05:05:41.451 | INFO | Downloaded 34GiB to GPU in 1.51979s (22.3715GiB/s) | 20260505 05:05:46.590 | INFO | Analyzing chunk frames = [563200,563712) | 20260505 05:05:46.590 | INFO | MemInfo: { | 20260505 05:05:46.590 | INFO | "gpu" : " 17 GB", | 20260505 05:05:46.590 | INFO | "huge" : "", | 20260505 05:05:46.590 | INFO | "huge_shared" : " 27 GB", | 20260505 05:05:46.590 | INFO | "malloc" : " 14 GB", | 20260505 05:05:46.590 | INFO | "pinned" : " 29 GB" | 20260505 05:05:46.590 | INFO | } | 20260505 05:05:46.590 | INFO | Perf report for the last 5.13936 seconds: | 20260505 05:05:46.590 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:05:46.590 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:46.590 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:46.590 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:46.590 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260505 05:05:46.590 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:46.590 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:46.590 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 05:05:46.590 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:46.591 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:46.591 | INFO | Uploaded 36GiB to GPU in 1.60124s (22.4825GiB/s) | 20260505 05:05:46.591 | INFO | Downloaded 34GiB to GPU in 1.51808s (22.3967GiB/s) | 20260505 05:05:51.730 | INFO | Analyzing chunk frames = [563712,564224) | 20260505 05:05:51.730 | INFO | MemInfo: { | 20260505 05:05:51.730 | INFO | "gpu" : " 17 GB", | 20260505 05:05:51.730 | INFO | "huge" : "", | 20260505 05:05:51.730 | INFO | "huge_shared" : " 27 GB", | 20260505 05:05:51.730 | INFO | "malloc" : " 14 GB", | 20260505 05:05:51.730 | INFO | "pinned" : " 29 GB" | 20260505 05:05:51.730 | INFO | } | 20260505 05:05:51.730 | INFO | Perf report for the last 5.1393 seconds: | 20260505 05:05:51.730 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:05:51.730 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:51.730 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:51.730 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:51.730 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260505 05:05:51.730 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:51.730 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:51.730 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 05:05:51.730 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:51.730 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:51.730 | INFO | Uploaded 36GiB to GPU in 1.60137s (22.4807GiB/s) | 20260505 05:05:51.730 | INFO | Downloaded 34GiB to GPU in 1.51893s (22.3842GiB/s) | 20260505 05:05:56.869 | INFO | Analyzing chunk frames = [564224,564736) | 20260505 05:05:56.869 | INFO | MemInfo: { | 20260505 05:05:56.869 | INFO | "gpu" : " 17 GB", | 20260505 05:05:56.869 | INFO | "huge" : "", | 20260505 05:05:56.869 | INFO | "huge_shared" : " 27 GB", | 20260505 05:05:56.869 | INFO | "malloc" : " 14 GB", | 20260505 05:05:56.869 | INFO | "pinned" : " 29 GB" | 20260505 05:05:56.869 | INFO | } | 20260505 05:05:56.869 | INFO | Perf report for the last 5.13938 seconds: | 20260505 05:05:56.869 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:05:56.869 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:56.869 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:56.869 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:56.869 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260505 05:05:56.869 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:56.869 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:56.869 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:05:56.869 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:56.869 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:05:56.870 | INFO | Uploaded 36GiB to GPU in 1.60171s (22.476GiB/s) | 20260505 05:05:56.870 | INFO | Downloaded 34GiB to GPU in 1.51856s (22.3896GiB/s) | 20260505 05:06:02.008 | INFO | Analyzing chunk frames = [564736,565248) | 20260505 05:06:02.008 | INFO | MemInfo: { | 20260505 05:06:02.008 | INFO | "gpu" : " 17 GB", | 20260505 05:06:02.008 | INFO | "huge" : "", | 20260505 05:06:02.008 | INFO | "huge_shared" : " 27 GB", | 20260505 05:06:02.008 | INFO | "malloc" : " 14 GB", | 20260505 05:06:02.008 | INFO | "pinned" : " 29 GB" | 20260505 05:06:02.008 | INFO | } | 20260505 05:06:02.008 | INFO | Perf report for the last 5.13938 seconds: | 20260505 05:06:02.008 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:06:02.008 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:02.008 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:02.008 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:02.009 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260505 05:06:02.009 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:02.009 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:02.009 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 05:06:02.009 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:02.009 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:02.009 | INFO | Uploaded 36GiB to GPU in 1.6018s (22.4747GiB/s) | 20260505 05:06:02.009 | INFO | Downloaded 34GiB to GPU in 1.51961s (22.3741GiB/s) | 20260505 05:06:02.009 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 05:06:02.009 | INFO | Total mem usage: 15.1 GiB | 20260505 05:06:02.009 | INFO | All Time High Water Marks sum: 15.7 GiB (Savings: 4.09%) | 20260505 05:06:02.009 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -4.44%) | 20260505 05:06:02.009 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 05:06:02.009 | INFO | High Water Breakdown: | 20260505 05:06:02.009 | INFO | 59.6% : 8.59 GiB ( 95.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 05:06:02.009 | INFO | 40.3% : 5.8 GiB ( 87.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 05:06:02.009 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 05:06:02.009 | INFO | ------------------------------------------ | 20260505 05:06:02.009 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:06:02.009 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 05:06:02.009 | INFO | Total mem usage: 26.5 GiB | 20260505 05:06:02.009 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 05:06:02.009 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 05:06:02.009 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 05:06:02.009 | INFO | High Water Breakdown: | 20260505 05:06:02.009 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:06:02.009 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:06:02.009 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:06:02.009 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:06:02.009 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:06:02.009 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:06:02.009 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 05:06:02.009 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 05:06:02.009 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:06:02.009 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 05:06:02.009 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 05:06:02.009 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 05:06:02.009 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 05:06:02.009 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 05:06:02.009 | INFO | ------------------------------------------ | 20260505 05:06:02.009 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:06:02.009 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 05:06:02.009 | INFO | Total mem usage: 25.6 GiB | 20260505 05:06:02.009 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 05:06:02.009 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 05:06:02.009 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 05:06:02.009 | INFO | High Water Breakdown: | 20260505 05:06:02.009 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 05:06:02.009 | INFO | ------------------------------------------ | 20260505 05:06:02.009 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:06:02.009 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 05:06:02.009 | INFO | Total mem usage: 15.7 GiB | 20260505 05:06:02.009 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 05:06:02.009 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 05:06:02.009 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260505 05:06:02.009 | INFO | High Water Breakdown: | 20260505 05:06:02.009 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 05:06:02.009 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 05:06:02.009 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 05:06:02.009 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 05:06:02.009 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 05:06:02.009 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 05:06:02.009 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 05:06:02.009 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 05:06:02.009 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 05:06:02.009 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:06:02.009 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 05:06:02.009 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:06:02.009 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 05:06:02.009 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 05:06:02.009 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 05:06:02.010 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 05:06:02.010 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:06:02.010 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 05:06:02.010 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:06:02.010 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:06:02.010 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:06:02.010 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 05:06:02.010 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 05:06:02.010 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:06:02.010 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 05:06:02.010 | INFO | ------------------------------------------ | 20260505 05:06:02.010 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:06:02.010 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 05:06:02.010 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:06:02.010 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:06:02.010 | INFO | PerfReport: Full Execution -- 3.04e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260505 05:06:02.010 | INFO | PerfReport: 0.28%, AggregateStats -- 8.64e-02 / -nan, 4.36e-03 / 0.00e+00 | 20260505 05:06:02.010 | INFO | PerfReport: 10.85%, Baseline -- 3.30e+00 / -nan, 1.76e-02 / 0.00e+00 | 20260505 05:06:02.010 | INFO | PerfReport: 3.04%, Histogram -- 9.24e-01 / -nan, 5.16e-02 / 0.00e+00 | 20260505 05:06:02.010 | INFO | PerfReport: 16.03%, DME -- 3.89e+01 / -nan, 3.25e+00 / 0.00e+00 | 20260505 05:06:02.010 | INFO | PerfReport: 56.66%, FrameLabeling -- 1.72e+01 / -nan, 4.57e-02 / 0.00e+00 | 20260505 05:06:02.010 | INFO | PerfReport: 7.59%, PulseAccumulating -- 2.30e+00 / -nan, 1.52e-01 / 0.00e+00 | 20260505 05:06:02.010 | INFO | PerfReport: 3.93%, Metrics -- 1.19e+00 / -nan, 1.98e-01 / 0.00e+00 | 20260505 05:06:02.010 | INFO | PerfReport: 1.62%, PulseSerializing -- 4.92e-01 / -nan, 7.94e-02 / 0.00e+00 | 20260505 05:06:02.010 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 05:06:02.010 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:06:02.010 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:06:02.010 | INFO | PerfReport: Full Execution -- 9.12e+01 / -nan, 1.90e+01 / 0.00e+00 | 20260505 05:06:02.010 | INFO | PerfReport: 18.34%, Upload -- 1.67e+01 / 0.00e+00, 3.74e-01 / 0.00e+00 | 20260505 05:06:02.010 | INFO | PerfReport: 17.61%, Download -- 1.61e+01 / 0.00e+00, 7.09e-01 / 0.00e+00 | 20260505 05:06:02.010 | INFO | PerfReport: 33.30%, Compute -- 3.04e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260505 05:06:02.010 | INFO | PerfReport: 0.55%, LockUpload -- 5.04e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260505 05:06:02.010 | INFO | PerfReport: 0.00%, LockDownload -- 3.92e-04 / 0.00e+00, 2.11e-04 / 0.00e+00 | 20260505 05:06:02.010 | INFO | PerfReport: 30.17%, LockCompute -- 2.75e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260505 05:06:02.010 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.52e-02 / 0.00e+00 | 20260505 05:06:02.106 | INFO | Starting a round of Baz IO | 20260505 05:06:05.053 | INFO | Populating RTMetrics report | 20260505 05:06:05.053 | INFO | RTMetrics report populated | 20260505 05:06:05.053 | INFO | Opening tmp RTMetrics json file | 20260505 05:06:05.054 | INFO | Serializing RTMetrics to json | 20260505 05:06:05.055 | INFO | Serialization done, swapping tmp file now... | 20260505 05:06:05.056 | INFO | RTMetrics done | 20260505 05:06:05.152 | INFO | Finished a round of Baz IO: Wrote 2.155 GiB in 3.046 seconds | 20260505 05:06:05.152 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.54% / 0.10% / 58.36% | 20260505 05:06:05.152 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.5596% / 0.000238419% / 4.37101e-05% | 20260505 05:06:05.152 | INFO | Baz IO overall: | 20260505 05:06:05.152 | INFO | BytesWritten : 251270883352 | 20260505 05:06:05.153 | INFO | NumEvents : 117514700202 | 20260505 05:06:07.148 | INFO | Analyzing chunk frames = [565248,565760) | 20260505 05:06:07.148 | INFO | MemInfo: { | 20260505 05:06:07.148 | INFO | "gpu" : " 17 GB", | 20260505 05:06:07.148 | INFO | "huge" : "", | 20260505 05:06:07.148 | INFO | "huge_shared" : " 27 GB", | 20260505 05:06:07.148 | INFO | "malloc" : " 14 GB", | 20260505 05:06:07.148 | INFO | "pinned" : " 29 GB" | 20260505 05:06:07.148 | INFO | } | 20260505 05:06:07.148 | INFO | Perf report for the last 5.1394 seconds: | 20260505 05:06:07.148 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:06:07.148 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:07.148 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:07.148 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:07.148 | INFO | ANALYSIS : 58.1%, 3.0, 90.4ms, 0 | 20260505 05:06:07.148 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:07.148 | INFO | RT_METRICS : 45.5%, 1.0, 23.9ms, 0 | 20260505 05:06:07.148 | INFO | PRE_HQ : 57.5%, 2.2, 64.6ms, 0 | 20260505 05:06:07.148 | INFO | BAZWRITER : 58.8%, 4.2, 126.4ms, 0 | 20260505 05:06:07.148 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 05:06:07.148 | INFO | Uploaded 36GiB to GPU in 1.63894s (21.9655GiB/s) | 20260505 05:06:07.148 | INFO | Downloaded 38GiB to GPU in 1.70413s (22.2988GiB/s) | 20260505 05:06:12.287 | INFO | Analyzing chunk frames = [565760,566272) | 20260505 05:06:12.287 | INFO | MemInfo: { | 20260505 05:06:12.287 | INFO | "gpu" : " 17 GB", | 20260505 05:06:12.287 | INFO | "huge" : "", | 20260505 05:06:12.287 | INFO | "huge_shared" : " 27 GB", | 20260505 05:06:12.287 | INFO | "malloc" : " 14 GB", | 20260505 05:06:12.287 | INFO | "pinned" : " 29 GB" | 20260505 05:06:12.287 | INFO | } | 20260505 05:06:12.287 | INFO | Perf report for the last 5.13935 seconds: | 20260505 05:06:12.287 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:06:12.287 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:12.287 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:12.287 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:12.287 | INFO | ANALYSIS : 57.8%, 3.0, 90.2ms, 0 | 20260505 05:06:12.287 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:12.287 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:12.287 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:06:12.287 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:12.287 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:12.287 | INFO | Uploaded 36GiB to GPU in 1.59988s (22.5016GiB/s) | 20260505 05:06:12.287 | INFO | Downloaded 34GiB to GPU in 1.51657s (22.419GiB/s) | 20260505 05:06:17.426 | INFO | Analyzing chunk frames = [566272,566784) | 20260505 05:06:17.426 | INFO | MemInfo: { | 20260505 05:06:17.426 | INFO | "gpu" : " 17 GB", | 20260505 05:06:17.426 | INFO | "huge" : "", | 20260505 05:06:17.426 | INFO | "huge_shared" : " 27 GB", | 20260505 05:06:17.427 | INFO | "malloc" : " 14 GB", | 20260505 05:06:17.427 | INFO | "pinned" : " 29 GB" | 20260505 05:06:17.427 | INFO | } | 20260505 05:06:17.427 | INFO | Perf report for the last 5.13935 seconds: | 20260505 05:06:17.427 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:06:17.427 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:17.427 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:17.427 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:17.427 | INFO | ANALYSIS : 57.3%, 3.0, 89.0ms, 0 | 20260505 05:06:17.427 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:17.427 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:17.427 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260505 05:06:17.427 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:17.427 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:17.427 | INFO | Uploaded 36GiB to GPU in 1.60466s (22.4347GiB/s) | 20260505 05:06:17.427 | INFO | Downloaded 34GiB to GPU in 1.52184s (22.3414GiB/s) | 20260505 05:06:22.566 | INFO | Analyzing chunk frames = [566784,567296) | 20260505 05:06:22.566 | INFO | MemInfo: { | 20260505 05:06:22.566 | INFO | "gpu" : " 17 GB", | 20260505 05:06:22.566 | INFO | "huge" : "", | 20260505 05:06:22.566 | INFO | "huge_shared" : " 27 GB", | 20260505 05:06:22.566 | INFO | "malloc" : " 14 GB", | 20260505 05:06:22.566 | INFO | "pinned" : " 29 GB" | 20260505 05:06:22.566 | INFO | } | 20260505 05:06:22.566 | INFO | Perf report for the last 5.13919 seconds: | 20260505 05:06:22.566 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:06:22.566 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:22.566 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:22.566 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:22.566 | INFO | ANALYSIS : 56.9%, 3.0, 88.3ms, 0 | 20260505 05:06:22.566 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:22.566 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:22.566 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 05:06:22.566 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:22.566 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:22.566 | INFO | Uploaded 36GiB to GPU in 1.603s (22.4578GiB/s) | 20260505 05:06:22.566 | INFO | Downloaded 34GiB to GPU in 1.51993s (22.3694GiB/s) | 20260505 05:06:27.705 | INFO | Analyzing chunk frames = [567296,567808) | 20260505 05:06:27.705 | INFO | MemInfo: { | 20260505 05:06:27.705 | INFO | "gpu" : " 17 GB", | 20260505 05:06:27.705 | INFO | "huge" : "", | 20260505 05:06:27.705 | INFO | "huge_shared" : " 27 GB", | 20260505 05:06:27.705 | INFO | "malloc" : " 14 GB", | 20260505 05:06:27.705 | INFO | "pinned" : " 29 GB" | 20260505 05:06:27.705 | INFO | } | 20260505 05:06:27.705 | INFO | Perf report for the last 5.13945 seconds: | 20260505 05:06:27.705 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:06:27.705 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:27.705 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:27.705 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:27.705 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260505 05:06:27.705 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:27.705 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:27.705 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260505 05:06:27.705 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:27.705 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:27.706 | INFO | Uploaded 36GiB to GPU in 1.6013s (22.4817GiB/s) | 20260505 05:06:27.706 | INFO | Downloaded 34GiB to GPU in 1.51828s (22.3938GiB/s) | 20260505 05:06:32.844 | INFO | Analyzing chunk frames = [567808,568320) | 20260505 05:06:32.844 | INFO | MemInfo: { | 20260505 05:06:32.844 | INFO | "gpu" : " 17 GB", | 20260505 05:06:32.845 | INFO | "huge" : "", | 20260505 05:06:32.845 | INFO | "huge_shared" : " 27 GB", | 20260505 05:06:32.845 | INFO | "malloc" : " 14 GB", | 20260505 05:06:32.845 | INFO | "pinned" : " 29 GB" | 20260505 05:06:32.845 | INFO | } | 20260505 05:06:32.845 | INFO | Perf report for the last 5.13935 seconds: | 20260505 05:06:32.845 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:06:32.845 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:32.845 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:32.845 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:32.845 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260505 05:06:32.845 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:32.845 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:32.845 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 05:06:32.845 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:32.845 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:32.845 | INFO | Uploaded 36GiB to GPU in 1.6021s (22.4705GiB/s) | 20260505 05:06:32.845 | INFO | Downloaded 34GiB to GPU in 1.5186s (22.3891GiB/s) | 20260505 05:06:37.984 | INFO | Analyzing chunk frames = [568320,568832) | 20260505 05:06:37.984 | INFO | MemInfo: { | 20260505 05:06:37.984 | INFO | "gpu" : " 17 GB", | 20260505 05:06:37.984 | INFO | "huge" : "", | 20260505 05:06:37.984 | INFO | "huge_shared" : " 27 GB", | 20260505 05:06:37.984 | INFO | "malloc" : " 14 GB", | 20260505 05:06:37.984 | INFO | "pinned" : " 29 GB" | 20260505 05:06:37.984 | INFO | } | 20260505 05:06:37.984 | INFO | Perf report for the last 5.13939 seconds: | 20260505 05:06:37.984 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:06:37.984 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:37.984 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:37.984 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:37.984 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260505 05:06:37.984 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:37.984 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:37.984 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 05:06:37.984 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:37.984 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:37.984 | INFO | Uploaded 36GiB to GPU in 1.60194s (22.4728GiB/s) | 20260505 05:06:37.984 | INFO | Downloaded 34GiB to GPU in 1.51894s (22.384GiB/s) | 20260505 05:06:43.123 | INFO | Analyzing chunk frames = [568832,569344) | 20260505 05:06:43.123 | INFO | MemInfo: { | 20260505 05:06:43.123 | INFO | "gpu" : " 17 GB", | 20260505 05:06:43.123 | INFO | "huge" : "", | 20260505 05:06:43.123 | INFO | "huge_shared" : " 27 GB", | 20260505 05:06:43.123 | INFO | "malloc" : " 14 GB", | 20260505 05:06:43.123 | INFO | "pinned" : " 29 GB" | 20260505 05:06:43.123 | INFO | } | 20260505 05:06:43.123 | INFO | Perf report for the last 5.13913 seconds: | 20260505 05:06:43.123 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:06:43.123 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:43.123 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:43.123 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:43.123 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260505 05:06:43.123 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:43.123 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:43.123 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:06:43.123 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:43.123 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:43.124 | INFO | Uploaded 36GiB to GPU in 1.60186s (22.4739GiB/s) | 20260505 05:06:43.124 | INFO | Downloaded 34GiB to GPU in 1.51905s (22.3824GiB/s) | 20260505 05:06:43.221 | INFO | Starting a round of Baz IO | 20260505 05:06:46.170 | INFO | Populating RTMetrics report | 20260505 05:06:46.170 | INFO | RTMetrics report populated | 20260505 05:06:46.170 | INFO | Opening tmp RTMetrics json file | 20260505 05:06:46.172 | INFO | Serializing RTMetrics to json | 20260505 05:06:46.173 | INFO | Serialization done, swapping tmp file now... | 20260505 05:06:46.173 | INFO | RTMetrics done | 20260505 05:06:46.275 | INFO | Finished a round of Baz IO: Wrote 2.158 GiB in 3.053 seconds | 20260505 05:06:46.275 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.44% / 0.10% / 58.46% | 20260505 05:06:46.275 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.52236% / 0.000206629% / 3.57628e-05% | 20260505 05:06:46.275 | INFO | Baz IO overall: | 20260505 05:06:46.275 | INFO | BytesWritten : 253587815760 | 20260505 05:06:46.275 | INFO | NumEvents : 118616032441 | 20260505 05:06:48.263 | INFO | Analyzing chunk frames = [569344,569856) | 20260505 05:06:48.263 | INFO | MemInfo: { | 20260505 05:06:48.263 | INFO | "gpu" : " 17 GB", | 20260505 05:06:48.263 | INFO | "huge" : "", | 20260505 05:06:48.266 | INFO | "huge_shared" : " 27 GB", | 20260505 05:06:48.266 | INFO | "malloc" : " 14 GB", | 20260505 05:06:48.266 | INFO | "pinned" : " 29 GB" | 20260505 05:06:48.266 | INFO | } | 20260505 05:06:48.266 | INFO | Perf report for the last 5.14294 seconds: | 20260505 05:06:48.266 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:06:48.266 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:48.266 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:48.266 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:48.266 | INFO | ANALYSIS : 58.1%, 3.0, 90.2ms, 0 | 20260505 05:06:48.266 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:48.266 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260505 05:06:48.266 | INFO | PRE_HQ : 57.4%, 2.2, 64.8ms, 0 | 20260505 05:06:48.266 | INFO | BAZWRITER : 58.8%, 4.2, 128.2ms, 0 | 20260505 05:06:48.266 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 05:06:48.266 | INFO | Uploaded 36GiB to GPU in 1.64199s (21.9246GiB/s) | 20260505 05:06:48.266 | INFO | Downloaded 38GiB to GPU in 1.70825s (22.2449GiB/s) | 20260505 05:06:53.402 | INFO | Analyzing chunk frames = [569856,570368) | 20260505 05:06:53.402 | INFO | MemInfo: { | 20260505 05:06:53.402 | INFO | "gpu" : " 17 GB", | 20260505 05:06:53.402 | INFO | "huge" : "", | 20260505 05:06:53.402 | INFO | "huge_shared" : " 27 GB", | 20260505 05:06:53.402 | INFO | "malloc" : " 14 GB", | 20260505 05:06:53.402 | INFO | "pinned" : " 29 GB" | 20260505 05:06:53.402 | INFO | } | 20260505 05:06:53.402 | INFO | Perf report for the last 5.13574 seconds: | 20260505 05:06:53.402 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:06:53.402 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:53.402 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:53.402 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:53.402 | INFO | ANALYSIS : 57.9%, 3.0, 90.3ms, 0 | 20260505 05:06:53.402 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:53.402 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:53.402 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:06:53.402 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:53.402 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:53.402 | INFO | Uploaded 36GiB to GPU in 1.59971s (22.5041GiB/s) | 20260505 05:06:53.402 | INFO | Downloaded 34GiB to GPU in 1.51589s (22.429GiB/s) | 20260505 05:06:58.541 | INFO | Analyzing chunk frames = [570368,570880) | 20260505 05:06:58.541 | INFO | MemInfo: { | 20260505 05:06:58.541 | INFO | "gpu" : " 17 GB", | 20260505 05:06:58.541 | INFO | "huge" : "", | 20260505 05:06:58.541 | INFO | "huge_shared" : " 27 GB", | 20260505 05:06:58.541 | INFO | "malloc" : " 14 GB", | 20260505 05:06:58.541 | INFO | "pinned" : " 29 GB" | 20260505 05:06:58.541 | INFO | } | 20260505 05:06:58.541 | INFO | Perf report for the last 5.13952 seconds: | 20260505 05:06:58.541 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:06:58.541 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:58.541 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:58.541 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:58.541 | INFO | ANALYSIS : 57.3%, 3.0, 89.0ms, 0 | 20260505 05:06:58.541 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:58.541 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:58.541 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 05:06:58.541 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:58.541 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:06:58.542 | INFO | Uploaded 36GiB to GPU in 1.60318s (22.4554GiB/s) | 20260505 05:06:58.542 | INFO | Downloaded 34GiB to GPU in 1.52072s (22.3578GiB/s) | 20260505 05:07:03.680 | INFO | Analyzing chunk frames = [570880,571392) | 20260505 05:07:03.681 | INFO | MemInfo: { | 20260505 05:07:03.681 | INFO | "gpu" : " 17 GB", | 20260505 05:07:03.681 | INFO | "huge" : "", | 20260505 05:07:03.681 | INFO | "huge_shared" : " 27 GB", | 20260505 05:07:03.681 | INFO | "malloc" : " 14 GB", | 20260505 05:07:03.681 | INFO | "pinned" : " 29 GB" | 20260505 05:07:03.681 | INFO | } | 20260505 05:07:03.681 | INFO | Perf report for the last 5.13931 seconds: | 20260505 05:07:03.681 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:07:03.681 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:03.681 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:03.681 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:03.681 | INFO | ANALYSIS : 56.9%, 3.0, 88.3ms, 0 | 20260505 05:07:03.681 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:03.681 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:03.681 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 05:07:03.681 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:03.681 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:03.681 | INFO | Uploaded 36GiB to GPU in 1.60428s (22.4399GiB/s) | 20260505 05:07:03.681 | INFO | Downloaded 34GiB to GPU in 1.5212s (22.3508GiB/s) | 20260505 05:07:08.820 | INFO | Analyzing chunk frames = [571392,571904) | 20260505 05:07:08.820 | INFO | MemInfo: { | 20260505 05:07:08.820 | INFO | "gpu" : " 17 GB", | 20260505 05:07:08.820 | INFO | "huge" : "", | 20260505 05:07:08.820 | INFO | "huge_shared" : " 27 GB", | 20260505 05:07:08.820 | INFO | "malloc" : " 14 GB", | 20260505 05:07:08.820 | INFO | "pinned" : " 29 GB" | 20260505 05:07:08.820 | INFO | } | 20260505 05:07:08.820 | INFO | Perf report for the last 5.13943 seconds: | 20260505 05:07:08.820 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:07:08.820 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:08.820 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:08.820 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:08.820 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260505 05:07:08.820 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:08.820 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:08.820 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 05:07:08.820 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:08.820 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:08.821 | INFO | Uploaded 36GiB to GPU in 1.60099s (22.486GiB/s) | 20260505 05:07:08.821 | INFO | Downloaded 34GiB to GPU in 1.51769s (22.4024GiB/s) | 20260505 05:07:13.959 | INFO | Analyzing chunk frames = [571904,572416) | 20260505 05:07:13.959 | INFO | MemInfo: { | 20260505 05:07:13.959 | INFO | "gpu" : " 17 GB", | 20260505 05:07:13.959 | INFO | "huge" : "", | 20260505 05:07:13.959 | INFO | "huge_shared" : " 27 GB", | 20260505 05:07:13.959 | INFO | "malloc" : " 14 GB", | 20260505 05:07:13.960 | INFO | "pinned" : " 29 GB" | 20260505 05:07:13.960 | INFO | } | 20260505 05:07:13.960 | INFO | Perf report for the last 5.1394 seconds: | 20260505 05:07:13.960 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:07:13.960 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:13.960 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:13.960 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:13.960 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260505 05:07:13.960 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:13.960 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:13.960 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260505 05:07:13.960 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:13.960 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:13.960 | INFO | Uploaded 36GiB to GPU in 1.60213s (22.4701GiB/s) | 20260505 05:07:13.960 | INFO | Downloaded 34GiB to GPU in 1.51863s (22.3886GiB/s) | 20260505 05:07:19.099 | INFO | Analyzing chunk frames = [572416,572928) | 20260505 05:07:19.099 | INFO | MemInfo: { | 20260505 05:07:19.099 | INFO | "gpu" : " 17 GB", | 20260505 05:07:19.099 | INFO | "huge" : "", | 20260505 05:07:19.099 | INFO | "huge_shared" : " 27 GB", | 20260505 05:07:19.102 | INFO | "malloc" : " 14 GB", | 20260505 05:07:19.102 | INFO | "pinned" : " 29 GB" | 20260505 05:07:19.102 | INFO | } | 20260505 05:07:19.102 | INFO | Perf report for the last 5.14237 seconds: | 20260505 05:07:19.102 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:07:19.102 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:19.102 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:19.102 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:19.102 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260505 05:07:19.102 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:19.102 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:19.102 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260505 05:07:19.102 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:19.102 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:19.103 | INFO | Uploaded 36GiB to GPU in 1.60218s (22.4694GiB/s) | 20260505 05:07:19.103 | INFO | Downloaded 34GiB to GPU in 1.51924s (22.3796GiB/s) | 20260505 05:07:24.239 | INFO | Analyzing chunk frames = [572928,573440) | 20260505 05:07:24.239 | INFO | MemInfo: { | 20260505 05:07:24.239 | INFO | "gpu" : " 17 GB", | 20260505 05:07:24.239 | INFO | "huge" : "", | 20260505 05:07:24.239 | INFO | "huge_shared" : " 27 GB", | 20260505 05:07:24.239 | INFO | "malloc" : " 14 GB", | 20260505 05:07:24.239 | INFO | "pinned" : " 29 GB" | 20260505 05:07:24.239 | INFO | } | 20260505 05:07:24.239 | INFO | Perf report for the last 5.13702 seconds: | 20260505 05:07:24.239 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:07:24.239 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:24.239 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:24.239 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:24.239 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260505 05:07:24.239 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:24.239 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:24.239 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 05:07:24.239 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:24.239 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:24.239 | INFO | Uploaded 36GiB to GPU in 1.60223s (22.4687GiB/s) | 20260505 05:07:24.239 | INFO | Downloaded 34GiB to GPU in 1.51944s (22.3766GiB/s) | 20260505 05:07:24.239 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 05:07:24.239 | INFO | Total mem usage: 15.1 GiB | 20260505 05:07:24.239 | INFO | All Time High Water Marks sum: 15.7 GiB (Savings: 4.09%) | 20260505 05:07:24.239 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -4.1%) | 20260505 05:07:24.239 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 05:07:24.239 | INFO | High Water Breakdown: | 20260505 05:07:24.239 | INFO | 59.7% : 8.64 GiB ( 95.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 05:07:24.239 | INFO | 40.1% : 5.8 GiB ( 87.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 05:07:24.239 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 05:07:24.239 | INFO | ------------------------------------------ | 20260505 05:07:24.239 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:07:24.239 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 05:07:24.239 | INFO | Total mem usage: 26.5 GiB | 20260505 05:07:24.239 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 05:07:24.239 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 05:07:24.240 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 05:07:24.240 | INFO | High Water Breakdown: | 20260505 05:07:24.240 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:07:24.240 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:07:24.240 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:07:24.240 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:07:24.240 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:07:24.240 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:07:24.240 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 05:07:24.240 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 05:07:24.240 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:07:24.240 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 05:07:24.240 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 05:07:24.240 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 05:07:24.240 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 05:07:24.240 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 05:07:24.240 | INFO | ------------------------------------------ | 20260505 05:07:24.240 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:07:24.240 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 05:07:24.240 | INFO | Total mem usage: 25.6 GiB | 20260505 05:07:24.240 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 05:07:24.240 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 05:07:24.240 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 05:07:24.240 | INFO | High Water Breakdown: | 20260505 05:07:24.240 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 05:07:24.240 | INFO | ------------------------------------------ | 20260505 05:07:24.240 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:07:24.240 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 05:07:24.240 | INFO | Total mem usage: 15.7 GiB | 20260505 05:07:24.240 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 05:07:24.240 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 05:07:24.240 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260505 05:07:24.240 | INFO | High Water Breakdown: | 20260505 05:07:24.240 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 05:07:24.240 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 05:07:24.240 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 05:07:24.240 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 05:07:24.240 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 05:07:24.240 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 05:07:24.240 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 05:07:24.240 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 05:07:24.240 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 05:07:24.240 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:07:24.240 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 05:07:24.240 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:07:24.240 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 05:07:24.240 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 05:07:24.240 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 05:07:24.240 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 05:07:24.240 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:07:24.240 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 05:07:24.240 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:07:24.240 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:07:24.240 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:07:24.240 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 05:07:24.240 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 05:07:24.240 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:07:24.240 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 05:07:24.240 | INFO | ------------------------------------------ | 20260505 05:07:24.240 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:07:24.240 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 05:07:24.240 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:07:24.240 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:07:24.240 | INFO | PerfReport: Full Execution -- 3.04e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260505 05:07:24.240 | INFO | PerfReport: 0.28%, AggregateStats -- 8.65e-02 / -nan, 4.67e-03 / 0.00e+00 | 20260505 05:07:24.240 | INFO | PerfReport: 10.86%, Baseline -- 3.30e+00 / -nan, 1.86e-02 / 0.00e+00 | 20260505 05:07:24.240 | INFO | PerfReport: 3.04%, Histogram -- 9.24e-01 / -nan, 5.22e-02 / 0.00e+00 | 20260505 05:07:24.240 | INFO | PerfReport: 16.03%, DME -- 3.90e+01 / -nan, 3.25e+00 / 0.00e+00 | 20260505 05:07:24.240 | INFO | PerfReport: 56.66%, FrameLabeling -- 1.72e+01 / -nan, 4.54e-02 / 0.00e+00 | 20260505 05:07:24.240 | INFO | PerfReport: 7.58%, PulseAccumulating -- 2.30e+00 / -nan, 1.52e-01 / 0.00e+00 | 20260505 05:07:24.240 | INFO | PerfReport: 3.92%, Metrics -- 1.19e+00 / -nan, 1.98e-01 / 0.00e+00 | 20260505 05:07:24.240 | INFO | PerfReport: 1.61%, PulseSerializing -- 4.91e-01 / -nan, 7.94e-02 / 0.00e+00 | 20260505 05:07:24.240 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 05:07:24.240 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:07:24.240 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:07:24.240 | INFO | PerfReport: Full Execution -- 9.12e+01 / -nan, 1.90e+01 / 0.00e+00 | 20260505 05:07:24.240 | INFO | PerfReport: 18.35%, Upload -- 1.67e+01 / 0.00e+00, 3.79e-01 / 0.00e+00 | 20260505 05:07:24.240 | INFO | PerfReport: 17.61%, Download -- 1.61e+01 / 0.00e+00, 7.05e-01 / 0.00e+00 | 20260505 05:07:24.240 | INFO | PerfReport: 33.31%, Compute -- 3.04e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260505 05:07:24.240 | INFO | PerfReport: 0.55%, LockUpload -- 5.05e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260505 05:07:24.240 | INFO | PerfReport: 0.00%, LockDownload -- 3.84e-04 / 0.00e+00, 9.71e-05 / 0.00e+00 | 20260505 05:07:24.240 | INFO | PerfReport: 30.16%, LockCompute -- 2.75e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260505 05:07:24.240 | INFO | PerfReport: 0.02%, Misc -- 1.40e-02 / 0.00e+00, 1.88e-02 / 0.00e+00 | 20260505 05:07:24.336 | INFO | Starting a round of Baz IO | 20260505 05:07:27.281 | INFO | Populating RTMetrics report | 20260505 05:07:27.282 | INFO | RTMetrics report populated | 20260505 05:07:27.282 | INFO | Opening tmp RTMetrics json file | 20260505 05:07:27.282 | INFO | Serializing RTMetrics to json | 20260505 05:07:27.283 | INFO | Serialization done, swapping tmp file now... | 20260505 05:07:27.284 | INFO | RTMetrics done | 20260505 05:07:27.382 | INFO | Finished a round of Baz IO: Wrote 2.162 GiB in 3.045 seconds | 20260505 05:07:27.382 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.34% / 0.10% / 58.56% | 20260505 05:07:27.382 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.48686% / 0.000206629% / 5.96046e-05% | 20260505 05:07:27.382 | INFO | Baz IO overall: | 20260505 05:07:27.382 | INFO | BytesWritten : 255909419615 | 20260505 05:07:27.382 | INFO | NumEvents : 119719877467 | 20260505 05:07:29.378 | INFO | Analyzing chunk frames = [573440,573952) | 20260505 05:07:29.378 | INFO | MemInfo: { | 20260505 05:07:29.378 | INFO | "gpu" : " 17 GB", | 20260505 05:07:29.378 | INFO | "huge" : "", | 20260505 05:07:29.378 | INFO | "huge_shared" : " 27 GB", | 20260505 05:07:29.378 | INFO | "malloc" : " 14 GB", | 20260505 05:07:29.378 | INFO | "pinned" : " 29 GB" | 20260505 05:07:29.378 | INFO | } | 20260505 05:07:29.378 | INFO | Perf report for the last 5.13861 seconds: | 20260505 05:07:29.378 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:07:29.378 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:29.378 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:29.378 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:29.378 | INFO | ANALYSIS : 58.1%, 3.0, 90.2ms, 0 | 20260505 05:07:29.378 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:29.378 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260505 05:07:29.378 | INFO | PRE_HQ : 57.4%, 2.2, 64.6ms, 0 | 20260505 05:07:29.378 | INFO | BAZWRITER : 58.8%, 4.2, 126.6ms, 0 | 20260505 05:07:29.378 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 05:07:29.378 | INFO | Uploaded 36GiB to GPU in 1.64426s (21.8943GiB/s) | 20260505 05:07:29.378 | INFO | Downloaded 38GiB to GPU in 1.70709s (22.2601GiB/s) | 20260505 05:07:34.517 | INFO | Analyzing chunk frames = [573952,574464) | 20260505 05:07:34.517 | INFO | MemInfo: { | 20260505 05:07:34.517 | INFO | "gpu" : " 17 GB", | 20260505 05:07:34.517 | INFO | "huge" : "", | 20260505 05:07:34.517 | INFO | "huge_shared" : " 27 GB", | 20260505 05:07:34.517 | INFO | "malloc" : " 14 GB", | 20260505 05:07:34.517 | INFO | "pinned" : " 29 GB" | 20260505 05:07:34.517 | INFO | } | 20260505 05:07:34.517 | INFO | Perf report for the last 5.13914 seconds: | 20260505 05:07:34.517 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:07:34.517 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:34.517 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:34.517 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:34.517 | INFO | ANALYSIS : 57.9%, 3.0, 90.3ms, 0 | 20260505 05:07:34.517 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:34.517 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:34.517 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:07:34.517 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:34.517 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:34.517 | INFO | Uploaded 36GiB to GPU in 1.59983s (22.5023GiB/s) | 20260505 05:07:34.517 | INFO | Downloaded 34GiB to GPU in 1.51664s (22.418GiB/s) | 20260505 05:07:39.656 | INFO | Analyzing chunk frames = [574464,574976) | 20260505 05:07:39.656 | INFO | MemInfo: { | 20260505 05:07:39.656 | INFO | "gpu" : " 17 GB", | 20260505 05:07:39.656 | INFO | "huge" : "", | 20260505 05:07:39.656 | INFO | "huge_shared" : " 27 GB", | 20260505 05:07:39.656 | INFO | "malloc" : " 14 GB", | 20260505 05:07:39.656 | INFO | "pinned" : " 29 GB" | 20260505 05:07:39.656 | INFO | } | 20260505 05:07:39.656 | INFO | Perf report for the last 5.13911 seconds: | 20260505 05:07:39.656 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:07:39.656 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:39.656 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:39.656 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:39.656 | INFO | ANALYSIS : 57.2%, 3.0, 88.9ms, 0 | 20260505 05:07:39.656 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:39.656 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:39.656 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 05:07:39.656 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:39.656 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:39.656 | INFO | Uploaded 36GiB to GPU in 1.60381s (22.4466GiB/s) | 20260505 05:07:39.656 | INFO | Downloaded 34GiB to GPU in 1.52114s (22.3517GiB/s) | 20260505 05:07:44.795 | INFO | Analyzing chunk frames = [574976,575488) | 20260505 05:07:44.795 | INFO | MemInfo: { | 20260505 05:07:44.795 | INFO | "gpu" : " 17 GB", | 20260505 05:07:44.795 | INFO | "huge" : "", | 20260505 05:07:44.795 | INFO | "huge_shared" : " 27 GB", | 20260505 05:07:44.795 | INFO | "malloc" : " 14 GB", | 20260505 05:07:44.795 | INFO | "pinned" : " 29 GB" | 20260505 05:07:44.795 | INFO | } | 20260505 05:07:44.796 | INFO | Perf report for the last 5.13955 seconds: | 20260505 05:07:44.796 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:07:44.796 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:44.796 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:44.796 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:44.796 | INFO | ANALYSIS : 56.9%, 3.0, 88.3ms, 0 | 20260505 05:07:44.796 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:44.796 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:44.796 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 05:07:44.796 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:44.796 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:44.796 | INFO | Uploaded 36GiB to GPU in 1.60293s (22.4588GiB/s) | 20260505 05:07:44.796 | INFO | Downloaded 34GiB to GPU in 1.51945s (22.3766GiB/s) | 20260505 05:07:49.935 | INFO | Analyzing chunk frames = [575488,576000) | 20260505 05:07:49.935 | INFO | MemInfo: { | 20260505 05:07:49.935 | INFO | "gpu" : " 17 GB", | 20260505 05:07:49.935 | INFO | "huge" : "", | 20260505 05:07:49.935 | INFO | "huge_shared" : " 27 GB", | 20260505 05:07:49.935 | INFO | "malloc" : " 14 GB", | 20260505 05:07:49.935 | INFO | "pinned" : " 29 GB" | 20260505 05:07:49.935 | INFO | } | 20260505 05:07:49.935 | INFO | Perf report for the last 5.13995 seconds: | 20260505 05:07:49.935 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:07:49.935 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:49.935 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:49.935 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:49.935 | INFO | ANALYSIS : 56.8%, 3.0, 88.7ms, 0 | 20260505 05:07:49.935 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:49.936 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:49.936 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260505 05:07:49.936 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:49.936 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:49.936 | INFO | Uploaded 36GiB to GPU in 1.60206s (22.471GiB/s) | 20260505 05:07:49.936 | INFO | Downloaded 34GiB to GPU in 1.51856s (22.3896GiB/s) | 20260505 05:07:55.074 | INFO | Analyzing chunk frames = [576000,576512) | 20260505 05:07:55.074 | INFO | MemInfo: { | 20260505 05:07:55.074 | INFO | "gpu" : " 17 GB", | 20260505 05:07:55.074 | INFO | "huge" : "", | 20260505 05:07:55.074 | INFO | "huge_shared" : " 27 GB", | 20260505 05:07:55.074 | INFO | "malloc" : " 14 GB", | 20260505 05:07:55.074 | INFO | "pinned" : " 29 GB" | 20260505 05:07:55.074 | INFO | } | 20260505 05:07:55.074 | INFO | Perf report for the last 5.13858 seconds: | 20260505 05:07:55.074 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:07:55.074 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:55.074 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:55.074 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:55.074 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260505 05:07:55.074 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:55.074 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:55.074 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260505 05:07:55.074 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:55.074 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:07:55.075 | INFO | Uploaded 36GiB to GPU in 1.60152s (22.4786GiB/s) | 20260505 05:07:55.075 | INFO | Downloaded 34GiB to GPU in 1.51781s (22.4008GiB/s) | 20260505 05:08:00.213 | INFO | Analyzing chunk frames = [576512,577024) | 20260505 05:08:00.213 | INFO | MemInfo: { | 20260505 05:08:00.213 | INFO | "gpu" : " 17 GB", | 20260505 05:08:00.213 | INFO | "huge" : "", | 20260505 05:08:00.213 | INFO | "huge_shared" : " 27 GB", | 20260505 05:08:00.216 | INFO | "malloc" : " 14 GB", | 20260505 05:08:00.216 | INFO | "pinned" : " 29 GB" | 20260505 05:08:00.217 | INFO | } | 20260505 05:08:00.217 | INFO | Perf report for the last 5.14249 seconds: | 20260505 05:08:00.217 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:08:00.217 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:00.217 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:00.217 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:00.217 | INFO | ANALYSIS : 57.3%, 3.0, 89.5ms, 0 | 20260505 05:08:00.217 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:00.217 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:00.217 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260505 05:08:00.217 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:00.217 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:00.217 | INFO | Uploaded 36GiB to GPU in 1.60273s (22.4617GiB/s) | 20260505 05:08:00.217 | INFO | Downloaded 34GiB to GPU in 1.51987s (22.3703GiB/s) | 20260505 05:08:05.353 | INFO | Analyzing chunk frames = [577024,577536) | 20260505 05:08:05.353 | INFO | MemInfo: { | 20260505 05:08:05.353 | INFO | "gpu" : " 17 GB", | 20260505 05:08:05.353 | INFO | "huge" : "", | 20260505 05:08:05.353 | INFO | "huge_shared" : " 27 GB", | 20260505 05:08:05.353 | INFO | "malloc" : " 14 GB", | 20260505 05:08:05.353 | INFO | "pinned" : " 29 GB" | 20260505 05:08:05.353 | INFO | } | 20260505 05:08:05.353 | INFO | Perf report for the last 5.13619 seconds: | 20260505 05:08:05.353 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:08:05.353 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:05.353 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:05.353 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:05.353 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260505 05:08:05.353 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:05.353 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:05.353 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 05:08:05.353 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:05.353 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:05.353 | INFO | Uploaded 36GiB to GPU in 1.60215s (22.4698GiB/s) | 20260505 05:08:05.353 | INFO | Downloaded 34GiB to GPU in 1.51918s (22.3805GiB/s) | 20260505 05:08:05.450 | INFO | Starting a round of Baz IO | 20260505 05:08:08.395 | INFO | Populating RTMetrics report | 20260505 05:08:08.395 | INFO | RTMetrics report populated | 20260505 05:08:08.395 | INFO | Opening tmp RTMetrics json file | 20260505 05:08:08.396 | INFO | Serializing RTMetrics to json | 20260505 05:08:08.397 | INFO | Serialization done, swapping tmp file now... | 20260505 05:08:08.398 | INFO | RTMetrics done | 20260505 05:08:08.496 | INFO | Finished a round of Baz IO: Wrote 2.164 GiB in 3.046 seconds | 20260505 05:08:08.496 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.25% / 0.10% / 58.66% | 20260505 05:08:08.496 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.44706% / 0.000190735% / 5.16574e-05% | 20260505 05:08:08.496 | INFO | Baz IO overall: | 20260505 05:08:08.496 | INFO | BytesWritten : 258233367731 | 20260505 05:08:08.496 | INFO | NumEvents : 120824714939 | 20260505 05:08:10.492 | INFO | Analyzing chunk frames = [577536,578048) | 20260505 05:08:10.492 | INFO | MemInfo: { | 20260505 05:08:10.492 | INFO | "gpu" : " 17 GB", | 20260505 05:08:10.492 | INFO | "huge" : "", | 20260505 05:08:10.492 | INFO | "huge_shared" : " 27 GB", | 20260505 05:08:10.492 | INFO | "malloc" : " 14 GB", | 20260505 05:08:10.492 | INFO | "pinned" : " 29 GB" | 20260505 05:08:10.492 | INFO | } | 20260505 05:08:10.492 | INFO | Perf report for the last 5.13939 seconds: | 20260505 05:08:10.492 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:08:10.492 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:10.492 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:10.492 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:10.492 | INFO | ANALYSIS : 58.1%, 3.0, 90.0ms, 0 | 20260505 05:08:10.492 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:10.492 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260505 05:08:10.492 | INFO | PRE_HQ : 57.4%, 2.2, 64.6ms, 0 | 20260505 05:08:10.492 | INFO | BAZWRITER : 58.9%, 4.3, 129.0ms, 0 | 20260505 05:08:10.492 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 05:08:10.493 | INFO | Uploaded 36GiB to GPU in 1.64553s (21.8774GiB/s) | 20260505 05:08:10.493 | INFO | Downloaded 38GiB to GPU in 1.70952s (22.2285GiB/s) | 20260505 05:08:15.631 | INFO | Analyzing chunk frames = [578048,578560) | 20260505 05:08:15.631 | INFO | MemInfo: { | 20260505 05:08:15.631 | INFO | "gpu" : " 17 GB", | 20260505 05:08:15.631 | INFO | "huge" : "", | 20260505 05:08:15.631 | INFO | "huge_shared" : " 27 GB", | 20260505 05:08:15.631 | INFO | "malloc" : " 14 GB", | 20260505 05:08:15.631 | INFO | "pinned" : " 29 GB" | 20260505 05:08:15.631 | INFO | } | 20260505 05:08:15.632 | INFO | Perf report for the last 5.13934 seconds: | 20260505 05:08:15.632 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:08:15.632 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:15.632 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:15.632 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:15.632 | INFO | ANALYSIS : 57.9%, 3.0, 90.3ms, 0 | 20260505 05:08:15.632 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:15.632 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:15.632 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 05:08:15.632 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:15.632 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:15.632 | INFO | Uploaded 36GiB to GPU in 1.59929s (22.5099GiB/s) | 20260505 05:08:15.632 | INFO | Downloaded 34GiB to GPU in 1.51616s (22.4251GiB/s) | 20260505 05:08:20.771 | INFO | Analyzing chunk frames = [578560,579072) | 20260505 05:08:20.771 | INFO | MemInfo: { | 20260505 05:08:20.771 | INFO | "gpu" : " 17 GB", | 20260505 05:08:20.771 | INFO | "huge" : "", | 20260505 05:08:20.771 | INFO | "huge_shared" : " 27 GB", | 20260505 05:08:20.771 | INFO | "malloc" : " 14 GB", | 20260505 05:08:20.771 | INFO | "pinned" : " 29 GB" | 20260505 05:08:20.771 | INFO | } | 20260505 05:08:20.771 | INFO | Perf report for the last 5.13948 seconds: | 20260505 05:08:20.771 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:08:20.771 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:20.771 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:20.771 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:20.771 | INFO | ANALYSIS : 57.2%, 3.0, 88.9ms, 0 | 20260505 05:08:20.771 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:20.771 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:20.771 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 05:08:20.771 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:20.771 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:20.772 | INFO | Uploaded 36GiB to GPU in 1.60403s (22.4434GiB/s) | 20260505 05:08:20.772 | INFO | Downloaded 34GiB to GPU in 1.52108s (22.3526GiB/s) | 20260505 05:08:25.910 | INFO | Analyzing chunk frames = [579072,579584) | 20260505 05:08:25.910 | INFO | MemInfo: { | 20260505 05:08:25.910 | INFO | "gpu" : " 17 GB", | 20260505 05:08:25.910 | INFO | "huge" : "", | 20260505 05:08:25.910 | INFO | "huge_shared" : " 27 GB", | 20260505 05:08:25.910 | INFO | "malloc" : " 14 GB", | 20260505 05:08:25.910 | INFO | "pinned" : " 29 GB" | 20260505 05:08:25.910 | INFO | } | 20260505 05:08:25.910 | INFO | Perf report for the last 5.1393 seconds: | 20260505 05:08:25.910 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:08:25.910 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:25.910 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:25.910 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:25.910 | INFO | ANALYSIS : 57.0%, 3.0, 88.3ms, 0 | 20260505 05:08:25.910 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:25.910 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:25.910 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 05:08:25.910 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:25.910 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:25.911 | INFO | Uploaded 36GiB to GPU in 1.60311s (22.4564GiB/s) | 20260505 05:08:25.911 | INFO | Downloaded 34GiB to GPU in 1.52034s (22.3634GiB/s) | 20260505 05:08:31.049 | INFO | Analyzing chunk frames = [579584,580096) | 20260505 05:08:31.049 | INFO | MemInfo: { | 20260505 05:08:31.049 | INFO | "gpu" : " 17 GB", | 20260505 05:08:31.049 | INFO | "huge" : "", | 20260505 05:08:31.049 | INFO | "huge_shared" : " 27 GB", | 20260505 05:08:31.049 | INFO | "malloc" : " 14 GB", | 20260505 05:08:31.049 | INFO | "pinned" : " 29 GB" | 20260505 05:08:31.049 | INFO | } | 20260505 05:08:31.049 | INFO | Perf report for the last 5.13916 seconds: | 20260505 05:08:31.049 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:08:31.049 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:31.050 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:31.050 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:31.050 | INFO | ANALYSIS : 56.8%, 3.0, 88.7ms, 0 | 20260505 05:08:31.050 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:31.050 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:31.050 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 05:08:31.050 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:31.050 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:31.050 | INFO | Uploaded 36GiB to GPU in 1.60211s (22.4703GiB/s) | 20260505 05:08:31.050 | INFO | Downloaded 34GiB to GPU in 1.51862s (22.3888GiB/s) | 20260505 05:08:36.189 | INFO | Analyzing chunk frames = [580096,580608) | 20260505 05:08:36.189 | INFO | MemInfo: { | 20260505 05:08:36.189 | INFO | "gpu" : " 17 GB", | 20260505 05:08:36.189 | INFO | "huge" : "", | 20260505 05:08:36.189 | INFO | "huge_shared" : " 27 GB", | 20260505 05:08:36.189 | INFO | "malloc" : " 14 GB", | 20260505 05:08:36.189 | INFO | "pinned" : " 29 GB" | 20260505 05:08:36.189 | INFO | } | 20260505 05:08:36.189 | INFO | Perf report for the last 5.13931 seconds: | 20260505 05:08:36.189 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:08:36.189 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:36.189 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:36.189 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:36.189 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260505 05:08:36.189 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:36.189 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:36.189 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:08:36.189 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:36.189 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:36.189 | INFO | Uploaded 36GiB to GPU in 1.60118s (22.4835GiB/s) | 20260505 05:08:36.189 | INFO | Downloaded 34GiB to GPU in 1.51841s (22.3919GiB/s) | 20260505 05:08:41.328 | INFO | Analyzing chunk frames = [580608,581120) | 20260505 05:08:41.328 | INFO | MemInfo: { | 20260505 05:08:41.329 | INFO | "gpu" : " 17 GB", | 20260505 05:08:41.329 | INFO | "huge" : "", | 20260505 05:08:41.329 | INFO | "huge_shared" : " 27 GB", | 20260505 05:08:41.329 | INFO | "malloc" : " 14 GB", | 20260505 05:08:41.329 | INFO | "pinned" : " 29 GB" | 20260505 05:08:41.329 | INFO | } | 20260505 05:08:41.329 | INFO | Perf report for the last 5.14041 seconds: | 20260505 05:08:41.329 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:08:41.329 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:41.329 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:41.329 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:41.329 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 05:08:41.329 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:41.329 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:41.329 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:08:41.329 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:41.329 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:41.330 | INFO | Uploaded 36GiB to GPU in 1.60179s (22.4749GiB/s) | 20260505 05:08:41.330 | INFO | Downloaded 34GiB to GPU in 1.51911s (22.3815GiB/s) | 20260505 05:08:46.467 | INFO | Analyzing chunk frames = [581120,581632) | 20260505 05:08:46.467 | INFO | MemInfo: { | 20260505 05:08:46.467 | INFO | "gpu" : " 17 GB", | 20260505 05:08:46.467 | INFO | "huge" : "", | 20260505 05:08:46.467 | INFO | "huge_shared" : " 27 GB", | 20260505 05:08:46.467 | INFO | "malloc" : " 14 GB", | 20260505 05:08:46.467 | INFO | "pinned" : " 29 GB" | 20260505 05:08:46.467 | INFO | } | 20260505 05:08:46.468 | INFO | Perf report for the last 5.1383 seconds: | 20260505 05:08:46.468 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:08:46.468 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:46.468 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:46.468 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:46.468 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260505 05:08:46.468 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:46.468 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:46.468 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 05:08:46.468 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:46.468 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:46.468 | INFO | Uploaded 36GiB to GPU in 1.60272s (22.4618GiB/s) | 20260505 05:08:46.468 | INFO | Downloaded 34GiB to GPU in 1.51987s (22.3703GiB/s) | 20260505 05:08:46.468 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 05:08:46.468 | INFO | Total mem usage: 15.1 GiB | 20260505 05:08:46.468 | INFO | All Time High Water Marks sum: 15.7 GiB (Savings: 4.09%) | 20260505 05:08:46.468 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -3.71%) | 20260505 05:08:46.468 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 05:08:46.468 | INFO | High Water Breakdown: | 20260505 05:08:46.468 | INFO | 59.8% : 8.68 GiB ( 96.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 05:08:46.468 | INFO | 40.1% : 5.82 GiB ( 87.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 05:08:46.468 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 05:08:46.468 | INFO | ------------------------------------------ | 20260505 05:08:46.468 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:08:46.468 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 05:08:46.468 | INFO | Total mem usage: 26.5 GiB | 20260505 05:08:46.468 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 05:08:46.468 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 05:08:46.468 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 05:08:46.468 | INFO | High Water Breakdown: | 20260505 05:08:46.468 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:08:46.468 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:08:46.468 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:08:46.468 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:08:46.468 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:08:46.468 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:08:46.468 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 05:08:46.468 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 05:08:46.468 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:08:46.468 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 05:08:46.468 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 05:08:46.468 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 05:08:46.468 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 05:08:46.468 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 05:08:46.468 | INFO | ------------------------------------------ | 20260505 05:08:46.468 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:08:46.468 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 05:08:46.468 | INFO | Total mem usage: 25.6 GiB | 20260505 05:08:46.468 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 05:08:46.468 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 05:08:46.468 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 05:08:46.468 | INFO | High Water Breakdown: | 20260505 05:08:46.468 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 05:08:46.468 | INFO | ------------------------------------------ | 20260505 05:08:46.468 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:08:46.468 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 05:08:46.468 | INFO | Total mem usage: 15.7 GiB | 20260505 05:08:46.468 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 05:08:46.468 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 05:08:46.468 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260505 05:08:46.468 | INFO | High Water Breakdown: | 20260505 05:08:46.468 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 05:08:46.468 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 05:08:46.468 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 05:08:46.468 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 05:08:46.468 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 05:08:46.468 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 05:08:46.468 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 05:08:46.468 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 05:08:46.468 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 05:08:46.468 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:08:46.468 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 05:08:46.468 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:08:46.468 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 05:08:46.468 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 05:08:46.468 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 05:08:46.468 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 05:08:46.468 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:08:46.468 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 05:08:46.468 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:08:46.468 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:08:46.468 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:08:46.468 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 05:08:46.468 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 05:08:46.468 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:08:46.468 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 05:08:46.468 | INFO | ------------------------------------------ | 20260505 05:08:46.468 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:08:46.468 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 05:08:46.468 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:08:46.468 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:08:46.468 | INFO | PerfReport: Full Execution -- 3.04e+01 / -nan, 1.29e+01 / 0.00e+00 | 20260505 05:08:46.468 | INFO | PerfReport: 0.28%, AggregateStats -- 8.63e-02 / -nan, 3.93e-03 / 0.00e+00 | 20260505 05:08:46.468 | INFO | PerfReport: 10.86%, Baseline -- 3.30e+00 / -nan, 1.62e-02 / 0.00e+00 | 20260505 05:08:46.468 | INFO | PerfReport: 3.04%, Histogram -- 9.24e-01 / -nan, 5.30e-02 / 0.00e+00 | 20260505 05:08:46.468 | INFO | PerfReport: 16.02%, DME -- 3.89e+01 / -nan, 3.17e+00 / 0.00e+00 | 20260505 05:08:46.468 | INFO | PerfReport: 56.68%, FrameLabeling -- 1.72e+01 / -nan, 4.63e-02 / 0.00e+00 | 20260505 05:08:46.468 | INFO | PerfReport: 7.58%, PulseAccumulating -- 2.30e+00 / -nan, 1.51e-01 / 0.00e+00 | 20260505 05:08:46.468 | INFO | PerfReport: 3.92%, Metrics -- 1.19e+00 / -nan, 1.97e-01 / 0.00e+00 | 20260505 05:08:46.468 | INFO | PerfReport: 1.61%, PulseSerializing -- 4.90e-01 / -nan, 7.91e-02 / 0.00e+00 | 20260505 05:08:46.468 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 05:08:46.468 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:08:46.469 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:08:46.469 | INFO | PerfReport: Full Execution -- 9.12e+01 / -nan, 1.90e+01 / 0.00e+00 | 20260505 05:08:46.469 | INFO | PerfReport: 18.36%, Upload -- 1.67e+01 / 0.00e+00, 3.85e-01 / 0.00e+00 | 20260505 05:08:46.469 | INFO | PerfReport: 17.63%, Download -- 1.61e+01 / 0.00e+00, 7.19e-01 / 0.00e+00 | 20260505 05:08:46.469 | INFO | PerfReport: 33.31%, Compute -- 3.04e+01 / 0.00e+00, 1.29e+01 / 0.00e+00 | 20260505 05:08:46.469 | INFO | PerfReport: 0.55%, LockUpload -- 5.04e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260505 05:08:46.469 | INFO | PerfReport: 0.00%, LockDownload -- 3.91e-04 / 0.00e+00, 9.57e-05 / 0.00e+00 | 20260505 05:08:46.469 | INFO | PerfReport: 30.13%, LockCompute -- 2.75e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260505 05:08:46.469 | INFO | PerfReport: 0.02%, Misc -- 1.41e-02 / 0.00e+00, 1.87e-02 / 0.00e+00 | 20260505 05:08:46.564 | INFO | Starting a round of Baz IO | 20260505 05:08:49.510 | INFO | Populating RTMetrics report | 20260505 05:08:49.510 | INFO | RTMetrics report populated | 20260505 05:08:49.510 | INFO | Opening tmp RTMetrics json file | 20260505 05:08:49.517 | INFO | Serializing RTMetrics to json | 20260505 05:08:49.519 | INFO | Serialization done, swapping tmp file now... | 20260505 05:08:49.522 | INFO | RTMetrics done | 20260505 05:08:49.613 | INFO | Finished a round of Baz IO: Wrote 2.167 GiB in 3.048 seconds | 20260505 05:08:49.613 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.15% / 0.10% / 58.75% | 20260505 05:08:49.613 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.41111% / 0.000186761% / 4.37101e-05% | 20260505 05:08:49.613 | INFO | Baz IO overall: | 20260505 05:08:49.613 | INFO | BytesWritten : 260560613063 | 20260505 05:08:49.613 | INFO | NumEvents : 121931570940 | 20260505 05:08:51.607 | INFO | Analyzing chunk frames = [581632,582144) | 20260505 05:08:51.607 | INFO | MemInfo: { | 20260505 05:08:51.607 | INFO | "gpu" : " 17 GB", | 20260505 05:08:51.607 | INFO | "huge" : "", | 20260505 05:08:51.607 | INFO | "huge_shared" : " 27 GB", | 20260505 05:08:51.607 | INFO | "malloc" : " 14 GB", | 20260505 05:08:51.607 | INFO | "pinned" : " 29 GB" | 20260505 05:08:51.607 | INFO | } | 20260505 05:08:51.607 | INFO | Perf report for the last 5.1392 seconds: | 20260505 05:08:51.607 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:08:51.607 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:51.607 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:51.607 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:51.607 | INFO | ANALYSIS : 58.1%, 3.0, 90.2ms, 0 | 20260505 05:08:51.607 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:51.607 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260505 05:08:51.607 | INFO | PRE_HQ : 57.4%, 2.2, 64.4ms, 0 | 20260505 05:08:51.607 | INFO | BAZWRITER : 58.9%, 4.2, 128.3ms, 0 | 20260505 05:08:51.607 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 05:08:51.607 | INFO | Uploaded 36GiB to GPU in 1.6411s (21.9365GiB/s) | 20260505 05:08:51.607 | INFO | Downloaded 38GiB to GPU in 1.70592s (22.2754GiB/s) | 20260505 05:08:56.746 | INFO | Analyzing chunk frames = [582144,582656) | 20260505 05:08:56.746 | INFO | MemInfo: { | 20260505 05:08:56.746 | INFO | "gpu" : " 17 GB", | 20260505 05:08:56.746 | INFO | "huge" : "", | 20260505 05:08:56.746 | INFO | "huge_shared" : " 27 GB", | 20260505 05:08:56.746 | INFO | "malloc" : " 14 GB", | 20260505 05:08:56.746 | INFO | "pinned" : " 29 GB" | 20260505 05:08:56.746 | INFO | } | 20260505 05:08:56.746 | INFO | Perf report for the last 5.13942 seconds: | 20260505 05:08:56.746 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:08:56.746 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:56.746 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:56.746 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:56.746 | INFO | ANALYSIS : 57.9%, 3.0, 90.3ms, 0 | 20260505 05:08:56.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:56.746 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:56.746 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 05:08:56.746 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:56.746 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:08:56.746 | INFO | Uploaded 36GiB to GPU in 1.59961s (22.5055GiB/s) | 20260505 05:08:56.747 | INFO | Downloaded 34GiB to GPU in 1.51656s (22.4191GiB/s) | 20260505 05:09:01.885 | INFO | Analyzing chunk frames = [582656,583168) | 20260505 05:09:01.885 | INFO | MemInfo: { | 20260505 05:09:01.885 | INFO | "gpu" : " 17 GB", | 20260505 05:09:01.885 | INFO | "huge" : "", | 20260505 05:09:01.885 | INFO | "huge_shared" : " 27 GB", | 20260505 05:09:01.885 | INFO | "malloc" : " 14 GB", | 20260505 05:09:01.885 | INFO | "pinned" : " 29 GB" | 20260505 05:09:01.886 | INFO | } | 20260505 05:09:01.886 | INFO | Perf report for the last 5.13934 seconds: | 20260505 05:09:01.886 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:09:01.886 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:01.886 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:01.886 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:01.886 | INFO | ANALYSIS : 57.2%, 3.0, 88.9ms, 0 | 20260505 05:09:01.886 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:01.886 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:01.886 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 05:09:01.886 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:01.886 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:01.886 | INFO | Uploaded 36GiB to GPU in 1.6048s (22.4327GiB/s) | 20260505 05:09:01.886 | INFO | Downloaded 34GiB to GPU in 1.52171s (22.3433GiB/s) | 20260505 05:09:07.025 | INFO | Analyzing chunk frames = [583168,583680) | 20260505 05:09:07.025 | INFO | MemInfo: { | 20260505 05:09:07.025 | INFO | "gpu" : " 17 GB", | 20260505 05:09:07.025 | INFO | "huge" : "", | 20260505 05:09:07.025 | INFO | "huge_shared" : " 27 GB", | 20260505 05:09:07.025 | INFO | "malloc" : " 14 GB", | 20260505 05:09:07.025 | INFO | "pinned" : " 29 GB" | 20260505 05:09:07.025 | INFO | } | 20260505 05:09:07.025 | INFO | Perf report for the last 5.13938 seconds: | 20260505 05:09:07.025 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:09:07.025 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:07.025 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:07.025 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:07.025 | INFO | ANALYSIS : 56.9%, 3.0, 88.3ms, 0 | 20260505 05:09:07.025 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:07.025 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:07.025 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 05:09:07.025 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:07.025 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:07.025 | INFO | Uploaded 36GiB to GPU in 1.60334s (22.4532GiB/s) | 20260505 05:09:07.025 | INFO | Downloaded 34GiB to GPU in 1.5203s (22.3641GiB/s) | 20260505 05:09:12.164 | INFO | Analyzing chunk frames = [583680,584192) | 20260505 05:09:12.164 | INFO | MemInfo: { | 20260505 05:09:12.164 | INFO | "gpu" : " 17 GB", | 20260505 05:09:12.164 | INFO | "huge" : "", | 20260505 05:09:12.164 | INFO | "huge_shared" : " 27 GB", | 20260505 05:09:12.164 | INFO | "malloc" : " 14 GB", | 20260505 05:09:12.164 | INFO | "pinned" : " 29 GB" | 20260505 05:09:12.164 | INFO | } | 20260505 05:09:12.164 | INFO | Perf report for the last 5.13923 seconds: | 20260505 05:09:12.164 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:09:12.164 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:12.164 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:12.164 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:12.164 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260505 05:09:12.164 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:12.164 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:12.164 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 05:09:12.164 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:12.164 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:12.164 | INFO | Uploaded 36GiB to GPU in 1.6014s (22.4803GiB/s) | 20260505 05:09:12.164 | INFO | Downloaded 34GiB to GPU in 1.51774s (22.4017GiB/s) | 20260505 05:09:17.303 | INFO | Analyzing chunk frames = [584192,584704) | 20260505 05:09:17.303 | INFO | MemInfo: { | 20260505 05:09:17.303 | INFO | "gpu" : " 17 GB", | 20260505 05:09:17.303 | INFO | "huge" : "", | 20260505 05:09:17.303 | INFO | "huge_shared" : " 27 GB", | 20260505 05:09:17.303 | INFO | "malloc" : " 14 GB", | 20260505 05:09:17.303 | INFO | "pinned" : " 29 GB" | 20260505 05:09:17.303 | INFO | } | 20260505 05:09:17.304 | INFO | Perf report for the last 5.13935 seconds: | 20260505 05:09:17.304 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:09:17.304 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:17.304 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:17.304 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:17.304 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260505 05:09:17.304 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:17.304 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:17.304 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:09:17.304 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:17.304 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:17.304 | INFO | Uploaded 36GiB to GPU in 1.60259s (22.4637GiB/s) | 20260505 05:09:17.304 | INFO | Downloaded 34GiB to GPU in 1.51986s (22.3705GiB/s) | 20260505 05:09:22.443 | INFO | Analyzing chunk frames = [584704,585216) | 20260505 05:09:22.443 | INFO | MemInfo: { | 20260505 05:09:22.443 | INFO | "gpu" : " 17 GB", | 20260505 05:09:22.443 | INFO | "huge" : "", | 20260505 05:09:22.443 | INFO | "huge_shared" : " 27 GB", | 20260505 05:09:22.443 | INFO | "malloc" : " 14 GB", | 20260505 05:09:22.443 | INFO | "pinned" : " 29 GB" | 20260505 05:09:22.443 | INFO | } | 20260505 05:09:22.443 | INFO | Perf report for the last 5.13947 seconds: | 20260505 05:09:22.443 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:09:22.443 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:22.443 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:22.443 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:22.443 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260505 05:09:22.443 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:22.443 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:22.443 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 05:09:22.443 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:22.443 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:22.443 | INFO | Uploaded 36GiB to GPU in 1.60211s (22.4703GiB/s) | 20260505 05:09:22.443 | INFO | Downloaded 34GiB to GPU in 1.51962s (22.374GiB/s) | 20260505 05:09:27.582 | INFO | Analyzing chunk frames = [585216,585728) | 20260505 05:09:27.582 | INFO | MemInfo: { | 20260505 05:09:27.582 | INFO | "gpu" : " 17 GB", | 20260505 05:09:27.582 | INFO | "huge" : "", | 20260505 05:09:27.582 | INFO | "huge_shared" : " 27 GB", | 20260505 05:09:27.582 | INFO | "malloc" : " 14 GB", | 20260505 05:09:27.582 | INFO | "pinned" : " 29 GB" | 20260505 05:09:27.582 | INFO | } | 20260505 05:09:27.582 | INFO | Perf report for the last 5.13907 seconds: | 20260505 05:09:27.582 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:09:27.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:27.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:27.582 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:27.582 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260505 05:09:27.582 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:27.582 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:27.582 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:09:27.582 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:27.582 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:27.582 | INFO | Uploaded 36GiB to GPU in 1.60148s (22.4793GiB/s) | 20260505 05:09:27.583 | INFO | Downloaded 34GiB to GPU in 1.51938s (22.3775GiB/s) | 20260505 05:09:27.680 | INFO | Starting a round of Baz IO | 20260505 05:09:30.625 | INFO | Populating RTMetrics report | 20260505 05:09:30.626 | INFO | RTMetrics report populated | 20260505 05:09:30.626 | INFO | Opening tmp RTMetrics json file | 20260505 05:09:30.627 | INFO | Serializing RTMetrics to json | 20260505 05:09:30.629 | INFO | Serialization done, swapping tmp file now... | 20260505 05:09:30.629 | INFO | RTMetrics done | 20260505 05:09:30.731 | INFO | Finished a round of Baz IO: Wrote 2.17 GiB in 3.051 seconds | 20260505 05:09:30.732 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.06% / 0.10% / 58.85% | 20260505 05:09:30.732 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.3634% / 0.000194709% / 1.58946e-05% | 20260505 05:09:30.732 | INFO | Baz IO overall: | 20260505 05:09:30.732 | INFO | BytesWritten : 262890276766 | 20260505 05:09:30.732 | INFO | NumEvents : 123039441995 | 20260505 05:09:32.722 | INFO | Analyzing chunk frames = [585728,586240) | 20260505 05:09:32.722 | INFO | MemInfo: { | 20260505 05:09:32.722 | INFO | "gpu" : " 17 GB", | 20260505 05:09:32.722 | INFO | "huge" : "", | 20260505 05:09:32.722 | INFO | "huge_shared" : " 27 GB", | 20260505 05:09:32.722 | INFO | "malloc" : " 14 GB", | 20260505 05:09:32.722 | INFO | "pinned" : " 29 GB" | 20260505 05:09:32.722 | INFO | } | 20260505 05:09:32.722 | INFO | Perf report for the last 5.1396 seconds: | 20260505 05:09:32.722 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:09:32.722 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:32.722 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:32.722 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:32.722 | INFO | ANALYSIS : 58.1%, 3.0, 90.1ms, 0 | 20260505 05:09:32.722 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:32.722 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260505 05:09:32.722 | INFO | PRE_HQ : 57.4%, 2.2, 64.3ms, 0 | 20260505 05:09:32.722 | INFO | BAZWRITER : 59.0%, 4.2, 127.5ms, 0 | 20260505 05:09:32.722 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 05:09:32.722 | INFO | Uploaded 36GiB to GPU in 1.63938s (21.9595GiB/s) | 20260505 05:09:32.722 | INFO | Downloaded 38GiB to GPU in 1.70487s (22.2891GiB/s) | 20260505 05:09:37.861 | INFO | Analyzing chunk frames = [586240,586752) | 20260505 05:09:37.861 | INFO | MemInfo: { | 20260505 05:09:37.861 | INFO | "gpu" : " 17 GB", | 20260505 05:09:37.861 | INFO | "huge" : "", | 20260505 05:09:37.861 | INFO | "huge_shared" : " 27 GB", | 20260505 05:09:37.861 | INFO | "malloc" : " 14 GB", | 20260505 05:09:37.861 | INFO | "pinned" : " 29 GB" | 20260505 05:09:37.861 | INFO | } | 20260505 05:09:37.861 | INFO | Perf report for the last 5.13933 seconds: | 20260505 05:09:37.861 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:09:37.861 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:37.861 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:37.861 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:37.861 | INFO | ANALYSIS : 57.8%, 3.0, 90.2ms, 0 | 20260505 05:09:37.861 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:37.861 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:37.861 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:09:37.861 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:37.861 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:37.861 | INFO | Uploaded 36GiB to GPU in 1.59988s (22.5017GiB/s) | 20260505 05:09:37.861 | INFO | Downloaded 34GiB to GPU in 1.5165s (22.42GiB/s) | 20260505 05:09:43.000 | INFO | Analyzing chunk frames = [586752,587264) | 20260505 05:09:43.000 | INFO | MemInfo: { | 20260505 05:09:43.000 | INFO | "gpu" : " 17 GB", | 20260505 05:09:43.000 | INFO | "huge" : "", | 20260505 05:09:43.000 | INFO | "huge_shared" : " 27 GB", | 20260505 05:09:43.000 | INFO | "malloc" : " 14 GB", | 20260505 05:09:43.000 | INFO | "pinned" : " 29 GB" | 20260505 05:09:43.000 | INFO | } | 20260505 05:09:43.000 | INFO | Perf report for the last 5.13921 seconds: | 20260505 05:09:43.000 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:09:43.000 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:43.000 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:43.000 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:43.000 | INFO | ANALYSIS : 57.3%, 3.0, 89.0ms, 0 | 20260505 05:09:43.000 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:43.000 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:43.000 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260505 05:09:43.000 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:43.000 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:43.001 | INFO | Uploaded 36GiB to GPU in 1.60337s (22.4527GiB/s) | 20260505 05:09:43.001 | INFO | Downloaded 34GiB to GPU in 1.52141s (22.3477GiB/s) | 20260505 05:09:48.143 | INFO | Analyzing chunk frames = [587264,587776) | 20260505 05:09:48.143 | INFO | MemInfo: { | 20260505 05:09:48.143 | INFO | "gpu" : " 17 GB", | 20260505 05:09:48.143 | INFO | "huge" : "", | 20260505 05:09:48.143 | INFO | "huge_shared" : " 27 GB", | 20260505 05:09:48.143 | INFO | "malloc" : " 14 GB", | 20260505 05:09:48.143 | INFO | "pinned" : " 29 GB" | 20260505 05:09:48.143 | INFO | } | 20260505 05:09:48.143 | INFO | Perf report for the last 5.1427 seconds: | 20260505 05:09:48.143 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:09:48.143 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:48.143 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:48.143 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:48.143 | INFO | ANALYSIS : 56.9%, 3.0, 88.2ms, 0 | 20260505 05:09:48.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:48.143 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:48.143 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 05:09:48.143 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:48.143 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:48.143 | INFO | Uploaded 36GiB to GPU in 1.60439s (22.4384GiB/s) | 20260505 05:09:48.143 | INFO | Downloaded 34GiB to GPU in 1.52172s (22.3432GiB/s) | 20260505 05:09:53.280 | INFO | Analyzing chunk frames = [587776,588288) | 20260505 05:09:53.280 | INFO | MemInfo: { | 20260505 05:09:53.280 | INFO | "gpu" : " 17 GB", | 20260505 05:09:53.280 | INFO | "huge" : "", | 20260505 05:09:53.280 | INFO | "huge_shared" : " 27 GB", | 20260505 05:09:53.280 | INFO | "malloc" : " 14 GB", | 20260505 05:09:53.280 | INFO | "pinned" : " 29 GB" | 20260505 05:09:53.280 | INFO | } | 20260505 05:09:53.280 | INFO | Perf report for the last 5.13699 seconds: | 20260505 05:09:53.280 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:09:53.280 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:53.280 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:53.280 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:53.280 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260505 05:09:53.280 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:53.280 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:53.280 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:09:53.280 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:53.280 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:53.280 | INFO | Uploaded 36GiB to GPU in 1.60122s (22.4829GiB/s) | 20260505 05:09:53.280 | INFO | Downloaded 34GiB to GPU in 1.51781s (22.4007GiB/s) | 20260505 05:09:58.418 | INFO | Analyzing chunk frames = [588288,588800) | 20260505 05:09:58.418 | INFO | MemInfo: { | 20260505 05:09:58.418 | INFO | "gpu" : " 17 GB", | 20260505 05:09:58.418 | INFO | "huge" : "", | 20260505 05:09:58.418 | INFO | "huge_shared" : " 27 GB", | 20260505 05:09:58.418 | INFO | "malloc" : " 14 GB", | 20260505 05:09:58.418 | INFO | "pinned" : " 29 GB" | 20260505 05:09:58.418 | INFO | } | 20260505 05:09:58.418 | INFO | Perf report for the last 5.1382 seconds: | 20260505 05:09:58.418 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:09:58.418 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:58.418 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:58.418 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:58.418 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260505 05:09:58.418 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:58.418 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:58.418 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 05:09:58.418 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:58.418 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:09:58.419 | INFO | Uploaded 36GiB to GPU in 1.6016s (22.4775GiB/s) | 20260505 05:09:58.419 | INFO | Downloaded 34GiB to GPU in 1.51809s (22.3965GiB/s) | 20260505 05:10:03.558 | INFO | Analyzing chunk frames = [588800,589312) | 20260505 05:10:03.558 | INFO | MemInfo: { | 20260505 05:10:03.558 | INFO | "gpu" : " 17 GB", | 20260505 05:10:03.558 | INFO | "huge" : "", | 20260505 05:10:03.558 | INFO | "huge_shared" : " 27 GB", | 20260505 05:10:03.558 | INFO | "malloc" : " 14 GB", | 20260505 05:10:03.558 | INFO | "pinned" : " 29 GB" | 20260505 05:10:03.558 | INFO | } | 20260505 05:10:03.559 | INFO | Perf report for the last 5.14086 seconds: | 20260505 05:10:03.559 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:10:03.559 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:03.559 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:03.559 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:03.559 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260505 05:10:03.559 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:03.559 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:03.559 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 05:10:03.559 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:03.559 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:03.559 | INFO | Uploaded 36GiB to GPU in 1.60303s (22.4575GiB/s) | 20260505 05:10:03.559 | INFO | Downloaded 34GiB to GPU in 1.52087s (22.3557GiB/s) | 20260505 05:10:08.697 | INFO | Analyzing chunk frames = [589312,589824) | 20260505 05:10:08.697 | INFO | MemInfo: { | 20260505 05:10:08.697 | INFO | "gpu" : " 17 GB", | 20260505 05:10:08.697 | INFO | "huge" : "", | 20260505 05:10:08.697 | INFO | "huge_shared" : " 27 GB", | 20260505 05:10:08.697 | INFO | "malloc" : " 14 GB", | 20260505 05:10:08.697 | INFO | "pinned" : " 29 GB" | 20260505 05:10:08.697 | INFO | } | 20260505 05:10:08.697 | INFO | Perf report for the last 5.13802 seconds: | 20260505 05:10:08.697 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:10:08.697 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:08.697 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:08.697 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:08.697 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260505 05:10:08.697 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:08.697 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:08.697 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 05:10:08.697 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:08.697 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:08.697 | INFO | Uploaded 36GiB to GPU in 1.60235s (22.467GiB/s) | 20260505 05:10:08.697 | INFO | Downloaded 34GiB to GPU in 1.52004s (22.3678GiB/s) | 20260505 05:10:08.698 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 05:10:08.698 | INFO | Total mem usage: 15.1 GiB | 20260505 05:10:08.698 | INFO | All Time High Water Marks sum: 15.7 GiB (Savings: 4.09%) | 20260505 05:10:08.698 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -3.96%) | 20260505 05:10:08.698 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 05:10:08.698 | INFO | High Water Breakdown: | 20260505 05:10:08.698 | INFO | 59.7% : 8.65 GiB ( 95.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 05:10:08.698 | INFO | 40.2% : 5.82 GiB ( 87.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 05:10:08.698 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 05:10:08.698 | INFO | ------------------------------------------ | 20260505 05:10:08.698 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:10:08.698 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 05:10:08.698 | INFO | Total mem usage: 26.5 GiB | 20260505 05:10:08.698 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 05:10:08.698 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 05:10:08.698 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 05:10:08.698 | INFO | High Water Breakdown: | 20260505 05:10:08.698 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:10:08.698 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:10:08.698 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:10:08.698 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:10:08.698 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:10:08.698 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:10:08.698 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 05:10:08.698 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 05:10:08.698 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:10:08.698 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 05:10:08.698 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 05:10:08.698 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 05:10:08.698 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 05:10:08.698 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 05:10:08.698 | INFO | ------------------------------------------ | 20260505 05:10:08.698 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:10:08.698 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 05:10:08.698 | INFO | Total mem usage: 25.6 GiB | 20260505 05:10:08.698 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 05:10:08.698 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 05:10:08.698 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 05:10:08.698 | INFO | High Water Breakdown: | 20260505 05:10:08.698 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 05:10:08.698 | INFO | ------------------------------------------ | 20260505 05:10:08.698 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:10:08.698 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 05:10:08.698 | INFO | Total mem usage: 15.7 GiB | 20260505 05:10:08.698 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 05:10:08.698 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 05:10:08.698 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260505 05:10:08.698 | INFO | High Water Breakdown: | 20260505 05:10:08.698 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 05:10:08.698 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 05:10:08.698 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 05:10:08.698 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 05:10:08.698 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 05:10:08.698 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 05:10:08.698 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 05:10:08.698 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 05:10:08.698 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 05:10:08.698 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:10:08.698 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 05:10:08.698 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:10:08.698 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 05:10:08.698 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 05:10:08.698 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 05:10:08.698 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 05:10:08.698 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:10:08.698 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 05:10:08.698 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:10:08.698 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:10:08.698 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:10:08.698 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 05:10:08.698 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 05:10:08.698 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:10:08.698 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 05:10:08.698 | INFO | ------------------------------------------ | 20260505 05:10:08.698 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:10:08.698 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 05:10:08.698 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:10:08.698 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:10:08.698 | INFO | PerfReport: Full Execution -- 3.04e+01 / -nan, 1.28e+01 / 0.00e+00 | 20260505 05:10:08.698 | INFO | PerfReport: 0.28%, AggregateStats -- 8.65e-02 / -nan, 4.90e-03 / 0.00e+00 | 20260505 05:10:08.698 | INFO | PerfReport: 10.85%, Baseline -- 3.30e+00 / -nan, 1.78e-02 / 0.00e+00 | 20260505 05:10:08.699 | INFO | PerfReport: 3.04%, Histogram -- 9.24e-01 / -nan, 5.21e-02 / 0.00e+00 | 20260505 05:10:08.699 | INFO | PerfReport: 16.04%, DME -- 3.90e+01 / -nan, 3.15e+00 / 0.00e+00 | 20260505 05:10:08.699 | INFO | PerfReport: 56.67%, FrameLabeling -- 1.72e+01 / -nan, 4.67e-02 / 0.00e+00 | 20260505 05:10:08.699 | INFO | PerfReport: 7.58%, PulseAccumulating -- 2.30e+00 / -nan, 1.51e-01 / 0.00e+00 | 20260505 05:10:08.699 | INFO | PerfReport: 3.92%, Metrics -- 1.19e+00 / -nan, 1.98e-01 / 0.00e+00 | 20260505 05:10:08.699 | INFO | PerfReport: 1.61%, PulseSerializing -- 4.90e-01 / -nan, 7.93e-02 / 0.00e+00 | 20260505 05:10:08.699 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 05:10:08.699 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:10:08.699 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:10:08.699 | INFO | PerfReport: Full Execution -- 9.12e+01 / -nan, 1.90e+01 / 0.00e+00 | 20260505 05:10:08.699 | INFO | PerfReport: 18.36%, Upload -- 1.67e+01 / 0.00e+00, 3.77e-01 / 0.00e+00 | 20260505 05:10:08.699 | INFO | PerfReport: 17.63%, Download -- 1.61e+01 / 0.00e+00, 7.08e-01 / 0.00e+00 | 20260505 05:10:08.699 | INFO | PerfReport: 33.31%, Compute -- 3.04e+01 / 0.00e+00, 1.28e+01 / 0.00e+00 | 20260505 05:10:08.699 | INFO | PerfReport: 0.55%, LockUpload -- 5.04e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260505 05:10:08.699 | INFO | PerfReport: 0.00%, LockDownload -- 4.02e-04 / 0.00e+00, 2.56e-04 / 0.00e+00 | 20260505 05:10:08.699 | INFO | PerfReport: 30.14%, LockCompute -- 2.75e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260505 05:10:08.699 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.50e-02 / 0.00e+00 | 20260505 05:10:08.794 | INFO | Starting a round of Baz IO | 20260505 05:10:11.739 | INFO | Populating RTMetrics report | 20260505 05:10:11.739 | INFO | RTMetrics report populated | 20260505 05:10:11.739 | INFO | Opening tmp RTMetrics json file | 20260505 05:10:11.742 | INFO | Serializing RTMetrics to json | 20260505 05:10:11.745 | INFO | Serialization done, swapping tmp file now... | 20260505 05:10:11.746 | INFO | RTMetrics done | 20260505 05:10:11.843 | INFO | Finished a round of Baz IO: Wrote 2.173 GiB in 3.048 seconds | 20260505 05:10:11.843 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.96% / 0.09% / 58.94% | 20260505 05:10:11.843 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.33696% / 0.000194709% / 3.97364e-05% | 20260505 05:10:11.843 | INFO | Baz IO overall: | 20260505 05:10:11.843 | INFO | BytesWritten : 265223353310 | 20260505 05:10:11.843 | INFO | NumEvents : 124149413213 | 20260505 05:10:13.836 | INFO | Analyzing chunk frames = [589824,590336) | 20260505 05:10:13.836 | INFO | MemInfo: { | 20260505 05:10:13.836 | INFO | "gpu" : " 17 GB", | 20260505 05:10:13.836 | INFO | "huge" : "", | 20260505 05:10:13.836 | INFO | "huge_shared" : " 27 GB", | 20260505 05:10:13.836 | INFO | "malloc" : " 14 GB", | 20260505 05:10:13.836 | INFO | "pinned" : " 29 GB" | 20260505 05:10:13.836 | INFO | } | 20260505 05:10:13.837 | INFO | Perf report for the last 5.13938 seconds: | 20260505 05:10:13.837 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:10:13.837 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:13.837 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:13.837 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:13.837 | INFO | ANALYSIS : 58.1%, 3.0, 89.9ms, 0 | 20260505 05:10:13.837 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:13.837 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260505 05:10:13.837 | INFO | PRE_HQ : 57.4%, 2.2, 64.4ms, 0 | 20260505 05:10:13.837 | INFO | BAZWRITER : 58.9%, 4.2, 128.3ms, 0 | 20260505 05:10:13.837 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 05:10:13.837 | INFO | Uploaded 36GiB to GPU in 1.64859s (21.8368GiB/s) | 20260505 05:10:13.837 | INFO | Downloaded 38GiB to GPU in 1.71489s (22.1588GiB/s) | 20260505 05:10:18.976 | INFO | Analyzing chunk frames = [590336,590848) | 20260505 05:10:18.976 | INFO | MemInfo: { | 20260505 05:10:18.976 | INFO | "gpu" : " 17 GB", | 20260505 05:10:18.976 | INFO | "huge" : "", | 20260505 05:10:18.976 | INFO | "huge_shared" : " 27 GB", | 20260505 05:10:18.976 | INFO | "malloc" : " 14 GB", | 20260505 05:10:18.976 | INFO | "pinned" : " 29 GB" | 20260505 05:10:18.976 | INFO | } | 20260505 05:10:18.976 | INFO | Perf report for the last 5.1392 seconds: | 20260505 05:10:18.976 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:10:18.976 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:18.976 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:18.976 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:18.976 | INFO | ANALYSIS : 57.8%, 3.0, 90.2ms, 0 | 20260505 05:10:18.976 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:18.976 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:18.976 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 05:10:18.976 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:18.976 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:18.976 | INFO | Uploaded 36GiB to GPU in 1.59782s (22.5307GiB/s) | 20260505 05:10:18.976 | INFO | Downloaded 34GiB to GPU in 1.5171s (22.4111GiB/s) | 20260505 05:10:24.115 | INFO | Analyzing chunk frames = [590848,591360) | 20260505 05:10:24.115 | INFO | MemInfo: { | 20260505 05:10:24.115 | INFO | "gpu" : " 17 GB", | 20260505 05:10:24.115 | INFO | "huge" : "", | 20260505 05:10:24.115 | INFO | "huge_shared" : " 27 GB", | 20260505 05:10:24.115 | INFO | "malloc" : " 14 GB", | 20260505 05:10:24.115 | INFO | "pinned" : " 29 GB" | 20260505 05:10:24.115 | INFO | } | 20260505 05:10:24.115 | INFO | Perf report for the last 5.13936 seconds: | 20260505 05:10:24.115 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:10:24.115 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:24.115 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:24.115 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:24.115 | INFO | ANALYSIS : 57.2%, 3.0, 89.0ms, 0 | 20260505 05:10:24.115 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:24.115 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:24.115 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260505 05:10:24.115 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:24.115 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:24.115 | INFO | Uploaded 36GiB to GPU in 1.60112s (22.4843GiB/s) | 20260505 05:10:24.115 | INFO | Downloaded 34GiB to GPU in 1.52164s (22.3443GiB/s) | 20260505 05:10:29.255 | INFO | Analyzing chunk frames = [591360,591872) | 20260505 05:10:29.255 | INFO | MemInfo: { | 20260505 05:10:29.255 | INFO | "gpu" : " 17 GB", | 20260505 05:10:29.255 | INFO | "huge" : "", | 20260505 05:10:29.255 | INFO | "huge_shared" : " 27 GB", | 20260505 05:10:29.255 | INFO | "malloc" : " 14 GB", | 20260505 05:10:29.255 | INFO | "pinned" : " 29 GB" | 20260505 05:10:29.255 | INFO | } | 20260505 05:10:29.255 | INFO | Perf report for the last 5.13948 seconds: | 20260505 05:10:29.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:10:29.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:29.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:29.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:29.255 | INFO | ANALYSIS : 56.9%, 3.0, 88.3ms, 0 | 20260505 05:10:29.255 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:29.255 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:29.255 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:10:29.255 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:29.255 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:29.255 | INFO | Uploaded 36GiB to GPU in 1.59987s (22.5018GiB/s) | 20260505 05:10:29.255 | INFO | Downloaded 34GiB to GPU in 1.52098s (22.3541GiB/s) | 20260505 05:10:34.394 | INFO | Analyzing chunk frames = [591872,592384) | 20260505 05:10:34.394 | INFO | MemInfo: { | 20260505 05:10:34.394 | INFO | "gpu" : " 17 GB", | 20260505 05:10:34.394 | INFO | "huge" : "", | 20260505 05:10:34.394 | INFO | "huge_shared" : " 27 GB", | 20260505 05:10:34.394 | INFO | "malloc" : " 14 GB", | 20260505 05:10:34.394 | INFO | "pinned" : " 29 GB" | 20260505 05:10:34.394 | INFO | } | 20260505 05:10:34.394 | INFO | Perf report for the last 5.13917 seconds: | 20260505 05:10:34.394 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:10:34.394 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:34.394 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:34.394 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:34.394 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260505 05:10:34.394 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:34.394 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:34.394 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 05:10:34.394 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:34.394 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:34.394 | INFO | Uploaded 36GiB to GPU in 1.59891s (22.5154GiB/s) | 20260505 05:10:34.394 | INFO | Downloaded 34GiB to GPU in 1.51882s (22.3857GiB/s) | 20260505 05:10:39.533 | INFO | Analyzing chunk frames = [592384,592896) | 20260505 05:10:39.533 | INFO | MemInfo: { | 20260505 05:10:39.533 | INFO | "gpu" : " 17 GB", | 20260505 05:10:39.533 | INFO | "huge" : "", | 20260505 05:10:39.533 | INFO | "huge_shared" : " 27 GB", | 20260505 05:10:39.533 | INFO | "malloc" : " 14 GB", | 20260505 05:10:39.533 | INFO | "pinned" : " 29 GB" | 20260505 05:10:39.533 | INFO | } | 20260505 05:10:39.534 | INFO | Perf report for the last 5.14056 seconds: | 20260505 05:10:39.534 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:10:39.534 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:39.534 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:39.534 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:39.534 | INFO | ANALYSIS : 57.1%, 3.0, 89.2ms, 0 | 20260505 05:10:39.534 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:39.534 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:39.534 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260505 05:10:39.534 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:39.534 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:39.535 | INFO | Uploaded 36GiB to GPU in 1.59938s (22.5087GiB/s) | 20260505 05:10:39.535 | INFO | Downloaded 34GiB to GPU in 1.51915s (22.3809GiB/s) | 20260505 05:10:44.672 | INFO | Analyzing chunk frames = [592896,593408) | 20260505 05:10:44.672 | INFO | MemInfo: { | 20260505 05:10:44.672 | INFO | "gpu" : " 17 GB", | 20260505 05:10:44.672 | INFO | "huge" : "", | 20260505 05:10:44.672 | INFO | "huge_shared" : " 27 GB", | 20260505 05:10:44.672 | INFO | "malloc" : " 14 GB", | 20260505 05:10:44.673 | INFO | "pinned" : " 29 GB" | 20260505 05:10:44.673 | INFO | } | 20260505 05:10:44.673 | INFO | Perf report for the last 5.13831 seconds: | 20260505 05:10:44.673 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:10:44.673 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:44.673 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:44.673 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:44.673 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260505 05:10:44.673 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:44.673 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:44.673 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 05:10:44.673 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:44.673 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:44.673 | INFO | Uploaded 36GiB to GPU in 1.59963s (22.5052GiB/s) | 20260505 05:10:44.673 | INFO | Downloaded 34GiB to GPU in 1.52005s (22.3677GiB/s) | 20260505 05:10:49.812 | INFO | Analyzing chunk frames = [593408,593920) | 20260505 05:10:49.812 | INFO | MemInfo: { | 20260505 05:10:49.812 | INFO | "gpu" : " 17 GB", | 20260505 05:10:49.812 | INFO | "huge" : "", | 20260505 05:10:49.812 | INFO | "huge_shared" : " 27 GB", | 20260505 05:10:49.812 | INFO | "malloc" : " 14 GB", | 20260505 05:10:49.812 | INFO | "pinned" : " 29 GB" | 20260505 05:10:49.812 | INFO | } | 20260505 05:10:49.812 | INFO | Perf report for the last 5.13935 seconds: | 20260505 05:10:49.812 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:10:49.812 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:49.812 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:49.812 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:49.812 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260505 05:10:49.812 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:49.812 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:49.812 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:10:49.812 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:49.812 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:49.812 | INFO | Uploaded 36GiB to GPU in 1.60006s (22.4992GiB/s) | 20260505 05:10:49.812 | INFO | Downloaded 34GiB to GPU in 1.52094s (22.3546GiB/s) | 20260505 05:10:49.909 | INFO | Starting a round of Baz IO | 20260505 05:10:52.855 | INFO | Populating RTMetrics report | 20260505 05:10:52.856 | INFO | RTMetrics report populated | 20260505 05:10:52.856 | INFO | Opening tmp RTMetrics json file | 20260505 05:10:52.861 | INFO | Serializing RTMetrics to json | 20260505 05:10:52.864 | INFO | Serialization done, swapping tmp file now... | 20260505 05:10:52.865 | INFO | RTMetrics done | 20260505 05:10:52.956 | INFO | Finished a round of Baz IO: Wrote 2.175 GiB in 3.046 seconds | 20260505 05:10:52.956 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.87% / 0.09% / 59.04% | 20260505 05:10:52.956 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.29419% / 0.000194709% / 2.78155e-05% | 20260505 05:10:52.956 | INFO | Baz IO overall: | 20260505 05:10:52.956 | INFO | BytesWritten : 267558519457 | 20260505 05:10:52.956 | INFO | NumEvents : 125260657943 | 20260505 05:10:54.952 | INFO | Analyzing chunk frames = [593920,594432) | 20260505 05:10:54.952 | INFO | MemInfo: { | 20260505 05:10:54.952 | INFO | "gpu" : " 17 GB", | 20260505 05:10:54.952 | INFO | "huge" : "", | 20260505 05:10:54.952 | INFO | "huge_shared" : " 27 GB", | 20260505 05:10:54.952 | INFO | "malloc" : " 14 GB", | 20260505 05:10:54.952 | INFO | "pinned" : " 29 GB" | 20260505 05:10:54.952 | INFO | } | 20260505 05:10:54.952 | INFO | Perf report for the last 5.14038 seconds: | 20260505 05:10:54.952 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:10:54.952 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:54.952 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:54.952 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:54.952 | INFO | ANALYSIS : 58.0%, 3.0, 90.1ms, 0 | 20260505 05:10:54.953 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:10:54.953 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260505 05:10:54.953 | INFO | PRE_HQ : 57.4%, 2.2, 64.3ms, 0 | 20260505 05:10:54.953 | INFO | BAZWRITER : 58.8%, 4.3, 129.3ms, 0 | 20260505 05:10:54.953 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 05:10:54.953 | INFO | Uploaded 36GiB to GPU in 1.64247s (21.9182GiB/s) | 20260505 05:10:54.953 | INFO | Downloaded 38GiB to GPU in 1.70826s (22.2448GiB/s) | 20260505 05:11:00.090 | INFO | Analyzing chunk frames = [594432,594944) | 20260505 05:11:00.090 | INFO | MemInfo: { | 20260505 05:11:00.090 | INFO | "gpu" : " 17 GB", | 20260505 05:11:00.090 | INFO | "huge" : "", | 20260505 05:11:00.090 | INFO | "huge_shared" : " 27 GB", | 20260505 05:11:00.090 | INFO | "malloc" : " 14 GB", | 20260505 05:11:00.091 | INFO | "pinned" : " 29 GB" | 20260505 05:11:00.091 | INFO | } | 20260505 05:11:00.091 | INFO | Perf report for the last 5.13829 seconds: | 20260505 05:11:00.091 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:11:00.091 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:00.091 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:00.091 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:00.091 | INFO | ANALYSIS : 57.9%, 3.0, 90.3ms, 0 | 20260505 05:11:00.091 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:00.091 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:00.091 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 05:11:00.091 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:00.091 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:00.091 | INFO | Uploaded 36GiB to GPU in 1.5999s (22.5015GiB/s) | 20260505 05:11:00.091 | INFO | Downloaded 34GiB to GPU in 1.51651s (22.4199GiB/s) | 20260505 05:11:05.230 | INFO | Analyzing chunk frames = [594944,595456) | 20260505 05:11:05.230 | INFO | MemInfo: { | 20260505 05:11:05.230 | INFO | "gpu" : " 17 GB", | 20260505 05:11:05.230 | INFO | "huge" : "", | 20260505 05:11:05.230 | INFO | "huge_shared" : " 27 GB", | 20260505 05:11:05.230 | INFO | "malloc" : " 14 GB", | 20260505 05:11:05.230 | INFO | "pinned" : " 29 GB" | 20260505 05:11:05.230 | INFO | } | 20260505 05:11:05.230 | INFO | Perf report for the last 5.13911 seconds: | 20260505 05:11:05.230 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:11:05.230 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:05.230 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:05.230 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:05.230 | INFO | ANALYSIS : 57.2%, 3.0, 88.9ms, 0 | 20260505 05:11:05.230 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:05.230 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:05.230 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 05:11:05.230 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:05.230 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:05.230 | INFO | Uploaded 36GiB to GPU in 1.60425s (22.4404GiB/s) | 20260505 05:11:05.231 | INFO | Downloaded 34GiB to GPU in 1.52133s (22.3489GiB/s) | 20260505 05:11:10.369 | INFO | Analyzing chunk frames = [595456,595968) | 20260505 05:11:10.369 | INFO | MemInfo: { | 20260505 05:11:10.369 | INFO | "gpu" : " 17 GB", | 20260505 05:11:10.369 | INFO | "huge" : "", | 20260505 05:11:10.369 | INFO | "huge_shared" : " 27 GB", | 20260505 05:11:10.369 | INFO | "malloc" : " 14 GB", | 20260505 05:11:10.369 | INFO | "pinned" : " 29 GB" | 20260505 05:11:10.369 | INFO | } | 20260505 05:11:10.369 | INFO | Perf report for the last 5.13963 seconds: | 20260505 05:11:10.369 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:11:10.369 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:10.370 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:10.370 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:10.370 | INFO | ANALYSIS : 56.9%, 3.0, 88.3ms, 0 | 20260505 05:11:10.370 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:10.370 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:10.370 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 05:11:10.370 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:10.370 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:10.370 | INFO | Uploaded 36GiB to GPU in 1.60325s (22.4544GiB/s) | 20260505 05:11:10.370 | INFO | Downloaded 34GiB to GPU in 1.52004s (22.3678GiB/s) | 20260505 05:11:15.508 | INFO | Analyzing chunk frames = [595968,596480) | 20260505 05:11:15.508 | INFO | MemInfo: { | 20260505 05:11:15.508 | INFO | "gpu" : " 17 GB", | 20260505 05:11:15.508 | INFO | "huge" : "", | 20260505 05:11:15.508 | INFO | "huge_shared" : " 27 GB", | 20260505 05:11:15.508 | INFO | "malloc" : " 14 GB", | 20260505 05:11:15.508 | INFO | "pinned" : " 29 GB" | 20260505 05:11:15.508 | INFO | } | 20260505 05:11:15.508 | INFO | Perf report for the last 5.139 seconds: | 20260505 05:11:15.509 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:11:15.509 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:15.509 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:15.509 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:15.509 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260505 05:11:15.509 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:15.509 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:15.509 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 05:11:15.509 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:15.509 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:15.509 | INFO | Uploaded 36GiB to GPU in 1.60196s (22.4725GiB/s) | 20260505 05:11:15.509 | INFO | Downloaded 34GiB to GPU in 1.51884s (22.3854GiB/s) | 20260505 05:11:20.648 | INFO | Analyzing chunk frames = [596480,596992) | 20260505 05:11:20.648 | INFO | MemInfo: { | 20260505 05:11:20.648 | INFO | "gpu" : " 17 GB", | 20260505 05:11:20.648 | INFO | "huge" : "", | 20260505 05:11:20.648 | INFO | "huge_shared" : " 27 GB", | 20260505 05:11:20.648 | INFO | "malloc" : " 14 GB", | 20260505 05:11:20.648 | INFO | "pinned" : " 29 GB" | 20260505 05:11:20.648 | INFO | } | 20260505 05:11:20.648 | INFO | Perf report for the last 5.13986 seconds: | 20260505 05:11:20.648 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:11:20.648 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:20.648 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:20.648 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:20.648 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260505 05:11:20.648 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:20.648 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:20.648 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260505 05:11:20.649 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:20.649 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:20.649 | INFO | Uploaded 36GiB to GPU in 1.60205s (22.4712GiB/s) | 20260505 05:11:20.649 | INFO | Downloaded 34GiB to GPU in 1.51879s (22.3863GiB/s) | 20260505 05:11:25.787 | INFO | Analyzing chunk frames = [596992,597504) | 20260505 05:11:25.787 | INFO | MemInfo: { | 20260505 05:11:25.787 | INFO | "gpu" : " 17 GB", | 20260505 05:11:25.787 | INFO | "huge" : "", | 20260505 05:11:25.787 | INFO | "huge_shared" : " 27 GB", | 20260505 05:11:25.787 | INFO | "malloc" : " 14 GB", | 20260505 05:11:25.787 | INFO | "pinned" : " 29 GB" | 20260505 05:11:25.787 | INFO | } | 20260505 05:11:25.787 | INFO | Perf report for the last 5.13909 seconds: | 20260505 05:11:25.787 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:11:25.787 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:25.787 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:25.788 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:25.788 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 05:11:25.788 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:25.788 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:25.788 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260505 05:11:25.788 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:25.788 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:25.788 | INFO | Uploaded 36GiB to GPU in 1.6022s (22.4691GiB/s) | 20260505 05:11:25.788 | INFO | Downloaded 34GiB to GPU in 1.5201s (22.367GiB/s) | 20260505 05:11:30.927 | INFO | Analyzing chunk frames = [597504,598016) | 20260505 05:11:30.927 | INFO | MemInfo: { | 20260505 05:11:30.927 | INFO | "gpu" : " 17 GB", | 20260505 05:11:30.927 | INFO | "huge" : "", | 20260505 05:11:30.927 | INFO | "huge_shared" : " 27 GB", | 20260505 05:11:30.927 | INFO | "malloc" : " 14 GB", | 20260505 05:11:30.927 | INFO | "pinned" : " 29 GB" | 20260505 05:11:30.927 | INFO | } | 20260505 05:11:30.927 | INFO | Perf report for the last 5.1393 seconds: | 20260505 05:11:30.927 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:11:30.927 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:30.927 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:30.927 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:30.927 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260505 05:11:30.927 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:30.927 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:30.927 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:11:30.927 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:30.927 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:30.927 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 05:11:30.927 | INFO | Total mem usage: 15.1 GiB | 20260505 05:11:30.927 | INFO | All Time High Water Marks sum: 15.7 GiB (Savings: 4.09%) | 20260505 05:11:30.927 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -3.77%) | 20260505 05:11:30.927 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 05:11:30.927 | INFO | High Water Breakdown: | 20260505 05:11:30.927 | INFO | 59.8% : 8.68 GiB ( 96.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 05:11:30.927 | INFO | 40.1% : 5.81 GiB ( 87.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 05:11:30.927 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 05:11:30.927 | INFO | ------------------------------------------ | 20260505 05:11:30.928 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:11:30.928 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 05:11:30.928 | INFO | Total mem usage: 26.5 GiB | 20260505 05:11:30.928 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 05:11:30.928 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 05:11:30.928 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 05:11:30.928 | INFO | Uploaded 36GiB to GPU in 1.60242s (22.466GiB/s) | 20260505 05:11:30.928 | INFO | High Water Breakdown: | 20260505 05:11:30.928 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:11:30.928 | INFO | Downloaded 34GiB to GPU in 1.51952s (22.3755GiB/s) | 20260505 05:11:30.928 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:11:30.928 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:11:30.928 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:11:30.928 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:11:30.928 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:11:30.928 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 05:11:30.928 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 05:11:30.928 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:11:30.928 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 05:11:30.928 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 05:11:30.928 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 05:11:30.928 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 05:11:30.928 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 05:11:30.928 | INFO | ------------------------------------------ | 20260505 05:11:30.928 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:11:30.928 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 05:11:30.928 | INFO | Total mem usage: 25.6 GiB | 20260505 05:11:30.928 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 05:11:30.928 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 05:11:30.928 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 05:11:30.928 | INFO | High Water Breakdown: | 20260505 05:11:30.928 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 05:11:30.928 | INFO | ------------------------------------------ | 20260505 05:11:30.928 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:11:30.928 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 05:11:30.928 | INFO | Total mem usage: 15.7 GiB | 20260505 05:11:30.928 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 05:11:30.928 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 05:11:30.928 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260505 05:11:30.928 | INFO | High Water Breakdown: | 20260505 05:11:30.928 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 05:11:30.928 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 05:11:30.928 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 05:11:30.928 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 05:11:30.928 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 05:11:30.928 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 05:11:30.928 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 05:11:30.928 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 05:11:30.928 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 05:11:30.928 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:11:30.928 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 05:11:30.928 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:11:30.928 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 05:11:30.928 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 05:11:30.928 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 05:11:30.928 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 05:11:30.928 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:11:30.928 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 05:11:30.928 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:11:30.928 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:11:30.928 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:11:30.928 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 05:11:30.928 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 05:11:30.928 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:11:30.928 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 05:11:30.928 | INFO | ------------------------------------------ | 20260505 05:11:30.928 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:11:30.928 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 05:11:30.928 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:11:30.928 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:11:30.928 | INFO | PerfReport: Full Execution -- 3.04e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260505 05:11:30.928 | INFO | PerfReport: 0.28%, AggregateStats -- 8.65e-02 / -nan, 4.29e-03 / 0.00e+00 | 20260505 05:11:30.928 | INFO | PerfReport: 10.86%, Baseline -- 3.30e+00 / -nan, 1.72e-02 / 0.00e+00 | 20260505 05:11:30.928 | INFO | PerfReport: 3.04%, Histogram -- 9.24e-01 / -nan, 5.19e-02 / 0.00e+00 | 20260505 05:11:30.928 | INFO | PerfReport: 16.04%, DME -- 3.90e+01 / -nan, 3.18e+00 / 0.00e+00 | 20260505 05:11:30.928 | INFO | PerfReport: 56.67%, FrameLabeling -- 1.72e+01 / -nan, 4.51e-02 / 0.00e+00 | 20260505 05:11:30.928 | INFO | PerfReport: 7.57%, PulseAccumulating -- 2.30e+00 / -nan, 1.51e-01 / 0.00e+00 | 20260505 05:11:30.929 | INFO | PerfReport: 3.91%, Metrics -- 1.19e+00 / -nan, 1.96e-01 / 0.00e+00 | 20260505 05:11:30.929 | INFO | PerfReport: 1.61%, PulseSerializing -- 4.88e-01 / -nan, 7.87e-02 / 0.00e+00 | 20260505 05:11:30.929 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 05:11:30.929 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:11:30.929 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:11:30.929 | INFO | PerfReport: Full Execution -- 9.11e+01 / -nan, 1.90e+01 / 0.00e+00 | 20260505 05:11:30.929 | INFO | PerfReport: 18.36%, Upload -- 1.67e+01 / 0.00e+00, 3.88e-01 / 0.00e+00 | 20260505 05:11:30.929 | INFO | PerfReport: 17.64%, Download -- 1.61e+01 / 0.00e+00, 7.28e-01 / 0.00e+00 | 20260505 05:11:30.929 | INFO | PerfReport: 33.32%, Compute -- 3.04e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260505 05:11:30.929 | INFO | PerfReport: 0.56%, LockUpload -- 5.07e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260505 05:11:30.929 | INFO | PerfReport: 0.00%, LockDownload -- 3.93e-04 / 0.00e+00, 9.09e-05 / 0.00e+00 | 20260505 05:11:30.929 | INFO | PerfReport: 30.11%, LockCompute -- 2.74e+01 / 0.00e+00, 1.72e+01 / 0.00e+00 | 20260505 05:11:30.929 | INFO | PerfReport: 0.02%, Misc -- 1.39e-02 / 0.00e+00, 1.77e-02 / 0.00e+00 | 20260505 05:11:31.024 | INFO | Starting a round of Baz IO | 20260505 05:11:33.969 | INFO | Populating RTMetrics report | 20260505 05:11:33.969 | INFO | RTMetrics report populated | 20260505 05:11:33.969 | INFO | Opening tmp RTMetrics json file | 20260505 05:11:33.976 | INFO | Serializing RTMetrics to json | 20260505 05:11:33.979 | INFO | Serialization done, swapping tmp file now... | 20260505 05:11:33.983 | INFO | RTMetrics done | 20260505 05:11:34.076 | INFO | Finished a round of Baz IO: Wrote 2.177 GiB in 3.051 seconds | 20260505 05:11:34.076 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.78% / 0.09% / 59.13% | 20260505 05:11:34.076 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.24477% / 0.000170867% / 5.16574e-05% | 20260505 05:11:34.076 | INFO | Baz IO overall: | 20260505 05:11:34.076 | INFO | BytesWritten : 269895992696 | 20260505 05:11:34.076 | INFO | NumEvents : 126372568430 | 20260505 05:11:36.066 | INFO | Analyzing chunk frames = [598016,598528) | 20260505 05:11:36.066 | INFO | MemInfo: { | 20260505 05:11:36.066 | INFO | "gpu" : " 17 GB", | 20260505 05:11:36.066 | INFO | "huge" : "", | 20260505 05:11:36.066 | INFO | "huge_shared" : " 27 GB", | 20260505 05:11:36.066 | INFO | "malloc" : " 14 GB", | 20260505 05:11:36.066 | INFO | "pinned" : " 29 GB" | 20260505 05:11:36.066 | INFO | } | 20260505 05:11:36.066 | INFO | Perf report for the last 5.13926 seconds: | 20260505 05:11:36.066 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:11:36.066 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:36.066 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:36.066 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:36.066 | INFO | ANALYSIS : 58.0%, 2.9, 89.6ms, 0 | 20260505 05:11:36.066 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:36.066 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 05:11:36.066 | INFO | PRE_HQ : 57.4%, 2.2, 64.4ms, 0 | 20260505 05:11:36.066 | INFO | BAZWRITER : 58.9%, 4.3, 130.6ms, 0 | 20260505 05:11:36.066 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 05:11:36.066 | INFO | Uploaded 36GiB to GPU in 1.65188s (21.7933GiB/s) | 20260505 05:11:36.067 | INFO | Downloaded 38GiB to GPU in 1.71667s (22.1359GiB/s) | 20260505 05:11:41.205 | INFO | Analyzing chunk frames = [598528,599040) | 20260505 05:11:41.205 | INFO | MemInfo: { | 20260505 05:11:41.205 | INFO | "gpu" : " 17 GB", | 20260505 05:11:41.205 | INFO | "huge" : "", | 20260505 05:11:41.205 | INFO | "huge_shared" : " 27 GB", | 20260505 05:11:41.205 | INFO | "malloc" : " 14 GB", | 20260505 05:11:41.205 | INFO | "pinned" : " 29 GB" | 20260505 05:11:41.205 | INFO | } | 20260505 05:11:41.205 | INFO | Perf report for the last 5.13938 seconds: | 20260505 05:11:41.205 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:11:41.205 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:41.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:41.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:41.205 | INFO | ANALYSIS : 57.9%, 3.0, 90.3ms, 0 | 20260505 05:11:41.205 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:41.206 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:41.206 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 05:11:41.206 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:41.206 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:41.206 | INFO | Uploaded 36GiB to GPU in 1.59721s (22.5392GiB/s) | 20260505 05:11:41.206 | INFO | Downloaded 34GiB to GPU in 1.51735s (22.4074GiB/s) | 20260505 05:11:46.345 | INFO | Analyzing chunk frames = [599040,599552) | 20260505 05:11:46.345 | INFO | MemInfo: { | 20260505 05:11:46.345 | INFO | "gpu" : " 17 GB", | 20260505 05:11:46.345 | INFO | "huge" : "", | 20260505 05:11:46.345 | INFO | "huge_shared" : " 27 GB", | 20260505 05:11:46.345 | INFO | "malloc" : " 14 GB", | 20260505 05:11:46.345 | INFO | "pinned" : " 29 GB" | 20260505 05:11:46.345 | INFO | } | 20260505 05:11:46.345 | INFO | Perf report for the last 5.13914 seconds: | 20260505 05:11:46.345 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:11:46.345 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:46.345 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:46.345 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:46.345 | INFO | ANALYSIS : 57.2%, 3.0, 88.8ms, 0 | 20260505 05:11:46.345 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:46.345 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:46.345 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 05:11:46.345 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:46.345 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:46.345 | INFO | Uploaded 36GiB to GPU in 1.60198s (22.4722GiB/s) | 20260505 05:11:46.345 | INFO | Downloaded 34GiB to GPU in 1.52215s (22.3368GiB/s) | 20260505 05:11:51.484 | INFO | Analyzing chunk frames = [599552,600064) | 20260505 05:11:51.484 | INFO | MemInfo: { | 20260505 05:11:51.484 | INFO | "gpu" : " 17 GB", | 20260505 05:11:51.484 | INFO | "huge" : "", | 20260505 05:11:51.484 | INFO | "huge_shared" : " 27 GB", | 20260505 05:11:51.484 | INFO | "malloc" : " 14 GB", | 20260505 05:11:51.484 | INFO | "pinned" : " 29 GB" | 20260505 05:11:51.484 | INFO | } | 20260505 05:11:51.484 | INFO | Perf report for the last 5.13956 seconds: | 20260505 05:11:51.484 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:11:51.484 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:51.484 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:51.484 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:51.484 | INFO | ANALYSIS : 56.9%, 3.0, 88.3ms, 0 | 20260505 05:11:51.484 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:51.484 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:51.484 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:11:51.484 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:51.484 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:51.484 | INFO | Uploaded 36GiB to GPU in 1.60059s (22.4918GiB/s) | 20260505 05:11:51.485 | INFO | Downloaded 34GiB to GPU in 1.52128s (22.3496GiB/s) | 20260505 05:11:56.623 | INFO | Analyzing chunk frames = [600064,600576) | 20260505 05:11:56.623 | INFO | MemInfo: { | 20260505 05:11:56.623 | INFO | "gpu" : " 17 GB", | 20260505 05:11:56.623 | INFO | "huge" : "", | 20260505 05:11:56.623 | INFO | "huge_shared" : " 27 GB", | 20260505 05:11:56.623 | INFO | "malloc" : " 14 GB", | 20260505 05:11:56.623 | INFO | "pinned" : " 29 GB" | 20260505 05:11:56.623 | INFO | } | 20260505 05:11:56.623 | INFO | Perf report for the last 5.13924 seconds: | 20260505 05:11:56.623 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:11:56.623 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:56.623 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:56.623 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:56.623 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260505 05:11:56.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:56.624 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:56.624 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:11:56.624 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:56.624 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:11:56.624 | INFO | Uploaded 36GiB to GPU in 1.59955s (22.5063GiB/s) | 20260505 05:11:56.624 | INFO | Downloaded 34GiB to GPU in 1.51863s (22.3885GiB/s) | 20260505 05:12:01.763 | INFO | Analyzing chunk frames = [600576,601088) | 20260505 05:12:01.763 | INFO | MemInfo: { | 20260505 05:12:01.763 | INFO | "gpu" : " 17 GB", | 20260505 05:12:01.763 | INFO | "huge" : "", | 20260505 05:12:01.763 | INFO | "huge_shared" : " 27 GB", | 20260505 05:12:01.763 | INFO | "malloc" : " 14 GB", | 20260505 05:12:01.763 | INFO | "pinned" : " 29 GB" | 20260505 05:12:01.763 | INFO | } | 20260505 05:12:01.763 | INFO | Perf report for the last 5.1394 seconds: | 20260505 05:12:01.763 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:12:01.763 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:01.763 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:01.763 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:01.763 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260505 05:12:01.763 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:01.763 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:01.763 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 05:12:01.763 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:01.763 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:01.763 | INFO | Uploaded 36GiB to GPU in 1.59997s (22.5004GiB/s) | 20260505 05:12:01.763 | INFO | Downloaded 34GiB to GPU in 1.52046s (22.3616GiB/s) | 20260505 05:12:06.902 | INFO | Analyzing chunk frames = [601088,601600) | 20260505 05:12:06.902 | INFO | MemInfo: { | 20260505 05:12:06.902 | INFO | "gpu" : " 17 GB", | 20260505 05:12:06.902 | INFO | "huge" : "", | 20260505 05:12:06.902 | INFO | "huge_shared" : " 27 GB", | 20260505 05:12:06.902 | INFO | "malloc" : " 14 GB", | 20260505 05:12:06.902 | INFO | "pinned" : " 29 GB" | 20260505 05:12:06.902 | INFO | } | 20260505 05:12:06.902 | INFO | Perf report for the last 5.13933 seconds: | 20260505 05:12:06.902 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:12:06.902 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:06.902 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:06.902 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:06.902 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 05:12:06.902 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:06.902 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:06.902 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260505 05:12:06.902 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:06.902 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:06.903 | INFO | Uploaded 36GiB to GPU in 1.5999s (22.5014GiB/s) | 20260505 05:12:06.903 | INFO | Downloaded 34GiB to GPU in 1.52104s (22.3531GiB/s) | 20260505 05:12:12.041 | INFO | Analyzing chunk frames = [601600,602112) | 20260505 05:12:12.041 | INFO | MemInfo: { | 20260505 05:12:12.041 | INFO | "gpu" : " 17 GB", | 20260505 05:12:12.041 | INFO | "huge" : "", | 20260505 05:12:12.041 | INFO | "huge_shared" : " 27 GB", | 20260505 05:12:12.041 | INFO | "malloc" : " 14 GB", | 20260505 05:12:12.041 | INFO | "pinned" : " 29 GB" | 20260505 05:12:12.041 | INFO | } | 20260505 05:12:12.042 | INFO | Perf report for the last 5.13936 seconds: | 20260505 05:12:12.042 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:12:12.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:12.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:12.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:12.042 | INFO | ANALYSIS : 57.5%, 3.0, 89.6ms, 0 | 20260505 05:12:12.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:12.042 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:12.042 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 05:12:12.042 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:12.042 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:12.042 | INFO | Uploaded 36GiB to GPU in 1.59965s (22.505GiB/s) | 20260505 05:12:12.042 | INFO | Downloaded 34GiB to GPU in 1.52044s (22.362GiB/s) | 20260505 05:12:12.139 | INFO | Starting a round of Baz IO | 20260505 05:12:15.083 | INFO | Populating RTMetrics report | 20260505 05:12:15.084 | INFO | RTMetrics report populated | 20260505 05:12:15.084 | INFO | Opening tmp RTMetrics json file | 20260505 05:12:15.087 | INFO | Serializing RTMetrics to json | 20260505 05:12:15.088 | INFO | Serialization done, swapping tmp file now... | 20260505 05:12:15.089 | INFO | RTMetrics done | 20260505 05:12:15.182 | INFO | Finished a round of Baz IO: Wrote 2.178 GiB in 3.043 seconds | 20260505 05:12:15.182 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.69% / 0.09% / 59.22% | 20260505 05:12:15.182 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.19909% / 0.000139078% / 4.37101e-05% | 20260505 05:12:15.182 | INFO | Baz IO overall: | 20260505 05:12:15.182 | INFO | BytesWritten : 272234895071 | 20260505 05:12:15.182 | INFO | NumEvents : 127485020893 | 20260505 05:12:17.181 | INFO | Analyzing chunk frames = [602112,602624) | 20260505 05:12:17.181 | INFO | MemInfo: { | 20260505 05:12:17.181 | INFO | "gpu" : " 17 GB", | 20260505 05:12:17.181 | INFO | "huge" : "", | 20260505 05:12:17.181 | INFO | "huge_shared" : " 27 GB", | 20260505 05:12:17.181 | INFO | "malloc" : " 14 GB", | 20260505 05:12:17.181 | INFO | "pinned" : " 29 GB" | 20260505 05:12:17.181 | INFO | } | 20260505 05:12:17.181 | INFO | Perf report for the last 5.13937 seconds: | 20260505 05:12:17.181 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:12:17.181 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:17.181 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:17.181 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:17.181 | INFO | ANALYSIS : 58.0%, 2.9, 89.6ms, 0 | 20260505 05:12:17.181 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:17.181 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260505 05:12:17.181 | INFO | PRE_HQ : 57.4%, 2.2, 64.2ms, 0 | 20260505 05:12:17.181 | INFO | BAZWRITER : 58.6%, 4.3, 130.1ms, 0 | 20260505 05:12:17.181 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 05:12:17.181 | INFO | Uploaded 36GiB to GPU in 1.64705s (21.8573GiB/s) | 20260505 05:12:17.181 | INFO | Downloaded 38GiB to GPU in 1.71096s (22.2098GiB/s) | 20260505 05:12:22.320 | INFO | Analyzing chunk frames = [602624,603136) | 20260505 05:12:22.320 | INFO | MemInfo: { | 20260505 05:12:22.320 | INFO | "gpu" : " 17 GB", | 20260505 05:12:22.320 | INFO | "huge" : "", | 20260505 05:12:22.320 | INFO | "huge_shared" : " 27 GB", | 20260505 05:12:22.321 | INFO | "malloc" : " 14 GB", | 20260505 05:12:22.321 | INFO | "pinned" : " 29 GB" | 20260505 05:12:22.321 | INFO | } | 20260505 05:12:22.321 | INFO | Perf report for the last 5.13986 seconds: | 20260505 05:12:22.321 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:12:22.321 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:22.321 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:22.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:22.321 | INFO | ANALYSIS : 58.2%, 3.0, 90.8ms, 0 | 20260505 05:12:22.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:22.321 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:22.321 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:12:22.321 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:22.321 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:22.321 | INFO | Uploaded 36GiB to GPU in 1.61273s (22.3225GiB/s) | 20260505 05:12:22.321 | INFO | Downloaded 34GiB to GPU in 1.51638s (22.4218GiB/s) | 20260505 05:12:27.459 | INFO | Analyzing chunk frames = [603136,603648) | 20260505 05:12:27.459 | INFO | MemInfo: { | 20260505 05:12:27.459 | INFO | "gpu" : " 17 GB", | 20260505 05:12:27.459 | INFO | "huge" : "", | 20260505 05:12:27.459 | INFO | "huge_shared" : " 27 GB", | 20260505 05:12:27.459 | INFO | "malloc" : " 14 GB", | 20260505 05:12:27.459 | INFO | "pinned" : " 29 GB" | 20260505 05:12:27.459 | INFO | } | 20260505 05:12:27.459 | INFO | Perf report for the last 5.13867 seconds: | 20260505 05:12:27.459 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:12:27.460 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:27.460 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:27.460 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:27.460 | INFO | ANALYSIS : 57.2%, 3.0, 88.9ms, 0 | 20260505 05:12:27.460 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:27.460 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:27.460 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 05:12:27.460 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:27.460 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:27.460 | INFO | Uploaded 36GiB to GPU in 1.6038s (22.4467GiB/s) | 20260505 05:12:27.460 | INFO | Downloaded 34GiB to GPU in 1.52084s (22.356GiB/s) | 20260505 05:12:32.599 | INFO | Analyzing chunk frames = [603648,604160) | 20260505 05:12:32.599 | INFO | MemInfo: { | 20260505 05:12:32.599 | INFO | "gpu" : " 17 GB", | 20260505 05:12:32.599 | INFO | "huge" : "", | 20260505 05:12:32.599 | INFO | "huge_shared" : " 27 GB", | 20260505 05:12:32.599 | INFO | "malloc" : " 14 GB", | 20260505 05:12:32.599 | INFO | "pinned" : " 29 GB" | 20260505 05:12:32.599 | INFO | } | 20260505 05:12:32.599 | INFO | Perf report for the last 5.13952 seconds: | 20260505 05:12:32.599 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:12:32.599 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:32.599 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:32.599 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:32.599 | INFO | ANALYSIS : 57.0%, 3.0, 88.4ms, 0 | 20260505 05:12:32.599 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:32.599 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:32.599 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 05:12:32.599 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:32.599 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:32.600 | INFO | Uploaded 36GiB to GPU in 1.60351s (22.4508GiB/s) | 20260505 05:12:32.600 | INFO | Downloaded 34GiB to GPU in 1.52028s (22.3644GiB/s) | 20260505 05:12:37.738 | INFO | Analyzing chunk frames = [604160,604672) | 20260505 05:12:37.738 | INFO | MemInfo: { | 20260505 05:12:37.738 | INFO | "gpu" : " 17 GB", | 20260505 05:12:37.738 | INFO | "huge" : "", | 20260505 05:12:37.738 | INFO | "huge_shared" : " 27 GB", | 20260505 05:12:37.738 | INFO | "malloc" : " 14 GB", | 20260505 05:12:37.738 | INFO | "pinned" : " 29 GB" | 20260505 05:12:37.738 | INFO | } | 20260505 05:12:37.738 | INFO | Perf report for the last 5.13928 seconds: | 20260505 05:12:37.738 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:12:37.738 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:37.738 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:37.738 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:37.738 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260505 05:12:37.738 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:37.738 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:37.738 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:12:37.738 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:37.739 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:37.739 | INFO | Uploaded 36GiB to GPU in 1.60127s (22.4821GiB/s) | 20260505 05:12:37.739 | INFO | Downloaded 34GiB to GPU in 1.51831s (22.3933GiB/s) | 20260505 05:12:42.879 | INFO | Analyzing chunk frames = [604672,605184) | 20260505 05:12:42.879 | INFO | MemInfo: { | 20260505 05:12:42.879 | INFO | "gpu" : " 17 GB", | 20260505 05:12:42.879 | INFO | "huge" : "", | 20260505 05:12:42.879 | INFO | "huge_shared" : " 27 GB", | 20260505 05:12:42.879 | INFO | "malloc" : " 14 GB", | 20260505 05:12:42.879 | INFO | "pinned" : " 29 GB" | 20260505 05:12:42.879 | INFO | } | 20260505 05:12:42.879 | INFO | Perf report for the last 5.14054 seconds: | 20260505 05:12:42.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:12:42.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:42.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:42.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:42.879 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260505 05:12:42.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:42.879 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:42.879 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 05:12:42.879 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:42.879 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:42.879 | INFO | Uploaded 36GiB to GPU in 1.60191s (22.4731GiB/s) | 20260505 05:12:42.879 | INFO | Downloaded 34GiB to GPU in 1.51848s (22.3908GiB/s) | 20260505 05:12:48.017 | INFO | Analyzing chunk frames = [605184,605696) | 20260505 05:12:48.017 | INFO | MemInfo: { | 20260505 05:12:48.017 | INFO | "gpu" : " 17 GB", | 20260505 05:12:48.017 | INFO | "huge" : "", | 20260505 05:12:48.017 | INFO | "huge_shared" : " 27 GB", | 20260505 05:12:48.017 | INFO | "malloc" : " 14 GB", | 20260505 05:12:48.017 | INFO | "pinned" : " 29 GB" | 20260505 05:12:48.017 | INFO | } | 20260505 05:12:48.017 | INFO | Perf report for the last 5.13818 seconds: | 20260505 05:12:48.017 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:12:48.017 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:48.017 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:48.017 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:48.017 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260505 05:12:48.017 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:48.017 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:48.017 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260505 05:12:48.017 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:48.017 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:48.018 | INFO | Uploaded 36GiB to GPU in 1.60214s (22.4699GiB/s) | 20260505 05:12:48.018 | INFO | Downloaded 34GiB to GPU in 1.51992s (22.3696GiB/s) | 20260505 05:12:53.156 | INFO | Analyzing chunk frames = [605696,606208) | 20260505 05:12:53.156 | INFO | MemInfo: { | 20260505 05:12:53.156 | INFO | "gpu" : " 17 GB", | 20260505 05:12:53.156 | INFO | "huge" : "", | 20260505 05:12:53.156 | INFO | "huge_shared" : " 27 GB", | 20260505 05:12:53.156 | INFO | "malloc" : " 14 GB", | 20260505 05:12:53.157 | INFO | "pinned" : " 29 GB" | 20260505 05:12:53.157 | INFO | } | 20260505 05:12:53.157 | INFO | Perf report for the last 5.13996 seconds: | 20260505 05:12:53.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:12:53.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:53.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:53.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:53.157 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260505 05:12:53.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:53.157 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:53.157 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260505 05:12:53.157 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:53.157 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:53.157 | INFO | Uploaded 36GiB to GPU in 1.60235s (22.467GiB/s) | 20260505 05:12:53.157 | INFO | Downloaded 34GiB to GPU in 1.51934s (22.3781GiB/s) | 20260505 05:12:53.157 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 05:12:53.157 | INFO | Total mem usage: 15.1 GiB | 20260505 05:12:53.157 | INFO | All Time High Water Marks sum: 15.7 GiB (Savings: 4.09%) | 20260505 05:12:53.157 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -4.1%) | 20260505 05:12:53.157 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 05:12:53.157 | INFO | High Water Breakdown: | 20260505 05:12:53.157 | INFO | 59.7% : 8.64 GiB ( 95.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 05:12:53.157 | INFO | 40.2% : 5.81 GiB ( 87.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 05:12:53.157 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 05:12:53.157 | INFO | ------------------------------------------ | 20260505 05:12:53.157 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:12:53.157 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 05:12:53.157 | INFO | Total mem usage: 26.5 GiB | 20260505 05:12:53.157 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 05:12:53.157 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 05:12:53.157 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 05:12:53.157 | INFO | High Water Breakdown: | 20260505 05:12:53.158 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:12:53.158 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:12:53.158 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:12:53.158 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:12:53.158 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:12:53.158 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:12:53.158 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 05:12:53.158 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 05:12:53.158 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:12:53.158 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 05:12:53.158 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 05:12:53.158 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 05:12:53.158 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 05:12:53.158 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 05:12:53.158 | INFO | ------------------------------------------ | 20260505 05:12:53.158 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:12:53.158 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 05:12:53.158 | INFO | Total mem usage: 25.6 GiB | 20260505 05:12:53.158 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 05:12:53.158 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 05:12:53.158 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 05:12:53.158 | INFO | High Water Breakdown: | 20260505 05:12:53.158 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 05:12:53.158 | INFO | ------------------------------------------ | 20260505 05:12:53.158 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:12:53.158 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 05:12:53.158 | INFO | Total mem usage: 15.7 GiB | 20260505 05:12:53.158 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 05:12:53.158 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 05:12:53.158 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260505 05:12:53.158 | INFO | High Water Breakdown: | 20260505 05:12:53.158 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 05:12:53.158 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 05:12:53.158 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 05:12:53.158 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 05:12:53.158 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 05:12:53.158 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 05:12:53.158 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 05:12:53.158 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 05:12:53.158 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 05:12:53.158 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:12:53.158 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 05:12:53.158 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:12:53.158 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 05:12:53.158 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 05:12:53.158 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 05:12:53.158 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 05:12:53.158 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:12:53.158 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 05:12:53.158 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:12:53.158 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:12:53.158 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:12:53.158 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 05:12:53.158 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 05:12:53.158 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:12:53.158 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 05:12:53.158 | INFO | ------------------------------------------ | 20260505 05:12:53.158 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:12:53.158 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 05:12:53.158 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:12:53.158 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:12:53.158 | INFO | PerfReport: Full Execution -- 3.04e+01 / -nan, 1.28e+01 / 0.00e+00 | 20260505 05:12:53.158 | INFO | PerfReport: 0.28%, AggregateStats -- 8.63e-02 / -nan, 4.33e-03 / 0.00e+00 | 20260505 05:12:53.158 | INFO | PerfReport: 10.85%, Baseline -- 3.30e+00 / -nan, 1.55e-02 / 0.00e+00 | 20260505 05:12:53.158 | INFO | PerfReport: 3.04%, Histogram -- 9.24e-01 / -nan, 5.29e-02 / 0.00e+00 | 20260505 05:12:53.158 | INFO | PerfReport: 16.04%, DME -- 3.90e+01 / -nan, 3.22e+00 / 0.00e+00 | 20260505 05:12:53.158 | INFO | PerfReport: 56.66%, FrameLabeling -- 1.72e+01 / -nan, 4.57e-02 / 0.00e+00 | 20260505 05:12:53.158 | INFO | PerfReport: 7.56%, PulseAccumulating -- 2.30e+00 / -nan, 1.51e-01 / 0.00e+00 | 20260505 05:12:53.158 | INFO | PerfReport: 3.94%, Metrics -- 1.20e+00 / -nan, 4.66e-01 / 0.00e+00 | 20260505 05:12:53.158 | INFO | PerfReport: 1.61%, PulseSerializing -- 4.88e-01 / -nan, 7.86e-02 / 0.00e+00 | 20260505 05:12:53.158 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 05:12:53.158 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:12:53.158 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:12:53.158 | INFO | PerfReport: Full Execution -- 9.11e+01 / -nan, 1.89e+01 / 0.00e+00 | 20260505 05:12:53.158 | INFO | PerfReport: 18.38%, Upload -- 1.67e+01 / 0.00e+00, 5.20e-01 / 0.00e+00 | 20260505 05:12:53.158 | INFO | PerfReport: 17.65%, Download -- 1.61e+01 / 0.00e+00, 7.36e-01 / 0.00e+00 | 20260505 05:12:53.158 | INFO | PerfReport: 33.33%, Compute -- 3.04e+01 / 0.00e+00, 1.28e+01 / 0.00e+00 | 20260505 05:12:53.158 | INFO | PerfReport: 0.55%, LockUpload -- 5.04e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260505 05:12:53.158 | INFO | PerfReport: 0.00%, LockDownload -- 3.92e-04 / 0.00e+00, 1.48e-04 / 0.00e+00 | 20260505 05:12:53.158 | INFO | PerfReport: 30.07%, LockCompute -- 2.74e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260505 05:12:53.158 | INFO | PerfReport: 0.02%, Misc -- 1.44e-02 / 0.00e+00, 2.22e-02 / 0.00e+00 | 20260505 05:12:53.254 | INFO | Starting a round of Baz IO | 20260505 05:12:56.200 | INFO | Populating RTMetrics report | 20260505 05:12:56.201 | INFO | RTMetrics report populated | 20260505 05:12:56.201 | INFO | Opening tmp RTMetrics json file | 20260505 05:12:56.202 | INFO | Serializing RTMetrics to json | 20260505 05:12:56.204 | INFO | Serialization done, swapping tmp file now... | 20260505 05:12:56.204 | INFO | RTMetrics done | 20260505 05:12:56.297 | INFO | Finished a round of Baz IO: Wrote 2.185 GiB in 3.043 seconds | 20260505 05:12:56.297 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.59% / 0.09% / 59.31% | 20260505 05:12:56.297 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.18842% / 0.000182788% / 4.76837e-05% | 20260505 05:12:56.297 | INFO | Baz IO overall: | 20260505 05:12:56.297 | INFO | BytesWritten : 274581515986 | 20260505 05:12:56.297 | INFO | NumEvents : 128602091690 | 20260505 05:12:58.295 | INFO | Analyzing chunk frames = [606208,606720) | 20260505 05:12:58.295 | INFO | MemInfo: { | 20260505 05:12:58.295 | INFO | "gpu" : " 17 GB", | 20260505 05:12:58.295 | INFO | "huge" : "", | 20260505 05:12:58.295 | INFO | "huge_shared" : " 27 GB", | 20260505 05:12:58.295 | INFO | "malloc" : " 14 GB", | 20260505 05:12:58.295 | INFO | "pinned" : " 29 GB" | 20260505 05:12:58.295 | INFO | } | 20260505 05:12:58.296 | INFO | Perf report for the last 5.13848 seconds: | 20260505 05:12:58.296 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:12:58.296 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:58.296 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:58.296 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:58.296 | INFO | ANALYSIS : 58.1%, 3.0, 90.2ms, 0 | 20260505 05:12:58.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:12:58.296 | INFO | RT_METRICS : 45.5%, 1.0, 23.9ms, 0 | 20260505 05:12:58.296 | INFO | PRE_HQ : 57.3%, 2.1, 63.8ms, 0 | 20260505 05:12:58.296 | INFO | BAZWRITER : 58.8%, 4.3, 128.6ms, 0 | 20260505 05:12:58.296 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 05:12:58.296 | INFO | Uploaded 36GiB to GPU in 1.64345s (21.9052GiB/s) | 20260505 05:12:58.297 | INFO | Downloaded 38GiB to GPU in 1.7067s (22.2652GiB/s) | 20260505 05:13:03.435 | INFO | Analyzing chunk frames = [606720,607232) | 20260505 05:13:03.435 | INFO | MemInfo: { | 20260505 05:13:03.435 | INFO | "gpu" : " 17 GB", | 20260505 05:13:03.435 | INFO | "huge" : "", | 20260505 05:13:03.435 | INFO | "huge_shared" : " 27 GB", | 20260505 05:13:03.435 | INFO | "malloc" : " 14 GB", | 20260505 05:13:03.435 | INFO | "pinned" : " 29 GB" | 20260505 05:13:03.435 | INFO | } | 20260505 05:13:03.435 | INFO | Perf report for the last 5.13945 seconds: | 20260505 05:13:03.435 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:13:03.435 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:03.435 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:03.435 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:03.435 | INFO | ANALYSIS : 57.9%, 3.0, 90.3ms, 0 | 20260505 05:13:03.435 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:03.435 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:03.435 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:13:03.435 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:03.435 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:03.436 | INFO | Uploaded 36GiB to GPU in 1.59831s (22.5239GiB/s) | 20260505 05:13:03.436 | INFO | Downloaded 34GiB to GPU in 1.51831s (22.3934GiB/s) | 20260505 05:13:08.574 | INFO | Analyzing chunk frames = [607232,607744) | 20260505 05:13:08.574 | INFO | MemInfo: { | 20260505 05:13:08.574 | INFO | "gpu" : " 17 GB", | 20260505 05:13:08.574 | INFO | "huge" : "", | 20260505 05:13:08.574 | INFO | "huge_shared" : " 27 GB", | 20260505 05:13:08.574 | INFO | "malloc" : " 14 GB", | 20260505 05:13:08.574 | INFO | "pinned" : " 29 GB" | 20260505 05:13:08.574 | INFO | } | 20260505 05:13:08.574 | INFO | Perf report for the last 5.13944 seconds: | 20260505 05:13:08.574 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:13:08.574 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:08.574 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:08.575 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:08.575 | INFO | ANALYSIS : 57.2%, 3.0, 88.9ms, 0 | 20260505 05:13:08.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:08.575 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:08.575 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:13:08.575 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:08.575 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:08.575 | INFO | Uploaded 36GiB to GPU in 1.60036s (22.4949GiB/s) | 20260505 05:13:08.575 | INFO | Downloaded 34GiB to GPU in 1.52115s (22.3515GiB/s) | 20260505 05:13:13.713 | INFO | Analyzing chunk frames = [607744,608256) | 20260505 05:13:13.713 | INFO | MemInfo: { | 20260505 05:13:13.713 | INFO | "gpu" : " 17 GB", | 20260505 05:13:13.713 | INFO | "huge" : "", | 20260505 05:13:13.713 | INFO | "huge_shared" : " 27 GB", | 20260505 05:13:13.713 | INFO | "malloc" : " 14 GB", | 20260505 05:13:13.713 | INFO | "pinned" : " 29 GB" | 20260505 05:13:13.713 | INFO | } | 20260505 05:13:13.714 | INFO | Perf report for the last 5.13916 seconds: | 20260505 05:13:13.714 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:13:13.714 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:13.714 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:13.714 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:13.714 | INFO | ANALYSIS : 56.9%, 3.0, 88.2ms, 0 | 20260505 05:13:13.714 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:13.714 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:13.714 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 05:13:13.714 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:13.714 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:13.714 | INFO | Uploaded 36GiB to GPU in 1.60059s (22.4917GiB/s) | 20260505 05:13:13.714 | INFO | Downloaded 34GiB to GPU in 1.52141s (22.3477GiB/s) | 20260505 05:13:18.854 | INFO | Analyzing chunk frames = [608256,608768) | 20260505 05:13:18.854 | INFO | MemInfo: { | 20260505 05:13:18.854 | INFO | "gpu" : " 17 GB", | 20260505 05:13:18.854 | INFO | "huge" : "", | 20260505 05:13:18.854 | INFO | "huge_shared" : " 27 GB", | 20260505 05:13:18.854 | INFO | "malloc" : " 14 GB", | 20260505 05:13:18.854 | INFO | "pinned" : " 29 GB" | 20260505 05:13:18.854 | INFO | } | 20260505 05:13:18.854 | INFO | Perf report for the last 5.14052 seconds: | 20260505 05:13:18.854 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:13:18.854 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:18.854 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:18.854 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:18.854 | INFO | ANALYSIS : 56.8%, 3.0, 88.7ms, 0 | 20260505 05:13:18.854 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:18.854 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:18.854 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 05:13:18.854 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:18.854 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:18.854 | INFO | Uploaded 36GiB to GPU in 1.59937s (22.5089GiB/s) | 20260505 05:13:18.855 | INFO | Downloaded 34GiB to GPU in 1.51955s (22.3751GiB/s) | 20260505 05:13:23.992 | INFO | Analyzing chunk frames = [608768,609280) | 20260505 05:13:23.992 | INFO | MemInfo: { | 20260505 05:13:23.992 | INFO | "gpu" : " 17 GB", | 20260505 05:13:23.992 | INFO | "huge" : "", | 20260505 05:13:23.992 | INFO | "huge_shared" : " 27 GB", | 20260505 05:13:23.992 | INFO | "malloc" : " 14 GB", | 20260505 05:13:23.992 | INFO | "pinned" : " 29 GB" | 20260505 05:13:23.992 | INFO | } | 20260505 05:13:23.992 | INFO | Perf report for the last 5.13822 seconds: | 20260505 05:13:23.992 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:13:23.992 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:23.992 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:23.992 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:23.992 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260505 05:13:23.992 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:23.992 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:23.992 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:13:23.992 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:23.993 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:23.993 | INFO | Uploaded 36GiB to GPU in 1.59964s (22.5051GiB/s) | 20260505 05:13:23.993 | INFO | Downloaded 34GiB to GPU in 1.51928s (22.379GiB/s) | 20260505 05:13:29.132 | INFO | Analyzing chunk frames = [609280,609792) | 20260505 05:13:29.132 | INFO | MemInfo: { | 20260505 05:13:29.132 | INFO | "gpu" : " 17 GB", | 20260505 05:13:29.132 | INFO | "huge" : "", | 20260505 05:13:29.132 | INFO | "huge_shared" : " 27 GB", | 20260505 05:13:29.132 | INFO | "malloc" : " 14 GB", | 20260505 05:13:29.132 | INFO | "pinned" : " 29 GB" | 20260505 05:13:29.132 | INFO | } | 20260505 05:13:29.132 | INFO | Perf report for the last 5.13964 seconds: | 20260505 05:13:29.132 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:13:29.132 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:29.132 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:29.132 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:29.132 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 05:13:29.132 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:29.132 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:29.132 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 05:13:29.132 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:29.132 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:29.132 | INFO | Uploaded 36GiB to GPU in 1.5992s (22.5112GiB/s) | 20260505 05:13:29.132 | INFO | Downloaded 34GiB to GPU in 1.52001s (22.3683GiB/s) | 20260505 05:13:34.271 | INFO | Analyzing chunk frames = [609792,610304) | 20260505 05:13:34.271 | INFO | MemInfo: { | 20260505 05:13:34.271 | INFO | "gpu" : " 17 GB", | 20260505 05:13:34.271 | INFO | "huge" : "", | 20260505 05:13:34.271 | INFO | "huge_shared" : " 27 GB", | 20260505 05:13:34.271 | INFO | "malloc" : " 14 GB", | 20260505 05:13:34.271 | INFO | "pinned" : " 29 GB" | 20260505 05:13:34.271 | INFO | } | 20260505 05:13:34.271 | INFO | Perf report for the last 5.139 seconds: | 20260505 05:13:34.271 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:13:34.271 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:34.271 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:34.271 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:34.271 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260505 05:13:34.271 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:34.271 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:34.271 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 05:13:34.271 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:34.271 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:34.271 | INFO | Uploaded 36GiB to GPU in 1.59965s (22.5049GiB/s) | 20260505 05:13:34.271 | INFO | Downloaded 34GiB to GPU in 1.52036s (22.3632GiB/s) | 20260505 05:13:34.368 | INFO | Starting a round of Baz IO | 20260505 05:13:37.313 | INFO | Populating RTMetrics report | 20260505 05:13:37.313 | INFO | RTMetrics report populated | 20260505 05:13:37.313 | INFO | Opening tmp RTMetrics json file | 20260505 05:13:37.316 | INFO | Serializing RTMetrics to json | 20260505 05:13:37.320 | INFO | Serialization done, swapping tmp file now... | 20260505 05:13:37.321 | INFO | RTMetrics done | 20260505 05:13:37.417 | INFO | Finished a round of Baz IO: Wrote 2.183 GiB in 3.048 seconds | 20260505 05:13:37.417 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.50% / 0.09% / 59.41% | 20260505 05:13:37.417 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.11503% / 0.000178814% / 4.37101e-05% | 20260505 05:13:37.417 | INFO | Baz IO overall: | 20260505 05:13:37.417 | INFO | BytesWritten : 276925704151 | 20260505 05:13:37.417 | INFO | NumEvents : 129716824357 | 20260505 05:13:39.410 | INFO | Analyzing chunk frames = [610304,610816) | 20260505 05:13:39.410 | INFO | MemInfo: { | 20260505 05:13:39.410 | INFO | "gpu" : " 17 GB", | 20260505 05:13:39.410 | INFO | "huge" : "", | 20260505 05:13:39.410 | INFO | "huge_shared" : " 27 GB", | 20260505 05:13:39.410 | INFO | "malloc" : " 14 GB", | 20260505 05:13:39.410 | INFO | "pinned" : " 29 GB" | 20260505 05:13:39.410 | INFO | } | 20260505 05:13:39.410 | INFO | Perf report for the last 5.13939 seconds: | 20260505 05:13:39.410 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:13:39.410 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:39.410 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:39.411 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:39.411 | INFO | ANALYSIS : 58.0%, 2.9, 89.7ms, 0 | 20260505 05:13:39.411 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:39.411 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260505 05:13:39.411 | INFO | PRE_HQ : 57.4%, 2.1, 64.0ms, 0 | 20260505 05:13:39.411 | INFO | BAZWRITER : 58.9%, 4.3, 129.6ms, 0 | 20260505 05:13:39.411 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 05:13:39.411 | INFO | Uploaded 36GiB to GPU in 1.64864s (21.8362GiB/s) | 20260505 05:13:39.411 | INFO | Downloaded 38GiB to GPU in 1.71412s (22.1689GiB/s) | 20260505 05:13:44.550 | INFO | Analyzing chunk frames = [610816,611328) | 20260505 05:13:44.550 | INFO | MemInfo: { | 20260505 05:13:44.550 | INFO | "gpu" : " 17 GB", | 20260505 05:13:44.550 | INFO | "huge" : "", | 20260505 05:13:44.550 | INFO | "huge_shared" : " 27 GB", | 20260505 05:13:44.550 | INFO | "malloc" : " 14 GB", | 20260505 05:13:44.550 | INFO | "pinned" : " 29 GB" | 20260505 05:13:44.550 | INFO | } | 20260505 05:13:44.550 | INFO | Perf report for the last 5.13936 seconds: | 20260505 05:13:44.550 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:13:44.550 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:44.550 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:44.550 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:44.550 | INFO | ANALYSIS : 57.8%, 3.0, 90.2ms, 0 | 20260505 05:13:44.550 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:44.550 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:44.550 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:13:44.550 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:44.550 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:44.550 | INFO | Uploaded 36GiB to GPU in 1.60022s (22.4968GiB/s) | 20260505 05:13:44.550 | INFO | Downloaded 34GiB to GPU in 1.5173s (22.4083GiB/s) | 20260505 05:13:49.689 | INFO | Analyzing chunk frames = [611328,611840) | 20260505 05:13:49.689 | INFO | MemInfo: { | 20260505 05:13:49.689 | INFO | "gpu" : " 17 GB", | 20260505 05:13:49.689 | INFO | "huge" : "", | 20260505 05:13:49.689 | INFO | "huge_shared" : " 27 GB", | 20260505 05:13:49.689 | INFO | "malloc" : " 14 GB", | 20260505 05:13:49.689 | INFO | "pinned" : " 29 GB" | 20260505 05:13:49.689 | INFO | } | 20260505 05:13:49.689 | INFO | Perf report for the last 5.13917 seconds: | 20260505 05:13:49.689 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:13:49.689 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:49.689 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:49.689 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:49.689 | INFO | ANALYSIS : 57.2%, 3.0, 88.9ms, 0 | 20260505 05:13:49.689 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:49.689 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:49.689 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 05:13:49.689 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:49.689 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:49.689 | INFO | Uploaded 36GiB to GPU in 1.60347s (22.4514GiB/s) | 20260505 05:13:49.689 | INFO | Downloaded 34GiB to GPU in 1.52148s (22.3466GiB/s) | 20260505 05:13:54.828 | INFO | Analyzing chunk frames = [611840,612352) | 20260505 05:13:54.828 | INFO | MemInfo: { | 20260505 05:13:54.828 | INFO | "gpu" : " 17 GB", | 20260505 05:13:54.828 | INFO | "huge" : "", | 20260505 05:13:54.828 | INFO | "huge_shared" : " 27 GB", | 20260505 05:13:54.828 | INFO | "malloc" : " 14 GB", | 20260505 05:13:54.828 | INFO | "pinned" : " 29 GB" | 20260505 05:13:54.828 | INFO | } | 20260505 05:13:54.828 | INFO | Perf report for the last 5.1393 seconds: | 20260505 05:13:54.828 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:13:54.828 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:54.828 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:54.828 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:54.829 | INFO | ANALYSIS : 56.9%, 3.0, 88.3ms, 0 | 20260505 05:13:54.829 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:54.829 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:54.829 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260505 05:13:54.829 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:54.829 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:54.829 | INFO | Uploaded 36GiB to GPU in 1.60396s (22.4444GiB/s) | 20260505 05:13:54.829 | INFO | Downloaded 34GiB to GPU in 1.52123s (22.3503GiB/s) | 20260505 05:13:59.968 | INFO | Analyzing chunk frames = [612352,612864) | 20260505 05:13:59.968 | INFO | MemInfo: { | 20260505 05:13:59.968 | INFO | "gpu" : " 17 GB", | 20260505 05:13:59.968 | INFO | "huge" : "", | 20260505 05:13:59.968 | INFO | "huge_shared" : " 27 GB", | 20260505 05:13:59.968 | INFO | "malloc" : " 14 GB", | 20260505 05:13:59.968 | INFO | "pinned" : " 29 GB" | 20260505 05:13:59.968 | INFO | } | 20260505 05:13:59.968 | INFO | Perf report for the last 5.13931 seconds: | 20260505 05:13:59.968 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:13:59.968 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:59.968 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:59.968 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:59.968 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260505 05:13:59.968 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:59.968 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:59.968 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 05:13:59.968 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:59.968 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:13:59.968 | INFO | Uploaded 36GiB to GPU in 1.60195s (22.4726GiB/s) | 20260505 05:13:59.968 | INFO | Downloaded 34GiB to GPU in 1.51776s (22.4014GiB/s) | 20260505 05:14:05.107 | INFO | Analyzing chunk frames = [612864,613376) | 20260505 05:14:05.107 | INFO | MemInfo: { | 20260505 05:14:05.107 | INFO | "gpu" : " 17 GB", | 20260505 05:14:05.107 | INFO | "huge" : "", | 20260505 05:14:05.107 | INFO | "huge_shared" : " 27 GB", | 20260505 05:14:05.107 | INFO | "malloc" : " 14 GB", | 20260505 05:14:05.107 | INFO | "pinned" : " 29 GB" | 20260505 05:14:05.107 | INFO | } | 20260505 05:14:05.107 | INFO | Perf report for the last 5.13934 seconds: | 20260505 05:14:05.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:14:05.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:05.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:05.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:05.107 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260505 05:14:05.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:05.107 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:05.107 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:14:05.107 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:05.107 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:05.107 | INFO | Uploaded 36GiB to GPU in 1.60207s (22.4709GiB/s) | 20260505 05:14:05.107 | INFO | Downloaded 34GiB to GPU in 1.51881s (22.3859GiB/s) | 20260505 05:14:10.246 | INFO | Analyzing chunk frames = [613376,613888) | 20260505 05:14:10.246 | INFO | MemInfo: { | 20260505 05:14:10.246 | INFO | "gpu" : " 17 GB", | 20260505 05:14:10.246 | INFO | "huge" : "", | 20260505 05:14:10.246 | INFO | "huge_shared" : " 27 GB", | 20260505 05:14:10.247 | INFO | "malloc" : " 14 GB", | 20260505 05:14:10.247 | INFO | "pinned" : " 29 GB" | 20260505 05:14:10.247 | INFO | } | 20260505 05:14:10.247 | INFO | Perf report for the last 5.13965 seconds: | 20260505 05:14:10.247 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:14:10.247 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:10.247 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:10.247 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:10.247 | INFO | ANALYSIS : 57.3%, 3.0, 89.5ms, 0 | 20260505 05:14:10.247 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:10.247 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:10.247 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:14:10.247 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:10.247 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:10.247 | INFO | Uploaded 36GiB to GPU in 1.6021s (22.4706GiB/s) | 20260505 05:14:10.247 | INFO | Downloaded 34GiB to GPU in 1.51914s (22.381GiB/s) | 20260505 05:14:15.387 | INFO | Analyzing chunk frames = [613888,614400) | 20260505 05:14:15.387 | INFO | MemInfo: { | 20260505 05:14:15.387 | INFO | "gpu" : " 17 GB", | 20260505 05:14:15.387 | INFO | "huge" : "", | 20260505 05:14:15.387 | INFO | "huge_shared" : " 27 GB", | 20260505 05:14:15.387 | INFO | "malloc" : " 14 GB", | 20260505 05:14:15.387 | INFO | "pinned" : " 29 GB" | 20260505 05:14:15.387 | INFO | } | 20260505 05:14:15.387 | INFO | Perf report for the last 5.1402 seconds: | 20260505 05:14:15.387 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:14:15.387 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:15.387 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:15.387 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:15.387 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260505 05:14:15.387 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:15.387 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:15.387 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:14:15.387 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:15.387 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:15.387 | INFO | Uploaded 36GiB to GPU in 1.60246s (22.4655GiB/s) | 20260505 05:14:15.387 | INFO | Downloaded 34GiB to GPU in 1.52056s (22.3602GiB/s) | 20260505 05:14:15.387 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 05:14:15.387 | INFO | Total mem usage: 15.1 GiB | 20260505 05:14:15.387 | INFO | All Time High Water Marks sum: 15.7 GiB (Savings: 4.09%) | 20260505 05:14:15.387 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -4.16%) | 20260505 05:14:15.387 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 05:14:15.387 | INFO | High Water Breakdown: | 20260505 05:14:15.387 | INFO | 59.8% : 8.65 GiB ( 95.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 05:14:15.387 | INFO | 40% : 5.79 GiB ( 87.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 05:14:15.387 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 05:14:15.387 | INFO | ------------------------------------------ | 20260505 05:14:15.387 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:14:15.388 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 05:14:15.388 | INFO | Total mem usage: 26.5 GiB | 20260505 05:14:15.388 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 05:14:15.388 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 05:14:15.388 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 05:14:15.388 | INFO | High Water Breakdown: | 20260505 05:14:15.388 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:14:15.388 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:14:15.388 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:14:15.388 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:14:15.388 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:14:15.388 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:14:15.388 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 05:14:15.388 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 05:14:15.388 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:14:15.388 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 05:14:15.388 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 05:14:15.388 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 05:14:15.388 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 05:14:15.388 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 05:14:15.388 | INFO | ------------------------------------------ | 20260505 05:14:15.388 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:14:15.388 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 05:14:15.388 | INFO | Total mem usage: 25.6 GiB | 20260505 05:14:15.388 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 05:14:15.388 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 05:14:15.388 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 05:14:15.388 | INFO | High Water Breakdown: | 20260505 05:14:15.388 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 05:14:15.388 | INFO | ------------------------------------------ | 20260505 05:14:15.388 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:14:15.388 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 05:14:15.388 | INFO | Total mem usage: 15.7 GiB | 20260505 05:14:15.388 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 05:14:15.388 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 05:14:15.388 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260505 05:14:15.388 | INFO | High Water Breakdown: | 20260505 05:14:15.388 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 05:14:15.388 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 05:14:15.388 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 05:14:15.388 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 05:14:15.388 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 05:14:15.388 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 05:14:15.388 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 05:14:15.388 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 05:14:15.388 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 05:14:15.388 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:14:15.388 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 05:14:15.388 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:14:15.388 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 05:14:15.388 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 05:14:15.388 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 05:14:15.388 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 05:14:15.388 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:14:15.388 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 05:14:15.388 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:14:15.388 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:14:15.388 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:14:15.388 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 05:14:15.388 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 05:14:15.388 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:14:15.388 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 05:14:15.388 | INFO | ------------------------------------------ | 20260505 05:14:15.388 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:14:15.388 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 05:14:15.388 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:14:15.388 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:14:15.388 | INFO | PerfReport: Full Execution -- 3.04e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260505 05:14:15.388 | INFO | PerfReport: 0.28%, AggregateStats -- 8.64e-02 / -nan, 4.28e-03 / 0.00e+00 | 20260505 05:14:15.388 | INFO | PerfReport: 10.86%, Baseline -- 3.30e+00 / -nan, 1.93e-02 / 0.00e+00 | 20260505 05:14:15.388 | INFO | PerfReport: 3.04%, Histogram -- 9.24e-01 / -nan, 5.18e-02 / 0.00e+00 | 20260505 05:14:15.388 | INFO | PerfReport: 16.04%, DME -- 3.89e+01 / -nan, 3.18e+00 / 0.00e+00 | 20260505 05:14:15.388 | INFO | PerfReport: 56.70%, FrameLabeling -- 1.72e+01 / -nan, 4.58e-02 / 0.00e+00 | 20260505 05:14:15.388 | INFO | PerfReport: 7.57%, PulseAccumulating -- 2.30e+00 / -nan, 1.52e-01 / 0.00e+00 | 20260505 05:14:15.388 | INFO | PerfReport: 3.90%, Metrics -- 1.18e+00 / -nan, 1.96e-01 / 0.00e+00 | 20260505 05:14:15.388 | INFO | PerfReport: 1.60%, PulseSerializing -- 4.87e-01 / -nan, 7.85e-02 / 0.00e+00 | 20260505 05:14:15.388 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 05:14:15.388 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:14:15.388 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:14:15.389 | INFO | PerfReport: Full Execution -- 9.11e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260505 05:14:15.389 | INFO | PerfReport: 18.37%, Upload -- 1.67e+01 / 0.00e+00, 3.91e-01 / 0.00e+00 | 20260505 05:14:15.389 | INFO | PerfReport: 17.65%, Download -- 1.61e+01 / 0.00e+00, 7.28e-01 / 0.00e+00 | 20260505 05:14:15.389 | INFO | PerfReport: 33.32%, Compute -- 3.04e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260505 05:14:15.389 | INFO | PerfReport: 0.55%, LockUpload -- 5.05e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260505 05:14:15.389 | INFO | PerfReport: 0.00%, LockDownload -- 3.91e-04 / 0.00e+00, 9.61e-05 / 0.00e+00 | 20260505 05:14:15.389 | INFO | PerfReport: 30.10%, LockCompute -- 2.74e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260505 05:14:15.389 | INFO | PerfReport: 0.02%, Misc -- 1.51e-02 / 0.00e+00, 4.14e-02 / 0.00e+00 | 20260505 05:14:15.484 | INFO | Starting a round of Baz IO | 20260505 05:14:18.429 | INFO | Populating RTMetrics report | 20260505 05:14:18.430 | INFO | RTMetrics report populated | 20260505 05:14:18.430 | INFO | Opening tmp RTMetrics json file | 20260505 05:14:18.431 | INFO | Serializing RTMetrics to json | 20260505 05:14:18.432 | INFO | Serialization done, swapping tmp file now... | 20260505 05:14:18.432 | INFO | RTMetrics done | 20260505 05:14:18.531 | INFO | Finished a round of Baz IO: Wrote 2.188 GiB in 3.047 seconds | 20260505 05:14:18.531 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.41% / 0.09% / 59.50% | 20260505 05:14:18.531 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.0972% / 0.000119209% / 4.37101e-05% | 20260505 05:14:18.531 | INFO | Baz IO overall: | 20260505 05:14:18.531 | INFO | BytesWritten : 279274964944 | 20260505 05:14:18.531 | INFO | NumEvents : 130834630678 | 20260505 05:14:20.525 | INFO | Analyzing chunk frames = [614400,614912) | 20260505 05:14:20.525 | INFO | MemInfo: { | 20260505 05:14:20.525 | INFO | "gpu" : " 17 GB", | 20260505 05:14:20.525 | INFO | "huge" : "", | 20260505 05:14:20.525 | INFO | "huge_shared" : " 27 GB", | 20260505 05:14:20.525 | INFO | "malloc" : " 14 GB", | 20260505 05:14:20.525 | INFO | "pinned" : " 29 GB" | 20260505 05:14:20.525 | INFO | } | 20260505 05:14:20.525 | INFO | Perf report for the last 5.13828 seconds: | 20260505 05:14:20.525 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:14:20.525 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:20.525 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:20.525 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:20.525 | INFO | ANALYSIS : 58.1%, 3.0, 89.8ms, 0 | 20260505 05:14:20.525 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:20.525 | INFO | RT_METRICS : 45.5%, 1.0, 23.8ms, 0 | 20260505 05:14:20.525 | INFO | PRE_HQ : 57.4%, 2.1, 64.0ms, 0 | 20260505 05:14:20.525 | INFO | BAZWRITER : 58.9%, 4.3, 129.9ms, 0 | 20260505 05:14:20.525 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 05:14:20.525 | INFO | Uploaded 36GiB to GPU in 1.6471s (21.8565GiB/s) | 20260505 05:14:20.525 | INFO | Downloaded 38GiB to GPU in 1.71132s (22.2051GiB/s) | 20260505 05:14:25.664 | INFO | Analyzing chunk frames = [614912,615424) | 20260505 05:14:25.664 | INFO | MemInfo: { | 20260505 05:14:25.664 | INFO | "gpu" : " 17 GB", | 20260505 05:14:25.664 | INFO | "huge" : "", | 20260505 05:14:25.664 | INFO | "huge_shared" : " 27 GB", | 20260505 05:14:25.664 | INFO | "malloc" : " 14 GB", | 20260505 05:14:25.664 | INFO | "pinned" : " 29 GB" | 20260505 05:14:25.664 | INFO | } | 20260505 05:14:25.664 | INFO | Perf report for the last 5.13932 seconds: | 20260505 05:14:25.664 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:14:25.664 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:25.665 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:25.665 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:25.665 | INFO | ANALYSIS : 57.9%, 3.0, 90.3ms, 0 | 20260505 05:14:25.665 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:25.665 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:25.665 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:14:25.665 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:25.665 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:25.665 | INFO | Uploaded 36GiB to GPU in 1.59704s (22.5417GiB/s) | 20260505 05:14:25.665 | INFO | Downloaded 34GiB to GPU in 1.51725s (22.409GiB/s) | 20260505 05:14:30.804 | INFO | Analyzing chunk frames = [615424,615936) | 20260505 05:14:30.804 | INFO | MemInfo: { | 20260505 05:14:30.804 | INFO | "gpu" : " 17 GB", | 20260505 05:14:30.804 | INFO | "huge" : "", | 20260505 05:14:30.804 | INFO | "huge_shared" : " 27 GB", | 20260505 05:14:30.804 | INFO | "malloc" : " 14 GB", | 20260505 05:14:30.804 | INFO | "pinned" : " 29 GB" | 20260505 05:14:30.804 | INFO | } | 20260505 05:14:30.804 | INFO | Perf report for the last 5.13947 seconds: | 20260505 05:14:30.804 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:14:30.804 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:30.804 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:30.804 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:30.804 | INFO | ANALYSIS : 57.2%, 3.0, 88.9ms, 0 | 20260505 05:14:30.804 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:30.804 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:30.804 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 05:14:30.804 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:30.804 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:30.804 | INFO | Uploaded 36GiB to GPU in 1.60125s (22.4824GiB/s) | 20260505 05:14:30.804 | INFO | Downloaded 34GiB to GPU in 1.52225s (22.3354GiB/s) | 20260505 05:14:35.944 | INFO | Analyzing chunk frames = [615936,616448) | 20260505 05:14:35.945 | INFO | MemInfo: { | 20260505 05:14:35.945 | INFO | "gpu" : " 17 GB", | 20260505 05:14:35.945 | INFO | "huge" : "", | 20260505 05:14:35.945 | INFO | "huge_shared" : " 27 GB", | 20260505 05:14:35.945 | INFO | "malloc" : " 14 GB", | 20260505 05:14:35.945 | INFO | "pinned" : " 29 GB" | 20260505 05:14:35.945 | INFO | } | 20260505 05:14:35.945 | INFO | Perf report for the last 5.14075 seconds: | 20260505 05:14:35.945 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:14:35.945 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:35.945 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:35.945 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:35.945 | INFO | ANALYSIS : 56.9%, 3.0, 88.3ms, 0 | 20260505 05:14:35.945 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:35.945 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:35.945 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 05:14:35.945 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:35.945 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:35.945 | INFO | Uploaded 36GiB to GPU in 1.60029s (22.4959GiB/s) | 20260505 05:14:35.945 | INFO | Downloaded 34GiB to GPU in 1.52099s (22.3538GiB/s) | 20260505 05:14:41.082 | INFO | Analyzing chunk frames = [616448,616960) | 20260505 05:14:41.082 | INFO | MemInfo: { | 20260505 05:14:41.082 | INFO | "gpu" : " 17 GB", | 20260505 05:14:41.082 | INFO | "huge" : "", | 20260505 05:14:41.082 | INFO | "huge_shared" : " 27 GB", | 20260505 05:14:41.082 | INFO | "malloc" : " 14 GB", | 20260505 05:14:41.082 | INFO | "pinned" : " 29 GB" | 20260505 05:14:41.083 | INFO | } | 20260505 05:14:41.083 | INFO | Perf report for the last 5.1378 seconds: | 20260505 05:14:41.083 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:14:41.083 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:41.083 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:41.083 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:41.083 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260505 05:14:41.083 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:41.083 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:41.083 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 05:14:41.083 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:41.083 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:41.083 | INFO | Uploaded 36GiB to GPU in 1.59925s (22.5106GiB/s) | 20260505 05:14:41.083 | INFO | Downloaded 34GiB to GPU in 1.51939s (22.3775GiB/s) | 20260505 05:14:46.222 | INFO | Analyzing chunk frames = [616960,617472) | 20260505 05:14:46.222 | INFO | MemInfo: { | 20260505 05:14:46.222 | INFO | "gpu" : " 17 GB", | 20260505 05:14:46.222 | INFO | "huge" : "", | 20260505 05:14:46.222 | INFO | "huge_shared" : " 27 GB", | 20260505 05:14:46.222 | INFO | "malloc" : " 14 GB", | 20260505 05:14:46.222 | INFO | "pinned" : " 29 GB" | 20260505 05:14:46.222 | INFO | } | 20260505 05:14:46.222 | INFO | Perf report for the last 5.13929 seconds: | 20260505 05:14:46.223 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:14:46.223 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:46.223 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:46.223 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:46.223 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260505 05:14:46.223 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:46.223 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:46.223 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 05:14:46.223 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:46.223 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:46.223 | INFO | Uploaded 36GiB to GPU in 1.59905s (22.5134GiB/s) | 20260505 05:14:46.223 | INFO | Downloaded 34GiB to GPU in 1.51886s (22.3852GiB/s) | 20260505 05:14:51.365 | INFO | Analyzing chunk frames = [617472,617984) | 20260505 05:14:51.365 | INFO | MemInfo: { | 20260505 05:14:51.365 | INFO | "gpu" : " 17 GB", | 20260505 05:14:51.365 | INFO | "huge" : "", | 20260505 05:14:51.365 | INFO | "huge_shared" : " 27 GB", | 20260505 05:14:51.365 | INFO | "malloc" : " 14 GB", | 20260505 05:14:51.365 | INFO | "pinned" : " 29 GB" | 20260505 05:14:51.365 | INFO | } | 20260505 05:14:51.365 | INFO | Perf report for the last 5.14301 seconds: | 20260505 05:14:51.365 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:14:51.365 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:51.365 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:51.365 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:51.365 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 05:14:51.365 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:51.365 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:51.365 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:14:51.365 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:51.365 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:51.365 | INFO | Uploaded 36GiB to GPU in 1.59976s (22.5034GiB/s) | 20260505 05:14:51.365 | INFO | Downloaded 34GiB to GPU in 1.52004s (22.3678GiB/s) | 20260505 05:14:56.500 | INFO | Analyzing chunk frames = [617984,618496) | 20260505 05:14:56.500 | INFO | MemInfo: { | 20260505 05:14:56.500 | INFO | "gpu" : " 17 GB", | 20260505 05:14:56.500 | INFO | "huge" : "", | 20260505 05:14:56.500 | INFO | "huge_shared" : " 27 GB", | 20260505 05:14:56.500 | INFO | "malloc" : " 14 GB", | 20260505 05:14:56.500 | INFO | "pinned" : " 29 GB" | 20260505 05:14:56.501 | INFO | } | 20260505 05:14:56.501 | INFO | Perf report for the last 5.13592 seconds: | 20260505 05:14:56.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:14:56.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:56.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:56.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:56.501 | INFO | ANALYSIS : 57.5%, 3.0, 89.6ms, 0 | 20260505 05:14:56.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:56.501 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:56.501 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 05:14:56.501 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:56.501 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:14:56.501 | INFO | Uploaded 36GiB to GPU in 1.59966s (22.5048GiB/s) | 20260505 05:14:56.501 | INFO | Downloaded 34GiB to GPU in 1.5207s (22.3581GiB/s) | 20260505 05:14:56.599 | INFO | Starting a round of Baz IO | 20260505 05:14:59.541 | INFO | Populating RTMetrics report | 20260505 05:14:59.541 | INFO | RTMetrics report populated | 20260505 05:14:59.541 | INFO | Opening tmp RTMetrics json file | 20260505 05:14:59.546 | INFO | Serializing RTMetrics to json | 20260505 05:14:59.547 | INFO | Serialization done, swapping tmp file now... | 20260505 05:14:59.547 | INFO | RTMetrics done | 20260505 05:14:59.647 | INFO | Finished a round of Baz IO: Wrote 2.191 GiB in 3.047 seconds | 20260505 05:14:59.647 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.32% / 0.09% / 59.59% | 20260505 05:14:59.647 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.06286% / 0.000170867% / 6.35783e-05% | 20260505 05:14:59.647 | INFO | Baz IO overall: | 20260505 05:14:59.647 | INFO | BytesWritten : 281627911083 | 20260505 05:14:59.647 | INFO | NumEvents : 131954284258 | 20260505 05:15:01.640 | INFO | Analyzing chunk frames = [618496,619008) | 20260505 05:15:01.640 | INFO | MemInfo: { | 20260505 05:15:01.640 | INFO | "gpu" : " 17 GB", | 20260505 05:15:01.640 | INFO | "huge" : "", | 20260505 05:15:01.640 | INFO | "huge_shared" : " 27 GB", | 20260505 05:15:01.640 | INFO | "malloc" : " 14 GB", | 20260505 05:15:01.640 | INFO | "pinned" : " 29 GB" | 20260505 05:15:01.640 | INFO | } | 20260505 05:15:01.640 | INFO | Perf report for the last 5.13919 seconds: | 20260505 05:15:01.640 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:15:01.640 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:01.640 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:01.640 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:01.640 | INFO | ANALYSIS : 58.0%, 2.9, 89.5ms, 0 | 20260505 05:15:01.640 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:01.640 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260505 05:15:01.640 | INFO | PRE_HQ : 57.4%, 2.2, 64.3ms, 0 | 20260505 05:15:01.640 | INFO | BAZWRITER : 58.9%, 4.3, 129.5ms, 0 | 20260505 05:15:01.640 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 05:15:01.640 | INFO | Uploaded 36GiB to GPU in 1.64745s (21.852GiB/s) | 20260505 05:15:01.640 | INFO | Downloaded 38GiB to GPU in 1.71288s (22.1849GiB/s) | 20260505 05:15:06.779 | INFO | Analyzing chunk frames = [619008,619520) | 20260505 05:15:06.779 | INFO | MemInfo: { | 20260505 05:15:06.779 | INFO | "gpu" : " 17 GB", | 20260505 05:15:06.779 | INFO | "huge" : "", | 20260505 05:15:06.779 | INFO | "huge_shared" : " 27 GB", | 20260505 05:15:06.779 | INFO | "malloc" : " 14 GB", | 20260505 05:15:06.779 | INFO | "pinned" : " 29 GB" | 20260505 05:15:06.779 | INFO | } | 20260505 05:15:06.779 | INFO | Perf report for the last 5.13933 seconds: | 20260505 05:15:06.779 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:15:06.779 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:06.779 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:06.779 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:06.779 | INFO | ANALYSIS : 57.8%, 3.0, 90.2ms, 0 | 20260505 05:15:06.779 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:06.780 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:06.780 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:15:06.780 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:06.780 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:06.780 | INFO | Uploaded 36GiB to GPU in 1.59992s (22.5011GiB/s) | 20260505 05:15:06.780 | INFO | Downloaded 34GiB to GPU in 1.51677s (22.4161GiB/s) | 20260505 05:15:11.918 | INFO | Analyzing chunk frames = [619520,620032) | 20260505 05:15:11.918 | INFO | MemInfo: { | 20260505 05:15:11.918 | INFO | "gpu" : " 17 GB", | 20260505 05:15:11.918 | INFO | "huge" : "", | 20260505 05:15:11.918 | INFO | "huge_shared" : " 27 GB", | 20260505 05:15:11.918 | INFO | "malloc" : " 14 GB", | 20260505 05:15:11.919 | INFO | "pinned" : " 29 GB" | 20260505 05:15:11.919 | INFO | } | 20260505 05:15:11.919 | INFO | Perf report for the last 5.13929 seconds: | 20260505 05:15:11.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:15:11.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:11.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:11.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:11.919 | INFO | ANALYSIS : 57.2%, 3.0, 88.9ms, 0 | 20260505 05:15:11.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:11.919 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:11.919 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:15:11.919 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:11.919 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:11.919 | INFO | Uploaded 36GiB to GPU in 1.6039s (22.4453GiB/s) | 20260505 05:15:11.919 | INFO | Downloaded 34GiB to GPU in 1.52173s (22.343GiB/s) | 20260505 05:15:17.058 | INFO | Analyzing chunk frames = [620032,620544) | 20260505 05:15:17.058 | INFO | MemInfo: { | 20260505 05:15:17.058 | INFO | "gpu" : " 17 GB", | 20260505 05:15:17.058 | INFO | "huge" : "", | 20260505 05:15:17.058 | INFO | "huge_shared" : " 27 GB", | 20260505 05:15:17.058 | INFO | "malloc" : " 14 GB", | 20260505 05:15:17.058 | INFO | "pinned" : " 29 GB" | 20260505 05:15:17.058 | INFO | } | 20260505 05:15:17.058 | INFO | Perf report for the last 5.13908 seconds: | 20260505 05:15:17.058 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:15:17.058 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:17.058 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:17.058 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:17.058 | INFO | ANALYSIS : 56.9%, 3.0, 88.3ms, 0 | 20260505 05:15:17.058 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:17.058 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:17.058 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 05:15:17.058 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:17.058 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:17.058 | INFO | Uploaded 36GiB to GPU in 1.60339s (22.4525GiB/s) | 20260505 05:15:17.058 | INFO | Downloaded 34GiB to GPU in 1.52044s (22.362GiB/s) | 20260505 05:15:22.197 | INFO | Analyzing chunk frames = [620544,621056) | 20260505 05:15:22.197 | INFO | MemInfo: { | 20260505 05:15:22.197 | INFO | "gpu" : " 17 GB", | 20260505 05:15:22.197 | INFO | "huge" : "", | 20260505 05:15:22.197 | INFO | "huge_shared" : " 27 GB", | 20260505 05:15:22.197 | INFO | "malloc" : " 14 GB", | 20260505 05:15:22.197 | INFO | "pinned" : " 29 GB" | 20260505 05:15:22.197 | INFO | } | 20260505 05:15:22.197 | INFO | Perf report for the last 5.13964 seconds: | 20260505 05:15:22.197 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:15:22.197 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:22.197 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:22.197 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:22.197 | INFO | ANALYSIS : 56.8%, 3.0, 88.7ms, 0 | 20260505 05:15:22.198 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:22.198 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:22.198 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 05:15:22.198 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:22.198 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:22.198 | INFO | Uploaded 36GiB to GPU in 1.60192s (22.473GiB/s) | 20260505 05:15:22.198 | INFO | Downloaded 34GiB to GPU in 1.51816s (22.3955GiB/s) | 20260505 05:15:27.336 | INFO | Analyzing chunk frames = [621056,621568) | 20260505 05:15:27.336 | INFO | MemInfo: { | 20260505 05:15:27.336 | INFO | "gpu" : " 17 GB", | 20260505 05:15:27.336 | INFO | "huge" : "", | 20260505 05:15:27.336 | INFO | "huge_shared" : " 27 GB", | 20260505 05:15:27.336 | INFO | "malloc" : " 14 GB", | 20260505 05:15:27.336 | INFO | "pinned" : " 29 GB" | 20260505 05:15:27.336 | INFO | } | 20260505 05:15:27.337 | INFO | Perf report for the last 5.13921 seconds: | 20260505 05:15:27.337 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:15:27.337 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:27.337 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:27.337 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:27.337 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260505 05:15:27.337 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:27.337 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:27.337 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:15:27.337 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:27.337 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:27.337 | INFO | Uploaded 36GiB to GPU in 1.60135s (22.4811GiB/s) | 20260505 05:15:27.337 | INFO | Downloaded 34GiB to GPU in 1.51833s (22.3931GiB/s) | 20260505 05:15:32.476 | INFO | Analyzing chunk frames = [621568,622080) | 20260505 05:15:32.476 | INFO | MemInfo: { | 20260505 05:15:32.476 | INFO | "gpu" : " 17 GB", | 20260505 05:15:32.476 | INFO | "huge" : "", | 20260505 05:15:32.476 | INFO | "huge_shared" : " 27 GB", | 20260505 05:15:32.476 | INFO | "malloc" : " 14 GB", | 20260505 05:15:32.476 | INFO | "pinned" : " 29 GB" | 20260505 05:15:32.476 | INFO | } | 20260505 05:15:32.476 | INFO | Perf report for the last 5.13951 seconds: | 20260505 05:15:32.476 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:15:32.476 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:32.476 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:32.476 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:32.476 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 05:15:32.476 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:32.476 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:32.476 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 05:15:32.476 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:32.476 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:32.476 | INFO | Uploaded 36GiB to GPU in 1.60334s (22.4532GiB/s) | 20260505 05:15:32.477 | INFO | Downloaded 34GiB to GPU in 1.52082s (22.3564GiB/s) | 20260505 05:15:37.615 | INFO | Analyzing chunk frames = [622080,622592) | 20260505 05:15:37.615 | INFO | MemInfo: { | 20260505 05:15:37.615 | INFO | "gpu" : " 17 GB", | 20260505 05:15:37.615 | INFO | "huge" : "", | 20260505 05:15:37.615 | INFO | "huge_shared" : " 27 GB", | 20260505 05:15:37.615 | INFO | "malloc" : " 14 GB", | 20260505 05:15:37.615 | INFO | "pinned" : " 29 GB" | 20260505 05:15:37.615 | INFO | } | 20260505 05:15:37.615 | INFO | Perf report for the last 5.13918 seconds: | 20260505 05:15:37.615 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:15:37.615 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:37.615 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:37.615 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:37.615 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260505 05:15:37.615 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:37.615 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:37.615 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 05:15:37.615 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:37.616 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:37.616 | INFO | Uploaded 36GiB to GPU in 1.60226s (22.4683GiB/s) | 20260505 05:15:37.616 | INFO | Downloaded 34GiB to GPU in 1.51981s (22.3712GiB/s) | 20260505 05:15:37.616 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 05:15:37.616 | INFO | Total mem usage: 15.1 GiB | 20260505 05:15:37.616 | INFO | All Time High Water Marks sum: 15.7 GiB (Savings: 4.09%) | 20260505 05:15:37.616 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -3.85%) | 20260505 05:15:37.616 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 05:15:37.616 | INFO | High Water Breakdown: | 20260505 05:15:37.616 | INFO | 59.8% : 8.68 GiB ( 96% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 05:15:37.616 | INFO | 40% : 5.8 GiB ( 87.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 05:15:37.616 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 05:15:37.616 | INFO | ------------------------------------------ | 20260505 05:15:37.616 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:15:37.616 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 05:15:37.616 | INFO | Total mem usage: 26.5 GiB | 20260505 05:15:37.616 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 05:15:37.616 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 05:15:37.616 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 05:15:37.616 | INFO | High Water Breakdown: | 20260505 05:15:37.616 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:15:37.616 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:15:37.616 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:15:37.616 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:15:37.616 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:15:37.616 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:15:37.616 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 05:15:37.616 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 05:15:37.616 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:15:37.616 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 05:15:37.616 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 05:15:37.616 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 05:15:37.616 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 05:15:37.617 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 05:15:37.617 | INFO | ------------------------------------------ | 20260505 05:15:37.617 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:15:37.617 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 05:15:37.617 | INFO | Total mem usage: 25.6 GiB | 20260505 05:15:37.617 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 05:15:37.617 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 05:15:37.617 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 05:15:37.617 | INFO | High Water Breakdown: | 20260505 05:15:37.617 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 05:15:37.617 | INFO | ------------------------------------------ | 20260505 05:15:37.617 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:15:37.617 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 05:15:37.617 | INFO | Total mem usage: 15.7 GiB | 20260505 05:15:37.617 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 05:15:37.617 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 05:15:37.617 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260505 05:15:37.617 | INFO | High Water Breakdown: | 20260505 05:15:37.617 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 05:15:37.617 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 05:15:37.617 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 05:15:37.617 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 05:15:37.617 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 05:15:37.617 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 05:15:37.617 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 05:15:37.617 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 05:15:37.617 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 05:15:37.617 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:15:37.617 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 05:15:37.617 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:15:37.617 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 05:15:37.617 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 05:15:37.617 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 05:15:37.617 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 05:15:37.617 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:15:37.617 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 05:15:37.617 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:15:37.617 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:15:37.617 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:15:37.617 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 05:15:37.617 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 05:15:37.617 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:15:37.617 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 05:15:37.617 | INFO | ------------------------------------------ | 20260505 05:15:37.617 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:15:37.617 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 05:15:37.617 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:15:37.617 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:15:37.617 | INFO | PerfReport: Full Execution -- 3.04e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260505 05:15:37.617 | INFO | PerfReport: 0.28%, AggregateStats -- 8.63e-02 / -nan, 3.93e-03 / 0.00e+00 | 20260505 05:15:37.617 | INFO | PerfReport: 10.87%, Baseline -- 3.30e+00 / -nan, 1.77e-02 / 0.00e+00 | 20260505 05:15:37.617 | INFO | PerfReport: 3.04%, Histogram -- 9.24e-01 / -nan, 5.22e-02 / 0.00e+00 | 20260505 05:15:37.617 | INFO | PerfReport: 16.04%, DME -- 3.90e+01 / -nan, 3.24e+00 / 0.00e+00 | 20260505 05:15:37.617 | INFO | PerfReport: 56.70%, FrameLabeling -- 1.72e+01 / -nan, 4.48e-02 / 0.00e+00 | 20260505 05:15:37.617 | INFO | PerfReport: 7.56%, PulseAccumulating -- 2.30e+00 / -nan, 1.51e-01 / 0.00e+00 | 20260505 05:15:37.617 | INFO | PerfReport: 3.90%, Metrics -- 1.18e+00 / -nan, 1.97e-01 / 0.00e+00 | 20260505 05:15:37.617 | INFO | PerfReport: 1.60%, PulseSerializing -- 4.87e-01 / -nan, 7.87e-02 / 0.00e+00 | 20260505 05:15:37.617 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 05:15:37.617 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:15:37.617 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:15:37.617 | INFO | PerfReport: Full Execution -- 9.11e+01 / -nan, 1.89e+01 / 0.00e+00 | 20260505 05:15:37.617 | INFO | PerfReport: 18.37%, Upload -- 1.67e+01 / 0.00e+00, 3.96e-01 / 0.00e+00 | 20260505 05:15:37.617 | INFO | PerfReport: 17.65%, Download -- 1.61e+01 / 0.00e+00, 7.35e-01 / 0.00e+00 | 20260505 05:15:37.617 | INFO | PerfReport: 33.33%, Compute -- 3.04e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260505 05:15:37.617 | INFO | PerfReport: 0.55%, LockUpload -- 5.05e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260505 05:15:37.617 | INFO | PerfReport: 0.00%, LockDownload -- 3.85e-04 / 0.00e+00, 9.16e-05 / 0.00e+00 | 20260505 05:15:37.617 | INFO | PerfReport: 30.08%, LockCompute -- 2.74e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260505 05:15:37.617 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.43e-02 / 0.00e+00 | 20260505 05:15:37.713 | INFO | Starting a round of Baz IO | 20260505 05:15:40.655 | INFO | Populating RTMetrics report | 20260505 05:15:40.655 | INFO | RTMetrics report populated | 20260505 05:15:40.655 | INFO | Opening tmp RTMetrics json file | 20260505 05:15:40.659 | INFO | Serializing RTMetrics to json | 20260505 05:15:40.661 | INFO | Serialization done, swapping tmp file now... | 20260505 05:15:40.661 | INFO | RTMetrics done | 20260505 05:15:40.763 | INFO | Finished a round of Baz IO: Wrote 2.194 GiB in 3.049 seconds | 20260505 05:15:40.763 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.23% / 0.09% / 59.68% | 20260505 05:15:40.763 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.02294% / 0.000178814% / 3.17891e-05% | 20260505 05:15:40.763 | INFO | Baz IO overall: | 20260505 05:15:40.763 | INFO | BytesWritten : 283983867118 | 20260505 05:15:40.763 | INFO | NumEvents : 133075540889 | 20260505 05:15:42.754 | INFO | Analyzing chunk frames = [622592,623104) | 20260505 05:15:42.754 | INFO | MemInfo: { | 20260505 05:15:42.754 | INFO | "gpu" : " 17 GB", | 20260505 05:15:42.754 | INFO | "huge" : "", | 20260505 05:15:42.755 | INFO | "huge_shared" : " 27 GB", | 20260505 05:15:42.755 | INFO | "malloc" : " 14 GB", | 20260505 05:15:42.755 | INFO | "pinned" : " 29 GB" | 20260505 05:15:42.755 | INFO | } | 20260505 05:15:42.755 | INFO | Perf report for the last 5.1394 seconds: | 20260505 05:15:42.755 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:15:42.755 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:42.755 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:42.755 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:42.755 | INFO | ANALYSIS : 58.0%, 3.0, 89.8ms, 0 | 20260505 05:15:42.755 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:42.755 | INFO | RT_METRICS : 45.7%, 1.0, 23.9ms, 0 | 20260505 05:15:42.755 | INFO | PRE_HQ : 57.4%, 2.2, 64.4ms, 0 | 20260505 05:15:42.755 | INFO | BAZWRITER : 58.9%, 4.2, 128.6ms, 0 | 20260505 05:15:42.755 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 05:15:42.755 | INFO | Uploaded 36GiB to GPU in 1.64428s (21.894GiB/s) | 20260505 05:15:42.755 | INFO | Downloaded 38GiB to GPU in 1.70943s (22.2297GiB/s) | 20260505 05:15:47.894 | INFO | Analyzing chunk frames = [623104,623616) | 20260505 05:15:47.894 | INFO | MemInfo: { | 20260505 05:15:47.894 | INFO | "gpu" : " 17 GB", | 20260505 05:15:47.894 | INFO | "huge" : "", | 20260505 05:15:47.894 | INFO | "huge_shared" : " 27 GB", | 20260505 05:15:47.894 | INFO | "malloc" : " 14 GB", | 20260505 05:15:47.894 | INFO | "pinned" : " 29 GB" | 20260505 05:15:47.894 | INFO | } | 20260505 05:15:47.894 | INFO | Perf report for the last 5.13944 seconds: | 20260505 05:15:47.894 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:15:47.894 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:47.894 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:47.894 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:47.894 | INFO | ANALYSIS : 57.9%, 3.0, 90.2ms, 0 | 20260505 05:15:47.894 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:47.894 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:47.894 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:15:47.894 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:47.894 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:47.895 | INFO | Uploaded 36GiB to GPU in 1.60067s (22.4906GiB/s) | 20260505 05:15:47.895 | INFO | Downloaded 34GiB to GPU in 1.51732s (22.4079GiB/s) | 20260505 05:15:53.033 | INFO | Analyzing chunk frames = [623616,624128) | 20260505 05:15:53.033 | INFO | MemInfo: { | 20260505 05:15:53.033 | INFO | "gpu" : " 17 GB", | 20260505 05:15:53.033 | INFO | "huge" : "", | 20260505 05:15:53.033 | INFO | "huge_shared" : " 27 GB", | 20260505 05:15:53.033 | INFO | "malloc" : " 14 GB", | 20260505 05:15:53.033 | INFO | "pinned" : " 29 GB" | 20260505 05:15:53.033 | INFO | } | 20260505 05:15:53.033 | INFO | Perf report for the last 5.13924 seconds: | 20260505 05:15:53.033 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:15:53.033 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:53.034 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:53.034 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:53.034 | INFO | ANALYSIS : 57.2%, 3.0, 88.9ms, 0 | 20260505 05:15:53.034 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:53.034 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:53.034 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 05:15:53.034 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:53.034 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:53.034 | INFO | Uploaded 36GiB to GPU in 1.60406s (22.443GiB/s) | 20260505 05:15:53.034 | INFO | Downloaded 34GiB to GPU in 1.52199s (22.3392GiB/s) | 20260505 05:15:58.173 | INFO | Analyzing chunk frames = [624128,624640) | 20260505 05:15:58.173 | INFO | MemInfo: { | 20260505 05:15:58.173 | INFO | "gpu" : " 17 GB", | 20260505 05:15:58.173 | INFO | "huge" : "", | 20260505 05:15:58.173 | INFO | "huge_shared" : " 27 GB", | 20260505 05:15:58.173 | INFO | "malloc" : " 14 GB", | 20260505 05:15:58.173 | INFO | "pinned" : " 29 GB" | 20260505 05:15:58.173 | INFO | } | 20260505 05:15:58.173 | INFO | Perf report for the last 5.13934 seconds: | 20260505 05:15:58.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:15:58.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:58.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:58.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:58.173 | INFO | ANALYSIS : 56.9%, 3.0, 88.3ms, 0 | 20260505 05:15:58.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:58.173 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:58.173 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 05:15:58.173 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:58.173 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:15:58.173 | INFO | Uploaded 36GiB to GPU in 1.60288s (22.4596GiB/s) | 20260505 05:15:58.173 | INFO | Downloaded 34GiB to GPU in 1.51976s (22.372GiB/s) | 20260505 05:16:03.312 | INFO | Analyzing chunk frames = [624640,625152) | 20260505 05:16:03.312 | INFO | MemInfo: { | 20260505 05:16:03.312 | INFO | "gpu" : " 17 GB", | 20260505 05:16:03.312 | INFO | "huge" : "", | 20260505 05:16:03.312 | INFO | "huge_shared" : " 27 GB", | 20260505 05:16:03.312 | INFO | "malloc" : " 14 GB", | 20260505 05:16:03.312 | INFO | "pinned" : " 29 GB" | 20260505 05:16:03.312 | INFO | } | 20260505 05:16:03.312 | INFO | Perf report for the last 5.13936 seconds: | 20260505 05:16:03.312 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:16:03.312 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:03.312 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:03.312 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:03.312 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260505 05:16:03.312 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:03.312 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:03.312 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260505 05:16:03.312 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:03.312 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:03.313 | INFO | Uploaded 36GiB to GPU in 1.60237s (22.4668GiB/s) | 20260505 05:16:03.313 | INFO | Downloaded 34GiB to GPU in 1.51918s (22.3805GiB/s) | 20260505 05:16:08.451 | INFO | Analyzing chunk frames = [625152,625664) | 20260505 05:16:08.451 | INFO | MemInfo: { | 20260505 05:16:08.451 | INFO | "gpu" : " 17 GB", | 20260505 05:16:08.451 | INFO | "huge" : "", | 20260505 05:16:08.451 | INFO | "huge_shared" : " 27 GB", | 20260505 05:16:08.451 | INFO | "malloc" : " 14 GB", | 20260505 05:16:08.451 | INFO | "pinned" : " 29 GB" | 20260505 05:16:08.451 | INFO | } | 20260505 05:16:08.451 | INFO | Perf report for the last 5.13925 seconds: | 20260505 05:16:08.451 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:16:08.451 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:08.451 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:08.451 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:08.452 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260505 05:16:08.452 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:08.452 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:08.452 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:16:08.452 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:08.452 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:08.452 | INFO | Uploaded 36GiB to GPU in 1.60148s (22.4792GiB/s) | 20260505 05:16:08.452 | INFO | Downloaded 34GiB to GPU in 1.51771s (22.4021GiB/s) | 20260505 05:16:13.591 | INFO | Analyzing chunk frames = [625664,626176) | 20260505 05:16:13.591 | INFO | MemInfo: { | 20260505 05:16:13.591 | INFO | "gpu" : " 17 GB", | 20260505 05:16:13.591 | INFO | "huge" : "", | 20260505 05:16:13.591 | INFO | "huge_shared" : " 27 GB", | 20260505 05:16:13.591 | INFO | "malloc" : " 14 GB", | 20260505 05:16:13.591 | INFO | "pinned" : " 29 GB" | 20260505 05:16:13.591 | INFO | } | 20260505 05:16:13.591 | INFO | Perf report for the last 5.13938 seconds: | 20260505 05:16:13.591 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:16:13.591 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:13.591 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:13.591 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:13.591 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260505 05:16:13.591 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:13.591 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:13.591 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 05:16:13.591 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:13.591 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:13.591 | INFO | Uploaded 36GiB to GPU in 1.60183s (22.4743GiB/s) | 20260505 05:16:13.591 | INFO | Downloaded 34GiB to GPU in 1.51969s (22.373GiB/s) | 20260505 05:16:18.730 | INFO | Analyzing chunk frames = [626176,626688) | 20260505 05:16:18.730 | INFO | MemInfo: { | 20260505 05:16:18.730 | INFO | "gpu" : " 17 GB", | 20260505 05:16:18.730 | INFO | "huge" : "", | 20260505 05:16:18.730 | INFO | "huge_shared" : " 27 GB", | 20260505 05:16:18.730 | INFO | "malloc" : " 14 GB", | 20260505 05:16:18.730 | INFO | "pinned" : " 29 GB" | 20260505 05:16:18.730 | INFO | } | 20260505 05:16:18.730 | INFO | Perf report for the last 5.13927 seconds: | 20260505 05:16:18.730 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:16:18.730 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:18.730 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:18.730 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:18.730 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260505 05:16:18.730 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:18.730 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:18.730 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260505 05:16:18.730 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:18.730 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:18.731 | INFO | Uploaded 36GiB to GPU in 1.60324s (22.4545GiB/s) | 20260505 05:16:18.731 | INFO | Downloaded 34GiB to GPU in 1.52141s (22.3478GiB/s) | 20260505 05:16:18.827 | INFO | Starting a round of Baz IO | 20260505 05:16:21.770 | INFO | Populating RTMetrics report | 20260505 05:16:21.771 | INFO | RTMetrics report populated | 20260505 05:16:21.771 | INFO | Opening tmp RTMetrics json file | 20260505 05:16:21.775 | INFO | Serializing RTMetrics to json | 20260505 05:16:21.776 | INFO | Serialization done, swapping tmp file now... | 20260505 05:16:21.776 | INFO | RTMetrics done | 20260505 05:16:21.879 | INFO | Finished a round of Baz IO: Wrote 2.199 GiB in 3.051 seconds | 20260505 05:16:21.879 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.14% / 0.09% / 59.77% | 20260505 05:16:21.879 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.01478% / 0.000135104% / 3.57628e-05% | 20260505 05:16:21.879 | INFO | Baz IO overall: | 20260505 05:16:21.879 | INFO | BytesWritten : 286344850779 | 20260505 05:16:21.879 | INFO | NumEvents : 134199835568 | 20260505 05:16:23.869 | INFO | Analyzing chunk frames = [626688,627200) | 20260505 05:16:23.869 | INFO | MemInfo: { | 20260505 05:16:23.869 | INFO | "gpu" : " 17 GB", | 20260505 05:16:23.869 | INFO | "huge" : "", | 20260505 05:16:23.869 | INFO | "huge_shared" : " 27 GB", | 20260505 05:16:23.869 | INFO | "malloc" : " 14 GB", | 20260505 05:16:23.869 | INFO | "pinned" : " 29 GB" | 20260505 05:16:23.869 | INFO | } | 20260505 05:16:23.869 | INFO | Perf report for the last 5.13935 seconds: | 20260505 05:16:23.869 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:16:23.870 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:23.870 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:23.870 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:23.870 | INFO | ANALYSIS : 58.0%, 2.9, 88.9ms, 0 | 20260505 05:16:23.870 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:23.870 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260505 05:16:23.870 | INFO | PRE_HQ : 57.3%, 2.2, 64.1ms, 0 | 20260505 05:16:23.870 | INFO | BAZWRITER : 58.9%, 4.3, 130.3ms, 0 | 20260505 05:16:23.870 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 05:16:23.870 | INFO | Uploaded 36GiB to GPU in 1.64634s (21.8667GiB/s) | 20260505 05:16:23.870 | INFO | Downloaded 38GiB to GPU in 1.71138s (22.2043GiB/s) | 20260505 05:16:29.009 | INFO | Analyzing chunk frames = [627200,627712) | 20260505 05:16:29.009 | INFO | MemInfo: { | 20260505 05:16:29.009 | INFO | "gpu" : " 17 GB", | 20260505 05:16:29.009 | INFO | "huge" : "", | 20260505 05:16:29.009 | INFO | "huge_shared" : " 27 GB", | 20260505 05:16:29.009 | INFO | "malloc" : " 14 GB", | 20260505 05:16:29.009 | INFO | "pinned" : " 29 GB" | 20260505 05:16:29.009 | INFO | } | 20260505 05:16:29.009 | INFO | Perf report for the last 5.13955 seconds: | 20260505 05:16:29.009 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:16:29.009 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:29.009 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:29.009 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:29.009 | INFO | ANALYSIS : 57.8%, 3.0, 90.2ms, 0 | 20260505 05:16:29.009 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:29.009 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:29.009 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:16:29.009 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:29.009 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:29.009 | INFO | Uploaded 36GiB to GPU in 1.60025s (22.4965GiB/s) | 20260505 05:16:29.009 | INFO | Downloaded 34GiB to GPU in 1.51653s (22.4195GiB/s) | 20260505 05:16:34.148 | INFO | Analyzing chunk frames = [627712,628224) | 20260505 05:16:34.148 | INFO | MemInfo: { | 20260505 05:16:34.148 | INFO | "gpu" : " 17 GB", | 20260505 05:16:34.148 | INFO | "huge" : "", | 20260505 05:16:34.148 | INFO | "huge_shared" : " 27 GB", | 20260505 05:16:34.148 | INFO | "malloc" : " 14 GB", | 20260505 05:16:34.148 | INFO | "pinned" : " 29 GB" | 20260505 05:16:34.148 | INFO | } | 20260505 05:16:34.148 | INFO | Perf report for the last 5.13932 seconds: | 20260505 05:16:34.148 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:16:34.148 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:34.148 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:34.148 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:34.148 | INFO | ANALYSIS : 57.2%, 3.0, 88.9ms, 0 | 20260505 05:16:34.149 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:34.149 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:34.149 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 05:16:34.149 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:34.149 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:34.149 | INFO | Uploaded 36GiB to GPU in 1.60384s (22.4461GiB/s) | 20260505 05:16:34.149 | INFO | Downloaded 34GiB to GPU in 1.52111s (22.352GiB/s) | 20260505 05:16:39.287 | INFO | Analyzing chunk frames = [628224,628736) | 20260505 05:16:39.287 | INFO | MemInfo: { | 20260505 05:16:39.287 | INFO | "gpu" : " 17 GB", | 20260505 05:16:39.287 | INFO | "huge" : "", | 20260505 05:16:39.287 | INFO | "huge_shared" : " 27 GB", | 20260505 05:16:39.287 | INFO | "malloc" : " 14 GB", | 20260505 05:16:39.288 | INFO | "pinned" : " 29 GB" | 20260505 05:16:39.288 | INFO | } | 20260505 05:16:39.288 | INFO | Perf report for the last 5.13928 seconds: | 20260505 05:16:39.288 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:16:39.288 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:39.288 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:39.288 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:39.288 | INFO | ANALYSIS : 56.9%, 3.0, 88.2ms, 0 | 20260505 05:16:39.288 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:39.288 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:39.288 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 05:16:39.288 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:39.288 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:39.288 | INFO | Uploaded 36GiB to GPU in 1.60431s (22.4396GiB/s) | 20260505 05:16:39.288 | INFO | Downloaded 34GiB to GPU in 1.52102s (22.3534GiB/s) | 20260505 05:16:44.427 | INFO | Analyzing chunk frames = [628736,629248) | 20260505 05:16:44.427 | INFO | MemInfo: { | 20260505 05:16:44.427 | INFO | "gpu" : " 17 GB", | 20260505 05:16:44.427 | INFO | "huge" : "", | 20260505 05:16:44.427 | INFO | "huge_shared" : " 27 GB", | 20260505 05:16:44.427 | INFO | "malloc" : " 14 GB", | 20260505 05:16:44.427 | INFO | "pinned" : " 29 GB" | 20260505 05:16:44.427 | INFO | } | 20260505 05:16:44.427 | INFO | Perf report for the last 5.13934 seconds: | 20260505 05:16:44.427 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:16:44.427 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:44.427 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:44.427 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:44.427 | INFO | ANALYSIS : 56.8%, 3.0, 88.7ms, 0 | 20260505 05:16:44.427 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:44.427 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:44.427 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 05:16:44.427 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:44.427 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:44.428 | INFO | Uploaded 36GiB to GPU in 1.60132s (22.4815GiB/s) | 20260505 05:16:44.428 | INFO | Downloaded 34GiB to GPU in 1.51777s (22.4012GiB/s) | 20260505 05:16:49.566 | INFO | Analyzing chunk frames = [629248,629760) | 20260505 05:16:49.566 | INFO | MemInfo: { | 20260505 05:16:49.566 | INFO | "gpu" : " 17 GB", | 20260505 05:16:49.566 | INFO | "huge" : "", | 20260505 05:16:49.566 | INFO | "huge_shared" : " 27 GB", | 20260505 05:16:49.566 | INFO | "malloc" : " 14 GB", | 20260505 05:16:49.566 | INFO | "pinned" : " 29 GB" | 20260505 05:16:49.566 | INFO | } | 20260505 05:16:49.566 | INFO | Perf report for the last 5.13932 seconds: | 20260505 05:16:49.566 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:16:49.566 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:49.566 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:49.566 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:49.566 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260505 05:16:49.566 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:49.566 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:49.567 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 05:16:49.567 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:49.567 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:49.567 | INFO | Uploaded 36GiB to GPU in 1.60245s (22.4656GiB/s) | 20260505 05:16:49.567 | INFO | Downloaded 34GiB to GPU in 1.51903s (22.3826GiB/s) | 20260505 05:16:54.705 | INFO | Analyzing chunk frames = [629760,630272) | 20260505 05:16:54.705 | INFO | MemInfo: { | 20260505 05:16:54.705 | INFO | "gpu" : " 17 GB", | 20260505 05:16:54.705 | INFO | "huge" : "", | 20260505 05:16:54.705 | INFO | "huge_shared" : " 27 GB", | 20260505 05:16:54.705 | INFO | "malloc" : " 14 GB", | 20260505 05:16:54.706 | INFO | "pinned" : " 29 GB" | 20260505 05:16:54.706 | INFO | } | 20260505 05:16:54.706 | INFO | Perf report for the last 5.13925 seconds: | 20260505 05:16:54.706 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:16:54.706 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:54.706 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:54.706 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:54.706 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260505 05:16:54.706 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:54.706 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:54.706 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 05:16:54.706 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:54.706 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:54.706 | INFO | Uploaded 36GiB to GPU in 1.60221s (22.4689GiB/s) | 20260505 05:16:54.706 | INFO | Downloaded 34GiB to GPU in 1.51908s (22.382GiB/s) | 20260505 05:16:59.845 | INFO | Analyzing chunk frames = [630272,630784) | 20260505 05:16:59.845 | INFO | MemInfo: { | 20260505 05:16:59.845 | INFO | "gpu" : " 17 GB", | 20260505 05:16:59.845 | INFO | "huge" : "", | 20260505 05:16:59.845 | INFO | "huge_shared" : " 27 GB", | 20260505 05:16:59.845 | INFO | "malloc" : " 14 GB", | 20260505 05:16:59.845 | INFO | "pinned" : " 29 GB" | 20260505 05:16:59.845 | INFO | } | 20260505 05:16:59.845 | INFO | Perf report for the last 5.13941 seconds: | 20260505 05:16:59.845 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:16:59.845 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:59.845 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:59.845 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:59.845 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260505 05:16:59.845 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:59.845 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:59.845 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 05:16:59.845 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:59.845 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:16:59.845 | INFO | Uploaded 36GiB to GPU in 1.60292s (22.459GiB/s) | 20260505 05:16:59.845 | INFO | Downloaded 34GiB to GPU in 1.52044s (22.3619GiB/s) | 20260505 05:16:59.846 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 05:16:59.846 | INFO | Total mem usage: 15.1 GiB | 20260505 05:16:59.846 | INFO | All Time High Water Marks sum: 15.7 GiB (Savings: 4.09%) | 20260505 05:16:59.846 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -3.75%) | 20260505 05:16:59.846 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 05:16:59.846 | INFO | High Water Breakdown: | 20260505 05:16:59.846 | INFO | 59.9% : 8.69 GiB ( 96.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 05:16:59.846 | INFO | 40% : 5.8 GiB ( 87.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 05:16:59.846 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 05:16:59.846 | INFO | ------------------------------------------ | 20260505 05:16:59.846 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:16:59.846 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 05:16:59.846 | INFO | Total mem usage: 26.5 GiB | 20260505 05:16:59.846 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 05:16:59.846 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 05:16:59.846 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 05:16:59.846 | INFO | High Water Breakdown: | 20260505 05:16:59.846 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:16:59.846 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:16:59.846 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:16:59.846 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:16:59.846 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:16:59.846 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:16:59.846 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 05:16:59.846 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 05:16:59.846 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:16:59.846 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 05:16:59.846 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 05:16:59.846 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 05:16:59.846 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 05:16:59.846 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 05:16:59.846 | INFO | ------------------------------------------ | 20260505 05:16:59.846 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:16:59.846 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 05:16:59.846 | INFO | Total mem usage: 25.6 GiB | 20260505 05:16:59.846 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 05:16:59.846 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 05:16:59.846 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 05:16:59.846 | INFO | High Water Breakdown: | 20260505 05:16:59.846 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 05:16:59.846 | INFO | ------------------------------------------ | 20260505 05:16:59.846 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:16:59.846 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 05:16:59.846 | INFO | Total mem usage: 15.7 GiB | 20260505 05:16:59.846 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 05:16:59.846 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 05:16:59.846 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260505 05:16:59.846 | INFO | High Water Breakdown: | 20260505 05:16:59.846 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 05:16:59.846 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 05:16:59.846 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 05:16:59.846 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 05:16:59.846 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 05:16:59.846 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 05:16:59.846 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 05:16:59.846 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 05:16:59.846 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 05:16:59.846 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:16:59.846 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 05:16:59.846 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:16:59.846 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 05:16:59.846 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 05:16:59.846 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 05:16:59.846 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 05:16:59.846 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:16:59.846 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 05:16:59.846 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:16:59.846 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:16:59.846 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:16:59.846 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 05:16:59.846 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 05:16:59.846 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:16:59.846 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 05:16:59.846 | INFO | ------------------------------------------ | 20260505 05:16:59.846 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:16:59.846 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 05:16:59.846 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:16:59.846 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:16:59.846 | INFO | PerfReport: Full Execution -- 3.04e+01 / -nan, 1.28e+01 / 0.00e+00 | 20260505 05:16:59.846 | INFO | PerfReport: 0.28%, AggregateStats -- 8.63e-02 / -nan, 4.17e-03 / 0.00e+00 | 20260505 05:16:59.846 | INFO | PerfReport: 10.86%, Baseline -- 3.30e+00 / -nan, 1.67e-02 / 0.00e+00 | 20260505 05:16:59.846 | INFO | PerfReport: 3.04%, Histogram -- 9.24e-01 / -nan, 5.22e-02 / 0.00e+00 | 20260505 05:16:59.847 | INFO | PerfReport: 16.04%, DME -- 3.89e+01 / -nan, 3.19e+00 / 0.00e+00 | 20260505 05:16:59.847 | INFO | PerfReport: 56.71%, FrameLabeling -- 1.72e+01 / -nan, 4.54e-02 / 0.00e+00 | 20260505 05:16:59.847 | INFO | PerfReport: 7.56%, PulseAccumulating -- 2.30e+00 / -nan, 1.51e-01 / 0.00e+00 | 20260505 05:16:59.847 | INFO | PerfReport: 3.90%, Metrics -- 1.18e+00 / -nan, 1.96e-01 / 0.00e+00 | 20260505 05:16:59.847 | INFO | PerfReport: 1.60%, PulseSerializing -- 4.86e-01 / -nan, 7.87e-02 / 0.00e+00 | 20260505 05:16:59.847 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 05:16:59.847 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:16:59.847 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:16:59.847 | INFO | PerfReport: Full Execution -- 9.10e+01 / -nan, 1.88e+01 / 0.00e+00 | 20260505 05:16:59.847 | INFO | PerfReport: 18.40%, Upload -- 1.67e+01 / 0.00e+00, 3.98e-01 / 0.00e+00 | 20260505 05:16:59.847 | INFO | PerfReport: 17.66%, Download -- 1.61e+01 / 0.00e+00, 7.27e-01 / 0.00e+00 | 20260505 05:16:59.847 | INFO | PerfReport: 33.34%, Compute -- 3.04e+01 / 0.00e+00, 1.28e+01 / 0.00e+00 | 20260505 05:16:59.847 | INFO | PerfReport: 0.55%, LockUpload -- 5.04e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260505 05:16:59.847 | INFO | PerfReport: 0.00%, LockDownload -- 3.93e-04 / 0.00e+00, 2.01e-04 / 0.00e+00 | 20260505 05:16:59.847 | INFO | PerfReport: 30.03%, LockCompute -- 2.73e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260505 05:16:59.847 | INFO | PerfReport: 0.02%, Misc -- 1.39e-02 / 0.00e+00, 1.72e-02 / 0.00e+00 | 20260505 05:16:59.943 | INFO | Starting a round of Baz IO | 20260505 05:17:02.888 | INFO | Populating RTMetrics report | 20260505 05:17:02.888 | INFO | RTMetrics report populated | 20260505 05:17:02.888 | INFO | Opening tmp RTMetrics json file | 20260505 05:17:02.893 | INFO | Serializing RTMetrics to json | 20260505 05:17:02.894 | INFO | Serialization done, swapping tmp file now... | 20260505 05:17:02.894 | INFO | RTMetrics done | 20260505 05:17:02.988 | INFO | Finished a round of Baz IO: Wrote 2.2 GiB in 3.045 seconds | 20260505 05:17:02.989 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.06% / 0.09% / 59.86% | 20260505 05:17:02.989 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.9626% / 0.000162919% / 4.76837e-05% | 20260505 05:17:02.989 | INFO | Baz IO overall: | 20260505 05:17:02.989 | INFO | BytesWritten : 288707276647 | 20260505 05:17:02.989 | INFO | NumEvents : 135324175662 | 20260505 05:17:04.984 | INFO | Analyzing chunk frames = [630784,631296) | 20260505 05:17:04.984 | INFO | MemInfo: { | 20260505 05:17:04.984 | INFO | "gpu" : " 17 GB", | 20260505 05:17:04.984 | INFO | "huge" : "", | 20260505 05:17:04.984 | INFO | "huge_shared" : " 27 GB", | 20260505 05:17:04.984 | INFO | "malloc" : " 14 GB", | 20260505 05:17:04.984 | INFO | "pinned" : " 29 GB" | 20260505 05:17:04.984 | INFO | } | 20260505 05:17:04.984 | INFO | Perf report for the last 5.13909 seconds: | 20260505 05:17:04.984 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:17:04.984 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:04.984 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:04.984 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:04.984 | INFO | ANALYSIS : 58.1%, 2.9, 89.8ms, 0 | 20260505 05:17:04.984 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:04.984 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260505 05:17:04.984 | INFO | PRE_HQ : 57.3%, 2.2, 64.5ms, 0 | 20260505 05:17:04.984 | INFO | BAZWRITER : 58.8%, 4.3, 129.7ms, 0 | 20260505 05:17:04.984 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 05:17:04.985 | INFO | Uploaded 36GiB to GPU in 1.64728s (21.8542GiB/s) | 20260505 05:17:04.985 | INFO | Downloaded 38GiB to GPU in 1.71225s (22.193GiB/s) | 20260505 05:17:10.123 | INFO | Analyzing chunk frames = [631296,631808) | 20260505 05:17:10.123 | INFO | MemInfo: { | 20260505 05:17:10.123 | INFO | "gpu" : " 17 GB", | 20260505 05:17:10.123 | INFO | "huge" : "", | 20260505 05:17:10.123 | INFO | "huge_shared" : " 27 GB", | 20260505 05:17:10.124 | INFO | "malloc" : " 14 GB", | 20260505 05:17:10.124 | INFO | "pinned" : " 29 GB" | 20260505 05:17:10.124 | INFO | } | 20260505 05:17:10.124 | INFO | Perf report for the last 5.13946 seconds: | 20260505 05:17:10.124 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:17:10.124 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:10.124 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:10.124 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:10.124 | INFO | ANALYSIS : 57.8%, 3.0, 90.2ms, 0 | 20260505 05:17:10.124 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:10.124 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:10.124 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 05:17:10.124 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:10.124 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:10.124 | INFO | Uploaded 36GiB to GPU in 1.60035s (22.4951GiB/s) | 20260505 05:17:10.124 | INFO | Downloaded 34GiB to GPU in 1.51633s (22.4226GiB/s) | 20260505 05:17:15.263 | INFO | Analyzing chunk frames = [631808,632320) | 20260505 05:17:15.263 | INFO | MemInfo: { | 20260505 05:17:15.263 | INFO | "gpu" : " 17 GB", | 20260505 05:17:15.263 | INFO | "huge" : "", | 20260505 05:17:15.263 | INFO | "huge_shared" : " 27 GB", | 20260505 05:17:15.263 | INFO | "malloc" : " 14 GB", | 20260505 05:17:15.263 | INFO | "pinned" : " 29 GB" | 20260505 05:17:15.263 | INFO | } | 20260505 05:17:15.263 | INFO | Perf report for the last 5.13947 seconds: | 20260505 05:17:15.263 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:17:15.263 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:15.263 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:15.263 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:15.263 | INFO | ANALYSIS : 57.2%, 3.0, 88.9ms, 0 | 20260505 05:17:15.263 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:15.263 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:15.263 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 05:17:15.263 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:15.263 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:15.263 | INFO | Uploaded 36GiB to GPU in 1.60389s (22.4455GiB/s) | 20260505 05:17:15.263 | INFO | Downloaded 34GiB to GPU in 1.52173s (22.3429GiB/s) | 20260505 05:17:20.402 | INFO | Analyzing chunk frames = [632320,632832) | 20260505 05:17:20.402 | INFO | MemInfo: { | 20260505 05:17:20.402 | INFO | "gpu" : " 17 GB", | 20260505 05:17:20.402 | INFO | "huge" : "", | 20260505 05:17:20.402 | INFO | "huge_shared" : " 27 GB", | 20260505 05:17:20.402 | INFO | "malloc" : " 14 GB", | 20260505 05:17:20.402 | INFO | "pinned" : " 29 GB" | 20260505 05:17:20.402 | INFO | } | 20260505 05:17:20.402 | INFO | Perf report for the last 5.13925 seconds: | 20260505 05:17:20.402 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:17:20.402 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:20.402 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:20.402 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:20.402 | INFO | ANALYSIS : 56.9%, 3.0, 88.2ms, 0 | 20260505 05:17:20.402 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:20.402 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:20.402 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 05:17:20.403 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:20.403 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:20.403 | INFO | Uploaded 36GiB to GPU in 1.60387s (22.4457GiB/s) | 20260505 05:17:20.403 | INFO | Downloaded 34GiB to GPU in 1.52089s (22.3553GiB/s) | 20260505 05:17:25.541 | INFO | Analyzing chunk frames = [632832,633344) | 20260505 05:17:25.542 | INFO | MemInfo: { | 20260505 05:17:25.542 | INFO | "gpu" : " 17 GB", | 20260505 05:17:25.542 | INFO | "huge" : "", | 20260505 05:17:25.542 | INFO | "huge_shared" : " 27 GB", | 20260505 05:17:25.542 | INFO | "malloc" : " 14 GB", | 20260505 05:17:25.542 | INFO | "pinned" : " 29 GB" | 20260505 05:17:25.542 | INFO | } | 20260505 05:17:25.542 | INFO | Perf report for the last 5.1395 seconds: | 20260505 05:17:25.542 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:17:25.542 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:25.542 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:25.542 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:25.542 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260505 05:17:25.542 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:25.542 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:25.542 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 05:17:25.542 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:25.542 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:25.543 | INFO | Uploaded 36GiB to GPU in 1.60134s (22.4811GiB/s) | 20260505 05:17:25.543 | INFO | Downloaded 34GiB to GPU in 1.51785s (22.4GiB/s) | 20260505 05:17:30.681 | INFO | Analyzing chunk frames = [633344,633856) | 20260505 05:17:30.681 | INFO | MemInfo: { | 20260505 05:17:30.681 | INFO | "gpu" : " 17 GB", | 20260505 05:17:30.681 | INFO | "huge" : "", | 20260505 05:17:30.681 | INFO | "huge_shared" : " 27 GB", | 20260505 05:17:30.681 | INFO | "malloc" : " 14 GB", | 20260505 05:17:30.681 | INFO | "pinned" : " 29 GB" | 20260505 05:17:30.681 | INFO | } | 20260505 05:17:30.681 | INFO | Perf report for the last 5.13903 seconds: | 20260505 05:17:30.681 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:17:30.681 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:30.681 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:30.681 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:30.681 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260505 05:17:30.681 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:30.681 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:30.681 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 05:17:30.681 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:30.681 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:30.681 | INFO | Uploaded 36GiB to GPU in 1.60195s (22.4726GiB/s) | 20260505 05:17:30.681 | INFO | Downloaded 34GiB to GPU in 1.5185s (22.3905GiB/s) | 20260505 05:17:35.820 | INFO | Analyzing chunk frames = [633856,634368) | 20260505 05:17:35.820 | INFO | MemInfo: { | 20260505 05:17:35.820 | INFO | "gpu" : " 17 GB", | 20260505 05:17:35.820 | INFO | "huge" : "", | 20260505 05:17:35.820 | INFO | "huge_shared" : " 27 GB", | 20260505 05:17:35.820 | INFO | "malloc" : " 14 GB", | 20260505 05:17:35.820 | INFO | "pinned" : " 29 GB" | 20260505 05:17:35.820 | INFO | } | 20260505 05:17:35.821 | INFO | Perf report for the last 5.13958 seconds: | 20260505 05:17:35.821 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:17:35.821 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:35.821 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:35.821 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:35.821 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 05:17:35.821 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:35.821 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:35.821 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 05:17:35.821 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:35.821 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:35.821 | INFO | Uploaded 36GiB to GPU in 1.60195s (22.4727GiB/s) | 20260505 05:17:35.821 | INFO | Downloaded 34GiB to GPU in 1.51905s (22.3824GiB/s) | 20260505 05:17:40.959 | INFO | Analyzing chunk frames = [634368,634880) | 20260505 05:17:40.959 | INFO | MemInfo: { | 20260505 05:17:40.959 | INFO | "gpu" : " 17 GB", | 20260505 05:17:40.959 | INFO | "huge" : "", | 20260505 05:17:40.959 | INFO | "huge_shared" : " 27 GB", | 20260505 05:17:40.959 | INFO | "malloc" : " 14 GB", | 20260505 05:17:40.959 | INFO | "pinned" : " 29 GB" | 20260505 05:17:40.960 | INFO | } | 20260505 05:17:40.960 | INFO | Perf report for the last 5.13924 seconds: | 20260505 05:17:40.960 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:17:40.960 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:40.960 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:40.960 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:40.960 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260505 05:17:40.960 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:40.960 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:40.960 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:17:40.960 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:40.960 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:40.960 | INFO | Uploaded 36GiB to GPU in 1.60297s (22.4583GiB/s) | 20260505 05:17:40.960 | INFO | Downloaded 34GiB to GPU in 1.51993s (22.3695GiB/s) | 20260505 05:17:41.057 | INFO | Starting a round of Baz IO | 20260505 05:17:44.003 | INFO | Populating RTMetrics report | 20260505 05:17:44.003 | INFO | RTMetrics report populated | 20260505 05:17:44.003 | INFO | Opening tmp RTMetrics json file | 20260505 05:17:44.009 | INFO | Serializing RTMetrics to json | 20260505 05:17:44.010 | INFO | Serialization done, swapping tmp file now... | 20260505 05:17:44.011 | INFO | RTMetrics done | 20260505 05:17:44.103 | INFO | Finished a round of Baz IO: Wrote 2.202 GiB in 3.046 seconds | 20260505 05:17:44.103 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.97% / 0.09% / 59.94% | 20260505 05:17:44.103 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.91181% / 0.000170867% / 4.76837e-05% | 20260505 05:17:44.103 | INFO | Baz IO overall: | 20260505 05:17:44.104 | INFO | BytesWritten : 291071466056 | 20260505 05:17:44.104 | INFO | NumEvents : 136449186994 | 20260505 05:17:46.099 | INFO | Analyzing chunk frames = [634880,635392) | 20260505 05:17:46.099 | INFO | MemInfo: { | 20260505 05:17:46.099 | INFO | "gpu" : " 17 GB", | 20260505 05:17:46.099 | INFO | "huge" : "", | 20260505 05:17:46.099 | INFO | "huge_shared" : " 27 GB", | 20260505 05:17:46.099 | INFO | "malloc" : " 14 GB", | 20260505 05:17:46.099 | INFO | "pinned" : " 29 GB" | 20260505 05:17:46.099 | INFO | } | 20260505 05:17:46.099 | INFO | Perf report for the last 5.13924 seconds: | 20260505 05:17:46.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:17:46.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:46.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:46.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:46.099 | INFO | ANALYSIS : 58.1%, 3.0, 89.8ms, 0 | 20260505 05:17:46.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:46.099 | INFO | RT_METRICS : 45.7%, 1.0, 23.9ms, 0 | 20260505 05:17:46.099 | INFO | PRE_HQ : 57.4%, 2.2, 64.2ms, 0 | 20260505 05:17:46.099 | INFO | BAZWRITER : 58.8%, 4.3, 128.7ms, 0 | 20260505 05:17:46.099 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 05:17:46.100 | INFO | Uploaded 36GiB to GPU in 1.64692s (21.859GiB/s) | 20260505 05:17:46.100 | INFO | Downloaded 38GiB to GPU in 1.70931s (22.2312GiB/s) | 20260505 05:17:51.238 | INFO | Analyzing chunk frames = [635392,635904) | 20260505 05:17:51.238 | INFO | MemInfo: { | 20260505 05:17:51.238 | INFO | "gpu" : " 17 GB", | 20260505 05:17:51.238 | INFO | "huge" : "", | 20260505 05:17:51.238 | INFO | "huge_shared" : " 27 GB", | 20260505 05:17:51.238 | INFO | "malloc" : " 14 GB", | 20260505 05:17:51.238 | INFO | "pinned" : " 29 GB" | 20260505 05:17:51.238 | INFO | } | 20260505 05:17:51.238 | INFO | Perf report for the last 5.1394 seconds: | 20260505 05:17:51.238 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:17:51.238 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:51.239 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:51.239 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:51.239 | INFO | ANALYSIS : 57.8%, 3.0, 90.2ms, 0 | 20260505 05:17:51.239 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:51.239 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:51.239 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 05:17:51.239 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:51.239 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:51.239 | INFO | Uploaded 36GiB to GPU in 1.59952s (22.5067GiB/s) | 20260505 05:17:51.239 | INFO | Downloaded 34GiB to GPU in 1.51655s (22.4193GiB/s) | 20260505 05:17:56.377 | INFO | Analyzing chunk frames = [635904,636416) | 20260505 05:17:56.377 | INFO | MemInfo: { | 20260505 05:17:56.377 | INFO | "gpu" : " 17 GB", | 20260505 05:17:56.378 | INFO | "huge" : "", | 20260505 05:17:56.378 | INFO | "huge_shared" : " 27 GB", | 20260505 05:17:56.378 | INFO | "malloc" : " 14 GB", | 20260505 05:17:56.378 | INFO | "pinned" : " 29 GB" | 20260505 05:17:56.378 | INFO | } | 20260505 05:17:56.378 | INFO | Perf report for the last 5.13935 seconds: | 20260505 05:17:56.378 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:17:56.378 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:56.378 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:56.378 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:56.378 | INFO | ANALYSIS : 57.2%, 3.0, 88.8ms, 0 | 20260505 05:17:56.378 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:56.378 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:56.378 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 05:17:56.378 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:56.378 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:17:56.378 | INFO | Uploaded 36GiB to GPU in 1.60448s (22.4371GiB/s) | 20260505 05:17:56.378 | INFO | Downloaded 34GiB to GPU in 1.5219s (22.3404GiB/s) | 20260505 05:18:01.517 | INFO | Analyzing chunk frames = [636416,636928) | 20260505 05:18:01.517 | INFO | MemInfo: { | 20260505 05:18:01.517 | INFO | "gpu" : " 17 GB", | 20260505 05:18:01.517 | INFO | "huge" : "", | 20260505 05:18:01.517 | INFO | "huge_shared" : " 27 GB", | 20260505 05:18:01.517 | INFO | "malloc" : " 14 GB", | 20260505 05:18:01.517 | INFO | "pinned" : " 29 GB" | 20260505 05:18:01.517 | INFO | } | 20260505 05:18:01.517 | INFO | Perf report for the last 5.13924 seconds: | 20260505 05:18:01.517 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:18:01.517 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:01.517 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:01.517 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:01.517 | INFO | ANALYSIS : 56.9%, 3.0, 88.2ms, 0 | 20260505 05:18:01.517 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:01.517 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:01.517 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 05:18:01.517 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:01.517 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:01.518 | INFO | Uploaded 36GiB to GPU in 1.60364s (22.4489GiB/s) | 20260505 05:18:01.518 | INFO | Downloaded 34GiB to GPU in 1.52142s (22.3476GiB/s) | 20260505 05:18:06.656 | INFO | Analyzing chunk frames = [636928,637440) | 20260505 05:18:06.656 | INFO | MemInfo: { | 20260505 05:18:06.656 | INFO | "gpu" : " 17 GB", | 20260505 05:18:06.656 | INFO | "huge" : "", | 20260505 05:18:06.656 | INFO | "huge_shared" : " 27 GB", | 20260505 05:18:06.656 | INFO | "malloc" : " 14 GB", | 20260505 05:18:06.656 | INFO | "pinned" : " 29 GB" | 20260505 05:18:06.656 | INFO | } | 20260505 05:18:06.657 | INFO | Perf report for the last 5.13943 seconds: | 20260505 05:18:06.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:18:06.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:06.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:06.657 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:06.657 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260505 05:18:06.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:06.657 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:06.657 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260505 05:18:06.657 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:06.657 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:06.657 | INFO | Uploaded 36GiB to GPU in 1.60176s (22.4753GiB/s) | 20260505 05:18:06.657 | INFO | Downloaded 34GiB to GPU in 1.51839s (22.3922GiB/s) | 20260505 05:18:11.796 | INFO | Analyzing chunk frames = [637440,637952) | 20260505 05:18:11.796 | INFO | MemInfo: { | 20260505 05:18:11.796 | INFO | "gpu" : " 17 GB", | 20260505 05:18:11.796 | INFO | "huge" : "", | 20260505 05:18:11.796 | INFO | "huge_shared" : " 27 GB", | 20260505 05:18:11.796 | INFO | "malloc" : " 14 GB", | 20260505 05:18:11.796 | INFO | "pinned" : " 29 GB" | 20260505 05:18:11.796 | INFO | } | 20260505 05:18:11.796 | INFO | Perf report for the last 5.13934 seconds: | 20260505 05:18:11.796 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:18:11.796 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:11.796 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:11.796 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:11.796 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260505 05:18:11.796 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:11.796 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:11.796 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 05:18:11.796 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:11.796 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:11.796 | INFO | Uploaded 36GiB to GPU in 1.60066s (22.4907GiB/s) | 20260505 05:18:11.797 | INFO | Downloaded 34GiB to GPU in 1.51826s (22.3941GiB/s) | 20260505 05:18:16.935 | INFO | Analyzing chunk frames = [637952,638464) | 20260505 05:18:16.935 | INFO | MemInfo: { | 20260505 05:18:16.935 | INFO | "gpu" : " 17 GB", | 20260505 05:18:16.935 | INFO | "huge" : "", | 20260505 05:18:16.935 | INFO | "huge_shared" : " 27 GB", | 20260505 05:18:16.935 | INFO | "malloc" : " 14 GB", | 20260505 05:18:16.935 | INFO | "pinned" : " 29 GB" | 20260505 05:18:16.935 | INFO | } | 20260505 05:18:16.935 | INFO | Perf report for the last 5.13935 seconds: | 20260505 05:18:16.935 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:18:16.935 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:16.935 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:16.935 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:16.935 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 05:18:16.935 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:16.935 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:16.935 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 05:18:16.935 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:16.935 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:16.936 | INFO | Uploaded 36GiB to GPU in 1.60238s (22.4666GiB/s) | 20260505 05:18:16.936 | INFO | Downloaded 34GiB to GPU in 1.51975s (22.3721GiB/s) | 20260505 05:18:22.074 | INFO | Analyzing chunk frames = [638464,638976) | 20260505 05:18:22.074 | INFO | MemInfo: { | 20260505 05:18:22.074 | INFO | "gpu" : " 17 GB", | 20260505 05:18:22.074 | INFO | "huge" : "", | 20260505 05:18:22.074 | INFO | "huge_shared" : " 27 GB", | 20260505 05:18:22.074 | INFO | "malloc" : " 14 GB", | 20260505 05:18:22.074 | INFO | "pinned" : " 29 GB" | 20260505 05:18:22.074 | INFO | } | 20260505 05:18:22.074 | INFO | Perf report for the last 5.13926 seconds: | 20260505 05:18:22.075 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:18:22.075 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:22.075 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:22.075 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:22.075 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260505 05:18:22.075 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:22.075 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:22.075 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260505 05:18:22.075 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:22.075 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:22.075 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 05:18:22.075 | INFO | Total mem usage: 15.1 GiB | 20260505 05:18:22.075 | INFO | All Time High Water Marks sum: 15.7 GiB (Savings: 4.09%) | 20260505 05:18:22.075 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -4.15%) | 20260505 05:18:22.075 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 05:18:22.075 | INFO | High Water Breakdown: | 20260505 05:18:22.075 | INFO | 59.8% : 8.64 GiB ( 95.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 05:18:22.075 | INFO | Uploaded 36GiB to GPU in 1.60172s (22.4758GiB/s) | 20260505 05:18:22.075 | INFO | 40.1% : 5.8 GiB ( 87.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 05:18:22.075 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 05:18:22.075 | INFO | Downloaded 34GiB to GPU in 1.52036s (22.3631GiB/s) | 20260505 05:18:22.075 | INFO | ------------------------------------------ | 20260505 05:18:22.075 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:18:22.075 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 05:18:22.075 | INFO | Total mem usage: 26.5 GiB | 20260505 05:18:22.075 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 05:18:22.075 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 05:18:22.075 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 05:18:22.075 | INFO | High Water Breakdown: | 20260505 05:18:22.075 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:18:22.075 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:18:22.075 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:18:22.075 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:18:22.075 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:18:22.075 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:18:22.075 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 05:18:22.075 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 05:18:22.075 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:18:22.075 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 05:18:22.075 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 05:18:22.076 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 05:18:22.076 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 05:18:22.076 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 05:18:22.076 | INFO | ------------------------------------------ | 20260505 05:18:22.076 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:18:22.076 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 05:18:22.076 | INFO | Total mem usage: 25.6 GiB | 20260505 05:18:22.076 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 05:18:22.076 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 05:18:22.076 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 05:18:22.076 | INFO | High Water Breakdown: | 20260505 05:18:22.076 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 05:18:22.076 | INFO | ------------------------------------------ | 20260505 05:18:22.076 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:18:22.076 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 05:18:22.076 | INFO | Total mem usage: 15.7 GiB | 20260505 05:18:22.076 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 05:18:22.076 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 05:18:22.076 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260505 05:18:22.076 | INFO | High Water Breakdown: | 20260505 05:18:22.076 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 05:18:22.076 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 05:18:22.076 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 05:18:22.076 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 05:18:22.076 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 05:18:22.076 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 05:18:22.076 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 05:18:22.076 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 05:18:22.076 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 05:18:22.076 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:18:22.076 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 05:18:22.076 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:18:22.076 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 05:18:22.076 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 05:18:22.076 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 05:18:22.076 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 05:18:22.076 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:18:22.076 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 05:18:22.076 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:18:22.076 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:18:22.076 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:18:22.076 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 05:18:22.076 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 05:18:22.076 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:18:22.076 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 05:18:22.076 | INFO | ------------------------------------------ | 20260505 05:18:22.076 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:18:22.076 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 05:18:22.076 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:18:22.076 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:18:22.076 | INFO | PerfReport: Full Execution -- 3.03e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260505 05:18:22.076 | INFO | PerfReport: 0.28%, AggregateStats -- 8.64e-02 / -nan, 3.86e-03 / 0.00e+00 | 20260505 05:18:22.076 | INFO | PerfReport: 10.87%, Baseline -- 3.30e+00 / -nan, 1.73e-02 / 0.00e+00 | 20260505 05:18:22.076 | INFO | PerfReport: 3.05%, Histogram -- 9.24e-01 / -nan, 5.16e-02 / 0.00e+00 | 20260505 05:18:22.076 | INFO | PerfReport: 16.04%, DME -- 3.89e+01 / -nan, 3.23e+00 / 0.00e+00 | 20260505 05:18:22.076 | INFO | PerfReport: 56.71%, FrameLabeling -- 1.72e+01 / -nan, 4.55e-02 / 0.00e+00 | 20260505 05:18:22.076 | INFO | PerfReport: 7.56%, PulseAccumulating -- 2.29e+00 / -nan, 1.51e-01 / 0.00e+00 | 20260505 05:18:22.076 | INFO | PerfReport: 3.89%, Metrics -- 1.18e+00 / -nan, 1.97e-01 / 0.00e+00 | 20260505 05:18:22.076 | INFO | PerfReport: 1.60%, PulseSerializing -- 4.85e-01 / -nan, 7.85e-02 / 0.00e+00 | 20260505 05:18:22.076 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 05:18:22.076 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:18:22.076 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:18:22.076 | INFO | PerfReport: Full Execution -- 9.11e+01 / -nan, 1.90e+01 / 0.00e+00 | 20260505 05:18:22.076 | INFO | PerfReport: 18.39%, Upload -- 1.67e+01 / 0.00e+00, 3.96e-01 / 0.00e+00 | 20260505 05:18:22.076 | INFO | PerfReport: 17.65%, Download -- 1.61e+01 / 0.00e+00, 7.32e-01 / 0.00e+00 | 20260505 05:18:22.076 | INFO | PerfReport: 33.32%, Compute -- 3.04e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260505 05:18:22.076 | INFO | PerfReport: 0.55%, LockUpload -- 5.04e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260505 05:18:22.076 | INFO | PerfReport: 0.00%, LockDownload -- 3.80e-04 / 0.00e+00, 9.41e-05 / 0.00e+00 | 20260505 05:18:22.076 | INFO | PerfReport: 30.07%, LockCompute -- 2.74e+01 / 0.00e+00, 1.68e+01 / 0.00e+00 | 20260505 05:18:22.076 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.19e-02 / 0.00e+00 | 20260505 05:18:22.172 | INFO | Starting a round of Baz IO | 20260505 05:18:25.115 | INFO | Populating RTMetrics report | 20260505 05:18:25.115 | INFO | RTMetrics report populated | 20260505 05:18:25.115 | INFO | Opening tmp RTMetrics json file | 20260505 05:18:25.121 | INFO | Serializing RTMetrics to json | 20260505 05:18:25.124 | INFO | Serialization done, swapping tmp file now... | 20260505 05:18:25.126 | INFO | RTMetrics done | 20260505 05:18:25.221 | INFO | Finished a round of Baz IO: Wrote 2.206 GiB in 3.049 seconds | 20260505 05:18:25.221 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.88% / 0.09% / 60.03% | 20260505 05:18:25.222 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.88522% / 0.00021855% / 6.35783e-05% | 20260505 05:18:25.222 | INFO | Baz IO overall: | 20260505 05:18:25.222 | INFO | BytesWritten : 293439612658 | 20260505 05:18:25.222 | INFO | NumEvents : 137575986350 | 20260505 05:18:27.214 | INFO | Analyzing chunk frames = [638976,639488) | 20260505 05:18:27.214 | INFO | MemInfo: { | 20260505 05:18:27.214 | INFO | "gpu" : " 17 GB", | 20260505 05:18:27.214 | INFO | "huge" : "", | 20260505 05:18:27.214 | INFO | "huge_shared" : " 27 GB", | 20260505 05:18:27.214 | INFO | "malloc" : " 14 GB", | 20260505 05:18:27.214 | INFO | "pinned" : " 29 GB" | 20260505 05:18:27.214 | INFO | } | 20260505 05:18:27.214 | INFO | Perf report for the last 5.13925 seconds: | 20260505 05:18:27.214 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:18:27.214 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:27.214 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:27.214 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:27.214 | INFO | ANALYSIS : 58.0%, 2.9, 89.5ms, 0 | 20260505 05:18:27.214 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:27.214 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260505 05:18:27.214 | INFO | PRE_HQ : 57.5%, 2.2, 64.6ms, 0 | 20260505 05:18:27.214 | INFO | BAZWRITER : 58.9%, 4.3, 129.9ms, 0 | 20260505 05:18:27.214 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260505 05:18:27.214 | INFO | Uploaded 36GiB to GPU in 1.64391s (21.8989GiB/s) | 20260505 05:18:27.214 | INFO | Downloaded 38GiB to GPU in 1.70797s (22.2487GiB/s) | 20260505 05:18:32.353 | INFO | Analyzing chunk frames = [639488,640000) | 20260505 05:18:32.353 | INFO | MemInfo: { | 20260505 05:18:32.353 | INFO | "gpu" : " 17 GB", | 20260505 05:18:32.353 | INFO | "huge" : "", | 20260505 05:18:32.353 | INFO | "huge_shared" : " 27 GB", | 20260505 05:18:32.353 | INFO | "malloc" : " 14 GB", | 20260505 05:18:32.353 | INFO | "pinned" : " 29 GB" | 20260505 05:18:32.353 | INFO | } | 20260505 05:18:32.353 | INFO | Perf report for the last 5.13948 seconds: | 20260505 05:18:32.353 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:18:32.353 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:32.353 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:32.353 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:32.353 | INFO | ANALYSIS : 57.9%, 3.0, 90.2ms, 0 | 20260505 05:18:32.353 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:32.353 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:32.353 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 05:18:32.353 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:32.353 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:32.354 | INFO | Uploaded 36GiB to GPU in 1.60059s (22.4917GiB/s) | 20260505 05:18:32.354 | INFO | Downloaded 34GiB to GPU in 1.51691s (22.414GiB/s) | 20260505 05:18:37.492 | INFO | Analyzing chunk frames = [640000,640512) | 20260505 05:18:37.492 | INFO | MemInfo: { | 20260505 05:18:37.492 | INFO | "gpu" : " 17 GB", | 20260505 05:18:37.492 | INFO | "huge" : "", | 20260505 05:18:37.492 | INFO | "huge_shared" : " 27 GB", | 20260505 05:18:37.492 | INFO | "malloc" : " 14 GB", | 20260505 05:18:37.492 | INFO | "pinned" : " 29 GB" | 20260505 05:18:37.492 | INFO | } | 20260505 05:18:37.492 | INFO | Perf report for the last 5.13912 seconds: | 20260505 05:18:37.492 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:18:37.492 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:37.492 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:37.492 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:37.492 | INFO | ANALYSIS : 57.2%, 3.0, 88.8ms, 0 | 20260505 05:18:37.492 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:37.492 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:37.492 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 05:18:37.493 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:37.493 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:37.493 | INFO | Uploaded 36GiB to GPU in 1.60463s (22.4351GiB/s) | 20260505 05:18:37.493 | INFO | Downloaded 34GiB to GPU in 1.52195s (22.3398GiB/s) | 20260505 05:18:42.632 | INFO | Analyzing chunk frames = [640512,641024) | 20260505 05:18:42.632 | INFO | MemInfo: { | 20260505 05:18:42.632 | INFO | "gpu" : " 17 GB", | 20260505 05:18:42.632 | INFO | "huge" : "", | 20260505 05:18:42.632 | INFO | "huge_shared" : " 27 GB", | 20260505 05:18:42.632 | INFO | "malloc" : " 14 GB", | 20260505 05:18:42.632 | INFO | "pinned" : " 29 GB" | 20260505 05:18:42.632 | INFO | } | 20260505 05:18:42.632 | INFO | Perf report for the last 5.13948 seconds: | 20260505 05:18:42.632 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:18:42.632 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:42.632 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:42.632 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:42.632 | INFO | ANALYSIS : 56.9%, 3.0, 88.2ms, 0 | 20260505 05:18:42.632 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:42.632 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:42.632 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260505 05:18:42.632 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:42.632 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:42.633 | INFO | Uploaded 36GiB to GPU in 1.60313s (22.4561GiB/s) | 20260505 05:18:42.633 | INFO | Downloaded 34GiB to GPU in 1.52075s (22.3574GiB/s) | 20260505 05:18:47.771 | INFO | Analyzing chunk frames = [641024,641536) | 20260505 05:18:47.771 | INFO | MemInfo: { | 20260505 05:18:47.771 | INFO | "gpu" : " 17 GB", | 20260505 05:18:47.771 | INFO | "huge" : "", | 20260505 05:18:47.771 | INFO | "huge_shared" : " 27 GB", | 20260505 05:18:47.771 | INFO | "malloc" : " 14 GB", | 20260505 05:18:47.771 | INFO | "pinned" : " 29 GB" | 20260505 05:18:47.771 | INFO | } | 20260505 05:18:47.771 | INFO | Perf report for the last 5.13928 seconds: | 20260505 05:18:47.771 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:18:47.771 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:47.771 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:47.771 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:47.771 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260505 05:18:47.771 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:47.771 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:47.771 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260505 05:18:47.771 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:47.771 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:47.772 | INFO | Uploaded 36GiB to GPU in 1.6026s (22.4635GiB/s) | 20260505 05:18:47.772 | INFO | Downloaded 34GiB to GPU in 1.51916s (22.3809GiB/s) | 20260505 05:18:52.910 | INFO | Analyzing chunk frames = [641536,642048) | 20260505 05:18:52.910 | INFO | MemInfo: { | 20260505 05:18:52.910 | INFO | "gpu" : " 17 GB", | 20260505 05:18:52.910 | INFO | "huge" : "", | 20260505 05:18:52.910 | INFO | "huge_shared" : " 27 GB", | 20260505 05:18:52.910 | INFO | "malloc" : " 14 GB", | 20260505 05:18:52.910 | INFO | "pinned" : " 29 GB" | 20260505 05:18:52.910 | INFO | } | 20260505 05:18:52.910 | INFO | Perf report for the last 5.13927 seconds: | 20260505 05:18:52.910 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:18:52.910 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:52.910 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:52.910 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:52.910 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260505 05:18:52.911 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:52.911 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:52.911 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 05:18:52.911 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:52.911 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:52.911 | INFO | Uploaded 36GiB to GPU in 1.60182s (22.4744GiB/s) | 20260505 05:18:52.911 | INFO | Downloaded 34GiB to GPU in 1.51906s (22.3823GiB/s) | 20260505 05:18:58.050 | INFO | Analyzing chunk frames = [642048,642560) | 20260505 05:18:58.050 | INFO | MemInfo: { | 20260505 05:18:58.050 | INFO | "gpu" : " 17 GB", | 20260505 05:18:58.050 | INFO | "huge" : "", | 20260505 05:18:58.050 | INFO | "huge_shared" : " 27 GB", | 20260505 05:18:58.050 | INFO | "malloc" : " 14 GB", | 20260505 05:18:58.050 | INFO | "pinned" : " 29 GB" | 20260505 05:18:58.050 | INFO | } | 20260505 05:18:58.050 | INFO | Perf report for the last 5.13938 seconds: | 20260505 05:18:58.050 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:18:58.050 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:58.050 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:58.050 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:58.050 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 05:18:58.050 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:58.050 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:58.050 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260505 05:18:58.050 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:58.050 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:18:58.050 | INFO | Uploaded 36GiB to GPU in 1.60217s (22.4695GiB/s) | 20260505 05:18:58.050 | INFO | Downloaded 34GiB to GPU in 1.5201s (22.367GiB/s) | 20260505 05:19:03.189 | INFO | Analyzing chunk frames = [642560,643072) | 20260505 05:19:03.189 | INFO | MemInfo: { | 20260505 05:19:03.189 | INFO | "gpu" : " 17 GB", | 20260505 05:19:03.189 | INFO | "huge" : "", | 20260505 05:19:03.189 | INFO | "huge_shared" : " 27 GB", | 20260505 05:19:03.189 | INFO | "malloc" : " 14 GB", | 20260505 05:19:03.189 | INFO | "pinned" : " 29 GB" | 20260505 05:19:03.189 | INFO | } | 20260505 05:19:03.189 | INFO | Perf report for the last 5.13938 seconds: | 20260505 05:19:03.189 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:19:03.189 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:03.189 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:03.189 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:03.189 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260505 05:19:03.189 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:03.189 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:03.189 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260505 05:19:03.189 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:03.189 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:03.190 | INFO | Uploaded 36GiB to GPU in 1.60144s (22.4798GiB/s) | 20260505 05:19:03.190 | INFO | Downloaded 34GiB to GPU in 1.52099s (22.3539GiB/s) | 20260505 05:19:03.286 | INFO | Starting a round of Baz IO | 20260505 05:19:06.229 | INFO | Populating RTMetrics report | 20260505 05:19:06.230 | INFO | RTMetrics report populated | 20260505 05:19:06.230 | INFO | Opening tmp RTMetrics json file | 20260505 05:19:06.233 | INFO | Serializing RTMetrics to json | 20260505 05:19:06.234 | INFO | Serialization done, swapping tmp file now... | 20260505 05:19:06.235 | INFO | RTMetrics done | 20260505 05:19:06.339 | INFO | Finished a round of Baz IO: Wrote 2.209 GiB in 3.052 seconds | 20260505 05:19:06.339 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.80% / 0.09% / 60.12% | 20260505 05:19:06.339 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.8706% / 0.000111262% / 4.76837e-05% | 20260505 05:19:06.339 | INFO | Baz IO overall: | 20260505 05:19:06.339 | INFO | BytesWritten : 295811889015 | 20260505 05:19:06.339 | INFO | NumEvents : 138705634261 | 20260505 05:19:08.328 | INFO | Analyzing chunk frames = [643072,643584) | 20260505 05:19:08.328 | INFO | MemInfo: { | 20260505 05:19:08.328 | INFO | "gpu" : " 17 GB", | 20260505 05:19:08.328 | INFO | "huge" : "", | 20260505 05:19:08.328 | INFO | "huge_shared" : " 27 GB", | 20260505 05:19:08.329 | INFO | "malloc" : " 14 GB", | 20260505 05:19:08.329 | INFO | "pinned" : " 29 GB" | 20260505 05:19:08.329 | INFO | } | 20260505 05:19:08.329 | INFO | Perf report for the last 5.13936 seconds: | 20260505 05:19:08.329 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:19:08.329 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:08.329 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:08.329 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:08.329 | INFO | ANALYSIS : 58.0%, 2.9, 89.6ms, 0 | 20260505 05:19:08.329 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:08.329 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260505 05:19:08.329 | INFO | PRE_HQ : 57.4%, 2.2, 64.7ms, 0 | 20260505 05:19:08.329 | INFO | BAZWRITER : 59.0%, 4.3, 130.2ms, 0 | 20260505 05:19:08.329 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 05:19:08.329 | INFO | Uploaded 36GiB to GPU in 1.64972s (21.8219GiB/s) | 20260505 05:19:08.329 | INFO | Downloaded 38GiB to GPU in 1.71641s (22.1392GiB/s) | 20260505 05:19:13.468 | INFO | Analyzing chunk frames = [643584,644096) | 20260505 05:19:13.468 | INFO | MemInfo: { | 20260505 05:19:13.468 | INFO | "gpu" : " 17 GB", | 20260505 05:19:13.468 | INFO | "huge" : "", | 20260505 05:19:13.468 | INFO | "huge_shared" : " 27 GB", | 20260505 05:19:13.468 | INFO | "malloc" : " 14 GB", | 20260505 05:19:13.468 | INFO | "pinned" : " 29 GB" | 20260505 05:19:13.468 | INFO | } | 20260505 05:19:13.468 | INFO | Perf report for the last 5.1392 seconds: | 20260505 05:19:13.468 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:19:13.468 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:13.468 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:13.468 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:13.468 | INFO | ANALYSIS : 57.9%, 3.0, 90.2ms, 0 | 20260505 05:19:13.468 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:13.468 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:13.468 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:19:13.468 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:13.468 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:13.468 | INFO | Uploaded 36GiB to GPU in 1.60038s (22.4946GiB/s) | 20260505 05:19:13.468 | INFO | Downloaded 34GiB to GPU in 1.51688s (22.4145GiB/s) | 20260505 05:19:18.607 | INFO | Analyzing chunk frames = [644096,644608) | 20260505 05:19:18.607 | INFO | MemInfo: { | 20260505 05:19:18.607 | INFO | "gpu" : " 17 GB", | 20260505 05:19:18.607 | INFO | "huge" : "", | 20260505 05:19:18.607 | INFO | "huge_shared" : " 27 GB", | 20260505 05:19:18.607 | INFO | "malloc" : " 14 GB", | 20260505 05:19:18.607 | INFO | "pinned" : " 29 GB" | 20260505 05:19:18.607 | INFO | } | 20260505 05:19:18.607 | INFO | Perf report for the last 5.13947 seconds: | 20260505 05:19:18.607 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:19:18.607 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:18.607 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:18.607 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:18.607 | INFO | ANALYSIS : 57.2%, 3.0, 88.8ms, 0 | 20260505 05:19:18.607 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:18.607 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:18.607 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:19:18.607 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:18.607 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:18.608 | INFO | Uploaded 36GiB to GPU in 1.60468s (22.4344GiB/s) | 20260505 05:19:18.608 | INFO | Downloaded 34GiB to GPU in 1.52287s (22.3263GiB/s) | 20260505 05:19:23.746 | INFO | Analyzing chunk frames = [644608,645120) | 20260505 05:19:23.746 | INFO | MemInfo: { | 20260505 05:19:23.746 | INFO | "gpu" : " 17 GB", | 20260505 05:19:23.746 | INFO | "huge" : "", | 20260505 05:19:23.746 | INFO | "huge_shared" : " 27 GB", | 20260505 05:19:23.747 | INFO | "malloc" : " 14 GB", | 20260505 05:19:23.747 | INFO | "pinned" : " 29 GB" | 20260505 05:19:23.747 | INFO | } | 20260505 05:19:23.747 | INFO | Perf report for the last 5.13927 seconds: | 20260505 05:19:23.747 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:19:23.747 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:23.747 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:23.747 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:23.747 | INFO | ANALYSIS : 56.9%, 3.0, 88.3ms, 0 | 20260505 05:19:23.747 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:23.747 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:23.747 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 05:19:23.747 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:23.747 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:23.747 | INFO | Uploaded 36GiB to GPU in 1.60346s (22.4515GiB/s) | 20260505 05:19:23.747 | INFO | Downloaded 34GiB to GPU in 1.5205s (22.3611GiB/s) | 20260505 05:19:28.886 | INFO | Analyzing chunk frames = [645120,645632) | 20260505 05:19:28.886 | INFO | MemInfo: { | 20260505 05:19:28.886 | INFO | "gpu" : " 17 GB", | 20260505 05:19:28.886 | INFO | "huge" : "", | 20260505 05:19:28.886 | INFO | "huge_shared" : " 27 GB", | 20260505 05:19:28.886 | INFO | "malloc" : " 14 GB", | 20260505 05:19:28.886 | INFO | "pinned" : " 29 GB" | 20260505 05:19:28.886 | INFO | } | 20260505 05:19:28.886 | INFO | Perf report for the last 5.1395 seconds: | 20260505 05:19:28.886 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:19:28.886 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:28.886 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:28.886 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:28.886 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260505 05:19:28.886 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:28.886 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:28.886 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 05:19:28.886 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:28.886 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:28.886 | INFO | Uploaded 36GiB to GPU in 1.60183s (22.4743GiB/s) | 20260505 05:19:28.886 | INFO | Downloaded 34GiB to GPU in 1.51853s (22.3901GiB/s) | 20260505 05:19:34.025 | INFO | Analyzing chunk frames = [645632,646144) | 20260505 05:19:34.025 | INFO | MemInfo: { | 20260505 05:19:34.025 | INFO | "gpu" : " 17 GB", | 20260505 05:19:34.025 | INFO | "huge" : "", | 20260505 05:19:34.025 | INFO | "huge_shared" : " 27 GB", | 20260505 05:19:34.025 | INFO | "malloc" : " 14 GB", | 20260505 05:19:34.025 | INFO | "pinned" : " 29 GB" | 20260505 05:19:34.025 | INFO | } | 20260505 05:19:34.025 | INFO | Perf report for the last 5.13912 seconds: | 20260505 05:19:34.025 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:19:34.025 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:34.025 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:34.025 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:34.025 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260505 05:19:34.025 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:34.025 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:34.025 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:19:34.025 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:34.025 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:34.026 | INFO | Uploaded 36GiB to GPU in 1.60179s (22.4748GiB/s) | 20260505 05:19:34.026 | INFO | Downloaded 34GiB to GPU in 1.51867s (22.3881GiB/s) | 20260505 05:19:39.164 | INFO | Analyzing chunk frames = [646144,646656) | 20260505 05:19:39.164 | INFO | MemInfo: { | 20260505 05:19:39.164 | INFO | "gpu" : " 17 GB", | 20260505 05:19:39.164 | INFO | "huge" : "", | 20260505 05:19:39.164 | INFO | "huge_shared" : " 27 GB", | 20260505 05:19:39.165 | INFO | "malloc" : " 14 GB", | 20260505 05:19:39.165 | INFO | "pinned" : " 29 GB" | 20260505 05:19:39.165 | INFO | } | 20260505 05:19:39.165 | INFO | Perf report for the last 5.13946 seconds: | 20260505 05:19:39.165 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:19:39.165 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:39.165 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:39.165 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:39.165 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260505 05:19:39.165 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:39.165 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:39.165 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 05:19:39.165 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:39.165 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:39.165 | INFO | Uploaded 36GiB to GPU in 1.60241s (22.4661GiB/s) | 20260505 05:19:39.165 | INFO | Downloaded 34GiB to GPU in 1.51982s (22.3711GiB/s) | 20260505 05:19:44.304 | INFO | Analyzing chunk frames = [646656,647168) | 20260505 05:19:44.304 | INFO | MemInfo: { | 20260505 05:19:44.304 | INFO | "gpu" : " 17 GB", | 20260505 05:19:44.304 | INFO | "huge" : "", | 20260505 05:19:44.304 | INFO | "huge_shared" : " 27 GB", | 20260505 05:19:44.304 | INFO | "malloc" : " 14 GB", | 20260505 05:19:44.304 | INFO | "pinned" : " 29 GB" | 20260505 05:19:44.304 | INFO | } | 20260505 05:19:44.304 | INFO | Perf report for the last 5.1397 seconds: | 20260505 05:19:44.304 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:19:44.304 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:44.304 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:44.304 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:44.305 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260505 05:19:44.305 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:44.305 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:44.305 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 05:19:44.305 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:44.305 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:44.305 | INFO | Uploaded 36GiB to GPU in 1.60303s (22.4575GiB/s) | 20260505 05:19:44.305 | INFO | Downloaded 34GiB to GPU in 1.52071s (22.358GiB/s) | 20260505 05:19:44.305 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 05:19:44.305 | INFO | Total mem usage: 15.1 GiB | 20260505 05:19:44.305 | INFO | All Time High Water Marks sum: 15.7 GiB (Savings: 4.09%) | 20260505 05:19:44.305 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -3.97%) | 20260505 05:19:44.305 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 05:19:44.305 | INFO | High Water Breakdown: | 20260505 05:19:44.305 | INFO | 59.8% : 8.67 GiB ( 95.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 05:19:44.305 | INFO | 40% : 5.8 GiB ( 87.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 05:19:44.305 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 05:19:44.305 | INFO | ------------------------------------------ | 20260505 05:19:44.305 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:19:44.305 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 05:19:44.305 | INFO | Total mem usage: 26.5 GiB | 20260505 05:19:44.305 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 05:19:44.305 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 05:19:44.305 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 05:19:44.305 | INFO | High Water Breakdown: | 20260505 05:19:44.305 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:19:44.305 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:19:44.305 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:19:44.305 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:19:44.305 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:19:44.305 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:19:44.305 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 05:19:44.305 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 05:19:44.305 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:19:44.305 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 05:19:44.305 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 05:19:44.305 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 05:19:44.305 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 05:19:44.305 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 05:19:44.305 | INFO | ------------------------------------------ | 20260505 05:19:44.305 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:19:44.305 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 05:19:44.305 | INFO | Total mem usage: 25.6 GiB | 20260505 05:19:44.305 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 05:19:44.305 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 05:19:44.305 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 05:19:44.305 | INFO | High Water Breakdown: | 20260505 05:19:44.305 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 05:19:44.305 | INFO | ------------------------------------------ | 20260505 05:19:44.305 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:19:44.306 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 05:19:44.306 | INFO | Total mem usage: 15.7 GiB | 20260505 05:19:44.306 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 05:19:44.306 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 05:19:44.306 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260505 05:19:44.306 | INFO | High Water Breakdown: | 20260505 05:19:44.306 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 05:19:44.306 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 05:19:44.306 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 05:19:44.306 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 05:19:44.306 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 05:19:44.306 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 05:19:44.306 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 05:19:44.306 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 05:19:44.306 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 05:19:44.306 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:19:44.306 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 05:19:44.306 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:19:44.306 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 05:19:44.306 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 05:19:44.306 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 05:19:44.306 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 05:19:44.306 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:19:44.306 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 05:19:44.306 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:19:44.306 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:19:44.306 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:19:44.306 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 05:19:44.306 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 05:19:44.306 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:19:44.306 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 05:19:44.306 | INFO | ------------------------------------------ | 20260505 05:19:44.306 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:19:44.306 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 05:19:44.306 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:19:44.306 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:19:44.306 | INFO | PerfReport: Full Execution -- 3.03e+01 / -nan, 1.24e+01 / 0.00e+00 | 20260505 05:19:44.306 | INFO | PerfReport: 0.28%, AggregateStats -- 8.62e-02 / -nan, 4.30e-03 / 0.00e+00 | 20260505 05:19:44.306 | INFO | PerfReport: 10.86%, Baseline -- 3.30e+00 / -nan, 1.77e-02 / 0.00e+00 | 20260505 05:19:44.306 | INFO | PerfReport: 3.05%, Histogram -- 9.24e-01 / -nan, 5.08e-02 / 0.00e+00 | 20260505 05:19:44.306 | INFO | PerfReport: 16.04%, DME -- 3.89e+01 / -nan, 3.19e+00 / 0.00e+00 | 20260505 05:19:44.306 | INFO | PerfReport: 56.72%, FrameLabeling -- 1.72e+01 / -nan, 4.57e-02 / 0.00e+00 | 20260505 05:19:44.306 | INFO | PerfReport: 7.56%, PulseAccumulating -- 2.29e+00 / -nan, 1.51e-01 / 0.00e+00 | 20260505 05:19:44.306 | INFO | PerfReport: 3.89%, Metrics -- 1.18e+00 / -nan, 1.95e-01 / 0.00e+00 | 20260505 05:19:44.306 | INFO | PerfReport: 1.60%, PulseSerializing -- 4.85e-01 / -nan, 7.85e-02 / 0.00e+00 | 20260505 05:19:44.306 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 05:19:44.306 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:19:44.306 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:19:44.306 | INFO | PerfReport: Full Execution -- 9.11e+01 / -nan, 1.89e+01 / 0.00e+00 | 20260505 05:19:44.306 | INFO | PerfReport: 18.39%, Upload -- 1.68e+01 / 0.00e+00, 4.00e-01 / 0.00e+00 | 20260505 05:19:44.306 | INFO | PerfReport: 17.66%, Download -- 1.61e+01 / 0.00e+00, 7.40e-01 / 0.00e+00 | 20260505 05:19:44.306 | INFO | PerfReport: 33.33%, Compute -- 3.04e+01 / 0.00e+00, 1.24e+01 / 0.00e+00 | 20260505 05:19:44.306 | INFO | PerfReport: 0.55%, LockUpload -- 5.05e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260505 05:19:44.306 | INFO | PerfReport: 0.00%, LockDownload -- 3.96e-04 / 0.00e+00, 1.11e-04 / 0.00e+00 | 20260505 05:19:44.306 | INFO | PerfReport: 30.04%, LockCompute -- 2.74e+01 / 0.00e+00, 1.68e+01 / 0.00e+00 | 20260505 05:19:44.306 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.22e-02 / 0.00e+00 | 20260505 05:19:44.401 | INFO | Starting a round of Baz IO | 20260505 05:19:47.346 | INFO | Populating RTMetrics report | 20260505 05:19:47.346 | INFO | RTMetrics report populated | 20260505 05:19:47.346 | INFO | Opening tmp RTMetrics json file | 20260505 05:19:47.352 | INFO | Serializing RTMetrics to json | 20260505 05:19:47.354 | INFO | Serialization done, swapping tmp file now... | 20260505 05:19:47.354 | INFO | RTMetrics done | 20260505 05:19:47.448 | INFO | Finished a round of Baz IO: Wrote 2.211 GiB in 3.047 seconds | 20260505 05:19:47.448 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.71% / 0.08% / 60.20% | 20260505 05:19:47.448 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.83641% / 0.000150998% / 5.96046e-05% | 20260505 05:19:47.448 | INFO | Baz IO overall: | 20260505 05:19:47.448 | INFO | BytesWritten : 298185878750 | 20260505 05:19:47.448 | INFO | NumEvents : 139835958703 | 20260505 05:19:49.443 | INFO | Analyzing chunk frames = [647168,647680) | 20260505 05:19:49.443 | INFO | MemInfo: { | 20260505 05:19:49.443 | INFO | "gpu" : " 17 GB", | 20260505 05:19:49.443 | INFO | "huge" : "", | 20260505 05:19:49.443 | INFO | "huge_shared" : " 27 GB", | 20260505 05:19:49.443 | INFO | "malloc" : " 14 GB", | 20260505 05:19:49.443 | INFO | "pinned" : " 29 GB" | 20260505 05:19:49.443 | INFO | } | 20260505 05:19:49.443 | INFO | Perf report for the last 5.1388 seconds: | 20260505 05:19:49.443 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:19:49.443 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:49.443 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:49.443 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:49.443 | INFO | ANALYSIS : 58.0%, 2.9, 89.5ms, 0 | 20260505 05:19:49.443 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:49.443 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260505 05:19:49.443 | INFO | PRE_HQ : 57.3%, 2.2, 64.4ms, 0 | 20260505 05:19:49.443 | INFO | BAZWRITER : 58.6%, 4.3, 129.6ms, 0 | 20260505 05:19:49.444 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 05:19:49.444 | INFO | Uploaded 36GiB to GPU in 1.6495s (21.8248GiB/s) | 20260505 05:19:49.444 | INFO | Downloaded 38GiB to GPU in 1.71442s (22.1649GiB/s) | 20260505 05:19:54.582 | INFO | Analyzing chunk frames = [647680,648192) | 20260505 05:19:54.582 | INFO | MemInfo: { | 20260505 05:19:54.582 | INFO | "gpu" : " 17 GB", | 20260505 05:19:54.582 | INFO | "huge" : "", | 20260505 05:19:54.582 | INFO | "huge_shared" : " 27 GB", | 20260505 05:19:54.582 | INFO | "malloc" : " 14 GB", | 20260505 05:19:54.582 | INFO | "pinned" : " 29 GB" | 20260505 05:19:54.583 | INFO | } | 20260505 05:19:54.583 | INFO | Perf report for the last 5.13949 seconds: | 20260505 05:19:54.583 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:19:54.583 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:54.583 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:54.583 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:54.583 | INFO | ANALYSIS : 57.8%, 3.0, 90.2ms, 0 | 20260505 05:19:54.583 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:54.583 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:54.583 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:19:54.583 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:54.583 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:54.583 | INFO | Uploaded 36GiB to GPU in 1.60062s (22.4913GiB/s) | 20260505 05:19:54.583 | INFO | Downloaded 34GiB to GPU in 1.51774s (22.4017GiB/s) | 20260505 05:19:59.722 | INFO | Analyzing chunk frames = [648192,648704) | 20260505 05:19:59.722 | INFO | MemInfo: { | 20260505 05:19:59.722 | INFO | "gpu" : " 17 GB", | 20260505 05:19:59.722 | INFO | "huge" : "", | 20260505 05:19:59.722 | INFO | "huge_shared" : " 27 GB", | 20260505 05:19:59.722 | INFO | "malloc" : " 14 GB", | 20260505 05:19:59.722 | INFO | "pinned" : " 29 GB" | 20260505 05:19:59.722 | INFO | } | 20260505 05:19:59.722 | INFO | Perf report for the last 5.13921 seconds: | 20260505 05:19:59.722 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:19:59.722 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:59.722 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:59.722 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:59.722 | INFO | ANALYSIS : 57.2%, 3.0, 88.9ms, 0 | 20260505 05:19:59.722 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:59.722 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:59.722 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:19:59.722 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:59.722 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:19:59.722 | INFO | Uploaded 36GiB to GPU in 1.60429s (22.4399GiB/s) | 20260505 05:19:59.722 | INFO | Downloaded 34GiB to GPU in 1.5218s (22.3419GiB/s) | 20260505 05:20:04.861 | INFO | Analyzing chunk frames = [648704,649216) | 20260505 05:20:04.861 | INFO | MemInfo: { | 20260505 05:20:04.861 | INFO | "gpu" : " 17 GB", | 20260505 05:20:04.861 | INFO | "huge" : "", | 20260505 05:20:04.861 | INFO | "huge_shared" : " 27 GB", | 20260505 05:20:04.861 | INFO | "malloc" : " 14 GB", | 20260505 05:20:04.861 | INFO | "pinned" : " 29 GB" | 20260505 05:20:04.861 | INFO | } | 20260505 05:20:04.861 | INFO | Perf report for the last 5.13948 seconds: | 20260505 05:20:04.861 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:20:04.861 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:04.861 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:04.861 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:04.861 | INFO | ANALYSIS : 56.8%, 3.0, 88.2ms, 0 | 20260505 05:20:04.862 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:04.862 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:04.862 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 05:20:04.862 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:04.862 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:04.862 | INFO | Uploaded 36GiB to GPU in 1.60377s (22.4471GiB/s) | 20260505 05:20:04.862 | INFO | Downloaded 34GiB to GPU in 1.52108s (22.3526GiB/s) | 20260505 05:20:10.001 | INFO | Analyzing chunk frames = [649216,649728) | 20260505 05:20:10.001 | INFO | MemInfo: { | 20260505 05:20:10.001 | INFO | "gpu" : " 17 GB", | 20260505 05:20:10.001 | INFO | "huge" : "", | 20260505 05:20:10.001 | INFO | "huge_shared" : " 27 GB", | 20260505 05:20:10.001 | INFO | "malloc" : " 14 GB", | 20260505 05:20:10.001 | INFO | "pinned" : " 29 GB" | 20260505 05:20:10.001 | INFO | } | 20260505 05:20:10.001 | INFO | Perf report for the last 5.13929 seconds: | 20260505 05:20:10.001 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:20:10.001 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:10.001 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:10.001 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:10.001 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260505 05:20:10.001 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:10.001 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:10.001 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 05:20:10.001 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:10.001 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:10.001 | INFO | Uploaded 36GiB to GPU in 1.60213s (22.4701GiB/s) | 20260505 05:20:10.001 | INFO | Downloaded 34GiB to GPU in 1.51883s (22.3857GiB/s) | 20260505 05:20:15.140 | INFO | Analyzing chunk frames = [649728,650240) | 20260505 05:20:15.140 | INFO | MemInfo: { | 20260505 05:20:15.140 | INFO | "gpu" : " 17 GB", | 20260505 05:20:15.140 | INFO | "huge" : "", | 20260505 05:20:15.140 | INFO | "huge_shared" : " 27 GB", | 20260505 05:20:15.140 | INFO | "malloc" : " 14 GB", | 20260505 05:20:15.140 | INFO | "pinned" : " 29 GB" | 20260505 05:20:15.140 | INFO | } | 20260505 05:20:15.140 | INFO | Perf report for the last 5.13942 seconds: | 20260505 05:20:15.140 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:20:15.140 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:15.140 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:15.140 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:15.140 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260505 05:20:15.140 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:15.140 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:15.140 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:20:15.140 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:15.140 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:15.140 | INFO | Uploaded 36GiB to GPU in 1.6024s (22.4663GiB/s) | 20260505 05:20:15.141 | INFO | Downloaded 34GiB to GPU in 1.51864s (22.3884GiB/s) | 20260505 05:20:20.279 | INFO | Analyzing chunk frames = [650240,650752) | 20260505 05:20:20.279 | INFO | MemInfo: { | 20260505 05:20:20.279 | INFO | "gpu" : " 17 GB", | 20260505 05:20:20.279 | INFO | "huge" : "", | 20260505 05:20:20.279 | INFO | "huge_shared" : " 27 GB", | 20260505 05:20:20.279 | INFO | "malloc" : " 14 GB", | 20260505 05:20:20.279 | INFO | "pinned" : " 29 GB" | 20260505 05:20:20.279 | INFO | } | 20260505 05:20:20.280 | INFO | Perf report for the last 5.1393 seconds: | 20260505 05:20:20.280 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:20:20.280 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:20.280 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:20.280 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:20.280 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 05:20:20.280 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:20.280 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:20.280 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 05:20:20.280 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:20.280 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:20.280 | INFO | Uploaded 36GiB to GPU in 1.60232s (22.4674GiB/s) | 20260505 05:20:20.280 | INFO | Downloaded 34GiB to GPU in 1.51948s (22.376GiB/s) | 20260505 05:20:25.419 | INFO | Analyzing chunk frames = [650752,651264) | 20260505 05:20:25.419 | INFO | MemInfo: { | 20260505 05:20:25.419 | INFO | "gpu" : " 17 GB", | 20260505 05:20:25.419 | INFO | "huge" : "", | 20260505 05:20:25.419 | INFO | "huge_shared" : " 27 GB", | 20260505 05:20:25.419 | INFO | "malloc" : " 14 GB", | 20260505 05:20:25.419 | INFO | "pinned" : " 29 GB" | 20260505 05:20:25.419 | INFO | } | 20260505 05:20:25.419 | INFO | Perf report for the last 5.13938 seconds: | 20260505 05:20:25.419 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:20:25.419 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:25.419 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:25.419 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:25.419 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260505 05:20:25.419 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:25.419 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:25.419 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 05:20:25.419 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:25.419 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:25.419 | INFO | Uploaded 36GiB to GPU in 1.60293s (22.4589GiB/s) | 20260505 05:20:25.419 | INFO | Downloaded 34GiB to GPU in 1.52057s (22.36GiB/s) | 20260505 05:20:25.516 | INFO | Starting a round of Baz IO | 20260505 05:20:28.462 | INFO | Populating RTMetrics report | 20260505 05:20:28.462 | INFO | RTMetrics report populated | 20260505 05:20:28.462 | INFO | Opening tmp RTMetrics json file | 20260505 05:20:28.468 | INFO | Serializing RTMetrics to json | 20260505 05:20:28.471 | INFO | Serialization done, swapping tmp file now... | 20260505 05:20:28.472 | INFO | RTMetrics done | 20260505 05:20:28.565 | INFO | Finished a round of Baz IO: Wrote 2.215 GiB in 3.048 seconds | 20260505 05:20:28.565 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.63% / 0.09% / 60.29% | 20260505 05:20:28.565 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.7975% / 0.000150998% / 4.37101e-05% | 20260505 05:20:28.565 | INFO | Baz IO overall: | 20260505 05:20:28.565 | INFO | BytesWritten : 300564319708 | 20260505 05:20:28.566 | INFO | NumEvents : 140967988599 | 20260505 05:20:30.558 | INFO | Analyzing chunk frames = [651264,651776) | 20260505 05:20:30.558 | INFO | MemInfo: { | 20260505 05:20:30.558 | INFO | "gpu" : " 17 GB", | 20260505 05:20:30.558 | INFO | "huge" : "", | 20260505 05:20:30.558 | INFO | "huge_shared" : " 27 GB", | 20260505 05:20:30.558 | INFO | "malloc" : " 14 GB", | 20260505 05:20:30.558 | INFO | "pinned" : " 29 GB" | 20260505 05:20:30.558 | INFO | } | 20260505 05:20:30.558 | INFO | Perf report for the last 5.13917 seconds: | 20260505 05:20:30.558 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:20:30.558 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:30.558 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:30.558 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:30.558 | INFO | ANALYSIS : 58.1%, 2.9, 89.7ms, 0 | 20260505 05:20:30.558 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:30.558 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260505 05:20:30.558 | INFO | PRE_HQ : 57.4%, 2.2, 64.3ms, 0 | 20260505 05:20:30.558 | INFO | BAZWRITER : 58.9%, 4.3, 130.4ms, 0 | 20260505 05:20:30.558 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 05:20:30.558 | INFO | Uploaded 36GiB to GPU in 1.64304s (21.9107GiB/s) | 20260505 05:20:30.558 | INFO | Downloaded 38GiB to GPU in 1.70746s (22.2552GiB/s) | 20260505 05:20:35.697 | INFO | Analyzing chunk frames = [651776,652288) | 20260505 05:20:35.697 | INFO | MemInfo: { | 20260505 05:20:35.697 | INFO | "gpu" : " 17 GB", | 20260505 05:20:35.697 | INFO | "huge" : "", | 20260505 05:20:35.697 | INFO | "huge_shared" : " 27 GB", | 20260505 05:20:35.697 | INFO | "malloc" : " 14 GB", | 20260505 05:20:35.697 | INFO | "pinned" : " 29 GB" | 20260505 05:20:35.697 | INFO | } | 20260505 05:20:35.697 | INFO | Perf report for the last 5.13931 seconds: | 20260505 05:20:35.697 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:20:35.697 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:35.697 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:35.697 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:35.697 | INFO | ANALYSIS : 57.8%, 3.0, 90.1ms, 0 | 20260505 05:20:35.697 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:35.698 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:35.698 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260505 05:20:35.698 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:35.698 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:35.698 | INFO | Uploaded 36GiB to GPU in 1.60055s (22.4922GiB/s) | 20260505 05:20:35.698 | INFO | Downloaded 34GiB to GPU in 1.51707s (22.4116GiB/s) | 20260505 05:20:40.837 | INFO | Analyzing chunk frames = [652288,652800) | 20260505 05:20:40.837 | INFO | MemInfo: { | 20260505 05:20:40.837 | INFO | "gpu" : " 17 GB", | 20260505 05:20:40.837 | INFO | "huge" : "", | 20260505 05:20:40.837 | INFO | "huge_shared" : " 27 GB", | 20260505 05:20:40.837 | INFO | "malloc" : " 14 GB", | 20260505 05:20:40.837 | INFO | "pinned" : " 29 GB" | 20260505 05:20:40.837 | INFO | } | 20260505 05:20:40.837 | INFO | Perf report for the last 5.1393 seconds: | 20260505 05:20:40.837 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:20:40.837 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:40.837 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:40.837 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:40.837 | INFO | ANALYSIS : 57.2%, 3.0, 88.9ms, 0 | 20260505 05:20:40.837 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:40.837 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:40.837 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 05:20:40.837 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:40.837 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:40.837 | INFO | Uploaded 36GiB to GPU in 1.60437s (22.4387GiB/s) | 20260505 05:20:40.837 | INFO | Downloaded 34GiB to GPU in 1.52213s (22.3371GiB/s) | 20260505 05:20:45.976 | INFO | Analyzing chunk frames = [652800,653312) | 20260505 05:20:45.976 | INFO | MemInfo: { | 20260505 05:20:45.976 | INFO | "gpu" : " 17 GB", | 20260505 05:20:45.976 | INFO | "huge" : "", | 20260505 05:20:45.976 | INFO | "huge_shared" : " 27 GB", | 20260505 05:20:45.976 | INFO | "malloc" : " 14 GB", | 20260505 05:20:45.976 | INFO | "pinned" : " 29 GB" | 20260505 05:20:45.976 | INFO | } | 20260505 05:20:45.976 | INFO | Perf report for the last 5.13945 seconds: | 20260505 05:20:45.976 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:20:45.976 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:45.976 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:45.976 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:45.976 | INFO | ANALYSIS : 56.9%, 3.0, 88.2ms, 0 | 20260505 05:20:45.976 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:45.976 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:45.976 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:20:45.976 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:45.976 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:45.976 | INFO | Uploaded 36GiB to GPU in 1.60385s (22.446GiB/s) | 20260505 05:20:45.977 | INFO | Downloaded 34GiB to GPU in 1.52084s (22.3561GiB/s) | 20260505 05:20:51.115 | INFO | Analyzing chunk frames = [653312,653824) | 20260505 05:20:51.115 | INFO | MemInfo: { | 20260505 05:20:51.115 | INFO | "gpu" : " 17 GB", | 20260505 05:20:51.115 | INFO | "huge" : "", | 20260505 05:20:51.116 | INFO | "huge_shared" : " 27 GB", | 20260505 05:20:51.116 | INFO | "malloc" : " 14 GB", | 20260505 05:20:51.116 | INFO | "pinned" : " 29 GB" | 20260505 05:20:51.116 | INFO | } | 20260505 05:20:51.116 | INFO | Perf report for the last 5.13945 seconds: | 20260505 05:20:51.116 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:20:51.116 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:51.116 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:51.116 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:51.116 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260505 05:20:51.116 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:51.116 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:51.116 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 05:20:51.116 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:51.116 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:51.116 | INFO | Uploaded 36GiB to GPU in 1.60166s (22.4767GiB/s) | 20260505 05:20:51.116 | INFO | Downloaded 34GiB to GPU in 1.51754s (22.4047GiB/s) | 20260505 05:20:56.255 | INFO | Analyzing chunk frames = [653824,654336) | 20260505 05:20:56.255 | INFO | MemInfo: { | 20260505 05:20:56.255 | INFO | "gpu" : " 17 GB", | 20260505 05:20:56.255 | INFO | "huge" : "", | 20260505 05:20:56.255 | INFO | "huge_shared" : " 27 GB", | 20260505 05:20:56.255 | INFO | "malloc" : " 14 GB", | 20260505 05:20:56.255 | INFO | "pinned" : " 29 GB" | 20260505 05:20:56.255 | INFO | } | 20260505 05:20:56.255 | INFO | Perf report for the last 5.13899 seconds: | 20260505 05:20:56.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:20:56.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:56.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:56.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:56.255 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260505 05:20:56.255 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:56.255 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:56.255 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 05:20:56.255 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:56.255 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:20:56.255 | INFO | Uploaded 36GiB to GPU in 1.60231s (22.4675GiB/s) | 20260505 05:20:56.255 | INFO | Downloaded 34GiB to GPU in 1.51845s (22.3913GiB/s) | 20260505 05:21:01.394 | INFO | Analyzing chunk frames = [654336,654848) | 20260505 05:21:01.394 | INFO | MemInfo: { | 20260505 05:21:01.394 | INFO | "gpu" : " 17 GB", | 20260505 05:21:01.394 | INFO | "huge" : "", | 20260505 05:21:01.394 | INFO | "huge_shared" : " 27 GB", | 20260505 05:21:01.394 | INFO | "malloc" : " 14 GB", | 20260505 05:21:01.394 | INFO | "pinned" : " 29 GB" | 20260505 05:21:01.394 | INFO | } | 20260505 05:21:01.394 | INFO | Perf report for the last 5.13958 seconds: | 20260505 05:21:01.394 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:21:01.394 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:01.394 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:01.394 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:01.394 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 05:21:01.394 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:01.394 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:01.394 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 05:21:01.394 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:01.394 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:01.394 | INFO | Uploaded 36GiB to GPU in 1.60287s (22.4598GiB/s) | 20260505 05:21:01.394 | INFO | Downloaded 34GiB to GPU in 1.52069s (22.3583GiB/s) | 20260505 05:21:06.533 | INFO | Analyzing chunk frames = [654848,655360) | 20260505 05:21:06.533 | INFO | MemInfo: { | 20260505 05:21:06.533 | INFO | "gpu" : " 17 GB", | 20260505 05:21:06.533 | INFO | "huge" : "", | 20260505 05:21:06.533 | INFO | "huge_shared" : " 27 GB", | 20260505 05:21:06.533 | INFO | "malloc" : " 14 GB", | 20260505 05:21:06.533 | INFO | "pinned" : " 29 GB" | 20260505 05:21:06.533 | INFO | } | 20260505 05:21:06.534 | INFO | Perf report for the last 5.13932 seconds: | 20260505 05:21:06.534 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:21:06.534 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:06.534 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:06.534 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:06.534 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260505 05:21:06.534 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:06.534 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:06.534 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 05:21:06.534 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:06.534 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:06.534 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 05:21:06.534 | INFO | Uploaded 36GiB to GPU in 1.60214s (22.4699GiB/s) | 20260505 05:21:06.534 | INFO | Total mem usage: 15.1 GiB | 20260505 05:21:06.534 | INFO | All Time High Water Marks sum: 15.7 GiB (Savings: 4.09%) | 20260505 05:21:06.534 | INFO | Downloaded 34GiB to GPU in 1.52015s (22.3662GiB/s) | 20260505 05:21:06.534 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -4.25%) | 20260505 05:21:06.534 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 05:21:06.534 | INFO | High Water Breakdown: | 20260505 05:21:06.534 | INFO | 59.8% : 8.63 GiB ( 95.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 05:21:06.534 | INFO | 40.1% : 5.79 GiB ( 87.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 05:21:06.534 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 05:21:06.534 | INFO | ------------------------------------------ | 20260505 05:21:06.534 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:21:06.534 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 05:21:06.534 | INFO | Total mem usage: 26.5 GiB | 20260505 05:21:06.534 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 05:21:06.534 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 05:21:06.534 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 05:21:06.534 | INFO | High Water Breakdown: | 20260505 05:21:06.534 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:21:06.535 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:21:06.535 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:21:06.535 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:21:06.535 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:21:06.535 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:21:06.535 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 05:21:06.535 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 05:21:06.535 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:21:06.535 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 05:21:06.535 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 05:21:06.535 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 05:21:06.535 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 05:21:06.535 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 05:21:06.535 | INFO | ------------------------------------------ | 20260505 05:21:06.535 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:21:06.535 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 05:21:06.535 | INFO | Total mem usage: 25.6 GiB | 20260505 05:21:06.535 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 05:21:06.535 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 05:21:06.535 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 05:21:06.535 | INFO | High Water Breakdown: | 20260505 05:21:06.535 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 05:21:06.535 | INFO | ------------------------------------------ | 20260505 05:21:06.535 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:21:06.535 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 05:21:06.535 | INFO | Total mem usage: 15.7 GiB | 20260505 05:21:06.535 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 05:21:06.535 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 05:21:06.535 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260505 05:21:06.535 | INFO | High Water Breakdown: | 20260505 05:21:06.535 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 05:21:06.535 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 05:21:06.535 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 05:21:06.535 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 05:21:06.535 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 05:21:06.535 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 05:21:06.535 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 05:21:06.535 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 05:21:06.535 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 05:21:06.535 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:21:06.535 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 05:21:06.535 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:21:06.535 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 05:21:06.535 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 05:21:06.535 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 05:21:06.535 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 05:21:06.535 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:21:06.535 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 05:21:06.535 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:21:06.535 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:21:06.535 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:21:06.535 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 05:21:06.535 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 05:21:06.535 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:21:06.535 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 05:21:06.535 | INFO | ------------------------------------------ | 20260505 05:21:06.535 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:21:06.535 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 05:21:06.535 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:21:06.535 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:21:06.535 | INFO | PerfReport: Full Execution -- 3.03e+01 / -nan, 1.25e+01 / 0.00e+00 | 20260505 05:21:06.535 | INFO | PerfReport: 0.29%, AggregateStats -- 8.65e-02 / -nan, 6.39e-03 / 0.00e+00 | 20260505 05:21:06.535 | INFO | PerfReport: 10.86%, Baseline -- 3.30e+00 / -nan, 1.75e-02 / 0.00e+00 | 20260505 05:21:06.535 | INFO | PerfReport: 3.05%, Histogram -- 9.24e-01 / -nan, 5.10e-02 / 0.00e+00 | 20260505 05:21:06.535 | INFO | PerfReport: 16.04%, DME -- 3.89e+01 / -nan, 3.19e+00 / 0.00e+00 | 20260505 05:21:06.535 | INFO | PerfReport: 56.72%, FrameLabeling -- 1.72e+01 / -nan, 4.60e-02 / 0.00e+00 | 20260505 05:21:06.535 | INFO | PerfReport: 7.55%, PulseAccumulating -- 2.29e+00 / -nan, 1.50e-01 / 0.00e+00 | 20260505 05:21:06.535 | INFO | PerfReport: 3.89%, Metrics -- 1.18e+00 / -nan, 1.97e-01 / 0.00e+00 | 20260505 05:21:06.535 | INFO | PerfReport: 1.60%, PulseSerializing -- 4.84e-01 / -nan, 7.83e-02 / 0.00e+00 | 20260505 05:21:06.535 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 05:21:06.535 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:21:06.535 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:21:06.535 | INFO | PerfReport: Full Execution -- 9.10e+01 / -nan, 1.90e+01 / 0.00e+00 | 20260505 05:21:06.536 | INFO | PerfReport: 18.40%, Upload -- 1.68e+01 / 0.00e+00, 3.92e-01 / 0.00e+00 | 20260505 05:21:06.536 | INFO | PerfReport: 17.66%, Download -- 1.61e+01 / 0.00e+00, 7.26e-01 / 0.00e+00 | 20260505 05:21:06.536 | INFO | PerfReport: 33.33%, Compute -- 3.03e+01 / 0.00e+00, 1.25e+01 / 0.00e+00 | 20260505 05:21:06.536 | INFO | PerfReport: 0.56%, LockUpload -- 5.07e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260505 05:21:06.536 | INFO | PerfReport: 0.00%, LockDownload -- 4.00e-04 / 0.00e+00, 1.51e-04 / 0.00e+00 | 20260505 05:21:06.536 | INFO | PerfReport: 30.03%, LockCompute -- 2.73e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260505 05:21:06.536 | INFO | PerfReport: 0.02%, Misc -- 1.43e-02 / 0.00e+00, 2.56e-02 / 0.00e+00 | 20260505 05:21:06.631 | INFO | Starting a round of Baz IO | 20260505 05:21:09.575 | INFO | Populating RTMetrics report | 20260505 05:21:09.575 | INFO | RTMetrics report populated | 20260505 05:21:09.575 | INFO | Opening tmp RTMetrics json file | 20260505 05:21:09.582 | INFO | Serializing RTMetrics to json | 20260505 05:21:09.583 | INFO | Serialization done, swapping tmp file now... | 20260505 05:21:09.584 | INFO | RTMetrics done | 20260505 05:21:09.680 | INFO | Finished a round of Baz IO: Wrote 2.216 GiB in 3.049 seconds | 20260505 05:21:09.680 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.54% / 0.08% / 60.37% | 20260505 05:21:09.680 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.7554% / 0.000135104% / 3.97364e-05% | 20260505 05:21:09.680 | INFO | Baz IO overall: | 20260505 05:21:09.680 | INFO | BytesWritten : 302943784967 | 20260505 05:21:09.680 | INFO | NumEvents : 142100226077 | 20260505 05:21:11.673 | INFO | Analyzing chunk frames = [655360,655872) | 20260505 05:21:11.673 | INFO | MemInfo: { | 20260505 05:21:11.673 | INFO | "gpu" : " 17 GB", | 20260505 05:21:11.673 | INFO | "huge" : "", | 20260505 05:21:11.673 | INFO | "huge_shared" : " 27 GB", | 20260505 05:21:11.673 | INFO | "malloc" : " 14 GB", | 20260505 05:21:11.673 | INFO | "pinned" : " 29 GB" | 20260505 05:21:11.673 | INFO | } | 20260505 05:21:11.673 | INFO | Perf report for the last 5.1394 seconds: | 20260505 05:21:11.673 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:21:11.673 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:11.673 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:11.673 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:11.673 | INFO | ANALYSIS : 58.0%, 2.9, 89.7ms, 0 | 20260505 05:21:11.673 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:11.673 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260505 05:21:11.673 | INFO | PRE_HQ : 57.3%, 2.2, 64.0ms, 0 | 20260505 05:21:11.673 | INFO | BAZWRITER : 58.7%, 4.3, 129.5ms, 0 | 20260505 05:21:11.673 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 05:21:11.674 | INFO | Uploaded 36GiB to GPU in 1.64573s (21.8747GiB/s) | 20260505 05:21:11.674 | INFO | Downloaded 38GiB to GPU in 1.70959s (22.2275GiB/s) | 20260505 05:21:16.812 | INFO | Analyzing chunk frames = [655872,656384) | 20260505 05:21:16.812 | INFO | MemInfo: { | 20260505 05:21:16.812 | INFO | "gpu" : " 17 GB", | 20260505 05:21:16.812 | INFO | "huge" : "", | 20260505 05:21:16.812 | INFO | "huge_shared" : " 27 GB", | 20260505 05:21:16.812 | INFO | "malloc" : " 14 GB", | 20260505 05:21:16.812 | INFO | "pinned" : " 29 GB" | 20260505 05:21:16.812 | INFO | } | 20260505 05:21:16.812 | INFO | Perf report for the last 5.13941 seconds: | 20260505 05:21:16.812 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:21:16.812 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:16.812 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:16.812 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:16.812 | INFO | ANALYSIS : 57.8%, 3.0, 90.2ms, 0 | 20260505 05:21:16.812 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:16.812 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:16.812 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:21:16.812 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:16.812 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:16.813 | INFO | Uploaded 36GiB to GPU in 1.60103s (22.4855GiB/s) | 20260505 05:21:16.813 | INFO | Downloaded 34GiB to GPU in 1.51858s (22.3893GiB/s) | 20260505 05:21:21.951 | INFO | Analyzing chunk frames = [656384,656896) | 20260505 05:21:21.951 | INFO | MemInfo: { | 20260505 05:21:21.951 | INFO | "gpu" : " 17 GB", | 20260505 05:21:21.951 | INFO | "huge" : "", | 20260505 05:21:21.951 | INFO | "huge_shared" : " 27 GB", | 20260505 05:21:21.951 | INFO | "malloc" : " 14 GB", | 20260505 05:21:21.951 | INFO | "pinned" : " 29 GB" | 20260505 05:21:21.951 | INFO | } | 20260505 05:21:21.952 | INFO | Perf report for the last 5.1391 seconds: | 20260505 05:21:21.952 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:21:21.952 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:21.952 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:21.952 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:21.952 | INFO | ANALYSIS : 57.2%, 3.0, 88.9ms, 0 | 20260505 05:21:21.952 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:21.952 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:21.952 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:21:21.952 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:21.952 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:21.952 | INFO | Uploaded 36GiB to GPU in 1.60401s (22.4437GiB/s) | 20260505 05:21:21.952 | INFO | Downloaded 34GiB to GPU in 1.52074s (22.3575GiB/s) | 20260505 05:21:27.091 | INFO | Analyzing chunk frames = [656896,657408) | 20260505 05:21:27.091 | INFO | MemInfo: { | 20260505 05:21:27.091 | INFO | "gpu" : " 17 GB", | 20260505 05:21:27.091 | INFO | "huge" : "", | 20260505 05:21:27.091 | INFO | "huge_shared" : " 27 GB", | 20260505 05:21:27.091 | INFO | "malloc" : " 14 GB", | 20260505 05:21:27.091 | INFO | "pinned" : " 29 GB" | 20260505 05:21:27.091 | INFO | } | 20260505 05:21:27.091 | INFO | Perf report for the last 5.13949 seconds: | 20260505 05:21:27.091 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:21:27.091 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:27.091 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:27.091 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:27.091 | INFO | ANALYSIS : 56.9%, 3.0, 88.2ms, 0 | 20260505 05:21:27.091 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:27.091 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:27.091 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260505 05:21:27.091 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:27.091 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:27.092 | INFO | Uploaded 36GiB to GPU in 1.60351s (22.4507GiB/s) | 20260505 05:21:27.092 | INFO | Downloaded 34GiB to GPU in 1.52068s (22.3584GiB/s) | 20260505 05:21:32.230 | INFO | Analyzing chunk frames = [657408,657920) | 20260505 05:21:32.230 | INFO | MemInfo: { | 20260505 05:21:32.230 | INFO | "gpu" : " 17 GB", | 20260505 05:21:32.230 | INFO | "huge" : "", | 20260505 05:21:32.230 | INFO | "huge_shared" : " 27 GB", | 20260505 05:21:32.230 | INFO | "malloc" : " 14 GB", | 20260505 05:21:32.230 | INFO | "pinned" : " 29 GB" | 20260505 05:21:32.230 | INFO | } | 20260505 05:21:32.230 | INFO | Perf report for the last 5.13931 seconds: | 20260505 05:21:32.230 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:21:32.230 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:32.230 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:32.230 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:32.230 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260505 05:21:32.230 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:32.230 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:32.230 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 05:21:32.230 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:32.231 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:32.231 | INFO | Uploaded 36GiB to GPU in 1.60344s (22.4517GiB/s) | 20260505 05:21:32.231 | INFO | Downloaded 34GiB to GPU in 1.52034s (22.3634GiB/s) | 20260505 05:21:37.369 | INFO | Analyzing chunk frames = [657920,658432) | 20260505 05:21:37.369 | INFO | MemInfo: { | 20260505 05:21:37.369 | INFO | "gpu" : " 17 GB", | 20260505 05:21:37.370 | INFO | "huge" : "", | 20260505 05:21:37.370 | INFO | "huge_shared" : " 27 GB", | 20260505 05:21:37.370 | INFO | "malloc" : " 14 GB", | 20260505 05:21:37.370 | INFO | "pinned" : " 29 GB" | 20260505 05:21:37.370 | INFO | } | 20260505 05:21:37.370 | INFO | Perf report for the last 5.13939 seconds: | 20260505 05:21:37.370 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:21:37.370 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:37.370 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:37.370 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:37.370 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260505 05:21:37.370 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:37.370 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:37.370 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 05:21:37.370 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:37.370 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:37.370 | INFO | Uploaded 36GiB to GPU in 1.60207s (22.471GiB/s) | 20260505 05:21:37.370 | INFO | Downloaded 34GiB to GPU in 1.51884s (22.3855GiB/s) | 20260505 05:21:42.509 | INFO | Analyzing chunk frames = [658432,658944) | 20260505 05:21:42.509 | INFO | MemInfo: { | 20260505 05:21:42.509 | INFO | "gpu" : " 17 GB", | 20260505 05:21:42.509 | INFO | "huge" : "", | 20260505 05:21:42.509 | INFO | "huge_shared" : " 27 GB", | 20260505 05:21:42.509 | INFO | "malloc" : " 14 GB", | 20260505 05:21:42.509 | INFO | "pinned" : " 29 GB" | 20260505 05:21:42.509 | INFO | } | 20260505 05:21:42.509 | INFO | Perf report for the last 5.1392 seconds: | 20260505 05:21:42.509 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:21:42.509 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:42.509 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:42.509 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:42.509 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 05:21:42.509 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:42.509 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:42.509 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 05:21:42.509 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:42.509 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:42.509 | INFO | Uploaded 36GiB to GPU in 1.60148s (22.4792GiB/s) | 20260505 05:21:42.509 | INFO | Downloaded 34GiB to GPU in 1.51989s (22.37GiB/s) | 20260505 05:21:47.648 | INFO | Analyzing chunk frames = [658944,659456) | 20260505 05:21:47.648 | INFO | MemInfo: { | 20260505 05:21:47.648 | INFO | "gpu" : " 17 GB", | 20260505 05:21:47.648 | INFO | "huge" : "", | 20260505 05:21:47.648 | INFO | "huge_shared" : " 27 GB", | 20260505 05:21:47.648 | INFO | "malloc" : " 14 GB", | 20260505 05:21:47.648 | INFO | "pinned" : " 29 GB" | 20260505 05:21:47.648 | INFO | } | 20260505 05:21:47.648 | INFO | Perf report for the last 5.1393 seconds: | 20260505 05:21:47.648 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:21:47.648 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:47.648 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:47.648 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:47.648 | INFO | ANALYSIS : 57.5%, 3.0, 89.6ms, 0 | 20260505 05:21:47.648 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:47.648 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:47.648 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260505 05:21:47.648 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:47.648 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:47.649 | INFO | Uploaded 36GiB to GPU in 1.60167s (22.4766GiB/s) | 20260505 05:21:47.649 | INFO | Downloaded 34GiB to GPU in 1.5205s (22.3611GiB/s) | 20260505 05:21:47.746 | INFO | Starting a round of Baz IO | 20260505 05:21:50.688 | INFO | Populating RTMetrics report | 20260505 05:21:50.688 | INFO | RTMetrics report populated | 20260505 05:21:50.688 | INFO | Opening tmp RTMetrics json file | 20260505 05:21:50.700 | INFO | Serializing RTMetrics to json | 20260505 05:21:50.702 | INFO | Serialization done, swapping tmp file now... | 20260505 05:21:50.705 | INFO | RTMetrics done | 20260505 05:21:50.790 | INFO | Finished a round of Baz IO: Wrote 2.216 GiB in 3.043 seconds | 20260505 05:21:50.790 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.46% / 0.08% / 60.46% | 20260505 05:21:50.790 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.71025% / 0.000143051% / 5.16574e-05% | 20260505 05:21:50.790 | INFO | Baz IO overall: | 20260505 05:21:50.790 | INFO | BytesWritten : 305323609822 | 20260505 05:21:50.790 | INFO | NumEvents : 143232936735 | 20260505 05:21:52.787 | INFO | Analyzing chunk frames = [659456,659968) | 20260505 05:21:52.787 | INFO | MemInfo: { | 20260505 05:21:52.787 | INFO | "gpu" : " 17 GB", | 20260505 05:21:52.787 | INFO | "huge" : "", | 20260505 05:21:52.787 | INFO | "huge_shared" : " 27 GB", | 20260505 05:21:52.788 | INFO | "malloc" : " 14 GB", | 20260505 05:21:52.788 | INFO | "pinned" : " 29 GB" | 20260505 05:21:52.788 | INFO | } | 20260505 05:21:52.788 | INFO | Perf report for the last 5.13935 seconds: | 20260505 05:21:52.788 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:21:52.788 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:52.788 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:52.788 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:52.788 | INFO | ANALYSIS : 58.0%, 2.9, 89.5ms, 0 | 20260505 05:21:52.788 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:52.788 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 05:21:52.788 | INFO | PRE_HQ : 57.3%, 2.2, 64.5ms, 0 | 20260505 05:21:52.788 | INFO | BAZWRITER : 58.6%, 4.3, 130.4ms, 0 | 20260505 05:21:52.788 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 05:21:52.788 | INFO | Uploaded 36GiB to GPU in 1.64719s (21.8554GiB/s) | 20260505 05:21:52.788 | INFO | Downloaded 38GiB to GPU in 1.71424s (22.1673GiB/s) | 20260505 05:21:57.927 | INFO | Analyzing chunk frames = [659968,660480) | 20260505 05:21:57.927 | INFO | MemInfo: { | 20260505 05:21:57.927 | INFO | "gpu" : " 17 GB", | 20260505 05:21:57.927 | INFO | "huge" : "", | 20260505 05:21:57.927 | INFO | "huge_shared" : " 27 GB", | 20260505 05:21:57.927 | INFO | "malloc" : " 14 GB", | 20260505 05:21:57.927 | INFO | "pinned" : " 29 GB" | 20260505 05:21:57.927 | INFO | } | 20260505 05:21:57.927 | INFO | Perf report for the last 5.13931 seconds: | 20260505 05:21:57.927 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:21:57.927 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:57.927 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:57.927 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:57.927 | INFO | ANALYSIS : 57.8%, 3.0, 90.2ms, 0 | 20260505 05:21:57.927 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:57.927 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:57.927 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:21:57.927 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:57.927 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:21:57.927 | INFO | Uploaded 36GiB to GPU in 1.60011s (22.4984GiB/s) | 20260505 05:21:57.927 | INFO | Downloaded 34GiB to GPU in 1.51741s (22.4066GiB/s) | 20260505 05:22:03.066 | INFO | Analyzing chunk frames = [660480,660992) | 20260505 05:22:03.066 | INFO | MemInfo: { | 20260505 05:22:03.066 | INFO | "gpu" : " 17 GB", | 20260505 05:22:03.066 | INFO | "huge" : "", | 20260505 05:22:03.066 | INFO | "huge_shared" : " 27 GB", | 20260505 05:22:03.066 | INFO | "malloc" : " 14 GB", | 20260505 05:22:03.066 | INFO | "pinned" : " 29 GB" | 20260505 05:22:03.066 | INFO | } | 20260505 05:22:03.066 | INFO | Perf report for the last 5.13936 seconds: | 20260505 05:22:03.066 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:22:03.066 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:03.066 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:03.066 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:03.066 | INFO | ANALYSIS : 57.2%, 3.0, 88.9ms, 0 | 20260505 05:22:03.066 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:03.066 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:03.066 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 05:22:03.066 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:03.066 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:03.067 | INFO | Uploaded 36GiB to GPU in 1.60444s (22.4377GiB/s) | 20260505 05:22:03.067 | INFO | Downloaded 34GiB to GPU in 1.52162s (22.3446GiB/s) | 20260505 05:22:08.206 | INFO | Analyzing chunk frames = [660992,661504) | 20260505 05:22:08.206 | INFO | MemInfo: { | 20260505 05:22:08.206 | INFO | "gpu" : " 17 GB", | 20260505 05:22:08.206 | INFO | "huge" : "", | 20260505 05:22:08.206 | INFO | "huge_shared" : " 27 GB", | 20260505 05:22:08.206 | INFO | "malloc" : " 14 GB", | 20260505 05:22:08.206 | INFO | "pinned" : " 29 GB" | 20260505 05:22:08.206 | INFO | } | 20260505 05:22:08.206 | INFO | Perf report for the last 5.1394 seconds: | 20260505 05:22:08.206 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:22:08.206 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:08.206 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:08.206 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:08.206 | INFO | ANALYSIS : 56.9%, 3.0, 88.2ms, 0 | 20260505 05:22:08.206 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:08.206 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:08.206 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 05:22:08.206 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:08.206 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:08.206 | INFO | Uploaded 36GiB to GPU in 1.60362s (22.4492GiB/s) | 20260505 05:22:08.206 | INFO | Downloaded 34GiB to GPU in 1.52028s (22.3643GiB/s) | 20260505 05:22:13.345 | INFO | Analyzing chunk frames = [661504,662016) | 20260505 05:22:13.345 | INFO | MemInfo: { | 20260505 05:22:13.345 | INFO | "gpu" : " 17 GB", | 20260505 05:22:13.345 | INFO | "huge" : "", | 20260505 05:22:13.345 | INFO | "huge_shared" : " 27 GB", | 20260505 05:22:13.345 | INFO | "malloc" : " 14 GB", | 20260505 05:22:13.345 | INFO | "pinned" : " 29 GB" | 20260505 05:22:13.345 | INFO | } | 20260505 05:22:13.345 | INFO | Perf report for the last 5.13941 seconds: | 20260505 05:22:13.345 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:22:13.345 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:13.345 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:13.345 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:13.345 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260505 05:22:13.345 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:13.345 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:13.345 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260505 05:22:13.345 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:13.345 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:13.345 | INFO | Uploaded 36GiB to GPU in 1.60204s (22.4714GiB/s) | 20260505 05:22:13.345 | INFO | Downloaded 34GiB to GPU in 1.5188s (22.386GiB/s) | 20260505 05:22:18.484 | INFO | Analyzing chunk frames = [662016,662528) | 20260505 05:22:18.484 | INFO | MemInfo: { | 20260505 05:22:18.484 | INFO | "gpu" : " 17 GB", | 20260505 05:22:18.484 | INFO | "huge" : "", | 20260505 05:22:18.484 | INFO | "huge_shared" : " 27 GB", | 20260505 05:22:18.484 | INFO | "malloc" : " 14 GB", | 20260505 05:22:18.484 | INFO | "pinned" : " 29 GB" | 20260505 05:22:18.484 | INFO | } | 20260505 05:22:18.485 | INFO | Perf report for the last 5.13934 seconds: | 20260505 05:22:18.485 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:22:18.485 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:18.485 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:18.485 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:18.485 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260505 05:22:18.485 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:18.485 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:18.485 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260505 05:22:18.485 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:18.485 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:18.485 | INFO | Uploaded 36GiB to GPU in 1.60286s (22.4599GiB/s) | 20260505 05:22:18.485 | INFO | Downloaded 34GiB to GPU in 1.51923s (22.3797GiB/s) | 20260505 05:22:23.623 | INFO | Analyzing chunk frames = [662528,663040) | 20260505 05:22:23.623 | INFO | MemInfo: { | 20260505 05:22:23.623 | INFO | "gpu" : " 17 GB", | 20260505 05:22:23.623 | INFO | "huge" : "", | 20260505 05:22:23.623 | INFO | "huge_shared" : " 27 GB", | 20260505 05:22:23.623 | INFO | "malloc" : " 14 GB", | 20260505 05:22:23.624 | INFO | "pinned" : " 29 GB" | 20260505 05:22:23.624 | INFO | } | 20260505 05:22:23.624 | INFO | Perf report for the last 5.13904 seconds: | 20260505 05:22:23.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:22:23.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:23.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:23.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:23.624 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 05:22:23.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:23.624 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:23.624 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260505 05:22:23.624 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:23.624 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:23.624 | INFO | Uploaded 36GiB to GPU in 1.60262s (22.4632GiB/s) | 20260505 05:22:23.624 | INFO | Downloaded 34GiB to GPU in 1.51986s (22.3704GiB/s) | 20260505 05:22:28.763 | INFO | Analyzing chunk frames = [663040,663552) | 20260505 05:22:28.763 | INFO | MemInfo: { | 20260505 05:22:28.763 | INFO | "gpu" : " 17 GB", | 20260505 05:22:28.763 | INFO | "huge" : "", | 20260505 05:22:28.763 | INFO | "huge_shared" : " 27 GB", | 20260505 05:22:28.763 | INFO | "malloc" : " 14 GB", | 20260505 05:22:28.763 | INFO | "pinned" : " 29 GB" | 20260505 05:22:28.763 | INFO | } | 20260505 05:22:28.763 | INFO | Perf report for the last 5.13928 seconds: | 20260505 05:22:28.763 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:22:28.763 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:28.763 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:28.763 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:28.763 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260505 05:22:28.763 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:28.763 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:28.763 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 05:22:28.763 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:28.763 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:28.763 | INFO | Uploaded 36GiB to GPU in 1.60244s (22.4658GiB/s) | 20260505 05:22:28.763 | INFO | Downloaded 34GiB to GPU in 1.51966s (22.3734GiB/s) | 20260505 05:22:28.764 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 05:22:28.764 | INFO | Total mem usage: 15.1 GiB | 20260505 05:22:28.764 | INFO | All Time High Water Marks sum: 15.7 GiB (Savings: 4.09%) | 20260505 05:22:28.764 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -4.23%) | 20260505 05:22:28.764 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 05:22:28.764 | INFO | High Water Breakdown: | 20260505 05:22:28.764 | INFO | 59.8% : 8.64 GiB ( 95.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 05:22:28.764 | INFO | 40.1% : 5.79 GiB ( 87.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 05:22:28.764 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 05:22:28.764 | INFO | ------------------------------------------ | 20260505 05:22:28.764 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:22:28.764 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 05:22:28.764 | INFO | Total mem usage: 26.5 GiB | 20260505 05:22:28.764 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 05:22:28.764 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 05:22:28.764 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 05:22:28.764 | INFO | High Water Breakdown: | 20260505 05:22:28.764 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:22:28.764 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:22:28.764 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:22:28.764 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:22:28.764 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:22:28.764 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:22:28.764 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 05:22:28.764 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 05:22:28.764 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:22:28.764 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 05:22:28.764 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 05:22:28.764 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 05:22:28.764 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 05:22:28.764 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 05:22:28.764 | INFO | ------------------------------------------ | 20260505 05:22:28.764 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:22:28.764 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 05:22:28.764 | INFO | Total mem usage: 25.6 GiB | 20260505 05:22:28.764 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 05:22:28.764 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 05:22:28.764 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 05:22:28.764 | INFO | High Water Breakdown: | 20260505 05:22:28.764 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 05:22:28.764 | INFO | ------------------------------------------ | 20260505 05:22:28.764 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:22:28.764 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 05:22:28.764 | INFO | Total mem usage: 15.7 GiB | 20260505 05:22:28.764 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 05:22:28.764 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 05:22:28.764 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260505 05:22:28.764 | INFO | High Water Breakdown: | 20260505 05:22:28.764 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 05:22:28.764 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 05:22:28.764 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 05:22:28.764 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 05:22:28.764 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 05:22:28.764 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 05:22:28.764 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 05:22:28.764 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 05:22:28.764 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 05:22:28.764 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:22:28.764 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 05:22:28.764 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:22:28.764 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 05:22:28.764 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 05:22:28.764 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 05:22:28.764 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 05:22:28.764 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:22:28.765 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 05:22:28.765 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:22:28.765 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:22:28.765 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:22:28.765 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 05:22:28.765 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 05:22:28.765 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:22:28.765 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 05:22:28.765 | INFO | ------------------------------------------ | 20260505 05:22:28.765 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:22:28.765 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 05:22:28.765 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:22:28.765 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:22:28.765 | INFO | PerfReport: Full Execution -- 3.03e+01 / -nan, 1.28e+01 / 0.00e+00 | 20260505 05:22:28.765 | INFO | PerfReport: 0.28%, AggregateStats -- 8.64e-02 / -nan, 4.36e-03 / 0.00e+00 | 20260505 05:22:28.765 | INFO | PerfReport: 10.87%, Baseline -- 3.30e+00 / -nan, 2.14e-02 / 0.00e+00 | 20260505 05:22:28.765 | INFO | PerfReport: 3.05%, Histogram -- 9.24e-01 / -nan, 5.22e-02 / 0.00e+00 | 20260505 05:22:28.765 | INFO | PerfReport: 16.04%, DME -- 3.89e+01 / -nan, 3.25e+00 / 0.00e+00 | 20260505 05:22:28.765 | INFO | PerfReport: 56.73%, FrameLabeling -- 1.72e+01 / -nan, 4.59e-02 / 0.00e+00 | 20260505 05:22:28.765 | INFO | PerfReport: 7.55%, PulseAccumulating -- 2.29e+00 / -nan, 1.51e-01 / 0.00e+00 | 20260505 05:22:28.765 | INFO | PerfReport: 3.88%, Metrics -- 1.18e+00 / -nan, 1.95e-01 / 0.00e+00 | 20260505 05:22:28.765 | INFO | PerfReport: 1.59%, PulseSerializing -- 4.84e-01 / -nan, 7.82e-02 / 0.00e+00 | 20260505 05:22:28.765 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 05:22:28.765 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:22:28.765 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:22:28.765 | INFO | PerfReport: Full Execution -- 9.10e+01 / -nan, 1.90e+01 / 0.00e+00 | 20260505 05:22:28.765 | INFO | PerfReport: 18.40%, Upload -- 1.68e+01 / 0.00e+00, 3.97e-01 / 0.00e+00 | 20260505 05:22:28.765 | INFO | PerfReport: 17.66%, Download -- 1.61e+01 / 0.00e+00, 7.34e-01 / 0.00e+00 | 20260505 05:22:28.765 | INFO | PerfReport: 33.33%, Compute -- 3.03e+01 / 0.00e+00, 1.28e+01 / 0.00e+00 | 20260505 05:22:28.765 | INFO | PerfReport: 0.56%, LockUpload -- 5.06e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260505 05:22:28.765 | INFO | PerfReport: 0.00%, LockDownload -- 3.94e-04 / 0.00e+00, 1.46e-04 / 0.00e+00 | 20260505 05:22:28.765 | INFO | PerfReport: 30.04%, LockCompute -- 2.73e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260505 05:22:28.765 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.28e-02 / 0.00e+00 | 20260505 05:22:28.867 | INFO | Starting a round of Baz IO | 20260505 05:22:31.803 | INFO | Populating RTMetrics report | 20260505 05:22:31.803 | INFO | RTMetrics report populated | 20260505 05:22:31.803 | INFO | Opening tmp RTMetrics json file | 20260505 05:22:31.820 | INFO | Serializing RTMetrics to json | 20260505 05:22:31.826 | INFO | Serialization done, swapping tmp file now... | 20260505 05:22:31.826 | INFO | RTMetrics done | 20260505 05:22:31.915 | INFO | Finished a round of Baz IO: Wrote 2.221 GiB in 3.048 seconds | 20260505 05:22:31.915 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.37% / 0.08% / 60.54% | 20260505 05:22:31.916 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.68221% / 0.000127157% / 6.35783e-05% | 20260505 05:22:31.916 | INFO | Baz IO overall: | 20260505 05:22:31.916 | INFO | BytesWritten : 307708480876 | 20260505 05:22:31.916 | INFO | NumEvents : 144368447376 | 20260505 05:22:33.902 | INFO | Analyzing chunk frames = [663552,664064) | 20260505 05:22:33.902 | INFO | MemInfo: { | 20260505 05:22:33.902 | INFO | "gpu" : " 17 GB", | 20260505 05:22:33.902 | INFO | "huge" : "", | 20260505 05:22:33.902 | INFO | "huge_shared" : " 27 GB", | 20260505 05:22:33.902 | INFO | "malloc" : " 14 GB", | 20260505 05:22:33.902 | INFO | "pinned" : " 29 GB" | 20260505 05:22:33.902 | INFO | } | 20260505 05:22:33.902 | INFO | Perf report for the last 5.13937 seconds: | 20260505 05:22:33.902 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:22:33.902 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:33.902 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:33.902 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:33.902 | INFO | ANALYSIS : 58.0%, 2.9, 89.0ms, 0 | 20260505 05:22:33.902 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:33.902 | INFO | RT_METRICS : 46.2%, 1.0, 24.2ms, 0 | 20260505 05:22:33.902 | INFO | PRE_HQ : 57.3%, 2.2, 64.7ms, 0 | 20260505 05:22:33.903 | INFO | BAZWRITER : 58.9%, 4.3, 131.7ms, 0 | 20260505 05:22:33.903 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 05:22:33.903 | INFO | Uploaded 36GiB to GPU in 1.65125s (21.8017GiB/s) | 20260505 05:22:33.903 | INFO | Downloaded 38GiB to GPU in 1.71614s (22.1428GiB/s) | 20260505 05:22:39.041 | INFO | Analyzing chunk frames = [664064,664576) | 20260505 05:22:39.042 | INFO | MemInfo: { | 20260505 05:22:39.042 | INFO | "gpu" : " 17 GB", | 20260505 05:22:39.042 | INFO | "huge" : "", | 20260505 05:22:39.042 | INFO | "huge_shared" : " 27 GB", | 20260505 05:22:39.042 | INFO | "malloc" : " 14 GB", | 20260505 05:22:39.042 | INFO | "pinned" : " 29 GB" | 20260505 05:22:39.042 | INFO | } | 20260505 05:22:39.042 | INFO | Perf report for the last 5.13943 seconds: | 20260505 05:22:39.042 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:22:39.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:39.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:39.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:39.042 | INFO | ANALYSIS : 57.8%, 3.0, 90.2ms, 0 | 20260505 05:22:39.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:39.042 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:39.042 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:22:39.042 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:39.042 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:39.042 | INFO | Uploaded 36GiB to GPU in 1.60015s (22.4978GiB/s) | 20260505 05:22:39.042 | INFO | Downloaded 34GiB to GPU in 1.51671s (22.417GiB/s) | 20260505 05:22:44.181 | INFO | Analyzing chunk frames = [664576,665088) | 20260505 05:22:44.181 | INFO | MemInfo: { | 20260505 05:22:44.181 | INFO | "gpu" : " 17 GB", | 20260505 05:22:44.181 | INFO | "huge" : "", | 20260505 05:22:44.181 | INFO | "huge_shared" : " 27 GB", | 20260505 05:22:44.181 | INFO | "malloc" : " 14 GB", | 20260505 05:22:44.181 | INFO | "pinned" : " 29 GB" | 20260505 05:22:44.181 | INFO | } | 20260505 05:22:44.182 | INFO | Perf report for the last 5.13979 seconds: | 20260505 05:22:44.182 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:22:44.182 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:44.182 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:44.182 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:44.182 | INFO | ANALYSIS : 57.1%, 3.0, 88.8ms, 0 | 20260505 05:22:44.182 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:44.182 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:44.182 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 05:22:44.182 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:44.182 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:44.182 | INFO | Uploaded 36GiB to GPU in 1.60457s (22.4359GiB/s) | 20260505 05:22:44.182 | INFO | Downloaded 34GiB to GPU in 1.52179s (22.342GiB/s) | 20260505 05:22:49.320 | INFO | Analyzing chunk frames = [665088,665600) | 20260505 05:22:49.320 | INFO | MemInfo: { | 20260505 05:22:49.320 | INFO | "gpu" : " 17 GB", | 20260505 05:22:49.320 | INFO | "huge" : "", | 20260505 05:22:49.320 | INFO | "huge_shared" : " 27 GB", | 20260505 05:22:49.320 | INFO | "malloc" : " 14 GB", | 20260505 05:22:49.320 | INFO | "pinned" : " 29 GB" | 20260505 05:22:49.320 | INFO | } | 20260505 05:22:49.320 | INFO | Perf report for the last 5.13888 seconds: | 20260505 05:22:49.320 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:22:49.320 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:49.320 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:49.320 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:49.320 | INFO | ANALYSIS : 56.9%, 3.0, 88.2ms, 0 | 20260505 05:22:49.320 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:49.320 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:49.321 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 05:22:49.321 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:49.321 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:49.321 | INFO | Uploaded 36GiB to GPU in 1.6035s (22.4509GiB/s) | 20260505 05:22:49.321 | INFO | Downloaded 34GiB to GPU in 1.52043s (22.3621GiB/s) | 20260505 05:22:54.460 | INFO | Analyzing chunk frames = [665600,666112) | 20260505 05:22:54.460 | INFO | MemInfo: { | 20260505 05:22:54.460 | INFO | "gpu" : " 17 GB", | 20260505 05:22:54.460 | INFO | "huge" : "", | 20260505 05:22:54.460 | INFO | "huge_shared" : " 27 GB", | 20260505 05:22:54.460 | INFO | "malloc" : " 14 GB", | 20260505 05:22:54.460 | INFO | "pinned" : " 29 GB" | 20260505 05:22:54.460 | INFO | } | 20260505 05:22:54.460 | INFO | Perf report for the last 5.13939 seconds: | 20260505 05:22:54.460 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:22:54.460 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:54.460 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:54.460 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:54.460 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260505 05:22:54.460 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:54.460 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:54.460 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260505 05:22:54.460 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:54.460 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:54.460 | INFO | Uploaded 36GiB to GPU in 1.6021s (22.4705GiB/s) | 20260505 05:22:54.460 | INFO | Downloaded 34GiB to GPU in 1.51863s (22.3886GiB/s) | 20260505 05:22:59.599 | INFO | Analyzing chunk frames = [666112,666624) | 20260505 05:22:59.599 | INFO | MemInfo: { | 20260505 05:22:59.599 | INFO | "gpu" : " 17 GB", | 20260505 05:22:59.599 | INFO | "huge" : "", | 20260505 05:22:59.599 | INFO | "huge_shared" : " 27 GB", | 20260505 05:22:59.599 | INFO | "malloc" : " 14 GB", | 20260505 05:22:59.599 | INFO | "pinned" : " 29 GB" | 20260505 05:22:59.599 | INFO | } | 20260505 05:22:59.599 | INFO | Perf report for the last 5.13957 seconds: | 20260505 05:22:59.599 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:22:59.599 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:59.599 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:59.599 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:59.599 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260505 05:22:59.599 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:59.599 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:59.600 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 05:22:59.600 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:59.600 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:22:59.600 | INFO | Uploaded 36GiB to GPU in 1.60176s (22.4752GiB/s) | 20260505 05:22:59.600 | INFO | Downloaded 34GiB to GPU in 1.51825s (22.3942GiB/s) | 20260505 05:23:04.738 | INFO | Analyzing chunk frames = [666624,667136) | 20260505 05:23:04.738 | INFO | MemInfo: { | 20260505 05:23:04.738 | INFO | "gpu" : " 17 GB", | 20260505 05:23:04.738 | INFO | "huge" : "", | 20260505 05:23:04.738 | INFO | "huge_shared" : " 27 GB", | 20260505 05:23:04.738 | INFO | "malloc" : " 14 GB", | 20260505 05:23:04.738 | INFO | "pinned" : " 29 GB" | 20260505 05:23:04.738 | INFO | } | 20260505 05:23:04.739 | INFO | Perf report for the last 5.1392 seconds: | 20260505 05:23:04.739 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:23:04.739 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:04.739 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:04.739 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:04.739 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 05:23:04.739 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:04.739 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:04.739 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 05:23:04.739 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:04.739 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:04.739 | INFO | Uploaded 36GiB to GPU in 1.60229s (22.4679GiB/s) | 20260505 05:23:04.739 | INFO | Downloaded 34GiB to GPU in 1.51976s (22.3719GiB/s) | 20260505 05:23:09.878 | INFO | Analyzing chunk frames = [667136,667648) | 20260505 05:23:09.878 | INFO | MemInfo: { | 20260505 05:23:09.878 | INFO | "gpu" : " 17 GB", | 20260505 05:23:09.878 | INFO | "huge" : "", | 20260505 05:23:09.878 | INFO | "huge_shared" : " 27 GB", | 20260505 05:23:09.878 | INFO | "malloc" : " 14 GB", | 20260505 05:23:09.878 | INFO | "pinned" : " 29 GB" | 20260505 05:23:09.878 | INFO | } | 20260505 05:23:09.878 | INFO | Perf report for the last 5.13936 seconds: | 20260505 05:23:09.878 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:23:09.878 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:09.878 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:09.878 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:09.878 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260505 05:23:09.878 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:09.878 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:09.878 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260505 05:23:09.878 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:09.878 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:09.878 | INFO | Uploaded 36GiB to GPU in 1.60288s (22.4596GiB/s) | 20260505 05:23:09.878 | INFO | Downloaded 34GiB to GPU in 1.52004s (22.3679GiB/s) | 20260505 05:23:09.975 | INFO | Starting a round of Baz IO | 20260505 05:23:12.918 | INFO | Populating RTMetrics report | 20260505 05:23:12.918 | INFO | RTMetrics report populated | 20260505 05:23:12.918 | INFO | Opening tmp RTMetrics json file | 20260505 05:23:12.931 | INFO | Serializing RTMetrics to json | 20260505 05:23:12.934 | INFO | Serialization done, swapping tmp file now... | 20260505 05:23:12.936 | INFO | RTMetrics done | 20260505 05:23:13.024 | INFO | Finished a round of Baz IO: Wrote 2.223 GiB in 3.048 seconds | 20260505 05:23:13.024 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.29% / 0.08% / 60.63% | 20260505 05:23:13.024 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.64832% / 0.000162919% / 4.37101e-05% | 20260505 05:23:13.024 | INFO | Baz IO overall: | 20260505 05:23:13.024 | INFO | BytesWritten : 310095136717 | 20260505 05:23:13.024 | INFO | NumEvents : 145504834206 | 20260505 05:23:15.017 | INFO | Analyzing chunk frames = [667648,668160) | 20260505 05:23:15.017 | INFO | MemInfo: { | 20260505 05:23:15.017 | INFO | "gpu" : " 17 GB", | 20260505 05:23:15.017 | INFO | "huge" : "", | 20260505 05:23:15.017 | INFO | "huge_shared" : " 27 GB", | 20260505 05:23:15.017 | INFO | "malloc" : " 14 GB", | 20260505 05:23:15.017 | INFO | "pinned" : " 29 GB" | 20260505 05:23:15.017 | INFO | } | 20260505 05:23:15.017 | INFO | Perf report for the last 5.1393 seconds: | 20260505 05:23:15.017 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:23:15.017 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:15.017 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:15.017 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:15.017 | INFO | ANALYSIS : 58.0%, 2.9, 89.4ms, 0 | 20260505 05:23:15.017 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:15.017 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260505 05:23:15.017 | INFO | PRE_HQ : 57.3%, 2.2, 64.2ms, 0 | 20260505 05:23:15.017 | INFO | BAZWRITER : 58.9%, 4.3, 131.2ms, 0 | 20260505 05:23:15.017 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 05:23:15.018 | INFO | Uploaded 36GiB to GPU in 1.65124s (21.8018GiB/s) | 20260505 05:23:15.018 | INFO | Downloaded 38GiB to GPU in 1.7164s (22.1393GiB/s) | 20260505 05:23:20.156 | INFO | Analyzing chunk frames = [668160,668672) | 20260505 05:23:20.156 | INFO | MemInfo: { | 20260505 05:23:20.156 | INFO | "gpu" : " 17 GB", | 20260505 05:23:20.156 | INFO | "huge" : "", | 20260505 05:23:20.156 | INFO | "huge_shared" : " 27 GB", | 20260505 05:23:20.157 | INFO | "malloc" : " 14 GB", | 20260505 05:23:20.157 | INFO | "pinned" : " 29 GB" | 20260505 05:23:20.157 | INFO | } | 20260505 05:23:20.157 | INFO | Perf report for the last 5.13934 seconds: | 20260505 05:23:20.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:23:20.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:20.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:20.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:20.157 | INFO | ANALYSIS : 57.8%, 3.0, 90.1ms, 0 | 20260505 05:23:20.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:20.157 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:20.157 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:23:20.157 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:20.157 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:20.157 | INFO | Uploaded 36GiB to GPU in 1.60071s (22.49GiB/s) | 20260505 05:23:20.157 | INFO | Downloaded 34GiB to GPU in 1.51714s (22.4106GiB/s) | 20260505 05:23:25.296 | INFO | Analyzing chunk frames = [668672,669184) | 20260505 05:23:25.296 | INFO | MemInfo: { | 20260505 05:23:25.296 | INFO | "gpu" : " 17 GB", | 20260505 05:23:25.296 | INFO | "huge" : "", | 20260505 05:23:25.296 | INFO | "huge_shared" : " 27 GB", | 20260505 05:23:25.296 | INFO | "malloc" : " 14 GB", | 20260505 05:23:25.296 | INFO | "pinned" : " 29 GB" | 20260505 05:23:25.296 | INFO | } | 20260505 05:23:25.296 | INFO | Perf report for the last 5.13933 seconds: | 20260505 05:23:25.296 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:23:25.296 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:25.296 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:25.296 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:25.296 | INFO | ANALYSIS : 57.2%, 3.0, 88.9ms, 0 | 20260505 05:23:25.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:25.296 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:25.296 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 05:23:25.296 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:25.296 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:25.297 | INFO | Uploaded 36GiB to GPU in 1.60439s (22.4384GiB/s) | 20260505 05:23:25.297 | INFO | Downloaded 34GiB to GPU in 1.52159s (22.3451GiB/s) | 20260505 05:23:30.435 | INFO | Analyzing chunk frames = [669184,669696) | 20260505 05:23:30.435 | INFO | MemInfo: { | 20260505 05:23:30.435 | INFO | "gpu" : " 17 GB", | 20260505 05:23:30.435 | INFO | "huge" : "", | 20260505 05:23:30.435 | INFO | "huge_shared" : " 27 GB", | 20260505 05:23:30.435 | INFO | "malloc" : " 14 GB", | 20260505 05:23:30.435 | INFO | "pinned" : " 29 GB" | 20260505 05:23:30.435 | INFO | } | 20260505 05:23:30.435 | INFO | Perf report for the last 5.13927 seconds: | 20260505 05:23:30.435 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:23:30.435 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:30.435 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:30.435 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:30.435 | INFO | ANALYSIS : 56.8%, 3.0, 88.1ms, 0 | 20260505 05:23:30.435 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:30.435 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:30.435 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:23:30.435 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:30.435 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:30.436 | INFO | Uploaded 36GiB to GPU in 1.60411s (22.4424GiB/s) | 20260505 05:23:30.436 | INFO | Downloaded 34GiB to GPU in 1.52159s (22.345GiB/s) | 20260505 05:23:35.574 | INFO | Analyzing chunk frames = [669696,670208) | 20260505 05:23:35.574 | INFO | MemInfo: { | 20260505 05:23:35.574 | INFO | "gpu" : " 17 GB", | 20260505 05:23:35.574 | INFO | "huge" : "", | 20260505 05:23:35.574 | INFO | "huge_shared" : " 27 GB", | 20260505 05:23:35.574 | INFO | "malloc" : " 14 GB", | 20260505 05:23:35.574 | INFO | "pinned" : " 29 GB" | 20260505 05:23:35.574 | INFO | } | 20260505 05:23:35.575 | INFO | Perf report for the last 5.13928 seconds: | 20260505 05:23:35.575 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:23:35.575 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:35.575 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:35.575 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:35.575 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260505 05:23:35.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:35.575 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:35.575 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260505 05:23:35.575 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:35.575 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:35.575 | INFO | Uploaded 36GiB to GPU in 1.6022s (22.469GiB/s) | 20260505 05:23:35.575 | INFO | Downloaded 34GiB to GPU in 1.51903s (22.3827GiB/s) | 20260505 05:23:40.714 | INFO | Analyzing chunk frames = [670208,670720) | 20260505 05:23:40.714 | INFO | MemInfo: { | 20260505 05:23:40.714 | INFO | "gpu" : " 17 GB", | 20260505 05:23:40.714 | INFO | "huge" : "", | 20260505 05:23:40.714 | INFO | "huge_shared" : " 27 GB", | 20260505 05:23:40.714 | INFO | "malloc" : " 14 GB", | 20260505 05:23:40.714 | INFO | "pinned" : " 29 GB" | 20260505 05:23:40.714 | INFO | } | 20260505 05:23:40.714 | INFO | Perf report for the last 5.13945 seconds: | 20260505 05:23:40.714 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:23:40.714 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:40.714 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:40.714 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:40.714 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260505 05:23:40.714 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:40.714 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:40.714 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 05:23:40.714 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:40.714 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:40.715 | INFO | Uploaded 36GiB to GPU in 1.60267s (22.4626GiB/s) | 20260505 05:23:40.715 | INFO | Downloaded 34GiB to GPU in 1.51902s (22.3829GiB/s) | 20260505 05:23:45.853 | INFO | Analyzing chunk frames = [670720,671232) | 20260505 05:23:45.853 | INFO | MemInfo: { | 20260505 05:23:45.853 | INFO | "gpu" : " 17 GB", | 20260505 05:23:45.853 | INFO | "huge" : "", | 20260505 05:23:45.853 | INFO | "huge_shared" : " 27 GB", | 20260505 05:23:45.853 | INFO | "malloc" : " 14 GB", | 20260505 05:23:45.853 | INFO | "pinned" : " 29 GB" | 20260505 05:23:45.853 | INFO | } | 20260505 05:23:45.853 | INFO | Perf report for the last 5.13922 seconds: | 20260505 05:23:45.853 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:23:45.853 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:45.853 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:45.853 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:45.853 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 05:23:45.853 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:45.853 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:45.853 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:23:45.853 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:45.853 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:45.854 | INFO | Uploaded 36GiB to GPU in 1.60195s (22.4726GiB/s) | 20260505 05:23:45.854 | INFO | Downloaded 34GiB to GPU in 1.51918s (22.3805GiB/s) | 20260505 05:23:50.993 | INFO | Analyzing chunk frames = [671232,671744) | 20260505 05:23:50.993 | INFO | MemInfo: { | 20260505 05:23:50.993 | INFO | "gpu" : " 17 GB", | 20260505 05:23:50.993 | INFO | "huge" : "", | 20260505 05:23:50.993 | INFO | "huge_shared" : " 27 GB", | 20260505 05:23:50.993 | INFO | "malloc" : " 14 GB", | 20260505 05:23:50.993 | INFO | "pinned" : " 29 GB" | 20260505 05:23:50.993 | INFO | } | 20260505 05:23:50.993 | INFO | Perf report for the last 5.1395 seconds: | 20260505 05:23:50.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:23:50.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:50.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:50.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:50.993 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260505 05:23:50.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:50.993 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:50.993 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:23:50.993 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:50.993 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:50.993 | INFO | Uploaded 36GiB to GPU in 1.60311s (22.4564GiB/s) | 20260505 05:23:50.993 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 05:23:50.993 | INFO | Total mem usage: 15.1 GiB | 20260505 05:23:50.993 | INFO | All Time High Water Marks sum: 15.7 GiB (Savings: 4.09%) | 20260505 05:23:50.993 | INFO | Downloaded 34GiB to GPU in 1.52013s (22.3664GiB/s) | 20260505 05:23:50.993 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -3.88%) | 20260505 05:23:50.993 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 05:23:50.993 | INFO | High Water Breakdown: | 20260505 05:23:50.993 | INFO | 59.9% : 8.69 GiB ( 96.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 05:23:50.993 | INFO | 39.9% : 5.79 GiB ( 87.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 05:23:50.993 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 05:23:50.993 | INFO | ------------------------------------------ | 20260505 05:23:50.993 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:23:50.994 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 05:23:50.994 | INFO | Total mem usage: 26.5 GiB | 20260505 05:23:50.994 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 05:23:50.994 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 05:23:50.994 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 05:23:50.994 | INFO | High Water Breakdown: | 20260505 05:23:50.994 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:23:50.994 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:23:50.994 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:23:50.994 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:23:50.994 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:23:50.994 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:23:50.994 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 05:23:50.994 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 05:23:50.994 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:23:50.994 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 05:23:50.994 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 05:23:50.994 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 05:23:50.994 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 05:23:50.994 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 05:23:50.994 | INFO | ------------------------------------------ | 20260505 05:23:50.994 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:23:50.994 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 05:23:50.994 | INFO | Total mem usage: 25.6 GiB | 20260505 05:23:50.994 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 05:23:50.994 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 05:23:50.994 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 05:23:50.994 | INFO | High Water Breakdown: | 20260505 05:23:50.994 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 05:23:50.994 | INFO | ------------------------------------------ | 20260505 05:23:50.994 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:23:50.994 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 05:23:50.994 | INFO | Total mem usage: 15.7 GiB | 20260505 05:23:50.994 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 05:23:50.994 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 05:23:50.994 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260505 05:23:50.994 | INFO | High Water Breakdown: | 20260505 05:23:50.994 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 05:23:50.994 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 05:23:50.994 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 05:23:50.994 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 05:23:50.994 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 05:23:50.994 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 05:23:50.994 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 05:23:50.994 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 05:23:50.994 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 05:23:50.994 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:23:50.994 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 05:23:50.994 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:23:50.994 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 05:23:50.994 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 05:23:50.994 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 05:23:50.994 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 05:23:50.994 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:23:50.994 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 05:23:50.994 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:23:50.994 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:23:50.994 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:23:50.994 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 05:23:50.994 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 05:23:50.994 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:23:50.994 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 05:23:50.994 | INFO | ------------------------------------------ | 20260505 05:23:50.994 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:23:50.994 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 05:23:50.994 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:23:50.994 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:23:50.994 | INFO | PerfReport: Full Execution -- 3.03e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260505 05:23:50.994 | INFO | PerfReport: 0.28%, AggregateStats -- 8.64e-02 / -nan, 3.95e-03 / 0.00e+00 | 20260505 05:23:50.994 | INFO | PerfReport: 10.87%, Baseline -- 3.30e+00 / -nan, 1.72e-02 / 0.00e+00 | 20260505 05:23:50.994 | INFO | PerfReport: 3.05%, Histogram -- 9.24e-01 / -nan, 5.17e-02 / 0.00e+00 | 20260505 05:23:50.994 | INFO | PerfReport: 16.04%, DME -- 3.89e+01 / -nan, 3.25e+00 / 0.00e+00 | 20260505 05:23:50.995 | INFO | PerfReport: 56.73%, FrameLabeling -- 1.72e+01 / -nan, 4.48e-02 / 0.00e+00 | 20260505 05:23:50.995 | INFO | PerfReport: 7.55%, PulseAccumulating -- 2.29e+00 / -nan, 1.50e-01 / 0.00e+00 | 20260505 05:23:50.995 | INFO | PerfReport: 3.88%, Metrics -- 1.18e+00 / -nan, 1.94e-01 / 0.00e+00 | 20260505 05:23:50.995 | INFO | PerfReport: 1.59%, PulseSerializing -- 4.83e-01 / -nan, 7.82e-02 / 0.00e+00 | 20260505 05:23:50.995 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 05:23:50.995 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:23:50.995 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:23:50.995 | INFO | PerfReport: Full Execution -- 9.10e+01 / -nan, 1.89e+01 / 0.00e+00 | 20260505 05:23:50.995 | INFO | PerfReport: 18.42%, Upload -- 1.68e+01 / 0.00e+00, 4.13e-01 / 0.00e+00 | 20260505 05:23:50.995 | INFO | PerfReport: 17.68%, Download -- 1.61e+01 / 0.00e+00, 7.51e-01 / 0.00e+00 | 20260505 05:23:50.995 | INFO | PerfReport: 33.35%, Compute -- 3.03e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260505 05:23:50.995 | INFO | PerfReport: 0.56%, LockUpload -- 5.08e-01 / 0.00e+00, 3.63e+00 / 0.00e+00 | 20260505 05:23:50.995 | INFO | PerfReport: 0.00%, LockDownload -- 3.86e-04 / 0.00e+00, 9.68e-05 / 0.00e+00 | 20260505 05:23:50.995 | INFO | PerfReport: 29.97%, LockCompute -- 2.73e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260505 05:23:50.995 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.48e-02 / 0.00e+00 | 20260505 05:23:51.090 | INFO | Starting a round of Baz IO | 20260505 05:23:54.031 | INFO | Populating RTMetrics report | 20260505 05:23:54.031 | INFO | RTMetrics report populated | 20260505 05:23:54.031 | INFO | Opening tmp RTMetrics json file | 20260505 05:23:54.037 | INFO | Serializing RTMetrics to json | 20260505 05:23:54.038 | INFO | Serialization done, swapping tmp file now... | 20260505 05:23:54.038 | INFO | RTMetrics done | 20260505 05:23:54.138 | INFO | Finished a round of Baz IO: Wrote 2.225 GiB in 3.048 seconds | 20260505 05:23:54.138 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.21% / 0.08% / 60.71% | 20260505 05:23:54.138 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.60815% / 9.53674e-05% / 4.37101e-05% | 20260505 05:23:54.138 | INFO | Baz IO overall: | 20260505 05:23:54.138 | INFO | BytesWritten : 312484459368 | 20260505 05:23:54.138 | INFO | NumEvents : 146642930888 | 20260505 05:23:56.132 | INFO | Analyzing chunk frames = [671744,672256) | 20260505 05:23:56.132 | INFO | MemInfo: { | 20260505 05:23:56.132 | INFO | "gpu" : " 17 GB", | 20260505 05:23:56.132 | INFO | "huge" : "", | 20260505 05:23:56.132 | INFO | "huge_shared" : " 27 GB", | 20260505 05:23:56.132 | INFO | "malloc" : " 14 GB", | 20260505 05:23:56.132 | INFO | "pinned" : " 29 GB" | 20260505 05:23:56.132 | INFO | } | 20260505 05:23:56.132 | INFO | Perf report for the last 5.13908 seconds: | 20260505 05:23:56.132 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:23:56.132 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:56.132 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:56.132 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:56.132 | INFO | ANALYSIS : 58.0%, 2.9, 89.2ms, 0 | 20260505 05:23:56.132 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:23:56.132 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260505 05:23:56.132 | INFO | PRE_HQ : 57.4%, 2.2, 64.1ms, 0 | 20260505 05:23:56.132 | INFO | BAZWRITER : 58.9%, 4.3, 130.9ms, 0 | 20260505 05:23:56.132 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 05:23:56.132 | INFO | Uploaded 36GiB to GPU in 1.65158s (21.7973GiB/s) | 20260505 05:23:56.132 | INFO | Downloaded 38GiB to GPU in 1.71556s (22.1502GiB/s) | 20260505 05:24:01.271 | INFO | Analyzing chunk frames = [672256,672768) | 20260505 05:24:01.271 | INFO | MemInfo: { | 20260505 05:24:01.271 | INFO | "gpu" : " 17 GB", | 20260505 05:24:01.271 | INFO | "huge" : "", | 20260505 05:24:01.271 | INFO | "huge_shared" : " 27 GB", | 20260505 05:24:01.271 | INFO | "malloc" : " 14 GB", | 20260505 05:24:01.271 | INFO | "pinned" : " 29 GB" | 20260505 05:24:01.271 | INFO | } | 20260505 05:24:01.271 | INFO | Perf report for the last 5.13951 seconds: | 20260505 05:24:01.271 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:24:01.271 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:01.271 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:01.271 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:01.271 | INFO | ANALYSIS : 57.8%, 3.0, 90.1ms, 0 | 20260505 05:24:01.271 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:01.271 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:01.272 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 05:24:01.272 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:01.272 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:01.272 | INFO | Uploaded 36GiB to GPU in 1.60143s (22.48GiB/s) | 20260505 05:24:01.272 | INFO | Downloaded 34GiB to GPU in 1.51798s (22.3982GiB/s) | 20260505 05:24:06.410 | INFO | Analyzing chunk frames = [672768,673280) | 20260505 05:24:06.410 | INFO | MemInfo: { | 20260505 05:24:06.410 | INFO | "gpu" : " 17 GB", | 20260505 05:24:06.410 | INFO | "huge" : "", | 20260505 05:24:06.410 | INFO | "huge_shared" : " 27 GB", | 20260505 05:24:06.410 | INFO | "malloc" : " 14 GB", | 20260505 05:24:06.410 | INFO | "pinned" : " 29 GB" | 20260505 05:24:06.410 | INFO | } | 20260505 05:24:06.411 | INFO | Perf report for the last 5.13927 seconds: | 20260505 05:24:06.411 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:24:06.411 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:06.411 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:06.411 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:06.411 | INFO | ANALYSIS : 57.2%, 3.0, 88.9ms, 0 | 20260505 05:24:06.411 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:06.411 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:06.411 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 05:24:06.411 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:06.411 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:06.411 | INFO | Uploaded 36GiB to GPU in 1.60427s (22.4401GiB/s) | 20260505 05:24:06.411 | INFO | Downloaded 34GiB to GPU in 1.52146s (22.3469GiB/s) | 20260505 05:24:11.550 | INFO | Analyzing chunk frames = [673280,673792) | 20260505 05:24:11.550 | INFO | MemInfo: { | 20260505 05:24:11.550 | INFO | "gpu" : " 17 GB", | 20260505 05:24:11.550 | INFO | "huge" : "", | 20260505 05:24:11.550 | INFO | "huge_shared" : " 27 GB", | 20260505 05:24:11.550 | INFO | "malloc" : " 14 GB", | 20260505 05:24:11.550 | INFO | "pinned" : " 29 GB" | 20260505 05:24:11.550 | INFO | } | 20260505 05:24:11.550 | INFO | Perf report for the last 5.1395 seconds: | 20260505 05:24:11.550 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:24:11.550 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:11.550 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:11.550 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:11.550 | INFO | ANALYSIS : 56.8%, 3.0, 88.2ms, 0 | 20260505 05:24:11.550 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:11.550 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:11.550 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260505 05:24:11.550 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:11.550 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:11.551 | INFO | Uploaded 36GiB to GPU in 1.60328s (22.454GiB/s) | 20260505 05:24:11.551 | INFO | Downloaded 34GiB to GPU in 1.52112s (22.352GiB/s) | 20260505 05:24:16.689 | INFO | Analyzing chunk frames = [673792,674304) | 20260505 05:24:16.689 | INFO | MemInfo: { | 20260505 05:24:16.689 | INFO | "gpu" : " 17 GB", | 20260505 05:24:16.689 | INFO | "huge" : "", | 20260505 05:24:16.689 | INFO | "huge_shared" : " 27 GB", | 20260505 05:24:16.689 | INFO | "malloc" : " 14 GB", | 20260505 05:24:16.689 | INFO | "pinned" : " 29 GB" | 20260505 05:24:16.689 | INFO | } | 20260505 05:24:16.689 | INFO | Perf report for the last 5.1391 seconds: | 20260505 05:24:16.689 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:24:16.689 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:16.689 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:16.689 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:16.689 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260505 05:24:16.689 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:16.689 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:16.689 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 05:24:16.689 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:16.689 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:16.690 | INFO | Uploaded 36GiB to GPU in 1.60206s (22.471GiB/s) | 20260505 05:24:16.690 | INFO | Downloaded 34GiB to GPU in 1.519s (22.3831GiB/s) | 20260505 05:24:21.829 | INFO | Analyzing chunk frames = [674304,674816) | 20260505 05:24:21.829 | INFO | MemInfo: { | 20260505 05:24:21.829 | INFO | "gpu" : " 17 GB", | 20260505 05:24:21.829 | INFO | "huge" : "", | 20260505 05:24:21.829 | INFO | "huge_shared" : " 27 GB", | 20260505 05:24:21.829 | INFO | "malloc" : " 14 GB", | 20260505 05:24:21.829 | INFO | "pinned" : " 29 GB" | 20260505 05:24:21.829 | INFO | } | 20260505 05:24:21.829 | INFO | Perf report for the last 5.13946 seconds: | 20260505 05:24:21.829 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:24:21.829 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:21.829 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:21.829 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:21.829 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260505 05:24:21.829 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:21.829 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:21.829 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 05:24:21.829 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:21.829 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:21.829 | INFO | Uploaded 36GiB to GPU in 1.60152s (22.4787GiB/s) | 20260505 05:24:21.829 | INFO | Downloaded 34GiB to GPU in 1.51886s (22.3852GiB/s) | 20260505 05:24:26.968 | INFO | Analyzing chunk frames = [674816,675328) | 20260505 05:24:26.968 | INFO | MemInfo: { | 20260505 05:24:26.968 | INFO | "gpu" : " 17 GB", | 20260505 05:24:26.968 | INFO | "huge" : "", | 20260505 05:24:26.968 | INFO | "huge_shared" : " 27 GB", | 20260505 05:24:26.968 | INFO | "malloc" : " 14 GB", | 20260505 05:24:26.968 | INFO | "pinned" : " 29 GB" | 20260505 05:24:26.968 | INFO | } | 20260505 05:24:26.968 | INFO | Perf report for the last 5.13933 seconds: | 20260505 05:24:26.968 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:24:26.968 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:26.968 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:26.968 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:26.968 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 05:24:26.968 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:26.968 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:26.968 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:24:26.968 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:26.968 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:26.968 | INFO | Uploaded 36GiB to GPU in 1.6015s (22.479GiB/s) | 20260505 05:24:26.968 | INFO | Downloaded 34GiB to GPU in 1.51953s (22.3753GiB/s) | 20260505 05:24:32.107 | INFO | Analyzing chunk frames = [675328,675840) | 20260505 05:24:32.107 | INFO | MemInfo: { | 20260505 05:24:32.107 | INFO | "gpu" : " 17 GB", | 20260505 05:24:32.107 | INFO | "huge" : "", | 20260505 05:24:32.107 | INFO | "huge_shared" : " 27 GB", | 20260505 05:24:32.107 | INFO | "malloc" : " 14 GB", | 20260505 05:24:32.107 | INFO | "pinned" : " 29 GB" | 20260505 05:24:32.107 | INFO | } | 20260505 05:24:32.107 | INFO | Perf report for the last 5.13932 seconds: | 20260505 05:24:32.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:24:32.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:32.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:32.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:32.108 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260505 05:24:32.108 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:32.108 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:32.108 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:24:32.108 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:32.108 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:32.108 | INFO | Uploaded 36GiB to GPU in 1.60276s (22.4612GiB/s) | 20260505 05:24:32.108 | INFO | Downloaded 34GiB to GPU in 1.52049s (22.3612GiB/s) | 20260505 05:24:32.204 | INFO | Starting a round of Baz IO | 20260505 05:24:35.149 | INFO | Populating RTMetrics report | 20260505 05:24:35.150 | INFO | RTMetrics report populated | 20260505 05:24:35.150 | INFO | Opening tmp RTMetrics json file | 20260505 05:24:35.156 | INFO | Serializing RTMetrics to json | 20260505 05:24:35.157 | INFO | Serialization done, swapping tmp file now... | 20260505 05:24:35.158 | INFO | RTMetrics done | 20260505 05:24:35.251 | INFO | Finished a round of Baz IO: Wrote 2.229 GiB in 3.046 seconds | 20260505 05:24:35.251 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.13% / 0.08% / 60.79% | 20260505 05:24:35.251 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.60956% / 0.000143051% / 6.35783e-05% | 20260505 05:24:35.251 | INFO | Baz IO overall: | 20260505 05:24:35.251 | INFO | BytesWritten : 314878348351 | 20260505 05:24:35.251 | INFO | NumEvents : 147784216875 | 20260505 05:24:37.246 | INFO | Analyzing chunk frames = [675840,676352) | 20260505 05:24:37.246 | INFO | MemInfo: { | 20260505 05:24:37.246 | INFO | "gpu" : " 17 GB", | 20260505 05:24:37.246 | INFO | "huge" : "", | 20260505 05:24:37.247 | INFO | "huge_shared" : " 27 GB", | 20260505 05:24:37.247 | INFO | "malloc" : " 14 GB", | 20260505 05:24:37.247 | INFO | "pinned" : " 29 GB" | 20260505 05:24:37.247 | INFO | } | 20260505 05:24:37.247 | INFO | Perf report for the last 5.13922 seconds: | 20260505 05:24:37.247 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:24:37.247 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:37.247 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:37.247 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:37.247 | INFO | ANALYSIS : 58.1%, 2.9, 89.7ms, 0 | 20260505 05:24:37.247 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:37.247 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260505 05:24:37.247 | INFO | PRE_HQ : 57.4%, 2.2, 64.2ms, 0 | 20260505 05:24:37.247 | INFO | BAZWRITER : 58.9%, 4.3, 130.4ms, 0 | 20260505 05:24:37.247 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 05:24:37.247 | INFO | Uploaded 36GiB to GPU in 1.64508s (21.8834GiB/s) | 20260505 05:24:37.247 | INFO | Downloaded 38GiB to GPU in 1.70927s (22.2317GiB/s) | 20260505 05:24:42.386 | INFO | Analyzing chunk frames = [676352,676864) | 20260505 05:24:42.386 | INFO | MemInfo: { | 20260505 05:24:42.386 | INFO | "gpu" : " 17 GB", | 20260505 05:24:42.386 | INFO | "huge" : "", | 20260505 05:24:42.386 | INFO | "huge_shared" : " 27 GB", | 20260505 05:24:42.386 | INFO | "malloc" : " 14 GB", | 20260505 05:24:42.386 | INFO | "pinned" : " 29 GB" | 20260505 05:24:42.386 | INFO | } | 20260505 05:24:42.386 | INFO | Perf report for the last 5.13957 seconds: | 20260505 05:24:42.386 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:24:42.386 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:42.386 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:42.386 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:42.386 | INFO | ANALYSIS : 57.8%, 3.0, 90.1ms, 0 | 20260505 05:24:42.386 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:42.386 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:42.386 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:24:42.386 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:42.386 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:42.386 | INFO | Uploaded 36GiB to GPU in 1.60074s (22.4895GiB/s) | 20260505 05:24:42.387 | INFO | Downloaded 34GiB to GPU in 1.51779s (22.401GiB/s) | 20260505 05:24:47.525 | INFO | Analyzing chunk frames = [676864,677376) | 20260505 05:24:47.525 | INFO | MemInfo: { | 20260505 05:24:47.525 | INFO | "gpu" : " 17 GB", | 20260505 05:24:47.525 | INFO | "huge" : "", | 20260505 05:24:47.525 | INFO | "huge_shared" : " 27 GB", | 20260505 05:24:47.525 | INFO | "malloc" : " 14 GB", | 20260505 05:24:47.526 | INFO | "pinned" : " 29 GB" | 20260505 05:24:47.526 | INFO | } | 20260505 05:24:47.526 | INFO | Perf report for the last 5.13933 seconds: | 20260505 05:24:47.526 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:24:47.526 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:47.526 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:47.526 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:47.526 | INFO | ANALYSIS : 57.2%, 3.0, 88.9ms, 0 | 20260505 05:24:47.526 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:47.526 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:47.526 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 05:24:47.526 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:47.526 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:47.526 | INFO | Uploaded 36GiB to GPU in 1.60457s (22.4359GiB/s) | 20260505 05:24:47.526 | INFO | Downloaded 34GiB to GPU in 1.52253s (22.3313GiB/s) | 20260505 05:24:52.665 | INFO | Analyzing chunk frames = [677376,677888) | 20260505 05:24:52.665 | INFO | MemInfo: { | 20260505 05:24:52.665 | INFO | "gpu" : " 17 GB", | 20260505 05:24:52.665 | INFO | "huge" : "", | 20260505 05:24:52.665 | INFO | "huge_shared" : " 27 GB", | 20260505 05:24:52.665 | INFO | "malloc" : " 14 GB", | 20260505 05:24:52.665 | INFO | "pinned" : " 29 GB" | 20260505 05:24:52.665 | INFO | } | 20260505 05:24:52.665 | INFO | Perf report for the last 5.1392 seconds: | 20260505 05:24:52.665 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:24:52.665 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:52.665 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:52.665 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:52.665 | INFO | ANALYSIS : 56.9%, 3.0, 88.2ms, 0 | 20260505 05:24:52.665 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:52.665 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:52.665 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 05:24:52.665 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:52.665 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:52.665 | INFO | Uploaded 36GiB to GPU in 1.60356s (22.4501GiB/s) | 20260505 05:24:52.665 | INFO | Downloaded 34GiB to GPU in 1.52056s (22.3602GiB/s) | 20260505 05:24:57.804 | INFO | Analyzing chunk frames = [677888,678400) | 20260505 05:24:57.804 | INFO | MemInfo: { | 20260505 05:24:57.804 | INFO | "gpu" : " 17 GB", | 20260505 05:24:57.804 | INFO | "huge" : "", | 20260505 05:24:57.804 | INFO | "huge_shared" : " 27 GB", | 20260505 05:24:57.804 | INFO | "malloc" : " 14 GB", | 20260505 05:24:57.804 | INFO | "pinned" : " 29 GB" | 20260505 05:24:57.804 | INFO | } | 20260505 05:24:57.804 | INFO | Perf report for the last 5.13928 seconds: | 20260505 05:24:57.804 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:24:57.804 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:57.804 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:57.804 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:57.804 | INFO | ANALYSIS : 56.8%, 3.0, 88.5ms, 0 | 20260505 05:24:57.804 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:57.804 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:57.804 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 05:24:57.804 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:57.804 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:24:57.804 | INFO | Uploaded 36GiB to GPU in 1.60265s (22.4627GiB/s) | 20260505 05:24:57.804 | INFO | Downloaded 34GiB to GPU in 1.51933s (22.3784GiB/s) | 20260505 05:25:02.943 | INFO | Analyzing chunk frames = [678400,678912) | 20260505 05:25:02.943 | INFO | MemInfo: { | 20260505 05:25:02.943 | INFO | "gpu" : " 17 GB", | 20260505 05:25:02.943 | INFO | "huge" : "", | 20260505 05:25:02.943 | INFO | "huge_shared" : " 27 GB", | 20260505 05:25:02.943 | INFO | "malloc" : " 14 GB", | 20260505 05:25:02.943 | INFO | "pinned" : " 29 GB" | 20260505 05:25:02.943 | INFO | } | 20260505 05:25:02.943 | INFO | Perf report for the last 5.13922 seconds: | 20260505 05:25:02.944 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:25:02.944 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:02.944 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:02.944 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:02.944 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260505 05:25:02.944 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:02.944 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:02.944 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 05:25:02.944 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:02.944 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:02.944 | INFO | Uploaded 36GiB to GPU in 1.60286s (22.4598GiB/s) | 20260505 05:25:02.944 | INFO | Downloaded 34GiB to GPU in 1.51892s (22.3844GiB/s) | 20260505 05:25:08.083 | INFO | Analyzing chunk frames = [678912,679424) | 20260505 05:25:08.083 | INFO | MemInfo: { | 20260505 05:25:08.083 | INFO | "gpu" : " 17 GB", | 20260505 05:25:08.083 | INFO | "huge" : "", | 20260505 05:25:08.083 | INFO | "huge_shared" : " 27 GB", | 20260505 05:25:08.083 | INFO | "malloc" : " 14 GB", | 20260505 05:25:08.083 | INFO | "pinned" : " 29 GB" | 20260505 05:25:08.083 | INFO | } | 20260505 05:25:08.083 | INFO | Perf report for the last 5.1395 seconds: | 20260505 05:25:08.083 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:25:08.083 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:08.083 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:08.083 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:08.083 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 05:25:08.083 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:08.083 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:08.083 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 05:25:08.083 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:08.083 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:08.083 | INFO | Uploaded 36GiB to GPU in 1.60298s (22.4582GiB/s) | 20260505 05:25:08.083 | INFO | Downloaded 34GiB to GPU in 1.52008s (22.3673GiB/s) | 20260505 05:25:13.222 | INFO | Analyzing chunk frames = [679424,679936) | 20260505 05:25:13.222 | INFO | MemInfo: { | 20260505 05:25:13.222 | INFO | "gpu" : " 17 GB", | 20260505 05:25:13.222 | INFO | "huge" : "", | 20260505 05:25:13.222 | INFO | "huge_shared" : " 27 GB", | 20260505 05:25:13.222 | INFO | "malloc" : " 14 GB", | 20260505 05:25:13.222 | INFO | "pinned" : " 29 GB" | 20260505 05:25:13.222 | INFO | } | 20260505 05:25:13.222 | INFO | Perf report for the last 5.13945 seconds: | 20260505 05:25:13.222 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:25:13.222 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:13.222 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:13.222 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:13.222 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260505 05:25:13.222 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:13.222 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:13.222 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:25:13.222 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:13.222 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:13.223 | INFO | Uploaded 36GiB to GPU in 1.60274s (22.4615GiB/s) | 20260505 05:25:13.223 | INFO | Downloaded 34GiB to GPU in 1.51929s (22.3788GiB/s) | 20260505 05:25:13.223 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 05:25:13.223 | INFO | Total mem usage: 15.1 GiB | 20260505 05:25:13.223 | INFO | All Time High Water Marks sum: 15.7 GiB (Savings: 4.09%) | 20260505 05:25:13.223 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -4.12%) | 20260505 05:25:13.223 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 05:25:13.223 | INFO | High Water Breakdown: | 20260505 05:25:13.223 | INFO | 59.9% : 8.66 GiB ( 95.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 05:25:13.223 | INFO | 40% : 5.78 GiB ( 87% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 05:25:13.223 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 05:25:13.223 | INFO | ------------------------------------------ | 20260505 05:25:13.223 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:25:13.223 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 05:25:13.223 | INFO | Total mem usage: 26.5 GiB | 20260505 05:25:13.223 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 05:25:13.223 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 05:25:13.223 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 05:25:13.223 | INFO | High Water Breakdown: | 20260505 05:25:13.223 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:25:13.223 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:25:13.223 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:25:13.223 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:25:13.223 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:25:13.223 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:25:13.223 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 05:25:13.223 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 05:25:13.223 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:25:13.223 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 05:25:13.223 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 05:25:13.223 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 05:25:13.223 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 05:25:13.223 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 05:25:13.223 | INFO | ------------------------------------------ | 20260505 05:25:13.223 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:25:13.223 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 05:25:13.223 | INFO | Total mem usage: 25.6 GiB | 20260505 05:25:13.223 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 05:25:13.223 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 05:25:13.223 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 05:25:13.223 | INFO | High Water Breakdown: | 20260505 05:25:13.223 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 05:25:13.223 | INFO | ------------------------------------------ | 20260505 05:25:13.223 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:25:13.223 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 05:25:13.223 | INFO | Total mem usage: 15.7 GiB | 20260505 05:25:13.223 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 05:25:13.223 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 05:25:13.223 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260505 05:25:13.223 | INFO | High Water Breakdown: | 20260505 05:25:13.223 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 05:25:13.223 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 05:25:13.223 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 05:25:13.223 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 05:25:13.223 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 05:25:13.223 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 05:25:13.223 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 05:25:13.223 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 05:25:13.223 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 05:25:13.223 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:25:13.223 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 05:25:13.223 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:25:13.223 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 05:25:13.223 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 05:25:13.223 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 05:25:13.223 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 05:25:13.223 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:25:13.223 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 05:25:13.223 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:25:13.223 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:25:13.223 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:25:13.223 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 05:25:13.223 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 05:25:13.224 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:25:13.224 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 05:25:13.224 | INFO | ------------------------------------------ | 20260505 05:25:13.224 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:25:13.224 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 05:25:13.224 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:25:13.224 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:25:13.224 | INFO | PerfReport: Full Execution -- 3.03e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260505 05:25:13.224 | INFO | PerfReport: 0.28%, AggregateStats -- 8.64e-02 / -nan, 4.67e-03 / 0.00e+00 | 20260505 05:25:13.224 | INFO | PerfReport: 10.87%, Baseline -- 3.30e+00 / -nan, 1.91e-02 / 0.00e+00 | 20260505 05:25:13.224 | INFO | PerfReport: 3.05%, Histogram -- 9.24e-01 / -nan, 5.20e-02 / 0.00e+00 | 20260505 05:25:13.224 | INFO | PerfReport: 16.04%, DME -- 3.89e+01 / -nan, 3.22e+00 / 0.00e+00 | 20260505 05:25:13.224 | INFO | PerfReport: 56.73%, FrameLabeling -- 1.72e+01 / -nan, 4.50e-02 / 0.00e+00 | 20260505 05:25:13.224 | INFO | PerfReport: 7.55%, PulseAccumulating -- 2.29e+00 / -nan, 1.50e-01 / 0.00e+00 | 20260505 05:25:13.224 | INFO | PerfReport: 3.88%, Metrics -- 1.18e+00 / -nan, 1.94e-01 / 0.00e+00 | 20260505 05:25:13.224 | INFO | PerfReport: 1.59%, PulseSerializing -- 4.83e-01 / -nan, 7.79e-02 / 0.00e+00 | 20260505 05:25:13.224 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 05:25:13.224 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:25:13.224 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:25:13.224 | INFO | PerfReport: Full Execution -- 9.10e+01 / -nan, 1.88e+01 / 0.00e+00 | 20260505 05:25:13.224 | INFO | PerfReport: 18.41%, Upload -- 1.68e+01 / 0.00e+00, 4.04e-01 / 0.00e+00 | 20260505 05:25:13.224 | INFO | PerfReport: 17.67%, Download -- 1.61e+01 / 0.00e+00, 7.34e-01 / 0.00e+00 | 20260505 05:25:13.224 | INFO | PerfReport: 33.34%, Compute -- 3.03e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260505 05:25:13.224 | INFO | PerfReport: 0.56%, LockUpload -- 5.06e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260505 05:25:13.224 | INFO | PerfReport: 0.00%, LockDownload -- 4.00e-04 / 0.00e+00, 1.45e-04 / 0.00e+00 | 20260505 05:25:13.224 | INFO | PerfReport: 30.01%, LockCompute -- 2.73e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260505 05:25:13.224 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.52e-02 / 0.00e+00 | 20260505 05:25:13.320 | INFO | Starting a round of Baz IO | 20260505 05:25:16.260 | INFO | Populating RTMetrics report | 20260505 05:25:16.260 | INFO | RTMetrics report populated | 20260505 05:25:16.260 | INFO | Opening tmp RTMetrics json file | 20260505 05:25:16.268 | INFO | Serializing RTMetrics to json | 20260505 05:25:16.269 | INFO | Serialization done, swapping tmp file now... | 20260505 05:25:16.272 | INFO | RTMetrics done | 20260505 05:25:16.367 | INFO | Finished a round of Baz IO: Wrote 2.231 GiB in 3.047 seconds | 20260505 05:25:16.367 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.05% / 0.08% / 60.87% | 20260505 05:25:16.367 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.57342% / 0.000234445% / 3.97364e-05% | 20260505 05:25:16.367 | INFO | Baz IO overall: | 20260505 05:25:16.367 | INFO | BytesWritten : 317274296923 | 20260505 05:25:16.367 | INFO | NumEvents : 148926545948 | 20260505 05:25:18.361 | INFO | Analyzing chunk frames = [679936,680448) | 20260505 05:25:18.361 | INFO | MemInfo: { | 20260505 05:25:18.361 | INFO | "gpu" : " 17 GB", | 20260505 05:25:18.361 | INFO | "huge" : "", | 20260505 05:25:18.361 | INFO | "huge_shared" : " 27 GB", | 20260505 05:25:18.361 | INFO | "malloc" : " 14 GB", | 20260505 05:25:18.361 | INFO | "pinned" : " 29 GB" | 20260505 05:25:18.361 | INFO | } | 20260505 05:25:18.361 | INFO | Perf report for the last 5.1391 seconds: | 20260505 05:25:18.362 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:25:18.362 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:18.362 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:18.362 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:18.362 | INFO | ANALYSIS : 58.0%, 2.9, 89.3ms, 0 | 20260505 05:25:18.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:18.362 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 05:25:18.362 | INFO | PRE_HQ : 57.4%, 2.2, 64.6ms, 0 | 20260505 05:25:18.362 | INFO | BAZWRITER : 58.7%, 4.4, 131.5ms, 0 | 20260505 05:25:18.362 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 05:25:18.362 | INFO | Uploaded 36GiB to GPU in 1.64548s (21.8781GiB/s) | 20260505 05:25:18.362 | INFO | Downloaded 38GiB to GPU in 1.70994s (22.223GiB/s) | 20260505 05:25:23.501 | INFO | Analyzing chunk frames = [680448,680960) | 20260505 05:25:23.501 | INFO | MemInfo: { | 20260505 05:25:23.501 | INFO | "gpu" : " 17 GB", | 20260505 05:25:23.501 | INFO | "huge" : "", | 20260505 05:25:23.501 | INFO | "huge_shared" : " 27 GB", | 20260505 05:25:23.501 | INFO | "malloc" : " 14 GB", | 20260505 05:25:23.501 | INFO | "pinned" : " 29 GB" | 20260505 05:25:23.501 | INFO | } | 20260505 05:25:23.501 | INFO | Perf report for the last 5.13954 seconds: | 20260505 05:25:23.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:25:23.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:23.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:23.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:23.501 | INFO | ANALYSIS : 57.8%, 3.0, 90.2ms, 0 | 20260505 05:25:23.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:23.501 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:23.501 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 05:25:23.501 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:23.501 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:23.501 | INFO | Uploaded 36GiB to GPU in 1.60015s (22.498GiB/s) | 20260505 05:25:23.501 | INFO | Downloaded 34GiB to GPU in 1.51714s (22.4106GiB/s) | 20260505 05:25:28.640 | INFO | Analyzing chunk frames = [680960,681472) | 20260505 05:25:28.640 | INFO | MemInfo: { | 20260505 05:25:28.640 | INFO | "gpu" : " 17 GB", | 20260505 05:25:28.640 | INFO | "huge" : "", | 20260505 05:25:28.640 | INFO | "huge_shared" : " 27 GB", | 20260505 05:25:28.640 | INFO | "malloc" : " 14 GB", | 20260505 05:25:28.640 | INFO | "pinned" : " 29 GB" | 20260505 05:25:28.640 | INFO | } | 20260505 05:25:28.640 | INFO | Perf report for the last 5.13928 seconds: | 20260505 05:25:28.640 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:25:28.640 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:28.640 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:28.640 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:28.640 | INFO | ANALYSIS : 57.1%, 3.0, 88.8ms, 0 | 20260505 05:25:28.640 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:28.640 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:28.640 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:25:28.640 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:28.640 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:28.641 | INFO | Uploaded 36GiB to GPU in 1.6047s (22.4342GiB/s) | 20260505 05:25:28.641 | INFO | Downloaded 34GiB to GPU in 1.52177s (22.3423GiB/s) | 20260505 05:25:33.779 | INFO | Analyzing chunk frames = [681472,681984) | 20260505 05:25:33.779 | INFO | MemInfo: { | 20260505 05:25:33.779 | INFO | "gpu" : " 17 GB", | 20260505 05:25:33.779 | INFO | "huge" : "", | 20260505 05:25:33.779 | INFO | "huge_shared" : " 27 GB", | 20260505 05:25:33.779 | INFO | "malloc" : " 14 GB", | 20260505 05:25:33.780 | INFO | "pinned" : " 29 GB" | 20260505 05:25:33.780 | INFO | } | 20260505 05:25:33.780 | INFO | Perf report for the last 5.13931 seconds: | 20260505 05:25:33.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:25:33.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:33.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:33.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:33.780 | INFO | ANALYSIS : 56.9%, 3.0, 88.2ms, 0 | 20260505 05:25:33.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:33.780 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:33.780 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 05:25:33.780 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:33.780 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:33.780 | INFO | Uploaded 36GiB to GPU in 1.6044s (22.4383GiB/s) | 20260505 05:25:33.780 | INFO | Downloaded 34GiB to GPU in 1.52057s (22.36GiB/s) | 20260505 05:25:38.919 | INFO | Analyzing chunk frames = [681984,682496) | 20260505 05:25:38.919 | INFO | MemInfo: { | 20260505 05:25:38.919 | INFO | "gpu" : " 17 GB", | 20260505 05:25:38.919 | INFO | "huge" : "", | 20260505 05:25:38.919 | INFO | "huge_shared" : " 27 GB", | 20260505 05:25:38.919 | INFO | "malloc" : " 14 GB", | 20260505 05:25:38.919 | INFO | "pinned" : " 29 GB" | 20260505 05:25:38.919 | INFO | } | 20260505 05:25:38.919 | INFO | Perf report for the last 5.13941 seconds: | 20260505 05:25:38.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:25:38.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:38.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:38.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:38.919 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260505 05:25:38.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:38.919 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:38.919 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:25:38.919 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:38.919 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:38.920 | INFO | Uploaded 36GiB to GPU in 1.60246s (22.4654GiB/s) | 20260505 05:25:38.920 | INFO | Downloaded 34GiB to GPU in 1.51908s (22.3819GiB/s) | 20260505 05:25:44.058 | INFO | Analyzing chunk frames = [682496,683008) | 20260505 05:25:44.058 | INFO | MemInfo: { | 20260505 05:25:44.058 | INFO | "gpu" : " 17 GB", | 20260505 05:25:44.058 | INFO | "huge" : "", | 20260505 05:25:44.058 | INFO | "huge_shared" : " 27 GB", | 20260505 05:25:44.059 | INFO | "malloc" : " 14 GB", | 20260505 05:25:44.059 | INFO | "pinned" : " 29 GB" | 20260505 05:25:44.059 | INFO | } | 20260505 05:25:44.059 | INFO | Perf report for the last 5.13961 seconds: | 20260505 05:25:44.059 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:25:44.059 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:44.059 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:44.059 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:44.059 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260505 05:25:44.059 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:44.059 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:44.059 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 05:25:44.059 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:44.059 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:44.059 | INFO | Uploaded 36GiB to GPU in 1.60253s (22.4645GiB/s) | 20260505 05:25:44.059 | INFO | Downloaded 34GiB to GPU in 1.51968s (22.3732GiB/s) | 20260505 05:25:49.197 | INFO | Analyzing chunk frames = [683008,683520) | 20260505 05:25:49.197 | INFO | MemInfo: { | 20260505 05:25:49.197 | INFO | "gpu" : " 17 GB", | 20260505 05:25:49.197 | INFO | "huge" : "", | 20260505 05:25:49.197 | INFO | "huge_shared" : " 27 GB", | 20260505 05:25:49.198 | INFO | "malloc" : " 14 GB", | 20260505 05:25:49.198 | INFO | "pinned" : " 29 GB" | 20260505 05:25:49.198 | INFO | } | 20260505 05:25:49.198 | INFO | Perf report for the last 5.13901 seconds: | 20260505 05:25:49.198 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:25:49.198 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:49.198 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:49.198 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:49.198 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 05:25:49.198 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:49.198 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:49.198 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260505 05:25:49.198 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:49.198 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:49.198 | INFO | Uploaded 36GiB to GPU in 1.60257s (22.4639GiB/s) | 20260505 05:25:49.198 | INFO | Downloaded 34GiB to GPU in 1.52012s (22.3666GiB/s) | 20260505 05:25:54.337 | INFO | Analyzing chunk frames = [683520,684032) | 20260505 05:25:54.337 | INFO | MemInfo: { | 20260505 05:25:54.337 | INFO | "gpu" : " 17 GB", | 20260505 05:25:54.337 | INFO | "huge" : "", | 20260505 05:25:54.337 | INFO | "huge_shared" : " 27 GB", | 20260505 05:25:54.337 | INFO | "malloc" : " 14 GB", | 20260505 05:25:54.337 | INFO | "pinned" : " 29 GB" | 20260505 05:25:54.337 | INFO | } | 20260505 05:25:54.337 | INFO | Perf report for the last 5.13936 seconds: | 20260505 05:25:54.337 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:25:54.337 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:54.337 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:54.337 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:54.337 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260505 05:25:54.337 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:54.337 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:54.337 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 05:25:54.337 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:54.337 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:54.337 | INFO | Uploaded 36GiB to GPU in 1.60279s (22.4608GiB/s) | 20260505 05:25:54.337 | INFO | Downloaded 34GiB to GPU in 1.5198s (22.3714GiB/s) | 20260505 05:25:54.434 | INFO | Starting a round of Baz IO | 20260505 05:25:57.376 | INFO | Populating RTMetrics report | 20260505 05:25:57.376 | INFO | RTMetrics report populated | 20260505 05:25:57.376 | INFO | Opening tmp RTMetrics json file | 20260505 05:25:57.383 | INFO | Serializing RTMetrics to json | 20260505 05:25:57.386 | INFO | Serialization done, swapping tmp file now... | 20260505 05:25:57.388 | INFO | RTMetrics done | 20260505 05:25:57.487 | INFO | Finished a round of Baz IO: Wrote 2.235 GiB in 3.052 seconds | 20260505 05:25:57.487 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.96% / 0.08% / 60.95% | 20260505 05:25:57.487 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.5649% / 0.000198682% / 1.98682e-05% | 20260505 05:25:57.487 | INFO | Baz IO overall: | 20260505 05:25:57.487 | INFO | BytesWritten : 319674619148 | 20260505 05:25:57.487 | INFO | NumEvents : 150071559290 | 20260505 05:25:59.476 | INFO | Analyzing chunk frames = [684032,684544) | 20260505 05:25:59.476 | INFO | MemInfo: { | 20260505 05:25:59.476 | INFO | "gpu" : " 17 GB", | 20260505 05:25:59.476 | INFO | "huge" : "", | 20260505 05:25:59.476 | INFO | "huge_shared" : " 27 GB", | 20260505 05:25:59.476 | INFO | "malloc" : " 14 GB", | 20260505 05:25:59.477 | INFO | "pinned" : " 29 GB" | 20260505 05:25:59.477 | INFO | } | 20260505 05:25:59.477 | INFO | Perf report for the last 5.13955 seconds: | 20260505 05:25:59.477 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:25:59.477 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:59.477 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:59.477 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:59.477 | INFO | ANALYSIS : 58.0%, 2.9, 89.3ms, 0 | 20260505 05:25:59.477 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:25:59.477 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260505 05:25:59.477 | INFO | PRE_HQ : 57.4%, 2.2, 64.1ms, 0 | 20260505 05:25:59.477 | INFO | BAZWRITER : 59.0%, 4.3, 131.4ms, 0 | 20260505 05:25:59.477 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 05:25:59.477 | INFO | Uploaded 36GiB to GPU in 1.65228s (21.7881GiB/s) | 20260505 05:25:59.477 | INFO | Downloaded 38GiB to GPU in 1.71641s (22.1393GiB/s) | 20260505 05:26:04.615 | INFO | Analyzing chunk frames = [684544,685056) | 20260505 05:26:04.616 | INFO | MemInfo: { | 20260505 05:26:04.616 | INFO | "gpu" : " 17 GB", | 20260505 05:26:04.616 | INFO | "huge" : "", | 20260505 05:26:04.616 | INFO | "huge_shared" : " 27 GB", | 20260505 05:26:04.616 | INFO | "malloc" : " 14 GB", | 20260505 05:26:04.616 | INFO | "pinned" : " 29 GB" | 20260505 05:26:04.616 | INFO | } | 20260505 05:26:04.616 | INFO | Perf report for the last 5.13914 seconds: | 20260505 05:26:04.616 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:26:04.616 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:04.616 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:04.616 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:04.616 | INFO | ANALYSIS : 57.8%, 3.0, 90.2ms, 0 | 20260505 05:26:04.616 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:04.616 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:04.616 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:26:04.616 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:04.616 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:04.617 | INFO | Uploaded 36GiB to GPU in 1.59735s (22.5373GiB/s) | 20260505 05:26:04.617 | INFO | Downloaded 34GiB to GPU in 1.51775s (22.4016GiB/s) | 20260505 05:26:09.755 | INFO | Analyzing chunk frames = [685056,685568) | 20260505 05:26:09.755 | INFO | MemInfo: { | 20260505 05:26:09.755 | INFO | "gpu" : " 17 GB", | 20260505 05:26:09.755 | INFO | "huge" : "", | 20260505 05:26:09.755 | INFO | "huge_shared" : " 27 GB", | 20260505 05:26:09.755 | INFO | "malloc" : " 14 GB", | 20260505 05:26:09.755 | INFO | "pinned" : " 29 GB" | 20260505 05:26:09.755 | INFO | } | 20260505 05:26:09.755 | INFO | Perf report for the last 5.13925 seconds: | 20260505 05:26:09.755 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:26:09.755 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:09.755 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:09.755 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:09.755 | INFO | ANALYSIS : 57.1%, 3.0, 88.8ms, 0 | 20260505 05:26:09.755 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:09.755 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:09.755 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260505 05:26:09.755 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:09.755 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:09.755 | INFO | Uploaded 36GiB to GPU in 1.60189s (22.4734GiB/s) | 20260505 05:26:09.755 | INFO | Downloaded 34GiB to GPU in 1.5225s (22.3317GiB/s) | 20260505 05:26:14.894 | INFO | Analyzing chunk frames = [685568,686080) | 20260505 05:26:14.894 | INFO | MemInfo: { | 20260505 05:26:14.894 | INFO | "gpu" : " 17 GB", | 20260505 05:26:14.894 | INFO | "huge" : "", | 20260505 05:26:14.894 | INFO | "huge_shared" : " 27 GB", | 20260505 05:26:14.894 | INFO | "malloc" : " 14 GB", | 20260505 05:26:14.894 | INFO | "pinned" : " 29 GB" | 20260505 05:26:14.894 | INFO | } | 20260505 05:26:14.894 | INFO | Perf report for the last 5.13943 seconds: | 20260505 05:26:14.894 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:26:14.894 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:14.894 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:14.894 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:14.895 | INFO | ANALYSIS : 56.9%, 3.0, 88.2ms, 0 | 20260505 05:26:14.895 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:14.895 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:14.895 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260505 05:26:14.895 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:14.895 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:14.895 | INFO | Uploaded 36GiB to GPU in 1.60099s (22.486GiB/s) | 20260505 05:26:14.895 | INFO | Downloaded 34GiB to GPU in 1.52096s (22.3544GiB/s) | 20260505 05:26:20.033 | INFO | Analyzing chunk frames = [686080,686592) | 20260505 05:26:20.033 | INFO | MemInfo: { | 20260505 05:26:20.033 | INFO | "gpu" : " 17 GB", | 20260505 05:26:20.033 | INFO | "huge" : "", | 20260505 05:26:20.033 | INFO | "huge_shared" : " 27 GB", | 20260505 05:26:20.033 | INFO | "malloc" : " 14 GB", | 20260505 05:26:20.033 | INFO | "pinned" : " 29 GB" | 20260505 05:26:20.034 | INFO | } | 20260505 05:26:20.034 | INFO | Perf report for the last 5.13911 seconds: | 20260505 05:26:20.034 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:26:20.034 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:20.034 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:20.034 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:20.034 | INFO | ANALYSIS : 56.8%, 3.0, 88.5ms, 0 | 20260505 05:26:20.034 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:20.034 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:20.034 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 05:26:20.034 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:20.034 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:20.034 | INFO | Uploaded 36GiB to GPU in 1.59937s (22.5088GiB/s) | 20260505 05:26:20.034 | INFO | Downloaded 34GiB to GPU in 1.51924s (22.3796GiB/s) | 20260505 05:26:25.173 | INFO | Analyzing chunk frames = [686592,687104) | 20260505 05:26:25.173 | INFO | MemInfo: { | 20260505 05:26:25.173 | INFO | "gpu" : " 17 GB", | 20260505 05:26:25.173 | INFO | "huge" : "", | 20260505 05:26:25.173 | INFO | "huge_shared" : " 27 GB", | 20260505 05:26:25.173 | INFO | "malloc" : " 14 GB", | 20260505 05:26:25.173 | INFO | "pinned" : " 29 GB" | 20260505 05:26:25.173 | INFO | } | 20260505 05:26:25.173 | INFO | Perf report for the last 5.13957 seconds: | 20260505 05:26:25.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:26:25.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:25.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:25.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:25.173 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260505 05:26:25.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:25.173 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:25.173 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260505 05:26:25.173 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:25.173 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:25.174 | INFO | Uploaded 36GiB to GPU in 1.59896s (22.5147GiB/s) | 20260505 05:26:25.174 | INFO | Downloaded 34GiB to GPU in 1.51899s (22.3832GiB/s) | 20260505 05:26:30.312 | INFO | Analyzing chunk frames = [687104,687616) | 20260505 05:26:30.312 | INFO | MemInfo: { | 20260505 05:26:30.312 | INFO | "gpu" : " 17 GB", | 20260505 05:26:30.312 | INFO | "huge" : "", | 20260505 05:26:30.312 | INFO | "huge_shared" : " 27 GB", | 20260505 05:26:30.312 | INFO | "malloc" : " 14 GB", | 20260505 05:26:30.312 | INFO | "pinned" : " 29 GB" | 20260505 05:26:30.312 | INFO | } | 20260505 05:26:30.312 | INFO | Perf report for the last 5.13915 seconds: | 20260505 05:26:30.312 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:26:30.312 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:30.312 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:30.312 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:30.312 | INFO | ANALYSIS : 57.3%, 3.0, 89.3ms, 0 | 20260505 05:26:30.312 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:30.312 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:30.312 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 05:26:30.312 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:30.313 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:30.313 | INFO | Uploaded 36GiB to GPU in 1.5994s (22.5085GiB/s) | 20260505 05:26:30.313 | INFO | Downloaded 34GiB to GPU in 1.51996s (22.369GiB/s) | 20260505 05:26:35.452 | INFO | Analyzing chunk frames = [687616,688128) | 20260505 05:26:35.452 | INFO | MemInfo: { | 20260505 05:26:35.452 | INFO | "gpu" : " 17 GB", | 20260505 05:26:35.452 | INFO | "huge" : "", | 20260505 05:26:35.452 | INFO | "huge_shared" : " 27 GB", | 20260505 05:26:35.452 | INFO | "malloc" : " 14 GB", | 20260505 05:26:35.452 | INFO | "pinned" : " 29 GB" | 20260505 05:26:35.452 | INFO | } | 20260505 05:26:35.452 | INFO | Perf report for the last 5.13928 seconds: | 20260505 05:26:35.452 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:26:35.452 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:35.452 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:35.452 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:35.452 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260505 05:26:35.452 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:35.452 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:35.452 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:26:35.452 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:35.452 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:35.452 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 05:26:35.452 | INFO | Total mem usage: 15.1 GiB | 20260505 05:26:35.452 | INFO | All Time High Water Marks sum: 15.7 GiB (Savings: 4.09%) | 20260505 05:26:35.452 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -3.81%) | 20260505 05:26:35.452 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 05:26:35.452 | INFO | High Water Breakdown: | 20260505 05:26:35.452 | INFO | 60% : 8.7 GiB ( 96.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 05:26:35.452 | INFO | 39.9% : 5.78 GiB ( 87% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 05:26:35.452 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 05:26:35.452 | INFO | ------------------------------------------ | 20260505 05:26:35.452 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:26:35.452 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 05:26:35.452 | INFO | Total mem usage: 26.5 GiB | 20260505 05:26:35.452 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 05:26:35.452 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 05:26:35.452 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 05:26:35.452 | INFO | High Water Breakdown: | 20260505 05:26:35.452 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:26:35.452 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:26:35.452 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:26:35.452 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:26:35.452 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:26:35.452 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:26:35.452 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 05:26:35.452 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 05:26:35.452 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:26:35.452 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 05:26:35.452 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 05:26:35.452 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 05:26:35.452 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 05:26:35.452 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 05:26:35.452 | INFO | ------------------------------------------ | 20260505 05:26:35.452 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:26:35.452 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 05:26:35.452 | INFO | Total mem usage: 25.6 GiB | 20260505 05:26:35.452 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 05:26:35.452 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 05:26:35.452 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 05:26:35.452 | INFO | High Water Breakdown: | 20260505 05:26:35.452 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 05:26:35.452 | INFO | ------------------------------------------ | 20260505 05:26:35.452 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:26:35.452 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 05:26:35.452 | INFO | Total mem usage: 15.7 GiB | 20260505 05:26:35.452 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 05:26:35.452 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 05:26:35.452 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260505 05:26:35.452 | INFO | High Water Breakdown: | 20260505 05:26:35.452 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 05:26:35.452 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 05:26:35.452 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 05:26:35.452 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 05:26:35.452 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 05:26:35.452 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 05:26:35.452 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 05:26:35.452 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 05:26:35.452 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 05:26:35.452 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:26:35.452 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 05:26:35.452 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:26:35.452 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 05:26:35.452 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 05:26:35.452 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 05:26:35.452 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 05:26:35.452 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:26:35.452 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 05:26:35.452 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:26:35.452 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:26:35.452 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:26:35.452 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 05:26:35.452 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 05:26:35.452 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:26:35.452 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 05:26:35.452 | INFO | ------------------------------------------ | 20260505 05:26:35.452 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:26:35.452 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 05:26:35.452 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:26:35.452 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:26:35.452 | INFO | PerfReport: Full Execution -- 3.03e+01 / -nan, 1.25e+01 / 0.00e+00 | 20260505 05:26:35.452 | INFO | PerfReport: 0.29%, AggregateStats -- 8.65e-02 / -nan, 4.17e-03 / 0.00e+00 | 20260505 05:26:35.452 | INFO | PerfReport: 10.87%, Baseline -- 3.30e+00 / -nan, 1.66e-02 / 0.00e+00 | 20260505 05:26:35.452 | INFO | PerfReport: 3.05%, Histogram -- 9.24e-01 / -nan, 5.11e-02 / 0.00e+00 | 20260505 05:26:35.452 | INFO | PerfReport: 16.04%, DME -- 3.89e+01 / -nan, 3.21e+00 / 0.00e+00 | 20260505 05:26:35.452 | INFO | PerfReport: 56.74%, FrameLabeling -- 1.72e+01 / -nan, 4.59e-02 / 0.00e+00 | 20260505 05:26:35.452 | INFO | PerfReport: 7.54%, PulseAccumulating -- 2.29e+00 / -nan, 1.50e-01 / 0.00e+00 | 20260505 05:26:35.452 | INFO | PerfReport: 3.88%, Metrics -- 1.18e+00 / -nan, 1.94e-01 / 0.00e+00 | 20260505 05:26:35.452 | INFO | PerfReport: 1.59%, PulseSerializing -- 4.82e-01 / -nan, 7.79e-02 / 0.00e+00 | 20260505 05:26:35.452 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 05:26:35.452 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:26:35.452 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:26:35.452 | INFO | PerfReport: Full Execution -- 9.10e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260505 05:26:35.452 | INFO | PerfReport: 18.40%, Upload -- 1.67e+01 / 0.00e+00, 4.09e-01 / 0.00e+00 | 20260505 05:26:35.452 | INFO | PerfReport: 17.68%, Download -- 1.61e+01 / 0.00e+00, 7.42e-01 / 0.00e+00 | 20260505 05:26:35.452 | INFO | PerfReport: 33.34%, Compute -- 3.03e+01 / 0.00e+00, 1.25e+01 / 0.00e+00 | 20260505 05:26:35.452 | INFO | PerfReport: 0.56%, LockUpload -- 5.07e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260505 05:26:35.452 | INFO | PerfReport: 0.00%, LockDownload -- 3.87e-04 / 0.00e+00, 9.21e-05 / 0.00e+00 | 20260505 05:26:35.452 | INFO | PerfReport: 30.00%, LockCompute -- 2.73e+01 / 0.00e+00, 1.67e+01 / 0.00e+00 | 20260505 05:26:35.452 | INFO | PerfReport: 0.02%, Misc -- 1.37e-02 / 0.00e+00, 1.68e-02 / 0.00e+00 | 20260505 05:26:35.453 | INFO | Uploaded 36GiB to GPU in 1.59911s (22.5126GiB/s) | 20260505 05:26:35.453 | INFO | Downloaded 34GiB to GPU in 1.52089s (22.3553GiB/s) | 20260505 05:26:35.549 | INFO | Starting a round of Baz IO | 20260505 05:26:38.493 | INFO | Populating RTMetrics report | 20260505 05:26:38.493 | INFO | RTMetrics report populated | 20260505 05:26:38.493 | INFO | Opening tmp RTMetrics json file | 20260505 05:26:38.500 | INFO | Serializing RTMetrics to json | 20260505 05:26:38.503 | INFO | Serialization done, swapping tmp file now... | 20260505 05:26:38.505 | INFO | RTMetrics done | 20260505 05:26:38.600 | INFO | Finished a round of Baz IO: Wrote 2.236 GiB in 3.05 seconds | 20260505 05:26:38.600 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.88% / 0.08% / 61.04% | 20260505 05:26:38.600 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.5209% / 0.000123183% / 3.97364e-05% | 20260505 05:26:38.600 | INFO | Baz IO overall: | 20260505 05:26:38.600 | INFO | BytesWritten : 322074991851 | 20260505 05:26:38.600 | INFO | NumEvents : 151216409878 | 20260505 05:26:40.591 | INFO | Analyzing chunk frames = [688128,688640) | 20260505 05:26:40.591 | INFO | MemInfo: { | 20260505 05:26:40.591 | INFO | "gpu" : " 17 GB", | 20260505 05:26:40.591 | INFO | "huge" : "", | 20260505 05:26:40.591 | INFO | "huge_shared" : " 27 GB", | 20260505 05:26:40.591 | INFO | "malloc" : " 14 GB", | 20260505 05:26:40.591 | INFO | "pinned" : " 29 GB" | 20260505 05:26:40.591 | INFO | } | 20260505 05:26:40.591 | INFO | Perf report for the last 5.13947 seconds: | 20260505 05:26:40.591 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:26:40.591 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:40.591 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:40.591 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:40.591 | INFO | ANALYSIS : 58.0%, 2.9, 89.5ms, 0 | 20260505 05:26:40.591 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:40.591 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260505 05:26:40.591 | INFO | PRE_HQ : 57.3%, 2.1, 63.9ms, 0 | 20260505 05:26:40.591 | INFO | BAZWRITER : 58.9%, 4.3, 131.1ms, 0 | 20260505 05:26:40.591 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 05:26:40.592 | INFO | Uploaded 36GiB to GPU in 1.64624s (21.868GiB/s) | 20260505 05:26:40.592 | INFO | Downloaded 38GiB to GPU in 1.71035s (22.2176GiB/s) | 20260505 05:26:45.730 | INFO | Analyzing chunk frames = [688640,689152) | 20260505 05:26:45.730 | INFO | MemInfo: { | 20260505 05:26:45.730 | INFO | "gpu" : " 17 GB", | 20260505 05:26:45.730 | INFO | "huge" : "", | 20260505 05:26:45.730 | INFO | "huge_shared" : " 27 GB", | 20260505 05:26:45.730 | INFO | "malloc" : " 14 GB", | 20260505 05:26:45.730 | INFO | "pinned" : " 29 GB" | 20260505 05:26:45.730 | INFO | } | 20260505 05:26:45.730 | INFO | Perf report for the last 5.13918 seconds: | 20260505 05:26:45.730 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:26:45.730 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:45.730 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:45.730 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:45.730 | INFO | ANALYSIS : 57.8%, 3.0, 90.1ms, 0 | 20260505 05:26:45.731 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:45.731 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:45.731 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 05:26:45.731 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:45.731 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:45.731 | INFO | Uploaded 36GiB to GPU in 1.59742s (22.5363GiB/s) | 20260505 05:26:45.731 | INFO | Downloaded 34GiB to GPU in 1.51702s (22.4123GiB/s) | 20260505 05:26:50.870 | INFO | Analyzing chunk frames = [689152,689664) | 20260505 05:26:50.870 | INFO | MemInfo: { | 20260505 05:26:50.870 | INFO | "gpu" : " 17 GB", | 20260505 05:26:50.870 | INFO | "huge" : "", | 20260505 05:26:50.870 | INFO | "huge_shared" : " 27 GB", | 20260505 05:26:50.870 | INFO | "malloc" : " 14 GB", | 20260505 05:26:50.870 | INFO | "pinned" : " 29 GB" | 20260505 05:26:50.870 | INFO | } | 20260505 05:26:50.870 | INFO | Perf report for the last 5.13963 seconds: | 20260505 05:26:50.870 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:26:50.870 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:50.870 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:50.870 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:50.870 | INFO | ANALYSIS : 57.2%, 3.0, 88.8ms, 0 | 20260505 05:26:50.870 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:50.870 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:50.870 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 05:26:50.870 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:50.870 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:50.870 | INFO | Uploaded 36GiB to GPU in 1.60153s (22.4785GiB/s) | 20260505 05:26:50.870 | INFO | Downloaded 34GiB to GPU in 1.52182s (22.3417GiB/s) | 20260505 05:26:56.009 | INFO | Analyzing chunk frames = [689664,690176) | 20260505 05:26:56.009 | INFO | MemInfo: { | 20260505 05:26:56.009 | INFO | "gpu" : " 17 GB", | 20260505 05:26:56.009 | INFO | "huge" : "", | 20260505 05:26:56.009 | INFO | "huge_shared" : " 27 GB", | 20260505 05:26:56.009 | INFO | "malloc" : " 14 GB", | 20260505 05:26:56.009 | INFO | "pinned" : " 29 GB" | 20260505 05:26:56.009 | INFO | } | 20260505 05:26:56.009 | INFO | Perf report for the last 5.13926 seconds: | 20260505 05:26:56.009 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:26:56.009 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:56.009 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:56.009 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:56.009 | INFO | ANALYSIS : 56.8%, 3.0, 88.1ms, 0 | 20260505 05:26:56.009 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:56.009 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:56.009 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260505 05:26:56.009 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:56.009 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:26:56.010 | INFO | Uploaded 36GiB to GPU in 1.60108s (22.4848GiB/s) | 20260505 05:26:56.010 | INFO | Downloaded 34GiB to GPU in 1.52172s (22.3431GiB/s) | 20260505 05:27:01.148 | INFO | Analyzing chunk frames = [690176,690688) | 20260505 05:27:01.148 | INFO | MemInfo: { | 20260505 05:27:01.148 | INFO | "gpu" : " 17 GB", | 20260505 05:27:01.148 | INFO | "huge" : "", | 20260505 05:27:01.148 | INFO | "huge_shared" : " 27 GB", | 20260505 05:27:01.148 | INFO | "malloc" : " 14 GB", | 20260505 05:27:01.148 | INFO | "pinned" : " 29 GB" | 20260505 05:27:01.148 | INFO | } | 20260505 05:27:01.148 | INFO | Perf report for the last 5.13919 seconds: | 20260505 05:27:01.148 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:27:01.148 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:01.148 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:01.148 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:01.148 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260505 05:27:01.148 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:01.148 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:01.148 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:27:01.149 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:01.149 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:01.149 | INFO | Uploaded 36GiB to GPU in 1.59925s (22.5106GiB/s) | 20260505 05:27:01.149 | INFO | Downloaded 34GiB to GPU in 1.51918s (22.3805GiB/s) | 20260505 05:27:06.288 | INFO | Analyzing chunk frames = [690688,691200) | 20260505 05:27:06.288 | INFO | MemInfo: { | 20260505 05:27:06.288 | INFO | "gpu" : " 17 GB", | 20260505 05:27:06.288 | INFO | "huge" : "", | 20260505 05:27:06.288 | INFO | "huge_shared" : " 27 GB", | 20260505 05:27:06.288 | INFO | "malloc" : " 14 GB", | 20260505 05:27:06.288 | INFO | "pinned" : " 29 GB" | 20260505 05:27:06.288 | INFO | } | 20260505 05:27:06.288 | INFO | Perf report for the last 5.13943 seconds: | 20260505 05:27:06.288 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:27:06.288 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:06.288 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:06.288 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:06.288 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260505 05:27:06.288 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:06.288 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:06.288 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 05:27:06.288 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:06.288 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:06.288 | INFO | Uploaded 36GiB to GPU in 1.5995s (22.507GiB/s) | 20260505 05:27:06.288 | INFO | Downloaded 34GiB to GPU in 1.51974s (22.3722GiB/s) | 20260505 05:27:11.427 | INFO | Analyzing chunk frames = [691200,691712) | 20260505 05:27:11.427 | INFO | MemInfo: { | 20260505 05:27:11.427 | INFO | "gpu" : " 17 GB", | 20260505 05:27:11.427 | INFO | "huge" : "", | 20260505 05:27:11.427 | INFO | "huge_shared" : " 27 GB", | 20260505 05:27:11.427 | INFO | "malloc" : " 14 GB", | 20260505 05:27:11.427 | INFO | "pinned" : " 29 GB" | 20260505 05:27:11.427 | INFO | } | 20260505 05:27:11.427 | INFO | Perf report for the last 5.13915 seconds: | 20260505 05:27:11.427 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:27:11.427 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:11.427 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:11.427 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:11.427 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 05:27:11.427 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:11.427 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:11.427 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:27:11.427 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:11.427 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:11.427 | INFO | Uploaded 36GiB to GPU in 1.59956s (22.5062GiB/s) | 20260505 05:27:11.427 | INFO | Downloaded 34GiB to GPU in 1.52029s (22.3641GiB/s) | 20260505 05:27:16.566 | INFO | Analyzing chunk frames = [691712,692224) | 20260505 05:27:16.566 | INFO | MemInfo: { | 20260505 05:27:16.566 | INFO | "gpu" : " 17 GB", | 20260505 05:27:16.566 | INFO | "huge" : "", | 20260505 05:27:16.566 | INFO | "huge_shared" : " 27 GB", | 20260505 05:27:16.566 | INFO | "malloc" : " 14 GB", | 20260505 05:27:16.566 | INFO | "pinned" : " 29 GB" | 20260505 05:27:16.566 | INFO | } | 20260505 05:27:16.567 | INFO | Perf report for the last 5.13952 seconds: | 20260505 05:27:16.567 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:27:16.567 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:16.567 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:16.567 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:16.567 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260505 05:27:16.567 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:16.567 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:16.567 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260505 05:27:16.567 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:16.567 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:16.567 | INFO | Uploaded 36GiB to GPU in 1.60063s (22.4911GiB/s) | 20260505 05:27:16.567 | INFO | Downloaded 34GiB to GPU in 1.52243s (22.3327GiB/s) | 20260505 05:27:16.664 | INFO | Starting a round of Baz IO | 20260505 05:27:19.610 | INFO | Populating RTMetrics report | 20260505 05:27:19.610 | INFO | RTMetrics report populated | 20260505 05:27:19.610 | INFO | Opening tmp RTMetrics json file | 20260505 05:27:19.616 | INFO | Serializing RTMetrics to json | 20260505 05:27:19.618 | INFO | Serialization done, swapping tmp file now... | 20260505 05:27:19.621 | INFO | RTMetrics done | 20260505 05:27:19.718 | INFO | Finished a round of Baz IO: Wrote 2.238 GiB in 3.054 seconds | 20260505 05:27:19.718 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.80% / 0.08% / 61.12% | 20260505 05:27:19.718 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.47667% / 0.000150998% / 3.57628e-05% | 20260505 05:27:19.718 | INFO | Baz IO overall: | 20260505 05:27:19.718 | INFO | BytesWritten : 324477708010 | 20260505 05:27:19.719 | INFO | NumEvents : 152362321988 | 20260505 05:27:21.706 | INFO | Analyzing chunk frames = [692224,692736) | 20260505 05:27:21.706 | INFO | MemInfo: { | 20260505 05:27:21.706 | INFO | "gpu" : " 17 GB", | 20260505 05:27:21.706 | INFO | "huge" : "", | 20260505 05:27:21.706 | INFO | "huge_shared" : " 27 GB", | 20260505 05:27:21.706 | INFO | "malloc" : " 14 GB", | 20260505 05:27:21.706 | INFO | "pinned" : " 29 GB" | 20260505 05:27:21.706 | INFO | } | 20260505 05:27:21.706 | INFO | Perf report for the last 5.13928 seconds: | 20260505 05:27:21.706 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:27:21.706 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:21.706 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:21.706 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:21.706 | INFO | ANALYSIS : 58.1%, 2.9, 88.9ms, 0 | 20260505 05:27:21.706 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:21.706 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260505 05:27:21.706 | INFO | PRE_HQ : 57.4%, 2.2, 64.6ms, 0 | 20260505 05:27:21.706 | INFO | BAZWRITER : 59.0%, 4.3, 131.8ms, 0 | 20260505 05:27:21.706 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 05:27:21.706 | INFO | Uploaded 36GiB to GPU in 1.64669s (21.8621GiB/s) | 20260505 05:27:21.706 | INFO | Downloaded 38GiB to GPU in 1.71216s (22.1942GiB/s) | 20260505 05:27:26.845 | INFO | Analyzing chunk frames = [692736,693248) | 20260505 05:27:26.845 | INFO | MemInfo: { | 20260505 05:27:26.845 | INFO | "gpu" : " 17 GB", | 20260505 05:27:26.845 | INFO | "huge" : "", | 20260505 05:27:26.845 | INFO | "huge_shared" : " 27 GB", | 20260505 05:27:26.845 | INFO | "malloc" : " 14 GB", | 20260505 05:27:26.845 | INFO | "pinned" : " 29 GB" | 20260505 05:27:26.845 | INFO | } | 20260505 05:27:26.845 | INFO | Perf report for the last 5.13928 seconds: | 20260505 05:27:26.845 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:27:26.845 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:26.845 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:26.845 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:26.845 | INFO | ANALYSIS : 57.8%, 3.0, 90.1ms, 0 | 20260505 05:27:26.845 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:26.845 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:26.845 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 05:27:26.845 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:26.845 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:26.845 | INFO | Uploaded 36GiB to GPU in 1.59772s (22.5321GiB/s) | 20260505 05:27:26.845 | INFO | Downloaded 34GiB to GPU in 1.5174s (22.4068GiB/s) | 20260505 05:27:31.984 | INFO | Analyzing chunk frames = [693248,693760) | 20260505 05:27:31.984 | INFO | MemInfo: { | 20260505 05:27:31.984 | INFO | "gpu" : " 17 GB", | 20260505 05:27:31.984 | INFO | "huge" : "", | 20260505 05:27:31.984 | INFO | "huge_shared" : " 27 GB", | 20260505 05:27:31.984 | INFO | "malloc" : " 14 GB", | 20260505 05:27:31.985 | INFO | "pinned" : " 29 GB" | 20260505 05:27:31.985 | INFO | } | 20260505 05:27:31.985 | INFO | Perf report for the last 5.13944 seconds: | 20260505 05:27:31.985 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:27:31.985 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:31.985 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:31.985 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:31.985 | INFO | ANALYSIS : 57.2%, 3.0, 88.9ms, 0 | 20260505 05:27:31.985 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:31.985 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:31.985 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 05:27:31.985 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:31.985 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:31.985 | INFO | Uploaded 36GiB to GPU in 1.60172s (22.4759GiB/s) | 20260505 05:27:31.985 | INFO | Downloaded 34GiB to GPU in 1.5228s (22.3272GiB/s) | 20260505 05:27:37.124 | INFO | Analyzing chunk frames = [693760,694272) | 20260505 05:27:37.124 | INFO | MemInfo: { | 20260505 05:27:37.124 | INFO | "gpu" : " 17 GB", | 20260505 05:27:37.124 | INFO | "huge" : "", | 20260505 05:27:37.124 | INFO | "huge_shared" : " 27 GB", | 20260505 05:27:37.124 | INFO | "malloc" : " 14 GB", | 20260505 05:27:37.124 | INFO | "pinned" : " 29 GB" | 20260505 05:27:37.124 | INFO | } | 20260505 05:27:37.124 | INFO | Perf report for the last 5.13931 seconds: | 20260505 05:27:37.124 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:27:37.124 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:37.124 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:37.124 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:37.124 | INFO | ANALYSIS : 56.8%, 3.0, 88.2ms, 0 | 20260505 05:27:37.124 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:37.124 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:37.124 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 05:27:37.124 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:37.124 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:37.124 | INFO | Uploaded 36GiB to GPU in 1.6006s (22.4916GiB/s) | 20260505 05:27:37.125 | INFO | Downloaded 34GiB to GPU in 1.5217s (22.3435GiB/s) | 20260505 05:27:42.263 | INFO | Analyzing chunk frames = [694272,694784) | 20260505 05:27:42.263 | INFO | MemInfo: { | 20260505 05:27:42.263 | INFO | "gpu" : " 17 GB", | 20260505 05:27:42.263 | INFO | "huge" : "", | 20260505 05:27:42.263 | INFO | "huge_shared" : " 27 GB", | 20260505 05:27:42.263 | INFO | "malloc" : " 14 GB", | 20260505 05:27:42.263 | INFO | "pinned" : " 29 GB" | 20260505 05:27:42.263 | INFO | } | 20260505 05:27:42.263 | INFO | Perf report for the last 5.13926 seconds: | 20260505 05:27:42.263 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:27:42.263 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:42.263 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:42.263 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:42.263 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260505 05:27:42.263 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:42.263 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:42.263 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 05:27:42.263 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:42.263 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:42.264 | INFO | Uploaded 36GiB to GPU in 1.59914s (22.5121GiB/s) | 20260505 05:27:42.264 | INFO | Downloaded 34GiB to GPU in 1.51897s (22.3836GiB/s) | 20260505 05:27:47.402 | INFO | Analyzing chunk frames = [694784,695296) | 20260505 05:27:47.402 | INFO | MemInfo: { | 20260505 05:27:47.402 | INFO | "gpu" : " 17 GB", | 20260505 05:27:47.402 | INFO | "huge" : "", | 20260505 05:27:47.402 | INFO | "huge_shared" : " 27 GB", | 20260505 05:27:47.402 | INFO | "malloc" : " 14 GB", | 20260505 05:27:47.403 | INFO | "pinned" : " 29 GB" | 20260505 05:27:47.403 | INFO | } | 20260505 05:27:47.403 | INFO | Perf report for the last 5.13939 seconds: | 20260505 05:27:47.403 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:27:47.403 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:47.403 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:47.403 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:47.403 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260505 05:27:47.403 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:47.403 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:47.403 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 05:27:47.403 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:47.403 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:47.403 | INFO | Uploaded 36GiB to GPU in 1.60023s (22.4967GiB/s) | 20260505 05:27:47.403 | INFO | Downloaded 34GiB to GPU in 1.5211s (22.3522GiB/s) | 20260505 05:27:52.542 | INFO | Analyzing chunk frames = [695296,695808) | 20260505 05:27:52.542 | INFO | MemInfo: { | 20260505 05:27:52.542 | INFO | "gpu" : " 17 GB", | 20260505 05:27:52.542 | INFO | "huge" : "", | 20260505 05:27:52.542 | INFO | "huge_shared" : " 27 GB", | 20260505 05:27:52.542 | INFO | "malloc" : " 14 GB", | 20260505 05:27:52.542 | INFO | "pinned" : " 29 GB" | 20260505 05:27:52.542 | INFO | } | 20260505 05:27:52.542 | INFO | Perf report for the last 5.13937 seconds: | 20260505 05:27:52.542 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:27:52.542 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:52.542 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:52.542 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:52.542 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 05:27:52.542 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:52.542 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:52.542 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260505 05:27:52.542 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:52.542 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:52.542 | INFO | Uploaded 36GiB to GPU in 1.59881s (22.5167GiB/s) | 20260505 05:27:52.542 | INFO | Downloaded 34GiB to GPU in 1.52056s (22.3602GiB/s) | 20260505 05:27:57.681 | INFO | Analyzing chunk frames = [695808,696320) | 20260505 05:27:57.681 | INFO | MemInfo: { | 20260505 05:27:57.681 | INFO | "gpu" : " 17 GB", | 20260505 05:27:57.681 | INFO | "huge" : "", | 20260505 05:27:57.681 | INFO | "huge_shared" : " 27 GB", | 20260505 05:27:57.681 | INFO | "malloc" : " 14 GB", | 20260505 05:27:57.681 | INFO | "pinned" : " 29 GB" | 20260505 05:27:57.681 | INFO | } | 20260505 05:27:57.681 | INFO | Perf report for the last 5.13928 seconds: | 20260505 05:27:57.681 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:27:57.681 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:57.681 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:57.681 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:57.681 | INFO | ANALYSIS : 57.3%, 3.0, 89.5ms, 0 | 20260505 05:27:57.681 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:57.682 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:57.682 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260505 05:27:57.682 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:57.682 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:27:57.682 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 05:27:57.682 | INFO | Total mem usage: 15.1 GiB | 20260505 05:27:57.682 | INFO | All Time High Water Marks sum: 15.7 GiB (Savings: 4.09%) | 20260505 05:27:57.682 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -3.91%) | 20260505 05:27:57.682 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 05:27:57.682 | INFO | High Water Breakdown: | 20260505 05:27:57.682 | INFO | 60% : 8.69 GiB ( 96.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 05:27:57.682 | INFO | 39.9% : 5.78 GiB ( 86.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 05:27:57.682 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 05:27:57.682 | INFO | ------------------------------------------ | 20260505 05:27:57.682 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:27:57.682 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 05:27:57.682 | INFO | Total mem usage: 26.5 GiB | 20260505 05:27:57.682 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 05:27:57.682 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 05:27:57.682 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 05:27:57.682 | INFO | High Water Breakdown: | 20260505 05:27:57.682 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:27:57.682 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:27:57.682 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:27:57.682 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:27:57.682 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:27:57.682 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:27:57.682 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 05:27:57.682 | INFO | Uploaded 36GiB to GPU in 1.60026s (22.4963GiB/s) | 20260505 05:27:57.682 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 05:27:57.682 | INFO | Downloaded 34GiB to GPU in 1.52092s (22.3549GiB/s) | 20260505 05:27:57.682 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:27:57.682 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 05:27:57.682 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 05:27:57.682 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 05:27:57.682 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 05:27:57.682 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 05:27:57.682 | INFO | ------------------------------------------ | 20260505 05:27:57.682 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:27:57.682 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 05:27:57.682 | INFO | Total mem usage: 25.6 GiB | 20260505 05:27:57.682 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 05:27:57.682 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 05:27:57.682 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 05:27:57.682 | INFO | High Water Breakdown: | 20260505 05:27:57.682 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 05:27:57.683 | INFO | ------------------------------------------ | 20260505 05:27:57.683 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:27:57.683 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 05:27:57.683 | INFO | Total mem usage: 15.7 GiB | 20260505 05:27:57.683 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 05:27:57.683 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 05:27:57.683 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260505 05:27:57.683 | INFO | High Water Breakdown: | 20260505 05:27:57.683 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 05:27:57.683 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 05:27:57.683 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 05:27:57.683 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 05:27:57.683 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 05:27:57.683 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 05:27:57.683 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 05:27:57.683 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 05:27:57.683 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 05:27:57.683 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:27:57.683 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 05:27:57.683 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:27:57.683 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 05:27:57.683 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 05:27:57.683 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 05:27:57.683 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 05:27:57.683 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:27:57.683 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 05:27:57.683 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:27:57.683 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:27:57.683 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:27:57.683 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 05:27:57.683 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 05:27:57.683 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:27:57.683 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 05:27:57.683 | INFO | ------------------------------------------ | 20260505 05:27:57.683 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:27:57.683 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 05:27:57.683 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:27:57.683 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:27:57.683 | INFO | PerfReport: Full Execution -- 3.03e+01 / -nan, 1.28e+01 / 0.00e+00 | 20260505 05:27:57.683 | INFO | PerfReport: 0.29%, AggregateStats -- 8.65e-02 / -nan, 6.25e-03 / 0.00e+00 | 20260505 05:27:57.683 | INFO | PerfReport: 10.87%, Baseline -- 3.30e+00 / -nan, 1.93e-02 / 0.00e+00 | 20260505 05:27:57.683 | INFO | PerfReport: 3.05%, Histogram -- 9.24e-01 / -nan, 5.25e-02 / 0.00e+00 | 20260505 05:27:57.683 | INFO | PerfReport: 16.05%, DME -- 3.89e+01 / -nan, 3.18e+00 / 0.00e+00 | 20260505 05:27:57.683 | INFO | PerfReport: 56.75%, FrameLabeling -- 1.72e+01 / -nan, 7.47e-02 / 0.00e+00 | 20260505 05:27:57.683 | INFO | PerfReport: 7.54%, PulseAccumulating -- 2.29e+00 / -nan, 1.50e-01 / 0.00e+00 | 20260505 05:27:57.683 | INFO | PerfReport: 3.87%, Metrics -- 1.17e+00 / -nan, 1.94e-01 / 0.00e+00 | 20260505 05:27:57.683 | INFO | PerfReport: 1.59%, PulseSerializing -- 4.81e-01 / -nan, 7.80e-02 / 0.00e+00 | 20260505 05:27:57.683 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 05:27:57.683 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:27:57.683 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:27:57.683 | INFO | PerfReport: Full Execution -- 9.10e+01 / -nan, 1.89e+01 / 0.00e+00 | 20260505 05:27:57.683 | INFO | PerfReport: 18.39%, Upload -- 1.67e+01 / 0.00e+00, 3.97e-01 / 0.00e+00 | 20260505 05:27:57.683 | INFO | PerfReport: 17.69%, Download -- 1.61e+01 / 0.00e+00, 7.36e-01 / 0.00e+00 | 20260505 05:27:57.683 | INFO | PerfReport: 33.35%, Compute -- 3.03e+01 / 0.00e+00, 1.28e+01 / 0.00e+00 | 20260505 05:27:57.683 | INFO | PerfReport: 0.56%, LockUpload -- 5.06e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260505 05:27:57.683 | INFO | PerfReport: 0.00%, LockDownload -- 3.92e-04 / 0.00e+00, 1.89e-04 / 0.00e+00 | 20260505 05:27:57.683 | INFO | PerfReport: 30.00%, LockCompute -- 2.73e+01 / 0.00e+00, 1.72e+01 / 0.00e+00 | 20260505 05:27:57.683 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 2.42e-02 / 0.00e+00 | 20260505 05:27:57.779 | INFO | Starting a round of Baz IO | 20260505 05:28:00.722 | INFO | Populating RTMetrics report | 20260505 05:28:00.722 | INFO | RTMetrics report populated | 20260505 05:28:00.722 | INFO | Opening tmp RTMetrics json file | 20260505 05:28:00.732 | INFO | Serializing RTMetrics to json | 20260505 05:28:00.734 | INFO | Serialization done, swapping tmp file now... | 20260505 05:28:00.735 | INFO | RTMetrics done | 20260505 05:28:00.832 | INFO | Finished a round of Baz IO: Wrote 2.241 GiB in 3.052 seconds | 20260505 05:28:00.832 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.72% / 0.08% / 61.20% | 20260505 05:28:00.832 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.42915% / 0.000162919% / 3.57628e-05% | 20260505 05:28:00.832 | INFO | Baz IO overall: | 20260505 05:28:00.832 | INFO | BytesWritten : 326883997970 | 20260505 05:28:00.832 | INFO | NumEvents : 153509087689 | 20260505 05:28:02.820 | INFO | Analyzing chunk frames = [696320,696832) | 20260505 05:28:02.820 | INFO | MemInfo: { | 20260505 05:28:02.820 | INFO | "gpu" : " 17 GB", | 20260505 05:28:02.820 | INFO | "huge" : "", | 20260505 05:28:02.820 | INFO | "huge_shared" : " 27 GB", | 20260505 05:28:02.820 | INFO | "malloc" : " 14 GB", | 20260505 05:28:02.821 | INFO | "pinned" : " 29 GB" | 20260505 05:28:02.821 | INFO | } | 20260505 05:28:02.821 | INFO | Perf report for the last 5.13938 seconds: | 20260505 05:28:02.821 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:28:02.821 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:02.821 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:02.821 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:02.821 | INFO | ANALYSIS : 58.0%, 2.9, 89.0ms, 0 | 20260505 05:28:02.821 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:02.821 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260505 05:28:02.821 | INFO | PRE_HQ : 57.4%, 2.2, 64.2ms, 0 | 20260505 05:28:02.821 | INFO | BAZWRITER : 58.9%, 4.4, 132.8ms, 0 | 20260505 05:28:02.821 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 05:28:02.821 | INFO | Uploaded 36GiB to GPU in 1.65043s (21.8125GiB/s) | 20260505 05:28:02.821 | INFO | Downloaded 38GiB to GPU in 1.71388s (22.172GiB/s) | 20260505 05:28:07.960 | INFO | Analyzing chunk frames = [696832,697344) | 20260505 05:28:07.960 | INFO | MemInfo: { | 20260505 05:28:07.960 | INFO | "gpu" : " 17 GB", | 20260505 05:28:07.960 | INFO | "huge" : "", | 20260505 05:28:07.960 | INFO | "huge_shared" : " 27 GB", | 20260505 05:28:07.960 | INFO | "malloc" : " 14 GB", | 20260505 05:28:07.960 | INFO | "pinned" : " 29 GB" | 20260505 05:28:07.960 | INFO | } | 20260505 05:28:07.960 | INFO | Perf report for the last 5.13937 seconds: | 20260505 05:28:07.960 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:28:07.960 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:07.960 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:07.960 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:07.960 | INFO | ANALYSIS : 57.8%, 3.0, 90.1ms, 0 | 20260505 05:28:07.960 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:07.960 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:07.960 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:28:07.960 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:07.960 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:07.961 | INFO | Uploaded 36GiB to GPU in 1.59769s (22.5325GiB/s) | 20260505 05:28:07.961 | INFO | Downloaded 34GiB to GPU in 1.51713s (22.4107GiB/s) | 20260505 05:28:13.099 | INFO | Analyzing chunk frames = [697344,697856) | 20260505 05:28:13.099 | INFO | MemInfo: { | 20260505 05:28:13.099 | INFO | "gpu" : " 17 GB", | 20260505 05:28:13.099 | INFO | "huge" : "", | 20260505 05:28:13.099 | INFO | "huge_shared" : " 27 GB", | 20260505 05:28:13.099 | INFO | "malloc" : " 14 GB", | 20260505 05:28:13.099 | INFO | "pinned" : " 29 GB" | 20260505 05:28:13.099 | INFO | } | 20260505 05:28:13.099 | INFO | Perf report for the last 5.1393 seconds: | 20260505 05:28:13.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:28:13.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:13.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:13.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:13.099 | INFO | ANALYSIS : 57.2%, 3.0, 88.8ms, 0 | 20260505 05:28:13.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:13.099 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:13.099 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260505 05:28:13.099 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:13.099 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:13.100 | INFO | Uploaded 36GiB to GPU in 1.60148s (22.4792GiB/s) | 20260505 05:28:13.100 | INFO | Downloaded 34GiB to GPU in 1.52214s (22.337GiB/s) | 20260505 05:28:18.238 | INFO | Analyzing chunk frames = [697856,698368) | 20260505 05:28:18.238 | INFO | MemInfo: { | 20260505 05:28:18.238 | INFO | "gpu" : " 17 GB", | 20260505 05:28:18.238 | INFO | "huge" : "", | 20260505 05:28:18.238 | INFO | "huge_shared" : " 27 GB", | 20260505 05:28:18.238 | INFO | "malloc" : " 14 GB", | 20260505 05:28:18.238 | INFO | "pinned" : " 29 GB" | 20260505 05:28:18.238 | INFO | } | 20260505 05:28:18.239 | INFO | Perf report for the last 5.1392 seconds: | 20260505 05:28:18.239 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:28:18.239 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:18.239 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:18.239 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:18.239 | INFO | ANALYSIS : 56.9%, 3.0, 88.2ms, 0 | 20260505 05:28:18.239 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:18.239 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:18.239 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 05:28:18.239 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:18.239 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:18.239 | INFO | Uploaded 36GiB to GPU in 1.60155s (22.4783GiB/s) | 20260505 05:28:18.239 | INFO | Downloaded 34GiB to GPU in 1.5217s (22.3434GiB/s) | 20260505 05:28:23.378 | INFO | Analyzing chunk frames = [698368,698880) | 20260505 05:28:23.378 | INFO | MemInfo: { | 20260505 05:28:23.378 | INFO | "gpu" : " 17 GB", | 20260505 05:28:23.378 | INFO | "huge" : "", | 20260505 05:28:23.378 | INFO | "huge_shared" : " 27 GB", | 20260505 05:28:23.378 | INFO | "malloc" : " 14 GB", | 20260505 05:28:23.378 | INFO | "pinned" : " 29 GB" | 20260505 05:28:23.378 | INFO | } | 20260505 05:28:23.378 | INFO | Perf report for the last 5.13949 seconds: | 20260505 05:28:23.378 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:28:23.378 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:23.378 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:23.378 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:23.378 | INFO | ANALYSIS : 56.8%, 3.0, 88.5ms, 0 | 20260505 05:28:23.378 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:23.378 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:23.378 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:28:23.378 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:23.378 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:23.379 | INFO | Uploaded 36GiB to GPU in 1.59995s (22.5007GiB/s) | 20260505 05:28:23.379 | INFO | Downloaded 34GiB to GPU in 1.52048s (22.3614GiB/s) | 20260505 05:28:28.517 | INFO | Analyzing chunk frames = [698880,699392) | 20260505 05:28:28.517 | INFO | MemInfo: { | 20260505 05:28:28.517 | INFO | "gpu" : " 17 GB", | 20260505 05:28:28.517 | INFO | "huge" : "", | 20260505 05:28:28.517 | INFO | "huge_shared" : " 27 GB", | 20260505 05:28:28.517 | INFO | "malloc" : " 14 GB", | 20260505 05:28:28.517 | INFO | "pinned" : " 29 GB" | 20260505 05:28:28.517 | INFO | } | 20260505 05:28:28.517 | INFO | Perf report for the last 5.13938 seconds: | 20260505 05:28:28.517 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:28:28.517 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:28.517 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:28.517 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:28.517 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260505 05:28:28.518 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:28.518 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:28.518 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 05:28:28.518 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:28.518 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:28.518 | INFO | Uploaded 36GiB to GPU in 1.5997s (22.5043GiB/s) | 20260505 05:28:28.518 | INFO | Downloaded 34GiB to GPU in 1.51953s (22.3754GiB/s) | 20260505 05:28:33.656 | INFO | Analyzing chunk frames = [699392,699904) | 20260505 05:28:33.656 | INFO | MemInfo: { | 20260505 05:28:33.656 | INFO | "gpu" : " 17 GB", | 20260505 05:28:33.657 | INFO | "huge" : "", | 20260505 05:28:33.657 | INFO | "huge_shared" : " 27 GB", | 20260505 05:28:33.657 | INFO | "malloc" : " 14 GB", | 20260505 05:28:33.657 | INFO | "pinned" : " 29 GB" | 20260505 05:28:33.657 | INFO | } | 20260505 05:28:33.657 | INFO | Perf report for the last 5.13931 seconds: | 20260505 05:28:33.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:28:33.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:33.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:33.657 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:33.657 | INFO | ANALYSIS : 57.3%, 3.0, 89.3ms, 0 | 20260505 05:28:33.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:33.657 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:33.657 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:28:33.657 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:33.657 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:33.657 | INFO | Uploaded 36GiB to GPU in 1.60022s (22.4969GiB/s) | 20260505 05:28:33.657 | INFO | Downloaded 34GiB to GPU in 1.52111s (22.3521GiB/s) | 20260505 05:28:38.796 | INFO | Analyzing chunk frames = [699904,700416) | 20260505 05:28:38.796 | INFO | MemInfo: { | 20260505 05:28:38.796 | INFO | "gpu" : " 17 GB", | 20260505 05:28:38.796 | INFO | "huge" : "", | 20260505 05:28:38.796 | INFO | "huge_shared" : " 27 GB", | 20260505 05:28:38.796 | INFO | "malloc" : " 14 GB", | 20260505 05:28:38.796 | INFO | "pinned" : " 29 GB" | 20260505 05:28:38.796 | INFO | } | 20260505 05:28:38.796 | INFO | Perf report for the last 5.13913 seconds: | 20260505 05:28:38.796 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:28:38.796 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:38.796 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:38.796 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:38.796 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260505 05:28:38.796 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:38.796 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:38.796 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:28:38.796 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:38.796 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:38.796 | INFO | Uploaded 36GiB to GPU in 1.60023s (22.4968GiB/s) | 20260505 05:28:38.796 | INFO | Downloaded 34GiB to GPU in 1.5207s (22.3581GiB/s) | 20260505 05:28:38.893 | INFO | Starting a round of Baz IO | 20260505 05:28:41.834 | INFO | Populating RTMetrics report | 20260505 05:28:41.834 | INFO | RTMetrics report populated | 20260505 05:28:41.834 | INFO | Opening tmp RTMetrics json file | 20260505 05:28:41.841 | INFO | Serializing RTMetrics to json | 20260505 05:28:41.843 | INFO | Serialization done, swapping tmp file now... | 20260505 05:28:41.843 | INFO | RTMetrics done | 20260505 05:28:41.941 | INFO | Finished a round of Baz IO: Wrote 2.24 GiB in 3.048 seconds | 20260505 05:28:41.941 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.64% / 0.08% / 61.28% | 20260505 05:28:41.941 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.38528% / 0.00013113% / 5.16574e-05% | 20260505 05:28:41.941 | INFO | Baz IO overall: | 20260505 05:28:41.941 | INFO | BytesWritten : 329288947753 | 20260505 05:28:41.941 | INFO | NumEvents : 154654939937 | 20260505 05:28:43.935 | INFO | Analyzing chunk frames = [700416,700928) | 20260505 05:28:43.935 | INFO | MemInfo: { | 20260505 05:28:43.935 | INFO | "gpu" : " 17 GB", | 20260505 05:28:43.935 | INFO | "huge" : "", | 20260505 05:28:43.935 | INFO | "huge_shared" : " 27 GB", | 20260505 05:28:43.936 | INFO | "malloc" : " 14 GB", | 20260505 05:28:43.936 | INFO | "pinned" : " 29 GB" | 20260505 05:28:43.936 | INFO | } | 20260505 05:28:43.936 | INFO | Perf report for the last 5.13975 seconds: | 20260505 05:28:43.936 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:28:43.936 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:43.936 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:43.936 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:43.936 | INFO | ANALYSIS : 58.0%, 2.9, 89.1ms, 0 | 20260505 05:28:43.936 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:43.936 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260505 05:28:43.936 | INFO | PRE_HQ : 57.2%, 2.1, 63.7ms, 0 | 20260505 05:28:43.936 | INFO | BAZWRITER : 58.9%, 4.3, 131.2ms, 0 | 20260505 05:28:43.936 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 05:28:43.936 | INFO | Uploaded 36GiB to GPU in 1.65074s (21.8084GiB/s) | 20260505 05:28:43.936 | INFO | Downloaded 38GiB to GPU in 1.71584s (22.1466GiB/s) | 20260505 05:28:49.075 | INFO | Analyzing chunk frames = [700928,701440) | 20260505 05:28:49.075 | INFO | MemInfo: { | 20260505 05:28:49.075 | INFO | "gpu" : " 17 GB", | 20260505 05:28:49.075 | INFO | "huge" : "", | 20260505 05:28:49.075 | INFO | "huge_shared" : " 27 GB", | 20260505 05:28:49.075 | INFO | "malloc" : " 14 GB", | 20260505 05:28:49.075 | INFO | "pinned" : " 29 GB" | 20260505 05:28:49.075 | INFO | } | 20260505 05:28:49.075 | INFO | Perf report for the last 5.13917 seconds: | 20260505 05:28:49.075 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:28:49.075 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:49.075 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:49.075 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:49.075 | INFO | ANALYSIS : 57.8%, 3.0, 90.1ms, 0 | 20260505 05:28:49.075 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:49.075 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:49.075 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:28:49.075 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:49.075 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:49.075 | INFO | Uploaded 36GiB to GPU in 1.60031s (22.4957GiB/s) | 20260505 05:28:49.075 | INFO | Downloaded 34GiB to GPU in 1.51706s (22.4118GiB/s) | 20260505 05:28:54.214 | INFO | Analyzing chunk frames = [701440,701952) | 20260505 05:28:54.214 | INFO | MemInfo: { | 20260505 05:28:54.214 | INFO | "gpu" : " 17 GB", | 20260505 05:28:54.214 | INFO | "huge" : "", | 20260505 05:28:54.214 | INFO | "huge_shared" : " 27 GB", | 20260505 05:28:54.214 | INFO | "malloc" : " 14 GB", | 20260505 05:28:54.214 | INFO | "pinned" : " 29 GB" | 20260505 05:28:54.214 | INFO | } | 20260505 05:28:54.214 | INFO | Perf report for the last 5.13932 seconds: | 20260505 05:28:54.214 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:28:54.214 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:54.214 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:54.214 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:54.214 | INFO | ANALYSIS : 57.1%, 3.0, 88.8ms, 0 | 20260505 05:28:54.214 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:54.214 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:54.214 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260505 05:28:54.214 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:54.214 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:54.214 | INFO | Uploaded 36GiB to GPU in 1.60444s (22.4378GiB/s) | 20260505 05:28:54.214 | INFO | Downloaded 34GiB to GPU in 1.5224s (22.3331GiB/s) | 20260505 05:28:59.354 | INFO | Analyzing chunk frames = [701952,702464) | 20260505 05:28:59.354 | INFO | MemInfo: { | 20260505 05:28:59.354 | INFO | "gpu" : " 17 GB", | 20260505 05:28:59.354 | INFO | "huge" : "", | 20260505 05:28:59.354 | INFO | "huge_shared" : " 27 GB", | 20260505 05:28:59.354 | INFO | "malloc" : " 14 GB", | 20260505 05:28:59.354 | INFO | "pinned" : " 29 GB" | 20260505 05:28:59.354 | INFO | } | 20260505 05:28:59.354 | INFO | Perf report for the last 5.13965 seconds: | 20260505 05:28:59.354 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:28:59.354 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:59.354 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:59.354 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:59.354 | INFO | ANALYSIS : 56.8%, 3.0, 88.1ms, 0 | 20260505 05:28:59.354 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:59.354 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:59.354 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:28:59.354 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:59.354 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:28:59.354 | INFO | Uploaded 36GiB to GPU in 1.60379s (22.4468GiB/s) | 20260505 05:28:59.354 | INFO | Downloaded 34GiB to GPU in 1.52131s (22.3491GiB/s) | 20260505 05:29:04.493 | INFO | Analyzing chunk frames = [702464,702976) | 20260505 05:29:04.493 | INFO | MemInfo: { | 20260505 05:29:04.493 | INFO | "gpu" : " 17 GB", | 20260505 05:29:04.493 | INFO | "huge" : "", | 20260505 05:29:04.493 | INFO | "huge_shared" : " 27 GB", | 20260505 05:29:04.493 | INFO | "malloc" : " 14 GB", | 20260505 05:29:04.493 | INFO | "pinned" : " 29 GB" | 20260505 05:29:04.493 | INFO | } | 20260505 05:29:04.493 | INFO | Perf report for the last 5.13895 seconds: | 20260505 05:29:04.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:29:04.493 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:04.493 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:04.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:04.493 | INFO | ANALYSIS : 56.8%, 3.0, 88.5ms, 0 | 20260505 05:29:04.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:04.493 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:04.493 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260505 05:29:04.493 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:04.493 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:04.493 | INFO | Uploaded 36GiB to GPU in 1.60247s (22.4654GiB/s) | 20260505 05:29:04.493 | INFO | Downloaded 34GiB to GPU in 1.51894s (22.3841GiB/s) | 20260505 05:29:09.632 | INFO | Analyzing chunk frames = [702976,703488) | 20260505 05:29:09.632 | INFO | MemInfo: { | 20260505 05:29:09.632 | INFO | "gpu" : " 17 GB", | 20260505 05:29:09.632 | INFO | "huge" : "", | 20260505 05:29:09.632 | INFO | "huge_shared" : " 27 GB", | 20260505 05:29:09.632 | INFO | "malloc" : " 14 GB", | 20260505 05:29:09.632 | INFO | "pinned" : " 29 GB" | 20260505 05:29:09.632 | INFO | } | 20260505 05:29:09.632 | INFO | Perf report for the last 5.1394 seconds: | 20260505 05:29:09.632 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:29:09.632 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:09.632 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:09.632 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:09.632 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260505 05:29:09.632 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:09.632 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:09.632 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 05:29:09.632 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:09.632 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:09.633 | INFO | Uploaded 36GiB to GPU in 1.60217s (22.4696GiB/s) | 20260505 05:29:09.633 | INFO | Downloaded 34GiB to GPU in 1.51841s (22.3919GiB/s) | 20260505 05:29:14.771 | INFO | Analyzing chunk frames = [703488,704000) | 20260505 05:29:14.771 | INFO | MemInfo: { | 20260505 05:29:14.771 | INFO | "gpu" : " 17 GB", | 20260505 05:29:14.771 | INFO | "huge" : "", | 20260505 05:29:14.771 | INFO | "huge_shared" : " 27 GB", | 20260505 05:29:14.771 | INFO | "malloc" : " 14 GB", | 20260505 05:29:14.771 | INFO | "pinned" : " 29 GB" | 20260505 05:29:14.771 | INFO | } | 20260505 05:29:14.772 | INFO | Perf report for the last 5.13933 seconds: | 20260505 05:29:14.772 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:29:14.772 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:14.772 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:14.772 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:14.772 | INFO | ANALYSIS : 57.2%, 3.0, 89.3ms, 0 | 20260505 05:29:14.772 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:14.772 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:14.772 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 05:29:14.772 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:14.772 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:14.772 | INFO | Uploaded 36GiB to GPU in 1.60292s (22.459GiB/s) | 20260505 05:29:14.772 | INFO | Downloaded 34GiB to GPU in 1.52022s (22.3652GiB/s) | 20260505 05:29:19.911 | INFO | Analyzing chunk frames = [704000,704512) | 20260505 05:29:19.911 | INFO | MemInfo: { | 20260505 05:29:19.911 | INFO | "gpu" : " 17 GB", | 20260505 05:29:19.911 | INFO | "huge" : "", | 20260505 05:29:19.911 | INFO | "huge_shared" : " 27 GB", | 20260505 05:29:19.911 | INFO | "malloc" : " 14 GB", | 20260505 05:29:19.911 | INFO | "pinned" : " 29 GB" | 20260505 05:29:19.911 | INFO | } | 20260505 05:29:19.911 | INFO | Perf report for the last 5.13928 seconds: | 20260505 05:29:19.911 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:29:19.911 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:19.911 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:19.911 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:19.911 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260505 05:29:19.911 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:19.911 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:19.911 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 05:29:19.911 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:19.911 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:19.911 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 05:29:19.911 | INFO | Total mem usage: 15.1 GiB | 20260505 05:29:19.911 | INFO | All Time High Water Marks sum: 15.7 GiB (Savings: 4.09%) | 20260505 05:29:19.911 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -4.01%) | 20260505 05:29:19.911 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 05:29:19.911 | INFO | High Water Breakdown: | 20260505 05:29:19.911 | INFO | 60% : 8.68 GiB ( 96.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 05:29:19.911 | INFO | 39.9% : 5.78 GiB ( 86.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 05:29:19.911 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 05:29:19.911 | INFO | ------------------------------------------ | 20260505 05:29:19.911 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:29:19.911 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 05:29:19.911 | INFO | Total mem usage: 26.5 GiB | 20260505 05:29:19.911 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 05:29:19.911 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 05:29:19.911 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 05:29:19.911 | INFO | High Water Breakdown: | 20260505 05:29:19.911 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:29:19.911 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:29:19.911 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:29:19.911 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:29:19.911 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:29:19.911 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:29:19.911 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 05:29:19.911 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 05:29:19.911 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:29:19.911 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 05:29:19.911 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 05:29:19.911 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 05:29:19.911 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 05:29:19.911 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 05:29:19.911 | INFO | ------------------------------------------ | 20260505 05:29:19.911 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:29:19.911 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 05:29:19.911 | INFO | Total mem usage: 25.6 GiB | 20260505 05:29:19.911 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 05:29:19.911 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 05:29:19.911 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 05:29:19.911 | INFO | High Water Breakdown: | 20260505 05:29:19.911 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 05:29:19.911 | INFO | ------------------------------------------ | 20260505 05:29:19.911 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:29:19.911 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 05:29:19.911 | INFO | Total mem usage: 15.7 GiB | 20260505 05:29:19.911 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 05:29:19.911 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 05:29:19.911 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260505 05:29:19.911 | INFO | High Water Breakdown: | 20260505 05:29:19.911 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 05:29:19.911 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 05:29:19.911 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 05:29:19.911 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 05:29:19.911 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 05:29:19.911 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 05:29:19.911 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 05:29:19.911 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 05:29:19.911 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 05:29:19.911 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:29:19.911 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 05:29:19.911 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:29:19.911 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 05:29:19.911 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 05:29:19.911 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 05:29:19.911 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 05:29:19.911 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:29:19.911 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 05:29:19.911 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:29:19.911 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:29:19.911 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:29:19.911 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 05:29:19.911 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 05:29:19.911 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:29:19.911 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 05:29:19.911 | INFO | ------------------------------------------ | 20260505 05:29:19.911 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:29:19.911 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 05:29:19.911 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:29:19.911 | INFO | Uploaded 36GiB to GPU in 1.60244s (22.4657GiB/s) | 20260505 05:29:19.911 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:29:19.912 | INFO | PerfReport: Full Execution -- 3.03e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260505 05:29:19.912 | INFO | PerfReport: 0.29%, AggregateStats -- 8.65e-02 / -nan, 4.26e-03 / 0.00e+00 | 20260505 05:29:19.912 | INFO | Downloaded 34GiB to GPU in 1.51945s (22.3766GiB/s) | 20260505 05:29:19.912 | INFO | PerfReport: 10.88%, Baseline -- 3.30e+00 / -nan, 1.72e-02 / 0.00e+00 | 20260505 05:29:19.912 | INFO | PerfReport: 3.05%, Histogram -- 9.24e-01 / -nan, 5.20e-02 / 0.00e+00 | 20260505 05:29:19.912 | INFO | PerfReport: 16.04%, DME -- 3.89e+01 / -nan, 3.17e+00 / 0.00e+00 | 20260505 05:29:19.912 | INFO | PerfReport: 56.76%, FrameLabeling -- 1.72e+01 / -nan, 4.54e-02 / 0.00e+00 | 20260505 05:29:19.912 | INFO | PerfReport: 7.54%, PulseAccumulating -- 2.28e+00 / -nan, 1.49e-01 / 0.00e+00 | 20260505 05:29:19.912 | INFO | PerfReport: 3.87%, Metrics -- 1.17e+00 / -nan, 1.93e-01 / 0.00e+00 | 20260505 05:29:19.912 | INFO | PerfReport: 1.58%, PulseSerializing -- 4.80e-01 / -nan, 7.79e-02 / 0.00e+00 | 20260505 05:29:19.912 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 05:29:19.912 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:29:19.912 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:29:19.912 | INFO | PerfReport: Full Execution -- 9.09e+01 / -nan, 1.88e+01 / 0.00e+00 | 20260505 05:29:19.912 | INFO | PerfReport: 18.42%, Upload -- 1.67e+01 / 0.00e+00, 4.04e-01 / 0.00e+00 | 20260505 05:29:19.912 | INFO | PerfReport: 17.70%, Download -- 1.61e+01 / 0.00e+00, 7.45e-01 / 0.00e+00 | 20260505 05:29:19.912 | INFO | PerfReport: 33.35%, Compute -- 3.03e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260505 05:29:19.912 | INFO | PerfReport: 0.56%, LockUpload -- 5.06e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260505 05:29:19.912 | INFO | PerfReport: 0.00%, LockDownload -- 3.76e-04 / 0.00e+00, 9.13e-05 / 0.00e+00 | 20260505 05:29:19.912 | INFO | PerfReport: 29.96%, LockCompute -- 2.72e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260505 05:29:19.912 | INFO | PerfReport: 0.02%, Misc -- 1.38e-02 / 0.00e+00, 1.88e-02 / 0.00e+00 | 20260505 05:29:20.009 | INFO | Starting a round of Baz IO | 20260505 05:29:22.948 | INFO | Populating RTMetrics report | 20260505 05:29:22.948 | INFO | RTMetrics report populated | 20260505 05:29:22.948 | INFO | Opening tmp RTMetrics json file | 20260505 05:29:22.959 | INFO | Serializing RTMetrics to json | 20260505 05:29:22.961 | INFO | Serialization done, swapping tmp file now... | 20260505 05:29:22.961 | INFO | RTMetrics done | 20260505 05:29:23.049 | INFO | Finished a round of Baz IO: Wrote 2.243 GiB in 3.039 seconds | 20260505 05:29:23.049 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.56% / 0.08% / 61.36% | 20260505 05:29:23.049 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.36201% / 0.000127157% / 3.17891e-05% | 20260505 05:29:23.049 | INFO | Baz IO overall: | 20260505 05:29:23.049 | INFO | BytesWritten : 331697700283 | 20260505 05:29:23.049 | INFO | NumEvents : 155803151596 | 20260505 05:29:25.050 | INFO | Analyzing chunk frames = [704512,705024) | 20260505 05:29:25.050 | INFO | MemInfo: { | 20260505 05:29:25.050 | INFO | "gpu" : " 17 GB", | 20260505 05:29:25.050 | INFO | "huge" : "", | 20260505 05:29:25.050 | INFO | "huge_shared" : " 27 GB", | 20260505 05:29:25.050 | INFO | "malloc" : " 14 GB", | 20260505 05:29:25.050 | INFO | "pinned" : " 29 GB" | 20260505 05:29:25.050 | INFO | } | 20260505 05:29:25.050 | INFO | Perf report for the last 5.13923 seconds: | 20260505 05:29:25.050 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:29:25.050 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:25.050 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:25.050 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:25.050 | INFO | ANALYSIS : 58.0%, 2.9, 88.9ms, 0 | 20260505 05:29:25.050 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:25.050 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260505 05:29:25.050 | INFO | PRE_HQ : 57.3%, 2.1, 63.8ms, 0 | 20260505 05:29:25.050 | INFO | BAZWRITER : 58.7%, 4.4, 131.7ms, 0 | 20260505 05:29:25.050 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 05:29:25.051 | INFO | Uploaded 36GiB to GPU in 1.6549s (21.7536GiB/s) | 20260505 05:29:25.051 | INFO | Downloaded 38GiB to GPU in 1.72005s (22.0924GiB/s) | 20260505 05:29:30.189 | INFO | Analyzing chunk frames = [705024,705536) | 20260505 05:29:30.189 | INFO | MemInfo: { | 20260505 05:29:30.189 | INFO | "gpu" : " 17 GB", | 20260505 05:29:30.189 | INFO | "huge" : "", | 20260505 05:29:30.189 | INFO | "huge_shared" : " 27 GB", | 20260505 05:29:30.189 | INFO | "malloc" : " 14 GB", | 20260505 05:29:30.189 | INFO | "pinned" : " 29 GB" | 20260505 05:29:30.189 | INFO | } | 20260505 05:29:30.190 | INFO | Perf report for the last 5.13942 seconds: | 20260505 05:29:30.190 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:29:30.190 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:30.190 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:30.190 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:30.190 | INFO | ANALYSIS : 57.8%, 3.0, 90.1ms, 0 | 20260505 05:29:30.190 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:30.190 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:30.190 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 05:29:30.190 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:30.190 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:30.190 | INFO | Uploaded 36GiB to GPU in 1.59768s (22.5327GiB/s) | 20260505 05:29:30.190 | INFO | Downloaded 34GiB to GPU in 1.51724s (22.4091GiB/s) | 20260505 05:29:35.329 | INFO | Analyzing chunk frames = [705536,706048) | 20260505 05:29:35.329 | INFO | MemInfo: { | 20260505 05:29:35.329 | INFO | "gpu" : " 17 GB", | 20260505 05:29:35.329 | INFO | "huge" : "", | 20260505 05:29:35.329 | INFO | "huge_shared" : " 27 GB", | 20260505 05:29:35.329 | INFO | "malloc" : " 14 GB", | 20260505 05:29:35.329 | INFO | "pinned" : " 29 GB" | 20260505 05:29:35.329 | INFO | } | 20260505 05:29:35.329 | INFO | Perf report for the last 5.13937 seconds: | 20260505 05:29:35.329 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:29:35.329 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:35.329 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:35.329 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:35.329 | INFO | ANALYSIS : 57.2%, 3.0, 88.8ms, 0 | 20260505 05:29:35.329 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:35.329 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:35.329 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260505 05:29:35.329 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:35.329 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:35.329 | INFO | Uploaded 36GiB to GPU in 1.60215s (22.4698GiB/s) | 20260505 05:29:35.329 | INFO | Downloaded 34GiB to GPU in 1.52259s (22.3304GiB/s) | 20260505 05:29:40.468 | INFO | Analyzing chunk frames = [706048,706560) | 20260505 05:29:40.468 | INFO | MemInfo: { | 20260505 05:29:40.468 | INFO | "gpu" : " 17 GB", | 20260505 05:29:40.468 | INFO | "huge" : "", | 20260505 05:29:40.468 | INFO | "huge_shared" : " 27 GB", | 20260505 05:29:40.468 | INFO | "malloc" : " 14 GB", | 20260505 05:29:40.468 | INFO | "pinned" : " 29 GB" | 20260505 05:29:40.468 | INFO | } | 20260505 05:29:40.468 | INFO | Perf report for the last 5.1393 seconds: | 20260505 05:29:40.468 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:29:40.468 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:40.468 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:40.468 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:40.468 | INFO | ANALYSIS : 56.8%, 3.0, 88.2ms, 0 | 20260505 05:29:40.468 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:40.468 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:40.468 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:29:40.468 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:40.468 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:40.469 | INFO | Uploaded 36GiB to GPU in 1.60099s (22.4861GiB/s) | 20260505 05:29:40.469 | INFO | Downloaded 34GiB to GPU in 1.52196s (22.3397GiB/s) | 20260505 05:29:45.607 | INFO | Analyzing chunk frames = [706560,707072) | 20260505 05:29:45.607 | INFO | MemInfo: { | 20260505 05:29:45.607 | INFO | "gpu" : " 17 GB", | 20260505 05:29:45.607 | INFO | "huge" : "", | 20260505 05:29:45.607 | INFO | "huge_shared" : " 27 GB", | 20260505 05:29:45.607 | INFO | "malloc" : " 14 GB", | 20260505 05:29:45.607 | INFO | "pinned" : " 29 GB" | 20260505 05:29:45.607 | INFO | } | 20260505 05:29:45.608 | INFO | Perf report for the last 5.13933 seconds: | 20260505 05:29:45.608 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:29:45.608 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:45.608 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:45.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:45.608 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260505 05:29:45.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:45.608 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:45.608 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:29:45.608 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:45.608 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:45.608 | INFO | Uploaded 36GiB to GPU in 1.59988s (22.5017GiB/s) | 20260505 05:29:45.608 | INFO | Downloaded 34GiB to GPU in 1.51984s (22.3708GiB/s) | 20260505 05:29:50.747 | INFO | Analyzing chunk frames = [707072,707584) | 20260505 05:29:50.747 | INFO | MemInfo: { | 20260505 05:29:50.747 | INFO | "gpu" : " 17 GB", | 20260505 05:29:50.747 | INFO | "huge" : "", | 20260505 05:29:50.747 | INFO | "huge_shared" : " 27 GB", | 20260505 05:29:50.747 | INFO | "malloc" : " 14 GB", | 20260505 05:29:50.747 | INFO | "pinned" : " 29 GB" | 20260505 05:29:50.747 | INFO | } | 20260505 05:29:50.747 | INFO | Perf report for the last 5.13941 seconds: | 20260505 05:29:50.747 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:29:50.747 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:50.747 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:50.747 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:50.747 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260505 05:29:50.747 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:50.747 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:50.747 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:29:50.747 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:50.747 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:50.747 | INFO | Uploaded 36GiB to GPU in 1.59923s (22.5109GiB/s) | 20260505 05:29:50.747 | INFO | Downloaded 34GiB to GPU in 1.51927s (22.3792GiB/s) | 20260505 05:29:55.886 | INFO | Analyzing chunk frames = [707584,708096) | 20260505 05:29:55.886 | INFO | MemInfo: { | 20260505 05:29:55.886 | INFO | "gpu" : " 17 GB", | 20260505 05:29:55.886 | INFO | "huge" : "", | 20260505 05:29:55.886 | INFO | "huge_shared" : " 27 GB", | 20260505 05:29:55.886 | INFO | "malloc" : " 14 GB", | 20260505 05:29:55.886 | INFO | "pinned" : " 29 GB" | 20260505 05:29:55.886 | INFO | } | 20260505 05:29:55.886 | INFO | Perf report for the last 5.13916 seconds: | 20260505 05:29:55.886 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:29:55.886 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:55.886 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:55.886 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:55.886 | INFO | ANALYSIS : 57.2%, 3.0, 89.3ms, 0 | 20260505 05:29:55.886 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:55.886 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:55.886 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:29:55.886 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:55.886 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:29:55.887 | INFO | Uploaded 36GiB to GPU in 1.6005s (22.493GiB/s) | 20260505 05:29:55.887 | INFO | Downloaded 34GiB to GPU in 1.52041s (22.3624GiB/s) | 20260505 05:30:01.025 | INFO | Analyzing chunk frames = [708096,708608) | 20260505 05:30:01.025 | INFO | MemInfo: { | 20260505 05:30:01.025 | INFO | "gpu" : " 17 GB", | 20260505 05:30:01.025 | INFO | "huge" : "", | 20260505 05:30:01.025 | INFO | "huge_shared" : " 27 GB", | 20260505 05:30:01.025 | INFO | "malloc" : " 14 GB", | 20260505 05:30:01.025 | INFO | "pinned" : " 29 GB" | 20260505 05:30:01.025 | INFO | } | 20260505 05:30:01.026 | INFO | Perf report for the last 5.13936 seconds: | 20260505 05:30:01.026 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:30:01.026 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:01.026 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:01.026 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:01.026 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260505 05:30:01.026 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:01.026 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:01.026 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260505 05:30:01.026 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:01.026 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:01.026 | INFO | Uploaded 36GiB to GPU in 1.60014s (22.498GiB/s) | 20260505 05:30:01.026 | INFO | Downloaded 34GiB to GPU in 1.52081s (22.3566GiB/s) | 20260505 05:30:01.123 | INFO | Starting a round of Baz IO | 20260505 05:30:04.064 | INFO | Populating RTMetrics report | 20260505 05:30:04.065 | INFO | RTMetrics report populated | 20260505 05:30:04.065 | INFO | Opening tmp RTMetrics json file | 20260505 05:30:04.073 | INFO | Serializing RTMetrics to json | 20260505 05:30:04.077 | INFO | Serialization done, swapping tmp file now... | 20260505 05:30:04.077 | INFO | RTMetrics done | 20260505 05:30:04.174 | INFO | Finished a round of Baz IO: Wrote 2.245 GiB in 3.051 seconds | 20260505 05:30:04.174 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.49% / 0.08% / 61.44% | 20260505 05:30:04.174 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.32462% / 0.000147025% / 4.37101e-05% | 20260505 05:30:04.174 | INFO | Baz IO overall: | 20260505 05:30:04.174 | INFO | BytesWritten : 334108522842 | 20260505 05:30:04.175 | INFO | NumEvents : 156952223398 | 20260505 05:30:06.165 | INFO | Analyzing chunk frames = [708608,709120) | 20260505 05:30:06.165 | INFO | MemInfo: { | 20260505 05:30:06.165 | INFO | "gpu" : " 17 GB", | 20260505 05:30:06.165 | INFO | "huge" : "", | 20260505 05:30:06.165 | INFO | "huge_shared" : " 27 GB", | 20260505 05:30:06.165 | INFO | "malloc" : " 14 GB", | 20260505 05:30:06.165 | INFO | "pinned" : " 29 GB" | 20260505 05:30:06.165 | INFO | } | 20260505 05:30:06.165 | INFO | Perf report for the last 5.13948 seconds: | 20260505 05:30:06.165 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:30:06.165 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:06.165 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:06.165 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:06.165 | INFO | ANALYSIS : 58.0%, 2.9, 88.6ms, 0 | 20260505 05:30:06.165 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:06.165 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 05:30:06.165 | INFO | PRE_HQ : 57.3%, 2.2, 64.0ms, 0 | 20260505 05:30:06.165 | INFO | BAZWRITER : 58.9%, 4.4, 133.4ms, 0 | 20260505 05:30:06.165 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260505 05:30:06.165 | INFO | Uploaded 36GiB to GPU in 1.65042s (21.8126GiB/s) | 20260505 05:30:06.165 | INFO | Downloaded 38GiB to GPU in 1.71548s (22.1513GiB/s) | 20260505 05:30:11.304 | INFO | Analyzing chunk frames = [709120,709632) | 20260505 05:30:11.304 | INFO | MemInfo: { | 20260505 05:30:11.304 | INFO | "gpu" : " 17 GB", | 20260505 05:30:11.304 | INFO | "huge" : "", | 20260505 05:30:11.304 | INFO | "huge_shared" : " 27 GB", | 20260505 05:30:11.304 | INFO | "malloc" : " 14 GB", | 20260505 05:30:11.304 | INFO | "pinned" : " 29 GB" | 20260505 05:30:11.304 | INFO | } | 20260505 05:30:11.304 | INFO | Perf report for the last 5.13929 seconds: | 20260505 05:30:11.304 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:30:11.304 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:11.304 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:11.304 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:11.304 | INFO | ANALYSIS : 57.7%, 3.0, 90.0ms, 0 | 20260505 05:30:11.304 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:11.304 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:11.304 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:30:11.304 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:11.304 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:11.305 | INFO | Uploaded 36GiB to GPU in 1.60081s (22.4886GiB/s) | 20260505 05:30:11.305 | INFO | Downloaded 34GiB to GPU in 1.51751s (22.4052GiB/s) | 20260505 05:30:16.443 | INFO | Analyzing chunk frames = [709632,710144) | 20260505 05:30:16.443 | INFO | MemInfo: { | 20260505 05:30:16.443 | INFO | "gpu" : " 17 GB", | 20260505 05:30:16.443 | INFO | "huge" : "", | 20260505 05:30:16.443 | INFO | "huge_shared" : " 27 GB", | 20260505 05:30:16.444 | INFO | "malloc" : " 14 GB", | 20260505 05:30:16.444 | INFO | "pinned" : " 29 GB" | 20260505 05:30:16.444 | INFO | } | 20260505 05:30:16.444 | INFO | Perf report for the last 5.13934 seconds: | 20260505 05:30:16.444 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:30:16.444 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:16.444 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:16.444 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:16.444 | INFO | ANALYSIS : 57.2%, 3.0, 88.8ms, 0 | 20260505 05:30:16.444 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:16.444 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:16.444 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:30:16.444 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:16.444 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:16.444 | INFO | Uploaded 36GiB to GPU in 1.60514s (22.428GiB/s) | 20260505 05:30:16.444 | INFO | Downloaded 34GiB to GPU in 1.52179s (22.3422GiB/s) | 20260505 05:30:21.583 | INFO | Analyzing chunk frames = [710144,710656) | 20260505 05:30:21.583 | INFO | MemInfo: { | 20260505 05:30:21.583 | INFO | "gpu" : " 17 GB", | 20260505 05:30:21.583 | INFO | "huge" : "", | 20260505 05:30:21.583 | INFO | "huge_shared" : " 27 GB", | 20260505 05:30:21.583 | INFO | "malloc" : " 14 GB", | 20260505 05:30:21.583 | INFO | "pinned" : " 29 GB" | 20260505 05:30:21.583 | INFO | } | 20260505 05:30:21.583 | INFO | Perf report for the last 5.13943 seconds: | 20260505 05:30:21.583 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:30:21.583 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:21.583 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:21.583 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:21.583 | INFO | ANALYSIS : 56.8%, 3.0, 88.1ms, 0 | 20260505 05:30:21.583 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:21.583 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:21.583 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 05:30:21.583 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:21.583 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:21.583 | INFO | Uploaded 36GiB to GPU in 1.60399s (22.4441GiB/s) | 20260505 05:30:21.583 | INFO | Downloaded 34GiB to GPU in 1.52123s (22.3503GiB/s) | 20260505 05:30:26.722 | INFO | Analyzing chunk frames = [710656,711168) | 20260505 05:30:26.722 | INFO | MemInfo: { | 20260505 05:30:26.722 | INFO | "gpu" : " 17 GB", | 20260505 05:30:26.722 | INFO | "huge" : "", | 20260505 05:30:26.722 | INFO | "huge_shared" : " 27 GB", | 20260505 05:30:26.722 | INFO | "malloc" : " 14 GB", | 20260505 05:30:26.722 | INFO | "pinned" : " 29 GB" | 20260505 05:30:26.722 | INFO | } | 20260505 05:30:26.722 | INFO | Perf report for the last 5.1392 seconds: | 20260505 05:30:26.722 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:30:26.722 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:26.722 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:26.722 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:26.722 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260505 05:30:26.722 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:26.722 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:26.722 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 05:30:26.722 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:26.722 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:26.723 | INFO | Uploaded 36GiB to GPU in 1.60225s (22.4683GiB/s) | 20260505 05:30:26.723 | INFO | Downloaded 34GiB to GPU in 1.51876s (22.3866GiB/s) | 20260505 05:30:31.861 | INFO | Analyzing chunk frames = [711168,711680) | 20260505 05:30:31.861 | INFO | MemInfo: { | 20260505 05:30:31.861 | INFO | "gpu" : " 17 GB", | 20260505 05:30:31.861 | INFO | "huge" : "", | 20260505 05:30:31.861 | INFO | "huge_shared" : " 27 GB", | 20260505 05:30:31.861 | INFO | "malloc" : " 14 GB", | 20260505 05:30:31.861 | INFO | "pinned" : " 29 GB" | 20260505 05:30:31.861 | INFO | } | 20260505 05:30:31.862 | INFO | Perf report for the last 5.13921 seconds: | 20260505 05:30:31.862 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:30:31.862 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:31.862 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:31.862 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:31.862 | INFO | ANALYSIS : 57.0%, 3.0, 89.0ms, 0 | 20260505 05:30:31.862 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:31.862 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:31.862 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 05:30:31.862 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:31.862 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:31.862 | INFO | Uploaded 36GiB to GPU in 1.60342s (22.452GiB/s) | 20260505 05:30:31.862 | INFO | Downloaded 34GiB to GPU in 1.51993s (22.3694GiB/s) | 20260505 05:30:37.001 | INFO | Analyzing chunk frames = [711680,712192) | 20260505 05:30:37.001 | INFO | MemInfo: { | 20260505 05:30:37.001 | INFO | "gpu" : " 17 GB", | 20260505 05:30:37.001 | INFO | "huge" : "", | 20260505 05:30:37.001 | INFO | "huge_shared" : " 27 GB", | 20260505 05:30:37.001 | INFO | "malloc" : " 14 GB", | 20260505 05:30:37.001 | INFO | "pinned" : " 29 GB" | 20260505 05:30:37.001 | INFO | } | 20260505 05:30:37.001 | INFO | Perf report for the last 5.1394 seconds: | 20260505 05:30:37.001 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:30:37.001 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:37.001 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:37.001 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:37.001 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 05:30:37.001 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:37.001 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:37.001 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:30:37.001 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:37.001 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:37.002 | INFO | Uploaded 36GiB to GPU in 1.60217s (22.4695GiB/s) | 20260505 05:30:37.002 | INFO | Downloaded 34GiB to GPU in 1.51997s (22.3689GiB/s) | 20260505 05:30:42.140 | INFO | Analyzing chunk frames = [712192,712704) | 20260505 05:30:42.140 | INFO | MemInfo: { | 20260505 05:30:42.140 | INFO | "gpu" : " 17 GB", | 20260505 05:30:42.140 | INFO | "huge" : "", | 20260505 05:30:42.140 | INFO | "huge_shared" : " 27 GB", | 20260505 05:30:42.140 | INFO | "malloc" : " 14 GB", | 20260505 05:30:42.140 | INFO | "pinned" : " 29 GB" | 20260505 05:30:42.140 | INFO | } | 20260505 05:30:42.140 | INFO | Perf report for the last 5.13938 seconds: | 20260505 05:30:42.140 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:30:42.140 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:42.140 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:42.140 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:42.140 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 05:30:42.140 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:42.140 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:42.141 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:30:42.141 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:42.141 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:42.141 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 05:30:42.141 | INFO | Total mem usage: 15.1 GiB | 20260505 05:30:42.141 | INFO | All Time High Water Marks sum: 15.7 GiB (Savings: 4.09%) | 20260505 05:30:42.141 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -3.91%) | 20260505 05:30:42.141 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 05:30:42.141 | INFO | High Water Breakdown: | 20260505 05:30:42.141 | INFO | 60% : 8.7 GiB ( 96.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 05:30:42.141 | INFO | 39.8% : 5.77 GiB ( 86.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 05:30:42.141 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 05:30:42.141 | INFO | ------------------------------------------ | 20260505 05:30:42.141 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:30:42.141 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 05:30:42.141 | INFO | Total mem usage: 26.5 GiB | 20260505 05:30:42.141 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 05:30:42.141 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 05:30:42.141 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 05:30:42.141 | INFO | High Water Breakdown: | 20260505 05:30:42.141 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:30:42.141 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:30:42.141 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:30:42.141 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:30:42.141 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:30:42.141 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:30:42.141 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 05:30:42.141 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 05:30:42.141 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:30:42.141 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 05:30:42.141 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 05:30:42.141 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 05:30:42.141 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 05:30:42.141 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 05:30:42.141 | INFO | ------------------------------------------ | 20260505 05:30:42.141 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:30:42.141 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 05:30:42.141 | INFO | Total mem usage: 25.6 GiB | 20260505 05:30:42.141 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 05:30:42.141 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 05:30:42.141 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 05:30:42.141 | INFO | High Water Breakdown: | 20260505 05:30:42.141 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 05:30:42.141 | INFO | ------------------------------------------ | 20260505 05:30:42.141 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:30:42.141 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 05:30:42.141 | INFO | Total mem usage: 15.7 GiB | 20260505 05:30:42.141 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 05:30:42.141 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 05:30:42.141 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260505 05:30:42.141 | INFO | High Water Breakdown: | 20260505 05:30:42.141 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 05:30:42.141 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 05:30:42.141 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 05:30:42.141 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 05:30:42.141 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 05:30:42.141 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 05:30:42.141 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 05:30:42.141 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 05:30:42.141 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 05:30:42.141 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:30:42.141 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 05:30:42.141 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:30:42.141 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 05:30:42.141 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 05:30:42.141 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 05:30:42.141 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 05:30:42.141 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:30:42.141 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 05:30:42.141 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:30:42.141 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:30:42.141 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:30:42.141 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 05:30:42.141 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 05:30:42.141 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:30:42.141 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 05:30:42.141 | INFO | ------------------------------------------ | 20260505 05:30:42.141 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:30:42.141 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 05:30:42.141 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:30:42.141 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:30:42.141 | INFO | PerfReport: Full Execution -- 3.03e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260505 05:30:42.141 | INFO | PerfReport: 0.28%, AggregateStats -- 8.64e-02 / -nan, 5.89e-03 / 0.00e+00 | 20260505 05:30:42.141 | INFO | PerfReport: 10.87%, Baseline -- 3.30e+00 / -nan, 1.84e-02 / 0.00e+00 | 20260505 05:30:42.141 | INFO | PerfReport: 3.05%, Histogram -- 9.24e-01 / -nan, 5.20e-02 / 0.00e+00 | 20260505 05:30:42.141 | INFO | PerfReport: 16.05%, DME -- 3.89e+01 / -nan, 3.26e+00 / 0.00e+00 | 20260505 05:30:42.141 | INFO | PerfReport: 56.76%, FrameLabeling -- 1.72e+01 / -nan, 4.68e-02 / 0.00e+00 | 20260505 05:30:42.141 | INFO | PerfReport: 7.53%, PulseAccumulating -- 2.28e+00 / -nan, 1.49e-01 / 0.00e+00 | 20260505 05:30:42.141 | INFO | PerfReport: 3.86%, Metrics -- 1.17e+00 / -nan, 1.94e-01 / 0.00e+00 | 20260505 05:30:42.141 | INFO | PerfReport: 1.58%, PulseSerializing -- 4.80e-01 / -nan, 7.75e-02 / 0.00e+00 | 20260505 05:30:42.141 | INFO | Uploaded 36GiB to GPU in 1.6028s (22.4607GiB/s) | 20260505 05:30:42.141 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 05:30:42.141 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:30:42.141 | INFO | Downloaded 34GiB to GPU in 1.52026s (22.3646GiB/s) | 20260505 05:30:42.141 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:30:42.141 | INFO | PerfReport: Full Execution -- 9.09e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260505 05:30:42.141 | INFO | PerfReport: 18.43%, Upload -- 1.67e+01 / 0.00e+00, 4.17e-01 / 0.00e+00 | 20260505 05:30:42.141 | INFO | PerfReport: 17.71%, Download -- 1.61e+01 / 0.00e+00, 7.59e-01 / 0.00e+00 | 20260505 05:30:42.141 | INFO | PerfReport: 33.37%, Compute -- 3.03e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260505 05:30:42.141 | INFO | PerfReport: 0.56%, LockUpload -- 5.07e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260505 05:30:42.141 | INFO | PerfReport: 0.00%, LockDownload -- 3.89e-04 / 0.00e+00, 8.92e-05 / 0.00e+00 | 20260505 05:30:42.141 | INFO | PerfReport: 29.91%, LockCompute -- 2.72e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260505 05:30:42.141 | INFO | PerfReport: 0.02%, Misc -- 1.53e-02 / 0.00e+00, 4.04e-02 / 0.00e+00 | 20260505 05:30:42.238 | INFO | Starting a round of Baz IO | 20260505 05:30:45.181 | INFO | Populating RTMetrics report | 20260505 05:30:45.181 | INFO | RTMetrics report populated | 20260505 05:30:45.181 | INFO | Opening tmp RTMetrics json file | 20260505 05:30:45.189 | INFO | Serializing RTMetrics to json | 20260505 05:30:45.190 | INFO | Serialization done, swapping tmp file now... | 20260505 05:30:45.191 | INFO | RTMetrics done | 20260505 05:30:45.288 | INFO | Finished a round of Baz IO: Wrote 2.246 GiB in 3.05 seconds | 20260505 05:30:45.288 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.41% / 0.08% / 61.51% | 20260505 05:30:45.288 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.27216% / 0.000154972% / 6.75519e-05% | 20260505 05:30:45.288 | INFO | Baz IO overall: | 20260505 05:30:45.288 | INFO | BytesWritten : 336519713983 | 20260505 05:30:45.288 | INFO | NumEvents : 158101627450 | 20260505 05:30:47.279 | INFO | Analyzing chunk frames = [712704,713216) | 20260505 05:30:47.279 | INFO | MemInfo: { | 20260505 05:30:47.279 | INFO | "gpu" : " 17 GB", | 20260505 05:30:47.279 | INFO | "huge" : "", | 20260505 05:30:47.279 | INFO | "huge_shared" : " 27 GB", | 20260505 05:30:47.279 | INFO | "malloc" : " 14 GB", | 20260505 05:30:47.279 | INFO | "pinned" : " 29 GB" | 20260505 05:30:47.279 | INFO | } | 20260505 05:30:47.280 | INFO | Perf report for the last 5.13916 seconds: | 20260505 05:30:47.280 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:30:47.280 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:47.280 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:47.280 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:47.280 | INFO | ANALYSIS : 58.0%, 2.9, 88.4ms, 0 | 20260505 05:30:47.280 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:47.280 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260505 05:30:47.280 | INFO | PRE_HQ : 57.3%, 2.1, 63.8ms, 0 | 20260505 05:30:47.280 | INFO | BAZWRITER : 58.9%, 4.4, 132.6ms, 0 | 20260505 05:30:47.280 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260505 05:30:47.280 | INFO | Uploaded 36GiB to GPU in 1.64704s (21.8574GiB/s) | 20260505 05:30:47.280 | INFO | Downloaded 38GiB to GPU in 1.71195s (22.1969GiB/s) | 20260505 05:30:52.419 | INFO | Analyzing chunk frames = [713216,713728) | 20260505 05:30:52.419 | INFO | MemInfo: { | 20260505 05:30:52.419 | INFO | "gpu" : " 17 GB", | 20260505 05:30:52.419 | INFO | "huge" : "", | 20260505 05:30:52.419 | INFO | "huge_shared" : " 27 GB", | 20260505 05:30:52.419 | INFO | "malloc" : " 14 GB", | 20260505 05:30:52.419 | INFO | "pinned" : " 29 GB" | 20260505 05:30:52.419 | INFO | } | 20260505 05:30:52.419 | INFO | Perf report for the last 5.13946 seconds: | 20260505 05:30:52.419 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:30:52.419 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:52.419 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:52.419 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:52.419 | INFO | ANALYSIS : 57.8%, 3.0, 90.1ms, 0 | 20260505 05:30:52.419 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:52.419 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:52.419 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 05:30:52.419 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:52.419 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:52.420 | INFO | Uploaded 36GiB to GPU in 1.5985s (22.5211GiB/s) | 20260505 05:30:52.420 | INFO | Downloaded 34GiB to GPU in 1.51805s (22.3972GiB/s) | 20260505 05:30:57.558 | INFO | Analyzing chunk frames = [713728,714240) | 20260505 05:30:57.558 | INFO | MemInfo: { | 20260505 05:30:57.558 | INFO | "gpu" : " 17 GB", | 20260505 05:30:57.558 | INFO | "huge" : "", | 20260505 05:30:57.558 | INFO | "huge_shared" : " 27 GB", | 20260505 05:30:57.558 | INFO | "malloc" : " 14 GB", | 20260505 05:30:57.558 | INFO | "pinned" : " 29 GB" | 20260505 05:30:57.558 | INFO | } | 20260505 05:30:57.558 | INFO | Perf report for the last 5.13937 seconds: | 20260505 05:30:57.558 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:30:57.558 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:57.558 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:57.558 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:57.558 | INFO | ANALYSIS : 57.1%, 3.0, 88.8ms, 0 | 20260505 05:30:57.558 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:57.558 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:57.559 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:30:57.559 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:57.559 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:30:57.559 | INFO | Uploaded 36GiB to GPU in 1.60172s (22.4758GiB/s) | 20260505 05:30:57.559 | INFO | Downloaded 34GiB to GPU in 1.52235s (22.3339GiB/s) | 20260505 05:31:02.697 | INFO | Analyzing chunk frames = [714240,714752) | 20260505 05:31:02.697 | INFO | MemInfo: { | 20260505 05:31:02.697 | INFO | "gpu" : " 17 GB", | 20260505 05:31:02.697 | INFO | "huge" : "", | 20260505 05:31:02.698 | INFO | "huge_shared" : " 27 GB", | 20260505 05:31:02.698 | INFO | "malloc" : " 14 GB", | 20260505 05:31:02.698 | INFO | "pinned" : " 29 GB" | 20260505 05:31:02.698 | INFO | } | 20260505 05:31:02.698 | INFO | Perf report for the last 5.13936 seconds: | 20260505 05:31:02.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:31:02.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:02.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:02.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:02.698 | INFO | ANALYSIS : 56.8%, 3.0, 88.1ms, 0 | 20260505 05:31:02.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:02.698 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:02.698 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:31:02.698 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:02.698 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:02.698 | INFO | Uploaded 36GiB to GPU in 1.60156s (22.4781GiB/s) | 20260505 05:31:02.698 | INFO | Downloaded 34GiB to GPU in 1.52227s (22.335GiB/s) | 20260505 05:31:07.837 | INFO | Analyzing chunk frames = [714752,715264) | 20260505 05:31:07.837 | INFO | MemInfo: { | 20260505 05:31:07.837 | INFO | "gpu" : " 17 GB", | 20260505 05:31:07.837 | INFO | "huge" : "", | 20260505 05:31:07.837 | INFO | "huge_shared" : " 27 GB", | 20260505 05:31:07.837 | INFO | "malloc" : " 14 GB", | 20260505 05:31:07.837 | INFO | "pinned" : " 29 GB" | 20260505 05:31:07.837 | INFO | } | 20260505 05:31:07.837 | INFO | Perf report for the last 5.13945 seconds: | 20260505 05:31:07.837 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:31:07.837 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:07.837 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:07.837 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:07.837 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260505 05:31:07.837 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:07.837 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:07.837 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 05:31:07.837 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:07.837 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:07.838 | INFO | Uploaded 36GiB to GPU in 1.59968s (22.5046GiB/s) | 20260505 05:31:07.838 | INFO | Downloaded 34GiB to GPU in 1.51949s (22.376GiB/s) | 20260505 05:31:12.976 | INFO | Analyzing chunk frames = [715264,715776) | 20260505 05:31:12.976 | INFO | MemInfo: { | 20260505 05:31:12.976 | INFO | "gpu" : " 17 GB", | 20260505 05:31:12.976 | INFO | "huge" : "", | 20260505 05:31:12.976 | INFO | "huge_shared" : " 27 GB", | 20260505 05:31:12.976 | INFO | "malloc" : " 14 GB", | 20260505 05:31:12.976 | INFO | "pinned" : " 29 GB" | 20260505 05:31:12.976 | INFO | } | 20260505 05:31:12.977 | INFO | Perf report for the last 5.13928 seconds: | 20260505 05:31:12.977 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:31:12.977 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:12.977 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:12.977 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:12.977 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260505 05:31:12.977 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:12.977 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:12.977 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 05:31:12.977 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:12.977 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:12.977 | INFO | Uploaded 36GiB to GPU in 1.59982s (22.5026GiB/s) | 20260505 05:31:12.977 | INFO | Downloaded 34GiB to GPU in 1.52016s (22.3661GiB/s) | 20260505 05:31:18.116 | INFO | Analyzing chunk frames = [715776,716288) | 20260505 05:31:18.116 | INFO | MemInfo: { | 20260505 05:31:18.116 | INFO | "gpu" : " 17 GB", | 20260505 05:31:18.116 | INFO | "huge" : "", | 20260505 05:31:18.116 | INFO | "huge_shared" : " 27 GB", | 20260505 05:31:18.116 | INFO | "malloc" : " 14 GB", | 20260505 05:31:18.116 | INFO | "pinned" : " 29 GB" | 20260505 05:31:18.116 | INFO | } | 20260505 05:31:18.116 | INFO | Perf report for the last 5.13934 seconds: | 20260505 05:31:18.116 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:31:18.116 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:18.116 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:18.116 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:18.116 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 05:31:18.116 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:18.116 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:18.116 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:31:18.116 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:18.116 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:18.116 | INFO | Uploaded 36GiB to GPU in 1.60034s (22.4952GiB/s) | 20260505 05:31:18.116 | INFO | Downloaded 34GiB to GPU in 1.52166s (22.344GiB/s) | 20260505 05:31:23.255 | INFO | Analyzing chunk frames = [716288,716800) | 20260505 05:31:23.255 | INFO | MemInfo: { | 20260505 05:31:23.255 | INFO | "gpu" : " 17 GB", | 20260505 05:31:23.255 | INFO | "huge" : "", | 20260505 05:31:23.255 | INFO | "huge_shared" : " 27 GB", | 20260505 05:31:23.255 | INFO | "malloc" : " 14 GB", | 20260505 05:31:23.255 | INFO | "pinned" : " 29 GB" | 20260505 05:31:23.255 | INFO | } | 20260505 05:31:23.255 | INFO | Perf report for the last 5.13933 seconds: | 20260505 05:31:23.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:31:23.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:23.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:23.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:23.255 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 05:31:23.255 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:23.255 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:23.255 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 05:31:23.255 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:23.255 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:23.255 | INFO | Uploaded 36GiB to GPU in 1.60039s (22.4945GiB/s) | 20260505 05:31:23.256 | INFO | Downloaded 34GiB to GPU in 1.52137s (22.3483GiB/s) | 20260505 05:31:23.352 | INFO | Starting a round of Baz IO | 20260505 05:31:26.292 | INFO | Populating RTMetrics report | 20260505 05:31:26.293 | INFO | RTMetrics report populated | 20260505 05:31:26.293 | INFO | Opening tmp RTMetrics json file | 20260505 05:31:26.303 | INFO | Serializing RTMetrics to json | 20260505 05:31:26.306 | INFO | Serialization done, swapping tmp file now... | 20260505 05:31:26.308 | INFO | RTMetrics done | 20260505 05:31:26.397 | INFO | Finished a round of Baz IO: Wrote 2.25 GiB in 3.044 seconds | 20260505 05:31:26.397 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.33% / 0.08% / 61.59% | 20260505 05:31:26.397 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.25871% / 0.000158946% / 4.76837e-05% | 20260505 05:31:26.398 | INFO | Baz IO overall: | 20260505 05:31:26.398 | INFO | BytesWritten : 338935734146 | 20260505 05:31:26.398 | INFO | NumEvents : 159253877308 | 20260505 05:31:28.394 | INFO | Analyzing chunk frames = [716800,717312) | 20260505 05:31:28.394 | INFO | MemInfo: { | 20260505 05:31:28.394 | INFO | "gpu" : " 17 GB", | 20260505 05:31:28.394 | INFO | "huge" : "", | 20260505 05:31:28.394 | INFO | "huge_shared" : " 27 GB", | 20260505 05:31:28.394 | INFO | "malloc" : " 14 GB", | 20260505 05:31:28.394 | INFO | "pinned" : " 29 GB" | 20260505 05:31:28.394 | INFO | } | 20260505 05:31:28.394 | INFO | Perf report for the last 5.13919 seconds: | 20260505 05:31:28.394 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:31:28.394 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:28.394 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:28.394 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:28.395 | INFO | ANALYSIS : 58.0%, 2.9, 88.6ms, 0 | 20260505 05:31:28.395 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:28.395 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260505 05:31:28.395 | INFO | PRE_HQ : 57.3%, 2.1, 63.6ms, 0 | 20260505 05:31:28.395 | INFO | BAZWRITER : 58.8%, 4.4, 132.6ms, 0 | 20260505 05:31:28.395 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 05:31:28.395 | INFO | Uploaded 36GiB to GPU in 1.65275s (21.7819GiB/s) | 20260505 05:31:28.395 | INFO | Downloaded 38GiB to GPU in 1.71516s (22.1554GiB/s) | 20260505 05:31:33.534 | INFO | Analyzing chunk frames = [717312,717824) | 20260505 05:31:33.534 | INFO | MemInfo: { | 20260505 05:31:33.534 | INFO | "gpu" : " 17 GB", | 20260505 05:31:33.534 | INFO | "huge" : "", | 20260505 05:31:33.534 | INFO | "huge_shared" : " 27 GB", | 20260505 05:31:33.534 | INFO | "malloc" : " 14 GB", | 20260505 05:31:33.534 | INFO | "pinned" : " 29 GB" | 20260505 05:31:33.534 | INFO | } | 20260505 05:31:33.534 | INFO | Perf report for the last 5.13943 seconds: | 20260505 05:31:33.534 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:31:33.534 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:33.534 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:33.534 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:33.534 | INFO | ANALYSIS : 57.8%, 3.0, 90.1ms, 0 | 20260505 05:31:33.534 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:33.534 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:33.534 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:31:33.534 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:33.534 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:33.534 | INFO | Uploaded 36GiB to GPU in 1.60075s (22.4894GiB/s) | 20260505 05:31:33.534 | INFO | Downloaded 34GiB to GPU in 1.51794s (22.3988GiB/s) | 20260505 05:31:38.673 | INFO | Analyzing chunk frames = [717824,718336) | 20260505 05:31:38.673 | INFO | MemInfo: { | 20260505 05:31:38.673 | INFO | "gpu" : " 17 GB", | 20260505 05:31:38.673 | INFO | "huge" : "", | 20260505 05:31:38.673 | INFO | "huge_shared" : " 27 GB", | 20260505 05:31:38.673 | INFO | "malloc" : " 14 GB", | 20260505 05:31:38.673 | INFO | "pinned" : " 29 GB" | 20260505 05:31:38.673 | INFO | } | 20260505 05:31:38.673 | INFO | Perf report for the last 5.13921 seconds: | 20260505 05:31:38.673 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:31:38.673 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:38.673 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:38.673 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:38.673 | INFO | ANALYSIS : 57.1%, 3.0, 88.8ms, 0 | 20260505 05:31:38.673 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:38.673 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:38.673 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:31:38.673 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:38.673 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:38.673 | INFO | Uploaded 36GiB to GPU in 1.60376s (22.4472GiB/s) | 20260505 05:31:38.673 | INFO | Downloaded 34GiB to GPU in 1.52146s (22.347GiB/s) | 20260505 05:31:43.812 | INFO | Analyzing chunk frames = [718336,718848) | 20260505 05:31:43.812 | INFO | MemInfo: { | 20260505 05:31:43.812 | INFO | "gpu" : " 17 GB", | 20260505 05:31:43.812 | INFO | "huge" : "", | 20260505 05:31:43.812 | INFO | "huge_shared" : " 27 GB", | 20260505 05:31:43.812 | INFO | "malloc" : " 14 GB", | 20260505 05:31:43.812 | INFO | "pinned" : " 29 GB" | 20260505 05:31:43.813 | INFO | } | 20260505 05:31:43.813 | INFO | Perf report for the last 5.13966 seconds: | 20260505 05:31:43.813 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:31:43.813 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:43.813 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:43.813 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:43.813 | INFO | ANALYSIS : 56.8%, 3.0, 88.1ms, 0 | 20260505 05:31:43.813 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:43.813 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:43.813 | INFO | PRE_HQ : 30.5%, 1.0, 15.9ms, 0 | 20260505 05:31:43.813 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:43.813 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:43.813 | INFO | Uploaded 36GiB to GPU in 1.60348s (22.4511GiB/s) | 20260505 05:31:43.813 | INFO | Downloaded 34GiB to GPU in 1.52126s (22.3499GiB/s) | 20260505 05:31:48.952 | INFO | Analyzing chunk frames = [718848,719360) | 20260505 05:31:48.952 | INFO | MemInfo: { | 20260505 05:31:48.952 | INFO | "gpu" : " 17 GB", | 20260505 05:31:48.952 | INFO | "huge" : "", | 20260505 05:31:48.952 | INFO | "huge_shared" : " 27 GB", | 20260505 05:31:48.952 | INFO | "malloc" : " 14 GB", | 20260505 05:31:48.952 | INFO | "pinned" : " 29 GB" | 20260505 05:31:48.952 | INFO | } | 20260505 05:31:48.952 | INFO | Perf report for the last 5.13921 seconds: | 20260505 05:31:48.952 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:31:48.952 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:48.952 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:48.952 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:48.952 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260505 05:31:48.952 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:48.952 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:48.952 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 05:31:48.952 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:48.952 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:48.952 | INFO | Uploaded 36GiB to GPU in 1.6026s (22.4635GiB/s) | 20260505 05:31:48.952 | INFO | Downloaded 34GiB to GPU in 1.51878s (22.3863GiB/s) | 20260505 05:31:54.091 | INFO | Analyzing chunk frames = [719360,719872) | 20260505 05:31:54.091 | INFO | MemInfo: { | 20260505 05:31:54.091 | INFO | "gpu" : " 17 GB", | 20260505 05:31:54.091 | INFO | "huge" : "", | 20260505 05:31:54.091 | INFO | "huge_shared" : " 27 GB", | 20260505 05:31:54.091 | INFO | "malloc" : " 14 GB", | 20260505 05:31:54.091 | INFO | "pinned" : " 29 GB" | 20260505 05:31:54.091 | INFO | } | 20260505 05:31:54.091 | INFO | Perf report for the last 5.13921 seconds: | 20260505 05:31:54.091 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:31:54.091 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:54.091 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:54.091 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:54.091 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260505 05:31:54.091 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:54.091 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:54.091 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 05:31:54.091 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:54.091 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:54.091 | INFO | Uploaded 36GiB to GPU in 1.60179s (22.4748GiB/s) | 20260505 05:31:54.092 | INFO | Downloaded 34GiB to GPU in 1.51866s (22.3881GiB/s) | 20260505 05:31:59.231 | INFO | Analyzing chunk frames = [719872,720384) | 20260505 05:31:59.231 | INFO | MemInfo: { | 20260505 05:31:59.231 | INFO | "gpu" : " 17 GB", | 20260505 05:31:59.231 | INFO | "huge" : "", | 20260505 05:31:59.231 | INFO | "huge_shared" : " 27 GB", | 20260505 05:31:59.231 | INFO | "malloc" : " 14 GB", | 20260505 05:31:59.231 | INFO | "pinned" : " 29 GB" | 20260505 05:31:59.231 | INFO | } | 20260505 05:31:59.231 | INFO | Perf report for the last 5.13959 seconds: | 20260505 05:31:59.231 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:31:59.231 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:59.231 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:59.231 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:59.231 | INFO | ANALYSIS : 57.2%, 3.0, 89.3ms, 0 | 20260505 05:31:59.231 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:59.231 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:59.231 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:31:59.231 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:59.231 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:31:59.231 | INFO | Uploaded 36GiB to GPU in 1.60285s (22.46GiB/s) | 20260505 05:31:59.231 | INFO | Downloaded 34GiB to GPU in 1.5201s (22.367GiB/s) | 20260505 05:32:04.370 | INFO | Analyzing chunk frames = [720384,720896) | 20260505 05:32:04.370 | INFO | MemInfo: { | 20260505 05:32:04.370 | INFO | "gpu" : " 17 GB", | 20260505 05:32:04.370 | INFO | "huge" : "", | 20260505 05:32:04.370 | INFO | "huge_shared" : " 27 GB", | 20260505 05:32:04.370 | INFO | "malloc" : " 14 GB", | 20260505 05:32:04.370 | INFO | "pinned" : " 29 GB" | 20260505 05:32:04.370 | INFO | } | 20260505 05:32:04.370 | INFO | Perf report for the last 5.13914 seconds: | 20260505 05:32:04.370 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:32:04.370 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:04.370 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:04.370 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:04.370 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260505 05:32:04.370 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:04.370 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:04.370 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 05:32:04.370 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:04.370 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:04.371 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 05:32:04.371 | INFO | Total mem usage: 15.1 GiB | 20260505 05:32:04.371 | INFO | All Time High Water Marks sum: 15.7 GiB (Savings: 4.09%) | 20260505 05:32:04.371 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -3.86%) | 20260505 05:32:04.371 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 05:32:04.371 | INFO | High Water Breakdown: | 20260505 05:32:04.371 | INFO | 60% : 8.7 GiB ( 96.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 05:32:04.371 | INFO | 39.8% : 5.77 GiB ( 86.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 05:32:04.371 | INFO | Uploaded 36GiB to GPU in 1.60253s (22.4644GiB/s) | 20260505 05:32:04.371 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 05:32:04.371 | INFO | Downloaded 34GiB to GPU in 1.51997s (22.3689GiB/s) | 20260505 05:32:04.371 | INFO | ------------------------------------------ | 20260505 05:32:04.371 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:32:04.371 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 05:32:04.371 | INFO | Total mem usage: 26.5 GiB | 20260505 05:32:04.371 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 05:32:04.371 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 05:32:04.371 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 05:32:04.371 | INFO | High Water Breakdown: | 20260505 05:32:04.371 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:32:04.371 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:32:04.371 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:32:04.371 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:32:04.371 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:32:04.371 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:32:04.371 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 05:32:04.371 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 05:32:04.371 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:32:04.371 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 05:32:04.371 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 05:32:04.371 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 05:32:04.371 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 05:32:04.371 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 05:32:04.371 | INFO | ------------------------------------------ | 20260505 05:32:04.371 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:32:04.371 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 05:32:04.371 | INFO | Total mem usage: 25.6 GiB | 20260505 05:32:04.371 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 05:32:04.371 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 05:32:04.371 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 05:32:04.371 | INFO | High Water Breakdown: | 20260505 05:32:04.371 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 05:32:04.371 | INFO | ------------------------------------------ | 20260505 05:32:04.371 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:32:04.371 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 05:32:04.371 | INFO | Total mem usage: 15.7 GiB | 20260505 05:32:04.371 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 05:32:04.371 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 05:32:04.371 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260505 05:32:04.371 | INFO | High Water Breakdown: | 20260505 05:32:04.371 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 05:32:04.371 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 05:32:04.371 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 05:32:04.371 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 05:32:04.371 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 05:32:04.371 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 05:32:04.371 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 05:32:04.371 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 05:32:04.371 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 05:32:04.371 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:32:04.371 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 05:32:04.371 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:32:04.371 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 05:32:04.371 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 05:32:04.371 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 05:32:04.371 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 05:32:04.371 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:32:04.371 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 05:32:04.371 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:32:04.371 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:32:04.371 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:32:04.371 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 05:32:04.371 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 05:32:04.371 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:32:04.371 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 05:32:04.371 | INFO | ------------------------------------------ | 20260505 05:32:04.371 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:32:04.371 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 05:32:04.371 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:32:04.371 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:32:04.371 | INFO | PerfReport: Full Execution -- 3.03e+01 / -nan, 1.28e+01 / 0.00e+00 | 20260505 05:32:04.371 | INFO | PerfReport: 0.28%, AggregateStats -- 8.61e-02 / -nan, 3.99e-03 / 0.00e+00 | 20260505 05:32:04.371 | INFO | PerfReport: 10.88%, Baseline -- 3.30e+00 / -nan, 1.91e-02 / 0.00e+00 | 20260505 05:32:04.371 | INFO | PerfReport: 3.05%, Histogram -- 9.24e-01 / -nan, 5.21e-02 / 0.00e+00 | 20260505 05:32:04.371 | INFO | PerfReport: 16.04%, DME -- 3.89e+01 / -nan, 3.22e+00 / 0.00e+00 | 20260505 05:32:04.371 | INFO | PerfReport: 56.77%, FrameLabeling -- 1.72e+01 / -nan, 4.60e-02 / 0.00e+00 | 20260505 05:32:04.371 | INFO | PerfReport: 7.53%, PulseAccumulating -- 2.28e+00 / -nan, 1.48e-01 / 0.00e+00 | 20260505 05:32:04.371 | INFO | PerfReport: 3.86%, Metrics -- 1.17e+00 / -nan, 1.94e-01 / 0.00e+00 | 20260505 05:32:04.371 | INFO | PerfReport: 1.58%, PulseSerializing -- 4.79e-01 / -nan, 7.73e-02 / 0.00e+00 | 20260505 05:32:04.371 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 05:32:04.371 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:32:04.371 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:32:04.372 | INFO | PerfReport: Full Execution -- 9.08e+01 / -nan, 1.89e+01 / 0.00e+00 | 20260505 05:32:04.372 | INFO | PerfReport: 18.44%, Upload -- 1.67e+01 / 0.00e+00, 4.06e-01 / 0.00e+00 | 20260505 05:32:04.372 | INFO | PerfReport: 17.71%, Download -- 1.61e+01 / 0.00e+00, 7.43e-01 / 0.00e+00 | 20260505 05:32:04.372 | INFO | PerfReport: 33.38%, Compute -- 3.03e+01 / 0.00e+00, 1.28e+01 / 0.00e+00 | 20260505 05:32:04.372 | INFO | PerfReport: 0.57%, LockUpload -- 5.17e-01 / 0.00e+00, 3.63e+00 / 0.00e+00 | 20260505 05:32:04.372 | INFO | PerfReport: 0.00%, LockDownload -- 3.94e-04 / 0.00e+00, 1.34e-04 / 0.00e+00 | 20260505 05:32:04.372 | INFO | PerfReport: 29.89%, LockCompute -- 2.71e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260505 05:32:04.372 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.63e-02 / 0.00e+00 | 20260505 05:32:04.467 | INFO | Starting a round of Baz IO | 20260505 05:32:07.408 | INFO | Populating RTMetrics report | 20260505 05:32:07.408 | INFO | RTMetrics report populated | 20260505 05:32:07.408 | INFO | Opening tmp RTMetrics json file | 20260505 05:32:07.418 | INFO | Serializing RTMetrics to json | 20260505 05:32:07.420 | INFO | Serialization done, swapping tmp file now... | 20260505 05:32:07.423 | INFO | RTMetrics done | 20260505 05:32:07.511 | INFO | Finished a round of Baz IO: Wrote 2.254 GiB in 3.043 seconds | 20260505 05:32:07.511 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.25% / 0.08% / 61.67% | 20260505 05:32:07.511 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.24905% / 0.000107288% / 4.37101e-05% | 20260505 05:32:07.511 | INFO | Baz IO overall: | 20260505 05:32:07.511 | INFO | BytesWritten : 341355946596 | 20260505 05:32:07.511 | INFO | NumEvents : 160408349896 | 20260505 05:32:09.509 | INFO | Analyzing chunk frames = [720896,721408) | 20260505 05:32:09.509 | INFO | MemInfo: { | 20260505 05:32:09.509 | INFO | "gpu" : " 17 GB", | 20260505 05:32:09.509 | INFO | "huge" : "", | 20260505 05:32:09.509 | INFO | "huge_shared" : " 27 GB", | 20260505 05:32:09.509 | INFO | "malloc" : " 14 GB", | 20260505 05:32:09.509 | INFO | "pinned" : " 29 GB" | 20260505 05:32:09.509 | INFO | } | 20260505 05:32:09.509 | INFO | Perf report for the last 5.13932 seconds: | 20260505 05:32:09.509 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:32:09.509 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:09.509 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:09.509 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:09.509 | INFO | ANALYSIS : 58.0%, 2.9, 88.4ms, 0 | 20260505 05:32:09.509 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:09.509 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260505 05:32:09.509 | INFO | PRE_HQ : 57.2%, 2.1, 63.4ms, 0 | 20260505 05:32:09.509 | INFO | BAZWRITER : 58.8%, 4.4, 133.1ms, 0 | 20260505 05:32:09.509 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 05:32:09.510 | INFO | Uploaded 36GiB to GPU in 1.65554s (21.7451GiB/s) | 20260505 05:32:09.510 | INFO | Downloaded 38GiB to GPU in 1.71857s (22.1115GiB/s) | 20260505 05:32:14.648 | INFO | Analyzing chunk frames = [721408,721920) | 20260505 05:32:14.648 | INFO | MemInfo: { | 20260505 05:32:14.648 | INFO | "gpu" : " 17 GB", | 20260505 05:32:14.648 | INFO | "huge" : "", | 20260505 05:32:14.648 | INFO | "huge_shared" : " 27 GB", | 20260505 05:32:14.648 | INFO | "malloc" : " 14 GB", | 20260505 05:32:14.649 | INFO | "pinned" : " 29 GB" | 20260505 05:32:14.649 | INFO | } | 20260505 05:32:14.649 | INFO | Perf report for the last 5.13941 seconds: | 20260505 05:32:14.649 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:32:14.649 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:14.649 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:14.649 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:14.649 | INFO | ANALYSIS : 57.8%, 3.0, 90.2ms, 0 | 20260505 05:32:14.649 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:14.649 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:14.649 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 05:32:14.649 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:14.649 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:14.649 | INFO | Uploaded 36GiB to GPU in 1.59756s (22.5343GiB/s) | 20260505 05:32:14.649 | INFO | Downloaded 34GiB to GPU in 1.51728s (22.4085GiB/s) | 20260505 05:32:19.788 | INFO | Analyzing chunk frames = [721920,722432) | 20260505 05:32:19.788 | INFO | MemInfo: { | 20260505 05:32:19.788 | INFO | "gpu" : " 17 GB", | 20260505 05:32:19.788 | INFO | "huge" : "", | 20260505 05:32:19.788 | INFO | "huge_shared" : " 27 GB", | 20260505 05:32:19.788 | INFO | "malloc" : " 14 GB", | 20260505 05:32:19.788 | INFO | "pinned" : " 29 GB" | 20260505 05:32:19.788 | INFO | } | 20260505 05:32:19.788 | INFO | Perf report for the last 5.13923 seconds: | 20260505 05:32:19.788 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:32:19.788 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:19.788 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:19.788 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:19.788 | INFO | ANALYSIS : 57.1%, 3.0, 88.7ms, 0 | 20260505 05:32:19.788 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:19.788 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:19.788 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 05:32:19.788 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:19.788 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:19.788 | INFO | Uploaded 36GiB to GPU in 1.60184s (22.4742GiB/s) | 20260505 05:32:19.788 | INFO | Downloaded 34GiB to GPU in 1.52258s (22.3305GiB/s) | 20260505 05:32:24.927 | INFO | Analyzing chunk frames = [722432,722944) | 20260505 05:32:24.927 | INFO | MemInfo: { | 20260505 05:32:24.927 | INFO | "gpu" : " 17 GB", | 20260505 05:32:24.927 | INFO | "huge" : "", | 20260505 05:32:24.927 | INFO | "huge_shared" : " 27 GB", | 20260505 05:32:24.927 | INFO | "malloc" : " 14 GB", | 20260505 05:32:24.927 | INFO | "pinned" : " 29 GB" | 20260505 05:32:24.927 | INFO | } | 20260505 05:32:24.927 | INFO | Perf report for the last 5.1393 seconds: | 20260505 05:32:24.927 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:32:24.927 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:24.927 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:24.927 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:24.927 | INFO | ANALYSIS : 56.9%, 3.0, 88.2ms, 0 | 20260505 05:32:24.927 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:24.927 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:24.927 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 05:32:24.927 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:24.927 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:24.927 | INFO | Uploaded 36GiB to GPU in 1.60079s (22.4889GiB/s) | 20260505 05:32:24.928 | INFO | Downloaded 34GiB to GPU in 1.52109s (22.3524GiB/s) | 20260505 05:32:30.066 | INFO | Analyzing chunk frames = [722944,723456) | 20260505 05:32:30.066 | INFO | MemInfo: { | 20260505 05:32:30.066 | INFO | "gpu" : " 17 GB", | 20260505 05:32:30.066 | INFO | "huge" : "", | 20260505 05:32:30.066 | INFO | "huge_shared" : " 27 GB", | 20260505 05:32:30.066 | INFO | "malloc" : " 14 GB", | 20260505 05:32:30.067 | INFO | "pinned" : " 29 GB" | 20260505 05:32:30.067 | INFO | } | 20260505 05:32:30.067 | INFO | Perf report for the last 5.13935 seconds: | 20260505 05:32:30.067 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:32:30.067 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:30.067 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:30.067 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:30.067 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260505 05:32:30.067 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:30.067 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:30.067 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 05:32:30.067 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:30.067 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:30.067 | INFO | Uploaded 36GiB to GPU in 1.59963s (22.5053GiB/s) | 20260505 05:32:30.068 | INFO | Downloaded 34GiB to GPU in 1.51958s (22.3746GiB/s) | 20260505 05:32:35.206 | INFO | Analyzing chunk frames = [723456,723968) | 20260505 05:32:35.206 | INFO | MemInfo: { | 20260505 05:32:35.206 | INFO | "gpu" : " 17 GB", | 20260505 05:32:35.206 | INFO | "huge" : "", | 20260505 05:32:35.206 | INFO | "huge_shared" : " 27 GB", | 20260505 05:32:35.206 | INFO | "malloc" : " 14 GB", | 20260505 05:32:35.206 | INFO | "pinned" : " 29 GB" | 20260505 05:32:35.206 | INFO | } | 20260505 05:32:35.206 | INFO | Perf report for the last 5.13941 seconds: | 20260505 05:32:35.206 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:32:35.206 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:35.206 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:35.206 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:35.206 | INFO | ANALYSIS : 57.0%, 3.0, 89.0ms, 0 | 20260505 05:32:35.206 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:35.206 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:35.206 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 05:32:35.206 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:35.206 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:35.206 | INFO | Uploaded 36GiB to GPU in 1.59939s (22.5085GiB/s) | 20260505 05:32:35.206 | INFO | Downloaded 34GiB to GPU in 1.519s (22.3832GiB/s) | 20260505 05:32:40.345 | INFO | Analyzing chunk frames = [723968,724480) | 20260505 05:32:40.345 | INFO | MemInfo: { | 20260505 05:32:40.345 | INFO | "gpu" : " 17 GB", | 20260505 05:32:40.345 | INFO | "huge" : "", | 20260505 05:32:40.345 | INFO | "huge_shared" : " 27 GB", | 20260505 05:32:40.345 | INFO | "malloc" : " 14 GB", | 20260505 05:32:40.345 | INFO | "pinned" : " 29 GB" | 20260505 05:32:40.345 | INFO | } | 20260505 05:32:40.345 | INFO | Perf report for the last 5.13933 seconds: | 20260505 05:32:40.345 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:32:40.345 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:40.345 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:40.345 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:40.345 | INFO | ANALYSIS : 57.2%, 3.0, 89.3ms, 0 | 20260505 05:32:40.345 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:40.345 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:40.345 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 05:32:40.346 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:40.346 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:40.346 | INFO | Uploaded 36GiB to GPU in 1.6004s (22.4944GiB/s) | 20260505 05:32:40.346 | INFO | Downloaded 34GiB to GPU in 1.52137s (22.3482GiB/s) | 20260505 05:32:45.484 | INFO | Analyzing chunk frames = [724480,724992) | 20260505 05:32:45.484 | INFO | MemInfo: { | 20260505 05:32:45.484 | INFO | "gpu" : " 17 GB", | 20260505 05:32:45.484 | INFO | "huge" : "", | 20260505 05:32:45.484 | INFO | "huge_shared" : " 27 GB", | 20260505 05:32:45.484 | INFO | "malloc" : " 14 GB", | 20260505 05:32:45.484 | INFO | "pinned" : " 29 GB" | 20260505 05:32:45.484 | INFO | } | 20260505 05:32:45.485 | INFO | Perf report for the last 5.13914 seconds: | 20260505 05:32:45.485 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:32:45.485 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:45.485 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:45.485 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:45.485 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260505 05:32:45.485 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:45.485 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:45.485 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 05:32:45.485 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:45.485 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:45.485 | INFO | Uploaded 36GiB to GPU in 1.59966s (22.5047GiB/s) | 20260505 05:32:45.485 | INFO | Downloaded 34GiB to GPU in 1.52081s (22.3566GiB/s) | 20260505 05:32:45.581 | INFO | Starting a round of Baz IO | 20260505 05:32:48.520 | INFO | Populating RTMetrics report | 20260505 05:32:48.520 | INFO | RTMetrics report populated | 20260505 05:32:48.520 | INFO | Opening tmp RTMetrics json file | 20260505 05:32:48.531 | INFO | Serializing RTMetrics to json | 20260505 05:32:48.534 | INFO | Serialization done, swapping tmp file now... | 20260505 05:32:48.535 | INFO | RTMetrics done | 20260505 05:32:48.627 | INFO | Finished a round of Baz IO: Wrote 2.257 GiB in 3.045 seconds | 20260505 05:32:48.627 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.18% / 0.08% / 61.75% | 20260505 05:32:48.627 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.22804% / 0.000127157% / 3.97364e-05% | 20260505 05:32:48.627 | INFO | Baz IO overall: | 20260505 05:32:48.627 | INFO | BytesWritten : 343779055280 | 20260505 05:32:48.627 | INFO | NumEvents : 161564226203 | 20260505 05:32:50.624 | INFO | Analyzing chunk frames = [724992,725504) | 20260505 05:32:50.624 | INFO | MemInfo: { | 20260505 05:32:50.624 | INFO | "gpu" : " 17 GB", | 20260505 05:32:50.624 | INFO | "huge" : "", | 20260505 05:32:50.624 | INFO | "huge_shared" : " 27 GB", | 20260505 05:32:50.624 | INFO | "malloc" : " 14 GB", | 20260505 05:32:50.624 | INFO | "pinned" : " 29 GB" | 20260505 05:32:50.624 | INFO | } | 20260505 05:32:50.624 | INFO | Perf report for the last 5.13937 seconds: | 20260505 05:32:50.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:32:50.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:50.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:50.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:50.624 | INFO | ANALYSIS : 57.9%, 2.9, 89.2ms, 0 | 20260505 05:32:50.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:50.624 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260505 05:32:50.624 | INFO | PRE_HQ : 57.3%, 2.1, 63.6ms, 0 | 20260505 05:32:50.624 | INFO | BAZWRITER : 58.8%, 4.4, 132.2ms, 0 | 20260505 05:32:50.624 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 05:32:50.624 | INFO | Uploaded 36GiB to GPU in 1.64866s (21.8359GiB/s) | 20260505 05:32:50.624 | INFO | Downloaded 38GiB to GPU in 1.71188s (22.1978GiB/s) | 20260505 05:32:55.763 | INFO | Analyzing chunk frames = [725504,726016) | 20260505 05:32:55.763 | INFO | MemInfo: { | 20260505 05:32:55.763 | INFO | "gpu" : " 17 GB", | 20260505 05:32:55.763 | INFO | "huge" : "", | 20260505 05:32:55.763 | INFO | "huge_shared" : " 27 GB", | 20260505 05:32:55.763 | INFO | "malloc" : " 14 GB", | 20260505 05:32:55.763 | INFO | "pinned" : " 29 GB" | 20260505 05:32:55.763 | INFO | } | 20260505 05:32:55.763 | INFO | Perf report for the last 5.13936 seconds: | 20260505 05:32:55.763 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:32:55.763 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:55.763 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:55.763 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:55.763 | INFO | ANALYSIS : 57.8%, 3.0, 90.1ms, 0 | 20260505 05:32:55.763 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:55.763 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:55.763 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 05:32:55.763 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:55.763 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:32:55.764 | INFO | Uploaded 36GiB to GPU in 1.60005s (22.4993GiB/s) | 20260505 05:32:55.764 | INFO | Downloaded 34GiB to GPU in 1.51632s (22.4227GiB/s) | 20260505 05:33:00.902 | INFO | Analyzing chunk frames = [726016,726528) | 20260505 05:33:00.902 | INFO | MemInfo: { | 20260505 05:33:00.902 | INFO | "gpu" : " 17 GB", | 20260505 05:33:00.903 | INFO | "huge" : "", | 20260505 05:33:00.903 | INFO | "huge_shared" : " 27 GB", | 20260505 05:33:00.903 | INFO | "malloc" : " 14 GB", | 20260505 05:33:00.903 | INFO | "pinned" : " 29 GB" | 20260505 05:33:00.903 | INFO | } | 20260505 05:33:00.903 | INFO | Perf report for the last 5.13928 seconds: | 20260505 05:33:00.903 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:33:00.903 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:00.903 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:00.903 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:00.903 | INFO | ANALYSIS : 57.1%, 3.0, 88.7ms, 0 | 20260505 05:33:00.903 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:00.903 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:00.903 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 05:33:00.903 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:00.903 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:00.903 | INFO | Uploaded 36GiB to GPU in 1.60455s (22.4362GiB/s) | 20260505 05:33:00.903 | INFO | Downloaded 34GiB to GPU in 1.52218s (22.3365GiB/s) | 20260505 05:33:06.042 | INFO | Analyzing chunk frames = [726528,727040) | 20260505 05:33:06.042 | INFO | MemInfo: { | 20260505 05:33:06.042 | INFO | "gpu" : " 17 GB", | 20260505 05:33:06.042 | INFO | "huge" : "", | 20260505 05:33:06.042 | INFO | "huge_shared" : " 27 GB", | 20260505 05:33:06.042 | INFO | "malloc" : " 14 GB", | 20260505 05:33:06.042 | INFO | "pinned" : " 29 GB" | 20260505 05:33:06.042 | INFO | } | 20260505 05:33:06.042 | INFO | Perf report for the last 5.13973 seconds: | 20260505 05:33:06.042 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:33:06.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:06.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:06.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:06.042 | INFO | ANALYSIS : 56.9%, 3.0, 88.2ms, 0 | 20260505 05:33:06.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:06.042 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:06.042 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 05:33:06.042 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:06.042 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:06.043 | INFO | Uploaded 36GiB to GPU in 1.60382s (22.4465GiB/s) | 20260505 05:33:06.043 | INFO | Downloaded 34GiB to GPU in 1.5212s (22.3507GiB/s) | 20260505 05:33:11.181 | INFO | Analyzing chunk frames = [727040,727552) | 20260505 05:33:11.181 | INFO | MemInfo: { | 20260505 05:33:11.181 | INFO | "gpu" : " 17 GB", | 20260505 05:33:11.181 | INFO | "huge" : "", | 20260505 05:33:11.181 | INFO | "huge_shared" : " 27 GB", | 20260505 05:33:11.181 | INFO | "malloc" : " 14 GB", | 20260505 05:33:11.181 | INFO | "pinned" : " 29 GB" | 20260505 05:33:11.181 | INFO | } | 20260505 05:33:11.181 | INFO | Perf report for the last 5.13896 seconds: | 20260505 05:33:11.181 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:33:11.181 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:11.181 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:11.181 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:11.181 | INFO | ANALYSIS : 56.8%, 3.0, 88.5ms, 0 | 20260505 05:33:11.181 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:11.181 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:11.181 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:33:11.181 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:11.181 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:11.182 | INFO | Uploaded 36GiB to GPU in 1.60162s (22.4772GiB/s) | 20260505 05:33:11.182 | INFO | Downloaded 34GiB to GPU in 1.51881s (22.386GiB/s) | 20260505 05:33:16.321 | INFO | Analyzing chunk frames = [727552,728064) | 20260505 05:33:16.321 | INFO | MemInfo: { | 20260505 05:33:16.321 | INFO | "gpu" : " 17 GB", | 20260505 05:33:16.321 | INFO | "huge" : "", | 20260505 05:33:16.321 | INFO | "huge_shared" : " 27 GB", | 20260505 05:33:16.321 | INFO | "malloc" : " 14 GB", | 20260505 05:33:16.321 | INFO | "pinned" : " 29 GB" | 20260505 05:33:16.321 | INFO | } | 20260505 05:33:16.321 | INFO | Perf report for the last 5.13951 seconds: | 20260505 05:33:16.321 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:33:16.321 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:16.321 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:16.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:16.321 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260505 05:33:16.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:16.321 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:16.321 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:33:16.321 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:16.321 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:16.321 | INFO | Uploaded 36GiB to GPU in 1.60246s (22.4655GiB/s) | 20260505 05:33:16.321 | INFO | Downloaded 34GiB to GPU in 1.51891s (22.3844GiB/s) | 20260505 05:33:21.460 | INFO | Analyzing chunk frames = [728064,728576) | 20260505 05:33:21.460 | INFO | MemInfo: { | 20260505 05:33:21.460 | INFO | "gpu" : " 17 GB", | 20260505 05:33:21.460 | INFO | "huge" : "", | 20260505 05:33:21.460 | INFO | "huge_shared" : " 27 GB", | 20260505 05:33:21.460 | INFO | "malloc" : " 14 GB", | 20260505 05:33:21.460 | INFO | "pinned" : " 29 GB" | 20260505 05:33:21.460 | INFO | } | 20260505 05:33:21.460 | INFO | Perf report for the last 5.13929 seconds: | 20260505 05:33:21.460 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:33:21.460 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:21.460 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:21.460 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:21.460 | INFO | ANALYSIS : 57.2%, 3.0, 89.3ms, 0 | 20260505 05:33:21.460 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:21.460 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:21.461 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 05:33:21.461 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:21.461 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:21.461 | INFO | Uploaded 36GiB to GPU in 1.60278s (22.461GiB/s) | 20260505 05:33:21.461 | INFO | Downloaded 34GiB to GPU in 1.52015s (22.3662GiB/s) | 20260505 05:33:26.599 | INFO | Analyzing chunk frames = [728576,729088) | 20260505 05:33:26.599 | INFO | MemInfo: { | 20260505 05:33:26.599 | INFO | "gpu" : " 17 GB", | 20260505 05:33:26.599 | INFO | "huge" : "", | 20260505 05:33:26.599 | INFO | "huge_shared" : " 27 GB", | 20260505 05:33:26.599 | INFO | "malloc" : " 14 GB", | 20260505 05:33:26.599 | INFO | "pinned" : " 29 GB" | 20260505 05:33:26.599 | INFO | } | 20260505 05:33:26.599 | INFO | Perf report for the last 5.13934 seconds: | 20260505 05:33:26.599 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:33:26.599 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:26.600 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:26.600 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:26.600 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260505 05:33:26.600 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:26.600 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:26.600 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 05:33:26.600 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:26.600 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:26.600 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 05:33:26.600 | INFO | Total mem usage: 15.1 GiB | 20260505 05:33:26.600 | INFO | All Time High Water Marks sum: 15.7 GiB (Savings: 4.09%) | 20260505 05:33:26.600 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -3.83%) | 20260505 05:33:26.600 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 05:33:26.600 | INFO | High Water Breakdown: | 20260505 05:33:26.600 | INFO | 60.1% : 8.71 GiB ( 96.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 05:33:26.600 | INFO | 39.8% : 5.77 GiB ( 86.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 05:33:26.600 | INFO | Uploaded 36GiB to GPU in 1.60241s (22.4661GiB/s) | 20260505 05:33:26.600 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 05:33:26.600 | INFO | Downloaded 34GiB to GPU in 1.51962s (22.374GiB/s) | 20260505 05:33:26.600 | INFO | ------------------------------------------ | 20260505 05:33:26.600 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:33:26.600 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 05:33:26.600 | INFO | Total mem usage: 26.5 GiB | 20260505 05:33:26.600 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 05:33:26.600 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 05:33:26.600 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 05:33:26.600 | INFO | High Water Breakdown: | 20260505 05:33:26.600 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:33:26.600 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:33:26.600 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:33:26.600 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:33:26.600 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:33:26.600 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:33:26.600 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 05:33:26.600 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 05:33:26.600 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:33:26.600 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 05:33:26.601 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 05:33:26.601 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 05:33:26.601 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 05:33:26.601 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 05:33:26.601 | INFO | ------------------------------------------ | 20260505 05:33:26.601 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:33:26.601 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 05:33:26.601 | INFO | Total mem usage: 25.6 GiB | 20260505 05:33:26.601 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 05:33:26.601 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 05:33:26.601 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 05:33:26.601 | INFO | High Water Breakdown: | 20260505 05:33:26.601 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 05:33:26.601 | INFO | ------------------------------------------ | 20260505 05:33:26.601 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:33:26.601 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 05:33:26.601 | INFO | Total mem usage: 15.7 GiB | 20260505 05:33:26.601 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 05:33:26.601 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 05:33:26.601 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260505 05:33:26.601 | INFO | High Water Breakdown: | 20260505 05:33:26.601 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 05:33:26.601 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 05:33:26.601 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 05:33:26.601 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 05:33:26.601 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 05:33:26.601 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 05:33:26.601 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 05:33:26.601 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 05:33:26.601 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 05:33:26.601 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:33:26.601 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 05:33:26.601 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:33:26.601 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 05:33:26.601 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 05:33:26.601 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 05:33:26.601 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 05:33:26.601 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:33:26.601 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 05:33:26.601 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:33:26.601 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:33:26.601 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:33:26.601 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 05:33:26.601 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 05:33:26.601 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:33:26.601 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 05:33:26.601 | INFO | ------------------------------------------ | 20260505 05:33:26.601 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:33:26.601 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 05:33:26.601 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:33:26.601 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:33:26.601 | INFO | PerfReport: Full Execution -- 3.03e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260505 05:33:26.601 | INFO | PerfReport: 0.28%, AggregateStats -- 8.62e-02 / -nan, 4.64e-03 / 0.00e+00 | 20260505 05:33:26.601 | INFO | PerfReport: 10.87%, Baseline -- 3.30e+00 / -nan, 1.69e-02 / 0.00e+00 | 20260505 05:33:26.601 | INFO | PerfReport: 3.05%, Histogram -- 9.24e-01 / -nan, 5.16e-02 / 0.00e+00 | 20260505 05:33:26.601 | INFO | PerfReport: 16.04%, DME -- 3.89e+01 / -nan, 3.28e+00 / 0.00e+00 | 20260505 05:33:26.601 | INFO | PerfReport: 56.77%, FrameLabeling -- 1.72e+01 / -nan, 4.62e-02 / 0.00e+00 | 20260505 05:33:26.601 | INFO | PerfReport: 7.53%, PulseAccumulating -- 2.28e+00 / -nan, 1.49e-01 / 0.00e+00 | 20260505 05:33:26.601 | INFO | PerfReport: 3.86%, Metrics -- 1.17e+00 / -nan, 1.93e-01 / 0.00e+00 | 20260505 05:33:26.601 | INFO | PerfReport: 1.58%, PulseSerializing -- 4.80e-01 / -nan, 7.74e-02 / 0.00e+00 | 20260505 05:33:26.601 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 05:33:26.601 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:33:26.601 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:33:26.601 | INFO | PerfReport: Full Execution -- 9.09e+01 / -nan, 1.88e+01 / 0.00e+00 | 20260505 05:33:26.601 | INFO | PerfReport: 18.43%, Upload -- 1.67e+01 / 0.00e+00, 4.07e-01 / 0.00e+00 | 20260505 05:33:26.601 | INFO | PerfReport: 17.71%, Download -- 1.61e+01 / 0.00e+00, 7.46e-01 / 0.00e+00 | 20260505 05:33:26.601 | INFO | PerfReport: 33.36%, Compute -- 3.03e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260505 05:33:26.601 | INFO | PerfReport: 0.56%, LockUpload -- 5.10e-01 / 0.00e+00, 3.63e+00 / 0.00e+00 | 20260505 05:33:26.601 | INFO | PerfReport: 0.00%, LockDownload -- 3.91e-04 / 0.00e+00, 1.23e-04 / 0.00e+00 | 20260505 05:33:26.601 | INFO | PerfReport: 29.92%, LockCompute -- 2.72e+01 / 0.00e+00, 1.68e+01 / 0.00e+00 | 20260505 05:33:26.601 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.84e-02 / 0.00e+00 | 20260505 05:33:26.697 | INFO | Starting a round of Baz IO | 20260505 05:33:29.639 | INFO | Populating RTMetrics report | 20260505 05:33:29.639 | INFO | RTMetrics report populated | 20260505 05:33:29.639 | INFO | Opening tmp RTMetrics json file | 20260505 05:33:29.646 | INFO | Serializing RTMetrics to json | 20260505 05:33:29.647 | INFO | Serialization done, swapping tmp file now... | 20260505 05:33:29.647 | INFO | RTMetrics done | 20260505 05:33:29.748 | INFO | Finished a round of Baz IO: Wrote 2.259 GiB in 3.05 seconds | 20260505 05:33:29.748 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.10% / 0.08% / 61.82% | 20260505 05:33:29.748 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.19458% / 0.000147025% / 5.96046e-05% | 20260505 05:33:29.748 | INFO | Baz IO overall: | 20260505 05:33:29.748 | INFO | BytesWritten : 346205100446 | 20260505 05:33:29.748 | INFO | NumEvents : 162721206159 | 20260505 05:33:31.738 | INFO | Analyzing chunk frames = [729088,729600) | 20260505 05:33:31.738 | INFO | MemInfo: { | 20260505 05:33:31.738 | INFO | "gpu" : " 17 GB", | 20260505 05:33:31.738 | INFO | "huge" : "", | 20260505 05:33:31.738 | INFO | "huge_shared" : " 27 GB", | 20260505 05:33:31.738 | INFO | "malloc" : " 14 GB", | 20260505 05:33:31.738 | INFO | "pinned" : " 29 GB" | 20260505 05:33:31.738 | INFO | } | 20260505 05:33:31.738 | INFO | Perf report for the last 5.13896 seconds: | 20260505 05:33:31.739 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:33:31.739 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:31.739 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:31.739 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:31.739 | INFO | ANALYSIS : 58.0%, 2.9, 88.6ms, 0 | 20260505 05:33:31.739 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:31.739 | INFO | RT_METRICS : 45.9%, 1.0, 24.0ms, 0 | 20260505 05:33:31.739 | INFO | PRE_HQ : 57.3%, 2.1, 63.8ms, 0 | 20260505 05:33:31.739 | INFO | BAZWRITER : 58.9%, 4.4, 132.2ms, 0 | 20260505 05:33:31.739 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 05:33:31.739 | INFO | Uploaded 36GiB to GPU in 1.65178s (21.7946GiB/s) | 20260505 05:33:31.739 | INFO | Downloaded 38GiB to GPU in 1.71746s (22.1257GiB/s) | 20260505 05:33:36.878 | INFO | Analyzing chunk frames = [729600,730112) | 20260505 05:33:36.878 | INFO | MemInfo: { | 20260505 05:33:36.878 | INFO | "gpu" : " 17 GB", | 20260505 05:33:36.878 | INFO | "huge" : "", | 20260505 05:33:36.878 | INFO | "huge_shared" : " 27 GB", | 20260505 05:33:36.878 | INFO | "malloc" : " 14 GB", | 20260505 05:33:36.878 | INFO | "pinned" : " 29 GB" | 20260505 05:33:36.878 | INFO | } | 20260505 05:33:36.878 | INFO | Perf report for the last 5.13971 seconds: | 20260505 05:33:36.878 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:33:36.878 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:36.878 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:36.878 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:36.878 | INFO | ANALYSIS : 57.7%, 3.0, 90.0ms, 0 | 20260505 05:33:36.878 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:36.878 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:36.878 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 05:33:36.878 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:36.878 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:36.879 | INFO | Uploaded 36GiB to GPU in 1.59811s (22.5267GiB/s) | 20260505 05:33:36.879 | INFO | Downloaded 34GiB to GPU in 1.5181s (22.3965GiB/s) | 20260505 05:33:42.017 | INFO | Analyzing chunk frames = [730112,730624) | 20260505 05:33:42.017 | INFO | MemInfo: { | 20260505 05:33:42.017 | INFO | "gpu" : " 17 GB", | 20260505 05:33:42.017 | INFO | "huge" : "", | 20260505 05:33:42.017 | INFO | "huge_shared" : " 27 GB", | 20260505 05:33:42.017 | INFO | "malloc" : " 14 GB", | 20260505 05:33:42.017 | INFO | "pinned" : " 29 GB" | 20260505 05:33:42.017 | INFO | } | 20260505 05:33:42.017 | INFO | Perf report for the last 5.13916 seconds: | 20260505 05:33:42.017 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:33:42.017 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:42.017 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:42.017 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:42.017 | INFO | ANALYSIS : 57.1%, 3.0, 88.8ms, 0 | 20260505 05:33:42.017 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:42.017 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:42.017 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 05:33:42.017 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:42.017 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:42.018 | INFO | Uploaded 36GiB to GPU in 1.60157s (22.478GiB/s) | 20260505 05:33:42.018 | INFO | Downloaded 34GiB to GPU in 1.52258s (22.3305GiB/s) | 20260505 05:33:47.157 | INFO | Analyzing chunk frames = [730624,731136) | 20260505 05:33:47.157 | INFO | MemInfo: { | 20260505 05:33:47.157 | INFO | "gpu" : " 17 GB", | 20260505 05:33:47.157 | INFO | "huge" : "", | 20260505 05:33:47.157 | INFO | "huge_shared" : " 27 GB", | 20260505 05:33:47.157 | INFO | "malloc" : " 14 GB", | 20260505 05:33:47.157 | INFO | "pinned" : " 29 GB" | 20260505 05:33:47.157 | INFO | } | 20260505 05:33:47.157 | INFO | Perf report for the last 5.1393 seconds: | 20260505 05:33:47.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:33:47.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:47.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:47.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:47.157 | INFO | ANALYSIS : 56.8%, 3.0, 88.1ms, 0 | 20260505 05:33:47.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:47.157 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:47.157 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:33:47.157 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:47.157 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:47.157 | INFO | Uploaded 36GiB to GPU in 1.60227s (22.4681GiB/s) | 20260505 05:33:47.157 | INFO | Downloaded 34GiB to GPU in 1.52279s (22.3274GiB/s) | 20260505 05:33:52.296 | INFO | Analyzing chunk frames = [731136,731648) | 20260505 05:33:52.296 | INFO | MemInfo: { | 20260505 05:33:52.296 | INFO | "gpu" : " 17 GB", | 20260505 05:33:52.296 | INFO | "huge" : "", | 20260505 05:33:52.296 | INFO | "huge_shared" : " 27 GB", | 20260505 05:33:52.296 | INFO | "malloc" : " 14 GB", | 20260505 05:33:52.296 | INFO | "pinned" : " 29 GB" | 20260505 05:33:52.296 | INFO | } | 20260505 05:33:52.296 | INFO | Perf report for the last 5.13958 seconds: | 20260505 05:33:52.296 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:33:52.296 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:52.296 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:52.296 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:52.296 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260505 05:33:52.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:52.296 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:52.296 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:33:52.296 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:52.296 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:52.297 | INFO | Uploaded 36GiB to GPU in 1.59975s (22.5035GiB/s) | 20260505 05:33:52.297 | INFO | Downloaded 34GiB to GPU in 1.51936s (22.3778GiB/s) | 20260505 05:33:57.435 | INFO | Analyzing chunk frames = [731648,732160) | 20260505 05:33:57.435 | INFO | MemInfo: { | 20260505 05:33:57.435 | INFO | "gpu" : " 17 GB", | 20260505 05:33:57.435 | INFO | "huge" : "", | 20260505 05:33:57.435 | INFO | "huge_shared" : " 27 GB", | 20260505 05:33:57.435 | INFO | "malloc" : " 14 GB", | 20260505 05:33:57.435 | INFO | "pinned" : " 29 GB" | 20260505 05:33:57.435 | INFO | } | 20260505 05:33:57.435 | INFO | Perf report for the last 5.13916 seconds: | 20260505 05:33:57.435 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:33:57.435 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:57.435 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:57.435 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:57.435 | INFO | ANALYSIS : 57.0%, 3.0, 89.0ms, 0 | 20260505 05:33:57.436 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:57.436 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:57.436 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:33:57.436 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:57.436 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:33:57.436 | INFO | Uploaded 36GiB to GPU in 1.60012s (22.4984GiB/s) | 20260505 05:33:57.436 | INFO | Downloaded 34GiB to GPU in 1.52112s (22.352GiB/s) | 20260505 05:34:02.575 | INFO | Analyzing chunk frames = [732160,732672) | 20260505 05:34:02.575 | INFO | MemInfo: { | 20260505 05:34:02.575 | INFO | "gpu" : " 17 GB", | 20260505 05:34:02.575 | INFO | "huge" : "", | 20260505 05:34:02.575 | INFO | "huge_shared" : " 27 GB", | 20260505 05:34:02.575 | INFO | "malloc" : " 14 GB", | 20260505 05:34:02.575 | INFO | "pinned" : " 29 GB" | 20260505 05:34:02.575 | INFO | } | 20260505 05:34:02.575 | INFO | Perf report for the last 5.13933 seconds: | 20260505 05:34:02.575 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:34:02.575 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:02.575 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:02.575 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:02.575 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 05:34:02.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:02.575 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:02.575 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260505 05:34:02.575 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:02.575 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:02.575 | INFO | Uploaded 36GiB to GPU in 1.60002s (22.4998GiB/s) | 20260505 05:34:02.575 | INFO | Downloaded 34GiB to GPU in 1.52121s (22.3506GiB/s) | 20260505 05:34:07.714 | INFO | Analyzing chunk frames = [732672,733184) | 20260505 05:34:07.714 | INFO | MemInfo: { | 20260505 05:34:07.714 | INFO | "gpu" : " 17 GB", | 20260505 05:34:07.714 | INFO | "huge" : "", | 20260505 05:34:07.714 | INFO | "huge_shared" : " 27 GB", | 20260505 05:34:07.714 | INFO | "malloc" : " 14 GB", | 20260505 05:34:07.714 | INFO | "pinned" : " 29 GB" | 20260505 05:34:07.714 | INFO | } | 20260505 05:34:07.714 | INFO | Perf report for the last 5.13957 seconds: | 20260505 05:34:07.714 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:34:07.714 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:07.714 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:07.714 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:07.714 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 05:34:07.714 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:07.714 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:07.714 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 05:34:07.714 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:07.715 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:07.715 | INFO | Uploaded 36GiB to GPU in 1.60034s (22.4952GiB/s) | 20260505 05:34:07.715 | INFO | Downloaded 34GiB to GPU in 1.52062s (22.3594GiB/s) | 20260505 05:34:07.812 | INFO | Starting a round of Baz IO | 20260505 05:34:10.754 | INFO | Populating RTMetrics report | 20260505 05:34:10.754 | INFO | RTMetrics report populated | 20260505 05:34:10.754 | INFO | Opening tmp RTMetrics json file | 20260505 05:34:10.763 | INFO | Serializing RTMetrics to json | 20260505 05:34:10.764 | INFO | Serialization done, swapping tmp file now... | 20260505 05:34:10.765 | INFO | RTMetrics done | 20260505 05:34:10.862 | INFO | Finished a round of Baz IO: Wrote 2.26 GiB in 3.049 seconds | 20260505 05:34:10.862 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.03% / 0.08% / 61.90% | 20260505 05:34:10.862 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.15002% / 0.000154972% / 7.54992e-05% | 20260505 05:34:10.862 | INFO | Baz IO overall: | 20260505 05:34:10.862 | INFO | BytesWritten : 348632187068 | 20260505 05:34:10.862 | INFO | NumEvents : 163878111909 | 20260505 05:34:12.853 | INFO | Analyzing chunk frames = [733184,733696) | 20260505 05:34:12.853 | INFO | MemInfo: { | 20260505 05:34:12.853 | INFO | "gpu" : " 17 GB", | 20260505 05:34:12.853 | INFO | "huge" : "", | 20260505 05:34:12.853 | INFO | "huge_shared" : " 27 GB", | 20260505 05:34:12.853 | INFO | "malloc" : " 14 GB", | 20260505 05:34:12.853 | INFO | "pinned" : " 29 GB" | 20260505 05:34:12.853 | INFO | } | 20260505 05:34:12.854 | INFO | Perf report for the last 5.13915 seconds: | 20260505 05:34:12.854 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:34:12.854 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:12.854 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:12.854 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:12.854 | INFO | ANALYSIS : 58.0%, 2.9, 89.5ms, 0 | 20260505 05:34:12.854 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:12.854 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260505 05:34:12.854 | INFO | PRE_HQ : 57.2%, 2.1, 63.4ms, 0 | 20260505 05:34:12.854 | INFO | BAZWRITER : 58.9%, 4.4, 131.8ms, 0 | 20260505 05:34:12.854 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260505 05:34:12.854 | INFO | Uploaded 36GiB to GPU in 1.65054s (21.8111GiB/s) | 20260505 05:34:12.854 | INFO | Downloaded 38GiB to GPU in 1.71454s (22.1633GiB/s) | 20260505 05:34:17.993 | INFO | Analyzing chunk frames = [733696,734208) | 20260505 05:34:17.993 | INFO | MemInfo: { | 20260505 05:34:17.993 | INFO | "gpu" : " 17 GB", | 20260505 05:34:17.993 | INFO | "huge" : "", | 20260505 05:34:17.993 | INFO | "huge_shared" : " 27 GB", | 20260505 05:34:17.993 | INFO | "malloc" : " 14 GB", | 20260505 05:34:17.993 | INFO | "pinned" : " 29 GB" | 20260505 05:34:17.993 | INFO | } | 20260505 05:34:17.993 | INFO | Perf report for the last 5.13925 seconds: | 20260505 05:34:17.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:34:17.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:17.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:17.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:17.993 | INFO | ANALYSIS : 57.7%, 3.0, 90.1ms, 0 | 20260505 05:34:17.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:17.993 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:17.993 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:34:17.993 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:17.993 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:17.994 | INFO | Uploaded 36GiB to GPU in 1.59891s (22.5154GiB/s) | 20260505 05:34:17.994 | INFO | Downloaded 34GiB to GPU in 1.51921s (22.3801GiB/s) | 20260505 05:34:23.132 | INFO | Analyzing chunk frames = [734208,734720) | 20260505 05:34:23.132 | INFO | MemInfo: { | 20260505 05:34:23.132 | INFO | "gpu" : " 17 GB", | 20260505 05:34:23.132 | INFO | "huge" : "", | 20260505 05:34:23.132 | INFO | "huge_shared" : " 27 GB", | 20260505 05:34:23.132 | INFO | "malloc" : " 14 GB", | 20260505 05:34:23.132 | INFO | "pinned" : " 29 GB" | 20260505 05:34:23.132 | INFO | } | 20260505 05:34:23.132 | INFO | Perf report for the last 5.13936 seconds: | 20260505 05:34:23.132 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:34:23.132 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:23.132 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:23.132 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:23.132 | INFO | ANALYSIS : 57.1%, 3.0, 88.8ms, 0 | 20260505 05:34:23.132 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:23.132 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:23.132 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 05:34:23.132 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:23.132 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:23.133 | INFO | Uploaded 36GiB to GPU in 1.60148s (22.4792GiB/s) | 20260505 05:34:23.133 | INFO | Downloaded 34GiB to GPU in 1.52184s (22.3414GiB/s) | 20260505 05:34:28.271 | INFO | Analyzing chunk frames = [734720,735232) | 20260505 05:34:28.271 | INFO | MemInfo: { | 20260505 05:34:28.271 | INFO | "gpu" : " 17 GB", | 20260505 05:34:28.271 | INFO | "huge" : "", | 20260505 05:34:28.271 | INFO | "huge_shared" : " 27 GB", | 20260505 05:34:28.271 | INFO | "malloc" : " 14 GB", | 20260505 05:34:28.272 | INFO | "pinned" : " 29 GB" | 20260505 05:34:28.272 | INFO | } | 20260505 05:34:28.272 | INFO | Perf report for the last 5.13934 seconds: | 20260505 05:34:28.272 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:34:28.272 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:28.272 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:28.272 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:28.272 | INFO | ANALYSIS : 56.8%, 3.0, 88.1ms, 0 | 20260505 05:34:28.272 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:28.272 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:28.272 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 05:34:28.272 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:28.272 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:28.272 | INFO | Uploaded 36GiB to GPU in 1.60138s (22.4806GiB/s) | 20260505 05:34:28.272 | INFO | Downloaded 34GiB to GPU in 1.52253s (22.3312GiB/s) | 20260505 05:34:33.411 | INFO | Analyzing chunk frames = [735232,735744) | 20260505 05:34:33.411 | INFO | MemInfo: { | 20260505 05:34:33.411 | INFO | "gpu" : " 17 GB", | 20260505 05:34:33.411 | INFO | "huge" : "", | 20260505 05:34:33.411 | INFO | "huge_shared" : " 27 GB", | 20260505 05:34:33.411 | INFO | "malloc" : " 14 GB", | 20260505 05:34:33.411 | INFO | "pinned" : " 29 GB" | 20260505 05:34:33.411 | INFO | } | 20260505 05:34:33.411 | INFO | Perf report for the last 5.13956 seconds: | 20260505 05:34:33.411 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:34:33.411 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:33.411 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:33.411 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:33.411 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260505 05:34:33.411 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:33.411 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:33.411 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 05:34:33.411 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:33.411 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:33.411 | INFO | Uploaded 36GiB to GPU in 1.59983s (22.5024GiB/s) | 20260505 05:34:33.411 | INFO | Downloaded 34GiB to GPU in 1.52038s (22.3628GiB/s) | 20260505 05:34:38.550 | INFO | Analyzing chunk frames = [735744,736256) | 20260505 05:34:38.550 | INFO | MemInfo: { | 20260505 05:34:38.550 | INFO | "gpu" : " 17 GB", | 20260505 05:34:38.550 | INFO | "huge" : "", | 20260505 05:34:38.550 | INFO | "huge_shared" : " 27 GB", | 20260505 05:34:38.550 | INFO | "malloc" : " 14 GB", | 20260505 05:34:38.550 | INFO | "pinned" : " 29 GB" | 20260505 05:34:38.550 | INFO | } | 20260505 05:34:38.550 | INFO | Perf report for the last 5.1391 seconds: | 20260505 05:34:38.550 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:34:38.550 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:38.550 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:38.550 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:38.550 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260505 05:34:38.550 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:38.550 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:38.550 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260505 05:34:38.550 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:38.550 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:38.551 | INFO | Uploaded 36GiB to GPU in 1.59965s (22.5049GiB/s) | 20260505 05:34:38.551 | INFO | Downloaded 34GiB to GPU in 1.51991s (22.3698GiB/s) | 20260505 05:34:43.689 | INFO | Analyzing chunk frames = [736256,736768) | 20260505 05:34:43.689 | INFO | MemInfo: { | 20260505 05:34:43.689 | INFO | "gpu" : " 17 GB", | 20260505 05:34:43.689 | INFO | "huge" : "", | 20260505 05:34:43.689 | INFO | "huge_shared" : " 27 GB", | 20260505 05:34:43.689 | INFO | "malloc" : " 14 GB", | 20260505 05:34:43.690 | INFO | "pinned" : " 29 GB" | 20260505 05:34:43.690 | INFO | } | 20260505 05:34:43.690 | INFO | Perf report for the last 5.13939 seconds: | 20260505 05:34:43.690 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:34:43.690 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:43.690 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:43.690 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:43.690 | INFO | ANALYSIS : 57.3%, 3.0, 89.3ms, 0 | 20260505 05:34:43.690 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:43.690 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:43.690 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:34:43.690 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:43.690 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:43.690 | INFO | Uploaded 36GiB to GPU in 1.6002s (22.4972GiB/s) | 20260505 05:34:43.690 | INFO | Downloaded 34GiB to GPU in 1.52109s (22.3524GiB/s) | 20260505 05:34:48.829 | INFO | Analyzing chunk frames = [736768,737280) | 20260505 05:34:48.829 | INFO | MemInfo: { | 20260505 05:34:48.829 | INFO | "gpu" : " 17 GB", | 20260505 05:34:48.829 | INFO | "huge" : "", | 20260505 05:34:48.829 | INFO | "huge_shared" : " 27 GB", | 20260505 05:34:48.829 | INFO | "malloc" : " 14 GB", | 20260505 05:34:48.829 | INFO | "pinned" : " 29 GB" | 20260505 05:34:48.829 | INFO | } | 20260505 05:34:48.829 | INFO | Perf report for the last 5.13937 seconds: | 20260505 05:34:48.829 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:34:48.829 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:48.829 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:48.829 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:48.829 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 05:34:48.829 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:48.829 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:48.829 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 05:34:48.829 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:48.829 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:48.829 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 05:34:48.829 | INFO | Total mem usage: 15.1 GiB | 20260505 05:34:48.829 | INFO | All Time High Water Marks sum: 15.7 GiB (Savings: 4.09%) | 20260505 05:34:48.829 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -4.24%) | 20260505 05:34:48.829 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 05:34:48.829 | INFO | High Water Breakdown: | 20260505 05:34:48.829 | INFO | 59.9% : 8.66 GiB ( 95.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 05:34:48.829 | INFO | 39.9% : 5.77 GiB ( 86.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 05:34:48.829 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 05:34:48.829 | INFO | ------------------------------------------ | 20260505 05:34:48.829 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:34:48.829 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 05:34:48.829 | INFO | Total mem usage: 26.5 GiB | 20260505 05:34:48.829 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 05:34:48.829 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 05:34:48.829 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 05:34:48.829 | INFO | High Water Breakdown: | 20260505 05:34:48.829 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:34:48.829 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:34:48.829 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:34:48.829 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:34:48.829 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:34:48.829 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:34:48.829 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 05:34:48.829 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 05:34:48.829 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:34:48.830 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 05:34:48.830 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 05:34:48.830 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 05:34:48.830 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 05:34:48.830 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 05:34:48.830 | INFO | ------------------------------------------ | 20260505 05:34:48.830 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:34:48.830 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 05:34:48.830 | INFO | Total mem usage: 25.6 GiB | 20260505 05:34:48.830 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 05:34:48.830 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 05:34:48.830 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 05:34:48.830 | INFO | High Water Breakdown: | 20260505 05:34:48.830 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 05:34:48.830 | INFO | ------------------------------------------ | 20260505 05:34:48.830 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:34:48.830 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 05:34:48.830 | INFO | Total mem usage: 15.7 GiB | 20260505 05:34:48.830 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 05:34:48.830 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 05:34:48.830 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260505 05:34:48.830 | INFO | High Water Breakdown: | 20260505 05:34:48.830 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 05:34:48.830 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 05:34:48.830 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 05:34:48.830 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 05:34:48.830 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 05:34:48.830 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 05:34:48.830 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 05:34:48.830 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 05:34:48.830 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 05:34:48.830 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:34:48.830 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 05:34:48.830 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:34:48.830 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 05:34:48.830 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 05:34:48.830 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 05:34:48.830 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 05:34:48.830 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:34:48.830 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 05:34:48.830 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:34:48.830 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:34:48.830 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:34:48.830 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 05:34:48.830 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 05:34:48.830 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:34:48.830 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 05:34:48.830 | INFO | ------------------------------------------ | 20260505 05:34:48.830 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:34:48.830 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 05:34:48.830 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:34:48.830 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:34:48.830 | INFO | PerfReport: Full Execution -- 3.03e+01 / -nan, 1.28e+01 / 0.00e+00 | 20260505 05:34:48.830 | INFO | PerfReport: 0.29%, AggregateStats -- 8.64e-02 / -nan, 4.04e-03 / 0.00e+00 | 20260505 05:34:48.830 | INFO | PerfReport: 10.87%, Baseline -- 3.30e+00 / -nan, 1.79e-02 / 0.00e+00 | 20260505 05:34:48.830 | INFO | PerfReport: 3.05%, Histogram -- 9.25e-01 / -nan, 5.26e-02 / 0.00e+00 | 20260505 05:34:48.830 | INFO | PerfReport: 16.04%, DME -- 3.89e+01 / -nan, 3.21e+00 / 0.00e+00 | 20260505 05:34:48.830 | INFO | PerfReport: 56.78%, FrameLabeling -- 1.72e+01 / -nan, 4.65e-02 / 0.00e+00 | 20260505 05:34:48.830 | INFO | Uploaded 36GiB to GPU in 1.60061s (22.4915GiB/s) | 20260505 05:34:48.830 | INFO | PerfReport: 7.53%, PulseAccumulating -- 2.28e+00 / -nan, 1.49e-01 / 0.00e+00 | 20260505 05:34:48.830 | INFO | Downloaded 34GiB to GPU in 1.52166s (22.3441GiB/s) | 20260505 05:34:48.831 | INFO | PerfReport: 3.86%, Metrics -- 1.17e+00 / -nan, 1.94e-01 / 0.00e+00 | 20260505 05:34:48.831 | INFO | PerfReport: 1.58%, PulseSerializing -- 4.79e-01 / -nan, 7.75e-02 / 0.00e+00 | 20260505 05:34:48.831 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 05:34:48.831 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:34:48.831 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:34:48.831 | INFO | PerfReport: Full Execution -- 9.09e+01 / -nan, 1.86e+01 / 0.00e+00 | 20260505 05:34:48.831 | INFO | PerfReport: 18.42%, Upload -- 1.67e+01 / 0.00e+00, 4.12e-01 / 0.00e+00 | 20260505 05:34:48.831 | INFO | PerfReport: 17.71%, Download -- 1.61e+01 / 0.00e+00, 7.51e-01 / 0.00e+00 | 20260505 05:34:48.831 | INFO | PerfReport: 33.36%, Compute -- 3.03e+01 / 0.00e+00, 1.28e+01 / 0.00e+00 | 20260505 05:34:48.831 | INFO | PerfReport: 0.56%, LockUpload -- 5.08e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260505 05:34:48.831 | INFO | PerfReport: 0.00%, LockDownload -- 3.95e-04 / 0.00e+00, 9.32e-05 / 0.00e+00 | 20260505 05:34:48.831 | INFO | PerfReport: 29.94%, LockCompute -- 2.72e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260505 05:34:48.831 | INFO | PerfReport: 0.02%, Misc -- 1.41e-02 / 0.00e+00, 2.03e-02 / 0.00e+00 | 20260505 05:34:48.928 | INFO | Starting a round of Baz IO | 20260505 05:34:51.869 | INFO | Populating RTMetrics report | 20260505 05:34:51.869 | INFO | RTMetrics report populated | 20260505 05:34:51.869 | INFO | Opening tmp RTMetrics json file | 20260505 05:34:51.880 | INFO | Serializing RTMetrics to json | 20260505 05:34:51.882 | INFO | Serialization done, swapping tmp file now... | 20260505 05:34:51.884 | INFO | RTMetrics done | 20260505 05:34:51.978 | INFO | Finished a round of Baz IO: Wrote 2.262 GiB in 3.05 seconds | 20260505 05:34:51.978 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.95% / 0.07% / 61.97% | 20260505 05:34:51.978 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.11963% / 0.000107288% / 4.37101e-05% | 20260505 05:34:51.978 | INFO | Baz IO overall: | 20260505 05:34:51.978 | INFO | BytesWritten : 351061473792 | 20260505 05:34:51.978 | INFO | NumEvents : 165036557254 | 20260505 05:34:53.968 | INFO | Analyzing chunk frames = [737280,737792) | 20260505 05:34:53.968 | INFO | MemInfo: { | 20260505 05:34:53.968 | INFO | "gpu" : " 17 GB", | 20260505 05:34:53.968 | INFO | "huge" : "", | 20260505 05:34:53.968 | INFO | "huge_shared" : " 27 GB", | 20260505 05:34:53.968 | INFO | "malloc" : " 14 GB", | 20260505 05:34:53.968 | INFO | "pinned" : " 29 GB" | 20260505 05:34:53.968 | INFO | } | 20260505 05:34:53.968 | INFO | Perf report for the last 5.13908 seconds: | 20260505 05:34:53.968 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:34:53.968 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:53.968 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:53.968 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:53.968 | INFO | ANALYSIS : 58.0%, 2.9, 88.7ms, 0 | 20260505 05:34:53.968 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:53.968 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260505 05:34:53.968 | INFO | PRE_HQ : 57.3%, 2.1, 63.7ms, 0 | 20260505 05:34:53.968 | INFO | BAZWRITER : 58.9%, 4.4, 133.3ms, 0 | 20260505 05:34:53.968 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 05:34:53.968 | INFO | Uploaded 36GiB to GPU in 1.65049s (21.8117GiB/s) | 20260505 05:34:53.968 | INFO | Downloaded 38GiB to GPU in 1.71479s (22.1602GiB/s) | 20260505 05:34:59.108 | INFO | Analyzing chunk frames = [737792,738304) | 20260505 05:34:59.108 | INFO | MemInfo: { | 20260505 05:34:59.108 | INFO | "gpu" : " 17 GB", | 20260505 05:34:59.108 | INFO | "huge" : "", | 20260505 05:34:59.108 | INFO | "huge_shared" : " 27 GB", | 20260505 05:34:59.108 | INFO | "malloc" : " 14 GB", | 20260505 05:34:59.108 | INFO | "pinned" : " 29 GB" | 20260505 05:34:59.108 | INFO | } | 20260505 05:34:59.108 | INFO | Perf report for the last 5.13985 seconds: | 20260505 05:34:59.108 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:34:59.108 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:59.108 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:59.108 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:59.108 | INFO | ANALYSIS : 57.8%, 3.0, 90.1ms, 0 | 20260505 05:34:59.108 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:59.108 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:59.108 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 05:34:59.108 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:59.108 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:34:59.108 | INFO | Uploaded 36GiB to GPU in 1.59817s (22.5258GiB/s) | 20260505 05:34:59.108 | INFO | Downloaded 34GiB to GPU in 1.51815s (22.3957GiB/s) | 20260505 05:35:04.247 | INFO | Analyzing chunk frames = [738304,738816) | 20260505 05:35:04.247 | INFO | MemInfo: { | 20260505 05:35:04.247 | INFO | "gpu" : " 17 GB", | 20260505 05:35:04.247 | INFO | "huge" : "", | 20260505 05:35:04.247 | INFO | "huge_shared" : " 27 GB", | 20260505 05:35:04.247 | INFO | "malloc" : " 14 GB", | 20260505 05:35:04.247 | INFO | "pinned" : " 29 GB" | 20260505 05:35:04.247 | INFO | } | 20260505 05:35:04.247 | INFO | Perf report for the last 5.13887 seconds: | 20260505 05:35:04.247 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:35:04.247 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:04.247 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:04.247 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:04.247 | INFO | ANALYSIS : 57.2%, 3.0, 88.9ms, 0 | 20260505 05:35:04.247 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:04.247 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:04.247 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 05:35:04.247 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:04.247 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:04.247 | INFO | Uploaded 36GiB to GPU in 1.60148s (22.4792GiB/s) | 20260505 05:35:04.247 | INFO | Downloaded 34GiB to GPU in 1.52265s (22.3296GiB/s) | 20260505 05:35:09.386 | INFO | Analyzing chunk frames = [738816,739328) | 20260505 05:35:09.386 | INFO | MemInfo: { | 20260505 05:35:09.386 | INFO | "gpu" : " 17 GB", | 20260505 05:35:09.386 | INFO | "huge" : "", | 20260505 05:35:09.386 | INFO | "huge_shared" : " 27 GB", | 20260505 05:35:09.386 | INFO | "malloc" : " 14 GB", | 20260505 05:35:09.386 | INFO | "pinned" : " 29 GB" | 20260505 05:35:09.386 | INFO | } | 20260505 05:35:09.386 | INFO | Perf report for the last 5.13959 seconds: | 20260505 05:35:09.387 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:35:09.387 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:09.387 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:09.387 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:09.387 | INFO | ANALYSIS : 56.8%, 3.0, 88.1ms, 0 | 20260505 05:35:09.387 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:09.387 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:09.387 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 05:35:09.387 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:09.387 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:09.387 | INFO | Uploaded 36GiB to GPU in 1.6017s (22.4761GiB/s) | 20260505 05:35:09.387 | INFO | Downloaded 34GiB to GPU in 1.52231s (22.3345GiB/s) | 20260505 05:35:14.526 | INFO | Analyzing chunk frames = [739328,739840) | 20260505 05:35:14.526 | INFO | MemInfo: { | 20260505 05:35:14.526 | INFO | "gpu" : " 17 GB", | 20260505 05:35:14.526 | INFO | "huge" : "", | 20260505 05:35:14.526 | INFO | "huge_shared" : " 27 GB", | 20260505 05:35:14.526 | INFO | "malloc" : " 14 GB", | 20260505 05:35:14.526 | INFO | "pinned" : " 29 GB" | 20260505 05:35:14.526 | INFO | } | 20260505 05:35:14.526 | INFO | Perf report for the last 5.13966 seconds: | 20260505 05:35:14.526 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:35:14.526 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:14.526 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:14.526 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:14.526 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260505 05:35:14.526 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:14.526 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:14.526 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 05:35:14.526 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:14.526 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:14.526 | INFO | Uploaded 36GiB to GPU in 1.59971s (22.5041GiB/s) | 20260505 05:35:14.526 | INFO | Downloaded 34GiB to GPU in 1.51934s (22.3782GiB/s) | 20260505 05:35:19.665 | INFO | Analyzing chunk frames = [739840,740352) | 20260505 05:35:19.665 | INFO | MemInfo: { | 20260505 05:35:19.665 | INFO | "gpu" : " 17 GB", | 20260505 05:35:19.665 | INFO | "huge" : "", | 20260505 05:35:19.665 | INFO | "huge_shared" : " 27 GB", | 20260505 05:35:19.665 | INFO | "malloc" : " 14 GB", | 20260505 05:35:19.665 | INFO | "pinned" : " 29 GB" | 20260505 05:35:19.665 | INFO | } | 20260505 05:35:19.665 | INFO | Perf report for the last 5.13892 seconds: | 20260505 05:35:19.665 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:35:19.665 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:19.665 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:19.665 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:19.665 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260505 05:35:19.665 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:19.665 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:19.665 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:35:19.665 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:19.665 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:19.666 | INFO | Uploaded 36GiB to GPU in 1.59969s (22.5043GiB/s) | 20260505 05:35:19.666 | INFO | Downloaded 34GiB to GPU in 1.51886s (22.3852GiB/s) | 20260505 05:35:24.804 | INFO | Analyzing chunk frames = [740352,740864) | 20260505 05:35:24.804 | INFO | MemInfo: { | 20260505 05:35:24.804 | INFO | "gpu" : " 17 GB", | 20260505 05:35:24.804 | INFO | "huge" : "", | 20260505 05:35:24.804 | INFO | "huge_shared" : " 27 GB", | 20260505 05:35:24.804 | INFO | "malloc" : " 14 GB", | 20260505 05:35:24.804 | INFO | "pinned" : " 29 GB" | 20260505 05:35:24.804 | INFO | } | 20260505 05:35:24.804 | INFO | Perf report for the last 5.13923 seconds: | 20260505 05:35:24.804 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:35:24.804 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:24.804 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:24.804 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:24.804 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260505 05:35:24.804 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:24.804 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:24.804 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:35:24.804 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:24.805 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:24.805 | INFO | Uploaded 36GiB to GPU in 1.60041s (22.4943GiB/s) | 20260505 05:35:24.805 | INFO | Downloaded 34GiB to GPU in 1.52113s (22.3518GiB/s) | 20260505 05:35:29.944 | INFO | Analyzing chunk frames = [740864,741376) | 20260505 05:35:29.944 | INFO | MemInfo: { | 20260505 05:35:29.944 | INFO | "gpu" : " 17 GB", | 20260505 05:35:29.944 | INFO | "huge" : "", | 20260505 05:35:29.944 | INFO | "huge_shared" : " 27 GB", | 20260505 05:35:29.944 | INFO | "malloc" : " 14 GB", | 20260505 05:35:29.944 | INFO | "pinned" : " 29 GB" | 20260505 05:35:29.944 | INFO | } | 20260505 05:35:29.944 | INFO | Perf report for the last 5.13953 seconds: | 20260505 05:35:29.944 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:35:29.944 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:29.944 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:29.944 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:29.944 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260505 05:35:29.944 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:29.944 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:29.944 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 05:35:29.944 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:29.944 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:29.945 | INFO | Uploaded 36GiB to GPU in 1.59992s (22.5011GiB/s) | 20260505 05:35:29.945 | INFO | Downloaded 34GiB to GPU in 1.52053s (22.3606GiB/s) | 20260505 05:35:30.041 | INFO | Starting a round of Baz IO | 20260505 05:35:32.979 | INFO | Populating RTMetrics report | 20260505 05:35:32.980 | INFO | RTMetrics report populated | 20260505 05:35:32.980 | INFO | Opening tmp RTMetrics json file | 20260505 05:35:32.989 | INFO | Serializing RTMetrics to json | 20260505 05:35:32.990 | INFO | Serialization done, swapping tmp file now... | 20260505 05:35:32.990 | INFO | RTMetrics done | 20260505 05:35:33.087 | INFO | Finished a round of Baz IO: Wrote 2.265 GiB in 3.046 seconds | 20260505 05:35:33.087 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.88% / 0.07% / 62.05% | 20260505 05:35:33.087 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.09525% / 0.000127157% / 4.76837e-05% | 20260505 05:35:33.087 | INFO | Baz IO overall: | 20260505 05:35:33.087 | INFO | BytesWritten : 353493529905 | 20260505 05:35:33.087 | INFO | NumEvents : 166196947150 | 20260505 05:35:35.083 | INFO | Analyzing chunk frames = [741376,741888) | 20260505 05:35:35.083 | INFO | MemInfo: { | 20260505 05:35:35.083 | INFO | "gpu" : " 17 GB", | 20260505 05:35:35.083 | INFO | "huge" : "", | 20260505 05:35:35.083 | INFO | "huge_shared" : " 27 GB", | 20260505 05:35:35.083 | INFO | "malloc" : " 14 GB", | 20260505 05:35:35.083 | INFO | "pinned" : " 29 GB" | 20260505 05:35:35.083 | INFO | } | 20260505 05:35:35.083 | INFO | Perf report for the last 5.13904 seconds: | 20260505 05:35:35.083 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:35:35.083 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:35.083 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:35.083 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:35.083 | INFO | ANALYSIS : 57.9%, 2.9, 88.4ms, 0 | 20260505 05:35:35.083 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:35.083 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260505 05:35:35.083 | INFO | PRE_HQ : 57.2%, 2.1, 63.6ms, 0 | 20260505 05:35:35.083 | INFO | BAZWRITER : 58.8%, 4.4, 132.9ms, 0 | 20260505 05:35:35.083 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260505 05:35:35.083 | INFO | Uploaded 36GiB to GPU in 1.6482s (21.8421GiB/s) | 20260505 05:35:35.083 | INFO | Downloaded 38GiB to GPU in 1.71354s (22.1763GiB/s) | 20260505 05:35:40.222 | INFO | Analyzing chunk frames = [741888,742400) | 20260505 05:35:40.222 | INFO | MemInfo: { | 20260505 05:35:40.222 | INFO | "gpu" : " 17 GB", | 20260505 05:35:40.222 | INFO | "huge" : "", | 20260505 05:35:40.222 | INFO | "huge_shared" : " 27 GB", | 20260505 05:35:40.222 | INFO | "malloc" : " 14 GB", | 20260505 05:35:40.222 | INFO | "pinned" : " 29 GB" | 20260505 05:35:40.222 | INFO | } | 20260505 05:35:40.222 | INFO | Perf report for the last 5.13955 seconds: | 20260505 05:35:40.223 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:35:40.223 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:40.223 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:40.223 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:40.223 | INFO | ANALYSIS : 57.8%, 3.0, 90.1ms, 0 | 20260505 05:35:40.223 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:40.223 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:40.223 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 05:35:40.223 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:40.223 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:40.223 | INFO | Uploaded 36GiB to GPU in 1.60081s (22.4886GiB/s) | 20260505 05:35:40.223 | INFO | Downloaded 34GiB to GPU in 1.51713s (22.4107GiB/s) | 20260505 05:35:45.362 | INFO | Analyzing chunk frames = [742400,742912) | 20260505 05:35:45.362 | INFO | MemInfo: { | 20260505 05:35:45.362 | INFO | "gpu" : " 17 GB", | 20260505 05:35:45.362 | INFO | "huge" : "", | 20260505 05:35:45.362 | INFO | "huge_shared" : " 27 GB", | 20260505 05:35:45.362 | INFO | "malloc" : " 14 GB", | 20260505 05:35:45.362 | INFO | "pinned" : " 29 GB" | 20260505 05:35:45.362 | INFO | } | 20260505 05:35:45.362 | INFO | Perf report for the last 5.13934 seconds: | 20260505 05:35:45.362 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:35:45.362 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:45.362 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:45.362 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:45.362 | INFO | ANALYSIS : 57.1%, 3.0, 88.7ms, 0 | 20260505 05:35:45.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:45.362 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:45.362 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 05:35:45.362 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:45.362 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:45.362 | INFO | Uploaded 36GiB to GPU in 1.6051s (22.4285GiB/s) | 20260505 05:35:45.362 | INFO | Downloaded 34GiB to GPU in 1.523s (22.3243GiB/s) | 20260505 05:35:50.501 | INFO | Analyzing chunk frames = [742912,743424) | 20260505 05:35:50.501 | INFO | MemInfo: { | 20260505 05:35:50.501 | INFO | "gpu" : " 17 GB", | 20260505 05:35:50.501 | INFO | "huge" : "", | 20260505 05:35:50.501 | INFO | "huge_shared" : " 27 GB", | 20260505 05:35:50.501 | INFO | "malloc" : " 14 GB", | 20260505 05:35:50.501 | INFO | "pinned" : " 29 GB" | 20260505 05:35:50.501 | INFO | } | 20260505 05:35:50.501 | INFO | Perf report for the last 5.13954 seconds: | 20260505 05:35:50.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:35:50.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:50.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:50.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:50.501 | INFO | ANALYSIS : 56.9%, 3.0, 88.2ms, 0 | 20260505 05:35:50.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:50.502 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:50.502 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:35:50.502 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:50.502 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:50.502 | INFO | Uploaded 36GiB to GPU in 1.60368s (22.4484GiB/s) | 20260505 05:35:50.502 | INFO | Downloaded 34GiB to GPU in 1.52021s (22.3654GiB/s) | 20260505 05:35:55.640 | INFO | Analyzing chunk frames = [743424,743936) | 20260505 05:35:55.640 | INFO | MemInfo: { | 20260505 05:35:55.640 | INFO | "gpu" : " 17 GB", | 20260505 05:35:55.640 | INFO | "huge" : "", | 20260505 05:35:55.640 | INFO | "huge_shared" : " 27 GB", | 20260505 05:35:55.640 | INFO | "malloc" : " 14 GB", | 20260505 05:35:55.640 | INFO | "pinned" : " 29 GB" | 20260505 05:35:55.640 | INFO | } | 20260505 05:35:55.641 | INFO | Perf report for the last 5.1391 seconds: | 20260505 05:35:55.641 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:35:55.641 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:55.641 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:55.641 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:55.641 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260505 05:35:55.641 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:55.641 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:55.641 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260505 05:35:55.641 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:55.641 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:35:55.641 | INFO | Uploaded 36GiB to GPU in 1.60242s (22.466GiB/s) | 20260505 05:35:55.641 | INFO | Downloaded 34GiB to GPU in 1.51925s (22.3795GiB/s) | 20260505 05:36:00.780 | INFO | Analyzing chunk frames = [743936,744448) | 20260505 05:36:00.780 | INFO | MemInfo: { | 20260505 05:36:00.780 | INFO | "gpu" : " 17 GB", | 20260505 05:36:00.780 | INFO | "huge" : "", | 20260505 05:36:00.780 | INFO | "huge_shared" : " 27 GB", | 20260505 05:36:00.780 | INFO | "malloc" : " 14 GB", | 20260505 05:36:00.780 | INFO | "pinned" : " 29 GB" | 20260505 05:36:00.780 | INFO | } | 20260505 05:36:00.780 | INFO | Perf report for the last 5.1393 seconds: | 20260505 05:36:00.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:36:00.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:00.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:00.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:00.780 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260505 05:36:00.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:00.780 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:00.780 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 05:36:00.780 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:00.780 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:00.780 | INFO | Uploaded 36GiB to GPU in 1.60238s (22.4666GiB/s) | 20260505 05:36:00.780 | INFO | Downloaded 34GiB to GPU in 1.51913s (22.3813GiB/s) | 20260505 05:36:05.919 | INFO | Analyzing chunk frames = [744448,744960) | 20260505 05:36:05.919 | INFO | MemInfo: { | 20260505 05:36:05.919 | INFO | "gpu" : " 17 GB", | 20260505 05:36:05.919 | INFO | "huge" : "", | 20260505 05:36:05.919 | INFO | "huge_shared" : " 27 GB", | 20260505 05:36:05.919 | INFO | "malloc" : " 14 GB", | 20260505 05:36:05.919 | INFO | "pinned" : " 29 GB" | 20260505 05:36:05.919 | INFO | } | 20260505 05:36:05.919 | INFO | Perf report for the last 5.13933 seconds: | 20260505 05:36:05.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:36:05.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:05.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:05.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:05.919 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260505 05:36:05.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:05.919 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:05.919 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:36:05.919 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:05.919 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:05.920 | INFO | Uploaded 36GiB to GPU in 1.60292s (22.459GiB/s) | 20260505 05:36:05.920 | INFO | Downloaded 34GiB to GPU in 1.51996s (22.369GiB/s) | 20260505 05:36:11.058 | INFO | Analyzing chunk frames = [744960,745472) | 20260505 05:36:11.058 | INFO | MemInfo: { | 20260505 05:36:11.058 | INFO | "gpu" : " 17 GB", | 20260505 05:36:11.058 | INFO | "huge" : "", | 20260505 05:36:11.058 | INFO | "huge_shared" : " 27 GB", | 20260505 05:36:11.058 | INFO | "malloc" : " 14 GB", | 20260505 05:36:11.058 | INFO | "pinned" : " 29 GB" | 20260505 05:36:11.058 | INFO | } | 20260505 05:36:11.058 | INFO | Perf report for the last 5.13917 seconds: | 20260505 05:36:11.058 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:36:11.058 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:11.058 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:11.058 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:11.058 | INFO | ANALYSIS : 57.3%, 3.0, 89.5ms, 0 | 20260505 05:36:11.058 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:11.058 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:11.058 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:36:11.059 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:11.059 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:11.059 | INFO | Uploaded 36GiB to GPU in 1.60315s (22.4558GiB/s) | 20260505 05:36:11.059 | INFO | Downloaded 34GiB to GPU in 1.52101s (22.3536GiB/s) | 20260505 05:36:11.059 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 05:36:11.059 | INFO | Total mem usage: 15.1 GiB | 20260505 05:36:11.059 | INFO | All Time High Water Marks sum: 15.7 GiB (Savings: 4.09%) | 20260505 05:36:11.059 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -4.11%) | 20260505 05:36:11.059 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 05:36:11.059 | INFO | High Water Breakdown: | 20260505 05:36:11.059 | INFO | 60% : 8.68 GiB ( 96.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 05:36:11.059 | INFO | 39.9% : 5.76 GiB ( 86.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 05:36:11.059 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 05:36:11.059 | INFO | ------------------------------------------ | 20260505 05:36:11.059 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:36:11.059 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 05:36:11.059 | INFO | Total mem usage: 26.5 GiB | 20260505 05:36:11.059 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 05:36:11.059 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 05:36:11.059 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 05:36:11.059 | INFO | High Water Breakdown: | 20260505 05:36:11.059 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:36:11.059 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:36:11.059 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:36:11.059 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:36:11.059 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:36:11.059 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:36:11.059 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 05:36:11.059 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 05:36:11.059 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:36:11.059 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 05:36:11.059 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 05:36:11.059 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 05:36:11.059 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 05:36:11.059 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 05:36:11.059 | INFO | ------------------------------------------ | 20260505 05:36:11.059 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:36:11.059 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 05:36:11.059 | INFO | Total mem usage: 25.6 GiB | 20260505 05:36:11.059 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 05:36:11.059 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 05:36:11.059 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 05:36:11.060 | INFO | High Water Breakdown: | 20260505 05:36:11.060 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 05:36:11.060 | INFO | ------------------------------------------ | 20260505 05:36:11.060 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:36:11.060 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 05:36:11.060 | INFO | Total mem usage: 15.7 GiB | 20260505 05:36:11.060 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 05:36:11.060 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 05:36:11.060 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 05:36:11.060 | INFO | High Water Breakdown: | 20260505 05:36:11.060 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 05:36:11.060 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 05:36:11.060 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 05:36:11.060 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 05:36:11.060 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 05:36:11.060 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 05:36:11.060 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 05:36:11.060 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 05:36:11.060 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 05:36:11.060 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:36:11.060 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 05:36:11.060 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:36:11.060 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 05:36:11.060 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 05:36:11.060 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 05:36:11.060 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 05:36:11.060 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:36:11.060 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 05:36:11.060 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:36:11.060 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:36:11.060 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:36:11.060 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 05:36:11.060 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 05:36:11.060 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:36:11.060 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 05:36:11.060 | INFO | ------------------------------------------ | 20260505 05:36:11.060 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:36:11.060 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 05:36:11.060 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:36:11.060 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:36:11.060 | INFO | PerfReport: Full Execution -- 3.03e+01 / -nan, 1.28e+01 / 0.00e+00 | 20260505 05:36:11.060 | INFO | PerfReport: 0.29%, AggregateStats -- 8.64e-02 / -nan, 4.86e-03 / 0.00e+00 | 20260505 05:36:11.060 | INFO | PerfReport: 10.88%, Baseline -- 3.30e+00 / -nan, 1.68e-02 / 0.00e+00 | 20260505 05:36:11.060 | INFO | PerfReport: 3.05%, Histogram -- 9.25e-01 / -nan, 5.24e-02 / 0.00e+00 | 20260505 05:36:11.060 | INFO | PerfReport: 16.05%, DME -- 3.89e+01 / -nan, 3.15e+00 / 0.00e+00 | 20260505 05:36:11.060 | INFO | PerfReport: 56.78%, FrameLabeling -- 1.72e+01 / -nan, 4.60e-02 / 0.00e+00 | 20260505 05:36:11.060 | INFO | PerfReport: 7.53%, PulseAccumulating -- 2.28e+00 / -nan, 1.48e-01 / 0.00e+00 | 20260505 05:36:11.060 | INFO | PerfReport: 3.85%, Metrics -- 1.17e+00 / -nan, 1.93e-01 / 0.00e+00 | 20260505 05:36:11.060 | INFO | PerfReport: 1.58%, PulseSerializing -- 4.78e-01 / -nan, 7.73e-02 / 0.00e+00 | 20260505 05:36:11.060 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 05:36:11.060 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:36:11.060 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:36:11.060 | INFO | PerfReport: Full Execution -- 9.08e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260505 05:36:11.060 | INFO | PerfReport: 18.44%, Upload -- 1.67e+01 / 0.00e+00, 4.08e-01 / 0.00e+00 | 20260505 05:36:11.060 | INFO | PerfReport: 17.71%, Download -- 1.61e+01 / 0.00e+00, 7.43e-01 / 0.00e+00 | 20260505 05:36:11.060 | INFO | PerfReport: 33.38%, Compute -- 3.03e+01 / 0.00e+00, 1.28e+01 / 0.00e+00 | 20260505 05:36:11.060 | INFO | PerfReport: 0.56%, LockUpload -- 5.07e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260505 05:36:11.060 | INFO | PerfReport: 0.00%, LockDownload -- 3.96e-04 / 0.00e+00, 1.32e-04 / 0.00e+00 | 20260505 05:36:11.060 | INFO | PerfReport: 29.90%, LockCompute -- 2.72e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260505 05:36:11.060 | INFO | PerfReport: 0.02%, Misc -- 1.51e-02 / 0.00e+00, 4.23e-02 / 0.00e+00 | 20260505 05:36:11.156 | INFO | Starting a round of Baz IO | 20260505 05:36:14.096 | INFO | Populating RTMetrics report | 20260505 05:36:14.096 | INFO | RTMetrics report populated | 20260505 05:36:14.096 | INFO | Opening tmp RTMetrics json file | 20260505 05:36:14.115 | INFO | Serializing RTMetrics to json | 20260505 05:36:14.131 | INFO | Serialization done, swapping tmp file now... | 20260505 05:36:14.140 | INFO | RTMetrics done | 20260505 05:36:14.202 | INFO | Finished a round of Baz IO: Wrote 2.265 GiB in 3.046 seconds | 20260505 05:36:14.202 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.80% / 0.07% / 62.12% | 20260505 05:36:14.202 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.04872% / 0.000107288% / 6.75519e-05% | 20260505 05:36:14.202 | INFO | Baz IO overall: | 20260505 05:36:14.202 | INFO | BytesWritten : 355925475940 | 20260505 05:36:14.202 | INFO | NumEvents : 167356728256 | 20260505 05:36:16.198 | INFO | Analyzing chunk frames = [745472,745984) | 20260505 05:36:16.198 | INFO | MemInfo: { | 20260505 05:36:16.198 | INFO | "gpu" : " 17 GB", | 20260505 05:36:16.198 | INFO | "huge" : "", | 20260505 05:36:16.198 | INFO | "huge_shared" : " 27 GB", | 20260505 05:36:16.198 | INFO | "malloc" : " 14 GB", | 20260505 05:36:16.198 | INFO | "pinned" : " 29 GB" | 20260505 05:36:16.198 | INFO | } | 20260505 05:36:16.198 | INFO | Perf report for the last 5.13937 seconds: | 20260505 05:36:16.198 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:36:16.198 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:16.198 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:16.198 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:16.198 | INFO | ANALYSIS : 58.0%, 2.9, 88.3ms, 0 | 20260505 05:36:16.198 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:16.198 | INFO | RT_METRICS : 46.6%, 1.0, 24.4ms, 0 | 20260505 05:36:16.198 | INFO | PRE_HQ : 57.2%, 2.1, 63.6ms, 0 | 20260505 05:36:16.198 | INFO | BAZWRITER : 58.8%, 4.4, 133.0ms, 0 | 20260505 05:36:16.198 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 05:36:16.198 | INFO | Uploaded 36GiB to GPU in 1.65216s (21.7897GiB/s) | 20260505 05:36:16.198 | INFO | Downloaded 38GiB to GPU in 1.71634s (22.1402GiB/s) | 20260505 05:36:21.337 | INFO | Analyzing chunk frames = [745984,746496) | 20260505 05:36:21.337 | INFO | MemInfo: { | 20260505 05:36:21.337 | INFO | "gpu" : " 17 GB", | 20260505 05:36:21.337 | INFO | "huge" : "", | 20260505 05:36:21.337 | INFO | "huge_shared" : " 27 GB", | 20260505 05:36:21.337 | INFO | "malloc" : " 14 GB", | 20260505 05:36:21.337 | INFO | "pinned" : " 29 GB" | 20260505 05:36:21.337 | INFO | } | 20260505 05:36:21.337 | INFO | Perf report for the last 5.13946 seconds: | 20260505 05:36:21.337 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:36:21.337 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:21.337 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:21.337 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:21.337 | INFO | ANALYSIS : 57.8%, 3.0, 90.1ms, 0 | 20260505 05:36:21.337 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:21.337 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:21.337 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 05:36:21.337 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:21.337 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:21.338 | INFO | Uploaded 36GiB to GPU in 1.5976s (22.5337GiB/s) | 20260505 05:36:21.338 | INFO | Downloaded 34GiB to GPU in 1.51705s (22.4119GiB/s) | 20260505 05:36:26.476 | INFO | Analyzing chunk frames = [746496,747008) | 20260505 05:36:26.476 | INFO | MemInfo: { | 20260505 05:36:26.476 | INFO | "gpu" : " 17 GB", | 20260505 05:36:26.476 | INFO | "huge" : "", | 20260505 05:36:26.476 | INFO | "huge_shared" : " 27 GB", | 20260505 05:36:26.477 | INFO | "malloc" : " 14 GB", | 20260505 05:36:26.477 | INFO | "pinned" : " 29 GB" | 20260505 05:36:26.477 | INFO | } | 20260505 05:36:26.477 | INFO | Perf report for the last 5.13944 seconds: | 20260505 05:36:26.477 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:36:26.477 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:26.477 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:26.477 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:26.477 | INFO | ANALYSIS : 57.1%, 3.0, 88.7ms, 0 | 20260505 05:36:26.477 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:26.477 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:26.477 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 05:36:26.477 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:26.477 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:26.477 | INFO | Uploaded 36GiB to GPU in 1.60219s (22.4693GiB/s) | 20260505 05:36:26.477 | INFO | Downloaded 34GiB to GPU in 1.52262s (22.3299GiB/s) | 20260505 05:36:31.616 | INFO | Analyzing chunk frames = [747008,747520) | 20260505 05:36:31.616 | INFO | MemInfo: { | 20260505 05:36:31.616 | INFO | "gpu" : " 17 GB", | 20260505 05:36:31.616 | INFO | "huge" : "", | 20260505 05:36:31.616 | INFO | "huge_shared" : " 27 GB", | 20260505 05:36:31.616 | INFO | "malloc" : " 14 GB", | 20260505 05:36:31.616 | INFO | "pinned" : " 29 GB" | 20260505 05:36:31.616 | INFO | } | 20260505 05:36:31.616 | INFO | Perf report for the last 5.13914 seconds: | 20260505 05:36:31.616 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:36:31.616 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:31.616 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:31.616 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:31.616 | INFO | ANALYSIS : 56.8%, 3.0, 88.1ms, 0 | 20260505 05:36:31.616 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:31.616 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:31.616 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:36:31.616 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:31.616 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:31.616 | INFO | Uploaded 36GiB to GPU in 1.6012s (22.4831GiB/s) | 20260505 05:36:31.616 | INFO | Downloaded 34GiB to GPU in 1.52277s (22.3277GiB/s) | 20260505 05:36:36.755 | INFO | Analyzing chunk frames = [747520,748032) | 20260505 05:36:36.755 | INFO | MemInfo: { | 20260505 05:36:36.755 | INFO | "gpu" : " 17 GB", | 20260505 05:36:36.755 | INFO | "huge" : "", | 20260505 05:36:36.755 | INFO | "huge_shared" : " 27 GB", | 20260505 05:36:36.755 | INFO | "malloc" : " 14 GB", | 20260505 05:36:36.755 | INFO | "pinned" : " 29 GB" | 20260505 05:36:36.755 | INFO | } | 20260505 05:36:36.755 | INFO | Perf report for the last 5.13947 seconds: | 20260505 05:36:36.755 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:36:36.755 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:36.755 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:36.755 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:36.755 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260505 05:36:36.755 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:36.755 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:36.755 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 05:36:36.755 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:36.755 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:36.756 | INFO | Uploaded 36GiB to GPU in 1.59992s (22.5011GiB/s) | 20260505 05:36:36.756 | INFO | Downloaded 34GiB to GPU in 1.51986s (22.3705GiB/s) | 20260505 05:36:41.894 | INFO | Analyzing chunk frames = [748032,748544) | 20260505 05:36:41.894 | INFO | MemInfo: { | 20260505 05:36:41.894 | INFO | "gpu" : " 17 GB", | 20260505 05:36:41.894 | INFO | "huge" : "", | 20260505 05:36:41.894 | INFO | "huge_shared" : " 27 GB", | 20260505 05:36:41.895 | INFO | "malloc" : " 14 GB", | 20260505 05:36:41.895 | INFO | "pinned" : " 29 GB" | 20260505 05:36:41.895 | INFO | } | 20260505 05:36:41.895 | INFO | Perf report for the last 5.13929 seconds: | 20260505 05:36:41.895 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:36:41.895 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:41.895 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:41.895 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:41.895 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260505 05:36:41.895 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:41.895 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:41.895 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 05:36:41.895 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:41.895 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:41.895 | INFO | Uploaded 36GiB to GPU in 1.59912s (22.5124GiB/s) | 20260505 05:36:41.895 | INFO | Downloaded 34GiB to GPU in 1.51954s (22.3752GiB/s) | 20260505 05:36:47.034 | INFO | Analyzing chunk frames = [748544,749056) | 20260505 05:36:47.034 | INFO | MemInfo: { | 20260505 05:36:47.034 | INFO | "gpu" : " 17 GB", | 20260505 05:36:47.034 | INFO | "huge" : "", | 20260505 05:36:47.034 | INFO | "huge_shared" : " 27 GB", | 20260505 05:36:47.034 | INFO | "malloc" : " 14 GB", | 20260505 05:36:47.034 | INFO | "pinned" : " 29 GB" | 20260505 05:36:47.034 | INFO | } | 20260505 05:36:47.034 | INFO | Perf report for the last 5.13936 seconds: | 20260505 05:36:47.034 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:36:47.034 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:47.034 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:47.034 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:47.034 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260505 05:36:47.034 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:47.034 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:47.034 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:36:47.034 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:47.034 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:47.034 | INFO | Uploaded 36GiB to GPU in 1.60062s (22.4912GiB/s) | 20260505 05:36:47.034 | INFO | Downloaded 34GiB to GPU in 1.52179s (22.342GiB/s) | 20260505 05:36:52.173 | INFO | Analyzing chunk frames = [749056,749568) | 20260505 05:36:52.173 | INFO | MemInfo: { | 20260505 05:36:52.173 | INFO | "gpu" : " 17 GB", | 20260505 05:36:52.173 | INFO | "huge" : "", | 20260505 05:36:52.173 | INFO | "huge_shared" : " 27 GB", | 20260505 05:36:52.173 | INFO | "malloc" : " 14 GB", | 20260505 05:36:52.173 | INFO | "pinned" : " 29 GB" | 20260505 05:36:52.173 | INFO | } | 20260505 05:36:52.173 | INFO | Perf report for the last 5.13934 seconds: | 20260505 05:36:52.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:36:52.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:52.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:52.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:52.173 | INFO | ANALYSIS : 57.3%, 3.0, 89.5ms, 0 | 20260505 05:36:52.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:52.173 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:52.173 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 05:36:52.173 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:52.173 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:52.174 | INFO | Uploaded 36GiB to GPU in 1.59941s (22.5083GiB/s) | 20260505 05:36:52.174 | INFO | Downloaded 34GiB to GPU in 1.52089s (22.3553GiB/s) | 20260505 05:36:52.270 | INFO | Starting a round of Baz IO | 20260505 05:36:55.210 | INFO | Populating RTMetrics report | 20260505 05:36:55.210 | INFO | RTMetrics report populated | 20260505 05:36:55.210 | INFO | Opening tmp RTMetrics json file | 20260505 05:36:55.245 | INFO | Serializing RTMetrics to json | 20260505 05:36:55.268 | INFO | Serialization done, swapping tmp file now... | 20260505 05:36:55.275 | INFO | RTMetrics done | 20260505 05:36:55.464 | INFO | Finished a round of Baz IO: Wrote 2.268 GiB in 3.194 seconds | 20260505 05:36:55.464 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.73% / 0.07% / 62.20% | 20260505 05:36:55.464 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.00974% / 0.000135104% / 4.76837e-05% | 20260505 05:36:55.464 | INFO | Baz IO overall: | 20260505 05:36:55.464 | INFO | BytesWritten : 358360463635 | 20260505 05:36:55.464 | INFO | NumEvents : 168517499261 | 20260505 05:36:57.312 | INFO | Analyzing chunk frames = [749568,750080) | 20260505 05:36:57.312 | INFO | MemInfo: { | 20260505 05:36:57.312 | INFO | "gpu" : " 17 GB", | 20260505 05:36:57.312 | INFO | "huge" : "", | 20260505 05:36:57.312 | INFO | "huge_shared" : " 27 GB", | 20260505 05:36:57.313 | INFO | "malloc" : " 14 GB", | 20260505 05:36:57.313 | INFO | "pinned" : " 29 GB" | 20260505 05:36:57.313 | INFO | } | 20260505 05:36:57.313 | INFO | Perf report for the last 5.1393 seconds: | 20260505 05:36:57.313 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:36:57.313 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:57.313 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:57.313 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:57.313 | INFO | ANALYSIS : 58.0%, 2.8, 85.1ms, 0 | 20260505 05:36:57.313 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:36:57.313 | INFO | RT_METRICS : 46.7%, 1.0, 24.5ms, 0 | 20260505 05:36:57.313 | INFO | PRE_HQ : 57.3%, 2.1, 63.2ms, 0 | 20260505 05:36:57.313 | INFO | BAZWRITER : 61.7%, 4.7, 148.6ms, 0 | 20260505 05:36:57.313 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 05:36:57.313 | INFO | Uploaded 36GiB to GPU in 1.65376s (21.7686GiB/s) | 20260505 05:36:57.313 | INFO | Downloaded 38GiB to GPU in 1.72062s (22.0851GiB/s) | 20260505 05:37:02.452 | INFO | Analyzing chunk frames = [750080,750592) | 20260505 05:37:02.452 | INFO | MemInfo: { | 20260505 05:37:02.452 | INFO | "gpu" : " 17 GB", | 20260505 05:37:02.452 | INFO | "huge" : "", | 20260505 05:37:02.452 | INFO | "huge_shared" : " 27 GB", | 20260505 05:37:02.452 | INFO | "malloc" : " 14 GB", | 20260505 05:37:02.452 | INFO | "pinned" : " 29 GB" | 20260505 05:37:02.452 | INFO | } | 20260505 05:37:02.452 | INFO | Perf report for the last 5.13933 seconds: | 20260505 05:37:02.452 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:37:02.452 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:02.452 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:02.452 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:02.452 | INFO | ANALYSIS : 57.8%, 3.0, 90.1ms, 0 | 20260505 05:37:02.452 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:02.452 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:02.452 | INFO | PRE_HQ : 31.0%, 1.0, 16.2ms, 0 | 20260505 05:37:02.452 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:02.452 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:02.452 | INFO | Uploaded 36GiB to GPU in 1.60098s (22.4862GiB/s) | 20260505 05:37:02.452 | INFO | Downloaded 34GiB to GPU in 1.51755s (22.4045GiB/s) | 20260505 05:37:07.591 | INFO | Analyzing chunk frames = [750592,751104) | 20260505 05:37:07.591 | INFO | MemInfo: { | 20260505 05:37:07.591 | INFO | "gpu" : " 17 GB", | 20260505 05:37:07.591 | INFO | "huge" : "", | 20260505 05:37:07.591 | INFO | "huge_shared" : " 27 GB", | 20260505 05:37:07.591 | INFO | "malloc" : " 14 GB", | 20260505 05:37:07.591 | INFO | "pinned" : " 29 GB" | 20260505 05:37:07.591 | INFO | } | 20260505 05:37:07.591 | INFO | Perf report for the last 5.13929 seconds: | 20260505 05:37:07.591 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:37:07.591 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:07.591 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:07.591 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:07.591 | INFO | ANALYSIS : 57.1%, 3.0, 88.7ms, 0 | 20260505 05:37:07.591 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:07.591 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:07.591 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 05:37:07.591 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:07.591 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:07.592 | INFO | Uploaded 36GiB to GPU in 1.60449s (22.437GiB/s) | 20260505 05:37:07.592 | INFO | Downloaded 34GiB to GPU in 1.52278s (22.3276GiB/s) | 20260505 05:37:12.730 | INFO | Analyzing chunk frames = [751104,751616) | 20260505 05:37:12.730 | INFO | MemInfo: { | 20260505 05:37:12.730 | INFO | "gpu" : " 17 GB", | 20260505 05:37:12.730 | INFO | "huge" : "", | 20260505 05:37:12.730 | INFO | "huge_shared" : " 27 GB", | 20260505 05:37:12.730 | INFO | "malloc" : " 14 GB", | 20260505 05:37:12.730 | INFO | "pinned" : " 29 GB" | 20260505 05:37:12.730 | INFO | } | 20260505 05:37:12.731 | INFO | Perf report for the last 5.13923 seconds: | 20260505 05:37:12.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:37:12.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:12.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:12.731 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:12.731 | INFO | ANALYSIS : 56.8%, 3.0, 88.0ms, 0 | 20260505 05:37:12.731 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:12.731 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:12.731 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:37:12.731 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:12.731 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:12.731 | INFO | Uploaded 36GiB to GPU in 1.60384s (22.4461GiB/s) | 20260505 05:37:12.731 | INFO | Downloaded 34GiB to GPU in 1.52085s (22.3558GiB/s) | 20260505 05:37:17.870 | INFO | Analyzing chunk frames = [751616,752128) | 20260505 05:37:17.870 | INFO | MemInfo: { | 20260505 05:37:17.870 | INFO | "gpu" : " 17 GB", | 20260505 05:37:17.870 | INFO | "huge" : "", | 20260505 05:37:17.870 | INFO | "huge_shared" : " 27 GB", | 20260505 05:37:17.870 | INFO | "malloc" : " 14 GB", | 20260505 05:37:17.870 | INFO | "pinned" : " 29 GB" | 20260505 05:37:17.870 | INFO | } | 20260505 05:37:17.870 | INFO | Perf report for the last 5.13943 seconds: | 20260505 05:37:17.870 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:37:17.870 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:17.870 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:17.870 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:17.870 | INFO | ANALYSIS : 56.8%, 3.0, 88.5ms, 0 | 20260505 05:37:17.870 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:17.870 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:17.870 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260505 05:37:17.870 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:17.870 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:17.870 | INFO | Uploaded 36GiB to GPU in 1.60277s (22.4611GiB/s) | 20260505 05:37:17.870 | INFO | Downloaded 34GiB to GPU in 1.51989s (22.37GiB/s) | 20260505 05:37:23.009 | INFO | Analyzing chunk frames = [752128,752640) | 20260505 05:37:23.009 | INFO | MemInfo: { | 20260505 05:37:23.009 | INFO | "gpu" : " 17 GB", | 20260505 05:37:23.009 | INFO | "huge" : "", | 20260505 05:37:23.009 | INFO | "huge_shared" : " 27 GB", | 20260505 05:37:23.009 | INFO | "malloc" : " 14 GB", | 20260505 05:37:23.009 | INFO | "pinned" : " 29 GB" | 20260505 05:37:23.009 | INFO | } | 20260505 05:37:23.009 | INFO | Perf report for the last 5.13934 seconds: | 20260505 05:37:23.009 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:37:23.009 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:23.009 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:23.009 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:23.009 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260505 05:37:23.009 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:23.009 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:23.009 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 05:37:23.010 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:23.010 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:23.010 | INFO | Uploaded 36GiB to GPU in 1.60272s (22.4617GiB/s) | 20260505 05:37:23.010 | INFO | Downloaded 34GiB to GPU in 1.51951s (22.3756GiB/s) | 20260505 05:37:28.149 | INFO | Analyzing chunk frames = [752640,753152) | 20260505 05:37:28.149 | INFO | MemInfo: { | 20260505 05:37:28.149 | INFO | "gpu" : " 17 GB", | 20260505 05:37:28.149 | INFO | "huge" : "", | 20260505 05:37:28.149 | INFO | "huge_shared" : " 27 GB", | 20260505 05:37:28.149 | INFO | "malloc" : " 14 GB", | 20260505 05:37:28.149 | INFO | "pinned" : " 29 GB" | 20260505 05:37:28.149 | INFO | } | 20260505 05:37:28.149 | INFO | Perf report for the last 5.13948 seconds: | 20260505 05:37:28.149 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:37:28.149 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:28.149 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:28.149 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:28.149 | INFO | ANALYSIS : 57.2%, 3.0, 89.3ms, 0 | 20260505 05:37:28.149 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:28.149 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:28.149 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260505 05:37:28.149 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:28.149 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:28.149 | INFO | Uploaded 36GiB to GPU in 1.60249s (22.4651GiB/s) | 20260505 05:37:28.149 | INFO | Downloaded 34GiB to GPU in 1.52007s (22.3674GiB/s) | 20260505 05:37:33.288 | INFO | Analyzing chunk frames = [753152,753664) | 20260505 05:37:33.288 | INFO | MemInfo: { | 20260505 05:37:33.288 | INFO | "gpu" : " 17 GB", | 20260505 05:37:33.288 | INFO | "huge" : "", | 20260505 05:37:33.288 | INFO | "huge_shared" : " 27 GB", | 20260505 05:37:33.288 | INFO | "malloc" : " 14 GB", | 20260505 05:37:33.288 | INFO | "pinned" : " 29 GB" | 20260505 05:37:33.288 | INFO | } | 20260505 05:37:33.288 | INFO | Perf report for the last 5.13919 seconds: | 20260505 05:37:33.288 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:37:33.288 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:33.288 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:33.288 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:33.288 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 05:37:33.288 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:33.288 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:33.288 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 05:37:33.288 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:33.288 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:33.289 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 05:37:33.289 | INFO | Total mem usage: 15.3 GiB | 20260505 05:37:33.289 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 05:37:33.289 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: 0.0501%) | 20260505 05:37:33.289 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 05:37:33.289 | INFO | High Water Breakdown: | 20260505 05:37:33.289 | INFO | 61% : 9.35 GiB ( 100% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 05:37:33.289 | INFO | 38.9% : 5.96 GiB ( 89.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 05:37:33.289 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 05:37:33.289 | INFO | Uploaded 36GiB to GPU in 1.60395s (22.4446GiB/s) | 20260505 05:37:33.289 | INFO | Downloaded 34GiB to GPU in 1.52183s (22.3416GiB/s) | 20260505 05:37:33.289 | INFO | ------------------------------------------ | 20260505 05:37:33.289 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:37:33.289 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 05:37:33.289 | INFO | Total mem usage: 26.5 GiB | 20260505 05:37:33.289 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 05:37:33.289 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 05:37:33.289 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 05:37:33.289 | INFO | High Water Breakdown: | 20260505 05:37:33.289 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:37:33.289 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:37:33.289 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:37:33.289 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:37:33.289 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:37:33.289 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:37:33.289 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 05:37:33.289 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 05:37:33.289 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:37:33.289 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 05:37:33.289 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 05:37:33.289 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 05:37:33.289 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 05:37:33.289 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 05:37:33.289 | INFO | ------------------------------------------ | 20260505 05:37:33.289 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:37:33.289 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 05:37:33.289 | INFO | Total mem usage: 25.6 GiB | 20260505 05:37:33.289 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 05:37:33.289 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 05:37:33.289 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 05:37:33.289 | INFO | High Water Breakdown: | 20260505 05:37:33.289 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 05:37:33.289 | INFO | ------------------------------------------ | 20260505 05:37:33.289 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:37:33.289 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 05:37:33.289 | INFO | Total mem usage: 15.7 GiB | 20260505 05:37:33.289 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 05:37:33.289 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 05:37:33.289 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 05:37:33.289 | INFO | High Water Breakdown: | 20260505 05:37:33.289 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 05:37:33.289 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 05:37:33.289 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 05:37:33.289 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 05:37:33.290 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 05:37:33.290 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 05:37:33.290 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 05:37:33.290 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 05:37:33.290 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 05:37:33.290 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:37:33.290 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 05:37:33.290 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:37:33.290 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 05:37:33.290 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 05:37:33.290 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 05:37:33.290 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 05:37:33.290 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:37:33.290 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 05:37:33.290 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:37:33.290 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:37:33.290 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:37:33.290 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 05:37:33.290 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 05:37:33.290 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:37:33.290 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 05:37:33.290 | INFO | ------------------------------------------ | 20260505 05:37:33.290 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:37:33.290 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 05:37:33.290 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:37:33.290 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:37:33.290 | INFO | PerfReport: Full Execution -- 3.03e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260505 05:37:33.290 | INFO | PerfReport: 0.28%, AggregateStats -- 8.62e-02 / -nan, 4.31e-03 / 0.00e+00 | 20260505 05:37:33.290 | INFO | PerfReport: 10.88%, Baseline -- 3.30e+00 / -nan, 1.85e-02 / 0.00e+00 | 20260505 05:37:33.290 | INFO | PerfReport: 3.05%, Histogram -- 9.24e-01 / -nan, 5.20e-02 / 0.00e+00 | 20260505 05:37:33.290 | INFO | PerfReport: 16.04%, DME -- 3.89e+01 / -nan, 3.22e+00 / 0.00e+00 | 20260505 05:37:33.290 | INFO | PerfReport: 56.80%, FrameLabeling -- 1.72e+01 / -nan, 4.61e-02 / 0.00e+00 | 20260505 05:37:33.290 | INFO | PerfReport: 7.52%, PulseAccumulating -- 2.28e+00 / -nan, 1.47e-01 / 0.00e+00 | 20260505 05:37:33.290 | INFO | PerfReport: 3.85%, Metrics -- 1.17e+00 / -nan, 1.91e-01 / 0.00e+00 | 20260505 05:37:33.290 | INFO | PerfReport: 1.57%, PulseSerializing -- 4.77e-01 / -nan, 7.66e-02 / 0.00e+00 | 20260505 05:37:33.290 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 05:37:33.290 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:37:33.290 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:37:33.290 | INFO | PerfReport: Full Execution -- 9.06e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260505 05:37:33.290 | INFO | PerfReport: 18.49%, Upload -- 1.67e+01 / 0.00e+00, 4.23e-01 / 0.00e+00 | 20260505 05:37:33.290 | INFO | PerfReport: 17.77%, Download -- 1.61e+01 / 0.00e+00, 7.64e-01 / 0.00e+00 | 20260505 05:37:33.290 | INFO | PerfReport: 33.46%, Compute -- 3.03e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260505 05:37:33.290 | INFO | PerfReport: 0.56%, LockUpload -- 5.08e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260505 05:37:33.290 | INFO | PerfReport: 0.00%, LockDownload -- 3.94e-04 / 0.00e+00, 9.91e-05 / 0.00e+00 | 20260505 05:37:33.290 | INFO | PerfReport: 29.69%, LockCompute -- 2.69e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260505 05:37:33.290 | INFO | PerfReport: 0.02%, Misc -- 1.41e-02 / 0.00e+00, 2.51e-02 / 0.00e+00 | 20260505 05:37:33.385 | INFO | Starting a round of Baz IO | 20260505 05:37:36.327 | INFO | Populating RTMetrics report | 20260505 05:37:36.327 | INFO | RTMetrics report populated | 20260505 05:37:36.327 | INFO | Opening tmp RTMetrics json file | 20260505 05:37:36.341 | INFO | Serializing RTMetrics to json | 20260505 05:37:36.344 | INFO | Serialization done, swapping tmp file now... | 20260505 05:37:36.345 | INFO | RTMetrics done | 20260505 05:37:36.431 | INFO | Finished a round of Baz IO: Wrote 2.268 GiB in 3.045 seconds | 20260505 05:37:36.431 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.66% / 0.07% / 62.27% | 20260505 05:37:36.431 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.96602% / 0.000147025% / 6.75519e-05% | 20260505 05:37:36.431 | INFO | Baz IO overall: | 20260505 05:37:36.431 | INFO | BytesWritten : 360795266198 | 20260505 05:37:36.431 | INFO | NumEvents : 169678261758 | 20260505 05:37:38.427 | INFO | Analyzing chunk frames = [753664,754176) | 20260505 05:37:38.427 | INFO | MemInfo: { | 20260505 05:37:38.427 | INFO | "gpu" : " 17 GB", | 20260505 05:37:38.427 | INFO | "huge" : "", | 20260505 05:37:38.427 | INFO | "huge_shared" : " 27 GB", | 20260505 05:37:38.427 | INFO | "malloc" : " 14 GB", | 20260505 05:37:38.427 | INFO | "pinned" : " 29 GB" | 20260505 05:37:38.427 | INFO | } | 20260505 05:37:38.427 | INFO | Perf report for the last 5.1393 seconds: | 20260505 05:37:38.427 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:37:38.427 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:38.427 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:38.427 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:38.427 | INFO | ANALYSIS : 58.0%, 2.9, 88.6ms, 0 | 20260505 05:37:38.427 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:38.427 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 05:37:38.427 | INFO | PRE_HQ : 57.3%, 2.2, 64.1ms, 0 | 20260505 05:37:38.427 | INFO | BAZWRITER : 58.8%, 4.4, 132.4ms, 0 | 20260505 05:37:38.428 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 05:37:38.428 | INFO | Uploaded 36GiB to GPU in 1.64586s (21.873GiB/s) | 20260505 05:37:38.428 | INFO | Downloaded 38GiB to GPU in 1.71338s (22.1784GiB/s) | 20260505 05:37:43.566 | INFO | Analyzing chunk frames = [754176,754688) | 20260505 05:37:43.567 | INFO | MemInfo: { | 20260505 05:37:43.567 | INFO | "gpu" : " 17 GB", | 20260505 05:37:43.567 | INFO | "huge" : "", | 20260505 05:37:43.567 | INFO | "huge_shared" : " 27 GB", | 20260505 05:37:43.567 | INFO | "malloc" : " 14 GB", | 20260505 05:37:43.567 | INFO | "pinned" : " 29 GB" | 20260505 05:37:43.567 | INFO | } | 20260505 05:37:43.567 | INFO | Perf report for the last 5.13934 seconds: | 20260505 05:37:43.567 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:37:43.567 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:43.567 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:43.567 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:43.567 | INFO | ANALYSIS : 57.7%, 3.0, 90.0ms, 0 | 20260505 05:37:43.567 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:43.567 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:43.567 | INFO | PRE_HQ : 31.0%, 1.0, 16.3ms, 0 | 20260505 05:37:43.567 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:43.567 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:43.567 | INFO | Uploaded 36GiB to GPU in 1.59951s (22.5069GiB/s) | 20260505 05:37:43.567 | INFO | Downloaded 34GiB to GPU in 1.51739s (22.407GiB/s) | 20260505 05:37:48.706 | INFO | Analyzing chunk frames = [754688,755200) | 20260505 05:37:48.706 | INFO | MemInfo: { | 20260505 05:37:48.706 | INFO | "gpu" : " 17 GB", | 20260505 05:37:48.706 | INFO | "huge" : "", | 20260505 05:37:48.706 | INFO | "huge_shared" : " 27 GB", | 20260505 05:37:48.706 | INFO | "malloc" : " 14 GB", | 20260505 05:37:48.706 | INFO | "pinned" : " 29 GB" | 20260505 05:37:48.706 | INFO | } | 20260505 05:37:48.706 | INFO | Perf report for the last 5.1395 seconds: | 20260505 05:37:48.706 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:37:48.706 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:48.706 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:48.706 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:48.706 | INFO | ANALYSIS : 57.1%, 3.0, 88.8ms, 0 | 20260505 05:37:48.706 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:48.706 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:48.706 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:37:48.706 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:48.706 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:48.706 | INFO | Uploaded 36GiB to GPU in 1.60527s (22.4261GiB/s) | 20260505 05:37:48.707 | INFO | Downloaded 34GiB to GPU in 1.52232s (22.3343GiB/s) | 20260505 05:37:53.845 | INFO | Analyzing chunk frames = [755200,755712) | 20260505 05:37:53.845 | INFO | MemInfo: { | 20260505 05:37:53.845 | INFO | "gpu" : " 17 GB", | 20260505 05:37:53.845 | INFO | "huge" : "", | 20260505 05:37:53.845 | INFO | "huge_shared" : " 27 GB", | 20260505 05:37:53.845 | INFO | "malloc" : " 14 GB", | 20260505 05:37:53.845 | INFO | "pinned" : " 29 GB" | 20260505 05:37:53.845 | INFO | } | 20260505 05:37:53.845 | INFO | Perf report for the last 5.13904 seconds: | 20260505 05:37:53.845 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:37:53.845 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:53.845 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:53.845 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:53.845 | INFO | ANALYSIS : 56.8%, 3.0, 88.0ms, 0 | 20260505 05:37:53.845 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:53.845 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:53.845 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260505 05:37:53.845 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:53.845 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:53.846 | INFO | Uploaded 36GiB to GPU in 1.60407s (22.4429GiB/s) | 20260505 05:37:53.846 | INFO | Downloaded 34GiB to GPU in 1.52085s (22.356GiB/s) | 20260505 05:37:58.985 | INFO | Analyzing chunk frames = [755712,756224) | 20260505 05:37:58.985 | INFO | MemInfo: { | 20260505 05:37:58.985 | INFO | "gpu" : " 17 GB", | 20260505 05:37:58.985 | INFO | "huge" : "", | 20260505 05:37:58.985 | INFO | "huge_shared" : " 27 GB", | 20260505 05:37:58.985 | INFO | "malloc" : " 14 GB", | 20260505 05:37:58.985 | INFO | "pinned" : " 29 GB" | 20260505 05:37:58.985 | INFO | } | 20260505 05:37:58.985 | INFO | Perf report for the last 5.13974 seconds: | 20260505 05:37:58.985 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:37:58.985 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:58.985 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:58.985 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:58.985 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260505 05:37:58.985 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:58.985 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:58.985 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260505 05:37:58.985 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:58.985 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:37:58.985 | INFO | Uploaded 36GiB to GPU in 1.60225s (22.4684GiB/s) | 20260505 05:37:58.985 | INFO | Downloaded 34GiB to GPU in 1.51886s (22.3852GiB/s) | 20260505 05:38:04.124 | INFO | Analyzing chunk frames = [756224,756736) | 20260505 05:38:04.124 | INFO | MemInfo: { | 20260505 05:38:04.124 | INFO | "gpu" : " 17 GB", | 20260505 05:38:04.124 | INFO | "huge" : "", | 20260505 05:38:04.124 | INFO | "huge_shared" : " 27 GB", | 20260505 05:38:04.124 | INFO | "malloc" : " 14 GB", | 20260505 05:38:04.124 | INFO | "pinned" : " 29 GB" | 20260505 05:38:04.124 | INFO | } | 20260505 05:38:04.124 | INFO | Perf report for the last 5.13909 seconds: | 20260505 05:38:04.124 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:38:04.124 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:04.124 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:04.124 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:04.124 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260505 05:38:04.124 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:04.124 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:04.124 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:38:04.124 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:04.124 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:04.125 | INFO | Uploaded 36GiB to GPU in 1.60294s (22.4587GiB/s) | 20260505 05:38:04.125 | INFO | Downloaded 34GiB to GPU in 1.51944s (22.3766GiB/s) | 20260505 05:38:09.263 | INFO | Analyzing chunk frames = [756736,757248) | 20260505 05:38:09.263 | INFO | MemInfo: { | 20260505 05:38:09.263 | INFO | "gpu" : " 17 GB", | 20260505 05:38:09.263 | INFO | "huge" : "", | 20260505 05:38:09.263 | INFO | "huge_shared" : " 27 GB", | 20260505 05:38:09.263 | INFO | "malloc" : " 14 GB", | 20260505 05:38:09.263 | INFO | "pinned" : " 29 GB" | 20260505 05:38:09.263 | INFO | } | 20260505 05:38:09.263 | INFO | Perf report for the last 5.13925 seconds: | 20260505 05:38:09.263 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:38:09.263 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:09.263 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:09.263 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:09.263 | INFO | ANALYSIS : 57.2%, 3.0, 89.3ms, 0 | 20260505 05:38:09.263 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:09.263 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:09.263 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:38:09.264 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:09.264 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:09.264 | INFO | Uploaded 36GiB to GPU in 1.60288s (22.4595GiB/s) | 20260505 05:38:09.264 | INFO | Downloaded 34GiB to GPU in 1.52066s (22.3588GiB/s) | 20260505 05:38:14.403 | INFO | Analyzing chunk frames = [757248,757760) | 20260505 05:38:14.403 | INFO | MemInfo: { | 20260505 05:38:14.403 | INFO | "gpu" : " 17 GB", | 20260505 05:38:14.403 | INFO | "huge" : "", | 20260505 05:38:14.403 | INFO | "huge_shared" : " 27 GB", | 20260505 05:38:14.403 | INFO | "malloc" : " 14 GB", | 20260505 05:38:14.403 | INFO | "pinned" : " 29 GB" | 20260505 05:38:14.403 | INFO | } | 20260505 05:38:14.403 | INFO | Perf report for the last 5.1395 seconds: | 20260505 05:38:14.403 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:38:14.403 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:14.403 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:14.403 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:14.403 | INFO | ANALYSIS : 57.3%, 3.0, 89.3ms, 0 | 20260505 05:38:14.403 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:14.403 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:14.403 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 05:38:14.403 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:14.403 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:14.403 | INFO | Uploaded 36GiB to GPU in 1.60331s (22.4535GiB/s) | 20260505 05:38:14.403 | INFO | Downloaded 34GiB to GPU in 1.52101s (22.3536GiB/s) | 20260505 05:38:14.500 | INFO | Starting a round of Baz IO | 20260505 05:38:17.440 | INFO | Populating RTMetrics report | 20260505 05:38:17.440 | INFO | RTMetrics report populated | 20260505 05:38:17.440 | INFO | Opening tmp RTMetrics json file | 20260505 05:38:17.453 | INFO | Serializing RTMetrics to json | 20260505 05:38:17.455 | INFO | Serialization done, swapping tmp file now... | 20260505 05:38:17.455 | INFO | RTMetrics done | 20260505 05:38:17.548 | INFO | Finished a round of Baz IO: Wrote 2.268 GiB in 3.047 seconds | 20260505 05:38:17.548 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.58% / 0.07% / 62.34% | 20260505 05:38:17.548 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.93058% / 0.000115236% / 5.16574e-05% | 20260505 05:38:17.548 | INFO | Baz IO overall: | 20260505 05:38:17.548 | INFO | BytesWritten : 363230653846 | 20260505 05:38:17.548 | INFO | NumEvents : 170839458927 | 20260505 05:38:19.542 | INFO | Analyzing chunk frames = [757760,758272) | 20260505 05:38:19.542 | INFO | MemInfo: { | 20260505 05:38:19.542 | INFO | "gpu" : " 17 GB", | 20260505 05:38:19.542 | INFO | "huge" : "", | 20260505 05:38:19.542 | INFO | "huge_shared" : " 27 GB", | 20260505 05:38:19.542 | INFO | "malloc" : " 14 GB", | 20260505 05:38:19.542 | INFO | "pinned" : " 29 GB" | 20260505 05:38:19.542 | INFO | } | 20260505 05:38:19.542 | INFO | Perf report for the last 5.13906 seconds: | 20260505 05:38:19.542 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:38:19.542 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:19.542 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:19.542 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:19.542 | INFO | ANALYSIS : 58.0%, 2.9, 88.9ms, 0 | 20260505 05:38:19.542 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:19.542 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260505 05:38:19.542 | INFO | PRE_HQ : 57.3%, 2.1, 63.8ms, 0 | 20260505 05:38:19.542 | INFO | BAZWRITER : 58.8%, 4.4, 133.0ms, 0 | 20260505 05:38:19.542 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 05:38:19.543 | INFO | Uploaded 36GiB to GPU in 1.65299s (21.7787GiB/s) | 20260505 05:38:19.543 | INFO | Downloaded 38GiB to GPU in 1.71908s (22.1049GiB/s) | 20260505 05:38:24.681 | INFO | Analyzing chunk frames = [758272,758784) | 20260505 05:38:24.681 | INFO | MemInfo: { | 20260505 05:38:24.681 | INFO | "gpu" : " 17 GB", | 20260505 05:38:24.681 | INFO | "huge" : "", | 20260505 05:38:24.681 | INFO | "huge_shared" : " 27 GB", | 20260505 05:38:24.681 | INFO | "malloc" : " 14 GB", | 20260505 05:38:24.681 | INFO | "pinned" : " 29 GB" | 20260505 05:38:24.681 | INFO | } | 20260505 05:38:24.681 | INFO | Perf report for the last 5.13951 seconds: | 20260505 05:38:24.682 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:38:24.682 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:24.682 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:24.682 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:24.682 | INFO | ANALYSIS : 57.7%, 3.0, 90.0ms, 0 | 20260505 05:38:24.682 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:24.682 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:24.682 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 05:38:24.682 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:24.682 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:24.682 | INFO | Uploaded 36GiB to GPU in 1.59838s (22.5228GiB/s) | 20260505 05:38:24.682 | INFO | Downloaded 34GiB to GPU in 1.51839s (22.3921GiB/s) | 20260505 05:38:29.820 | INFO | Analyzing chunk frames = [758784,759296) | 20260505 05:38:29.820 | INFO | MemInfo: { | 20260505 05:38:29.820 | INFO | "gpu" : " 17 GB", | 20260505 05:38:29.820 | INFO | "huge" : "", | 20260505 05:38:29.820 | INFO | "huge_shared" : " 27 GB", | 20260505 05:38:29.821 | INFO | "malloc" : " 14 GB", | 20260505 05:38:29.821 | INFO | "pinned" : " 29 GB" | 20260505 05:38:29.821 | INFO | } | 20260505 05:38:29.821 | INFO | Perf report for the last 5.13916 seconds: | 20260505 05:38:29.821 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:38:29.821 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:29.821 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:29.821 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:29.821 | INFO | ANALYSIS : 57.1%, 3.0, 88.8ms, 0 | 20260505 05:38:29.821 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:29.821 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:29.821 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:38:29.821 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:29.821 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:29.821 | INFO | Uploaded 36GiB to GPU in 1.60196s (22.4725GiB/s) | 20260505 05:38:29.821 | INFO | Downloaded 34GiB to GPU in 1.52248s (22.332GiB/s) | 20260505 05:38:34.960 | INFO | Analyzing chunk frames = [759296,759808) | 20260505 05:38:34.960 | INFO | MemInfo: { | 20260505 05:38:34.960 | INFO | "gpu" : " 17 GB", | 20260505 05:38:34.960 | INFO | "huge" : "", | 20260505 05:38:34.960 | INFO | "huge_shared" : " 27 GB", | 20260505 05:38:34.960 | INFO | "malloc" : " 14 GB", | 20260505 05:38:34.960 | INFO | "pinned" : " 29 GB" | 20260505 05:38:34.960 | INFO | } | 20260505 05:38:34.960 | INFO | Perf report for the last 5.13949 seconds: | 20260505 05:38:34.960 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:38:34.960 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:34.960 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:34.960 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:34.960 | INFO | ANALYSIS : 56.8%, 3.0, 88.1ms, 0 | 20260505 05:38:34.960 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:34.960 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:34.960 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 05:38:34.960 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:34.960 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:34.960 | INFO | Uploaded 36GiB to GPU in 1.60167s (22.4766GiB/s) | 20260505 05:38:34.961 | INFO | Downloaded 34GiB to GPU in 1.52198s (22.3393GiB/s) | 20260505 05:38:40.099 | INFO | Analyzing chunk frames = [759808,760320) | 20260505 05:38:40.099 | INFO | MemInfo: { | 20260505 05:38:40.099 | INFO | "gpu" : " 17 GB", | 20260505 05:38:40.099 | INFO | "huge" : "", | 20260505 05:38:40.099 | INFO | "huge_shared" : " 27 GB", | 20260505 05:38:40.099 | INFO | "malloc" : " 14 GB", | 20260505 05:38:40.099 | INFO | "pinned" : " 29 GB" | 20260505 05:38:40.099 | INFO | } | 20260505 05:38:40.099 | INFO | Perf report for the last 5.13922 seconds: | 20260505 05:38:40.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:38:40.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:40.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:40.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:40.099 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260505 05:38:40.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:40.100 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:40.100 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 05:38:40.100 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:40.100 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:40.100 | INFO | Uploaded 36GiB to GPU in 1.59978s (22.5031GiB/s) | 20260505 05:38:40.100 | INFO | Downloaded 34GiB to GPU in 1.51945s (22.3765GiB/s) | 20260505 05:38:45.239 | INFO | Analyzing chunk frames = [760320,760832) | 20260505 05:38:45.239 | INFO | MemInfo: { | 20260505 05:38:45.239 | INFO | "gpu" : " 17 GB", | 20260505 05:38:45.239 | INFO | "huge" : "", | 20260505 05:38:45.239 | INFO | "huge_shared" : " 27 GB", | 20260505 05:38:45.239 | INFO | "malloc" : " 14 GB", | 20260505 05:38:45.239 | INFO | "pinned" : " 29 GB" | 20260505 05:38:45.239 | INFO | } | 20260505 05:38:45.239 | INFO | Perf report for the last 5.1394 seconds: | 20260505 05:38:45.239 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:38:45.239 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:45.239 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:45.239 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:45.239 | INFO | ANALYSIS : 57.0%, 3.0, 89.0ms, 0 | 20260505 05:38:45.239 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:45.239 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:45.239 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:38:45.239 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:45.239 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:45.239 | INFO | Uploaded 36GiB to GPU in 1.59968s (22.5044GiB/s) | 20260505 05:38:45.239 | INFO | Downloaded 34GiB to GPU in 1.52008s (22.3672GiB/s) | 20260505 05:38:50.378 | INFO | Analyzing chunk frames = [760832,761344) | 20260505 05:38:50.378 | INFO | MemInfo: { | 20260505 05:38:50.378 | INFO | "gpu" : " 17 GB", | 20260505 05:38:50.378 | INFO | "huge" : "", | 20260505 05:38:50.378 | INFO | "huge_shared" : " 27 GB", | 20260505 05:38:50.378 | INFO | "malloc" : " 14 GB", | 20260505 05:38:50.378 | INFO | "pinned" : " 29 GB" | 20260505 05:38:50.378 | INFO | } | 20260505 05:38:50.378 | INFO | Perf report for the last 5.13941 seconds: | 20260505 05:38:50.378 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:38:50.378 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:50.378 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:50.378 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:50.378 | INFO | ANALYSIS : 57.2%, 3.0, 89.3ms, 0 | 20260505 05:38:50.378 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:50.378 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:50.378 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 05:38:50.378 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:50.378 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:50.379 | INFO | Uploaded 36GiB to GPU in 1.60075s (22.4895GiB/s) | 20260505 05:38:50.379 | INFO | Downloaded 34GiB to GPU in 1.52113s (22.3518GiB/s) | 20260505 05:38:55.517 | INFO | Analyzing chunk frames = [761344,761856) | 20260505 05:38:55.517 | INFO | MemInfo: { | 20260505 05:38:55.517 | INFO | "gpu" : " 17 GB", | 20260505 05:38:55.517 | INFO | "huge" : "", | 20260505 05:38:55.517 | INFO | "huge_shared" : " 27 GB", | 20260505 05:38:55.517 | INFO | "malloc" : " 14 GB", | 20260505 05:38:55.517 | INFO | "pinned" : " 29 GB" | 20260505 05:38:55.517 | INFO | } | 20260505 05:38:55.518 | INFO | Perf report for the last 5.13932 seconds: | 20260505 05:38:55.518 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:38:55.518 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:55.518 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:55.518 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:55.518 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 05:38:55.518 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:55.518 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:55.518 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260505 05:38:55.518 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:55.518 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:38:55.518 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 05:38:55.518 | INFO | Total mem usage: 15.3 GiB | 20260505 05:38:55.518 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 05:38:55.518 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -5.78%) | 20260505 05:38:55.518 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 05:38:55.518 | INFO | High Water Breakdown: | 20260505 05:38:55.518 | INFO | 60.1% : 8.71 GiB ( 93.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 05:38:55.518 | INFO | 39.7% : 5.76 GiB ( 86.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 05:38:55.518 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 05:38:55.518 | INFO | ------------------------------------------ | 20260505 05:38:55.518 | INFO | Uploaded 36GiB to GPU in 1.60052s (22.4927GiB/s) | 20260505 05:38:55.518 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:38:55.518 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 05:38:55.518 | INFO | Downloaded 34GiB to GPU in 1.52082s (22.3563GiB/s) | 20260505 05:38:55.518 | INFO | Total mem usage: 26.5 GiB | 20260505 05:38:55.518 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 05:38:55.518 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 05:38:55.518 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 05:38:55.518 | INFO | High Water Breakdown: | 20260505 05:38:55.518 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:38:55.518 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:38:55.518 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:38:55.519 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:38:55.519 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:38:55.519 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:38:55.519 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 05:38:55.519 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 05:38:55.519 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:38:55.519 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 05:38:55.519 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 05:38:55.519 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 05:38:55.519 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 05:38:55.519 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 05:38:55.519 | INFO | ------------------------------------------ | 20260505 05:38:55.519 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:38:55.519 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 05:38:55.519 | INFO | Total mem usage: 25.6 GiB | 20260505 05:38:55.519 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 05:38:55.519 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 05:38:55.519 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 05:38:55.519 | INFO | High Water Breakdown: | 20260505 05:38:55.519 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 05:38:55.519 | INFO | ------------------------------------------ | 20260505 05:38:55.519 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:38:55.519 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 05:38:55.519 | INFO | Total mem usage: 15.7 GiB | 20260505 05:38:55.519 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 05:38:55.519 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 05:38:55.519 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 05:38:55.519 | INFO | High Water Breakdown: | 20260505 05:38:55.519 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 05:38:55.519 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 05:38:55.519 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 05:38:55.519 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 05:38:55.519 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 05:38:55.519 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 05:38:55.519 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 05:38:55.519 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 05:38:55.519 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 05:38:55.519 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:38:55.519 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 05:38:55.519 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:38:55.519 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 05:38:55.519 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 05:38:55.519 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 05:38:55.519 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 05:38:55.519 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:38:55.519 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 05:38:55.519 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:38:55.519 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:38:55.519 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:38:55.519 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 05:38:55.519 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 05:38:55.519 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:38:55.519 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 05:38:55.519 | INFO | ------------------------------------------ | 20260505 05:38:55.519 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:38:55.519 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 05:38:55.519 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:38:55.519 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:38:55.519 | INFO | PerfReport: Full Execution -- 3.03e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260505 05:38:55.519 | INFO | PerfReport: 0.28%, AggregateStats -- 8.62e-02 / -nan, 4.10e-03 / 0.00e+00 | 20260505 05:38:55.519 | INFO | PerfReport: 10.89%, Baseline -- 3.30e+00 / -nan, 1.81e-02 / 0.00e+00 | 20260505 05:38:55.519 | INFO | PerfReport: 3.05%, Histogram -- 9.24e-01 / -nan, 5.15e-02 / 0.00e+00 | 20260505 05:38:55.519 | INFO | PerfReport: 16.04%, DME -- 3.89e+01 / -nan, 3.14e+00 / 0.00e+00 | 20260505 05:38:55.519 | INFO | PerfReport: 56.81%, FrameLabeling -- 1.72e+01 / -nan, 4.54e-02 / 0.00e+00 | 20260505 05:38:55.519 | INFO | PerfReport: 7.51%, PulseAccumulating -- 2.28e+00 / -nan, 1.49e-01 / 0.00e+00 | 20260505 05:38:55.519 | INFO | PerfReport: 3.85%, Metrics -- 1.17e+00 / -nan, 1.91e-01 / 0.00e+00 | 20260505 05:38:55.519 | INFO | PerfReport: 1.57%, PulseSerializing -- 4.76e-01 / -nan, 7.68e-02 / 0.00e+00 | 20260505 05:38:55.519 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 05:38:55.519 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:38:55.519 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:38:55.519 | INFO | PerfReport: Full Execution -- 9.08e+01 / -nan, 1.89e+01 / 0.00e+00 | 20260505 05:38:55.519 | INFO | PerfReport: 18.44%, Upload -- 1.67e+01 / 0.00e+00, 4.09e-01 / 0.00e+00 | 20260505 05:38:55.519 | INFO | PerfReport: 17.72%, Download -- 1.61e+01 / 0.00e+00, 7.51e-01 / 0.00e+00 | 20260505 05:38:55.519 | INFO | PerfReport: 33.37%, Compute -- 3.03e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260505 05:38:55.519 | INFO | PerfReport: 0.56%, LockUpload -- 5.13e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260505 05:38:55.519 | INFO | PerfReport: 0.00%, LockDownload -- 3.94e-04 / 0.00e+00, 9.06e-05 / 0.00e+00 | 20260505 05:38:55.519 | INFO | PerfReport: 29.89%, LockCompute -- 2.71e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260505 05:38:55.519 | INFO | PerfReport: 0.02%, Misc -- 1.43e-02 / 0.00e+00, 2.09e-02 / 0.00e+00 | 20260505 05:38:55.615 | INFO | Starting a round of Baz IO | 20260505 05:38:58.553 | INFO | Populating RTMetrics report | 20260505 05:38:58.553 | INFO | RTMetrics report populated | 20260505 05:38:58.553 | INFO | Opening tmp RTMetrics json file | 20260505 05:38:58.571 | INFO | Serializing RTMetrics to json | 20260505 05:38:58.572 | INFO | Serialization done, swapping tmp file now... | 20260505 05:38:58.572 | INFO | RTMetrics done | 20260505 05:38:58.659 | INFO | Finished a round of Baz IO: Wrote 2.273 GiB in 3.044 seconds | 20260505 05:38:58.659 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.51% / 0.07% / 62.42% | 20260505 05:38:58.659 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.91931% / 0.000115236% / 6.35783e-05% | 20260505 05:38:58.659 | INFO | Baz IO overall: | 20260505 05:38:58.659 | INFO | BytesWritten : 365670852846 | 20260505 05:38:58.659 | INFO | NumEvents : 172003967290 | 20260505 05:39:00.657 | INFO | Analyzing chunk frames = [761856,762368) | 20260505 05:39:00.657 | INFO | MemInfo: { | 20260505 05:39:00.657 | INFO | "gpu" : " 17 GB", | 20260505 05:39:00.657 | INFO | "huge" : "", | 20260505 05:39:00.657 | INFO | "huge_shared" : " 27 GB", | 20260505 05:39:00.657 | INFO | "malloc" : " 14 GB", | 20260505 05:39:00.657 | INFO | "pinned" : " 29 GB" | 20260505 05:39:00.657 | INFO | } | 20260505 05:39:00.657 | INFO | Perf report for the last 5.13912 seconds: | 20260505 05:39:00.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:39:00.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:00.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:00.657 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:00.657 | INFO | ANALYSIS : 57.9%, 2.9, 88.3ms, 0 | 20260505 05:39:00.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:00.657 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 05:39:00.657 | INFO | PRE_HQ : 57.1%, 2.1, 63.6ms, 0 | 20260505 05:39:00.657 | INFO | BAZWRITER : 58.8%, 4.4, 134.1ms, 0 | 20260505 05:39:00.657 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 05:39:00.657 | INFO | Uploaded 36GiB to GPU in 1.65259s (21.784GiB/s) | 20260505 05:39:00.657 | INFO | Downloaded 38GiB to GPU in 1.71782s (22.1211GiB/s) | 20260505 05:39:05.796 | INFO | Analyzing chunk frames = [762368,762880) | 20260505 05:39:05.796 | INFO | MemInfo: { | 20260505 05:39:05.796 | INFO | "gpu" : " 17 GB", | 20260505 05:39:05.796 | INFO | "huge" : "", | 20260505 05:39:05.796 | INFO | "huge_shared" : " 27 GB", | 20260505 05:39:05.796 | INFO | "malloc" : " 14 GB", | 20260505 05:39:05.796 | INFO | "pinned" : " 29 GB" | 20260505 05:39:05.796 | INFO | } | 20260505 05:39:05.796 | INFO | Perf report for the last 5.13941 seconds: | 20260505 05:39:05.796 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:39:05.796 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:05.796 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:05.796 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:05.796 | INFO | ANALYSIS : 57.7%, 3.0, 90.1ms, 0 | 20260505 05:39:05.796 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:05.796 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:05.796 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 05:39:05.796 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:05.796 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:05.796 | INFO | Uploaded 36GiB to GPU in 1.60111s (22.4844GiB/s) | 20260505 05:39:05.797 | INFO | Downloaded 34GiB to GPU in 1.51799s (22.3981GiB/s) | 20260505 05:39:10.935 | INFO | Analyzing chunk frames = [762880,763392) | 20260505 05:39:10.935 | INFO | MemInfo: { | 20260505 05:39:10.935 | INFO | "gpu" : " 17 GB", | 20260505 05:39:10.935 | INFO | "huge" : "", | 20260505 05:39:10.935 | INFO | "huge_shared" : " 27 GB", | 20260505 05:39:10.935 | INFO | "malloc" : " 14 GB", | 20260505 05:39:10.935 | INFO | "pinned" : " 29 GB" | 20260505 05:39:10.935 | INFO | } | 20260505 05:39:10.936 | INFO | Perf report for the last 5.13939 seconds: | 20260505 05:39:10.936 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:39:10.936 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:10.936 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:10.936 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:10.936 | INFO | ANALYSIS : 57.1%, 3.0, 88.7ms, 0 | 20260505 05:39:10.936 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:10.936 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:10.936 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 05:39:10.936 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:10.936 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:10.936 | INFO | Uploaded 36GiB to GPU in 1.60528s (22.426GiB/s) | 20260505 05:39:10.936 | INFO | Downloaded 34GiB to GPU in 1.52246s (22.3323GiB/s) | 20260505 05:39:16.075 | INFO | Analyzing chunk frames = [763392,763904) | 20260505 05:39:16.075 | INFO | MemInfo: { | 20260505 05:39:16.075 | INFO | "gpu" : " 17 GB", | 20260505 05:39:16.075 | INFO | "huge" : "", | 20260505 05:39:16.075 | INFO | "huge_shared" : " 27 GB", | 20260505 05:39:16.075 | INFO | "malloc" : " 14 GB", | 20260505 05:39:16.075 | INFO | "pinned" : " 29 GB" | 20260505 05:39:16.075 | INFO | } | 20260505 05:39:16.075 | INFO | Perf report for the last 5.1394 seconds: | 20260505 05:39:16.075 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:39:16.075 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:16.075 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:16.075 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:16.075 | INFO | ANALYSIS : 56.8%, 3.0, 88.1ms, 0 | 20260505 05:39:16.075 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:16.075 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:16.075 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 05:39:16.075 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:16.075 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:16.076 | INFO | Uploaded 36GiB to GPU in 1.60403s (22.4434GiB/s) | 20260505 05:39:16.076 | INFO | Downloaded 34GiB to GPU in 1.52124s (22.3502GiB/s) | 20260505 05:39:21.214 | INFO | Analyzing chunk frames = [763904,764416) | 20260505 05:39:21.214 | INFO | MemInfo: { | 20260505 05:39:21.214 | INFO | "gpu" : " 17 GB", | 20260505 05:39:21.214 | INFO | "huge" : "", | 20260505 05:39:21.214 | INFO | "huge_shared" : " 27 GB", | 20260505 05:39:21.214 | INFO | "malloc" : " 14 GB", | 20260505 05:39:21.214 | INFO | "pinned" : " 29 GB" | 20260505 05:39:21.214 | INFO | } | 20260505 05:39:21.214 | INFO | Perf report for the last 5.13929 seconds: | 20260505 05:39:21.214 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:39:21.214 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:21.214 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:21.214 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:21.214 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260505 05:39:21.214 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:21.214 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:21.214 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:39:21.214 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:21.214 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:21.215 | INFO | Uploaded 36GiB to GPU in 1.60291s (22.4591GiB/s) | 20260505 05:39:21.215 | INFO | Downloaded 34GiB to GPU in 1.51943s (22.3768GiB/s) | 20260505 05:39:26.353 | INFO | Analyzing chunk frames = [764416,764928) | 20260505 05:39:26.354 | INFO | MemInfo: { | 20260505 05:39:26.354 | INFO | "gpu" : " 17 GB", | 20260505 05:39:26.354 | INFO | "huge" : "", | 20260505 05:39:26.354 | INFO | "huge_shared" : " 27 GB", | 20260505 05:39:26.354 | INFO | "malloc" : " 14 GB", | 20260505 05:39:26.354 | INFO | "pinned" : " 29 GB" | 20260505 05:39:26.354 | INFO | } | 20260505 05:39:26.354 | INFO | Perf report for the last 5.13954 seconds: | 20260505 05:39:26.354 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:39:26.354 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:26.354 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:26.354 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:26.354 | INFO | ANALYSIS : 57.0%, 3.0, 89.0ms, 0 | 20260505 05:39:26.354 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:26.354 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:26.354 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 05:39:26.354 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:26.354 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:26.354 | INFO | Uploaded 36GiB to GPU in 1.60267s (22.4626GiB/s) | 20260505 05:39:26.354 | INFO | Downloaded 34GiB to GPU in 1.51951s (22.3756GiB/s) | 20260505 05:39:31.493 | INFO | Analyzing chunk frames = [764928,765440) | 20260505 05:39:31.493 | INFO | MemInfo: { | 20260505 05:39:31.493 | INFO | "gpu" : " 17 GB", | 20260505 05:39:31.493 | INFO | "huge" : "", | 20260505 05:39:31.493 | INFO | "huge_shared" : " 27 GB", | 20260505 05:39:31.493 | INFO | "malloc" : " 14 GB", | 20260505 05:39:31.493 | INFO | "pinned" : " 29 GB" | 20260505 05:39:31.493 | INFO | } | 20260505 05:39:31.493 | INFO | Perf report for the last 5.13923 seconds: | 20260505 05:39:31.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:39:31.493 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:31.493 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:31.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:31.493 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260505 05:39:31.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:31.493 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:31.493 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 05:39:31.493 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:31.493 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:31.493 | INFO | Uploaded 36GiB to GPU in 1.60416s (22.4416GiB/s) | 20260505 05:39:31.493 | INFO | Downloaded 34GiB to GPU in 1.52126s (22.3499GiB/s) | 20260505 05:39:36.632 | INFO | Analyzing chunk frames = [765440,765952) | 20260505 05:39:36.632 | INFO | MemInfo: { | 20260505 05:39:36.632 | INFO | "gpu" : " 17 GB", | 20260505 05:39:36.632 | INFO | "huge" : "", | 20260505 05:39:36.632 | INFO | "huge_shared" : " 27 GB", | 20260505 05:39:36.632 | INFO | "malloc" : " 14 GB", | 20260505 05:39:36.632 | INFO | "pinned" : " 29 GB" | 20260505 05:39:36.632 | INFO | } | 20260505 05:39:36.632 | INFO | Perf report for the last 5.13931 seconds: | 20260505 05:39:36.632 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:39:36.632 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:36.632 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:36.632 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:36.632 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 05:39:36.632 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:36.632 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:36.632 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 05:39:36.633 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:36.633 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:36.633 | INFO | Uploaded 36GiB to GPU in 1.60318s (22.4554GiB/s) | 20260505 05:39:36.633 | INFO | Downloaded 34GiB to GPU in 1.52049s (22.3611GiB/s) | 20260505 05:39:36.730 | INFO | Starting a round of Baz IO | 20260505 05:39:39.670 | INFO | Populating RTMetrics report | 20260505 05:39:39.670 | INFO | RTMetrics report populated | 20260505 05:39:39.670 | INFO | Opening tmp RTMetrics json file | 20260505 05:39:39.681 | INFO | Serializing RTMetrics to json | 20260505 05:39:39.682 | INFO | Serialization done, swapping tmp file now... | 20260505 05:39:39.683 | INFO | RTMetrics done | 20260505 05:39:39.775 | INFO | Finished a round of Baz IO: Wrote 2.272 GiB in 3.045 seconds | 20260505 05:39:39.775 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.44% / 0.07% / 62.49% | 20260505 05:39:39.775 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.87158% / 0.000111262% / 6.35783e-05% | 20260505 05:39:39.775 | INFO | Baz IO overall: | 20260505 05:39:39.775 | INFO | BytesWritten : 368110358884 | 20260505 05:39:39.775 | INFO | NumEvents : 173167428253 | 20260505 05:39:41.772 | INFO | Analyzing chunk frames = [765952,766464) | 20260505 05:39:41.772 | INFO | MemInfo: { | 20260505 05:39:41.772 | INFO | "gpu" : " 17 GB", | 20260505 05:39:41.772 | INFO | "huge" : "", | 20260505 05:39:41.772 | INFO | "huge_shared" : " 27 GB", | 20260505 05:39:41.772 | INFO | "malloc" : " 14 GB", | 20260505 05:39:41.772 | INFO | "pinned" : " 29 GB" | 20260505 05:39:41.772 | INFO | } | 20260505 05:39:41.772 | INFO | Perf report for the last 5.13939 seconds: | 20260505 05:39:41.772 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:39:41.772 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:41.772 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:41.772 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:41.772 | INFO | ANALYSIS : 57.9%, 2.9, 88.0ms, 0 | 20260505 05:39:41.772 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:41.772 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 05:39:41.772 | INFO | PRE_HQ : 57.1%, 2.2, 64.2ms, 0 | 20260505 05:39:41.772 | INFO | BAZWRITER : 58.8%, 4.4, 133.7ms, 0 | 20260505 05:39:41.772 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 05:39:41.772 | INFO | Uploaded 36GiB to GPU in 1.65052s (21.8113GiB/s) | 20260505 05:39:41.772 | INFO | Downloaded 38GiB to GPU in 1.71715s (22.1297GiB/s) | 20260505 05:39:46.911 | INFO | Analyzing chunk frames = [766464,766976) | 20260505 05:39:46.911 | INFO | MemInfo: { | 20260505 05:39:46.911 | INFO | "gpu" : " 17 GB", | 20260505 05:39:46.911 | INFO | "huge" : "", | 20260505 05:39:46.911 | INFO | "huge_shared" : " 27 GB", | 20260505 05:39:46.911 | INFO | "malloc" : " 14 GB", | 20260505 05:39:46.911 | INFO | "pinned" : " 29 GB" | 20260505 05:39:46.911 | INFO | } | 20260505 05:39:46.911 | INFO | Perf report for the last 5.13929 seconds: | 20260505 05:39:46.911 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:39:46.911 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:46.911 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:46.911 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:46.911 | INFO | ANALYSIS : 57.8%, 3.0, 90.1ms, 0 | 20260505 05:39:46.911 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:46.911 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:46.911 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:39:46.911 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:46.911 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:46.912 | INFO | Uploaded 36GiB to GPU in 1.59841s (22.5224GiB/s) | 20260505 05:39:46.912 | INFO | Downloaded 34GiB to GPU in 1.51876s (22.3867GiB/s) | 20260505 05:39:52.050 | INFO | Analyzing chunk frames = [766976,767488) | 20260505 05:39:52.050 | INFO | MemInfo: { | 20260505 05:39:52.050 | INFO | "gpu" : " 17 GB", | 20260505 05:39:52.050 | INFO | "huge" : "", | 20260505 05:39:52.050 | INFO | "huge_shared" : " 27 GB", | 20260505 05:39:52.050 | INFO | "malloc" : " 14 GB", | 20260505 05:39:52.050 | INFO | "pinned" : " 29 GB" | 20260505 05:39:52.050 | INFO | } | 20260505 05:39:52.050 | INFO | Perf report for the last 5.13937 seconds: | 20260505 05:39:52.050 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:39:52.050 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:52.050 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:52.050 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:52.051 | INFO | ANALYSIS : 57.1%, 3.0, 88.7ms, 0 | 20260505 05:39:52.051 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:52.051 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:52.051 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 05:39:52.051 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:52.051 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:52.051 | INFO | Uploaded 36GiB to GPU in 1.60236s (22.4668GiB/s) | 20260505 05:39:52.051 | INFO | Downloaded 34GiB to GPU in 1.52226s (22.3352GiB/s) | 20260505 05:39:57.190 | INFO | Analyzing chunk frames = [767488,768000) | 20260505 05:39:57.190 | INFO | MemInfo: { | 20260505 05:39:57.190 | INFO | "gpu" : " 17 GB", | 20260505 05:39:57.190 | INFO | "huge" : "", | 20260505 05:39:57.190 | INFO | "huge_shared" : " 27 GB", | 20260505 05:39:57.190 | INFO | "malloc" : " 14 GB", | 20260505 05:39:57.190 | INFO | "pinned" : " 29 GB" | 20260505 05:39:57.190 | INFO | } | 20260505 05:39:57.190 | INFO | Perf report for the last 5.13932 seconds: | 20260505 05:39:57.190 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:39:57.190 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:57.190 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:57.190 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:57.190 | INFO | ANALYSIS : 56.8%, 3.0, 88.1ms, 0 | 20260505 05:39:57.190 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:57.190 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:57.190 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 05:39:57.190 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:57.190 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:39:57.190 | INFO | Uploaded 36GiB to GPU in 1.60159s (22.4776GiB/s) | 20260505 05:39:57.190 | INFO | Downloaded 34GiB to GPU in 1.52215s (22.3369GiB/s) | 20260505 05:40:02.329 | INFO | Analyzing chunk frames = [768000,768512) | 20260505 05:40:02.329 | INFO | MemInfo: { | 20260505 05:40:02.329 | INFO | "gpu" : " 17 GB", | 20260505 05:40:02.329 | INFO | "huge" : "", | 20260505 05:40:02.329 | INFO | "huge_shared" : " 27 GB", | 20260505 05:40:02.329 | INFO | "malloc" : " 14 GB", | 20260505 05:40:02.329 | INFO | "pinned" : " 29 GB" | 20260505 05:40:02.329 | INFO | } | 20260505 05:40:02.329 | INFO | Perf report for the last 5.13939 seconds: | 20260505 05:40:02.329 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:40:02.329 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:02.329 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:02.329 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:02.329 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260505 05:40:02.329 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:02.329 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:02.329 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 05:40:02.329 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:02.329 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:02.329 | INFO | Uploaded 36GiB to GPU in 1.60067s (22.4906GiB/s) | 20260505 05:40:02.330 | INFO | Downloaded 34GiB to GPU in 1.52049s (22.3612GiB/s) | 20260505 05:40:07.468 | INFO | Analyzing chunk frames = [768512,769024) | 20260505 05:40:07.468 | INFO | MemInfo: { | 20260505 05:40:07.468 | INFO | "gpu" : " 17 GB", | 20260505 05:40:07.468 | INFO | "huge" : "", | 20260505 05:40:07.468 | INFO | "huge_shared" : " 27 GB", | 20260505 05:40:07.468 | INFO | "malloc" : " 14 GB", | 20260505 05:40:07.468 | INFO | "pinned" : " 29 GB" | 20260505 05:40:07.468 | INFO | } | 20260505 05:40:07.469 | INFO | Perf report for the last 5.1393 seconds: | 20260505 05:40:07.469 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:40:07.469 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:07.469 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:07.469 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:07.469 | INFO | ANALYSIS : 57.0%, 3.0, 89.0ms, 0 | 20260505 05:40:07.469 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:07.469 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:07.469 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:40:07.469 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:07.469 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:07.469 | INFO | Uploaded 36GiB to GPU in 1.59935s (22.5091GiB/s) | 20260505 05:40:07.469 | INFO | Downloaded 34GiB to GPU in 1.51973s (22.3724GiB/s) | 20260505 05:40:12.608 | INFO | Analyzing chunk frames = [769024,769536) | 20260505 05:40:12.608 | INFO | MemInfo: { | 20260505 05:40:12.608 | INFO | "gpu" : " 17 GB", | 20260505 05:40:12.608 | INFO | "huge" : "", | 20260505 05:40:12.608 | INFO | "huge_shared" : " 27 GB", | 20260505 05:40:12.608 | INFO | "malloc" : " 14 GB", | 20260505 05:40:12.608 | INFO | "pinned" : " 29 GB" | 20260505 05:40:12.608 | INFO | } | 20260505 05:40:12.608 | INFO | Perf report for the last 5.13918 seconds: | 20260505 05:40:12.608 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:40:12.608 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:12.608 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:12.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:12.608 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260505 05:40:12.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:12.608 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:12.608 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 05:40:12.608 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:12.608 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:12.608 | INFO | Uploaded 36GiB to GPU in 1.6007s (22.4902GiB/s) | 20260505 05:40:12.608 | INFO | Downloaded 34GiB to GPU in 1.52156s (22.3455GiB/s) | 20260505 05:40:17.747 | INFO | Analyzing chunk frames = [769536,770048) | 20260505 05:40:17.747 | INFO | MemInfo: { | 20260505 05:40:17.747 | INFO | "gpu" : " 17 GB", | 20260505 05:40:17.747 | INFO | "huge" : "", | 20260505 05:40:17.747 | INFO | "huge_shared" : " 27 GB", | 20260505 05:40:17.747 | INFO | "malloc" : " 14 GB", | 20260505 05:40:17.747 | INFO | "pinned" : " 29 GB" | 20260505 05:40:17.747 | INFO | } | 20260505 05:40:17.747 | INFO | Perf report for the last 5.13953 seconds: | 20260505 05:40:17.747 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:40:17.747 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:17.747 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:17.747 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:17.747 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 05:40:17.747 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:17.747 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:17.747 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 05:40:17.747 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:17.747 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:17.747 | INFO | Uploaded 36GiB to GPU in 1.60086s (22.4879GiB/s) | 20260505 05:40:17.747 | INFO | Downloaded 34GiB to GPU in 1.52181s (22.3418GiB/s) | 20260505 05:40:17.748 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 05:40:17.748 | INFO | Total mem usage: 15.3 GiB | 20260505 05:40:17.748 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 05:40:17.748 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -5.78%) | 20260505 05:40:17.748 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 05:40:17.748 | INFO | High Water Breakdown: | 20260505 05:40:17.748 | INFO | 60.1% : 8.71 GiB ( 93.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 05:40:17.748 | INFO | 39.7% : 5.76 GiB ( 86.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 05:40:17.748 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 05:40:17.748 | INFO | ------------------------------------------ | 20260505 05:40:17.748 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:40:17.748 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 05:40:17.748 | INFO | Total mem usage: 26.5 GiB | 20260505 05:40:17.748 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 05:40:17.748 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 05:40:17.748 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 05:40:17.748 | INFO | High Water Breakdown: | 20260505 05:40:17.748 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:40:17.748 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:40:17.748 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:40:17.748 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:40:17.748 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:40:17.748 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:40:17.748 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 05:40:17.748 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 05:40:17.748 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:40:17.748 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 05:40:17.748 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 05:40:17.748 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 05:40:17.748 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 05:40:17.748 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 05:40:17.748 | INFO | ------------------------------------------ | 20260505 05:40:17.748 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:40:17.748 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 05:40:17.748 | INFO | Total mem usage: 25.6 GiB | 20260505 05:40:17.748 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 05:40:17.748 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 05:40:17.748 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 05:40:17.748 | INFO | High Water Breakdown: | 20260505 05:40:17.748 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 05:40:17.748 | INFO | ------------------------------------------ | 20260505 05:40:17.748 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:40:17.748 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 05:40:17.748 | INFO | Total mem usage: 15.7 GiB | 20260505 05:40:17.748 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 05:40:17.748 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 05:40:17.748 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 05:40:17.748 | INFO | High Water Breakdown: | 20260505 05:40:17.748 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 05:40:17.748 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 05:40:17.748 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 05:40:17.748 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 05:40:17.748 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 05:40:17.748 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 05:40:17.748 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 05:40:17.748 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 05:40:17.748 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 05:40:17.748 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:40:17.748 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 05:40:17.748 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:40:17.748 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 05:40:17.748 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 05:40:17.748 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 05:40:17.748 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 05:40:17.748 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:40:17.748 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 05:40:17.748 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:40:17.748 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:40:17.748 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:40:17.748 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 05:40:17.748 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 05:40:17.748 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:40:17.748 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 05:40:17.748 | INFO | ------------------------------------------ | 20260505 05:40:17.748 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:40:17.749 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 05:40:17.749 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:40:17.749 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:40:17.749 | INFO | PerfReport: Full Execution -- 3.03e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260505 05:40:17.749 | INFO | PerfReport: 0.28%, AggregateStats -- 8.63e-02 / -nan, 4.43e-03 / 0.00e+00 | 20260505 05:40:17.749 | INFO | PerfReport: 10.88%, Baseline -- 3.30e+00 / -nan, 1.68e-02 / 0.00e+00 | 20260505 05:40:17.749 | INFO | PerfReport: 3.05%, Histogram -- 9.24e-01 / -nan, 5.26e-02 / 0.00e+00 | 20260505 05:40:17.749 | INFO | PerfReport: 16.03%, DME -- 3.88e+01 / -nan, 3.27e+00 / 0.00e+00 | 20260505 05:40:17.749 | INFO | PerfReport: 56.82%, FrameLabeling -- 1.72e+01 / -nan, 4.54e-02 / 0.00e+00 | 20260505 05:40:17.749 | INFO | PerfReport: 7.52%, PulseAccumulating -- 2.28e+00 / -nan, 1.48e-01 / 0.00e+00 | 20260505 05:40:17.749 | INFO | PerfReport: 3.84%, Metrics -- 1.16e+00 / -nan, 1.91e-01 / 0.00e+00 | 20260505 05:40:17.749 | INFO | PerfReport: 1.57%, PulseSerializing -- 4.76e-01 / -nan, 7.65e-02 / 0.00e+00 | 20260505 05:40:17.749 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 05:40:17.749 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:40:17.749 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:40:17.749 | INFO | PerfReport: Full Execution -- 9.07e+01 / -nan, 1.88e+01 / 0.00e+00 | 20260505 05:40:17.749 | INFO | PerfReport: 18.46%, Upload -- 1.68e+01 / 0.00e+00, 4.17e-01 / 0.00e+00 | 20260505 05:40:17.749 | INFO | PerfReport: 17.74%, Download -- 1.61e+01 / 0.00e+00, 7.53e-01 / 0.00e+00 | 20260505 05:40:17.749 | INFO | PerfReport: 33.39%, Compute -- 3.03e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260505 05:40:17.749 | INFO | PerfReport: 0.56%, LockUpload -- 5.12e-01 / 0.00e+00, 3.63e+00 / 0.00e+00 | 20260505 05:40:17.749 | INFO | PerfReport: 0.00%, LockDownload -- 3.98e-04 / 0.00e+00, 1.63e-04 / 0.00e+00 | 20260505 05:40:17.749 | INFO | PerfReport: 29.83%, LockCompute -- 2.71e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260505 05:40:17.749 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.18e-02 / 0.00e+00 | 20260505 05:40:17.844 | INFO | Starting a round of Baz IO | 20260505 05:40:20.784 | INFO | Populating RTMetrics report | 20260505 05:40:20.784 | INFO | RTMetrics report populated | 20260505 05:40:20.784 | INFO | Opening tmp RTMetrics json file | 20260505 05:40:20.793 | INFO | Serializing RTMetrics to json | 20260505 05:40:20.794 | INFO | Serialization done, swapping tmp file now... | 20260505 05:40:20.794 | INFO | RTMetrics done | 20260505 05:40:20.892 | INFO | Finished a round of Baz IO: Wrote 2.276 GiB in 3.047 seconds | 20260505 05:40:20.892 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.37% / 0.07% / 62.56% | 20260505 05:40:20.892 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.85488% / 0.000111262% / 5.16574e-05% | 20260505 05:40:20.892 | INFO | Baz IO overall: | 20260505 05:40:20.892 | INFO | BytesWritten : 370554118733 | 20260505 05:40:20.892 | INFO | NumEvents : 174333393344 | 20260505 05:40:22.886 | INFO | Analyzing chunk frames = [770048,770560) | 20260505 05:40:22.886 | INFO | MemInfo: { | 20260505 05:40:22.886 | INFO | "gpu" : " 17 GB", | 20260505 05:40:22.886 | INFO | "huge" : "", | 20260505 05:40:22.886 | INFO | "huge_shared" : " 27 GB", | 20260505 05:40:22.886 | INFO | "malloc" : " 14 GB", | 20260505 05:40:22.886 | INFO | "pinned" : " 29 GB" | 20260505 05:40:22.886 | INFO | } | 20260505 05:40:22.886 | INFO | Perf report for the last 5.13902 seconds: | 20260505 05:40:22.886 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:40:22.886 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:22.886 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:22.886 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:22.886 | INFO | ANALYSIS : 57.9%, 2.9, 88.6ms, 0 | 20260505 05:40:22.886 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:22.886 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260505 05:40:22.886 | INFO | PRE_HQ : 57.2%, 2.1, 63.7ms, 0 | 20260505 05:40:22.887 | INFO | BAZWRITER : 58.9%, 4.4, 133.3ms, 0 | 20260505 05:40:22.887 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 05:40:22.887 | INFO | Uploaded 36GiB to GPU in 1.65119s (21.8024GiB/s) | 20260505 05:40:22.887 | INFO | Downloaded 38GiB to GPU in 1.71465s (22.162GiB/s) | 20260505 05:40:28.026 | INFO | Analyzing chunk frames = [770560,771072) | 20260505 05:40:28.026 | INFO | MemInfo: { | 20260505 05:40:28.026 | INFO | "gpu" : " 17 GB", | 20260505 05:40:28.026 | INFO | "huge" : "", | 20260505 05:40:28.026 | INFO | "huge_shared" : " 27 GB", | 20260505 05:40:28.026 | INFO | "malloc" : " 14 GB", | 20260505 05:40:28.026 | INFO | "pinned" : " 29 GB" | 20260505 05:40:28.026 | INFO | } | 20260505 05:40:28.026 | INFO | Perf report for the last 5.13955 seconds: | 20260505 05:40:28.026 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:40:28.026 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:28.026 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:28.026 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:28.026 | INFO | ANALYSIS : 57.7%, 3.0, 90.0ms, 0 | 20260505 05:40:28.026 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:28.026 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:28.026 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:40:28.026 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:28.026 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:28.026 | INFO | Uploaded 36GiB to GPU in 1.60067s (22.4906GiB/s) | 20260505 05:40:28.026 | INFO | Downloaded 34GiB to GPU in 1.5169s (22.4142GiB/s) | 20260505 05:40:33.165 | INFO | Analyzing chunk frames = [771072,771584) | 20260505 05:40:33.165 | INFO | MemInfo: { | 20260505 05:40:33.165 | INFO | "gpu" : " 17 GB", | 20260505 05:40:33.165 | INFO | "huge" : "", | 20260505 05:40:33.165 | INFO | "huge_shared" : " 27 GB", | 20260505 05:40:33.165 | INFO | "malloc" : " 14 GB", | 20260505 05:40:33.165 | INFO | "pinned" : " 29 GB" | 20260505 05:40:33.165 | INFO | } | 20260505 05:40:33.165 | INFO | Perf report for the last 5.13929 seconds: | 20260505 05:40:33.165 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:40:33.165 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:33.165 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:33.165 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:33.165 | INFO | ANALYSIS : 57.1%, 3.0, 88.7ms, 0 | 20260505 05:40:33.165 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:33.165 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:33.165 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 05:40:33.165 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:33.165 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:33.165 | INFO | Uploaded 36GiB to GPU in 1.60599s (22.416GiB/s) | 20260505 05:40:33.166 | INFO | Downloaded 34GiB to GPU in 1.52371s (22.3139GiB/s) | 20260505 05:40:38.304 | INFO | Analyzing chunk frames = [771584,772096) | 20260505 05:40:38.304 | INFO | MemInfo: { | 20260505 05:40:38.304 | INFO | "gpu" : " 17 GB", | 20260505 05:40:38.304 | INFO | "huge" : "", | 20260505 05:40:38.304 | INFO | "huge_shared" : " 27 GB", | 20260505 05:40:38.304 | INFO | "malloc" : " 14 GB", | 20260505 05:40:38.304 | INFO | "pinned" : " 29 GB" | 20260505 05:40:38.305 | INFO | } | 20260505 05:40:38.305 | INFO | Perf report for the last 5.13942 seconds: | 20260505 05:40:38.305 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:40:38.305 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:38.305 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:38.305 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:38.305 | INFO | ANALYSIS : 56.8%, 3.0, 88.1ms, 0 | 20260505 05:40:38.305 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:38.305 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:38.305 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260505 05:40:38.305 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:38.305 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:38.305 | INFO | Uploaded 36GiB to GPU in 1.60437s (22.4386GiB/s) | 20260505 05:40:38.305 | INFO | Downloaded 34GiB to GPU in 1.52109s (22.3524GiB/s) | 20260505 05:40:43.444 | INFO | Analyzing chunk frames = [772096,772608) | 20260505 05:40:43.444 | INFO | MemInfo: { | 20260505 05:40:43.444 | INFO | "gpu" : " 17 GB", | 20260505 05:40:43.444 | INFO | "huge" : "", | 20260505 05:40:43.444 | INFO | "huge_shared" : " 27 GB", | 20260505 05:40:43.444 | INFO | "malloc" : " 14 GB", | 20260505 05:40:43.444 | INFO | "pinned" : " 29 GB" | 20260505 05:40:43.444 | INFO | } | 20260505 05:40:43.444 | INFO | Perf report for the last 5.13951 seconds: | 20260505 05:40:43.444 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:40:43.444 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:43.444 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:43.444 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:43.444 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260505 05:40:43.444 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:43.444 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:43.444 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:40:43.444 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:43.444 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:43.444 | INFO | Uploaded 36GiB to GPU in 1.60208s (22.4707GiB/s) | 20260505 05:40:43.445 | INFO | Downloaded 34GiB to GPU in 1.51844s (22.3915GiB/s) | 20260505 05:40:48.583 | INFO | Analyzing chunk frames = [772608,773120) | 20260505 05:40:48.583 | INFO | MemInfo: { | 20260505 05:40:48.583 | INFO | "gpu" : " 17 GB", | 20260505 05:40:48.583 | INFO | "huge" : "", | 20260505 05:40:48.583 | INFO | "huge_shared" : " 27 GB", | 20260505 05:40:48.583 | INFO | "malloc" : " 14 GB", | 20260505 05:40:48.583 | INFO | "pinned" : " 29 GB" | 20260505 05:40:48.583 | INFO | } | 20260505 05:40:48.583 | INFO | Perf report for the last 5.13921 seconds: | 20260505 05:40:48.583 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:40:48.583 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:48.583 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:48.583 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:48.583 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260505 05:40:48.583 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:48.583 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:48.583 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 05:40:48.583 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:48.583 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:48.584 | INFO | Uploaded 36GiB to GPU in 1.60396s (22.4445GiB/s) | 20260505 05:40:48.584 | INFO | Downloaded 34GiB to GPU in 1.52018s (22.3657GiB/s) | 20260505 05:40:53.722 | INFO | Analyzing chunk frames = [773120,773632) | 20260505 05:40:53.722 | INFO | MemInfo: { | 20260505 05:40:53.722 | INFO | "gpu" : " 17 GB", | 20260505 05:40:53.722 | INFO | "huge" : "", | 20260505 05:40:53.722 | INFO | "huge_shared" : " 27 GB", | 20260505 05:40:53.722 | INFO | "malloc" : " 14 GB", | 20260505 05:40:53.722 | INFO | "pinned" : " 29 GB" | 20260505 05:40:53.723 | INFO | } | 20260505 05:40:53.723 | INFO | Perf report for the last 5.13926 seconds: | 20260505 05:40:53.723 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:40:53.723 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:53.723 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:53.723 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:53.723 | INFO | ANALYSIS : 57.3%, 3.0, 89.3ms, 0 | 20260505 05:40:53.723 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:53.723 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:53.723 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 05:40:53.723 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:53.723 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:53.723 | INFO | Uploaded 36GiB to GPU in 1.60212s (22.4703GiB/s) | 20260505 05:40:53.723 | INFO | Downloaded 34GiB to GPU in 1.5199s (22.3698GiB/s) | 20260505 05:40:58.862 | INFO | Analyzing chunk frames = [773632,774144) | 20260505 05:40:58.862 | INFO | MemInfo: { | 20260505 05:40:58.862 | INFO | "gpu" : " 17 GB", | 20260505 05:40:58.862 | INFO | "huge" : "", | 20260505 05:40:58.862 | INFO | "huge_shared" : " 27 GB", | 20260505 05:40:58.862 | INFO | "malloc" : " 14 GB", | 20260505 05:40:58.862 | INFO | "pinned" : " 29 GB" | 20260505 05:40:58.862 | INFO | } | 20260505 05:40:58.862 | INFO | Perf report for the last 5.13933 seconds: | 20260505 05:40:58.862 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:40:58.862 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:58.862 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:58.862 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:58.862 | INFO | ANALYSIS : 57.3%, 3.0, 89.3ms, 0 | 20260505 05:40:58.862 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:58.862 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:58.862 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 05:40:58.862 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:58.862 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:40:58.862 | INFO | Uploaded 36GiB to GPU in 1.60356s (22.4501GiB/s) | 20260505 05:40:58.862 | INFO | Downloaded 34GiB to GPU in 1.52155s (22.3457GiB/s) | 20260505 05:40:58.959 | INFO | Starting a round of Baz IO | 20260505 05:41:01.900 | INFO | Populating RTMetrics report | 20260505 05:41:01.900 | INFO | RTMetrics report populated | 20260505 05:41:01.900 | INFO | Opening tmp RTMetrics json file | 20260505 05:41:01.916 | INFO | Serializing RTMetrics to json | 20260505 05:41:01.950 | INFO | Serialization done, swapping tmp file now... | 20260505 05:41:01.975 | INFO | RTMetrics done | 20260505 05:41:02.084 | INFO | Finished a round of Baz IO: Wrote 2.276 GiB in 3.124 seconds | 20260505 05:41:02.084 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.30% / 0.07% / 62.63% | 20260505 05:41:02.084 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.81378% / 0.000127157% / 3.97364e-05% | 20260505 05:41:02.084 | INFO | Baz IO overall: | 20260505 05:41:02.084 | INFO | BytesWritten : 372997721552 | 20260505 05:41:02.084 | INFO | NumEvents : 175498924497 | 20260505 05:41:04.001 | INFO | Analyzing chunk frames = [774144,774656) | 20260505 05:41:04.001 | INFO | MemInfo: { | 20260505 05:41:04.001 | INFO | "gpu" : " 17 GB", | 20260505 05:41:04.001 | INFO | "huge" : "", | 20260505 05:41:04.001 | INFO | "huge_shared" : " 27 GB", | 20260505 05:41:04.001 | INFO | "malloc" : " 14 GB", | 20260505 05:41:04.001 | INFO | "pinned" : " 29 GB" | 20260505 05:41:04.001 | INFO | } | 20260505 05:41:04.001 | INFO | Perf report for the last 5.13927 seconds: | 20260505 05:41:04.001 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:41:04.001 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:04.001 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:04.001 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:04.001 | INFO | ANALYSIS : 58.0%, 2.8, 85.2ms, 0 | 20260505 05:41:04.001 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:04.001 | INFO | RT_METRICS : 47.2%, 1.0, 24.7ms, 0 | 20260505 05:41:04.001 | INFO | PRE_HQ : 57.3%, 2.1, 63.2ms, 0 | 20260505 05:41:04.001 | INFO | BAZWRITER : 60.2%, 4.7, 144.8ms, 0 | 20260505 05:41:04.001 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260505 05:41:04.002 | INFO | Uploaded 36GiB to GPU in 1.66184s (21.6627GiB/s) | 20260505 05:41:04.002 | INFO | Downloaded 38GiB to GPU in 1.72734s (21.9992GiB/s) | 20260505 05:41:09.140 | INFO | Analyzing chunk frames = [774656,775168) | 20260505 05:41:09.140 | INFO | MemInfo: { | 20260505 05:41:09.140 | INFO | "gpu" : " 17 GB", | 20260505 05:41:09.140 | INFO | "huge" : "", | 20260505 05:41:09.140 | INFO | "huge_shared" : " 27 GB", | 20260505 05:41:09.140 | INFO | "malloc" : " 14 GB", | 20260505 05:41:09.140 | INFO | "pinned" : " 29 GB" | 20260505 05:41:09.140 | INFO | } | 20260505 05:41:09.141 | INFO | Perf report for the last 5.13931 seconds: | 20260505 05:41:09.141 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:41:09.141 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:09.141 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:09.141 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:09.141 | INFO | ANALYSIS : 57.7%, 3.0, 90.0ms, 0 | 20260505 05:41:09.141 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:09.141 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:09.141 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:41:09.141 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:09.141 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:09.141 | INFO | Uploaded 36GiB to GPU in 1.5982s (22.5254GiB/s) | 20260505 05:41:09.141 | INFO | Downloaded 34GiB to GPU in 1.51838s (22.3922GiB/s) | 20260505 05:41:14.280 | INFO | Analyzing chunk frames = [775168,775680) | 20260505 05:41:14.280 | INFO | MemInfo: { | 20260505 05:41:14.280 | INFO | "gpu" : " 17 GB", | 20260505 05:41:14.280 | INFO | "huge" : "", | 20260505 05:41:14.280 | INFO | "huge_shared" : " 27 GB", | 20260505 05:41:14.280 | INFO | "malloc" : " 14 GB", | 20260505 05:41:14.280 | INFO | "pinned" : " 29 GB" | 20260505 05:41:14.280 | INFO | } | 20260505 05:41:14.280 | INFO | Perf report for the last 5.13964 seconds: | 20260505 05:41:14.280 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:41:14.280 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:14.280 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:14.280 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:14.280 | INFO | ANALYSIS : 57.1%, 3.0, 88.7ms, 0 | 20260505 05:41:14.280 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:14.280 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:14.280 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:41:14.280 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:14.280 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:14.280 | INFO | Uploaded 36GiB to GPU in 1.60198s (22.4722GiB/s) | 20260505 05:41:14.281 | INFO | Downloaded 34GiB to GPU in 1.5221s (22.3376GiB/s) | 20260505 05:41:19.419 | INFO | Analyzing chunk frames = [775680,776192) | 20260505 05:41:19.419 | INFO | MemInfo: { | 20260505 05:41:19.419 | INFO | "gpu" : " 17 GB", | 20260505 05:41:19.419 | INFO | "huge" : "", | 20260505 05:41:19.419 | INFO | "huge_shared" : " 27 GB", | 20260505 05:41:19.419 | INFO | "malloc" : " 14 GB", | 20260505 05:41:19.419 | INFO | "pinned" : " 29 GB" | 20260505 05:41:19.419 | INFO | } | 20260505 05:41:19.419 | INFO | Perf report for the last 5.13909 seconds: | 20260505 05:41:19.419 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:41:19.419 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:19.419 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:19.419 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:19.419 | INFO | ANALYSIS : 56.8%, 3.0, 88.0ms, 0 | 20260505 05:41:19.419 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:19.419 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:19.419 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:41:19.419 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:19.419 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:19.420 | INFO | Uploaded 36GiB to GPU in 1.60192s (22.473GiB/s) | 20260505 05:41:19.420 | INFO | Downloaded 34GiB to GPU in 1.52195s (22.3397GiB/s) | 20260505 05:41:24.558 | INFO | Analyzing chunk frames = [776192,776704) | 20260505 05:41:24.558 | INFO | MemInfo: { | 20260505 05:41:24.558 | INFO | "gpu" : " 17 GB", | 20260505 05:41:24.558 | INFO | "huge" : "", | 20260505 05:41:24.558 | INFO | "huge_shared" : " 27 GB", | 20260505 05:41:24.559 | INFO | "malloc" : " 14 GB", | 20260505 05:41:24.559 | INFO | "pinned" : " 29 GB" | 20260505 05:41:24.559 | INFO | } | 20260505 05:41:24.559 | INFO | Perf report for the last 5.13933 seconds: | 20260505 05:41:24.559 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:41:24.559 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:24.559 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:24.559 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:24.559 | INFO | ANALYSIS : 56.8%, 3.0, 88.5ms, 0 | 20260505 05:41:24.559 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:24.559 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:24.559 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 05:41:24.559 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:24.559 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:24.559 | INFO | Uploaded 36GiB to GPU in 1.59978s (22.5031GiB/s) | 20260505 05:41:24.559 | INFO | Downloaded 34GiB to GPU in 1.51978s (22.3716GiB/s) | 20260505 05:41:29.698 | INFO | Analyzing chunk frames = [776704,777216) | 20260505 05:41:29.698 | INFO | MemInfo: { | 20260505 05:41:29.698 | INFO | "gpu" : " 17 GB", | 20260505 05:41:29.698 | INFO | "huge" : "", | 20260505 05:41:29.698 | INFO | "huge_shared" : " 27 GB", | 20260505 05:41:29.698 | INFO | "malloc" : " 14 GB", | 20260505 05:41:29.698 | INFO | "pinned" : " 29 GB" | 20260505 05:41:29.698 | INFO | } | 20260505 05:41:29.698 | INFO | Perf report for the last 5.13956 seconds: | 20260505 05:41:29.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:41:29.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:29.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:29.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:29.698 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260505 05:41:29.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:29.698 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:29.698 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:41:29.698 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:29.698 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:29.699 | INFO | Uploaded 36GiB to GPU in 1.6003s (22.4958GiB/s) | 20260505 05:41:29.699 | INFO | Downloaded 34GiB to GPU in 1.52043s (22.362GiB/s) | 20260505 05:41:34.837 | INFO | Analyzing chunk frames = [777216,777728) | 20260505 05:41:34.837 | INFO | MemInfo: { | 20260505 05:41:34.837 | INFO | "gpu" : " 17 GB", | 20260505 05:41:34.837 | INFO | "huge" : "", | 20260505 05:41:34.837 | INFO | "huge_shared" : " 27 GB", | 20260505 05:41:34.837 | INFO | "malloc" : " 14 GB", | 20260505 05:41:34.837 | INFO | "pinned" : " 29 GB" | 20260505 05:41:34.837 | INFO | } | 20260505 05:41:34.837 | INFO | Perf report for the last 5.13911 seconds: | 20260505 05:41:34.837 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:41:34.837 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:34.837 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:34.837 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:34.837 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260505 05:41:34.837 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:34.837 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:34.837 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260505 05:41:34.837 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:34.838 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:34.838 | INFO | Uploaded 36GiB to GPU in 1.60043s (22.4939GiB/s) | 20260505 05:41:34.838 | INFO | Downloaded 34GiB to GPU in 1.52135s (22.3485GiB/s) | 20260505 05:41:39.976 | INFO | Analyzing chunk frames = [777728,778240) | 20260505 05:41:39.976 | INFO | MemInfo: { | 20260505 05:41:39.976 | INFO | "gpu" : " 17 GB", | 20260505 05:41:39.976 | INFO | "huge" : "", | 20260505 05:41:39.976 | INFO | "huge_shared" : " 27 GB", | 20260505 05:41:39.976 | INFO | "malloc" : " 14 GB", | 20260505 05:41:39.977 | INFO | "pinned" : " 29 GB" | 20260505 05:41:39.977 | INFO | } | 20260505 05:41:39.977 | INFO | Perf report for the last 5.13921 seconds: | 20260505 05:41:39.977 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:41:39.977 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:39.977 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:39.977 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:39.977 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 05:41:39.977 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:39.977 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:39.977 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 05:41:39.977 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:39.977 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:39.977 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 05:41:39.977 | INFO | Total mem usage: 15.3 GiB | 20260505 05:41:39.977 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 05:41:39.977 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -1.43%) | 20260505 05:41:39.977 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 05:41:39.977 | INFO | High Water Breakdown: | 20260505 05:41:39.977 | INFO | 60.8% : 9.18 GiB ( 98.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 05:41:39.977 | INFO | 39.1% : 5.91 GiB ( 88.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 05:41:39.977 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 05:41:39.977 | INFO | ------------------------------------------ | 20260505 05:41:39.977 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:41:39.977 | INFO | Uploaded 36GiB to GPU in 1.6004s (22.4944GiB/s) | 20260505 05:41:39.977 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 05:41:39.977 | INFO | Downloaded 34GiB to GPU in 1.5208s (22.3566GiB/s) | 20260505 05:41:39.977 | INFO | Total mem usage: 26.5 GiB | 20260505 05:41:39.977 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 05:41:39.977 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 05:41:39.977 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 05:41:39.977 | INFO | High Water Breakdown: | 20260505 05:41:39.977 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:41:39.977 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:41:39.977 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:41:39.977 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:41:39.977 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:41:39.977 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:41:39.977 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 05:41:39.977 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 05:41:39.978 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:41:39.978 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 05:41:39.978 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 05:41:39.978 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 05:41:39.978 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 05:41:39.978 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 05:41:39.978 | INFO | ------------------------------------------ | 20260505 05:41:39.978 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:41:39.978 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 05:41:39.978 | INFO | Total mem usage: 25.6 GiB | 20260505 05:41:39.978 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 05:41:39.978 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 05:41:39.978 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 05:41:39.978 | INFO | High Water Breakdown: | 20260505 05:41:39.978 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 05:41:39.978 | INFO | ------------------------------------------ | 20260505 05:41:39.978 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:41:39.978 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 05:41:39.978 | INFO | Total mem usage: 15.7 GiB | 20260505 05:41:39.978 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 05:41:39.978 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 05:41:39.978 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 05:41:39.978 | INFO | High Water Breakdown: | 20260505 05:41:39.978 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 05:41:39.978 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 05:41:39.978 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 05:41:39.978 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 05:41:39.978 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 05:41:39.978 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 05:41:39.978 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 05:41:39.978 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 05:41:39.978 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 05:41:39.978 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:41:39.978 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 05:41:39.978 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:41:39.978 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 05:41:39.978 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 05:41:39.978 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 05:41:39.978 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 05:41:39.978 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:41:39.978 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 05:41:39.978 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:41:39.978 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:41:39.978 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:41:39.978 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 05:41:39.978 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 05:41:39.978 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:41:39.978 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 05:41:39.978 | INFO | ------------------------------------------ | 20260505 05:41:39.978 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:41:39.978 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 05:41:39.978 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:41:39.978 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:41:39.978 | INFO | PerfReport: Full Execution -- 3.03e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260505 05:41:39.978 | INFO | PerfReport: 0.29%, AggregateStats -- 8.63e-02 / -nan, 4.43e-03 / 0.00e+00 | 20260505 05:41:39.978 | INFO | PerfReport: 10.88%, Baseline -- 3.30e+00 / -nan, 1.62e-02 / 0.00e+00 | 20260505 05:41:39.978 | INFO | PerfReport: 3.05%, Histogram -- 9.24e-01 / -nan, 5.20e-02 / 0.00e+00 | 20260505 05:41:39.978 | INFO | PerfReport: 16.04%, DME -- 3.89e+01 / -nan, 3.20e+00 / 0.00e+00 | 20260505 05:41:39.978 | INFO | PerfReport: 56.82%, FrameLabeling -- 1.72e+01 / -nan, 4.64e-02 / 0.00e+00 | 20260505 05:41:39.978 | INFO | PerfReport: 7.51%, PulseAccumulating -- 2.27e+00 / -nan, 1.47e-01 / 0.00e+00 | 20260505 05:41:39.978 | INFO | PerfReport: 3.84%, Metrics -- 1.16e+00 / -nan, 1.91e-01 / 0.00e+00 | 20260505 05:41:39.978 | INFO | PerfReport: 1.57%, PulseSerializing -- 4.75e-01 / -nan, 7.63e-02 / 0.00e+00 | 20260505 05:41:39.978 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 05:41:39.978 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:41:39.978 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:41:39.978 | INFO | PerfReport: Full Execution -- 9.06e+01 / -nan, 1.88e+01 / 0.00e+00 | 20260505 05:41:39.978 | INFO | PerfReport: 18.50%, Upload -- 1.68e+01 / 0.00e+00, 4.23e-01 / 0.00e+00 | 20260505 05:41:39.978 | INFO | PerfReport: 17.78%, Download -- 1.61e+01 / 0.00e+00, 7.74e-01 / 0.00e+00 | 20260505 05:41:39.978 | INFO | PerfReport: 33.45%, Compute -- 3.03e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260505 05:41:39.978 | INFO | PerfReport: 0.56%, LockUpload -- 5.05e-01 / 0.00e+00, 3.63e+00 / 0.00e+00 | 20260505 05:41:39.978 | INFO | PerfReport: 0.00%, LockDownload -- 3.93e-04 / 0.00e+00, 9.22e-05 / 0.00e+00 | 20260505 05:41:39.978 | INFO | PerfReport: 29.69%, LockCompute -- 2.69e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260505 05:41:39.978 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.02e-02 / 0.00e+00 | 20260505 05:41:40.074 | INFO | Starting a round of Baz IO | 20260505 05:41:43.013 | INFO | Populating RTMetrics report | 20260505 05:41:43.013 | INFO | RTMetrics report populated | 20260505 05:41:43.013 | INFO | Opening tmp RTMetrics json file | 20260505 05:41:43.030 | INFO | Serializing RTMetrics to json | 20260505 05:41:43.040 | INFO | Serialization done, swapping tmp file now... | 20260505 05:41:43.056 | INFO | RTMetrics done | 20260505 05:41:43.132 | INFO | Finished a round of Baz IO: Wrote 2.28 GiB in 3.058 seconds | 20260505 05:41:43.132 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.23% / 0.07% / 62.70% | 20260505 05:41:43.132 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.80681% / 0.000111262% / 3.97364e-05% | 20260505 05:41:43.132 | INFO | Baz IO overall: | 20260505 05:41:43.132 | INFO | BytesWritten : 375445569649 | 20260505 05:41:43.132 | INFO | NumEvents : 176666966535 | 20260505 05:41:45.116 | INFO | Analyzing chunk frames = [778240,778752) | 20260505 05:41:45.116 | INFO | MemInfo: { | 20260505 05:41:45.116 | INFO | "gpu" : " 17 GB", | 20260505 05:41:45.116 | INFO | "huge" : "", | 20260505 05:41:45.116 | INFO | "huge_shared" : " 27 GB", | 20260505 05:41:45.116 | INFO | "malloc" : " 14 GB", | 20260505 05:41:45.116 | INFO | "pinned" : " 29 GB" | 20260505 05:41:45.116 | INFO | } | 20260505 05:41:45.116 | INFO | Perf report for the last 5.13937 seconds: | 20260505 05:41:45.116 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:41:45.116 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:45.116 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:45.116 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:45.116 | INFO | ANALYSIS : 57.9%, 2.9, 88.1ms, 0 | 20260505 05:41:45.116 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:45.116 | INFO | RT_METRICS : 46.4%, 1.0, 24.4ms, 0 | 20260505 05:41:45.116 | INFO | PRE_HQ : 57.3%, 2.1, 63.6ms, 0 | 20260505 05:41:45.116 | INFO | BAZWRITER : 59.1%, 4.4, 134.5ms, 0 | 20260505 05:41:45.116 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 05:41:45.116 | INFO | Uploaded 36GiB to GPU in 1.65155s (21.7978GiB/s) | 20260505 05:41:45.116 | INFO | Downloaded 38GiB to GPU in 1.71915s (22.104GiB/s) | 20260505 05:41:50.255 | INFO | Analyzing chunk frames = [778752,779264) | 20260505 05:41:50.255 | INFO | MemInfo: { | 20260505 05:41:50.255 | INFO | "gpu" : " 17 GB", | 20260505 05:41:50.255 | INFO | "huge" : "", | 20260505 05:41:50.255 | INFO | "huge_shared" : " 27 GB", | 20260505 05:41:50.255 | INFO | "malloc" : " 14 GB", | 20260505 05:41:50.255 | INFO | "pinned" : " 29 GB" | 20260505 05:41:50.255 | INFO | } | 20260505 05:41:50.255 | INFO | Perf report for the last 5.13947 seconds: | 20260505 05:41:50.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:41:50.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:50.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:50.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:50.255 | INFO | ANALYSIS : 57.7%, 3.0, 90.0ms, 0 | 20260505 05:41:50.255 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:50.255 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:50.255 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260505 05:41:50.255 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:50.255 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:50.256 | INFO | Uploaded 36GiB to GPU in 1.60083s (22.4884GiB/s) | 20260505 05:41:50.256 | INFO | Downloaded 34GiB to GPU in 1.51761s (22.4036GiB/s) | 20260505 05:41:55.395 | INFO | Analyzing chunk frames = [779264,779776) | 20260505 05:41:55.395 | INFO | MemInfo: { | 20260505 05:41:55.395 | INFO | "gpu" : " 17 GB", | 20260505 05:41:55.395 | INFO | "huge" : "", | 20260505 05:41:55.395 | INFO | "huge_shared" : " 27 GB", | 20260505 05:41:55.395 | INFO | "malloc" : " 14 GB", | 20260505 05:41:55.395 | INFO | "pinned" : " 29 GB" | 20260505 05:41:55.395 | INFO | } | 20260505 05:41:55.395 | INFO | Perf report for the last 5.13937 seconds: | 20260505 05:41:55.395 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:41:55.395 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:55.395 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:55.395 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:55.395 | INFO | ANALYSIS : 57.1%, 3.0, 88.7ms, 0 | 20260505 05:41:55.395 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:55.395 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:55.395 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:41:55.395 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:55.395 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:41:55.395 | INFO | Uploaded 36GiB to GPU in 1.60476s (22.4333GiB/s) | 20260505 05:41:55.395 | INFO | Downloaded 34GiB to GPU in 1.52186s (22.3411GiB/s) | 20260505 05:42:00.534 | INFO | Analyzing chunk frames = [779776,780288) | 20260505 05:42:00.534 | INFO | MemInfo: { | 20260505 05:42:00.534 | INFO | "gpu" : " 17 GB", | 20260505 05:42:00.534 | INFO | "huge" : "", | 20260505 05:42:00.534 | INFO | "huge_shared" : " 27 GB", | 20260505 05:42:00.534 | INFO | "malloc" : " 14 GB", | 20260505 05:42:00.534 | INFO | "pinned" : " 29 GB" | 20260505 05:42:00.534 | INFO | } | 20260505 05:42:00.535 | INFO | Perf report for the last 5.13971 seconds: | 20260505 05:42:00.535 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:42:00.535 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:00.535 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:00.535 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:00.535 | INFO | ANALYSIS : 56.7%, 3.0, 88.0ms, 0 | 20260505 05:42:00.535 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:00.535 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:00.535 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 05:42:00.535 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:00.535 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:00.535 | INFO | Uploaded 36GiB to GPU in 1.60464s (22.435GiB/s) | 20260505 05:42:00.535 | INFO | Downloaded 34GiB to GPU in 1.52168s (22.3437GiB/s) | 20260505 05:42:05.674 | INFO | Analyzing chunk frames = [780288,780800) | 20260505 05:42:05.674 | INFO | MemInfo: { | 20260505 05:42:05.674 | INFO | "gpu" : " 17 GB", | 20260505 05:42:05.674 | INFO | "huge" : "", | 20260505 05:42:05.674 | INFO | "huge_shared" : " 27 GB", | 20260505 05:42:05.674 | INFO | "malloc" : " 14 GB", | 20260505 05:42:05.674 | INFO | "pinned" : " 29 GB" | 20260505 05:42:05.674 | INFO | } | 20260505 05:42:05.674 | INFO | Perf report for the last 5.13973 seconds: | 20260505 05:42:05.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:42:05.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:05.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:05.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:05.674 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260505 05:42:05.674 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:05.674 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:05.674 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 05:42:05.674 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:05.674 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:05.675 | INFO | Uploaded 36GiB to GPU in 1.603s (22.4579GiB/s) | 20260505 05:42:05.675 | INFO | Downloaded 34GiB to GPU in 1.51898s (22.3835GiB/s) | 20260505 05:42:10.813 | INFO | Analyzing chunk frames = [780800,781312) | 20260505 05:42:10.813 | INFO | MemInfo: { | 20260505 05:42:10.813 | INFO | "gpu" : " 17 GB", | 20260505 05:42:10.813 | INFO | "huge" : "", | 20260505 05:42:10.813 | INFO | "huge_shared" : " 27 GB", | 20260505 05:42:10.813 | INFO | "malloc" : " 14 GB", | 20260505 05:42:10.813 | INFO | "pinned" : " 29 GB" | 20260505 05:42:10.813 | INFO | } | 20260505 05:42:10.813 | INFO | Perf report for the last 5.1387 seconds: | 20260505 05:42:10.813 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:42:10.813 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:10.813 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:10.813 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:10.813 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260505 05:42:10.813 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:10.813 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:10.813 | INFO | PRE_HQ : 34.3%, 1.0, 17.8ms, 0 | 20260505 05:42:10.813 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:10.813 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:10.813 | INFO | Uploaded 36GiB to GPU in 1.60389s (22.4454GiB/s) | 20260505 05:42:10.813 | INFO | Downloaded 34GiB to GPU in 1.52268s (22.3291GiB/s) | 20260505 05:42:15.952 | INFO | Analyzing chunk frames = [781312,781824) | 20260505 05:42:15.952 | INFO | MemInfo: { | 20260505 05:42:15.952 | INFO | "gpu" : " 17 GB", | 20260505 05:42:15.952 | INFO | "huge" : "", | 20260505 05:42:15.952 | INFO | "huge_shared" : " 27 GB", | 20260505 05:42:15.952 | INFO | "malloc" : " 14 GB", | 20260505 05:42:15.952 | INFO | "pinned" : " 29 GB" | 20260505 05:42:15.952 | INFO | } | 20260505 05:42:15.952 | INFO | Perf report for the last 5.1392 seconds: | 20260505 05:42:15.952 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:42:15.952 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:15.952 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:15.952 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:15.952 | INFO | ANALYSIS : 57.2%, 3.0, 89.3ms, 0 | 20260505 05:42:15.952 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:15.952 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:15.952 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 05:42:15.952 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:15.952 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:15.953 | INFO | Uploaded 36GiB to GPU in 1.60204s (22.4713GiB/s) | 20260505 05:42:15.953 | INFO | Downloaded 34GiB to GPU in 1.52041s (22.3623GiB/s) | 20260505 05:42:21.091 | INFO | Analyzing chunk frames = [781824,782336) | 20260505 05:42:21.091 | INFO | MemInfo: { | 20260505 05:42:21.091 | INFO | "gpu" : " 17 GB", | 20260505 05:42:21.091 | INFO | "huge" : "", | 20260505 05:42:21.091 | INFO | "huge_shared" : " 27 GB", | 20260505 05:42:21.091 | INFO | "malloc" : " 14 GB", | 20260505 05:42:21.091 | INFO | "pinned" : " 29 GB" | 20260505 05:42:21.091 | INFO | } | 20260505 05:42:21.091 | INFO | Perf report for the last 5.13916 seconds: | 20260505 05:42:21.091 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:42:21.091 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:21.091 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:21.091 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:21.091 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 05:42:21.091 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:21.091 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:21.091 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 05:42:21.091 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:21.092 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:21.092 | INFO | Uploaded 36GiB to GPU in 1.60231s (22.4675GiB/s) | 20260505 05:42:21.092 | INFO | Downloaded 34GiB to GPU in 1.52076s (22.3573GiB/s) | 20260505 05:42:21.189 | INFO | Starting a round of Baz IO | 20260505 05:42:24.127 | INFO | Populating RTMetrics report | 20260505 05:42:24.127 | INFO | RTMetrics report populated | 20260505 05:42:24.127 | INFO | Opening tmp RTMetrics json file | 20260505 05:42:24.143 | INFO | Serializing RTMetrics to json | 20260505 05:42:24.146 | INFO | Serialization done, swapping tmp file now... | 20260505 05:42:24.147 | INFO | RTMetrics done | 20260505 05:42:24.241 | INFO | Finished a round of Baz IO: Wrote 2.282 GiB in 3.052 seconds | 20260505 05:42:24.241 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.15% / 0.07% / 62.77% | 20260505 05:42:24.241 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.77574% / 0.000186761% / 4.76837e-05% | 20260505 05:42:24.241 | INFO | Baz IO overall: | 20260505 05:42:24.241 | INFO | BytesWritten : 377895913131 | 20260505 05:42:24.241 | INFO | NumEvents : 177835805442 | 20260505 05:42:26.231 | INFO | Analyzing chunk frames = [782336,782848) | 20260505 05:42:26.231 | INFO | MemInfo: { | 20260505 05:42:26.231 | INFO | "gpu" : " 17 GB", | 20260505 05:42:26.231 | INFO | "huge" : "", | 20260505 05:42:26.233 | INFO | "huge_shared" : " 27 GB", | 20260505 05:42:26.233 | INFO | "malloc" : " 14 GB", | 20260505 05:42:26.233 | INFO | "pinned" : " 29 GB" | 20260505 05:42:26.233 | INFO | } | 20260505 05:42:26.233 | INFO | Perf report for the last 5.14127 seconds: | 20260505 05:42:26.233 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:42:26.233 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:26.233 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:26.233 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:26.233 | INFO | ANALYSIS : 57.9%, 2.9, 87.9ms, 0 | 20260505 05:42:26.233 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:26.233 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260505 05:42:26.233 | INFO | PRE_HQ : 57.3%, 2.2, 64.1ms, 0 | 20260505 05:42:26.233 | INFO | BAZWRITER : 59.0%, 4.4, 134.4ms, 0 | 20260505 05:42:26.233 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 05:42:26.233 | INFO | Uploaded 36GiB to GPU in 1.66193s (21.6615GiB/s) | 20260505 05:42:26.233 | INFO | Downloaded 38GiB to GPU in 1.73062s (21.9574GiB/s) | 20260505 05:42:31.370 | INFO | Analyzing chunk frames = [782848,783360) | 20260505 05:42:31.370 | INFO | MemInfo: { | 20260505 05:42:31.370 | INFO | "gpu" : " 17 GB", | 20260505 05:42:31.370 | INFO | "huge" : "", | 20260505 05:42:31.370 | INFO | "huge_shared" : " 27 GB", | 20260505 05:42:31.370 | INFO | "malloc" : " 14 GB", | 20260505 05:42:31.370 | INFO | "pinned" : " 29 GB" | 20260505 05:42:31.370 | INFO | } | 20260505 05:42:31.370 | INFO | Perf report for the last 5.13779 seconds: | 20260505 05:42:31.370 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:42:31.370 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:31.370 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:31.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:31.371 | INFO | ANALYSIS : 57.8%, 3.0, 90.1ms, 0 | 20260505 05:42:31.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:31.371 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:31.371 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 05:42:31.371 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:31.371 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:31.371 | INFO | Uploaded 36GiB to GPU in 1.59867s (22.5187GiB/s) | 20260505 05:42:31.371 | INFO | Downloaded 34GiB to GPU in 1.51865s (22.3882GiB/s) | 20260505 05:42:36.509 | INFO | Analyzing chunk frames = [783360,783872) | 20260505 05:42:36.509 | INFO | MemInfo: { | 20260505 05:42:36.509 | INFO | "gpu" : " 17 GB", | 20260505 05:42:36.509 | INFO | "huge" : "", | 20260505 05:42:36.509 | INFO | "huge_shared" : " 27 GB", | 20260505 05:42:36.509 | INFO | "malloc" : " 14 GB", | 20260505 05:42:36.509 | INFO | "pinned" : " 29 GB" | 20260505 05:42:36.509 | INFO | } | 20260505 05:42:36.510 | INFO | Perf report for the last 5.13905 seconds: | 20260505 05:42:36.510 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:42:36.510 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:36.510 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:36.510 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:36.510 | INFO | ANALYSIS : 57.1%, 3.0, 88.7ms, 0 | 20260505 05:42:36.510 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:36.510 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:36.510 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260505 05:42:36.510 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:36.510 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:36.510 | INFO | Uploaded 36GiB to GPU in 1.6035s (22.4509GiB/s) | 20260505 05:42:36.510 | INFO | Downloaded 34GiB to GPU in 1.52401s (22.3096GiB/s) | 20260505 05:42:41.649 | INFO | Analyzing chunk frames = [783872,784384) | 20260505 05:42:41.649 | INFO | MemInfo: { | 20260505 05:42:41.649 | INFO | "gpu" : " 17 GB", | 20260505 05:42:41.649 | INFO | "huge" : "", | 20260505 05:42:41.649 | INFO | "huge_shared" : " 27 GB", | 20260505 05:42:41.649 | INFO | "malloc" : " 14 GB", | 20260505 05:42:41.649 | INFO | "pinned" : " 29 GB" | 20260505 05:42:41.649 | INFO | } | 20260505 05:42:41.649 | INFO | Perf report for the last 5.13935 seconds: | 20260505 05:42:41.649 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:42:41.649 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:41.649 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:41.649 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:41.649 | INFO | ANALYSIS : 56.8%, 3.0, 88.1ms, 0 | 20260505 05:42:41.649 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:41.649 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:41.649 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 05:42:41.649 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:41.649 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:41.649 | INFO | Uploaded 36GiB to GPU in 1.63025s (22.0825GiB/s) | 20260505 05:42:41.649 | INFO | Downloaded 34GiB to GPU in 1.64403s (20.6809GiB/s) | 20260505 05:42:46.788 | INFO | Analyzing chunk frames = [784384,784896) | 20260505 05:42:46.788 | INFO | MemInfo: { | 20260505 05:42:46.788 | INFO | "gpu" : " 17 GB", | 20260505 05:42:46.788 | INFO | "huge" : "", | 20260505 05:42:46.788 | INFO | "huge_shared" : " 27 GB", | 20260505 05:42:46.788 | INFO | "malloc" : " 14 GB", | 20260505 05:42:46.788 | INFO | "pinned" : " 29 GB" | 20260505 05:42:46.788 | INFO | } | 20260505 05:42:46.788 | INFO | Perf report for the last 5.13927 seconds: | 20260505 05:42:46.788 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:42:46.788 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:46.788 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:46.788 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:46.788 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260505 05:42:46.788 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:46.788 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:46.788 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 05:42:46.788 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:46.788 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:46.788 | INFO | Uploaded 36GiB to GPU in 1.63142s (22.0667GiB/s) | 20260505 05:42:46.788 | INFO | Downloaded 34GiB to GPU in 1.63046s (20.853GiB/s) | 20260505 05:42:51.927 | INFO | Analyzing chunk frames = [784896,785408) | 20260505 05:42:51.927 | INFO | MemInfo: { | 20260505 05:42:51.927 | INFO | "gpu" : " 17 GB", | 20260505 05:42:51.927 | INFO | "huge" : "", | 20260505 05:42:51.927 | INFO | "huge_shared" : " 27 GB", | 20260505 05:42:51.927 | INFO | "malloc" : " 14 GB", | 20260505 05:42:51.927 | INFO | "pinned" : " 29 GB" | 20260505 05:42:51.927 | INFO | } | 20260505 05:42:51.928 | INFO | Perf report for the last 5.13938 seconds: | 20260505 05:42:51.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:42:51.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:51.928 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:51.928 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:51.928 | INFO | ANALYSIS : 57.0%, 3.0, 89.0ms, 0 | 20260505 05:42:51.928 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:51.928 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:51.928 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 05:42:51.928 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:51.928 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:51.928 | INFO | Uploaded 36GiB to GPU in 1.63177s (22.062GiB/s) | 20260505 05:42:51.928 | INFO | Downloaded 34GiB to GPU in 1.6491s (20.6173GiB/s) | 20260505 05:42:57.067 | INFO | Analyzing chunk frames = [785408,785920) | 20260505 05:42:57.067 | INFO | MemInfo: { | 20260505 05:42:57.067 | INFO | "gpu" : " 17 GB", | 20260505 05:42:57.067 | INFO | "huge" : "", | 20260505 05:42:57.067 | INFO | "huge_shared" : " 27 GB", | 20260505 05:42:57.067 | INFO | "malloc" : " 14 GB", | 20260505 05:42:57.067 | INFO | "pinned" : " 29 GB" | 20260505 05:42:57.067 | INFO | } | 20260505 05:42:57.067 | INFO | Perf report for the last 5.13941 seconds: | 20260505 05:42:57.067 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:42:57.067 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:57.067 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:57.067 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:57.067 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260505 05:42:57.067 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:57.067 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:57.067 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 05:42:57.067 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:57.067 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:42:57.067 | INFO | Uploaded 36GiB to GPU in 1.60738s (22.3966GiB/s) | 20260505 05:42:57.067 | INFO | Downloaded 34GiB to GPU in 1.54382s (22.0233GiB/s) | 20260505 05:43:02.206 | INFO | Analyzing chunk frames = [785920,786432) | 20260505 05:43:02.206 | INFO | MemInfo: { | 20260505 05:43:02.206 | INFO | "gpu" : " 17 GB", | 20260505 05:43:02.206 | INFO | "huge" : "", | 20260505 05:43:02.206 | INFO | "huge_shared" : " 27 GB", | 20260505 05:43:02.206 | INFO | "malloc" : " 14 GB", | 20260505 05:43:02.206 | INFO | "pinned" : " 29 GB" | 20260505 05:43:02.206 | INFO | } | 20260505 05:43:02.206 | INFO | Perf report for the last 5.13925 seconds: | 20260505 05:43:02.206 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:43:02.206 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:02.206 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:02.206 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:02.206 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260505 05:43:02.206 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:02.206 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:02.206 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 05:43:02.206 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:02.206 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:02.207 | INFO | Uploaded 36GiB to GPU in 1.60221s (22.469GiB/s) | 20260505 05:43:02.207 | INFO | Downloaded 34GiB to GPU in 1.52246s (22.3322GiB/s) | 20260505 05:43:02.207 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 05:43:02.207 | INFO | Total mem usage: 15.3 GiB | 20260505 05:43:02.207 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 05:43:02.207 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -5.84%) | 20260505 05:43:02.207 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 05:43:02.207 | INFO | High Water Breakdown: | 20260505 05:43:02.207 | INFO | 60.1% : 8.71 GiB ( 93.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 05:43:02.207 | INFO | 39.7% : 5.75 GiB ( 86.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 05:43:02.207 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 05:43:02.207 | INFO | ------------------------------------------ | 20260505 05:43:02.207 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:43:02.207 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 05:43:02.207 | INFO | Total mem usage: 26.5 GiB | 20260505 05:43:02.207 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 05:43:02.207 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 05:43:02.207 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 05:43:02.207 | INFO | High Water Breakdown: | 20260505 05:43:02.207 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:43:02.207 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:43:02.207 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:43:02.207 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:43:02.207 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:43:02.207 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:43:02.207 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 05:43:02.207 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 05:43:02.207 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:43:02.207 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 05:43:02.207 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 05:43:02.207 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 05:43:02.207 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 05:43:02.207 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 05:43:02.207 | INFO | ------------------------------------------ | 20260505 05:43:02.207 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:43:02.207 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 05:43:02.207 | INFO | Total mem usage: 25.6 GiB | 20260505 05:43:02.207 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 05:43:02.207 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260505 05:43:02.207 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260505 05:43:02.207 | INFO | High Water Breakdown: | 20260505 05:43:02.207 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260505 05:43:02.207 | INFO | ------------------------------------------ | 20260505 05:43:02.207 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:43:02.207 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 05:43:02.207 | INFO | Total mem usage: 15.7 GiB | 20260505 05:43:02.207 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 05:43:02.207 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 05:43:02.207 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 05:43:02.207 | INFO | High Water Breakdown: | 20260505 05:43:02.207 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 05:43:02.207 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 05:43:02.207 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 05:43:02.208 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 05:43:02.208 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 05:43:02.208 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 05:43:02.208 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 05:43:02.208 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 05:43:02.208 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 05:43:02.208 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:43:02.208 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 05:43:02.208 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:43:02.208 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 05:43:02.208 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 05:43:02.208 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 05:43:02.208 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 05:43:02.208 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:43:02.208 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 05:43:02.208 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:43:02.208 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:43:02.208 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:43:02.208 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 05:43:02.208 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 05:43:02.208 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:43:02.208 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 05:43:02.208 | INFO | ------------------------------------------ | 20260505 05:43:02.208 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:43:02.208 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 05:43:02.208 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:43:02.208 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:43:02.208 | INFO | PerfReport: Full Execution -- 3.03e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260505 05:43:02.208 | INFO | PerfReport: 0.28%, AggregateStats -- 8.61e-02 / -nan, 4.14e-03 / 0.00e+00 | 20260505 05:43:02.208 | INFO | PerfReport: 10.89%, Baseline -- 3.30e+00 / -nan, 1.93e-02 / 0.00e+00 | 20260505 05:43:02.208 | INFO | PerfReport: 3.05%, Histogram -- 9.24e-01 / -nan, 5.18e-02 / 0.00e+00 | 20260505 05:43:02.208 | INFO | PerfReport: 16.03%, DME -- 3.88e+01 / -nan, 3.18e+00 / 0.00e+00 | 20260505 05:43:02.208 | INFO | PerfReport: 56.82%, FrameLabeling -- 1.72e+01 / -nan, 6.07e-02 / 0.00e+00 | 20260505 05:43:02.208 | INFO | PerfReport: 7.51%, PulseAccumulating -- 2.28e+00 / -nan, 1.47e-01 / 0.00e+00 | 20260505 05:43:02.208 | INFO | PerfReport: 3.84%, Metrics -- 1.16e+00 / -nan, 1.92e-01 / 0.00e+00 | 20260505 05:43:02.208 | INFO | PerfReport: 1.57%, PulseSerializing -- 4.75e-01 / -nan, 7.68e-02 / 0.00e+00 | 20260505 05:43:02.208 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 05:43:02.208 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:43:02.208 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:43:02.208 | INFO | PerfReport: Full Execution -- 9.07e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260505 05:43:02.208 | INFO | PerfReport: 18.55%, Upload -- 1.68e+01 / 0.00e+00, 4.62e-01 / 0.00e+00 | 20260505 05:43:02.208 | INFO | PerfReport: 18.04%, Download -- 1.64e+01 / 0.00e+00, 1.15e+00 / 0.00e+00 | 20260505 05:43:02.208 | INFO | PerfReport: 33.40%, Compute -- 3.03e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260505 05:43:02.208 | INFO | PerfReport: 0.57%, LockUpload -- 5.15e-01 / 0.00e+00, 3.64e+00 / 0.00e+00 | 20260505 05:43:02.208 | INFO | PerfReport: 0.00%, LockDownload -- 3.98e-04 / 0.00e+00, 1.40e-04 / 0.00e+00 | 20260505 05:43:02.208 | INFO | PerfReport: 29.43%, LockCompute -- 2.67e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260505 05:43:02.208 | INFO | PerfReport: 0.02%, Misc -- 1.38e-02 / 0.00e+00, 1.68e-02 / 0.00e+00 | 20260505 05:43:02.303 | INFO | Starting a round of Baz IO | 20260505 05:43:05.242 | INFO | Populating RTMetrics report | 20260505 05:43:05.242 | INFO | RTMetrics report populated | 20260505 05:43:05.242 | INFO | Opening tmp RTMetrics json file | 20260505 05:43:05.251 | INFO | Serializing RTMetrics to json | 20260505 05:43:05.252 | INFO | Serialization done, swapping tmp file now... | 20260505 05:43:05.253 | INFO | RTMetrics done | 20260505 05:43:05.357 | INFO | Finished a round of Baz IO: Wrote 2.283 GiB in 3.053 seconds | 20260505 05:43:05.357 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.08% / 0.07% / 62.85% | 20260505 05:43:05.357 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.73528% / 0.000143051% / 5.5631e-05% | 20260505 05:43:05.357 | INFO | Baz IO overall: | 20260505 05:43:05.357 | INFO | BytesWritten : 380347190801 | 20260505 05:43:05.357 | INFO | NumEvents : 179004586954 | 20260505 05:43:07.346 | INFO | Analyzing chunk frames = [786432,786944) | 20260505 05:43:07.346 | INFO | MemInfo: { | 20260505 05:43:07.346 | INFO | "gpu" : " 17 GB", | 20260505 05:43:07.346 | INFO | "huge" : "", | 20260505 05:43:07.346 | INFO | "huge_shared" : " 27 GB", | 20260505 05:43:07.346 | INFO | "malloc" : " 14 GB", | 20260505 05:43:07.346 | INFO | "pinned" : " 29 GB" | 20260505 05:43:07.346 | INFO | } | 20260505 05:43:07.346 | INFO | Perf report for the last 5.13959 seconds: | 20260505 05:43:07.346 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:43:07.346 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:07.346 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:07.346 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:07.346 | INFO | ANALYSIS : 57.9%, 2.9, 86.6ms, 0 | 20260505 05:43:07.346 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:07.346 | INFO | RT_METRICS : 46.5%, 1.0, 24.4ms, 0 | 20260505 05:43:07.346 | INFO | PRE_HQ : 57.4%, 2.2, 65.9ms, 0 | 20260505 05:43:07.346 | INFO | BAZWRITER : 59.0%, 4.5, 135.3ms, 0 | 20260505 05:43:07.346 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 05:43:07.346 | INFO | Uploaded 36GiB to GPU in 1.64841s (21.8392GiB/s) | 20260505 05:43:07.346 | INFO | Downloaded 38GiB to GPU in 1.72011s (22.0916GiB/s) | 20260505 05:43:12.485 | INFO | Analyzing chunk frames = [786944,787456) | 20260505 05:43:12.485 | INFO | MemInfo: { | 20260505 05:43:12.485 | INFO | "gpu" : " 17 GB", | 20260505 05:43:12.485 | INFO | "huge" : "", | 20260505 05:43:12.485 | INFO | "huge_shared" : " 27 GB", | 20260505 05:43:12.485 | INFO | "malloc" : " 14 GB", | 20260505 05:43:12.485 | INFO | "pinned" : " 29 GB" | 20260505 05:43:12.485 | INFO | } | 20260505 05:43:12.485 | INFO | Perf report for the last 5.13965 seconds: | 20260505 05:43:12.486 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:43:12.486 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:12.486 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:12.486 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:12.486 | INFO | ANALYSIS : 57.7%, 3.0, 90.1ms, 0 | 20260505 05:43:12.486 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:12.486 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:12.486 | INFO | PRE_HQ : 34.2%, 1.0, 17.8ms, 0 | 20260505 05:43:12.486 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:12.486 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:12.486 | INFO | Uploaded 36GiB to GPU in 1.61744s (22.2574GiB/s) | 20260505 05:43:12.486 | INFO | Downloaded 34GiB to GPU in 1.56059s (21.7866GiB/s) | 20260505 05:43:17.625 | INFO | Analyzing chunk frames = [787456,787968) | 20260505 05:43:17.625 | INFO | MemInfo: { | 20260505 05:43:17.625 | INFO | "gpu" : " 17 GB", | 20260505 05:43:17.625 | INFO | "huge" : "", | 20260505 05:43:17.625 | INFO | "huge_shared" : " 27 GB", | 20260505 05:43:17.625 | INFO | "malloc" : " 14 GB", | 20260505 05:43:17.625 | INFO | "pinned" : " 29 GB" | 20260505 05:43:17.625 | INFO | } | 20260505 05:43:17.625 | INFO | Perf report for the last 5.13943 seconds: | 20260505 05:43:17.625 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:43:17.625 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:17.625 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:17.625 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:17.625 | INFO | ANALYSIS : 57.1%, 3.0, 88.7ms, 0 | 20260505 05:43:17.625 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:17.625 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:17.625 | INFO | PRE_HQ : 35.1%, 1.0, 18.3ms, 0 | 20260505 05:43:17.625 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:17.625 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:17.625 | INFO | Uploaded 36GiB to GPU in 1.62138s (22.2033GiB/s) | 20260505 05:43:17.625 | INFO | Downloaded 34GiB to GPU in 1.56638s (21.7061GiB/s) | 20260505 05:43:22.765 | INFO | Analyzing chunk frames = [787968,788480) | 20260505 05:43:22.765 | INFO | MemInfo: { | 20260505 05:43:22.765 | INFO | "gpu" : " 17 GB", | 20260505 05:43:22.765 | INFO | "huge" : "", | 20260505 05:43:22.765 | INFO | "huge_shared" : " 27 GB", | 20260505 05:43:22.765 | INFO | "malloc" : " 14 GB", | 20260505 05:43:22.765 | INFO | "pinned" : " 29 GB" | 20260505 05:43:22.765 | INFO | } | 20260505 05:43:22.765 | INFO | Perf report for the last 5.13987 seconds: | 20260505 05:43:22.765 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:43:22.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:22.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:22.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:22.765 | INFO | ANALYSIS : 56.8%, 3.0, 88.1ms, 0 | 20260505 05:43:22.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:22.765 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:22.765 | INFO | PRE_HQ : 35.2%, 1.0, 18.3ms, 0 | 20260505 05:43:22.765 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:22.765 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:22.765 | INFO | Uploaded 36GiB to GPU in 1.61323s (22.3155GiB/s) | 20260505 05:43:22.765 | INFO | Downloaded 34GiB to GPU in 1.55436s (21.8739GiB/s) | 20260505 05:43:27.903 | INFO | Analyzing chunk frames = [788480,788992) | 20260505 05:43:27.903 | INFO | MemInfo: { | 20260505 05:43:27.903 | INFO | "gpu" : " 17 GB", | 20260505 05:43:27.903 | INFO | "huge" : "", | 20260505 05:43:27.903 | INFO | "huge_shared" : " 27 GB", | 20260505 05:43:27.903 | INFO | "malloc" : " 14 GB", | 20260505 05:43:27.903 | INFO | "pinned" : " 29 GB" | 20260505 05:43:27.903 | INFO | } | 20260505 05:43:27.903 | INFO | Perf report for the last 5.13818 seconds: | 20260505 05:43:27.903 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:43:27.903 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:27.903 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:27.903 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:27.903 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260505 05:43:27.903 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:27.903 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:27.903 | INFO | PRE_HQ : 34.9%, 1.0, 18.1ms, 0 | 20260505 05:43:27.903 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:27.903 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:27.903 | INFO | Uploaded 36GiB to GPU in 1.60804s (22.3875GiB/s) | 20260505 05:43:27.903 | INFO | Downloaded 34GiB to GPU in 1.54098s (22.0639GiB/s) | 20260505 05:43:33.042 | INFO | Analyzing chunk frames = [788992,789504) | 20260505 05:43:33.042 | INFO | MemInfo: { | 20260505 05:43:33.042 | INFO | "gpu" : " 17 GB", | 20260505 05:43:33.042 | INFO | "huge" : "", | 20260505 05:43:33.043 | INFO | "huge_shared" : " 27 GB", | 20260505 05:43:33.043 | INFO | "malloc" : " 14 GB", | 20260505 05:43:33.043 | INFO | "pinned" : " 29 GB" | 20260505 05:43:33.043 | INFO | } | 20260505 05:43:33.043 | INFO | Perf report for the last 5.13964 seconds: | 20260505 05:43:33.043 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:43:33.043 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:33.043 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:33.043 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:33.043 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260505 05:43:33.043 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:33.043 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:33.043 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 05:43:33.043 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:33.043 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:33.043 | INFO | Uploaded 36GiB to GPU in 1.60047s (22.4935GiB/s) | 20260505 05:43:33.043 | INFO | Downloaded 34GiB to GPU in 1.5206s (22.3597GiB/s) | 20260505 05:43:38.182 | INFO | Analyzing chunk frames = [789504,790016) | 20260505 05:43:38.182 | INFO | MemInfo: { | 20260505 05:43:38.182 | INFO | "gpu" : " 17 GB", | 20260505 05:43:38.182 | INFO | "huge" : "", | 20260505 05:43:38.182 | INFO | "huge_shared" : " 27 GB", | 20260505 05:43:38.182 | INFO | "malloc" : " 14 GB", | 20260505 05:43:38.182 | INFO | "pinned" : " 29 GB" | 20260505 05:43:38.182 | INFO | } | 20260505 05:43:38.182 | INFO | Perf report for the last 5.13938 seconds: | 20260505 05:43:38.182 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:43:38.182 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:38.182 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:38.182 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:38.182 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260505 05:43:38.182 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:38.182 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:38.182 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 05:43:38.182 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:38.182 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:38.182 | INFO | Uploaded 36GiB to GPU in 1.59986s (22.502GiB/s) | 20260505 05:43:38.182 | INFO | Downloaded 34GiB to GPU in 1.52198s (22.3393GiB/s) | 20260505 05:43:43.321 | INFO | Analyzing chunk frames = [790016,790528) | 20260505 05:43:43.321 | INFO | MemInfo: { | 20260505 05:43:43.321 | INFO | "gpu" : " 17 GB", | 20260505 05:43:43.321 | INFO | "huge" : "", | 20260505 05:43:43.321 | INFO | "huge_shared" : " 27 GB", | 20260505 05:43:43.321 | INFO | "malloc" : " 14 GB", | 20260505 05:43:43.321 | INFO | "pinned" : " 29 GB" | 20260505 05:43:43.321 | INFO | } | 20260505 05:43:43.321 | INFO | Perf report for the last 5.13915 seconds: | 20260505 05:43:43.321 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:43:43.321 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:43.321 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:43.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:43.321 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 05:43:43.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:43.321 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:43.321 | INFO | PRE_HQ : 33.4%, 1.0, 17.9ms, 0 | 20260505 05:43:43.321 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:43.321 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:43.322 | INFO | Uploaded 36GiB to GPU in 2.85782s (12.597GiB/s) | 20260505 05:43:43.322 | INFO | Downloaded 34GiB to GPU in 2.25049s (15.1078GiB/s) | 20260505 05:43:43.443 | INFO | Starting a round of Baz IO | 20260505 05:43:46.521 | INFO | Populating RTMetrics report | 20260505 05:43:46.521 | INFO | RTMetrics report populated | 20260505 05:43:46.521 | INFO | Opening tmp RTMetrics json file | 20260505 05:43:46.540 | INFO | Serializing RTMetrics to json | 20260505 05:43:46.541 | INFO | Serialization done, swapping tmp file now... | 20260505 05:43:46.541 | INFO | RTMetrics done | 20260505 05:43:46.628 | INFO | Finished a round of Baz IO: Wrote 2.285 GiB in 3.184 seconds | 20260505 05:43:46.628 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.01% / 0.07% / 62.92% | 20260505 05:43:46.628 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.70048% / 0.000111262% / 4.76837e-05% | 20260505 05:43:46.628 | INFO | Baz IO overall: | 20260505 05:43:46.628 | INFO | BytesWritten : 382800205573 | 20260505 05:43:46.628 | INFO | NumEvents : 180174415801 | 20260505 05:43:48.461 | INFO | Analyzing chunk frames = [790528,791040) | 20260505 05:43:48.461 | INFO | MemInfo: { | 20260505 05:43:48.461 | INFO | "gpu" : " 17 GB", | 20260505 05:43:48.461 | INFO | "huge" : "", | 20260505 05:43:48.461 | INFO | "huge_shared" : " 27 GB", | 20260505 05:43:48.461 | INFO | "malloc" : " 14 GB", | 20260505 05:43:48.461 | INFO | "pinned" : " 29 GB" | 20260505 05:43:48.461 | INFO | } | 20260505 05:43:48.461 | INFO | Perf report for the last 5.13977 seconds: | 20260505 05:43:48.461 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:43:48.461 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:48.461 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:48.461 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:48.461 | INFO | ANALYSIS : 61.1%, 2.9, 93.4ms, 0 | 20260505 05:43:48.461 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:48.461 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260505 05:43:48.461 | INFO | PRE_HQ : 60.0%, 2.1, 65.5ms, 0 | 20260505 05:43:48.461 | INFO | BAZWRITER : 61.3%, 4.2, 133.4ms, 0 | 20260505 05:43:48.461 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 05:43:48.461 | INFO | Uploaded 36GiB to GPU in 1.98642s (18.1231GiB/s) | 20260505 05:43:48.461 | INFO | Downloaded 38GiB to GPU in 1.93798s (19.6081GiB/s) | 20260505 05:43:53.600 | INFO | Analyzing chunk frames = [791040,791552) | 20260505 05:43:53.600 | INFO | MemInfo: { | 20260505 05:43:53.600 | INFO | "gpu" : " 17 GB", | 20260505 05:43:53.600 | INFO | "huge" : "", | 20260505 05:43:53.600 | INFO | "huge_shared" : " 27 GB", | 20260505 05:43:53.600 | INFO | "malloc" : " 14 GB", | 20260505 05:43:53.600 | INFO | "pinned" : " 29 GB" | 20260505 05:43:53.600 | INFO | } | 20260505 05:43:53.600 | INFO | Perf report for the last 5.13905 seconds: | 20260505 05:43:53.600 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:43:53.600 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:53.600 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:53.600 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:53.600 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260505 05:43:53.600 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:53.600 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:53.600 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 05:43:53.600 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:53.600 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:53.600 | INFO | Uploaded 36GiB to GPU in 1.94704s (18.4896GiB/s) | 20260505 05:43:53.600 | INFO | Downloaded 34GiB to GPU in 1.68832s (20.1384GiB/s) | 20260505 05:43:58.739 | INFO | Analyzing chunk frames = [791552,792064) | 20260505 05:43:58.739 | INFO | MemInfo: { | 20260505 05:43:58.739 | INFO | "gpu" : " 17 GB", | 20260505 05:43:58.739 | INFO | "huge" : "", | 20260505 05:43:58.739 | INFO | "huge_shared" : " 27 GB", | 20260505 05:43:58.739 | INFO | "malloc" : " 14 GB", | 20260505 05:43:58.739 | INFO | "pinned" : " 29 GB" | 20260505 05:43:58.739 | INFO | } | 20260505 05:43:58.739 | INFO | Perf report for the last 5.13934 seconds: | 20260505 05:43:58.739 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:43:58.739 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:58.739 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:58.739 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:58.739 | INFO | ANALYSIS : 60.0%, 3.0, 93.3ms, 0 | 20260505 05:43:58.739 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:58.739 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:58.739 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 05:43:58.739 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:58.739 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:43:58.740 | INFO | Uploaded 36GiB to GPU in 1.93695s (18.5859GiB/s) | 20260505 05:43:58.740 | INFO | Downloaded 34GiB to GPU in 1.70393s (19.9539GiB/s) | 20260505 05:44:03.879 | INFO | Analyzing chunk frames = [792064,792576) | 20260505 05:44:03.879 | INFO | MemInfo: { | 20260505 05:44:03.879 | INFO | "gpu" : " 17 GB", | 20260505 05:44:03.879 | INFO | "huge" : "", | 20260505 05:44:03.879 | INFO | "huge_shared" : " 27 GB", | 20260505 05:44:03.879 | INFO | "malloc" : " 14 GB", | 20260505 05:44:03.879 | INFO | "pinned" : " 29 GB" | 20260505 05:44:03.879 | INFO | } | 20260505 05:44:03.879 | INFO | Perf report for the last 5.13966 seconds: | 20260505 05:44:03.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:44:03.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:03.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:03.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:03.879 | INFO | ANALYSIS : 59.8%, 3.0, 92.9ms, 0 | 20260505 05:44:03.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:03.879 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:03.879 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260505 05:44:03.879 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:03.879 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:03.879 | INFO | Uploaded 36GiB to GPU in 1.94363s (18.5221GiB/s) | 20260505 05:44:03.879 | INFO | Downloaded 34GiB to GPU in 1.66887s (20.373GiB/s) | 20260505 05:44:09.018 | INFO | Analyzing chunk frames = [792576,793088) | 20260505 05:44:09.018 | INFO | MemInfo: { | 20260505 05:44:09.018 | INFO | "gpu" : " 17 GB", | 20260505 05:44:09.018 | INFO | "huge" : "", | 20260505 05:44:09.018 | INFO | "huge_shared" : " 27 GB", | 20260505 05:44:09.018 | INFO | "malloc" : " 14 GB", | 20260505 05:44:09.018 | INFO | "pinned" : " 29 GB" | 20260505 05:44:09.018 | INFO | } | 20260505 05:44:09.018 | INFO | Perf report for the last 5.13912 seconds: | 20260505 05:44:09.018 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:44:09.018 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:09.018 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:09.018 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:09.018 | INFO | ANALYSIS : 59.7%, 3.0, 93.1ms, 0 | 20260505 05:44:09.018 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:09.018 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:09.018 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 05:44:09.019 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:09.019 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:09.019 | INFO | Uploaded 36GiB to GPU in 1.95136s (18.4487GiB/s) | 20260505 05:44:09.019 | INFO | Downloaded 34GiB to GPU in 1.70823s (19.9036GiB/s) | 20260505 05:44:14.158 | INFO | Analyzing chunk frames = [793088,793600) | 20260505 05:44:14.158 | INFO | MemInfo: { | 20260505 05:44:14.158 | INFO | "gpu" : " 17 GB", | 20260505 05:44:14.158 | INFO | "huge" : "", | 20260505 05:44:14.158 | INFO | "huge_shared" : " 27 GB", | 20260505 05:44:14.158 | INFO | "malloc" : " 14 GB", | 20260505 05:44:14.158 | INFO | "pinned" : " 29 GB" | 20260505 05:44:14.158 | INFO | } | 20260505 05:44:14.158 | INFO | Perf report for the last 5.13988 seconds: | 20260505 05:44:14.158 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:44:14.158 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:14.158 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:14.158 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:14.158 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260505 05:44:14.158 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:14.158 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:14.158 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 05:44:14.158 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:14.158 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:14.158 | INFO | Uploaded 36GiB to GPU in 1.93546s (18.6002GiB/s) | 20260505 05:44:14.158 | INFO | Downloaded 34GiB to GPU in 1.7163s (19.81GiB/s) | 20260505 05:44:19.297 | INFO | Analyzing chunk frames = [793600,794112) | 20260505 05:44:19.297 | INFO | MemInfo: { | 20260505 05:44:19.297 | INFO | "gpu" : " 17 GB", | 20260505 05:44:19.297 | INFO | "huge" : "", | 20260505 05:44:19.297 | INFO | "huge_shared" : " 27 GB", | 20260505 05:44:19.297 | INFO | "malloc" : " 14 GB", | 20260505 05:44:19.297 | INFO | "pinned" : " 29 GB" | 20260505 05:44:19.297 | INFO | } | 20260505 05:44:19.297 | INFO | Perf report for the last 5.13889 seconds: | 20260505 05:44:19.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:44:19.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:19.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:19.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:19.297 | INFO | ANALYSIS : 60.3%, 3.0, 94.0ms, 0 | 20260505 05:44:19.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:19.297 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:19.297 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 05:44:19.297 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:19.297 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:19.297 | INFO | Uploaded 36GiB to GPU in 1.95562s (18.4085GiB/s) | 20260505 05:44:19.298 | INFO | Downloaded 34GiB to GPU in 1.63961s (20.7366GiB/s) | 20260505 05:44:24.436 | INFO | Analyzing chunk frames = [794112,794624) | 20260505 05:44:24.436 | INFO | MemInfo: { | 20260505 05:44:24.436 | INFO | "gpu" : " 17 GB", | 20260505 05:44:24.436 | INFO | "huge" : "", | 20260505 05:44:24.436 | INFO | "huge_shared" : " 27 GB", | 20260505 05:44:24.436 | INFO | "malloc" : " 14 GB", | 20260505 05:44:24.436 | INFO | "pinned" : " 29 GB" | 20260505 05:44:24.436 | INFO | } | 20260505 05:44:24.436 | INFO | Perf report for the last 5.13942 seconds: | 20260505 05:44:24.436 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:44:24.436 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:24.436 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:24.437 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:24.437 | INFO | ANALYSIS : 60.3%, 3.0, 94.1ms, 0 | 20260505 05:44:24.437 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:24.437 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:24.437 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 05:44:24.437 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:24.437 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:24.437 | INFO | Uploaded 36GiB to GPU in 1.93604s (18.5946GiB/s) | 20260505 05:44:24.437 | INFO | Downloaded 34GiB to GPU in 1.71851s (19.7846GiB/s) | 20260505 05:44:24.437 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 05:44:24.437 | INFO | Total mem usage: 15.3 GiB | 20260505 05:44:24.437 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 05:44:24.437 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -5.82%) | 20260505 05:44:24.437 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 05:44:24.437 | INFO | High Water Breakdown: | 20260505 05:44:24.437 | INFO | 60.2% : 8.71 GiB ( 93.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 05:44:24.437 | INFO | 39.7% : 5.75 GiB ( 86.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 05:44:24.437 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 05:44:24.437 | INFO | ------------------------------------------ | 20260505 05:44:24.437 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:44:24.437 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 05:44:24.437 | INFO | Total mem usage: 26.5 GiB | 20260505 05:44:24.437 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 05:44:24.437 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 05:44:24.437 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 05:44:24.437 | INFO | High Water Breakdown: | 20260505 05:44:24.437 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:44:24.437 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:44:24.437 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:44:24.437 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:44:24.437 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:44:24.438 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:44:24.438 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 05:44:24.438 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 05:44:24.438 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:44:24.438 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 05:44:24.438 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 05:44:24.438 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 05:44:24.438 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 05:44:24.438 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 05:44:24.438 | INFO | ------------------------------------------ | 20260505 05:44:24.438 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:44:24.438 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 05:44:24.438 | INFO | Total mem usage: 25.6 GiB | 20260505 05:44:24.438 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 05:44:24.438 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 05:44:24.438 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 05:44:24.438 | INFO | High Water Breakdown: | 20260505 05:44:24.438 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 05:44:24.438 | INFO | ------------------------------------------ | 20260505 05:44:24.438 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:44:24.438 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 05:44:24.438 | INFO | Total mem usage: 15.7 GiB | 20260505 05:44:24.438 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 05:44:24.438 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 05:44:24.438 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 05:44:24.438 | INFO | High Water Breakdown: | 20260505 05:44:24.438 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 05:44:24.438 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 05:44:24.438 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 05:44:24.438 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 05:44:24.438 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 05:44:24.438 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 05:44:24.438 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 05:44:24.438 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 05:44:24.438 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 05:44:24.438 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:44:24.438 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 05:44:24.438 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:44:24.438 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 05:44:24.438 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 05:44:24.438 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 05:44:24.438 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 05:44:24.438 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:44:24.438 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 05:44:24.438 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:44:24.438 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:44:24.438 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:44:24.438 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 05:44:24.438 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 05:44:24.438 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:44:24.438 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 05:44:24.439 | INFO | ------------------------------------------ | 20260505 05:44:24.439 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:44:24.439 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 05:44:24.439 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:44:24.439 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:44:24.439 | INFO | PerfReport: Full Execution -- 3.03e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260505 05:44:24.439 | INFO | PerfReport: 0.29%, AggregateStats -- 8.68e-02 / -nan, 5.18e-03 / 0.00e+00 | 20260505 05:44:24.439 | INFO | PerfReport: 10.90%, Baseline -- 3.31e+00 / -nan, 7.17e-02 / 0.00e+00 | 20260505 05:44:24.439 | INFO | PerfReport: 3.05%, Histogram -- 9.27e-01 / -nan, 5.87e-02 / 0.00e+00 | 20260505 05:44:24.439 | INFO | PerfReport: 16.04%, DME -- 3.89e+01 / -nan, 3.17e+00 / 0.00e+00 | 20260505 05:44:24.439 | INFO | PerfReport: 56.79%, FrameLabeling -- 1.72e+01 / -nan, 8.23e-02 / 0.00e+00 | 20260505 05:44:24.439 | INFO | PerfReport: 7.50%, PulseAccumulating -- 2.28e+00 / -nan, 1.51e-01 / 0.00e+00 | 20260505 05:44:24.439 | INFO | PerfReport: 3.85%, Metrics -- 1.17e+00 / -nan, 1.99e-01 / 0.00e+00 | 20260505 05:44:24.439 | INFO | PerfReport: 1.57%, PulseSerializing -- 4.76e-01 / -nan, 7.76e-02 / 0.00e+00 | 20260505 05:44:24.439 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 05:44:24.439 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:44:24.439 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:44:24.439 | INFO | PerfReport: Full Execution -- 9.37e+01 / -nan, 2.10e+01 / 0.00e+00 | 20260505 05:44:24.439 | INFO | PerfReport: 20.67%, Upload -- 1.94e+01 / 0.00e+00, 6.10e+00 / 0.00e+00 | 20260505 05:44:24.439 | INFO | PerfReport: 18.77%, Download -- 1.76e+01 / 0.00e+00, 3.83e+00 / 0.00e+00 | 20260505 05:44:24.439 | INFO | PerfReport: 32.38%, Compute -- 3.03e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260505 05:44:24.439 | INFO | PerfReport: 2.70%, LockUpload -- 2.53e+00 / 0.00e+00, 7.98e+00 / 0.00e+00 | 20260505 05:44:24.439 | INFO | PerfReport: 0.21%, LockDownload -- 2.00e-01 / 0.00e+00, 1.28e+00 / 0.00e+00 | 20260505 05:44:24.439 | INFO | PerfReport: 25.25%, LockCompute -- 2.37e+01 / 0.00e+00, 1.77e+01 / 0.00e+00 | 20260505 05:44:24.439 | INFO | PerfReport: 0.02%, Misc -- 1.42e-02 / 0.00e+00, 2.56e-02 / 0.00e+00 | 20260505 05:44:24.556 | INFO | Starting a round of Baz IO | 20260505 05:44:27.642 | INFO | Populating RTMetrics report | 20260505 05:44:27.642 | INFO | RTMetrics report populated | 20260505 05:44:27.642 | INFO | Opening tmp RTMetrics json file | 20260505 05:44:27.645 | INFO | Serializing RTMetrics to json | 20260505 05:44:27.690 | INFO | Serialization done, swapping tmp file now... | 20260505 05:44:27.714 | INFO | RTMetrics done | 20260505 05:44:27.779 | INFO | Finished a round of Baz IO: Wrote 2.285 GiB in 3.223 seconds | 20260505 05:44:27.779 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.94% / 0.07% / 62.99% | 20260505 05:44:27.779 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.66395% / 0.000103315% / 4.76837e-05% | 20260505 05:44:27.779 | INFO | Baz IO overall: | 20260505 05:44:27.779 | INFO | BytesWritten : 385253945777 | 20260505 05:44:27.779 | INFO | NumEvents : 181344602354 | 20260505 05:44:29.576 | INFO | Analyzing chunk frames = [794624,795136) | 20260505 05:44:29.576 | INFO | MemInfo: { | 20260505 05:44:29.576 | INFO | "gpu" : " 17 GB", | 20260505 05:44:29.576 | INFO | "huge" : "", | 20260505 05:44:29.576 | INFO | "huge_shared" : " 27 GB", | 20260505 05:44:29.576 | INFO | "malloc" : " 14 GB", | 20260505 05:44:29.576 | INFO | "pinned" : " 29 GB" | 20260505 05:44:29.576 | INFO | } | 20260505 05:44:29.576 | INFO | Perf report for the last 5.13933 seconds: | 20260505 05:44:29.576 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:44:29.576 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:29.576 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:29.576 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:29.576 | INFO | ANALYSIS : 61.2%, 2.9, 93.6ms, 0 | 20260505 05:44:29.576 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:29.576 | INFO | RT_METRICS : 47.2%, 1.0, 24.7ms, 0 | 20260505 05:44:29.576 | INFO | PRE_HQ : 59.9%, 2.1, 65.0ms, 0 | 20260505 05:44:29.576 | INFO | BAZWRITER : 62.1%, 4.2, 134.9ms, 0 | 20260505 05:44:29.576 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 05:44:29.576 | INFO | Uploaded 36GiB to GPU in 1.99554s (18.0403GiB/s) | 20260505 05:44:29.576 | INFO | Downloaded 38GiB to GPU in 1.91793s (19.8131GiB/s) | 20260505 05:44:34.715 | INFO | Analyzing chunk frames = [795136,795648) | 20260505 05:44:34.715 | INFO | MemInfo: { | 20260505 05:44:34.715 | INFO | "gpu" : " 17 GB", | 20260505 05:44:34.715 | INFO | "huge" : "", | 20260505 05:44:34.715 | INFO | "huge_shared" : " 27 GB", | 20260505 05:44:34.715 | INFO | "malloc" : " 14 GB", | 20260505 05:44:34.715 | INFO | "pinned" : " 29 GB" | 20260505 05:44:34.715 | INFO | } | 20260505 05:44:34.715 | INFO | Perf report for the last 5.13956 seconds: | 20260505 05:44:34.715 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:44:34.715 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:34.715 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:34.715 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:34.715 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260505 05:44:34.715 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:34.715 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:34.715 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260505 05:44:34.715 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:34.716 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:34.716 | INFO | Uploaded 36GiB to GPU in 1.93511s (18.6036GiB/s) | 20260505 05:44:34.716 | INFO | Downloaded 34GiB to GPU in 1.70119s (19.986GiB/s) | 20260505 05:44:39.854 | INFO | Analyzing chunk frames = [795648,796160) | 20260505 05:44:39.854 | INFO | MemInfo: { | 20260505 05:44:39.854 | INFO | "gpu" : " 17 GB", | 20260505 05:44:39.854 | INFO | "huge" : "", | 20260505 05:44:39.854 | INFO | "huge_shared" : " 27 GB", | 20260505 05:44:39.854 | INFO | "malloc" : " 14 GB", | 20260505 05:44:39.854 | INFO | "pinned" : " 29 GB" | 20260505 05:44:39.854 | INFO | } | 20260505 05:44:39.855 | INFO | Perf report for the last 5.13918 seconds: | 20260505 05:44:39.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:44:39.855 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:39.855 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:39.855 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:39.855 | INFO | ANALYSIS : 60.1%, 3.0, 93.5ms, 0 | 20260505 05:44:39.855 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:39.855 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:39.855 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 05:44:39.855 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:39.855 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:39.855 | INFO | Uploaded 36GiB to GPU in 1.93844s (18.5716GiB/s) | 20260505 05:44:39.855 | INFO | Downloaded 34GiB to GPU in 1.69755s (20.0289GiB/s) | 20260505 05:44:44.993 | INFO | Analyzing chunk frames = [796160,796672) | 20260505 05:44:44.993 | INFO | MemInfo: { | 20260505 05:44:44.993 | INFO | "gpu" : " 17 GB", | 20260505 05:44:44.993 | INFO | "huge" : "", | 20260505 05:44:44.993 | INFO | "huge_shared" : " 27 GB", | 20260505 05:44:44.993 | INFO | "malloc" : " 14 GB", | 20260505 05:44:44.993 | INFO | "pinned" : " 29 GB" | 20260505 05:44:44.993 | INFO | } | 20260505 05:44:44.994 | INFO | Perf report for the last 5.13898 seconds: | 20260505 05:44:44.994 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:44:44.994 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:44.994 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:44.994 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:44.994 | INFO | ANALYSIS : 59.8%, 3.0, 92.9ms, 0 | 20260505 05:44:44.994 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:44.994 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:44.994 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260505 05:44:44.994 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:44.994 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:44.994 | INFO | Uploaded 36GiB to GPU in 1.94917s (18.4694GiB/s) | 20260505 05:44:44.994 | INFO | Downloaded 34GiB to GPU in 1.66295s (20.4456GiB/s) | 20260505 05:44:50.133 | INFO | Analyzing chunk frames = [796672,797184) | 20260505 05:44:50.133 | INFO | MemInfo: { | 20260505 05:44:50.133 | INFO | "gpu" : " 17 GB", | 20260505 05:44:50.133 | INFO | "huge" : "", | 20260505 05:44:50.133 | INFO | "huge_shared" : " 27 GB", | 20260505 05:44:50.133 | INFO | "malloc" : " 14 GB", | 20260505 05:44:50.133 | INFO | "pinned" : " 29 GB" | 20260505 05:44:50.133 | INFO | } | 20260505 05:44:50.133 | INFO | Perf report for the last 5.1397 seconds: | 20260505 05:44:50.133 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:44:50.133 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:50.133 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:50.133 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:50.133 | INFO | ANALYSIS : 59.8%, 3.0, 93.3ms, 0 | 20260505 05:44:50.133 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:50.133 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:50.133 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 05:44:50.133 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:50.133 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:50.134 | INFO | Uploaded 36GiB to GPU in 1.95367s (18.4268GiB/s) | 20260505 05:44:50.134 | INFO | Downloaded 34GiB to GPU in 1.70743s (19.913GiB/s) | 20260505 05:44:55.272 | INFO | Analyzing chunk frames = [797184,797696) | 20260505 05:44:55.272 | INFO | MemInfo: { | 20260505 05:44:55.272 | INFO | "gpu" : " 17 GB", | 20260505 05:44:55.272 | INFO | "huge" : "", | 20260505 05:44:55.272 | INFO | "huge_shared" : " 27 GB", | 20260505 05:44:55.272 | INFO | "malloc" : " 14 GB", | 20260505 05:44:55.272 | INFO | "pinned" : " 29 GB" | 20260505 05:44:55.273 | INFO | } | 20260505 05:44:55.273 | INFO | Perf report for the last 5.13936 seconds: | 20260505 05:44:55.273 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:44:55.273 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:55.273 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:55.273 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:55.273 | INFO | ANALYSIS : 60.0%, 3.0, 93.6ms, 0 | 20260505 05:44:55.273 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:55.273 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:55.273 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 05:44:55.273 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:55.273 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:44:55.273 | INFO | Uploaded 36GiB to GPU in 1.95445s (18.4195GiB/s) | 20260505 05:44:55.273 | INFO | Downloaded 34GiB to GPU in 1.69653s (20.0409GiB/s) | 20260505 05:45:00.411 | INFO | Analyzing chunk frames = [797696,798208) | 20260505 05:45:00.411 | INFO | MemInfo: { | 20260505 05:45:00.411 | INFO | "gpu" : " 17 GB", | 20260505 05:45:00.411 | INFO | "huge" : "", | 20260505 05:45:00.411 | INFO | "huge_shared" : " 27 GB", | 20260505 05:45:00.411 | INFO | "malloc" : " 14 GB", | 20260505 05:45:00.411 | INFO | "pinned" : " 29 GB" | 20260505 05:45:00.412 | INFO | } | 20260505 05:45:00.412 | INFO | Perf report for the last 5.1389 seconds: | 20260505 05:45:00.412 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:45:00.412 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:00.412 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:00.412 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:00.412 | INFO | ANALYSIS : 60.3%, 3.0, 94.1ms, 0 | 20260505 05:45:00.412 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:00.412 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:00.412 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260505 05:45:00.412 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:00.412 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:00.412 | INFO | Uploaded 36GiB to GPU in 1.93657s (18.5896GiB/s) | 20260505 05:45:00.412 | INFO | Downloaded 34GiB to GPU in 1.72592s (19.6996GiB/s) | 20260505 05:45:05.551 | INFO | Analyzing chunk frames = [798208,798720) | 20260505 05:45:05.551 | INFO | MemInfo: { | 20260505 05:45:05.551 | INFO | "gpu" : " 17 GB", | 20260505 05:45:05.551 | INFO | "huge" : "", | 20260505 05:45:05.551 | INFO | "huge_shared" : " 27 GB", | 20260505 05:45:05.551 | INFO | "malloc" : " 14 GB", | 20260505 05:45:05.551 | INFO | "pinned" : " 29 GB" | 20260505 05:45:05.551 | INFO | } | 20260505 05:45:05.551 | INFO | Perf report for the last 5.13983 seconds: | 20260505 05:45:05.551 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:45:05.551 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:05.551 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:05.551 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:05.551 | INFO | ANALYSIS : 60.3%, 3.0, 94.1ms, 0 | 20260505 05:45:05.552 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:05.552 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:05.552 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260505 05:45:05.552 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:05.552 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:05.552 | INFO | Uploaded 36GiB to GPU in 1.94862s (18.4747GiB/s) | 20260505 05:45:05.552 | INFO | Downloaded 34GiB to GPU in 1.65812s (20.5052GiB/s) | 20260505 05:45:05.673 | INFO | Starting a round of Baz IO | 20260505 05:45:08.753 | INFO | Populating RTMetrics report | 20260505 05:45:08.754 | INFO | RTMetrics report populated | 20260505 05:45:08.754 | INFO | Opening tmp RTMetrics json file | 20260505 05:45:08.768 | INFO | Serializing RTMetrics to json | 20260505 05:45:08.769 | INFO | Serialization done, swapping tmp file now... | 20260505 05:45:08.770 | INFO | RTMetrics done | 20260505 05:45:08.865 | INFO | Finished a round of Baz IO: Wrote 2.287 GiB in 3.192 seconds | 20260505 05:45:08.866 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.87% / 0.07% / 63.06% | 20260505 05:45:08.866 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.64576% / 0.000119209% / 6.75519e-05% | 20260505 05:45:08.866 | INFO | Baz IO overall: | 20260505 05:45:08.866 | INFO | BytesWritten : 387710100052 | 20260505 05:45:08.866 | INFO | NumEvents : 182516963358 | 20260505 05:45:10.690 | INFO | Analyzing chunk frames = [798720,799232) | 20260505 05:45:10.690 | INFO | MemInfo: { | 20260505 05:45:10.690 | INFO | "gpu" : " 17 GB", | 20260505 05:45:10.690 | INFO | "huge" : "", | 20260505 05:45:10.690 | INFO | "huge_shared" : " 27 GB", | 20260505 05:45:10.690 | INFO | "malloc" : " 14 GB", | 20260505 05:45:10.691 | INFO | "pinned" : " 29 GB" | 20260505 05:45:10.691 | INFO | } | 20260505 05:45:10.691 | INFO | Perf report for the last 5.13924 seconds: | 20260505 05:45:10.691 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:45:10.691 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:10.691 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:10.691 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:10.691 | INFO | ANALYSIS : 61.2%, 2.9, 93.1ms, 0 | 20260505 05:45:10.691 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:10.691 | INFO | RT_METRICS : 46.2%, 1.0, 24.2ms, 0 | 20260505 05:45:10.691 | INFO | PRE_HQ : 59.7%, 2.1, 64.8ms, 0 | 20260505 05:45:10.691 | INFO | BAZWRITER : 61.5%, 4.3, 134.9ms, 0 | 20260505 05:45:10.691 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 05:45:10.691 | INFO | Uploaded 36GiB to GPU in 1.98937s (18.0962GiB/s) | 20260505 05:45:10.691 | INFO | Downloaded 38GiB to GPU in 1.88411s (20.1687GiB/s) | 20260505 05:45:15.829 | INFO | Analyzing chunk frames = [799232,799744) | 20260505 05:45:15.830 | INFO | MemInfo: { | 20260505 05:45:15.830 | INFO | "gpu" : " 17 GB", | 20260505 05:45:15.830 | INFO | "huge" : "", | 20260505 05:45:15.830 | INFO | "huge_shared" : " 27 GB", | 20260505 05:45:15.830 | INFO | "malloc" : " 14 GB", | 20260505 05:45:15.830 | INFO | "pinned" : " 29 GB" | 20260505 05:45:15.830 | INFO | } | 20260505 05:45:15.830 | INFO | Perf report for the last 5.13912 seconds: | 20260505 05:45:15.830 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:45:15.830 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:15.830 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:15.830 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:15.830 | INFO | ANALYSIS : 60.8%, 3.0, 94.9ms, 0 | 20260505 05:45:15.830 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:15.830 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:15.830 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 05:45:15.830 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:15.830 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:15.830 | INFO | Uploaded 36GiB to GPU in 1.95041s (18.4577GiB/s) | 20260505 05:45:15.830 | INFO | Downloaded 34GiB to GPU in 1.73303s (19.6188GiB/s) | 20260505 05:45:20.969 | INFO | Analyzing chunk frames = [799744,800256) | 20260505 05:45:20.969 | INFO | MemInfo: { | 20260505 05:45:20.969 | INFO | "gpu" : " 17 GB", | 20260505 05:45:20.969 | INFO | "huge" : "", | 20260505 05:45:20.969 | INFO | "huge_shared" : " 27 GB", | 20260505 05:45:20.969 | INFO | "malloc" : " 14 GB", | 20260505 05:45:20.969 | INFO | "pinned" : " 29 GB" | 20260505 05:45:20.969 | INFO | } | 20260505 05:45:20.969 | INFO | Perf report for the last 5.13918 seconds: | 20260505 05:45:20.969 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:45:20.969 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:20.969 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:20.969 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:20.969 | INFO | ANALYSIS : 60.1%, 3.0, 93.5ms, 0 | 20260505 05:45:20.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:20.969 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:20.969 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 05:45:20.969 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:20.969 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:20.969 | INFO | Uploaded 36GiB to GPU in 1.95276s (18.4354GiB/s) | 20260505 05:45:20.969 | INFO | Downloaded 34GiB to GPU in 1.74058s (19.5337GiB/s) | 20260505 05:45:26.108 | INFO | Analyzing chunk frames = [800256,800768) | 20260505 05:45:26.108 | INFO | MemInfo: { | 20260505 05:45:26.108 | INFO | "gpu" : " 17 GB", | 20260505 05:45:26.108 | INFO | "huge" : "", | 20260505 05:45:26.108 | INFO | "huge_shared" : " 27 GB", | 20260505 05:45:26.108 | INFO | "malloc" : " 14 GB", | 20260505 05:45:26.109 | INFO | "pinned" : " 29 GB" | 20260505 05:45:26.109 | INFO | } | 20260505 05:45:26.109 | INFO | Perf report for the last 5.13978 seconds: | 20260505 05:45:26.109 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:45:26.109 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:26.109 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:26.109 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:26.109 | INFO | ANALYSIS : 59.9%, 3.0, 93.0ms, 0 | 20260505 05:45:26.109 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:26.109 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:26.109 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 05:45:26.109 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:26.109 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:26.109 | INFO | Uploaded 36GiB to GPU in 1.97143s (18.2609GiB/s) | 20260505 05:45:26.109 | INFO | Downloaded 34GiB to GPU in 1.70405s (19.9524GiB/s) | 20260505 05:45:31.247 | INFO | Analyzing chunk frames = [800768,801280) | 20260505 05:45:31.247 | INFO | MemInfo: { | 20260505 05:45:31.247 | INFO | "gpu" : " 17 GB", | 20260505 05:45:31.247 | INFO | "huge" : "", | 20260505 05:45:31.247 | INFO | "huge_shared" : " 27 GB", | 20260505 05:45:31.247 | INFO | "malloc" : " 14 GB", | 20260505 05:45:31.247 | INFO | "pinned" : " 29 GB" | 20260505 05:45:31.247 | INFO | } | 20260505 05:45:31.248 | INFO | Perf report for the last 5.13883 seconds: | 20260505 05:45:31.248 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:45:31.248 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:31.248 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:31.248 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:31.248 | INFO | ANALYSIS : 59.7%, 3.0, 93.2ms, 0 | 20260505 05:45:31.248 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:31.248 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:31.248 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260505 05:45:31.248 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:31.248 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:31.248 | INFO | Uploaded 36GiB to GPU in 1.95915s (18.3754GiB/s) | 20260505 05:45:31.248 | INFO | Downloaded 34GiB to GPU in 1.73179s (19.6329GiB/s) | 20260505 05:45:36.388 | INFO | Analyzing chunk frames = [801280,801792) | 20260505 05:45:36.388 | INFO | MemInfo: { | 20260505 05:45:36.388 | INFO | "gpu" : " 17 GB", | 20260505 05:45:36.388 | INFO | "huge" : "", | 20260505 05:45:36.388 | INFO | "huge_shared" : " 27 GB", | 20260505 05:45:36.388 | INFO | "malloc" : " 14 GB", | 20260505 05:45:36.388 | INFO | "pinned" : " 29 GB" | 20260505 05:45:36.388 | INFO | } | 20260505 05:45:36.388 | INFO | Perf report for the last 5.14013 seconds: | 20260505 05:45:36.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:45:36.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:36.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:36.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:36.388 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260505 05:45:36.388 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:36.388 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:36.388 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 05:45:36.388 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:36.388 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:36.388 | INFO | Uploaded 36GiB to GPU in 1.97379s (18.239GiB/s) | 20260505 05:45:36.388 | INFO | Downloaded 34GiB to GPU in 1.72139s (19.7515GiB/s) | 20260505 05:45:41.530 | INFO | Analyzing chunk frames = [801792,802304) | 20260505 05:45:41.530 | INFO | MemInfo: { | 20260505 05:45:41.530 | INFO | "gpu" : " 17 GB", | 20260505 05:45:41.530 | INFO | "huge" : "", | 20260505 05:45:41.530 | INFO | "huge_shared" : " 27 GB", | 20260505 05:45:41.530 | INFO | "malloc" : " 14 GB", | 20260505 05:45:41.530 | INFO | "pinned" : " 29 GB" | 20260505 05:45:41.530 | INFO | } | 20260505 05:45:41.530 | INFO | Perf report for the last 5.14243 seconds: | 20260505 05:45:41.530 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:45:41.530 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:41.530 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:41.530 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:41.530 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260505 05:45:41.530 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:41.530 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:41.530 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260505 05:45:41.530 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:41.530 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:41.531 | INFO | Uploaded 36GiB to GPU in 1.98651s (18.1222GiB/s) | 20260505 05:45:41.531 | INFO | Downloaded 34GiB to GPU in 1.67071s (20.3507GiB/s) | 20260505 05:45:46.665 | INFO | Analyzing chunk frames = [802304,802816) | 20260505 05:45:46.665 | INFO | MemInfo: { | 20260505 05:45:46.665 | INFO | "gpu" : " 17 GB", | 20260505 05:45:46.665 | INFO | "huge" : "", | 20260505 05:45:46.665 | INFO | "huge_shared" : " 27 GB", | 20260505 05:45:46.665 | INFO | "malloc" : " 14 GB", | 20260505 05:45:46.665 | INFO | "pinned" : " 29 GB" | 20260505 05:45:46.665 | INFO | } | 20260505 05:45:46.665 | INFO | Perf report for the last 5.13522 seconds: | 20260505 05:45:46.666 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:45:46.666 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:46.666 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:46.666 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:46.666 | INFO | ANALYSIS : 60.9%, 3.0, 95.0ms, 0 | 20260505 05:45:46.666 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:46.666 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:46.666 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 05:45:46.666 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:46.666 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:46.666 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 05:45:46.666 | INFO | Total mem usage: 15.3 GiB | 20260505 05:45:46.666 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 05:45:46.666 | INFO | Uploaded 36GiB to GPU in 1.98918s (18.0979GiB/s) | 20260505 05:45:46.666 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -5.8%) | 20260505 05:45:46.666 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 05:45:46.666 | INFO | High Water Breakdown: | 20260505 05:45:46.666 | INFO | 60% : 8.69 GiB ( 92.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 05:45:46.666 | INFO | Downloaded 34GiB to GPU in 1.72663s (19.6916GiB/s) | 20260505 05:45:46.666 | INFO | 39.9% : 5.77 GiB ( 86.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 05:45:46.666 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 05:45:46.666 | INFO | ------------------------------------------ | 20260505 05:45:46.666 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:45:46.666 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 05:45:46.666 | INFO | Total mem usage: 26.5 GiB | 20260505 05:45:46.666 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 05:45:46.666 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 05:45:46.666 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 05:45:46.666 | INFO | High Water Breakdown: | 20260505 05:45:46.666 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:45:46.666 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:45:46.666 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:45:46.666 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:45:46.666 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:45:46.666 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:45:46.666 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 05:45:46.666 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 05:45:46.666 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:45:46.666 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 05:45:46.666 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 05:45:46.666 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 05:45:46.666 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 05:45:46.666 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 05:45:46.666 | INFO | ------------------------------------------ | 20260505 05:45:46.666 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:45:46.666 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 05:45:46.666 | INFO | Total mem usage: 25.6 GiB | 20260505 05:45:46.666 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 05:45:46.666 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 05:45:46.666 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 05:45:46.666 | INFO | High Water Breakdown: | 20260505 05:45:46.666 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 05:45:46.666 | INFO | ------------------------------------------ | 20260505 05:45:46.666 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:45:46.667 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 05:45:46.667 | INFO | Total mem usage: 15.7 GiB | 20260505 05:45:46.667 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 05:45:46.667 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 05:45:46.667 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 05:45:46.667 | INFO | High Water Breakdown: | 20260505 05:45:46.667 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 05:45:46.667 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 05:45:46.667 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 05:45:46.667 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 05:45:46.667 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 05:45:46.667 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 05:45:46.667 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 05:45:46.667 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 05:45:46.667 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 05:45:46.667 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:45:46.667 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 05:45:46.667 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:45:46.667 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 05:45:46.667 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 05:45:46.667 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 05:45:46.667 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 05:45:46.667 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:45:46.667 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 05:45:46.667 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:45:46.667 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:45:46.667 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:45:46.667 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 05:45:46.667 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 05:45:46.667 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:45:46.667 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 05:45:46.667 | INFO | ------------------------------------------ | 20260505 05:45:46.667 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:45:46.667 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 05:45:46.667 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:45:46.667 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:45:46.667 | INFO | PerfReport: Full Execution -- 3.04e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260505 05:45:46.667 | INFO | PerfReport: 0.29%, AggregateStats -- 8.71e-02 / -nan, 5.82e-03 / 0.00e+00 | 20260505 05:45:46.667 | INFO | PerfReport: 10.93%, Baseline -- 3.32e+00 / -nan, 1.06e-01 / 0.00e+00 | 20260505 05:45:46.667 | INFO | PerfReport: 3.07%, Histogram -- 9.31e-01 / -nan, 6.97e-02 / 0.00e+00 | 20260505 05:45:46.667 | INFO | PerfReport: 16.04%, DME -- 3.90e+01 / -nan, 3.17e+00 / 0.00e+00 | 20260505 05:45:46.667 | INFO | PerfReport: 56.74%, FrameLabeling -- 1.72e+01 / -nan, 9.26e-02 / 0.00e+00 | 20260505 05:45:46.667 | INFO | PerfReport: 7.50%, PulseAccumulating -- 2.28e+00 / -nan, 1.54e-01 / 0.00e+00 | 20260505 05:45:46.667 | INFO | PerfReport: 3.86%, Metrics -- 1.17e+00 / -nan, 2.11e-01 / 0.00e+00 | 20260505 05:45:46.667 | INFO | PerfReport: 1.57%, PulseSerializing -- 4.77e-01 / -nan, 7.83e-02 / 0.00e+00 | 20260505 05:45:46.667 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 05:45:46.667 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:45:46.667 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:45:46.667 | INFO | PerfReport: Full Execution -- 9.57e+01 / -nan, 2.29e+01 / 0.00e+00 | 20260505 05:45:46.667 | INFO | PerfReport: 21.35%, Upload -- 2.04e+01 / 0.00e+00, 7.27e+00 / 0.00e+00 | 20260505 05:45:46.667 | INFO | PerfReport: 18.83%, Download -- 1.80e+01 / 0.00e+00, 4.56e+00 / 0.00e+00 | 20260505 05:45:46.667 | INFO | PerfReport: 31.74%, Compute -- 3.04e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260505 05:45:46.667 | INFO | PerfReport: 3.72%, LockUpload -- 3.56e+00 / 0.00e+00, 9.89e+00 / 0.00e+00 | 20260505 05:45:46.667 | INFO | PerfReport: 0.39%, LockDownload -- 3.75e-01 / 0.00e+00, 1.81e+00 / 0.00e+00 | 20260505 05:45:46.667 | INFO | PerfReport: 23.95%, LockCompute -- 2.29e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260505 05:45:46.667 | INFO | PerfReport: 0.02%, Misc -- 1.44e-02 / 0.00e+00, 1.85e-02 / 0.00e+00 | 20260505 05:45:46.794 | INFO | Starting a round of Baz IO | 20260505 05:45:49.892 | INFO | Populating RTMetrics report | 20260505 05:45:49.893 | INFO | RTMetrics report populated | 20260505 05:45:49.893 | INFO | Opening tmp RTMetrics json file | 20260505 05:45:49.911 | INFO | Serializing RTMetrics to json | 20260505 05:45:49.916 | INFO | Serialization done, swapping tmp file now... | 20260505 05:45:49.918 | INFO | RTMetrics done | 20260505 05:45:50.003 | INFO | Finished a round of Baz IO: Wrote 2.287 GiB in 3.208 seconds | 20260505 05:45:50.003 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.80% / 0.07% / 63.13% | 20260505 05:45:50.003 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.60079% / 0.000111262% / 4.37101e-05% | 20260505 05:45:50.003 | INFO | Baz IO overall: | 20260505 05:45:50.003 | INFO | BytesWritten : 390166052680 | 20260505 05:45:50.003 | INFO | NumEvents : 183689037113 | 20260505 05:45:51.805 | INFO | Analyzing chunk frames = [802816,803328) | 20260505 05:45:51.805 | INFO | MemInfo: { | 20260505 05:45:51.805 | INFO | "gpu" : " 17 GB", | 20260505 05:45:51.805 | INFO | "huge" : "", | 20260505 05:45:51.805 | INFO | "huge_shared" : " 27 GB", | 20260505 05:45:51.805 | INFO | "malloc" : " 14 GB", | 20260505 05:45:51.805 | INFO | "pinned" : " 29 GB" | 20260505 05:45:51.805 | INFO | } | 20260505 05:45:51.805 | INFO | Perf report for the last 5.13953 seconds: | 20260505 05:45:51.805 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:45:51.805 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:51.805 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:51.805 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:51.805 | INFO | ANALYSIS : 61.6%, 2.9, 94.2ms, 0 | 20260505 05:45:51.805 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:51.805 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260505 05:45:51.805 | INFO | PRE_HQ : 60.0%, 2.1, 65.1ms, 0 | 20260505 05:45:51.805 | INFO | BAZWRITER : 61.8%, 4.2, 134.7ms, 0 | 20260505 05:45:51.805 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 05:45:51.805 | INFO | Uploaded 36GiB to GPU in 2.03435s (17.696GiB/s) | 20260505 05:45:51.806 | INFO | Downloaded 38GiB to GPU in 1.92029s (19.7887GiB/s) | 20260505 05:45:56.944 | INFO | Analyzing chunk frames = [803328,803840) | 20260505 05:45:56.944 | INFO | MemInfo: { | 20260505 05:45:56.944 | INFO | "gpu" : " 17 GB", | 20260505 05:45:56.944 | INFO | "huge" : "", | 20260505 05:45:56.944 | INFO | "huge_shared" : " 27 GB", | 20260505 05:45:56.944 | INFO | "malloc" : " 14 GB", | 20260505 05:45:56.944 | INFO | "pinned" : " 29 GB" | 20260505 05:45:56.944 | INFO | } | 20260505 05:45:56.945 | INFO | Perf report for the last 5.13953 seconds: | 20260505 05:45:56.945 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:45:56.945 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:56.945 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:56.945 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:56.945 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260505 05:45:56.945 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:56.945 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:56.945 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:45:56.945 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:56.945 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:45:56.945 | INFO | Uploaded 36GiB to GPU in 2.00022s (17.998GiB/s) | 20260505 05:45:56.945 | INFO | Downloaded 34GiB to GPU in 1.72183s (19.7464GiB/s) | 20260505 05:46:02.084 | INFO | Analyzing chunk frames = [803840,804352) | 20260505 05:46:02.084 | INFO | MemInfo: { | 20260505 05:46:02.084 | INFO | "gpu" : " 17 GB", | 20260505 05:46:02.084 | INFO | "huge" : "", | 20260505 05:46:02.084 | INFO | "huge_shared" : " 27 GB", | 20260505 05:46:02.084 | INFO | "malloc" : " 14 GB", | 20260505 05:46:02.084 | INFO | "pinned" : " 29 GB" | 20260505 05:46:02.084 | INFO | } | 20260505 05:46:02.084 | INFO | Perf report for the last 5.13936 seconds: | 20260505 05:46:02.084 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:46:02.084 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:02.084 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:02.084 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:02.084 | INFO | ANALYSIS : 65.3%, 3.0, 101.7ms, 0 | 20260505 05:46:02.084 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:02.084 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:02.084 | INFO | PRE_HQ : 31.7%, 1.0, 16.4ms, 0 | 20260505 05:46:02.084 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:02.084 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:02.084 | INFO | Uploaded 36GiB to GPU in 2.35166s (15.3083GiB/s) | 20260505 05:46:02.084 | INFO | Downloaded 34GiB to GPU in 2.01646s (16.8612GiB/s) | 20260505 05:46:07.224 | INFO | Analyzing chunk frames = [804352,804864) | 20260505 05:46:07.224 | INFO | MemInfo: { | 20260505 05:46:07.224 | INFO | "gpu" : " 17 GB", | 20260505 05:46:07.224 | INFO | "huge" : "", | 20260505 05:46:07.224 | INFO | "huge_shared" : " 27 GB", | 20260505 05:46:07.224 | INFO | "malloc" : " 14 GB", | 20260505 05:46:07.224 | INFO | "pinned" : " 29 GB" | 20260505 05:46:07.224 | INFO | } | 20260505 05:46:07.224 | INFO | Perf report for the last 5.14008 seconds: | 20260505 05:46:07.224 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:46:07.224 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:07.224 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:07.224 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:07.224 | INFO | ANALYSIS : 64.3%, 3.0, 99.9ms, 0 | 20260505 05:46:07.224 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:07.224 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:07.224 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260505 05:46:07.224 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:07.224 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:07.224 | INFO | Uploaded 36GiB to GPU in 2.23074s (16.1381GiB/s) | 20260505 05:46:07.224 | INFO | Downloaded 34GiB to GPU in 1.94522s (17.4787GiB/s) | 20260505 05:46:12.363 | INFO | Analyzing chunk frames = [804864,805376) | 20260505 05:46:12.363 | INFO | MemInfo: { | 20260505 05:46:12.363 | INFO | "gpu" : " 17 GB", | 20260505 05:46:12.363 | INFO | "huge" : "", | 20260505 05:46:12.363 | INFO | "huge_shared" : " 27 GB", | 20260505 05:46:12.363 | INFO | "malloc" : " 14 GB", | 20260505 05:46:12.363 | INFO | "pinned" : " 29 GB" | 20260505 05:46:12.363 | INFO | } | 20260505 05:46:12.363 | INFO | Perf report for the last 5.13903 seconds: | 20260505 05:46:12.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:46:12.363 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:12.363 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:12.363 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:12.363 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 05:46:12.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:12.363 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:12.363 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260505 05:46:12.363 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:12.363 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:12.363 | INFO | Uploaded 36GiB to GPU in 2.30295s (15.6321GiB/s) | 20260505 05:46:12.363 | INFO | Downloaded 34GiB to GPU in 2.0807s (16.3407GiB/s) | 20260505 05:46:17.502 | INFO | Analyzing chunk frames = [805376,805888) | 20260505 05:46:17.502 | INFO | MemInfo: { | 20260505 05:46:17.502 | INFO | "gpu" : " 17 GB", | 20260505 05:46:17.502 | INFO | "huge" : "", | 20260505 05:46:17.502 | INFO | "huge_shared" : " 27 GB", | 20260505 05:46:17.502 | INFO | "malloc" : " 14 GB", | 20260505 05:46:17.502 | INFO | "pinned" : " 29 GB" | 20260505 05:46:17.502 | INFO | } | 20260505 05:46:17.502 | INFO | Perf report for the last 5.13924 seconds: | 20260505 05:46:17.502 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:46:17.502 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:17.502 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:17.502 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:17.502 | INFO | ANALYSIS : 65.0%, 3.0, 101.6ms, 0 | 20260505 05:46:17.503 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:17.503 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:17.503 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 05:46:17.503 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:17.503 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:17.503 | INFO | Uploaded 36GiB to GPU in 2.23979s (16.0729GiB/s) | 20260505 05:46:17.503 | INFO | Downloaded 34GiB to GPU in 1.99407s (17.0506GiB/s) | 20260505 05:46:22.641 | INFO | Analyzing chunk frames = [805888,806400) | 20260505 05:46:22.641 | INFO | MemInfo: { | 20260505 05:46:22.641 | INFO | "gpu" : " 17 GB", | 20260505 05:46:22.641 | INFO | "huge" : "", | 20260505 05:46:22.641 | INFO | "huge_shared" : " 27 GB", | 20260505 05:46:22.641 | INFO | "malloc" : " 14 GB", | 20260505 05:46:22.641 | INFO | "pinned" : " 29 GB" | 20260505 05:46:22.641 | INFO | } | 20260505 05:46:22.641 | INFO | Perf report for the last 5.13869 seconds: | 20260505 05:46:22.641 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:46:22.641 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:22.641 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:22.641 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:22.641 | INFO | ANALYSIS : 65.8%, 3.0, 102.8ms, 0 | 20260505 05:46:22.641 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:22.641 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:22.641 | INFO | PRE_HQ : 32.8%, 1.0, 17.1ms, 0 | 20260505 05:46:22.641 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:22.641 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:22.642 | INFO | Uploaded 36GiB to GPU in 2.34547s (15.3487GiB/s) | 20260505 05:46:22.642 | INFO | Downloaded 34GiB to GPU in 2.18466s (15.563GiB/s) | 20260505 05:46:27.780 | INFO | Analyzing chunk frames = [806400,806912) | 20260505 05:46:27.780 | INFO | MemInfo: { | 20260505 05:46:27.780 | INFO | "gpu" : " 17 GB", | 20260505 05:46:27.780 | INFO | "huge" : "", | 20260505 05:46:27.780 | INFO | "huge_shared" : " 27 GB", | 20260505 05:46:27.780 | INFO | "malloc" : " 14 GB", | 20260505 05:46:27.780 | INFO | "pinned" : " 29 GB" | 20260505 05:46:27.780 | INFO | } | 20260505 05:46:27.781 | INFO | Perf report for the last 5.13969 seconds: | 20260505 05:46:27.781 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:46:27.781 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:27.781 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:27.781 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:27.781 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260505 05:46:27.781 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:27.781 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:27.781 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260505 05:46:27.781 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:27.781 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:27.781 | INFO | Uploaded 36GiB to GPU in 2.342s (15.3715GiB/s) | 20260505 05:46:27.781 | INFO | Downloaded 34GiB to GPU in 2.08671s (16.2936GiB/s) | 20260505 05:46:27.918 | INFO | Starting a round of Baz IO | 20260505 05:46:31.265 | INFO | Populating RTMetrics report | 20260505 05:46:31.265 | INFO | RTMetrics report populated | 20260505 05:46:31.265 | INFO | Opening tmp RTMetrics json file | 20260505 05:46:31.278 | INFO | Serializing RTMetrics to json | 20260505 05:46:31.281 | INFO | Serialization done, swapping tmp file now... | 20260505 05:46:31.283 | INFO | RTMetrics done | 20260505 05:46:31.377 | INFO | Finished a round of Baz IO: Wrote 2.291 GiB in 3.458 seconds | 20260505 05:46:31.377 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.74% / 0.07% / 63.20% | 20260505 05:46:31.377 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.58434% / 0.000115236% / 5.5631e-05% | 20260505 05:46:31.377 | INFO | Baz IO overall: | 20260505 05:46:31.377 | INFO | BytesWritten : 392625752588 | 20260505 05:46:31.377 | INFO | NumEvents : 184863433392 | 20260505 05:46:32.920 | INFO | Analyzing chunk frames = [806912,807424) | 20260505 05:46:32.920 | INFO | MemInfo: { | 20260505 05:46:32.920 | INFO | "gpu" : " 17 GB", | 20260505 05:46:32.920 | INFO | "huge" : "", | 20260505 05:46:32.920 | INFO | "huge_shared" : " 27 GB", | 20260505 05:46:32.920 | INFO | "malloc" : " 14 GB", | 20260505 05:46:32.920 | INFO | "pinned" : " 29 GB" | 20260505 05:46:32.920 | INFO | } | 20260505 05:46:32.920 | INFO | Perf report for the last 5.13926 seconds: | 20260505 05:46:32.920 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:46:32.920 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:32.920 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:32.920 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:32.920 | INFO | ANALYSIS : 66.7%, 2.9, 102.4ms, 0 | 20260505 05:46:32.920 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:32.920 | INFO | RT_METRICS : 46.2%, 1.0, 24.3ms, 0 | 20260505 05:46:32.920 | INFO | PRE_HQ : 64.0%, 2.0, 64.9ms, 0 | 20260505 05:46:32.920 | INFO | BAZWRITER : 66.8%, 3.9, 134.6ms, 0 | 20260505 05:46:32.920 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 05:46:32.920 | INFO | Uploaded 36GiB to GPU in 2.20904s (16.2967GiB/s) | 20260505 05:46:32.920 | INFO | Downloaded 38GiB to GPU in 2.29531s (16.5555GiB/s) | 20260505 05:46:38.059 | INFO | Analyzing chunk frames = [807424,807936) | 20260505 05:46:38.059 | INFO | MemInfo: { | 20260505 05:46:38.059 | INFO | "gpu" : " 17 GB", | 20260505 05:46:38.059 | INFO | "huge" : "", | 20260505 05:46:38.059 | INFO | "huge_shared" : " 27 GB", | 20260505 05:46:38.059 | INFO | "malloc" : " 14 GB", | 20260505 05:46:38.059 | INFO | "pinned" : " 29 GB" | 20260505 05:46:38.059 | INFO | } | 20260505 05:46:38.059 | INFO | Perf report for the last 5.13915 seconds: | 20260505 05:46:38.059 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:46:38.059 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:38.059 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:38.059 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:38.059 | INFO | ANALYSIS : 66.6%, 3.0, 104.0ms, 0 | 20260505 05:46:38.059 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:38.059 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:38.059 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260505 05:46:38.059 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:38.059 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:38.060 | INFO | Uploaded 36GiB to GPU in 2.29108s (15.7131GiB/s) | 20260505 05:46:38.060 | INFO | Downloaded 34GiB to GPU in 2.18686s (15.5474GiB/s) | 20260505 05:46:43.198 | INFO | Analyzing chunk frames = [807936,808448) | 20260505 05:46:43.198 | INFO | MemInfo: { | 20260505 05:46:43.198 | INFO | "gpu" : " 17 GB", | 20260505 05:46:43.198 | INFO | "huge" : "", | 20260505 05:46:43.198 | INFO | "huge_shared" : " 27 GB", | 20260505 05:46:43.198 | INFO | "malloc" : " 14 GB", | 20260505 05:46:43.198 | INFO | "pinned" : " 29 GB" | 20260505 05:46:43.198 | INFO | } | 20260505 05:46:43.198 | INFO | Perf report for the last 5.13929 seconds: | 20260505 05:46:43.199 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:46:43.199 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:43.199 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:43.199 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:43.199 | INFO | ANALYSIS : 65.3%, 3.0, 101.6ms, 0 | 20260505 05:46:43.199 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:43.200 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:43.200 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 05:46:43.200 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:43.200 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:43.200 | INFO | Uploaded 36GiB to GPU in 2.28377s (15.7634GiB/s) | 20260505 05:46:43.200 | INFO | Downloaded 34GiB to GPU in 2.13159s (15.9505GiB/s) | 20260505 05:46:48.339 | INFO | Analyzing chunk frames = [808448,808960) | 20260505 05:46:48.339 | INFO | MemInfo: { | 20260505 05:46:48.339 | INFO | "gpu" : " 17 GB", | 20260505 05:46:48.339 | INFO | "huge" : "", | 20260505 05:46:48.339 | INFO | "huge_shared" : " 27 GB", | 20260505 05:46:48.339 | INFO | "malloc" : " 14 GB", | 20260505 05:46:48.339 | INFO | "pinned" : " 29 GB" | 20260505 05:46:48.339 | INFO | } | 20260505 05:46:48.339 | INFO | Perf report for the last 5.14054 seconds: | 20260505 05:46:48.339 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:46:48.339 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:48.339 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:48.339 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:48.339 | INFO | ANALYSIS : 65.6%, 3.0, 102.0ms, 0 | 20260505 05:46:48.339 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:48.339 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:48.339 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 05:46:48.339 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:48.339 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:48.339 | INFO | Uploaded 36GiB to GPU in 2.28044s (15.7864GiB/s) | 20260505 05:46:48.339 | INFO | Downloaded 34GiB to GPU in 2.17272s (15.6486GiB/s) | 20260505 05:46:53.480 | INFO | Analyzing chunk frames = [808960,809472) | 20260505 05:46:53.480 | INFO | MemInfo: { | 20260505 05:46:53.480 | INFO | "gpu" : " 17 GB", | 20260505 05:46:53.480 | INFO | "huge" : "", | 20260505 05:46:53.480 | INFO | "huge_shared" : " 27 GB", | 20260505 05:46:53.480 | INFO | "malloc" : " 14 GB", | 20260505 05:46:53.480 | INFO | "pinned" : " 29 GB" | 20260505 05:46:53.480 | INFO | } | 20260505 05:46:53.480 | INFO | Perf report for the last 5.14134 seconds: | 20260505 05:46:53.480 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:46:53.480 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:53.481 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:53.481 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:53.481 | INFO | ANALYSIS : 65.3%, 3.0, 102.0ms, 0 | 20260505 05:46:53.481 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:53.481 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:53.481 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 05:46:53.481 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:53.481 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:53.481 | INFO | Uploaded 36GiB to GPU in 2.24712s (16.0205GiB/s) | 20260505 05:46:53.481 | INFO | Downloaded 34GiB to GPU in 2.22117s (15.3072GiB/s) | 20260505 05:46:58.616 | INFO | Analyzing chunk frames = [809472,809984) | 20260505 05:46:58.616 | INFO | MemInfo: { | 20260505 05:46:58.616 | INFO | "gpu" : " 17 GB", | 20260505 05:46:58.616 | INFO | "huge" : "", | 20260505 05:46:58.616 | INFO | "huge_shared" : " 27 GB", | 20260505 05:46:58.616 | INFO | "malloc" : " 14 GB", | 20260505 05:46:58.616 | INFO | "pinned" : " 29 GB" | 20260505 05:46:58.616 | INFO | } | 20260505 05:46:58.616 | INFO | Perf report for the last 5.1361 seconds: | 20260505 05:46:58.617 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:46:58.617 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:58.617 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:58.617 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:58.617 | INFO | ANALYSIS : 65.5%, 3.0, 102.3ms, 0 | 20260505 05:46:58.617 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:58.617 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:58.617 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260505 05:46:58.617 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:58.617 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:46:58.617 | INFO | Uploaded 36GiB to GPU in 2.21729s (16.236GiB/s) | 20260505 05:46:58.617 | INFO | Downloaded 34GiB to GPU in 2.20976s (15.3863GiB/s) | 20260505 05:47:03.756 | INFO | Analyzing chunk frames = [809984,810496) | 20260505 05:47:03.756 | INFO | MemInfo: { | 20260505 05:47:03.756 | INFO | "gpu" : " 17 GB", | 20260505 05:47:03.756 | INFO | "huge" : "", | 20260505 05:47:03.756 | INFO | "huge_shared" : " 27 GB", | 20260505 05:47:03.756 | INFO | "malloc" : " 14 GB", | 20260505 05:47:03.756 | INFO | "pinned" : " 29 GB" | 20260505 05:47:03.756 | INFO | } | 20260505 05:47:03.756 | INFO | Perf report for the last 5.13947 seconds: | 20260505 05:47:03.756 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:47:03.756 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:03.756 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:03.756 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:03.756 | INFO | ANALYSIS : 65.6%, 3.0, 102.4ms, 0 | 20260505 05:47:03.756 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:03.756 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:03.756 | INFO | PRE_HQ : 32.4%, 1.0, 16.9ms, 0 | 20260505 05:47:03.756 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:03.756 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:03.756 | INFO | Uploaded 36GiB to GPU in 2.14752s (16.7635GiB/s) | 20260505 05:47:03.756 | INFO | Downloaded 34GiB to GPU in 2.23135s (15.2374GiB/s) | 20260505 05:47:08.895 | INFO | Analyzing chunk frames = [810496,811008) | 20260505 05:47:08.895 | INFO | MemInfo: { | 20260505 05:47:08.895 | INFO | "gpu" : " 17 GB", | 20260505 05:47:08.895 | INFO | "huge" : "", | 20260505 05:47:08.895 | INFO | "huge_shared" : " 27 GB", | 20260505 05:47:08.895 | INFO | "malloc" : " 14 GB", | 20260505 05:47:08.895 | INFO | "pinned" : " 29 GB" | 20260505 05:47:08.895 | INFO | } | 20260505 05:47:08.895 | INFO | Perf report for the last 5.13922 seconds: | 20260505 05:47:08.895 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:47:08.895 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:08.895 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:08.895 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:08.895 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260505 05:47:08.895 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:08.895 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:08.895 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 05:47:08.895 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:08.895 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:08.895 | INFO | Uploaded 36GiB to GPU in 2.21011s (16.2888GiB/s) | 20260505 05:47:08.895 | INFO | Downloaded 34GiB to GPU in 2.14241s (15.87GiB/s) | 20260505 05:47:08.896 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 05:47:08.896 | INFO | Total mem usage: 15.3 GiB | 20260505 05:47:08.896 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 05:47:08.896 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -5.95%) | 20260505 05:47:08.896 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 05:47:08.896 | INFO | High Water Breakdown: | 20260505 05:47:08.896 | INFO | 60.1% : 8.7 GiB ( 93% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 05:47:08.896 | INFO | 39.7% : 5.75 GiB ( 86.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 05:47:08.896 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 05:47:08.896 | INFO | ------------------------------------------ | 20260505 05:47:08.896 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:47:08.896 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 05:47:08.896 | INFO | Total mem usage: 26.5 GiB | 20260505 05:47:08.896 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 05:47:08.896 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 05:47:08.896 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 05:47:08.896 | INFO | High Water Breakdown: | 20260505 05:47:08.896 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:47:08.896 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:47:08.896 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:47:08.896 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:47:08.896 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:47:08.896 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:47:08.896 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 05:47:08.896 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 05:47:08.896 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:47:08.896 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 05:47:08.896 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 05:47:08.896 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 05:47:08.896 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 05:47:08.896 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 05:47:08.896 | INFO | ------------------------------------------ | 20260505 05:47:08.896 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:47:08.896 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 05:47:08.896 | INFO | Total mem usage: 25.6 GiB | 20260505 05:47:08.896 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 05:47:08.896 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 05:47:08.896 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 05:47:08.896 | INFO | High Water Breakdown: | 20260505 05:47:08.896 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 05:47:08.896 | INFO | ------------------------------------------ | 20260505 05:47:08.896 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:47:08.896 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 05:47:08.896 | INFO | Total mem usage: 15.7 GiB | 20260505 05:47:08.896 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 05:47:08.896 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 05:47:08.896 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 05:47:08.896 | INFO | High Water Breakdown: | 20260505 05:47:08.896 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 05:47:08.896 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 05:47:08.896 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 05:47:08.896 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 05:47:08.896 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 05:47:08.896 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 05:47:08.896 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 05:47:08.896 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 05:47:08.896 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 05:47:08.896 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:47:08.896 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 05:47:08.896 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:47:08.896 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 05:47:08.896 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 05:47:08.896 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 05:47:08.896 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 05:47:08.896 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:47:08.896 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 05:47:08.896 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:47:08.896 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:47:08.896 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:47:08.896 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 05:47:08.896 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 05:47:08.896 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:47:08.896 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 05:47:08.896 | INFO | ------------------------------------------ | 20260505 05:47:08.896 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:47:08.897 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 05:47:08.897 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:47:08.897 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:47:08.897 | INFO | PerfReport: Full Execution -- 3.25e+01 / 7.74e+01, 1.41e+01 / 5.22e+00 | 20260505 05:47:08.897 | INFO | PerfReport: 0.28%, AggregateStats -- 9.18e-02 / 9.68e-02, 9.61e-03 / 5.35e-03 | 20260505 05:47:08.897 | INFO | PerfReport: 11.33%, Baseline -- 3.68e+00 / 4.24e+00, 5.42e-01 / 3.46e-01 | 20260505 05:47:08.897 | INFO | PerfReport: 3.21%, Histogram -- 1.04e+00 / 1.42e+00, 1.90e-01 / 1.39e-01 | 20260505 05:47:08.897 | INFO | PerfReport: 16.02%, DME -- 4.16e+01 / 4.68e+01, 5.60e+00 / 4.64e+00 | 20260505 05:47:08.897 | INFO | PerfReport: 55.67%, FrameLabeling -- 1.81e+01 / 1.96e+01, 1.70e+00 / 1.89e+00 | 20260505 05:47:08.897 | INFO | PerfReport: 7.80%, PulseAccumulating -- 2.53e+00 / 2.98e+00, 4.54e-01 / 3.27e-01 | 20260505 05:47:08.897 | INFO | PerfReport: 4.14%, Metrics -- 1.35e+00 / 1.67e+00, 3.61e-01 / 2.31e-01 | 20260505 05:47:08.897 | INFO | PerfReport: 1.54%, PulseSerializing -- 5.02e-01 / 5.64e-01, 9.59e-02 / 5.38e-02 | 20260505 05:47:08.897 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 05:47:08.897 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:47:08.897 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:47:08.897 | INFO | PerfReport: Full Execution -- 1.03e+02 / -nan, 2.61e+01 / 0.00e+00 | 20260505 05:47:08.897 | INFO | PerfReport: 22.51%, Upload -- 2.33e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260505 05:47:08.897 | INFO | PerfReport: 21.13%, Download -- 2.18e+01 / 0.00e+00, 7.48e+00 / 0.00e+00 | 20260505 05:47:08.897 | INFO | PerfReport: 31.45%, Compute -- 3.25e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260505 05:47:08.897 | INFO | PerfReport: 3.45%, LockUpload -- 3.56e+00 / 0.00e+00, 8.73e+00 / 0.00e+00 | 20260505 05:47:08.897 | INFO | PerfReport: 1.16%, LockDownload -- 1.19e+00 / 0.00e+00, 2.99e+00 / 0.00e+00 | 20260505 05:47:08.897 | INFO | PerfReport: 20.29%, LockCompute -- 2.10e+01 / 0.00e+00, 2.01e+01 / 0.00e+00 | 20260505 05:47:08.897 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 2.07e-02 / 0.00e+00 | 20260505 05:47:09.014 | INFO | Starting a round of Baz IO | 20260505 05:47:12.365 | INFO | Populating RTMetrics report | 20260505 05:47:12.366 | INFO | RTMetrics report populated | 20260505 05:47:12.366 | INFO | Opening tmp RTMetrics json file | 20260505 05:47:12.382 | INFO | Serializing RTMetrics to json | 20260505 05:47:12.383 | INFO | Serialization done, swapping tmp file now... | 20260505 05:47:12.383 | INFO | RTMetrics done | 20260505 05:47:12.477 | INFO | Finished a round of Baz IO: Wrote 2.293 GiB in 3.463 seconds | 20260505 05:47:12.477 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.67% / 0.07% / 63.26% | 20260505 05:47:12.477 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.55765% / 0.000103315% / 5.5631e-05% | 20260505 05:47:12.477 | INFO | Baz IO overall: | 20260505 05:47:12.477 | INFO | BytesWritten : 395087434898 | 20260505 05:47:12.477 | INFO | NumEvents : 186038602722 | 20260505 05:47:14.035 | INFO | Analyzing chunk frames = [811008,811520) | 20260505 05:47:14.035 | INFO | MemInfo: { | 20260505 05:47:14.035 | INFO | "gpu" : " 17 GB", | 20260505 05:47:14.035 | INFO | "huge" : "", | 20260505 05:47:14.035 | INFO | "huge_shared" : " 27 GB", | 20260505 05:47:14.035 | INFO | "malloc" : " 14 GB", | 20260505 05:47:14.035 | INFO | "pinned" : " 29 GB" | 20260505 05:47:14.035 | INFO | } | 20260505 05:47:14.035 | INFO | Perf report for the last 5.13968 seconds: | 20260505 05:47:14.035 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:47:14.035 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:14.035 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:14.035 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:14.035 | INFO | ANALYSIS : 66.4%, 2.9, 102.1ms, 0 | 20260505 05:47:14.035 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:14.035 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260505 05:47:14.035 | INFO | PRE_HQ : 64.9%, 1.9, 64.5ms, 0 | 20260505 05:47:14.035 | INFO | BAZWRITER : 66.7%, 3.9, 134.3ms, 0 | 20260505 05:47:14.035 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 05:47:14.035 | INFO | Uploaded 36GiB to GPU in 2.22109s (16.2083GiB/s) | 20260505 05:47:14.035 | INFO | Downloaded 38GiB to GPU in 2.28741s (16.6127GiB/s) | 20260505 05:47:19.174 | INFO | Analyzing chunk frames = [811520,812032) | 20260505 05:47:19.174 | INFO | MemInfo: { | 20260505 05:47:19.174 | INFO | "gpu" : " 17 GB", | 20260505 05:47:19.174 | INFO | "huge" : "", | 20260505 05:47:19.174 | INFO | "huge_shared" : " 27 GB", | 20260505 05:47:19.174 | INFO | "malloc" : " 14 GB", | 20260505 05:47:19.174 | INFO | "pinned" : " 29 GB" | 20260505 05:47:19.174 | INFO | } | 20260505 05:47:19.174 | INFO | Perf report for the last 5.13892 seconds: | 20260505 05:47:19.174 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:47:19.174 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:19.174 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:19.174 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:19.174 | INFO | ANALYSIS : 65.5%, 3.0, 102.3ms, 0 | 20260505 05:47:19.174 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:19.174 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:19.174 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 05:47:19.174 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:19.174 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:19.174 | INFO | Uploaded 36GiB to GPU in 2.22875s (16.1525GiB/s) | 20260505 05:47:19.174 | INFO | Downloaded 34GiB to GPU in 2.08606s (16.2986GiB/s) | 20260505 05:47:24.313 | INFO | Analyzing chunk frames = [812032,812544) | 20260505 05:47:24.313 | INFO | MemInfo: { | 20260505 05:47:24.313 | INFO | "gpu" : " 17 GB", | 20260505 05:47:24.313 | INFO | "huge" : "", | 20260505 05:47:24.313 | INFO | "huge_shared" : " 27 GB", | 20260505 05:47:24.313 | INFO | "malloc" : " 14 GB", | 20260505 05:47:24.313 | INFO | "pinned" : " 29 GB" | 20260505 05:47:24.313 | INFO | } | 20260505 05:47:24.313 | INFO | Perf report for the last 5.13929 seconds: | 20260505 05:47:24.313 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:47:24.313 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:24.313 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:24.313 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:24.313 | INFO | ANALYSIS : 63.6%, 3.0, 99.0ms, 0 | 20260505 05:47:24.313 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:24.313 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:24.313 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 05:47:24.313 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:24.313 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:24.314 | INFO | Uploaded 36GiB to GPU in 2.13821s (16.8365GiB/s) | 20260505 05:47:24.314 | INFO | Downloaded 34GiB to GPU in 2.04598s (16.6179GiB/s) | 20260505 05:47:29.453 | INFO | Analyzing chunk frames = [812544,813056) | 20260505 05:47:29.453 | INFO | MemInfo: { | 20260505 05:47:29.453 | INFO | "gpu" : " 17 GB", | 20260505 05:47:29.453 | INFO | "huge" : "", | 20260505 05:47:29.453 | INFO | "huge_shared" : " 27 GB", | 20260505 05:47:29.453 | INFO | "malloc" : " 14 GB", | 20260505 05:47:29.453 | INFO | "pinned" : " 29 GB" | 20260505 05:47:29.453 | INFO | } | 20260505 05:47:29.453 | INFO | Perf report for the last 5.1397 seconds: | 20260505 05:47:29.453 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:47:29.453 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:29.453 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:29.453 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:29.453 | INFO | ANALYSIS : 63.5%, 3.0, 98.6ms, 0 | 20260505 05:47:29.453 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:29.453 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:29.453 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260505 05:47:29.453 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:29.453 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:29.453 | INFO | Uploaded 36GiB to GPU in 2.13898s (16.8305GiB/s) | 20260505 05:47:29.453 | INFO | Downloaded 34GiB to GPU in 2.01996s (16.832GiB/s) | 20260505 05:47:34.592 | INFO | Analyzing chunk frames = [813056,813568) | 20260505 05:47:34.592 | INFO | MemInfo: { | 20260505 05:47:34.592 | INFO | "gpu" : " 17 GB", | 20260505 05:47:34.592 | INFO | "huge" : "", | 20260505 05:47:34.592 | INFO | "huge_shared" : " 27 GB", | 20260505 05:47:34.592 | INFO | "malloc" : " 14 GB", | 20260505 05:47:34.592 | INFO | "pinned" : " 29 GB" | 20260505 05:47:34.592 | INFO | } | 20260505 05:47:34.592 | INFO | Perf report for the last 5.13928 seconds: | 20260505 05:47:34.592 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:47:34.592 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:34.592 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:34.592 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:34.592 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 05:47:34.592 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:34.592 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:34.592 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 05:47:34.592 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:34.592 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:34.592 | INFO | Uploaded 36GiB to GPU in 2.16344s (16.6401GiB/s) | 20260505 05:47:34.592 | INFO | Downloaded 34GiB to GPU in 2.04972s (16.5876GiB/s) | 20260505 05:47:39.731 | INFO | Analyzing chunk frames = [813568,814080) | 20260505 05:47:39.731 | INFO | MemInfo: { | 20260505 05:47:39.731 | INFO | "gpu" : " 17 GB", | 20260505 05:47:39.731 | INFO | "huge" : "", | 20260505 05:47:39.731 | INFO | "huge_shared" : " 27 GB", | 20260505 05:47:39.731 | INFO | "malloc" : " 14 GB", | 20260505 05:47:39.731 | INFO | "pinned" : " 29 GB" | 20260505 05:47:39.731 | INFO | } | 20260505 05:47:39.731 | INFO | Perf report for the last 5.1392 seconds: | 20260505 05:47:39.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:47:39.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:39.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:39.731 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:39.731 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 05:47:39.731 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:39.732 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:39.732 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:47:39.732 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:39.732 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:39.732 | INFO | Uploaded 36GiB to GPU in 2.15825s (16.6802GiB/s) | 20260505 05:47:39.732 | INFO | Downloaded 34GiB to GPU in 2.0528s (16.5628GiB/s) | 20260505 05:47:44.870 | INFO | Analyzing chunk frames = [814080,814592) | 20260505 05:47:44.870 | INFO | MemInfo: { | 20260505 05:47:44.870 | INFO | "gpu" : " 17 GB", | 20260505 05:47:44.870 | INFO | "huge" : "", | 20260505 05:47:44.870 | INFO | "huge_shared" : " 27 GB", | 20260505 05:47:44.870 | INFO | "malloc" : " 14 GB", | 20260505 05:47:44.870 | INFO | "pinned" : " 29 GB" | 20260505 05:47:44.870 | INFO | } | 20260505 05:47:44.870 | INFO | Perf report for the last 5.1392 seconds: | 20260505 05:47:44.871 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:47:44.871 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:44.871 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:44.871 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:44.871 | INFO | ANALYSIS : 64.8%, 3.0, 101.3ms, 0 | 20260505 05:47:44.871 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:44.871 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:44.871 | INFO | PRE_HQ : 32.0%, 1.0, 16.7ms, 0 | 20260505 05:47:44.871 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:44.871 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:44.871 | INFO | Uploaded 36GiB to GPU in 2.20612s (16.3182GiB/s) | 20260505 05:47:44.871 | INFO | Downloaded 34GiB to GPU in 2.13659s (15.9132GiB/s) | 20260505 05:47:50.010 | INFO | Analyzing chunk frames = [814592,815104) | 20260505 05:47:50.010 | INFO | MemInfo: { | 20260505 05:47:50.010 | INFO | "gpu" : " 17 GB", | 20260505 05:47:50.010 | INFO | "huge" : "", | 20260505 05:47:50.010 | INFO | "huge_shared" : " 27 GB", | 20260505 05:47:50.010 | INFO | "malloc" : " 14 GB", | 20260505 05:47:50.010 | INFO | "pinned" : " 29 GB" | 20260505 05:47:50.010 | INFO | } | 20260505 05:47:50.010 | INFO | Perf report for the last 5.13959 seconds: | 20260505 05:47:50.010 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:47:50.010 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:50.010 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:50.010 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:50.010 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 05:47:50.010 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:50.010 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:50.010 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 05:47:50.010 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:50.010 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:50.011 | INFO | Uploaded 36GiB to GPU in 2.15569s (16.7GiB/s) | 20260505 05:47:50.011 | INFO | Downloaded 34GiB to GPU in 2.04321s (16.6404GiB/s) | 20260505 05:47:50.130 | INFO | Starting a round of Baz IO | 20260505 05:47:53.412 | INFO | Populating RTMetrics report | 20260505 05:47:53.412 | INFO | RTMetrics report populated | 20260505 05:47:53.412 | INFO | Opening tmp RTMetrics json file | 20260505 05:47:53.423 | INFO | Serializing RTMetrics to json | 20260505 05:47:53.425 | INFO | Serialization done, swapping tmp file now... | 20260505 05:47:53.425 | INFO | RTMetrics done | 20260505 05:47:53.521 | INFO | Finished a round of Baz IO: Wrote 2.295 GiB in 3.391 seconds | 20260505 05:47:53.521 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.60% / 0.07% / 63.33% | 20260505 05:47:53.521 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.53515% / 0.000119209% / 8.74201e-05% | 20260505 05:47:53.521 | INFO | Baz IO overall: | 20260505 05:47:53.521 | INFO | BytesWritten : 397551979693 | 20260505 05:47:53.521 | INFO | NumEvents : 187215671610 | 20260505 05:47:55.149 | INFO | Analyzing chunk frames = [815104,815616) | 20260505 05:47:55.149 | INFO | MemInfo: { | 20260505 05:47:55.149 | INFO | "gpu" : " 17 GB", | 20260505 05:47:55.149 | INFO | "huge" : "", | 20260505 05:47:55.149 | INFO | "huge_shared" : " 27 GB", | 20260505 05:47:55.149 | INFO | "malloc" : " 14 GB", | 20260505 05:47:55.149 | INFO | "pinned" : " 29 GB" | 20260505 05:47:55.149 | INFO | } | 20260505 05:47:55.150 | INFO | Perf report for the last 5.13941 seconds: | 20260505 05:47:55.150 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:47:55.150 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:55.150 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:55.150 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:55.150 | INFO | ANALYSIS : 65.0%, 2.9, 100.1ms, 0 | 20260505 05:47:55.150 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:47:55.150 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260505 05:47:55.150 | INFO | PRE_HQ : 64.0%, 2.0, 65.0ms, 0 | 20260505 05:47:55.150 | INFO | BAZWRITER : 65.3%, 4.0, 132.6ms, 0 | 20260505 05:47:55.150 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 05:47:55.150 | INFO | Uploaded 36GiB to GPU in 2.19517s (16.3996GiB/s) | 20260505 05:47:55.150 | INFO | Downloaded 38GiB to GPU in 2.29876s (16.5307GiB/s) | 20260505 05:48:00.289 | INFO | Analyzing chunk frames = [815616,816128) | 20260505 05:48:00.289 | INFO | MemInfo: { | 20260505 05:48:00.289 | INFO | "gpu" : " 17 GB", | 20260505 05:48:00.289 | INFO | "huge" : "", | 20260505 05:48:00.289 | INFO | "huge_shared" : " 27 GB", | 20260505 05:48:00.289 | INFO | "malloc" : " 14 GB", | 20260505 05:48:00.289 | INFO | "pinned" : " 29 GB" | 20260505 05:48:00.289 | INFO | } | 20260505 05:48:00.289 | INFO | Perf report for the last 5.13907 seconds: | 20260505 05:48:00.292 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:48:00.292 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:00.292 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:00.292 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:00.292 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 05:48:00.292 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:00.292 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:00.292 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 05:48:00.292 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:00.292 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:00.292 | INFO | Uploaded 36GiB to GPU in 2.10634s (17.0913GiB/s) | 20260505 05:48:00.292 | INFO | Downloaded 34GiB to GPU in 2.03165s (16.7352GiB/s) | 20260505 05:48:05.428 | INFO | Analyzing chunk frames = [816128,816640) | 20260505 05:48:05.428 | INFO | MemInfo: { | 20260505 05:48:05.428 | INFO | "gpu" : " 17 GB", | 20260505 05:48:05.428 | INFO | "huge" : "", | 20260505 05:48:05.428 | INFO | "huge_shared" : " 27 GB", | 20260505 05:48:05.428 | INFO | "malloc" : " 14 GB", | 20260505 05:48:05.428 | INFO | "pinned" : " 29 GB" | 20260505 05:48:05.428 | INFO | } | 20260505 05:48:05.428 | INFO | Perf report for the last 5.13913 seconds: | 20260505 05:48:05.428 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:48:05.428 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:05.428 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:05.428 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:05.428 | INFO | ANALYSIS : 63.6%, 3.0, 99.0ms, 0 | 20260505 05:48:05.428 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:05.428 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:05.428 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260505 05:48:05.428 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:05.428 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:05.428 | INFO | Uploaded 36GiB to GPU in 2.12071s (16.9755GiB/s) | 20260505 05:48:05.428 | INFO | Downloaded 34GiB to GPU in 2.04591s (16.6185GiB/s) | 20260505 05:48:10.567 | INFO | Analyzing chunk frames = [816640,817152) | 20260505 05:48:10.567 | INFO | MemInfo: { | 20260505 05:48:10.567 | INFO | "gpu" : " 17 GB", | 20260505 05:48:10.567 | INFO | "huge" : "", | 20260505 05:48:10.567 | INFO | "huge_shared" : " 27 GB", | 20260505 05:48:10.567 | INFO | "malloc" : " 14 GB", | 20260505 05:48:10.567 | INFO | "pinned" : " 29 GB" | 20260505 05:48:10.567 | INFO | } | 20260505 05:48:10.567 | INFO | Perf report for the last 5.1394 seconds: | 20260505 05:48:10.567 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:48:10.567 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:10.567 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:10.567 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:10.567 | INFO | ANALYSIS : 63.6%, 3.0, 98.8ms, 0 | 20260505 05:48:10.567 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:10.567 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:10.567 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260505 05:48:10.567 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:10.567 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:10.568 | INFO | Uploaded 36GiB to GPU in 2.14594s (16.7758GiB/s) | 20260505 05:48:10.568 | INFO | Downloaded 34GiB to GPU in 2.02893s (16.7576GiB/s) | 20260505 05:48:15.706 | INFO | Analyzing chunk frames = [817152,817664) | 20260505 05:48:15.706 | INFO | MemInfo: { | 20260505 05:48:15.706 | INFO | "gpu" : " 17 GB", | 20260505 05:48:15.706 | INFO | "huge" : "", | 20260505 05:48:15.706 | INFO | "huge_shared" : " 27 GB", | 20260505 05:48:15.706 | INFO | "malloc" : " 14 GB", | 20260505 05:48:15.706 | INFO | "pinned" : " 29 GB" | 20260505 05:48:15.706 | INFO | } | 20260505 05:48:15.706 | INFO | Perf report for the last 5.13927 seconds: | 20260505 05:48:15.710 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:48:15.710 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:15.710 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:15.710 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:15.710 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 05:48:15.710 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:15.710 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:15.710 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 05:48:15.710 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:15.710 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:15.710 | INFO | Uploaded 36GiB to GPU in 2.11579s (17.0149GiB/s) | 20260505 05:48:15.710 | INFO | Downloaded 34GiB to GPU in 2.02744s (16.7699GiB/s) | 20260505 05:48:20.846 | INFO | Analyzing chunk frames = [817664,818176) | 20260505 05:48:20.846 | INFO | MemInfo: { | 20260505 05:48:20.846 | INFO | "gpu" : " 17 GB", | 20260505 05:48:20.846 | INFO | "huge" : "", | 20260505 05:48:20.846 | INFO | "huge_shared" : " 27 GB", | 20260505 05:48:20.846 | INFO | "malloc" : " 14 GB", | 20260505 05:48:20.846 | INFO | "pinned" : " 29 GB" | 20260505 05:48:20.846 | INFO | } | 20260505 05:48:20.846 | INFO | Perf report for the last 5.1394 seconds: | 20260505 05:48:20.846 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:48:20.846 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:20.846 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:20.846 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:20.846 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 05:48:20.846 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:20.846 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:20.846 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 05:48:20.846 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:20.846 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:20.846 | INFO | Uploaded 36GiB to GPU in 2.16137s (16.6561GiB/s) | 20260505 05:48:20.846 | INFO | Downloaded 34GiB to GPU in 2.05418s (16.5517GiB/s) | 20260505 05:48:25.985 | INFO | Analyzing chunk frames = [818176,818688) | 20260505 05:48:25.985 | INFO | MemInfo: { | 20260505 05:48:25.985 | INFO | "gpu" : " 17 GB", | 20260505 05:48:25.985 | INFO | "huge" : "", | 20260505 05:48:25.985 | INFO | "huge_shared" : " 27 GB", | 20260505 05:48:25.985 | INFO | "malloc" : " 14 GB", | 20260505 05:48:25.985 | INFO | "pinned" : " 29 GB" | 20260505 05:48:25.985 | INFO | } | 20260505 05:48:25.985 | INFO | Perf report for the last 5.13961 seconds: | 20260505 05:48:25.985 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:48:25.985 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:25.985 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:25.985 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:25.986 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 05:48:25.986 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:25.986 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:25.986 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260505 05:48:25.986 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:25.986 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:25.986 | INFO | Uploaded 36GiB to GPU in 2.16345s (16.6401GiB/s) | 20260505 05:48:25.986 | INFO | Downloaded 34GiB to GPU in 2.1451s (15.8501GiB/s) | 20260505 05:48:31.125 | INFO | Analyzing chunk frames = [818688,819200) | 20260505 05:48:31.125 | INFO | MemInfo: { | 20260505 05:48:31.125 | INFO | "gpu" : " 17 GB", | 20260505 05:48:31.125 | INFO | "huge" : "", | 20260505 05:48:31.125 | INFO | "huge_shared" : " 27 GB", | 20260505 05:48:31.125 | INFO | "malloc" : " 14 GB", | 20260505 05:48:31.125 | INFO | "pinned" : " 29 GB" | 20260505 05:48:31.125 | INFO | } | 20260505 05:48:31.125 | INFO | Perf report for the last 5.13941 seconds: | 20260505 05:48:31.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:48:31.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:31.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:31.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:31.125 | INFO | ANALYSIS : 64.4%, 3.0, 100.6ms, 0 | 20260505 05:48:31.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:31.125 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:31.125 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 05:48:31.125 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:31.125 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:31.125 | INFO | Uploaded 36GiB to GPU in 2.15768s (16.6846GiB/s) | 20260505 05:48:31.125 | INFO | Downloaded 34GiB to GPU in 2.04386s (16.6352GiB/s) | 20260505 05:48:31.125 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 05:48:31.125 | INFO | Total mem usage: 15.3 GiB | 20260505 05:48:31.125 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 05:48:31.125 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.3%) | 20260505 05:48:31.125 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 05:48:31.125 | INFO | High Water Breakdown: | 20260505 05:48:31.125 | INFO | 60% : 8.66 GiB ( 92.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 05:48:31.125 | INFO | 39.8% : 5.74 GiB ( 86.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 05:48:31.125 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 05:48:31.125 | INFO | ------------------------------------------ | 20260505 05:48:31.125 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:48:31.125 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 05:48:31.125 | INFO | Total mem usage: 26.5 GiB | 20260505 05:48:31.125 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 05:48:31.125 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 05:48:31.125 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 05:48:31.125 | INFO | High Water Breakdown: | 20260505 05:48:31.126 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:48:31.126 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:48:31.126 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:48:31.126 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:48:31.126 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:48:31.126 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:48:31.126 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 05:48:31.126 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 05:48:31.126 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:48:31.126 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 05:48:31.126 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 05:48:31.126 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 05:48:31.126 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 05:48:31.126 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 05:48:31.126 | INFO | ------------------------------------------ | 20260505 05:48:31.126 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:48:31.126 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 05:48:31.126 | INFO | Total mem usage: 25.6 GiB | 20260505 05:48:31.126 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 05:48:31.126 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 05:48:31.126 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 05:48:31.126 | INFO | High Water Breakdown: | 20260505 05:48:31.126 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 05:48:31.126 | INFO | ------------------------------------------ | 20260505 05:48:31.126 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:48:31.126 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 05:48:31.126 | INFO | Total mem usage: 15.7 GiB | 20260505 05:48:31.126 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 05:48:31.126 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 05:48:31.126 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260505 05:48:31.126 | INFO | High Water Breakdown: | 20260505 05:48:31.126 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 05:48:31.126 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 05:48:31.126 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 05:48:31.126 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 05:48:31.126 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 05:48:31.126 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 05:48:31.126 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 05:48:31.126 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 05:48:31.126 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 05:48:31.126 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:48:31.126 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 05:48:31.126 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:48:31.126 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 05:48:31.126 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 05:48:31.126 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 05:48:31.126 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 05:48:31.126 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:48:31.126 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 05:48:31.126 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:48:31.126 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:48:31.126 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:48:31.126 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 05:48:31.126 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 05:48:31.126 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:48:31.126 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 05:48:31.126 | INFO | ------------------------------------------ | 20260505 05:48:31.126 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:48:31.126 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 05:48:31.126 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:48:31.126 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:48:31.126 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.49e+01, 1.37e+01 / 1.88e+00 | 20260505 05:48:31.126 | INFO | PerfReport: 0.28%, AggregateStats -- 9.39e-02 / 1.04e-01, 1.09e-02 / 7.21e-03 | 20260505 05:48:31.126 | INFO | PerfReport: 11.57%, Baseline -- 3.83e+00 / 4.47e+00, 6.23e-01 / 2.54e-01 | 20260505 05:48:31.126 | INFO | PerfReport: 3.30%, Histogram -- 1.09e+00 / 1.51e+00, 2.14e-01 / 1.06e-01 | 20260505 05:48:31.126 | INFO | PerfReport: 15.48%, DME -- 4.10e+01 / 4.45e+01, 4.08e+00 / 1.41e+00 | 20260505 05:48:31.126 | INFO | PerfReport: 55.79%, FrameLabeling -- 1.85e+01 / 1.87e+01, 1.98e+00 / 1.02e+00 | 20260505 05:48:31.126 | INFO | PerfReport: 7.82%, PulseAccumulating -- 2.59e+00 / 3.18e+00, 4.69e-01 / 3.24e-01 | 20260505 05:48:31.126 | INFO | PerfReport: 4.21%, Metrics -- 1.39e+00 / 1.77e+00, 3.86e-01 / 2.16e-01 | 20260505 05:48:31.126 | INFO | PerfReport: 1.55%, PulseSerializing -- 5.12e-01 / 5.92e-01, 1.02e-01 / 4.65e-02 | 20260505 05:48:31.126 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 05:48:31.126 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:48:31.126 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:48:31.126 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.24e+01 / 0.00e+00 | 20260505 05:48:31.126 | INFO | PerfReport: 22.07%, Upload -- 2.25e+01 / 0.00e+00, 7.89e+00 / 0.00e+00 | 20260505 05:48:31.126 | INFO | PerfReport: 21.32%, Download -- 2.17e+01 / 0.00e+00, 7.74e+00 / 0.00e+00 | 20260505 05:48:31.126 | INFO | PerfReport: 32.43%, Compute -- 3.31e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 05:48:31.126 | INFO | PerfReport: 1.58%, LockUpload -- 1.61e+00 / 0.00e+00, 7.36e+00 / 0.00e+00 | 20260505 05:48:31.126 | INFO | PerfReport: 0.83%, LockDownload -- 8.50e-01 / 0.00e+00, 2.24e+00 / 0.00e+00 | 20260505 05:48:31.126 | INFO | PerfReport: 21.75%, LockCompute -- 2.22e+01 / 0.00e+00, 1.74e+01 / 0.00e+00 | 20260505 05:48:31.126 | INFO | PerfReport: 0.01%, Misc -- 1.47e-02 / 0.00e+00, 1.95e-02 / 0.00e+00 | 20260505 05:48:31.247 | INFO | Starting a round of Baz IO | 20260505 05:48:34.493 | INFO | Populating RTMetrics report | 20260505 05:48:34.493 | INFO | RTMetrics report populated | 20260505 05:48:34.493 | INFO | Opening tmp RTMetrics json file | 20260505 05:48:34.515 | INFO | Serializing RTMetrics to json | 20260505 05:48:34.516 | INFO | Serialization done, swapping tmp file now... | 20260505 05:48:34.516 | INFO | RTMetrics done | 20260505 05:48:34.603 | INFO | Finished a round of Baz IO: Wrote 2.299 GiB in 3.355 seconds | 20260505 05:48:34.603 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.53% / 0.07% / 63.40% | 20260505 05:48:34.603 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.52665% / 9.93411e-05% / 2.38419e-05% | 20260505 05:48:34.603 | INFO | Baz IO overall: | 20260505 05:48:34.603 | INFO | BytesWritten : 400020356987 | 20260505 05:48:34.603 | INFO | NumEvents : 188395304223 | 20260505 05:48:36.264 | INFO | Analyzing chunk frames = [819200,819712) | 20260505 05:48:36.264 | INFO | MemInfo: { | 20260505 05:48:36.264 | INFO | "gpu" : " 17 GB", | 20260505 05:48:36.264 | INFO | "huge" : "", | 20260505 05:48:36.264 | INFO | "huge_shared" : " 27 GB", | 20260505 05:48:36.264 | INFO | "malloc" : " 14 GB", | 20260505 05:48:36.264 | INFO | "pinned" : " 29 GB" | 20260505 05:48:36.264 | INFO | } | 20260505 05:48:36.264 | INFO | Perf report for the last 5.13959 seconds: | 20260505 05:48:36.265 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:48:36.265 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:36.265 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:36.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:36.265 | INFO | ANALYSIS : 64.4%, 2.9, 99.1ms, 0 | 20260505 05:48:36.265 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:36.265 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260505 05:48:36.265 | INFO | PRE_HQ : 63.5%, 2.0, 64.8ms, 0 | 20260505 05:48:36.265 | INFO | BAZWRITER : 64.6%, 4.0, 132.2ms, 0 | 20260505 05:48:36.265 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 05:48:36.265 | INFO | Uploaded 36GiB to GPU in 2.09919s (17.1495GiB/s) | 20260505 05:48:36.265 | INFO | Downloaded 38GiB to GPU in 2.30995s (16.4506GiB/s) | 20260505 05:48:41.403 | INFO | Analyzing chunk frames = [819712,820224) | 20260505 05:48:41.403 | INFO | MemInfo: { | 20260505 05:48:41.403 | INFO | "gpu" : " 17 GB", | 20260505 05:48:41.403 | INFO | "huge" : "", | 20260505 05:48:41.403 | INFO | "huge_shared" : " 27 GB", | 20260505 05:48:41.403 | INFO | "malloc" : " 14 GB", | 20260505 05:48:41.403 | INFO | "pinned" : " 29 GB" | 20260505 05:48:41.403 | INFO | } | 20260505 05:48:41.403 | INFO | Perf report for the last 5.13875 seconds: | 20260505 05:48:41.403 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:48:41.403 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:41.403 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:41.403 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:41.403 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 05:48:41.403 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:41.404 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:41.404 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260505 05:48:41.404 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:41.404 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:41.404 | INFO | Uploaded 36GiB to GPU in 2.12328s (16.9549GiB/s) | 20260505 05:48:41.404 | INFO | Downloaded 34GiB to GPU in 2.04007s (16.6661GiB/s) | 20260505 05:48:46.542 | INFO | Analyzing chunk frames = [820224,820736) | 20260505 05:48:46.542 | INFO | MemInfo: { | 20260505 05:48:46.542 | INFO | "gpu" : " 17 GB", | 20260505 05:48:46.542 | INFO | "huge" : "", | 20260505 05:48:46.542 | INFO | "huge_shared" : " 27 GB", | 20260505 05:48:46.542 | INFO | "malloc" : " 14 GB", | 20260505 05:48:46.542 | INFO | "pinned" : " 29 GB" | 20260505 05:48:46.543 | INFO | } | 20260505 05:48:46.543 | INFO | Perf report for the last 5.13928 seconds: | 20260505 05:48:46.543 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:48:46.543 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:46.543 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:46.543 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:46.543 | INFO | ANALYSIS : 63.9%, 3.0, 99.5ms, 0 | 20260505 05:48:46.543 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:46.543 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:46.543 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260505 05:48:46.543 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:46.543 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:46.543 | INFO | Uploaded 36GiB to GPU in 2.14108s (16.8139GiB/s) | 20260505 05:48:46.543 | INFO | Downloaded 34GiB to GPU in 2.03745s (16.6875GiB/s) | 20260505 05:48:51.682 | INFO | Analyzing chunk frames = [820736,821248) | 20260505 05:48:51.682 | INFO | MemInfo: { | 20260505 05:48:51.682 | INFO | "gpu" : " 17 GB", | 20260505 05:48:51.682 | INFO | "huge" : "", | 20260505 05:48:51.682 | INFO | "huge_shared" : " 27 GB", | 20260505 05:48:51.682 | INFO | "malloc" : " 14 GB", | 20260505 05:48:51.682 | INFO | "pinned" : " 29 GB" | 20260505 05:48:51.682 | INFO | } | 20260505 05:48:51.682 | INFO | Perf report for the last 5.13967 seconds: | 20260505 05:48:51.682 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:48:51.682 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:51.682 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:51.682 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:51.682 | INFO | ANALYSIS : 63.9%, 3.0, 99.3ms, 0 | 20260505 05:48:51.682 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:51.682 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:51.682 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 05:48:51.682 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:51.682 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:51.683 | INFO | Uploaded 36GiB to GPU in 2.16858s (16.6007GiB/s) | 20260505 05:48:51.683 | INFO | Downloaded 34GiB to GPU in 2.03123s (16.7386GiB/s) | 20260505 05:48:56.821 | INFO | Analyzing chunk frames = [821248,821760) | 20260505 05:48:56.821 | INFO | MemInfo: { | 20260505 05:48:56.821 | INFO | "gpu" : " 17 GB", | 20260505 05:48:56.821 | INFO | "huge" : "", | 20260505 05:48:56.821 | INFO | "huge_shared" : " 27 GB", | 20260505 05:48:56.821 | INFO | "malloc" : " 14 GB", | 20260505 05:48:56.821 | INFO | "pinned" : " 29 GB" | 20260505 05:48:56.821 | INFO | } | 20260505 05:48:56.821 | INFO | Perf report for the last 5.1391 seconds: | 20260505 05:48:56.821 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:48:56.821 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:56.821 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:56.821 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:56.821 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 05:48:56.821 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:56.821 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:56.821 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 05:48:56.821 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:56.821 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:48:56.822 | INFO | Uploaded 36GiB to GPU in 2.1544s (16.71GiB/s) | 20260505 05:48:56.822 | INFO | Downloaded 34GiB to GPU in 2.0463s (16.6154GiB/s) | 20260505 05:49:01.961 | INFO | Analyzing chunk frames = [821760,822272) | 20260505 05:49:01.961 | INFO | MemInfo: { | 20260505 05:49:01.961 | INFO | "gpu" : " 17 GB", | 20260505 05:49:01.961 | INFO | "huge" : "", | 20260505 05:49:01.961 | INFO | "huge_shared" : " 27 GB", | 20260505 05:49:01.961 | INFO | "malloc" : " 14 GB", | 20260505 05:49:01.961 | INFO | "pinned" : " 29 GB" | 20260505 05:49:01.961 | INFO | } | 20260505 05:49:01.961 | INFO | Perf report for the last 5.13951 seconds: | 20260505 05:49:01.961 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:49:01.961 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:01.961 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:01.961 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:01.961 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 05:49:01.961 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:01.961 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:01.961 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:49:01.961 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:01.961 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:01.961 | INFO | Uploaded 36GiB to GPU in 2.14533s (16.7806GiB/s) | 20260505 05:49:01.961 | INFO | Downloaded 34GiB to GPU in 2.03916s (16.6735GiB/s) | 20260505 05:49:07.100 | INFO | Analyzing chunk frames = [822272,822784) | 20260505 05:49:07.100 | INFO | MemInfo: { | 20260505 05:49:07.100 | INFO | "gpu" : " 17 GB", | 20260505 05:49:07.100 | INFO | "huge" : "", | 20260505 05:49:07.100 | INFO | "huge_shared" : " 27 GB", | 20260505 05:49:07.100 | INFO | "malloc" : " 14 GB", | 20260505 05:49:07.100 | INFO | "pinned" : " 29 GB" | 20260505 05:49:07.101 | INFO | } | 20260505 05:49:07.101 | INFO | Perf report for the last 5.13979 seconds: | 20260505 05:49:07.101 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:49:07.101 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:07.101 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:07.101 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:07.101 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 05:49:07.101 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:07.101 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:07.101 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260505 05:49:07.101 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:07.101 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:07.101 | INFO | Uploaded 36GiB to GPU in 2.14863s (16.7548GiB/s) | 20260505 05:49:07.101 | INFO | Downloaded 34GiB to GPU in 2.1513s (15.8044GiB/s) | 20260505 05:49:12.239 | INFO | Analyzing chunk frames = [822784,823296) | 20260505 05:49:12.239 | INFO | MemInfo: { | 20260505 05:49:12.239 | INFO | "gpu" : " 17 GB", | 20260505 05:49:12.239 | INFO | "huge" : "", | 20260505 05:49:12.240 | INFO | "huge_shared" : " 27 GB", | 20260505 05:49:12.240 | INFO | "malloc" : " 14 GB", | 20260505 05:49:12.240 | INFO | "pinned" : " 29 GB" | 20260505 05:49:12.240 | INFO | } | 20260505 05:49:12.240 | INFO | Perf report for the last 5.13901 seconds: | 20260505 05:49:12.240 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:49:12.240 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:12.240 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:12.240 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:12.240 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 05:49:12.240 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:12.240 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:12.240 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 05:49:12.240 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:12.240 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:12.240 | INFO | Uploaded 36GiB to GPU in 2.10811s (17.0769GiB/s) | 20260505 05:49:12.240 | INFO | Downloaded 34GiB to GPU in 2.05827s (16.5187GiB/s) | 20260505 05:49:12.363 | INFO | Starting a round of Baz IO | 20260505 05:49:15.626 | INFO | Populating RTMetrics report | 20260505 05:49:15.627 | INFO | RTMetrics report populated | 20260505 05:49:15.627 | INFO | Opening tmp RTMetrics json file | 20260505 05:49:15.647 | INFO | Serializing RTMetrics to json | 20260505 05:49:15.649 | INFO | Serialization done, swapping tmp file now... | 20260505 05:49:15.649 | INFO | RTMetrics done | 20260505 05:49:15.739 | INFO | Finished a round of Baz IO: Wrote 2.301 GiB in 3.376 seconds | 20260505 05:49:15.739 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.47% / 0.07% / 63.47% | 20260505 05:49:15.739 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.49881% / 6.75519e-05% / 3.17891e-05% | 20260505 05:49:15.739 | INFO | Baz IO overall: | 20260505 05:49:15.739 | INFO | BytesWritten : 402490620492 | 20260505 05:49:15.739 | INFO | NumEvents : 189575643639 | 20260505 05:49:17.379 | INFO | Analyzing chunk frames = [823296,823808) | 20260505 05:49:17.379 | INFO | MemInfo: { | 20260505 05:49:17.379 | INFO | "gpu" : " 17 GB", | 20260505 05:49:17.379 | INFO | "huge" : "", | 20260505 05:49:17.379 | INFO | "huge_shared" : " 27 GB", | 20260505 05:49:17.379 | INFO | "malloc" : " 14 GB", | 20260505 05:49:17.379 | INFO | "pinned" : " 29 GB" | 20260505 05:49:17.379 | INFO | } | 20260505 05:49:17.379 | INFO | Perf report for the last 5.13926 seconds: | 20260505 05:49:17.379 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:49:17.379 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:17.379 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:17.379 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:17.379 | INFO | ANALYSIS : 64.7%, 2.9, 99.4ms, 0 | 20260505 05:49:17.379 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:17.379 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260505 05:49:17.379 | INFO | PRE_HQ : 63.8%, 2.0, 64.8ms, 0 | 20260505 05:49:17.379 | INFO | BAZWRITER : 65.0%, 4.0, 134.3ms, 0 | 20260505 05:49:17.379 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 05:49:17.379 | INFO | Uploaded 36GiB to GPU in 2.13233s (16.8829GiB/s) | 20260505 05:49:17.379 | INFO | Downloaded 38GiB to GPU in 2.3182s (16.392GiB/s) | 20260505 05:49:22.518 | INFO | Analyzing chunk frames = [823808,824320) | 20260505 05:49:22.518 | INFO | MemInfo: { | 20260505 05:49:22.518 | INFO | "gpu" : " 17 GB", | 20260505 05:49:22.518 | INFO | "huge" : "", | 20260505 05:49:22.518 | INFO | "huge_shared" : " 27 GB", | 20260505 05:49:22.518 | INFO | "malloc" : " 14 GB", | 20260505 05:49:22.518 | INFO | "pinned" : " 29 GB" | 20260505 05:49:22.519 | INFO | } | 20260505 05:49:22.519 | INFO | Perf report for the last 5.13964 seconds: | 20260505 05:49:22.519 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:49:22.519 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:22.519 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:22.519 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:22.519 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 05:49:22.519 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:22.519 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:22.519 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 05:49:22.519 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:22.519 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:22.519 | INFO | Uploaded 36GiB to GPU in 2.13201s (16.8855GiB/s) | 20260505 05:49:22.519 | INFO | Downloaded 34GiB to GPU in 2.04019s (16.6651GiB/s) | 20260505 05:49:27.657 | INFO | Analyzing chunk frames = [824320,824832) | 20260505 05:49:27.657 | INFO | MemInfo: { | 20260505 05:49:27.657 | INFO | "gpu" : " 17 GB", | 20260505 05:49:27.657 | INFO | "huge" : "", | 20260505 05:49:27.657 | INFO | "huge_shared" : " 27 GB", | 20260505 05:49:27.657 | INFO | "malloc" : " 14 GB", | 20260505 05:49:27.657 | INFO | "pinned" : " 29 GB" | 20260505 05:49:27.658 | INFO | } | 20260505 05:49:27.658 | INFO | Perf report for the last 5.13892 seconds: | 20260505 05:49:27.658 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:49:27.658 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:27.658 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:27.658 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:27.658 | INFO | ANALYSIS : 64.0%, 3.0, 99.5ms, 0 | 20260505 05:49:27.658 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:27.658 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:27.658 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:49:27.658 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:27.658 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:27.658 | INFO | Uploaded 36GiB to GPU in 2.15398s (16.7133GiB/s) | 20260505 05:49:27.658 | INFO | Downloaded 34GiB to GPU in 2.03736s (16.6883GiB/s) | 20260505 05:49:32.797 | INFO | Analyzing chunk frames = [824832,825344) | 20260505 05:49:32.797 | INFO | MemInfo: { | 20260505 05:49:32.797 | INFO | "gpu" : " 17 GB", | 20260505 05:49:32.797 | INFO | "huge" : "", | 20260505 05:49:32.797 | INFO | "huge_shared" : " 27 GB", | 20260505 05:49:32.797 | INFO | "malloc" : " 14 GB", | 20260505 05:49:32.797 | INFO | "pinned" : " 29 GB" | 20260505 05:49:32.797 | INFO | } | 20260505 05:49:32.798 | INFO | Perf report for the last 5.13995 seconds: | 20260505 05:49:32.798 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:49:32.798 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:32.798 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:32.798 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:32.798 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 05:49:32.798 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:32.798 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:32.798 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 05:49:32.798 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:32.798 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:32.798 | INFO | Uploaded 36GiB to GPU in 2.15204s (16.7283GiB/s) | 20260505 05:49:32.798 | INFO | Downloaded 34GiB to GPU in 2.05342s (16.5578GiB/s) | 20260505 05:49:37.936 | INFO | Analyzing chunk frames = [825344,825856) | 20260505 05:49:37.936 | INFO | MemInfo: { | 20260505 05:49:37.936 | INFO | "gpu" : " 17 GB", | 20260505 05:49:37.936 | INFO | "huge" : "", | 20260505 05:49:37.936 | INFO | "huge_shared" : " 27 GB", | 20260505 05:49:37.936 | INFO | "malloc" : " 14 GB", | 20260505 05:49:37.936 | INFO | "pinned" : " 29 GB" | 20260505 05:49:37.936 | INFO | } | 20260505 05:49:37.936 | INFO | Perf report for the last 5.13892 seconds: | 20260505 05:49:37.936 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:49:37.936 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:37.936 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:37.936 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:37.936 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 05:49:37.937 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:37.937 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:37.937 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 05:49:37.937 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:37.937 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:37.937 | INFO | Uploaded 36GiB to GPU in 2.15757s (16.6855GiB/s) | 20260505 05:49:37.937 | INFO | Downloaded 34GiB to GPU in 2.04606s (16.6173GiB/s) | 20260505 05:49:43.075 | INFO | Analyzing chunk frames = [825856,826368) | 20260505 05:49:43.075 | INFO | MemInfo: { | 20260505 05:49:43.075 | INFO | "gpu" : " 17 GB", | 20260505 05:49:43.075 | INFO | "huge" : "", | 20260505 05:49:43.075 | INFO | "huge_shared" : " 27 GB", | 20260505 05:49:43.075 | INFO | "malloc" : " 14 GB", | 20260505 05:49:43.075 | INFO | "pinned" : " 29 GB" | 20260505 05:49:43.076 | INFO | } | 20260505 05:49:43.076 | INFO | Perf report for the last 5.13928 seconds: | 20260505 05:49:43.076 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:49:43.076 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:43.076 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:43.076 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:43.076 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 05:49:43.076 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:43.076 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:43.076 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:49:43.076 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:43.076 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:43.076 | INFO | Uploaded 36GiB to GPU in 2.14922s (16.7503GiB/s) | 20260505 05:49:43.076 | INFO | Downloaded 34GiB to GPU in 2.04436s (16.6311GiB/s) | 20260505 05:49:48.215 | INFO | Analyzing chunk frames = [826368,826880) | 20260505 05:49:48.215 | INFO | MemInfo: { | 20260505 05:49:48.215 | INFO | "gpu" : " 17 GB", | 20260505 05:49:48.215 | INFO | "huge" : "", | 20260505 05:49:48.215 | INFO | "huge_shared" : " 27 GB", | 20260505 05:49:48.215 | INFO | "malloc" : " 14 GB", | 20260505 05:49:48.215 | INFO | "pinned" : " 29 GB" | 20260505 05:49:48.215 | INFO | } | 20260505 05:49:48.215 | INFO | Perf report for the last 5.13944 seconds: | 20260505 05:49:48.215 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:49:48.215 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:48.215 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:48.215 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:48.215 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 05:49:48.215 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:48.215 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:48.215 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 05:49:48.215 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:48.215 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:48.215 | INFO | Uploaded 36GiB to GPU in 2.17205s (16.5742GiB/s) | 20260505 05:49:48.215 | INFO | Downloaded 34GiB to GPU in 2.14638s (15.8406GiB/s) | 20260505 05:49:53.354 | INFO | Analyzing chunk frames = [826880,827392) | 20260505 05:49:53.354 | INFO | MemInfo: { | 20260505 05:49:53.354 | INFO | "gpu" : " 17 GB", | 20260505 05:49:53.354 | INFO | "huge" : "", | 20260505 05:49:53.354 | INFO | "huge_shared" : " 27 GB", | 20260505 05:49:53.354 | INFO | "malloc" : " 14 GB", | 20260505 05:49:53.354 | INFO | "pinned" : " 29 GB" | 20260505 05:49:53.355 | INFO | } | 20260505 05:49:53.355 | INFO | Perf report for the last 5.13938 seconds: | 20260505 05:49:53.355 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:49:53.355 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:53.355 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:53.355 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:53.355 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 05:49:53.355 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:53.355 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:53.355 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 05:49:53.355 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:53.355 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:53.355 | INFO | Uploaded 36GiB to GPU in 2.10007s (17.1423GiB/s) | 20260505 05:49:53.355 | INFO | Downloaded 34GiB to GPU in 2.06999s (16.4252GiB/s) | 20260505 05:49:53.355 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 05:49:53.355 | INFO | Total mem usage: 15.3 GiB | 20260505 05:49:53.355 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 05:49:53.355 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.27%) | 20260505 05:49:53.355 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 05:49:53.355 | INFO | High Water Breakdown: | 20260505 05:49:53.355 | INFO | 60% : 8.66 GiB ( 92.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 05:49:53.355 | INFO | 39.8% : 5.74 GiB ( 86.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 05:49:53.355 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 05:49:53.355 | INFO | ------------------------------------------ | 20260505 05:49:53.355 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:49:53.355 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 05:49:53.355 | INFO | Total mem usage: 26.5 GiB | 20260505 05:49:53.355 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 05:49:53.355 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 05:49:53.355 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 05:49:53.355 | INFO | High Water Breakdown: | 20260505 05:49:53.355 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:49:53.355 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:49:53.355 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:49:53.355 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:49:53.355 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:49:53.355 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:49:53.355 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 05:49:53.355 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 05:49:53.355 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:49:53.355 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 05:49:53.356 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 05:49:53.356 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 05:49:53.356 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 05:49:53.356 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 05:49:53.356 | INFO | ------------------------------------------ | 20260505 05:49:53.356 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:49:53.356 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 05:49:53.356 | INFO | Total mem usage: 25.6 GiB | 20260505 05:49:53.356 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 05:49:53.356 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 05:49:53.356 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 05:49:53.356 | INFO | High Water Breakdown: | 20260505 05:49:53.356 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 05:49:53.356 | INFO | ------------------------------------------ | 20260505 05:49:53.356 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:49:53.356 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 05:49:53.356 | INFO | Total mem usage: 15.7 GiB | 20260505 05:49:53.356 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 05:49:53.356 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 05:49:53.356 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260505 05:49:53.356 | INFO | High Water Breakdown: | 20260505 05:49:53.356 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 05:49:53.356 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 05:49:53.356 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 05:49:53.356 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 05:49:53.356 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 05:49:53.356 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 05:49:53.356 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 05:49:53.356 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 05:49:53.356 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 05:49:53.356 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:49:53.356 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 05:49:53.356 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:49:53.356 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 05:49:53.356 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 05:49:53.356 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 05:49:53.356 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 05:49:53.356 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:49:53.356 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 05:49:53.356 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:49:53.356 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:49:53.356 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:49:53.356 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 05:49:53.356 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 05:49:53.356 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:49:53.356 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 05:49:53.356 | INFO | ------------------------------------------ | 20260505 05:49:53.356 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:49:53.356 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 05:49:53.356 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:49:53.356 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:49:53.356 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.53e+01, 1.39e+01 / 1.66e+00 | 20260505 05:49:53.356 | INFO | PerfReport: 0.28%, AggregateStats -- 9.38e-02 / 1.03e-01, 1.08e-02 / 5.04e-03 | 20260505 05:49:53.356 | INFO | PerfReport: 11.58%, Baseline -- 3.84e+00 / 4.47e+00, 6.34e-01 / 3.18e-01 | 20260505 05:49:53.356 | INFO | PerfReport: 3.30%, Histogram -- 1.10e+00 / 1.47e+00, 2.16e-01 / 1.22e-01 | 20260505 05:49:53.356 | INFO | PerfReport: 15.43%, DME -- 4.10e+01 / 4.48e+01, 4.23e+00 / 1.13e+00 | 20260505 05:49:53.356 | INFO | PerfReport: 55.87%, FrameLabeling -- 1.85e+01 / 1.89e+01, 2.03e+00 / 9.74e-01 | 20260505 05:49:53.356 | INFO | PerfReport: 7.79%, PulseAccumulating -- 2.58e+00 / 3.19e+00, 4.70e-01 / 2.92e-01 | 20260505 05:49:53.356 | INFO | PerfReport: 4.20%, Metrics -- 1.39e+00 / 1.74e+00, 3.87e-01 / 1.85e-01 | 20260505 05:49:53.356 | INFO | PerfReport: 1.54%, PulseSerializing -- 5.11e-01 / 5.88e-01, 1.01e-01 / 3.72e-02 | 20260505 05:49:53.356 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 05:49:53.356 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:49:53.356 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:49:53.356 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.23e+01 / 0.00e+00 | 20260505 05:49:53.356 | INFO | PerfReport: 21.93%, Upload -- 2.23e+01 / 0.00e+00, 7.66e+00 / 0.00e+00 | 20260505 05:49:53.356 | INFO | PerfReport: 21.44%, Download -- 2.18e+01 / 0.00e+00, 7.71e+00 / 0.00e+00 | 20260505 05:49:53.356 | INFO | PerfReport: 32.65%, Compute -- 3.32e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 05:49:53.356 | INFO | PerfReport: 1.44%, LockUpload -- 1.47e+00 / 0.00e+00, 6.82e+00 / 0.00e+00 | 20260505 05:49:53.356 | INFO | PerfReport: 0.77%, LockDownload -- 7.86e-01 / 0.00e+00, 2.22e+00 / 0.00e+00 | 20260505 05:49:53.356 | INFO | PerfReport: 21.75%, LockCompute -- 2.21e+01 / 0.00e+00, 1.68e+01 / 0.00e+00 | 20260505 05:49:53.356 | INFO | PerfReport: 0.01%, Misc -- 1.47e-02 / 0.00e+00, 2.02e-02 / 0.00e+00 | 20260505 05:49:53.468 | INFO | Starting a round of Baz IO | 20260505 05:49:56.742 | INFO | Populating RTMetrics report | 20260505 05:49:56.743 | INFO | RTMetrics report populated | 20260505 05:49:56.743 | INFO | Opening tmp RTMetrics json file | 20260505 05:49:56.761 | INFO | Serializing RTMetrics to json | 20260505 05:49:56.762 | INFO | Serialization done, swapping tmp file now... | 20260505 05:49:56.763 | INFO | RTMetrics done | 20260505 05:49:56.854 | INFO | Finished a round of Baz IO: Wrote 2.299 GiB in 3.386 seconds | 20260505 05:49:56.854 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.40% / 0.07% / 63.53% | 20260505 05:49:56.854 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.44923% / 0.000147025% / 3.97364e-05% | 20260505 05:49:56.854 | INFO | Baz IO overall: | 20260505 05:49:56.854 | INFO | BytesWritten : 404959461754 | 20260505 05:49:56.854 | INFO | NumEvents : 190754394334 | 20260505 05:49:58.494 | INFO | Analyzing chunk frames = [827392,827904) | 20260505 05:49:58.494 | INFO | MemInfo: { | 20260505 05:49:58.494 | INFO | "gpu" : " 17 GB", | 20260505 05:49:58.494 | INFO | "huge" : "", | 20260505 05:49:58.494 | INFO | "huge_shared" : " 27 GB", | 20260505 05:49:58.494 | INFO | "malloc" : " 14 GB", | 20260505 05:49:58.494 | INFO | "pinned" : " 29 GB" | 20260505 05:49:58.494 | INFO | } | 20260505 05:49:58.494 | INFO | Perf report for the last 5.13927 seconds: | 20260505 05:49:58.494 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:49:58.494 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:58.494 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:58.494 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:58.494 | INFO | ANALYSIS : 64.8%, 2.9, 99.5ms, 0 | 20260505 05:49:58.494 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:49:58.494 | INFO | RT_METRICS : 46.2%, 1.0, 24.2ms, 0 | 20260505 05:49:58.494 | INFO | PRE_HQ : 63.9%, 2.0, 64.8ms, 0 | 20260505 05:49:58.494 | INFO | BAZWRITER : 65.2%, 4.0, 133.0ms, 0 | 20260505 05:49:58.494 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 05:49:58.494 | INFO | Uploaded 36GiB to GPU in 2.17435s (16.5567GiB/s) | 20260505 05:49:58.494 | INFO | Downloaded 38GiB to GPU in 2.32538s (16.3414GiB/s) | 20260505 05:50:03.633 | INFO | Analyzing chunk frames = [827904,828416) | 20260505 05:50:03.633 | INFO | MemInfo: { | 20260505 05:50:03.633 | INFO | "gpu" : " 17 GB", | 20260505 05:50:03.633 | INFO | "huge" : "", | 20260505 05:50:03.633 | INFO | "huge_shared" : " 27 GB", | 20260505 05:50:03.633 | INFO | "malloc" : " 14 GB", | 20260505 05:50:03.633 | INFO | "pinned" : " 29 GB" | 20260505 05:50:03.633 | INFO | } | 20260505 05:50:03.633 | INFO | Perf report for the last 5.13907 seconds: | 20260505 05:50:03.633 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:50:03.633 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:03.633 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:03.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:03.633 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 05:50:03.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:03.633 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:03.633 | INFO | PRE_HQ : 30.8%, 1.0, 16.1ms, 0 | 20260505 05:50:03.633 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:03.633 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:03.633 | INFO | Uploaded 36GiB to GPU in 2.12799s (16.9174GiB/s) | 20260505 05:50:03.633 | INFO | Downloaded 34GiB to GPU in 2.0399s (16.6675GiB/s) | 20260505 05:50:08.772 | INFO | Analyzing chunk frames = [828416,828928) | 20260505 05:50:08.772 | INFO | MemInfo: { | 20260505 05:50:08.772 | INFO | "gpu" : " 17 GB", | 20260505 05:50:08.772 | INFO | "huge" : "", | 20260505 05:50:08.772 | INFO | "huge_shared" : " 27 GB", | 20260505 05:50:08.772 | INFO | "malloc" : " 14 GB", | 20260505 05:50:08.772 | INFO | "pinned" : " 29 GB" | 20260505 05:50:08.772 | INFO | } | 20260505 05:50:08.773 | INFO | Perf report for the last 5.13961 seconds: | 20260505 05:50:08.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:50:08.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:08.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:08.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:08.773 | INFO | ANALYSIS : 64.0%, 3.0, 99.6ms, 0 | 20260505 05:50:08.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:08.773 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:08.773 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260505 05:50:08.773 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:08.773 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:08.773 | INFO | Uploaded 36GiB to GPU in 2.15699s (16.6899GiB/s) | 20260505 05:50:08.773 | INFO | Downloaded 34GiB to GPU in 2.03838s (16.6799GiB/s) | 20260505 05:50:13.912 | INFO | Analyzing chunk frames = [828928,829440) | 20260505 05:50:13.912 | INFO | MemInfo: { | 20260505 05:50:13.912 | INFO | "gpu" : " 17 GB", | 20260505 05:50:13.912 | INFO | "huge" : "", | 20260505 05:50:13.912 | INFO | "huge_shared" : " 27 GB", | 20260505 05:50:13.912 | INFO | "malloc" : " 14 GB", | 20260505 05:50:13.912 | INFO | "pinned" : " 29 GB" | 20260505 05:50:13.912 | INFO | } | 20260505 05:50:13.912 | INFO | Perf report for the last 5.1393 seconds: | 20260505 05:50:13.912 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:50:13.912 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:13.912 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:13.912 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:13.912 | INFO | ANALYSIS : 63.4%, 3.0, 98.5ms, 0 | 20260505 05:50:13.912 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:13.912 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:13.912 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 05:50:13.912 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:13.912 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:13.912 | INFO | Uploaded 36GiB to GPU in 2.16531s (16.6258GiB/s) | 20260505 05:50:13.912 | INFO | Downloaded 34GiB to GPU in 2.05043s (16.5819GiB/s) | 20260505 05:50:19.051 | INFO | Analyzing chunk frames = [829440,829952) | 20260505 05:50:19.051 | INFO | MemInfo: { | 20260505 05:50:19.051 | INFO | "gpu" : " 17 GB", | 20260505 05:50:19.051 | INFO | "huge" : "", | 20260505 05:50:19.051 | INFO | "huge_shared" : " 27 GB", | 20260505 05:50:19.051 | INFO | "malloc" : " 14 GB", | 20260505 05:50:19.051 | INFO | "pinned" : " 29 GB" | 20260505 05:50:19.051 | INFO | } | 20260505 05:50:19.051 | INFO | Perf report for the last 5.13948 seconds: | 20260505 05:50:19.051 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:50:19.051 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:19.051 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:19.051 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:19.051 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 05:50:19.051 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:19.052 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:19.052 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 05:50:19.052 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:19.052 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:19.052 | INFO | Uploaded 36GiB to GPU in 2.12495s (16.9415GiB/s) | 20260505 05:50:19.052 | INFO | Downloaded 34GiB to GPU in 2.04263s (16.6452GiB/s) | 20260505 05:50:24.190 | INFO | Analyzing chunk frames = [829952,830464) | 20260505 05:50:24.190 | INFO | MemInfo: { | 20260505 05:50:24.190 | INFO | "gpu" : " 17 GB", | 20260505 05:50:24.190 | INFO | "huge" : "", | 20260505 05:50:24.190 | INFO | "huge_shared" : " 27 GB", | 20260505 05:50:24.190 | INFO | "malloc" : " 14 GB", | 20260505 05:50:24.190 | INFO | "pinned" : " 29 GB" | 20260505 05:50:24.190 | INFO | } | 20260505 05:50:24.190 | INFO | Perf report for the last 5.13905 seconds: | 20260505 05:50:24.191 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:50:24.191 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:24.191 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:24.191 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:24.191 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 05:50:24.191 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:24.191 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:24.191 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 05:50:24.191 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:24.191 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:24.191 | INFO | Uploaded 36GiB to GPU in 2.12981s (16.9029GiB/s) | 20260505 05:50:24.191 | INFO | Downloaded 34GiB to GPU in 2.06136s (16.494GiB/s) | 20260505 05:50:29.330 | INFO | Analyzing chunk frames = [830464,830976) | 20260505 05:50:29.330 | INFO | MemInfo: { | 20260505 05:50:29.330 | INFO | "gpu" : " 17 GB", | 20260505 05:50:29.330 | INFO | "huge" : "", | 20260505 05:50:29.330 | INFO | "huge_shared" : " 27 GB", | 20260505 05:50:29.330 | INFO | "malloc" : " 14 GB", | 20260505 05:50:29.330 | INFO | "pinned" : " 29 GB" | 20260505 05:50:29.330 | INFO | } | 20260505 05:50:29.330 | INFO | Perf report for the last 5.13959 seconds: | 20260505 05:50:29.330 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:50:29.330 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:29.330 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:29.330 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:29.330 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 05:50:29.330 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:29.330 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:29.330 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260505 05:50:29.330 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:29.330 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:29.330 | INFO | Uploaded 36GiB to GPU in 2.15302s (16.7207GiB/s) | 20260505 05:50:29.331 | INFO | Downloaded 34GiB to GPU in 2.13628s (15.9155GiB/s) | 20260505 05:50:34.470 | INFO | Analyzing chunk frames = [830976,831488) | 20260505 05:50:34.470 | INFO | MemInfo: { | 20260505 05:50:34.470 | INFO | "gpu" : " 17 GB", | 20260505 05:50:34.470 | INFO | "huge" : "", | 20260505 05:50:34.470 | INFO | "huge_shared" : " 27 GB", | 20260505 05:50:34.470 | INFO | "malloc" : " 14 GB", | 20260505 05:50:34.470 | INFO | "pinned" : " 29 GB" | 20260505 05:50:34.470 | INFO | } | 20260505 05:50:34.470 | INFO | Perf report for the last 5.13983 seconds: | 20260505 05:50:34.470 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:50:34.470 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:34.470 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:34.470 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:34.470 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 05:50:34.470 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:34.470 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:34.470 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 05:50:34.470 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:34.470 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:34.470 | INFO | Uploaded 36GiB to GPU in 2.11098s (17.0537GiB/s) | 20260505 05:50:34.470 | INFO | Downloaded 34GiB to GPU in 2.07075s (16.4192GiB/s) | 20260505 05:50:34.580 | INFO | Starting a round of Baz IO | 20260505 05:50:37.812 | INFO | Populating RTMetrics report | 20260505 05:50:37.812 | INFO | RTMetrics report populated | 20260505 05:50:37.812 | INFO | Opening tmp RTMetrics json file | 20260505 05:50:37.833 | INFO | Serializing RTMetrics to json | 20260505 05:50:37.834 | INFO | Serialization done, swapping tmp file now... | 20260505 05:50:37.835 | INFO | RTMetrics done | 20260505 05:50:37.921 | INFO | Finished a round of Baz IO: Wrote 2.303 GiB in 3.34 seconds | 20260505 05:50:37.921 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.33% / 0.07% / 63.60% | 20260505 05:50:37.921 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.43401% / 0.000115236% / 5.16574e-05% | 20260505 05:50:37.921 | INFO | Baz IO overall: | 20260505 05:50:37.921 | INFO | BytesWritten : 407432305464 | 20260505 05:50:37.921 | INFO | NumEvents : 191935538206 | 20260505 05:50:39.608 | INFO | Analyzing chunk frames = [831488,832000) | 20260505 05:50:39.608 | INFO | MemInfo: { | 20260505 05:50:39.608 | INFO | "gpu" : " 17 GB", | 20260505 05:50:39.608 | INFO | "huge" : "", | 20260505 05:50:39.608 | INFO | "huge_shared" : " 27 GB", | 20260505 05:50:39.608 | INFO | "malloc" : " 14 GB", | 20260505 05:50:39.608 | INFO | "pinned" : " 29 GB" | 20260505 05:50:39.608 | INFO | } | 20260505 05:50:39.609 | INFO | Perf report for the last 5.13868 seconds: | 20260505 05:50:39.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:50:39.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:39.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:39.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:39.609 | INFO | ANALYSIS : 63.9%, 2.9, 98.5ms, 0 | 20260505 05:50:39.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:39.609 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260505 05:50:39.609 | INFO | PRE_HQ : 63.2%, 2.0, 65.2ms, 0 | 20260505 05:50:39.609 | INFO | BAZWRITER : 64.3%, 4.0, 133.6ms, 0 | 20260505 05:50:39.609 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 05:50:39.609 | INFO | Uploaded 36GiB to GPU in 2.09384s (17.1933GiB/s) | 20260505 05:50:39.609 | INFO | Downloaded 38GiB to GPU in 2.31554s (16.4109GiB/s) | 20260505 05:50:44.748 | INFO | Analyzing chunk frames = [832000,832512) | 20260505 05:50:44.748 | INFO | MemInfo: { | 20260505 05:50:44.748 | INFO | "gpu" : " 17 GB", | 20260505 05:50:44.748 | INFO | "huge" : "", | 20260505 05:50:44.748 | INFO | "huge_shared" : " 27 GB", | 20260505 05:50:44.748 | INFO | "malloc" : " 14 GB", | 20260505 05:50:44.748 | INFO | "pinned" : " 29 GB" | 20260505 05:50:44.748 | INFO | } | 20260505 05:50:44.748 | INFO | Perf report for the last 5.13913 seconds: | 20260505 05:50:44.748 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:50:44.748 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:44.748 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:44.748 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:44.748 | INFO | ANALYSIS : 64.4%, 3.0, 100.6ms, 0 | 20260505 05:50:44.748 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:44.748 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:44.748 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 05:50:44.748 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:44.748 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:44.748 | INFO | Uploaded 36GiB to GPU in 2.12724s (16.9233GiB/s) | 20260505 05:50:44.748 | INFO | Downloaded 34GiB to GPU in 2.03581s (16.701GiB/s) | 20260505 05:50:49.887 | INFO | Analyzing chunk frames = [832512,833024) | 20260505 05:50:49.887 | INFO | MemInfo: { | 20260505 05:50:49.887 | INFO | "gpu" : " 17 GB", | 20260505 05:50:49.887 | INFO | "huge" : "", | 20260505 05:50:49.887 | INFO | "huge_shared" : " 27 GB", | 20260505 05:50:49.887 | INFO | "malloc" : " 14 GB", | 20260505 05:50:49.887 | INFO | "pinned" : " 29 GB" | 20260505 05:50:49.887 | INFO | } | 20260505 05:50:49.887 | INFO | Perf report for the last 5.13943 seconds: | 20260505 05:50:49.887 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:50:49.887 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:49.887 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:49.887 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:49.887 | INFO | ANALYSIS : 64.1%, 3.0, 99.8ms, 0 | 20260505 05:50:49.887 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:49.887 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:49.887 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 05:50:49.887 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:49.887 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:49.887 | INFO | Uploaded 36GiB to GPU in 2.15928s (16.6722GiB/s) | 20260505 05:50:49.888 | INFO | Downloaded 34GiB to GPU in 2.03922s (16.673GiB/s) | 20260505 05:50:55.026 | INFO | Analyzing chunk frames = [833024,833536) | 20260505 05:50:55.026 | INFO | MemInfo: { | 20260505 05:50:55.026 | INFO | "gpu" : " 17 GB", | 20260505 05:50:55.026 | INFO | "huge" : "", | 20260505 05:50:55.026 | INFO | "huge_shared" : " 27 GB", | 20260505 05:50:55.026 | INFO | "malloc" : " 14 GB", | 20260505 05:50:55.026 | INFO | "pinned" : " 29 GB" | 20260505 05:50:55.026 | INFO | } | 20260505 05:50:55.026 | INFO | Perf report for the last 5.13934 seconds: | 20260505 05:50:55.026 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:50:55.027 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:55.027 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:55.027 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:55.027 | INFO | ANALYSIS : 63.4%, 3.0, 98.4ms, 0 | 20260505 05:50:55.027 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:55.027 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:55.027 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 05:50:55.027 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:55.027 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:50:55.027 | INFO | Uploaded 36GiB to GPU in 2.13751s (16.8421GiB/s) | 20260505 05:50:55.027 | INFO | Downloaded 34GiB to GPU in 2.05154s (16.5729GiB/s) | 20260505 05:51:00.166 | INFO | Analyzing chunk frames = [833536,834048) | 20260505 05:51:00.166 | INFO | MemInfo: { | 20260505 05:51:00.166 | INFO | "gpu" : " 17 GB", | 20260505 05:51:00.166 | INFO | "huge" : "", | 20260505 05:51:00.166 | INFO | "huge_shared" : " 27 GB", | 20260505 05:51:00.166 | INFO | "malloc" : " 14 GB", | 20260505 05:51:00.166 | INFO | "pinned" : " 29 GB" | 20260505 05:51:00.166 | INFO | } | 20260505 05:51:00.166 | INFO | Perf report for the last 5.13929 seconds: | 20260505 05:51:00.166 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:51:00.166 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:00.166 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:00.166 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:00.166 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 05:51:00.166 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:00.166 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:00.166 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 05:51:00.166 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:00.166 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:00.166 | INFO | Uploaded 36GiB to GPU in 2.11687s (17.0062GiB/s) | 20260505 05:51:00.166 | INFO | Downloaded 34GiB to GPU in 2.04678s (16.6114GiB/s) | 20260505 05:51:05.305 | INFO | Analyzing chunk frames = [834048,834560) | 20260505 05:51:05.305 | INFO | MemInfo: { | 20260505 05:51:05.305 | INFO | "gpu" : " 17 GB", | 20260505 05:51:05.305 | INFO | "huge" : "", | 20260505 05:51:05.305 | INFO | "huge_shared" : " 27 GB", | 20260505 05:51:05.305 | INFO | "malloc" : " 14 GB", | 20260505 05:51:05.305 | INFO | "pinned" : " 29 GB" | 20260505 05:51:05.305 | INFO | } | 20260505 05:51:05.305 | INFO | Perf report for the last 5.13949 seconds: | 20260505 05:51:05.305 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:51:05.305 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:05.305 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:05.305 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:05.305 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 05:51:05.305 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:05.305 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:05.305 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:51:05.305 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:05.305 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:05.306 | INFO | Uploaded 36GiB to GPU in 2.13413s (16.8687GiB/s) | 20260505 05:51:05.306 | INFO | Downloaded 34GiB to GPU in 2.06173s (16.491GiB/s) | 20260505 05:51:10.444 | INFO | Analyzing chunk frames = [834560,835072) | 20260505 05:51:10.444 | INFO | MemInfo: { | 20260505 05:51:10.444 | INFO | "gpu" : " 17 GB", | 20260505 05:51:10.444 | INFO | "huge" : "", | 20260505 05:51:10.444 | INFO | "huge_shared" : " 27 GB", | 20260505 05:51:10.444 | INFO | "malloc" : " 14 GB", | 20260505 05:51:10.444 | INFO | "pinned" : " 29 GB" | 20260505 05:51:10.445 | INFO | } | 20260505 05:51:10.445 | INFO | Perf report for the last 5.13928 seconds: | 20260505 05:51:10.445 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:51:10.445 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:10.445 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:10.445 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:10.445 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 05:51:10.445 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:10.445 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:10.445 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 05:51:10.445 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:10.445 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:10.445 | INFO | Uploaded 36GiB to GPU in 2.16119s (16.6575GiB/s) | 20260505 05:51:10.445 | INFO | Downloaded 34GiB to GPU in 2.1406s (15.8834GiB/s) | 20260505 05:51:15.584 | INFO | Analyzing chunk frames = [835072,835584) | 20260505 05:51:15.584 | INFO | MemInfo: { | 20260505 05:51:15.584 | INFO | "gpu" : " 17 GB", | 20260505 05:51:15.584 | INFO | "huge" : "", | 20260505 05:51:15.584 | INFO | "huge_shared" : " 27 GB", | 20260505 05:51:15.584 | INFO | "malloc" : " 14 GB", | 20260505 05:51:15.584 | INFO | "pinned" : " 29 GB" | 20260505 05:51:15.584 | INFO | } | 20260505 05:51:15.584 | INFO | Perf report for the last 5.13947 seconds: | 20260505 05:51:15.584 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:51:15.584 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:15.584 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:15.584 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:15.584 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 05:51:15.584 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:15.584 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:15.584 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 05:51:15.584 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:15.584 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:15.584 | INFO | Uploaded 36GiB to GPU in 2.09862s (17.1541GiB/s) | 20260505 05:51:15.584 | INFO | Downloaded 34GiB to GPU in 2.07373s (16.3956GiB/s) | 20260505 05:51:15.584 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 05:51:15.584 | INFO | Total mem usage: 15.3 GiB | 20260505 05:51:15.584 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 05:51:15.584 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.33%) | 20260505 05:51:15.584 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 05:51:15.584 | INFO | High Water Breakdown: | 20260505 05:51:15.584 | INFO | 60.1% : 8.66 GiB ( 92.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 05:51:15.584 | INFO | 39.8% : 5.74 GiB ( 86.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 05:51:15.584 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 05:51:15.584 | INFO | ------------------------------------------ | 20260505 05:51:15.585 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:51:15.585 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 05:51:15.585 | INFO | Total mem usage: 26.5 GiB | 20260505 05:51:15.585 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 05:51:15.585 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 05:51:15.585 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 05:51:15.585 | INFO | High Water Breakdown: | 20260505 05:51:15.585 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:51:15.585 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:51:15.585 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:51:15.585 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:51:15.585 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:51:15.585 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:51:15.585 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 05:51:15.585 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 05:51:15.585 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:51:15.585 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 05:51:15.585 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 05:51:15.585 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 05:51:15.585 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 05:51:15.585 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 05:51:15.585 | INFO | ------------------------------------------ | 20260505 05:51:15.585 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:51:15.585 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 05:51:15.585 | INFO | Total mem usage: 25.6 GiB | 20260505 05:51:15.585 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 05:51:15.585 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 05:51:15.585 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 05:51:15.585 | INFO | High Water Breakdown: | 20260505 05:51:15.585 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 05:51:15.585 | INFO | ------------------------------------------ | 20260505 05:51:15.585 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:51:15.585 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 05:51:15.585 | INFO | Total mem usage: 15.7 GiB | 20260505 05:51:15.585 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 05:51:15.585 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 05:51:15.585 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260505 05:51:15.585 | INFO | High Water Breakdown: | 20260505 05:51:15.585 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 05:51:15.585 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 05:51:15.585 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 05:51:15.585 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 05:51:15.585 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 05:51:15.585 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 05:51:15.585 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 05:51:15.585 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 05:51:15.585 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 05:51:15.585 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:51:15.585 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 05:51:15.585 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:51:15.585 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 05:51:15.585 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 05:51:15.585 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 05:51:15.585 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 05:51:15.585 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:51:15.585 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 05:51:15.585 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:51:15.585 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:51:15.585 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:51:15.585 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 05:51:15.585 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 05:51:15.585 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:51:15.585 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 05:51:15.585 | INFO | ------------------------------------------ | 20260505 05:51:15.585 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:51:15.585 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 05:51:15.585 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:51:15.585 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:51:15.585 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.54e+01, 1.36e+01 / 1.83e+00 | 20260505 05:51:15.585 | INFO | PerfReport: 0.28%, AggregateStats -- 9.41e-02 / 1.02e-01, 1.07e-02 / 6.34e-03 | 20260505 05:51:15.585 | INFO | PerfReport: 11.58%, Baseline -- 3.84e+00 / 4.52e+00, 6.37e-01 / 2.73e-01 | 20260505 05:51:15.585 | INFO | PerfReport: 3.31%, Histogram -- 1.10e+00 / 1.50e+00, 2.18e-01 / 1.25e-01 | 20260505 05:51:15.585 | INFO | PerfReport: 15.45%, DME -- 4.10e+01 / 4.49e+01, 4.39e+00 / 1.23e+00 | 20260505 05:51:15.585 | INFO | PerfReport: 55.77%, FrameLabeling -- 1.85e+01 / 1.89e+01, 2.02e+00 / 1.01e+00 | 20260505 05:51:15.585 | INFO | PerfReport: 7.86%, PulseAccumulating -- 2.61e+00 / 3.19e+00, 4.97e-01 / 3.28e-01 | 20260505 05:51:15.585 | INFO | PerfReport: 4.20%, Metrics -- 1.39e+00 / 1.76e+00, 3.89e-01 / 2.24e-01 | 20260505 05:51:15.585 | INFO | PerfReport: 1.54%, PulseSerializing -- 5.12e-01 / 5.93e-01, 1.02e-01 / 4.96e-02 | 20260505 05:51:15.585 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 05:51:15.585 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:51:15.585 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:51:15.585 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.25e+01 / 0.00e+00 | 20260505 05:51:15.585 | INFO | PerfReport: 21.93%, Upload -- 2.22e+01 / 0.00e+00, 7.56e+00 / 0.00e+00 | 20260505 05:51:15.585 | INFO | PerfReport: 21.52%, Download -- 2.18e+01 / 0.00e+00, 7.70e+00 / 0.00e+00 | 20260505 05:51:15.585 | INFO | PerfReport: 32.72%, Compute -- 3.32e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 05:51:15.585 | INFO | PerfReport: 1.43%, LockUpload -- 1.45e+00 / 0.00e+00, 6.70e+00 / 0.00e+00 | 20260505 05:51:15.585 | INFO | PerfReport: 0.70%, LockDownload -- 7.13e-01 / 0.00e+00, 1.94e+00 / 0.00e+00 | 20260505 05:51:15.585 | INFO | PerfReport: 21.69%, LockCompute -- 2.20e+01 / 0.00e+00, 1.66e+01 / 0.00e+00 | 20260505 05:51:15.585 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.61e-02 / 0.00e+00 | 20260505 05:51:15.694 | INFO | Starting a round of Baz IO | 20260505 05:51:18.968 | INFO | Populating RTMetrics report | 20260505 05:51:18.969 | INFO | RTMetrics report populated | 20260505 05:51:18.969 | INFO | Opening tmp RTMetrics json file | 20260505 05:51:18.970 | INFO | Serializing RTMetrics to json | 20260505 05:51:18.970 | INFO | Serialization done, swapping tmp file now... | 20260505 05:51:18.971 | INFO | RTMetrics done | 20260505 05:51:19.095 | INFO | Finished a round of Baz IO: Wrote 2.302 GiB in 3.401 seconds | 20260505 05:51:19.095 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.27% / 0.07% / 63.67% | 20260505 05:51:19.095 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.38405% / 0.000103315% / 7.15256e-05% | 20260505 05:51:19.095 | INFO | Baz IO overall: | 20260505 05:51:19.095 | INFO | BytesWritten : 409904097229 | 20260505 05:51:19.095 | INFO | NumEvents : 193115660842 | 20260505 05:51:20.723 | INFO | Analyzing chunk frames = [835584,836096) | 20260505 05:51:20.723 | INFO | MemInfo: { | 20260505 05:51:20.723 | INFO | "gpu" : " 17 GB", | 20260505 05:51:20.723 | INFO | "huge" : "", | 20260505 05:51:20.723 | INFO | "huge_shared" : " 27 GB", | 20260505 05:51:20.723 | INFO | "malloc" : " 14 GB", | 20260505 05:51:20.723 | INFO | "pinned" : " 29 GB" | 20260505 05:51:20.723 | INFO | } | 20260505 05:51:20.723 | INFO | Perf report for the last 5.13912 seconds: | 20260505 05:51:20.723 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:51:20.723 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:20.723 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:20.723 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:20.723 | INFO | ANALYSIS : 64.7%, 2.9, 99.5ms, 0 | 20260505 05:51:20.723 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:20.723 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260505 05:51:20.723 | INFO | PRE_HQ : 63.9%, 2.0, 64.8ms, 0 | 20260505 05:51:20.723 | INFO | BAZWRITER : 65.5%, 4.0, 134.7ms, 0 | 20260505 05:51:20.723 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 05:51:20.723 | INFO | Uploaded 36GiB to GPU in 2.15924s (16.6725GiB/s) | 20260505 05:51:20.723 | INFO | Downloaded 38GiB to GPU in 2.31763s (16.3961GiB/s) | 20260505 05:51:25.862 | INFO | Analyzing chunk frames = [836096,836608) | 20260505 05:51:25.862 | INFO | MemInfo: { | 20260505 05:51:25.862 | INFO | "gpu" : " 17 GB", | 20260505 05:51:25.862 | INFO | "huge" : "", | 20260505 05:51:25.862 | INFO | "huge_shared" : " 27 GB", | 20260505 05:51:25.862 | INFO | "malloc" : " 14 GB", | 20260505 05:51:25.862 | INFO | "pinned" : " 29 GB" | 20260505 05:51:25.862 | INFO | } | 20260505 05:51:25.862 | INFO | Perf report for the last 5.13929 seconds: | 20260505 05:51:25.862 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:51:25.862 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:25.862 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:25.863 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:25.863 | INFO | ANALYSIS : 64.4%, 3.0, 100.6ms, 0 | 20260505 05:51:25.863 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:25.863 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:25.863 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:51:25.863 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:25.863 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:25.863 | INFO | Uploaded 36GiB to GPU in 2.13681s (16.8476GiB/s) | 20260505 05:51:25.863 | INFO | Downloaded 34GiB to GPU in 2.03971s (16.6691GiB/s) | 20260505 05:51:31.002 | INFO | Analyzing chunk frames = [836608,837120) | 20260505 05:51:31.002 | INFO | MemInfo: { | 20260505 05:51:31.002 | INFO | "gpu" : " 17 GB", | 20260505 05:51:31.002 | INFO | "huge" : "", | 20260505 05:51:31.002 | INFO | "huge_shared" : " 27 GB", | 20260505 05:51:31.002 | INFO | "malloc" : " 14 GB", | 20260505 05:51:31.002 | INFO | "pinned" : " 29 GB" | 20260505 05:51:31.002 | INFO | } | 20260505 05:51:31.002 | INFO | Perf report for the last 5.13953 seconds: | 20260505 05:51:31.002 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:51:31.002 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:31.002 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:31.002 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:31.002 | INFO | ANALYSIS : 64.2%, 3.0, 99.9ms, 0 | 20260505 05:51:31.002 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:31.002 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:31.002 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 05:51:31.002 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:31.002 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:31.002 | INFO | Uploaded 36GiB to GPU in 2.16752s (16.6089GiB/s) | 20260505 05:51:31.002 | INFO | Downloaded 34GiB to GPU in 2.04344s (16.6386GiB/s) | 20260505 05:51:36.141 | INFO | Analyzing chunk frames = [837120,837632) | 20260505 05:51:36.141 | INFO | MemInfo: { | 20260505 05:51:36.141 | INFO | "gpu" : " 17 GB", | 20260505 05:51:36.141 | INFO | "huge" : "", | 20260505 05:51:36.141 | INFO | "huge_shared" : " 27 GB", | 20260505 05:51:36.141 | INFO | "malloc" : " 14 GB", | 20260505 05:51:36.141 | INFO | "pinned" : " 29 GB" | 20260505 05:51:36.141 | INFO | } | 20260505 05:51:36.141 | INFO | Perf report for the last 5.13932 seconds: | 20260505 05:51:36.141 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:51:36.141 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:36.141 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:36.141 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:36.141 | INFO | ANALYSIS : 63.4%, 3.0, 98.5ms, 0 | 20260505 05:51:36.141 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:36.141 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:36.142 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260505 05:51:36.142 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:36.142 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:36.142 | INFO | Uploaded 36GiB to GPU in 2.09927s (17.1488GiB/s) | 20260505 05:51:36.142 | INFO | Downloaded 34GiB to GPU in 2.03096s (16.7408GiB/s) | 20260505 05:51:41.281 | INFO | Analyzing chunk frames = [837632,838144) | 20260505 05:51:41.281 | INFO | MemInfo: { | 20260505 05:51:41.281 | INFO | "gpu" : " 17 GB", | 20260505 05:51:41.281 | INFO | "huge" : "", | 20260505 05:51:41.281 | INFO | "huge_shared" : " 27 GB", | 20260505 05:51:41.281 | INFO | "malloc" : " 14 GB", | 20260505 05:51:41.281 | INFO | "pinned" : " 29 GB" | 20260505 05:51:41.281 | INFO | } | 20260505 05:51:41.281 | INFO | Perf report for the last 5.13972 seconds: | 20260505 05:51:41.281 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:51:41.281 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:41.281 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:41.281 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:41.281 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 05:51:41.281 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:41.281 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:41.281 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 05:51:41.281 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:41.281 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:41.281 | INFO | Uploaded 36GiB to GPU in 2.11886s (16.9902GiB/s) | 20260505 05:51:41.281 | INFO | Downloaded 34GiB to GPU in 2.04654s (16.6134GiB/s) | 20260505 05:51:46.420 | INFO | Analyzing chunk frames = [838144,838656) | 20260505 05:51:46.420 | INFO | MemInfo: { | 20260505 05:51:46.420 | INFO | "gpu" : " 17 GB", | 20260505 05:51:46.420 | INFO | "huge" : "", | 20260505 05:51:46.420 | INFO | "huge_shared" : " 27 GB", | 20260505 05:51:46.420 | INFO | "malloc" : " 14 GB", | 20260505 05:51:46.420 | INFO | "pinned" : " 29 GB" | 20260505 05:51:46.420 | INFO | } | 20260505 05:51:46.420 | INFO | Perf report for the last 5.13912 seconds: | 20260505 05:51:46.420 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:51:46.420 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:46.420 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:46.420 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:46.420 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 05:51:46.420 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:46.420 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:46.420 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 05:51:46.420 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:46.420 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:46.421 | INFO | Uploaded 36GiB to GPU in 2.13484s (16.8631GiB/s) | 20260505 05:51:46.421 | INFO | Downloaded 34GiB to GPU in 2.06298s (16.481GiB/s) | 20260505 05:51:51.559 | INFO | Analyzing chunk frames = [838656,839168) | 20260505 05:51:51.559 | INFO | MemInfo: { | 20260505 05:51:51.559 | INFO | "gpu" : " 17 GB", | 20260505 05:51:51.559 | INFO | "huge" : "", | 20260505 05:51:51.559 | INFO | "huge_shared" : " 27 GB", | 20260505 05:51:51.559 | INFO | "malloc" : " 14 GB", | 20260505 05:51:51.559 | INFO | "pinned" : " 29 GB" | 20260505 05:51:51.559 | INFO | } | 20260505 05:51:51.559 | INFO | Perf report for the last 5.13923 seconds: | 20260505 05:51:51.559 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:51:51.559 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:51.559 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:51.560 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:51.560 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 05:51:51.560 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:51.560 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:51.560 | INFO | PRE_HQ : 32.0%, 1.0, 16.7ms, 0 | 20260505 05:51:51.560 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:51.560 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:51.560 | INFO | Uploaded 36GiB to GPU in 2.16839s (16.6022GiB/s) | 20260505 05:51:51.560 | INFO | Downloaded 34GiB to GPU in 2.15678s (15.7642GiB/s) | 20260505 05:51:56.698 | INFO | Analyzing chunk frames = [839168,839680) | 20260505 05:51:56.698 | INFO | MemInfo: { | 20260505 05:51:56.698 | INFO | "gpu" : " 17 GB", | 20260505 05:51:56.698 | INFO | "huge" : "", | 20260505 05:51:56.698 | INFO | "huge_shared" : " 27 GB", | 20260505 05:51:56.698 | INFO | "malloc" : " 14 GB", | 20260505 05:51:56.698 | INFO | "pinned" : " 29 GB" | 20260505 05:51:56.699 | INFO | } | 20260505 05:51:56.699 | INFO | Perf report for the last 5.13921 seconds: | 20260505 05:51:56.699 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:51:56.699 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:56.699 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:56.699 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:56.699 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 05:51:56.699 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:56.699 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:56.699 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 05:51:56.699 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:56.699 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:51:56.699 | INFO | Uploaded 36GiB to GPU in 2.10223s (17.1247GiB/s) | 20260505 05:51:56.699 | INFO | Downloaded 34GiB to GPU in 2.07246s (16.4056GiB/s) | 20260505 05:51:56.812 | INFO | Starting a round of Baz IO | 20260505 05:52:00.065 | INFO | Populating RTMetrics report | 20260505 05:52:00.066 | INFO | RTMetrics report populated | 20260505 05:52:00.066 | INFO | Opening tmp RTMetrics json file | 20260505 05:52:00.081 | INFO | Serializing RTMetrics to json | 20260505 05:52:00.082 | INFO | Serialization done, swapping tmp file now... | 20260505 05:52:00.082 | INFO | RTMetrics done | 20260505 05:52:00.177 | INFO | Finished a round of Baz IO: Wrote 2.311 GiB in 3.364 seconds | 20260505 05:52:00.177 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.20% / 0.07% / 63.73% | 20260505 05:52:00.177 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.4087% / 0.00013113% / 5.96046e-05% | 20260505 05:52:00.177 | INFO | Baz IO overall: | 20260505 05:52:00.177 | INFO | BytesWritten : 412385168366 | 20260505 05:52:00.177 | INFO | NumEvents : 194301909070 | 20260505 05:52:01.838 | INFO | Analyzing chunk frames = [839680,840192) | 20260505 05:52:01.838 | INFO | MemInfo: { | 20260505 05:52:01.838 | INFO | "gpu" : " 17 GB", | 20260505 05:52:01.838 | INFO | "huge" : "", | 20260505 05:52:01.838 | INFO | "huge_shared" : " 27 GB", | 20260505 05:52:01.838 | INFO | "malloc" : " 14 GB", | 20260505 05:52:01.838 | INFO | "pinned" : " 29 GB" | 20260505 05:52:01.838 | INFO | } | 20260505 05:52:01.838 | INFO | Perf report for the last 5.13935 seconds: | 20260505 05:52:01.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:52:01.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:01.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:01.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:01.838 | INFO | ANALYSIS : 64.4%, 2.9, 98.9ms, 0 | 20260505 05:52:01.838 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:01.838 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260505 05:52:01.838 | INFO | PRE_HQ : 63.6%, 2.0, 65.0ms, 0 | 20260505 05:52:01.838 | INFO | BAZWRITER : 64.8%, 4.0, 133.1ms, 0 | 20260505 05:52:01.838 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 05:52:01.838 | INFO | Uploaded 36GiB to GPU in 2.12666s (16.9279GiB/s) | 20260505 05:52:01.838 | INFO | Downloaded 38GiB to GPU in 2.33571s (16.2691GiB/s) | 20260505 05:52:06.977 | INFO | Analyzing chunk frames = [840192,840704) | 20260505 05:52:06.977 | INFO | MemInfo: { | 20260505 05:52:06.977 | INFO | "gpu" : " 17 GB", | 20260505 05:52:06.977 | INFO | "huge" : "", | 20260505 05:52:06.977 | INFO | "huge_shared" : " 27 GB", | 20260505 05:52:06.977 | INFO | "malloc" : " 14 GB", | 20260505 05:52:06.977 | INFO | "pinned" : " 29 GB" | 20260505 05:52:06.977 | INFO | } | 20260505 05:52:06.977 | INFO | Perf report for the last 5.13942 seconds: | 20260505 05:52:06.977 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:52:06.978 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:06.978 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:06.978 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:06.978 | INFO | ANALYSIS : 64.5%, 3.0, 100.7ms, 0 | 20260505 05:52:06.978 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:06.978 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:06.978 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 05:52:06.978 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:06.978 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:06.978 | INFO | Uploaded 36GiB to GPU in 2.13181s (16.887GiB/s) | 20260505 05:52:06.978 | INFO | Downloaded 34GiB to GPU in 2.04143s (16.655GiB/s) | 20260505 05:52:12.117 | INFO | Analyzing chunk frames = [840704,841216) | 20260505 05:52:12.117 | INFO | MemInfo: { | 20260505 05:52:12.117 | INFO | "gpu" : " 17 GB", | 20260505 05:52:12.117 | INFO | "huge" : "", | 20260505 05:52:12.117 | INFO | "huge_shared" : " 27 GB", | 20260505 05:52:12.117 | INFO | "malloc" : " 14 GB", | 20260505 05:52:12.117 | INFO | "pinned" : " 29 GB" | 20260505 05:52:12.117 | INFO | } | 20260505 05:52:12.117 | INFO | Perf report for the last 5.13939 seconds: | 20260505 05:52:12.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:52:12.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:12.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:12.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:12.117 | INFO | ANALYSIS : 64.3%, 3.0, 100.0ms, 0 | 20260505 05:52:12.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:12.117 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:12.117 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:52:12.117 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:12.117 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:12.117 | INFO | Uploaded 36GiB to GPU in 2.16728s (16.6107GiB/s) | 20260505 05:52:12.117 | INFO | Downloaded 34GiB to GPU in 2.03853s (16.6787GiB/s) | 20260505 05:52:17.256 | INFO | Analyzing chunk frames = [841216,841728) | 20260505 05:52:17.256 | INFO | MemInfo: { | 20260505 05:52:17.256 | INFO | "gpu" : " 17 GB", | 20260505 05:52:17.256 | INFO | "huge" : "", | 20260505 05:52:17.256 | INFO | "huge_shared" : " 27 GB", | 20260505 05:52:17.256 | INFO | "malloc" : " 14 GB", | 20260505 05:52:17.256 | INFO | "pinned" : " 29 GB" | 20260505 05:52:17.256 | INFO | } | 20260505 05:52:17.257 | INFO | Perf report for the last 5.13962 seconds: | 20260505 05:52:17.257 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:52:17.257 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:17.257 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:17.257 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:17.257 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 05:52:17.257 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:17.257 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:17.257 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 05:52:17.257 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:17.257 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:17.257 | INFO | Uploaded 36GiB to GPU in 2.11006s (17.0612GiB/s) | 20260505 05:52:17.257 | INFO | Downloaded 34GiB to GPU in 2.04377s (16.6359GiB/s) | 20260505 05:52:22.395 | INFO | Analyzing chunk frames = [841728,842240) | 20260505 05:52:22.395 | INFO | MemInfo: { | 20260505 05:52:22.395 | INFO | "gpu" : " 17 GB", | 20260505 05:52:22.395 | INFO | "huge" : "", | 20260505 05:52:22.395 | INFO | "huge_shared" : " 27 GB", | 20260505 05:52:22.395 | INFO | "malloc" : " 14 GB", | 20260505 05:52:22.395 | INFO | "pinned" : " 29 GB" | 20260505 05:52:22.395 | INFO | } | 20260505 05:52:22.395 | INFO | Perf report for the last 5.13882 seconds: | 20260505 05:52:22.395 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:52:22.395 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:22.395 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:22.395 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:22.395 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 05:52:22.395 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:22.395 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:22.395 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 05:52:22.395 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:22.396 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:22.396 | INFO | Uploaded 36GiB to GPU in 2.12441s (16.9459GiB/s) | 20260505 05:52:22.396 | INFO | Downloaded 34GiB to GPU in 2.04665s (16.6125GiB/s) | 20260505 05:52:27.534 | INFO | Analyzing chunk frames = [842240,842752) | 20260505 05:52:27.534 | INFO | MemInfo: { | 20260505 05:52:27.534 | INFO | "gpu" : " 17 GB", | 20260505 05:52:27.535 | INFO | "huge" : "", | 20260505 05:52:27.535 | INFO | "huge_shared" : " 27 GB", | 20260505 05:52:27.535 | INFO | "malloc" : " 14 GB", | 20260505 05:52:27.535 | INFO | "pinned" : " 29 GB" | 20260505 05:52:27.535 | INFO | } | 20260505 05:52:27.535 | INFO | Perf report for the last 5.13929 seconds: | 20260505 05:52:27.535 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:52:27.535 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:27.535 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:27.535 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:27.535 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 05:52:27.535 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:27.535 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:27.535 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:52:27.535 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:27.535 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:27.535 | INFO | Uploaded 36GiB to GPU in 2.14323s (16.7971GiB/s) | 20260505 05:52:27.535 | INFO | Downloaded 34GiB to GPU in 2.06972s (16.4273GiB/s) | 20260505 05:52:32.674 | INFO | Analyzing chunk frames = [842752,843264) | 20260505 05:52:32.674 | INFO | MemInfo: { | 20260505 05:52:32.674 | INFO | "gpu" : " 17 GB", | 20260505 05:52:32.674 | INFO | "huge" : "", | 20260505 05:52:32.674 | INFO | "huge_shared" : " 27 GB", | 20260505 05:52:32.674 | INFO | "malloc" : " 14 GB", | 20260505 05:52:32.674 | INFO | "pinned" : " 29 GB" | 20260505 05:52:32.674 | INFO | } | 20260505 05:52:32.674 | INFO | Perf report for the last 5.13956 seconds: | 20260505 05:52:32.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:52:32.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:32.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:32.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:32.674 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 05:52:32.674 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:32.674 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:32.674 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260505 05:52:32.674 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:32.674 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:32.675 | INFO | Uploaded 36GiB to GPU in 2.17268s (16.5694GiB/s) | 20260505 05:52:32.675 | INFO | Downloaded 34GiB to GPU in 2.15557s (15.7731GiB/s) | 20260505 05:52:37.813 | INFO | Analyzing chunk frames = [843264,843776) | 20260505 05:52:37.813 | INFO | MemInfo: { | 20260505 05:52:37.813 | INFO | "gpu" : " 17 GB", | 20260505 05:52:37.813 | INFO | "huge" : "", | 20260505 05:52:37.813 | INFO | "huge_shared" : " 27 GB", | 20260505 05:52:37.813 | INFO | "malloc" : " 14 GB", | 20260505 05:52:37.813 | INFO | "pinned" : " 29 GB" | 20260505 05:52:37.813 | INFO | } | 20260505 05:52:37.813 | INFO | Perf report for the last 5.13924 seconds: | 20260505 05:52:37.814 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:52:37.814 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:37.814 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:37.814 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:37.814 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 05:52:37.814 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:37.814 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:37.814 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 05:52:37.814 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:37.814 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:37.814 | INFO | Uploaded 36GiB to GPU in 2.07354s (17.3616GiB/s) | 20260505 05:52:37.814 | INFO | Downloaded 34GiB to GPU in 2.08296s (16.323GiB/s) | 20260505 05:52:37.814 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 05:52:37.814 | INFO | Total mem usage: 15.3 GiB | 20260505 05:52:37.814 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 05:52:37.814 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.41%) | 20260505 05:52:37.814 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 05:52:37.814 | INFO | High Water Breakdown: | 20260505 05:52:37.814 | INFO | 60% : 8.65 GiB ( 92.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 05:52:37.814 | INFO | 39.8% : 5.74 GiB ( 86.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 05:52:37.814 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 05:52:37.814 | INFO | ------------------------------------------ | 20260505 05:52:37.815 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:52:37.815 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 05:52:37.815 | INFO | Total mem usage: 26.5 GiB | 20260505 05:52:37.815 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 05:52:37.815 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 05:52:37.815 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 05:52:37.815 | INFO | High Water Breakdown: | 20260505 05:52:37.815 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:52:37.815 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:52:37.815 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:52:37.815 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:52:37.815 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:52:37.815 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:52:37.815 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 05:52:37.815 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 05:52:37.815 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:52:37.815 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 05:52:37.815 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 05:52:37.815 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 05:52:37.815 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 05:52:37.815 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 05:52:37.815 | INFO | ------------------------------------------ | 20260505 05:52:37.815 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:52:37.815 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 05:52:37.815 | INFO | Total mem usage: 25.6 GiB | 20260505 05:52:37.815 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 05:52:37.815 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 05:52:37.815 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 05:52:37.815 | INFO | High Water Breakdown: | 20260505 05:52:37.815 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 05:52:37.815 | INFO | ------------------------------------------ | 20260505 05:52:37.815 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:52:37.815 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 05:52:37.815 | INFO | Total mem usage: 15.7 GiB | 20260505 05:52:37.815 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 05:52:37.815 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 05:52:37.815 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260505 05:52:37.815 | INFO | High Water Breakdown: | 20260505 05:52:37.815 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 05:52:37.815 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 05:52:37.815 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 05:52:37.815 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 05:52:37.815 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 05:52:37.815 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 05:52:37.815 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 05:52:37.815 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 05:52:37.815 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 05:52:37.815 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:52:37.815 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 05:52:37.815 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:52:37.815 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 05:52:37.815 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 05:52:37.815 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 05:52:37.815 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 05:52:37.815 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:52:37.815 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 05:52:37.815 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:52:37.815 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:52:37.815 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:52:37.815 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 05:52:37.815 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 05:52:37.815 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:52:37.815 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 05:52:37.815 | INFO | ------------------------------------------ | 20260505 05:52:37.815 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:52:37.815 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 05:52:37.815 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:52:37.815 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:52:37.815 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.57e+01, 1.39e+01 / 1.79e+00 | 20260505 05:52:37.815 | INFO | PerfReport: 0.28%, AggregateStats -- 9.41e-02 / 1.03e-01, 1.10e-02 / 5.02e-03 | 20260505 05:52:37.815 | INFO | PerfReport: 11.58%, Baseline -- 3.85e+00 / 4.53e+00, 6.32e-01 / 2.49e-01 | 20260505 05:52:37.815 | INFO | PerfReport: 3.31%, Histogram -- 1.10e+00 / 1.51e+00, 2.19e-01 / 1.16e-01 | 20260505 05:52:37.815 | INFO | PerfReport: 15.48%, DME -- 4.11e+01 / 4.50e+01, 4.24e+00 / 1.34e+00 | 20260505 05:52:37.815 | INFO | PerfReport: 55.78%, FrameLabeling -- 1.85e+01 / 1.89e+01, 2.01e+00 / 9.48e-01 | 20260505 05:52:37.815 | INFO | PerfReport: 7.84%, PulseAccumulating -- 2.61e+00 / 3.18e+00, 4.87e-01 / 3.16e-01 | 20260505 05:52:37.815 | INFO | PerfReport: 4.20%, Metrics -- 1.39e+00 / 1.76e+00, 3.89e-01 / 2.37e-01 | 20260505 05:52:37.815 | INFO | PerfReport: 1.54%, PulseSerializing -- 5.12e-01 / 5.89e-01, 1.01e-01 / 4.74e-02 | 20260505 05:52:37.815 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 05:52:37.815 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:52:37.815 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:52:37.815 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.26e+01 / 0.00e+00 | 20260505 05:52:37.815 | INFO | PerfReport: 21.89%, Upload -- 2.22e+01 / 0.00e+00, 7.50e+00 / 0.00e+00 | 20260505 05:52:37.815 | INFO | PerfReport: 21.54%, Download -- 2.19e+01 / 0.00e+00, 7.67e+00 / 0.00e+00 | 20260505 05:52:37.815 | INFO | PerfReport: 32.74%, Compute -- 3.32e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 05:52:37.815 | INFO | PerfReport: 1.44%, LockUpload -- 1.46e+00 / 0.00e+00, 6.76e+00 / 0.00e+00 | 20260505 05:52:37.815 | INFO | PerfReport: 0.70%, LockDownload -- 7.11e-01 / 0.00e+00, 1.92e+00 / 0.00e+00 | 20260505 05:52:37.815 | INFO | PerfReport: 21.67%, LockCompute -- 2.20e+01 / 0.00e+00, 1.68e+01 / 0.00e+00 | 20260505 05:52:37.815 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 1.56e-02 / 0.00e+00 | 20260505 05:52:37.926 | INFO | Starting a round of Baz IO | 20260505 05:52:41.198 | INFO | Populating RTMetrics report | 20260505 05:52:41.198 | INFO | RTMetrics report populated | 20260505 05:52:41.198 | INFO | Opening tmp RTMetrics json file | 20260505 05:52:41.216 | INFO | Serializing RTMetrics to json | 20260505 05:52:41.217 | INFO | Serialization done, swapping tmp file now... | 20260505 05:52:41.217 | INFO | RTMetrics done | 20260505 05:52:41.310 | INFO | Finished a round of Baz IO: Wrote 2.309 GiB in 3.383 seconds | 20260505 05:52:41.310 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.14% / 0.06% / 63.80% | 20260505 05:52:41.310 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.36087% / 0.000147025% / 3.57628e-05% | 20260505 05:52:41.310 | INFO | Baz IO overall: | 20260505 05:52:41.311 | INFO | BytesWritten : 414864780546 | 20260505 05:52:41.311 | INFO | NumEvents : 195486824287 | 20260505 05:52:42.953 | INFO | Analyzing chunk frames = [843776,844288) | 20260505 05:52:42.953 | INFO | MemInfo: { | 20260505 05:52:42.953 | INFO | "gpu" : " 17 GB", | 20260505 05:52:42.953 | INFO | "huge" : "", | 20260505 05:52:42.953 | INFO | "huge_shared" : " 27 GB", | 20260505 05:52:42.953 | INFO | "malloc" : " 14 GB", | 20260505 05:52:42.953 | INFO | "pinned" : " 29 GB" | 20260505 05:52:42.953 | INFO | } | 20260505 05:52:42.953 | INFO | Perf report for the last 5.13926 seconds: | 20260505 05:52:42.953 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:52:42.953 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:42.953 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:42.953 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:42.953 | INFO | ANALYSIS : 64.7%, 2.9, 99.4ms, 0 | 20260505 05:52:42.953 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:42.953 | INFO | RT_METRICS : 46.2%, 1.0, 24.2ms, 0 | 20260505 05:52:42.953 | INFO | PRE_HQ : 63.9%, 1.9, 64.6ms, 0 | 20260505 05:52:42.953 | INFO | BAZWRITER : 65.1%, 4.0, 134.2ms, 0 | 20260505 05:52:42.953 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 05:52:42.953 | INFO | Uploaded 36GiB to GPU in 2.16033s (16.6641GiB/s) | 20260505 05:52:42.953 | INFO | Downloaded 38GiB to GPU in 2.33958s (16.2422GiB/s) | 20260505 05:52:48.092 | INFO | Analyzing chunk frames = [844288,844800) | 20260505 05:52:48.092 | INFO | MemInfo: { | 20260505 05:52:48.092 | INFO | "gpu" : " 17 GB", | 20260505 05:52:48.092 | INFO | "huge" : "", | 20260505 05:52:48.092 | INFO | "huge_shared" : " 27 GB", | 20260505 05:52:48.092 | INFO | "malloc" : " 14 GB", | 20260505 05:52:48.092 | INFO | "pinned" : " 29 GB" | 20260505 05:52:48.092 | INFO | } | 20260505 05:52:48.092 | INFO | Perf report for the last 5.13962 seconds: | 20260505 05:52:48.092 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:52:48.092 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:48.092 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:48.092 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:48.092 | INFO | ANALYSIS : 64.6%, 3.0, 100.8ms, 0 | 20260505 05:52:48.092 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:48.093 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:48.093 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:52:48.093 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:48.093 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:48.093 | INFO | Uploaded 36GiB to GPU in 2.13346s (16.874GiB/s) | 20260505 05:52:48.093 | INFO | Downloaded 34GiB to GPU in 2.04167s (16.653GiB/s) | 20260505 05:52:53.231 | INFO | Analyzing chunk frames = [844800,845312) | 20260505 05:52:53.232 | INFO | MemInfo: { | 20260505 05:52:53.232 | INFO | "gpu" : " 17 GB", | 20260505 05:52:53.232 | INFO | "huge" : "", | 20260505 05:52:53.232 | INFO | "huge_shared" : " 27 GB", | 20260505 05:52:53.232 | INFO | "malloc" : " 14 GB", | 20260505 05:52:53.232 | INFO | "pinned" : " 29 GB" | 20260505 05:52:53.232 | INFO | } | 20260505 05:52:53.232 | INFO | Perf report for the last 5.13919 seconds: | 20260505 05:52:53.232 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:52:53.232 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:53.232 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:53.232 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:53.232 | INFO | ANALYSIS : 64.3%, 3.0, 100.1ms, 0 | 20260505 05:52:53.232 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:53.232 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:53.232 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 05:52:53.232 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:53.232 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:53.232 | INFO | Uploaded 36GiB to GPU in 2.17177s (16.5763GiB/s) | 20260505 05:52:53.232 | INFO | Downloaded 34GiB to GPU in 2.0403s (16.6642GiB/s) | 20260505 05:52:58.371 | INFO | Analyzing chunk frames = [845312,845824) | 20260505 05:52:58.371 | INFO | MemInfo: { | 20260505 05:52:58.371 | INFO | "gpu" : " 17 GB", | 20260505 05:52:58.371 | INFO | "huge" : "", | 20260505 05:52:58.371 | INFO | "huge_shared" : " 27 GB", | 20260505 05:52:58.371 | INFO | "malloc" : " 14 GB", | 20260505 05:52:58.371 | INFO | "pinned" : " 29 GB" | 20260505 05:52:58.371 | INFO | } | 20260505 05:52:58.371 | INFO | Perf report for the last 5.1394 seconds: | 20260505 05:52:58.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:52:58.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:58.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:58.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:58.371 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260505 05:52:58.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:58.371 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:58.371 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260505 05:52:58.371 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:58.371 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:52:58.371 | INFO | Uploaded 36GiB to GPU in 2.09776s (17.1612GiB/s) | 20260505 05:52:58.371 | INFO | Downloaded 34GiB to GPU in 2.02364s (16.8014GiB/s) | 20260505 05:53:03.510 | INFO | Analyzing chunk frames = [845824,846336) | 20260505 05:53:03.510 | INFO | MemInfo: { | 20260505 05:53:03.510 | INFO | "gpu" : " 17 GB", | 20260505 05:53:03.510 | INFO | "huge" : "", | 20260505 05:53:03.510 | INFO | "huge_shared" : " 27 GB", | 20260505 05:53:03.510 | INFO | "malloc" : " 14 GB", | 20260505 05:53:03.510 | INFO | "pinned" : " 29 GB" | 20260505 05:53:03.510 | INFO | } | 20260505 05:53:03.510 | INFO | Perf report for the last 5.13914 seconds: | 20260505 05:53:03.510 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:53:03.510 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:03.510 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:03.510 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:03.510 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 05:53:03.510 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:03.510 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:03.510 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260505 05:53:03.510 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:03.510 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:03.510 | INFO | Uploaded 36GiB to GPU in 2.12138s (16.9701GiB/s) | 20260505 05:53:03.511 | INFO | Downloaded 34GiB to GPU in 2.05375s (16.5551GiB/s) | 20260505 05:53:08.650 | INFO | Analyzing chunk frames = [846336,846848) | 20260505 05:53:08.650 | INFO | MemInfo: { | 20260505 05:53:08.650 | INFO | "gpu" : " 17 GB", | 20260505 05:53:08.650 | INFO | "huge" : "", | 20260505 05:53:08.650 | INFO | "huge_shared" : " 27 GB", | 20260505 05:53:08.650 | INFO | "malloc" : " 14 GB", | 20260505 05:53:08.650 | INFO | "pinned" : " 29 GB" | 20260505 05:53:08.650 | INFO | } | 20260505 05:53:08.650 | INFO | Perf report for the last 5.14019 seconds: | 20260505 05:53:08.650 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:53:08.650 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:08.650 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:08.650 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:08.650 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 05:53:08.650 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:08.650 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:08.651 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 05:53:08.651 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:08.651 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:08.651 | INFO | Uploaded 36GiB to GPU in 2.13285s (16.8788GiB/s) | 20260505 05:53:08.651 | INFO | Downloaded 34GiB to GPU in 2.06228s (16.4866GiB/s) | 20260505 05:53:13.789 | INFO | Analyzing chunk frames = [846848,847360) | 20260505 05:53:13.789 | INFO | MemInfo: { | 20260505 05:53:13.789 | INFO | "gpu" : " 17 GB", | 20260505 05:53:13.789 | INFO | "huge" : "", | 20260505 05:53:13.789 | INFO | "huge_shared" : " 27 GB", | 20260505 05:53:13.789 | INFO | "malloc" : " 14 GB", | 20260505 05:53:13.789 | INFO | "pinned" : " 29 GB" | 20260505 05:53:13.789 | INFO | } | 20260505 05:53:13.789 | INFO | Perf report for the last 5.13854 seconds: | 20260505 05:53:13.789 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:53:13.789 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:13.789 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:13.789 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:13.789 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 05:53:13.789 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:13.789 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:13.789 | INFO | PRE_HQ : 32.4%, 1.0, 16.9ms, 0 | 20260505 05:53:13.789 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:13.789 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:13.789 | INFO | Uploaded 36GiB to GPU in 2.16277s (16.6453GiB/s) | 20260505 05:53:13.789 | INFO | Downloaded 34GiB to GPU in 2.16367s (15.714GiB/s) | 20260505 05:53:18.928 | INFO | Analyzing chunk frames = [847360,847872) | 20260505 05:53:18.928 | INFO | MemInfo: { | 20260505 05:53:18.928 | INFO | "gpu" : " 17 GB", | 20260505 05:53:18.928 | INFO | "huge" : "", | 20260505 05:53:18.928 | INFO | "huge_shared" : " 27 GB", | 20260505 05:53:18.928 | INFO | "malloc" : " 14 GB", | 20260505 05:53:18.928 | INFO | "pinned" : " 29 GB" | 20260505 05:53:18.928 | INFO | } | 20260505 05:53:18.929 | INFO | Perf report for the last 5.13981 seconds: | 20260505 05:53:18.929 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:53:18.929 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:18.929 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:18.929 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:18.929 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 05:53:18.929 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:18.929 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:18.929 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 05:53:18.929 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:18.929 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:18.929 | INFO | Uploaded 36GiB to GPU in 2.07221s (17.3728GiB/s) | 20260505 05:53:18.929 | INFO | Downloaded 34GiB to GPU in 2.08987s (16.269GiB/s) | 20260505 05:53:19.042 | INFO | Starting a round of Baz IO | 20260505 05:53:22.309 | INFO | Populating RTMetrics report | 20260505 05:53:22.309 | INFO | RTMetrics report populated | 20260505 05:53:22.309 | INFO | Opening tmp RTMetrics json file | 20260505 05:53:22.331 | INFO | Serializing RTMetrics to json | 20260505 05:53:22.336 | INFO | Serialization done, swapping tmp file now... | 20260505 05:53:22.338 | INFO | RTMetrics done | 20260505 05:53:22.422 | INFO | Finished a round of Baz IO: Wrote 2.308 GiB in 3.38 seconds | 20260505 05:53:22.422 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.07% / 0.06% / 63.86% | 20260505 05:53:22.422 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.31607% / 9.13938e-05% / 5.5631e-05% | 20260505 05:53:22.422 | INFO | Baz IO overall: | 20260505 05:53:22.422 | INFO | BytesWritten : 417343361357 | 20260505 05:53:22.422 | INFO | NumEvents : 196670566031 | 20260505 05:53:24.067 | INFO | Analyzing chunk frames = [847872,848384) | 20260505 05:53:24.067 | INFO | MemInfo: { | 20260505 05:53:24.067 | INFO | "gpu" : " 17 GB", | 20260505 05:53:24.067 | INFO | "huge" : "", | 20260505 05:53:24.067 | INFO | "huge_shared" : " 27 GB", | 20260505 05:53:24.067 | INFO | "malloc" : " 14 GB", | 20260505 05:53:24.067 | INFO | "pinned" : " 29 GB" | 20260505 05:53:24.068 | INFO | } | 20260505 05:53:24.068 | INFO | Perf report for the last 5.13882 seconds: | 20260505 05:53:24.068 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:53:24.068 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:24.068 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:24.068 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:24.068 | INFO | ANALYSIS : 64.6%, 2.9, 99.3ms, 0 | 20260505 05:53:24.068 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:24.068 | INFO | RT_METRICS : 46.6%, 1.0, 24.4ms, 0 | 20260505 05:53:24.068 | INFO | PRE_HQ : 63.8%, 2.0, 64.8ms, 0 | 20260505 05:53:24.068 | INFO | BAZWRITER : 65.1%, 4.0, 133.8ms, 0 | 20260505 05:53:24.068 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260505 05:53:24.068 | INFO | Uploaded 36GiB to GPU in 2.15946s (16.6708GiB/s) | 20260505 05:53:24.068 | INFO | Downloaded 38GiB to GPU in 2.3264s (16.3343GiB/s) | 20260505 05:53:29.207 | INFO | Analyzing chunk frames = [848384,848896) | 20260505 05:53:29.207 | INFO | MemInfo: { | 20260505 05:53:29.207 | INFO | "gpu" : " 17 GB", | 20260505 05:53:29.207 | INFO | "huge" : "", | 20260505 05:53:29.207 | INFO | "huge_shared" : " 27 GB", | 20260505 05:53:29.207 | INFO | "malloc" : " 14 GB", | 20260505 05:53:29.207 | INFO | "pinned" : " 29 GB" | 20260505 05:53:29.207 | INFO | } | 20260505 05:53:29.207 | INFO | Perf report for the last 5.13963 seconds: | 20260505 05:53:29.207 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:53:29.207 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:29.207 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:29.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:29.207 | INFO | ANALYSIS : 64.6%, 3.0, 100.8ms, 0 | 20260505 05:53:29.207 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:29.207 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:29.207 | INFO | PRE_HQ : 30.5%, 1.0, 15.9ms, 0 | 20260505 05:53:29.207 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:29.207 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:29.208 | INFO | Uploaded 36GiB to GPU in 2.13198s (16.8857GiB/s) | 20260505 05:53:29.208 | INFO | Downloaded 34GiB to GPU in 2.03848s (16.6791GiB/s) | 20260505 05:53:34.346 | INFO | Analyzing chunk frames = [848896,849408) | 20260505 05:53:34.346 | INFO | MemInfo: { | 20260505 05:53:34.346 | INFO | "gpu" : " 17 GB", | 20260505 05:53:34.346 | INFO | "huge" : "", | 20260505 05:53:34.346 | INFO | "huge_shared" : " 27 GB", | 20260505 05:53:34.346 | INFO | "malloc" : " 14 GB", | 20260505 05:53:34.346 | INFO | "pinned" : " 29 GB" | 20260505 05:53:34.346 | INFO | } | 20260505 05:53:34.346 | INFO | Perf report for the last 5.13913 seconds: | 20260505 05:53:34.346 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:53:34.346 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:34.346 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:34.346 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:34.346 | INFO | ANALYSIS : 64.3%, 3.0, 100.1ms, 0 | 20260505 05:53:34.346 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:34.346 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:34.346 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:53:34.347 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:34.347 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:34.347 | INFO | Uploaded 36GiB to GPU in 2.17948s (16.5177GiB/s) | 20260505 05:53:34.347 | INFO | Downloaded 34GiB to GPU in 2.04642s (16.6144GiB/s) | 20260505 05:53:39.486 | INFO | Analyzing chunk frames = [849408,849920) | 20260505 05:53:39.486 | INFO | MemInfo: { | 20260505 05:53:39.486 | INFO | "gpu" : " 17 GB", | 20260505 05:53:39.486 | INFO | "huge" : "", | 20260505 05:53:39.486 | INFO | "huge_shared" : " 27 GB", | 20260505 05:53:39.486 | INFO | "malloc" : " 14 GB", | 20260505 05:53:39.486 | INFO | "pinned" : " 29 GB" | 20260505 05:53:39.486 | INFO | } | 20260505 05:53:39.486 | INFO | Perf report for the last 5.13997 seconds: | 20260505 05:53:39.486 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:53:39.486 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:39.486 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:39.486 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:39.486 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260505 05:53:39.486 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:39.486 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:39.486 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:53:39.486 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:39.487 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:39.487 | INFO | Uploaded 36GiB to GPU in 2.08509s (17.2654GiB/s) | 20260505 05:53:39.487 | INFO | Downloaded 34GiB to GPU in 2.02112s (16.8224GiB/s) | 20260505 05:53:44.625 | INFO | Analyzing chunk frames = [849920,850432) | 20260505 05:53:44.625 | INFO | MemInfo: { | 20260505 05:53:44.625 | INFO | "gpu" : " 17 GB", | 20260505 05:53:44.625 | INFO | "huge" : "", | 20260505 05:53:44.625 | INFO | "huge_shared" : " 27 GB", | 20260505 05:53:44.625 | INFO | "malloc" : " 14 GB", | 20260505 05:53:44.625 | INFO | "pinned" : " 29 GB" | 20260505 05:53:44.625 | INFO | } | 20260505 05:53:44.625 | INFO | Perf report for the last 5.1389 seconds: | 20260505 05:53:44.625 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:53:44.625 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:44.625 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:44.625 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:44.625 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 05:53:44.625 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:44.625 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:44.625 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260505 05:53:44.625 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:44.625 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:44.626 | INFO | Uploaded 36GiB to GPU in 2.12879s (16.911GiB/s) | 20260505 05:53:44.626 | INFO | Downloaded 34GiB to GPU in 2.04317s (16.6408GiB/s) | 20260505 05:53:49.764 | INFO | Analyzing chunk frames = [850432,850944) | 20260505 05:53:49.764 | INFO | MemInfo: { | 20260505 05:53:49.764 | INFO | "gpu" : " 17 GB", | 20260505 05:53:49.764 | INFO | "huge" : "", | 20260505 05:53:49.764 | INFO | "huge_shared" : " 27 GB", | 20260505 05:53:49.764 | INFO | "malloc" : " 14 GB", | 20260505 05:53:49.764 | INFO | "pinned" : " 29 GB" | 20260505 05:53:49.764 | INFO | } | 20260505 05:53:49.765 | INFO | Perf report for the last 5.13926 seconds: | 20260505 05:53:49.765 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:53:49.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:49.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:49.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:49.765 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 05:53:49.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:49.765 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:49.765 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 05:53:49.765 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:49.765 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:49.765 | INFO | Uploaded 36GiB to GPU in 2.13178s (16.8873GiB/s) | 20260505 05:53:49.765 | INFO | Downloaded 34GiB to GPU in 2.05839s (16.5177GiB/s) | 20260505 05:53:54.904 | INFO | Analyzing chunk frames = [850944,851456) | 20260505 05:53:54.904 | INFO | MemInfo: { | 20260505 05:53:54.904 | INFO | "gpu" : " 17 GB", | 20260505 05:53:54.904 | INFO | "huge" : "", | 20260505 05:53:54.904 | INFO | "huge_shared" : " 27 GB", | 20260505 05:53:54.904 | INFO | "malloc" : " 14 GB", | 20260505 05:53:54.904 | INFO | "pinned" : " 29 GB" | 20260505 05:53:54.904 | INFO | } | 20260505 05:53:54.904 | INFO | Perf report for the last 5.13952 seconds: | 20260505 05:53:54.904 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:53:54.904 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:54.904 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:54.904 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:54.904 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 05:53:54.904 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:54.904 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:54.904 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 05:53:54.904 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:54.904 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:53:54.904 | INFO | Uploaded 36GiB to GPU in 2.16406s (16.6354GiB/s) | 20260505 05:53:54.904 | INFO | Downloaded 34GiB to GPU in 2.16668s (15.6922GiB/s) | 20260505 05:54:00.043 | INFO | Analyzing chunk frames = [851456,851968) | 20260505 05:54:00.043 | INFO | MemInfo: { | 20260505 05:54:00.043 | INFO | "gpu" : " 17 GB", | 20260505 05:54:00.043 | INFO | "huge" : "", | 20260505 05:54:00.043 | INFO | "huge_shared" : " 27 GB", | 20260505 05:54:00.043 | INFO | "malloc" : " 14 GB", | 20260505 05:54:00.043 | INFO | "pinned" : " 29 GB" | 20260505 05:54:00.043 | INFO | } | 20260505 05:54:00.043 | INFO | Perf report for the last 5.139 seconds: | 20260505 05:54:00.043 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:54:00.043 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:00.043 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:00.043 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:00.043 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 05:54:00.043 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:00.043 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:00.043 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260505 05:54:00.043 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:00.043 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:00.043 | INFO | Uploaded 36GiB to GPU in 2.0752s (17.3478GiB/s) | 20260505 05:54:00.043 | INFO | Downloaded 34GiB to GPU in 2.08411s (16.3139GiB/s) | 20260505 05:54:00.044 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 05:54:00.044 | INFO | Total mem usage: 15.3 GiB | 20260505 05:54:00.044 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 05:54:00.044 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.41%) | 20260505 05:54:00.044 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 05:54:00.044 | INFO | High Water Breakdown: | 20260505 05:54:00.044 | INFO | 60.1% : 8.65 GiB ( 92.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 05:54:00.044 | INFO | 39.8% : 5.73 GiB ( 86.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 05:54:00.044 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 05:54:00.044 | INFO | ------------------------------------------ | 20260505 05:54:00.044 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:54:00.044 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 05:54:00.044 | INFO | Total mem usage: 26.5 GiB | 20260505 05:54:00.044 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 05:54:00.044 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 05:54:00.044 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 05:54:00.044 | INFO | High Water Breakdown: | 20260505 05:54:00.044 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:54:00.044 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:54:00.044 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:54:00.044 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:54:00.044 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:54:00.044 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:54:00.044 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 05:54:00.044 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 05:54:00.044 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:54:00.044 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 05:54:00.044 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 05:54:00.044 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 05:54:00.044 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 05:54:00.044 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 05:54:00.044 | INFO | ------------------------------------------ | 20260505 05:54:00.044 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:54:00.044 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 05:54:00.044 | INFO | Total mem usage: 25.6 GiB | 20260505 05:54:00.044 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 05:54:00.044 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 05:54:00.044 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 05:54:00.044 | INFO | High Water Breakdown: | 20260505 05:54:00.044 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 05:54:00.044 | INFO | ------------------------------------------ | 20260505 05:54:00.044 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:54:00.044 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 05:54:00.044 | INFO | Total mem usage: 15.7 GiB | 20260505 05:54:00.044 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 05:54:00.044 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 05:54:00.044 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260505 05:54:00.044 | INFO | High Water Breakdown: | 20260505 05:54:00.044 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 05:54:00.044 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 05:54:00.044 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 05:54:00.044 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 05:54:00.044 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 05:54:00.044 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 05:54:00.044 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 05:54:00.044 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 05:54:00.044 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 05:54:00.044 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:54:00.044 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 05:54:00.044 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:54:00.044 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 05:54:00.044 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 05:54:00.044 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 05:54:00.044 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 05:54:00.044 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:54:00.044 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 05:54:00.044 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:54:00.044 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:54:00.044 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:54:00.044 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 05:54:00.044 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 05:54:00.044 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:54:00.044 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 05:54:00.044 | INFO | ------------------------------------------ | 20260505 05:54:00.044 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:54:00.045 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 05:54:00.045 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:54:00.045 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:54:00.045 | INFO | PerfReport: Full Execution -- 3.33e+01 / 7.60e+01, 1.39e+01 / 1.62e+00 | 20260505 05:54:00.045 | INFO | PerfReport: 0.28%, AggregateStats -- 9.43e-02 / 1.03e-01, 1.10e-02 / 6.01e-03 | 20260505 05:54:00.045 | INFO | PerfReport: 11.55%, Baseline -- 3.84e+00 / 4.54e+00, 6.33e-01 / 2.63e-01 | 20260505 05:54:00.045 | INFO | PerfReport: 3.31%, Histogram -- 1.10e+00 / 1.51e+00, 2.19e-01 / 1.20e-01 | 20260505 05:54:00.045 | INFO | PerfReport: 15.50%, DME -- 4.13e+01 / 4.52e+01, 4.44e+00 / 1.26e+00 | 20260505 05:54:00.045 | INFO | PerfReport: 55.79%, FrameLabeling -- 1.86e+01 / 1.91e+01, 2.04e+00 / 9.76e-01 | 20260505 05:54:00.045 | INFO | PerfReport: 7.84%, PulseAccumulating -- 2.61e+00 / 3.17e+00, 4.96e-01 / 3.39e-01 | 20260505 05:54:00.045 | INFO | PerfReport: 4.18%, Metrics -- 1.39e+00 / 1.76e+00, 3.89e-01 / 2.13e-01 | 20260505 05:54:00.045 | INFO | PerfReport: 1.53%, PulseSerializing -- 5.10e-01 / 5.91e-01, 1.01e-01 / 5.01e-02 | 20260505 05:54:00.045 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 05:54:00.045 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:54:00.045 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:54:00.045 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.28e+01 / 0.00e+00 | 20260505 05:54:00.045 | INFO | PerfReport: 21.86%, Upload -- 2.22e+01 / 0.00e+00, 7.54e+00 / 0.00e+00 | 20260505 05:54:00.045 | INFO | PerfReport: 21.54%, Download -- 2.19e+01 / 0.00e+00, 7.71e+00 / 0.00e+00 | 20260505 05:54:00.045 | INFO | PerfReport: 32.75%, Compute -- 3.33e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 05:54:00.045 | INFO | PerfReport: 1.44%, LockUpload -- 1.46e+00 / 0.00e+00, 6.72e+00 / 0.00e+00 | 20260505 05:54:00.045 | INFO | PerfReport: 0.68%, LockDownload -- 6.94e-01 / 0.00e+00, 1.92e+00 / 0.00e+00 | 20260505 05:54:00.045 | INFO | PerfReport: 21.71%, LockCompute -- 2.21e+01 / 0.00e+00, 1.67e+01 / 0.00e+00 | 20260505 05:54:00.045 | INFO | PerfReport: 0.01%, Misc -- 1.49e-02 / 0.00e+00, 2.18e-02 / 0.00e+00 | 20260505 05:54:00.158 | INFO | Starting a round of Baz IO | 20260505 05:54:03.417 | INFO | Populating RTMetrics report | 20260505 05:54:03.417 | INFO | RTMetrics report populated | 20260505 05:54:03.417 | INFO | Opening tmp RTMetrics json file | 20260505 05:54:03.430 | INFO | Serializing RTMetrics to json | 20260505 05:54:03.431 | INFO | Serialization done, swapping tmp file now... | 20260505 05:54:03.431 | INFO | RTMetrics done | 20260505 05:54:03.529 | INFO | Finished a round of Baz IO: Wrote 2.311 GiB in 3.371 seconds | 20260505 05:54:03.529 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.01% / 0.06% / 63.93% | 20260505 05:54:03.529 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.29061% / 0.000119209% / 5.96046e-05% | 20260505 05:54:03.529 | INFO | Baz IO overall: | 20260505 05:54:03.529 | INFO | BytesWritten : 419824681782 | 20260505 05:54:03.530 | INFO | NumEvents : 197855579454 | 20260505 05:54:05.182 | INFO | Analyzing chunk frames = [851968,852480) | 20260505 05:54:05.182 | INFO | MemInfo: { | 20260505 05:54:05.182 | INFO | "gpu" : " 17 GB", | 20260505 05:54:05.182 | INFO | "huge" : "", | 20260505 05:54:05.182 | INFO | "huge_shared" : " 27 GB", | 20260505 05:54:05.182 | INFO | "malloc" : " 14 GB", | 20260505 05:54:05.182 | INFO | "pinned" : " 29 GB" | 20260505 05:54:05.182 | INFO | } | 20260505 05:54:05.182 | INFO | Perf report for the last 5.13909 seconds: | 20260505 05:54:05.183 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:54:05.183 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:05.183 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:05.183 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:05.183 | INFO | ANALYSIS : 64.5%, 2.9, 99.8ms, 0 | 20260505 05:54:05.183 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:05.183 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260505 05:54:05.183 | INFO | PRE_HQ : 63.5%, 2.0, 64.6ms, 0 | 20260505 05:54:05.183 | INFO | BAZWRITER : 64.9%, 4.0, 132.8ms, 0 | 20260505 05:54:05.183 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 05:54:05.183 | INFO | Uploaded 36GiB to GPU in 2.11765s (17GiB/s) | 20260505 05:54:05.183 | INFO | Downloaded 38GiB to GPU in 2.31739s (16.3978GiB/s) | 20260505 05:54:10.322 | INFO | Analyzing chunk frames = [852480,852992) | 20260505 05:54:10.322 | INFO | MemInfo: { | 20260505 05:54:10.322 | INFO | "gpu" : " 17 GB", | 20260505 05:54:10.322 | INFO | "huge" : "", | 20260505 05:54:10.322 | INFO | "huge_shared" : " 27 GB", | 20260505 05:54:10.322 | INFO | "malloc" : " 14 GB", | 20260505 05:54:10.322 | INFO | "pinned" : " 29 GB" | 20260505 05:54:10.322 | INFO | } | 20260505 05:54:10.322 | INFO | Perf report for the last 5.13946 seconds: | 20260505 05:54:10.322 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:54:10.322 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:10.322 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:10.322 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:10.322 | INFO | ANALYSIS : 64.8%, 3.0, 101.2ms, 0 | 20260505 05:54:10.322 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:10.322 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:10.322 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 05:54:10.322 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:10.322 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:10.322 | INFO | Uploaded 36GiB to GPU in 2.14478s (16.7849GiB/s) | 20260505 05:54:10.322 | INFO | Downloaded 34GiB to GPU in 2.0463s (16.6153GiB/s) | 20260505 05:54:15.461 | INFO | Analyzing chunk frames = [852992,853504) | 20260505 05:54:15.461 | INFO | MemInfo: { | 20260505 05:54:15.461 | INFO | "gpu" : " 17 GB", | 20260505 05:54:15.461 | INFO | "huge" : "", | 20260505 05:54:15.461 | INFO | "huge_shared" : " 27 GB", | 20260505 05:54:15.461 | INFO | "malloc" : " 14 GB", | 20260505 05:54:15.461 | INFO | "pinned" : " 29 GB" | 20260505 05:54:15.461 | INFO | } | 20260505 05:54:15.461 | INFO | Perf report for the last 5.13905 seconds: | 20260505 05:54:15.461 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:54:15.461 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:15.461 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:15.461 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:15.461 | INFO | ANALYSIS : 64.4%, 3.0, 100.2ms, 0 | 20260505 05:54:15.461 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:15.461 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:15.461 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 05:54:15.461 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:15.461 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:15.461 | INFO | Uploaded 36GiB to GPU in 2.16837s (16.6023GiB/s) | 20260505 05:54:15.461 | INFO | Downloaded 34GiB to GPU in 2.03202s (16.7321GiB/s) | 20260505 05:54:20.600 | INFO | Analyzing chunk frames = [853504,854016) | 20260505 05:54:20.600 | INFO | MemInfo: { | 20260505 05:54:20.600 | INFO | "gpu" : " 17 GB", | 20260505 05:54:20.600 | INFO | "huge" : "", | 20260505 05:54:20.600 | INFO | "huge_shared" : " 27 GB", | 20260505 05:54:20.601 | INFO | "malloc" : " 14 GB", | 20260505 05:54:20.601 | INFO | "pinned" : " 29 GB" | 20260505 05:54:20.601 | INFO | } | 20260505 05:54:20.601 | INFO | Perf report for the last 5.13983 seconds: | 20260505 05:54:20.601 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:54:20.601 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:20.601 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:20.601 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:20.601 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260505 05:54:20.601 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:20.601 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:20.601 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 05:54:20.601 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:20.601 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:20.601 | INFO | Uploaded 36GiB to GPU in 2.09359s (17.1954GiB/s) | 20260505 05:54:20.601 | INFO | Downloaded 34GiB to GPU in 2.01977s (16.8336GiB/s) | 20260505 05:54:25.739 | INFO | Analyzing chunk frames = [854016,854528) | 20260505 05:54:25.739 | INFO | MemInfo: { | 20260505 05:54:25.739 | INFO | "gpu" : " 17 GB", | 20260505 05:54:25.739 | INFO | "huge" : "", | 20260505 05:54:25.739 | INFO | "huge_shared" : " 27 GB", | 20260505 05:54:25.739 | INFO | "malloc" : " 14 GB", | 20260505 05:54:25.739 | INFO | "pinned" : " 29 GB" | 20260505 05:54:25.740 | INFO | } | 20260505 05:54:25.740 | INFO | Perf report for the last 5.13915 seconds: | 20260505 05:54:25.740 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:54:25.740 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:25.740 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:25.740 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:25.740 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 05:54:25.740 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:25.740 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:25.740 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 05:54:25.740 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:25.740 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:25.740 | INFO | Uploaded 36GiB to GPU in 2.13046s (16.8977GiB/s) | 20260505 05:54:25.740 | INFO | Downloaded 34GiB to GPU in 2.04487s (16.627GiB/s) | 20260505 05:54:30.879 | INFO | Analyzing chunk frames = [854528,855040) | 20260505 05:54:30.879 | INFO | MemInfo: { | 20260505 05:54:30.879 | INFO | "gpu" : " 17 GB", | 20260505 05:54:30.879 | INFO | "huge" : "", | 20260505 05:54:30.879 | INFO | "huge_shared" : " 27 GB", | 20260505 05:54:30.879 | INFO | "malloc" : " 14 GB", | 20260505 05:54:30.880 | INFO | "pinned" : " 29 GB" | 20260505 05:54:30.880 | INFO | } | 20260505 05:54:30.880 | INFO | Perf report for the last 5.13995 seconds: | 20260505 05:54:30.880 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:54:30.880 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:30.880 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:30.880 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:30.880 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 05:54:30.880 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:30.880 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:30.880 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 05:54:30.880 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:30.880 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:30.880 | INFO | Uploaded 36GiB to GPU in 2.13995s (16.8228GiB/s) | 20260505 05:54:30.880 | INFO | Downloaded 34GiB to GPU in 2.05982s (16.5063GiB/s) | 20260505 05:54:36.018 | INFO | Analyzing chunk frames = [855040,855552) | 20260505 05:54:36.018 | INFO | MemInfo: { | 20260505 05:54:36.018 | INFO | "gpu" : " 17 GB", | 20260505 05:54:36.018 | INFO | "huge" : "", | 20260505 05:54:36.018 | INFO | "huge_shared" : " 27 GB", | 20260505 05:54:36.018 | INFO | "malloc" : " 14 GB", | 20260505 05:54:36.018 | INFO | "pinned" : " 29 GB" | 20260505 05:54:36.018 | INFO | } | 20260505 05:54:36.018 | INFO | Perf report for the last 5.13876 seconds: | 20260505 05:54:36.018 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:54:36.018 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:36.018 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:36.019 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:36.019 | INFO | ANALYSIS : 64.1%, 3.0, 100.0ms, 0 | 20260505 05:54:36.019 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:36.019 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:36.019 | INFO | PRE_HQ : 32.6%, 1.0, 17.0ms, 0 | 20260505 05:54:36.019 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:36.019 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:36.019 | INFO | Uploaded 36GiB to GPU in 2.17887s (16.5223GiB/s) | 20260505 05:54:36.019 | INFO | Downloaded 34GiB to GPU in 2.16313s (15.718GiB/s) | 20260505 05:54:41.158 | INFO | Analyzing chunk frames = [855552,856064) | 20260505 05:54:41.158 | INFO | MemInfo: { | 20260505 05:54:41.158 | INFO | "gpu" : " 17 GB", | 20260505 05:54:41.158 | INFO | "huge" : "", | 20260505 05:54:41.158 | INFO | "huge_shared" : " 27 GB", | 20260505 05:54:41.158 | INFO | "malloc" : " 14 GB", | 20260505 05:54:41.158 | INFO | "pinned" : " 29 GB" | 20260505 05:54:41.158 | INFO | } | 20260505 05:54:41.158 | INFO | Perf report for the last 5.1399 seconds: | 20260505 05:54:41.158 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:54:41.158 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:41.158 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:41.158 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:41.158 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 05:54:41.158 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:41.158 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:41.158 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260505 05:54:41.158 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:41.159 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:41.159 | INFO | Uploaded 36GiB to GPU in 2.06057s (17.4709GiB/s) | 20260505 05:54:41.159 | INFO | Downloaded 34GiB to GPU in 2.07989s (16.3471GiB/s) | 20260505 05:54:41.272 | INFO | Starting a round of Baz IO | 20260505 05:54:44.540 | INFO | Populating RTMetrics report | 20260505 05:54:44.540 | INFO | RTMetrics report populated | 20260505 05:54:44.540 | INFO | Opening tmp RTMetrics json file | 20260505 05:54:44.561 | INFO | Serializing RTMetrics to json | 20260505 05:54:44.562 | INFO | Serialization done, swapping tmp file now... | 20260505 05:54:44.563 | INFO | RTMetrics done | 20260505 05:54:44.654 | INFO | Finished a round of Baz IO: Wrote 2.314 GiB in 3.382 seconds | 20260505 05:54:44.654 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.95% / 0.06% / 63.99% | 20260505 05:54:44.654 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.2724% / 0.000135104% / 5.5631e-05% | 20260505 05:54:44.655 | INFO | Baz IO overall: | 20260505 05:54:44.655 | INFO | BytesWritten : 422309455271 | 20260505 05:54:44.655 | INFO | NumEvents : 199042631583 | 20260505 05:54:46.297 | INFO | Analyzing chunk frames = [856064,856576) | 20260505 05:54:46.297 | INFO | MemInfo: { | 20260505 05:54:46.297 | INFO | "gpu" : " 17 GB", | 20260505 05:54:46.297 | INFO | "huge" : "", | 20260505 05:54:46.297 | INFO | "huge_shared" : " 27 GB", | 20260505 05:54:46.297 | INFO | "malloc" : " 14 GB", | 20260505 05:54:46.297 | INFO | "pinned" : " 29 GB" | 20260505 05:54:46.297 | INFO | } | 20260505 05:54:46.297 | INFO | Perf report for the last 5.13881 seconds: | 20260505 05:54:46.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:54:46.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:46.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:46.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:46.297 | INFO | ANALYSIS : 64.7%, 2.9, 99.2ms, 0 | 20260505 05:54:46.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:46.297 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260505 05:54:46.297 | INFO | PRE_HQ : 63.8%, 2.0, 64.7ms, 0 | 20260505 05:54:46.297 | INFO | BAZWRITER : 65.1%, 4.0, 134.1ms, 0 | 20260505 05:54:46.297 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 05:54:46.297 | INFO | Uploaded 36GiB to GPU in 2.13816s (16.8369GiB/s) | 20260505 05:54:46.298 | INFO | Downloaded 38GiB to GPU in 2.33117s (16.3009GiB/s) | 20260505 05:54:51.436 | INFO | Analyzing chunk frames = [856576,857088) | 20260505 05:54:51.436 | INFO | MemInfo: { | 20260505 05:54:51.436 | INFO | "gpu" : " 17 GB", | 20260505 05:54:51.436 | INFO | "huge" : "", | 20260505 05:54:51.436 | INFO | "huge_shared" : " 27 GB", | 20260505 05:54:51.436 | INFO | "malloc" : " 14 GB", | 20260505 05:54:51.437 | INFO | "pinned" : " 29 GB" | 20260505 05:54:51.437 | INFO | } | 20260505 05:54:51.437 | INFO | Perf report for the last 5.13938 seconds: | 20260505 05:54:51.437 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:54:51.437 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:51.437 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:51.437 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:51.437 | INFO | ANALYSIS : 64.6%, 3.0, 100.9ms, 0 | 20260505 05:54:51.437 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:51.437 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:51.437 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 05:54:51.437 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:51.437 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:51.437 | INFO | Uploaded 36GiB to GPU in 2.13385s (16.8709GiB/s) | 20260505 05:54:51.437 | INFO | Downloaded 34GiB to GPU in 2.04173s (16.6526GiB/s) | 20260505 05:54:56.576 | INFO | Analyzing chunk frames = [857088,857600) | 20260505 05:54:56.576 | INFO | MemInfo: { | 20260505 05:54:56.576 | INFO | "gpu" : " 17 GB", | 20260505 05:54:56.576 | INFO | "huge" : "", | 20260505 05:54:56.576 | INFO | "huge_shared" : " 27 GB", | 20260505 05:54:56.576 | INFO | "malloc" : " 14 GB", | 20260505 05:54:56.576 | INFO | "pinned" : " 29 GB" | 20260505 05:54:56.576 | INFO | } | 20260505 05:54:56.576 | INFO | Perf report for the last 5.13955 seconds: | 20260505 05:54:56.576 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:54:56.576 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:56.576 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:56.576 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:56.576 | INFO | ANALYSIS : 64.3%, 3.0, 100.1ms, 0 | 20260505 05:54:56.576 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:56.576 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:56.576 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:54:56.576 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:56.576 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:54:56.576 | INFO | Uploaded 36GiB to GPU in 2.17245s (16.5711GiB/s) | 20260505 05:54:56.577 | INFO | Downloaded 34GiB to GPU in 2.04659s (16.613GiB/s) | 20260505 05:55:01.715 | INFO | Analyzing chunk frames = [857600,858112) | 20260505 05:55:01.715 | INFO | MemInfo: { | 20260505 05:55:01.715 | INFO | "gpu" : " 17 GB", | 20260505 05:55:01.715 | INFO | "huge" : "", | 20260505 05:55:01.715 | INFO | "huge_shared" : " 27 GB", | 20260505 05:55:01.715 | INFO | "malloc" : " 14 GB", | 20260505 05:55:01.715 | INFO | "pinned" : " 29 GB" | 20260505 05:55:01.715 | INFO | } | 20260505 05:55:01.715 | INFO | Perf report for the last 5.1388 seconds: | 20260505 05:55:01.715 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:55:01.715 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:01.715 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:01.715 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:01.715 | INFO | ANALYSIS : 63.5%, 3.0, 98.6ms, 0 | 20260505 05:55:01.715 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:01.715 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:01.715 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:55:01.715 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:01.715 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:01.715 | INFO | Uploaded 36GiB to GPU in 2.0983s (17.1568GiB/s) | 20260505 05:55:01.716 | INFO | Downloaded 34GiB to GPU in 2.02903s (16.7568GiB/s) | 20260505 05:55:06.854 | INFO | Analyzing chunk frames = [858112,858624) | 20260505 05:55:06.854 | INFO | MemInfo: { | 20260505 05:55:06.854 | INFO | "gpu" : " 17 GB", | 20260505 05:55:06.855 | INFO | "huge" : "", | 20260505 05:55:06.855 | INFO | "huge_shared" : " 27 GB", | 20260505 05:55:06.855 | INFO | "malloc" : " 14 GB", | 20260505 05:55:06.855 | INFO | "pinned" : " 29 GB" | 20260505 05:55:06.855 | INFO | } | 20260505 05:55:06.855 | INFO | Perf report for the last 5.13969 seconds: | 20260505 05:55:06.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:55:06.855 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:06.855 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:06.855 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:06.855 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 05:55:06.855 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:06.855 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:06.855 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 05:55:06.855 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:06.855 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:06.855 | INFO | Uploaded 36GiB to GPU in 2.14401s (16.791GiB/s) | 20260505 05:55:06.855 | INFO | Downloaded 34GiB to GPU in 2.04925s (16.5914GiB/s) | 20260505 05:55:11.994 | INFO | Analyzing chunk frames = [858624,859136) | 20260505 05:55:11.994 | INFO | MemInfo: { | 20260505 05:55:11.994 | INFO | "gpu" : " 17 GB", | 20260505 05:55:11.994 | INFO | "huge" : "", | 20260505 05:55:11.994 | INFO | "huge_shared" : " 27 GB", | 20260505 05:55:11.994 | INFO | "malloc" : " 14 GB", | 20260505 05:55:11.994 | INFO | "pinned" : " 29 GB" | 20260505 05:55:11.994 | INFO | } | 20260505 05:55:11.995 | INFO | Perf report for the last 5.13982 seconds: | 20260505 05:55:11.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:55:11.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:11.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:11.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:11.995 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 05:55:11.995 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:11.995 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:11.995 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 05:55:11.995 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:11.995 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:11.995 | INFO | Uploaded 36GiB to GPU in 2.11478s (17.0231GiB/s) | 20260505 05:55:11.995 | INFO | Downloaded 34GiB to GPU in 2.05914s (16.5118GiB/s) | 20260505 05:55:17.133 | INFO | Analyzing chunk frames = [859136,859648) | 20260505 05:55:17.133 | INFO | MemInfo: { | 20260505 05:55:17.133 | INFO | "gpu" : " 17 GB", | 20260505 05:55:17.133 | INFO | "huge" : "", | 20260505 05:55:17.133 | INFO | "huge_shared" : " 27 GB", | 20260505 05:55:17.133 | INFO | "malloc" : " 14 GB", | 20260505 05:55:17.133 | INFO | "pinned" : " 29 GB" | 20260505 05:55:17.133 | INFO | } | 20260505 05:55:17.134 | INFO | Perf report for the last 5.13918 seconds: | 20260505 05:55:17.134 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:55:17.134 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:17.134 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:17.134 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:17.134 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 05:55:17.134 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:17.134 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:17.134 | INFO | PRE_HQ : 32.6%, 1.0, 17.0ms, 0 | 20260505 05:55:17.134 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:17.134 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:17.134 | INFO | Uploaded 36GiB to GPU in 2.15543s (16.702GiB/s) | 20260505 05:55:17.134 | INFO | Downloaded 34GiB to GPU in 2.15847s (15.7519GiB/s) | 20260505 05:55:22.272 | INFO | Analyzing chunk frames = [859648,860160) | 20260505 05:55:22.272 | INFO | MemInfo: { | 20260505 05:55:22.272 | INFO | "gpu" : " 17 GB", | 20260505 05:55:22.272 | INFO | "huge" : "", | 20260505 05:55:22.272 | INFO | "huge_shared" : " 27 GB", | 20260505 05:55:22.272 | INFO | "malloc" : " 14 GB", | 20260505 05:55:22.272 | INFO | "pinned" : " 29 GB" | 20260505 05:55:22.272 | INFO | } | 20260505 05:55:22.272 | INFO | Perf report for the last 5.13871 seconds: | 20260505 05:55:22.272 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:55:22.272 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:22.272 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:22.272 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:22.272 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 05:55:22.273 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:22.273 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:22.273 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 05:55:22.273 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:22.273 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:22.273 | INFO | Uploaded 36GiB to GPU in 2.06523s (17.4315GiB/s) | 20260505 05:55:22.273 | INFO | Downloaded 34GiB to GPU in 2.08739s (16.2883GiB/s) | 20260505 05:55:22.273 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 05:55:22.273 | INFO | Total mem usage: 15.3 GiB | 20260505 05:55:22.273 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 05:55:22.273 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.39%) | 20260505 05:55:22.273 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 05:55:22.273 | INFO | High Water Breakdown: | 20260505 05:55:22.273 | INFO | 60.1% : 8.65 GiB ( 92.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 05:55:22.273 | INFO | 39.8% : 5.73 GiB ( 86.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 05:55:22.273 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 05:55:22.273 | INFO | ------------------------------------------ | 20260505 05:55:22.273 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:55:22.273 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 05:55:22.273 | INFO | Total mem usage: 26.5 GiB | 20260505 05:55:22.273 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 05:55:22.273 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 05:55:22.273 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 05:55:22.273 | INFO | High Water Breakdown: | 20260505 05:55:22.273 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:55:22.273 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:55:22.273 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:55:22.273 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:55:22.273 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:55:22.273 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:55:22.273 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 05:55:22.273 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 05:55:22.273 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:55:22.273 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 05:55:22.273 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 05:55:22.273 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 05:55:22.273 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 05:55:22.273 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 05:55:22.273 | INFO | ------------------------------------------ | 20260505 05:55:22.273 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:55:22.273 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 05:55:22.273 | INFO | Total mem usage: 25.6 GiB | 20260505 05:55:22.273 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 05:55:22.273 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 05:55:22.273 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 05:55:22.273 | INFO | High Water Breakdown: | 20260505 05:55:22.273 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 05:55:22.273 | INFO | ------------------------------------------ | 20260505 05:55:22.273 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:55:22.273 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 05:55:22.273 | INFO | Total mem usage: 15.7 GiB | 20260505 05:55:22.273 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 05:55:22.273 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 05:55:22.273 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260505 05:55:22.273 | INFO | High Water Breakdown: | 20260505 05:55:22.273 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 05:55:22.273 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 05:55:22.273 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 05:55:22.273 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 05:55:22.273 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 05:55:22.273 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 05:55:22.273 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 05:55:22.273 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 05:55:22.273 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 05:55:22.273 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:55:22.273 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 05:55:22.273 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:55:22.273 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 05:55:22.274 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 05:55:22.274 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 05:55:22.274 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 05:55:22.274 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:55:22.274 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 05:55:22.274 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:55:22.274 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:55:22.274 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:55:22.274 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 05:55:22.274 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 05:55:22.274 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:55:22.274 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 05:55:22.274 | INFO | ------------------------------------------ | 20260505 05:55:22.274 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:55:22.274 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 05:55:22.274 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:55:22.274 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:55:22.274 | INFO | PerfReport: Full Execution -- 3.33e+01 / 7.62e+01, 1.40e+01 / 1.41e+00 | 20260505 05:55:22.274 | INFO | PerfReport: 0.28%, AggregateStats -- 9.40e-02 / 1.02e-01, 1.05e-02 / 5.96e-03 | 20260505 05:55:22.274 | INFO | PerfReport: 11.56%, Baseline -- 3.85e+00 / 4.56e+00, 6.42e-01 / 2.41e-01 | 20260505 05:55:22.274 | INFO | PerfReport: 3.31%, Histogram -- 1.10e+00 / 1.52e+00, 2.22e-01 / 1.12e-01 | 20260505 05:55:22.274 | INFO | PerfReport: 15.50%, DME -- 4.13e+01 / 4.53e+01, 4.36e+00 / 1.08e+00 | 20260505 05:55:22.274 | INFO | PerfReport: 55.81%, FrameLabeling -- 1.86e+01 / 1.91e+01, 2.09e+00 / 1.07e+00 | 20260505 05:55:22.274 | INFO | PerfReport: 7.83%, PulseAccumulating -- 2.61e+00 / 3.19e+00, 4.98e-01 / 3.32e-01 | 20260505 05:55:22.274 | INFO | PerfReport: 4.18%, Metrics -- 1.39e+00 / 1.76e+00, 3.87e-01 / 2.16e-01 | 20260505 05:55:22.274 | INFO | PerfReport: 1.53%, PulseSerializing -- 5.09e-01 / 5.89e-01, 1.00e-01 / 4.10e-02 | 20260505 05:55:22.274 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 05:55:22.274 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:55:22.274 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:55:22.274 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.30e+01 / 0.00e+00 | 20260505 05:55:22.274 | INFO | PerfReport: 21.80%, Upload -- 2.22e+01 / 0.00e+00, 7.57e+00 / 0.00e+00 | 20260505 05:55:22.274 | INFO | PerfReport: 21.49%, Download -- 2.19e+01 / 0.00e+00, 7.71e+00 / 0.00e+00 | 20260505 05:55:22.274 | INFO | PerfReport: 32.77%, Compute -- 3.33e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260505 05:55:22.274 | INFO | PerfReport: 1.46%, LockUpload -- 1.48e+00 / 0.00e+00, 6.65e+00 / 0.00e+00 | 20260505 05:55:22.274 | INFO | PerfReport: 0.65%, LockDownload -- 6.57e-01 / 0.00e+00, 1.90e+00 / 0.00e+00 | 20260505 05:55:22.274 | INFO | PerfReport: 21.82%, LockCompute -- 2.22e+01 / 0.00e+00, 1.66e+01 / 0.00e+00 | 20260505 05:55:22.274 | INFO | PerfReport: 0.01%, Misc -- 1.49e-02 / 0.00e+00, 2.03e-02 / 0.00e+00 | 20260505 05:55:22.389 | INFO | Starting a round of Baz IO | 20260505 05:55:25.648 | INFO | Populating RTMetrics report | 20260505 05:55:25.648 | INFO | RTMetrics report populated | 20260505 05:55:25.648 | INFO | Opening tmp RTMetrics json file | 20260505 05:55:25.668 | INFO | Serializing RTMetrics to json | 20260505 05:55:25.669 | INFO | Serialization done, swapping tmp file now... | 20260505 05:55:25.670 | INFO | RTMetrics done | 20260505 05:55:25.756 | INFO | Finished a round of Baz IO: Wrote 2.315 GiB in 3.367 seconds | 20260505 05:55:25.756 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.88% / 0.06% / 64.05% | 20260505 05:55:25.756 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.23813% / 0.000119209% / 4.37101e-05% | 20260505 05:55:25.756 | INFO | Baz IO overall: | 20260505 05:55:25.756 | INFO | BytesWritten : 424795135547 | 20260505 05:55:25.756 | INFO | NumEvents : 200230217142 | 20260505 05:55:27.412 | INFO | Analyzing chunk frames = [860160,860672) | 20260505 05:55:27.412 | INFO | MemInfo: { | 20260505 05:55:27.412 | INFO | "gpu" : " 17 GB", | 20260505 05:55:27.412 | INFO | "huge" : "", | 20260505 05:55:27.412 | INFO | "huge_shared" : " 27 GB", | 20260505 05:55:27.412 | INFO | "malloc" : " 14 GB", | 20260505 05:55:27.412 | INFO | "pinned" : " 29 GB" | 20260505 05:55:27.412 | INFO | } | 20260505 05:55:27.412 | INFO | Perf report for the last 5.13958 seconds: | 20260505 05:55:27.412 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:55:27.412 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:27.412 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:27.412 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:27.412 | INFO | ANALYSIS : 64.5%, 2.9, 99.3ms, 0 | 20260505 05:55:27.412 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:27.412 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260505 05:55:27.412 | INFO | PRE_HQ : 63.6%, 2.0, 65.0ms, 0 | 20260505 05:55:27.412 | INFO | BAZWRITER : 64.8%, 4.0, 134.6ms, 0 | 20260505 05:55:27.412 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 05:55:27.412 | INFO | Uploaded 36GiB to GPU in 2.12604s (16.9329GiB/s) | 20260505 05:55:27.412 | INFO | Downloaded 38GiB to GPU in 2.33258s (16.291GiB/s) | 20260505 05:55:32.551 | INFO | Analyzing chunk frames = [860672,861184) | 20260505 05:55:32.551 | INFO | MemInfo: { | 20260505 05:55:32.551 | INFO | "gpu" : " 17 GB", | 20260505 05:55:32.551 | INFO | "huge" : "", | 20260505 05:55:32.551 | INFO | "huge_shared" : " 27 GB", | 20260505 05:55:32.551 | INFO | "malloc" : " 14 GB", | 20260505 05:55:32.551 | INFO | "pinned" : " 29 GB" | 20260505 05:55:32.551 | INFO | } | 20260505 05:55:32.551 | INFO | Perf report for the last 5.13913 seconds: | 20260505 05:55:32.551 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:55:32.551 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:32.551 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:32.551 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:32.551 | INFO | ANALYSIS : 64.7%, 3.0, 101.0ms, 0 | 20260505 05:55:32.551 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:32.551 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:32.551 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:55:32.552 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:32.552 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:32.552 | INFO | Uploaded 36GiB to GPU in 2.13738s (16.8431GiB/s) | 20260505 05:55:32.552 | INFO | Downloaded 34GiB to GPU in 2.03382s (16.7173GiB/s) | 20260505 05:55:37.690 | INFO | Analyzing chunk frames = [861184,861696) | 20260505 05:55:37.690 | INFO | MemInfo: { | 20260505 05:55:37.690 | INFO | "gpu" : " 17 GB", | 20260505 05:55:37.690 | INFO | "huge" : "", | 20260505 05:55:37.690 | INFO | "huge_shared" : " 27 GB", | 20260505 05:55:37.690 | INFO | "malloc" : " 14 GB", | 20260505 05:55:37.690 | INFO | "pinned" : " 29 GB" | 20260505 05:55:37.690 | INFO | } | 20260505 05:55:37.691 | INFO | Perf report for the last 5.13935 seconds: | 20260505 05:55:37.691 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:55:37.691 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:37.691 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:37.691 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:37.691 | INFO | ANALYSIS : 64.4%, 3.0, 100.3ms, 0 | 20260505 05:55:37.691 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:37.691 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:37.691 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:55:37.691 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:37.691 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:37.691 | INFO | Uploaded 36GiB to GPU in 2.16913s (16.5965GiB/s) | 20260505 05:55:37.691 | INFO | Downloaded 34GiB to GPU in 2.02874s (16.7592GiB/s) | 20260505 05:55:42.830 | INFO | Analyzing chunk frames = [861696,862208) | 20260505 05:55:42.830 | INFO | MemInfo: { | 20260505 05:55:42.830 | INFO | "gpu" : " 17 GB", | 20260505 05:55:42.830 | INFO | "huge" : "", | 20260505 05:55:42.830 | INFO | "huge_shared" : " 27 GB", | 20260505 05:55:42.830 | INFO | "malloc" : " 14 GB", | 20260505 05:55:42.830 | INFO | "pinned" : " 29 GB" | 20260505 05:55:42.830 | INFO | } | 20260505 05:55:42.830 | INFO | Perf report for the last 5.13954 seconds: | 20260505 05:55:42.830 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:55:42.830 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:42.830 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:42.830 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:42.830 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260505 05:55:42.830 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:42.830 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:42.830 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:55:42.830 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:42.830 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:42.830 | INFO | Uploaded 36GiB to GPU in 2.10868s (17.0723GiB/s) | 20260505 05:55:42.830 | INFO | Downloaded 34GiB to GPU in 2.02154s (16.8188GiB/s) | 20260505 05:55:47.969 | INFO | Analyzing chunk frames = [862208,862720) | 20260505 05:55:47.969 | INFO | MemInfo: { | 20260505 05:55:47.969 | INFO | "gpu" : " 17 GB", | 20260505 05:55:47.969 | INFO | "huge" : "", | 20260505 05:55:47.969 | INFO | "huge_shared" : " 27 GB", | 20260505 05:55:47.969 | INFO | "malloc" : " 14 GB", | 20260505 05:55:47.969 | INFO | "pinned" : " 29 GB" | 20260505 05:55:47.969 | INFO | } | 20260505 05:55:47.969 | INFO | Perf report for the last 5.13924 seconds: | 20260505 05:55:47.969 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:55:47.969 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:47.969 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:47.969 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:47.969 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 05:55:47.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:47.969 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:47.969 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 05:55:47.969 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:47.969 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:47.970 | INFO | Uploaded 36GiB to GPU in 2.13545s (16.8583GiB/s) | 20260505 05:55:47.970 | INFO | Downloaded 34GiB to GPU in 2.04469s (16.6284GiB/s) | 20260505 05:55:53.109 | INFO | Analyzing chunk frames = [862720,863232) | 20260505 05:55:53.109 | INFO | MemInfo: { | 20260505 05:55:53.109 | INFO | "gpu" : " 17 GB", | 20260505 05:55:53.109 | INFO | "huge" : "", | 20260505 05:55:53.109 | INFO | "huge_shared" : " 27 GB", | 20260505 05:55:53.109 | INFO | "malloc" : " 14 GB", | 20260505 05:55:53.109 | INFO | "pinned" : " 29 GB" | 20260505 05:55:53.109 | INFO | } | 20260505 05:55:53.109 | INFO | Perf report for the last 5.13988 seconds: | 20260505 05:55:53.109 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:55:53.109 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:53.109 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:53.109 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:53.109 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 05:55:53.109 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:53.109 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:53.109 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 05:55:53.109 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:53.109 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:53.109 | INFO | Uploaded 36GiB to GPU in 2.1028s (17.1201GiB/s) | 20260505 05:55:53.110 | INFO | Downloaded 34GiB to GPU in 2.05345s (16.5575GiB/s) | 20260505 05:55:58.248 | INFO | Analyzing chunk frames = [863232,863744) | 20260505 05:55:58.248 | INFO | MemInfo: { | 20260505 05:55:58.248 | INFO | "gpu" : " 17 GB", | 20260505 05:55:58.248 | INFO | "huge" : "", | 20260505 05:55:58.248 | INFO | "huge_shared" : " 27 GB", | 20260505 05:55:58.248 | INFO | "malloc" : " 14 GB", | 20260505 05:55:58.248 | INFO | "pinned" : " 29 GB" | 20260505 05:55:58.248 | INFO | } | 20260505 05:55:58.248 | INFO | Perf report for the last 5.13866 seconds: | 20260505 05:55:58.248 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:55:58.248 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:58.248 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:58.248 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:58.248 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 05:55:58.248 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:58.248 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:58.248 | INFO | PRE_HQ : 32.4%, 1.0, 16.9ms, 0 | 20260505 05:55:58.248 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:58.248 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:55:58.248 | INFO | Uploaded 36GiB to GPU in 2.1591s (16.6736GiB/s) | 20260505 05:55:58.249 | INFO | Downloaded 34GiB to GPU in 2.15895s (15.7484GiB/s) | 20260505 05:56:03.387 | INFO | Analyzing chunk frames = [863744,864256) | 20260505 05:56:03.387 | INFO | MemInfo: { | 20260505 05:56:03.387 | INFO | "gpu" : " 17 GB", | 20260505 05:56:03.387 | INFO | "huge" : "", | 20260505 05:56:03.387 | INFO | "huge_shared" : " 27 GB", | 20260505 05:56:03.387 | INFO | "malloc" : " 14 GB", | 20260505 05:56:03.387 | INFO | "pinned" : " 29 GB" | 20260505 05:56:03.387 | INFO | } | 20260505 05:56:03.387 | INFO | Perf report for the last 5.13948 seconds: | 20260505 05:56:03.387 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:56:03.387 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:03.387 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:03.387 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:03.387 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 05:56:03.387 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:03.387 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:03.388 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 05:56:03.388 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:03.388 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:03.388 | INFO | Uploaded 36GiB to GPU in 2.07549s (17.3453GiB/s) | 20260505 05:56:03.388 | INFO | Downloaded 34GiB to GPU in 2.08165s (16.3332GiB/s) | 20260505 05:56:03.503 | INFO | Starting a round of Baz IO | 20260505 05:56:06.774 | INFO | Populating RTMetrics report | 20260505 05:56:06.775 | INFO | RTMetrics report populated | 20260505 05:56:06.775 | INFO | Opening tmp RTMetrics json file | 20260505 05:56:06.792 | INFO | Serializing RTMetrics to json | 20260505 05:56:06.794 | INFO | Serialization done, swapping tmp file now... | 20260505 05:56:06.797 | INFO | RTMetrics done | 20260505 05:56:06.889 | INFO | Finished a round of Baz IO: Wrote 2.317 GiB in 3.386 seconds | 20260505 05:56:06.889 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.82% / 0.06% / 64.12% | 20260505 05:56:06.889 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.21781% / 0.000111262% / 5.16574e-05% | 20260505 05:56:06.889 | INFO | Baz IO overall: | 20260505 05:56:06.889 | INFO | BytesWritten : 427282637305 | 20260505 05:56:06.889 | INFO | NumEvents : 201419457019 | 20260505 05:56:08.527 | INFO | Analyzing chunk frames = [864256,864768) | 20260505 05:56:08.527 | INFO | MemInfo: { | 20260505 05:56:08.527 | INFO | "gpu" : " 17 GB", | 20260505 05:56:08.527 | INFO | "huge" : "", | 20260505 05:56:08.527 | INFO | "huge_shared" : " 27 GB", | 20260505 05:56:08.527 | INFO | "malloc" : " 14 GB", | 20260505 05:56:08.527 | INFO | "pinned" : " 29 GB" | 20260505 05:56:08.527 | INFO | } | 20260505 05:56:08.527 | INFO | Perf report for the last 5.13967 seconds: | 20260505 05:56:08.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:56:08.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:08.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:08.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:08.527 | INFO | ANALYSIS : 64.8%, 2.9, 99.2ms, 0 | 20260505 05:56:08.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:08.527 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260505 05:56:08.527 | INFO | PRE_HQ : 63.8%, 1.9, 64.5ms, 0 | 20260505 05:56:08.527 | INFO | BAZWRITER : 65.2%, 4.0, 134.9ms, 0 | 20260505 05:56:08.527 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 05:56:08.527 | INFO | Uploaded 36GiB to GPU in 2.157s (16.6899GiB/s) | 20260505 05:56:08.527 | INFO | Downloaded 38GiB to GPU in 2.33478s (16.2756GiB/s) | 20260505 05:56:13.666 | INFO | Analyzing chunk frames = [864768,865280) | 20260505 05:56:13.666 | INFO | MemInfo: { | 20260505 05:56:13.666 | INFO | "gpu" : " 17 GB", | 20260505 05:56:13.666 | INFO | "huge" : "", | 20260505 05:56:13.666 | INFO | "huge_shared" : " 27 GB", | 20260505 05:56:13.666 | INFO | "malloc" : " 14 GB", | 20260505 05:56:13.666 | INFO | "pinned" : " 29 GB" | 20260505 05:56:13.666 | INFO | } | 20260505 05:56:13.666 | INFO | Perf report for the last 5.13874 seconds: | 20260505 05:56:13.666 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:56:13.666 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:13.666 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:13.666 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:13.666 | INFO | ANALYSIS : 64.8%, 3.0, 101.1ms, 0 | 20260505 05:56:13.666 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:13.666 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:13.666 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 05:56:13.666 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:13.666 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:13.666 | INFO | Uploaded 36GiB to GPU in 2.14191s (16.8074GiB/s) | 20260505 05:56:13.666 | INFO | Downloaded 34GiB to GPU in 2.02721s (16.7718GiB/s) | 20260505 05:56:18.805 | INFO | Analyzing chunk frames = [865280,865792) | 20260505 05:56:18.805 | INFO | MemInfo: { | 20260505 05:56:18.805 | INFO | "gpu" : " 17 GB", | 20260505 05:56:18.805 | INFO | "huge" : "", | 20260505 05:56:18.805 | INFO | "huge_shared" : " 27 GB", | 20260505 05:56:18.805 | INFO | "malloc" : " 14 GB", | 20260505 05:56:18.805 | INFO | "pinned" : " 29 GB" | 20260505 05:56:18.805 | INFO | } | 20260505 05:56:18.805 | INFO | Perf report for the last 5.1396 seconds: | 20260505 05:56:18.805 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:56:18.805 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:18.805 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:18.805 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:18.806 | INFO | ANALYSIS : 64.4%, 3.0, 100.2ms, 0 | 20260505 05:56:18.806 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:18.806 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:18.806 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:56:18.806 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:18.806 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:18.806 | INFO | Uploaded 36GiB to GPU in 2.17031s (16.5875GiB/s) | 20260505 05:56:18.806 | INFO | Downloaded 34GiB to GPU in 2.03177s (16.7342GiB/s) | 20260505 05:56:23.944 | INFO | Analyzing chunk frames = [865792,866304) | 20260505 05:56:23.945 | INFO | MemInfo: { | 20260505 05:56:23.945 | INFO | "gpu" : " 17 GB", | 20260505 05:56:23.945 | INFO | "huge" : "", | 20260505 05:56:23.945 | INFO | "huge_shared" : " 27 GB", | 20260505 05:56:23.945 | INFO | "malloc" : " 14 GB", | 20260505 05:56:23.945 | INFO | "pinned" : " 29 GB" | 20260505 05:56:23.945 | INFO | } | 20260505 05:56:23.945 | INFO | Perf report for the last 5.13936 seconds: | 20260505 05:56:23.945 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:56:23.945 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:23.945 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:23.945 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:23.945 | INFO | ANALYSIS : 63.6%, 3.0, 98.8ms, 0 | 20260505 05:56:23.945 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:23.945 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:23.945 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260505 05:56:23.945 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:23.945 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:23.945 | INFO | Uploaded 36GiB to GPU in 2.1135s (17.0333GiB/s) | 20260505 05:56:23.945 | INFO | Downloaded 34GiB to GPU in 2.03159s (16.7357GiB/s) | 20260505 05:56:29.084 | INFO | Analyzing chunk frames = [866304,866816) | 20260505 05:56:29.084 | INFO | MemInfo: { | 20260505 05:56:29.084 | INFO | "gpu" : " 17 GB", | 20260505 05:56:29.084 | INFO | "huge" : "", | 20260505 05:56:29.084 | INFO | "huge_shared" : " 27 GB", | 20260505 05:56:29.084 | INFO | "malloc" : " 14 GB", | 20260505 05:56:29.084 | INFO | "pinned" : " 29 GB" | 20260505 05:56:29.084 | INFO | } | 20260505 05:56:29.084 | INFO | Perf report for the last 5.13934 seconds: | 20260505 05:56:29.084 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:56:29.084 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:29.084 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:29.084 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:29.084 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 05:56:29.084 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:29.084 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:29.084 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 05:56:29.084 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:29.084 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:29.084 | INFO | Uploaded 36GiB to GPU in 2.12798s (16.9175GiB/s) | 20260505 05:56:29.084 | INFO | Downloaded 34GiB to GPU in 2.03732s (16.6886GiB/s) | 20260505 05:56:34.223 | INFO | Analyzing chunk frames = [866816,867328) | 20260505 05:56:34.223 | INFO | MemInfo: { | 20260505 05:56:34.223 | INFO | "gpu" : " 17 GB", | 20260505 05:56:34.223 | INFO | "huge" : "", | 20260505 05:56:34.223 | INFO | "huge_shared" : " 27 GB", | 20260505 05:56:34.223 | INFO | "malloc" : " 14 GB", | 20260505 05:56:34.223 | INFO | "pinned" : " 29 GB" | 20260505 05:56:34.223 | INFO | } | 20260505 05:56:34.223 | INFO | Perf report for the last 5.13912 seconds: | 20260505 05:56:34.223 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:56:34.223 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:34.223 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:34.224 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:34.224 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 05:56:34.224 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:34.224 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:34.224 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 05:56:34.224 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:34.224 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:34.224 | INFO | Uploaded 36GiB to GPU in 2.14581s (16.7769GiB/s) | 20260505 05:56:34.224 | INFO | Downloaded 34GiB to GPU in 2.05901s (16.5128GiB/s) | 20260505 05:56:39.362 | INFO | Analyzing chunk frames = [867328,867840) | 20260505 05:56:39.362 | INFO | MemInfo: { | 20260505 05:56:39.362 | INFO | "gpu" : " 17 GB", | 20260505 05:56:39.362 | INFO | "huge" : "", | 20260505 05:56:39.362 | INFO | "huge_shared" : " 27 GB", | 20260505 05:56:39.362 | INFO | "malloc" : " 14 GB", | 20260505 05:56:39.362 | INFO | "pinned" : " 29 GB" | 20260505 05:56:39.362 | INFO | } | 20260505 05:56:39.363 | INFO | Perf report for the last 5.13923 seconds: | 20260505 05:56:39.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:56:39.363 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:39.363 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:39.363 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:39.363 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 05:56:39.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:39.363 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:39.363 | INFO | PRE_HQ : 32.6%, 1.0, 17.0ms, 0 | 20260505 05:56:39.363 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:39.363 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:39.363 | INFO | Uploaded 36GiB to GPU in 2.20133s (16.3537GiB/s) | 20260505 05:56:39.363 | INFO | Downloaded 34GiB to GPU in 2.17144s (15.6578GiB/s) | 20260505 05:56:44.502 | INFO | Analyzing chunk frames = [867840,868352) | 20260505 05:56:44.502 | INFO | MemInfo: { | 20260505 05:56:44.502 | INFO | "gpu" : " 17 GB", | 20260505 05:56:44.502 | INFO | "huge" : "", | 20260505 05:56:44.502 | INFO | "huge_shared" : " 27 GB", | 20260505 05:56:44.502 | INFO | "malloc" : " 14 GB", | 20260505 05:56:44.502 | INFO | "pinned" : " 29 GB" | 20260505 05:56:44.502 | INFO | } | 20260505 05:56:44.502 | INFO | Perf report for the last 5.13981 seconds: | 20260505 05:56:44.503 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:56:44.503 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:44.503 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:44.503 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:44.503 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 05:56:44.503 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:44.503 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:44.503 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 05:56:44.503 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:44.503 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:44.503 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 05:56:44.503 | INFO | Total mem usage: 15.3 GiB | 20260505 05:56:44.503 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 05:56:44.503 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.41%) | 20260505 05:56:44.503 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 05:56:44.503 | INFO | High Water Breakdown: | 20260505 05:56:44.503 | INFO | 60.1% : 8.65 GiB ( 92.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 05:56:44.503 | INFO | 39.8% : 5.73 GiB ( 86.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 05:56:44.503 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 05:56:44.503 | INFO | ------------------------------------------ | 20260505 05:56:44.503 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:56:44.503 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 05:56:44.503 | INFO | Total mem usage: 26.5 GiB | 20260505 05:56:44.503 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 05:56:44.503 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 05:56:44.503 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 05:56:44.503 | INFO | High Water Breakdown: | 20260505 05:56:44.503 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:56:44.503 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:56:44.503 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:56:44.503 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:56:44.503 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:56:44.503 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:56:44.503 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 05:56:44.503 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 05:56:44.503 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:56:44.503 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 05:56:44.503 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 05:56:44.503 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 05:56:44.503 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 05:56:44.503 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 05:56:44.503 | INFO | ------------------------------------------ | 20260505 05:56:44.503 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:56:44.503 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 05:56:44.503 | INFO | Total mem usage: 25.6 GiB | 20260505 05:56:44.503 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 05:56:44.503 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 05:56:44.503 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 05:56:44.503 | INFO | High Water Breakdown: | 20260505 05:56:44.503 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 05:56:44.503 | INFO | ------------------------------------------ | 20260505 05:56:44.503 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:56:44.503 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 05:56:44.503 | INFO | Total mem usage: 15.7 GiB | 20260505 05:56:44.503 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 05:56:44.503 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 05:56:44.503 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260505 05:56:44.503 | INFO | High Water Breakdown: | 20260505 05:56:44.503 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 05:56:44.503 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 05:56:44.503 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 05:56:44.503 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 05:56:44.503 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 05:56:44.503 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 05:56:44.503 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 05:56:44.503 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 05:56:44.503 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 05:56:44.503 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:56:44.503 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 05:56:44.503 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:56:44.503 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 05:56:44.503 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 05:56:44.503 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 05:56:44.503 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 05:56:44.503 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:56:44.503 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 05:56:44.503 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:56:44.503 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:56:44.503 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:56:44.503 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 05:56:44.503 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 05:56:44.503 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:56:44.503 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 05:56:44.503 | INFO | ------------------------------------------ | 20260505 05:56:44.503 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:56:44.503 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 05:56:44.503 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:56:44.503 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:56:44.503 | INFO | PerfReport: Full Execution -- 3.34e+01 / 7.64e+01, 1.40e+01 / 1.59e+00 | 20260505 05:56:44.503 | INFO | PerfReport: 0.28%, AggregateStats -- 9.42e-02 / 1.04e-01, 1.10e-02 / 3.78e-03 | 20260505 05:56:44.503 | INFO | PerfReport: 11.54%, Baseline -- 3.85e+00 / 4.55e+00, 6.44e-01 / 2.86e-01 | 20260505 05:56:44.503 | INFO | PerfReport: 3.31%, Histogram -- 1.10e+00 / 1.53e+00, 2.23e-01 / 9.95e-02 | 20260505 05:56:44.503 | INFO | PerfReport: 15.52%, DME -- 4.14e+01 / 4.55e+01, 4.41e+00 / 1.19e+00 | 20260505 05:56:44.503 | INFO | PerfReport: 55.84%, FrameLabeling -- 1.86e+01 / 1.92e+01, 2.11e+00 / 1.01e+00 | 20260505 05:56:44.503 | INFO | PerfReport: 7.81%, PulseAccumulating -- 2.61e+00 / 3.20e+00, 4.94e-01 / 3.09e-01 | 20260505 05:56:44.503 | INFO | PerfReport: 4.17%, Metrics -- 1.39e+00 / 1.77e+00, 3.93e-01 / 2.21e-01 | 20260505 05:56:44.503 | INFO | PerfReport: 1.52%, PulseSerializing -- 5.09e-01 / 5.88e-01, 1.00e-01 / 3.62e-02 | 20260505 05:56:44.503 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 05:56:44.503 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:56:44.503 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:56:44.503 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.32e+01 / 0.00e+00 | 20260505 05:56:44.503 | INFO | PerfReport: 21.85%, Upload -- 2.22e+01 / 0.00e+00, 7.55e+00 / 0.00e+00 | 20260505 05:56:44.503 | INFO | PerfReport: 21.46%, Download -- 2.18e+01 / 0.00e+00, 7.68e+00 / 0.00e+00 | 20260505 05:56:44.503 | INFO | PerfReport: 32.79%, Compute -- 3.34e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260505 05:56:44.503 | INFO | PerfReport: 1.45%, LockUpload -- 1.47e+00 / 0.00e+00, 6.64e+00 / 0.00e+00 | 20260505 05:56:44.503 | INFO | PerfReport: 0.64%, LockDownload -- 6.56e-01 / 0.00e+00, 1.90e+00 / 0.00e+00 | 20260505 05:56:44.503 | INFO | PerfReport: 21.79%, LockCompute -- 2.22e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260505 05:56:44.503 | INFO | PerfReport: 0.01%, Misc -- 1.48e-02 / 0.00e+00, 2.24e-02 / 0.00e+00 | 20260505 05:56:44.503 | INFO | Uploaded 36GiB to GPU in 2.06332s (17.4476GiB/s) | 20260505 05:56:44.503 | INFO | Downloaded 34GiB to GPU in 2.07447s (16.3898GiB/s) | 20260505 05:56:44.616 | INFO | Starting a round of Baz IO | 20260505 05:56:47.847 | INFO | Populating RTMetrics report | 20260505 05:56:47.847 | INFO | RTMetrics report populated | 20260505 05:56:47.847 | INFO | Opening tmp RTMetrics json file | 20260505 05:56:47.862 | INFO | Serializing RTMetrics to json | 20260505 05:56:47.865 | INFO | Serialization done, swapping tmp file now... | 20260505 05:56:47.868 | INFO | RTMetrics done | 20260505 05:56:47.959 | INFO | Finished a round of Baz IO: Wrote 2.315 GiB in 3.343 seconds | 20260505 05:56:47.959 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.76% / 0.06% / 64.18% | 20260505 05:56:47.959 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.17281% / 0.000147025% / 6.35783e-05% | 20260505 05:56:47.959 | INFO | Baz IO overall: | 20260505 05:56:47.959 | INFO | BytesWritten : 429767948103 | 20260505 05:56:47.959 | INFO | NumEvents : 202606816962 | 20260505 05:56:49.641 | INFO | Analyzing chunk frames = [868352,868864) | 20260505 05:56:49.641 | INFO | MemInfo: { | 20260505 05:56:49.641 | INFO | "gpu" : " 17 GB", | 20260505 05:56:49.641 | INFO | "huge" : "", | 20260505 05:56:49.641 | INFO | "huge_shared" : " 27 GB", | 20260505 05:56:49.641 | INFO | "malloc" : " 14 GB", | 20260505 05:56:49.641 | INFO | "pinned" : " 29 GB" | 20260505 05:56:49.641 | INFO | } | 20260505 05:56:49.641 | INFO | Perf report for the last 5.13888 seconds: | 20260505 05:56:49.641 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:56:49.641 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:49.641 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:49.641 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:49.641 | INFO | ANALYSIS : 63.9%, 2.9, 98.0ms, 0 | 20260505 05:56:49.641 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:49.641 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260505 05:56:49.641 | INFO | PRE_HQ : 63.1%, 2.0, 64.4ms, 0 | 20260505 05:56:49.641 | INFO | BAZWRITER : 64.4%, 4.0, 133.7ms, 0 | 20260505 05:56:49.641 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260505 05:56:49.642 | INFO | Uploaded 36GiB to GPU in 2.05014s (17.5598GiB/s) | 20260505 05:56:49.642 | INFO | Downloaded 38GiB to GPU in 2.32281s (16.3595GiB/s) | 20260505 05:56:54.780 | INFO | Analyzing chunk frames = [868864,869376) | 20260505 05:56:54.780 | INFO | MemInfo: { | 20260505 05:56:54.780 | INFO | "gpu" : " 17 GB", | 20260505 05:56:54.780 | INFO | "huge" : "", | 20260505 05:56:54.780 | INFO | "huge_shared" : " 27 GB", | 20260505 05:56:54.780 | INFO | "malloc" : " 14 GB", | 20260505 05:56:54.780 | INFO | "pinned" : " 29 GB" | 20260505 05:56:54.780 | INFO | } | 20260505 05:56:54.781 | INFO | Perf report for the last 5.13928 seconds: | 20260505 05:56:54.781 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:56:54.781 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:54.781 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:54.781 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:54.781 | INFO | ANALYSIS : 64.7%, 3.0, 101.0ms, 0 | 20260505 05:56:54.781 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:54.781 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:54.781 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 05:56:54.781 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:54.781 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:54.781 | INFO | Uploaded 36GiB to GPU in 2.14066s (16.8172GiB/s) | 20260505 05:56:54.781 | INFO | Downloaded 34GiB to GPU in 2.03809s (16.6823GiB/s) | 20260505 05:56:59.920 | INFO | Analyzing chunk frames = [869376,869888) | 20260505 05:56:59.920 | INFO | MemInfo: { | 20260505 05:56:59.920 | INFO | "gpu" : " 17 GB", | 20260505 05:56:59.920 | INFO | "huge" : "", | 20260505 05:56:59.920 | INFO | "huge_shared" : " 27 GB", | 20260505 05:56:59.920 | INFO | "malloc" : " 14 GB", | 20260505 05:56:59.920 | INFO | "pinned" : " 29 GB" | 20260505 05:56:59.920 | INFO | } | 20260505 05:56:59.920 | INFO | Perf report for the last 5.1396 seconds: | 20260505 05:56:59.920 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:56:59.920 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:59.920 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:59.920 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:59.920 | INFO | ANALYSIS : 64.3%, 3.0, 100.0ms, 0 | 20260505 05:56:59.920 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:59.920 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:59.920 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:56:59.920 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:59.920 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:56:59.920 | INFO | Uploaded 36GiB to GPU in 2.17235s (16.5719GiB/s) | 20260505 05:56:59.920 | INFO | Downloaded 34GiB to GPU in 2.03305s (16.7237GiB/s) | 20260505 05:57:05.059 | INFO | Analyzing chunk frames = [869888,870400) | 20260505 05:57:05.059 | INFO | MemInfo: { | 20260505 05:57:05.059 | INFO | "gpu" : " 17 GB", | 20260505 05:57:05.059 | INFO | "huge" : "", | 20260505 05:57:05.059 | INFO | "huge_shared" : " 27 GB", | 20260505 05:57:05.059 | INFO | "malloc" : " 14 GB", | 20260505 05:57:05.059 | INFO | "pinned" : " 29 GB" | 20260505 05:57:05.059 | INFO | } | 20260505 05:57:05.059 | INFO | Perf report for the last 5.13894 seconds: | 20260505 05:57:05.059 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:57:05.059 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:05.059 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:05.059 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:05.059 | INFO | ANALYSIS : 63.6%, 3.0, 98.8ms, 0 | 20260505 05:57:05.059 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:05.059 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:05.059 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:57:05.059 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:05.059 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:05.059 | INFO | Uploaded 36GiB to GPU in 2.09008s (17.2242GiB/s) | 20260505 05:57:05.059 | INFO | Downloaded 34GiB to GPU in 2.02786s (16.7664GiB/s) | 20260505 05:57:10.199 | INFO | Analyzing chunk frames = [870400,870912) | 20260505 05:57:10.199 | INFO | MemInfo: { | 20260505 05:57:10.199 | INFO | "gpu" : " 17 GB", | 20260505 05:57:10.199 | INFO | "huge" : "", | 20260505 05:57:10.199 | INFO | "huge_shared" : " 27 GB", | 20260505 05:57:10.199 | INFO | "malloc" : " 14 GB", | 20260505 05:57:10.199 | INFO | "pinned" : " 29 GB" | 20260505 05:57:10.199 | INFO | } | 20260505 05:57:10.199 | INFO | Perf report for the last 5.13947 seconds: | 20260505 05:57:10.199 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:57:10.199 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:10.199 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:10.199 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:10.199 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 05:57:10.199 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:10.199 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:10.199 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 05:57:10.199 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:10.199 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:10.199 | INFO | Uploaded 36GiB to GPU in 2.12524s (16.9392GiB/s) | 20260505 05:57:10.199 | INFO | Downloaded 34GiB to GPU in 2.0511s (16.5765GiB/s) | 20260505 05:57:15.339 | INFO | Analyzing chunk frames = [870912,871424) | 20260505 05:57:15.339 | INFO | MemInfo: { | 20260505 05:57:15.339 | INFO | "gpu" : " 17 GB", | 20260505 05:57:15.339 | INFO | "huge" : "", | 20260505 05:57:15.339 | INFO | "huge_shared" : " 27 GB", | 20260505 05:57:15.339 | INFO | "malloc" : " 14 GB", | 20260505 05:57:15.339 | INFO | "pinned" : " 29 GB" | 20260505 05:57:15.339 | INFO | } | 20260505 05:57:15.339 | INFO | Perf report for the last 5.14041 seconds: | 20260505 05:57:15.339 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:57:15.339 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:15.339 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:15.339 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:15.339 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 05:57:15.339 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:15.339 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:15.339 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 05:57:15.339 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:15.339 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:15.339 | INFO | Uploaded 36GiB to GPU in 2.09309s (17.1995GiB/s) | 20260505 05:57:15.339 | INFO | Downloaded 34GiB to GPU in 2.04335s (16.6393GiB/s) | 20260505 05:57:20.477 | INFO | Analyzing chunk frames = [871424,871936) | 20260505 05:57:20.477 | INFO | MemInfo: { | 20260505 05:57:20.477 | INFO | "gpu" : " 17 GB", | 20260505 05:57:20.477 | INFO | "huge" : "", | 20260505 05:57:20.477 | INFO | "huge_shared" : " 27 GB", | 20260505 05:57:20.477 | INFO | "malloc" : " 14 GB", | 20260505 05:57:20.477 | INFO | "pinned" : " 29 GB" | 20260505 05:57:20.477 | INFO | } | 20260505 05:57:20.477 | INFO | Perf report for the last 5.13821 seconds: | 20260505 05:57:20.477 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:57:20.477 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:20.477 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:20.477 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:20.477 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 05:57:20.477 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:20.477 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:20.477 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260505 05:57:20.478 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:20.478 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:20.478 | INFO | Uploaded 36GiB to GPU in 2.17506s (16.5513GiB/s) | 20260505 05:57:20.478 | INFO | Downloaded 34GiB to GPU in 2.1739s (15.6401GiB/s) | 20260505 05:57:25.617 | INFO | Analyzing chunk frames = [871936,872448) | 20260505 05:57:25.617 | INFO | MemInfo: { | 20260505 05:57:25.617 | INFO | "gpu" : " 17 GB", | 20260505 05:57:25.617 | INFO | "huge" : "", | 20260505 05:57:25.617 | INFO | "huge_shared" : " 27 GB", | 20260505 05:57:25.617 | INFO | "malloc" : " 14 GB", | 20260505 05:57:25.617 | INFO | "pinned" : " 29 GB" | 20260505 05:57:25.617 | INFO | } | 20260505 05:57:25.617 | INFO | Perf report for the last 5.13954 seconds: | 20260505 05:57:25.617 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:57:25.617 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:25.617 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:25.617 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:25.617 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 05:57:25.617 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:25.617 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:25.617 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 05:57:25.617 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:25.617 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:25.617 | INFO | Uploaded 36GiB to GPU in 2.07538s (17.3462GiB/s) | 20260505 05:57:25.617 | INFO | Downloaded 34GiB to GPU in 2.07884s (16.3553GiB/s) | 20260505 05:57:25.733 | INFO | Starting a round of Baz IO | 20260505 05:57:29.003 | INFO | Populating RTMetrics report | 20260505 05:57:29.003 | INFO | RTMetrics report populated | 20260505 05:57:29.003 | INFO | Opening tmp RTMetrics json file | 20260505 05:57:29.027 | INFO | Serializing RTMetrics to json | 20260505 05:57:29.037 | INFO | Serialization done, swapping tmp file now... | 20260505 05:57:29.054 | INFO | RTMetrics done | 20260505 05:57:29.116 | INFO | Finished a round of Baz IO: Wrote 2.317 GiB in 3.383 seconds | 20260505 05:57:29.116 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.70% / 0.06% / 64.24% | 20260505 05:57:29.117 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.15373% / 0.000103315% / 8.34465e-05% | 20260505 05:57:29.117 | INFO | Baz IO overall: | 20260505 05:57:29.117 | INFO | BytesWritten : 432256216298 | 20260505 05:57:29.117 | INFO | NumEvents : 203796063868 | 20260505 05:57:30.756 | INFO | Analyzing chunk frames = [872448,872960) | 20260505 05:57:30.756 | INFO | MemInfo: { | 20260505 05:57:30.756 | INFO | "gpu" : " 17 GB", | 20260505 05:57:30.756 | INFO | "huge" : "", | 20260505 05:57:30.756 | INFO | "huge_shared" : " 27 GB", | 20260505 05:57:30.756 | INFO | "malloc" : " 14 GB", | 20260505 05:57:30.756 | INFO | "pinned" : " 29 GB" | 20260505 05:57:30.756 | INFO | } | 20260505 05:57:30.756 | INFO | Perf report for the last 5.13929 seconds: | 20260505 05:57:30.756 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:57:30.756 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:30.756 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:30.756 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:30.756 | INFO | ANALYSIS : 64.7%, 2.9, 99.4ms, 0 | 20260505 05:57:30.756 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:30.756 | INFO | RT_METRICS : 47.0%, 1.0, 24.6ms, 0 | 20260505 05:57:30.756 | INFO | PRE_HQ : 63.8%, 1.9, 64.6ms, 0 | 20260505 05:57:30.756 | INFO | BAZWRITER : 65.1%, 4.0, 134.9ms, 0 | 20260505 05:57:30.756 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 05:57:30.756 | INFO | Uploaded 36GiB to GPU in 2.13519s (16.8603GiB/s) | 20260505 05:57:30.756 | INFO | Downloaded 38GiB to GPU in 2.33031s (16.3068GiB/s) | 20260505 05:57:35.896 | INFO | Analyzing chunk frames = [872960,873472) | 20260505 05:57:35.896 | INFO | MemInfo: { | 20260505 05:57:35.896 | INFO | "gpu" : " 17 GB", | 20260505 05:57:35.896 | INFO | "huge" : "", | 20260505 05:57:35.896 | INFO | "huge_shared" : " 27 GB", | 20260505 05:57:35.896 | INFO | "malloc" : " 14 GB", | 20260505 05:57:35.896 | INFO | "pinned" : " 29 GB" | 20260505 05:57:35.896 | INFO | } | 20260505 05:57:35.896 | INFO | Perf report for the last 5.13985 seconds: | 20260505 05:57:35.896 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:57:35.896 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:35.896 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:35.896 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:35.896 | INFO | ANALYSIS : 64.7%, 3.0, 101.0ms, 0 | 20260505 05:57:35.896 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:35.896 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:35.896 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:57:35.896 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:35.896 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:35.896 | INFO | Uploaded 36GiB to GPU in 2.1426s (16.802GiB/s) | 20260505 05:57:35.896 | INFO | Downloaded 34GiB to GPU in 2.02974s (16.7509GiB/s) | 20260505 05:57:41.035 | INFO | Analyzing chunk frames = [873472,873984) | 20260505 05:57:41.035 | INFO | MemInfo: { | 20260505 05:57:41.035 | INFO | "gpu" : " 17 GB", | 20260505 05:57:41.035 | INFO | "huge" : "", | 20260505 05:57:41.035 | INFO | "huge_shared" : " 27 GB", | 20260505 05:57:41.035 | INFO | "malloc" : " 14 GB", | 20260505 05:57:41.035 | INFO | "pinned" : " 29 GB" | 20260505 05:57:41.035 | INFO | } | 20260505 05:57:41.035 | INFO | Perf report for the last 5.13893 seconds: | 20260505 05:57:41.035 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:57:41.035 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:41.035 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:41.035 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:41.035 | INFO | ANALYSIS : 64.5%, 3.0, 100.3ms, 0 | 20260505 05:57:41.035 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:41.035 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:41.035 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 05:57:41.035 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:41.035 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:41.035 | INFO | Uploaded 36GiB to GPU in 2.18491s (16.4766GiB/s) | 20260505 05:57:41.035 | INFO | Downloaded 34GiB to GPU in 2.03461s (16.7108GiB/s) | 20260505 05:57:46.174 | INFO | Analyzing chunk frames = [873984,874496) | 20260505 05:57:46.174 | INFO | MemInfo: { | 20260505 05:57:46.174 | INFO | "gpu" : " 17 GB", | 20260505 05:57:46.174 | INFO | "huge" : "", | 20260505 05:57:46.174 | INFO | "huge_shared" : " 27 GB", | 20260505 05:57:46.174 | INFO | "malloc" : " 14 GB", | 20260505 05:57:46.174 | INFO | "pinned" : " 29 GB" | 20260505 05:57:46.174 | INFO | } | 20260505 05:57:46.174 | INFO | Perf report for the last 5.13912 seconds: | 20260505 05:57:46.174 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:57:46.174 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:46.174 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:46.174 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:46.174 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260505 05:57:46.174 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:46.174 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:46.174 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:57:46.174 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:46.174 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:46.174 | INFO | Uploaded 36GiB to GPU in 2.10038s (17.1398GiB/s) | 20260505 05:57:46.175 | INFO | Downloaded 34GiB to GPU in 2.01575s (16.8672GiB/s) | 20260505 05:57:51.313 | INFO | Analyzing chunk frames = [874496,875008) | 20260505 05:57:51.313 | INFO | MemInfo: { | 20260505 05:57:51.313 | INFO | "gpu" : " 17 GB", | 20260505 05:57:51.313 | INFO | "huge" : "", | 20260505 05:57:51.313 | INFO | "huge_shared" : " 27 GB", | 20260505 05:57:51.313 | INFO | "malloc" : " 14 GB", | 20260505 05:57:51.313 | INFO | "pinned" : " 29 GB" | 20260505 05:57:51.313 | INFO | } | 20260505 05:57:51.314 | INFO | Perf report for the last 5.13944 seconds: | 20260505 05:57:51.314 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:57:51.314 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:51.314 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:51.314 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:51.314 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 05:57:51.314 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:51.314 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:51.314 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:57:51.314 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:51.314 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:51.314 | INFO | Uploaded 36GiB to GPU in 2.1447s (16.7856GiB/s) | 20260505 05:57:51.314 | INFO | Downloaded 34GiB to GPU in 2.04813s (16.6005GiB/s) | 20260505 05:57:56.453 | INFO | Analyzing chunk frames = [875008,875520) | 20260505 05:57:56.453 | INFO | MemInfo: { | 20260505 05:57:56.453 | INFO | "gpu" : " 17 GB", | 20260505 05:57:56.453 | INFO | "huge" : "", | 20260505 05:57:56.453 | INFO | "huge_shared" : " 27 GB", | 20260505 05:57:56.453 | INFO | "malloc" : " 14 GB", | 20260505 05:57:56.453 | INFO | "pinned" : " 29 GB" | 20260505 05:57:56.453 | INFO | } | 20260505 05:57:56.453 | INFO | Perf report for the last 5.13975 seconds: | 20260505 05:57:56.453 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:57:56.453 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:56.453 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:56.453 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:56.453 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 05:57:56.453 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:56.454 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:56.454 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 05:57:56.454 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:56.454 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:57:56.454 | INFO | Uploaded 36GiB to GPU in 2.14507s (16.7827GiB/s) | 20260505 05:57:56.454 | INFO | Downloaded 34GiB to GPU in 2.05663s (16.5319GiB/s) | 20260505 05:58:01.592 | INFO | Analyzing chunk frames = [875520,876032) | 20260505 05:58:01.592 | INFO | MemInfo: { | 20260505 05:58:01.592 | INFO | "gpu" : " 17 GB", | 20260505 05:58:01.592 | INFO | "huge" : "", | 20260505 05:58:01.592 | INFO | "huge_shared" : " 27 GB", | 20260505 05:58:01.592 | INFO | "malloc" : " 14 GB", | 20260505 05:58:01.592 | INFO | "pinned" : " 29 GB" | 20260505 05:58:01.592 | INFO | } | 20260505 05:58:01.592 | INFO | Perf report for the last 5.13904 seconds: | 20260505 05:58:01.592 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:58:01.592 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:01.592 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:01.592 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:01.592 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 05:58:01.592 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:01.593 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:01.593 | INFO | PRE_HQ : 32.5%, 1.0, 17.0ms, 0 | 20260505 05:58:01.593 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:01.593 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:01.593 | INFO | Uploaded 36GiB to GPU in 2.16388s (16.6368GiB/s) | 20260505 05:58:01.593 | INFO | Downloaded 34GiB to GPU in 2.16143s (15.7303GiB/s) | 20260505 05:58:06.731 | INFO | Analyzing chunk frames = [876032,876544) | 20260505 05:58:06.732 | INFO | MemInfo: { | 20260505 05:58:06.732 | INFO | "gpu" : " 17 GB", | 20260505 05:58:06.732 | INFO | "huge" : "", | 20260505 05:58:06.732 | INFO | "huge_shared" : " 27 GB", | 20260505 05:58:06.732 | INFO | "malloc" : " 14 GB", | 20260505 05:58:06.732 | INFO | "pinned" : " 29 GB" | 20260505 05:58:06.732 | INFO | } | 20260505 05:58:06.732 | INFO | Perf report for the last 5.13925 seconds: | 20260505 05:58:06.732 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:58:06.732 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:06.732 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:06.732 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:06.732 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 05:58:06.732 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:06.732 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:06.732 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260505 05:58:06.732 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:06.732 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:06.732 | INFO | Uploaded 36GiB to GPU in 2.06708s (17.4159GiB/s) | 20260505 05:58:06.732 | INFO | Downloaded 34GiB to GPU in 2.0866s (16.2944GiB/s) | 20260505 05:58:06.732 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 05:58:06.732 | INFO | Total mem usage: 15.3 GiB | 20260505 05:58:06.732 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 05:58:06.732 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.49%) | 20260505 05:58:06.732 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 05:58:06.732 | INFO | High Water Breakdown: | 20260505 05:58:06.732 | INFO | 60.1% : 8.64 GiB ( 92.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 05:58:06.732 | INFO | 39.8% : 5.73 GiB ( 86.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 05:58:06.732 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 05:58:06.732 | INFO | ------------------------------------------ | 20260505 05:58:06.732 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:58:06.732 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 05:58:06.732 | INFO | Total mem usage: 26.5 GiB | 20260505 05:58:06.732 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 05:58:06.732 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 05:58:06.732 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 05:58:06.732 | INFO | High Water Breakdown: | 20260505 05:58:06.732 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:58:06.732 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:58:06.732 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:58:06.732 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:58:06.732 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:58:06.733 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:58:06.733 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 05:58:06.733 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 05:58:06.733 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:58:06.733 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 05:58:06.733 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 05:58:06.733 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 05:58:06.733 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 05:58:06.733 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 05:58:06.733 | INFO | ------------------------------------------ | 20260505 05:58:06.733 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:58:06.733 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 05:58:06.733 | INFO | Total mem usage: 25.6 GiB | 20260505 05:58:06.733 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 05:58:06.733 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 05:58:06.733 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 05:58:06.733 | INFO | High Water Breakdown: | 20260505 05:58:06.733 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 05:58:06.733 | INFO | ------------------------------------------ | 20260505 05:58:06.733 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:58:06.733 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 05:58:06.733 | INFO | Total mem usage: 15.7 GiB | 20260505 05:58:06.733 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 05:58:06.733 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 05:58:06.733 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260505 05:58:06.733 | INFO | High Water Breakdown: | 20260505 05:58:06.733 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 05:58:06.733 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 05:58:06.733 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 05:58:06.733 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 05:58:06.733 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 05:58:06.733 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 05:58:06.733 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 05:58:06.733 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 05:58:06.733 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 05:58:06.733 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:58:06.733 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 05:58:06.733 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:58:06.733 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 05:58:06.733 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 05:58:06.733 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 05:58:06.733 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 05:58:06.733 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:58:06.733 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 05:58:06.733 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:58:06.733 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:58:06.733 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:58:06.733 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 05:58:06.733 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 05:58:06.733 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:58:06.733 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 05:58:06.733 | INFO | ------------------------------------------ | 20260505 05:58:06.733 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:58:06.733 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 05:58:06.733 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:58:06.733 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:58:06.733 | INFO | PerfReport: Full Execution -- 3.33e+01 / 7.63e+01, 1.39e+01 / 1.60e+00 | 20260505 05:58:06.733 | INFO | PerfReport: 0.28%, AggregateStats -- 9.41e-02 / 1.05e-01, 1.09e-02 / 4.29e-03 | 20260505 05:58:06.733 | INFO | PerfReport: 11.56%, Baseline -- 3.86e+00 / 4.57e+00, 6.46e-01 / 2.21e-01 | 20260505 05:58:06.733 | INFO | PerfReport: 3.31%, Histogram -- 1.10e+00 / 1.54e+00, 2.25e-01 / 8.07e-02 | 20260505 05:58:06.733 | INFO | PerfReport: 15.53%, DME -- 4.14e+01 / 4.55e+01, 4.36e+00 / 1.22e+00 | 20260505 05:58:06.733 | INFO | PerfReport: 55.81%, FrameLabeling -- 1.86e+01 / 1.90e+01, 2.09e+00 / 7.86e-01 | 20260505 05:58:06.733 | INFO | PerfReport: 7.82%, PulseAccumulating -- 2.61e+00 / 3.23e+00, 4.97e-01 / 2.92e-01 | 20260505 05:58:06.733 | INFO | PerfReport: 4.17%, Metrics -- 1.39e+00 / 1.78e+00, 3.93e-01 / 2.27e-01 | 20260505 05:58:06.734 | INFO | PerfReport: 1.52%, PulseSerializing -- 5.07e-01 / 5.89e-01, 9.99e-02 / 4.40e-02 | 20260505 05:58:06.734 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 05:58:06.734 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:58:06.734 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:58:06.734 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.31e+01 / 0.00e+00 | 20260505 05:58:06.734 | INFO | PerfReport: 21.78%, Upload -- 2.21e+01 / 0.00e+00, 7.51e+00 / 0.00e+00 | 20260505 05:58:06.734 | INFO | PerfReport: 21.48%, Download -- 2.18e+01 / 0.00e+00, 7.68e+00 / 0.00e+00 | 20260505 05:58:06.734 | INFO | PerfReport: 32.81%, Compute -- 3.34e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 05:58:06.734 | INFO | PerfReport: 1.45%, LockUpload -- 1.48e+00 / 0.00e+00, 6.67e+00 / 0.00e+00 | 20260505 05:58:06.734 | INFO | PerfReport: 0.66%, LockDownload -- 6.71e-01 / 0.00e+00, 1.95e+00 / 0.00e+00 | 20260505 05:58:06.734 | INFO | PerfReport: 21.79%, LockCompute -- 2.21e+01 / 0.00e+00, 1.66e+01 / 0.00e+00 | 20260505 05:58:06.734 | INFO | PerfReport: 0.01%, Misc -- 1.49e-02 / 0.00e+00, 2.40e-02 / 0.00e+00 | 20260505 05:58:06.846 | INFO | Starting a round of Baz IO | 20260505 05:58:10.115 | INFO | Populating RTMetrics report | 20260505 05:58:10.115 | INFO | RTMetrics report populated | 20260505 05:58:10.115 | INFO | Opening tmp RTMetrics json file | 20260505 05:58:10.146 | INFO | Serializing RTMetrics to json | 20260505 05:58:10.169 | INFO | Serialization done, swapping tmp file now... | 20260505 05:58:10.171 | INFO | RTMetrics done | 20260505 05:58:10.231 | INFO | Finished a round of Baz IO: Wrote 2.317 GiB in 3.385 seconds | 20260505 05:58:10.231 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.64% / 0.06% / 64.30% | 20260505 05:58:10.231 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.11453% / 9.13938e-05% / 7.54992e-05% | 20260505 05:58:10.231 | INFO | Baz IO overall: | 20260505 05:58:10.232 | INFO | BytesWritten : 434744166048 | 20260505 05:58:10.232 | INFO | NumEvents : 204984537053 | 20260505 05:58:11.871 | INFO | Analyzing chunk frames = [876544,877056) | 20260505 05:58:11.871 | INFO | MemInfo: { | 20260505 05:58:11.871 | INFO | "gpu" : " 17 GB", | 20260505 05:58:11.871 | INFO | "huge" : "", | 20260505 05:58:11.871 | INFO | "huge_shared" : " 27 GB", | 20260505 05:58:11.871 | INFO | "malloc" : " 14 GB", | 20260505 05:58:11.871 | INFO | "pinned" : " 29 GB" | 20260505 05:58:11.871 | INFO | } | 20260505 05:58:11.871 | INFO | Perf report for the last 5.13955 seconds: | 20260505 05:58:11.871 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:58:11.871 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:11.871 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:11.871 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:11.871 | INFO | ANALYSIS : 64.7%, 2.9, 98.8ms, 0 | 20260505 05:58:11.871 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:11.871 | INFO | RT_METRICS : 47.0%, 1.0, 24.6ms, 0 | 20260505 05:58:11.871 | INFO | PRE_HQ : 63.8%, 1.9, 64.5ms, 0 | 20260505 05:58:11.871 | INFO | BAZWRITER : 65.2%, 4.1, 136.2ms, 0 | 20260505 05:58:11.871 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260505 05:58:11.871 | INFO | Uploaded 36GiB to GPU in 2.13352s (16.8735GiB/s) | 20260505 05:58:11.871 | INFO | Downloaded 38GiB to GPU in 2.33033s (16.3067GiB/s) | 20260505 05:58:17.010 | INFO | Analyzing chunk frames = [877056,877568) | 20260505 05:58:17.010 | INFO | MemInfo: { | 20260505 05:58:17.010 | INFO | "gpu" : " 17 GB", | 20260505 05:58:17.010 | INFO | "huge" : "", | 20260505 05:58:17.010 | INFO | "huge_shared" : " 27 GB", | 20260505 05:58:17.010 | INFO | "malloc" : " 14 GB", | 20260505 05:58:17.010 | INFO | "pinned" : " 29 GB" | 20260505 05:58:17.010 | INFO | } | 20260505 05:58:17.010 | INFO | Perf report for the last 5.13923 seconds: | 20260505 05:58:17.010 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:58:17.010 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:17.010 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:17.010 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:17.010 | INFO | ANALYSIS : 64.7%, 3.0, 101.0ms, 0 | 20260505 05:58:17.010 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:17.010 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:17.010 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 05:58:17.011 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:17.011 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:17.011 | INFO | Uploaded 36GiB to GPU in 2.1386s (16.8334GiB/s) | 20260505 05:58:17.011 | INFO | Downloaded 34GiB to GPU in 2.03678s (16.693GiB/s) | 20260505 05:58:22.150 | INFO | Analyzing chunk frames = [877568,878080) | 20260505 05:58:22.150 | INFO | MemInfo: { | 20260505 05:58:22.150 | INFO | "gpu" : " 17 GB", | 20260505 05:58:22.150 | INFO | "huge" : "", | 20260505 05:58:22.150 | INFO | "huge_shared" : " 27 GB", | 20260505 05:58:22.150 | INFO | "malloc" : " 14 GB", | 20260505 05:58:22.150 | INFO | "pinned" : " 29 GB" | 20260505 05:58:22.150 | INFO | } | 20260505 05:58:22.150 | INFO | Perf report for the last 5.13977 seconds: | 20260505 05:58:22.150 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:58:22.150 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:22.150 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:22.150 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:22.150 | INFO | ANALYSIS : 64.4%, 3.0, 100.3ms, 0 | 20260505 05:58:22.150 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:22.150 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:22.150 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260505 05:58:22.150 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:22.150 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:22.150 | INFO | Uploaded 36GiB to GPU in 2.17577s (16.5459GiB/s) | 20260505 05:58:22.150 | INFO | Downloaded 34GiB to GPU in 2.03396s (16.7161GiB/s) | 20260505 05:58:27.289 | INFO | Analyzing chunk frames = [878080,878592) | 20260505 05:58:27.289 | INFO | MemInfo: { | 20260505 05:58:27.289 | INFO | "gpu" : " 17 GB", | 20260505 05:58:27.289 | INFO | "huge" : "", | 20260505 05:58:27.289 | INFO | "huge_shared" : " 27 GB", | 20260505 05:58:27.289 | INFO | "malloc" : " 14 GB", | 20260505 05:58:27.289 | INFO | "pinned" : " 29 GB" | 20260505 05:58:27.289 | INFO | } | 20260505 05:58:27.289 | INFO | Perf report for the last 5.13895 seconds: | 20260505 05:58:27.289 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:58:27.289 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:27.289 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:27.289 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:27.289 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260505 05:58:27.289 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:27.289 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:27.289 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:58:27.289 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:27.289 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:27.289 | INFO | Uploaded 36GiB to GPU in 2.1149s (17.0221GiB/s) | 20260505 05:58:27.289 | INFO | Downloaded 34GiB to GPU in 2.035s (16.7076GiB/s) | 20260505 05:58:32.428 | INFO | Analyzing chunk frames = [878592,879104) | 20260505 05:58:32.428 | INFO | MemInfo: { | 20260505 05:58:32.428 | INFO | "gpu" : " 17 GB", | 20260505 05:58:32.428 | INFO | "huge" : "", | 20260505 05:58:32.428 | INFO | "huge_shared" : " 27 GB", | 20260505 05:58:32.428 | INFO | "malloc" : " 14 GB", | 20260505 05:58:32.428 | INFO | "pinned" : " 29 GB" | 20260505 05:58:32.428 | INFO | } | 20260505 05:58:32.428 | INFO | Perf report for the last 5.13922 seconds: | 20260505 05:58:32.428 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:58:32.428 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:32.428 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:32.428 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:32.428 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 05:58:32.428 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:32.428 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:32.428 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 05:58:32.428 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:32.429 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:32.429 | INFO | Uploaded 36GiB to GPU in 2.13305s (16.8772GiB/s) | 20260505 05:58:32.429 | INFO | Downloaded 34GiB to GPU in 2.04377s (16.6359GiB/s) | 20260505 05:58:37.568 | INFO | Analyzing chunk frames = [879104,879616) | 20260505 05:58:37.568 | INFO | MemInfo: { | 20260505 05:58:37.568 | INFO | "gpu" : " 17 GB", | 20260505 05:58:37.568 | INFO | "huge" : "", | 20260505 05:58:37.568 | INFO | "huge_shared" : " 27 GB", | 20260505 05:58:37.568 | INFO | "malloc" : " 14 GB", | 20260505 05:58:37.568 | INFO | "pinned" : " 29 GB" | 20260505 05:58:37.569 | INFO | } | 20260505 05:58:37.569 | INFO | Perf report for the last 5.1403 seconds: | 20260505 05:58:37.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:58:37.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:37.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:37.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:37.569 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 05:58:37.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:37.569 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:37.569 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 05:58:37.569 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:37.569 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:37.569 | INFO | Uploaded 36GiB to GPU in 2.14892s (16.7526GiB/s) | 20260505 05:58:37.569 | INFO | Downloaded 34GiB to GPU in 2.05856s (16.5164GiB/s) | 20260505 05:58:42.707 | INFO | Analyzing chunk frames = [879616,880128) | 20260505 05:58:42.707 | INFO | MemInfo: { | 20260505 05:58:42.707 | INFO | "gpu" : " 17 GB", | 20260505 05:58:42.707 | INFO | "huge" : "", | 20260505 05:58:42.707 | INFO | "huge_shared" : " 27 GB", | 20260505 05:58:42.707 | INFO | "malloc" : " 14 GB", | 20260505 05:58:42.707 | INFO | "pinned" : " 29 GB" | 20260505 05:58:42.707 | INFO | } | 20260505 05:58:42.707 | INFO | Perf report for the last 5.1384 seconds: | 20260505 05:58:42.707 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:58:42.707 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:42.707 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:42.707 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:42.707 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 05:58:42.707 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:42.707 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:42.707 | INFO | PRE_HQ : 32.6%, 1.0, 17.0ms, 0 | 20260505 05:58:42.707 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:42.707 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:42.707 | INFO | Uploaded 36GiB to GPU in 2.17415s (16.5582GiB/s) | 20260505 05:58:42.707 | INFO | Downloaded 34GiB to GPU in 2.15035s (15.8114GiB/s) | 20260505 05:58:47.846 | INFO | Analyzing chunk frames = [880128,880640) | 20260505 05:58:47.846 | INFO | MemInfo: { | 20260505 05:58:47.846 | INFO | "gpu" : " 17 GB", | 20260505 05:58:47.846 | INFO | "huge" : "", | 20260505 05:58:47.846 | INFO | "huge_shared" : " 27 GB", | 20260505 05:58:47.846 | INFO | "malloc" : " 14 GB", | 20260505 05:58:47.846 | INFO | "pinned" : " 29 GB" | 20260505 05:58:47.846 | INFO | } | 20260505 05:58:47.846 | INFO | Perf report for the last 5.13911 seconds: | 20260505 05:58:47.846 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:58:47.846 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:47.846 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:47.846 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:47.846 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 05:58:47.846 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:47.846 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:47.846 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 05:58:47.846 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:47.846 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:47.847 | INFO | Uploaded 36GiB to GPU in 2.06862s (17.4029GiB/s) | 20260505 05:58:47.847 | INFO | Downloaded 34GiB to GPU in 2.08027s (16.3441GiB/s) | 20260505 05:58:47.962 | INFO | Starting a round of Baz IO | 20260505 05:58:51.229 | INFO | Populating RTMetrics report | 20260505 05:58:51.230 | INFO | RTMetrics report populated | 20260505 05:58:51.230 | INFO | Opening tmp RTMetrics json file | 20260505 05:58:51.246 | INFO | Serializing RTMetrics to json | 20260505 05:58:51.251 | INFO | Serialization done, swapping tmp file now... | 20260505 05:58:51.253 | INFO | RTMetrics done | 20260505 05:58:51.341 | INFO | Finished a round of Baz IO: Wrote 2.324 GiB in 3.379 seconds | 20260505 05:58:51.341 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.57% / 0.06% / 64.36% | 20260505 05:58:51.341 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.12036% / 0.000127157% / 7.15256e-05% | 20260505 05:58:51.341 | INFO | Baz IO overall: | 20260505 05:58:51.341 | INFO | BytesWritten : 437239094858 | 20260505 05:58:51.341 | INFO | NumEvents : 206177725117 | 20260505 05:58:52.986 | INFO | Analyzing chunk frames = [880640,881152) | 20260505 05:58:52.986 | INFO | MemInfo: { | 20260505 05:58:52.986 | INFO | "gpu" : " 17 GB", | 20260505 05:58:52.986 | INFO | "huge" : "", | 20260505 05:58:52.986 | INFO | "huge_shared" : " 27 GB", | 20260505 05:58:52.986 | INFO | "malloc" : " 14 GB", | 20260505 05:58:52.986 | INFO | "pinned" : " 29 GB" | 20260505 05:58:52.986 | INFO | } | 20260505 05:58:52.986 | INFO | Perf report for the last 5.13993 seconds: | 20260505 05:58:52.986 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:58:52.986 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:52.986 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:52.986 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:52.986 | INFO | ANALYSIS : 64.7%, 2.9, 98.7ms, 0 | 20260505 05:58:52.986 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:52.986 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260505 05:58:52.986 | INFO | PRE_HQ : 63.8%, 2.0, 64.7ms, 0 | 20260505 05:58:52.986 | INFO | BAZWRITER : 65.0%, 4.1, 135.8ms, 0 | 20260505 05:58:52.986 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 05:58:52.987 | INFO | Uploaded 36GiB to GPU in 2.13036s (16.8986GiB/s) | 20260505 05:58:52.987 | INFO | Downloaded 38GiB to GPU in 2.32768s (16.3253GiB/s) | 20260505 05:58:58.125 | INFO | Analyzing chunk frames = [881152,881664) | 20260505 05:58:58.125 | INFO | MemInfo: { | 20260505 05:58:58.125 | INFO | "gpu" : " 17 GB", | 20260505 05:58:58.125 | INFO | "huge" : "", | 20260505 05:58:58.125 | INFO | "huge_shared" : " 27 GB", | 20260505 05:58:58.125 | INFO | "malloc" : " 14 GB", | 20260505 05:58:58.125 | INFO | "pinned" : " 29 GB" | 20260505 05:58:58.125 | INFO | } | 20260505 05:58:58.126 | INFO | Perf report for the last 5.13935 seconds: | 20260505 05:58:58.126 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:58:58.126 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:58.126 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:58.126 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:58.126 | INFO | ANALYSIS : 64.6%, 3.0, 100.8ms, 0 | 20260505 05:58:58.126 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:58.126 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:58.126 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 05:58:58.126 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:58.126 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:58:58.126 | INFO | Uploaded 36GiB to GPU in 2.13856s (16.8338GiB/s) | 20260505 05:58:58.126 | INFO | Downloaded 34GiB to GPU in 2.04096s (16.6588GiB/s) | 20260505 05:59:03.264 | INFO | Analyzing chunk frames = [881664,882176) | 20260505 05:59:03.264 | INFO | MemInfo: { | 20260505 05:59:03.264 | INFO | "gpu" : " 17 GB", | 20260505 05:59:03.264 | INFO | "huge" : "", | 20260505 05:59:03.264 | INFO | "huge_shared" : " 27 GB", | 20260505 05:59:03.264 | INFO | "malloc" : " 14 GB", | 20260505 05:59:03.264 | INFO | "pinned" : " 29 GB" | 20260505 05:59:03.264 | INFO | } | 20260505 05:59:03.265 | INFO | Perf report for the last 5.13902 seconds: | 20260505 05:59:03.265 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:59:03.265 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:03.265 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:03.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:03.265 | INFO | ANALYSIS : 64.4%, 3.0, 100.3ms, 0 | 20260505 05:59:03.265 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:03.265 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:03.265 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 05:59:03.265 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:03.265 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:03.265 | INFO | Uploaded 36GiB to GPU in 2.17201s (16.5745GiB/s) | 20260505 05:59:03.265 | INFO | Downloaded 34GiB to GPU in 2.03418s (16.7143GiB/s) | 20260505 05:59:08.404 | INFO | Analyzing chunk frames = [882176,882688) | 20260505 05:59:08.404 | INFO | MemInfo: { | 20260505 05:59:08.404 | INFO | "gpu" : " 17 GB", | 20260505 05:59:08.404 | INFO | "huge" : "", | 20260505 05:59:08.404 | INFO | "huge_shared" : " 27 GB", | 20260505 05:59:08.404 | INFO | "malloc" : " 14 GB", | 20260505 05:59:08.404 | INFO | "pinned" : " 29 GB" | 20260505 05:59:08.404 | INFO | } | 20260505 05:59:08.404 | INFO | Perf report for the last 5.1392 seconds: | 20260505 05:59:08.404 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:59:08.404 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:08.404 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:08.404 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:08.404 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260505 05:59:08.404 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:08.404 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:08.404 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 05:59:08.404 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:08.404 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:08.404 | INFO | Uploaded 36GiB to GPU in 2.09703s (17.1672GiB/s) | 20260505 05:59:08.404 | INFO | Downloaded 34GiB to GPU in 2.02078s (16.8251GiB/s) | 20260505 05:59:13.543 | INFO | Analyzing chunk frames = [882688,883200) | 20260505 05:59:13.543 | INFO | MemInfo: { | 20260505 05:59:13.543 | INFO | "gpu" : " 17 GB", | 20260505 05:59:13.543 | INFO | "huge" : "", | 20260505 05:59:13.543 | INFO | "huge_shared" : " 27 GB", | 20260505 05:59:13.543 | INFO | "malloc" : " 14 GB", | 20260505 05:59:13.543 | INFO | "pinned" : " 29 GB" | 20260505 05:59:13.543 | INFO | } | 20260505 05:59:13.543 | INFO | Perf report for the last 5.13959 seconds: | 20260505 05:59:13.543 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:59:13.543 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:13.543 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:13.543 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:13.543 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 05:59:13.543 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:13.543 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:13.543 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 05:59:13.544 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:13.544 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:13.544 | INFO | Uploaded 36GiB to GPU in 2.14532s (16.7807GiB/s) | 20260505 05:59:13.544 | INFO | Downloaded 34GiB to GPU in 2.04625s (16.6158GiB/s) | 20260505 05:59:18.683 | INFO | Analyzing chunk frames = [883200,883712) | 20260505 05:59:18.683 | INFO | MemInfo: { | 20260505 05:59:18.683 | INFO | "gpu" : " 17 GB", | 20260505 05:59:18.683 | INFO | "huge" : "", | 20260505 05:59:18.683 | INFO | "huge_shared" : " 27 GB", | 20260505 05:59:18.683 | INFO | "malloc" : " 14 GB", | 20260505 05:59:18.683 | INFO | "pinned" : " 29 GB" | 20260505 05:59:18.683 | INFO | } | 20260505 05:59:18.684 | INFO | Perf report for the last 5.14016 seconds: | 20260505 05:59:18.684 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:59:18.684 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:18.684 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:18.684 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:18.684 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 05:59:18.684 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:18.684 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:18.684 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 05:59:18.684 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:18.684 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:18.684 | INFO | Uploaded 36GiB to GPU in 2.09626s (17.1734GiB/s) | 20260505 05:59:18.684 | INFO | Downloaded 34GiB to GPU in 2.0553s (16.5426GiB/s) | 20260505 05:59:23.822 | INFO | Analyzing chunk frames = [883712,884224) | 20260505 05:59:23.822 | INFO | MemInfo: { | 20260505 05:59:23.822 | INFO | "gpu" : " 17 GB", | 20260505 05:59:23.822 | INFO | "huge" : "", | 20260505 05:59:23.822 | INFO | "huge_shared" : " 27 GB", | 20260505 05:59:23.822 | INFO | "malloc" : " 14 GB", | 20260505 05:59:23.822 | INFO | "pinned" : " 29 GB" | 20260505 05:59:23.822 | INFO | } | 20260505 05:59:23.822 | INFO | Perf report for the last 5.13856 seconds: | 20260505 05:59:23.822 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:59:23.822 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:23.822 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:23.822 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:23.822 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 05:59:23.822 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:23.822 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:23.822 | INFO | PRE_HQ : 32.7%, 1.0, 17.0ms, 0 | 20260505 05:59:23.822 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:23.822 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:23.822 | INFO | Uploaded 36GiB to GPU in 2.1533s (16.7185GiB/s) | 20260505 05:59:23.822 | INFO | Downloaded 34GiB to GPU in 2.15948s (15.7446GiB/s) | 20260505 05:59:28.961 | INFO | Analyzing chunk frames = [884224,884736) | 20260505 05:59:28.961 | INFO | MemInfo: { | 20260505 05:59:28.961 | INFO | "gpu" : " 17 GB", | 20260505 05:59:28.961 | INFO | "huge" : "", | 20260505 05:59:28.961 | INFO | "huge_shared" : " 27 GB", | 20260505 05:59:28.961 | INFO | "malloc" : " 14 GB", | 20260505 05:59:28.961 | INFO | "pinned" : " 29 GB" | 20260505 05:59:28.961 | INFO | } | 20260505 05:59:28.961 | INFO | Perf report for the last 5.13936 seconds: | 20260505 05:59:28.961 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:59:28.961 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:28.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:28.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:28.962 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 05:59:28.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:28.962 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:28.962 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 05:59:28.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:28.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:28.962 | INFO | Uploaded 36GiB to GPU in 2.06501s (17.4333GiB/s) | 20260505 05:59:28.962 | INFO | Downloaded 34GiB to GPU in 2.07928s (16.3518GiB/s) | 20260505 05:59:28.962 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 05:59:28.962 | INFO | Total mem usage: 15.3 GiB | 20260505 05:59:28.962 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 05:59:28.962 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.43%) | 20260505 05:59:28.962 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 05:59:28.962 | INFO | High Water Breakdown: | 20260505 05:59:28.962 | INFO | 60.1% : 8.65 GiB ( 92.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 05:59:28.962 | INFO | 39.8% : 5.73 GiB ( 86.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 05:59:28.962 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 05:59:28.962 | INFO | ------------------------------------------ | 20260505 05:59:28.962 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:59:28.962 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 05:59:28.962 | INFO | Total mem usage: 26.5 GiB | 20260505 05:59:28.962 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 05:59:28.962 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 05:59:28.962 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 05:59:28.962 | INFO | High Water Breakdown: | 20260505 05:59:28.962 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:59:28.962 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:59:28.962 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:59:28.962 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:59:28.962 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:59:28.962 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:59:28.962 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 05:59:28.962 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 05:59:28.962 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:59:28.962 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 05:59:28.963 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 05:59:28.963 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 05:59:28.963 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 05:59:28.963 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 05:59:28.963 | INFO | ------------------------------------------ | 20260505 05:59:28.963 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:59:28.963 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 05:59:28.963 | INFO | Total mem usage: 25.6 GiB | 20260505 05:59:28.963 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 05:59:28.963 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 05:59:28.963 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 05:59:28.963 | INFO | High Water Breakdown: | 20260505 05:59:28.963 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 05:59:28.963 | INFO | ------------------------------------------ | 20260505 05:59:28.963 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:59:28.963 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 05:59:28.963 | INFO | Total mem usage: 15.7 GiB | 20260505 05:59:28.963 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 05:59:28.963 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 05:59:28.963 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260505 05:59:28.963 | INFO | High Water Breakdown: | 20260505 05:59:28.963 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 05:59:28.963 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 05:59:28.963 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 05:59:28.963 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 05:59:28.963 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 05:59:28.963 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 05:59:28.963 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 05:59:28.963 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 05:59:28.963 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 05:59:28.963 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 05:59:28.963 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 05:59:28.963 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 05:59:28.963 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 05:59:28.963 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 05:59:28.963 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 05:59:28.963 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 05:59:28.963 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 05:59:28.963 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 05:59:28.963 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 05:59:28.963 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 05:59:28.963 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 05:59:28.963 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 05:59:28.963 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 05:59:28.963 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 05:59:28.963 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 05:59:28.963 | INFO | ------------------------------------------ | 20260505 05:59:28.963 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 05:59:28.963 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 05:59:28.963 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:59:28.963 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:59:28.963 | INFO | PerfReport: Full Execution -- 3.34e+01 / 7.64e+01, 1.39e+01 / 1.34e+00 | 20260505 05:59:28.963 | INFO | PerfReport: 0.28%, AggregateStats -- 9.41e-02 / 1.05e-01, 1.09e-02 / 4.64e-03 | 20260505 05:59:28.963 | INFO | PerfReport: 11.55%, Baseline -- 3.85e+00 / 4.58e+00, 6.40e-01 / 1.93e-01 | 20260505 05:59:28.963 | INFO | PerfReport: 3.31%, Histogram -- 1.10e+00 / 1.54e+00, 2.22e-01 / 8.35e-02 | 20260505 05:59:28.963 | INFO | PerfReport: 15.52%, DME -- 4.14e+01 / 4.55e+01, 4.48e+00 / 1.27e+00 | 20260505 05:59:28.963 | INFO | PerfReport: 55.84%, FrameLabeling -- 1.86e+01 / 1.91e+01, 2.11e+00 / 1.01e+00 | 20260505 05:59:28.963 | INFO | PerfReport: 7.80%, PulseAccumulating -- 2.60e+00 / 3.20e+00, 4.94e-01 / 3.27e-01 | 20260505 05:59:28.963 | INFO | PerfReport: 4.17%, Metrics -- 1.39e+00 / 1.77e+00, 3.92e-01 / 2.15e-01 | 20260505 05:59:28.963 | INFO | PerfReport: 1.52%, PulseSerializing -- 5.08e-01 / 5.88e-01, 1.00e-01 / 4.41e-02 | 20260505 05:59:28.963 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 05:59:28.963 | INFO | PerfReport: Gathered 1536 samples. | 20260505 05:59:28.963 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 05:59:28.963 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.31e+01 / 0.00e+00 | 20260505 05:59:28.963 | INFO | PerfReport: 21.83%, Upload -- 2.22e+01 / 0.00e+00, 7.54e+00 / 0.00e+00 | 20260505 05:59:28.963 | INFO | PerfReport: 21.48%, Download -- 2.18e+01 / 0.00e+00, 7.64e+00 / 0.00e+00 | 20260505 05:59:28.963 | INFO | PerfReport: 32.82%, Compute -- 3.34e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 05:59:28.963 | INFO | PerfReport: 1.46%, LockUpload -- 1.49e+00 / 0.00e+00, 6.65e+00 / 0.00e+00 | 20260505 05:59:28.963 | INFO | PerfReport: 0.63%, LockDownload -- 6.39e-01 / 0.00e+00, 1.85e+00 / 0.00e+00 | 20260505 05:59:28.963 | INFO | PerfReport: 21.77%, LockCompute -- 2.21e+01 / 0.00e+00, 1.67e+01 / 0.00e+00 | 20260505 05:59:28.963 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.50e-02 / 0.00e+00 | 20260505 05:59:29.076 | INFO | Starting a round of Baz IO | 20260505 05:59:32.339 | INFO | Populating RTMetrics report | 20260505 05:59:32.339 | INFO | RTMetrics report populated | 20260505 05:59:32.339 | INFO | Opening tmp RTMetrics json file | 20260505 05:59:32.354 | INFO | Serializing RTMetrics to json | 20260505 05:59:32.355 | INFO | Serialization done, swapping tmp file now... | 20260505 05:59:32.355 | INFO | RTMetrics done | 20260505 05:59:32.452 | INFO | Finished a round of Baz IO: Wrote 2.327 GiB in 3.376 seconds | 20260505 05:59:32.452 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.51% / 0.06% / 64.43% | 20260505 05:59:32.452 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.10843% / 8.74201e-05% / 6.35783e-05% | 20260505 05:59:32.452 | INFO | Baz IO overall: | 20260505 05:59:32.452 | INFO | BytesWritten : 439737722074 | 20260505 05:59:32.452 | INFO | NumEvents : 207373051360 | 20260505 05:59:34.100 | INFO | Analyzing chunk frames = [884736,885248) | 20260505 05:59:34.100 | INFO | MemInfo: { | 20260505 05:59:34.100 | INFO | "gpu" : " 17 GB", | 20260505 05:59:34.100 | INFO | "huge" : "", | 20260505 05:59:34.100 | INFO | "huge_shared" : " 27 GB", | 20260505 05:59:34.100 | INFO | "malloc" : " 14 GB", | 20260505 05:59:34.100 | INFO | "pinned" : " 29 GB" | 20260505 05:59:34.100 | INFO | } | 20260505 05:59:34.100 | INFO | Perf report for the last 5.13901 seconds: | 20260505 05:59:34.100 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:59:34.100 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:34.101 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:34.101 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:34.101 | INFO | ANALYSIS : 64.6%, 2.9, 99.1ms, 0 | 20260505 05:59:34.101 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:34.101 | INFO | RT_METRICS : 46.4%, 1.0, 24.4ms, 0 | 20260505 05:59:34.101 | INFO | PRE_HQ : 63.7%, 2.0, 64.7ms, 0 | 20260505 05:59:34.101 | INFO | BAZWRITER : 65.1%, 4.0, 135.2ms, 0 | 20260505 05:59:34.101 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 05:59:34.101 | INFO | Uploaded 36GiB to GPU in 2.12644s (16.9297GiB/s) | 20260505 05:59:34.101 | INFO | Downloaded 38GiB to GPU in 2.32325s (16.3564GiB/s) | 20260505 05:59:39.240 | INFO | Analyzing chunk frames = [885248,885760) | 20260505 05:59:39.240 | INFO | MemInfo: { | 20260505 05:59:39.240 | INFO | "gpu" : " 17 GB", | 20260505 05:59:39.240 | INFO | "huge" : "", | 20260505 05:59:39.240 | INFO | "huge_shared" : " 27 GB", | 20260505 05:59:39.240 | INFO | "malloc" : " 14 GB", | 20260505 05:59:39.240 | INFO | "pinned" : " 29 GB" | 20260505 05:59:39.240 | INFO | } | 20260505 05:59:39.240 | INFO | Perf report for the last 5.13936 seconds: | 20260505 05:59:39.240 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:59:39.240 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:39.240 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:39.240 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:39.240 | INFO | ANALYSIS : 64.7%, 3.0, 101.0ms, 0 | 20260505 05:59:39.240 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:39.240 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:39.240 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 05:59:39.240 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:39.240 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:39.240 | INFO | Uploaded 36GiB to GPU in 2.14064s (16.8174GiB/s) | 20260505 05:59:39.240 | INFO | Downloaded 34GiB to GPU in 2.03663s (16.6942GiB/s) | 20260505 05:59:44.379 | INFO | Analyzing chunk frames = [885760,886272) | 20260505 05:59:44.379 | INFO | MemInfo: { | 20260505 05:59:44.379 | INFO | "gpu" : " 17 GB", | 20260505 05:59:44.379 | INFO | "huge" : "", | 20260505 05:59:44.379 | INFO | "huge_shared" : " 27 GB", | 20260505 05:59:44.379 | INFO | "malloc" : " 14 GB", | 20260505 05:59:44.379 | INFO | "pinned" : " 29 GB" | 20260505 05:59:44.379 | INFO | } | 20260505 05:59:44.380 | INFO | Perf report for the last 5.13965 seconds: | 20260505 05:59:44.380 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:59:44.380 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:44.380 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:44.380 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:44.380 | INFO | ANALYSIS : 64.3%, 3.0, 100.1ms, 0 | 20260505 05:59:44.380 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:44.380 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:44.380 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 05:59:44.380 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:44.380 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:44.380 | INFO | Uploaded 36GiB to GPU in 2.17931s (16.519GiB/s) | 20260505 05:59:44.380 | INFO | Downloaded 34GiB to GPU in 2.03569s (16.7019GiB/s) | 20260505 05:59:49.518 | INFO | Analyzing chunk frames = [886272,886784) | 20260505 05:59:49.518 | INFO | MemInfo: { | 20260505 05:59:49.518 | INFO | "gpu" : " 17 GB", | 20260505 05:59:49.518 | INFO | "huge" : "", | 20260505 05:59:49.518 | INFO | "huge_shared" : " 27 GB", | 20260505 05:59:49.518 | INFO | "malloc" : " 14 GB", | 20260505 05:59:49.518 | INFO | "pinned" : " 29 GB" | 20260505 05:59:49.518 | INFO | } | 20260505 05:59:49.519 | INFO | Perf report for the last 5.13898 seconds: | 20260505 05:59:49.519 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:59:49.519 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:49.519 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:49.519 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:49.519 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260505 05:59:49.519 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:49.519 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:49.519 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260505 05:59:49.519 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:49.519 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:49.519 | INFO | Uploaded 36GiB to GPU in 2.09102s (17.2165GiB/s) | 20260505 05:59:49.519 | INFO | Downloaded 34GiB to GPU in 2.01416s (16.8805GiB/s) | 20260505 05:59:54.657 | INFO | Analyzing chunk frames = [886784,887296) | 20260505 05:59:54.657 | INFO | MemInfo: { | 20260505 05:59:54.657 | INFO | "gpu" : " 17 GB", | 20260505 05:59:54.657 | INFO | "huge" : "", | 20260505 05:59:54.657 | INFO | "huge_shared" : " 27 GB", | 20260505 05:59:54.657 | INFO | "malloc" : " 14 GB", | 20260505 05:59:54.658 | INFO | "pinned" : " 29 GB" | 20260505 05:59:54.658 | INFO | } | 20260505 05:59:54.658 | INFO | Perf report for the last 5.13903 seconds: | 20260505 05:59:54.658 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:59:54.658 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:54.658 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:54.658 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:54.658 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 05:59:54.658 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:54.658 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:54.658 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 05:59:54.658 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:54.658 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:54.658 | INFO | Uploaded 36GiB to GPU in 2.11835s (16.9944GiB/s) | 20260505 05:59:54.658 | INFO | Downloaded 34GiB to GPU in 2.0496s (16.5886GiB/s) | 20260505 05:59:59.798 | INFO | Analyzing chunk frames = [887296,887808) | 20260505 05:59:59.798 | INFO | MemInfo: { | 20260505 05:59:59.798 | INFO | "gpu" : " 17 GB", | 20260505 05:59:59.798 | INFO | "huge" : "", | 20260505 05:59:59.798 | INFO | "huge_shared" : " 27 GB", | 20260505 05:59:59.798 | INFO | "malloc" : " 14 GB", | 20260505 05:59:59.798 | INFO | "pinned" : " 29 GB" | 20260505 05:59:59.798 | INFO | } | 20260505 05:59:59.798 | INFO | Perf report for the last 5.14056 seconds: | 20260505 05:59:59.798 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 05:59:59.798 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:59.798 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:59.798 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:59.798 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 05:59:59.798 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:59.798 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:59.798 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 05:59:59.798 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:59.798 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 05:59:59.798 | INFO | Uploaded 36GiB to GPU in 2.08551s (17.262GiB/s) | 20260505 05:59:59.798 | INFO | Downloaded 34GiB to GPU in 2.04803s (16.6013GiB/s) | 20260505 06:00:04.937 | INFO | Analyzing chunk frames = [887808,888320) | 20260505 06:00:04.937 | INFO | MemInfo: { | 20260505 06:00:04.937 | INFO | "gpu" : " 17 GB", | 20260505 06:00:04.937 | INFO | "huge" : "", | 20260505 06:00:04.937 | INFO | "huge_shared" : " 27 GB", | 20260505 06:00:04.937 | INFO | "malloc" : " 14 GB", | 20260505 06:00:04.937 | INFO | "pinned" : " 29 GB" | 20260505 06:00:04.937 | INFO | } | 20260505 06:00:04.937 | INFO | Perf report for the last 5.13867 seconds: | 20260505 06:00:04.937 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:00:04.937 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:04.937 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:04.937 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:04.937 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 06:00:04.937 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:04.937 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:04.937 | INFO | PRE_HQ : 32.7%, 1.0, 17.0ms, 0 | 20260505 06:00:04.937 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:04.937 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:04.937 | INFO | Uploaded 36GiB to GPU in 2.19924s (16.3693GiB/s) | 20260505 06:00:04.937 | INFO | Downloaded 34GiB to GPU in 2.18186s (15.5831GiB/s) | 20260505 06:00:10.076 | INFO | Analyzing chunk frames = [888320,888832) | 20260505 06:00:10.076 | INFO | MemInfo: { | 20260505 06:00:10.076 | INFO | "gpu" : " 17 GB", | 20260505 06:00:10.076 | INFO | "huge" : "", | 20260505 06:00:10.076 | INFO | "huge_shared" : " 27 GB", | 20260505 06:00:10.076 | INFO | "malloc" : " 14 GB", | 20260505 06:00:10.076 | INFO | "pinned" : " 29 GB" | 20260505 06:00:10.076 | INFO | } | 20260505 06:00:10.076 | INFO | Perf report for the last 5.13933 seconds: | 20260505 06:00:10.076 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:00:10.076 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:10.076 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:10.076 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:10.076 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 06:00:10.076 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:10.076 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:10.076 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 06:00:10.076 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:10.076 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:10.076 | INFO | Uploaded 36GiB to GPU in 2.07468s (17.352GiB/s) | 20260505 06:00:10.076 | INFO | Downloaded 34GiB to GPU in 2.08587s (16.3002GiB/s) | 20260505 06:00:10.190 | INFO | Starting a round of Baz IO | 20260505 06:00:13.446 | INFO | Populating RTMetrics report | 20260505 06:00:13.446 | INFO | RTMetrics report populated | 20260505 06:00:13.446 | INFO | Opening tmp RTMetrics json file | 20260505 06:00:13.473 | INFO | Serializing RTMetrics to json | 20260505 06:00:13.512 | INFO | Serialization done, swapping tmp file now... | 20260505 06:00:13.519 | INFO | RTMetrics done | 20260505 06:00:13.553 | INFO | Finished a round of Baz IO: Wrote 2.324 GiB in 3.363 seconds | 20260505 06:00:13.553 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.45% / 0.06% / 64.49% | 20260505 06:00:13.553 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.04683% / 9.13938e-05% / 7.15256e-05% | 20260505 06:00:13.553 | INFO | Baz IO overall: | 20260505 06:00:13.553 | INFO | BytesWritten : 442232694282 | 20260505 06:00:13.553 | INFO | NumEvents : 208565024757 | 20260505 06:00:15.215 | INFO | Analyzing chunk frames = [888832,889344) | 20260505 06:00:15.215 | INFO | MemInfo: { | 20260505 06:00:15.215 | INFO | "gpu" : " 17 GB", | 20260505 06:00:15.215 | INFO | "huge" : "", | 20260505 06:00:15.215 | INFO | "huge_shared" : " 27 GB", | 20260505 06:00:15.215 | INFO | "malloc" : " 14 GB", | 20260505 06:00:15.215 | INFO | "pinned" : " 29 GB" | 20260505 06:00:15.215 | INFO | } | 20260505 06:00:15.215 | INFO | Perf report for the last 5.13929 seconds: | 20260505 06:00:15.215 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:00:15.215 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:15.215 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:15.215 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:15.216 | INFO | ANALYSIS : 64.4%, 2.9, 98.5ms, 0 | 20260505 06:00:15.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:15.216 | INFO | RT_METRICS : 47.5%, 1.0, 24.9ms, 0 | 20260505 06:00:15.216 | INFO | PRE_HQ : 63.6%, 2.0, 64.9ms, 0 | 20260505 06:00:15.216 | INFO | BAZWRITER : 64.7%, 4.0, 134.2ms, 0 | 20260505 06:00:15.216 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 06:00:15.216 | INFO | Uploaded 36GiB to GPU in 2.09399s (17.1921GiB/s) | 20260505 06:00:15.216 | INFO | Downloaded 38GiB to GPU in 2.31366s (16.4242GiB/s) | 20260505 06:00:20.355 | INFO | Analyzing chunk frames = [889344,889856) | 20260505 06:00:20.355 | INFO | MemInfo: { | 20260505 06:00:20.355 | INFO | "gpu" : " 17 GB", | 20260505 06:00:20.355 | INFO | "huge" : "", | 20260505 06:00:20.355 | INFO | "huge_shared" : " 27 GB", | 20260505 06:00:20.355 | INFO | "malloc" : " 14 GB", | 20260505 06:00:20.355 | INFO | "pinned" : " 29 GB" | 20260505 06:00:20.355 | INFO | } | 20260505 06:00:20.355 | INFO | Perf report for the last 5.1394 seconds: | 20260505 06:00:20.355 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:00:20.355 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:20.355 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:20.355 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:20.355 | INFO | ANALYSIS : 64.6%, 3.0, 100.9ms, 0 | 20260505 06:00:20.355 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:20.355 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:20.355 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 06:00:20.355 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:20.355 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:20.355 | INFO | Uploaded 36GiB to GPU in 2.13925s (16.8283GiB/s) | 20260505 06:00:20.355 | INFO | Downloaded 34GiB to GPU in 2.031s (16.7405GiB/s) | 20260505 06:00:25.494 | INFO | Analyzing chunk frames = [889856,890368) | 20260505 06:00:25.494 | INFO | MemInfo: { | 20260505 06:00:25.494 | INFO | "gpu" : " 17 GB", | 20260505 06:00:25.494 | INFO | "huge" : "", | 20260505 06:00:25.494 | INFO | "huge_shared" : " 27 GB", | 20260505 06:00:25.494 | INFO | "malloc" : " 14 GB", | 20260505 06:00:25.494 | INFO | "pinned" : " 29 GB" | 20260505 06:00:25.494 | INFO | } | 20260505 06:00:25.494 | INFO | Perf report for the last 5.13929 seconds: | 20260505 06:00:25.494 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:00:25.494 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:25.494 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:25.494 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:25.494 | INFO | ANALYSIS : 64.3%, 3.0, 100.0ms, 0 | 20260505 06:00:25.494 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:25.494 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:25.494 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 06:00:25.494 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:25.494 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:25.494 | INFO | Uploaded 36GiB to GPU in 2.16944s (16.5942GiB/s) | 20260505 06:00:25.494 | INFO | Downloaded 34GiB to GPU in 2.03357s (16.7194GiB/s) | 20260505 06:00:30.633 | INFO | Analyzing chunk frames = [890368,890880) | 20260505 06:00:30.633 | INFO | MemInfo: { | 20260505 06:00:30.633 | INFO | "gpu" : " 17 GB", | 20260505 06:00:30.633 | INFO | "huge" : "", | 20260505 06:00:30.633 | INFO | "huge_shared" : " 27 GB", | 20260505 06:00:30.633 | INFO | "malloc" : " 14 GB", | 20260505 06:00:30.633 | INFO | "pinned" : " 29 GB" | 20260505 06:00:30.633 | INFO | } | 20260505 06:00:30.633 | INFO | Perf report for the last 5.13905 seconds: | 20260505 06:00:30.633 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:00:30.633 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:30.633 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:30.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:30.633 | INFO | ANALYSIS : 63.6%, 3.0, 98.7ms, 0 | 20260505 06:00:30.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:30.633 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:30.633 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 06:00:30.633 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:30.633 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:30.634 | INFO | Uploaded 36GiB to GPU in 2.10163s (17.1296GiB/s) | 20260505 06:00:30.634 | INFO | Downloaded 34GiB to GPU in 2.02159s (16.8185GiB/s) | 20260505 06:00:35.772 | INFO | Analyzing chunk frames = [890880,891392) | 20260505 06:00:35.772 | INFO | MemInfo: { | 20260505 06:00:35.772 | INFO | "gpu" : " 17 GB", | 20260505 06:00:35.772 | INFO | "huge" : "", | 20260505 06:00:35.772 | INFO | "huge_shared" : " 27 GB", | 20260505 06:00:35.772 | INFO | "malloc" : " 14 GB", | 20260505 06:00:35.772 | INFO | "pinned" : " 29 GB" | 20260505 06:00:35.772 | INFO | } | 20260505 06:00:35.773 | INFO | Perf report for the last 5.13944 seconds: | 20260505 06:00:35.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:00:35.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:35.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:35.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:35.773 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 06:00:35.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:35.773 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:35.773 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 06:00:35.773 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:35.773 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:35.773 | INFO | Uploaded 36GiB to GPU in 2.1176s (17.0004GiB/s) | 20260505 06:00:35.773 | INFO | Downloaded 34GiB to GPU in 2.03564s (16.7023GiB/s) | 20260505 06:00:40.913 | INFO | Analyzing chunk frames = [891392,891904) | 20260505 06:00:40.913 | INFO | MemInfo: { | 20260505 06:00:40.913 | INFO | "gpu" : " 17 GB", | 20260505 06:00:40.913 | INFO | "huge" : "", | 20260505 06:00:40.913 | INFO | "huge_shared" : " 27 GB", | 20260505 06:00:40.913 | INFO | "malloc" : " 14 GB", | 20260505 06:00:40.913 | INFO | "pinned" : " 29 GB" | 20260505 06:00:40.913 | INFO | } | 20260505 06:00:40.913 | INFO | Perf report for the last 5.14049 seconds: | 20260505 06:00:40.913 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:00:40.913 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:40.913 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:40.913 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:40.913 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 06:00:40.913 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:40.913 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:40.913 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 06:00:40.913 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:40.913 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:40.914 | INFO | Uploaded 36GiB to GPU in 2.13212s (16.8846GiB/s) | 20260505 06:00:40.914 | INFO | Downloaded 34GiB to GPU in 2.06678s (16.4507GiB/s) | 20260505 06:00:46.051 | INFO | Analyzing chunk frames = [891904,892416) | 20260505 06:00:46.051 | INFO | MemInfo: { | 20260505 06:00:46.051 | INFO | "gpu" : " 17 GB", | 20260505 06:00:46.051 | INFO | "huge" : "", | 20260505 06:00:46.051 | INFO | "huge_shared" : " 27 GB", | 20260505 06:00:46.051 | INFO | "malloc" : " 14 GB", | 20260505 06:00:46.051 | INFO | "pinned" : " 29 GB" | 20260505 06:00:46.051 | INFO | } | 20260505 06:00:46.052 | INFO | Perf report for the last 5.13829 seconds: | 20260505 06:00:46.052 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:00:46.052 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:46.052 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:46.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:46.052 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 06:00:46.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:46.052 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:46.052 | INFO | PRE_HQ : 32.5%, 1.0, 17.0ms, 0 | 20260505 06:00:46.052 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:46.052 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:46.052 | INFO | Uploaded 36GiB to GPU in 2.15775s (16.684GiB/s) | 20260505 06:00:46.052 | INFO | Downloaded 34GiB to GPU in 2.17536s (15.6296GiB/s) | 20260505 06:00:51.191 | INFO | Analyzing chunk frames = [892416,892928) | 20260505 06:00:51.191 | INFO | MemInfo: { | 20260505 06:00:51.191 | INFO | "gpu" : " 17 GB", | 20260505 06:00:51.191 | INFO | "huge" : "", | 20260505 06:00:51.191 | INFO | "huge_shared" : " 27 GB", | 20260505 06:00:51.191 | INFO | "malloc" : " 14 GB", | 20260505 06:00:51.191 | INFO | "pinned" : " 29 GB" | 20260505 06:00:51.191 | INFO | } | 20260505 06:00:51.191 | INFO | Perf report for the last 5.13915 seconds: | 20260505 06:00:51.191 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:00:51.191 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:51.191 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:51.191 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:51.191 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 06:00:51.191 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:51.191 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:51.191 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260505 06:00:51.191 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:51.191 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:51.191 | INFO | Uploaded 36GiB to GPU in 2.06503s (17.4331GiB/s) | 20260505 06:00:51.191 | INFO | Downloaded 34GiB to GPU in 2.08763s (16.2864GiB/s) | 20260505 06:00:51.191 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 06:00:51.191 | INFO | Total mem usage: 15.3 GiB | 20260505 06:00:51.191 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 06:00:51.191 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -5.55%) | 20260505 06:00:51.191 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 06:00:51.191 | INFO | High Water Breakdown: | 20260505 06:00:51.191 | INFO | 60.3% : 8.75 GiB ( 93.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 06:00:51.191 | INFO | 39.6% : 5.75 GiB ( 86.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 06:00:51.191 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 06:00:51.191 | INFO | ------------------------------------------ | 20260505 06:00:51.191 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:00:51.192 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 06:00:51.192 | INFO | Total mem usage: 26.5 GiB | 20260505 06:00:51.192 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 06:00:51.192 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 06:00:51.192 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 06:00:51.192 | INFO | High Water Breakdown: | 20260505 06:00:51.192 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:00:51.192 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:00:51.192 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:00:51.192 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:00:51.192 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:00:51.192 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:00:51.192 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 06:00:51.192 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 06:00:51.192 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:00:51.192 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 06:00:51.192 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 06:00:51.192 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 06:00:51.192 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 06:00:51.192 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 06:00:51.192 | INFO | ------------------------------------------ | 20260505 06:00:51.192 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:00:51.192 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 06:00:51.192 | INFO | Total mem usage: 25.6 GiB | 20260505 06:00:51.192 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 06:00:51.192 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 06:00:51.192 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 06:00:51.192 | INFO | High Water Breakdown: | 20260505 06:00:51.192 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 06:00:51.192 | INFO | ------------------------------------------ | 20260505 06:00:51.192 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:00:51.192 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 06:00:51.192 | INFO | Total mem usage: 15.7 GiB | 20260505 06:00:51.192 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 06:00:51.192 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 06:00:51.192 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260505 06:00:51.192 | INFO | High Water Breakdown: | 20260505 06:00:51.192 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 06:00:51.192 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 06:00:51.192 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 06:00:51.192 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 06:00:51.192 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 06:00:51.192 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 06:00:51.192 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 06:00:51.192 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 06:00:51.192 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 06:00:51.192 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:00:51.192 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 06:00:51.192 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:00:51.192 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 06:00:51.193 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 06:00:51.193 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 06:00:51.193 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 06:00:51.193 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:00:51.193 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 06:00:51.193 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:00:51.193 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:00:51.193 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:00:51.193 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 06:00:51.193 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 06:00:51.193 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:00:51.193 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 06:00:51.193 | INFO | ------------------------------------------ | 20260505 06:00:51.193 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:00:51.193 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 06:00:51.193 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:00:51.193 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:00:51.193 | INFO | PerfReport: Full Execution -- 3.33e+01 / 7.63e+01, 1.40e+01 / 1.48e+00 | 20260505 06:00:51.193 | INFO | PerfReport: 0.28%, AggregateStats -- 9.43e-02 / 1.03e-01, 1.15e-02 / 6.57e-03 | 20260505 06:00:51.193 | INFO | PerfReport: 11.56%, Baseline -- 3.85e+00 / 4.58e+00, 6.40e-01 / 2.09e-01 | 20260505 06:00:51.193 | INFO | PerfReport: 3.30%, Histogram -- 1.10e+00 / 1.52e+00, 2.21e-01 / 1.12e-01 | 20260505 06:00:51.193 | INFO | PerfReport: 15.52%, DME -- 4.14e+01 / 4.54e+01, 4.43e+00 / 1.13e+00 | 20260505 06:00:51.193 | INFO | PerfReport: 55.82%, FrameLabeling -- 1.86e+01 / 1.91e+01, 2.08e+00 / 1.03e+00 | 20260505 06:00:51.193 | INFO | PerfReport: 7.83%, PulseAccumulating -- 2.61e+00 / 3.21e+00, 4.96e-01 / 2.92e-01 | 20260505 06:00:51.193 | INFO | PerfReport: 4.16%, Metrics -- 1.39e+00 / 1.80e+00, 3.93e-01 / 2.47e-01 | 20260505 06:00:51.193 | INFO | PerfReport: 1.52%, PulseSerializing -- 5.07e-01 / 5.89e-01, 1.01e-01 / 4.59e-02 | 20260505 06:00:51.193 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 06:00:51.193 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:00:51.193 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:00:51.193 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.31e+01 / 0.00e+00 | 20260505 06:00:51.193 | INFO | PerfReport: 21.78%, Upload -- 2.21e+01 / 0.00e+00, 7.51e+00 / 0.00e+00 | 20260505 06:00:51.193 | INFO | PerfReport: 21.49%, Download -- 2.18e+01 / 0.00e+00, 7.64e+00 / 0.00e+00 | 20260505 06:00:51.193 | INFO | PerfReport: 32.80%, Compute -- 3.33e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260505 06:00:51.193 | INFO | PerfReport: 1.47%, LockUpload -- 1.50e+00 / 0.00e+00, 6.69e+00 / 0.00e+00 | 20260505 06:00:51.193 | INFO | PerfReport: 0.65%, LockDownload -- 6.65e-01 / 0.00e+00, 1.88e+00 / 0.00e+00 | 20260505 06:00:51.193 | INFO | PerfReport: 21.78%, LockCompute -- 2.21e+01 / 0.00e+00, 1.67e+01 / 0.00e+00 | 20260505 06:00:51.193 | INFO | PerfReport: 0.01%, Misc -- 1.47e-02 / 0.00e+00, 2.27e-02 / 0.00e+00 | 20260505 06:00:51.303 | INFO | Starting a round of Baz IO | 20260505 06:00:54.547 | INFO | Populating RTMetrics report | 20260505 06:00:54.547 | INFO | RTMetrics report populated | 20260505 06:00:54.547 | INFO | Opening tmp RTMetrics json file | 20260505 06:00:54.560 | INFO | Serializing RTMetrics to json | 20260505 06:00:54.561 | INFO | Serialization done, swapping tmp file now... | 20260505 06:00:54.561 | INFO | RTMetrics done | 20260505 06:00:54.661 | INFO | Finished a round of Baz IO: Wrote 2.328 GiB in 3.358 seconds | 20260505 06:00:54.661 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.39% / 0.06% / 64.55% | 20260505 06:00:54.661 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.04155% / 0.000115236% / 4.76837e-05% | 20260505 06:00:54.661 | INFO | Baz IO overall: | 20260505 06:00:54.661 | INFO | BytesWritten : 444732059737 | 20260505 06:00:54.661 | INFO | NumEvents : 209759642637 | 20260505 06:00:56.330 | INFO | Analyzing chunk frames = [892928,893440) | 20260505 06:00:56.330 | INFO | MemInfo: { | 20260505 06:00:56.330 | INFO | "gpu" : " 17 GB", | 20260505 06:00:56.330 | INFO | "huge" : "", | 20260505 06:00:56.330 | INFO | "huge_shared" : " 27 GB", | 20260505 06:00:56.330 | INFO | "malloc" : " 14 GB", | 20260505 06:00:56.330 | INFO | "pinned" : " 29 GB" | 20260505 06:00:56.330 | INFO | } | 20260505 06:00:56.330 | INFO | Perf report for the last 5.13969 seconds: | 20260505 06:00:56.330 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:00:56.330 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:56.330 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:56.330 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:56.330 | INFO | ANALYSIS : 64.1%, 2.9, 98.2ms, 0 | 20260505 06:00:56.330 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:00:56.331 | INFO | RT_METRICS : 46.2%, 1.0, 24.2ms, 0 | 20260505 06:00:56.331 | INFO | PRE_HQ : 63.5%, 2.0, 64.6ms, 0 | 20260505 06:00:56.331 | INFO | BAZWRITER : 64.6%, 4.1, 135.5ms, 0 | 20260505 06:00:56.331 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 06:00:56.331 | INFO | Uploaded 36GiB to GPU in 2.09614s (17.1745GiB/s) | 20260505 06:00:56.331 | INFO | Downloaded 38GiB to GPU in 2.31655s (16.4037GiB/s) | 20260505 06:01:01.469 | INFO | Analyzing chunk frames = [893440,893952) | 20260505 06:01:01.469 | INFO | MemInfo: { | 20260505 06:01:01.469 | INFO | "gpu" : " 17 GB", | 20260505 06:01:01.469 | INFO | "huge" : "", | 20260505 06:01:01.469 | INFO | "huge_shared" : " 27 GB", | 20260505 06:01:01.469 | INFO | "malloc" : " 14 GB", | 20260505 06:01:01.469 | INFO | "pinned" : " 29 GB" | 20260505 06:01:01.469 | INFO | } | 20260505 06:01:01.470 | INFO | Perf report for the last 5.13917 seconds: | 20260505 06:01:01.470 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:01:01.470 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:01.470 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:01.470 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:01.470 | INFO | ANALYSIS : 64.6%, 3.0, 100.9ms, 0 | 20260505 06:01:01.470 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:01.470 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:01.470 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 06:01:01.470 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:01.470 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:01.470 | INFO | Uploaded 36GiB to GPU in 2.1365s (16.85GiB/s) | 20260505 06:01:01.470 | INFO | Downloaded 34GiB to GPU in 2.03254s (16.7279GiB/s) | 20260505 06:01:06.608 | INFO | Analyzing chunk frames = [893952,894464) | 20260505 06:01:06.609 | INFO | MemInfo: { | 20260505 06:01:06.609 | INFO | "gpu" : " 17 GB", | 20260505 06:01:06.609 | INFO | "huge" : "", | 20260505 06:01:06.609 | INFO | "huge_shared" : " 27 GB", | 20260505 06:01:06.609 | INFO | "malloc" : " 14 GB", | 20260505 06:01:06.609 | INFO | "pinned" : " 29 GB" | 20260505 06:01:06.609 | INFO | } | 20260505 06:01:06.609 | INFO | Perf report for the last 5.13918 seconds: | 20260505 06:01:06.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:01:06.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:06.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:06.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:06.609 | INFO | ANALYSIS : 64.3%, 3.0, 100.1ms, 0 | 20260505 06:01:06.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:06.609 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:06.609 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 06:01:06.609 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:06.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:06.609 | INFO | Uploaded 36GiB to GPU in 2.18319s (16.4896GiB/s) | 20260505 06:01:06.609 | INFO | Downloaded 34GiB to GPU in 2.03129s (16.7381GiB/s) | 20260505 06:01:11.748 | INFO | Analyzing chunk frames = [894464,894976) | 20260505 06:01:11.748 | INFO | MemInfo: { | 20260505 06:01:11.748 | INFO | "gpu" : " 17 GB", | 20260505 06:01:11.748 | INFO | "huge" : "", | 20260505 06:01:11.748 | INFO | "huge_shared" : " 27 GB", | 20260505 06:01:11.748 | INFO | "malloc" : " 14 GB", | 20260505 06:01:11.748 | INFO | "pinned" : " 29 GB" | 20260505 06:01:11.748 | INFO | } | 20260505 06:01:11.748 | INFO | Perf report for the last 5.13926 seconds: | 20260505 06:01:11.748 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:01:11.748 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:11.748 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:11.748 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:11.748 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260505 06:01:11.748 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:11.748 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:11.748 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 06:01:11.748 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:11.748 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:11.748 | INFO | Uploaded 36GiB to GPU in 2.10904s (17.0693GiB/s) | 20260505 06:01:11.748 | INFO | Downloaded 34GiB to GPU in 2.03512s (16.7066GiB/s) | 20260505 06:01:16.887 | INFO | Analyzing chunk frames = [894976,895488) | 20260505 06:01:16.887 | INFO | MemInfo: { | 20260505 06:01:16.887 | INFO | "gpu" : " 17 GB", | 20260505 06:01:16.887 | INFO | "huge" : "", | 20260505 06:01:16.887 | INFO | "huge_shared" : " 27 GB", | 20260505 06:01:16.887 | INFO | "malloc" : " 14 GB", | 20260505 06:01:16.887 | INFO | "pinned" : " 29 GB" | 20260505 06:01:16.887 | INFO | } | 20260505 06:01:16.887 | INFO | Perf report for the last 5.13949 seconds: | 20260505 06:01:16.887 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:01:16.887 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:16.887 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:16.888 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:16.888 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 06:01:16.888 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:16.888 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:16.888 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 06:01:16.888 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:16.888 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:16.888 | INFO | Uploaded 36GiB to GPU in 2.147s (16.7676GiB/s) | 20260505 06:01:16.888 | INFO | Downloaded 34GiB to GPU in 2.04394s (16.6346GiB/s) | 20260505 06:01:22.027 | INFO | Analyzing chunk frames = [895488,896000) | 20260505 06:01:22.027 | INFO | MemInfo: { | 20260505 06:01:22.027 | INFO | "gpu" : " 17 GB", | 20260505 06:01:22.027 | INFO | "huge" : "", | 20260505 06:01:22.027 | INFO | "huge_shared" : " 27 GB", | 20260505 06:01:22.027 | INFO | "malloc" : " 14 GB", | 20260505 06:01:22.027 | INFO | "pinned" : " 29 GB" | 20260505 06:01:22.028 | INFO | } | 20260505 06:01:22.028 | INFO | Perf report for the last 5.14017 seconds: | 20260505 06:01:22.028 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:01:22.028 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:22.028 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:22.028 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:22.028 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 06:01:22.028 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:22.028 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:22.028 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 06:01:22.028 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:22.028 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:22.028 | INFO | Uploaded 36GiB to GPU in 2.13657s (16.8494GiB/s) | 20260505 06:01:22.028 | INFO | Downloaded 34GiB to GPU in 2.0644s (16.4696GiB/s) | 20260505 06:01:27.166 | INFO | Analyzing chunk frames = [896000,896512) | 20260505 06:01:27.166 | INFO | MemInfo: { | 20260505 06:01:27.166 | INFO | "gpu" : " 17 GB", | 20260505 06:01:27.166 | INFO | "huge" : "", | 20260505 06:01:27.166 | INFO | "huge_shared" : " 27 GB", | 20260505 06:01:27.166 | INFO | "malloc" : " 14 GB", | 20260505 06:01:27.166 | INFO | "pinned" : " 29 GB" | 20260505 06:01:27.166 | INFO | } | 20260505 06:01:27.166 | INFO | Perf report for the last 5.13853 seconds: | 20260505 06:01:27.166 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:01:27.166 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:27.166 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:27.166 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:27.166 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 06:01:27.166 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:27.166 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:27.166 | INFO | PRE_HQ : 32.6%, 1.0, 16.9ms, 0 | 20260505 06:01:27.166 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:27.166 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:27.167 | INFO | Uploaded 36GiB to GPU in 2.18293s (16.4916GiB/s) | 20260505 06:01:27.167 | INFO | Downloaded 34GiB to GPU in 2.15929s (15.7459GiB/s) | 20260505 06:01:32.305 | INFO | Analyzing chunk frames = [896512,897024) | 20260505 06:01:32.305 | INFO | MemInfo: { | 20260505 06:01:32.305 | INFO | "gpu" : " 17 GB", | 20260505 06:01:32.305 | INFO | "huge" : "", | 20260505 06:01:32.305 | INFO | "huge_shared" : " 27 GB", | 20260505 06:01:32.305 | INFO | "malloc" : " 14 GB", | 20260505 06:01:32.305 | INFO | "pinned" : " 29 GB" | 20260505 06:01:32.305 | INFO | } | 20260505 06:01:32.305 | INFO | Perf report for the last 5.13916 seconds: | 20260505 06:01:32.305 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:01:32.305 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:32.305 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:32.305 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:32.305 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 06:01:32.305 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:32.305 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:32.306 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 06:01:32.306 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:32.306 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:32.306 | INFO | Uploaded 36GiB to GPU in 2.08116s (17.2981GiB/s) | 20260505 06:01:32.306 | INFO | Downloaded 34GiB to GPU in 2.08389s (16.3156GiB/s) | 20260505 06:01:32.419 | INFO | Starting a round of Baz IO | 20260505 06:01:35.684 | INFO | Populating RTMetrics report | 20260505 06:01:35.684 | INFO | RTMetrics report populated | 20260505 06:01:35.684 | INFO | Opening tmp RTMetrics json file | 20260505 06:01:35.705 | INFO | Serializing RTMetrics to json | 20260505 06:01:35.722 | INFO | Serialization done, swapping tmp file now... | 20260505 06:01:35.730 | INFO | RTMetrics done | 20260505 06:01:35.795 | INFO | Finished a round of Baz IO: Wrote 2.326 GiB in 3.376 seconds | 20260505 06:01:35.796 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.33% / 0.06% / 64.61% | 20260505 06:01:35.796 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.99509% / 0.000103315% / 4.76837e-05% | 20260505 06:01:35.796 | INFO | Baz IO overall: | 20260505 06:01:35.796 | INFO | BytesWritten : 447229994045 | 20260505 06:01:35.796 | INFO | NumEvents : 210953157583 | 20260505 06:01:37.445 | INFO | Analyzing chunk frames = [897024,897536) | 20260505 06:01:37.445 | INFO | MemInfo: { | 20260505 06:01:37.445 | INFO | "gpu" : " 17 GB", | 20260505 06:01:37.445 | INFO | "huge" : "", | 20260505 06:01:37.445 | INFO | "huge_shared" : " 27 GB", | 20260505 06:01:37.445 | INFO | "malloc" : " 14 GB", | 20260505 06:01:37.445 | INFO | "pinned" : " 29 GB" | 20260505 06:01:37.445 | INFO | } | 20260505 06:01:37.445 | INFO | Perf report for the last 5.13945 seconds: | 20260505 06:01:37.445 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:01:37.445 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:37.445 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:37.445 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:37.445 | INFO | ANALYSIS : 64.6%, 2.9, 99.6ms, 0 | 20260505 06:01:37.445 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:37.445 | INFO | RT_METRICS : 46.9%, 1.0, 24.6ms, 0 | 20260505 06:01:37.445 | INFO | PRE_HQ : 63.7%, 1.9, 64.2ms, 0 | 20260505 06:01:37.445 | INFO | BAZWRITER : 65.0%, 4.1, 135.5ms, 0 | 20260505 06:01:37.445 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 06:01:37.445 | INFO | Uploaded 36GiB to GPU in 2.13791s (16.8389GiB/s) | 20260505 06:01:37.445 | INFO | Downloaded 38GiB to GPU in 2.32133s (16.37GiB/s) | 20260505 06:01:42.584 | INFO | Analyzing chunk frames = [897536,898048) | 20260505 06:01:42.584 | INFO | MemInfo: { | 20260505 06:01:42.584 | INFO | "gpu" : " 17 GB", | 20260505 06:01:42.584 | INFO | "huge" : "", | 20260505 06:01:42.584 | INFO | "huge_shared" : " 27 GB", | 20260505 06:01:42.584 | INFO | "malloc" : " 14 GB", | 20260505 06:01:42.584 | INFO | "pinned" : " 29 GB" | 20260505 06:01:42.584 | INFO | } | 20260505 06:01:42.584 | INFO | Perf report for the last 5.13931 seconds: | 20260505 06:01:42.584 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:01:42.584 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:42.584 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:42.584 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:42.584 | INFO | ANALYSIS : 64.5%, 3.0, 100.7ms, 0 | 20260505 06:01:42.584 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:42.584 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:42.584 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 06:01:42.584 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:42.584 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:42.584 | INFO | Uploaded 36GiB to GPU in 2.13455s (16.8654GiB/s) | 20260505 06:01:42.584 | INFO | Downloaded 34GiB to GPU in 2.03566s (16.7022GiB/s) | 20260505 06:01:47.723 | INFO | Analyzing chunk frames = [898048,898560) | 20260505 06:01:47.723 | INFO | MemInfo: { | 20260505 06:01:47.723 | INFO | "gpu" : " 17 GB", | 20260505 06:01:47.724 | INFO | "huge" : "", | 20260505 06:01:47.724 | INFO | "huge_shared" : " 27 GB", | 20260505 06:01:47.724 | INFO | "malloc" : " 14 GB", | 20260505 06:01:47.724 | INFO | "pinned" : " 29 GB" | 20260505 06:01:47.724 | INFO | } | 20260505 06:01:47.724 | INFO | Perf report for the last 5.13957 seconds: | 20260505 06:01:47.724 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:01:47.724 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:47.724 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:47.724 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:47.724 | INFO | ANALYSIS : 64.3%, 3.0, 100.1ms, 0 | 20260505 06:01:47.724 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:47.724 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:47.724 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 06:01:47.724 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:47.724 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:47.724 | INFO | Uploaded 36GiB to GPU in 2.17367s (16.5619GiB/s) | 20260505 06:01:47.724 | INFO | Downloaded 34GiB to GPU in 2.03644s (16.6958GiB/s) | 20260505 06:01:52.863 | INFO | Analyzing chunk frames = [898560,899072) | 20260505 06:01:52.863 | INFO | MemInfo: { | 20260505 06:01:52.863 | INFO | "gpu" : " 17 GB", | 20260505 06:01:52.863 | INFO | "huge" : "", | 20260505 06:01:52.863 | INFO | "huge_shared" : " 27 GB", | 20260505 06:01:52.863 | INFO | "malloc" : " 14 GB", | 20260505 06:01:52.863 | INFO | "pinned" : " 29 GB" | 20260505 06:01:52.863 | INFO | } | 20260505 06:01:52.863 | INFO | Perf report for the last 5.13925 seconds: | 20260505 06:01:52.863 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:01:52.863 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:52.863 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:52.863 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:52.863 | INFO | ANALYSIS : 63.4%, 3.0, 98.6ms, 0 | 20260505 06:01:52.863 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:52.863 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:52.863 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 06:01:52.863 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:52.863 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:52.863 | INFO | Uploaded 36GiB to GPU in 2.10102s (17.1345GiB/s) | 20260505 06:01:52.864 | INFO | Downloaded 34GiB to GPU in 2.03147s (16.7366GiB/s) | 20260505 06:01:58.002 | INFO | Analyzing chunk frames = [899072,899584) | 20260505 06:01:58.002 | INFO | MemInfo: { | 20260505 06:01:58.002 | INFO | "gpu" : " 17 GB", | 20260505 06:01:58.002 | INFO | "huge" : "", | 20260505 06:01:58.002 | INFO | "huge_shared" : " 27 GB", | 20260505 06:01:58.002 | INFO | "malloc" : " 14 GB", | 20260505 06:01:58.002 | INFO | "pinned" : " 29 GB" | 20260505 06:01:58.002 | INFO | } | 20260505 06:01:58.002 | INFO | Perf report for the last 5.13907 seconds: | 20260505 06:01:58.002 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:01:58.002 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:58.002 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:58.002 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:58.002 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 06:01:58.002 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:58.002 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:58.002 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 06:01:58.002 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:58.002 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:01:58.002 | INFO | Uploaded 36GiB to GPU in 2.11767s (16.9998GiB/s) | 20260505 06:01:58.003 | INFO | Downloaded 34GiB to GPU in 2.04061s (16.6617GiB/s) | 20260505 06:02:03.142 | INFO | Analyzing chunk frames = [899584,900096) | 20260505 06:02:03.142 | INFO | MemInfo: { | 20260505 06:02:03.142 | INFO | "gpu" : " 17 GB", | 20260505 06:02:03.142 | INFO | "huge" : "", | 20260505 06:02:03.142 | INFO | "huge_shared" : " 27 GB", | 20260505 06:02:03.142 | INFO | "malloc" : " 14 GB", | 20260505 06:02:03.142 | INFO | "pinned" : " 29 GB" | 20260505 06:02:03.142 | INFO | } | 20260505 06:02:03.142 | INFO | Perf report for the last 5.13972 seconds: | 20260505 06:02:03.142 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:02:03.142 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:03.142 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:03.142 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:03.142 | INFO | ANALYSIS : 63.8%, 3.0, 99.5ms, 0 | 20260505 06:02:03.142 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:03.142 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:03.142 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 06:02:03.142 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:03.142 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:03.142 | INFO | Uploaded 36GiB to GPU in 2.13605s (16.8536GiB/s) | 20260505 06:02:03.142 | INFO | Downloaded 34GiB to GPU in 2.05992s (16.5055GiB/s) | 20260505 06:02:08.281 | INFO | Analyzing chunk frames = [900096,900608) | 20260505 06:02:08.281 | INFO | MemInfo: { | 20260505 06:02:08.281 | INFO | "gpu" : " 17 GB", | 20260505 06:02:08.281 | INFO | "huge" : "", | 20260505 06:02:08.281 | INFO | "huge_shared" : " 27 GB", | 20260505 06:02:08.281 | INFO | "malloc" : " 14 GB", | 20260505 06:02:08.281 | INFO | "pinned" : " 29 GB" | 20260505 06:02:08.281 | INFO | } | 20260505 06:02:08.281 | INFO | Perf report for the last 5.13918 seconds: | 20260505 06:02:08.281 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:02:08.281 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:08.281 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:08.281 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:08.281 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 06:02:08.281 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:08.281 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:08.281 | INFO | PRE_HQ : 32.1%, 1.0, 16.8ms, 0 | 20260505 06:02:08.281 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:08.281 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:08.281 | INFO | Uploaded 36GiB to GPU in 2.16839s (16.6022GiB/s) | 20260505 06:02:08.281 | INFO | Downloaded 34GiB to GPU in 2.17304s (15.6463GiB/s) | 20260505 06:02:13.420 | INFO | Analyzing chunk frames = [900608,901120) | 20260505 06:02:13.420 | INFO | MemInfo: { | 20260505 06:02:13.420 | INFO | "gpu" : " 17 GB", | 20260505 06:02:13.420 | INFO | "huge" : "", | 20260505 06:02:13.420 | INFO | "huge_shared" : " 27 GB", | 20260505 06:02:13.420 | INFO | "malloc" : " 14 GB", | 20260505 06:02:13.420 | INFO | "pinned" : " 29 GB" | 20260505 06:02:13.420 | INFO | } | 20260505 06:02:13.420 | INFO | Perf report for the last 5.1393 seconds: | 20260505 06:02:13.420 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:02:13.420 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:13.420 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:13.420 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:13.420 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 06:02:13.420 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:13.420 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:13.420 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 06:02:13.420 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:13.420 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:13.421 | INFO | Uploaded 36GiB to GPU in 2.08156s (17.2947GiB/s) | 20260505 06:02:13.421 | INFO | Downloaded 34GiB to GPU in 2.08329s (16.3203GiB/s) | 20260505 06:02:13.421 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 06:02:13.421 | INFO | Total mem usage: 15.3 GiB | 20260505 06:02:13.421 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 06:02:13.421 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.39%) | 20260505 06:02:13.421 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 06:02:13.421 | INFO | High Water Breakdown: | 20260505 06:02:13.421 | INFO | 60.1% : 8.66 GiB ( 92.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 06:02:13.421 | INFO | 39.7% : 5.72 GiB ( 86.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 06:02:13.421 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 06:02:13.421 | INFO | ------------------------------------------ | 20260505 06:02:13.421 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:02:13.421 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 06:02:13.421 | INFO | Total mem usage: 26.5 GiB | 20260505 06:02:13.421 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 06:02:13.421 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 06:02:13.421 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 06:02:13.421 | INFO | High Water Breakdown: | 20260505 06:02:13.421 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:02:13.421 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:02:13.421 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:02:13.421 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:02:13.421 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:02:13.421 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:02:13.421 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 06:02:13.421 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 06:02:13.421 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:02:13.421 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 06:02:13.421 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 06:02:13.421 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 06:02:13.421 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 06:02:13.421 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 06:02:13.421 | INFO | ------------------------------------------ | 20260505 06:02:13.421 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:02:13.421 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 06:02:13.421 | INFO | Total mem usage: 25.6 GiB | 20260505 06:02:13.421 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 06:02:13.421 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 06:02:13.421 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 06:02:13.421 | INFO | High Water Breakdown: | 20260505 06:02:13.421 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 06:02:13.421 | INFO | ------------------------------------------ | 20260505 06:02:13.421 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:02:13.421 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 06:02:13.421 | INFO | Total mem usage: 15.7 GiB | 20260505 06:02:13.421 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 06:02:13.421 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 06:02:13.421 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260505 06:02:13.421 | INFO | High Water Breakdown: | 20260505 06:02:13.421 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 06:02:13.421 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 06:02:13.421 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 06:02:13.421 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 06:02:13.422 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 06:02:13.422 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 06:02:13.422 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 06:02:13.422 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 06:02:13.422 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 06:02:13.422 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:02:13.422 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 06:02:13.422 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:02:13.422 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 06:02:13.422 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 06:02:13.422 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 06:02:13.422 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 06:02:13.422 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:02:13.422 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 06:02:13.422 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:02:13.422 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:02:13.422 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:02:13.422 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 06:02:13.422 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 06:02:13.422 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:02:13.422 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 06:02:13.422 | INFO | ------------------------------------------ | 20260505 06:02:13.422 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:02:13.422 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 06:02:13.422 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:02:13.422 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:02:13.422 | INFO | PerfReport: Full Execution -- 3.33e+01 / 7.63e+01, 1.40e+01 / 1.47e+00 | 20260505 06:02:13.422 | INFO | PerfReport: 0.28%, AggregateStats -- 9.43e-02 / 1.04e-01, 1.11e-02 / 6.91e-03 | 20260505 06:02:13.422 | INFO | PerfReport: 11.56%, Baseline -- 3.85e+00 / 4.58e+00, 6.40e-01 / 2.25e-01 | 20260505 06:02:13.422 | INFO | PerfReport: 3.31%, Histogram -- 1.10e+00 / 1.53e+00, 2.24e-01 / 1.02e-01 | 20260505 06:02:13.422 | INFO | PerfReport: 15.51%, DME -- 4.13e+01 / 4.54e+01, 4.46e+00 / 1.15e+00 | 20260505 06:02:13.422 | INFO | PerfReport: 55.84%, FrameLabeling -- 1.86e+01 / 1.91e+01, 2.08e+00 / 8.58e-01 | 20260505 06:02:13.422 | INFO | PerfReport: 7.82%, PulseAccumulating -- 2.60e+00 / 3.19e+00, 4.92e-01 / 3.07e-01 | 20260505 06:02:13.422 | INFO | PerfReport: 4.16%, Metrics -- 1.39e+00 / 1.76e+00, 3.87e-01 / 2.08e-01 | 20260505 06:02:13.422 | INFO | PerfReport: 1.52%, PulseSerializing -- 5.06e-01 / 5.88e-01, 9.95e-02 / 4.87e-02 | 20260505 06:02:13.422 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 06:02:13.422 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:02:13.422 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:02:13.422 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.28e+01 / 0.00e+00 | 20260505 06:02:13.422 | INFO | PerfReport: 21.88%, Upload -- 2.22e+01 / 0.00e+00, 7.51e+00 / 0.00e+00 | 20260505 06:02:13.422 | INFO | PerfReport: 21.51%, Download -- 2.18e+01 / 0.00e+00, 7.64e+00 / 0.00e+00 | 20260505 06:02:13.422 | INFO | PerfReport: 32.80%, Compute -- 3.33e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260505 06:02:13.422 | INFO | PerfReport: 1.45%, LockUpload -- 1.48e+00 / 0.00e+00, 6.70e+00 / 0.00e+00 | 20260505 06:02:13.422 | INFO | PerfReport: 0.64%, LockDownload -- 6.47e-01 / 0.00e+00, 1.85e+00 / 0.00e+00 | 20260505 06:02:13.422 | INFO | PerfReport: 21.71%, LockCompute -- 2.20e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260505 06:02:13.422 | INFO | PerfReport: 0.01%, Misc -- 1.48e-02 / 0.00e+00, 2.32e-02 / 0.00e+00 | 20260505 06:02:13.532 | INFO | Starting a round of Baz IO | 20260505 06:02:16.793 | INFO | Populating RTMetrics report | 20260505 06:02:16.794 | INFO | RTMetrics report populated | 20260505 06:02:16.794 | INFO | Opening tmp RTMetrics json file | 20260505 06:02:16.813 | INFO | Serializing RTMetrics to json | 20260505 06:02:16.815 | INFO | Serialization done, swapping tmp file now... | 20260505 06:02:16.815 | INFO | RTMetrics done | 20260505 06:02:16.904 | INFO | Finished a round of Baz IO: Wrote 2.328 GiB in 3.371 seconds | 20260505 06:02:16.904 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.27% / 0.06% / 64.67% | 20260505 06:02:16.904 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.97422% / 9.53674e-05% / 7.54992e-05% | 20260505 06:02:16.904 | INFO | Baz IO overall: | 20260505 06:02:16.904 | INFO | BytesWritten : 449730139524 | 20260505 06:02:16.904 | INFO | NumEvents : 212147471283 | 20260505 06:02:18.559 | INFO | Analyzing chunk frames = [901120,901632) | 20260505 06:02:18.559 | INFO | MemInfo: { | 20260505 06:02:18.559 | INFO | "gpu" : " 17 GB", | 20260505 06:02:18.559 | INFO | "huge" : "", | 20260505 06:02:18.559 | INFO | "huge_shared" : " 27 GB", | 20260505 06:02:18.559 | INFO | "malloc" : " 14 GB", | 20260505 06:02:18.559 | INFO | "pinned" : " 29 GB" | 20260505 06:02:18.559 | INFO | } | 20260505 06:02:18.559 | INFO | Perf report for the last 5.1391 seconds: | 20260505 06:02:18.560 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:02:18.560 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:18.560 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:18.560 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:18.560 | INFO | ANALYSIS : 64.5%, 2.9, 99.2ms, 0 | 20260505 06:02:18.560 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:18.560 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260505 06:02:18.560 | INFO | PRE_HQ : 63.7%, 2.0, 64.6ms, 0 | 20260505 06:02:18.560 | INFO | BAZWRITER : 64.9%, 4.1, 135.2ms, 0 | 20260505 06:02:18.560 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260505 06:02:18.560 | INFO | Uploaded 36GiB to GPU in 2.12924s (16.9075GiB/s) | 20260505 06:02:18.560 | INFO | Downloaded 38GiB to GPU in 2.32665s (16.3325GiB/s) | 20260505 06:02:23.699 | INFO | Analyzing chunk frames = [901632,902144) | 20260505 06:02:23.699 | INFO | MemInfo: { | 20260505 06:02:23.699 | INFO | "gpu" : " 17 GB", | 20260505 06:02:23.699 | INFO | "huge" : "", | 20260505 06:02:23.699 | INFO | "huge_shared" : " 27 GB", | 20260505 06:02:23.699 | INFO | "malloc" : " 14 GB", | 20260505 06:02:23.699 | INFO | "pinned" : " 29 GB" | 20260505 06:02:23.699 | INFO | } | 20260505 06:02:23.699 | INFO | Perf report for the last 5.13966 seconds: | 20260505 06:02:23.699 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:02:23.699 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:23.699 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:23.699 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:23.699 | INFO | ANALYSIS : 64.4%, 3.0, 100.6ms, 0 | 20260505 06:02:23.699 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:23.699 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:23.699 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 06:02:23.699 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:23.700 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:23.700 | INFO | Uploaded 36GiB to GPU in 2.13462s (16.8648GiB/s) | 20260505 06:02:23.700 | INFO | Downloaded 34GiB to GPU in 2.04565s (16.6207GiB/s) | 20260505 06:02:28.838 | INFO | Analyzing chunk frames = [902144,902656) | 20260505 06:02:28.838 | INFO | MemInfo: { | 20260505 06:02:28.838 | INFO | "gpu" : " 17 GB", | 20260505 06:02:28.838 | INFO | "huge" : "", | 20260505 06:02:28.838 | INFO | "huge_shared" : " 27 GB", | 20260505 06:02:28.838 | INFO | "malloc" : " 14 GB", | 20260505 06:02:28.838 | INFO | "pinned" : " 29 GB" | 20260505 06:02:28.838 | INFO | } | 20260505 06:02:28.838 | INFO | Perf report for the last 5.1391 seconds: | 20260505 06:02:28.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:02:28.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:28.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:28.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:28.838 | INFO | ANALYSIS : 64.3%, 3.0, 100.1ms, 0 | 20260505 06:02:28.838 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:28.838 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:28.838 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 06:02:28.838 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:28.838 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:28.838 | INFO | Uploaded 36GiB to GPU in 2.18023s (16.5121GiB/s) | 20260505 06:02:28.838 | INFO | Downloaded 34GiB to GPU in 2.04198s (16.6505GiB/s) | 20260505 06:02:33.977 | INFO | Analyzing chunk frames = [902656,903168) | 20260505 06:02:33.977 | INFO | MemInfo: { | 20260505 06:02:33.977 | INFO | "gpu" : " 17 GB", | 20260505 06:02:33.977 | INFO | "huge" : "", | 20260505 06:02:33.977 | INFO | "huge_shared" : " 27 GB", | 20260505 06:02:33.977 | INFO | "malloc" : " 14 GB", | 20260505 06:02:33.977 | INFO | "pinned" : " 29 GB" | 20260505 06:02:33.977 | INFO | } | 20260505 06:02:33.977 | INFO | Perf report for the last 5.13926 seconds: | 20260505 06:02:33.977 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:02:33.977 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:33.977 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:33.977 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:33.977 | INFO | ANALYSIS : 63.5%, 3.0, 98.6ms, 0 | 20260505 06:02:33.977 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:33.977 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:33.977 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 06:02:33.977 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:33.977 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:33.978 | INFO | Uploaded 36GiB to GPU in 2.11326s (17.0353GiB/s) | 20260505 06:02:33.978 | INFO | Downloaded 34GiB to GPU in 2.03278s (16.7258GiB/s) | 20260505 06:02:39.117 | INFO | Analyzing chunk frames = [903168,903680) | 20260505 06:02:39.117 | INFO | MemInfo: { | 20260505 06:02:39.117 | INFO | "gpu" : " 17 GB", | 20260505 06:02:39.117 | INFO | "huge" : "", | 20260505 06:02:39.117 | INFO | "huge_shared" : " 27 GB", | 20260505 06:02:39.117 | INFO | "malloc" : " 14 GB", | 20260505 06:02:39.117 | INFO | "pinned" : " 29 GB" | 20260505 06:02:39.117 | INFO | } | 20260505 06:02:39.117 | INFO | Perf report for the last 5.13929 seconds: | 20260505 06:02:39.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:02:39.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:39.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:39.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:39.117 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 06:02:39.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:39.117 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:39.117 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 06:02:39.117 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:39.117 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:39.117 | INFO | Uploaded 36GiB to GPU in 2.1133s (17.0349GiB/s) | 20260505 06:02:39.117 | INFO | Downloaded 34GiB to GPU in 2.04045s (16.663GiB/s) | 20260505 06:02:44.257 | INFO | Analyzing chunk frames = [903680,904192) | 20260505 06:02:44.257 | INFO | MemInfo: { | 20260505 06:02:44.257 | INFO | "gpu" : " 17 GB", | 20260505 06:02:44.258 | INFO | "huge" : "", | 20260505 06:02:44.258 | INFO | "huge_shared" : " 27 GB", | 20260505 06:02:44.258 | INFO | "malloc" : " 14 GB", | 20260505 06:02:44.258 | INFO | "pinned" : " 29 GB" | 20260505 06:02:44.258 | INFO | } | 20260505 06:02:44.258 | INFO | Perf report for the last 5.14087 seconds: | 20260505 06:02:44.258 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:02:44.258 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:44.258 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:44.258 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:44.258 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 06:02:44.258 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:44.258 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:44.258 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 06:02:44.258 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:44.258 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:44.258 | INFO | Uploaded 36GiB to GPU in 2.12205s (16.9647GiB/s) | 20260505 06:02:44.258 | INFO | Downloaded 34GiB to GPU in 2.07023s (16.4233GiB/s) | 20260505 06:02:49.396 | INFO | Analyzing chunk frames = [904192,904704) | 20260505 06:02:49.396 | INFO | MemInfo: { | 20260505 06:02:49.396 | INFO | "gpu" : " 17 GB", | 20260505 06:02:49.396 | INFO | "huge" : "", | 20260505 06:02:49.396 | INFO | "huge_shared" : " 27 GB", | 20260505 06:02:49.396 | INFO | "malloc" : " 14 GB", | 20260505 06:02:49.396 | INFO | "pinned" : " 29 GB" | 20260505 06:02:49.396 | INFO | } | 20260505 06:02:49.396 | INFO | Perf report for the last 5.13796 seconds: | 20260505 06:02:49.396 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:02:49.396 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:49.396 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:49.396 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:49.396 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 06:02:49.396 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:49.396 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:49.396 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260505 06:02:49.396 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:49.396 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:49.396 | INFO | Uploaded 36GiB to GPU in 2.15537s (16.7024GiB/s) | 20260505 06:02:49.396 | INFO | Downloaded 34GiB to GPU in 2.15395s (15.7849GiB/s) | 20260505 06:02:54.535 | INFO | Analyzing chunk frames = [904704,905216) | 20260505 06:02:54.535 | INFO | MemInfo: { | 20260505 06:02:54.535 | INFO | "gpu" : " 17 GB", | 20260505 06:02:54.535 | INFO | "huge" : "", | 20260505 06:02:54.535 | INFO | "huge_shared" : " 27 GB", | 20260505 06:02:54.535 | INFO | "malloc" : " 14 GB", | 20260505 06:02:54.535 | INFO | "pinned" : " 29 GB" | 20260505 06:02:54.535 | INFO | } | 20260505 06:02:54.535 | INFO | Perf report for the last 5.139 seconds: | 20260505 06:02:54.535 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:02:54.535 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:54.535 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:54.535 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:54.535 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 06:02:54.535 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:54.535 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:54.535 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 06:02:54.535 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:54.535 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:54.535 | INFO | Uploaded 36GiB to GPU in 2.08529s (17.2638GiB/s) | 20260505 06:02:54.535 | INFO | Downloaded 34GiB to GPU in 2.09099s (16.2602GiB/s) | 20260505 06:02:54.646 | INFO | Starting a round of Baz IO | 20260505 06:02:57.915 | INFO | Populating RTMetrics report | 20260505 06:02:57.915 | INFO | RTMetrics report populated | 20260505 06:02:57.915 | INFO | Opening tmp RTMetrics json file | 20260505 06:02:57.916 | INFO | Serializing RTMetrics to json | 20260505 06:02:57.917 | INFO | Serialization done, swapping tmp file now... | 20260505 06:02:57.918 | INFO | RTMetrics done | 20260505 06:02:58.031 | INFO | Finished a round of Baz IO: Wrote 2.332 GiB in 3.385 seconds | 20260505 06:02:58.031 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.21% / 0.06% / 64.73% | 20260505 06:02:58.032 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.95468% / 8.34465e-05% / 6.35783e-05% | 20260505 06:02:58.032 | INFO | Baz IO overall: | 20260505 06:02:58.032 | INFO | BytesWritten : 452233877350 | 20260505 06:02:58.032 | INFO | NumEvents : 213343666302 | 20260505 06:02:59.674 | INFO | Analyzing chunk frames = [905216,905728) | 20260505 06:02:59.674 | INFO | MemInfo: { | 20260505 06:02:59.674 | INFO | "gpu" : " 17 GB", | 20260505 06:02:59.674 | INFO | "huge" : "", | 20260505 06:02:59.674 | INFO | "huge_shared" : " 27 GB", | 20260505 06:02:59.674 | INFO | "malloc" : " 14 GB", | 20260505 06:02:59.674 | INFO | "pinned" : " 29 GB" | 20260505 06:02:59.674 | INFO | } | 20260505 06:02:59.674 | INFO | Perf report for the last 5.13973 seconds: | 20260505 06:02:59.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:02:59.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:59.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:59.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:59.674 | INFO | ANALYSIS : 64.6%, 2.9, 98.6ms, 0 | 20260505 06:02:59.674 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:02:59.674 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260505 06:02:59.674 | INFO | PRE_HQ : 63.8%, 1.9, 64.2ms, 0 | 20260505 06:02:59.674 | INFO | BAZWRITER : 65.2%, 4.1, 135.8ms, 0 | 20260505 06:02:59.674 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 06:02:59.675 | INFO | Uploaded 36GiB to GPU in 2.1421s (16.8059GiB/s) | 20260505 06:02:59.675 | INFO | Downloaded 38GiB to GPU in 2.3276s (16.3258GiB/s) | 20260505 06:03:04.814 | INFO | Analyzing chunk frames = [905728,906240) | 20260505 06:03:04.814 | INFO | MemInfo: { | 20260505 06:03:04.814 | INFO | "gpu" : " 17 GB", | 20260505 06:03:04.814 | INFO | "huge" : "", | 20260505 06:03:04.814 | INFO | "huge_shared" : " 27 GB", | 20260505 06:03:04.814 | INFO | "malloc" : " 14 GB", | 20260505 06:03:04.814 | INFO | "pinned" : " 29 GB" | 20260505 06:03:04.814 | INFO | } | 20260505 06:03:04.814 | INFO | Perf report for the last 5.13968 seconds: | 20260505 06:03:04.814 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:03:04.814 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:04.814 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:04.814 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:04.814 | INFO | ANALYSIS : 64.5%, 3.0, 100.7ms, 0 | 20260505 06:03:04.814 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:04.814 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:04.814 | INFO | PRE_HQ : 30.5%, 1.0, 15.9ms, 0 | 20260505 06:03:04.814 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:04.814 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:04.814 | INFO | Uploaded 36GiB to GPU in 2.13393s (16.8703GiB/s) | 20260505 06:03:04.814 | INFO | Downloaded 34GiB to GPU in 2.0355s (16.7035GiB/s) | 20260505 06:03:09.953 | INFO | Analyzing chunk frames = [906240,906752) | 20260505 06:03:09.953 | INFO | MemInfo: { | 20260505 06:03:09.953 | INFO | "gpu" : " 17 GB", | 20260505 06:03:09.953 | INFO | "huge" : "", | 20260505 06:03:09.953 | INFO | "huge_shared" : " 27 GB", | 20260505 06:03:09.953 | INFO | "malloc" : " 14 GB", | 20260505 06:03:09.953 | INFO | "pinned" : " 29 GB" | 20260505 06:03:09.953 | INFO | } | 20260505 06:03:09.953 | INFO | Perf report for the last 5.13868 seconds: | 20260505 06:03:09.953 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:03:09.953 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:09.953 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:09.953 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:09.953 | INFO | ANALYSIS : 64.2%, 3.0, 100.0ms, 0 | 20260505 06:03:09.953 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:09.953 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:09.953 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 06:03:09.953 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:09.953 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:09.953 | INFO | Uploaded 36GiB to GPU in 2.15783s (16.6834GiB/s) | 20260505 06:03:09.953 | INFO | Downloaded 34GiB to GPU in 2.04007s (16.6661GiB/s) | 20260505 06:03:15.092 | INFO | Analyzing chunk frames = [906752,907264) | 20260505 06:03:15.092 | INFO | MemInfo: { | 20260505 06:03:15.092 | INFO | "gpu" : " 17 GB", | 20260505 06:03:15.092 | INFO | "huge" : "", | 20260505 06:03:15.092 | INFO | "huge_shared" : " 27 GB", | 20260505 06:03:15.092 | INFO | "malloc" : " 14 GB", | 20260505 06:03:15.092 | INFO | "pinned" : " 29 GB" | 20260505 06:03:15.092 | INFO | } | 20260505 06:03:15.092 | INFO | Perf report for the last 5.13916 seconds: | 20260505 06:03:15.092 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:03:15.092 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:15.092 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:15.092 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:15.092 | INFO | ANALYSIS : 63.4%, 3.0, 98.6ms, 0 | 20260505 06:03:15.092 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:15.092 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:15.092 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 06:03:15.092 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:15.092 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:15.092 | INFO | Uploaded 36GiB to GPU in 2.1289s (16.9101GiB/s) | 20260505 06:03:15.092 | INFO | Downloaded 34GiB to GPU in 2.03524s (16.7056GiB/s) | 20260505 06:03:20.232 | INFO | Analyzing chunk frames = [907264,907776) | 20260505 06:03:20.232 | INFO | MemInfo: { | 20260505 06:03:20.232 | INFO | "gpu" : " 17 GB", | 20260505 06:03:20.232 | INFO | "huge" : "", | 20260505 06:03:20.232 | INFO | "huge_shared" : " 27 GB", | 20260505 06:03:20.232 | INFO | "malloc" : " 14 GB", | 20260505 06:03:20.232 | INFO | "pinned" : " 29 GB" | 20260505 06:03:20.232 | INFO | } | 20260505 06:03:20.232 | INFO | Perf report for the last 5.13993 seconds: | 20260505 06:03:20.232 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:03:20.232 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:20.232 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:20.232 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:20.232 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 06:03:20.232 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:20.232 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:20.232 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 06:03:20.232 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:20.232 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:20.232 | INFO | Uploaded 36GiB to GPU in 2.14334s (16.7962GiB/s) | 20260505 06:03:20.232 | INFO | Downloaded 34GiB to GPU in 2.05239s (16.5661GiB/s) | 20260505 06:03:25.372 | INFO | Analyzing chunk frames = [907776,908288) | 20260505 06:03:25.372 | INFO | MemInfo: { | 20260505 06:03:25.372 | INFO | "gpu" : " 17 GB", | 20260505 06:03:25.372 | INFO | "huge" : "", | 20260505 06:03:25.372 | INFO | "huge_shared" : " 27 GB", | 20260505 06:03:25.372 | INFO | "malloc" : " 14 GB", | 20260505 06:03:25.372 | INFO | "pinned" : " 29 GB" | 20260505 06:03:25.372 | INFO | } | 20260505 06:03:25.372 | INFO | Perf report for the last 5.13989 seconds: | 20260505 06:03:25.372 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:03:25.372 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:25.372 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:25.372 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:25.372 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 06:03:25.372 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:25.372 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:25.372 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 06:03:25.372 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:25.372 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:25.372 | INFO | Uploaded 36GiB to GPU in 2.14248s (16.803GiB/s) | 20260505 06:03:25.372 | INFO | Downloaded 34GiB to GPU in 2.06689s (16.4498GiB/s) | 20260505 06:03:30.510 | INFO | Analyzing chunk frames = [908288,908800) | 20260505 06:03:30.510 | INFO | MemInfo: { | 20260505 06:03:30.510 | INFO | "gpu" : " 17 GB", | 20260505 06:03:30.510 | INFO | "huge" : "", | 20260505 06:03:30.510 | INFO | "huge_shared" : " 27 GB", | 20260505 06:03:30.510 | INFO | "malloc" : " 14 GB", | 20260505 06:03:30.510 | INFO | "pinned" : " 29 GB" | 20260505 06:03:30.510 | INFO | } | 20260505 06:03:30.510 | INFO | Perf report for the last 5.13862 seconds: | 20260505 06:03:30.510 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:03:30.510 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:30.510 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:30.510 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:30.510 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 06:03:30.510 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:30.511 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:30.511 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 06:03:30.511 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:30.511 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:30.511 | INFO | Uploaded 36GiB to GPU in 2.18475s (16.4779GiB/s) | 20260505 06:03:30.511 | INFO | Downloaded 34GiB to GPU in 2.15587s (15.7709GiB/s) | 20260505 06:03:35.649 | INFO | Analyzing chunk frames = [908800,909312) | 20260505 06:03:35.649 | INFO | MemInfo: { | 20260505 06:03:35.649 | INFO | "gpu" : " 17 GB", | 20260505 06:03:35.649 | INFO | "huge" : "", | 20260505 06:03:35.649 | INFO | "huge_shared" : " 27 GB", | 20260505 06:03:35.649 | INFO | "malloc" : " 14 GB", | 20260505 06:03:35.649 | INFO | "pinned" : " 29 GB" | 20260505 06:03:35.649 | INFO | } | 20260505 06:03:35.649 | INFO | Perf report for the last 5.13897 seconds: | 20260505 06:03:35.650 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:03:35.650 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:35.650 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:35.650 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:35.650 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 06:03:35.650 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:35.650 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:35.650 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 06:03:35.650 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:35.650 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:35.650 | INFO | Uploaded 36GiB to GPU in 2.07766s (17.3272GiB/s) | 20260505 06:03:35.650 | INFO | Downloaded 34GiB to GPU in 2.08278s (16.3243GiB/s) | 20260505 06:03:35.650 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 06:03:35.650 | INFO | Total mem usage: 15.3 GiB | 20260505 06:03:35.650 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 06:03:35.650 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.35%) | 20260505 06:03:35.650 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 06:03:35.650 | INFO | High Water Breakdown: | 20260505 06:03:35.650 | INFO | 60.2% : 8.67 GiB ( 92.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 06:03:35.650 | INFO | 39.7% : 5.72 GiB ( 86.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 06:03:35.650 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 06:03:35.650 | INFO | ------------------------------------------ | 20260505 06:03:35.650 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:03:35.650 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 06:03:35.650 | INFO | Total mem usage: 26.5 GiB | 20260505 06:03:35.650 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 06:03:35.650 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 06:03:35.650 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 06:03:35.650 | INFO | High Water Breakdown: | 20260505 06:03:35.650 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:03:35.650 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:03:35.650 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:03:35.650 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:03:35.650 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:03:35.650 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:03:35.650 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 06:03:35.650 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 06:03:35.650 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:03:35.650 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 06:03:35.650 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 06:03:35.650 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 06:03:35.650 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 06:03:35.650 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 06:03:35.650 | INFO | ------------------------------------------ | 20260505 06:03:35.650 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:03:35.650 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 06:03:35.650 | INFO | Total mem usage: 25.6 GiB | 20260505 06:03:35.650 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 06:03:35.650 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 06:03:35.650 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 06:03:35.650 | INFO | High Water Breakdown: | 20260505 06:03:35.650 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 06:03:35.650 | INFO | ------------------------------------------ | 20260505 06:03:35.650 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:03:35.651 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 06:03:35.651 | INFO | Total mem usage: 15.7 GiB | 20260505 06:03:35.651 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 06:03:35.651 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 06:03:35.651 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260505 06:03:35.651 | INFO | High Water Breakdown: | 20260505 06:03:35.651 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 06:03:35.651 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 06:03:35.651 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 06:03:35.651 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 06:03:35.651 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 06:03:35.651 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 06:03:35.651 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 06:03:35.651 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 06:03:35.651 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 06:03:35.651 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:03:35.651 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 06:03:35.651 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:03:35.651 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 06:03:35.651 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 06:03:35.651 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 06:03:35.651 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 06:03:35.651 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:03:35.651 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 06:03:35.651 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:03:35.651 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:03:35.651 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:03:35.651 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 06:03:35.651 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 06:03:35.651 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:03:35.651 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 06:03:35.651 | INFO | ------------------------------------------ | 20260505 06:03:35.651 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:03:35.651 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 06:03:35.651 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:03:35.651 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:03:35.651 | INFO | PerfReport: Full Execution -- 3.33e+01 / 7.60e+01, 1.38e+01 / 1.69e+00 | 20260505 06:03:35.651 | INFO | PerfReport: 0.28%, AggregateStats -- 9.44e-02 / 1.04e-01, 1.12e-02 / 7.04e-03 | 20260505 06:03:35.651 | INFO | PerfReport: 11.57%, Baseline -- 3.85e+00 / 4.54e+00, 6.43e-01 / 2.42e-01 | 20260505 06:03:35.651 | INFO | PerfReport: 3.31%, Histogram -- 1.10e+00 / 1.52e+00, 2.23e-01 / 1.13e-01 | 20260505 06:03:35.651 | INFO | PerfReport: 15.52%, DME -- 4.13e+01 / 4.53e+01, 4.45e+00 / 1.27e+00 | 20260505 06:03:35.651 | INFO | PerfReport: 55.82%, FrameLabeling -- 1.86e+01 / 1.90e+01, 2.06e+00 / 9.38e-01 | 20260505 06:03:35.651 | INFO | PerfReport: 7.82%, PulseAccumulating -- 2.60e+00 / 3.19e+00, 4.94e-01 / 3.27e-01 | 20260505 06:03:35.651 | INFO | PerfReport: 4.15%, Metrics -- 1.38e+00 / 1.75e+00, 3.89e-01 / 2.26e-01 | 20260505 06:03:35.651 | INFO | PerfReport: 1.52%, PulseSerializing -- 5.05e-01 / 5.84e-01, 9.99e-02 / 4.21e-02 | 20260505 06:03:35.651 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 06:03:35.651 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:03:35.651 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:03:35.651 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.29e+01 / 0.00e+00 | 20260505 06:03:35.651 | INFO | PerfReport: 21.90%, Upload -- 2.22e+01 / 0.00e+00, 7.56e+00 / 0.00e+00 | 20260505 06:03:35.651 | INFO | PerfReport: 21.55%, Download -- 2.19e+01 / 0.00e+00, 7.77e+00 / 0.00e+00 | 20260505 06:03:35.651 | INFO | PerfReport: 32.78%, Compute -- 3.33e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 06:03:35.651 | INFO | PerfReport: 1.44%, LockUpload -- 1.47e+00 / 0.00e+00, 6.67e+00 / 0.00e+00 | 20260505 06:03:35.651 | INFO | PerfReport: 0.66%, LockDownload -- 6.73e-01 / 0.00e+00, 1.90e+00 / 0.00e+00 | 20260505 06:03:35.651 | INFO | PerfReport: 21.64%, LockCompute -- 2.20e+01 / 0.00e+00, 1.66e+01 / 0.00e+00 | 20260505 06:03:35.651 | INFO | PerfReport: 0.01%, Misc -- 1.47e-02 / 0.00e+00, 2.22e-02 / 0.00e+00 | 20260505 06:03:35.761 | INFO | Starting a round of Baz IO | 20260505 06:03:39.022 | INFO | Populating RTMetrics report | 20260505 06:03:39.022 | INFO | RTMetrics report populated | 20260505 06:03:39.022 | INFO | Opening tmp RTMetrics json file | 20260505 06:03:39.044 | INFO | Serializing RTMetrics to json | 20260505 06:03:39.055 | INFO | Serialization done, swapping tmp file now... | 20260505 06:03:39.070 | INFO | RTMetrics done | 20260505 06:03:39.132 | INFO | Finished a round of Baz IO: Wrote 2.334 GiB in 3.371 seconds | 20260505 06:03:39.132 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.15% / 0.06% / 64.79% | 20260505 06:03:39.132 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.93654% / 9.13938e-05% / 5.16574e-05% | 20260505 06:03:39.132 | INFO | Baz IO overall: | 20260505 06:03:39.132 | INFO | BytesWritten : 454740135183 | 20260505 06:03:39.132 | INFO | NumEvents : 214541361483 | 20260505 06:03:40.789 | INFO | Analyzing chunk frames = [909312,909824) | 20260505 06:03:40.789 | INFO | MemInfo: { | 20260505 06:03:40.789 | INFO | "gpu" : " 17 GB", | 20260505 06:03:40.789 | INFO | "huge" : "", | 20260505 06:03:40.789 | INFO | "huge_shared" : " 27 GB", | 20260505 06:03:40.789 | INFO | "malloc" : " 14 GB", | 20260505 06:03:40.789 | INFO | "pinned" : " 29 GB" | 20260505 06:03:40.789 | INFO | } | 20260505 06:03:40.789 | INFO | Perf report for the last 5.13939 seconds: | 20260505 06:03:40.789 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:03:40.789 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:40.789 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:40.789 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:40.789 | INFO | ANALYSIS : 64.5%, 2.9, 98.6ms, 0 | 20260505 06:03:40.789 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:40.789 | INFO | RT_METRICS : 46.8%, 1.0, 24.6ms, 0 | 20260505 06:03:40.789 | INFO | PRE_HQ : 63.8%, 1.9, 64.5ms, 0 | 20260505 06:03:40.789 | INFO | BAZWRITER : 64.7%, 4.1, 135.9ms, 0 | 20260505 06:03:40.789 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 06:03:40.789 | INFO | Uploaded 36GiB to GPU in 2.16427s (16.6338GiB/s) | 20260505 06:03:40.789 | INFO | Downloaded 38GiB to GPU in 2.32501s (16.344GiB/s) | 20260505 06:03:45.928 | INFO | Analyzing chunk frames = [909824,910336) | 20260505 06:03:45.928 | INFO | MemInfo: { | 20260505 06:03:45.928 | INFO | "gpu" : " 17 GB", | 20260505 06:03:45.928 | INFO | "huge" : "", | 20260505 06:03:45.928 | INFO | "huge_shared" : " 27 GB", | 20260505 06:03:45.928 | INFO | "malloc" : " 14 GB", | 20260505 06:03:45.928 | INFO | "pinned" : " 29 GB" | 20260505 06:03:45.928 | INFO | } | 20260505 06:03:45.928 | INFO | Perf report for the last 5.13954 seconds: | 20260505 06:03:45.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:03:45.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:45.929 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:45.929 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:45.929 | INFO | ANALYSIS : 64.5%, 3.0, 100.7ms, 0 | 20260505 06:03:45.929 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:45.929 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:45.929 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 06:03:45.929 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:45.929 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:45.929 | INFO | Uploaded 36GiB to GPU in 2.13336s (16.8748GiB/s) | 20260505 06:03:45.929 | INFO | Downloaded 34GiB to GPU in 2.03567s (16.7021GiB/s) | 20260505 06:03:51.068 | INFO | Analyzing chunk frames = [910336,910848) | 20260505 06:03:51.068 | INFO | MemInfo: { | 20260505 06:03:51.068 | INFO | "gpu" : " 17 GB", | 20260505 06:03:51.068 | INFO | "huge" : "", | 20260505 06:03:51.068 | INFO | "huge_shared" : " 27 GB", | 20260505 06:03:51.068 | INFO | "malloc" : " 14 GB", | 20260505 06:03:51.068 | INFO | "pinned" : " 29 GB" | 20260505 06:03:51.068 | INFO | } | 20260505 06:03:51.068 | INFO | Perf report for the last 5.13956 seconds: | 20260505 06:03:51.068 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:03:51.068 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:51.068 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:51.068 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:51.068 | INFO | ANALYSIS : 64.2%, 3.0, 99.9ms, 0 | 20260505 06:03:51.069 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:51.069 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:51.069 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 06:03:51.069 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:51.069 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:51.069 | INFO | Uploaded 36GiB to GPU in 2.16823s (16.6034GiB/s) | 20260505 06:03:51.069 | INFO | Downloaded 34GiB to GPU in 2.0467s (16.6121GiB/s) | 20260505 06:03:56.207 | INFO | Analyzing chunk frames = [910848,911360) | 20260505 06:03:56.207 | INFO | MemInfo: { | 20260505 06:03:56.207 | INFO | "gpu" : " 17 GB", | 20260505 06:03:56.207 | INFO | "huge" : "", | 20260505 06:03:56.207 | INFO | "huge_shared" : " 27 GB", | 20260505 06:03:56.207 | INFO | "malloc" : " 14 GB", | 20260505 06:03:56.207 | INFO | "pinned" : " 29 GB" | 20260505 06:03:56.207 | INFO | } | 20260505 06:03:56.207 | INFO | Perf report for the last 5.13887 seconds: | 20260505 06:03:56.207 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:03:56.207 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:56.207 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:56.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:56.207 | INFO | ANALYSIS : 63.5%, 3.0, 98.6ms, 0 | 20260505 06:03:56.207 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:56.207 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:56.207 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 06:03:56.207 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:56.207 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:03:56.207 | INFO | Uploaded 36GiB to GPU in 2.1196s (16.9843GiB/s) | 20260505 06:03:56.207 | INFO | Downloaded 34GiB to GPU in 2.04116s (16.6572GiB/s) | 20260505 06:04:01.346 | INFO | Analyzing chunk frames = [911360,911872) | 20260505 06:04:01.346 | INFO | MemInfo: { | 20260505 06:04:01.346 | INFO | "gpu" : " 17 GB", | 20260505 06:04:01.346 | INFO | "huge" : "", | 20260505 06:04:01.346 | INFO | "huge_shared" : " 27 GB", | 20260505 06:04:01.346 | INFO | "malloc" : " 14 GB", | 20260505 06:04:01.346 | INFO | "pinned" : " 29 GB" | 20260505 06:04:01.346 | INFO | } | 20260505 06:04:01.346 | INFO | Perf report for the last 5.13942 seconds: | 20260505 06:04:01.346 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:04:01.346 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:01.346 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:01.346 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:01.346 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 06:04:01.346 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:01.346 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:01.346 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 06:04:01.346 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:01.346 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:01.347 | INFO | Uploaded 36GiB to GPU in 2.11716s (17.0039GiB/s) | 20260505 06:04:01.347 | INFO | Downloaded 34GiB to GPU in 2.04235s (16.6475GiB/s) | 20260505 06:04:06.486 | INFO | Analyzing chunk frames = [911872,912384) | 20260505 06:04:06.486 | INFO | MemInfo: { | 20260505 06:04:06.486 | INFO | "gpu" : " 17 GB", | 20260505 06:04:06.486 | INFO | "huge" : "", | 20260505 06:04:06.486 | INFO | "huge_shared" : " 27 GB", | 20260505 06:04:06.486 | INFO | "malloc" : " 14 GB", | 20260505 06:04:06.486 | INFO | "pinned" : " 29 GB" | 20260505 06:04:06.486 | INFO | } | 20260505 06:04:06.486 | INFO | Perf report for the last 5.13987 seconds: | 20260505 06:04:06.486 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:04:06.486 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:06.486 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:06.486 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:06.486 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 06:04:06.486 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:06.486 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:06.486 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 06:04:06.486 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:06.486 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:06.486 | INFO | Uploaded 36GiB to GPU in 2.13412s (16.8687GiB/s) | 20260505 06:04:06.486 | INFO | Downloaded 34GiB to GPU in 2.06993s (16.4257GiB/s) | 20260505 06:04:11.625 | INFO | Analyzing chunk frames = [912384,912896) | 20260505 06:04:11.625 | INFO | MemInfo: { | 20260505 06:04:11.625 | INFO | "gpu" : " 17 GB", | 20260505 06:04:11.625 | INFO | "huge" : "", | 20260505 06:04:11.625 | INFO | "huge_shared" : " 27 GB", | 20260505 06:04:11.625 | INFO | "malloc" : " 14 GB", | 20260505 06:04:11.625 | INFO | "pinned" : " 29 GB" | 20260505 06:04:11.625 | INFO | } | 20260505 06:04:11.625 | INFO | Perf report for the last 5.13884 seconds: | 20260505 06:04:11.625 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:04:11.625 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:11.625 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:11.625 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:11.625 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 06:04:11.625 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:11.625 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:11.625 | INFO | PRE_HQ : 32.0%, 1.0, 16.7ms, 0 | 20260505 06:04:11.625 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:11.625 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:11.625 | INFO | Uploaded 36GiB to GPU in 2.18502s (16.4759GiB/s) | 20260505 06:04:11.625 | INFO | Downloaded 34GiB to GPU in 2.16387s (15.7126GiB/s) | 20260505 06:04:16.764 | INFO | Analyzing chunk frames = [912896,913408) | 20260505 06:04:16.764 | INFO | MemInfo: { | 20260505 06:04:16.764 | INFO | "gpu" : " 17 GB", | 20260505 06:04:16.764 | INFO | "huge" : "", | 20260505 06:04:16.764 | INFO | "huge_shared" : " 27 GB", | 20260505 06:04:16.764 | INFO | "malloc" : " 14 GB", | 20260505 06:04:16.764 | INFO | "pinned" : " 29 GB" | 20260505 06:04:16.764 | INFO | } | 20260505 06:04:16.765 | INFO | Perf report for the last 5.1395 seconds: | 20260505 06:04:16.765 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:04:16.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:16.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:16.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:16.765 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 06:04:16.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:16.765 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:16.765 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 06:04:16.765 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:16.765 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:16.765 | INFO | Uploaded 36GiB to GPU in 2.09093s (17.2172GiB/s) | 20260505 06:04:16.765 | INFO | Downloaded 34GiB to GPU in 2.08907s (16.2752GiB/s) | 20260505 06:04:16.874 | INFO | Starting a round of Baz IO | 20260505 06:04:20.146 | INFO | Populating RTMetrics report | 20260505 06:04:20.146 | INFO | RTMetrics report populated | 20260505 06:04:20.146 | INFO | Opening tmp RTMetrics json file | 20260505 06:04:20.171 | INFO | Serializing RTMetrics to json | 20260505 06:04:20.194 | INFO | Serialization done, swapping tmp file now... | 20260505 06:04:20.216 | INFO | RTMetrics done | 20260505 06:04:20.257 | INFO | Finished a round of Baz IO: Wrote 2.333 GiB in 3.383 seconds | 20260505 06:04:20.257 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.10% / 0.06% / 64.85% | 20260505 06:04:20.257 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.8965% / 0.000150998% / 5.16574e-05% | 20260505 06:04:20.257 | INFO | Baz IO overall: | 20260505 06:04:20.257 | INFO | BytesWritten : 457245601396 | 20260505 06:04:20.257 | INFO | NumEvents : 215738278950 | 20260505 06:04:21.904 | INFO | Analyzing chunk frames = [913408,913920) | 20260505 06:04:21.904 | INFO | MemInfo: { | 20260505 06:04:21.904 | INFO | "gpu" : " 17 GB", | 20260505 06:04:21.904 | INFO | "huge" : "", | 20260505 06:04:21.904 | INFO | "huge_shared" : " 27 GB", | 20260505 06:04:21.904 | INFO | "malloc" : " 14 GB", | 20260505 06:04:21.904 | INFO | "pinned" : " 29 GB" | 20260505 06:04:21.904 | INFO | } | 20260505 06:04:21.904 | INFO | Perf report for the last 5.13985 seconds: | 20260505 06:04:21.904 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:04:21.904 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:21.904 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:21.904 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:21.904 | INFO | ANALYSIS : 64.6%, 2.9, 99.1ms, 0 | 20260505 06:04:21.904 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:21.904 | INFO | RT_METRICS : 47.4%, 1.0, 24.8ms, 0 | 20260505 06:04:21.904 | INFO | PRE_HQ : 63.8%, 1.9, 64.3ms, 0 | 20260505 06:04:21.904 | INFO | BAZWRITER : 64.9%, 4.1, 135.2ms, 0 | 20260505 06:04:21.905 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 06:04:21.905 | INFO | Uploaded 36GiB to GPU in 2.17659s (16.5396GiB/s) | 20260505 06:04:21.905 | INFO | Downloaded 38GiB to GPU in 2.33096s (16.3023GiB/s) | 20260505 06:04:27.043 | INFO | Analyzing chunk frames = [913920,914432) | 20260505 06:04:27.043 | INFO | MemInfo: { | 20260505 06:04:27.043 | INFO | "gpu" : " 17 GB", | 20260505 06:04:27.043 | INFO | "huge" : "", | 20260505 06:04:27.043 | INFO | "huge_shared" : " 27 GB", | 20260505 06:04:27.043 | INFO | "malloc" : " 14 GB", | 20260505 06:04:27.043 | INFO | "pinned" : " 29 GB" | 20260505 06:04:27.043 | INFO | } | 20260505 06:04:27.043 | INFO | Perf report for the last 5.1388 seconds: | 20260505 06:04:27.043 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:04:27.043 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:27.043 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:27.043 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:27.043 | INFO | ANALYSIS : 64.5%, 3.0, 100.7ms, 0 | 20260505 06:04:27.043 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:27.043 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:27.043 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 06:04:27.043 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:27.043 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:27.043 | INFO | Uploaded 36GiB to GPU in 2.13314s (16.8765GiB/s) | 20260505 06:04:27.043 | INFO | Downloaded 34GiB to GPU in 2.03469s (16.7102GiB/s) | 20260505 06:04:32.183 | INFO | Analyzing chunk frames = [914432,914944) | 20260505 06:04:32.183 | INFO | MemInfo: { | 20260505 06:04:32.183 | INFO | "gpu" : " 17 GB", | 20260505 06:04:32.183 | INFO | "huge" : "", | 20260505 06:04:32.183 | INFO | "huge_shared" : " 27 GB", | 20260505 06:04:32.183 | INFO | "malloc" : " 14 GB", | 20260505 06:04:32.183 | INFO | "pinned" : " 29 GB" | 20260505 06:04:32.183 | INFO | } | 20260505 06:04:32.183 | INFO | Perf report for the last 5.13977 seconds: | 20260505 06:04:32.183 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:04:32.183 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:32.183 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:32.183 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:32.183 | INFO | ANALYSIS : 64.2%, 3.0, 99.9ms, 0 | 20260505 06:04:32.183 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:32.183 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:32.183 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 06:04:32.183 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:32.183 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:32.183 | INFO | Uploaded 36GiB to GPU in 2.16846s (16.6016GiB/s) | 20260505 06:04:32.183 | INFO | Downloaded 34GiB to GPU in 2.03807s (16.6825GiB/s) | 20260505 06:04:37.322 | INFO | Analyzing chunk frames = [914944,915456) | 20260505 06:04:37.322 | INFO | MemInfo: { | 20260505 06:04:37.322 | INFO | "gpu" : " 17 GB", | 20260505 06:04:37.322 | INFO | "huge" : "", | 20260505 06:04:37.322 | INFO | "huge_shared" : " 27 GB", | 20260505 06:04:37.322 | INFO | "malloc" : " 14 GB", | 20260505 06:04:37.322 | INFO | "pinned" : " 29 GB" | 20260505 06:04:37.322 | INFO | } | 20260505 06:04:37.322 | INFO | Perf report for the last 5.13938 seconds: | 20260505 06:04:37.322 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:04:37.322 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:37.322 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:37.322 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:37.322 | INFO | ANALYSIS : 63.4%, 3.0, 98.6ms, 0 | 20260505 06:04:37.322 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:37.322 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:37.322 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 06:04:37.322 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:37.322 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:37.323 | INFO | Uploaded 36GiB to GPU in 2.15289s (16.7217GiB/s) | 20260505 06:04:37.323 | INFO | Downloaded 34GiB to GPU in 2.04231s (16.6478GiB/s) | 20260505 06:04:42.461 | INFO | Analyzing chunk frames = [915456,915968) | 20260505 06:04:42.461 | INFO | MemInfo: { | 20260505 06:04:42.461 | INFO | "gpu" : " 17 GB", | 20260505 06:04:42.461 | INFO | "huge" : "", | 20260505 06:04:42.461 | INFO | "huge_shared" : " 27 GB", | 20260505 06:04:42.461 | INFO | "malloc" : " 14 GB", | 20260505 06:04:42.461 | INFO | "pinned" : " 29 GB" | 20260505 06:04:42.461 | INFO | } | 20260505 06:04:42.461 | INFO | Perf report for the last 5.13909 seconds: | 20260505 06:04:42.461 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:04:42.461 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:42.461 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:42.462 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:42.462 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 06:04:42.462 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:42.462 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:42.462 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 06:04:42.462 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:42.462 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:42.462 | INFO | Uploaded 36GiB to GPU in 2.14822s (16.758GiB/s) | 20260505 06:04:42.462 | INFO | Downloaded 34GiB to GPU in 2.044s (16.634GiB/s) | 20260505 06:04:47.601 | INFO | Analyzing chunk frames = [915968,916480) | 20260505 06:04:47.601 | INFO | MemInfo: { | 20260505 06:04:47.601 | INFO | "gpu" : " 17 GB", | 20260505 06:04:47.601 | INFO | "huge" : "", | 20260505 06:04:47.601 | INFO | "huge_shared" : " 27 GB", | 20260505 06:04:47.601 | INFO | "malloc" : " 14 GB", | 20260505 06:04:47.601 | INFO | "pinned" : " 29 GB" | 20260505 06:04:47.602 | INFO | } | 20260505 06:04:47.602 | INFO | Perf report for the last 5.14012 seconds: | 20260505 06:04:47.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:04:47.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:47.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:47.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:47.602 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 06:04:47.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:47.602 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:47.602 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 06:04:47.602 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:47.602 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:47.602 | INFO | Uploaded 36GiB to GPU in 2.12979s (16.9031GiB/s) | 20260505 06:04:47.602 | INFO | Downloaded 34GiB to GPU in 2.06375s (16.4748GiB/s) | 20260505 06:04:52.740 | INFO | Analyzing chunk frames = [916480,916992) | 20260505 06:04:52.740 | INFO | MemInfo: { | 20260505 06:04:52.740 | INFO | "gpu" : " 17 GB", | 20260505 06:04:52.740 | INFO | "huge" : "", | 20260505 06:04:52.740 | INFO | "huge_shared" : " 27 GB", | 20260505 06:04:52.740 | INFO | "malloc" : " 14 GB", | 20260505 06:04:52.740 | INFO | "pinned" : " 29 GB" | 20260505 06:04:52.740 | INFO | } | 20260505 06:04:52.740 | INFO | Perf report for the last 5.13866 seconds: | 20260505 06:04:52.740 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:04:52.740 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:52.740 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:52.740 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:52.740 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 06:04:52.740 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:52.740 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:52.740 | INFO | PRE_HQ : 32.2%, 1.0, 16.8ms, 0 | 20260505 06:04:52.740 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:52.740 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:52.741 | INFO | Uploaded 36GiB to GPU in 2.17324s (16.5651GiB/s) | 20260505 06:04:52.741 | INFO | Downloaded 34GiB to GPU in 2.15857s (15.7512GiB/s) | 20260505 06:04:57.879 | INFO | Analyzing chunk frames = [916992,917504) | 20260505 06:04:57.879 | INFO | MemInfo: { | 20260505 06:04:57.879 | INFO | "gpu" : " 17 GB", | 20260505 06:04:57.879 | INFO | "huge" : "", | 20260505 06:04:57.879 | INFO | "huge_shared" : " 27 GB", | 20260505 06:04:57.879 | INFO | "malloc" : " 14 GB", | 20260505 06:04:57.879 | INFO | "pinned" : " 29 GB" | 20260505 06:04:57.879 | INFO | } | 20260505 06:04:57.879 | INFO | Perf report for the last 5.13913 seconds: | 20260505 06:04:57.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:04:57.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:57.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:57.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:57.879 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 06:04:57.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:57.880 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:57.880 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 06:04:57.880 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:57.880 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:04:57.880 | INFO | Uploaded 36GiB to GPU in 2.09536s (17.1808GiB/s) | 20260505 06:04:57.880 | INFO | Downloaded 34GiB to GPU in 2.06332s (16.4783GiB/s) | 20260505 06:04:57.880 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 06:04:57.880 | INFO | Total mem usage: 15.3 GiB | 20260505 06:04:57.880 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 06:04:57.880 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.53%) | 20260505 06:04:57.880 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 06:04:57.880 | INFO | High Water Breakdown: | 20260505 06:04:57.880 | INFO | 60.1% : 8.65 GiB ( 92.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 06:04:57.880 | INFO | 39.8% : 5.72 GiB ( 86.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 06:04:57.880 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 06:04:57.880 | INFO | ------------------------------------------ | 20260505 06:04:57.880 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:04:57.880 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 06:04:57.880 | INFO | Total mem usage: 26.5 GiB | 20260505 06:04:57.880 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 06:04:57.880 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 06:04:57.880 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 06:04:57.880 | INFO | High Water Breakdown: | 20260505 06:04:57.880 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:04:57.880 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:04:57.880 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:04:57.880 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:04:57.880 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:04:57.880 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:04:57.880 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 06:04:57.880 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 06:04:57.880 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:04:57.880 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 06:04:57.880 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 06:04:57.880 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 06:04:57.880 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 06:04:57.880 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 06:04:57.880 | INFO | ------------------------------------------ | 20260505 06:04:57.880 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:04:57.880 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 06:04:57.880 | INFO | Total mem usage: 25.6 GiB | 20260505 06:04:57.880 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 06:04:57.880 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 06:04:57.880 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 06:04:57.880 | INFO | High Water Breakdown: | 20260505 06:04:57.880 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 06:04:57.880 | INFO | ------------------------------------------ | 20260505 06:04:57.880 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:04:57.880 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 06:04:57.880 | INFO | Total mem usage: 15.7 GiB | 20260505 06:04:57.880 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 06:04:57.880 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 06:04:57.880 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260505 06:04:57.880 | INFO | High Water Breakdown: | 20260505 06:04:57.880 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 06:04:57.880 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 06:04:57.881 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 06:04:57.881 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 06:04:57.881 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 06:04:57.881 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 06:04:57.881 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 06:04:57.881 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 06:04:57.881 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 06:04:57.881 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:04:57.881 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 06:04:57.881 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:04:57.881 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 06:04:57.881 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 06:04:57.881 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 06:04:57.881 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 06:04:57.881 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:04:57.881 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 06:04:57.881 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:04:57.881 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:04:57.881 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:04:57.881 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 06:04:57.881 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 06:04:57.881 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:04:57.881 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 06:04:57.881 | INFO | ------------------------------------------ | 20260505 06:04:57.881 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:04:57.881 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 06:04:57.881 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:04:57.881 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:04:57.881 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.58e+01, 1.37e+01 / 1.83e+00 | 20260505 06:04:57.881 | INFO | PerfReport: 0.28%, AggregateStats -- 9.42e-02 / 1.03e-01, 1.15e-02 / 7.70e-03 | 20260505 06:04:57.881 | INFO | PerfReport: 11.57%, Baseline -- 3.85e+00 / 4.52e+00, 6.42e-01 / 2.44e-01 | 20260505 06:04:57.881 | INFO | PerfReport: 3.31%, Histogram -- 1.10e+00 / 1.51e+00, 2.21e-01 / 1.14e-01 | 20260505 06:04:57.881 | INFO | PerfReport: 15.49%, DME -- 4.12e+01 / 4.51e+01, 4.33e+00 / 1.40e+00 | 20260505 06:04:57.881 | INFO | PerfReport: 55.86%, FrameLabeling -- 1.86e+01 / 1.91e+01, 2.07e+00 / 9.27e-01 | 20260505 06:04:57.881 | INFO | PerfReport: 7.82%, PulseAccumulating -- 2.60e+00 / 3.17e+00, 4.92e-01 / 3.00e-01 | 20260505 06:04:57.881 | INFO | PerfReport: 4.15%, Metrics -- 1.38e+00 / 1.73e+00, 3.87e-01 / 2.17e-01 | 20260505 06:04:57.881 | INFO | PerfReport: 1.52%, PulseSerializing -- 5.05e-01 / 5.81e-01, 9.99e-02 / 4.10e-02 | 20260505 06:04:57.881 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 06:04:57.881 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:04:57.881 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:04:57.881 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.28e+01 / 0.00e+00 | 20260505 06:04:57.881 | INFO | PerfReport: 21.99%, Upload -- 2.23e+01 / 0.00e+00, 7.65e+00 / 0.00e+00 | 20260505 06:04:57.881 | INFO | PerfReport: 21.54%, Download -- 2.19e+01 / 0.00e+00, 7.73e+00 / 0.00e+00 | 20260505 06:04:57.881 | INFO | PerfReport: 32.75%, Compute -- 3.32e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 06:04:57.881 | INFO | PerfReport: 1.46%, LockUpload -- 1.48e+00 / 0.00e+00, 6.72e+00 / 0.00e+00 | 20260505 06:04:57.881 | INFO | PerfReport: 0.67%, LockDownload -- 6.85e-01 / 0.00e+00, 1.93e+00 / 0.00e+00 | 20260505 06:04:57.881 | INFO | PerfReport: 21.57%, LockCompute -- 2.19e+01 / 0.00e+00, 1.67e+01 / 0.00e+00 | 20260505 06:04:57.881 | INFO | PerfReport: 0.01%, Misc -- 1.50e-02 / 0.00e+00, 2.65e-02 / 0.00e+00 | 20260505 06:04:57.990 | INFO | Starting a round of Baz IO | 20260505 06:05:01.258 | INFO | Populating RTMetrics report | 20260505 06:05:01.259 | INFO | RTMetrics report populated | 20260505 06:05:01.259 | INFO | Opening tmp RTMetrics json file | 20260505 06:05:01.282 | INFO | Serializing RTMetrics to json | 20260505 06:05:01.286 | INFO | Serialization done, swapping tmp file now... | 20260505 06:05:01.295 | INFO | RTMetrics done | 20260505 06:05:01.379 | INFO | Finished a round of Baz IO: Wrote 2.335 GiB in 3.388 seconds | 20260505 06:05:01.379 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.04% / 0.06% / 64.90% | 20260505 06:05:01.379 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.88071% / 9.53674e-05% / 6.75519e-05% | 20260505 06:05:01.379 | INFO | Baz IO overall: | 20260505 06:05:01.379 | INFO | BytesWritten : 459753132753 | 20260505 06:05:01.379 | INFO | NumEvents : 216936487159 | 20260505 06:05:03.018 | INFO | Analyzing chunk frames = [917504,918016) | 20260505 06:05:03.018 | INFO | MemInfo: { | 20260505 06:05:03.018 | INFO | "gpu" : " 17 GB", | 20260505 06:05:03.018 | INFO | "huge" : "", | 20260505 06:05:03.018 | INFO | "huge_shared" : " 27 GB", | 20260505 06:05:03.018 | INFO | "malloc" : " 14 GB", | 20260505 06:05:03.018 | INFO | "pinned" : " 29 GB" | 20260505 06:05:03.018 | INFO | } | 20260505 06:05:03.018 | INFO | Perf report for the last 5.13892 seconds: | 20260505 06:05:03.018 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:05:03.018 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:03.018 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:03.018 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:03.018 | INFO | ANALYSIS : 64.6%, 2.9, 99.1ms, 0 | 20260505 06:05:03.018 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:03.018 | INFO | RT_METRICS : 46.7%, 1.0, 24.5ms, 0 | 20260505 06:05:03.019 | INFO | PRE_HQ : 63.8%, 1.9, 64.5ms, 0 | 20260505 06:05:03.019 | INFO | BAZWRITER : 65.2%, 4.0, 135.4ms, 0 | 20260505 06:05:03.019 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 06:05:03.019 | INFO | Uploaded 36GiB to GPU in 2.1387s (16.8327GiB/s) | 20260505 06:05:03.019 | INFO | Downloaded 38GiB to GPU in 2.33014s (16.3081GiB/s) | 20260505 06:05:08.158 | INFO | Analyzing chunk frames = [918016,918528) | 20260505 06:05:08.158 | INFO | MemInfo: { | 20260505 06:05:08.158 | INFO | "gpu" : " 17 GB", | 20260505 06:05:08.158 | INFO | "huge" : "", | 20260505 06:05:08.158 | INFO | "huge_shared" : " 27 GB", | 20260505 06:05:08.158 | INFO | "malloc" : " 14 GB", | 20260505 06:05:08.158 | INFO | "pinned" : " 29 GB" | 20260505 06:05:08.158 | INFO | } | 20260505 06:05:08.158 | INFO | Perf report for the last 5.13977 seconds: | 20260505 06:05:08.158 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:05:08.158 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:08.158 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:08.158 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:08.158 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 06:05:08.158 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:08.158 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:08.158 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 06:05:08.158 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:08.158 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:08.158 | INFO | Uploaded 36GiB to GPU in 2.13191s (16.8863GiB/s) | 20260505 06:05:08.158 | INFO | Downloaded 34GiB to GPU in 2.03851s (16.6788GiB/s) | 20260505 06:05:13.297 | INFO | Analyzing chunk frames = [918528,919040) | 20260505 06:05:13.297 | INFO | MemInfo: { | 20260505 06:05:13.297 | INFO | "gpu" : " 17 GB", | 20260505 06:05:13.297 | INFO | "huge" : "", | 20260505 06:05:13.297 | INFO | "huge_shared" : " 27 GB", | 20260505 06:05:13.297 | INFO | "malloc" : " 14 GB", | 20260505 06:05:13.297 | INFO | "pinned" : " 29 GB" | 20260505 06:05:13.297 | INFO | } | 20260505 06:05:13.298 | INFO | Perf report for the last 5.13938 seconds: | 20260505 06:05:13.298 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:05:13.298 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:13.298 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:13.298 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:13.298 | INFO | ANALYSIS : 64.3%, 3.0, 100.0ms, 0 | 20260505 06:05:13.298 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:13.298 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:13.298 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 06:05:13.298 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:13.298 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:13.298 | INFO | Uploaded 36GiB to GPU in 2.17056s (16.5856GiB/s) | 20260505 06:05:13.298 | INFO | Downloaded 34GiB to GPU in 2.04416s (16.6327GiB/s) | 20260505 06:05:18.437 | INFO | Analyzing chunk frames = [919040,919552) | 20260505 06:05:18.437 | INFO | MemInfo: { | 20260505 06:05:18.437 | INFO | "gpu" : " 17 GB", | 20260505 06:05:18.437 | INFO | "huge" : "", | 20260505 06:05:18.437 | INFO | "huge_shared" : " 27 GB", | 20260505 06:05:18.437 | INFO | "malloc" : " 14 GB", | 20260505 06:05:18.437 | INFO | "pinned" : " 29 GB" | 20260505 06:05:18.437 | INFO | } | 20260505 06:05:18.437 | INFO | Perf report for the last 5.13943 seconds: | 20260505 06:05:18.437 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:05:18.437 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:18.437 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:18.437 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:18.437 | INFO | ANALYSIS : 63.5%, 3.0, 98.6ms, 0 | 20260505 06:05:18.437 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:18.437 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:18.437 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 06:05:18.437 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:18.437 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:18.437 | INFO | Uploaded 36GiB to GPU in 2.15463s (16.7082GiB/s) | 20260505 06:05:18.437 | INFO | Downloaded 34GiB to GPU in 2.05149s (16.5733GiB/s) | 20260505 06:05:23.576 | INFO | Analyzing chunk frames = [919552,920064) | 20260505 06:05:23.576 | INFO | MemInfo: { | 20260505 06:05:23.576 | INFO | "gpu" : " 17 GB", | 20260505 06:05:23.576 | INFO | "huge" : "", | 20260505 06:05:23.576 | INFO | "huge_shared" : " 27 GB", | 20260505 06:05:23.576 | INFO | "malloc" : " 14 GB", | 20260505 06:05:23.576 | INFO | "pinned" : " 29 GB" | 20260505 06:05:23.576 | INFO | } | 20260505 06:05:23.577 | INFO | Perf report for the last 5.13954 seconds: | 20260505 06:05:23.577 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:05:23.577 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:23.577 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:23.577 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:23.577 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 06:05:23.577 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:23.577 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:23.577 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 06:05:23.577 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:23.577 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:23.577 | INFO | Uploaded 36GiB to GPU in 2.12486s (16.9423GiB/s) | 20260505 06:05:23.577 | INFO | Downloaded 34GiB to GPU in 2.04828s (16.5993GiB/s) | 20260505 06:05:28.716 | INFO | Analyzing chunk frames = [920064,920576) | 20260505 06:05:28.716 | INFO | MemInfo: { | 20260505 06:05:28.716 | INFO | "gpu" : " 17 GB", | 20260505 06:05:28.716 | INFO | "huge" : "", | 20260505 06:05:28.716 | INFO | "huge_shared" : " 27 GB", | 20260505 06:05:28.716 | INFO | "malloc" : " 14 GB", | 20260505 06:05:28.716 | INFO | "pinned" : " 29 GB" | 20260505 06:05:28.716 | INFO | } | 20260505 06:05:28.716 | INFO | Perf report for the last 5.13944 seconds: | 20260505 06:05:28.716 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:05:28.716 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:28.716 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:28.716 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:28.716 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 06:05:28.716 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:28.716 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:28.716 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 06:05:28.716 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:28.716 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:28.716 | INFO | Uploaded 36GiB to GPU in 2.12897s (16.9096GiB/s) | 20260505 06:05:28.716 | INFO | Downloaded 34GiB to GPU in 2.06376s (16.4748GiB/s) | 20260505 06:05:33.855 | INFO | Analyzing chunk frames = [920576,921088) | 20260505 06:05:33.855 | INFO | MemInfo: { | 20260505 06:05:33.855 | INFO | "gpu" : " 17 GB", | 20260505 06:05:33.855 | INFO | "huge" : "", | 20260505 06:05:33.855 | INFO | "huge_shared" : " 27 GB", | 20260505 06:05:33.855 | INFO | "malloc" : " 14 GB", | 20260505 06:05:33.855 | INFO | "pinned" : " 29 GB" | 20260505 06:05:33.855 | INFO | } | 20260505 06:05:33.855 | INFO | Perf report for the last 5.13902 seconds: | 20260505 06:05:33.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:05:33.855 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:33.855 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:33.855 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:33.855 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 06:05:33.855 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:33.855 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:33.855 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 06:05:33.855 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:33.855 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:33.855 | INFO | Uploaded 36GiB to GPU in 2.1693s (16.5952GiB/s) | 20260505 06:05:33.855 | INFO | Downloaded 34GiB to GPU in 2.14605s (15.8431GiB/s) | 20260505 06:05:38.994 | INFO | Analyzing chunk frames = [921088,921600) | 20260505 06:05:38.994 | INFO | MemInfo: { | 20260505 06:05:38.994 | INFO | "gpu" : " 17 GB", | 20260505 06:05:38.994 | INFO | "huge" : "", | 20260505 06:05:38.994 | INFO | "huge_shared" : " 27 GB", | 20260505 06:05:38.994 | INFO | "malloc" : " 14 GB", | 20260505 06:05:38.994 | INFO | "pinned" : " 29 GB" | 20260505 06:05:38.994 | INFO | } | 20260505 06:05:38.995 | INFO | Perf report for the last 5.13952 seconds: | 20260505 06:05:38.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:05:38.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:38.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:38.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:38.995 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 06:05:38.995 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:38.995 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:38.995 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 06:05:38.995 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:38.995 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:38.995 | INFO | Uploaded 36GiB to GPU in 2.09979s (17.1445GiB/s) | 20260505 06:05:38.995 | INFO | Downloaded 34GiB to GPU in 2.073s (16.4014GiB/s) | 20260505 06:05:39.105 | INFO | Starting a round of Baz IO | 20260505 06:05:42.366 | INFO | Populating RTMetrics report | 20260505 06:05:42.366 | INFO | RTMetrics report populated | 20260505 06:05:42.366 | INFO | Opening tmp RTMetrics json file | 20260505 06:05:42.384 | INFO | Serializing RTMetrics to json | 20260505 06:05:42.387 | INFO | Serialization done, swapping tmp file now... | 20260505 06:05:42.387 | INFO | RTMetrics done | 20260505 06:05:42.480 | INFO | Finished a round of Baz IO: Wrote 2.337 GiB in 3.375 seconds | 20260505 06:05:42.480 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.98% / 0.06% / 64.96% | 20260505 06:05:42.480 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.8557% / 7.54992e-05% / 5.5631e-05% | 20260505 06:05:42.480 | INFO | Baz IO overall: | 20260505 06:05:42.480 | INFO | BytesWritten : 462261935767 | 20260505 06:05:42.480 | INFO | NumEvents : 218135419257 | 20260505 06:05:44.134 | INFO | Analyzing chunk frames = [921600,922112) | 20260505 06:05:44.134 | INFO | MemInfo: { | 20260505 06:05:44.134 | INFO | "gpu" : " 17 GB", | 20260505 06:05:44.134 | INFO | "huge" : "", | 20260505 06:05:44.134 | INFO | "huge_shared" : " 27 GB", | 20260505 06:05:44.134 | INFO | "malloc" : " 14 GB", | 20260505 06:05:44.134 | INFO | "pinned" : " 29 GB" | 20260505 06:05:44.134 | INFO | } | 20260505 06:05:44.134 | INFO | Perf report for the last 5.13922 seconds: | 20260505 06:05:44.134 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:05:44.134 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:44.134 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:44.134 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:44.134 | INFO | ANALYSIS : 64.5%, 2.9, 98.6ms, 0 | 20260505 06:05:44.134 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:44.134 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260505 06:05:44.134 | INFO | PRE_HQ : 63.6%, 1.9, 64.1ms, 0 | 20260505 06:05:44.134 | INFO | BAZWRITER : 65.0%, 4.1, 135.8ms, 0 | 20260505 06:05:44.134 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 06:05:44.134 | INFO | Uploaded 36GiB to GPU in 2.15369s (16.7155GiB/s) | 20260505 06:05:44.134 | INFO | Downloaded 38GiB to GPU in 2.32663s (16.3326GiB/s) | 20260505 06:05:49.273 | INFO | Analyzing chunk frames = [922112,922624) | 20260505 06:05:49.273 | INFO | MemInfo: { | 20260505 06:05:49.273 | INFO | "gpu" : " 17 GB", | 20260505 06:05:49.273 | INFO | "huge" : "", | 20260505 06:05:49.273 | INFO | "huge_shared" : " 27 GB", | 20260505 06:05:49.273 | INFO | "malloc" : " 14 GB", | 20260505 06:05:49.273 | INFO | "pinned" : " 29 GB" | 20260505 06:05:49.273 | INFO | } | 20260505 06:05:49.273 | INFO | Perf report for the last 5.13907 seconds: | 20260505 06:05:49.273 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:05:49.273 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:49.273 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:49.273 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:49.273 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 06:05:49.273 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:49.273 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:49.273 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 06:05:49.273 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:49.273 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:49.273 | INFO | Uploaded 36GiB to GPU in 2.12883s (16.9107GiB/s) | 20260505 06:05:49.273 | INFO | Downloaded 34GiB to GPU in 2.03828s (16.6808GiB/s) | 20260505 06:05:54.412 | INFO | Analyzing chunk frames = [922624,923136) | 20260505 06:05:54.412 | INFO | MemInfo: { | 20260505 06:05:54.412 | INFO | "gpu" : " 17 GB", | 20260505 06:05:54.412 | INFO | "huge" : "", | 20260505 06:05:54.412 | INFO | "huge_shared" : " 27 GB", | 20260505 06:05:54.412 | INFO | "malloc" : " 14 GB", | 20260505 06:05:54.412 | INFO | "pinned" : " 29 GB" | 20260505 06:05:54.412 | INFO | } | 20260505 06:05:54.412 | INFO | Perf report for the last 5.13964 seconds: | 20260505 06:05:54.412 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:05:54.412 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:54.412 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:54.413 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:54.413 | INFO | ANALYSIS : 64.2%, 3.0, 100.0ms, 0 | 20260505 06:05:54.413 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:54.413 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:54.413 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 06:05:54.413 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:54.413 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:54.413 | INFO | Uploaded 36GiB to GPU in 2.1659s (16.6212GiB/s) | 20260505 06:05:54.413 | INFO | Downloaded 34GiB to GPU in 2.04223s (16.6485GiB/s) | 20260505 06:05:59.552 | INFO | Analyzing chunk frames = [923136,923648) | 20260505 06:05:59.552 | INFO | MemInfo: { | 20260505 06:05:59.552 | INFO | "gpu" : " 17 GB", | 20260505 06:05:59.552 | INFO | "huge" : "", | 20260505 06:05:59.552 | INFO | "huge_shared" : " 27 GB", | 20260505 06:05:59.552 | INFO | "malloc" : " 14 GB", | 20260505 06:05:59.552 | INFO | "pinned" : " 29 GB" | 20260505 06:05:59.552 | INFO | } | 20260505 06:05:59.552 | INFO | Perf report for the last 5.1394 seconds: | 20260505 06:05:59.552 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:05:59.552 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:59.552 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:59.552 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:59.552 | INFO | ANALYSIS : 63.4%, 3.0, 98.5ms, 0 | 20260505 06:05:59.552 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:59.552 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:59.552 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 06:05:59.552 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:59.552 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:05:59.552 | INFO | Uploaded 36GiB to GPU in 2.13972s (16.8247GiB/s) | 20260505 06:05:59.552 | INFO | Downloaded 34GiB to GPU in 2.05013s (16.5843GiB/s) | 20260505 06:06:04.690 | INFO | Analyzing chunk frames = [923648,924160) | 20260505 06:06:04.690 | INFO | MemInfo: { | 20260505 06:06:04.690 | INFO | "gpu" : " 17 GB", | 20260505 06:06:04.690 | INFO | "huge" : "", | 20260505 06:06:04.690 | INFO | "huge_shared" : " 27 GB", | 20260505 06:06:04.690 | INFO | "malloc" : " 14 GB", | 20260505 06:06:04.690 | INFO | "pinned" : " 29 GB" | 20260505 06:06:04.690 | INFO | } | 20260505 06:06:04.690 | INFO | Perf report for the last 5.1386 seconds: | 20260505 06:06:04.691 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:06:04.691 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:04.691 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:04.691 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:04.691 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 06:06:04.691 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:04.691 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:04.691 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 06:06:04.691 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:04.691 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:04.691 | INFO | Uploaded 36GiB to GPU in 2.11555s (17.0168GiB/s) | 20260505 06:06:04.691 | INFO | Downloaded 34GiB to GPU in 2.03943s (16.6713GiB/s) | 20260505 06:06:09.831 | INFO | Analyzing chunk frames = [924160,924672) | 20260505 06:06:09.831 | INFO | MemInfo: { | 20260505 06:06:09.831 | INFO | "gpu" : " 17 GB", | 20260505 06:06:09.831 | INFO | "huge" : "", | 20260505 06:06:09.831 | INFO | "huge_shared" : " 27 GB", | 20260505 06:06:09.831 | INFO | "malloc" : " 14 GB", | 20260505 06:06:09.831 | INFO | "pinned" : " 29 GB" | 20260505 06:06:09.831 | INFO | } | 20260505 06:06:09.831 | INFO | Perf report for the last 5.14048 seconds: | 20260505 06:06:09.831 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:06:09.831 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:09.831 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:09.831 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:09.831 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 06:06:09.831 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:09.831 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:09.831 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 06:06:09.831 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:09.831 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:09.831 | INFO | Uploaded 36GiB to GPU in 2.11716s (17.0039GiB/s) | 20260505 06:06:09.831 | INFO | Downloaded 34GiB to GPU in 2.06745s (16.4454GiB/s) | 20260505 06:06:14.969 | INFO | Analyzing chunk frames = [924672,925184) | 20260505 06:06:14.969 | INFO | MemInfo: { | 20260505 06:06:14.969 | INFO | "gpu" : " 17 GB", | 20260505 06:06:14.969 | INFO | "huge" : "", | 20260505 06:06:14.969 | INFO | "huge_shared" : " 27 GB", | 20260505 06:06:14.969 | INFO | "malloc" : " 14 GB", | 20260505 06:06:14.969 | INFO | "pinned" : " 29 GB" | 20260505 06:06:14.969 | INFO | } | 20260505 06:06:14.969 | INFO | Perf report for the last 5.13837 seconds: | 20260505 06:06:14.969 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:06:14.969 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:14.970 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:14.970 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:14.970 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 06:06:14.970 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:14.970 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:14.970 | INFO | PRE_HQ : 32.2%, 1.0, 16.8ms, 0 | 20260505 06:06:14.970 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:14.970 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:14.970 | INFO | Uploaded 36GiB to GPU in 2.18595s (16.4688GiB/s) | 20260505 06:06:14.970 | INFO | Downloaded 34GiB to GPU in 2.15905s (15.7476GiB/s) | 20260505 06:06:20.109 | INFO | Analyzing chunk frames = [925184,925696) | 20260505 06:06:20.109 | INFO | MemInfo: { | 20260505 06:06:20.109 | INFO | "gpu" : " 17 GB", | 20260505 06:06:20.109 | INFO | "huge" : "", | 20260505 06:06:20.109 | INFO | "huge_shared" : " 27 GB", | 20260505 06:06:20.109 | INFO | "malloc" : " 14 GB", | 20260505 06:06:20.109 | INFO | "pinned" : " 29 GB" | 20260505 06:06:20.109 | INFO | } | 20260505 06:06:20.109 | INFO | Perf report for the last 5.13996 seconds: | 20260505 06:06:20.109 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:06:20.109 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:20.109 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:20.109 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:20.109 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 06:06:20.109 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:20.109 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:20.109 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260505 06:06:20.109 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:20.109 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:20.110 | INFO | Uploaded 36GiB to GPU in 2.12843s (16.9139GiB/s) | 20260505 06:06:20.110 | INFO | Downloaded 34GiB to GPU in 2.07971s (16.3484GiB/s) | 20260505 06:06:20.110 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 06:06:20.110 | INFO | Total mem usage: 15.3 GiB | 20260505 06:06:20.110 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 06:06:20.110 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.48%) | 20260505 06:06:20.110 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 06:06:20.110 | INFO | High Water Breakdown: | 20260505 06:06:20.110 | INFO | 60.1% : 8.66 GiB ( 92.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 06:06:20.110 | INFO | 39.7% : 5.72 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 06:06:20.110 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 06:06:20.110 | INFO | ------------------------------------------ | 20260505 06:06:20.110 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:06:20.110 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 06:06:20.110 | INFO | Total mem usage: 26.5 GiB | 20260505 06:06:20.110 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 06:06:20.110 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 06:06:20.110 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 06:06:20.110 | INFO | High Water Breakdown: | 20260505 06:06:20.110 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:06:20.110 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:06:20.110 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:06:20.110 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:06:20.110 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:06:20.110 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:06:20.110 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 06:06:20.110 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 06:06:20.110 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:06:20.110 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 06:06:20.110 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 06:06:20.110 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 06:06:20.110 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 06:06:20.110 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 06:06:20.110 | INFO | ------------------------------------------ | 20260505 06:06:20.110 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:06:20.110 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 06:06:20.110 | INFO | Total mem usage: 25.6 GiB | 20260505 06:06:20.110 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 06:06:20.110 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 06:06:20.110 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 06:06:20.110 | INFO | High Water Breakdown: | 20260505 06:06:20.110 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 06:06:20.110 | INFO | ------------------------------------------ | 20260505 06:06:20.110 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:06:20.110 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 06:06:20.110 | INFO | Total mem usage: 15.7 GiB | 20260505 06:06:20.110 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 06:06:20.110 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 06:06:20.110 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260505 06:06:20.110 | INFO | High Water Breakdown: | 20260505 06:06:20.110 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 06:06:20.110 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 06:06:20.110 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 06:06:20.110 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 06:06:20.110 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 06:06:20.110 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 06:06:20.110 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 06:06:20.110 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 06:06:20.110 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 06:06:20.110 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:06:20.110 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 06:06:20.110 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:06:20.110 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 06:06:20.110 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 06:06:20.110 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 06:06:20.110 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 06:06:20.110 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:06:20.110 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 06:06:20.110 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:06:20.111 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:06:20.111 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:06:20.111 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 06:06:20.111 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 06:06:20.111 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:06:20.111 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 06:06:20.111 | INFO | ------------------------------------------ | 20260505 06:06:20.111 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:06:20.111 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 06:06:20.111 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:06:20.111 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:06:20.111 | INFO | PerfReport: Full Execution -- 3.33e+01 / 7.60e+01, 1.40e+01 / 1.73e+00 | 20260505 06:06:20.111 | INFO | PerfReport: 0.28%, AggregateStats -- 9.40e-02 / 1.04e-01, 1.12e-02 / 5.37e-03 | 20260505 06:06:20.111 | INFO | PerfReport: 11.55%, Baseline -- 3.84e+00 / 4.50e+00, 6.36e-01 / 2.54e-01 | 20260505 06:06:20.111 | INFO | PerfReport: 3.32%, Histogram -- 1.10e+00 / 1.53e+00, 2.23e-01 / 1.04e-01 | 20260505 06:06:20.111 | INFO | PerfReport: 15.50%, DME -- 4.12e+01 / 4.54e+01, 4.50e+00 / 1.27e+00 | 20260505 06:06:20.111 | INFO | PerfReport: 55.92%, FrameLabeling -- 1.86e+01 / 1.90e+01, 2.07e+00 / 9.25e-01 | 20260505 06:06:20.111 | INFO | PerfReport: 7.77%, PulseAccumulating -- 2.58e+00 / 3.17e+00, 4.86e-01 / 3.30e-01 | 20260505 06:06:20.111 | INFO | PerfReport: 4.14%, Metrics -- 1.38e+00 / 1.74e+00, 3.91e-01 / 2.11e-01 | 20260505 06:06:20.111 | INFO | PerfReport: 1.52%, PulseSerializing -- 5.04e-01 / 5.85e-01, 1.01e-01 / 4.15e-02 | 20260505 06:06:20.111 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 06:06:20.111 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:06:20.111 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:06:20.111 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.29e+01 / 0.00e+00 | 20260505 06:06:20.111 | INFO | PerfReport: 21.96%, Upload -- 2.23e+01 / 0.00e+00, 7.68e+00 / 0.00e+00 | 20260505 06:06:20.111 | INFO | PerfReport: 21.54%, Download -- 2.19e+01 / 0.00e+00, 7.76e+00 / 0.00e+00 | 20260505 06:06:20.111 | INFO | PerfReport: 32.76%, Compute -- 3.33e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260505 06:06:20.111 | INFO | PerfReport: 1.42%, LockUpload -- 1.45e+00 / 0.00e+00, 6.69e+00 / 0.00e+00 | 20260505 06:06:20.111 | INFO | PerfReport: 0.65%, LockDownload -- 6.63e-01 / 0.00e+00, 1.85e+00 / 0.00e+00 | 20260505 06:06:20.111 | INFO | PerfReport: 21.65%, LockCompute -- 2.20e+01 / 0.00e+00, 1.68e+01 / 0.00e+00 | 20260505 06:06:20.111 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 1.87e-02 / 0.00e+00 | 20260505 06:06:20.221 | INFO | Starting a round of Baz IO | 20260505 06:06:23.496 | INFO | Populating RTMetrics report | 20260505 06:06:23.496 | INFO | RTMetrics report populated | 20260505 06:06:23.496 | INFO | Opening tmp RTMetrics json file | 20260505 06:06:23.517 | INFO | Serializing RTMetrics to json | 20260505 06:06:23.520 | INFO | Serialization done, swapping tmp file now... | 20260505 06:06:23.521 | INFO | RTMetrics done | 20260505 06:06:23.609 | INFO | Finished a round of Baz IO: Wrote 2.336 GiB in 3.388 seconds | 20260505 06:06:23.609 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.92% / 0.06% / 65.02% | 20260505 06:06:23.609 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.8186% / 8.74201e-05% / 5.96046e-05% | 20260505 06:06:23.609 | INFO | Baz IO overall: | 20260505 06:06:23.609 | INFO | BytesWritten : 464770010058 | 20260505 06:06:23.609 | INFO | NumEvents : 219333981934 | 20260505 06:06:25.248 | INFO | Analyzing chunk frames = [925696,926208) | 20260505 06:06:25.248 | INFO | MemInfo: { | 20260505 06:06:25.248 | INFO | "gpu" : " 17 GB", | 20260505 06:06:25.248 | INFO | "huge" : "", | 20260505 06:06:25.248 | INFO | "huge_shared" : " 27 GB", | 20260505 06:06:25.248 | INFO | "malloc" : " 14 GB", | 20260505 06:06:25.248 | INFO | "pinned" : " 29 GB" | 20260505 06:06:25.248 | INFO | } | 20260505 06:06:25.248 | INFO | Perf report for the last 5.1388 seconds: | 20260505 06:06:25.248 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:06:25.248 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:25.248 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:25.248 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:25.248 | INFO | ANALYSIS : 64.7%, 2.9, 99.0ms, 0 | 20260505 06:06:25.248 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:25.248 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260505 06:06:25.248 | INFO | PRE_HQ : 63.9%, 1.9, 64.2ms, 0 | 20260505 06:06:25.248 | INFO | BAZWRITER : 65.2%, 4.0, 135.4ms, 0 | 20260505 06:06:25.248 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 06:06:25.248 | INFO | Uploaded 36GiB to GPU in 2.16695s (16.6132GiB/s) | 20260505 06:06:25.248 | INFO | Downloaded 38GiB to GPU in 2.32633s (16.3348GiB/s) | 20260505 06:06:30.387 | INFO | Analyzing chunk frames = [926208,926720) | 20260505 06:06:30.387 | INFO | MemInfo: { | 20260505 06:06:30.387 | INFO | "gpu" : " 17 GB", | 20260505 06:06:30.387 | INFO | "huge" : "", | 20260505 06:06:30.387 | INFO | "huge_shared" : " 27 GB", | 20260505 06:06:30.387 | INFO | "malloc" : " 14 GB", | 20260505 06:06:30.387 | INFO | "pinned" : " 29 GB" | 20260505 06:06:30.387 | INFO | } | 20260505 06:06:30.388 | INFO | Perf report for the last 5.13939 seconds: | 20260505 06:06:30.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:06:30.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:30.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:30.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:30.388 | INFO | ANALYSIS : 64.3%, 3.0, 100.5ms, 0 | 20260505 06:06:30.388 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:30.388 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:30.388 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 06:06:30.388 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:30.388 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:30.388 | INFO | Uploaded 36GiB to GPU in 2.1284s (16.9141GiB/s) | 20260505 06:06:30.388 | INFO | Downloaded 34GiB to GPU in 2.04331s (16.6397GiB/s) | 20260505 06:06:35.527 | INFO | Analyzing chunk frames = [926720,927232) | 20260505 06:06:35.527 | INFO | MemInfo: { | 20260505 06:06:35.527 | INFO | "gpu" : " 17 GB", | 20260505 06:06:35.527 | INFO | "huge" : "", | 20260505 06:06:35.527 | INFO | "huge_shared" : " 27 GB", | 20260505 06:06:35.527 | INFO | "malloc" : " 14 GB", | 20260505 06:06:35.527 | INFO | "pinned" : " 29 GB" | 20260505 06:06:35.527 | INFO | } | 20260505 06:06:35.527 | INFO | Perf report for the last 5.13944 seconds: | 20260505 06:06:35.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:06:35.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:35.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:35.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:35.527 | INFO | ANALYSIS : 64.2%, 3.0, 99.8ms, 0 | 20260505 06:06:35.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:35.527 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:35.527 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 06:06:35.527 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:35.527 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:35.527 | INFO | Uploaded 36GiB to GPU in 2.15703s (16.6896GiB/s) | 20260505 06:06:35.527 | INFO | Downloaded 34GiB to GPU in 2.03221s (16.7306GiB/s) | 20260505 06:06:40.666 | INFO | Analyzing chunk frames = [927232,927744) | 20260505 06:06:40.666 | INFO | MemInfo: { | 20260505 06:06:40.666 | INFO | "gpu" : " 17 GB", | 20260505 06:06:40.666 | INFO | "huge" : "", | 20260505 06:06:40.666 | INFO | "huge_shared" : " 27 GB", | 20260505 06:06:40.666 | INFO | "malloc" : " 14 GB", | 20260505 06:06:40.666 | INFO | "pinned" : " 29 GB" | 20260505 06:06:40.666 | INFO | } | 20260505 06:06:40.666 | INFO | Perf report for the last 5.13946 seconds: | 20260505 06:06:40.666 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:06:40.666 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:40.666 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:40.667 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:40.667 | INFO | ANALYSIS : 63.4%, 3.0, 98.5ms, 0 | 20260505 06:06:40.667 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:40.667 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:40.667 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260505 06:06:40.667 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:40.667 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:40.667 | INFO | Uploaded 36GiB to GPU in 2.14364s (16.7938GiB/s) | 20260505 06:06:40.667 | INFO | Downloaded 34GiB to GPU in 2.03964s (16.6696GiB/s) | 20260505 06:06:45.805 | INFO | Analyzing chunk frames = [927744,928256) | 20260505 06:06:45.805 | INFO | MemInfo: { | 20260505 06:06:45.805 | INFO | "gpu" : " 17 GB", | 20260505 06:06:45.805 | INFO | "huge" : "", | 20260505 06:06:45.805 | INFO | "huge_shared" : " 27 GB", | 20260505 06:06:45.805 | INFO | "malloc" : " 14 GB", | 20260505 06:06:45.805 | INFO | "pinned" : " 29 GB" | 20260505 06:06:45.805 | INFO | } | 20260505 06:06:45.805 | INFO | Perf report for the last 5.13901 seconds: | 20260505 06:06:45.806 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:06:45.806 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:45.806 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:45.806 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:45.806 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 06:06:45.806 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:45.806 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:45.806 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 06:06:45.806 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:45.806 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:45.806 | INFO | Uploaded 36GiB to GPU in 2.11192s (17.0461GiB/s) | 20260505 06:06:45.806 | INFO | Downloaded 34GiB to GPU in 2.04422s (16.6323GiB/s) | 20260505 06:06:50.946 | INFO | Analyzing chunk frames = [928256,928768) | 20260505 06:06:50.946 | INFO | MemInfo: { | 20260505 06:06:50.946 | INFO | "gpu" : " 17 GB", | 20260505 06:06:50.946 | INFO | "huge" : "", | 20260505 06:06:50.946 | INFO | "huge_shared" : " 27 GB", | 20260505 06:06:50.946 | INFO | "malloc" : " 14 GB", | 20260505 06:06:50.946 | INFO | "pinned" : " 29 GB" | 20260505 06:06:50.946 | INFO | } | 20260505 06:06:50.946 | INFO | Perf report for the last 5.14032 seconds: | 20260505 06:06:50.946 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:06:50.946 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:50.946 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:50.946 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:50.946 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 06:06:50.946 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:50.946 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:50.946 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 06:06:50.946 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:50.946 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:50.946 | INFO | Uploaded 36GiB to GPU in 2.12084s (16.9744GiB/s) | 20260505 06:06:50.946 | INFO | Downloaded 34GiB to GPU in 2.06117s (16.4955GiB/s) | 20260505 06:06:56.084 | INFO | Analyzing chunk frames = [928768,929280) | 20260505 06:06:56.084 | INFO | MemInfo: { | 20260505 06:06:56.084 | INFO | "gpu" : " 17 GB", | 20260505 06:06:56.084 | INFO | "huge" : "", | 20260505 06:06:56.084 | INFO | "huge_shared" : " 27 GB", | 20260505 06:06:56.084 | INFO | "malloc" : " 14 GB", | 20260505 06:06:56.084 | INFO | "pinned" : " 29 GB" | 20260505 06:06:56.084 | INFO | } | 20260505 06:06:56.084 | INFO | Perf report for the last 5.13857 seconds: | 20260505 06:06:56.084 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:06:56.084 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:56.084 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:56.084 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:56.084 | INFO | ANALYSIS : 65.0%, 3.0, 101.5ms, 0 | 20260505 06:06:56.084 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:56.084 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:56.085 | INFO | PRE_HQ : 32.5%, 1.0, 17.0ms, 0 | 20260505 06:06:56.085 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:56.085 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:06:56.085 | INFO | Uploaded 36GiB to GPU in 2.24025s (16.0697GiB/s) | 20260505 06:06:56.085 | INFO | Downloaded 34GiB to GPU in 2.21427s (15.355GiB/s) | 20260505 06:07:01.223 | INFO | Analyzing chunk frames = [929280,929792) | 20260505 06:07:01.223 | INFO | MemInfo: { | 20260505 06:07:01.223 | INFO | "gpu" : " 17 GB", | 20260505 06:07:01.223 | INFO | "huge" : "", | 20260505 06:07:01.223 | INFO | "huge_shared" : " 27 GB", | 20260505 06:07:01.223 | INFO | "malloc" : " 14 GB", | 20260505 06:07:01.224 | INFO | "pinned" : " 29 GB" | 20260505 06:07:01.224 | INFO | } | 20260505 06:07:01.224 | INFO | Perf report for the last 5.13934 seconds: | 20260505 06:07:01.224 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:07:01.224 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:01.224 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:01.224 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:01.224 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 06:07:01.224 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:01.224 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:01.224 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 06:07:01.224 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:01.224 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:01.224 | INFO | Uploaded 36GiB to GPU in 2.14573s (16.7775GiB/s) | 20260505 06:07:01.224 | INFO | Downloaded 34GiB to GPU in 2.05005s (16.585GiB/s) | 20260505 06:07:01.357 | INFO | Starting a round of Baz IO | 20260505 06:07:04.611 | INFO | Populating RTMetrics report | 20260505 06:07:04.611 | INFO | RTMetrics report populated | 20260505 06:07:04.611 | INFO | Opening tmp RTMetrics json file | 20260505 06:07:04.629 | INFO | Serializing RTMetrics to json | 20260505 06:07:04.631 | INFO | Serialization done, swapping tmp file now... | 20260505 06:07:04.631 | INFO | RTMetrics done | 20260505 06:07:04.725 | INFO | Finished a round of Baz IO: Wrote 2.337 GiB in 3.367 seconds | 20260505 06:07:04.725 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.87% / 0.06% / 65.08% | 20260505 06:07:04.725 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.79585% / 0.000103315% / 7.94729e-05% | 20260505 06:07:04.725 | INFO | Baz IO overall: | 20260505 06:07:04.725 | INFO | BytesWritten : 467279286519 | 20260505 06:07:04.725 | INFO | NumEvents : 220533374642 | 20260505 06:07:06.363 | INFO | Analyzing chunk frames = [929792,930304) | 20260505 06:07:06.363 | INFO | MemInfo: { | 20260505 06:07:06.363 | INFO | "gpu" : " 17 GB", | 20260505 06:07:06.363 | INFO | "huge" : "", | 20260505 06:07:06.363 | INFO | "huge_shared" : " 27 GB", | 20260505 06:07:06.363 | INFO | "malloc" : " 14 GB", | 20260505 06:07:06.363 | INFO | "pinned" : " 29 GB" | 20260505 06:07:06.363 | INFO | } | 20260505 06:07:06.363 | INFO | Perf report for the last 5.13951 seconds: | 20260505 06:07:06.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:07:06.363 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:06.363 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:06.363 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:06.363 | INFO | ANALYSIS : 64.7%, 2.9, 98.7ms, 0 | 20260505 06:07:06.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:06.363 | INFO | RT_METRICS : 46.6%, 1.0, 24.4ms, 0 | 20260505 06:07:06.363 | INFO | PRE_HQ : 63.9%, 2.0, 64.8ms, 0 | 20260505 06:07:06.363 | INFO | BAZWRITER : 65.5%, 4.1, 136.7ms, 0 | 20260505 06:07:06.363 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 06:07:06.364 | INFO | Uploaded 36GiB to GPU in 2.1725s (16.5708GiB/s) | 20260505 06:07:06.364 | INFO | Downloaded 38GiB to GPU in 2.3328s (16.2894GiB/s) | 20260505 06:07:11.502 | INFO | Analyzing chunk frames = [930304,930816) | 20260505 06:07:11.502 | INFO | MemInfo: { | 20260505 06:07:11.502 | INFO | "gpu" : " 17 GB", | 20260505 06:07:11.502 | INFO | "huge" : "", | 20260505 06:07:11.502 | INFO | "huge_shared" : " 27 GB", | 20260505 06:07:11.502 | INFO | "malloc" : " 14 GB", | 20260505 06:07:11.502 | INFO | "pinned" : " 29 GB" | 20260505 06:07:11.502 | INFO | } | 20260505 06:07:11.502 | INFO | Perf report for the last 5.13914 seconds: | 20260505 06:07:11.502 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:07:11.502 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:11.502 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:11.502 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:11.502 | INFO | ANALYSIS : 64.4%, 3.0, 100.6ms, 0 | 20260505 06:07:11.502 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:11.502 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:11.503 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 06:07:11.503 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:11.503 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:11.503 | INFO | Uploaded 36GiB to GPU in 2.13793s (16.8387GiB/s) | 20260505 06:07:11.503 | INFO | Downloaded 34GiB to GPU in 2.03875s (16.6769GiB/s) | 20260505 06:07:16.642 | INFO | Analyzing chunk frames = [930816,931328) | 20260505 06:07:16.642 | INFO | MemInfo: { | 20260505 06:07:16.642 | INFO | "gpu" : " 17 GB", | 20260505 06:07:16.642 | INFO | "huge" : "", | 20260505 06:07:16.642 | INFO | "huge_shared" : " 27 GB", | 20260505 06:07:16.642 | INFO | "malloc" : " 14 GB", | 20260505 06:07:16.642 | INFO | "pinned" : " 29 GB" | 20260505 06:07:16.642 | INFO | } | 20260505 06:07:16.642 | INFO | Perf report for the last 5.13939 seconds: | 20260505 06:07:16.642 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:07:16.642 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:16.642 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:16.642 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:16.642 | INFO | ANALYSIS : 64.1%, 3.0, 99.7ms, 0 | 20260505 06:07:16.642 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:16.642 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:16.642 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 06:07:16.642 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:16.642 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:16.642 | INFO | Uploaded 36GiB to GPU in 2.15883s (16.6757GiB/s) | 20260505 06:07:16.642 | INFO | Downloaded 34GiB to GPU in 2.04395s (16.6345GiB/s) | 20260505 06:07:21.781 | INFO | Analyzing chunk frames = [931328,931840) | 20260505 06:07:21.781 | INFO | MemInfo: { | 20260505 06:07:21.781 | INFO | "gpu" : " 17 GB", | 20260505 06:07:21.781 | INFO | "huge" : "", | 20260505 06:07:21.781 | INFO | "huge_shared" : " 27 GB", | 20260505 06:07:21.781 | INFO | "malloc" : " 14 GB", | 20260505 06:07:21.781 | INFO | "pinned" : " 29 GB" | 20260505 06:07:21.781 | INFO | } | 20260505 06:07:21.781 | INFO | Perf report for the last 5.13937 seconds: | 20260505 06:07:21.781 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:07:21.781 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:21.781 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:21.781 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:21.781 | INFO | ANALYSIS : 63.4%, 3.0, 98.4ms, 0 | 20260505 06:07:21.781 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:21.781 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:21.781 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 06:07:21.781 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:21.781 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:21.781 | INFO | Uploaded 36GiB to GPU in 2.13643s (16.8506GiB/s) | 20260505 06:07:21.781 | INFO | Downloaded 34GiB to GPU in 2.05008s (16.5847GiB/s) | 20260505 06:07:26.920 | INFO | Analyzing chunk frames = [931840,932352) | 20260505 06:07:26.920 | INFO | MemInfo: { | 20260505 06:07:26.920 | INFO | "gpu" : " 17 GB", | 20260505 06:07:26.920 | INFO | "huge" : "", | 20260505 06:07:26.920 | INFO | "huge_shared" : " 27 GB", | 20260505 06:07:26.920 | INFO | "malloc" : " 14 GB", | 20260505 06:07:26.920 | INFO | "pinned" : " 29 GB" | 20260505 06:07:26.920 | INFO | } | 20260505 06:07:26.920 | INFO | Perf report for the last 5.13924 seconds: | 20260505 06:07:26.920 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:07:26.920 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:26.920 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:26.920 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:26.921 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 06:07:26.921 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:26.921 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:26.921 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260505 06:07:26.921 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:26.921 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:26.921 | INFO | Uploaded 36GiB to GPU in 2.12498s (16.9414GiB/s) | 20260505 06:07:26.921 | INFO | Downloaded 34GiB to GPU in 2.04944s (16.5899GiB/s) | 20260505 06:07:32.061 | INFO | Analyzing chunk frames = [932352,932864) | 20260505 06:07:32.061 | INFO | MemInfo: { | 20260505 06:07:32.061 | INFO | "gpu" : " 17 GB", | 20260505 06:07:32.061 | INFO | "huge" : "", | 20260505 06:07:32.061 | INFO | "huge_shared" : " 27 GB", | 20260505 06:07:32.061 | INFO | "malloc" : " 14 GB", | 20260505 06:07:32.061 | INFO | "pinned" : " 29 GB" | 20260505 06:07:32.061 | INFO | } | 20260505 06:07:32.061 | INFO | Perf report for the last 5.14028 seconds: | 20260505 06:07:32.061 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:07:32.061 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:32.061 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:32.061 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:32.061 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 06:07:32.061 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:32.061 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:32.061 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 06:07:32.061 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:32.061 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:32.061 | INFO | Uploaded 36GiB to GPU in 2.13544s (16.8584GiB/s) | 20260505 06:07:32.061 | INFO | Downloaded 34GiB to GPU in 2.06144s (16.4934GiB/s) | 20260505 06:07:37.199 | INFO | Analyzing chunk frames = [932864,933376) | 20260505 06:07:37.199 | INFO | MemInfo: { | 20260505 06:07:37.199 | INFO | "gpu" : " 17 GB", | 20260505 06:07:37.199 | INFO | "huge" : "", | 20260505 06:07:37.199 | INFO | "huge_shared" : " 27 GB", | 20260505 06:07:37.199 | INFO | "malloc" : " 14 GB", | 20260505 06:07:37.199 | INFO | "pinned" : " 29 GB" | 20260505 06:07:37.199 | INFO | } | 20260505 06:07:37.199 | INFO | Perf report for the last 5.13843 seconds: | 20260505 06:07:37.199 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:07:37.199 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:37.199 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:37.199 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:37.199 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 06:07:37.199 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:37.199 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:37.199 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260505 06:07:37.199 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:37.199 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:37.199 | INFO | Uploaded 36GiB to GPU in 2.16589s (16.6214GiB/s) | 20260505 06:07:37.200 | INFO | Downloaded 34GiB to GPU in 2.14296s (15.8659GiB/s) | 20260505 06:07:42.338 | INFO | Analyzing chunk frames = [933376,933888) | 20260505 06:07:42.338 | INFO | MemInfo: { | 20260505 06:07:42.338 | INFO | "gpu" : " 17 GB", | 20260505 06:07:42.338 | INFO | "huge" : "", | 20260505 06:07:42.338 | INFO | "huge_shared" : " 27 GB", | 20260505 06:07:42.338 | INFO | "malloc" : " 14 GB", | 20260505 06:07:42.338 | INFO | "pinned" : " 29 GB" | 20260505 06:07:42.338 | INFO | } | 20260505 06:07:42.338 | INFO | Perf report for the last 5.13925 seconds: | 20260505 06:07:42.338 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:07:42.338 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:42.338 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:42.338 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:42.338 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 06:07:42.339 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:42.339 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:42.339 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 06:07:42.339 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:42.339 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:42.339 | INFO | Uploaded 36GiB to GPU in 2.0887s (17.2356GiB/s) | 20260505 06:07:42.339 | INFO | Downloaded 34GiB to GPU in 2.07833s (16.3593GiB/s) | 20260505 06:07:42.339 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 06:07:42.339 | INFO | Total mem usage: 15.3 GiB | 20260505 06:07:42.339 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 06:07:42.339 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.44%) | 20260505 06:07:42.339 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 06:07:42.339 | INFO | High Water Breakdown: | 20260505 06:07:42.339 | INFO | 60.2% : 8.67 GiB ( 92.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 06:07:42.339 | INFO | 39.7% : 5.71 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 06:07:42.339 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 06:07:42.339 | INFO | ------------------------------------------ | 20260505 06:07:42.339 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:07:42.339 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 06:07:42.339 | INFO | Total mem usage: 26.5 GiB | 20260505 06:07:42.339 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 06:07:42.339 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 06:07:42.339 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 06:07:42.339 | INFO | High Water Breakdown: | 20260505 06:07:42.339 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:07:42.339 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:07:42.339 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:07:42.339 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:07:42.339 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:07:42.339 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:07:42.339 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 06:07:42.339 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 06:07:42.339 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:07:42.339 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 06:07:42.339 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 06:07:42.339 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 06:07:42.339 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 06:07:42.339 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 06:07:42.339 | INFO | ------------------------------------------ | 20260505 06:07:42.339 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:07:42.339 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 06:07:42.339 | INFO | Total mem usage: 25.6 GiB | 20260505 06:07:42.339 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 06:07:42.339 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 06:07:42.339 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 06:07:42.339 | INFO | High Water Breakdown: | 20260505 06:07:42.339 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 06:07:42.339 | INFO | ------------------------------------------ | 20260505 06:07:42.339 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:07:42.339 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 06:07:42.339 | INFO | Total mem usage: 15.7 GiB | 20260505 06:07:42.339 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 06:07:42.339 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 06:07:42.339 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260505 06:07:42.339 | INFO | High Water Breakdown: | 20260505 06:07:42.339 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 06:07:42.339 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 06:07:42.339 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 06:07:42.339 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 06:07:42.339 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 06:07:42.339 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 06:07:42.339 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 06:07:42.339 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 06:07:42.339 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 06:07:42.339 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:07:42.339 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 06:07:42.339 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:07:42.339 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 06:07:42.339 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 06:07:42.340 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 06:07:42.340 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 06:07:42.340 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:07:42.340 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 06:07:42.340 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:07:42.340 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:07:42.340 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:07:42.340 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 06:07:42.340 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 06:07:42.340 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:07:42.340 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 06:07:42.340 | INFO | ------------------------------------------ | 20260505 06:07:42.340 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:07:42.340 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 06:07:42.340 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:07:42.340 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:07:42.340 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.57e+01, 1.37e+01 / 1.66e+00 | 20260505 06:07:42.340 | INFO | PerfReport: 0.28%, AggregateStats -- 9.42e-02 / 1.03e-01, 1.24e-02 / 6.70e-03 | 20260505 06:07:42.340 | INFO | PerfReport: 11.57%, Baseline -- 3.84e+00 / 4.52e+00, 6.34e-01 / 2.40e-01 | 20260505 06:07:42.340 | INFO | PerfReport: 3.30%, Histogram -- 1.10e+00 / 1.48e+00, 2.17e-01 / 1.27e-01 | 20260505 06:07:42.340 | INFO | PerfReport: 15.46%, DME -- 4.11e+01 / 4.49e+01, 4.35e+00 / 1.14e+00 | 20260505 06:07:42.340 | INFO | PerfReport: 55.93%, FrameLabeling -- 1.86e+01 / 1.92e+01, 2.07e+00 / 1.24e+00 | 20260505 06:07:42.340 | INFO | PerfReport: 7.80%, PulseAccumulating -- 2.59e+00 / 3.17e+00, 4.89e-01 / 3.24e-01 | 20260505 06:07:42.340 | INFO | PerfReport: 4.13%, Metrics -- 1.37e+00 / 1.74e+00, 3.84e-01 / 2.08e-01 | 20260505 06:07:42.340 | INFO | PerfReport: 1.52%, PulseSerializing -- 5.04e-01 / 5.85e-01, 9.99e-02 / 4.35e-02 | 20260505 06:07:42.340 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 06:07:42.340 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:07:42.340 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:07:42.340 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.29e+01 / 0.00e+00 | 20260505 06:07:42.340 | INFO | PerfReport: 22.01%, Upload -- 2.24e+01 / 0.00e+00, 7.70e+00 / 0.00e+00 | 20260505 06:07:42.340 | INFO | PerfReport: 21.54%, Download -- 2.19e+01 / 0.00e+00, 7.74e+00 / 0.00e+00 | 20260505 06:07:42.340 | INFO | PerfReport: 32.70%, Compute -- 3.32e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 06:07:42.340 | INFO | PerfReport: 1.46%, LockUpload -- 1.48e+00 / 0.00e+00, 6.86e+00 / 0.00e+00 | 20260505 06:07:42.340 | INFO | PerfReport: 0.71%, LockDownload -- 7.21e-01 / 0.00e+00, 1.98e+00 / 0.00e+00 | 20260505 06:07:42.340 | INFO | PerfReport: 21.57%, LockCompute -- 2.19e+01 / 0.00e+00, 1.68e+01 / 0.00e+00 | 20260505 06:07:42.340 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 1.83e-02 / 0.00e+00 | 20260505 06:07:42.452 | INFO | Starting a round of Baz IO | 20260505 06:07:45.728 | INFO | Populating RTMetrics report | 20260505 06:07:45.728 | INFO | RTMetrics report populated | 20260505 06:07:45.728 | INFO | Opening tmp RTMetrics json file | 20260505 06:07:45.746 | INFO | Serializing RTMetrics to json | 20260505 06:07:45.785 | INFO | Serialization done, swapping tmp file now... | 20260505 06:07:45.787 | INFO | RTMetrics done | 20260505 06:07:45.867 | INFO | Finished a round of Baz IO: Wrote 2.338 GiB in 3.414 seconds | 20260505 06:07:45.867 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.81% / 0.06% / 65.13% | 20260505 06:07:45.867 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.76599% / 9.13938e-05% / 4.76837e-05% | 20260505 06:07:45.867 | INFO | Baz IO overall: | 20260505 06:07:45.867 | INFO | BytesWritten : 469789310447 | 20260505 06:07:45.867 | INFO | NumEvents : 221733172977 | 20260505 06:07:47.478 | INFO | Analyzing chunk frames = [933888,934400) | 20260505 06:07:47.478 | INFO | MemInfo: { | 20260505 06:07:47.478 | INFO | "gpu" : " 17 GB", | 20260505 06:07:47.478 | INFO | "huge" : "", | 20260505 06:07:47.478 | INFO | "huge_shared" : " 27 GB", | 20260505 06:07:47.478 | INFO | "malloc" : " 14 GB", | 20260505 06:07:47.478 | INFO | "pinned" : " 29 GB" | 20260505 06:07:47.478 | INFO | } | 20260505 06:07:47.478 | INFO | Perf report for the last 5.13929 seconds: | 20260505 06:07:47.478 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:07:47.478 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:47.478 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:47.478 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:47.478 | INFO | ANALYSIS : 64.8%, 2.9, 98.6ms, 0 | 20260505 06:07:47.478 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:47.478 | INFO | RT_METRICS : 47.1%, 1.0, 24.7ms, 0 | 20260505 06:07:47.478 | INFO | PRE_HQ : 63.8%, 2.0, 64.7ms, 0 | 20260505 06:07:47.478 | INFO | BAZWRITER : 65.7%, 4.1, 137.4ms, 0 | 20260505 06:07:47.478 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 06:07:47.478 | INFO | Uploaded 36GiB to GPU in 2.18517s (16.4747GiB/s) | 20260505 06:07:47.478 | INFO | Downloaded 38GiB to GPU in 2.33196s (16.2953GiB/s) | 20260505 06:07:52.617 | INFO | Analyzing chunk frames = [934400,934912) | 20260505 06:07:52.617 | INFO | MemInfo: { | 20260505 06:07:52.617 | INFO | "gpu" : " 17 GB", | 20260505 06:07:52.617 | INFO | "huge" : "", | 20260505 06:07:52.617 | INFO | "huge_shared" : " 27 GB", | 20260505 06:07:52.617 | INFO | "malloc" : " 14 GB", | 20260505 06:07:52.617 | INFO | "pinned" : " 29 GB" | 20260505 06:07:52.618 | INFO | } | 20260505 06:07:52.618 | INFO | Perf report for the last 5.13991 seconds: | 20260505 06:07:52.618 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:07:52.618 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:52.618 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:52.618 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:52.618 | INFO | ANALYSIS : 64.4%, 3.0, 100.6ms, 0 | 20260505 06:07:52.618 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:52.618 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:52.618 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 06:07:52.618 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:52.618 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:52.618 | INFO | Uploaded 36GiB to GPU in 2.13402s (16.8696GiB/s) | 20260505 06:07:52.618 | INFO | Downloaded 34GiB to GPU in 2.03572s (16.7017GiB/s) | 20260505 06:07:57.756 | INFO | Analyzing chunk frames = [934912,935424) | 20260505 06:07:57.756 | INFO | MemInfo: { | 20260505 06:07:57.756 | INFO | "gpu" : " 17 GB", | 20260505 06:07:57.756 | INFO | "huge" : "", | 20260505 06:07:57.756 | INFO | "huge_shared" : " 27 GB", | 20260505 06:07:57.756 | INFO | "malloc" : " 14 GB", | 20260505 06:07:57.756 | INFO | "pinned" : " 29 GB" | 20260505 06:07:57.756 | INFO | } | 20260505 06:07:57.756 | INFO | Perf report for the last 5.13873 seconds: | 20260505 06:07:57.756 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:07:57.756 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:57.756 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:57.756 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:57.756 | INFO | ANALYSIS : 64.0%, 3.0, 99.6ms, 0 | 20260505 06:07:57.756 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:57.756 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:57.756 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 06:07:57.756 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:57.757 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:07:57.757 | INFO | Uploaded 36GiB to GPU in 2.15651s (16.6937GiB/s) | 20260505 06:07:57.757 | INFO | Downloaded 34GiB to GPU in 2.05155s (16.5728GiB/s) | 20260505 06:08:02.896 | INFO | Analyzing chunk frames = [935424,935936) | 20260505 06:08:02.896 | INFO | MemInfo: { | 20260505 06:08:02.896 | INFO | "gpu" : " 17 GB", | 20260505 06:08:02.896 | INFO | "huge" : "", | 20260505 06:08:02.896 | INFO | "huge_shared" : " 27 GB", | 20260505 06:08:02.896 | INFO | "malloc" : " 14 GB", | 20260505 06:08:02.896 | INFO | "pinned" : " 29 GB" | 20260505 06:08:02.896 | INFO | } | 20260505 06:08:02.897 | INFO | Perf report for the last 5.14044 seconds: | 20260505 06:08:02.897 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:08:02.897 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:02.897 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:02.897 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:02.897 | INFO | ANALYSIS : 63.4%, 3.0, 98.6ms, 0 | 20260505 06:08:02.897 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:02.897 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:02.897 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260505 06:08:02.897 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:02.897 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:02.897 | INFO | Uploaded 36GiB to GPU in 2.14398s (16.7912GiB/s) | 20260505 06:08:02.897 | INFO | Downloaded 34GiB to GPU in 2.05367s (16.5558GiB/s) | 20260505 06:08:08.035 | INFO | Analyzing chunk frames = [935936,936448) | 20260505 06:08:08.035 | INFO | MemInfo: { | 20260505 06:08:08.035 | INFO | "gpu" : " 17 GB", | 20260505 06:08:08.035 | INFO | "huge" : "", | 20260505 06:08:08.035 | INFO | "huge_shared" : " 27 GB", | 20260505 06:08:08.035 | INFO | "malloc" : " 14 GB", | 20260505 06:08:08.035 | INFO | "pinned" : " 29 GB" | 20260505 06:08:08.035 | INFO | } | 20260505 06:08:08.035 | INFO | Perf report for the last 5.1382 seconds: | 20260505 06:08:08.035 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:08:08.035 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:08.035 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:08.035 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:08.035 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 06:08:08.035 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:08.035 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:08.035 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 06:08:08.035 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:08.035 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:08.035 | INFO | Uploaded 36GiB to GPU in 2.11289s (17.0382GiB/s) | 20260505 06:08:08.035 | INFO | Downloaded 34GiB to GPU in 2.0475s (16.6056GiB/s) | 20260505 06:08:13.175 | INFO | Analyzing chunk frames = [936448,936960) | 20260505 06:08:13.175 | INFO | MemInfo: { | 20260505 06:08:13.175 | INFO | "gpu" : " 17 GB", | 20260505 06:08:13.175 | INFO | "huge" : "", | 20260505 06:08:13.175 | INFO | "huge_shared" : " 27 GB", | 20260505 06:08:13.175 | INFO | "malloc" : " 14 GB", | 20260505 06:08:13.175 | INFO | "pinned" : " 29 GB" | 20260505 06:08:13.175 | INFO | } | 20260505 06:08:13.176 | INFO | Perf report for the last 5.14042 seconds: | 20260505 06:08:13.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:08:13.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:13.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:13.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:13.176 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 06:08:13.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:13.176 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:13.176 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 06:08:13.176 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:13.176 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:13.176 | INFO | Uploaded 36GiB to GPU in 2.12303s (16.9569GiB/s) | 20260505 06:08:13.176 | INFO | Downloaded 34GiB to GPU in 2.05417s (16.5517GiB/s) | 20260505 06:08:18.314 | INFO | Analyzing chunk frames = [936960,937472) | 20260505 06:08:18.314 | INFO | MemInfo: { | 20260505 06:08:18.314 | INFO | "gpu" : " 17 GB", | 20260505 06:08:18.314 | INFO | "huge" : "", | 20260505 06:08:18.314 | INFO | "huge_shared" : " 27 GB", | 20260505 06:08:18.314 | INFO | "malloc" : " 14 GB", | 20260505 06:08:18.314 | INFO | "pinned" : " 29 GB" | 20260505 06:08:18.314 | INFO | } | 20260505 06:08:18.314 | INFO | Perf report for the last 5.1383 seconds: | 20260505 06:08:18.314 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:08:18.314 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:18.314 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:18.314 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:18.314 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 06:08:18.314 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:18.314 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:18.314 | INFO | PRE_HQ : 33.2%, 1.0, 17.3ms, 0 | 20260505 06:08:18.314 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:18.314 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:18.314 | INFO | Uploaded 36GiB to GPU in 2.1752s (16.5502GiB/s) | 20260505 06:08:18.314 | INFO | Downloaded 34GiB to GPU in 2.14262s (15.8684GiB/s) | 20260505 06:08:23.453 | INFO | Analyzing chunk frames = [937472,937984) | 20260505 06:08:23.453 | INFO | MemInfo: { | 20260505 06:08:23.453 | INFO | "gpu" : " 17 GB", | 20260505 06:08:23.453 | INFO | "huge" : "", | 20260505 06:08:23.453 | INFO | "huge_shared" : " 27 GB", | 20260505 06:08:23.453 | INFO | "malloc" : " 14 GB", | 20260505 06:08:23.453 | INFO | "pinned" : " 29 GB" | 20260505 06:08:23.453 | INFO | } | 20260505 06:08:23.453 | INFO | Perf report for the last 5.13952 seconds: | 20260505 06:08:23.453 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:08:23.453 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:23.453 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:23.453 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:23.453 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 06:08:23.453 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:23.453 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:23.454 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260505 06:08:23.454 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:23.454 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:23.454 | INFO | Uploaded 36GiB to GPU in 2.10972s (17.0638GiB/s) | 20260505 06:08:23.454 | INFO | Downloaded 34GiB to GPU in 2.07617s (16.3763GiB/s) | 20260505 06:08:23.565 | INFO | Starting a round of Baz IO | 20260505 06:08:26.843 | INFO | Populating RTMetrics report | 20260505 06:08:26.843 | INFO | RTMetrics report populated | 20260505 06:08:26.843 | INFO | Opening tmp RTMetrics json file | 20260505 06:08:26.864 | INFO | Serializing RTMetrics to json | 20260505 06:08:26.873 | INFO | Serialization done, swapping tmp file now... | 20260505 06:08:26.889 | INFO | RTMetrics done | 20260505 06:08:26.955 | INFO | Finished a round of Baz IO: Wrote 2.339 GiB in 3.389 seconds | 20260505 06:08:26.955 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.75% / 0.06% / 65.19% | 20260505 06:08:26.955 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.73419% / 0.000107288% / 5.5631e-05% | 20260505 06:08:26.955 | INFO | Baz IO overall: | 20260505 06:08:26.955 | INFO | BytesWritten : 472300672083 | 20260505 06:08:26.955 | INFO | NumEvents : 222933472773 | 20260505 06:08:28.592 | INFO | Analyzing chunk frames = [937984,938496) | 20260505 06:08:28.592 | INFO | MemInfo: { | 20260505 06:08:28.592 | INFO | "gpu" : " 17 GB", | 20260505 06:08:28.592 | INFO | "huge" : "", | 20260505 06:08:28.592 | INFO | "huge_shared" : " 27 GB", | 20260505 06:08:28.592 | INFO | "malloc" : " 14 GB", | 20260505 06:08:28.592 | INFO | "pinned" : " 29 GB" | 20260505 06:08:28.592 | INFO | } | 20260505 06:08:28.592 | INFO | Perf report for the last 5.13888 seconds: | 20260505 06:08:28.592 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:08:28.592 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:28.592 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:28.592 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:28.592 | INFO | ANALYSIS : 64.8%, 2.9, 99.5ms, 0 | 20260505 06:08:28.592 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:28.592 | INFO | RT_METRICS : 46.8%, 1.0, 24.5ms, 0 | 20260505 06:08:28.592 | INFO | PRE_HQ : 63.9%, 1.9, 64.5ms, 0 | 20260505 06:08:28.592 | INFO | BAZWRITER : 65.2%, 4.0, 135.4ms, 0 | 20260505 06:08:28.592 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 06:08:28.593 | INFO | Uploaded 36GiB to GPU in 2.20535s (16.3239GiB/s) | 20260505 06:08:28.593 | INFO | Downloaded 38GiB to GPU in 2.32482s (16.3454GiB/s) | 20260505 06:08:33.732 | INFO | Analyzing chunk frames = [938496,939008) | 20260505 06:08:33.732 | INFO | MemInfo: { | 20260505 06:08:33.732 | INFO | "gpu" : " 17 GB", | 20260505 06:08:33.732 | INFO | "huge" : "", | 20260505 06:08:33.732 | INFO | "huge_shared" : " 27 GB", | 20260505 06:08:33.732 | INFO | "malloc" : " 14 GB", | 20260505 06:08:33.732 | INFO | "pinned" : " 29 GB" | 20260505 06:08:33.732 | INFO | } | 20260505 06:08:33.732 | INFO | Perf report for the last 5.13949 seconds: | 20260505 06:08:33.732 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:08:33.732 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:33.732 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:33.732 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:33.732 | INFO | ANALYSIS : 64.3%, 3.0, 100.3ms, 0 | 20260505 06:08:33.732 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:33.732 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:33.732 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 06:08:33.732 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:33.732 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:33.732 | INFO | Uploaded 36GiB to GPU in 2.12545s (16.9376GiB/s) | 20260505 06:08:33.732 | INFO | Downloaded 34GiB to GPU in 2.04808s (16.6009GiB/s) | 20260505 06:08:38.871 | INFO | Analyzing chunk frames = [939008,939520) | 20260505 06:08:38.871 | INFO | MemInfo: { | 20260505 06:08:38.871 | INFO | "gpu" : " 17 GB", | 20260505 06:08:38.871 | INFO | "huge" : "", | 20260505 06:08:38.871 | INFO | "huge_shared" : " 27 GB", | 20260505 06:08:38.871 | INFO | "malloc" : " 14 GB", | 20260505 06:08:38.871 | INFO | "pinned" : " 29 GB" | 20260505 06:08:38.871 | INFO | } | 20260505 06:08:38.871 | INFO | Perf report for the last 5.13955 seconds: | 20260505 06:08:38.871 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:08:38.871 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:38.871 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:38.871 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:38.871 | INFO | ANALYSIS : 64.1%, 3.0, 99.8ms, 0 | 20260505 06:08:38.871 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:38.871 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:38.871 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 06:08:38.871 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:38.871 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:38.872 | INFO | Uploaded 36GiB to GPU in 2.16409s (16.6351GiB/s) | 20260505 06:08:38.872 | INFO | Downloaded 34GiB to GPU in 2.03s (16.7488GiB/s) | 20260505 06:08:44.010 | INFO | Analyzing chunk frames = [939520,940032) | 20260505 06:08:44.010 | INFO | MemInfo: { | 20260505 06:08:44.010 | INFO | "gpu" : " 17 GB", | 20260505 06:08:44.010 | INFO | "huge" : "", | 20260505 06:08:44.010 | INFO | "huge_shared" : " 27 GB", | 20260505 06:08:44.010 | INFO | "malloc" : " 14 GB", | 20260505 06:08:44.011 | INFO | "pinned" : " 29 GB" | 20260505 06:08:44.011 | INFO | } | 20260505 06:08:44.011 | INFO | Perf report for the last 5.1393 seconds: | 20260505 06:08:44.011 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:08:44.011 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:44.011 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:44.011 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:44.011 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 06:08:44.011 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:44.011 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:44.011 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 06:08:44.011 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:44.011 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:44.011 | INFO | Uploaded 36GiB to GPU in 2.1334s (16.8745GiB/s) | 20260505 06:08:44.011 | INFO | Downloaded 34GiB to GPU in 2.05558s (16.5404GiB/s) | 20260505 06:08:49.150 | INFO | Analyzing chunk frames = [940032,940544) | 20260505 06:08:49.150 | INFO | MemInfo: { | 20260505 06:08:49.150 | INFO | "gpu" : " 17 GB", | 20260505 06:08:49.150 | INFO | "huge" : "", | 20260505 06:08:49.150 | INFO | "huge_shared" : " 27 GB", | 20260505 06:08:49.150 | INFO | "malloc" : " 14 GB", | 20260505 06:08:49.150 | INFO | "pinned" : " 29 GB" | 20260505 06:08:49.150 | INFO | } | 20260505 06:08:49.150 | INFO | Perf report for the last 5.13928 seconds: | 20260505 06:08:49.150 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:08:49.150 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:49.150 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:49.150 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:49.150 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 06:08:49.150 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:49.150 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:49.150 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 06:08:49.150 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:49.150 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:49.150 | INFO | Uploaded 36GiB to GPU in 2.1186s (16.9923GiB/s) | 20260505 06:08:49.150 | INFO | Downloaded 34GiB to GPU in 2.04655s (16.6134GiB/s) | 20260505 06:08:54.290 | INFO | Analyzing chunk frames = [940544,941056) | 20260505 06:08:54.290 | INFO | MemInfo: { | 20260505 06:08:54.290 | INFO | "gpu" : " 17 GB", | 20260505 06:08:54.290 | INFO | "huge" : "", | 20260505 06:08:54.290 | INFO | "huge_shared" : " 27 GB", | 20260505 06:08:54.290 | INFO | "malloc" : " 14 GB", | 20260505 06:08:54.290 | INFO | "pinned" : " 29 GB" | 20260505 06:08:54.290 | INFO | } | 20260505 06:08:54.290 | INFO | Perf report for the last 5.14006 seconds: | 20260505 06:08:54.290 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:08:54.290 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:54.290 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:54.290 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:54.290 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 06:08:54.290 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:54.290 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:54.290 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260505 06:08:54.290 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:54.290 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:54.290 | INFO | Uploaded 36GiB to GPU in 2.10566s (17.0968GiB/s) | 20260505 06:08:54.290 | INFO | Downloaded 34GiB to GPU in 2.06002s (16.5047GiB/s) | 20260505 06:08:59.429 | INFO | Analyzing chunk frames = [941056,941568) | 20260505 06:08:59.429 | INFO | MemInfo: { | 20260505 06:08:59.429 | INFO | "gpu" : " 17 GB", | 20260505 06:08:59.429 | INFO | "huge" : "", | 20260505 06:08:59.429 | INFO | "huge_shared" : " 27 GB", | 20260505 06:08:59.429 | INFO | "malloc" : " 14 GB", | 20260505 06:08:59.429 | INFO | "pinned" : " 29 GB" | 20260505 06:08:59.429 | INFO | } | 20260505 06:08:59.429 | INFO | Perf report for the last 5.13891 seconds: | 20260505 06:08:59.429 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:08:59.429 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:59.429 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:59.429 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:59.429 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 06:08:59.429 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:59.429 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:59.429 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 06:08:59.429 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:59.429 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:08:59.429 | INFO | Uploaded 36GiB to GPU in 2.14429s (16.7888GiB/s) | 20260505 06:08:59.429 | INFO | Downloaded 34GiB to GPU in 2.14829s (15.8265GiB/s) | 20260505 06:09:04.568 | INFO | Analyzing chunk frames = [941568,942080) | 20260505 06:09:04.568 | INFO | MemInfo: { | 20260505 06:09:04.568 | INFO | "gpu" : " 17 GB", | 20260505 06:09:04.568 | INFO | "huge" : "", | 20260505 06:09:04.568 | INFO | "huge_shared" : " 27 GB", | 20260505 06:09:04.568 | INFO | "malloc" : " 14 GB", | 20260505 06:09:04.568 | INFO | "pinned" : " 29 GB" | 20260505 06:09:04.568 | INFO | } | 20260505 06:09:04.568 | INFO | Perf report for the last 5.13898 seconds: | 20260505 06:09:04.568 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:09:04.568 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:04.568 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:04.568 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:04.568 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 06:09:04.568 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:04.568 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:04.568 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 06:09:04.568 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:04.568 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:04.568 | INFO | Uploaded 36GiB to GPU in 2.09476s (17.1858GiB/s) | 20260505 06:09:04.568 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 06:09:04.568 | INFO | Downloaded 34GiB to GPU in 2.07481s (16.3871GiB/s) | 20260505 06:09:04.568 | INFO | Total mem usage: 15.3 GiB | 20260505 06:09:04.568 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 06:09:04.568 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.1%) | 20260505 06:09:04.568 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 06:09:04.568 | INFO | High Water Breakdown: | 20260505 06:09:04.568 | INFO | 60.1% : 8.69 GiB ( 92.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 06:09:04.568 | INFO | 39.7% : 5.74 GiB ( 86.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 06:09:04.568 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 06:09:04.568 | INFO | ------------------------------------------ | 20260505 06:09:04.568 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:09:04.568 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 06:09:04.568 | INFO | Total mem usage: 26.5 GiB | 20260505 06:09:04.568 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 06:09:04.568 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 06:09:04.568 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 06:09:04.568 | INFO | High Water Breakdown: | 20260505 06:09:04.568 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:09:04.568 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:09:04.568 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:09:04.568 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:09:04.568 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:09:04.568 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:09:04.568 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 06:09:04.568 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 06:09:04.568 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:09:04.568 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 06:09:04.568 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 06:09:04.568 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 06:09:04.568 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 06:09:04.569 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 06:09:04.569 | INFO | ------------------------------------------ | 20260505 06:09:04.569 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:09:04.569 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 06:09:04.569 | INFO | Total mem usage: 25.6 GiB | 20260505 06:09:04.569 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 06:09:04.569 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 06:09:04.569 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 06:09:04.569 | INFO | High Water Breakdown: | 20260505 06:09:04.569 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 06:09:04.569 | INFO | ------------------------------------------ | 20260505 06:09:04.569 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:09:04.569 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 06:09:04.569 | INFO | Total mem usage: 15.7 GiB | 20260505 06:09:04.569 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 06:09:04.569 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 06:09:04.569 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260505 06:09:04.569 | INFO | High Water Breakdown: | 20260505 06:09:04.569 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 06:09:04.569 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 06:09:04.569 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 06:09:04.569 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 06:09:04.569 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 06:09:04.569 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 06:09:04.569 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 06:09:04.569 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 06:09:04.569 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 06:09:04.569 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:09:04.569 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 06:09:04.569 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:09:04.569 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 06:09:04.569 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 06:09:04.569 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 06:09:04.569 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 06:09:04.569 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:09:04.569 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 06:09:04.569 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:09:04.569 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:09:04.569 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:09:04.569 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 06:09:04.569 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 06:09:04.569 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:09:04.569 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 06:09:04.569 | INFO | ------------------------------------------ | 20260505 06:09:04.569 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:09:04.569 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 06:09:04.569 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:09:04.569 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:09:04.569 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.59e+01, 1.39e+01 / 1.57e+00 | 20260505 06:09:04.569 | INFO | PerfReport: 0.28%, AggregateStats -- 9.41e-02 / 1.03e-01, 1.05e-02 / 7.92e-03 | 20260505 06:09:04.569 | INFO | PerfReport: 11.57%, Baseline -- 3.84e+00 / 4.52e+00, 6.35e-01 / 2.93e-01 | 20260505 06:09:04.569 | INFO | PerfReport: 3.30%, Histogram -- 1.10e+00 / 1.51e+00, 2.18e-01 / 1.24e-01 | 20260505 06:09:04.569 | INFO | PerfReport: 15.48%, DME -- 4.11e+01 / 4.52e+01, 4.39e+00 / 1.18e+00 | 20260505 06:09:04.569 | INFO | PerfReport: 55.90%, FrameLabeling -- 1.86e+01 / 1.91e+01, 2.06e+00 / 1.01e+00 | 20260505 06:09:04.569 | INFO | PerfReport: 7.79%, PulseAccumulating -- 2.59e+00 / 3.16e+00, 4.91e-01 / 3.38e-01 | 20260505 06:09:04.569 | INFO | PerfReport: 4.15%, Metrics -- 1.38e+00 / 1.76e+00, 3.85e-01 / 2.59e-01 | 20260505 06:09:04.569 | INFO | PerfReport: 1.52%, PulseSerializing -- 5.04e-01 / 5.85e-01, 9.93e-02 / 4.12e-02 | 20260505 06:09:04.569 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 06:09:04.569 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:09:04.569 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:09:04.569 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.28e+01 / 0.00e+00 | 20260505 06:09:04.569 | INFO | PerfReport: 21.97%, Upload -- 2.23e+01 / 0.00e+00, 7.61e+00 / 0.00e+00 | 20260505 06:09:04.569 | INFO | PerfReport: 21.55%, Download -- 2.19e+01 / 0.00e+00, 7.69e+00 / 0.00e+00 | 20260505 06:09:04.569 | INFO | PerfReport: 32.75%, Compute -- 3.32e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 06:09:04.569 | INFO | PerfReport: 1.41%, LockUpload -- 1.43e+00 / 0.00e+00, 6.77e+00 / 0.00e+00 | 20260505 06:09:04.569 | INFO | PerfReport: 0.67%, LockDownload -- 6.75e-01 / 0.00e+00, 1.84e+00 / 0.00e+00 | 20260505 06:09:04.569 | INFO | PerfReport: 21.64%, LockCompute -- 2.20e+01 / 0.00e+00, 1.67e+01 / 0.00e+00 | 20260505 06:09:04.569 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 1.69e-02 / 0.00e+00 | 20260505 06:09:04.680 | INFO | Starting a round of Baz IO | 20260505 06:09:07.952 | INFO | Populating RTMetrics report | 20260505 06:09:07.952 | INFO | RTMetrics report populated | 20260505 06:09:07.952 | INFO | Opening tmp RTMetrics json file | 20260505 06:09:07.973 | INFO | Serializing RTMetrics to json | 20260505 06:09:07.974 | INFO | Serialization done, swapping tmp file now... | 20260505 06:09:07.975 | INFO | RTMetrics done | 20260505 06:09:08.074 | INFO | Finished a round of Baz IO: Wrote 2.34 GiB in 3.393 seconds | 20260505 06:09:08.074 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.70% / 0.06% / 65.25% | 20260505 06:09:08.074 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.71063% / 7.54992e-05% / 3.97364e-05% | 20260505 06:09:08.074 | INFO | Baz IO overall: | 20260505 06:09:08.074 | INFO | BytesWritten : 474812806788 | 20260505 06:09:08.074 | INFO | NumEvents : 224134191057 | 20260505 06:09:09.708 | INFO | Analyzing chunk frames = [942080,942592) | 20260505 06:09:09.708 | INFO | MemInfo: { | 20260505 06:09:09.708 | INFO | "gpu" : " 17 GB", | 20260505 06:09:09.708 | INFO | "huge" : "", | 20260505 06:09:09.708 | INFO | "huge_shared" : " 27 GB", | 20260505 06:09:09.708 | INFO | "malloc" : " 14 GB", | 20260505 06:09:09.708 | INFO | "pinned" : " 29 GB" | 20260505 06:09:09.708 | INFO | } | 20260505 06:09:09.708 | INFO | Perf report for the last 5.13988 seconds: | 20260505 06:09:09.708 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:09:09.708 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:09.708 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:09.708 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:09.708 | INFO | ANALYSIS : 64.7%, 2.9, 99.2ms, 0 | 20260505 06:09:09.708 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:09.708 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260505 06:09:09.708 | INFO | PRE_HQ : 63.8%, 1.9, 64.3ms, 0 | 20260505 06:09:09.708 | INFO | BAZWRITER : 65.3%, 4.0, 135.4ms, 0 | 20260505 06:09:09.708 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 06:09:09.708 | INFO | Uploaded 36GiB to GPU in 2.15665s (16.6925GiB/s) | 20260505 06:09:09.708 | INFO | Downloaded 38GiB to GPU in 2.32887s (16.3169GiB/s) | 20260505 06:09:14.847 | INFO | Analyzing chunk frames = [942592,943104) | 20260505 06:09:14.847 | INFO | MemInfo: { | 20260505 06:09:14.847 | INFO | "gpu" : " 17 GB", | 20260505 06:09:14.847 | INFO | "huge" : "", | 20260505 06:09:14.847 | INFO | "huge_shared" : " 27 GB", | 20260505 06:09:14.847 | INFO | "malloc" : " 14 GB", | 20260505 06:09:14.847 | INFO | "pinned" : " 29 GB" | 20260505 06:09:14.847 | INFO | } | 20260505 06:09:14.847 | INFO | Perf report for the last 5.13925 seconds: | 20260505 06:09:14.847 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:09:14.847 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:14.847 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:14.847 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:14.847 | INFO | ANALYSIS : 64.3%, 3.0, 100.5ms, 0 | 20260505 06:09:14.847 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:14.847 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:14.847 | INFO | PRE_HQ : 30.5%, 1.0, 15.9ms, 0 | 20260505 06:09:14.847 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:14.847 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:14.847 | INFO | Uploaded 36GiB to GPU in 2.1381s (16.8374GiB/s) | 20260505 06:09:14.847 | INFO | Downloaded 34GiB to GPU in 2.04577s (16.6197GiB/s) | 20260505 06:09:19.986 | INFO | Analyzing chunk frames = [943104,943616) | 20260505 06:09:19.986 | INFO | MemInfo: { | 20260505 06:09:19.986 | INFO | "gpu" : " 17 GB", | 20260505 06:09:19.986 | INFO | "huge" : "", | 20260505 06:09:19.986 | INFO | "huge_shared" : " 27 GB", | 20260505 06:09:19.986 | INFO | "malloc" : " 14 GB", | 20260505 06:09:19.986 | INFO | "pinned" : " 29 GB" | 20260505 06:09:19.986 | INFO | } | 20260505 06:09:19.986 | INFO | Perf report for the last 5.13904 seconds: | 20260505 06:09:19.986 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:09:19.986 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:19.986 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:19.986 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:19.986 | INFO | ANALYSIS : 64.2%, 3.0, 99.9ms, 0 | 20260505 06:09:19.986 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:19.986 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:19.986 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 06:09:19.986 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:19.986 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:19.986 | INFO | Uploaded 36GiB to GPU in 2.16554s (16.624GiB/s) | 20260505 06:09:19.986 | INFO | Downloaded 34GiB to GPU in 2.03695s (16.6916GiB/s) | 20260505 06:09:25.125 | INFO | Analyzing chunk frames = [943616,944128) | 20260505 06:09:25.125 | INFO | MemInfo: { | 20260505 06:09:25.125 | INFO | "gpu" : " 17 GB", | 20260505 06:09:25.125 | INFO | "huge" : "", | 20260505 06:09:25.125 | INFO | "huge_shared" : " 27 GB", | 20260505 06:09:25.125 | INFO | "malloc" : " 14 GB", | 20260505 06:09:25.125 | INFO | "pinned" : " 29 GB" | 20260505 06:09:25.125 | INFO | } | 20260505 06:09:25.125 | INFO | Perf report for the last 5.13913 seconds: | 20260505 06:09:25.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:09:25.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:25.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:25.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:25.125 | INFO | ANALYSIS : 63.3%, 3.0, 98.3ms, 0 | 20260505 06:09:25.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:25.125 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:25.125 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260505 06:09:25.125 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:25.125 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:25.126 | INFO | Uploaded 36GiB to GPU in 2.11867s (16.9918GiB/s) | 20260505 06:09:25.126 | INFO | Downloaded 34GiB to GPU in 2.05109s (16.5765GiB/s) | 20260505 06:09:30.264 | INFO | Analyzing chunk frames = [944128,944640) | 20260505 06:09:30.264 | INFO | MemInfo: { | 20260505 06:09:30.264 | INFO | "gpu" : " 17 GB", | 20260505 06:09:30.264 | INFO | "huge" : "", | 20260505 06:09:30.264 | INFO | "huge_shared" : " 27 GB", | 20260505 06:09:30.264 | INFO | "malloc" : " 14 GB", | 20260505 06:09:30.264 | INFO | "pinned" : " 29 GB" | 20260505 06:09:30.265 | INFO | } | 20260505 06:09:30.265 | INFO | Perf report for the last 5.13928 seconds: | 20260505 06:09:30.265 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:09:30.265 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:30.265 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:30.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:30.265 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 06:09:30.265 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:30.265 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:30.265 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 06:09:30.265 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:30.265 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:30.265 | INFO | Uploaded 36GiB to GPU in 2.11406s (17.0289GiB/s) | 20260505 06:09:30.265 | INFO | Downloaded 34GiB to GPU in 2.05122s (16.5755GiB/s) | 20260505 06:09:35.405 | INFO | Analyzing chunk frames = [944640,945152) | 20260505 06:09:35.405 | INFO | MemInfo: { | 20260505 06:09:35.405 | INFO | "gpu" : " 17 GB", | 20260505 06:09:35.405 | INFO | "huge" : "", | 20260505 06:09:35.405 | INFO | "huge_shared" : " 27 GB", | 20260505 06:09:35.405 | INFO | "malloc" : " 14 GB", | 20260505 06:09:35.405 | INFO | "pinned" : " 29 GB" | 20260505 06:09:35.405 | INFO | } | 20260505 06:09:35.405 | INFO | Perf report for the last 5.14031 seconds: | 20260505 06:09:35.405 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:09:35.405 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:35.405 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:35.405 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:35.405 | INFO | ANALYSIS : 64.1%, 3.0, 100.2ms, 0 | 20260505 06:09:35.405 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:35.405 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:35.405 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 06:09:35.405 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:35.405 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:35.405 | INFO | Uploaded 36GiB to GPU in 2.15736s (16.687GiB/s) | 20260505 06:09:35.405 | INFO | Downloaded 34GiB to GPU in 2.05126s (16.5752GiB/s) | 20260505 06:09:40.544 | INFO | Analyzing chunk frames = [945152,945664) | 20260505 06:09:40.544 | INFO | MemInfo: { | 20260505 06:09:40.544 | INFO | "gpu" : " 17 GB", | 20260505 06:09:40.544 | INFO | "huge" : "", | 20260505 06:09:40.544 | INFO | "huge_shared" : " 27 GB", | 20260505 06:09:40.544 | INFO | "malloc" : " 14 GB", | 20260505 06:09:40.544 | INFO | "pinned" : " 29 GB" | 20260505 06:09:40.544 | INFO | } | 20260505 06:09:40.544 | INFO | Perf report for the last 5.13911 seconds: | 20260505 06:09:40.544 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:09:40.544 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:40.544 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:40.544 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:40.544 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 06:09:40.544 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:40.544 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:40.544 | INFO | PRE_HQ : 32.4%, 1.0, 16.9ms, 0 | 20260505 06:09:40.544 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:40.544 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:40.544 | INFO | Uploaded 36GiB to GPU in 2.13809s (16.8375GiB/s) | 20260505 06:09:40.544 | INFO | Downloaded 34GiB to GPU in 2.14519s (15.8494GiB/s) | 20260505 06:09:45.683 | INFO | Analyzing chunk frames = [945664,946176) | 20260505 06:09:45.683 | INFO | MemInfo: { | 20260505 06:09:45.683 | INFO | "gpu" : " 17 GB", | 20260505 06:09:45.683 | INFO | "huge" : "", | 20260505 06:09:45.683 | INFO | "huge_shared" : " 27 GB", | 20260505 06:09:45.683 | INFO | "malloc" : " 14 GB", | 20260505 06:09:45.683 | INFO | "pinned" : " 29 GB" | 20260505 06:09:45.683 | INFO | } | 20260505 06:09:45.683 | INFO | Perf report for the last 5.13882 seconds: | 20260505 06:09:45.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:09:45.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:45.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:45.683 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:45.683 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 06:09:45.683 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:45.683 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:45.683 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 06:09:45.683 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:45.683 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:45.683 | INFO | Uploaded 36GiB to GPU in 2.0856s (17.2612GiB/s) | 20260505 06:09:45.683 | INFO | Downloaded 34GiB to GPU in 2.07396s (16.3938GiB/s) | 20260505 06:09:45.795 | INFO | Starting a round of Baz IO | 20260505 06:09:49.065 | INFO | Populating RTMetrics report | 20260505 06:09:49.066 | INFO | RTMetrics report populated | 20260505 06:09:49.066 | INFO | Opening tmp RTMetrics json file | 20260505 06:09:49.087 | INFO | Serializing RTMetrics to json | 20260505 06:09:49.088 | INFO | Serialization done, swapping tmp file now... | 20260505 06:09:49.089 | INFO | RTMetrics done | 20260505 06:09:49.178 | INFO | Finished a round of Baz IO: Wrote 2.341 GiB in 3.383 seconds | 20260505 06:09:49.178 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.64% / 0.06% / 65.30% | 20260505 06:09:49.179 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.69128% / 8.74201e-05% / 5.5631e-05% | 20260505 06:09:49.179 | INFO | Baz IO overall: | 20260505 06:09:49.179 | INFO | BytesWritten : 477326690852 | 20260505 06:09:49.179 | INFO | NumEvents : 225335615084 | 20260505 06:09:50.822 | INFO | Analyzing chunk frames = [946176,946688) | 20260505 06:09:50.822 | INFO | MemInfo: { | 20260505 06:09:50.822 | INFO | "gpu" : " 17 GB", | 20260505 06:09:50.822 | INFO | "huge" : "", | 20260505 06:09:50.822 | INFO | "huge_shared" : " 27 GB", | 20260505 06:09:50.822 | INFO | "malloc" : " 14 GB", | 20260505 06:09:50.822 | INFO | "pinned" : " 29 GB" | 20260505 06:09:50.822 | INFO | } | 20260505 06:09:50.822 | INFO | Perf report for the last 5.13913 seconds: | 20260505 06:09:50.822 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:09:50.822 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:50.822 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:50.822 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:50.822 | INFO | ANALYSIS : 64.7%, 2.9, 99.0ms, 0 | 20260505 06:09:50.822 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:50.822 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260505 06:09:50.822 | INFO | PRE_HQ : 63.7%, 1.9, 64.1ms, 0 | 20260505 06:09:50.822 | INFO | BAZWRITER : 65.1%, 4.0, 135.1ms, 0 | 20260505 06:09:50.822 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 06:09:50.822 | INFO | Uploaded 36GiB to GPU in 2.18174s (16.5006GiB/s) | 20260505 06:09:50.822 | INFO | Downloaded 38GiB to GPU in 2.32843s (16.32GiB/s) | 20260505 06:09:55.961 | INFO | Analyzing chunk frames = [946688,947200) | 20260505 06:09:55.961 | INFO | MemInfo: { | 20260505 06:09:55.961 | INFO | "gpu" : " 17 GB", | 20260505 06:09:55.961 | INFO | "huge" : "", | 20260505 06:09:55.961 | INFO | "huge_shared" : " 27 GB", | 20260505 06:09:55.961 | INFO | "malloc" : " 14 GB", | 20260505 06:09:55.961 | INFO | "pinned" : " 29 GB" | 20260505 06:09:55.961 | INFO | } | 20260505 06:09:55.961 | INFO | Perf report for the last 5.13952 seconds: | 20260505 06:09:55.961 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:09:55.961 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:55.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:55.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:55.962 | INFO | ANALYSIS : 64.3%, 3.0, 100.5ms, 0 | 20260505 06:09:55.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:55.962 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:55.962 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 06:09:55.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:55.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:09:55.962 | INFO | Uploaded 36GiB to GPU in 2.12911s (16.9084GiB/s) | 20260505 06:09:55.962 | INFO | Downloaded 34GiB to GPU in 2.04196s (16.6507GiB/s) | 20260505 06:10:01.101 | INFO | Analyzing chunk frames = [947200,947712) | 20260505 06:10:01.101 | INFO | MemInfo: { | 20260505 06:10:01.101 | INFO | "gpu" : " 17 GB", | 20260505 06:10:01.101 | INFO | "huge" : "", | 20260505 06:10:01.101 | INFO | "huge_shared" : " 27 GB", | 20260505 06:10:01.101 | INFO | "malloc" : " 14 GB", | 20260505 06:10:01.101 | INFO | "pinned" : " 29 GB" | 20260505 06:10:01.101 | INFO | } | 20260505 06:10:01.101 | INFO | Perf report for the last 5.13922 seconds: | 20260505 06:10:01.101 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:10:01.101 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:01.101 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:01.101 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:01.101 | INFO | ANALYSIS : 64.1%, 3.0, 99.7ms, 0 | 20260505 06:10:01.101 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:01.101 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:01.101 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 06:10:01.101 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:01.101 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:01.101 | INFO | Uploaded 36GiB to GPU in 2.15976s (16.6685GiB/s) | 20260505 06:10:01.101 | INFO | Downloaded 34GiB to GPU in 2.03866s (16.6776GiB/s) | 20260505 06:10:06.240 | INFO | Analyzing chunk frames = [947712,948224) | 20260505 06:10:06.240 | INFO | MemInfo: { | 20260505 06:10:06.240 | INFO | "gpu" : " 17 GB", | 20260505 06:10:06.240 | INFO | "huge" : "", | 20260505 06:10:06.240 | INFO | "huge_shared" : " 27 GB", | 20260505 06:10:06.240 | INFO | "malloc" : " 14 GB", | 20260505 06:10:06.240 | INFO | "pinned" : " 29 GB" | 20260505 06:10:06.240 | INFO | } | 20260505 06:10:06.240 | INFO | Perf report for the last 5.13944 seconds: | 20260505 06:10:06.240 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:10:06.240 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:06.240 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:06.240 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:06.240 | INFO | ANALYSIS : 63.4%, 3.0, 98.5ms, 0 | 20260505 06:10:06.240 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:06.240 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:06.240 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 06:10:06.240 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:06.240 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:06.240 | INFO | Uploaded 36GiB to GPU in 2.12625s (16.9312GiB/s) | 20260505 06:10:06.241 | INFO | Downloaded 34GiB to GPU in 2.04755s (16.6052GiB/s) | 20260505 06:10:11.379 | INFO | Analyzing chunk frames = [948224,948736) | 20260505 06:10:11.379 | INFO | MemInfo: { | 20260505 06:10:11.379 | INFO | "gpu" : " 17 GB", | 20260505 06:10:11.379 | INFO | "huge" : "", | 20260505 06:10:11.379 | INFO | "huge_shared" : " 27 GB", | 20260505 06:10:11.379 | INFO | "malloc" : " 14 GB", | 20260505 06:10:11.379 | INFO | "pinned" : " 29 GB" | 20260505 06:10:11.379 | INFO | } | 20260505 06:10:11.379 | INFO | Perf report for the last 5.13928 seconds: | 20260505 06:10:11.379 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:10:11.379 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:11.379 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:11.379 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:11.380 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 06:10:11.380 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:11.380 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:11.380 | INFO | PRE_HQ : 30.5%, 1.0, 15.9ms, 0 | 20260505 06:10:11.380 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:11.380 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:11.380 | INFO | Uploaded 36GiB to GPU in 2.10879s (17.0714GiB/s) | 20260505 06:10:11.380 | INFO | Downloaded 34GiB to GPU in 2.04695s (16.6101GiB/s) | 20260505 06:10:16.519 | INFO | Analyzing chunk frames = [948736,949248) | 20260505 06:10:16.519 | INFO | MemInfo: { | 20260505 06:10:16.519 | INFO | "gpu" : " 17 GB", | 20260505 06:10:16.519 | INFO | "huge" : "", | 20260505 06:10:16.519 | INFO | "huge_shared" : " 27 GB", | 20260505 06:10:16.519 | INFO | "malloc" : " 14 GB", | 20260505 06:10:16.519 | INFO | "pinned" : " 29 GB" | 20260505 06:10:16.519 | INFO | } | 20260505 06:10:16.519 | INFO | Perf report for the last 5.13982 seconds: | 20260505 06:10:16.519 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:10:16.519 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:16.519 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:16.519 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:16.519 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 06:10:16.519 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:16.519 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:16.519 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 06:10:16.519 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:16.519 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:16.520 | INFO | Uploaded 36GiB to GPU in 2.16318s (16.6422GiB/s) | 20260505 06:10:16.520 | INFO | Downloaded 34GiB to GPU in 2.05687s (16.53GiB/s) | 20260505 06:10:21.658 | INFO | Analyzing chunk frames = [949248,949760) | 20260505 06:10:21.658 | INFO | MemInfo: { | 20260505 06:10:21.658 | INFO | "gpu" : " 17 GB", | 20260505 06:10:21.658 | INFO | "huge" : "", | 20260505 06:10:21.658 | INFO | "huge_shared" : " 27 GB", | 20260505 06:10:21.658 | INFO | "malloc" : " 14 GB", | 20260505 06:10:21.658 | INFO | "pinned" : " 29 GB" | 20260505 06:10:21.658 | INFO | } | 20260505 06:10:21.658 | INFO | Perf report for the last 5.13889 seconds: | 20260505 06:10:21.658 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:10:21.658 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:21.658 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:21.658 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:21.658 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 06:10:21.658 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:21.658 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:21.658 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260505 06:10:21.658 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:21.658 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:21.658 | INFO | Uploaded 36GiB to GPU in 2.17435s (16.5567GiB/s) | 20260505 06:10:21.658 | INFO | Downloaded 34GiB to GPU in 2.15003s (15.8138GiB/s) | 20260505 06:10:26.797 | INFO | Analyzing chunk frames = [949760,950272) | 20260505 06:10:26.797 | INFO | MemInfo: { | 20260505 06:10:26.797 | INFO | "gpu" : " 17 GB", | 20260505 06:10:26.797 | INFO | "huge" : "", | 20260505 06:10:26.797 | INFO | "huge_shared" : " 27 GB", | 20260505 06:10:26.797 | INFO | "malloc" : " 14 GB", | 20260505 06:10:26.797 | INFO | "pinned" : " 29 GB" | 20260505 06:10:26.797 | INFO | } | 20260505 06:10:26.798 | INFO | Perf report for the last 5.13931 seconds: | 20260505 06:10:26.798 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:10:26.798 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:26.798 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:26.798 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:26.798 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 06:10:26.798 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:26.798 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:26.798 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 06:10:26.798 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:26.798 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:26.798 | INFO | Uploaded 36GiB to GPU in 2.09515s (17.1825GiB/s) | 20260505 06:10:26.798 | INFO | Downloaded 34GiB to GPU in 2.06337s (16.4779GiB/s) | 20260505 06:10:26.798 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 06:10:26.798 | INFO | Total mem usage: 15.3 GiB | 20260505 06:10:26.798 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 06:10:26.798 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.45%) | 20260505 06:10:26.798 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 06:10:26.798 | INFO | High Water Breakdown: | 20260505 06:10:26.798 | INFO | 60.2% : 8.67 GiB ( 92.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 06:10:26.798 | INFO | 39.7% : 5.71 GiB ( 85.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 06:10:26.798 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 06:10:26.798 | INFO | ------------------------------------------ | 20260505 06:10:26.798 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:10:26.798 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 06:10:26.798 | INFO | Total mem usage: 26.5 GiB | 20260505 06:10:26.798 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 06:10:26.798 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 06:10:26.798 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 06:10:26.798 | INFO | High Water Breakdown: | 20260505 06:10:26.798 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:10:26.798 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:10:26.798 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:10:26.798 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:10:26.798 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:10:26.798 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:10:26.798 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 06:10:26.798 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 06:10:26.798 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:10:26.798 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 06:10:26.798 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 06:10:26.798 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 06:10:26.798 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 06:10:26.798 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 06:10:26.798 | INFO | ------------------------------------------ | 20260505 06:10:26.798 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:10:26.798 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 06:10:26.798 | INFO | Total mem usage: 25.6 GiB | 20260505 06:10:26.798 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 06:10:26.798 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 06:10:26.798 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 06:10:26.798 | INFO | High Water Breakdown: | 20260505 06:10:26.798 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 06:10:26.798 | INFO | ------------------------------------------ | 20260505 06:10:26.798 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:10:26.798 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 06:10:26.798 | INFO | Total mem usage: 15.7 GiB | 20260505 06:10:26.798 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 06:10:26.798 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 06:10:26.798 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260505 06:10:26.798 | INFO | High Water Breakdown: | 20260505 06:10:26.798 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 06:10:26.798 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 06:10:26.798 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 06:10:26.798 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 06:10:26.798 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 06:10:26.798 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 06:10:26.798 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 06:10:26.798 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 06:10:26.798 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 06:10:26.798 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:10:26.798 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 06:10:26.798 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:10:26.798 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 06:10:26.798 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 06:10:26.798 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 06:10:26.798 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 06:10:26.798 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:10:26.798 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 06:10:26.798 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:10:26.798 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:10:26.798 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:10:26.798 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 06:10:26.798 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 06:10:26.799 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:10:26.799 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 06:10:26.799 | INFO | ------------------------------------------ | 20260505 06:10:26.799 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:10:26.799 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 06:10:26.799 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:10:26.799 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:10:26.799 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.60e+01, 1.38e+01 / 1.42e+00 | 20260505 06:10:26.799 | INFO | PerfReport: 0.28%, AggregateStats -- 9.42e-02 / 1.03e-01, 1.21e-02 / 5.24e-03 | 20260505 06:10:26.799 | INFO | PerfReport: 11.58%, Baseline -- 3.85e+00 / 4.52e+00, 6.38e-01 / 2.61e-01 | 20260505 06:10:26.799 | INFO | PerfReport: 3.31%, Histogram -- 1.10e+00 / 1.51e+00, 2.19e-01 / 1.11e-01 | 20260505 06:10:26.799 | INFO | PerfReport: 15.47%, DME -- 4.11e+01 / 4.52e+01, 4.36e+00 / 1.08e+00 | 20260505 06:10:26.799 | INFO | PerfReport: 55.97%, FrameLabeling -- 1.86e+01 / 1.91e+01, 2.08e+00 / 1.11e+00 | 20260505 06:10:26.799 | INFO | PerfReport: 7.75%, PulseAccumulating -- 2.57e+00 / 3.18e+00, 4.73e-01 / 3.18e-01 | 20260505 06:10:26.799 | INFO | PerfReport: 4.14%, Metrics -- 1.37e+00 / 1.75e+00, 3.87e-01 / 2.13e-01 | 20260505 06:10:26.799 | INFO | PerfReport: 1.51%, PulseSerializing -- 5.02e-01 / 5.82e-01, 9.96e-02 / 4.13e-02 | 20260505 06:10:26.799 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 06:10:26.799 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:10:26.799 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:10:26.799 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.27e+01 / 0.00e+00 | 20260505 06:10:26.799 | INFO | PerfReport: 21.93%, Upload -- 2.23e+01 / 0.00e+00, 7.64e+00 / 0.00e+00 | 20260505 06:10:26.799 | INFO | PerfReport: 21.51%, Download -- 2.18e+01 / 0.00e+00, 7.69e+00 / 0.00e+00 | 20260505 06:10:26.799 | INFO | PerfReport: 32.72%, Compute -- 3.32e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 06:10:26.799 | INFO | PerfReport: 1.41%, LockUpload -- 1.43e+00 / 0.00e+00, 6.71e+00 / 0.00e+00 | 20260505 06:10:26.799 | INFO | PerfReport: 0.68%, LockDownload -- 6.93e-01 / 0.00e+00, 1.92e+00 / 0.00e+00 | 20260505 06:10:26.799 | INFO | PerfReport: 21.73%, LockCompute -- 2.21e+01 / 0.00e+00, 1.67e+01 / 0.00e+00 | 20260505 06:10:26.799 | INFO | PerfReport: 0.01%, Misc -- 1.50e-02 / 0.00e+00, 2.20e-02 / 0.00e+00 | 20260505 06:10:26.911 | INFO | Starting a round of Baz IO | 20260505 06:10:30.175 | INFO | Populating RTMetrics report | 20260505 06:10:30.176 | INFO | RTMetrics report populated | 20260505 06:10:30.176 | INFO | Opening tmp RTMetrics json file | 20260505 06:10:30.189 | INFO | Serializing RTMetrics to json | 20260505 06:10:30.190 | INFO | Serialization done, swapping tmp file now... | 20260505 06:10:30.190 | INFO | RTMetrics done | 20260505 06:10:30.288 | INFO | Finished a round of Baz IO: Wrote 2.342 GiB in 3.377 seconds | 20260505 06:10:30.288 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.58% / 0.06% / 65.36% | 20260505 06:10:30.288 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.66066% / 0.000111262% / 5.16574e-05% | 20260505 06:10:30.288 | INFO | Baz IO overall: | 20260505 06:10:30.288 | INFO | BytesWritten : 479841244333 | 20260505 06:10:30.288 | INFO | NumEvents : 226536882107 | 20260505 06:10:31.937 | INFO | Analyzing chunk frames = [950272,950784) | 20260505 06:10:31.937 | INFO | MemInfo: { | 20260505 06:10:31.937 | INFO | "gpu" : " 17 GB", | 20260505 06:10:31.937 | INFO | "huge" : "", | 20260505 06:10:31.937 | INFO | "huge_shared" : " 27 GB", | 20260505 06:10:31.937 | INFO | "malloc" : " 14 GB", | 20260505 06:10:31.937 | INFO | "pinned" : " 29 GB" | 20260505 06:10:31.937 | INFO | } | 20260505 06:10:31.937 | INFO | Perf report for the last 5.13949 seconds: | 20260505 06:10:31.937 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:10:31.937 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:31.937 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:31.937 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:31.937 | INFO | ANALYSIS : 64.6%, 2.9, 98.9ms, 0 | 20260505 06:10:31.937 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:31.937 | INFO | RT_METRICS : 46.2%, 1.0, 24.2ms, 0 | 20260505 06:10:31.937 | INFO | PRE_HQ : 63.6%, 1.9, 64.3ms, 0 | 20260505 06:10:31.937 | INFO | BAZWRITER : 65.0%, 4.0, 135.3ms, 0 | 20260505 06:10:31.937 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 06:10:31.937 | INFO | Uploaded 36GiB to GPU in 2.13989s (16.8233GiB/s) | 20260505 06:10:31.937 | INFO | Downloaded 38GiB to GPU in 2.32121s (16.3707GiB/s) | 20260505 06:10:37.076 | INFO | Analyzing chunk frames = [950784,951296) | 20260505 06:10:37.076 | INFO | MemInfo: { | 20260505 06:10:37.076 | INFO | "gpu" : " 17 GB", | 20260505 06:10:37.076 | INFO | "huge" : "", | 20260505 06:10:37.076 | INFO | "huge_shared" : " 27 GB", | 20260505 06:10:37.076 | INFO | "malloc" : " 14 GB", | 20260505 06:10:37.076 | INFO | "pinned" : " 29 GB" | 20260505 06:10:37.076 | INFO | } | 20260505 06:10:37.076 | INFO | Perf report for the last 5.1391 seconds: | 20260505 06:10:37.076 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:10:37.076 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:37.076 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:37.076 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:37.076 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 06:10:37.076 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:37.076 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:37.076 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 06:10:37.076 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:37.076 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:37.076 | INFO | Uploaded 36GiB to GPU in 2.13696s (16.8464GiB/s) | 20260505 06:10:37.076 | INFO | Downloaded 34GiB to GPU in 2.03377s (16.7177GiB/s) | 20260505 06:10:42.215 | INFO | Analyzing chunk frames = [951296,951808) | 20260505 06:10:42.215 | INFO | MemInfo: { | 20260505 06:10:42.215 | INFO | "gpu" : " 17 GB", | 20260505 06:10:42.215 | INFO | "huge" : "", | 20260505 06:10:42.215 | INFO | "huge_shared" : " 27 GB", | 20260505 06:10:42.215 | INFO | "malloc" : " 14 GB", | 20260505 06:10:42.216 | INFO | "pinned" : " 29 GB" | 20260505 06:10:42.216 | INFO | } | 20260505 06:10:42.216 | INFO | Perf report for the last 5.13943 seconds: | 20260505 06:10:42.216 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:10:42.216 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:42.216 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:42.216 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:42.216 | INFO | ANALYSIS : 64.0%, 3.0, 99.6ms, 0 | 20260505 06:10:42.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:42.216 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:42.216 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 06:10:42.216 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:42.216 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:42.216 | INFO | Uploaded 36GiB to GPU in 2.15488s (16.7062GiB/s) | 20260505 06:10:42.216 | INFO | Downloaded 34GiB to GPU in 2.03126s (16.7384GiB/s) | 20260505 06:10:47.355 | INFO | Analyzing chunk frames = [951808,952320) | 20260505 06:10:47.355 | INFO | MemInfo: { | 20260505 06:10:47.355 | INFO | "gpu" : " 17 GB", | 20260505 06:10:47.355 | INFO | "huge" : "", | 20260505 06:10:47.355 | INFO | "huge_shared" : " 27 GB", | 20260505 06:10:47.355 | INFO | "malloc" : " 14 GB", | 20260505 06:10:47.355 | INFO | "pinned" : " 29 GB" | 20260505 06:10:47.355 | INFO | } | 20260505 06:10:47.355 | INFO | Perf report for the last 5.13924 seconds: | 20260505 06:10:47.355 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:10:47.355 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:47.355 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:47.355 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:47.355 | INFO | ANALYSIS : 63.3%, 3.0, 98.3ms, 0 | 20260505 06:10:47.355 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:47.355 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:47.355 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 06:10:47.355 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:47.355 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:47.355 | INFO | Uploaded 36GiB to GPU in 2.13145s (16.8899GiB/s) | 20260505 06:10:47.355 | INFO | Downloaded 34GiB to GPU in 2.05097s (16.5775GiB/s) | 20260505 06:10:52.494 | INFO | Analyzing chunk frames = [952320,952832) | 20260505 06:10:52.494 | INFO | MemInfo: { | 20260505 06:10:52.494 | INFO | "gpu" : " 17 GB", | 20260505 06:10:52.494 | INFO | "huge" : "", | 20260505 06:10:52.494 | INFO | "huge_shared" : " 27 GB", | 20260505 06:10:52.494 | INFO | "malloc" : " 14 GB", | 20260505 06:10:52.494 | INFO | "pinned" : " 29 GB" | 20260505 06:10:52.494 | INFO | } | 20260505 06:10:52.494 | INFO | Perf report for the last 5.13947 seconds: | 20260505 06:10:52.494 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:10:52.494 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:52.494 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:52.494 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:52.494 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 06:10:52.494 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:52.494 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:52.494 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 06:10:52.494 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:52.494 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:52.495 | INFO | Uploaded 36GiB to GPU in 2.1187s (16.9916GiB/s) | 20260505 06:10:52.495 | INFO | Downloaded 34GiB to GPU in 2.04583s (16.6191GiB/s) | 20260505 06:10:57.634 | INFO | Analyzing chunk frames = [952832,953344) | 20260505 06:10:57.634 | INFO | MemInfo: { | 20260505 06:10:57.634 | INFO | "gpu" : " 17 GB", | 20260505 06:10:57.634 | INFO | "huge" : "", | 20260505 06:10:57.634 | INFO | "huge_shared" : " 27 GB", | 20260505 06:10:57.634 | INFO | "malloc" : " 14 GB", | 20260505 06:10:57.634 | INFO | "pinned" : " 29 GB" | 20260505 06:10:57.634 | INFO | } | 20260505 06:10:57.634 | INFO | Perf report for the last 5.14004 seconds: | 20260505 06:10:57.634 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:10:57.634 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:57.634 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:57.634 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:57.634 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 06:10:57.634 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:57.634 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:57.634 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 06:10:57.634 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:57.634 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:10:57.635 | INFO | Uploaded 36GiB to GPU in 2.1289s (16.9101GiB/s) | 20260505 06:10:57.635 | INFO | Downloaded 34GiB to GPU in 2.06546s (16.4612GiB/s) | 20260505 06:11:02.773 | INFO | Analyzing chunk frames = [953344,953856) | 20260505 06:11:02.773 | INFO | MemInfo: { | 20260505 06:11:02.773 | INFO | "gpu" : " 17 GB", | 20260505 06:11:02.773 | INFO | "huge" : "", | 20260505 06:11:02.773 | INFO | "huge_shared" : " 27 GB", | 20260505 06:11:02.773 | INFO | "malloc" : " 14 GB", | 20260505 06:11:02.773 | INFO | "pinned" : " 29 GB" | 20260505 06:11:02.773 | INFO | } | 20260505 06:11:02.773 | INFO | Perf report for the last 5.13862 seconds: | 20260505 06:11:02.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:11:02.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:02.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:02.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:02.773 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 06:11:02.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:02.773 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:02.773 | INFO | PRE_HQ : 32.4%, 1.0, 16.9ms, 0 | 20260505 06:11:02.773 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:02.773 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:02.773 | INFO | Uploaded 36GiB to GPU in 2.20017s (16.3624GiB/s) | 20260505 06:11:02.773 | INFO | Downloaded 34GiB to GPU in 2.14925s (15.8195GiB/s) | 20260505 06:11:07.912 | INFO | Analyzing chunk frames = [953856,954368) | 20260505 06:11:07.912 | INFO | MemInfo: { | 20260505 06:11:07.912 | INFO | "gpu" : " 17 GB", | 20260505 06:11:07.912 | INFO | "huge" : "", | 20260505 06:11:07.912 | INFO | "huge_shared" : " 27 GB", | 20260505 06:11:07.912 | INFO | "malloc" : " 14 GB", | 20260505 06:11:07.912 | INFO | "pinned" : " 29 GB" | 20260505 06:11:07.912 | INFO | } | 20260505 06:11:07.912 | INFO | Perf report for the last 5.13922 seconds: | 20260505 06:11:07.912 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:11:07.912 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:07.912 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:07.912 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:07.912 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 06:11:07.912 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:07.912 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:07.912 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 06:11:07.912 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:07.912 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:07.912 | INFO | Uploaded 36GiB to GPU in 2.09628s (17.1733GiB/s) | 20260505 06:11:07.913 | INFO | Downloaded 34GiB to GPU in 2.07746s (16.3661GiB/s) | 20260505 06:11:08.027 | INFO | Starting a round of Baz IO | 20260505 06:11:11.308 | INFO | Populating RTMetrics report | 20260505 06:11:11.308 | INFO | RTMetrics report populated | 20260505 06:11:11.308 | INFO | Opening tmp RTMetrics json file | 20260505 06:11:11.329 | INFO | Serializing RTMetrics to json | 20260505 06:11:11.330 | INFO | Serialization done, swapping tmp file now... | 20260505 06:11:11.331 | INFO | RTMetrics done | 20260505 06:11:11.419 | INFO | Finished a round of Baz IO: Wrote 2.346 GiB in 3.391 seconds | 20260505 06:11:11.419 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.53% / 0.06% / 65.42% | 20260505 06:11:11.419 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.65416% / 9.93411e-05% / 7.15256e-05% | 20260505 06:11:11.419 | INFO | Baz IO overall: | 20260505 06:11:11.419 | INFO | BytesWritten : 482359827806 | 20260505 06:11:11.419 | INFO | NumEvents : 227740778343 | 20260505 06:11:13.051 | INFO | Analyzing chunk frames = [954368,954880) | 20260505 06:11:13.051 | INFO | MemInfo: { | 20260505 06:11:13.051 | INFO | "gpu" : " 17 GB", | 20260505 06:11:13.051 | INFO | "huge" : "", | 20260505 06:11:13.051 | INFO | "huge_shared" : " 27 GB", | 20260505 06:11:13.051 | INFO | "malloc" : " 14 GB", | 20260505 06:11:13.051 | INFO | "pinned" : " 29 GB" | 20260505 06:11:13.052 | INFO | } | 20260505 06:11:13.052 | INFO | Perf report for the last 5.13934 seconds: | 20260505 06:11:13.052 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:11:13.052 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:13.052 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:13.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:13.052 | INFO | ANALYSIS : 64.9%, 2.9, 99.8ms, 0 | 20260505 06:11:13.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:13.052 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260505 06:11:13.052 | INFO | PRE_HQ : 63.8%, 1.9, 64.3ms, 0 | 20260505 06:11:13.052 | INFO | BAZWRITER : 65.3%, 4.0, 134.9ms, 0 | 20260505 06:11:13.052 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 06:11:13.052 | INFO | Uploaded 36GiB to GPU in 2.22845s (16.1547GiB/s) | 20260505 06:11:13.052 | INFO | Downloaded 38GiB to GPU in 2.34188s (16.2263GiB/s) | 20260505 06:11:18.191 | INFO | Analyzing chunk frames = [954880,955392) | 20260505 06:11:18.191 | INFO | MemInfo: { | 20260505 06:11:18.191 | INFO | "gpu" : " 17 GB", | 20260505 06:11:18.191 | INFO | "huge" : "", | 20260505 06:11:18.191 | INFO | "huge_shared" : " 27 GB", | 20260505 06:11:18.191 | INFO | "malloc" : " 14 GB", | 20260505 06:11:18.191 | INFO | "pinned" : " 29 GB" | 20260505 06:11:18.191 | INFO | } | 20260505 06:11:18.191 | INFO | Perf report for the last 5.13926 seconds: | 20260505 06:11:18.191 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:11:18.191 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:18.191 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:18.191 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:18.191 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 06:11:18.191 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:18.191 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:18.191 | INFO | PRE_HQ : 30.5%, 1.0, 15.9ms, 0 | 20260505 06:11:18.191 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:18.191 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:18.191 | INFO | Uploaded 36GiB to GPU in 2.12772s (16.9195GiB/s) | 20260505 06:11:18.191 | INFO | Downloaded 34GiB to GPU in 2.04287s (16.6432GiB/s) | 20260505 06:11:23.330 | INFO | Analyzing chunk frames = [955392,955904) | 20260505 06:11:23.330 | INFO | MemInfo: { | 20260505 06:11:23.330 | INFO | "gpu" : " 17 GB", | 20260505 06:11:23.330 | INFO | "huge" : "", | 20260505 06:11:23.330 | INFO | "huge_shared" : " 27 GB", | 20260505 06:11:23.330 | INFO | "malloc" : " 14 GB", | 20260505 06:11:23.330 | INFO | "pinned" : " 29 GB" | 20260505 06:11:23.330 | INFO | } | 20260505 06:11:23.330 | INFO | Perf report for the last 5.13926 seconds: | 20260505 06:11:23.330 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:11:23.330 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:23.330 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:23.330 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:23.330 | INFO | ANALYSIS : 64.0%, 3.0, 99.7ms, 0 | 20260505 06:11:23.330 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:23.330 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:23.330 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 06:11:23.330 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:23.330 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:23.330 | INFO | Uploaded 36GiB to GPU in 2.1609s (16.6597GiB/s) | 20260505 06:11:23.330 | INFO | Downloaded 34GiB to GPU in 2.0361s (16.6986GiB/s) | 20260505 06:11:28.469 | INFO | Analyzing chunk frames = [955904,956416) | 20260505 06:11:28.469 | INFO | MemInfo: { | 20260505 06:11:28.469 | INFO | "gpu" : " 17 GB", | 20260505 06:11:28.469 | INFO | "huge" : "", | 20260505 06:11:28.470 | INFO | "huge_shared" : " 27 GB", | 20260505 06:11:28.470 | INFO | "malloc" : " 14 GB", | 20260505 06:11:28.470 | INFO | "pinned" : " 29 GB" | 20260505 06:11:28.470 | INFO | } | 20260505 06:11:28.470 | INFO | Perf report for the last 5.13946 seconds: | 20260505 06:11:28.470 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:11:28.470 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:28.470 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:28.470 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:28.470 | INFO | ANALYSIS : 63.4%, 3.0, 98.5ms, 0 | 20260505 06:11:28.470 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:28.470 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:28.470 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 06:11:28.470 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:28.470 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:28.470 | INFO | Uploaded 36GiB to GPU in 2.14759s (16.7629GiB/s) | 20260505 06:11:28.470 | INFO | Downloaded 34GiB to GPU in 2.05005s (16.5849GiB/s) | 20260505 06:11:33.609 | INFO | Analyzing chunk frames = [956416,956928) | 20260505 06:11:33.609 | INFO | MemInfo: { | 20260505 06:11:33.609 | INFO | "gpu" : " 17 GB", | 20260505 06:11:33.609 | INFO | "huge" : "", | 20260505 06:11:33.609 | INFO | "huge_shared" : " 27 GB", | 20260505 06:11:33.609 | INFO | "malloc" : " 14 GB", | 20260505 06:11:33.609 | INFO | "pinned" : " 29 GB" | 20260505 06:11:33.609 | INFO | } | 20260505 06:11:33.609 | INFO | Perf report for the last 5.13937 seconds: | 20260505 06:11:33.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:11:33.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:33.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:33.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:33.609 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 06:11:33.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:33.609 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:33.609 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 06:11:33.609 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:33.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:33.609 | INFO | Uploaded 36GiB to GPU in 2.12319s (16.9556GiB/s) | 20260505 06:11:33.609 | INFO | Downloaded 34GiB to GPU in 2.0506s (16.5805GiB/s) | 20260505 06:11:38.750 | INFO | Analyzing chunk frames = [956928,957440) | 20260505 06:11:38.750 | INFO | MemInfo: { | 20260505 06:11:38.750 | INFO | "gpu" : " 17 GB", | 20260505 06:11:38.750 | INFO | "huge" : "", | 20260505 06:11:38.750 | INFO | "huge_shared" : " 27 GB", | 20260505 06:11:38.750 | INFO | "malloc" : " 14 GB", | 20260505 06:11:38.750 | INFO | "pinned" : " 29 GB" | 20260505 06:11:38.750 | INFO | } | 20260505 06:11:38.750 | INFO | Perf report for the last 5.14073 seconds: | 20260505 06:11:38.750 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:11:38.750 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:38.750 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:38.750 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:38.750 | INFO | ANALYSIS : 64.1%, 3.0, 100.2ms, 0 | 20260505 06:11:38.750 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:38.750 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:38.750 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260505 06:11:38.750 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:38.750 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:38.750 | INFO | Uploaded 36GiB to GPU in 2.16031s (16.6643GiB/s) | 20260505 06:11:38.750 | INFO | Downloaded 34GiB to GPU in 2.05278s (16.5629GiB/s) | 20260505 06:11:43.888 | INFO | Analyzing chunk frames = [957440,957952) | 20260505 06:11:43.888 | INFO | MemInfo: { | 20260505 06:11:43.888 | INFO | "gpu" : " 17 GB", | 20260505 06:11:43.888 | INFO | "huge" : "", | 20260505 06:11:43.888 | INFO | "huge_shared" : " 27 GB", | 20260505 06:11:43.888 | INFO | "malloc" : " 14 GB", | 20260505 06:11:43.888 | INFO | "pinned" : " 29 GB" | 20260505 06:11:43.888 | INFO | } | 20260505 06:11:43.888 | INFO | Perf report for the last 5.13807 seconds: | 20260505 06:11:43.888 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:11:43.888 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:43.888 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:43.888 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:43.888 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 06:11:43.888 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:43.888 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:43.888 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260505 06:11:43.888 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:43.888 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:43.888 | INFO | Uploaded 36GiB to GPU in 2.17192s (16.5752GiB/s) | 20260505 06:11:43.888 | INFO | Downloaded 34GiB to GPU in 2.13777s (15.9045GiB/s) | 20260505 06:11:49.027 | INFO | Analyzing chunk frames = [957952,958464) | 20260505 06:11:49.027 | INFO | MemInfo: { | 20260505 06:11:49.027 | INFO | "gpu" : " 17 GB", | 20260505 06:11:49.027 | INFO | "huge" : "", | 20260505 06:11:49.027 | INFO | "huge_shared" : " 27 GB", | 20260505 06:11:49.027 | INFO | "malloc" : " 14 GB", | 20260505 06:11:49.027 | INFO | "pinned" : " 29 GB" | 20260505 06:11:49.027 | INFO | } | 20260505 06:11:49.027 | INFO | Perf report for the last 5.13943 seconds: | 20260505 06:11:49.027 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:11:49.027 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:49.027 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:49.027 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:49.027 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 06:11:49.027 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:49.027 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:49.027 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260505 06:11:49.027 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:49.027 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:49.027 | INFO | Uploaded 36GiB to GPU in 2.09589s (17.1765GiB/s) | 20260505 06:11:49.027 | INFO | Downloaded 34GiB to GPU in 2.06695s (16.4493GiB/s) | 20260505 06:11:49.028 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 06:11:49.028 | INFO | Total mem usage: 15.3 GiB | 20260505 06:11:49.028 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 06:11:49.028 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.5%) | 20260505 06:11:49.028 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 06:11:49.028 | INFO | High Water Breakdown: | 20260505 06:11:49.028 | INFO | 60.2% : 8.66 GiB ( 92.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 06:11:49.028 | INFO | 39.7% : 5.71 GiB ( 85.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 06:11:49.028 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 06:11:49.028 | INFO | ------------------------------------------ | 20260505 06:11:49.028 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:11:49.028 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 06:11:49.028 | INFO | Total mem usage: 26.5 GiB | 20260505 06:11:49.028 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 06:11:49.028 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 06:11:49.028 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 06:11:49.028 | INFO | High Water Breakdown: | 20260505 06:11:49.028 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:11:49.028 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:11:49.028 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:11:49.028 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:11:49.028 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:11:49.028 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:11:49.028 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 06:11:49.028 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 06:11:49.028 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:11:49.028 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 06:11:49.028 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 06:11:49.028 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 06:11:49.028 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 06:11:49.028 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 06:11:49.028 | INFO | ------------------------------------------ | 20260505 06:11:49.028 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:11:49.028 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 06:11:49.028 | INFO | Total mem usage: 25.6 GiB | 20260505 06:11:49.028 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 06:11:49.028 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 06:11:49.028 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 06:11:49.028 | INFO | High Water Breakdown: | 20260505 06:11:49.028 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 06:11:49.028 | INFO | ------------------------------------------ | 20260505 06:11:49.028 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:11:49.028 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 06:11:49.028 | INFO | Total mem usage: 15.7 GiB | 20260505 06:11:49.028 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 06:11:49.028 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 06:11:49.028 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260505 06:11:49.028 | INFO | High Water Breakdown: | 20260505 06:11:49.028 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 06:11:49.028 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 06:11:49.028 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 06:11:49.028 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 06:11:49.028 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 06:11:49.028 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 06:11:49.028 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 06:11:49.028 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 06:11:49.028 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 06:11:49.028 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:11:49.028 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 06:11:49.028 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:11:49.028 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 06:11:49.028 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 06:11:49.028 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 06:11:49.028 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 06:11:49.028 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:11:49.028 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 06:11:49.028 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:11:49.028 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:11:49.028 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:11:49.028 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 06:11:49.028 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 06:11:49.028 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:11:49.028 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 06:11:49.028 | INFO | ------------------------------------------ | 20260505 06:11:49.028 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:11:49.028 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 06:11:49.028 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:11:49.028 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:11:49.028 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.58e+01, 1.38e+01 / 1.73e+00 | 20260505 06:11:49.028 | INFO | PerfReport: 0.28%, AggregateStats -- 9.39e-02 / 1.04e-01, 1.06e-02 / 5.67e-03 | 20260505 06:11:49.028 | INFO | PerfReport: 11.57%, Baseline -- 3.84e+00 / 4.50e+00, 6.33e-01 / 2.68e-01 | 20260505 06:11:49.028 | INFO | PerfReport: 3.30%, Histogram -- 1.10e+00 / 1.50e+00, 2.19e-01 / 1.21e-01 | 20260505 06:11:49.028 | INFO | PerfReport: 15.46%, DME -- 4.10e+01 / 4.51e+01, 4.41e+00 / 1.18e+00 | 20260505 06:11:49.028 | INFO | PerfReport: 56.00%, FrameLabeling -- 1.86e+01 / 1.90e+01, 2.07e+00 / 9.94e-01 | 20260505 06:11:49.028 | INFO | PerfReport: 7.74%, PulseAccumulating -- 2.57e+00 / 3.17e+00, 4.75e-01 / 3.29e-01 | 20260505 06:11:49.028 | INFO | PerfReport: 4.14%, Metrics -- 1.37e+00 / 1.76e+00, 3.84e-01 / 2.03e-01 | 20260505 06:11:49.028 | INFO | PerfReport: 1.51%, PulseSerializing -- 5.02e-01 / 5.86e-01, 9.88e-02 / 3.68e-02 | 20260505 06:11:49.028 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 06:11:49.028 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:11:49.028 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:11:49.028 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.24e+01 / 0.00e+00 | 20260505 06:11:49.028 | INFO | PerfReport: 22.01%, Upload -- 2.23e+01 / 0.00e+00, 7.71e+00 / 0.00e+00 | 20260505 06:11:49.028 | INFO | PerfReport: 21.52%, Download -- 2.18e+01 / 0.00e+00, 7.73e+00 / 0.00e+00 | 20260505 06:11:49.028 | INFO | PerfReport: 32.70%, Compute -- 3.32e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 06:11:49.028 | INFO | PerfReport: 1.42%, LockUpload -- 1.44e+00 / 0.00e+00, 6.75e+00 / 0.00e+00 | 20260505 06:11:49.028 | INFO | PerfReport: 0.68%, LockDownload -- 6.86e-01 / 0.00e+00, 1.90e+00 / 0.00e+00 | 20260505 06:11:49.028 | INFO | PerfReport: 21.66%, LockCompute -- 2.20e+01 / 0.00e+00, 1.67e+01 / 0.00e+00 | 20260505 06:11:49.028 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 1.64e-02 / 0.00e+00 | 20260505 06:11:49.141 | INFO | Starting a round of Baz IO | 20260505 06:11:52.418 | INFO | Populating RTMetrics report | 20260505 06:11:52.419 | INFO | RTMetrics report populated | 20260505 06:11:52.419 | INFO | Opening tmp RTMetrics json file | 20260505 06:11:52.423 | INFO | Serializing RTMetrics to json | 20260505 06:11:52.425 | INFO | Serialization done, swapping tmp file now... | 20260505 06:11:52.427 | INFO | RTMetrics done | 20260505 06:11:52.540 | INFO | Finished a round of Baz IO: Wrote 2.347 GiB in 3.398 seconds | 20260505 06:11:52.540 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.47% / 0.06% / 65.47% | 20260505 06:11:52.540 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.62995% / 8.34465e-05% / 3.17891e-05% | 20260505 06:11:52.540 | INFO | Baz IO overall: | 20260505 06:11:52.540 | INFO | BytesWritten : 484879497918 | 20260505 06:11:52.540 | INFO | NumEvents : 228945162034 | 20260505 06:11:54.166 | INFO | Analyzing chunk frames = [958464,958976) | 20260505 06:11:54.166 | INFO | MemInfo: { | 20260505 06:11:54.166 | INFO | "gpu" : " 17 GB", | 20260505 06:11:54.166 | INFO | "huge" : "", | 20260505 06:11:54.166 | INFO | "huge_shared" : " 27 GB", | 20260505 06:11:54.166 | INFO | "malloc" : " 14 GB", | 20260505 06:11:54.166 | INFO | "pinned" : " 29 GB" | 20260505 06:11:54.166 | INFO | } | 20260505 06:11:54.166 | INFO | Perf report for the last 5.13921 seconds: | 20260505 06:11:54.166 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:11:54.166 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:54.166 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:54.166 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:54.167 | INFO | ANALYSIS : 64.8%, 2.9, 98.2ms, 0 | 20260505 06:11:54.167 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:54.167 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260505 06:11:54.167 | INFO | PRE_HQ : 63.8%, 1.9, 64.4ms, 0 | 20260505 06:11:54.167 | INFO | BAZWRITER : 65.4%, 4.1, 137.8ms, 0 | 20260505 06:11:54.167 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 06:11:54.167 | INFO | Uploaded 36GiB to GPU in 2.22521s (16.1783GiB/s) | 20260505 06:11:54.167 | INFO | Downloaded 38GiB to GPU in 2.33009s (16.3084GiB/s) | 20260505 06:11:59.306 | INFO | Analyzing chunk frames = [958976,959488) | 20260505 06:11:59.306 | INFO | MemInfo: { | 20260505 06:11:59.306 | INFO | "gpu" : " 17 GB", | 20260505 06:11:59.306 | INFO | "huge" : "", | 20260505 06:11:59.306 | INFO | "huge_shared" : " 27 GB", | 20260505 06:11:59.306 | INFO | "malloc" : " 14 GB", | 20260505 06:11:59.306 | INFO | "pinned" : " 29 GB" | 20260505 06:11:59.306 | INFO | } | 20260505 06:11:59.306 | INFO | Perf report for the last 5.13952 seconds: | 20260505 06:11:59.306 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:11:59.306 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:59.306 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:59.306 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:59.306 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 06:11:59.306 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:59.306 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:59.306 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 06:11:59.306 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:59.306 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:11:59.306 | INFO | Uploaded 36GiB to GPU in 2.13255s (16.8812GiB/s) | 20260505 06:11:59.306 | INFO | Downloaded 34GiB to GPU in 2.04578s (16.6196GiB/s) | 20260505 06:12:04.445 | INFO | Analyzing chunk frames = [959488,960000) | 20260505 06:12:04.445 | INFO | MemInfo: { | 20260505 06:12:04.445 | INFO | "gpu" : " 17 GB", | 20260505 06:12:04.445 | INFO | "huge" : "", | 20260505 06:12:04.445 | INFO | "huge_shared" : " 27 GB", | 20260505 06:12:04.445 | INFO | "malloc" : " 14 GB", | 20260505 06:12:04.445 | INFO | "pinned" : " 29 GB" | 20260505 06:12:04.445 | INFO | } | 20260505 06:12:04.445 | INFO | Perf report for the last 5.13893 seconds: | 20260505 06:12:04.445 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:12:04.445 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:04.445 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:04.445 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:04.445 | INFO | ANALYSIS : 64.1%, 3.0, 99.7ms, 0 | 20260505 06:12:04.445 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:04.445 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:04.445 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 06:12:04.445 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:04.445 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:04.445 | INFO | Uploaded 36GiB to GPU in 2.15888s (16.6753GiB/s) | 20260505 06:12:04.445 | INFO | Downloaded 34GiB to GPU in 2.03949s (16.6708GiB/s) | 20260505 06:12:09.584 | INFO | Analyzing chunk frames = [960000,960512) | 20260505 06:12:09.584 | INFO | MemInfo: { | 20260505 06:12:09.584 | INFO | "gpu" : " 17 GB", | 20260505 06:12:09.584 | INFO | "huge" : "", | 20260505 06:12:09.584 | INFO | "huge_shared" : " 27 GB", | 20260505 06:12:09.584 | INFO | "malloc" : " 14 GB", | 20260505 06:12:09.584 | INFO | "pinned" : " 29 GB" | 20260505 06:12:09.584 | INFO | } | 20260505 06:12:09.584 | INFO | Perf report for the last 5.13931 seconds: | 20260505 06:12:09.584 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:12:09.584 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:09.584 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:09.584 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:09.584 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 06:12:09.584 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:09.584 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:09.584 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 06:12:09.584 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:09.584 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:09.584 | INFO | Uploaded 36GiB to GPU in 2.1327s (16.88GiB/s) | 20260505 06:12:09.584 | INFO | Downloaded 34GiB to GPU in 2.05839s (16.5177GiB/s) | 20260505 06:12:14.723 | INFO | Analyzing chunk frames = [960512,961024) | 20260505 06:12:14.723 | INFO | MemInfo: { | 20260505 06:12:14.723 | INFO | "gpu" : " 17 GB", | 20260505 06:12:14.723 | INFO | "huge" : "", | 20260505 06:12:14.723 | INFO | "huge_shared" : " 27 GB", | 20260505 06:12:14.724 | INFO | "malloc" : " 14 GB", | 20260505 06:12:14.724 | INFO | "pinned" : " 29 GB" | 20260505 06:12:14.724 | INFO | } | 20260505 06:12:14.724 | INFO | Perf report for the last 5.13935 seconds: | 20260505 06:12:14.724 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:12:14.724 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:14.724 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:14.724 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:14.724 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 06:12:14.724 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:14.724 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:14.724 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 06:12:14.724 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:14.724 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:14.724 | INFO | Uploaded 36GiB to GPU in 2.11554s (17.0169GiB/s) | 20260505 06:12:14.724 | INFO | Downloaded 34GiB to GPU in 2.0518s (16.5708GiB/s) | 20260505 06:12:19.865 | INFO | Analyzing chunk frames = [961024,961536) | 20260505 06:12:19.865 | INFO | MemInfo: { | 20260505 06:12:19.865 | INFO | "gpu" : " 17 GB", | 20260505 06:12:19.865 | INFO | "huge" : "", | 20260505 06:12:19.865 | INFO | "huge_shared" : " 27 GB", | 20260505 06:12:19.865 | INFO | "malloc" : " 14 GB", | 20260505 06:12:19.865 | INFO | "pinned" : " 29 GB" | 20260505 06:12:19.865 | INFO | } | 20260505 06:12:19.865 | INFO | Perf report for the last 5.14116 seconds: | 20260505 06:12:19.865 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:12:19.865 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:19.865 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:19.865 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:19.865 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 06:12:19.865 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:19.865 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:19.865 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260505 06:12:19.865 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:19.865 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:19.865 | INFO | Uploaded 36GiB to GPU in 2.12508s (16.9406GiB/s) | 20260505 06:12:19.865 | INFO | Downloaded 34GiB to GPU in 2.06379s (16.4745GiB/s) | 20260505 06:12:25.003 | INFO | Analyzing chunk frames = [961536,962048) | 20260505 06:12:25.003 | INFO | MemInfo: { | 20260505 06:12:25.003 | INFO | "gpu" : " 17 GB", | 20260505 06:12:25.003 | INFO | "huge" : "", | 20260505 06:12:25.003 | INFO | "huge_shared" : " 27 GB", | 20260505 06:12:25.003 | INFO | "malloc" : " 14 GB", | 20260505 06:12:25.003 | INFO | "pinned" : " 29 GB" | 20260505 06:12:25.003 | INFO | } | 20260505 06:12:25.003 | INFO | Perf report for the last 5.13808 seconds: | 20260505 06:12:25.003 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:12:25.003 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:25.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:25.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:25.003 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 06:12:25.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:25.003 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:25.003 | INFO | PRE_HQ : 32.4%, 1.0, 16.9ms, 0 | 20260505 06:12:25.003 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:25.003 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:25.003 | INFO | Uploaded 36GiB to GPU in 2.17323s (16.5652GiB/s) | 20260505 06:12:25.003 | INFO | Downloaded 34GiB to GPU in 2.13955s (15.8912GiB/s) | 20260505 06:12:30.142 | INFO | Analyzing chunk frames = [962048,962560) | 20260505 06:12:30.142 | INFO | MemInfo: { | 20260505 06:12:30.142 | INFO | "gpu" : " 17 GB", | 20260505 06:12:30.142 | INFO | "huge" : "", | 20260505 06:12:30.142 | INFO | "huge_shared" : " 27 GB", | 20260505 06:12:30.142 | INFO | "malloc" : " 14 GB", | 20260505 06:12:30.142 | INFO | "pinned" : " 29 GB" | 20260505 06:12:30.142 | INFO | } | 20260505 06:12:30.142 | INFO | Perf report for the last 5.13907 seconds: | 20260505 06:12:30.142 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:12:30.142 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:30.142 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:30.142 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:30.142 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 06:12:30.142 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:30.142 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:30.142 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 06:12:30.142 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:30.142 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:30.142 | INFO | Uploaded 36GiB to GPU in 2.09727s (17.1652GiB/s) | 20260505 06:12:30.142 | INFO | Downloaded 34GiB to GPU in 2.06896s (16.4334GiB/s) | 20260505 06:12:30.255 | INFO | Starting a round of Baz IO | 20260505 06:12:33.539 | INFO | Populating RTMetrics report | 20260505 06:12:33.539 | INFO | RTMetrics report populated | 20260505 06:12:33.539 | INFO | Opening tmp RTMetrics json file | 20260505 06:12:33.555 | INFO | Serializing RTMetrics to json | 20260505 06:12:33.558 | INFO | Serialization done, swapping tmp file now... | 20260505 06:12:33.561 | INFO | RTMetrics done | 20260505 06:12:33.664 | INFO | Finished a round of Baz IO: Wrote 2.349 GiB in 3.408 seconds | 20260505 06:12:33.664 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.42% / 0.06% / 65.53% | 20260505 06:12:33.664 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.60986% / 0.000111262% / 5.16574e-05% | 20260505 06:12:33.664 | INFO | Baz IO overall: | 20260505 06:12:33.664 | INFO | BytesWritten : 487401787502 | 20260505 06:12:33.664 | INFO | NumEvents : 230151027649 | 20260505 06:12:35.281 | INFO | Analyzing chunk frames = [962560,963072) | 20260505 06:12:35.281 | INFO | MemInfo: { | 20260505 06:12:35.281 | INFO | "gpu" : " 17 GB", | 20260505 06:12:35.281 | INFO | "huge" : "", | 20260505 06:12:35.281 | INFO | "huge_shared" : " 27 GB", | 20260505 06:12:35.281 | INFO | "malloc" : " 14 GB", | 20260505 06:12:35.281 | INFO | "pinned" : " 29 GB" | 20260505 06:12:35.281 | INFO | } | 20260505 06:12:35.281 | INFO | Perf report for the last 5.13916 seconds: | 20260505 06:12:35.281 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:12:35.281 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:35.281 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:35.281 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:35.281 | INFO | ANALYSIS : 65.0%, 2.9, 99.2ms, 0 | 20260505 06:12:35.281 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:35.281 | INFO | RT_METRICS : 46.7%, 1.0, 24.5ms, 0 | 20260505 06:12:35.281 | INFO | PRE_HQ : 63.8%, 1.9, 64.5ms, 0 | 20260505 06:12:35.281 | INFO | BAZWRITER : 65.6%, 4.1, 136.6ms, 0 | 20260505 06:12:35.281 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 06:12:35.281 | INFO | Uploaded 36GiB to GPU in 2.2013s (16.354GiB/s) | 20260505 06:12:35.281 | INFO | Downloaded 38GiB to GPU in 2.33115s (16.301GiB/s) | 20260505 06:12:40.420 | INFO | Analyzing chunk frames = [963072,963584) | 20260505 06:12:40.420 | INFO | MemInfo: { | 20260505 06:12:40.420 | INFO | "gpu" : " 17 GB", | 20260505 06:12:40.420 | INFO | "huge" : "", | 20260505 06:12:40.420 | INFO | "huge_shared" : " 27 GB", | 20260505 06:12:40.420 | INFO | "malloc" : " 14 GB", | 20260505 06:12:40.420 | INFO | "pinned" : " 29 GB" | 20260505 06:12:40.420 | INFO | } | 20260505 06:12:40.421 | INFO | Perf report for the last 5.13937 seconds: | 20260505 06:12:40.421 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:12:40.421 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:40.421 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:40.421 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:40.421 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 06:12:40.421 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:40.421 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:40.421 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 06:12:40.421 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:40.421 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:40.421 | INFO | Uploaded 36GiB to GPU in 2.13914s (16.8292GiB/s) | 20260505 06:12:40.421 | INFO | Downloaded 34GiB to GPU in 2.04733s (16.607GiB/s) | 20260505 06:12:45.560 | INFO | Analyzing chunk frames = [963584,964096) | 20260505 06:12:45.560 | INFO | MemInfo: { | 20260505 06:12:45.560 | INFO | "gpu" : " 17 GB", | 20260505 06:12:45.560 | INFO | "huge" : "", | 20260505 06:12:45.560 | INFO | "huge_shared" : " 27 GB", | 20260505 06:12:45.560 | INFO | "malloc" : " 14 GB", | 20260505 06:12:45.560 | INFO | "pinned" : " 29 GB" | 20260505 06:12:45.560 | INFO | } | 20260505 06:12:45.560 | INFO | Perf report for the last 5.13919 seconds: | 20260505 06:12:45.560 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:12:45.560 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:45.560 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:45.560 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:45.560 | INFO | ANALYSIS : 64.1%, 3.0, 99.8ms, 0 | 20260505 06:12:45.560 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:45.560 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:45.560 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 06:12:45.560 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:45.560 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:45.560 | INFO | Uploaded 36GiB to GPU in 2.15708s (16.6892GiB/s) | 20260505 06:12:45.560 | INFO | Downloaded 34GiB to GPU in 2.03584s (16.7007GiB/s) | 20260505 06:12:50.699 | INFO | Analyzing chunk frames = [964096,964608) | 20260505 06:12:50.699 | INFO | MemInfo: { | 20260505 06:12:50.699 | INFO | "gpu" : " 17 GB", | 20260505 06:12:50.699 | INFO | "huge" : "", | 20260505 06:12:50.699 | INFO | "huge_shared" : " 27 GB", | 20260505 06:12:50.699 | INFO | "malloc" : " 14 GB", | 20260505 06:12:50.699 | INFO | "pinned" : " 29 GB" | 20260505 06:12:50.699 | INFO | } | 20260505 06:12:50.699 | INFO | Perf report for the last 5.13957 seconds: | 20260505 06:12:50.699 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:12:50.699 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:50.699 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:50.699 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:50.699 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 06:12:50.699 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:50.699 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:50.699 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 06:12:50.699 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:50.699 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:50.700 | INFO | Uploaded 36GiB to GPU in 2.13252s (16.8814GiB/s) | 20260505 06:12:50.700 | INFO | Downloaded 34GiB to GPU in 2.05282s (16.5626GiB/s) | 20260505 06:12:55.839 | INFO | Analyzing chunk frames = [964608,965120) | 20260505 06:12:55.839 | INFO | MemInfo: { | 20260505 06:12:55.839 | INFO | "gpu" : " 17 GB", | 20260505 06:12:55.839 | INFO | "huge" : "", | 20260505 06:12:55.839 | INFO | "huge_shared" : " 27 GB", | 20260505 06:12:55.839 | INFO | "malloc" : " 14 GB", | 20260505 06:12:55.839 | INFO | "pinned" : " 29 GB" | 20260505 06:12:55.839 | INFO | } | 20260505 06:12:55.839 | INFO | Perf report for the last 5.13944 seconds: | 20260505 06:12:55.839 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:12:55.839 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:55.839 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:55.839 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:55.839 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 06:12:55.839 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:55.839 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:55.839 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 06:12:55.839 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:55.839 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:12:55.839 | INFO | Uploaded 36GiB to GPU in 2.12176s (16.967GiB/s) | 20260505 06:12:55.839 | INFO | Downloaded 34GiB to GPU in 2.04396s (16.6344GiB/s) | 20260505 06:13:00.979 | INFO | Analyzing chunk frames = [965120,965632) | 20260505 06:13:00.979 | INFO | MemInfo: { | 20260505 06:13:00.979 | INFO | "gpu" : " 17 GB", | 20260505 06:13:00.979 | INFO | "huge" : "", | 20260505 06:13:00.979 | INFO | "huge_shared" : " 27 GB", | 20260505 06:13:00.979 | INFO | "malloc" : " 14 GB", | 20260505 06:13:00.979 | INFO | "pinned" : " 29 GB" | 20260505 06:13:00.979 | INFO | } | 20260505 06:13:00.979 | INFO | Perf report for the last 5.14027 seconds: | 20260505 06:13:00.979 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:13:00.979 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:00.979 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:00.979 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:00.979 | INFO | ANALYSIS : 64.1%, 3.0, 100.0ms, 0 | 20260505 06:13:00.979 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:00.979 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:00.979 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 06:13:00.979 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:00.979 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:00.979 | INFO | Uploaded 36GiB to GPU in 2.16098s (16.6591GiB/s) | 20260505 06:13:00.979 | INFO | Downloaded 34GiB to GPU in 2.04946s (16.5897GiB/s) | 20260505 06:13:06.117 | INFO | Analyzing chunk frames = [965632,966144) | 20260505 06:13:06.117 | INFO | MemInfo: { | 20260505 06:13:06.117 | INFO | "gpu" : " 17 GB", | 20260505 06:13:06.117 | INFO | "huge" : "", | 20260505 06:13:06.117 | INFO | "huge_shared" : " 27 GB", | 20260505 06:13:06.117 | INFO | "malloc" : " 14 GB", | 20260505 06:13:06.117 | INFO | "pinned" : " 29 GB" | 20260505 06:13:06.117 | INFO | } | 20260505 06:13:06.117 | INFO | Perf report for the last 5.13807 seconds: | 20260505 06:13:06.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:13:06.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:06.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:06.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:06.117 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 06:13:06.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:06.117 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:06.117 | INFO | PRE_HQ : 32.7%, 1.0, 17.0ms, 0 | 20260505 06:13:06.117 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:06.117 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:06.117 | INFO | Uploaded 36GiB to GPU in 2.15922s (16.6727GiB/s) | 20260505 06:13:06.117 | INFO | Downloaded 34GiB to GPU in 2.14586s (15.8444GiB/s) | 20260505 06:13:11.257 | INFO | Analyzing chunk frames = [966144,966656) | 20260505 06:13:11.257 | INFO | MemInfo: { | 20260505 06:13:11.257 | INFO | "gpu" : " 17 GB", | 20260505 06:13:11.257 | INFO | "huge" : "", | 20260505 06:13:11.257 | INFO | "huge_shared" : " 27 GB", | 20260505 06:13:11.257 | INFO | "malloc" : " 14 GB", | 20260505 06:13:11.257 | INFO | "pinned" : " 29 GB" | 20260505 06:13:11.257 | INFO | } | 20260505 06:13:11.257 | INFO | Perf report for the last 5.13975 seconds: | 20260505 06:13:11.257 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:13:11.257 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:11.257 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:11.257 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:11.257 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 06:13:11.257 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:11.257 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:11.257 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 06:13:11.257 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:11.257 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:11.257 | INFO | Uploaded 36GiB to GPU in 2.09416s (17.1906GiB/s) | 20260505 06:13:11.257 | INFO | Downloaded 34GiB to GPU in 2.07258s (16.4047GiB/s) | 20260505 06:13:11.257 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 06:13:11.257 | INFO | Total mem usage: 15.3 GiB | 20260505 06:13:11.257 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 06:13:11.257 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.48%) | 20260505 06:13:11.257 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 06:13:11.257 | INFO | High Water Breakdown: | 20260505 06:13:11.257 | INFO | 60.2% : 8.67 GiB ( 92.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 06:13:11.257 | INFO | 39.7% : 5.71 GiB ( 85.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 06:13:11.257 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 06:13:11.257 | INFO | ------------------------------------------ | 20260505 06:13:11.257 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:13:11.257 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 06:13:11.257 | INFO | Total mem usage: 26.5 GiB | 20260505 06:13:11.257 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 06:13:11.257 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 06:13:11.257 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 06:13:11.257 | INFO | High Water Breakdown: | 20260505 06:13:11.258 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:13:11.258 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:13:11.258 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:13:11.258 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:13:11.258 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:13:11.258 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:13:11.258 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 06:13:11.258 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 06:13:11.258 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:13:11.258 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 06:13:11.258 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 06:13:11.258 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 06:13:11.258 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 06:13:11.258 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 06:13:11.258 | INFO | ------------------------------------------ | 20260505 06:13:11.258 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:13:11.258 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 06:13:11.258 | INFO | Total mem usage: 25.6 GiB | 20260505 06:13:11.258 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 06:13:11.258 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 06:13:11.258 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 06:13:11.258 | INFO | High Water Breakdown: | 20260505 06:13:11.258 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 06:13:11.258 | INFO | ------------------------------------------ | 20260505 06:13:11.258 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:13:11.258 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 06:13:11.258 | INFO | Total mem usage: 15.7 GiB | 20260505 06:13:11.258 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 06:13:11.258 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 06:13:11.258 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260505 06:13:11.258 | INFO | High Water Breakdown: | 20260505 06:13:11.258 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 06:13:11.258 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 06:13:11.258 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 06:13:11.258 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 06:13:11.258 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 06:13:11.258 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 06:13:11.258 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 06:13:11.258 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 06:13:11.258 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 06:13:11.258 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:13:11.258 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 06:13:11.258 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:13:11.258 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 06:13:11.258 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 06:13:11.258 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 06:13:11.258 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 06:13:11.258 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:13:11.258 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 06:13:11.258 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:13:11.258 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:13:11.258 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:13:11.258 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 06:13:11.258 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 06:13:11.258 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:13:11.258 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 06:13:11.258 | INFO | ------------------------------------------ | 20260505 06:13:11.258 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:13:11.258 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 06:13:11.258 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:13:11.258 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:13:11.258 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.58e+01, 1.38e+01 / 1.49e+00 | 20260505 06:13:11.258 | INFO | PerfReport: 0.28%, AggregateStats -- 9.42e-02 / 1.04e-01, 1.14e-02 / 5.66e-03 | 20260505 06:13:11.258 | INFO | PerfReport: 11.56%, Baseline -- 3.83e+00 / 4.45e+00, 6.32e-01 / 3.01e-01 | 20260505 06:13:11.258 | INFO | PerfReport: 3.30%, Histogram -- 1.10e+00 / 1.52e+00, 2.20e-01 / 1.13e-01 | 20260505 06:13:11.258 | INFO | PerfReport: 15.47%, DME -- 4.11e+01 / 4.51e+01, 4.40e+00 / 1.17e+00 | 20260505 06:13:11.258 | INFO | PerfReport: 55.93%, FrameLabeling -- 1.85e+01 / 1.92e+01, 2.04e+00 / 8.68e-01 | 20260505 06:13:11.258 | INFO | PerfReport: 7.78%, PulseAccumulating -- 2.58e+00 / 3.16e+00, 4.85e-01 / 3.19e-01 | 20260505 06:13:11.258 | INFO | PerfReport: 4.15%, Metrics -- 1.38e+00 / 1.76e+00, 3.86e-01 / 2.17e-01 | 20260505 06:13:11.259 | INFO | PerfReport: 1.51%, PulseSerializing -- 5.01e-01 / 5.83e-01, 9.90e-02 / 3.67e-02 | 20260505 06:13:11.259 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 06:13:11.259 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:13:11.259 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:13:11.259 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.25e+01 / 0.00e+00 | 20260505 06:13:11.259 | INFO | PerfReport: 22.04%, Upload -- 2.23e+01 / 0.00e+00, 7.72e+00 / 0.00e+00 | 20260505 06:13:11.259 | INFO | PerfReport: 21.56%, Download -- 2.19e+01 / 0.00e+00, 7.74e+00 / 0.00e+00 | 20260505 06:13:11.259 | INFO | PerfReport: 32.71%, Compute -- 3.32e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 06:13:11.259 | INFO | PerfReport: 1.42%, LockUpload -- 1.44e+00 / 0.00e+00, 6.72e+00 / 0.00e+00 | 20260505 06:13:11.259 | INFO | PerfReport: 0.68%, LockDownload -- 6.85e-01 / 0.00e+00, 1.81e+00 / 0.00e+00 | 20260505 06:13:11.259 | INFO | PerfReport: 21.58%, LockCompute -- 2.19e+01 / 0.00e+00, 1.67e+01 / 0.00e+00 | 20260505 06:13:11.259 | INFO | PerfReport: 0.01%, Misc -- 1.46e-02 / 0.00e+00, 1.96e-02 / 0.00e+00 | 20260505 06:13:11.370 | INFO | Starting a round of Baz IO | 20260505 06:13:14.651 | INFO | Populating RTMetrics report | 20260505 06:13:14.651 | INFO | RTMetrics report populated | 20260505 06:13:14.651 | INFO | Opening tmp RTMetrics json file | 20260505 06:13:14.676 | INFO | Serializing RTMetrics to json | 20260505 06:13:14.695 | INFO | Serialization done, swapping tmp file now... | 20260505 06:13:14.700 | INFO | RTMetrics done | 20260505 06:13:14.768 | INFO | Finished a round of Baz IO: Wrote 2.35 GiB in 3.398 seconds | 20260505 06:13:14.768 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.36% / 0.06% / 65.58% | 20260505 06:13:14.768 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.58695% / 0.000139078% / 8.34465e-05% | 20260505 06:13:14.768 | INFO | Baz IO overall: | 20260505 06:13:14.769 | INFO | BytesWritten : 489925340493 | 20260505 06:13:14.769 | INFO | NumEvents : 231357648147 | 20260505 06:13:16.396 | INFO | Analyzing chunk frames = [966656,967168) | 20260505 06:13:16.396 | INFO | MemInfo: { | 20260505 06:13:16.396 | INFO | "gpu" : " 17 GB", | 20260505 06:13:16.396 | INFO | "huge" : "", | 20260505 06:13:16.396 | INFO | "huge_shared" : " 27 GB", | 20260505 06:13:16.396 | INFO | "malloc" : " 14 GB", | 20260505 06:13:16.396 | INFO | "pinned" : " 29 GB" | 20260505 06:13:16.396 | INFO | } | 20260505 06:13:16.396 | INFO | Perf report for the last 5.13909 seconds: | 20260505 06:13:16.396 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:13:16.396 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:16.396 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:16.396 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:16.396 | INFO | ANALYSIS : 64.9%, 2.9, 98.9ms, 0 | 20260505 06:13:16.396 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:16.396 | INFO | RT_METRICS : 47.1%, 1.0, 24.7ms, 0 | 20260505 06:13:16.396 | INFO | PRE_HQ : 64.0%, 2.0, 64.8ms, 0 | 20260505 06:13:16.396 | INFO | BAZWRITER : 65.4%, 4.1, 137.3ms, 0 | 20260505 06:13:16.396 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260505 06:13:16.396 | INFO | Uploaded 36GiB to GPU in 2.19091s (16.4315GiB/s) | 20260505 06:13:16.396 | INFO | Downloaded 38GiB to GPU in 2.33009s (16.3084GiB/s) | 20260505 06:13:21.535 | INFO | Analyzing chunk frames = [967168,967680) | 20260505 06:13:21.535 | INFO | MemInfo: { | 20260505 06:13:21.535 | INFO | "gpu" : " 17 GB", | 20260505 06:13:21.535 | INFO | "huge" : "", | 20260505 06:13:21.535 | INFO | "huge_shared" : " 27 GB", | 20260505 06:13:21.535 | INFO | "malloc" : " 14 GB", | 20260505 06:13:21.535 | INFO | "pinned" : " 29 GB" | 20260505 06:13:21.535 | INFO | } | 20260505 06:13:21.535 | INFO | Perf report for the last 5.13927 seconds: | 20260505 06:13:21.535 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:13:21.535 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:21.535 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:21.535 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:21.535 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 06:13:21.535 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:21.535 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:21.535 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 06:13:21.535 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:21.535 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:21.536 | INFO | Uploaded 36GiB to GPU in 2.127s (16.9253GiB/s) | 20260505 06:13:21.536 | INFO | Downloaded 34GiB to GPU in 2.03825s (16.6809GiB/s) | 20260505 06:13:26.674 | INFO | Analyzing chunk frames = [967680,968192) | 20260505 06:13:26.675 | INFO | MemInfo: { | 20260505 06:13:26.675 | INFO | "gpu" : " 17 GB", | 20260505 06:13:26.675 | INFO | "huge" : "", | 20260505 06:13:26.675 | INFO | "huge_shared" : " 27 GB", | 20260505 06:13:26.675 | INFO | "malloc" : " 14 GB", | 20260505 06:13:26.675 | INFO | "pinned" : " 29 GB" | 20260505 06:13:26.675 | INFO | } | 20260505 06:13:26.675 | INFO | Perf report for the last 5.13939 seconds: | 20260505 06:13:26.675 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:13:26.675 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:26.675 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:26.675 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:26.675 | INFO | ANALYSIS : 64.0%, 3.0, 99.6ms, 0 | 20260505 06:13:26.675 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:26.675 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:26.675 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 06:13:26.675 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:26.675 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:26.675 | INFO | Uploaded 36GiB to GPU in 2.15134s (16.7338GiB/s) | 20260505 06:13:26.675 | INFO | Downloaded 34GiB to GPU in 2.03569s (16.7019GiB/s) | 20260505 06:13:31.814 | INFO | Analyzing chunk frames = [968192,968704) | 20260505 06:13:31.814 | INFO | MemInfo: { | 20260505 06:13:31.814 | INFO | "gpu" : " 17 GB", | 20260505 06:13:31.814 | INFO | "huge" : "", | 20260505 06:13:31.814 | INFO | "huge_shared" : " 27 GB", | 20260505 06:13:31.814 | INFO | "malloc" : " 14 GB", | 20260505 06:13:31.814 | INFO | "pinned" : " 29 GB" | 20260505 06:13:31.814 | INFO | } | 20260505 06:13:31.814 | INFO | Perf report for the last 5.13906 seconds: | 20260505 06:13:31.814 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:13:31.814 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:31.814 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:31.814 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:31.814 | INFO | ANALYSIS : 63.1%, 3.0, 98.1ms, 0 | 20260505 06:13:31.814 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:31.814 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:31.814 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 06:13:31.814 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:31.814 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:31.814 | INFO | Uploaded 36GiB to GPU in 2.13947s (16.8266GiB/s) | 20260505 06:13:31.814 | INFO | Downloaded 34GiB to GPU in 2.05686s (16.5301GiB/s) | 20260505 06:13:36.953 | INFO | Analyzing chunk frames = [968704,969216) | 20260505 06:13:36.953 | INFO | MemInfo: { | 20260505 06:13:36.953 | INFO | "gpu" : " 17 GB", | 20260505 06:13:36.953 | INFO | "huge" : "", | 20260505 06:13:36.953 | INFO | "huge_shared" : " 27 GB", | 20260505 06:13:36.953 | INFO | "malloc" : " 14 GB", | 20260505 06:13:36.953 | INFO | "pinned" : " 29 GB" | 20260505 06:13:36.953 | INFO | } | 20260505 06:13:36.953 | INFO | Perf report for the last 5.1396 seconds: | 20260505 06:13:36.953 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:13:36.953 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:36.953 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:36.953 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:36.953 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 06:13:36.953 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:36.953 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:36.953 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 06:13:36.953 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:36.953 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:36.954 | INFO | Uploaded 36GiB to GPU in 2.11861s (16.9923GiB/s) | 20260505 06:13:36.954 | INFO | Downloaded 34GiB to GPU in 2.04225s (16.6483GiB/s) | 20260505 06:13:42.093 | INFO | Analyzing chunk frames = [969216,969728) | 20260505 06:13:42.093 | INFO | MemInfo: { | 20260505 06:13:42.093 | INFO | "gpu" : " 17 GB", | 20260505 06:13:42.093 | INFO | "huge" : "", | 20260505 06:13:42.093 | INFO | "huge_shared" : " 27 GB", | 20260505 06:13:42.093 | INFO | "malloc" : " 14 GB", | 20260505 06:13:42.093 | INFO | "pinned" : " 29 GB" | 20260505 06:13:42.093 | INFO | } | 20260505 06:13:42.093 | INFO | Perf report for the last 5.14009 seconds: | 20260505 06:13:42.093 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:13:42.094 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:42.094 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:42.094 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:42.094 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 06:13:42.094 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:42.094 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:42.094 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 06:13:42.094 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:42.094 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:42.094 | INFO | Uploaded 36GiB to GPU in 2.1626s (16.6467GiB/s) | 20260505 06:13:42.094 | INFO | Downloaded 34GiB to GPU in 2.04998s (16.5856GiB/s) | 20260505 06:13:47.232 | INFO | Analyzing chunk frames = [969728,970240) | 20260505 06:13:47.232 | INFO | MemInfo: { | 20260505 06:13:47.232 | INFO | "gpu" : " 17 GB", | 20260505 06:13:47.232 | INFO | "huge" : "", | 20260505 06:13:47.232 | INFO | "huge_shared" : " 27 GB", | 20260505 06:13:47.232 | INFO | "malloc" : " 14 GB", | 20260505 06:13:47.232 | INFO | "pinned" : " 29 GB" | 20260505 06:13:47.232 | INFO | } | 20260505 06:13:47.232 | INFO | Perf report for the last 5.13837 seconds: | 20260505 06:13:47.232 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:13:47.232 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:47.232 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:47.232 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:47.232 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 06:13:47.232 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:47.232 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:47.232 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260505 06:13:47.232 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:47.232 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:47.232 | INFO | Uploaded 36GiB to GPU in 2.19035s (16.4357GiB/s) | 20260505 06:13:47.232 | INFO | Downloaded 34GiB to GPU in 2.14763s (15.8314GiB/s) | 20260505 06:13:52.371 | INFO | Analyzing chunk frames = [970240,970752) | 20260505 06:13:52.371 | INFO | MemInfo: { | 20260505 06:13:52.371 | INFO | "gpu" : " 17 GB", | 20260505 06:13:52.371 | INFO | "huge" : "", | 20260505 06:13:52.371 | INFO | "huge_shared" : " 27 GB", | 20260505 06:13:52.371 | INFO | "malloc" : " 14 GB", | 20260505 06:13:52.371 | INFO | "pinned" : " 29 GB" | 20260505 06:13:52.371 | INFO | } | 20260505 06:13:52.371 | INFO | Perf report for the last 5.13952 seconds: | 20260505 06:13:52.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:13:52.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:52.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:52.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:52.371 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 06:13:52.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:52.371 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:52.371 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 06:13:52.371 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:52.371 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:52.372 | INFO | Uploaded 36GiB to GPU in 2.1055s (17.0981GiB/s) | 20260505 06:13:52.372 | INFO | Downloaded 34GiB to GPU in 2.06731s (16.4465GiB/s) | 20260505 06:13:52.485 | INFO | Starting a round of Baz IO | 20260505 06:13:55.749 | INFO | Populating RTMetrics report | 20260505 06:13:55.749 | INFO | RTMetrics report populated | 20260505 06:13:55.749 | INFO | Opening tmp RTMetrics json file | 20260505 06:13:55.806 | INFO | Serializing RTMetrics to json | 20260505 06:13:55.825 | INFO | Serialization done, swapping tmp file now... | 20260505 06:13:55.854 | INFO | RTMetrics done | 20260505 06:13:55.906 | INFO | Finished a round of Baz IO: Wrote 2.35 GiB in 3.42 seconds | 20260505 06:13:55.906 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.31% / 0.05% / 65.64% | 20260505 06:13:55.906 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.55257% / 9.53674e-05% / 6.35783e-05% | 20260505 06:13:55.906 | INFO | Baz IO overall: | 20260505 06:13:55.906 | INFO | BytesWritten : 492448507675 | 20260505 06:13:55.906 | INFO | NumEvents : 232563591291 | 20260505 06:13:57.511 | INFO | Analyzing chunk frames = [970752,971264) | 20260505 06:13:57.511 | INFO | MemInfo: { | 20260505 06:13:57.511 | INFO | "gpu" : " 17 GB", | 20260505 06:13:57.511 | INFO | "huge" : "", | 20260505 06:13:57.511 | INFO | "huge_shared" : " 27 GB", | 20260505 06:13:57.511 | INFO | "malloc" : " 14 GB", | 20260505 06:13:57.511 | INFO | "pinned" : " 29 GB" | 20260505 06:13:57.511 | INFO | } | 20260505 06:13:57.511 | INFO | Perf report for the last 5.13936 seconds: | 20260505 06:13:57.511 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:13:57.511 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:57.511 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:57.511 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:57.511 | INFO | ANALYSIS : 64.6%, 2.9, 98.2ms, 0 | 20260505 06:13:57.511 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:13:57.511 | INFO | RT_METRICS : 48.2%, 1.0, 25.3ms, 0 | 20260505 06:13:57.511 | INFO | PRE_HQ : 63.5%, 2.0, 64.4ms, 0 | 20260505 06:13:57.511 | INFO | BAZWRITER : 65.9%, 4.1, 138.4ms, 0 | 20260505 06:13:57.511 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260505 06:13:57.511 | INFO | Uploaded 36GiB to GPU in 2.14245s (16.8032GiB/s) | 20260505 06:13:57.511 | INFO | Downloaded 38GiB to GPU in 2.31954s (16.3825GiB/s) | 20260505 06:14:02.650 | INFO | Analyzing chunk frames = [971264,971776) | 20260505 06:14:02.650 | INFO | MemInfo: { | 20260505 06:14:02.650 | INFO | "gpu" : " 17 GB", | 20260505 06:14:02.650 | INFO | "huge" : "", | 20260505 06:14:02.650 | INFO | "huge_shared" : " 27 GB", | 20260505 06:14:02.650 | INFO | "malloc" : " 14 GB", | 20260505 06:14:02.650 | INFO | "pinned" : " 29 GB" | 20260505 06:14:02.650 | INFO | } | 20260505 06:14:02.650 | INFO | Perf report for the last 5.13941 seconds: | 20260505 06:14:02.650 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:14:02.650 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:02.650 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:02.650 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:02.650 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 06:14:02.650 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:02.650 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:02.650 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 06:14:02.650 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:02.650 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:02.650 | INFO | Uploaded 36GiB to GPU in 2.12499s (16.9412GiB/s) | 20260505 06:14:02.650 | INFO | Downloaded 34GiB to GPU in 2.03864s (16.6778GiB/s) | 20260505 06:14:07.789 | INFO | Analyzing chunk frames = [971776,972288) | 20260505 06:14:07.789 | INFO | MemInfo: { | 20260505 06:14:07.789 | INFO | "gpu" : " 17 GB", | 20260505 06:14:07.789 | INFO | "huge" : "", | 20260505 06:14:07.789 | INFO | "huge_shared" : " 27 GB", | 20260505 06:14:07.789 | INFO | "malloc" : " 14 GB", | 20260505 06:14:07.789 | INFO | "pinned" : " 29 GB" | 20260505 06:14:07.789 | INFO | } | 20260505 06:14:07.790 | INFO | Perf report for the last 5.13933 seconds: | 20260505 06:14:07.790 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:14:07.790 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:07.790 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:07.790 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:07.790 | INFO | ANALYSIS : 64.1%, 3.0, 99.7ms, 0 | 20260505 06:14:07.790 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:07.790 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:07.790 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 06:14:07.790 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:07.790 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:07.790 | INFO | Uploaded 36GiB to GPU in 2.15807s (16.6816GiB/s) | 20260505 06:14:07.790 | INFO | Downloaded 34GiB to GPU in 2.02977s (16.7507GiB/s) | 20260505 06:14:12.929 | INFO | Analyzing chunk frames = [972288,972800) | 20260505 06:14:12.929 | INFO | MemInfo: { | 20260505 06:14:12.929 | INFO | "gpu" : " 17 GB", | 20260505 06:14:12.929 | INFO | "huge" : "", | 20260505 06:14:12.929 | INFO | "huge_shared" : " 27 GB", | 20260505 06:14:12.929 | INFO | "malloc" : " 14 GB", | 20260505 06:14:12.929 | INFO | "pinned" : " 29 GB" | 20260505 06:14:12.929 | INFO | } | 20260505 06:14:12.929 | INFO | Perf report for the last 5.13924 seconds: | 20260505 06:14:12.929 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:14:12.929 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:12.929 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:12.929 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:12.929 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 06:14:12.929 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:12.929 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:12.929 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260505 06:14:12.929 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:12.929 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:12.929 | INFO | Uploaded 36GiB to GPU in 2.12749s (16.9213GiB/s) | 20260505 06:14:12.929 | INFO | Downloaded 34GiB to GPU in 2.05161s (16.5723GiB/s) | 20260505 06:14:18.068 | INFO | Analyzing chunk frames = [972800,973312) | 20260505 06:14:18.068 | INFO | MemInfo: { | 20260505 06:14:18.068 | INFO | "gpu" : " 17 GB", | 20260505 06:14:18.068 | INFO | "huge" : "", | 20260505 06:14:18.068 | INFO | "huge_shared" : " 27 GB", | 20260505 06:14:18.068 | INFO | "malloc" : " 14 GB", | 20260505 06:14:18.068 | INFO | "pinned" : " 29 GB" | 20260505 06:14:18.068 | INFO | } | 20260505 06:14:18.068 | INFO | Perf report for the last 5.13933 seconds: | 20260505 06:14:18.068 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:14:18.068 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:18.068 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:18.068 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:18.068 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 06:14:18.068 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:18.068 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:18.068 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 06:14:18.068 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:18.068 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:18.068 | INFO | Uploaded 36GiB to GPU in 2.12189s (16.966GiB/s) | 20260505 06:14:18.068 | INFO | Downloaded 34GiB to GPU in 2.04696s (16.61GiB/s) | 20260505 06:14:23.208 | INFO | Analyzing chunk frames = [973312,973824) | 20260505 06:14:23.208 | INFO | MemInfo: { | 20260505 06:14:23.208 | INFO | "gpu" : " 17 GB", | 20260505 06:14:23.208 | INFO | "huge" : "", | 20260505 06:14:23.208 | INFO | "huge_shared" : " 27 GB", | 20260505 06:14:23.208 | INFO | "malloc" : " 14 GB", | 20260505 06:14:23.208 | INFO | "pinned" : " 29 GB" | 20260505 06:14:23.208 | INFO | } | 20260505 06:14:23.208 | INFO | Perf report for the last 5.14012 seconds: | 20260505 06:14:23.208 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:14:23.208 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:23.208 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:23.208 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:23.208 | INFO | ANALYSIS : 64.1%, 3.0, 100.2ms, 0 | 20260505 06:14:23.208 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:23.208 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:23.208 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 06:14:23.208 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:23.208 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:23.208 | INFO | Uploaded 36GiB to GPU in 2.16176s (16.6531GiB/s) | 20260505 06:14:23.208 | INFO | Downloaded 34GiB to GPU in 2.04118s (16.657GiB/s) | 20260505 06:14:28.347 | INFO | Analyzing chunk frames = [973824,974336) | 20260505 06:14:28.347 | INFO | MemInfo: { | 20260505 06:14:28.347 | INFO | "gpu" : " 17 GB", | 20260505 06:14:28.347 | INFO | "huge" : "", | 20260505 06:14:28.347 | INFO | "huge_shared" : " 27 GB", | 20260505 06:14:28.347 | INFO | "malloc" : " 14 GB", | 20260505 06:14:28.347 | INFO | "pinned" : " 29 GB" | 20260505 06:14:28.347 | INFO | } | 20260505 06:14:28.347 | INFO | Perf report for the last 5.13897 seconds: | 20260505 06:14:28.347 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:14:28.347 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:28.347 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:28.347 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:28.347 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 06:14:28.347 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:28.347 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:28.347 | INFO | PRE_HQ : 32.7%, 1.0, 17.0ms, 0 | 20260505 06:14:28.347 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:28.347 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:28.347 | INFO | Uploaded 36GiB to GPU in 2.17378s (16.561GiB/s) | 20260505 06:14:28.347 | INFO | Downloaded 34GiB to GPU in 2.13999s (15.8879GiB/s) | 20260505 06:14:33.486 | INFO | Analyzing chunk frames = [974336,974848) | 20260505 06:14:33.486 | INFO | MemInfo: { | 20260505 06:14:33.486 | INFO | "gpu" : " 17 GB", | 20260505 06:14:33.486 | INFO | "huge" : "", | 20260505 06:14:33.486 | INFO | "huge_shared" : " 27 GB", | 20260505 06:14:33.486 | INFO | "malloc" : " 14 GB", | 20260505 06:14:33.486 | INFO | "pinned" : " 29 GB" | 20260505 06:14:33.486 | INFO | } | 20260505 06:14:33.486 | INFO | Perf report for the last 5.13903 seconds: | 20260505 06:14:33.486 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:14:33.486 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:33.486 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:33.486 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:33.486 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 06:14:33.486 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:33.486 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:33.486 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 06:14:33.486 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:33.486 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:33.486 | INFO | Uploaded 36GiB to GPU in 2.09429s (17.1896GiB/s) | 20260505 06:14:33.486 | INFO | Downloaded 34GiB to GPU in 2.06825s (16.439GiB/s) | 20260505 06:14:33.487 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 06:14:33.487 | INFO | Total mem usage: 15.3 GiB | 20260505 06:14:33.487 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 06:14:33.487 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.11%) | 20260505 06:14:33.487 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 06:14:33.487 | INFO | High Water Breakdown: | 20260505 06:14:33.487 | INFO | 60.2% : 8.69 GiB ( 92.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 06:14:33.487 | INFO | 39.7% : 5.73 GiB ( 86.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 06:14:33.487 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 06:14:33.487 | INFO | ------------------------------------------ | 20260505 06:14:33.487 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:14:33.487 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 06:14:33.487 | INFO | Total mem usage: 26.5 GiB | 20260505 06:14:33.487 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 06:14:33.487 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 06:14:33.487 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 06:14:33.487 | INFO | High Water Breakdown: | 20260505 06:14:33.487 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:14:33.487 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:14:33.487 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:14:33.487 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:14:33.487 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:14:33.487 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:14:33.487 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 06:14:33.487 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 06:14:33.487 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:14:33.487 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 06:14:33.487 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 06:14:33.487 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 06:14:33.487 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 06:14:33.487 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 06:14:33.487 | INFO | ------------------------------------------ | 20260505 06:14:33.487 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:14:33.487 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 06:14:33.487 | INFO | Total mem usage: 25.6 GiB | 20260505 06:14:33.487 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 06:14:33.487 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 06:14:33.487 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 06:14:33.487 | INFO | High Water Breakdown: | 20260505 06:14:33.487 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 06:14:33.487 | INFO | ------------------------------------------ | 20260505 06:14:33.487 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:14:33.487 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 06:14:33.487 | INFO | Total mem usage: 15.7 GiB | 20260505 06:14:33.487 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 06:14:33.487 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 06:14:33.487 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260505 06:14:33.487 | INFO | High Water Breakdown: | 20260505 06:14:33.487 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 06:14:33.487 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 06:14:33.487 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 06:14:33.487 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 06:14:33.487 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 06:14:33.487 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 06:14:33.487 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 06:14:33.487 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 06:14:33.487 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 06:14:33.487 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:14:33.487 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 06:14:33.487 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:14:33.487 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 06:14:33.487 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 06:14:33.487 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 06:14:33.487 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 06:14:33.487 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:14:33.487 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 06:14:33.487 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:14:33.487 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:14:33.487 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:14:33.487 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 06:14:33.487 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 06:14:33.487 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:14:33.487 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 06:14:33.487 | INFO | ------------------------------------------ | 20260505 06:14:33.487 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:14:33.487 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 06:14:33.487 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:14:33.487 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:14:33.487 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.58e+01, 1.38e+01 / 1.41e+00 | 20260505 06:14:33.487 | INFO | PerfReport: 0.28%, AggregateStats -- 9.42e-02 / 1.03e-01, 1.10e-02 / 6.13e-03 | 20260505 06:14:33.487 | INFO | PerfReport: 11.58%, Baseline -- 3.84e+00 / 4.48e+00, 6.40e-01 / 3.00e-01 | 20260505 06:14:33.487 | INFO | PerfReport: 3.31%, Histogram -- 1.10e+00 / 1.50e+00, 2.19e-01 / 1.16e-01 | 20260505 06:14:33.487 | INFO | PerfReport: 15.45%, DME -- 4.10e+01 / 4.51e+01, 4.45e+00 / 1.03e+00 | 20260505 06:14:33.488 | INFO | PerfReport: 55.97%, FrameLabeling -- 1.86e+01 / 1.91e+01, 2.07e+00 / 1.04e+00 | 20260505 06:14:33.488 | INFO | PerfReport: 7.76%, PulseAccumulating -- 2.58e+00 / 3.21e+00, 4.82e-01 / 2.75e-01 | 20260505 06:14:33.488 | INFO | PerfReport: 4.14%, Metrics -- 1.37e+00 / 1.76e+00, 3.89e-01 / 1.99e-01 | 20260505 06:14:33.488 | INFO | PerfReport: 1.51%, PulseSerializing -- 5.00e-01 / 5.86e-01, 9.88e-02 / 3.93e-02 | 20260505 06:14:33.488 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 06:14:33.488 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:14:33.488 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:14:33.488 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.26e+01 / 0.00e+00 | 20260505 06:14:33.488 | INFO | PerfReport: 22.01%, Upload -- 2.23e+01 / 0.00e+00, 7.73e+00 / 0.00e+00 | 20260505 06:14:33.488 | INFO | PerfReport: 21.51%, Download -- 2.18e+01 / 0.00e+00, 7.75e+00 / 0.00e+00 | 20260505 06:14:33.488 | INFO | PerfReport: 32.72%, Compute -- 3.32e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 06:14:33.488 | INFO | PerfReport: 1.42%, LockUpload -- 1.44e+00 / 0.00e+00, 6.77e+00 / 0.00e+00 | 20260505 06:14:33.488 | INFO | PerfReport: 0.71%, LockDownload -- 7.15e-01 / 0.00e+00, 1.97e+00 / 0.00e+00 | 20260505 06:14:33.488 | INFO | PerfReport: 21.62%, LockCompute -- 2.19e+01 / 0.00e+00, 1.67e+01 / 0.00e+00 | 20260505 06:14:33.488 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.08e-02 / 0.00e+00 | 20260505 06:14:33.600 | INFO | Starting a round of Baz IO | 20260505 06:14:36.877 | INFO | Populating RTMetrics report | 20260505 06:14:36.877 | INFO | RTMetrics report populated | 20260505 06:14:36.877 | INFO | Opening tmp RTMetrics json file | 20260505 06:14:36.878 | INFO | Serializing RTMetrics to json | 20260505 06:14:36.879 | INFO | Serialization done, swapping tmp file now... | 20260505 06:14:36.880 | INFO | RTMetrics done | 20260505 06:14:37.000 | INFO | Finished a round of Baz IO: Wrote 2.35 GiB in 3.4 seconds | 20260505 06:14:37.001 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.25% / 0.05% / 65.69% | 20260505 06:14:37.001 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.52302% / 7.15256e-05% / 8.34465e-05% | 20260505 06:14:37.001 | INFO | Baz IO overall: | 20260505 06:14:37.001 | INFO | BytesWritten : 494972053991 | 20260505 06:14:37.001 | INFO | NumEvents : 233769449521 | 20260505 06:14:38.625 | INFO | Analyzing chunk frames = [974848,975360) | 20260505 06:14:38.625 | INFO | MemInfo: { | 20260505 06:14:38.625 | INFO | "gpu" : " 17 GB", | 20260505 06:14:38.625 | INFO | "huge" : "", | 20260505 06:14:38.625 | INFO | "huge_shared" : " 27 GB", | 20260505 06:14:38.625 | INFO | "malloc" : " 14 GB", | 20260505 06:14:38.625 | INFO | "pinned" : " 29 GB" | 20260505 06:14:38.625 | INFO | } | 20260505 06:14:38.626 | INFO | Perf report for the last 5.13922 seconds: | 20260505 06:14:38.626 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:14:38.626 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:38.626 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:38.626 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:38.626 | INFO | ANALYSIS : 64.8%, 2.9, 98.6ms, 0 | 20260505 06:14:38.626 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:38.626 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 06:14:38.626 | INFO | PRE_HQ : 63.8%, 1.9, 64.3ms, 0 | 20260505 06:14:38.626 | INFO | BAZWRITER : 65.5%, 4.1, 137.5ms, 0 | 20260505 06:14:38.626 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 06:14:38.626 | INFO | Uploaded 36GiB to GPU in 2.22384s (16.1882GiB/s) | 20260505 06:14:38.626 | INFO | Downloaded 38GiB to GPU in 2.32665s (16.3325GiB/s) | 20260505 06:14:43.765 | INFO | Analyzing chunk frames = [975360,975872) | 20260505 06:14:43.765 | INFO | MemInfo: { | 20260505 06:14:43.765 | INFO | "gpu" : " 17 GB", | 20260505 06:14:43.765 | INFO | "huge" : "", | 20260505 06:14:43.765 | INFO | "huge_shared" : " 27 GB", | 20260505 06:14:43.765 | INFO | "malloc" : " 14 GB", | 20260505 06:14:43.765 | INFO | "pinned" : " 29 GB" | 20260505 06:14:43.765 | INFO | } | 20260505 06:14:43.765 | INFO | Perf report for the last 5.1394 seconds: | 20260505 06:14:43.765 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:14:43.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:43.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:43.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:43.765 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 06:14:43.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:43.765 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:43.765 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 06:14:43.765 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:43.765 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:43.765 | INFO | Uploaded 36GiB to GPU in 2.12426s (16.947GiB/s) | 20260505 06:14:43.765 | INFO | Downloaded 34GiB to GPU in 2.03851s (16.6788GiB/s) | 20260505 06:14:48.904 | INFO | Analyzing chunk frames = [975872,976384) | 20260505 06:14:48.904 | INFO | MemInfo: { | 20260505 06:14:48.904 | INFO | "gpu" : " 17 GB", | 20260505 06:14:48.904 | INFO | "huge" : "", | 20260505 06:14:48.904 | INFO | "huge_shared" : " 27 GB", | 20260505 06:14:48.904 | INFO | "malloc" : " 14 GB", | 20260505 06:14:48.904 | INFO | "pinned" : " 29 GB" | 20260505 06:14:48.904 | INFO | } | 20260505 06:14:48.904 | INFO | Perf report for the last 5.13928 seconds: | 20260505 06:14:48.904 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:14:48.904 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:48.904 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:48.904 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:48.904 | INFO | ANALYSIS : 64.0%, 3.0, 99.7ms, 0 | 20260505 06:14:48.904 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:48.904 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:48.904 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 06:14:48.904 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:48.904 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:48.904 | INFO | Uploaded 36GiB to GPU in 2.15949s (16.6706GiB/s) | 20260505 06:14:48.904 | INFO | Downloaded 34GiB to GPU in 2.03256s (16.7277GiB/s) | 20260505 06:14:54.043 | INFO | Analyzing chunk frames = [976384,976896) | 20260505 06:14:54.043 | INFO | MemInfo: { | 20260505 06:14:54.043 | INFO | "gpu" : " 17 GB", | 20260505 06:14:54.043 | INFO | "huge" : "", | 20260505 06:14:54.043 | INFO | "huge_shared" : " 27 GB", | 20260505 06:14:54.043 | INFO | "malloc" : " 14 GB", | 20260505 06:14:54.043 | INFO | "pinned" : " 29 GB" | 20260505 06:14:54.043 | INFO | } | 20260505 06:14:54.043 | INFO | Perf report for the last 5.13909 seconds: | 20260505 06:14:54.043 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:14:54.043 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:54.043 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:54.043 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:54.043 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 06:14:54.043 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:54.043 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:54.043 | INFO | PRE_HQ : 30.5%, 1.0, 15.9ms, 0 | 20260505 06:14:54.043 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:54.043 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:54.043 | INFO | Uploaded 36GiB to GPU in 2.13603s (16.8537GiB/s) | 20260505 06:14:54.044 | INFO | Downloaded 34GiB to GPU in 2.05034s (16.5826GiB/s) | 20260505 06:14:59.183 | INFO | Analyzing chunk frames = [976896,977408) | 20260505 06:14:59.183 | INFO | MemInfo: { | 20260505 06:14:59.183 | INFO | "gpu" : " 17 GB", | 20260505 06:14:59.183 | INFO | "huge" : "", | 20260505 06:14:59.183 | INFO | "huge_shared" : " 27 GB", | 20260505 06:14:59.183 | INFO | "malloc" : " 14 GB", | 20260505 06:14:59.183 | INFO | "pinned" : " 29 GB" | 20260505 06:14:59.183 | INFO | } | 20260505 06:14:59.183 | INFO | Perf report for the last 5.13967 seconds: | 20260505 06:14:59.183 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:14:59.183 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:59.183 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:59.183 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:59.183 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 06:14:59.183 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:59.183 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:59.183 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 06:14:59.183 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:59.183 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:14:59.183 | INFO | Uploaded 36GiB to GPU in 2.12161s (16.9683GiB/s) | 20260505 06:14:59.183 | INFO | Downloaded 34GiB to GPU in 2.04374s (16.6361GiB/s) | 20260505 06:15:04.324 | INFO | Analyzing chunk frames = [977408,977920) | 20260505 06:15:04.324 | INFO | MemInfo: { | 20260505 06:15:04.324 | INFO | "gpu" : " 17 GB", | 20260505 06:15:04.324 | INFO | "huge" : "", | 20260505 06:15:04.324 | INFO | "huge_shared" : " 27 GB", | 20260505 06:15:04.324 | INFO | "malloc" : " 14 GB", | 20260505 06:15:04.324 | INFO | "pinned" : " 29 GB" | 20260505 06:15:04.324 | INFO | } | 20260505 06:15:04.324 | INFO | Perf report for the last 5.14113 seconds: | 20260505 06:15:04.324 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:15:04.324 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:04.324 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:04.324 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:04.324 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 06:15:04.324 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:04.324 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:04.324 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 06:15:04.324 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:04.324 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:04.324 | INFO | Uploaded 36GiB to GPU in 2.16375s (16.6378GiB/s) | 20260505 06:15:04.324 | INFO | Downloaded 34GiB to GPU in 2.04801s (16.6015GiB/s) | 20260505 06:15:09.462 | INFO | Analyzing chunk frames = [977920,978432) | 20260505 06:15:09.462 | INFO | MemInfo: { | 20260505 06:15:09.462 | INFO | "gpu" : " 17 GB", | 20260505 06:15:09.462 | INFO | "huge" : "", | 20260505 06:15:09.462 | INFO | "huge_shared" : " 27 GB", | 20260505 06:15:09.462 | INFO | "malloc" : " 14 GB", | 20260505 06:15:09.462 | INFO | "pinned" : " 29 GB" | 20260505 06:15:09.462 | INFO | } | 20260505 06:15:09.462 | INFO | Perf report for the last 5.13764 seconds: | 20260505 06:15:09.462 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:15:09.462 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:09.462 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:09.462 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:09.462 | INFO | ANALYSIS : 64.3%, 3.0, 100.3ms, 0 | 20260505 06:15:09.462 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:09.462 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:09.462 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260505 06:15:09.462 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:09.462 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:09.462 | INFO | Uploaded 36GiB to GPU in 2.1814s (16.5032GiB/s) | 20260505 06:15:09.462 | INFO | Downloaded 34GiB to GPU in 2.15461s (15.7801GiB/s) | 20260505 06:15:14.601 | INFO | Analyzing chunk frames = [978432,978944) | 20260505 06:15:14.601 | INFO | MemInfo: { | 20260505 06:15:14.601 | INFO | "gpu" : " 17 GB", | 20260505 06:15:14.601 | INFO | "huge" : "", | 20260505 06:15:14.601 | INFO | "huge_shared" : " 27 GB", | 20260505 06:15:14.601 | INFO | "malloc" : " 14 GB", | 20260505 06:15:14.601 | INFO | "pinned" : " 29 GB" | 20260505 06:15:14.601 | INFO | } | 20260505 06:15:14.601 | INFO | Perf report for the last 5.13945 seconds: | 20260505 06:15:14.601 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:15:14.601 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:14.601 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:14.601 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:14.601 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 06:15:14.601 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:14.601 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:14.601 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 06:15:14.601 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:14.601 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:14.601 | INFO | Uploaded 36GiB to GPU in 2.08829s (17.239GiB/s) | 20260505 06:15:14.602 | INFO | Downloaded 34GiB to GPU in 2.0709s (16.418GiB/s) | 20260505 06:15:14.714 | INFO | Starting a round of Baz IO | 20260505 06:15:17.978 | INFO | Populating RTMetrics report | 20260505 06:15:17.979 | INFO | RTMetrics report populated | 20260505 06:15:17.979 | INFO | Opening tmp RTMetrics json file | 20260505 06:15:18.002 | INFO | Serializing RTMetrics to json | 20260505 06:15:18.003 | INFO | Serialization done, swapping tmp file now... | 20260505 06:15:18.003 | INFO | RTMetrics done | 20260505 06:15:18.106 | INFO | Finished a round of Baz IO: Wrote 2.353 GiB in 3.392 seconds | 20260505 06:15:18.107 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.20% / 0.05% / 65.75% | 20260505 06:15:18.107 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.50737% / 8.74201e-05% / 5.96046e-05% | 20260505 06:15:18.107 | INFO | Baz IO overall: | 20260505 06:15:18.107 | INFO | BytesWritten : 497498292575 | 20260505 06:15:18.107 | INFO | NumEvents : 234976560758 | 20260505 06:15:19.740 | INFO | Analyzing chunk frames = [978944,979456) | 20260505 06:15:19.740 | INFO | MemInfo: { | 20260505 06:15:19.740 | INFO | "gpu" : " 17 GB", | 20260505 06:15:19.740 | INFO | "huge" : "", | 20260505 06:15:19.740 | INFO | "huge_shared" : " 27 GB", | 20260505 06:15:19.740 | INFO | "malloc" : " 14 GB", | 20260505 06:15:19.740 | INFO | "pinned" : " 29 GB" | 20260505 06:15:19.740 | INFO | } | 20260505 06:15:19.740 | INFO | Perf report for the last 5.13899 seconds: | 20260505 06:15:19.740 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:15:19.740 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:19.740 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:19.740 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:19.740 | INFO | ANALYSIS : 64.6%, 2.9, 98.7ms, 0 | 20260505 06:15:19.740 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:19.740 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260505 06:15:19.740 | INFO | PRE_HQ : 63.7%, 1.9, 64.4ms, 0 | 20260505 06:15:19.740 | INFO | BAZWRITER : 65.3%, 4.1, 136.2ms, 0 | 20260505 06:15:19.740 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260505 06:15:19.740 | INFO | Uploaded 36GiB to GPU in 2.18601s (16.4684GiB/s) | 20260505 06:15:19.741 | INFO | Downloaded 38GiB to GPU in 2.33704s (16.2599GiB/s) | 20260505 06:15:24.880 | INFO | Analyzing chunk frames = [979456,979968) | 20260505 06:15:24.880 | INFO | MemInfo: { | 20260505 06:15:24.880 | INFO | "gpu" : " 17 GB", | 20260505 06:15:24.880 | INFO | "huge" : "", | 20260505 06:15:24.880 | INFO | "huge_shared" : " 27 GB", | 20260505 06:15:24.880 | INFO | "malloc" : " 14 GB", | 20260505 06:15:24.880 | INFO | "pinned" : " 29 GB" | 20260505 06:15:24.880 | INFO | } | 20260505 06:15:24.880 | INFO | Perf report for the last 5.13957 seconds: | 20260505 06:15:24.880 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:15:24.880 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:24.880 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:24.880 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:24.880 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 06:15:24.880 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:24.880 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:24.880 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 06:15:24.880 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:24.880 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:24.880 | INFO | Uploaded 36GiB to GPU in 2.12338s (16.9541GiB/s) | 20260505 06:15:24.880 | INFO | Downloaded 34GiB to GPU in 2.04348s (16.6383GiB/s) | 20260505 06:15:30.019 | INFO | Analyzing chunk frames = [979968,980480) | 20260505 06:15:30.019 | INFO | MemInfo: { | 20260505 06:15:30.019 | INFO | "gpu" : " 17 GB", | 20260505 06:15:30.019 | INFO | "huge" : "", | 20260505 06:15:30.019 | INFO | "huge_shared" : " 27 GB", | 20260505 06:15:30.019 | INFO | "malloc" : " 14 GB", | 20260505 06:15:30.019 | INFO | "pinned" : " 29 GB" | 20260505 06:15:30.019 | INFO | } | 20260505 06:15:30.019 | INFO | Perf report for the last 5.13915 seconds: | 20260505 06:15:30.019 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:15:30.019 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:30.019 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:30.019 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:30.019 | INFO | ANALYSIS : 64.1%, 3.0, 99.8ms, 0 | 20260505 06:15:30.019 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:30.019 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:30.019 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 06:15:30.019 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:30.019 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:30.019 | INFO | Uploaded 36GiB to GPU in 2.1597s (16.669GiB/s) | 20260505 06:15:30.019 | INFO | Downloaded 34GiB to GPU in 2.03961s (16.6699GiB/s) | 20260505 06:15:35.158 | INFO | Analyzing chunk frames = [980480,980992) | 20260505 06:15:35.158 | INFO | MemInfo: { | 20260505 06:15:35.158 | INFO | "gpu" : " 17 GB", | 20260505 06:15:35.158 | INFO | "huge" : "", | 20260505 06:15:35.158 | INFO | "huge_shared" : " 27 GB", | 20260505 06:15:35.158 | INFO | "malloc" : " 14 GB", | 20260505 06:15:35.158 | INFO | "pinned" : " 29 GB" | 20260505 06:15:35.158 | INFO | } | 20260505 06:15:35.158 | INFO | Perf report for the last 5.13926 seconds: | 20260505 06:15:35.158 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:15:35.158 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:35.158 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:35.158 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:35.158 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 06:15:35.158 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:35.158 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:35.158 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 06:15:35.158 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:35.158 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:35.158 | INFO | Uploaded 36GiB to GPU in 2.12829s (16.915GiB/s) | 20260505 06:15:35.159 | INFO | Downloaded 34GiB to GPU in 2.04698s (16.6099GiB/s) | 20260505 06:15:40.298 | INFO | Analyzing chunk frames = [980992,981504) | 20260505 06:15:40.298 | INFO | MemInfo: { | 20260505 06:15:40.298 | INFO | "gpu" : " 17 GB", | 20260505 06:15:40.298 | INFO | "huge" : "", | 20260505 06:15:40.298 | INFO | "huge_shared" : " 27 GB", | 20260505 06:15:40.298 | INFO | "malloc" : " 14 GB", | 20260505 06:15:40.298 | INFO | "pinned" : " 29 GB" | 20260505 06:15:40.298 | INFO | } | 20260505 06:15:40.298 | INFO | Perf report for the last 5.13965 seconds: | 20260505 06:15:40.298 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:15:40.298 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:40.298 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:40.298 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:40.298 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 06:15:40.298 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:40.298 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:40.298 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 06:15:40.298 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:40.298 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:40.298 | INFO | Uploaded 36GiB to GPU in 2.12462s (16.9442GiB/s) | 20260505 06:15:40.299 | INFO | Downloaded 34GiB to GPU in 2.05287s (16.5622GiB/s) | 20260505 06:15:45.438 | INFO | Analyzing chunk frames = [981504,982016) | 20260505 06:15:45.438 | INFO | MemInfo: { | 20260505 06:15:45.438 | INFO | "gpu" : " 17 GB", | 20260505 06:15:45.439 | INFO | "huge" : "", | 20260505 06:15:45.439 | INFO | "huge_shared" : " 27 GB", | 20260505 06:15:45.439 | INFO | "malloc" : " 14 GB", | 20260505 06:15:45.439 | INFO | "pinned" : " 29 GB" | 20260505 06:15:45.439 | INFO | } | 20260505 06:15:45.439 | INFO | Perf report for the last 5.14066 seconds: | 20260505 06:15:45.439 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:15:45.439 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:45.439 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:45.439 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:45.439 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 06:15:45.439 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:45.439 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:45.439 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260505 06:15:45.439 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:45.439 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:45.439 | INFO | Uploaded 36GiB to GPU in 2.15482s (16.7067GiB/s) | 20260505 06:15:45.439 | INFO | Downloaded 34GiB to GPU in 2.0504s (16.5821GiB/s) | 20260505 06:15:50.580 | INFO | Analyzing chunk frames = [982016,982528) | 20260505 06:15:50.580 | INFO | MemInfo: { | 20260505 06:15:50.580 | INFO | "gpu" : " 17 GB", | 20260505 06:15:50.580 | INFO | "huge" : "", | 20260505 06:15:50.580 | INFO | "huge_shared" : " 27 GB", | 20260505 06:15:50.580 | INFO | "malloc" : " 14 GB", | 20260505 06:15:50.580 | INFO | "pinned" : " 29 GB" | 20260505 06:15:50.580 | INFO | } | 20260505 06:15:50.580 | INFO | Perf report for the last 5.14112 seconds: | 20260505 06:15:50.580 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:15:50.580 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:50.580 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:50.580 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:50.580 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 06:15:50.580 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:50.580 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:50.580 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 06:15:50.580 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:50.580 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:50.580 | INFO | Uploaded 36GiB to GPU in 2.1595s (16.6705GiB/s) | 20260505 06:15:50.580 | INFO | Downloaded 34GiB to GPU in 2.14155s (15.8763GiB/s) | 20260505 06:15:55.717 | INFO | Analyzing chunk frames = [982528,983040) | 20260505 06:15:55.717 | INFO | MemInfo: { | 20260505 06:15:55.717 | INFO | "gpu" : " 17 GB", | 20260505 06:15:55.717 | INFO | "huge" : "", | 20260505 06:15:55.717 | INFO | "huge_shared" : " 27 GB", | 20260505 06:15:55.717 | INFO | "malloc" : " 14 GB", | 20260505 06:15:55.717 | INFO | "pinned" : " 29 GB" | 20260505 06:15:55.717 | INFO | } | 20260505 06:15:55.717 | INFO | Perf report for the last 5.13757 seconds: | 20260505 06:15:55.717 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:15:55.717 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:55.717 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:55.717 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:55.717 | INFO | ANALYSIS : 63.8%, 3.0, 99.5ms, 0 | 20260505 06:15:55.717 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:55.717 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:55.717 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 06:15:55.717 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:55.717 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:15:55.718 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 06:15:55.718 | INFO | Total mem usage: 15.3 GiB | 20260505 06:15:55.718 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 06:15:55.718 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.49%) | 20260505 06:15:55.718 | INFO | Uploaded 36GiB to GPU in 2.08975s (17.2269GiB/s) | 20260505 06:15:55.718 | INFO | Downloaded 34GiB to GPU in 2.06134s (16.4941GiB/s) | 20260505 06:15:55.718 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 06:15:55.718 | INFO | High Water Breakdown: | 20260505 06:15:55.718 | INFO | 60.2% : 8.67 GiB ( 92.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 06:15:55.718 | INFO | 39.6% : 5.71 GiB ( 85.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 06:15:55.718 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 06:15:55.718 | INFO | ------------------------------------------ | 20260505 06:15:55.718 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:15:55.718 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 06:15:55.718 | INFO | Total mem usage: 26.5 GiB | 20260505 06:15:55.718 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 06:15:55.718 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 06:15:55.718 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 06:15:55.718 | INFO | High Water Breakdown: | 20260505 06:15:55.718 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:15:55.718 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:15:55.718 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:15:55.718 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:15:55.718 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:15:55.718 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:15:55.718 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 06:15:55.718 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 06:15:55.718 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:15:55.718 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 06:15:55.718 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 06:15:55.718 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 06:15:55.718 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 06:15:55.718 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 06:15:55.718 | INFO | ------------------------------------------ | 20260505 06:15:55.718 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:15:55.718 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 06:15:55.718 | INFO | Total mem usage: 25.6 GiB | 20260505 06:15:55.718 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 06:15:55.718 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 06:15:55.718 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 06:15:55.718 | INFO | High Water Breakdown: | 20260505 06:15:55.718 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 06:15:55.718 | INFO | ------------------------------------------ | 20260505 06:15:55.719 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:15:55.719 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 06:15:55.719 | INFO | Total mem usage: 15.7 GiB | 20260505 06:15:55.719 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 06:15:55.719 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 06:15:55.719 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260505 06:15:55.719 | INFO | High Water Breakdown: | 20260505 06:15:55.719 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 06:15:55.719 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 06:15:55.719 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 06:15:55.719 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 06:15:55.719 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 06:15:55.719 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 06:15:55.719 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 06:15:55.719 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 06:15:55.719 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 06:15:55.719 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:15:55.719 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 06:15:55.719 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:15:55.719 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 06:15:55.719 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 06:15:55.719 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 06:15:55.719 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 06:15:55.719 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:15:55.719 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 06:15:55.719 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:15:55.719 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:15:55.719 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:15:55.719 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 06:15:55.719 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 06:15:55.719 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:15:55.719 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 06:15:55.719 | INFO | ------------------------------------------ | 20260505 06:15:55.719 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:15:55.719 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 06:15:55.719 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:15:55.719 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:15:55.719 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.56e+01, 1.39e+01 / 1.69e+00 | 20260505 06:15:55.719 | INFO | PerfReport: 0.28%, AggregateStats -- 9.37e-02 / 1.04e-01, 1.06e-02 / 6.30e-03 | 20260505 06:15:55.719 | INFO | PerfReport: 11.57%, Baseline -- 3.84e+00 / 4.46e+00, 6.33e-01 / 3.00e-01 | 20260505 06:15:55.719 | INFO | PerfReport: 3.30%, Histogram -- 1.09e+00 / 1.50e+00, 2.18e-01 / 1.19e-01 | 20260505 06:15:55.719 | INFO | PerfReport: 15.46%, DME -- 4.10e+01 / 4.50e+01, 4.24e+00 / 1.20e+00 | 20260505 06:15:55.719 | INFO | PerfReport: 55.98%, FrameLabeling -- 1.86e+01 / 1.91e+01, 2.06e+00 / 1.03e+00 | 20260505 06:15:55.719 | INFO | PerfReport: 7.76%, PulseAccumulating -- 2.57e+00 / 3.16e+00, 4.77e-01 / 2.80e-01 | 20260505 06:15:55.719 | INFO | PerfReport: 4.14%, Metrics -- 1.37e+00 / 1.78e+00, 3.87e-01 / 2.24e-01 | 20260505 06:15:55.719 | INFO | PerfReport: 1.51%, PulseSerializing -- 5.00e-01 / 5.80e-01, 9.89e-02 / 3.69e-02 | 20260505 06:15:55.719 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 06:15:55.719 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:15:55.719 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:15:55.719 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.27e+01 / 0.00e+00 | 20260505 06:15:55.719 | INFO | PerfReport: 22.03%, Upload -- 2.23e+01 / 0.00e+00, 7.71e+00 / 0.00e+00 | 20260505 06:15:55.719 | INFO | PerfReport: 21.53%, Download -- 2.18e+01 / 0.00e+00, 7.70e+00 / 0.00e+00 | 20260505 06:15:55.719 | INFO | PerfReport: 32.71%, Compute -- 3.32e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 06:15:55.719 | INFO | PerfReport: 1.43%, LockUpload -- 1.45e+00 / 0.00e+00, 6.77e+00 / 0.00e+00 | 20260505 06:15:55.719 | INFO | PerfReport: 0.69%, LockDownload -- 7.02e-01 / 0.00e+00, 1.88e+00 / 0.00e+00 | 20260505 06:15:55.719 | INFO | PerfReport: 21.59%, LockCompute -- 2.19e+01 / 0.00e+00, 1.67e+01 / 0.00e+00 | 20260505 06:15:55.719 | INFO | PerfReport: 0.01%, Misc -- 1.52e-02 / 0.00e+00, 2.31e-02 / 0.00e+00 | 20260505 06:15:55.836 | INFO | Starting a round of Baz IO | 20260505 06:15:59.140 | INFO | Populating RTMetrics report | 20260505 06:15:59.141 | INFO | RTMetrics report populated | 20260505 06:15:59.141 | INFO | Opening tmp RTMetrics json file | 20260505 06:15:59.186 | INFO | Serializing RTMetrics to json | 20260505 06:15:59.188 | INFO | Serialization done, swapping tmp file now... | 20260505 06:15:59.189 | INFO | RTMetrics done | 20260505 06:15:59.293 | INFO | Finished a round of Baz IO: Wrote 2.357 GiB in 3.457 seconds | 20260505 06:15:59.294 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.15% / 0.05% / 65.80% | 20260505 06:15:59.294 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.50149% / 6.35783e-05% / 3.97364e-05% | 20260505 06:15:59.294 | INFO | Baz IO overall: | 20260505 06:15:59.294 | INFO | BytesWritten : 500029400847 | 20260505 06:15:59.294 | INFO | NumEvents : 236186809473 | 20260505 06:16:00.855 | INFO | Analyzing chunk frames = [983040,983552) | 20260505 06:16:00.855 | INFO | MemInfo: { | 20260505 06:16:00.855 | INFO | "gpu" : " 17 GB", | 20260505 06:16:00.855 | INFO | "huge" : "", | 20260505 06:16:00.855 | INFO | "huge_shared" : " 27 GB", | 20260505 06:16:00.855 | INFO | "malloc" : " 14 GB", | 20260505 06:16:00.855 | INFO | "pinned" : " 29 GB" | 20260505 06:16:00.855 | INFO | } | 20260505 06:16:00.855 | INFO | Perf report for the last 5.13766 seconds: | 20260505 06:16:00.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:16:00.855 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:00.855 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:00.855 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:00.855 | INFO | ANALYSIS : 65.3%, 2.8, 97.1ms, 0 | 20260505 06:16:00.855 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:00.855 | INFO | RT_METRICS : 53.0%, 1.0, 27.8ms, 0 | 20260505 06:16:00.855 | INFO | PRE_HQ : 64.6%, 2.1, 69.6ms, 0 | 20260505 06:16:00.855 | INFO | BAZWRITER : 66.6%, 4.3, 146.1ms, 0 | 20260505 06:16:00.855 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 06:16:00.855 | INFO | Uploaded 36GiB to GPU in 2.12968s (16.904GiB/s) | 20260505 06:16:00.855 | INFO | Downloaded 38GiB to GPU in 2.31732s (16.3982GiB/s) | 20260505 06:16:05.994 | INFO | Analyzing chunk frames = [983552,984064) | 20260505 06:16:05.994 | INFO | MemInfo: { | 20260505 06:16:05.994 | INFO | "gpu" : " 17 GB", | 20260505 06:16:05.994 | INFO | "huge" : "", | 20260505 06:16:05.994 | INFO | "huge_shared" : " 27 GB", | 20260505 06:16:05.994 | INFO | "malloc" : " 14 GB", | 20260505 06:16:05.994 | INFO | "pinned" : " 29 GB" | 20260505 06:16:05.994 | INFO | } | 20260505 06:16:05.994 | INFO | Perf report for the last 5.13934 seconds: | 20260505 06:16:05.994 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:16:05.994 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:05.994 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:05.994 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:05.994 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 06:16:05.994 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:05.994 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:05.994 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 06:16:05.994 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:05.994 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:05.995 | INFO | Uploaded 36GiB to GPU in 2.12553s (16.937GiB/s) | 20260505 06:16:05.995 | INFO | Downloaded 34GiB to GPU in 2.04575s (16.6198GiB/s) | 20260505 06:16:11.134 | INFO | Analyzing chunk frames = [984064,984576) | 20260505 06:16:11.134 | INFO | MemInfo: { | 20260505 06:16:11.134 | INFO | "gpu" : " 17 GB", | 20260505 06:16:11.134 | INFO | "huge" : "", | 20260505 06:16:11.134 | INFO | "huge_shared" : " 27 GB", | 20260505 06:16:11.134 | INFO | "malloc" : " 14 GB", | 20260505 06:16:11.134 | INFO | "pinned" : " 29 GB" | 20260505 06:16:11.134 | INFO | } | 20260505 06:16:11.134 | INFO | Perf report for the last 5.13934 seconds: | 20260505 06:16:11.134 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:16:11.134 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:11.134 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:11.134 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:11.134 | INFO | ANALYSIS : 64.1%, 3.0, 99.7ms, 0 | 20260505 06:16:11.134 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:11.134 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:11.134 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 06:16:11.134 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:11.134 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:11.134 | INFO | Uploaded 36GiB to GPU in 2.16163s (16.6541GiB/s) | 20260505 06:16:11.134 | INFO | Downloaded 34GiB to GPU in 2.03761s (16.6862GiB/s) | 20260505 06:16:16.273 | INFO | Analyzing chunk frames = [984576,985088) | 20260505 06:16:16.273 | INFO | MemInfo: { | 20260505 06:16:16.273 | INFO | "gpu" : " 17 GB", | 20260505 06:16:16.274 | INFO | "huge" : "", | 20260505 06:16:16.274 | INFO | "huge_shared" : " 27 GB", | 20260505 06:16:16.274 | INFO | "malloc" : " 14 GB", | 20260505 06:16:16.274 | INFO | "pinned" : " 29 GB" | 20260505 06:16:16.274 | INFO | } | 20260505 06:16:16.274 | INFO | Perf report for the last 5.13999 seconds: | 20260505 06:16:16.274 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:16:16.274 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:16.274 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:16.274 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:16.274 | INFO | ANALYSIS : 64.0%, 3.0, 99.4ms, 0 | 20260505 06:16:16.274 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:16.274 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:16.274 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 06:16:16.274 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:16.274 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:16.274 | INFO | Uploaded 36GiB to GPU in 2.17733s (16.534GiB/s) | 20260505 06:16:16.274 | INFO | Downloaded 34GiB to GPU in 2.04839s (16.5984GiB/s) | 20260505 06:16:21.416 | INFO | Analyzing chunk frames = [985088,985600) | 20260505 06:16:21.416 | INFO | MemInfo: { | 20260505 06:16:21.416 | INFO | "gpu" : " 17 GB", | 20260505 06:16:21.416 | INFO | "huge" : "", | 20260505 06:16:21.416 | INFO | "huge_shared" : " 27 GB", | 20260505 06:16:21.417 | INFO | "malloc" : " 14 GB", | 20260505 06:16:21.417 | INFO | "pinned" : " 29 GB" | 20260505 06:16:21.417 | INFO | } | 20260505 06:16:21.417 | INFO | Perf report for the last 5.1429 seconds: | 20260505 06:16:21.417 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:16:21.417 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:21.417 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:21.417 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:21.417 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 06:16:21.417 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:21.417 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:21.417 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 06:16:21.417 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:21.417 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:21.417 | INFO | Uploaded 36GiB to GPU in 2.10396s (17.1106GiB/s) | 20260505 06:16:21.417 | INFO | Downloaded 34GiB to GPU in 2.04267s (16.6449GiB/s) | 20260505 06:16:26.554 | INFO | Analyzing chunk frames = [985600,986112) | 20260505 06:16:26.554 | INFO | MemInfo: { | 20260505 06:16:26.554 | INFO | "gpu" : " 17 GB", | 20260505 06:16:26.554 | INFO | "huge" : "", | 20260505 06:16:26.554 | INFO | "huge_shared" : " 27 GB", | 20260505 06:16:26.554 | INFO | "malloc" : " 14 GB", | 20260505 06:16:26.554 | INFO | "pinned" : " 29 GB" | 20260505 06:16:26.554 | INFO | } | 20260505 06:16:26.554 | INFO | Perf report for the last 5.1372 seconds: | 20260505 06:16:26.554 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:16:26.554 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:26.554 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:26.554 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:26.554 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 06:16:26.554 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:26.554 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:26.554 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 06:16:26.554 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:26.554 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:26.554 | INFO | Uploaded 36GiB to GPU in 2.15801s (16.6821GiB/s) | 20260505 06:16:26.554 | INFO | Downloaded 34GiB to GPU in 2.05012s (16.5844GiB/s) | 20260505 06:16:31.691 | INFO | Analyzing chunk frames = [986112,986624) | 20260505 06:16:31.691 | INFO | MemInfo: { | 20260505 06:16:31.691 | INFO | "gpu" : " 17 GB", | 20260505 06:16:31.691 | INFO | "huge" : "", | 20260505 06:16:31.691 | INFO | "huge_shared" : " 27 GB", | 20260505 06:16:31.691 | INFO | "malloc" : " 14 GB", | 20260505 06:16:31.691 | INFO | "pinned" : " 29 GB" | 20260505 06:16:31.691 | INFO | } | 20260505 06:16:31.691 | INFO | Perf report for the last 5.13735 seconds: | 20260505 06:16:31.691 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:16:31.691 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:31.691 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:31.691 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:31.691 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 06:16:31.691 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:31.691 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:31.691 | INFO | PRE_HQ : 32.7%, 1.0, 17.0ms, 0 | 20260505 06:16:31.691 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:31.691 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:31.692 | INFO | Uploaded 36GiB to GPU in 2.16887s (16.5985GiB/s) | 20260505 06:16:31.692 | INFO | Downloaded 34GiB to GPU in 2.1612s (15.732GiB/s) | 20260505 06:16:36.830 | INFO | Analyzing chunk frames = [986624,987136) | 20260505 06:16:36.830 | INFO | MemInfo: { | 20260505 06:16:36.830 | INFO | "gpu" : " 17 GB", | 20260505 06:16:36.830 | INFO | "huge" : "", | 20260505 06:16:36.830 | INFO | "huge_shared" : " 27 GB", | 20260505 06:16:36.830 | INFO | "malloc" : " 14 GB", | 20260505 06:16:36.830 | INFO | "pinned" : " 29 GB" | 20260505 06:16:36.830 | INFO | } | 20260505 06:16:36.830 | INFO | Perf report for the last 5.13921 seconds: | 20260505 06:16:36.831 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:16:36.831 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:36.831 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:36.831 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:36.831 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 06:16:36.831 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:36.831 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:36.831 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 06:16:36.831 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:36.831 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:36.831 | INFO | Uploaded 36GiB to GPU in 2.10179s (17.1282GiB/s) | 20260505 06:16:36.831 | INFO | Downloaded 34GiB to GPU in 2.0739s (16.3942GiB/s) | 20260505 06:16:36.945 | INFO | Starting a round of Baz IO | 20260505 06:16:40.236 | INFO | Populating RTMetrics report | 20260505 06:16:40.236 | INFO | RTMetrics report populated | 20260505 06:16:40.236 | INFO | Opening tmp RTMetrics json file | 20260505 06:16:40.252 | INFO | Serializing RTMetrics to json | 20260505 06:16:40.254 | INFO | Serialization done, swapping tmp file now... | 20260505 06:16:40.254 | INFO | RTMetrics done | 20260505 06:16:40.351 | INFO | Finished a round of Baz IO: Wrote 2.357 GiB in 3.406 seconds | 20260505 06:16:40.351 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.09% / 0.05% / 65.85% | 20260505 06:16:40.351 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.47319% / 0.000115236% / 5.5631e-05% | 20260505 06:16:40.351 | INFO | Baz IO overall: | 20260505 06:16:40.351 | INFO | BytesWritten : 502560518537 | 20260505 06:16:40.351 | INFO | NumEvents : 237396937173 | 20260505 06:16:41.970 | INFO | Analyzing chunk frames = [987136,987648) | 20260505 06:16:41.970 | INFO | MemInfo: { | 20260505 06:16:41.970 | INFO | "gpu" : " 17 GB", | 20260505 06:16:41.970 | INFO | "huge" : "", | 20260505 06:16:41.970 | INFO | "huge_shared" : " 27 GB", | 20260505 06:16:41.970 | INFO | "malloc" : " 14 GB", | 20260505 06:16:41.970 | INFO | "pinned" : " 29 GB" | 20260505 06:16:41.970 | INFO | } | 20260505 06:16:41.970 | INFO | Perf report for the last 5.13953 seconds: | 20260505 06:16:41.970 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:16:41.970 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:41.970 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:41.970 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:41.970 | INFO | ANALYSIS : 65.1%, 2.9, 99.4ms, 0 | 20260505 06:16:41.970 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:41.970 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260505 06:16:41.970 | INFO | PRE_HQ : 64.0%, 1.9, 64.3ms, 0 | 20260505 06:16:41.970 | INFO | BAZWRITER : 65.6%, 4.1, 137.8ms, 0 | 20260505 06:16:41.970 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 06:16:41.970 | INFO | Uploaded 36GiB to GPU in 2.19704s (16.3857GiB/s) | 20260505 06:16:41.970 | INFO | Downloaded 38GiB to GPU in 2.32167s (16.3675GiB/s) | 20260505 06:16:47.109 | INFO | Analyzing chunk frames = [987648,988160) | 20260505 06:16:47.109 | INFO | MemInfo: { | 20260505 06:16:47.109 | INFO | "gpu" : " 17 GB", | 20260505 06:16:47.109 | INFO | "huge" : "", | 20260505 06:16:47.109 | INFO | "huge_shared" : " 27 GB", | 20260505 06:16:47.109 | INFO | "malloc" : " 14 GB", | 20260505 06:16:47.109 | INFO | "pinned" : " 29 GB" | 20260505 06:16:47.109 | INFO | } | 20260505 06:16:47.109 | INFO | Perf report for the last 5.13947 seconds: | 20260505 06:16:47.109 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:16:47.109 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:47.109 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:47.109 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:47.109 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 06:16:47.109 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:47.109 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:47.109 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 06:16:47.109 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:47.110 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:47.110 | INFO | Uploaded 36GiB to GPU in 2.131s (16.8934GiB/s) | 20260505 06:16:47.110 | INFO | Downloaded 34GiB to GPU in 2.03885s (16.6761GiB/s) | 20260505 06:16:52.248 | INFO | Analyzing chunk frames = [988160,988672) | 20260505 06:16:52.248 | INFO | MemInfo: { | 20260505 06:16:52.248 | INFO | "gpu" : " 17 GB", | 20260505 06:16:52.248 | INFO | "huge" : "", | 20260505 06:16:52.248 | INFO | "huge_shared" : " 27 GB", | 20260505 06:16:52.248 | INFO | "malloc" : " 14 GB", | 20260505 06:16:52.248 | INFO | "pinned" : " 29 GB" | 20260505 06:16:52.248 | INFO | } | 20260505 06:16:52.248 | INFO | Perf report for the last 5.13875 seconds: | 20260505 06:16:52.248 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:16:52.248 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:52.248 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:52.248 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:52.248 | INFO | ANALYSIS : 64.0%, 3.0, 99.6ms, 0 | 20260505 06:16:52.248 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:52.248 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:52.248 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 06:16:52.248 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:52.248 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:52.249 | INFO | Uploaded 36GiB to GPU in 2.15237s (16.7258GiB/s) | 20260505 06:16:52.249 | INFO | Downloaded 34GiB to GPU in 2.02859s (16.7604GiB/s) | 20260505 06:16:57.388 | INFO | Analyzing chunk frames = [988672,989184) | 20260505 06:16:57.388 | INFO | MemInfo: { | 20260505 06:16:57.388 | INFO | "gpu" : " 17 GB", | 20260505 06:16:57.388 | INFO | "huge" : "", | 20260505 06:16:57.388 | INFO | "huge_shared" : " 27 GB", | 20260505 06:16:57.388 | INFO | "malloc" : " 14 GB", | 20260505 06:16:57.388 | INFO | "pinned" : " 29 GB" | 20260505 06:16:57.388 | INFO | } | 20260505 06:16:57.388 | INFO | Perf report for the last 5.13971 seconds: | 20260505 06:16:57.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:16:57.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:57.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:57.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:57.388 | INFO | ANALYSIS : 63.3%, 3.0, 98.3ms, 0 | 20260505 06:16:57.388 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:57.388 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:57.388 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 06:16:57.388 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:57.388 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:16:57.388 | INFO | Uploaded 36GiB to GPU in 2.13761s (16.8412GiB/s) | 20260505 06:16:57.388 | INFO | Downloaded 34GiB to GPU in 2.04859s (16.5968GiB/s) | 20260505 06:17:02.527 | INFO | Analyzing chunk frames = [989184,989696) | 20260505 06:17:02.527 | INFO | MemInfo: { | 20260505 06:17:02.527 | INFO | "gpu" : " 17 GB", | 20260505 06:17:02.527 | INFO | "huge" : "", | 20260505 06:17:02.527 | INFO | "huge_shared" : " 27 GB", | 20260505 06:17:02.527 | INFO | "malloc" : " 14 GB", | 20260505 06:17:02.527 | INFO | "pinned" : " 29 GB" | 20260505 06:17:02.527 | INFO | } | 20260505 06:17:02.527 | INFO | Perf report for the last 5.13922 seconds: | 20260505 06:17:02.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:17:02.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:02.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:02.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:02.527 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 06:17:02.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:02.527 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:02.527 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 06:17:02.527 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:02.527 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:02.527 | INFO | Uploaded 36GiB to GPU in 2.16646s (16.617GiB/s) | 20260505 06:17:02.527 | INFO | Downloaded 34GiB to GPU in 2.05528s (16.5428GiB/s) | 20260505 06:17:07.668 | INFO | Analyzing chunk frames = [989696,990208) | 20260505 06:17:07.668 | INFO | MemInfo: { | 20260505 06:17:07.668 | INFO | "gpu" : " 17 GB", | 20260505 06:17:07.668 | INFO | "huge" : "", | 20260505 06:17:07.668 | INFO | "huge_shared" : " 27 GB", | 20260505 06:17:07.668 | INFO | "malloc" : " 14 GB", | 20260505 06:17:07.668 | INFO | "pinned" : " 29 GB" | 20260505 06:17:07.668 | INFO | } | 20260505 06:17:07.668 | INFO | Perf report for the last 5.14098 seconds: | 20260505 06:17:07.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:17:07.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:07.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:07.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:07.668 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 06:17:07.668 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:07.668 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:07.668 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 06:17:07.668 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:07.668 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:07.668 | INFO | Uploaded 36GiB to GPU in 2.16926s (16.5955GiB/s) | 20260505 06:17:07.668 | INFO | Downloaded 34GiB to GPU in 2.0539s (16.5539GiB/s) | 20260505 06:17:12.806 | INFO | Analyzing chunk frames = [990208,990720) | 20260505 06:17:12.806 | INFO | MemInfo: { | 20260505 06:17:12.806 | INFO | "gpu" : " 17 GB", | 20260505 06:17:12.806 | INFO | "huge" : "", | 20260505 06:17:12.806 | INFO | "huge_shared" : " 27 GB", | 20260505 06:17:12.806 | INFO | "malloc" : " 14 GB", | 20260505 06:17:12.806 | INFO | "pinned" : " 29 GB" | 20260505 06:17:12.806 | INFO | } | 20260505 06:17:12.806 | INFO | Perf report for the last 5.13776 seconds: | 20260505 06:17:12.806 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:17:12.806 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:12.806 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:12.806 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:12.806 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 06:17:12.806 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:12.806 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:12.806 | INFO | PRE_HQ : 32.8%, 1.0, 17.0ms, 0 | 20260505 06:17:12.806 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:12.806 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:12.806 | INFO | Uploaded 36GiB to GPU in 2.17206s (16.5741GiB/s) | 20260505 06:17:12.806 | INFO | Downloaded 34GiB to GPU in 2.14524s (15.8491GiB/s) | 20260505 06:17:17.945 | INFO | Analyzing chunk frames = [990720,991232) | 20260505 06:17:17.945 | INFO | MemInfo: { | 20260505 06:17:17.945 | INFO | "gpu" : " 17 GB", | 20260505 06:17:17.945 | INFO | "huge" : "", | 20260505 06:17:17.945 | INFO | "huge_shared" : " 27 GB", | 20260505 06:17:17.945 | INFO | "malloc" : " 14 GB", | 20260505 06:17:17.945 | INFO | "pinned" : " 29 GB" | 20260505 06:17:17.945 | INFO | } | 20260505 06:17:17.945 | INFO | Perf report for the last 5.13935 seconds: | 20260505 06:17:17.945 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:17:17.945 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:17.945 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:17.945 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:17.945 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 06:17:17.945 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:17.945 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:17.945 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260505 06:17:17.945 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:17.945 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:17.946 | INFO | Uploaded 36GiB to GPU in 2.09957s (17.1464GiB/s) | 20260505 06:17:17.946 | INFO | Downloaded 34GiB to GPU in 2.06934s (16.4304GiB/s) | 20260505 06:17:17.946 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 06:17:17.946 | INFO | Total mem usage: 15.3 GiB | 20260505 06:17:17.946 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 06:17:17.946 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -5.73%) | 20260505 06:17:17.946 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 06:17:17.946 | INFO | High Water Breakdown: | 20260505 06:17:17.946 | INFO | 60.3% : 8.75 GiB ( 93.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 06:17:17.946 | INFO | 39.5% : 5.73 GiB ( 86.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 06:17:17.946 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 06:17:17.946 | INFO | ------------------------------------------ | 20260505 06:17:17.946 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:17:17.946 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 06:17:17.946 | INFO | Total mem usage: 26.5 GiB | 20260505 06:17:17.946 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 06:17:17.946 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 06:17:17.946 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 06:17:17.946 | INFO | High Water Breakdown: | 20260505 06:17:17.946 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:17:17.946 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:17:17.946 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:17:17.946 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:17:17.946 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:17:17.946 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:17:17.946 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 06:17:17.946 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 06:17:17.946 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:17:17.946 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 06:17:17.946 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 06:17:17.946 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 06:17:17.946 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 06:17:17.946 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 06:17:17.946 | INFO | ------------------------------------------ | 20260505 06:17:17.946 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:17:17.946 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 06:17:17.946 | INFO | Total mem usage: 25.6 GiB | 20260505 06:17:17.946 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 06:17:17.946 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 06:17:17.946 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 06:17:17.946 | INFO | High Water Breakdown: | 20260505 06:17:17.946 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 06:17:17.946 | INFO | ------------------------------------------ | 20260505 06:17:17.946 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:17:17.946 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 06:17:17.946 | INFO | Total mem usage: 15.7 GiB | 20260505 06:17:17.946 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 06:17:17.946 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 06:17:17.946 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260505 06:17:17.946 | INFO | High Water Breakdown: | 20260505 06:17:17.946 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 06:17:17.946 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 06:17:17.946 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 06:17:17.946 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 06:17:17.946 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 06:17:17.946 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 06:17:17.946 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 06:17:17.947 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 06:17:17.947 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 06:17:17.947 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:17:17.947 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 06:17:17.947 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:17:17.947 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 06:17:17.947 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 06:17:17.947 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 06:17:17.947 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 06:17:17.947 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:17:17.947 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 06:17:17.947 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:17:17.947 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:17:17.947 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:17:17.947 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 06:17:17.947 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 06:17:17.947 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:17:17.947 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 06:17:17.947 | INFO | ------------------------------------------ | 20260505 06:17:17.947 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:17:17.947 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 06:17:17.947 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:17:17.947 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:17:17.947 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.60e+01, 1.37e+01 / 1.41e+00 | 20260505 06:17:17.947 | INFO | PerfReport: 0.28%, AggregateStats -- 9.43e-02 / 1.04e-01, 1.23e-02 / 5.44e-03 | 20260505 06:17:17.947 | INFO | PerfReport: 11.56%, Baseline -- 3.84e+00 / 4.52e+00, 6.33e-01 / 2.51e-01 | 20260505 06:17:17.947 | INFO | PerfReport: 3.31%, Histogram -- 1.10e+00 / 1.50e+00, 2.18e-01 / 1.12e-01 | 20260505 06:17:17.947 | INFO | PerfReport: 15.48%, DME -- 4.11e+01 / 4.53e+01, 4.40e+00 / 1.00e+00 | 20260505 06:17:17.947 | INFO | PerfReport: 56.03%, FrameLabeling -- 1.86e+01 / 1.91e+01, 2.07e+00 / 9.01e-01 | 20260505 06:17:17.947 | INFO | PerfReport: 7.71%, PulseAccumulating -- 2.56e+00 / 3.16e+00, 4.71e-01 / 3.08e-01 | 20260505 06:17:17.947 | INFO | PerfReport: 4.11%, Metrics -- 1.37e+00 / 1.74e+00, 3.84e-01 / 2.16e-01 | 20260505 06:17:17.947 | INFO | PerfReport: 1.51%, PulseSerializing -- 5.00e-01 / 5.81e-01, 9.86e-02 / 3.66e-02 | 20260505 06:17:17.947 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 06:17:17.947 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:17:17.947 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:17:17.947 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.26e+01 / 0.00e+00 | 20260505 06:17:17.947 | INFO | PerfReport: 22.03%, Upload -- 2.24e+01 / 0.00e+00, 7.79e+00 / 0.00e+00 | 20260505 06:17:17.947 | INFO | PerfReport: 21.51%, Download -- 2.18e+01 / 0.00e+00, 7.77e+00 / 0.00e+00 | 20260505 06:17:17.947 | INFO | PerfReport: 32.71%, Compute -- 3.32e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 06:17:17.947 | INFO | PerfReport: 1.43%, LockUpload -- 1.45e+00 / 0.00e+00, 6.70e+00 / 0.00e+00 | 20260505 06:17:17.947 | INFO | PerfReport: 0.70%, LockDownload -- 7.09e-01 / 0.00e+00, 1.94e+00 / 0.00e+00 | 20260505 06:17:17.947 | INFO | PerfReport: 21.60%, LockCompute -- 2.19e+01 / 0.00e+00, 1.68e+01 / 0.00e+00 | 20260505 06:17:17.947 | INFO | PerfReport: 0.02%, Misc -- 1.53e-02 / 0.00e+00, 3.34e-02 / 0.00e+00 | 20260505 06:17:18.060 | INFO | Starting a round of Baz IO | 20260505 06:17:21.340 | INFO | Populating RTMetrics report | 20260505 06:17:21.340 | INFO | RTMetrics report populated | 20260505 06:17:21.341 | INFO | Opening tmp RTMetrics json file | 20260505 06:17:21.342 | INFO | Serializing RTMetrics to json | 20260505 06:17:21.349 | INFO | Serialization done, swapping tmp file now... | 20260505 06:17:21.365 | INFO | RTMetrics done | 20260505 06:17:21.458 | INFO | Finished a round of Baz IO: Wrote 2.357 GiB in 3.397 seconds | 20260505 06:17:21.458 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.04% / 0.05% / 65.91% | 20260505 06:17:21.458 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.4409% / 5.5631e-05% / 7.54992e-05% | 20260505 06:17:21.458 | INFO | Baz IO overall: | 20260505 06:17:21.458 | INFO | BytesWritten : 505091125620 | 20260505 06:17:21.458 | INFO | NumEvents : 238606788808 | 20260505 06:17:23.084 | INFO | Analyzing chunk frames = [991232,991744) | 20260505 06:17:23.084 | INFO | MemInfo: { | 20260505 06:17:23.084 | INFO | "gpu" : " 17 GB", | 20260505 06:17:23.084 | INFO | "huge" : "", | 20260505 06:17:23.084 | INFO | "huge_shared" : " 27 GB", | 20260505 06:17:23.084 | INFO | "malloc" : " 14 GB", | 20260505 06:17:23.084 | INFO | "pinned" : " 29 GB" | 20260505 06:17:23.084 | INFO | } | 20260505 06:17:23.084 | INFO | Perf report for the last 5.13914 seconds: | 20260505 06:17:23.085 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:17:23.085 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:23.085 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:23.085 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:23.085 | INFO | ANALYSIS : 64.9%, 2.9, 99.0ms, 0 | 20260505 06:17:23.085 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:23.085 | INFO | RT_METRICS : 46.5%, 1.0, 24.4ms, 0 | 20260505 06:17:23.085 | INFO | PRE_HQ : 63.8%, 2.0, 64.8ms, 0 | 20260505 06:17:23.085 | INFO | BAZWRITER : 65.4%, 4.1, 136.3ms, 0 | 20260505 06:17:23.085 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 06:17:23.085 | INFO | Uploaded 36GiB to GPU in 2.24007s (16.0709GiB/s) | 20260505 06:17:23.085 | INFO | Downloaded 38GiB to GPU in 2.32939s (16.3133GiB/s) | 20260505 06:17:28.224 | INFO | Analyzing chunk frames = [991744,992256) | 20260505 06:17:28.224 | INFO | MemInfo: { | 20260505 06:17:28.224 | INFO | "gpu" : " 17 GB", | 20260505 06:17:28.224 | INFO | "huge" : "", | 20260505 06:17:28.224 | INFO | "huge_shared" : " 27 GB", | 20260505 06:17:28.224 | INFO | "malloc" : " 14 GB", | 20260505 06:17:28.224 | INFO | "pinned" : " 29 GB" | 20260505 06:17:28.224 | INFO | } | 20260505 06:17:28.224 | INFO | Perf report for the last 5.13976 seconds: | 20260505 06:17:28.224 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:17:28.224 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:28.224 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:28.224 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:28.224 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 06:17:28.224 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:28.224 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:28.224 | INFO | PRE_HQ : 30.5%, 1.0, 15.9ms, 0 | 20260505 06:17:28.224 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:28.224 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:28.225 | INFO | Uploaded 36GiB to GPU in 2.11779s (16.9989GiB/s) | 20260505 06:17:28.225 | INFO | Downloaded 34GiB to GPU in 2.0406s (16.6618GiB/s) | 20260505 06:17:33.363 | INFO | Analyzing chunk frames = [992256,992768) | 20260505 06:17:33.363 | INFO | MemInfo: { | 20260505 06:17:33.363 | INFO | "gpu" : " 17 GB", | 20260505 06:17:33.363 | INFO | "huge" : "", | 20260505 06:17:33.363 | INFO | "huge_shared" : " 27 GB", | 20260505 06:17:33.363 | INFO | "malloc" : " 14 GB", | 20260505 06:17:33.363 | INFO | "pinned" : " 29 GB" | 20260505 06:17:33.363 | INFO | } | 20260505 06:17:33.363 | INFO | Perf report for the last 5.13911 seconds: | 20260505 06:17:33.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:17:33.364 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:33.364 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:33.364 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:33.364 | INFO | ANALYSIS : 63.9%, 3.0, 99.4ms, 0 | 20260505 06:17:33.364 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:33.364 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:33.364 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 06:17:33.364 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:33.364 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:33.364 | INFO | Uploaded 36GiB to GPU in 2.14809s (16.7591GiB/s) | 20260505 06:17:33.364 | INFO | Downloaded 34GiB to GPU in 2.03819s (16.6815GiB/s) | 20260505 06:17:38.502 | INFO | Analyzing chunk frames = [992768,993280) | 20260505 06:17:38.502 | INFO | MemInfo: { | 20260505 06:17:38.502 | INFO | "gpu" : " 17 GB", | 20260505 06:17:38.502 | INFO | "huge" : "", | 20260505 06:17:38.502 | INFO | "huge_shared" : " 27 GB", | 20260505 06:17:38.502 | INFO | "malloc" : " 14 GB", | 20260505 06:17:38.502 | INFO | "pinned" : " 29 GB" | 20260505 06:17:38.503 | INFO | } | 20260505 06:17:38.503 | INFO | Perf report for the last 5.13926 seconds: | 20260505 06:17:38.503 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:17:38.503 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:38.503 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:38.503 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:38.503 | INFO | ANALYSIS : 63.4%, 3.0, 98.5ms, 0 | 20260505 06:17:38.503 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:38.503 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:38.503 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 06:17:38.503 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:38.503 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:38.503 | INFO | Uploaded 36GiB to GPU in 2.14157s (16.8101GiB/s) | 20260505 06:17:38.503 | INFO | Downloaded 34GiB to GPU in 2.05171s (16.5715GiB/s) | 20260505 06:17:43.642 | INFO | Analyzing chunk frames = [993280,993792) | 20260505 06:17:43.642 | INFO | MemInfo: { | 20260505 06:17:43.642 | INFO | "gpu" : " 17 GB", | 20260505 06:17:43.642 | INFO | "huge" : "", | 20260505 06:17:43.642 | INFO | "huge_shared" : " 27 GB", | 20260505 06:17:43.642 | INFO | "malloc" : " 14 GB", | 20260505 06:17:43.642 | INFO | "pinned" : " 29 GB" | 20260505 06:17:43.642 | INFO | } | 20260505 06:17:43.642 | INFO | Perf report for the last 5.13963 seconds: | 20260505 06:17:43.642 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:17:43.642 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:43.642 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:43.642 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:43.642 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 06:17:43.642 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:43.642 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:43.642 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 06:17:43.642 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:43.642 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:43.643 | INFO | Uploaded 36GiB to GPU in 2.15542s (16.7021GiB/s) | 20260505 06:17:43.643 | INFO | Downloaded 34GiB to GPU in 2.04238s (16.6472GiB/s) | 20260505 06:17:48.783 | INFO | Analyzing chunk frames = [993792,994304) | 20260505 06:17:48.783 | INFO | MemInfo: { | 20260505 06:17:48.783 | INFO | "gpu" : " 17 GB", | 20260505 06:17:48.783 | INFO | "huge" : "", | 20260505 06:17:48.783 | INFO | "huge_shared" : " 27 GB", | 20260505 06:17:48.783 | INFO | "malloc" : " 14 GB", | 20260505 06:17:48.783 | INFO | "pinned" : " 29 GB" | 20260505 06:17:48.783 | INFO | } | 20260505 06:17:48.783 | INFO | Perf report for the last 5.14066 seconds: | 20260505 06:17:48.783 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:17:48.783 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:48.783 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:48.783 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:48.783 | INFO | ANALYSIS : 64.1%, 3.0, 100.2ms, 0 | 20260505 06:17:48.783 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:48.783 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:48.783 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 06:17:48.783 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:48.783 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:48.783 | INFO | Uploaded 36GiB to GPU in 2.16347s (16.64GiB/s) | 20260505 06:17:48.783 | INFO | Downloaded 34GiB to GPU in 2.05399s (16.5532GiB/s) | 20260505 06:17:53.921 | INFO | Analyzing chunk frames = [994304,994816) | 20260505 06:17:53.921 | INFO | MemInfo: { | 20260505 06:17:53.921 | INFO | "gpu" : " 17 GB", | 20260505 06:17:53.921 | INFO | "huge" : "", | 20260505 06:17:53.921 | INFO | "huge_shared" : " 27 GB", | 20260505 06:17:53.921 | INFO | "malloc" : " 14 GB", | 20260505 06:17:53.921 | INFO | "pinned" : " 29 GB" | 20260505 06:17:53.921 | INFO | } | 20260505 06:17:53.921 | INFO | Perf report for the last 5.13784 seconds: | 20260505 06:17:53.921 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:17:53.921 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:53.921 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:53.921 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:53.921 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 06:17:53.921 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:53.921 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:53.921 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 06:17:53.921 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:53.921 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:53.921 | INFO | Uploaded 36GiB to GPU in 2.19509s (16.4002GiB/s) | 20260505 06:17:53.921 | INFO | Downloaded 34GiB to GPU in 2.15088s (15.8075GiB/s) | 20260505 06:17:59.060 | INFO | Analyzing chunk frames = [994816,995328) | 20260505 06:17:59.060 | INFO | MemInfo: { | 20260505 06:17:59.060 | INFO | "gpu" : " 17 GB", | 20260505 06:17:59.060 | INFO | "huge" : "", | 20260505 06:17:59.060 | INFO | "huge_shared" : " 27 GB", | 20260505 06:17:59.060 | INFO | "malloc" : " 14 GB", | 20260505 06:17:59.060 | INFO | "pinned" : " 29 GB" | 20260505 06:17:59.060 | INFO | } | 20260505 06:17:59.060 | INFO | Perf report for the last 5.13971 seconds: | 20260505 06:17:59.060 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:17:59.060 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:59.060 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:59.060 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:59.060 | INFO | ANALYSIS : 63.8%, 3.0, 98.6ms, 0 | 20260505 06:17:59.061 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:59.061 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:59.061 | INFO | PRE_HQ : 31.1%, 1.0, 16.0ms, 0 | 20260505 06:17:59.061 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:59.061 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:17:59.061 | INFO | Uploaded 36GiB to GPU in 2.09827s (17.157GiB/s) | 20260505 06:17:59.061 | INFO | Downloaded 34GiB to GPU in 2.07521s (16.3839GiB/s) | 20260505 06:17:59.175 | INFO | Starting a round of Baz IO | 20260505 06:18:02.435 | INFO | Populating RTMetrics report | 20260505 06:18:02.435 | INFO | RTMetrics report populated | 20260505 06:18:02.435 | INFO | Opening tmp RTMetrics json file | 20260505 06:18:02.436 | INFO | Serializing RTMetrics to json | 20260505 06:18:02.437 | INFO | Serialization done, swapping tmp file now... | 20260505 06:18:02.437 | INFO | RTMetrics done | 20260505 06:18:02.552 | INFO | Finished a round of Baz IO: Wrote 2.359 GiB in 3.376 seconds | 20260505 06:18:02.552 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.99% / 0.05% / 65.96% | 20260505 06:18:02.552 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.42748% / 8.74201e-05% / 7.94729e-05% | 20260505 06:18:02.552 | INFO | Baz IO overall: | 20260505 06:18:02.552 | INFO | BytesWritten : 507624087029 | 20260505 06:18:02.552 | INFO | NumEvents : 239818273065 | 20260505 06:18:04.199 | INFO | Analyzing chunk frames = [995328,995840) | 20260505 06:18:04.199 | INFO | MemInfo: { | 20260505 06:18:04.199 | INFO | "gpu" : " 17 GB", | 20260505 06:18:04.199 | INFO | "huge" : "", | 20260505 06:18:04.199 | INFO | "huge_shared" : " 27 GB", | 20260505 06:18:04.199 | INFO | "malloc" : " 14 GB", | 20260505 06:18:04.199 | INFO | "pinned" : " 29 GB" | 20260505 06:18:04.199 | INFO | } | 20260505 06:18:04.199 | INFO | Perf report for the last 5.13894 seconds: | 20260505 06:18:04.199 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:18:04.199 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:04.199 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:04.199 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:04.199 | INFO | ANALYSIS : 64.5%, 2.9, 98.3ms, 0 | 20260505 06:18:04.200 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:04.200 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260505 06:18:04.200 | INFO | PRE_HQ : 63.4%, 2.0, 64.4ms, 0 | 20260505 06:18:04.200 | INFO | BAZWRITER : 65.0%, 4.1, 136.8ms, 0 | 20260505 06:18:04.200 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 06:18:04.200 | INFO | Uploaded 36GiB to GPU in 2.14222s (16.805GiB/s) | 20260505 06:18:04.200 | INFO | Downloaded 38GiB to GPU in 2.32712s (16.3292GiB/s) | 20260505 06:18:09.338 | INFO | Analyzing chunk frames = [995840,996352) | 20260505 06:18:09.338 | INFO | MemInfo: { | 20260505 06:18:09.338 | INFO | "gpu" : " 17 GB", | 20260505 06:18:09.338 | INFO | "huge" : "", | 20260505 06:18:09.338 | INFO | "huge_shared" : " 27 GB", | 20260505 06:18:09.338 | INFO | "malloc" : " 14 GB", | 20260505 06:18:09.338 | INFO | "pinned" : " 29 GB" | 20260505 06:18:09.338 | INFO | } | 20260505 06:18:09.338 | INFO | Perf report for the last 5.13906 seconds: | 20260505 06:18:09.339 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:18:09.339 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:09.339 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:09.339 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:09.339 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 06:18:09.339 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:09.339 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:09.339 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 06:18:09.339 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:09.339 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:09.339 | INFO | Uploaded 36GiB to GPU in 2.12948s (16.9056GiB/s) | 20260505 06:18:09.339 | INFO | Downloaded 34GiB to GPU in 2.04142s (16.6551GiB/s) | 20260505 06:18:14.479 | INFO | Analyzing chunk frames = [996352,996864) | 20260505 06:18:14.479 | INFO | MemInfo: { | 20260505 06:18:14.479 | INFO | "gpu" : " 17 GB", | 20260505 06:18:14.479 | INFO | "huge" : "", | 20260505 06:18:14.479 | INFO | "huge_shared" : " 27 GB", | 20260505 06:18:14.479 | INFO | "malloc" : " 14 GB", | 20260505 06:18:14.479 | INFO | "pinned" : " 29 GB" | 20260505 06:18:14.479 | INFO | } | 20260505 06:18:14.479 | INFO | Perf report for the last 5.14029 seconds: | 20260505 06:18:14.479 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:18:14.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:14.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:14.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:14.479 | INFO | ANALYSIS : 64.0%, 3.0, 99.6ms, 0 | 20260505 06:18:14.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:14.479 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:14.479 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 06:18:14.479 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:14.479 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:14.479 | INFO | Uploaded 36GiB to GPU in 2.15692s (16.6905GiB/s) | 20260505 06:18:14.479 | INFO | Downloaded 34GiB to GPU in 2.03353s (16.7197GiB/s) | 20260505 06:18:19.617 | INFO | Analyzing chunk frames = [996864,997376) | 20260505 06:18:19.617 | INFO | MemInfo: { | 20260505 06:18:19.617 | INFO | "gpu" : " 17 GB", | 20260505 06:18:19.617 | INFO | "huge" : "", | 20260505 06:18:19.617 | INFO | "huge_shared" : " 27 GB", | 20260505 06:18:19.617 | INFO | "malloc" : " 14 GB", | 20260505 06:18:19.617 | INFO | "pinned" : " 29 GB" | 20260505 06:18:19.617 | INFO | } | 20260505 06:18:19.617 | INFO | Perf report for the last 5.13856 seconds: | 20260505 06:18:19.617 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:18:19.617 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:19.617 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:19.617 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:19.617 | INFO | ANALYSIS : 63.3%, 3.0, 98.3ms, 0 | 20260505 06:18:19.617 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:19.617 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:19.617 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 06:18:19.617 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:19.618 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:19.618 | INFO | Uploaded 36GiB to GPU in 2.13021s (16.8997GiB/s) | 20260505 06:18:19.618 | INFO | Downloaded 34GiB to GPU in 2.05042s (16.582GiB/s) | 20260505 06:18:24.757 | INFO | Analyzing chunk frames = [997376,997888) | 20260505 06:18:24.757 | INFO | MemInfo: { | 20260505 06:18:24.757 | INFO | "gpu" : " 17 GB", | 20260505 06:18:24.757 | INFO | "huge" : "", | 20260505 06:18:24.757 | INFO | "huge_shared" : " 27 GB", | 20260505 06:18:24.757 | INFO | "malloc" : " 14 GB", | 20260505 06:18:24.757 | INFO | "pinned" : " 29 GB" | 20260505 06:18:24.757 | INFO | } | 20260505 06:18:24.757 | INFO | Perf report for the last 5.13958 seconds: | 20260505 06:18:24.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:18:24.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:24.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:24.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:24.757 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 06:18:24.757 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:24.757 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:24.757 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 06:18:24.757 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:24.757 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:24.757 | INFO | Uploaded 36GiB to GPU in 2.15637s (16.6947GiB/s) | 20260505 06:18:24.757 | INFO | Downloaded 34GiB to GPU in 2.04121s (16.6568GiB/s) | 20260505 06:18:29.898 | INFO | Analyzing chunk frames = [997888,998400) | 20260505 06:18:29.898 | INFO | MemInfo: { | 20260505 06:18:29.898 | INFO | "gpu" : " 17 GB", | 20260505 06:18:29.898 | INFO | "huge" : "", | 20260505 06:18:29.898 | INFO | "huge_shared" : " 27 GB", | 20260505 06:18:29.898 | INFO | "malloc" : " 14 GB", | 20260505 06:18:29.898 | INFO | "pinned" : " 29 GB" | 20260505 06:18:29.898 | INFO | } | 20260505 06:18:29.898 | INFO | Perf report for the last 5.14093 seconds: | 20260505 06:18:29.898 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:18:29.898 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:29.898 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:29.898 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:29.898 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 06:18:29.898 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:29.898 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:29.898 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 06:18:29.898 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:29.898 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:29.898 | INFO | Uploaded 36GiB to GPU in 2.1605s (16.6628GiB/s) | 20260505 06:18:29.898 | INFO | Downloaded 34GiB to GPU in 2.05074s (16.5794GiB/s) | 20260505 06:18:35.035 | INFO | Analyzing chunk frames = [998400,998912) | 20260505 06:18:35.035 | INFO | MemInfo: { | 20260505 06:18:35.035 | INFO | "gpu" : " 17 GB", | 20260505 06:18:35.035 | INFO | "huge" : "", | 20260505 06:18:35.035 | INFO | "huge_shared" : " 27 GB", | 20260505 06:18:35.035 | INFO | "malloc" : " 14 GB", | 20260505 06:18:35.035 | INFO | "pinned" : " 29 GB" | 20260505 06:18:35.035 | INFO | } | 20260505 06:18:35.035 | INFO | Perf report for the last 5.13758 seconds: | 20260505 06:18:35.036 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:18:35.036 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:35.036 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:35.036 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:35.036 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 06:18:35.036 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:35.036 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:35.036 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260505 06:18:35.036 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:35.036 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:35.036 | INFO | Uploaded 36GiB to GPU in 2.15222s (16.7269GiB/s) | 20260505 06:18:35.036 | INFO | Downloaded 34GiB to GPU in 2.15297s (15.7922GiB/s) | 20260505 06:18:40.175 | INFO | Analyzing chunk frames = [998912,999424) | 20260505 06:18:40.175 | INFO | MemInfo: { | 20260505 06:18:40.175 | INFO | "gpu" : " 17 GB", | 20260505 06:18:40.175 | INFO | "huge" : "", | 20260505 06:18:40.175 | INFO | "huge_shared" : " 27 GB", | 20260505 06:18:40.175 | INFO | "malloc" : " 14 GB", | 20260505 06:18:40.175 | INFO | "pinned" : " 29 GB" | 20260505 06:18:40.175 | INFO | } | 20260505 06:18:40.175 | INFO | Perf report for the last 5.13972 seconds: | 20260505 06:18:40.175 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:18:40.175 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:40.175 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:40.175 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:40.175 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 06:18:40.175 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:40.175 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:40.175 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 06:18:40.175 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:40.175 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:40.176 | INFO | Uploaded 36GiB to GPU in 2.10489s (17.103GiB/s) | 20260505 06:18:40.176 | INFO | Downloaded 34GiB to GPU in 2.06887s (16.4341GiB/s) | 20260505 06:18:40.176 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 06:18:40.176 | INFO | Total mem usage: 15.3 GiB | 20260505 06:18:40.176 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 06:18:40.176 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.51%) | 20260505 06:18:40.176 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 06:18:40.176 | INFO | High Water Breakdown: | 20260505 06:18:40.176 | INFO | 60.2% : 8.67 GiB ( 92.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 06:18:40.176 | INFO | 39.6% : 5.7 GiB ( 85.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 06:18:40.176 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 06:18:40.176 | INFO | ------------------------------------------ | 20260505 06:18:40.176 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:18:40.176 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 06:18:40.176 | INFO | Total mem usage: 26.5 GiB | 20260505 06:18:40.176 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 06:18:40.176 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 06:18:40.176 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 06:18:40.176 | INFO | High Water Breakdown: | 20260505 06:18:40.176 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:18:40.176 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:18:40.176 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:18:40.176 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:18:40.176 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:18:40.176 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:18:40.176 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 06:18:40.176 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 06:18:40.176 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:18:40.176 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 06:18:40.176 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 06:18:40.176 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 06:18:40.176 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 06:18:40.176 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 06:18:40.176 | INFO | ------------------------------------------ | 20260505 06:18:40.176 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:18:40.176 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 06:18:40.176 | INFO | Total mem usage: 25.6 GiB | 20260505 06:18:40.176 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 06:18:40.176 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 06:18:40.176 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 06:18:40.176 | INFO | High Water Breakdown: | 20260505 06:18:40.176 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 06:18:40.176 | INFO | ------------------------------------------ | 20260505 06:18:40.176 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:18:40.176 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 06:18:40.176 | INFO | Total mem usage: 15.7 GiB | 20260505 06:18:40.176 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 06:18:40.176 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 06:18:40.176 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260505 06:18:40.177 | INFO | High Water Breakdown: | 20260505 06:18:40.177 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 06:18:40.177 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 06:18:40.177 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 06:18:40.177 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 06:18:40.177 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 06:18:40.177 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 06:18:40.177 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 06:18:40.177 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 06:18:40.177 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 06:18:40.177 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:18:40.177 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 06:18:40.177 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:18:40.177 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 06:18:40.177 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 06:18:40.177 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 06:18:40.177 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 06:18:40.177 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:18:40.177 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 06:18:40.177 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:18:40.177 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:18:40.177 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:18:40.177 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 06:18:40.177 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 06:18:40.177 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:18:40.177 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 06:18:40.177 | INFO | ------------------------------------------ | 20260505 06:18:40.177 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:18:40.177 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 06:18:40.177 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:18:40.177 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:18:40.177 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.58e+01, 1.37e+01 / 1.73e+00 | 20260505 06:18:40.177 | INFO | PerfReport: 0.28%, AggregateStats -- 9.40e-02 / 1.04e-01, 1.08e-02 / 5.16e-03 | 20260505 06:18:40.177 | INFO | PerfReport: 11.55%, Baseline -- 3.83e+00 / 4.48e+00, 6.35e-01 / 3.11e-01 | 20260505 06:18:40.177 | INFO | PerfReport: 3.31%, Histogram -- 1.10e+00 / 1.50e+00, 2.19e-01 / 1.21e-01 | 20260505 06:18:40.177 | INFO | PerfReport: 15.45%, DME -- 4.10e+01 / 4.51e+01, 4.48e+00 / 1.22e+00 | 20260505 06:18:40.177 | INFO | PerfReport: 56.06%, FrameLabeling -- 1.86e+01 / 1.91e+01, 2.10e+00 / 1.03e+00 | 20260505 06:18:40.177 | INFO | PerfReport: 7.71%, PulseAccumulating -- 2.56e+00 / 3.15e+00, 4.67e-01 / 3.30e-01 | 20260505 06:18:40.177 | INFO | PerfReport: 4.13%, Metrics -- 1.37e+00 / 1.76e+00, 3.91e-01 / 1.96e-01 | 20260505 06:18:40.177 | INFO | PerfReport: 1.50%, PulseSerializing -- 4.99e-01 / 5.86e-01, 9.90e-02 / 3.30e-02 | 20260505 06:18:40.177 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 06:18:40.177 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:18:40.177 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:18:40.177 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.26e+01 / 0.00e+00 | 20260505 06:18:40.177 | INFO | PerfReport: 22.05%, Upload -- 2.24e+01 / 0.00e+00, 7.81e+00 / 0.00e+00 | 20260505 06:18:40.177 | INFO | PerfReport: 21.51%, Download -- 2.18e+01 / 0.00e+00, 7.78e+00 / 0.00e+00 | 20260505 06:18:40.177 | INFO | PerfReport: 32.70%, Compute -- 3.32e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 06:18:40.177 | INFO | PerfReport: 1.39%, LockUpload -- 1.42e+00 / 0.00e+00, 6.75e+00 / 0.00e+00 | 20260505 06:18:40.177 | INFO | PerfReport: 0.69%, LockDownload -- 7.02e-01 / 0.00e+00, 1.91e+00 / 0.00e+00 | 20260505 06:18:40.177 | INFO | PerfReport: 21.65%, LockCompute -- 2.20e+01 / 0.00e+00, 1.67e+01 / 0.00e+00 | 20260505 06:18:40.177 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.59e-02 / 0.00e+00 | 20260505 06:18:40.290 | INFO | Starting a round of Baz IO | 20260505 06:18:43.557 | INFO | Populating RTMetrics report | 20260505 06:18:43.557 | INFO | RTMetrics report populated | 20260505 06:18:43.557 | INFO | Opening tmp RTMetrics json file | 20260505 06:18:43.581 | INFO | Serializing RTMetrics to json | 20260505 06:18:43.584 | INFO | Serialization done, swapping tmp file now... | 20260505 06:18:43.588 | INFO | RTMetrics done | 20260505 06:18:43.680 | INFO | Finished a round of Baz IO: Wrote 2.362 GiB in 3.389 seconds | 20260505 06:18:43.680 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.94% / 0.05% / 66.01% | 20260505 06:18:43.680 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.41576% / 7.15256e-05% / 4.76837e-05% | 20260505 06:18:43.680 | INFO | Baz IO overall: | 20260505 06:18:43.680 | INFO | BytesWritten : 510160392239 | 20260505 06:18:43.680 | INFO | NumEvents : 241031717756 | 20260505 06:18:45.314 | INFO | Analyzing chunk frames = [999424,999936) | 20260505 06:18:45.314 | INFO | MemInfo: { | 20260505 06:18:45.314 | INFO | "gpu" : " 17 GB", | 20260505 06:18:45.314 | INFO | "huge" : "", | 20260505 06:18:45.314 | INFO | "huge_shared" : " 27 GB", | 20260505 06:18:45.314 | INFO | "malloc" : " 14 GB", | 20260505 06:18:45.314 | INFO | "pinned" : " 29 GB" | 20260505 06:18:45.314 | INFO | } | 20260505 06:18:45.314 | INFO | Perf report for the last 5.1389 seconds: | 20260505 06:18:45.314 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:18:45.314 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:45.314 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:45.314 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:45.314 | INFO | ANALYSIS : 64.7%, 2.9, 98.8ms, 0 | 20260505 06:18:45.314 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:45.314 | INFO | RT_METRICS : 46.5%, 1.0, 24.4ms, 0 | 20260505 06:18:45.314 | INFO | PRE_HQ : 63.6%, 1.9, 64.1ms, 0 | 20260505 06:18:45.314 | INFO | BAZWRITER : 65.3%, 4.1, 136.9ms, 0 | 20260505 06:18:45.314 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 06:18:45.315 | INFO | Uploaded 36GiB to GPU in 2.14309s (16.7982GiB/s) | 20260505 06:18:45.315 | INFO | Downloaded 38GiB to GPU in 2.31838s (16.3908GiB/s) | 20260505 06:18:50.453 | INFO | Analyzing chunk frames = [999936,1000448) | 20260505 06:18:50.453 | INFO | MemInfo: { | 20260505 06:18:50.453 | INFO | "gpu" : " 17 GB", | 20260505 06:18:50.453 | INFO | "huge" : "", | 20260505 06:18:50.453 | INFO | "huge_shared" : " 27 GB", | 20260505 06:18:50.453 | INFO | "malloc" : " 14 GB", | 20260505 06:18:50.453 | INFO | "pinned" : " 29 GB" | 20260505 06:18:50.453 | INFO | } | 20260505 06:18:50.454 | INFO | Perf report for the last 5.13942 seconds: | 20260505 06:18:50.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:18:50.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:50.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:50.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:50.454 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 06:18:50.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:50.454 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:50.454 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 06:18:50.454 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:50.454 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:50.454 | INFO | Uploaded 36GiB to GPU in 2.11742s (17.0018GiB/s) | 20260505 06:18:50.454 | INFO | Downloaded 34GiB to GPU in 2.04208s (16.6497GiB/s) | 20260505 06:18:55.593 | INFO | Analyzing chunk frames = [1000448,1000960) | 20260505 06:18:55.593 | INFO | MemInfo: { | 20260505 06:18:55.593 | INFO | "gpu" : " 17 GB", | 20260505 06:18:55.593 | INFO | "huge" : "", | 20260505 06:18:55.593 | INFO | "huge_shared" : " 27 GB", | 20260505 06:18:55.593 | INFO | "malloc" : " 14 GB", | 20260505 06:18:55.593 | INFO | "pinned" : " 29 GB" | 20260505 06:18:55.593 | INFO | } | 20260505 06:18:55.593 | INFO | Perf report for the last 5.13958 seconds: | 20260505 06:18:55.593 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:18:55.593 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:55.593 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:55.593 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:55.593 | INFO | ANALYSIS : 64.1%, 3.0, 99.8ms, 0 | 20260505 06:18:55.593 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:55.593 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:55.593 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 06:18:55.593 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:55.593 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:18:55.594 | INFO | Uploaded 36GiB to GPU in 2.16115s (16.6578GiB/s) | 20260505 06:18:55.594 | INFO | Downloaded 34GiB to GPU in 2.04061s (16.6617GiB/s) | 20260505 06:19:00.732 | INFO | Analyzing chunk frames = [1000960,1001472) | 20260505 06:19:00.732 | INFO | MemInfo: { | 20260505 06:19:00.732 | INFO | "gpu" : " 17 GB", | 20260505 06:19:00.732 | INFO | "huge" : "", | 20260505 06:19:00.732 | INFO | "huge_shared" : " 27 GB", | 20260505 06:19:00.732 | INFO | "malloc" : " 14 GB", | 20260505 06:19:00.732 | INFO | "pinned" : " 29 GB" | 20260505 06:19:00.732 | INFO | } | 20260505 06:19:00.732 | INFO | Perf report for the last 5.1393 seconds: | 20260505 06:19:00.732 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:19:00.732 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:00.732 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:00.732 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:00.733 | INFO | ANALYSIS : 64.0%, 3.0, 99.4ms, 0 | 20260505 06:19:00.733 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:00.733 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:00.733 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 06:19:00.733 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:00.733 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:00.733 | INFO | Uploaded 36GiB to GPU in 2.16975s (16.5918GiB/s) | 20260505 06:19:00.733 | INFO | Downloaded 34GiB to GPU in 2.04706s (16.6092GiB/s) | 20260505 06:19:05.871 | INFO | Analyzing chunk frames = [1001472,1001984) | 20260505 06:19:05.871 | INFO | MemInfo: { | 20260505 06:19:05.871 | INFO | "gpu" : " 17 GB", | 20260505 06:19:05.871 | INFO | "huge" : "", | 20260505 06:19:05.871 | INFO | "huge_shared" : " 27 GB", | 20260505 06:19:05.872 | INFO | "malloc" : " 14 GB", | 20260505 06:19:05.872 | INFO | "pinned" : " 29 GB" | 20260505 06:19:05.872 | INFO | } | 20260505 06:19:05.872 | INFO | Perf report for the last 5.13911 seconds: | 20260505 06:19:05.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:19:05.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:05.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:05.872 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:05.872 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 06:19:05.872 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:05.872 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:05.872 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260505 06:19:05.872 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:05.872 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:05.872 | INFO | Uploaded 36GiB to GPU in 2.12222s (16.9634GiB/s) | 20260505 06:19:05.872 | INFO | Downloaded 34GiB to GPU in 2.04596s (16.6181GiB/s) | 20260505 06:19:11.012 | INFO | Analyzing chunk frames = [1001984,1002496) | 20260505 06:19:11.012 | INFO | MemInfo: { | 20260505 06:19:11.012 | INFO | "gpu" : " 17 GB", | 20260505 06:19:11.012 | INFO | "huge" : "", | 20260505 06:19:11.012 | INFO | "huge_shared" : " 27 GB", | 20260505 06:19:11.012 | INFO | "malloc" : " 14 GB", | 20260505 06:19:11.013 | INFO | "pinned" : " 29 GB" | 20260505 06:19:11.013 | INFO | } | 20260505 06:19:11.013 | INFO | Perf report for the last 5.14104 seconds: | 20260505 06:19:11.013 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:19:11.013 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:11.013 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:11.013 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:11.013 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 06:19:11.013 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:11.013 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:11.013 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 06:19:11.013 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:11.013 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:11.013 | INFO | Uploaded 36GiB to GPU in 2.15661s (16.6928GiB/s) | 20260505 06:19:11.013 | INFO | Downloaded 34GiB to GPU in 2.05688s (16.5299GiB/s) | 20260505 06:19:16.150 | INFO | Analyzing chunk frames = [1002496,1003008) | 20260505 06:19:16.150 | INFO | MemInfo: { | 20260505 06:19:16.150 | INFO | "gpu" : " 17 GB", | 20260505 06:19:16.150 | INFO | "huge" : "", | 20260505 06:19:16.150 | INFO | "huge_shared" : " 27 GB", | 20260505 06:19:16.150 | INFO | "malloc" : " 14 GB", | 20260505 06:19:16.150 | INFO | "pinned" : " 29 GB" | 20260505 06:19:16.150 | INFO | } | 20260505 06:19:16.150 | INFO | Perf report for the last 5.13775 seconds: | 20260505 06:19:16.150 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:19:16.150 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:16.150 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:16.150 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:16.150 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 06:19:16.150 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:16.151 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:16.151 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 06:19:16.151 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:16.151 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:16.151 | INFO | Uploaded 36GiB to GPU in 2.16084s (16.6602GiB/s) | 20260505 06:19:16.151 | INFO | Downloaded 34GiB to GPU in 2.14269s (15.8679GiB/s) | 20260505 06:19:21.294 | INFO | Analyzing chunk frames = [1003008,1003520) | 20260505 06:19:21.294 | INFO | MemInfo: { | 20260505 06:19:21.294 | INFO | "gpu" : " 17 GB", | 20260505 06:19:21.294 | INFO | "huge" : "", | 20260505 06:19:21.294 | INFO | "huge_shared" : " 27 GB", | 20260505 06:19:21.294 | INFO | "malloc" : " 14 GB", | 20260505 06:19:21.294 | INFO | "pinned" : " 29 GB" | 20260505 06:19:21.294 | INFO | } | 20260505 06:19:21.294 | INFO | Perf report for the last 5.14342 seconds: | 20260505 06:19:21.294 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:19:21.294 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:21.294 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:21.294 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:21.294 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 06:19:21.294 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:21.294 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:21.294 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 06:19:21.294 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:21.294 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:21.294 | INFO | Uploaded 36GiB to GPU in 2.10487s (17.1032GiB/s) | 20260505 06:19:21.294 | INFO | Downloaded 34GiB to GPU in 2.0588s (16.5145GiB/s) | 20260505 06:19:21.410 | INFO | Starting a round of Baz IO | 20260505 06:19:24.677 | INFO | Populating RTMetrics report | 20260505 06:19:24.677 | INFO | RTMetrics report populated | 20260505 06:19:24.677 | INFO | Opening tmp RTMetrics json file | 20260505 06:19:24.698 | INFO | Serializing RTMetrics to json | 20260505 06:19:24.699 | INFO | Serialization done, swapping tmp file now... | 20260505 06:19:24.699 | INFO | RTMetrics done | 20260505 06:19:24.792 | INFO | Finished a round of Baz IO: Wrote 2.361 GiB in 3.381 seconds | 20260505 06:19:24.792 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.88% / 0.05% / 66.06% | 20260505 06:19:24.792 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.38091% / 9.53674e-05% / 6.75519e-05% | 20260505 06:19:24.792 | INFO | Baz IO overall: | 20260505 06:19:24.792 | INFO | BytesWritten : 512695114732 | 20260505 06:19:24.792 | INFO | NumEvents : 242243762854 | 20260505 06:19:26.429 | INFO | Analyzing chunk frames = [1003520,1004032) | 20260505 06:19:26.429 | INFO | MemInfo: { | 20260505 06:19:26.429 | INFO | "gpu" : " 17 GB", | 20260505 06:19:26.429 | INFO | "huge" : "", | 20260505 06:19:26.429 | INFO | "huge_shared" : " 27 GB", | 20260505 06:19:26.429 | INFO | "malloc" : " 14 GB", | 20260505 06:19:26.429 | INFO | "pinned" : " 29 GB" | 20260505 06:19:26.429 | INFO | } | 20260505 06:19:26.429 | INFO | Perf report for the last 5.13513 seconds: | 20260505 06:19:26.429 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:19:26.429 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:26.429 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:26.429 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:26.429 | INFO | ANALYSIS : 64.7%, 2.9, 98.7ms, 0 | 20260505 06:19:26.429 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:26.429 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260505 06:19:26.429 | INFO | PRE_HQ : 63.7%, 1.9, 64.0ms, 0 | 20260505 06:19:26.429 | INFO | BAZWRITER : 65.2%, 4.1, 136.3ms, 0 | 20260505 06:19:26.429 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 06:19:26.429 | INFO | Uploaded 36GiB to GPU in 2.1832s (16.4896GiB/s) | 20260505 06:19:26.429 | INFO | Downloaded 38GiB to GPU in 2.32145s (16.3691GiB/s) | 20260505 06:19:31.568 | INFO | Analyzing chunk frames = [1004032,1004544) | 20260505 06:19:31.568 | INFO | MemInfo: { | 20260505 06:19:31.568 | INFO | "gpu" : " 17 GB", | 20260505 06:19:31.568 | INFO | "huge" : "", | 20260505 06:19:31.568 | INFO | "huge_shared" : " 27 GB", | 20260505 06:19:31.568 | INFO | "malloc" : " 14 GB", | 20260505 06:19:31.568 | INFO | "pinned" : " 29 GB" | 20260505 06:19:31.568 | INFO | } | 20260505 06:19:31.568 | INFO | Perf report for the last 5.13937 seconds: | 20260505 06:19:31.568 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:19:31.568 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:31.568 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:31.568 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:31.568 | INFO | ANALYSIS : 64.5%, 3.0, 100.6ms, 0 | 20260505 06:19:31.568 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:31.568 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:31.568 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 06:19:31.568 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:31.569 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:31.569 | INFO | Uploaded 36GiB to GPU in 2.14229s (16.8045GiB/s) | 20260505 06:19:31.569 | INFO | Downloaded 34GiB to GPU in 2.04669s (16.6122GiB/s) | 20260505 06:19:36.707 | INFO | Analyzing chunk frames = [1004544,1005056) | 20260505 06:19:36.707 | INFO | MemInfo: { | 20260505 06:19:36.707 | INFO | "gpu" : " 17 GB", | 20260505 06:19:36.707 | INFO | "huge" : "", | 20260505 06:19:36.707 | INFO | "huge_shared" : " 27 GB", | 20260505 06:19:36.707 | INFO | "malloc" : " 14 GB", | 20260505 06:19:36.707 | INFO | "pinned" : " 29 GB" | 20260505 06:19:36.708 | INFO | } | 20260505 06:19:36.708 | INFO | Perf report for the last 5.13923 seconds: | 20260505 06:19:36.708 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:19:36.708 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:36.708 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:36.708 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:36.708 | INFO | ANALYSIS : 63.9%, 3.0, 99.4ms, 0 | 20260505 06:19:36.708 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:36.708 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:36.708 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 06:19:36.708 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:36.708 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:36.708 | INFO | Uploaded 36GiB to GPU in 2.1474s (16.7645GiB/s) | 20260505 06:19:36.708 | INFO | Downloaded 34GiB to GPU in 2.03483s (16.709GiB/s) | 20260505 06:19:41.847 | INFO | Analyzing chunk frames = [1005056,1005568) | 20260505 06:19:41.847 | INFO | MemInfo: { | 20260505 06:19:41.847 | INFO | "gpu" : " 17 GB", | 20260505 06:19:41.847 | INFO | "huge" : "", | 20260505 06:19:41.847 | INFO | "huge_shared" : " 27 GB", | 20260505 06:19:41.847 | INFO | "malloc" : " 14 GB", | 20260505 06:19:41.847 | INFO | "pinned" : " 29 GB" | 20260505 06:19:41.847 | INFO | } | 20260505 06:19:41.847 | INFO | Perf report for the last 5.13937 seconds: | 20260505 06:19:41.847 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:19:41.847 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:41.847 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:41.847 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:41.847 | INFO | ANALYSIS : 63.3%, 3.0, 98.3ms, 0 | 20260505 06:19:41.847 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:41.847 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:41.847 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260505 06:19:41.847 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:41.847 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:41.847 | INFO | Uploaded 36GiB to GPU in 2.1237s (16.9516GiB/s) | 20260505 06:19:41.848 | INFO | Downloaded 34GiB to GPU in 2.05723s (16.5271GiB/s) | 20260505 06:19:46.987 | INFO | Analyzing chunk frames = [1005568,1006080) | 20260505 06:19:46.987 | INFO | MemInfo: { | 20260505 06:19:46.987 | INFO | "gpu" : " 17 GB", | 20260505 06:19:46.987 | INFO | "huge" : "", | 20260505 06:19:46.987 | INFO | "huge_shared" : " 27 GB", | 20260505 06:19:46.987 | INFO | "malloc" : " 14 GB", | 20260505 06:19:46.987 | INFO | "pinned" : " 29 GB" | 20260505 06:19:46.987 | INFO | } | 20260505 06:19:46.987 | INFO | Perf report for the last 5.13981 seconds: | 20260505 06:19:46.987 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:19:46.987 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:46.987 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:46.987 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:46.987 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 06:19:46.987 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:46.987 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:46.987 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 06:19:46.987 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:46.987 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:46.987 | INFO | Uploaded 36GiB to GPU in 2.1472s (16.766GiB/s) | 20260505 06:19:46.987 | INFO | Downloaded 34GiB to GPU in 2.05037s (16.5824GiB/s) | 20260505 06:19:52.127 | INFO | Analyzing chunk frames = [1006080,1006592) | 20260505 06:19:52.127 | INFO | MemInfo: { | 20260505 06:19:52.127 | INFO | "gpu" : " 17 GB", | 20260505 06:19:52.127 | INFO | "huge" : "", | 20260505 06:19:52.127 | INFO | "huge_shared" : " 27 GB", | 20260505 06:19:52.127 | INFO | "malloc" : " 14 GB", | 20260505 06:19:52.127 | INFO | "pinned" : " 29 GB" | 20260505 06:19:52.127 | INFO | } | 20260505 06:19:52.127 | INFO | Perf report for the last 5.14025 seconds: | 20260505 06:19:52.127 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:19:52.127 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:52.127 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:52.127 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:52.127 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 06:19:52.127 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:52.127 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:52.127 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 06:19:52.127 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:52.127 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:52.127 | INFO | Uploaded 36GiB to GPU in 2.15922s (16.6727GiB/s) | 20260505 06:19:52.127 | INFO | Downloaded 34GiB to GPU in 2.06072s (16.4991GiB/s) | 20260505 06:19:57.265 | INFO | Analyzing chunk frames = [1006592,1007104) | 20260505 06:19:57.265 | INFO | MemInfo: { | 20260505 06:19:57.265 | INFO | "gpu" : " 17 GB", | 20260505 06:19:57.265 | INFO | "huge" : "", | 20260505 06:19:57.265 | INFO | "huge_shared" : " 27 GB", | 20260505 06:19:57.265 | INFO | "malloc" : " 14 GB", | 20260505 06:19:57.265 | INFO | "pinned" : " 29 GB" | 20260505 06:19:57.265 | INFO | } | 20260505 06:19:57.266 | INFO | Perf report for the last 5.13849 seconds: | 20260505 06:19:57.266 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:19:57.266 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:57.266 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:57.266 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:57.266 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 06:19:57.266 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:57.266 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:57.266 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260505 06:19:57.266 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:57.266 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:19:57.266 | INFO | Uploaded 36GiB to GPU in 2.17773s (16.531GiB/s) | 20260505 06:19:57.266 | INFO | Downloaded 34GiB to GPU in 2.15169s (15.8015GiB/s) | 20260505 06:20:02.404 | INFO | Analyzing chunk frames = [1007104,1007616) | 20260505 06:20:02.404 | INFO | MemInfo: { | 20260505 06:20:02.404 | INFO | "gpu" : " 17 GB", | 20260505 06:20:02.404 | INFO | "huge" : "", | 20260505 06:20:02.404 | INFO | "huge_shared" : " 27 GB", | 20260505 06:20:02.404 | INFO | "malloc" : " 14 GB", | 20260505 06:20:02.404 | INFO | "pinned" : " 29 GB" | 20260505 06:20:02.404 | INFO | } | 20260505 06:20:02.404 | INFO | Perf report for the last 5.13855 seconds: | 20260505 06:20:02.404 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:20:02.404 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:02.404 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:02.404 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:02.404 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 06:20:02.404 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:02.404 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:02.404 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 06:20:02.404 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:02.404 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:02.404 | INFO | Uploaded 36GiB to GPU in 2.09607s (17.175GiB/s) | 20260505 06:20:02.405 | INFO | Downloaded 34GiB to GPU in 2.06862s (16.4361GiB/s) | 20260505 06:20:02.405 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 06:20:02.405 | INFO | Total mem usage: 15.3 GiB | 20260505 06:20:02.405 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 06:20:02.405 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.42%) | 20260505 06:20:02.405 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 06:20:02.405 | INFO | High Water Breakdown: | 20260505 06:20:02.405 | INFO | 60.3% : 8.68 GiB ( 92.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 06:20:02.405 | INFO | 39.6% : 5.7 GiB ( 85.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 06:20:02.405 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 06:20:02.405 | INFO | ------------------------------------------ | 20260505 06:20:02.405 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:20:02.405 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 06:20:02.405 | INFO | Total mem usage: 26.5 GiB | 20260505 06:20:02.405 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 06:20:02.405 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 06:20:02.405 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 06:20:02.405 | INFO | High Water Breakdown: | 20260505 06:20:02.405 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:20:02.405 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:20:02.405 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:20:02.405 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:20:02.405 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:20:02.405 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:20:02.405 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 06:20:02.405 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 06:20:02.405 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:20:02.405 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 06:20:02.405 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 06:20:02.405 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 06:20:02.405 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 06:20:02.405 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 06:20:02.405 | INFO | ------------------------------------------ | 20260505 06:20:02.405 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:20:02.405 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 06:20:02.405 | INFO | Total mem usage: 25.6 GiB | 20260505 06:20:02.405 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 06:20:02.405 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 06:20:02.405 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 06:20:02.405 | INFO | High Water Breakdown: | 20260505 06:20:02.405 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 06:20:02.405 | INFO | ------------------------------------------ | 20260505 06:20:02.405 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:20:02.405 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 06:20:02.405 | INFO | Total mem usage: 15.7 GiB | 20260505 06:20:02.405 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 06:20:02.405 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 06:20:02.405 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260505 06:20:02.405 | INFO | High Water Breakdown: | 20260505 06:20:02.405 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 06:20:02.405 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 06:20:02.405 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 06:20:02.405 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 06:20:02.405 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 06:20:02.405 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 06:20:02.405 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 06:20:02.405 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 06:20:02.405 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 06:20:02.405 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:20:02.405 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 06:20:02.405 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:20:02.405 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 06:20:02.405 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 06:20:02.405 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 06:20:02.405 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 06:20:02.405 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:20:02.405 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 06:20:02.406 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:20:02.406 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:20:02.406 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:20:02.406 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 06:20:02.406 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 06:20:02.406 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:20:02.406 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 06:20:02.406 | INFO | ------------------------------------------ | 20260505 06:20:02.406 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:20:02.406 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 06:20:02.406 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:20:02.406 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:20:02.406 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.57e+01, 1.38e+01 / 1.58e+00 | 20260505 06:20:02.406 | INFO | PerfReport: 0.28%, AggregateStats -- 9.42e-02 / 1.05e-01, 1.27e-02 / 1.43e-02 | 20260505 06:20:02.406 | INFO | PerfReport: 11.57%, Baseline -- 3.84e+00 / 4.48e+00, 6.34e-01 / 3.01e-01 | 20260505 06:20:02.406 | INFO | PerfReport: 3.30%, Histogram -- 1.10e+00 / 1.50e+00, 2.17e-01 / 1.17e-01 | 20260505 06:20:02.406 | INFO | PerfReport: 15.46%, DME -- 4.10e+01 / 4.51e+01, 4.29e+00 / 1.20e+00 | 20260505 06:20:02.406 | INFO | PerfReport: 56.04%, FrameLabeling -- 1.86e+01 / 1.91e+01, 2.06e+00 / 9.13e-01 | 20260505 06:20:02.406 | INFO | PerfReport: 7.71%, PulseAccumulating -- 2.56e+00 / 3.17e+00, 4.66e-01 / 3.24e-01 | 20260505 06:20:02.406 | INFO | PerfReport: 4.12%, Metrics -- 1.37e+00 / 1.75e+00, 3.88e-01 / 1.92e-01 | 20260505 06:20:02.406 | INFO | PerfReport: 1.50%, PulseSerializing -- 4.99e-01 / 5.80e-01, 9.87e-02 / 3.84e-02 | 20260505 06:20:02.406 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 06:20:02.406 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:20:02.406 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:20:02.406 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.25e+01 / 0.00e+00 | 20260505 06:20:02.406 | INFO | PerfReport: 22.00%, Upload -- 2.23e+01 / 0.00e+00, 7.72e+00 / 0.00e+00 | 20260505 06:20:02.406 | INFO | PerfReport: 21.51%, Download -- 2.18e+01 / 0.00e+00, 7.73e+00 / 0.00e+00 | 20260505 06:20:02.406 | INFO | PerfReport: 32.67%, Compute -- 3.32e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 06:20:02.406 | INFO | PerfReport: 1.44%, LockUpload -- 1.46e+00 / 0.00e+00, 6.81e+00 / 0.00e+00 | 20260505 06:20:02.406 | INFO | PerfReport: 0.69%, LockDownload -- 7.01e-01 / 0.00e+00, 1.94e+00 / 0.00e+00 | 20260505 06:20:02.406 | INFO | PerfReport: 21.68%, LockCompute -- 2.20e+01 / 0.00e+00, 1.67e+01 / 0.00e+00 | 20260505 06:20:02.406 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 1.88e-02 / 0.00e+00 | 20260505 06:20:02.521 | INFO | Starting a round of Baz IO | 20260505 06:20:05.793 | INFO | Populating RTMetrics report | 20260505 06:20:05.794 | INFO | RTMetrics report populated | 20260505 06:20:05.794 | INFO | Opening tmp RTMetrics json file | 20260505 06:20:05.796 | INFO | Serializing RTMetrics to json | 20260505 06:20:05.819 | INFO | Serialization done, swapping tmp file now... | 20260505 06:20:05.826 | INFO | RTMetrics done | 20260505 06:20:05.906 | INFO | Finished a round of Baz IO: Wrote 2.359 GiB in 3.385 seconds | 20260505 06:20:05.906 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.83% / 0.05% / 66.12% | 20260505 06:20:05.907 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.33801% / 7.15256e-05% / 6.35783e-05% | 20260505 06:20:05.907 | INFO | Baz IO overall: | 20260505 06:20:05.907 | INFO | BytesWritten : 515228355945 | 20260505 06:20:05.907 | INFO | NumEvents : 243454640689 | 20260505 06:20:07.544 | INFO | Analyzing chunk frames = [1007616,1008128) | 20260505 06:20:07.544 | INFO | MemInfo: { | 20260505 06:20:07.544 | INFO | "gpu" : " 17 GB", | 20260505 06:20:07.544 | INFO | "huge" : "", | 20260505 06:20:07.544 | INFO | "huge_shared" : " 27 GB", | 20260505 06:20:07.544 | INFO | "malloc" : " 14 GB", | 20260505 06:20:07.544 | INFO | "pinned" : " 29 GB" | 20260505 06:20:07.544 | INFO | } | 20260505 06:20:07.544 | INFO | Perf report for the last 5.13983 seconds: | 20260505 06:20:07.544 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:20:07.544 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:07.544 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:07.544 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:07.544 | INFO | ANALYSIS : 64.8%, 2.9, 98.9ms, 0 | 20260505 06:20:07.544 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:07.544 | INFO | RT_METRICS : 46.6%, 1.0, 24.4ms, 0 | 20260505 06:20:07.544 | INFO | PRE_HQ : 63.8%, 1.9, 64.2ms, 0 | 20260505 06:20:07.544 | INFO | BAZWRITER : 65.2%, 4.1, 136.4ms, 0 | 20260505 06:20:07.544 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 06:20:07.544 | INFO | Uploaded 36GiB to GPU in 2.18457s (16.4792GiB/s) | 20260505 06:20:07.544 | INFO | Downloaded 38GiB to GPU in 2.33862s (16.2489GiB/s) | 20260505 06:20:12.683 | INFO | Analyzing chunk frames = [1008128,1008640) | 20260505 06:20:12.683 | INFO | MemInfo: { | 20260505 06:20:12.683 | INFO | "gpu" : " 17 GB", | 20260505 06:20:12.683 | INFO | "huge" : "", | 20260505 06:20:12.683 | INFO | "huge_shared" : " 27 GB", | 20260505 06:20:12.683 | INFO | "malloc" : " 14 GB", | 20260505 06:20:12.683 | INFO | "pinned" : " 29 GB" | 20260505 06:20:12.683 | INFO | } | 20260505 06:20:12.683 | INFO | Perf report for the last 5.13903 seconds: | 20260505 06:20:12.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:20:12.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:12.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:12.683 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:12.683 | INFO | ANALYSIS : 64.1%, 3.0, 100.0ms, 0 | 20260505 06:20:12.683 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:12.683 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:12.683 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 06:20:12.683 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:12.683 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:12.683 | INFO | Uploaded 36GiB to GPU in 2.11383s (17.0307GiB/s) | 20260505 06:20:12.683 | INFO | Downloaded 34GiB to GPU in 2.04077s (16.6604GiB/s) | 20260505 06:20:17.822 | INFO | Analyzing chunk frames = [1008640,1009152) | 20260505 06:20:17.822 | INFO | MemInfo: { | 20260505 06:20:17.822 | INFO | "gpu" : " 17 GB", | 20260505 06:20:17.822 | INFO | "huge" : "", | 20260505 06:20:17.822 | INFO | "huge_shared" : " 27 GB", | 20260505 06:20:17.822 | INFO | "malloc" : " 14 GB", | 20260505 06:20:17.822 | INFO | "pinned" : " 29 GB" | 20260505 06:20:17.822 | INFO | } | 20260505 06:20:17.822 | INFO | Perf report for the last 5.1391 seconds: | 20260505 06:20:17.822 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:20:17.822 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:17.822 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:17.822 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:17.822 | INFO | ANALYSIS : 64.0%, 3.0, 99.6ms, 0 | 20260505 06:20:17.822 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:17.822 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:17.822 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 06:20:17.822 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:17.822 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:17.823 | INFO | Uploaded 36GiB to GPU in 2.15414s (16.712GiB/s) | 20260505 06:20:17.823 | INFO | Downloaded 34GiB to GPU in 2.04091s (16.6592GiB/s) | 20260505 06:20:22.961 | INFO | Analyzing chunk frames = [1009152,1009664) | 20260505 06:20:22.961 | INFO | MemInfo: { | 20260505 06:20:22.961 | INFO | "gpu" : " 17 GB", | 20260505 06:20:22.961 | INFO | "huge" : "", | 20260505 06:20:22.961 | INFO | "huge_shared" : " 27 GB", | 20260505 06:20:22.961 | INFO | "malloc" : " 14 GB", | 20260505 06:20:22.961 | INFO | "pinned" : " 29 GB" | 20260505 06:20:22.961 | INFO | } | 20260505 06:20:22.962 | INFO | Perf report for the last 5.13939 seconds: | 20260505 06:20:22.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:20:22.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:22.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:22.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:22.962 | INFO | ANALYSIS : 64.0%, 3.0, 99.5ms, 0 | 20260505 06:20:22.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:22.962 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:22.962 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 06:20:22.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:22.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:22.962 | INFO | Uploaded 36GiB to GPU in 2.17805s (16.5285GiB/s) | 20260505 06:20:22.962 | INFO | Downloaded 34GiB to GPU in 2.04748s (16.6058GiB/s) | 20260505 06:20:28.101 | INFO | Analyzing chunk frames = [1009664,1010176) | 20260505 06:20:28.101 | INFO | MemInfo: { | 20260505 06:20:28.101 | INFO | "gpu" : " 17 GB", | 20260505 06:20:28.101 | INFO | "huge" : "", | 20260505 06:20:28.101 | INFO | "huge_shared" : " 27 GB", | 20260505 06:20:28.101 | INFO | "malloc" : " 14 GB", | 20260505 06:20:28.101 | INFO | "pinned" : " 29 GB" | 20260505 06:20:28.101 | INFO | } | 20260505 06:20:28.101 | INFO | Perf report for the last 5.13939 seconds: | 20260505 06:20:28.101 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:20:28.101 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:28.101 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:28.101 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:28.101 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 06:20:28.101 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:28.101 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:28.101 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 06:20:28.101 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:28.101 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:28.101 | INFO | Uploaded 36GiB to GPU in 2.15703s (16.6896GiB/s) | 20260505 06:20:28.101 | INFO | Downloaded 34GiB to GPU in 2.04935s (16.5906GiB/s) | 20260505 06:20:33.242 | INFO | Analyzing chunk frames = [1010176,1010688) | 20260505 06:20:33.242 | INFO | MemInfo: { | 20260505 06:20:33.242 | INFO | "gpu" : " 17 GB", | 20260505 06:20:33.242 | INFO | "huge" : "", | 20260505 06:20:33.242 | INFO | "huge_shared" : " 27 GB", | 20260505 06:20:33.242 | INFO | "malloc" : " 14 GB", | 20260505 06:20:33.242 | INFO | "pinned" : " 29 GB" | 20260505 06:20:33.242 | INFO | } | 20260505 06:20:33.242 | INFO | Perf report for the last 5.14084 seconds: | 20260505 06:20:33.242 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:20:33.242 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:33.242 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:33.242 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:33.242 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 06:20:33.242 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:33.242 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:33.242 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 06:20:33.242 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:33.242 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:33.242 | INFO | Uploaded 36GiB to GPU in 2.15015s (16.743GiB/s) | 20260505 06:20:33.242 | INFO | Downloaded 34GiB to GPU in 2.0451s (16.6251GiB/s) | 20260505 06:20:38.380 | INFO | Analyzing chunk frames = [1010688,1011200) | 20260505 06:20:38.380 | INFO | MemInfo: { | 20260505 06:20:38.380 | INFO | "gpu" : " 17 GB", | 20260505 06:20:38.380 | INFO | "huge" : "", | 20260505 06:20:38.380 | INFO | "huge_shared" : " 27 GB", | 20260505 06:20:38.380 | INFO | "malloc" : " 14 GB", | 20260505 06:20:38.380 | INFO | "pinned" : " 29 GB" | 20260505 06:20:38.380 | INFO | } | 20260505 06:20:38.380 | INFO | Perf report for the last 5.13787 seconds: | 20260505 06:20:38.380 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:20:38.380 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:38.380 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:38.380 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:38.380 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 06:20:38.380 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:38.380 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:38.380 | INFO | PRE_HQ : 32.5%, 1.0, 17.0ms, 0 | 20260505 06:20:38.380 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:38.380 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:38.380 | INFO | Uploaded 36GiB to GPU in 2.17878s (16.523GiB/s) | 20260505 06:20:38.381 | INFO | Downloaded 34GiB to GPU in 2.15682s (15.764GiB/s) | 20260505 06:20:43.519 | INFO | Analyzing chunk frames = [1011200,1011712) | 20260505 06:20:43.519 | INFO | MemInfo: { | 20260505 06:20:43.519 | INFO | "gpu" : " 17 GB", | 20260505 06:20:43.519 | INFO | "huge" : "", | 20260505 06:20:43.519 | INFO | "huge_shared" : " 27 GB", | 20260505 06:20:43.519 | INFO | "malloc" : " 14 GB", | 20260505 06:20:43.519 | INFO | "pinned" : " 29 GB" | 20260505 06:20:43.519 | INFO | } | 20260505 06:20:43.519 | INFO | Perf report for the last 5.13937 seconds: | 20260505 06:20:43.519 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:20:43.519 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:43.519 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:43.519 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:43.519 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 06:20:43.519 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:43.519 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:43.519 | INFO | PRE_HQ : 31.7%, 1.0, 16.4ms, 0 | 20260505 06:20:43.519 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:43.519 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:43.519 | INFO | Uploaded 36GiB to GPU in 2.09153s (17.2122GiB/s) | 20260505 06:20:43.519 | INFO | Downloaded 34GiB to GPU in 2.07025s (16.4231GiB/s) | 20260505 06:20:43.642 | INFO | Starting a round of Baz IO | 20260505 06:20:46.930 | INFO | Populating RTMetrics report | 20260505 06:20:46.930 | INFO | RTMetrics report populated | 20260505 06:20:46.930 | INFO | Opening tmp RTMetrics json file | 20260505 06:20:46.954 | INFO | Serializing RTMetrics to json | 20260505 06:20:46.958 | INFO | Serialization done, swapping tmp file now... | 20260505 06:20:46.959 | INFO | RTMetrics done | 20260505 06:20:47.052 | INFO | Finished a round of Baz IO: Wrote 2.361 GiB in 3.41 seconds | 20260505 06:20:47.052 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.78% / 0.05% / 66.17% | 20260505 06:20:47.052 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.32364% / 8.74201e-05% / 5.96046e-05% | 20260505 06:20:47.052 | INFO | Baz IO overall: | 20260505 06:20:47.053 | INFO | BytesWritten : 517763763000 | 20260505 06:20:47.053 | INFO | NumEvents : 244666691054 | 20260505 06:20:48.658 | INFO | Analyzing chunk frames = [1011712,1012224) | 20260505 06:20:48.658 | INFO | MemInfo: { | 20260505 06:20:48.658 | INFO | "gpu" : " 17 GB", | 20260505 06:20:48.658 | INFO | "huge" : "", | 20260505 06:20:48.658 | INFO | "huge_shared" : " 27 GB", | 20260505 06:20:48.658 | INFO | "malloc" : " 14 GB", | 20260505 06:20:48.658 | INFO | "pinned" : " 29 GB" | 20260505 06:20:48.658 | INFO | } | 20260505 06:20:48.659 | INFO | Perf report for the last 5.13948 seconds: | 20260505 06:20:48.659 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:20:48.659 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:48.659 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:48.659 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:48.659 | INFO | ANALYSIS : 65.2%, 2.9, 98.9ms, 0 | 20260505 06:20:48.659 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:48.659 | INFO | RT_METRICS : 46.6%, 1.0, 24.5ms, 0 | 20260505 06:20:48.659 | INFO | PRE_HQ : 64.2%, 1.9, 64.9ms, 0 | 20260505 06:20:48.659 | INFO | BAZWRITER : 65.7%, 4.1, 138.0ms, 0 | 20260505 06:20:48.659 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 06:20:48.659 | INFO | Uploaded 36GiB to GPU in 2.18199s (16.4987GiB/s) | 20260505 06:20:48.659 | INFO | Downloaded 38GiB to GPU in 2.31985s (16.3803GiB/s) | 20260505 06:20:53.798 | INFO | Analyzing chunk frames = [1012224,1012736) | 20260505 06:20:53.798 | INFO | MemInfo: { | 20260505 06:20:53.798 | INFO | "gpu" : " 17 GB", | 20260505 06:20:53.798 | INFO | "huge" : "", | 20260505 06:20:53.798 | INFO | "huge_shared" : " 27 GB", | 20260505 06:20:53.798 | INFO | "malloc" : " 14 GB", | 20260505 06:20:53.798 | INFO | "pinned" : " 29 GB" | 20260505 06:20:53.798 | INFO | } | 20260505 06:20:53.798 | INFO | Perf report for the last 5.13909 seconds: | 20260505 06:20:53.798 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:20:53.798 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:53.798 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:53.798 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:53.798 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 06:20:53.798 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:53.798 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:53.798 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 06:20:53.798 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:53.798 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:53.798 | INFO | Uploaded 36GiB to GPU in 2.11768s (16.9997GiB/s) | 20260505 06:20:53.798 | INFO | Downloaded 34GiB to GPU in 2.03701s (16.6911GiB/s) | 20260505 06:20:58.937 | INFO | Analyzing chunk frames = [1012736,1013248) | 20260505 06:20:58.937 | INFO | MemInfo: { | 20260505 06:20:58.937 | INFO | "gpu" : " 17 GB", | 20260505 06:20:58.937 | INFO | "huge" : "", | 20260505 06:20:58.937 | INFO | "huge_shared" : " 27 GB", | 20260505 06:20:58.937 | INFO | "malloc" : " 14 GB", | 20260505 06:20:58.937 | INFO | "pinned" : " 29 GB" | 20260505 06:20:58.937 | INFO | } | 20260505 06:20:58.937 | INFO | Perf report for the last 5.1398 seconds: | 20260505 06:20:58.937 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:20:58.937 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:58.937 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:58.937 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:58.937 | INFO | ANALYSIS : 63.9%, 3.0, 99.5ms, 0 | 20260505 06:20:58.937 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:58.937 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:58.937 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 06:20:58.938 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:58.938 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:20:58.938 | INFO | Uploaded 36GiB to GPU in 2.16346s (16.64GiB/s) | 20260505 06:20:58.938 | INFO | Downloaded 34GiB to GPU in 2.0252s (16.7884GiB/s) | 20260505 06:21:04.076 | INFO | Analyzing chunk frames = [1013248,1013760) | 20260505 06:21:04.076 | INFO | MemInfo: { | 20260505 06:21:04.076 | INFO | "gpu" : " 17 GB", | 20260505 06:21:04.076 | INFO | "huge" : "", | 20260505 06:21:04.076 | INFO | "huge_shared" : " 27 GB", | 20260505 06:21:04.076 | INFO | "malloc" : " 14 GB", | 20260505 06:21:04.076 | INFO | "pinned" : " 29 GB" | 20260505 06:21:04.076 | INFO | } | 20260505 06:21:04.077 | INFO | Perf report for the last 5.13914 seconds: | 20260505 06:21:04.077 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:21:04.077 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:04.077 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:04.077 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:04.077 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 06:21:04.077 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:04.077 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:04.077 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 06:21:04.077 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:04.077 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:04.077 | INFO | Uploaded 36GiB to GPU in 2.14326s (16.7968GiB/s) | 20260505 06:21:04.077 | INFO | Downloaded 34GiB to GPU in 2.04809s (16.6009GiB/s) | 20260505 06:21:09.216 | INFO | Analyzing chunk frames = [1013760,1014272) | 20260505 06:21:09.216 | INFO | MemInfo: { | 20260505 06:21:09.216 | INFO | "gpu" : " 17 GB", | 20260505 06:21:09.216 | INFO | "huge" : "", | 20260505 06:21:09.216 | INFO | "huge_shared" : " 27 GB", | 20260505 06:21:09.216 | INFO | "malloc" : " 14 GB", | 20260505 06:21:09.216 | INFO | "pinned" : " 29 GB" | 20260505 06:21:09.216 | INFO | } | 20260505 06:21:09.216 | INFO | Perf report for the last 5.13906 seconds: | 20260505 06:21:09.216 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:21:09.216 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:09.216 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:09.216 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:09.216 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 06:21:09.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:09.216 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:09.216 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 06:21:09.216 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:09.216 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:09.216 | INFO | Uploaded 36GiB to GPU in 2.1687s (16.5998GiB/s) | 20260505 06:21:09.216 | INFO | Downloaded 34GiB to GPU in 2.04662s (16.6127GiB/s) | 20260505 06:21:14.357 | INFO | Analyzing chunk frames = [1014272,1014784) | 20260505 06:21:14.357 | INFO | MemInfo: { | 20260505 06:21:14.357 | INFO | "gpu" : " 17 GB", | 20260505 06:21:14.357 | INFO | "huge" : "", | 20260505 06:21:14.357 | INFO | "huge_shared" : " 27 GB", | 20260505 06:21:14.357 | INFO | "malloc" : " 14 GB", | 20260505 06:21:14.357 | INFO | "pinned" : " 29 GB" | 20260505 06:21:14.357 | INFO | } | 20260505 06:21:14.357 | INFO | Perf report for the last 5.14138 seconds: | 20260505 06:21:14.357 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:21:14.357 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:14.357 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:14.357 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:14.357 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 06:21:14.357 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:14.357 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:14.357 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 06:21:14.357 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:14.357 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:14.357 | INFO | Uploaded 36GiB to GPU in 2.15562s (16.7006GiB/s) | 20260505 06:21:14.357 | INFO | Downloaded 34GiB to GPU in 2.04584s (16.6191GiB/s) | 20260505 06:21:19.494 | INFO | Analyzing chunk frames = [1014784,1015296) | 20260505 06:21:19.494 | INFO | MemInfo: { | 20260505 06:21:19.494 | INFO | "gpu" : " 17 GB", | 20260505 06:21:19.494 | INFO | "huge" : "", | 20260505 06:21:19.494 | INFO | "huge_shared" : " 27 GB", | 20260505 06:21:19.494 | INFO | "malloc" : " 14 GB", | 20260505 06:21:19.494 | INFO | "pinned" : " 29 GB" | 20260505 06:21:19.494 | INFO | } | 20260505 06:21:19.494 | INFO | Perf report for the last 5.13731 seconds: | 20260505 06:21:19.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:21:19.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:19.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:19.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:19.495 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 06:21:19.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:19.495 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:19.495 | INFO | PRE_HQ : 32.4%, 1.0, 16.9ms, 0 | 20260505 06:21:19.495 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:19.495 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:19.495 | INFO | Uploaded 36GiB to GPU in 2.1578s (16.6837GiB/s) | 20260505 06:21:19.495 | INFO | Downloaded 34GiB to GPU in 2.15831s (15.7531GiB/s) | 20260505 06:21:24.634 | INFO | Analyzing chunk frames = [1015296,1015808) | 20260505 06:21:24.634 | INFO | MemInfo: { | 20260505 06:21:24.634 | INFO | "gpu" : " 17 GB", | 20260505 06:21:24.634 | INFO | "huge" : "", | 20260505 06:21:24.634 | INFO | "huge_shared" : " 27 GB", | 20260505 06:21:24.634 | INFO | "malloc" : " 14 GB", | 20260505 06:21:24.634 | INFO | "pinned" : " 29 GB" | 20260505 06:21:24.634 | INFO | } | 20260505 06:21:24.634 | INFO | Perf report for the last 5.13972 seconds: | 20260505 06:21:24.634 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:21:24.634 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:24.634 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:24.634 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:24.634 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 06:21:24.634 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:24.634 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:24.634 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260505 06:21:24.634 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:24.634 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:24.634 | INFO | Uploaded 36GiB to GPU in 2.09485s (17.185GiB/s) | 20260505 06:21:24.634 | INFO | Downloaded 34GiB to GPU in 2.0719s (16.41GiB/s) | 20260505 06:21:24.635 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 06:21:24.635 | INFO | Total mem usage: 15.3 GiB | 20260505 06:21:24.635 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 06:21:24.635 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.53%) | 20260505 06:21:24.635 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 06:21:24.635 | INFO | High Water Breakdown: | 20260505 06:21:24.635 | INFO | 60.2% : 8.67 GiB ( 92.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 06:21:24.635 | INFO | 39.6% : 5.7 GiB ( 85.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 06:21:24.635 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 06:21:24.635 | INFO | ------------------------------------------ | 20260505 06:21:24.635 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:21:24.635 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 06:21:24.635 | INFO | Total mem usage: 26.5 GiB | 20260505 06:21:24.635 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 06:21:24.635 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 06:21:24.635 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 06:21:24.635 | INFO | High Water Breakdown: | 20260505 06:21:24.635 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:21:24.635 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:21:24.635 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:21:24.635 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:21:24.635 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:21:24.635 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:21:24.635 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 06:21:24.635 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 06:21:24.635 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:21:24.635 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 06:21:24.635 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 06:21:24.635 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 06:21:24.635 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 06:21:24.635 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 06:21:24.635 | INFO | ------------------------------------------ | 20260505 06:21:24.635 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:21:24.635 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 06:21:24.635 | INFO | Total mem usage: 25.6 GiB | 20260505 06:21:24.635 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 06:21:24.635 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 06:21:24.635 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 06:21:24.635 | INFO | High Water Breakdown: | 20260505 06:21:24.635 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 06:21:24.635 | INFO | ------------------------------------------ | 20260505 06:21:24.635 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:21:24.635 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 06:21:24.635 | INFO | Total mem usage: 15.7 GiB | 20260505 06:21:24.635 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 06:21:24.635 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 06:21:24.635 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260505 06:21:24.635 | INFO | High Water Breakdown: | 20260505 06:21:24.635 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 06:21:24.635 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 06:21:24.635 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 06:21:24.635 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 06:21:24.635 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 06:21:24.635 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 06:21:24.635 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 06:21:24.635 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 06:21:24.635 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 06:21:24.635 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:21:24.635 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 06:21:24.635 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:21:24.635 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 06:21:24.635 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 06:21:24.635 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 06:21:24.635 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 06:21:24.635 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:21:24.635 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 06:21:24.635 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:21:24.635 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:21:24.635 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:21:24.635 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 06:21:24.635 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 06:21:24.635 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:21:24.635 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 06:21:24.635 | INFO | ------------------------------------------ | 20260505 06:21:24.636 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:21:24.636 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 06:21:24.636 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:21:24.636 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:21:24.636 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.60e+01, 1.36e+01 / 1.55e+00 | 20260505 06:21:24.636 | INFO | PerfReport: 0.28%, AggregateStats -- 9.40e-02 / 1.05e-01, 1.07e-02 / 3.72e-03 | 20260505 06:21:24.636 | INFO | PerfReport: 11.56%, Baseline -- 3.84e+00 / 4.51e+00, 6.36e-01 / 2.71e-01 | 20260505 06:21:24.636 | INFO | PerfReport: 3.30%, Histogram -- 1.10e+00 / 1.52e+00, 2.17e-01 / 9.39e-02 | 20260505 06:21:24.636 | INFO | PerfReport: 15.46%, DME -- 4.11e+01 / 4.52e+01, 4.47e+00 / 1.18e+00 | 20260505 06:21:24.636 | INFO | PerfReport: 56.11%, FrameLabeling -- 1.86e+01 / 1.92e+01, 2.10e+00 / 8.59e-01 | 20260505 06:21:24.636 | INFO | PerfReport: 7.69%, PulseAccumulating -- 2.55e+00 / 3.15e+00, 4.66e-01 / 3.16e-01 | 20260505 06:21:24.636 | INFO | PerfReport: 4.10%, Metrics -- 1.36e+00 / 1.76e+00, 3.85e-01 / 2.14e-01 | 20260505 06:21:24.636 | INFO | PerfReport: 1.49%, PulseSerializing -- 4.96e-01 / 5.76e-01, 9.75e-02 / 3.15e-02 | 20260505 06:21:24.636 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 06:21:24.636 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:21:24.636 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:21:24.636 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.26e+01 / 0.00e+00 | 20260505 06:21:24.636 | INFO | PerfReport: 22.03%, Upload -- 2.24e+01 / 0.00e+00, 7.86e+00 / 0.00e+00 | 20260505 06:21:24.636 | INFO | PerfReport: 21.49%, Download -- 2.18e+01 / 0.00e+00, 7.80e+00 / 0.00e+00 | 20260505 06:21:24.636 | INFO | PerfReport: 32.67%, Compute -- 3.32e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 06:21:24.636 | INFO | PerfReport: 1.38%, LockUpload -- 1.40e+00 / 0.00e+00, 6.72e+00 / 0.00e+00 | 20260505 06:21:24.636 | INFO | PerfReport: 0.72%, LockDownload -- 7.31e-01 / 0.00e+00, 2.00e+00 / 0.00e+00 | 20260505 06:21:24.636 | INFO | PerfReport: 21.69%, LockCompute -- 2.20e+01 / 0.00e+00, 1.65e+01 / 0.00e+00 | 20260505 06:21:24.636 | INFO | PerfReport: 0.01%, Misc -- 1.47e-02 / 0.00e+00, 2.49e-02 / 0.00e+00 | 20260505 06:21:24.757 | INFO | Starting a round of Baz IO | 20260505 06:21:28.014 | INFO | Populating RTMetrics report | 20260505 06:21:28.014 | INFO | RTMetrics report populated | 20260505 06:21:28.014 | INFO | Opening tmp RTMetrics json file | 20260505 06:21:28.017 | INFO | Serializing RTMetrics to json | 20260505 06:21:28.039 | INFO | Serialization done, swapping tmp file now... | 20260505 06:21:28.039 | INFO | RTMetrics done | 20260505 06:21:28.129 | INFO | Finished a round of Baz IO: Wrote 2.361 GiB in 3.371 seconds | 20260505 06:21:28.129 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.73% / 0.05% / 66.22% | 20260505 06:21:28.129 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.29046% / 7.15256e-05% / 5.96046e-05% | 20260505 06:21:28.129 | INFO | Baz IO overall: | 20260505 06:21:28.129 | INFO | BytesWritten : 520299326228 | 20260505 06:21:28.129 | INFO | NumEvents : 245878493430 | 20260505 06:21:29.773 | INFO | Analyzing chunk frames = [1015808,1016320) | 20260505 06:21:29.773 | INFO | MemInfo: { | 20260505 06:21:29.773 | INFO | "gpu" : " 17 GB", | 20260505 06:21:29.773 | INFO | "huge" : "", | 20260505 06:21:29.773 | INFO | "huge_shared" : " 27 GB", | 20260505 06:21:29.773 | INFO | "malloc" : " 14 GB", | 20260505 06:21:29.773 | INFO | "pinned" : " 29 GB" | 20260505 06:21:29.773 | INFO | } | 20260505 06:21:29.773 | INFO | Perf report for the last 5.13931 seconds: | 20260505 06:21:29.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:21:29.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:29.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:29.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:29.774 | INFO | ANALYSIS : 64.6%, 2.9, 98.7ms, 0 | 20260505 06:21:29.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:29.774 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260505 06:21:29.774 | INFO | PRE_HQ : 63.6%, 1.9, 64.2ms, 0 | 20260505 06:21:29.774 | INFO | BAZWRITER : 64.9%, 4.1, 137.1ms, 0 | 20260505 06:21:29.774 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 06:21:29.774 | INFO | Uploaded 36GiB to GPU in 2.11552s (17.0171GiB/s) | 20260505 06:21:29.774 | INFO | Downloaded 38GiB to GPU in 2.31815s (16.3924GiB/s) | 20260505 06:21:34.912 | INFO | Analyzing chunk frames = [1016320,1016832) | 20260505 06:21:34.912 | INFO | MemInfo: { | 20260505 06:21:34.912 | INFO | "gpu" : " 17 GB", | 20260505 06:21:34.912 | INFO | "huge" : "", | 20260505 06:21:34.912 | INFO | "huge_shared" : " 27 GB", | 20260505 06:21:34.912 | INFO | "malloc" : " 14 GB", | 20260505 06:21:34.912 | INFO | "pinned" : " 29 GB" | 20260505 06:21:34.912 | INFO | } | 20260505 06:21:34.912 | INFO | Perf report for the last 5.13886 seconds: | 20260505 06:21:34.912 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:21:34.912 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:34.913 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:34.913 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:34.913 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 06:21:34.913 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:34.913 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:34.913 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 06:21:34.913 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:34.913 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:34.913 | INFO | Uploaded 36GiB to GPU in 2.12056s (16.9766GiB/s) | 20260505 06:21:34.913 | INFO | Downloaded 34GiB to GPU in 2.03469s (16.7102GiB/s) | 20260505 06:21:40.052 | INFO | Analyzing chunk frames = [1016832,1017344) | 20260505 06:21:40.052 | INFO | MemInfo: { | 20260505 06:21:40.052 | INFO | "gpu" : " 17 GB", | 20260505 06:21:40.052 | INFO | "huge" : "", | 20260505 06:21:40.052 | INFO | "huge_shared" : " 27 GB", | 20260505 06:21:40.052 | INFO | "malloc" : " 14 GB", | 20260505 06:21:40.052 | INFO | "pinned" : " 29 GB" | 20260505 06:21:40.052 | INFO | } | 20260505 06:21:40.052 | INFO | Perf report for the last 5.13972 seconds: | 20260505 06:21:40.052 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:21:40.052 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:40.052 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:40.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:40.052 | INFO | ANALYSIS : 64.0%, 3.0, 99.6ms, 0 | 20260505 06:21:40.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:40.052 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:40.052 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 06:21:40.052 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:40.052 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:40.052 | INFO | Uploaded 36GiB to GPU in 2.16172s (16.6534GiB/s) | 20260505 06:21:40.053 | INFO | Downloaded 34GiB to GPU in 2.04169s (16.6528GiB/s) | 20260505 06:21:45.191 | INFO | Analyzing chunk frames = [1017344,1017856) | 20260505 06:21:45.191 | INFO | MemInfo: { | 20260505 06:21:45.191 | INFO | "gpu" : " 17 GB", | 20260505 06:21:45.191 | INFO | "huge" : "", | 20260505 06:21:45.191 | INFO | "huge_shared" : " 27 GB", | 20260505 06:21:45.191 | INFO | "malloc" : " 14 GB", | 20260505 06:21:45.191 | INFO | "pinned" : " 29 GB" | 20260505 06:21:45.191 | INFO | } | 20260505 06:21:45.191 | INFO | Perf report for the last 5.13904 seconds: | 20260505 06:21:45.191 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:21:45.191 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:45.191 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:45.191 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:45.191 | INFO | ANALYSIS : 63.9%, 3.0, 99.3ms, 0 | 20260505 06:21:45.191 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:45.191 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:45.191 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 06:21:45.191 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:45.191 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:45.191 | INFO | Uploaded 36GiB to GPU in 2.16511s (16.6273GiB/s) | 20260505 06:21:45.192 | INFO | Downloaded 34GiB to GPU in 2.0327s (16.7265GiB/s) | 20260505 06:21:50.330 | INFO | Analyzing chunk frames = [1017856,1018368) | 20260505 06:21:50.330 | INFO | MemInfo: { | 20260505 06:21:50.330 | INFO | "gpu" : " 17 GB", | 20260505 06:21:50.330 | INFO | "huge" : "", | 20260505 06:21:50.330 | INFO | "huge_shared" : " 27 GB", | 20260505 06:21:50.330 | INFO | "malloc" : " 14 GB", | 20260505 06:21:50.330 | INFO | "pinned" : " 29 GB" | 20260505 06:21:50.330 | INFO | } | 20260505 06:21:50.330 | INFO | Perf report for the last 5.13938 seconds: | 20260505 06:21:50.331 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:21:50.331 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:50.331 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:50.331 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:50.331 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 06:21:50.331 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:50.331 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:50.331 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 06:21:50.331 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:50.331 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:50.331 | INFO | Uploaded 36GiB to GPU in 2.16047s (16.663GiB/s) | 20260505 06:21:50.331 | INFO | Downloaded 34GiB to GPU in 2.0394s (16.6716GiB/s) | 20260505 06:21:55.471 | INFO | Analyzing chunk frames = [1018368,1018880) | 20260505 06:21:55.471 | INFO | MemInfo: { | 20260505 06:21:55.471 | INFO | "gpu" : " 17 GB", | 20260505 06:21:55.471 | INFO | "huge" : "", | 20260505 06:21:55.471 | INFO | "huge_shared" : " 27 GB", | 20260505 06:21:55.471 | INFO | "malloc" : " 14 GB", | 20260505 06:21:55.471 | INFO | "pinned" : " 29 GB" | 20260505 06:21:55.471 | INFO | } | 20260505 06:21:55.471 | INFO | Perf report for the last 5.14013 seconds: | 20260505 06:21:55.471 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:21:55.471 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:55.471 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:55.471 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:55.471 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 06:21:55.471 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:55.471 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:55.471 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 06:21:55.471 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:55.471 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:21:55.471 | INFO | Uploaded 36GiB to GPU in 2.15941s (16.6712GiB/s) | 20260505 06:21:55.471 | INFO | Downloaded 34GiB to GPU in 2.05224s (16.5672GiB/s) | 20260505 06:22:00.609 | INFO | Analyzing chunk frames = [1018880,1019392) | 20260505 06:22:00.609 | INFO | MemInfo: { | 20260505 06:22:00.609 | INFO | "gpu" : " 17 GB", | 20260505 06:22:00.609 | INFO | "huge" : "", | 20260505 06:22:00.609 | INFO | "huge_shared" : " 27 GB", | 20260505 06:22:00.609 | INFO | "malloc" : " 14 GB", | 20260505 06:22:00.609 | INFO | "pinned" : " 29 GB" | 20260505 06:22:00.609 | INFO | } | 20260505 06:22:00.609 | INFO | Perf report for the last 5.13837 seconds: | 20260505 06:22:00.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:22:00.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:00.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:00.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:00.609 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 06:22:00.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:00.609 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:00.609 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 06:22:00.609 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:00.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:00.609 | INFO | Uploaded 36GiB to GPU in 2.17868s (16.5238GiB/s) | 20260505 06:22:00.610 | INFO | Downloaded 34GiB to GPU in 2.15544s (15.7741GiB/s) | 20260505 06:22:05.748 | INFO | Analyzing chunk frames = [1019392,1019904) | 20260505 06:22:05.748 | INFO | MemInfo: { | 20260505 06:22:05.748 | INFO | "gpu" : " 17 GB", | 20260505 06:22:05.748 | INFO | "huge" : "", | 20260505 06:22:05.748 | INFO | "huge_shared" : " 27 GB", | 20260505 06:22:05.748 | INFO | "malloc" : " 14 GB", | 20260505 06:22:05.748 | INFO | "pinned" : " 29 GB" | 20260505 06:22:05.748 | INFO | } | 20260505 06:22:05.748 | INFO | Perf report for the last 5.13938 seconds: | 20260505 06:22:05.749 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:22:05.749 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:05.749 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:05.749 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:05.749 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 06:22:05.749 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:05.749 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:05.749 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260505 06:22:05.749 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:05.749 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:05.749 | INFO | Uploaded 36GiB to GPU in 2.11106s (17.053GiB/s) | 20260505 06:22:05.749 | INFO | Downloaded 34GiB to GPU in 2.06239s (16.4857GiB/s) | 20260505 06:22:05.863 | INFO | Starting a round of Baz IO | 20260505 06:22:09.134 | INFO | Populating RTMetrics report | 20260505 06:22:09.134 | INFO | RTMetrics report populated | 20260505 06:22:09.134 | INFO | Opening tmp RTMetrics json file | 20260505 06:22:09.170 | INFO | Serializing RTMetrics to json | 20260505 06:22:09.207 | INFO | Serialization done, swapping tmp file now... | 20260505 06:22:09.226 | INFO | RTMetrics done | 20260505 06:22:09.249 | INFO | Finished a round of Baz IO: Wrote 2.363 GiB in 3.385 seconds | 20260505 06:22:09.249 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.68% / 0.05% / 66.27% | 20260505 06:22:09.249 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.26952% / 7.54992e-05% / 4.76837e-05% | 20260505 06:22:09.249 | INFO | Baz IO overall: | 20260505 06:22:09.249 | INFO | BytesWritten : 522836543584 | 20260505 06:22:09.249 | INFO | NumEvents : 247091499565 | 20260505 06:22:10.888 | INFO | Analyzing chunk frames = [1019904,1020416) | 20260505 06:22:10.888 | INFO | MemInfo: { | 20260505 06:22:10.888 | INFO | "gpu" : " 17 GB", | 20260505 06:22:10.888 | INFO | "huge" : "", | 20260505 06:22:10.888 | INFO | "huge_shared" : " 27 GB", | 20260505 06:22:10.888 | INFO | "malloc" : " 14 GB", | 20260505 06:22:10.888 | INFO | "pinned" : " 29 GB" | 20260505 06:22:10.888 | INFO | } | 20260505 06:22:10.888 | INFO | Perf report for the last 5.13957 seconds: | 20260505 06:22:10.888 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:22:10.888 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:10.888 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:10.888 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:10.888 | INFO | ANALYSIS : 64.7%, 2.9, 98.6ms, 0 | 20260505 06:22:10.888 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:10.889 | INFO | RT_METRICS : 47.7%, 1.0, 25.0ms, 0 | 20260505 06:22:10.889 | INFO | PRE_HQ : 63.6%, 1.9, 64.1ms, 0 | 20260505 06:22:10.889 | INFO | BAZWRITER : 65.2%, 4.1, 135.8ms, 0 | 20260505 06:22:10.889 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260505 06:22:10.889 | INFO | Uploaded 36GiB to GPU in 2.17507s (16.5512GiB/s) | 20260505 06:22:10.889 | INFO | Downloaded 38GiB to GPU in 2.32171s (16.3672GiB/s) | 20260505 06:22:16.027 | INFO | Analyzing chunk frames = [1020416,1020928) | 20260505 06:22:16.027 | INFO | MemInfo: { | 20260505 06:22:16.027 | INFO | "gpu" : " 17 GB", | 20260505 06:22:16.027 | INFO | "huge" : "", | 20260505 06:22:16.027 | INFO | "huge_shared" : " 27 GB", | 20260505 06:22:16.027 | INFO | "malloc" : " 14 GB", | 20260505 06:22:16.027 | INFO | "pinned" : " 29 GB" | 20260505 06:22:16.027 | INFO | } | 20260505 06:22:16.027 | INFO | Perf report for the last 5.13921 seconds: | 20260505 06:22:16.027 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:22:16.027 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:16.027 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:16.027 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:16.027 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 06:22:16.027 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:16.027 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:16.027 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 06:22:16.027 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:16.027 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:16.027 | INFO | Uploaded 36GiB to GPU in 2.12893s (16.9099GiB/s) | 20260505 06:22:16.027 | INFO | Downloaded 34GiB to GPU in 2.03535s (16.7047GiB/s) | 20260505 06:22:21.173 | INFO | Analyzing chunk frames = [1020928,1021440) | 20260505 06:22:21.173 | INFO | MemInfo: { | 20260505 06:22:21.173 | INFO | "gpu" : " 17 GB", | 20260505 06:22:21.173 | INFO | "huge" : "", | 20260505 06:22:21.173 | INFO | "huge_shared" : " 27 GB", | 20260505 06:22:21.173 | INFO | "malloc" : " 14 GB", | 20260505 06:22:21.173 | INFO | "pinned" : " 29 GB" | 20260505 06:22:21.173 | INFO | } | 20260505 06:22:21.173 | INFO | Perf report for the last 5.14611 seconds: | 20260505 06:22:21.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:22:21.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:21.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:21.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:21.173 | INFO | ANALYSIS : 63.8%, 3.0, 99.4ms, 0 | 20260505 06:22:21.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:21.173 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:21.173 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260505 06:22:21.173 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:21.173 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:21.174 | INFO | Uploaded 36GiB to GPU in 2.14764s (16.7626GiB/s) | 20260505 06:22:21.174 | INFO | Downloaded 34GiB to GPU in 2.03447s (16.712GiB/s) | 20260505 06:22:26.306 | INFO | Analyzing chunk frames = [1021440,1021952) | 20260505 06:22:26.306 | INFO | MemInfo: { | 20260505 06:22:26.306 | INFO | "gpu" : " 17 GB", | 20260505 06:22:26.306 | INFO | "huge" : "", | 20260505 06:22:26.306 | INFO | "huge_shared" : " 27 GB", | 20260505 06:22:26.306 | INFO | "malloc" : " 14 GB", | 20260505 06:22:26.306 | INFO | "pinned" : " 29 GB" | 20260505 06:22:26.306 | INFO | } | 20260505 06:22:26.306 | INFO | Perf report for the last 5.13264 seconds: | 20260505 06:22:26.306 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:22:26.306 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:26.306 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:26.306 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:26.306 | INFO | ANALYSIS : 63.9%, 3.0, 99.1ms, 0 | 20260505 06:22:26.306 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:26.306 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:26.306 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 06:22:26.306 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:26.306 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:26.306 | INFO | Uploaded 36GiB to GPU in 2.16153s (16.6549GiB/s) | 20260505 06:22:26.306 | INFO | Downloaded 34GiB to GPU in 2.0274s (16.7703GiB/s) | 20260505 06:22:31.445 | INFO | Analyzing chunk frames = [1021952,1022464) | 20260505 06:22:31.445 | INFO | MemInfo: { | 20260505 06:22:31.445 | INFO | "gpu" : " 17 GB", | 20260505 06:22:31.445 | INFO | "huge" : "", | 20260505 06:22:31.445 | INFO | "huge_shared" : " 27 GB", | 20260505 06:22:31.445 | INFO | "malloc" : " 14 GB", | 20260505 06:22:31.445 | INFO | "pinned" : " 29 GB" | 20260505 06:22:31.445 | INFO | } | 20260505 06:22:31.445 | INFO | Perf report for the last 5.13907 seconds: | 20260505 06:22:31.445 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:22:31.445 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:31.445 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:31.445 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:31.445 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 06:22:31.445 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:31.445 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:31.445 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 06:22:31.445 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:31.445 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:31.445 | INFO | Uploaded 36GiB to GPU in 2.12682s (16.9267GiB/s) | 20260505 06:22:31.446 | INFO | Downloaded 34GiB to GPU in 2.05211s (16.5683GiB/s) | 20260505 06:22:36.586 | INFO | Analyzing chunk frames = [1022464,1022976) | 20260505 06:22:36.586 | INFO | MemInfo: { | 20260505 06:22:36.586 | INFO | "gpu" : " 17 GB", | 20260505 06:22:36.586 | INFO | "huge" : "", | 20260505 06:22:36.586 | INFO | "huge_shared" : " 27 GB", | 20260505 06:22:36.586 | INFO | "malloc" : " 14 GB", | 20260505 06:22:36.586 | INFO | "pinned" : " 29 GB" | 20260505 06:22:36.586 | INFO | } | 20260505 06:22:36.586 | INFO | Perf report for the last 5.14073 seconds: | 20260505 06:22:36.586 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:22:36.586 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:36.586 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:36.586 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:36.586 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 06:22:36.586 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:36.586 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:36.586 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260505 06:22:36.586 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:36.586 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:36.586 | INFO | Uploaded 36GiB to GPU in 2.15423s (16.7113GiB/s) | 20260505 06:22:36.586 | INFO | Downloaded 34GiB to GPU in 2.05407s (16.5525GiB/s) | 20260505 06:22:41.724 | INFO | Analyzing chunk frames = [1022976,1023488) | 20260505 06:22:41.724 | INFO | MemInfo: { | 20260505 06:22:41.724 | INFO | "gpu" : " 17 GB", | 20260505 06:22:41.724 | INFO | "huge" : "", | 20260505 06:22:41.724 | INFO | "huge_shared" : " 27 GB", | 20260505 06:22:41.724 | INFO | "malloc" : " 14 GB", | 20260505 06:22:41.724 | INFO | "pinned" : " 29 GB" | 20260505 06:22:41.724 | INFO | } | 20260505 06:22:41.724 | INFO | Perf report for the last 5.13832 seconds: | 20260505 06:22:41.724 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:22:41.724 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:41.724 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:41.724 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:41.724 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 06:22:41.724 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:41.724 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:41.724 | INFO | PRE_HQ : 32.8%, 1.0, 17.1ms, 0 | 20260505 06:22:41.724 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:41.724 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:41.725 | INFO | Uploaded 36GiB to GPU in 2.1902s (16.4368GiB/s) | 20260505 06:22:41.725 | INFO | Downloaded 34GiB to GPU in 2.14957s (15.8171GiB/s) | 20260505 06:22:46.863 | INFO | Analyzing chunk frames = [1023488,1024000) | 20260505 06:22:46.863 | INFO | MemInfo: { | 20260505 06:22:46.863 | INFO | "gpu" : " 17 GB", | 20260505 06:22:46.863 | INFO | "huge" : "", | 20260505 06:22:46.863 | INFO | "huge_shared" : " 27 GB", | 20260505 06:22:46.863 | INFO | "malloc" : " 14 GB", | 20260505 06:22:46.863 | INFO | "pinned" : " 29 GB" | 20260505 06:22:46.863 | INFO | } | 20260505 06:22:46.863 | INFO | Perf report for the last 5.13907 seconds: | 20260505 06:22:46.863 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:22:46.863 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:46.863 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:46.863 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:46.863 | INFO | ANALYSIS : 63.8%, 3.0, 99.5ms, 0 | 20260505 06:22:46.863 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:46.863 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:46.863 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 06:22:46.863 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:46.863 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:46.863 | INFO | Uploaded 36GiB to GPU in 2.115s (17.0213GiB/s) | 20260505 06:22:46.863 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 06:22:46.863 | INFO | Total mem usage: 15.3 GiB | 20260505 06:22:46.863 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 06:22:46.863 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.38%) | 20260505 06:22:46.863 | INFO | Downloaded 34GiB to GPU in 2.06243s (16.4854GiB/s) | 20260505 06:22:46.863 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 06:22:46.863 | INFO | High Water Breakdown: | 20260505 06:22:46.863 | INFO | 60.3% : 8.69 GiB ( 92.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 06:22:46.863 | INFO | 39.5% : 5.7 GiB ( 85.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 06:22:46.863 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 06:22:46.864 | INFO | ------------------------------------------ | 20260505 06:22:46.864 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:22:46.864 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 06:22:46.864 | INFO | Total mem usage: 26.5 GiB | 20260505 06:22:46.864 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 06:22:46.864 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 06:22:46.864 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 06:22:46.864 | INFO | High Water Breakdown: | 20260505 06:22:46.864 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:22:46.864 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:22:46.864 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:22:46.864 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:22:46.864 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:22:46.864 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:22:46.864 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 06:22:46.864 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 06:22:46.864 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:22:46.864 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 06:22:46.864 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 06:22:46.864 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 06:22:46.864 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 06:22:46.864 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 06:22:46.864 | INFO | ------------------------------------------ | 20260505 06:22:46.864 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:22:46.864 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 06:22:46.864 | INFO | Total mem usage: 25.6 GiB | 20260505 06:22:46.864 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 06:22:46.864 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 06:22:46.864 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 06:22:46.864 | INFO | High Water Breakdown: | 20260505 06:22:46.864 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 06:22:46.864 | INFO | ------------------------------------------ | 20260505 06:22:46.864 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:22:46.864 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 06:22:46.864 | INFO | Total mem usage: 15.7 GiB | 20260505 06:22:46.864 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 06:22:46.864 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 06:22:46.864 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260505 06:22:46.864 | INFO | High Water Breakdown: | 20260505 06:22:46.864 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 06:22:46.864 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 06:22:46.864 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 06:22:46.864 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 06:22:46.864 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 06:22:46.864 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 06:22:46.864 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 06:22:46.864 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 06:22:46.864 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 06:22:46.864 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:22:46.864 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 06:22:46.864 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:22:46.864 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 06:22:46.864 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 06:22:46.864 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 06:22:46.864 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 06:22:46.865 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:22:46.865 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 06:22:46.865 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:22:46.865 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:22:46.865 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:22:46.865 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 06:22:46.865 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 06:22:46.865 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:22:46.865 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 06:22:46.865 | INFO | ------------------------------------------ | 20260505 06:22:46.865 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:22:46.865 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 06:22:46.865 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:22:46.865 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:22:46.865 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.54e+01, 1.37e+01 / 1.63e+00 | 20260505 06:22:46.865 | INFO | PerfReport: 0.28%, AggregateStats -- 9.38e-02 / 1.04e-01, 1.04e-02 / 5.19e-03 | 20260505 06:22:46.865 | INFO | PerfReport: 11.58%, Baseline -- 3.84e+00 / 4.47e+00, 6.41e-01 / 3.11e-01 | 20260505 06:22:46.865 | INFO | PerfReport: 3.31%, Histogram -- 1.10e+00 / 1.51e+00, 2.19e-01 / 1.00e-01 | 20260505 06:22:46.865 | INFO | PerfReport: 15.44%, DME -- 4.10e+01 / 4.48e+01, 4.28e+00 / 1.21e+00 | 20260505 06:22:46.865 | INFO | PerfReport: 56.08%, FrameLabeling -- 1.86e+01 / 1.91e+01, 2.10e+00 / 1.02e+00 | 20260505 06:22:46.865 | INFO | PerfReport: 7.69%, PulseAccumulating -- 2.55e+00 / 3.13e+00, 4.62e-01 / 3.37e-01 | 20260505 06:22:46.865 | INFO | PerfReport: 4.12%, Metrics -- 1.37e+00 / 1.72e+00, 3.88e-01 / 1.99e-01 | 20260505 06:22:46.865 | INFO | PerfReport: 1.50%, PulseSerializing -- 4.97e-01 / 5.74e-01, 9.80e-02 / 4.30e-02 | 20260505 06:22:46.865 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 06:22:46.865 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:22:46.865 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:22:46.865 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.24e+01 / 0.00e+00 | 20260505 06:22:46.865 | INFO | PerfReport: 22.03%, Upload -- 2.24e+01 / 0.00e+00, 7.82e+00 / 0.00e+00 | 20260505 06:22:46.865 | INFO | PerfReport: 21.46%, Download -- 2.18e+01 / 0.00e+00, 7.77e+00 / 0.00e+00 | 20260505 06:22:46.865 | INFO | PerfReport: 32.66%, Compute -- 3.32e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 06:22:46.865 | INFO | PerfReport: 1.44%, LockUpload -- 1.46e+00 / 0.00e+00, 6.80e+00 / 0.00e+00 | 20260505 06:22:46.865 | INFO | PerfReport: 0.71%, LockDownload -- 7.16e-01 / 0.00e+00, 2.02e+00 / 0.00e+00 | 20260505 06:22:46.865 | INFO | PerfReport: 21.70%, LockCompute -- 2.20e+01 / 0.00e+00, 1.67e+01 / 0.00e+00 | 20260505 06:22:46.865 | INFO | PerfReport: 0.02%, Misc -- 1.54e-02 / 0.00e+00, 2.59e-02 / 0.00e+00 | 20260505 06:22:46.992 | INFO | Starting a round of Baz IO | 20260505 06:22:50.241 | INFO | Populating RTMetrics report | 20260505 06:22:50.242 | INFO | RTMetrics report populated | 20260505 06:22:50.242 | INFO | Opening tmp RTMetrics json file | 20260505 06:22:50.246 | INFO | Serializing RTMetrics to json | 20260505 06:22:50.249 | INFO | Serialization done, swapping tmp file now... | 20260505 06:22:50.256 | INFO | RTMetrics done | 20260505 06:22:50.356 | INFO | Finished a round of Baz IO: Wrote 2.365 GiB in 3.364 seconds | 20260505 06:22:50.356 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.63% / 0.05% / 66.32% | 20260505 06:22:50.356 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.25695% / 4.76837e-05% / 3.57628e-05% | 20260505 06:22:50.356 | INFO | Baz IO overall: | 20260505 06:22:50.356 | INFO | BytesWritten : 525375751724 | 20260505 06:22:50.356 | INFO | NumEvents : 248305593631 | 20260505 06:22:52.003 | INFO | Analyzing chunk frames = [1024000,1024512) | 20260505 06:22:52.003 | INFO | MemInfo: { | 20260505 06:22:52.003 | INFO | "gpu" : " 17 GB", | 20260505 06:22:52.003 | INFO | "huge" : "", | 20260505 06:22:52.003 | INFO | "huge_shared" : " 27 GB", | 20260505 06:22:52.003 | INFO | "malloc" : " 14 GB", | 20260505 06:22:52.003 | INFO | "pinned" : " 29 GB" | 20260505 06:22:52.003 | INFO | } | 20260505 06:22:52.003 | INFO | Perf report for the last 5.13976 seconds: | 20260505 06:22:52.003 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:22:52.003 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:52.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:52.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:52.003 | INFO | ANALYSIS : 64.6%, 2.9, 98.5ms, 0 | 20260505 06:22:52.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:52.003 | INFO | RT_METRICS : 46.5%, 1.0, 24.4ms, 0 | 20260505 06:22:52.003 | INFO | PRE_HQ : 63.4%, 2.0, 64.3ms, 0 | 20260505 06:22:52.003 | INFO | BAZWRITER : 65.0%, 4.1, 137.3ms, 0 | 20260505 06:22:52.003 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 06:22:52.003 | INFO | Uploaded 36GiB to GPU in 2.14368s (16.7935GiB/s) | 20260505 06:22:52.003 | INFO | Downloaded 38GiB to GPU in 2.32316s (16.357GiB/s) | 20260505 06:22:57.142 | INFO | Analyzing chunk frames = [1024512,1025024) | 20260505 06:22:57.142 | INFO | MemInfo: { | 20260505 06:22:57.142 | INFO | "gpu" : " 17 GB", | 20260505 06:22:57.142 | INFO | "huge" : "", | 20260505 06:22:57.142 | INFO | "huge_shared" : " 27 GB", | 20260505 06:22:57.142 | INFO | "malloc" : " 14 GB", | 20260505 06:22:57.142 | INFO | "pinned" : " 29 GB" | 20260505 06:22:57.142 | INFO | } | 20260505 06:22:57.142 | INFO | Perf report for the last 5.13901 seconds: | 20260505 06:22:57.142 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:22:57.142 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:57.142 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:57.142 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:57.142 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 06:22:57.142 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:57.142 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:57.142 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 06:22:57.142 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:57.142 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:22:57.142 | INFO | Uploaded 36GiB to GPU in 2.11398s (17.0295GiB/s) | 20260505 06:22:57.143 | INFO | Downloaded 34GiB to GPU in 2.03877s (16.6767GiB/s) | 20260505 06:23:02.281 | INFO | Analyzing chunk frames = [1025024,1025536) | 20260505 06:23:02.281 | INFO | MemInfo: { | 20260505 06:23:02.281 | INFO | "gpu" : " 17 GB", | 20260505 06:23:02.281 | INFO | "huge" : "", | 20260505 06:23:02.281 | INFO | "huge_shared" : " 27 GB", | 20260505 06:23:02.281 | INFO | "malloc" : " 14 GB", | 20260505 06:23:02.281 | INFO | "pinned" : " 29 GB" | 20260505 06:23:02.281 | INFO | } | 20260505 06:23:02.281 | INFO | Perf report for the last 5.13931 seconds: | 20260505 06:23:02.281 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:23:02.281 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:02.281 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:02.281 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:02.281 | INFO | ANALYSIS : 63.9%, 3.0, 99.5ms, 0 | 20260505 06:23:02.281 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:02.281 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:02.281 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 06:23:02.281 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:02.281 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:02.282 | INFO | Uploaded 36GiB to GPU in 2.1441s (16.7903GiB/s) | 20260505 06:23:02.282 | INFO | Downloaded 34GiB to GPU in 2.0297s (16.7512GiB/s) | 20260505 06:23:07.420 | INFO | Analyzing chunk frames = [1025536,1026048) | 20260505 06:23:07.420 | INFO | MemInfo: { | 20260505 06:23:07.420 | INFO | "gpu" : " 17 GB", | 20260505 06:23:07.420 | INFO | "huge" : "", | 20260505 06:23:07.420 | INFO | "huge_shared" : " 27 GB", | 20260505 06:23:07.420 | INFO | "malloc" : " 14 GB", | 20260505 06:23:07.420 | INFO | "pinned" : " 29 GB" | 20260505 06:23:07.420 | INFO | } | 20260505 06:23:07.420 | INFO | Perf report for the last 5.13921 seconds: | 20260505 06:23:07.421 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:23:07.421 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:07.421 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:07.421 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:07.421 | INFO | ANALYSIS : 63.9%, 3.0, 99.3ms, 0 | 20260505 06:23:07.421 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:07.421 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:07.421 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 06:23:07.421 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:07.421 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:07.421 | INFO | Uploaded 36GiB to GPU in 2.16767s (16.6077GiB/s) | 20260505 06:23:07.421 | INFO | Downloaded 34GiB to GPU in 2.0443s (16.6316GiB/s) | 20260505 06:23:12.560 | INFO | Analyzing chunk frames = [1026048,1026560) | 20260505 06:23:12.560 | INFO | MemInfo: { | 20260505 06:23:12.560 | INFO | "gpu" : " 17 GB", | 20260505 06:23:12.560 | INFO | "huge" : "", | 20260505 06:23:12.560 | INFO | "huge_shared" : " 27 GB", | 20260505 06:23:12.560 | INFO | "malloc" : " 14 GB", | 20260505 06:23:12.560 | INFO | "pinned" : " 29 GB" | 20260505 06:23:12.560 | INFO | } | 20260505 06:23:12.560 | INFO | Perf report for the last 5.13933 seconds: | 20260505 06:23:12.560 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:23:12.560 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:12.560 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:12.560 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:12.560 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 06:23:12.560 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:12.560 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:12.560 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 06:23:12.560 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:12.560 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:12.560 | INFO | Uploaded 36GiB to GPU in 2.15518s (16.7039GiB/s) | 20260505 06:23:12.560 | INFO | Downloaded 34GiB to GPU in 2.04634s (16.615GiB/s) | 20260505 06:23:17.701 | INFO | Analyzing chunk frames = [1026560,1027072) | 20260505 06:23:17.701 | INFO | MemInfo: { | 20260505 06:23:17.701 | INFO | "gpu" : " 17 GB", | 20260505 06:23:17.701 | INFO | "huge" : "", | 20260505 06:23:17.701 | INFO | "huge_shared" : " 27 GB", | 20260505 06:23:17.701 | INFO | "malloc" : " 14 GB", | 20260505 06:23:17.701 | INFO | "pinned" : " 29 GB" | 20260505 06:23:17.701 | INFO | } | 20260505 06:23:17.701 | INFO | Perf report for the last 5.14091 seconds: | 20260505 06:23:17.701 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:23:17.701 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:17.701 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:17.701 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:17.701 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 06:23:17.701 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:17.701 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:17.701 | INFO | PRE_HQ : 31.7%, 1.0, 16.4ms, 0 | 20260505 06:23:17.701 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:17.701 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:17.701 | INFO | Uploaded 36GiB to GPU in 2.1674s (16.6098GiB/s) | 20260505 06:23:17.701 | INFO | Downloaded 34GiB to GPU in 2.05367s (16.5557GiB/s) | 20260505 06:23:22.839 | INFO | Analyzing chunk frames = [1027072,1027584) | 20260505 06:23:22.839 | INFO | MemInfo: { | 20260505 06:23:22.839 | INFO | "gpu" : " 17 GB", | 20260505 06:23:22.839 | INFO | "huge" : "", | 20260505 06:23:22.839 | INFO | "huge_shared" : " 27 GB", | 20260505 06:23:22.839 | INFO | "malloc" : " 14 GB", | 20260505 06:23:22.839 | INFO | "pinned" : " 29 GB" | 20260505 06:23:22.839 | INFO | } | 20260505 06:23:22.839 | INFO | Perf report for the last 5.13805 seconds: | 20260505 06:23:22.839 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:23:22.839 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:22.839 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:22.839 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:22.839 | INFO | ANALYSIS : 64.1%, 3.0, 100.0ms, 0 | 20260505 06:23:22.839 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:22.839 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:22.839 | INFO | PRE_HQ : 32.8%, 1.0, 17.0ms, 0 | 20260505 06:23:22.839 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:22.839 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:22.839 | INFO | Uploaded 36GiB to GPU in 2.17015s (16.5887GiB/s) | 20260505 06:23:22.839 | INFO | Downloaded 34GiB to GPU in 2.15316s (15.7907GiB/s) | 20260505 06:23:27.978 | INFO | Analyzing chunk frames = [1027584,1028096) | 20260505 06:23:27.978 | INFO | MemInfo: { | 20260505 06:23:27.978 | INFO | "gpu" : " 17 GB", | 20260505 06:23:27.978 | INFO | "huge" : "", | 20260505 06:23:27.978 | INFO | "huge_shared" : " 27 GB", | 20260505 06:23:27.978 | INFO | "malloc" : " 14 GB", | 20260505 06:23:27.978 | INFO | "pinned" : " 29 GB" | 20260505 06:23:27.978 | INFO | } | 20260505 06:23:27.978 | INFO | Perf report for the last 5.13929 seconds: | 20260505 06:23:27.978 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:23:27.978 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:27.978 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:27.978 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:27.978 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 06:23:27.978 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:27.978 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:27.978 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 06:23:27.978 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:27.978 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:27.978 | INFO | Uploaded 36GiB to GPU in 2.11367s (17.032GiB/s) | 20260505 06:23:27.979 | INFO | Downloaded 34GiB to GPU in 2.06891s (16.4338GiB/s) | 20260505 06:23:28.100 | INFO | Starting a round of Baz IO | 20260505 06:23:31.355 | INFO | Populating RTMetrics report | 20260505 06:23:31.355 | INFO | RTMetrics report populated | 20260505 06:23:31.355 | INFO | Opening tmp RTMetrics json file | 20260505 06:23:31.368 | INFO | Serializing RTMetrics to json | 20260505 06:23:31.386 | INFO | Serialization done, swapping tmp file now... | 20260505 06:23:31.394 | INFO | RTMetrics done | 20260505 06:23:31.470 | INFO | Finished a round of Baz IO: Wrote 2.368 GiB in 3.369 seconds | 20260505 06:23:31.470 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.58% / 0.05% / 66.37% | 20260505 06:23:31.470 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.24101% / 7.94729e-05% / 4.37101e-05% | 20260505 06:23:31.470 | INFO | Baz IO overall: | 20260505 06:23:31.470 | INFO | BytesWritten : 527917939661 | 20260505 06:23:31.470 | INFO | NumEvents : 249521479739 | 20260505 06:23:33.117 | INFO | Analyzing chunk frames = [1028096,1028608) | 20260505 06:23:33.117 | INFO | MemInfo: { | 20260505 06:23:33.117 | INFO | "gpu" : " 17 GB", | 20260505 06:23:33.117 | INFO | "huge" : "", | 20260505 06:23:33.117 | INFO | "huge_shared" : " 27 GB", | 20260505 06:23:33.117 | INFO | "malloc" : " 14 GB", | 20260505 06:23:33.117 | INFO | "pinned" : " 29 GB" | 20260505 06:23:33.117 | INFO | } | 20260505 06:23:33.117 | INFO | Perf report for the last 5.13908 seconds: | 20260505 06:23:33.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:23:33.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:33.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:33.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:33.117 | INFO | ANALYSIS : 64.6%, 2.9, 98.6ms, 0 | 20260505 06:23:33.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:33.117 | INFO | RT_METRICS : 46.7%, 1.0, 24.5ms, 0 | 20260505 06:23:33.117 | INFO | PRE_HQ : 63.4%, 2.0, 64.6ms, 0 | 20260505 06:23:33.117 | INFO | BAZWRITER : 64.8%, 4.1, 138.1ms, 0 | 20260505 06:23:33.117 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 06:23:33.118 | INFO | Uploaded 36GiB to GPU in 2.11928s (16.9869GiB/s) | 20260505 06:23:33.118 | INFO | Downloaded 38GiB to GPU in 2.30912s (16.4565GiB/s) | 20260505 06:23:38.257 | INFO | Analyzing chunk frames = [1028608,1029120) | 20260505 06:23:38.257 | INFO | MemInfo: { | 20260505 06:23:38.257 | INFO | "gpu" : " 17 GB", | 20260505 06:23:38.257 | INFO | "huge" : "", | 20260505 06:23:38.257 | INFO | "huge_shared" : " 27 GB", | 20260505 06:23:38.257 | INFO | "malloc" : " 14 GB", | 20260505 06:23:38.257 | INFO | "pinned" : " 29 GB" | 20260505 06:23:38.257 | INFO | } | 20260505 06:23:38.257 | INFO | Perf report for the last 5.13954 seconds: | 20260505 06:23:38.257 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:23:38.257 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:38.257 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:38.257 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:38.257 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 06:23:38.257 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:38.257 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:38.257 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260505 06:23:38.257 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:38.257 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:38.257 | INFO | Uploaded 36GiB to GPU in 2.12427s (16.947GiB/s) | 20260505 06:23:38.257 | INFO | Downloaded 34GiB to GPU in 2.04232s (16.6477GiB/s) | 20260505 06:23:43.396 | INFO | Analyzing chunk frames = [1029120,1029632) | 20260505 06:23:43.396 | INFO | MemInfo: { | 20260505 06:23:43.396 | INFO | "gpu" : " 17 GB", | 20260505 06:23:43.396 | INFO | "huge" : "", | 20260505 06:23:43.396 | INFO | "huge_shared" : " 27 GB", | 20260505 06:23:43.396 | INFO | "malloc" : " 14 GB", | 20260505 06:23:43.396 | INFO | "pinned" : " 29 GB" | 20260505 06:23:43.396 | INFO | } | 20260505 06:23:43.396 | INFO | Perf report for the last 5.13947 seconds: | 20260505 06:23:43.396 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:23:43.396 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:43.396 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:43.396 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:43.396 | INFO | ANALYSIS : 63.9%, 3.0, 99.5ms, 0 | 20260505 06:23:43.396 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:43.396 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:43.396 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 06:23:43.396 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:43.396 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:43.397 | INFO | Uploaded 36GiB to GPU in 2.14469s (16.7857GiB/s) | 20260505 06:23:43.397 | INFO | Downloaded 34GiB to GPU in 2.03617s (16.6981GiB/s) | 20260505 06:23:48.535 | INFO | Analyzing chunk frames = [1029632,1030144) | 20260505 06:23:48.535 | INFO | MemInfo: { | 20260505 06:23:48.535 | INFO | "gpu" : " 17 GB", | 20260505 06:23:48.535 | INFO | "huge" : "", | 20260505 06:23:48.535 | INFO | "huge_shared" : " 27 GB", | 20260505 06:23:48.535 | INFO | "malloc" : " 14 GB", | 20260505 06:23:48.535 | INFO | "pinned" : " 29 GB" | 20260505 06:23:48.535 | INFO | } | 20260505 06:23:48.535 | INFO | Perf report for the last 5.13919 seconds: | 20260505 06:23:48.535 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:23:48.535 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:48.535 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:48.535 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:48.535 | INFO | ANALYSIS : 64.0%, 3.0, 99.4ms, 0 | 20260505 06:23:48.535 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:48.536 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:48.536 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 06:23:48.536 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:48.536 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:48.536 | INFO | Uploaded 36GiB to GPU in 2.17956s (16.5171GiB/s) | 20260505 06:23:48.536 | INFO | Downloaded 34GiB to GPU in 2.0454s (16.6227GiB/s) | 20260505 06:23:53.675 | INFO | Analyzing chunk frames = [1030144,1030656) | 20260505 06:23:53.675 | INFO | MemInfo: { | 20260505 06:23:53.675 | INFO | "gpu" : " 17 GB", | 20260505 06:23:53.675 | INFO | "huge" : "", | 20260505 06:23:53.675 | INFO | "huge_shared" : " 27 GB", | 20260505 06:23:53.675 | INFO | "malloc" : " 14 GB", | 20260505 06:23:53.675 | INFO | "pinned" : " 29 GB" | 20260505 06:23:53.675 | INFO | } | 20260505 06:23:53.675 | INFO | Perf report for the last 5.13935 seconds: | 20260505 06:23:53.675 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:23:53.675 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:53.675 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:53.675 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:53.675 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 06:23:53.675 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:53.675 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:53.675 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 06:23:53.675 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:53.675 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:53.675 | INFO | Uploaded 36GiB to GPU in 2.15919s (16.6729GiB/s) | 20260505 06:23:53.675 | INFO | Downloaded 34GiB to GPU in 2.04958s (16.5888GiB/s) | 20260505 06:23:58.816 | INFO | Analyzing chunk frames = [1030656,1031168) | 20260505 06:23:58.816 | INFO | MemInfo: { | 20260505 06:23:58.816 | INFO | "gpu" : " 17 GB", | 20260505 06:23:58.816 | INFO | "huge" : "", | 20260505 06:23:58.816 | INFO | "huge_shared" : " 27 GB", | 20260505 06:23:58.816 | INFO | "malloc" : " 14 GB", | 20260505 06:23:58.816 | INFO | "pinned" : " 29 GB" | 20260505 06:23:58.816 | INFO | } | 20260505 06:23:58.816 | INFO | Perf report for the last 5.14095 seconds: | 20260505 06:23:58.816 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:23:58.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:58.816 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:58.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:58.816 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 06:23:58.816 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:58.816 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:58.816 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 06:23:58.816 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:58.816 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:23:58.816 | INFO | Uploaded 36GiB to GPU in 2.17097s (16.5825GiB/s) | 20260505 06:23:58.816 | INFO | Downloaded 34GiB to GPU in 2.05243s (16.5657GiB/s) | 20260505 06:24:03.954 | INFO | Analyzing chunk frames = [1031168,1031680) | 20260505 06:24:03.954 | INFO | MemInfo: { | 20260505 06:24:03.954 | INFO | "gpu" : " 17 GB", | 20260505 06:24:03.954 | INFO | "huge" : "", | 20260505 06:24:03.954 | INFO | "huge_shared" : " 27 GB", | 20260505 06:24:03.954 | INFO | "malloc" : " 14 GB", | 20260505 06:24:03.954 | INFO | "pinned" : " 29 GB" | 20260505 06:24:03.954 | INFO | } | 20260505 06:24:03.954 | INFO | Perf report for the last 5.13786 seconds: | 20260505 06:24:03.954 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:24:03.954 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:03.954 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:03.954 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:03.954 | INFO | ANALYSIS : 64.3%, 3.0, 100.3ms, 0 | 20260505 06:24:03.954 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:03.954 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:03.954 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260505 06:24:03.954 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:03.954 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:03.954 | INFO | Uploaded 36GiB to GPU in 2.18158s (16.5018GiB/s) | 20260505 06:24:03.954 | INFO | Downloaded 34GiB to GPU in 2.15492s (15.7778GiB/s) | 20260505 06:24:09.093 | INFO | Analyzing chunk frames = [1031680,1032192) | 20260505 06:24:09.093 | INFO | MemInfo: { | 20260505 06:24:09.093 | INFO | "gpu" : " 17 GB", | 20260505 06:24:09.093 | INFO | "huge" : "", | 20260505 06:24:09.093 | INFO | "huge_shared" : " 27 GB", | 20260505 06:24:09.093 | INFO | "malloc" : " 14 GB", | 20260505 06:24:09.093 | INFO | "pinned" : " 29 GB" | 20260505 06:24:09.093 | INFO | } | 20260505 06:24:09.093 | INFO | Perf report for the last 5.13951 seconds: | 20260505 06:24:09.093 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:24:09.093 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:09.093 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:09.093 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:09.093 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 06:24:09.093 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:09.093 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:09.093 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 06:24:09.093 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:09.093 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:09.093 | INFO | Uploaded 36GiB to GPU in 2.11607s (17.0127GiB/s) | 20260505 06:24:09.093 | INFO | Downloaded 34GiB to GPU in 2.06362s (16.4759GiB/s) | 20260505 06:24:09.094 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 06:24:09.094 | INFO | Total mem usage: 15.3 GiB | 20260505 06:24:09.094 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 06:24:09.094 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.44%) | 20260505 06:24:09.094 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 06:24:09.094 | INFO | High Water Breakdown: | 20260505 06:24:09.094 | INFO | 60.3% : 8.69 GiB ( 92.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 06:24:09.094 | INFO | 39.5% : 5.69 GiB ( 85.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 06:24:09.094 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 06:24:09.094 | INFO | ------------------------------------------ | 20260505 06:24:09.094 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:24:09.094 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 06:24:09.094 | INFO | Total mem usage: 26.5 GiB | 20260505 06:24:09.094 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 06:24:09.094 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 06:24:09.094 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 06:24:09.094 | INFO | High Water Breakdown: | 20260505 06:24:09.094 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:24:09.094 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:24:09.094 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:24:09.094 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:24:09.094 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:24:09.094 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:24:09.094 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 06:24:09.094 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 06:24:09.094 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:24:09.094 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 06:24:09.094 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 06:24:09.094 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 06:24:09.094 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 06:24:09.094 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 06:24:09.094 | INFO | ------------------------------------------ | 20260505 06:24:09.094 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:24:09.094 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 06:24:09.094 | INFO | Total mem usage: 25.6 GiB | 20260505 06:24:09.094 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 06:24:09.094 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 06:24:09.094 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 06:24:09.094 | INFO | High Water Breakdown: | 20260505 06:24:09.094 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 06:24:09.094 | INFO | ------------------------------------------ | 20260505 06:24:09.094 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:24:09.094 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 06:24:09.094 | INFO | Total mem usage: 15.7 GiB | 20260505 06:24:09.094 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 06:24:09.094 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 06:24:09.094 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260505 06:24:09.094 | INFO | High Water Breakdown: | 20260505 06:24:09.094 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 06:24:09.094 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 06:24:09.094 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 06:24:09.094 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 06:24:09.094 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 06:24:09.094 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 06:24:09.094 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 06:24:09.094 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 06:24:09.094 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 06:24:09.094 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:24:09.094 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 06:24:09.094 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:24:09.094 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 06:24:09.094 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 06:24:09.094 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 06:24:09.094 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 06:24:09.094 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:24:09.094 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 06:24:09.094 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:24:09.094 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:24:09.094 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:24:09.094 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 06:24:09.094 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 06:24:09.094 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:24:09.094 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 06:24:09.094 | INFO | ------------------------------------------ | 20260505 06:24:09.094 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:24:09.094 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 06:24:09.094 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:24:09.094 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:24:09.095 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.57e+01, 1.35e+01 / 1.67e+00 | 20260505 06:24:09.095 | INFO | PerfReport: 0.28%, AggregateStats -- 9.39e-02 / 1.05e-01, 1.11e-02 / 1.40e-02 | 20260505 06:24:09.095 | INFO | PerfReport: 11.57%, Baseline -- 3.84e+00 / 4.48e+00, 6.39e-01 / 2.54e-01 | 20260505 06:24:09.095 | INFO | PerfReport: 3.31%, Histogram -- 1.10e+00 / 1.50e+00, 2.18e-01 / 9.58e-02 | 20260505 06:24:09.095 | INFO | PerfReport: 15.45%, DME -- 4.10e+01 / 4.51e+01, 4.41e+00 / 1.17e+00 | 20260505 06:24:09.095 | INFO | PerfReport: 56.10%, FrameLabeling -- 1.86e+01 / 1.91e+01, 2.12e+00 / 9.19e-01 | 20260505 06:24:09.095 | INFO | PerfReport: 7.68%, PulseAccumulating -- 2.55e+00 / 3.15e+00, 4.62e-01 / 3.09e-01 | 20260505 06:24:09.095 | INFO | PerfReport: 4.10%, Metrics -- 1.36e+00 / 1.72e+00, 3.88e-01 / 2.01e-01 | 20260505 06:24:09.095 | INFO | PerfReport: 1.50%, PulseSerializing -- 4.97e-01 / 5.71e-01, 9.78e-02 / 4.09e-02 | 20260505 06:24:09.095 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 06:24:09.095 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:24:09.095 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:24:09.095 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.25e+01 / 0.00e+00 | 20260505 06:24:09.095 | INFO | PerfReport: 22.01%, Upload -- 2.24e+01 / 0.00e+00, 7.83e+00 / 0.00e+00 | 20260505 06:24:09.095 | INFO | PerfReport: 21.46%, Download -- 2.18e+01 / 0.00e+00, 7.76e+00 / 0.00e+00 | 20260505 06:24:09.095 | INFO | PerfReport: 32.65%, Compute -- 3.32e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 06:24:09.095 | INFO | PerfReport: 1.44%, LockUpload -- 1.47e+00 / 0.00e+00, 6.77e+00 / 0.00e+00 | 20260505 06:24:09.095 | INFO | PerfReport: 0.71%, LockDownload -- 7.22e-01 / 0.00e+00, 2.05e+00 / 0.00e+00 | 20260505 06:24:09.095 | INFO | PerfReport: 21.72%, LockCompute -- 2.21e+01 / 0.00e+00, 1.66e+01 / 0.00e+00 | 20260505 06:24:09.095 | INFO | PerfReport: 0.01%, Misc -- 1.46e-02 / 0.00e+00, 2.05e-02 / 0.00e+00 | 20260505 06:24:09.215 | INFO | Starting a round of Baz IO | 20260505 06:24:12.464 | INFO | Populating RTMetrics report | 20260505 06:24:12.464 | INFO | RTMetrics report populated | 20260505 06:24:12.464 | INFO | Opening tmp RTMetrics json file | 20260505 06:24:12.488 | INFO | Serializing RTMetrics to json | 20260505 06:24:12.512 | INFO | Serialization done, swapping tmp file now... | 20260505 06:24:12.537 | INFO | RTMetrics done | 20260505 06:24:12.576 | INFO | Finished a round of Baz IO: Wrote 2.37 GiB in 3.36 seconds | 20260505 06:24:12.576 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.53% / 0.05% / 66.42% | 20260505 06:24:12.576 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.23527% / 5.16574e-05% / 4.76837e-05% | 20260505 06:24:12.576 | INFO | Baz IO overall: | 20260505 06:24:12.576 | INFO | BytesWritten : 530462959436 | 20260505 06:24:12.576 | INFO | NumEvents : 250739122268 | 20260505 06:24:14.232 | INFO | Analyzing chunk frames = [1032192,1032704) | 20260505 06:24:14.232 | INFO | MemInfo: { | 20260505 06:24:14.232 | INFO | "gpu" : " 17 GB", | 20260505 06:24:14.232 | INFO | "huge" : "", | 20260505 06:24:14.232 | INFO | "huge_shared" : " 27 GB", | 20260505 06:24:14.232 | INFO | "malloc" : " 14 GB", | 20260505 06:24:14.232 | INFO | "pinned" : " 29 GB" | 20260505 06:24:14.232 | INFO | } | 20260505 06:24:14.232 | INFO | Perf report for the last 5.13927 seconds: | 20260505 06:24:14.233 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:24:14.233 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:14.233 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:14.233 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:14.233 | INFO | ANALYSIS : 64.4%, 2.9, 97.9ms, 0 | 20260505 06:24:14.233 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:14.233 | INFO | RT_METRICS : 47.4%, 1.0, 24.8ms, 0 | 20260505 06:24:14.233 | INFO | PRE_HQ : 63.5%, 1.9, 64.2ms, 0 | 20260505 06:24:14.233 | INFO | BAZWRITER : 64.7%, 4.1, 137.2ms, 0 | 20260505 06:24:14.233 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 06:24:14.233 | INFO | Uploaded 36GiB to GPU in 2.12525s (16.9392GiB/s) | 20260505 06:24:14.233 | INFO | Downloaded 38GiB to GPU in 2.31439s (16.419GiB/s) | 20260505 06:24:19.372 | INFO | Analyzing chunk frames = [1032704,1033216) | 20260505 06:24:19.372 | INFO | MemInfo: { | 20260505 06:24:19.372 | INFO | "gpu" : " 17 GB", | 20260505 06:24:19.372 | INFO | "huge" : "", | 20260505 06:24:19.372 | INFO | "huge_shared" : " 27 GB", | 20260505 06:24:19.372 | INFO | "malloc" : " 14 GB", | 20260505 06:24:19.372 | INFO | "pinned" : " 29 GB" | 20260505 06:24:19.372 | INFO | } | 20260505 06:24:19.372 | INFO | Perf report for the last 5.13964 seconds: | 20260505 06:24:19.372 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:24:19.372 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:19.372 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:19.372 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:19.372 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 06:24:19.372 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:19.372 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:19.372 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 06:24:19.372 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:19.372 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:19.372 | INFO | Uploaded 36GiB to GPU in 2.1138s (17.0309GiB/s) | 20260505 06:24:19.372 | INFO | Downloaded 34GiB to GPU in 2.03829s (16.6806GiB/s) | 20260505 06:24:24.511 | INFO | Analyzing chunk frames = [1033216,1033728) | 20260505 06:24:24.511 | INFO | MemInfo: { | 20260505 06:24:24.511 | INFO | "gpu" : " 17 GB", | 20260505 06:24:24.511 | INFO | "huge" : "", | 20260505 06:24:24.511 | INFO | "huge_shared" : " 27 GB", | 20260505 06:24:24.511 | INFO | "malloc" : " 14 GB", | 20260505 06:24:24.511 | INFO | "pinned" : " 29 GB" | 20260505 06:24:24.511 | INFO | } | 20260505 06:24:24.511 | INFO | Perf report for the last 5.13907 seconds: | 20260505 06:24:24.511 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:24:24.511 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:24.511 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:24.511 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:24.511 | INFO | ANALYSIS : 63.7%, 3.0, 99.1ms, 0 | 20260505 06:24:24.511 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:24.511 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:24.511 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 06:24:24.511 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:24.511 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:24.512 | INFO | Uploaded 36GiB to GPU in 2.12604s (16.9329GiB/s) | 20260505 06:24:24.512 | INFO | Downloaded 34GiB to GPU in 2.03254s (16.7278GiB/s) | 20260505 06:24:29.650 | INFO | Analyzing chunk frames = [1033728,1034240) | 20260505 06:24:29.650 | INFO | MemInfo: { | 20260505 06:24:29.650 | INFO | "gpu" : " 17 GB", | 20260505 06:24:29.650 | INFO | "huge" : "", | 20260505 06:24:29.650 | INFO | "huge_shared" : " 27 GB", | 20260505 06:24:29.650 | INFO | "malloc" : " 14 GB", | 20260505 06:24:29.650 | INFO | "pinned" : " 29 GB" | 20260505 06:24:29.650 | INFO | } | 20260505 06:24:29.650 | INFO | Perf report for the last 5.13914 seconds: | 20260505 06:24:29.650 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:24:29.650 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:29.650 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:29.650 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:29.650 | INFO | ANALYSIS : 64.1%, 3.0, 99.5ms, 0 | 20260505 06:24:29.650 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:29.650 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:29.650 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 06:24:29.651 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:29.651 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:29.651 | INFO | Uploaded 36GiB to GPU in 2.18111s (16.5054GiB/s) | 20260505 06:24:29.651 | INFO | Downloaded 34GiB to GPU in 2.04398s (16.6342GiB/s) | 20260505 06:24:34.789 | INFO | Analyzing chunk frames = [1034240,1034752) | 20260505 06:24:34.789 | INFO | MemInfo: { | 20260505 06:24:34.789 | INFO | "gpu" : " 17 GB", | 20260505 06:24:34.789 | INFO | "huge" : "", | 20260505 06:24:34.789 | INFO | "huge_shared" : " 27 GB", | 20260505 06:24:34.789 | INFO | "malloc" : " 14 GB", | 20260505 06:24:34.789 | INFO | "pinned" : " 29 GB" | 20260505 06:24:34.789 | INFO | } | 20260505 06:24:34.790 | INFO | Perf report for the last 5.13917 seconds: | 20260505 06:24:34.790 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:24:34.790 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:34.790 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:34.790 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:34.790 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 06:24:34.790 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:34.790 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:34.790 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 06:24:34.790 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:34.790 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:34.790 | INFO | Uploaded 36GiB to GPU in 2.16017s (16.6653GiB/s) | 20260505 06:24:34.790 | INFO | Downloaded 34GiB to GPU in 2.0476s (16.6048GiB/s) | 20260505 06:24:39.930 | INFO | Analyzing chunk frames = [1034752,1035264) | 20260505 06:24:39.930 | INFO | MemInfo: { | 20260505 06:24:39.930 | INFO | "gpu" : " 17 GB", | 20260505 06:24:39.930 | INFO | "huge" : "", | 20260505 06:24:39.930 | INFO | "huge_shared" : " 27 GB", | 20260505 06:24:39.930 | INFO | "malloc" : " 14 GB", | 20260505 06:24:39.930 | INFO | "pinned" : " 29 GB" | 20260505 06:24:39.930 | INFO | } | 20260505 06:24:39.930 | INFO | Perf report for the last 5.14088 seconds: | 20260505 06:24:39.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:24:39.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:39.930 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:39.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:39.930 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 06:24:39.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:39.930 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:39.930 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 06:24:39.930 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:39.931 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:39.931 | INFO | Uploaded 36GiB to GPU in 2.10683s (17.0873GiB/s) | 20260505 06:24:39.931 | INFO | Downloaded 34GiB to GPU in 2.06203s (16.4886GiB/s) | 20260505 06:24:45.069 | INFO | Analyzing chunk frames = [1035264,1035776) | 20260505 06:24:45.069 | INFO | MemInfo: { | 20260505 06:24:45.069 | INFO | "gpu" : " 17 GB", | 20260505 06:24:45.069 | INFO | "huge" : "", | 20260505 06:24:45.069 | INFO | "huge_shared" : " 27 GB", | 20260505 06:24:45.069 | INFO | "malloc" : " 14 GB", | 20260505 06:24:45.069 | INFO | "pinned" : " 29 GB" | 20260505 06:24:45.069 | INFO | } | 20260505 06:24:45.069 | INFO | Perf report for the last 5.13848 seconds: | 20260505 06:24:45.069 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:24:45.069 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:45.069 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:45.069 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:45.069 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 06:24:45.069 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:45.069 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:45.069 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 06:24:45.069 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:45.069 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:45.069 | INFO | Uploaded 36GiB to GPU in 2.17254s (16.5704GiB/s) | 20260505 06:24:45.069 | INFO | Downloaded 34GiB to GPU in 2.16534s (15.7019GiB/s) | 20260505 06:24:50.207 | INFO | Analyzing chunk frames = [1035776,1036288) | 20260505 06:24:50.207 | INFO | MemInfo: { | 20260505 06:24:50.207 | INFO | "gpu" : " 17 GB", | 20260505 06:24:50.207 | INFO | "huge" : "", | 20260505 06:24:50.207 | INFO | "huge_shared" : " 27 GB", | 20260505 06:24:50.207 | INFO | "malloc" : " 14 GB", | 20260505 06:24:50.207 | INFO | "pinned" : " 29 GB" | 20260505 06:24:50.207 | INFO | } | 20260505 06:24:50.207 | INFO | Perf report for the last 5.13857 seconds: | 20260505 06:24:50.208 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:24:50.208 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:50.208 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:50.208 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:50.208 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 06:24:50.208 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:50.208 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:50.208 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 06:24:50.208 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:50.208 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:50.208 | INFO | Uploaded 36GiB to GPU in 2.11045s (17.0579GiB/s) | 20260505 06:24:50.208 | INFO | Downloaded 34GiB to GPU in 2.06872s (16.4353GiB/s) | 20260505 06:24:50.330 | INFO | Starting a round of Baz IO | 20260505 06:24:53.586 | INFO | Populating RTMetrics report | 20260505 06:24:53.586 | INFO | RTMetrics report populated | 20260505 06:24:53.587 | INFO | Opening tmp RTMetrics json file | 20260505 06:24:53.609 | INFO | Serializing RTMetrics to json | 20260505 06:24:53.612 | INFO | Serialization done, swapping tmp file now... | 20260505 06:24:53.616 | INFO | RTMetrics done | 20260505 06:24:53.706 | INFO | Finished a round of Baz IO: Wrote 2.369 GiB in 3.375 seconds | 20260505 06:24:53.706 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.48% / 0.05% / 66.47% | 20260505 06:24:53.706 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.19658% / 5.5631e-05% / 5.5631e-05% | 20260505 06:24:53.706 | INFO | Baz IO overall: | 20260505 06:24:53.706 | INFO | BytesWritten : 533007014074 | 20260505 06:24:53.706 | INFO | NumEvents : 251955438237 | 20260505 06:24:55.347 | INFO | Analyzing chunk frames = [1036288,1036800) | 20260505 06:24:55.347 | INFO | MemInfo: { | 20260505 06:24:55.347 | INFO | "gpu" : " 17 GB", | 20260505 06:24:55.347 | INFO | "huge" : "", | 20260505 06:24:55.347 | INFO | "huge_shared" : " 27 GB", | 20260505 06:24:55.347 | INFO | "malloc" : " 14 GB", | 20260505 06:24:55.347 | INFO | "pinned" : " 29 GB" | 20260505 06:24:55.347 | INFO | } | 20260505 06:24:55.347 | INFO | Perf report for the last 5.13946 seconds: | 20260505 06:24:55.347 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:24:55.347 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:55.347 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:55.347 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:55.347 | INFO | ANALYSIS : 64.6%, 2.9, 98.9ms, 0 | 20260505 06:24:55.347 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:24:55.347 | INFO | RT_METRICS : 46.5%, 1.0, 24.4ms, 0 | 20260505 06:24:55.347 | INFO | PRE_HQ : 63.5%, 1.9, 63.8ms, 0 | 20260505 06:24:55.347 | INFO | BAZWRITER : 65.0%, 4.1, 136.8ms, 0 | 20260505 06:24:55.347 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 06:24:55.347 | INFO | Uploaded 36GiB to GPU in 2.1025s (17.1225GiB/s) | 20260505 06:24:55.347 | INFO | Downloaded 38GiB to GPU in 2.3005s (16.5182GiB/s) | 20260505 06:25:00.486 | INFO | Analyzing chunk frames = [1036800,1037312) | 20260505 06:25:00.486 | INFO | MemInfo: { | 20260505 06:25:00.486 | INFO | "gpu" : " 17 GB", | 20260505 06:25:00.486 | INFO | "huge" : "", | 20260505 06:25:00.486 | INFO | "huge_shared" : " 27 GB", | 20260505 06:25:00.486 | INFO | "malloc" : " 14 GB", | 20260505 06:25:00.486 | INFO | "pinned" : " 29 GB" | 20260505 06:25:00.486 | INFO | } | 20260505 06:25:00.486 | INFO | Perf report for the last 5.13934 seconds: | 20260505 06:25:00.486 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:25:00.487 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:00.487 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:00.487 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:00.487 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 06:25:00.487 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:00.487 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:00.487 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 06:25:00.487 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:00.487 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:00.487 | INFO | Uploaded 36GiB to GPU in 2.12415s (16.948GiB/s) | 20260505 06:25:00.487 | INFO | Downloaded 34GiB to GPU in 2.03858s (16.6783GiB/s) | 20260505 06:25:05.625 | INFO | Analyzing chunk frames = [1037312,1037824) | 20260505 06:25:05.625 | INFO | MemInfo: { | 20260505 06:25:05.625 | INFO | "gpu" : " 17 GB", | 20260505 06:25:05.625 | INFO | "huge" : "", | 20260505 06:25:05.625 | INFO | "huge_shared" : " 27 GB", | 20260505 06:25:05.625 | INFO | "malloc" : " 14 GB", | 20260505 06:25:05.625 | INFO | "pinned" : " 29 GB" | 20260505 06:25:05.625 | INFO | } | 20260505 06:25:05.625 | INFO | Perf report for the last 5.13913 seconds: | 20260505 06:25:05.626 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:25:05.626 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:05.626 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:05.626 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:05.626 | INFO | ANALYSIS : 63.8%, 3.0, 99.3ms, 0 | 20260505 06:25:05.626 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:05.626 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:05.626 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 06:25:05.626 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:05.626 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:05.626 | INFO | Uploaded 36GiB to GPU in 2.13294s (16.8781GiB/s) | 20260505 06:25:05.626 | INFO | Downloaded 34GiB to GPU in 2.0314s (16.7372GiB/s) | 20260505 06:25:10.765 | INFO | Analyzing chunk frames = [1037824,1038336) | 20260505 06:25:10.765 | INFO | MemInfo: { | 20260505 06:25:10.765 | INFO | "gpu" : " 17 GB", | 20260505 06:25:10.765 | INFO | "huge" : "", | 20260505 06:25:10.765 | INFO | "huge_shared" : " 27 GB", | 20260505 06:25:10.765 | INFO | "malloc" : " 14 GB", | 20260505 06:25:10.765 | INFO | "pinned" : " 29 GB" | 20260505 06:25:10.765 | INFO | } | 20260505 06:25:10.765 | INFO | Perf report for the last 5.13966 seconds: | 20260505 06:25:10.765 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:25:10.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:10.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:10.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:10.765 | INFO | ANALYSIS : 64.1%, 3.0, 99.5ms, 0 | 20260505 06:25:10.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:10.765 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:10.765 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 06:25:10.765 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:10.765 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:10.766 | INFO | Uploaded 36GiB to GPU in 2.17961s (16.5167GiB/s) | 20260505 06:25:10.766 | INFO | Downloaded 34GiB to GPU in 2.0347s (16.7101GiB/s) | 20260505 06:25:15.904 | INFO | Analyzing chunk frames = [1038336,1038848) | 20260505 06:25:15.904 | INFO | MemInfo: { | 20260505 06:25:15.904 | INFO | "gpu" : " 17 GB", | 20260505 06:25:15.904 | INFO | "huge" : "", | 20260505 06:25:15.904 | INFO | "huge_shared" : " 27 GB", | 20260505 06:25:15.904 | INFO | "malloc" : " 14 GB", | 20260505 06:25:15.904 | INFO | "pinned" : " 29 GB" | 20260505 06:25:15.904 | INFO | } | 20260505 06:25:15.904 | INFO | Perf report for the last 5.13903 seconds: | 20260505 06:25:15.904 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:25:15.904 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:15.904 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:15.904 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:15.904 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 06:25:15.904 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:15.904 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:15.904 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 06:25:15.904 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:15.904 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:15.904 | INFO | Uploaded 36GiB to GPU in 2.16396s (16.6362GiB/s) | 20260505 06:25:15.905 | INFO | Downloaded 34GiB to GPU in 2.03958s (16.6701GiB/s) | 20260505 06:25:21.049 | INFO | Analyzing chunk frames = [1038848,1039360) | 20260505 06:25:21.049 | INFO | MemInfo: { | 20260505 06:25:21.049 | INFO | "gpu" : " 17 GB", | 20260505 06:25:21.049 | INFO | "huge" : "", | 20260505 06:25:21.049 | INFO | "huge_shared" : " 27 GB", | 20260505 06:25:21.049 | INFO | "malloc" : " 14 GB", | 20260505 06:25:21.049 | INFO | "pinned" : " 29 GB" | 20260505 06:25:21.049 | INFO | } | 20260505 06:25:21.049 | INFO | Perf report for the last 5.14532 seconds: | 20260505 06:25:21.050 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:25:21.050 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:21.050 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:21.050 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:21.050 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 06:25:21.050 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:21.050 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:21.050 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 06:25:21.050 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:21.050 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:21.050 | INFO | Uploaded 36GiB to GPU in 2.15542s (16.7021GiB/s) | 20260505 06:25:21.050 | INFO | Downloaded 34GiB to GPU in 2.04823s (16.5997GiB/s) | 20260505 06:25:26.184 | INFO | Analyzing chunk frames = [1039360,1039872) | 20260505 06:25:26.184 | INFO | MemInfo: { | 20260505 06:25:26.184 | INFO | "gpu" : " 17 GB", | 20260505 06:25:26.184 | INFO | "huge" : "", | 20260505 06:25:26.184 | INFO | "huge_shared" : " 27 GB", | 20260505 06:25:26.184 | INFO | "malloc" : " 14 GB", | 20260505 06:25:26.184 | INFO | "pinned" : " 29 GB" | 20260505 06:25:26.184 | INFO | } | 20260505 06:25:26.184 | INFO | Perf report for the last 5.1349 seconds: | 20260505 06:25:26.185 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:25:26.185 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:26.185 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:26.185 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:26.185 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 06:25:26.185 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:26.185 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:26.185 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260505 06:25:26.185 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:26.185 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:26.185 | INFO | Uploaded 36GiB to GPU in 2.18137s (16.5034GiB/s) | 20260505 06:25:26.185 | INFO | Downloaded 34GiB to GPU in 2.14957s (15.8171GiB/s) | 20260505 06:25:31.322 | INFO | Analyzing chunk frames = [1039872,1040384) | 20260505 06:25:31.322 | INFO | MemInfo: { | 20260505 06:25:31.322 | INFO | "gpu" : " 17 GB", | 20260505 06:25:31.322 | INFO | "huge" : "", | 20260505 06:25:31.322 | INFO | "huge_shared" : " 27 GB", | 20260505 06:25:31.322 | INFO | "malloc" : " 14 GB", | 20260505 06:25:31.322 | INFO | "pinned" : " 29 GB" | 20260505 06:25:31.322 | INFO | } | 20260505 06:25:31.323 | INFO | Perf report for the last 5.13811 seconds: | 20260505 06:25:31.323 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:25:31.323 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:31.323 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:31.323 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:31.323 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 06:25:31.323 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:31.323 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:31.323 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260505 06:25:31.323 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:31.323 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:31.323 | INFO | Uploaded 36GiB to GPU in 2.08051s (17.3034GiB/s) | 20260505 06:25:31.323 | INFO | Downloaded 34GiB to GPU in 2.09564s (16.2242GiB/s) | 20260505 06:25:31.323 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 06:25:31.323 | INFO | Total mem usage: 15.3 GiB | 20260505 06:25:31.323 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 06:25:31.323 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.56%) | 20260505 06:25:31.323 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 06:25:31.323 | INFO | High Water Breakdown: | 20260505 06:25:31.323 | INFO | 60.3% : 8.67 GiB ( 92.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 06:25:31.324 | INFO | 39.6% : 5.69 GiB ( 85.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 06:25:31.324 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 06:25:31.324 | INFO | ------------------------------------------ | 20260505 06:25:31.324 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:25:31.324 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 06:25:31.324 | INFO | Total mem usage: 26.5 GiB | 20260505 06:25:31.324 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 06:25:31.324 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 06:25:31.324 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 06:25:31.324 | INFO | High Water Breakdown: | 20260505 06:25:31.324 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:25:31.324 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:25:31.324 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:25:31.324 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:25:31.324 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:25:31.324 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:25:31.324 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 06:25:31.324 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 06:25:31.324 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:25:31.324 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 06:25:31.324 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 06:25:31.324 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 06:25:31.324 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 06:25:31.324 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 06:25:31.324 | INFO | ------------------------------------------ | 20260505 06:25:31.324 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:25:31.324 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 06:25:31.324 | INFO | Total mem usage: 25.6 GiB | 20260505 06:25:31.324 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 06:25:31.324 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 06:25:31.324 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 06:25:31.324 | INFO | High Water Breakdown: | 20260505 06:25:31.324 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 06:25:31.324 | INFO | ------------------------------------------ | 20260505 06:25:31.324 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:25:31.324 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 06:25:31.324 | INFO | Total mem usage: 15.7 GiB | 20260505 06:25:31.324 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 06:25:31.324 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 06:25:31.324 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260505 06:25:31.324 | INFO | High Water Breakdown: | 20260505 06:25:31.324 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 06:25:31.324 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 06:25:31.324 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 06:25:31.324 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 06:25:31.324 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 06:25:31.324 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 06:25:31.324 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 06:25:31.324 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 06:25:31.324 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 06:25:31.324 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:25:31.324 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 06:25:31.324 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:25:31.324 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 06:25:31.324 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 06:25:31.324 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 06:25:31.324 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 06:25:31.324 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:25:31.324 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 06:25:31.324 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:25:31.324 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:25:31.324 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:25:31.324 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 06:25:31.324 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 06:25:31.324 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:25:31.324 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 06:25:31.324 | INFO | ------------------------------------------ | 20260505 06:25:31.324 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:25:31.324 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 06:25:31.324 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:25:31.324 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:25:31.324 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.55e+01, 1.35e+01 / 1.72e+00 | 20260505 06:25:31.324 | INFO | PerfReport: 0.28%, AggregateStats -- 9.41e-02 / 1.04e-01, 1.16e-02 / 5.09e-03 | 20260505 06:25:31.324 | INFO | PerfReport: 11.58%, Baseline -- 3.84e+00 / 4.49e+00, 6.44e-01 / 2.95e-01 | 20260505 06:25:31.324 | INFO | PerfReport: 3.31%, Histogram -- 1.10e+00 / 1.50e+00, 2.20e-01 / 1.07e-01 | 20260505 06:25:31.325 | INFO | PerfReport: 15.46%, DME -- 4.10e+01 / 4.49e+01, 4.39e+00 / 1.15e+00 | 20260505 06:25:31.325 | INFO | PerfReport: 56.03%, FrameLabeling -- 1.86e+01 / 1.90e+01, 2.09e+00 / 9.79e-01 | 20260505 06:25:31.325 | INFO | PerfReport: 7.71%, PulseAccumulating -- 2.56e+00 / 3.13e+00, 4.74e-01 / 3.43e-01 | 20260505 06:25:31.325 | INFO | PerfReport: 4.12%, Metrics -- 1.37e+00 / 1.70e+00, 3.97e-01 / 2.23e-01 | 20260505 06:25:31.325 | INFO | PerfReport: 1.50%, PulseSerializing -- 4.97e-01 / 5.69e-01, 9.84e-02 / 4.12e-02 | 20260505 06:25:31.325 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 06:25:31.325 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:25:31.325 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:25:31.325 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.24e+01 / 0.00e+00 | 20260505 06:25:31.325 | INFO | PerfReport: 21.95%, Upload -- 2.23e+01 / 0.00e+00, 7.70e+00 / 0.00e+00 | 20260505 06:25:31.325 | INFO | PerfReport: 21.49%, Download -- 2.18e+01 / 0.00e+00, 7.80e+00 / 0.00e+00 | 20260505 06:25:31.325 | INFO | PerfReport: 32.70%, Compute -- 3.32e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 06:25:31.325 | INFO | PerfReport: 1.44%, LockUpload -- 1.46e+00 / 0.00e+00, 6.75e+00 / 0.00e+00 | 20260505 06:25:31.325 | INFO | PerfReport: 0.73%, LockDownload -- 7.37e-01 / 0.00e+00, 2.10e+00 / 0.00e+00 | 20260505 06:25:31.325 | INFO | PerfReport: 21.68%, LockCompute -- 2.20e+01 / 0.00e+00, 1.67e+01 / 0.00e+00 | 20260505 06:25:31.325 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.71e-02 / 0.00e+00 | 20260505 06:25:31.445 | INFO | Starting a round of Baz IO | 20260505 06:25:34.694 | INFO | Populating RTMetrics report | 20260505 06:25:34.695 | INFO | RTMetrics report populated | 20260505 06:25:34.695 | INFO | Opening tmp RTMetrics json file | 20260505 06:25:34.717 | INFO | Serializing RTMetrics to json | 20260505 06:25:34.727 | INFO | Serialization done, swapping tmp file now... | 20260505 06:25:34.744 | INFO | RTMetrics done | 20260505 06:25:34.819 | INFO | Finished a round of Baz IO: Wrote 2.373 GiB in 3.373 seconds | 20260505 06:25:34.819 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.43% / 0.05% / 66.52% | 20260505 06:25:34.819 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.19564% / 7.54992e-05% / 6.35783e-05% | 20260505 06:25:34.819 | INFO | Baz IO overall: | 20260505 06:25:34.819 | INFO | BytesWritten : 535555044918 | 20260505 06:25:34.819 | INFO | NumEvents : 253174461518 | 20260505 06:25:36.462 | INFO | Analyzing chunk frames = [1040384,1040896) | 20260505 06:25:36.462 | INFO | MemInfo: { | 20260505 06:25:36.462 | INFO | "gpu" : " 17 GB", | 20260505 06:25:36.462 | INFO | "huge" : "", | 20260505 06:25:36.462 | INFO | "huge_shared" : " 27 GB", | 20260505 06:25:36.462 | INFO | "malloc" : " 14 GB", | 20260505 06:25:36.462 | INFO | "pinned" : " 29 GB" | 20260505 06:25:36.462 | INFO | } | 20260505 06:25:36.462 | INFO | Perf report for the last 5.13952 seconds: | 20260505 06:25:36.462 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:25:36.462 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:36.462 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:36.462 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:36.462 | INFO | ANALYSIS : 64.5%, 2.9, 98.1ms, 0 | 20260505 06:25:36.462 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:36.462 | INFO | RT_METRICS : 47.1%, 1.0, 24.7ms, 0 | 20260505 06:25:36.462 | INFO | PRE_HQ : 63.4%, 2.0, 64.2ms, 0 | 20260505 06:25:36.462 | INFO | BAZWRITER : 64.9%, 4.1, 137.5ms, 0 | 20260505 06:25:36.462 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 06:25:36.462 | INFO | Uploaded 36GiB to GPU in 2.07951s (17.3117GiB/s) | 20260505 06:25:36.462 | INFO | Downloaded 38GiB to GPU in 2.30867s (16.4597GiB/s) | 20260505 06:25:41.601 | INFO | Analyzing chunk frames = [1040896,1041408) | 20260505 06:25:41.601 | INFO | MemInfo: { | 20260505 06:25:41.601 | INFO | "gpu" : " 17 GB", | 20260505 06:25:41.601 | INFO | "huge" : "", | 20260505 06:25:41.601 | INFO | "huge_shared" : " 27 GB", | 20260505 06:25:41.601 | INFO | "malloc" : " 14 GB", | 20260505 06:25:41.601 | INFO | "pinned" : " 29 GB" | 20260505 06:25:41.601 | INFO | } | 20260505 06:25:41.601 | INFO | Perf report for the last 5.13936 seconds: | 20260505 06:25:41.601 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:25:41.601 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:41.601 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:41.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:41.602 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 06:25:41.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:41.602 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:41.602 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 06:25:41.602 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:41.602 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:41.602 | INFO | Uploaded 36GiB to GPU in 2.11308s (17.0368GiB/s) | 20260505 06:25:41.602 | INFO | Downloaded 34GiB to GPU in 2.04238s (16.6473GiB/s) | 20260505 06:25:46.740 | INFO | Analyzing chunk frames = [1041408,1041920) | 20260505 06:25:46.740 | INFO | MemInfo: { | 20260505 06:25:46.740 | INFO | "gpu" : " 17 GB", | 20260505 06:25:46.740 | INFO | "huge" : "", | 20260505 06:25:46.740 | INFO | "huge_shared" : " 27 GB", | 20260505 06:25:46.740 | INFO | "malloc" : " 14 GB", | 20260505 06:25:46.740 | INFO | "pinned" : " 29 GB" | 20260505 06:25:46.740 | INFO | } | 20260505 06:25:46.740 | INFO | Perf report for the last 5.13891 seconds: | 20260505 06:25:46.741 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:25:46.741 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:46.741 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:46.741 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:46.741 | INFO | ANALYSIS : 63.8%, 3.0, 99.3ms, 0 | 20260505 06:25:46.741 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:46.741 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:46.741 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 06:25:46.741 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:46.741 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:46.741 | INFO | Uploaded 36GiB to GPU in 2.14167s (16.8093GiB/s) | 20260505 06:25:46.741 | INFO | Downloaded 34GiB to GPU in 2.03795s (16.6835GiB/s) | 20260505 06:25:51.880 | INFO | Analyzing chunk frames = [1041920,1042432) | 20260505 06:25:51.880 | INFO | MemInfo: { | 20260505 06:25:51.880 | INFO | "gpu" : " 17 GB", | 20260505 06:25:51.880 | INFO | "huge" : "", | 20260505 06:25:51.880 | INFO | "huge_shared" : " 27 GB", | 20260505 06:25:51.880 | INFO | "malloc" : " 14 GB", | 20260505 06:25:51.880 | INFO | "pinned" : " 29 GB" | 20260505 06:25:51.880 | INFO | } | 20260505 06:25:51.880 | INFO | Perf report for the last 5.13953 seconds: | 20260505 06:25:51.880 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:25:51.880 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:51.880 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:51.880 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:51.880 | INFO | ANALYSIS : 63.8%, 3.0, 99.1ms, 0 | 20260505 06:25:51.880 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:51.880 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:51.880 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 06:25:51.880 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:51.880 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:51.880 | INFO | Uploaded 36GiB to GPU in 2.15976s (16.6685GiB/s) | 20260505 06:25:51.880 | INFO | Downloaded 34GiB to GPU in 2.04542s (16.6225GiB/s) | 20260505 06:25:57.019 | INFO | Analyzing chunk frames = [1042432,1042944) | 20260505 06:25:57.019 | INFO | MemInfo: { | 20260505 06:25:57.019 | INFO | "gpu" : " 17 GB", | 20260505 06:25:57.019 | INFO | "huge" : "", | 20260505 06:25:57.019 | INFO | "huge_shared" : " 27 GB", | 20260505 06:25:57.019 | INFO | "malloc" : " 14 GB", | 20260505 06:25:57.019 | INFO | "pinned" : " 29 GB" | 20260505 06:25:57.019 | INFO | } | 20260505 06:25:57.019 | INFO | Perf report for the last 5.13932 seconds: | 20260505 06:25:57.019 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:25:57.019 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:57.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:57.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:57.020 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 06:25:57.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:57.020 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:57.020 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 06:25:57.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:57.020 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:25:57.020 | INFO | Uploaded 36GiB to GPU in 2.16777s (16.6069GiB/s) | 20260505 06:25:57.020 | INFO | Downloaded 34GiB to GPU in 2.03923s (16.6729GiB/s) | 20260505 06:26:02.160 | INFO | Analyzing chunk frames = [1042944,1043456) | 20260505 06:26:02.160 | INFO | MemInfo: { | 20260505 06:26:02.160 | INFO | "gpu" : " 17 GB", | 20260505 06:26:02.160 | INFO | "huge" : "", | 20260505 06:26:02.160 | INFO | "huge_shared" : " 27 GB", | 20260505 06:26:02.160 | INFO | "malloc" : " 14 GB", | 20260505 06:26:02.160 | INFO | "pinned" : " 29 GB" | 20260505 06:26:02.160 | INFO | } | 20260505 06:26:02.160 | INFO | Perf report for the last 5.14104 seconds: | 20260505 06:26:02.160 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:26:02.160 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:02.160 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:02.160 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:02.160 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 06:26:02.160 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:02.160 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:02.160 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 06:26:02.160 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:02.160 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:02.161 | INFO | Uploaded 36GiB to GPU in 2.14515s (16.7821GiB/s) | 20260505 06:26:02.161 | INFO | Downloaded 34GiB to GPU in 2.0521s (16.5684GiB/s) | 20260505 06:26:07.298 | INFO | Analyzing chunk frames = [1043456,1043968) | 20260505 06:26:07.298 | INFO | MemInfo: { | 20260505 06:26:07.298 | INFO | "gpu" : " 17 GB", | 20260505 06:26:07.298 | INFO | "huge" : "", | 20260505 06:26:07.298 | INFO | "huge_shared" : " 27 GB", | 20260505 06:26:07.298 | INFO | "malloc" : " 14 GB", | 20260505 06:26:07.298 | INFO | "pinned" : " 29 GB" | 20260505 06:26:07.298 | INFO | } | 20260505 06:26:07.298 | INFO | Perf report for the last 5.13793 seconds: | 20260505 06:26:07.298 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:26:07.298 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:07.298 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:07.298 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:07.298 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 06:26:07.298 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:07.298 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:07.298 | INFO | PRE_HQ : 32.2%, 1.0, 16.8ms, 0 | 20260505 06:26:07.298 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:07.298 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:07.299 | INFO | Uploaded 36GiB to GPU in 2.16697s (16.613GiB/s) | 20260505 06:26:07.299 | INFO | Downloaded 34GiB to GPU in 2.15575s (15.7718GiB/s) | 20260505 06:26:12.437 | INFO | Analyzing chunk frames = [1043968,1044480) | 20260505 06:26:12.437 | INFO | MemInfo: { | 20260505 06:26:12.437 | INFO | "gpu" : " 17 GB", | 20260505 06:26:12.437 | INFO | "huge" : "", | 20260505 06:26:12.437 | INFO | "huge_shared" : " 27 GB", | 20260505 06:26:12.437 | INFO | "malloc" : " 14 GB", | 20260505 06:26:12.437 | INFO | "pinned" : " 29 GB" | 20260505 06:26:12.437 | INFO | } | 20260505 06:26:12.437 | INFO | Perf report for the last 5.1391 seconds: | 20260505 06:26:12.437 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:26:12.437 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:12.437 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:12.437 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:12.437 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 06:26:12.437 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:12.437 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:12.437 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 06:26:12.437 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:12.438 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:12.438 | INFO | Uploaded 36GiB to GPU in 2.10001s (17.1427GiB/s) | 20260505 06:26:12.438 | INFO | Downloaded 34GiB to GPU in 2.06291s (16.4816GiB/s) | 20260505 06:26:12.559 | INFO | Starting a round of Baz IO | 20260505 06:26:15.828 | INFO | Populating RTMetrics report | 20260505 06:26:15.829 | INFO | RTMetrics report populated | 20260505 06:26:15.829 | INFO | Opening tmp RTMetrics json file | 20260505 06:26:15.848 | INFO | Serializing RTMetrics to json | 20260505 06:26:15.863 | INFO | Serialization done, swapping tmp file now... | 20260505 06:26:15.863 | INFO | RTMetrics done | 20260505 06:26:15.961 | INFO | Finished a round of Baz IO: Wrote 2.371 GiB in 3.401 seconds | 20260505 06:26:15.961 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.38% / 0.05% / 66.57% | 20260505 06:26:15.961 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.1558% / 9.93411e-05% / 4.76837e-05% | 20260505 06:26:15.961 | INFO | Baz IO overall: | 20260505 06:26:15.961 | INFO | BytesWritten : 538100656607 | 20260505 06:26:15.961 | INFO | NumEvents : 254391579071 | 20260505 06:26:17.577 | INFO | Analyzing chunk frames = [1044480,1044992) | 20260505 06:26:17.577 | INFO | MemInfo: { | 20260505 06:26:17.577 | INFO | "gpu" : " 17 GB", | 20260505 06:26:17.577 | INFO | "huge" : "", | 20260505 06:26:17.577 | INFO | "huge_shared" : " 27 GB", | 20260505 06:26:17.577 | INFO | "malloc" : " 14 GB", | 20260505 06:26:17.577 | INFO | "pinned" : " 29 GB" | 20260505 06:26:17.577 | INFO | } | 20260505 06:26:17.577 | INFO | Perf report for the last 5.13941 seconds: | 20260505 06:26:17.577 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:26:17.577 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:17.577 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:17.577 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:17.577 | INFO | ANALYSIS : 64.8%, 2.9, 98.4ms, 0 | 20260505 06:26:17.577 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:17.577 | INFO | RT_METRICS : 47.0%, 1.0, 24.7ms, 0 | 20260505 06:26:17.577 | INFO | PRE_HQ : 63.9%, 2.0, 64.7ms, 0 | 20260505 06:26:17.577 | INFO | BAZWRITER : 65.5%, 4.1, 138.8ms, 0 | 20260505 06:26:17.577 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 06:26:17.577 | INFO | Uploaded 36GiB to GPU in 2.13801s (16.8381GiB/s) | 20260505 06:26:17.577 | INFO | Downloaded 38GiB to GPU in 2.3076s (16.4673GiB/s) | 20260505 06:26:22.716 | INFO | Analyzing chunk frames = [1044992,1045504) | 20260505 06:26:22.716 | INFO | MemInfo: { | 20260505 06:26:22.716 | INFO | "gpu" : " 17 GB", | 20260505 06:26:22.716 | INFO | "huge" : "", | 20260505 06:26:22.716 | INFO | "huge_shared" : " 27 GB", | 20260505 06:26:22.716 | INFO | "malloc" : " 14 GB", | 20260505 06:26:22.716 | INFO | "pinned" : " 29 GB" | 20260505 06:26:22.716 | INFO | } | 20260505 06:26:22.716 | INFO | Perf report for the last 5.13927 seconds: | 20260505 06:26:22.716 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:26:22.716 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:22.716 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:22.716 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:22.716 | INFO | ANALYSIS : 64.1%, 3.0, 100.0ms, 0 | 20260505 06:26:22.716 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:22.716 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:22.716 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 06:26:22.716 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:22.716 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:22.716 | INFO | Uploaded 36GiB to GPU in 2.11376s (17.0313GiB/s) | 20260505 06:26:22.716 | INFO | Downloaded 34GiB to GPU in 2.04097s (16.6587GiB/s) | 20260505 06:26:27.855 | INFO | Analyzing chunk frames = [1045504,1046016) | 20260505 06:26:27.855 | INFO | MemInfo: { | 20260505 06:26:27.855 | INFO | "gpu" : " 17 GB", | 20260505 06:26:27.855 | INFO | "huge" : "", | 20260505 06:26:27.855 | INFO | "huge_shared" : " 27 GB", | 20260505 06:26:27.855 | INFO | "malloc" : " 14 GB", | 20260505 06:26:27.855 | INFO | "pinned" : " 29 GB" | 20260505 06:26:27.855 | INFO | } | 20260505 06:26:27.855 | INFO | Perf report for the last 5.13933 seconds: | 20260505 06:26:27.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:26:27.855 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:27.855 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:27.855 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:27.855 | INFO | ANALYSIS : 63.9%, 3.0, 99.4ms, 0 | 20260505 06:26:27.855 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:27.856 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:27.856 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 06:26:27.856 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:27.856 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:27.856 | INFO | Uploaded 36GiB to GPU in 2.14077s (16.8164GiB/s) | 20260505 06:26:27.856 | INFO | Downloaded 34GiB to GPU in 2.03896s (16.6752GiB/s) | 20260505 06:26:32.994 | INFO | Analyzing chunk frames = [1046016,1046528) | 20260505 06:26:32.994 | INFO | MemInfo: { | 20260505 06:26:32.994 | INFO | "gpu" : " 17 GB", | 20260505 06:26:32.994 | INFO | "huge" : "", | 20260505 06:26:32.994 | INFO | "huge_shared" : " 27 GB", | 20260505 06:26:32.994 | INFO | "malloc" : " 14 GB", | 20260505 06:26:32.995 | INFO | "pinned" : " 29 GB" | 20260505 06:26:32.995 | INFO | } | 20260505 06:26:32.995 | INFO | Perf report for the last 5.13915 seconds: | 20260505 06:26:32.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:26:32.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:32.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:32.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:32.995 | INFO | ANALYSIS : 63.9%, 3.0, 99.4ms, 0 | 20260505 06:26:32.995 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:32.995 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:32.995 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 06:26:32.995 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:32.995 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:32.995 | INFO | Uploaded 36GiB to GPU in 2.18073s (16.5082GiB/s) | 20260505 06:26:32.995 | INFO | Downloaded 34GiB to GPU in 2.04036s (16.6637GiB/s) | 20260505 06:26:38.134 | INFO | Analyzing chunk frames = [1046528,1047040) | 20260505 06:26:38.134 | INFO | MemInfo: { | 20260505 06:26:38.134 | INFO | "gpu" : " 17 GB", | 20260505 06:26:38.134 | INFO | "huge" : "", | 20260505 06:26:38.134 | INFO | "huge_shared" : " 27 GB", | 20260505 06:26:38.134 | INFO | "malloc" : " 14 GB", | 20260505 06:26:38.134 | INFO | "pinned" : " 29 GB" | 20260505 06:26:38.134 | INFO | } | 20260505 06:26:38.134 | INFO | Perf report for the last 5.13974 seconds: | 20260505 06:26:38.134 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:26:38.134 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:38.134 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:38.134 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:38.134 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 06:26:38.134 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:38.134 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:38.134 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 06:26:38.134 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:38.134 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:38.135 | INFO | Uploaded 36GiB to GPU in 2.15521s (16.7037GiB/s) | 20260505 06:26:38.135 | INFO | Downloaded 34GiB to GPU in 2.04436s (16.6311GiB/s) | 20260505 06:26:43.275 | INFO | Analyzing chunk frames = [1047040,1047552) | 20260505 06:26:43.275 | INFO | MemInfo: { | 20260505 06:26:43.275 | INFO | "gpu" : " 17 GB", | 20260505 06:26:43.275 | INFO | "huge" : "", | 20260505 06:26:43.275 | INFO | "huge_shared" : " 27 GB", | 20260505 06:26:43.275 | INFO | "malloc" : " 14 GB", | 20260505 06:26:43.275 | INFO | "pinned" : " 29 GB" | 20260505 06:26:43.275 | INFO | } | 20260505 06:26:43.275 | INFO | Perf report for the last 5.14081 seconds: | 20260505 06:26:43.275 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:26:43.275 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:43.275 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:43.275 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:43.275 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 06:26:43.275 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:43.275 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:43.275 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 06:26:43.275 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:43.275 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:43.275 | INFO | Uploaded 36GiB to GPU in 2.14789s (16.7607GiB/s) | 20260505 06:26:43.275 | INFO | Downloaded 34GiB to GPU in 2.04835s (16.5988GiB/s) | 20260505 06:26:48.413 | INFO | Analyzing chunk frames = [1047552,1048064) | 20260505 06:26:48.413 | INFO | MemInfo: { | 20260505 06:26:48.413 | INFO | "gpu" : " 17 GB", | 20260505 06:26:48.413 | INFO | "huge" : "", | 20260505 06:26:48.413 | INFO | "huge_shared" : " 27 GB", | 20260505 06:26:48.413 | INFO | "malloc" : " 14 GB", | 20260505 06:26:48.413 | INFO | "pinned" : " 29 GB" | 20260505 06:26:48.413 | INFO | } | 20260505 06:26:48.413 | INFO | Perf report for the last 5.13782 seconds: | 20260505 06:26:48.413 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:26:48.413 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:48.413 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:48.413 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:48.413 | INFO | ANALYSIS : 64.1%, 3.0, 100.0ms, 0 | 20260505 06:26:48.413 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:48.413 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:48.413 | INFO | PRE_HQ : 32.2%, 1.0, 16.8ms, 0 | 20260505 06:26:48.413 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:48.413 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:48.413 | INFO | Uploaded 36GiB to GPU in 2.14432s (16.7885GiB/s) | 20260505 06:26:48.413 | INFO | Downloaded 34GiB to GPU in 2.15957s (15.7439GiB/s) | 20260505 06:26:53.552 | INFO | Analyzing chunk frames = [1048064,1048576) | 20260505 06:26:53.552 | INFO | MemInfo: { | 20260505 06:26:53.552 | INFO | "gpu" : " 17 GB", | 20260505 06:26:53.552 | INFO | "huge" : "", | 20260505 06:26:53.552 | INFO | "huge_shared" : " 27 GB", | 20260505 06:26:53.552 | INFO | "malloc" : " 14 GB", | 20260505 06:26:53.552 | INFO | "pinned" : " 29 GB" | 20260505 06:26:53.552 | INFO | } | 20260505 06:26:53.552 | INFO | Perf report for the last 5.13911 seconds: | 20260505 06:26:53.552 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:26:53.552 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:53.552 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:53.552 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:53.552 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 06:26:53.552 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:53.552 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:53.552 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 06:26:53.552 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:53.552 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:53.552 | INFO | Uploaded 36GiB to GPU in 2.1169s (17.006GiB/s) | 20260505 06:26:53.552 | INFO | Downloaded 34GiB to GPU in 2.07486s (16.3867GiB/s) | 20260505 06:26:53.553 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 06:26:53.553 | INFO | Total mem usage: 15.3 GiB | 20260505 06:26:53.553 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 06:26:53.553 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.52%) | 20260505 06:26:53.553 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 06:26:53.553 | INFO | High Water Breakdown: | 20260505 06:26:53.553 | INFO | 60.2% : 8.67 GiB ( 92.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 06:26:53.553 | INFO | 39.6% : 5.7 GiB ( 85.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 06:26:53.553 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 06:26:53.553 | INFO | ------------------------------------------ | 20260505 06:26:53.553 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:26:53.553 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 06:26:53.553 | INFO | Total mem usage: 26.5 GiB | 20260505 06:26:53.553 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 06:26:53.553 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 06:26:53.553 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 06:26:53.553 | INFO | High Water Breakdown: | 20260505 06:26:53.553 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:26:53.553 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:26:53.553 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:26:53.553 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:26:53.553 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:26:53.553 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:26:53.553 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 06:26:53.553 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 06:26:53.553 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:26:53.553 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 06:26:53.553 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 06:26:53.553 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 06:26:53.553 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 06:26:53.553 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 06:26:53.553 | INFO | ------------------------------------------ | 20260505 06:26:53.553 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:26:53.553 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 06:26:53.553 | INFO | Total mem usage: 25.6 GiB | 20260505 06:26:53.553 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 06:26:53.553 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 06:26:53.553 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 06:26:53.553 | INFO | High Water Breakdown: | 20260505 06:26:53.553 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 06:26:53.553 | INFO | ------------------------------------------ | 20260505 06:26:53.553 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:26:53.553 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 06:26:53.553 | INFO | Total mem usage: 15.7 GiB | 20260505 06:26:53.553 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 06:26:53.553 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 06:26:53.553 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260505 06:26:53.553 | INFO | High Water Breakdown: | 20260505 06:26:53.553 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 06:26:53.553 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 06:26:53.553 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 06:26:53.553 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 06:26:53.553 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 06:26:53.553 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 06:26:53.553 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 06:26:53.553 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 06:26:53.553 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 06:26:53.553 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:26:53.553 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 06:26:53.553 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:26:53.553 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 06:26:53.553 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 06:26:53.553 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 06:26:53.553 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 06:26:53.554 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:26:53.554 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 06:26:53.554 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:26:53.554 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:26:53.554 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:26:53.554 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 06:26:53.554 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 06:26:53.554 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:26:53.554 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 06:26:53.554 | INFO | ------------------------------------------ | 20260505 06:26:53.554 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:26:53.554 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 06:26:53.554 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:26:53.554 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:26:53.554 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.56e+01, 1.37e+01 / 1.56e+00 | 20260505 06:26:53.554 | INFO | PerfReport: 0.28%, AggregateStats -- 9.39e-02 / 1.04e-01, 1.08e-02 / 6.50e-03 | 20260505 06:26:53.554 | INFO | PerfReport: 11.58%, Baseline -- 3.84e+00 / 4.48e+00, 6.43e-01 / 3.03e-01 | 20260505 06:26:53.554 | INFO | PerfReport: 3.31%, Histogram -- 1.10e+00 / 1.49e+00, 2.19e-01 / 1.24e-01 | 20260505 06:26:53.554 | INFO | PerfReport: 15.45%, DME -- 4.10e+01 / 4.50e+01, 4.34e+00 / 1.28e+00 | 20260505 06:26:53.554 | INFO | PerfReport: 56.08%, FrameLabeling -- 1.86e+01 / 1.92e+01, 2.08e+00 / 1.20e+00 | 20260505 06:26:53.554 | INFO | PerfReport: 7.70%, PulseAccumulating -- 2.55e+00 / 3.12e+00, 4.68e-01 / 3.33e-01 | 20260505 06:26:53.554 | INFO | PerfReport: 4.10%, Metrics -- 1.36e+00 / 1.71e+00, 3.85e-01 / 1.98e-01 | 20260505 06:26:53.554 | INFO | PerfReport: 1.50%, PulseSerializing -- 4.96e-01 / 5.69e-01, 9.81e-02 / 3.61e-02 | 20260505 06:26:53.554 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 06:26:53.554 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:26:53.554 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:26:53.554 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.26e+01 / 0.00e+00 | 20260505 06:26:53.554 | INFO | PerfReport: 21.94%, Upload -- 2.23e+01 / 0.00e+00, 7.78e+00 / 0.00e+00 | 20260505 06:26:53.554 | INFO | PerfReport: 21.49%, Download -- 2.18e+01 / 0.00e+00, 7.79e+00 / 0.00e+00 | 20260505 06:26:53.554 | INFO | PerfReport: 32.68%, Compute -- 3.32e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 06:26:53.554 | INFO | PerfReport: 1.43%, LockUpload -- 1.45e+00 / 0.00e+00, 6.80e+00 / 0.00e+00 | 20260505 06:26:53.554 | INFO | PerfReport: 0.76%, LockDownload -- 7.68e-01 / 0.00e+00, 2.17e+00 / 0.00e+00 | 20260505 06:26:53.554 | INFO | PerfReport: 21.69%, LockCompute -- 2.20e+01 / 0.00e+00, 1.67e+01 / 0.00e+00 | 20260505 06:26:53.554 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.65e-02 / 0.00e+00 | 20260505 06:26:53.674 | INFO | Starting a round of Baz IO | 20260505 06:26:56.928 | INFO | Populating RTMetrics report | 20260505 06:26:56.928 | INFO | RTMetrics report populated | 20260505 06:26:56.928 | INFO | Opening tmp RTMetrics json file | 20260505 06:26:56.931 | INFO | Serializing RTMetrics to json | 20260505 06:26:56.935 | INFO | Serialization done, swapping tmp file now... | 20260505 06:26:56.936 | INFO | RTMetrics done | 20260505 06:26:57.176 | INFO | Finished a round of Baz IO: Wrote 2.371 GiB in 3.502 seconds | 20260505 06:26:57.176 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.33% / 0.05% / 66.62% | 20260505 06:26:57.176 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.12795% / 8.34465e-05% / 4.76837e-05% | 20260505 06:26:57.177 | INFO | Baz IO overall: | 20260505 06:26:57.177 | INFO | BytesWritten : 540646955822 | 20260505 06:26:57.177 | INFO | NumEvents : 255608401103 | 20260505 06:26:58.691 | INFO | Analyzing chunk frames = [1048576,1049088) | 20260505 06:26:58.691 | INFO | MemInfo: { | 20260505 06:26:58.691 | INFO | "gpu" : " 17 GB", | 20260505 06:26:58.691 | INFO | "huge" : "", | 20260505 06:26:58.691 | INFO | "huge_shared" : " 27 GB", | 20260505 06:26:58.691 | INFO | "malloc" : " 14 GB", | 20260505 06:26:58.691 | INFO | "pinned" : " 29 GB" | 20260505 06:26:58.691 | INFO | } | 20260505 06:26:58.691 | INFO | Perf report for the last 5.13926 seconds: | 20260505 06:26:58.691 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:26:58.691 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:58.691 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:58.691 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:58.691 | INFO | ANALYSIS : 64.5%, 2.9, 97.1ms, 0 | 20260505 06:26:58.691 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:26:58.691 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260505 06:26:58.691 | INFO | PRE_HQ : 63.4%, 1.9, 63.5ms, 0 | 20260505 06:26:58.692 | INFO | BAZWRITER : 67.5%, 4.2, 147.2ms, 0 | 20260505 06:26:58.692 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 06:26:58.692 | INFO | Uploaded 36GiB to GPU in 2.10278s (17.1202GiB/s) | 20260505 06:26:58.692 | INFO | Downloaded 38GiB to GPU in 2.30795s (16.4648GiB/s) | 20260505 06:27:03.831 | INFO | Analyzing chunk frames = [1049088,1049600) | 20260505 06:27:03.831 | INFO | MemInfo: { | 20260505 06:27:03.831 | INFO | "gpu" : " 17 GB", | 20260505 06:27:03.831 | INFO | "huge" : "", | 20260505 06:27:03.831 | INFO | "huge_shared" : " 27 GB", | 20260505 06:27:03.831 | INFO | "malloc" : " 14 GB", | 20260505 06:27:03.831 | INFO | "pinned" : " 29 GB" | 20260505 06:27:03.831 | INFO | } | 20260505 06:27:03.831 | INFO | Perf report for the last 5.14007 seconds: | 20260505 06:27:03.831 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:27:03.831 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:03.831 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:03.832 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:03.832 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 06:27:03.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:03.832 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:03.832 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 06:27:03.832 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:03.832 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:03.832 | INFO | Uploaded 36GiB to GPU in 2.11479s (17.023GiB/s) | 20260505 06:27:03.832 | INFO | Downloaded 34GiB to GPU in 2.04502s (16.6257GiB/s) | 20260505 06:27:08.970 | INFO | Analyzing chunk frames = [1049600,1050112) | 20260505 06:27:08.970 | INFO | MemInfo: { | 20260505 06:27:08.970 | INFO | "gpu" : " 17 GB", | 20260505 06:27:08.970 | INFO | "huge" : "", | 20260505 06:27:08.970 | INFO | "huge_shared" : " 27 GB", | 20260505 06:27:08.970 | INFO | "malloc" : " 14 GB", | 20260505 06:27:08.970 | INFO | "pinned" : " 29 GB" | 20260505 06:27:08.970 | INFO | } | 20260505 06:27:08.970 | INFO | Perf report for the last 5.13862 seconds: | 20260505 06:27:08.970 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:27:08.970 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:08.970 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:08.970 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:08.970 | INFO | ANALYSIS : 63.9%, 3.0, 99.4ms, 0 | 20260505 06:27:08.970 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:08.970 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:08.970 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 06:27:08.970 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:08.970 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:08.971 | INFO | Uploaded 36GiB to GPU in 2.1333s (16.8753GiB/s) | 20260505 06:27:08.971 | INFO | Downloaded 34GiB to GPU in 2.02974s (16.7509GiB/s) | 20260505 06:27:14.109 | INFO | Analyzing chunk frames = [1050112,1050624) | 20260505 06:27:14.109 | INFO | MemInfo: { | 20260505 06:27:14.109 | INFO | "gpu" : " 17 GB", | 20260505 06:27:14.110 | INFO | "huge" : "", | 20260505 06:27:14.110 | INFO | "huge_shared" : " 27 GB", | 20260505 06:27:14.110 | INFO | "malloc" : " 14 GB", | 20260505 06:27:14.110 | INFO | "pinned" : " 29 GB" | 20260505 06:27:14.110 | INFO | } | 20260505 06:27:14.110 | INFO | Perf report for the last 5.13962 seconds: | 20260505 06:27:14.110 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:27:14.110 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:14.110 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:14.110 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:14.110 | INFO | ANALYSIS : 63.9%, 3.0, 99.3ms, 0 | 20260505 06:27:14.110 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:14.110 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:14.110 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 06:27:14.110 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:14.110 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:14.110 | INFO | Uploaded 36GiB to GPU in 2.16841s (16.602GiB/s) | 20260505 06:27:14.110 | INFO | Downloaded 34GiB to GPU in 2.04295s (16.6426GiB/s) | 20260505 06:27:19.249 | INFO | Analyzing chunk frames = [1050624,1051136) | 20260505 06:27:19.249 | INFO | MemInfo: { | 20260505 06:27:19.249 | INFO | "gpu" : " 17 GB", | 20260505 06:27:19.249 | INFO | "huge" : "", | 20260505 06:27:19.249 | INFO | "huge_shared" : " 27 GB", | 20260505 06:27:19.249 | INFO | "malloc" : " 14 GB", | 20260505 06:27:19.249 | INFO | "pinned" : " 29 GB" | 20260505 06:27:19.249 | INFO | } | 20260505 06:27:19.249 | INFO | Perf report for the last 5.13952 seconds: | 20260505 06:27:19.249 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:27:19.249 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:19.249 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:19.249 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:19.249 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 06:27:19.249 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:19.249 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:19.249 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 06:27:19.249 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:19.249 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:19.250 | INFO | Uploaded 36GiB to GPU in 2.16304s (16.6432GiB/s) | 20260505 06:27:19.250 | INFO | Downloaded 34GiB to GPU in 2.04814s (16.6004GiB/s) | 20260505 06:27:24.389 | INFO | Analyzing chunk frames = [1051136,1051648) | 20260505 06:27:24.389 | INFO | MemInfo: { | 20260505 06:27:24.389 | INFO | "gpu" : " 17 GB", | 20260505 06:27:24.389 | INFO | "huge" : "", | 20260505 06:27:24.389 | INFO | "huge_shared" : " 27 GB", | 20260505 06:27:24.389 | INFO | "malloc" : " 14 GB", | 20260505 06:27:24.389 | INFO | "pinned" : " 29 GB" | 20260505 06:27:24.389 | INFO | } | 20260505 06:27:24.390 | INFO | Perf report for the last 5.14027 seconds: | 20260505 06:27:24.390 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:27:24.390 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:24.390 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:24.390 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:24.390 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 06:27:24.390 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:24.390 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:24.390 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 06:27:24.390 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:24.390 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:24.390 | INFO | Uploaded 36GiB to GPU in 2.15617s (16.6963GiB/s) | 20260505 06:27:24.390 | INFO | Downloaded 34GiB to GPU in 2.04476s (16.6279GiB/s) | 20260505 06:27:29.528 | INFO | Analyzing chunk frames = [1051648,1052160) | 20260505 06:27:29.528 | INFO | MemInfo: { | 20260505 06:27:29.528 | INFO | "gpu" : " 17 GB", | 20260505 06:27:29.528 | INFO | "huge" : "", | 20260505 06:27:29.528 | INFO | "huge_shared" : " 27 GB", | 20260505 06:27:29.528 | INFO | "malloc" : " 14 GB", | 20260505 06:27:29.528 | INFO | "pinned" : " 29 GB" | 20260505 06:27:29.528 | INFO | } | 20260505 06:27:29.528 | INFO | Perf report for the last 5.13841 seconds: | 20260505 06:27:29.528 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:27:29.528 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:29.528 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:29.528 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:29.528 | INFO | ANALYSIS : 64.0%, 3.0, 99.8ms, 0 | 20260505 06:27:29.528 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:29.528 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:29.528 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 06:27:29.528 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:29.528 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:29.528 | INFO | Uploaded 36GiB to GPU in 2.16061s (16.662GiB/s) | 20260505 06:27:29.528 | INFO | Downloaded 34GiB to GPU in 2.14977s (15.8157GiB/s) | 20260505 06:27:34.667 | INFO | Analyzing chunk frames = [1052160,1052672) | 20260505 06:27:34.667 | INFO | MemInfo: { | 20260505 06:27:34.667 | INFO | "gpu" : " 17 GB", | 20260505 06:27:34.667 | INFO | "huge" : "", | 20260505 06:27:34.667 | INFO | "huge_shared" : " 27 GB", | 20260505 06:27:34.667 | INFO | "malloc" : " 14 GB", | 20260505 06:27:34.667 | INFO | "pinned" : " 29 GB" | 20260505 06:27:34.667 | INFO | } | 20260505 06:27:34.667 | INFO | Perf report for the last 5.13894 seconds: | 20260505 06:27:34.667 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:27:34.667 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:34.667 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:34.667 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:34.667 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 06:27:34.667 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:34.667 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:34.667 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260505 06:27:34.667 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:34.667 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:34.667 | INFO | Uploaded 36GiB to GPU in 2.0946s (17.1871GiB/s) | 20260505 06:27:34.667 | INFO | Downloaded 34GiB to GPU in 2.05295s (16.5615GiB/s) | 20260505 06:27:34.789 | INFO | Starting a round of Baz IO | 20260505 06:27:38.035 | INFO | Populating RTMetrics report | 20260505 06:27:38.036 | INFO | RTMetrics report populated | 20260505 06:27:38.036 | INFO | Opening tmp RTMetrics json file | 20260505 06:27:38.058 | INFO | Serializing RTMetrics to json | 20260505 06:27:38.059 | INFO | Serialization done, swapping tmp file now... | 20260505 06:27:38.060 | INFO | RTMetrics done | 20260505 06:27:38.153 | INFO | Finished a round of Baz IO: Wrote 2.372 GiB in 3.363 seconds | 20260505 06:27:38.153 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.28% / 0.05% / 66.67% | 20260505 06:27:38.153 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.11088% / 5.16574e-05% / 3.97364e-05% | 20260505 06:27:38.153 | INFO | Baz IO overall: | 20260505 06:27:38.153 | INFO | BytesWritten : 543193936216 | 20260505 06:27:38.153 | INFO | NumEvents : 256825819062 | 20260505 06:27:39.806 | INFO | Analyzing chunk frames = [1052672,1053184) | 20260505 06:27:39.806 | INFO | MemInfo: { | 20260505 06:27:39.806 | INFO | "gpu" : " 17 GB", | 20260505 06:27:39.806 | INFO | "huge" : "", | 20260505 06:27:39.806 | INFO | "huge_shared" : " 27 GB", | 20260505 06:27:39.806 | INFO | "malloc" : " 14 GB", | 20260505 06:27:39.806 | INFO | "pinned" : " 29 GB" | 20260505 06:27:39.806 | INFO | } | 20260505 06:27:39.806 | INFO | Perf report for the last 5.13922 seconds: | 20260505 06:27:39.806 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:27:39.806 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:39.806 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:39.806 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:39.806 | INFO | ANALYSIS : 64.4%, 2.9, 98.7ms, 0 | 20260505 06:27:39.806 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:39.806 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260505 06:27:39.806 | INFO | PRE_HQ : 63.4%, 1.9, 64.0ms, 0 | 20260505 06:27:39.806 | INFO | BAZWRITER : 64.5%, 4.1, 136.4ms, 0 | 20260505 06:27:39.806 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 06:27:39.807 | INFO | Uploaded 36GiB to GPU in 2.10078s (17.1365GiB/s) | 20260505 06:27:39.807 | INFO | Downloaded 38GiB to GPU in 2.31324s (16.4272GiB/s) | 20260505 06:27:44.946 | INFO | Analyzing chunk frames = [1053184,1053696) | 20260505 06:27:44.946 | INFO | MemInfo: { | 20260505 06:27:44.946 | INFO | "gpu" : " 17 GB", | 20260505 06:27:44.946 | INFO | "huge" : "", | 20260505 06:27:44.946 | INFO | "huge_shared" : " 27 GB", | 20260505 06:27:44.946 | INFO | "malloc" : " 14 GB", | 20260505 06:27:44.946 | INFO | "pinned" : " 29 GB" | 20260505 06:27:44.946 | INFO | } | 20260505 06:27:44.946 | INFO | Perf report for the last 5.13986 seconds: | 20260505 06:27:44.946 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:27:44.946 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:44.946 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:44.946 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:44.946 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 06:27:44.946 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:44.946 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:44.946 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 06:27:44.946 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:44.946 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:44.946 | INFO | Uploaded 36GiB to GPU in 2.11414s (17.0282GiB/s) | 20260505 06:27:44.946 | INFO | Downloaded 34GiB to GPU in 2.03688s (16.6922GiB/s) | 20260505 06:27:50.085 | INFO | Analyzing chunk frames = [1053696,1054208) | 20260505 06:27:50.085 | INFO | MemInfo: { | 20260505 06:27:50.085 | INFO | "gpu" : " 17 GB", | 20260505 06:27:50.085 | INFO | "huge" : "", | 20260505 06:27:50.085 | INFO | "huge_shared" : " 27 GB", | 20260505 06:27:50.085 | INFO | "malloc" : " 14 GB", | 20260505 06:27:50.085 | INFO | "pinned" : " 29 GB" | 20260505 06:27:50.085 | INFO | } | 20260505 06:27:50.085 | INFO | Perf report for the last 5.13877 seconds: | 20260505 06:27:50.085 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:27:50.085 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:50.085 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:50.085 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:50.085 | INFO | ANALYSIS : 63.8%, 3.0, 99.3ms, 0 | 20260505 06:27:50.085 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:50.085 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:50.085 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 06:27:50.085 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:50.085 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:50.085 | INFO | Uploaded 36GiB to GPU in 2.13369s (16.8722GiB/s) | 20260505 06:27:50.085 | INFO | Downloaded 34GiB to GPU in 2.03426s (16.7137GiB/s) | 20260505 06:27:55.224 | INFO | Analyzing chunk frames = [1054208,1054720) | 20260505 06:27:55.224 | INFO | MemInfo: { | 20260505 06:27:55.224 | INFO | "gpu" : " 17 GB", | 20260505 06:27:55.224 | INFO | "huge" : "", | 20260505 06:27:55.224 | INFO | "huge_shared" : " 27 GB", | 20260505 06:27:55.224 | INFO | "malloc" : " 14 GB", | 20260505 06:27:55.224 | INFO | "pinned" : " 29 GB" | 20260505 06:27:55.224 | INFO | } | 20260505 06:27:55.224 | INFO | Perf report for the last 5.13949 seconds: | 20260505 06:27:55.224 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:27:55.224 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:55.224 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:55.224 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:55.224 | INFO | ANALYSIS : 63.8%, 3.0, 99.2ms, 0 | 20260505 06:27:55.224 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:55.224 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:55.224 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 06:27:55.224 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:55.224 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:27:55.225 | INFO | Uploaded 36GiB to GPU in 2.16029s (16.6644GiB/s) | 20260505 06:27:55.225 | INFO | Downloaded 34GiB to GPU in 2.03021s (16.747GiB/s) | 20260505 06:28:00.363 | INFO | Analyzing chunk frames = [1054720,1055232) | 20260505 06:28:00.363 | INFO | MemInfo: { | 20260505 06:28:00.363 | INFO | "gpu" : " 17 GB", | 20260505 06:28:00.363 | INFO | "huge" : "", | 20260505 06:28:00.363 | INFO | "huge_shared" : " 27 GB", | 20260505 06:28:00.363 | INFO | "malloc" : " 14 GB", | 20260505 06:28:00.363 | INFO | "pinned" : " 29 GB" | 20260505 06:28:00.363 | INFO | } | 20260505 06:28:00.363 | INFO | Perf report for the last 5.13908 seconds: | 20260505 06:28:00.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:28:00.363 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:00.363 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:00.363 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:00.363 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 06:28:00.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:00.364 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:00.364 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 06:28:00.364 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:00.364 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:00.364 | INFO | Uploaded 36GiB to GPU in 2.16742s (16.6096GiB/s) | 20260505 06:28:00.364 | INFO | Downloaded 34GiB to GPU in 2.05207s (16.5686GiB/s) | 20260505 06:28:05.504 | INFO | Analyzing chunk frames = [1055232,1055744) | 20260505 06:28:05.504 | INFO | MemInfo: { | 20260505 06:28:05.504 | INFO | "gpu" : " 17 GB", | 20260505 06:28:05.504 | INFO | "huge" : "", | 20260505 06:28:05.504 | INFO | "huge_shared" : " 27 GB", | 20260505 06:28:05.504 | INFO | "malloc" : " 14 GB", | 20260505 06:28:05.504 | INFO | "pinned" : " 29 GB" | 20260505 06:28:05.505 | INFO | } | 20260505 06:28:05.505 | INFO | Perf report for the last 5.14114 seconds: | 20260505 06:28:05.505 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:28:05.505 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:05.505 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:05.505 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:05.505 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 06:28:05.505 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:05.505 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:05.505 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 06:28:05.505 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:05.505 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:05.505 | INFO | Uploaded 36GiB to GPU in 2.15019s (16.7427GiB/s) | 20260505 06:28:05.505 | INFO | Downloaded 34GiB to GPU in 2.04515s (16.6247GiB/s) | 20260505 06:28:10.642 | INFO | Analyzing chunk frames = [1055744,1056256) | 20260505 06:28:10.642 | INFO | MemInfo: { | 20260505 06:28:10.642 | INFO | "gpu" : " 17 GB", | 20260505 06:28:10.642 | INFO | "huge" : "", | 20260505 06:28:10.642 | INFO | "huge_shared" : " 27 GB", | 20260505 06:28:10.642 | INFO | "malloc" : " 14 GB", | 20260505 06:28:10.642 | INFO | "pinned" : " 29 GB" | 20260505 06:28:10.642 | INFO | } | 20260505 06:28:10.642 | INFO | Perf report for the last 5.13783 seconds: | 20260505 06:28:10.642 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:28:10.642 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:10.642 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:10.642 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:10.642 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 06:28:10.642 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:10.642 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:10.643 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 06:28:10.643 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:10.643 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:10.643 | INFO | Uploaded 36GiB to GPU in 2.17993s (16.5143GiB/s) | 20260505 06:28:10.643 | INFO | Downloaded 34GiB to GPU in 2.15678s (15.7642GiB/s) | 20260505 06:28:15.782 | INFO | Analyzing chunk frames = [1056256,1056768) | 20260505 06:28:15.782 | INFO | MemInfo: { | 20260505 06:28:15.782 | INFO | "gpu" : " 17 GB", | 20260505 06:28:15.782 | INFO | "huge" : "", | 20260505 06:28:15.782 | INFO | "huge_shared" : " 27 GB", | 20260505 06:28:15.782 | INFO | "malloc" : " 14 GB", | 20260505 06:28:15.782 | INFO | "pinned" : " 29 GB" | 20260505 06:28:15.782 | INFO | } | 20260505 06:28:15.782 | INFO | Perf report for the last 5.13941 seconds: | 20260505 06:28:15.782 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:28:15.782 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:15.782 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:15.782 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:15.782 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 06:28:15.782 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:15.782 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:15.782 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 06:28:15.782 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:15.782 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:15.782 | INFO | Uploaded 36GiB to GPU in 2.10017s (17.1415GiB/s) | 20260505 06:28:15.782 | INFO | Downloaded 34GiB to GPU in 2.06197s (16.4891GiB/s) | 20260505 06:28:15.782 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 06:28:15.782 | INFO | Total mem usage: 15.3 GiB | 20260505 06:28:15.782 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 06:28:15.782 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -1.94%) | 20260505 06:28:15.782 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 06:28:15.782 | INFO | High Water Breakdown: | 20260505 06:28:15.782 | INFO | 61% : 9.17 GiB ( 98.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 06:28:15.782 | INFO | 38.8% : 5.84 GiB ( 87.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 06:28:15.782 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 06:28:15.782 | INFO | ------------------------------------------ | 20260505 06:28:15.782 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:28:15.782 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 06:28:15.782 | INFO | Total mem usage: 26.5 GiB | 20260505 06:28:15.782 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 06:28:15.782 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 06:28:15.782 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 06:28:15.783 | INFO | High Water Breakdown: | 20260505 06:28:15.783 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:28:15.783 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:28:15.783 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:28:15.783 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:28:15.783 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:28:15.783 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:28:15.783 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 06:28:15.783 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 06:28:15.783 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:28:15.783 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 06:28:15.783 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 06:28:15.783 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 06:28:15.783 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 06:28:15.783 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 06:28:15.783 | INFO | ------------------------------------------ | 20260505 06:28:15.783 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:28:15.783 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 06:28:15.783 | INFO | Total mem usage: 25.6 GiB | 20260505 06:28:15.783 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 06:28:15.783 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 06:28:15.783 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 06:28:15.783 | INFO | High Water Breakdown: | 20260505 06:28:15.783 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 06:28:15.783 | INFO | ------------------------------------------ | 20260505 06:28:15.783 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:28:15.783 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 06:28:15.783 | INFO | Total mem usage: 15.7 GiB | 20260505 06:28:15.783 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 06:28:15.783 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 06:28:15.783 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260505 06:28:15.783 | INFO | High Water Breakdown: | 20260505 06:28:15.783 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 06:28:15.783 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 06:28:15.783 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 06:28:15.783 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 06:28:15.783 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 06:28:15.783 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 06:28:15.783 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 06:28:15.783 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 06:28:15.783 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 06:28:15.783 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:28:15.783 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 06:28:15.783 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:28:15.783 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 06:28:15.783 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 06:28:15.783 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 06:28:15.783 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 06:28:15.783 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:28:15.783 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 06:28:15.783 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:28:15.783 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:28:15.783 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:28:15.783 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 06:28:15.783 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 06:28:15.783 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:28:15.783 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 06:28:15.783 | INFO | ------------------------------------------ | 20260505 06:28:15.783 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:28:15.783 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 06:28:15.783 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:28:15.783 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:28:15.783 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.56e+01, 1.36e+01 / 1.61e+00 | 20260505 06:28:15.783 | INFO | PerfReport: 0.28%, AggregateStats -- 9.39e-02 / 1.02e-01, 1.17e-02 / 6.52e-03 | 20260505 06:28:15.783 | INFO | PerfReport: 11.59%, Baseline -- 3.85e+00 / 4.49e+00, 6.39e-01 / 3.26e-01 | 20260505 06:28:15.783 | INFO | PerfReport: 3.32%, Histogram -- 1.10e+00 / 1.49e+00, 2.18e-01 / 1.21e-01 | 20260505 06:28:15.783 | INFO | PerfReport: 15.45%, DME -- 4.10e+01 / 4.50e+01, 4.32e+00 / 1.08e+00 | 20260505 06:28:15.783 | INFO | PerfReport: 56.07%, FrameLabeling -- 1.86e+01 / 1.91e+01, 2.07e+00 / 1.05e+00 | 20260505 06:28:15.783 | INFO | PerfReport: 7.69%, PulseAccumulating -- 2.55e+00 / 3.15e+00, 4.62e-01 / 3.21e-01 | 20260505 06:28:15.783 | INFO | PerfReport: 4.10%, Metrics -- 1.36e+00 / 1.70e+00, 3.85e-01 / 2.10e-01 | 20260505 06:28:15.783 | INFO | PerfReport: 1.49%, PulseSerializing -- 4.95e-01 / 5.67e-01, 9.78e-02 / 4.27e-02 | 20260505 06:28:15.783 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 06:28:15.783 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:28:15.783 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:28:15.783 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.23e+01 / 0.00e+00 | 20260505 06:28:15.783 | INFO | PerfReport: 21.94%, Upload -- 2.23e+01 / 0.00e+00, 7.70e+00 / 0.00e+00 | 20260505 06:28:15.783 | INFO | PerfReport: 21.46%, Download -- 2.18e+01 / 0.00e+00, 7.72e+00 / 0.00e+00 | 20260505 06:28:15.783 | INFO | PerfReport: 32.69%, Compute -- 3.32e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 06:28:15.783 | INFO | PerfReport: 1.43%, LockUpload -- 1.45e+00 / 0.00e+00, 6.79e+00 / 0.00e+00 | 20260505 06:28:15.783 | INFO | PerfReport: 0.74%, LockDownload -- 7.49e-01 / 0.00e+00, 2.15e+00 / 0.00e+00 | 20260505 06:28:15.783 | INFO | PerfReport: 21.73%, LockCompute -- 2.21e+01 / 0.00e+00, 1.68e+01 / 0.00e+00 | 20260505 06:28:15.783 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 2.07e-02 / 0.00e+00 | 20260505 06:28:15.907 | INFO | Starting a round of Baz IO | 20260505 06:28:19.170 | INFO | Populating RTMetrics report | 20260505 06:28:19.170 | INFO | RTMetrics report populated | 20260505 06:28:19.170 | INFO | Opening tmp RTMetrics json file | 20260505 06:28:19.190 | INFO | Serializing RTMetrics to json | 20260505 06:28:19.193 | INFO | Serialization done, swapping tmp file now... | 20260505 06:28:19.195 | INFO | RTMetrics done | 20260505 06:28:19.298 | INFO | Finished a round of Baz IO: Wrote 2.375 GiB in 3.391 seconds | 20260505 06:28:19.298 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.23% / 0.05% / 66.72% | 20260505 06:28:19.298 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.0962% / 6.75519e-05% / 5.5631e-05% | 20260505 06:28:19.298 | INFO | Baz IO overall: | 20260505 06:28:19.298 | INFO | BytesWritten : 545743730271 | 20260505 06:28:19.298 | INFO | NumEvents : 258044874985 | 20260505 06:28:20.929 | INFO | Analyzing chunk frames = [1056768,1057280) | 20260505 06:28:20.929 | INFO | MemInfo: { | 20260505 06:28:20.929 | INFO | "gpu" : " 17 GB", | 20260505 06:28:20.929 | INFO | "huge" : "", | 20260505 06:28:20.929 | INFO | "huge_shared" : " 27 GB", | 20260505 06:28:20.929 | INFO | "malloc" : " 14 GB", | 20260505 06:28:20.929 | INFO | "pinned" : " 29 GB" | 20260505 06:28:20.929 | INFO | } | 20260505 06:28:20.929 | INFO | Perf report for the last 5.14721 seconds: | 20260505 06:28:20.929 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:28:20.929 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:20.929 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:20.929 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:20.929 | INFO | ANALYSIS : 64.7%, 2.9, 99.3ms, 0 | 20260505 06:28:20.929 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:20.929 | INFO | RT_METRICS : 46.6%, 1.0, 24.5ms, 0 | 20260505 06:28:20.929 | INFO | PRE_HQ : 63.7%, 1.9, 64.2ms, 0 | 20260505 06:28:20.929 | INFO | BAZWRITER : 65.3%, 4.1, 136.9ms, 0 | 20260505 06:28:20.929 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 06:28:20.929 | INFO | Uploaded 36GiB to GPU in 2.14547s (16.7795GiB/s) | 20260505 06:28:20.929 | INFO | Downloaded 38GiB to GPU in 2.29886s (16.5299GiB/s) | 20260505 06:28:26.061 | INFO | Analyzing chunk frames = [1057280,1057792) | 20260505 06:28:26.061 | INFO | MemInfo: { | 20260505 06:28:26.061 | INFO | "gpu" : " 17 GB", | 20260505 06:28:26.061 | INFO | "huge" : "", | 20260505 06:28:26.061 | INFO | "huge_shared" : " 27 GB", | 20260505 06:28:26.061 | INFO | "malloc" : " 14 GB", | 20260505 06:28:26.061 | INFO | "pinned" : " 29 GB" | 20260505 06:28:26.061 | INFO | } | 20260505 06:28:26.061 | INFO | Perf report for the last 5.13239 seconds: | 20260505 06:28:26.061 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:28:26.061 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:26.061 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:26.062 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:26.062 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 06:28:26.062 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:26.062 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:26.062 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 06:28:26.062 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:26.062 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:26.062 | INFO | Uploaded 36GiB to GPU in 2.09296s (17.2005GiB/s) | 20260505 06:28:26.062 | INFO | Downloaded 34GiB to GPU in 2.03737s (16.6882GiB/s) | 20260505 06:28:31.199 | INFO | Analyzing chunk frames = [1057792,1058304) | 20260505 06:28:31.199 | INFO | MemInfo: { | 20260505 06:28:31.200 | INFO | "gpu" : " 17 GB", | 20260505 06:28:31.200 | INFO | "huge" : "", | 20260505 06:28:31.200 | INFO | "huge_shared" : " 27 GB", | 20260505 06:28:31.200 | INFO | "malloc" : " 14 GB", | 20260505 06:28:31.200 | INFO | "pinned" : " 29 GB" | 20260505 06:28:31.200 | INFO | } | 20260505 06:28:31.200 | INFO | Perf report for the last 5.13831 seconds: | 20260505 06:28:31.200 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:28:31.200 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:31.200 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:31.200 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:31.200 | INFO | ANALYSIS : 64.1%, 3.0, 99.8ms, 0 | 20260505 06:28:31.200 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:31.200 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:31.200 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 06:28:31.200 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:31.200 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:31.200 | INFO | Uploaded 36GiB to GPU in 2.17123s (16.5805GiB/s) | 20260505 06:28:31.200 | INFO | Downloaded 34GiB to GPU in 2.04322s (16.6404GiB/s) | 20260505 06:28:36.339 | INFO | Analyzing chunk frames = [1058304,1058816) | 20260505 06:28:36.339 | INFO | MemInfo: { | 20260505 06:28:36.339 | INFO | "gpu" : " 17 GB", | 20260505 06:28:36.339 | INFO | "huge" : "", | 20260505 06:28:36.339 | INFO | "huge_shared" : " 27 GB", | 20260505 06:28:36.339 | INFO | "malloc" : " 14 GB", | 20260505 06:28:36.339 | INFO | "pinned" : " 29 GB" | 20260505 06:28:36.339 | INFO | } | 20260505 06:28:36.339 | INFO | Perf report for the last 5.13925 seconds: | 20260505 06:28:36.339 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:28:36.339 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:36.339 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:36.339 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:36.339 | INFO | ANALYSIS : 63.9%, 3.0, 99.2ms, 0 | 20260505 06:28:36.339 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:36.339 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:36.339 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 06:28:36.339 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:36.339 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:36.339 | INFO | Uploaded 36GiB to GPU in 2.16695s (16.6132GiB/s) | 20260505 06:28:36.339 | INFO | Downloaded 34GiB to GPU in 2.03567s (16.7021GiB/s) | 20260505 06:28:41.478 | INFO | Analyzing chunk frames = [1058816,1059328) | 20260505 06:28:41.478 | INFO | MemInfo: { | 20260505 06:28:41.478 | INFO | "gpu" : " 17 GB", | 20260505 06:28:41.478 | INFO | "huge" : "", | 20260505 06:28:41.478 | INFO | "huge_shared" : " 27 GB", | 20260505 06:28:41.478 | INFO | "malloc" : " 14 GB", | 20260505 06:28:41.478 | INFO | "pinned" : " 29 GB" | 20260505 06:28:41.478 | INFO | } | 20260505 06:28:41.478 | INFO | Perf report for the last 5.13925 seconds: | 20260505 06:28:41.478 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:28:41.478 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:41.478 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:41.478 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:41.478 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 06:28:41.478 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:41.478 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:41.478 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 06:28:41.478 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:41.478 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:41.479 | INFO | Uploaded 36GiB to GPU in 2.17902s (16.5212GiB/s) | 20260505 06:28:41.479 | INFO | Downloaded 34GiB to GPU in 2.04106s (16.658GiB/s) | 20260505 06:28:46.619 | INFO | Analyzing chunk frames = [1059328,1059840) | 20260505 06:28:46.619 | INFO | MemInfo: { | 20260505 06:28:46.619 | INFO | "gpu" : " 17 GB", | 20260505 06:28:46.619 | INFO | "huge" : "", | 20260505 06:28:46.619 | INFO | "huge_shared" : " 27 GB", | 20260505 06:28:46.619 | INFO | "malloc" : " 14 GB", | 20260505 06:28:46.619 | INFO | "pinned" : " 29 GB" | 20260505 06:28:46.619 | INFO | } | 20260505 06:28:46.619 | INFO | Perf report for the last 5.14075 seconds: | 20260505 06:28:46.619 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:28:46.619 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:46.619 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:46.619 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:46.619 | INFO | ANALYSIS : 63.9%, 3.0, 99.9ms, 0 | 20260505 06:28:46.619 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:46.619 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:46.619 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 06:28:46.619 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:46.619 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:46.619 | INFO | Uploaded 36GiB to GPU in 2.16227s (16.6492GiB/s) | 20260505 06:28:46.619 | INFO | Downloaded 34GiB to GPU in 2.05296s (16.5614GiB/s) | 20260505 06:28:51.757 | INFO | Analyzing chunk frames = [1059840,1060352) | 20260505 06:28:51.757 | INFO | MemInfo: { | 20260505 06:28:51.757 | INFO | "gpu" : " 17 GB", | 20260505 06:28:51.757 | INFO | "huge" : "", | 20260505 06:28:51.757 | INFO | "huge_shared" : " 27 GB", | 20260505 06:28:51.757 | INFO | "malloc" : " 14 GB", | 20260505 06:28:51.757 | INFO | "pinned" : " 29 GB" | 20260505 06:28:51.757 | INFO | } | 20260505 06:28:51.757 | INFO | Perf report for the last 5.13829 seconds: | 20260505 06:28:51.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:28:51.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:51.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:51.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:51.757 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 06:28:51.757 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:51.758 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:51.758 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260505 06:28:51.758 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:51.758 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:51.758 | INFO | Uploaded 36GiB to GPU in 2.18021s (16.5122GiB/s) | 20260505 06:28:51.758 | INFO | Downloaded 34GiB to GPU in 2.15544s (15.774GiB/s) | 20260505 06:28:56.896 | INFO | Analyzing chunk frames = [1060352,1060864) | 20260505 06:28:56.896 | INFO | MemInfo: { | 20260505 06:28:56.896 | INFO | "gpu" : " 17 GB", | 20260505 06:28:56.896 | INFO | "huge" : "", | 20260505 06:28:56.896 | INFO | "huge_shared" : " 27 GB", | 20260505 06:28:56.896 | INFO | "malloc" : " 14 GB", | 20260505 06:28:56.897 | INFO | "pinned" : " 29 GB" | 20260505 06:28:56.897 | INFO | } | 20260505 06:28:56.897 | INFO | Perf report for the last 5.13938 seconds: | 20260505 06:28:56.897 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:28:56.897 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:56.897 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:56.897 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:56.897 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 06:28:56.897 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:56.897 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:56.897 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 06:28:56.897 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:56.897 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:28:56.897 | INFO | Uploaded 36GiB to GPU in 2.11895s (16.9896GiB/s) | 20260505 06:28:56.897 | INFO | Downloaded 34GiB to GPU in 2.06971s (16.4275GiB/s) | 20260505 06:28:57.019 | INFO | Starting a round of Baz IO | 20260505 06:29:00.276 | INFO | Populating RTMetrics report | 20260505 06:29:00.277 | INFO | RTMetrics report populated | 20260505 06:29:00.277 | INFO | Opening tmp RTMetrics json file | 20260505 06:29:00.296 | INFO | Serializing RTMetrics to json | 20260505 06:29:00.299 | INFO | Serialization done, swapping tmp file now... | 20260505 06:29:00.302 | INFO | RTMetrics done | 20260505 06:29:00.397 | INFO | Finished a round of Baz IO: Wrote 2.375 GiB in 3.378 seconds | 20260505 06:29:00.397 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.18% / 0.05% / 66.77% | 20260505 06:29:00.397 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.07676% / 9.53674e-05% / 6.75519e-05% | 20260505 06:29:00.397 | INFO | Baz IO overall: | 20260505 06:29:00.397 | INFO | BytesWritten : 548294171081 | 20260505 06:29:00.397 | INFO | NumEvents : 259264684875 | 20260505 06:29:02.036 | INFO | Analyzing chunk frames = [1060864,1061376) | 20260505 06:29:02.036 | INFO | MemInfo: { | 20260505 06:29:02.036 | INFO | "gpu" : " 17 GB", | 20260505 06:29:02.036 | INFO | "huge" : "", | 20260505 06:29:02.036 | INFO | "huge_shared" : " 27 GB", | 20260505 06:29:02.036 | INFO | "malloc" : " 14 GB", | 20260505 06:29:02.036 | INFO | "pinned" : " 29 GB" | 20260505 06:29:02.036 | INFO | } | 20260505 06:29:02.036 | INFO | Perf report for the last 5.13908 seconds: | 20260505 06:29:02.036 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:29:02.036 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:02.036 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:02.036 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:02.036 | INFO | ANALYSIS : 64.6%, 2.9, 98.5ms, 0 | 20260505 06:29:02.036 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:02.036 | INFO | RT_METRICS : 46.6%, 1.0, 24.4ms, 0 | 20260505 06:29:02.036 | INFO | PRE_HQ : 63.7%, 1.9, 64.0ms, 0 | 20260505 06:29:02.036 | INFO | BAZWRITER : 65.0%, 4.1, 136.9ms, 0 | 20260505 06:29:02.036 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 06:29:02.036 | INFO | Uploaded 36GiB to GPU in 2.11103s (17.0533GiB/s) | 20260505 06:29:02.036 | INFO | Downloaded 38GiB to GPU in 2.30629s (16.4767GiB/s) | 20260505 06:29:07.175 | INFO | Analyzing chunk frames = [1061376,1061888) | 20260505 06:29:07.175 | INFO | MemInfo: { | 20260505 06:29:07.175 | INFO | "gpu" : " 17 GB", | 20260505 06:29:07.175 | INFO | "huge" : "", | 20260505 06:29:07.175 | INFO | "huge_shared" : " 27 GB", | 20260505 06:29:07.175 | INFO | "malloc" : " 14 GB", | 20260505 06:29:07.175 | INFO | "pinned" : " 29 GB" | 20260505 06:29:07.175 | INFO | } | 20260505 06:29:07.175 | INFO | Perf report for the last 5.13928 seconds: | 20260505 06:29:07.175 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:29:07.175 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:07.175 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:07.175 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:07.175 | INFO | ANALYSIS : 64.0%, 3.0, 98.9ms, 0 | 20260505 06:29:07.175 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:07.175 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:07.175 | INFO | PRE_HQ : 30.6%, 1.0, 15.7ms, 0 | 20260505 06:29:07.175 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:07.175 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:07.175 | INFO | Uploaded 36GiB to GPU in 2.11047s (17.0578GiB/s) | 20260505 06:29:07.175 | INFO | Downloaded 34GiB to GPU in 2.03965s (16.6695GiB/s) | 20260505 06:29:12.314 | INFO | Analyzing chunk frames = [1061888,1062400) | 20260505 06:29:12.314 | INFO | MemInfo: { | 20260505 06:29:12.314 | INFO | "gpu" : " 17 GB", | 20260505 06:29:12.314 | INFO | "huge" : "", | 20260505 06:29:12.314 | INFO | "huge_shared" : " 27 GB", | 20260505 06:29:12.314 | INFO | "malloc" : " 14 GB", | 20260505 06:29:12.314 | INFO | "pinned" : " 29 GB" | 20260505 06:29:12.314 | INFO | } | 20260505 06:29:12.314 | INFO | Perf report for the last 5.13914 seconds: | 20260505 06:29:12.314 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:29:12.314 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:12.314 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:12.314 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:12.314 | INFO | ANALYSIS : 63.9%, 3.0, 99.4ms, 0 | 20260505 06:29:12.314 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:12.314 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:12.314 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 06:29:12.314 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:12.314 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:12.315 | INFO | Uploaded 36GiB to GPU in 2.13924s (16.8284GiB/s) | 20260505 06:29:12.315 | INFO | Downloaded 34GiB to GPU in 2.03939s (16.6717GiB/s) | 20260505 06:29:17.454 | INFO | Analyzing chunk frames = [1062400,1062912) | 20260505 06:29:17.454 | INFO | MemInfo: { | 20260505 06:29:17.454 | INFO | "gpu" : " 17 GB", | 20260505 06:29:17.454 | INFO | "huge" : "", | 20260505 06:29:17.454 | INFO | "huge_shared" : " 27 GB", | 20260505 06:29:17.454 | INFO | "malloc" : " 14 GB", | 20260505 06:29:17.454 | INFO | "pinned" : " 29 GB" | 20260505 06:29:17.454 | INFO | } | 20260505 06:29:17.454 | INFO | Perf report for the last 5.13958 seconds: | 20260505 06:29:17.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:29:17.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:17.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:17.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:17.454 | INFO | ANALYSIS : 63.9%, 3.0, 99.4ms, 0 | 20260505 06:29:17.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:17.455 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:17.455 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 06:29:17.455 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:17.455 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:17.455 | INFO | Uploaded 36GiB to GPU in 2.16633s (16.6179GiB/s) | 20260505 06:29:17.455 | INFO | Downloaded 34GiB to GPU in 2.03871s (16.6772GiB/s) | 20260505 06:29:22.593 | INFO | Analyzing chunk frames = [1062912,1063424) | 20260505 06:29:22.593 | INFO | MemInfo: { | 20260505 06:29:22.593 | INFO | "gpu" : " 17 GB", | 20260505 06:29:22.593 | INFO | "huge" : "", | 20260505 06:29:22.593 | INFO | "huge_shared" : " 27 GB", | 20260505 06:29:22.593 | INFO | "malloc" : " 14 GB", | 20260505 06:29:22.593 | INFO | "pinned" : " 29 GB" | 20260505 06:29:22.593 | INFO | } | 20260505 06:29:22.593 | INFO | Perf report for the last 5.13926 seconds: | 20260505 06:29:22.593 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:29:22.593 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:22.593 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:22.593 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:22.593 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 06:29:22.593 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:22.593 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:22.593 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 06:29:22.593 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:22.593 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:22.593 | INFO | Uploaded 36GiB to GPU in 2.16237s (16.6484GiB/s) | 20260505 06:29:22.594 | INFO | Downloaded 34GiB to GPU in 2.04574s (16.6199GiB/s) | 20260505 06:29:27.733 | INFO | Analyzing chunk frames = [1063424,1063936) | 20260505 06:29:27.733 | INFO | MemInfo: { | 20260505 06:29:27.733 | INFO | "gpu" : " 17 GB", | 20260505 06:29:27.733 | INFO | "huge" : "", | 20260505 06:29:27.733 | INFO | "huge_shared" : " 27 GB", | 20260505 06:29:27.734 | INFO | "malloc" : " 14 GB", | 20260505 06:29:27.734 | INFO | "pinned" : " 29 GB" | 20260505 06:29:27.734 | INFO | } | 20260505 06:29:27.734 | INFO | Perf report for the last 5.14054 seconds: | 20260505 06:29:27.734 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:29:27.734 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:27.734 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:27.734 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:27.734 | INFO | ANALYSIS : 63.9%, 3.0, 99.9ms, 0 | 20260505 06:29:27.734 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:27.734 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:27.734 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 06:29:27.734 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:27.734 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:27.734 | INFO | Uploaded 36GiB to GPU in 2.1529s (16.7217GiB/s) | 20260505 06:29:27.734 | INFO | Downloaded 34GiB to GPU in 2.04667s (16.6124GiB/s) | 20260505 06:29:32.872 | INFO | Analyzing chunk frames = [1063936,1064448) | 20260505 06:29:32.872 | INFO | MemInfo: { | 20260505 06:29:32.872 | INFO | "gpu" : " 17 GB", | 20260505 06:29:32.872 | INFO | "huge" : "", | 20260505 06:29:32.872 | INFO | "huge_shared" : " 27 GB", | 20260505 06:29:32.872 | INFO | "malloc" : " 14 GB", | 20260505 06:29:32.872 | INFO | "pinned" : " 29 GB" | 20260505 06:29:32.872 | INFO | } | 20260505 06:29:32.872 | INFO | Perf report for the last 5.13821 seconds: | 20260505 06:29:32.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:29:32.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:32.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:32.872 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:32.872 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 06:29:32.872 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:32.872 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:32.872 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 06:29:32.872 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:32.872 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:32.872 | INFO | Uploaded 36GiB to GPU in 2.17576s (16.546GiB/s) | 20260505 06:29:32.872 | INFO | Downloaded 34GiB to GPU in 2.16s (15.7407GiB/s) | 20260505 06:29:38.011 | INFO | Analyzing chunk frames = [1064448,1064960) | 20260505 06:29:38.011 | INFO | MemInfo: { | 20260505 06:29:38.011 | INFO | "gpu" : " 17 GB", | 20260505 06:29:38.011 | INFO | "huge" : "", | 20260505 06:29:38.011 | INFO | "huge_shared" : " 27 GB", | 20260505 06:29:38.011 | INFO | "malloc" : " 14 GB", | 20260505 06:29:38.011 | INFO | "pinned" : " 29 GB" | 20260505 06:29:38.011 | INFO | } | 20260505 06:29:38.011 | INFO | Perf report for the last 5.13921 seconds: | 20260505 06:29:38.011 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:29:38.011 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:38.011 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:38.011 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:38.011 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 06:29:38.011 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:38.011 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:38.011 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260505 06:29:38.011 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:38.011 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:38.011 | INFO | Uploaded 36GiB to GPU in 2.12912s (16.9084GiB/s) | 20260505 06:29:38.011 | INFO | Downloaded 34GiB to GPU in 2.07244s (16.4058GiB/s) | 20260505 06:29:38.011 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 06:29:38.011 | INFO | Total mem usage: 15.3 GiB | 20260505 06:29:38.011 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 06:29:38.012 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.53%) | 20260505 06:29:38.012 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 06:29:38.012 | INFO | High Water Breakdown: | 20260505 06:29:38.012 | INFO | 60.2% : 8.67 GiB ( 92.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 06:29:38.012 | INFO | 39.6% : 5.7 GiB ( 85.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 06:29:38.012 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 06:29:38.012 | INFO | ------------------------------------------ | 20260505 06:29:38.012 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:29:38.012 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 06:29:38.012 | INFO | Total mem usage: 26.5 GiB | 20260505 06:29:38.012 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 06:29:38.012 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 06:29:38.012 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 06:29:38.012 | INFO | High Water Breakdown: | 20260505 06:29:38.012 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:29:38.012 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:29:38.012 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:29:38.012 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:29:38.012 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:29:38.012 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:29:38.012 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 06:29:38.012 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 06:29:38.012 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:29:38.012 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 06:29:38.012 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 06:29:38.012 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 06:29:38.012 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 06:29:38.012 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 06:29:38.012 | INFO | ------------------------------------------ | 20260505 06:29:38.012 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:29:38.012 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 06:29:38.012 | INFO | Total mem usage: 25.6 GiB | 20260505 06:29:38.012 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 06:29:38.012 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 06:29:38.012 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 06:29:38.012 | INFO | High Water Breakdown: | 20260505 06:29:38.012 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 06:29:38.012 | INFO | ------------------------------------------ | 20260505 06:29:38.012 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:29:38.012 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 06:29:38.012 | INFO | Total mem usage: 15.7 GiB | 20260505 06:29:38.012 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 06:29:38.012 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 06:29:38.012 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260505 06:29:38.012 | INFO | High Water Breakdown: | 20260505 06:29:38.012 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 06:29:38.012 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 06:29:38.012 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 06:29:38.012 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 06:29:38.012 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 06:29:38.012 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 06:29:38.012 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 06:29:38.012 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 06:29:38.012 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 06:29:38.012 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:29:38.012 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 06:29:38.012 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:29:38.012 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 06:29:38.012 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 06:29:38.012 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 06:29:38.012 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 06:29:38.012 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:29:38.012 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 06:29:38.012 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:29:38.012 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:29:38.012 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:29:38.012 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 06:29:38.012 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 06:29:38.012 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:29:38.012 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 06:29:38.012 | INFO | ------------------------------------------ | 20260505 06:29:38.012 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:29:38.012 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 06:29:38.012 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:29:38.012 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:29:38.012 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.55e+01, 1.37e+01 / 1.64e+00 | 20260505 06:29:38.012 | INFO | PerfReport: 0.28%, AggregateStats -- 9.40e-02 / 1.04e-01, 1.15e-02 / 4.61e-03 | 20260505 06:29:38.012 | INFO | PerfReport: 11.57%, Baseline -- 3.84e+00 / 4.46e+00, 6.36e-01 / 3.27e-01 | 20260505 06:29:38.012 | INFO | PerfReport: 3.31%, Histogram -- 1.10e+00 / 1.50e+00, 2.19e-01 / 1.07e-01 | 20260505 06:29:38.012 | INFO | PerfReport: 15.46%, DME -- 4.10e+01 / 4.50e+01, 4.37e+00 / 1.08e+00 | 20260505 06:29:38.012 | INFO | PerfReport: 56.10%, FrameLabeling -- 1.86e+01 / 1.90e+01, 2.10e+00 / 1.10e+00 | 20260505 06:29:38.012 | INFO | PerfReport: 7.68%, PulseAccumulating -- 2.55e+00 / 3.15e+00, 4.62e-01 / 3.36e-01 | 20260505 06:29:38.012 | INFO | PerfReport: 4.10%, Metrics -- 1.36e+00 / 1.70e+00, 3.90e-01 / 2.18e-01 | 20260505 06:29:38.012 | INFO | PerfReport: 1.49%, PulseSerializing -- 4.95e-01 / 5.67e-01, 9.79e-02 / 3.97e-02 | 20260505 06:29:38.013 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 06:29:38.013 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:29:38.013 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:29:38.013 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.25e+01 / 0.00e+00 | 20260505 06:29:38.013 | INFO | PerfReport: 22.02%, Upload -- 2.24e+01 / 0.00e+00, 7.80e+00 / 0.00e+00 | 20260505 06:29:38.013 | INFO | PerfReport: 21.45%, Download -- 2.18e+01 / 0.00e+00, 7.76e+00 / 0.00e+00 | 20260505 06:29:38.013 | INFO | PerfReport: 32.63%, Compute -- 3.32e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 06:29:38.013 | INFO | PerfReport: 1.46%, LockUpload -- 1.48e+00 / 0.00e+00, 6.82e+00 / 0.00e+00 | 20260505 06:29:38.013 | INFO | PerfReport: 0.73%, LockDownload -- 7.44e-01 / 0.00e+00, 2.15e+00 / 0.00e+00 | 20260505 06:29:38.013 | INFO | PerfReport: 21.69%, LockCompute -- 2.20e+01 / 0.00e+00, 1.68e+01 / 0.00e+00 | 20260505 06:29:38.013 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 1.87e-02 / 0.00e+00 | 20260505 06:29:38.138 | INFO | Starting a round of Baz IO | 20260505 06:29:41.386 | INFO | Populating RTMetrics report | 20260505 06:29:41.386 | INFO | RTMetrics report populated | 20260505 06:29:41.386 | INFO | Opening tmp RTMetrics json file | 20260505 06:29:41.413 | INFO | Serializing RTMetrics to json | 20260505 06:29:41.414 | INFO | Serialization done, swapping tmp file now... | 20260505 06:29:41.418 | INFO | RTMetrics done | 20260505 06:29:41.502 | INFO | Finished a round of Baz IO: Wrote 2.377 GiB in 3.364 seconds | 20260505 06:29:41.502 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.14% / 0.05% / 66.82% | 20260505 06:29:41.502 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.06077% / 9.13938e-05% / 4.37101e-05% | 20260505 06:29:41.502 | INFO | Baz IO overall: | 20260505 06:29:41.502 | INFO | BytesWritten : 550846389033 | 20260505 06:29:41.502 | INFO | NumEvents : 260486021955 | 20260505 06:29:43.150 | INFO | Analyzing chunk frames = [1064960,1065472) | 20260505 06:29:43.150 | INFO | MemInfo: { | 20260505 06:29:43.150 | INFO | "gpu" : " 17 GB", | 20260505 06:29:43.150 | INFO | "huge" : "", | 20260505 06:29:43.150 | INFO | "huge_shared" : " 27 GB", | 20260505 06:29:43.150 | INFO | "malloc" : " 14 GB", | 20260505 06:29:43.150 | INFO | "pinned" : " 29 GB" | 20260505 06:29:43.150 | INFO | } | 20260505 06:29:43.150 | INFO | Perf report for the last 5.13922 seconds: | 20260505 06:29:43.150 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:29:43.150 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:43.150 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:43.150 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:43.150 | INFO | ANALYSIS : 64.5%, 2.9, 98.2ms, 0 | 20260505 06:29:43.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:43.151 | INFO | RT_METRICS : 46.7%, 1.0, 24.5ms, 0 | 20260505 06:29:43.151 | INFO | PRE_HQ : 63.4%, 2.0, 64.3ms, 0 | 20260505 06:29:43.151 | INFO | BAZWRITER : 64.9%, 4.2, 138.6ms, 0 | 20260505 06:29:43.151 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 06:29:43.151 | INFO | Uploaded 36GiB to GPU in 2.11809s (16.9965GiB/s) | 20260505 06:29:43.151 | INFO | Downloaded 38GiB to GPU in 2.31058s (16.4461GiB/s) | 20260505 06:29:48.290 | INFO | Analyzing chunk frames = [1065472,1065984) | 20260505 06:29:48.290 | INFO | MemInfo: { | 20260505 06:29:48.290 | INFO | "gpu" : " 17 GB", | 20260505 06:29:48.290 | INFO | "huge" : "", | 20260505 06:29:48.290 | INFO | "huge_shared" : " 27 GB", | 20260505 06:29:48.290 | INFO | "malloc" : " 14 GB", | 20260505 06:29:48.290 | INFO | "pinned" : " 29 GB" | 20260505 06:29:48.290 | INFO | } | 20260505 06:29:48.290 | INFO | Perf report for the last 5.13937 seconds: | 20260505 06:29:48.290 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:29:48.290 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:48.290 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:48.290 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:48.290 | INFO | ANALYSIS : 64.1%, 3.0, 100.0ms, 0 | 20260505 06:29:48.290 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:48.290 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:48.290 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 06:29:48.290 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:48.290 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:48.290 | INFO | Uploaded 36GiB to GPU in 2.11801s (16.9971GiB/s) | 20260505 06:29:48.290 | INFO | Downloaded 34GiB to GPU in 2.04027s (16.6645GiB/s) | 20260505 06:29:53.429 | INFO | Analyzing chunk frames = [1065984,1066496) | 20260505 06:29:53.429 | INFO | MemInfo: { | 20260505 06:29:53.429 | INFO | "gpu" : " 17 GB", | 20260505 06:29:53.429 | INFO | "huge" : "", | 20260505 06:29:53.429 | INFO | "huge_shared" : " 27 GB", | 20260505 06:29:53.429 | INFO | "malloc" : " 14 GB", | 20260505 06:29:53.429 | INFO | "pinned" : " 29 GB" | 20260505 06:29:53.429 | INFO | } | 20260505 06:29:53.429 | INFO | Perf report for the last 5.13917 seconds: | 20260505 06:29:53.429 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:29:53.429 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:53.429 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:53.429 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:53.429 | INFO | ANALYSIS : 63.8%, 3.0, 99.3ms, 0 | 20260505 06:29:53.429 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:53.429 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:53.429 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 06:29:53.429 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:53.429 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:53.429 | INFO | Uploaded 36GiB to GPU in 2.13704s (16.8457GiB/s) | 20260505 06:29:53.429 | INFO | Downloaded 34GiB to GPU in 2.03758s (16.6865GiB/s) | 20260505 06:29:58.568 | INFO | Analyzing chunk frames = [1066496,1067008) | 20260505 06:29:58.568 | INFO | MemInfo: { | 20260505 06:29:58.568 | INFO | "gpu" : " 17 GB", | 20260505 06:29:58.568 | INFO | "huge" : "", | 20260505 06:29:58.568 | INFO | "huge_shared" : " 27 GB", | 20260505 06:29:58.568 | INFO | "malloc" : " 14 GB", | 20260505 06:29:58.568 | INFO | "pinned" : " 29 GB" | 20260505 06:29:58.568 | INFO | } | 20260505 06:29:58.568 | INFO | Perf report for the last 5.13939 seconds: | 20260505 06:29:58.568 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:29:58.568 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:58.568 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:58.568 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:58.568 | INFO | ANALYSIS : 63.8%, 3.0, 99.1ms, 0 | 20260505 06:29:58.568 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:58.568 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:58.568 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 06:29:58.568 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:58.569 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:29:58.569 | INFO | Uploaded 36GiB to GPU in 2.1598s (16.6682GiB/s) | 20260505 06:29:58.569 | INFO | Downloaded 34GiB to GPU in 2.02789s (16.7662GiB/s) | 20260505 06:30:03.708 | INFO | Analyzing chunk frames = [1067008,1067520) | 20260505 06:30:03.708 | INFO | MemInfo: { | 20260505 06:30:03.708 | INFO | "gpu" : " 17 GB", | 20260505 06:30:03.708 | INFO | "huge" : "", | 20260505 06:30:03.708 | INFO | "huge_shared" : " 27 GB", | 20260505 06:30:03.708 | INFO | "malloc" : " 14 GB", | 20260505 06:30:03.708 | INFO | "pinned" : " 29 GB" | 20260505 06:30:03.708 | INFO | } | 20260505 06:30:03.708 | INFO | Perf report for the last 5.13949 seconds: | 20260505 06:30:03.708 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:30:03.708 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:03.708 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:03.708 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:03.708 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 06:30:03.708 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:03.708 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:03.708 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 06:30:03.708 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:03.708 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:03.708 | INFO | Uploaded 36GiB to GPU in 2.16055s (16.6624GiB/s) | 20260505 06:30:03.708 | INFO | Downloaded 34GiB to GPU in 2.05298s (16.5613GiB/s) | 20260505 06:30:08.848 | INFO | Analyzing chunk frames = [1067520,1068032) | 20260505 06:30:08.848 | INFO | MemInfo: { | 20260505 06:30:08.848 | INFO | "gpu" : " 17 GB", | 20260505 06:30:08.848 | INFO | "huge" : "", | 20260505 06:30:08.848 | INFO | "huge_shared" : " 27 GB", | 20260505 06:30:08.848 | INFO | "malloc" : " 14 GB", | 20260505 06:30:08.848 | INFO | "pinned" : " 29 GB" | 20260505 06:30:08.848 | INFO | } | 20260505 06:30:08.849 | INFO | Perf report for the last 5.14064 seconds: | 20260505 06:30:08.849 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:30:08.849 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:08.849 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:08.849 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:08.849 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 06:30:08.849 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:08.849 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:08.849 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 06:30:08.849 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:08.849 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:08.849 | INFO | Uploaded 36GiB to GPU in 2.1536s (16.7162GiB/s) | 20260505 06:30:08.849 | INFO | Downloaded 34GiB to GPU in 2.04012s (16.6656GiB/s) | 20260505 06:30:13.987 | INFO | Analyzing chunk frames = [1068032,1068544) | 20260505 06:30:13.987 | INFO | MemInfo: { | 20260505 06:30:13.987 | INFO | "gpu" : " 17 GB", | 20260505 06:30:13.987 | INFO | "huge" : "", | 20260505 06:30:13.987 | INFO | "huge_shared" : " 27 GB", | 20260505 06:30:13.987 | INFO | "malloc" : " 14 GB", | 20260505 06:30:13.987 | INFO | "pinned" : " 29 GB" | 20260505 06:30:13.987 | INFO | } | 20260505 06:30:13.987 | INFO | Perf report for the last 5.13848 seconds: | 20260505 06:30:13.987 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:30:13.987 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:13.987 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:13.987 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:13.987 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 06:30:13.987 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:13.987 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:13.987 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260505 06:30:13.987 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:13.987 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:13.987 | INFO | Uploaded 36GiB to GPU in 2.17953s (16.5174GiB/s) | 20260505 06:30:13.987 | INFO | Downloaded 34GiB to GPU in 2.15246s (15.7959GiB/s) | 20260505 06:30:19.125 | INFO | Analyzing chunk frames = [1068544,1069056) | 20260505 06:30:19.125 | INFO | MemInfo: { | 20260505 06:30:19.125 | INFO | "gpu" : " 17 GB", | 20260505 06:30:19.125 | INFO | "huge" : "", | 20260505 06:30:19.125 | INFO | "huge_shared" : " 27 GB", | 20260505 06:30:19.125 | INFO | "malloc" : " 14 GB", | 20260505 06:30:19.125 | INFO | "pinned" : " 29 GB" | 20260505 06:30:19.125 | INFO | } | 20260505 06:30:19.126 | INFO | Perf report for the last 5.13872 seconds: | 20260505 06:30:19.126 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:30:19.126 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:19.126 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:19.126 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:19.126 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 06:30:19.126 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:19.126 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:19.126 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 06:30:19.126 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:19.126 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:19.126 | INFO | Uploaded 36GiB to GPU in 2.09215s (17.2072GiB/s) | 20260505 06:30:19.126 | INFO | Downloaded 34GiB to GPU in 2.06089s (16.4977GiB/s) | 20260505 06:30:19.248 | INFO | Starting a round of Baz IO | 20260505 06:30:22.493 | INFO | Populating RTMetrics report | 20260505 06:30:22.493 | INFO | RTMetrics report populated | 20260505 06:30:22.493 | INFO | Opening tmp RTMetrics json file | 20260505 06:30:22.511 | INFO | Serializing RTMetrics to json | 20260505 06:30:22.512 | INFO | Serialization done, swapping tmp file now... | 20260505 06:30:22.512 | INFO | RTMetrics done | 20260505 06:30:22.610 | INFO | Finished a round of Baz IO: Wrote 2.374 GiB in 3.362 seconds | 20260505 06:30:22.611 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.09% / 0.05% / 66.86% | 20260505 06:30:22.611 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.01925% / 6.35783e-05% / 4.37101e-05% | 20260505 06:30:22.611 | INFO | Baz IO overall: | 20260505 06:30:22.611 | INFO | BytesWritten : 553395013873 | 20260505 06:30:22.611 | INFO | NumEvents : 261704399717 | 20260505 06:30:24.265 | INFO | Analyzing chunk frames = [1069056,1069568) | 20260505 06:30:24.265 | INFO | MemInfo: { | 20260505 06:30:24.265 | INFO | "gpu" : " 17 GB", | 20260505 06:30:24.265 | INFO | "huge" : "", | 20260505 06:30:24.265 | INFO | "huge_shared" : " 27 GB", | 20260505 06:30:24.265 | INFO | "malloc" : " 14 GB", | 20260505 06:30:24.265 | INFO | "pinned" : " 29 GB" | 20260505 06:30:24.265 | INFO | } | 20260505 06:30:24.265 | INFO | Perf report for the last 5.1393 seconds: | 20260505 06:30:24.265 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:30:24.265 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:24.265 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:24.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:24.265 | INFO | ANALYSIS : 64.3%, 2.9, 98.4ms, 0 | 20260505 06:30:24.265 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:24.265 | INFO | RT_METRICS : 46.5%, 1.0, 24.4ms, 0 | 20260505 06:30:24.265 | INFO | PRE_HQ : 63.3%, 2.0, 64.6ms, 0 | 20260505 06:30:24.265 | INFO | BAZWRITER : 64.6%, 4.1, 137.0ms, 0 | 20260505 06:30:24.265 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 06:30:24.265 | INFO | Uploaded 36GiB to GPU in 2.11508s (17.0206GiB/s) | 20260505 06:30:24.265 | INFO | Downloaded 38GiB to GPU in 2.31014s (16.4492GiB/s) | 20260505 06:30:29.404 | INFO | Analyzing chunk frames = [1069568,1070080) | 20260505 06:30:29.404 | INFO | MemInfo: { | 20260505 06:30:29.404 | INFO | "gpu" : " 17 GB", | 20260505 06:30:29.404 | INFO | "huge" : "", | 20260505 06:30:29.404 | INFO | "huge_shared" : " 27 GB", | 20260505 06:30:29.404 | INFO | "malloc" : " 14 GB", | 20260505 06:30:29.405 | INFO | "pinned" : " 29 GB" | 20260505 06:30:29.405 | INFO | } | 20260505 06:30:29.405 | INFO | Perf report for the last 5.13965 seconds: | 20260505 06:30:29.405 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:30:29.405 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:29.405 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:29.405 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:29.405 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 06:30:29.405 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:29.405 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:29.405 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 06:30:29.405 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:29.405 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:29.405 | INFO | Uploaded 36GiB to GPU in 2.11395s (17.0297GiB/s) | 20260505 06:30:29.405 | INFO | Downloaded 34GiB to GPU in 2.03832s (16.6804GiB/s) | 20260505 06:30:34.544 | INFO | Analyzing chunk frames = [1070080,1070592) | 20260505 06:30:34.544 | INFO | MemInfo: { | 20260505 06:30:34.544 | INFO | "gpu" : " 17 GB", | 20260505 06:30:34.544 | INFO | "huge" : "", | 20260505 06:30:34.544 | INFO | "huge_shared" : " 27 GB", | 20260505 06:30:34.544 | INFO | "malloc" : " 14 GB", | 20260505 06:30:34.544 | INFO | "pinned" : " 29 GB" | 20260505 06:30:34.544 | INFO | } | 20260505 06:30:34.544 | INFO | Perf report for the last 5.13952 seconds: | 20260505 06:30:34.544 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:30:34.544 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:34.544 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:34.544 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:34.544 | INFO | ANALYSIS : 63.8%, 3.0, 99.3ms, 0 | 20260505 06:30:34.544 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:34.544 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:34.544 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 06:30:34.544 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:34.544 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:34.545 | INFO | Uploaded 36GiB to GPU in 2.13647s (16.8502GiB/s) | 20260505 06:30:34.545 | INFO | Downloaded 34GiB to GPU in 2.03839s (16.6798GiB/s) | 20260505 06:30:39.683 | INFO | Analyzing chunk frames = [1070592,1071104) | 20260505 06:30:39.683 | INFO | MemInfo: { | 20260505 06:30:39.683 | INFO | "gpu" : " 17 GB", | 20260505 06:30:39.683 | INFO | "huge" : "", | 20260505 06:30:39.683 | INFO | "huge_shared" : " 27 GB", | 20260505 06:30:39.683 | INFO | "malloc" : " 14 GB", | 20260505 06:30:39.683 | INFO | "pinned" : " 29 GB" | 20260505 06:30:39.683 | INFO | } | 20260505 06:30:39.683 | INFO | Perf report for the last 5.139 seconds: | 20260505 06:30:39.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:30:39.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:39.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:39.683 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:39.683 | INFO | ANALYSIS : 63.7%, 3.0, 99.0ms, 0 | 20260505 06:30:39.683 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:39.683 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:39.683 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 06:30:39.683 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:39.683 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:39.684 | INFO | Uploaded 36GiB to GPU in 2.14941s (16.7488GiB/s) | 20260505 06:30:39.684 | INFO | Downloaded 34GiB to GPU in 2.03201s (16.7322GiB/s) | 20260505 06:30:44.822 | INFO | Analyzing chunk frames = [1071104,1071616) | 20260505 06:30:44.822 | INFO | MemInfo: { | 20260505 06:30:44.822 | INFO | "gpu" : " 17 GB", | 20260505 06:30:44.822 | INFO | "huge" : "", | 20260505 06:30:44.822 | INFO | "huge_shared" : " 27 GB", | 20260505 06:30:44.822 | INFO | "malloc" : " 14 GB", | 20260505 06:30:44.822 | INFO | "pinned" : " 29 GB" | 20260505 06:30:44.822 | INFO | } | 20260505 06:30:44.823 | INFO | Perf report for the last 5.13932 seconds: | 20260505 06:30:44.823 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:30:44.823 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:44.823 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:44.823 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:44.823 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 06:30:44.823 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:44.823 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:44.823 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 06:30:44.823 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:44.823 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:44.823 | INFO | Uploaded 36GiB to GPU in 2.15132s (16.7339GiB/s) | 20260505 06:30:44.823 | INFO | Downloaded 34GiB to GPU in 2.04244s (16.6468GiB/s) | 20260505 06:30:49.963 | INFO | Analyzing chunk frames = [1071616,1072128) | 20260505 06:30:49.963 | INFO | MemInfo: { | 20260505 06:30:49.963 | INFO | "gpu" : " 17 GB", | 20260505 06:30:49.963 | INFO | "huge" : "", | 20260505 06:30:49.963 | INFO | "huge_shared" : " 27 GB", | 20260505 06:30:49.963 | INFO | "malloc" : " 14 GB", | 20260505 06:30:49.963 | INFO | "pinned" : " 29 GB" | 20260505 06:30:49.963 | INFO | } | 20260505 06:30:49.963 | INFO | Perf report for the last 5.14053 seconds: | 20260505 06:30:49.963 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:30:49.963 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:49.963 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:49.963 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:49.963 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 06:30:49.963 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:49.963 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:49.963 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 06:30:49.963 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:49.963 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:49.963 | INFO | Uploaded 36GiB to GPU in 2.15174s (16.7307GiB/s) | 20260505 06:30:49.963 | INFO | Downloaded 34GiB to GPU in 2.04574s (16.6199GiB/s) | 20260505 06:30:55.102 | INFO | Analyzing chunk frames = [1072128,1072640) | 20260505 06:30:55.102 | INFO | MemInfo: { | 20260505 06:30:55.102 | INFO | "gpu" : " 17 GB", | 20260505 06:30:55.102 | INFO | "huge" : "", | 20260505 06:30:55.102 | INFO | "huge_shared" : " 27 GB", | 20260505 06:30:55.102 | INFO | "malloc" : " 14 GB", | 20260505 06:30:55.102 | INFO | "pinned" : " 29 GB" | 20260505 06:30:55.102 | INFO | } | 20260505 06:30:55.102 | INFO | Perf report for the last 5.13873 seconds: | 20260505 06:30:55.102 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:30:55.102 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:55.102 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:55.102 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:55.102 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 06:30:55.102 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:55.102 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:55.102 | INFO | PRE_HQ : 32.6%, 1.0, 16.9ms, 0 | 20260505 06:30:55.102 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:55.102 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:30:55.102 | INFO | Uploaded 36GiB to GPU in 2.18843s (16.4502GiB/s) | 20260505 06:30:55.102 | INFO | Downloaded 34GiB to GPU in 2.16197s (15.7264GiB/s) | 20260505 06:31:00.241 | INFO | Analyzing chunk frames = [1072640,1073152) | 20260505 06:31:00.241 | INFO | MemInfo: { | 20260505 06:31:00.241 | INFO | "gpu" : " 17 GB", | 20260505 06:31:00.241 | INFO | "huge" : "", | 20260505 06:31:00.241 | INFO | "huge_shared" : " 27 GB", | 20260505 06:31:00.241 | INFO | "malloc" : " 14 GB", | 20260505 06:31:00.241 | INFO | "pinned" : " 29 GB" | 20260505 06:31:00.241 | INFO | } | 20260505 06:31:00.241 | INFO | Perf report for the last 5.13926 seconds: | 20260505 06:31:00.241 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:31:00.241 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:00.241 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:00.241 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:00.241 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 06:31:00.241 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:00.241 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:00.241 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 06:31:00.241 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:00.241 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:00.241 | INFO | Uploaded 36GiB to GPU in 2.12348s (16.9533GiB/s) | 20260505 06:31:00.241 | INFO | Downloaded 34GiB to GPU in 2.07091s (16.4179GiB/s) | 20260505 06:31:00.242 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 06:31:00.242 | INFO | Total mem usage: 15.3 GiB | 20260505 06:31:00.242 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 06:31:00.242 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.6%) | 20260505 06:31:00.242 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 06:31:00.242 | INFO | High Water Breakdown: | 20260505 06:31:00.242 | INFO | 60.3% : 8.67 GiB ( 92.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 06:31:00.242 | INFO | 39.6% : 5.69 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 06:31:00.242 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 06:31:00.242 | INFO | ------------------------------------------ | 20260505 06:31:00.242 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:31:00.242 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 06:31:00.242 | INFO | Total mem usage: 26.5 GiB | 20260505 06:31:00.242 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 06:31:00.242 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 06:31:00.242 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 06:31:00.242 | INFO | High Water Breakdown: | 20260505 06:31:00.242 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:31:00.242 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:31:00.242 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:31:00.242 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:31:00.242 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:31:00.242 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:31:00.242 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 06:31:00.242 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 06:31:00.242 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:31:00.242 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 06:31:00.242 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 06:31:00.242 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 06:31:00.242 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 06:31:00.242 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 06:31:00.242 | INFO | ------------------------------------------ | 20260505 06:31:00.242 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:31:00.242 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 06:31:00.242 | INFO | Total mem usage: 25.6 GiB | 20260505 06:31:00.242 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 06:31:00.242 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 06:31:00.242 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 06:31:00.242 | INFO | High Water Breakdown: | 20260505 06:31:00.242 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 06:31:00.242 | INFO | ------------------------------------------ | 20260505 06:31:00.242 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:31:00.242 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 06:31:00.242 | INFO | Total mem usage: 15.7 GiB | 20260505 06:31:00.242 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 06:31:00.242 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 06:31:00.242 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260505 06:31:00.242 | INFO | High Water Breakdown: | 20260505 06:31:00.242 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 06:31:00.243 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 06:31:00.243 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 06:31:00.243 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 06:31:00.243 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 06:31:00.243 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 06:31:00.243 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 06:31:00.243 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 06:31:00.243 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 06:31:00.243 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:31:00.243 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 06:31:00.243 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:31:00.243 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 06:31:00.243 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 06:31:00.243 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 06:31:00.243 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 06:31:00.243 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:31:00.243 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 06:31:00.243 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:31:00.243 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:31:00.243 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:31:00.243 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 06:31:00.243 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 06:31:00.243 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:31:00.243 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 06:31:00.243 | INFO | ------------------------------------------ | 20260505 06:31:00.243 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:31:00.243 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 06:31:00.243 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:31:00.243 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:31:00.243 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.54e+01, 1.37e+01 / 1.53e+00 | 20260505 06:31:00.243 | INFO | PerfReport: 0.28%, AggregateStats -- 9.39e-02 / 1.02e-01, 1.09e-02 / 4.42e-03 | 20260505 06:31:00.243 | INFO | PerfReport: 11.60%, Baseline -- 3.84e+00 / 4.47e+00, 6.38e-01 / 2.73e-01 | 20260505 06:31:00.243 | INFO | PerfReport: 3.32%, Histogram -- 1.10e+00 / 1.48e+00, 2.17e-01 / 9.92e-02 | 20260505 06:31:00.243 | INFO | PerfReport: 15.46%, DME -- 4.10e+01 / 4.49e+01, 4.39e+00 / 1.08e+00 | 20260505 06:31:00.243 | INFO | PerfReport: 56.06%, FrameLabeling -- 1.86e+01 / 1.91e+01, 2.03e+00 / 1.04e+00 | 20260505 06:31:00.243 | INFO | PerfReport: 7.70%, PulseAccumulating -- 2.55e+00 / 3.13e+00, 4.60e-01 / 3.21e-01 | 20260505 06:31:00.243 | INFO | PerfReport: 4.09%, Metrics -- 1.35e+00 / 1.69e+00, 3.86e-01 / 2.01e-01 | 20260505 06:31:00.243 | INFO | PerfReport: 1.49%, PulseSerializing -- 4.93e-01 / 5.66e-01, 9.62e-02 / 4.10e-02 | 20260505 06:31:00.243 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 06:31:00.243 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:31:00.243 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:31:00.243 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.21e+01 / 0.00e+00 | 20260505 06:31:00.243 | INFO | PerfReport: 21.97%, Upload -- 2.23e+01 / 0.00e+00, 7.74e+00 / 0.00e+00 | 20260505 06:31:00.243 | INFO | PerfReport: 21.47%, Download -- 2.18e+01 / 0.00e+00, 7.75e+00 / 0.00e+00 | 20260505 06:31:00.243 | INFO | PerfReport: 32.65%, Compute -- 3.31e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 06:31:00.243 | INFO | PerfReport: 1.45%, LockUpload -- 1.47e+00 / 0.00e+00, 6.90e+00 / 0.00e+00 | 20260505 06:31:00.243 | INFO | PerfReport: 0.76%, LockDownload -- 7.75e-01 / 0.00e+00, 2.18e+00 / 0.00e+00 | 20260505 06:31:00.243 | INFO | PerfReport: 21.68%, LockCompute -- 2.20e+01 / 0.00e+00, 1.68e+01 / 0.00e+00 | 20260505 06:31:00.243 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.32e-02 / 0.00e+00 | 20260505 06:31:00.364 | INFO | Starting a round of Baz IO | 20260505 06:31:03.616 | INFO | Populating RTMetrics report | 20260505 06:31:03.616 | INFO | RTMetrics report populated | 20260505 06:31:03.616 | INFO | Opening tmp RTMetrics json file | 20260505 06:31:03.642 | INFO | Serializing RTMetrics to json | 20260505 06:31:03.657 | INFO | Serialization done, swapping tmp file now... | 20260505 06:31:03.667 | INFO | RTMetrics done | 20260505 06:31:03.727 | INFO | Finished a round of Baz IO: Wrote 2.374 GiB in 3.363 seconds | 20260505 06:31:03.727 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.04% / 0.05% / 66.91% | 20260505 06:31:03.727 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.98654% / 5.16574e-05% / 7.54992e-05% | 20260505 06:31:03.727 | INFO | Baz IO overall: | 20260505 06:31:03.727 | INFO | BytesWritten : 555943589581 | 20260505 06:31:03.727 | INFO | NumEvents : 262922378287 | 20260505 06:31:05.380 | INFO | Analyzing chunk frames = [1073152,1073664) | 20260505 06:31:05.380 | INFO | MemInfo: { | 20260505 06:31:05.380 | INFO | "gpu" : " 17 GB", | 20260505 06:31:05.380 | INFO | "huge" : "", | 20260505 06:31:05.380 | INFO | "huge_shared" : " 27 GB", | 20260505 06:31:05.380 | INFO | "malloc" : " 14 GB", | 20260505 06:31:05.380 | INFO | "pinned" : " 29 GB" | 20260505 06:31:05.380 | INFO | } | 20260505 06:31:05.380 | INFO | Perf report for the last 5.13907 seconds: | 20260505 06:31:05.380 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:31:05.380 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:05.380 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:05.380 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:05.380 | INFO | ANALYSIS : 64.5%, 2.9, 98.4ms, 0 | 20260505 06:31:05.380 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:05.380 | INFO | RT_METRICS : 46.9%, 1.0, 24.6ms, 0 | 20260505 06:31:05.380 | INFO | PRE_HQ : 63.4%, 1.9, 63.9ms, 0 | 20260505 06:31:05.380 | INFO | BAZWRITER : 64.7%, 4.2, 138.2ms, 0 | 20260505 06:31:05.380 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 06:31:05.380 | INFO | Uploaded 36GiB to GPU in 2.12752s (16.9211GiB/s) | 20260505 06:31:05.381 | INFO | Downloaded 38GiB to GPU in 2.30218s (16.5061GiB/s) | 20260505 06:31:10.519 | INFO | Analyzing chunk frames = [1073664,1074176) | 20260505 06:31:10.519 | INFO | MemInfo: { | 20260505 06:31:10.519 | INFO | "gpu" : " 17 GB", | 20260505 06:31:10.519 | INFO | "huge" : "", | 20260505 06:31:10.519 | INFO | "huge_shared" : " 27 GB", | 20260505 06:31:10.519 | INFO | "malloc" : " 14 GB", | 20260505 06:31:10.519 | INFO | "pinned" : " 29 GB" | 20260505 06:31:10.519 | INFO | } | 20260505 06:31:10.519 | INFO | Perf report for the last 5.13925 seconds: | 20260505 06:31:10.519 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:31:10.519 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:10.519 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:10.520 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:10.520 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 06:31:10.520 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:10.520 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:10.520 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 06:31:10.520 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:10.520 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:10.520 | INFO | Uploaded 36GiB to GPU in 2.11375s (17.0313GiB/s) | 20260505 06:31:10.520 | INFO | Downloaded 34GiB to GPU in 2.03955s (16.6704GiB/s) | 20260505 06:31:15.659 | INFO | Analyzing chunk frames = [1074176,1074688) | 20260505 06:31:15.659 | INFO | MemInfo: { | 20260505 06:31:15.659 | INFO | "gpu" : " 17 GB", | 20260505 06:31:15.659 | INFO | "huge" : "", | 20260505 06:31:15.659 | INFO | "huge_shared" : " 27 GB", | 20260505 06:31:15.659 | INFO | "malloc" : " 14 GB", | 20260505 06:31:15.659 | INFO | "pinned" : " 29 GB" | 20260505 06:31:15.659 | INFO | } | 20260505 06:31:15.659 | INFO | Perf report for the last 5.13926 seconds: | 20260505 06:31:15.659 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:31:15.659 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:15.659 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:15.659 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:15.659 | INFO | ANALYSIS : 63.9%, 3.0, 99.4ms, 0 | 20260505 06:31:15.659 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:15.659 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:15.659 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 06:31:15.659 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:15.659 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:15.659 | INFO | Uploaded 36GiB to GPU in 2.14089s (16.8155GiB/s) | 20260505 06:31:15.659 | INFO | Downloaded 34GiB to GPU in 2.04085s (16.6598GiB/s) | 20260505 06:31:20.803 | INFO | Analyzing chunk frames = [1074688,1075200) | 20260505 06:31:20.803 | INFO | MemInfo: { | 20260505 06:31:20.803 | INFO | "gpu" : " 17 GB", | 20260505 06:31:20.803 | INFO | "huge" : "", | 20260505 06:31:20.803 | INFO | "huge_shared" : " 27 GB", | 20260505 06:31:20.803 | INFO | "malloc" : " 14 GB", | 20260505 06:31:20.803 | INFO | "pinned" : " 29 GB" | 20260505 06:31:20.803 | INFO | } | 20260505 06:31:20.803 | INFO | Perf report for the last 5.14417 seconds: | 20260505 06:31:20.803 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:31:20.803 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:20.803 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:20.803 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:20.803 | INFO | ANALYSIS : 63.9%, 3.0, 99.3ms, 0 | 20260505 06:31:20.803 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:20.803 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:20.803 | INFO | PRE_HQ : 30.5%, 1.0, 15.9ms, 0 | 20260505 06:31:20.803 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:20.803 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:20.803 | INFO | Uploaded 36GiB to GPU in 2.17153s (16.5782GiB/s) | 20260505 06:31:20.803 | INFO | Downloaded 34GiB to GPU in 2.03954s (16.6704GiB/s) | 20260505 06:31:25.938 | INFO | Analyzing chunk frames = [1075200,1075712) | 20260505 06:31:25.938 | INFO | MemInfo: { | 20260505 06:31:25.938 | INFO | "gpu" : " 17 GB", | 20260505 06:31:25.938 | INFO | "huge" : "", | 20260505 06:31:25.938 | INFO | "huge_shared" : " 27 GB", | 20260505 06:31:25.938 | INFO | "malloc" : " 14 GB", | 20260505 06:31:25.938 | INFO | "pinned" : " 29 GB" | 20260505 06:31:25.938 | INFO | } | 20260505 06:31:25.938 | INFO | Perf report for the last 5.13497 seconds: | 20260505 06:31:25.938 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:31:25.938 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:25.938 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:25.938 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:25.938 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 06:31:25.938 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:25.938 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:25.938 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 06:31:25.938 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:25.938 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:25.938 | INFO | Uploaded 36GiB to GPU in 2.1284s (16.9141GiB/s) | 20260505 06:31:25.938 | INFO | Downloaded 34GiB to GPU in 2.02986s (16.7499GiB/s) | 20260505 06:31:31.078 | INFO | Analyzing chunk frames = [1075712,1076224) | 20260505 06:31:31.078 | INFO | MemInfo: { | 20260505 06:31:31.078 | INFO | "gpu" : " 17 GB", | 20260505 06:31:31.078 | INFO | "huge" : "", | 20260505 06:31:31.078 | INFO | "huge_shared" : " 27 GB", | 20260505 06:31:31.078 | INFO | "malloc" : " 14 GB", | 20260505 06:31:31.078 | INFO | "pinned" : " 29 GB" | 20260505 06:31:31.078 | INFO | } | 20260505 06:31:31.078 | INFO | Perf report for the last 5.14021 seconds: | 20260505 06:31:31.078 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:31:31.078 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:31.078 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:31.078 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:31.078 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 06:31:31.078 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:31.078 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:31.078 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 06:31:31.078 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:31.078 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:31.078 | INFO | Uploaded 36GiB to GPU in 2.13137s (16.8905GiB/s) | 20260505 06:31:31.078 | INFO | Downloaded 34GiB to GPU in 2.06794s (16.4415GiB/s) | 20260505 06:31:36.216 | INFO | Analyzing chunk frames = [1076224,1076736) | 20260505 06:31:36.216 | INFO | MemInfo: { | 20260505 06:31:36.216 | INFO | "gpu" : " 17 GB", | 20260505 06:31:36.216 | INFO | "huge" : "", | 20260505 06:31:36.216 | INFO | "huge_shared" : " 27 GB", | 20260505 06:31:36.216 | INFO | "malloc" : " 14 GB", | 20260505 06:31:36.216 | INFO | "pinned" : " 29 GB" | 20260505 06:31:36.216 | INFO | } | 20260505 06:31:36.216 | INFO | Perf report for the last 5.13773 seconds: | 20260505 06:31:36.216 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:31:36.216 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:36.216 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:36.216 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:36.216 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 06:31:36.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:36.216 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:36.216 | INFO | PRE_HQ : 32.6%, 1.0, 17.0ms, 0 | 20260505 06:31:36.216 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:36.216 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:36.216 | INFO | Uploaded 36GiB to GPU in 2.17237s (16.5717GiB/s) | 20260505 06:31:36.216 | INFO | Downloaded 34GiB to GPU in 2.15202s (15.7991GiB/s) | 20260505 06:31:41.355 | INFO | Analyzing chunk frames = [1076736,1077248) | 20260505 06:31:41.355 | INFO | MemInfo: { | 20260505 06:31:41.355 | INFO | "gpu" : " 17 GB", | 20260505 06:31:41.355 | INFO | "huge" : "", | 20260505 06:31:41.355 | INFO | "huge_shared" : " 27 GB", | 20260505 06:31:41.355 | INFO | "malloc" : " 14 GB", | 20260505 06:31:41.355 | INFO | "pinned" : " 29 GB" | 20260505 06:31:41.355 | INFO | } | 20260505 06:31:41.355 | INFO | Perf report for the last 5.13957 seconds: | 20260505 06:31:41.355 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:31:41.355 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:41.355 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:41.355 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:41.356 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 06:31:41.356 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:41.356 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:41.356 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260505 06:31:41.356 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:41.356 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:41.356 | INFO | Uploaded 36GiB to GPU in 2.10684s (17.0872GiB/s) | 20260505 06:31:41.356 | INFO | Downloaded 34GiB to GPU in 2.05896s (16.5132GiB/s) | 20260505 06:31:41.478 | INFO | Starting a round of Baz IO | 20260505 06:31:44.722 | INFO | Populating RTMetrics report | 20260505 06:31:44.723 | INFO | RTMetrics report populated | 20260505 06:31:44.723 | INFO | Opening tmp RTMetrics json file | 20260505 06:31:44.734 | INFO | Serializing RTMetrics to json | 20260505 06:31:44.748 | INFO | Serialization done, swapping tmp file now... | 20260505 06:31:44.749 | INFO | RTMetrics done | 20260505 06:31:44.889 | INFO | Finished a round of Baz IO: Wrote 2.378 GiB in 3.41 seconds | 20260505 06:31:44.889 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.99% / 0.05% / 66.96% | 20260505 06:31:44.889 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.98739% / 5.16574e-05% / 3.97364e-05% | 20260505 06:31:44.889 | INFO | Baz IO overall: | 20260505 06:31:44.889 | INFO | BytesWritten : 558496821055 | 20260505 06:31:44.889 | INFO | NumEvents : 264143584786 | 20260505 06:31:46.495 | INFO | Analyzing chunk frames = [1077248,1077760) | 20260505 06:31:46.495 | INFO | MemInfo: { | 20260505 06:31:46.495 | INFO | "gpu" : " 17 GB", | 20260505 06:31:46.495 | INFO | "huge" : "", | 20260505 06:31:46.495 | INFO | "huge_shared" : " 27 GB", | 20260505 06:31:46.495 | INFO | "malloc" : " 14 GB", | 20260505 06:31:46.495 | INFO | "pinned" : " 29 GB" | 20260505 06:31:46.495 | INFO | } | 20260505 06:31:46.495 | INFO | Perf report for the last 5.13979 seconds: | 20260505 06:31:46.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:31:46.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:46.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:46.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:46.495 | INFO | ANALYSIS : 64.4%, 2.9, 97.8ms, 0 | 20260505 06:31:46.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:46.495 | INFO | RT_METRICS : 46.5%, 1.0, 24.4ms, 0 | 20260505 06:31:46.495 | INFO | PRE_HQ : 63.3%, 1.9, 63.9ms, 0 | 20260505 06:31:46.495 | INFO | BAZWRITER : 65.6%, 4.2, 141.6ms, 0 | 20260505 06:31:46.495 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 06:31:46.496 | INFO | Uploaded 36GiB to GPU in 2.13738s (16.843GiB/s) | 20260505 06:31:46.496 | INFO | Downloaded 38GiB to GPU in 2.32211s (16.3644GiB/s) | 20260505 06:31:51.634 | INFO | Analyzing chunk frames = [1077760,1078272) | 20260505 06:31:51.634 | INFO | MemInfo: { | 20260505 06:31:51.634 | INFO | "gpu" : " 17 GB", | 20260505 06:31:51.634 | INFO | "huge" : "", | 20260505 06:31:51.634 | INFO | "huge_shared" : " 27 GB", | 20260505 06:31:51.634 | INFO | "malloc" : " 14 GB", | 20260505 06:31:51.634 | INFO | "pinned" : " 29 GB" | 20260505 06:31:51.634 | INFO | } | 20260505 06:31:51.634 | INFO | Perf report for the last 5.13891 seconds: | 20260505 06:31:51.634 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:31:51.634 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:51.634 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:51.634 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:51.634 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 06:31:51.634 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:51.634 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:51.634 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 06:31:51.634 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:51.634 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:51.634 | INFO | Uploaded 36GiB to GPU in 2.11523s (17.0194GiB/s) | 20260505 06:31:51.634 | INFO | Downloaded 34GiB to GPU in 2.04016s (16.6653GiB/s) | 20260505 06:31:56.773 | INFO | Analyzing chunk frames = [1078272,1078784) | 20260505 06:31:56.773 | INFO | MemInfo: { | 20260505 06:31:56.773 | INFO | "gpu" : " 17 GB", | 20260505 06:31:56.773 | INFO | "huge" : "", | 20260505 06:31:56.773 | INFO | "huge_shared" : " 27 GB", | 20260505 06:31:56.773 | INFO | "malloc" : " 14 GB", | 20260505 06:31:56.774 | INFO | "pinned" : " 29 GB" | 20260505 06:31:56.774 | INFO | } | 20260505 06:31:56.774 | INFO | Perf report for the last 5.1394 seconds: | 20260505 06:31:56.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:31:56.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:56.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:56.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:56.774 | INFO | ANALYSIS : 63.8%, 3.0, 99.3ms, 0 | 20260505 06:31:56.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:56.774 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:56.774 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 06:31:56.774 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:56.774 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:31:56.774 | INFO | Uploaded 36GiB to GPU in 2.14052s (16.8183GiB/s) | 20260505 06:31:56.774 | INFO | Downloaded 34GiB to GPU in 2.02882s (16.7585GiB/s) | 20260505 06:32:01.913 | INFO | Analyzing chunk frames = [1078784,1079296) | 20260505 06:32:01.913 | INFO | MemInfo: { | 20260505 06:32:01.913 | INFO | "gpu" : " 17 GB", | 20260505 06:32:01.913 | INFO | "huge" : "", | 20260505 06:32:01.913 | INFO | "huge_shared" : " 27 GB", | 20260505 06:32:01.913 | INFO | "malloc" : " 14 GB", | 20260505 06:32:01.913 | INFO | "pinned" : " 29 GB" | 20260505 06:32:01.913 | INFO | } | 20260505 06:32:01.913 | INFO | Perf report for the last 5.13965 seconds: | 20260505 06:32:01.913 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:32:01.913 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:01.913 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:01.913 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:01.913 | INFO | ANALYSIS : 63.9%, 3.0, 99.3ms, 0 | 20260505 06:32:01.913 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:01.913 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:01.913 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 06:32:01.913 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:01.913 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:01.914 | INFO | Uploaded 36GiB to GPU in 2.17272s (16.5691GiB/s) | 20260505 06:32:01.914 | INFO | Downloaded 34GiB to GPU in 2.03625s (16.6974GiB/s) | 20260505 06:32:07.052 | INFO | Analyzing chunk frames = [1079296,1079808) | 20260505 06:32:07.052 | INFO | MemInfo: { | 20260505 06:32:07.052 | INFO | "gpu" : " 17 GB", | 20260505 06:32:07.052 | INFO | "huge" : "", | 20260505 06:32:07.052 | INFO | "huge_shared" : " 27 GB", | 20260505 06:32:07.052 | INFO | "malloc" : " 14 GB", | 20260505 06:32:07.052 | INFO | "pinned" : " 29 GB" | 20260505 06:32:07.052 | INFO | } | 20260505 06:32:07.052 | INFO | Perf report for the last 5.13875 seconds: | 20260505 06:32:07.052 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:32:07.052 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:07.052 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:07.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:07.052 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 06:32:07.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:07.052 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:07.052 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 06:32:07.052 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:07.052 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:07.052 | INFO | Uploaded 36GiB to GPU in 2.17385s (16.5605GiB/s) | 20260505 06:32:07.052 | INFO | Downloaded 34GiB to GPU in 2.04798s (16.6017GiB/s) | 20260505 06:32:12.192 | INFO | Analyzing chunk frames = [1079808,1080320) | 20260505 06:32:12.192 | INFO | MemInfo: { | 20260505 06:32:12.192 | INFO | "gpu" : " 17 GB", | 20260505 06:32:12.192 | INFO | "huge" : "", | 20260505 06:32:12.192 | INFO | "huge_shared" : " 27 GB", | 20260505 06:32:12.192 | INFO | "malloc" : " 14 GB", | 20260505 06:32:12.192 | INFO | "pinned" : " 29 GB" | 20260505 06:32:12.192 | INFO | } | 20260505 06:32:12.192 | INFO | Perf report for the last 5.14037 seconds: | 20260505 06:32:12.192 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:32:12.192 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:12.192 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:12.192 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:12.192 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 06:32:12.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:12.192 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:12.193 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 06:32:12.193 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:12.193 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:12.193 | INFO | Uploaded 36GiB to GPU in 2.15051s (16.7402GiB/s) | 20260505 06:32:12.193 | INFO | Downloaded 34GiB to GPU in 2.03779s (16.6848GiB/s) | 20260505 06:32:17.331 | INFO | Analyzing chunk frames = [1080320,1080832) | 20260505 06:32:17.331 | INFO | MemInfo: { | 20260505 06:32:17.331 | INFO | "gpu" : " 17 GB", | 20260505 06:32:17.331 | INFO | "huge" : "", | 20260505 06:32:17.331 | INFO | "huge_shared" : " 27 GB", | 20260505 06:32:17.331 | INFO | "malloc" : " 14 GB", | 20260505 06:32:17.331 | INFO | "pinned" : " 29 GB" | 20260505 06:32:17.331 | INFO | } | 20260505 06:32:17.331 | INFO | Perf report for the last 5.13854 seconds: | 20260505 06:32:17.331 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:32:17.331 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:17.331 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:17.331 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:17.331 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 06:32:17.331 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:17.331 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:17.331 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260505 06:32:17.331 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:17.331 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:17.331 | INFO | Uploaded 36GiB to GPU in 2.18819s (16.452GiB/s) | 20260505 06:32:17.331 | INFO | Downloaded 34GiB to GPU in 2.16054s (15.7368GiB/s) | 20260505 06:32:22.470 | INFO | Analyzing chunk frames = [1080832,1081344) | 20260505 06:32:22.470 | INFO | MemInfo: { | 20260505 06:32:22.470 | INFO | "gpu" : " 17 GB", | 20260505 06:32:22.470 | INFO | "huge" : "", | 20260505 06:32:22.470 | INFO | "huge_shared" : " 27 GB", | 20260505 06:32:22.470 | INFO | "malloc" : " 14 GB", | 20260505 06:32:22.470 | INFO | "pinned" : " 29 GB" | 20260505 06:32:22.470 | INFO | } | 20260505 06:32:22.470 | INFO | Perf report for the last 5.1391 seconds: | 20260505 06:32:22.470 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:32:22.470 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:22.470 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:22.470 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:22.470 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 06:32:22.470 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:22.470 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:22.470 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 06:32:22.470 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:22.470 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:22.470 | INFO | Uploaded 36GiB to GPU in 2.13135s (16.8907GiB/s) | 20260505 06:32:22.470 | INFO | Downloaded 34GiB to GPU in 2.07135s (16.4144GiB/s) | 20260505 06:32:22.471 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 06:32:22.471 | INFO | Total mem usage: 15.3 GiB | 20260505 06:32:22.471 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 06:32:22.471 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -5.87%) | 20260505 06:32:22.471 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 06:32:22.471 | INFO | High Water Breakdown: | 20260505 06:32:22.471 | INFO | 60.4% : 8.75 GiB ( 93.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 06:32:22.471 | INFO | 39.4% : 5.71 GiB ( 85.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 06:32:22.471 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 06:32:22.471 | INFO | ------------------------------------------ | 20260505 06:32:22.471 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:32:22.471 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 06:32:22.471 | INFO | Total mem usage: 26.5 GiB | 20260505 06:32:22.471 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 06:32:22.471 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 06:32:22.471 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 06:32:22.471 | INFO | High Water Breakdown: | 20260505 06:32:22.471 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:32:22.471 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:32:22.471 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:32:22.471 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:32:22.471 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:32:22.471 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:32:22.471 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 06:32:22.471 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 06:32:22.471 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:32:22.471 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 06:32:22.471 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 06:32:22.471 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 06:32:22.471 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 06:32:22.471 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 06:32:22.471 | INFO | ------------------------------------------ | 20260505 06:32:22.471 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:32:22.471 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 06:32:22.471 | INFO | Total mem usage: 25.6 GiB | 20260505 06:32:22.471 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 06:32:22.471 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 06:32:22.471 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 06:32:22.471 | INFO | High Water Breakdown: | 20260505 06:32:22.471 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 06:32:22.471 | INFO | ------------------------------------------ | 20260505 06:32:22.471 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:32:22.471 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 06:32:22.471 | INFO | Total mem usage: 15.7 GiB | 20260505 06:32:22.471 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 06:32:22.472 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 06:32:22.472 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260505 06:32:22.472 | INFO | High Water Breakdown: | 20260505 06:32:22.472 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 06:32:22.472 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 06:32:22.472 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 06:32:22.472 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 06:32:22.472 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 06:32:22.472 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 06:32:22.472 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 06:32:22.472 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 06:32:22.472 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 06:32:22.472 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:32:22.472 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 06:32:22.472 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:32:22.472 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 06:32:22.472 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 06:32:22.472 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 06:32:22.472 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 06:32:22.472 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:32:22.472 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 06:32:22.472 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:32:22.472 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:32:22.472 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:32:22.472 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 06:32:22.472 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 06:32:22.472 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:32:22.472 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 06:32:22.472 | INFO | ------------------------------------------ | 20260505 06:32:22.472 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:32:22.472 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 06:32:22.472 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:32:22.472 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:32:22.472 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.54e+01, 1.38e+01 / 1.59e+00 | 20260505 06:32:22.472 | INFO | PerfReport: 0.28%, AggregateStats -- 9.41e-02 / 1.03e-01, 1.09e-02 / 7.25e-03 | 20260505 06:32:22.472 | INFO | PerfReport: 11.59%, Baseline -- 3.84e+00 / 4.45e+00, 6.40e-01 / 3.46e-01 | 20260505 06:32:22.472 | INFO | PerfReport: 3.31%, Histogram -- 1.10e+00 / 1.48e+00, 2.19e-01 / 1.30e-01 | 20260505 06:32:22.472 | INFO | PerfReport: 15.48%, DME -- 4.10e+01 / 4.48e+01, 4.31e+00 / 1.13e+00 | 20260505 06:32:22.472 | INFO | PerfReport: 56.06%, FrameLabeling -- 1.86e+01 / 1.92e+01, 2.05e+00 / 1.05e+00 | 20260505 06:32:22.472 | INFO | PerfReport: 7.69%, PulseAccumulating -- 2.55e+00 / 3.10e+00, 4.69e-01 / 3.39e-01 | 20260505 06:32:22.472 | INFO | PerfReport: 4.09%, Metrics -- 1.36e+00 / 1.68e+00, 3.87e-01 / 2.01e-01 | 20260505 06:32:22.472 | INFO | PerfReport: 1.49%, PulseSerializing -- 4.93e-01 / 5.63e-01, 9.74e-02 / 4.39e-02 | 20260505 06:32:22.472 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 06:32:22.472 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:32:22.472 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:32:22.472 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.25e+01 / 0.00e+00 | 20260505 06:32:22.472 | INFO | PerfReport: 22.01%, Upload -- 2.23e+01 / 0.00e+00, 7.74e+00 / 0.00e+00 | 20260505 06:32:22.472 | INFO | PerfReport: 21.48%, Download -- 2.18e+01 / 0.00e+00, 7.74e+00 / 0.00e+00 | 20260505 06:32:22.472 | INFO | PerfReport: 32.65%, Compute -- 3.31e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 06:32:22.472 | INFO | PerfReport: 1.50%, LockUpload -- 1.53e+00 / 0.00e+00, 6.92e+00 / 0.00e+00 | 20260505 06:32:22.472 | INFO | PerfReport: 0.76%, LockDownload -- 7.74e-01 / 0.00e+00, 2.22e+00 / 0.00e+00 | 20260505 06:32:22.472 | INFO | PerfReport: 21.57%, LockCompute -- 2.19e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260505 06:32:22.472 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.51e-02 / 0.00e+00 | 20260505 06:32:22.593 | INFO | Starting a round of Baz IO | 20260505 06:32:25.826 | INFO | Populating RTMetrics report | 20260505 06:32:25.826 | INFO | RTMetrics report populated | 20260505 06:32:25.826 | INFO | Opening tmp RTMetrics json file | 20260505 06:32:25.849 | INFO | Serializing RTMetrics to json | 20260505 06:32:25.850 | INFO | Serialization done, swapping tmp file now... | 20260505 06:32:25.852 | INFO | RTMetrics done | 20260505 06:32:25.946 | INFO | Finished a round of Baz IO: Wrote 2.379 GiB in 3.352 seconds | 20260505 06:32:25.946 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.95% / 0.05% / 67.01% | 20260505 06:32:25.946 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.971% / 6.75519e-05% / 6.75519e-05% | 20260505 06:32:25.946 | INFO | Baz IO overall: | 20260505 06:32:25.946 | INFO | BytesWritten : 561051597759 | 20260505 06:32:25.946 | INFO | NumEvents : 265365589495 | 20260505 06:32:27.609 | INFO | Analyzing chunk frames = [1081344,1081856) | 20260505 06:32:27.609 | INFO | MemInfo: { | 20260505 06:32:27.609 | INFO | "gpu" : " 17 GB", | 20260505 06:32:27.609 | INFO | "huge" : "", | 20260505 06:32:27.609 | INFO | "huge_shared" : " 27 GB", | 20260505 06:32:27.609 | INFO | "malloc" : " 14 GB", | 20260505 06:32:27.609 | INFO | "pinned" : " 29 GB" | 20260505 06:32:27.609 | INFO | } | 20260505 06:32:27.609 | INFO | Perf report for the last 5.13937 seconds: | 20260505 06:32:27.610 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:32:27.610 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:27.610 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:27.610 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:27.610 | INFO | ANALYSIS : 64.2%, 2.9, 97.8ms, 0 | 20260505 06:32:27.610 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:27.610 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260505 06:32:27.610 | INFO | PRE_HQ : 63.1%, 1.9, 63.8ms, 0 | 20260505 06:32:27.610 | INFO | BAZWRITER : 64.5%, 4.1, 137.4ms, 0 | 20260505 06:32:27.610 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 06:32:27.610 | INFO | Uploaded 36GiB to GPU in 2.0656s (17.4283GiB/s) | 20260505 06:32:27.610 | INFO | Downloaded 38GiB to GPU in 2.29997s (16.5219GiB/s) | 20260505 06:32:32.749 | INFO | Analyzing chunk frames = [1081856,1082368) | 20260505 06:32:32.749 | INFO | MemInfo: { | 20260505 06:32:32.749 | INFO | "gpu" : " 17 GB", | 20260505 06:32:32.749 | INFO | "huge" : "", | 20260505 06:32:32.749 | INFO | "huge_shared" : " 27 GB", | 20260505 06:32:32.749 | INFO | "malloc" : " 14 GB", | 20260505 06:32:32.749 | INFO | "pinned" : " 29 GB" | 20260505 06:32:32.749 | INFO | } | 20260505 06:32:32.749 | INFO | Perf report for the last 5.13932 seconds: | 20260505 06:32:32.749 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:32:32.749 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:32.749 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:32.749 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:32.749 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 06:32:32.749 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:32.749 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:32.749 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 06:32:32.749 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:32.749 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:32.749 | INFO | Uploaded 36GiB to GPU in 2.11284s (17.0387GiB/s) | 20260505 06:32:32.749 | INFO | Downloaded 34GiB to GPU in 2.03947s (16.671GiB/s) | 20260505 06:32:37.888 | INFO | Analyzing chunk frames = [1082368,1082880) | 20260505 06:32:37.888 | INFO | MemInfo: { | 20260505 06:32:37.888 | INFO | "gpu" : " 17 GB", | 20260505 06:32:37.888 | INFO | "huge" : "", | 20260505 06:32:37.888 | INFO | "huge_shared" : " 27 GB", | 20260505 06:32:37.888 | INFO | "malloc" : " 14 GB", | 20260505 06:32:37.888 | INFO | "pinned" : " 29 GB" | 20260505 06:32:37.888 | INFO | } | 20260505 06:32:37.888 | INFO | Perf report for the last 5.13937 seconds: | 20260505 06:32:37.888 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:32:37.888 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:37.888 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:37.888 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:37.888 | INFO | ANALYSIS : 63.6%, 3.0, 99.0ms, 0 | 20260505 06:32:37.888 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:37.888 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:37.888 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 06:32:37.888 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:37.888 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:37.888 | INFO | Uploaded 36GiB to GPU in 2.12081s (16.9746GiB/s) | 20260505 06:32:37.889 | INFO | Downloaded 34GiB to GPU in 2.03518s (16.7061GiB/s) | 20260505 06:32:43.027 | INFO | Analyzing chunk frames = [1082880,1083392) | 20260505 06:32:43.027 | INFO | MemInfo: { | 20260505 06:32:43.027 | INFO | "gpu" : " 17 GB", | 20260505 06:32:43.027 | INFO | "huge" : "", | 20260505 06:32:43.027 | INFO | "huge_shared" : " 27 GB", | 20260505 06:32:43.027 | INFO | "malloc" : " 14 GB", | 20260505 06:32:43.027 | INFO | "pinned" : " 29 GB" | 20260505 06:32:43.027 | INFO | } | 20260505 06:32:43.027 | INFO | Perf report for the last 5.1393 seconds: | 20260505 06:32:43.027 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:32:43.027 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:43.028 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:43.028 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:43.028 | INFO | ANALYSIS : 63.8%, 3.0, 99.2ms, 0 | 20260505 06:32:43.028 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:43.028 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:43.028 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 06:32:43.028 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:43.028 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:43.028 | INFO | Uploaded 36GiB to GPU in 2.16098s (16.6591GiB/s) | 20260505 06:32:43.028 | INFO | Downloaded 34GiB to GPU in 2.03779s (16.6847GiB/s) | 20260505 06:32:48.167 | INFO | Analyzing chunk frames = [1083392,1083904) | 20260505 06:32:48.167 | INFO | MemInfo: { | 20260505 06:32:48.167 | INFO | "gpu" : " 17 GB", | 20260505 06:32:48.167 | INFO | "huge" : "", | 20260505 06:32:48.167 | INFO | "huge_shared" : " 27 GB", | 20260505 06:32:48.167 | INFO | "malloc" : " 14 GB", | 20260505 06:32:48.167 | INFO | "pinned" : " 29 GB" | 20260505 06:32:48.167 | INFO | } | 20260505 06:32:48.167 | INFO | Perf report for the last 5.13934 seconds: | 20260505 06:32:48.167 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:32:48.167 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:48.167 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:48.167 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:48.167 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 06:32:48.167 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:48.167 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:48.167 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 06:32:48.167 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:48.167 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:48.167 | INFO | Uploaded 36GiB to GPU in 2.14799s (16.7598GiB/s) | 20260505 06:32:48.167 | INFO | Downloaded 34GiB to GPU in 2.04861s (16.5966GiB/s) | 20260505 06:32:53.307 | INFO | Analyzing chunk frames = [1083904,1084416) | 20260505 06:32:53.308 | INFO | MemInfo: { | 20260505 06:32:53.308 | INFO | "gpu" : " 17 GB", | 20260505 06:32:53.308 | INFO | "huge" : "", | 20260505 06:32:53.308 | INFO | "huge_shared" : " 27 GB", | 20260505 06:32:53.308 | INFO | "malloc" : " 14 GB", | 20260505 06:32:53.308 | INFO | "pinned" : " 29 GB" | 20260505 06:32:53.308 | INFO | } | 20260505 06:32:53.308 | INFO | Perf report for the last 5.14094 seconds: | 20260505 06:32:53.308 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:32:53.308 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:53.308 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:53.308 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:53.308 | INFO | ANALYSIS : 63.9%, 3.0, 99.9ms, 0 | 20260505 06:32:53.308 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:53.308 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:53.308 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260505 06:32:53.308 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:53.308 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:53.308 | INFO | Uploaded 36GiB to GPU in 2.1562s (16.6961GiB/s) | 20260505 06:32:53.308 | INFO | Downloaded 34GiB to GPU in 2.04205s (16.65GiB/s) | 20260505 06:32:58.445 | INFO | Analyzing chunk frames = [1084416,1084928) | 20260505 06:32:58.445 | INFO | MemInfo: { | 20260505 06:32:58.445 | INFO | "gpu" : " 17 GB", | 20260505 06:32:58.446 | INFO | "huge" : "", | 20260505 06:32:58.446 | INFO | "huge_shared" : " 27 GB", | 20260505 06:32:58.446 | INFO | "malloc" : " 14 GB", | 20260505 06:32:58.446 | INFO | "pinned" : " 29 GB" | 20260505 06:32:58.446 | INFO | } | 20260505 06:32:58.446 | INFO | Perf report for the last 5.13785 seconds: | 20260505 06:32:58.446 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:32:58.446 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:58.446 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:58.446 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:58.446 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 06:32:58.446 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:58.446 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:58.446 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260505 06:32:58.446 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:58.446 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:32:58.446 | INFO | Uploaded 36GiB to GPU in 2.19231s (16.4211GiB/s) | 20260505 06:32:58.446 | INFO | Downloaded 34GiB to GPU in 2.1572s (15.7612GiB/s) | 20260505 06:33:03.585 | INFO | Analyzing chunk frames = [1084928,1085440) | 20260505 06:33:03.585 | INFO | MemInfo: { | 20260505 06:33:03.585 | INFO | "gpu" : " 17 GB", | 20260505 06:33:03.585 | INFO | "huge" : "", | 20260505 06:33:03.585 | INFO | "huge_shared" : " 27 GB", | 20260505 06:33:03.585 | INFO | "malloc" : " 14 GB", | 20260505 06:33:03.585 | INFO | "pinned" : " 29 GB" | 20260505 06:33:03.585 | INFO | } | 20260505 06:33:03.585 | INFO | Perf report for the last 5.13915 seconds: | 20260505 06:33:03.585 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:33:03.585 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:03.585 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:03.585 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:03.585 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 06:33:03.585 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:03.585 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:03.585 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 06:33:03.585 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:03.585 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:03.585 | INFO | Uploaded 36GiB to GPU in 2.13106s (16.893GiB/s) | 20260505 06:33:03.585 | INFO | Downloaded 34GiB to GPU in 2.06339s (16.4777GiB/s) | 20260505 06:33:03.708 | INFO | Starting a round of Baz IO | 20260505 06:33:06.952 | INFO | Populating RTMetrics report | 20260505 06:33:06.952 | INFO | RTMetrics report populated | 20260505 06:33:06.952 | INFO | Opening tmp RTMetrics json file | 20260505 06:33:06.980 | INFO | Serializing RTMetrics to json | 20260505 06:33:07.003 | INFO | Serialization done, swapping tmp file now... | 20260505 06:33:07.006 | INFO | RTMetrics done | 20260505 06:33:07.084 | INFO | Finished a round of Baz IO: Wrote 2.382 GiB in 3.376 seconds | 20260505 06:33:07.084 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.90% / 0.05% / 67.05% | 20260505 06:33:07.084 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.95461% / 5.5631e-05% / 5.5631e-05% | 20260505 06:33:07.084 | INFO | Baz IO overall: | 20260505 06:33:07.084 | INFO | BytesWritten : 563608977441 | 20260505 06:33:07.084 | INFO | NumEvents : 266589119355 | 20260505 06:33:08.724 | INFO | Analyzing chunk frames = [1085440,1085952) | 20260505 06:33:08.724 | INFO | MemInfo: { | 20260505 06:33:08.724 | INFO | "gpu" : " 17 GB", | 20260505 06:33:08.724 | INFO | "huge" : "", | 20260505 06:33:08.724 | INFO | "huge_shared" : " 27 GB", | 20260505 06:33:08.724 | INFO | "malloc" : " 14 GB", | 20260505 06:33:08.724 | INFO | "pinned" : " 29 GB" | 20260505 06:33:08.724 | INFO | } | 20260505 06:33:08.724 | INFO | Perf report for the last 5.13955 seconds: | 20260505 06:33:08.724 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:33:08.724 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:08.724 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:08.724 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:08.724 | INFO | ANALYSIS : 64.4%, 2.9, 98.2ms, 0 | 20260505 06:33:08.724 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:08.724 | INFO | RT_METRICS : 47.0%, 1.0, 24.6ms, 0 | 20260505 06:33:08.725 | INFO | PRE_HQ : 63.3%, 1.9, 64.0ms, 0 | 20260505 06:33:08.725 | INFO | BAZWRITER : 65.0%, 4.1, 137.9ms, 0 | 20260505 06:33:08.725 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 06:33:08.725 | INFO | Uploaded 36GiB to GPU in 2.11306s (17.0369GiB/s) | 20260505 06:33:08.725 | INFO | Downloaded 38GiB to GPU in 2.3233s (16.356GiB/s) | 20260505 06:33:13.864 | INFO | Analyzing chunk frames = [1085952,1086464) | 20260505 06:33:13.864 | INFO | MemInfo: { | 20260505 06:33:13.864 | INFO | "gpu" : " 17 GB", | 20260505 06:33:13.864 | INFO | "huge" : "", | 20260505 06:33:13.864 | INFO | "huge_shared" : " 27 GB", | 20260505 06:33:13.864 | INFO | "malloc" : " 14 GB", | 20260505 06:33:13.864 | INFO | "pinned" : " 29 GB" | 20260505 06:33:13.864 | INFO | } | 20260505 06:33:13.864 | INFO | Perf report for the last 5.13977 seconds: | 20260505 06:33:13.864 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:33:13.864 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:13.864 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:13.864 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:13.864 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 06:33:13.864 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:13.864 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:13.864 | INFO | PRE_HQ : 30.5%, 1.0, 16.0ms, 0 | 20260505 06:33:13.864 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:13.864 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:13.864 | INFO | Uploaded 36GiB to GPU in 2.10851s (17.0737GiB/s) | 20260505 06:33:13.865 | INFO | Downloaded 34GiB to GPU in 2.04112s (16.6575GiB/s) | 20260505 06:33:19.003 | INFO | Analyzing chunk frames = [1086464,1086976) | 20260505 06:33:19.003 | INFO | MemInfo: { | 20260505 06:33:19.003 | INFO | "gpu" : " 17 GB", | 20260505 06:33:19.003 | INFO | "huge" : "", | 20260505 06:33:19.003 | INFO | "huge_shared" : " 27 GB", | 20260505 06:33:19.003 | INFO | "malloc" : " 14 GB", | 20260505 06:33:19.003 | INFO | "pinned" : " 29 GB" | 20260505 06:33:19.003 | INFO | } | 20260505 06:33:19.003 | INFO | Perf report for the last 5.1389 seconds: | 20260505 06:33:19.003 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:33:19.003 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:19.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:19.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:19.003 | INFO | ANALYSIS : 63.8%, 3.0, 99.3ms, 0 | 20260505 06:33:19.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:19.003 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:19.003 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 06:33:19.003 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:19.003 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:19.003 | INFO | Uploaded 36GiB to GPU in 2.1372s (16.8445GiB/s) | 20260505 06:33:19.003 | INFO | Downloaded 34GiB to GPU in 2.0459s (16.6186GiB/s) | 20260505 06:33:24.142 | INFO | Analyzing chunk frames = [1086976,1087488) | 20260505 06:33:24.142 | INFO | MemInfo: { | 20260505 06:33:24.142 | INFO | "gpu" : " 17 GB", | 20260505 06:33:24.142 | INFO | "huge" : "", | 20260505 06:33:24.142 | INFO | "huge_shared" : " 27 GB", | 20260505 06:33:24.142 | INFO | "malloc" : " 14 GB", | 20260505 06:33:24.142 | INFO | "pinned" : " 29 GB" | 20260505 06:33:24.142 | INFO | } | 20260505 06:33:24.142 | INFO | Perf report for the last 5.13909 seconds: | 20260505 06:33:24.142 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:33:24.142 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:24.142 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:24.142 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:24.142 | INFO | ANALYSIS : 63.7%, 3.0, 99.0ms, 0 | 20260505 06:33:24.142 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:24.142 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:24.142 | INFO | PRE_HQ : 30.5%, 1.0, 15.9ms, 0 | 20260505 06:33:24.142 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:24.142 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:24.142 | INFO | Uploaded 36GiB to GPU in 2.15709s (16.6891GiB/s) | 20260505 06:33:24.142 | INFO | Downloaded 34GiB to GPU in 2.03378s (16.7176GiB/s) | 20260505 06:33:29.282 | INFO | Analyzing chunk frames = [1087488,1088000) | 20260505 06:33:29.282 | INFO | MemInfo: { | 20260505 06:33:29.282 | INFO | "gpu" : " 17 GB", | 20260505 06:33:29.282 | INFO | "huge" : "", | 20260505 06:33:29.282 | INFO | "huge_shared" : " 27 GB", | 20260505 06:33:29.282 | INFO | "malloc" : " 14 GB", | 20260505 06:33:29.282 | INFO | "pinned" : " 29 GB" | 20260505 06:33:29.282 | INFO | } | 20260505 06:33:29.282 | INFO | Perf report for the last 5.13962 seconds: | 20260505 06:33:29.282 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:33:29.282 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:29.282 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:29.282 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:29.282 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 06:33:29.282 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:29.282 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:29.282 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 06:33:29.282 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:29.282 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:29.282 | INFO | Uploaded 36GiB to GPU in 2.17073s (16.5843GiB/s) | 20260505 06:33:29.282 | INFO | Downloaded 34GiB to GPU in 2.04445s (16.6304GiB/s) | 20260505 06:33:34.422 | INFO | Analyzing chunk frames = [1088000,1088512) | 20260505 06:33:34.422 | INFO | MemInfo: { | 20260505 06:33:34.422 | INFO | "gpu" : " 17 GB", | 20260505 06:33:34.422 | INFO | "huge" : "", | 20260505 06:33:34.422 | INFO | "huge_shared" : " 27 GB", | 20260505 06:33:34.422 | INFO | "malloc" : " 14 GB", | 20260505 06:33:34.422 | INFO | "pinned" : " 29 GB" | 20260505 06:33:34.422 | INFO | } | 20260505 06:33:34.422 | INFO | Perf report for the last 5.1404 seconds: | 20260505 06:33:34.422 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:33:34.422 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:34.422 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:34.422 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:34.422 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 06:33:34.422 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:34.422 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:34.422 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 06:33:34.422 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:34.422 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:34.423 | INFO | Uploaded 36GiB to GPU in 2.15868s (16.6769GiB/s) | 20260505 06:33:34.423 | INFO | Downloaded 34GiB to GPU in 2.04137s (16.6555GiB/s) | 20260505 06:33:39.560 | INFO | Analyzing chunk frames = [1088512,1089024) | 20260505 06:33:39.560 | INFO | MemInfo: { | 20260505 06:33:39.560 | INFO | "gpu" : " 17 GB", | 20260505 06:33:39.560 | INFO | "huge" : "", | 20260505 06:33:39.560 | INFO | "huge_shared" : " 27 GB", | 20260505 06:33:39.560 | INFO | "malloc" : " 14 GB", | 20260505 06:33:39.560 | INFO | "pinned" : " 29 GB" | 20260505 06:33:39.560 | INFO | } | 20260505 06:33:39.560 | INFO | Perf report for the last 5.13795 seconds: | 20260505 06:33:39.560 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:33:39.560 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:39.560 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:39.560 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:39.560 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 06:33:39.560 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:39.560 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:39.560 | INFO | PRE_HQ : 32.4%, 1.0, 16.9ms, 0 | 20260505 06:33:39.560 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:39.560 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:39.560 | INFO | Uploaded 36GiB to GPU in 2.173s (16.567GiB/s) | 20260505 06:33:39.560 | INFO | Downloaded 34GiB to GPU in 2.1649s (15.7051GiB/s) | 20260505 06:33:44.699 | INFO | Analyzing chunk frames = [1089024,1089536) | 20260505 06:33:44.699 | INFO | MemInfo: { | 20260505 06:33:44.699 | INFO | "gpu" : " 17 GB", | 20260505 06:33:44.699 | INFO | "huge" : "", | 20260505 06:33:44.700 | INFO | "huge_shared" : " 27 GB", | 20260505 06:33:44.700 | INFO | "malloc" : " 14 GB", | 20260505 06:33:44.700 | INFO | "pinned" : " 29 GB" | 20260505 06:33:44.700 | INFO | } | 20260505 06:33:44.700 | INFO | Perf report for the last 5.13942 seconds: | 20260505 06:33:44.700 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:33:44.700 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:44.700 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:44.700 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:44.700 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 06:33:44.700 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:44.700 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:44.700 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260505 06:33:44.700 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:44.700 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:44.700 | INFO | Uploaded 36GiB to GPU in 2.12948s (16.9055GiB/s) | 20260505 06:33:44.700 | INFO | Downloaded 34GiB to GPU in 2.06765s (16.4438GiB/s) | 20260505 06:33:44.700 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 06:33:44.700 | INFO | Total mem usage: 15.3 GiB | 20260505 06:33:44.700 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 06:33:44.700 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.3%) | 20260505 06:33:44.700 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 06:33:44.700 | INFO | High Water Breakdown: | 20260505 06:33:44.700 | INFO | 60.3% : 8.69 GiB ( 92.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 06:33:44.700 | INFO | 39.6% : 5.71 GiB ( 85.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 06:33:44.700 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 06:33:44.700 | INFO | ------------------------------------------ | 20260505 06:33:44.700 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:33:44.700 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 06:33:44.700 | INFO | Total mem usage: 26.5 GiB | 20260505 06:33:44.700 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 06:33:44.700 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 06:33:44.700 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 06:33:44.700 | INFO | High Water Breakdown: | 20260505 06:33:44.700 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:33:44.700 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:33:44.700 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:33:44.700 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:33:44.700 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:33:44.700 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:33:44.700 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 06:33:44.700 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 06:33:44.700 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:33:44.700 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 06:33:44.700 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 06:33:44.700 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 06:33:44.700 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 06:33:44.700 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 06:33:44.700 | INFO | ------------------------------------------ | 20260505 06:33:44.700 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:33:44.700 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 06:33:44.700 | INFO | Total mem usage: 25.6 GiB | 20260505 06:33:44.700 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 06:33:44.700 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 06:33:44.700 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 06:33:44.700 | INFO | High Water Breakdown: | 20260505 06:33:44.700 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 06:33:44.700 | INFO | ------------------------------------------ | 20260505 06:33:44.700 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:33:44.700 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 06:33:44.700 | INFO | Total mem usage: 15.7 GiB | 20260505 06:33:44.700 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 06:33:44.700 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 06:33:44.700 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260505 06:33:44.700 | INFO | High Water Breakdown: | 20260505 06:33:44.700 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 06:33:44.700 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 06:33:44.700 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 06:33:44.700 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 06:33:44.700 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 06:33:44.700 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 06:33:44.700 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 06:33:44.700 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 06:33:44.700 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 06:33:44.700 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:33:44.700 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 06:33:44.700 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:33:44.700 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 06:33:44.700 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 06:33:44.701 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 06:33:44.701 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 06:33:44.701 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:33:44.701 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 06:33:44.701 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:33:44.701 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:33:44.701 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:33:44.701 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 06:33:44.701 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 06:33:44.701 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:33:44.701 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 06:33:44.701 | INFO | ------------------------------------------ | 20260505 06:33:44.701 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:33:44.701 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 06:33:44.701 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:33:44.701 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:33:44.701 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.54e+01, 1.36e+01 / 1.61e+00 | 20260505 06:33:44.701 | INFO | PerfReport: 0.28%, AggregateStats -- 9.39e-02 / 1.03e-01, 1.06e-02 / 7.13e-03 | 20260505 06:33:44.701 | INFO | PerfReport: 11.60%, Baseline -- 3.84e+00 / 4.45e+00, 6.41e-01 / 3.27e-01 | 20260505 06:33:44.701 | INFO | PerfReport: 3.31%, Histogram -- 1.10e+00 / 1.50e+00, 2.19e-01 / 1.25e-01 | 20260505 06:33:44.701 | INFO | PerfReport: 15.45%, DME -- 4.09e+01 / 4.48e+01, 4.33e+00 / 1.10e+00 | 20260505 06:33:44.701 | INFO | PerfReport: 56.05%, FrameLabeling -- 1.86e+01 / 1.91e+01, 2.04e+00 / 1.05e+00 | 20260505 06:33:44.701 | INFO | PerfReport: 7.70%, PulseAccumulating -- 2.55e+00 / 3.17e+00, 4.66e-01 / 2.96e-01 | 20260505 06:33:44.701 | INFO | PerfReport: 4.10%, Metrics -- 1.36e+00 / 1.71e+00, 3.89e-01 / 2.00e-01 | 20260505 06:33:44.701 | INFO | PerfReport: 1.49%, PulseSerializing -- 4.93e-01 / 5.67e-01, 9.79e-02 / 4.55e-02 | 20260505 06:33:44.701 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 06:33:44.701 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:33:44.701 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:33:44.701 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.24e+01 / 0.00e+00 | 20260505 06:33:44.701 | INFO | PerfReport: 21.99%, Upload -- 2.23e+01 / 0.00e+00, 7.73e+00 / 0.00e+00 | 20260505 06:33:44.701 | INFO | PerfReport: 21.51%, Download -- 2.18e+01 / 0.00e+00, 7.76e+00 / 0.00e+00 | 20260505 06:33:44.701 | INFO | PerfReport: 32.65%, Compute -- 3.31e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 06:33:44.701 | INFO | PerfReport: 1.50%, LockUpload -- 1.52e+00 / 0.00e+00, 6.92e+00 / 0.00e+00 | 20260505 06:33:44.701 | INFO | PerfReport: 0.76%, LockDownload -- 7.75e-01 / 0.00e+00, 2.24e+00 / 0.00e+00 | 20260505 06:33:44.701 | INFO | PerfReport: 21.58%, LockCompute -- 2.19e+01 / 0.00e+00, 1.67e+01 / 0.00e+00 | 20260505 06:33:44.701 | INFO | PerfReport: 0.01%, Misc -- 1.46e-02 / 0.00e+00, 1.97e-02 / 0.00e+00 | 20260505 06:33:44.825 | INFO | Starting a round of Baz IO | 20260505 06:33:48.094 | INFO | Populating RTMetrics report | 20260505 06:33:48.094 | INFO | RTMetrics report populated | 20260505 06:33:48.094 | INFO | Opening tmp RTMetrics json file | 20260505 06:33:48.111 | INFO | Serializing RTMetrics to json | 20260505 06:33:48.113 | INFO | Serialization done, swapping tmp file now... | 20260505 06:33:48.113 | INFO | RTMetrics done | 20260505 06:33:48.203 | INFO | Finished a round of Baz IO: Wrote 2.381 GiB in 3.378 seconds | 20260505 06:33:48.203 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.86% / 0.05% / 67.10% | 20260505 06:33:48.203 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.92355% / 5.96046e-05% / 7.15256e-05% | 20260505 06:33:48.203 | INFO | Baz IO overall: | 20260505 06:33:48.203 | INFO | BytesWritten : 566165351636 | 20260505 06:33:48.203 | INFO | NumEvents : 267811551902 | 20260505 06:33:49.839 | INFO | Analyzing chunk frames = [1089536,1090048) | 20260505 06:33:49.839 | INFO | MemInfo: { | 20260505 06:33:49.839 | INFO | "gpu" : " 17 GB", | 20260505 06:33:49.839 | INFO | "huge" : "", | 20260505 06:33:49.839 | INFO | "huge_shared" : " 27 GB", | 20260505 06:33:49.839 | INFO | "malloc" : " 14 GB", | 20260505 06:33:49.839 | INFO | "pinned" : " 29 GB" | 20260505 06:33:49.839 | INFO | } | 20260505 06:33:49.839 | INFO | Perf report for the last 5.13939 seconds: | 20260505 06:33:49.839 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:33:49.839 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:49.839 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:49.839 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:49.839 | INFO | ANALYSIS : 64.9%, 2.9, 99.0ms, 0 | 20260505 06:33:49.839 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:49.839 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260505 06:33:49.839 | INFO | PRE_HQ : 63.9%, 1.9, 64.0ms, 0 | 20260505 06:33:49.839 | INFO | BAZWRITER : 64.9%, 4.1, 137.4ms, 0 | 20260505 06:33:49.839 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 06:33:49.839 | INFO | Uploaded 36GiB to GPU in 2.15671s (16.6921GiB/s) | 20260505 06:33:49.839 | INFO | Downloaded 38GiB to GPU in 2.30524s (16.4842GiB/s) | 20260505 06:33:54.978 | INFO | Analyzing chunk frames = [1090048,1090560) | 20260505 06:33:54.978 | INFO | MemInfo: { | 20260505 06:33:54.978 | INFO | "gpu" : " 17 GB", | 20260505 06:33:54.978 | INFO | "huge" : "", | 20260505 06:33:54.978 | INFO | "huge_shared" : " 27 GB", | 20260505 06:33:54.978 | INFO | "malloc" : " 14 GB", | 20260505 06:33:54.978 | INFO | "pinned" : " 29 GB" | 20260505 06:33:54.978 | INFO | } | 20260505 06:33:54.978 | INFO | Perf report for the last 5.13935 seconds: | 20260505 06:33:54.978 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:33:54.978 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:54.978 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:54.978 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:54.978 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 06:33:54.978 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:54.978 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:54.978 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 06:33:54.978 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:54.978 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:33:54.979 | INFO | Uploaded 36GiB to GPU in 2.10659s (17.0892GiB/s) | 20260505 06:33:54.979 | INFO | Downloaded 34GiB to GPU in 2.0393s (16.6724GiB/s) | 20260505 06:34:00.118 | INFO | Analyzing chunk frames = [1090560,1091072) | 20260505 06:34:00.118 | INFO | MemInfo: { | 20260505 06:34:00.118 | INFO | "gpu" : " 17 GB", | 20260505 06:34:00.118 | INFO | "huge" : "", | 20260505 06:34:00.118 | INFO | "huge_shared" : " 27 GB", | 20260505 06:34:00.118 | INFO | "malloc" : " 14 GB", | 20260505 06:34:00.118 | INFO | "pinned" : " 29 GB" | 20260505 06:34:00.118 | INFO | } | 20260505 06:34:00.118 | INFO | Perf report for the last 5.13937 seconds: | 20260505 06:34:00.118 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:34:00.118 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:00.118 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:00.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:00.118 | INFO | ANALYSIS : 63.8%, 3.0, 99.3ms, 0 | 20260505 06:34:00.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:00.118 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:00.118 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 06:34:00.118 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:00.118 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:00.118 | INFO | Uploaded 36GiB to GPU in 2.13705s (16.8457GiB/s) | 20260505 06:34:00.118 | INFO | Downloaded 34GiB to GPU in 2.03612s (16.6984GiB/s) | 20260505 06:34:05.257 | INFO | Analyzing chunk frames = [1091072,1091584) | 20260505 06:34:05.257 | INFO | MemInfo: { | 20260505 06:34:05.257 | INFO | "gpu" : " 17 GB", | 20260505 06:34:05.257 | INFO | "huge" : "", | 20260505 06:34:05.257 | INFO | "huge_shared" : " 27 GB", | 20260505 06:34:05.257 | INFO | "malloc" : " 14 GB", | 20260505 06:34:05.257 | INFO | "pinned" : " 29 GB" | 20260505 06:34:05.257 | INFO | } | 20260505 06:34:05.257 | INFO | Perf report for the last 5.13925 seconds: | 20260505 06:34:05.257 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:34:05.257 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:05.257 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:05.257 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:05.257 | INFO | ANALYSIS : 63.8%, 3.0, 99.1ms, 0 | 20260505 06:34:05.257 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:05.257 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:05.257 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 06:34:05.257 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:05.257 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:05.257 | INFO | Uploaded 36GiB to GPU in 2.16026s (16.6646GiB/s) | 20260505 06:34:05.257 | INFO | Downloaded 34GiB to GPU in 2.0347s (16.7101GiB/s) | 20260505 06:34:10.396 | INFO | Analyzing chunk frames = [1091584,1092096) | 20260505 06:34:10.396 | INFO | MemInfo: { | 20260505 06:34:10.396 | INFO | "gpu" : " 17 GB", | 20260505 06:34:10.396 | INFO | "huge" : "", | 20260505 06:34:10.396 | INFO | "huge_shared" : " 27 GB", | 20260505 06:34:10.396 | INFO | "malloc" : " 14 GB", | 20260505 06:34:10.396 | INFO | "pinned" : " 29 GB" | 20260505 06:34:10.396 | INFO | } | 20260505 06:34:10.396 | INFO | Perf report for the last 5.13936 seconds: | 20260505 06:34:10.396 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:34:10.396 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:10.396 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:10.396 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:10.396 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 06:34:10.396 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:10.396 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:10.396 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 06:34:10.396 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:10.396 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:10.397 | INFO | Uploaded 36GiB to GPU in 2.16407s (16.6353GiB/s) | 20260505 06:34:10.397 | INFO | Downloaded 34GiB to GPU in 2.05276s (16.5631GiB/s) | 20260505 06:34:15.537 | INFO | Analyzing chunk frames = [1092096,1092608) | 20260505 06:34:15.537 | INFO | MemInfo: { | 20260505 06:34:15.537 | INFO | "gpu" : " 17 GB", | 20260505 06:34:15.537 | INFO | "huge" : "", | 20260505 06:34:15.537 | INFO | "huge_shared" : " 27 GB", | 20260505 06:34:15.537 | INFO | "malloc" : " 14 GB", | 20260505 06:34:15.537 | INFO | "pinned" : " 29 GB" | 20260505 06:34:15.537 | INFO | } | 20260505 06:34:15.537 | INFO | Perf report for the last 5.14075 seconds: | 20260505 06:34:15.537 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:34:15.537 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:15.537 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:15.537 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:15.537 | INFO | ANALYSIS : 63.9%, 3.0, 99.9ms, 0 | 20260505 06:34:15.537 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:15.537 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:15.537 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 06:34:15.537 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:15.537 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:15.537 | INFO | Uploaded 36GiB to GPU in 2.15969s (16.6691GiB/s) | 20260505 06:34:15.537 | INFO | Downloaded 34GiB to GPU in 2.05101s (16.5772GiB/s) | 20260505 06:34:20.679 | INFO | Analyzing chunk frames = [1092608,1093120) | 20260505 06:34:20.679 | INFO | MemInfo: { | 20260505 06:34:20.679 | INFO | "gpu" : " 17 GB", | 20260505 06:34:20.679 | INFO | "huge" : "", | 20260505 06:34:20.679 | INFO | "huge_shared" : " 27 GB", | 20260505 06:34:20.679 | INFO | "malloc" : " 14 GB", | 20260505 06:34:20.679 | INFO | "pinned" : " 29 GB" | 20260505 06:34:20.679 | INFO | } | 20260505 06:34:20.679 | INFO | Perf report for the last 5.14213 seconds: | 20260505 06:34:20.679 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:34:20.679 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:20.679 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:20.679 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:20.679 | INFO | ANALYSIS : 64.1%, 3.0, 100.2ms, 0 | 20260505 06:34:20.679 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:20.679 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:20.679 | INFO | PRE_HQ : 32.2%, 1.0, 16.8ms, 0 | 20260505 06:34:20.679 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:20.679 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:20.680 | INFO | Uploaded 36GiB to GPU in 2.18957s (16.4416GiB/s) | 20260505 06:34:20.680 | INFO | Downloaded 34GiB to GPU in 2.16867s (15.6778GiB/s) | 20260505 06:34:25.819 | INFO | Analyzing chunk frames = [1093120,1093632) | 20260505 06:34:25.819 | INFO | MemInfo: { | 20260505 06:34:25.819 | INFO | "gpu" : " 17 GB", | 20260505 06:34:25.819 | INFO | "huge" : "", | 20260505 06:34:25.819 | INFO | "huge_shared" : " 27 GB", | 20260505 06:34:25.819 | INFO | "malloc" : " 14 GB", | 20260505 06:34:25.819 | INFO | "pinned" : " 29 GB" | 20260505 06:34:25.819 | INFO | } | 20260505 06:34:25.819 | INFO | Perf report for the last 5.13971 seconds: | 20260505 06:34:25.819 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:34:25.819 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:25.819 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:25.819 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:25.819 | INFO | ANALYSIS : 64.3%, 3.0, 100.5ms, 0 | 20260505 06:34:25.819 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:25.819 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:25.819 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 06:34:25.819 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:25.819 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:25.820 | INFO | Uploaded 36GiB to GPU in 2.16092s (16.6596GiB/s) | 20260505 06:34:25.820 | INFO | Downloaded 34GiB to GPU in 2.05193s (16.5698GiB/s) | 20260505 06:34:25.932 | INFO | Starting a round of Baz IO | 20260505 06:34:29.206 | INFO | Populating RTMetrics report | 20260505 06:34:29.206 | INFO | RTMetrics report populated | 20260505 06:34:29.206 | INFO | Opening tmp RTMetrics json file | 20260505 06:34:29.240 | INFO | Serializing RTMetrics to json | 20260505 06:34:29.263 | INFO | Serialization done, swapping tmp file now... | 20260505 06:34:29.288 | INFO | RTMetrics done | 20260505 06:34:29.333 | INFO | Finished a round of Baz IO: Wrote 2.38 GiB in 3.401 seconds | 20260505 06:34:29.333 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.81% / 0.05% / 67.14% | 20260505 06:34:29.333 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.89571% / 7.15256e-05% / 6.75519e-05% | 20260505 06:34:29.333 | INFO | Baz IO overall: | 20260505 06:34:29.333 | INFO | BytesWritten : 568721131329 | 20260505 06:34:29.333 | INFO | NumEvents : 269033786932 | 20260505 06:34:30.954 | INFO | Analyzing chunk frames = [1093632,1094144) | 20260505 06:34:30.954 | INFO | MemInfo: { | 20260505 06:34:30.954 | INFO | "gpu" : " 17 GB", | 20260505 06:34:30.954 | INFO | "huge" : "", | 20260505 06:34:30.954 | INFO | "huge_shared" : " 27 GB", | 20260505 06:34:30.954 | INFO | "malloc" : " 14 GB", | 20260505 06:34:30.954 | INFO | "pinned" : " 29 GB" | 20260505 06:34:30.954 | INFO | } | 20260505 06:34:30.954 | INFO | Perf report for the last 5.13473 seconds: | 20260505 06:34:30.954 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:34:30.954 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:30.954 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:30.954 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:30.954 | INFO | ANALYSIS : 64.8%, 2.9, 98.6ms, 0 | 20260505 06:34:30.954 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:30.954 | INFO | RT_METRICS : 48.2%, 1.0, 25.3ms, 0 | 20260505 06:34:30.954 | INFO | PRE_HQ : 63.8%, 1.9, 63.9ms, 0 | 20260505 06:34:30.954 | INFO | BAZWRITER : 65.6%, 4.1, 139.2ms, 0 | 20260505 06:34:30.954 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260505 06:34:30.954 | INFO | Uploaded 36GiB to GPU in 2.20214s (16.3478GiB/s) | 20260505 06:34:30.954 | INFO | Downloaded 38GiB to GPU in 2.33301s (16.288GiB/s) | 20260505 06:34:36.094 | INFO | Analyzing chunk frames = [1094144,1094656) | 20260505 06:34:36.094 | INFO | MemInfo: { | 20260505 06:34:36.094 | INFO | "gpu" : " 17 GB", | 20260505 06:34:36.094 | INFO | "huge" : "", | 20260505 06:34:36.094 | INFO | "huge_shared" : " 27 GB", | 20260505 06:34:36.094 | INFO | "malloc" : " 14 GB", | 20260505 06:34:36.094 | INFO | "pinned" : " 29 GB" | 20260505 06:34:36.094 | INFO | } | 20260505 06:34:36.094 | INFO | Perf report for the last 5.14007 seconds: | 20260505 06:34:36.094 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:34:36.094 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:36.094 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:36.094 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:36.094 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 06:34:36.094 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:36.094 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:36.094 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 06:34:36.094 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:36.094 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:36.094 | INFO | Uploaded 36GiB to GPU in 2.11488s (17.0222GiB/s) | 20260505 06:34:36.094 | INFO | Downloaded 34GiB to GPU in 2.04034s (16.6639GiB/s) | 20260505 06:34:41.232 | INFO | Analyzing chunk frames = [1094656,1095168) | 20260505 06:34:41.232 | INFO | MemInfo: { | 20260505 06:34:41.232 | INFO | "gpu" : " 17 GB", | 20260505 06:34:41.232 | INFO | "huge" : "", | 20260505 06:34:41.232 | INFO | "huge_shared" : " 27 GB", | 20260505 06:34:41.232 | INFO | "malloc" : " 14 GB", | 20260505 06:34:41.232 | INFO | "pinned" : " 29 GB" | 20260505 06:34:41.232 | INFO | } | 20260505 06:34:41.232 | INFO | Perf report for the last 5.13853 seconds: | 20260505 06:34:41.232 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:34:41.232 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:41.232 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:41.232 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:41.232 | INFO | ANALYSIS : 63.7%, 3.0, 99.2ms, 0 | 20260505 06:34:41.232 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:41.232 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:41.232 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 06:34:41.232 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:41.232 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:41.233 | INFO | Uploaded 36GiB to GPU in 2.13022s (16.8997GiB/s) | 20260505 06:34:41.233 | INFO | Downloaded 34GiB to GPU in 2.04547s (16.6221GiB/s) | 20260505 06:34:46.372 | INFO | Analyzing chunk frames = [1095168,1095680) | 20260505 06:34:46.372 | INFO | MemInfo: { | 20260505 06:34:46.372 | INFO | "gpu" : " 17 GB", | 20260505 06:34:46.372 | INFO | "huge" : "", | 20260505 06:34:46.372 | INFO | "huge_shared" : " 27 GB", | 20260505 06:34:46.372 | INFO | "malloc" : " 14 GB", | 20260505 06:34:46.372 | INFO | "pinned" : " 29 GB" | 20260505 06:34:46.372 | INFO | } | 20260505 06:34:46.372 | INFO | Perf report for the last 5.13999 seconds: | 20260505 06:34:46.372 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:34:46.372 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:46.372 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:46.372 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:46.372 | INFO | ANALYSIS : 63.9%, 3.0, 99.3ms, 0 | 20260505 06:34:46.372 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:46.372 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:46.372 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 06:34:46.372 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:46.372 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:46.373 | INFO | Uploaded 36GiB to GPU in 2.1705s (16.586GiB/s) | 20260505 06:34:46.373 | INFO | Downloaded 34GiB to GPU in 2.04634s (16.615GiB/s) | 20260505 06:34:51.511 | INFO | Analyzing chunk frames = [1095680,1096192) | 20260505 06:34:51.511 | INFO | MemInfo: { | 20260505 06:34:51.511 | INFO | "gpu" : " 17 GB", | 20260505 06:34:51.511 | INFO | "huge" : "", | 20260505 06:34:51.511 | INFO | "huge_shared" : " 27 GB", | 20260505 06:34:51.511 | INFO | "malloc" : " 14 GB", | 20260505 06:34:51.511 | INFO | "pinned" : " 29 GB" | 20260505 06:34:51.511 | INFO | } | 20260505 06:34:51.511 | INFO | Perf report for the last 5.13892 seconds: | 20260505 06:34:51.511 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:34:51.511 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:51.511 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:51.511 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:51.511 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 06:34:51.511 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:51.511 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:51.511 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 06:34:51.511 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:51.511 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:51.512 | INFO | Uploaded 36GiB to GPU in 2.17244s (16.5712GiB/s) | 20260505 06:34:51.512 | INFO | Downloaded 34GiB to GPU in 2.04166s (16.6531GiB/s) | 20260505 06:34:56.652 | INFO | Analyzing chunk frames = [1096192,1096704) | 20260505 06:34:56.652 | INFO | MemInfo: { | 20260505 06:34:56.652 | INFO | "gpu" : " 17 GB", | 20260505 06:34:56.652 | INFO | "huge" : "", | 20260505 06:34:56.652 | INFO | "huge_shared" : " 27 GB", | 20260505 06:34:56.652 | INFO | "malloc" : " 14 GB", | 20260505 06:34:56.652 | INFO | "pinned" : " 29 GB" | 20260505 06:34:56.652 | INFO | } | 20260505 06:34:56.652 | INFO | Perf report for the last 5.14068 seconds: | 20260505 06:34:56.652 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:34:56.652 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:56.652 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:56.652 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:56.652 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 06:34:56.652 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:56.652 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:56.652 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 06:34:56.652 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:56.652 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:34:56.652 | INFO | Uploaded 36GiB to GPU in 2.14504s (16.7829GiB/s) | 20260505 06:34:56.652 | INFO | Downloaded 34GiB to GPU in 2.03365s (16.7187GiB/s) | 20260505 06:35:01.790 | INFO | Analyzing chunk frames = [1096704,1097216) | 20260505 06:35:01.790 | INFO | MemInfo: { | 20260505 06:35:01.790 | INFO | "gpu" : " 17 GB", | 20260505 06:35:01.790 | INFO | "huge" : "", | 20260505 06:35:01.790 | INFO | "huge_shared" : " 27 GB", | 20260505 06:35:01.790 | INFO | "malloc" : " 14 GB", | 20260505 06:35:01.790 | INFO | "pinned" : " 29 GB" | 20260505 06:35:01.790 | INFO | } | 20260505 06:35:01.790 | INFO | Perf report for the last 5.1379 seconds: | 20260505 06:35:01.790 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:35:01.790 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:01.790 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:01.790 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:01.790 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 06:35:01.790 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:01.790 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:01.790 | INFO | PRE_HQ : 32.4%, 1.0, 16.9ms, 0 | 20260505 06:35:01.790 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:01.790 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:01.790 | INFO | Uploaded 36GiB to GPU in 2.1766s (16.5396GiB/s) | 20260505 06:35:01.790 | INFO | Downloaded 34GiB to GPU in 2.15522s (15.7757GiB/s) | 20260505 06:35:06.929 | INFO | Analyzing chunk frames = [1097216,1097728) | 20260505 06:35:06.929 | INFO | MemInfo: { | 20260505 06:35:06.929 | INFO | "gpu" : " 17 GB", | 20260505 06:35:06.929 | INFO | "huge" : "", | 20260505 06:35:06.929 | INFO | "huge_shared" : " 27 GB", | 20260505 06:35:06.929 | INFO | "malloc" : " 14 GB", | 20260505 06:35:06.929 | INFO | "pinned" : " 29 GB" | 20260505 06:35:06.929 | INFO | } | 20260505 06:35:06.929 | INFO | Perf report for the last 5.13937 seconds: | 20260505 06:35:06.929 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:35:06.929 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:06.929 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:06.929 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:06.929 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 06:35:06.929 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:06.929 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:06.929 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 06:35:06.929 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:06.929 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:06.930 | INFO | Uploaded 36GiB to GPU in 2.12518s (16.9397GiB/s) | 20260505 06:35:06.930 | INFO | Downloaded 34GiB to GPU in 2.05583s (16.5383GiB/s) | 20260505 06:35:06.930 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 06:35:06.930 | INFO | Total mem usage: 15.3 GiB | 20260505 06:35:06.930 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 06:35:06.930 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.59%) | 20260505 06:35:06.930 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 06:35:06.930 | INFO | High Water Breakdown: | 20260505 06:35:06.930 | INFO | 60.3% : 8.67 GiB ( 92.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 06:35:06.930 | INFO | 39.5% : 5.68 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 06:35:06.930 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 06:35:06.930 | INFO | ------------------------------------------ | 20260505 06:35:06.930 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:35:06.930 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 06:35:06.930 | INFO | Total mem usage: 26.5 GiB | 20260505 06:35:06.930 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 06:35:06.930 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 06:35:06.930 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 06:35:06.930 | INFO | High Water Breakdown: | 20260505 06:35:06.930 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:35:06.930 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:35:06.930 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:35:06.930 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:35:06.930 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:35:06.930 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:35:06.930 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 06:35:06.930 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 06:35:06.930 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:35:06.930 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 06:35:06.930 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 06:35:06.930 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 06:35:06.930 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 06:35:06.930 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 06:35:06.930 | INFO | ------------------------------------------ | 20260505 06:35:06.930 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:35:06.930 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 06:35:06.930 | INFO | Total mem usage: 25.6 GiB | 20260505 06:35:06.930 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 06:35:06.930 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 06:35:06.930 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 06:35:06.930 | INFO | High Water Breakdown: | 20260505 06:35:06.930 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 06:35:06.930 | INFO | ------------------------------------------ | 20260505 06:35:06.930 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:35:06.930 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 06:35:06.930 | INFO | Total mem usage: 15.7 GiB | 20260505 06:35:06.930 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 06:35:06.930 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 06:35:06.930 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260505 06:35:06.930 | INFO | High Water Breakdown: | 20260505 06:35:06.930 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 06:35:06.930 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 06:35:06.930 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 06:35:06.930 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 06:35:06.930 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 06:35:06.930 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 06:35:06.930 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 06:35:06.930 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 06:35:06.931 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 06:35:06.931 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:35:06.931 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 06:35:06.931 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:35:06.931 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 06:35:06.931 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 06:35:06.931 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 06:35:06.931 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 06:35:06.931 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:35:06.931 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 06:35:06.931 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:35:06.931 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:35:06.931 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:35:06.931 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 06:35:06.931 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 06:35:06.931 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:35:06.931 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 06:35:06.931 | INFO | ------------------------------------------ | 20260505 06:35:06.931 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:35:06.931 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 06:35:06.931 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:35:06.931 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:35:06.931 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.55e+01, 1.39e+01 / 1.59e+00 | 20260505 06:35:06.931 | INFO | PerfReport: 0.28%, AggregateStats -- 9.39e-02 / 1.04e-01, 1.08e-02 / 4.34e-03 | 20260505 06:35:06.931 | INFO | PerfReport: 11.59%, Baseline -- 3.84e+00 / 4.46e+00, 6.39e-01 / 3.01e-01 | 20260505 06:35:06.931 | INFO | PerfReport: 3.31%, Histogram -- 1.10e+00 / 1.51e+00, 2.19e-01 / 9.69e-02 | 20260505 06:35:06.931 | INFO | PerfReport: 15.47%, DME -- 4.10e+01 / 4.49e+01, 4.40e+00 / 1.13e+00 | 20260505 06:35:06.931 | INFO | PerfReport: 56.11%, FrameLabeling -- 1.86e+01 / 1.92e+01, 2.08e+00 / 1.12e+00 | 20260505 06:35:06.931 | INFO | PerfReport: 7.66%, PulseAccumulating -- 2.54e+00 / 3.13e+00, 4.56e-01 / 2.91e-01 | 20260505 06:35:06.931 | INFO | PerfReport: 4.08%, Metrics -- 1.35e+00 / 1.71e+00, 3.84e-01 / 1.94e-01 | 20260505 06:35:06.931 | INFO | PerfReport: 1.48%, PulseSerializing -- 4.91e-01 / 5.63e-01, 9.66e-02 / 3.86e-02 | 20260505 06:35:06.931 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 06:35:06.931 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:35:06.931 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:35:06.931 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.25e+01 / 0.00e+00 | 20260505 06:35:06.931 | INFO | PerfReport: 22.09%, Upload -- 2.24e+01 / 0.00e+00, 7.93e+00 / 0.00e+00 | 20260505 06:35:06.931 | INFO | PerfReport: 21.46%, Download -- 2.18e+01 / 0.00e+00, 7.78e+00 / 0.00e+00 | 20260505 06:35:06.931 | INFO | PerfReport: 32.59%, Compute -- 3.31e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 06:35:06.931 | INFO | PerfReport: 1.45%, LockUpload -- 1.48e+00 / 0.00e+00, 6.89e+00 / 0.00e+00 | 20260505 06:35:06.931 | INFO | PerfReport: 0.75%, LockDownload -- 7.57e-01 / 0.00e+00, 2.06e+00 / 0.00e+00 | 20260505 06:35:06.931 | INFO | PerfReport: 21.64%, LockCompute -- 2.20e+01 / 0.00e+00, 1.68e+01 / 0.00e+00 | 20260505 06:35:06.931 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 1.88e-02 / 0.00e+00 | 20260505 06:35:07.053 | INFO | Starting a round of Baz IO | 20260505 06:35:10.294 | INFO | Populating RTMetrics report | 20260505 06:35:10.295 | INFO | RTMetrics report populated | 20260505 06:35:10.295 | INFO | Opening tmp RTMetrics json file | 20260505 06:35:10.330 | INFO | Serializing RTMetrics to json | 20260505 06:35:10.365 | INFO | Serialization done, swapping tmp file now... | 20260505 06:35:10.391 | INFO | RTMetrics done | 20260505 06:35:10.455 | INFO | Finished a round of Baz IO: Wrote 2.382 GiB in 3.401 seconds | 20260505 06:35:10.455 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.76% / 0.05% / 67.19% | 20260505 06:35:10.455 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.88221% / 4.37101e-05% / 7.54992e-05% | 20260505 06:35:10.455 | INFO | Baz IO overall: | 20260505 06:35:10.455 | INFO | BytesWritten : 571279207767 | 20260505 06:35:10.455 | INFO | NumEvents : 270257459270 | 20260505 06:35:12.068 | INFO | Analyzing chunk frames = [1097728,1098240) | 20260505 06:35:12.068 | INFO | MemInfo: { | 20260505 06:35:12.068 | INFO | "gpu" : " 17 GB", | 20260505 06:35:12.068 | INFO | "huge" : "", | 20260505 06:35:12.068 | INFO | "huge_shared" : " 27 GB", | 20260505 06:35:12.068 | INFO | "malloc" : " 14 GB", | 20260505 06:35:12.068 | INFO | "pinned" : " 29 GB" | 20260505 06:35:12.068 | INFO | } | 20260505 06:35:12.068 | INFO | Perf report for the last 5.13914 seconds: | 20260505 06:35:12.068 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:35:12.068 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:12.068 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:12.068 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:12.068 | INFO | ANALYSIS : 64.3%, 2.9, 97.7ms, 0 | 20260505 06:35:12.069 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:12.069 | INFO | RT_METRICS : 47.9%, 1.0, 25.1ms, 0 | 20260505 06:35:12.069 | INFO | PRE_HQ : 63.2%, 1.9, 63.7ms, 0 | 20260505 06:35:12.069 | INFO | BAZWRITER : 65.5%, 4.2, 140.9ms, 0 | 20260505 06:35:12.069 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 06:35:12.069 | INFO | Uploaded 36GiB to GPU in 2.12037s (16.9782GiB/s) | 20260505 06:35:12.069 | INFO | Downloaded 38GiB to GPU in 2.31584s (16.4087GiB/s) | 20260505 06:35:17.208 | INFO | Analyzing chunk frames = [1098240,1098752) | 20260505 06:35:17.208 | INFO | MemInfo: { | 20260505 06:35:17.208 | INFO | "gpu" : " 17 GB", | 20260505 06:35:17.208 | INFO | "huge" : "", | 20260505 06:35:17.208 | INFO | "huge_shared" : " 27 GB", | 20260505 06:35:17.208 | INFO | "malloc" : " 14 GB", | 20260505 06:35:17.208 | INFO | "pinned" : " 29 GB" | 20260505 06:35:17.208 | INFO | } | 20260505 06:35:17.208 | INFO | Perf report for the last 5.13958 seconds: | 20260505 06:35:17.208 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:35:17.208 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:17.208 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:17.208 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:17.208 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 06:35:17.208 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:17.208 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:17.208 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 06:35:17.208 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:17.208 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:17.208 | INFO | Uploaded 36GiB to GPU in 2.11803s (16.9969GiB/s) | 20260505 06:35:17.208 | INFO | Downloaded 34GiB to GPU in 2.03819s (16.6815GiB/s) | 20260505 06:35:22.347 | INFO | Analyzing chunk frames = [1098752,1099264) | 20260505 06:35:22.347 | INFO | MemInfo: { | 20260505 06:35:22.347 | INFO | "gpu" : " 17 GB", | 20260505 06:35:22.347 | INFO | "huge" : "", | 20260505 06:35:22.347 | INFO | "huge_shared" : " 27 GB", | 20260505 06:35:22.347 | INFO | "malloc" : " 14 GB", | 20260505 06:35:22.347 | INFO | "pinned" : " 29 GB" | 20260505 06:35:22.347 | INFO | } | 20260505 06:35:22.347 | INFO | Perf report for the last 5.13939 seconds: | 20260505 06:35:22.347 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:35:22.347 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:22.347 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:22.347 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:22.347 | INFO | ANALYSIS : 63.7%, 3.0, 99.2ms, 0 | 20260505 06:35:22.347 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:22.347 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:22.348 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 06:35:22.348 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:22.348 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:22.348 | INFO | Uploaded 36GiB to GPU in 2.12516s (16.9399GiB/s) | 20260505 06:35:22.348 | INFO | Downloaded 34GiB to GPU in 2.04696s (16.61GiB/s) | 20260505 06:35:27.487 | INFO | Analyzing chunk frames = [1099264,1099776) | 20260505 06:35:27.487 | INFO | MemInfo: { | 20260505 06:35:27.487 | INFO | "gpu" : " 17 GB", | 20260505 06:35:27.487 | INFO | "huge" : "", | 20260505 06:35:27.487 | INFO | "huge_shared" : " 27 GB", | 20260505 06:35:27.487 | INFO | "malloc" : " 14 GB", | 20260505 06:35:27.487 | INFO | "pinned" : " 29 GB" | 20260505 06:35:27.487 | INFO | } | 20260505 06:35:27.487 | INFO | Perf report for the last 5.13924 seconds: | 20260505 06:35:27.487 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:35:27.487 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:27.487 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:27.487 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:27.487 | INFO | ANALYSIS : 63.8%, 3.0, 99.2ms, 0 | 20260505 06:35:27.487 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:27.487 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:27.487 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 06:35:27.487 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:27.487 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:27.487 | INFO | Uploaded 36GiB to GPU in 2.16062s (16.6619GiB/s) | 20260505 06:35:27.487 | INFO | Downloaded 34GiB to GPU in 2.03116s (16.7392GiB/s) | 20260505 06:35:32.626 | INFO | Analyzing chunk frames = [1099776,1100288) | 20260505 06:35:32.626 | INFO | MemInfo: { | 20260505 06:35:32.626 | INFO | "gpu" : " 17 GB", | 20260505 06:35:32.626 | INFO | "huge" : "", | 20260505 06:35:32.626 | INFO | "huge_shared" : " 27 GB", | 20260505 06:35:32.626 | INFO | "malloc" : " 14 GB", | 20260505 06:35:32.626 | INFO | "pinned" : " 29 GB" | 20260505 06:35:32.626 | INFO | } | 20260505 06:35:32.626 | INFO | Perf report for the last 5.1391 seconds: | 20260505 06:35:32.626 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:35:32.626 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:32.626 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:32.626 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:32.626 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 06:35:32.626 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:32.626 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:32.626 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 06:35:32.626 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:32.626 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:32.626 | INFO | Uploaded 36GiB to GPU in 2.17469s (16.5541GiB/s) | 20260505 06:35:32.626 | INFO | Downloaded 34GiB to GPU in 2.04567s (16.6205GiB/s) | 20260505 06:35:37.767 | INFO | Analyzing chunk frames = [1100288,1100800) | 20260505 06:35:37.767 | INFO | MemInfo: { | 20260505 06:35:37.767 | INFO | "gpu" : " 17 GB", | 20260505 06:35:37.767 | INFO | "huge" : "", | 20260505 06:35:37.767 | INFO | "huge_shared" : " 27 GB", | 20260505 06:35:37.767 | INFO | "malloc" : " 14 GB", | 20260505 06:35:37.767 | INFO | "pinned" : " 29 GB" | 20260505 06:35:37.767 | INFO | } | 20260505 06:35:37.767 | INFO | Perf report for the last 5.14097 seconds: | 20260505 06:35:37.767 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:35:37.767 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:37.767 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:37.767 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:37.767 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 06:35:37.767 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:37.767 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:37.767 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 06:35:37.767 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:37.767 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:37.767 | INFO | Uploaded 36GiB to GPU in 2.15485s (16.7065GiB/s) | 20260505 06:35:37.767 | INFO | Downloaded 34GiB to GPU in 2.0292s (16.7553GiB/s) | 20260505 06:35:42.904 | INFO | Analyzing chunk frames = [1100800,1101312) | 20260505 06:35:42.904 | INFO | MemInfo: { | 20260505 06:35:42.904 | INFO | "gpu" : " 17 GB", | 20260505 06:35:42.904 | INFO | "huge" : "", | 20260505 06:35:42.904 | INFO | "huge_shared" : " 27 GB", | 20260505 06:35:42.904 | INFO | "malloc" : " 14 GB", | 20260505 06:35:42.904 | INFO | "pinned" : " 29 GB" | 20260505 06:35:42.904 | INFO | } | 20260505 06:35:42.904 | INFO | Perf report for the last 5.1377 seconds: | 20260505 06:35:42.905 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:35:42.905 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:42.905 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:42.905 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:42.905 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 06:35:42.905 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:42.905 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:42.905 | INFO | PRE_HQ : 32.0%, 1.0, 16.7ms, 0 | 20260505 06:35:42.905 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:42.905 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:42.905 | INFO | Uploaded 36GiB to GPU in 2.17207s (16.5741GiB/s) | 20260505 06:35:42.905 | INFO | Downloaded 34GiB to GPU in 2.16376s (15.7134GiB/s) | 20260505 06:35:48.044 | INFO | Analyzing chunk frames = [1101312,1101824) | 20260505 06:35:48.044 | INFO | MemInfo: { | 20260505 06:35:48.044 | INFO | "gpu" : " 17 GB", | 20260505 06:35:48.044 | INFO | "huge" : "", | 20260505 06:35:48.044 | INFO | "huge_shared" : " 27 GB", | 20260505 06:35:48.045 | INFO | "malloc" : " 14 GB", | 20260505 06:35:48.045 | INFO | "pinned" : " 29 GB" | 20260505 06:35:48.045 | INFO | } | 20260505 06:35:48.045 | INFO | Perf report for the last 5.14005 seconds: | 20260505 06:35:48.045 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:35:48.045 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:48.045 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:48.045 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:48.045 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 06:35:48.045 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:48.045 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:48.045 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 06:35:48.045 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:48.045 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:48.045 | INFO | Uploaded 36GiB to GPU in 2.11978s (16.9829GiB/s) | 20260505 06:35:48.045 | INFO | Downloaded 34GiB to GPU in 2.06072s (16.4991GiB/s) | 20260505 06:35:48.167 | INFO | Starting a round of Baz IO | 20260505 06:35:51.401 | INFO | Populating RTMetrics report | 20260505 06:35:51.402 | INFO | RTMetrics report populated | 20260505 06:35:51.402 | INFO | Opening tmp RTMetrics json file | 20260505 06:35:51.410 | INFO | Serializing RTMetrics to json | 20260505 06:35:51.430 | INFO | Serialization done, swapping tmp file now... | 20260505 06:35:51.441 | INFO | RTMetrics done | 20260505 06:35:51.525 | INFO | Finished a round of Baz IO: Wrote 2.383 GiB in 3.358 seconds | 20260505 06:35:51.525 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.72% / 0.05% / 67.24% | 20260505 06:35:51.525 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.86329% / 4.37101e-05% / 5.5631e-05% | 20260505 06:35:51.525 | INFO | Baz IO overall: | 20260505 06:35:51.525 | INFO | BytesWritten : 573838305044 | 20260505 06:35:51.525 | INFO | NumEvents : 271481596572 | 20260505 06:35:53.183 | INFO | Analyzing chunk frames = [1101824,1102336) | 20260505 06:35:53.183 | INFO | MemInfo: { | 20260505 06:35:53.183 | INFO | "gpu" : " 17 GB", | 20260505 06:35:53.183 | INFO | "huge" : "", | 20260505 06:35:53.183 | INFO | "huge_shared" : " 27 GB", | 20260505 06:35:53.183 | INFO | "malloc" : " 14 GB", | 20260505 06:35:53.183 | INFO | "pinned" : " 29 GB" | 20260505 06:35:53.183 | INFO | } | 20260505 06:35:53.183 | INFO | Perf report for the last 5.13878 seconds: | 20260505 06:35:53.183 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:35:53.183 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:53.183 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:53.183 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:53.183 | INFO | ANALYSIS : 64.2%, 2.9, 97.6ms, 0 | 20260505 06:35:53.183 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:53.183 | INFO | RT_METRICS : 46.7%, 1.0, 24.5ms, 0 | 20260505 06:35:53.183 | INFO | PRE_HQ : 63.2%, 2.0, 64.0ms, 0 | 20260505 06:35:53.183 | INFO | BAZWRITER : 64.6%, 4.2, 138.6ms, 0 | 20260505 06:35:53.184 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 06:35:53.184 | INFO | Uploaded 36GiB to GPU in 2.10232s (17.124GiB/s) | 20260505 06:35:53.184 | INFO | Downloaded 38GiB to GPU in 2.31017s (16.449GiB/s) | 20260505 06:35:58.323 | INFO | Analyzing chunk frames = [1102336,1102848) | 20260505 06:35:58.323 | INFO | MemInfo: { | 20260505 06:35:58.323 | INFO | "gpu" : " 17 GB", | 20260505 06:35:58.323 | INFO | "huge" : "", | 20260505 06:35:58.323 | INFO | "huge_shared" : " 27 GB", | 20260505 06:35:58.323 | INFO | "malloc" : " 14 GB", | 20260505 06:35:58.323 | INFO | "pinned" : " 29 GB" | 20260505 06:35:58.323 | INFO | } | 20260505 06:35:58.323 | INFO | Perf report for the last 5.13933 seconds: | 20260505 06:35:58.323 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:35:58.323 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:58.323 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:58.323 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:58.323 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 06:35:58.323 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:58.323 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:58.323 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 06:35:58.323 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:58.323 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:35:58.323 | INFO | Uploaded 36GiB to GPU in 2.11873s (16.9913GiB/s) | 20260505 06:35:58.323 | INFO | Downloaded 34GiB to GPU in 2.03504s (16.7073GiB/s) | 20260505 06:36:03.462 | INFO | Analyzing chunk frames = [1102848,1103360) | 20260505 06:36:03.462 | INFO | MemInfo: { | 20260505 06:36:03.462 | INFO | "gpu" : " 17 GB", | 20260505 06:36:03.462 | INFO | "huge" : "", | 20260505 06:36:03.462 | INFO | "huge_shared" : " 27 GB", | 20260505 06:36:03.462 | INFO | "malloc" : " 14 GB", | 20260505 06:36:03.462 | INFO | "pinned" : " 29 GB" | 20260505 06:36:03.462 | INFO | } | 20260505 06:36:03.462 | INFO | Perf report for the last 5.13924 seconds: | 20260505 06:36:03.462 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:36:03.462 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:03.462 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:03.462 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:03.462 | INFO | ANALYSIS : 63.8%, 3.0, 99.3ms, 0 | 20260505 06:36:03.462 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:03.462 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:03.462 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 06:36:03.462 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:03.462 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:03.462 | INFO | Uploaded 36GiB to GPU in 2.14288s (16.7998GiB/s) | 20260505 06:36:03.462 | INFO | Downloaded 34GiB to GPU in 2.037s (16.6913GiB/s) | 20260505 06:36:08.601 | INFO | Analyzing chunk frames = [1103360,1103872) | 20260505 06:36:08.601 | INFO | MemInfo: { | 20260505 06:36:08.601 | INFO | "gpu" : " 17 GB", | 20260505 06:36:08.601 | INFO | "huge" : "", | 20260505 06:36:08.601 | INFO | "huge_shared" : " 27 GB", | 20260505 06:36:08.601 | INFO | "malloc" : " 14 GB", | 20260505 06:36:08.601 | INFO | "pinned" : " 29 GB" | 20260505 06:36:08.601 | INFO | } | 20260505 06:36:08.601 | INFO | Perf report for the last 5.13941 seconds: | 20260505 06:36:08.601 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:36:08.601 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:08.601 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:08.601 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:08.601 | INFO | ANALYSIS : 63.7%, 3.0, 98.9ms, 0 | 20260505 06:36:08.601 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:08.601 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:08.601 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 06:36:08.601 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:08.601 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:08.602 | INFO | Uploaded 36GiB to GPU in 2.14196s (16.8071GiB/s) | 20260505 06:36:08.602 | INFO | Downloaded 34GiB to GPU in 2.02765s (16.7682GiB/s) | 20260505 06:36:13.741 | INFO | Analyzing chunk frames = [1103872,1104384) | 20260505 06:36:13.741 | INFO | MemInfo: { | 20260505 06:36:13.741 | INFO | "gpu" : " 17 GB", | 20260505 06:36:13.741 | INFO | "huge" : "", | 20260505 06:36:13.741 | INFO | "huge_shared" : " 27 GB", | 20260505 06:36:13.741 | INFO | "malloc" : " 14 GB", | 20260505 06:36:13.741 | INFO | "pinned" : " 29 GB" | 20260505 06:36:13.741 | INFO | } | 20260505 06:36:13.741 | INFO | Perf report for the last 5.13933 seconds: | 20260505 06:36:13.741 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:36:13.741 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:13.741 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:13.741 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:13.741 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 06:36:13.741 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:13.741 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:13.741 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 06:36:13.741 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:13.741 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:13.741 | INFO | Uploaded 36GiB to GPU in 2.16985s (16.591GiB/s) | 20260505 06:36:13.741 | INFO | Downloaded 34GiB to GPU in 2.04085s (16.6597GiB/s) | 20260505 06:36:18.882 | INFO | Analyzing chunk frames = [1104384,1104896) | 20260505 06:36:18.882 | INFO | MemInfo: { | 20260505 06:36:18.882 | INFO | "gpu" : " 17 GB", | 20260505 06:36:18.882 | INFO | "huge" : "", | 20260505 06:36:18.882 | INFO | "huge_shared" : " 27 GB", | 20260505 06:36:18.882 | INFO | "malloc" : " 14 GB", | 20260505 06:36:18.882 | INFO | "pinned" : " 29 GB" | 20260505 06:36:18.882 | INFO | } | 20260505 06:36:18.882 | INFO | Perf report for the last 5.14126 seconds: | 20260505 06:36:18.882 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:36:18.882 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:18.882 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:18.882 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:18.882 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 06:36:18.882 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:18.882 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:18.882 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 06:36:18.882 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:18.882 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:18.882 | INFO | Uploaded 36GiB to GPU in 2.16712s (16.6119GiB/s) | 20260505 06:36:18.882 | INFO | Downloaded 34GiB to GPU in 2.0357s (16.7019GiB/s) | 20260505 06:36:24.019 | INFO | Analyzing chunk frames = [1104896,1105408) | 20260505 06:36:24.019 | INFO | MemInfo: { | 20260505 06:36:24.019 | INFO | "gpu" : " 17 GB", | 20260505 06:36:24.023 | INFO | "huge" : "", | 20260505 06:36:24.023 | INFO | "huge_shared" : " 27 GB", | 20260505 06:36:24.023 | INFO | "malloc" : " 14 GB", | 20260505 06:36:24.023 | INFO | "pinned" : " 29 GB" | 20260505 06:36:24.023 | INFO | } | 20260505 06:36:24.023 | INFO | Perf report for the last 5.14071 seconds: | 20260505 06:36:24.023 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:36:24.023 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:24.023 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:24.023 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:24.023 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 06:36:24.023 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:24.023 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:24.023 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260505 06:36:24.023 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:24.023 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:24.023 | INFO | Uploaded 36GiB to GPU in 2.19818s (16.3772GiB/s) | 20260505 06:36:24.023 | INFO | Downloaded 34GiB to GPU in 2.1587s (15.7502GiB/s) | 20260505 06:36:29.159 | INFO | Analyzing chunk frames = [1105408,1105920) | 20260505 06:36:29.159 | INFO | MemInfo: { | 20260505 06:36:29.159 | INFO | "gpu" : " 17 GB", | 20260505 06:36:29.159 | INFO | "huge" : "", | 20260505 06:36:29.159 | INFO | "huge_shared" : " 27 GB", | 20260505 06:36:29.160 | INFO | "malloc" : " 14 GB", | 20260505 06:36:29.160 | INFO | "pinned" : " 29 GB" | 20260505 06:36:29.160 | INFO | } | 20260505 06:36:29.160 | INFO | Perf report for the last 5.1376 seconds: | 20260505 06:36:29.160 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:36:29.160 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:29.160 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:29.160 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:29.160 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 06:36:29.160 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:29.160 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:29.160 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 06:36:29.160 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:29.160 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:29.160 | INFO | Uploaded 36GiB to GPU in 2.16273s (16.6456GiB/s) | 20260505 06:36:29.160 | INFO | Downloaded 34GiB to GPU in 2.05086s (16.5784GiB/s) | 20260505 06:36:29.161 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 06:36:29.161 | INFO | Total mem usage: 15.3 GiB | 20260505 06:36:29.161 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 06:36:29.161 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.11%) | 20260505 06:36:29.161 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 06:36:29.161 | INFO | High Water Breakdown: | 20260505 06:36:29.161 | INFO | 60.3% : 8.72 GiB ( 93.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 06:36:29.161 | INFO | 39.5% : 5.71 GiB ( 85.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 06:36:29.161 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 06:36:29.161 | INFO | ------------------------------------------ | 20260505 06:36:29.161 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:36:29.161 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 06:36:29.161 | INFO | Total mem usage: 26.5 GiB | 20260505 06:36:29.161 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 06:36:29.161 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 06:36:29.161 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 06:36:29.161 | INFO | High Water Breakdown: | 20260505 06:36:29.161 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:36:29.161 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:36:29.161 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:36:29.161 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:36:29.161 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:36:29.161 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:36:29.161 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 06:36:29.161 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 06:36:29.161 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:36:29.161 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 06:36:29.161 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 06:36:29.161 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 06:36:29.161 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 06:36:29.161 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 06:36:29.161 | INFO | ------------------------------------------ | 20260505 06:36:29.161 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:36:29.161 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 06:36:29.161 | INFO | Total mem usage: 25.6 GiB | 20260505 06:36:29.161 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 06:36:29.161 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 06:36:29.161 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 06:36:29.161 | INFO | High Water Breakdown: | 20260505 06:36:29.161 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 06:36:29.161 | INFO | ------------------------------------------ | 20260505 06:36:29.161 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:36:29.161 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 06:36:29.161 | INFO | Total mem usage: 15.7 GiB | 20260505 06:36:29.161 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 06:36:29.161 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 06:36:29.161 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260505 06:36:29.161 | INFO | High Water Breakdown: | 20260505 06:36:29.161 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 06:36:29.161 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 06:36:29.161 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 06:36:29.161 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 06:36:29.161 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 06:36:29.161 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 06:36:29.161 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 06:36:29.161 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 06:36:29.161 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 06:36:29.161 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:36:29.161 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 06:36:29.161 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:36:29.161 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 06:36:29.161 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 06:36:29.161 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 06:36:29.161 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 06:36:29.161 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:36:29.161 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 06:36:29.161 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:36:29.161 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:36:29.161 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:36:29.161 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 06:36:29.161 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 06:36:29.161 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:36:29.161 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 06:36:29.161 | INFO | ------------------------------------------ | 20260505 06:36:29.161 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:36:29.161 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 06:36:29.161 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:36:29.161 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:36:29.161 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.53e+01, 1.37e+01 / 1.45e+00 | 20260505 06:36:29.161 | INFO | PerfReport: 0.28%, AggregateStats -- 9.37e-02 / 1.03e-01, 1.04e-02 / 5.98e-03 | 20260505 06:36:29.161 | INFO | PerfReport: 11.60%, Baseline -- 3.84e+00 / 4.45e+00, 6.37e-01 / 3.63e-01 | 20260505 06:36:29.161 | INFO | PerfReport: 3.31%, Histogram -- 1.10e+00 / 1.49e+00, 2.17e-01 / 1.20e-01 | 20260505 06:36:29.161 | INFO | PerfReport: 15.44%, DME -- 4.09e+01 / 4.47e+01, 4.18e+00 / 1.03e+00 | 20260505 06:36:29.161 | INFO | PerfReport: 56.09%, FrameLabeling -- 1.86e+01 / 1.91e+01, 2.04e+00 / 1.12e+00 | 20260505 06:36:29.161 | INFO | PerfReport: 7.69%, PulseAccumulating -- 2.54e+00 / 3.14e+00, 4.59e-01 / 3.08e-01 | 20260505 06:36:29.161 | INFO | PerfReport: 4.10%, Metrics -- 1.36e+00 / 1.68e+00, 3.88e-01 / 1.87e-01 | 20260505 06:36:29.161 | INFO | PerfReport: 1.48%, PulseSerializing -- 4.91e-01 / 5.63e-01, 9.61e-02 / 4.14e-02 | 20260505 06:36:29.161 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 06:36:29.161 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:36:29.161 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:36:29.161 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.23e+01 / 0.00e+00 | 20260505 06:36:29.161 | INFO | PerfReport: 22.05%, Upload -- 2.24e+01 / 0.00e+00, 7.81e+00 / 0.00e+00 | 20260505 06:36:29.162 | INFO | PerfReport: 21.46%, Download -- 2.18e+01 / 0.00e+00, 7.71e+00 / 0.00e+00 | 20260505 06:36:29.162 | INFO | PerfReport: 32.63%, Compute -- 3.31e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 06:36:29.162 | INFO | PerfReport: 1.50%, LockUpload -- 1.52e+00 / 0.00e+00, 6.95e+00 / 0.00e+00 | 20260505 06:36:29.162 | INFO | PerfReport: 0.80%, LockDownload -- 8.07e-01 / 0.00e+00, 2.28e+00 / 0.00e+00 | 20260505 06:36:29.162 | INFO | PerfReport: 21.55%, LockCompute -- 2.19e+01 / 0.00e+00, 1.68e+01 / 0.00e+00 | 20260505 06:36:29.162 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 1.99e-02 / 0.00e+00 | 20260505 06:36:29.285 | INFO | Starting a round of Baz IO | 20260505 06:36:32.517 | INFO | Populating RTMetrics report | 20260505 06:36:32.517 | INFO | RTMetrics report populated | 20260505 06:36:32.517 | INFO | Opening tmp RTMetrics json file | 20260505 06:36:32.518 | INFO | Serializing RTMetrics to json | 20260505 06:36:32.519 | INFO | Serialization done, swapping tmp file now... | 20260505 06:36:32.519 | INFO | RTMetrics done | 20260505 06:36:32.646 | INFO | Finished a round of Baz IO: Wrote 2.385 GiB in 3.361 seconds | 20260505 06:36:32.646 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.67% / 0.05% / 67.28% | 20260505 06:36:32.646 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.84617% / 0.000107288% / 7.15256e-05% | 20260505 06:36:32.646 | INFO | Baz IO overall: | 20260505 06:36:32.646 | INFO | BytesWritten : 576399087430 | 20260505 06:36:32.646 | INFO | NumEvents : 272706045953 | 20260505 06:36:34.298 | INFO | Analyzing chunk frames = [1105920,1106432) | 20260505 06:36:34.298 | INFO | MemInfo: { | 20260505 06:36:34.298 | INFO | "gpu" : " 17 GB", | 20260505 06:36:34.298 | INFO | "huge" : "", | 20260505 06:36:34.298 | INFO | "huge_shared" : " 27 GB", | 20260505 06:36:34.298 | INFO | "malloc" : " 14 GB", | 20260505 06:36:34.298 | INFO | "pinned" : " 29 GB" | 20260505 06:36:34.298 | INFO | } | 20260505 06:36:34.298 | INFO | Perf report for the last 5.13802 seconds: | 20260505 06:36:34.298 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:36:34.298 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:34.298 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:34.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:34.299 | INFO | ANALYSIS : 64.2%, 2.9, 97.9ms, 0 | 20260505 06:36:34.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:34.299 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260505 06:36:34.299 | INFO | PRE_HQ : 63.1%, 1.9, 63.9ms, 0 | 20260505 06:36:34.299 | INFO | BAZWRITER : 64.7%, 4.2, 138.2ms, 0 | 20260505 06:36:34.299 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 06:36:34.299 | INFO | Uploaded 36GiB to GPU in 2.08767s (17.2441GiB/s) | 20260505 06:36:34.299 | INFO | Downloaded 38GiB to GPU in 2.30936s (16.4548GiB/s) | 20260505 06:36:39.437 | INFO | Analyzing chunk frames = [1106432,1106944) | 20260505 06:36:39.437 | INFO | MemInfo: { | 20260505 06:36:39.437 | INFO | "gpu" : " 17 GB", | 20260505 06:36:39.437 | INFO | "huge" : "", | 20260505 06:36:39.437 | INFO | "huge_shared" : " 27 GB", | 20260505 06:36:39.437 | INFO | "malloc" : " 14 GB", | 20260505 06:36:39.437 | INFO | "pinned" : " 29 GB" | 20260505 06:36:39.437 | INFO | } | 20260505 06:36:39.438 | INFO | Perf report for the last 5.13917 seconds: | 20260505 06:36:39.438 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:36:39.438 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:39.438 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:39.438 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:39.438 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 06:36:39.438 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:39.438 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:39.438 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 06:36:39.438 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:39.438 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:39.438 | INFO | Uploaded 36GiB to GPU in 2.09883s (17.1524GiB/s) | 20260505 06:36:39.438 | INFO | Downloaded 34GiB to GPU in 2.03371s (16.7182GiB/s) | 20260505 06:36:44.577 | INFO | Analyzing chunk frames = [1106944,1107456) | 20260505 06:36:44.577 | INFO | MemInfo: { | 20260505 06:36:44.577 | INFO | "gpu" : " 17 GB", | 20260505 06:36:44.577 | INFO | "huge" : "", | 20260505 06:36:44.577 | INFO | "huge_shared" : " 27 GB", | 20260505 06:36:44.577 | INFO | "malloc" : " 14 GB", | 20260505 06:36:44.577 | INFO | "pinned" : " 29 GB" | 20260505 06:36:44.577 | INFO | } | 20260505 06:36:44.577 | INFO | Perf report for the last 5.13963 seconds: | 20260505 06:36:44.577 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:36:44.577 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:44.577 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:44.577 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:44.577 | INFO | ANALYSIS : 63.8%, 3.0, 99.3ms, 0 | 20260505 06:36:44.577 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:44.577 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:44.577 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 06:36:44.577 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:44.577 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:44.577 | INFO | Uploaded 36GiB to GPU in 2.12459s (16.9445GiB/s) | 20260505 06:36:44.577 | INFO | Downloaded 34GiB to GPU in 2.0465s (16.6137GiB/s) | 20260505 06:36:49.721 | INFO | Analyzing chunk frames = [1107456,1107968) | 20260505 06:36:49.721 | INFO | MemInfo: { | 20260505 06:36:49.721 | INFO | "gpu" : " 17 GB", | 20260505 06:36:49.721 | INFO | "huge" : "", | 20260505 06:36:49.721 | INFO | "huge_shared" : " 27 GB", | 20260505 06:36:49.722 | INFO | "malloc" : " 14 GB", | 20260505 06:36:49.722 | INFO | "pinned" : " 29 GB" | 20260505 06:36:49.722 | INFO | } | 20260505 06:36:49.722 | INFO | Perf report for the last 5.14437 seconds: | 20260505 06:36:49.722 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:36:49.722 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:49.722 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:49.722 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:49.722 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260505 06:36:49.722 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:49.722 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:49.722 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 06:36:49.722 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:49.722 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:49.722 | INFO | Uploaded 36GiB to GPU in 2.14689s (16.7685GiB/s) | 20260505 06:36:49.722 | INFO | Downloaded 34GiB to GPU in 2.02826s (16.7632GiB/s) | 20260505 06:36:54.855 | INFO | Analyzing chunk frames = [1107968,1108480) | 20260505 06:36:54.855 | INFO | MemInfo: { | 20260505 06:36:54.855 | INFO | "gpu" : " 17 GB", | 20260505 06:36:54.855 | INFO | "huge" : "", | 20260505 06:36:54.855 | INFO | "huge_shared" : " 27 GB", | 20260505 06:36:54.855 | INFO | "malloc" : " 14 GB", | 20260505 06:36:54.855 | INFO | "pinned" : " 29 GB" | 20260505 06:36:54.855 | INFO | } | 20260505 06:36:54.855 | INFO | Perf report for the last 5.1337 seconds: | 20260505 06:36:54.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:36:54.855 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:54.855 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:54.855 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:54.855 | INFO | ANALYSIS : 63.4%, 3.0, 98.8ms, 0 | 20260505 06:36:54.855 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:54.855 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:54.855 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 06:36:54.855 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:54.855 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:54.855 | INFO | Uploaded 36GiB to GPU in 2.10566s (17.0967GiB/s) | 20260505 06:36:54.856 | INFO | Downloaded 34GiB to GPU in 2.03416s (16.7145GiB/s) | 20260505 06:36:59.996 | INFO | Analyzing chunk frames = [1108480,1108992) | 20260505 06:36:59.996 | INFO | MemInfo: { | 20260505 06:36:59.996 | INFO | "gpu" : " 17 GB", | 20260505 06:36:59.996 | INFO | "huge" : "", | 20260505 06:36:59.999 | INFO | "huge_shared" : " 27 GB", | 20260505 06:36:59.999 | INFO | "malloc" : " 14 GB", | 20260505 06:36:59.999 | INFO | "pinned" : " 29 GB" | 20260505 06:36:59.999 | INFO | } | 20260505 06:36:59.999 | INFO | Perf report for the last 5.14358 seconds: | 20260505 06:36:59.999 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:36:59.999 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:59.999 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:59.999 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:59.999 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 06:36:59.999 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:59.999 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:59.999 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 06:36:59.999 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:59.999 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:36:59.999 | INFO | Uploaded 36GiB to GPU in 2.15279s (16.7225GiB/s) | 20260505 06:36:59.999 | INFO | Downloaded 34GiB to GPU in 2.04408s (16.6334GiB/s) | 20260505 06:37:05.134 | INFO | Analyzing chunk frames = [1108992,1109504) | 20260505 06:37:05.134 | INFO | MemInfo: { | 20260505 06:37:05.134 | INFO | "gpu" : " 17 GB", | 20260505 06:37:05.134 | INFO | "huge" : "", | 20260505 06:37:05.134 | INFO | "huge_shared" : " 27 GB", | 20260505 06:37:05.134 | INFO | "malloc" : " 14 GB", | 20260505 06:37:05.134 | INFO | "pinned" : " 29 GB" | 20260505 06:37:05.134 | INFO | } | 20260505 06:37:05.134 | INFO | Perf report for the last 5.1354 seconds: | 20260505 06:37:05.134 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:37:05.134 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:05.134 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:05.134 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:05.134 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 06:37:05.134 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:05.134 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:05.134 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260505 06:37:05.134 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:05.134 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:05.134 | INFO | Uploaded 36GiB to GPU in 2.15931s (16.672GiB/s) | 20260505 06:37:05.135 | INFO | Downloaded 34GiB to GPU in 2.15755s (15.7586GiB/s) | 20260505 06:37:10.273 | INFO | Analyzing chunk frames = [1109504,1110016) | 20260505 06:37:10.273 | INFO | MemInfo: { | 20260505 06:37:10.273 | INFO | "gpu" : " 17 GB", | 20260505 06:37:10.273 | INFO | "huge" : "", | 20260505 06:37:10.273 | INFO | "huge_shared" : " 27 GB", | 20260505 06:37:10.273 | INFO | "malloc" : " 14 GB", | 20260505 06:37:10.273 | INFO | "pinned" : " 29 GB" | 20260505 06:37:10.273 | INFO | } | 20260505 06:37:10.273 | INFO | Perf report for the last 5.13919 seconds: | 20260505 06:37:10.273 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:37:10.273 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:10.273 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:10.273 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:10.274 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 06:37:10.274 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:10.274 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:10.274 | INFO | PRE_HQ : 30.5%, 1.0, 15.9ms, 0 | 20260505 06:37:10.274 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:10.274 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:10.274 | INFO | Uploaded 36GiB to GPU in 2.12681s (16.9267GiB/s) | 20260505 06:37:10.274 | INFO | Downloaded 34GiB to GPU in 2.05841s (16.5176GiB/s) | 20260505 06:37:10.396 | INFO | Starting a round of Baz IO | 20260505 06:37:13.631 | INFO | Populating RTMetrics report | 20260505 06:37:13.631 | INFO | RTMetrics report populated | 20260505 06:37:13.631 | INFO | Opening tmp RTMetrics json file | 20260505 06:37:13.655 | INFO | Serializing RTMetrics to json | 20260505 06:37:13.656 | INFO | Serialization done, swapping tmp file now... | 20260505 06:37:13.657 | INFO | RTMetrics done | 20260505 06:37:13.744 | INFO | Finished a round of Baz IO: Wrote 2.388 GiB in 3.348 seconds | 20260505 06:37:13.745 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.63% / 0.04% / 67.33% | 20260505 06:37:13.745 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.83721% / 5.16574e-05% / 8.74201e-05% | 20260505 06:37:13.745 | INFO | Baz IO overall: | 20260505 06:37:13.745 | INFO | BytesWritten : 578963211621 | 20260505 06:37:13.745 | INFO | NumEvents : 273932959385 | 20260505 06:37:15.413 | INFO | Analyzing chunk frames = [1110016,1110528) | 20260505 06:37:15.413 | INFO | MemInfo: { | 20260505 06:37:15.413 | INFO | "gpu" : " 17 GB", | 20260505 06:37:15.413 | INFO | "huge" : "", | 20260505 06:37:15.413 | INFO | "huge_shared" : " 27 GB", | 20260505 06:37:15.413 | INFO | "malloc" : " 14 GB", | 20260505 06:37:15.413 | INFO | "pinned" : " 29 GB" | 20260505 06:37:15.413 | INFO | } | 20260505 06:37:15.413 | INFO | Perf report for the last 5.13923 seconds: | 20260505 06:37:15.413 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:37:15.413 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:15.413 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:15.413 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:15.413 | INFO | ANALYSIS : 64.2%, 2.9, 98.1ms, 0 | 20260505 06:37:15.413 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:15.413 | INFO | RT_METRICS : 46.6%, 1.0, 24.4ms, 0 | 20260505 06:37:15.413 | INFO | PRE_HQ : 63.2%, 2.0, 64.2ms, 0 | 20260505 06:37:15.413 | INFO | BAZWRITER : 64.4%, 4.1, 135.7ms, 0 | 20260505 06:37:15.413 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 06:37:15.413 | INFO | Uploaded 36GiB to GPU in 2.06953s (17.3952GiB/s) | 20260505 06:37:15.413 | INFO | Downloaded 38GiB to GPU in 2.3029s (16.5009GiB/s) | 20260505 06:37:20.554 | INFO | Analyzing chunk frames = [1110528,1111040) | 20260505 06:37:20.554 | INFO | MemInfo: { | 20260505 06:37:20.554 | INFO | "gpu" : " 17 GB", | 20260505 06:37:20.554 | INFO | "huge" : "", | 20260505 06:37:20.554 | INFO | "huge_shared" : " 27 GB", | 20260505 06:37:20.554 | INFO | "malloc" : " 14 GB", | 20260505 06:37:20.554 | INFO | "pinned" : " 29 GB" | 20260505 06:37:20.554 | INFO | } | 20260505 06:37:20.555 | INFO | Perf report for the last 5.14178 seconds: | 20260505 06:37:20.555 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:37:20.555 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:20.555 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:20.555 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:20.555 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 06:37:20.555 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:20.555 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:20.555 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 06:37:20.555 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:20.555 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:20.555 | INFO | Uploaded 36GiB to GPU in 2.09311s (17.1992GiB/s) | 20260505 06:37:20.555 | INFO | Downloaded 34GiB to GPU in 2.03975s (16.6687GiB/s) | 20260505 06:37:25.694 | INFO | Analyzing chunk frames = [1111040,1111552) | 20260505 06:37:25.694 | INFO | MemInfo: { | 20260505 06:37:25.694 | INFO | "gpu" : " 17 GB", | 20260505 06:37:25.694 | INFO | "huge" : "", | 20260505 06:37:25.694 | INFO | "huge_shared" : " 27 GB", | 20260505 06:37:25.694 | INFO | "malloc" : " 14 GB", | 20260505 06:37:25.694 | INFO | "pinned" : " 29 GB" | 20260505 06:37:25.694 | INFO | } | 20260505 06:37:25.694 | INFO | Perf report for the last 5.13973 seconds: | 20260505 06:37:25.694 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:37:25.694 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:25.694 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:25.694 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:25.694 | INFO | ANALYSIS : 63.8%, 3.0, 99.3ms, 0 | 20260505 06:37:25.694 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:25.694 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:25.694 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 06:37:25.694 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:25.694 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:25.695 | INFO | Uploaded 36GiB to GPU in 2.13325s (16.8756GiB/s) | 20260505 06:37:25.695 | INFO | Downloaded 34GiB to GPU in 2.03654s (16.695GiB/s) | 20260505 06:37:30.831 | INFO | Analyzing chunk frames = [1111552,1112064) | 20260505 06:37:30.831 | INFO | MemInfo: { | 20260505 06:37:30.831 | INFO | "gpu" : " 17 GB", | 20260505 06:37:30.831 | INFO | "huge" : "", | 20260505 06:37:30.831 | INFO | "huge_shared" : " 27 GB", | 20260505 06:37:30.831 | INFO | "malloc" : " 14 GB", | 20260505 06:37:30.831 | INFO | "pinned" : " 29 GB" | 20260505 06:37:30.831 | INFO | } | 20260505 06:37:30.831 | INFO | Perf report for the last 5.13708 seconds: | 20260505 06:37:30.831 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:37:30.831 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:30.831 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:30.831 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:30.831 | INFO | ANALYSIS : 63.3%, 3.0, 98.3ms, 0 | 20260505 06:37:30.831 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:30.831 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:30.831 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 06:37:30.832 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:30.832 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:30.832 | INFO | Uploaded 36GiB to GPU in 2.12221s (16.9635GiB/s) | 20260505 06:37:30.832 | INFO | Downloaded 34GiB to GPU in 2.04436s (16.6311GiB/s) | 20260505 06:37:35.970 | INFO | Analyzing chunk frames = [1112064,1112576) | 20260505 06:37:35.970 | INFO | MemInfo: { | 20260505 06:37:35.970 | INFO | "gpu" : " 17 GB", | 20260505 06:37:35.970 | INFO | "huge" : "", | 20260505 06:37:35.970 | INFO | "huge_shared" : " 27 GB", | 20260505 06:37:35.970 | INFO | "malloc" : " 14 GB", | 20260505 06:37:35.970 | INFO | "pinned" : " 29 GB" | 20260505 06:37:35.970 | INFO | } | 20260505 06:37:35.970 | INFO | Perf report for the last 5.13882 seconds: | 20260505 06:37:35.970 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:37:35.970 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:35.970 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:35.970 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:35.970 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 06:37:35.970 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:35.970 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:35.970 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 06:37:35.970 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:35.970 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:35.970 | INFO | Uploaded 36GiB to GPU in 2.16968s (16.5923GiB/s) | 20260505 06:37:35.970 | INFO | Downloaded 34GiB to GPU in 2.03486s (16.7088GiB/s) | 20260505 06:37:41.111 | INFO | Analyzing chunk frames = [1112576,1113088) | 20260505 06:37:41.111 | INFO | MemInfo: { | 20260505 06:37:41.111 | INFO | "gpu" : " 17 GB", | 20260505 06:37:41.111 | INFO | "huge" : "", | 20260505 06:37:41.111 | INFO | "huge_shared" : " 27 GB", | 20260505 06:37:41.111 | INFO | "malloc" : " 14 GB", | 20260505 06:37:41.111 | INFO | "pinned" : " 29 GB" | 20260505 06:37:41.111 | INFO | } | 20260505 06:37:41.111 | INFO | Perf report for the last 5.14068 seconds: | 20260505 06:37:41.111 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:37:41.111 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:41.111 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:41.111 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:41.111 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 06:37:41.111 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:41.111 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:41.111 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 06:37:41.111 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:41.111 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:41.111 | INFO | Uploaded 36GiB to GPU in 2.15495s (16.7057GiB/s) | 20260505 06:37:41.111 | INFO | Downloaded 34GiB to GPU in 2.03249s (16.7283GiB/s) | 20260505 06:37:46.249 | INFO | Analyzing chunk frames = [1113088,1113600) | 20260505 06:37:46.249 | INFO | MemInfo: { | 20260505 06:37:46.249 | INFO | "gpu" : " 17 GB", | 20260505 06:37:46.249 | INFO | "huge" : "", | 20260505 06:37:46.249 | INFO | "huge_shared" : " 27 GB", | 20260505 06:37:46.249 | INFO | "malloc" : " 14 GB", | 20260505 06:37:46.249 | INFO | "pinned" : " 29 GB" | 20260505 06:37:46.249 | INFO | } | 20260505 06:37:46.249 | INFO | Perf report for the last 5.13785 seconds: | 20260505 06:37:46.249 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:37:46.249 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:46.249 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:46.249 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:46.249 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 06:37:46.249 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:46.249 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:46.249 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260505 06:37:46.249 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:46.249 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:46.249 | INFO | Uploaded 36GiB to GPU in 2.19289s (16.4167GiB/s) | 20260505 06:37:46.249 | INFO | Downloaded 34GiB to GPU in 2.16595s (15.6975GiB/s) | 20260505 06:37:51.388 | INFO | Analyzing chunk frames = [1113600,1114112) | 20260505 06:37:51.388 | INFO | MemInfo: { | 20260505 06:37:51.388 | INFO | "gpu" : " 17 GB", | 20260505 06:37:51.388 | INFO | "huge" : "", | 20260505 06:37:51.388 | INFO | "huge_shared" : " 27 GB", | 20260505 06:37:51.388 | INFO | "malloc" : " 14 GB", | 20260505 06:37:51.388 | INFO | "pinned" : " 29 GB" | 20260505 06:37:51.388 | INFO | } | 20260505 06:37:51.388 | INFO | Perf report for the last 5.13942 seconds: | 20260505 06:37:51.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:37:51.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:51.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:51.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:51.388 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 06:37:51.388 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:51.388 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:51.388 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 06:37:51.388 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:51.388 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:51.389 | INFO | Uploaded 36GiB to GPU in 2.12388s (16.9501GiB/s) | 20260505 06:37:51.389 | INFO | Downloaded 34GiB to GPU in 2.07276s (16.4033GiB/s) | 20260505 06:37:51.389 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 06:37:51.389 | INFO | Total mem usage: 15.3 GiB | 20260505 06:37:51.389 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 06:37:51.389 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.51%) | 20260505 06:37:51.389 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 06:37:51.389 | INFO | High Water Breakdown: | 20260505 06:37:51.389 | INFO | 60.4% : 8.69 GiB ( 92.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 06:37:51.389 | INFO | 39.5% : 5.68 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 06:37:51.389 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 06:37:51.389 | INFO | ------------------------------------------ | 20260505 06:37:51.389 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:37:51.389 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 06:37:51.389 | INFO | Total mem usage: 26.5 GiB | 20260505 06:37:51.389 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 06:37:51.389 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 06:37:51.389 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 06:37:51.389 | INFO | High Water Breakdown: | 20260505 06:37:51.389 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:37:51.389 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:37:51.389 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:37:51.389 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:37:51.389 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:37:51.389 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:37:51.389 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 06:37:51.389 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 06:37:51.389 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:37:51.389 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 06:37:51.389 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 06:37:51.389 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 06:37:51.389 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 06:37:51.389 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 06:37:51.389 | INFO | ------------------------------------------ | 20260505 06:37:51.389 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:37:51.389 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 06:37:51.389 | INFO | Total mem usage: 25.6 GiB | 20260505 06:37:51.389 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 06:37:51.389 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 06:37:51.389 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 06:37:51.389 | INFO | High Water Breakdown: | 20260505 06:37:51.389 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 06:37:51.389 | INFO | ------------------------------------------ | 20260505 06:37:51.389 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:37:51.389 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 06:37:51.389 | INFO | Total mem usage: 15.7 GiB | 20260505 06:37:51.389 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 06:37:51.389 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 06:37:51.389 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260505 06:37:51.389 | INFO | High Water Breakdown: | 20260505 06:37:51.389 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 06:37:51.389 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 06:37:51.389 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 06:37:51.389 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 06:37:51.389 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 06:37:51.389 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 06:37:51.389 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 06:37:51.389 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 06:37:51.389 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 06:37:51.390 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:37:51.390 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 06:37:51.390 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:37:51.390 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 06:37:51.390 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 06:37:51.390 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 06:37:51.390 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 06:37:51.390 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:37:51.390 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 06:37:51.390 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:37:51.390 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:37:51.390 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:37:51.390 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 06:37:51.390 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 06:37:51.390 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:37:51.390 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 06:37:51.390 | INFO | ------------------------------------------ | 20260505 06:37:51.390 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:37:51.390 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 06:37:51.390 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:37:51.390 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:37:51.390 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.51e+01, 1.37e+01 / 1.71e+00 | 20260505 06:37:51.390 | INFO | PerfReport: 0.28%, AggregateStats -- 9.40e-02 / 1.03e-01, 1.14e-02 / 6.72e-03 | 20260505 06:37:51.390 | INFO | PerfReport: 11.62%, Baseline -- 3.84e+00 / 4.48e+00, 6.42e-01 / 2.77e-01 | 20260505 06:37:51.390 | INFO | PerfReport: 3.32%, Histogram -- 1.10e+00 / 1.48e+00, 2.18e-01 / 1.21e-01 | 20260505 06:37:51.390 | INFO | PerfReport: 15.45%, DME -- 4.09e+01 / 4.46e+01, 4.26e+00 / 1.14e+00 | 20260505 06:37:51.390 | INFO | PerfReport: 56.02%, FrameLabeling -- 1.85e+01 / 1.90e+01, 2.03e+00 / 1.08e+00 | 20260505 06:37:51.390 | INFO | PerfReport: 7.72%, PulseAccumulating -- 2.55e+00 / 3.15e+00, 4.68e-01 / 3.22e-01 | 20260505 06:37:51.390 | INFO | PerfReport: 4.10%, Metrics -- 1.36e+00 / 1.70e+00, 3.86e-01 / 1.87e-01 | 20260505 06:37:51.390 | INFO | PerfReport: 1.48%, PulseSerializing -- 4.91e-01 / 5.65e-01, 9.63e-02 / 4.47e-02 | 20260505 06:37:51.390 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 06:37:51.390 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:37:51.390 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:37:51.390 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.22e+01 / 0.00e+00 | 20260505 06:37:51.390 | INFO | PerfReport: 21.90%, Upload -- 2.22e+01 / 0.00e+00, 7.65e+00 / 0.00e+00 | 20260505 06:37:51.390 | INFO | PerfReport: 21.50%, Download -- 2.18e+01 / 0.00e+00, 7.74e+00 / 0.00e+00 | 20260505 06:37:51.390 | INFO | PerfReport: 32.67%, Compute -- 3.31e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 06:37:51.390 | INFO | PerfReport: 1.48%, LockUpload -- 1.50e+00 / 0.00e+00, 6.93e+00 / 0.00e+00 | 20260505 06:37:51.390 | INFO | PerfReport: 0.79%, LockDownload -- 8.02e-01 / 0.00e+00, 2.29e+00 / 0.00e+00 | 20260505 06:37:51.390 | INFO | PerfReport: 21.65%, LockCompute -- 2.19e+01 / 0.00e+00, 1.67e+01 / 0.00e+00 | 20260505 06:37:51.390 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 1.75e-02 / 0.00e+00 | 20260505 06:37:51.511 | INFO | Starting a round of Baz IO | 20260505 06:37:54.747 | INFO | Populating RTMetrics report | 20260505 06:37:54.747 | INFO | RTMetrics report populated | 20260505 06:37:54.747 | INFO | Opening tmp RTMetrics json file | 20260505 06:37:54.772 | INFO | Serializing RTMetrics to json | 20260505 06:37:54.783 | INFO | Serialization done, swapping tmp file now... | 20260505 06:37:54.785 | INFO | RTMetrics done | 20260505 06:37:54.873 | INFO | Finished a round of Baz IO: Wrote 2.387 GiB in 3.361 seconds | 20260505 06:37:54.873 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.58% / 0.04% / 67.37% | 20260505 06:37:54.873 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.80845% / 7.54992e-05% / 3.17891e-05% | 20260505 06:37:54.873 | INFO | Baz IO overall: | 20260505 06:37:54.873 | INFO | BytesWritten : 581526538053 | 20260505 06:37:54.873 | INFO | NumEvents : 275158825635 | 20260505 06:37:56.527 | INFO | Analyzing chunk frames = [1114112,1114624) | 20260505 06:37:56.528 | INFO | MemInfo: { | 20260505 06:37:56.528 | INFO | "gpu" : " 17 GB", | 20260505 06:37:56.528 | INFO | "huge" : "", | 20260505 06:37:56.528 | INFO | "huge_shared" : " 27 GB", | 20260505 06:37:56.528 | INFO | "malloc" : " 14 GB", | 20260505 06:37:56.528 | INFO | "pinned" : " 29 GB" | 20260505 06:37:56.528 | INFO | } | 20260505 06:37:56.528 | INFO | Perf report for the last 5.13943 seconds: | 20260505 06:37:56.528 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:37:56.528 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:56.528 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:56.528 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:56.528 | INFO | ANALYSIS : 64.2%, 2.9, 98.0ms, 0 | 20260505 06:37:56.528 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:37:56.528 | INFO | RT_METRICS : 46.7%, 1.0, 24.5ms, 0 | 20260505 06:37:56.528 | INFO | PRE_HQ : 63.2%, 1.9, 63.8ms, 0 | 20260505 06:37:56.528 | INFO | BAZWRITER : 64.7%, 4.2, 138.9ms, 0 | 20260505 06:37:56.528 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 06:37:56.528 | INFO | Uploaded 36GiB to GPU in 2.09029s (17.2225GiB/s) | 20260505 06:37:56.528 | INFO | Downloaded 38GiB to GPU in 2.2976s (16.539GiB/s) | 20260505 06:38:01.667 | INFO | Analyzing chunk frames = [1114624,1115136) | 20260505 06:38:01.667 | INFO | MemInfo: { | 20260505 06:38:01.667 | INFO | "gpu" : " 17 GB", | 20260505 06:38:01.667 | INFO | "huge" : "", | 20260505 06:38:01.667 | INFO | "huge_shared" : " 27 GB", | 20260505 06:38:01.667 | INFO | "malloc" : " 14 GB", | 20260505 06:38:01.667 | INFO | "pinned" : " 29 GB" | 20260505 06:38:01.667 | INFO | } | 20260505 06:38:01.667 | INFO | Perf report for the last 5.13918 seconds: | 20260505 06:38:01.667 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:38:01.667 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:01.667 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:01.667 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:01.667 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 06:38:01.667 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:01.667 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:01.667 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 06:38:01.667 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:01.667 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:01.667 | INFO | Uploaded 36GiB to GPU in 2.10805s (17.0774GiB/s) | 20260505 06:38:01.667 | INFO | Downloaded 34GiB to GPU in 2.03954s (16.6704GiB/s) | 20260505 06:38:06.806 | INFO | Analyzing chunk frames = [1115136,1115648) | 20260505 06:38:06.806 | INFO | MemInfo: { | 20260505 06:38:06.806 | INFO | "gpu" : " 17 GB", | 20260505 06:38:06.806 | INFO | "huge" : "", | 20260505 06:38:06.806 | INFO | "huge_shared" : " 27 GB", | 20260505 06:38:06.806 | INFO | "malloc" : " 14 GB", | 20260505 06:38:06.806 | INFO | "pinned" : " 29 GB" | 20260505 06:38:06.806 | INFO | } | 20260505 06:38:06.806 | INFO | Perf report for the last 5.13934 seconds: | 20260505 06:38:06.806 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:38:06.806 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:06.806 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:06.806 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:06.806 | INFO | ANALYSIS : 63.8%, 3.0, 99.3ms, 0 | 20260505 06:38:06.806 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:06.806 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:06.806 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260505 06:38:06.806 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:06.806 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:06.806 | INFO | Uploaded 36GiB to GPU in 2.13859s (16.8335GiB/s) | 20260505 06:38:06.806 | INFO | Downloaded 34GiB to GPU in 2.04001s (16.6666GiB/s) | 20260505 06:38:11.945 | INFO | Analyzing chunk frames = [1115648,1116160) | 20260505 06:38:11.945 | INFO | MemInfo: { | 20260505 06:38:11.945 | INFO | "gpu" : " 17 GB", | 20260505 06:38:11.946 | INFO | "huge" : "", | 20260505 06:38:11.946 | INFO | "huge_shared" : " 27 GB", | 20260505 06:38:11.946 | INFO | "malloc" : " 14 GB", | 20260505 06:38:11.946 | INFO | "pinned" : " 29 GB" | 20260505 06:38:11.946 | INFO | } | 20260505 06:38:11.946 | INFO | Perf report for the last 5.13943 seconds: | 20260505 06:38:11.946 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:38:11.946 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:11.946 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:11.946 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:11.946 | INFO | ANALYSIS : 63.8%, 3.0, 99.1ms, 0 | 20260505 06:38:11.946 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:11.946 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:11.946 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 06:38:11.946 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:11.946 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:11.946 | INFO | Uploaded 36GiB to GPU in 2.15726s (16.6879GiB/s) | 20260505 06:38:11.946 | INFO | Downloaded 34GiB to GPU in 2.0344s (16.7125GiB/s) | 20260505 06:38:17.085 | INFO | Analyzing chunk frames = [1116160,1116672) | 20260505 06:38:17.085 | INFO | MemInfo: { | 20260505 06:38:17.085 | INFO | "gpu" : " 17 GB", | 20260505 06:38:17.085 | INFO | "huge" : "", | 20260505 06:38:17.085 | INFO | "huge_shared" : " 27 GB", | 20260505 06:38:17.085 | INFO | "malloc" : " 14 GB", | 20260505 06:38:17.085 | INFO | "pinned" : " 29 GB" | 20260505 06:38:17.085 | INFO | } | 20260505 06:38:17.085 | INFO | Perf report for the last 5.13977 seconds: | 20260505 06:38:17.085 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:38:17.085 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:17.085 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:17.085 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:17.085 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 06:38:17.085 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:17.085 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:17.086 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 06:38:17.086 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:17.086 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:17.086 | INFO | Uploaded 36GiB to GPU in 2.16955s (16.5933GiB/s) | 20260505 06:38:17.086 | INFO | Downloaded 34GiB to GPU in 2.04327s (16.64GiB/s) | 20260505 06:38:22.225 | INFO | Analyzing chunk frames = [1116672,1117184) | 20260505 06:38:22.225 | INFO | MemInfo: { | 20260505 06:38:22.225 | INFO | "gpu" : " 17 GB", | 20260505 06:38:22.226 | INFO | "huge" : "", | 20260505 06:38:22.226 | INFO | "huge_shared" : " 27 GB", | 20260505 06:38:22.226 | INFO | "malloc" : " 14 GB", | 20260505 06:38:22.226 | INFO | "pinned" : " 29 GB" | 20260505 06:38:22.226 | INFO | } | 20260505 06:38:22.226 | INFO | Perf report for the last 5.14024 seconds: | 20260505 06:38:22.226 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:38:22.226 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:22.226 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:22.226 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:22.226 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 06:38:22.226 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:22.226 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:22.226 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 06:38:22.226 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:22.226 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:22.226 | INFO | Uploaded 36GiB to GPU in 2.15342s (16.7176GiB/s) | 20260505 06:38:22.226 | INFO | Downloaded 34GiB to GPU in 2.03068s (16.7432GiB/s) | 20260505 06:38:27.363 | INFO | Analyzing chunk frames = [1117184,1117696) | 20260505 06:38:27.363 | INFO | MemInfo: { | 20260505 06:38:27.363 | INFO | "gpu" : " 17 GB", | 20260505 06:38:27.363 | INFO | "huge" : "", | 20260505 06:38:27.363 | INFO | "huge_shared" : " 27 GB", | 20260505 06:38:27.364 | INFO | "malloc" : " 14 GB", | 20260505 06:38:27.364 | INFO | "pinned" : " 29 GB" | 20260505 06:38:27.364 | INFO | } | 20260505 06:38:27.364 | INFO | Perf report for the last 5.1379 seconds: | 20260505 06:38:27.364 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:38:27.364 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:27.364 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:27.364 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:27.364 | INFO | ANALYSIS : 65.1%, 3.0, 101.7ms, 0 | 20260505 06:38:27.364 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:27.364 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:27.364 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260505 06:38:27.364 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:27.364 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:27.364 | INFO | Uploaded 36GiB to GPU in 2.25086s (15.9939GiB/s) | 20260505 06:38:27.364 | INFO | Downloaded 34GiB to GPU in 2.15356s (15.7878GiB/s) | 20260505 06:38:32.503 | INFO | Analyzing chunk frames = [1117696,1118208) | 20260505 06:38:32.503 | INFO | MemInfo: { | 20260505 06:38:32.503 | INFO | "gpu" : " 17 GB", | 20260505 06:38:32.503 | INFO | "huge" : "", | 20260505 06:38:32.503 | INFO | "huge_shared" : " 27 GB", | 20260505 06:38:32.503 | INFO | "malloc" : " 14 GB", | 20260505 06:38:32.503 | INFO | "pinned" : " 29 GB" | 20260505 06:38:32.503 | INFO | } | 20260505 06:38:32.503 | INFO | Perf report for the last 5.13951 seconds: | 20260505 06:38:32.503 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:38:32.503 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:32.503 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:32.503 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:32.503 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 06:38:32.503 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:32.503 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:32.503 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 06:38:32.503 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:32.503 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:32.503 | INFO | Uploaded 36GiB to GPU in 2.12938s (16.9063GiB/s) | 20260505 06:38:32.503 | INFO | Downloaded 34GiB to GPU in 2.0736s (16.3966GiB/s) | 20260505 06:38:32.625 | INFO | Starting a round of Baz IO | 20260505 06:38:35.862 | INFO | Populating RTMetrics report | 20260505 06:38:35.863 | INFO | RTMetrics report populated | 20260505 06:38:35.863 | INFO | Opening tmp RTMetrics json file | 20260505 06:38:35.871 | INFO | Serializing RTMetrics to json | 20260505 06:38:35.887 | INFO | Serialization done, swapping tmp file now... | 20260505 06:38:35.897 | INFO | RTMetrics done | 20260505 06:38:35.983 | INFO | Finished a round of Baz IO: Wrote 2.39 GiB in 3.358 seconds | 20260505 06:38:35.983 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.54% / 0.04% / 67.42% | 20260505 06:38:35.983 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.79941% / 6.75519e-05% / 8.34465e-05% | 20260505 06:38:35.983 | INFO | Baz IO overall: | 20260505 06:38:35.983 | INFO | BytesWritten : 584092580072 | 20260505 06:38:35.983 | INFO | NumEvents : 276386702401 | 20260505 06:38:37.642 | INFO | Analyzing chunk frames = [1118208,1118720) | 20260505 06:38:37.642 | INFO | MemInfo: { | 20260505 06:38:37.642 | INFO | "gpu" : " 17 GB", | 20260505 06:38:37.642 | INFO | "huge" : "", | 20260505 06:38:37.642 | INFO | "huge_shared" : " 27 GB", | 20260505 06:38:37.642 | INFO | "malloc" : " 14 GB", | 20260505 06:38:37.642 | INFO | "pinned" : " 29 GB" | 20260505 06:38:37.642 | INFO | } | 20260505 06:38:37.642 | INFO | Perf report for the last 5.13911 seconds: | 20260505 06:38:37.642 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:38:37.642 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:37.642 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:37.642 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:37.642 | INFO | ANALYSIS : 64.2%, 2.9, 97.9ms, 0 | 20260505 06:38:37.642 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:37.642 | INFO | RT_METRICS : 46.6%, 1.0, 24.4ms, 0 | 20260505 06:38:37.642 | INFO | PRE_HQ : 63.2%, 1.9, 63.9ms, 0 | 20260505 06:38:37.642 | INFO | BAZWRITER : 64.6%, 4.2, 138.7ms, 0 | 20260505 06:38:37.642 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 06:38:37.643 | INFO | Uploaded 36GiB to GPU in 2.09113s (17.2156GiB/s) | 20260505 06:38:37.643 | INFO | Downloaded 38GiB to GPU in 2.29717s (16.5421GiB/s) | 20260505 06:38:42.782 | INFO | Analyzing chunk frames = [1118720,1119232) | 20260505 06:38:42.782 | INFO | MemInfo: { | 20260505 06:38:42.782 | INFO | "gpu" : " 17 GB", | 20260505 06:38:42.782 | INFO | "huge" : "", | 20260505 06:38:42.782 | INFO | "huge_shared" : " 27 GB", | 20260505 06:38:42.782 | INFO | "malloc" : " 14 GB", | 20260505 06:38:42.782 | INFO | "pinned" : " 29 GB" | 20260505 06:38:42.782 | INFO | } | 20260505 06:38:42.782 | INFO | Perf report for the last 5.13957 seconds: | 20260505 06:38:42.782 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:38:42.782 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:42.782 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:42.782 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:42.782 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 06:38:42.782 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:42.782 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:42.782 | INFO | PRE_HQ : 30.8%, 1.0, 16.2ms, 0 | 20260505 06:38:42.782 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:42.782 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:42.782 | INFO | Uploaded 36GiB to GPU in 2.11296s (17.0377GiB/s) | 20260505 06:38:42.782 | INFO | Downloaded 34GiB to GPU in 2.03667s (16.6939GiB/s) | 20260505 06:38:47.925 | INFO | Analyzing chunk frames = [1119232,1119744) | 20260505 06:38:47.925 | INFO | MemInfo: { | 20260505 06:38:47.925 | INFO | "gpu" : " 17 GB", | 20260505 06:38:47.925 | INFO | "huge" : "", | 20260505 06:38:47.925 | INFO | "huge_shared" : " 27 GB", | 20260505 06:38:47.925 | INFO | "malloc" : " 14 GB", | 20260505 06:38:47.925 | INFO | "pinned" : " 29 GB" | 20260505 06:38:47.925 | INFO | } | 20260505 06:38:47.925 | INFO | Perf report for the last 5.14337 seconds: | 20260505 06:38:47.925 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:38:47.925 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:47.925 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:47.925 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:47.925 | INFO | ANALYSIS : 63.6%, 3.0, 99.1ms, 0 | 20260505 06:38:47.925 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:47.925 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:47.925 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 06:38:47.925 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:47.925 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:47.925 | INFO | Uploaded 36GiB to GPU in 2.12399s (16.9492GiB/s) | 20260505 06:38:47.926 | INFO | Downloaded 34GiB to GPU in 2.04796s (16.6019GiB/s) | 20260505 06:38:53.061 | INFO | Analyzing chunk frames = [1119744,1120256) | 20260505 06:38:53.061 | INFO | MemInfo: { | 20260505 06:38:53.061 | INFO | "gpu" : " 17 GB", | 20260505 06:38:53.061 | INFO | "huge" : "", | 20260505 06:38:53.061 | INFO | "huge_shared" : " 27 GB", | 20260505 06:38:53.061 | INFO | "malloc" : " 14 GB", | 20260505 06:38:53.061 | INFO | "pinned" : " 29 GB" | 20260505 06:38:53.061 | INFO | } | 20260505 06:38:53.061 | INFO | Perf report for the last 5.13579 seconds: | 20260505 06:38:53.061 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:38:53.061 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:53.061 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:53.061 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:53.061 | INFO | ANALYSIS : 63.7%, 3.0, 98.9ms, 0 | 20260505 06:38:53.061 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:53.061 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:53.061 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 06:38:53.061 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:53.061 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:53.061 | INFO | Uploaded 36GiB to GPU in 2.15535s (16.7026GiB/s) | 20260505 06:38:53.061 | INFO | Downloaded 34GiB to GPU in 2.02881s (16.7586GiB/s) | 20260505 06:38:58.200 | INFO | Analyzing chunk frames = [1120256,1120768) | 20260505 06:38:58.200 | INFO | MemInfo: { | 20260505 06:38:58.200 | INFO | "gpu" : " 17 GB", | 20260505 06:38:58.200 | INFO | "huge" : "", | 20260505 06:38:58.200 | INFO | "huge_shared" : " 27 GB", | 20260505 06:38:58.200 | INFO | "malloc" : " 14 GB", | 20260505 06:38:58.200 | INFO | "pinned" : " 29 GB" | 20260505 06:38:58.200 | INFO | } | 20260505 06:38:58.200 | INFO | Perf report for the last 5.13891 seconds: | 20260505 06:38:58.200 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:38:58.200 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:58.200 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:58.200 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:58.200 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 06:38:58.200 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:58.200 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:58.200 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 06:38:58.200 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:58.200 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:38:58.200 | INFO | Uploaded 36GiB to GPU in 2.17084s (16.5834GiB/s) | 20260505 06:38:58.200 | INFO | Downloaded 34GiB to GPU in 2.04876s (16.5954GiB/s) | 20260505 06:39:03.340 | INFO | Analyzing chunk frames = [1120768,1121280) | 20260505 06:39:03.341 | INFO | MemInfo: { | 20260505 06:39:03.341 | INFO | "gpu" : " 17 GB", | 20260505 06:39:03.341 | INFO | "huge" : "", | 20260505 06:39:03.341 | INFO | "huge_shared" : " 27 GB", | 20260505 06:39:03.341 | INFO | "malloc" : " 14 GB", | 20260505 06:39:03.341 | INFO | "pinned" : " 29 GB" | 20260505 06:39:03.341 | INFO | } | 20260505 06:39:03.341 | INFO | Perf report for the last 5.14071 seconds: | 20260505 06:39:03.341 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:39:03.341 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:03.341 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:03.341 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:03.341 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 06:39:03.341 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:03.341 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:03.341 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 06:39:03.341 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:03.341 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:03.341 | INFO | Uploaded 36GiB to GPU in 2.16944s (16.5942GiB/s) | 20260505 06:39:03.341 | INFO | Downloaded 34GiB to GPU in 2.04689s (16.6106GiB/s) | 20260505 06:39:08.478 | INFO | Analyzing chunk frames = [1121280,1121792) | 20260505 06:39:08.478 | INFO | MemInfo: { | 20260505 06:39:08.478 | INFO | "gpu" : " 17 GB", | 20260505 06:39:08.478 | INFO | "huge" : "", | 20260505 06:39:08.478 | INFO | "huge_shared" : " 27 GB", | 20260505 06:39:08.478 | INFO | "malloc" : " 14 GB", | 20260505 06:39:08.478 | INFO | "pinned" : " 29 GB" | 20260505 06:39:08.478 | INFO | } | 20260505 06:39:08.478 | INFO | Perf report for the last 5.13762 seconds: | 20260505 06:39:08.478 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:39:08.478 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:08.478 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:08.478 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:08.478 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 06:39:08.478 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:08.478 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:08.478 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260505 06:39:08.478 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:08.478 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:08.478 | INFO | Uploaded 36GiB to GPU in 2.1816s (16.5017GiB/s) | 20260505 06:39:08.479 | INFO | Downloaded 34GiB to GPU in 2.15607s (15.7694GiB/s) | 20260505 06:39:13.617 | INFO | Analyzing chunk frames = [1121792,1122304) | 20260505 06:39:13.617 | INFO | MemInfo: { | 20260505 06:39:13.617 | INFO | "gpu" : " 17 GB", | 20260505 06:39:13.617 | INFO | "huge" : "", | 20260505 06:39:13.617 | INFO | "huge_shared" : " 27 GB", | 20260505 06:39:13.618 | INFO | "malloc" : " 14 GB", | 20260505 06:39:13.618 | INFO | "pinned" : " 29 GB" | 20260505 06:39:13.618 | INFO | } | 20260505 06:39:13.618 | INFO | Perf report for the last 5.1393 seconds: | 20260505 06:39:13.618 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:39:13.618 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:13.618 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:13.618 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:13.618 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 06:39:13.618 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:13.618 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:13.618 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 06:39:13.618 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:13.618 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:13.618 | INFO | Uploaded 36GiB to GPU in 2.13025s (16.8994GiB/s) | 20260505 06:39:13.618 | INFO | Downloaded 34GiB to GPU in 2.06665s (16.4518GiB/s) | 20260505 06:39:13.618 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 06:39:13.618 | INFO | Total mem usage: 15.3 GiB | 20260505 06:39:13.618 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 06:39:13.618 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.59%) | 20260505 06:39:13.618 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 06:39:13.618 | INFO | High Water Breakdown: | 20260505 06:39:13.618 | INFO | 60.4% : 8.68 GiB ( 92.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 06:39:13.618 | INFO | 39.5% : 5.68 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 06:39:13.618 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 06:39:13.618 | INFO | ------------------------------------------ | 20260505 06:39:13.618 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:39:13.618 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 06:39:13.618 | INFO | Total mem usage: 26.5 GiB | 20260505 06:39:13.618 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 06:39:13.618 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 06:39:13.618 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 06:39:13.618 | INFO | High Water Breakdown: | 20260505 06:39:13.618 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:39:13.618 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:39:13.618 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:39:13.618 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:39:13.618 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:39:13.618 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:39:13.618 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 06:39:13.618 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 06:39:13.618 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:39:13.618 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 06:39:13.618 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 06:39:13.618 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 06:39:13.618 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 06:39:13.618 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 06:39:13.618 | INFO | ------------------------------------------ | 20260505 06:39:13.618 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:39:13.618 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 06:39:13.619 | INFO | Total mem usage: 25.6 GiB | 20260505 06:39:13.619 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 06:39:13.619 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 06:39:13.619 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 06:39:13.619 | INFO | High Water Breakdown: | 20260505 06:39:13.619 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 06:39:13.619 | INFO | ------------------------------------------ | 20260505 06:39:13.619 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:39:13.619 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 06:39:13.619 | INFO | Total mem usage: 15.7 GiB | 20260505 06:39:13.619 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 06:39:13.619 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 06:39:13.619 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260505 06:39:13.619 | INFO | High Water Breakdown: | 20260505 06:39:13.619 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 06:39:13.619 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 06:39:13.619 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 06:39:13.619 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 06:39:13.619 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 06:39:13.619 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 06:39:13.619 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 06:39:13.619 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 06:39:13.619 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 06:39:13.619 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:39:13.619 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 06:39:13.619 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:39:13.619 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 06:39:13.619 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 06:39:13.619 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 06:39:13.619 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 06:39:13.619 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:39:13.619 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 06:39:13.619 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:39:13.619 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:39:13.619 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:39:13.619 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 06:39:13.619 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 06:39:13.619 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:39:13.619 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 06:39:13.619 | INFO | ------------------------------------------ | 20260505 06:39:13.619 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:39:13.619 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 06:39:13.619 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:39:13.619 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:39:13.619 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.54e+01, 1.35e+01 / 1.74e+00 | 20260505 06:39:13.619 | INFO | PerfReport: 0.28%, AggregateStats -- 9.41e-02 / 1.02e-01, 1.16e-02 / 5.31e-03 | 20260505 06:39:13.619 | INFO | PerfReport: 11.59%, Baseline -- 3.84e+00 / 4.45e+00, 6.29e-01 / 3.20e-01 | 20260505 06:39:13.619 | INFO | PerfReport: 3.31%, Histogram -- 1.10e+00 / 1.48e+00, 2.14e-01 / 1.31e-01 | 20260505 06:39:13.619 | INFO | PerfReport: 15.44%, DME -- 4.09e+01 / 4.48e+01, 4.25e+00 / 1.22e+00 | 20260505 06:39:13.619 | INFO | PerfReport: 56.15%, FrameLabeling -- 1.86e+01 / 1.91e+01, 2.09e+00 / 1.01e+00 | 20260505 06:39:13.619 | INFO | PerfReport: 7.67%, PulseAccumulating -- 2.54e+00 / 3.13e+00, 4.57e-01 / 3.20e-01 | 20260505 06:39:13.619 | INFO | PerfReport: 4.07%, Metrics -- 1.35e+00 / 1.69e+00, 3.81e-01 / 1.88e-01 | 20260505 06:39:13.619 | INFO | PerfReport: 1.48%, PulseSerializing -- 4.89e-01 / 5.68e-01, 9.56e-02 / 4.47e-02 | 20260505 06:39:13.619 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 06:39:13.619 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:39:13.619 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:39:13.619 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.21e+01 / 0.00e+00 | 20260505 06:39:13.619 | INFO | PerfReport: 22.02%, Upload -- 2.24e+01 / 0.00e+00, 7.76e+00 / 0.00e+00 | 20260505 06:39:13.619 | INFO | PerfReport: 21.45%, Download -- 2.18e+01 / 0.00e+00, 7.62e+00 / 0.00e+00 | 20260505 06:39:13.619 | INFO | PerfReport: 32.62%, Compute -- 3.31e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 06:39:13.619 | INFO | PerfReport: 1.52%, LockUpload -- 1.54e+00 / 0.00e+00, 6.88e+00 / 0.00e+00 | 20260505 06:39:13.619 | INFO | PerfReport: 0.75%, LockDownload -- 7.59e-01 / 0.00e+00, 2.20e+00 / 0.00e+00 | 20260505 06:39:13.619 | INFO | PerfReport: 21.64%, LockCompute -- 2.20e+01 / 0.00e+00, 1.64e+01 / 0.00e+00 | 20260505 06:39:13.619 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 1.89e-02 / 0.00e+00 | 20260505 06:39:13.741 | INFO | Starting a round of Baz IO | 20260505 06:39:16.983 | INFO | Populating RTMetrics report | 20260505 06:39:16.983 | INFO | RTMetrics report populated | 20260505 06:39:16.983 | INFO | Opening tmp RTMetrics json file | 20260505 06:39:17.004 | INFO | Serializing RTMetrics to json | 20260505 06:39:17.007 | INFO | Serialization done, swapping tmp file now... | 20260505 06:39:17.011 | INFO | RTMetrics done | 20260505 06:39:17.106 | INFO | Finished a round of Baz IO: Wrote 2.39 GiB in 3.365 seconds | 20260505 06:39:17.107 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.49% / 0.04% / 67.46% | 20260505 06:39:17.107 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.78095% / 5.96046e-05% / 7.15256e-05% | 20260505 06:39:17.107 | INFO | Baz IO overall: | 20260505 06:39:17.107 | INFO | BytesWritten : 586658997046 | 20260505 06:39:17.107 | INFO | NumEvents : 277614764534 | 20260505 06:39:18.757 | INFO | Analyzing chunk frames = [1122304,1122816) | 20260505 06:39:18.757 | INFO | MemInfo: { | 20260505 06:39:18.757 | INFO | "gpu" : " 17 GB", | 20260505 06:39:18.757 | INFO | "huge" : "", | 20260505 06:39:18.757 | INFO | "huge_shared" : " 27 GB", | 20260505 06:39:18.757 | INFO | "malloc" : " 14 GB", | 20260505 06:39:18.757 | INFO | "pinned" : " 29 GB" | 20260505 06:39:18.757 | INFO | } | 20260505 06:39:18.757 | INFO | Perf report for the last 5.13967 seconds: | 20260505 06:39:18.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:39:18.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:18.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:18.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:18.757 | INFO | ANALYSIS : 64.3%, 2.9, 98.0ms, 0 | 20260505 06:39:18.757 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:18.757 | INFO | RT_METRICS : 46.5%, 1.0, 24.4ms, 0 | 20260505 06:39:18.757 | INFO | PRE_HQ : 63.2%, 1.9, 63.8ms, 0 | 20260505 06:39:18.757 | INFO | BAZWRITER : 64.8%, 4.1, 137.6ms, 0 | 20260505 06:39:18.757 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 06:39:18.757 | INFO | Uploaded 36GiB to GPU in 2.12227s (16.963GiB/s) | 20260505 06:39:18.758 | INFO | Downloaded 38GiB to GPU in 2.30711s (16.4708GiB/s) | 20260505 06:39:23.896 | INFO | Analyzing chunk frames = [1122816,1123328) | 20260505 06:39:23.896 | INFO | MemInfo: { | 20260505 06:39:23.896 | INFO | "gpu" : " 17 GB", | 20260505 06:39:23.896 | INFO | "huge" : "", | 20260505 06:39:23.896 | INFO | "huge_shared" : " 27 GB", | 20260505 06:39:23.896 | INFO | "malloc" : " 14 GB", | 20260505 06:39:23.896 | INFO | "pinned" : " 29 GB" | 20260505 06:39:23.896 | INFO | } | 20260505 06:39:23.897 | INFO | Perf report for the last 5.13927 seconds: | 20260505 06:39:23.897 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:39:23.897 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:23.897 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:23.897 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:23.897 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 06:39:23.897 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:23.897 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:23.897 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 06:39:23.897 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:23.897 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:23.897 | INFO | Uploaded 36GiB to GPU in 2.10116s (17.1334GiB/s) | 20260505 06:39:23.897 | INFO | Downloaded 34GiB to GPU in 2.03734s (16.6885GiB/s) | 20260505 06:39:29.036 | INFO | Analyzing chunk frames = [1123328,1123840) | 20260505 06:39:29.036 | INFO | MemInfo: { | 20260505 06:39:29.036 | INFO | "gpu" : " 17 GB", | 20260505 06:39:29.036 | INFO | "huge" : "", | 20260505 06:39:29.036 | INFO | "huge_shared" : " 27 GB", | 20260505 06:39:29.036 | INFO | "malloc" : " 14 GB", | 20260505 06:39:29.036 | INFO | "pinned" : " 29 GB" | 20260505 06:39:29.036 | INFO | } | 20260505 06:39:29.036 | INFO | Perf report for the last 5.13949 seconds: | 20260505 06:39:29.036 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:39:29.036 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:29.036 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:29.036 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:29.036 | INFO | ANALYSIS : 63.8%, 3.0, 99.3ms, 0 | 20260505 06:39:29.036 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:29.036 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:29.036 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 06:39:29.036 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:29.036 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:29.036 | INFO | Uploaded 36GiB to GPU in 2.13689s (16.8469GiB/s) | 20260505 06:39:29.036 | INFO | Downloaded 34GiB to GPU in 2.03654s (16.695GiB/s) | 20260505 06:39:34.175 | INFO | Analyzing chunk frames = [1123840,1124352) | 20260505 06:39:34.175 | INFO | MemInfo: { | 20260505 06:39:34.175 | INFO | "gpu" : " 17 GB", | 20260505 06:39:34.175 | INFO | "huge" : "", | 20260505 06:39:34.175 | INFO | "huge_shared" : " 27 GB", | 20260505 06:39:34.175 | INFO | "malloc" : " 14 GB", | 20260505 06:39:34.175 | INFO | "pinned" : " 29 GB" | 20260505 06:39:34.175 | INFO | } | 20260505 06:39:34.175 | INFO | Perf report for the last 5.13902 seconds: | 20260505 06:39:34.175 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:39:34.175 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:34.175 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:34.175 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:34.175 | INFO | ANALYSIS : 63.8%, 3.0, 99.0ms, 0 | 20260505 06:39:34.175 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:34.175 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:34.175 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 06:39:34.175 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:34.175 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:34.175 | INFO | Uploaded 36GiB to GPU in 2.15355s (16.7166GiB/s) | 20260505 06:39:34.175 | INFO | Downloaded 34GiB to GPU in 2.03198s (16.7325GiB/s) | 20260505 06:39:39.314 | INFO | Analyzing chunk frames = [1124352,1124864) | 20260505 06:39:39.314 | INFO | MemInfo: { | 20260505 06:39:39.314 | INFO | "gpu" : " 17 GB", | 20260505 06:39:39.314 | INFO | "huge" : "", | 20260505 06:39:39.314 | INFO | "huge_shared" : " 27 GB", | 20260505 06:39:39.314 | INFO | "malloc" : " 14 GB", | 20260505 06:39:39.314 | INFO | "pinned" : " 29 GB" | 20260505 06:39:39.314 | INFO | } | 20260505 06:39:39.314 | INFO | Perf report for the last 5.13932 seconds: | 20260505 06:39:39.314 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:39:39.314 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:39.314 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:39.314 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:39.314 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 06:39:39.315 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:39.315 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:39.315 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 06:39:39.315 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:39.315 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:39.315 | INFO | Uploaded 36GiB to GPU in 2.16674s (16.6148GiB/s) | 20260505 06:39:39.315 | INFO | Downloaded 34GiB to GPU in 2.04916s (16.5922GiB/s) | 20260505 06:39:44.455 | INFO | Analyzing chunk frames = [1124864,1125376) | 20260505 06:39:44.455 | INFO | MemInfo: { | 20260505 06:39:44.455 | INFO | "gpu" : " 17 GB", | 20260505 06:39:44.455 | INFO | "huge" : "", | 20260505 06:39:44.455 | INFO | "huge_shared" : " 27 GB", | 20260505 06:39:44.455 | INFO | "malloc" : " 14 GB", | 20260505 06:39:44.455 | INFO | "pinned" : " 29 GB" | 20260505 06:39:44.455 | INFO | } | 20260505 06:39:44.455 | INFO | Perf report for the last 5.14101 seconds: | 20260505 06:39:44.455 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:39:44.455 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:44.455 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:44.456 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:44.456 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 06:39:44.456 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:44.456 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:44.456 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 06:39:44.456 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:44.456 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:44.456 | INFO | Uploaded 36GiB to GPU in 2.14537s (16.7803GiB/s) | 20260505 06:39:44.456 | INFO | Downloaded 34GiB to GPU in 2.0397s (16.6691GiB/s) | 20260505 06:39:49.593 | INFO | Analyzing chunk frames = [1125376,1125888) | 20260505 06:39:49.593 | INFO | MemInfo: { | 20260505 06:39:49.593 | INFO | "gpu" : " 17 GB", | 20260505 06:39:49.593 | INFO | "huge" : "", | 20260505 06:39:49.593 | INFO | "huge_shared" : " 27 GB", | 20260505 06:39:49.593 | INFO | "malloc" : " 14 GB", | 20260505 06:39:49.593 | INFO | "pinned" : " 29 GB" | 20260505 06:39:49.593 | INFO | } | 20260505 06:39:49.593 | INFO | Perf report for the last 5.13784 seconds: | 20260505 06:39:49.593 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:39:49.593 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:49.593 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:49.593 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:49.593 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 06:39:49.593 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:49.593 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:49.593 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260505 06:39:49.593 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:49.593 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:49.594 | INFO | Uploaded 36GiB to GPU in 2.18927s (16.4438GiB/s) | 20260505 06:39:49.594 | INFO | Downloaded 34GiB to GPU in 2.16385s (15.7127GiB/s) | 20260505 06:39:54.732 | INFO | Analyzing chunk frames = [1125888,1126400) | 20260505 06:39:54.732 | INFO | MemInfo: { | 20260505 06:39:54.732 | INFO | "gpu" : " 17 GB", | 20260505 06:39:54.732 | INFO | "huge" : "", | 20260505 06:39:54.732 | INFO | "huge_shared" : " 27 GB", | 20260505 06:39:54.732 | INFO | "malloc" : " 14 GB", | 20260505 06:39:54.733 | INFO | "pinned" : " 29 GB" | 20260505 06:39:54.733 | INFO | } | 20260505 06:39:54.733 | INFO | Perf report for the last 5.13926 seconds: | 20260505 06:39:54.733 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:39:54.733 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:54.733 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:54.733 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:54.733 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 06:39:54.733 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:54.733 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:54.733 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 06:39:54.733 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:54.733 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:54.733 | INFO | Uploaded 36GiB to GPU in 2.17006s (16.5894GiB/s) | 20260505 06:39:54.733 | INFO | Downloaded 34GiB to GPU in 2.03802s (16.6829GiB/s) | 20260505 06:39:54.857 | INFO | Starting a round of Baz IO | 20260505 06:39:58.122 | INFO | Populating RTMetrics report | 20260505 06:39:58.123 | INFO | RTMetrics report populated | 20260505 06:39:58.123 | INFO | Opening tmp RTMetrics json file | 20260505 06:39:58.144 | INFO | Serializing RTMetrics to json | 20260505 06:39:58.145 | INFO | Serialization done, swapping tmp file now... | 20260505 06:39:58.145 | INFO | RTMetrics done | 20260505 06:39:58.233 | INFO | Finished a round of Baz IO: Wrote 2.393 GiB in 3.376 seconds | 20260505 06:39:58.234 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.45% / 0.04% / 67.51% | 20260505 06:39:58.234 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.77207% / 4.76837e-05% / 7.15256e-05% | 20260505 06:39:58.234 | INFO | Baz IO overall: | 20260505 06:39:58.234 | INFO | BytesWritten : 589228201893 | 20260505 06:39:58.234 | INFO | NumEvents : 278844660972 | 20260505 06:39:59.873 | INFO | Analyzing chunk frames = [1126400,1126912) | 20260505 06:39:59.873 | INFO | MemInfo: { | 20260505 06:39:59.873 | INFO | "gpu" : " 17 GB", | 20260505 06:39:59.873 | INFO | "huge" : "", | 20260505 06:39:59.873 | INFO | "huge_shared" : " 27 GB", | 20260505 06:39:59.873 | INFO | "malloc" : " 14 GB", | 20260505 06:39:59.873 | INFO | "pinned" : " 29 GB" | 20260505 06:39:59.873 | INFO | } | 20260505 06:39:59.873 | INFO | Perf report for the last 5.14008 seconds: | 20260505 06:39:59.873 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:39:59.873 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:59.873 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:59.873 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:59.873 | INFO | ANALYSIS : 64.8%, 2.9, 99.0ms, 0 | 20260505 06:39:59.873 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:39:59.873 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260505 06:39:59.873 | INFO | PRE_HQ : 63.7%, 1.9, 63.4ms, 0 | 20260505 06:39:59.873 | INFO | BAZWRITER : 65.0%, 4.1, 135.6ms, 0 | 20260505 06:39:59.873 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 06:39:59.873 | INFO | Uploaded 36GiB to GPU in 2.19196s (16.4236GiB/s) | 20260505 06:39:59.873 | INFO | Downloaded 38GiB to GPU in 2.32073s (16.3742GiB/s) | 20260505 06:40:05.011 | INFO | Analyzing chunk frames = [1126912,1127424) | 20260505 06:40:05.011 | INFO | MemInfo: { | 20260505 06:40:05.011 | INFO | "gpu" : " 17 GB", | 20260505 06:40:05.011 | INFO | "huge" : "", | 20260505 06:40:05.011 | INFO | "huge_shared" : " 27 GB", | 20260505 06:40:05.011 | INFO | "malloc" : " 14 GB", | 20260505 06:40:05.011 | INFO | "pinned" : " 29 GB" | 20260505 06:40:05.011 | INFO | } | 20260505 06:40:05.011 | INFO | Perf report for the last 5.1384 seconds: | 20260505 06:40:05.011 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:40:05.011 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:05.011 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:05.011 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:05.011 | INFO | ANALYSIS : 63.8%, 3.0, 99.5ms, 0 | 20260505 06:40:05.011 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:05.011 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:05.011 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 06:40:05.011 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:05.011 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:05.011 | INFO | Uploaded 36GiB to GPU in 2.08669s (17.2522GiB/s) | 20260505 06:40:05.011 | INFO | Downloaded 34GiB to GPU in 2.03461s (16.7108GiB/s) | 20260505 06:40:10.150 | INFO | Analyzing chunk frames = [1127424,1127936) | 20260505 06:40:10.150 | INFO | MemInfo: { | 20260505 06:40:10.150 | INFO | "gpu" : " 17 GB", | 20260505 06:40:10.150 | INFO | "huge" : "", | 20260505 06:40:10.150 | INFO | "huge_shared" : " 27 GB", | 20260505 06:40:10.150 | INFO | "malloc" : " 14 GB", | 20260505 06:40:10.150 | INFO | "pinned" : " 29 GB" | 20260505 06:40:10.150 | INFO | } | 20260505 06:40:10.150 | INFO | Perf report for the last 5.13935 seconds: | 20260505 06:40:10.150 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:40:10.150 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:10.150 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:10.150 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:10.150 | INFO | ANALYSIS : 63.8%, 3.0, 99.2ms, 0 | 20260505 06:40:10.150 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:10.150 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:10.150 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 06:40:10.150 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:10.151 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:10.151 | INFO | Uploaded 36GiB to GPU in 2.13908s (16.8297GiB/s) | 20260505 06:40:10.151 | INFO | Downloaded 34GiB to GPU in 2.04593s (16.6183GiB/s) | 20260505 06:40:15.290 | INFO | Analyzing chunk frames = [1127936,1128448) | 20260505 06:40:15.290 | INFO | MemInfo: { | 20260505 06:40:15.290 | INFO | "gpu" : " 17 GB", | 20260505 06:40:15.290 | INFO | "huge" : "", | 20260505 06:40:15.290 | INFO | "huge_shared" : " 27 GB", | 20260505 06:40:15.290 | INFO | "malloc" : " 14 GB", | 20260505 06:40:15.290 | INFO | "pinned" : " 29 GB" | 20260505 06:40:15.290 | INFO | } | 20260505 06:40:15.290 | INFO | Perf report for the last 5.13939 seconds: | 20260505 06:40:15.290 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:40:15.290 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:15.290 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:15.290 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:15.290 | INFO | ANALYSIS : 63.7%, 3.0, 98.9ms, 0 | 20260505 06:40:15.290 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:15.290 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:15.290 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 06:40:15.290 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:15.290 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:15.290 | INFO | Uploaded 36GiB to GPU in 2.1566s (16.6929GiB/s) | 20260505 06:40:15.290 | INFO | Downloaded 34GiB to GPU in 2.03592s (16.7GiB/s) | 20260505 06:40:20.431 | INFO | Analyzing chunk frames = [1128448,1128960) | 20260505 06:40:20.431 | INFO | MemInfo: { | 20260505 06:40:20.431 | INFO | "gpu" : " 17 GB", | 20260505 06:40:20.431 | INFO | "huge" : "", | 20260505 06:40:20.431 | INFO | "huge_shared" : " 27 GB", | 20260505 06:40:20.431 | INFO | "malloc" : " 14 GB", | 20260505 06:40:20.431 | INFO | "pinned" : " 29 GB" | 20260505 06:40:20.431 | INFO | } | 20260505 06:40:20.431 | INFO | Perf report for the last 5.14106 seconds: | 20260505 06:40:20.431 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:40:20.431 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:20.431 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:20.431 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:20.431 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 06:40:20.431 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:20.431 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:20.431 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260505 06:40:20.431 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:20.431 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:20.431 | INFO | Uploaded 36GiB to GPU in 2.16467s (16.6307GiB/s) | 20260505 06:40:20.431 | INFO | Downloaded 34GiB to GPU in 2.04113s (16.6575GiB/s) | 20260505 06:40:25.573 | INFO | Analyzing chunk frames = [1128960,1129472) | 20260505 06:40:25.573 | INFO | MemInfo: { | 20260505 06:40:25.573 | INFO | "gpu" : " 17 GB", | 20260505 06:40:25.573 | INFO | "huge" : "", | 20260505 06:40:25.573 | INFO | "huge_shared" : " 27 GB", | 20260505 06:40:25.573 | INFO | "malloc" : " 14 GB", | 20260505 06:40:25.573 | INFO | "pinned" : " 29 GB" | 20260505 06:40:25.573 | INFO | } | 20260505 06:40:25.573 | INFO | Perf report for the last 5.14231 seconds: | 20260505 06:40:25.573 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:40:25.573 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:25.573 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:25.573 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:25.573 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 06:40:25.573 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:25.573 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:25.573 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 06:40:25.573 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:25.573 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:25.573 | INFO | Uploaded 36GiB to GPU in 2.14669s (16.77GiB/s) | 20260505 06:40:25.573 | INFO | Downloaded 34GiB to GPU in 2.04406s (16.6336GiB/s) | 20260505 06:40:30.708 | INFO | Analyzing chunk frames = [1129472,1129984) | 20260505 06:40:30.708 | INFO | MemInfo: { | 20260505 06:40:30.708 | INFO | "gpu" : " 17 GB", | 20260505 06:40:30.708 | INFO | "huge" : "", | 20260505 06:40:30.708 | INFO | "huge_shared" : " 27 GB", | 20260505 06:40:30.708 | INFO | "malloc" : " 14 GB", | 20260505 06:40:30.708 | INFO | "pinned" : " 29 GB" | 20260505 06:40:30.708 | INFO | } | 20260505 06:40:30.708 | INFO | Perf report for the last 5.13468 seconds: | 20260505 06:40:30.708 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:40:30.708 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:30.708 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:30.708 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:30.708 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 06:40:30.708 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:30.708 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:30.708 | INFO | PRE_HQ : 32.9%, 1.0, 17.1ms, 0 | 20260505 06:40:30.708 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:30.708 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:30.708 | INFO | Uploaded 36GiB to GPU in 2.16131s (16.6566GiB/s) | 20260505 06:40:30.708 | INFO | Downloaded 34GiB to GPU in 2.15026s (15.8121GiB/s) | 20260505 06:40:35.847 | INFO | Analyzing chunk frames = [1129984,1130496) | 20260505 06:40:35.847 | INFO | MemInfo: { | 20260505 06:40:35.847 | INFO | "gpu" : " 17 GB", | 20260505 06:40:35.847 | INFO | "huge" : "", | 20260505 06:40:35.847 | INFO | "huge_shared" : " 27 GB", | 20260505 06:40:35.847 | INFO | "malloc" : " 14 GB", | 20260505 06:40:35.847 | INFO | "pinned" : " 29 GB" | 20260505 06:40:35.847 | INFO | } | 20260505 06:40:35.847 | INFO | Perf report for the last 5.13916 seconds: | 20260505 06:40:35.847 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:40:35.847 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:35.847 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:35.847 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:35.847 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 06:40:35.847 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:35.847 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:35.847 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 06:40:35.847 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:35.847 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:35.847 | INFO | Uploaded 36GiB to GPU in 2.18079s (16.5078GiB/s) | 20260505 06:40:35.847 | INFO | Downloaded 34GiB to GPU in 2.04807s (16.601GiB/s) | 20260505 06:40:35.847 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 06:40:35.847 | INFO | Total mem usage: 15.3 GiB | 20260505 06:40:35.847 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 06:40:35.847 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.5%) | 20260505 06:40:35.847 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 06:40:35.847 | INFO | High Water Breakdown: | 20260505 06:40:35.847 | INFO | 60.3% : 8.68 GiB ( 92.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 06:40:35.847 | INFO | 39.5% : 5.69 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 06:40:35.847 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 06:40:35.847 | INFO | ------------------------------------------ | 20260505 06:40:35.847 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:40:35.847 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 06:40:35.847 | INFO | Total mem usage: 26.5 GiB | 20260505 06:40:35.848 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 06:40:35.848 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 06:40:35.848 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 06:40:35.848 | INFO | High Water Breakdown: | 20260505 06:40:35.848 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:40:35.848 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:40:35.848 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:40:35.848 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:40:35.848 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:40:35.848 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:40:35.848 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 06:40:35.848 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 06:40:35.848 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:40:35.848 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 06:40:35.848 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 06:40:35.848 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 06:40:35.848 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 06:40:35.848 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 06:40:35.848 | INFO | ------------------------------------------ | 20260505 06:40:35.848 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:40:35.848 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 06:40:35.848 | INFO | Total mem usage: 25.6 GiB | 20260505 06:40:35.848 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 06:40:35.848 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 06:40:35.848 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 06:40:35.848 | INFO | High Water Breakdown: | 20260505 06:40:35.848 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 06:40:35.848 | INFO | ------------------------------------------ | 20260505 06:40:35.848 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:40:35.848 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 06:40:35.848 | INFO | Total mem usage: 15.7 GiB | 20260505 06:40:35.848 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 06:40:35.848 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 06:40:35.848 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260505 06:40:35.848 | INFO | High Water Breakdown: | 20260505 06:40:35.848 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 06:40:35.848 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 06:40:35.848 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 06:40:35.848 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 06:40:35.848 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 06:40:35.848 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 06:40:35.848 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 06:40:35.848 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 06:40:35.848 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 06:40:35.848 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:40:35.848 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 06:40:35.848 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:40:35.848 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 06:40:35.848 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 06:40:35.848 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 06:40:35.848 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 06:40:35.848 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:40:35.848 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 06:40:35.848 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:40:35.848 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:40:35.848 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:40:35.848 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 06:40:35.848 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 06:40:35.848 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:40:35.848 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 06:40:35.848 | INFO | ------------------------------------------ | 20260505 06:40:35.848 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:40:35.848 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 06:40:35.848 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:40:35.848 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:40:35.848 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.54e+01, 1.37e+01 / 1.70e+00 | 20260505 06:40:35.848 | INFO | PerfReport: 0.28%, AggregateStats -- 9.40e-02 / 1.03e-01, 1.23e-02 / 5.43e-03 | 20260505 06:40:35.848 | INFO | PerfReport: 11.59%, Baseline -- 3.84e+00 / 4.47e+00, 6.39e-01 / 2.99e-01 | 20260505 06:40:35.848 | INFO | PerfReport: 3.31%, Histogram -- 1.09e+00 / 1.48e+00, 2.16e-01 / 1.18e-01 | 20260505 06:40:35.848 | INFO | PerfReport: 15.47%, DME -- 4.10e+01 / 4.49e+01, 4.37e+00 / 1.11e+00 | 20260505 06:40:35.848 | INFO | PerfReport: 56.13%, FrameLabeling -- 1.86e+01 / 1.91e+01, 2.06e+00 / 8.75e-01 | 20260505 06:40:35.848 | INFO | PerfReport: 7.66%, PulseAccumulating -- 2.53e+00 / 3.11e+00, 4.54e-01 / 3.38e-01 | 20260505 06:40:35.848 | INFO | PerfReport: 4.08%, Metrics -- 1.35e+00 / 1.70e+00, 3.83e-01 / 2.15e-01 | 20260505 06:40:35.848 | INFO | PerfReport: 1.48%, PulseSerializing -- 4.89e-01 / 5.64e-01, 9.63e-02 / 4.19e-02 | 20260505 06:40:35.848 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 06:40:35.848 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:40:35.848 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:40:35.848 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.24e+01 / 0.00e+00 | 20260505 06:40:35.848 | INFO | PerfReport: 22.07%, Upload -- 2.24e+01 / 0.00e+00, 7.86e+00 / 0.00e+00 | 20260505 06:40:35.848 | INFO | PerfReport: 21.44%, Download -- 2.18e+01 / 0.00e+00, 7.73e+00 / 0.00e+00 | 20260505 06:40:35.848 | INFO | PerfReport: 32.59%, Compute -- 3.31e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 06:40:35.848 | INFO | PerfReport: 1.49%, LockUpload -- 1.51e+00 / 0.00e+00, 6.96e+00 / 0.00e+00 | 20260505 06:40:35.848 | INFO | PerfReport: 0.78%, LockDownload -- 7.90e-01 / 0.00e+00, 2.17e+00 / 0.00e+00 | 20260505 06:40:35.848 | INFO | PerfReport: 21.62%, LockCompute -- 2.19e+01 / 0.00e+00, 1.67e+01 / 0.00e+00 | 20260505 06:40:35.848 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 1.98e-02 / 0.00e+00 | 20260505 06:40:35.970 | INFO | Starting a round of Baz IO | 20260505 06:40:39.236 | INFO | Populating RTMetrics report | 20260505 06:40:39.237 | INFO | RTMetrics report populated | 20260505 06:40:39.237 | INFO | Opening tmp RTMetrics json file | 20260505 06:40:39.262 | INFO | Serializing RTMetrics to json | 20260505 06:40:39.285 | INFO | Serialization done, swapping tmp file now... | 20260505 06:40:39.286 | INFO | RTMetrics done | 20260505 06:40:39.363 | INFO | Finished a round of Baz IO: Wrote 2.39 GiB in 3.393 seconds | 20260505 06:40:39.363 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.41% / 0.04% / 67.55% | 20260505 06:40:39.363 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.73126% / 9.13938e-05% / 0.000111262% | 20260505 06:40:39.363 | INFO | Baz IO overall: | 20260505 06:40:39.363 | INFO | BytesWritten : 591794413673 | 20260505 06:40:39.363 | INFO | NumEvents : 280071922860 | 20260505 06:40:40.986 | INFO | Analyzing chunk frames = [1130496,1131008) | 20260505 06:40:40.986 | INFO | MemInfo: { | 20260505 06:40:40.986 | INFO | "gpu" : " 17 GB", | 20260505 06:40:40.988 | INFO | "huge" : "", | 20260505 06:40:40.988 | INFO | "huge_shared" : " 27 GB", | 20260505 06:40:40.988 | INFO | "malloc" : " 14 GB", | 20260505 06:40:40.988 | INFO | "pinned" : " 29 GB" | 20260505 06:40:40.988 | INFO | } | 20260505 06:40:40.988 | INFO | Perf report for the last 5.14096 seconds: | 20260505 06:40:40.988 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:40:40.988 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:40.988 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:40.988 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:40.988 | INFO | ANALYSIS : 64.8%, 2.9, 99.5ms, 0 | 20260505 06:40:40.988 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:40.988 | INFO | RT_METRICS : 46.9%, 1.0, 24.6ms, 0 | 20260505 06:40:40.988 | INFO | PRE_HQ : 63.8%, 1.9, 63.3ms, 0 | 20260505 06:40:40.988 | INFO | BAZWRITER : 65.3%, 4.1, 136.2ms, 0 | 20260505 06:40:40.988 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 06:40:40.988 | INFO | Uploaded 36GiB to GPU in 2.15772s (16.6843GiB/s) | 20260505 06:40:40.988 | INFO | Downloaded 38GiB to GPU in 2.3175s (16.397GiB/s) | 20260505 06:40:46.126 | INFO | Analyzing chunk frames = [1131008,1131520) | 20260505 06:40:46.127 | INFO | MemInfo: { | 20260505 06:40:46.127 | INFO | "gpu" : " 17 GB", | 20260505 06:40:46.127 | INFO | "huge" : "", | 20260505 06:40:46.127 | INFO | "huge_shared" : " 27 GB", | 20260505 06:40:46.127 | INFO | "malloc" : " 14 GB", | 20260505 06:40:46.127 | INFO | "pinned" : " 29 GB" | 20260505 06:40:46.127 | INFO | } | 20260505 06:40:46.127 | INFO | Perf report for the last 5.13857 seconds: | 20260505 06:40:46.127 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:40:46.127 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:46.127 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:46.127 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:46.127 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 06:40:46.127 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:46.127 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:46.127 | INFO | PRE_HQ : 30.6%, 1.0, 16.0ms, 0 | 20260505 06:40:46.127 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:46.127 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:46.127 | INFO | Uploaded 36GiB to GPU in 2.09938s (17.148GiB/s) | 20260505 06:40:46.127 | INFO | Downloaded 34GiB to GPU in 2.03382s (16.7173GiB/s) | 20260505 06:40:51.265 | INFO | Analyzing chunk frames = [1131520,1132032) | 20260505 06:40:51.265 | INFO | MemInfo: { | 20260505 06:40:51.265 | INFO | "gpu" : " 17 GB", | 20260505 06:40:51.265 | INFO | "huge" : "", | 20260505 06:40:51.265 | INFO | "huge_shared" : " 27 GB", | 20260505 06:40:51.265 | INFO | "malloc" : " 14 GB", | 20260505 06:40:51.265 | INFO | "pinned" : " 29 GB" | 20260505 06:40:51.265 | INFO | } | 20260505 06:40:51.265 | INFO | Perf report for the last 5.13871 seconds: | 20260505 06:40:51.265 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:40:51.265 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:51.265 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:51.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:51.265 | INFO | ANALYSIS : 63.7%, 3.0, 99.1ms, 0 | 20260505 06:40:51.265 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:51.265 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:51.265 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 06:40:51.265 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:51.265 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:51.266 | INFO | Uploaded 36GiB to GPU in 2.11309s (17.0367GiB/s) | 20260505 06:40:51.266 | INFO | Downloaded 34GiB to GPU in 2.04572s (16.62GiB/s) | 20260505 06:40:56.404 | INFO | Analyzing chunk frames = [1132032,1132544) | 20260505 06:40:56.404 | INFO | MemInfo: { | 20260505 06:40:56.404 | INFO | "gpu" : " 17 GB", | 20260505 06:40:56.404 | INFO | "huge" : "", | 20260505 06:40:56.404 | INFO | "huge_shared" : " 27 GB", | 20260505 06:40:56.404 | INFO | "malloc" : " 14 GB", | 20260505 06:40:56.405 | INFO | "pinned" : " 29 GB" | 20260505 06:40:56.405 | INFO | } | 20260505 06:40:56.405 | INFO | Perf report for the last 5.13918 seconds: | 20260505 06:40:56.405 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:40:56.405 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:56.405 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:56.405 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:56.405 | INFO | ANALYSIS : 63.7%, 3.0, 98.9ms, 0 | 20260505 06:40:56.405 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:56.405 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:56.405 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 06:40:56.405 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:56.405 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:40:56.405 | INFO | Uploaded 36GiB to GPU in 2.1497s (16.7465GiB/s) | 20260505 06:40:56.405 | INFO | Downloaded 34GiB to GPU in 2.0301s (16.7479GiB/s) | 20260505 06:41:01.544 | INFO | Analyzing chunk frames = [1132544,1133056) | 20260505 06:41:01.544 | INFO | MemInfo: { | 20260505 06:41:01.544 | INFO | "gpu" : " 17 GB", | 20260505 06:41:01.544 | INFO | "huge" : "", | 20260505 06:41:01.544 | INFO | "huge_shared" : " 27 GB", | 20260505 06:41:01.544 | INFO | "malloc" : " 14 GB", | 20260505 06:41:01.544 | INFO | "pinned" : " 29 GB" | 20260505 06:41:01.544 | INFO | } | 20260505 06:41:01.544 | INFO | Perf report for the last 5.13981 seconds: | 20260505 06:41:01.544 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:41:01.544 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:01.544 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:01.544 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:01.544 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 06:41:01.544 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:01.544 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:01.544 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 06:41:01.544 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:01.544 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:01.545 | INFO | Uploaded 36GiB to GPU in 2.16297s (16.6438GiB/s) | 20260505 06:41:01.545 | INFO | Downloaded 34GiB to GPU in 2.04083s (16.6599GiB/s) | 20260505 06:41:06.685 | INFO | Analyzing chunk frames = [1133056,1133568) | 20260505 06:41:06.685 | INFO | MemInfo: { | 20260505 06:41:06.685 | INFO | "gpu" : " 17 GB", | 20260505 06:41:06.685 | INFO | "huge" : "", | 20260505 06:41:06.685 | INFO | "huge_shared" : " 27 GB", | 20260505 06:41:06.685 | INFO | "malloc" : " 14 GB", | 20260505 06:41:06.685 | INFO | "pinned" : " 29 GB" | 20260505 06:41:06.685 | INFO | } | 20260505 06:41:06.685 | INFO | Perf report for the last 5.14053 seconds: | 20260505 06:41:06.685 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:41:06.685 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:06.685 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:06.685 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:06.685 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 06:41:06.685 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:06.685 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:06.685 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 06:41:06.685 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:06.685 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:06.685 | INFO | Uploaded 36GiB to GPU in 2.16243s (16.648GiB/s) | 20260505 06:41:06.685 | INFO | Downloaded 34GiB to GPU in 2.0407s (16.661GiB/s) | 20260505 06:41:11.823 | INFO | Analyzing chunk frames = [1133568,1134080) | 20260505 06:41:11.823 | INFO | MemInfo: { | 20260505 06:41:11.823 | INFO | "gpu" : " 17 GB", | 20260505 06:41:11.823 | INFO | "huge" : "", | 20260505 06:41:11.826 | INFO | "huge_shared" : " 27 GB", | 20260505 06:41:11.826 | INFO | "malloc" : " 14 GB", | 20260505 06:41:11.826 | INFO | "pinned" : " 29 GB" | 20260505 06:41:11.826 | INFO | } | 20260505 06:41:11.826 | INFO | Perf report for the last 5.14088 seconds: | 20260505 06:41:11.826 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:41:11.826 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:11.826 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:11.826 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:11.826 | INFO | ANALYSIS : 64.1%, 3.0, 100.2ms, 0 | 20260505 06:41:11.826 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:11.826 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:11.826 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260505 06:41:11.826 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:11.826 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:11.826 | INFO | Uploaded 36GiB to GPU in 2.18472s (16.4781GiB/s) | 20260505 06:41:11.826 | INFO | Downloaded 34GiB to GPU in 2.1607s (15.7356GiB/s) | 20260505 06:41:16.962 | INFO | Analyzing chunk frames = [1134080,1134592) | 20260505 06:41:16.962 | INFO | MemInfo: { | 20260505 06:41:16.962 | INFO | "gpu" : " 17 GB", | 20260505 06:41:16.962 | INFO | "huge" : "", | 20260505 06:41:16.962 | INFO | "huge_shared" : " 27 GB", | 20260505 06:41:16.962 | INFO | "malloc" : " 14 GB", | 20260505 06:41:16.962 | INFO | "pinned" : " 29 GB" | 20260505 06:41:16.962 | INFO | } | 20260505 06:41:16.962 | INFO | Perf report for the last 5.13647 seconds: | 20260505 06:41:16.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:41:16.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:16.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:16.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:16.962 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 06:41:16.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:16.962 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:16.962 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 06:41:16.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:16.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:16.963 | INFO | Uploaded 36GiB to GPU in 2.16116s (16.6577GiB/s) | 20260505 06:41:16.963 | INFO | Downloaded 34GiB to GPU in 2.04405s (16.6337GiB/s) | 20260505 06:41:17.085 | INFO | Starting a round of Baz IO | 20260505 06:41:20.345 | INFO | Populating RTMetrics report | 20260505 06:41:20.345 | INFO | RTMetrics report populated | 20260505 06:41:20.345 | INFO | Opening tmp RTMetrics json file | 20260505 06:41:20.354 | INFO | Serializing RTMetrics to json | 20260505 06:41:20.382 | INFO | Serialization done, swapping tmp file now... | 20260505 06:41:20.398 | INFO | RTMetrics done | 20260505 06:41:20.460 | INFO | Finished a round of Baz IO: Wrote 2.391 GiB in 3.374 seconds | 20260505 06:41:20.460 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.36% / 0.04% / 67.59% | 20260505 06:41:20.460 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.7152% / 5.5631e-05% / 5.5631e-05% | 20260505 06:41:20.460 | INFO | Baz IO overall: | 20260505 06:41:20.460 | INFO | BytesWritten : 594362011377 | 20260505 06:41:20.460 | INFO | NumEvents : 281300070289 | 20260505 06:41:22.101 | INFO | Analyzing chunk frames = [1134592,1135104) | 20260505 06:41:22.101 | INFO | MemInfo: { | 20260505 06:41:22.101 | INFO | "gpu" : " 17 GB", | 20260505 06:41:22.101 | INFO | "huge" : "", | 20260505 06:41:22.101 | INFO | "huge_shared" : " 27 GB", | 20260505 06:41:22.101 | INFO | "malloc" : " 14 GB", | 20260505 06:41:22.101 | INFO | "pinned" : " 29 GB" | 20260505 06:41:22.101 | INFO | } | 20260505 06:41:22.101 | INFO | Perf report for the last 5.13885 seconds: | 20260505 06:41:22.102 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:41:22.102 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:22.102 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:22.102 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:22.102 | INFO | ANALYSIS : 64.7%, 2.9, 98.8ms, 0 | 20260505 06:41:22.102 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:22.102 | INFO | RT_METRICS : 47.1%, 1.0, 24.7ms, 0 | 20260505 06:41:22.102 | INFO | PRE_HQ : 63.6%, 1.9, 64.1ms, 0 | 20260505 06:41:22.102 | INFO | BAZWRITER : 64.8%, 4.1, 136.8ms, 0 | 20260505 06:41:22.102 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 06:41:22.102 | INFO | Uploaded 36GiB to GPU in 2.16255s (16.647GiB/s) | 20260505 06:41:22.102 | INFO | Downloaded 38GiB to GPU in 2.30432s (16.4908GiB/s) | 20260505 06:41:27.241 | INFO | Analyzing chunk frames = [1135104,1135616) | 20260505 06:41:27.241 | INFO | MemInfo: { | 20260505 06:41:27.241 | INFO | "gpu" : " 17 GB", | 20260505 06:41:27.241 | INFO | "huge" : "", | 20260505 06:41:27.241 | INFO | "huge_shared" : " 27 GB", | 20260505 06:41:27.241 | INFO | "malloc" : " 14 GB", | 20260505 06:41:27.241 | INFO | "pinned" : " 29 GB" | 20260505 06:41:27.241 | INFO | } | 20260505 06:41:27.241 | INFO | Perf report for the last 5.13962 seconds: | 20260505 06:41:27.241 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:41:27.241 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:27.241 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:27.241 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:27.241 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 06:41:27.241 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:27.241 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:27.241 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 06:41:27.241 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:27.241 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:27.241 | INFO | Uploaded 36GiB to GPU in 2.10342s (17.115GiB/s) | 20260505 06:41:27.241 | INFO | Downloaded 34GiB to GPU in 2.03254s (16.7278GiB/s) | 20260505 06:41:32.380 | INFO | Analyzing chunk frames = [1135616,1136128) | 20260505 06:41:32.380 | INFO | MemInfo: { | 20260505 06:41:32.380 | INFO | "gpu" : " 17 GB", | 20260505 06:41:32.380 | INFO | "huge" : "", | 20260505 06:41:32.380 | INFO | "huge_shared" : " 27 GB", | 20260505 06:41:32.380 | INFO | "malloc" : " 14 GB", | 20260505 06:41:32.380 | INFO | "pinned" : " 29 GB" | 20260505 06:41:32.380 | INFO | } | 20260505 06:41:32.380 | INFO | Perf report for the last 5.13928 seconds: | 20260505 06:41:32.380 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:41:32.380 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:32.380 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:32.380 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:32.380 | INFO | ANALYSIS : 63.6%, 3.0, 99.0ms, 0 | 20260505 06:41:32.380 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:32.380 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:32.380 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 06:41:32.380 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:32.380 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:32.380 | INFO | Uploaded 36GiB to GPU in 2.13288s (16.8786GiB/s) | 20260505 06:41:32.381 | INFO | Downloaded 34GiB to GPU in 2.05535s (16.5422GiB/s) | 20260505 06:41:37.519 | INFO | Analyzing chunk frames = [1136128,1136640) | 20260505 06:41:37.519 | INFO | MemInfo: { | 20260505 06:41:37.519 | INFO | "gpu" : " 17 GB", | 20260505 06:41:37.519 | INFO | "huge" : "", | 20260505 06:41:37.519 | INFO | "huge_shared" : " 27 GB", | 20260505 06:41:37.519 | INFO | "malloc" : " 14 GB", | 20260505 06:41:37.519 | INFO | "pinned" : " 29 GB" | 20260505 06:41:37.519 | INFO | } | 20260505 06:41:37.519 | INFO | Perf report for the last 5.13941 seconds: | 20260505 06:41:37.519 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:41:37.519 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:37.519 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:37.519 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:37.519 | INFO | ANALYSIS : 63.6%, 3.0, 98.8ms, 0 | 20260505 06:41:37.519 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:37.520 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:37.520 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260505 06:41:37.520 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:37.520 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:37.520 | INFO | Uploaded 36GiB to GPU in 2.14164s (16.8095GiB/s) | 20260505 06:41:37.520 | INFO | Downloaded 34GiB to GPU in 2.03099s (16.7406GiB/s) | 20260505 06:41:42.658 | INFO | Analyzing chunk frames = [1136640,1137152) | 20260505 06:41:42.658 | INFO | MemInfo: { | 20260505 06:41:42.658 | INFO | "gpu" : " 17 GB", | 20260505 06:41:42.658 | INFO | "huge" : "", | 20260505 06:41:42.659 | INFO | "huge_shared" : " 27 GB", | 20260505 06:41:42.659 | INFO | "malloc" : " 14 GB", | 20260505 06:41:42.659 | INFO | "pinned" : " 29 GB" | 20260505 06:41:42.659 | INFO | } | 20260505 06:41:42.659 | INFO | Perf report for the last 5.13919 seconds: | 20260505 06:41:42.659 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:41:42.659 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:42.659 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:42.659 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:42.659 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 06:41:42.659 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:42.659 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:42.659 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 06:41:42.659 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:42.659 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:42.659 | INFO | Uploaded 36GiB to GPU in 2.17203s (16.5744GiB/s) | 20260505 06:41:42.659 | INFO | Downloaded 34GiB to GPU in 2.04983s (16.5867GiB/s) | 20260505 06:41:47.800 | INFO | Analyzing chunk frames = [1137152,1137664) | 20260505 06:41:47.800 | INFO | MemInfo: { | 20260505 06:41:47.800 | INFO | "gpu" : " 17 GB", | 20260505 06:41:47.800 | INFO | "huge" : "", | 20260505 06:41:47.800 | INFO | "huge_shared" : " 27 GB", | 20260505 06:41:47.800 | INFO | "malloc" : " 14 GB", | 20260505 06:41:47.800 | INFO | "pinned" : " 29 GB" | 20260505 06:41:47.800 | INFO | } | 20260505 06:41:47.800 | INFO | Perf report for the last 5.14115 seconds: | 20260505 06:41:47.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:41:47.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:47.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:47.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:47.800 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 06:41:47.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:47.800 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:47.800 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 06:41:47.800 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:47.800 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:47.800 | INFO | Uploaded 36GiB to GPU in 2.15996s (16.6669GiB/s) | 20260505 06:41:47.800 | INFO | Downloaded 34GiB to GPU in 2.03782s (16.6845GiB/s) | 20260505 06:41:52.937 | INFO | Analyzing chunk frames = [1137664,1138176) | 20260505 06:41:52.937 | INFO | MemInfo: { | 20260505 06:41:52.937 | INFO | "gpu" : " 17 GB", | 20260505 06:41:52.937 | INFO | "huge" : "", | 20260505 06:41:52.937 | INFO | "huge_shared" : " 27 GB", | 20260505 06:41:52.937 | INFO | "malloc" : " 14 GB", | 20260505 06:41:52.937 | INFO | "pinned" : " 29 GB" | 20260505 06:41:52.937 | INFO | } | 20260505 06:41:52.937 | INFO | Perf report for the last 5.13722 seconds: | 20260505 06:41:52.937 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:41:52.937 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:52.937 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:52.937 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:52.937 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 06:41:52.937 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:52.937 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:52.937 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260505 06:41:52.937 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:52.937 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:52.938 | INFO | Uploaded 36GiB to GPU in 2.21137s (16.2795GiB/s) | 20260505 06:41:52.938 | INFO | Downloaded 34GiB to GPU in 2.17381s (15.6407GiB/s) | 20260505 06:41:58.076 | INFO | Analyzing chunk frames = [1138176,1138688) | 20260505 06:41:58.076 | INFO | MemInfo: { | 20260505 06:41:58.076 | INFO | "gpu" : " 17 GB", | 20260505 06:41:58.076 | INFO | "huge" : "", | 20260505 06:41:58.076 | INFO | "huge_shared" : " 27 GB", | 20260505 06:41:58.076 | INFO | "malloc" : " 14 GB", | 20260505 06:41:58.077 | INFO | "pinned" : " 29 GB" | 20260505 06:41:58.077 | INFO | } | 20260505 06:41:58.077 | INFO | Perf report for the last 5.1396 seconds: | 20260505 06:41:58.077 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:41:58.077 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:58.077 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:58.077 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:58.077 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 06:41:58.077 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:58.077 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:58.077 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 06:41:58.077 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:58.077 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:41:58.077 | INFO | Uploaded 36GiB to GPU in 2.1309s (16.8943GiB/s) | 20260505 06:41:58.077 | INFO | Downloaded 34GiB to GPU in 2.05968s (16.5075GiB/s) | 20260505 06:41:58.077 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 06:41:58.077 | INFO | Total mem usage: 15.3 GiB | 20260505 06:41:58.077 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 06:41:58.077 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.55%) | 20260505 06:41:58.077 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 06:41:58.077 | INFO | High Water Breakdown: | 20260505 06:41:58.077 | INFO | 60.4% : 8.69 GiB ( 92.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 06:41:58.077 | INFO | 39.5% : 5.68 GiB ( 85.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 06:41:58.077 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 06:41:58.077 | INFO | ------------------------------------------ | 20260505 06:41:58.077 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:41:58.077 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 06:41:58.077 | INFO | Total mem usage: 26.5 GiB | 20260505 06:41:58.077 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 06:41:58.077 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 06:41:58.077 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 06:41:58.077 | INFO | High Water Breakdown: | 20260505 06:41:58.077 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:41:58.077 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:41:58.077 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:41:58.077 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:41:58.077 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:41:58.077 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:41:58.077 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 06:41:58.077 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 06:41:58.077 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:41:58.077 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 06:41:58.077 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 06:41:58.077 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 06:41:58.077 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 06:41:58.077 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 06:41:58.077 | INFO | ------------------------------------------ | 20260505 06:41:58.077 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:41:58.077 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 06:41:58.077 | INFO | Total mem usage: 25.6 GiB | 20260505 06:41:58.077 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 06:41:58.077 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 06:41:58.077 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 06:41:58.077 | INFO | High Water Breakdown: | 20260505 06:41:58.077 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 06:41:58.077 | INFO | ------------------------------------------ | 20260505 06:41:58.078 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:41:58.078 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 06:41:58.078 | INFO | Total mem usage: 15.7 GiB | 20260505 06:41:58.078 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 06:41:58.078 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 06:41:58.078 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260505 06:41:58.078 | INFO | High Water Breakdown: | 20260505 06:41:58.078 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 06:41:58.078 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 06:41:58.078 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 06:41:58.078 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 06:41:58.078 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 06:41:58.078 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 06:41:58.078 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 06:41:58.078 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 06:41:58.078 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 06:41:58.078 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:41:58.078 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 06:41:58.078 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:41:58.078 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 06:41:58.078 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 06:41:58.078 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 06:41:58.078 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 06:41:58.078 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:41:58.078 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 06:41:58.078 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:41:58.078 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:41:58.078 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:41:58.078 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 06:41:58.078 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 06:41:58.078 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:41:58.078 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 06:41:58.078 | INFO | ------------------------------------------ | 20260505 06:41:58.078 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:41:58.078 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 06:41:58.078 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:41:58.078 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:41:58.078 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.53e+01, 1.36e+01 / 1.59e+00 | 20260505 06:41:58.078 | INFO | PerfReport: 0.28%, AggregateStats -- 9.38e-02 / 1.03e-01, 1.08e-02 / 5.04e-03 | 20260505 06:41:58.078 | INFO | PerfReport: 11.61%, Baseline -- 3.84e+00 / 4.48e+00, 6.37e-01 / 2.86e-01 | 20260505 06:41:58.078 | INFO | PerfReport: 3.31%, Histogram -- 1.10e+00 / 1.47e+00, 2.16e-01 / 1.20e-01 | 20260505 06:41:58.078 | INFO | PerfReport: 15.45%, DME -- 4.09e+01 / 4.48e+01, 4.35e+00 / 1.14e+00 | 20260505 06:41:58.078 | INFO | PerfReport: 56.11%, FrameLabeling -- 1.86e+01 / 1.91e+01, 2.07e+00 / 9.69e-01 | 20260505 06:41:58.078 | INFO | PerfReport: 7.67%, PulseAccumulating -- 2.54e+00 / 3.13e+00, 4.56e-01 / 3.06e-01 | 20260505 06:41:58.078 | INFO | PerfReport: 4.08%, Metrics -- 1.35e+00 / 1.72e+00, 3.85e-01 / 1.99e-01 | 20260505 06:41:58.078 | INFO | PerfReport: 1.48%, PulseSerializing -- 4.89e-01 / 5.64e-01, 9.57e-02 / 3.89e-02 | 20260505 06:41:58.078 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 06:41:58.078 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:41:58.078 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:41:58.078 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.24e+01 / 0.00e+00 | 20260505 06:41:58.078 | INFO | PerfReport: 22.06%, Upload -- 2.24e+01 / 0.00e+00, 7.87e+00 / 0.00e+00 | 20260505 06:41:58.078 | INFO | PerfReport: 21.45%, Download -- 2.18e+01 / 0.00e+00, 7.78e+00 / 0.00e+00 | 20260505 06:41:58.078 | INFO | PerfReport: 32.57%, Compute -- 3.31e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 06:41:58.078 | INFO | PerfReport: 1.45%, LockUpload -- 1.48e+00 / 0.00e+00, 6.92e+00 / 0.00e+00 | 20260505 06:41:58.078 | INFO | PerfReport: 0.78%, LockDownload -- 7.91e-01 / 0.00e+00, 2.18e+00 / 0.00e+00 | 20260505 06:41:58.078 | INFO | PerfReport: 21.68%, LockCompute -- 2.20e+01 / 0.00e+00, 1.68e+01 / 0.00e+00 | 20260505 06:41:58.078 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.59e-02 / 0.00e+00 | 20260505 06:41:58.197 | INFO | Starting a round of Baz IO | 20260505 06:42:01.443 | INFO | Populating RTMetrics report | 20260505 06:42:01.443 | INFO | RTMetrics report populated | 20260505 06:42:01.443 | INFO | Opening tmp RTMetrics json file | 20260505 06:42:01.444 | INFO | Serializing RTMetrics to json | 20260505 06:42:01.445 | INFO | Serialization done, swapping tmp file now... | 20260505 06:42:01.446 | INFO | RTMetrics done | 20260505 06:42:01.583 | INFO | Finished a round of Baz IO: Wrote 2.391 GiB in 3.385 seconds | 20260505 06:42:01.583 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.32% / 0.04% / 67.64% | 20260505 06:42:01.583 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.68831% / 8.74201e-05% / 7.94729e-05% | 20260505 06:42:01.583 | INFO | Baz IO overall: | 20260505 06:42:01.583 | INFO | BytesWritten : 596928903173 | 20260505 06:42:01.583 | INFO | NumEvents : 282527553051 | 20260505 06:42:03.216 | INFO | Analyzing chunk frames = [1138688,1139200) | 20260505 06:42:03.216 | INFO | MemInfo: { | 20260505 06:42:03.216 | INFO | "gpu" : " 17 GB", | 20260505 06:42:03.216 | INFO | "huge" : "", | 20260505 06:42:03.216 | INFO | "huge_shared" : " 27 GB", | 20260505 06:42:03.216 | INFO | "malloc" : " 14 GB", | 20260505 06:42:03.216 | INFO | "pinned" : " 29 GB" | 20260505 06:42:03.216 | INFO | } | 20260505 06:42:03.216 | INFO | Perf report for the last 5.13918 seconds: | 20260505 06:42:03.216 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:42:03.216 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:03.216 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:03.216 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:03.216 | INFO | ANALYSIS : 64.4%, 2.9, 98.2ms, 0 | 20260505 06:42:03.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:03.216 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260505 06:42:03.216 | INFO | PRE_HQ : 63.2%, 1.9, 63.8ms, 0 | 20260505 06:42:03.216 | INFO | BAZWRITER : 65.2%, 4.2, 139.5ms, 0 | 20260505 06:42:03.216 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 06:42:03.216 | INFO | Uploaded 36GiB to GPU in 2.15124s (16.7345GiB/s) | 20260505 06:42:03.216 | INFO | Downloaded 38GiB to GPU in 2.30706s (16.4712GiB/s) | 20260505 06:42:08.355 | INFO | Analyzing chunk frames = [1139200,1139712) | 20260505 06:42:08.355 | INFO | MemInfo: { | 20260505 06:42:08.355 | INFO | "gpu" : " 17 GB", | 20260505 06:42:08.355 | INFO | "huge" : "", | 20260505 06:42:08.355 | INFO | "huge_shared" : " 27 GB", | 20260505 06:42:08.355 | INFO | "malloc" : " 14 GB", | 20260505 06:42:08.355 | INFO | "pinned" : " 29 GB" | 20260505 06:42:08.355 | INFO | } | 20260505 06:42:08.355 | INFO | Perf report for the last 5.13956 seconds: | 20260505 06:42:08.355 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:42:08.355 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:08.355 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:08.355 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:08.356 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 06:42:08.356 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:08.356 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:08.356 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 06:42:08.356 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:08.356 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:08.356 | INFO | Uploaded 36GiB to GPU in 2.10342s (17.115GiB/s) | 20260505 06:42:08.356 | INFO | Downloaded 34GiB to GPU in 2.04034s (16.6639GiB/s) | 20260505 06:42:13.495 | INFO | Analyzing chunk frames = [1139712,1140224) | 20260505 06:42:13.495 | INFO | MemInfo: { | 20260505 06:42:13.495 | INFO | "gpu" : " 17 GB", | 20260505 06:42:13.495 | INFO | "huge" : "", | 20260505 06:42:13.495 | INFO | "huge_shared" : " 27 GB", | 20260505 06:42:13.495 | INFO | "malloc" : " 14 GB", | 20260505 06:42:13.495 | INFO | "pinned" : " 29 GB" | 20260505 06:42:13.495 | INFO | } | 20260505 06:42:13.495 | INFO | Perf report for the last 5.13953 seconds: | 20260505 06:42:13.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:42:13.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:13.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:13.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:13.495 | INFO | ANALYSIS : 63.7%, 3.0, 99.1ms, 0 | 20260505 06:42:13.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:13.495 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:13.495 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 06:42:13.495 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:13.495 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:13.495 | INFO | Uploaded 36GiB to GPU in 2.12365s (16.952GiB/s) | 20260505 06:42:13.495 | INFO | Downloaded 34GiB to GPU in 2.04506s (16.6254GiB/s) | 20260505 06:42:18.634 | INFO | Analyzing chunk frames = [1140224,1140736) | 20260505 06:42:18.634 | INFO | MemInfo: { | 20260505 06:42:18.634 | INFO | "gpu" : " 17 GB", | 20260505 06:42:18.634 | INFO | "huge" : "", | 20260505 06:42:18.634 | INFO | "huge_shared" : " 27 GB", | 20260505 06:42:18.634 | INFO | "malloc" : " 14 GB", | 20260505 06:42:18.634 | INFO | "pinned" : " 29 GB" | 20260505 06:42:18.634 | INFO | } | 20260505 06:42:18.634 | INFO | Perf report for the last 5.13933 seconds: | 20260505 06:42:18.634 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:42:18.634 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:18.634 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:18.634 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:18.634 | INFO | ANALYSIS : 63.6%, 3.0, 98.8ms, 0 | 20260505 06:42:18.634 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:18.634 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:18.634 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 06:42:18.634 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:18.634 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:18.635 | INFO | Uploaded 36GiB to GPU in 2.13961s (16.8255GiB/s) | 20260505 06:42:18.635 | INFO | Downloaded 34GiB to GPU in 2.03362s (16.719GiB/s) | 20260505 06:42:23.774 | INFO | Analyzing chunk frames = [1140736,1141248) | 20260505 06:42:23.774 | INFO | MemInfo: { | 20260505 06:42:23.774 | INFO | "gpu" : " 17 GB", | 20260505 06:42:23.774 | INFO | "huge" : "", | 20260505 06:42:23.774 | INFO | "huge_shared" : " 27 GB", | 20260505 06:42:23.774 | INFO | "malloc" : " 14 GB", | 20260505 06:42:23.774 | INFO | "pinned" : " 29 GB" | 20260505 06:42:23.774 | INFO | } | 20260505 06:42:23.774 | INFO | Perf report for the last 5.13995 seconds: | 20260505 06:42:23.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:42:23.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:23.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:23.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:23.774 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 06:42:23.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:23.774 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:23.774 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260505 06:42:23.774 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:23.774 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:23.775 | INFO | Uploaded 36GiB to GPU in 2.12445s (16.9456GiB/s) | 20260505 06:42:23.775 | INFO | Downloaded 34GiB to GPU in 2.04087s (16.6595GiB/s) | 20260505 06:42:28.917 | INFO | Analyzing chunk frames = [1141248,1141760) | 20260505 06:42:28.917 | INFO | MemInfo: { | 20260505 06:42:28.917 | INFO | "gpu" : " 17 GB", | 20260505 06:42:28.917 | INFO | "huge" : "", | 20260505 06:42:28.917 | INFO | "huge_shared" : " 27 GB", | 20260505 06:42:28.917 | INFO | "malloc" : " 14 GB", | 20260505 06:42:28.917 | INFO | "pinned" : " 29 GB" | 20260505 06:42:28.917 | INFO | } | 20260505 06:42:28.917 | INFO | Perf report for the last 5.14303 seconds: | 20260505 06:42:28.917 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:42:28.917 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:28.917 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:28.917 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:28.917 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 06:42:28.917 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:28.917 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:28.917 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 06:42:28.917 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:28.917 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:28.918 | INFO | Uploaded 36GiB to GPU in 2.15171s (16.7309GiB/s) | 20260505 06:42:28.918 | INFO | Downloaded 34GiB to GPU in 2.03906s (16.6744GiB/s) | 20260505 06:42:34.052 | INFO | Analyzing chunk frames = [1141760,1142272) | 20260505 06:42:34.052 | INFO | MemInfo: { | 20260505 06:42:34.052 | INFO | "gpu" : " 17 GB", | 20260505 06:42:34.052 | INFO | "huge" : "", | 20260505 06:42:34.052 | INFO | "huge_shared" : " 27 GB", | 20260505 06:42:34.052 | INFO | "malloc" : " 14 GB", | 20260505 06:42:34.053 | INFO | "pinned" : " 29 GB" | 20260505 06:42:34.053 | INFO | } | 20260505 06:42:34.053 | INFO | Perf report for the last 5.1353 seconds: | 20260505 06:42:34.053 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:42:34.053 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:34.053 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:34.053 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:34.053 | INFO | ANALYSIS : 64.5%, 3.0, 100.7ms, 0 | 20260505 06:42:34.053 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:34.053 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:34.053 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 06:42:34.053 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:34.053 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:34.053 | INFO | Uploaded 36GiB to GPU in 2.20331s (16.3391GiB/s) | 20260505 06:42:34.053 | INFO | Downloaded 34GiB to GPU in 2.16556s (15.7003GiB/s) | 20260505 06:42:39.195 | INFO | Analyzing chunk frames = [1142272,1142784) | 20260505 06:42:39.195 | INFO | MemInfo: { | 20260505 06:42:39.195 | INFO | "gpu" : " 17 GB", | 20260505 06:42:39.195 | INFO | "huge" : "", | 20260505 06:42:39.195 | INFO | "huge_shared" : " 27 GB", | 20260505 06:42:39.195 | INFO | "malloc" : " 14 GB", | 20260505 06:42:39.195 | INFO | "pinned" : " 29 GB" | 20260505 06:42:39.195 | INFO | } | 20260505 06:42:39.195 | INFO | Perf report for the last 5.14235 seconds: | 20260505 06:42:39.195 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:42:39.195 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:39.195 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:39.195 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:39.195 | INFO | ANALYSIS : 64.4%, 3.0, 100.6ms, 0 | 20260505 06:42:39.195 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:39.195 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:39.195 | INFO | PRE_HQ : 30.5%, 1.0, 15.9ms, 0 | 20260505 06:42:39.195 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:39.195 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:39.195 | INFO | Uploaded 36GiB to GPU in 2.17179s (16.5762GiB/s) | 20260505 06:42:39.195 | INFO | Downloaded 34GiB to GPU in 2.03663s (16.6942GiB/s) | 20260505 06:42:39.316 | INFO | Starting a round of Baz IO | 20260505 06:42:42.577 | INFO | Populating RTMetrics report | 20260505 06:42:42.577 | INFO | RTMetrics report populated | 20260505 06:42:42.577 | INFO | Opening tmp RTMetrics json file | 20260505 06:42:42.618 | INFO | Serializing RTMetrics to json | 20260505 06:42:42.656 | INFO | Serialization done, swapping tmp file now... | 20260505 06:42:42.676 | INFO | RTMetrics done | 20260505 06:42:42.692 | INFO | Finished a round of Baz IO: Wrote 2.397 GiB in 3.375 seconds | 20260505 06:42:42.692 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.28% / 0.04% / 67.68% | 20260505 06:42:42.692 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.68841% / 5.16574e-05% / 7.54992e-05% | 20260505 06:42:42.692 | INFO | Baz IO overall: | 20260505 06:42:42.692 | INFO | BytesWritten : 599502430870 | 20260505 06:42:42.692 | INFO | NumEvents : 283758458605 | 20260505 06:42:44.332 | INFO | Analyzing chunk frames = [1142784,1143296) | 20260505 06:42:44.332 | INFO | MemInfo: { | 20260505 06:42:44.332 | INFO | "gpu" : " 17 GB", | 20260505 06:42:44.332 | INFO | "huge" : "", | 20260505 06:42:44.332 | INFO | "huge_shared" : " 27 GB", | 20260505 06:42:44.332 | INFO | "malloc" : " 14 GB", | 20260505 06:42:44.332 | INFO | "pinned" : " 29 GB" | 20260505 06:42:44.332 | INFO | } | 20260505 06:42:44.332 | INFO | Perf report for the last 5.13677 seconds: | 20260505 06:42:44.332 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:42:44.332 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:44.332 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:44.332 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:44.332 | INFO | ANALYSIS : 64.7%, 2.9, 99.0ms, 0 | 20260505 06:42:44.332 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:44.332 | INFO | RT_METRICS : 47.9%, 1.0, 25.1ms, 0 | 20260505 06:42:44.332 | INFO | PRE_HQ : 63.5%, 1.9, 63.9ms, 0 | 20260505 06:42:44.332 | INFO | BAZWRITER : 65.0%, 4.1, 136.6ms, 0 | 20260505 06:42:44.332 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 06:42:44.332 | INFO | Uploaded 36GiB to GPU in 2.15229s (16.7263GiB/s) | 20260505 06:42:44.332 | INFO | Downloaded 38GiB to GPU in 2.29944s (16.5258GiB/s) | 20260505 06:42:49.470 | INFO | Analyzing chunk frames = [1143296,1143808) | 20260505 06:42:49.470 | INFO | MemInfo: { | 20260505 06:42:49.470 | INFO | "gpu" : " 17 GB", | 20260505 06:42:49.470 | INFO | "huge" : "", | 20260505 06:42:49.470 | INFO | "huge_shared" : " 27 GB", | 20260505 06:42:49.470 | INFO | "malloc" : " 14 GB", | 20260505 06:42:49.470 | INFO | "pinned" : " 29 GB" | 20260505 06:42:49.470 | INFO | } | 20260505 06:42:49.470 | INFO | Perf report for the last 5.13842 seconds: | 20260505 06:42:49.470 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:42:49.470 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:49.470 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:49.470 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:49.470 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 06:42:49.470 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:49.470 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:49.470 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 06:42:49.470 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:49.470 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:49.470 | INFO | Uploaded 36GiB to GPU in 2.1042s (17.1087GiB/s) | 20260505 06:42:49.470 | INFO | Downloaded 34GiB to GPU in 2.03579s (16.7011GiB/s) | 20260505 06:42:54.610 | INFO | Analyzing chunk frames = [1143808,1144320) | 20260505 06:42:54.610 | INFO | MemInfo: { | 20260505 06:42:54.610 | INFO | "gpu" : " 17 GB", | 20260505 06:42:54.610 | INFO | "huge" : "", | 20260505 06:42:54.610 | INFO | "huge_shared" : " 27 GB", | 20260505 06:42:54.610 | INFO | "malloc" : " 14 GB", | 20260505 06:42:54.610 | INFO | "pinned" : " 29 GB" | 20260505 06:42:54.610 | INFO | } | 20260505 06:42:54.610 | INFO | Perf report for the last 5.13953 seconds: | 20260505 06:42:54.610 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:42:54.610 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:54.610 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:54.610 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:54.610 | INFO | ANALYSIS : 63.7%, 3.0, 99.1ms, 0 | 20260505 06:42:54.610 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:54.610 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:54.610 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 06:42:54.610 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:54.610 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:54.610 | INFO | Uploaded 36GiB to GPU in 2.12712s (16.9243GiB/s) | 20260505 06:42:54.610 | INFO | Downloaded 34GiB to GPU in 2.04225s (16.6483GiB/s) | 20260505 06:42:59.749 | INFO | Analyzing chunk frames = [1144320,1144832) | 20260505 06:42:59.749 | INFO | MemInfo: { | 20260505 06:42:59.749 | INFO | "gpu" : " 17 GB", | 20260505 06:42:59.749 | INFO | "huge" : "", | 20260505 06:42:59.749 | INFO | "huge_shared" : " 27 GB", | 20260505 06:42:59.749 | INFO | "malloc" : " 14 GB", | 20260505 06:42:59.749 | INFO | "pinned" : " 29 GB" | 20260505 06:42:59.749 | INFO | } | 20260505 06:42:59.749 | INFO | Perf report for the last 5.13903 seconds: | 20260505 06:42:59.749 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:42:59.749 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:59.749 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:59.749 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:59.749 | INFO | ANALYSIS : 63.6%, 3.0, 98.8ms, 0 | 20260505 06:42:59.749 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:59.749 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:59.749 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 06:42:59.749 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:59.749 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:42:59.749 | INFO | Uploaded 36GiB to GPU in 2.14787s (16.7608GiB/s) | 20260505 06:42:59.749 | INFO | Downloaded 34GiB to GPU in 2.02779s (16.767GiB/s) | 20260505 06:43:04.888 | INFO | Analyzing chunk frames = [1144832,1145344) | 20260505 06:43:04.888 | INFO | MemInfo: { | 20260505 06:43:04.888 | INFO | "gpu" : " 17 GB", | 20260505 06:43:04.888 | INFO | "huge" : "", | 20260505 06:43:04.888 | INFO | "huge_shared" : " 27 GB", | 20260505 06:43:04.888 | INFO | "malloc" : " 14 GB", | 20260505 06:43:04.888 | INFO | "pinned" : " 29 GB" | 20260505 06:43:04.888 | INFO | } | 20260505 06:43:04.889 | INFO | Perf report for the last 5.13972 seconds: | 20260505 06:43:04.889 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:43:04.889 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:04.889 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:04.889 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:04.889 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 06:43:04.889 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:04.889 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:04.889 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260505 06:43:04.889 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:04.889 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:04.889 | INFO | Uploaded 36GiB to GPU in 2.16701s (16.6128GiB/s) | 20260505 06:43:04.889 | INFO | Downloaded 34GiB to GPU in 2.04168s (16.653GiB/s) | 20260505 06:43:10.033 | INFO | Analyzing chunk frames = [1145344,1145856) | 20260505 06:43:10.033 | INFO | MemInfo: { | 20260505 06:43:10.033 | INFO | "gpu" : " 17 GB", | 20260505 06:43:10.033 | INFO | "huge" : "", | 20260505 06:43:10.033 | INFO | "huge_shared" : " 27 GB", | 20260505 06:43:10.033 | INFO | "malloc" : " 14 GB", | 20260505 06:43:10.033 | INFO | "pinned" : " 29 GB" | 20260505 06:43:10.033 | INFO | } | 20260505 06:43:10.033 | INFO | Perf report for the last 5.14406 seconds: | 20260505 06:43:10.033 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:43:10.033 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:10.033 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:10.033 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:10.033 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 06:43:10.033 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:10.033 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:10.033 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 06:43:10.033 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:10.033 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:10.033 | INFO | Uploaded 36GiB to GPU in 2.14868s (16.7544GiB/s) | 20260505 06:43:10.033 | INFO | Downloaded 34GiB to GPU in 2.03986s (16.6678GiB/s) | 20260505 06:43:15.170 | INFO | Analyzing chunk frames = [1145856,1146368) | 20260505 06:43:15.170 | INFO | MemInfo: { | 20260505 06:43:15.170 | INFO | "gpu" : " 17 GB", | 20260505 06:43:15.170 | INFO | "huge" : "", | 20260505 06:43:15.170 | INFO | "huge_shared" : " 27 GB", | 20260505 06:43:15.170 | INFO | "malloc" : " 14 GB", | 20260505 06:43:15.170 | INFO | "pinned" : " 29 GB" | 20260505 06:43:15.170 | INFO | } | 20260505 06:43:15.170 | INFO | Perf report for the last 5.13711 seconds: | 20260505 06:43:15.170 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:43:15.170 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:15.170 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:15.170 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:15.170 | INFO | ANALYSIS : 64.9%, 3.0, 101.4ms, 0 | 20260505 06:43:15.170 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:15.170 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:15.170 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260505 06:43:15.170 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:15.170 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:15.170 | INFO | Uploaded 36GiB to GPU in 2.24585s (16.0296GiB/s) | 20260505 06:43:15.170 | INFO | Downloaded 34GiB to GPU in 2.15658s (15.7657GiB/s) | 20260505 06:43:20.311 | INFO | Analyzing chunk frames = [1146368,1146880) | 20260505 06:43:20.311 | INFO | MemInfo: { | 20260505 06:43:20.311 | INFO | "gpu" : " 17 GB", | 20260505 06:43:20.311 | INFO | "huge" : "", | 20260505 06:43:20.311 | INFO | "huge_shared" : " 27 GB", | 20260505 06:43:20.311 | INFO | "malloc" : " 14 GB", | 20260505 06:43:20.311 | INFO | "pinned" : " 29 GB" | 20260505 06:43:20.311 | INFO | } | 20260505 06:43:20.311 | INFO | Perf report for the last 5.14104 seconds: | 20260505 06:43:20.311 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:43:20.311 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:20.311 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:20.311 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:20.311 | INFO | ANALYSIS : 64.2%, 3.0, 100.4ms, 0 | 20260505 06:43:20.311 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:20.311 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:20.311 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 06:43:20.311 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:20.311 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:20.311 | INFO | Uploaded 36GiB to GPU in 2.15844s (16.6787GiB/s) | 20260505 06:43:20.311 | INFO | Downloaded 34GiB to GPU in 2.0407s (16.6609GiB/s) | 20260505 06:43:20.311 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 06:43:20.311 | INFO | Total mem usage: 15.3 GiB | 20260505 06:43:20.311 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 06:43:20.311 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.31%) | 20260505 06:43:20.311 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 06:43:20.311 | INFO | High Water Breakdown: | 20260505 06:43:20.311 | INFO | 60.3% : 8.7 GiB ( 93% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 06:43:20.311 | INFO | 39.5% : 5.7 GiB ( 85.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 06:43:20.311 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 06:43:20.311 | INFO | ------------------------------------------ | 20260505 06:43:20.311 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:43:20.311 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 06:43:20.311 | INFO | Total mem usage: 26.5 GiB | 20260505 06:43:20.311 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 06:43:20.311 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 06:43:20.311 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 06:43:20.311 | INFO | High Water Breakdown: | 20260505 06:43:20.311 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:43:20.311 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:43:20.311 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:43:20.311 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:43:20.311 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:43:20.312 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:43:20.312 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 06:43:20.312 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 06:43:20.312 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:43:20.312 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 06:43:20.312 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 06:43:20.312 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 06:43:20.312 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 06:43:20.312 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 06:43:20.312 | INFO | ------------------------------------------ | 20260505 06:43:20.312 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:43:20.312 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 06:43:20.312 | INFO | Total mem usage: 25.6 GiB | 20260505 06:43:20.312 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 06:43:20.312 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 06:43:20.312 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 06:43:20.312 | INFO | High Water Breakdown: | 20260505 06:43:20.312 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 06:43:20.312 | INFO | ------------------------------------------ | 20260505 06:43:20.312 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:43:20.312 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 06:43:20.312 | INFO | Total mem usage: 15.7 GiB | 20260505 06:43:20.312 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 06:43:20.312 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 06:43:20.312 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260505 06:43:20.312 | INFO | High Water Breakdown: | 20260505 06:43:20.312 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 06:43:20.312 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 06:43:20.312 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 06:43:20.312 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 06:43:20.312 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 06:43:20.312 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 06:43:20.312 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 06:43:20.312 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 06:43:20.312 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 06:43:20.312 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:43:20.312 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 06:43:20.312 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:43:20.312 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 06:43:20.312 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 06:43:20.312 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 06:43:20.312 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 06:43:20.312 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:43:20.312 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 06:43:20.312 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:43:20.312 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:43:20.312 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:43:20.312 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 06:43:20.312 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 06:43:20.312 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:43:20.312 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 06:43:20.312 | INFO | ------------------------------------------ | 20260505 06:43:20.312 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:43:20.312 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 06:43:20.312 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:43:20.312 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:43:20.312 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.53e+01, 1.37e+01 / 1.70e+00 | 20260505 06:43:20.312 | INFO | PerfReport: 0.28%, AggregateStats -- 9.39e-02 / 1.04e-01, 1.06e-02 / 5.12e-03 | 20260505 06:43:20.312 | INFO | PerfReport: 11.58%, Baseline -- 3.83e+00 / 4.45e+00, 6.38e-01 / 3.31e-01 | 20260505 06:43:20.312 | INFO | PerfReport: 3.31%, Histogram -- 1.10e+00 / 1.48e+00, 2.17e-01 / 9.95e-02 | 20260505 06:43:20.312 | INFO | PerfReport: 15.45%, DME -- 4.09e+01 / 4.48e+01, 4.29e+00 / 1.25e+00 | 20260505 06:43:20.312 | INFO | PerfReport: 56.18%, FrameLabeling -- 1.86e+01 / 1.91e+01, 2.10e+00 / 9.16e-01 | 20260505 06:43:20.312 | INFO | PerfReport: 7.64%, PulseAccumulating -- 2.53e+00 / 3.13e+00, 4.49e-01 / 3.03e-01 | 20260505 06:43:20.312 | INFO | PerfReport: 4.07%, Metrics -- 1.35e+00 / 1.70e+00, 3.84e-01 / 1.49e-01 | 20260505 06:43:20.312 | INFO | PerfReport: 1.48%, PulseSerializing -- 4.89e-01 / 5.67e-01, 9.56e-02 / 4.20e-02 | 20260505 06:43:20.312 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 06:43:20.313 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:43:20.313 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:43:20.313 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.24e+01 / 0.00e+00 | 20260505 06:43:20.313 | INFO | PerfReport: 22.05%, Upload -- 2.24e+01 / 0.00e+00, 7.91e+00 / 0.00e+00 | 20260505 06:43:20.313 | INFO | PerfReport: 21.39%, Download -- 2.17e+01 / 0.00e+00, 7.77e+00 / 0.00e+00 | 20260505 06:43:20.313 | INFO | PerfReport: 32.56%, Compute -- 3.31e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 06:43:20.313 | INFO | PerfReport: 1.49%, LockUpload -- 1.52e+00 / 0.00e+00, 7.02e+00 / 0.00e+00 | 20260505 06:43:20.313 | INFO | PerfReport: 0.76%, LockDownload -- 7.70e-01 / 0.00e+00, 2.15e+00 / 0.00e+00 | 20260505 06:43:20.313 | INFO | PerfReport: 21.74%, LockCompute -- 2.21e+01 / 0.00e+00, 1.67e+01 / 0.00e+00 | 20260505 06:43:20.313 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.53e-02 / 0.00e+00 | 20260505 06:43:20.430 | INFO | Starting a round of Baz IO | 20260505 06:43:23.731 | INFO | Populating RTMetrics report | 20260505 06:43:23.732 | INFO | RTMetrics report populated | 20260505 06:43:23.732 | INFO | Opening tmp RTMetrics json file | 20260505 06:43:23.734 | INFO | Serializing RTMetrics to json | 20260505 06:43:23.757 | INFO | Serialization done, swapping tmp file now... | 20260505 06:43:23.758 | INFO | RTMetrics done | 20260505 06:43:23.848 | INFO | Finished a round of Baz IO: Wrote 2.396 GiB in 3.418 seconds | 20260505 06:43:23.849 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.23% / 0.04% / 67.72% | 20260505 06:43:23.849 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.66029% / 3.97364e-05% / 7.54992e-05% | 20260505 06:43:23.849 | INFO | Baz IO overall: | 20260505 06:43:23.849 | INFO | BytesWritten : 602075297602 | 20260505 06:43:23.849 | INFO | NumEvents : 284988323371 | 20260505 06:43:25.452 | INFO | Analyzing chunk frames = [1146880,1147392) | 20260505 06:43:25.452 | INFO | MemInfo: { | 20260505 06:43:25.452 | INFO | "gpu" : " 17 GB", | 20260505 06:43:25.452 | INFO | "huge" : "", | 20260505 06:43:25.452 | INFO | "huge_shared" : " 27 GB", | 20260505 06:43:25.452 | INFO | "malloc" : " 14 GB", | 20260505 06:43:25.452 | INFO | "pinned" : " 29 GB" | 20260505 06:43:25.452 | INFO | } | 20260505 06:43:25.452 | INFO | Perf report for the last 5.1416 seconds: | 20260505 06:43:25.452 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:43:25.452 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:25.452 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:25.452 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:25.452 | INFO | ANALYSIS : 65.4%, 2.9, 100.5ms, 0 | 20260505 06:43:25.452 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:25.452 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260505 06:43:25.452 | INFO | PRE_HQ : 64.2%, 1.9, 63.5ms, 0 | 20260505 06:43:25.453 | INFO | BAZWRITER : 65.8%, 4.1, 137.1ms, 0 | 20260505 06:43:25.453 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 06:43:25.453 | INFO | Uploaded 36GiB to GPU in 2.2574s (15.9475GiB/s) | 20260505 06:43:25.453 | INFO | Downloaded 38GiB to GPU in 2.32709s (16.3294GiB/s) | 20260505 06:43:30.585 | INFO | Analyzing chunk frames = [1147392,1147904) | 20260505 06:43:30.585 | INFO | MemInfo: { | 20260505 06:43:30.585 | INFO | "gpu" : " 17 GB", | 20260505 06:43:30.585 | INFO | "huge" : "", | 20260505 06:43:30.585 | INFO | "huge_shared" : " 27 GB", | 20260505 06:43:30.585 | INFO | "malloc" : " 14 GB", | 20260505 06:43:30.585 | INFO | "pinned" : " 29 GB" | 20260505 06:43:30.585 | INFO | } | 20260505 06:43:30.585 | INFO | Perf report for the last 5.13259 seconds: | 20260505 06:43:30.585 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:43:30.585 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:30.585 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:30.585 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:30.585 | INFO | ANALYSIS : 64.1%, 3.0, 100.0ms, 0 | 20260505 06:43:30.585 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:30.585 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:30.585 | INFO | PRE_HQ : 30.8%, 1.0, 15.9ms, 0 | 20260505 06:43:30.585 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:30.585 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:30.585 | INFO | Uploaded 36GiB to GPU in 2.12201s (16.9651GiB/s) | 20260505 06:43:30.585 | INFO | Downloaded 34GiB to GPU in 2.04797s (16.6018GiB/s) | 20260505 06:43:35.725 | INFO | Analyzing chunk frames = [1147904,1148416) | 20260505 06:43:35.725 | INFO | MemInfo: { | 20260505 06:43:35.725 | INFO | "gpu" : " 17 GB", | 20260505 06:43:35.725 | INFO | "huge" : "", | 20260505 06:43:35.725 | INFO | "huge_shared" : " 27 GB", | 20260505 06:43:35.725 | INFO | "malloc" : " 14 GB", | 20260505 06:43:35.725 | INFO | "pinned" : " 29 GB" | 20260505 06:43:35.725 | INFO | } | 20260505 06:43:35.725 | INFO | Perf report for the last 5.13985 seconds: | 20260505 06:43:35.725 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:43:35.725 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:35.725 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:35.725 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:35.725 | INFO | ANALYSIS : 63.8%, 3.0, 99.3ms, 0 | 20260505 06:43:35.725 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:35.725 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:35.725 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 06:43:35.725 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:35.725 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:35.725 | INFO | Uploaded 36GiB to GPU in 2.12653s (16.929GiB/s) | 20260505 06:43:35.725 | INFO | Downloaded 34GiB to GPU in 2.0398s (16.6683GiB/s) | 20260505 06:43:40.864 | INFO | Analyzing chunk frames = [1148416,1148928) | 20260505 06:43:40.864 | INFO | MemInfo: { | 20260505 06:43:40.864 | INFO | "gpu" : " 17 GB", | 20260505 06:43:40.864 | INFO | "huge" : "", | 20260505 06:43:40.864 | INFO | "huge_shared" : " 27 GB", | 20260505 06:43:40.864 | INFO | "malloc" : " 14 GB", | 20260505 06:43:40.864 | INFO | "pinned" : " 29 GB" | 20260505 06:43:40.864 | INFO | } | 20260505 06:43:40.864 | INFO | Perf report for the last 5.1389 seconds: | 20260505 06:43:40.864 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:43:40.864 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:40.864 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:40.864 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:40.864 | INFO | ANALYSIS : 63.6%, 3.0, 98.8ms, 0 | 20260505 06:43:40.864 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:40.864 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:40.864 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 06:43:40.864 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:40.864 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:40.864 | INFO | Uploaded 36GiB to GPU in 2.1502s (16.7426GiB/s) | 20260505 06:43:40.864 | INFO | Downloaded 34GiB to GPU in 2.02609s (16.7811GiB/s) | 20260505 06:43:46.003 | INFO | Analyzing chunk frames = [1148928,1149440) | 20260505 06:43:46.003 | INFO | MemInfo: { | 20260505 06:43:46.003 | INFO | "gpu" : " 17 GB", | 20260505 06:43:46.003 | INFO | "huge" : "", | 20260505 06:43:46.003 | INFO | "huge_shared" : " 27 GB", | 20260505 06:43:46.003 | INFO | "malloc" : " 14 GB", | 20260505 06:43:46.003 | INFO | "pinned" : " 29 GB" | 20260505 06:43:46.003 | INFO | } | 20260505 06:43:46.003 | INFO | Perf report for the last 5.13955 seconds: | 20260505 06:43:46.003 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:43:46.003 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:46.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:46.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:46.003 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 06:43:46.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:46.003 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:46.003 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 06:43:46.003 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:46.003 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:46.004 | INFO | Uploaded 36GiB to GPU in 2.15949s (16.6706GiB/s) | 20260505 06:43:46.004 | INFO | Downloaded 34GiB to GPU in 2.04617s (16.6164GiB/s) | 20260505 06:43:51.144 | INFO | Analyzing chunk frames = [1149440,1149952) | 20260505 06:43:51.144 | INFO | MemInfo: { | 20260505 06:43:51.144 | INFO | "gpu" : " 17 GB", | 20260505 06:43:51.144 | INFO | "huge" : "", | 20260505 06:43:51.144 | INFO | "huge_shared" : " 27 GB", | 20260505 06:43:51.144 | INFO | "malloc" : " 14 GB", | 20260505 06:43:51.144 | INFO | "pinned" : " 29 GB" | 20260505 06:43:51.144 | INFO | } | 20260505 06:43:51.144 | INFO | Perf report for the last 5.14086 seconds: | 20260505 06:43:51.144 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:43:51.144 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:51.144 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:51.144 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:51.144 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 06:43:51.144 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:51.144 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:51.144 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260505 06:43:51.144 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:51.144 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:51.144 | INFO | Uploaded 36GiB to GPU in 2.14913s (16.751GiB/s) | 20260505 06:43:51.144 | INFO | Downloaded 34GiB to GPU in 2.03963s (16.6697GiB/s) | 20260505 06:43:56.282 | INFO | Analyzing chunk frames = [1149952,1150464) | 20260505 06:43:56.282 | INFO | MemInfo: { | 20260505 06:43:56.282 | INFO | "gpu" : " 17 GB", | 20260505 06:43:56.282 | INFO | "huge" : "", | 20260505 06:43:56.282 | INFO | "huge_shared" : " 27 GB", | 20260505 06:43:56.282 | INFO | "malloc" : " 14 GB", | 20260505 06:43:56.282 | INFO | "pinned" : " 29 GB" | 20260505 06:43:56.282 | INFO | } | 20260505 06:43:56.282 | INFO | Perf report for the last 5.13784 seconds: | 20260505 06:43:56.282 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:43:56.282 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:56.282 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:56.282 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:56.282 | INFO | ANALYSIS : 64.5%, 3.0, 100.7ms, 0 | 20260505 06:43:56.282 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:56.282 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:56.282 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260505 06:43:56.282 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:56.282 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:43:56.282 | INFO | Uploaded 36GiB to GPU in 2.21974s (16.2181GiB/s) | 20260505 06:43:56.282 | INFO | Downloaded 34GiB to GPU in 2.16297s (15.7192GiB/s) | 20260505 06:44:01.421 | INFO | Analyzing chunk frames = [1150464,1150976) | 20260505 06:44:01.421 | INFO | MemInfo: { | 20260505 06:44:01.421 | INFO | "gpu" : " 17 GB", | 20260505 06:44:01.421 | INFO | "huge" : "", | 20260505 06:44:01.421 | INFO | "huge_shared" : " 27 GB", | 20260505 06:44:01.421 | INFO | "malloc" : " 14 GB", | 20260505 06:44:01.421 | INFO | "pinned" : " 29 GB" | 20260505 06:44:01.421 | INFO | } | 20260505 06:44:01.421 | INFO | Perf report for the last 5.13919 seconds: | 20260505 06:44:01.421 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:44:01.421 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:01.421 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:01.421 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:01.421 | INFO | ANALYSIS : 64.3%, 3.0, 100.5ms, 0 | 20260505 06:44:01.421 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:01.421 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:01.421 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 06:44:01.421 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:01.421 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:01.422 | INFO | Uploaded 36GiB to GPU in 2.16297s (16.6438GiB/s) | 20260505 06:44:01.422 | INFO | Downloaded 34GiB to GPU in 2.04441s (16.6307GiB/s) | 20260505 06:44:01.545 | INFO | Starting a round of Baz IO | 20260505 06:44:04.784 | INFO | Populating RTMetrics report | 20260505 06:44:04.784 | INFO | RTMetrics report populated | 20260505 06:44:04.784 | INFO | Opening tmp RTMetrics json file | 20260505 06:44:04.785 | INFO | Serializing RTMetrics to json | 20260505 06:44:04.801 | INFO | Serialization done, swapping tmp file now... | 20260505 06:44:04.810 | INFO | RTMetrics done | 20260505 06:44:04.897 | INFO | Finished a round of Baz IO: Wrote 2.398 GiB in 3.351 seconds | 20260505 06:44:04.897 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.19% / 0.04% / 67.77% | 20260505 06:44:04.897 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.65315% / 5.5631e-05% / 7.94729e-05% | 20260505 06:44:04.897 | INFO | Baz IO overall: | 20260505 06:44:04.897 | INFO | BytesWritten : 604650164058 | 20260505 06:44:04.897 | INFO | NumEvents : 286220022255 | 20260505 06:44:06.560 | INFO | Analyzing chunk frames = [1150976,1151488) | 20260505 06:44:06.560 | INFO | MemInfo: { | 20260505 06:44:06.560 | INFO | "gpu" : " 17 GB", | 20260505 06:44:06.560 | INFO | "huge" : "", | 20260505 06:44:06.560 | INFO | "huge_shared" : " 27 GB", | 20260505 06:44:06.560 | INFO | "malloc" : " 14 GB", | 20260505 06:44:06.560 | INFO | "pinned" : " 29 GB" | 20260505 06:44:06.560 | INFO | } | 20260505 06:44:06.560 | INFO | Perf report for the last 5.1392 seconds: | 20260505 06:44:06.560 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:44:06.560 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:06.560 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:06.561 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:06.561 | INFO | ANALYSIS : 64.3%, 2.9, 98.3ms, 0 | 20260505 06:44:06.561 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:06.561 | INFO | RT_METRICS : 46.4%, 1.0, 24.4ms, 0 | 20260505 06:44:06.561 | INFO | PRE_HQ : 63.2%, 1.9, 63.6ms, 0 | 20260505 06:44:06.561 | INFO | BAZWRITER : 64.5%, 4.1, 137.2ms, 0 | 20260505 06:44:06.561 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260505 06:44:06.561 | INFO | Uploaded 36GiB to GPU in 2.10671s (17.0883GiB/s) | 20260505 06:44:06.561 | INFO | Downloaded 38GiB to GPU in 2.31595s (16.4079GiB/s) | 20260505 06:44:11.700 | INFO | Analyzing chunk frames = [1151488,1152000) | 20260505 06:44:11.700 | INFO | MemInfo: { | 20260505 06:44:11.700 | INFO | "gpu" : " 17 GB", | 20260505 06:44:11.700 | INFO | "huge" : "", | 20260505 06:44:11.700 | INFO | "huge_shared" : " 27 GB", | 20260505 06:44:11.700 | INFO | "malloc" : " 14 GB", | 20260505 06:44:11.700 | INFO | "pinned" : " 29 GB" | 20260505 06:44:11.700 | INFO | } | 20260505 06:44:11.700 | INFO | Perf report for the last 5.13958 seconds: | 20260505 06:44:11.700 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:44:11.700 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:11.700 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:11.700 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:11.700 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 06:44:11.700 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:11.700 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:11.700 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 06:44:11.700 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:11.700 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:11.700 | INFO | Uploaded 36GiB to GPU in 2.09008s (17.2242GiB/s) | 20260505 06:44:11.700 | INFO | Downloaded 34GiB to GPU in 2.03899s (16.6749GiB/s) | 20260505 06:44:16.839 | INFO | Analyzing chunk frames = [1152000,1152512) | 20260505 06:44:16.839 | INFO | MemInfo: { | 20260505 06:44:16.839 | INFO | "gpu" : " 17 GB", | 20260505 06:44:16.839 | INFO | "huge" : "", | 20260505 06:44:16.839 | INFO | "huge_shared" : " 27 GB", | 20260505 06:44:16.839 | INFO | "malloc" : " 14 GB", | 20260505 06:44:16.839 | INFO | "pinned" : " 29 GB" | 20260505 06:44:16.839 | INFO | } | 20260505 06:44:16.839 | INFO | Perf report for the last 5.13929 seconds: | 20260505 06:44:16.839 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:44:16.839 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:16.839 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:16.839 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:16.839 | INFO | ANALYSIS : 63.7%, 3.0, 99.1ms, 0 | 20260505 06:44:16.839 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:16.839 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:16.839 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 06:44:16.839 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:16.840 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:16.840 | INFO | Uploaded 36GiB to GPU in 2.12365s (16.9519GiB/s) | 20260505 06:44:16.840 | INFO | Downloaded 34GiB to GPU in 2.03816s (16.6817GiB/s) | 20260505 06:44:21.978 | INFO | Analyzing chunk frames = [1152512,1153024) | 20260505 06:44:21.978 | INFO | MemInfo: { | 20260505 06:44:21.978 | INFO | "gpu" : " 17 GB", | 20260505 06:44:21.978 | INFO | "huge" : "", | 20260505 06:44:21.978 | INFO | "huge_shared" : " 27 GB", | 20260505 06:44:21.979 | INFO | "malloc" : " 14 GB", | 20260505 06:44:21.979 | INFO | "pinned" : " 29 GB" | 20260505 06:44:21.979 | INFO | } | 20260505 06:44:21.979 | INFO | Perf report for the last 5.1393 seconds: | 20260505 06:44:21.979 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:44:21.979 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:21.979 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:21.979 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:21.979 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260505 06:44:21.979 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:21.979 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:21.979 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 06:44:21.979 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:21.979 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:21.979 | INFO | Uploaded 36GiB to GPU in 2.15037s (16.7413GiB/s) | 20260505 06:44:21.979 | INFO | Downloaded 34GiB to GPU in 2.03002s (16.7486GiB/s) | 20260505 06:44:27.118 | INFO | Analyzing chunk frames = [1153024,1153536) | 20260505 06:44:27.118 | INFO | MemInfo: { | 20260505 06:44:27.118 | INFO | "gpu" : " 17 GB", | 20260505 06:44:27.118 | INFO | "huge" : "", | 20260505 06:44:27.118 | INFO | "huge_shared" : " 27 GB", | 20260505 06:44:27.118 | INFO | "malloc" : " 14 GB", | 20260505 06:44:27.118 | INFO | "pinned" : " 29 GB" | 20260505 06:44:27.118 | INFO | } | 20260505 06:44:27.118 | INFO | Perf report for the last 5.13916 seconds: | 20260505 06:44:27.118 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:44:27.118 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:27.118 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:27.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:27.118 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 06:44:27.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:27.118 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:27.118 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 06:44:27.118 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:27.118 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:27.118 | INFO | Uploaded 36GiB to GPU in 2.16919s (16.5961GiB/s) | 20260505 06:44:27.118 | INFO | Downloaded 34GiB to GPU in 2.04267s (16.6449GiB/s) | 20260505 06:44:32.259 | INFO | Analyzing chunk frames = [1153536,1154048) | 20260505 06:44:32.259 | INFO | MemInfo: { | 20260505 06:44:32.259 | INFO | "gpu" : " 17 GB", | 20260505 06:44:32.259 | INFO | "huge" : "", | 20260505 06:44:32.259 | INFO | "huge_shared" : " 27 GB", | 20260505 06:44:32.259 | INFO | "malloc" : " 14 GB", | 20260505 06:44:32.259 | INFO | "pinned" : " 29 GB" | 20260505 06:44:32.259 | INFO | } | 20260505 06:44:32.259 | INFO | Perf report for the last 5.14103 seconds: | 20260505 06:44:32.259 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:44:32.259 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:32.259 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:32.259 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:32.259 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 06:44:32.259 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:32.259 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:32.259 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 06:44:32.259 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:32.259 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:32.259 | INFO | Uploaded 36GiB to GPU in 2.15121s (16.7348GiB/s) | 20260505 06:44:32.259 | INFO | Downloaded 34GiB to GPU in 2.04233s (16.6476GiB/s) | 20260505 06:44:37.396 | INFO | Analyzing chunk frames = [1154048,1154560) | 20260505 06:44:37.396 | INFO | MemInfo: { | 20260505 06:44:37.396 | INFO | "gpu" : " 17 GB", | 20260505 06:44:37.396 | INFO | "huge" : "", | 20260505 06:44:37.396 | INFO | "huge_shared" : " 27 GB", | 20260505 06:44:37.396 | INFO | "malloc" : " 14 GB", | 20260505 06:44:37.396 | INFO | "pinned" : " 29 GB" | 20260505 06:44:37.396 | INFO | } | 20260505 06:44:37.397 | INFO | Perf report for the last 5.13772 seconds: | 20260505 06:44:37.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:44:37.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:37.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:37.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:37.397 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 06:44:37.397 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:37.397 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:37.397 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260505 06:44:37.397 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:37.397 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:37.397 | INFO | Uploaded 36GiB to GPU in 2.20418s (16.3326GiB/s) | 20260505 06:44:37.397 | INFO | Downloaded 34GiB to GPU in 2.16813s (15.6817GiB/s) | 20260505 06:44:42.536 | INFO | Analyzing chunk frames = [1154560,1155072) | 20260505 06:44:42.536 | INFO | MemInfo: { | 20260505 06:44:42.536 | INFO | "gpu" : " 17 GB", | 20260505 06:44:42.536 | INFO | "huge" : "", | 20260505 06:44:42.536 | INFO | "huge_shared" : " 27 GB", | 20260505 06:44:42.536 | INFO | "malloc" : " 14 GB", | 20260505 06:44:42.536 | INFO | "pinned" : " 29 GB" | 20260505 06:44:42.536 | INFO | } | 20260505 06:44:42.536 | INFO | Perf report for the last 5.13943 seconds: | 20260505 06:44:42.536 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:44:42.536 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:42.536 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:42.536 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:42.536 | INFO | ANALYSIS : 64.3%, 3.0, 100.5ms, 0 | 20260505 06:44:42.536 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:42.536 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:42.536 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 06:44:42.536 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:42.536 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:42.536 | INFO | Uploaded 36GiB to GPU in 2.17155s (16.578GiB/s) | 20260505 06:44:42.536 | INFO | Downloaded 34GiB to GPU in 2.04159s (16.6537GiB/s) | 20260505 06:44:42.536 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 06:44:42.536 | INFO | Total mem usage: 15.3 GiB | 20260505 06:44:42.536 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 06:44:42.536 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.58%) | 20260505 06:44:42.536 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 06:44:42.536 | INFO | High Water Breakdown: | 20260505 06:44:42.536 | INFO | 60.4% : 8.69 GiB ( 92.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 06:44:42.537 | INFO | 39.4% : 5.67 GiB ( 85.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 06:44:42.537 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 06:44:42.537 | INFO | ------------------------------------------ | 20260505 06:44:42.537 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:44:42.537 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 06:44:42.537 | INFO | Total mem usage: 26.5 GiB | 20260505 06:44:42.537 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 06:44:42.537 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 06:44:42.537 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 06:44:42.537 | INFO | High Water Breakdown: | 20260505 06:44:42.537 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:44:42.537 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:44:42.537 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:44:42.537 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:44:42.537 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:44:42.537 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:44:42.537 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 06:44:42.537 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 06:44:42.537 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:44:42.537 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 06:44:42.537 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 06:44:42.537 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 06:44:42.537 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 06:44:42.537 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 06:44:42.537 | INFO | ------------------------------------------ | 20260505 06:44:42.537 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:44:42.537 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 06:44:42.537 | INFO | Total mem usage: 25.6 GiB | 20260505 06:44:42.537 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 06:44:42.537 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 06:44:42.537 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 06:44:42.537 | INFO | High Water Breakdown: | 20260505 06:44:42.537 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 06:44:42.537 | INFO | ------------------------------------------ | 20260505 06:44:42.537 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:44:42.537 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 06:44:42.537 | INFO | Total mem usage: 15.7 GiB | 20260505 06:44:42.537 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 06:44:42.537 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 06:44:42.537 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260505 06:44:42.537 | INFO | High Water Breakdown: | 20260505 06:44:42.537 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 06:44:42.537 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 06:44:42.537 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 06:44:42.537 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 06:44:42.537 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 06:44:42.537 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 06:44:42.537 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 06:44:42.537 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 06:44:42.537 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 06:44:42.537 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:44:42.537 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 06:44:42.537 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:44:42.537 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 06:44:42.537 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 06:44:42.537 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 06:44:42.537 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 06:44:42.537 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:44:42.537 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 06:44:42.537 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:44:42.537 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:44:42.537 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:44:42.537 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 06:44:42.537 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 06:44:42.537 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:44:42.537 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 06:44:42.537 | INFO | ------------------------------------------ | 20260505 06:44:42.537 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:44:42.537 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 06:44:42.537 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:44:42.537 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:44:42.537 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.55e+01, 1.37e+01 / 1.77e+00 | 20260505 06:44:42.537 | INFO | PerfReport: 0.29%, AggregateStats -- 9.44e-02 / 1.03e-01, 1.28e-02 / 5.36e-03 | 20260505 06:44:42.537 | INFO | PerfReport: 11.58%, Baseline -- 3.83e+00 / 4.46e+00, 6.32e-01 / 2.70e-01 | 20260505 06:44:42.537 | INFO | PerfReport: 3.31%, Histogram -- 1.10e+00 / 1.49e+00, 2.16e-01 / 9.68e-02 | 20260505 06:44:42.537 | INFO | PerfReport: 15.46%, DME -- 4.09e+01 / 4.48e+01, 4.41e+00 / 1.16e+00 | 20260505 06:44:42.537 | INFO | PerfReport: 56.16%, FrameLabeling -- 1.86e+01 / 1.93e+01, 2.09e+00 / 1.19e+00 | 20260505 06:44:42.537 | INFO | PerfReport: 7.65%, PulseAccumulating -- 2.53e+00 / 3.12e+00, 4.56e-01 / 2.84e-01 | 20260505 06:44:42.537 | INFO | PerfReport: 4.07%, Metrics -- 1.35e+00 / 1.71e+00, 3.84e-01 / 2.07e-01 | 20260505 06:44:42.537 | INFO | PerfReport: 1.48%, PulseSerializing -- 4.89e-01 / 5.62e-01, 9.49e-02 / 4.24e-02 | 20260505 06:44:42.538 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 06:44:42.538 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:44:42.538 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:44:42.538 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.23e+01 / 0.00e+00 | 20260505 06:44:42.538 | INFO | PerfReport: 22.10%, Upload -- 2.25e+01 / 0.00e+00, 7.95e+00 / 0.00e+00 | 20260505 06:44:42.538 | INFO | PerfReport: 21.42%, Download -- 2.18e+01 / 0.00e+00, 7.80e+00 / 0.00e+00 | 20260505 06:44:42.538 | INFO | PerfReport: 32.53%, Compute -- 3.31e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 06:44:42.538 | INFO | PerfReport: 1.46%, LockUpload -- 1.49e+00 / 0.00e+00, 6.92e+00 / 0.00e+00 | 20260505 06:44:42.538 | INFO | PerfReport: 0.77%, LockDownload -- 7.82e-01 / 0.00e+00, 2.16e+00 / 0.00e+00 | 20260505 06:44:42.538 | INFO | PerfReport: 21.70%, LockCompute -- 2.21e+01 / 0.00e+00, 1.68e+01 / 0.00e+00 | 20260505 06:44:42.538 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 1.94e-02 / 0.00e+00 | 20260505 06:44:42.658 | INFO | Starting a round of Baz IO | 20260505 06:44:45.908 | INFO | Populating RTMetrics report | 20260505 06:44:45.908 | INFO | RTMetrics report populated | 20260505 06:44:45.908 | INFO | Opening tmp RTMetrics json file | 20260505 06:44:45.935 | INFO | Serializing RTMetrics to json | 20260505 06:44:45.937 | INFO | Serialization done, swapping tmp file now... | 20260505 06:44:45.938 | INFO | RTMetrics done | 20260505 06:44:46.030 | INFO | Finished a round of Baz IO: Wrote 2.396 GiB in 3.371 seconds | 20260505 06:44:46.030 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.15% / 0.04% / 67.81% | 20260505 06:44:46.030 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.62281% / 8.34465e-05% / 7.94729e-05% | 20260505 06:44:46.030 | INFO | Baz IO overall: | 20260505 06:44:46.030 | INFO | BytesWritten : 607222415825 | 20260505 06:44:46.030 | INFO | NumEvents : 287450171354 | 20260505 06:44:47.675 | INFO | Analyzing chunk frames = [1155072,1155584) | 20260505 06:44:47.675 | INFO | MemInfo: { | 20260505 06:44:47.675 | INFO | "gpu" : " 17 GB", | 20260505 06:44:47.675 | INFO | "huge" : "", | 20260505 06:44:47.675 | INFO | "huge_shared" : " 27 GB", | 20260505 06:44:47.675 | INFO | "malloc" : " 14 GB", | 20260505 06:44:47.675 | INFO | "pinned" : " 29 GB" | 20260505 06:44:47.675 | INFO | } | 20260505 06:44:47.675 | INFO | Perf report for the last 5.13931 seconds: | 20260505 06:44:47.676 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:44:47.676 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:47.676 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:47.676 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:47.676 | INFO | ANALYSIS : 64.5%, 2.9, 98.2ms, 0 | 20260505 06:44:47.676 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:47.676 | INFO | RT_METRICS : 46.7%, 1.0, 24.5ms, 0 | 20260505 06:44:47.676 | INFO | PRE_HQ : 63.4%, 1.9, 64.0ms, 0 | 20260505 06:44:47.676 | INFO | BAZWRITER : 64.9%, 4.1, 137.7ms, 0 | 20260505 06:44:47.676 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 06:44:47.676 | INFO | Uploaded 36GiB to GPU in 2.09932s (17.1484GiB/s) | 20260505 06:44:47.676 | INFO | Downloaded 38GiB to GPU in 2.29896s (16.5292GiB/s) | 20260505 06:44:52.814 | INFO | Analyzing chunk frames = [1155584,1156096) | 20260505 06:44:52.814 | INFO | MemInfo: { | 20260505 06:44:52.814 | INFO | "gpu" : " 17 GB", | 20260505 06:44:52.814 | INFO | "huge" : "", | 20260505 06:44:52.814 | INFO | "huge_shared" : " 27 GB", | 20260505 06:44:52.814 | INFO | "malloc" : " 14 GB", | 20260505 06:44:52.815 | INFO | "pinned" : " 29 GB" | 20260505 06:44:52.815 | INFO | } | 20260505 06:44:52.815 | INFO | Perf report for the last 5.13929 seconds: | 20260505 06:44:52.815 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:44:52.815 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:52.815 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:52.815 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:52.815 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 06:44:52.815 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:52.815 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:52.815 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 06:44:52.815 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:52.815 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:52.815 | INFO | Uploaded 36GiB to GPU in 2.1027s (17.1208GiB/s) | 20260505 06:44:52.815 | INFO | Downloaded 34GiB to GPU in 2.03783s (16.6844GiB/s) | 20260505 06:44:57.954 | INFO | Analyzing chunk frames = [1156096,1156608) | 20260505 06:44:57.954 | INFO | MemInfo: { | 20260505 06:44:57.954 | INFO | "gpu" : " 17 GB", | 20260505 06:44:57.954 | INFO | "huge" : "", | 20260505 06:44:57.954 | INFO | "huge_shared" : " 27 GB", | 20260505 06:44:57.954 | INFO | "malloc" : " 14 GB", | 20260505 06:44:57.954 | INFO | "pinned" : " 29 GB" | 20260505 06:44:57.954 | INFO | } | 20260505 06:44:57.954 | INFO | Perf report for the last 5.13951 seconds: | 20260505 06:44:57.954 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:44:57.954 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:57.954 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:57.954 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:57.954 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 06:44:57.954 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:57.954 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:57.954 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 06:44:57.954 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:57.954 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:44:57.954 | INFO | Uploaded 36GiB to GPU in 2.12182s (16.9665GiB/s) | 20260505 06:44:57.954 | INFO | Downloaded 34GiB to GPU in 2.04706s (16.6092GiB/s) | 20260505 06:45:03.093 | INFO | Analyzing chunk frames = [1156608,1157120) | 20260505 06:45:03.093 | INFO | MemInfo: { | 20260505 06:45:03.093 | INFO | "gpu" : " 17 GB", | 20260505 06:45:03.093 | INFO | "huge" : "", | 20260505 06:45:03.093 | INFO | "huge_shared" : " 27 GB", | 20260505 06:45:03.093 | INFO | "malloc" : " 14 GB", | 20260505 06:45:03.093 | INFO | "pinned" : " 29 GB" | 20260505 06:45:03.093 | INFO | } | 20260505 06:45:03.093 | INFO | Perf report for the last 5.13914 seconds: | 20260505 06:45:03.093 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:45:03.094 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:03.094 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:03.094 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:03.094 | INFO | ANALYSIS : 63.7%, 3.0, 98.9ms, 0 | 20260505 06:45:03.094 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:03.094 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:03.094 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 06:45:03.094 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:03.094 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:03.094 | INFO | Uploaded 36GiB to GPU in 2.15323s (16.7191GiB/s) | 20260505 06:45:03.094 | INFO | Downloaded 34GiB to GPU in 2.03113s (16.7394GiB/s) | 20260505 06:45:08.233 | INFO | Analyzing chunk frames = [1157120,1157632) | 20260505 06:45:08.233 | INFO | MemInfo: { | 20260505 06:45:08.233 | INFO | "gpu" : " 17 GB", | 20260505 06:45:08.233 | INFO | "huge" : "", | 20260505 06:45:08.233 | INFO | "huge_shared" : " 27 GB", | 20260505 06:45:08.233 | INFO | "malloc" : " 14 GB", | 20260505 06:45:08.233 | INFO | "pinned" : " 29 GB" | 20260505 06:45:08.233 | INFO | } | 20260505 06:45:08.233 | INFO | Perf report for the last 5.13964 seconds: | 20260505 06:45:08.233 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:45:08.233 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:08.233 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:08.233 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:08.233 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 06:45:08.233 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:08.233 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:08.233 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 06:45:08.233 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:08.233 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:08.233 | INFO | Uploaded 36GiB to GPU in 2.12799s (16.9174GiB/s) | 20260505 06:45:08.233 | INFO | Downloaded 34GiB to GPU in 2.03966s (16.6695GiB/s) | 20260505 06:45:13.374 | INFO | Analyzing chunk frames = [1157632,1158144) | 20260505 06:45:13.374 | INFO | MemInfo: { | 20260505 06:45:13.374 | INFO | "gpu" : " 17 GB", | 20260505 06:45:13.374 | INFO | "huge" : "", | 20260505 06:45:13.374 | INFO | "huge_shared" : " 27 GB", | 20260505 06:45:13.374 | INFO | "malloc" : " 14 GB", | 20260505 06:45:13.374 | INFO | "pinned" : " 29 GB" | 20260505 06:45:13.374 | INFO | } | 20260505 06:45:13.374 | INFO | Perf report for the last 5.14079 seconds: | 20260505 06:45:13.374 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:45:13.374 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:13.374 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:13.374 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:13.374 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 06:45:13.374 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:13.374 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:13.374 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 06:45:13.374 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:13.374 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:13.374 | INFO | Uploaded 36GiB to GPU in 2.15405s (16.7127GiB/s) | 20260505 06:45:13.374 | INFO | Downloaded 34GiB to GPU in 2.03503s (16.7074GiB/s) | 20260505 06:45:18.511 | INFO | Analyzing chunk frames = [1158144,1158656) | 20260505 06:45:18.511 | INFO | MemInfo: { | 20260505 06:45:18.511 | INFO | "gpu" : " 17 GB", | 20260505 06:45:18.511 | INFO | "huge" : "", | 20260505 06:45:18.511 | INFO | "huge_shared" : " 27 GB", | 20260505 06:45:18.511 | INFO | "malloc" : " 14 GB", | 20260505 06:45:18.511 | INFO | "pinned" : " 29 GB" | 20260505 06:45:18.511 | INFO | } | 20260505 06:45:18.511 | INFO | Perf report for the last 5.13753 seconds: | 20260505 06:45:18.511 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:45:18.512 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:18.512 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:18.512 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:18.512 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 06:45:18.512 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:18.512 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:18.512 | INFO | PRE_HQ : 32.1%, 1.0, 16.6ms, 0 | 20260505 06:45:18.512 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:18.512 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:18.512 | INFO | Uploaded 36GiB to GPU in 2.20421s (16.3324GiB/s) | 20260505 06:45:18.512 | INFO | Downloaded 34GiB to GPU in 2.16834s (15.6802GiB/s) | 20260505 06:45:23.651 | INFO | Analyzing chunk frames = [1158656,1159168) | 20260505 06:45:23.651 | INFO | MemInfo: { | 20260505 06:45:23.651 | INFO | "gpu" : " 17 GB", | 20260505 06:45:23.651 | INFO | "huge" : "", | 20260505 06:45:23.651 | INFO | "huge_shared" : " 27 GB", | 20260505 06:45:23.651 | INFO | "malloc" : " 14 GB", | 20260505 06:45:23.651 | INFO | "pinned" : " 29 GB" | 20260505 06:45:23.651 | INFO | } | 20260505 06:45:23.651 | INFO | Perf report for the last 5.13926 seconds: | 20260505 06:45:23.651 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:45:23.651 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:23.651 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:23.651 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:23.651 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 06:45:23.651 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:23.651 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:23.651 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 06:45:23.651 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:23.651 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:23.651 | INFO | Uploaded 36GiB to GPU in 2.1679s (16.6059GiB/s) | 20260505 06:45:23.651 | INFO | Downloaded 34GiB to GPU in 2.03876s (16.6768GiB/s) | 20260505 06:45:23.773 | INFO | Starting a round of Baz IO | 20260505 06:45:27.017 | INFO | Populating RTMetrics report | 20260505 06:45:27.017 | INFO | RTMetrics report populated | 20260505 06:45:27.017 | INFO | Opening tmp RTMetrics json file | 20260505 06:45:27.056 | INFO | Serializing RTMetrics to json | 20260505 06:45:27.074 | INFO | Serialization done, swapping tmp file now... | 20260505 06:45:27.090 | INFO | RTMetrics done | 20260505 06:45:27.132 | INFO | Finished a round of Baz IO: Wrote 2.397 GiB in 3.358 seconds | 20260505 06:45:27.132 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.11% / 0.04% / 67.85% | 20260505 06:45:27.132 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.60593% / 5.96046e-05% / 9.53674e-05% | 20260505 06:45:27.132 | INFO | Baz IO overall: | 20260505 06:45:27.132 | INFO | BytesWritten : 609795886526 | 20260505 06:45:27.132 | INFO | NumEvents : 288680999660 | 20260505 06:45:28.790 | INFO | Analyzing chunk frames = [1159168,1159680) | 20260505 06:45:28.790 | INFO | MemInfo: { | 20260505 06:45:28.790 | INFO | "gpu" : " 17 GB", | 20260505 06:45:28.790 | INFO | "huge" : "", | 20260505 06:45:28.790 | INFO | "huge_shared" : " 27 GB", | 20260505 06:45:28.790 | INFO | "malloc" : " 14 GB", | 20260505 06:45:28.790 | INFO | "pinned" : " 29 GB" | 20260505 06:45:28.790 | INFO | } | 20260505 06:45:28.790 | INFO | Perf report for the last 5.13956 seconds: | 20260505 06:45:28.790 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:45:28.790 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:28.790 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:28.790 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:28.790 | INFO | ANALYSIS : 64.4%, 2.9, 98.3ms, 0 | 20260505 06:45:28.790 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:28.790 | INFO | RT_METRICS : 47.3%, 1.0, 24.8ms, 0 | 20260505 06:45:28.790 | INFO | PRE_HQ : 63.2%, 1.9, 63.6ms, 0 | 20260505 06:45:28.790 | INFO | BAZWRITER : 64.7%, 4.2, 139.9ms, 0 | 20260505 06:45:28.790 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260505 06:45:28.790 | INFO | Uploaded 36GiB to GPU in 2.13481s (16.8633GiB/s) | 20260505 06:45:28.790 | INFO | Downloaded 38GiB to GPU in 2.29665s (16.5459GiB/s) | 20260505 06:45:33.929 | INFO | Analyzing chunk frames = [1159680,1160192) | 20260505 06:45:33.929 | INFO | MemInfo: { | 20260505 06:45:33.929 | INFO | "gpu" : " 17 GB", | 20260505 06:45:33.929 | INFO | "huge" : "", | 20260505 06:45:33.929 | INFO | "huge_shared" : " 27 GB", | 20260505 06:45:33.929 | INFO | "malloc" : " 14 GB", | 20260505 06:45:33.929 | INFO | "pinned" : " 29 GB" | 20260505 06:45:33.929 | INFO | } | 20260505 06:45:33.929 | INFO | Perf report for the last 5.13911 seconds: | 20260505 06:45:33.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:45:33.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:33.930 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:33.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:33.930 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 06:45:33.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:33.930 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:33.930 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 06:45:33.930 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:33.930 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:33.930 | INFO | Uploaded 36GiB to GPU in 2.10367s (17.113GiB/s) | 20260505 06:45:33.930 | INFO | Downloaded 34GiB to GPU in 2.03878s (16.6766GiB/s) | 20260505 06:45:39.069 | INFO | Analyzing chunk frames = [1160192,1160704) | 20260505 06:45:39.069 | INFO | MemInfo: { | 20260505 06:45:39.069 | INFO | "gpu" : " 17 GB", | 20260505 06:45:39.069 | INFO | "huge" : "", | 20260505 06:45:39.069 | INFO | "huge_shared" : " 27 GB", | 20260505 06:45:39.069 | INFO | "malloc" : " 14 GB", | 20260505 06:45:39.069 | INFO | "pinned" : " 29 GB" | 20260505 06:45:39.069 | INFO | } | 20260505 06:45:39.069 | INFO | Perf report for the last 5.13943 seconds: | 20260505 06:45:39.069 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:45:39.069 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:39.069 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:39.069 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:39.069 | INFO | ANALYSIS : 63.6%, 3.0, 99.0ms, 0 | 20260505 06:45:39.069 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:39.069 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:39.069 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 06:45:39.069 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:39.069 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:39.069 | INFO | Uploaded 36GiB to GPU in 2.11714s (17.0041GiB/s) | 20260505 06:45:39.069 | INFO | Downloaded 34GiB to GPU in 2.04392s (16.6347GiB/s) | 20260505 06:45:44.208 | INFO | Analyzing chunk frames = [1160704,1161216) | 20260505 06:45:44.208 | INFO | MemInfo: { | 20260505 06:45:44.208 | INFO | "gpu" : " 17 GB", | 20260505 06:45:44.208 | INFO | "huge" : "", | 20260505 06:45:44.208 | INFO | "huge_shared" : " 27 GB", | 20260505 06:45:44.208 | INFO | "malloc" : " 14 GB", | 20260505 06:45:44.208 | INFO | "pinned" : " 29 GB" | 20260505 06:45:44.208 | INFO | } | 20260505 06:45:44.208 | INFO | Perf report for the last 5.13959 seconds: | 20260505 06:45:44.208 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:45:44.208 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:44.208 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:44.208 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:44.208 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260505 06:45:44.208 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:44.208 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:44.208 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 06:45:44.209 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:44.209 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:44.209 | INFO | Uploaded 36GiB to GPU in 2.141s (16.8146GiB/s) | 20260505 06:45:44.209 | INFO | Downloaded 34GiB to GPU in 2.03667s (16.6939GiB/s) | 20260505 06:45:49.347 | INFO | Analyzing chunk frames = [1161216,1161728) | 20260505 06:45:49.347 | INFO | MemInfo: { | 20260505 06:45:49.347 | INFO | "gpu" : " 17 GB", | 20260505 06:45:49.347 | INFO | "huge" : "", | 20260505 06:45:49.347 | INFO | "huge_shared" : " 27 GB", | 20260505 06:45:49.347 | INFO | "malloc" : " 14 GB", | 20260505 06:45:49.347 | INFO | "pinned" : " 29 GB" | 20260505 06:45:49.347 | INFO | } | 20260505 06:45:49.347 | INFO | Perf report for the last 5.13873 seconds: | 20260505 06:45:49.347 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:45:49.347 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:49.347 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:49.347 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:49.347 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 06:45:49.347 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:49.347 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:49.347 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 06:45:49.347 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:49.347 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:49.347 | INFO | Uploaded 36GiB to GPU in 2.11171s (17.0478GiB/s) | 20260505 06:45:49.348 | INFO | Downloaded 34GiB to GPU in 2.03425s (16.7137GiB/s) | 20260505 06:45:54.488 | INFO | Analyzing chunk frames = [1161728,1162240) | 20260505 06:45:54.488 | INFO | MemInfo: { | 20260505 06:45:54.488 | INFO | "gpu" : " 17 GB", | 20260505 06:45:54.488 | INFO | "huge" : "", | 20260505 06:45:54.488 | INFO | "huge_shared" : " 27 GB", | 20260505 06:45:54.488 | INFO | "malloc" : " 14 GB", | 20260505 06:45:54.488 | INFO | "pinned" : " 29 GB" | 20260505 06:45:54.488 | INFO | } | 20260505 06:45:54.488 | INFO | Perf report for the last 5.14113 seconds: | 20260505 06:45:54.488 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:45:54.488 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:54.488 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:54.488 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:54.488 | INFO | ANALYSIS : 63.7%, 3.0, 99.6ms, 0 | 20260505 06:45:54.488 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:54.488 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:54.488 | INFO | PRE_HQ : 30.5%, 1.0, 15.9ms, 0 | 20260505 06:45:54.488 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:54.488 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:54.489 | INFO | Uploaded 36GiB to GPU in 2.15351s (16.7169GiB/s) | 20260505 06:45:54.489 | INFO | Downloaded 34GiB to GPU in 2.03417s (16.7144GiB/s) | 20260505 06:45:59.626 | INFO | Analyzing chunk frames = [1162240,1162752) | 20260505 06:45:59.626 | INFO | MemInfo: { | 20260505 06:45:59.626 | INFO | "gpu" : " 17 GB", | 20260505 06:45:59.626 | INFO | "huge" : "", | 20260505 06:45:59.626 | INFO | "huge_shared" : " 27 GB", | 20260505 06:45:59.626 | INFO | "malloc" : " 14 GB", | 20260505 06:45:59.626 | INFO | "pinned" : " 29 GB" | 20260505 06:45:59.626 | INFO | } | 20260505 06:45:59.626 | INFO | Perf report for the last 5.13794 seconds: | 20260505 06:45:59.626 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:45:59.626 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:59.626 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:59.626 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:59.626 | INFO | ANALYSIS : 64.5%, 3.0, 100.7ms, 0 | 20260505 06:45:59.626 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:59.626 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:59.626 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 06:45:59.626 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:59.626 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:45:59.626 | INFO | Uploaded 36GiB to GPU in 2.21048s (16.286GiB/s) | 20260505 06:45:59.627 | INFO | Downloaded 34GiB to GPU in 2.17204s (15.6535GiB/s) | 20260505 06:46:04.765 | INFO | Analyzing chunk frames = [1162752,1163264) | 20260505 06:46:04.765 | INFO | MemInfo: { | 20260505 06:46:04.765 | INFO | "gpu" : " 17 GB", | 20260505 06:46:04.765 | INFO | "huge" : "", | 20260505 06:46:04.765 | INFO | "huge_shared" : " 27 GB", | 20260505 06:46:04.765 | INFO | "malloc" : " 14 GB", | 20260505 06:46:04.765 | INFO | "pinned" : " 29 GB" | 20260505 06:46:04.765 | INFO | } | 20260505 06:46:04.765 | INFO | Perf report for the last 5.13913 seconds: | 20260505 06:46:04.765 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:46:04.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:04.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:04.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:04.765 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 06:46:04.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:04.765 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:04.765 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 06:46:04.765 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:04.765 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:04.766 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 06:46:04.766 | INFO | Total mem usage: 15.3 GiB | 20260505 06:46:04.766 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 06:46:04.766 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -5.91%) | 20260505 06:46:04.766 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 06:46:04.766 | INFO | High Water Breakdown: | 20260505 06:46:04.766 | INFO | 60.5% : 8.76 GiB ( 93.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 06:46:04.766 | INFO | 39.4% : 5.7 GiB ( 85.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 06:46:04.766 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 06:46:04.766 | INFO | ------------------------------------------ | 20260505 06:46:04.766 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:46:04.766 | INFO | Uploaded 36GiB to GPU in 2.16084s (16.6602GiB/s) | 20260505 06:46:04.766 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 06:46:04.766 | INFO | Total mem usage: 26.5 GiB | 20260505 06:46:04.766 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 06:46:04.766 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 06:46:04.766 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 06:46:04.766 | INFO | Downloaded 34GiB to GPU in 2.04761s (16.6047GiB/s) | 20260505 06:46:04.766 | INFO | High Water Breakdown: | 20260505 06:46:04.766 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:46:04.766 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:46:04.766 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:46:04.766 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:46:04.766 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:46:04.766 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:46:04.766 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 06:46:04.766 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 06:46:04.766 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:46:04.766 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 06:46:04.766 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 06:46:04.766 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 06:46:04.766 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 06:46:04.766 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 06:46:04.766 | INFO | ------------------------------------------ | 20260505 06:46:04.766 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:46:04.766 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 06:46:04.766 | INFO | Total mem usage: 25.6 GiB | 20260505 06:46:04.766 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 06:46:04.766 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 06:46:04.766 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 06:46:04.766 | INFO | High Water Breakdown: | 20260505 06:46:04.766 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 06:46:04.766 | INFO | ------------------------------------------ | 20260505 06:46:04.766 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:46:04.766 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 06:46:04.766 | INFO | Total mem usage: 15.7 GiB | 20260505 06:46:04.766 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 06:46:04.766 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 06:46:04.766 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260505 06:46:04.766 | INFO | High Water Breakdown: | 20260505 06:46:04.766 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 06:46:04.766 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 06:46:04.766 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 06:46:04.766 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 06:46:04.766 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 06:46:04.766 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 06:46:04.766 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 06:46:04.766 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 06:46:04.766 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 06:46:04.766 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:46:04.766 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 06:46:04.766 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:46:04.766 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 06:46:04.766 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 06:46:04.766 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 06:46:04.766 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 06:46:04.766 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:46:04.766 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 06:46:04.766 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:46:04.766 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:46:04.766 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:46:04.766 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 06:46:04.766 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 06:46:04.766 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:46:04.766 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 06:46:04.766 | INFO | ------------------------------------------ | 20260505 06:46:04.766 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:46:04.766 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 06:46:04.766 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:46:04.766 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:46:04.766 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.51e+01, 1.37e+01 / 1.48e+00 | 20260505 06:46:04.766 | INFO | PerfReport: 0.28%, AggregateStats -- 9.38e-02 / 1.03e-01, 1.03e-02 / 4.96e-03 | 20260505 06:46:04.766 | INFO | PerfReport: 11.60%, Baseline -- 3.84e+00 / 4.45e+00, 6.33e-01 / 2.61e-01 | 20260505 06:46:04.766 | INFO | PerfReport: 3.32%, Histogram -- 1.10e+00 / 1.50e+00, 2.17e-01 / 9.72e-02 | 20260505 06:46:04.766 | INFO | PerfReport: 15.44%, DME -- 4.08e+01 / 4.46e+01, 4.09e+00 / 9.83e-01 | 20260505 06:46:04.766 | INFO | PerfReport: 56.17%, FrameLabeling -- 1.86e+01 / 1.90e+01, 2.08e+00 / 9.50e-01 | 20260505 06:46:04.766 | INFO | PerfReport: 7.66%, PulseAccumulating -- 2.53e+00 / 3.17e+00, 4.47e-01 / 2.67e-01 | 20260505 06:46:04.766 | INFO | PerfReport: 4.06%, Metrics -- 1.34e+00 / 1.71e+00, 3.77e-01 / 1.96e-01 | 20260505 06:46:04.766 | INFO | PerfReport: 1.47%, PulseSerializing -- 4.87e-01 / 5.63e-01, 9.46e-02 / 4.29e-02 | 20260505 06:46:04.766 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 06:46:04.766 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:46:04.766 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:46:04.766 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.22e+01 / 0.00e+00 | 20260505 06:46:04.767 | INFO | PerfReport: 21.98%, Upload -- 2.23e+01 / 0.00e+00, 7.82e+00 / 0.00e+00 | 20260505 06:46:04.767 | INFO | PerfReport: 21.43%, Download -- 2.17e+01 / 0.00e+00, 7.69e+00 / 0.00e+00 | 20260505 06:46:04.767 | INFO | PerfReport: 32.58%, Compute -- 3.31e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 06:46:04.767 | INFO | PerfReport: 1.46%, LockUpload -- 1.49e+00 / 0.00e+00, 6.94e+00 / 0.00e+00 | 20260505 06:46:04.767 | INFO | PerfReport: 0.82%, LockDownload -- 8.29e-01 / 0.00e+00, 2.30e+00 / 0.00e+00 | 20260505 06:46:04.767 | INFO | PerfReport: 21.72%, LockCompute -- 2.20e+01 / 0.00e+00, 1.67e+01 / 0.00e+00 | 20260505 06:46:04.767 | INFO | PerfReport: 0.01%, Misc -- 1.51e-02 / 0.00e+00, 2.39e-02 / 0.00e+00 | 20260505 06:46:04.887 | INFO | Starting a round of Baz IO | 20260505 06:46:08.124 | INFO | Populating RTMetrics report | 20260505 06:46:08.124 | INFO | RTMetrics report populated | 20260505 06:46:08.124 | INFO | Opening tmp RTMetrics json file | 20260505 06:46:08.146 | INFO | Serializing RTMetrics to json | 20260505 06:46:08.159 | INFO | Serialization done, swapping tmp file now... | 20260505 06:46:08.175 | INFO | RTMetrics done | 20260505 06:46:08.250 | INFO | Finished a round of Baz IO: Wrote 2.396 GiB in 3.362 seconds | 20260505 06:46:08.250 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.06% / 0.04% / 67.89% | 20260505 06:46:08.250 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.57744% / 6.75519e-05% / 0.000103315% | 20260505 06:46:08.250 | INFO | Baz IO overall: | 20260505 06:46:08.250 | INFO | BytesWritten : 612369035898 | 20260505 06:46:08.250 | INFO | NumEvents : 289911443060 | 20260505 06:46:09.904 | INFO | Analyzing chunk frames = [1163264,1163776) | 20260505 06:46:09.904 | INFO | MemInfo: { | 20260505 06:46:09.904 | INFO | "gpu" : " 17 GB", | 20260505 06:46:09.904 | INFO | "huge" : "", | 20260505 06:46:09.904 | INFO | "huge_shared" : " 27 GB", | 20260505 06:46:09.904 | INFO | "malloc" : " 14 GB", | 20260505 06:46:09.904 | INFO | "pinned" : " 29 GB" | 20260505 06:46:09.904 | INFO | } | 20260505 06:46:09.904 | INFO | Perf report for the last 5.13908 seconds: | 20260505 06:46:09.904 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:46:09.904 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:09.904 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:09.904 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:09.904 | INFO | ANALYSIS : 64.2%, 2.9, 97.8ms, 0 | 20260505 06:46:09.904 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:09.904 | INFO | RT_METRICS : 46.9%, 1.0, 24.6ms, 0 | 20260505 06:46:09.904 | INFO | PRE_HQ : 63.0%, 1.9, 63.3ms, 0 | 20260505 06:46:09.904 | INFO | BAZWRITER : 64.7%, 4.2, 138.7ms, 0 | 20260505 06:46:09.904 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 06:46:09.905 | INFO | Uploaded 36GiB to GPU in 2.11093s (17.0541GiB/s) | 20260505 06:46:09.905 | INFO | Downloaded 38GiB to GPU in 2.3093s (16.4552GiB/s) | 20260505 06:46:15.044 | INFO | Analyzing chunk frames = [1163776,1164288) | 20260505 06:46:15.044 | INFO | MemInfo: { | 20260505 06:46:15.044 | INFO | "gpu" : " 17 GB", | 20260505 06:46:15.044 | INFO | "huge" : "", | 20260505 06:46:15.044 | INFO | "huge_shared" : " 27 GB", | 20260505 06:46:15.044 | INFO | "malloc" : " 14 GB", | 20260505 06:46:15.044 | INFO | "pinned" : " 29 GB" | 20260505 06:46:15.044 | INFO | } | 20260505 06:46:15.044 | INFO | Perf report for the last 5.13959 seconds: | 20260505 06:46:15.044 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:46:15.044 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:15.044 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:15.044 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:15.044 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 06:46:15.044 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:15.044 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:15.044 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 06:46:15.044 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:15.044 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:15.044 | INFO | Uploaded 36GiB to GPU in 2.09972s (17.1452GiB/s) | 20260505 06:46:15.044 | INFO | Downloaded 34GiB to GPU in 2.03921s (16.6731GiB/s) | 20260505 06:46:20.183 | INFO | Analyzing chunk frames = [1164288,1164800) | 20260505 06:46:20.183 | INFO | MemInfo: { | 20260505 06:46:20.183 | INFO | "gpu" : " 17 GB", | 20260505 06:46:20.183 | INFO | "huge" : "", | 20260505 06:46:20.183 | INFO | "huge_shared" : " 27 GB", | 20260505 06:46:20.183 | INFO | "malloc" : " 14 GB", | 20260505 06:46:20.183 | INFO | "pinned" : " 29 GB" | 20260505 06:46:20.183 | INFO | } | 20260505 06:46:20.183 | INFO | Perf report for the last 5.13903 seconds: | 20260505 06:46:20.183 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:46:20.183 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:20.183 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:20.183 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:20.183 | INFO | ANALYSIS : 63.8%, 3.0, 99.2ms, 0 | 20260505 06:46:20.183 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:20.183 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:20.183 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 06:46:20.183 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:20.183 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:20.183 | INFO | Uploaded 36GiB to GPU in 2.14153s (16.8104GiB/s) | 20260505 06:46:20.184 | INFO | Downloaded 34GiB to GPU in 2.04665s (16.6125GiB/s) | 20260505 06:46:25.322 | INFO | Analyzing chunk frames = [1164800,1165312) | 20260505 06:46:25.322 | INFO | MemInfo: { | 20260505 06:46:25.322 | INFO | "gpu" : " 17 GB", | 20260505 06:46:25.322 | INFO | "huge" : "", | 20260505 06:46:25.322 | INFO | "huge_shared" : " 27 GB", | 20260505 06:46:25.322 | INFO | "malloc" : " 14 GB", | 20260505 06:46:25.322 | INFO | "pinned" : " 29 GB" | 20260505 06:46:25.322 | INFO | } | 20260505 06:46:25.322 | INFO | Perf report for the last 5.13945 seconds: | 20260505 06:46:25.323 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:46:25.323 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:25.323 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:25.323 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:25.323 | INFO | ANALYSIS : 63.8%, 3.0, 99.0ms, 0 | 20260505 06:46:25.323 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:25.323 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:25.323 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 06:46:25.323 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:25.323 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:25.323 | INFO | Uploaded 36GiB to GPU in 2.16113s (16.658GiB/s) | 20260505 06:46:25.323 | INFO | Downloaded 34GiB to GPU in 2.03049s (16.7447GiB/s) | 20260505 06:46:30.462 | INFO | Analyzing chunk frames = [1165312,1165824) | 20260505 06:46:30.462 | INFO | MemInfo: { | 20260505 06:46:30.462 | INFO | "gpu" : " 17 GB", | 20260505 06:46:30.462 | INFO | "huge" : "", | 20260505 06:46:30.462 | INFO | "huge_shared" : " 27 GB", | 20260505 06:46:30.462 | INFO | "malloc" : " 14 GB", | 20260505 06:46:30.462 | INFO | "pinned" : " 29 GB" | 20260505 06:46:30.462 | INFO | } | 20260505 06:46:30.462 | INFO | Perf report for the last 5.13922 seconds: | 20260505 06:46:30.462 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:46:30.462 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:30.462 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:30.462 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:30.462 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 06:46:30.462 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:30.462 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:30.462 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 06:46:30.462 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:30.462 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:30.462 | INFO | Uploaded 36GiB to GPU in 2.11784s (16.9984GiB/s) | 20260505 06:46:30.463 | INFO | Downloaded 34GiB to GPU in 2.03132s (16.7379GiB/s) | 20260505 06:46:35.603 | INFO | Analyzing chunk frames = [1165824,1166336) | 20260505 06:46:35.603 | INFO | MemInfo: { | 20260505 06:46:35.603 | INFO | "gpu" : " 17 GB", | 20260505 06:46:35.603 | INFO | "huge" : "", | 20260505 06:46:35.603 | INFO | "huge_shared" : " 27 GB", | 20260505 06:46:35.603 | INFO | "malloc" : " 14 GB", | 20260505 06:46:35.603 | INFO | "pinned" : " 29 GB" | 20260505 06:46:35.603 | INFO | } | 20260505 06:46:35.603 | INFO | Perf report for the last 5.14139 seconds: | 20260505 06:46:35.603 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:46:35.603 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:35.603 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:35.603 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:35.603 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 06:46:35.603 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:35.603 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:35.603 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 06:46:35.603 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:35.603 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:35.603 | INFO | Uploaded 36GiB to GPU in 2.15028s (16.742GiB/s) | 20260505 06:46:35.603 | INFO | Downloaded 34GiB to GPU in 2.03812s (16.6821GiB/s) | 20260505 06:46:40.741 | INFO | Analyzing chunk frames = [1166336,1166848) | 20260505 06:46:40.741 | INFO | MemInfo: { | 20260505 06:46:40.741 | INFO | "gpu" : " 17 GB", | 20260505 06:46:40.741 | INFO | "huge" : "", | 20260505 06:46:40.741 | INFO | "huge_shared" : " 27 GB", | 20260505 06:46:40.741 | INFO | "malloc" : " 14 GB", | 20260505 06:46:40.741 | INFO | "pinned" : " 29 GB" | 20260505 06:46:40.741 | INFO | } | 20260505 06:46:40.741 | INFO | Perf report for the last 5.13774 seconds: | 20260505 06:46:40.741 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:46:40.741 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:40.741 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:40.741 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:40.741 | INFO | ANALYSIS : 64.5%, 3.0, 100.7ms, 0 | 20260505 06:46:40.741 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:40.741 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:40.741 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260505 06:46:40.741 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:40.741 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:40.741 | INFO | Uploaded 36GiB to GPU in 2.21563s (16.2482GiB/s) | 20260505 06:46:40.741 | INFO | Downloaded 34GiB to GPU in 2.16603s (15.6969GiB/s) | 20260505 06:46:45.880 | INFO | Analyzing chunk frames = [1166848,1167360) | 20260505 06:46:45.880 | INFO | MemInfo: { | 20260505 06:46:45.880 | INFO | "gpu" : " 17 GB", | 20260505 06:46:45.880 | INFO | "huge" : "", | 20260505 06:46:45.880 | INFO | "huge_shared" : " 27 GB", | 20260505 06:46:45.880 | INFO | "malloc" : " 14 GB", | 20260505 06:46:45.880 | INFO | "pinned" : " 29 GB" | 20260505 06:46:45.880 | INFO | } | 20260505 06:46:45.880 | INFO | Perf report for the last 5.13923 seconds: | 20260505 06:46:45.880 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:46:45.880 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:45.880 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:45.880 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:45.880 | INFO | ANALYSIS : 64.3%, 3.0, 100.5ms, 0 | 20260505 06:46:45.880 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:45.880 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:45.880 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 06:46:45.880 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:45.880 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:45.880 | INFO | Uploaded 36GiB to GPU in 2.16592s (16.6211GiB/s) | 20260505 06:46:45.880 | INFO | Downloaded 34GiB to GPU in 2.04607s (16.6172GiB/s) | 20260505 06:46:46.003 | INFO | Starting a round of Baz IO | 20260505 06:46:49.277 | INFO | Populating RTMetrics report | 20260505 06:46:49.277 | INFO | RTMetrics report populated | 20260505 06:46:49.277 | INFO | Opening tmp RTMetrics json file | 20260505 06:46:49.305 | INFO | Serializing RTMetrics to json | 20260505 06:46:49.306 | INFO | Serialization done, swapping tmp file now... | 20260505 06:46:49.307 | INFO | RTMetrics done | 20260505 06:46:49.409 | INFO | Finished a round of Baz IO: Wrote 2.397 GiB in 3.406 seconds | 20260505 06:46:49.409 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.02% / 0.04% / 67.94% | 20260505 06:46:49.409 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.55583% / 7.15256e-05% / 7.94729e-05% | 20260505 06:46:49.409 | INFO | Baz IO overall: | 20260505 06:46:49.409 | INFO | BytesWritten : 614942790104 | 20260505 06:46:49.409 | INFO | NumEvents : 291141986648 | 20260505 06:46:51.019 | INFO | Analyzing chunk frames = [1167360,1167872) | 20260505 06:46:51.019 | INFO | MemInfo: { | 20260505 06:46:51.019 | INFO | "gpu" : " 17 GB", | 20260505 06:46:51.019 | INFO | "huge" : "", | 20260505 06:46:51.019 | INFO | "huge_shared" : " 27 GB", | 20260505 06:46:51.019 | INFO | "malloc" : " 14 GB", | 20260505 06:46:51.019 | INFO | "pinned" : " 29 GB" | 20260505 06:46:51.019 | INFO | } | 20260505 06:46:51.019 | INFO | Perf report for the last 5.13932 seconds: | 20260505 06:46:51.019 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:46:51.019 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:51.019 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:51.019 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:51.019 | INFO | ANALYSIS : 64.9%, 2.9, 98.9ms, 0 | 20260505 06:46:51.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:51.020 | INFO | RT_METRICS : 46.7%, 1.0, 24.5ms, 0 | 20260505 06:46:51.020 | INFO | PRE_HQ : 63.9%, 1.9, 63.5ms, 0 | 20260505 06:46:51.020 | INFO | BAZWRITER : 65.6%, 4.1, 138.1ms, 0 | 20260505 06:46:51.020 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.5ms, 0 | 20260505 06:46:51.020 | INFO | Uploaded 36GiB to GPU in 2.18952s (16.442GiB/s) | 20260505 06:46:51.020 | INFO | Downloaded 38GiB to GPU in 2.30046s (16.5184GiB/s) | 20260505 06:46:56.159 | INFO | Analyzing chunk frames = [1167872,1168384) | 20260505 06:46:56.159 | INFO | MemInfo: { | 20260505 06:46:56.159 | INFO | "gpu" : " 17 GB", | 20260505 06:46:56.159 | INFO | "huge" : "", | 20260505 06:46:56.159 | INFO | "huge_shared" : " 27 GB", | 20260505 06:46:56.159 | INFO | "malloc" : " 14 GB", | 20260505 06:46:56.159 | INFO | "pinned" : " 29 GB" | 20260505 06:46:56.159 | INFO | } | 20260505 06:46:56.159 | INFO | Perf report for the last 5.13925 seconds: | 20260505 06:46:56.159 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:46:56.159 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:56.159 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:56.159 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:56.159 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 06:46:56.159 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:56.159 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:56.159 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 06:46:56.159 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:56.159 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:46:56.159 | INFO | Uploaded 36GiB to GPU in 2.09958s (17.1463GiB/s) | 20260505 06:46:56.159 | INFO | Downloaded 34GiB to GPU in 2.03818s (16.6815GiB/s) | 20260505 06:47:01.298 | INFO | Analyzing chunk frames = [1168384,1168896) | 20260505 06:47:01.298 | INFO | MemInfo: { | 20260505 06:47:01.298 | INFO | "gpu" : " 17 GB", | 20260505 06:47:01.298 | INFO | "huge" : "", | 20260505 06:47:01.298 | INFO | "huge_shared" : " 27 GB", | 20260505 06:47:01.298 | INFO | "malloc" : " 14 GB", | 20260505 06:47:01.298 | INFO | "pinned" : " 29 GB" | 20260505 06:47:01.298 | INFO | } | 20260505 06:47:01.298 | INFO | Perf report for the last 5.13923 seconds: | 20260505 06:47:01.298 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:47:01.298 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:01.298 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:01.298 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:01.298 | INFO | ANALYSIS : 63.6%, 3.0, 99.0ms, 0 | 20260505 06:47:01.298 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:01.298 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:01.298 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 06:47:01.298 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:01.298 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:01.298 | INFO | Uploaded 36GiB to GPU in 2.12252s (16.961GiB/s) | 20260505 06:47:01.299 | INFO | Downloaded 34GiB to GPU in 2.04441s (16.6307GiB/s) | 20260505 06:47:06.437 | INFO | Analyzing chunk frames = [1168896,1169408) | 20260505 06:47:06.437 | INFO | MemInfo: { | 20260505 06:47:06.437 | INFO | "gpu" : " 17 GB", | 20260505 06:47:06.437 | INFO | "huge" : "", | 20260505 06:47:06.437 | INFO | "huge_shared" : " 27 GB", | 20260505 06:47:06.437 | INFO | "malloc" : " 14 GB", | 20260505 06:47:06.437 | INFO | "pinned" : " 29 GB" | 20260505 06:47:06.437 | INFO | } | 20260505 06:47:06.437 | INFO | Perf report for the last 5.13931 seconds: | 20260505 06:47:06.437 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:47:06.437 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:06.437 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:06.437 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:06.437 | INFO | ANALYSIS : 63.5%, 3.0, 98.6ms, 0 | 20260505 06:47:06.437 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:06.437 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:06.437 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 06:47:06.437 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:06.437 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:06.437 | INFO | Uploaded 36GiB to GPU in 2.13785s (16.8394GiB/s) | 20260505 06:47:06.438 | INFO | Downloaded 34GiB to GPU in 2.02669s (16.7761GiB/s) | 20260505 06:47:11.576 | INFO | Analyzing chunk frames = [1169408,1169920) | 20260505 06:47:11.576 | INFO | MemInfo: { | 20260505 06:47:11.576 | INFO | "gpu" : " 17 GB", | 20260505 06:47:11.576 | INFO | "huge" : "", | 20260505 06:47:11.576 | INFO | "huge_shared" : " 27 GB", | 20260505 06:47:11.576 | INFO | "malloc" : " 14 GB", | 20260505 06:47:11.576 | INFO | "pinned" : " 29 GB" | 20260505 06:47:11.576 | INFO | } | 20260505 06:47:11.577 | INFO | Perf report for the last 5.13931 seconds: | 20260505 06:47:11.577 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:47:11.577 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:11.577 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:11.577 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:11.577 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 06:47:11.577 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:11.577 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:11.577 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 06:47:11.577 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:11.577 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:11.577 | INFO | Uploaded 36GiB to GPU in 2.16891s (16.5982GiB/s) | 20260505 06:47:11.577 | INFO | Downloaded 34GiB to GPU in 2.04126s (16.6564GiB/s) | 20260505 06:47:16.717 | INFO | Analyzing chunk frames = [1169920,1170432) | 20260505 06:47:16.717 | INFO | MemInfo: { | 20260505 06:47:16.717 | INFO | "gpu" : " 17 GB", | 20260505 06:47:16.717 | INFO | "huge" : "", | 20260505 06:47:16.717 | INFO | "huge_shared" : " 27 GB", | 20260505 06:47:16.717 | INFO | "malloc" : " 14 GB", | 20260505 06:47:16.717 | INFO | "pinned" : " 29 GB" | 20260505 06:47:16.718 | INFO | } | 20260505 06:47:16.718 | INFO | Perf report for the last 5.14091 seconds: | 20260505 06:47:16.718 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:47:16.718 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:16.718 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:16.718 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:16.718 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 06:47:16.718 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:16.718 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:16.718 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 06:47:16.718 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:16.718 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:16.718 | INFO | Uploaded 36GiB to GPU in 2.15657s (16.6932GiB/s) | 20260505 06:47:16.718 | INFO | Downloaded 34GiB to GPU in 2.0438s (16.6356GiB/s) | 20260505 06:47:21.855 | INFO | Analyzing chunk frames = [1170432,1170944) | 20260505 06:47:21.855 | INFO | MemInfo: { | 20260505 06:47:21.855 | INFO | "gpu" : " 17 GB", | 20260505 06:47:21.855 | INFO | "huge" : "", | 20260505 06:47:21.855 | INFO | "huge_shared" : " 27 GB", | 20260505 06:47:21.855 | INFO | "malloc" : " 14 GB", | 20260505 06:47:21.855 | INFO | "pinned" : " 29 GB" | 20260505 06:47:21.855 | INFO | } | 20260505 06:47:21.855 | INFO | Perf report for the last 5.13785 seconds: | 20260505 06:47:21.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:47:21.855 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:21.855 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:21.855 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:21.855 | INFO | ANALYSIS : 64.4%, 3.0, 100.6ms, 0 | 20260505 06:47:21.855 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:21.855 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:21.856 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260505 06:47:21.856 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:21.856 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:21.856 | INFO | Uploaded 36GiB to GPU in 2.21105s (16.2819GiB/s) | 20260505 06:47:21.856 | INFO | Downloaded 34GiB to GPU in 2.16488s (15.7052GiB/s) | 20260505 06:47:26.995 | INFO | Analyzing chunk frames = [1170944,1171456) | 20260505 06:47:26.995 | INFO | MemInfo: { | 20260505 06:47:26.995 | INFO | "gpu" : " 17 GB", | 20260505 06:47:26.995 | INFO | "huge" : "", | 20260505 06:47:26.995 | INFO | "huge_shared" : " 27 GB", | 20260505 06:47:26.995 | INFO | "malloc" : " 14 GB", | 20260505 06:47:26.995 | INFO | "pinned" : " 29 GB" | 20260505 06:47:26.995 | INFO | } | 20260505 06:47:26.995 | INFO | Perf report for the last 5.1395 seconds: | 20260505 06:47:26.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:47:26.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:26.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:26.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:26.995 | INFO | ANALYSIS : 64.3%, 3.0, 100.3ms, 0 | 20260505 06:47:26.995 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:26.995 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:26.995 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260505 06:47:26.995 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:26.995 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:26.995 | INFO | Uploaded 36GiB to GPU in 2.16105s (16.6585GiB/s) | 20260505 06:47:26.995 | INFO | Downloaded 34GiB to GPU in 2.04292s (16.6428GiB/s) | 20260505 06:47:26.995 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 06:47:26.995 | INFO | Total mem usage: 15.3 GiB | 20260505 06:47:26.995 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 06:47:26.995 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -5.93%) | 20260505 06:47:26.995 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 06:47:26.995 | INFO | High Water Breakdown: | 20260505 06:47:26.995 | INFO | 60.5% : 8.75 GiB ( 93.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 06:47:26.995 | INFO | 39.4% : 5.7 GiB ( 85.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 06:47:26.995 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 06:47:26.995 | INFO | ------------------------------------------ | 20260505 06:47:26.995 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:47:26.995 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 06:47:26.995 | INFO | Total mem usage: 26.5 GiB | 20260505 06:47:26.995 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 06:47:26.996 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 06:47:26.996 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 06:47:26.996 | INFO | High Water Breakdown: | 20260505 06:47:26.996 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:47:26.996 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:47:26.996 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:47:26.996 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:47:26.996 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:47:26.996 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:47:26.996 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 06:47:26.996 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 06:47:26.996 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:47:26.996 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 06:47:26.996 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 06:47:26.996 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 06:47:26.996 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 06:47:26.996 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 06:47:26.996 | INFO | ------------------------------------------ | 20260505 06:47:26.996 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:47:26.996 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 06:47:26.996 | INFO | Total mem usage: 25.6 GiB | 20260505 06:47:26.996 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 06:47:26.996 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 06:47:26.996 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 06:47:26.996 | INFO | High Water Breakdown: | 20260505 06:47:26.996 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 06:47:26.996 | INFO | ------------------------------------------ | 20260505 06:47:26.996 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:47:26.996 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 06:47:26.996 | INFO | Total mem usage: 15.7 GiB | 20260505 06:47:26.996 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 06:47:26.996 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 06:47:26.996 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260505 06:47:26.996 | INFO | High Water Breakdown: | 20260505 06:47:26.996 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 06:47:26.996 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 06:47:26.996 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 06:47:26.996 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 06:47:26.996 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 06:47:26.996 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 06:47:26.996 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 06:47:26.996 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 06:47:26.996 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 06:47:26.996 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:47:26.996 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 06:47:26.996 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:47:26.996 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 06:47:26.996 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 06:47:26.996 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 06:47:26.996 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 06:47:26.996 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:47:26.996 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 06:47:26.996 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:47:26.996 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:47:26.996 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:47:26.996 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 06:47:26.996 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 06:47:26.996 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:47:26.996 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 06:47:26.996 | INFO | ------------------------------------------ | 20260505 06:47:26.996 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:47:26.996 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 06:47:26.996 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:47:26.996 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:47:26.996 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.54e+01, 1.36e+01 / 1.64e+00 | 20260505 06:47:26.996 | INFO | PerfReport: 0.28%, AggregateStats -- 9.37e-02 / 1.03e-01, 1.05e-02 / 4.50e-03 | 20260505 06:47:26.996 | INFO | PerfReport: 11.59%, Baseline -- 3.83e+00 / 4.46e+00, 6.38e-01 / 2.95e-01 | 20260505 06:47:26.996 | INFO | PerfReport: 3.32%, Histogram -- 1.10e+00 / 1.49e+00, 2.24e-01 / 1.04e-01 | 20260505 06:47:26.996 | INFO | PerfReport: 15.47%, DME -- 4.09e+01 / 4.48e+01, 4.27e+00 / 1.17e+00 | 20260505 06:47:26.996 | INFO | PerfReport: 56.14%, FrameLabeling -- 1.86e+01 / 1.91e+01, 2.06e+00 / 1.02e+00 | 20260505 06:47:26.996 | INFO | PerfReport: 7.65%, PulseAccumulating -- 2.53e+00 / 3.13e+00, 4.52e-01 / 2.88e-01 | 20260505 06:47:26.996 | INFO | PerfReport: 4.06%, Metrics -- 1.34e+00 / 1.71e+00, 3.82e-01 / 1.80e-01 | 20260505 06:47:26.996 | INFO | PerfReport: 1.47%, PulseSerializing -- 4.87e-01 / 5.58e-01, 9.43e-02 / 3.89e-02 | 20260505 06:47:26.997 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 06:47:26.997 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:47:26.997 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:47:26.997 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.23e+01 / 0.00e+00 | 20260505 06:47:26.997 | INFO | PerfReport: 22.07%, Upload -- 2.24e+01 / 0.00e+00, 7.93e+00 / 0.00e+00 | 20260505 06:47:26.997 | INFO | PerfReport: 21.43%, Download -- 2.18e+01 / 0.00e+00, 7.78e+00 / 0.00e+00 | 20260505 06:47:26.997 | INFO | PerfReport: 32.56%, Compute -- 3.31e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 06:47:26.997 | INFO | PerfReport: 1.46%, LockUpload -- 1.48e+00 / 0.00e+00, 6.96e+00 / 0.00e+00 | 20260505 06:47:26.997 | INFO | PerfReport: 0.78%, LockDownload -- 7.94e-01 / 0.00e+00, 2.20e+00 / 0.00e+00 | 20260505 06:47:26.997 | INFO | PerfReport: 21.69%, LockCompute -- 2.20e+01 / 0.00e+00, 1.67e+01 / 0.00e+00 | 20260505 06:47:26.997 | INFO | PerfReport: 0.02%, Misc -- 1.61e-02 / 0.00e+00, 3.81e-02 / 0.00e+00 | 20260505 06:47:27.116 | INFO | Starting a round of Baz IO | 20260505 06:47:30.366 | INFO | Populating RTMetrics report | 20260505 06:47:30.367 | INFO | RTMetrics report populated | 20260505 06:47:30.367 | INFO | Opening tmp RTMetrics json file | 20260505 06:47:30.383 | INFO | Serializing RTMetrics to json | 20260505 06:47:30.384 | INFO | Serialization done, swapping tmp file now... | 20260505 06:47:30.384 | INFO | RTMetrics done | 20260505 06:47:30.482 | INFO | Finished a round of Baz IO: Wrote 2.398 GiB in 3.365 seconds | 20260505 06:47:30.482 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 31.98% / 0.04% / 67.98% | 20260505 06:47:30.482 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.53854% / 6.35783e-05% / 6.75519e-05% | 20260505 06:47:30.482 | INFO | Baz IO overall: | 20260505 06:47:30.482 | INFO | BytesWritten : 617517412437 | 20260505 06:47:30.482 | INFO | NumEvents : 292373065368 | 20260505 06:47:32.134 | INFO | Analyzing chunk frames = [1171456,1171968) | 20260505 06:47:32.134 | INFO | MemInfo: { | 20260505 06:47:32.134 | INFO | "gpu" : " 17 GB", | 20260505 06:47:32.134 | INFO | "huge" : "", | 20260505 06:47:32.134 | INFO | "huge_shared" : " 27 GB", | 20260505 06:47:32.134 | INFO | "malloc" : " 14 GB", | 20260505 06:47:32.135 | INFO | "pinned" : " 29 GB" | 20260505 06:47:32.135 | INFO | } | 20260505 06:47:32.135 | INFO | Perf report for the last 5.13973 seconds: | 20260505 06:47:32.135 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:47:32.135 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:32.135 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:32.135 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:32.135 | INFO | ANALYSIS : 64.4%, 2.9, 98.4ms, 0 | 20260505 06:47:32.135 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:32.135 | INFO | RT_METRICS : 46.6%, 1.0, 24.4ms, 0 | 20260505 06:47:32.135 | INFO | PRE_HQ : 63.3%, 1.9, 63.5ms, 0 | 20260505 06:47:32.135 | INFO | BAZWRITER : 64.8%, 4.1, 137.8ms, 0 | 20260505 06:47:32.135 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 06:47:32.135 | INFO | Uploaded 36GiB to GPU in 2.15352s (16.7168GiB/s) | 20260505 06:47:32.135 | INFO | Downloaded 38GiB to GPU in 2.31928s (16.3844GiB/s) | 20260505 06:47:37.273 | INFO | Analyzing chunk frames = [1171968,1172480) | 20260505 06:47:37.273 | INFO | MemInfo: { | 20260505 06:47:37.273 | INFO | "gpu" : " 17 GB", | 20260505 06:47:37.273 | INFO | "huge" : "", | 20260505 06:47:37.273 | INFO | "huge_shared" : " 27 GB", | 20260505 06:47:37.273 | INFO | "malloc" : " 14 GB", | 20260505 06:47:37.273 | INFO | "pinned" : " 29 GB" | 20260505 06:47:37.273 | INFO | } | 20260505 06:47:37.273 | INFO | Perf report for the last 5.13857 seconds: | 20260505 06:47:37.273 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:47:37.273 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:37.273 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:37.273 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:37.273 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 06:47:37.273 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:37.273 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:37.273 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 06:47:37.274 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:37.274 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:37.274 | INFO | Uploaded 36GiB to GPU in 2.0947s (17.1863GiB/s) | 20260505 06:47:37.274 | INFO | Downloaded 34GiB to GPU in 2.03155s (16.736GiB/s) | 20260505 06:47:42.413 | INFO | Analyzing chunk frames = [1172480,1172992) | 20260505 06:47:42.413 | INFO | MemInfo: { | 20260505 06:47:42.413 | INFO | "gpu" : " 17 GB", | 20260505 06:47:42.413 | INFO | "huge" : "", | 20260505 06:47:42.413 | INFO | "huge_shared" : " 27 GB", | 20260505 06:47:42.413 | INFO | "malloc" : " 14 GB", | 20260505 06:47:42.413 | INFO | "pinned" : " 29 GB" | 20260505 06:47:42.413 | INFO | } | 20260505 06:47:42.413 | INFO | Perf report for the last 5.13968 seconds: | 20260505 06:47:42.413 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:47:42.413 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:42.413 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:42.413 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:42.413 | INFO | ANALYSIS : 63.8%, 3.0, 99.2ms, 0 | 20260505 06:47:42.413 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:42.413 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:42.413 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 06:47:42.413 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:42.413 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:42.413 | INFO | Uploaded 36GiB to GPU in 2.12781s (16.9188GiB/s) | 20260505 06:47:42.413 | INFO | Downloaded 34GiB to GPU in 2.04054s (16.6623GiB/s) | 20260505 06:47:47.552 | INFO | Analyzing chunk frames = [1172992,1173504) | 20260505 06:47:47.552 | INFO | MemInfo: { | 20260505 06:47:47.552 | INFO | "gpu" : " 17 GB", | 20260505 06:47:47.552 | INFO | "huge" : "", | 20260505 06:47:47.552 | INFO | "huge_shared" : " 27 GB", | 20260505 06:47:47.552 | INFO | "malloc" : " 14 GB", | 20260505 06:47:47.552 | INFO | "pinned" : " 29 GB" | 20260505 06:47:47.552 | INFO | } | 20260505 06:47:47.552 | INFO | Perf report for the last 5.13917 seconds: | 20260505 06:47:47.552 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:47:47.552 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:47.552 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:47.552 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:47.552 | INFO | ANALYSIS : 63.7%, 3.0, 98.9ms, 0 | 20260505 06:47:47.552 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:47.552 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:47.552 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 06:47:47.552 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:47.552 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:47.552 | INFO | Uploaded 36GiB to GPU in 2.16157s (16.6546GiB/s) | 20260505 06:47:47.552 | INFO | Downloaded 34GiB to GPU in 2.02944s (16.7534GiB/s) | 20260505 06:47:52.691 | INFO | Analyzing chunk frames = [1173504,1174016) | 20260505 06:47:52.691 | INFO | MemInfo: { | 20260505 06:47:52.691 | INFO | "gpu" : " 17 GB", | 20260505 06:47:52.691 | INFO | "huge" : "", | 20260505 06:47:52.691 | INFO | "huge_shared" : " 27 GB", | 20260505 06:47:52.691 | INFO | "malloc" : " 14 GB", | 20260505 06:47:52.691 | INFO | "pinned" : " 29 GB" | 20260505 06:47:52.691 | INFO | } | 20260505 06:47:52.691 | INFO | Perf report for the last 5.13915 seconds: | 20260505 06:47:52.691 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:47:52.691 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:52.691 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:52.691 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:52.691 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 06:47:52.691 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:52.691 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:52.691 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 06:47:52.691 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:52.691 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:52.692 | INFO | Uploaded 36GiB to GPU in 2.16753s (16.6087GiB/s) | 20260505 06:47:52.692 | INFO | Downloaded 34GiB to GPU in 2.04391s (16.6348GiB/s) | 20260505 06:47:57.832 | INFO | Analyzing chunk frames = [1174016,1174528) | 20260505 06:47:57.832 | INFO | MemInfo: { | 20260505 06:47:57.832 | INFO | "gpu" : " 17 GB", | 20260505 06:47:57.832 | INFO | "huge" : "", | 20260505 06:47:57.832 | INFO | "huge_shared" : " 27 GB", | 20260505 06:47:57.832 | INFO | "malloc" : " 14 GB", | 20260505 06:47:57.832 | INFO | "pinned" : " 29 GB" | 20260505 06:47:57.832 | INFO | } | 20260505 06:47:57.833 | INFO | Perf report for the last 5.14121 seconds: | 20260505 06:47:57.833 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:47:57.833 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:57.833 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:57.833 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:57.833 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 06:47:57.833 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:57.833 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:57.833 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 06:47:57.833 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:57.833 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:47:57.833 | INFO | Uploaded 36GiB to GPU in 2.15396s (16.7134GiB/s) | 20260505 06:47:57.833 | INFO | Downloaded 34GiB to GPU in 2.05406s (16.5526GiB/s) | 20260505 06:48:02.970 | INFO | Analyzing chunk frames = [1174528,1175040) | 20260505 06:48:02.970 | INFO | MemInfo: { | 20260505 06:48:02.970 | INFO | "gpu" : " 17 GB", | 20260505 06:48:02.970 | INFO | "huge" : "", | 20260505 06:48:02.970 | INFO | "huge_shared" : " 27 GB", | 20260505 06:48:02.970 | INFO | "malloc" : " 14 GB", | 20260505 06:48:02.970 | INFO | "pinned" : " 29 GB" | 20260505 06:48:02.970 | INFO | } | 20260505 06:48:02.970 | INFO | Perf report for the last 5.13776 seconds: | 20260505 06:48:02.970 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:48:02.970 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:02.970 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:02.971 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:02.971 | INFO | ANALYSIS : 64.9%, 3.0, 101.4ms, 0 | 20260505 06:48:02.971 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:02.971 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:02.971 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260505 06:48:02.971 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:02.971 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:02.971 | INFO | Uploaded 36GiB to GPU in 2.23776s (16.0875GiB/s) | 20260505 06:48:02.971 | INFO | Downloaded 34GiB to GPU in 2.15174s (15.8011GiB/s) | 20260505 06:48:08.109 | INFO | Analyzing chunk frames = [1175040,1175552) | 20260505 06:48:08.109 | INFO | MemInfo: { | 20260505 06:48:08.109 | INFO | "gpu" : " 17 GB", | 20260505 06:48:08.109 | INFO | "huge" : "", | 20260505 06:48:08.109 | INFO | "huge_shared" : " 27 GB", | 20260505 06:48:08.109 | INFO | "malloc" : " 14 GB", | 20260505 06:48:08.109 | INFO | "pinned" : " 29 GB" | 20260505 06:48:08.109 | INFO | } | 20260505 06:48:08.110 | INFO | Perf report for the last 5.1393 seconds: | 20260505 06:48:08.110 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:48:08.110 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:08.110 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:08.110 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:08.110 | INFO | ANALYSIS : 64.4%, 3.0, 100.6ms, 0 | 20260505 06:48:08.110 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:08.110 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:08.110 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260505 06:48:08.110 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:08.110 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:08.110 | INFO | Uploaded 36GiB to GPU in 2.17463s (16.5546GiB/s) | 20260505 06:48:08.110 | INFO | Downloaded 34GiB to GPU in 2.04838s (16.5985GiB/s) | 20260505 06:48:08.231 | INFO | Starting a round of Baz IO | 20260505 06:48:11.482 | INFO | Populating RTMetrics report | 20260505 06:48:11.482 | INFO | RTMetrics report populated | 20260505 06:48:11.482 | INFO | Opening tmp RTMetrics json file | 20260505 06:48:11.483 | INFO | Serializing RTMetrics to json | 20260505 06:48:11.485 | INFO | Serialization done, swapping tmp file now... | 20260505 06:48:11.485 | INFO | RTMetrics done | 20260505 06:48:11.603 | INFO | Finished a round of Baz IO: Wrote 2.398 GiB in 3.371 seconds | 20260505 06:48:11.603 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 31.94% / 0.04% / 68.02% | 20260505 06:48:11.603 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.51847% / 5.96046e-05% / 7.54992e-05% | 20260505 06:48:11.603 | INFO | Baz IO overall: | 20260505 06:48:11.603 | INFO | BytesWritten : 620092022141 | 20260505 06:48:11.603 | INFO | NumEvents : 293603949819 | 20260505 06:48:13.249 | INFO | Analyzing chunk frames = [1175552,1176064) | 20260505 06:48:13.249 | INFO | MemInfo: { | 20260505 06:48:13.249 | INFO | "gpu" : " 17 GB", | 20260505 06:48:13.249 | INFO | "huge" : "", | 20260505 06:48:13.249 | INFO | "huge_shared" : " 27 GB", | 20260505 06:48:13.249 | INFO | "malloc" : " 14 GB", | 20260505 06:48:13.249 | INFO | "pinned" : " 29 GB" | 20260505 06:48:13.249 | INFO | } | 20260505 06:48:13.249 | INFO | Perf report for the last 5.13972 seconds: | 20260505 06:48:13.249 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:48:13.249 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:13.249 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:13.249 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:13.249 | INFO | ANALYSIS : 64.5%, 2.9, 98.6ms, 0 | 20260505 06:48:13.249 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:13.249 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 06:48:13.249 | INFO | PRE_HQ : 63.3%, 1.9, 63.3ms, 0 | 20260505 06:48:13.249 | INFO | BAZWRITER : 64.9%, 4.1, 137.4ms, 0 | 20260505 06:48:13.249 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 06:48:13.250 | INFO | Uploaded 36GiB to GPU in 2.15509s (16.7046GiB/s) | 20260505 06:48:13.250 | INFO | Downloaded 38GiB to GPU in 2.30392s (16.4936GiB/s) | 20260505 06:48:18.388 | INFO | Analyzing chunk frames = [1176064,1176576) | 20260505 06:48:18.388 | INFO | MemInfo: { | 20260505 06:48:18.388 | INFO | "gpu" : " 17 GB", | 20260505 06:48:18.388 | INFO | "huge" : "", | 20260505 06:48:18.388 | INFO | "huge_shared" : " 27 GB", | 20260505 06:48:18.388 | INFO | "malloc" : " 14 GB", | 20260505 06:48:18.388 | INFO | "pinned" : " 29 GB" | 20260505 06:48:18.388 | INFO | } | 20260505 06:48:18.388 | INFO | Perf report for the last 5.13871 seconds: | 20260505 06:48:18.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:48:18.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:18.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:18.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:18.388 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 06:48:18.388 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:18.388 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:18.388 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 06:48:18.388 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:18.388 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:18.388 | INFO | Uploaded 36GiB to GPU in 2.10004s (17.1425GiB/s) | 20260505 06:48:18.389 | INFO | Downloaded 34GiB to GPU in 2.03378s (16.7176GiB/s) | 20260505 06:48:23.527 | INFO | Analyzing chunk frames = [1176576,1177088) | 20260505 06:48:23.527 | INFO | MemInfo: { | 20260505 06:48:23.527 | INFO | "gpu" : " 17 GB", | 20260505 06:48:23.527 | INFO | "huge" : "", | 20260505 06:48:23.527 | INFO | "huge_shared" : " 27 GB", | 20260505 06:48:23.527 | INFO | "malloc" : " 14 GB", | 20260505 06:48:23.527 | INFO | "pinned" : " 29 GB" | 20260505 06:48:23.527 | INFO | } | 20260505 06:48:23.527 | INFO | Perf report for the last 5.13914 seconds: | 20260505 06:48:23.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:48:23.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:23.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:23.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:23.527 | INFO | ANALYSIS : 63.7%, 3.0, 99.1ms, 0 | 20260505 06:48:23.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:23.527 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:23.527 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 06:48:23.527 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:23.527 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:23.528 | INFO | Uploaded 36GiB to GPU in 2.13353s (16.8735GiB/s) | 20260505 06:48:23.528 | INFO | Downloaded 34GiB to GPU in 2.04724s (16.6078GiB/s) | 20260505 06:48:28.666 | INFO | Analyzing chunk frames = [1177088,1177600) | 20260505 06:48:28.667 | INFO | MemInfo: { | 20260505 06:48:28.667 | INFO | "gpu" : " 17 GB", | 20260505 06:48:28.667 | INFO | "huge" : "", | 20260505 06:48:28.667 | INFO | "huge_shared" : " 27 GB", | 20260505 06:48:28.667 | INFO | "malloc" : " 14 GB", | 20260505 06:48:28.667 | INFO | "pinned" : " 29 GB" | 20260505 06:48:28.667 | INFO | } | 20260505 06:48:28.667 | INFO | Perf report for the last 5.13956 seconds: | 20260505 06:48:28.667 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:48:28.667 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:28.667 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:28.667 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:28.667 | INFO | ANALYSIS : 63.6%, 3.0, 98.8ms, 0 | 20260505 06:48:28.667 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:28.667 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:28.667 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 06:48:28.667 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:28.667 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:28.667 | INFO | Uploaded 36GiB to GPU in 2.1506s (16.7395GiB/s) | 20260505 06:48:28.667 | INFO | Downloaded 34GiB to GPU in 2.02748s (16.7696GiB/s) | 20260505 06:48:33.806 | INFO | Analyzing chunk frames = [1177600,1178112) | 20260505 06:48:33.806 | INFO | MemInfo: { | 20260505 06:48:33.806 | INFO | "gpu" : " 17 GB", | 20260505 06:48:33.806 | INFO | "huge" : "", | 20260505 06:48:33.806 | INFO | "huge_shared" : " 27 GB", | 20260505 06:48:33.806 | INFO | "malloc" : " 14 GB", | 20260505 06:48:33.806 | INFO | "pinned" : " 29 GB" | 20260505 06:48:33.806 | INFO | } | 20260505 06:48:33.806 | INFO | Perf report for the last 5.1394 seconds: | 20260505 06:48:33.806 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:48:33.806 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:33.806 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:33.806 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:33.806 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 06:48:33.806 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:33.806 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:33.806 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 06:48:33.807 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:33.807 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:33.807 | INFO | Uploaded 36GiB to GPU in 2.11979s (16.9828GiB/s) | 20260505 06:48:33.807 | INFO | Downloaded 34GiB to GPU in 2.03041s (16.7454GiB/s) | 20260505 06:48:38.947 | INFO | Analyzing chunk frames = [1178112,1178624) | 20260505 06:48:38.947 | INFO | MemInfo: { | 20260505 06:48:38.947 | INFO | "gpu" : " 17 GB", | 20260505 06:48:38.947 | INFO | "huge" : "", | 20260505 06:48:38.947 | INFO | "huge_shared" : " 27 GB", | 20260505 06:48:38.947 | INFO | "malloc" : " 14 GB", | 20260505 06:48:38.947 | INFO | "pinned" : " 29 GB" | 20260505 06:48:38.947 | INFO | } | 20260505 06:48:38.947 | INFO | Perf report for the last 5.14055 seconds: | 20260505 06:48:38.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:48:38.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:38.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:38.947 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:38.947 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 06:48:38.947 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:38.947 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:38.947 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 06:48:38.947 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:38.947 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:38.947 | INFO | Uploaded 36GiB to GPU in 2.1372s (16.8445GiB/s) | 20260505 06:48:38.947 | INFO | Downloaded 34GiB to GPU in 2.04369s (16.6366GiB/s) | 20260505 06:48:44.085 | INFO | Analyzing chunk frames = [1178624,1179136) | 20260505 06:48:44.085 | INFO | MemInfo: { | 20260505 06:48:44.085 | INFO | "gpu" : " 17 GB", | 20260505 06:48:44.085 | INFO | "huge" : "", | 20260505 06:48:44.085 | INFO | "huge_shared" : " 27 GB", | 20260505 06:48:44.085 | INFO | "malloc" : " 14 GB", | 20260505 06:48:44.085 | INFO | "pinned" : " 29 GB" | 20260505 06:48:44.085 | INFO | } | 20260505 06:48:44.085 | INFO | Perf report for the last 5.13871 seconds: | 20260505 06:48:44.085 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:48:44.085 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:44.085 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:44.085 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:44.086 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 06:48:44.086 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:44.086 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:44.086 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 06:48:44.086 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:44.086 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:44.086 | INFO | Uploaded 36GiB to GPU in 2.22073s (16.2109GiB/s) | 20260505 06:48:44.086 | INFO | Downloaded 34GiB to GPU in 2.17795s (15.611GiB/s) | 20260505 06:48:49.224 | INFO | Analyzing chunk frames = [1179136,1179648) | 20260505 06:48:49.224 | INFO | MemInfo: { | 20260505 06:48:49.224 | INFO | "gpu" : " 17 GB", | 20260505 06:48:49.224 | INFO | "huge" : "", | 20260505 06:48:49.224 | INFO | "huge_shared" : " 27 GB", | 20260505 06:48:49.224 | INFO | "malloc" : " 14 GB", | 20260505 06:48:49.224 | INFO | "pinned" : " 29 GB" | 20260505 06:48:49.224 | INFO | } | 20260505 06:48:49.224 | INFO | Perf report for the last 5.13881 seconds: | 20260505 06:48:49.224 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:48:49.224 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:49.224 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:49.224 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:49.224 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 06:48:49.224 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:49.224 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:49.224 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 06:48:49.224 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:49.224 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:49.225 | INFO | Uploaded 36GiB to GPU in 2.16293s (16.6441GiB/s) | 20260505 06:48:49.225 | INFO | Downloaded 34GiB to GPU in 2.04616s (16.6165GiB/s) | 20260505 06:48:49.225 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 06:48:49.225 | INFO | Total mem usage: 15.3 GiB | 20260505 06:48:49.225 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 06:48:49.225 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.72%) | 20260505 06:48:49.225 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 06:48:49.225 | INFO | High Water Breakdown: | 20260505 06:48:49.225 | INFO | 60.4% : 8.67 GiB ( 92.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 06:48:49.225 | INFO | 39.5% : 5.67 GiB ( 85.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 06:48:49.225 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 06:48:49.225 | INFO | ------------------------------------------ | 20260505 06:48:49.225 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:48:49.225 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 06:48:49.225 | INFO | Total mem usage: 26.5 GiB | 20260505 06:48:49.225 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 06:48:49.225 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 06:48:49.225 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 06:48:49.225 | INFO | High Water Breakdown: | 20260505 06:48:49.225 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:48:49.225 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:48:49.225 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:48:49.225 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:48:49.225 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:48:49.225 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:48:49.225 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 06:48:49.225 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 06:48:49.225 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:48:49.225 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 06:48:49.225 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 06:48:49.225 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 06:48:49.225 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 06:48:49.225 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 06:48:49.225 | INFO | ------------------------------------------ | 20260505 06:48:49.225 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:48:49.225 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 06:48:49.225 | INFO | Total mem usage: 25.6 GiB | 20260505 06:48:49.225 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 06:48:49.225 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 06:48:49.225 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 06:48:49.225 | INFO | High Water Breakdown: | 20260505 06:48:49.225 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 06:48:49.225 | INFO | ------------------------------------------ | 20260505 06:48:49.225 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:48:49.225 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 06:48:49.225 | INFO | Total mem usage: 15.7 GiB | 20260505 06:48:49.225 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 06:48:49.225 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 06:48:49.225 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260505 06:48:49.225 | INFO | High Water Breakdown: | 20260505 06:48:49.226 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 06:48:49.226 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 06:48:49.226 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 06:48:49.226 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 06:48:49.226 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 06:48:49.226 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 06:48:49.226 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 06:48:49.226 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 06:48:49.226 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 06:48:49.226 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:48:49.226 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 06:48:49.226 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:48:49.226 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 06:48:49.226 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 06:48:49.226 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 06:48:49.226 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 06:48:49.226 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:48:49.226 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 06:48:49.226 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:48:49.226 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:48:49.226 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:48:49.226 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 06:48:49.226 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 06:48:49.226 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:48:49.226 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 06:48:49.226 | INFO | ------------------------------------------ | 20260505 06:48:49.226 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:48:49.226 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 06:48:49.226 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:48:49.226 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:48:49.226 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.55e+01, 1.37e+01 / 1.57e+00 | 20260505 06:48:49.226 | INFO | PerfReport: 0.28%, AggregateStats -- 9.38e-02 / 1.02e-01, 1.19e-02 / 4.57e-03 | 20260505 06:48:49.226 | INFO | PerfReport: 11.59%, Baseline -- 3.84e+00 / 4.46e+00, 6.42e-01 / 3.17e-01 | 20260505 06:48:49.226 | INFO | PerfReport: 3.31%, Histogram -- 1.09e+00 / 1.47e+00, 2.17e-01 / 1.07e-01 | 20260505 06:48:49.226 | INFO | PerfReport: 15.49%, DME -- 4.10e+01 / 4.50e+01, 4.19e+00 / 1.17e+00 | 20260505 06:48:49.226 | INFO | PerfReport: 56.20%, FrameLabeling -- 1.86e+01 / 1.92e+01, 2.10e+00 / 1.04e+00 | 20260505 06:48:49.226 | INFO | PerfReport: 7.61%, PulseAccumulating -- 2.52e+00 / 3.07e+00, 4.50e-01 / 3.52e-01 | 20260505 06:48:49.226 | INFO | PerfReport: 4.05%, Metrics -- 1.34e+00 / 1.68e+00, 3.86e-01 / 1.87e-01 | 20260505 06:48:49.226 | INFO | PerfReport: 1.47%, PulseSerializing -- 4.86e-01 / 5.56e-01, 9.42e-02 / 4.23e-02 | 20260505 06:48:49.226 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 06:48:49.226 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:48:49.226 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:48:49.226 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.22e+01 / 0.00e+00 | 20260505 06:48:49.226 | INFO | PerfReport: 22.09%, Upload -- 2.24e+01 / 0.00e+00, 7.90e+00 / 0.00e+00 | 20260505 06:48:49.226 | INFO | PerfReport: 21.43%, Download -- 2.18e+01 / 0.00e+00, 7.78e+00 / 0.00e+00 | 20260505 06:48:49.226 | INFO | PerfReport: 32.59%, Compute -- 3.31e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 06:48:49.226 | INFO | PerfReport: 1.48%, LockUpload -- 1.50e+00 / 0.00e+00, 6.92e+00 / 0.00e+00 | 20260505 06:48:49.226 | INFO | PerfReport: 0.76%, LockDownload -- 7.69e-01 / 0.00e+00, 2.17e+00 / 0.00e+00 | 20260505 06:48:49.226 | INFO | PerfReport: 21.64%, LockCompute -- 2.20e+01 / 0.00e+00, 1.66e+01 / 0.00e+00 | 20260505 06:48:49.226 | INFO | PerfReport: 0.01%, Misc -- 1.52e-02 / 0.00e+00, 2.90e-02 / 0.00e+00 | 20260505 06:48:49.346 | INFO | Starting a round of Baz IO | 20260505 06:48:52.596 | INFO | Populating RTMetrics report | 20260505 06:48:52.596 | INFO | RTMetrics report populated | 20260505 06:48:52.596 | INFO | Opening tmp RTMetrics json file | 20260505 06:48:52.625 | INFO | Serializing RTMetrics to json | 20260505 06:48:52.648 | INFO | Serialization done, swapping tmp file now... | 20260505 06:48:52.649 | INFO | RTMetrics done | 20260505 06:48:52.711 | INFO | Finished a round of Baz IO: Wrote 2.398 GiB in 3.365 seconds | 20260505 06:48:52.711 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 31.90% / 0.04% / 68.06% | 20260505 06:48:52.711 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.49979% / 5.5631e-05% / 2.78155e-05% | 20260505 06:48:52.711 | INFO | Baz IO overall: | 20260505 06:48:52.711 | INFO | BytesWritten : 622667325013 | 20260505 06:48:52.711 | INFO | NumEvents : 294835018104 | 20260505 06:48:54.363 | INFO | Analyzing chunk frames = [1179648,1180160) | 20260505 06:48:54.363 | INFO | MemInfo: { | 20260505 06:48:54.363 | INFO | "gpu" : " 17 GB", | 20260505 06:48:54.363 | INFO | "huge" : "", | 20260505 06:48:54.363 | INFO | "huge_shared" : " 27 GB", | 20260505 06:48:54.363 | INFO | "malloc" : " 14 GB", | 20260505 06:48:54.363 | INFO | "pinned" : " 29 GB" | 20260505 06:48:54.363 | INFO | } | 20260505 06:48:54.363 | INFO | Perf report for the last 5.13894 seconds: | 20260505 06:48:54.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:48:54.363 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:54.363 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:54.363 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:54.363 | INFO | ANALYSIS : 64.4%, 2.9, 98.1ms, 0 | 20260505 06:48:54.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:54.363 | INFO | RT_METRICS : 47.0%, 1.0, 24.7ms, 0 | 20260505 06:48:54.363 | INFO | PRE_HQ : 63.4%, 1.9, 63.4ms, 0 | 20260505 06:48:54.364 | INFO | BAZWRITER : 64.8%, 4.2, 138.7ms, 0 | 20260505 06:48:54.364 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 06:48:54.364 | INFO | Uploaded 36GiB to GPU in 2.14402s (16.7909GiB/s) | 20260505 06:48:54.364 | INFO | Downloaded 38GiB to GPU in 2.30311s (16.4995GiB/s) | 20260505 06:48:59.503 | INFO | Analyzing chunk frames = [1180160,1180672) | 20260505 06:48:59.503 | INFO | MemInfo: { | 20260505 06:48:59.503 | INFO | "gpu" : " 17 GB", | 20260505 06:48:59.503 | INFO | "huge" : "", | 20260505 06:48:59.503 | INFO | "huge_shared" : " 27 GB", | 20260505 06:48:59.503 | INFO | "malloc" : " 14 GB", | 20260505 06:48:59.503 | INFO | "pinned" : " 29 GB" | 20260505 06:48:59.503 | INFO | } | 20260505 06:48:59.503 | INFO | Perf report for the last 5.13986 seconds: | 20260505 06:48:59.503 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:48:59.503 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:59.503 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:59.503 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:59.503 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 06:48:59.503 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:59.503 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:59.503 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 06:48:59.503 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:59.503 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:48:59.503 | INFO | Uploaded 36GiB to GPU in 2.09881s (17.1526GiB/s) | 20260505 06:48:59.504 | INFO | Downloaded 34GiB to GPU in 2.03821s (16.6813GiB/s) | 20260505 06:49:04.642 | INFO | Analyzing chunk frames = [1180672,1181184) | 20260505 06:49:04.642 | INFO | MemInfo: { | 20260505 06:49:04.642 | INFO | "gpu" : " 17 GB", | 20260505 06:49:04.642 | INFO | "huge" : "", | 20260505 06:49:04.642 | INFO | "huge_shared" : " 27 GB", | 20260505 06:49:04.642 | INFO | "malloc" : " 14 GB", | 20260505 06:49:04.642 | INFO | "pinned" : " 29 GB" | 20260505 06:49:04.642 | INFO | } | 20260505 06:49:04.642 | INFO | Perf report for the last 5.13872 seconds: | 20260505 06:49:04.642 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:49:04.642 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:04.642 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:04.642 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:04.642 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260505 06:49:04.642 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:04.642 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:04.642 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 06:49:04.642 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:04.642 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:04.642 | INFO | Uploaded 36GiB to GPU in 2.11649s (17.0093GiB/s) | 20260505 06:49:04.642 | INFO | Downloaded 34GiB to GPU in 2.04848s (16.5977GiB/s) | 20260505 06:49:09.782 | INFO | Analyzing chunk frames = [1181184,1181696) | 20260505 06:49:09.782 | INFO | MemInfo: { | 20260505 06:49:09.782 | INFO | "gpu" : " 17 GB", | 20260505 06:49:09.782 | INFO | "huge" : "", | 20260505 06:49:09.782 | INFO | "huge_shared" : " 27 GB", | 20260505 06:49:09.782 | INFO | "malloc" : " 14 GB", | 20260505 06:49:09.782 | INFO | "pinned" : " 29 GB" | 20260505 06:49:09.782 | INFO | } | 20260505 06:49:09.782 | INFO | Perf report for the last 5.13974 seconds: | 20260505 06:49:09.782 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:49:09.782 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:09.782 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:09.782 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:09.782 | INFO | ANALYSIS : 63.7%, 3.0, 99.0ms, 0 | 20260505 06:49:09.782 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:09.782 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:09.782 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 06:49:09.782 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:09.782 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:09.782 | INFO | Uploaded 36GiB to GPU in 2.16343s (16.6403GiB/s) | 20260505 06:49:09.782 | INFO | Downloaded 34GiB to GPU in 2.02982s (16.7503GiB/s) | 20260505 06:49:14.921 | INFO | Analyzing chunk frames = [1181696,1182208) | 20260505 06:49:14.921 | INFO | MemInfo: { | 20260505 06:49:14.921 | INFO | "gpu" : " 17 GB", | 20260505 06:49:14.921 | INFO | "huge" : "", | 20260505 06:49:14.921 | INFO | "huge_shared" : " 27 GB", | 20260505 06:49:14.921 | INFO | "malloc" : " 14 GB", | 20260505 06:49:14.921 | INFO | "pinned" : " 29 GB" | 20260505 06:49:14.921 | INFO | } | 20260505 06:49:14.921 | INFO | Perf report for the last 5.13914 seconds: | 20260505 06:49:14.921 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:49:14.921 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:14.921 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:14.921 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:14.921 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 06:49:14.921 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:14.921 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:14.921 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 06:49:14.921 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:14.921 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:14.921 | INFO | Uploaded 36GiB to GPU in 2.11698s (17.0054GiB/s) | 20260505 06:49:14.921 | INFO | Downloaded 34GiB to GPU in 2.03359s (16.7192GiB/s) | 20260505 06:49:20.062 | INFO | Analyzing chunk frames = [1182208,1182720) | 20260505 06:49:20.062 | INFO | MemInfo: { | 20260505 06:49:20.062 | INFO | "gpu" : " 17 GB", | 20260505 06:49:20.062 | INFO | "huge" : "", | 20260505 06:49:20.062 | INFO | "huge_shared" : " 27 GB", | 20260505 06:49:20.062 | INFO | "malloc" : " 14 GB", | 20260505 06:49:20.062 | INFO | "pinned" : " 29 GB" | 20260505 06:49:20.062 | INFO | } | 20260505 06:49:20.062 | INFO | Perf report for the last 5.14104 seconds: | 20260505 06:49:20.062 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:49:20.062 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:20.062 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:20.062 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:20.062 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 06:49:20.062 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:20.062 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:20.062 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 06:49:20.062 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:20.062 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:20.062 | INFO | Uploaded 36GiB to GPU in 2.14483s (16.7846GiB/s) | 20260505 06:49:20.062 | INFO | Downloaded 34GiB to GPU in 2.03967s (16.6693GiB/s) | 20260505 06:49:25.199 | INFO | Analyzing chunk frames = [1182720,1183232) | 20260505 06:49:25.199 | INFO | MemInfo: { | 20260505 06:49:25.199 | INFO | "gpu" : " 17 GB", | 20260505 06:49:25.199 | INFO | "huge" : "", | 20260505 06:49:25.199 | INFO | "huge_shared" : " 27 GB", | 20260505 06:49:25.199 | INFO | "malloc" : " 14 GB", | 20260505 06:49:25.199 | INFO | "pinned" : " 29 GB" | 20260505 06:49:25.199 | INFO | } | 20260505 06:49:25.200 | INFO | Perf report for the last 5.13778 seconds: | 20260505 06:49:25.200 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:49:25.200 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:25.200 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:25.200 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:25.200 | INFO | ANALYSIS : 64.4%, 3.0, 100.6ms, 0 | 20260505 06:49:25.200 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:25.200 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:25.200 | INFO | PRE_HQ : 32.3%, 1.0, 16.7ms, 0 | 20260505 06:49:25.200 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:25.200 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:25.200 | INFO | Uploaded 36GiB to GPU in 2.20577s (16.3208GiB/s) | 20260505 06:49:25.200 | INFO | Downloaded 34GiB to GPU in 2.16588s (15.698GiB/s) | 20260505 06:49:30.339 | INFO | Analyzing chunk frames = [1183232,1183744) | 20260505 06:49:30.339 | INFO | MemInfo: { | 20260505 06:49:30.339 | INFO | "gpu" : " 17 GB", | 20260505 06:49:30.339 | INFO | "huge" : "", | 20260505 06:49:30.339 | INFO | "huge_shared" : " 27 GB", | 20260505 06:49:30.339 | INFO | "malloc" : " 14 GB", | 20260505 06:49:30.339 | INFO | "pinned" : " 29 GB" | 20260505 06:49:30.339 | INFO | } | 20260505 06:49:30.339 | INFO | Perf report for the last 5.13981 seconds: | 20260505 06:49:30.339 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:49:30.339 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:30.339 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:30.339 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:30.339 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 06:49:30.340 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:30.340 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:30.340 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 06:49:30.340 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:30.340 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:30.340 | INFO | Uploaded 36GiB to GPU in 2.16519s (16.6268GiB/s) | 20260505 06:49:30.340 | INFO | Downloaded 34GiB to GPU in 2.04303s (16.642GiB/s) | 20260505 06:49:30.460 | INFO | Starting a round of Baz IO | 20260505 06:49:33.699 | INFO | Populating RTMetrics report | 20260505 06:49:33.699 | INFO | RTMetrics report populated | 20260505 06:49:33.699 | INFO | Opening tmp RTMetrics json file | 20260505 06:49:33.700 | INFO | Serializing RTMetrics to json | 20260505 06:49:33.701 | INFO | Serialization done, swapping tmp file now... | 20260505 06:49:33.702 | INFO | RTMetrics done | 20260505 06:49:33.826 | INFO | Finished a round of Baz IO: Wrote 2.398 GiB in 3.366 seconds | 20260505 06:49:33.826 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 31.86% / 0.04% / 68.10% | 20260505 06:49:33.826 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.47409% / 4.76837e-05% / 6.75519e-05% | 20260505 06:49:33.826 | INFO | Baz IO overall: | 20260505 06:49:33.826 | INFO | BytesWritten : 625242673153 | 20260505 06:49:33.826 | INFO | NumEvents : 296065859446 | 20260505 06:49:35.478 | INFO | Analyzing chunk frames = [1183744,1184256) | 20260505 06:49:35.478 | INFO | MemInfo: { | 20260505 06:49:35.478 | INFO | "gpu" : " 17 GB", | 20260505 06:49:35.478 | INFO | "huge" : "", | 20260505 06:49:35.478 | INFO | "huge_shared" : " 27 GB", | 20260505 06:49:35.478 | INFO | "malloc" : " 14 GB", | 20260505 06:49:35.478 | INFO | "pinned" : " 29 GB" | 20260505 06:49:35.478 | INFO | } | 20260505 06:49:35.478 | INFO | Perf report for the last 5.1389 seconds: | 20260505 06:49:35.478 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:49:35.478 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:35.478 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:35.478 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:35.478 | INFO | ANALYSIS : 64.2%, 2.9, 98.0ms, 0 | 20260505 06:49:35.478 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:35.478 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260505 06:49:35.478 | INFO | PRE_HQ : 63.2%, 1.9, 63.5ms, 0 | 20260505 06:49:35.478 | INFO | BAZWRITER : 64.8%, 4.2, 139.0ms, 0 | 20260505 06:49:35.478 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 06:49:35.479 | INFO | Uploaded 36GiB to GPU in 2.13347s (16.874GiB/s) | 20260505 06:49:35.479 | INFO | Downloaded 38GiB to GPU in 2.31139s (16.4403GiB/s) | 20260505 06:49:40.617 | INFO | Analyzing chunk frames = [1184256,1184768) | 20260505 06:49:40.617 | INFO | MemInfo: { | 20260505 06:49:40.617 | INFO | "gpu" : " 17 GB", | 20260505 06:49:40.617 | INFO | "huge" : "", | 20260505 06:49:40.617 | INFO | "huge_shared" : " 27 GB", | 20260505 06:49:40.617 | INFO | "malloc" : " 14 GB", | 20260505 06:49:40.617 | INFO | "pinned" : " 29 GB" | 20260505 06:49:40.617 | INFO | } | 20260505 06:49:40.617 | INFO | Perf report for the last 5.13903 seconds: | 20260505 06:49:40.618 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:49:40.618 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:40.618 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:40.618 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:40.618 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 06:49:40.618 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:40.618 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:40.618 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 06:49:40.618 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:40.618 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:40.618 | INFO | Uploaded 36GiB to GPU in 2.09279s (17.2019GiB/s) | 20260505 06:49:40.618 | INFO | Downloaded 34GiB to GPU in 2.03836s (16.6801GiB/s) | 20260505 06:49:45.756 | INFO | Analyzing chunk frames = [1184768,1185280) | 20260505 06:49:45.757 | INFO | MemInfo: { | 20260505 06:49:45.757 | INFO | "gpu" : " 17 GB", | 20260505 06:49:45.757 | INFO | "huge" : "", | 20260505 06:49:45.757 | INFO | "huge_shared" : " 27 GB", | 20260505 06:49:45.757 | INFO | "malloc" : " 14 GB", | 20260505 06:49:45.757 | INFO | "pinned" : " 29 GB" | 20260505 06:49:45.757 | INFO | } | 20260505 06:49:45.757 | INFO | Perf report for the last 5.1392 seconds: | 20260505 06:49:45.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:49:45.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:45.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:45.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:45.757 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260505 06:49:45.757 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:45.757 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:45.757 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 06:49:45.757 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:45.757 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:45.757 | INFO | Uploaded 36GiB to GPU in 2.13308s (16.877GiB/s) | 20260505 06:49:45.757 | INFO | Downloaded 34GiB to GPU in 2.04914s (16.5923GiB/s) | 20260505 06:49:50.896 | INFO | Analyzing chunk frames = [1185280,1185792) | 20260505 06:49:50.896 | INFO | MemInfo: { | 20260505 06:49:50.896 | INFO | "gpu" : " 17 GB", | 20260505 06:49:50.896 | INFO | "huge" : "", | 20260505 06:49:50.896 | INFO | "huge_shared" : " 27 GB", | 20260505 06:49:50.896 | INFO | "malloc" : " 14 GB", | 20260505 06:49:50.896 | INFO | "pinned" : " 29 GB" | 20260505 06:49:50.896 | INFO | } | 20260505 06:49:50.896 | INFO | Perf report for the last 5.13946 seconds: | 20260505 06:49:50.896 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:49:50.896 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:50.896 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:50.896 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:50.896 | INFO | ANALYSIS : 63.6%, 3.0, 98.8ms, 0 | 20260505 06:49:50.896 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:50.896 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:50.896 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 06:49:50.896 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:50.896 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:50.897 | INFO | Uploaded 36GiB to GPU in 2.13863s (16.8332GiB/s) | 20260505 06:49:50.897 | INFO | Downloaded 34GiB to GPU in 2.03141s (16.7371GiB/s) | 20260505 06:49:56.035 | INFO | Analyzing chunk frames = [1185792,1186304) | 20260505 06:49:56.035 | INFO | MemInfo: { | 20260505 06:49:56.035 | INFO | "gpu" : " 17 GB", | 20260505 06:49:56.035 | INFO | "huge" : "", | 20260505 06:49:56.035 | INFO | "huge_shared" : " 27 GB", | 20260505 06:49:56.035 | INFO | "malloc" : " 14 GB", | 20260505 06:49:56.035 | INFO | "pinned" : " 29 GB" | 20260505 06:49:56.035 | INFO | } | 20260505 06:49:56.035 | INFO | Perf report for the last 5.13944 seconds: | 20260505 06:49:56.036 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:49:56.036 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:56.036 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:56.036 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:56.036 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 06:49:56.036 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:56.036 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:56.036 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 06:49:56.036 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:56.036 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:49:56.036 | INFO | Uploaded 36GiB to GPU in 2.16168s (16.6537GiB/s) | 20260505 06:49:56.036 | INFO | Downloaded 34GiB to GPU in 2.03935s (16.672GiB/s) | 20260505 06:50:01.176 | INFO | Analyzing chunk frames = [1186304,1186816) | 20260505 06:50:01.176 | INFO | MemInfo: { | 20260505 06:50:01.176 | INFO | "gpu" : " 17 GB", | 20260505 06:50:01.176 | INFO | "huge" : "", | 20260505 06:50:01.176 | INFO | "huge_shared" : " 27 GB", | 20260505 06:50:01.176 | INFO | "malloc" : " 14 GB", | 20260505 06:50:01.176 | INFO | "pinned" : " 29 GB" | 20260505 06:50:01.176 | INFO | } | 20260505 06:50:01.176 | INFO | Perf report for the last 5.14085 seconds: | 20260505 06:50:01.177 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:50:01.177 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:01.177 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:01.177 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:01.177 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 06:50:01.177 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:01.177 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:01.177 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 06:50:01.177 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:01.177 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:01.177 | INFO | Uploaded 36GiB to GPU in 2.14944s (16.7485GiB/s) | 20260505 06:50:01.177 | INFO | Downloaded 34GiB to GPU in 2.03878s (16.6767GiB/s) | 20260505 06:50:06.314 | INFO | Analyzing chunk frames = [1186816,1187328) | 20260505 06:50:06.314 | INFO | MemInfo: { | 20260505 06:50:06.314 | INFO | "gpu" : " 17 GB", | 20260505 06:50:06.314 | INFO | "huge" : "", | 20260505 06:50:06.314 | INFO | "huge_shared" : " 27 GB", | 20260505 06:50:06.314 | INFO | "malloc" : " 14 GB", | 20260505 06:50:06.314 | INFO | "pinned" : " 29 GB" | 20260505 06:50:06.314 | INFO | } | 20260505 06:50:06.314 | INFO | Perf report for the last 5.13782 seconds: | 20260505 06:50:06.314 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:50:06.314 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:06.314 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:06.314 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:06.314 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 06:50:06.314 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:06.314 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:06.314 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260505 06:50:06.314 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:06.315 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:06.315 | INFO | Uploaded 36GiB to GPU in 2.20695s (16.3121GiB/s) | 20260505 06:50:06.315 | INFO | Downloaded 34GiB to GPU in 2.15476s (15.779GiB/s) | 20260505 06:50:11.453 | INFO | Analyzing chunk frames = [1187328,1187840) | 20260505 06:50:11.453 | INFO | MemInfo: { | 20260505 06:50:11.453 | INFO | "gpu" : " 17 GB", | 20260505 06:50:11.453 | INFO | "huge" : "", | 20260505 06:50:11.453 | INFO | "huge_shared" : " 27 GB", | 20260505 06:50:11.453 | INFO | "malloc" : " 14 GB", | 20260505 06:50:11.453 | INFO | "pinned" : " 29 GB" | 20260505 06:50:11.453 | INFO | } | 20260505 06:50:11.453 | INFO | Perf report for the last 5.13894 seconds: | 20260505 06:50:11.453 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:50:11.453 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:11.453 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:11.453 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:11.453 | INFO | ANALYSIS : 64.4%, 3.0, 100.6ms, 0 | 20260505 06:50:11.453 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:11.453 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:11.453 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 06:50:11.453 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:11.453 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:11.454 | INFO | Uploaded 36GiB to GPU in 2.16151s (16.655GiB/s) | 20260505 06:50:11.454 | INFO | Downloaded 34GiB to GPU in 2.05205s (16.5688GiB/s) | 20260505 06:50:11.454 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 06:50:11.454 | INFO | Total mem usage: 15.3 GiB | 20260505 06:50:11.454 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 06:50:11.454 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.75%) | 20260505 06:50:11.454 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 06:50:11.454 | INFO | High Water Breakdown: | 20260505 06:50:11.454 | INFO | 60.4% : 8.67 GiB ( 92.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 06:50:11.454 | INFO | 39.5% : 5.67 GiB ( 85.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 06:50:11.454 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 06:50:11.454 | INFO | ------------------------------------------ | 20260505 06:50:11.454 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:50:11.454 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 06:50:11.454 | INFO | Total mem usage: 26.5 GiB | 20260505 06:50:11.454 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 06:50:11.454 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 06:50:11.454 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 06:50:11.454 | INFO | High Water Breakdown: | 20260505 06:50:11.454 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:50:11.454 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:50:11.454 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:50:11.454 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:50:11.454 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:50:11.454 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:50:11.454 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 06:50:11.454 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 06:50:11.454 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:50:11.454 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 06:50:11.454 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 06:50:11.454 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 06:50:11.454 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 06:50:11.454 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 06:50:11.454 | INFO | ------------------------------------------ | 20260505 06:50:11.454 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:50:11.454 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 06:50:11.454 | INFO | Total mem usage: 25.6 GiB | 20260505 06:50:11.454 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 06:50:11.454 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 06:50:11.454 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 06:50:11.454 | INFO | High Water Breakdown: | 20260505 06:50:11.454 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 06:50:11.454 | INFO | ------------------------------------------ | 20260505 06:50:11.454 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:50:11.454 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 06:50:11.454 | INFO | Total mem usage: 15.7 GiB | 20260505 06:50:11.454 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 06:50:11.454 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 06:50:11.454 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260505 06:50:11.454 | INFO | High Water Breakdown: | 20260505 06:50:11.455 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 06:50:11.455 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 06:50:11.455 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 06:50:11.455 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 06:50:11.455 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 06:50:11.455 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 06:50:11.455 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 06:50:11.455 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 06:50:11.455 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 06:50:11.455 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:50:11.455 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 06:50:11.455 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:50:11.455 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 06:50:11.455 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 06:50:11.455 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 06:50:11.455 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 06:50:11.455 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:50:11.455 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 06:50:11.455 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:50:11.455 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:50:11.455 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:50:11.455 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 06:50:11.455 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 06:50:11.455 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:50:11.455 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 06:50:11.455 | INFO | ------------------------------------------ | 20260505 06:50:11.455 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:50:11.455 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 06:50:11.455 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:50:11.455 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:50:11.455 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.50e+01, 1.35e+01 / 1.54e+00 | 20260505 06:50:11.455 | INFO | PerfReport: 0.28%, AggregateStats -- 9.39e-02 / 1.04e-01, 1.06e-02 / 6.41e-03 | 20260505 06:50:11.455 | INFO | PerfReport: 11.61%, Baseline -- 3.84e+00 / 4.45e+00, 6.39e-01 / 2.75e-01 | 20260505 06:50:11.455 | INFO | PerfReport: 3.32%, Histogram -- 1.10e+00 / 1.49e+00, 2.18e-01 / 9.47e-02 | 20260505 06:50:11.455 | INFO | PerfReport: 15.44%, DME -- 4.08e+01 / 4.46e+01, 4.19e+00 / 1.03e+00 | 20260505 06:50:11.455 | INFO | PerfReport: 56.16%, FrameLabeling -- 1.86e+01 / 1.90e+01, 2.06e+00 / 9.19e-01 | 20260505 06:50:11.455 | INFO | PerfReport: 7.66%, PulseAccumulating -- 2.53e+00 / 3.12e+00, 4.54e-01 / 2.79e-01 | 20260505 06:50:11.455 | INFO | PerfReport: 4.05%, Metrics -- 1.34e+00 / 1.67e+00, 3.80e-01 / 1.88e-01 | 20260505 06:50:11.455 | INFO | PerfReport: 1.47%, PulseSerializing -- 4.86e-01 / 5.57e-01, 9.45e-02 / 4.04e-02 | 20260505 06:50:11.455 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 06:50:11.455 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:50:11.455 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:50:11.455 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.22e+01 / 0.00e+00 | 20260505 06:50:11.455 | INFO | PerfReport: 22.04%, Upload -- 2.24e+01 / 0.00e+00, 7.81e+00 / 0.00e+00 | 20260505 06:50:11.455 | INFO | PerfReport: 21.45%, Download -- 2.18e+01 / 0.00e+00, 7.71e+00 / 0.00e+00 | 20260505 06:50:11.455 | INFO | PerfReport: 32.59%, Compute -- 3.30e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 06:50:11.455 | INFO | PerfReport: 1.47%, LockUpload -- 1.49e+00 / 0.00e+00, 6.92e+00 / 0.00e+00 | 20260505 06:50:11.455 | INFO | PerfReport: 0.79%, LockDownload -- 8.04e-01 / 0.00e+00, 2.20e+00 / 0.00e+00 | 20260505 06:50:11.455 | INFO | PerfReport: 21.65%, LockCompute -- 2.20e+01 / 0.00e+00, 1.68e+01 / 0.00e+00 | 20260505 06:50:11.455 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 1.66e-02 / 0.00e+00 | 20260505 06:50:11.576 | INFO | Starting a round of Baz IO | 20260505 06:50:14.821 | INFO | Populating RTMetrics report | 20260505 06:50:14.822 | INFO | RTMetrics report populated | 20260505 06:50:14.822 | INFO | Opening tmp RTMetrics json file | 20260505 06:50:14.822 | INFO | Serializing RTMetrics to json | 20260505 06:50:14.823 | INFO | Serialization done, swapping tmp file now... | 20260505 06:50:14.824 | INFO | RTMetrics done | 20260505 06:50:14.941 | INFO | Finished a round of Baz IO: Wrote 2.4 GiB in 3.365 seconds | 20260505 06:50:14.941 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 31.81% / 0.04% / 68.14% | 20260505 06:50:14.941 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.4618% / 5.5631e-05% / 7.15256e-05% | 20260505 06:50:14.941 | INFO | Baz IO overall: | 20260505 06:50:14.941 | INFO | BytesWritten : 627819750058 | 20260505 06:50:14.941 | INFO | NumEvents : 297297840963 | 20260505 06:50:16.592 | INFO | Analyzing chunk frames = [1187840,1188352) | 20260505 06:50:16.592 | INFO | MemInfo: { | 20260505 06:50:16.592 | INFO | "gpu" : " 17 GB", | 20260505 06:50:16.592 | INFO | "huge" : "", | 20260505 06:50:16.592 | INFO | "huge_shared" : " 27 GB", | 20260505 06:50:16.592 | INFO | "malloc" : " 14 GB", | 20260505 06:50:16.592 | INFO | "pinned" : " 29 GB" | 20260505 06:50:16.592 | INFO | } | 20260505 06:50:16.592 | INFO | Perf report for the last 5.13934 seconds: | 20260505 06:50:16.592 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:50:16.592 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:16.592 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:16.592 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:16.592 | INFO | ANALYSIS : 64.4%, 2.9, 97.9ms, 0 | 20260505 06:50:16.593 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:16.593 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260505 06:50:16.593 | INFO | PRE_HQ : 63.2%, 1.9, 63.8ms, 0 | 20260505 06:50:16.593 | INFO | BAZWRITER : 64.8%, 4.1, 137.7ms, 0 | 20260505 06:50:16.593 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 06:50:16.593 | INFO | Uploaded 36GiB to GPU in 2.14881s (16.7535GiB/s) | 20260505 06:50:16.594 | INFO | Downloaded 38GiB to GPU in 2.31819s (16.3921GiB/s) | 20260505 06:50:21.732 | INFO | Analyzing chunk frames = [1188352,1188864) | 20260505 06:50:21.732 | INFO | MemInfo: { | 20260505 06:50:21.732 | INFO | "gpu" : " 17 GB", | 20260505 06:50:21.732 | INFO | "huge" : "", | 20260505 06:50:21.732 | INFO | "huge_shared" : " 27 GB", | 20260505 06:50:21.732 | INFO | "malloc" : " 14 GB", | 20260505 06:50:21.732 | INFO | "pinned" : " 29 GB" | 20260505 06:50:21.732 | INFO | } | 20260505 06:50:21.732 | INFO | Perf report for the last 5.13958 seconds: | 20260505 06:50:21.732 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:50:21.732 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:21.732 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:21.732 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:21.732 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 06:50:21.732 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:21.732 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:21.732 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 06:50:21.732 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:21.732 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:21.732 | INFO | Uploaded 36GiB to GPU in 2.09643s (17.172GiB/s) | 20260505 06:50:21.732 | INFO | Downloaded 34GiB to GPU in 2.03695s (16.6916GiB/s) | 20260505 06:50:26.871 | INFO | Analyzing chunk frames = [1188864,1189376) | 20260505 06:50:26.871 | INFO | MemInfo: { | 20260505 06:50:26.871 | INFO | "gpu" : " 17 GB", | 20260505 06:50:26.871 | INFO | "huge" : "", | 20260505 06:50:26.871 | INFO | "huge_shared" : " 27 GB", | 20260505 06:50:26.871 | INFO | "malloc" : " 14 GB", | 20260505 06:50:26.871 | INFO | "pinned" : " 29 GB" | 20260505 06:50:26.871 | INFO | } | 20260505 06:50:26.871 | INFO | Perf report for the last 5.13921 seconds: | 20260505 06:50:26.871 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:50:26.871 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:26.871 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:26.871 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:26.871 | INFO | ANALYSIS : 63.7%, 3.0, 99.2ms, 0 | 20260505 06:50:26.871 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:26.871 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:26.871 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 06:50:26.871 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:26.871 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:26.872 | INFO | Uploaded 36GiB to GPU in 2.12536s (16.9383GiB/s) | 20260505 06:50:26.872 | INFO | Downloaded 34GiB to GPU in 2.04281s (16.6438GiB/s) | 20260505 06:50:32.011 | INFO | Analyzing chunk frames = [1189376,1189888) | 20260505 06:50:32.011 | INFO | MemInfo: { | 20260505 06:50:32.011 | INFO | "gpu" : " 17 GB", | 20260505 06:50:32.011 | INFO | "huge" : "", | 20260505 06:50:32.011 | INFO | "huge_shared" : " 27 GB", | 20260505 06:50:32.011 | INFO | "malloc" : " 14 GB", | 20260505 06:50:32.011 | INFO | "pinned" : " 29 GB" | 20260505 06:50:32.011 | INFO | } | 20260505 06:50:32.011 | INFO | Perf report for the last 5.13954 seconds: | 20260505 06:50:32.011 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:50:32.011 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:32.011 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:32.011 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:32.011 | INFO | ANALYSIS : 63.7%, 3.0, 99.0ms, 0 | 20260505 06:50:32.011 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:32.011 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:32.011 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 06:50:32.011 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:32.011 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:32.011 | INFO | Uploaded 36GiB to GPU in 2.16026s (16.6647GiB/s) | 20260505 06:50:32.011 | INFO | Downloaded 34GiB to GPU in 2.03291s (16.7248GiB/s) | 20260505 06:50:37.150 | INFO | Analyzing chunk frames = [1189888,1190400) | 20260505 06:50:37.150 | INFO | MemInfo: { | 20260505 06:50:37.150 | INFO | "gpu" : " 17 GB", | 20260505 06:50:37.150 | INFO | "huge" : "", | 20260505 06:50:37.150 | INFO | "huge_shared" : " 27 GB", | 20260505 06:50:37.150 | INFO | "malloc" : " 14 GB", | 20260505 06:50:37.150 | INFO | "pinned" : " 29 GB" | 20260505 06:50:37.150 | INFO | } | 20260505 06:50:37.150 | INFO | Perf report for the last 5.13898 seconds: | 20260505 06:50:37.150 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:50:37.150 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:37.150 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:37.150 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:37.150 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 06:50:37.150 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:37.150 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:37.150 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 06:50:37.150 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:37.150 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:37.150 | INFO | Uploaded 36GiB to GPU in 2.12105s (16.9727GiB/s) | 20260505 06:50:37.150 | INFO | Downloaded 34GiB to GPU in 2.03321s (16.7223GiB/s) | 20260505 06:50:42.291 | INFO | Analyzing chunk frames = [1190400,1190912) | 20260505 06:50:42.291 | INFO | MemInfo: { | 20260505 06:50:42.291 | INFO | "gpu" : " 17 GB", | 20260505 06:50:42.291 | INFO | "huge" : "", | 20260505 06:50:42.291 | INFO | "huge_shared" : " 27 GB", | 20260505 06:50:42.291 | INFO | "malloc" : " 14 GB", | 20260505 06:50:42.291 | INFO | "pinned" : " 29 GB" | 20260505 06:50:42.291 | INFO | } | 20260505 06:50:42.292 | INFO | Perf report for the last 5.14167 seconds: | 20260505 06:50:42.292 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:50:42.292 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:42.292 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:42.292 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:42.292 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 06:50:42.292 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:42.292 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:42.292 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 06:50:42.292 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:42.292 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:42.292 | INFO | Uploaded 36GiB to GPU in 2.14817s (16.7584GiB/s) | 20260505 06:50:42.292 | INFO | Downloaded 34GiB to GPU in 2.04141s (16.6551GiB/s) | 20260505 06:50:47.429 | INFO | Analyzing chunk frames = [1190912,1191424) | 20260505 06:50:47.429 | INFO | MemInfo: { | 20260505 06:50:47.429 | INFO | "gpu" : " 17 GB", | 20260505 06:50:47.429 | INFO | "huge" : "", | 20260505 06:50:47.429 | INFO | "huge_shared" : " 27 GB", | 20260505 06:50:47.429 | INFO | "malloc" : " 14 GB", | 20260505 06:50:47.429 | INFO | "pinned" : " 29 GB" | 20260505 06:50:47.429 | INFO | } | 20260505 06:50:47.429 | INFO | Perf report for the last 5.13725 seconds: | 20260505 06:50:47.429 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:50:47.429 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:47.429 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:47.429 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:47.429 | INFO | ANALYSIS : 64.9%, 3.0, 101.4ms, 0 | 20260505 06:50:47.429 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:47.429 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:47.429 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 06:50:47.429 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:47.429 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:47.429 | INFO | Uploaded 36GiB to GPU in 2.23279s (16.1234GiB/s) | 20260505 06:50:47.429 | INFO | Downloaded 34GiB to GPU in 2.15655s (15.7659GiB/s) | 20260505 06:50:52.568 | INFO | Analyzing chunk frames = [1191424,1191936) | 20260505 06:50:52.568 | INFO | MemInfo: { | 20260505 06:50:52.568 | INFO | "gpu" : " 17 GB", | 20260505 06:50:52.568 | INFO | "huge" : "", | 20260505 06:50:52.568 | INFO | "huge_shared" : " 27 GB", | 20260505 06:50:52.568 | INFO | "malloc" : " 14 GB", | 20260505 06:50:52.568 | INFO | "pinned" : " 29 GB" | 20260505 06:50:52.568 | INFO | } | 20260505 06:50:52.568 | INFO | Perf report for the last 5.13917 seconds: | 20260505 06:50:52.568 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:50:52.568 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:52.568 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:52.568 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:52.568 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 06:50:52.568 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:52.568 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:52.568 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 06:50:52.568 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:52.568 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:52.568 | INFO | Uploaded 36GiB to GPU in 2.16444s (16.6325GiB/s) | 20260505 06:50:52.568 | INFO | Downloaded 34GiB to GPU in 2.04386s (16.6352GiB/s) | 20260505 06:50:52.688 | INFO | Starting a round of Baz IO | 20260505 06:50:55.933 | INFO | Populating RTMetrics report | 20260505 06:50:55.933 | INFO | RTMetrics report populated | 20260505 06:50:55.933 | INFO | Opening tmp RTMetrics json file | 20260505 06:50:55.938 | INFO | Serializing RTMetrics to json | 20260505 06:50:55.948 | INFO | Serialization done, swapping tmp file now... | 20260505 06:50:55.964 | INFO | RTMetrics done | 20260505 06:50:56.052 | INFO | Finished a round of Baz IO: Wrote 2.4 GiB in 3.363 seconds | 20260505 06:50:56.052 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 31.77% / 0.04% / 68.19% | 20260505 06:50:56.052 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.446% / 4.37101e-05% / 5.16574e-05% | 20260505 06:50:56.055 | INFO | Baz IO overall: | 20260505 06:50:56.055 | INFO | BytesWritten : 630396856063 | 20260505 06:50:56.055 | INFO | NumEvents : 298530489112 | 20260505 06:50:57.708 | INFO | Analyzing chunk frames = [1191936,1192448) | 20260505 06:50:57.708 | INFO | MemInfo: { | 20260505 06:50:57.708 | INFO | "gpu" : " 17 GB", | 20260505 06:50:57.708 | INFO | "huge" : "", | 20260505 06:50:57.708 | INFO | "huge_shared" : " 27 GB", | 20260505 06:50:57.708 | INFO | "malloc" : " 14 GB", | 20260505 06:50:57.708 | INFO | "pinned" : " 29 GB" | 20260505 06:50:57.708 | INFO | } | 20260505 06:50:57.708 | INFO | Perf report for the last 5.13978 seconds: | 20260505 06:50:57.708 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:50:57.708 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:57.708 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:57.708 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:57.708 | INFO | ANALYSIS : 64.3%, 2.9, 98.1ms, 0 | 20260505 06:50:57.708 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:50:57.708 | INFO | RT_METRICS : 46.6%, 1.0, 24.4ms, 0 | 20260505 06:50:57.708 | INFO | PRE_HQ : 63.1%, 1.9, 63.3ms, 0 | 20260505 06:50:57.708 | INFO | BAZWRITER : 64.8%, 4.1, 137.0ms, 0 | 20260505 06:50:57.708 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 06:50:57.708 | INFO | Uploaded 36GiB to GPU in 2.14591s (16.7761GiB/s) | 20260505 06:50:57.708 | INFO | Downloaded 38GiB to GPU in 2.31515s (16.4136GiB/s) | 20260505 06:51:02.847 | INFO | Analyzing chunk frames = [1192448,1192960) | 20260505 06:51:02.847 | INFO | MemInfo: { | 20260505 06:51:02.847 | INFO | "gpu" : " 17 GB", | 20260505 06:51:02.847 | INFO | "huge" : "", | 20260505 06:51:02.847 | INFO | "huge_shared" : " 27 GB", | 20260505 06:51:02.847 | INFO | "malloc" : " 14 GB", | 20260505 06:51:02.847 | INFO | "pinned" : " 29 GB" | 20260505 06:51:02.847 | INFO | } | 20260505 06:51:02.847 | INFO | Perf report for the last 5.13899 seconds: | 20260505 06:51:02.847 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:51:02.847 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:02.847 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:02.847 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:02.847 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 06:51:02.847 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:02.847 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:02.847 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 06:51:02.847 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:02.847 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:02.847 | INFO | Uploaded 36GiB to GPU in 2.09103s (17.2164GiB/s) | 20260505 06:51:02.847 | INFO | Downloaded 34GiB to GPU in 2.03332s (16.7214GiB/s) | 20260505 06:51:07.986 | INFO | Analyzing chunk frames = [1192960,1193472) | 20260505 06:51:07.986 | INFO | MemInfo: { | 20260505 06:51:07.986 | INFO | "gpu" : " 17 GB", | 20260505 06:51:07.986 | INFO | "huge" : "", | 20260505 06:51:07.986 | INFO | "huge_shared" : " 27 GB", | 20260505 06:51:07.986 | INFO | "malloc" : " 14 GB", | 20260505 06:51:07.986 | INFO | "pinned" : " 29 GB" | 20260505 06:51:07.986 | INFO | } | 20260505 06:51:07.986 | INFO | Perf report for the last 5.13971 seconds: | 20260505 06:51:07.986 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:51:07.986 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:07.987 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:07.987 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:07.987 | INFO | ANALYSIS : 63.7%, 3.0, 99.1ms, 0 | 20260505 06:51:07.987 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:07.987 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:07.987 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 06:51:07.987 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:07.987 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:07.987 | INFO | Uploaded 36GiB to GPU in 2.13388s (16.8707GiB/s) | 20260505 06:51:07.987 | INFO | Downloaded 34GiB to GPU in 2.04758s (16.605GiB/s) | 20260505 06:51:13.125 | INFO | Analyzing chunk frames = [1193472,1193984) | 20260505 06:51:13.125 | INFO | MemInfo: { | 20260505 06:51:13.125 | INFO | "gpu" : " 17 GB", | 20260505 06:51:13.125 | INFO | "huge" : "", | 20260505 06:51:13.125 | INFO | "huge_shared" : " 27 GB", | 20260505 06:51:13.125 | INFO | "malloc" : " 14 GB", | 20260505 06:51:13.125 | INFO | "pinned" : " 29 GB" | 20260505 06:51:13.125 | INFO | } | 20260505 06:51:13.125 | INFO | Perf report for the last 5.13897 seconds: | 20260505 06:51:13.126 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:51:13.126 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:13.126 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:13.126 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:13.126 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260505 06:51:13.126 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:13.126 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:13.126 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 06:51:13.126 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:13.126 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:13.126 | INFO | Uploaded 36GiB to GPU in 2.14153s (16.8104GiB/s) | 20260505 06:51:13.126 | INFO | Downloaded 34GiB to GPU in 2.0334s (16.7208GiB/s) | 20260505 06:51:18.265 | INFO | Analyzing chunk frames = [1193984,1194496) | 20260505 06:51:18.265 | INFO | MemInfo: { | 20260505 06:51:18.265 | INFO | "gpu" : " 17 GB", | 20260505 06:51:18.265 | INFO | "huge" : "", | 20260505 06:51:18.265 | INFO | "huge_shared" : " 27 GB", | 20260505 06:51:18.265 | INFO | "malloc" : " 14 GB", | 20260505 06:51:18.265 | INFO | "pinned" : " 29 GB" | 20260505 06:51:18.265 | INFO | } | 20260505 06:51:18.265 | INFO | Perf report for the last 5.13936 seconds: | 20260505 06:51:18.265 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:51:18.265 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:18.265 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:18.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:18.265 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 06:51:18.265 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:18.265 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:18.265 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 06:51:18.265 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:18.265 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:18.265 | INFO | Uploaded 36GiB to GPU in 2.11824s (16.9952GiB/s) | 20260505 06:51:18.265 | INFO | Downloaded 34GiB to GPU in 2.03976s (16.6686GiB/s) | 20260505 06:51:23.406 | INFO | Analyzing chunk frames = [1194496,1195008) | 20260505 06:51:23.406 | INFO | MemInfo: { | 20260505 06:51:23.406 | INFO | "gpu" : " 17 GB", | 20260505 06:51:23.406 | INFO | "huge" : "", | 20260505 06:51:23.406 | INFO | "huge_shared" : " 27 GB", | 20260505 06:51:23.406 | INFO | "malloc" : " 14 GB", | 20260505 06:51:23.406 | INFO | "pinned" : " 29 GB" | 20260505 06:51:23.406 | INFO | } | 20260505 06:51:23.406 | INFO | Perf report for the last 5.14125 seconds: | 20260505 06:51:23.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:51:23.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:23.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:23.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:23.406 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 06:51:23.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:23.406 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:23.406 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 06:51:23.406 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:23.406 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:23.407 | INFO | Uploaded 36GiB to GPU in 2.14708s (16.767GiB/s) | 20260505 06:51:23.407 | INFO | Downloaded 34GiB to GPU in 2.04312s (16.6412GiB/s) | 20260505 06:51:28.544 | INFO | Analyzing chunk frames = [1195008,1195520) | 20260505 06:51:28.544 | INFO | MemInfo: { | 20260505 06:51:28.544 | INFO | "gpu" : " 17 GB", | 20260505 06:51:28.544 | INFO | "huge" : "", | 20260505 06:51:28.544 | INFO | "huge_shared" : " 27 GB", | 20260505 06:51:28.544 | INFO | "malloc" : " 14 GB", | 20260505 06:51:28.544 | INFO | "pinned" : " 29 GB" | 20260505 06:51:28.544 | INFO | } | 20260505 06:51:28.544 | INFO | Perf report for the last 5.13766 seconds: | 20260505 06:51:28.544 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:51:28.544 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:28.544 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:28.544 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:28.544 | INFO | ANALYSIS : 64.1%, 3.0, 100.0ms, 0 | 20260505 06:51:28.544 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:28.544 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:28.544 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260505 06:51:28.544 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:28.544 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:28.544 | INFO | Uploaded 36GiB to GPU in 2.16951s (16.5936GiB/s) | 20260505 06:51:28.544 | INFO | Downloaded 34GiB to GPU in 2.16421s (15.7101GiB/s) | 20260505 06:51:33.683 | INFO | Analyzing chunk frames = [1195520,1196032) | 20260505 06:51:33.683 | INFO | MemInfo: { | 20260505 06:51:33.683 | INFO | "gpu" : " 17 GB", | 20260505 06:51:33.683 | INFO | "huge" : "", | 20260505 06:51:33.683 | INFO | "huge_shared" : " 27 GB", | 20260505 06:51:33.683 | INFO | "malloc" : " 14 GB", | 20260505 06:51:33.683 | INFO | "pinned" : " 29 GB" | 20260505 06:51:33.683 | INFO | } | 20260505 06:51:33.683 | INFO | Perf report for the last 5.13957 seconds: | 20260505 06:51:33.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:51:33.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:33.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:33.683 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:33.683 | INFO | ANALYSIS : 64.3%, 3.0, 100.5ms, 0 | 20260505 06:51:33.683 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:33.683 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:33.683 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 06:51:33.683 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:33.683 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:33.684 | INFO | Uploaded 36GiB to GPU in 2.16302s (16.6434GiB/s) | 20260505 06:51:33.684 | INFO | Downloaded 34GiB to GPU in 2.05431s (16.5505GiB/s) | 20260505 06:51:33.684 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 06:51:33.684 | INFO | Total mem usage: 15.3 GiB | 20260505 06:51:33.684 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 06:51:33.684 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.73%) | 20260505 06:51:33.684 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 06:51:33.684 | INFO | High Water Breakdown: | 20260505 06:51:33.684 | INFO | 60.4% : 8.67 GiB ( 92.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 06:51:33.684 | INFO | 39.5% : 5.67 GiB ( 85.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 06:51:33.684 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 06:51:33.684 | INFO | ------------------------------------------ | 20260505 06:51:33.684 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:51:33.684 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 06:51:33.684 | INFO | Total mem usage: 26.5 GiB | 20260505 06:51:33.684 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 06:51:33.684 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260505 06:51:33.684 | INFO | Unused Cache: 1.6 GiB (Waste: 6.05%) | 20260505 06:51:33.684 | INFO | High Water Breakdown: | 20260505 06:51:33.684 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:51:33.684 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:51:33.684 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:51:33.684 | INFO | 2% : 523 MiB ( 99.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:51:33.684 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:51:33.684 | INFO | 0.557% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:51:33.684 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 06:51:33.684 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 06:51:33.684 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:51:33.684 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 06:51:33.684 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 06:51:33.684 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 06:51:33.684 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 06:51:33.684 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 06:51:33.684 | INFO | ------------------------------------------ | 20260505 06:51:33.684 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:51:33.684 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 06:51:33.684 | INFO | Total mem usage: 25.6 GiB | 20260505 06:51:33.684 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 06:51:33.684 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 06:51:33.684 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 06:51:33.684 | INFO | High Water Breakdown: | 20260505 06:51:33.684 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 06:51:33.684 | INFO | ------------------------------------------ | 20260505 06:51:33.684 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:51:33.684 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 06:51:33.684 | INFO | Total mem usage: 15.7 GiB | 20260505 06:51:33.684 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 06:51:33.684 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 06:51:33.684 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260505 06:51:33.684 | INFO | High Water Breakdown: | 20260505 06:51:33.684 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 06:51:33.684 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 06:51:33.684 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 06:51:33.685 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 06:51:33.685 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 06:51:33.685 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 06:51:33.685 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 06:51:33.685 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 06:51:33.685 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 06:51:33.685 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:51:33.685 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 06:51:33.685 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:51:33.685 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 06:51:33.685 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 06:51:33.685 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 06:51:33.685 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 06:51:33.685 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:51:33.685 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 06:51:33.685 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:51:33.685 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:51:33.685 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:51:33.685 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 06:51:33.685 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 06:51:33.685 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:51:33.685 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 06:51:33.685 | INFO | ------------------------------------------ | 20260505 06:51:33.685 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:51:33.685 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 06:51:33.685 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:51:33.685 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:51:33.685 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.53e+01, 1.38e+01 / 1.85e+00 | 20260505 06:51:33.685 | INFO | PerfReport: 0.28%, AggregateStats -- 9.38e-02 / 1.03e-01, 1.07e-02 / 6.34e-03 | 20260505 06:51:33.685 | INFO | PerfReport: 11.60%, Baseline -- 3.83e+00 / 4.46e+00, 6.38e-01 / 2.97e-01 | 20260505 06:51:33.685 | INFO | PerfReport: 3.33%, Histogram -- 1.10e+00 / 1.48e+00, 2.18e-01 / 1.09e-01 | 20260505 06:51:33.685 | INFO | PerfReport: 15.48%, DME -- 4.09e+01 / 4.48e+01, 4.18e+00 / 1.36e+00 | 20260505 06:51:33.685 | INFO | PerfReport: 56.15%, FrameLabeling -- 1.86e+01 / 1.91e+01, 2.04e+00 / 1.11e+00 | 20260505 06:51:33.685 | INFO | PerfReport: 7.64%, PulseAccumulating -- 2.52e+00 / 3.11e+00, 4.50e-01 / 2.94e-01 | 20260505 06:51:33.685 | INFO | PerfReport: 4.06%, Metrics -- 1.34e+00 / 1.67e+00, 3.84e-01 / 1.75e-01 | 20260505 06:51:33.685 | INFO | PerfReport: 1.46%, PulseSerializing -- 4.84e-01 / 5.51e-01, 9.35e-02 / 3.68e-02 | 20260505 06:51:33.685 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 06:51:33.685 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:51:33.685 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:51:33.685 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.24e+01 / 0.00e+00 | 20260505 06:51:33.685 | INFO | PerfReport: 22.02%, Upload -- 2.23e+01 / 0.00e+00, 7.78e+00 / 0.00e+00 | 20260505 06:51:33.685 | INFO | PerfReport: 21.46%, Download -- 2.18e+01 / 0.00e+00, 7.75e+00 / 0.00e+00 | 20260505 06:51:33.685 | INFO | PerfReport: 32.60%, Compute -- 3.31e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 06:51:33.685 | INFO | PerfReport: 1.46%, LockUpload -- 1.48e+00 / 0.00e+00, 6.95e+00 / 0.00e+00 | 20260505 06:51:33.685 | INFO | PerfReport: 0.81%, LockDownload -- 8.19e-01 / 0.00e+00, 2.25e+00 / 0.00e+00 | 20260505 06:51:33.685 | INFO | PerfReport: 21.64%, LockCompute -- 2.19e+01 / 0.00e+00, 1.67e+01 / 0.00e+00 | 20260505 06:51:33.685 | INFO | PerfReport: 0.02%, Misc -- 1.54e-02 / 0.00e+00, 3.04e-02 / 0.00e+00 | 20260505 06:51:33.803 | INFO | Starting a round of Baz IO | 20260505 06:51:37.048 | INFO | Populating RTMetrics report | 20260505 06:51:37.048 | INFO | RTMetrics report populated | 20260505 06:51:37.048 | INFO | Opening tmp RTMetrics json file | 20260505 06:51:37.049 | INFO | Serializing RTMetrics to json | 20260505 06:51:37.050 | INFO | Serialization done, swapping tmp file now... | 20260505 06:51:37.050 | INFO | RTMetrics done | 20260505 06:51:37.174 | INFO | Finished a round of Baz IO: Wrote 2.402 GiB in 3.371 seconds | 20260505 06:51:37.175 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 31.73% / 0.04% / 68.23% | 20260505 06:51:37.175 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.43596% / 5.96046e-05% / 5.16574e-05% | 20260505 06:51:37.175 | INFO | Baz IO overall: | 20260505 06:51:37.175 | INFO | BytesWritten : 632975666460 | 20260505 06:51:37.175 | INFO | NumEvents : 299764207544 | 20260505 06:51:38.822 | INFO | Analyzing chunk frames = [1196032,1196544) | 20260505 06:51:38.822 | INFO | MemInfo: { | 20260505 06:51:38.822 | INFO | "gpu" : " 17 GB", | 20260505 06:51:38.822 | INFO | "huge" : "", | 20260505 06:51:38.822 | INFO | "huge_shared" : " 27 GB", | 20260505 06:51:38.822 | INFO | "malloc" : " 14 GB", | 20260505 06:51:38.822 | INFO | "pinned" : " 29 GB" | 20260505 06:51:38.822 | INFO | } | 20260505 06:51:38.822 | INFO | Perf report for the last 5.13878 seconds: | 20260505 06:51:38.822 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:51:38.822 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:38.822 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:38.822 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:38.822 | INFO | ANALYSIS : 64.3%, 2.9, 97.8ms, 0 | 20260505 06:51:38.822 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:38.822 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260505 06:51:38.822 | INFO | PRE_HQ : 63.3%, 2.0, 64.3ms, 0 | 20260505 06:51:38.822 | INFO | BAZWRITER : 64.9%, 4.2, 138.7ms, 0 | 20260505 06:51:38.822 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 06:51:38.823 | INFO | Uploaded 36GiB to GPU in 2.12078s (16.9749GiB/s) | 20260505 06:51:38.823 | INFO | Downloaded 38GiB to GPU in 2.30452s (16.4893GiB/s) | 20260505 06:51:43.962 | INFO | Analyzing chunk frames = [1196544,1197056) | 20260505 06:51:43.962 | INFO | MemInfo: { | 20260505 06:51:43.962 | INFO | "gpu" : " 17 GB", | 20260505 06:51:43.962 | INFO | "huge" : "", | 20260505 06:51:43.962 | INFO | "huge_shared" : " 27 GB", | 20260505 06:51:43.962 | INFO | "malloc" : " 14 GB", | 20260505 06:51:43.962 | INFO | "pinned" : " 29 GB" | 20260505 06:51:43.962 | INFO | } | 20260505 06:51:43.962 | INFO | Perf report for the last 5.14001 seconds: | 20260505 06:51:43.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:51:43.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:43.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:43.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:43.962 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 06:51:43.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:43.962 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:43.962 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 06:51:43.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:43.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:43.962 | INFO | Uploaded 36GiB to GPU in 2.09696s (17.1677GiB/s) | 20260505 06:51:43.962 | INFO | Downloaded 34GiB to GPU in 2.03923s (16.673GiB/s) | 20260505 06:51:49.101 | INFO | Analyzing chunk frames = [1197056,1197568) | 20260505 06:51:49.101 | INFO | MemInfo: { | 20260505 06:51:49.101 | INFO | "gpu" : " 17 GB", | 20260505 06:51:49.101 | INFO | "huge" : "", | 20260505 06:51:49.101 | INFO | "huge_shared" : " 27 GB", | 20260505 06:51:49.101 | INFO | "malloc" : " 14 GB", | 20260505 06:51:49.101 | INFO | "pinned" : " 29 GB" | 20260505 06:51:49.101 | INFO | } | 20260505 06:51:49.101 | INFO | Perf report for the last 5.13877 seconds: | 20260505 06:51:49.101 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:51:49.101 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:49.101 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:49.101 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:49.101 | INFO | ANALYSIS : 63.7%, 3.0, 99.1ms, 0 | 20260505 06:51:49.101 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:49.101 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:49.101 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 06:51:49.101 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:49.101 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:49.101 | INFO | Uploaded 36GiB to GPU in 2.14576s (16.7772GiB/s) | 20260505 06:51:49.102 | INFO | Downloaded 34GiB to GPU in 2.04209s (16.6496GiB/s) | 20260505 06:51:54.240 | INFO | Analyzing chunk frames = [1197568,1198080) | 20260505 06:51:54.240 | INFO | MemInfo: { | 20260505 06:51:54.240 | INFO | "gpu" : " 17 GB", | 20260505 06:51:54.240 | INFO | "huge" : "", | 20260505 06:51:54.240 | INFO | "huge_shared" : " 27 GB", | 20260505 06:51:54.240 | INFO | "malloc" : " 14 GB", | 20260505 06:51:54.240 | INFO | "pinned" : " 29 GB" | 20260505 06:51:54.240 | INFO | } | 20260505 06:51:54.240 | INFO | Perf report for the last 5.13933 seconds: | 20260505 06:51:54.240 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:51:54.241 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:54.241 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:54.241 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:54.241 | INFO | ANALYSIS : 63.7%, 3.0, 98.9ms, 0 | 20260505 06:51:54.241 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:54.241 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:54.241 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 06:51:54.241 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:54.241 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:54.241 | INFO | Uploaded 36GiB to GPU in 2.15603s (16.6974GiB/s) | 20260505 06:51:54.241 | INFO | Downloaded 34GiB to GPU in 2.02996s (16.7491GiB/s) | 20260505 06:51:59.380 | INFO | Analyzing chunk frames = [1198080,1198592) | 20260505 06:51:59.380 | INFO | MemInfo: { | 20260505 06:51:59.380 | INFO | "gpu" : " 17 GB", | 20260505 06:51:59.380 | INFO | "huge" : "", | 20260505 06:51:59.380 | INFO | "huge_shared" : " 27 GB", | 20260505 06:51:59.380 | INFO | "malloc" : " 14 GB", | 20260505 06:51:59.380 | INFO | "pinned" : " 29 GB" | 20260505 06:51:59.380 | INFO | } | 20260505 06:51:59.380 | INFO | Perf report for the last 5.13948 seconds: | 20260505 06:51:59.380 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:51:59.380 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:59.380 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:59.380 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:59.380 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 06:51:59.380 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:59.380 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:59.380 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 06:51:59.380 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:59.380 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:51:59.380 | INFO | Uploaded 36GiB to GPU in 2.16554s (16.624GiB/s) | 20260505 06:51:59.380 | INFO | Downloaded 34GiB to GPU in 2.0438s (16.6356GiB/s) | 20260505 06:52:04.521 | INFO | Analyzing chunk frames = [1198592,1199104) | 20260505 06:52:04.521 | INFO | MemInfo: { | 20260505 06:52:04.521 | INFO | "gpu" : " 17 GB", | 20260505 06:52:04.521 | INFO | "huge" : "", | 20260505 06:52:04.521 | INFO | "huge_shared" : " 27 GB", | 20260505 06:52:04.521 | INFO | "malloc" : " 14 GB", | 20260505 06:52:04.521 | INFO | "pinned" : " 29 GB" | 20260505 06:52:04.521 | INFO | } | 20260505 06:52:04.521 | INFO | Perf report for the last 5.14084 seconds: | 20260505 06:52:04.521 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:52:04.521 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:04.521 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:04.521 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:04.521 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 06:52:04.521 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:04.521 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:04.521 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 06:52:04.521 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:04.521 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:04.521 | INFO | Uploaded 36GiB to GPU in 2.15092s (16.737GiB/s) | 20260505 06:52:04.521 | INFO | Downloaded 34GiB to GPU in 2.03317s (16.7227GiB/s) | 20260505 06:52:09.658 | INFO | Analyzing chunk frames = [1199104,1199616) | 20260505 06:52:09.658 | INFO | MemInfo: { | 20260505 06:52:09.658 | INFO | "gpu" : " 17 GB", | 20260505 06:52:09.658 | INFO | "huge" : "", | 20260505 06:52:09.658 | INFO | "huge_shared" : " 27 GB", | 20260505 06:52:09.658 | INFO | "malloc" : " 14 GB", | 20260505 06:52:09.658 | INFO | "pinned" : " 29 GB" | 20260505 06:52:09.658 | INFO | } | 20260505 06:52:09.658 | INFO | Perf report for the last 5.13776 seconds: | 20260505 06:52:09.658 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:52:09.658 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:09.658 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:09.658 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:09.658 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 06:52:09.658 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:09.658 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:09.658 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260505 06:52:09.658 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:09.659 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:09.659 | INFO | Uploaded 36GiB to GPU in 2.21242s (16.2718GiB/s) | 20260505 06:52:09.659 | INFO | Downloaded 34GiB to GPU in 2.17272s (15.6486GiB/s) | 20260505 06:52:14.797 | INFO | Analyzing chunk frames = [1199616,1200128) | 20260505 06:52:14.798 | INFO | MemInfo: { | 20260505 06:52:14.798 | INFO | "gpu" : " 17 GB", | 20260505 06:52:14.798 | INFO | "huge" : "", | 20260505 06:52:14.798 | INFO | "huge_shared" : " 27 GB", | 20260505 06:52:14.798 | INFO | "malloc" : " 14 GB", | 20260505 06:52:14.798 | INFO | "pinned" : " 29 GB" | 20260505 06:52:14.798 | INFO | } | 20260505 06:52:14.798 | INFO | Perf report for the last 5.13914 seconds: | 20260505 06:52:14.798 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:52:14.798 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:14.798 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:14.798 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:14.798 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 06:52:14.798 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:14.798 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:14.798 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 06:52:14.798 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:14.798 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:14.798 | INFO | Uploaded 36GiB to GPU in 2.16199s (16.6513GiB/s) | 20260505 06:52:14.798 | INFO | Downloaded 34GiB to GPU in 2.03985s (16.6679GiB/s) | 20260505 06:52:14.918 | INFO | Starting a round of Baz IO | 20260505 06:52:18.187 | INFO | Populating RTMetrics report | 20260505 06:52:18.188 | INFO | RTMetrics report populated | 20260505 06:52:18.188 | INFO | Opening tmp RTMetrics json file | 20260505 06:52:18.191 | INFO | Serializing RTMetrics to json | 20260505 06:52:18.193 | INFO | Serialization done, swapping tmp file now... | 20260505 06:52:18.194 | INFO | RTMetrics done | 20260505 06:52:18.310 | INFO | Finished a round of Baz IO: Wrote 2.404 GiB in 3.391 seconds | 20260505 06:52:18.310 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 31.69% / 0.04% / 68.27% | 20260505 06:52:18.310 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.42075% / 5.16574e-05% / 7.15256e-05% | 20260505 06:52:18.310 | INFO | Baz IO overall: | 20260505 06:52:18.310 | INFO | BytesWritten : 635556751891 | 20260505 06:52:18.310 | INFO | NumEvents : 300999603528 | 20260505 06:52:19.937 | INFO | Analyzing chunk frames = [1200128,1200640) | 20260505 06:52:19.937 | INFO | MemInfo: { | 20260505 06:52:19.937 | INFO | "gpu" : " 17 GB", | 20260505 06:52:19.937 | INFO | "huge" : "", | 20260505 06:52:19.937 | INFO | "huge_shared" : " 27 GB", | 20260505 06:52:19.937 | INFO | "malloc" : " 14 GB", | 20260505 06:52:19.937 | INFO | "pinned" : " 29 GB" | 20260505 06:52:19.937 | INFO | } | 20260505 06:52:19.937 | INFO | Perf report for the last 5.13948 seconds: | 20260505 06:52:19.937 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:52:19.937 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:19.937 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:19.937 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:19.937 | INFO | ANALYSIS : 64.8%, 2.9, 99.4ms, 0 | 20260505 06:52:19.937 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:19.937 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260505 06:52:19.937 | INFO | PRE_HQ : 63.6%, 1.9, 63.7ms, 0 | 20260505 06:52:19.937 | INFO | BAZWRITER : 65.2%, 4.1, 137.3ms, 0 | 20260505 06:52:19.937 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 06:52:19.937 | INFO | Uploaded 36GiB to GPU in 2.19389s (16.4092GiB/s) | 20260505 06:52:19.937 | INFO | Downloaded 38GiB to GPU in 2.30562s (16.4815GiB/s) | 20260505 06:52:25.076 | INFO | Analyzing chunk frames = [1200640,1201152) | 20260505 06:52:25.076 | INFO | MemInfo: { | 20260505 06:52:25.076 | INFO | "gpu" : " 17 GB", | 20260505 06:52:25.076 | INFO | "huge" : "", | 20260505 06:52:25.076 | INFO | "huge_shared" : " 27 GB", | 20260505 06:52:25.076 | INFO | "malloc" : " 14 GB", | 20260505 06:52:25.077 | INFO | "pinned" : " 29 GB" | 20260505 06:52:25.077 | INFO | } | 20260505 06:52:25.077 | INFO | Perf report for the last 5.13959 seconds: | 20260505 06:52:25.077 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:52:25.077 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:25.077 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:25.077 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:25.077 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 06:52:25.077 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:25.077 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:25.077 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 06:52:25.077 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:25.077 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:25.077 | INFO | Uploaded 36GiB to GPU in 2.09486s (17.1849GiB/s) | 20260505 06:52:25.077 | INFO | Downloaded 34GiB to GPU in 2.0329s (16.7249GiB/s) | 20260505 06:52:30.216 | INFO | Analyzing chunk frames = [1201152,1201664) | 20260505 06:52:30.216 | INFO | MemInfo: { | 20260505 06:52:30.216 | INFO | "gpu" : " 17 GB", | 20260505 06:52:30.216 | INFO | "huge" : "", | 20260505 06:52:30.216 | INFO | "huge_shared" : " 27 GB", | 20260505 06:52:30.216 | INFO | "malloc" : " 14 GB", | 20260505 06:52:30.216 | INFO | "pinned" : " 29 GB" | 20260505 06:52:30.216 | INFO | } | 20260505 06:52:30.216 | INFO | Perf report for the last 5.1392 seconds: | 20260505 06:52:30.216 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:52:30.216 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:30.216 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:30.216 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:30.216 | INFO | ANALYSIS : 63.7%, 3.0, 99.1ms, 0 | 20260505 06:52:30.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:30.216 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:30.216 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 06:52:30.216 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:30.216 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:30.216 | INFO | Uploaded 36GiB to GPU in 2.14313s (16.7978GiB/s) | 20260505 06:52:30.217 | INFO | Downloaded 34GiB to GPU in 2.04474s (16.628GiB/s) | 20260505 06:52:35.355 | INFO | Analyzing chunk frames = [1201664,1202176) | 20260505 06:52:35.355 | INFO | MemInfo: { | 20260505 06:52:35.355 | INFO | "gpu" : " 17 GB", | 20260505 06:52:35.355 | INFO | "huge" : "", | 20260505 06:52:35.355 | INFO | "huge_shared" : " 27 GB", | 20260505 06:52:35.355 | INFO | "malloc" : " 14 GB", | 20260505 06:52:35.355 | INFO | "pinned" : " 29 GB" | 20260505 06:52:35.355 | INFO | } | 20260505 06:52:35.355 | INFO | Perf report for the last 5.13926 seconds: | 20260505 06:52:35.355 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:52:35.355 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:35.355 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:35.355 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:35.355 | INFO | ANALYSIS : 63.6%, 3.0, 98.9ms, 0 | 20260505 06:52:35.355 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:35.355 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:35.355 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 06:52:35.355 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:35.355 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:35.356 | INFO | Uploaded 36GiB to GPU in 2.1478s (16.7613GiB/s) | 20260505 06:52:35.356 | INFO | Downloaded 34GiB to GPU in 2.02807s (16.7647GiB/s) | 20260505 06:52:40.494 | INFO | Analyzing chunk frames = [1202176,1202688) | 20260505 06:52:40.494 | INFO | MemInfo: { | 20260505 06:52:40.494 | INFO | "gpu" : " 17 GB", | 20260505 06:52:40.494 | INFO | "huge" : "", | 20260505 06:52:40.494 | INFO | "huge_shared" : " 27 GB", | 20260505 06:52:40.494 | INFO | "malloc" : " 14 GB", | 20260505 06:52:40.494 | INFO | "pinned" : " 29 GB" | 20260505 06:52:40.494 | INFO | } | 20260505 06:52:40.494 | INFO | Perf report for the last 5.13907 seconds: | 20260505 06:52:40.494 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:52:40.494 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:40.494 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:40.494 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:40.494 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 06:52:40.494 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:40.494 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:40.494 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 06:52:40.494 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:40.495 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:40.495 | INFO | Uploaded 36GiB to GPU in 2.1476s (16.7629GiB/s) | 20260505 06:52:40.495 | INFO | Downloaded 34GiB to GPU in 2.05072s (16.5795GiB/s) | 20260505 06:52:45.635 | INFO | Analyzing chunk frames = [1202688,1203200) | 20260505 06:52:45.635 | INFO | MemInfo: { | 20260505 06:52:45.635 | INFO | "gpu" : " 17 GB", | 20260505 06:52:45.635 | INFO | "huge" : "", | 20260505 06:52:45.635 | INFO | "huge_shared" : " 27 GB", | 20260505 06:52:45.635 | INFO | "malloc" : " 14 GB", | 20260505 06:52:45.635 | INFO | "pinned" : " 29 GB" | 20260505 06:52:45.635 | INFO | } | 20260505 06:52:45.635 | INFO | Perf report for the last 5.14097 seconds: | 20260505 06:52:45.635 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:52:45.635 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:45.635 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:45.635 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:45.635 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 06:52:45.636 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:45.636 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:45.636 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 06:52:45.636 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:45.636 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:45.636 | INFO | Uploaded 36GiB to GPU in 2.14587s (16.7764GiB/s) | 20260505 06:52:45.636 | INFO | Downloaded 34GiB to GPU in 2.03852s (16.6787GiB/s) | 20260505 06:52:50.773 | INFO | Analyzing chunk frames = [1203200,1203712) | 20260505 06:52:50.773 | INFO | MemInfo: { | 20260505 06:52:50.773 | INFO | "gpu" : " 17 GB", | 20260505 06:52:50.773 | INFO | "huge" : "", | 20260505 06:52:50.773 | INFO | "huge_shared" : " 27 GB", | 20260505 06:52:50.773 | INFO | "malloc" : " 14 GB", | 20260505 06:52:50.773 | INFO | "pinned" : " 29 GB" | 20260505 06:52:50.773 | INFO | } | 20260505 06:52:50.773 | INFO | Perf report for the last 5.13772 seconds: | 20260505 06:52:50.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:52:50.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:50.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:50.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:50.773 | INFO | ANALYSIS : 64.8%, 3.0, 101.2ms, 0 | 20260505 06:52:50.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:50.773 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:50.773 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 06:52:50.773 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:50.773 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:50.773 | INFO | Uploaded 36GiB to GPU in 2.23053s (16.1397GiB/s) | 20260505 06:52:50.773 | INFO | Downloaded 34GiB to GPU in 2.15028s (15.8119GiB/s) | 20260505 06:52:55.912 | INFO | Analyzing chunk frames = [1203712,1204224) | 20260505 06:52:55.912 | INFO | MemInfo: { | 20260505 06:52:55.912 | INFO | "gpu" : " 17 GB", | 20260505 06:52:55.912 | INFO | "huge" : "", | 20260505 06:52:55.912 | INFO | "huge_shared" : " 27 GB", | 20260505 06:52:55.912 | INFO | "malloc" : " 14 GB", | 20260505 06:52:55.912 | INFO | "pinned" : " 29 GB" | 20260505 06:52:55.912 | INFO | } | 20260505 06:52:55.912 | INFO | Perf report for the last 5.13931 seconds: | 20260505 06:52:55.912 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:52:55.912 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:55.912 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:55.913 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:55.913 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 06:52:55.913 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:55.913 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:55.913 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 06:52:55.913 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:55.913 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:52:55.913 | INFO | Uploaded 36GiB to GPU in 2.1636s (16.639GiB/s) | 20260505 06:52:55.913 | INFO | Downloaded 34GiB to GPU in 2.04332s (16.6396GiB/s) | 20260505 06:52:55.913 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 06:52:55.913 | INFO | Total mem usage: 15.3 GiB | 20260505 06:52:55.913 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 06:52:55.913 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.76%) | 20260505 06:52:55.913 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 06:52:55.913 | INFO | High Water Breakdown: | 20260505 06:52:55.913 | INFO | 60.4% : 8.67 GiB ( 92.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 06:52:55.913 | INFO | 39.5% : 5.66 GiB ( 85.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 06:52:55.913 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 06:52:55.913 | INFO | ------------------------------------------ | 20260505 06:52:55.913 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:52:55.913 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 06:52:55.913 | INFO | Total mem usage: 26.5 GiB | 20260505 06:52:55.913 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 06:52:55.913 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260505 06:52:55.913 | INFO | Unused Cache: 1.6 GiB (Waste: 6.05%) | 20260505 06:52:55.913 | INFO | High Water Breakdown: | 20260505 06:52:55.913 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:52:55.913 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:52:55.913 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:52:55.913 | INFO | 2% : 523 MiB ( 99.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:52:55.913 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:52:55.913 | INFO | 0.557% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:52:55.913 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 06:52:55.913 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 06:52:55.913 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:52:55.913 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 06:52:55.913 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 06:52:55.913 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 06:52:55.913 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 06:52:55.913 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 06:52:55.913 | INFO | ------------------------------------------ | 20260505 06:52:55.913 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:52:55.913 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 06:52:55.913 | INFO | Total mem usage: 25.6 GiB | 20260505 06:52:55.914 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 06:52:55.914 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 06:52:55.914 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 06:52:55.914 | INFO | High Water Breakdown: | 20260505 06:52:55.914 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 06:52:55.914 | INFO | ------------------------------------------ | 20260505 06:52:55.914 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:52:55.914 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 06:52:55.914 | INFO | Total mem usage: 15.7 GiB | 20260505 06:52:55.914 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 06:52:55.914 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 06:52:55.914 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260505 06:52:55.914 | INFO | High Water Breakdown: | 20260505 06:52:55.914 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 06:52:55.914 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 06:52:55.914 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 06:52:55.914 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 06:52:55.914 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 06:52:55.914 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 06:52:55.914 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 06:52:55.914 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 06:52:55.914 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 06:52:55.914 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:52:55.914 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 06:52:55.914 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:52:55.914 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 06:52:55.914 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 06:52:55.914 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 06:52:55.914 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 06:52:55.914 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:52:55.914 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 06:52:55.914 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:52:55.914 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:52:55.914 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:52:55.914 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 06:52:55.914 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 06:52:55.914 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:52:55.914 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 06:52:55.914 | INFO | ------------------------------------------ | 20260505 06:52:55.914 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:52:55.914 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 06:52:55.914 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:52:55.914 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:52:55.914 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.53e+01, 1.36e+01 / 1.61e+00 | 20260505 06:52:55.914 | INFO | PerfReport: 0.28%, AggregateStats -- 9.40e-02 / 1.03e-01, 1.07e-02 / 5.41e-03 | 20260505 06:52:55.914 | INFO | PerfReport: 11.60%, Baseline -- 3.84e+00 / 4.47e+00, 6.39e-01 / 3.10e-01 | 20260505 06:52:55.914 | INFO | PerfReport: 3.31%, Histogram -- 1.10e+00 / 1.48e+00, 2.17e-01 / 1.11e-01 | 20260505 06:52:55.914 | INFO | PerfReport: 15.46%, DME -- 4.09e+01 / 4.47e+01, 4.23e+00 / 1.21e+00 | 20260505 06:52:55.914 | INFO | PerfReport: 56.19%, FrameLabeling -- 1.86e+01 / 1.91e+01, 2.09e+00 / 1.10e+00 | 20260505 06:52:55.914 | INFO | PerfReport: 7.64%, PulseAccumulating -- 2.53e+00 / 3.11e+00, 4.49e-01 / 3.24e-01 | 20260505 06:52:55.914 | INFO | PerfReport: 4.04%, Metrics -- 1.34e+00 / 1.69e+00, 3.78e-01 / 2.04e-01 | 20260505 06:52:55.914 | INFO | PerfReport: 1.46%, PulseSerializing -- 4.84e-01 / 5.59e-01, 9.29e-02 / 3.89e-02 | 20260505 06:52:55.914 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 06:52:55.914 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:52:55.914 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:52:55.914 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.22e+01 / 0.00e+00 | 20260505 06:52:55.914 | INFO | PerfReport: 22.11%, Upload -- 2.24e+01 / 0.00e+00, 7.92e+00 / 0.00e+00 | 20260505 06:52:55.914 | INFO | PerfReport: 21.42%, Download -- 2.17e+01 / 0.00e+00, 7.76e+00 / 0.00e+00 | 20260505 06:52:55.914 | INFO | PerfReport: 32.56%, Compute -- 3.31e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 06:52:55.914 | INFO | PerfReport: 1.44%, LockUpload -- 1.47e+00 / 0.00e+00, 6.89e+00 / 0.00e+00 | 20260505 06:52:55.914 | INFO | PerfReport: 0.79%, LockDownload -- 7.99e-01 / 0.00e+00, 2.17e+00 / 0.00e+00 | 20260505 06:52:55.914 | INFO | PerfReport: 21.68%, LockCompute -- 2.20e+01 / 0.00e+00, 1.67e+01 / 0.00e+00 | 20260505 06:52:55.914 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 2.29e-02 / 0.00e+00 | 20260505 06:52:56.033 | INFO | Starting a round of Baz IO | 20260505 06:52:59.279 | INFO | Populating RTMetrics report | 20260505 06:52:59.279 | INFO | RTMetrics report populated | 20260505 06:52:59.279 | INFO | Opening tmp RTMetrics json file | 20260505 06:52:59.302 | INFO | Serializing RTMetrics to json | 20260505 06:52:59.307 | INFO | Serialization done, swapping tmp file now... | 20260505 06:52:59.309 | INFO | RTMetrics done | 20260505 06:52:59.394 | INFO | Finished a round of Baz IO: Wrote 2.401 GiB in 3.36 seconds | 20260505 06:52:59.394 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 31.65% / 0.04% / 68.31% | 20260505 06:52:59.394 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.39138% / 4.37101e-05% / 5.96046e-05% | 20260505 06:52:59.394 | INFO | Baz IO overall: | 20260505 06:52:59.394 | INFO | BytesWritten : 638134564255 | 20260505 06:52:59.394 | INFO | NumEvents : 302232309850 | 20260505 06:53:01.052 | INFO | Analyzing chunk frames = [1204224,1204736) | 20260505 06:53:01.052 | INFO | MemInfo: { | 20260505 06:53:01.052 | INFO | "gpu" : " 17 GB", | 20260505 06:53:01.052 | INFO | "huge" : "", | 20260505 06:53:01.052 | INFO | "huge_shared" : " 27 GB", | 20260505 06:53:01.052 | INFO | "malloc" : " 14 GB", | 20260505 06:53:01.052 | INFO | "pinned" : " 29 GB" | 20260505 06:53:01.052 | INFO | } | 20260505 06:53:01.052 | INFO | Perf report for the last 5.13945 seconds: | 20260505 06:53:01.052 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:53:01.052 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:01.052 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:01.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:01.052 | INFO | ANALYSIS : 64.3%, 2.9, 97.7ms, 0 | 20260505 06:53:01.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:01.052 | INFO | RT_METRICS : 46.7%, 1.0, 24.5ms, 0 | 20260505 06:53:01.052 | INFO | PRE_HQ : 63.2%, 1.9, 63.9ms, 0 | 20260505 06:53:01.052 | INFO | BAZWRITER : 64.6%, 4.1, 137.3ms, 0 | 20260505 06:53:01.052 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 06:53:01.052 | INFO | Uploaded 36GiB to GPU in 2.13206s (16.885GiB/s) | 20260505 06:53:01.052 | INFO | Downloaded 38GiB to GPU in 2.30931s (16.4551GiB/s) | 20260505 06:53:06.191 | INFO | Analyzing chunk frames = [1204736,1205248) | 20260505 06:53:06.191 | INFO | MemInfo: { | 20260505 06:53:06.191 | INFO | "gpu" : " 17 GB", | 20260505 06:53:06.191 | INFO | "huge" : "", | 20260505 06:53:06.191 | INFO | "huge_shared" : " 27 GB", | 20260505 06:53:06.191 | INFO | "malloc" : " 14 GB", | 20260505 06:53:06.191 | INFO | "pinned" : " 29 GB" | 20260505 06:53:06.191 | INFO | } | 20260505 06:53:06.191 | INFO | Perf report for the last 5.13965 seconds: | 20260505 06:53:06.191 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:53:06.191 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:06.191 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:06.191 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:06.192 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 06:53:06.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:06.192 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:06.192 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 06:53:06.192 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:06.192 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:06.192 | INFO | Uploaded 36GiB to GPU in 2.10897s (17.0699GiB/s) | 20260505 06:53:06.192 | INFO | Downloaded 34GiB to GPU in 2.04123s (16.6566GiB/s) | 20260505 06:53:11.330 | INFO | Analyzing chunk frames = [1205248,1205760) | 20260505 06:53:11.330 | INFO | MemInfo: { | 20260505 06:53:11.330 | INFO | "gpu" : " 17 GB", | 20260505 06:53:11.330 | INFO | "huge" : "", | 20260505 06:53:11.330 | INFO | "huge_shared" : " 27 GB", | 20260505 06:53:11.330 | INFO | "malloc" : " 14 GB", | 20260505 06:53:11.331 | INFO | "pinned" : " 29 GB" | 20260505 06:53:11.331 | INFO | } | 20260505 06:53:11.331 | INFO | Perf report for the last 5.1393 seconds: | 20260505 06:53:11.331 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:53:11.331 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:11.331 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:11.331 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:11.331 | INFO | ANALYSIS : 63.6%, 3.0, 99.0ms, 0 | 20260505 06:53:11.331 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:11.331 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:11.331 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 06:53:11.331 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:11.331 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:11.331 | INFO | Uploaded 36GiB to GPU in 2.14111s (16.8137GiB/s) | 20260505 06:53:11.331 | INFO | Downloaded 34GiB to GPU in 2.04477s (16.6278GiB/s) | 20260505 06:53:16.470 | INFO | Analyzing chunk frames = [1205760,1206272) | 20260505 06:53:16.470 | INFO | MemInfo: { | 20260505 06:53:16.470 | INFO | "gpu" : " 17 GB", | 20260505 06:53:16.470 | INFO | "huge" : "", | 20260505 06:53:16.470 | INFO | "huge_shared" : " 27 GB", | 20260505 06:53:16.470 | INFO | "malloc" : " 14 GB", | 20260505 06:53:16.470 | INFO | "pinned" : " 29 GB" | 20260505 06:53:16.470 | INFO | } | 20260505 06:53:16.470 | INFO | Perf report for the last 5.13936 seconds: | 20260505 06:53:16.470 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:53:16.470 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:16.470 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:16.470 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:16.470 | INFO | ANALYSIS : 63.7%, 3.0, 98.9ms, 0 | 20260505 06:53:16.470 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:16.470 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:16.470 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 06:53:16.470 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:16.470 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:16.470 | INFO | Uploaded 36GiB to GPU in 2.15706s (16.6894GiB/s) | 20260505 06:53:16.470 | INFO | Downloaded 34GiB to GPU in 2.02892s (16.7576GiB/s) | 20260505 06:53:21.609 | INFO | Analyzing chunk frames = [1206272,1206784) | 20260505 06:53:21.609 | INFO | MemInfo: { | 20260505 06:53:21.609 | INFO | "gpu" : " 17 GB", | 20260505 06:53:21.609 | INFO | "huge" : "", | 20260505 06:53:21.609 | INFO | "huge_shared" : " 27 GB", | 20260505 06:53:21.609 | INFO | "malloc" : " 14 GB", | 20260505 06:53:21.610 | INFO | "pinned" : " 29 GB" | 20260505 06:53:21.610 | INFO | } | 20260505 06:53:21.610 | INFO | Perf report for the last 5.13947 seconds: | 20260505 06:53:21.610 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:53:21.610 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:21.610 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:21.610 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:21.610 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 06:53:21.610 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:21.610 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:21.610 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 06:53:21.610 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:21.610 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:21.610 | INFO | Uploaded 36GiB to GPU in 2.11139s (17.0504GiB/s) | 20260505 06:53:21.610 | INFO | Downloaded 34GiB to GPU in 2.03216s (16.731GiB/s) | 20260505 06:53:26.750 | INFO | Analyzing chunk frames = [1206784,1207296) | 20260505 06:53:26.750 | INFO | MemInfo: { | 20260505 06:53:26.750 | INFO | "gpu" : " 17 GB", | 20260505 06:53:26.750 | INFO | "huge" : "", | 20260505 06:53:26.750 | INFO | "huge_shared" : " 27 GB", | 20260505 06:53:26.750 | INFO | "malloc" : " 14 GB", | 20260505 06:53:26.750 | INFO | "pinned" : " 29 GB" | 20260505 06:53:26.750 | INFO | } | 20260505 06:53:26.750 | INFO | Perf report for the last 5.14081 seconds: | 20260505 06:53:26.750 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:53:26.750 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:26.750 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:26.750 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:26.750 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 06:53:26.751 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:26.751 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:26.751 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 06:53:26.751 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:26.751 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:26.751 | INFO | Uploaded 36GiB to GPU in 2.14677s (16.7694GiB/s) | 20260505 06:53:26.751 | INFO | Downloaded 34GiB to GPU in 2.04403s (16.6338GiB/s) | 20260505 06:53:31.888 | INFO | Analyzing chunk frames = [1207296,1207808) | 20260505 06:53:31.888 | INFO | MemInfo: { | 20260505 06:53:31.888 | INFO | "gpu" : " 17 GB", | 20260505 06:53:31.888 | INFO | "huge" : "", | 20260505 06:53:31.888 | INFO | "huge_shared" : " 27 GB", | 20260505 06:53:31.888 | INFO | "malloc" : " 14 GB", | 20260505 06:53:31.888 | INFO | "pinned" : " 29 GB" | 20260505 06:53:31.888 | INFO | } | 20260505 06:53:31.888 | INFO | Perf report for the last 5.13786 seconds: | 20260505 06:53:31.888 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:53:31.888 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:31.888 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:31.888 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:31.888 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 06:53:31.888 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:31.888 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:31.888 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 06:53:31.888 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:31.888 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:31.889 | INFO | Uploaded 36GiB to GPU in 2.20075s (16.358GiB/s) | 20260505 06:53:31.889 | INFO | Downloaded 34GiB to GPU in 2.16001s (15.7406GiB/s) | 20260505 06:53:37.027 | INFO | Analyzing chunk frames = [1207808,1208320) | 20260505 06:53:37.027 | INFO | MemInfo: { | 20260505 06:53:37.027 | INFO | "gpu" : " 17 GB", | 20260505 06:53:37.027 | INFO | "huge" : "", | 20260505 06:53:37.027 | INFO | "huge_shared" : " 27 GB", | 20260505 06:53:37.027 | INFO | "malloc" : " 14 GB", | 20260505 06:53:37.028 | INFO | "pinned" : " 29 GB" | 20260505 06:53:37.028 | INFO | } | 20260505 06:53:37.028 | INFO | Perf report for the last 5.13934 seconds: | 20260505 06:53:37.028 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:53:37.028 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:37.028 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:37.028 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:37.028 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 06:53:37.028 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:37.028 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:37.028 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 06:53:37.028 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:37.028 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:37.028 | INFO | Uploaded 36GiB to GPU in 2.16443s (16.6326GiB/s) | 20260505 06:53:37.028 | INFO | Downloaded 34GiB to GPU in 2.04136s (16.6556GiB/s) | 20260505 06:53:37.149 | INFO | Starting a round of Baz IO | 20260505 06:53:40.380 | INFO | Populating RTMetrics report | 20260505 06:53:40.380 | INFO | RTMetrics report populated | 20260505 06:53:40.380 | INFO | Opening tmp RTMetrics json file | 20260505 06:53:40.410 | INFO | Serializing RTMetrics to json | 20260505 06:53:40.434 | INFO | Serialization done, swapping tmp file now... | 20260505 06:53:40.459 | INFO | RTMetrics done | 20260505 06:53:40.497 | INFO | Finished a round of Baz IO: Wrote 2.404 GiB in 3.347 seconds | 20260505 06:53:40.497 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 31.61% / 0.04% / 68.35% | 20260505 06:53:40.497 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.38411% / 4.37101e-05% / 5.96046e-05% | 20260505 06:53:40.497 | INFO | Baz IO overall: | 20260505 06:53:40.497 | INFO | BytesWritten : 640715984872 | 20260505 06:53:40.497 | INFO | NumEvents : 303467318799 | 20260505 06:53:42.167 | INFO | Analyzing chunk frames = [1208320,1208832) | 20260505 06:53:42.167 | INFO | MemInfo: { | 20260505 06:53:42.167 | INFO | "gpu" : " 17 GB", | 20260505 06:53:42.167 | INFO | "huge" : "", | 20260505 06:53:42.167 | INFO | "huge_shared" : " 27 GB", | 20260505 06:53:42.167 | INFO | "malloc" : " 14 GB", | 20260505 06:53:42.167 | INFO | "pinned" : " 29 GB" | 20260505 06:53:42.167 | INFO | } | 20260505 06:53:42.167 | INFO | Perf report for the last 5.13936 seconds: | 20260505 06:53:42.167 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:53:42.167 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:42.167 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:42.167 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:42.167 | INFO | ANALYSIS : 64.1%, 2.9, 97.9ms, 0 | 20260505 06:53:42.167 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:42.167 | INFO | RT_METRICS : 47.5%, 1.0, 24.9ms, 0 | 20260505 06:53:42.167 | INFO | PRE_HQ : 63.0%, 1.9, 63.1ms, 0 | 20260505 06:53:42.167 | INFO | BAZWRITER : 64.5%, 4.1, 137.1ms, 0 | 20260505 06:53:42.167 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 06:53:42.167 | INFO | Uploaded 36GiB to GPU in 2.11239s (17.0423GiB/s) | 20260505 06:53:42.167 | INFO | Downloaded 38GiB to GPU in 2.29655s (16.5466GiB/s) | 20260505 06:53:47.306 | INFO | Analyzing chunk frames = [1208832,1209344) | 20260505 06:53:47.306 | INFO | MemInfo: { | 20260505 06:53:47.306 | INFO | "gpu" : " 17 GB", | 20260505 06:53:47.306 | INFO | "huge" : "", | 20260505 06:53:47.306 | INFO | "huge_shared" : " 27 GB", | 20260505 06:53:47.306 | INFO | "malloc" : " 14 GB", | 20260505 06:53:47.306 | INFO | "pinned" : " 29 GB" | 20260505 06:53:47.306 | INFO | } | 20260505 06:53:47.306 | INFO | Perf report for the last 5.13913 seconds: | 20260505 06:53:47.306 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:53:47.306 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:47.306 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:47.306 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:47.306 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 06:53:47.306 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:47.306 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:47.306 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 06:53:47.306 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:47.306 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:47.307 | INFO | Uploaded 36GiB to GPU in 2.1072s (17.0843GiB/s) | 20260505 06:53:47.307 | INFO | Downloaded 34GiB to GPU in 2.03556s (16.7031GiB/s) | 20260505 06:53:52.445 | INFO | Analyzing chunk frames = [1209344,1209856) | 20260505 06:53:52.445 | INFO | MemInfo: { | 20260505 06:53:52.445 | INFO | "gpu" : " 17 GB", | 20260505 06:53:52.445 | INFO | "huge" : "", | 20260505 06:53:52.445 | INFO | "huge_shared" : " 27 GB", | 20260505 06:53:52.445 | INFO | "malloc" : " 14 GB", | 20260505 06:53:52.445 | INFO | "pinned" : " 29 GB" | 20260505 06:53:52.446 | INFO | } | 20260505 06:53:52.446 | INFO | Perf report for the last 5.13951 seconds: | 20260505 06:53:52.446 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:53:52.446 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:52.446 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:52.446 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:52.446 | INFO | ANALYSIS : 63.8%, 3.0, 99.3ms, 0 | 20260505 06:53:52.446 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:52.446 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:52.446 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 06:53:52.446 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:52.446 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:52.446 | INFO | Uploaded 36GiB to GPU in 2.14574s (16.7774GiB/s) | 20260505 06:53:52.446 | INFO | Downloaded 34GiB to GPU in 2.04592s (16.6184GiB/s) | 20260505 06:53:57.585 | INFO | Analyzing chunk frames = [1209856,1210368) | 20260505 06:53:57.585 | INFO | MemInfo: { | 20260505 06:53:57.585 | INFO | "gpu" : " 17 GB", | 20260505 06:53:57.585 | INFO | "huge" : "", | 20260505 06:53:57.585 | INFO | "huge_shared" : " 27 GB", | 20260505 06:53:57.585 | INFO | "malloc" : " 14 GB", | 20260505 06:53:57.585 | INFO | "pinned" : " 29 GB" | 20260505 06:53:57.585 | INFO | } | 20260505 06:53:57.585 | INFO | Perf report for the last 5.13918 seconds: | 20260505 06:53:57.585 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:53:57.585 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:57.585 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:57.585 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:57.585 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260505 06:53:57.585 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:57.585 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:57.585 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 06:53:57.585 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:57.585 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:53:57.585 | INFO | Uploaded 36GiB to GPU in 2.14863s (16.7549GiB/s) | 20260505 06:53:57.585 | INFO | Downloaded 34GiB to GPU in 2.0279s (16.7661GiB/s) | 20260505 06:54:02.724 | INFO | Analyzing chunk frames = [1210368,1210880) | 20260505 06:54:02.724 | INFO | MemInfo: { | 20260505 06:54:02.724 | INFO | "gpu" : " 17 GB", | 20260505 06:54:02.724 | INFO | "huge" : "", | 20260505 06:54:02.724 | INFO | "huge_shared" : " 27 GB", | 20260505 06:54:02.724 | INFO | "malloc" : " 14 GB", | 20260505 06:54:02.724 | INFO | "pinned" : " 29 GB" | 20260505 06:54:02.724 | INFO | } | 20260505 06:54:02.724 | INFO | Perf report for the last 5.13948 seconds: | 20260505 06:54:02.724 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:54:02.724 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:02.724 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:02.724 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:02.724 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 06:54:02.724 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:02.724 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:02.724 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 06:54:02.724 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:02.724 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:02.725 | INFO | Uploaded 36GiB to GPU in 2.1237s (16.9516GiB/s) | 20260505 06:54:02.725 | INFO | Downloaded 34GiB to GPU in 2.03298s (16.7242GiB/s) | 20260505 06:54:07.865 | INFO | Analyzing chunk frames = [1210880,1211392) | 20260505 06:54:07.865 | INFO | MemInfo: { | 20260505 06:54:07.865 | INFO | "gpu" : " 17 GB", | 20260505 06:54:07.865 | INFO | "huge" : "", | 20260505 06:54:07.865 | INFO | "huge_shared" : " 27 GB", | 20260505 06:54:07.865 | INFO | "malloc" : " 14 GB", | 20260505 06:54:07.865 | INFO | "pinned" : " 29 GB" | 20260505 06:54:07.865 | INFO | } | 20260505 06:54:07.865 | INFO | Perf report for the last 5.14095 seconds: | 20260505 06:54:07.865 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:54:07.865 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:07.865 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:07.865 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:07.865 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 06:54:07.865 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:07.865 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:07.865 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 06:54:07.865 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:07.865 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:07.866 | INFO | Uploaded 36GiB to GPU in 2.15159s (16.7318GiB/s) | 20260505 06:54:07.866 | INFO | Downloaded 34GiB to GPU in 2.03797s (16.6833GiB/s) | 20260505 06:54:13.003 | INFO | Analyzing chunk frames = [1211392,1211904) | 20260505 06:54:13.003 | INFO | MemInfo: { | 20260505 06:54:13.003 | INFO | "gpu" : " 17 GB", | 20260505 06:54:13.003 | INFO | "huge" : "", | 20260505 06:54:13.003 | INFO | "huge_shared" : " 27 GB", | 20260505 06:54:13.003 | INFO | "malloc" : " 14 GB", | 20260505 06:54:13.003 | INFO | "pinned" : " 29 GB" | 20260505 06:54:13.003 | INFO | } | 20260505 06:54:13.003 | INFO | Perf report for the last 5.13776 seconds: | 20260505 06:54:13.003 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:54:13.003 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:13.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:13.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:13.003 | INFO | ANALYSIS : 64.4%, 3.0, 100.6ms, 0 | 20260505 06:54:13.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:13.003 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:13.003 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260505 06:54:13.003 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:13.003 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:13.003 | INFO | Uploaded 36GiB to GPU in 2.20464s (16.3292GiB/s) | 20260505 06:54:13.003 | INFO | Downloaded 34GiB to GPU in 2.16734s (15.6874GiB/s) | 20260505 06:54:18.142 | INFO | Analyzing chunk frames = [1211904,1212416) | 20260505 06:54:18.142 | INFO | MemInfo: { | 20260505 06:54:18.142 | INFO | "gpu" : " 17 GB", | 20260505 06:54:18.142 | INFO | "huge" : "", | 20260505 06:54:18.142 | INFO | "huge_shared" : " 27 GB", | 20260505 06:54:18.142 | INFO | "malloc" : " 14 GB", | 20260505 06:54:18.142 | INFO | "pinned" : " 29 GB" | 20260505 06:54:18.142 | INFO | } | 20260505 06:54:18.142 | INFO | Perf report for the last 5.13909 seconds: | 20260505 06:54:18.142 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:54:18.142 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:18.142 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:18.142 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:18.142 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 06:54:18.142 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:18.142 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:18.142 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 06:54:18.142 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:18.142 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:18.142 | INFO | Uploaded 36GiB to GPU in 2.15949s (16.6706GiB/s) | 20260505 06:54:18.142 | INFO | Downloaded 34GiB to GPU in 2.04137s (16.6555GiB/s) | 20260505 06:54:18.143 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 06:54:18.143 | INFO | Total mem usage: 15.3 GiB | 20260505 06:54:18.143 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 06:54:18.143 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.79%) | 20260505 06:54:18.143 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 06:54:18.143 | INFO | High Water Breakdown: | 20260505 06:54:18.143 | INFO | 60.4% : 8.67 GiB ( 92.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 06:54:18.143 | INFO | 39.5% : 5.66 GiB ( 85.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 06:54:18.143 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 06:54:18.143 | INFO | ------------------------------------------ | 20260505 06:54:18.143 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:54:18.143 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 06:54:18.143 | INFO | Total mem usage: 26.5 GiB | 20260505 06:54:18.143 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 06:54:18.143 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 06:54:18.143 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 06:54:18.143 | INFO | High Water Breakdown: | 20260505 06:54:18.143 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:54:18.143 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:54:18.143 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:54:18.143 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:54:18.143 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:54:18.143 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:54:18.143 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 06:54:18.143 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 06:54:18.143 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:54:18.143 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 06:54:18.143 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 06:54:18.143 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 06:54:18.143 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 06:54:18.143 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 06:54:18.143 | INFO | ------------------------------------------ | 20260505 06:54:18.143 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:54:18.143 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 06:54:18.143 | INFO | Total mem usage: 25.6 GiB | 20260505 06:54:18.143 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 06:54:18.143 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 06:54:18.143 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 06:54:18.143 | INFO | High Water Breakdown: | 20260505 06:54:18.143 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 06:54:18.143 | INFO | ------------------------------------------ | 20260505 06:54:18.143 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:54:18.143 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 06:54:18.143 | INFO | Total mem usage: 15.7 GiB | 20260505 06:54:18.143 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 06:54:18.143 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 06:54:18.143 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260505 06:54:18.143 | INFO | High Water Breakdown: | 20260505 06:54:18.143 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 06:54:18.143 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 06:54:18.143 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 06:54:18.143 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 06:54:18.143 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 06:54:18.143 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 06:54:18.143 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 06:54:18.143 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 06:54:18.143 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 06:54:18.143 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:54:18.143 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 06:54:18.143 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:54:18.143 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 06:54:18.143 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 06:54:18.143 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 06:54:18.144 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 06:54:18.144 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:54:18.144 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 06:54:18.144 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:54:18.144 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:54:18.144 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:54:18.144 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 06:54:18.144 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 06:54:18.144 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:54:18.144 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 06:54:18.144 | INFO | ------------------------------------------ | 20260505 06:54:18.144 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:54:18.144 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 06:54:18.144 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:54:18.144 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:54:18.144 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.50e+01, 1.35e+01 / 1.60e+00 | 20260505 06:54:18.144 | INFO | PerfReport: 0.28%, AggregateStats -- 9.38e-02 / 1.02e-01, 1.03e-02 / 6.34e-03 | 20260505 06:54:18.144 | INFO | PerfReport: 11.63%, Baseline -- 3.84e+00 / 4.48e+00, 6.43e-01 / 2.71e-01 | 20260505 06:54:18.144 | INFO | PerfReport: 3.32%, Histogram -- 1.10e+00 / 1.49e+00, 2.18e-01 / 1.12e-01 | 20260505 06:54:18.144 | INFO | PerfReport: 15.46%, DME -- 4.09e+01 / 4.46e+01, 4.03e+00 / 1.16e+00 | 20260505 06:54:18.144 | INFO | PerfReport: 56.11%, FrameLabeling -- 1.85e+01 / 1.90e+01, 2.05e+00 / 1.10e+00 | 20260505 06:54:18.144 | INFO | PerfReport: 7.66%, PulseAccumulating -- 2.53e+00 / 3.13e+00, 4.53e-01 / 3.01e-01 | 20260505 06:54:18.144 | INFO | PerfReport: 4.06%, Metrics -- 1.34e+00 / 1.69e+00, 3.82e-01 / 2.03e-01 | 20260505 06:54:18.144 | INFO | PerfReport: 1.47%, PulseSerializing -- 4.85e-01 / 5.58e-01, 9.35e-02 / 3.89e-02 | 20260505 06:54:18.144 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 06:54:18.144 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:54:18.144 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:54:18.144 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.21e+01 / 0.00e+00 | 20260505 06:54:18.144 | INFO | PerfReport: 22.04%, Upload -- 2.23e+01 / 0.00e+00, 7.81e+00 / 0.00e+00 | 20260505 06:54:18.144 | INFO | PerfReport: 21.44%, Download -- 2.17e+01 / 0.00e+00, 7.77e+00 / 0.00e+00 | 20260505 06:54:18.144 | INFO | PerfReport: 32.59%, Compute -- 3.30e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 06:54:18.144 | INFO | PerfReport: 1.49%, LockUpload -- 1.51e+00 / 0.00e+00, 6.97e+00 / 0.00e+00 | 20260505 06:54:18.144 | INFO | PerfReport: 0.79%, LockDownload -- 8.03e-01 / 0.00e+00, 2.23e+00 / 0.00e+00 | 20260505 06:54:18.144 | INFO | PerfReport: 21.62%, LockCompute -- 2.19e+01 / 0.00e+00, 1.67e+01 / 0.00e+00 | 20260505 06:54:18.144 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.56e-02 / 0.00e+00 | 20260505 06:54:18.263 | INFO | Starting a round of Baz IO | 20260505 06:54:21.520 | INFO | Populating RTMetrics report | 20260505 06:54:21.520 | INFO | RTMetrics report populated | 20260505 06:54:21.520 | INFO | Opening tmp RTMetrics json file | 20260505 06:54:21.550 | INFO | Serializing RTMetrics to json | 20260505 06:54:21.585 | INFO | Serialization done, swapping tmp file now... | 20260505 06:54:21.616 | INFO | RTMetrics done | 20260505 06:54:21.634 | INFO | Finished a round of Baz IO: Wrote 2.408 GiB in 3.37 seconds | 20260505 06:54:21.634 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 31.57% / 0.04% / 68.39% | 20260505 06:54:21.634 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.38638% / 5.16574e-05% / 6.35783e-05% | 20260505 06:54:21.634 | INFO | Baz IO overall: | 20260505 06:54:21.634 | INFO | BytesWritten : 643301776194 | 20260505 06:54:21.634 | INFO | NumEvents : 304705604922 | 20260505 06:54:23.281 | INFO | Analyzing chunk frames = [1212416,1212928) | 20260505 06:54:23.281 | INFO | MemInfo: { | 20260505 06:54:23.281 | INFO | "gpu" : " 17 GB", | 20260505 06:54:23.281 | INFO | "huge" : "", | 20260505 06:54:23.281 | INFO | "huge_shared" : " 27 GB", | 20260505 06:54:23.281 | INFO | "malloc" : " 14 GB", | 20260505 06:54:23.281 | INFO | "pinned" : " 29 GB" | 20260505 06:54:23.281 | INFO | } | 20260505 06:54:23.281 | INFO | Perf report for the last 5.13905 seconds: | 20260505 06:54:23.281 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:54:23.281 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:23.281 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:23.281 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:23.281 | INFO | ANALYSIS : 64.6%, 2.9, 98.8ms, 0 | 20260505 06:54:23.281 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:23.281 | INFO | RT_METRICS : 47.8%, 1.0, 25.1ms, 0 | 20260505 06:54:23.281 | INFO | PRE_HQ : 63.4%, 1.9, 63.1ms, 0 | 20260505 06:54:23.281 | INFO | BAZWRITER : 64.9%, 4.2, 138.7ms, 0 | 20260505 06:54:23.281 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 06:54:23.282 | INFO | Uploaded 36GiB to GPU in 2.15778s (16.6838GiB/s) | 20260505 06:54:23.282 | INFO | Downloaded 38GiB to GPU in 2.30567s (16.4811GiB/s) | 20260505 06:54:28.420 | INFO | Analyzing chunk frames = [1212928,1213440) | 20260505 06:54:28.420 | INFO | MemInfo: { | 20260505 06:54:28.420 | INFO | "gpu" : " 17 GB", | 20260505 06:54:28.420 | INFO | "huge" : "", | 20260505 06:54:28.420 | INFO | "huge_shared" : " 27 GB", | 20260505 06:54:28.420 | INFO | "malloc" : " 14 GB", | 20260505 06:54:28.420 | INFO | "pinned" : " 29 GB" | 20260505 06:54:28.421 | INFO | } | 20260505 06:54:28.421 | INFO | Perf report for the last 5.13939 seconds: | 20260505 06:54:28.421 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:54:28.421 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:28.421 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:28.421 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:28.421 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 06:54:28.421 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:28.421 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:28.421 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 06:54:28.421 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:28.421 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:28.421 | INFO | Uploaded 36GiB to GPU in 2.09737s (17.1643GiB/s) | 20260505 06:54:28.421 | INFO | Downloaded 34GiB to GPU in 2.03677s (16.6931GiB/s) | 20260505 06:54:33.560 | INFO | Analyzing chunk frames = [1213440,1213952) | 20260505 06:54:33.560 | INFO | MemInfo: { | 20260505 06:54:33.560 | INFO | "gpu" : " 17 GB", | 20260505 06:54:33.560 | INFO | "huge" : "", | 20260505 06:54:33.560 | INFO | "huge_shared" : " 27 GB", | 20260505 06:54:33.560 | INFO | "malloc" : " 14 GB", | 20260505 06:54:33.561 | INFO | "pinned" : " 29 GB" | 20260505 06:54:33.561 | INFO | } | 20260505 06:54:33.561 | INFO | Perf report for the last 5.13996 seconds: | 20260505 06:54:33.561 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:54:33.561 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:33.561 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:33.561 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:33.561 | INFO | ANALYSIS : 63.7%, 3.0, 99.1ms, 0 | 20260505 06:54:33.561 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:33.561 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:33.561 | INFO | PRE_HQ : 30.5%, 1.0, 15.9ms, 0 | 20260505 06:54:33.561 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:33.561 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:33.561 | INFO | Uploaded 36GiB to GPU in 2.12584s (16.9345GiB/s) | 20260505 06:54:33.561 | INFO | Downloaded 34GiB to GPU in 2.04926s (16.5913GiB/s) | 20260505 06:54:38.699 | INFO | Analyzing chunk frames = [1213952,1214464) | 20260505 06:54:38.699 | INFO | MemInfo: { | 20260505 06:54:38.699 | INFO | "gpu" : " 17 GB", | 20260505 06:54:38.699 | INFO | "huge" : "", | 20260505 06:54:38.699 | INFO | "huge_shared" : " 27 GB", | 20260505 06:54:38.700 | INFO | "malloc" : " 14 GB", | 20260505 06:54:38.700 | INFO | "pinned" : " 29 GB" | 20260505 06:54:38.700 | INFO | } | 20260505 06:54:38.700 | INFO | Perf report for the last 5.13905 seconds: | 20260505 06:54:38.700 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:54:38.700 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:38.700 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:38.700 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:38.700 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260505 06:54:38.700 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:38.700 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:38.700 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 06:54:38.700 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:38.700 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:38.700 | INFO | Uploaded 36GiB to GPU in 2.14428s (16.7889GiB/s) | 20260505 06:54:38.700 | INFO | Downloaded 34GiB to GPU in 2.03132s (16.7379GiB/s) | 20260505 06:54:43.839 | INFO | Analyzing chunk frames = [1214464,1214976) | 20260505 06:54:43.839 | INFO | MemInfo: { | 20260505 06:54:43.839 | INFO | "gpu" : " 17 GB", | 20260505 06:54:43.839 | INFO | "huge" : "", | 20260505 06:54:43.839 | INFO | "huge_shared" : " 27 GB", | 20260505 06:54:43.839 | INFO | "malloc" : " 14 GB", | 20260505 06:54:43.839 | INFO | "pinned" : " 29 GB" | 20260505 06:54:43.839 | INFO | } | 20260505 06:54:43.839 | INFO | Perf report for the last 5.13918 seconds: | 20260505 06:54:43.839 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:54:43.839 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:43.839 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:43.839 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:43.839 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 06:54:43.839 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:43.839 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:43.839 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 06:54:43.839 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:43.839 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:43.839 | INFO | Uploaded 36GiB to GPU in 2.11683s (17.0066GiB/s) | 20260505 06:54:43.839 | INFO | Downloaded 34GiB to GPU in 2.02863s (16.7601GiB/s) | 20260505 06:54:48.980 | INFO | Analyzing chunk frames = [1214976,1215488) | 20260505 06:54:48.980 | INFO | MemInfo: { | 20260505 06:54:48.980 | INFO | "gpu" : " 17 GB", | 20260505 06:54:48.980 | INFO | "huge" : "", | 20260505 06:54:48.980 | INFO | "huge_shared" : " 27 GB", | 20260505 06:54:48.980 | INFO | "malloc" : " 14 GB", | 20260505 06:54:48.980 | INFO | "pinned" : " 29 GB" | 20260505 06:54:48.980 | INFO | } | 20260505 06:54:48.980 | INFO | Perf report for the last 5.14073 seconds: | 20260505 06:54:48.980 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:54:48.980 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:48.980 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:48.980 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:48.980 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 06:54:48.980 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:48.980 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:48.980 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260505 06:54:48.980 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:48.980 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:48.980 | INFO | Uploaded 36GiB to GPU in 2.1221s (16.9643GiB/s) | 20260505 06:54:48.980 | INFO | Downloaded 34GiB to GPU in 2.03915s (16.6736GiB/s) | 20260505 06:54:54.117 | INFO | Analyzing chunk frames = [1215488,1216000) | 20260505 06:54:54.117 | INFO | MemInfo: { | 20260505 06:54:54.117 | INFO | "gpu" : " 17 GB", | 20260505 06:54:54.117 | INFO | "huge" : "", | 20260505 06:54:54.117 | INFO | "huge_shared" : " 27 GB", | 20260505 06:54:54.117 | INFO | "malloc" : " 14 GB", | 20260505 06:54:54.117 | INFO | "pinned" : " 29 GB" | 20260505 06:54:54.117 | INFO | } | 20260505 06:54:54.117 | INFO | Perf report for the last 5.13769 seconds: | 20260505 06:54:54.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:54:54.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:54.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:54.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:54.118 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 06:54:54.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:54.118 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:54.118 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260505 06:54:54.118 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:54.118 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:54.118 | INFO | Uploaded 36GiB to GPU in 2.21112s (16.2814GiB/s) | 20260505 06:54:54.118 | INFO | Downloaded 34GiB to GPU in 2.17115s (15.6599GiB/s) | 20260505 06:54:59.257 | INFO | Analyzing chunk frames = [1216000,1216512) | 20260505 06:54:59.257 | INFO | MemInfo: { | 20260505 06:54:59.257 | INFO | "gpu" : " 17 GB", | 20260505 06:54:59.257 | INFO | "huge" : "", | 20260505 06:54:59.257 | INFO | "huge_shared" : " 27 GB", | 20260505 06:54:59.257 | INFO | "malloc" : " 14 GB", | 20260505 06:54:59.257 | INFO | "pinned" : " 29 GB" | 20260505 06:54:59.257 | INFO | } | 20260505 06:54:59.257 | INFO | Perf report for the last 5.14005 seconds: | 20260505 06:54:59.257 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:54:59.257 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:59.257 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:59.257 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:59.257 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 06:54:59.257 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:59.257 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:59.257 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 06:54:59.258 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:59.258 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:54:59.258 | INFO | Uploaded 36GiB to GPU in 2.16282s (16.6449GiB/s) | 20260505 06:54:59.258 | INFO | Downloaded 34GiB to GPU in 2.03799s (16.6831GiB/s) | 20260505 06:54:59.378 | INFO | Starting a round of Baz IO | 20260505 06:55:02.631 | INFO | Populating RTMetrics report | 20260505 06:55:02.631 | INFO | RTMetrics report populated | 20260505 06:55:02.631 | INFO | Opening tmp RTMetrics json file | 20260505 06:55:02.660 | INFO | Serializing RTMetrics to json | 20260505 06:55:02.692 | INFO | Serialization done, swapping tmp file now... | 20260505 06:55:02.710 | INFO | RTMetrics done | 20260505 06:55:02.745 | INFO | Finished a round of Baz IO: Wrote 2.407 GiB in 3.367 seconds | 20260505 06:55:02.745 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 31.53% / 0.04% / 68.43% | 20260505 06:55:02.745 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.35676% / 7.54992e-05% / 6.35783e-05% | 20260505 06:55:02.745 | INFO | Baz IO overall: | 20260505 06:55:02.745 | INFO | BytesWritten : 645886377284 | 20260505 06:55:02.745 | INFO | NumEvents : 305942332019 | 20260505 06:55:04.396 | INFO | Analyzing chunk frames = [1216512,1217024) | 20260505 06:55:04.396 | INFO | MemInfo: { | 20260505 06:55:04.396 | INFO | "gpu" : " 17 GB", | 20260505 06:55:04.396 | INFO | "huge" : "", | 20260505 06:55:04.396 | INFO | "huge_shared" : " 27 GB", | 20260505 06:55:04.396 | INFO | "malloc" : " 14 GB", | 20260505 06:55:04.396 | INFO | "pinned" : " 29 GB" | 20260505 06:55:04.396 | INFO | } | 20260505 06:55:04.396 | INFO | Perf report for the last 5.13886 seconds: | 20260505 06:55:04.396 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:55:04.396 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:04.396 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:04.396 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:04.396 | INFO | ANALYSIS : 64.5%, 2.9, 98.8ms, 0 | 20260505 06:55:04.396 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:04.396 | INFO | RT_METRICS : 47.5%, 1.0, 24.9ms, 0 | 20260505 06:55:04.396 | INFO | PRE_HQ : 63.5%, 1.9, 63.6ms, 0 | 20260505 06:55:04.396 | INFO | BAZWRITER : 64.8%, 4.1, 136.7ms, 0 | 20260505 06:55:04.396 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260505 06:55:04.397 | INFO | Uploaded 36GiB to GPU in 2.15018s (16.7428GiB/s) | 20260505 06:55:04.397 | INFO | Downloaded 38GiB to GPU in 2.30599s (16.4788GiB/s) | 20260505 06:55:09.535 | INFO | Analyzing chunk frames = [1217024,1217536) | 20260505 06:55:09.535 | INFO | MemInfo: { | 20260505 06:55:09.535 | INFO | "gpu" : " 17 GB", | 20260505 06:55:09.535 | INFO | "huge" : "", | 20260505 06:55:09.535 | INFO | "huge_shared" : " 27 GB", | 20260505 06:55:09.535 | INFO | "malloc" : " 14 GB", | 20260505 06:55:09.535 | INFO | "pinned" : " 29 GB" | 20260505 06:55:09.535 | INFO | } | 20260505 06:55:09.535 | INFO | Perf report for the last 5.13916 seconds: | 20260505 06:55:09.535 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:55:09.535 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:09.535 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:09.535 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:09.535 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 06:55:09.535 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:09.535 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:09.535 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 06:55:09.535 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:09.535 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:09.536 | INFO | Uploaded 36GiB to GPU in 2.10123s (17.1328GiB/s) | 20260505 06:55:09.536 | INFO | Downloaded 34GiB to GPU in 2.02997s (16.7491GiB/s) | 20260505 06:55:14.675 | INFO | Analyzing chunk frames = [1217536,1218048) | 20260505 06:55:14.675 | INFO | MemInfo: { | 20260505 06:55:14.675 | INFO | "gpu" : " 17 GB", | 20260505 06:55:14.675 | INFO | "huge" : "", | 20260505 06:55:14.675 | INFO | "huge_shared" : " 27 GB", | 20260505 06:55:14.675 | INFO | "malloc" : " 14 GB", | 20260505 06:55:14.675 | INFO | "pinned" : " 29 GB" | 20260505 06:55:14.675 | INFO | } | 20260505 06:55:14.675 | INFO | Perf report for the last 5.13962 seconds: | 20260505 06:55:14.675 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:55:14.675 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:14.675 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:14.675 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:14.675 | INFO | ANALYSIS : 63.6%, 3.0, 99.0ms, 0 | 20260505 06:55:14.675 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:14.675 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:14.675 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 06:55:14.675 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:14.675 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:14.675 | INFO | Uploaded 36GiB to GPU in 2.14383s (16.7924GiB/s) | 20260505 06:55:14.675 | INFO | Downloaded 34GiB to GPU in 2.05169s (16.5717GiB/s) | 20260505 06:55:19.814 | INFO | Analyzing chunk frames = [1218048,1218560) | 20260505 06:55:19.814 | INFO | MemInfo: { | 20260505 06:55:19.814 | INFO | "gpu" : " 17 GB", | 20260505 06:55:19.814 | INFO | "huge" : "", | 20260505 06:55:19.814 | INFO | "huge_shared" : " 27 GB", | 20260505 06:55:19.814 | INFO | "malloc" : " 14 GB", | 20260505 06:55:19.814 | INFO | "pinned" : " 29 GB" | 20260505 06:55:19.814 | INFO | } | 20260505 06:55:19.814 | INFO | Perf report for the last 5.13917 seconds: | 20260505 06:55:19.814 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:55:19.814 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:19.814 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:19.814 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:19.814 | INFO | ANALYSIS : 63.7%, 3.0, 99.0ms, 0 | 20260505 06:55:19.814 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:19.814 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:19.814 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 06:55:19.814 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:19.814 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:19.815 | INFO | Uploaded 36GiB to GPU in 2.16048s (16.6629GiB/s) | 20260505 06:55:19.815 | INFO | Downloaded 34GiB to GPU in 2.03065s (16.7434GiB/s) | 20260505 06:55:24.954 | INFO | Analyzing chunk frames = [1218560,1219072) | 20260505 06:55:24.954 | INFO | MemInfo: { | 20260505 06:55:24.954 | INFO | "gpu" : " 17 GB", | 20260505 06:55:24.954 | INFO | "huge" : "", | 20260505 06:55:24.954 | INFO | "huge_shared" : " 27 GB", | 20260505 06:55:24.954 | INFO | "malloc" : " 14 GB", | 20260505 06:55:24.954 | INFO | "pinned" : " 29 GB" | 20260505 06:55:24.954 | INFO | } | 20260505 06:55:24.954 | INFO | Perf report for the last 5.13995 seconds: | 20260505 06:55:24.954 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:55:24.954 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:24.954 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:24.954 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:24.954 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 06:55:24.954 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:24.954 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:24.954 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 06:55:24.954 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:24.954 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:24.955 | INFO | Uploaded 36GiB to GPU in 2.15267s (16.7234GiB/s) | 20260505 06:55:24.955 | INFO | Downloaded 34GiB to GPU in 2.0453s (16.6235GiB/s) | 20260505 06:55:30.094 | INFO | Analyzing chunk frames = [1219072,1219584) | 20260505 06:55:30.094 | INFO | MemInfo: { | 20260505 06:55:30.094 | INFO | "gpu" : " 17 GB", | 20260505 06:55:30.094 | INFO | "huge" : "", | 20260505 06:55:30.094 | INFO | "huge_shared" : " 27 GB", | 20260505 06:55:30.094 | INFO | "malloc" : " 14 GB", | 20260505 06:55:30.094 | INFO | "pinned" : " 29 GB" | 20260505 06:55:30.094 | INFO | } | 20260505 06:55:30.094 | INFO | Perf report for the last 5.14022 seconds: | 20260505 06:55:30.095 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:55:30.095 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:30.095 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:30.095 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:30.095 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 06:55:30.095 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:30.095 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:30.095 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 06:55:30.095 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:30.095 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:30.095 | INFO | Uploaded 36GiB to GPU in 2.15938s (16.6714GiB/s) | 20260505 06:55:30.095 | INFO | Downloaded 34GiB to GPU in 2.03705s (16.6908GiB/s) | 20260505 06:55:35.232 | INFO | Analyzing chunk frames = [1219584,1220096) | 20260505 06:55:35.232 | INFO | MemInfo: { | 20260505 06:55:35.232 | INFO | "gpu" : " 17 GB", | 20260505 06:55:35.232 | INFO | "huge" : "", | 20260505 06:55:35.232 | INFO | "huge_shared" : " 27 GB", | 20260505 06:55:35.232 | INFO | "malloc" : " 14 GB", | 20260505 06:55:35.232 | INFO | "pinned" : " 29 GB" | 20260505 06:55:35.232 | INFO | } | 20260505 06:55:35.232 | INFO | Perf report for the last 5.13762 seconds: | 20260505 06:55:35.232 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:55:35.232 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:35.232 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:35.232 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:35.232 | INFO | ANALYSIS : 64.5%, 3.0, 100.7ms, 0 | 20260505 06:55:35.232 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:35.232 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:35.232 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 06:55:35.232 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:35.232 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:35.232 | INFO | Uploaded 36GiB to GPU in 2.21116s (16.2811GiB/s) | 20260505 06:55:35.233 | INFO | Downloaded 34GiB to GPU in 2.16877s (15.6771GiB/s) | 20260505 06:55:40.371 | INFO | Analyzing chunk frames = [1220096,1220608) | 20260505 06:55:40.371 | INFO | MemInfo: { | 20260505 06:55:40.371 | INFO | "gpu" : " 17 GB", | 20260505 06:55:40.371 | INFO | "huge" : "", | 20260505 06:55:40.371 | INFO | "huge_shared" : " 27 GB", | 20260505 06:55:40.371 | INFO | "malloc" : " 14 GB", | 20260505 06:55:40.371 | INFO | "pinned" : " 29 GB" | 20260505 06:55:40.372 | INFO | } | 20260505 06:55:40.372 | INFO | Perf report for the last 5.13959 seconds: | 20260505 06:55:40.372 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:55:40.372 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:40.372 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:40.372 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:40.372 | INFO | ANALYSIS : 64.3%, 3.0, 100.5ms, 0 | 20260505 06:55:40.372 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:40.372 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:40.372 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 06:55:40.372 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:40.372 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:40.372 | INFO | Uploaded 36GiB to GPU in 2.16333s (16.641GiB/s) | 20260505 06:55:40.372 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 06:55:40.372 | INFO | Total mem usage: 15.3 GiB | 20260505 06:55:40.372 | INFO | Downloaded 34GiB to GPU in 2.0514s (16.5741GiB/s) | 20260505 06:55:40.372 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 06:55:40.372 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.76%) | 20260505 06:55:40.372 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 06:55:40.372 | INFO | High Water Breakdown: | 20260505 06:55:40.372 | INFO | 60.4% : 8.67 GiB ( 92.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 06:55:40.372 | INFO | 39.5% : 5.67 GiB ( 85.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 06:55:40.372 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 06:55:40.372 | INFO | ------------------------------------------ | 20260505 06:55:40.372 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:55:40.372 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 06:55:40.372 | INFO | Total mem usage: 26.5 GiB | 20260505 06:55:40.372 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 06:55:40.372 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 06:55:40.372 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 06:55:40.372 | INFO | High Water Breakdown: | 20260505 06:55:40.372 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:55:40.372 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:55:40.372 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:55:40.372 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:55:40.372 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:55:40.373 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:55:40.373 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 06:55:40.373 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 06:55:40.373 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:55:40.373 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 06:55:40.373 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 06:55:40.373 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 06:55:40.373 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 06:55:40.373 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 06:55:40.373 | INFO | ------------------------------------------ | 20260505 06:55:40.373 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:55:40.373 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 06:55:40.373 | INFO | Total mem usage: 25.6 GiB | 20260505 06:55:40.373 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 06:55:40.373 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 06:55:40.373 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 06:55:40.373 | INFO | High Water Breakdown: | 20260505 06:55:40.373 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 06:55:40.373 | INFO | ------------------------------------------ | 20260505 06:55:40.373 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:55:40.373 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 06:55:40.373 | INFO | Total mem usage: 15.7 GiB | 20260505 06:55:40.373 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 06:55:40.373 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 06:55:40.373 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 06:55:40.373 | INFO | High Water Breakdown: | 20260505 06:55:40.373 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 06:55:40.373 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 06:55:40.373 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 06:55:40.373 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 06:55:40.373 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 06:55:40.373 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 06:55:40.373 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 06:55:40.373 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 06:55:40.373 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 06:55:40.373 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:55:40.373 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 06:55:40.373 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:55:40.373 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 06:55:40.373 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 06:55:40.373 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 06:55:40.373 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 06:55:40.373 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:55:40.373 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 06:55:40.373 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:55:40.373 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:55:40.373 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:55:40.373 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 06:55:40.373 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 06:55:40.373 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:55:40.373 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 06:55:40.373 | INFO | ------------------------------------------ | 20260505 06:55:40.373 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:55:40.373 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 06:55:40.373 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:55:40.373 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:55:40.373 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.51e+01, 1.38e+01 / 1.65e+00 | 20260505 06:55:40.373 | INFO | PerfReport: 0.28%, AggregateStats -- 9.39e-02 / 1.05e-01, 1.05e-02 / 4.97e-03 | 20260505 06:55:40.373 | INFO | PerfReport: 11.61%, Baseline -- 3.83e+00 / 4.48e+00, 6.35e-01 / 3.24e-01 | 20260505 06:55:40.373 | INFO | PerfReport: 3.32%, Histogram -- 1.10e+00 / 1.49e+00, 2.17e-01 / 1.05e-01 | 20260505 06:55:40.373 | INFO | PerfReport: 15.45%, DME -- 4.08e+01 / 4.46e+01, 4.08e+00 / 1.25e+00 | 20260505 06:55:40.373 | INFO | PerfReport: 56.17%, FrameLabeling -- 1.86e+01 / 1.91e+01, 2.07e+00 / 1.18e+00 | 20260505 06:55:40.373 | INFO | PerfReport: 7.65%, PulseAccumulating -- 2.53e+00 / 3.11e+00, 4.55e-01 / 3.24e-01 | 20260505 06:55:40.373 | INFO | PerfReport: 4.05%, Metrics -- 1.34e+00 / 1.69e+00, 3.79e-01 / 1.98e-01 | 20260505 06:55:40.373 | INFO | PerfReport: 1.47%, PulseSerializing -- 4.84e-01 / 5.63e-01, 9.40e-02 / 3.89e-02 | 20260505 06:55:40.373 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 06:55:40.373 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:55:40.373 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:55:40.373 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.21e+01 / 0.00e+00 | 20260505 06:55:40.373 | INFO | PerfReport: 22.05%, Upload -- 2.24e+01 / 0.00e+00, 7.80e+00 / 0.00e+00 | 20260505 06:55:40.373 | INFO | PerfReport: 21.44%, Download -- 2.18e+01 / 0.00e+00, 7.75e+00 / 0.00e+00 | 20260505 06:55:40.373 | INFO | PerfReport: 32.55%, Compute -- 3.30e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 06:55:40.373 | INFO | PerfReport: 1.46%, LockUpload -- 1.48e+00 / 0.00e+00, 6.96e+00 / 0.00e+00 | 20260505 06:55:40.373 | INFO | PerfReport: 0.80%, LockDownload -- 8.08e-01 / 0.00e+00, 2.23e+00 / 0.00e+00 | 20260505 06:55:40.373 | INFO | PerfReport: 21.68%, LockCompute -- 2.20e+01 / 0.00e+00, 1.68e+01 / 0.00e+00 | 20260505 06:55:40.373 | INFO | PerfReport: 0.02%, Misc -- 1.52e-02 / 0.00e+00, 2.53e-02 / 0.00e+00 | 20260505 06:55:40.497 | INFO | Starting a round of Baz IO | 20260505 06:55:43.780 | INFO | Populating RTMetrics report | 20260505 06:55:43.781 | INFO | RTMetrics report populated | 20260505 06:55:43.781 | INFO | Opening tmp RTMetrics json file | 20260505 06:55:43.814 | INFO | Serializing RTMetrics to json | 20260505 06:55:43.838 | INFO | Serialization done, swapping tmp file now... | 20260505 06:55:43.841 | INFO | RTMetrics done | 20260505 06:55:43.903 | INFO | Finished a round of Baz IO: Wrote 2.407 GiB in 3.406 seconds | 20260505 06:55:43.903 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 31.49% / 0.04% / 68.47% | 20260505 06:55:43.903 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.33712% / 5.5631e-05% / 5.5631e-05% | 20260505 06:55:43.903 | INFO | Baz IO overall: | 20260505 06:55:43.903 | INFO | BytesWritten : 648470466495 | 20260505 06:55:43.903 | INFO | NumEvents : 307178710453 | 20260505 06:55:45.511 | INFO | Analyzing chunk frames = [1220608,1221120) | 20260505 06:55:45.511 | INFO | MemInfo: { | 20260505 06:55:45.511 | INFO | "gpu" : " 17 GB", | 20260505 06:55:45.511 | INFO | "huge" : "", | 20260505 06:55:45.511 | INFO | "huge_shared" : " 27 GB", | 20260505 06:55:45.511 | INFO | "malloc" : " 14 GB", | 20260505 06:55:45.511 | INFO | "pinned" : " 29 GB" | 20260505 06:55:45.511 | INFO | } | 20260505 06:55:45.511 | INFO | Perf report for the last 5.1393 seconds: | 20260505 06:55:45.511 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:55:45.511 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:45.511 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:45.511 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:45.511 | INFO | ANALYSIS : 65.2%, 2.9, 99.6ms, 0 | 20260505 06:55:45.511 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:45.511 | INFO | RT_METRICS : 47.2%, 1.0, 24.8ms, 0 | 20260505 06:55:45.511 | INFO | PRE_HQ : 64.0%, 1.9, 63.3ms, 0 | 20260505 06:55:45.511 | INFO | BAZWRITER : 65.7%, 4.1, 137.6ms, 0 | 20260505 06:55:45.511 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 06:55:45.511 | INFO | Uploaded 36GiB to GPU in 2.23479s (16.1089GiB/s) | 20260505 06:55:45.511 | INFO | Downloaded 38GiB to GPU in 2.31755s (16.3966GiB/s) | 20260505 06:55:50.650 | INFO | Analyzing chunk frames = [1221120,1221632) | 20260505 06:55:50.650 | INFO | MemInfo: { | 20260505 06:55:50.650 | INFO | "gpu" : " 17 GB", | 20260505 06:55:50.650 | INFO | "huge" : "", | 20260505 06:55:50.650 | INFO | "huge_shared" : " 27 GB", | 20260505 06:55:50.650 | INFO | "malloc" : " 14 GB", | 20260505 06:55:50.650 | INFO | "pinned" : " 29 GB" | 20260505 06:55:50.650 | INFO | } | 20260505 06:55:50.651 | INFO | Perf report for the last 5.13955 seconds: | 20260505 06:55:50.651 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:55:50.651 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:50.651 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:50.651 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:50.651 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 06:55:50.651 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:50.651 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:50.651 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 06:55:50.651 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:50.651 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:50.651 | INFO | Uploaded 36GiB to GPU in 2.09599s (17.1756GiB/s) | 20260505 06:55:50.651 | INFO | Downloaded 34GiB to GPU in 2.0351s (16.7068GiB/s) | 20260505 06:55:55.789 | INFO | Analyzing chunk frames = [1221632,1222144) | 20260505 06:55:55.790 | INFO | MemInfo: { | 20260505 06:55:55.790 | INFO | "gpu" : " 17 GB", | 20260505 06:55:55.790 | INFO | "huge" : "", | 20260505 06:55:55.790 | INFO | "huge_shared" : " 27 GB", | 20260505 06:55:55.790 | INFO | "malloc" : " 14 GB", | 20260505 06:55:55.790 | INFO | "pinned" : " 29 GB" | 20260505 06:55:55.790 | INFO | } | 20260505 06:55:55.790 | INFO | Perf report for the last 5.1392 seconds: | 20260505 06:55:55.790 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:55:55.790 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:55.790 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:55.790 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:55.790 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 06:55:55.790 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:55.790 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:55.790 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 06:55:55.790 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:55.790 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:55:55.790 | INFO | Uploaded 36GiB to GPU in 2.13838s (16.8352GiB/s) | 20260505 06:55:55.790 | INFO | Downloaded 34GiB to GPU in 2.04601s (16.6177GiB/s) | 20260505 06:56:00.929 | INFO | Analyzing chunk frames = [1222144,1222656) | 20260505 06:56:00.929 | INFO | MemInfo: { | 20260505 06:56:00.929 | INFO | "gpu" : " 17 GB", | 20260505 06:56:00.929 | INFO | "huge" : "", | 20260505 06:56:00.929 | INFO | "huge_shared" : " 27 GB", | 20260505 06:56:00.929 | INFO | "malloc" : " 14 GB", | 20260505 06:56:00.929 | INFO | "pinned" : " 29 GB" | 20260505 06:56:00.929 | INFO | } | 20260505 06:56:00.929 | INFO | Perf report for the last 5.13909 seconds: | 20260505 06:56:00.929 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:56:00.929 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:00.929 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:00.929 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:00.929 | INFO | ANALYSIS : 63.6%, 3.0, 98.8ms, 0 | 20260505 06:56:00.929 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:00.929 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:00.929 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 06:56:00.929 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:00.929 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:00.929 | INFO | Uploaded 36GiB to GPU in 2.14442s (16.7878GiB/s) | 20260505 06:56:00.929 | INFO | Downloaded 34GiB to GPU in 2.02802s (16.7652GiB/s) | 20260505 06:56:06.068 | INFO | Analyzing chunk frames = [1222656,1223168) | 20260505 06:56:06.068 | INFO | MemInfo: { | 20260505 06:56:06.068 | INFO | "gpu" : " 17 GB", | 20260505 06:56:06.068 | INFO | "huge" : "", | 20260505 06:56:06.068 | INFO | "huge_shared" : " 27 GB", | 20260505 06:56:06.068 | INFO | "malloc" : " 14 GB", | 20260505 06:56:06.068 | INFO | "pinned" : " 29 GB" | 20260505 06:56:06.068 | INFO | } | 20260505 06:56:06.068 | INFO | Perf report for the last 5.1394 seconds: | 20260505 06:56:06.068 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:56:06.068 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:06.068 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:06.068 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:06.068 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 06:56:06.068 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:06.068 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:06.068 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 06:56:06.068 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:06.068 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:06.069 | INFO | Uploaded 36GiB to GPU in 2.15274s (16.7229GiB/s) | 20260505 06:56:06.069 | INFO | Downloaded 34GiB to GPU in 2.04735s (16.6069GiB/s) | 20260505 06:56:11.209 | INFO | Analyzing chunk frames = [1223168,1223680) | 20260505 06:56:11.209 | INFO | MemInfo: { | 20260505 06:56:11.209 | INFO | "gpu" : " 17 GB", | 20260505 06:56:11.209 | INFO | "huge" : "", | 20260505 06:56:11.209 | INFO | "huge_shared" : " 27 GB", | 20260505 06:56:11.209 | INFO | "malloc" : " 14 GB", | 20260505 06:56:11.209 | INFO | "pinned" : " 29 GB" | 20260505 06:56:11.209 | INFO | } | 20260505 06:56:11.209 | INFO | Perf report for the last 5.14096 seconds: | 20260505 06:56:11.209 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:56:11.209 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:11.209 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:11.209 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:11.209 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 06:56:11.209 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:11.209 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:11.209 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 06:56:11.209 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:11.210 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:11.210 | INFO | Uploaded 36GiB to GPU in 2.15116s (16.7351GiB/s) | 20260505 06:56:11.210 | INFO | Downloaded 34GiB to GPU in 2.03756s (16.6866GiB/s) | 20260505 06:56:16.347 | INFO | Analyzing chunk frames = [1223680,1224192) | 20260505 06:56:16.347 | INFO | MemInfo: { | 20260505 06:56:16.347 | INFO | "gpu" : " 17 GB", | 20260505 06:56:16.347 | INFO | "huge" : "", | 20260505 06:56:16.347 | INFO | "huge_shared" : " 27 GB", | 20260505 06:56:16.347 | INFO | "malloc" : " 14 GB", | 20260505 06:56:16.347 | INFO | "pinned" : " 29 GB" | 20260505 06:56:16.347 | INFO | } | 20260505 06:56:16.347 | INFO | Perf report for the last 5.13784 seconds: | 20260505 06:56:16.347 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:56:16.347 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:16.347 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:16.347 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:16.347 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 06:56:16.347 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:16.347 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:16.347 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260505 06:56:16.347 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:16.347 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:16.347 | INFO | Uploaded 36GiB to GPU in 2.21701s (16.2381GiB/s) | 20260505 06:56:16.347 | INFO | Downloaded 34GiB to GPU in 2.17061s (15.6638GiB/s) | 20260505 06:56:21.486 | INFO | Analyzing chunk frames = [1224192,1224704) | 20260505 06:56:21.486 | INFO | MemInfo: { | 20260505 06:56:21.486 | INFO | "gpu" : " 17 GB", | 20260505 06:56:21.486 | INFO | "huge" : "", | 20260505 06:56:21.486 | INFO | "huge_shared" : " 27 GB", | 20260505 06:56:21.486 | INFO | "malloc" : " 14 GB", | 20260505 06:56:21.487 | INFO | "pinned" : " 29 GB" | 20260505 06:56:21.487 | INFO | } | 20260505 06:56:21.487 | INFO | Perf report for the last 5.13957 seconds: | 20260505 06:56:21.487 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:56:21.487 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:21.487 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:21.487 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:21.487 | INFO | ANALYSIS : 64.4%, 3.0, 99.5ms, 0 | 20260505 06:56:21.487 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:21.487 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:21.487 | INFO | PRE_HQ : 30.6%, 1.0, 15.7ms, 0 | 20260505 06:56:21.487 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:21.487 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:21.487 | INFO | Uploaded 36GiB to GPU in 2.17021s (16.5882GiB/s) | 20260505 06:56:21.487 | INFO | Downloaded 34GiB to GPU in 2.04329s (16.6398GiB/s) | 20260505 06:56:21.604 | INFO | Starting a round of Baz IO | 20260505 06:56:24.901 | INFO | Populating RTMetrics report | 20260505 06:56:24.901 | INFO | RTMetrics report populated | 20260505 06:56:24.901 | INFO | Opening tmp RTMetrics json file | 20260505 06:56:24.902 | INFO | Serializing RTMetrics to json | 20260505 06:56:24.919 | INFO | Serialization done, swapping tmp file now... | 20260505 06:56:24.928 | INFO | RTMetrics done | 20260505 06:56:25.015 | INFO | Finished a round of Baz IO: Wrote 2.406 GiB in 3.41 seconds | 20260505 06:56:25.015 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 31.45% / 0.04% / 68.51% | 20260505 06:56:25.015 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.30808% / 7.15256e-05% / 4.37101e-05% | 20260505 06:56:25.015 | INFO | Baz IO overall: | 20260505 06:56:25.015 | INFO | BytesWritten : 651053358779 | 20260505 06:56:25.015 | INFO | NumEvents : 308414093546 | 20260505 06:56:26.626 | INFO | Analyzing chunk frames = [1224704,1225216) | 20260505 06:56:26.626 | INFO | MemInfo: { | 20260505 06:56:26.626 | INFO | "gpu" : " 17 GB", | 20260505 06:56:26.626 | INFO | "huge" : "", | 20260505 06:56:26.626 | INFO | "huge_shared" : " 27 GB", | 20260505 06:56:26.626 | INFO | "malloc" : " 14 GB", | 20260505 06:56:26.626 | INFO | "pinned" : " 29 GB" | 20260505 06:56:26.626 | INFO | } | 20260505 06:56:26.626 | INFO | Perf report for the last 5.13915 seconds: | 20260505 06:56:26.626 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:56:26.626 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:26.626 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:26.626 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:26.626 | INFO | ANALYSIS : 65.3%, 2.9, 99.7ms, 0 | 20260505 06:56:26.626 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:26.626 | INFO | RT_METRICS : 46.5%, 1.0, 24.4ms, 0 | 20260505 06:56:26.626 | INFO | PRE_HQ : 64.1%, 1.9, 63.2ms, 0 | 20260505 06:56:26.626 | INFO | BAZWRITER : 65.6%, 4.1, 137.0ms, 0 | 20260505 06:56:26.626 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 06:56:26.626 | INFO | Uploaded 36GiB to GPU in 2.22487s (16.1807GiB/s) | 20260505 06:56:26.626 | INFO | Downloaded 38GiB to GPU in 2.31765s (16.3959GiB/s) | 20260505 06:56:31.765 | INFO | Analyzing chunk frames = [1225216,1225728) | 20260505 06:56:31.765 | INFO | MemInfo: { | 20260505 06:56:31.765 | INFO | "gpu" : " 17 GB", | 20260505 06:56:31.765 | INFO | "huge" : "", | 20260505 06:56:31.765 | INFO | "huge_shared" : " 27 GB", | 20260505 06:56:31.765 | INFO | "malloc" : " 14 GB", | 20260505 06:56:31.765 | INFO | "pinned" : " 29 GB" | 20260505 06:56:31.765 | INFO | } | 20260505 06:56:31.765 | INFO | Perf report for the last 5.13928 seconds: | 20260505 06:56:31.765 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:56:31.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:31.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:31.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:31.765 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 06:56:31.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:31.765 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:31.765 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 06:56:31.765 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:31.765 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:31.765 | INFO | Uploaded 36GiB to GPU in 2.09793s (17.1598GiB/s) | 20260505 06:56:31.765 | INFO | Downloaded 34GiB to GPU in 2.03755s (16.6867GiB/s) | 20260505 06:56:36.904 | INFO | Analyzing chunk frames = [1225728,1226240) | 20260505 06:56:36.904 | INFO | MemInfo: { | 20260505 06:56:36.904 | INFO | "gpu" : " 17 GB", | 20260505 06:56:36.904 | INFO | "huge" : "", | 20260505 06:56:36.904 | INFO | "huge_shared" : " 27 GB", | 20260505 06:56:36.904 | INFO | "malloc" : " 14 GB", | 20260505 06:56:36.904 | INFO | "pinned" : " 29 GB" | 20260505 06:56:36.904 | INFO | } | 20260505 06:56:36.904 | INFO | Perf report for the last 5.13929 seconds: | 20260505 06:56:36.904 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:56:36.904 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:36.904 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:36.904 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:36.904 | INFO | ANALYSIS : 63.6%, 3.0, 99.0ms, 0 | 20260505 06:56:36.904 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:36.905 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:36.905 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 06:56:36.905 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:36.905 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:36.905 | INFO | Uploaded 36GiB to GPU in 2.14688s (16.7685GiB/s) | 20260505 06:56:36.905 | INFO | Downloaded 34GiB to GPU in 2.04733s (16.607GiB/s) | 20260505 06:56:42.044 | INFO | Analyzing chunk frames = [1226240,1226752) | 20260505 06:56:42.044 | INFO | MemInfo: { | 20260505 06:56:42.044 | INFO | "gpu" : " 17 GB", | 20260505 06:56:42.044 | INFO | "huge" : "", | 20260505 06:56:42.044 | INFO | "huge_shared" : " 27 GB", | 20260505 06:56:42.044 | INFO | "malloc" : " 14 GB", | 20260505 06:56:42.044 | INFO | "pinned" : " 29 GB" | 20260505 06:56:42.044 | INFO | } | 20260505 06:56:42.044 | INFO | Perf report for the last 5.13938 seconds: | 20260505 06:56:42.044 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:56:42.044 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:42.044 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:42.044 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:42.044 | INFO | ANALYSIS : 63.4%, 3.0, 98.4ms, 0 | 20260505 06:56:42.044 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:42.044 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:42.044 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 06:56:42.044 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:42.044 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:42.044 | INFO | Uploaded 36GiB to GPU in 2.12754s (16.9209GiB/s) | 20260505 06:56:42.044 | INFO | Downloaded 34GiB to GPU in 2.03052s (16.7445GiB/s) | 20260505 06:56:47.183 | INFO | Analyzing chunk frames = [1226752,1227264) | 20260505 06:56:47.183 | INFO | MemInfo: { | 20260505 06:56:47.183 | INFO | "gpu" : " 17 GB", | 20260505 06:56:47.183 | INFO | "huge" : "", | 20260505 06:56:47.183 | INFO | "huge_shared" : " 27 GB", | 20260505 06:56:47.183 | INFO | "malloc" : " 14 GB", | 20260505 06:56:47.183 | INFO | "pinned" : " 29 GB" | 20260505 06:56:47.183 | INFO | } | 20260505 06:56:47.183 | INFO | Perf report for the last 5.1393 seconds: | 20260505 06:56:47.183 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:56:47.183 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:47.183 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:47.183 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:47.183 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 06:56:47.183 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:47.183 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:47.183 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 06:56:47.183 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:47.183 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:47.183 | INFO | Uploaded 36GiB to GPU in 2.15258s (16.7241GiB/s) | 20260505 06:56:47.183 | INFO | Downloaded 34GiB to GPU in 2.04456s (16.6295GiB/s) | 20260505 06:56:52.324 | INFO | Analyzing chunk frames = [1227264,1227776) | 20260505 06:56:52.324 | INFO | MemInfo: { | 20260505 06:56:52.324 | INFO | "gpu" : " 17 GB", | 20260505 06:56:52.324 | INFO | "huge" : "", | 20260505 06:56:52.324 | INFO | "huge_shared" : " 27 GB", | 20260505 06:56:52.324 | INFO | "malloc" : " 14 GB", | 20260505 06:56:52.324 | INFO | "pinned" : " 29 GB" | 20260505 06:56:52.324 | INFO | } | 20260505 06:56:52.324 | INFO | Perf report for the last 5.14102 seconds: | 20260505 06:56:52.324 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:56:52.324 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:52.324 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:52.324 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:52.324 | INFO | ANALYSIS : 63.5%, 3.0, 99.3ms, 0 | 20260505 06:56:52.324 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:52.324 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:52.324 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 06:56:52.324 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:52.324 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:52.324 | INFO | Uploaded 36GiB to GPU in 2.12761s (16.9204GiB/s) | 20260505 06:56:52.324 | INFO | Downloaded 34GiB to GPU in 2.03179s (16.734GiB/s) | 20260505 06:56:57.462 | INFO | Analyzing chunk frames = [1227776,1228288) | 20260505 06:56:57.462 | INFO | MemInfo: { | 20260505 06:56:57.462 | INFO | "gpu" : " 17 GB", | 20260505 06:56:57.462 | INFO | "huge" : "", | 20260505 06:56:57.462 | INFO | "huge_shared" : " 27 GB", | 20260505 06:56:57.462 | INFO | "malloc" : " 14 GB", | 20260505 06:56:57.462 | INFO | "pinned" : " 29 GB" | 20260505 06:56:57.462 | INFO | } | 20260505 06:56:57.462 | INFO | Perf report for the last 5.13752 seconds: | 20260505 06:56:57.462 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:56:57.462 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:57.462 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:57.462 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:57.462 | INFO | ANALYSIS : 64.9%, 3.0, 101.3ms, 0 | 20260505 06:56:57.462 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:57.462 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:57.462 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 06:56:57.462 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:57.462 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:56:57.462 | INFO | Uploaded 36GiB to GPU in 2.24248s (16.0536GiB/s) | 20260505 06:56:57.462 | INFO | Downloaded 34GiB to GPU in 2.15399s (15.7847GiB/s) | 20260505 06:57:02.601 | INFO | Analyzing chunk frames = [1228288,1228800) | 20260505 06:57:02.601 | INFO | MemInfo: { | 20260505 06:57:02.601 | INFO | "gpu" : " 17 GB", | 20260505 06:57:02.601 | INFO | "huge" : "", | 20260505 06:57:02.601 | INFO | "huge_shared" : " 27 GB", | 20260505 06:57:02.601 | INFO | "malloc" : " 14 GB", | 20260505 06:57:02.601 | INFO | "pinned" : " 29 GB" | 20260505 06:57:02.601 | INFO | } | 20260505 06:57:02.601 | INFO | Perf report for the last 5.13917 seconds: | 20260505 06:57:02.601 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:57:02.601 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:02.601 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:02.601 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:02.601 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 06:57:02.601 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:02.601 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:02.601 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 06:57:02.601 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:02.601 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:02.601 | INFO | Uploaded 36GiB to GPU in 2.16352s (16.6395GiB/s) | 20260505 06:57:02.601 | INFO | Downloaded 34GiB to GPU in 2.04293s (16.6428GiB/s) | 20260505 06:57:02.602 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 06:57:02.602 | INFO | Total mem usage: 15.3 GiB | 20260505 06:57:02.602 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 06:57:02.602 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.68%) | 20260505 06:57:02.602 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 06:57:02.602 | INFO | High Water Breakdown: | 20260505 06:57:02.602 | INFO | 60.5% : 8.69 GiB ( 92.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 06:57:02.602 | INFO | 39.4% : 5.66 GiB ( 85.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 06:57:02.602 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 06:57:02.602 | INFO | ------------------------------------------ | 20260505 06:57:02.602 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:57:02.602 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 06:57:02.602 | INFO | Total mem usage: 26.5 GiB | 20260505 06:57:02.602 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 06:57:02.602 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 06:57:02.602 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 06:57:02.602 | INFO | High Water Breakdown: | 20260505 06:57:02.602 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:57:02.602 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:57:02.602 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:57:02.602 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:57:02.602 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:57:02.602 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:57:02.602 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 06:57:02.602 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 06:57:02.602 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:57:02.602 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 06:57:02.602 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 06:57:02.602 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 06:57:02.602 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 06:57:02.602 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 06:57:02.602 | INFO | ------------------------------------------ | 20260505 06:57:02.602 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:57:02.602 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 06:57:02.602 | INFO | Total mem usage: 25.6 GiB | 20260505 06:57:02.602 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 06:57:02.602 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 06:57:02.602 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 06:57:02.602 | INFO | High Water Breakdown: | 20260505 06:57:02.602 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 06:57:02.602 | INFO | ------------------------------------------ | 20260505 06:57:02.602 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:57:02.602 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 06:57:02.602 | INFO | Total mem usage: 15.7 GiB | 20260505 06:57:02.602 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 06:57:02.602 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 06:57:02.602 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 06:57:02.602 | INFO | High Water Breakdown: | 20260505 06:57:02.602 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 06:57:02.602 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 06:57:02.602 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 06:57:02.602 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 06:57:02.602 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 06:57:02.602 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 06:57:02.602 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 06:57:02.602 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 06:57:02.602 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 06:57:02.602 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:57:02.602 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 06:57:02.602 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:57:02.602 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 06:57:02.602 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 06:57:02.602 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 06:57:02.603 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 06:57:02.603 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:57:02.603 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 06:57:02.603 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:57:02.603 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:57:02.603 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:57:02.603 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 06:57:02.603 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 06:57:02.603 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:57:02.603 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 06:57:02.603 | INFO | ------------------------------------------ | 20260505 06:57:02.603 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:57:02.603 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 06:57:02.603 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:57:02.603 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:57:02.603 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.53e+01, 1.37e+01 / 1.90e+00 | 20260505 06:57:02.603 | INFO | PerfReport: 0.28%, AggregateStats -- 9.37e-02 / 1.05e-01, 1.03e-02 / 6.10e-03 | 20260505 06:57:02.603 | INFO | PerfReport: 11.59%, Baseline -- 3.82e+00 / 4.46e+00, 6.32e-01 / 3.20e-01 | 20260505 06:57:02.603 | INFO | PerfReport: 3.31%, Histogram -- 1.09e+00 / 1.47e+00, 2.14e-01 / 1.15e-01 | 20260505 06:57:02.603 | INFO | PerfReport: 15.46%, DME -- 4.08e+01 / 4.46e+01, 4.10e+00 / 1.34e+00 | 20260505 06:57:02.603 | INFO | PerfReport: 56.21%, FrameLabeling -- 1.85e+01 / 1.92e+01, 2.07e+00 / 1.55e+00 | 20260505 06:57:02.603 | INFO | PerfReport: 7.64%, PulseAccumulating -- 2.52e+00 / 3.17e+00, 4.48e-01 / 2.63e-01 | 20260505 06:57:02.603 | INFO | PerfReport: 4.04%, Metrics -- 1.33e+00 / 1.74e+00, 3.72e-01 / 1.82e-01 | 20260505 06:57:02.603 | INFO | PerfReport: 1.47%, PulseSerializing -- 4.84e-01 / 5.58e-01, 9.34e-02 / 3.44e-02 | 20260505 06:57:02.603 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 06:57:02.603 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:57:02.603 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:57:02.603 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.25e+01 / 0.00e+00 | 20260505 06:57:02.603 | INFO | PerfReport: 22.17%, Upload -- 2.25e+01 / 0.00e+00, 7.96e+00 / 0.00e+00 | 20260505 06:57:02.603 | INFO | PerfReport: 21.43%, Download -- 2.18e+01 / 0.00e+00, 7.80e+00 / 0.00e+00 | 20260505 06:57:02.603 | INFO | PerfReport: 32.48%, Compute -- 3.30e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 06:57:02.603 | INFO | PerfReport: 1.42%, LockUpload -- 1.44e+00 / 0.00e+00, 6.91e+00 / 0.00e+00 | 20260505 06:57:02.603 | INFO | PerfReport: 0.80%, LockDownload -- 8.18e-01 / 0.00e+00, 2.18e+00 / 0.00e+00 | 20260505 06:57:02.603 | INFO | PerfReport: 21.69%, LockCompute -- 2.20e+01 / 0.00e+00, 1.64e+01 / 0.00e+00 | 20260505 06:57:02.603 | INFO | PerfReport: 0.01%, Misc -- 1.46e-02 / 0.00e+00, 2.22e-02 / 0.00e+00 | 20260505 06:57:02.721 | INFO | Starting a round of Baz IO | 20260505 06:57:05.982 | INFO | Populating RTMetrics report | 20260505 06:57:05.982 | INFO | RTMetrics report populated | 20260505 06:57:05.982 | INFO | Opening tmp RTMetrics json file | 20260505 06:57:06.010 | INFO | Serializing RTMetrics to json | 20260505 06:57:06.012 | INFO | Serialization done, swapping tmp file now... | 20260505 06:57:06.016 | INFO | RTMetrics done | 20260505 06:57:06.106 | INFO | Finished a round of Baz IO: Wrote 2.408 GiB in 3.384 seconds | 20260505 06:57:06.106 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 31.41% / 0.04% / 68.55% | 20260505 06:57:06.106 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.29701% / 7.15256e-05% / 6.75519e-05% | 20260505 06:57:06.106 | INFO | Baz IO overall: | 20260505 06:57:06.106 | INFO | BytesWritten : 653638908566 | 20260505 06:57:06.106 | INFO | NumEvents : 309651086181 | 20260505 06:57:07.740 | INFO | Analyzing chunk frames = [1228800,1229312) | 20260505 06:57:07.740 | INFO | MemInfo: { | 20260505 06:57:07.740 | INFO | "gpu" : " 17 GB", | 20260505 06:57:07.740 | INFO | "huge" : "", | 20260505 06:57:07.740 | INFO | "huge_shared" : " 27 GB", | 20260505 06:57:07.740 | INFO | "malloc" : " 14 GB", | 20260505 06:57:07.740 | INFO | "pinned" : " 29 GB" | 20260505 06:57:07.740 | INFO | } | 20260505 06:57:07.741 | INFO | Perf report for the last 5.13958 seconds: | 20260505 06:57:07.741 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:57:07.741 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:07.741 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:07.741 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:07.741 | INFO | ANALYSIS : 64.6%, 2.9, 98.8ms, 0 | 20260505 06:57:07.741 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:07.741 | INFO | RT_METRICS : 46.7%, 1.0, 24.5ms, 0 | 20260505 06:57:07.741 | INFO | PRE_HQ : 63.5%, 1.9, 63.1ms, 0 | 20260505 06:57:07.741 | INFO | BAZWRITER : 65.2%, 4.1, 138.2ms, 0 | 20260505 06:57:07.741 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 06:57:07.741 | INFO | Uploaded 36GiB to GPU in 2.18025s (16.5119GiB/s) | 20260505 06:57:07.741 | INFO | Downloaded 38GiB to GPU in 2.30352s (16.4965GiB/s) | 20260505 06:57:12.880 | INFO | Analyzing chunk frames = [1229312,1229824) | 20260505 06:57:12.880 | INFO | MemInfo: { | 20260505 06:57:12.880 | INFO | "gpu" : " 17 GB", | 20260505 06:57:12.880 | INFO | "huge" : "", | 20260505 06:57:12.880 | INFO | "huge_shared" : " 27 GB", | 20260505 06:57:12.880 | INFO | "malloc" : " 14 GB", | 20260505 06:57:12.880 | INFO | "pinned" : " 29 GB" | 20260505 06:57:12.880 | INFO | } | 20260505 06:57:12.880 | INFO | Perf report for the last 5.13998 seconds: | 20260505 06:57:12.880 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:57:12.880 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:12.880 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:12.881 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:12.881 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 06:57:12.881 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:12.881 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:12.881 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 06:57:12.881 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:12.881 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:12.881 | INFO | Uploaded 36GiB to GPU in 2.10579s (17.0957GiB/s) | 20260505 06:57:12.881 | INFO | Downloaded 34GiB to GPU in 2.03757s (16.6866GiB/s) | 20260505 06:57:18.019 | INFO | Analyzing chunk frames = [1229824,1230336) | 20260505 06:57:18.019 | INFO | MemInfo: { | 20260505 06:57:18.019 | INFO | "gpu" : " 17 GB", | 20260505 06:57:18.019 | INFO | "huge" : "", | 20260505 06:57:18.019 | INFO | "huge_shared" : " 27 GB", | 20260505 06:57:18.019 | INFO | "malloc" : " 14 GB", | 20260505 06:57:18.019 | INFO | "pinned" : " 29 GB" | 20260505 06:57:18.019 | INFO | } | 20260505 06:57:18.019 | INFO | Perf report for the last 5.13883 seconds: | 20260505 06:57:18.019 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:57:18.019 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:18.019 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:18.019 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:18.019 | INFO | ANALYSIS : 63.8%, 3.0, 99.2ms, 0 | 20260505 06:57:18.019 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:18.019 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:18.019 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 06:57:18.019 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:18.019 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:18.020 | INFO | Uploaded 36GiB to GPU in 2.15578s (16.6993GiB/s) | 20260505 06:57:18.020 | INFO | Downloaded 34GiB to GPU in 2.05263s (16.5641GiB/s) | 20260505 06:57:23.158 | INFO | Analyzing chunk frames = [1230336,1230848) | 20260505 06:57:23.158 | INFO | MemInfo: { | 20260505 06:57:23.158 | INFO | "gpu" : " 17 GB", | 20260505 06:57:23.158 | INFO | "huge" : "", | 20260505 06:57:23.158 | INFO | "huge_shared" : " 27 GB", | 20260505 06:57:23.158 | INFO | "malloc" : " 14 GB", | 20260505 06:57:23.158 | INFO | "pinned" : " 29 GB" | 20260505 06:57:23.158 | INFO | } | 20260505 06:57:23.159 | INFO | Perf report for the last 5.13918 seconds: | 20260505 06:57:23.159 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:57:23.159 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:23.159 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:23.159 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:23.159 | INFO | ANALYSIS : 63.6%, 3.0, 98.8ms, 0 | 20260505 06:57:23.159 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:23.159 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:23.159 | INFO | PRE_HQ : 30.5%, 1.0, 15.9ms, 0 | 20260505 06:57:23.159 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:23.159 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:23.159 | INFO | Uploaded 36GiB to GPU in 2.15049s (16.7404GiB/s) | 20260505 06:57:23.159 | INFO | Downloaded 34GiB to GPU in 2.03333s (16.7214GiB/s) | 20260505 06:57:28.298 | INFO | Analyzing chunk frames = [1230848,1231360) | 20260505 06:57:28.298 | INFO | MemInfo: { | 20260505 06:57:28.298 | INFO | "gpu" : " 17 GB", | 20260505 06:57:28.298 | INFO | "huge" : "", | 20260505 06:57:28.298 | INFO | "huge_shared" : " 27 GB", | 20260505 06:57:28.298 | INFO | "malloc" : " 14 GB", | 20260505 06:57:28.298 | INFO | "pinned" : " 29 GB" | 20260505 06:57:28.298 | INFO | } | 20260505 06:57:28.298 | INFO | Perf report for the last 5.1394 seconds: | 20260505 06:57:28.298 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:57:28.298 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:28.298 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:28.298 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:28.298 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 06:57:28.298 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:28.298 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:28.298 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 06:57:28.298 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:28.298 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:28.298 | INFO | Uploaded 36GiB to GPU in 2.11697s (17.0055GiB/s) | 20260505 06:57:28.298 | INFO | Downloaded 34GiB to GPU in 2.03113s (16.7394GiB/s) | 20260505 06:57:33.439 | INFO | Analyzing chunk frames = [1231360,1231872) | 20260505 06:57:33.439 | INFO | MemInfo: { | 20260505 06:57:33.439 | INFO | "gpu" : " 17 GB", | 20260505 06:57:33.439 | INFO | "huge" : "", | 20260505 06:57:33.439 | INFO | "huge_shared" : " 27 GB", | 20260505 06:57:33.439 | INFO | "malloc" : " 14 GB", | 20260505 06:57:33.439 | INFO | "pinned" : " 29 GB" | 20260505 06:57:33.439 | INFO | } | 20260505 06:57:33.439 | INFO | Perf report for the last 5.14071 seconds: | 20260505 06:57:33.439 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:57:33.439 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:33.439 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:33.439 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:33.439 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 06:57:33.439 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:33.439 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:33.439 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 06:57:33.439 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:33.439 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:33.439 | INFO | Uploaded 36GiB to GPU in 2.13075s (16.8954GiB/s) | 20260505 06:57:33.439 | INFO | Downloaded 34GiB to GPU in 2.04042s (16.6632GiB/s) | 20260505 06:57:38.577 | INFO | Analyzing chunk frames = [1231872,1232384) | 20260505 06:57:38.577 | INFO | MemInfo: { | 20260505 06:57:38.577 | INFO | "gpu" : " 17 GB", | 20260505 06:57:38.577 | INFO | "huge" : "", | 20260505 06:57:38.577 | INFO | "huge_shared" : " 27 GB", | 20260505 06:57:38.577 | INFO | "malloc" : " 14 GB", | 20260505 06:57:38.577 | INFO | "pinned" : " 29 GB" | 20260505 06:57:38.577 | INFO | } | 20260505 06:57:38.577 | INFO | Perf report for the last 5.1386 seconds: | 20260505 06:57:38.577 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:57:38.577 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:38.577 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:38.577 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:38.577 | INFO | ANALYSIS : 64.7%, 3.0, 101.1ms, 0 | 20260505 06:57:38.577 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:38.577 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:38.577 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260505 06:57:38.577 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:38.577 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:38.578 | INFO | Uploaded 36GiB to GPU in 2.24031s (16.0692GiB/s) | 20260505 06:57:38.578 | INFO | Downloaded 34GiB to GPU in 2.16251s (15.7225GiB/s) | 20260505 06:57:43.716 | INFO | Analyzing chunk frames = [1232384,1232896) | 20260505 06:57:43.716 | INFO | MemInfo: { | 20260505 06:57:43.716 | INFO | "gpu" : " 17 GB", | 20260505 06:57:43.716 | INFO | "huge" : "", | 20260505 06:57:43.716 | INFO | "huge_shared" : " 27 GB", | 20260505 06:57:43.716 | INFO | "malloc" : " 14 GB", | 20260505 06:57:43.716 | INFO | "pinned" : " 29 GB" | 20260505 06:57:43.716 | INFO | } | 20260505 06:57:43.716 | INFO | Perf report for the last 5.13899 seconds: | 20260505 06:57:43.716 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:57:43.716 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:43.716 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:43.716 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:43.716 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 06:57:43.716 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:43.716 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:43.716 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 06:57:43.716 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:43.716 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:43.717 | INFO | Uploaded 36GiB to GPU in 2.16241s (16.6481GiB/s) | 20260505 06:57:43.717 | INFO | Downloaded 34GiB to GPU in 2.04268s (16.6448GiB/s) | 20260505 06:57:43.836 | INFO | Starting a round of Baz IO | 20260505 06:57:47.092 | INFO | Populating RTMetrics report | 20260505 06:57:47.092 | INFO | RTMetrics report populated | 20260505 06:57:47.092 | INFO | Opening tmp RTMetrics json file | 20260505 06:57:47.117 | INFO | Serializing RTMetrics to json | 20260505 06:57:47.131 | INFO | Serialization done, swapping tmp file now... | 20260505 06:57:47.143 | INFO | RTMetrics done | 20260505 06:57:47.209 | INFO | Finished a round of Baz IO: Wrote 2.407 GiB in 3.372 seconds | 20260505 06:57:47.209 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 31.37% / 0.04% / 68.59% | 20260505 06:57:47.209 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.2732% / 3.57628e-05% / 6.35783e-05% | 20260505 06:57:47.209 | INFO | Baz IO overall: | 20260505 06:57:47.209 | INFO | BytesWritten : 656223746356 | 20260505 06:57:47.209 | INFO | NumEvents : 310887364637 | 20260505 06:57:48.855 | INFO | Analyzing chunk frames = [1232896,1233408) | 20260505 06:57:48.855 | INFO | MemInfo: { | 20260505 06:57:48.855 | INFO | "gpu" : " 17 GB", | 20260505 06:57:48.855 | INFO | "huge" : "", | 20260505 06:57:48.855 | INFO | "huge_shared" : " 27 GB", | 20260505 06:57:48.855 | INFO | "malloc" : " 14 GB", | 20260505 06:57:48.855 | INFO | "pinned" : " 29 GB" | 20260505 06:57:48.855 | INFO | } | 20260505 06:57:48.855 | INFO | Perf report for the last 5.13915 seconds: | 20260505 06:57:48.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:57:48.855 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:48.855 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:48.855 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:48.856 | INFO | ANALYSIS : 64.5%, 2.9, 98.9ms, 0 | 20260505 06:57:48.856 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:48.856 | INFO | RT_METRICS : 47.0%, 1.0, 24.6ms, 0 | 20260505 06:57:48.856 | INFO | PRE_HQ : 63.4%, 1.9, 63.2ms, 0 | 20260505 06:57:48.856 | INFO | BAZWRITER : 64.8%, 4.1, 136.5ms, 0 | 20260505 06:57:48.856 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 06:57:48.856 | INFO | Uploaded 36GiB to GPU in 2.1508s (16.738GiB/s) | 20260505 06:57:48.856 | INFO | Downloaded 38GiB to GPU in 2.30783s (16.4657GiB/s) | 20260505 06:57:53.994 | INFO | Analyzing chunk frames = [1233408,1233920) | 20260505 06:57:53.995 | INFO | MemInfo: { | 20260505 06:57:53.995 | INFO | "gpu" : " 17 GB", | 20260505 06:57:53.995 | INFO | "huge" : "", | 20260505 06:57:53.995 | INFO | "huge_shared" : " 27 GB", | 20260505 06:57:53.995 | INFO | "malloc" : " 14 GB", | 20260505 06:57:53.995 | INFO | "pinned" : " 29 GB" | 20260505 06:57:53.995 | INFO | } | 20260505 06:57:53.995 | INFO | Perf report for the last 5.13916 seconds: | 20260505 06:57:53.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:57:53.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:53.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:53.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:53.995 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 06:57:53.995 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:53.995 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:53.995 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 06:57:53.995 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:53.995 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:53.995 | INFO | Uploaded 36GiB to GPU in 2.10593s (17.0946GiB/s) | 20260505 06:57:53.995 | INFO | Downloaded 34GiB to GPU in 2.03319s (16.7225GiB/s) | 20260505 06:57:59.134 | INFO | Analyzing chunk frames = [1233920,1234432) | 20260505 06:57:59.134 | INFO | MemInfo: { | 20260505 06:57:59.134 | INFO | "gpu" : " 17 GB", | 20260505 06:57:59.134 | INFO | "huge" : "", | 20260505 06:57:59.134 | INFO | "huge_shared" : " 27 GB", | 20260505 06:57:59.134 | INFO | "malloc" : " 14 GB", | 20260505 06:57:59.134 | INFO | "pinned" : " 29 GB" | 20260505 06:57:59.134 | INFO | } | 20260505 06:57:59.134 | INFO | Perf report for the last 5.13982 seconds: | 20260505 06:57:59.134 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:57:59.134 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:59.134 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:59.134 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:59.135 | INFO | ANALYSIS : 63.6%, 3.0, 99.0ms, 0 | 20260505 06:57:59.135 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:59.135 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:59.135 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 06:57:59.135 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:59.135 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:57:59.135 | INFO | Uploaded 36GiB to GPU in 2.14051s (16.8184GiB/s) | 20260505 06:57:59.135 | INFO | Downloaded 34GiB to GPU in 2.04611s (16.6169GiB/s) | 20260505 06:58:04.273 | INFO | Analyzing chunk frames = [1234432,1234944) | 20260505 06:58:04.273 | INFO | MemInfo: { | 20260505 06:58:04.273 | INFO | "gpu" : " 17 GB", | 20260505 06:58:04.273 | INFO | "huge" : "", | 20260505 06:58:04.273 | INFO | "huge_shared" : " 27 GB", | 20260505 06:58:04.274 | INFO | "malloc" : " 14 GB", | 20260505 06:58:04.274 | INFO | "pinned" : " 29 GB" | 20260505 06:58:04.274 | INFO | } | 20260505 06:58:04.274 | INFO | Perf report for the last 5.13925 seconds: | 20260505 06:58:04.274 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:58:04.274 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:04.274 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:04.274 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:04.274 | INFO | ANALYSIS : 63.5%, 3.0, 98.6ms, 0 | 20260505 06:58:04.274 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:04.274 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:04.274 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 06:58:04.274 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:04.274 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:04.274 | INFO | Uploaded 36GiB to GPU in 2.12875s (16.9114GiB/s) | 20260505 06:58:04.274 | INFO | Downloaded 34GiB to GPU in 2.02883s (16.7584GiB/s) | 20260505 06:58:09.413 | INFO | Analyzing chunk frames = [1234944,1235456) | 20260505 06:58:09.413 | INFO | MemInfo: { | 20260505 06:58:09.413 | INFO | "gpu" : " 17 GB", | 20260505 06:58:09.413 | INFO | "huge" : "", | 20260505 06:58:09.413 | INFO | "huge_shared" : " 27 GB", | 20260505 06:58:09.413 | INFO | "malloc" : " 14 GB", | 20260505 06:58:09.413 | INFO | "pinned" : " 29 GB" | 20260505 06:58:09.413 | INFO | } | 20260505 06:58:09.413 | INFO | Perf report for the last 5.13892 seconds: | 20260505 06:58:09.413 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:58:09.413 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:09.413 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:09.413 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:09.413 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 06:58:09.413 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:09.413 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:09.413 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 06:58:09.413 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:09.413 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:09.413 | INFO | Uploaded 36GiB to GPU in 2.12353s (16.9529GiB/s) | 20260505 06:58:09.413 | INFO | Downloaded 34GiB to GPU in 2.02827s (16.7631GiB/s) | 20260505 06:58:14.554 | INFO | Analyzing chunk frames = [1235456,1235968) | 20260505 06:58:14.554 | INFO | MemInfo: { | 20260505 06:58:14.554 | INFO | "gpu" : " 17 GB", | 20260505 06:58:14.554 | INFO | "huge" : "", | 20260505 06:58:14.554 | INFO | "huge_shared" : " 27 GB", | 20260505 06:58:14.554 | INFO | "malloc" : " 14 GB", | 20260505 06:58:14.554 | INFO | "pinned" : " 29 GB" | 20260505 06:58:14.554 | INFO | } | 20260505 06:58:14.554 | INFO | Perf report for the last 5.14117 seconds: | 20260505 06:58:14.554 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:58:14.554 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:14.554 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:14.554 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:14.554 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 06:58:14.554 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:14.554 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:14.554 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 06:58:14.554 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:14.554 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:14.554 | INFO | Uploaded 36GiB to GPU in 2.11602s (17.0131GiB/s) | 20260505 06:58:14.554 | INFO | Downloaded 34GiB to GPU in 2.03872s (16.6771GiB/s) | 20260505 06:58:19.691 | INFO | Analyzing chunk frames = [1235968,1236480) | 20260505 06:58:19.691 | INFO | MemInfo: { | 20260505 06:58:19.691 | INFO | "gpu" : " 17 GB", | 20260505 06:58:19.691 | INFO | "huge" : "", | 20260505 06:58:19.691 | INFO | "huge_shared" : " 27 GB", | 20260505 06:58:19.691 | INFO | "malloc" : " 14 GB", | 20260505 06:58:19.691 | INFO | "pinned" : " 29 GB" | 20260505 06:58:19.691 | INFO | } | 20260505 06:58:19.691 | INFO | Perf report for the last 5.13768 seconds: | 20260505 06:58:19.691 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:58:19.691 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:19.692 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:19.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:19.692 | INFO | ANALYSIS : 64.9%, 3.0, 101.3ms, 0 | 20260505 06:58:19.692 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:19.692 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:19.692 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260505 06:58:19.692 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:19.692 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:19.692 | INFO | Uploaded 36GiB to GPU in 2.2492s (16.0057GiB/s) | 20260505 06:58:19.692 | INFO | Downloaded 34GiB to GPU in 2.15465s (15.7798GiB/s) | 20260505 06:58:24.830 | INFO | Analyzing chunk frames = [1236480,1236992) | 20260505 06:58:24.830 | INFO | MemInfo: { | 20260505 06:58:24.831 | INFO | "gpu" : " 17 GB", | 20260505 06:58:24.831 | INFO | "huge" : "", | 20260505 06:58:24.831 | INFO | "huge_shared" : " 27 GB", | 20260505 06:58:24.831 | INFO | "malloc" : " 14 GB", | 20260505 06:58:24.831 | INFO | "pinned" : " 29 GB" | 20260505 06:58:24.831 | INFO | } | 20260505 06:58:24.831 | INFO | Perf report for the last 5.13926 seconds: | 20260505 06:58:24.831 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:58:24.831 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:24.831 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:24.831 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:24.831 | INFO | ANALYSIS : 64.3%, 3.0, 100.3ms, 0 | 20260505 06:58:24.831 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:24.831 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:24.831 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 06:58:24.831 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:24.831 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:24.831 | INFO | Uploaded 36GiB to GPU in 2.16652s (16.6165GiB/s) | 20260505 06:58:24.831 | INFO | Downloaded 34GiB to GPU in 2.04429s (16.6317GiB/s) | 20260505 06:58:24.831 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 06:58:24.831 | INFO | Total mem usage: 15.3 GiB | 20260505 06:58:24.831 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 06:58:24.831 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.75%) | 20260505 06:58:24.831 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 06:58:24.831 | INFO | High Water Breakdown: | 20260505 06:58:24.831 | INFO | 60.4% : 8.68 GiB ( 92.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 06:58:24.831 | INFO | 39.4% : 5.66 GiB ( 85.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 06:58:24.831 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 06:58:24.831 | INFO | ------------------------------------------ | 20260505 06:58:24.831 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:58:24.831 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 06:58:24.831 | INFO | Total mem usage: 26.5 GiB | 20260505 06:58:24.831 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 06:58:24.831 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 06:58:24.831 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 06:58:24.831 | INFO | High Water Breakdown: | 20260505 06:58:24.832 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:58:24.832 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:58:24.832 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:58:24.832 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:58:24.832 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:58:24.832 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:58:24.832 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 06:58:24.832 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 06:58:24.832 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:58:24.832 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 06:58:24.832 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 06:58:24.832 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 06:58:24.832 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 06:58:24.832 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 06:58:24.832 | INFO | ------------------------------------------ | 20260505 06:58:24.832 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:58:24.832 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 06:58:24.832 | INFO | Total mem usage: 25.6 GiB | 20260505 06:58:24.832 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 06:58:24.832 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 06:58:24.832 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 06:58:24.832 | INFO | High Water Breakdown: | 20260505 06:58:24.832 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 06:58:24.832 | INFO | ------------------------------------------ | 20260505 06:58:24.832 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:58:24.832 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 06:58:24.832 | INFO | Total mem usage: 15.7 GiB | 20260505 06:58:24.832 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 06:58:24.832 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 06:58:24.832 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 06:58:24.832 | INFO | High Water Breakdown: | 20260505 06:58:24.832 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 06:58:24.832 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 06:58:24.832 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 06:58:24.832 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 06:58:24.832 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 06:58:24.832 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 06:58:24.832 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 06:58:24.832 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 06:58:24.832 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 06:58:24.832 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:58:24.832 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 06:58:24.832 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:58:24.832 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 06:58:24.832 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 06:58:24.832 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 06:58:24.832 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 06:58:24.832 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:58:24.832 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 06:58:24.832 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:58:24.832 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:58:24.832 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:58:24.832 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 06:58:24.832 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 06:58:24.832 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:58:24.832 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 06:58:24.832 | INFO | ------------------------------------------ | 20260505 06:58:24.832 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:58:24.832 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 06:58:24.832 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:58:24.832 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:58:24.832 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.52e+01, 1.37e+01 / 1.66e+00 | 20260505 06:58:24.832 | INFO | PerfReport: 0.28%, AggregateStats -- 9.38e-02 / 1.04e-01, 1.08e-02 / 6.12e-03 | 20260505 06:58:24.832 | INFO | PerfReport: 11.58%, Baseline -- 3.83e+00 / 4.44e+00, 6.36e-01 / 2.94e-01 | 20260505 06:58:24.832 | INFO | PerfReport: 3.31%, Histogram -- 1.09e+00 / 1.46e+00, 2.16e-01 / 9.05e-02 | 20260505 06:58:24.832 | INFO | PerfReport: 15.48%, DME -- 4.09e+01 / 4.47e+01, 4.02e+00 / 1.21e+00 | 20260505 06:58:24.832 | INFO | PerfReport: 56.21%, FrameLabeling -- 1.86e+01 / 1.91e+01, 2.10e+00 / 8.83e-01 | 20260505 06:58:24.832 | INFO | PerfReport: 7.63%, PulseAccumulating -- 2.52e+00 / 3.08e+00, 4.46e-01 / 2.80e-01 | 20260505 06:58:24.832 | INFO | PerfReport: 4.03%, Metrics -- 1.33e+00 / 1.69e+00, 3.82e-01 / 2.14e-01 | 20260505 06:58:24.832 | INFO | PerfReport: 1.46%, PulseSerializing -- 4.83e-01 / 5.57e-01, 9.40e-02 / 3.95e-02 | 20260505 06:58:24.832 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 06:58:24.832 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:58:24.832 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:58:24.832 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.21e+01 / 0.00e+00 | 20260505 06:58:24.832 | INFO | PerfReport: 22.07%, Upload -- 2.24e+01 / 0.00e+00, 7.90e+00 / 0.00e+00 | 20260505 06:58:24.832 | INFO | PerfReport: 21.41%, Download -- 2.17e+01 / 0.00e+00, 7.77e+00 / 0.00e+00 | 20260505 06:58:24.832 | INFO | PerfReport: 32.54%, Compute -- 3.30e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 06:58:24.832 | INFO | PerfReport: 1.49%, LockUpload -- 1.51e+00 / 0.00e+00, 6.98e+00 / 0.00e+00 | 20260505 06:58:24.832 | INFO | PerfReport: 0.79%, LockDownload -- 8.01e-01 / 0.00e+00, 2.21e+00 / 0.00e+00 | 20260505 06:58:24.832 | INFO | PerfReport: 21.69%, LockCompute -- 2.20e+01 / 0.00e+00, 1.67e+01 / 0.00e+00 | 20260505 06:58:24.832 | INFO | PerfReport: 0.01%, Misc -- 1.49e-02 / 0.00e+00, 2.17e-02 / 0.00e+00 | 20260505 06:58:24.950 | INFO | Starting a round of Baz IO | 20260505 06:58:28.193 | INFO | Populating RTMetrics report | 20260505 06:58:28.193 | INFO | RTMetrics report populated | 20260505 06:58:28.193 | INFO | Opening tmp RTMetrics json file | 20260505 06:58:28.215 | INFO | Serializing RTMetrics to json | 20260505 06:58:28.217 | INFO | Serialization done, swapping tmp file now... | 20260505 06:58:28.218 | INFO | RTMetrics done | 20260505 06:58:28.311 | INFO | Finished a round of Baz IO: Wrote 2.413 GiB in 3.361 seconds | 20260505 06:58:28.312 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 31.34% / 0.04% / 68.63% | 20260505 06:58:28.312 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.28089% / 3.17891e-05% / 5.16574e-05% | 20260505 06:58:28.312 | INFO | Baz IO overall: | 20260505 06:58:28.312 | INFO | BytesWritten : 658814474352 | 20260505 06:58:28.312 | INFO | NumEvents : 312128045319 | 20260505 06:58:29.970 | INFO | Analyzing chunk frames = [1236992,1237504) | 20260505 06:58:29.970 | INFO | MemInfo: { | 20260505 06:58:29.970 | INFO | "gpu" : " 17 GB", | 20260505 06:58:29.970 | INFO | "huge" : "", | 20260505 06:58:29.970 | INFO | "huge_shared" : " 27 GB", | 20260505 06:58:29.970 | INFO | "malloc" : " 14 GB", | 20260505 06:58:29.970 | INFO | "pinned" : " 29 GB" | 20260505 06:58:29.970 | INFO | } | 20260505 06:58:29.970 | INFO | Perf report for the last 5.13936 seconds: | 20260505 06:58:29.970 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:58:29.970 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:29.970 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:29.970 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:29.970 | INFO | ANALYSIS : 64.3%, 2.9, 98.3ms, 0 | 20260505 06:58:29.970 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:29.970 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260505 06:58:29.970 | INFO | PRE_HQ : 63.2%, 1.9, 63.0ms, 0 | 20260505 06:58:29.970 | INFO | BAZWRITER : 64.7%, 4.1, 137.4ms, 0 | 20260505 06:58:29.970 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 06:58:29.970 | INFO | Uploaded 36GiB to GPU in 2.1217s (16.9676GiB/s) | 20260505 06:58:29.971 | INFO | Downloaded 38GiB to GPU in 2.30377s (16.4947GiB/s) | 20260505 06:58:35.110 | INFO | Analyzing chunk frames = [1237504,1238016) | 20260505 06:58:35.110 | INFO | MemInfo: { | 20260505 06:58:35.110 | INFO | "gpu" : " 17 GB", | 20260505 06:58:35.110 | INFO | "huge" : "", | 20260505 06:58:35.110 | INFO | "huge_shared" : " 27 GB", | 20260505 06:58:35.110 | INFO | "malloc" : " 14 GB", | 20260505 06:58:35.110 | INFO | "pinned" : " 29 GB" | 20260505 06:58:35.110 | INFO | } | 20260505 06:58:35.110 | INFO | Perf report for the last 5.13963 seconds: | 20260505 06:58:35.110 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:58:35.110 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:35.110 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:35.110 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:35.110 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 06:58:35.110 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:35.110 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:35.110 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 06:58:35.110 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:35.110 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:35.110 | INFO | Uploaded 36GiB to GPU in 2.09238s (17.2053GiB/s) | 20260505 06:58:35.110 | INFO | Downloaded 34GiB to GPU in 2.03691s (16.692GiB/s) | 20260505 06:58:40.249 | INFO | Analyzing chunk frames = [1238016,1238528) | 20260505 06:58:40.249 | INFO | MemInfo: { | 20260505 06:58:40.249 | INFO | "gpu" : " 17 GB", | 20260505 06:58:40.249 | INFO | "huge" : "", | 20260505 06:58:40.249 | INFO | "huge_shared" : " 27 GB", | 20260505 06:58:40.249 | INFO | "malloc" : " 14 GB", | 20260505 06:58:40.249 | INFO | "pinned" : " 29 GB" | 20260505 06:58:40.249 | INFO | } | 20260505 06:58:40.249 | INFO | Perf report for the last 5.13923 seconds: | 20260505 06:58:40.249 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:58:40.249 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:40.249 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:40.249 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:40.249 | INFO | ANALYSIS : 63.6%, 3.0, 99.0ms, 0 | 20260505 06:58:40.249 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:40.249 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:40.249 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 06:58:40.249 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:40.249 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:40.249 | INFO | Uploaded 36GiB to GPU in 2.154s (16.7131GiB/s) | 20260505 06:58:40.249 | INFO | Downloaded 34GiB to GPU in 2.0491s (16.5927GiB/s) | 20260505 06:58:45.388 | INFO | Analyzing chunk frames = [1238528,1239040) | 20260505 06:58:45.388 | INFO | MemInfo: { | 20260505 06:58:45.388 | INFO | "gpu" : " 17 GB", | 20260505 06:58:45.388 | INFO | "huge" : "", | 20260505 06:58:45.388 | INFO | "huge_shared" : " 27 GB", | 20260505 06:58:45.388 | INFO | "malloc" : " 14 GB", | 20260505 06:58:45.388 | INFO | "pinned" : " 29 GB" | 20260505 06:58:45.388 | INFO | } | 20260505 06:58:45.388 | INFO | Perf report for the last 5.13935 seconds: | 20260505 06:58:45.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:58:45.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:45.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:45.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:45.388 | INFO | ANALYSIS : 63.6%, 3.0, 98.8ms, 0 | 20260505 06:58:45.388 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:45.389 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:45.389 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 06:58:45.389 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:45.389 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:45.389 | INFO | Uploaded 36GiB to GPU in 2.15876s (16.6763GiB/s) | 20260505 06:58:45.389 | INFO | Downloaded 34GiB to GPU in 2.03146s (16.7367GiB/s) | 20260505 06:58:50.528 | INFO | Analyzing chunk frames = [1239040,1239552) | 20260505 06:58:50.528 | INFO | MemInfo: { | 20260505 06:58:50.528 | INFO | "gpu" : " 17 GB", | 20260505 06:58:50.528 | INFO | "huge" : "", | 20260505 06:58:50.528 | INFO | "huge_shared" : " 27 GB", | 20260505 06:58:50.528 | INFO | "malloc" : " 14 GB", | 20260505 06:58:50.528 | INFO | "pinned" : " 29 GB" | 20260505 06:58:50.528 | INFO | } | 20260505 06:58:50.528 | INFO | Perf report for the last 5.13956 seconds: | 20260505 06:58:50.528 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:58:50.528 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:50.528 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:50.528 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:50.528 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 06:58:50.528 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:50.528 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:50.528 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 06:58:50.528 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:50.528 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:50.528 | INFO | Uploaded 36GiB to GPU in 2.16497s (16.6284GiB/s) | 20260505 06:58:50.528 | INFO | Downloaded 34GiB to GPU in 2.04678s (16.6114GiB/s) | 20260505 06:58:55.668 | INFO | Analyzing chunk frames = [1239552,1240064) | 20260505 06:58:55.668 | INFO | MemInfo: { | 20260505 06:58:55.668 | INFO | "gpu" : " 17 GB", | 20260505 06:58:55.668 | INFO | "huge" : "", | 20260505 06:58:55.668 | INFO | "huge_shared" : " 27 GB", | 20260505 06:58:55.668 | INFO | "malloc" : " 14 GB", | 20260505 06:58:55.669 | INFO | "pinned" : " 29 GB" | 20260505 06:58:55.669 | INFO | } | 20260505 06:58:55.669 | INFO | Perf report for the last 5.1406 seconds: | 20260505 06:58:55.669 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:58:55.669 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:55.669 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:55.669 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:55.669 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 06:58:55.669 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:55.669 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:55.669 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 06:58:55.669 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:55.669 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:58:55.669 | INFO | Uploaded 36GiB to GPU in 2.15055s (16.7399GiB/s) | 20260505 06:58:55.669 | INFO | Downloaded 34GiB to GPU in 2.04024s (16.6647GiB/s) | 20260505 06:59:00.806 | INFO | Analyzing chunk frames = [1240064,1240576) | 20260505 06:59:00.806 | INFO | MemInfo: { | 20260505 06:59:00.806 | INFO | "gpu" : " 17 GB", | 20260505 06:59:00.806 | INFO | "huge" : "", | 20260505 06:59:00.806 | INFO | "huge_shared" : " 27 GB", | 20260505 06:59:00.806 | INFO | "malloc" : " 14 GB", | 20260505 06:59:00.806 | INFO | "pinned" : " 29 GB" | 20260505 06:59:00.806 | INFO | } | 20260505 06:59:00.806 | INFO | Perf report for the last 5.13753 seconds: | 20260505 06:59:00.806 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:59:00.806 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:00.806 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:00.806 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:00.806 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 06:59:00.806 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:00.806 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:00.806 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 06:59:00.806 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:00.806 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:00.807 | INFO | Uploaded 36GiB to GPU in 2.21534s (16.2503GiB/s) | 20260505 06:59:00.807 | INFO | Downloaded 34GiB to GPU in 2.16813s (15.6817GiB/s) | 20260505 06:59:05.946 | INFO | Analyzing chunk frames = [1240576,1241088) | 20260505 06:59:05.946 | INFO | MemInfo: { | 20260505 06:59:05.946 | INFO | "gpu" : " 17 GB", | 20260505 06:59:05.946 | INFO | "huge" : "", | 20260505 06:59:05.946 | INFO | "huge_shared" : " 27 GB", | 20260505 06:59:05.946 | INFO | "malloc" : " 14 GB", | 20260505 06:59:05.946 | INFO | "pinned" : " 29 GB" | 20260505 06:59:05.946 | INFO | } | 20260505 06:59:05.946 | INFO | Perf report for the last 5.13974 seconds: | 20260505 06:59:05.946 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:59:05.946 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:05.946 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:05.946 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:05.946 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 06:59:05.946 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:05.946 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:05.946 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 06:59:05.946 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:05.946 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:05.946 | INFO | Uploaded 36GiB to GPU in 2.16655s (16.6163GiB/s) | 20260505 06:59:05.946 | INFO | Downloaded 34GiB to GPU in 2.05614s (16.5358GiB/s) | 20260505 06:59:06.063 | INFO | Starting a round of Baz IO | 20260505 06:59:09.364 | INFO | Populating RTMetrics report | 20260505 06:59:09.364 | INFO | RTMetrics report populated | 20260505 06:59:09.364 | INFO | Opening tmp RTMetrics json file | 20260505 06:59:09.365 | INFO | Serializing RTMetrics to json | 20260505 06:59:09.366 | INFO | Serialization done, swapping tmp file now... | 20260505 06:59:09.367 | INFO | RTMetrics done | 20260505 06:59:09.489 | INFO | Finished a round of Baz IO: Wrote 2.411 GiB in 3.425 seconds | 20260505 06:59:09.489 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 31.30% / 0.04% / 68.66% | 20260505 06:59:09.489 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.25324% / 6.35783e-05% / 6.75519e-05% | 20260505 06:59:09.489 | INFO | Baz IO overall: | 20260505 06:59:09.489 | INFO | BytesWritten : 661403572217 | 20260505 06:59:09.489 | INFO | NumEvents : 313367146316 | 20260505 06:59:11.085 | INFO | Analyzing chunk frames = [1241088,1241600) | 20260505 06:59:11.085 | INFO | MemInfo: { | 20260505 06:59:11.085 | INFO | "gpu" : " 17 GB", | 20260505 06:59:11.085 | INFO | "huge" : "", | 20260505 06:59:11.085 | INFO | "huge_shared" : " 27 GB", | 20260505 06:59:11.085 | INFO | "malloc" : " 14 GB", | 20260505 06:59:11.085 | INFO | "pinned" : " 29 GB" | 20260505 06:59:11.085 | INFO | } | 20260505 06:59:11.086 | INFO | Perf report for the last 5.13968 seconds: | 20260505 06:59:11.086 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:59:11.086 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:11.086 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:11.086 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:11.086 | INFO | ANALYSIS : 65.3%, 2.9, 99.5ms, 0 | 20260505 06:59:11.086 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:11.086 | INFO | RT_METRICS : 46.2%, 1.0, 24.2ms, 0 | 20260505 06:59:11.086 | INFO | PRE_HQ : 64.0%, 1.9, 63.4ms, 0 | 20260505 06:59:11.086 | INFO | BAZWRITER : 65.9%, 4.1, 138.1ms, 0 | 20260505 06:59:11.086 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 06:59:11.086 | INFO | Uploaded 36GiB to GPU in 2.24731s (16.0192GiB/s) | 20260505 06:59:11.086 | INFO | Downloaded 38GiB to GPU in 2.31376s (16.4235GiB/s) | 20260505 06:59:16.225 | INFO | Analyzing chunk frames = [1241600,1242112) | 20260505 06:59:16.225 | INFO | MemInfo: { | 20260505 06:59:16.225 | INFO | "gpu" : " 17 GB", | 20260505 06:59:16.225 | INFO | "huge" : "", | 20260505 06:59:16.225 | INFO | "huge_shared" : " 27 GB", | 20260505 06:59:16.225 | INFO | "malloc" : " 14 GB", | 20260505 06:59:16.225 | INFO | "pinned" : " 29 GB" | 20260505 06:59:16.225 | INFO | } | 20260505 06:59:16.225 | INFO | Perf report for the last 5.1393 seconds: | 20260505 06:59:16.225 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:59:16.225 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:16.225 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:16.225 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:16.225 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 06:59:16.225 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:16.225 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:16.225 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 06:59:16.225 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:16.225 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:16.225 | INFO | Uploaded 36GiB to GPU in 2.10673s (17.0881GiB/s) | 20260505 06:59:16.225 | INFO | Downloaded 34GiB to GPU in 2.03601s (16.6993GiB/s) | 20260505 06:59:21.364 | INFO | Analyzing chunk frames = [1242112,1242624) | 20260505 06:59:21.364 | INFO | MemInfo: { | 20260505 06:59:21.364 | INFO | "gpu" : " 17 GB", | 20260505 06:59:21.364 | INFO | "huge" : "", | 20260505 06:59:21.364 | INFO | "huge_shared" : " 27 GB", | 20260505 06:59:21.364 | INFO | "malloc" : " 14 GB", | 20260505 06:59:21.364 | INFO | "pinned" : " 29 GB" | 20260505 06:59:21.364 | INFO | } | 20260505 06:59:21.364 | INFO | Perf report for the last 5.13889 seconds: | 20260505 06:59:21.364 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:59:21.364 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:21.364 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:21.364 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:21.364 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260505 06:59:21.364 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:21.364 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:21.364 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 06:59:21.364 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:21.364 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:21.364 | INFO | Uploaded 36GiB to GPU in 2.15392s (16.7137GiB/s) | 20260505 06:59:21.364 | INFO | Downloaded 34GiB to GPU in 2.05273s (16.5633GiB/s) | 20260505 06:59:26.503 | INFO | Analyzing chunk frames = [1242624,1243136) | 20260505 06:59:26.503 | INFO | MemInfo: { | 20260505 06:59:26.503 | INFO | "gpu" : " 17 GB", | 20260505 06:59:26.503 | INFO | "huge" : "", | 20260505 06:59:26.503 | INFO | "huge_shared" : " 27 GB", | 20260505 06:59:26.503 | INFO | "malloc" : " 14 GB", | 20260505 06:59:26.503 | INFO | "pinned" : " 29 GB" | 20260505 06:59:26.503 | INFO | } | 20260505 06:59:26.503 | INFO | Perf report for the last 5.13951 seconds: | 20260505 06:59:26.503 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:59:26.503 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:26.503 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:26.503 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:26.503 | INFO | ANALYSIS : 63.4%, 3.0, 98.5ms, 0 | 20260505 06:59:26.503 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:26.503 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:26.503 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 06:59:26.503 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:26.503 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:26.504 | INFO | Uploaded 36GiB to GPU in 2.13384s (16.871GiB/s) | 20260505 06:59:26.504 | INFO | Downloaded 34GiB to GPU in 2.03074s (16.7427GiB/s) | 20260505 06:59:31.643 | INFO | Analyzing chunk frames = [1243136,1243648) | 20260505 06:59:31.643 | INFO | MemInfo: { | 20260505 06:59:31.643 | INFO | "gpu" : " 17 GB", | 20260505 06:59:31.643 | INFO | "huge" : "", | 20260505 06:59:31.643 | INFO | "huge_shared" : " 27 GB", | 20260505 06:59:31.643 | INFO | "malloc" : " 14 GB", | 20260505 06:59:31.643 | INFO | "pinned" : " 29 GB" | 20260505 06:59:31.643 | INFO | } | 20260505 06:59:31.643 | INFO | Perf report for the last 5.13933 seconds: | 20260505 06:59:31.643 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:59:31.643 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:31.643 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:31.643 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:31.643 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 06:59:31.643 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:31.643 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:31.643 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 06:59:31.643 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:31.643 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:31.643 | INFO | Uploaded 36GiB to GPU in 2.15378s (16.7148GiB/s) | 20260505 06:59:31.643 | INFO | Downloaded 34GiB to GPU in 2.04868s (16.596GiB/s) | 20260505 06:59:36.783 | INFO | Analyzing chunk frames = [1243648,1244160) | 20260505 06:59:36.783 | INFO | MemInfo: { | 20260505 06:59:36.783 | INFO | "gpu" : " 17 GB", | 20260505 06:59:36.783 | INFO | "huge" : "", | 20260505 06:59:36.783 | INFO | "huge_shared" : " 27 GB", | 20260505 06:59:36.783 | INFO | "malloc" : " 14 GB", | 20260505 06:59:36.783 | INFO | "pinned" : " 29 GB" | 20260505 06:59:36.783 | INFO | } | 20260505 06:59:36.783 | INFO | Perf report for the last 5.14064 seconds: | 20260505 06:59:36.783 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:59:36.783 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:36.783 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:36.783 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:36.783 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 06:59:36.783 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:36.783 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:36.784 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 06:59:36.784 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:36.784 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:36.784 | INFO | Uploaded 36GiB to GPU in 2.13891s (16.831GiB/s) | 20260505 06:59:36.784 | INFO | Downloaded 34GiB to GPU in 2.03882s (16.6763GiB/s) | 20260505 06:59:41.921 | INFO | Analyzing chunk frames = [1244160,1244672) | 20260505 06:59:41.921 | INFO | MemInfo: { | 20260505 06:59:41.921 | INFO | "gpu" : " 17 GB", | 20260505 06:59:41.921 | INFO | "huge" : "", | 20260505 06:59:41.921 | INFO | "huge_shared" : " 27 GB", | 20260505 06:59:41.921 | INFO | "malloc" : " 14 GB", | 20260505 06:59:41.921 | INFO | "pinned" : " 29 GB" | 20260505 06:59:41.921 | INFO | } | 20260505 06:59:41.921 | INFO | Perf report for the last 5.13767 seconds: | 20260505 06:59:41.921 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:59:41.921 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:41.921 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:41.921 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:41.921 | INFO | ANALYSIS : 64.5%, 3.0, 100.7ms, 0 | 20260505 06:59:41.921 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:41.921 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:41.921 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 06:59:41.921 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:41.921 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:41.921 | INFO | Uploaded 36GiB to GPU in 2.20857s (16.3002GiB/s) | 20260505 06:59:41.921 | INFO | Downloaded 34GiB to GPU in 2.16107s (15.733GiB/s) | 20260505 06:59:47.061 | INFO | Analyzing chunk frames = [1244672,1245184) | 20260505 06:59:47.061 | INFO | MemInfo: { | 20260505 06:59:47.061 | INFO | "gpu" : " 17 GB", | 20260505 06:59:47.061 | INFO | "huge" : "", | 20260505 06:59:47.061 | INFO | "huge_shared" : " 27 GB", | 20260505 06:59:47.061 | INFO | "malloc" : " 14 GB", | 20260505 06:59:47.061 | INFO | "pinned" : " 29 GB" | 20260505 06:59:47.061 | INFO | } | 20260505 06:59:47.061 | INFO | Perf report for the last 5.14011 seconds: | 20260505 06:59:47.061 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:59:47.061 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:47.061 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:47.061 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:47.061 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 06:59:47.061 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:47.061 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:47.061 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 06:59:47.061 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:47.061 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:47.061 | INFO | Uploaded 36GiB to GPU in 2.15466s (16.7079GiB/s) | 20260505 06:59:47.061 | INFO | Downloaded 34GiB to GPU in 2.04375s (16.6361GiB/s) | 20260505 06:59:47.062 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 06:59:47.062 | INFO | Total mem usage: 15.3 GiB | 20260505 06:59:47.062 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 06:59:47.062 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.68%) | 20260505 06:59:47.062 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 06:59:47.062 | INFO | High Water Breakdown: | 20260505 06:59:47.062 | INFO | 60.5% : 8.69 GiB ( 92.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 06:59:47.062 | INFO | 39.4% : 5.66 GiB ( 85.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 06:59:47.062 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 06:59:47.062 | INFO | ------------------------------------------ | 20260505 06:59:47.062 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:59:47.062 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 06:59:47.062 | INFO | Total mem usage: 26.5 GiB | 20260505 06:59:47.062 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 06:59:47.062 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 06:59:47.062 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 06:59:47.062 | INFO | High Water Breakdown: | 20260505 06:59:47.062 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:59:47.062 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:59:47.062 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:59:47.062 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:59:47.062 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:59:47.062 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:59:47.062 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 06:59:47.062 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 06:59:47.062 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:59:47.062 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 06:59:47.062 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 06:59:47.062 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 06:59:47.062 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 06:59:47.062 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 06:59:47.062 | INFO | ------------------------------------------ | 20260505 06:59:47.062 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:59:47.062 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 06:59:47.062 | INFO | Total mem usage: 25.6 GiB | 20260505 06:59:47.062 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 06:59:47.062 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 06:59:47.062 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 06:59:47.062 | INFO | High Water Breakdown: | 20260505 06:59:47.062 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 06:59:47.062 | INFO | ------------------------------------------ | 20260505 06:59:47.062 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:59:47.062 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 06:59:47.062 | INFO | Total mem usage: 15.7 GiB | 20260505 06:59:47.062 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 06:59:47.062 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 06:59:47.062 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 06:59:47.062 | INFO | High Water Breakdown: | 20260505 06:59:47.062 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 06:59:47.062 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 06:59:47.062 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 06:59:47.062 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 06:59:47.063 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 06:59:47.063 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 06:59:47.063 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 06:59:47.063 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 06:59:47.063 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 06:59:47.063 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 06:59:47.063 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 06:59:47.063 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 06:59:47.063 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 06:59:47.063 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 06:59:47.063 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 06:59:47.063 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 06:59:47.063 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 06:59:47.063 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 06:59:47.063 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 06:59:47.063 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 06:59:47.063 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 06:59:47.063 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 06:59:47.063 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 06:59:47.063 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 06:59:47.063 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 06:59:47.063 | INFO | ------------------------------------------ | 20260505 06:59:47.063 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 06:59:47.063 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 06:59:47.063 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:59:47.063 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:59:47.063 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.51e+01, 1.36e+01 / 1.47e+00 | 20260505 06:59:47.063 | INFO | PerfReport: 0.28%, AggregateStats -- 9.40e-02 / 1.02e-01, 1.10e-02 / 4.82e-03 | 20260505 06:59:47.063 | INFO | PerfReport: 11.61%, Baseline -- 3.83e+00 / 4.45e+00, 6.36e-01 / 3.08e-01 | 20260505 06:59:47.063 | INFO | PerfReport: 3.32%, Histogram -- 1.09e+00 / 1.47e+00, 2.15e-01 / 1.02e-01 | 20260505 06:59:47.063 | INFO | PerfReport: 15.45%, DME -- 4.08e+01 / 4.45e+01, 4.15e+00 / 1.08e+00 | 20260505 06:59:47.063 | INFO | PerfReport: 56.18%, FrameLabeling -- 1.85e+01 / 1.91e+01, 2.05e+00 / 1.19e+00 | 20260505 06:59:47.063 | INFO | PerfReport: 7.66%, PulseAccumulating -- 2.53e+00 / 3.12e+00, 4.50e-01 / 2.82e-01 | 20260505 06:59:47.063 | INFO | PerfReport: 4.04%, Metrics -- 1.33e+00 / 1.73e+00, 3.74e-01 / 1.86e-01 | 20260505 06:59:47.063 | INFO | PerfReport: 1.46%, PulseSerializing -- 4.83e-01 / 5.56e-01, 9.39e-02 / 3.29e-02 | 20260505 06:59:47.063 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 06:59:47.063 | INFO | PerfReport: Gathered 1536 samples. | 20260505 06:59:47.063 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 06:59:47.063 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.23e+01 / 0.00e+00 | 20260505 06:59:47.063 | INFO | PerfReport: 22.15%, Upload -- 2.25e+01 / 0.00e+00, 7.93e+00 / 0.00e+00 | 20260505 06:59:47.063 | INFO | PerfReport: 21.47%, Download -- 2.18e+01 / 0.00e+00, 7.80e+00 / 0.00e+00 | 20260505 06:59:47.063 | INFO | PerfReport: 32.52%, Compute -- 3.30e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 06:59:47.063 | INFO | PerfReport: 1.46%, LockUpload -- 1.48e+00 / 0.00e+00, 6.93e+00 / 0.00e+00 | 20260505 06:59:47.063 | INFO | PerfReport: 0.78%, LockDownload -- 7.94e-01 / 0.00e+00, 2.13e+00 / 0.00e+00 | 20260505 06:59:47.063 | INFO | PerfReport: 21.60%, LockCompute -- 2.19e+01 / 0.00e+00, 1.68e+01 / 0.00e+00 | 20260505 06:59:47.063 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 1.94e-02 / 0.00e+00 | 20260505 06:59:47.177 | INFO | Starting a round of Baz IO | 20260505 06:59:50.463 | INFO | Populating RTMetrics report | 20260505 06:59:50.464 | INFO | RTMetrics report populated | 20260505 06:59:50.464 | INFO | Opening tmp RTMetrics json file | 20260505 06:59:50.486 | INFO | Serializing RTMetrics to json | 20260505 06:59:50.496 | INFO | Serialization done, swapping tmp file now... | 20260505 06:59:50.513 | INFO | RTMetrics done | 20260505 06:59:50.587 | INFO | Finished a round of Baz IO: Wrote 2.411 GiB in 3.409 seconds | 20260505 06:59:50.587 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 31.26% / 0.04% / 68.70% | 20260505 06:59:50.587 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.23418% / 7.15256e-05% / 6.75519e-05% | 20260505 06:59:50.587 | INFO | Baz IO overall: | 20260505 06:59:50.587 | INFO | BytesWritten : 663991913260 | 20260505 06:59:50.587 | INFO | NumEvents : 314605487020 | 20260505 06:59:52.200 | INFO | Analyzing chunk frames = [1245184,1245696) | 20260505 06:59:52.200 | INFO | MemInfo: { | 20260505 06:59:52.200 | INFO | "gpu" : " 17 GB", | 20260505 06:59:52.200 | INFO | "huge" : "", | 20260505 06:59:52.200 | INFO | "huge_shared" : " 27 GB", | 20260505 06:59:52.200 | INFO | "malloc" : " 14 GB", | 20260505 06:59:52.200 | INFO | "pinned" : " 29 GB" | 20260505 06:59:52.200 | INFO | } | 20260505 06:59:52.200 | INFO | Perf report for the last 5.13868 seconds: | 20260505 06:59:52.200 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:59:52.200 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:52.200 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:52.200 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:52.200 | INFO | ANALYSIS : 65.1%, 2.9, 99.4ms, 0 | 20260505 06:59:52.200 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:52.200 | INFO | RT_METRICS : 46.9%, 1.0, 24.6ms, 0 | 20260505 06:59:52.200 | INFO | PRE_HQ : 64.0%, 1.9, 63.5ms, 0 | 20260505 06:59:52.200 | INFO | BAZWRITER : 65.6%, 4.1, 137.7ms, 0 | 20260505 06:59:52.200 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 06:59:52.200 | INFO | Uploaded 36GiB to GPU in 2.25538s (15.9618GiB/s) | 20260505 06:59:52.200 | INFO | Downloaded 38GiB to GPU in 2.31779s (16.3949GiB/s) | 20260505 06:59:57.339 | INFO | Analyzing chunk frames = [1245696,1246208) | 20260505 06:59:57.339 | INFO | MemInfo: { | 20260505 06:59:57.339 | INFO | "gpu" : " 17 GB", | 20260505 06:59:57.339 | INFO | "huge" : "", | 20260505 06:59:57.339 | INFO | "huge_shared" : " 27 GB", | 20260505 06:59:57.339 | INFO | "malloc" : " 14 GB", | 20260505 06:59:57.339 | INFO | "pinned" : " 29 GB" | 20260505 06:59:57.339 | INFO | } | 20260505 06:59:57.339 | INFO | Perf report for the last 5.13959 seconds: | 20260505 06:59:57.339 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 06:59:57.339 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:57.339 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:57.339 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:57.339 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 06:59:57.339 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:57.339 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:57.339 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 06:59:57.339 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:57.339 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 06:59:57.340 | INFO | Uploaded 36GiB to GPU in 2.0982s (17.1576GiB/s) | 20260505 06:59:57.340 | INFO | Downloaded 34GiB to GPU in 2.0344s (16.7125GiB/s) | 20260505 07:00:02.478 | INFO | Analyzing chunk frames = [1246208,1246720) | 20260505 07:00:02.478 | INFO | MemInfo: { | 20260505 07:00:02.478 | INFO | "gpu" : " 17 GB", | 20260505 07:00:02.478 | INFO | "huge" : "", | 20260505 07:00:02.478 | INFO | "huge_shared" : " 27 GB", | 20260505 07:00:02.478 | INFO | "malloc" : " 14 GB", | 20260505 07:00:02.478 | INFO | "pinned" : " 29 GB" | 20260505 07:00:02.478 | INFO | } | 20260505 07:00:02.478 | INFO | Perf report for the last 5.13914 seconds: | 20260505 07:00:02.479 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:00:02.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:02.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:02.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:02.479 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260505 07:00:02.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:02.479 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:02.479 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 07:00:02.479 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:02.479 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:02.479 | INFO | Uploaded 36GiB to GPU in 2.13591s (16.8546GiB/s) | 20260505 07:00:02.479 | INFO | Downloaded 34GiB to GPU in 2.04967s (16.5881GiB/s) | 20260505 07:00:07.618 | INFO | Analyzing chunk frames = [1246720,1247232) | 20260505 07:00:07.618 | INFO | MemInfo: { | 20260505 07:00:07.618 | INFO | "gpu" : " 17 GB", | 20260505 07:00:07.618 | INFO | "huge" : "", | 20260505 07:00:07.618 | INFO | "huge_shared" : " 27 GB", | 20260505 07:00:07.618 | INFO | "malloc" : " 14 GB", | 20260505 07:00:07.618 | INFO | "pinned" : " 29 GB" | 20260505 07:00:07.618 | INFO | } | 20260505 07:00:07.618 | INFO | Perf report for the last 5.13907 seconds: | 20260505 07:00:07.618 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:00:07.618 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:07.618 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:07.618 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:07.618 | INFO | ANALYSIS : 63.5%, 3.0, 98.6ms, 0 | 20260505 07:00:07.618 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:07.618 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:07.618 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 07:00:07.618 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:07.618 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:07.618 | INFO | Uploaded 36GiB to GPU in 2.13925s (16.8283GiB/s) | 20260505 07:00:07.618 | INFO | Downloaded 34GiB to GPU in 2.02864s (16.76GiB/s) | 20260505 07:00:12.757 | INFO | Analyzing chunk frames = [1247232,1247744) | 20260505 07:00:12.757 | INFO | MemInfo: { | 20260505 07:00:12.757 | INFO | "gpu" : " 17 GB", | 20260505 07:00:12.757 | INFO | "huge" : "", | 20260505 07:00:12.757 | INFO | "huge_shared" : " 27 GB", | 20260505 07:00:12.757 | INFO | "malloc" : " 14 GB", | 20260505 07:00:12.757 | INFO | "pinned" : " 29 GB" | 20260505 07:00:12.757 | INFO | } | 20260505 07:00:12.757 | INFO | Perf report for the last 5.13966 seconds: | 20260505 07:00:12.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:00:12.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:12.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:12.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:12.757 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 07:00:12.757 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:12.757 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:12.757 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:00:12.757 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:12.757 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:12.758 | INFO | Uploaded 36GiB to GPU in 2.16285s (16.6447GiB/s) | 20260505 07:00:12.758 | INFO | Downloaded 34GiB to GPU in 2.04711s (16.6088GiB/s) | 20260505 07:00:17.898 | INFO | Analyzing chunk frames = [1247744,1248256) | 20260505 07:00:17.898 | INFO | MemInfo: { | 20260505 07:00:17.898 | INFO | "gpu" : " 17 GB", | 20260505 07:00:17.898 | INFO | "huge" : "", | 20260505 07:00:17.898 | INFO | "huge_shared" : " 27 GB", | 20260505 07:00:17.898 | INFO | "malloc" : " 14 GB", | 20260505 07:00:17.898 | INFO | "pinned" : " 29 GB" | 20260505 07:00:17.898 | INFO | } | 20260505 07:00:17.898 | INFO | Perf report for the last 5.14109 seconds: | 20260505 07:00:17.898 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:00:17.898 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:17.898 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:17.898 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:17.898 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 07:00:17.898 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:17.898 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:17.898 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 07:00:17.898 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:17.898 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:17.899 | INFO | Uploaded 36GiB to GPU in 2.16476s (16.63GiB/s) | 20260505 07:00:17.899 | INFO | Downloaded 34GiB to GPU in 2.05679s (16.5306GiB/s) | 20260505 07:00:23.036 | INFO | Analyzing chunk frames = [1248256,1248768) | 20260505 07:00:23.036 | INFO | MemInfo: { | 20260505 07:00:23.036 | INFO | "gpu" : " 17 GB", | 20260505 07:00:23.036 | INFO | "huge" : "", | 20260505 07:00:23.036 | INFO | "huge_shared" : " 27 GB", | 20260505 07:00:23.036 | INFO | "malloc" : " 14 GB", | 20260505 07:00:23.036 | INFO | "pinned" : " 29 GB" | 20260505 07:00:23.036 | INFO | } | 20260505 07:00:23.036 | INFO | Perf report for the last 5.13746 seconds: | 20260505 07:00:23.036 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:00:23.036 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:23.036 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:23.036 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:23.036 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 07:00:23.036 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:23.036 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:23.036 | INFO | PRE_HQ : 31.9%, 1.0, 16.5ms, 0 | 20260505 07:00:23.036 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:23.036 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:23.036 | INFO | Uploaded 36GiB to GPU in 2.20949s (16.2933GiB/s) | 20260505 07:00:23.036 | INFO | Downloaded 34GiB to GPU in 2.16617s (15.6959GiB/s) | 20260505 07:00:28.175 | INFO | Analyzing chunk frames = [1248768,1249280) | 20260505 07:00:28.175 | INFO | MemInfo: { | 20260505 07:00:28.175 | INFO | "gpu" : " 17 GB", | 20260505 07:00:28.175 | INFO | "huge" : "", | 20260505 07:00:28.175 | INFO | "huge_shared" : " 27 GB", | 20260505 07:00:28.175 | INFO | "malloc" : " 14 GB", | 20260505 07:00:28.175 | INFO | "pinned" : " 29 GB" | 20260505 07:00:28.175 | INFO | } | 20260505 07:00:28.175 | INFO | Perf report for the last 5.13919 seconds: | 20260505 07:00:28.175 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:00:28.175 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:28.175 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:28.175 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:28.175 | INFO | ANALYSIS : 64.3%, 3.0, 100.3ms, 0 | 20260505 07:00:28.175 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:28.175 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:28.175 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260505 07:00:28.175 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:28.175 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:28.175 | INFO | Uploaded 36GiB to GPU in 2.16214s (16.6502GiB/s) | 20260505 07:00:28.176 | INFO | Downloaded 34GiB to GPU in 2.04439s (16.6309GiB/s) | 20260505 07:00:28.295 | INFO | Starting a round of Baz IO | 20260505 07:00:31.540 | INFO | Populating RTMetrics report | 20260505 07:00:31.540 | INFO | RTMetrics report populated | 20260505 07:00:31.540 | INFO | Opening tmp RTMetrics json file | 20260505 07:00:31.562 | INFO | Serializing RTMetrics to json | 20260505 07:00:31.563 | INFO | Serialization done, swapping tmp file now... | 20260505 07:00:31.563 | INFO | RTMetrics done | 20260505 07:00:31.650 | INFO | Finished a round of Baz IO: Wrote 2.41 GiB in 3.354 seconds | 20260505 07:00:31.650 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 31.22% / 0.04% / 68.74% | 20260505 07:00:31.650 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.21401% / 4.76837e-05% / 7.54992e-05% | 20260505 07:00:31.650 | INFO | Baz IO overall: | 20260505 07:00:31.650 | INFO | BytesWritten : 666580057034 | 20260505 07:00:31.650 | INFO | NumEvents : 315843548641 | 20260505 07:00:33.315 | INFO | Analyzing chunk frames = [1249280,1249792) | 20260505 07:00:33.315 | INFO | MemInfo: { | 20260505 07:00:33.315 | INFO | "gpu" : " 17 GB", | 20260505 07:00:33.315 | INFO | "huge" : "", | 20260505 07:00:33.315 | INFO | "huge_shared" : " 27 GB", | 20260505 07:00:33.315 | INFO | "malloc" : " 14 GB", | 20260505 07:00:33.315 | INFO | "pinned" : " 29 GB" | 20260505 07:00:33.315 | INFO | } | 20260505 07:00:33.315 | INFO | Perf report for the last 5.14019 seconds: | 20260505 07:00:33.315 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:00:33.315 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:33.315 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:33.315 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:33.315 | INFO | ANALYSIS : 64.3%, 2.9, 98.5ms, 0 | 20260505 07:00:33.315 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:33.315 | INFO | RT_METRICS : 46.5%, 1.0, 24.4ms, 0 | 20260505 07:00:33.315 | INFO | PRE_HQ : 63.0%, 1.9, 62.9ms, 0 | 20260505 07:00:33.315 | INFO | BAZWRITER : 64.6%, 4.1, 137.0ms, 0 | 20260505 07:00:33.315 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 07:00:33.316 | INFO | Uploaded 36GiB to GPU in 2.14165s (16.8094GiB/s) | 20260505 07:00:33.316 | INFO | Downloaded 38GiB to GPU in 2.31403s (16.4216GiB/s) | 20260505 07:00:38.454 | INFO | Analyzing chunk frames = [1249792,1250304) | 20260505 07:00:38.454 | INFO | MemInfo: { | 20260505 07:00:38.454 | INFO | "gpu" : " 17 GB", | 20260505 07:00:38.454 | INFO | "huge" : "", | 20260505 07:00:38.454 | INFO | "huge_shared" : " 27 GB", | 20260505 07:00:38.454 | INFO | "malloc" : " 14 GB", | 20260505 07:00:38.454 | INFO | "pinned" : " 29 GB" | 20260505 07:00:38.454 | INFO | } | 20260505 07:00:38.454 | INFO | Perf report for the last 5.13869 seconds: | 20260505 07:00:38.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:00:38.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:38.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:38.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:38.454 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 07:00:38.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:38.454 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:38.454 | INFO | PRE_HQ : 30.2%, 1.0, 15.8ms, 0 | 20260505 07:00:38.454 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:38.454 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:38.454 | INFO | Uploaded 36GiB to GPU in 2.1129s (17.0382GiB/s) | 20260505 07:00:38.455 | INFO | Downloaded 34GiB to GPU in 2.03161s (16.7355GiB/s) | 20260505 07:00:43.593 | INFO | Analyzing chunk frames = [1250304,1250816) | 20260505 07:00:43.593 | INFO | MemInfo: { | 20260505 07:00:43.593 | INFO | "gpu" : " 17 GB", | 20260505 07:00:43.593 | INFO | "huge" : "", | 20260505 07:00:43.593 | INFO | "huge_shared" : " 27 GB", | 20260505 07:00:43.593 | INFO | "malloc" : " 14 GB", | 20260505 07:00:43.593 | INFO | "pinned" : " 29 GB" | 20260505 07:00:43.593 | INFO | } | 20260505 07:00:43.593 | INFO | Perf report for the last 5.13936 seconds: | 20260505 07:00:43.593 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:00:43.593 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:43.593 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:43.593 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:43.593 | INFO | ANALYSIS : 63.6%, 3.0, 98.9ms, 0 | 20260505 07:00:43.593 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:43.593 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:43.593 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:00:43.593 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:43.593 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:43.594 | INFO | Uploaded 36GiB to GPU in 2.14289s (16.7997GiB/s) | 20260505 07:00:43.594 | INFO | Downloaded 34GiB to GPU in 2.04974s (16.5875GiB/s) | 20260505 07:00:48.733 | INFO | Analyzing chunk frames = [1250816,1251328) | 20260505 07:00:48.733 | INFO | MemInfo: { | 20260505 07:00:48.733 | INFO | "gpu" : " 17 GB", | 20260505 07:00:48.733 | INFO | "huge" : "", | 20260505 07:00:48.733 | INFO | "huge_shared" : " 27 GB", | 20260505 07:00:48.733 | INFO | "malloc" : " 14 GB", | 20260505 07:00:48.733 | INFO | "pinned" : " 29 GB" | 20260505 07:00:48.733 | INFO | } | 20260505 07:00:48.733 | INFO | Perf report for the last 5.13943 seconds: | 20260505 07:00:48.733 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:00:48.733 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:48.733 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:48.733 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:48.733 | INFO | ANALYSIS : 63.5%, 3.0, 98.6ms, 0 | 20260505 07:00:48.733 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:48.733 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:48.733 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 07:00:48.733 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:48.733 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:48.733 | INFO | Uploaded 36GiB to GPU in 2.15654s (16.6934GiB/s) | 20260505 07:00:48.733 | INFO | Downloaded 34GiB to GPU in 2.03206s (16.7318GiB/s) | 20260505 07:00:53.872 | INFO | Analyzing chunk frames = [1251328,1251840) | 20260505 07:00:53.872 | INFO | MemInfo: { | 20260505 07:00:53.872 | INFO | "gpu" : " 17 GB", | 20260505 07:00:53.872 | INFO | "huge" : "", | 20260505 07:00:53.872 | INFO | "huge_shared" : " 27 GB", | 20260505 07:00:53.872 | INFO | "malloc" : " 14 GB", | 20260505 07:00:53.872 | INFO | "pinned" : " 29 GB" | 20260505 07:00:53.872 | INFO | } | 20260505 07:00:53.872 | INFO | Perf report for the last 5.13919 seconds: | 20260505 07:00:53.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:00:53.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:53.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:53.872 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:53.872 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 07:00:53.872 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:53.872 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:53.872 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:00:53.872 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:53.872 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:53.872 | INFO | Uploaded 36GiB to GPU in 2.14803s (16.7595GiB/s) | 20260505 07:00:53.872 | INFO | Downloaded 34GiB to GPU in 2.04674s (16.6118GiB/s) | 20260505 07:00:59.013 | INFO | Analyzing chunk frames = [1251840,1252352) | 20260505 07:00:59.013 | INFO | MemInfo: { | 20260505 07:00:59.013 | INFO | "gpu" : " 17 GB", | 20260505 07:00:59.013 | INFO | "huge" : "", | 20260505 07:00:59.013 | INFO | "huge_shared" : " 27 GB", | 20260505 07:00:59.013 | INFO | "malloc" : " 14 GB", | 20260505 07:00:59.013 | INFO | "pinned" : " 29 GB" | 20260505 07:00:59.013 | INFO | } | 20260505 07:00:59.013 | INFO | Perf report for the last 5.14139 seconds: | 20260505 07:00:59.013 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:00:59.013 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:59.013 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:59.013 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:59.013 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 07:00:59.013 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:59.013 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:59.014 | INFO | PRE_HQ : 30.1%, 1.0, 15.7ms, 0 | 20260505 07:00:59.014 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:59.014 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:00:59.014 | INFO | Uploaded 36GiB to GPU in 2.16903s (16.5973GiB/s) | 20260505 07:00:59.014 | INFO | Downloaded 34GiB to GPU in 2.05901s (16.5128GiB/s) | 20260505 07:01:04.151 | INFO | Analyzing chunk frames = [1252352,1252864) | 20260505 07:01:04.151 | INFO | MemInfo: { | 20260505 07:01:04.151 | INFO | "gpu" : " 17 GB", | 20260505 07:01:04.151 | INFO | "huge" : "", | 20260505 07:01:04.151 | INFO | "huge_shared" : " 27 GB", | 20260505 07:01:04.151 | INFO | "malloc" : " 14 GB", | 20260505 07:01:04.151 | INFO | "pinned" : " 29 GB" | 20260505 07:01:04.151 | INFO | } | 20260505 07:01:04.151 | INFO | Perf report for the last 5.13731 seconds: | 20260505 07:01:04.151 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:01:04.151 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:04.151 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:04.151 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:04.151 | INFO | ANALYSIS : 64.9%, 3.0, 101.3ms, 0 | 20260505 07:01:04.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:04.151 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:04.151 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260505 07:01:04.151 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:04.151 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:04.151 | INFO | Uploaded 36GiB to GPU in 2.24185s (16.0582GiB/s) | 20260505 07:01:04.151 | INFO | Downloaded 34GiB to GPU in 2.15818s (15.754GiB/s) | 20260505 07:01:09.290 | INFO | Analyzing chunk frames = [1252864,1253376) | 20260505 07:01:09.290 | INFO | MemInfo: { | 20260505 07:01:09.290 | INFO | "gpu" : " 17 GB", | 20260505 07:01:09.290 | INFO | "huge" : "", | 20260505 07:01:09.290 | INFO | "huge_shared" : " 27 GB", | 20260505 07:01:09.290 | INFO | "malloc" : " 14 GB", | 20260505 07:01:09.290 | INFO | "pinned" : " 29 GB" | 20260505 07:01:09.290 | INFO | } | 20260505 07:01:09.290 | INFO | Perf report for the last 5.13926 seconds: | 20260505 07:01:09.290 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:01:09.290 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:09.290 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:09.290 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:09.290 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 07:01:09.290 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:09.290 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:09.290 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 07:01:09.290 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:09.290 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:09.290 | INFO | Uploaded 36GiB to GPU in 2.16272s (16.6457GiB/s) | 20260505 07:01:09.291 | INFO | Downloaded 34GiB to GPU in 2.03869s (16.6774GiB/s) | 20260505 07:01:09.291 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 07:01:09.291 | INFO | Total mem usage: 15.3 GiB | 20260505 07:01:09.291 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 07:01:09.291 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.75%) | 20260505 07:01:09.291 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 07:01:09.291 | INFO | High Water Breakdown: | 20260505 07:01:09.291 | INFO | 60.5% : 8.68 GiB ( 92.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 07:01:09.291 | INFO | 39.4% : 5.66 GiB ( 85.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 07:01:09.291 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 07:01:09.291 | INFO | ------------------------------------------ | 20260505 07:01:09.291 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:01:09.291 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 07:01:09.291 | INFO | Total mem usage: 26.5 GiB | 20260505 07:01:09.291 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 07:01:09.291 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 07:01:09.291 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 07:01:09.291 | INFO | High Water Breakdown: | 20260505 07:01:09.291 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:01:09.291 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:01:09.291 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:01:09.291 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:01:09.291 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:01:09.291 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:01:09.291 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 07:01:09.291 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 07:01:09.291 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:01:09.291 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 07:01:09.291 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 07:01:09.291 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 07:01:09.291 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 07:01:09.291 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 07:01:09.291 | INFO | ------------------------------------------ | 20260505 07:01:09.291 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:01:09.291 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 07:01:09.291 | INFO | Total mem usage: 25.6 GiB | 20260505 07:01:09.291 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 07:01:09.291 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 07:01:09.291 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 07:01:09.291 | INFO | High Water Breakdown: | 20260505 07:01:09.291 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 07:01:09.291 | INFO | ------------------------------------------ | 20260505 07:01:09.291 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:01:09.291 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 07:01:09.291 | INFO | Total mem usage: 15.7 GiB | 20260505 07:01:09.291 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 07:01:09.291 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 07:01:09.291 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 07:01:09.291 | INFO | High Water Breakdown: | 20260505 07:01:09.291 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 07:01:09.291 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 07:01:09.291 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 07:01:09.291 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 07:01:09.291 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 07:01:09.291 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 07:01:09.291 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 07:01:09.291 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 07:01:09.291 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 07:01:09.291 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:01:09.291 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 07:01:09.291 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:01:09.291 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 07:01:09.291 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 07:01:09.291 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 07:01:09.291 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 07:01:09.292 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:01:09.292 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 07:01:09.292 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:01:09.292 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:01:09.292 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:01:09.292 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 07:01:09.292 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 07:01:09.292 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:01:09.292 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 07:01:09.292 | INFO | ------------------------------------------ | 20260505 07:01:09.292 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:01:09.292 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 07:01:09.292 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:01:09.292 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:01:09.292 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.49e+01, 1.37e+01 / 1.59e+00 | 20260505 07:01:09.292 | INFO | PerfReport: 0.28%, AggregateStats -- 9.37e-02 / 1.03e-01, 1.14e-02 / 4.23e-03 | 20260505 07:01:09.292 | INFO | PerfReport: 11.60%, Baseline -- 3.82e+00 / 4.42e+00, 6.29e-01 / 3.41e-01 | 20260505 07:01:09.292 | INFO | PerfReport: 3.31%, Histogram -- 1.09e+00 / 1.48e+00, 2.14e-01 / 1.12e-01 | 20260505 07:01:09.292 | INFO | PerfReport: 15.47%, DME -- 4.08e+01 / 4.44e+01, 4.06e+00 / 1.19e+00 | 20260505 07:01:09.292 | INFO | PerfReport: 56.18%, FrameLabeling -- 1.85e+01 / 1.91e+01, 2.06e+00 / 1.19e+00 | 20260505 07:01:09.292 | INFO | PerfReport: 7.64%, PulseAccumulating -- 2.52e+00 / 3.07e+00, 4.47e-01 / 3.18e-01 | 20260505 07:01:09.292 | INFO | PerfReport: 4.04%, Metrics -- 1.33e+00 / 1.71e+00, 3.71e-01 / 2.24e-01 | 20260505 07:01:09.292 | INFO | PerfReport: 1.46%, PulseSerializing -- 4.82e-01 / 5.50e-01, 9.34e-02 / 3.72e-02 | 20260505 07:01:09.292 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 07:01:09.292 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:01:09.292 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:01:09.292 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.23e+01 / 0.00e+00 | 20260505 07:01:09.292 | INFO | PerfReport: 22.20%, Upload -- 2.25e+01 / 0.00e+00, 7.92e+00 / 0.00e+00 | 20260505 07:01:09.292 | INFO | PerfReport: 21.47%, Download -- 2.18e+01 / 0.00e+00, 7.78e+00 / 0.00e+00 | 20260505 07:01:09.292 | INFO | PerfReport: 32.48%, Compute -- 3.30e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 07:01:09.292 | INFO | PerfReport: 1.48%, LockUpload -- 1.50e+00 / 0.00e+00, 7.01e+00 / 0.00e+00 | 20260505 07:01:09.292 | INFO | PerfReport: 0.80%, LockDownload -- 8.10e-01 / 0.00e+00, 2.18e+00 / 0.00e+00 | 20260505 07:01:09.292 | INFO | PerfReport: 21.55%, LockCompute -- 2.19e+01 / 0.00e+00, 1.67e+01 / 0.00e+00 | 20260505 07:01:09.292 | INFO | PerfReport: 0.01%, Misc -- 1.49e-02 / 0.00e+00, 2.52e-02 / 0.00e+00 | 20260505 07:01:09.410 | INFO | Starting a round of Baz IO | 20260505 07:01:12.656 | INFO | Populating RTMetrics report | 20260505 07:01:12.657 | INFO | RTMetrics report populated | 20260505 07:01:12.657 | INFO | Opening tmp RTMetrics json file | 20260505 07:01:12.707 | INFO | Serializing RTMetrics to json | 20260505 07:01:12.739 | INFO | Serialization done, swapping tmp file now... | 20260505 07:01:12.739 | INFO | RTMetrics done | 20260505 07:01:12.773 | INFO | Finished a round of Baz IO: Wrote 2.41 GiB in 3.362 seconds | 20260505 07:01:12.773 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 31.18% / 0.04% / 68.78% | 20260505 07:01:12.773 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.18779% / 5.96046e-05% / 7.54992e-05% | 20260505 07:01:12.773 | INFO | Baz IO overall: | 20260505 07:01:12.773 | INFO | BytesWritten : 669167455350 | 20260505 07:01:12.773 | INFO | NumEvents : 317080713553 | 20260505 07:01:14.430 | INFO | Analyzing chunk frames = [1253376,1253888) | 20260505 07:01:14.430 | INFO | MemInfo: { | 20260505 07:01:14.430 | INFO | "gpu" : " 17 GB", | 20260505 07:01:14.430 | INFO | "huge" : "", | 20260505 07:01:14.430 | INFO | "huge_shared" : " 27 GB", | 20260505 07:01:14.430 | INFO | "malloc" : " 14 GB", | 20260505 07:01:14.430 | INFO | "pinned" : " 29 GB" | 20260505 07:01:14.430 | INFO | } | 20260505 07:01:14.430 | INFO | Perf report for the last 5.13985 seconds: | 20260505 07:01:14.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:01:14.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:14.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:14.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:14.430 | INFO | ANALYSIS : 64.3%, 2.9, 98.0ms, 0 | 20260505 07:01:14.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:14.430 | INFO | RT_METRICS : 47.6%, 1.0, 25.0ms, 0 | 20260505 07:01:14.430 | INFO | PRE_HQ : 63.1%, 1.9, 63.7ms, 0 | 20260505 07:01:14.430 | INFO | BAZWRITER : 64.7%, 4.1, 137.6ms, 0 | 20260505 07:01:14.430 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260505 07:01:14.430 | INFO | Uploaded 36GiB to GPU in 2.14084s (16.8158GiB/s) | 20260505 07:01:14.430 | INFO | Downloaded 38GiB to GPU in 2.3086s (16.4602GiB/s) | 20260505 07:01:19.569 | INFO | Analyzing chunk frames = [1253888,1254400) | 20260505 07:01:19.569 | INFO | MemInfo: { | 20260505 07:01:19.569 | INFO | "gpu" : " 17 GB", | 20260505 07:01:19.569 | INFO | "huge" : "", | 20260505 07:01:19.569 | INFO | "huge_shared" : " 27 GB", | 20260505 07:01:19.569 | INFO | "malloc" : " 14 GB", | 20260505 07:01:19.569 | INFO | "pinned" : " 29 GB" | 20260505 07:01:19.569 | INFO | } | 20260505 07:01:19.569 | INFO | Perf report for the last 5.13886 seconds: | 20260505 07:01:19.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:01:19.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:19.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:19.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:19.569 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 07:01:19.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:19.569 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:19.569 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 07:01:19.569 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:19.569 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:19.569 | INFO | Uploaded 36GiB to GPU in 2.11694s (17.0056GiB/s) | 20260505 07:01:19.569 | INFO | Downloaded 34GiB to GPU in 2.03819s (16.6815GiB/s) | 20260505 07:01:24.708 | INFO | Analyzing chunk frames = [1254400,1254912) | 20260505 07:01:24.708 | INFO | MemInfo: { | 20260505 07:01:24.708 | INFO | "gpu" : " 17 GB", | 20260505 07:01:24.708 | INFO | "huge" : "", | 20260505 07:01:24.708 | INFO | "huge_shared" : " 27 GB", | 20260505 07:01:24.708 | INFO | "malloc" : " 14 GB", | 20260505 07:01:24.708 | INFO | "pinned" : " 29 GB" | 20260505 07:01:24.708 | INFO | } | 20260505 07:01:24.708 | INFO | Perf report for the last 5.13938 seconds: | 20260505 07:01:24.708 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:01:24.708 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:24.708 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:24.708 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:24.708 | INFO | ANALYSIS : 63.7%, 3.0, 99.1ms, 0 | 20260505 07:01:24.708 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:24.708 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:24.708 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 07:01:24.708 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:24.708 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:24.709 | INFO | Uploaded 36GiB to GPU in 2.14494s (16.7837GiB/s) | 20260505 07:01:24.709 | INFO | Downloaded 34GiB to GPU in 2.05308s (16.5605GiB/s) | 20260505 07:01:29.848 | INFO | Analyzing chunk frames = [1254912,1255424) | 20260505 07:01:29.848 | INFO | MemInfo: { | 20260505 07:01:29.848 | INFO | "gpu" : " 17 GB", | 20260505 07:01:29.848 | INFO | "huge" : "", | 20260505 07:01:29.848 | INFO | "huge_shared" : " 27 GB", | 20260505 07:01:29.848 | INFO | "malloc" : " 14 GB", | 20260505 07:01:29.848 | INFO | "pinned" : " 29 GB" | 20260505 07:01:29.848 | INFO | } | 20260505 07:01:29.848 | INFO | Perf report for the last 5.13972 seconds: | 20260505 07:01:29.848 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:01:29.848 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:29.848 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:29.848 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:29.848 | INFO | ANALYSIS : 63.5%, 3.0, 98.6ms, 0 | 20260505 07:01:29.848 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:29.848 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:29.848 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 07:01:29.848 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:29.848 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:29.848 | INFO | Uploaded 36GiB to GPU in 2.14486s (16.7843GiB/s) | 20260505 07:01:29.848 | INFO | Downloaded 34GiB to GPU in 2.02692s (16.7742GiB/s) | 20260505 07:01:34.987 | INFO | Analyzing chunk frames = [1255424,1255936) | 20260505 07:01:34.987 | INFO | MemInfo: { | 20260505 07:01:34.987 | INFO | "gpu" : " 17 GB", | 20260505 07:01:34.987 | INFO | "huge" : "", | 20260505 07:01:34.987 | INFO | "huge_shared" : " 27 GB", | 20260505 07:01:34.987 | INFO | "malloc" : " 14 GB", | 20260505 07:01:34.987 | INFO | "pinned" : " 29 GB" | 20260505 07:01:34.987 | INFO | } | 20260505 07:01:34.987 | INFO | Perf report for the last 5.13935 seconds: | 20260505 07:01:34.987 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:01:34.987 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:34.987 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:34.987 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:34.987 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 07:01:34.987 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:34.987 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:34.987 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 07:01:34.987 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:34.987 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:34.988 | INFO | Uploaded 36GiB to GPU in 2.15342s (16.7176GiB/s) | 20260505 07:01:34.988 | INFO | Downloaded 34GiB to GPU in 2.04491s (16.6266GiB/s) | 20260505 07:01:40.128 | INFO | Analyzing chunk frames = [1255936,1256448) | 20260505 07:01:40.128 | INFO | MemInfo: { | 20260505 07:01:40.128 | INFO | "gpu" : " 17 GB", | 20260505 07:01:40.128 | INFO | "huge" : "", | 20260505 07:01:40.128 | INFO | "huge_shared" : " 27 GB", | 20260505 07:01:40.128 | INFO | "malloc" : " 14 GB", | 20260505 07:01:40.128 | INFO | "pinned" : " 29 GB" | 20260505 07:01:40.128 | INFO | } | 20260505 07:01:40.128 | INFO | Perf report for the last 5.14088 seconds: | 20260505 07:01:40.128 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:01:40.128 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:40.128 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:40.128 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:40.128 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 07:01:40.128 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:40.128 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:40.128 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 07:01:40.128 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:40.128 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:40.129 | INFO | Uploaded 36GiB to GPU in 2.1607s (16.6612GiB/s) | 20260505 07:01:40.129 | INFO | Downloaded 34GiB to GPU in 2.05413s (16.5521GiB/s) | 20260505 07:01:45.265 | INFO | Analyzing chunk frames = [1256448,1256960) | 20260505 07:01:45.265 | INFO | MemInfo: { | 20260505 07:01:45.265 | INFO | "gpu" : " 17 GB", | 20260505 07:01:45.265 | INFO | "huge" : "", | 20260505 07:01:45.265 | INFO | "huge_shared" : " 27 GB", | 20260505 07:01:45.265 | INFO | "malloc" : " 14 GB", | 20260505 07:01:45.265 | INFO | "pinned" : " 29 GB" | 20260505 07:01:45.265 | INFO | } | 20260505 07:01:45.266 | INFO | Perf report for the last 5.13758 seconds: | 20260505 07:01:45.266 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:01:45.266 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:45.266 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:45.266 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:45.266 | INFO | ANALYSIS : 64.5%, 3.0, 100.7ms, 0 | 20260505 07:01:45.266 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:45.266 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:45.266 | INFO | PRE_HQ : 32.4%, 1.0, 16.9ms, 0 | 20260505 07:01:45.266 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:45.266 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:45.266 | INFO | Uploaded 36GiB to GPU in 2.21876s (16.2252GiB/s) | 20260505 07:01:45.266 | INFO | Downloaded 34GiB to GPU in 2.16091s (15.7341GiB/s) | 20260505 07:01:50.405 | INFO | Analyzing chunk frames = [1256960,1257472) | 20260505 07:01:50.405 | INFO | MemInfo: { | 20260505 07:01:50.405 | INFO | "gpu" : " 17 GB", | 20260505 07:01:50.405 | INFO | "huge" : "", | 20260505 07:01:50.405 | INFO | "huge_shared" : " 27 GB", | 20260505 07:01:50.405 | INFO | "malloc" : " 14 GB", | 20260505 07:01:50.405 | INFO | "pinned" : " 29 GB" | 20260505 07:01:50.405 | INFO | } | 20260505 07:01:50.405 | INFO | Perf report for the last 5.13929 seconds: | 20260505 07:01:50.405 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:01:50.405 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:50.405 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:50.405 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:50.405 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 07:01:50.405 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:50.405 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:50.405 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260505 07:01:50.405 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:50.405 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:50.405 | INFO | Uploaded 36GiB to GPU in 2.16736s (16.61GiB/s) | 20260505 07:01:50.405 | INFO | Downloaded 34GiB to GPU in 2.04629s (16.6154GiB/s) | 20260505 07:01:50.522 | INFO | Starting a round of Baz IO | 20260505 07:01:53.826 | INFO | Populating RTMetrics report | 20260505 07:01:53.827 | INFO | RTMetrics report populated | 20260505 07:01:53.827 | INFO | Opening tmp RTMetrics json file | 20260505 07:01:53.863 | INFO | Serializing RTMetrics to json | 20260505 07:01:53.909 | INFO | Serialization done, swapping tmp file now... | 20260505 07:01:53.918 | INFO | RTMetrics done | 20260505 07:01:53.951 | INFO | Finished a round of Baz IO: Wrote 2.41 GiB in 3.429 seconds | 20260505 07:01:53.951 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 31.15% / 0.04% / 68.82% | 20260505 07:01:53.951 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.17106% / 7.15256e-05% / 6.35783e-05% | 20260505 07:01:53.951 | INFO | Baz IO overall: | 20260505 07:01:53.951 | INFO | BytesWritten : 671755605426 | 20260505 07:01:53.951 | INFO | NumEvents : 318318641628 | 20260505 07:01:55.544 | INFO | Analyzing chunk frames = [1257472,1257984) | 20260505 07:01:55.544 | INFO | MemInfo: { | 20260505 07:01:55.544 | INFO | "gpu" : " 17 GB", | 20260505 07:01:55.544 | INFO | "huge" : "", | 20260505 07:01:55.544 | INFO | "huge_shared" : " 27 GB", | 20260505 07:01:55.544 | INFO | "malloc" : " 14 GB", | 20260505 07:01:55.544 | INFO | "pinned" : " 29 GB" | 20260505 07:01:55.544 | INFO | } | 20260505 07:01:55.545 | INFO | Perf report for the last 5.13951 seconds: | 20260505 07:01:55.545 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:01:55.545 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:55.545 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:55.545 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:55.545 | INFO | ANALYSIS : 65.4%, 2.9, 99.8ms, 0 | 20260505 07:01:55.545 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:01:55.545 | INFO | RT_METRICS : 48.1%, 1.0, 25.2ms, 0 | 20260505 07:01:55.545 | INFO | PRE_HQ : 64.2%, 1.9, 63.2ms, 0 | 20260505 07:01:55.545 | INFO | BAZWRITER : 66.0%, 4.1, 138.5ms, 0 | 20260505 07:01:55.545 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 07:01:55.545 | INFO | Uploaded 36GiB to GPU in 2.26246s (15.9119GiB/s) | 20260505 07:01:55.545 | INFO | Downloaded 38GiB to GPU in 2.32847s (16.3197GiB/s) | 20260505 07:02:00.684 | INFO | Analyzing chunk frames = [1257984,1258496) | 20260505 07:02:00.684 | INFO | MemInfo: { | 20260505 07:02:00.684 | INFO | "gpu" : " 17 GB", | 20260505 07:02:00.684 | INFO | "huge" : "", | 20260505 07:02:00.684 | INFO | "huge_shared" : " 27 GB", | 20260505 07:02:00.684 | INFO | "malloc" : " 14 GB", | 20260505 07:02:00.684 | INFO | "pinned" : " 29 GB" | 20260505 07:02:00.684 | INFO | } | 20260505 07:02:00.684 | INFO | Perf report for the last 5.13946 seconds: | 20260505 07:02:00.684 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:02:00.684 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:00.684 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:00.684 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:00.684 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 07:02:00.684 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:00.684 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:00.684 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 07:02:00.684 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:00.684 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:00.684 | INFO | Uploaded 36GiB to GPU in 2.10653s (17.0898GiB/s) | 20260505 07:02:00.684 | INFO | Downloaded 34GiB to GPU in 2.03641s (16.6961GiB/s) | 20260505 07:02:05.823 | INFO | Analyzing chunk frames = [1258496,1259008) | 20260505 07:02:05.823 | INFO | MemInfo: { | 20260505 07:02:05.823 | INFO | "gpu" : " 17 GB", | 20260505 07:02:05.823 | INFO | "huge" : "", | 20260505 07:02:05.823 | INFO | "huge_shared" : " 27 GB", | 20260505 07:02:05.823 | INFO | "malloc" : " 14 GB", | 20260505 07:02:05.823 | INFO | "pinned" : " 29 GB" | 20260505 07:02:05.823 | INFO | } | 20260505 07:02:05.823 | INFO | Perf report for the last 5.13907 seconds: | 20260505 07:02:05.823 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:02:05.823 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:05.823 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:05.823 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:05.823 | INFO | ANALYSIS : 63.6%, 3.0, 98.9ms, 0 | 20260505 07:02:05.823 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:05.823 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:05.823 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 07:02:05.823 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:05.823 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:05.823 | INFO | Uploaded 36GiB to GPU in 2.14899s (16.7521GiB/s) | 20260505 07:02:05.824 | INFO | Downloaded 34GiB to GPU in 2.05057s (16.5807GiB/s) | 20260505 07:02:10.962 | INFO | Analyzing chunk frames = [1259008,1259520) | 20260505 07:02:10.962 | INFO | MemInfo: { | 20260505 07:02:10.962 | INFO | "gpu" : " 17 GB", | 20260505 07:02:10.962 | INFO | "huge" : "", | 20260505 07:02:10.962 | INFO | "huge_shared" : " 27 GB", | 20260505 07:02:10.962 | INFO | "malloc" : " 14 GB", | 20260505 07:02:10.962 | INFO | "pinned" : " 29 GB" | 20260505 07:02:10.962 | INFO | } | 20260505 07:02:10.963 | INFO | Perf report for the last 5.13961 seconds: | 20260505 07:02:10.963 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:02:10.963 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:10.963 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:10.963 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:10.963 | INFO | ANALYSIS : 63.4%, 3.0, 98.5ms, 0 | 20260505 07:02:10.963 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:10.963 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:10.963 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260505 07:02:10.963 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:10.963 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:10.963 | INFO | Uploaded 36GiB to GPU in 2.13101s (16.8934GiB/s) | 20260505 07:02:10.963 | INFO | Downloaded 34GiB to GPU in 2.02475s (16.7922GiB/s) | 20260505 07:02:16.102 | INFO | Analyzing chunk frames = [1259520,1260032) | 20260505 07:02:16.102 | INFO | MemInfo: { | 20260505 07:02:16.102 | INFO | "gpu" : " 17 GB", | 20260505 07:02:16.102 | INFO | "huge" : "", | 20260505 07:02:16.102 | INFO | "huge_shared" : " 27 GB", | 20260505 07:02:16.102 | INFO | "malloc" : " 14 GB", | 20260505 07:02:16.102 | INFO | "pinned" : " 29 GB" | 20260505 07:02:16.102 | INFO | } | 20260505 07:02:16.102 | INFO | Perf report for the last 5.13897 seconds: | 20260505 07:02:16.102 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:02:16.102 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:16.102 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:16.102 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:16.102 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 07:02:16.102 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:16.102 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:16.102 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 07:02:16.102 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:16.102 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:16.102 | INFO | Uploaded 36GiB to GPU in 2.162s (16.6512GiB/s) | 20260505 07:02:16.102 | INFO | Downloaded 34GiB to GPU in 2.04745s (16.6061GiB/s) | 20260505 07:02:21.243 | INFO | Analyzing chunk frames = [1260032,1260544) | 20260505 07:02:21.243 | INFO | MemInfo: { | 20260505 07:02:21.243 | INFO | "gpu" : " 17 GB", | 20260505 07:02:21.243 | INFO | "huge" : "", | 20260505 07:02:21.243 | INFO | "huge_shared" : " 27 GB", | 20260505 07:02:21.243 | INFO | "malloc" : " 14 GB", | 20260505 07:02:21.243 | INFO | "pinned" : " 29 GB" | 20260505 07:02:21.243 | INFO | } | 20260505 07:02:21.243 | INFO | Perf report for the last 5.14114 seconds: | 20260505 07:02:21.243 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:02:21.243 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:21.243 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:21.243 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:21.243 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 07:02:21.243 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:21.243 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:21.243 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 07:02:21.243 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:21.243 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:21.243 | INFO | Uploaded 36GiB to GPU in 2.12265s (16.9599GiB/s) | 20260505 07:02:21.243 | INFO | Downloaded 34GiB to GPU in 2.03275s (16.7261GiB/s) | 20260505 07:02:26.380 | INFO | Analyzing chunk frames = [1260544,1261056) | 20260505 07:02:26.380 | INFO | MemInfo: { | 20260505 07:02:26.380 | INFO | "gpu" : " 17 GB", | 20260505 07:02:26.380 | INFO | "huge" : "", | 20260505 07:02:26.380 | INFO | "huge_shared" : " 27 GB", | 20260505 07:02:26.380 | INFO | "malloc" : " 14 GB", | 20260505 07:02:26.380 | INFO | "pinned" : " 29 GB" | 20260505 07:02:26.380 | INFO | } | 20260505 07:02:26.381 | INFO | Perf report for the last 5.13774 seconds: | 20260505 07:02:26.381 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:02:26.381 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:26.381 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:26.381 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:26.381 | INFO | ANALYSIS : 64.9%, 3.0, 101.3ms, 0 | 20260505 07:02:26.381 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:26.381 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:26.381 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260505 07:02:26.381 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:26.381 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:26.381 | INFO | Uploaded 36GiB to GPU in 2.24014s (16.0704GiB/s) | 20260505 07:02:26.381 | INFO | Downloaded 34GiB to GPU in 2.15528s (15.7752GiB/s) | 20260505 07:02:31.520 | INFO | Analyzing chunk frames = [1261056,1261568) | 20260505 07:02:31.520 | INFO | MemInfo: { | 20260505 07:02:31.520 | INFO | "gpu" : " 17 GB", | 20260505 07:02:31.520 | INFO | "huge" : "", | 20260505 07:02:31.520 | INFO | "huge_shared" : " 27 GB", | 20260505 07:02:31.520 | INFO | "malloc" : " 14 GB", | 20260505 07:02:31.520 | INFO | "pinned" : " 29 GB" | 20260505 07:02:31.520 | INFO | } | 20260505 07:02:31.520 | INFO | Perf report for the last 5.13926 seconds: | 20260505 07:02:31.520 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:02:31.520 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:31.520 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:31.520 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:31.520 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 07:02:31.520 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:31.520 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:31.520 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 07:02:31.520 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:31.520 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:31.520 | INFO | Uploaded 36GiB to GPU in 2.1614s (16.6559GiB/s) | 20260505 07:02:31.520 | INFO | Downloaded 34GiB to GPU in 2.03976s (16.6686GiB/s) | 20260505 07:02:31.520 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 07:02:31.520 | INFO | Total mem usage: 15.3 GiB | 20260505 07:02:31.520 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 07:02:31.520 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.69%) | 20260505 07:02:31.520 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 07:02:31.520 | INFO | High Water Breakdown: | 20260505 07:02:31.520 | INFO | 60.5% : 8.69 GiB ( 92.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 07:02:31.520 | INFO | 39.4% : 5.66 GiB ( 85.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 07:02:31.520 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 07:02:31.520 | INFO | ------------------------------------------ | 20260505 07:02:31.520 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:02:31.520 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 07:02:31.520 | INFO | Total mem usage: 26.5 GiB | 20260505 07:02:31.520 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 07:02:31.520 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 07:02:31.520 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 07:02:31.521 | INFO | High Water Breakdown: | 20260505 07:02:31.521 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:02:31.521 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:02:31.521 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:02:31.521 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:02:31.521 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:02:31.521 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:02:31.521 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 07:02:31.521 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 07:02:31.521 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:02:31.521 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 07:02:31.521 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 07:02:31.521 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 07:02:31.521 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 07:02:31.521 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 07:02:31.521 | INFO | ------------------------------------------ | 20260505 07:02:31.521 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:02:31.521 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 07:02:31.521 | INFO | Total mem usage: 25.6 GiB | 20260505 07:02:31.521 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 07:02:31.521 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 07:02:31.521 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 07:02:31.521 | INFO | High Water Breakdown: | 20260505 07:02:31.521 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 07:02:31.521 | INFO | ------------------------------------------ | 20260505 07:02:31.521 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:02:31.521 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 07:02:31.521 | INFO | Total mem usage: 15.7 GiB | 20260505 07:02:31.521 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 07:02:31.521 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 07:02:31.521 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 07:02:31.521 | INFO | High Water Breakdown: | 20260505 07:02:31.521 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 07:02:31.521 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 07:02:31.521 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 07:02:31.521 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 07:02:31.521 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 07:02:31.521 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 07:02:31.521 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 07:02:31.521 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 07:02:31.521 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 07:02:31.521 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:02:31.521 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 07:02:31.521 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:02:31.521 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 07:02:31.521 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 07:02:31.521 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 07:02:31.521 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 07:02:31.521 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:02:31.521 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 07:02:31.521 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:02:31.521 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:02:31.521 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:02:31.521 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 07:02:31.521 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 07:02:31.521 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:02:31.521 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 07:02:31.521 | INFO | ------------------------------------------ | 20260505 07:02:31.521 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:02:31.521 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 07:02:31.521 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:02:31.521 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:02:31.521 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.52e+01, 1.38e+01 / 1.57e+00 | 20260505 07:02:31.521 | INFO | PerfReport: 0.29%, AggregateStats -- 9.42e-02 / 1.04e-01, 1.16e-02 / 6.82e-03 | 20260505 07:02:31.521 | INFO | PerfReport: 11.60%, Baseline -- 3.83e+00 / 4.43e+00, 6.39e-01 / 3.34e-01 | 20260505 07:02:31.521 | INFO | PerfReport: 3.31%, Histogram -- 1.09e+00 / 1.48e+00, 2.17e-01 / 1.17e-01 | 20260505 07:02:31.521 | INFO | PerfReport: 15.48%, DME -- 4.08e+01 / 4.46e+01, 4.20e+00 / 1.13e+00 | 20260505 07:02:31.521 | INFO | PerfReport: 56.21%, FrameLabeling -- 1.85e+01 / 1.92e+01, 2.08e+00 / 1.31e+00 | 20260505 07:02:31.521 | INFO | PerfReport: 7.64%, PulseAccumulating -- 2.52e+00 / 3.11e+00, 4.47e-01 / 2.96e-01 | 20260505 07:02:31.521 | INFO | PerfReport: 4.02%, Metrics -- 1.33e+00 / 1.70e+00, 3.71e-01 / 1.92e-01 | 20260505 07:02:31.521 | INFO | PerfReport: 1.46%, PulseSerializing -- 4.82e-01 / 5.54e-01, 9.30e-02 / 3.56e-02 | 20260505 07:02:31.521 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 07:02:31.521 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:02:31.521 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:02:31.521 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.23e+01 / 0.00e+00 | 20260505 07:02:31.521 | INFO | PerfReport: 22.19%, Upload -- 2.25e+01 / 0.00e+00, 7.97e+00 / 0.00e+00 | 20260505 07:02:31.521 | INFO | PerfReport: 21.46%, Download -- 2.18e+01 / 0.00e+00, 7.84e+00 / 0.00e+00 | 20260505 07:02:31.522 | INFO | PerfReport: 32.51%, Compute -- 3.30e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 07:02:31.522 | INFO | PerfReport: 1.48%, LockUpload -- 1.50e+00 / 0.00e+00, 6.99e+00 / 0.00e+00 | 20260505 07:02:31.522 | INFO | PerfReport: 0.80%, LockDownload -- 8.10e-01 / 0.00e+00, 2.17e+00 / 0.00e+00 | 20260505 07:02:31.522 | INFO | PerfReport: 21.55%, LockCompute -- 2.19e+01 / 0.00e+00, 1.68e+01 / 0.00e+00 | 20260505 07:02:31.522 | INFO | PerfReport: 0.01%, Misc -- 1.46e-02 / 0.00e+00, 2.00e-02 / 0.00e+00 | 20260505 07:02:31.634 | INFO | Starting a round of Baz IO | 20260505 07:02:34.922 | INFO | Populating RTMetrics report | 20260505 07:02:34.922 | INFO | RTMetrics report populated | 20260505 07:02:34.922 | INFO | Opening tmp RTMetrics json file | 20260505 07:02:34.923 | INFO | Serializing RTMetrics to json | 20260505 07:02:34.924 | INFO | Serialization done, swapping tmp file now... | 20260505 07:02:34.924 | INFO | RTMetrics done | 20260505 07:02:35.051 | INFO | Finished a round of Baz IO: Wrote 2.412 GiB in 3.416 seconds | 20260505 07:02:35.051 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 31.11% / 0.04% / 68.85% | 20260505 07:02:35.051 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.1592% / 7.15256e-05% / 4.76837e-05% | 20260505 07:02:35.051 | INFO | Baz IO overall: | 20260505 07:02:35.051 | INFO | BytesWritten : 674344993051 | 20260505 07:02:35.051 | INFO | NumEvents : 319557648214 | 20260505 07:02:36.659 | INFO | Analyzing chunk frames = [1261568,1262080) | 20260505 07:02:36.659 | INFO | MemInfo: { | 20260505 07:02:36.659 | INFO | "gpu" : " 17 GB", | 20260505 07:02:36.659 | INFO | "huge" : "", | 20260505 07:02:36.659 | INFO | "huge_shared" : " 27 GB", | 20260505 07:02:36.659 | INFO | "malloc" : " 14 GB", | 20260505 07:02:36.659 | INFO | "pinned" : " 29 GB" | 20260505 07:02:36.659 | INFO | } | 20260505 07:02:36.659 | INFO | Perf report for the last 5.13922 seconds: | 20260505 07:02:36.659 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:02:36.659 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:36.659 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:36.659 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:36.659 | INFO | ANALYSIS : 65.1%, 2.9, 99.7ms, 0 | 20260505 07:02:36.659 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:36.659 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 07:02:36.659 | INFO | PRE_HQ : 63.9%, 1.9, 63.4ms, 0 | 20260505 07:02:36.659 | INFO | BAZWRITER : 65.8%, 4.1, 138.5ms, 0 | 20260505 07:02:36.659 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 07:02:36.659 | INFO | Uploaded 36GiB to GPU in 2.25745s (15.9472GiB/s) | 20260505 07:02:36.659 | INFO | Downloaded 38GiB to GPU in 2.30118s (16.5133GiB/s) | 20260505 07:02:41.799 | INFO | Analyzing chunk frames = [1262080,1262592) | 20260505 07:02:41.799 | INFO | MemInfo: { | 20260505 07:02:41.799 | INFO | "gpu" : " 17 GB", | 20260505 07:02:41.799 | INFO | "huge" : "", | 20260505 07:02:41.799 | INFO | "huge_shared" : " 27 GB", | 20260505 07:02:41.799 | INFO | "malloc" : " 14 GB", | 20260505 07:02:41.799 | INFO | "pinned" : " 29 GB" | 20260505 07:02:41.799 | INFO | } | 20260505 07:02:41.799 | INFO | Perf report for the last 5.13985 seconds: | 20260505 07:02:41.799 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:02:41.799 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:41.799 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:41.799 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:41.799 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 07:02:41.799 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:41.799 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:41.799 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 07:02:41.799 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:41.799 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:41.799 | INFO | Uploaded 36GiB to GPU in 2.10406s (17.1098GiB/s) | 20260505 07:02:41.799 | INFO | Downloaded 34GiB to GPU in 2.03809s (16.6823GiB/s) | 20260505 07:02:46.938 | INFO | Analyzing chunk frames = [1262592,1263104) | 20260505 07:02:46.938 | INFO | MemInfo: { | 20260505 07:02:46.938 | INFO | "gpu" : " 17 GB", | 20260505 07:02:46.938 | INFO | "huge" : "", | 20260505 07:02:46.938 | INFO | "huge_shared" : " 27 GB", | 20260505 07:02:46.938 | INFO | "malloc" : " 14 GB", | 20260505 07:02:46.938 | INFO | "pinned" : " 29 GB" | 20260505 07:02:46.938 | INFO | } | 20260505 07:02:46.938 | INFO | Perf report for the last 5.13887 seconds: | 20260505 07:02:46.938 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:02:46.938 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:46.938 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:46.938 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:46.938 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260505 07:02:46.938 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:46.938 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:46.938 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 07:02:46.938 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:46.938 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:46.938 | INFO | Uploaded 36GiB to GPU in 2.15393s (16.7137GiB/s) | 20260505 07:02:46.938 | INFO | Downloaded 34GiB to GPU in 2.049s (16.5934GiB/s) | 20260505 07:02:52.077 | INFO | Analyzing chunk frames = [1263104,1263616) | 20260505 07:02:52.077 | INFO | MemInfo: { | 20260505 07:02:52.077 | INFO | "gpu" : " 17 GB", | 20260505 07:02:52.077 | INFO | "huge" : "", | 20260505 07:02:52.077 | INFO | "huge_shared" : " 27 GB", | 20260505 07:02:52.077 | INFO | "malloc" : " 14 GB", | 20260505 07:02:52.077 | INFO | "pinned" : " 29 GB" | 20260505 07:02:52.077 | INFO | } | 20260505 07:02:52.077 | INFO | Perf report for the last 5.13944 seconds: | 20260505 07:02:52.077 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:02:52.077 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:52.077 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:52.077 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:52.077 | INFO | ANALYSIS : 63.4%, 3.0, 98.6ms, 0 | 20260505 07:02:52.077 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:52.077 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:52.077 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 07:02:52.077 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:52.077 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:52.078 | INFO | Uploaded 36GiB to GPU in 2.13517s (16.8605GiB/s) | 20260505 07:02:52.078 | INFO | Downloaded 34GiB to GPU in 2.02335s (16.8038GiB/s) | 20260505 07:02:57.216 | INFO | Analyzing chunk frames = [1263616,1264128) | 20260505 07:02:57.216 | INFO | MemInfo: { | 20260505 07:02:57.216 | INFO | "gpu" : " 17 GB", | 20260505 07:02:57.216 | INFO | "huge" : "", | 20260505 07:02:57.216 | INFO | "huge_shared" : " 27 GB", | 20260505 07:02:57.216 | INFO | "malloc" : " 14 GB", | 20260505 07:02:57.216 | INFO | "pinned" : " 29 GB" | 20260505 07:02:57.216 | INFO | } | 20260505 07:02:57.217 | INFO | Perf report for the last 5.13924 seconds: | 20260505 07:02:57.217 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:02:57.217 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:57.217 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:57.217 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:57.217 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 07:02:57.217 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:57.217 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:57.217 | INFO | PRE_HQ : 30.3%, 1.0, 15.8ms, 0 | 20260505 07:02:57.217 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:57.217 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:02:57.217 | INFO | Uploaded 36GiB to GPU in 2.16366s (16.6384GiB/s) | 20260505 07:02:57.217 | INFO | Downloaded 34GiB to GPU in 2.05244s (16.5656GiB/s) | 20260505 07:03:02.357 | INFO | Analyzing chunk frames = [1264128,1264640) | 20260505 07:03:02.357 | INFO | MemInfo: { | 20260505 07:03:02.357 | INFO | "gpu" : " 17 GB", | 20260505 07:03:02.357 | INFO | "huge" : "", | 20260505 07:03:02.357 | INFO | "huge_shared" : " 27 GB", | 20260505 07:03:02.357 | INFO | "malloc" : " 14 GB", | 20260505 07:03:02.357 | INFO | "pinned" : " 29 GB" | 20260505 07:03:02.357 | INFO | } | 20260505 07:03:02.357 | INFO | Perf report for the last 5.14087 seconds: | 20260505 07:03:02.358 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:03:02.358 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:02.358 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:02.358 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:02.358 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 07:03:02.358 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:02.358 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:02.358 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260505 07:03:02.358 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:02.358 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:02.358 | INFO | Uploaded 36GiB to GPU in 2.16006s (16.6662GiB/s) | 20260505 07:03:02.358 | INFO | Downloaded 34GiB to GPU in 2.05166s (16.572GiB/s) | 20260505 07:03:07.495 | INFO | Analyzing chunk frames = [1264640,1265152) | 20260505 07:03:07.495 | INFO | MemInfo: { | 20260505 07:03:07.495 | INFO | "gpu" : " 17 GB", | 20260505 07:03:07.495 | INFO | "huge" : "", | 20260505 07:03:07.495 | INFO | "huge_shared" : " 27 GB", | 20260505 07:03:07.495 | INFO | "malloc" : " 14 GB", | 20260505 07:03:07.495 | INFO | "pinned" : " 29 GB" | 20260505 07:03:07.495 | INFO | } | 20260505 07:03:07.495 | INFO | Perf report for the last 5.13802 seconds: | 20260505 07:03:07.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:03:07.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:07.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:07.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:07.496 | INFO | ANALYSIS : 64.8%, 3.0, 101.2ms, 0 | 20260505 07:03:07.496 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:07.496 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:07.496 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260505 07:03:07.496 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:07.496 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:07.496 | INFO | Uploaded 36GiB to GPU in 2.22408s (16.1865GiB/s) | 20260505 07:03:07.496 | INFO | Downloaded 34GiB to GPU in 2.14555s (15.8468GiB/s) | 20260505 07:03:12.635 | INFO | Analyzing chunk frames = [1265152,1265664) | 20260505 07:03:12.635 | INFO | MemInfo: { | 20260505 07:03:12.635 | INFO | "gpu" : " 17 GB", | 20260505 07:03:12.635 | INFO | "huge" : "", | 20260505 07:03:12.635 | INFO | "huge_shared" : " 27 GB", | 20260505 07:03:12.635 | INFO | "malloc" : " 14 GB", | 20260505 07:03:12.635 | INFO | "pinned" : " 29 GB" | 20260505 07:03:12.635 | INFO | } | 20260505 07:03:12.635 | INFO | Perf report for the last 5.13961 seconds: | 20260505 07:03:12.635 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:03:12.635 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:12.635 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:12.635 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:12.635 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 07:03:12.635 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:12.635 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:12.635 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 07:03:12.635 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:12.635 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:12.635 | INFO | Uploaded 36GiB to GPU in 2.16079s (16.6606GiB/s) | 20260505 07:03:12.635 | INFO | Downloaded 34GiB to GPU in 2.0501s (16.5846GiB/s) | 20260505 07:03:12.754 | INFO | Starting a round of Baz IO | 20260505 07:03:16.058 | INFO | Populating RTMetrics report | 20260505 07:03:16.058 | INFO | RTMetrics report populated | 20260505 07:03:16.058 | INFO | Opening tmp RTMetrics json file | 20260505 07:03:16.065 | INFO | Serializing RTMetrics to json | 20260505 07:03:16.067 | INFO | Serialization done, swapping tmp file now... | 20260505 07:03:16.069 | INFO | RTMetrics done | 20260505 07:03:16.185 | INFO | Finished a round of Baz IO: Wrote 2.413 GiB in 3.43 seconds | 20260505 07:03:16.185 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 31.07% / 0.04% / 68.89% | 20260505 07:03:16.185 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.1486% / 6.35783e-05% / 7.54992e-05% | 20260505 07:03:16.185 | INFO | Baz IO overall: | 20260505 07:03:16.185 | INFO | BytesWritten : 676936337816 | 20260505 07:03:16.185 | INFO | NumEvents : 320797961873 | 20260505 07:03:17.774 | INFO | Analyzing chunk frames = [1265664,1266176) | 20260505 07:03:17.774 | INFO | MemInfo: { | 20260505 07:03:17.774 | INFO | "gpu" : " 17 GB", | 20260505 07:03:17.774 | INFO | "huge" : "", | 20260505 07:03:17.774 | INFO | "huge_shared" : " 27 GB", | 20260505 07:03:17.774 | INFO | "malloc" : " 14 GB", | 20260505 07:03:17.774 | INFO | "pinned" : " 29 GB" | 20260505 07:03:17.774 | INFO | } | 20260505 07:03:17.774 | INFO | Perf report for the last 5.13887 seconds: | 20260505 07:03:17.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:03:17.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:17.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:17.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:17.774 | INFO | ANALYSIS : 65.5%, 2.9, 100.2ms, 0 | 20260505 07:03:17.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:17.774 | INFO | RT_METRICS : 46.2%, 1.0, 24.2ms, 0 | 20260505 07:03:17.774 | INFO | PRE_HQ : 64.4%, 1.9, 63.7ms, 0 | 20260505 07:03:17.774 | INFO | BAZWRITER : 66.0%, 4.1, 138.4ms, 0 | 20260505 07:03:17.774 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 07:03:17.774 | INFO | Uploaded 36GiB to GPU in 2.27012s (15.8582GiB/s) | 20260505 07:03:17.774 | INFO | Downloaded 38GiB to GPU in 2.32615s (16.336GiB/s) | 20260505 07:03:22.913 | INFO | Analyzing chunk frames = [1266176,1266688) | 20260505 07:03:22.913 | INFO | MemInfo: { | 20260505 07:03:22.913 | INFO | "gpu" : " 17 GB", | 20260505 07:03:22.913 | INFO | "huge" : "", | 20260505 07:03:22.913 | INFO | "huge_shared" : " 27 GB", | 20260505 07:03:22.913 | INFO | "malloc" : " 14 GB", | 20260505 07:03:22.913 | INFO | "pinned" : " 29 GB" | 20260505 07:03:22.913 | INFO | } | 20260505 07:03:22.913 | INFO | Perf report for the last 5.13947 seconds: | 20260505 07:03:22.913 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:03:22.913 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:22.913 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:22.914 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:22.914 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 07:03:22.914 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:22.914 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:22.914 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 07:03:22.914 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:22.914 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:22.914 | INFO | Uploaded 36GiB to GPU in 2.10996s (17.0619GiB/s) | 20260505 07:03:22.914 | INFO | Downloaded 34GiB to GPU in 2.03446s (16.7121GiB/s) | 20260505 07:03:28.052 | INFO | Analyzing chunk frames = [1266688,1267200) | 20260505 07:03:28.052 | INFO | MemInfo: { | 20260505 07:03:28.052 | INFO | "gpu" : " 17 GB", | 20260505 07:03:28.052 | INFO | "huge" : "", | 20260505 07:03:28.052 | INFO | "huge_shared" : " 27 GB", | 20260505 07:03:28.052 | INFO | "malloc" : " 14 GB", | 20260505 07:03:28.052 | INFO | "pinned" : " 29 GB" | 20260505 07:03:28.053 | INFO | } | 20260505 07:03:28.053 | INFO | Perf report for the last 5.13925 seconds: | 20260505 07:03:28.053 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:03:28.053 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:28.053 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:28.053 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:28.053 | INFO | ANALYSIS : 63.6%, 3.0, 98.9ms, 0 | 20260505 07:03:28.053 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:28.053 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:28.053 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 07:03:28.053 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:28.053 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:28.053 | INFO | Uploaded 36GiB to GPU in 2.1493s (16.7496GiB/s) | 20260505 07:03:28.053 | INFO | Downloaded 34GiB to GPU in 2.05131s (16.5748GiB/s) | 20260505 07:03:33.192 | INFO | Analyzing chunk frames = [1267200,1267712) | 20260505 07:03:33.192 | INFO | MemInfo: { | 20260505 07:03:33.192 | INFO | "gpu" : " 17 GB", | 20260505 07:03:33.192 | INFO | "huge" : "", | 20260505 07:03:33.192 | INFO | "huge_shared" : " 27 GB", | 20260505 07:03:33.192 | INFO | "malloc" : " 14 GB", | 20260505 07:03:33.192 | INFO | "pinned" : " 29 GB" | 20260505 07:03:33.192 | INFO | } | 20260505 07:03:33.192 | INFO | Perf report for the last 5.1392 seconds: | 20260505 07:03:33.192 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:03:33.192 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:33.192 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:33.192 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:33.192 | INFO | ANALYSIS : 63.5%, 3.0, 98.6ms, 0 | 20260505 07:03:33.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:33.192 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:33.192 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 07:03:33.192 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:33.192 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:33.192 | INFO | Uploaded 36GiB to GPU in 2.13155s (16.8891GiB/s) | 20260505 07:03:33.192 | INFO | Downloaded 34GiB to GPU in 2.02762s (16.7684GiB/s) | 20260505 07:03:38.331 | INFO | Analyzing chunk frames = [1267712,1268224) | 20260505 07:03:38.331 | INFO | MemInfo: { | 20260505 07:03:38.331 | INFO | "gpu" : " 17 GB", | 20260505 07:03:38.331 | INFO | "huge" : "", | 20260505 07:03:38.331 | INFO | "huge_shared" : " 27 GB", | 20260505 07:03:38.331 | INFO | "malloc" : " 14 GB", | 20260505 07:03:38.331 | INFO | "pinned" : " 29 GB" | 20260505 07:03:38.331 | INFO | } | 20260505 07:03:38.331 | INFO | Perf report for the last 5.13934 seconds: | 20260505 07:03:38.331 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:03:38.331 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:38.331 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:38.331 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:38.331 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 07:03:38.331 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:38.331 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:38.331 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:03:38.331 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:38.331 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:38.331 | INFO | Uploaded 36GiB to GPU in 2.16854s (16.6011GiB/s) | 20260505 07:03:38.332 | INFO | Downloaded 34GiB to GPU in 2.05111s (16.5764GiB/s) | 20260505 07:03:43.472 | INFO | Analyzing chunk frames = [1268224,1268736) | 20260505 07:03:43.472 | INFO | MemInfo: { | 20260505 07:03:43.472 | INFO | "gpu" : " 17 GB", | 20260505 07:03:43.472 | INFO | "huge" : "", | 20260505 07:03:43.472 | INFO | "huge_shared" : " 27 GB", | 20260505 07:03:43.472 | INFO | "malloc" : " 14 GB", | 20260505 07:03:43.472 | INFO | "pinned" : " 29 GB" | 20260505 07:03:43.472 | INFO | } | 20260505 07:03:43.472 | INFO | Perf report for the last 5.1412 seconds: | 20260505 07:03:43.472 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:03:43.472 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:43.472 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:43.472 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:43.472 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 07:03:43.472 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:43.473 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:43.473 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:03:43.473 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:43.473 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:43.473 | INFO | Uploaded 36GiB to GPU in 2.11776s (16.9991GiB/s) | 20260505 07:03:43.473 | INFO | Downloaded 34GiB to GPU in 2.03828s (16.6808GiB/s) | 20260505 07:03:48.610 | INFO | Analyzing chunk frames = [1268736,1269248) | 20260505 07:03:48.610 | INFO | MemInfo: { | 20260505 07:03:48.610 | INFO | "gpu" : " 17 GB", | 20260505 07:03:48.610 | INFO | "huge" : "", | 20260505 07:03:48.610 | INFO | "huge_shared" : " 27 GB", | 20260505 07:03:48.610 | INFO | "malloc" : " 14 GB", | 20260505 07:03:48.610 | INFO | "pinned" : " 29 GB" | 20260505 07:03:48.610 | INFO | } | 20260505 07:03:48.610 | INFO | Perf report for the last 5.13729 seconds: | 20260505 07:03:48.610 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:03:48.610 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:48.610 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:48.610 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:48.610 | INFO | ANALYSIS : 64.4%, 3.0, 100.6ms, 0 | 20260505 07:03:48.610 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:48.610 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:48.610 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 07:03:48.610 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:48.610 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:48.610 | INFO | Uploaded 36GiB to GPU in 2.20629s (16.317GiB/s) | 20260505 07:03:48.610 | INFO | Downloaded 34GiB to GPU in 2.16383s (15.7129GiB/s) | 20260505 07:03:53.749 | INFO | Analyzing chunk frames = [1269248,1269760) | 20260505 07:03:53.749 | INFO | MemInfo: { | 20260505 07:03:53.749 | INFO | "gpu" : " 17 GB", | 20260505 07:03:53.749 | INFO | "huge" : "", | 20260505 07:03:53.749 | INFO | "huge_shared" : " 27 GB", | 20260505 07:03:53.749 | INFO | "malloc" : " 14 GB", | 20260505 07:03:53.749 | INFO | "pinned" : " 29 GB" | 20260505 07:03:53.749 | INFO | } | 20260505 07:03:53.749 | INFO | Perf report for the last 5.13951 seconds: | 20260505 07:03:53.749 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:03:53.749 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:53.749 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:53.749 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:53.749 | INFO | ANALYSIS : 64.3%, 3.0, 100.3ms, 0 | 20260505 07:03:53.749 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:53.749 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:53.749 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 07:03:53.749 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:53.749 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:53.750 | INFO | Uploaded 36GiB to GPU in 2.16317s (16.6423GiB/s) | 20260505 07:03:53.750 | INFO | Downloaded 34GiB to GPU in 2.04331s (16.6396GiB/s) | 20260505 07:03:53.750 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 07:03:53.750 | INFO | Total mem usage: 15.3 GiB | 20260505 07:03:53.750 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 07:03:53.750 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -5.96%) | 20260505 07:03:53.750 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 07:03:53.750 | INFO | High Water Breakdown: | 20260505 07:03:53.750 | INFO | 60.6% : 8.76 GiB ( 93.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 07:03:53.750 | INFO | 39.3% : 5.68 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 07:03:53.750 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 07:03:53.750 | INFO | ------------------------------------------ | 20260505 07:03:53.750 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:03:53.750 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 07:03:53.750 | INFO | Total mem usage: 26.5 GiB | 20260505 07:03:53.750 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 07:03:53.750 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 07:03:53.750 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 07:03:53.750 | INFO | High Water Breakdown: | 20260505 07:03:53.750 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:03:53.750 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:03:53.750 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:03:53.750 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:03:53.750 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:03:53.750 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:03:53.750 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 07:03:53.750 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 07:03:53.750 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:03:53.750 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 07:03:53.750 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 07:03:53.750 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 07:03:53.750 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 07:03:53.750 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 07:03:53.750 | INFO | ------------------------------------------ | 20260505 07:03:53.750 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:03:53.750 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 07:03:53.750 | INFO | Total mem usage: 25.6 GiB | 20260505 07:03:53.750 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 07:03:53.750 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 07:03:53.750 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 07:03:53.750 | INFO | High Water Breakdown: | 20260505 07:03:53.750 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 07:03:53.750 | INFO | ------------------------------------------ | 20260505 07:03:53.750 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:03:53.750 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 07:03:53.750 | INFO | Total mem usage: 15.7 GiB | 20260505 07:03:53.750 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 07:03:53.750 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 07:03:53.750 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 07:03:53.750 | INFO | High Water Breakdown: | 20260505 07:03:53.750 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 07:03:53.750 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 07:03:53.750 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 07:03:53.750 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 07:03:53.750 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 07:03:53.750 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 07:03:53.750 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 07:03:53.750 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 07:03:53.750 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 07:03:53.750 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:03:53.750 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 07:03:53.750 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:03:53.750 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 07:03:53.750 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 07:03:53.750 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 07:03:53.750 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 07:03:53.750 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:03:53.751 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 07:03:53.751 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:03:53.751 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:03:53.751 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:03:53.751 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 07:03:53.751 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 07:03:53.751 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:03:53.751 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 07:03:53.751 | INFO | ------------------------------------------ | 20260505 07:03:53.751 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:03:53.751 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 07:03:53.751 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:03:53.751 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:03:53.751 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.53e+01, 1.36e+01 / 1.75e+00 | 20260505 07:03:53.751 | INFO | PerfReport: 0.29%, AggregateStats -- 9.40e-02 / 1.04e-01, 1.10e-02 / 4.61e-03 | 20260505 07:03:53.751 | INFO | PerfReport: 11.58%, Baseline -- 3.82e+00 / 4.47e+00, 6.30e-01 / 2.47e-01 | 20260505 07:03:53.751 | INFO | PerfReport: 3.31%, Histogram -- 1.09e+00 / 1.47e+00, 2.15e-01 / 1.14e-01 | 20260505 07:03:53.751 | INFO | PerfReport: 15.48%, DME -- 4.08e+01 / 4.46e+01, 4.14e+00 / 1.29e+00 | 20260505 07:03:53.751 | INFO | PerfReport: 56.21%, FrameLabeling -- 1.85e+01 / 1.92e+01, 2.06e+00 / 1.21e+00 | 20260505 07:03:53.751 | INFO | PerfReport: 7.64%, PulseAccumulating -- 2.52e+00 / 3.12e+00, 4.48e-01 / 2.84e-01 | 20260505 07:03:53.751 | INFO | PerfReport: 4.02%, Metrics -- 1.33e+00 / 1.75e+00, 3.73e-01 / 2.05e-01 | 20260505 07:03:53.751 | INFO | PerfReport: 1.46%, PulseSerializing -- 4.81e-01 / 5.57e-01, 9.32e-02 / 3.21e-02 | 20260505 07:03:53.751 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 07:03:53.751 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:03:53.751 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:03:53.751 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.24e+01 / 0.00e+00 | 20260505 07:03:53.751 | INFO | PerfReport: 22.22%, Upload -- 2.26e+01 / 0.00e+00, 8.01e+00 / 0.00e+00 | 20260505 07:03:53.751 | INFO | PerfReport: 21.44%, Download -- 2.18e+01 / 0.00e+00, 7.79e+00 / 0.00e+00 | 20260505 07:03:53.751 | INFO | PerfReport: 32.46%, Compute -- 3.30e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 07:03:53.751 | INFO | PerfReport: 1.46%, LockUpload -- 1.48e+00 / 0.00e+00, 7.01e+00 / 0.00e+00 | 20260505 07:03:53.751 | INFO | PerfReport: 0.78%, LockDownload -- 7.93e-01 / 0.00e+00, 2.05e+00 / 0.00e+00 | 20260505 07:03:53.751 | INFO | PerfReport: 21.63%, LockCompute -- 2.20e+01 / 0.00e+00, 1.68e+01 / 0.00e+00 | 20260505 07:03:53.751 | INFO | PerfReport: 0.01%, Misc -- 1.51e-02 / 0.00e+00, 2.21e-02 / 0.00e+00 | 20260505 07:03:53.869 | INFO | Starting a round of Baz IO | 20260505 07:03:57.111 | INFO | Populating RTMetrics report | 20260505 07:03:57.112 | INFO | RTMetrics report populated | 20260505 07:03:57.112 | INFO | Opening tmp RTMetrics json file | 20260505 07:03:57.112 | INFO | Serializing RTMetrics to json | 20260505 07:03:57.114 | INFO | Serialization done, swapping tmp file now... | 20260505 07:03:57.114 | INFO | RTMetrics done | 20260505 07:03:57.235 | INFO | Finished a round of Baz IO: Wrote 2.413 GiB in 3.366 seconds | 20260505 07:03:57.235 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 31.03% / 0.04% / 68.93% | 20260505 07:03:57.235 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.13462% / 5.16574e-05% / 9.93411e-05% | 20260505 07:03:57.235 | INFO | Baz IO overall: | 20260505 07:03:57.235 | INFO | BytesWritten : 679527709153 | 20260505 07:03:57.235 | INFO | NumEvents : 322038259397 | 20260505 07:03:58.889 | INFO | Analyzing chunk frames = [1269760,1270272) | 20260505 07:03:58.889 | INFO | MemInfo: { | 20260505 07:03:58.889 | INFO | "gpu" : " 17 GB", | 20260505 07:03:58.889 | INFO | "huge" : "", | 20260505 07:03:58.889 | INFO | "huge_shared" : " 27 GB", | 20260505 07:03:58.889 | INFO | "malloc" : " 14 GB", | 20260505 07:03:58.889 | INFO | "pinned" : " 29 GB" | 20260505 07:03:58.889 | INFO | } | 20260505 07:03:58.889 | INFO | Perf report for the last 5.13942 seconds: | 20260505 07:03:58.889 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:03:58.889 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:58.889 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:58.889 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:58.889 | INFO | ANALYSIS : 64.3%, 2.9, 98.0ms, 0 | 20260505 07:03:58.889 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:03:58.889 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 07:03:58.889 | INFO | PRE_HQ : 63.0%, 1.9, 62.9ms, 0 | 20260505 07:03:58.889 | INFO | BAZWRITER : 64.8%, 4.1, 138.2ms, 0 | 20260505 07:03:58.889 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 07:03:58.889 | INFO | Uploaded 36GiB to GPU in 2.14591s (16.7761GiB/s) | 20260505 07:03:58.889 | INFO | Downloaded 38GiB to GPU in 2.30999s (16.4503GiB/s) | 20260505 07:04:04.028 | INFO | Analyzing chunk frames = [1270272,1270784) | 20260505 07:04:04.028 | INFO | MemInfo: { | 20260505 07:04:04.028 | INFO | "gpu" : " 17 GB", | 20260505 07:04:04.028 | INFO | "huge" : "", | 20260505 07:04:04.028 | INFO | "huge_shared" : " 27 GB", | 20260505 07:04:04.028 | INFO | "malloc" : " 14 GB", | 20260505 07:04:04.028 | INFO | "pinned" : " 29 GB" | 20260505 07:04:04.028 | INFO | } | 20260505 07:04:04.028 | INFO | Perf report for the last 5.13922 seconds: | 20260505 07:04:04.028 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:04:04.028 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:04.028 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:04.028 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:04.028 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 07:04:04.028 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:04.028 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:04.028 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:04:04.028 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:04.028 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:04.028 | INFO | Uploaded 36GiB to GPU in 2.11067s (17.0562GiB/s) | 20260505 07:04:04.029 | INFO | Downloaded 34GiB to GPU in 2.03306s (16.7235GiB/s) | 20260505 07:04:09.167 | INFO | Analyzing chunk frames = [1270784,1271296) | 20260505 07:04:09.167 | INFO | MemInfo: { | 20260505 07:04:09.167 | INFO | "gpu" : " 17 GB", | 20260505 07:04:09.167 | INFO | "huge" : "", | 20260505 07:04:09.167 | INFO | "huge_shared" : " 27 GB", | 20260505 07:04:09.167 | INFO | "malloc" : " 14 GB", | 20260505 07:04:09.167 | INFO | "pinned" : " 29 GB" | 20260505 07:04:09.167 | INFO | } | 20260505 07:04:09.167 | INFO | Perf report for the last 5.13948 seconds: | 20260505 07:04:09.167 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:04:09.167 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:09.167 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:09.168 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:09.168 | INFO | ANALYSIS : 63.6%, 3.0, 98.9ms, 0 | 20260505 07:04:09.168 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:09.168 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:09.168 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 07:04:09.168 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:09.168 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:09.168 | INFO | Uploaded 36GiB to GPU in 2.15189s (16.7295GiB/s) | 20260505 07:04:09.168 | INFO | Downloaded 34GiB to GPU in 2.05145s (16.5736GiB/s) | 20260505 07:04:14.307 | INFO | Analyzing chunk frames = [1271296,1271808) | 20260505 07:04:14.307 | INFO | MemInfo: { | 20260505 07:04:14.307 | INFO | "gpu" : " 17 GB", | 20260505 07:04:14.307 | INFO | "huge" : "", | 20260505 07:04:14.307 | INFO | "huge_shared" : " 27 GB", | 20260505 07:04:14.307 | INFO | "malloc" : " 14 GB", | 20260505 07:04:14.307 | INFO | "pinned" : " 29 GB" | 20260505 07:04:14.307 | INFO | } | 20260505 07:04:14.307 | INFO | Perf report for the last 5.13941 seconds: | 20260505 07:04:14.307 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:04:14.307 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:14.307 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:14.307 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:14.307 | INFO | ANALYSIS : 63.4%, 3.0, 98.4ms, 0 | 20260505 07:04:14.307 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:14.307 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:14.307 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 07:04:14.307 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:14.307 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:14.307 | INFO | Uploaded 36GiB to GPU in 2.12656s (16.9288GiB/s) | 20260505 07:04:14.307 | INFO | Downloaded 34GiB to GPU in 2.02534s (16.7873GiB/s) | 20260505 07:04:19.446 | INFO | Analyzing chunk frames = [1271808,1272320) | 20260505 07:04:19.446 | INFO | MemInfo: { | 20260505 07:04:19.446 | INFO | "gpu" : " 17 GB", | 20260505 07:04:19.446 | INFO | "huge" : "", | 20260505 07:04:19.446 | INFO | "huge_shared" : " 27 GB", | 20260505 07:04:19.446 | INFO | "malloc" : " 14 GB", | 20260505 07:04:19.446 | INFO | "pinned" : " 29 GB" | 20260505 07:04:19.446 | INFO | } | 20260505 07:04:19.446 | INFO | Perf report for the last 5.13936 seconds: | 20260505 07:04:19.446 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:04:19.446 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:19.446 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:19.446 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:19.446 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 07:04:19.446 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:19.446 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:19.446 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 07:04:19.446 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:19.446 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:19.447 | INFO | Uploaded 36GiB to GPU in 2.16846s (16.6016GiB/s) | 20260505 07:04:19.447 | INFO | Downloaded 34GiB to GPU in 2.04671s (16.6121GiB/s) | 20260505 07:04:24.587 | INFO | Analyzing chunk frames = [1272320,1272832) | 20260505 07:04:24.587 | INFO | MemInfo: { | 20260505 07:04:24.587 | INFO | "gpu" : " 17 GB", | 20260505 07:04:24.587 | INFO | "huge" : "", | 20260505 07:04:24.587 | INFO | "huge_shared" : " 27 GB", | 20260505 07:04:24.587 | INFO | "malloc" : " 14 GB", | 20260505 07:04:24.587 | INFO | "pinned" : " 29 GB" | 20260505 07:04:24.587 | INFO | } | 20260505 07:04:24.587 | INFO | Perf report for the last 5.14117 seconds: | 20260505 07:04:24.587 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:04:24.587 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:24.587 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:24.588 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:24.588 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 07:04:24.588 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:24.588 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:24.588 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 07:04:24.588 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:24.588 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:24.588 | INFO | Uploaded 36GiB to GPU in 2.1589s (16.6752GiB/s) | 20260505 07:04:24.588 | INFO | Downloaded 34GiB to GPU in 2.0613s (16.4945GiB/s) | 20260505 07:04:29.724 | INFO | Analyzing chunk frames = [1272832,1273344) | 20260505 07:04:29.724 | INFO | MemInfo: { | 20260505 07:04:29.724 | INFO | "gpu" : " 17 GB", | 20260505 07:04:29.724 | INFO | "huge" : "", | 20260505 07:04:29.724 | INFO | "huge_shared" : " 27 GB", | 20260505 07:04:29.725 | INFO | "malloc" : " 14 GB", | 20260505 07:04:29.725 | INFO | "pinned" : " 29 GB" | 20260505 07:04:29.725 | INFO | } | 20260505 07:04:29.725 | INFO | Perf report for the last 5.1371 seconds: | 20260505 07:04:29.725 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:04:29.725 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:29.725 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:29.725 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:29.725 | INFO | ANALYSIS : 64.8%, 3.0, 101.1ms, 0 | 20260505 07:04:29.725 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:29.725 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:29.725 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260505 07:04:29.725 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:29.725 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:29.725 | INFO | Uploaded 36GiB to GPU in 2.22769s (16.1602GiB/s) | 20260505 07:04:29.725 | INFO | Downloaded 34GiB to GPU in 2.147s (15.8361GiB/s) | 20260505 07:04:34.864 | INFO | Analyzing chunk frames = [1273344,1273856) | 20260505 07:04:34.864 | INFO | MemInfo: { | 20260505 07:04:34.864 | INFO | "gpu" : " 17 GB", | 20260505 07:04:34.864 | INFO | "huge" : "", | 20260505 07:04:34.864 | INFO | "huge_shared" : " 27 GB", | 20260505 07:04:34.864 | INFO | "malloc" : " 14 GB", | 20260505 07:04:34.864 | INFO | "pinned" : " 29 GB" | 20260505 07:04:34.864 | INFO | } | 20260505 07:04:34.864 | INFO | Perf report for the last 5.13958 seconds: | 20260505 07:04:34.864 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:04:34.864 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:34.864 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:34.864 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:34.864 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 07:04:34.864 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:34.864 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:34.864 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 07:04:34.864 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:34.864 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:34.864 | INFO | Uploaded 36GiB to GPU in 2.16382s (16.6372GiB/s) | 20260505 07:04:34.864 | INFO | Downloaded 34GiB to GPU in 2.04251s (16.6462GiB/s) | 20260505 07:04:34.979 | INFO | Starting a round of Baz IO | 20260505 07:04:38.273 | INFO | Populating RTMetrics report | 20260505 07:04:38.273 | INFO | RTMetrics report populated | 20260505 07:04:38.273 | INFO | Opening tmp RTMetrics json file | 20260505 07:04:38.310 | INFO | Serializing RTMetrics to json | 20260505 07:04:38.335 | INFO | Serialization done, swapping tmp file now... | 20260505 07:04:38.349 | INFO | RTMetrics done | 20260505 07:04:38.391 | INFO | Finished a round of Baz IO: Wrote 2.414 GiB in 3.412 seconds | 20260505 07:04:38.391 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 31.00% / 0.04% / 68.97% | 20260505 07:04:38.391 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.11443% / 4.76837e-05% / 8.74201e-05% | 20260505 07:04:38.391 | INFO | Baz IO overall: | 20260505 07:04:38.391 | INFO | BytesWritten : 682119392033 | 20260505 07:04:38.391 | INFO | NumEvents : 323278285592 | 20260505 07:04:40.003 | INFO | Analyzing chunk frames = [1273856,1274368) | 20260505 07:04:40.003 | INFO | MemInfo: { | 20260505 07:04:40.003 | INFO | "gpu" : " 17 GB", | 20260505 07:04:40.003 | INFO | "huge" : "", | 20260505 07:04:40.003 | INFO | "huge_shared" : " 27 GB", | 20260505 07:04:40.003 | INFO | "malloc" : " 14 GB", | 20260505 07:04:40.003 | INFO | "pinned" : " 29 GB" | 20260505 07:04:40.003 | INFO | } | 20260505 07:04:40.003 | INFO | Perf report for the last 5.13933 seconds: | 20260505 07:04:40.003 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:04:40.003 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:40.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:40.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:40.003 | INFO | ANALYSIS : 65.2%, 2.9, 99.1ms, 0 | 20260505 07:04:40.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:40.003 | INFO | RT_METRICS : 47.4%, 1.0, 24.9ms, 0 | 20260505 07:04:40.003 | INFO | PRE_HQ : 64.0%, 1.9, 62.9ms, 0 | 20260505 07:04:40.003 | INFO | BAZWRITER : 65.7%, 4.1, 139.4ms, 0 | 20260505 07:04:40.003 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 07:04:40.004 | INFO | Uploaded 36GiB to GPU in 2.21424s (16.2584GiB/s) | 20260505 07:04:40.004 | INFO | Downloaded 38GiB to GPU in 2.29656s (16.5465GiB/s) | 20260505 07:04:45.143 | INFO | Analyzing chunk frames = [1274368,1274880) | 20260505 07:04:45.143 | INFO | MemInfo: { | 20260505 07:04:45.143 | INFO | "gpu" : " 17 GB", | 20260505 07:04:45.143 | INFO | "huge" : "", | 20260505 07:04:45.143 | INFO | "huge_shared" : " 27 GB", | 20260505 07:04:45.143 | INFO | "malloc" : " 14 GB", | 20260505 07:04:45.143 | INFO | "pinned" : " 29 GB" | 20260505 07:04:45.143 | INFO | } | 20260505 07:04:45.143 | INFO | Perf report for the last 5.1394 seconds: | 20260505 07:04:45.143 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:04:45.143 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:45.143 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:45.143 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:45.143 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 07:04:45.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:45.143 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:45.143 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 07:04:45.143 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:45.143 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:45.143 | INFO | Uploaded 36GiB to GPU in 2.11754s (17.0009GiB/s) | 20260505 07:04:45.143 | INFO | Downloaded 34GiB to GPU in 2.03428s (16.7136GiB/s) | 20260505 07:04:50.282 | INFO | Analyzing chunk frames = [1274880,1275392) | 20260505 07:04:50.282 | INFO | MemInfo: { | 20260505 07:04:50.282 | INFO | "gpu" : " 17 GB", | 20260505 07:04:50.282 | INFO | "huge" : "", | 20260505 07:04:50.282 | INFO | "huge_shared" : " 27 GB", | 20260505 07:04:50.282 | INFO | "malloc" : " 14 GB", | 20260505 07:04:50.282 | INFO | "pinned" : " 29 GB" | 20260505 07:04:50.282 | INFO | } | 20260505 07:04:50.282 | INFO | Perf report for the last 5.13912 seconds: | 20260505 07:04:50.282 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:04:50.282 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:50.282 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:50.282 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:50.282 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 07:04:50.282 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:50.282 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:50.282 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 07:04:50.282 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:50.282 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:50.283 | INFO | Uploaded 36GiB to GPU in 2.14374s (16.793GiB/s) | 20260505 07:04:50.283 | INFO | Downloaded 34GiB to GPU in 2.0486s (16.5967GiB/s) | 20260505 07:04:55.421 | INFO | Analyzing chunk frames = [1275392,1275904) | 20260505 07:04:55.421 | INFO | MemInfo: { | 20260505 07:04:55.421 | INFO | "gpu" : " 17 GB", | 20260505 07:04:55.421 | INFO | "huge" : "", | 20260505 07:04:55.421 | INFO | "huge_shared" : " 27 GB", | 20260505 07:04:55.421 | INFO | "malloc" : " 14 GB", | 20260505 07:04:55.421 | INFO | "pinned" : " 29 GB" | 20260505 07:04:55.421 | INFO | } | 20260505 07:04:55.422 | INFO | Perf report for the last 5.13948 seconds: | 20260505 07:04:55.422 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:04:55.422 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:55.422 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:55.422 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:55.422 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260505 07:04:55.422 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:55.422 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:55.422 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 07:04:55.422 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:55.422 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:04:55.422 | INFO | Uploaded 36GiB to GPU in 2.14361s (16.7941GiB/s) | 20260505 07:04:55.422 | INFO | Downloaded 34GiB to GPU in 2.02927s (16.7548GiB/s) | 20260505 07:05:00.561 | INFO | Analyzing chunk frames = [1275904,1276416) | 20260505 07:05:00.561 | INFO | MemInfo: { | 20260505 07:05:00.561 | INFO | "gpu" : " 17 GB", | 20260505 07:05:00.561 | INFO | "huge" : "", | 20260505 07:05:00.561 | INFO | "huge_shared" : " 27 GB", | 20260505 07:05:00.561 | INFO | "malloc" : " 14 GB", | 20260505 07:05:00.561 | INFO | "pinned" : " 29 GB" | 20260505 07:05:00.561 | INFO | } | 20260505 07:05:00.561 | INFO | Perf report for the last 5.13927 seconds: | 20260505 07:05:00.561 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:05:00.561 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:00.561 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:00.561 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:00.561 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 07:05:00.561 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:00.561 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:00.561 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:05:00.561 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:00.561 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:00.561 | INFO | Uploaded 36GiB to GPU in 2.15909s (16.6737GiB/s) | 20260505 07:05:00.561 | INFO | Downloaded 34GiB to GPU in 2.04588s (16.6187GiB/s) | 20260505 07:05:05.702 | INFO | Analyzing chunk frames = [1276416,1276928) | 20260505 07:05:05.702 | INFO | MemInfo: { | 20260505 07:05:05.702 | INFO | "gpu" : " 17 GB", | 20260505 07:05:05.702 | INFO | "huge" : "", | 20260505 07:05:05.702 | INFO | "huge_shared" : " 27 GB", | 20260505 07:05:05.702 | INFO | "malloc" : " 14 GB", | 20260505 07:05:05.702 | INFO | "pinned" : " 29 GB" | 20260505 07:05:05.702 | INFO | } | 20260505 07:05:05.702 | INFO | Perf report for the last 5.14151 seconds: | 20260505 07:05:05.702 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:05:05.702 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:05.702 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:05.702 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:05.702 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 07:05:05.702 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:05.702 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:05.703 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 07:05:05.703 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:05.703 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:05.703 | INFO | Uploaded 36GiB to GPU in 2.1641s (16.6351GiB/s) | 20260505 07:05:05.703 | INFO | Downloaded 34GiB to GPU in 2.05502s (16.5448GiB/s) | 20260505 07:05:10.840 | INFO | Analyzing chunk frames = [1276928,1277440) | 20260505 07:05:10.840 | INFO | MemInfo: { | 20260505 07:05:10.840 | INFO | "gpu" : " 17 GB", | 20260505 07:05:10.840 | INFO | "huge" : "", | 20260505 07:05:10.840 | INFO | "huge_shared" : " 27 GB", | 20260505 07:05:10.840 | INFO | "malloc" : " 14 GB", | 20260505 07:05:10.840 | INFO | "pinned" : " 29 GB" | 20260505 07:05:10.840 | INFO | } | 20260505 07:05:10.840 | INFO | Perf report for the last 5.13774 seconds: | 20260505 07:05:10.840 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:05:10.840 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:10.840 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:10.840 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:10.840 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 07:05:10.840 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:10.840 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:10.840 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260505 07:05:10.840 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:10.840 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:10.840 | INFO | Uploaded 36GiB to GPU in 2.19787s (16.3795GiB/s) | 20260505 07:05:10.840 | INFO | Downloaded 34GiB to GPU in 2.16072s (15.7355GiB/s) | 20260505 07:05:15.979 | INFO | Analyzing chunk frames = [1277440,1277952) | 20260505 07:05:15.979 | INFO | MemInfo: { | 20260505 07:05:15.979 | INFO | "gpu" : " 17 GB", | 20260505 07:05:15.979 | INFO | "huge" : "", | 20260505 07:05:15.979 | INFO | "huge_shared" : " 27 GB", | 20260505 07:05:15.979 | INFO | "malloc" : " 14 GB", | 20260505 07:05:15.979 | INFO | "pinned" : " 29 GB" | 20260505 07:05:15.979 | INFO | } | 20260505 07:05:15.979 | INFO | Perf report for the last 5.13883 seconds: | 20260505 07:05:15.979 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:05:15.979 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:15.979 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:15.979 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:15.979 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 07:05:15.979 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:15.979 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:15.979 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 07:05:15.979 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:15.979 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:15.979 | INFO | Uploaded 36GiB to GPU in 2.16806s (16.6047GiB/s) | 20260505 07:05:15.979 | INFO | Downloaded 34GiB to GPU in 2.04749s (16.6057GiB/s) | 20260505 07:05:15.979 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 07:05:15.979 | INFO | Total mem usage: 15.3 GiB | 20260505 07:05:15.979 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 07:05:15.979 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -5.92%) | 20260505 07:05:15.979 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 07:05:15.979 | INFO | High Water Breakdown: | 20260505 07:05:15.980 | INFO | 60.6% : 8.77 GiB ( 93.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 07:05:15.980 | INFO | 39.2% : 5.68 GiB ( 85.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 07:05:15.980 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 07:05:15.980 | INFO | ------------------------------------------ | 20260505 07:05:15.980 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:05:15.980 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 07:05:15.980 | INFO | Total mem usage: 26.5 GiB | 20260505 07:05:15.980 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 07:05:15.980 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 07:05:15.980 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 07:05:15.980 | INFO | High Water Breakdown: | 20260505 07:05:15.980 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:05:15.980 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:05:15.980 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:05:15.980 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:05:15.980 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:05:15.980 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:05:15.980 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 07:05:15.980 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 07:05:15.980 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:05:15.980 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 07:05:15.980 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 07:05:15.980 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 07:05:15.980 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 07:05:15.980 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 07:05:15.980 | INFO | ------------------------------------------ | 20260505 07:05:15.980 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:05:15.980 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 07:05:15.980 | INFO | Total mem usage: 25.6 GiB | 20260505 07:05:15.980 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 07:05:15.980 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 07:05:15.980 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 07:05:15.980 | INFO | High Water Breakdown: | 20260505 07:05:15.980 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 07:05:15.980 | INFO | ------------------------------------------ | 20260505 07:05:15.980 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:05:15.980 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 07:05:15.980 | INFO | Total mem usage: 15.7 GiB | 20260505 07:05:15.980 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 07:05:15.980 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 07:05:15.980 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 07:05:15.980 | INFO | High Water Breakdown: | 20260505 07:05:15.980 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 07:05:15.980 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 07:05:15.980 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 07:05:15.980 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 07:05:15.980 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 07:05:15.980 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 07:05:15.980 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 07:05:15.980 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 07:05:15.980 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 07:05:15.980 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:05:15.980 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 07:05:15.980 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:05:15.980 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 07:05:15.980 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 07:05:15.980 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 07:05:15.980 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 07:05:15.980 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:05:15.980 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 07:05:15.980 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:05:15.980 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:05:15.980 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:05:15.980 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 07:05:15.980 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 07:05:15.980 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:05:15.980 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 07:05:15.980 | INFO | ------------------------------------------ | 20260505 07:05:15.980 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:05:15.980 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 07:05:15.980 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:05:15.980 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:05:15.980 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.51e+01, 1.37e+01 / 1.59e+00 | 20260505 07:05:15.980 | INFO | PerfReport: 0.28%, AggregateStats -- 9.39e-02 / 1.02e-01, 1.15e-02 / 5.85e-03 | 20260505 07:05:15.980 | INFO | PerfReport: 11.61%, Baseline -- 3.83e+00 / 4.42e+00, 6.31e-01 / 2.94e-01 | 20260505 07:05:15.980 | INFO | PerfReport: 3.31%, Histogram -- 1.09e+00 / 1.47e+00, 2.15e-01 / 1.10e-01 | 20260505 07:05:15.980 | INFO | PerfReport: 15.47%, DME -- 4.08e+01 / 4.46e+01, 4.05e+00 / 1.21e+00 | 20260505 07:05:15.980 | INFO | PerfReport: 56.20%, FrameLabeling -- 1.85e+01 / 1.91e+01, 2.04e+00 / 1.01e+00 | 20260505 07:05:15.980 | INFO | PerfReport: 7.64%, PulseAccumulating -- 2.52e+00 / 3.13e+00, 4.48e-01 / 2.84e-01 | 20260505 07:05:15.980 | INFO | PerfReport: 4.02%, Metrics -- 1.33e+00 / 1.72e+00, 3.72e-01 / 1.71e-01 | 20260505 07:05:15.980 | INFO | PerfReport: 1.46%, PulseSerializing -- 4.81e-01 / 5.58e-01, 9.28e-02 / 3.39e-02 | 20260505 07:05:15.980 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 07:05:15.980 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:05:15.980 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:05:15.980 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.24e+01 / 0.00e+00 | 20260505 07:05:15.980 | INFO | PerfReport: 22.19%, Upload -- 2.25e+01 / 0.00e+00, 7.89e+00 / 0.00e+00 | 20260505 07:05:15.980 | INFO | PerfReport: 21.47%, Download -- 2.18e+01 / 0.00e+00, 7.74e+00 / 0.00e+00 | 20260505 07:05:15.980 | INFO | PerfReport: 32.51%, Compute -- 3.30e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 07:05:15.980 | INFO | PerfReport: 1.50%, LockUpload -- 1.52e+00 / 0.00e+00, 7.05e+00 / 0.00e+00 | 20260505 07:05:15.980 | INFO | PerfReport: 0.80%, LockDownload -- 8.09e-01 / 0.00e+00, 2.22e+00 / 0.00e+00 | 20260505 07:05:15.980 | INFO | PerfReport: 21.51%, LockCompute -- 2.18e+01 / 0.00e+00, 1.67e+01 / 0.00e+00 | 20260505 07:05:15.980 | INFO | PerfReport: 0.01%, Misc -- 1.48e-02 / 0.00e+00, 2.20e-02 / 0.00e+00 | 20260505 07:05:16.101 | INFO | Starting a round of Baz IO | 20260505 07:05:19.342 | INFO | Populating RTMetrics report | 20260505 07:05:19.343 | INFO | RTMetrics report populated | 20260505 07:05:19.343 | INFO | Opening tmp RTMetrics json file | 20260505 07:05:19.353 | INFO | Serializing RTMetrics to json | 20260505 07:05:19.355 | INFO | Serialization done, swapping tmp file now... | 20260505 07:05:19.358 | INFO | RTMetrics done | 20260505 07:05:19.475 | INFO | Finished a round of Baz IO: Wrote 2.415 GiB in 3.374 seconds | 20260505 07:05:19.475 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 30.96% / 0.04% / 69.00% | 20260505 07:05:19.475 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.09572% / 5.5631e-05% / 5.96046e-05% | 20260505 07:05:19.475 | INFO | Baz IO overall: | 20260505 07:05:19.475 | INFO | BytesWritten : 684712259830 | 20260505 07:05:19.475 | INFO | NumEvents : 324518480332 | 20260505 07:05:21.118 | INFO | Analyzing chunk frames = [1277952,1278464) | 20260505 07:05:21.118 | INFO | MemInfo: { | 20260505 07:05:21.118 | INFO | "gpu" : " 17 GB", | 20260505 07:05:21.118 | INFO | "huge" : "", | 20260505 07:05:21.118 | INFO | "huge_shared" : " 27 GB", | 20260505 07:05:21.118 | INFO | "malloc" : " 14 GB", | 20260505 07:05:21.118 | INFO | "pinned" : " 29 GB" | 20260505 07:05:21.118 | INFO | } | 20260505 07:05:21.118 | INFO | Perf report for the last 5.13936 seconds: | 20260505 07:05:21.118 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:05:21.118 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:21.118 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:21.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:21.118 | INFO | ANALYSIS : 64.3%, 2.9, 98.0ms, 0 | 20260505 07:05:21.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:21.118 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260505 07:05:21.118 | INFO | PRE_HQ : 63.1%, 1.9, 63.4ms, 0 | 20260505 07:05:21.118 | INFO | BAZWRITER : 64.9%, 4.2, 139.2ms, 0 | 20260505 07:05:21.118 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 07:05:21.119 | INFO | Uploaded 36GiB to GPU in 2.13632s (16.8514GiB/s) | 20260505 07:05:21.119 | INFO | Downloaded 38GiB to GPU in 2.30776s (16.4662GiB/s) | 20260505 07:05:26.258 | INFO | Analyzing chunk frames = [1278464,1278976) | 20260505 07:05:26.258 | INFO | MemInfo: { | 20260505 07:05:26.258 | INFO | "gpu" : " 17 GB", | 20260505 07:05:26.258 | INFO | "huge" : "", | 20260505 07:05:26.258 | INFO | "huge_shared" : " 27 GB", | 20260505 07:05:26.258 | INFO | "malloc" : " 14 GB", | 20260505 07:05:26.258 | INFO | "pinned" : " 29 GB" | 20260505 07:05:26.258 | INFO | } | 20260505 07:05:26.258 | INFO | Perf report for the last 5.13968 seconds: | 20260505 07:05:26.258 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:05:26.258 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:26.258 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:26.258 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:26.258 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 07:05:26.258 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:26.258 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:26.258 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 07:05:26.258 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:26.258 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:26.258 | INFO | Uploaded 36GiB to GPU in 2.11537s (17.0183GiB/s) | 20260505 07:05:26.258 | INFO | Downloaded 34GiB to GPU in 2.03252s (16.728GiB/s) | 20260505 07:05:31.397 | INFO | Analyzing chunk frames = [1278976,1279488) | 20260505 07:05:31.397 | INFO | MemInfo: { | 20260505 07:05:31.397 | INFO | "gpu" : " 17 GB", | 20260505 07:05:31.397 | INFO | "huge" : "", | 20260505 07:05:31.397 | INFO | "huge_shared" : " 27 GB", | 20260505 07:05:31.397 | INFO | "malloc" : " 14 GB", | 20260505 07:05:31.397 | INFO | "pinned" : " 29 GB" | 20260505 07:05:31.397 | INFO | } | 20260505 07:05:31.397 | INFO | Perf report for the last 5.13925 seconds: | 20260505 07:05:31.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:05:31.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:31.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:31.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:31.397 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260505 07:05:31.397 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:31.397 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:31.397 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 07:05:31.397 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:31.397 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:31.398 | INFO | Uploaded 36GiB to GPU in 2.14912s (16.751GiB/s) | 20260505 07:05:31.398 | INFO | Downloaded 34GiB to GPU in 2.05289s (16.5621GiB/s) | 20260505 07:05:36.536 | INFO | Analyzing chunk frames = [1279488,1280000) | 20260505 07:05:36.536 | INFO | MemInfo: { | 20260505 07:05:36.536 | INFO | "gpu" : " 17 GB", | 20260505 07:05:36.536 | INFO | "huge" : "", | 20260505 07:05:36.536 | INFO | "huge_shared" : " 27 GB", | 20260505 07:05:36.536 | INFO | "malloc" : " 14 GB", | 20260505 07:05:36.536 | INFO | "pinned" : " 29 GB" | 20260505 07:05:36.536 | INFO | } | 20260505 07:05:36.537 | INFO | Perf report for the last 5.13921 seconds: | 20260505 07:05:36.537 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:05:36.537 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:36.537 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:36.537 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:36.537 | INFO | ANALYSIS : 63.4%, 3.0, 98.4ms, 0 | 20260505 07:05:36.537 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:36.537 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:36.537 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 07:05:36.537 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:36.537 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:36.537 | INFO | Uploaded 36GiB to GPU in 2.13764s (16.841GiB/s) | 20260505 07:05:36.537 | INFO | Downloaded 34GiB to GPU in 2.02501s (16.79GiB/s) | 20260505 07:05:41.676 | INFO | Analyzing chunk frames = [1280000,1280512) | 20260505 07:05:41.676 | INFO | MemInfo: { | 20260505 07:05:41.676 | INFO | "gpu" : " 17 GB", | 20260505 07:05:41.676 | INFO | "huge" : "", | 20260505 07:05:41.676 | INFO | "huge_shared" : " 27 GB", | 20260505 07:05:41.676 | INFO | "malloc" : " 14 GB", | 20260505 07:05:41.676 | INFO | "pinned" : " 29 GB" | 20260505 07:05:41.676 | INFO | } | 20260505 07:05:41.676 | INFO | Perf report for the last 5.13956 seconds: | 20260505 07:05:41.676 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:05:41.676 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:41.676 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:41.676 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:41.676 | INFO | ANALYSIS : 63.2%, 3.0, 98.8ms, 0 | 20260505 07:05:41.676 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:41.676 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:41.676 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 07:05:41.676 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:41.676 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:41.676 | INFO | Uploaded 36GiB to GPU in 2.1569s (16.6906GiB/s) | 20260505 07:05:41.676 | INFO | Downloaded 34GiB to GPU in 2.04505s (16.6255GiB/s) | 20260505 07:05:46.816 | INFO | Analyzing chunk frames = [1280512,1281024) | 20260505 07:05:46.817 | INFO | MemInfo: { | 20260505 07:05:46.817 | INFO | "gpu" : " 17 GB", | 20260505 07:05:46.817 | INFO | "huge" : "", | 20260505 07:05:46.817 | INFO | "huge_shared" : " 27 GB", | 20260505 07:05:46.817 | INFO | "malloc" : " 14 GB", | 20260505 07:05:46.817 | INFO | "pinned" : " 29 GB" | 20260505 07:05:46.817 | INFO | } | 20260505 07:05:46.817 | INFO | Perf report for the last 5.14069 seconds: | 20260505 07:05:46.817 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:05:46.817 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:46.817 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:46.817 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:46.817 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 07:05:46.817 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:46.817 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:46.817 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 07:05:46.817 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:46.817 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:46.817 | INFO | Uploaded 36GiB to GPU in 2.16519s (16.6267GiB/s) | 20260505 07:05:46.817 | INFO | Downloaded 34GiB to GPU in 2.05673s (16.5311GiB/s) | 20260505 07:05:51.958 | INFO | Analyzing chunk frames = [1281024,1281536) | 20260505 07:05:51.958 | INFO | MemInfo: { | 20260505 07:05:51.958 | INFO | "gpu" : " 17 GB", | 20260505 07:05:51.958 | INFO | "huge" : "", | 20260505 07:05:51.958 | INFO | "huge_shared" : " 27 GB", | 20260505 07:05:51.958 | INFO | "malloc" : " 14 GB", | 20260505 07:05:51.959 | INFO | "pinned" : " 29 GB" | 20260505 07:05:51.959 | INFO | } | 20260505 07:05:51.959 | INFO | Perf report for the last 5.14179 seconds: | 20260505 07:05:51.959 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:05:51.959 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:51.959 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:51.959 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:51.959 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 07:05:51.959 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:51.959 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:51.959 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260505 07:05:51.959 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:51.959 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:51.959 | INFO | Uploaded 36GiB to GPU in 2.20031s (16.3614GiB/s) | 20260505 07:05:51.959 | INFO | Downloaded 34GiB to GPU in 2.16141s (15.7305GiB/s) | 20260505 07:05:57.098 | INFO | Analyzing chunk frames = [1281536,1282048) | 20260505 07:05:57.098 | INFO | MemInfo: { | 20260505 07:05:57.098 | INFO | "gpu" : " 17 GB", | 20260505 07:05:57.098 | INFO | "huge" : "", | 20260505 07:05:57.098 | INFO | "huge_shared" : " 27 GB", | 20260505 07:05:57.098 | INFO | "malloc" : " 14 GB", | 20260505 07:05:57.098 | INFO | "pinned" : " 29 GB" | 20260505 07:05:57.098 | INFO | } | 20260505 07:05:57.098 | INFO | Perf report for the last 5.13952 seconds: | 20260505 07:05:57.098 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:05:57.098 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:57.098 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:57.098 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:57.098 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 07:05:57.098 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:57.098 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:57.098 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 07:05:57.098 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:57.098 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:05:57.098 | INFO | Uploaded 36GiB to GPU in 2.15971s (16.6689GiB/s) | 20260505 07:05:57.098 | INFO | Downloaded 34GiB to GPU in 2.0197s (16.8341GiB/s) | 20260505 07:05:57.232 | INFO | Starting a round of Baz IO | 20260505 07:06:00.524 | INFO | Populating RTMetrics report | 20260505 07:06:00.524 | INFO | RTMetrics report populated | 20260505 07:06:00.524 | INFO | Opening tmp RTMetrics json file | 20260505 07:06:00.541 | INFO | Serializing RTMetrics to json | 20260505 07:06:00.542 | INFO | Serialization done, swapping tmp file now... | 20260505 07:06:00.543 | INFO | RTMetrics done | 20260505 07:06:00.649 | INFO | Finished a round of Baz IO: Wrote 2.418 GiB in 3.416 seconds | 20260505 07:06:00.649 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 30.92% / 0.04% / 69.04% | 20260505 07:06:00.649 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.09438% / 5.96046e-05% / 4.37101e-05% | 20260505 07:06:00.649 | INFO | Baz IO overall: | 20260505 07:06:00.649 | INFO | BytesWritten : 687308726571 | 20260505 07:06:00.649 | INFO | NumEvents : 325761436164 | 20260505 07:06:02.233 | INFO | Analyzing chunk frames = [1282048,1282560) | 20260505 07:06:02.233 | INFO | MemInfo: { | 20260505 07:06:02.233 | INFO | "gpu" : " 17 GB", | 20260505 07:06:02.233 | INFO | "huge" : "", | 20260505 07:06:02.233 | INFO | "huge_shared" : " 27 GB", | 20260505 07:06:02.237 | INFO | "malloc" : " 14 GB", | 20260505 07:06:02.237 | INFO | "pinned" : " 29 GB" | 20260505 07:06:02.237 | INFO | } | 20260505 07:06:02.237 | INFO | Perf report for the last 5.13853 seconds: | 20260505 07:06:02.237 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:06:02.237 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:02.237 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:02.237 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:02.237 | INFO | ANALYSIS : 65.5%, 2.9, 99.1ms, 0 | 20260505 07:06:02.237 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:02.237 | INFO | RT_METRICS : 46.4%, 1.0, 24.4ms, 0 | 20260505 07:06:02.237 | INFO | PRE_HQ : 64.2%, 1.9, 63.4ms, 0 | 20260505 07:06:02.237 | INFO | BAZWRITER : 65.8%, 4.2, 141.5ms, 0 | 20260505 07:06:02.237 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 07:06:02.237 | INFO | Uploaded 36GiB to GPU in 2.19935s (16.3685GiB/s) | 20260505 07:06:02.237 | INFO | Downloaded 38GiB to GPU in 2.35615s (16.128GiB/s) | 20260505 07:06:07.372 | INFO | Analyzing chunk frames = [1282560,1283072) | 20260505 07:06:07.372 | INFO | MemInfo: { | 20260505 07:06:07.372 | INFO | "gpu" : " 17 GB", | 20260505 07:06:07.372 | INFO | "huge" : "", | 20260505 07:06:07.372 | INFO | "huge_shared" : " 27 GB", | 20260505 07:06:07.376 | INFO | "malloc" : " 14 GB", | 20260505 07:06:07.376 | INFO | "pinned" : " 29 GB" | 20260505 07:06:07.376 | INFO | } | 20260505 07:06:07.376 | INFO | Perf report for the last 5.13947 seconds: | 20260505 07:06:07.376 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:06:07.376 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:07.376 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:07.376 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:07.376 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 07:06:07.376 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:07.376 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:07.376 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 07:06:07.376 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:07.376 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:07.376 | INFO | Uploaded 36GiB to GPU in 2.1246s (16.9444GiB/s) | 20260505 07:06:07.376 | INFO | Downloaded 34GiB to GPU in 2.03731s (16.6887GiB/s) | 20260505 07:06:12.512 | INFO | Analyzing chunk frames = [1283072,1283584) | 20260505 07:06:12.512 | INFO | MemInfo: { | 20260505 07:06:12.512 | INFO | "gpu" : " 17 GB", | 20260505 07:06:12.512 | INFO | "huge" : "", | 20260505 07:06:12.512 | INFO | "huge_shared" : " 27 GB", | 20260505 07:06:12.512 | INFO | "malloc" : " 14 GB", | 20260505 07:06:12.512 | INFO | "pinned" : " 29 GB" | 20260505 07:06:12.512 | INFO | } | 20260505 07:06:12.512 | INFO | Perf report for the last 5.13577 seconds: | 20260505 07:06:12.512 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:06:12.512 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:12.512 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:12.512 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:12.512 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260505 07:06:12.512 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:12.512 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:12.512 | INFO | PRE_HQ : 30.2%, 1.0, 15.6ms, 0 | 20260505 07:06:12.512 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:12.512 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:12.512 | INFO | Uploaded 36GiB to GPU in 2.15251s (16.7246GiB/s) | 20260505 07:06:12.512 | INFO | Downloaded 34GiB to GPU in 2.05274s (16.5633GiB/s) | 20260505 07:06:17.651 | INFO | Analyzing chunk frames = [1283584,1284096) | 20260505 07:06:17.651 | INFO | MemInfo: { | 20260505 07:06:17.651 | INFO | "gpu" : " 17 GB", | 20260505 07:06:17.651 | INFO | "huge" : "", | 20260505 07:06:17.651 | INFO | "huge_shared" : " 27 GB", | 20260505 07:06:17.651 | INFO | "malloc" : " 14 GB", | 20260505 07:06:17.651 | INFO | "pinned" : " 29 GB" | 20260505 07:06:17.651 | INFO | } | 20260505 07:06:17.651 | INFO | Perf report for the last 5.13921 seconds: | 20260505 07:06:17.651 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:06:17.651 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:17.651 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:17.651 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:17.651 | INFO | ANALYSIS : 63.4%, 3.0, 98.5ms, 0 | 20260505 07:06:17.651 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:17.651 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:17.651 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 07:06:17.651 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:17.651 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:17.651 | INFO | Uploaded 36GiB to GPU in 2.12694s (16.9257GiB/s) | 20260505 07:06:17.652 | INFO | Downloaded 34GiB to GPU in 2.02572s (16.7842GiB/s) | 20260505 07:06:22.790 | INFO | Analyzing chunk frames = [1284096,1284608) | 20260505 07:06:22.790 | INFO | MemInfo: { | 20260505 07:06:22.790 | INFO | "gpu" : " 17 GB", | 20260505 07:06:22.790 | INFO | "huge" : "", | 20260505 07:06:22.790 | INFO | "huge_shared" : " 27 GB", | 20260505 07:06:22.790 | INFO | "malloc" : " 14 GB", | 20260505 07:06:22.790 | INFO | "pinned" : " 29 GB" | 20260505 07:06:22.790 | INFO | } | 20260505 07:06:22.791 | INFO | Perf report for the last 5.13935 seconds: | 20260505 07:06:22.791 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:06:22.791 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:22.791 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:22.791 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:22.791 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 07:06:22.791 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:22.791 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:22.791 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260505 07:06:22.791 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:22.791 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:22.791 | INFO | Uploaded 36GiB to GPU in 2.15846s (16.6785GiB/s) | 20260505 07:06:22.791 | INFO | Downloaded 34GiB to GPU in 2.04828s (16.5993GiB/s) | 20260505 07:06:27.931 | INFO | Analyzing chunk frames = [1284608,1285120) | 20260505 07:06:27.931 | INFO | MemInfo: { | 20260505 07:06:27.931 | INFO | "gpu" : " 17 GB", | 20260505 07:06:27.931 | INFO | "huge" : "", | 20260505 07:06:27.931 | INFO | "huge_shared" : " 27 GB", | 20260505 07:06:27.931 | INFO | "malloc" : " 14 GB", | 20260505 07:06:27.931 | INFO | "pinned" : " 29 GB" | 20260505 07:06:27.931 | INFO | } | 20260505 07:06:27.931 | INFO | Perf report for the last 5.14097 seconds: | 20260505 07:06:27.932 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:06:27.932 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:27.932 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:27.932 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:27.932 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 07:06:27.932 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:27.932 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:27.932 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 07:06:27.932 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:27.932 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:27.932 | INFO | Uploaded 36GiB to GPU in 2.125s (16.9412GiB/s) | 20260505 07:06:27.932 | INFO | Downloaded 34GiB to GPU in 2.03879s (16.6766GiB/s) | 20260505 07:06:33.069 | INFO | Analyzing chunk frames = [1285120,1285632) | 20260505 07:06:33.069 | INFO | MemInfo: { | 20260505 07:06:33.069 | INFO | "gpu" : " 17 GB", | 20260505 07:06:33.069 | INFO | "huge" : "", | 20260505 07:06:33.069 | INFO | "huge_shared" : " 27 GB", | 20260505 07:06:33.069 | INFO | "malloc" : " 14 GB", | 20260505 07:06:33.069 | INFO | "pinned" : " 29 GB" | 20260505 07:06:33.069 | INFO | } | 20260505 07:06:33.069 | INFO | Perf report for the last 5.13765 seconds: | 20260505 07:06:33.069 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:06:33.069 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:33.069 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:33.069 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:33.069 | INFO | ANALYSIS : 64.3%, 3.0, 100.3ms, 0 | 20260505 07:06:33.069 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:33.069 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:33.069 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260505 07:06:33.069 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:33.069 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:33.070 | INFO | Uploaded 36GiB to GPU in 2.20875s (16.2988GiB/s) | 20260505 07:06:33.070 | INFO | Downloaded 34GiB to GPU in 2.1609s (15.7342GiB/s) | 20260505 07:06:38.208 | INFO | Analyzing chunk frames = [1285632,1286144) | 20260505 07:06:38.208 | INFO | MemInfo: { | 20260505 07:06:38.208 | INFO | "gpu" : " 17 GB", | 20260505 07:06:38.208 | INFO | "huge" : "", | 20260505 07:06:38.208 | INFO | "huge_shared" : " 27 GB", | 20260505 07:06:38.208 | INFO | "malloc" : " 14 GB", | 20260505 07:06:38.212 | INFO | "pinned" : " 29 GB" | 20260505 07:06:38.212 | INFO | } | 20260505 07:06:38.212 | INFO | Perf report for the last 5.14328 seconds: | 20260505 07:06:38.212 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:06:38.212 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:38.212 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:38.212 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:38.213 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 07:06:38.213 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:38.213 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:38.213 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 07:06:38.213 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:38.213 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:38.213 | INFO | Uploaded 36GiB to GPU in 2.16106s (16.6585GiB/s) | 20260505 07:06:38.213 | INFO | Downloaded 34GiB to GPU in 2.04177s (16.6522GiB/s) | 20260505 07:06:38.213 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 07:06:38.213 | INFO | Total mem usage: 15.3 GiB | 20260505 07:06:38.213 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 07:06:38.213 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -5.18%) | 20260505 07:06:38.213 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 07:06:38.213 | INFO | High Water Breakdown: | 20260505 07:06:38.213 | INFO | 60.7% : 8.85 GiB ( 94.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 07:06:38.213 | INFO | 39.1% : 5.7 GiB ( 85.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 07:06:38.213 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 07:06:38.213 | INFO | ------------------------------------------ | 20260505 07:06:38.213 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:06:38.213 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 07:06:38.213 | INFO | Total mem usage: 26.5 GiB | 20260505 07:06:38.213 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 07:06:38.213 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 07:06:38.213 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 07:06:38.213 | INFO | High Water Breakdown: | 20260505 07:06:38.213 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:06:38.213 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:06:38.213 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:06:38.213 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:06:38.213 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:06:38.213 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:06:38.213 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 07:06:38.213 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 07:06:38.213 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:06:38.213 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 07:06:38.213 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 07:06:38.213 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 07:06:38.213 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 07:06:38.213 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 07:06:38.213 | INFO | ------------------------------------------ | 20260505 07:06:38.213 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:06:38.213 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 07:06:38.213 | INFO | Total mem usage: 25.6 GiB | 20260505 07:06:38.213 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 07:06:38.213 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 07:06:38.213 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 07:06:38.213 | INFO | High Water Breakdown: | 20260505 07:06:38.214 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 07:06:38.214 | INFO | ------------------------------------------ | 20260505 07:06:38.214 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:06:38.214 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 07:06:38.214 | INFO | Total mem usage: 15.7 GiB | 20260505 07:06:38.214 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 07:06:38.214 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 07:06:38.214 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 07:06:38.214 | INFO | High Water Breakdown: | 20260505 07:06:38.214 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 07:06:38.214 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 07:06:38.214 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 07:06:38.214 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 07:06:38.214 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 07:06:38.214 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 07:06:38.214 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 07:06:38.214 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 07:06:38.214 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 07:06:38.214 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:06:38.214 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 07:06:38.214 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:06:38.214 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 07:06:38.214 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 07:06:38.214 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 07:06:38.214 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 07:06:38.214 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:06:38.214 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 07:06:38.214 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:06:38.214 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:06:38.214 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:06:38.214 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 07:06:38.214 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 07:06:38.214 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:06:38.214 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 07:06:38.214 | INFO | ------------------------------------------ | 20260505 07:06:38.214 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:06:38.214 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 07:06:38.214 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:06:38.214 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:06:38.214 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.50e+01, 1.34e+01 / 1.83e+00 | 20260505 07:06:38.214 | INFO | PerfReport: 0.28%, AggregateStats -- 9.35e-02 / 1.03e-01, 1.00e-02 / 6.11e-03 | 20260505 07:06:38.214 | INFO | PerfReport: 11.65%, Baseline -- 3.84e+00 / 4.45e+00, 6.39e-01 / 2.45e-01 | 20260505 07:06:38.214 | INFO | PerfReport: 3.31%, Histogram -- 1.09e+00 / 1.47e+00, 2.15e-01 / 1.11e-01 | 20260505 07:06:38.214 | INFO | PerfReport: 15.44%, DME -- 4.07e+01 / 4.44e+01, 4.11e+00 / 1.19e+00 | 20260505 07:06:38.214 | INFO | PerfReport: 56.13%, FrameLabeling -- 1.85e+01 / 1.91e+01, 2.02e+00 / 1.24e+00 | 20260505 07:06:38.214 | INFO | PerfReport: 7.68%, PulseAccumulating -- 2.53e+00 / 3.16e+00, 4.53e-01 / 2.51e-01 | 20260505 07:06:38.214 | INFO | PerfReport: 4.04%, Metrics -- 1.33e+00 / 1.72e+00, 3.74e-01 / 1.73e-01 | 20260505 07:06:38.214 | INFO | PerfReport: 1.46%, PulseSerializing -- 4.81e-01 / 5.59e-01, 9.34e-02 / 3.08e-02 | 20260505 07:06:38.214 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 07:06:38.214 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:06:38.214 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:06:38.214 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.23e+01 / 0.00e+00 | 20260505 07:06:38.214 | INFO | PerfReport: 22.16%, Upload -- 2.24e+01 / 0.00e+00, 7.87e+00 / 0.00e+00 | 20260505 07:06:38.214 | INFO | PerfReport: 21.51%, Download -- 2.18e+01 / 0.00e+00, 7.78e+00 / 0.00e+00 | 20260505 07:06:38.214 | INFO | PerfReport: 32.52%, Compute -- 3.29e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 07:06:38.214 | INFO | PerfReport: 1.49%, LockUpload -- 1.51e+00 / 0.00e+00, 7.08e+00 / 0.00e+00 | 20260505 07:06:38.214 | INFO | PerfReport: 0.85%, LockDownload -- 8.61e-01 / 0.00e+00, 2.31e+00 / 0.00e+00 | 20260505 07:06:38.214 | INFO | PerfReport: 21.46%, LockCompute -- 2.17e+01 / 0.00e+00, 1.66e+01 / 0.00e+00 | 20260505 07:06:38.214 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 1.78e-02 / 0.00e+00 | 20260505 07:06:38.345 | INFO | Starting a round of Baz IO | 20260505 07:06:41.635 | INFO | Populating RTMetrics report | 20260505 07:06:41.635 | INFO | RTMetrics report populated | 20260505 07:06:41.635 | INFO | Opening tmp RTMetrics json file | 20260505 07:06:41.673 | INFO | Serializing RTMetrics to json | 20260505 07:06:41.719 | INFO | Serialization done, swapping tmp file now... | 20260505 07:06:41.745 | INFO | RTMetrics done | 20260505 07:06:41.761 | INFO | Finished a round of Baz IO: Wrote 2.419 GiB in 3.415 seconds | 20260505 07:06:41.761 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 30.89% / 0.04% / 69.08% | 20260505 07:06:41.761 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.07963% / 5.5631e-05% / 5.16574e-05% | 20260505 07:06:41.761 | INFO | Baz IO overall: | 20260505 07:06:41.761 | INFO | BytesWritten : 689905977059 | 20260505 07:06:41.761 | INFO | NumEvents : 327004832148 | 20260505 07:06:43.348 | INFO | Analyzing chunk frames = [1286144,1286656) | 20260505 07:06:43.349 | INFO | MemInfo: { | 20260505 07:06:43.349 | INFO | "gpu" : " 17 GB", | 20260505 07:06:43.349 | INFO | "huge" : "", | 20260505 07:06:43.349 | INFO | "huge_shared" : " 27 GB", | 20260505 07:06:43.349 | INFO | "malloc" : " 14 GB", | 20260505 07:06:43.349 | INFO | "pinned" : " 29 GB" | 20260505 07:06:43.349 | INFO | } | 20260505 07:06:43.349 | INFO | Perf report for the last 5.13611 seconds: | 20260505 07:06:43.349 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:06:43.349 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:43.349 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:43.349 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:43.349 | INFO | ANALYSIS : 65.5%, 2.9, 100.4ms, 0 | 20260505 07:06:43.349 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:43.349 | INFO | RT_METRICS : 48.1%, 1.0, 25.2ms, 0 | 20260505 07:06:43.349 | INFO | PRE_HQ : 64.4%, 1.9, 63.2ms, 0 | 20260505 07:06:43.349 | INFO | BAZWRITER : 65.8%, 4.1, 139.4ms, 0 | 20260505 07:06:43.349 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 07:06:43.349 | INFO | Uploaded 36GiB to GPU in 2.24934s (16.0047GiB/s) | 20260505 07:06:43.349 | INFO | Downloaded 38GiB to GPU in 2.33079s (16.3035GiB/s) | 20260505 07:06:48.487 | INFO | Analyzing chunk frames = [1286656,1287168) | 20260505 07:06:48.487 | INFO | MemInfo: { | 20260505 07:06:48.487 | INFO | "gpu" : " 17 GB", | 20260505 07:06:48.488 | INFO | "huge" : "", | 20260505 07:06:48.488 | INFO | "huge_shared" : " 27 GB", | 20260505 07:06:48.488 | INFO | "malloc" : " 14 GB", | 20260505 07:06:48.488 | INFO | "pinned" : " 29 GB" | 20260505 07:06:48.488 | INFO | } | 20260505 07:06:48.488 | INFO | Perf report for the last 5.13914 seconds: | 20260505 07:06:48.488 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:06:48.488 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:48.488 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:48.488 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:48.488 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 07:06:48.488 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:48.488 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:48.488 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 07:06:48.488 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:48.488 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:48.488 | INFO | Uploaded 36GiB to GPU in 2.11605s (17.0128GiB/s) | 20260505 07:06:48.488 | INFO | Downloaded 34GiB to GPU in 2.03599s (16.6995GiB/s) | 20260505 07:06:53.627 | INFO | Analyzing chunk frames = [1287168,1287680) | 20260505 07:06:53.627 | INFO | MemInfo: { | 20260505 07:06:53.627 | INFO | "gpu" : " 17 GB", | 20260505 07:06:53.627 | INFO | "huge" : "", | 20260505 07:06:53.627 | INFO | "huge_shared" : " 27 GB", | 20260505 07:06:53.627 | INFO | "malloc" : " 14 GB", | 20260505 07:06:53.627 | INFO | "pinned" : " 29 GB" | 20260505 07:06:53.627 | INFO | } | 20260505 07:06:53.627 | INFO | Perf report for the last 5.139 seconds: | 20260505 07:06:53.627 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:06:53.627 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:53.627 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:53.627 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:53.627 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 07:06:53.627 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:53.627 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:53.627 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 07:06:53.627 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:53.627 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:53.627 | INFO | Uploaded 36GiB to GPU in 2.15966s (16.6693GiB/s) | 20260505 07:06:53.627 | INFO | Downloaded 34GiB to GPU in 2.05053s (16.5811GiB/s) | 20260505 07:06:58.766 | INFO | Analyzing chunk frames = [1287680,1288192) | 20260505 07:06:58.766 | INFO | MemInfo: { | 20260505 07:06:58.766 | INFO | "gpu" : " 17 GB", | 20260505 07:06:58.766 | INFO | "huge" : "", | 20260505 07:06:58.766 | INFO | "huge_shared" : " 27 GB", | 20260505 07:06:58.766 | INFO | "malloc" : " 14 GB", | 20260505 07:06:58.766 | INFO | "pinned" : " 29 GB" | 20260505 07:06:58.766 | INFO | } | 20260505 07:06:58.766 | INFO | Perf report for the last 5.13944 seconds: | 20260505 07:06:58.766 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:06:58.766 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:58.766 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:58.766 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:58.766 | INFO | ANALYSIS : 63.4%, 3.0, 98.5ms, 0 | 20260505 07:06:58.766 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:58.766 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:58.766 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 07:06:58.766 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:58.766 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:06:58.766 | INFO | Uploaded 36GiB to GPU in 2.12946s (16.9057GiB/s) | 20260505 07:06:58.767 | INFO | Downloaded 34GiB to GPU in 2.02448s (16.7944GiB/s) | 20260505 07:07:03.905 | INFO | Analyzing chunk frames = [1288192,1288704) | 20260505 07:07:03.905 | INFO | MemInfo: { | 20260505 07:07:03.905 | INFO | "gpu" : " 17 GB", | 20260505 07:07:03.905 | INFO | "huge" : "", | 20260505 07:07:03.905 | INFO | "huge_shared" : " 27 GB", | 20260505 07:07:03.906 | INFO | "malloc" : " 14 GB", | 20260505 07:07:03.906 | INFO | "pinned" : " 29 GB" | 20260505 07:07:03.906 | INFO | } | 20260505 07:07:03.906 | INFO | Perf report for the last 5.1395 seconds: | 20260505 07:07:03.906 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:07:03.906 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:03.906 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:03.906 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:03.906 | INFO | ANALYSIS : 63.2%, 3.0, 98.8ms, 0 | 20260505 07:07:03.906 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:03.906 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:03.906 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 07:07:03.906 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:03.906 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:03.906 | INFO | Uploaded 36GiB to GPU in 2.15698s (16.69GiB/s) | 20260505 07:07:03.906 | INFO | Downloaded 34GiB to GPU in 2.04923s (16.5916GiB/s) | 20260505 07:07:09.046 | INFO | Analyzing chunk frames = [1288704,1289216) | 20260505 07:07:09.046 | INFO | MemInfo: { | 20260505 07:07:09.046 | INFO | "gpu" : " 17 GB", | 20260505 07:07:09.046 | INFO | "huge" : "", | 20260505 07:07:09.046 | INFO | "huge_shared" : " 27 GB", | 20260505 07:07:09.046 | INFO | "malloc" : " 14 GB", | 20260505 07:07:09.046 | INFO | "pinned" : " 29 GB" | 20260505 07:07:09.046 | INFO | } | 20260505 07:07:09.046 | INFO | Perf report for the last 5.14063 seconds: | 20260505 07:07:09.046 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:07:09.046 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:09.046 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:09.046 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:09.046 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 07:07:09.046 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:09.046 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:09.046 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:07:09.046 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:09.046 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:09.047 | INFO | Uploaded 36GiB to GPU in 2.15859s (16.6776GiB/s) | 20260505 07:07:09.047 | INFO | Downloaded 34GiB to GPU in 2.0528s (16.5628GiB/s) | 20260505 07:07:14.184 | INFO | Analyzing chunk frames = [1289216,1289728) | 20260505 07:07:14.184 | INFO | MemInfo: { | 20260505 07:07:14.184 | INFO | "gpu" : " 17 GB", | 20260505 07:07:14.184 | INFO | "huge" : "", | 20260505 07:07:14.184 | INFO | "huge_shared" : " 27 GB", | 20260505 07:07:14.184 | INFO | "malloc" : " 14 GB", | 20260505 07:07:14.184 | INFO | "pinned" : " 29 GB" | 20260505 07:07:14.185 | INFO | } | 20260505 07:07:14.185 | INFO | Perf report for the last 5.13821 seconds: | 20260505 07:07:14.185 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:07:14.185 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:14.185 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:14.185 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:14.185 | INFO | ANALYSIS : 64.4%, 3.0, 100.6ms, 0 | 20260505 07:07:14.185 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:14.185 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:14.185 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260505 07:07:14.185 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:14.185 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:14.185 | INFO | Uploaded 36GiB to GPU in 2.20839s (16.3014GiB/s) | 20260505 07:07:14.185 | INFO | Downloaded 34GiB to GPU in 2.15392s (15.7851GiB/s) | 20260505 07:07:19.323 | INFO | Analyzing chunk frames = [1289728,1290240) | 20260505 07:07:19.323 | INFO | MemInfo: { | 20260505 07:07:19.323 | INFO | "gpu" : " 17 GB", | 20260505 07:07:19.323 | INFO | "huge" : "", | 20260505 07:07:19.323 | INFO | "huge_shared" : " 27 GB", | 20260505 07:07:19.323 | INFO | "malloc" : " 14 GB", | 20260505 07:07:19.323 | INFO | "pinned" : " 29 GB" | 20260505 07:07:19.323 | INFO | } | 20260505 07:07:19.323 | INFO | Perf report for the last 5.13877 seconds: | 20260505 07:07:19.323 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:07:19.323 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:19.323 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:19.323 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:19.323 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 07:07:19.323 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:19.323 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:19.323 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 07:07:19.323 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:19.324 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:19.324 | INFO | Uploaded 36GiB to GPU in 2.16439s (16.6329GiB/s) | 20260505 07:07:19.324 | INFO | Downloaded 34GiB to GPU in 2.04293s (16.6427GiB/s) | 20260505 07:07:19.439 | INFO | Starting a round of Baz IO | 20260505 07:07:22.740 | INFO | Populating RTMetrics report | 20260505 07:07:22.740 | INFO | RTMetrics report populated | 20260505 07:07:22.740 | INFO | Opening tmp RTMetrics json file | 20260505 07:07:22.772 | INFO | Serializing RTMetrics to json | 20260505 07:07:22.773 | INFO | Serialization done, swapping tmp file now... | 20260505 07:07:22.774 | INFO | RTMetrics done | 20260505 07:07:22.867 | INFO | Finished a round of Baz IO: Wrote 2.418 GiB in 3.428 seconds | 20260505 07:07:22.867 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 30.85% / 0.04% / 69.11% | 20260505 07:07:22.868 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.05699% / 4.37101e-05% / 7.15256e-05% | 20260505 07:07:22.868 | INFO | Baz IO overall: | 20260505 07:07:22.868 | INFO | BytesWritten : 692502770307 | 20260505 07:07:22.868 | INFO | NumEvents : 328247860761 | 20260505 07:07:24.462 | INFO | Analyzing chunk frames = [1290240,1290752) | 20260505 07:07:24.462 | INFO | MemInfo: { | 20260505 07:07:24.462 | INFO | "gpu" : " 17 GB", | 20260505 07:07:24.462 | INFO | "huge" : "", | 20260505 07:07:24.462 | INFO | "huge_shared" : " 27 GB", | 20260505 07:07:24.462 | INFO | "malloc" : " 14 GB", | 20260505 07:07:24.463 | INFO | "pinned" : " 29 GB" | 20260505 07:07:24.463 | INFO | } | 20260505 07:07:24.463 | INFO | Perf report for the last 5.13922 seconds: | 20260505 07:07:24.463 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:07:24.463 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:24.463 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:24.463 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:24.463 | INFO | ANALYSIS : 65.3%, 2.9, 99.5ms, 0 | 20260505 07:07:24.463 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:24.463 | INFO | RT_METRICS : 46.6%, 1.0, 24.4ms, 0 | 20260505 07:07:24.463 | INFO | PRE_HQ : 64.1%, 1.9, 63.1ms, 0 | 20260505 07:07:24.463 | INFO | BAZWRITER : 66.0%, 4.1, 139.7ms, 0 | 20260505 07:07:24.463 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 07:07:24.463 | INFO | Uploaded 36GiB to GPU in 2.26146s (15.9189GiB/s) | 20260505 07:07:24.463 | INFO | Downloaded 38GiB to GPU in 2.32947s (16.3127GiB/s) | 20260505 07:07:29.602 | INFO | Analyzing chunk frames = [1290752,1291264) | 20260505 07:07:29.602 | INFO | MemInfo: { | 20260505 07:07:29.602 | INFO | "gpu" : " 17 GB", | 20260505 07:07:29.602 | INFO | "huge" : "", | 20260505 07:07:29.602 | INFO | "huge_shared" : " 27 GB", | 20260505 07:07:29.602 | INFO | "malloc" : " 14 GB", | 20260505 07:07:29.602 | INFO | "pinned" : " 29 GB" | 20260505 07:07:29.602 | INFO | } | 20260505 07:07:29.602 | INFO | Perf report for the last 5.13972 seconds: | 20260505 07:07:29.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:07:29.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:29.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:29.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:29.602 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 07:07:29.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:29.602 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:29.602 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 07:07:29.602 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:29.602 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:29.603 | INFO | Uploaded 36GiB to GPU in 2.11342s (17.034GiB/s) | 20260505 07:07:29.603 | INFO | Downloaded 34GiB to GPU in 2.03659s (16.6946GiB/s) | 20260505 07:07:34.741 | INFO | Analyzing chunk frames = [1291264,1291776) | 20260505 07:07:34.741 | INFO | MemInfo: { | 20260505 07:07:34.741 | INFO | "gpu" : " 17 GB", | 20260505 07:07:34.741 | INFO | "huge" : "", | 20260505 07:07:34.741 | INFO | "huge_shared" : " 27 GB", | 20260505 07:07:34.741 | INFO | "malloc" : " 14 GB", | 20260505 07:07:34.741 | INFO | "pinned" : " 29 GB" | 20260505 07:07:34.741 | INFO | } | 20260505 07:07:34.741 | INFO | Perf report for the last 5.13904 seconds: | 20260505 07:07:34.741 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:07:34.741 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:34.741 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:34.741 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:34.741 | INFO | ANALYSIS : 63.6%, 3.0, 98.9ms, 0 | 20260505 07:07:34.741 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:34.742 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:34.742 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 07:07:34.742 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:34.742 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:34.742 | INFO | Uploaded 36GiB to GPU in 2.15073s (16.7385GiB/s) | 20260505 07:07:34.742 | INFO | Downloaded 34GiB to GPU in 2.05002s (16.5852GiB/s) | 20260505 07:07:39.881 | INFO | Analyzing chunk frames = [1291776,1292288) | 20260505 07:07:39.881 | INFO | MemInfo: { | 20260505 07:07:39.881 | INFO | "gpu" : " 17 GB", | 20260505 07:07:39.881 | INFO | "huge" : "", | 20260505 07:07:39.881 | INFO | "huge_shared" : " 27 GB", | 20260505 07:07:39.881 | INFO | "malloc" : " 14 GB", | 20260505 07:07:39.881 | INFO | "pinned" : " 29 GB" | 20260505 07:07:39.881 | INFO | } | 20260505 07:07:39.881 | INFO | Perf report for the last 5.13937 seconds: | 20260505 07:07:39.881 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:07:39.881 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:39.881 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:39.881 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:39.881 | INFO | ANALYSIS : 63.6%, 3.0, 98.8ms, 0 | 20260505 07:07:39.881 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:39.881 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:39.881 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 07:07:39.881 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:39.881 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:39.881 | INFO | Uploaded 36GiB to GPU in 2.15355s (16.7166GiB/s) | 20260505 07:07:39.881 | INFO | Downloaded 34GiB to GPU in 2.03324s (16.7221GiB/s) | 20260505 07:07:45.020 | INFO | Analyzing chunk frames = [1292288,1292800) | 20260505 07:07:45.020 | INFO | MemInfo: { | 20260505 07:07:45.020 | INFO | "gpu" : " 17 GB", | 20260505 07:07:45.020 | INFO | "huge" : "", | 20260505 07:07:45.020 | INFO | "huge_shared" : " 27 GB", | 20260505 07:07:45.020 | INFO | "malloc" : " 14 GB", | 20260505 07:07:45.020 | INFO | "pinned" : " 29 GB" | 20260505 07:07:45.020 | INFO | } | 20260505 07:07:45.020 | INFO | Perf report for the last 5.13934 seconds: | 20260505 07:07:45.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:07:45.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:45.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:45.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:45.020 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 07:07:45.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:45.020 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:45.020 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 07:07:45.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:45.020 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:45.020 | INFO | Uploaded 36GiB to GPU in 2.15381s (16.7146GiB/s) | 20260505 07:07:45.020 | INFO | Downloaded 34GiB to GPU in 2.04659s (16.613GiB/s) | 20260505 07:07:50.161 | INFO | Analyzing chunk frames = [1292800,1293312) | 20260505 07:07:50.161 | INFO | MemInfo: { | 20260505 07:07:50.161 | INFO | "gpu" : " 17 GB", | 20260505 07:07:50.161 | INFO | "huge" : "", | 20260505 07:07:50.161 | INFO | "huge_shared" : " 27 GB", | 20260505 07:07:50.161 | INFO | "malloc" : " 14 GB", | 20260505 07:07:50.161 | INFO | "pinned" : " 29 GB" | 20260505 07:07:50.161 | INFO | } | 20260505 07:07:50.161 | INFO | Perf report for the last 5.14107 seconds: | 20260505 07:07:50.161 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:07:50.161 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:50.161 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:50.161 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:50.161 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 07:07:50.161 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:50.161 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:50.161 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:07:50.161 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:50.161 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:50.161 | INFO | Uploaded 36GiB to GPU in 2.1609s (16.6598GiB/s) | 20260505 07:07:50.161 | INFO | Downloaded 34GiB to GPU in 2.05653s (16.5327GiB/s) | 20260505 07:07:55.303 | INFO | Analyzing chunk frames = [1293312,1293824) | 20260505 07:07:55.303 | INFO | MemInfo: { | 20260505 07:07:55.303 | INFO | "gpu" : " 17 GB", | 20260505 07:07:55.303 | INFO | "huge" : "", | 20260505 07:07:55.303 | INFO | "huge_shared" : " 27 GB", | 20260505 07:07:55.303 | INFO | "malloc" : " 14 GB", | 20260505 07:07:55.303 | INFO | "pinned" : " 29 GB" | 20260505 07:07:55.303 | INFO | } | 20260505 07:07:55.303 | INFO | Perf report for the last 5.14163 seconds: | 20260505 07:07:55.303 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:07:55.303 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:55.303 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:55.303 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:55.303 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 07:07:55.303 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:55.303 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:55.303 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260505 07:07:55.303 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:55.303 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:07:55.303 | INFO | Uploaded 36GiB to GPU in 2.15684s (16.6911GiB/s) | 20260505 07:07:55.303 | INFO | Downloaded 34GiB to GPU in 2.15599s (15.77GiB/s) | 20260505 07:08:00.442 | INFO | Analyzing chunk frames = [1293824,1294336) | 20260505 07:08:00.442 | INFO | MemInfo: { | 20260505 07:08:00.442 | INFO | "gpu" : " 17 GB", | 20260505 07:08:00.442 | INFO | "huge" : "", | 20260505 07:08:00.442 | INFO | "huge_shared" : " 27 GB", | 20260505 07:08:00.442 | INFO | "malloc" : " 14 GB", | 20260505 07:08:00.442 | INFO | "pinned" : " 29 GB" | 20260505 07:08:00.442 | INFO | } | 20260505 07:08:00.442 | INFO | Perf report for the last 5.13927 seconds: | 20260505 07:08:00.442 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:08:00.442 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:00.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:00.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:00.442 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 07:08:00.442 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:00.442 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:00.442 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 07:08:00.442 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:00.442 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:00.442 | INFO | Uploaded 36GiB to GPU in 2.16905s (16.5971GiB/s) | 20260505 07:08:00.442 | INFO | Downloaded 34GiB to GPU in 2.0457s (16.6202GiB/s) | 20260505 07:08:00.443 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 07:08:00.443 | INFO | Total mem usage: 15.3 GiB | 20260505 07:08:00.443 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 07:08:00.443 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -6.01%) | 20260505 07:08:00.443 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 07:08:00.443 | INFO | High Water Breakdown: | 20260505 07:08:00.443 | INFO | 60.6% : 8.76 GiB ( 93.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 07:08:00.443 | INFO | 39.3% : 5.68 GiB ( 85.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 07:08:00.443 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 07:08:00.443 | INFO | ------------------------------------------ | 20260505 07:08:00.443 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:08:00.443 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 07:08:00.443 | INFO | Total mem usage: 26.5 GiB | 20260505 07:08:00.443 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 07:08:00.443 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 07:08:00.443 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 07:08:00.443 | INFO | High Water Breakdown: | 20260505 07:08:00.443 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:08:00.443 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:08:00.443 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:08:00.443 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:08:00.443 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:08:00.443 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:08:00.443 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 07:08:00.443 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 07:08:00.443 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:08:00.443 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 07:08:00.443 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 07:08:00.443 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 07:08:00.443 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 07:08:00.443 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 07:08:00.443 | INFO | ------------------------------------------ | 20260505 07:08:00.443 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:08:00.443 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 07:08:00.443 | INFO | Total mem usage: 25.6 GiB | 20260505 07:08:00.443 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 07:08:00.443 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 07:08:00.443 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 07:08:00.443 | INFO | High Water Breakdown: | 20260505 07:08:00.443 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 07:08:00.443 | INFO | ------------------------------------------ | 20260505 07:08:00.443 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:08:00.443 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 07:08:00.443 | INFO | Total mem usage: 15.7 GiB | 20260505 07:08:00.443 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 07:08:00.443 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 07:08:00.443 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 07:08:00.443 | INFO | High Water Breakdown: | 20260505 07:08:00.443 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 07:08:00.443 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 07:08:00.443 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 07:08:00.443 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 07:08:00.443 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 07:08:00.443 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 07:08:00.443 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 07:08:00.443 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 07:08:00.443 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 07:08:00.443 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:08:00.443 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 07:08:00.443 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:08:00.443 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 07:08:00.443 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 07:08:00.443 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 07:08:00.443 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 07:08:00.443 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:08:00.443 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 07:08:00.443 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:08:00.443 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:08:00.443 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:08:00.443 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 07:08:00.443 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 07:08:00.443 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:08:00.443 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 07:08:00.443 | INFO | ------------------------------------------ | 20260505 07:08:00.443 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:08:00.444 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 07:08:00.444 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:08:00.444 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:08:00.444 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.53e+01, 1.37e+01 / 1.98e+00 | 20260505 07:08:00.444 | INFO | PerfReport: 0.28%, AggregateStats -- 9.37e-02 / 1.04e-01, 1.07e-02 / 5.91e-03 | 20260505 07:08:00.444 | INFO | PerfReport: 11.60%, Baseline -- 3.82e+00 / 4.44e+00, 6.26e-01 / 2.94e-01 | 20260505 07:08:00.444 | INFO | PerfReport: 3.31%, Histogram -- 1.09e+00 / 1.47e+00, 2.14e-01 / 1.09e-01 | 20260505 07:08:00.444 | INFO | PerfReport: 15.47%, DME -- 4.07e+01 / 4.44e+01, 4.08e+00 / 1.23e+00 | 20260505 07:08:00.444 | INFO | PerfReport: 56.15%, FrameLabeling -- 1.85e+01 / 1.93e+01, 2.02e+00 / 1.69e+00 | 20260505 07:08:00.444 | INFO | PerfReport: 7.70%, PulseAccumulating -- 2.53e+00 / 3.16e+00, 4.61e-01 / 2.65e-01 | 20260505 07:08:00.444 | INFO | PerfReport: 4.03%, Metrics -- 1.33e+00 / 1.74e+00, 3.66e-01 / 1.86e-01 | 20260505 07:08:00.444 | INFO | PerfReport: 1.46%, PulseSerializing -- 4.81e-01 / 5.55e-01, 9.27e-02 / 3.18e-02 | 20260505 07:08:00.444 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 07:08:00.444 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:08:00.444 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:08:00.444 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.24e+01 / 0.00e+00 | 20260505 07:08:00.444 | INFO | PerfReport: 22.24%, Upload -- 2.26e+01 / 0.00e+00, 7.97e+00 / 0.00e+00 | 20260505 07:08:00.444 | INFO | PerfReport: 21.49%, Download -- 2.18e+01 / 0.00e+00, 7.82e+00 / 0.00e+00 | 20260505 07:08:00.444 | INFO | PerfReport: 32.44%, Compute -- 3.29e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 07:08:00.444 | INFO | PerfReport: 1.47%, LockUpload -- 1.49e+00 / 0.00e+00, 7.09e+00 / 0.00e+00 | 20260505 07:08:00.444 | INFO | PerfReport: 0.82%, LockDownload -- 8.36e-01 / 0.00e+00, 2.18e+00 / 0.00e+00 | 20260505 07:08:00.444 | INFO | PerfReport: 21.52%, LockCompute -- 2.18e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260505 07:08:00.444 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 1.84e-02 / 0.00e+00 | 20260505 07:08:00.567 | INFO | Starting a round of Baz IO | 20260505 07:08:03.867 | INFO | Populating RTMetrics report | 20260505 07:08:03.867 | INFO | RTMetrics report populated | 20260505 07:08:03.867 | INFO | Opening tmp RTMetrics json file | 20260505 07:08:03.890 | INFO | Serializing RTMetrics to json | 20260505 07:08:03.891 | INFO | Serialization done, swapping tmp file now... | 20260505 07:08:03.891 | INFO | RTMetrics done | 20260505 07:08:03.984 | INFO | Finished a round of Baz IO: Wrote 2.418 GiB in 3.416 seconds | 20260505 07:08:03.984 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 30.81% / 0.04% / 69.15% | 20260505 07:08:03.984 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.03695% / 3.97364e-05% / 5.96046e-05% | 20260505 07:08:03.984 | INFO | Baz IO overall: | 20260505 07:08:03.984 | INFO | BytesWritten : 695099148697 | 20260505 07:08:03.984 | INFO | NumEvents : 329490030690 | 20260505 07:08:05.577 | INFO | Analyzing chunk frames = [1294336,1294848) | 20260505 07:08:05.577 | INFO | MemInfo: { | 20260505 07:08:05.577 | INFO | "gpu" : " 17 GB", | 20260505 07:08:05.577 | INFO | "huge" : "", | 20260505 07:08:05.577 | INFO | "huge_shared" : " 27 GB", | 20260505 07:08:05.577 | INFO | "malloc" : " 14 GB", | 20260505 07:08:05.577 | INFO | "pinned" : " 29 GB" | 20260505 07:08:05.577 | INFO | } | 20260505 07:08:05.577 | INFO | Perf report for the last 5.13523 seconds: | 20260505 07:08:05.577 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:08:05.577 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:05.577 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:05.577 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:05.577 | INFO | ANALYSIS : 65.5%, 2.9, 100.1ms, 0 | 20260505 07:08:05.577 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:05.577 | INFO | RT_METRICS : 46.5%, 1.0, 24.3ms, 0 | 20260505 07:08:05.577 | INFO | PRE_HQ : 64.5%, 1.9, 63.2ms, 0 | 20260505 07:08:05.577 | INFO | BAZWRITER : 65.8%, 4.0, 136.7ms, 0 | 20260505 07:08:05.578 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 07:08:05.578 | INFO | Uploaded 36GiB to GPU in 2.23607s (16.0997GiB/s) | 20260505 07:08:05.578 | INFO | Downloaded 38GiB to GPU in 2.36518s (16.0664GiB/s) | 20260505 07:08:10.717 | INFO | Analyzing chunk frames = [1294848,1295360) | 20260505 07:08:10.717 | INFO | MemInfo: { | 20260505 07:08:10.717 | INFO | "gpu" : " 17 GB", | 20260505 07:08:10.717 | INFO | "huge" : "", | 20260505 07:08:10.717 | INFO | "huge_shared" : " 27 GB", | 20260505 07:08:10.717 | INFO | "malloc" : " 14 GB", | 20260505 07:08:10.717 | INFO | "pinned" : " 29 GB" | 20260505 07:08:10.717 | INFO | } | 20260505 07:08:10.717 | INFO | Perf report for the last 5.13982 seconds: | 20260505 07:08:10.717 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:08:10.717 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:10.717 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:10.717 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:10.717 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 07:08:10.717 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:10.717 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:10.717 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 07:08:10.717 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:10.717 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:10.717 | INFO | Uploaded 36GiB to GPU in 2.1315s (16.8895GiB/s) | 20260505 07:08:10.718 | INFO | Downloaded 34GiB to GPU in 2.04114s (16.6574GiB/s) | 20260505 07:08:15.856 | INFO | Analyzing chunk frames = [1295360,1295872) | 20260505 07:08:15.856 | INFO | MemInfo: { | 20260505 07:08:15.856 | INFO | "gpu" : " 17 GB", | 20260505 07:08:15.856 | INFO | "huge" : "", | 20260505 07:08:15.856 | INFO | "huge_shared" : " 27 GB", | 20260505 07:08:15.856 | INFO | "malloc" : " 14 GB", | 20260505 07:08:15.856 | INFO | "pinned" : " 29 GB" | 20260505 07:08:15.856 | INFO | } | 20260505 07:08:15.856 | INFO | Perf report for the last 5.13911 seconds: | 20260505 07:08:15.856 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:08:15.856 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:15.856 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:15.856 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:15.856 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260505 07:08:15.856 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:15.856 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:15.856 | INFO | PRE_HQ : 30.3%, 1.0, 15.8ms, 0 | 20260505 07:08:15.856 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:15.857 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:15.857 | INFO | Uploaded 36GiB to GPU in 2.15731s (16.6874GiB/s) | 20260505 07:08:15.857 | INFO | Downloaded 34GiB to GPU in 2.05337s (16.5581GiB/s) | 20260505 07:08:20.995 | INFO | Analyzing chunk frames = [1295872,1296384) | 20260505 07:08:20.995 | INFO | MemInfo: { | 20260505 07:08:20.995 | INFO | "gpu" : " 17 GB", | 20260505 07:08:20.995 | INFO | "huge" : "", | 20260505 07:08:20.995 | INFO | "huge_shared" : " 27 GB", | 20260505 07:08:20.995 | INFO | "malloc" : " 14 GB", | 20260505 07:08:20.996 | INFO | "pinned" : " 29 GB" | 20260505 07:08:20.996 | INFO | } | 20260505 07:08:20.996 | INFO | Perf report for the last 5.13931 seconds: | 20260505 07:08:20.996 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:08:20.996 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:20.996 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:20.996 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:20.996 | INFO | ANALYSIS : 63.5%, 3.0, 98.6ms, 0 | 20260505 07:08:20.996 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:20.996 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:20.996 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 07:08:20.996 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:20.996 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:20.996 | INFO | Uploaded 36GiB to GPU in 2.13314s (16.8765GiB/s) | 20260505 07:08:20.996 | INFO | Downloaded 34GiB to GPU in 2.02587s (16.7829GiB/s) | 20260505 07:08:26.135 | INFO | Analyzing chunk frames = [1296384,1296896) | 20260505 07:08:26.135 | INFO | MemInfo: { | 20260505 07:08:26.135 | INFO | "gpu" : " 17 GB", | 20260505 07:08:26.135 | INFO | "huge" : "", | 20260505 07:08:26.135 | INFO | "huge_shared" : " 27 GB", | 20260505 07:08:26.135 | INFO | "malloc" : " 14 GB", | 20260505 07:08:26.135 | INFO | "pinned" : " 29 GB" | 20260505 07:08:26.135 | INFO | } | 20260505 07:08:26.135 | INFO | Perf report for the last 5.13953 seconds: | 20260505 07:08:26.135 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:08:26.135 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:26.135 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:26.135 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:26.135 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 07:08:26.135 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:26.135 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:26.135 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:08:26.135 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:26.135 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:26.135 | INFO | Uploaded 36GiB to GPU in 2.15977s (16.6684GiB/s) | 20260505 07:08:26.135 | INFO | Downloaded 34GiB to GPU in 2.04804s (16.6013GiB/s) | 20260505 07:08:31.276 | INFO | Analyzing chunk frames = [1296896,1297408) | 20260505 07:08:31.276 | INFO | MemInfo: { | 20260505 07:08:31.276 | INFO | "gpu" : " 17 GB", | 20260505 07:08:31.276 | INFO | "huge" : "", | 20260505 07:08:31.276 | INFO | "huge_shared" : " 27 GB", | 20260505 07:08:31.276 | INFO | "malloc" : " 14 GB", | 20260505 07:08:31.276 | INFO | "pinned" : " 29 GB" | 20260505 07:08:31.276 | INFO | } | 20260505 07:08:31.276 | INFO | Perf report for the last 5.14074 seconds: | 20260505 07:08:31.276 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:08:31.276 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:31.276 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:31.276 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:31.276 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 07:08:31.276 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:31.276 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:31.276 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:08:31.276 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:31.276 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:31.276 | INFO | Uploaded 36GiB to GPU in 2.15565s (16.7003GiB/s) | 20260505 07:08:31.276 | INFO | Downloaded 34GiB to GPU in 2.05408s (16.5524GiB/s) | 20260505 07:08:36.414 | INFO | Analyzing chunk frames = [1297408,1297920) | 20260505 07:08:36.414 | INFO | MemInfo: { | 20260505 07:08:36.414 | INFO | "gpu" : " 17 GB", | 20260505 07:08:36.414 | INFO | "huge" : "", | 20260505 07:08:36.414 | INFO | "huge_shared" : " 27 GB", | 20260505 07:08:36.414 | INFO | "malloc" : " 14 GB", | 20260505 07:08:36.414 | INFO | "pinned" : " 29 GB" | 20260505 07:08:36.414 | INFO | } | 20260505 07:08:36.414 | INFO | Perf report for the last 5.13814 seconds: | 20260505 07:08:36.414 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:08:36.414 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:36.414 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:36.414 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:36.414 | INFO | ANALYSIS : 64.7%, 3.0, 101.1ms, 0 | 20260505 07:08:36.414 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:36.414 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:36.414 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260505 07:08:36.414 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:36.414 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:36.414 | INFO | Uploaded 36GiB to GPU in 2.24048s (16.068GiB/s) | 20260505 07:08:36.414 | INFO | Downloaded 34GiB to GPU in 2.1532s (15.7904GiB/s) | 20260505 07:08:41.553 | INFO | Analyzing chunk frames = [1297920,1298432) | 20260505 07:08:41.553 | INFO | MemInfo: { | 20260505 07:08:41.553 | INFO | "gpu" : " 17 GB", | 20260505 07:08:41.553 | INFO | "huge" : "", | 20260505 07:08:41.553 | INFO | "huge_shared" : " 27 GB", | 20260505 07:08:41.553 | INFO | "malloc" : " 14 GB", | 20260505 07:08:41.553 | INFO | "pinned" : " 29 GB" | 20260505 07:08:41.553 | INFO | } | 20260505 07:08:41.553 | INFO | Perf report for the last 5.1388 seconds: | 20260505 07:08:41.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:08:41.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:41.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:41.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:41.553 | INFO | ANALYSIS : 64.1%, 3.0, 100.2ms, 0 | 20260505 07:08:41.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:41.553 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:41.553 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 07:08:41.553 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:41.553 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:41.553 | INFO | Uploaded 36GiB to GPU in 2.16567s (16.6231GiB/s) | 20260505 07:08:41.553 | INFO | Downloaded 34GiB to GPU in 2.04536s (16.623GiB/s) | 20260505 07:08:41.667 | INFO | Starting a round of Baz IO | 20260505 07:08:44.965 | INFO | Populating RTMetrics report | 20260505 07:08:44.966 | INFO | RTMetrics report populated | 20260505 07:08:44.966 | INFO | Opening tmp RTMetrics json file | 20260505 07:08:44.966 | INFO | Serializing RTMetrics to json | 20260505 07:08:44.967 | INFO | Serialization done, swapping tmp file now... | 20260505 07:08:44.968 | INFO | RTMetrics done | 20260505 07:08:45.098 | INFO | Finished a round of Baz IO: Wrote 2.421 GiB in 3.431 seconds | 20260505 07:08:45.098 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 30.78% / 0.04% / 69.19% | 20260505 07:08:45.098 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.03617% / 4.37101e-05% / 7.15256e-05% | 20260505 07:08:45.098 | INFO | Baz IO overall: | 20260505 07:08:45.098 | INFO | BytesWritten : 697698396696 | 20260505 07:08:45.098 | INFO | NumEvents : 330734436595 | 20260505 07:08:46.692 | INFO | Analyzing chunk frames = [1298432,1298944) | 20260505 07:08:46.692 | INFO | MemInfo: { | 20260505 07:08:46.692 | INFO | "gpu" : " 17 GB", | 20260505 07:08:46.692 | INFO | "huge" : "", | 20260505 07:08:46.692 | INFO | "huge_shared" : " 27 GB", | 20260505 07:08:46.692 | INFO | "malloc" : " 14 GB", | 20260505 07:08:46.692 | INFO | "pinned" : " 29 GB" | 20260505 07:08:46.692 | INFO | } | 20260505 07:08:46.692 | INFO | Perf report for the last 5.13931 seconds: | 20260505 07:08:46.692 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:08:46.692 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:46.692 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:46.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:46.692 | INFO | ANALYSIS : 65.3%, 2.9, 99.3ms, 0 | 20260505 07:08:46.692 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:46.692 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260505 07:08:46.692 | INFO | PRE_HQ : 64.1%, 1.9, 63.3ms, 0 | 20260505 07:08:46.692 | INFO | BAZWRITER : 66.0%, 4.1, 139.0ms, 0 | 20260505 07:08:46.692 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 07:08:46.693 | INFO | Uploaded 36GiB to GPU in 2.25626s (15.9556GiB/s) | 20260505 07:08:46.693 | INFO | Downloaded 38GiB to GPU in 2.31587s (16.4086GiB/s) | 20260505 07:08:51.831 | INFO | Analyzing chunk frames = [1298944,1299456) | 20260505 07:08:51.832 | INFO | MemInfo: { | 20260505 07:08:51.832 | INFO | "gpu" : " 17 GB", | 20260505 07:08:51.832 | INFO | "huge" : "", | 20260505 07:08:51.832 | INFO | "huge_shared" : " 27 GB", | 20260505 07:08:51.832 | INFO | "malloc" : " 14 GB", | 20260505 07:08:51.832 | INFO | "pinned" : " 29 GB" | 20260505 07:08:51.832 | INFO | } | 20260505 07:08:51.832 | INFO | Perf report for the last 5.13945 seconds: | 20260505 07:08:51.832 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:08:51.832 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:51.832 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:51.832 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:51.832 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 07:08:51.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:51.832 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:51.832 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:08:51.832 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:51.832 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:51.832 | INFO | Uploaded 36GiB to GPU in 2.13274s (16.8797GiB/s) | 20260505 07:08:51.832 | INFO | Downloaded 34GiB to GPU in 2.04138s (16.6554GiB/s) | 20260505 07:08:56.971 | INFO | Analyzing chunk frames = [1299456,1299968) | 20260505 07:08:56.971 | INFO | MemInfo: { | 20260505 07:08:56.971 | INFO | "gpu" : " 17 GB", | 20260505 07:08:56.971 | INFO | "huge" : "", | 20260505 07:08:56.971 | INFO | "huge_shared" : " 27 GB", | 20260505 07:08:56.971 | INFO | "malloc" : " 14 GB", | 20260505 07:08:56.971 | INFO | "pinned" : " 29 GB" | 20260505 07:08:56.971 | INFO | } | 20260505 07:08:56.971 | INFO | Perf report for the last 5.13927 seconds: | 20260505 07:08:56.971 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:08:56.971 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:56.971 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:56.971 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:56.971 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260505 07:08:56.971 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:56.971 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:56.971 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 07:08:56.971 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:56.971 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:08:56.971 | INFO | Uploaded 36GiB to GPU in 2.15053s (16.7401GiB/s) | 20260505 07:08:56.971 | INFO | Downloaded 34GiB to GPU in 2.05039s (16.5822GiB/s) | 20260505 07:09:02.110 | INFO | Analyzing chunk frames = [1299968,1300480) | 20260505 07:09:02.110 | INFO | MemInfo: { | 20260505 07:09:02.110 | INFO | "gpu" : " 17 GB", | 20260505 07:09:02.110 | INFO | "huge" : "", | 20260505 07:09:02.110 | INFO | "huge_shared" : " 27 GB", | 20260505 07:09:02.110 | INFO | "malloc" : " 14 GB", | 20260505 07:09:02.110 | INFO | "pinned" : " 29 GB" | 20260505 07:09:02.110 | INFO | } | 20260505 07:09:02.110 | INFO | Perf report for the last 5.1393 seconds: | 20260505 07:09:02.110 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:09:02.110 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:02.110 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:02.110 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:02.110 | INFO | ANALYSIS : 63.5%, 3.0, 98.6ms, 0 | 20260505 07:09:02.110 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:02.110 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:02.110 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260505 07:09:02.111 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:02.111 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:02.111 | INFO | Uploaded 36GiB to GPU in 2.14031s (16.82GiB/s) | 20260505 07:09:02.111 | INFO | Downloaded 34GiB to GPU in 2.02168s (16.8177GiB/s) | 20260505 07:09:07.249 | INFO | Analyzing chunk frames = [1300480,1300992) | 20260505 07:09:07.249 | INFO | MemInfo: { | 20260505 07:09:07.249 | INFO | "gpu" : " 17 GB", | 20260505 07:09:07.249 | INFO | "huge" : "", | 20260505 07:09:07.249 | INFO | "huge_shared" : " 27 GB", | 20260505 07:09:07.249 | INFO | "malloc" : " 14 GB", | 20260505 07:09:07.249 | INFO | "pinned" : " 29 GB" | 20260505 07:09:07.249 | INFO | } | 20260505 07:09:07.249 | INFO | Perf report for the last 5.13909 seconds: | 20260505 07:09:07.249 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:09:07.249 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:07.249 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:07.249 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:07.249 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 07:09:07.249 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:07.249 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:07.249 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 07:09:07.249 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:07.250 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:07.250 | INFO | Uploaded 36GiB to GPU in 2.15905s (16.674GiB/s) | 20260505 07:09:07.250 | INFO | Downloaded 34GiB to GPU in 2.0482s (16.5999GiB/s) | 20260505 07:09:12.391 | INFO | Analyzing chunk frames = [1300992,1301504) | 20260505 07:09:12.391 | INFO | MemInfo: { | 20260505 07:09:12.391 | INFO | "gpu" : " 17 GB", | 20260505 07:09:12.391 | INFO | "huge" : "", | 20260505 07:09:12.391 | INFO | "huge_shared" : " 27 GB", | 20260505 07:09:12.392 | INFO | "malloc" : " 14 GB", | 20260505 07:09:12.392 | INFO | "pinned" : " 29 GB" | 20260505 07:09:12.392 | INFO | } | 20260505 07:09:12.392 | INFO | Perf report for the last 5.14222 seconds: | 20260505 07:09:12.392 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:09:12.392 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:12.392 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:12.392 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:12.392 | INFO | ANALYSIS : 63.5%, 3.0, 99.3ms, 0 | 20260505 07:09:12.392 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:12.392 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:12.392 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 07:09:12.392 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:12.392 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:12.392 | INFO | Uploaded 36GiB to GPU in 2.16366s (16.6384GiB/s) | 20260505 07:09:12.392 | INFO | Downloaded 34GiB to GPU in 2.05384s (16.5544GiB/s) | 20260505 07:09:17.528 | INFO | Analyzing chunk frames = [1301504,1302016) | 20260505 07:09:17.528 | INFO | MemInfo: { | 20260505 07:09:17.528 | INFO | "gpu" : " 17 GB", | 20260505 07:09:17.528 | INFO | "huge" : "", | 20260505 07:09:17.528 | INFO | "huge_shared" : " 27 GB", | 20260505 07:09:17.528 | INFO | "malloc" : " 14 GB", | 20260505 07:09:17.528 | INFO | "pinned" : " 29 GB" | 20260505 07:09:17.528 | INFO | } | 20260505 07:09:17.528 | INFO | Perf report for the last 5.13659 seconds: | 20260505 07:09:17.528 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:09:17.528 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:17.528 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:17.528 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:17.528 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 07:09:17.528 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:17.528 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:17.528 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260505 07:09:17.528 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:17.528 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:17.528 | INFO | Uploaded 36GiB to GPU in 2.19431s (16.4061GiB/s) | 20260505 07:09:17.529 | INFO | Downloaded 34GiB to GPU in 2.15662s (15.7654GiB/s) | 20260505 07:09:22.667 | INFO | Analyzing chunk frames = [1302016,1302528) | 20260505 07:09:22.667 | INFO | MemInfo: { | 20260505 07:09:22.667 | INFO | "gpu" : " 17 GB", | 20260505 07:09:22.667 | INFO | "huge" : "", | 20260505 07:09:22.667 | INFO | "huge_shared" : " 27 GB", | 20260505 07:09:22.668 | INFO | "malloc" : " 14 GB", | 20260505 07:09:22.668 | INFO | "pinned" : " 29 GB" | 20260505 07:09:22.668 | INFO | } | 20260505 07:09:22.668 | INFO | Perf report for the last 5.13935 seconds: | 20260505 07:09:22.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:09:22.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:22.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:22.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:22.668 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 07:09:22.668 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:22.668 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:22.668 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 07:09:22.668 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:22.668 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:22.668 | INFO | Uploaded 36GiB to GPU in 2.16792s (16.6058GiB/s) | 20260505 07:09:22.668 | INFO | Downloaded 34GiB to GPU in 2.04234s (16.6476GiB/s) | 20260505 07:09:22.668 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 07:09:22.668 | INFO | Total mem usage: 15.3 GiB | 20260505 07:09:22.668 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 07:09:22.668 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -5.97%) | 20260505 07:09:22.668 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 07:09:22.668 | INFO | High Water Breakdown: | 20260505 07:09:22.668 | INFO | 60.6% : 8.77 GiB ( 93.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 07:09:22.668 | INFO | 39.2% : 5.68 GiB ( 85.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 07:09:22.668 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 07:09:22.668 | INFO | ------------------------------------------ | 20260505 07:09:22.668 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:09:22.668 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 07:09:22.668 | INFO | Total mem usage: 26.5 GiB | 20260505 07:09:22.668 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 07:09:22.668 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 07:09:22.668 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 07:09:22.668 | INFO | High Water Breakdown: | 20260505 07:09:22.668 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:09:22.668 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:09:22.668 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:09:22.668 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:09:22.668 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:09:22.668 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:09:22.668 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 07:09:22.668 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 07:09:22.668 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:09:22.668 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 07:09:22.669 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 07:09:22.669 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 07:09:22.669 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 07:09:22.669 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 07:09:22.669 | INFO | ------------------------------------------ | 20260505 07:09:22.669 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:09:22.669 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 07:09:22.669 | INFO | Total mem usage: 25.6 GiB | 20260505 07:09:22.669 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 07:09:22.669 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 07:09:22.669 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 07:09:22.669 | INFO | High Water Breakdown: | 20260505 07:09:22.669 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 07:09:22.669 | INFO | ------------------------------------------ | 20260505 07:09:22.669 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:09:22.669 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 07:09:22.669 | INFO | Total mem usage: 15.7 GiB | 20260505 07:09:22.669 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 07:09:22.669 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 07:09:22.669 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 07:09:22.669 | INFO | High Water Breakdown: | 20260505 07:09:22.669 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 07:09:22.669 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 07:09:22.669 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 07:09:22.669 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 07:09:22.669 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 07:09:22.669 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 07:09:22.669 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 07:09:22.669 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 07:09:22.669 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 07:09:22.669 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:09:22.669 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 07:09:22.669 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:09:22.669 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 07:09:22.669 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 07:09:22.669 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 07:09:22.669 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 07:09:22.669 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:09:22.669 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 07:09:22.669 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:09:22.669 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:09:22.669 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:09:22.669 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 07:09:22.669 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 07:09:22.669 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:09:22.669 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 07:09:22.669 | INFO | ------------------------------------------ | 20260505 07:09:22.669 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:09:22.669 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 07:09:22.669 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:09:22.669 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:09:22.669 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.52e+01, 1.37e+01 / 1.98e+00 | 20260505 07:09:22.669 | INFO | PerfReport: 0.29%, AggregateStats -- 9.40e-02 / 1.04e-01, 1.14e-02 / 5.90e-03 | 20260505 07:09:22.669 | INFO | PerfReport: 11.61%, Baseline -- 3.82e+00 / 4.43e+00, 6.29e-01 / 2.93e-01 | 20260505 07:09:22.669 | INFO | PerfReport: 3.31%, Histogram -- 1.09e+00 / 1.48e+00, 2.15e-01 / 1.09e-01 | 20260505 07:09:22.669 | INFO | PerfReport: 15.47%, DME -- 4.07e+01 / 4.45e+01, 4.16e+00 / 1.40e+00 | 20260505 07:09:22.669 | INFO | PerfReport: 56.20%, FrameLabeling -- 1.85e+01 / 1.92e+01, 2.03e+00 / 1.39e+00 | 20260505 07:09:22.669 | INFO | PerfReport: 7.65%, PulseAccumulating -- 2.52e+00 / 3.18e+00, 4.50e-01 / 2.42e-01 | 20260505 07:09:22.669 | INFO | PerfReport: 4.02%, Metrics -- 1.32e+00 / 1.75e+00, 3.68e-01 / 1.65e-01 | 20260505 07:09:22.669 | INFO | PerfReport: 1.46%, PulseSerializing -- 4.80e-01 / 5.58e-01, 9.23e-02 / 2.99e-02 | 20260505 07:09:22.669 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 07:09:22.669 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:09:22.669 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:09:22.669 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.22e+01 / 0.00e+00 | 20260505 07:09:22.669 | INFO | PerfReport: 22.29%, Upload -- 2.26e+01 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260505 07:09:22.669 | INFO | PerfReport: 21.50%, Download -- 2.18e+01 / 0.00e+00, 7.84e+00 / 0.00e+00 | 20260505 07:09:22.669 | INFO | PerfReport: 32.44%, Compute -- 3.29e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 07:09:22.669 | INFO | PerfReport: 1.45%, LockUpload -- 1.47e+00 / 0.00e+00, 7.02e+00 / 0.00e+00 | 20260505 07:09:22.669 | INFO | PerfReport: 0.84%, LockDownload -- 8.52e-01 / 0.00e+00, 2.24e+00 / 0.00e+00 | 20260505 07:09:22.669 | INFO | PerfReport: 21.47%, LockCompute -- 2.18e+01 / 0.00e+00, 1.68e+01 / 0.00e+00 | 20260505 07:09:22.670 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.60e-02 / 0.00e+00 | 20260505 07:09:22.788 | INFO | Starting a round of Baz IO | 20260505 07:09:26.089 | INFO | Populating RTMetrics report | 20260505 07:09:26.089 | INFO | RTMetrics report populated | 20260505 07:09:26.089 | INFO | Opening tmp RTMetrics json file | 20260505 07:09:26.112 | INFO | Serializing RTMetrics to json | 20260505 07:09:26.117 | INFO | Serialization done, swapping tmp file now... | 20260505 07:09:26.126 | INFO | RTMetrics done | 20260505 07:09:26.207 | INFO | Finished a round of Baz IO: Wrote 2.421 GiB in 3.419 seconds | 20260505 07:09:26.207 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 30.74% / 0.04% / 69.22% | 20260505 07:09:26.207 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.01395% / 3.57628e-05% / 6.75519e-05% | 20260505 07:09:26.207 | INFO | Baz IO overall: | 20260505 07:09:26.207 | INFO | BytesWritten : 700297890823 | 20260505 07:09:26.207 | INFO | NumEvents : 331978481630 | 20260505 07:09:27.807 | INFO | Analyzing chunk frames = [1302528,1303040) | 20260505 07:09:27.807 | INFO | MemInfo: { | 20260505 07:09:27.807 | INFO | "gpu" : " 17 GB", | 20260505 07:09:27.807 | INFO | "huge" : "", | 20260505 07:09:27.807 | INFO | "huge_shared" : " 27 GB", | 20260505 07:09:27.807 | INFO | "malloc" : " 14 GB", | 20260505 07:09:27.807 | INFO | "pinned" : " 29 GB" | 20260505 07:09:27.807 | INFO | } | 20260505 07:09:27.807 | INFO | Perf report for the last 5.13953 seconds: | 20260505 07:09:27.807 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:09:27.807 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:27.807 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:27.807 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:27.807 | INFO | ANALYSIS : 65.4%, 2.9, 99.9ms, 0 | 20260505 07:09:27.807 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:27.807 | INFO | RT_METRICS : 46.6%, 1.0, 24.4ms, 0 | 20260505 07:09:27.807 | INFO | PRE_HQ : 64.3%, 1.9, 62.7ms, 0 | 20260505 07:09:27.807 | INFO | BAZWRITER : 65.8%, 4.1, 138.2ms, 0 | 20260505 07:09:27.807 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 07:09:27.807 | INFO | Uploaded 36GiB to GPU in 2.21687s (16.2391GiB/s) | 20260505 07:09:27.807 | INFO | Downloaded 38GiB to GPU in 2.33906s (16.2458GiB/s) | 20260505 07:09:32.946 | INFO | Analyzing chunk frames = [1303040,1303552) | 20260505 07:09:32.946 | INFO | MemInfo: { | 20260505 07:09:32.946 | INFO | "gpu" : " 17 GB", | 20260505 07:09:32.946 | INFO | "huge" : "", | 20260505 07:09:32.946 | INFO | "huge_shared" : " 27 GB", | 20260505 07:09:32.946 | INFO | "malloc" : " 14 GB", | 20260505 07:09:32.946 | INFO | "pinned" : " 29 GB" | 20260505 07:09:32.946 | INFO | } | 20260505 07:09:32.946 | INFO | Perf report for the last 5.13904 seconds: | 20260505 07:09:32.946 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:09:32.946 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:32.946 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:32.946 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:32.946 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 07:09:32.946 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:32.946 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:32.946 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:09:32.946 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:32.946 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:32.946 | INFO | Uploaded 36GiB to GPU in 2.11231s (17.043GiB/s) | 20260505 07:09:32.946 | INFO | Downloaded 34GiB to GPU in 2.03526s (16.7055GiB/s) | 20260505 07:09:38.085 | INFO | Analyzing chunk frames = [1303552,1304064) | 20260505 07:09:38.085 | INFO | MemInfo: { | 20260505 07:09:38.085 | INFO | "gpu" : " 17 GB", | 20260505 07:09:38.085 | INFO | "huge" : "", | 20260505 07:09:38.085 | INFO | "huge_shared" : " 27 GB", | 20260505 07:09:38.085 | INFO | "malloc" : " 14 GB", | 20260505 07:09:38.085 | INFO | "pinned" : " 29 GB" | 20260505 07:09:38.085 | INFO | } | 20260505 07:09:38.085 | INFO | Perf report for the last 5.13927 seconds: | 20260505 07:09:38.085 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:09:38.085 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:38.085 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:38.085 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:38.086 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260505 07:09:38.086 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:38.086 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:38.086 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 07:09:38.086 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:38.086 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:38.086 | INFO | Uploaded 36GiB to GPU in 2.15285s (16.722GiB/s) | 20260505 07:09:38.086 | INFO | Downloaded 34GiB to GPU in 2.05376s (16.555GiB/s) | 20260505 07:09:43.226 | INFO | Analyzing chunk frames = [1304064,1304576) | 20260505 07:09:43.226 | INFO | MemInfo: { | 20260505 07:09:43.226 | INFO | "gpu" : " 17 GB", | 20260505 07:09:43.226 | INFO | "huge" : "", | 20260505 07:09:43.226 | INFO | "huge_shared" : " 27 GB", | 20260505 07:09:43.226 | INFO | "malloc" : " 14 GB", | 20260505 07:09:43.226 | INFO | "pinned" : " 29 GB" | 20260505 07:09:43.226 | INFO | } | 20260505 07:09:43.226 | INFO | Perf report for the last 5.14052 seconds: | 20260505 07:09:43.226 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:09:43.226 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:43.226 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:43.226 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:43.226 | INFO | ANALYSIS : 63.4%, 3.0, 98.6ms, 0 | 20260505 07:09:43.226 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:43.226 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:43.226 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 07:09:43.226 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:43.226 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:43.226 | INFO | Uploaded 36GiB to GPU in 2.1377s (16.8405GiB/s) | 20260505 07:09:43.226 | INFO | Downloaded 34GiB to GPU in 2.02568s (16.7845GiB/s) | 20260505 07:09:48.368 | INFO | Analyzing chunk frames = [1304576,1305088) | 20260505 07:09:48.368 | INFO | MemInfo: { | 20260505 07:09:48.368 | INFO | "gpu" : " 17 GB", | 20260505 07:09:48.368 | INFO | "huge" : "", | 20260505 07:09:48.368 | INFO | "huge_shared" : " 27 GB", | 20260505 07:09:48.368 | INFO | "malloc" : " 14 GB", | 20260505 07:09:48.368 | INFO | "pinned" : " 29 GB" | 20260505 07:09:48.368 | INFO | } | 20260505 07:09:48.368 | INFO | Perf report for the last 5.14191 seconds: | 20260505 07:09:48.368 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:09:48.368 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:48.368 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:48.368 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:48.368 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 07:09:48.368 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:48.368 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:48.368 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 07:09:48.368 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:48.368 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:48.368 | INFO | Uploaded 36GiB to GPU in 2.14464s (16.786GiB/s) | 20260505 07:09:48.368 | INFO | Downloaded 34GiB to GPU in 2.05063s (16.5803GiB/s) | 20260505 07:09:53.505 | INFO | Analyzing chunk frames = [1305088,1305600) | 20260505 07:09:53.505 | INFO | MemInfo: { | 20260505 07:09:53.505 | INFO | "gpu" : " 17 GB", | 20260505 07:09:53.505 | INFO | "huge" : "", | 20260505 07:09:53.505 | INFO | "huge_shared" : " 27 GB", | 20260505 07:09:53.505 | INFO | "malloc" : " 14 GB", | 20260505 07:09:53.505 | INFO | "pinned" : " 29 GB" | 20260505 07:09:53.505 | INFO | } | 20260505 07:09:53.505 | INFO | Perf report for the last 5.13741 seconds: | 20260505 07:09:53.505 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:09:53.505 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:53.505 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:53.505 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:53.505 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 07:09:53.505 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:53.505 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:53.505 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 07:09:53.505 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:53.505 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:53.506 | INFO | Uploaded 36GiB to GPU in 2.15975s (16.6686GiB/s) | 20260505 07:09:53.506 | INFO | Downloaded 34GiB to GPU in 2.05347s (16.5573GiB/s) | 20260505 07:09:58.643 | INFO | Analyzing chunk frames = [1305600,1306112) | 20260505 07:09:58.643 | INFO | MemInfo: { | 20260505 07:09:58.643 | INFO | "gpu" : " 17 GB", | 20260505 07:09:58.643 | INFO | "huge" : "", | 20260505 07:09:58.643 | INFO | "huge_shared" : " 27 GB", | 20260505 07:09:58.643 | INFO | "malloc" : " 14 GB", | 20260505 07:09:58.643 | INFO | "pinned" : " 29 GB" | 20260505 07:09:58.643 | INFO | } | 20260505 07:09:58.643 | INFO | Perf report for the last 5.13799 seconds: | 20260505 07:09:58.643 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:09:58.643 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:58.643 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:58.643 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:58.643 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 07:09:58.643 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:58.643 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:58.643 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 07:09:58.643 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:58.643 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:09:58.644 | INFO | Uploaded 36GiB to GPU in 2.17879s (16.5229GiB/s) | 20260505 07:09:58.644 | INFO | Downloaded 34GiB to GPU in 2.14504s (15.8505GiB/s) | 20260505 07:10:03.783 | INFO | Analyzing chunk frames = [1306112,1306624) | 20260505 07:10:03.783 | INFO | MemInfo: { | 20260505 07:10:03.783 | INFO | "gpu" : " 17 GB", | 20260505 07:10:03.783 | INFO | "huge" : "", | 20260505 07:10:03.783 | INFO | "huge_shared" : " 27 GB", | 20260505 07:10:03.783 | INFO | "malloc" : " 14 GB", | 20260505 07:10:03.783 | INFO | "pinned" : " 29 GB" | 20260505 07:10:03.783 | INFO | } | 20260505 07:10:03.783 | INFO | Perf report for the last 5.13943 seconds: | 20260505 07:10:03.783 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:10:03.783 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:03.783 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:03.783 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:03.783 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 07:10:03.783 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:03.783 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:03.783 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260505 07:10:03.783 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:03.783 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:03.783 | INFO | Uploaded 36GiB to GPU in 2.1689s (16.5983GiB/s) | 20260505 07:10:03.783 | INFO | Downloaded 34GiB to GPU in 2.03283s (16.7254GiB/s) | 20260505 07:10:03.910 | INFO | Starting a round of Baz IO | 20260505 07:10:07.192 | INFO | Populating RTMetrics report | 20260505 07:10:07.193 | INFO | RTMetrics report populated | 20260505 07:10:07.193 | INFO | Opening tmp RTMetrics json file | 20260505 07:10:07.205 | INFO | Serializing RTMetrics to json | 20260505 07:10:07.227 | INFO | Serialization done, swapping tmp file now... | 20260505 07:10:07.245 | INFO | RTMetrics done | 20260505 07:10:07.302 | INFO | Finished a round of Baz IO: Wrote 2.422 GiB in 3.392 seconds | 20260505 07:10:07.302 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 30.71% / 0.04% / 69.26% | 20260505 07:10:07.302 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.0018% / 3.17891e-05% / 8.74201e-05% | 20260505 07:10:07.302 | INFO | Baz IO overall: | 20260505 07:10:07.302 | INFO | BytesWritten : 702898447563 | 20260505 07:10:07.302 | INFO | NumEvents : 333223505564 | 20260505 07:10:08.921 | INFO | Analyzing chunk frames = [1306624,1307136) | 20260505 07:10:08.921 | INFO | MemInfo: { | 20260505 07:10:08.921 | INFO | "gpu" : " 17 GB", | 20260505 07:10:08.921 | INFO | "huge" : "", | 20260505 07:10:08.921 | INFO | "huge_shared" : " 27 GB", | 20260505 07:10:08.922 | INFO | "malloc" : " 14 GB", | 20260505 07:10:08.922 | INFO | "pinned" : " 29 GB" | 20260505 07:10:08.922 | INFO | } | 20260505 07:10:08.922 | INFO | Perf report for the last 5.1388 seconds: | 20260505 07:10:08.922 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:10:08.922 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:08.922 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:08.922 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:08.922 | INFO | ANALYSIS : 65.2%, 2.9, 99.6ms, 0 | 20260505 07:10:08.922 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:08.922 | INFO | RT_METRICS : 47.0%, 1.0, 24.7ms, 0 | 20260505 07:10:08.922 | INFO | PRE_HQ : 64.1%, 1.9, 62.8ms, 0 | 20260505 07:10:08.922 | INFO | BAZWRITER : 65.6%, 4.1, 138.4ms, 0 | 20260505 07:10:08.922 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 07:10:08.922 | INFO | Uploaded 36GiB to GPU in 2.21433s (16.2578GiB/s) | 20260505 07:10:08.922 | INFO | Downloaded 38GiB to GPU in 2.29874s (16.5308GiB/s) | 20260505 07:10:14.061 | INFO | Analyzing chunk frames = [1307136,1307648) | 20260505 07:10:14.061 | INFO | MemInfo: { | 20260505 07:10:14.061 | INFO | "gpu" : " 17 GB", | 20260505 07:10:14.061 | INFO | "huge" : "", | 20260505 07:10:14.061 | INFO | "huge_shared" : " 27 GB", | 20260505 07:10:14.061 | INFO | "malloc" : " 14 GB", | 20260505 07:10:14.061 | INFO | "pinned" : " 29 GB" | 20260505 07:10:14.061 | INFO | } | 20260505 07:10:14.061 | INFO | Perf report for the last 5.13958 seconds: | 20260505 07:10:14.061 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:10:14.061 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:14.061 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:14.061 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:14.061 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 07:10:14.061 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:14.061 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:14.061 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 07:10:14.061 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:14.061 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:14.062 | INFO | Uploaded 36GiB to GPU in 2.1042s (17.1086GiB/s) | 20260505 07:10:14.062 | INFO | Downloaded 34GiB to GPU in 2.03111s (16.7396GiB/s) | 20260505 07:10:19.200 | INFO | Analyzing chunk frames = [1307648,1308160) | 20260505 07:10:19.200 | INFO | MemInfo: { | 20260505 07:10:19.200 | INFO | "gpu" : " 17 GB", | 20260505 07:10:19.200 | INFO | "huge" : "", | 20260505 07:10:19.200 | INFO | "huge_shared" : " 27 GB", | 20260505 07:10:19.200 | INFO | "malloc" : " 14 GB", | 20260505 07:10:19.200 | INFO | "pinned" : " 29 GB" | 20260505 07:10:19.200 | INFO | } | 20260505 07:10:19.200 | INFO | Perf report for the last 5.13904 seconds: | 20260505 07:10:19.200 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:10:19.200 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:19.200 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:19.200 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:19.200 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 07:10:19.200 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:19.200 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:19.200 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:10:19.200 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:19.200 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:19.201 | INFO | Uploaded 36GiB to GPU in 2.15227s (16.7265GiB/s) | 20260505 07:10:19.201 | INFO | Downloaded 34GiB to GPU in 2.057s (16.5289GiB/s) | 20260505 07:10:24.340 | INFO | Analyzing chunk frames = [1308160,1308672) | 20260505 07:10:24.340 | INFO | MemInfo: { | 20260505 07:10:24.340 | INFO | "gpu" : " 17 GB", | 20260505 07:10:24.340 | INFO | "huge" : "", | 20260505 07:10:24.340 | INFO | "huge_shared" : " 27 GB", | 20260505 07:10:24.340 | INFO | "malloc" : " 14 GB", | 20260505 07:10:24.340 | INFO | "pinned" : " 29 GB" | 20260505 07:10:24.340 | INFO | } | 20260505 07:10:24.340 | INFO | Perf report for the last 5.13944 seconds: | 20260505 07:10:24.340 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:10:24.340 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:24.340 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:24.340 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:24.340 | INFO | ANALYSIS : 63.4%, 3.0, 98.6ms, 0 | 20260505 07:10:24.340 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:24.340 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:24.340 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 07:10:24.340 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:24.340 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:24.340 | INFO | Uploaded 36GiB to GPU in 2.13226s (16.8835GiB/s) | 20260505 07:10:24.340 | INFO | Downloaded 34GiB to GPU in 2.02542s (16.7866GiB/s) | 20260505 07:10:29.479 | INFO | Analyzing chunk frames = [1308672,1309184) | 20260505 07:10:29.479 | INFO | MemInfo: { | 20260505 07:10:29.479 | INFO | "gpu" : " 17 GB", | 20260505 07:10:29.479 | INFO | "huge" : "", | 20260505 07:10:29.479 | INFO | "huge_shared" : " 27 GB", | 20260505 07:10:29.479 | INFO | "malloc" : " 14 GB", | 20260505 07:10:29.479 | INFO | "pinned" : " 29 GB" | 20260505 07:10:29.479 | INFO | } | 20260505 07:10:29.479 | INFO | Perf report for the last 5.13955 seconds: | 20260505 07:10:29.479 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:10:29.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:29.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:29.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:29.479 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 07:10:29.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:29.479 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:29.479 | INFO | PRE_HQ : 30.1%, 1.0, 15.7ms, 0 | 20260505 07:10:29.479 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:29.479 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:29.480 | INFO | Uploaded 36GiB to GPU in 2.15109s (16.7357GiB/s) | 20260505 07:10:29.480 | INFO | Downloaded 34GiB to GPU in 2.04719s (16.6082GiB/s) | 20260505 07:10:34.620 | INFO | Analyzing chunk frames = [1309184,1309696) | 20260505 07:10:34.620 | INFO | MemInfo: { | 20260505 07:10:34.620 | INFO | "gpu" : " 17 GB", | 20260505 07:10:34.620 | INFO | "huge" : "", | 20260505 07:10:34.620 | INFO | "huge_shared" : " 27 GB", | 20260505 07:10:34.620 | INFO | "malloc" : " 14 GB", | 20260505 07:10:34.620 | INFO | "pinned" : " 29 GB" | 20260505 07:10:34.620 | INFO | } | 20260505 07:10:34.620 | INFO | Perf report for the last 5.14076 seconds: | 20260505 07:10:34.620 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:10:34.620 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:34.620 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:34.620 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:34.620 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 07:10:34.620 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:34.620 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:34.620 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:10:34.620 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:34.620 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:34.620 | INFO | Uploaded 36GiB to GPU in 2.16402s (16.6357GiB/s) | 20260505 07:10:34.620 | INFO | Downloaded 34GiB to GPU in 2.05824s (16.519GiB/s) | 20260505 07:10:39.758 | INFO | Analyzing chunk frames = [1309696,1310208) | 20260505 07:10:39.758 | INFO | MemInfo: { | 20260505 07:10:39.758 | INFO | "gpu" : " 17 GB", | 20260505 07:10:39.758 | INFO | "huge" : "", | 20260505 07:10:39.758 | INFO | "huge_shared" : " 27 GB", | 20260505 07:10:39.758 | INFO | "malloc" : " 14 GB", | 20260505 07:10:39.758 | INFO | "pinned" : " 29 GB" | 20260505 07:10:39.758 | INFO | } | 20260505 07:10:39.758 | INFO | Perf report for the last 5.13788 seconds: | 20260505 07:10:39.758 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:10:39.758 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:39.758 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:39.758 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:39.758 | INFO | ANALYSIS : 64.3%, 3.0, 100.3ms, 0 | 20260505 07:10:39.758 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:39.758 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:39.758 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260505 07:10:39.758 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:39.758 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:39.758 | INFO | Uploaded 36GiB to GPU in 2.20208s (16.3482GiB/s) | 20260505 07:10:39.758 | INFO | Downloaded 34GiB to GPU in 2.15785s (15.7564GiB/s) | 20260505 07:10:44.897 | INFO | Analyzing chunk frames = [1310208,1310720) | 20260505 07:10:44.897 | INFO | MemInfo: { | 20260505 07:10:44.897 | INFO | "gpu" : " 17 GB", | 20260505 07:10:44.897 | INFO | "huge" : "", | 20260505 07:10:44.897 | INFO | "huge_shared" : " 27 GB", | 20260505 07:10:44.897 | INFO | "malloc" : " 14 GB", | 20260505 07:10:44.897 | INFO | "pinned" : " 29 GB" | 20260505 07:10:44.897 | INFO | } | 20260505 07:10:44.897 | INFO | Perf report for the last 5.13927 seconds: | 20260505 07:10:44.897 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:10:44.897 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:44.897 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:44.897 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:44.897 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 07:10:44.897 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:44.897 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:44.897 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 07:10:44.897 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:44.897 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:44.897 | INFO | Uploaded 36GiB to GPU in 2.16536s (16.6254GiB/s) | 20260505 07:10:44.897 | INFO | Downloaded 34GiB to GPU in 2.0407s (16.6609GiB/s) | 20260505 07:10:44.898 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 07:10:44.898 | INFO | Total mem usage: 15.3 GiB | 20260505 07:10:44.898 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 07:10:44.898 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -6.01%) | 20260505 07:10:44.898 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 07:10:44.898 | INFO | High Water Breakdown: | 20260505 07:10:44.898 | INFO | 60.6% : 8.76 GiB ( 93.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 07:10:44.898 | INFO | 39.2% : 5.67 GiB ( 85.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 07:10:44.898 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 07:10:44.898 | INFO | ------------------------------------------ | 20260505 07:10:44.898 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:10:44.898 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 07:10:44.898 | INFO | Total mem usage: 26.5 GiB | 20260505 07:10:44.898 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 07:10:44.898 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 07:10:44.898 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 07:10:44.898 | INFO | High Water Breakdown: | 20260505 07:10:44.898 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:10:44.898 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:10:44.898 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:10:44.898 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:10:44.898 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:10:44.898 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:10:44.898 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 07:10:44.898 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 07:10:44.898 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:10:44.898 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 07:10:44.898 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 07:10:44.898 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 07:10:44.898 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 07:10:44.898 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 07:10:44.898 | INFO | ------------------------------------------ | 20260505 07:10:44.898 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:10:44.898 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 07:10:44.898 | INFO | Total mem usage: 25.6 GiB | 20260505 07:10:44.898 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 07:10:44.898 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 07:10:44.898 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 07:10:44.898 | INFO | High Water Breakdown: | 20260505 07:10:44.898 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 07:10:44.898 | INFO | ------------------------------------------ | 20260505 07:10:44.898 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:10:44.898 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 07:10:44.898 | INFO | Total mem usage: 15.7 GiB | 20260505 07:10:44.898 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 07:10:44.898 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 07:10:44.898 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 07:10:44.898 | INFO | High Water Breakdown: | 20260505 07:10:44.898 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 07:10:44.898 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 07:10:44.898 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 07:10:44.898 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 07:10:44.898 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 07:10:44.898 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 07:10:44.898 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 07:10:44.898 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 07:10:44.898 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 07:10:44.898 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:10:44.898 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 07:10:44.898 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:10:44.898 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 07:10:44.898 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 07:10:44.898 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 07:10:44.898 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 07:10:44.898 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:10:44.898 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 07:10:44.898 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:10:44.898 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:10:44.898 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:10:44.898 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 07:10:44.898 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 07:10:44.898 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:10:44.898 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 07:10:44.898 | INFO | ------------------------------------------ | 20260505 07:10:44.898 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:10:44.898 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 07:10:44.898 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:10:44.898 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:10:44.898 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.50e+01, 1.37e+01 / 1.93e+00 | 20260505 07:10:44.898 | INFO | PerfReport: 0.29%, AggregateStats -- 9.42e-02 / 1.04e-01, 1.30e-02 / 7.24e-03 | 20260505 07:10:44.898 | INFO | PerfReport: 11.64%, Baseline -- 3.83e+00 / 4.48e+00, 6.34e-01 / 2.83e-01 | 20260505 07:10:44.898 | INFO | PerfReport: 3.32%, Histogram -- 1.09e+00 / 1.47e+00, 2.15e-01 / 1.29e-01 | 20260505 07:10:44.898 | INFO | PerfReport: 15.46%, DME -- 4.07e+01 / 4.44e+01, 4.05e+00 / 1.29e+00 | 20260505 07:10:44.898 | INFO | PerfReport: 56.12%, FrameLabeling -- 1.85e+01 / 1.90e+01, 1.99e+00 / 1.36e+00 | 20260505 07:10:44.898 | INFO | PerfReport: 7.68%, PulseAccumulating -- 2.53e+00 / 3.18e+00, 4.51e-01 / 2.61e-01 | 20260505 07:10:44.898 | INFO | PerfReport: 4.03%, Metrics -- 1.33e+00 / 1.75e+00, 3.69e-01 / 1.95e-01 | 20260505 07:10:44.899 | INFO | PerfReport: 1.46%, PulseSerializing -- 4.79e-01 / 5.61e-01, 9.19e-02 / 2.98e-02 | 20260505 07:10:44.899 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 07:10:44.899 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:10:44.899 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:10:44.899 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.22e+01 / 0.00e+00 | 20260505 07:10:44.899 | INFO | PerfReport: 22.19%, Upload -- 2.25e+01 / 0.00e+00, 7.87e+00 / 0.00e+00 | 20260505 07:10:44.899 | INFO | PerfReport: 21.48%, Download -- 2.18e+01 / 0.00e+00, 7.78e+00 / 0.00e+00 | 20260505 07:10:44.899 | INFO | PerfReport: 32.45%, Compute -- 3.29e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 07:10:44.899 | INFO | PerfReport: 1.50%, LockUpload -- 1.52e+00 / 0.00e+00, 7.06e+00 / 0.00e+00 | 20260505 07:10:44.899 | INFO | PerfReport: 0.82%, LockDownload -- 8.28e-01 / 0.00e+00, 2.20e+00 / 0.00e+00 | 20260505 07:10:44.899 | INFO | PerfReport: 21.54%, LockCompute -- 2.18e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260505 07:10:44.899 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 2.02e-02 / 0.00e+00 | 20260505 07:10:45.011 | INFO | Starting a round of Baz IO | 20260505 07:10:48.309 | INFO | Populating RTMetrics report | 20260505 07:10:48.309 | INFO | RTMetrics report populated | 20260505 07:10:48.309 | INFO | Opening tmp RTMetrics json file | 20260505 07:10:48.311 | INFO | Serializing RTMetrics to json | 20260505 07:10:48.316 | INFO | Serialization done, swapping tmp file now... | 20260505 07:10:48.318 | INFO | RTMetrics done | 20260505 07:10:48.470 | INFO | Finished a round of Baz IO: Wrote 2.42 GiB in 3.458 seconds | 20260505 07:10:48.470 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 30.67% / 0.03% / 69.29% | 20260505 07:10:48.470 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.97164% / 4.76837e-05% / 6.35783e-05% | 20260505 07:10:48.470 | INFO | Baz IO overall: | 20260505 07:10:48.470 | INFO | BytesWritten : 705496416670 | 20260505 07:10:48.470 | INFO | NumEvents : 334466358831 | 20260505 07:10:50.036 | INFO | Analyzing chunk frames = [1310720,1311232) | 20260505 07:10:50.036 | INFO | MemInfo: { | 20260505 07:10:50.036 | INFO | "gpu" : " 17 GB", | 20260505 07:10:50.036 | INFO | "huge" : "", | 20260505 07:10:50.037 | INFO | "huge_shared" : " 27 GB", | 20260505 07:10:50.037 | INFO | "malloc" : " 14 GB", | 20260505 07:10:50.037 | INFO | "pinned" : " 29 GB" | 20260505 07:10:50.037 | INFO | } | 20260505 07:10:50.037 | INFO | Perf report for the last 5.13943 seconds: | 20260505 07:10:50.037 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:10:50.037 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:50.037 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:50.037 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:50.037 | INFO | ANALYSIS : 65.2%, 2.9, 99.9ms, 0 | 20260505 07:10:50.037 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:50.037 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 07:10:50.037 | INFO | PRE_HQ : 64.1%, 1.9, 62.7ms, 0 | 20260505 07:10:50.037 | INFO | BAZWRITER : 66.5%, 4.1, 139.6ms, 0 | 20260505 07:10:50.037 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 07:10:50.037 | INFO | Uploaded 36GiB to GPU in 2.24733s (16.019GiB/s) | 20260505 07:10:50.037 | INFO | Downloaded 38GiB to GPU in 2.31017s (16.449GiB/s) | 20260505 07:10:55.176 | INFO | Analyzing chunk frames = [1311232,1311744) | 20260505 07:10:55.176 | INFO | MemInfo: { | 20260505 07:10:55.176 | INFO | "gpu" : " 17 GB", | 20260505 07:10:55.176 | INFO | "huge" : "", | 20260505 07:10:55.176 | INFO | "huge_shared" : " 27 GB", | 20260505 07:10:55.176 | INFO | "malloc" : " 14 GB", | 20260505 07:10:55.176 | INFO | "pinned" : " 29 GB" | 20260505 07:10:55.176 | INFO | } | 20260505 07:10:55.176 | INFO | Perf report for the last 5.13916 seconds: | 20260505 07:10:55.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:10:55.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:55.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:55.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:55.176 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 07:10:55.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:55.176 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:55.176 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:10:55.176 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:55.176 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:10:55.176 | INFO | Uploaded 36GiB to GPU in 2.11746s (17.0015GiB/s) | 20260505 07:10:55.176 | INFO | Downloaded 34GiB to GPU in 2.03877s (16.6767GiB/s) | 20260505 07:11:00.315 | INFO | Analyzing chunk frames = [1311744,1312256) | 20260505 07:11:00.315 | INFO | MemInfo: { | 20260505 07:11:00.315 | INFO | "gpu" : " 17 GB", | 20260505 07:11:00.315 | INFO | "huge" : "", | 20260505 07:11:00.315 | INFO | "huge_shared" : " 27 GB", | 20260505 07:11:00.315 | INFO | "malloc" : " 14 GB", | 20260505 07:11:00.315 | INFO | "pinned" : " 29 GB" | 20260505 07:11:00.315 | INFO | } | 20260505 07:11:00.315 | INFO | Perf report for the last 5.13936 seconds: | 20260505 07:11:00.315 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:11:00.315 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:00.315 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:00.315 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:00.315 | INFO | ANALYSIS : 63.6%, 3.0, 99.0ms, 0 | 20260505 07:11:00.315 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:00.315 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:00.315 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 07:11:00.315 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:00.315 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:00.315 | INFO | Uploaded 36GiB to GPU in 2.15776s (16.6839GiB/s) | 20260505 07:11:00.315 | INFO | Downloaded 34GiB to GPU in 2.0459s (16.6186GiB/s) | 20260505 07:11:05.454 | INFO | Analyzing chunk frames = [1312256,1312768) | 20260505 07:11:05.454 | INFO | MemInfo: { | 20260505 07:11:05.454 | INFO | "gpu" : " 17 GB", | 20260505 07:11:05.454 | INFO | "huge" : "", | 20260505 07:11:05.454 | INFO | "huge_shared" : " 27 GB", | 20260505 07:11:05.454 | INFO | "malloc" : " 14 GB", | 20260505 07:11:05.454 | INFO | "pinned" : " 29 GB" | 20260505 07:11:05.454 | INFO | } | 20260505 07:11:05.454 | INFO | Perf report for the last 5.13924 seconds: | 20260505 07:11:05.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:11:05.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:05.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:05.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:05.454 | INFO | ANALYSIS : 63.4%, 3.0, 98.5ms, 0 | 20260505 07:11:05.455 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:05.455 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:05.455 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:11:05.455 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:05.455 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:05.455 | INFO | Uploaded 36GiB to GPU in 2.12621s (16.9315GiB/s) | 20260505 07:11:05.455 | INFO | Downloaded 34GiB to GPU in 2.02181s (16.8166GiB/s) | 20260505 07:11:10.594 | INFO | Analyzing chunk frames = [1312768,1313280) | 20260505 07:11:10.594 | INFO | MemInfo: { | 20260505 07:11:10.594 | INFO | "gpu" : " 17 GB", | 20260505 07:11:10.594 | INFO | "huge" : "", | 20260505 07:11:10.594 | INFO | "huge_shared" : " 27 GB", | 20260505 07:11:10.594 | INFO | "malloc" : " 14 GB", | 20260505 07:11:10.594 | INFO | "pinned" : " 29 GB" | 20260505 07:11:10.594 | INFO | } | 20260505 07:11:10.594 | INFO | Perf report for the last 5.13932 seconds: | 20260505 07:11:10.594 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:11:10.594 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:10.594 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:10.594 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:10.594 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 07:11:10.594 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:10.594 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:10.594 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:11:10.594 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:10.594 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:10.594 | INFO | Uploaded 36GiB to GPU in 2.16181s (16.6527GiB/s) | 20260505 07:11:10.594 | INFO | Downloaded 34GiB to GPU in 2.05228s (16.5669GiB/s) | 20260505 07:11:15.735 | INFO | Analyzing chunk frames = [1313280,1313792) | 20260505 07:11:15.735 | INFO | MemInfo: { | 20260505 07:11:15.735 | INFO | "gpu" : " 17 GB", | 20260505 07:11:15.735 | INFO | "huge" : "", | 20260505 07:11:15.735 | INFO | "huge_shared" : " 27 GB", | 20260505 07:11:15.735 | INFO | "malloc" : " 14 GB", | 20260505 07:11:15.735 | INFO | "pinned" : " 29 GB" | 20260505 07:11:15.735 | INFO | } | 20260505 07:11:15.735 | INFO | Perf report for the last 5.14127 seconds: | 20260505 07:11:15.735 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:11:15.735 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:15.735 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:15.735 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:15.735 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 07:11:15.735 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:15.735 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:15.735 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 07:11:15.735 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:15.735 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:15.735 | INFO | Uploaded 36GiB to GPU in 2.15962s (16.6696GiB/s) | 20260505 07:11:15.735 | INFO | Downloaded 34GiB to GPU in 2.05515s (16.5438GiB/s) | 20260505 07:11:20.872 | INFO | Analyzing chunk frames = [1313792,1314304) | 20260505 07:11:20.872 | INFO | MemInfo: { | 20260505 07:11:20.872 | INFO | "gpu" : " 17 GB", | 20260505 07:11:20.872 | INFO | "huge" : "", | 20260505 07:11:20.872 | INFO | "huge_shared" : " 27 GB", | 20260505 07:11:20.872 | INFO | "malloc" : " 14 GB", | 20260505 07:11:20.872 | INFO | "pinned" : " 29 GB" | 20260505 07:11:20.872 | INFO | } | 20260505 07:11:20.872 | INFO | Perf report for the last 5.13725 seconds: | 20260505 07:11:20.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:11:20.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:20.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:20.872 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:20.872 | INFO | ANALYSIS : 64.8%, 3.0, 101.1ms, 0 | 20260505 07:11:20.872 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:20.872 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:20.872 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260505 07:11:20.872 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:20.872 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:20.873 | INFO | Uploaded 36GiB to GPU in 2.22668s (16.1676GiB/s) | 20260505 07:11:20.873 | INFO | Downloaded 34GiB to GPU in 2.13895s (15.8956GiB/s) | 20260505 07:11:26.012 | INFO | Analyzing chunk frames = [1314304,1314816) | 20260505 07:11:26.012 | INFO | MemInfo: { | 20260505 07:11:26.012 | INFO | "gpu" : " 17 GB", | 20260505 07:11:26.012 | INFO | "huge" : "", | 20260505 07:11:26.012 | INFO | "huge_shared" : " 27 GB", | 20260505 07:11:26.012 | INFO | "malloc" : " 14 GB", | 20260505 07:11:26.012 | INFO | "pinned" : " 29 GB" | 20260505 07:11:26.012 | INFO | } | 20260505 07:11:26.012 | INFO | Perf report for the last 5.13964 seconds: | 20260505 07:11:26.012 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:11:26.012 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:26.012 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:26.012 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:26.012 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 07:11:26.012 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:26.012 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:26.012 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 07:11:26.012 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:26.012 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:26.013 | INFO | Uploaded 36GiB to GPU in 2.1737s (16.5616GiB/s) | 20260505 07:11:26.013 | INFO | Downloaded 34GiB to GPU in 2.04412s (16.6331GiB/s) | 20260505 07:11:26.127 | INFO | Starting a round of Baz IO | 20260505 07:11:29.427 | INFO | Populating RTMetrics report | 20260505 07:11:29.427 | INFO | RTMetrics report populated | 20260505 07:11:29.427 | INFO | Opening tmp RTMetrics json file | 20260505 07:11:29.428 | INFO | Serializing RTMetrics to json | 20260505 07:11:29.429 | INFO | Serialization done, swapping tmp file now... | 20260505 07:11:29.430 | INFO | RTMetrics done | 20260505 07:11:29.558 | INFO | Finished a round of Baz IO: Wrote 2.42 GiB in 3.431 seconds | 20260505 07:11:29.558 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 30.64% / 0.04% / 69.33% | 20260505 07:11:29.558 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.95253% / 5.5631e-05% / 8.74201e-05% | 20260505 07:11:29.558 | INFO | Baz IO overall: | 20260505 07:11:29.558 | INFO | BytesWritten : 708094517526 | 20260505 07:11:29.558 | INFO | NumEvents : 335709006426 | 20260505 07:11:31.151 | INFO | Analyzing chunk frames = [1314816,1315328) | 20260505 07:11:31.151 | INFO | MemInfo: { | 20260505 07:11:31.151 | INFO | "gpu" : " 17 GB", | 20260505 07:11:31.151 | INFO | "huge" : "", | 20260505 07:11:31.151 | INFO | "huge_shared" : " 27 GB", | 20260505 07:11:31.151 | INFO | "malloc" : " 14 GB", | 20260505 07:11:31.151 | INFO | "pinned" : " 29 GB" | 20260505 07:11:31.151 | INFO | } | 20260505 07:11:31.151 | INFO | Perf report for the last 5.13909 seconds: | 20260505 07:11:31.151 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:11:31.151 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:31.151 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:31.151 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:31.151 | INFO | ANALYSIS : 65.3%, 2.9, 99.5ms, 0 | 20260505 07:11:31.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:31.151 | INFO | RT_METRICS : 46.2%, 1.0, 24.2ms, 0 | 20260505 07:11:31.151 | INFO | PRE_HQ : 64.2%, 1.9, 63.7ms, 0 | 20260505 07:11:31.151 | INFO | BAZWRITER : 66.0%, 4.1, 138.6ms, 0 | 20260505 07:11:31.151 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 07:11:31.151 | INFO | Uploaded 36GiB to GPU in 2.22614s (16.1715GiB/s) | 20260505 07:11:31.151 | INFO | Downloaded 38GiB to GPU in 2.31092s (16.4437GiB/s) | 20260505 07:11:36.290 | INFO | Analyzing chunk frames = [1315328,1315840) | 20260505 07:11:36.290 | INFO | MemInfo: { | 20260505 07:11:36.290 | INFO | "gpu" : " 17 GB", | 20260505 07:11:36.290 | INFO | "huge" : "", | 20260505 07:11:36.290 | INFO | "huge_shared" : " 27 GB", | 20260505 07:11:36.290 | INFO | "malloc" : " 14 GB", | 20260505 07:11:36.290 | INFO | "pinned" : " 29 GB" | 20260505 07:11:36.290 | INFO | } | 20260505 07:11:36.290 | INFO | Perf report for the last 5.13931 seconds: | 20260505 07:11:36.290 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:11:36.290 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:36.290 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:36.290 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:36.290 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 07:11:36.290 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:36.290 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:36.290 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:11:36.290 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:36.290 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:36.291 | INFO | Uploaded 36GiB to GPU in 2.11119s (17.052GiB/s) | 20260505 07:11:36.291 | INFO | Downloaded 34GiB to GPU in 2.03585s (16.7007GiB/s) | 20260505 07:11:41.430 | INFO | Analyzing chunk frames = [1315840,1316352) | 20260505 07:11:41.430 | INFO | MemInfo: { | 20260505 07:11:41.430 | INFO | "gpu" : " 17 GB", | 20260505 07:11:41.430 | INFO | "huge" : "", | 20260505 07:11:41.430 | INFO | "huge_shared" : " 27 GB", | 20260505 07:11:41.430 | INFO | "malloc" : " 14 GB", | 20260505 07:11:41.430 | INFO | "pinned" : " 29 GB" | 20260505 07:11:41.430 | INFO | } | 20260505 07:11:41.430 | INFO | Perf report for the last 5.13939 seconds: | 20260505 07:11:41.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:11:41.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:41.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:41.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:41.430 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260505 07:11:41.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:41.430 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:41.430 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 07:11:41.430 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:41.430 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:41.431 | INFO | Uploaded 36GiB to GPU in 2.14445s (16.7875GiB/s) | 20260505 07:11:41.431 | INFO | Downloaded 34GiB to GPU in 2.05519s (16.5435GiB/s) | 20260505 07:11:46.569 | INFO | Analyzing chunk frames = [1316352,1316864) | 20260505 07:11:46.569 | INFO | MemInfo: { | 20260505 07:11:46.569 | INFO | "gpu" : " 17 GB", | 20260505 07:11:46.569 | INFO | "huge" : "", | 20260505 07:11:46.569 | INFO | "huge_shared" : " 27 GB", | 20260505 07:11:46.569 | INFO | "malloc" : " 14 GB", | 20260505 07:11:46.569 | INFO | "pinned" : " 29 GB" | 20260505 07:11:46.569 | INFO | } | 20260505 07:11:46.569 | INFO | Perf report for the last 5.13946 seconds: | 20260505 07:11:46.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:11:46.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:46.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:46.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:46.569 | INFO | ANALYSIS : 63.4%, 3.0, 98.4ms, 0 | 20260505 07:11:46.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:46.569 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:46.569 | INFO | PRE_HQ : 30.3%, 1.0, 15.8ms, 0 | 20260505 07:11:46.569 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:46.569 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:46.569 | INFO | Uploaded 36GiB to GPU in 2.12832s (16.9148GiB/s) | 20260505 07:11:46.570 | INFO | Downloaded 34GiB to GPU in 2.03316s (16.7228GiB/s) | 20260505 07:11:51.708 | INFO | Analyzing chunk frames = [1316864,1317376) | 20260505 07:11:51.708 | INFO | MemInfo: { | 20260505 07:11:51.708 | INFO | "gpu" : " 17 GB", | 20260505 07:11:51.708 | INFO | "huge" : "", | 20260505 07:11:51.708 | INFO | "huge_shared" : " 27 GB", | 20260505 07:11:51.708 | INFO | "malloc" : " 14 GB", | 20260505 07:11:51.708 | INFO | "pinned" : " 29 GB" | 20260505 07:11:51.708 | INFO | } | 20260505 07:11:51.708 | INFO | Perf report for the last 5.13919 seconds: | 20260505 07:11:51.708 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:11:51.708 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:51.708 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:51.708 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:51.708 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 07:11:51.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:51.709 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:51.709 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:11:51.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:51.709 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:51.709 | INFO | Uploaded 36GiB to GPU in 2.1607s (16.6613GiB/s) | 20260505 07:11:51.709 | INFO | Downloaded 34GiB to GPU in 2.05159s (16.5725GiB/s) | 20260505 07:11:56.849 | INFO | Analyzing chunk frames = [1317376,1317888) | 20260505 07:11:56.849 | INFO | MemInfo: { | 20260505 07:11:56.849 | INFO | "gpu" : " 17 GB", | 20260505 07:11:56.849 | INFO | "huge" : "", | 20260505 07:11:56.849 | INFO | "huge_shared" : " 27 GB", | 20260505 07:11:56.849 | INFO | "malloc" : " 14 GB", | 20260505 07:11:56.849 | INFO | "pinned" : " 29 GB" | 20260505 07:11:56.849 | INFO | } | 20260505 07:11:56.849 | INFO | Perf report for the last 5.14087 seconds: | 20260505 07:11:56.849 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:11:56.849 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:56.849 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:56.849 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:56.849 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 07:11:56.849 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:56.849 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:56.849 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:11:56.849 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:56.849 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:11:56.850 | INFO | Uploaded 36GiB to GPU in 2.16045s (16.6632GiB/s) | 20260505 07:11:56.850 | INFO | Downloaded 34GiB to GPU in 2.05865s (16.5157GiB/s) | 20260505 07:12:01.987 | INFO | Analyzing chunk frames = [1317888,1318400) | 20260505 07:12:01.987 | INFO | MemInfo: { | 20260505 07:12:01.987 | INFO | "gpu" : " 17 GB", | 20260505 07:12:01.987 | INFO | "huge" : "", | 20260505 07:12:01.987 | INFO | "huge_shared" : " 27 GB", | 20260505 07:12:01.987 | INFO | "malloc" : " 14 GB", | 20260505 07:12:01.987 | INFO | "pinned" : " 29 GB" | 20260505 07:12:01.987 | INFO | } | 20260505 07:12:01.987 | INFO | Perf report for the last 5.13799 seconds: | 20260505 07:12:01.987 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:12:01.987 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:01.987 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:01.987 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:01.987 | INFO | ANALYSIS : 64.7%, 3.0, 101.1ms, 0 | 20260505 07:12:01.987 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:01.987 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:01.987 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260505 07:12:01.987 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:01.987 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:01.988 | INFO | Uploaded 36GiB to GPU in 2.2113s (16.28GiB/s) | 20260505 07:12:01.988 | INFO | Downloaded 34GiB to GPU in 2.14604s (15.8432GiB/s) | 20260505 07:12:07.126 | INFO | Analyzing chunk frames = [1318400,1318912) | 20260505 07:12:07.126 | INFO | MemInfo: { | 20260505 07:12:07.126 | INFO | "gpu" : " 17 GB", | 20260505 07:12:07.126 | INFO | "huge" : "", | 20260505 07:12:07.126 | INFO | "huge_shared" : " 27 GB", | 20260505 07:12:07.127 | INFO | "malloc" : " 14 GB", | 20260505 07:12:07.127 | INFO | "pinned" : " 29 GB" | 20260505 07:12:07.127 | INFO | } | 20260505 07:12:07.127 | INFO | Perf report for the last 5.13928 seconds: | 20260505 07:12:07.127 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:12:07.127 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:07.127 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:07.127 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:07.127 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 07:12:07.127 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:07.127 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:07.127 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 07:12:07.127 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:07.127 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:07.127 | INFO | Uploaded 36GiB to GPU in 2.16557s (16.6238GiB/s) | 20260505 07:12:07.127 | INFO | Downloaded 34GiB to GPU in 2.03723s (16.6893GiB/s) | 20260505 07:12:07.127 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 07:12:07.127 | INFO | Total mem usage: 15.3 GiB | 20260505 07:12:07.127 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 07:12:07.127 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.71%) | 20260505 07:12:07.127 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 07:12:07.127 | INFO | High Water Breakdown: | 20260505 07:12:07.127 | INFO | 60.5% : 8.7 GiB ( 93% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 07:12:07.127 | INFO | 39.3% : 5.65 GiB ( 85% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 07:12:07.127 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 07:12:07.127 | INFO | ------------------------------------------ | 20260505 07:12:07.127 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:12:07.127 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 07:12:07.127 | INFO | Total mem usage: 26.5 GiB | 20260505 07:12:07.127 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 07:12:07.127 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 07:12:07.127 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 07:12:07.127 | INFO | High Water Breakdown: | 20260505 07:12:07.127 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:12:07.127 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:12:07.127 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:12:07.127 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:12:07.127 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:12:07.127 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:12:07.127 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 07:12:07.127 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 07:12:07.127 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:12:07.127 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 07:12:07.127 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 07:12:07.127 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 07:12:07.127 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 07:12:07.127 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 07:12:07.127 | INFO | ------------------------------------------ | 20260505 07:12:07.127 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:12:07.127 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 07:12:07.127 | INFO | Total mem usage: 25.6 GiB | 20260505 07:12:07.127 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 07:12:07.127 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 07:12:07.127 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 07:12:07.127 | INFO | High Water Breakdown: | 20260505 07:12:07.127 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 07:12:07.127 | INFO | ------------------------------------------ | 20260505 07:12:07.127 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:12:07.127 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 07:12:07.127 | INFO | Total mem usage: 15.7 GiB | 20260505 07:12:07.127 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 07:12:07.127 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 07:12:07.127 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 07:12:07.127 | INFO | High Water Breakdown: | 20260505 07:12:07.127 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 07:12:07.127 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 07:12:07.128 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 07:12:07.128 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 07:12:07.128 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 07:12:07.128 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 07:12:07.128 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 07:12:07.128 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 07:12:07.128 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 07:12:07.128 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:12:07.128 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 07:12:07.128 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:12:07.128 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 07:12:07.128 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 07:12:07.128 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 07:12:07.128 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 07:12:07.128 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:12:07.128 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 07:12:07.128 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:12:07.128 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:12:07.128 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:12:07.128 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 07:12:07.128 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 07:12:07.128 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:12:07.128 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 07:12:07.128 | INFO | ------------------------------------------ | 20260505 07:12:07.128 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:12:07.128 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 07:12:07.128 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:12:07.128 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:12:07.128 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.54e+01, 1.37e+01 / 1.60e+00 | 20260505 07:12:07.128 | INFO | PerfReport: 0.28%, AggregateStats -- 9.39e-02 / 1.04e-01, 1.06e-02 / 3.99e-03 | 20260505 07:12:07.128 | INFO | PerfReport: 11.59%, Baseline -- 3.82e+00 / 4.43e+00, 6.27e-01 / 2.40e-01 | 20260505 07:12:07.128 | INFO | PerfReport: 3.31%, Histogram -- 1.09e+00 / 1.46e+00, 2.14e-01 / 1.05e-01 | 20260505 07:12:07.128 | INFO | PerfReport: 15.49%, DME -- 4.08e+01 / 4.47e+01, 4.09e+00 / 1.20e+00 | 20260505 07:12:07.128 | INFO | PerfReport: 56.25%, FrameLabeling -- 1.85e+01 / 1.93e+01, 2.06e+00 / 1.28e+00 | 20260505 07:12:07.128 | INFO | PerfReport: 7.61%, PulseAccumulating -- 2.51e+00 / 3.16e+00, 4.41e-01 / 2.37e-01 | 20260505 07:12:07.128 | INFO | PerfReport: 4.02%, Metrics -- 1.32e+00 / 1.74e+00, 3.73e-01 / 1.69e-01 | 20260505 07:12:07.128 | INFO | PerfReport: 1.45%, PulseSerializing -- 4.78e-01 / 5.55e-01, 9.14e-02 / 2.83e-02 | 20260505 07:12:07.128 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 07:12:07.128 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:12:07.128 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:12:07.128 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.23e+01 / 0.00e+00 | 20260505 07:12:07.128 | INFO | PerfReport: 22.24%, Upload -- 2.26e+01 / 0.00e+00, 8.03e+00 / 0.00e+00 | 20260505 07:12:07.128 | INFO | PerfReport: 21.44%, Download -- 2.18e+01 / 0.00e+00, 7.80e+00 / 0.00e+00 | 20260505 07:12:07.128 | INFO | PerfReport: 32.46%, Compute -- 3.30e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 07:12:07.128 | INFO | PerfReport: 1.49%, LockUpload -- 1.51e+00 / 0.00e+00, 7.05e+00 / 0.00e+00 | 20260505 07:12:07.128 | INFO | PerfReport: 0.79%, LockDownload -- 8.05e-01 / 0.00e+00, 2.15e+00 / 0.00e+00 | 20260505 07:12:07.128 | INFO | PerfReport: 21.57%, LockCompute -- 2.19e+01 / 0.00e+00, 1.68e+01 / 0.00e+00 | 20260505 07:12:07.128 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 1.80e-02 / 0.00e+00 | 20260505 07:12:07.241 | INFO | Starting a round of Baz IO | 20260505 07:12:10.526 | INFO | Populating RTMetrics report | 20260505 07:12:10.527 | INFO | RTMetrics report populated | 20260505 07:12:10.527 | INFO | Opening tmp RTMetrics json file | 20260505 07:12:10.549 | INFO | Serializing RTMetrics to json | 20260505 07:12:10.550 | INFO | Serialization done, swapping tmp file now... | 20260505 07:12:10.551 | INFO | RTMetrics done | 20260505 07:12:10.649 | INFO | Finished a round of Baz IO: Wrote 2.42 GiB in 3.408 seconds | 20260505 07:12:10.649 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 30.60% / 0.04% / 69.36% | 20260505 07:12:10.649 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.93997% / 3.57628e-05% / 6.35783e-05% | 20260505 07:12:10.649 | INFO | Baz IO overall: | 20260505 07:12:10.649 | INFO | BytesWritten : 710693343274 | 20260505 07:12:10.649 | INFO | NumEvents : 336952279500 | 20260505 07:12:12.266 | INFO | Analyzing chunk frames = [1318912,1319424) | 20260505 07:12:12.266 | INFO | MemInfo: { | 20260505 07:12:12.266 | INFO | "gpu" : " 17 GB", | 20260505 07:12:12.266 | INFO | "huge" : "", | 20260505 07:12:12.266 | INFO | "huge_shared" : " 27 GB", | 20260505 07:12:12.266 | INFO | "malloc" : " 14 GB", | 20260505 07:12:12.266 | INFO | "pinned" : " 29 GB" | 20260505 07:12:12.266 | INFO | } | 20260505 07:12:12.266 | INFO | Perf report for the last 5.13904 seconds: | 20260505 07:12:12.266 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:12:12.266 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:12.266 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:12.266 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:12.266 | INFO | ANALYSIS : 65.0%, 2.9, 99.3ms, 0 | 20260505 07:12:12.266 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:12.266 | INFO | RT_METRICS : 46.6%, 1.0, 24.4ms, 0 | 20260505 07:12:12.266 | INFO | PRE_HQ : 63.9%, 1.9, 63.1ms, 0 | 20260505 07:12:12.266 | INFO | BAZWRITER : 65.6%, 4.1, 137.8ms, 0 | 20260505 07:12:12.266 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260505 07:12:12.266 | INFO | Uploaded 36GiB to GPU in 2.25513s (15.9636GiB/s) | 20260505 07:12:12.266 | INFO | Downloaded 38GiB to GPU in 2.30284s (16.5014GiB/s) | 20260505 07:12:17.405 | INFO | Analyzing chunk frames = [1319424,1319936) | 20260505 07:12:17.405 | INFO | MemInfo: { | 20260505 07:12:17.405 | INFO | "gpu" : " 17 GB", | 20260505 07:12:17.405 | INFO | "huge" : "", | 20260505 07:12:17.405 | INFO | "huge_shared" : " 27 GB", | 20260505 07:12:17.405 | INFO | "malloc" : " 14 GB", | 20260505 07:12:17.405 | INFO | "pinned" : " 29 GB" | 20260505 07:12:17.405 | INFO | } | 20260505 07:12:17.405 | INFO | Perf report for the last 5.13976 seconds: | 20260505 07:12:17.405 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:12:17.405 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:17.405 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:17.405 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:17.405 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 07:12:17.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:17.406 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:17.406 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:12:17.406 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:17.406 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:17.406 | INFO | Uploaded 36GiB to GPU in 2.11209s (17.0447GiB/s) | 20260505 07:12:17.406 | INFO | Downloaded 34GiB to GPU in 2.04249s (16.6463GiB/s) | 20260505 07:12:22.545 | INFO | Analyzing chunk frames = [1319936,1320448) | 20260505 07:12:22.545 | INFO | MemInfo: { | 20260505 07:12:22.545 | INFO | "gpu" : " 17 GB", | 20260505 07:12:22.545 | INFO | "huge" : "", | 20260505 07:12:22.545 | INFO | "huge_shared" : " 27 GB", | 20260505 07:12:22.545 | INFO | "malloc" : " 14 GB", | 20260505 07:12:22.545 | INFO | "pinned" : " 29 GB" | 20260505 07:12:22.545 | INFO | } | 20260505 07:12:22.545 | INFO | Perf report for the last 5.13935 seconds: | 20260505 07:12:22.545 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:12:22.545 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:22.545 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:22.545 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:22.545 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260505 07:12:22.545 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:22.545 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:22.545 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:12:22.545 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:22.545 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:22.545 | INFO | Uploaded 36GiB to GPU in 2.14938s (16.749GiB/s) | 20260505 07:12:22.545 | INFO | Downloaded 34GiB to GPU in 2.04994s (16.5858GiB/s) | 20260505 07:12:27.684 | INFO | Analyzing chunk frames = [1320448,1320960) | 20260505 07:12:27.684 | INFO | MemInfo: { | 20260505 07:12:27.684 | INFO | "gpu" : " 17 GB", | 20260505 07:12:27.684 | INFO | "huge" : "", | 20260505 07:12:27.684 | INFO | "huge_shared" : " 27 GB", | 20260505 07:12:27.684 | INFO | "malloc" : " 14 GB", | 20260505 07:12:27.684 | INFO | "pinned" : " 29 GB" | 20260505 07:12:27.684 | INFO | } | 20260505 07:12:27.684 | INFO | Perf report for the last 5.13931 seconds: | 20260505 07:12:27.684 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:12:27.684 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:27.684 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:27.684 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:27.684 | INFO | ANALYSIS : 63.4%, 3.0, 98.4ms, 0 | 20260505 07:12:27.684 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:27.684 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:27.684 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:12:27.684 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:27.684 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:27.684 | INFO | Uploaded 36GiB to GPU in 2.13318s (16.8762GiB/s) | 20260505 07:12:27.684 | INFO | Downloaded 34GiB to GPU in 2.02893s (16.7576GiB/s) | 20260505 07:12:32.823 | INFO | Analyzing chunk frames = [1320960,1321472) | 20260505 07:12:32.823 | INFO | MemInfo: { | 20260505 07:12:32.823 | INFO | "gpu" : " 17 GB", | 20260505 07:12:32.823 | INFO | "huge" : "", | 20260505 07:12:32.823 | INFO | "huge_shared" : " 27 GB", | 20260505 07:12:32.823 | INFO | "malloc" : " 14 GB", | 20260505 07:12:32.823 | INFO | "pinned" : " 29 GB" | 20260505 07:12:32.823 | INFO | } | 20260505 07:12:32.823 | INFO | Perf report for the last 5.1392 seconds: | 20260505 07:12:32.823 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:12:32.823 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:32.823 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:32.823 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:32.823 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 07:12:32.823 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:32.823 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:32.823 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 07:12:32.823 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:32.823 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:32.824 | INFO | Uploaded 36GiB to GPU in 2.17105s (16.5818GiB/s) | 20260505 07:12:32.824 | INFO | Downloaded 34GiB to GPU in 2.05217s (16.5678GiB/s) | 20260505 07:12:37.964 | INFO | Analyzing chunk frames = [1321472,1321984) | 20260505 07:12:37.964 | INFO | MemInfo: { | 20260505 07:12:37.964 | INFO | "gpu" : " 17 GB", | 20260505 07:12:37.964 | INFO | "huge" : "", | 20260505 07:12:37.964 | INFO | "huge_shared" : " 27 GB", | 20260505 07:12:37.964 | INFO | "malloc" : " 14 GB", | 20260505 07:12:37.964 | INFO | "pinned" : " 29 GB" | 20260505 07:12:37.964 | INFO | } | 20260505 07:12:37.964 | INFO | Perf report for the last 5.14097 seconds: | 20260505 07:12:37.964 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:12:37.964 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:37.964 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:37.964 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:37.964 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 07:12:37.964 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:37.964 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:37.964 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:12:37.964 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:37.964 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:37.965 | INFO | Uploaded 36GiB to GPU in 2.15836s (16.6793GiB/s) | 20260505 07:12:37.965 | INFO | Downloaded 34GiB to GPU in 2.05488s (16.5459GiB/s) | 20260505 07:12:43.102 | INFO | Analyzing chunk frames = [1321984,1322496) | 20260505 07:12:43.102 | INFO | MemInfo: { | 20260505 07:12:43.102 | INFO | "gpu" : " 17 GB", | 20260505 07:12:43.102 | INFO | "huge" : "", | 20260505 07:12:43.102 | INFO | "huge_shared" : " 27 GB", | 20260505 07:12:43.102 | INFO | "malloc" : " 14 GB", | 20260505 07:12:43.102 | INFO | "pinned" : " 29 GB" | 20260505 07:12:43.102 | INFO | } | 20260505 07:12:43.102 | INFO | Perf report for the last 5.13765 seconds: | 20260505 07:12:43.102 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:12:43.102 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:43.102 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:43.102 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:43.102 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 07:12:43.102 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:43.102 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:43.102 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260505 07:12:43.102 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:43.102 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:43.102 | INFO | Uploaded 36GiB to GPU in 2.20531s (16.3242GiB/s) | 20260505 07:12:43.102 | INFO | Downloaded 34GiB to GPU in 2.16995s (15.6686GiB/s) | 20260505 07:12:48.241 | INFO | Analyzing chunk frames = [1322496,1323008) | 20260505 07:12:48.241 | INFO | MemInfo: { | 20260505 07:12:48.241 | INFO | "gpu" : " 17 GB", | 20260505 07:12:48.241 | INFO | "huge" : "", | 20260505 07:12:48.241 | INFO | "huge_shared" : " 27 GB", | 20260505 07:12:48.241 | INFO | "malloc" : " 14 GB", | 20260505 07:12:48.241 | INFO | "pinned" : " 29 GB" | 20260505 07:12:48.241 | INFO | } | 20260505 07:12:48.241 | INFO | Perf report for the last 5.13919 seconds: | 20260505 07:12:48.241 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:12:48.241 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:48.241 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:48.241 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:48.241 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 07:12:48.241 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:48.241 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:48.241 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 07:12:48.241 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:48.241 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:48.241 | INFO | Uploaded 36GiB to GPU in 2.17225s (16.5726GiB/s) | 20260505 07:12:48.241 | INFO | Downloaded 34GiB to GPU in 2.03742s (16.6877GiB/s) | 20260505 07:12:48.360 | INFO | Starting a round of Baz IO | 20260505 07:12:51.660 | INFO | Populating RTMetrics report | 20260505 07:12:51.660 | INFO | RTMetrics report populated | 20260505 07:12:51.660 | INFO | Opening tmp RTMetrics json file | 20260505 07:12:51.671 | INFO | Serializing RTMetrics to json | 20260505 07:12:51.695 | INFO | Serialization done, swapping tmp file now... | 20260505 07:12:51.696 | INFO | RTMetrics done | 20260505 07:12:51.794 | INFO | Finished a round of Baz IO: Wrote 2.42 GiB in 3.433 seconds | 20260505 07:12:51.794 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 30.57% / 0.03% / 69.40% | 20260505 07:12:51.794 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.92568% / 2.38419e-05% / 7.54992e-05% | 20260505 07:12:51.794 | INFO | Baz IO overall: | 20260505 07:12:51.794 | INFO | BytesWritten : 713291893505 | 20260505 07:12:51.794 | INFO | NumEvents : 338195563877 | 20260505 07:12:53.380 | INFO | Analyzing chunk frames = [1323008,1323520) | 20260505 07:12:53.380 | INFO | MemInfo: { | 20260505 07:12:53.380 | INFO | "gpu" : " 17 GB", | 20260505 07:12:53.381 | INFO | "huge" : "", | 20260505 07:12:53.381 | INFO | "huge_shared" : " 27 GB", | 20260505 07:12:53.381 | INFO | "malloc" : " 14 GB", | 20260505 07:12:53.381 | INFO | "pinned" : " 29 GB" | 20260505 07:12:53.381 | INFO | } | 20260505 07:12:53.381 | INFO | Perf report for the last 5.13938 seconds: | 20260505 07:12:53.381 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:12:53.381 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:53.381 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:53.381 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:53.381 | INFO | ANALYSIS : 65.4%, 2.9, 99.7ms, 0 | 20260505 07:12:53.381 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:53.381 | INFO | RT_METRICS : 46.7%, 1.0, 24.5ms, 0 | 20260505 07:12:53.381 | INFO | PRE_HQ : 64.3%, 1.9, 62.8ms, 0 | 20260505 07:12:53.381 | INFO | BAZWRITER : 66.1%, 4.1, 138.4ms, 0 | 20260505 07:12:53.381 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 07:12:53.381 | INFO | Uploaded 36GiB to GPU in 2.22526s (16.1779GiB/s) | 20260505 07:12:53.381 | INFO | Downloaded 38GiB to GPU in 2.33671s (16.2622GiB/s) | 20260505 07:12:58.520 | INFO | Analyzing chunk frames = [1323520,1324032) | 20260505 07:12:58.520 | INFO | MemInfo: { | 20260505 07:12:58.520 | INFO | "gpu" : " 17 GB", | 20260505 07:12:58.520 | INFO | "huge" : "", | 20260505 07:12:58.520 | INFO | "huge_shared" : " 27 GB", | 20260505 07:12:58.520 | INFO | "malloc" : " 14 GB", | 20260505 07:12:58.520 | INFO | "pinned" : " 29 GB" | 20260505 07:12:58.520 | INFO | } | 20260505 07:12:58.520 | INFO | Perf report for the last 5.13938 seconds: | 20260505 07:12:58.520 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:12:58.520 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:58.520 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:58.520 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:58.520 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 07:12:58.520 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:58.520 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:58.520 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 07:12:58.520 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:58.520 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:12:58.520 | INFO | Uploaded 36GiB to GPU in 2.12377s (16.951GiB/s) | 20260505 07:12:58.520 | INFO | Downloaded 34GiB to GPU in 2.0352s (16.706GiB/s) | 20260505 07:13:03.660 | INFO | Analyzing chunk frames = [1324032,1324544) | 20260505 07:13:03.660 | INFO | MemInfo: { | 20260505 07:13:03.660 | INFO | "gpu" : " 17 GB", | 20260505 07:13:03.660 | INFO | "huge" : "", | 20260505 07:13:03.660 | INFO | "huge_shared" : " 27 GB", | 20260505 07:13:03.660 | INFO | "malloc" : " 14 GB", | 20260505 07:13:03.660 | INFO | "pinned" : " 29 GB" | 20260505 07:13:03.660 | INFO | } | 20260505 07:13:03.660 | INFO | Perf report for the last 5.13986 seconds: | 20260505 07:13:03.660 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:13:03.660 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:03.660 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:03.660 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:03.660 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 07:13:03.660 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:03.660 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:03.660 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 07:13:03.660 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:03.660 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:03.660 | INFO | Uploaded 36GiB to GPU in 2.15343s (16.7175GiB/s) | 20260505 07:13:03.660 | INFO | Downloaded 34GiB to GPU in 2.0577s (16.5233GiB/s) | 20260505 07:13:08.799 | INFO | Analyzing chunk frames = [1324544,1325056) | 20260505 07:13:08.799 | INFO | MemInfo: { | 20260505 07:13:08.799 | INFO | "gpu" : " 17 GB", | 20260505 07:13:08.799 | INFO | "huge" : "", | 20260505 07:13:08.799 | INFO | "huge_shared" : " 27 GB", | 20260505 07:13:08.799 | INFO | "malloc" : " 14 GB", | 20260505 07:13:08.799 | INFO | "pinned" : " 29 GB" | 20260505 07:13:08.799 | INFO | } | 20260505 07:13:08.799 | INFO | Perf report for the last 5.13905 seconds: | 20260505 07:13:08.799 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:13:08.799 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:08.799 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:08.799 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:08.799 | INFO | ANALYSIS : 63.4%, 3.0, 98.5ms, 0 | 20260505 07:13:08.799 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:08.799 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:08.799 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:13:08.799 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:08.799 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:08.799 | INFO | Uploaded 36GiB to GPU in 2.1293s (16.907GiB/s) | 20260505 07:13:08.799 | INFO | Downloaded 34GiB to GPU in 2.02741s (16.7702GiB/s) | 20260505 07:13:13.938 | INFO | Analyzing chunk frames = [1325056,1325568) | 20260505 07:13:13.938 | INFO | MemInfo: { | 20260505 07:13:13.938 | INFO | "gpu" : " 17 GB", | 20260505 07:13:13.938 | INFO | "huge" : "", | 20260505 07:13:13.938 | INFO | "huge_shared" : " 27 GB", | 20260505 07:13:13.938 | INFO | "malloc" : " 14 GB", | 20260505 07:13:13.938 | INFO | "pinned" : " 29 GB" | 20260505 07:13:13.938 | INFO | } | 20260505 07:13:13.938 | INFO | Perf report for the last 5.13926 seconds: | 20260505 07:13:13.938 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:13:13.938 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:13.938 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:13.938 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:13.938 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 07:13:13.938 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:13.938 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:13.938 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:13:13.938 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:13.938 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:13.938 | INFO | Uploaded 36GiB to GPU in 2.16281s (16.645GiB/s) | 20260505 07:13:13.938 | INFO | Downloaded 34GiB to GPU in 2.05078s (16.579GiB/s) | 20260505 07:13:19.079 | INFO | Analyzing chunk frames = [1325568,1326080) | 20260505 07:13:19.079 | INFO | MemInfo: { | 20260505 07:13:19.079 | INFO | "gpu" : " 17 GB", | 20260505 07:13:19.079 | INFO | "huge" : "", | 20260505 07:13:19.079 | INFO | "huge_shared" : " 27 GB", | 20260505 07:13:19.079 | INFO | "malloc" : " 14 GB", | 20260505 07:13:19.079 | INFO | "pinned" : " 29 GB" | 20260505 07:13:19.079 | INFO | } | 20260505 07:13:19.079 | INFO | Perf report for the last 5.14104 seconds: | 20260505 07:13:19.079 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:13:19.079 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:19.079 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:19.079 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:19.079 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 07:13:19.079 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:19.079 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:19.079 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:13:19.079 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:19.079 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:19.079 | INFO | Uploaded 36GiB to GPU in 2.16609s (16.6198GiB/s) | 20260505 07:13:19.079 | INFO | Downloaded 34GiB to GPU in 2.05132s (16.5747GiB/s) | 20260505 07:13:24.217 | INFO | Analyzing chunk frames = [1326080,1326592) | 20260505 07:13:24.217 | INFO | MemInfo: { | 20260505 07:13:24.217 | INFO | "gpu" : " 17 GB", | 20260505 07:13:24.217 | INFO | "huge" : "", | 20260505 07:13:24.217 | INFO | "huge_shared" : " 27 GB", | 20260505 07:13:24.217 | INFO | "malloc" : " 14 GB", | 20260505 07:13:24.217 | INFO | "pinned" : " 29 GB" | 20260505 07:13:24.217 | INFO | } | 20260505 07:13:24.217 | INFO | Perf report for the last 5.13748 seconds: | 20260505 07:13:24.217 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:13:24.217 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:24.217 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:24.217 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:24.217 | INFO | ANALYSIS : 64.1%, 3.0, 100.0ms, 0 | 20260505 07:13:24.217 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:24.217 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:24.217 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260505 07:13:24.217 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:24.217 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:24.217 | INFO | Uploaded 36GiB to GPU in 2.20008s (16.363GiB/s) | 20260505 07:13:24.217 | INFO | Downloaded 34GiB to GPU in 2.16951s (15.6717GiB/s) | 20260505 07:13:29.356 | INFO | Analyzing chunk frames = [1326592,1327104) | 20260505 07:13:29.356 | INFO | MemInfo: { | 20260505 07:13:29.356 | INFO | "gpu" : " 17 GB", | 20260505 07:13:29.356 | INFO | "huge" : "", | 20260505 07:13:29.356 | INFO | "huge_shared" : " 27 GB", | 20260505 07:13:29.356 | INFO | "malloc" : " 14 GB", | 20260505 07:13:29.357 | INFO | "pinned" : " 29 GB" | 20260505 07:13:29.357 | INFO | } | 20260505 07:13:29.357 | INFO | Perf report for the last 5.13979 seconds: | 20260505 07:13:29.357 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:13:29.357 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:29.357 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:29.357 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:29.357 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 07:13:29.357 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:29.357 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:29.357 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 07:13:29.357 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:29.357 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:29.357 | INFO | Uploaded 36GiB to GPU in 2.1575s (16.686GiB/s) | 20260505 07:13:29.357 | INFO | Downloaded 34GiB to GPU in 2.04032s (16.664GiB/s) | 20260505 07:13:29.357 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 07:13:29.357 | INFO | Total mem usage: 15.3 GiB | 20260505 07:13:29.357 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 07:13:29.357 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -6.02%) | 20260505 07:13:29.357 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 07:13:29.357 | INFO | High Water Breakdown: | 20260505 07:13:29.357 | INFO | 60.6% : 8.76 GiB ( 93.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 07:13:29.357 | INFO | 39.2% : 5.67 GiB ( 85.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 07:13:29.357 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 07:13:29.357 | INFO | ------------------------------------------ | 20260505 07:13:29.357 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:13:29.357 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 07:13:29.357 | INFO | Total mem usage: 26.5 GiB | 20260505 07:13:29.357 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 07:13:29.357 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 07:13:29.357 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 07:13:29.357 | INFO | High Water Breakdown: | 20260505 07:13:29.357 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:13:29.357 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:13:29.357 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:13:29.357 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:13:29.357 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:13:29.357 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:13:29.357 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 07:13:29.357 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 07:13:29.357 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:13:29.357 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 07:13:29.357 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 07:13:29.357 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 07:13:29.357 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 07:13:29.357 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 07:13:29.357 | INFO | ------------------------------------------ | 20260505 07:13:29.357 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:13:29.357 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 07:13:29.357 | INFO | Total mem usage: 25.6 GiB | 20260505 07:13:29.357 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 07:13:29.357 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 07:13:29.357 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 07:13:29.357 | INFO | High Water Breakdown: | 20260505 07:13:29.357 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 07:13:29.357 | INFO | ------------------------------------------ | 20260505 07:13:29.357 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:13:29.357 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 07:13:29.357 | INFO | Total mem usage: 15.7 GiB | 20260505 07:13:29.357 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 07:13:29.357 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 07:13:29.357 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 07:13:29.357 | INFO | High Water Breakdown: | 20260505 07:13:29.357 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 07:13:29.357 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 07:13:29.357 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 07:13:29.357 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 07:13:29.357 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 07:13:29.357 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 07:13:29.357 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 07:13:29.358 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 07:13:29.358 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 07:13:29.358 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:13:29.358 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 07:13:29.358 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:13:29.358 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 07:13:29.358 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 07:13:29.358 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 07:13:29.358 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 07:13:29.358 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:13:29.358 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 07:13:29.358 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:13:29.358 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:13:29.358 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:13:29.358 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 07:13:29.358 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 07:13:29.358 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:13:29.358 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 07:13:29.358 | INFO | ------------------------------------------ | 20260505 07:13:29.358 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:13:29.358 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 07:13:29.358 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:13:29.358 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:13:29.358 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.51e+01, 1.37e+01 / 1.82e+00 | 20260505 07:13:29.358 | INFO | PerfReport: 0.29%, AggregateStats -- 9.40e-02 / 1.05e-01, 1.18e-02 / 4.40e-03 | 20260505 07:13:29.358 | INFO | PerfReport: 11.62%, Baseline -- 3.82e+00 / 4.46e+00, 6.16e-01 / 2.76e-01 | 20260505 07:13:29.358 | INFO | PerfReport: 3.31%, Histogram -- 1.09e+00 / 1.48e+00, 2.12e-01 / 1.10e-01 | 20260505 07:13:29.358 | INFO | PerfReport: 15.47%, DME -- 4.07e+01 / 4.45e+01, 4.12e+00 / 1.19e+00 | 20260505 07:13:29.358 | INFO | PerfReport: 56.21%, FrameLabeling -- 1.85e+01 / 1.91e+01, 2.01e+00 / 1.24e+00 | 20260505 07:13:29.358 | INFO | PerfReport: 7.65%, PulseAccumulating -- 2.52e+00 / 3.14e+00, 4.46e-01 / 2.71e-01 | 20260505 07:13:29.358 | INFO | PerfReport: 4.01%, Metrics -- 1.32e+00 / 1.74e+00, 3.66e-01 / 2.13e-01 | 20260505 07:13:29.358 | INFO | PerfReport: 1.45%, PulseSerializing -- 4.77e-01 / 5.56e-01, 9.09e-02 / 3.05e-02 | 20260505 07:13:29.358 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 07:13:29.358 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:13:29.358 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:13:29.358 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.25e+01 / 0.00e+00 | 20260505 07:13:29.358 | INFO | PerfReport: 22.26%, Upload -- 2.26e+01 / 0.00e+00, 7.87e+00 / 0.00e+00 | 20260505 07:13:29.358 | INFO | PerfReport: 21.51%, Download -- 2.18e+01 / 0.00e+00, 7.69e+00 / 0.00e+00 | 20260505 07:13:29.358 | INFO | PerfReport: 32.47%, Compute -- 3.29e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 07:13:29.358 | INFO | PerfReport: 1.48%, LockUpload -- 1.51e+00 / 0.00e+00, 7.03e+00 / 0.00e+00 | 20260505 07:13:29.358 | INFO | PerfReport: 0.79%, LockDownload -- 8.00e-01 / 0.00e+00, 2.08e+00 / 0.00e+00 | 20260505 07:13:29.358 | INFO | PerfReport: 21.48%, LockCompute -- 2.18e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260505 07:13:29.358 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.68e-02 / 0.00e+00 | 20260505 07:13:29.480 | INFO | Starting a round of Baz IO | 20260505 07:13:32.783 | INFO | Populating RTMetrics report | 20260505 07:13:32.783 | INFO | RTMetrics report populated | 20260505 07:13:32.783 | INFO | Opening tmp RTMetrics json file | 20260505 07:13:32.794 | INFO | Serializing RTMetrics to json | 20260505 07:13:32.795 | INFO | Serialization done, swapping tmp file now... | 20260505 07:13:32.796 | INFO | RTMetrics done | 20260505 07:13:32.925 | INFO | Finished a round of Baz IO: Wrote 2.422 GiB in 3.445 seconds | 20260505 07:13:32.925 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 30.53% / 0.03% / 69.43% | 20260505 07:13:32.925 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.91715% / 6.35783e-05% / 7.94729e-05% | 20260505 07:13:32.925 | INFO | Baz IO overall: | 20260505 07:13:32.925 | INFO | BytesWritten : 715892243191 | 20260505 07:13:32.925 | INFO | NumEvents : 339440533543 | 20260505 07:13:34.495 | INFO | Analyzing chunk frames = [1327104,1327616) | 20260505 07:13:34.495 | INFO | MemInfo: { | 20260505 07:13:34.495 | INFO | "gpu" : " 17 GB", | 20260505 07:13:34.495 | INFO | "huge" : "", | 20260505 07:13:34.495 | INFO | "huge_shared" : " 27 GB", | 20260505 07:13:34.495 | INFO | "malloc" : " 14 GB", | 20260505 07:13:34.495 | INFO | "pinned" : " 29 GB" | 20260505 07:13:34.495 | INFO | } | 20260505 07:13:34.496 | INFO | Perf report for the last 5.13893 seconds: | 20260505 07:13:34.496 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:13:34.496 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:34.496 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:34.496 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:34.496 | INFO | ANALYSIS : 65.5%, 2.9, 99.9ms, 0 | 20260505 07:13:34.496 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:34.496 | INFO | RT_METRICS : 46.2%, 1.0, 24.2ms, 0 | 20260505 07:13:34.496 | INFO | PRE_HQ : 64.3%, 1.9, 62.7ms, 0 | 20260505 07:13:34.496 | INFO | BAZWRITER : 66.3%, 4.1, 139.9ms, 0 | 20260505 07:13:34.496 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 07:13:34.496 | INFO | Uploaded 36GiB to GPU in 2.24509s (16.035GiB/s) | 20260505 07:13:34.496 | INFO | Downloaded 38GiB to GPU in 2.34367s (16.2139GiB/s) | 20260505 07:13:39.635 | INFO | Analyzing chunk frames = [1327616,1328128) | 20260505 07:13:39.635 | INFO | MemInfo: { | 20260505 07:13:39.635 | INFO | "gpu" : " 17 GB", | 20260505 07:13:39.635 | INFO | "huge" : "", | 20260505 07:13:39.635 | INFO | "huge_shared" : " 27 GB", | 20260505 07:13:39.635 | INFO | "malloc" : " 14 GB", | 20260505 07:13:39.635 | INFO | "pinned" : " 29 GB" | 20260505 07:13:39.635 | INFO | } | 20260505 07:13:39.635 | INFO | Perf report for the last 5.13942 seconds: | 20260505 07:13:39.635 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:13:39.635 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:39.635 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:39.635 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:39.635 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 07:13:39.635 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:39.635 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:39.635 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:13:39.635 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:39.635 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:39.635 | INFO | Uploaded 36GiB to GPU in 2.12391s (16.9499GiB/s) | 20260505 07:13:39.635 | INFO | Downloaded 34GiB to GPU in 2.04152s (16.6543GiB/s) | 20260505 07:13:44.774 | INFO | Analyzing chunk frames = [1328128,1328640) | 20260505 07:13:44.774 | INFO | MemInfo: { | 20260505 07:13:44.774 | INFO | "gpu" : " 17 GB", | 20260505 07:13:44.774 | INFO | "huge" : "", | 20260505 07:13:44.774 | INFO | "huge_shared" : " 27 GB", | 20260505 07:13:44.774 | INFO | "malloc" : " 14 GB", | 20260505 07:13:44.774 | INFO | "pinned" : " 29 GB" | 20260505 07:13:44.774 | INFO | } | 20260505 07:13:44.774 | INFO | Perf report for the last 5.13936 seconds: | 20260505 07:13:44.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:13:44.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:44.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:44.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:44.774 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260505 07:13:44.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:44.774 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:44.774 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 07:13:44.774 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:44.774 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:44.775 | INFO | Uploaded 36GiB to GPU in 2.15439s (16.7101GiB/s) | 20260505 07:13:44.775 | INFO | Downloaded 34GiB to GPU in 2.05335s (16.5583GiB/s) | 20260505 07:13:49.913 | INFO | Analyzing chunk frames = [1328640,1329152) | 20260505 07:13:49.913 | INFO | MemInfo: { | 20260505 07:13:49.913 | INFO | "gpu" : " 17 GB", | 20260505 07:13:49.913 | INFO | "huge" : "", | 20260505 07:13:49.913 | INFO | "huge_shared" : " 27 GB", | 20260505 07:13:49.913 | INFO | "malloc" : " 14 GB", | 20260505 07:13:49.913 | INFO | "pinned" : " 29 GB" | 20260505 07:13:49.913 | INFO | } | 20260505 07:13:49.914 | INFO | Perf report for the last 5.13922 seconds: | 20260505 07:13:49.914 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:13:49.914 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:49.914 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:49.914 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:49.914 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 07:13:49.914 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:49.914 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:49.914 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:13:49.914 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:49.914 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:49.914 | INFO | Uploaded 36GiB to GPU in 2.12711s (16.9244GiB/s) | 20260505 07:13:49.914 | INFO | Downloaded 34GiB to GPU in 2.0262s (16.7802GiB/s) | 20260505 07:13:55.053 | INFO | Analyzing chunk frames = [1329152,1329664) | 20260505 07:13:55.053 | INFO | MemInfo: { | 20260505 07:13:55.053 | INFO | "gpu" : " 17 GB", | 20260505 07:13:55.053 | INFO | "huge" : "", | 20260505 07:13:55.053 | INFO | "huge_shared" : " 27 GB", | 20260505 07:13:55.053 | INFO | "malloc" : " 14 GB", | 20260505 07:13:55.053 | INFO | "pinned" : " 29 GB" | 20260505 07:13:55.053 | INFO | } | 20260505 07:13:55.053 | INFO | Perf report for the last 5.13937 seconds: | 20260505 07:13:55.053 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:13:55.053 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:55.053 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:55.053 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:55.053 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 07:13:55.053 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:55.053 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:55.053 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 07:13:55.053 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:55.053 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:13:55.053 | INFO | Uploaded 36GiB to GPU in 2.16417s (16.6346GiB/s) | 20260505 07:13:55.053 | INFO | Downloaded 34GiB to GPU in 2.05125s (16.5753GiB/s) | 20260505 07:14:00.194 | INFO | Analyzing chunk frames = [1329664,1330176) | 20260505 07:14:00.194 | INFO | MemInfo: { | 20260505 07:14:00.194 | INFO | "gpu" : " 17 GB", | 20260505 07:14:00.194 | INFO | "huge" : "", | 20260505 07:14:00.194 | INFO | "huge_shared" : " 27 GB", | 20260505 07:14:00.194 | INFO | "malloc" : " 14 GB", | 20260505 07:14:00.194 | INFO | "pinned" : " 29 GB" | 20260505 07:14:00.194 | INFO | } | 20260505 07:14:00.194 | INFO | Perf report for the last 5.14132 seconds: | 20260505 07:14:00.194 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:14:00.194 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:00.194 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:00.194 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:00.194 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 07:14:00.194 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:00.194 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:00.194 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:14:00.194 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:00.194 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:00.195 | INFO | Uploaded 36GiB to GPU in 2.16035s (16.6639GiB/s) | 20260505 07:14:00.195 | INFO | Downloaded 34GiB to GPU in 2.05784s (16.5222GiB/s) | 20260505 07:14:05.331 | INFO | Analyzing chunk frames = [1330176,1330688) | 20260505 07:14:05.331 | INFO | MemInfo: { | 20260505 07:14:05.331 | INFO | "gpu" : " 17 GB", | 20260505 07:14:05.331 | INFO | "huge" : "", | 20260505 07:14:05.331 | INFO | "huge_shared" : " 27 GB", | 20260505 07:14:05.332 | INFO | "malloc" : " 14 GB", | 20260505 07:14:05.332 | INFO | "pinned" : " 29 GB" | 20260505 07:14:05.332 | INFO | } | 20260505 07:14:05.332 | INFO | Perf report for the last 5.13773 seconds: | 20260505 07:14:05.332 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:14:05.332 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:05.332 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:05.332 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:05.332 | INFO | ANALYSIS : 64.7%, 3.0, 100.9ms, 0 | 20260505 07:14:05.332 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:05.332 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:05.332 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260505 07:14:05.332 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:05.332 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:05.332 | INFO | Uploaded 36GiB to GPU in 2.22288s (16.1952GiB/s) | 20260505 07:14:05.332 | INFO | Downloaded 34GiB to GPU in 2.14618s (15.8421GiB/s) | 20260505 07:14:10.471 | INFO | Analyzing chunk frames = [1330688,1331200) | 20260505 07:14:10.471 | INFO | MemInfo: { | 20260505 07:14:10.471 | INFO | "gpu" : " 17 GB", | 20260505 07:14:10.471 | INFO | "huge" : "", | 20260505 07:14:10.471 | INFO | "huge_shared" : " 27 GB", | 20260505 07:14:10.471 | INFO | "malloc" : " 14 GB", | 20260505 07:14:10.471 | INFO | "pinned" : " 29 GB" | 20260505 07:14:10.471 | INFO | } | 20260505 07:14:10.471 | INFO | Perf report for the last 5.1391 seconds: | 20260505 07:14:10.471 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:14:10.471 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:10.471 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:10.471 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:10.471 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 07:14:10.471 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:10.471 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:10.471 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 07:14:10.471 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:10.471 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:10.471 | INFO | Uploaded 36GiB to GPU in 2.16605s (16.6201GiB/s) | 20260505 07:14:10.471 | INFO | Downloaded 34GiB to GPU in 2.03985s (16.6679GiB/s) | 20260505 07:14:10.585 | INFO | Starting a round of Baz IO | 20260505 07:14:13.875 | INFO | Populating RTMetrics report | 20260505 07:14:13.875 | INFO | RTMetrics report populated | 20260505 07:14:13.875 | INFO | Opening tmp RTMetrics json file | 20260505 07:14:13.876 | INFO | Serializing RTMetrics to json | 20260505 07:14:13.877 | INFO | Serialization done, swapping tmp file now... | 20260505 07:14:13.878 | INFO | RTMetrics done | 20260505 07:14:13.994 | INFO | Finished a round of Baz IO: Wrote 2.422 GiB in 3.408 seconds | 20260505 07:14:13.994 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 30.50% / 0.03% / 69.47% | 20260505 07:14:13.994 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.89637% / 5.5631e-05% / 8.74201e-05% | 20260505 07:14:13.994 | INFO | Baz IO overall: | 20260505 07:14:13.994 | INFO | BytesWritten : 718492328879 | 20260505 07:14:13.994 | INFO | NumEvents : 340685178987 | 20260505 07:14:15.610 | INFO | Analyzing chunk frames = [1331200,1331712) | 20260505 07:14:15.610 | INFO | MemInfo: { | 20260505 07:14:15.610 | INFO | "gpu" : " 17 GB", | 20260505 07:14:15.610 | INFO | "huge" : "", | 20260505 07:14:15.610 | INFO | "huge_shared" : " 27 GB", | 20260505 07:14:15.610 | INFO | "malloc" : " 14 GB", | 20260505 07:14:15.610 | INFO | "pinned" : " 29 GB" | 20260505 07:14:15.610 | INFO | } | 20260505 07:14:15.610 | INFO | Perf report for the last 5.13921 seconds: | 20260505 07:14:15.610 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:14:15.610 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:15.610 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:15.610 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:15.610 | INFO | ANALYSIS : 65.0%, 2.9, 99.0ms, 0 | 20260505 07:14:15.610 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:15.610 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 07:14:15.610 | INFO | PRE_HQ : 63.9%, 1.9, 63.0ms, 0 | 20260505 07:14:15.610 | INFO | BAZWRITER : 65.6%, 4.1, 139.7ms, 0 | 20260505 07:14:15.610 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 07:14:15.611 | INFO | Uploaded 36GiB to GPU in 2.25735s (15.9479GiB/s) | 20260505 07:14:15.611 | INFO | Downloaded 38GiB to GPU in 2.30923s (16.4557GiB/s) | 20260505 07:14:20.750 | INFO | Analyzing chunk frames = [1331712,1332224) | 20260505 07:14:20.750 | INFO | MemInfo: { | 20260505 07:14:20.750 | INFO | "gpu" : " 17 GB", | 20260505 07:14:20.750 | INFO | "huge" : "", | 20260505 07:14:20.750 | INFO | "huge_shared" : " 27 GB", | 20260505 07:14:20.750 | INFO | "malloc" : " 14 GB", | 20260505 07:14:20.750 | INFO | "pinned" : " 29 GB" | 20260505 07:14:20.750 | INFO | } | 20260505 07:14:20.750 | INFO | Perf report for the last 5.13935 seconds: | 20260505 07:14:20.750 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:14:20.750 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:20.750 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:20.750 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:20.750 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 07:14:20.750 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:20.750 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:20.750 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:14:20.750 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:20.750 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:20.750 | INFO | Uploaded 36GiB to GPU in 2.1287s (16.9117GiB/s) | 20260505 07:14:20.750 | INFO | Downloaded 34GiB to GPU in 2.04451s (16.6299GiB/s) | 20260505 07:14:25.889 | INFO | Analyzing chunk frames = [1332224,1332736) | 20260505 07:14:25.889 | INFO | MemInfo: { | 20260505 07:14:25.889 | INFO | "gpu" : " 17 GB", | 20260505 07:14:25.889 | INFO | "huge" : "", | 20260505 07:14:25.889 | INFO | "huge_shared" : " 27 GB", | 20260505 07:14:25.889 | INFO | "malloc" : " 14 GB", | 20260505 07:14:25.889 | INFO | "pinned" : " 29 GB" | 20260505 07:14:25.889 | INFO | } | 20260505 07:14:25.889 | INFO | Perf report for the last 5.13922 seconds: | 20260505 07:14:25.889 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:14:25.889 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:25.889 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:25.889 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:25.889 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 07:14:25.889 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:25.889 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:25.889 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:14:25.889 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:25.889 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:25.890 | INFO | Uploaded 36GiB to GPU in 2.14912s (16.751GiB/s) | 20260505 07:14:25.890 | INFO | Downloaded 34GiB to GPU in 2.05387s (16.5541GiB/s) | 20260505 07:14:31.028 | INFO | Analyzing chunk frames = [1332736,1333248) | 20260505 07:14:31.028 | INFO | MemInfo: { | 20260505 07:14:31.028 | INFO | "gpu" : " 17 GB", | 20260505 07:14:31.028 | INFO | "huge" : "", | 20260505 07:14:31.028 | INFO | "huge_shared" : " 27 GB", | 20260505 07:14:31.028 | INFO | "malloc" : " 14 GB", | 20260505 07:14:31.028 | INFO | "pinned" : " 29 GB" | 20260505 07:14:31.028 | INFO | } | 20260505 07:14:31.028 | INFO | Perf report for the last 5.13933 seconds: | 20260505 07:14:31.028 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:14:31.028 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:31.028 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:31.028 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:31.028 | INFO | ANALYSIS : 63.4%, 3.0, 98.5ms, 0 | 20260505 07:14:31.028 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:31.028 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:31.028 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:14:31.028 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:31.028 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:31.029 | INFO | Uploaded 36GiB to GPU in 2.1256s (16.9364GiB/s) | 20260505 07:14:31.029 | INFO | Downloaded 34GiB to GPU in 2.02788s (16.7663GiB/s) | 20260505 07:14:36.168 | INFO | Analyzing chunk frames = [1333248,1333760) | 20260505 07:14:36.168 | INFO | MemInfo: { | 20260505 07:14:36.168 | INFO | "gpu" : " 17 GB", | 20260505 07:14:36.168 | INFO | "huge" : "", | 20260505 07:14:36.168 | INFO | "huge_shared" : " 27 GB", | 20260505 07:14:36.168 | INFO | "malloc" : " 14 GB", | 20260505 07:14:36.168 | INFO | "pinned" : " 29 GB" | 20260505 07:14:36.168 | INFO | } | 20260505 07:14:36.168 | INFO | Perf report for the last 5.13943 seconds: | 20260505 07:14:36.168 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:14:36.168 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:36.168 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:36.168 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:36.168 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 07:14:36.168 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:36.168 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:36.168 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:14:36.168 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:36.168 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:36.168 | INFO | Uploaded 36GiB to GPU in 2.16222s (16.6495GiB/s) | 20260505 07:14:36.168 | INFO | Downloaded 34GiB to GPU in 2.04623s (16.6159GiB/s) | 20260505 07:14:41.309 | INFO | Analyzing chunk frames = [1333760,1334272) | 20260505 07:14:41.309 | INFO | MemInfo: { | 20260505 07:14:41.309 | INFO | "gpu" : " 17 GB", | 20260505 07:14:41.309 | INFO | "huge" : "", | 20260505 07:14:41.309 | INFO | "huge_shared" : " 27 GB", | 20260505 07:14:41.309 | INFO | "malloc" : " 14 GB", | 20260505 07:14:41.309 | INFO | "pinned" : " 29 GB" | 20260505 07:14:41.309 | INFO | } | 20260505 07:14:41.309 | INFO | Perf report for the last 5.14109 seconds: | 20260505 07:14:41.309 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:14:41.309 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:41.309 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:41.309 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:41.309 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 07:14:41.309 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:41.309 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:41.309 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 07:14:41.309 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:41.309 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:41.309 | INFO | Uploaded 36GiB to GPU in 2.16239s (16.6482GiB/s) | 20260505 07:14:41.309 | INFO | Downloaded 34GiB to GPU in 2.05942s (16.5095GiB/s) | 20260505 07:14:46.446 | INFO | Analyzing chunk frames = [1334272,1334784) | 20260505 07:14:46.446 | INFO | MemInfo: { | 20260505 07:14:46.446 | INFO | "gpu" : " 17 GB", | 20260505 07:14:46.446 | INFO | "huge" : "", | 20260505 07:14:46.446 | INFO | "huge_shared" : " 27 GB", | 20260505 07:14:46.446 | INFO | "malloc" : " 14 GB", | 20260505 07:14:46.446 | INFO | "pinned" : " 29 GB" | 20260505 07:14:46.446 | INFO | } | 20260505 07:14:46.447 | INFO | Perf report for the last 5.13769 seconds: | 20260505 07:14:46.447 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:14:46.447 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:46.447 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:46.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:46.447 | INFO | ANALYSIS : 64.6%, 3.0, 100.9ms, 0 | 20260505 07:14:46.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:46.447 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:46.447 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 07:14:46.447 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:46.447 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:46.447 | INFO | Uploaded 36GiB to GPU in 2.2441s (16.0421GiB/s) | 20260505 07:14:46.447 | INFO | Downloaded 34GiB to GPU in 2.15723s (15.761GiB/s) | 20260505 07:14:51.585 | INFO | Analyzing chunk frames = [1334784,1335296) | 20260505 07:14:51.585 | INFO | MemInfo: { | 20260505 07:14:51.585 | INFO | "gpu" : " 17 GB", | 20260505 07:14:51.585 | INFO | "huge" : "", | 20260505 07:14:51.585 | INFO | "huge_shared" : " 27 GB", | 20260505 07:14:51.586 | INFO | "malloc" : " 14 GB", | 20260505 07:14:51.586 | INFO | "pinned" : " 29 GB" | 20260505 07:14:51.586 | INFO | } | 20260505 07:14:51.586 | INFO | Perf report for the last 5.13907 seconds: | 20260505 07:14:51.586 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:14:51.586 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:51.586 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:51.586 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:51.586 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 07:14:51.586 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:51.586 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:51.586 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 07:14:51.586 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:51.586 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:51.586 | INFO | Uploaded 36GiB to GPU in 2.16203s (16.6511GiB/s) | 20260505 07:14:51.586 | INFO | Downloaded 34GiB to GPU in 2.03538s (16.7045GiB/s) | 20260505 07:14:51.586 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 07:14:51.586 | INFO | Total mem usage: 15.3 GiB | 20260505 07:14:51.586 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 07:14:51.586 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.39%) | 20260505 07:14:51.586 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 07:14:51.586 | INFO | High Water Breakdown: | 20260505 07:14:51.586 | INFO | 60.6% : 8.73 GiB ( 93.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 07:14:51.586 | INFO | 39.3% : 5.66 GiB ( 85.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 07:14:51.586 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 07:14:51.586 | INFO | ------------------------------------------ | 20260505 07:14:51.586 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:14:51.586 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 07:14:51.586 | INFO | Total mem usage: 26.5 GiB | 20260505 07:14:51.586 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 07:14:51.586 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 07:14:51.586 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 07:14:51.586 | INFO | High Water Breakdown: | 20260505 07:14:51.586 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:14:51.586 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:14:51.586 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:14:51.586 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:14:51.586 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:14:51.586 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:14:51.586 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 07:14:51.586 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 07:14:51.586 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:14:51.586 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 07:14:51.586 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 07:14:51.586 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 07:14:51.586 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 07:14:51.586 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 07:14:51.586 | INFO | ------------------------------------------ | 20260505 07:14:51.586 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:14:51.586 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 07:14:51.586 | INFO | Total mem usage: 25.6 GiB | 20260505 07:14:51.586 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 07:14:51.586 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 07:14:51.586 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 07:14:51.586 | INFO | High Water Breakdown: | 20260505 07:14:51.586 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 07:14:51.586 | INFO | ------------------------------------------ | 20260505 07:14:51.586 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:14:51.586 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 07:14:51.586 | INFO | Total mem usage: 15.7 GiB | 20260505 07:14:51.586 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 07:14:51.586 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 07:14:51.586 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 07:14:51.586 | INFO | High Water Breakdown: | 20260505 07:14:51.586 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 07:14:51.586 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 07:14:51.586 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 07:14:51.586 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 07:14:51.586 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 07:14:51.586 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 07:14:51.586 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 07:14:51.586 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 07:14:51.586 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 07:14:51.587 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:14:51.587 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 07:14:51.587 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:14:51.587 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 07:14:51.587 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 07:14:51.587 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 07:14:51.587 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 07:14:51.587 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:14:51.587 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 07:14:51.587 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:14:51.587 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:14:51.587 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:14:51.587 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 07:14:51.587 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 07:14:51.587 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:14:51.587 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 07:14:51.587 | INFO | ------------------------------------------ | 20260505 07:14:51.587 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:14:51.587 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 07:14:51.587 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:14:51.587 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:14:51.587 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.52e+01, 1.35e+01 / 1.96e+00 | 20260505 07:14:51.587 | INFO | PerfReport: 0.29%, AggregateStats -- 9.42e-02 / 1.06e-01, 1.25e-02 / 1.10e-02 | 20260505 07:14:51.587 | INFO | PerfReport: 11.60%, Baseline -- 3.82e+00 / 4.46e+00, 6.28e-01 / 2.56e-01 | 20260505 07:14:51.587 | INFO | PerfReport: 3.30%, Histogram -- 1.09e+00 / 1.46e+00, 2.13e-01 / 1.29e-01 | 20260505 07:14:51.587 | INFO | PerfReport: 15.48%, DME -- 4.08e+01 / 4.45e+01, 4.04e+00 / 1.27e+00 | 20260505 07:14:51.587 | INFO | PerfReport: 56.25%, FrameLabeling -- 1.85e+01 / 1.92e+01, 2.06e+00 / 1.42e+00 | 20260505 07:14:51.587 | INFO | PerfReport: 7.63%, PulseAccumulating -- 2.51e+00 / 3.12e+00, 4.43e-01 / 2.76e-01 | 20260505 07:14:51.587 | INFO | PerfReport: 4.00%, Metrics -- 1.32e+00 / 1.75e+00, 3.71e-01 / 2.03e-01 | 20260505 07:14:51.587 | INFO | PerfReport: 1.45%, PulseSerializing -- 4.77e-01 / 5.59e-01, 9.13e-02 / 2.97e-02 | 20260505 07:14:51.587 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 07:14:51.587 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:14:51.587 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:14:51.587 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.25e+01 / 0.00e+00 | 20260505 07:14:51.587 | INFO | PerfReport: 22.30%, Upload -- 2.26e+01 / 0.00e+00, 8.06e+00 / 0.00e+00 | 20260505 07:14:51.587 | INFO | PerfReport: 21.49%, Download -- 2.18e+01 / 0.00e+00, 7.83e+00 / 0.00e+00 | 20260505 07:14:51.587 | INFO | PerfReport: 32.45%, Compute -- 3.29e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 07:14:51.587 | INFO | PerfReport: 1.51%, LockUpload -- 1.53e+00 / 0.00e+00, 7.07e+00 / 0.00e+00 | 20260505 07:14:51.587 | INFO | PerfReport: 0.79%, LockDownload -- 7.98e-01 / 0.00e+00, 2.07e+00 / 0.00e+00 | 20260505 07:14:51.587 | INFO | PerfReport: 21.44%, LockCompute -- 2.18e+01 / 0.00e+00, 1.68e+01 / 0.00e+00 | 20260505 07:14:51.587 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 1.89e-02 / 0.00e+00 | 20260505 07:14:51.701 | INFO | Starting a round of Baz IO | 20260505 07:14:55.009 | INFO | Populating RTMetrics report | 20260505 07:14:55.010 | INFO | RTMetrics report populated | 20260505 07:14:55.010 | INFO | Opening tmp RTMetrics json file | 20260505 07:14:55.012 | INFO | Serializing RTMetrics to json | 20260505 07:14:55.017 | INFO | Serialization done, swapping tmp file now... | 20260505 07:14:55.017 | INFO | RTMetrics done | 20260505 07:14:55.137 | INFO | Finished a round of Baz IO: Wrote 2.421 GiB in 3.435 seconds | 20260505 07:14:55.137 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 30.46% / 0.03% / 69.50% | 20260505 07:14:55.137 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.87697% / 3.97364e-05% / 9.13938e-05% | 20260505 07:14:55.137 | INFO | Baz IO overall: | 20260505 07:14:55.137 | INFO | BytesWritten : 721092241517 | 20260505 07:14:55.137 | INFO | NumEvents : 341929504011 | 20260505 07:14:56.725 | INFO | Analyzing chunk frames = [1335296,1335808) | 20260505 07:14:56.725 | INFO | MemInfo: { | 20260505 07:14:56.725 | INFO | "gpu" : " 17 GB", | 20260505 07:14:56.725 | INFO | "huge" : "", | 20260505 07:14:56.725 | INFO | "huge_shared" : " 27 GB", | 20260505 07:14:56.725 | INFO | "malloc" : " 14 GB", | 20260505 07:14:56.725 | INFO | "pinned" : " 29 GB" | 20260505 07:14:56.725 | INFO | } | 20260505 07:14:56.725 | INFO | Perf report for the last 5.13958 seconds: | 20260505 07:14:56.725 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:14:56.725 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:56.725 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:56.725 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:56.725 | INFO | ANALYSIS : 65.5%, 2.9, 100.0ms, 0 | 20260505 07:14:56.725 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:14:56.725 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 07:14:56.725 | INFO | PRE_HQ : 64.4%, 1.9, 62.6ms, 0 | 20260505 07:14:56.725 | INFO | BAZWRITER : 66.1%, 4.1, 138.5ms, 0 | 20260505 07:14:56.725 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260505 07:14:56.725 | INFO | Uploaded 36GiB to GPU in 2.27244s (15.842GiB/s) | 20260505 07:14:56.725 | INFO | Downloaded 38GiB to GPU in 2.32619s (16.3357GiB/s) | 20260505 07:15:01.864 | INFO | Analyzing chunk frames = [1335808,1336320) | 20260505 07:15:01.864 | INFO | MemInfo: { | 20260505 07:15:01.864 | INFO | "gpu" : " 17 GB", | 20260505 07:15:01.864 | INFO | "huge" : "", | 20260505 07:15:01.864 | INFO | "huge_shared" : " 27 GB", | 20260505 07:15:01.865 | INFO | "malloc" : " 14 GB", | 20260505 07:15:01.865 | INFO | "pinned" : " 29 GB" | 20260505 07:15:01.865 | INFO | } | 20260505 07:15:01.865 | INFO | Perf report for the last 5.13933 seconds: | 20260505 07:15:01.865 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:15:01.865 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:01.865 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:01.865 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:01.865 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 07:15:01.865 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:01.865 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:01.865 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:15:01.865 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:01.865 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:01.865 | INFO | Uploaded 36GiB to GPU in 2.11239s (17.0423GiB/s) | 20260505 07:15:01.865 | INFO | Downloaded 34GiB to GPU in 2.03683s (16.6926GiB/s) | 20260505 07:15:07.004 | INFO | Analyzing chunk frames = [1336320,1336832) | 20260505 07:15:07.004 | INFO | MemInfo: { | 20260505 07:15:07.004 | INFO | "gpu" : " 17 GB", | 20260505 07:15:07.004 | INFO | "huge" : "", | 20260505 07:15:07.004 | INFO | "huge_shared" : " 27 GB", | 20260505 07:15:07.004 | INFO | "malloc" : " 14 GB", | 20260505 07:15:07.004 | INFO | "pinned" : " 29 GB" | 20260505 07:15:07.004 | INFO | } | 20260505 07:15:07.004 | INFO | Perf report for the last 5.13927 seconds: | 20260505 07:15:07.004 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:15:07.004 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:07.004 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:07.004 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:07.004 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260505 07:15:07.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:07.004 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:07.004 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:15:07.004 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:07.004 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:07.004 | INFO | Uploaded 36GiB to GPU in 2.1538s (16.7146GiB/s) | 20260505 07:15:07.004 | INFO | Downloaded 34GiB to GPU in 2.0535s (16.5571GiB/s) | 20260505 07:15:12.143 | INFO | Analyzing chunk frames = [1336832,1337344) | 20260505 07:15:12.143 | INFO | MemInfo: { | 20260505 07:15:12.143 | INFO | "gpu" : " 17 GB", | 20260505 07:15:12.143 | INFO | "huge" : "", | 20260505 07:15:12.143 | INFO | "huge_shared" : " 27 GB", | 20260505 07:15:12.143 | INFO | "malloc" : " 14 GB", | 20260505 07:15:12.143 | INFO | "pinned" : " 29 GB" | 20260505 07:15:12.143 | INFO | } | 20260505 07:15:12.143 | INFO | Perf report for the last 5.13922 seconds: | 20260505 07:15:12.143 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:15:12.143 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:12.143 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:12.143 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:12.143 | INFO | ANALYSIS : 63.5%, 3.0, 98.6ms, 0 | 20260505 07:15:12.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:12.143 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:12.143 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:15:12.143 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:12.143 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:12.143 | INFO | Uploaded 36GiB to GPU in 2.13825s (16.8362GiB/s) | 20260505 07:15:12.143 | INFO | Downloaded 34GiB to GPU in 2.03617s (16.698GiB/s) | 20260505 07:15:17.282 | INFO | Analyzing chunk frames = [1337344,1337856) | 20260505 07:15:17.282 | INFO | MemInfo: { | 20260505 07:15:17.282 | INFO | "gpu" : " 17 GB", | 20260505 07:15:17.282 | INFO | "huge" : "", | 20260505 07:15:17.282 | INFO | "huge_shared" : " 27 GB", | 20260505 07:15:17.282 | INFO | "malloc" : " 14 GB", | 20260505 07:15:17.282 | INFO | "pinned" : " 29 GB" | 20260505 07:15:17.282 | INFO | } | 20260505 07:15:17.282 | INFO | Perf report for the last 5.13909 seconds: | 20260505 07:15:17.282 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:15:17.282 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:17.282 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:17.282 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:17.282 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 07:15:17.282 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:17.282 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:17.282 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 07:15:17.282 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:17.282 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:17.282 | INFO | Uploaded 36GiB to GPU in 2.16442s (16.6326GiB/s) | 20260505 07:15:17.283 | INFO | Downloaded 34GiB to GPU in 2.04927s (16.5913GiB/s) | 20260505 07:15:22.423 | INFO | Analyzing chunk frames = [1337856,1338368) | 20260505 07:15:22.423 | INFO | MemInfo: { | 20260505 07:15:22.423 | INFO | "gpu" : " 17 GB", | 20260505 07:15:22.423 | INFO | "huge" : "", | 20260505 07:15:22.423 | INFO | "huge_shared" : " 27 GB", | 20260505 07:15:22.423 | INFO | "malloc" : " 14 GB", | 20260505 07:15:22.423 | INFO | "pinned" : " 29 GB" | 20260505 07:15:22.423 | INFO | } | 20260505 07:15:22.423 | INFO | Perf report for the last 5.14116 seconds: | 20260505 07:15:22.423 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:15:22.423 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:22.423 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:22.423 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:22.423 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 07:15:22.423 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:22.423 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:22.423 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 07:15:22.423 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:22.423 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:22.424 | INFO | Uploaded 36GiB to GPU in 2.16881s (16.5989GiB/s) | 20260505 07:15:22.424 | INFO | Downloaded 34GiB to GPU in 2.05508s (16.5444GiB/s) | 20260505 07:15:27.561 | INFO | Analyzing chunk frames = [1338368,1338880) | 20260505 07:15:27.561 | INFO | MemInfo: { | 20260505 07:15:27.561 | INFO | "gpu" : " 17 GB", | 20260505 07:15:27.561 | INFO | "huge" : "", | 20260505 07:15:27.561 | INFO | "huge_shared" : " 27 GB", | 20260505 07:15:27.561 | INFO | "malloc" : " 14 GB", | 20260505 07:15:27.561 | INFO | "pinned" : " 29 GB" | 20260505 07:15:27.561 | INFO | } | 20260505 07:15:27.561 | INFO | Perf report for the last 5.13785 seconds: | 20260505 07:15:27.561 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:15:27.561 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:27.561 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:27.561 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:27.561 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 07:15:27.561 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:27.561 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:27.561 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 07:15:27.561 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:27.561 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:27.562 | INFO | Uploaded 36GiB to GPU in 2.19338s (16.413GiB/s) | 20260505 07:15:27.562 | INFO | Downloaded 34GiB to GPU in 2.15945s (15.7447GiB/s) | 20260505 07:15:32.700 | INFO | Analyzing chunk frames = [1338880,1339392) | 20260505 07:15:32.700 | INFO | MemInfo: { | 20260505 07:15:32.700 | INFO | "gpu" : " 17 GB", | 20260505 07:15:32.700 | INFO | "huge" : "", | 20260505 07:15:32.700 | INFO | "huge_shared" : " 27 GB", | 20260505 07:15:32.700 | INFO | "malloc" : " 14 GB", | 20260505 07:15:32.700 | INFO | "pinned" : " 29 GB" | 20260505 07:15:32.700 | INFO | } | 20260505 07:15:32.701 | INFO | Perf report for the last 5.1393 seconds: | 20260505 07:15:32.701 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:15:32.701 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:32.701 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:32.701 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:32.701 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 07:15:32.701 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:32.701 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:32.701 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 07:15:32.701 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:32.701 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:32.701 | INFO | Uploaded 36GiB to GPU in 2.16964s (16.5926GiB/s) | 20260505 07:15:32.701 | INFO | Downloaded 34GiB to GPU in 2.03643s (16.6959GiB/s) | 20260505 07:15:32.821 | INFO | Starting a round of Baz IO | 20260505 07:15:36.114 | INFO | Populating RTMetrics report | 20260505 07:15:36.115 | INFO | RTMetrics report populated | 20260505 07:15:36.115 | INFO | Opening tmp RTMetrics json file | 20260505 07:15:36.116 | INFO | Serializing RTMetrics to json | 20260505 07:15:36.117 | INFO | Serialization done, swapping tmp file now... | 20260505 07:15:36.117 | INFO | RTMetrics done | 20260505 07:15:36.241 | INFO | Finished a round of Baz IO: Wrote 2.42 GiB in 3.42 seconds | 20260505 07:15:36.241 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 30.43% / 0.03% / 69.54% | 20260505 07:15:36.241 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.85303% / 4.76837e-05% / 7.15256e-05% | 20260505 07:15:36.241 | INFO | Baz IO overall: | 20260505 07:15:36.241 | INFO | BytesWritten : 723690270984 | 20260505 07:15:36.241 | INFO | NumEvents : 343172396568 | 20260505 07:15:37.840 | INFO | Analyzing chunk frames = [1339392,1339904) | 20260505 07:15:37.840 | INFO | MemInfo: { | 20260505 07:15:37.840 | INFO | "gpu" : " 17 GB", | 20260505 07:15:37.840 | INFO | "huge" : "", | 20260505 07:15:37.840 | INFO | "huge_shared" : " 27 GB", | 20260505 07:15:37.840 | INFO | "malloc" : " 14 GB", | 20260505 07:15:37.840 | INFO | "pinned" : " 29 GB" | 20260505 07:15:37.840 | INFO | } | 20260505 07:15:37.840 | INFO | Perf report for the last 5.13917 seconds: | 20260505 07:15:37.840 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:15:37.840 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:37.840 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:37.840 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:37.840 | INFO | ANALYSIS : 65.3%, 2.9, 100.0ms, 0 | 20260505 07:15:37.840 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:37.840 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260505 07:15:37.840 | INFO | PRE_HQ : 64.2%, 1.9, 62.8ms, 0 | 20260505 07:15:37.840 | INFO | BAZWRITER : 65.8%, 4.1, 138.6ms, 0 | 20260505 07:15:37.840 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 07:15:37.840 | INFO | Uploaded 36GiB to GPU in 2.2428s (16.0514GiB/s) | 20260505 07:15:37.840 | INFO | Downloaded 38GiB to GPU in 2.31022s (16.4486GiB/s) | 20260505 07:15:42.979 | INFO | Analyzing chunk frames = [1339904,1340416) | 20260505 07:15:42.979 | INFO | MemInfo: { | 20260505 07:15:42.979 | INFO | "gpu" : " 17 GB", | 20260505 07:15:42.979 | INFO | "huge" : "", | 20260505 07:15:42.979 | INFO | "huge_shared" : " 27 GB", | 20260505 07:15:42.979 | INFO | "malloc" : " 14 GB", | 20260505 07:15:42.979 | INFO | "pinned" : " 29 GB" | 20260505 07:15:42.979 | INFO | } | 20260505 07:15:42.979 | INFO | Perf report for the last 5.13947 seconds: | 20260505 07:15:42.979 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:15:42.979 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:42.979 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:42.979 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:42.979 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 07:15:42.979 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:42.979 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:42.979 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 07:15:42.979 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:42.979 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:42.979 | INFO | Uploaded 36GiB to GPU in 2.10787s (17.0789GiB/s) | 20260505 07:15:42.980 | INFO | Downloaded 34GiB to GPU in 2.03925s (16.6728GiB/s) | 20260505 07:15:48.118 | INFO | Analyzing chunk frames = [1340416,1340928) | 20260505 07:15:48.118 | INFO | MemInfo: { | 20260505 07:15:48.118 | INFO | "gpu" : " 17 GB", | 20260505 07:15:48.118 | INFO | "huge" : "", | 20260505 07:15:48.118 | INFO | "huge_shared" : " 27 GB", | 20260505 07:15:48.118 | INFO | "malloc" : " 14 GB", | 20260505 07:15:48.118 | INFO | "pinned" : " 29 GB" | 20260505 07:15:48.118 | INFO | } | 20260505 07:15:48.118 | INFO | Perf report for the last 5.13921 seconds: | 20260505 07:15:48.118 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:15:48.118 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:48.118 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:48.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:48.118 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260505 07:15:48.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:48.118 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:48.118 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:15:48.118 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:48.118 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:48.119 | INFO | Uploaded 36GiB to GPU in 2.15661s (16.6929GiB/s) | 20260505 07:15:48.119 | INFO | Downloaded 34GiB to GPU in 2.05332s (16.5585GiB/s) | 20260505 07:15:53.258 | INFO | Analyzing chunk frames = [1340928,1341440) | 20260505 07:15:53.258 | INFO | MemInfo: { | 20260505 07:15:53.258 | INFO | "gpu" : " 17 GB", | 20260505 07:15:53.258 | INFO | "huge" : "", | 20260505 07:15:53.258 | INFO | "huge_shared" : " 27 GB", | 20260505 07:15:53.258 | INFO | "malloc" : " 14 GB", | 20260505 07:15:53.258 | INFO | "pinned" : " 29 GB" | 20260505 07:15:53.258 | INFO | } | 20260505 07:15:53.258 | INFO | Perf report for the last 5.13951 seconds: | 20260505 07:15:53.258 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:15:53.258 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:53.258 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:53.258 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:53.258 | INFO | ANALYSIS : 63.4%, 3.0, 98.5ms, 0 | 20260505 07:15:53.258 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:53.258 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:53.258 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:15:53.258 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:53.258 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:53.258 | INFO | Uploaded 36GiB to GPU in 2.13184s (16.8868GiB/s) | 20260505 07:15:53.258 | INFO | Downloaded 34GiB to GPU in 2.03283s (16.7255GiB/s) | 20260505 07:15:58.397 | INFO | Analyzing chunk frames = [1341440,1341952) | 20260505 07:15:58.397 | INFO | MemInfo: { | 20260505 07:15:58.397 | INFO | "gpu" : " 17 GB", | 20260505 07:15:58.397 | INFO | "huge" : "", | 20260505 07:15:58.397 | INFO | "huge_shared" : " 27 GB", | 20260505 07:15:58.397 | INFO | "malloc" : " 14 GB", | 20260505 07:15:58.397 | INFO | "pinned" : " 29 GB" | 20260505 07:15:58.397 | INFO | } | 20260505 07:15:58.397 | INFO | Perf report for the last 5.13946 seconds: | 20260505 07:15:58.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:15:58.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:58.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:58.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:58.397 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 07:15:58.397 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:58.397 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:58.397 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:15:58.398 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:58.398 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:15:58.398 | INFO | Uploaded 36GiB to GPU in 2.16354s (16.6394GiB/s) | 20260505 07:15:58.398 | INFO | Downloaded 34GiB to GPU in 2.05026s (16.5833GiB/s) | 20260505 07:16:03.538 | INFO | Analyzing chunk frames = [1341952,1342464) | 20260505 07:16:03.538 | INFO | MemInfo: { | 20260505 07:16:03.538 | INFO | "gpu" : " 17 GB", | 20260505 07:16:03.538 | INFO | "huge" : "", | 20260505 07:16:03.538 | INFO | "huge_shared" : " 27 GB", | 20260505 07:16:03.538 | INFO | "malloc" : " 14 GB", | 20260505 07:16:03.538 | INFO | "pinned" : " 29 GB" | 20260505 07:16:03.538 | INFO | } | 20260505 07:16:03.538 | INFO | Perf report for the last 5.14086 seconds: | 20260505 07:16:03.538 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:16:03.538 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:03.538 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:03.538 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:03.538 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 07:16:03.538 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:03.538 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:03.538 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 07:16:03.538 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:03.538 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:03.539 | INFO | Uploaded 36GiB to GPU in 2.1636s (16.6389GiB/s) | 20260505 07:16:03.539 | INFO | Downloaded 34GiB to GPU in 2.05141s (16.574GiB/s) | 20260505 07:16:08.676 | INFO | Analyzing chunk frames = [1342464,1342976) | 20260505 07:16:08.676 | INFO | MemInfo: { | 20260505 07:16:08.676 | INFO | "gpu" : " 17 GB", | 20260505 07:16:08.676 | INFO | "huge" : "", | 20260505 07:16:08.676 | INFO | "huge_shared" : " 27 GB", | 20260505 07:16:08.676 | INFO | "malloc" : " 14 GB", | 20260505 07:16:08.676 | INFO | "pinned" : " 29 GB" | 20260505 07:16:08.676 | INFO | } | 20260505 07:16:08.676 | INFO | Perf report for the last 5.13771 seconds: | 20260505 07:16:08.676 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:16:08.676 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:08.676 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:08.676 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:08.676 | INFO | ANALYSIS : 64.8%, 3.0, 101.1ms, 0 | 20260505 07:16:08.676 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:08.676 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:08.676 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260505 07:16:08.676 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:08.676 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:08.676 | INFO | Uploaded 36GiB to GPU in 2.22658s (16.1683GiB/s) | 20260505 07:16:08.676 | INFO | Downloaded 34GiB to GPU in 2.14626s (15.8415GiB/s) | 20260505 07:16:13.815 | INFO | Analyzing chunk frames = [1342976,1343488) | 20260505 07:16:13.815 | INFO | MemInfo: { | 20260505 07:16:13.815 | INFO | "gpu" : " 17 GB", | 20260505 07:16:13.815 | INFO | "huge" : "", | 20260505 07:16:13.815 | INFO | "huge_shared" : " 27 GB", | 20260505 07:16:13.815 | INFO | "malloc" : " 14 GB", | 20260505 07:16:13.815 | INFO | "pinned" : " 29 GB" | 20260505 07:16:13.815 | INFO | } | 20260505 07:16:13.815 | INFO | Perf report for the last 5.13911 seconds: | 20260505 07:16:13.815 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:16:13.815 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:13.815 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:13.815 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:13.815 | INFO | ANALYSIS : 64.1%, 3.0, 100.2ms, 0 | 20260505 07:16:13.815 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:13.815 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:13.815 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 07:16:13.815 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:13.815 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:13.815 | INFO | Uploaded 36GiB to GPU in 2.16717s (16.6115GiB/s) | 20260505 07:16:13.815 | INFO | Downloaded 34GiB to GPU in 2.03221s (16.7306GiB/s) | 20260505 07:16:13.815 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 07:16:13.815 | INFO | Total mem usage: 15.3 GiB | 20260505 07:16:13.815 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 07:16:13.815 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.15%) | 20260505 07:16:13.815 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 07:16:13.815 | INFO | High Water Breakdown: | 20260505 07:16:13.815 | INFO | 60.6% : 8.75 GiB ( 93.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 07:16:13.816 | INFO | 39.3% : 5.67 GiB ( 85.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 07:16:13.816 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 07:16:13.816 | INFO | ------------------------------------------ | 20260505 07:16:13.816 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:16:13.816 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 07:16:13.816 | INFO | Total mem usage: 26.5 GiB | 20260505 07:16:13.816 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 07:16:13.816 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 07:16:13.816 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 07:16:13.816 | INFO | High Water Breakdown: | 20260505 07:16:13.816 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:16:13.816 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:16:13.816 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:16:13.816 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:16:13.816 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:16:13.816 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:16:13.816 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 07:16:13.816 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 07:16:13.816 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:16:13.816 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 07:16:13.816 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 07:16:13.816 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 07:16:13.816 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 07:16:13.816 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 07:16:13.816 | INFO | ------------------------------------------ | 20260505 07:16:13.816 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:16:13.816 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 07:16:13.816 | INFO | Total mem usage: 25.6 GiB | 20260505 07:16:13.816 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 07:16:13.816 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 07:16:13.816 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 07:16:13.816 | INFO | High Water Breakdown: | 20260505 07:16:13.816 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 07:16:13.816 | INFO | ------------------------------------------ | 20260505 07:16:13.816 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:16:13.816 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 07:16:13.816 | INFO | Total mem usage: 15.7 GiB | 20260505 07:16:13.816 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 07:16:13.816 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 07:16:13.816 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 07:16:13.816 | INFO | High Water Breakdown: | 20260505 07:16:13.816 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 07:16:13.816 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 07:16:13.816 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 07:16:13.816 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 07:16:13.816 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 07:16:13.816 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 07:16:13.816 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 07:16:13.816 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 07:16:13.816 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 07:16:13.816 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:16:13.816 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 07:16:13.816 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:16:13.816 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 07:16:13.816 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 07:16:13.816 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 07:16:13.816 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 07:16:13.816 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:16:13.816 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 07:16:13.816 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:16:13.816 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:16:13.816 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:16:13.816 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 07:16:13.816 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 07:16:13.816 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:16:13.816 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 07:16:13.816 | INFO | ------------------------------------------ | 20260505 07:16:13.816 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:16:13.816 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 07:16:13.816 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:16:13.816 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:16:13.816 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.53e+01, 1.37e+01 / 2.27e+00 | 20260505 07:16:13.816 | INFO | PerfReport: 0.29%, AggregateStats -- 9.38e-02 / 1.04e-01, 1.05e-02 / 6.38e-03 | 20260505 07:16:13.816 | INFO | PerfReport: 11.61%, Baseline -- 3.82e+00 / 4.46e+00, 6.29e-01 / 2.59e-01 | 20260505 07:16:13.816 | INFO | PerfReport: 3.31%, Histogram -- 1.09e+00 / 1.46e+00, 2.13e-01 / 1.25e-01 | 20260505 07:16:13.816 | INFO | PerfReport: 15.47%, DME -- 4.07e+01 / 4.45e+01, 4.00e+00 / 1.50e+00 | 20260505 07:16:13.816 | INFO | PerfReport: 56.24%, FrameLabeling -- 1.85e+01 / 1.93e+01, 2.06e+00 / 1.60e+00 | 20260505 07:16:13.816 | INFO | PerfReport: 7.65%, PulseAccumulating -- 2.52e+00 / 3.18e+00, 4.52e-01 / 2.01e-01 | 20260505 07:16:13.816 | INFO | PerfReport: 3.99%, Metrics -- 1.31e+00 / 1.73e+00, 3.64e-01 / 1.41e-01 | 20260505 07:16:13.816 | INFO | PerfReport: 1.45%, PulseSerializing -- 4.76e-01 / 5.53e-01, 9.07e-02 / 2.41e-02 | 20260505 07:16:13.816 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 07:16:13.816 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:16:13.816 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:16:13.816 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.24e+01 / 0.00e+00 | 20260505 07:16:13.816 | INFO | PerfReport: 22.28%, Upload -- 2.26e+01 / 0.00e+00, 8.07e+00 / 0.00e+00 | 20260505 07:16:13.816 | INFO | PerfReport: 21.47%, Download -- 2.18e+01 / 0.00e+00, 7.83e+00 / 0.00e+00 | 20260505 07:16:13.816 | INFO | PerfReport: 32.42%, Compute -- 3.29e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 07:16:13.816 | INFO | PerfReport: 1.50%, LockUpload -- 1.53e+00 / 0.00e+00, 7.11e+00 / 0.00e+00 | 20260505 07:16:13.816 | INFO | PerfReport: 0.80%, LockDownload -- 8.16e-01 / 0.00e+00, 2.14e+00 / 0.00e+00 | 20260505 07:16:13.816 | INFO | PerfReport: 21.51%, LockCompute -- 2.18e+01 / 0.00e+00, 1.68e+01 / 0.00e+00 | 20260505 07:16:13.816 | INFO | PerfReport: 0.01%, Misc -- 1.51e-02 / 0.00e+00, 2.16e-02 / 0.00e+00 | 20260505 07:16:13.935 | INFO | Starting a round of Baz IO | 20260505 07:16:17.241 | INFO | Populating RTMetrics report | 20260505 07:16:17.242 | INFO | RTMetrics report populated | 20260505 07:16:17.242 | INFO | Opening tmp RTMetrics json file | 20260505 07:16:17.254 | INFO | Serializing RTMetrics to json | 20260505 07:16:17.295 | INFO | Serialization done, swapping tmp file now... | 20260505 07:16:17.298 | INFO | RTMetrics done | 20260505 07:16:17.367 | INFO | Finished a round of Baz IO: Wrote 2.421 GiB in 3.432 seconds | 20260505 07:16:17.367 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 30.40% / 0.03% / 69.57% | 20260505 07:16:17.367 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.84024% / 4.76837e-05% / 7.15256e-05% | 20260505 07:16:17.367 | INFO | Baz IO overall: | 20260505 07:16:17.367 | INFO | BytesWritten : 726289612921 | 20260505 07:16:17.367 | INFO | NumEvents : 344415771997 | 20260505 07:16:18.954 | INFO | Analyzing chunk frames = [1343488,1344000) | 20260505 07:16:18.955 | INFO | MemInfo: { | 20260505 07:16:18.955 | INFO | "gpu" : " 17 GB", | 20260505 07:16:18.955 | INFO | "huge" : "", | 20260505 07:16:18.955 | INFO | "huge_shared" : " 27 GB", | 20260505 07:16:18.955 | INFO | "malloc" : " 14 GB", | 20260505 07:16:18.955 | INFO | "pinned" : " 29 GB" | 20260505 07:16:18.955 | INFO | } | 20260505 07:16:18.955 | INFO | Perf report for the last 5.13953 seconds: | 20260505 07:16:18.955 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:16:18.955 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:18.955 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:18.955 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:18.955 | INFO | ANALYSIS : 65.5%, 2.9, 99.9ms, 0 | 20260505 07:16:18.955 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:18.955 | INFO | RT_METRICS : 47.2%, 1.0, 24.7ms, 0 | 20260505 07:16:18.955 | INFO | PRE_HQ : 64.3%, 1.9, 62.8ms, 0 | 20260505 07:16:18.955 | INFO | BAZWRITER : 66.0%, 4.1, 138.9ms, 0 | 20260505 07:16:18.955 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.5ms, 0 | 20260505 07:16:18.955 | INFO | Uploaded 36GiB to GPU in 2.26459s (15.8969GiB/s) | 20260505 07:16:18.955 | INFO | Downloaded 38GiB to GPU in 2.32191s (16.3659GiB/s) | 20260505 07:16:24.094 | INFO | Analyzing chunk frames = [1344000,1344512) | 20260505 07:16:24.094 | INFO | MemInfo: { | 20260505 07:16:24.094 | INFO | "gpu" : " 17 GB", | 20260505 07:16:24.094 | INFO | "huge" : "", | 20260505 07:16:24.094 | INFO | "huge_shared" : " 27 GB", | 20260505 07:16:24.094 | INFO | "malloc" : " 14 GB", | 20260505 07:16:24.094 | INFO | "pinned" : " 29 GB" | 20260505 07:16:24.094 | INFO | } | 20260505 07:16:24.094 | INFO | Perf report for the last 5.13942 seconds: | 20260505 07:16:24.094 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:16:24.094 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:24.094 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:24.094 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:24.094 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 07:16:24.094 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:24.094 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:24.094 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:16:24.094 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:24.094 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:24.094 | INFO | Uploaded 36GiB to GPU in 2.12661s (16.9283GiB/s) | 20260505 07:16:24.094 | INFO | Downloaded 34GiB to GPU in 2.04155s (16.654GiB/s) | 20260505 07:16:29.234 | INFO | Analyzing chunk frames = [1344512,1345024) | 20260505 07:16:29.234 | INFO | MemInfo: { | 20260505 07:16:29.234 | INFO | "gpu" : " 17 GB", | 20260505 07:16:29.234 | INFO | "huge" : "", | 20260505 07:16:29.234 | INFO | "huge_shared" : " 27 GB", | 20260505 07:16:29.234 | INFO | "malloc" : " 14 GB", | 20260505 07:16:29.234 | INFO | "pinned" : " 29 GB" | 20260505 07:16:29.234 | INFO | } | 20260505 07:16:29.234 | INFO | Perf report for the last 5.13952 seconds: | 20260505 07:16:29.234 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:16:29.234 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:29.234 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:29.234 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:29.234 | INFO | ANALYSIS : 63.6%, 3.0, 98.9ms, 0 | 20260505 07:16:29.234 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:29.234 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:29.234 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:16:29.234 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:29.234 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:29.234 | INFO | Uploaded 36GiB to GPU in 2.1621s (16.6505GiB/s) | 20260505 07:16:29.234 | INFO | Downloaded 34GiB to GPU in 2.0547s (16.5474GiB/s) | 20260505 07:16:34.373 | INFO | Analyzing chunk frames = [1345024,1345536) | 20260505 07:16:34.373 | INFO | MemInfo: { | 20260505 07:16:34.373 | INFO | "gpu" : " 17 GB", | 20260505 07:16:34.373 | INFO | "huge" : "", | 20260505 07:16:34.373 | INFO | "huge_shared" : " 27 GB", | 20260505 07:16:34.373 | INFO | "malloc" : " 14 GB", | 20260505 07:16:34.373 | INFO | "pinned" : " 29 GB" | 20260505 07:16:34.373 | INFO | } | 20260505 07:16:34.373 | INFO | Perf report for the last 5.1392 seconds: | 20260505 07:16:34.373 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:16:34.373 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:34.373 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:34.373 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:34.373 | INFO | ANALYSIS : 63.4%, 3.0, 98.6ms, 0 | 20260505 07:16:34.373 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:34.373 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:34.373 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:16:34.373 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:34.373 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:34.373 | INFO | Uploaded 36GiB to GPU in 2.13752s (16.8419GiB/s) | 20260505 07:16:34.373 | INFO | Downloaded 34GiB to GPU in 2.02705s (16.7731GiB/s) | 20260505 07:16:39.512 | INFO | Analyzing chunk frames = [1345536,1346048) | 20260505 07:16:39.512 | INFO | MemInfo: { | 20260505 07:16:39.512 | INFO | "gpu" : " 17 GB", | 20260505 07:16:39.512 | INFO | "huge" : "", | 20260505 07:16:39.512 | INFO | "huge_shared" : " 27 GB", | 20260505 07:16:39.512 | INFO | "malloc" : " 14 GB", | 20260505 07:16:39.512 | INFO | "pinned" : " 29 GB" | 20260505 07:16:39.512 | INFO | } | 20260505 07:16:39.512 | INFO | Perf report for the last 5.13921 seconds: | 20260505 07:16:39.512 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:16:39.512 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:39.512 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:39.512 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:39.512 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 07:16:39.512 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:39.512 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:39.512 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:16:39.512 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:39.512 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:39.512 | INFO | Uploaded 36GiB to GPU in 2.15816s (16.6809GiB/s) | 20260505 07:16:39.512 | INFO | Downloaded 34GiB to GPU in 2.05006s (16.5849GiB/s) | 20260505 07:16:44.653 | INFO | Analyzing chunk frames = [1346048,1346560) | 20260505 07:16:44.653 | INFO | MemInfo: { | 20260505 07:16:44.653 | INFO | "gpu" : " 17 GB", | 20260505 07:16:44.653 | INFO | "huge" : "", | 20260505 07:16:44.653 | INFO | "huge_shared" : " 27 GB", | 20260505 07:16:44.653 | INFO | "malloc" : " 14 GB", | 20260505 07:16:44.653 | INFO | "pinned" : " 29 GB" | 20260505 07:16:44.653 | INFO | } | 20260505 07:16:44.653 | INFO | Perf report for the last 5.14141 seconds: | 20260505 07:16:44.653 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:16:44.653 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:44.654 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:44.654 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:44.654 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 07:16:44.654 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:44.654 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:44.654 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:16:44.654 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:44.654 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:44.654 | INFO | Uploaded 36GiB to GPU in 2.15991s (16.6673GiB/s) | 20260505 07:16:44.654 | INFO | Downloaded 34GiB to GPU in 2.05619s (16.5355GiB/s) | 20260505 07:16:49.791 | INFO | Analyzing chunk frames = [1346560,1347072) | 20260505 07:16:49.791 | INFO | MemInfo: { | 20260505 07:16:49.791 | INFO | "gpu" : " 17 GB", | 20260505 07:16:49.791 | INFO | "huge" : "", | 20260505 07:16:49.791 | INFO | "huge_shared" : " 27 GB", | 20260505 07:16:49.791 | INFO | "malloc" : " 14 GB", | 20260505 07:16:49.791 | INFO | "pinned" : " 29 GB" | 20260505 07:16:49.791 | INFO | } | 20260505 07:16:49.791 | INFO | Perf report for the last 5.13731 seconds: | 20260505 07:16:49.791 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:16:49.791 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:49.791 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:49.791 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:49.791 | INFO | ANALYSIS : 64.5%, 3.0, 100.8ms, 0 | 20260505 07:16:49.791 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:49.791 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:49.791 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260505 07:16:49.791 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:49.791 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:49.791 | INFO | Uploaded 36GiB to GPU in 2.21041s (16.2866GiB/s) | 20260505 07:16:49.791 | INFO | Downloaded 34GiB to GPU in 2.15111s (15.8058GiB/s) | 20260505 07:16:54.930 | INFO | Analyzing chunk frames = [1347072,1347584) | 20260505 07:16:54.930 | INFO | MemInfo: { | 20260505 07:16:54.930 | INFO | "gpu" : " 17 GB", | 20260505 07:16:54.930 | INFO | "huge" : "", | 20260505 07:16:54.930 | INFO | "huge_shared" : " 27 GB", | 20260505 07:16:54.930 | INFO | "malloc" : " 14 GB", | 20260505 07:16:54.930 | INFO | "pinned" : " 29 GB" | 20260505 07:16:54.930 | INFO | } | 20260505 07:16:54.930 | INFO | Perf report for the last 5.13906 seconds: | 20260505 07:16:54.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:16:54.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:54.930 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:54.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:54.930 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 07:16:54.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:54.930 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:54.930 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 07:16:54.930 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:54.930 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:16:54.930 | INFO | Uploaded 36GiB to GPU in 2.16233s (16.6487GiB/s) | 20260505 07:16:54.930 | INFO | Downloaded 34GiB to GPU in 2.03183s (16.7337GiB/s) | 20260505 07:16:55.054 | INFO | Starting a round of Baz IO | 20260505 07:16:58.356 | INFO | Populating RTMetrics report | 20260505 07:16:58.356 | INFO | RTMetrics report populated | 20260505 07:16:58.356 | INFO | Opening tmp RTMetrics json file | 20260505 07:16:58.386 | INFO | Serializing RTMetrics to json | 20260505 07:16:58.433 | INFO | Serialization done, swapping tmp file now... | 20260505 07:16:58.443 | INFO | RTMetrics done | 20260505 07:16:58.474 | INFO | Finished a round of Baz IO: Wrote 2.421 GiB in 3.419 seconds | 20260505 07:16:58.474 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 30.36% / 0.03% / 69.60% | 20260505 07:16:58.474 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.8285% / 4.76837e-05% / 5.5631e-05% | 20260505 07:16:58.474 | INFO | Baz IO overall: | 20260505 07:16:58.474 | INFO | BytesWritten : 728889672978 | 20260505 07:16:58.474 | INFO | NumEvents : 345659719558 | 20260505 07:17:00.070 | INFO | Analyzing chunk frames = [1347584,1348096) | 20260505 07:17:00.070 | INFO | MemInfo: { | 20260505 07:17:00.070 | INFO | "gpu" : " 17 GB", | 20260505 07:17:00.070 | INFO | "huge" : "", | 20260505 07:17:00.070 | INFO | "huge_shared" : " 27 GB", | 20260505 07:17:00.070 | INFO | "malloc" : " 14 GB", | 20260505 07:17:00.070 | INFO | "pinned" : " 29 GB" | 20260505 07:17:00.070 | INFO | } | 20260505 07:17:00.070 | INFO | Perf report for the last 5.14028 seconds: | 20260505 07:17:00.070 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:17:00.070 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:00.070 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:00.070 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:00.070 | INFO | ANALYSIS : 65.5%, 2.9, 99.7ms, 0 | 20260505 07:17:00.070 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:00.070 | INFO | RT_METRICS : 47.5%, 1.0, 24.9ms, 0 | 20260505 07:17:00.070 | INFO | PRE_HQ : 64.3%, 1.9, 62.8ms, 0 | 20260505 07:17:00.070 | INFO | BAZWRITER : 65.7%, 4.1, 140.0ms, 0 | 20260505 07:17:00.070 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 07:17:00.070 | INFO | Uploaded 36GiB to GPU in 2.25049s (15.9965GiB/s) | 20260505 07:17:00.070 | INFO | Downloaded 38GiB to GPU in 2.34453s (16.2079GiB/s) | 20260505 07:17:05.209 | INFO | Analyzing chunk frames = [1348096,1348608) | 20260505 07:17:05.209 | INFO | MemInfo: { | 20260505 07:17:05.209 | INFO | "gpu" : " 17 GB", | 20260505 07:17:05.209 | INFO | "huge" : "", | 20260505 07:17:05.209 | INFO | "huge_shared" : " 27 GB", | 20260505 07:17:05.209 | INFO | "malloc" : " 14 GB", | 20260505 07:17:05.209 | INFO | "pinned" : " 29 GB" | 20260505 07:17:05.209 | INFO | } | 20260505 07:17:05.209 | INFO | Perf report for the last 5.13932 seconds: | 20260505 07:17:05.209 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:17:05.210 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:05.210 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:05.210 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:05.210 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 07:17:05.210 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:05.210 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:05.210 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:17:05.210 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:05.210 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:05.210 | INFO | Uploaded 36GiB to GPU in 2.12868s (16.9119GiB/s) | 20260505 07:17:05.210 | INFO | Downloaded 34GiB to GPU in 2.04087s (16.6595GiB/s) | 20260505 07:17:10.348 | INFO | Analyzing chunk frames = [1348608,1349120) | 20260505 07:17:10.348 | INFO | MemInfo: { | 20260505 07:17:10.348 | INFO | "gpu" : " 17 GB", | 20260505 07:17:10.348 | INFO | "huge" : "", | 20260505 07:17:10.348 | INFO | "huge_shared" : " 27 GB", | 20260505 07:17:10.348 | INFO | "malloc" : " 14 GB", | 20260505 07:17:10.348 | INFO | "pinned" : " 29 GB" | 20260505 07:17:10.348 | INFO | } | 20260505 07:17:10.348 | INFO | Perf report for the last 5.13878 seconds: | 20260505 07:17:10.348 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:17:10.348 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:10.348 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:10.348 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:10.348 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 07:17:10.348 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:10.348 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:10.348 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:17:10.348 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:10.348 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:10.349 | INFO | Uploaded 36GiB to GPU in 2.15584s (16.6988GiB/s) | 20260505 07:17:10.349 | INFO | Downloaded 34GiB to GPU in 2.05521s (16.5434GiB/s) | 20260505 07:17:15.487 | INFO | Analyzing chunk frames = [1349120,1349632) | 20260505 07:17:15.487 | INFO | MemInfo: { | 20260505 07:17:15.487 | INFO | "gpu" : " 17 GB", | 20260505 07:17:15.487 | INFO | "huge" : "", | 20260505 07:17:15.487 | INFO | "huge_shared" : " 27 GB", | 20260505 07:17:15.487 | INFO | "malloc" : " 14 GB", | 20260505 07:17:15.487 | INFO | "pinned" : " 29 GB" | 20260505 07:17:15.487 | INFO | } | 20260505 07:17:15.487 | INFO | Perf report for the last 5.13906 seconds: | 20260505 07:17:15.487 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:17:15.487 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:15.487 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:15.487 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:15.487 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 07:17:15.487 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:15.487 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:15.487 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 07:17:15.487 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:15.487 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:15.488 | INFO | Uploaded 36GiB to GPU in 2.12748s (16.9215GiB/s) | 20260505 07:17:15.488 | INFO | Downloaded 34GiB to GPU in 2.02914s (16.7558GiB/s) | 20260505 07:17:20.627 | INFO | Analyzing chunk frames = [1349632,1350144) | 20260505 07:17:20.627 | INFO | MemInfo: { | 20260505 07:17:20.627 | INFO | "gpu" : " 17 GB", | 20260505 07:17:20.627 | INFO | "huge" : "", | 20260505 07:17:20.627 | INFO | "huge_shared" : " 27 GB", | 20260505 07:17:20.627 | INFO | "malloc" : " 14 GB", | 20260505 07:17:20.627 | INFO | "pinned" : " 29 GB" | 20260505 07:17:20.627 | INFO | } | 20260505 07:17:20.627 | INFO | Perf report for the last 5.13935 seconds: | 20260505 07:17:20.627 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:17:20.627 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:20.627 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:20.627 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:20.627 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 07:17:20.627 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:20.627 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:20.627 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:17:20.627 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:20.627 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:20.627 | INFO | Uploaded 36GiB to GPU in 2.15784s (16.6834GiB/s) | 20260505 07:17:20.627 | INFO | Downloaded 34GiB to GPU in 2.04851s (16.5975GiB/s) | 20260505 07:17:25.768 | INFO | Analyzing chunk frames = [1350144,1350656) | 20260505 07:17:25.768 | INFO | MemInfo: { | 20260505 07:17:25.768 | INFO | "gpu" : " 17 GB", | 20260505 07:17:25.768 | INFO | "huge" : "", | 20260505 07:17:25.768 | INFO | "huge_shared" : " 27 GB", | 20260505 07:17:25.768 | INFO | "malloc" : " 14 GB", | 20260505 07:17:25.768 | INFO | "pinned" : " 29 GB" | 20260505 07:17:25.768 | INFO | } | 20260505 07:17:25.768 | INFO | Perf report for the last 5.14125 seconds: | 20260505 07:17:25.768 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:17:25.768 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:25.768 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:25.768 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:25.768 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 07:17:25.768 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:25.768 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:25.768 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:17:25.768 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:25.768 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:25.768 | INFO | Uploaded 36GiB to GPU in 2.16139s (16.6559GiB/s) | 20260505 07:17:25.768 | INFO | Downloaded 34GiB to GPU in 2.05748s (16.5251GiB/s) | 20260505 07:17:30.905 | INFO | Analyzing chunk frames = [1350656,1351168) | 20260505 07:17:30.905 | INFO | MemInfo: { | 20260505 07:17:30.905 | INFO | "gpu" : " 17 GB", | 20260505 07:17:30.905 | INFO | "huge" : "", | 20260505 07:17:30.905 | INFO | "huge_shared" : " 27 GB", | 20260505 07:17:30.905 | INFO | "malloc" : " 14 GB", | 20260505 07:17:30.905 | INFO | "pinned" : " 29 GB" | 20260505 07:17:30.905 | INFO | } | 20260505 07:17:30.905 | INFO | Perf report for the last 5.13735 seconds: | 20260505 07:17:30.905 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:17:30.905 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:30.905 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:30.905 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:30.905 | INFO | ANALYSIS : 64.5%, 3.0, 100.7ms, 0 | 20260505 07:17:30.905 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:30.905 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:30.905 | INFO | PRE_HQ : 31.9%, 1.0, 16.5ms, 0 | 20260505 07:17:30.905 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:30.905 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:30.906 | INFO | Uploaded 36GiB to GPU in 2.23247s (16.1257GiB/s) | 20260505 07:17:30.906 | INFO | Downloaded 34GiB to GPU in 2.1537s (15.7868GiB/s) | 20260505 07:17:36.045 | INFO | Analyzing chunk frames = [1351168,1351680) | 20260505 07:17:36.045 | INFO | MemInfo: { | 20260505 07:17:36.045 | INFO | "gpu" : " 17 GB", | 20260505 07:17:36.045 | INFO | "huge" : "", | 20260505 07:17:36.045 | INFO | "huge_shared" : " 27 GB", | 20260505 07:17:36.045 | INFO | "malloc" : " 14 GB", | 20260505 07:17:36.045 | INFO | "pinned" : " 29 GB" | 20260505 07:17:36.045 | INFO | } | 20260505 07:17:36.045 | INFO | Perf report for the last 5.1395 seconds: | 20260505 07:17:36.045 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:17:36.045 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:36.045 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:36.045 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:36.045 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 07:17:36.045 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:36.045 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:36.045 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 07:17:36.045 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:36.045 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:36.045 | INFO | Uploaded 36GiB to GPU in 2.16403s (16.6356GiB/s) | 20260505 07:17:36.045 | INFO | Downloaded 34GiB to GPU in 2.04531s (16.6234GiB/s) | 20260505 07:17:36.045 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 07:17:36.045 | INFO | Total mem usage: 15.3 GiB | 20260505 07:17:36.045 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 07:17:36.045 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.12%) | 20260505 07:17:36.045 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 07:17:36.045 | INFO | High Water Breakdown: | 20260505 07:17:36.045 | INFO | 60.6% : 8.75 GiB ( 93.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 07:17:36.045 | INFO | 39.2% : 5.67 GiB ( 85.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 07:17:36.045 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 07:17:36.045 | INFO | ------------------------------------------ | 20260505 07:17:36.045 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:17:36.045 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 07:17:36.045 | INFO | Total mem usage: 26.5 GiB | 20260505 07:17:36.045 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 07:17:36.045 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 07:17:36.045 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 07:17:36.045 | INFO | High Water Breakdown: | 20260505 07:17:36.045 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:17:36.045 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:17:36.045 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:17:36.045 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:17:36.045 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:17:36.045 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:17:36.045 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 07:17:36.045 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 07:17:36.045 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:17:36.045 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 07:17:36.045 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 07:17:36.045 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 07:17:36.045 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 07:17:36.045 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 07:17:36.045 | INFO | ------------------------------------------ | 20260505 07:17:36.045 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:17:36.045 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 07:17:36.046 | INFO | Total mem usage: 25.6 GiB | 20260505 07:17:36.046 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 07:17:36.046 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 07:17:36.046 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 07:17:36.046 | INFO | High Water Breakdown: | 20260505 07:17:36.046 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 07:17:36.046 | INFO | ------------------------------------------ | 20260505 07:17:36.046 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:17:36.046 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 07:17:36.046 | INFO | Total mem usage: 15.7 GiB | 20260505 07:17:36.046 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 07:17:36.046 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 07:17:36.046 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 07:17:36.046 | INFO | High Water Breakdown: | 20260505 07:17:36.046 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 07:17:36.046 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 07:17:36.046 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 07:17:36.046 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 07:17:36.046 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 07:17:36.046 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 07:17:36.046 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 07:17:36.046 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 07:17:36.046 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 07:17:36.046 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:17:36.046 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 07:17:36.046 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:17:36.046 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 07:17:36.046 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 07:17:36.046 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 07:17:36.046 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 07:17:36.046 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:17:36.046 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 07:17:36.046 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:17:36.046 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:17:36.046 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:17:36.046 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 07:17:36.046 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 07:17:36.046 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:17:36.046 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 07:17:36.046 | INFO | ------------------------------------------ | 20260505 07:17:36.046 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:17:36.046 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 07:17:36.046 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:17:36.046 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:17:36.046 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.56e+01, 1.36e+01 / 1.73e+00 | 20260505 07:17:36.046 | INFO | PerfReport: 0.28%, AggregateStats -- 9.37e-02 / 1.04e-01, 1.11e-02 / 6.58e-03 | 20260505 07:17:36.046 | INFO | PerfReport: 11.60%, Baseline -- 3.82e+00 / 4.45e+00, 6.31e-01 / 2.18e-01 | 20260505 07:17:36.046 | INFO | PerfReport: 3.31%, Histogram -- 1.09e+00 / 1.47e+00, 2.15e-01 / 1.11e-01 | 20260505 07:17:36.046 | INFO | PerfReport: 15.50%, DME -- 4.08e+01 / 4.47e+01, 4.25e+00 / 1.29e+00 | 20260505 07:17:36.046 | INFO | PerfReport: 56.26%, FrameLabeling -- 1.85e+01 / 1.95e+01, 2.06e+00 / 1.20e+00 | 20260505 07:17:36.046 | INFO | PerfReport: 7.61%, PulseAccumulating -- 2.50e+00 / 3.17e+00, 4.43e-01 / 2.21e-01 | 20260505 07:17:36.046 | INFO | PerfReport: 3.99%, Metrics -- 1.31e+00 / 1.74e+00, 3.66e-01 / 1.54e-01 | 20260505 07:17:36.046 | INFO | PerfReport: 1.44%, PulseSerializing -- 4.75e-01 / 5.56e-01, 9.05e-02 / 3.27e-02 | 20260505 07:17:36.046 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 07:17:36.046 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:17:36.046 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:17:36.046 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.26e+01 / 0.00e+00 | 20260505 07:17:36.046 | INFO | PerfReport: 22.30%, Upload -- 2.26e+01 / 0.00e+00, 8.10e+00 / 0.00e+00 | 20260505 07:17:36.046 | INFO | PerfReport: 21.50%, Download -- 2.18e+01 / 0.00e+00, 7.89e+00 / 0.00e+00 | 20260505 07:17:36.046 | INFO | PerfReport: 32.44%, Compute -- 3.29e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 07:17:36.046 | INFO | PerfReport: 1.48%, LockUpload -- 1.50e+00 / 0.00e+00, 7.05e+00 / 0.00e+00 | 20260505 07:17:36.046 | INFO | PerfReport: 0.83%, LockDownload -- 8.41e-01 / 0.00e+00, 2.16e+00 / 0.00e+00 | 20260505 07:17:36.046 | INFO | PerfReport: 21.44%, LockCompute -- 2.18e+01 / 0.00e+00, 1.68e+01 / 0.00e+00 | 20260505 07:17:36.046 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.60e-02 / 0.00e+00 | 20260505 07:17:36.169 | INFO | Starting a round of Baz IO | 20260505 07:17:39.467 | INFO | Populating RTMetrics report | 20260505 07:17:39.467 | INFO | RTMetrics report populated | 20260505 07:17:39.467 | INFO | Opening tmp RTMetrics json file | 20260505 07:17:39.468 | INFO | Serializing RTMetrics to json | 20260505 07:17:39.470 | INFO | Serialization done, swapping tmp file now... | 20260505 07:17:39.470 | INFO | RTMetrics done | 20260505 07:17:39.594 | INFO | Finished a round of Baz IO: Wrote 2.424 GiB in 3.425 seconds | 20260505 07:17:39.595 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 30.33% / 0.03% / 69.64% | 20260505 07:17:39.595 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.8216% / 5.5631e-05% / 6.35783e-05% | 20260505 07:17:39.595 | INFO | Baz IO overall: | 20260505 07:17:39.595 | INFO | BytesWritten : 731492279136 | 20260505 07:17:39.595 | INFO | NumEvents : 346905172692 | 20260505 07:17:41.184 | INFO | Analyzing chunk frames = [1351680,1352192) | 20260505 07:17:41.184 | INFO | MemInfo: { | 20260505 07:17:41.184 | INFO | "gpu" : " 17 GB", | 20260505 07:17:41.184 | INFO | "huge" : "", | 20260505 07:17:41.184 | INFO | "huge_shared" : " 27 GB", | 20260505 07:17:41.184 | INFO | "malloc" : " 14 GB", | 20260505 07:17:41.184 | INFO | "pinned" : " 29 GB" | 20260505 07:17:41.184 | INFO | } | 20260505 07:17:41.184 | INFO | Perf report for the last 5.13924 seconds: | 20260505 07:17:41.184 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:17:41.184 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:41.184 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:41.184 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:41.184 | INFO | ANALYSIS : 65.4%, 2.9, 99.7ms, 0 | 20260505 07:17:41.184 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:41.184 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 07:17:41.184 | INFO | PRE_HQ : 64.2%, 1.9, 62.3ms, 0 | 20260505 07:17:41.184 | INFO | BAZWRITER : 65.9%, 4.1, 139.2ms, 0 | 20260505 07:17:41.184 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 07:17:41.184 | INFO | Uploaded 36GiB to GPU in 2.2348s (16.1088GiB/s) | 20260505 07:17:41.184 | INFO | Downloaded 38GiB to GPU in 2.35637s (16.1265GiB/s) | 20260505 07:17:46.323 | INFO | Analyzing chunk frames = [1352192,1352704) | 20260505 07:17:46.324 | INFO | MemInfo: { | 20260505 07:17:46.324 | INFO | "gpu" : " 17 GB", | 20260505 07:17:46.324 | INFO | "huge" : "", | 20260505 07:17:46.324 | INFO | "huge_shared" : " 27 GB", | 20260505 07:17:46.324 | INFO | "malloc" : " 14 GB", | 20260505 07:17:46.324 | INFO | "pinned" : " 29 GB" | 20260505 07:17:46.324 | INFO | } | 20260505 07:17:46.324 | INFO | Perf report for the last 5.13957 seconds: | 20260505 07:17:46.324 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:17:46.324 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:46.324 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:46.324 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:46.324 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 07:17:46.324 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:46.324 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:46.324 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:17:46.324 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:46.324 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:46.324 | INFO | Uploaded 36GiB to GPU in 2.1063s (17.0916GiB/s) | 20260505 07:17:46.324 | INFO | Downloaded 34GiB to GPU in 2.03854s (16.6786GiB/s) | 20260505 07:17:51.463 | INFO | Analyzing chunk frames = [1352704,1353216) | 20260505 07:17:51.463 | INFO | MemInfo: { | 20260505 07:17:51.463 | INFO | "gpu" : " 17 GB", | 20260505 07:17:51.463 | INFO | "huge" : "", | 20260505 07:17:51.463 | INFO | "huge_shared" : " 27 GB", | 20260505 07:17:51.463 | INFO | "malloc" : " 14 GB", | 20260505 07:17:51.463 | INFO | "pinned" : " 29 GB" | 20260505 07:17:51.463 | INFO | } | 20260505 07:17:51.463 | INFO | Perf report for the last 5.13937 seconds: | 20260505 07:17:51.463 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:17:51.463 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:51.463 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:51.463 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:51.463 | INFO | ANALYSIS : 63.7%, 3.0, 99.1ms, 0 | 20260505 07:17:51.463 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:51.463 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:51.463 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 07:17:51.463 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:51.463 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:51.463 | INFO | Uploaded 36GiB to GPU in 2.16686s (16.6139GiB/s) | 20260505 07:17:51.463 | INFO | Downloaded 34GiB to GPU in 2.0575s (16.5249GiB/s) | 20260505 07:17:56.602 | INFO | Analyzing chunk frames = [1353216,1353728) | 20260505 07:17:56.602 | INFO | MemInfo: { | 20260505 07:17:56.602 | INFO | "gpu" : " 17 GB", | 20260505 07:17:56.602 | INFO | "huge" : "", | 20260505 07:17:56.602 | INFO | "huge_shared" : " 27 GB", | 20260505 07:17:56.602 | INFO | "malloc" : " 14 GB", | 20260505 07:17:56.602 | INFO | "pinned" : " 29 GB" | 20260505 07:17:56.602 | INFO | } | 20260505 07:17:56.602 | INFO | Perf report for the last 5.13922 seconds: | 20260505 07:17:56.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:17:56.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:56.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:56.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:56.602 | INFO | ANALYSIS : 63.4%, 3.0, 98.4ms, 0 | 20260505 07:17:56.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:56.602 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:56.602 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:17:56.602 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:56.602 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:17:56.603 | INFO | Uploaded 36GiB to GPU in 2.1286s (16.9125GiB/s) | 20260505 07:17:56.603 | INFO | Downloaded 34GiB to GPU in 2.02848s (16.7613GiB/s) | 20260505 07:18:01.741 | INFO | Analyzing chunk frames = [1353728,1354240) | 20260505 07:18:01.741 | INFO | MemInfo: { | 20260505 07:18:01.741 | INFO | "gpu" : " 17 GB", | 20260505 07:18:01.741 | INFO | "huge" : "", | 20260505 07:18:01.741 | INFO | "huge_shared" : " 27 GB", | 20260505 07:18:01.741 | INFO | "malloc" : " 14 GB", | 20260505 07:18:01.741 | INFO | "pinned" : " 29 GB" | 20260505 07:18:01.742 | INFO | } | 20260505 07:18:01.742 | INFO | Perf report for the last 5.13923 seconds: | 20260505 07:18:01.742 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:18:01.742 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:01.742 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:01.742 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:01.742 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 07:18:01.742 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:01.742 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:01.742 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 07:18:01.742 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:01.742 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:01.742 | INFO | Uploaded 36GiB to GPU in 2.16056s (16.6623GiB/s) | 20260505 07:18:01.742 | INFO | Downloaded 34GiB to GPU in 2.04961s (16.5885GiB/s) | 20260505 07:18:06.882 | INFO | Analyzing chunk frames = [1354240,1354752) | 20260505 07:18:06.882 | INFO | MemInfo: { | 20260505 07:18:06.882 | INFO | "gpu" : " 17 GB", | 20260505 07:18:06.882 | INFO | "huge" : "", | 20260505 07:18:06.883 | INFO | "huge_shared" : " 27 GB", | 20260505 07:18:06.883 | INFO | "malloc" : " 14 GB", | 20260505 07:18:06.883 | INFO | "pinned" : " 29 GB" | 20260505 07:18:06.883 | INFO | } | 20260505 07:18:06.883 | INFO | Perf report for the last 5.14105 seconds: | 20260505 07:18:06.883 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:18:06.883 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:06.883 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:06.883 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:06.883 | INFO | ANALYSIS : 63.5%, 3.0, 99.3ms, 0 | 20260505 07:18:06.883 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:06.883 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:06.883 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:18:06.883 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:06.883 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:06.883 | INFO | Uploaded 36GiB to GPU in 2.15785s (16.6833GiB/s) | 20260505 07:18:06.883 | INFO | Downloaded 34GiB to GPU in 2.05516s (16.5437GiB/s) | 20260505 07:18:12.020 | INFO | Analyzing chunk frames = [1354752,1355264) | 20260505 07:18:12.020 | INFO | MemInfo: { | 20260505 07:18:12.020 | INFO | "gpu" : " 17 GB", | 20260505 07:18:12.020 | INFO | "huge" : "", | 20260505 07:18:12.020 | INFO | "huge_shared" : " 27 GB", | 20260505 07:18:12.020 | INFO | "malloc" : " 14 GB", | 20260505 07:18:12.020 | INFO | "pinned" : " 29 GB" | 20260505 07:18:12.020 | INFO | } | 20260505 07:18:12.020 | INFO | Perf report for the last 5.13773 seconds: | 20260505 07:18:12.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:18:12.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:12.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:12.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:12.020 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 07:18:12.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:12.020 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:12.020 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260505 07:18:12.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:12.021 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:12.021 | INFO | Uploaded 36GiB to GPU in 2.20652s (16.3153GiB/s) | 20260505 07:18:12.021 | INFO | Downloaded 34GiB to GPU in 2.16421s (15.7101GiB/s) | 20260505 07:18:17.159 | INFO | Analyzing chunk frames = [1355264,1355776) | 20260505 07:18:17.159 | INFO | MemInfo: { | 20260505 07:18:17.159 | INFO | "gpu" : " 17 GB", | 20260505 07:18:17.159 | INFO | "huge" : "", | 20260505 07:18:17.160 | INFO | "huge_shared" : " 27 GB", | 20260505 07:18:17.160 | INFO | "malloc" : " 14 GB", | 20260505 07:18:17.160 | INFO | "pinned" : " 29 GB" | 20260505 07:18:17.160 | INFO | } | 20260505 07:18:17.160 | INFO | Perf report for the last 5.13934 seconds: | 20260505 07:18:17.160 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:18:17.160 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:17.160 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:17.160 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:17.160 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 07:18:17.160 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:17.160 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:17.160 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 07:18:17.160 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:17.160 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:17.160 | INFO | Uploaded 36GiB to GPU in 2.14474s (16.7853GiB/s) | 20260505 07:18:17.160 | INFO | Downloaded 34GiB to GPU in 2.02475s (16.7922GiB/s) | 20260505 07:18:17.277 | INFO | Starting a round of Baz IO | 20260505 07:18:20.581 | INFO | Populating RTMetrics report | 20260505 07:18:20.581 | INFO | RTMetrics report populated | 20260505 07:18:20.581 | INFO | Opening tmp RTMetrics json file | 20260505 07:18:20.604 | INFO | Serializing RTMetrics to json | 20260505 07:18:20.606 | INFO | Serialization done, swapping tmp file now... | 20260505 07:18:20.606 | INFO | RTMetrics done | 20260505 07:18:20.704 | INFO | Finished a round of Baz IO: Wrote 2.426 GiB in 3.426 seconds | 20260505 07:18:20.704 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 30.29% / 0.03% / 69.67% | 20260505 07:18:20.704 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.81548% / 2.38419e-05% / 7.15256e-05% | 20260505 07:18:20.704 | INFO | Baz IO overall: | 20260505 07:18:20.704 | INFO | BytesWritten : 734097086629 | 20260505 07:18:20.704 | INFO | NumEvents : 348152243040 | 20260505 07:18:22.299 | INFO | Analyzing chunk frames = [1355776,1356288) | 20260505 07:18:22.299 | INFO | MemInfo: { | 20260505 07:18:22.299 | INFO | "gpu" : " 17 GB", | 20260505 07:18:22.299 | INFO | "huge" : "", | 20260505 07:18:22.299 | INFO | "huge_shared" : " 27 GB", | 20260505 07:18:22.299 | INFO | "malloc" : " 14 GB", | 20260505 07:18:22.299 | INFO | "pinned" : " 29 GB" | 20260505 07:18:22.299 | INFO | } | 20260505 07:18:22.299 | INFO | Perf report for the last 5.13934 seconds: | 20260505 07:18:22.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:18:22.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:22.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:22.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:22.299 | INFO | ANALYSIS : 65.4%, 2.9, 100.0ms, 0 | 20260505 07:18:22.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:22.299 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260505 07:18:22.299 | INFO | PRE_HQ : 64.3%, 1.9, 62.4ms, 0 | 20260505 07:18:22.299 | INFO | BAZWRITER : 66.0%, 4.1, 138.4ms, 0 | 20260505 07:18:22.299 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 07:18:22.299 | INFO | Uploaded 36GiB to GPU in 2.26964s (15.8616GiB/s) | 20260505 07:18:22.299 | INFO | Downloaded 38GiB to GPU in 2.32639s (16.3343GiB/s) | 20260505 07:18:27.438 | INFO | Analyzing chunk frames = [1356288,1356800) | 20260505 07:18:27.438 | INFO | MemInfo: { | 20260505 07:18:27.438 | INFO | "gpu" : " 17 GB", | 20260505 07:18:27.438 | INFO | "huge" : "", | 20260505 07:18:27.438 | INFO | "huge_shared" : " 27 GB", | 20260505 07:18:27.438 | INFO | "malloc" : " 14 GB", | 20260505 07:18:27.438 | INFO | "pinned" : " 29 GB" | 20260505 07:18:27.438 | INFO | } | 20260505 07:18:27.438 | INFO | Perf report for the last 5.13917 seconds: | 20260505 07:18:27.438 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:18:27.438 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:27.438 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:27.438 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:27.438 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 07:18:27.438 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:27.438 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:27.438 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:18:27.438 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:27.438 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:27.438 | INFO | Uploaded 36GiB to GPU in 2.13407s (16.8692GiB/s) | 20260505 07:18:27.439 | INFO | Downloaded 34GiB to GPU in 2.04186s (16.6515GiB/s) | 20260505 07:18:32.578 | INFO | Analyzing chunk frames = [1356800,1357312) | 20260505 07:18:32.578 | INFO | MemInfo: { | 20260505 07:18:32.578 | INFO | "gpu" : " 17 GB", | 20260505 07:18:32.578 | INFO | "huge" : "", | 20260505 07:18:32.578 | INFO | "huge_shared" : " 27 GB", | 20260505 07:18:32.578 | INFO | "malloc" : " 14 GB", | 20260505 07:18:32.578 | INFO | "pinned" : " 29 GB" | 20260505 07:18:32.578 | INFO | } | 20260505 07:18:32.578 | INFO | Perf report for the last 5.1396 seconds: | 20260505 07:18:32.578 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:18:32.578 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:32.578 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:32.578 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:32.578 | INFO | ANALYSIS : 63.6%, 3.0, 98.9ms, 0 | 20260505 07:18:32.578 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:32.578 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:32.578 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:18:32.578 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:32.578 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:32.578 | INFO | Uploaded 36GiB to GPU in 2.15645s (16.6941GiB/s) | 20260505 07:18:32.578 | INFO | Downloaded 34GiB to GPU in 2.05264s (16.564GiB/s) | 20260505 07:18:37.717 | INFO | Analyzing chunk frames = [1357312,1357824) | 20260505 07:18:37.717 | INFO | MemInfo: { | 20260505 07:18:37.717 | INFO | "gpu" : " 17 GB", | 20260505 07:18:37.717 | INFO | "huge" : "", | 20260505 07:18:37.717 | INFO | "huge_shared" : " 27 GB", | 20260505 07:18:37.717 | INFO | "malloc" : " 14 GB", | 20260505 07:18:37.717 | INFO | "pinned" : " 29 GB" | 20260505 07:18:37.717 | INFO | } | 20260505 07:18:37.717 | INFO | Perf report for the last 5.13918 seconds: | 20260505 07:18:37.717 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:18:37.717 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:37.717 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:37.717 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:37.717 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 07:18:37.717 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:37.717 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:37.717 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:18:37.717 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:37.717 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:37.717 | INFO | Uploaded 36GiB to GPU in 2.12253s (16.9609GiB/s) | 20260505 07:18:37.717 | INFO | Downloaded 34GiB to GPU in 2.03024s (16.7468GiB/s) | 20260505 07:18:42.856 | INFO | Analyzing chunk frames = [1357824,1358336) | 20260505 07:18:42.856 | INFO | MemInfo: { | 20260505 07:18:42.856 | INFO | "gpu" : " 17 GB", | 20260505 07:18:42.856 | INFO | "huge" : "", | 20260505 07:18:42.856 | INFO | "huge_shared" : " 27 GB", | 20260505 07:18:42.856 | INFO | "malloc" : " 14 GB", | 20260505 07:18:42.856 | INFO | "pinned" : " 29 GB" | 20260505 07:18:42.856 | INFO | } | 20260505 07:18:42.856 | INFO | Perf report for the last 5.13917 seconds: | 20260505 07:18:42.856 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:18:42.856 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:42.856 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:42.856 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:42.856 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 07:18:42.856 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:42.856 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:42.856 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 07:18:42.856 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:42.856 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:42.856 | INFO | Uploaded 36GiB to GPU in 2.16452s (16.6319GiB/s) | 20260505 07:18:42.857 | INFO | Downloaded 34GiB to GPU in 2.04779s (16.6032GiB/s) | 20260505 07:18:47.997 | INFO | Analyzing chunk frames = [1358336,1358848) | 20260505 07:18:47.997 | INFO | MemInfo: { | 20260505 07:18:47.997 | INFO | "gpu" : " 17 GB", | 20260505 07:18:47.997 | INFO | "huge" : "", | 20260505 07:18:47.998 | INFO | "huge_shared" : " 27 GB", | 20260505 07:18:47.998 | INFO | "malloc" : " 14 GB", | 20260505 07:18:47.998 | INFO | "pinned" : " 29 GB" | 20260505 07:18:47.998 | INFO | } | 20260505 07:18:47.998 | INFO | Perf report for the last 5.14135 seconds: | 20260505 07:18:47.998 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:18:47.998 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:47.998 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:47.998 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:47.998 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 07:18:47.998 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:47.998 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:47.998 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:18:47.998 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:47.998 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:47.998 | INFO | Uploaded 36GiB to GPU in 2.15393s (16.7136GiB/s) | 20260505 07:18:47.998 | INFO | Downloaded 34GiB to GPU in 2.05912s (16.5119GiB/s) | 20260505 07:18:53.135 | INFO | Analyzing chunk frames = [1358848,1359360) | 20260505 07:18:53.135 | INFO | MemInfo: { | 20260505 07:18:53.135 | INFO | "gpu" : " 17 GB", | 20260505 07:18:53.135 | INFO | "huge" : "", | 20260505 07:18:53.135 | INFO | "huge_shared" : " 27 GB", | 20260505 07:18:53.135 | INFO | "malloc" : " 14 GB", | 20260505 07:18:53.135 | INFO | "pinned" : " 29 GB" | 20260505 07:18:53.135 | INFO | } | 20260505 07:18:53.135 | INFO | Perf report for the last 5.13736 seconds: | 20260505 07:18:53.135 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:18:53.135 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:53.135 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:53.135 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:53.135 | INFO | ANALYSIS : 64.7%, 3.0, 101.1ms, 0 | 20260505 07:18:53.135 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:53.135 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:53.135 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260505 07:18:53.135 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:53.135 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:53.136 | INFO | Uploaded 36GiB to GPU in 2.2237s (16.1892GiB/s) | 20260505 07:18:53.136 | INFO | Downloaded 34GiB to GPU in 2.15194s (15.7997GiB/s) | 20260505 07:18:58.274 | INFO | Analyzing chunk frames = [1359360,1359872) | 20260505 07:18:58.274 | INFO | MemInfo: { | 20260505 07:18:58.274 | INFO | "gpu" : " 17 GB", | 20260505 07:18:58.274 | INFO | "huge" : "", | 20260505 07:18:58.274 | INFO | "huge_shared" : " 27 GB", | 20260505 07:18:58.274 | INFO | "malloc" : " 14 GB", | 20260505 07:18:58.274 | INFO | "pinned" : " 29 GB" | 20260505 07:18:58.274 | INFO | } | 20260505 07:18:58.274 | INFO | Perf report for the last 5.13941 seconds: | 20260505 07:18:58.274 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:18:58.274 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:58.274 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:58.274 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:58.274 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 07:18:58.274 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:58.274 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:58.274 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 07:18:58.274 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:58.274 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:18:58.275 | INFO | Uploaded 36GiB to GPU in 2.16783s (16.6065GiB/s) | 20260505 07:18:58.275 | INFO | Downloaded 34GiB to GPU in 2.03091s (16.7413GiB/s) | 20260505 07:18:58.275 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 07:18:58.275 | INFO | Total mem usage: 15.3 GiB | 20260505 07:18:58.275 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 07:18:58.275 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.75%) | 20260505 07:18:58.275 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 07:18:58.275 | INFO | High Water Breakdown: | 20260505 07:18:58.275 | INFO | 60.6% : 8.7 GiB ( 93% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 07:18:58.275 | INFO | 39.3% : 5.64 GiB ( 84.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 07:18:58.275 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 07:18:58.275 | INFO | ------------------------------------------ | 20260505 07:18:58.275 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:18:58.275 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 07:18:58.275 | INFO | Total mem usage: 26.5 GiB | 20260505 07:18:58.275 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 07:18:58.275 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 07:18:58.275 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 07:18:58.275 | INFO | High Water Breakdown: | 20260505 07:18:58.275 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:18:58.275 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:18:58.275 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:18:58.275 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:18:58.275 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:18:58.275 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:18:58.275 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 07:18:58.275 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 07:18:58.275 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:18:58.275 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 07:18:58.275 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 07:18:58.275 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 07:18:58.275 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 07:18:58.275 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 07:18:58.275 | INFO | ------------------------------------------ | 20260505 07:18:58.275 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:18:58.275 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 07:18:58.275 | INFO | Total mem usage: 25.6 GiB | 20260505 07:18:58.275 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 07:18:58.275 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 07:18:58.275 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 07:18:58.275 | INFO | High Water Breakdown: | 20260505 07:18:58.275 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 07:18:58.275 | INFO | ------------------------------------------ | 20260505 07:18:58.275 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:18:58.275 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 07:18:58.275 | INFO | Total mem usage: 15.7 GiB | 20260505 07:18:58.275 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 07:18:58.275 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 07:18:58.275 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 07:18:58.275 | INFO | High Water Breakdown: | 20260505 07:18:58.275 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 07:18:58.275 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 07:18:58.275 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 07:18:58.275 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 07:18:58.275 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 07:18:58.275 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 07:18:58.275 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 07:18:58.275 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 07:18:58.275 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 07:18:58.275 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:18:58.275 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 07:18:58.275 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:18:58.275 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 07:18:58.275 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 07:18:58.275 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 07:18:58.275 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 07:18:58.275 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:18:58.275 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 07:18:58.275 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:18:58.275 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:18:58.275 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:18:58.275 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 07:18:58.275 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 07:18:58.275 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:18:58.275 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 07:18:58.275 | INFO | ------------------------------------------ | 20260505 07:18:58.275 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:18:58.275 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 07:18:58.275 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:18:58.276 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:18:58.276 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.54e+01, 1.38e+01 / 1.93e+00 | 20260505 07:18:58.276 | INFO | PerfReport: 0.28%, AggregateStats -- 9.37e-02 / 1.04e-01, 1.07e-02 / 5.11e-03 | 20260505 07:18:58.276 | INFO | PerfReport: 11.60%, Baseline -- 3.82e+00 / 4.45e+00, 6.29e-01 / 3.09e-01 | 20260505 07:18:58.276 | INFO | PerfReport: 3.31%, Histogram -- 1.09e+00 / 1.46e+00, 2.16e-01 / 1.18e-01 | 20260505 07:18:58.276 | INFO | PerfReport: 15.47%, DME -- 4.07e+01 / 4.45e+01, 3.96e+00 / 1.18e+00 | 20260505 07:18:58.276 | INFO | PerfReport: 56.25%, FrameLabeling -- 1.85e+01 / 1.94e+01, 2.04e+00 / 1.66e+00 | 20260505 07:18:58.276 | INFO | PerfReport: 7.63%, PulseAccumulating -- 2.51e+00 / 3.18e+00, 4.51e-01 / 2.06e-01 | 20260505 07:18:58.276 | INFO | PerfReport: 4.00%, Metrics -- 1.32e+00 / 1.72e+00, 3.69e-01 / 1.62e-01 | 20260505 07:18:58.276 | INFO | PerfReport: 1.45%, PulseSerializing -- 4.76e-01 / 5.54e-01, 9.24e-02 / 3.18e-02 | 20260505 07:18:58.276 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 07:18:58.276 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:18:58.276 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:18:58.276 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.26e+01 / 0.00e+00 | 20260505 07:18:58.276 | INFO | PerfReport: 22.26%, Upload -- 2.26e+01 / 0.00e+00, 8.02e+00 / 0.00e+00 | 20260505 07:18:58.276 | INFO | PerfReport: 21.50%, Download -- 2.18e+01 / 0.00e+00, 7.86e+00 / 0.00e+00 | 20260505 07:18:58.276 | INFO | PerfReport: 32.43%, Compute -- 3.29e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 07:18:58.276 | INFO | PerfReport: 1.50%, LockUpload -- 1.52e+00 / 0.00e+00, 7.13e+00 / 0.00e+00 | 20260505 07:18:58.276 | INFO | PerfReport: 0.83%, LockDownload -- 8.43e-01 / 0.00e+00, 2.21e+00 / 0.00e+00 | 20260505 07:18:58.276 | INFO | PerfReport: 21.46%, LockCompute -- 2.18e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260505 07:18:58.276 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 1.64e-02 / 0.00e+00 | 20260505 07:18:58.394 | INFO | Starting a round of Baz IO | 20260505 07:19:01.700 | INFO | Populating RTMetrics report | 20260505 07:19:01.700 | INFO | RTMetrics report populated | 20260505 07:19:01.700 | INFO | Opening tmp RTMetrics json file | 20260505 07:19:01.719 | INFO | Serializing RTMetrics to json | 20260505 07:19:01.761 | INFO | Serialization done, swapping tmp file now... | 20260505 07:19:01.764 | INFO | RTMetrics done | 20260505 07:19:01.839 | INFO | Finished a round of Baz IO: Wrote 2.427 GiB in 3.444 seconds | 20260505 07:19:01.839 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 30.26% / 0.03% / 69.71% | 20260505 07:19:01.839 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.80582% / 6.35783e-05% / 6.35783e-05% | 20260505 07:19:01.839 | INFO | Baz IO overall: | 20260505 07:19:01.839 | INFO | BytesWritten : 736703492512 | 20260505 07:19:01.839 | INFO | NumEvents : 349400185240 | 20260505 07:19:03.414 | INFO | Analyzing chunk frames = [1359872,1360384) | 20260505 07:19:03.414 | INFO | MemInfo: { | 20260505 07:19:03.414 | INFO | "gpu" : " 17 GB", | 20260505 07:19:03.414 | INFO | "huge" : "", | 20260505 07:19:03.414 | INFO | "huge_shared" : " 27 GB", | 20260505 07:19:03.414 | INFO | "malloc" : " 14 GB", | 20260505 07:19:03.414 | INFO | "pinned" : " 29 GB" | 20260505 07:19:03.414 | INFO | } | 20260505 07:19:03.414 | INFO | Perf report for the last 5.1395 seconds: | 20260505 07:19:03.414 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:19:03.414 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:03.414 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:03.414 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:03.414 | INFO | ANALYSIS : 65.5%, 2.9, 99.9ms, 0 | 20260505 07:19:03.414 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:03.414 | INFO | RT_METRICS : 47.3%, 1.0, 24.8ms, 0 | 20260505 07:19:03.414 | INFO | PRE_HQ : 64.3%, 1.9, 62.7ms, 0 | 20260505 07:19:03.414 | INFO | BAZWRITER : 66.3%, 4.1, 140.0ms, 0 | 20260505 07:19:03.414 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 07:19:03.414 | INFO | Uploaded 36GiB to GPU in 2.22958s (16.1465GiB/s) | 20260505 07:19:03.414 | INFO | Downloaded 38GiB to GPU in 2.35796s (16.1156GiB/s) | 20260505 07:19:08.553 | INFO | Analyzing chunk frames = [1360384,1360896) | 20260505 07:19:08.553 | INFO | MemInfo: { | 20260505 07:19:08.553 | INFO | "gpu" : " 17 GB", | 20260505 07:19:08.553 | INFO | "huge" : "", | 20260505 07:19:08.553 | INFO | "huge_shared" : " 27 GB", | 20260505 07:19:08.553 | INFO | "malloc" : " 14 GB", | 20260505 07:19:08.553 | INFO | "pinned" : " 29 GB" | 20260505 07:19:08.553 | INFO | } | 20260505 07:19:08.553 | INFO | Perf report for the last 5.13927 seconds: | 20260505 07:19:08.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:19:08.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:08.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:08.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:08.553 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 07:19:08.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:08.553 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:08.553 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:19:08.553 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:08.553 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:08.553 | INFO | Uploaded 36GiB to GPU in 2.13249s (16.8816GiB/s) | 20260505 07:19:08.553 | INFO | Downloaded 34GiB to GPU in 2.04194s (16.6508GiB/s) | 20260505 07:19:13.692 | INFO | Analyzing chunk frames = [1360896,1361408) | 20260505 07:19:13.692 | INFO | MemInfo: { | 20260505 07:19:13.693 | INFO | "gpu" : " 17 GB", | 20260505 07:19:13.693 | INFO | "huge" : "", | 20260505 07:19:13.693 | INFO | "huge_shared" : " 27 GB", | 20260505 07:19:13.693 | INFO | "malloc" : " 14 GB", | 20260505 07:19:13.693 | INFO | "pinned" : " 29 GB" | 20260505 07:19:13.693 | INFO | } | 20260505 07:19:13.693 | INFO | Perf report for the last 5.13945 seconds: | 20260505 07:19:13.693 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:19:13.693 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:13.693 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:13.693 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:13.693 | INFO | ANALYSIS : 63.6%, 3.0, 98.9ms, 0 | 20260505 07:19:13.693 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:13.693 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:13.693 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:19:13.693 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:13.693 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:13.693 | INFO | Uploaded 36GiB to GPU in 2.1579s (16.6829GiB/s) | 20260505 07:19:13.693 | INFO | Downloaded 34GiB to GPU in 2.05485s (16.5462GiB/s) | 20260505 07:19:18.832 | INFO | Analyzing chunk frames = [1361408,1361920) | 20260505 07:19:18.832 | INFO | MemInfo: { | 20260505 07:19:18.832 | INFO | "gpu" : " 17 GB", | 20260505 07:19:18.832 | INFO | "huge" : "", | 20260505 07:19:18.832 | INFO | "huge_shared" : " 27 GB", | 20260505 07:19:18.832 | INFO | "malloc" : " 14 GB", | 20260505 07:19:18.832 | INFO | "pinned" : " 29 GB" | 20260505 07:19:18.832 | INFO | } | 20260505 07:19:18.832 | INFO | Perf report for the last 5.13931 seconds: | 20260505 07:19:18.832 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:19:18.832 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:18.832 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:18.832 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:18.832 | INFO | ANALYSIS : 63.3%, 3.0, 98.3ms, 0 | 20260505 07:19:18.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:18.832 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:18.832 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 07:19:18.832 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:18.832 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:18.832 | INFO | Uploaded 36GiB to GPU in 2.12151s (16.9691GiB/s) | 20260505 07:19:18.832 | INFO | Downloaded 34GiB to GPU in 2.0341s (16.715GiB/s) | 20260505 07:19:23.971 | INFO | Analyzing chunk frames = [1361920,1362432) | 20260505 07:19:23.971 | INFO | MemInfo: { | 20260505 07:19:23.971 | INFO | "gpu" : " 17 GB", | 20260505 07:19:23.971 | INFO | "huge" : "", | 20260505 07:19:23.971 | INFO | "huge_shared" : " 27 GB", | 20260505 07:19:23.971 | INFO | "malloc" : " 14 GB", | 20260505 07:19:23.971 | INFO | "pinned" : " 29 GB" | 20260505 07:19:23.971 | INFO | } | 20260505 07:19:23.971 | INFO | Perf report for the last 5.13933 seconds: | 20260505 07:19:23.971 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:19:23.971 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:23.971 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:23.971 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:23.971 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 07:19:23.971 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:23.971 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:23.971 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 07:19:23.971 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:23.971 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:23.972 | INFO | Uploaded 36GiB to GPU in 2.17131s (16.5798GiB/s) | 20260505 07:19:23.972 | INFO | Downloaded 34GiB to GPU in 2.05381s (16.5546GiB/s) | 20260505 07:19:29.112 | INFO | Analyzing chunk frames = [1362432,1362944) | 20260505 07:19:29.112 | INFO | MemInfo: { | 20260505 07:19:29.112 | INFO | "gpu" : " 17 GB", | 20260505 07:19:29.112 | INFO | "huge" : "", | 20260505 07:19:29.112 | INFO | "huge_shared" : " 27 GB", | 20260505 07:19:29.112 | INFO | "malloc" : " 14 GB", | 20260505 07:19:29.112 | INFO | "pinned" : " 29 GB" | 20260505 07:19:29.112 | INFO | } | 20260505 07:19:29.112 | INFO | Perf report for the last 5.14115 seconds: | 20260505 07:19:29.112 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:19:29.113 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:29.113 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:29.113 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:29.113 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 07:19:29.113 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:29.113 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:29.113 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:19:29.113 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:29.113 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:29.113 | INFO | Uploaded 36GiB to GPU in 2.16191s (16.6519GiB/s) | 20260505 07:19:29.113 | INFO | Downloaded 34GiB to GPU in 2.05467s (16.5476GiB/s) | 20260505 07:19:34.250 | INFO | Analyzing chunk frames = [1362944,1363456) | 20260505 07:19:34.250 | INFO | MemInfo: { | 20260505 07:19:34.250 | INFO | "gpu" : " 17 GB", | 20260505 07:19:34.250 | INFO | "huge" : "", | 20260505 07:19:34.250 | INFO | "huge_shared" : " 27 GB", | 20260505 07:19:34.250 | INFO | "malloc" : " 14 GB", | 20260505 07:19:34.250 | INFO | "pinned" : " 29 GB" | 20260505 07:19:34.250 | INFO | } | 20260505 07:19:34.250 | INFO | Perf report for the last 5.13747 seconds: | 20260505 07:19:34.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:19:34.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:34.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:34.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:34.250 | INFO | ANALYSIS : 64.6%, 3.0, 100.9ms, 0 | 20260505 07:19:34.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:34.250 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:34.250 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260505 07:19:34.250 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:34.250 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:34.250 | INFO | Uploaded 36GiB to GPU in 2.23135s (16.1337GiB/s) | 20260505 07:19:34.250 | INFO | Downloaded 34GiB to GPU in 2.15694s (15.763GiB/s) | 20260505 07:19:39.390 | INFO | Analyzing chunk frames = [1363456,1363968) | 20260505 07:19:39.390 | INFO | MemInfo: { | 20260505 07:19:39.390 | INFO | "gpu" : " 17 GB", | 20260505 07:19:39.390 | INFO | "huge" : "", | 20260505 07:19:39.390 | INFO | "huge_shared" : " 27 GB", | 20260505 07:19:39.390 | INFO | "malloc" : " 14 GB", | 20260505 07:19:39.390 | INFO | "pinned" : " 29 GB" | 20260505 07:19:39.390 | INFO | } | 20260505 07:19:39.390 | INFO | Perf report for the last 5.13974 seconds: | 20260505 07:19:39.390 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:19:39.390 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:39.390 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:39.390 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:39.390 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 07:19:39.390 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:39.390 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:39.390 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 07:19:39.390 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:39.390 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:39.390 | INFO | Uploaded 36GiB to GPU in 2.1723s (16.5723GiB/s) | 20260505 07:19:39.390 | INFO | Downloaded 34GiB to GPU in 2.03345s (16.7203GiB/s) | 20260505 07:19:39.510 | INFO | Starting a round of Baz IO | 20260505 07:19:42.823 | INFO | Populating RTMetrics report | 20260505 07:19:42.823 | INFO | RTMetrics report populated | 20260505 07:19:42.823 | INFO | Opening tmp RTMetrics json file | 20260505 07:19:42.857 | INFO | Serializing RTMetrics to json | 20260505 07:19:42.882 | INFO | Serialization done, swapping tmp file now... | 20260505 07:19:42.882 | INFO | RTMetrics done | 20260505 07:19:42.947 | INFO | Finished a round of Baz IO: Wrote 2.426 GiB in 3.437 seconds | 20260505 07:19:42.947 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 30.23% / 0.03% / 69.74% | 20260505 07:19:42.947 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.77999% / 6.75519e-05% / 8.34465e-05% | 20260505 07:19:42.947 | INFO | Baz IO overall: | 20260505 07:19:42.947 | INFO | BytesWritten : 739307990788 | 20260505 07:19:42.947 | INFO | NumEvents : 350646348898 | 20260505 07:19:44.529 | INFO | Analyzing chunk frames = [1363968,1364480) | 20260505 07:19:44.529 | INFO | MemInfo: { | 20260505 07:19:44.529 | INFO | "gpu" : " 17 GB", | 20260505 07:19:44.529 | INFO | "huge" : "", | 20260505 07:19:44.529 | INFO | "huge_shared" : " 27 GB", | 20260505 07:19:44.529 | INFO | "malloc" : " 14 GB", | 20260505 07:19:44.529 | INFO | "pinned" : " 29 GB" | 20260505 07:19:44.529 | INFO | } | 20260505 07:19:44.529 | INFO | Perf report for the last 5.13921 seconds: | 20260505 07:19:44.529 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:19:44.529 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:44.529 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:44.529 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:44.529 | INFO | ANALYSIS : 65.6%, 2.9, 99.8ms, 0 | 20260505 07:19:44.529 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:44.529 | INFO | RT_METRICS : 47.1%, 1.0, 24.7ms, 0 | 20260505 07:19:44.529 | INFO | PRE_HQ : 64.4%, 1.9, 62.4ms, 0 | 20260505 07:19:44.529 | INFO | BAZWRITER : 66.1%, 4.1, 141.1ms, 0 | 20260505 07:19:44.529 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 07:19:44.529 | INFO | Uploaded 36GiB to GPU in 2.26384s (15.9022GiB/s) | 20260505 07:19:44.529 | INFO | Downloaded 38GiB to GPU in 2.35141s (16.1605GiB/s) | 20260505 07:19:49.668 | INFO | Analyzing chunk frames = [1364480,1364992) | 20260505 07:19:49.668 | INFO | MemInfo: { | 20260505 07:19:49.668 | INFO | "gpu" : " 17 GB", | 20260505 07:19:49.668 | INFO | "huge" : "", | 20260505 07:19:49.668 | INFO | "huge_shared" : " 27 GB", | 20260505 07:19:49.668 | INFO | "malloc" : " 14 GB", | 20260505 07:19:49.668 | INFO | "pinned" : " 29 GB" | 20260505 07:19:49.668 | INFO | } | 20260505 07:19:49.668 | INFO | Perf report for the last 5.13893 seconds: | 20260505 07:19:49.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:19:49.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:49.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:49.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:49.668 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 07:19:49.668 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:49.668 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:49.668 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:19:49.668 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:49.668 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:49.668 | INFO | Uploaded 36GiB to GPU in 2.10974s (17.0637GiB/s) | 20260505 07:19:49.668 | INFO | Downloaded 34GiB to GPU in 2.04071s (16.6608GiB/s) | 20260505 07:19:54.807 | INFO | Analyzing chunk frames = [1364992,1365504) | 20260505 07:19:54.807 | INFO | MemInfo: { | 20260505 07:19:54.807 | INFO | "gpu" : " 17 GB", | 20260505 07:19:54.807 | INFO | "huge" : "", | 20260505 07:19:54.807 | INFO | "huge_shared" : " 27 GB", | 20260505 07:19:54.807 | INFO | "malloc" : " 14 GB", | 20260505 07:19:54.807 | INFO | "pinned" : " 29 GB" | 20260505 07:19:54.807 | INFO | } | 20260505 07:19:54.807 | INFO | Perf report for the last 5.13934 seconds: | 20260505 07:19:54.807 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:19:54.807 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:54.807 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:54.807 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:54.807 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260505 07:19:54.807 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:54.807 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:54.807 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:19:54.807 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:54.807 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:54.807 | INFO | Uploaded 36GiB to GPU in 2.1452s (16.7816GiB/s) | 20260505 07:19:54.808 | INFO | Downloaded 34GiB to GPU in 2.05935s (16.51GiB/s) | 20260505 07:19:59.946 | INFO | Analyzing chunk frames = [1365504,1366016) | 20260505 07:19:59.946 | INFO | MemInfo: { | 20260505 07:19:59.946 | INFO | "gpu" : " 17 GB", | 20260505 07:19:59.946 | INFO | "huge" : "", | 20260505 07:19:59.946 | INFO | "huge_shared" : " 27 GB", | 20260505 07:19:59.946 | INFO | "malloc" : " 14 GB", | 20260505 07:19:59.946 | INFO | "pinned" : " 29 GB" | 20260505 07:19:59.946 | INFO | } | 20260505 07:19:59.946 | INFO | Perf report for the last 5.1392 seconds: | 20260505 07:19:59.946 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:19:59.946 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:59.946 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:59.946 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:59.946 | INFO | ANALYSIS : 63.4%, 3.0, 98.5ms, 0 | 20260505 07:19:59.946 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:59.946 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:59.946 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 07:19:59.946 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:59.947 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:19:59.947 | INFO | Uploaded 36GiB to GPU in 2.13126s (16.8914GiB/s) | 20260505 07:19:59.947 | INFO | Downloaded 34GiB to GPU in 2.02737s (16.7705GiB/s) | 20260505 07:20:05.086 | INFO | Analyzing chunk frames = [1366016,1366528) | 20260505 07:20:05.086 | INFO | MemInfo: { | 20260505 07:20:05.086 | INFO | "gpu" : " 17 GB", | 20260505 07:20:05.086 | INFO | "huge" : "", | 20260505 07:20:05.086 | INFO | "huge_shared" : " 27 GB", | 20260505 07:20:05.086 | INFO | "malloc" : " 14 GB", | 20260505 07:20:05.086 | INFO | "pinned" : " 29 GB" | 20260505 07:20:05.086 | INFO | } | 20260505 07:20:05.086 | INFO | Perf report for the last 5.13951 seconds: | 20260505 07:20:05.086 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:20:05.086 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:05.086 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:05.086 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:05.086 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 07:20:05.086 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:05.086 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:05.086 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:20:05.086 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:05.086 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:05.086 | INFO | Uploaded 36GiB to GPU in 2.15873s (16.6764GiB/s) | 20260505 07:20:05.086 | INFO | Downloaded 34GiB to GPU in 2.04852s (16.5974GiB/s) | 20260505 07:20:10.227 | INFO | Analyzing chunk frames = [1366528,1367040) | 20260505 07:20:10.227 | INFO | MemInfo: { | 20260505 07:20:10.227 | INFO | "gpu" : " 17 GB", | 20260505 07:20:10.227 | INFO | "huge" : "", | 20260505 07:20:10.227 | INFO | "huge_shared" : " 27 GB", | 20260505 07:20:10.227 | INFO | "malloc" : " 14 GB", | 20260505 07:20:10.227 | INFO | "pinned" : " 29 GB" | 20260505 07:20:10.227 | INFO | } | 20260505 07:20:10.227 | INFO | Perf report for the last 5.14114 seconds: | 20260505 07:20:10.227 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:20:10.227 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:10.227 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:10.227 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:10.227 | INFO | ANALYSIS : 63.5%, 3.0, 99.3ms, 0 | 20260505 07:20:10.227 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:10.227 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:10.227 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 07:20:10.227 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:10.227 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:10.227 | INFO | Uploaded 36GiB to GPU in 2.16065s (16.6616GiB/s) | 20260505 07:20:10.227 | INFO | Downloaded 34GiB to GPU in 2.05313s (16.5601GiB/s) | 20260505 07:20:15.364 | INFO | Analyzing chunk frames = [1367040,1367552) | 20260505 07:20:15.364 | INFO | MemInfo: { | 20260505 07:20:15.364 | INFO | "gpu" : " 17 GB", | 20260505 07:20:15.365 | INFO | "huge" : "", | 20260505 07:20:15.365 | INFO | "huge_shared" : " 27 GB", | 20260505 07:20:15.365 | INFO | "malloc" : " 14 GB", | 20260505 07:20:15.365 | INFO | "pinned" : " 29 GB" | 20260505 07:20:15.365 | INFO | } | 20260505 07:20:15.365 | INFO | Perf report for the last 5.1375 seconds: | 20260505 07:20:15.365 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:20:15.365 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:15.365 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:15.365 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:15.365 | INFO | ANALYSIS : 64.7%, 3.0, 101.0ms, 0 | 20260505 07:20:15.365 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:15.365 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:15.365 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 07:20:15.365 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:15.365 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:15.365 | INFO | Uploaded 36GiB to GPU in 2.22372s (16.1891GiB/s) | 20260505 07:20:15.365 | INFO | Downloaded 34GiB to GPU in 2.14066s (15.883GiB/s) | 20260505 07:20:20.504 | INFO | Analyzing chunk frames = [1367552,1368064) | 20260505 07:20:20.504 | INFO | MemInfo: { | 20260505 07:20:20.504 | INFO | "gpu" : " 17 GB", | 20260505 07:20:20.504 | INFO | "huge" : "", | 20260505 07:20:20.504 | INFO | "huge_shared" : " 27 GB", | 20260505 07:20:20.504 | INFO | "malloc" : " 14 GB", | 20260505 07:20:20.504 | INFO | "pinned" : " 29 GB" | 20260505 07:20:20.504 | INFO | } | 20260505 07:20:20.504 | INFO | Perf report for the last 5.13924 seconds: | 20260505 07:20:20.504 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:20:20.504 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:20.504 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:20.504 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:20.504 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 07:20:20.504 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:20.504 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:20.504 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 07:20:20.504 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:20.504 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:20.504 | INFO | Uploaded 36GiB to GPU in 2.16356s (16.6393GiB/s) | 20260505 07:20:20.504 | INFO | Downloaded 34GiB to GPU in 2.0289s (16.7578GiB/s) | 20260505 07:20:20.504 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 07:20:20.504 | INFO | Total mem usage: 15.3 GiB | 20260505 07:20:20.504 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 07:20:20.504 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -5.93%) | 20260505 07:20:20.504 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 07:20:20.504 | INFO | High Water Breakdown: | 20260505 07:20:20.504 | INFO | 60.7% : 8.78 GiB ( 93.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 07:20:20.504 | INFO | 39.2% : 5.67 GiB ( 85.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 07:20:20.504 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 07:20:20.504 | INFO | ------------------------------------------ | 20260505 07:20:20.504 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:20:20.504 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 07:20:20.504 | INFO | Total mem usage: 26.5 GiB | 20260505 07:20:20.504 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 07:20:20.504 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 07:20:20.504 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 07:20:20.504 | INFO | High Water Breakdown: | 20260505 07:20:20.504 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:20:20.504 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:20:20.504 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:20:20.504 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:20:20.504 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:20:20.504 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:20:20.504 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 07:20:20.504 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 07:20:20.504 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:20:20.504 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 07:20:20.504 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 07:20:20.504 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 07:20:20.505 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 07:20:20.505 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 07:20:20.505 | INFO | ------------------------------------------ | 20260505 07:20:20.505 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:20:20.505 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 07:20:20.505 | INFO | Total mem usage: 25.6 GiB | 20260505 07:20:20.505 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 07:20:20.505 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 07:20:20.505 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 07:20:20.505 | INFO | High Water Breakdown: | 20260505 07:20:20.505 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 07:20:20.505 | INFO | ------------------------------------------ | 20260505 07:20:20.505 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:20:20.505 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 07:20:20.505 | INFO | Total mem usage: 15.7 GiB | 20260505 07:20:20.505 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 07:20:20.505 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 07:20:20.505 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260505 07:20:20.505 | INFO | High Water Breakdown: | 20260505 07:20:20.505 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 07:20:20.505 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 07:20:20.505 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 07:20:20.505 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 07:20:20.505 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 07:20:20.505 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 07:20:20.505 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 07:20:20.505 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 07:20:20.505 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 07:20:20.505 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:20:20.505 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 07:20:20.505 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:20:20.505 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 07:20:20.505 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 07:20:20.505 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 07:20:20.505 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 07:20:20.505 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:20:20.505 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 07:20:20.505 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:20:20.505 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:20:20.505 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:20:20.505 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 07:20:20.505 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 07:20:20.505 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:20:20.505 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 07:20:20.505 | INFO | ------------------------------------------ | 20260505 07:20:20.505 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:20:20.505 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 07:20:20.505 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:20:20.505 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:20:20.505 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.54e+01, 1.37e+01 / 2.04e+00 | 20260505 07:20:20.505 | INFO | PerfReport: 0.28%, AggregateStats -- 9.32e-02 / 1.02e-01, 1.08e-02 / 6.51e-03 | 20260505 07:20:20.505 | INFO | PerfReport: 11.61%, Baseline -- 3.82e+00 / 4.46e+00, 6.16e-01 / 2.58e-01 | 20260505 07:20:20.505 | INFO | PerfReport: 3.30%, Histogram -- 1.09e+00 / 1.47e+00, 2.11e-01 / 1.32e-01 | 20260505 07:20:20.505 | INFO | PerfReport: 15.49%, DME -- 4.08e+01 / 4.46e+01, 4.17e+00 / 1.33e+00 | 20260505 07:20:20.505 | INFO | PerfReport: 56.25%, FrameLabeling -- 1.85e+01 / 1.93e+01, 2.04e+00 / 1.49e+00 | 20260505 07:20:20.505 | INFO | PerfReport: 7.61%, PulseAccumulating -- 2.51e+00 / 3.15e+00, 4.35e-01 / 2.25e-01 | 20260505 07:20:20.505 | INFO | PerfReport: 4.00%, Metrics -- 1.32e+00 / 1.73e+00, 3.65e-01 / 1.42e-01 | 20260505 07:20:20.505 | INFO | PerfReport: 1.44%, PulseSerializing -- 4.75e-01 / 5.56e-01, 9.02e-02 / 2.66e-02 | 20260505 07:20:20.505 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 07:20:20.505 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:20:20.505 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:20:20.505 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.22e+01 / 0.00e+00 | 20260505 07:20:20.505 | INFO | PerfReport: 22.28%, Upload -- 2.26e+01 / 0.00e+00, 7.98e+00 / 0.00e+00 | 20260505 07:20:20.505 | INFO | PerfReport: 21.51%, Download -- 2.18e+01 / 0.00e+00, 7.75e+00 / 0.00e+00 | 20260505 07:20:20.505 | INFO | PerfReport: 32.43%, Compute -- 3.29e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 07:20:20.505 | INFO | PerfReport: 1.50%, LockUpload -- 1.52e+00 / 0.00e+00, 7.04e+00 / 0.00e+00 | 20260505 07:20:20.505 | INFO | PerfReport: 0.83%, LockDownload -- 8.39e-01 / 0.00e+00, 2.26e+00 / 0.00e+00 | 20260505 07:20:20.505 | INFO | PerfReport: 21.44%, LockCompute -- 2.18e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260505 07:20:20.505 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 1.72e-02 / 0.00e+00 | 20260505 07:20:20.626 | INFO | Starting a round of Baz IO | 20260505 07:20:23.930 | INFO | Populating RTMetrics report | 20260505 07:20:23.930 | INFO | RTMetrics report populated | 20260505 07:20:23.930 | INFO | Opening tmp RTMetrics json file | 20260505 07:20:23.948 | INFO | Serializing RTMetrics to json | 20260505 07:20:23.949 | INFO | Serialization done, swapping tmp file now... | 20260505 07:20:23.949 | INFO | RTMetrics done | 20260505 07:20:24.071 | INFO | Finished a round of Baz IO: Wrote 2.427 GiB in 3.445 seconds | 20260505 07:20:24.071 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 30.20% / 0.03% / 69.77% | 20260505 07:20:24.071 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.76759% / 7.15256e-05% / 8.74201e-05% | 20260505 07:20:24.071 | INFO | Baz IO overall: | 20260505 07:20:24.071 | INFO | BytesWritten : 741914102862 | 20260505 07:20:24.071 | INFO | NumEvents : 351893683656 | 20260505 07:20:25.644 | INFO | Analyzing chunk frames = [1368064,1368576) | 20260505 07:20:25.644 | INFO | MemInfo: { | 20260505 07:20:25.644 | INFO | "gpu" : " 17 GB", | 20260505 07:20:25.644 | INFO | "huge" : "", | 20260505 07:20:25.644 | INFO | "huge_shared" : " 27 GB", | 20260505 07:20:25.644 | INFO | "malloc" : " 14 GB", | 20260505 07:20:25.644 | INFO | "pinned" : " 29 GB" | 20260505 07:20:25.644 | INFO | } | 20260505 07:20:25.644 | INFO | Perf report for the last 5.14013 seconds: | 20260505 07:20:25.644 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:20:25.644 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:25.644 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:25.644 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:25.644 | INFO | ANALYSIS : 65.5%, 2.9, 99.7ms, 0 | 20260505 07:20:25.644 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:25.644 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260505 07:20:25.644 | INFO | PRE_HQ : 64.3%, 1.9, 62.7ms, 0 | 20260505 07:20:25.644 | INFO | BAZWRITER : 66.2%, 4.1, 139.8ms, 0 | 20260505 07:20:25.644 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 07:20:25.644 | INFO | Uploaded 36GiB to GPU in 2.24906s (16.0067GiB/s) | 20260505 07:20:25.644 | INFO | Downloaded 38GiB to GPU in 2.34171s (16.2274GiB/s) | 20260505 07:20:30.783 | INFO | Analyzing chunk frames = [1368576,1369088) | 20260505 07:20:30.783 | INFO | MemInfo: { | 20260505 07:20:30.783 | INFO | "gpu" : " 17 GB", | 20260505 07:20:30.783 | INFO | "huge" : "", | 20260505 07:20:30.783 | INFO | "huge_shared" : " 27 GB", | 20260505 07:20:30.783 | INFO | "malloc" : " 14 GB", | 20260505 07:20:30.783 | INFO | "pinned" : " 29 GB" | 20260505 07:20:30.783 | INFO | } | 20260505 07:20:30.783 | INFO | Perf report for the last 5.13896 seconds: | 20260505 07:20:30.783 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:20:30.783 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:30.783 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:30.783 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:30.783 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 07:20:30.783 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:30.783 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:30.783 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:20:30.783 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:30.783 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:30.783 | INFO | Uploaded 36GiB to GPU in 2.1257s (16.9356GiB/s) | 20260505 07:20:30.783 | INFO | Downloaded 34GiB to GPU in 2.04207s (16.6498GiB/s) | 20260505 07:20:35.922 | INFO | Analyzing chunk frames = [1369088,1369600) | 20260505 07:20:35.922 | INFO | MemInfo: { | 20260505 07:20:35.922 | INFO | "gpu" : " 17 GB", | 20260505 07:20:35.922 | INFO | "huge" : "", | 20260505 07:20:35.922 | INFO | "huge_shared" : " 27 GB", | 20260505 07:20:35.922 | INFO | "malloc" : " 14 GB", | 20260505 07:20:35.922 | INFO | "pinned" : " 29 GB" | 20260505 07:20:35.922 | INFO | } | 20260505 07:20:35.922 | INFO | Perf report for the last 5.1389 seconds: | 20260505 07:20:35.922 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:20:35.922 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:35.922 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:35.922 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:35.922 | INFO | ANALYSIS : 63.6%, 3.0, 98.9ms, 0 | 20260505 07:20:35.922 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:35.922 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:35.922 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:20:35.922 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:35.922 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:35.922 | INFO | Uploaded 36GiB to GPU in 2.15692s (16.6904GiB/s) | 20260505 07:20:35.922 | INFO | Downloaded 34GiB to GPU in 2.04896s (16.5937GiB/s) | 20260505 07:20:41.061 | INFO | Analyzing chunk frames = [1369600,1370112) | 20260505 07:20:41.061 | INFO | MemInfo: { | 20260505 07:20:41.061 | INFO | "gpu" : " 17 GB", | 20260505 07:20:41.061 | INFO | "huge" : "", | 20260505 07:20:41.061 | INFO | "huge_shared" : " 27 GB", | 20260505 07:20:41.061 | INFO | "malloc" : " 14 GB", | 20260505 07:20:41.061 | INFO | "pinned" : " 29 GB" | 20260505 07:20:41.061 | INFO | } | 20260505 07:20:41.061 | INFO | Perf report for the last 5.13928 seconds: | 20260505 07:20:41.061 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:20:41.061 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:41.061 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:41.061 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:41.061 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 07:20:41.061 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:41.061 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:41.061 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:20:41.061 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:41.061 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:41.061 | INFO | Uploaded 36GiB to GPU in 2.12229s (16.9628GiB/s) | 20260505 07:20:41.062 | INFO | Downloaded 34GiB to GPU in 2.03054s (16.7443GiB/s) | 20260505 07:20:46.201 | INFO | Analyzing chunk frames = [1370112,1370624) | 20260505 07:20:46.201 | INFO | MemInfo: { | 20260505 07:20:46.201 | INFO | "gpu" : " 17 GB", | 20260505 07:20:46.201 | INFO | "huge" : "", | 20260505 07:20:46.201 | INFO | "huge_shared" : " 27 GB", | 20260505 07:20:46.201 | INFO | "malloc" : " 14 GB", | 20260505 07:20:46.201 | INFO | "pinned" : " 29 GB" | 20260505 07:20:46.201 | INFO | } | 20260505 07:20:46.201 | INFO | Perf report for the last 5.13971 seconds: | 20260505 07:20:46.201 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:20:46.201 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:46.201 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:46.201 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:46.201 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 07:20:46.201 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:46.201 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:46.201 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:20:46.201 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:46.201 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:46.201 | INFO | Uploaded 36GiB to GPU in 2.16086s (16.66GiB/s) | 20260505 07:20:46.201 | INFO | Downloaded 34GiB to GPU in 2.05017s (16.584GiB/s) | 20260505 07:20:51.342 | INFO | Analyzing chunk frames = [1370624,1371136) | 20260505 07:20:51.342 | INFO | MemInfo: { | 20260505 07:20:51.342 | INFO | "gpu" : " 17 GB", | 20260505 07:20:51.342 | INFO | "huge" : "", | 20260505 07:20:51.342 | INFO | "huge_shared" : " 27 GB", | 20260505 07:20:51.342 | INFO | "malloc" : " 14 GB", | 20260505 07:20:51.342 | INFO | "pinned" : " 29 GB" | 20260505 07:20:51.342 | INFO | } | 20260505 07:20:51.342 | INFO | Perf report for the last 5.14087 seconds: | 20260505 07:20:51.342 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:20:51.342 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:51.342 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:51.342 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:51.342 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 07:20:51.342 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:51.342 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:51.342 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:20:51.342 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:51.342 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:51.342 | INFO | Uploaded 36GiB to GPU in 2.15897s (16.6746GiB/s) | 20260505 07:20:51.342 | INFO | Downloaded 34GiB to GPU in 2.05883s (16.5142GiB/s) | 20260505 07:20:56.479 | INFO | Analyzing chunk frames = [1371136,1371648) | 20260505 07:20:56.479 | INFO | MemInfo: { | 20260505 07:20:56.479 | INFO | "gpu" : " 17 GB", | 20260505 07:20:56.483 | INFO | "huge" : "", | 20260505 07:20:56.483 | INFO | "huge_shared" : " 27 GB", | 20260505 07:20:56.483 | INFO | "malloc" : " 14 GB", | 20260505 07:20:56.483 | INFO | "pinned" : " 29 GB" | 20260505 07:20:56.483 | INFO | } | 20260505 07:20:56.483 | INFO | Perf report for the last 5.14136 seconds: | 20260505 07:20:56.483 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:20:56.483 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:56.483 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:56.483 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:56.483 | INFO | ANALYSIS : 64.5%, 3.0, 100.8ms, 0 | 20260505 07:20:56.483 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:56.483 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:56.483 | INFO | PRE_HQ : 31.8%, 1.0, 16.6ms, 0 | 20260505 07:20:56.483 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:56.483 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:20:56.483 | INFO | Uploaded 36GiB to GPU in 2.23245s (16.1258GiB/s) | 20260505 07:20:56.484 | INFO | Downloaded 34GiB to GPU in 2.13739s (15.9073GiB/s) | 20260505 07:21:01.618 | INFO | Analyzing chunk frames = [1371648,1372160) | 20260505 07:21:01.618 | INFO | MemInfo: { | 20260505 07:21:01.618 | INFO | "gpu" : " 17 GB", | 20260505 07:21:01.618 | INFO | "huge" : "", | 20260505 07:21:01.618 | INFO | "huge_shared" : " 27 GB", | 20260505 07:21:01.618 | INFO | "malloc" : " 14 GB", | 20260505 07:21:01.618 | INFO | "pinned" : " 29 GB" | 20260505 07:21:01.618 | INFO | } | 20260505 07:21:01.619 | INFO | Perf report for the last 5.1354 seconds: | 20260505 07:21:01.619 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:21:01.619 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:01.619 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:01.619 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:01.619 | INFO | ANALYSIS : 64.2%, 3.0, 100.1ms, 0 | 20260505 07:21:01.619 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:01.619 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:01.619 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 07:21:01.619 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:01.619 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:01.619 | INFO | Uploaded 36GiB to GPU in 2.18504s (16.4757GiB/s) | 20260505 07:21:01.619 | INFO | Downloaded 34GiB to GPU in 2.04435s (16.6312GiB/s) | 20260505 07:21:01.753 | INFO | Starting a round of Baz IO | 20260505 07:21:05.022 | INFO | Populating RTMetrics report | 20260505 07:21:05.023 | INFO | RTMetrics report populated | 20260505 07:21:05.023 | INFO | Opening tmp RTMetrics json file | 20260505 07:21:05.041 | INFO | Serializing RTMetrics to json | 20260505 07:21:05.085 | INFO | Serialization done, swapping tmp file now... | 20260505 07:21:05.088 | INFO | RTMetrics done | 20260505 07:21:05.156 | INFO | Finished a round of Baz IO: Wrote 2.424 GiB in 3.403 seconds | 20260505 07:21:05.156 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 30.16% / 0.03% / 69.80% | 20260505 07:21:05.156 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.73875% / 5.5631e-05% / 7.94729e-05% | 20260505 07:21:05.156 | INFO | Baz IO overall: | 20260505 07:21:05.156 | INFO | BytesWritten : 744516560177 | 20260505 07:21:05.156 | INFO | NumEvents : 353137915475 | 20260505 07:21:06.758 | INFO | Analyzing chunk frames = [1372160,1372672) | 20260505 07:21:06.758 | INFO | MemInfo: { | 20260505 07:21:06.758 | INFO | "gpu" : " 17 GB", | 20260505 07:21:06.758 | INFO | "huge" : "", | 20260505 07:21:06.758 | INFO | "huge_shared" : " 27 GB", | 20260505 07:21:06.758 | INFO | "malloc" : " 14 GB", | 20260505 07:21:06.758 | INFO | "pinned" : " 29 GB" | 20260505 07:21:06.758 | INFO | } | 20260505 07:21:06.758 | INFO | Perf report for the last 5.1394 seconds: | 20260505 07:21:06.758 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:21:06.758 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:06.758 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:06.758 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:06.758 | INFO | ANALYSIS : 65.1%, 2.9, 99.4ms, 0 | 20260505 07:21:06.758 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:06.758 | INFO | RT_METRICS : 47.2%, 1.0, 24.8ms, 0 | 20260505 07:21:06.758 | INFO | PRE_HQ : 63.8%, 1.9, 62.8ms, 0 | 20260505 07:21:06.758 | INFO | BAZWRITER : 65.5%, 4.2, 140.6ms, 0 | 20260505 07:21:06.758 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 07:21:06.758 | INFO | Uploaded 36GiB to GPU in 2.21024s (16.2878GiB/s) | 20260505 07:21:06.758 | INFO | Downloaded 38GiB to GPU in 2.31746s (16.3973GiB/s) | 20260505 07:21:11.897 | INFO | Analyzing chunk frames = [1372672,1373184) | 20260505 07:21:11.897 | INFO | MemInfo: { | 20260505 07:21:11.897 | INFO | "gpu" : " 17 GB", | 20260505 07:21:11.897 | INFO | "huge" : "", | 20260505 07:21:11.897 | INFO | "huge_shared" : " 27 GB", | 20260505 07:21:11.897 | INFO | "malloc" : " 14 GB", | 20260505 07:21:11.897 | INFO | "pinned" : " 29 GB" | 20260505 07:21:11.897 | INFO | } | 20260505 07:21:11.897 | INFO | Perf report for the last 5.13936 seconds: | 20260505 07:21:11.897 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:21:11.897 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:11.897 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:11.897 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:11.897 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 07:21:11.897 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:11.897 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:11.897 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:21:11.897 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:11.897 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:11.898 | INFO | Uploaded 36GiB to GPU in 2.13721s (16.8444GiB/s) | 20260505 07:21:11.898 | INFO | Downloaded 34GiB to GPU in 2.0387s (16.6773GiB/s) | 20260505 07:21:17.036 | INFO | Analyzing chunk frames = [1373184,1373696) | 20260505 07:21:17.036 | INFO | MemInfo: { | 20260505 07:21:17.036 | INFO | "gpu" : " 17 GB", | 20260505 07:21:17.036 | INFO | "huge" : "", | 20260505 07:21:17.037 | INFO | "huge_shared" : " 27 GB", | 20260505 07:21:17.037 | INFO | "malloc" : " 14 GB", | 20260505 07:21:17.037 | INFO | "pinned" : " 29 GB" | 20260505 07:21:17.037 | INFO | } | 20260505 07:21:17.037 | INFO | Perf report for the last 5.13929 seconds: | 20260505 07:21:17.037 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:21:17.037 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:17.037 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:17.037 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:17.037 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260505 07:21:17.037 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:17.037 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:17.037 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 07:21:17.037 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:17.037 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:17.037 | INFO | Uploaded 36GiB to GPU in 2.15227s (16.7265GiB/s) | 20260505 07:21:17.037 | INFO | Downloaded 34GiB to GPU in 2.05476s (16.5469GiB/s) | 20260505 07:21:22.176 | INFO | Analyzing chunk frames = [1373696,1374208) | 20260505 07:21:22.176 | INFO | MemInfo: { | 20260505 07:21:22.176 | INFO | "gpu" : " 17 GB", | 20260505 07:21:22.176 | INFO | "huge" : "", | 20260505 07:21:22.176 | INFO | "huge_shared" : " 27 GB", | 20260505 07:21:22.176 | INFO | "malloc" : " 14 GB", | 20260505 07:21:22.176 | INFO | "pinned" : " 29 GB" | 20260505 07:21:22.176 | INFO | } | 20260505 07:21:22.176 | INFO | Perf report for the last 5.13943 seconds: | 20260505 07:21:22.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:21:22.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:22.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:22.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:22.176 | INFO | ANALYSIS : 63.5%, 3.0, 98.6ms, 0 | 20260505 07:21:22.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:22.176 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:22.176 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 07:21:22.176 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:22.176 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:22.176 | INFO | Uploaded 36GiB to GPU in 2.13228s (16.8833GiB/s) | 20260505 07:21:22.176 | INFO | Downloaded 34GiB to GPU in 2.03612s (16.6985GiB/s) | 20260505 07:21:27.319 | INFO | Analyzing chunk frames = [1374208,1374720) | 20260505 07:21:27.319 | INFO | MemInfo: { | 20260505 07:21:27.319 | INFO | "gpu" : " 17 GB", | 20260505 07:21:27.319 | INFO | "huge" : "", | 20260505 07:21:27.319 | INFO | "huge_shared" : " 27 GB", | 20260505 07:21:27.319 | INFO | "malloc" : " 14 GB", | 20260505 07:21:27.319 | INFO | "pinned" : " 29 GB" | 20260505 07:21:27.319 | INFO | } | 20260505 07:21:27.319 | INFO | Perf report for the last 5.14309 seconds: | 20260505 07:21:27.319 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:21:27.319 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:27.319 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:27.319 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:27.319 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 07:21:27.319 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:27.319 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:27.319 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 07:21:27.319 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:27.319 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:27.319 | INFO | Uploaded 36GiB to GPU in 2.16045s (16.6632GiB/s) | 20260505 07:21:27.319 | INFO | Downloaded 34GiB to GPU in 2.05173s (16.5714GiB/s) | 20260505 07:21:32.456 | INFO | Analyzing chunk frames = [1374720,1375232) | 20260505 07:21:32.456 | INFO | MemInfo: { | 20260505 07:21:32.456 | INFO | "gpu" : " 17 GB", | 20260505 07:21:32.456 | INFO | "huge" : "", | 20260505 07:21:32.456 | INFO | "huge_shared" : " 27 GB", | 20260505 07:21:32.456 | INFO | "malloc" : " 14 GB", | 20260505 07:21:32.456 | INFO | "pinned" : " 29 GB" | 20260505 07:21:32.456 | INFO | } | 20260505 07:21:32.456 | INFO | Perf report for the last 5.13711 seconds: | 20260505 07:21:32.457 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:21:32.457 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:32.457 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:32.457 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:32.457 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 07:21:32.457 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:32.457 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:32.457 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 07:21:32.457 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:32.457 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:32.457 | INFO | Uploaded 36GiB to GPU in 2.16092s (16.6595GiB/s) | 20260505 07:21:32.457 | INFO | Downloaded 34GiB to GPU in 2.05648s (16.5331GiB/s) | 20260505 07:21:37.594 | INFO | Analyzing chunk frames = [1375232,1375744) | 20260505 07:21:37.594 | INFO | MemInfo: { | 20260505 07:21:37.594 | INFO | "gpu" : " 17 GB", | 20260505 07:21:37.594 | INFO | "huge" : "", | 20260505 07:21:37.594 | INFO | "huge_shared" : " 27 GB", | 20260505 07:21:37.594 | INFO | "malloc" : " 14 GB", | 20260505 07:21:37.594 | INFO | "pinned" : " 29 GB" | 20260505 07:21:37.594 | INFO | } | 20260505 07:21:37.594 | INFO | Perf report for the last 5.13794 seconds: | 20260505 07:21:37.594 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:21:37.594 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:37.594 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:37.594 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:37.594 | INFO | ANALYSIS : 64.7%, 3.0, 101.1ms, 0 | 20260505 07:21:37.594 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:37.594 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:37.594 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 07:21:37.594 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:37.594 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:37.595 | INFO | Uploaded 36GiB to GPU in 2.2236s (16.1899GiB/s) | 20260505 07:21:37.595 | INFO | Downloaded 34GiB to GPU in 2.14333s (15.8631GiB/s) | 20260505 07:21:42.733 | INFO | Analyzing chunk frames = [1375744,1376256) | 20260505 07:21:42.733 | INFO | MemInfo: { | 20260505 07:21:42.733 | INFO | "gpu" : " 17 GB", | 20260505 07:21:42.733 | INFO | "huge" : "", | 20260505 07:21:42.733 | INFO | "huge_shared" : " 27 GB", | 20260505 07:21:42.733 | INFO | "malloc" : " 14 GB", | 20260505 07:21:42.733 | INFO | "pinned" : " 29 GB" | 20260505 07:21:42.733 | INFO | } | 20260505 07:21:42.733 | INFO | Perf report for the last 5.13904 seconds: | 20260505 07:21:42.733 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:21:42.733 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:42.733 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:42.733 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:42.733 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 07:21:42.733 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:42.733 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:42.733 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 07:21:42.733 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:42.733 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:42.734 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 07:21:42.734 | INFO | Total mem usage: 15.3 GiB | 20260505 07:21:42.734 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 07:21:42.734 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.55%) | 20260505 07:21:42.734 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 07:21:42.734 | INFO | High Water Breakdown: | 20260505 07:21:42.734 | INFO | 60.5% : 8.7 GiB ( 93% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 07:21:42.734 | INFO | 39.4% : 5.67 GiB ( 85.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 07:21:42.734 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 07:21:42.734 | INFO | ------------------------------------------ | 20260505 07:21:42.734 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:21:42.734 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 07:21:42.734 | INFO | Total mem usage: 26.5 GiB | 20260505 07:21:42.734 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 07:21:42.734 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 07:21:42.734 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 07:21:42.734 | INFO | High Water Breakdown: | 20260505 07:21:42.734 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:21:42.734 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:21:42.734 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:21:42.734 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:21:42.734 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:21:42.734 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:21:42.734 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 07:21:42.734 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 07:21:42.734 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:21:42.734 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 07:21:42.734 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 07:21:42.734 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 07:21:42.734 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 07:21:42.734 | INFO | Uploaded 36GiB to GPU in 2.16705s (16.6124GiB/s) | 20260505 07:21:42.734 | INFO | Downloaded 34GiB to GPU in 2.02587s (16.7829GiB/s) | 20260505 07:21:42.734 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 07:21:42.734 | INFO | ------------------------------------------ | 20260505 07:21:42.734 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:21:42.734 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 07:21:42.734 | INFO | Total mem usage: 25.6 GiB | 20260505 07:21:42.734 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 07:21:42.734 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 07:21:42.734 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 07:21:42.734 | INFO | High Water Breakdown: | 20260505 07:21:42.734 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 07:21:42.734 | INFO | ------------------------------------------ | 20260505 07:21:42.734 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:21:42.734 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 07:21:42.734 | INFO | Total mem usage: 15.7 GiB | 20260505 07:21:42.734 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 07:21:42.735 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 07:21:42.735 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 07:21:42.735 | INFO | High Water Breakdown: | 20260505 07:21:42.735 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 07:21:42.735 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 07:21:42.735 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 07:21:42.735 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 07:21:42.735 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 07:21:42.735 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 07:21:42.735 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 07:21:42.735 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 07:21:42.735 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 07:21:42.735 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:21:42.735 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 07:21:42.735 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:21:42.735 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 07:21:42.735 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 07:21:42.735 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 07:21:42.735 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 07:21:42.735 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:21:42.735 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 07:21:42.735 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:21:42.735 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:21:42.735 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:21:42.735 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 07:21:42.735 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 07:21:42.735 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:21:42.735 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 07:21:42.735 | INFO | ------------------------------------------ | 20260505 07:21:42.735 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:21:42.735 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 07:21:42.735 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:21:42.735 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:21:42.735 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.53e+01, 1.36e+01 / 2.04e+00 | 20260505 07:21:42.735 | INFO | PerfReport: 0.28%, AggregateStats -- 9.36e-02 / 1.03e-01, 1.13e-02 / 8.73e-03 | 20260505 07:21:42.735 | INFO | PerfReport: 11.61%, Baseline -- 3.82e+00 / 4.42e+00, 6.25e-01 / 3.42e-01 | 20260505 07:21:42.735 | INFO | PerfReport: 3.29%, Histogram -- 1.08e+00 / 1.44e+00, 2.10e-01 / 1.44e-01 | 20260505 07:21:42.735 | INFO | PerfReport: 15.46%, DME -- 4.07e+01 / 4.45e+01, 4.15e+00 / 1.40e+00 | 20260505 07:21:42.735 | INFO | PerfReport: 56.31%, FrameLabeling -- 1.85e+01 / 1.93e+01, 2.05e+00 / 1.76e+00 | 20260505 07:21:42.735 | INFO | PerfReport: 7.61%, PulseAccumulating -- 2.50e+00 / 3.16e+00, 4.37e-01 / 2.63e-01 | 20260505 07:21:42.735 | INFO | PerfReport: 3.98%, Metrics -- 1.31e+00 / 1.73e+00, 3.62e-01 / 1.64e-01 | 20260505 07:21:42.735 | INFO | PerfReport: 1.44%, PulseSerializing -- 4.75e-01 / 5.48e-01, 9.06e-02 / 3.04e-02 | 20260505 07:21:42.735 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 07:21:42.735 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:21:42.735 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:21:42.735 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.21e+01 / 0.00e+00 | 20260505 07:21:42.735 | INFO | PerfReport: 22.29%, Upload -- 2.26e+01 / 0.00e+00, 8.02e+00 / 0.00e+00 | 20260505 07:21:42.735 | INFO | PerfReport: 21.48%, Download -- 2.18e+01 / 0.00e+00, 7.80e+00 / 0.00e+00 | 20260505 07:21:42.735 | INFO | PerfReport: 32.43%, Compute -- 3.29e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 07:21:42.735 | INFO | PerfReport: 1.53%, LockUpload -- 1.56e+00 / 0.00e+00, 7.13e+00 / 0.00e+00 | 20260505 07:21:42.735 | INFO | PerfReport: 0.83%, LockDownload -- 8.38e-01 / 0.00e+00, 2.20e+00 / 0.00e+00 | 20260505 07:21:42.735 | INFO | PerfReport: 21.42%, LockCompute -- 2.17e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260505 07:21:42.735 | INFO | PerfReport: 0.01%, Misc -- 1.51e-02 / 0.00e+00, 2.43e-02 / 0.00e+00 | 20260505 07:21:42.857 | INFO | Starting a round of Baz IO | 20260505 07:21:46.150 | INFO | Populating RTMetrics report | 20260505 07:21:46.150 | INFO | RTMetrics report populated | 20260505 07:21:46.150 | INFO | Opening tmp RTMetrics json file | 20260505 07:21:46.188 | INFO | Serializing RTMetrics to json | 20260505 07:21:46.208 | INFO | Serialization done, swapping tmp file now... | 20260505 07:21:46.243 | INFO | RTMetrics done | 20260505 07:21:46.288 | INFO | Finished a round of Baz IO: Wrote 2.428 GiB in 3.43 seconds | 20260505 07:21:46.288 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 30.13% / 0.03% / 69.84% | 20260505 07:21:46.288 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.7391% / 5.16574e-05% / 9.53674e-05% | 20260505 07:21:46.288 | INFO | Baz IO overall: | 20260505 07:21:46.288 | INFO | BytesWritten : 747123324456 | 20260505 07:21:46.288 | INFO | NumEvents : 354385373079 | 20260505 07:21:47.873 | INFO | Analyzing chunk frames = [1376256,1376768) | 20260505 07:21:47.873 | INFO | MemInfo: { | 20260505 07:21:47.873 | INFO | "gpu" : " 17 GB", | 20260505 07:21:47.873 | INFO | "huge" : "", | 20260505 07:21:47.873 | INFO | "huge_shared" : " 27 GB", | 20260505 07:21:47.873 | INFO | "malloc" : " 14 GB", | 20260505 07:21:47.873 | INFO | "pinned" : " 29 GB" | 20260505 07:21:47.873 | INFO | } | 20260505 07:21:47.873 | INFO | Perf report for the last 5.13947 seconds: | 20260505 07:21:47.873 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:21:47.873 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:47.873 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:47.873 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:47.873 | INFO | ANALYSIS : 65.4%, 2.9, 99.5ms, 0 | 20260505 07:21:47.873 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:47.873 | INFO | RT_METRICS : 47.8%, 1.0, 25.1ms, 0 | 20260505 07:21:47.873 | INFO | PRE_HQ : 64.2%, 1.9, 63.3ms, 0 | 20260505 07:21:47.873 | INFO | BAZWRITER : 65.9%, 4.2, 141.5ms, 0 | 20260505 07:21:47.873 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260505 07:21:47.873 | INFO | Uploaded 36GiB to GPU in 2.26362s (15.9037GiB/s) | 20260505 07:21:47.873 | INFO | Downloaded 38GiB to GPU in 2.32563s (16.3396GiB/s) | 20260505 07:21:53.012 | INFO | Analyzing chunk frames = [1376768,1377280) | 20260505 07:21:53.012 | INFO | MemInfo: { | 20260505 07:21:53.012 | INFO | "gpu" : " 17 GB", | 20260505 07:21:53.012 | INFO | "huge" : "", | 20260505 07:21:53.012 | INFO | "huge_shared" : " 27 GB", | 20260505 07:21:53.012 | INFO | "malloc" : " 14 GB", | 20260505 07:21:53.012 | INFO | "pinned" : " 29 GB" | 20260505 07:21:53.012 | INFO | } | 20260505 07:21:53.012 | INFO | Perf report for the last 5.13925 seconds: | 20260505 07:21:53.012 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:21:53.012 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:53.012 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:53.012 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:53.012 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 07:21:53.012 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:53.012 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:53.012 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:21:53.012 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:53.012 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:53.013 | INFO | Uploaded 36GiB to GPU in 2.12317s (16.9558GiB/s) | 20260505 07:21:53.013 | INFO | Downloaded 34GiB to GPU in 2.04194s (16.6508GiB/s) | 20260505 07:21:58.151 | INFO | Analyzing chunk frames = [1377280,1377792) | 20260505 07:21:58.151 | INFO | MemInfo: { | 20260505 07:21:58.151 | INFO | "gpu" : " 17 GB", | 20260505 07:21:58.151 | INFO | "huge" : "", | 20260505 07:21:58.151 | INFO | "huge_shared" : " 27 GB", | 20260505 07:21:58.151 | INFO | "malloc" : " 14 GB", | 20260505 07:21:58.151 | INFO | "pinned" : " 29 GB" | 20260505 07:21:58.151 | INFO | } | 20260505 07:21:58.151 | INFO | Perf report for the last 5.13924 seconds: | 20260505 07:21:58.151 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:21:58.152 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:58.152 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:58.152 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:58.152 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260505 07:21:58.152 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:58.152 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:58.152 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:21:58.152 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:58.152 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:21:58.152 | INFO | Uploaded 36GiB to GPU in 2.1512s (16.7348GiB/s) | 20260505 07:21:58.152 | INFO | Downloaded 34GiB to GPU in 2.05654s (16.5326GiB/s) | 20260505 07:22:03.291 | INFO | Analyzing chunk frames = [1377792,1378304) | 20260505 07:22:03.291 | INFO | MemInfo: { | 20260505 07:22:03.291 | INFO | "gpu" : " 17 GB", | 20260505 07:22:03.291 | INFO | "huge" : "", | 20260505 07:22:03.291 | INFO | "huge_shared" : " 27 GB", | 20260505 07:22:03.291 | INFO | "malloc" : " 14 GB", | 20260505 07:22:03.291 | INFO | "pinned" : " 29 GB" | 20260505 07:22:03.291 | INFO | } | 20260505 07:22:03.291 | INFO | Perf report for the last 5.13943 seconds: | 20260505 07:22:03.291 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:22:03.291 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:03.291 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:03.291 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:03.291 | INFO | ANALYSIS : 63.4%, 3.0, 98.5ms, 0 | 20260505 07:22:03.291 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:03.291 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:03.291 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 07:22:03.291 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:03.291 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:03.291 | INFO | Uploaded 36GiB to GPU in 2.12376s (16.951GiB/s) | 20260505 07:22:03.291 | INFO | Downloaded 34GiB to GPU in 2.02665s (16.7765GiB/s) | 20260505 07:22:08.430 | INFO | Analyzing chunk frames = [1378304,1378816) | 20260505 07:22:08.430 | INFO | MemInfo: { | 20260505 07:22:08.430 | INFO | "gpu" : " 17 GB", | 20260505 07:22:08.430 | INFO | "huge" : "", | 20260505 07:22:08.430 | INFO | "huge_shared" : " 27 GB", | 20260505 07:22:08.430 | INFO | "malloc" : " 14 GB", | 20260505 07:22:08.431 | INFO | "pinned" : " 29 GB" | 20260505 07:22:08.431 | INFO | } | 20260505 07:22:08.431 | INFO | Perf report for the last 5.13983 seconds: | 20260505 07:22:08.431 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:22:08.431 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:08.431 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:08.431 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:08.431 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 07:22:08.431 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:08.431 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:08.431 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:22:08.431 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:08.431 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:08.431 | INFO | Uploaded 36GiB to GPU in 2.15762s (16.6851GiB/s) | 20260505 07:22:08.431 | INFO | Downloaded 34GiB to GPU in 2.05045s (16.5818GiB/s) | 20260505 07:22:13.571 | INFO | Analyzing chunk frames = [1378816,1379328) | 20260505 07:22:13.571 | INFO | MemInfo: { | 20260505 07:22:13.571 | INFO | "gpu" : " 17 GB", | 20260505 07:22:13.571 | INFO | "huge" : "", | 20260505 07:22:13.571 | INFO | "huge_shared" : " 27 GB", | 20260505 07:22:13.571 | INFO | "malloc" : " 14 GB", | 20260505 07:22:13.571 | INFO | "pinned" : " 29 GB" | 20260505 07:22:13.571 | INFO | } | 20260505 07:22:13.571 | INFO | Perf report for the last 5.14064 seconds: | 20260505 07:22:13.571 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:22:13.571 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:13.571 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:13.571 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:13.571 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 07:22:13.571 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:13.571 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:13.571 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 07:22:13.571 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:13.571 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:13.571 | INFO | Uploaded 36GiB to GPU in 2.16238s (16.6483GiB/s) | 20260505 07:22:13.571 | INFO | Downloaded 34GiB to GPU in 2.05885s (16.5141GiB/s) | 20260505 07:22:18.709 | INFO | Analyzing chunk frames = [1379328,1379840) | 20260505 07:22:18.709 | INFO | MemInfo: { | 20260505 07:22:18.709 | INFO | "gpu" : " 17 GB", | 20260505 07:22:18.709 | INFO | "huge" : "", | 20260505 07:22:18.709 | INFO | "huge_shared" : " 27 GB", | 20260505 07:22:18.709 | INFO | "malloc" : " 14 GB", | 20260505 07:22:18.709 | INFO | "pinned" : " 29 GB" | 20260505 07:22:18.709 | INFO | } | 20260505 07:22:18.709 | INFO | Perf report for the last 5.13757 seconds: | 20260505 07:22:18.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:22:18.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:18.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:18.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:18.709 | INFO | ANALYSIS : 64.8%, 3.0, 101.2ms, 0 | 20260505 07:22:18.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:18.709 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:18.709 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260505 07:22:18.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:18.709 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:18.709 | INFO | Uploaded 36GiB to GPU in 2.22627s (16.1705GiB/s) | 20260505 07:22:18.709 | INFO | Downloaded 34GiB to GPU in 2.15197s (15.7995GiB/s) | 20260505 07:22:23.848 | INFO | Analyzing chunk frames = [1379840,1380352) | 20260505 07:22:23.848 | INFO | MemInfo: { | 20260505 07:22:23.848 | INFO | "gpu" : " 17 GB", | 20260505 07:22:23.848 | INFO | "huge" : "", | 20260505 07:22:23.848 | INFO | "huge_shared" : " 27 GB", | 20260505 07:22:23.848 | INFO | "malloc" : " 14 GB", | 20260505 07:22:23.848 | INFO | "pinned" : " 29 GB" | 20260505 07:22:23.848 | INFO | } | 20260505 07:22:23.848 | INFO | Perf report for the last 5.13929 seconds: | 20260505 07:22:23.848 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:22:23.848 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:23.848 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:23.848 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:23.848 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 07:22:23.848 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:23.848 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:23.848 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 07:22:23.848 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:23.848 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:23.848 | INFO | Uploaded 36GiB to GPU in 2.15132s (16.7339GiB/s) | 20260505 07:22:23.848 | INFO | Downloaded 34GiB to GPU in 2.02804s (16.765GiB/s) | 20260505 07:22:23.971 | INFO | Starting a round of Baz IO | 20260505 07:22:27.277 | INFO | Populating RTMetrics report | 20260505 07:22:27.277 | INFO | RTMetrics report populated | 20260505 07:22:27.277 | INFO | Opening tmp RTMetrics json file | 20260505 07:22:27.314 | INFO | Serializing RTMetrics to json | 20260505 07:22:27.338 | INFO | Serialization done, swapping tmp file now... | 20260505 07:22:27.339 | INFO | RTMetrics done | 20260505 07:22:27.412 | INFO | Finished a round of Baz IO: Wrote 2.426 GiB in 3.441 seconds | 20260505 07:22:27.412 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 30.10% / 0.03% / 69.87% | 20260505 07:22:27.412 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.72122% / 5.16574e-05% / 7.94729e-05% | 20260505 07:22:27.412 | INFO | Baz IO overall: | 20260505 07:22:27.412 | INFO | BytesWritten : 749728583019 | 20260505 07:22:27.412 | INFO | NumEvents : 355631473451 | 20260505 07:22:28.988 | INFO | Analyzing chunk frames = [1380352,1380864) | 20260505 07:22:28.988 | INFO | MemInfo: { | 20260505 07:22:28.988 | INFO | "gpu" : " 17 GB", | 20260505 07:22:28.988 | INFO | "huge" : "", | 20260505 07:22:28.988 | INFO | "huge_shared" : " 27 GB", | 20260505 07:22:28.988 | INFO | "malloc" : " 14 GB", | 20260505 07:22:28.988 | INFO | "pinned" : " 29 GB" | 20260505 07:22:28.988 | INFO | } | 20260505 07:22:28.988 | INFO | Perf report for the last 5.13964 seconds: | 20260505 07:22:28.988 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:22:28.988 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:28.988 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:28.988 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:28.988 | INFO | ANALYSIS : 65.6%, 2.9, 100.0ms, 0 | 20260505 07:22:28.988 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:28.988 | INFO | RT_METRICS : 47.1%, 1.0, 24.7ms, 0 | 20260505 07:22:28.988 | INFO | PRE_HQ : 64.3%, 1.9, 62.5ms, 0 | 20260505 07:22:28.988 | INFO | BAZWRITER : 66.1%, 4.1, 140.4ms, 0 | 20260505 07:22:28.988 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260505 07:22:28.988 | INFO | Uploaded 36GiB to GPU in 2.24021s (16.0699GiB/s) | 20260505 07:22:28.988 | INFO | Downloaded 38GiB to GPU in 2.3578s (16.1167GiB/s) | 20260505 07:22:34.127 | INFO | Analyzing chunk frames = [1380864,1381376) | 20260505 07:22:34.127 | INFO | MemInfo: { | 20260505 07:22:34.127 | INFO | "gpu" : " 17 GB", | 20260505 07:22:34.127 | INFO | "huge" : "", | 20260505 07:22:34.127 | INFO | "huge_shared" : " 27 GB", | 20260505 07:22:34.127 | INFO | "malloc" : " 14 GB", | 20260505 07:22:34.127 | INFO | "pinned" : " 29 GB" | 20260505 07:22:34.127 | INFO | } | 20260505 07:22:34.127 | INFO | Perf report for the last 5.13913 seconds: | 20260505 07:22:34.127 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:22:34.127 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:34.127 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:34.127 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:34.127 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 07:22:34.127 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:34.127 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:34.127 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 07:22:34.127 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:34.127 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:34.127 | INFO | Uploaded 36GiB to GPU in 2.13919s (16.8288GiB/s) | 20260505 07:22:34.127 | INFO | Downloaded 34GiB to GPU in 2.04257s (16.6457GiB/s) | 20260505 07:22:39.266 | INFO | Analyzing chunk frames = [1381376,1381888) | 20260505 07:22:39.266 | INFO | MemInfo: { | 20260505 07:22:39.266 | INFO | "gpu" : " 17 GB", | 20260505 07:22:39.266 | INFO | "huge" : "", | 20260505 07:22:39.266 | INFO | "huge_shared" : " 27 GB", | 20260505 07:22:39.266 | INFO | "malloc" : " 14 GB", | 20260505 07:22:39.266 | INFO | "pinned" : " 29 GB" | 20260505 07:22:39.267 | INFO | } | 20260505 07:22:39.267 | INFO | Perf report for the last 5.13958 seconds: | 20260505 07:22:39.267 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:22:39.267 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:39.267 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:39.267 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:39.267 | INFO | ANALYSIS : 63.6%, 3.0, 98.9ms, 0 | 20260505 07:22:39.267 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:39.267 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:39.267 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:22:39.267 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:39.267 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:39.267 | INFO | Uploaded 36GiB to GPU in 2.16146s (16.6554GiB/s) | 20260505 07:22:39.267 | INFO | Downloaded 34GiB to GPU in 2.05267s (16.5638GiB/s) | 20260505 07:22:44.406 | INFO | Analyzing chunk frames = [1381888,1382400) | 20260505 07:22:44.406 | INFO | MemInfo: { | 20260505 07:22:44.406 | INFO | "gpu" : " 17 GB", | 20260505 07:22:44.406 | INFO | "huge" : "", | 20260505 07:22:44.406 | INFO | "huge_shared" : " 27 GB", | 20260505 07:22:44.406 | INFO | "malloc" : " 14 GB", | 20260505 07:22:44.406 | INFO | "pinned" : " 29 GB" | 20260505 07:22:44.406 | INFO | } | 20260505 07:22:44.406 | INFO | Perf report for the last 5.13934 seconds: | 20260505 07:22:44.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:22:44.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:44.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:44.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:44.406 | INFO | ANALYSIS : 63.4%, 3.0, 98.4ms, 0 | 20260505 07:22:44.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:44.406 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:44.406 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:22:44.406 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:44.406 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:44.406 | INFO | Uploaded 36GiB to GPU in 2.12614s (16.9321GiB/s) | 20260505 07:22:44.406 | INFO | Downloaded 34GiB to GPU in 2.02972s (16.7511GiB/s) | 20260505 07:22:49.545 | INFO | Analyzing chunk frames = [1382400,1382912) | 20260505 07:22:49.545 | INFO | MemInfo: { | 20260505 07:22:49.545 | INFO | "gpu" : " 17 GB", | 20260505 07:22:49.545 | INFO | "huge" : "", | 20260505 07:22:49.545 | INFO | "huge_shared" : " 27 GB", | 20260505 07:22:49.545 | INFO | "malloc" : " 14 GB", | 20260505 07:22:49.545 | INFO | "pinned" : " 29 GB" | 20260505 07:22:49.545 | INFO | } | 20260505 07:22:49.545 | INFO | Perf report for the last 5.13911 seconds: | 20260505 07:22:49.545 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:22:49.545 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:49.545 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:49.545 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:49.545 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 07:22:49.545 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:49.545 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:49.545 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:22:49.545 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:49.545 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:49.545 | INFO | Uploaded 36GiB to GPU in 2.1443s (16.7887GiB/s) | 20260505 07:22:49.545 | INFO | Downloaded 34GiB to GPU in 2.04994s (16.5858GiB/s) | 20260505 07:22:54.686 | INFO | Analyzing chunk frames = [1382912,1383424) | 20260505 07:22:54.686 | INFO | MemInfo: { | 20260505 07:22:54.686 | INFO | "gpu" : " 17 GB", | 20260505 07:22:54.686 | INFO | "huge" : "", | 20260505 07:22:54.686 | INFO | "huge_shared" : " 27 GB", | 20260505 07:22:54.686 | INFO | "malloc" : " 14 GB", | 20260505 07:22:54.686 | INFO | "pinned" : " 29 GB" | 20260505 07:22:54.686 | INFO | } | 20260505 07:22:54.686 | INFO | Perf report for the last 5.14129 seconds: | 20260505 07:22:54.686 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:22:54.686 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:54.686 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:54.686 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:54.686 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 07:22:54.686 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:54.686 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:54.686 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:22:54.686 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:54.686 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:54.687 | INFO | Uploaded 36GiB to GPU in 2.15995s (16.6671GiB/s) | 20260505 07:22:54.687 | INFO | Downloaded 34GiB to GPU in 2.05581s (16.5385GiB/s) | 20260505 07:22:59.823 | INFO | Analyzing chunk frames = [1383424,1383936) | 20260505 07:22:59.823 | INFO | MemInfo: { | 20260505 07:22:59.823 | INFO | "gpu" : " 17 GB", | 20260505 07:22:59.823 | INFO | "huge" : "", | 20260505 07:22:59.824 | INFO | "huge_shared" : " 27 GB", | 20260505 07:22:59.824 | INFO | "malloc" : " 14 GB", | 20260505 07:22:59.824 | INFO | "pinned" : " 29 GB" | 20260505 07:22:59.824 | INFO | } | 20260505 07:22:59.824 | INFO | Perf report for the last 5.13734 seconds: | 20260505 07:22:59.824 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:22:59.824 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:59.824 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:59.824 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:59.824 | INFO | ANALYSIS : 64.2%, 3.0, 100.1ms, 0 | 20260505 07:22:59.824 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:59.824 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:59.824 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260505 07:22:59.824 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:59.824 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:22:59.824 | INFO | Uploaded 36GiB to GPU in 2.2002s (16.3621GiB/s) | 20260505 07:22:59.824 | INFO | Downloaded 34GiB to GPU in 2.1496s (15.8169GiB/s) | 20260505 07:23:04.963 | INFO | Analyzing chunk frames = [1383936,1384448) | 20260505 07:23:04.963 | INFO | MemInfo: { | 20260505 07:23:04.963 | INFO | "gpu" : " 17 GB", | 20260505 07:23:04.963 | INFO | "huge" : "", | 20260505 07:23:04.963 | INFO | "huge_shared" : " 27 GB", | 20260505 07:23:04.963 | INFO | "malloc" : " 14 GB", | 20260505 07:23:04.963 | INFO | "pinned" : " 29 GB" | 20260505 07:23:04.963 | INFO | } | 20260505 07:23:04.963 | INFO | Perf report for the last 5.13942 seconds: | 20260505 07:23:04.963 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:23:04.963 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:04.963 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:04.963 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:04.963 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 07:23:04.963 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:04.963 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:04.963 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 07:23:04.963 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:04.963 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:04.964 | INFO | Uploaded 36GiB to GPU in 2.1688s (16.5991GiB/s) | 20260505 07:23:04.964 | INFO | Downloaded 34GiB to GPU in 2.03669s (16.6937GiB/s) | 20260505 07:23:04.964 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 07:23:04.964 | INFO | Total mem usage: 15.3 GiB | 20260505 07:23:04.964 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 07:23:04.964 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -4.88%) | 20260505 07:23:04.964 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 07:23:04.964 | INFO | High Water Breakdown: | 20260505 07:23:04.964 | INFO | 60.9% : 8.9 GiB ( 95.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 07:23:04.964 | INFO | 39% : 5.69 GiB ( 85.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 07:23:04.964 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 07:23:04.964 | INFO | ------------------------------------------ | 20260505 07:23:04.964 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:23:04.964 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 07:23:04.964 | INFO | Total mem usage: 26.5 GiB | 20260505 07:23:04.964 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 07:23:04.964 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 07:23:04.964 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 07:23:04.964 | INFO | High Water Breakdown: | 20260505 07:23:04.964 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:23:04.964 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:23:04.964 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:23:04.964 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:23:04.964 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:23:04.964 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:23:04.964 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 07:23:04.964 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 07:23:04.964 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:23:04.964 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 07:23:04.964 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 07:23:04.964 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 07:23:04.964 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 07:23:04.964 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 07:23:04.964 | INFO | ------------------------------------------ | 20260505 07:23:04.964 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:23:04.964 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 07:23:04.964 | INFO | Total mem usage: 25.6 GiB | 20260505 07:23:04.964 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 07:23:04.964 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 07:23:04.964 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 07:23:04.964 | INFO | High Water Breakdown: | 20260505 07:23:04.964 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 07:23:04.964 | INFO | ------------------------------------------ | 20260505 07:23:04.964 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:23:04.964 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 07:23:04.964 | INFO | Total mem usage: 15.7 GiB | 20260505 07:23:04.964 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 07:23:04.964 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 07:23:04.964 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 07:23:04.964 | INFO | High Water Breakdown: | 20260505 07:23:04.964 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 07:23:04.964 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 07:23:04.964 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 07:23:04.964 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 07:23:04.964 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 07:23:04.964 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 07:23:04.964 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 07:23:04.964 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 07:23:04.964 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 07:23:04.964 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:23:04.964 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 07:23:04.964 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:23:04.964 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 07:23:04.964 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 07:23:04.964 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 07:23:04.964 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 07:23:04.964 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:23:04.964 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 07:23:04.964 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:23:04.964 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:23:04.964 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:23:04.964 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 07:23:04.964 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 07:23:04.964 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:23:04.964 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 07:23:04.964 | INFO | ------------------------------------------ | 20260505 07:23:04.964 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:23:04.964 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 07:23:04.964 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:23:04.964 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:23:04.964 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.52e+01, 1.37e+01 / 1.96e+00 | 20260505 07:23:04.964 | INFO | PerfReport: 0.28%, AggregateStats -- 9.36e-02 / 1.03e-01, 1.06e-02 / 5.01e-03 | 20260505 07:23:04.964 | INFO | PerfReport: 11.62%, Baseline -- 3.82e+00 / 4.45e+00, 6.28e-01 / 2.55e-01 | 20260505 07:23:04.964 | INFO | PerfReport: 3.30%, Histogram -- 1.08e+00 / 1.44e+00, 2.12e-01 / 1.30e-01 | 20260505 07:23:04.965 | INFO | PerfReport: 15.47%, DME -- 4.07e+01 / 4.45e+01, 4.12e+00 / 1.22e+00 | 20260505 07:23:04.965 | INFO | PerfReport: 56.24%, FrameLabeling -- 1.85e+01 / 1.93e+01, 2.03e+00 / 1.55e+00 | 20260505 07:23:04.965 | INFO | PerfReport: 7.64%, PulseAccumulating -- 2.51e+00 / 3.18e+00, 4.46e-01 / 2.34e-01 | 20260505 07:23:04.965 | INFO | PerfReport: 4.01%, Metrics -- 1.32e+00 / 1.73e+00, 3.72e-01 / 1.41e-01 | 20260505 07:23:04.965 | INFO | PerfReport: 1.45%, PulseSerializing -- 4.75e-01 / 5.51e-01, 9.13e-02 / 2.50e-02 | 20260505 07:23:04.965 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 07:23:04.965 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:23:04.965 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:23:04.965 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.25e+01 / 0.00e+00 | 20260505 07:23:04.965 | INFO | PerfReport: 22.28%, Upload -- 2.26e+01 / 0.00e+00, 8.04e+00 / 0.00e+00 | 20260505 07:23:04.965 | INFO | PerfReport: 21.52%, Download -- 2.18e+01 / 0.00e+00, 7.87e+00 / 0.00e+00 | 20260505 07:23:04.965 | INFO | PerfReport: 32.42%, Compute -- 3.29e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 07:23:04.965 | INFO | PerfReport: 1.52%, LockUpload -- 1.55e+00 / 0.00e+00, 7.15e+00 / 0.00e+00 | 20260505 07:23:04.965 | INFO | PerfReport: 0.85%, LockDownload -- 8.66e-01 / 0.00e+00, 2.26e+00 / 0.00e+00 | 20260505 07:23:04.965 | INFO | PerfReport: 21.38%, LockCompute -- 2.17e+01 / 0.00e+00, 1.66e+01 / 0.00e+00 | 20260505 07:23:04.965 | INFO | PerfReport: 0.02%, Misc -- 1.59e-02 / 0.00e+00, 3.53e-02 / 0.00e+00 | 20260505 07:23:05.097 | INFO | Starting a round of Baz IO | 20260505 07:23:08.376 | INFO | Populating RTMetrics report | 20260505 07:23:08.377 | INFO | RTMetrics report populated | 20260505 07:23:08.377 | INFO | Opening tmp RTMetrics json file | 20260505 07:23:08.385 | INFO | Serializing RTMetrics to json | 20260505 07:23:08.387 | INFO | Serialization done, swapping tmp file now... | 20260505 07:23:08.387 | INFO | RTMetrics done | 20260505 07:23:08.516 | INFO | Finished a round of Baz IO: Wrote 2.427 GiB in 3.419 seconds | 20260505 07:23:08.516 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 30.06% / 0.03% / 69.90% | 20260505 07:23:08.516 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.70438% / 4.37101e-05% / 8.74201e-05% | 20260505 07:23:08.516 | INFO | Baz IO overall: | 20260505 07:23:08.517 | INFO | BytesWritten : 752334551869 | 20260505 07:23:08.517 | INFO | NumEvents : 356878282711 | 20260505 07:23:10.102 | INFO | Analyzing chunk frames = [1384448,1384960) | 20260505 07:23:10.102 | INFO | MemInfo: { | 20260505 07:23:10.102 | INFO | "gpu" : " 17 GB", | 20260505 07:23:10.102 | INFO | "huge" : "", | 20260505 07:23:10.102 | INFO | "huge_shared" : " 27 GB", | 20260505 07:23:10.102 | INFO | "malloc" : " 14 GB", | 20260505 07:23:10.102 | INFO | "pinned" : " 29 GB" | 20260505 07:23:10.102 | INFO | } | 20260505 07:23:10.102 | INFO | Perf report for the last 5.139 seconds: | 20260505 07:23:10.102 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:23:10.102 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:10.102 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:10.102 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:10.102 | INFO | ANALYSIS : 65.3%, 2.9, 99.2ms, 0 | 20260505 07:23:10.102 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:10.102 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260505 07:23:10.102 | INFO | PRE_HQ : 63.9%, 1.9, 62.5ms, 0 | 20260505 07:23:10.102 | INFO | BAZWRITER : 65.8%, 4.2, 142.4ms, 0 | 20260505 07:23:10.102 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 07:23:10.102 | INFO | Uploaded 36GiB to GPU in 2.21943s (16.2204GiB/s) | 20260505 07:23:10.103 | INFO | Downloaded 38GiB to GPU in 2.32238s (16.3625GiB/s) | 20260505 07:23:15.242 | INFO | Analyzing chunk frames = [1384960,1385472) | 20260505 07:23:15.242 | INFO | MemInfo: { | 20260505 07:23:15.242 | INFO | "gpu" : " 17 GB", | 20260505 07:23:15.242 | INFO | "huge" : "", | 20260505 07:23:15.242 | INFO | "huge_shared" : " 27 GB", | 20260505 07:23:15.242 | INFO | "malloc" : " 14 GB", | 20260505 07:23:15.242 | INFO | "pinned" : " 29 GB" | 20260505 07:23:15.242 | INFO | } | 20260505 07:23:15.242 | INFO | Perf report for the last 5.13976 seconds: | 20260505 07:23:15.242 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:23:15.242 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:15.242 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:15.242 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:15.242 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 07:23:15.242 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:15.242 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:15.242 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 07:23:15.242 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:15.242 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:15.242 | INFO | Uploaded 36GiB to GPU in 2.12611s (16.9323GiB/s) | 20260505 07:23:15.242 | INFO | Downloaded 34GiB to GPU in 2.03732s (16.6886GiB/s) | 20260505 07:23:20.381 | INFO | Analyzing chunk frames = [1385472,1385984) | 20260505 07:23:20.381 | INFO | MemInfo: { | 20260505 07:23:20.381 | INFO | "gpu" : " 17 GB", | 20260505 07:23:20.381 | INFO | "huge" : "", | 20260505 07:23:20.381 | INFO | "huge_shared" : " 27 GB", | 20260505 07:23:20.381 | INFO | "malloc" : " 14 GB", | 20260505 07:23:20.381 | INFO | "pinned" : " 29 GB" | 20260505 07:23:20.381 | INFO | } | 20260505 07:23:20.381 | INFO | Perf report for the last 5.13913 seconds: | 20260505 07:23:20.381 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:23:20.381 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:20.381 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:20.381 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:20.381 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260505 07:23:20.381 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:20.381 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:20.381 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 07:23:20.381 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:20.381 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:20.381 | INFO | Uploaded 36GiB to GPU in 2.15861s (16.6774GiB/s) | 20260505 07:23:20.381 | INFO | Downloaded 34GiB to GPU in 2.0555s (16.541GiB/s) | 20260505 07:23:25.520 | INFO | Analyzing chunk frames = [1385984,1386496) | 20260505 07:23:25.520 | INFO | MemInfo: { | 20260505 07:23:25.520 | INFO | "gpu" : " 17 GB", | 20260505 07:23:25.520 | INFO | "huge" : "", | 20260505 07:23:25.520 | INFO | "huge_shared" : " 27 GB", | 20260505 07:23:25.520 | INFO | "malloc" : " 14 GB", | 20260505 07:23:25.520 | INFO | "pinned" : " 29 GB" | 20260505 07:23:25.520 | INFO | } | 20260505 07:23:25.521 | INFO | Perf report for the last 5.1395 seconds: | 20260505 07:23:25.521 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:23:25.521 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:25.521 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:25.521 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:25.521 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 07:23:25.521 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:25.521 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:25.521 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 07:23:25.521 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:25.521 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:25.521 | INFO | Uploaded 36GiB to GPU in 2.12575s (16.9352GiB/s) | 20260505 07:23:25.521 | INFO | Downloaded 34GiB to GPU in 2.03026s (16.7466GiB/s) | 20260505 07:23:30.659 | INFO | Analyzing chunk frames = [1386496,1387008) | 20260505 07:23:30.659 | INFO | MemInfo: { | 20260505 07:23:30.659 | INFO | "gpu" : " 17 GB", | 20260505 07:23:30.659 | INFO | "huge" : "", | 20260505 07:23:30.659 | INFO | "huge_shared" : " 27 GB", | 20260505 07:23:30.659 | INFO | "malloc" : " 14 GB", | 20260505 07:23:30.660 | INFO | "pinned" : " 29 GB" | 20260505 07:23:30.660 | INFO | } | 20260505 07:23:30.660 | INFO | Perf report for the last 5.13909 seconds: | 20260505 07:23:30.660 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:23:30.660 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:30.660 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:30.660 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:30.660 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 07:23:30.660 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:30.660 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:30.660 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 07:23:30.660 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:30.660 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:30.660 | INFO | Uploaded 36GiB to GPU in 2.20095s (16.3565GiB/s) | 20260505 07:23:30.660 | INFO | Downloaded 34GiB to GPU in 2.05703s (16.5287GiB/s) | 20260505 07:23:35.800 | INFO | Analyzing chunk frames = [1387008,1387520) | 20260505 07:23:35.801 | INFO | MemInfo: { | 20260505 07:23:35.801 | INFO | "gpu" : " 17 GB", | 20260505 07:23:35.801 | INFO | "huge" : "", | 20260505 07:23:35.801 | INFO | "huge_shared" : " 27 GB", | 20260505 07:23:35.801 | INFO | "malloc" : " 14 GB", | 20260505 07:23:35.801 | INFO | "pinned" : " 29 GB" | 20260505 07:23:35.801 | INFO | } | 20260505 07:23:35.801 | INFO | Perf report for the last 5.14099 seconds: | 20260505 07:23:35.801 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:23:35.801 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:35.801 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:35.801 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:35.801 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 07:23:35.801 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:35.801 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:35.801 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:23:35.801 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:35.801 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:35.801 | INFO | Uploaded 36GiB to GPU in 2.16826s (16.6032GiB/s) | 20260505 07:23:35.801 | INFO | Downloaded 34GiB to GPU in 2.05837s (16.5179GiB/s) | 20260505 07:23:40.938 | INFO | Analyzing chunk frames = [1387520,1388032) | 20260505 07:23:40.938 | INFO | MemInfo: { | 20260505 07:23:40.938 | INFO | "gpu" : " 17 GB", | 20260505 07:23:40.938 | INFO | "huge" : "", | 20260505 07:23:40.938 | INFO | "huge_shared" : " 27 GB", | 20260505 07:23:40.938 | INFO | "malloc" : " 14 GB", | 20260505 07:23:40.938 | INFO | "pinned" : " 29 GB" | 20260505 07:23:40.938 | INFO | } | 20260505 07:23:40.938 | INFO | Perf report for the last 5.1378 seconds: | 20260505 07:23:40.938 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:23:40.938 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:40.938 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:40.938 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:40.938 | INFO | ANALYSIS : 64.1%, 3.0, 100.0ms, 0 | 20260505 07:23:40.939 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:40.939 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:40.939 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260505 07:23:40.939 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:40.939 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:40.939 | INFO | Uploaded 36GiB to GPU in 2.19707s (16.3855GiB/s) | 20260505 07:23:40.939 | INFO | Downloaded 34GiB to GPU in 2.16873s (15.6774GiB/s) | 20260505 07:23:46.078 | INFO | Analyzing chunk frames = [1388032,1388544) | 20260505 07:23:46.078 | INFO | MemInfo: { | 20260505 07:23:46.078 | INFO | "gpu" : " 17 GB", | 20260505 07:23:46.078 | INFO | "huge" : "", | 20260505 07:23:46.078 | INFO | "huge_shared" : " 27 GB", | 20260505 07:23:46.078 | INFO | "malloc" : " 14 GB", | 20260505 07:23:46.078 | INFO | "pinned" : " 29 GB" | 20260505 07:23:46.078 | INFO | } | 20260505 07:23:46.078 | INFO | Perf report for the last 5.13936 seconds: | 20260505 07:23:46.078 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:23:46.078 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:46.078 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:46.078 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:46.078 | INFO | ANALYSIS : 64.1%, 3.0, 100.2ms, 0 | 20260505 07:23:46.078 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:46.078 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:46.078 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 07:23:46.078 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:46.078 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:46.078 | INFO | Uploaded 36GiB to GPU in 2.16339s (16.6406GiB/s) | 20260505 07:23:46.078 | INFO | Downloaded 34GiB to GPU in 2.02386s (16.7996GiB/s) | 20260505 07:23:46.210 | INFO | Starting a round of Baz IO | 20260505 07:23:49.505 | INFO | Populating RTMetrics report | 20260505 07:23:49.505 | INFO | RTMetrics report populated | 20260505 07:23:49.505 | INFO | Opening tmp RTMetrics json file | 20260505 07:23:49.540 | INFO | Serializing RTMetrics to json | 20260505 07:23:49.555 | INFO | Serialization done, swapping tmp file now... | 20260505 07:23:49.557 | INFO | RTMetrics done | 20260505 07:23:49.672 | INFO | Finished a round of Baz IO: Wrote 2.427 GiB in 3.462 seconds | 20260505 07:23:49.673 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 30.03% / 0.03% / 69.94% | 20260505 07:23:49.673 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.68965% / 1.98682e-05% / 8.34465e-05% | 20260505 07:23:49.673 | INFO | Baz IO overall: | 20260505 07:23:49.673 | INFO | BytesWritten : 754940461192 | 20260505 07:23:49.673 | INFO | NumEvents : 358125144299 | 20260505 07:23:51.217 | INFO | Analyzing chunk frames = [1388544,1389056) | 20260505 07:23:51.217 | INFO | MemInfo: { | 20260505 07:23:51.217 | INFO | "gpu" : " 17 GB", | 20260505 07:23:51.217 | INFO | "huge" : "", | 20260505 07:23:51.217 | INFO | "huge_shared" : " 27 GB", | 20260505 07:23:51.217 | INFO | "malloc" : " 14 GB", | 20260505 07:23:51.217 | INFO | "pinned" : " 29 GB" | 20260505 07:23:51.218 | INFO | } | 20260505 07:23:51.218 | INFO | Perf report for the last 5.13987 seconds: | 20260505 07:23:51.218 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:23:51.218 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:51.218 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:51.218 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:51.218 | INFO | ANALYSIS : 65.5%, 2.9, 99.5ms, 0 | 20260505 07:23:51.218 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:51.218 | INFO | RT_METRICS : 47.1%, 1.0, 24.7ms, 0 | 20260505 07:23:51.218 | INFO | PRE_HQ : 64.2%, 1.9, 62.9ms, 0 | 20260505 07:23:51.218 | INFO | BAZWRITER : 66.7%, 4.1, 141.8ms, 0 | 20260505 07:23:51.218 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 07:23:51.218 | INFO | Uploaded 36GiB to GPU in 2.23476s (16.1091GiB/s) | 20260505 07:23:51.218 | INFO | Downloaded 38GiB to GPU in 2.36233s (16.0858GiB/s) | 20260505 07:23:56.360 | INFO | Analyzing chunk frames = [1389056,1389568) | 20260505 07:23:56.360 | INFO | MemInfo: { | 20260505 07:23:56.360 | INFO | "gpu" : " 17 GB", | 20260505 07:23:56.360 | INFO | "huge" : "", | 20260505 07:23:56.360 | INFO | "huge_shared" : " 27 GB", | 20260505 07:23:56.360 | INFO | "malloc" : " 14 GB", | 20260505 07:23:56.360 | INFO | "pinned" : " 29 GB" | 20260505 07:23:56.360 | INFO | } | 20260505 07:23:56.360 | INFO | Perf report for the last 5.14205 seconds: | 20260505 07:23:56.360 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:23:56.360 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:56.360 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:56.360 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:56.360 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 07:23:56.360 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:56.360 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:56.360 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:23:56.360 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:56.360 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:23:56.360 | INFO | Uploaded 36GiB to GPU in 2.1346s (16.865GiB/s) | 20260505 07:23:56.360 | INFO | Downloaded 34GiB to GPU in 2.04421s (16.6323GiB/s) | 20260505 07:24:01.499 | INFO | Analyzing chunk frames = [1389568,1390080) | 20260505 07:24:01.499 | INFO | MemInfo: { | 20260505 07:24:01.499 | INFO | "gpu" : " 17 GB", | 20260505 07:24:01.499 | INFO | "huge" : "", | 20260505 07:24:01.499 | INFO | "huge_shared" : " 27 GB", | 20260505 07:24:01.499 | INFO | "malloc" : " 14 GB", | 20260505 07:24:01.499 | INFO | "pinned" : " 29 GB" | 20260505 07:24:01.499 | INFO | } | 20260505 07:24:01.499 | INFO | Perf report for the last 5.13898 seconds: | 20260505 07:24:01.499 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:24:01.499 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:01.499 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:01.499 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:01.499 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 07:24:01.499 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:01.499 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:01.499 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:24:01.499 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:01.499 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:01.499 | INFO | Uploaded 36GiB to GPU in 2.14936s (16.7491GiB/s) | 20260505 07:24:01.499 | INFO | Downloaded 34GiB to GPU in 2.05672s (16.5311GiB/s) | 20260505 07:24:06.635 | INFO | Analyzing chunk frames = [1390080,1390592) | 20260505 07:24:06.635 | INFO | MemInfo: { | 20260505 07:24:06.635 | INFO | "gpu" : " 17 GB", | 20260505 07:24:06.635 | INFO | "huge" : "", | 20260505 07:24:06.635 | INFO | "huge_shared" : " 27 GB", | 20260505 07:24:06.635 | INFO | "malloc" : " 14 GB", | 20260505 07:24:06.635 | INFO | "pinned" : " 29 GB" | 20260505 07:24:06.635 | INFO | } | 20260505 07:24:06.635 | INFO | Perf report for the last 5.13621 seconds: | 20260505 07:24:06.635 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:24:06.635 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:06.635 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:06.635 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:06.635 | INFO | ANALYSIS : 63.3%, 3.0, 98.2ms, 0 | 20260505 07:24:06.635 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:06.635 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:06.635 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:24:06.635 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:06.635 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:06.635 | INFO | Uploaded 36GiB to GPU in 2.13081s (16.895GiB/s) | 20260505 07:24:06.635 | INFO | Downloaded 34GiB to GPU in 2.03597s (16.6996GiB/s) | 20260505 07:24:11.774 | INFO | Analyzing chunk frames = [1390592,1391104) | 20260505 07:24:11.774 | INFO | MemInfo: { | 20260505 07:24:11.774 | INFO | "gpu" : " 17 GB", | 20260505 07:24:11.774 | INFO | "huge" : "", | 20260505 07:24:11.774 | INFO | "huge_shared" : " 27 GB", | 20260505 07:24:11.774 | INFO | "malloc" : " 14 GB", | 20260505 07:24:11.774 | INFO | "pinned" : " 29 GB" | 20260505 07:24:11.774 | INFO | } | 20260505 07:24:11.774 | INFO | Perf report for the last 5.13922 seconds: | 20260505 07:24:11.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:24:11.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:11.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:11.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:11.774 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 07:24:11.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:11.774 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:11.774 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:24:11.774 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:11.774 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:11.775 | INFO | Uploaded 36GiB to GPU in 2.19733s (16.3836GiB/s) | 20260505 07:24:11.775 | INFO | Downloaded 34GiB to GPU in 2.05172s (16.5715GiB/s) | 20260505 07:24:16.915 | INFO | Analyzing chunk frames = [1391104,1391616) | 20260505 07:24:16.915 | INFO | MemInfo: { | 20260505 07:24:16.915 | INFO | "gpu" : " 17 GB", | 20260505 07:24:16.915 | INFO | "huge" : "", | 20260505 07:24:16.915 | INFO | "huge_shared" : " 27 GB", | 20260505 07:24:16.915 | INFO | "malloc" : " 14 GB", | 20260505 07:24:16.915 | INFO | "pinned" : " 29 GB" | 20260505 07:24:16.915 | INFO | } | 20260505 07:24:16.915 | INFO | Perf report for the last 5.14125 seconds: | 20260505 07:24:16.916 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:24:16.916 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:16.916 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:16.916 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:16.916 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 07:24:16.916 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:16.916 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:16.916 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:24:16.916 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:16.916 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:16.916 | INFO | Uploaded 36GiB to GPU in 2.16417s (16.6346GiB/s) | 20260505 07:24:16.916 | INFO | Downloaded 34GiB to GPU in 2.05751s (16.5248GiB/s) | 20260505 07:24:22.053 | INFO | Analyzing chunk frames = [1391616,1392128) | 20260505 07:24:22.053 | INFO | MemInfo: { | 20260505 07:24:22.053 | INFO | "gpu" : " 17 GB", | 20260505 07:24:22.053 | INFO | "huge" : "", | 20260505 07:24:22.053 | INFO | "huge_shared" : " 27 GB", | 20260505 07:24:22.053 | INFO | "malloc" : " 14 GB", | 20260505 07:24:22.053 | INFO | "pinned" : " 29 GB" | 20260505 07:24:22.053 | INFO | } | 20260505 07:24:22.053 | INFO | Perf report for the last 5.1376 seconds: | 20260505 07:24:22.053 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:24:22.053 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:22.053 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:22.053 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:22.053 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 07:24:22.053 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:22.053 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:22.053 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260505 07:24:22.053 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:22.053 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:22.053 | INFO | Uploaded 36GiB to GPU in 2.21527s (16.2508GiB/s) | 20260505 07:24:22.053 | INFO | Downloaded 34GiB to GPU in 2.16146s (15.7301GiB/s) | 20260505 07:24:27.192 | INFO | Analyzing chunk frames = [1392128,1392640) | 20260505 07:24:27.192 | INFO | MemInfo: { | 20260505 07:24:27.192 | INFO | "gpu" : " 17 GB", | 20260505 07:24:27.192 | INFO | "huge" : "", | 20260505 07:24:27.193 | INFO | "huge_shared" : " 27 GB", | 20260505 07:24:27.193 | INFO | "malloc" : " 14 GB", | 20260505 07:24:27.193 | INFO | "pinned" : " 29 GB" | 20260505 07:24:27.193 | INFO | } | 20260505 07:24:27.193 | INFO | Perf report for the last 5.13956 seconds: | 20260505 07:24:27.193 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:24:27.193 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:27.193 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:27.193 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:27.193 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 07:24:27.193 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:27.193 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:27.193 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 07:24:27.193 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:27.193 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:27.193 | INFO | Uploaded 36GiB to GPU in 2.15844s (16.6787GiB/s) | 20260505 07:24:27.193 | INFO | Downloaded 34GiB to GPU in 2.02407s (16.7978GiB/s) | 20260505 07:24:27.193 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 07:24:27.193 | INFO | Total mem usage: 15.3 GiB | 20260505 07:24:27.193 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 07:24:27.193 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -5.41%) | 20260505 07:24:27.193 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 07:24:27.193 | INFO | High Water Breakdown: | 20260505 07:24:27.193 | INFO | 60.7% : 8.83 GiB ( 94.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 07:24:27.193 | INFO | 39.1% : 5.69 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 07:24:27.193 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 07:24:27.193 | INFO | ------------------------------------------ | 20260505 07:24:27.193 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:24:27.193 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 07:24:27.193 | INFO | Total mem usage: 26.5 GiB | 20260505 07:24:27.193 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 07:24:27.193 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260505 07:24:27.193 | INFO | Unused Cache: 1.6 GiB (Waste: 6.05%) | 20260505 07:24:27.193 | INFO | High Water Breakdown: | 20260505 07:24:27.193 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:24:27.193 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:24:27.193 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:24:27.193 | INFO | 2% : 523 MiB ( 99.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:24:27.193 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:24:27.193 | INFO | 0.557% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:24:27.194 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 07:24:27.194 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 07:24:27.194 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:24:27.194 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 07:24:27.194 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 07:24:27.194 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 07:24:27.194 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 07:24:27.194 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 07:24:27.194 | INFO | ------------------------------------------ | 20260505 07:24:27.194 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:24:27.194 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 07:24:27.194 | INFO | Total mem usage: 25.6 GiB | 20260505 07:24:27.194 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 07:24:27.194 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 07:24:27.194 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 07:24:27.194 | INFO | High Water Breakdown: | 20260505 07:24:27.194 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 07:24:27.194 | INFO | ------------------------------------------ | 20260505 07:24:27.194 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:24:27.194 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 07:24:27.194 | INFO | Total mem usage: 15.7 GiB | 20260505 07:24:27.194 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 07:24:27.194 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 07:24:27.194 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 07:24:27.194 | INFO | High Water Breakdown: | 20260505 07:24:27.194 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 07:24:27.194 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 07:24:27.194 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 07:24:27.194 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 07:24:27.194 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 07:24:27.194 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 07:24:27.194 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 07:24:27.194 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 07:24:27.194 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 07:24:27.194 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:24:27.194 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 07:24:27.194 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:24:27.194 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 07:24:27.194 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 07:24:27.194 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 07:24:27.194 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 07:24:27.194 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:24:27.194 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 07:24:27.194 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:24:27.194 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:24:27.194 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:24:27.194 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 07:24:27.194 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 07:24:27.194 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:24:27.194 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 07:24:27.194 | INFO | ------------------------------------------ | 20260505 07:24:27.194 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:24:27.194 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 07:24:27.194 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:24:27.194 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:24:27.194 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.51e+01, 1.34e+01 / 2.27e+00 | 20260505 07:24:27.194 | INFO | PerfReport: 0.28%, AggregateStats -- 9.36e-02 / 1.03e-01, 1.06e-02 / 5.16e-03 | 20260505 07:24:27.194 | INFO | PerfReport: 11.62%, Baseline -- 3.82e+00 / 4.44e+00, 6.31e-01 / 3.01e-01 | 20260505 07:24:27.194 | INFO | PerfReport: 3.31%, Histogram -- 1.09e+00 / 1.46e+00, 2.13e-01 / 1.25e-01 | 20260505 07:24:27.194 | INFO | PerfReport: 15.46%, DME -- 4.07e+01 / 4.45e+01, 4.12e+00 / 1.32e+00 | 20260505 07:24:27.194 | INFO | PerfReport: 56.30%, FrameLabeling -- 1.85e+01 / 1.92e+01, 2.03e+00 / 1.60e+00 | 20260505 07:24:27.194 | INFO | PerfReport: 7.60%, PulseAccumulating -- 2.50e+00 / 3.18e+00, 4.39e-01 / 2.15e-01 | 20260505 07:24:27.194 | INFO | PerfReport: 3.99%, Metrics -- 1.31e+00 / 1.73e+00, 3.69e-01 / 1.56e-01 | 20260505 07:24:27.194 | INFO | PerfReport: 1.44%, PulseSerializing -- 4.74e-01 / 5.51e-01, 9.07e-02 / 2.74e-02 | 20260505 07:24:27.194 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 07:24:27.194 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:24:27.194 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:24:27.194 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.26e+01 / 0.00e+00 | 20260505 07:24:27.194 | INFO | PerfReport: 22.30%, Upload -- 2.26e+01 / 0.00e+00, 8.04e+00 / 0.00e+00 | 20260505 07:24:27.194 | INFO | PerfReport: 21.54%, Download -- 2.18e+01 / 0.00e+00, 7.95e+00 / 0.00e+00 | 20260505 07:24:27.194 | INFO | PerfReport: 32.42%, Compute -- 3.29e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 07:24:27.194 | INFO | PerfReport: 1.51%, LockUpload -- 1.53e+00 / 0.00e+00, 7.14e+00 / 0.00e+00 | 20260505 07:24:27.195 | INFO | PerfReport: 0.89%, LockDownload -- 8.98e-01 / 0.00e+00, 2.32e+00 / 0.00e+00 | 20260505 07:24:27.195 | INFO | PerfReport: 21.32%, LockCompute -- 2.16e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260505 07:24:27.195 | INFO | PerfReport: 0.01%, Misc -- 1.46e-02 / 0.00e+00, 2.48e-02 / 0.00e+00 | 20260505 07:24:27.325 | INFO | Starting a round of Baz IO | 20260505 07:24:30.598 | INFO | Populating RTMetrics report | 20260505 07:24:30.598 | INFO | RTMetrics report populated | 20260505 07:24:30.598 | INFO | Opening tmp RTMetrics json file | 20260505 07:24:30.614 | INFO | Serializing RTMetrics to json | 20260505 07:24:30.615 | INFO | Serialization done, swapping tmp file now... | 20260505 07:24:30.615 | INFO | RTMetrics done | 20260505 07:24:30.726 | INFO | Finished a round of Baz IO: Wrote 2.428 GiB in 3.4 seconds | 20260505 07:24:30.726 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 30.00% / 0.03% / 69.97% | 20260505 07:24:30.726 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.67608% / 2.38419e-05% / 8.34465e-05% | 20260505 07:24:30.726 | INFO | Baz IO overall: | 20260505 07:24:30.726 | INFO | BytesWritten : 757547715601 | 20260505 07:24:30.726 | INFO | NumEvents : 359372733455 | 20260505 07:24:32.331 | INFO | Analyzing chunk frames = [1392640,1393152) | 20260505 07:24:32.331 | INFO | MemInfo: { | 20260505 07:24:32.331 | INFO | "gpu" : " 17 GB", | 20260505 07:24:32.331 | INFO | "huge" : "", | 20260505 07:24:32.331 | INFO | "huge_shared" : " 27 GB", | 20260505 07:24:32.332 | INFO | "malloc" : " 14 GB", | 20260505 07:24:32.332 | INFO | "pinned" : " 29 GB" | 20260505 07:24:32.332 | INFO | } | 20260505 07:24:32.332 | INFO | Perf report for the last 5.139 seconds: | 20260505 07:24:32.332 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:24:32.332 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:32.332 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:32.332 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:32.332 | INFO | ANALYSIS : 65.1%, 2.9, 99.2ms, 0 | 20260505 07:24:32.332 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:32.332 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260505 07:24:32.332 | INFO | PRE_HQ : 63.8%, 1.9, 62.8ms, 0 | 20260505 07:24:32.332 | INFO | BAZWRITER : 65.5%, 4.2, 140.2ms, 0 | 20260505 07:24:32.332 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 07:24:32.332 | INFO | Uploaded 36GiB to GPU in 2.20232s (16.3464GiB/s) | 20260505 07:24:32.332 | INFO | Downloaded 38GiB to GPU in 2.32215s (16.3642GiB/s) | 20260505 07:24:37.471 | INFO | Analyzing chunk frames = [1393152,1393664) | 20260505 07:24:37.471 | INFO | MemInfo: { | 20260505 07:24:37.471 | INFO | "gpu" : " 17 GB", | 20260505 07:24:37.471 | INFO | "huge" : "", | 20260505 07:24:37.471 | INFO | "huge_shared" : " 27 GB", | 20260505 07:24:37.471 | INFO | "malloc" : " 14 GB", | 20260505 07:24:37.471 | INFO | "pinned" : " 29 GB" | 20260505 07:24:37.471 | INFO | } | 20260505 07:24:37.471 | INFO | Perf report for the last 5.13942 seconds: | 20260505 07:24:37.471 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:24:37.471 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:37.471 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:37.471 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:37.471 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 07:24:37.471 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:37.471 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:37.471 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 07:24:37.471 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:37.471 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:37.472 | INFO | Uploaded 36GiB to GPU in 2.12224s (16.9632GiB/s) | 20260505 07:24:37.472 | INFO | Downloaded 34GiB to GPU in 2.0452s (16.6243GiB/s) | 20260505 07:24:42.611 | INFO | Analyzing chunk frames = [1393664,1394176) | 20260505 07:24:42.611 | INFO | MemInfo: { | 20260505 07:24:42.611 | INFO | "gpu" : " 17 GB", | 20260505 07:24:42.611 | INFO | "huge" : "", | 20260505 07:24:42.611 | INFO | "huge_shared" : " 27 GB", | 20260505 07:24:42.611 | INFO | "malloc" : " 14 GB", | 20260505 07:24:42.611 | INFO | "pinned" : " 29 GB" | 20260505 07:24:42.611 | INFO | } | 20260505 07:24:42.611 | INFO | Perf report for the last 5.13949 seconds: | 20260505 07:24:42.611 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:24:42.611 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:42.611 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:42.611 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:42.611 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260505 07:24:42.611 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:42.611 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:42.611 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 07:24:42.611 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:42.611 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:42.611 | INFO | Uploaded 36GiB to GPU in 2.15789s (16.6829GiB/s) | 20260505 07:24:42.611 | INFO | Downloaded 34GiB to GPU in 2.05482s (16.5465GiB/s) | 20260505 07:24:47.750 | INFO | Analyzing chunk frames = [1394176,1394688) | 20260505 07:24:47.750 | INFO | MemInfo: { | 20260505 07:24:47.750 | INFO | "gpu" : " 17 GB", | 20260505 07:24:47.750 | INFO | "huge" : "", | 20260505 07:24:47.750 | INFO | "huge_shared" : " 27 GB", | 20260505 07:24:47.750 | INFO | "malloc" : " 14 GB", | 20260505 07:24:47.750 | INFO | "pinned" : " 29 GB" | 20260505 07:24:47.750 | INFO | } | 20260505 07:24:47.750 | INFO | Perf report for the last 5.13951 seconds: | 20260505 07:24:47.750 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:24:47.750 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:47.750 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:47.750 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:47.750 | INFO | ANALYSIS : 63.3%, 3.0, 98.3ms, 0 | 20260505 07:24:47.750 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:47.750 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:47.750 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 07:24:47.750 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:47.750 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:47.750 | INFO | Uploaded 36GiB to GPU in 2.12714s (16.9241GiB/s) | 20260505 07:24:47.750 | INFO | Downloaded 34GiB to GPU in 2.0303s (16.7463GiB/s) | 20260505 07:24:52.889 | INFO | Analyzing chunk frames = [1394688,1395200) | 20260505 07:24:52.889 | INFO | MemInfo: { | 20260505 07:24:52.889 | INFO | "gpu" : " 17 GB", | 20260505 07:24:52.889 | INFO | "huge" : "", | 20260505 07:24:52.889 | INFO | "huge_shared" : " 27 GB", | 20260505 07:24:52.889 | INFO | "malloc" : " 14 GB", | 20260505 07:24:52.889 | INFO | "pinned" : " 29 GB" | 20260505 07:24:52.889 | INFO | } | 20260505 07:24:52.889 | INFO | Perf report for the last 5.13902 seconds: | 20260505 07:24:52.889 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:24:52.889 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:52.889 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:52.889 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:52.889 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 07:24:52.889 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:52.889 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:52.889 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 07:24:52.889 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:52.889 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:52.889 | INFO | Uploaded 36GiB to GPU in 2.16569s (16.6229GiB/s) | 20260505 07:24:52.889 | INFO | Downloaded 34GiB to GPU in 2.05097s (16.5775GiB/s) | 20260505 07:24:58.030 | INFO | Analyzing chunk frames = [1395200,1395712) | 20260505 07:24:58.030 | INFO | MemInfo: { | 20260505 07:24:58.030 | INFO | "gpu" : " 17 GB", | 20260505 07:24:58.030 | INFO | "huge" : "", | 20260505 07:24:58.030 | INFO | "huge_shared" : " 27 GB", | 20260505 07:24:58.030 | INFO | "malloc" : " 14 GB", | 20260505 07:24:58.030 | INFO | "pinned" : " 29 GB" | 20260505 07:24:58.030 | INFO | } | 20260505 07:24:58.030 | INFO | Perf report for the last 5.14128 seconds: | 20260505 07:24:58.030 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:24:58.030 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:58.030 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:58.030 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:58.030 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 07:24:58.030 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:58.031 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:58.031 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:24:58.031 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:58.031 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:24:58.031 | INFO | Uploaded 36GiB to GPU in 2.16553s (16.6241GiB/s) | 20260505 07:24:58.031 | INFO | Downloaded 34GiB to GPU in 2.05968s (16.5074GiB/s) | 20260505 07:25:03.168 | INFO | Analyzing chunk frames = [1395712,1396224) | 20260505 07:25:03.168 | INFO | MemInfo: { | 20260505 07:25:03.168 | INFO | "gpu" : " 17 GB", | 20260505 07:25:03.168 | INFO | "huge" : "", | 20260505 07:25:03.168 | INFO | "huge_shared" : " 27 GB", | 20260505 07:25:03.168 | INFO | "malloc" : " 14 GB", | 20260505 07:25:03.168 | INFO | "pinned" : " 29 GB" | 20260505 07:25:03.168 | INFO | } | 20260505 07:25:03.168 | INFO | Perf report for the last 5.13779 seconds: | 20260505 07:25:03.168 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:25:03.168 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:03.168 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:03.168 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:03.168 | INFO | ANALYSIS : 64.7%, 3.0, 100.9ms, 0 | 20260505 07:25:03.168 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:03.168 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:03.168 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260505 07:25:03.168 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:03.168 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:03.169 | INFO | Uploaded 36GiB to GPU in 2.21937s (16.2209GiB/s) | 20260505 07:25:03.169 | INFO | Downloaded 34GiB to GPU in 2.15103s (15.8064GiB/s) | 20260505 07:25:08.307 | INFO | Analyzing chunk frames = [1396224,1396736) | 20260505 07:25:08.307 | INFO | MemInfo: { | 20260505 07:25:08.307 | INFO | "gpu" : " 17 GB", | 20260505 07:25:08.307 | INFO | "huge" : "", | 20260505 07:25:08.307 | INFO | "huge_shared" : " 27 GB", | 20260505 07:25:08.307 | INFO | "malloc" : " 14 GB", | 20260505 07:25:08.307 | INFO | "pinned" : " 29 GB" | 20260505 07:25:08.307 | INFO | } | 20260505 07:25:08.307 | INFO | Perf report for the last 5.13909 seconds: | 20260505 07:25:08.307 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:25:08.307 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:08.307 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:08.307 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:08.307 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 07:25:08.307 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:08.307 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:08.307 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 07:25:08.307 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:08.308 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:08.308 | INFO | Uploaded 36GiB to GPU in 2.16505s (16.6278GiB/s) | 20260505 07:25:08.308 | INFO | Downloaded 34GiB to GPU in 2.03699s (16.6913GiB/s) | 20260505 07:25:08.441 | INFO | Starting a round of Baz IO | 20260505 07:25:11.733 | INFO | Populating RTMetrics report | 20260505 07:25:11.733 | INFO | RTMetrics report populated | 20260505 07:25:11.733 | INFO | Opening tmp RTMetrics json file | 20260505 07:25:11.735 | INFO | Serializing RTMetrics to json | 20260505 07:25:11.738 | INFO | Serialization done, swapping tmp file now... | 20260505 07:25:11.740 | INFO | RTMetrics done | 20260505 07:25:11.880 | INFO | Finished a round of Baz IO: Wrote 2.43 GiB in 3.439 seconds | 20260505 07:25:11.880 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 29.97% / 0.03% / 70.00% | 20260505 07:25:11.880 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.66741% / 7.54992e-05% / 5.96046e-05% | 20260505 07:25:11.880 | INFO | Baz IO overall: | 20260505 07:25:11.880 | INFO | BytesWritten : 760157056883 | 20260505 07:25:11.880 | INFO | NumEvents : 360621651524 | 20260505 07:25:13.446 | INFO | Analyzing chunk frames = [1396736,1397248) | 20260505 07:25:13.446 | INFO | MemInfo: { | 20260505 07:25:13.446 | INFO | "gpu" : " 17 GB", | 20260505 07:25:13.446 | INFO | "huge" : "", | 20260505 07:25:13.446 | INFO | "huge_shared" : " 27 GB", | 20260505 07:25:13.446 | INFO | "malloc" : " 14 GB", | 20260505 07:25:13.446 | INFO | "pinned" : " 29 GB" | 20260505 07:25:13.447 | INFO | } | 20260505 07:25:13.447 | INFO | Perf report for the last 5.13921 seconds: | 20260505 07:25:13.447 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:25:13.447 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:13.447 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:13.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:13.447 | INFO | ANALYSIS : 65.5%, 2.9, 99.8ms, 0 | 20260505 07:25:13.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:13.447 | INFO | RT_METRICS : 46.1%, 1.0, 24.1ms, 0 | 20260505 07:25:13.447 | INFO | PRE_HQ : 64.1%, 1.9, 62.6ms, 0 | 20260505 07:25:13.447 | INFO | BAZWRITER : 66.2%, 4.2, 141.7ms, 0 | 20260505 07:25:13.447 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 07:25:13.447 | INFO | Uploaded 36GiB to GPU in 2.2351s (16.1067GiB/s) | 20260505 07:25:13.447 | INFO | Downloaded 38GiB to GPU in 2.35141s (16.1605GiB/s) | 20260505 07:25:18.586 | INFO | Analyzing chunk frames = [1397248,1397760) | 20260505 07:25:18.586 | INFO | MemInfo: { | 20260505 07:25:18.586 | INFO | "gpu" : " 17 GB", | 20260505 07:25:18.586 | INFO | "huge" : "", | 20260505 07:25:18.586 | INFO | "huge_shared" : " 27 GB", | 20260505 07:25:18.586 | INFO | "malloc" : " 14 GB", | 20260505 07:25:18.586 | INFO | "pinned" : " 29 GB" | 20260505 07:25:18.586 | INFO | } | 20260505 07:25:18.586 | INFO | Perf report for the last 5.13953 seconds: | 20260505 07:25:18.586 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:25:18.586 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:18.586 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:18.586 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:18.586 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 07:25:18.586 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:18.586 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:18.586 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:25:18.586 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:18.586 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:18.586 | INFO | Uploaded 36GiB to GPU in 2.13593s (16.8545GiB/s) | 20260505 07:25:18.586 | INFO | Downloaded 34GiB to GPU in 2.04342s (16.6388GiB/s) | 20260505 07:25:23.725 | INFO | Analyzing chunk frames = [1397760,1398272) | 20260505 07:25:23.725 | INFO | MemInfo: { | 20260505 07:25:23.725 | INFO | "gpu" : " 17 GB", | 20260505 07:25:23.725 | INFO | "huge" : "", | 20260505 07:25:23.725 | INFO | "huge_shared" : " 27 GB", | 20260505 07:25:23.725 | INFO | "malloc" : " 14 GB", | 20260505 07:25:23.725 | INFO | "pinned" : " 29 GB" | 20260505 07:25:23.725 | INFO | } | 20260505 07:25:23.725 | INFO | Perf report for the last 5.13909 seconds: | 20260505 07:25:23.725 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:25:23.725 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:23.725 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:23.725 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:23.725 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 07:25:23.725 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:23.726 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:23.726 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:25:23.726 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:23.726 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:23.726 | INFO | Uploaded 36GiB to GPU in 2.15133s (16.7339GiB/s) | 20260505 07:25:23.726 | INFO | Downloaded 34GiB to GPU in 2.05797s (16.5211GiB/s) | 20260505 07:25:28.865 | INFO | Analyzing chunk frames = [1398272,1398784) | 20260505 07:25:28.865 | INFO | MemInfo: { | 20260505 07:25:28.865 | INFO | "gpu" : " 17 GB", | 20260505 07:25:28.865 | INFO | "huge" : "", | 20260505 07:25:28.865 | INFO | "huge_shared" : " 27 GB", | 20260505 07:25:28.865 | INFO | "malloc" : " 14 GB", | 20260505 07:25:28.865 | INFO | "pinned" : " 29 GB" | 20260505 07:25:28.865 | INFO | } | 20260505 07:25:28.865 | INFO | Perf report for the last 5.13949 seconds: | 20260505 07:25:28.865 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:25:28.865 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:28.865 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:28.865 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:28.865 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 07:25:28.865 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:28.865 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:28.865 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:25:28.865 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:28.865 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:28.865 | INFO | Uploaded 36GiB to GPU in 2.11645s (17.0096GiB/s) | 20260505 07:25:28.865 | INFO | Downloaded 34GiB to GPU in 2.03355s (16.7196GiB/s) | 20260505 07:25:34.004 | INFO | Analyzing chunk frames = [1398784,1399296) | 20260505 07:25:34.004 | INFO | MemInfo: { | 20260505 07:25:34.004 | INFO | "gpu" : " 17 GB", | 20260505 07:25:34.004 | INFO | "huge" : "", | 20260505 07:25:34.004 | INFO | "huge_shared" : " 27 GB", | 20260505 07:25:34.004 | INFO | "malloc" : " 14 GB", | 20260505 07:25:34.004 | INFO | "pinned" : " 29 GB" | 20260505 07:25:34.004 | INFO | } | 20260505 07:25:34.004 | INFO | Perf report for the last 5.13908 seconds: | 20260505 07:25:34.004 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:25:34.004 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:34.004 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:34.004 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:34.004 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 07:25:34.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:34.004 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:34.004 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 07:25:34.004 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:34.004 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:34.004 | INFO | Uploaded 36GiB to GPU in 2.15693s (16.6904GiB/s) | 20260505 07:25:34.004 | INFO | Downloaded 34GiB to GPU in 2.04858s (16.5968GiB/s) | 20260505 07:25:39.145 | INFO | Analyzing chunk frames = [1399296,1399808) | 20260505 07:25:39.145 | INFO | MemInfo: { | 20260505 07:25:39.145 | INFO | "gpu" : " 17 GB", | 20260505 07:25:39.145 | INFO | "huge" : "", | 20260505 07:25:39.145 | INFO | "huge_shared" : " 27 GB", | 20260505 07:25:39.145 | INFO | "malloc" : " 14 GB", | 20260505 07:25:39.145 | INFO | "pinned" : " 29 GB" | 20260505 07:25:39.145 | INFO | } | 20260505 07:25:39.145 | INFO | Perf report for the last 5.14085 seconds: | 20260505 07:25:39.145 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:25:39.145 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:39.145 | INFO | PREP_TRACE : 0.1%, 1.0, 0.0ms, 0 | 20260505 07:25:39.145 | INFO | SAVE_TRACE : 0.1%, 1.0, 0.1ms, 0 | 20260505 07:25:39.145 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 07:25:39.145 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:39.145 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:39.145 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:25:39.145 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:39.145 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:39.145 | INFO | Uploaded 36GiB to GPU in 2.14275s (16.8008GiB/s) | 20260505 07:25:39.145 | INFO | Downloaded 34GiB to GPU in 2.05761s (16.5241GiB/s) | 20260505 07:25:44.283 | INFO | Analyzing chunk frames = [1399808,1400320) | 20260505 07:25:44.283 | INFO | MemInfo: { | 20260505 07:25:44.283 | INFO | "gpu" : " 17 GB", | 20260505 07:25:44.283 | INFO | "huge" : "", | 20260505 07:25:44.283 | INFO | "huge_shared" : " 27 GB", | 20260505 07:25:44.283 | INFO | "malloc" : " 14 GB", | 20260505 07:25:44.283 | INFO | "pinned" : " 29 GB" | 20260505 07:25:44.283 | INFO | } | 20260505 07:25:44.283 | INFO | Perf report for the last 5.13828 seconds: | 20260505 07:25:44.283 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:25:44.283 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:44.283 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:44.283 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:44.283 | INFO | ANALYSIS : 64.5%, 3.0, 100.6ms, 0 | 20260505 07:25:44.283 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:44.283 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:44.283 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260505 07:25:44.283 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:44.283 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:44.283 | INFO | Uploaded 36GiB to GPU in 2.21816s (16.2297GiB/s) | 20260505 07:25:44.283 | INFO | Downloaded 34GiB to GPU in 2.16481s (15.7058GiB/s) | 20260505 07:25:49.422 | INFO | Analyzing chunk frames = [1400320,1400832) | 20260505 07:25:49.422 | INFO | MemInfo: { | 20260505 07:25:49.422 | INFO | "gpu" : " 17 GB", | 20260505 07:25:49.422 | INFO | "huge" : "", | 20260505 07:25:49.422 | INFO | "huge_shared" : " 27 GB", | 20260505 07:25:49.422 | INFO | "malloc" : " 14 GB", | 20260505 07:25:49.422 | INFO | "pinned" : " 29 GB" | 20260505 07:25:49.422 | INFO | } | 20260505 07:25:49.422 | INFO | Perf report for the last 5.13875 seconds: | 20260505 07:25:49.422 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:25:49.422 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:49.422 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:49.422 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:49.422 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 07:25:49.422 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:49.422 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:49.422 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 07:25:49.422 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:49.422 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:49.422 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 07:25:49.422 | INFO | Uploaded 36GiB to GPU in 2.17s (16.5898GiB/s) | 20260505 07:25:49.422 | INFO | Total mem usage: 15.3 GiB | 20260505 07:25:49.422 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 07:25:49.422 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.1%) | 20260505 07:25:49.422 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 07:25:49.422 | INFO | Downloaded 34GiB to GPU in 2.02819s (16.7638GiB/s) | 20260505 07:25:49.422 | INFO | High Water Breakdown: | 20260505 07:25:49.422 | INFO | 60.7% : 8.76 GiB ( 93.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 07:25:49.422 | INFO | 39.2% : 5.66 GiB ( 85.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 07:25:49.422 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 07:25:49.422 | INFO | ------------------------------------------ | 20260505 07:25:49.422 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:25:49.422 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 07:25:49.422 | INFO | Total mem usage: 26.5 GiB | 20260505 07:25:49.422 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 07:25:49.422 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260505 07:25:49.422 | INFO | Unused Cache: 1.6 GiB (Waste: 6.05%) | 20260505 07:25:49.422 | INFO | High Water Breakdown: | 20260505 07:25:49.422 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:25:49.422 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:25:49.423 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:25:49.423 | INFO | 2% : 523 MiB ( 99.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:25:49.423 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:25:49.423 | INFO | 0.557% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:25:49.423 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 07:25:49.423 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 07:25:49.423 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:25:49.423 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 07:25:49.423 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 07:25:49.423 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 07:25:49.423 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 07:25:49.423 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 07:25:49.423 | INFO | ------------------------------------------ | 20260505 07:25:49.423 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:25:49.423 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 07:25:49.423 | INFO | Total mem usage: 25.6 GiB | 20260505 07:25:49.423 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 07:25:49.423 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 07:25:49.423 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 07:25:49.423 | INFO | High Water Breakdown: | 20260505 07:25:49.423 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 07:25:49.423 | INFO | ------------------------------------------ | 20260505 07:25:49.423 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:25:49.423 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 07:25:49.423 | INFO | Total mem usage: 15.7 GiB | 20260505 07:25:49.423 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 07:25:49.423 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 07:25:49.423 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 07:25:49.423 | INFO | High Water Breakdown: | 20260505 07:25:49.423 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 07:25:49.423 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 07:25:49.423 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 07:25:49.423 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 07:25:49.423 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 07:25:49.423 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 07:25:49.423 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 07:25:49.423 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 07:25:49.423 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 07:25:49.423 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:25:49.423 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 07:25:49.423 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:25:49.423 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 07:25:49.423 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 07:25:49.423 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 07:25:49.423 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 07:25:49.423 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:25:49.423 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 07:25:49.423 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:25:49.424 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:25:49.424 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:25:49.424 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 07:25:49.424 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 07:25:49.424 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:25:49.424 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 07:25:49.424 | INFO | ------------------------------------------ | 20260505 07:25:49.424 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:25:49.424 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 07:25:49.424 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:25:49.424 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:25:49.424 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.51e+01, 1.37e+01 / 2.28e+00 | 20260505 07:25:49.424 | INFO | PerfReport: 0.29%, AggregateStats -- 9.38e-02 / 1.03e-01, 1.11e-02 / 5.75e-03 | 20260505 07:25:49.424 | INFO | PerfReport: 11.61%, Baseline -- 3.82e+00 / 4.45e+00, 6.25e-01 / 2.95e-01 | 20260505 07:25:49.424 | INFO | PerfReport: 3.30%, Histogram -- 1.08e+00 / 1.44e+00, 2.12e-01 / 1.34e-01 | 20260505 07:25:49.424 | INFO | PerfReport: 15.47%, DME -- 4.07e+01 / 4.44e+01, 4.02e+00 / 1.44e+00 | 20260505 07:25:49.424 | INFO | PerfReport: 56.28%, FrameLabeling -- 1.85e+01 / 1.93e+01, 2.03e+00 / 1.75e+00 | 20260505 07:25:49.424 | INFO | PerfReport: 7.62%, PulseAccumulating -- 2.50e+00 / 3.16e+00, 4.43e-01 / 2.68e-01 | 20260505 07:25:49.424 | INFO | PerfReport: 3.99%, Metrics -- 1.31e+00 / 1.71e+00, 3.69e-01 / 1.86e-01 | 20260505 07:25:49.424 | INFO | PerfReport: 1.44%, PulseSerializing -- 4.74e-01 / 5.50e-01, 9.13e-02 / 3.10e-02 | 20260505 07:25:49.424 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 07:25:49.424 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:25:49.424 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:25:49.424 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.24e+01 / 0.00e+00 | 20260505 07:25:49.424 | INFO | PerfReport: 22.26%, Upload -- 2.26e+01 / 0.00e+00, 7.94e+00 / 0.00e+00 | 20260505 07:25:49.424 | INFO | PerfReport: 21.54%, Download -- 2.18e+01 / 0.00e+00, 7.87e+00 / 0.00e+00 | 20260505 07:25:49.424 | INFO | PerfReport: 32.43%, Compute -- 3.29e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 07:25:49.424 | INFO | PerfReport: 1.50%, LockUpload -- 1.52e+00 / 0.00e+00, 7.12e+00 / 0.00e+00 | 20260505 07:25:49.424 | INFO | PerfReport: 0.89%, LockDownload -- 9.01e-01 / 0.00e+00, 2.34e+00 / 0.00e+00 | 20260505 07:25:49.424 | INFO | PerfReport: 21.36%, LockCompute -- 2.16e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260505 07:25:49.424 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.70e-02 / 0.00e+00 | 20260505 07:25:49.555 | INFO | Starting a round of Baz IO | 20260505 07:25:52.840 | INFO | Populating RTMetrics report | 20260505 07:25:52.841 | INFO | RTMetrics report populated | 20260505 07:25:52.841 | INFO | Opening tmp RTMetrics json file | 20260505 07:25:52.848 | INFO | Serializing RTMetrics to json | 20260505 07:25:52.850 | INFO | Serialization done, swapping tmp file now... | 20260505 07:25:52.851 | INFO | RTMetrics done | 20260505 07:25:52.980 | INFO | Finished a round of Baz IO: Wrote 2.43 GiB in 3.425 seconds | 20260505 07:25:52.980 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 29.94% / 0.03% / 70.03% | 20260505 07:25:52.980 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.65762% / 3.97364e-05% / 5.96046e-05% | 20260505 07:25:52.980 | INFO | Baz IO overall: | 20260505 07:25:52.980 | INFO | BytesWritten : 762766317120 | 20260505 07:25:52.980 | INFO | NumEvents : 361870527170 | 20260505 07:25:54.561 | INFO | Analyzing chunk frames = [1400832,1401344) | 20260505 07:25:54.561 | INFO | MemInfo: { | 20260505 07:25:54.561 | INFO | "gpu" : " 17 GB", | 20260505 07:25:54.561 | INFO | "huge" : "", | 20260505 07:25:54.561 | INFO | "huge_shared" : " 27 GB", | 20260505 07:25:54.561 | INFO | "malloc" : " 14 GB", | 20260505 07:25:54.561 | INFO | "pinned" : " 29 GB" | 20260505 07:25:54.561 | INFO | } | 20260505 07:25:54.561 | INFO | Perf report for the last 5.13968 seconds: | 20260505 07:25:54.562 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:25:54.562 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:54.562 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:54.562 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:54.562 | INFO | ANALYSIS : 65.4%, 2.9, 99.2ms, 0 | 20260505 07:25:54.562 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:54.562 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260505 07:25:54.562 | INFO | PRE_HQ : 64.1%, 1.9, 63.1ms, 0 | 20260505 07:25:54.562 | INFO | BAZWRITER : 66.0%, 4.2, 140.9ms, 0 | 20260505 07:25:54.562 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 07:25:54.562 | INFO | Uploaded 36GiB to GPU in 2.24307s (16.0494GiB/s) | 20260505 07:25:54.562 | INFO | Downloaded 38GiB to GPU in 2.32642s (16.3341GiB/s) | 20260505 07:25:59.701 | INFO | Analyzing chunk frames = [1401344,1401856) | 20260505 07:25:59.701 | INFO | MemInfo: { | 20260505 07:25:59.701 | INFO | "gpu" : " 17 GB", | 20260505 07:25:59.701 | INFO | "huge" : "", | 20260505 07:25:59.701 | INFO | "huge_shared" : " 27 GB", | 20260505 07:25:59.701 | INFO | "malloc" : " 14 GB", | 20260505 07:25:59.701 | INFO | "pinned" : " 29 GB" | 20260505 07:25:59.701 | INFO | } | 20260505 07:25:59.701 | INFO | Perf report for the last 5.13941 seconds: | 20260505 07:25:59.701 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:25:59.701 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:59.701 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:59.701 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:59.701 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 07:25:59.701 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:59.701 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:59.701 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:25:59.701 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:59.701 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:25:59.701 | INFO | Uploaded 36GiB to GPU in 2.13306s (16.8771GiB/s) | 20260505 07:25:59.701 | INFO | Downloaded 34GiB to GPU in 2.04326s (16.64GiB/s) | 20260505 07:26:04.840 | INFO | Analyzing chunk frames = [1401856,1402368) | 20260505 07:26:04.840 | INFO | MemInfo: { | 20260505 07:26:04.840 | INFO | "gpu" : " 17 GB", | 20260505 07:26:04.840 | INFO | "huge" : "", | 20260505 07:26:04.840 | INFO | "huge_shared" : " 27 GB", | 20260505 07:26:04.840 | INFO | "malloc" : " 14 GB", | 20260505 07:26:04.840 | INFO | "pinned" : " 29 GB" | 20260505 07:26:04.840 | INFO | } | 20260505 07:26:04.840 | INFO | Perf report for the last 5.13919 seconds: | 20260505 07:26:04.840 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:26:04.840 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:04.840 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:04.840 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:04.840 | INFO | ANALYSIS : 63.4%, 3.0, 98.6ms, 0 | 20260505 07:26:04.840 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:04.840 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:04.840 | INFO | PRE_HQ : 30.3%, 1.0, 15.8ms, 0 | 20260505 07:26:04.840 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:04.840 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:04.840 | INFO | Uploaded 36GiB to GPU in 2.15209s (16.7279GiB/s) | 20260505 07:26:04.840 | INFO | Downloaded 34GiB to GPU in 2.05393s (16.5536GiB/s) | 20260505 07:26:09.979 | INFO | Analyzing chunk frames = [1402368,1402880) | 20260505 07:26:09.979 | INFO | MemInfo: { | 20260505 07:26:09.979 | INFO | "gpu" : " 17 GB", | 20260505 07:26:09.979 | INFO | "huge" : "", | 20260505 07:26:09.979 | INFO | "huge_shared" : " 27 GB", | 20260505 07:26:09.979 | INFO | "malloc" : " 14 GB", | 20260505 07:26:09.979 | INFO | "pinned" : " 29 GB" | 20260505 07:26:09.979 | INFO | } | 20260505 07:26:09.979 | INFO | Perf report for the last 5.13912 seconds: | 20260505 07:26:09.979 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:26:09.979 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:09.979 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:09.979 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:09.979 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 07:26:09.979 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:09.980 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:09.980 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 07:26:09.980 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:09.980 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:09.980 | INFO | Uploaded 36GiB to GPU in 2.15472s (16.7075GiB/s) | 20260505 07:26:09.980 | INFO | Downloaded 34GiB to GPU in 2.04085s (16.6597GiB/s) | 20260505 07:26:15.118 | INFO | Analyzing chunk frames = [1402880,1403392) | 20260505 07:26:15.119 | INFO | MemInfo: { | 20260505 07:26:15.119 | INFO | "gpu" : " 17 GB", | 20260505 07:26:15.119 | INFO | "huge" : "", | 20260505 07:26:15.119 | INFO | "huge_shared" : " 27 GB", | 20260505 07:26:15.119 | INFO | "malloc" : " 14 GB", | 20260505 07:26:15.119 | INFO | "pinned" : " 29 GB" | 20260505 07:26:15.119 | INFO | } | 20260505 07:26:15.119 | INFO | Perf report for the last 5.1394 seconds: | 20260505 07:26:15.119 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:26:15.119 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:15.119 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:15.119 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:15.119 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 07:26:15.119 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:15.119 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:15.119 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:26:15.119 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:15.119 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:15.119 | INFO | Uploaded 36GiB to GPU in 2.16131s (16.6566GiB/s) | 20260505 07:26:15.119 | INFO | Downloaded 34GiB to GPU in 2.05053s (16.5811GiB/s) | 20260505 07:26:20.260 | INFO | Analyzing chunk frames = [1403392,1403904) | 20260505 07:26:20.260 | INFO | MemInfo: { | 20260505 07:26:20.260 | INFO | "gpu" : " 17 GB", | 20260505 07:26:20.260 | INFO | "huge" : "", | 20260505 07:26:20.260 | INFO | "huge_shared" : " 27 GB", | 20260505 07:26:20.260 | INFO | "malloc" : " 14 GB", | 20260505 07:26:20.260 | INFO | "pinned" : " 29 GB" | 20260505 07:26:20.260 | INFO | } | 20260505 07:26:20.260 | INFO | Perf report for the last 5.14119 seconds: | 20260505 07:26:20.260 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:26:20.260 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:20.260 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:20.260 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:20.260 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 07:26:20.260 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:20.260 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:20.260 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 07:26:20.260 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:20.260 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:20.260 | INFO | Uploaded 36GiB to GPU in 2.1515s (16.7325GiB/s) | 20260505 07:26:20.260 | INFO | Downloaded 34GiB to GPU in 2.05861s (16.516GiB/s) | 20260505 07:26:25.397 | INFO | Analyzing chunk frames = [1403904,1404416) | 20260505 07:26:25.397 | INFO | MemInfo: { | 20260505 07:26:25.397 | INFO | "gpu" : " 17 GB", | 20260505 07:26:25.397 | INFO | "huge" : "", | 20260505 07:26:25.397 | INFO | "huge_shared" : " 27 GB", | 20260505 07:26:25.397 | INFO | "malloc" : " 14 GB", | 20260505 07:26:25.397 | INFO | "pinned" : " 29 GB" | 20260505 07:26:25.397 | INFO | } | 20260505 07:26:25.397 | INFO | Perf report for the last 5.13747 seconds: | 20260505 07:26:25.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:26:25.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:25.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:25.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:25.398 | INFO | ANALYSIS : 64.7%, 3.0, 100.9ms, 0 | 20260505 07:26:25.398 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:25.398 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:25.398 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260505 07:26:25.398 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:25.398 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:25.398 | INFO | Uploaded 36GiB to GPU in 2.22624s (16.1708GiB/s) | 20260505 07:26:25.398 | INFO | Downloaded 34GiB to GPU in 2.15007s (15.8135GiB/s) | 20260505 07:26:30.537 | INFO | Analyzing chunk frames = [1404416,1404928) | 20260505 07:26:30.537 | INFO | MemInfo: { | 20260505 07:26:30.537 | INFO | "gpu" : " 17 GB", | 20260505 07:26:30.537 | INFO | "huge" : "", | 20260505 07:26:30.537 | INFO | "huge_shared" : " 27 GB", | 20260505 07:26:30.537 | INFO | "malloc" : " 14 GB", | 20260505 07:26:30.537 | INFO | "pinned" : " 29 GB" | 20260505 07:26:30.537 | INFO | } | 20260505 07:26:30.537 | INFO | Perf report for the last 5.14005 seconds: | 20260505 07:26:30.537 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:26:30.537 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:30.537 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:30.537 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:30.537 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 07:26:30.537 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:30.537 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:30.537 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 07:26:30.537 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:30.537 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:30.538 | INFO | Uploaded 36GiB to GPU in 2.14917s (16.7506GiB/s) | 20260505 07:26:30.538 | INFO | Downloaded 34GiB to GPU in 2.03205s (16.7318GiB/s) | 20260505 07:26:30.674 | INFO | Starting a round of Baz IO | 20260505 07:26:33.944 | INFO | Populating RTMetrics report | 20260505 07:26:33.944 | INFO | RTMetrics report populated | 20260505 07:26:33.944 | INFO | Opening tmp RTMetrics json file | 20260505 07:26:33.953 | INFO | Serializing RTMetrics to json | 20260505 07:26:33.955 | INFO | Serialization done, swapping tmp file now... | 20260505 07:26:33.956 | INFO | RTMetrics done | 20260505 07:26:34.082 | INFO | Finished a round of Baz IO: Wrote 2.427 GiB in 3.408 seconds | 20260505 07:26:34.082 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 29.91% / 0.03% / 70.06% | 20260505 07:26:34.082 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.62551% / 3.17891e-05% / 7.15256e-05% | 20260505 07:26:34.082 | INFO | Baz IO overall: | 20260505 07:26:34.082 | INFO | BytesWritten : 765371967774 | 20260505 07:26:34.082 | INFO | NumEvents : 363116911848 | 20260505 07:26:35.676 | INFO | Analyzing chunk frames = [1404928,1405440) | 20260505 07:26:35.676 | INFO | MemInfo: { | 20260505 07:26:35.676 | INFO | "gpu" : " 17 GB", | 20260505 07:26:35.676 | INFO | "huge" : "", | 20260505 07:26:35.676 | INFO | "huge_shared" : " 27 GB", | 20260505 07:26:35.676 | INFO | "malloc" : " 14 GB", | 20260505 07:26:35.676 | INFO | "pinned" : " 29 GB" | 20260505 07:26:35.676 | INFO | } | 20260505 07:26:35.676 | INFO | Perf report for the last 5.13894 seconds: | 20260505 07:26:35.676 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:26:35.676 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:35.676 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:35.676 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:35.676 | INFO | ANALYSIS : 65.1%, 2.9, 99.1ms, 0 | 20260505 07:26:35.676 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:35.676 | INFO | RT_METRICS : 46.2%, 1.0, 24.3ms, 0 | 20260505 07:26:35.676 | INFO | PRE_HQ : 63.8%, 1.9, 62.9ms, 0 | 20260505 07:26:35.676 | INFO | BAZWRITER : 65.7%, 4.2, 141.4ms, 0 | 20260505 07:26:35.676 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 07:26:35.676 | INFO | Uploaded 36GiB to GPU in 2.22239s (16.1988GiB/s) | 20260505 07:26:35.677 | INFO | Downloaded 38GiB to GPU in 2.31702s (16.4004GiB/s) | 20260505 07:26:40.815 | INFO | Analyzing chunk frames = [1405440,1405952) | 20260505 07:26:40.815 | INFO | MemInfo: { | 20260505 07:26:40.815 | INFO | "gpu" : " 17 GB", | 20260505 07:26:40.815 | INFO | "huge" : "", | 20260505 07:26:40.815 | INFO | "huge_shared" : " 27 GB", | 20260505 07:26:40.815 | INFO | "malloc" : " 14 GB", | 20260505 07:26:40.815 | INFO | "pinned" : " 29 GB" | 20260505 07:26:40.815 | INFO | } | 20260505 07:26:40.815 | INFO | Perf report for the last 5.13904 seconds: | 20260505 07:26:40.815 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:26:40.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:40.816 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:40.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:40.816 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 07:26:40.816 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:40.816 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:40.816 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 07:26:40.816 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:40.816 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:40.816 | INFO | Uploaded 36GiB to GPU in 2.13173s (16.8877GiB/s) | 20260505 07:26:40.816 | INFO | Downloaded 34GiB to GPU in 2.04556s (16.6213GiB/s) | 20260505 07:26:45.955 | INFO | Analyzing chunk frames = [1405952,1406464) | 20260505 07:26:45.955 | INFO | MemInfo: { | 20260505 07:26:45.955 | INFO | "gpu" : " 17 GB", | 20260505 07:26:45.955 | INFO | "huge" : "", | 20260505 07:26:45.955 | INFO | "huge_shared" : " 27 GB", | 20260505 07:26:45.955 | INFO | "malloc" : " 14 GB", | 20260505 07:26:45.955 | INFO | "pinned" : " 29 GB" | 20260505 07:26:45.955 | INFO | } | 20260505 07:26:45.955 | INFO | Perf report for the last 5.13948 seconds: | 20260505 07:26:45.955 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:26:45.955 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:45.955 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:45.955 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:45.955 | INFO | ANALYSIS : 63.4%, 3.0, 98.6ms, 0 | 20260505 07:26:45.955 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:45.955 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:45.955 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 07:26:45.955 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:45.955 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:45.955 | INFO | Uploaded 36GiB to GPU in 2.14998s (16.7444GiB/s) | 20260505 07:26:45.955 | INFO | Downloaded 34GiB to GPU in 2.05758s (16.5243GiB/s) | 20260505 07:26:51.094 | INFO | Analyzing chunk frames = [1406464,1406976) | 20260505 07:26:51.094 | INFO | MemInfo: { | 20260505 07:26:51.094 | INFO | "gpu" : " 17 GB", | 20260505 07:26:51.094 | INFO | "huge" : "", | 20260505 07:26:51.094 | INFO | "huge_shared" : " 27 GB", | 20260505 07:26:51.094 | INFO | "malloc" : " 14 GB", | 20260505 07:26:51.094 | INFO | "pinned" : " 29 GB" | 20260505 07:26:51.094 | INFO | } | 20260505 07:26:51.094 | INFO | Perf report for the last 5.1392 seconds: | 20260505 07:26:51.094 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:26:51.094 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:51.094 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:51.094 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:51.094 | INFO | ANALYSIS : 63.5%, 3.0, 98.6ms, 0 | 20260505 07:26:51.094 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:51.094 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:51.094 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 07:26:51.094 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:51.094 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:51.094 | INFO | Uploaded 36GiB to GPU in 2.1355s (16.8579GiB/s) | 20260505 07:26:51.094 | INFO | Downloaded 34GiB to GPU in 2.03217s (16.7309GiB/s) | 20260505 07:26:56.233 | INFO | Analyzing chunk frames = [1406976,1407488) | 20260505 07:26:56.233 | INFO | MemInfo: { | 20260505 07:26:56.233 | INFO | "gpu" : " 17 GB", | 20260505 07:26:56.233 | INFO | "huge" : "", | 20260505 07:26:56.233 | INFO | "huge_shared" : " 27 GB", | 20260505 07:26:56.233 | INFO | "malloc" : " 14 GB", | 20260505 07:26:56.233 | INFO | "pinned" : " 29 GB" | 20260505 07:26:56.234 | INFO | } | 20260505 07:26:56.234 | INFO | Perf report for the last 5.13951 seconds: | 20260505 07:26:56.234 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:26:56.234 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:56.234 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:56.234 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:56.234 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 07:26:56.234 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:56.234 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:56.234 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 07:26:56.234 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:56.234 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:26:56.234 | INFO | Uploaded 36GiB to GPU in 2.19558s (16.3966GiB/s) | 20260505 07:26:56.234 | INFO | Downloaded 34GiB to GPU in 2.06193s (16.4894GiB/s) | 20260505 07:27:01.375 | INFO | Analyzing chunk frames = [1407488,1408000) | 20260505 07:27:01.375 | INFO | MemInfo: { | 20260505 07:27:01.375 | INFO | "gpu" : " 17 GB", | 20260505 07:27:01.375 | INFO | "huge" : "", | 20260505 07:27:01.375 | INFO | "huge_shared" : " 27 GB", | 20260505 07:27:01.375 | INFO | "malloc" : " 14 GB", | 20260505 07:27:01.375 | INFO | "pinned" : " 29 GB" | 20260505 07:27:01.375 | INFO | } | 20260505 07:27:01.375 | INFO | Perf report for the last 5.1414 seconds: | 20260505 07:27:01.375 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:27:01.375 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:01.375 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:01.375 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:01.375 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 07:27:01.375 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:01.375 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:01.375 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 07:27:01.375 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:01.375 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:01.375 | INFO | Uploaded 36GiB to GPU in 2.15776s (16.684GiB/s) | 20260505 07:27:01.375 | INFO | Downloaded 34GiB to GPU in 2.05743s (16.5255GiB/s) | 20260505 07:27:06.512 | INFO | Analyzing chunk frames = [1408000,1408512) | 20260505 07:27:06.512 | INFO | MemInfo: { | 20260505 07:27:06.512 | INFO | "gpu" : " 17 GB", | 20260505 07:27:06.512 | INFO | "huge" : "", | 20260505 07:27:06.512 | INFO | "huge_shared" : " 27 GB", | 20260505 07:27:06.512 | INFO | "malloc" : " 14 GB", | 20260505 07:27:06.512 | INFO | "pinned" : " 29 GB" | 20260505 07:27:06.512 | INFO | } | 20260505 07:27:06.512 | INFO | Perf report for the last 5.13699 seconds: | 20260505 07:27:06.512 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:27:06.512 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:06.512 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:06.512 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:06.512 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 07:27:06.512 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:06.512 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:06.512 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260505 07:27:06.512 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:06.512 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:06.512 | INFO | Uploaded 36GiB to GPU in 2.20065s (16.3588GiB/s) | 20260505 07:27:06.512 | INFO | Downloaded 34GiB to GPU in 2.16812s (15.6818GiB/s) | 20260505 07:27:11.651 | INFO | Analyzing chunk frames = [1408512,1409024) | 20260505 07:27:11.651 | INFO | MemInfo: { | 20260505 07:27:11.651 | INFO | "gpu" : " 17 GB", | 20260505 07:27:11.651 | INFO | "huge" : "", | 20260505 07:27:11.651 | INFO | "huge_shared" : " 27 GB", | 20260505 07:27:11.651 | INFO | "malloc" : " 14 GB", | 20260505 07:27:11.651 | INFO | "pinned" : " 29 GB" | 20260505 07:27:11.651 | INFO | } | 20260505 07:27:11.651 | INFO | Perf report for the last 5.13931 seconds: | 20260505 07:27:11.651 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:27:11.651 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:11.651 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:11.651 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:11.652 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 07:27:11.652 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:11.652 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:11.652 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:27:11.652 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:11.652 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:11.652 | INFO | Uploaded 36GiB to GPU in 2.16263s (16.6464GiB/s) | 20260505 07:27:11.652 | INFO | Downloaded 34GiB to GPU in 2.0319s (16.7331GiB/s) | 20260505 07:27:11.652 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 07:27:11.652 | INFO | Total mem usage: 15.3 GiB | 20260505 07:27:11.652 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 07:27:11.652 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.38%) | 20260505 07:27:11.652 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 07:27:11.652 | INFO | High Water Breakdown: | 20260505 07:27:11.652 | INFO | 60.6% : 8.73 GiB ( 93.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 07:27:11.652 | INFO | 39.3% : 5.66 GiB ( 85.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 07:27:11.652 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 07:27:11.652 | INFO | ------------------------------------------ | 20260505 07:27:11.652 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:27:11.652 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 07:27:11.652 | INFO | Total mem usage: 26.5 GiB | 20260505 07:27:11.652 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 07:27:11.652 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 07:27:11.652 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 07:27:11.652 | INFO | High Water Breakdown: | 20260505 07:27:11.652 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:27:11.652 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:27:11.652 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:27:11.652 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:27:11.652 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:27:11.652 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:27:11.652 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 07:27:11.652 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 07:27:11.652 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:27:11.652 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 07:27:11.652 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 07:27:11.652 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 07:27:11.652 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 07:27:11.652 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 07:27:11.652 | INFO | ------------------------------------------ | 20260505 07:27:11.652 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:27:11.652 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 07:27:11.652 | INFO | Total mem usage: 25.6 GiB | 20260505 07:27:11.652 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 07:27:11.652 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 07:27:11.652 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 07:27:11.652 | INFO | High Water Breakdown: | 20260505 07:27:11.652 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 07:27:11.652 | INFO | ------------------------------------------ | 20260505 07:27:11.652 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:27:11.652 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 07:27:11.652 | INFO | Total mem usage: 15.7 GiB | 20260505 07:27:11.652 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 07:27:11.652 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 07:27:11.652 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 07:27:11.652 | INFO | High Water Breakdown: | 20260505 07:27:11.652 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 07:27:11.652 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 07:27:11.652 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 07:27:11.652 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 07:27:11.652 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 07:27:11.653 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 07:27:11.653 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 07:27:11.653 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 07:27:11.653 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 07:27:11.653 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:27:11.653 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 07:27:11.653 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:27:11.653 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 07:27:11.653 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 07:27:11.653 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 07:27:11.653 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 07:27:11.653 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:27:11.653 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 07:27:11.653 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:27:11.653 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:27:11.653 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:27:11.653 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 07:27:11.653 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 07:27:11.653 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:27:11.653 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 07:27:11.653 | INFO | ------------------------------------------ | 20260505 07:27:11.653 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:27:11.653 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 07:27:11.653 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:27:11.653 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:27:11.653 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.51e+01, 1.36e+01 / 2.25e+00 | 20260505 07:27:11.653 | INFO | PerfReport: 0.29%, AggregateStats -- 9.40e-02 / 1.03e-01, 1.05e-02 / 5.22e-03 | 20260505 07:27:11.653 | INFO | PerfReport: 11.61%, Baseline -- 3.81e+00 / 4.43e+00, 6.24e-01 / 2.71e-01 | 20260505 07:27:11.653 | INFO | PerfReport: 3.31%, Histogram -- 1.09e+00 / 1.44e+00, 2.12e-01 / 1.39e-01 | 20260505 07:27:11.653 | INFO | PerfReport: 15.46%, DME -- 4.06e+01 / 4.44e+01, 4.19e+00 / 1.39e+00 | 20260505 07:27:11.653 | INFO | PerfReport: 56.29%, FrameLabeling -- 1.85e+01 / 1.93e+01, 2.02e+00 / 1.75e+00 | 20260505 07:27:11.653 | INFO | PerfReport: 7.63%, PulseAccumulating -- 2.51e+00 / 3.18e+00, 4.47e-01 / 2.07e-01 | 20260505 07:27:11.653 | INFO | PerfReport: 3.97%, Metrics -- 1.31e+00 / 1.72e+00, 3.63e-01 / 1.36e-01 | 20260505 07:27:11.653 | INFO | PerfReport: 1.44%, PulseSerializing -- 4.73e-01 / 5.46e-01, 9.14e-02 / 2.52e-02 | 20260505 07:27:11.653 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 07:27:11.653 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:27:11.653 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:27:11.653 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.24e+01 / 0.00e+00 | 20260505 07:27:11.653 | INFO | PerfReport: 22.32%, Upload -- 2.26e+01 / 0.00e+00, 8.01e+00 / 0.00e+00 | 20260505 07:27:11.653 | INFO | PerfReport: 21.55%, Download -- 2.18e+01 / 0.00e+00, 7.89e+00 / 0.00e+00 | 20260505 07:27:11.653 | INFO | PerfReport: 32.43%, Compute -- 3.29e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 07:27:11.653 | INFO | PerfReport: 1.52%, LockUpload -- 1.54e+00 / 0.00e+00, 7.18e+00 / 0.00e+00 | 20260505 07:27:11.653 | INFO | PerfReport: 0.87%, LockDownload -- 8.77e-01 / 0.00e+00, 2.27e+00 / 0.00e+00 | 20260505 07:27:11.653 | INFO | PerfReport: 21.31%, LockCompute -- 2.16e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260505 07:27:11.653 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 1.95e-02 / 0.00e+00 | 20260505 07:27:11.787 | INFO | Starting a round of Baz IO | 20260505 07:27:15.057 | INFO | Populating RTMetrics report | 20260505 07:27:15.057 | INFO | RTMetrics report populated | 20260505 07:27:15.057 | INFO | Opening tmp RTMetrics json file | 20260505 07:27:15.058 | INFO | Serializing RTMetrics to json | 20260505 07:27:15.059 | INFO | Serialization done, swapping tmp file now... | 20260505 07:27:15.059 | INFO | RTMetrics done | 20260505 07:27:15.240 | INFO | Finished a round of Baz IO: Wrote 2.43 GiB in 3.453 seconds | 20260505 07:27:15.240 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 29.87% / 0.03% / 70.09% | 20260505 07:27:15.240 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.6199% / 4.76837e-05% / 9.13938e-05% | 20260505 07:27:15.240 | INFO | Baz IO overall: | 20260505 07:27:15.241 | INFO | BytesWritten : 767980991166 | 20260505 07:27:15.241 | INFO | NumEvents : 364365444328 | 20260505 07:27:16.791 | INFO | Analyzing chunk frames = [1409024,1409536) | 20260505 07:27:16.791 | INFO | MemInfo: { | 20260505 07:27:16.791 | INFO | "gpu" : " 17 GB", | 20260505 07:27:16.791 | INFO | "huge" : "", | 20260505 07:27:16.791 | INFO | "huge_shared" : " 27 GB", | 20260505 07:27:16.791 | INFO | "malloc" : " 14 GB", | 20260505 07:27:16.791 | INFO | "pinned" : " 29 GB" | 20260505 07:27:16.791 | INFO | } | 20260505 07:27:16.791 | INFO | Perf report for the last 5.13957 seconds: | 20260505 07:27:16.791 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:27:16.791 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:16.791 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:16.791 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:16.791 | INFO | ANALYSIS : 65.1%, 2.9, 98.8ms, 0 | 20260505 07:27:16.791 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:16.791 | INFO | RT_METRICS : 46.2%, 1.0, 24.2ms, 0 | 20260505 07:27:16.791 | INFO | PRE_HQ : 63.8%, 1.9, 63.2ms, 0 | 20260505 07:27:16.791 | INFO | BAZWRITER : 66.5%, 4.2, 142.9ms, 0 | 20260505 07:27:16.791 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 07:27:16.791 | INFO | Uploaded 36GiB to GPU in 2.22904s (16.1504GiB/s) | 20260505 07:27:16.791 | INFO | Downloaded 38GiB to GPU in 2.32496s (16.3444GiB/s) | 20260505 07:27:21.930 | INFO | Analyzing chunk frames = [1409536,1410048) | 20260505 07:27:21.930 | INFO | MemInfo: { | 20260505 07:27:21.930 | INFO | "gpu" : " 17 GB", | 20260505 07:27:21.930 | INFO | "huge" : "", | 20260505 07:27:21.930 | INFO | "huge_shared" : " 27 GB", | 20260505 07:27:21.930 | INFO | "malloc" : " 14 GB", | 20260505 07:27:21.930 | INFO | "pinned" : " 29 GB" | 20260505 07:27:21.930 | INFO | } | 20260505 07:27:21.930 | INFO | Perf report for the last 5.13946 seconds: | 20260505 07:27:21.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:27:21.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:21.930 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:21.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:21.930 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 07:27:21.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:21.930 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:21.930 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:27:21.930 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:21.930 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:21.931 | INFO | Uploaded 36GiB to GPU in 2.13321s (16.876GiB/s) | 20260505 07:27:21.931 | INFO | Downloaded 34GiB to GPU in 2.0426s (16.6454GiB/s) | 20260505 07:27:27.069 | INFO | Analyzing chunk frames = [1410048,1410560) | 20260505 07:27:27.069 | INFO | MemInfo: { | 20260505 07:27:27.069 | INFO | "gpu" : " 17 GB", | 20260505 07:27:27.069 | INFO | "huge" : "", | 20260505 07:27:27.069 | INFO | "huge_shared" : " 27 GB", | 20260505 07:27:27.069 | INFO | "malloc" : " 14 GB", | 20260505 07:27:27.069 | INFO | "pinned" : " 29 GB" | 20260505 07:27:27.069 | INFO | } | 20260505 07:27:27.069 | INFO | Perf report for the last 5.13915 seconds: | 20260505 07:27:27.070 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:27:27.070 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:27.070 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:27.070 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:27.070 | INFO | ANALYSIS : 63.4%, 3.0, 98.6ms, 0 | 20260505 07:27:27.070 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:27.070 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:27.070 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:27:27.070 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:27.070 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:27.070 | INFO | Uploaded 36GiB to GPU in 2.15018s (16.7428GiB/s) | 20260505 07:27:27.070 | INFO | Downloaded 34GiB to GPU in 2.05372s (16.5554GiB/s) | 20260505 07:27:32.209 | INFO | Analyzing chunk frames = [1410560,1411072) | 20260505 07:27:32.209 | INFO | MemInfo: { | 20260505 07:27:32.209 | INFO | "gpu" : " 17 GB", | 20260505 07:27:32.209 | INFO | "huge" : "", | 20260505 07:27:32.209 | INFO | "huge_shared" : " 27 GB", | 20260505 07:27:32.209 | INFO | "malloc" : " 14 GB", | 20260505 07:27:32.209 | INFO | "pinned" : " 29 GB" | 20260505 07:27:32.209 | INFO | } | 20260505 07:27:32.209 | INFO | Perf report for the last 5.13929 seconds: | 20260505 07:27:32.209 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:27:32.209 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:32.209 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:32.209 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:32.209 | INFO | ANALYSIS : 63.3%, 3.0, 98.3ms, 0 | 20260505 07:27:32.209 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:32.209 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:32.209 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 07:27:32.209 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:32.209 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:32.209 | INFO | Uploaded 36GiB to GPU in 2.12737s (16.9223GiB/s) | 20260505 07:27:32.209 | INFO | Downloaded 34GiB to GPU in 2.03496s (16.708GiB/s) | 20260505 07:27:37.348 | INFO | Analyzing chunk frames = [1411072,1411584) | 20260505 07:27:37.348 | INFO | MemInfo: { | 20260505 07:27:37.348 | INFO | "gpu" : " 17 GB", | 20260505 07:27:37.348 | INFO | "huge" : "", | 20260505 07:27:37.348 | INFO | "huge_shared" : " 27 GB", | 20260505 07:27:37.348 | INFO | "malloc" : " 14 GB", | 20260505 07:27:37.348 | INFO | "pinned" : " 29 GB" | 20260505 07:27:37.348 | INFO | } | 20260505 07:27:37.348 | INFO | Perf report for the last 5.13963 seconds: | 20260505 07:27:37.348 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:27:37.348 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:37.348 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:37.348 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:37.349 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 07:27:37.349 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:37.349 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:37.349 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 07:27:37.349 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:37.349 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:37.349 | INFO | Uploaded 36GiB to GPU in 2.20094s (16.3567GiB/s) | 20260505 07:27:37.349 | INFO | Downloaded 34GiB to GPU in 2.05755s (16.5245GiB/s) | 20260505 07:27:42.489 | INFO | Analyzing chunk frames = [1411584,1412096) | 20260505 07:27:42.489 | INFO | MemInfo: { | 20260505 07:27:42.489 | INFO | "gpu" : " 17 GB", | 20260505 07:27:42.489 | INFO | "huge" : "", | 20260505 07:27:42.489 | INFO | "huge_shared" : " 27 GB", | 20260505 07:27:42.489 | INFO | "malloc" : " 14 GB", | 20260505 07:27:42.489 | INFO | "pinned" : " 29 GB" | 20260505 07:27:42.489 | INFO | } | 20260505 07:27:42.489 | INFO | Perf report for the last 5.14077 seconds: | 20260505 07:27:42.489 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:27:42.489 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:42.489 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:42.489 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:42.489 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 07:27:42.489 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:42.489 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:42.489 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:27:42.489 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:42.490 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:42.490 | INFO | Uploaded 36GiB to GPU in 2.16176s (16.6531GiB/s) | 20260505 07:27:42.490 | INFO | Downloaded 34GiB to GPU in 2.05931s (16.5104GiB/s) | 20260505 07:27:47.627 | INFO | Analyzing chunk frames = [1412096,1412608) | 20260505 07:27:47.627 | INFO | MemInfo: { | 20260505 07:27:47.627 | INFO | "gpu" : " 17 GB", | 20260505 07:27:47.627 | INFO | "huge" : "", | 20260505 07:27:47.627 | INFO | "huge_shared" : " 27 GB", | 20260505 07:27:47.627 | INFO | "malloc" : " 14 GB", | 20260505 07:27:47.627 | INFO | "pinned" : " 29 GB" | 20260505 07:27:47.627 | INFO | } | 20260505 07:27:47.627 | INFO | Perf report for the last 5.13784 seconds: | 20260505 07:27:47.627 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:27:47.627 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:47.627 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:47.627 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:47.627 | INFO | ANALYSIS : 64.4%, 3.0, 100.6ms, 0 | 20260505 07:27:47.627 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:47.627 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:47.627 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260505 07:27:47.627 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:47.627 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:47.628 | INFO | Uploaded 36GiB to GPU in 2.23511s (16.1066GiB/s) | 20260505 07:27:47.628 | INFO | Downloaded 34GiB to GPU in 2.15291s (15.7926GiB/s) | 20260505 07:27:52.766 | INFO | Analyzing chunk frames = [1412608,1413120) | 20260505 07:27:52.766 | INFO | MemInfo: { | 20260505 07:27:52.766 | INFO | "gpu" : " 17 GB", | 20260505 07:27:52.766 | INFO | "huge" : "", | 20260505 07:27:52.766 | INFO | "huge_shared" : " 27 GB", | 20260505 07:27:52.766 | INFO | "malloc" : " 14 GB", | 20260505 07:27:52.766 | INFO | "pinned" : " 29 GB" | 20260505 07:27:52.766 | INFO | } | 20260505 07:27:52.766 | INFO | Perf report for the last 5.13935 seconds: | 20260505 07:27:52.766 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:27:52.766 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:52.766 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:52.766 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:52.766 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 07:27:52.767 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:52.767 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:52.767 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 07:27:52.767 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:52.767 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:52.767 | INFO | Uploaded 36GiB to GPU in 2.16218s (16.6499GiB/s) | 20260505 07:27:52.767 | INFO | Downloaded 34GiB to GPU in 2.02887s (16.7581GiB/s) | 20260505 07:27:52.900 | INFO | Starting a round of Baz IO | 20260505 07:27:56.189 | INFO | Populating RTMetrics report | 20260505 07:27:56.189 | INFO | RTMetrics report populated | 20260505 07:27:56.189 | INFO | Opening tmp RTMetrics json file | 20260505 07:27:56.230 | INFO | Serializing RTMetrics to json | 20260505 07:27:56.256 | INFO | Serialization done, swapping tmp file now... | 20260505 07:27:56.257 | INFO | RTMetrics done | 20260505 07:27:56.328 | INFO | Finished a round of Baz IO: Wrote 2.43 GiB in 3.427 seconds | 20260505 07:27:56.328 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 29.84% / 0.03% / 70.13% | 20260505 07:27:56.328 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.60463% / 4.76837e-05% / 7.54992e-05% | 20260505 07:27:56.328 | INFO | Baz IO overall: | 20260505 07:27:56.328 | INFO | BytesWritten : 770590209327 | 20260505 07:27:56.328 | INFO | NumEvents : 365613941976 | 20260505 07:27:57.906 | INFO | Analyzing chunk frames = [1413120,1413632) | 20260505 07:27:57.906 | INFO | MemInfo: { | 20260505 07:27:57.906 | INFO | "gpu" : " 17 GB", | 20260505 07:27:57.906 | INFO | "huge" : "", | 20260505 07:27:57.906 | INFO | "huge_shared" : " 27 GB", | 20260505 07:27:57.906 | INFO | "malloc" : " 14 GB", | 20260505 07:27:57.906 | INFO | "pinned" : " 29 GB" | 20260505 07:27:57.906 | INFO | } | 20260505 07:27:57.906 | INFO | Perf report for the last 5.13976 seconds: | 20260505 07:27:57.906 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:27:57.906 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:57.906 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:57.906 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:57.906 | INFO | ANALYSIS : 65.4%, 2.9, 99.9ms, 0 | 20260505 07:27:57.906 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:27:57.906 | INFO | RT_METRICS : 47.2%, 1.0, 24.8ms, 0 | 20260505 07:27:57.906 | INFO | PRE_HQ : 64.0%, 1.9, 61.9ms, 0 | 20260505 07:27:57.906 | INFO | BAZWRITER : 66.0%, 4.1, 140.6ms, 0 | 20260505 07:27:57.906 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260505 07:27:57.907 | INFO | Uploaded 36GiB to GPU in 2.24727s (16.0194GiB/s) | 20260505 07:27:57.907 | INFO | Downloaded 38GiB to GPU in 2.35103s (16.1631GiB/s) | 20260505 07:28:03.045 | INFO | Analyzing chunk frames = [1413632,1414144) | 20260505 07:28:03.045 | INFO | MemInfo: { | 20260505 07:28:03.045 | INFO | "gpu" : " 17 GB", | 20260505 07:28:03.045 | INFO | "huge" : "", | 20260505 07:28:03.045 | INFO | "huge_shared" : " 27 GB", | 20260505 07:28:03.045 | INFO | "malloc" : " 14 GB", | 20260505 07:28:03.045 | INFO | "pinned" : " 29 GB" | 20260505 07:28:03.045 | INFO | } | 20260505 07:28:03.045 | INFO | Perf report for the last 5.13863 seconds: | 20260505 07:28:03.045 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:28:03.045 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:03.045 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:03.045 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:03.045 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 07:28:03.045 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:03.045 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:03.045 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:28:03.045 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:03.045 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:03.045 | INFO | Uploaded 36GiB to GPU in 2.14023s (16.8207GiB/s) | 20260505 07:28:03.045 | INFO | Downloaded 34GiB to GPU in 2.04441s (16.6307GiB/s) | 20260505 07:28:08.184 | INFO | Analyzing chunk frames = [1414144,1414656) | 20260505 07:28:08.184 | INFO | MemInfo: { | 20260505 07:28:08.184 | INFO | "gpu" : " 17 GB", | 20260505 07:28:08.184 | INFO | "huge" : "", | 20260505 07:28:08.184 | INFO | "huge_shared" : " 27 GB", | 20260505 07:28:08.184 | INFO | "malloc" : " 14 GB", | 20260505 07:28:08.184 | INFO | "pinned" : " 29 GB" | 20260505 07:28:08.184 | INFO | } | 20260505 07:28:08.184 | INFO | Perf report for the last 5.1392 seconds: | 20260505 07:28:08.184 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:28:08.184 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:08.184 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:08.184 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:08.184 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 07:28:08.184 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:08.184 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:08.184 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:28:08.184 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:08.184 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:08.185 | INFO | Uploaded 36GiB to GPU in 2.15066s (16.7391GiB/s) | 20260505 07:28:08.185 | INFO | Downloaded 34GiB to GPU in 2.05946s (16.5092GiB/s) | 20260505 07:28:13.324 | INFO | Analyzing chunk frames = [1414656,1415168) | 20260505 07:28:13.324 | INFO | MemInfo: { | 20260505 07:28:13.324 | INFO | "gpu" : " 17 GB", | 20260505 07:28:13.324 | INFO | "huge" : "", | 20260505 07:28:13.324 | INFO | "huge_shared" : " 27 GB", | 20260505 07:28:13.324 | INFO | "malloc" : " 14 GB", | 20260505 07:28:13.324 | INFO | "pinned" : " 29 GB" | 20260505 07:28:13.324 | INFO | } | 20260505 07:28:13.324 | INFO | Perf report for the last 5.13956 seconds: | 20260505 07:28:13.324 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:28:13.324 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:13.324 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:13.324 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:13.324 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 07:28:13.324 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:13.324 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:13.324 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 07:28:13.324 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:13.324 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:13.324 | INFO | Uploaded 36GiB to GPU in 2.12363s (16.9521GiB/s) | 20260505 07:28:13.324 | INFO | Downloaded 34GiB to GPU in 2.02665s (16.7764GiB/s) | 20260505 07:28:18.463 | INFO | Analyzing chunk frames = [1415168,1415680) | 20260505 07:28:18.463 | INFO | MemInfo: { | 20260505 07:28:18.463 | INFO | "gpu" : " 17 GB", | 20260505 07:28:18.463 | INFO | "huge" : "", | 20260505 07:28:18.463 | INFO | "huge_shared" : " 27 GB", | 20260505 07:28:18.463 | INFO | "malloc" : " 14 GB", | 20260505 07:28:18.463 | INFO | "pinned" : " 29 GB" | 20260505 07:28:18.463 | INFO | } | 20260505 07:28:18.463 | INFO | Perf report for the last 5.13943 seconds: | 20260505 07:28:18.463 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:28:18.463 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:18.463 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:18.463 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:18.463 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 07:28:18.463 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:18.463 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:18.463 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 07:28:18.463 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:18.463 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:18.463 | INFO | Uploaded 36GiB to GPU in 2.16741s (16.6097GiB/s) | 20260505 07:28:18.463 | INFO | Downloaded 34GiB to GPU in 2.05467s (16.5477GiB/s) | 20260505 07:28:23.604 | INFO | Analyzing chunk frames = [1415680,1416192) | 20260505 07:28:23.604 | INFO | MemInfo: { | 20260505 07:28:23.604 | INFO | "gpu" : " 17 GB", | 20260505 07:28:23.604 | INFO | "huge" : "", | 20260505 07:28:23.604 | INFO | "huge_shared" : " 27 GB", | 20260505 07:28:23.604 | INFO | "malloc" : " 14 GB", | 20260505 07:28:23.604 | INFO | "pinned" : " 29 GB" | 20260505 07:28:23.604 | INFO | } | 20260505 07:28:23.604 | INFO | Perf report for the last 5.14084 seconds: | 20260505 07:28:23.604 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:28:23.604 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:23.604 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:23.604 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:23.604 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 07:28:23.604 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:23.604 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:23.604 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:28:23.604 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:23.604 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:23.604 | INFO | Uploaded 36GiB to GPU in 2.16507s (16.6277GiB/s) | 20260505 07:28:23.604 | INFO | Downloaded 34GiB to GPU in 2.0609s (16.4976GiB/s) | 20260505 07:28:28.742 | INFO | Analyzing chunk frames = [1416192,1416704) | 20260505 07:28:28.742 | INFO | MemInfo: { | 20260505 07:28:28.742 | INFO | "gpu" : " 17 GB", | 20260505 07:28:28.742 | INFO | "huge" : "", | 20260505 07:28:28.742 | INFO | "huge_shared" : " 27 GB", | 20260505 07:28:28.742 | INFO | "malloc" : " 14 GB", | 20260505 07:28:28.742 | INFO | "pinned" : " 29 GB" | 20260505 07:28:28.742 | INFO | } | 20260505 07:28:28.742 | INFO | Perf report for the last 5.13769 seconds: | 20260505 07:28:28.742 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:28:28.742 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:28.742 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:28.742 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:28.742 | INFO | ANALYSIS : 64.6%, 3.0, 100.9ms, 0 | 20260505 07:28:28.742 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:28.742 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:28.742 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 07:28:28.742 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:28.742 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:28.742 | INFO | Uploaded 36GiB to GPU in 2.22269s (16.1966GiB/s) | 20260505 07:28:28.742 | INFO | Downloaded 34GiB to GPU in 2.15142s (15.8035GiB/s) | 20260505 07:28:33.881 | INFO | Analyzing chunk frames = [1416704,1417216) | 20260505 07:28:33.881 | INFO | MemInfo: { | 20260505 07:28:33.881 | INFO | "gpu" : " 17 GB", | 20260505 07:28:33.881 | INFO | "huge" : "", | 20260505 07:28:33.881 | INFO | "huge_shared" : " 27 GB", | 20260505 07:28:33.881 | INFO | "malloc" : " 14 GB", | 20260505 07:28:33.881 | INFO | "pinned" : " 29 GB" | 20260505 07:28:33.881 | INFO | } | 20260505 07:28:33.881 | INFO | Perf report for the last 5.1397 seconds: | 20260505 07:28:33.882 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:28:33.882 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:33.882 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:33.882 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:33.882 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 07:28:33.882 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:33.882 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:33.882 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 07:28:33.882 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:33.882 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:33.882 | INFO | Uploaded 36GiB to GPU in 2.18382s (16.4848GiB/s) | 20260505 07:28:33.882 | INFO | Downloaded 34GiB to GPU in 2.0522s (16.5676GiB/s) | 20260505 07:28:33.882 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 07:28:33.882 | INFO | Total mem usage: 15.3 GiB | 20260505 07:28:33.882 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 07:28:33.882 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -5.35%) | 20260505 07:28:33.882 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 07:28:33.882 | INFO | High Water Breakdown: | 20260505 07:28:33.882 | INFO | 60.8% : 8.84 GiB ( 94.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 07:28:33.882 | INFO | 39.1% : 5.69 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 07:28:33.882 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 07:28:33.882 | INFO | ------------------------------------------ | 20260505 07:28:33.882 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:28:33.882 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 07:28:33.882 | INFO | Total mem usage: 26.5 GiB | 20260505 07:28:33.882 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 07:28:33.882 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 07:28:33.882 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 07:28:33.882 | INFO | High Water Breakdown: | 20260505 07:28:33.882 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:28:33.882 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:28:33.882 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:28:33.882 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:28:33.882 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:28:33.882 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:28:33.882 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 07:28:33.882 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 07:28:33.882 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:28:33.882 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 07:28:33.882 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 07:28:33.882 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 07:28:33.882 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 07:28:33.882 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 07:28:33.882 | INFO | ------------------------------------------ | 20260505 07:28:33.883 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:28:33.883 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 07:28:33.883 | INFO | Total mem usage: 25.6 GiB | 20260505 07:28:33.883 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 07:28:33.883 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 07:28:33.883 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 07:28:33.883 | INFO | High Water Breakdown: | 20260505 07:28:33.883 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 07:28:33.883 | INFO | ------------------------------------------ | 20260505 07:28:33.883 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:28:33.883 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 07:28:33.883 | INFO | Total mem usage: 15.7 GiB | 20260505 07:28:33.883 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 07:28:33.883 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 07:28:33.883 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 07:28:33.883 | INFO | High Water Breakdown: | 20260505 07:28:33.883 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 07:28:33.883 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 07:28:33.883 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 07:28:33.883 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 07:28:33.883 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 07:28:33.883 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 07:28:33.883 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 07:28:33.883 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 07:28:33.883 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 07:28:33.883 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:28:33.883 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 07:28:33.883 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:28:33.883 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 07:28:33.883 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 07:28:33.883 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 07:28:33.883 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 07:28:33.883 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:28:33.883 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 07:28:33.883 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:28:33.883 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:28:33.883 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:28:33.883 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 07:28:33.883 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 07:28:33.883 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:28:33.883 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 07:28:33.883 | INFO | ------------------------------------------ | 20260505 07:28:33.883 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:28:33.883 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 07:28:33.883 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:28:33.883 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:28:33.883 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.53e+01, 1.35e+01 / 1.78e+00 | 20260505 07:28:33.883 | INFO | PerfReport: 0.28%, AggregateStats -- 9.35e-02 / 1.03e-01, 1.08e-02 / 6.81e-03 | 20260505 07:28:33.883 | INFO | PerfReport: 11.61%, Baseline -- 3.82e+00 / 4.42e+00, 6.27e-01 / 3.38e-01 | 20260505 07:28:33.883 | INFO | PerfReport: 3.30%, Histogram -- 1.09e+00 / 1.46e+00, 2.12e-01 / 1.31e-01 | 20260505 07:28:33.883 | INFO | PerfReport: 15.47%, DME -- 4.07e+01 / 4.45e+01, 4.12e+00 / 1.22e+00 | 20260505 07:28:33.883 | INFO | PerfReport: 56.30%, FrameLabeling -- 1.85e+01 / 1.94e+01, 2.03e+00 / 1.63e+00 | 20260505 07:28:33.883 | INFO | PerfReport: 7.61%, PulseAccumulating -- 2.50e+00 / 3.17e+00, 4.38e-01 / 2.16e-01 | 20260505 07:28:33.883 | INFO | PerfReport: 3.98%, Metrics -- 1.31e+00 / 1.72e+00, 3.66e-01 / 1.57e-01 | 20260505 07:28:33.883 | INFO | PerfReport: 1.44%, PulseSerializing -- 4.72e-01 / 5.49e-01, 9.03e-02 / 2.85e-02 | 20260505 07:28:33.883 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 07:28:33.883 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:28:33.883 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:28:33.883 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.25e+01 / 0.00e+00 | 20260505 07:28:33.883 | INFO | PerfReport: 22.34%, Upload -- 2.27e+01 / 0.00e+00, 8.04e+00 / 0.00e+00 | 20260505 07:28:33.883 | INFO | PerfReport: 21.54%, Download -- 2.18e+01 / 0.00e+00, 7.86e+00 / 0.00e+00 | 20260505 07:28:33.883 | INFO | PerfReport: 32.40%, Compute -- 3.29e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 07:28:33.883 | INFO | PerfReport: 1.54%, LockUpload -- 1.56e+00 / 0.00e+00, 7.14e+00 / 0.00e+00 | 20260505 07:28:33.883 | INFO | PerfReport: 0.86%, LockDownload -- 8.73e-01 / 0.00e+00, 2.27e+00 / 0.00e+00 | 20260505 07:28:33.883 | INFO | PerfReport: 21.31%, LockCompute -- 2.16e+01 / 0.00e+00, 1.68e+01 / 0.00e+00 | 20260505 07:28:33.883 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 2.21e-02 / 0.00e+00 | 20260505 07:28:34.016 | INFO | Starting a round of Baz IO | 20260505 07:28:37.288 | INFO | Populating RTMetrics report | 20260505 07:28:37.288 | INFO | RTMetrics report populated | 20260505 07:28:37.288 | INFO | Opening tmp RTMetrics json file | 20260505 07:28:37.306 | INFO | Serializing RTMetrics to json | 20260505 07:28:37.308 | INFO | Serialization done, swapping tmp file now... | 20260505 07:28:37.308 | INFO | RTMetrics done | 20260505 07:28:37.444 | INFO | Finished a round of Baz IO: Wrote 2.429 GiB in 3.428 seconds | 20260505 07:28:37.444 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 29.81% / 0.03% / 70.16% | 20260505 07:28:37.444 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.58514% / 7.15256e-05% / 4.37101e-05% | 20260505 07:28:37.444 | INFO | Baz IO overall: | 20260505 07:28:37.444 | INFO | BytesWritten : 773198355548 | 20260505 07:28:37.444 | INFO | NumEvents : 366861486798 | 20260505 07:28:39.020 | INFO | Analyzing chunk frames = [1417216,1417728) | 20260505 07:28:39.020 | INFO | MemInfo: { | 20260505 07:28:39.020 | INFO | "gpu" : " 17 GB", | 20260505 07:28:39.020 | INFO | "huge" : "", | 20260505 07:28:39.020 | INFO | "huge_shared" : " 27 GB", | 20260505 07:28:39.020 | INFO | "malloc" : " 14 GB", | 20260505 07:28:39.020 | INFO | "pinned" : " 29 GB" | 20260505 07:28:39.020 | INFO | } | 20260505 07:28:39.020 | INFO | Perf report for the last 5.13892 seconds: | 20260505 07:28:39.021 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:28:39.021 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:39.021 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:39.021 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:39.021 | INFO | ANALYSIS : 65.1%, 2.9, 98.9ms, 0 | 20260505 07:28:39.021 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:39.021 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260505 07:28:39.021 | INFO | PRE_HQ : 63.9%, 1.9, 63.2ms, 0 | 20260505 07:28:39.021 | INFO | BAZWRITER : 66.0%, 4.2, 142.4ms, 0 | 20260505 07:28:39.021 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260505 07:28:39.021 | INFO | Uploaded 36GiB to GPU in 2.2193s (16.2214GiB/s) | 20260505 07:28:39.021 | INFO | Downloaded 38GiB to GPU in 2.3143s (16.4197GiB/s) | 20260505 07:28:44.160 | INFO | Analyzing chunk frames = [1417728,1418240) | 20260505 07:28:44.160 | INFO | MemInfo: { | 20260505 07:28:44.160 | INFO | "gpu" : " 17 GB", | 20260505 07:28:44.160 | INFO | "huge" : "", | 20260505 07:28:44.160 | INFO | "huge_shared" : " 27 GB", | 20260505 07:28:44.160 | INFO | "malloc" : " 14 GB", | 20260505 07:28:44.160 | INFO | "pinned" : " 29 GB" | 20260505 07:28:44.160 | INFO | } | 20260505 07:28:44.160 | INFO | Perf report for the last 5.13977 seconds: | 20260505 07:28:44.160 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:28:44.160 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:44.160 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:44.160 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:44.160 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 07:28:44.160 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:44.160 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:44.160 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260505 07:28:44.160 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:44.160 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:44.160 | INFO | Uploaded 36GiB to GPU in 2.1427s (16.8013GiB/s) | 20260505 07:28:44.160 | INFO | Downloaded 34GiB to GPU in 2.05055s (16.5809GiB/s) | 20260505 07:28:49.299 | INFO | Analyzing chunk frames = [1418240,1418752) | 20260505 07:28:49.299 | INFO | MemInfo: { | 20260505 07:28:49.299 | INFO | "gpu" : " 17 GB", | 20260505 07:28:49.299 | INFO | "huge" : "", | 20260505 07:28:49.299 | INFO | "huge_shared" : " 27 GB", | 20260505 07:28:49.299 | INFO | "malloc" : " 14 GB", | 20260505 07:28:49.299 | INFO | "pinned" : " 29 GB" | 20260505 07:28:49.299 | INFO | } | 20260505 07:28:49.299 | INFO | Perf report for the last 5.1388 seconds: | 20260505 07:28:49.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:28:49.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:49.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:49.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:49.299 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260505 07:28:49.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:49.299 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:49.299 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 07:28:49.299 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:49.299 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:49.299 | INFO | Uploaded 36GiB to GPU in 2.15548s (16.7016GiB/s) | 20260505 07:28:49.299 | INFO | Downloaded 34GiB to GPU in 2.05428s (16.5508GiB/s) | 20260505 07:28:54.438 | INFO | Analyzing chunk frames = [1418752,1419264) | 20260505 07:28:54.438 | INFO | MemInfo: { | 20260505 07:28:54.438 | INFO | "gpu" : " 17 GB", | 20260505 07:28:54.438 | INFO | "huge" : "", | 20260505 07:28:54.438 | INFO | "huge_shared" : " 27 GB", | 20260505 07:28:54.438 | INFO | "malloc" : " 14 GB", | 20260505 07:28:54.438 | INFO | "pinned" : " 29 GB" | 20260505 07:28:54.438 | INFO | } | 20260505 07:28:54.438 | INFO | Perf report for the last 5.13948 seconds: | 20260505 07:28:54.439 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:28:54.439 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:54.439 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:54.439 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:54.439 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 07:28:54.439 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:54.439 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:54.439 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 07:28:54.439 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:54.439 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:54.439 | INFO | Uploaded 36GiB to GPU in 2.12225s (16.9631GiB/s) | 20260505 07:28:54.439 | INFO | Downloaded 34GiB to GPU in 2.03841s (16.6797GiB/s) | 20260505 07:28:59.578 | INFO | Analyzing chunk frames = [1419264,1419776) | 20260505 07:28:59.578 | INFO | MemInfo: { | 20260505 07:28:59.578 | INFO | "gpu" : " 17 GB", | 20260505 07:28:59.578 | INFO | "huge" : "", | 20260505 07:28:59.578 | INFO | "huge_shared" : " 27 GB", | 20260505 07:28:59.578 | INFO | "malloc" : " 14 GB", | 20260505 07:28:59.578 | INFO | "pinned" : " 29 GB" | 20260505 07:28:59.578 | INFO | } | 20260505 07:28:59.578 | INFO | Perf report for the last 5.13973 seconds: | 20260505 07:28:59.578 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:28:59.578 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:59.578 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:59.578 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:59.578 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 07:28:59.578 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:59.578 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:59.578 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 07:28:59.578 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:59.578 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:28:59.578 | INFO | Uploaded 36GiB to GPU in 2.13712s (16.8451GiB/s) | 20260505 07:28:59.578 | INFO | Downloaded 34GiB to GPU in 2.05062s (16.5803GiB/s) | 20260505 07:29:04.719 | INFO | Analyzing chunk frames = [1419776,1420288) | 20260505 07:29:04.719 | INFO | MemInfo: { | 20260505 07:29:04.719 | INFO | "gpu" : " 17 GB", | 20260505 07:29:04.719 | INFO | "huge" : "", | 20260505 07:29:04.719 | INFO | "huge_shared" : " 27 GB", | 20260505 07:29:04.719 | INFO | "malloc" : " 14 GB", | 20260505 07:29:04.719 | INFO | "pinned" : " 29 GB" | 20260505 07:29:04.719 | INFO | } | 20260505 07:29:04.719 | INFO | Perf report for the last 5.14109 seconds: | 20260505 07:29:04.719 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:29:04.719 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:04.719 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:04.719 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:04.719 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 07:29:04.719 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:04.719 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:04.719 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 07:29:04.719 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:04.719 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:04.719 | INFO | Uploaded 36GiB to GPU in 2.13567s (16.8566GiB/s) | 20260505 07:29:04.719 | INFO | Downloaded 34GiB to GPU in 2.05685s (16.5301GiB/s) | 20260505 07:29:09.856 | INFO | Analyzing chunk frames = [1420288,1420800) | 20260505 07:29:09.856 | INFO | MemInfo: { | 20260505 07:29:09.856 | INFO | "gpu" : " 17 GB", | 20260505 07:29:09.856 | INFO | "huge" : "", | 20260505 07:29:09.856 | INFO | "huge_shared" : " 27 GB", | 20260505 07:29:09.856 | INFO | "malloc" : " 14 GB", | 20260505 07:29:09.856 | INFO | "pinned" : " 29 GB" | 20260505 07:29:09.856 | INFO | } | 20260505 07:29:09.856 | INFO | Perf report for the last 5.13703 seconds: | 20260505 07:29:09.856 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:29:09.856 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:09.856 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:09.856 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:09.856 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 07:29:09.856 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:09.856 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:09.856 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260505 07:29:09.856 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:09.856 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:09.857 | INFO | Uploaded 36GiB to GPU in 2.22366s (16.1895GiB/s) | 20260505 07:29:09.857 | INFO | Downloaded 34GiB to GPU in 2.16814s (15.6817GiB/s) | 20260505 07:29:14.996 | INFO | Analyzing chunk frames = [1420800,1421312) | 20260505 07:29:14.996 | INFO | MemInfo: { | 20260505 07:29:14.996 | INFO | "gpu" : " 17 GB", | 20260505 07:29:14.996 | INFO | "huge" : "", | 20260505 07:29:14.996 | INFO | "huge_shared" : " 27 GB", | 20260505 07:29:14.996 | INFO | "malloc" : " 14 GB", | 20260505 07:29:14.996 | INFO | "pinned" : " 29 GB" | 20260505 07:29:14.996 | INFO | } | 20260505 07:29:14.996 | INFO | Perf report for the last 5.13951 seconds: | 20260505 07:29:14.996 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:29:14.996 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:14.996 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:14.996 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:14.996 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 07:29:14.996 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:14.996 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:14.996 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 07:29:14.996 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:14.996 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:14.996 | INFO | Uploaded 36GiB to GPU in 2.18236s (16.4959GiB/s) | 20260505 07:29:14.996 | INFO | Downloaded 34GiB to GPU in 2.05087s (16.5783GiB/s) | 20260505 07:29:15.127 | INFO | Starting a round of Baz IO | 20260505 07:29:18.417 | INFO | Populating RTMetrics report | 20260505 07:29:18.418 | INFO | RTMetrics report populated | 20260505 07:29:18.418 | INFO | Opening tmp RTMetrics json file | 20260505 07:29:18.476 | INFO | Serializing RTMetrics to json | 20260505 07:29:18.481 | INFO | Serialization done, swapping tmp file now... | 20260505 07:29:18.489 | INFO | RTMetrics done | 20260505 07:29:18.600 | INFO | Finished a round of Baz IO: Wrote 2.429 GiB in 3.472 seconds | 20260505 07:29:18.600 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 29.78% / 0.03% / 70.19% | 20260505 07:29:18.600 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.56806% / 5.16574e-05% / 5.96046e-05% | 20260505 07:29:18.600 | INFO | Baz IO overall: | 20260505 07:29:18.600 | INFO | BytesWritten : 775806906476 | 20260505 07:29:18.600 | INFO | NumEvents : 368108962645 | 20260505 07:29:20.135 | INFO | Analyzing chunk frames = [1421312,1421824) | 20260505 07:29:20.135 | INFO | MemInfo: { | 20260505 07:29:20.135 | INFO | "gpu" : " 17 GB", | 20260505 07:29:20.135 | INFO | "huge" : "", | 20260505 07:29:20.135 | INFO | "huge_shared" : " 27 GB", | 20260505 07:29:20.135 | INFO | "malloc" : " 14 GB", | 20260505 07:29:20.135 | INFO | "pinned" : " 29 GB" | 20260505 07:29:20.135 | INFO | } | 20260505 07:29:20.135 | INFO | Perf report for the last 5.13937 seconds: | 20260505 07:29:20.135 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:29:20.135 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:20.135 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:20.135 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:20.135 | INFO | ANALYSIS : 65.4%, 2.9, 99.5ms, 0 | 20260505 07:29:20.135 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:20.135 | INFO | RT_METRICS : 47.6%, 1.0, 25.0ms, 0 | 20260505 07:29:20.135 | INFO | PRE_HQ : 64.0%, 1.9, 62.6ms, 0 | 20260505 07:29:20.135 | INFO | BAZWRITER : 66.9%, 4.1, 142.2ms, 0 | 20260505 07:29:20.135 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260505 07:29:20.135 | INFO | Uploaded 36GiB to GPU in 2.23511s (16.1066GiB/s) | 20260505 07:29:20.135 | INFO | Downloaded 38GiB to GPU in 2.35696s (16.1225GiB/s) | 20260505 07:29:25.275 | INFO | Analyzing chunk frames = [1421824,1422336) | 20260505 07:29:25.275 | INFO | MemInfo: { | 20260505 07:29:25.275 | INFO | "gpu" : " 17 GB", | 20260505 07:29:25.275 | INFO | "huge" : "", | 20260505 07:29:25.275 | INFO | "huge_shared" : " 27 GB", | 20260505 07:29:25.275 | INFO | "malloc" : " 14 GB", | 20260505 07:29:25.275 | INFO | "pinned" : " 29 GB" | 20260505 07:29:25.275 | INFO | } | 20260505 07:29:25.275 | INFO | Perf report for the last 5.1396 seconds: | 20260505 07:29:25.275 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:29:25.275 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:25.275 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:25.275 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:25.275 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 07:29:25.275 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:25.275 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:25.275 | INFO | PRE_HQ : 30.1%, 1.0, 15.7ms, 0 | 20260505 07:29:25.275 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:25.275 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:25.275 | INFO | Uploaded 36GiB to GPU in 2.1391s (16.8295GiB/s) | 20260505 07:29:25.275 | INFO | Downloaded 34GiB to GPU in 2.04688s (16.6107GiB/s) | 20260505 07:29:30.414 | INFO | Analyzing chunk frames = [1422336,1422848) | 20260505 07:29:30.414 | INFO | MemInfo: { | 20260505 07:29:30.414 | INFO | "gpu" : " 17 GB", | 20260505 07:29:30.414 | INFO | "huge" : "", | 20260505 07:29:30.414 | INFO | "huge_shared" : " 27 GB", | 20260505 07:29:30.414 | INFO | "malloc" : " 14 GB", | 20260505 07:29:30.414 | INFO | "pinned" : " 29 GB" | 20260505 07:29:30.414 | INFO | } | 20260505 07:29:30.414 | INFO | Perf report for the last 5.13961 seconds: | 20260505 07:29:30.414 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:29:30.414 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:30.414 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:30.414 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:30.414 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 07:29:30.415 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:30.415 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:30.415 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 07:29:30.415 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:30.415 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:30.415 | INFO | Uploaded 36GiB to GPU in 2.13929s (16.828GiB/s) | 20260505 07:29:30.415 | INFO | Downloaded 34GiB to GPU in 2.05605s (16.5366GiB/s) | 20260505 07:29:35.553 | INFO | Analyzing chunk frames = [1422848,1423360) | 20260505 07:29:35.553 | INFO | MemInfo: { | 20260505 07:29:35.553 | INFO | "gpu" : " 17 GB", | 20260505 07:29:35.553 | INFO | "huge" : "", | 20260505 07:29:35.553 | INFO | "huge_shared" : " 27 GB", | 20260505 07:29:35.553 | INFO | "malloc" : " 14 GB", | 20260505 07:29:35.553 | INFO | "pinned" : " 29 GB" | 20260505 07:29:35.553 | INFO | } | 20260505 07:29:35.553 | INFO | Perf report for the last 5.13862 seconds: | 20260505 07:29:35.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:29:35.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:35.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:35.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:35.553 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 07:29:35.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:35.553 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:35.553 | INFO | PRE_HQ : 30.5%, 1.0, 15.9ms, 0 | 20260505 07:29:35.553 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:35.553 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:35.553 | INFO | Uploaded 36GiB to GPU in 2.144s (16.7911GiB/s) | 20260505 07:29:35.553 | INFO | Downloaded 34GiB to GPU in 2.03798s (16.6832GiB/s) | 20260505 07:29:40.692 | INFO | Analyzing chunk frames = [1423360,1423872) | 20260505 07:29:40.692 | INFO | MemInfo: { | 20260505 07:29:40.692 | INFO | "gpu" : " 17 GB", | 20260505 07:29:40.692 | INFO | "huge" : "", | 20260505 07:29:40.692 | INFO | "huge_shared" : " 27 GB", | 20260505 07:29:40.692 | INFO | "malloc" : " 14 GB", | 20260505 07:29:40.692 | INFO | "pinned" : " 29 GB" | 20260505 07:29:40.692 | INFO | } | 20260505 07:29:40.692 | INFO | Perf report for the last 5.1394 seconds: | 20260505 07:29:40.692 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:29:40.692 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:40.692 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:40.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:40.692 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 07:29:40.692 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:40.693 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:40.693 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:29:40.693 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:40.693 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:40.693 | INFO | Uploaded 36GiB to GPU in 2.16535s (16.6255GiB/s) | 20260505 07:29:40.693 | INFO | Downloaded 34GiB to GPU in 2.05509s (16.5443GiB/s) | 20260505 07:29:45.833 | INFO | Analyzing chunk frames = [1423872,1424384) | 20260505 07:29:45.833 | INFO | MemInfo: { | 20260505 07:29:45.833 | INFO | "gpu" : " 17 GB", | 20260505 07:29:45.833 | INFO | "huge" : "", | 20260505 07:29:45.833 | INFO | "huge_shared" : " 27 GB", | 20260505 07:29:45.833 | INFO | "malloc" : " 14 GB", | 20260505 07:29:45.833 | INFO | "pinned" : " 29 GB" | 20260505 07:29:45.833 | INFO | } | 20260505 07:29:45.833 | INFO | Perf report for the last 5.14093 seconds: | 20260505 07:29:45.833 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:29:45.833 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:45.833 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:45.833 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:45.833 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 07:29:45.833 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:45.834 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:45.834 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 07:29:45.834 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:45.834 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:45.834 | INFO | Uploaded 36GiB to GPU in 2.16157s (16.6545GiB/s) | 20260505 07:29:45.834 | INFO | Downloaded 34GiB to GPU in 2.06003s (16.5046GiB/s) | 20260505 07:29:50.971 | INFO | Analyzing chunk frames = [1424384,1424896) | 20260505 07:29:50.971 | INFO | MemInfo: { | 20260505 07:29:50.971 | INFO | "gpu" : " 17 GB", | 20260505 07:29:50.971 | INFO | "huge" : "", | 20260505 07:29:50.971 | INFO | "huge_shared" : " 27 GB", | 20260505 07:29:50.971 | INFO | "malloc" : " 14 GB", | 20260505 07:29:50.971 | INFO | "pinned" : " 29 GB" | 20260505 07:29:50.971 | INFO | } | 20260505 07:29:50.971 | INFO | Perf report for the last 5.13798 seconds: | 20260505 07:29:50.972 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:29:50.972 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:50.972 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:50.972 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:50.972 | INFO | ANALYSIS : 64.4%, 3.0, 100.6ms, 0 | 20260505 07:29:50.972 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:50.972 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:50.972 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260505 07:29:50.972 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:50.972 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:50.972 | INFO | Uploaded 36GiB to GPU in 2.22176s (16.2034GiB/s) | 20260505 07:29:50.972 | INFO | Downloaded 34GiB to GPU in 2.15055s (15.8099GiB/s) | 20260505 07:29:56.110 | INFO | Analyzing chunk frames = [1424896,1425408) | 20260505 07:29:56.110 | INFO | MemInfo: { | 20260505 07:29:56.110 | INFO | "gpu" : " 17 GB", | 20260505 07:29:56.110 | INFO | "huge" : "", | 20260505 07:29:56.110 | INFO | "huge_shared" : " 27 GB", | 20260505 07:29:56.110 | INFO | "malloc" : " 14 GB", | 20260505 07:29:56.110 | INFO | "pinned" : " 29 GB" | 20260505 07:29:56.110 | INFO | } | 20260505 07:29:56.110 | INFO | Perf report for the last 5.13909 seconds: | 20260505 07:29:56.111 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:29:56.111 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:56.111 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:56.111 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:56.111 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 07:29:56.111 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:56.111 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:56.111 | INFO | PRE_HQ : 30.5%, 1.0, 15.9ms, 0 | 20260505 07:29:56.111 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:56.111 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:29:56.111 | INFO | Uploaded 36GiB to GPU in 2.16786s (16.6062GiB/s) | 20260505 07:29:56.111 | INFO | Downloaded 34GiB to GPU in 2.02688s (16.7746GiB/s) | 20260505 07:29:56.111 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 07:29:56.111 | INFO | Total mem usage: 15.3 GiB | 20260505 07:29:56.111 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 07:29:56.111 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.16%) | 20260505 07:29:56.111 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 07:29:56.111 | INFO | High Water Breakdown: | 20260505 07:29:56.111 | INFO | 60.7% : 8.76 GiB ( 93.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 07:29:56.111 | INFO | 39.2% : 5.66 GiB ( 85.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 07:29:56.111 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 07:29:56.111 | INFO | ------------------------------------------ | 20260505 07:29:56.111 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:29:56.111 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 07:29:56.111 | INFO | Total mem usage: 26.5 GiB | 20260505 07:29:56.111 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 07:29:56.111 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 07:29:56.111 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 07:29:56.111 | INFO | High Water Breakdown: | 20260505 07:29:56.111 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:29:56.111 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:29:56.111 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:29:56.111 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:29:56.111 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:29:56.111 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:29:56.111 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 07:29:56.111 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 07:29:56.111 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:29:56.111 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 07:29:56.111 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 07:29:56.111 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 07:29:56.111 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 07:29:56.111 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 07:29:56.111 | INFO | ------------------------------------------ | 20260505 07:29:56.111 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:29:56.111 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 07:29:56.111 | INFO | Total mem usage: 25.6 GiB | 20260505 07:29:56.112 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 07:29:56.112 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 07:29:56.112 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 07:29:56.112 | INFO | High Water Breakdown: | 20260505 07:29:56.112 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 07:29:56.112 | INFO | ------------------------------------------ | 20260505 07:29:56.112 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:29:56.112 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 07:29:56.112 | INFO | Total mem usage: 15.7 GiB | 20260505 07:29:56.112 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 07:29:56.112 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 07:29:56.112 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 07:29:56.112 | INFO | High Water Breakdown: | 20260505 07:29:56.112 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 07:29:56.112 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 07:29:56.112 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 07:29:56.112 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 07:29:56.112 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 07:29:56.112 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 07:29:56.112 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 07:29:56.112 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 07:29:56.112 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 07:29:56.112 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:29:56.112 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 07:29:56.112 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:29:56.112 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 07:29:56.112 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 07:29:56.112 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 07:29:56.112 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 07:29:56.112 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:29:56.112 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 07:29:56.112 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:29:56.112 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:29:56.112 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:29:56.112 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 07:29:56.112 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 07:29:56.112 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:29:56.112 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 07:29:56.112 | INFO | ------------------------------------------ | 20260505 07:29:56.112 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:29:56.112 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 07:29:56.112 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:29:56.112 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:29:56.112 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.49e+01, 1.35e+01 / 1.81e+00 | 20260505 07:29:56.112 | INFO | PerfReport: 0.29%, AggregateStats -- 9.40e-02 / 1.03e-01, 1.24e-02 / 7.27e-03 | 20260505 07:29:56.112 | INFO | PerfReport: 11.62%, Baseline -- 3.82e+00 / 4.41e+00, 6.27e-01 / 3.19e-01 | 20260505 07:29:56.112 | INFO | PerfReport: 3.30%, Histogram -- 1.08e+00 / 1.44e+00, 2.12e-01 / 1.41e-01 | 20260505 07:29:56.112 | INFO | PerfReport: 15.46%, DME -- 4.06e+01 / 4.43e+01, 4.16e+00 / 1.29e+00 | 20260505 07:29:56.112 | INFO | PerfReport: 56.26%, FrameLabeling -- 1.85e+01 / 1.93e+01, 2.01e+00 / 1.46e+00 | 20260505 07:29:56.112 | INFO | PerfReport: 7.64%, PulseAccumulating -- 2.51e+00 / 3.15e+00, 4.50e-01 / 2.67e-01 | 20260505 07:29:56.112 | INFO | PerfReport: 3.99%, Metrics -- 1.31e+00 / 1.70e+00, 3.67e-01 / 1.67e-01 | 20260505 07:29:56.112 | INFO | PerfReport: 1.44%, PulseSerializing -- 4.73e-01 / 5.47e-01, 9.04e-02 / 3.42e-02 | 20260505 07:29:56.112 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 07:29:56.112 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:29:56.112 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:29:56.112 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.24e+01 / 0.00e+00 | 20260505 07:29:56.112 | INFO | PerfReport: 22.31%, Upload -- 2.26e+01 / 0.00e+00, 8.00e+00 / 0.00e+00 | 20260505 07:29:56.112 | INFO | PerfReport: 21.59%, Download -- 2.19e+01 / 0.00e+00, 7.90e+00 / 0.00e+00 | 20260505 07:29:56.112 | INFO | PerfReport: 32.43%, Compute -- 3.28e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 07:29:56.112 | INFO | PerfReport: 1.53%, LockUpload -- 1.55e+00 / 0.00e+00, 7.12e+00 / 0.00e+00 | 20260505 07:29:56.112 | INFO | PerfReport: 0.88%, LockDownload -- 8.90e-01 / 0.00e+00, 2.27e+00 / 0.00e+00 | 20260505 07:29:56.112 | INFO | PerfReport: 21.25%, LockCompute -- 2.15e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260505 07:29:56.112 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.77e-02 / 0.00e+00 | 20260505 07:29:56.244 | INFO | Starting a round of Baz IO | 20260505 07:29:59.533 | INFO | Populating RTMetrics report | 20260505 07:29:59.533 | INFO | RTMetrics report populated | 20260505 07:29:59.533 | INFO | Opening tmp RTMetrics json file | 20260505 07:29:59.573 | INFO | Serializing RTMetrics to json | 20260505 07:29:59.574 | INFO | Serialization done, swapping tmp file now... | 20260505 07:29:59.575 | INFO | RTMetrics done | 20260505 07:29:59.706 | INFO | Finished a round of Baz IO: Wrote 2.431 GiB in 3.461 seconds | 20260505 07:29:59.706 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 29.75% / 0.03% / 70.22% | 20260505 07:29:59.706 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.56549% / 5.5631e-05% / 7.15256e-05% | 20260505 07:29:59.706 | INFO | Baz IO overall: | 20260505 07:29:59.706 | INFO | BytesWritten : 778417454726 | 20260505 07:29:59.706 | INFO | NumEvents : 369357786423 | 20260505 07:30:01.250 | INFO | Analyzing chunk frames = [1425408,1425920) | 20260505 07:30:01.250 | INFO | MemInfo: { | 20260505 07:30:01.250 | INFO | "gpu" : " 17 GB", | 20260505 07:30:01.250 | INFO | "huge" : "", | 20260505 07:30:01.250 | INFO | "huge_shared" : " 27 GB", | 20260505 07:30:01.250 | INFO | "malloc" : " 14 GB", | 20260505 07:30:01.250 | INFO | "pinned" : " 29 GB" | 20260505 07:30:01.250 | INFO | } | 20260505 07:30:01.250 | INFO | Perf report for the last 5.13917 seconds: | 20260505 07:30:01.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:30:01.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:01.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:01.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:01.250 | INFO | ANALYSIS : 65.5%, 2.9, 99.0ms, 0 | 20260505 07:30:01.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:01.250 | INFO | RT_METRICS : 46.8%, 1.0, 24.5ms, 0 | 20260505 07:30:01.250 | INFO | PRE_HQ : 64.2%, 1.9, 62.5ms, 0 | 20260505 07:30:01.250 | INFO | BAZWRITER : 66.6%, 4.2, 144.1ms, 0 | 20260505 07:30:01.250 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 07:30:01.250 | INFO | Uploaded 36GiB to GPU in 2.22693s (16.1657GiB/s) | 20260505 07:30:01.250 | INFO | Downloaded 38GiB to GPU in 2.36249s (16.0848GiB/s) | 20260505 07:30:06.389 | INFO | Analyzing chunk frames = [1425920,1426432) | 20260505 07:30:06.389 | INFO | MemInfo: { | 20260505 07:30:06.389 | INFO | "gpu" : " 17 GB", | 20260505 07:30:06.389 | INFO | "huge" : "", | 20260505 07:30:06.389 | INFO | "huge_shared" : " 27 GB", | 20260505 07:30:06.389 | INFO | "malloc" : " 14 GB", | 20260505 07:30:06.389 | INFO | "pinned" : " 29 GB" | 20260505 07:30:06.389 | INFO | } | 20260505 07:30:06.389 | INFO | Perf report for the last 5.13931 seconds: | 20260505 07:30:06.389 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:30:06.389 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:06.389 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:06.389 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:06.389 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 07:30:06.389 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:06.389 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:06.389 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:30:06.390 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:06.390 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:06.390 | INFO | Uploaded 36GiB to GPU in 2.13796s (16.8385GiB/s) | 20260505 07:30:06.390 | INFO | Downloaded 34GiB to GPU in 2.04658s (16.6131GiB/s) | 20260505 07:30:11.529 | INFO | Analyzing chunk frames = [1426432,1426944) | 20260505 07:30:11.529 | INFO | MemInfo: { | 20260505 07:30:11.529 | INFO | "gpu" : " 17 GB", | 20260505 07:30:11.529 | INFO | "huge" : "", | 20260505 07:30:11.529 | INFO | "huge_shared" : " 27 GB", | 20260505 07:30:11.529 | INFO | "malloc" : " 14 GB", | 20260505 07:30:11.529 | INFO | "pinned" : " 29 GB" | 20260505 07:30:11.529 | INFO | } | 20260505 07:30:11.529 | INFO | Perf report for the last 5.13987 seconds: | 20260505 07:30:11.529 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:30:11.529 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:11.529 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:11.529 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:11.529 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 07:30:11.529 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:11.529 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:11.529 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:30:11.529 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:11.529 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:11.529 | INFO | Uploaded 36GiB to GPU in 2.15316s (16.7196GiB/s) | 20260505 07:30:11.529 | INFO | Downloaded 34GiB to GPU in 2.05794s (16.5214GiB/s) | 20260505 07:30:16.668 | INFO | Analyzing chunk frames = [1426944,1427456) | 20260505 07:30:16.668 | INFO | MemInfo: { | 20260505 07:30:16.668 | INFO | "gpu" : " 17 GB", | 20260505 07:30:16.668 | INFO | "huge" : "", | 20260505 07:30:16.668 | INFO | "huge_shared" : " 27 GB", | 20260505 07:30:16.668 | INFO | "malloc" : " 14 GB", | 20260505 07:30:16.668 | INFO | "pinned" : " 29 GB" | 20260505 07:30:16.668 | INFO | } | 20260505 07:30:16.668 | INFO | Perf report for the last 5.13914 seconds: | 20260505 07:30:16.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:30:16.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:16.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:16.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:16.668 | INFO | ANALYSIS : 63.3%, 3.0, 98.3ms, 0 | 20260505 07:30:16.668 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:16.668 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:16.668 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:30:16.668 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:16.668 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:16.668 | INFO | Uploaded 36GiB to GPU in 2.12337s (16.9542GiB/s) | 20260505 07:30:16.668 | INFO | Downloaded 34GiB to GPU in 2.03212s (16.7313GiB/s) | 20260505 07:30:21.807 | INFO | Analyzing chunk frames = [1427456,1427968) | 20260505 07:30:21.807 | INFO | MemInfo: { | 20260505 07:30:21.807 | INFO | "gpu" : " 17 GB", | 20260505 07:30:21.807 | INFO | "huge" : "", | 20260505 07:30:21.807 | INFO | "huge_shared" : " 27 GB", | 20260505 07:30:21.807 | INFO | "malloc" : " 14 GB", | 20260505 07:30:21.807 | INFO | "pinned" : " 29 GB" | 20260505 07:30:21.807 | INFO | } | 20260505 07:30:21.807 | INFO | Perf report for the last 5.13917 seconds: | 20260505 07:30:21.807 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:30:21.807 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:21.807 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:21.807 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:21.807 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 07:30:21.807 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:21.807 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:21.807 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 07:30:21.807 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:21.807 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:21.807 | INFO | Uploaded 36GiB to GPU in 2.15719s (16.6884GiB/s) | 20260505 07:30:21.807 | INFO | Downloaded 34GiB to GPU in 2.05062s (16.5803GiB/s) | 20260505 07:30:26.948 | INFO | Analyzing chunk frames = [1427968,1428480) | 20260505 07:30:26.948 | INFO | MemInfo: { | 20260505 07:30:26.948 | INFO | "gpu" : " 17 GB", | 20260505 07:30:26.948 | INFO | "huge" : "", | 20260505 07:30:26.948 | INFO | "huge_shared" : " 27 GB", | 20260505 07:30:26.948 | INFO | "malloc" : " 14 GB", | 20260505 07:30:26.948 | INFO | "pinned" : " 29 GB" | 20260505 07:30:26.948 | INFO | } | 20260505 07:30:26.948 | INFO | Perf report for the last 5.14104 seconds: | 20260505 07:30:26.948 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:30:26.949 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:26.949 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:26.949 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:26.949 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 07:30:26.949 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:26.949 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:26.949 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 07:30:26.949 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:26.949 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:26.949 | INFO | Uploaded 36GiB to GPU in 2.13991s (16.8231GiB/s) | 20260505 07:30:26.949 | INFO | Downloaded 34GiB to GPU in 2.05758s (16.5243GiB/s) | 20260505 07:30:32.086 | INFO | Analyzing chunk frames = [1428480,1428992) | 20260505 07:30:32.086 | INFO | MemInfo: { | 20260505 07:30:32.086 | INFO | "gpu" : " 17 GB", | 20260505 07:30:32.086 | INFO | "huge" : "", | 20260505 07:30:32.086 | INFO | "huge_shared" : " 27 GB", | 20260505 07:30:32.086 | INFO | "malloc" : " 14 GB", | 20260505 07:30:32.086 | INFO | "pinned" : " 29 GB" | 20260505 07:30:32.086 | INFO | } | 20260505 07:30:32.086 | INFO | Perf report for the last 5.13783 seconds: | 20260505 07:30:32.086 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:30:32.086 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:32.086 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:32.086 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:32.086 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 07:30:32.086 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:32.086 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:32.086 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260505 07:30:32.086 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:32.086 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:32.086 | INFO | Uploaded 36GiB to GPU in 2.20596s (16.3194GiB/s) | 20260505 07:30:32.086 | INFO | Downloaded 34GiB to GPU in 2.16597s (15.6974GiB/s) | 20260505 07:30:37.225 | INFO | Analyzing chunk frames = [1428992,1429504) | 20260505 07:30:37.225 | INFO | MemInfo: { | 20260505 07:30:37.225 | INFO | "gpu" : " 17 GB", | 20260505 07:30:37.225 | INFO | "huge" : "", | 20260505 07:30:37.225 | INFO | "huge_shared" : " 27 GB", | 20260505 07:30:37.225 | INFO | "malloc" : " 14 GB", | 20260505 07:30:37.225 | INFO | "pinned" : " 29 GB" | 20260505 07:30:37.225 | INFO | } | 20260505 07:30:37.225 | INFO | Perf report for the last 5.13909 seconds: | 20260505 07:30:37.225 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:30:37.225 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:37.225 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:37.225 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:37.226 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 07:30:37.226 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:37.226 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:37.226 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 07:30:37.226 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:37.226 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:37.226 | INFO | Uploaded 36GiB to GPU in 2.14436s (16.7882GiB/s) | 20260505 07:30:37.226 | INFO | Downloaded 34GiB to GPU in 2.02761s (16.7685GiB/s) | 20260505 07:30:37.360 | INFO | Starting a round of Baz IO | 20260505 07:30:40.646 | INFO | Populating RTMetrics report | 20260505 07:30:40.647 | INFO | RTMetrics report populated | 20260505 07:30:40.647 | INFO | Opening tmp RTMetrics json file | 20260505 07:30:40.648 | INFO | Serializing RTMetrics to json | 20260505 07:30:40.650 | INFO | Serialization done, swapping tmp file now... | 20260505 07:30:40.650 | INFO | RTMetrics done | 20260505 07:30:40.787 | INFO | Finished a round of Baz IO: Wrote 2.432 GiB in 3.427 seconds | 20260505 07:30:40.787 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 29.72% / 0.03% / 70.25% | 20260505 07:30:40.787 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.54867% / 3.17891e-05% / 6.75519e-05% | 20260505 07:30:40.787 | INFO | Baz IO overall: | 20260505 07:30:40.787 | INFO | BytesWritten : 781028452184 | 20260505 07:30:40.787 | INFO | NumEvents : 370606583539 | 20260505 07:30:42.365 | INFO | Analyzing chunk frames = [1429504,1430016) | 20260505 07:30:42.365 | INFO | MemInfo: { | 20260505 07:30:42.365 | INFO | "gpu" : " 17 GB", | 20260505 07:30:42.365 | INFO | "huge" : "", | 20260505 07:30:42.365 | INFO | "huge_shared" : " 27 GB", | 20260505 07:30:42.365 | INFO | "malloc" : " 14 GB", | 20260505 07:30:42.365 | INFO | "pinned" : " 29 GB" | 20260505 07:30:42.365 | INFO | } | 20260505 07:30:42.365 | INFO | Perf report for the last 5.13947 seconds: | 20260505 07:30:42.365 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:30:42.365 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:42.365 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:42.365 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:42.365 | INFO | ANALYSIS : 65.4%, 2.9, 99.5ms, 0 | 20260505 07:30:42.365 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:42.365 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260505 07:30:42.365 | INFO | PRE_HQ : 64.0%, 1.9, 62.4ms, 0 | 20260505 07:30:42.365 | INFO | BAZWRITER : 65.7%, 4.2, 140.8ms, 0 | 20260505 07:30:42.365 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 07:30:42.365 | INFO | Uploaded 36GiB to GPU in 2.22921s (16.1493GiB/s) | 20260505 07:30:42.365 | INFO | Downloaded 38GiB to GPU in 2.34446s (16.2084GiB/s) | 20260505 07:30:47.504 | INFO | Analyzing chunk frames = [1430016,1430528) | 20260505 07:30:47.504 | INFO | MemInfo: { | 20260505 07:30:47.504 | INFO | "gpu" : " 17 GB", | 20260505 07:30:47.504 | INFO | "huge" : "", | 20260505 07:30:47.504 | INFO | "huge_shared" : " 27 GB", | 20260505 07:30:47.504 | INFO | "malloc" : " 14 GB", | 20260505 07:30:47.504 | INFO | "pinned" : " 29 GB" | 20260505 07:30:47.504 | INFO | } | 20260505 07:30:47.504 | INFO | Perf report for the last 5.13926 seconds: | 20260505 07:30:47.504 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:30:47.504 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:47.504 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:47.504 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:47.504 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 07:30:47.504 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:47.504 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:47.504 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:30:47.504 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:47.504 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:47.505 | INFO | Uploaded 36GiB to GPU in 2.13732s (16.8436GiB/s) | 20260505 07:30:47.505 | INFO | Downloaded 34GiB to GPU in 2.04459s (16.6292GiB/s) | 20260505 07:30:52.643 | INFO | Analyzing chunk frames = [1430528,1431040) | 20260505 07:30:52.643 | INFO | MemInfo: { | 20260505 07:30:52.643 | INFO | "gpu" : " 17 GB", | 20260505 07:30:52.643 | INFO | "huge" : "", | 20260505 07:30:52.643 | INFO | "huge_shared" : " 27 GB", | 20260505 07:30:52.643 | INFO | "malloc" : " 14 GB", | 20260505 07:30:52.643 | INFO | "pinned" : " 29 GB" | 20260505 07:30:52.643 | INFO | } | 20260505 07:30:52.643 | INFO | Perf report for the last 5.13917 seconds: | 20260505 07:30:52.643 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:30:52.643 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:52.643 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:52.643 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:52.643 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260505 07:30:52.643 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:52.643 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:52.643 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 07:30:52.643 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:52.643 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:52.644 | INFO | Uploaded 36GiB to GPU in 2.15289s (16.7217GiB/s) | 20260505 07:30:52.644 | INFO | Downloaded 34GiB to GPU in 2.05787s (16.522GiB/s) | 20260505 07:30:57.783 | INFO | Analyzing chunk frames = [1431040,1431552) | 20260505 07:30:57.783 | INFO | MemInfo: { | 20260505 07:30:57.783 | INFO | "gpu" : " 17 GB", | 20260505 07:30:57.783 | INFO | "huge" : "", | 20260505 07:30:57.783 | INFO | "huge_shared" : " 27 GB", | 20260505 07:30:57.783 | INFO | "malloc" : " 14 GB", | 20260505 07:30:57.783 | INFO | "pinned" : " 29 GB" | 20260505 07:30:57.783 | INFO | } | 20260505 07:30:57.783 | INFO | Perf report for the last 5.1398 seconds: | 20260505 07:30:57.783 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:30:57.783 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:57.783 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:57.783 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:57.783 | INFO | ANALYSIS : 63.3%, 3.0, 98.3ms, 0 | 20260505 07:30:57.783 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:57.783 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:57.783 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:30:57.783 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:57.783 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:30:57.783 | INFO | Uploaded 36GiB to GPU in 2.11865s (16.992GiB/s) | 20260505 07:30:57.783 | INFO | Downloaded 34GiB to GPU in 2.02885s (16.7583GiB/s) | 20260505 07:31:02.922 | INFO | Analyzing chunk frames = [1431552,1432064) | 20260505 07:31:02.922 | INFO | MemInfo: { | 20260505 07:31:02.922 | INFO | "gpu" : " 17 GB", | 20260505 07:31:02.922 | INFO | "huge" : "", | 20260505 07:31:02.922 | INFO | "huge_shared" : " 27 GB", | 20260505 07:31:02.922 | INFO | "malloc" : " 14 GB", | 20260505 07:31:02.922 | INFO | "pinned" : " 29 GB" | 20260505 07:31:02.922 | INFO | } | 20260505 07:31:02.922 | INFO | Perf report for the last 5.13948 seconds: | 20260505 07:31:02.923 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:31:02.923 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:02.923 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:02.923 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:02.923 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 07:31:02.923 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:02.923 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:02.923 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 07:31:02.923 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:02.923 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:02.923 | INFO | Uploaded 36GiB to GPU in 2.1633s (16.6412GiB/s) | 20260505 07:31:02.923 | INFO | Downloaded 34GiB to GPU in 2.05658s (16.5323GiB/s) | 20260505 07:31:08.063 | INFO | Analyzing chunk frames = [1432064,1432576) | 20260505 07:31:08.063 | INFO | MemInfo: { | 20260505 07:31:08.063 | INFO | "gpu" : " 17 GB", | 20260505 07:31:08.063 | INFO | "huge" : "", | 20260505 07:31:08.063 | INFO | "huge_shared" : " 27 GB", | 20260505 07:31:08.063 | INFO | "malloc" : " 14 GB", | 20260505 07:31:08.063 | INFO | "pinned" : " 29 GB" | 20260505 07:31:08.063 | INFO | } | 20260505 07:31:08.063 | INFO | Perf report for the last 5.14053 seconds: | 20260505 07:31:08.063 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:31:08.063 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:08.063 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:08.063 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:08.063 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 07:31:08.063 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:08.063 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:08.063 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:31:08.063 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:08.063 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:08.063 | INFO | Uploaded 36GiB to GPU in 2.16292s (16.6442GiB/s) | 20260505 07:31:08.063 | INFO | Downloaded 34GiB to GPU in 2.06723s (16.4472GiB/s) | 20260505 07:31:13.201 | INFO | Analyzing chunk frames = [1432576,1433088) | 20260505 07:31:13.201 | INFO | MemInfo: { | 20260505 07:31:13.201 | INFO | "gpu" : " 17 GB", | 20260505 07:31:13.201 | INFO | "huge" : "", | 20260505 07:31:13.201 | INFO | "huge_shared" : " 27 GB", | 20260505 07:31:13.201 | INFO | "malloc" : " 14 GB", | 20260505 07:31:13.201 | INFO | "pinned" : " 29 GB" | 20260505 07:31:13.201 | INFO | } | 20260505 07:31:13.201 | INFO | Perf report for the last 5.13781 seconds: | 20260505 07:31:13.201 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:31:13.201 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:13.201 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:13.201 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:13.201 | INFO | ANALYSIS : 63.8%, 3.0, 99.5ms, 0 | 20260505 07:31:13.201 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:13.201 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:13.201 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 07:31:13.201 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:13.201 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:13.201 | INFO | Uploaded 36GiB to GPU in 2.16003s (16.6664GiB/s) | 20260505 07:31:13.201 | INFO | Downloaded 34GiB to GPU in 2.14387s (15.8591GiB/s) | 20260505 07:31:18.340 | INFO | Analyzing chunk frames = [1433088,1433600) | 20260505 07:31:18.340 | INFO | MemInfo: { | 20260505 07:31:18.340 | INFO | "gpu" : " 17 GB", | 20260505 07:31:18.340 | INFO | "huge" : "", | 20260505 07:31:18.340 | INFO | "huge_shared" : " 27 GB", | 20260505 07:31:18.340 | INFO | "malloc" : " 14 GB", | 20260505 07:31:18.340 | INFO | "pinned" : " 29 GB" | 20260505 07:31:18.340 | INFO | } | 20260505 07:31:18.340 | INFO | Perf report for the last 5.13945 seconds: | 20260505 07:31:18.340 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:31:18.340 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:18.340 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:18.340 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:18.340 | INFO | ANALYSIS : 64.1%, 3.0, 100.0ms, 0 | 20260505 07:31:18.340 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:18.340 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:18.340 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 07:31:18.340 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:18.340 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:18.341 | INFO | Uploaded 36GiB to GPU in 2.14777s (16.7616GiB/s) | 20260505 07:31:18.341 | INFO | Downloaded 34GiB to GPU in 2.03947s (16.671GiB/s) | 20260505 07:31:18.341 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 07:31:18.341 | INFO | Total mem usage: 15.3 GiB | 20260505 07:31:18.341 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 07:31:18.341 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.11%) | 20260505 07:31:18.341 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 07:31:18.341 | INFO | High Water Breakdown: | 20260505 07:31:18.341 | INFO | 60.7% : 8.77 GiB ( 93.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 07:31:18.341 | INFO | 39.2% : 5.66 GiB ( 85.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 07:31:18.341 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 07:31:18.341 | INFO | ------------------------------------------ | 20260505 07:31:18.341 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:31:18.341 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 07:31:18.341 | INFO | Total mem usage: 26.5 GiB | 20260505 07:31:18.341 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 07:31:18.341 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 07:31:18.341 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 07:31:18.341 | INFO | High Water Breakdown: | 20260505 07:31:18.341 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:31:18.341 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:31:18.341 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:31:18.341 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:31:18.341 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:31:18.341 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:31:18.341 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 07:31:18.341 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 07:31:18.341 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:31:18.341 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 07:31:18.341 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 07:31:18.341 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 07:31:18.341 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 07:31:18.341 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 07:31:18.341 | INFO | ------------------------------------------ | 20260505 07:31:18.341 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:31:18.341 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 07:31:18.341 | INFO | Total mem usage: 25.6 GiB | 20260505 07:31:18.341 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 07:31:18.341 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 07:31:18.341 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 07:31:18.341 | INFO | High Water Breakdown: | 20260505 07:31:18.341 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 07:31:18.341 | INFO | ------------------------------------------ | 20260505 07:31:18.341 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:31:18.341 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 07:31:18.341 | INFO | Total mem usage: 15.7 GiB | 20260505 07:31:18.341 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 07:31:18.341 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 07:31:18.341 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 07:31:18.341 | INFO | High Water Breakdown: | 20260505 07:31:18.341 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 07:31:18.341 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 07:31:18.341 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 07:31:18.341 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 07:31:18.341 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 07:31:18.341 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 07:31:18.341 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 07:31:18.341 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 07:31:18.341 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 07:31:18.341 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:31:18.341 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 07:31:18.341 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:31:18.341 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 07:31:18.341 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 07:31:18.341 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 07:31:18.341 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 07:31:18.341 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:31:18.341 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 07:31:18.341 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:31:18.341 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:31:18.341 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:31:18.342 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 07:31:18.342 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 07:31:18.342 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:31:18.342 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 07:31:18.342 | INFO | ------------------------------------------ | 20260505 07:31:18.342 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:31:18.342 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 07:31:18.342 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:31:18.342 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:31:18.342 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.49e+01, 1.37e+01 / 2.17e+00 | 20260505 07:31:18.342 | INFO | PerfReport: 0.29%, AggregateStats -- 9.41e-02 / 1.04e-01, 1.21e-02 / 6.62e-03 | 20260505 07:31:18.342 | INFO | PerfReport: 11.63%, Baseline -- 3.82e+00 / 4.45e+00, 6.25e-01 / 3.00e-01 | 20260505 07:31:18.342 | INFO | PerfReport: 3.31%, Histogram -- 1.09e+00 / 1.45e+00, 2.13e-01 / 1.23e-01 | 20260505 07:31:18.342 | INFO | PerfReport: 15.47%, DME -- 4.06e+01 / 4.43e+01, 4.03e+00 / 1.24e+00 | 20260505 07:31:18.342 | INFO | PerfReport: 56.21%, FrameLabeling -- 1.84e+01 / 1.92e+01, 1.97e+00 / 1.56e+00 | 20260505 07:31:18.342 | INFO | PerfReport: 7.65%, PulseAccumulating -- 2.51e+00 / 3.18e+00, 4.49e-01 / 2.30e-01 | 20260505 07:31:18.342 | INFO | PerfReport: 4.00%, Metrics -- 1.31e+00 / 1.71e+00, 3.70e-01 / 1.78e-01 | 20260505 07:31:18.342 | INFO | PerfReport: 1.44%, PulseSerializing -- 4.72e-01 / 5.47e-01, 9.07e-02 / 2.78e-02 | 20260505 07:31:18.342 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 07:31:18.342 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:31:18.342 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:31:18.342 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.24e+01 / 0.00e+00 | 20260505 07:31:18.342 | INFO | PerfReport: 22.23%, Upload -- 2.25e+01 / 0.00e+00, 7.93e+00 / 0.00e+00 | 20260505 07:31:18.342 | INFO | PerfReport: 21.60%, Download -- 2.19e+01 / 0.00e+00, 7.88e+00 / 0.00e+00 | 20260505 07:31:18.342 | INFO | PerfReport: 32.43%, Compute -- 3.28e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 07:31:18.342 | INFO | PerfReport: 1.52%, LockUpload -- 1.54e+00 / 0.00e+00, 7.15e+00 / 0.00e+00 | 20260505 07:31:18.342 | INFO | PerfReport: 0.91%, LockDownload -- 9.16e-01 / 0.00e+00, 2.36e+00 / 0.00e+00 | 20260505 07:31:18.342 | INFO | PerfReport: 21.30%, LockCompute -- 2.16e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260505 07:31:18.342 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.24e-02 / 0.00e+00 | 20260505 07:31:18.475 | INFO | Starting a round of Baz IO | 20260505 07:31:21.762 | INFO | Populating RTMetrics report | 20260505 07:31:21.762 | INFO | RTMetrics report populated | 20260505 07:31:21.762 | INFO | Opening tmp RTMetrics json file | 20260505 07:31:21.788 | INFO | Serializing RTMetrics to json | 20260505 07:31:21.830 | INFO | Serialization done, swapping tmp file now... | 20260505 07:31:21.832 | INFO | RTMetrics done | 20260505 07:31:21.906 | INFO | Finished a round of Baz IO: Wrote 2.434 GiB in 3.431 seconds | 20260505 07:31:21.907 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 29.69% / 0.03% / 70.28% | 20260505 07:31:21.907 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.54602% / 5.16574e-05% / 7.15256e-05% | 20260505 07:31:21.907 | INFO | Baz IO overall: | 20260505 07:31:21.907 | INFO | BytesWritten : 783642112858 | 20260505 07:31:21.907 | INFO | NumEvents : 371857271062 | 20260505 07:31:23.479 | INFO | Analyzing chunk frames = [1433600,1434112) | 20260505 07:31:23.479 | INFO | MemInfo: { | 20260505 07:31:23.479 | INFO | "gpu" : " 17 GB", | 20260505 07:31:23.479 | INFO | "huge" : "", | 20260505 07:31:23.479 | INFO | "huge_shared" : " 27 GB", | 20260505 07:31:23.479 | INFO | "malloc" : " 14 GB", | 20260505 07:31:23.479 | INFO | "pinned" : " 29 GB" | 20260505 07:31:23.479 | INFO | } | 20260505 07:31:23.479 | INFO | Perf report for the last 5.13912 seconds: | 20260505 07:31:23.479 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:31:23.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:23.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:23.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:23.479 | INFO | ANALYSIS : 65.4%, 2.9, 99.5ms, 0 | 20260505 07:31:23.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:23.479 | INFO | RT_METRICS : 47.3%, 1.0, 24.8ms, 0 | 20260505 07:31:23.480 | INFO | PRE_HQ : 64.1%, 1.9, 62.7ms, 0 | 20260505 07:31:23.480 | INFO | BAZWRITER : 66.1%, 4.2, 141.3ms, 0 | 20260505 07:31:23.480 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 07:31:23.480 | INFO | Uploaded 36GiB to GPU in 2.24771s (16.0163GiB/s) | 20260505 07:31:23.480 | INFO | Downloaded 38GiB to GPU in 2.31954s (16.3826GiB/s) | 20260505 07:31:28.619 | INFO | Analyzing chunk frames = [1434112,1434624) | 20260505 07:31:28.619 | INFO | MemInfo: { | 20260505 07:31:28.619 | INFO | "gpu" : " 17 GB", | 20260505 07:31:28.619 | INFO | "huge" : "", | 20260505 07:31:28.619 | INFO | "huge_shared" : " 27 GB", | 20260505 07:31:28.619 | INFO | "malloc" : " 14 GB", | 20260505 07:31:28.619 | INFO | "pinned" : " 29 GB" | 20260505 07:31:28.619 | INFO | } | 20260505 07:31:28.619 | INFO | Perf report for the last 5.13932 seconds: | 20260505 07:31:28.619 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:31:28.619 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:28.619 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:28.619 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:28.619 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 07:31:28.619 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:28.619 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:28.619 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:31:28.619 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:28.619 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:28.619 | INFO | Uploaded 36GiB to GPU in 2.1328s (16.8792GiB/s) | 20260505 07:31:28.619 | INFO | Downloaded 34GiB to GPU in 2.0464s (16.6146GiB/s) | 20260505 07:31:33.758 | INFO | Analyzing chunk frames = [1434624,1435136) | 20260505 07:31:33.758 | INFO | MemInfo: { | 20260505 07:31:33.758 | INFO | "gpu" : " 17 GB", | 20260505 07:31:33.758 | INFO | "huge" : "", | 20260505 07:31:33.758 | INFO | "huge_shared" : " 27 GB", | 20260505 07:31:33.758 | INFO | "malloc" : " 14 GB", | 20260505 07:31:33.758 | INFO | "pinned" : " 29 GB" | 20260505 07:31:33.758 | INFO | } | 20260505 07:31:33.758 | INFO | Perf report for the last 5.13929 seconds: | 20260505 07:31:33.758 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:31:33.758 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:33.758 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:33.758 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:33.758 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 07:31:33.758 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:33.758 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:33.758 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 07:31:33.758 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:33.758 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:33.758 | INFO | Uploaded 36GiB to GPU in 2.15121s (16.7348GiB/s) | 20260505 07:31:33.758 | INFO | Downloaded 34GiB to GPU in 2.05714s (16.5278GiB/s) | 20260505 07:31:38.898 | INFO | Analyzing chunk frames = [1435136,1435648) | 20260505 07:31:38.898 | INFO | MemInfo: { | 20260505 07:31:38.898 | INFO | "gpu" : " 17 GB", | 20260505 07:31:38.898 | INFO | "huge" : "", | 20260505 07:31:38.898 | INFO | "huge_shared" : " 27 GB", | 20260505 07:31:38.898 | INFO | "malloc" : " 14 GB", | 20260505 07:31:38.898 | INFO | "pinned" : " 29 GB" | 20260505 07:31:38.898 | INFO | } | 20260505 07:31:38.898 | INFO | Perf report for the last 5.13967 seconds: | 20260505 07:31:38.898 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:31:38.898 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:38.898 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:38.898 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:38.898 | INFO | ANALYSIS : 63.4%, 3.0, 98.4ms, 0 | 20260505 07:31:38.898 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:38.898 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:38.898 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:31:38.898 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:38.898 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:38.898 | INFO | Uploaded 36GiB to GPU in 2.13625s (16.8519GiB/s) | 20260505 07:31:38.898 | INFO | Downloaded 34GiB to GPU in 2.02918s (16.7555GiB/s) | 20260505 07:31:44.037 | INFO | Analyzing chunk frames = [1435648,1436160) | 20260505 07:31:44.037 | INFO | MemInfo: { | 20260505 07:31:44.037 | INFO | "gpu" : " 17 GB", | 20260505 07:31:44.037 | INFO | "huge" : "", | 20260505 07:31:44.037 | INFO | "huge_shared" : " 27 GB", | 20260505 07:31:44.037 | INFO | "malloc" : " 14 GB", | 20260505 07:31:44.037 | INFO | "pinned" : " 29 GB" | 20260505 07:31:44.037 | INFO | } | 20260505 07:31:44.037 | INFO | Perf report for the last 5.1393 seconds: | 20260505 07:31:44.037 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:31:44.037 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:44.037 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:44.037 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:44.037 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 07:31:44.037 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:44.037 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:44.037 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 07:31:44.037 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:44.037 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:44.037 | INFO | Uploaded 36GiB to GPU in 2.14638s (16.7724GiB/s) | 20260505 07:31:44.037 | INFO | Downloaded 34GiB to GPU in 2.05232s (16.5666GiB/s) | 20260505 07:31:49.178 | INFO | Analyzing chunk frames = [1436160,1436672) | 20260505 07:31:49.178 | INFO | MemInfo: { | 20260505 07:31:49.178 | INFO | "gpu" : " 17 GB", | 20260505 07:31:49.178 | INFO | "huge" : "", | 20260505 07:31:49.178 | INFO | "huge_shared" : " 27 GB", | 20260505 07:31:49.178 | INFO | "malloc" : " 14 GB", | 20260505 07:31:49.178 | INFO | "pinned" : " 29 GB" | 20260505 07:31:49.178 | INFO | } | 20260505 07:31:49.178 | INFO | Perf report for the last 5.14115 seconds: | 20260505 07:31:49.178 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:31:49.178 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:49.178 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:49.178 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:49.178 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 07:31:49.178 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:49.178 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:49.178 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:31:49.178 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:49.178 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:49.178 | INFO | Uploaded 36GiB to GPU in 2.15891s (16.6751GiB/s) | 20260505 07:31:49.178 | INFO | Downloaded 34GiB to GPU in 2.06106s (16.4964GiB/s) | 20260505 07:31:54.315 | INFO | Analyzing chunk frames = [1436672,1437184) | 20260505 07:31:54.315 | INFO | MemInfo: { | 20260505 07:31:54.315 | INFO | "gpu" : " 17 GB", | 20260505 07:31:54.315 | INFO | "huge" : "", | 20260505 07:31:54.315 | INFO | "huge_shared" : " 27 GB", | 20260505 07:31:54.315 | INFO | "malloc" : " 14 GB", | 20260505 07:31:54.315 | INFO | "pinned" : " 29 GB" | 20260505 07:31:54.315 | INFO | } | 20260505 07:31:54.315 | INFO | Perf report for the last 5.13722 seconds: | 20260505 07:31:54.315 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:31:54.315 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:54.315 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:54.315 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:54.316 | INFO | ANALYSIS : 64.1%, 3.0, 100.0ms, 0 | 20260505 07:31:54.316 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:54.316 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:54.316 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 07:31:54.316 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:54.316 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:54.316 | INFO | Uploaded 36GiB to GPU in 2.20211s (16.348GiB/s) | 20260505 07:31:54.316 | INFO | Downloaded 34GiB to GPU in 2.15038s (15.8111GiB/s) | 20260505 07:31:59.455 | INFO | Analyzing chunk frames = [1437184,1437696) | 20260505 07:31:59.455 | INFO | MemInfo: { | 20260505 07:31:59.455 | INFO | "gpu" : " 17 GB", | 20260505 07:31:59.455 | INFO | "huge" : "", | 20260505 07:31:59.455 | INFO | "huge_shared" : " 27 GB", | 20260505 07:31:59.455 | INFO | "malloc" : " 14 GB", | 20260505 07:31:59.455 | INFO | "pinned" : " 29 GB" | 20260505 07:31:59.455 | INFO | } | 20260505 07:31:59.455 | INFO | Perf report for the last 5.1399 seconds: | 20260505 07:31:59.455 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:31:59.455 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:59.455 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:59.455 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:59.455 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 07:31:59.455 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:59.455 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:59.456 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 07:31:59.456 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:59.456 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:31:59.456 | INFO | Uploaded 36GiB to GPU in 2.16921s (16.5959GiB/s) | 20260505 07:31:59.456 | INFO | Downloaded 34GiB to GPU in 2.03215s (16.7311GiB/s) | 20260505 07:31:59.590 | INFO | Starting a round of Baz IO | 20260505 07:32:02.881 | INFO | Populating RTMetrics report | 20260505 07:32:02.882 | INFO | RTMetrics report populated | 20260505 07:32:02.882 | INFO | Opening tmp RTMetrics json file | 20260505 07:32:02.901 | INFO | Serializing RTMetrics to json | 20260505 07:32:02.903 | INFO | Serialization done, swapping tmp file now... | 20260505 07:32:02.903 | INFO | RTMetrics done | 20260505 07:32:03.017 | INFO | Finished a round of Baz IO: Wrote 2.435 GiB in 3.427 seconds | 20260505 07:32:03.017 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 29.66% / 0.03% / 70.31% | 20260505 07:32:03.017 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.53363% / 5.5631e-05% / 7.15256e-05% | 20260505 07:32:03.017 | INFO | Baz IO overall: | 20260505 07:32:03.017 | INFO | BytesWritten : 786257123313 | 20260505 07:32:03.017 | INFO | NumEvents : 373108941810 | 20260505 07:32:04.595 | INFO | Analyzing chunk frames = [1437696,1438208) | 20260505 07:32:04.595 | INFO | MemInfo: { | 20260505 07:32:04.595 | INFO | "gpu" : " 17 GB", | 20260505 07:32:04.595 | INFO | "huge" : "", | 20260505 07:32:04.595 | INFO | "huge_shared" : " 27 GB", | 20260505 07:32:04.595 | INFO | "malloc" : " 14 GB", | 20260505 07:32:04.595 | INFO | "pinned" : " 29 GB" | 20260505 07:32:04.595 | INFO | } | 20260505 07:32:04.595 | INFO | Perf report for the last 5.13948 seconds: | 20260505 07:32:04.595 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:32:04.595 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:04.595 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:04.595 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:04.595 | INFO | ANALYSIS : 65.5%, 2.9, 99.9ms, 0 | 20260505 07:32:04.595 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:04.595 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260505 07:32:04.595 | INFO | PRE_HQ : 64.1%, 1.9, 62.7ms, 0 | 20260505 07:32:04.595 | INFO | BAZWRITER : 66.0%, 4.1, 140.4ms, 0 | 20260505 07:32:04.595 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 07:32:04.595 | INFO | Uploaded 36GiB to GPU in 2.23814s (16.0848GiB/s) | 20260505 07:32:04.595 | INFO | Downloaded 38GiB to GPU in 2.34987s (16.1711GiB/s) | 20260505 07:32:09.734 | INFO | Analyzing chunk frames = [1438208,1438720) | 20260505 07:32:09.734 | INFO | MemInfo: { | 20260505 07:32:09.734 | INFO | "gpu" : " 17 GB", | 20260505 07:32:09.734 | INFO | "huge" : "", | 20260505 07:32:09.734 | INFO | "huge_shared" : " 27 GB", | 20260505 07:32:09.734 | INFO | "malloc" : " 14 GB", | 20260505 07:32:09.734 | INFO | "pinned" : " 29 GB" | 20260505 07:32:09.734 | INFO | } | 20260505 07:32:09.734 | INFO | Perf report for the last 5.13883 seconds: | 20260505 07:32:09.734 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:32:09.734 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:09.734 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:09.734 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:09.734 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 07:32:09.734 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:09.734 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:09.734 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:32:09.734 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:09.734 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:09.734 | INFO | Uploaded 36GiB to GPU in 2.1792s (16.5198GiB/s) | 20260505 07:32:09.734 | INFO | Downloaded 34GiB to GPU in 2.0588s (16.5145GiB/s) | 20260505 07:32:14.873 | INFO | Analyzing chunk frames = [1438720,1439232) | 20260505 07:32:14.873 | INFO | MemInfo: { | 20260505 07:32:14.873 | INFO | "gpu" : " 17 GB", | 20260505 07:32:14.873 | INFO | "huge" : "", | 20260505 07:32:14.873 | INFO | "huge_shared" : " 27 GB", | 20260505 07:32:14.873 | INFO | "malloc" : " 14 GB", | 20260505 07:32:14.873 | INFO | "pinned" : " 29 GB" | 20260505 07:32:14.873 | INFO | } | 20260505 07:32:14.873 | INFO | Perf report for the last 5.13924 seconds: | 20260505 07:32:14.873 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:32:14.873 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:14.873 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:14.873 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:14.873 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 07:32:14.873 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:14.873 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:14.873 | INFO | PRE_HQ : 30.3%, 1.0, 15.8ms, 0 | 20260505 07:32:14.873 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:14.873 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:14.873 | INFO | Uploaded 36GiB to GPU in 2.14956s (16.7476GiB/s) | 20260505 07:32:14.873 | INFO | Downloaded 34GiB to GPU in 2.05616s (16.5357GiB/s) | 20260505 07:32:20.012 | INFO | Analyzing chunk frames = [1439232,1439744) | 20260505 07:32:20.012 | INFO | MemInfo: { | 20260505 07:32:20.012 | INFO | "gpu" : " 17 GB", | 20260505 07:32:20.012 | INFO | "huge" : "", | 20260505 07:32:20.012 | INFO | "huge_shared" : " 27 GB", | 20260505 07:32:20.012 | INFO | "malloc" : " 14 GB", | 20260505 07:32:20.012 | INFO | "pinned" : " 29 GB" | 20260505 07:32:20.012 | INFO | } | 20260505 07:32:20.012 | INFO | Perf report for the last 5.13936 seconds: | 20260505 07:32:20.012 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:32:20.012 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:20.012 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:20.012 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:20.012 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 07:32:20.012 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:20.012 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:20.012 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:32:20.012 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:20.012 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:20.013 | INFO | Uploaded 36GiB to GPU in 2.12414s (16.9481GiB/s) | 20260505 07:32:20.013 | INFO | Downloaded 34GiB to GPU in 2.03239s (16.7291GiB/s) | 20260505 07:32:25.151 | INFO | Analyzing chunk frames = [1439744,1440256) | 20260505 07:32:25.151 | INFO | MemInfo: { | 20260505 07:32:25.151 | INFO | "gpu" : " 17 GB", | 20260505 07:32:25.151 | INFO | "huge" : "", | 20260505 07:32:25.152 | INFO | "huge_shared" : " 27 GB", | 20260505 07:32:25.152 | INFO | "malloc" : " 14 GB", | 20260505 07:32:25.152 | INFO | "pinned" : " 29 GB" | 20260505 07:32:25.152 | INFO | } | 20260505 07:32:25.152 | INFO | Perf report for the last 5.13927 seconds: | 20260505 07:32:25.152 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:32:25.152 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:25.152 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:25.152 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:25.152 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 07:32:25.152 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:25.152 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:25.152 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 07:32:25.152 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:25.152 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:25.152 | INFO | Uploaded 36GiB to GPU in 2.14798s (16.7599GiB/s) | 20260505 07:32:25.152 | INFO | Downloaded 34GiB to GPU in 2.05479s (16.5467GiB/s) | 20260505 07:32:30.292 | INFO | Analyzing chunk frames = [1440256,1440768) | 20260505 07:32:30.292 | INFO | MemInfo: { | 20260505 07:32:30.292 | INFO | "gpu" : " 17 GB", | 20260505 07:32:30.292 | INFO | "huge" : "", | 20260505 07:32:30.293 | INFO | "huge_shared" : " 27 GB", | 20260505 07:32:30.293 | INFO | "malloc" : " 14 GB", | 20260505 07:32:30.293 | INFO | "pinned" : " 29 GB" | 20260505 07:32:30.293 | INFO | } | 20260505 07:32:30.293 | INFO | Perf report for the last 5.14098 seconds: | 20260505 07:32:30.293 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:32:30.293 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:30.293 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:30.293 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:30.293 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 07:32:30.293 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:30.293 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:30.293 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 07:32:30.293 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:30.293 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:30.293 | INFO | Uploaded 36GiB to GPU in 2.14698s (16.7677GiB/s) | 20260505 07:32:30.293 | INFO | Downloaded 34GiB to GPU in 2.06215s (16.4876GiB/s) | 20260505 07:32:35.430 | INFO | Analyzing chunk frames = [1440768,1441280) | 20260505 07:32:35.430 | INFO | MemInfo: { | 20260505 07:32:35.430 | INFO | "gpu" : " 17 GB", | 20260505 07:32:35.430 | INFO | "huge" : "", | 20260505 07:32:35.430 | INFO | "huge_shared" : " 27 GB", | 20260505 07:32:35.430 | INFO | "malloc" : " 14 GB", | 20260505 07:32:35.430 | INFO | "pinned" : " 29 GB" | 20260505 07:32:35.430 | INFO | } | 20260505 07:32:35.430 | INFO | Perf report for the last 5.13776 seconds: | 20260505 07:32:35.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:32:35.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:35.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:35.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:35.430 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 07:32:35.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:35.430 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:35.430 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 07:32:35.430 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:35.430 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:35.431 | INFO | Uploaded 36GiB to GPU in 2.19922s (16.3694GiB/s) | 20260505 07:32:35.431 | INFO | Downloaded 34GiB to GPU in 2.16253s (15.7223GiB/s) | 20260505 07:32:40.569 | INFO | Analyzing chunk frames = [1441280,1441792) | 20260505 07:32:40.569 | INFO | MemInfo: { | 20260505 07:32:40.569 | INFO | "gpu" : " 17 GB", | 20260505 07:32:40.569 | INFO | "huge" : "", | 20260505 07:32:40.569 | INFO | "huge_shared" : " 27 GB", | 20260505 07:32:40.569 | INFO | "malloc" : " 14 GB", | 20260505 07:32:40.569 | INFO | "pinned" : " 29 GB" | 20260505 07:32:40.570 | INFO | } | 20260505 07:32:40.570 | INFO | Perf report for the last 5.13916 seconds: | 20260505 07:32:40.570 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:32:40.570 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:40.570 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:40.570 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:40.570 | INFO | ANALYSIS : 64.1%, 3.0, 100.0ms, 0 | 20260505 07:32:40.570 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:40.570 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:40.570 | INFO | PRE_HQ : 30.5%, 1.0, 15.9ms, 0 | 20260505 07:32:40.570 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:40.570 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:40.570 | INFO | Uploaded 36GiB to GPU in 2.17876s (16.5232GiB/s) | 20260505 07:32:40.570 | INFO | Downloaded 34GiB to GPU in 2.05112s (16.5763GiB/s) | 20260505 07:32:40.570 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 07:32:40.570 | INFO | Total mem usage: 15.3 GiB | 20260505 07:32:40.570 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 07:32:40.570 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.61%) | 20260505 07:32:40.570 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 07:32:40.570 | INFO | High Water Breakdown: | 20260505 07:32:40.570 | INFO | 60.5% : 8.7 GiB ( 93% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 07:32:40.570 | INFO | 39.3% : 5.66 GiB ( 85.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 07:32:40.570 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 07:32:40.570 | INFO | ------------------------------------------ | 20260505 07:32:40.570 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:32:40.570 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 07:32:40.570 | INFO | Total mem usage: 26.5 GiB | 20260505 07:32:40.570 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 07:32:40.570 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 07:32:40.570 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 07:32:40.570 | INFO | High Water Breakdown: | 20260505 07:32:40.570 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:32:40.570 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:32:40.570 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:32:40.570 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:32:40.570 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:32:40.570 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:32:40.570 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 07:32:40.570 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 07:32:40.570 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:32:40.570 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 07:32:40.570 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 07:32:40.570 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 07:32:40.570 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 07:32:40.570 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 07:32:40.570 | INFO | ------------------------------------------ | 20260505 07:32:40.570 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:32:40.570 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 07:32:40.570 | INFO | Total mem usage: 25.6 GiB | 20260505 07:32:40.571 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 07:32:40.571 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 07:32:40.571 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 07:32:40.571 | INFO | High Water Breakdown: | 20260505 07:32:40.571 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 07:32:40.571 | INFO | ------------------------------------------ | 20260505 07:32:40.571 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:32:40.571 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 07:32:40.571 | INFO | Total mem usage: 15.7 GiB | 20260505 07:32:40.571 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 07:32:40.571 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 07:32:40.571 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 07:32:40.571 | INFO | High Water Breakdown: | 20260505 07:32:40.571 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 07:32:40.571 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 07:32:40.571 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 07:32:40.571 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 07:32:40.571 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 07:32:40.571 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 07:32:40.571 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 07:32:40.571 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 07:32:40.571 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 07:32:40.571 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:32:40.571 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 07:32:40.571 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:32:40.571 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 07:32:40.571 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 07:32:40.571 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 07:32:40.571 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 07:32:40.571 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:32:40.571 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 07:32:40.571 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:32:40.571 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:32:40.571 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:32:40.571 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 07:32:40.571 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 07:32:40.571 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:32:40.571 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 07:32:40.571 | INFO | ------------------------------------------ | 20260505 07:32:40.571 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:32:40.571 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 07:32:40.571 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:32:40.571 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:32:40.571 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.51e+01, 1.37e+01 / 2.13e+00 | 20260505 07:32:40.571 | INFO | PerfReport: 0.29%, AggregateStats -- 9.36e-02 / 1.03e-01, 1.06e-02 / 6.94e-03 | 20260505 07:32:40.571 | INFO | PerfReport: 11.62%, Baseline -- 3.81e+00 / 4.44e+00, 6.26e-01 / 2.98e-01 | 20260505 07:32:40.571 | INFO | PerfReport: 3.30%, Histogram -- 1.08e+00 / 1.46e+00, 2.12e-01 / 1.31e-01 | 20260505 07:32:40.571 | INFO | PerfReport: 15.47%, DME -- 4.06e+01 / 4.43e+01, 4.14e+00 / 1.44e+00 | 20260505 07:32:40.571 | INFO | PerfReport: 56.24%, FrameLabeling -- 1.85e+01 / 1.94e+01, 1.99e+00 / 1.59e+00 | 20260505 07:32:40.571 | INFO | PerfReport: 7.66%, PulseAccumulating -- 2.51e+00 / 3.16e+00, 4.54e-01 / 2.46e-01 | 20260505 07:32:40.571 | INFO | PerfReport: 3.98%, Metrics -- 1.31e+00 / 1.71e+00, 3.67e-01 / 1.51e-01 | 20260505 07:32:40.571 | INFO | PerfReport: 1.44%, PulseSerializing -- 4.72e-01 / 5.47e-01, 9.02e-02 / 2.84e-02 | 20260505 07:32:40.571 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 07:32:40.571 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:32:40.571 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:32:40.571 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.24e+01 / 0.00e+00 | 20260505 07:32:40.571 | INFO | PerfReport: 22.29%, Upload -- 2.26e+01 / 0.00e+00, 7.97e+00 / 0.00e+00 | 20260505 07:32:40.571 | INFO | PerfReport: 21.57%, Download -- 2.19e+01 / 0.00e+00, 7.90e+00 / 0.00e+00 | 20260505 07:32:40.571 | INFO | PerfReport: 32.38%, Compute -- 3.28e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 07:32:40.571 | INFO | PerfReport: 1.54%, LockUpload -- 1.56e+00 / 0.00e+00, 7.16e+00 / 0.00e+00 | 20260505 07:32:40.571 | INFO | PerfReport: 0.85%, LockDownload -- 8.64e-01 / 0.00e+00, 2.23e+00 / 0.00e+00 | 20260505 07:32:40.571 | INFO | PerfReport: 21.35%, LockCompute -- 2.16e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260505 07:32:40.571 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 1.83e-02 / 0.00e+00 | 20260505 07:32:40.703 | INFO | Starting a round of Baz IO | 20260505 07:32:43.984 | INFO | Populating RTMetrics report | 20260505 07:32:43.984 | INFO | RTMetrics report populated | 20260505 07:32:43.984 | INFO | Opening tmp RTMetrics json file | 20260505 07:32:44.038 | INFO | Serializing RTMetrics to json | 20260505 07:32:44.065 | INFO | Serialization done, swapping tmp file now... | 20260505 07:32:44.067 | INFO | RTMetrics done | 20260505 07:32:44.162 | INFO | Finished a round of Baz IO: Wrote 2.432 GiB in 3.458 seconds | 20260505 07:32:44.162 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 29.63% / 0.03% / 70.34% | 20260505 07:32:44.162 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.50814% / 8.34465e-05% / 5.96046e-05% | 20260505 07:32:44.162 | INFO | Baz IO overall: | 20260505 07:32:44.162 | INFO | BytesWritten : 788868383251 | 20260505 07:32:44.162 | INFO | NumEvents : 374357675674 | 20260505 07:32:45.709 | INFO | Analyzing chunk frames = [1441792,1442304) | 20260505 07:32:45.709 | INFO | MemInfo: { | 20260505 07:32:45.709 | INFO | "gpu" : " 17 GB", | 20260505 07:32:45.709 | INFO | "huge" : "", | 20260505 07:32:45.709 | INFO | "huge_shared" : " 27 GB", | 20260505 07:32:45.709 | INFO | "malloc" : " 14 GB", | 20260505 07:32:45.709 | INFO | "pinned" : " 29 GB" | 20260505 07:32:45.709 | INFO | } | 20260505 07:32:45.709 | INFO | Perf report for the last 5.1395 seconds: | 20260505 07:32:45.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:32:45.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:45.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:45.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:45.709 | INFO | ANALYSIS : 65.3%, 2.9, 99.6ms, 0 | 20260505 07:32:45.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:45.709 | INFO | RT_METRICS : 47.5%, 1.0, 24.9ms, 0 | 20260505 07:32:45.709 | INFO | PRE_HQ : 63.8%, 1.9, 62.5ms, 0 | 20260505 07:32:45.709 | INFO | BAZWRITER : 66.6%, 4.1, 141.2ms, 0 | 20260505 07:32:45.709 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260505 07:32:45.710 | INFO | Uploaded 36GiB to GPU in 2.21018s (16.2882GiB/s) | 20260505 07:32:45.710 | INFO | Downloaded 38GiB to GPU in 2.33929s (16.2443GiB/s) | 20260505 07:32:50.848 | INFO | Analyzing chunk frames = [1442304,1442816) | 20260505 07:32:50.848 | INFO | MemInfo: { | 20260505 07:32:50.848 | INFO | "gpu" : " 17 GB", | 20260505 07:32:50.848 | INFO | "huge" : "", | 20260505 07:32:50.848 | INFO | "huge_shared" : " 27 GB", | 20260505 07:32:50.848 | INFO | "malloc" : " 14 GB", | 20260505 07:32:50.849 | INFO | "pinned" : " 29 GB" | 20260505 07:32:50.849 | INFO | } | 20260505 07:32:50.849 | INFO | Perf report for the last 5.1395 seconds: | 20260505 07:32:50.849 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:32:50.849 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:50.849 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:50.849 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:50.849 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 07:32:50.849 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:50.849 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:50.849 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:32:50.849 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:50.849 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:50.849 | INFO | Uploaded 36GiB to GPU in 2.18744s (16.4576GiB/s) | 20260505 07:32:50.849 | INFO | Downloaded 34GiB to GPU in 2.05684s (16.5302GiB/s) | 20260505 07:32:55.988 | INFO | Analyzing chunk frames = [1442816,1443328) | 20260505 07:32:55.988 | INFO | MemInfo: { | 20260505 07:32:55.988 | INFO | "gpu" : " 17 GB", | 20260505 07:32:55.988 | INFO | "huge" : "", | 20260505 07:32:55.988 | INFO | "huge_shared" : " 27 GB", | 20260505 07:32:55.988 | INFO | "malloc" : " 14 GB", | 20260505 07:32:55.988 | INFO | "pinned" : " 29 GB" | 20260505 07:32:55.988 | INFO | } | 20260505 07:32:55.988 | INFO | Perf report for the last 5.13922 seconds: | 20260505 07:32:55.988 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:32:55.988 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:55.988 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:55.988 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:55.988 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260505 07:32:55.988 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:55.988 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:55.988 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:32:55.988 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:55.988 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:32:55.988 | INFO | Uploaded 36GiB to GPU in 2.15508s (16.7047GiB/s) | 20260505 07:32:55.988 | INFO | Downloaded 34GiB to GPU in 2.05484s (16.5463GiB/s) | 20260505 07:33:01.127 | INFO | Analyzing chunk frames = [1443328,1443840) | 20260505 07:33:01.127 | INFO | MemInfo: { | 20260505 07:33:01.127 | INFO | "gpu" : " 17 GB", | 20260505 07:33:01.127 | INFO | "huge" : "", | 20260505 07:33:01.127 | INFO | "huge_shared" : " 27 GB", | 20260505 07:33:01.127 | INFO | "malloc" : " 14 GB", | 20260505 07:33:01.127 | INFO | "pinned" : " 29 GB" | 20260505 07:33:01.127 | INFO | } | 20260505 07:33:01.127 | INFO | Perf report for the last 5.13954 seconds: | 20260505 07:33:01.127 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:33:01.127 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:01.127 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:01.127 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:01.127 | INFO | ANALYSIS : 63.3%, 3.0, 98.3ms, 0 | 20260505 07:33:01.127 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:01.127 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:01.127 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:33:01.128 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:01.128 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:01.128 | INFO | Uploaded 36GiB to GPU in 2.13265s (16.8804GiB/s) | 20260505 07:33:01.128 | INFO | Downloaded 34GiB to GPU in 2.03548s (16.7037GiB/s) | 20260505 07:33:06.266 | INFO | Analyzing chunk frames = [1443840,1444352) | 20260505 07:33:06.266 | INFO | MemInfo: { | 20260505 07:33:06.266 | INFO | "gpu" : " 17 GB", | 20260505 07:33:06.266 | INFO | "huge" : "", | 20260505 07:33:06.266 | INFO | "huge_shared" : " 27 GB", | 20260505 07:33:06.266 | INFO | "malloc" : " 14 GB", | 20260505 07:33:06.266 | INFO | "pinned" : " 29 GB" | 20260505 07:33:06.266 | INFO | } | 20260505 07:33:06.266 | INFO | Perf report for the last 5.13909 seconds: | 20260505 07:33:06.267 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:33:06.267 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:06.267 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:06.267 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:06.267 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 07:33:06.267 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:06.267 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:06.267 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 07:33:06.267 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:06.267 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:06.267 | INFO | Uploaded 36GiB to GPU in 2.16275s (16.6455GiB/s) | 20260505 07:33:06.267 | INFO | Downloaded 34GiB to GPU in 2.04631s (16.6153GiB/s) | 20260505 07:33:11.407 | INFO | Analyzing chunk frames = [1444352,1444864) | 20260505 07:33:11.407 | INFO | MemInfo: { | 20260505 07:33:11.407 | INFO | "gpu" : " 17 GB", | 20260505 07:33:11.407 | INFO | "huge" : "", | 20260505 07:33:11.407 | INFO | "huge_shared" : " 27 GB", | 20260505 07:33:11.407 | INFO | "malloc" : " 14 GB", | 20260505 07:33:11.407 | INFO | "pinned" : " 29 GB" | 20260505 07:33:11.407 | INFO | } | 20260505 07:33:11.408 | INFO | Perf report for the last 5.14101 seconds: | 20260505 07:33:11.408 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:33:11.408 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:11.408 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:11.408 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:11.408 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 07:33:11.408 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:11.408 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:11.408 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 07:33:11.408 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:11.408 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:11.408 | INFO | Uploaded 36GiB to GPU in 2.14865s (16.7547GiB/s) | 20260505 07:33:11.408 | INFO | Downloaded 34GiB to GPU in 2.05673s (16.5311GiB/s) | 20260505 07:33:16.545 | INFO | Analyzing chunk frames = [1444864,1445376) | 20260505 07:33:16.545 | INFO | MemInfo: { | 20260505 07:33:16.545 | INFO | "gpu" : " 17 GB", | 20260505 07:33:16.545 | INFO | "huge" : "", | 20260505 07:33:16.545 | INFO | "huge_shared" : " 27 GB", | 20260505 07:33:16.545 | INFO | "malloc" : " 14 GB", | 20260505 07:33:16.545 | INFO | "pinned" : " 29 GB" | 20260505 07:33:16.545 | INFO | } | 20260505 07:33:16.545 | INFO | Perf report for the last 5.13762 seconds: | 20260505 07:33:16.545 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:33:16.545 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:16.545 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:16.545 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:16.545 | INFO | ANALYSIS : 64.6%, 3.0, 100.9ms, 0 | 20260505 07:33:16.545 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:16.545 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:16.545 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260505 07:33:16.545 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:16.545 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:16.545 | INFO | Uploaded 36GiB to GPU in 2.22525s (16.178GiB/s) | 20260505 07:33:16.545 | INFO | Downloaded 34GiB to GPU in 2.15392s (15.7852GiB/s) | 20260505 07:33:21.684 | INFO | Analyzing chunk frames = [1445376,1445888) | 20260505 07:33:21.684 | INFO | MemInfo: { | 20260505 07:33:21.684 | INFO | "gpu" : " 17 GB", | 20260505 07:33:21.684 | INFO | "huge" : "", | 20260505 07:33:21.684 | INFO | "huge_shared" : " 27 GB", | 20260505 07:33:21.684 | INFO | "malloc" : " 14 GB", | 20260505 07:33:21.684 | INFO | "pinned" : " 29 GB" | 20260505 07:33:21.684 | INFO | } | 20260505 07:33:21.684 | INFO | Perf report for the last 5.13915 seconds: | 20260505 07:33:21.684 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:33:21.684 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:21.684 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:21.684 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:21.684 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 07:33:21.684 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:21.684 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:21.684 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 07:33:21.684 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:21.684 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:21.684 | INFO | Uploaded 36GiB to GPU in 2.1782s (16.5274GiB/s) | 20260505 07:33:21.685 | INFO | Downloaded 34GiB to GPU in 2.04836s (16.5986GiB/s) | 20260505 07:33:21.818 | INFO | Starting a round of Baz IO | 20260505 07:33:25.101 | INFO | Populating RTMetrics report | 20260505 07:33:25.101 | INFO | RTMetrics report populated | 20260505 07:33:25.101 | INFO | Opening tmp RTMetrics json file | 20260505 07:33:25.102 | INFO | Serializing RTMetrics to json | 20260505 07:33:25.105 | INFO | Serialization done, swapping tmp file now... | 20260505 07:33:25.108 | INFO | RTMetrics done | 20260505 07:33:25.252 | INFO | Finished a round of Baz IO: Wrote 2.435 GiB in 3.433 seconds | 20260505 07:33:25.252 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 29.60% / 0.03% / 70.37% | 20260505 07:33:25.252 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.50883% / 8.34465e-05% / 5.96046e-05% | 20260505 07:33:25.252 | INFO | Baz IO overall: | 20260505 07:33:25.252 | INFO | BytesWritten : 791483181546 | 20260505 07:33:25.252 | INFO | NumEvents : 375609085629 | 20260505 07:33:26.824 | INFO | Analyzing chunk frames = [1445888,1446400) | 20260505 07:33:26.824 | INFO | MemInfo: { | 20260505 07:33:26.824 | INFO | "gpu" : " 17 GB", | 20260505 07:33:26.824 | INFO | "huge" : "", | 20260505 07:33:26.824 | INFO | "huge_shared" : " 27 GB", | 20260505 07:33:26.824 | INFO | "malloc" : " 14 GB", | 20260505 07:33:26.824 | INFO | "pinned" : " 29 GB" | 20260505 07:33:26.824 | INFO | } | 20260505 07:33:26.824 | INFO | Perf report for the last 5.13937 seconds: | 20260505 07:33:26.824 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:33:26.824 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:26.824 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:26.824 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:26.824 | INFO | ANALYSIS : 65.3%, 2.9, 99.5ms, 0 | 20260505 07:33:26.824 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:26.824 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 07:33:26.824 | INFO | PRE_HQ : 64.0%, 1.9, 62.7ms, 0 | 20260505 07:33:26.824 | INFO | BAZWRITER : 66.1%, 4.2, 141.4ms, 0 | 20260505 07:33:26.824 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 07:33:26.824 | INFO | Uploaded 36GiB to GPU in 2.26096s (15.9224GiB/s) | 20260505 07:33:26.824 | INFO | Downloaded 38GiB to GPU in 2.33405s (16.2807GiB/s) | 20260505 07:33:31.963 | INFO | Analyzing chunk frames = [1446400,1446912) | 20260505 07:33:31.963 | INFO | MemInfo: { | 20260505 07:33:31.963 | INFO | "gpu" : " 17 GB", | 20260505 07:33:31.963 | INFO | "huge" : "", | 20260505 07:33:31.963 | INFO | "huge_shared" : " 27 GB", | 20260505 07:33:31.963 | INFO | "malloc" : " 14 GB", | 20260505 07:33:31.963 | INFO | "pinned" : " 29 GB" | 20260505 07:33:31.963 | INFO | } | 20260505 07:33:31.963 | INFO | Perf report for the last 5.13924 seconds: | 20260505 07:33:31.963 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:33:31.963 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:31.963 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:31.963 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:31.963 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 07:33:31.963 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:31.963 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:31.963 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:33:31.963 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:31.963 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:31.963 | INFO | Uploaded 36GiB to GPU in 2.13274s (16.8797GiB/s) | 20260505 07:33:31.963 | INFO | Downloaded 34GiB to GPU in 2.0496s (16.5886GiB/s) | 20260505 07:33:37.102 | INFO | Analyzing chunk frames = [1446912,1447424) | 20260505 07:33:37.102 | INFO | MemInfo: { | 20260505 07:33:37.102 | INFO | "gpu" : " 17 GB", | 20260505 07:33:37.102 | INFO | "huge" : "", | 20260505 07:33:37.102 | INFO | "huge_shared" : " 27 GB", | 20260505 07:33:37.102 | INFO | "malloc" : " 14 GB", | 20260505 07:33:37.102 | INFO | "pinned" : " 29 GB" | 20260505 07:33:37.102 | INFO | } | 20260505 07:33:37.102 | INFO | Perf report for the last 5.13935 seconds: | 20260505 07:33:37.102 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:33:37.102 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:37.102 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:37.102 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:37.102 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 07:33:37.102 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:37.102 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:37.102 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:33:37.102 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:37.102 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:37.102 | INFO | Uploaded 36GiB to GPU in 2.15192s (16.7292GiB/s) | 20260505 07:33:37.103 | INFO | Downloaded 34GiB to GPU in 2.05844s (16.5173GiB/s) | 20260505 07:33:42.242 | INFO | Analyzing chunk frames = [1447424,1447936) | 20260505 07:33:42.242 | INFO | MemInfo: { | 20260505 07:33:42.242 | INFO | "gpu" : " 17 GB", | 20260505 07:33:42.242 | INFO | "huge" : "", | 20260505 07:33:42.242 | INFO | "huge_shared" : " 27 GB", | 20260505 07:33:42.242 | INFO | "malloc" : " 14 GB", | 20260505 07:33:42.242 | INFO | "pinned" : " 29 GB" | 20260505 07:33:42.242 | INFO | } | 20260505 07:33:42.242 | INFO | Perf report for the last 5.13942 seconds: | 20260505 07:33:42.242 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:33:42.242 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:42.242 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:42.242 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:42.242 | INFO | ANALYSIS : 63.3%, 3.0, 98.3ms, 0 | 20260505 07:33:42.242 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:42.242 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:42.242 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:33:42.242 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:42.242 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:42.242 | INFO | Uploaded 36GiB to GPU in 2.13649s (16.8501GiB/s) | 20260505 07:33:42.242 | INFO | Downloaded 34GiB to GPU in 2.04061s (16.6617GiB/s) | 20260505 07:33:47.381 | INFO | Analyzing chunk frames = [1447936,1448448) | 20260505 07:33:47.381 | INFO | MemInfo: { | 20260505 07:33:47.381 | INFO | "gpu" : " 17 GB", | 20260505 07:33:47.381 | INFO | "huge" : "", | 20260505 07:33:47.381 | INFO | "huge_shared" : " 27 GB", | 20260505 07:33:47.381 | INFO | "malloc" : " 14 GB", | 20260505 07:33:47.381 | INFO | "pinned" : " 29 GB" | 20260505 07:33:47.381 | INFO | } | 20260505 07:33:47.381 | INFO | Perf report for the last 5.1397 seconds: | 20260505 07:33:47.381 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:33:47.381 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:47.381 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:47.381 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:47.381 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 07:33:47.381 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:47.381 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:47.381 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 07:33:47.382 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:47.382 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:47.382 | INFO | Uploaded 36GiB to GPU in 2.1447s (16.7856GiB/s) | 20260505 07:33:47.382 | INFO | Downloaded 34GiB to GPU in 2.04635s (16.615GiB/s) | 20260505 07:33:52.522 | INFO | Analyzing chunk frames = [1448448,1448960) | 20260505 07:33:52.522 | INFO | MemInfo: { | 20260505 07:33:52.522 | INFO | "gpu" : " 17 GB", | 20260505 07:33:52.522 | INFO | "huge" : "", | 20260505 07:33:52.522 | INFO | "huge_shared" : " 27 GB", | 20260505 07:33:52.522 | INFO | "malloc" : " 14 GB", | 20260505 07:33:52.522 | INFO | "pinned" : " 29 GB" | 20260505 07:33:52.522 | INFO | } | 20260505 07:33:52.522 | INFO | Perf report for the last 5.1409 seconds: | 20260505 07:33:52.522 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:33:52.522 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:52.522 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:52.522 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:52.522 | INFO | ANALYSIS : 63.5%, 3.0, 99.3ms, 0 | 20260505 07:33:52.522 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:52.522 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:52.522 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:33:52.522 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:52.522 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:52.523 | INFO | Uploaded 36GiB to GPU in 2.16277s (16.6453GiB/s) | 20260505 07:33:52.523 | INFO | Downloaded 34GiB to GPU in 2.05802s (16.5208GiB/s) | 20260505 07:33:57.660 | INFO | Analyzing chunk frames = [1448960,1449472) | 20260505 07:33:57.660 | INFO | MemInfo: { | 20260505 07:33:57.660 | INFO | "gpu" : " 17 GB", | 20260505 07:33:57.660 | INFO | "huge" : "", | 20260505 07:33:57.660 | INFO | "huge_shared" : " 27 GB", | 20260505 07:33:57.660 | INFO | "malloc" : " 14 GB", | 20260505 07:33:57.660 | INFO | "pinned" : " 29 GB" | 20260505 07:33:57.660 | INFO | } | 20260505 07:33:57.660 | INFO | Perf report for the last 5.13762 seconds: | 20260505 07:33:57.660 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:33:57.660 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:57.660 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:57.660 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:57.660 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 07:33:57.660 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:57.660 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:57.660 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260505 07:33:57.660 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:57.660 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:33:57.660 | INFO | Uploaded 36GiB to GPU in 2.19303s (16.4156GiB/s) | 20260505 07:33:57.660 | INFO | Downloaded 34GiB to GPU in 2.16465s (15.7069GiB/s) | 20260505 07:34:02.799 | INFO | Analyzing chunk frames = [1449472,1449984) | 20260505 07:34:02.799 | INFO | MemInfo: { | 20260505 07:34:02.799 | INFO | "gpu" : " 17 GB", | 20260505 07:34:02.799 | INFO | "huge" : "", | 20260505 07:34:02.799 | INFO | "huge_shared" : " 27 GB", | 20260505 07:34:02.799 | INFO | "malloc" : " 14 GB", | 20260505 07:34:02.799 | INFO | "pinned" : " 29 GB" | 20260505 07:34:02.799 | INFO | } | 20260505 07:34:02.799 | INFO | Perf report for the last 5.13898 seconds: | 20260505 07:34:02.799 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:34:02.799 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:02.799 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:02.799 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:02.799 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 07:34:02.799 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:02.799 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:02.799 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 07:34:02.799 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:02.799 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:02.799 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 07:34:02.799 | INFO | Total mem usage: 15.3 GiB | 20260505 07:34:02.799 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 07:34:02.799 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.31%) | 20260505 07:34:02.799 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 07:34:02.799 | INFO | High Water Breakdown: | 20260505 07:34:02.799 | INFO | 60.6% : 8.74 GiB ( 93.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 07:34:02.799 | INFO | 39.2% : 5.66 GiB ( 85.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 07:34:02.799 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 07:34:02.799 | INFO | ------------------------------------------ | 20260505 07:34:02.799 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:34:02.799 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 07:34:02.799 | INFO | Total mem usage: 26.5 GiB | 20260505 07:34:02.799 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 07:34:02.799 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 07:34:02.799 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 07:34:02.800 | INFO | High Water Breakdown: | 20260505 07:34:02.800 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:34:02.800 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:34:02.800 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:34:02.800 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:34:02.800 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:34:02.800 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:34:02.800 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 07:34:02.800 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 07:34:02.800 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:34:02.800 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 07:34:02.800 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 07:34:02.800 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 07:34:02.800 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 07:34:02.800 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 07:34:02.800 | INFO | ------------------------------------------ | 20260505 07:34:02.800 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:34:02.800 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 07:34:02.800 | INFO | Total mem usage: 25.6 GiB | 20260505 07:34:02.800 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 07:34:02.800 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 07:34:02.800 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 07:34:02.800 | INFO | High Water Breakdown: | 20260505 07:34:02.800 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 07:34:02.800 | INFO | ------------------------------------------ | 20260505 07:34:02.800 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:34:02.800 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 07:34:02.800 | INFO | Total mem usage: 15.7 GiB | 20260505 07:34:02.800 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 07:34:02.800 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 07:34:02.800 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 07:34:02.800 | INFO | High Water Breakdown: | 20260505 07:34:02.800 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 07:34:02.800 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 07:34:02.800 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 07:34:02.800 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 07:34:02.800 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 07:34:02.800 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 07:34:02.800 | INFO | Uploaded 36GiB to GPU in 2.15302s (16.7207GiB/s) | 20260505 07:34:02.800 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 07:34:02.800 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 07:34:02.800 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 07:34:02.800 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:34:02.800 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 07:34:02.800 | INFO | Downloaded 34GiB to GPU in 2.03486s (16.7088GiB/s) | 20260505 07:34:02.800 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:34:02.800 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 07:34:02.800 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 07:34:02.800 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 07:34:02.800 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 07:34:02.800 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:34:02.800 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 07:34:02.800 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:34:02.800 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:34:02.800 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:34:02.800 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 07:34:02.800 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 07:34:02.800 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:34:02.800 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 07:34:02.800 | INFO | ------------------------------------------ | 20260505 07:34:02.800 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:34:02.800 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 07:34:02.800 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:34:02.800 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:34:02.800 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.52e+01, 1.35e+01 / 1.91e+00 | 20260505 07:34:02.800 | INFO | PerfReport: 0.29%, AggregateStats -- 9.37e-02 / 1.04e-01, 1.11e-02 / 9.83e-03 | 20260505 07:34:02.800 | INFO | PerfReport: 11.61%, Baseline -- 3.81e+00 / 4.42e+00, 6.18e-01 / 3.22e-01 | 20260505 07:34:02.800 | INFO | PerfReport: 3.30%, Histogram -- 1.09e+00 / 1.44e+00, 2.10e-01 / 1.31e-01 | 20260505 07:34:02.800 | INFO | PerfReport: 15.46%, DME -- 4.06e+01 / 4.44e+01, 4.00e+00 / 1.20e+00 | 20260505 07:34:02.800 | INFO | PerfReport: 56.31%, FrameLabeling -- 1.85e+01 / 1.94e+01, 2.02e+00 / 1.64e+00 | 20260505 07:34:02.800 | INFO | PerfReport: 7.62%, PulseAccumulating -- 2.50e+00 / 3.16e+00, 4.44e-01 / 2.05e-01 | 20260505 07:34:02.800 | INFO | PerfReport: 3.98%, Metrics -- 1.31e+00 / 1.72e+00, 3.62e-01 / 1.26e-01 | 20260505 07:34:02.800 | INFO | PerfReport: 1.43%, PulseSerializing -- 4.71e-01 / 5.47e-01, 9.00e-02 / 2.59e-02 | 20260505 07:34:02.800 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 07:34:02.800 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:34:02.800 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:34:02.800 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.24e+01 / 0.00e+00 | 20260505 07:34:02.800 | INFO | PerfReport: 22.30%, Upload -- 2.26e+01 / 0.00e+00, 7.97e+00 / 0.00e+00 | 20260505 07:34:02.800 | INFO | PerfReport: 21.56%, Download -- 2.19e+01 / 0.00e+00, 7.85e+00 / 0.00e+00 | 20260505 07:34:02.800 | INFO | PerfReport: 32.40%, Compute -- 3.29e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 07:34:02.800 | INFO | PerfReport: 1.53%, LockUpload -- 1.56e+00 / 0.00e+00, 7.13e+00 / 0.00e+00 | 20260505 07:34:02.800 | INFO | PerfReport: 0.87%, LockDownload -- 8.86e-01 / 0.00e+00, 2.29e+00 / 0.00e+00 | 20260505 07:34:02.800 | INFO | PerfReport: 21.31%, LockCompute -- 2.16e+01 / 0.00e+00, 1.66e+01 / 0.00e+00 | 20260505 07:34:02.800 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 1.98e-02 / 0.00e+00 | 20260505 07:34:02.933 | INFO | Starting a round of Baz IO | 20260505 07:34:06.208 | INFO | Populating RTMetrics report | 20260505 07:34:06.208 | INFO | RTMetrics report populated | 20260505 07:34:06.208 | INFO | Opening tmp RTMetrics json file | 20260505 07:34:06.218 | INFO | Serializing RTMetrics to json | 20260505 07:34:06.239 | INFO | Serialization done, swapping tmp file now... | 20260505 07:34:06.241 | INFO | RTMetrics done | 20260505 07:34:06.340 | INFO | Finished a round of Baz IO: Wrote 2.437 GiB in 3.407 seconds | 20260505 07:34:06.340 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 29.57% / 0.03% / 70.40% | 20260505 07:34:06.340 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.50065% / 2.78155e-05% / 5.96046e-05% | 20260505 07:34:06.340 | INFO | Baz IO overall: | 20260505 07:34:06.340 | INFO | BytesWritten : 794099488438 | 20260505 07:34:06.340 | INFO | NumEvents : 376861634179 | 20260505 07:34:07.938 | INFO | Analyzing chunk frames = [1449984,1450496) | 20260505 07:34:07.938 | INFO | MemInfo: { | 20260505 07:34:07.938 | INFO | "gpu" : " 17 GB", | 20260505 07:34:07.938 | INFO | "huge" : "", | 20260505 07:34:07.938 | INFO | "huge_shared" : " 27 GB", | 20260505 07:34:07.938 | INFO | "malloc" : " 14 GB", | 20260505 07:34:07.938 | INFO | "pinned" : " 29 GB" | 20260505 07:34:07.938 | INFO | } | 20260505 07:34:07.938 | INFO | Perf report for the last 5.13948 seconds: | 20260505 07:34:07.938 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:34:07.938 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:07.938 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:07.938 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:07.938 | INFO | ANALYSIS : 65.2%, 2.9, 99.5ms, 0 | 20260505 07:34:07.938 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:07.938 | INFO | RT_METRICS : 46.6%, 1.0, 24.4ms, 0 | 20260505 07:34:07.939 | INFO | PRE_HQ : 63.8%, 1.9, 62.5ms, 0 | 20260505 07:34:07.939 | INFO | BAZWRITER : 65.6%, 4.2, 139.9ms, 0 | 20260505 07:34:07.939 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 07:34:07.939 | INFO | Uploaded 36GiB to GPU in 2.23479s (16.1089GiB/s) | 20260505 07:34:07.939 | INFO | Downloaded 38GiB to GPU in 2.30766s (16.4669GiB/s) | 20260505 07:34:13.078 | INFO | Analyzing chunk frames = [1450496,1451008) | 20260505 07:34:13.078 | INFO | MemInfo: { | 20260505 07:34:13.078 | INFO | "gpu" : " 17 GB", | 20260505 07:34:13.078 | INFO | "huge" : "", | 20260505 07:34:13.078 | INFO | "huge_shared" : " 27 GB", | 20260505 07:34:13.078 | INFO | "malloc" : " 14 GB", | 20260505 07:34:13.078 | INFO | "pinned" : " 29 GB" | 20260505 07:34:13.078 | INFO | } | 20260505 07:34:13.078 | INFO | Perf report for the last 5.13986 seconds: | 20260505 07:34:13.078 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:34:13.078 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:13.078 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:13.078 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:13.078 | INFO | ANALYSIS : 64.4%, 3.0, 100.6ms, 0 | 20260505 07:34:13.078 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:13.078 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:13.078 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 07:34:13.078 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:13.078 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:13.079 | INFO | Uploaded 36GiB to GPU in 2.18177s (16.5004GiB/s) | 20260505 07:34:13.079 | INFO | Downloaded 34GiB to GPU in 2.058s (16.5209GiB/s) | 20260505 07:34:18.217 | INFO | Analyzing chunk frames = [1451008,1451520) | 20260505 07:34:18.217 | INFO | MemInfo: { | 20260505 07:34:18.217 | INFO | "gpu" : " 17 GB", | 20260505 07:34:18.217 | INFO | "huge" : "", | 20260505 07:34:18.217 | INFO | "huge_shared" : " 27 GB", | 20260505 07:34:18.217 | INFO | "malloc" : " 14 GB", | 20260505 07:34:18.217 | INFO | "pinned" : " 29 GB" | 20260505 07:34:18.217 | INFO | } | 20260505 07:34:18.217 | INFO | Perf report for the last 5.13899 seconds: | 20260505 07:34:18.217 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:34:18.217 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:18.217 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:18.217 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:18.217 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260505 07:34:18.217 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:18.217 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:18.217 | INFO | PRE_HQ : 30.0%, 1.0, 15.5ms, 0 | 20260505 07:34:18.217 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:18.218 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:18.218 | INFO | Uploaded 36GiB to GPU in 2.15394s (16.7136GiB/s) | 20260505 07:34:18.218 | INFO | Downloaded 34GiB to GPU in 2.06081s (16.4983GiB/s) | 20260505 07:34:23.357 | INFO | Analyzing chunk frames = [1451520,1452032) | 20260505 07:34:23.357 | INFO | MemInfo: { | 20260505 07:34:23.357 | INFO | "gpu" : " 17 GB", | 20260505 07:34:23.357 | INFO | "huge" : "", | 20260505 07:34:23.357 | INFO | "huge_shared" : " 27 GB", | 20260505 07:34:23.357 | INFO | "malloc" : " 14 GB", | 20260505 07:34:23.357 | INFO | "pinned" : " 29 GB" | 20260505 07:34:23.357 | INFO | } | 20260505 07:34:23.357 | INFO | Perf report for the last 5.13949 seconds: | 20260505 07:34:23.357 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:34:23.357 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:23.357 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:23.357 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:23.357 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 07:34:23.357 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:23.357 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:23.357 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:34:23.357 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:23.357 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:23.357 | INFO | Uploaded 36GiB to GPU in 2.14586s (16.7765GiB/s) | 20260505 07:34:23.357 | INFO | Downloaded 34GiB to GPU in 2.04133s (16.6558GiB/s) | 20260505 07:34:28.496 | INFO | Analyzing chunk frames = [1452032,1452544) | 20260505 07:34:28.496 | INFO | MemInfo: { | 20260505 07:34:28.496 | INFO | "gpu" : " 17 GB", | 20260505 07:34:28.496 | INFO | "huge" : "", | 20260505 07:34:28.496 | INFO | "huge_shared" : " 27 GB", | 20260505 07:34:28.496 | INFO | "malloc" : " 14 GB", | 20260505 07:34:28.496 | INFO | "pinned" : " 29 GB" | 20260505 07:34:28.496 | INFO | } | 20260505 07:34:28.496 | INFO | Perf report for the last 5.13908 seconds: | 20260505 07:34:28.496 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:34:28.496 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:28.496 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:28.496 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:28.496 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 07:34:28.496 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:28.496 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:28.496 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 07:34:28.496 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:28.496 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:28.496 | INFO | Uploaded 36GiB to GPU in 2.14379s (16.7927GiB/s) | 20260505 07:34:28.496 | INFO | Downloaded 34GiB to GPU in 2.05139s (16.5741GiB/s) | 20260505 07:34:33.637 | INFO | Analyzing chunk frames = [1452544,1453056) | 20260505 07:34:33.637 | INFO | MemInfo: { | 20260505 07:34:33.637 | INFO | "gpu" : " 17 GB", | 20260505 07:34:33.637 | INFO | "huge" : "", | 20260505 07:34:33.637 | INFO | "huge_shared" : " 27 GB", | 20260505 07:34:33.637 | INFO | "malloc" : " 14 GB", | 20260505 07:34:33.637 | INFO | "pinned" : " 29 GB" | 20260505 07:34:33.637 | INFO | } | 20260505 07:34:33.637 | INFO | Perf report for the last 5.14095 seconds: | 20260505 07:34:33.637 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:34:33.637 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:33.637 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:33.637 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:33.637 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 07:34:33.637 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:33.637 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:33.637 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 07:34:33.637 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:33.637 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:33.637 | INFO | Uploaded 36GiB to GPU in 2.15461s (16.7084GiB/s) | 20260505 07:34:33.637 | INFO | Downloaded 34GiB to GPU in 2.05523s (16.5431GiB/s) | 20260505 07:34:38.774 | INFO | Analyzing chunk frames = [1453056,1453568) | 20260505 07:34:38.774 | INFO | MemInfo: { | 20260505 07:34:38.774 | INFO | "gpu" : " 17 GB", | 20260505 07:34:38.774 | INFO | "huge" : "", | 20260505 07:34:38.774 | INFO | "huge_shared" : " 27 GB", | 20260505 07:34:38.774 | INFO | "malloc" : " 14 GB", | 20260505 07:34:38.774 | INFO | "pinned" : " 29 GB" | 20260505 07:34:38.774 | INFO | } | 20260505 07:34:38.774 | INFO | Perf report for the last 5.13758 seconds: | 20260505 07:34:38.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:34:38.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:38.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:38.775 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:38.775 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 07:34:38.775 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:38.775 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:38.775 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260505 07:34:38.775 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:38.775 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:38.775 | INFO | Uploaded 36GiB to GPU in 2.21209s (16.2742GiB/s) | 20260505 07:34:38.775 | INFO | Downloaded 34GiB to GPU in 2.17036s (15.6656GiB/s) | 20260505 07:34:43.914 | INFO | Analyzing chunk frames = [1453568,1454080) | 20260505 07:34:43.914 | INFO | MemInfo: { | 20260505 07:34:43.914 | INFO | "gpu" : " 17 GB", | 20260505 07:34:43.914 | INFO | "huge" : "", | 20260505 07:34:43.914 | INFO | "huge_shared" : " 27 GB", | 20260505 07:34:43.914 | INFO | "malloc" : " 14 GB", | 20260505 07:34:43.914 | INFO | "pinned" : " 29 GB" | 20260505 07:34:43.914 | INFO | } | 20260505 07:34:43.914 | INFO | Perf report for the last 5.13968 seconds: | 20260505 07:34:43.914 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:34:43.914 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:43.914 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:43.914 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:43.914 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 07:34:43.914 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:43.914 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:43.914 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:34:43.914 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:43.914 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:43.914 | INFO | Uploaded 36GiB to GPU in 2.17817s (16.5276GiB/s) | 20260505 07:34:43.914 | INFO | Downloaded 34GiB to GPU in 2.05106s (16.5768GiB/s) | 20260505 07:34:44.049 | INFO | Starting a round of Baz IO | 20260505 07:34:47.319 | INFO | Populating RTMetrics report | 20260505 07:34:47.319 | INFO | RTMetrics report populated | 20260505 07:34:47.319 | INFO | Opening tmp RTMetrics json file | 20260505 07:34:47.331 | INFO | Serializing RTMetrics to json | 20260505 07:34:47.376 | INFO | Serialization done, swapping tmp file now... | 20260505 07:34:47.378 | INFO | RTMetrics done | 20260505 07:34:47.455 | INFO | Finished a round of Baz IO: Wrote 2.435 GiB in 3.406 seconds | 20260505 07:34:47.455 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 29.54% / 0.03% / 70.43% | 20260505 07:34:47.455 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.48188% / 1.58946e-05% / 7.54992e-05% | 20260505 07:34:47.456 | INFO | Baz IO overall: | 20260505 07:34:47.456 | INFO | BytesWritten : 796713898430 | 20260505 07:34:47.456 | INFO | NumEvents : 378112971564 | 20260505 07:34:49.053 | INFO | Analyzing chunk frames = [1454080,1454592) | 20260505 07:34:49.053 | INFO | MemInfo: { | 20260505 07:34:49.053 | INFO | "gpu" : " 17 GB", | 20260505 07:34:49.053 | INFO | "huge" : "", | 20260505 07:34:49.053 | INFO | "huge_shared" : " 27 GB", | 20260505 07:34:49.053 | INFO | "malloc" : " 14 GB", | 20260505 07:34:49.053 | INFO | "pinned" : " 29 GB" | 20260505 07:34:49.053 | INFO | } | 20260505 07:34:49.053 | INFO | Perf report for the last 5.13909 seconds: | 20260505 07:34:49.053 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:34:49.053 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:49.053 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:49.053 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:49.053 | INFO | ANALYSIS : 65.1%, 2.9, 99.5ms, 0 | 20260505 07:34:49.053 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:49.053 | INFO | RT_METRICS : 47.1%, 1.0, 24.7ms, 0 | 20260505 07:34:49.053 | INFO | PRE_HQ : 63.7%, 1.9, 62.2ms, 0 | 20260505 07:34:49.053 | INFO | BAZWRITER : 65.6%, 4.1, 139.6ms, 0 | 20260505 07:34:49.053 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 07:34:49.053 | INFO | Uploaded 36GiB to GPU in 2.21177s (16.2766GiB/s) | 20260505 07:34:49.054 | INFO | Downloaded 38GiB to GPU in 2.31565s (16.4101GiB/s) | 20260505 07:34:54.193 | INFO | Analyzing chunk frames = [1454592,1455104) | 20260505 07:34:54.193 | INFO | MemInfo: { | 20260505 07:34:54.193 | INFO | "gpu" : " 17 GB", | 20260505 07:34:54.193 | INFO | "huge" : "", | 20260505 07:34:54.193 | INFO | "huge_shared" : " 27 GB", | 20260505 07:34:54.193 | INFO | "malloc" : " 14 GB", | 20260505 07:34:54.193 | INFO | "pinned" : " 29 GB" | 20260505 07:34:54.193 | INFO | } | 20260505 07:34:54.193 | INFO | Perf report for the last 5.1395 seconds: | 20260505 07:34:54.193 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:34:54.193 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:54.193 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:54.193 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:54.193 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 07:34:54.193 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:54.193 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:54.193 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:34:54.193 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:54.193 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:54.193 | INFO | Uploaded 36GiB to GPU in 2.18386s (16.4846GiB/s) | 20260505 07:34:54.193 | INFO | Downloaded 34GiB to GPU in 2.05827s (16.5187GiB/s) | 20260505 07:34:59.332 | INFO | Analyzing chunk frames = [1455104,1455616) | 20260505 07:34:59.332 | INFO | MemInfo: { | 20260505 07:34:59.332 | INFO | "gpu" : " 17 GB", | 20260505 07:34:59.332 | INFO | "huge" : "", | 20260505 07:34:59.332 | INFO | "huge_shared" : " 27 GB", | 20260505 07:34:59.332 | INFO | "malloc" : " 14 GB", | 20260505 07:34:59.332 | INFO | "pinned" : " 29 GB" | 20260505 07:34:59.332 | INFO | } | 20260505 07:34:59.332 | INFO | Perf report for the last 5.13945 seconds: | 20260505 07:34:59.333 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:34:59.333 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:59.333 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:59.333 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:59.333 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 07:34:59.333 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:59.333 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:59.333 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:34:59.333 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:59.333 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:34:59.334 | INFO | Uploaded 36GiB to GPU in 2.14869s (16.7544GiB/s) | 20260505 07:34:59.334 | INFO | Downloaded 34GiB to GPU in 2.05477s (16.5468GiB/s) | 20260505 07:35:04.471 | INFO | Analyzing chunk frames = [1455616,1456128) | 20260505 07:35:04.471 | INFO | MemInfo: { | 20260505 07:35:04.471 | INFO | "gpu" : " 17 GB", | 20260505 07:35:04.471 | INFO | "huge" : "", | 20260505 07:35:04.471 | INFO | "huge_shared" : " 27 GB", | 20260505 07:35:04.471 | INFO | "malloc" : " 14 GB", | 20260505 07:35:04.471 | INFO | "pinned" : " 29 GB" | 20260505 07:35:04.471 | INFO | } | 20260505 07:35:04.471 | INFO | Perf report for the last 5.139 seconds: | 20260505 07:35:04.471 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:35:04.471 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:04.471 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:04.471 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:04.471 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 07:35:04.471 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:04.471 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:04.471 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 07:35:04.471 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:04.471 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:04.472 | INFO | Uploaded 36GiB to GPU in 2.12675s (16.9272GiB/s) | 20260505 07:35:04.472 | INFO | Downloaded 34GiB to GPU in 2.03578s (16.7012GiB/s) | 20260505 07:35:09.610 | INFO | Analyzing chunk frames = [1456128,1456640) | 20260505 07:35:09.610 | INFO | MemInfo: { | 20260505 07:35:09.610 | INFO | "gpu" : " 17 GB", | 20260505 07:35:09.610 | INFO | "huge" : "", | 20260505 07:35:09.610 | INFO | "huge_shared" : " 27 GB", | 20260505 07:35:09.610 | INFO | "malloc" : " 14 GB", | 20260505 07:35:09.611 | INFO | "pinned" : " 29 GB" | 20260505 07:35:09.611 | INFO | } | 20260505 07:35:09.611 | INFO | Perf report for the last 5.13937 seconds: | 20260505 07:35:09.611 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:35:09.611 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:09.611 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:09.611 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:09.611 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 07:35:09.611 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:09.611 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:09.611 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:35:09.611 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:09.611 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:09.611 | INFO | Uploaded 36GiB to GPU in 2.16848s (16.6015GiB/s) | 20260505 07:35:09.611 | INFO | Downloaded 34GiB to GPU in 2.05335s (16.5583GiB/s) | 20260505 07:35:14.752 | INFO | Analyzing chunk frames = [1456640,1457152) | 20260505 07:35:14.752 | INFO | MemInfo: { | 20260505 07:35:14.752 | INFO | "gpu" : " 17 GB", | 20260505 07:35:14.752 | INFO | "huge" : "", | 20260505 07:35:14.752 | INFO | "huge_shared" : " 27 GB", | 20260505 07:35:14.752 | INFO | "malloc" : " 14 GB", | 20260505 07:35:14.752 | INFO | "pinned" : " 29 GB" | 20260505 07:35:14.752 | INFO | } | 20260505 07:35:14.752 | INFO | Perf report for the last 5.14098 seconds: | 20260505 07:35:14.752 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:35:14.752 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:14.752 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:14.752 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:14.752 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 07:35:14.752 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:14.752 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:14.752 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 07:35:14.752 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:14.752 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:14.752 | INFO | Uploaded 36GiB to GPU in 2.16209s (16.6506GiB/s) | 20260505 07:35:14.752 | INFO | Downloaded 34GiB to GPU in 2.05473s (16.5472GiB/s) | 20260505 07:35:19.889 | INFO | Analyzing chunk frames = [1457152,1457664) | 20260505 07:35:19.889 | INFO | MemInfo: { | 20260505 07:35:19.889 | INFO | "gpu" : " 17 GB", | 20260505 07:35:19.889 | INFO | "huge" : "", | 20260505 07:35:19.889 | INFO | "huge_shared" : " 27 GB", | 20260505 07:35:19.889 | INFO | "malloc" : " 14 GB", | 20260505 07:35:19.889 | INFO | "pinned" : " 29 GB" | 20260505 07:35:19.889 | INFO | } | 20260505 07:35:19.889 | INFO | Perf report for the last 5.13759 seconds: | 20260505 07:35:19.889 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:35:19.889 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:19.889 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:19.889 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:19.889 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 07:35:19.889 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:19.889 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:19.889 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 07:35:19.889 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:19.889 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:19.889 | INFO | Uploaded 36GiB to GPU in 2.20152s (16.3524GiB/s) | 20260505 07:35:19.889 | INFO | Downloaded 34GiB to GPU in 2.16659s (15.6928GiB/s) | 20260505 07:35:25.029 | INFO | Analyzing chunk frames = [1457664,1458176) | 20260505 07:35:25.029 | INFO | MemInfo: { | 20260505 07:35:25.029 | INFO | "gpu" : " 17 GB", | 20260505 07:35:25.029 | INFO | "huge" : "", | 20260505 07:35:25.029 | INFO | "huge_shared" : " 27 GB", | 20260505 07:35:25.029 | INFO | "malloc" : " 14 GB", | 20260505 07:35:25.029 | INFO | "pinned" : " 29 GB" | 20260505 07:35:25.029 | INFO | } | 20260505 07:35:25.029 | INFO | Perf report for the last 5.13956 seconds: | 20260505 07:35:25.029 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:35:25.029 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:25.029 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:25.029 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:25.029 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 07:35:25.029 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:25.029 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:25.029 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 07:35:25.029 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:25.029 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:25.029 | INFO | Uploaded 36GiB to GPU in 2.16468s (16.6306GiB/s) | 20260505 07:35:25.029 | INFO | Downloaded 34GiB to GPU in 2.04474s (16.628GiB/s) | 20260505 07:35:25.029 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 07:35:25.029 | INFO | Total mem usage: 15.3 GiB | 20260505 07:35:25.029 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 07:35:25.029 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.16%) | 20260505 07:35:25.029 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 07:35:25.029 | INFO | High Water Breakdown: | 20260505 07:35:25.029 | INFO | 60.7% : 8.76 GiB ( 93.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 07:35:25.029 | INFO | 39.2% : 5.66 GiB ( 85.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 07:35:25.029 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 07:35:25.029 | INFO | ------------------------------------------ | 20260505 07:35:25.029 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:35:25.029 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 07:35:25.029 | INFO | Total mem usage: 26.5 GiB | 20260505 07:35:25.029 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 07:35:25.029 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 07:35:25.029 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 07:35:25.029 | INFO | High Water Breakdown: | 20260505 07:35:25.029 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:35:25.029 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:35:25.029 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:35:25.029 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:35:25.029 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:35:25.029 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:35:25.029 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 07:35:25.029 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 07:35:25.029 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:35:25.029 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 07:35:25.029 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 07:35:25.029 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 07:35:25.029 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 07:35:25.029 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 07:35:25.029 | INFO | ------------------------------------------ | 20260505 07:35:25.029 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:35:25.029 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 07:35:25.029 | INFO | Total mem usage: 25.6 GiB | 20260505 07:35:25.029 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 07:35:25.029 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 07:35:25.029 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 07:35:25.029 | INFO | High Water Breakdown: | 20260505 07:35:25.029 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 07:35:25.029 | INFO | ------------------------------------------ | 20260505 07:35:25.030 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:35:25.030 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 07:35:25.030 | INFO | Total mem usage: 15.7 GiB | 20260505 07:35:25.030 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 07:35:25.030 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 07:35:25.030 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 07:35:25.030 | INFO | High Water Breakdown: | 20260505 07:35:25.030 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 07:35:25.030 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 07:35:25.030 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 07:35:25.030 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 07:35:25.030 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 07:35:25.030 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 07:35:25.030 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 07:35:25.030 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 07:35:25.030 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 07:35:25.030 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:35:25.030 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 07:35:25.030 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:35:25.030 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 07:35:25.030 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 07:35:25.030 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 07:35:25.030 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 07:35:25.030 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:35:25.030 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 07:35:25.030 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:35:25.030 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:35:25.030 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:35:25.030 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 07:35:25.030 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 07:35:25.030 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:35:25.030 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 07:35:25.030 | INFO | ------------------------------------------ | 20260505 07:35:25.030 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:35:25.030 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 07:35:25.030 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:35:25.030 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:35:25.030 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.49e+01, 1.35e+01 / 1.94e+00 | 20260505 07:35:25.030 | INFO | PerfReport: 0.29%, AggregateStats -- 9.38e-02 / 1.02e-01, 1.10e-02 / 6.52e-03 | 20260505 07:35:25.030 | INFO | PerfReport: 11.60%, Baseline -- 3.81e+00 / 4.38e+00, 6.21e-01 / 3.33e-01 | 20260505 07:35:25.030 | INFO | PerfReport: 3.30%, Histogram -- 1.08e+00 / 1.44e+00, 2.09e-01 / 1.30e-01 | 20260505 07:35:25.030 | INFO | PerfReport: 15.46%, DME -- 4.06e+01 / 4.42e+01, 4.10e+00 / 1.32e+00 | 20260505 07:35:25.030 | INFO | PerfReport: 56.32%, FrameLabeling -- 1.85e+01 / 1.95e+01, 2.03e+00 / 1.66e+00 | 20260505 07:35:25.030 | INFO | PerfReport: 7.62%, PulseAccumulating -- 2.50e+00 / 3.09e+00, 4.47e-01 / 3.03e-01 | 20260505 07:35:25.030 | INFO | PerfReport: 3.97%, Metrics -- 1.30e+00 / 1.69e+00, 3.61e-01 / 1.42e-01 | 20260505 07:35:25.030 | INFO | PerfReport: 1.43%, PulseSerializing -- 4.71e-01 / 5.46e-01, 9.03e-02 / 2.94e-02 | 20260505 07:35:25.030 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 07:35:25.030 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:35:25.030 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:35:25.030 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.24e+01 / 0.00e+00 | 20260505 07:35:25.030 | INFO | PerfReport: 22.32%, Upload -- 2.26e+01 / 0.00e+00, 8.05e+00 / 0.00e+00 | 20260505 07:35:25.030 | INFO | PerfReport: 21.55%, Download -- 2.19e+01 / 0.00e+00, 7.94e+00 / 0.00e+00 | 20260505 07:35:25.030 | INFO | PerfReport: 32.36%, Compute -- 3.28e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 07:35:25.030 | INFO | PerfReport: 1.59%, LockUpload -- 1.61e+00 / 0.00e+00, 7.19e+00 / 0.00e+00 | 20260505 07:35:25.030 | INFO | PerfReport: 0.85%, LockDownload -- 8.63e-01 / 0.00e+00, 2.24e+00 / 0.00e+00 | 20260505 07:35:25.030 | INFO | PerfReport: 21.32%, LockCompute -- 2.16e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260505 07:35:25.030 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 1.82e-02 / 0.00e+00 | 20260505 07:35:25.165 | INFO | Starting a round of Baz IO | 20260505 07:35:28.451 | INFO | Populating RTMetrics report | 20260505 07:35:28.452 | INFO | RTMetrics report populated | 20260505 07:35:28.452 | INFO | Opening tmp RTMetrics json file | 20260505 07:35:28.453 | INFO | Serializing RTMetrics to json | 20260505 07:35:28.456 | INFO | Serialization done, swapping tmp file now... | 20260505 07:35:28.456 | INFO | RTMetrics done | 20260505 07:35:28.587 | INFO | Finished a round of Baz IO: Wrote 2.433 GiB in 3.422 seconds | 20260505 07:35:28.587 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 29.51% / 0.03% / 70.46% | 20260505 07:35:28.587 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.45996% / 5.16574e-05% / 5.96046e-05% | 20260505 07:35:28.587 | INFO | Baz IO overall: | 20260505 07:35:28.587 | INFO | BytesWritten : 799325902776 | 20260505 07:35:28.587 | INFO | NumEvents : 379362575375 | 20260505 07:35:30.168 | INFO | Analyzing chunk frames = [1458176,1458688) | 20260505 07:35:30.168 | INFO | MemInfo: { | 20260505 07:35:30.168 | INFO | "gpu" : " 17 GB", | 20260505 07:35:30.168 | INFO | "huge" : "", | 20260505 07:35:30.168 | INFO | "huge_shared" : " 27 GB", | 20260505 07:35:30.168 | INFO | "malloc" : " 14 GB", | 20260505 07:35:30.168 | INFO | "pinned" : " 29 GB" | 20260505 07:35:30.168 | INFO | } | 20260505 07:35:30.168 | INFO | Perf report for the last 5.13921 seconds: | 20260505 07:35:30.168 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:35:30.168 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:30.168 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:30.168 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:30.168 | INFO | ANALYSIS : 65.4%, 2.9, 99.1ms, 0 | 20260505 07:35:30.168 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:30.168 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260505 07:35:30.168 | INFO | PRE_HQ : 64.1%, 1.9, 62.2ms, 0 | 20260505 07:35:30.168 | INFO | BAZWRITER : 65.9%, 4.2, 141.7ms, 0 | 20260505 07:35:30.168 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 07:35:30.168 | INFO | Uploaded 36GiB to GPU in 2.24148s (16.0608GiB/s) | 20260505 07:35:30.168 | INFO | Downloaded 38GiB to GPU in 2.35245s (16.1534GiB/s) | 20260505 07:35:35.307 | INFO | Analyzing chunk frames = [1458688,1459200) | 20260505 07:35:35.307 | INFO | MemInfo: { | 20260505 07:35:35.307 | INFO | "gpu" : " 17 GB", | 20260505 07:35:35.307 | INFO | "huge" : "", | 20260505 07:35:35.307 | INFO | "huge_shared" : " 27 GB", | 20260505 07:35:35.307 | INFO | "malloc" : " 14 GB", | 20260505 07:35:35.307 | INFO | "pinned" : " 29 GB" | 20260505 07:35:35.307 | INFO | } | 20260505 07:35:35.307 | INFO | Perf report for the last 5.13933 seconds: | 20260505 07:35:35.307 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:35:35.307 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:35.307 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:35.307 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:35.307 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 07:35:35.307 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:35.307 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:35.307 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:35:35.307 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:35.308 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:35.308 | INFO | Uploaded 36GiB to GPU in 2.19554s (16.3969GiB/s) | 20260505 07:35:35.308 | INFO | Downloaded 34GiB to GPU in 2.06456s (16.4684GiB/s) | 20260505 07:35:40.447 | INFO | Analyzing chunk frames = [1459200,1459712) | 20260505 07:35:40.447 | INFO | MemInfo: { | 20260505 07:35:40.447 | INFO | "gpu" : " 17 GB", | 20260505 07:35:40.447 | INFO | "huge" : "", | 20260505 07:35:40.447 | INFO | "huge_shared" : " 27 GB", | 20260505 07:35:40.447 | INFO | "malloc" : " 14 GB", | 20260505 07:35:40.447 | INFO | "pinned" : " 29 GB" | 20260505 07:35:40.447 | INFO | } | 20260505 07:35:40.447 | INFO | Perf report for the last 5.1394 seconds: | 20260505 07:35:40.447 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:35:40.447 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:40.447 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:40.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:40.447 | INFO | ANALYSIS : 63.4%, 3.0, 98.6ms, 0 | 20260505 07:35:40.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:40.447 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:40.447 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 07:35:40.447 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:40.447 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:40.447 | INFO | Uploaded 36GiB to GPU in 2.15175s (16.7306GiB/s) | 20260505 07:35:40.447 | INFO | Downloaded 34GiB to GPU in 2.05446s (16.5493GiB/s) | 20260505 07:35:45.586 | INFO | Analyzing chunk frames = [1459712,1460224) | 20260505 07:35:45.586 | INFO | MemInfo: { | 20260505 07:35:45.586 | INFO | "gpu" : " 17 GB", | 20260505 07:35:45.586 | INFO | "huge" : "", | 20260505 07:35:45.586 | INFO | "huge_shared" : " 27 GB", | 20260505 07:35:45.587 | INFO | "malloc" : " 14 GB", | 20260505 07:35:45.587 | INFO | "pinned" : " 29 GB" | 20260505 07:35:45.587 | INFO | } | 20260505 07:35:45.587 | INFO | Perf report for the last 5.13978 seconds: | 20260505 07:35:45.587 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:35:45.587 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:45.587 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:45.587 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:45.587 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 07:35:45.587 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:45.587 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:45.587 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:35:45.587 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:45.587 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:45.587 | INFO | Uploaded 36GiB to GPU in 2.14773s (16.7619GiB/s) | 20260505 07:35:45.587 | INFO | Downloaded 34GiB to GPU in 2.04248s (16.6464GiB/s) | 20260505 07:35:50.725 | INFO | Analyzing chunk frames = [1460224,1460736) | 20260505 07:35:50.725 | INFO | MemInfo: { | 20260505 07:35:50.725 | INFO | "gpu" : " 17 GB", | 20260505 07:35:50.725 | INFO | "huge" : "", | 20260505 07:35:50.725 | INFO | "huge_shared" : " 27 GB", | 20260505 07:35:50.725 | INFO | "malloc" : " 14 GB", | 20260505 07:35:50.725 | INFO | "pinned" : " 29 GB" | 20260505 07:35:50.725 | INFO | } | 20260505 07:35:50.725 | INFO | Perf report for the last 5.13889 seconds: | 20260505 07:35:50.725 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:35:50.725 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:50.726 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:50.726 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:50.726 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 07:35:50.726 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:50.726 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:50.726 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 07:35:50.726 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:50.726 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:50.726 | INFO | Uploaded 36GiB to GPU in 2.14213s (16.8057GiB/s) | 20260505 07:35:50.726 | INFO | Downloaded 34GiB to GPU in 2.04835s (16.5988GiB/s) | 20260505 07:35:55.867 | INFO | Analyzing chunk frames = [1460736,1461248) | 20260505 07:35:55.867 | INFO | MemInfo: { | 20260505 07:35:55.867 | INFO | "gpu" : " 17 GB", | 20260505 07:35:55.867 | INFO | "huge" : "", | 20260505 07:35:55.867 | INFO | "huge_shared" : " 27 GB", | 20260505 07:35:55.867 | INFO | "malloc" : " 14 GB", | 20260505 07:35:55.867 | INFO | "pinned" : " 29 GB" | 20260505 07:35:55.867 | INFO | } | 20260505 07:35:55.867 | INFO | Perf report for the last 5.14125 seconds: | 20260505 07:35:55.867 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:35:55.867 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:55.867 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:55.867 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:55.867 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 07:35:55.867 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:55.867 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:55.867 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 07:35:55.867 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:55.867 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:35:55.867 | INFO | Uploaded 36GiB to GPU in 2.16498s (16.6283GiB/s) | 20260505 07:35:55.867 | INFO | Downloaded 34GiB to GPU in 2.0628s (16.4824GiB/s) | 20260505 07:36:01.004 | INFO | Analyzing chunk frames = [1461248,1461760) | 20260505 07:36:01.004 | INFO | MemInfo: { | 20260505 07:36:01.004 | INFO | "gpu" : " 17 GB", | 20260505 07:36:01.004 | INFO | "huge" : "", | 20260505 07:36:01.004 | INFO | "huge_shared" : " 27 GB", | 20260505 07:36:01.004 | INFO | "malloc" : " 14 GB", | 20260505 07:36:01.004 | INFO | "pinned" : " 29 GB" | 20260505 07:36:01.004 | INFO | } | 20260505 07:36:01.004 | INFO | Perf report for the last 5.13731 seconds: | 20260505 07:36:01.004 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:36:01.004 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:01.004 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:01.004 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:01.004 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 07:36:01.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:01.004 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:01.004 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 07:36:01.004 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:01.004 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:01.004 | INFO | Uploaded 36GiB to GPU in 2.20577s (16.3208GiB/s) | 20260505 07:36:01.004 | INFO | Downloaded 34GiB to GPU in 2.15802s (15.7552GiB/s) | 20260505 07:36:06.143 | INFO | Analyzing chunk frames = [1461760,1462272) | 20260505 07:36:06.143 | INFO | MemInfo: { | 20260505 07:36:06.143 | INFO | "gpu" : " 17 GB", | 20260505 07:36:06.143 | INFO | "huge" : "", | 20260505 07:36:06.143 | INFO | "huge_shared" : " 27 GB", | 20260505 07:36:06.143 | INFO | "malloc" : " 14 GB", | 20260505 07:36:06.143 | INFO | "pinned" : " 29 GB" | 20260505 07:36:06.143 | INFO | } | 20260505 07:36:06.143 | INFO | Perf report for the last 5.1394 seconds: | 20260505 07:36:06.143 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:36:06.143 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:06.143 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:06.144 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:06.144 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 07:36:06.144 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:06.144 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:06.144 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:36:06.144 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:06.144 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:06.144 | INFO | Uploaded 36GiB to GPU in 2.18164s (16.5013GiB/s) | 20260505 07:36:06.144 | INFO | Downloaded 34GiB to GPU in 2.05811s (16.52GiB/s) | 20260505 07:36:06.279 | INFO | Starting a round of Baz IO | 20260505 07:36:09.558 | INFO | Populating RTMetrics report | 20260505 07:36:09.558 | INFO | RTMetrics report populated | 20260505 07:36:09.558 | INFO | Opening tmp RTMetrics json file | 20260505 07:36:09.565 | INFO | Serializing RTMetrics to json | 20260505 07:36:09.566 | INFO | Serialization done, swapping tmp file now... | 20260505 07:36:09.566 | INFO | RTMetrics done | 20260505 07:36:09.688 | INFO | Finished a round of Baz IO: Wrote 2.434 GiB in 3.408 seconds | 20260505 07:36:09.688 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 29.48% / 0.03% / 70.49% | 20260505 07:36:09.688 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.45107% / 5.16574e-05% / 8.34465e-05% | 20260505 07:36:09.688 | INFO | Baz IO overall: | 20260505 07:36:09.688 | INFO | BytesWritten : 801939208212 | 20260505 07:36:09.688 | INFO | NumEvents : 380612930345 | 20260505 07:36:11.283 | INFO | Analyzing chunk frames = [1462272,1462784) | 20260505 07:36:11.283 | INFO | MemInfo: { | 20260505 07:36:11.283 | INFO | "gpu" : " 17 GB", | 20260505 07:36:11.283 | INFO | "huge" : "", | 20260505 07:36:11.283 | INFO | "huge_shared" : " 27 GB", | 20260505 07:36:11.283 | INFO | "malloc" : " 14 GB", | 20260505 07:36:11.283 | INFO | "pinned" : " 29 GB" | 20260505 07:36:11.283 | INFO | } | 20260505 07:36:11.283 | INFO | Perf report for the last 5.13943 seconds: | 20260505 07:36:11.283 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:36:11.283 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:11.283 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:11.283 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:11.283 | INFO | ANALYSIS : 65.3%, 2.9, 99.5ms, 0 | 20260505 07:36:11.283 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:11.283 | INFO | RT_METRICS : 46.4%, 1.0, 24.4ms, 0 | 20260505 07:36:11.283 | INFO | PRE_HQ : 63.9%, 1.9, 62.3ms, 0 | 20260505 07:36:11.283 | INFO | BAZWRITER : 65.6%, 4.1, 139.5ms, 0 | 20260505 07:36:11.283 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 07:36:11.283 | INFO | Uploaded 36GiB to GPU in 2.24209s (16.0565GiB/s) | 20260505 07:36:11.283 | INFO | Downloaded 38GiB to GPU in 2.32354s (16.3544GiB/s) | 20260505 07:36:16.422 | INFO | Analyzing chunk frames = [1462784,1463296) | 20260505 07:36:16.422 | INFO | MemInfo: { | 20260505 07:36:16.422 | INFO | "gpu" : " 17 GB", | 20260505 07:36:16.422 | INFO | "huge" : "", | 20260505 07:36:16.422 | INFO | "huge_shared" : " 27 GB", | 20260505 07:36:16.422 | INFO | "malloc" : " 14 GB", | 20260505 07:36:16.422 | INFO | "pinned" : " 29 GB" | 20260505 07:36:16.422 | INFO | } | 20260505 07:36:16.422 | INFO | Perf report for the last 5.13916 seconds: | 20260505 07:36:16.422 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:36:16.422 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:16.422 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:16.422 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:16.422 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 07:36:16.422 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:16.422 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:16.422 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:36:16.422 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:16.422 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:16.422 | INFO | Uploaded 36GiB to GPU in 2.1734s (16.5639GiB/s) | 20260505 07:36:16.422 | INFO | Downloaded 34GiB to GPU in 2.05324s (16.5592GiB/s) | 20260505 07:36:21.561 | INFO | Analyzing chunk frames = [1463296,1463808) | 20260505 07:36:21.561 | INFO | MemInfo: { | 20260505 07:36:21.561 | INFO | "gpu" : " 17 GB", | 20260505 07:36:21.561 | INFO | "huge" : "", | 20260505 07:36:21.561 | INFO | "huge_shared" : " 27 GB", | 20260505 07:36:21.561 | INFO | "malloc" : " 14 GB", | 20260505 07:36:21.561 | INFO | "pinned" : " 29 GB" | 20260505 07:36:21.561 | INFO | } | 20260505 07:36:21.561 | INFO | Perf report for the last 5.13933 seconds: | 20260505 07:36:21.561 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:36:21.561 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:21.561 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:21.561 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:21.561 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 07:36:21.561 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:21.561 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:21.561 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:36:21.561 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:21.561 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:21.562 | INFO | Uploaded 36GiB to GPU in 2.14544s (16.7798GiB/s) | 20260505 07:36:21.562 | INFO | Downloaded 34GiB to GPU in 2.05762s (16.5239GiB/s) | 20260505 07:36:26.700 | INFO | Analyzing chunk frames = [1463808,1464320) | 20260505 07:36:26.700 | INFO | MemInfo: { | 20260505 07:36:26.700 | INFO | "gpu" : " 17 GB", | 20260505 07:36:26.701 | INFO | "huge" : "", | 20260505 07:36:26.701 | INFO | "huge_shared" : " 27 GB", | 20260505 07:36:26.701 | INFO | "malloc" : " 14 GB", | 20260505 07:36:26.701 | INFO | "pinned" : " 29 GB" | 20260505 07:36:26.701 | INFO | } | 20260505 07:36:26.701 | INFO | Perf report for the last 5.13919 seconds: | 20260505 07:36:26.701 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:36:26.701 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:26.701 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:26.701 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:26.701 | INFO | ANALYSIS : 63.2%, 3.0, 98.1ms, 0 | 20260505 07:36:26.701 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:26.701 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:26.701 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 07:36:26.701 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:26.701 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:26.701 | INFO | Uploaded 36GiB to GPU in 2.12982s (16.9029GiB/s) | 20260505 07:36:26.701 | INFO | Downloaded 34GiB to GPU in 2.03819s (16.6815GiB/s) | 20260505 07:36:31.840 | INFO | Analyzing chunk frames = [1464320,1464832) | 20260505 07:36:31.840 | INFO | MemInfo: { | 20260505 07:36:31.840 | INFO | "gpu" : " 17 GB", | 20260505 07:36:31.840 | INFO | "huge" : "", | 20260505 07:36:31.840 | INFO | "huge_shared" : " 27 GB", | 20260505 07:36:31.840 | INFO | "malloc" : " 14 GB", | 20260505 07:36:31.840 | INFO | "pinned" : " 29 GB" | 20260505 07:36:31.840 | INFO | } | 20260505 07:36:31.840 | INFO | Perf report for the last 5.13943 seconds: | 20260505 07:36:31.840 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:36:31.840 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:31.840 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:31.840 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:31.840 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 07:36:31.840 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:31.840 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:31.840 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 07:36:31.840 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:31.840 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:31.840 | INFO | Uploaded 36GiB to GPU in 2.1945s (16.4047GiB/s) | 20260505 07:36:31.840 | INFO | Downloaded 34GiB to GPU in 2.05055s (16.5809GiB/s) | 20260505 07:36:36.981 | INFO | Analyzing chunk frames = [1464832,1465344) | 20260505 07:36:36.981 | INFO | MemInfo: { | 20260505 07:36:36.981 | INFO | "gpu" : " 17 GB", | 20260505 07:36:36.981 | INFO | "huge" : "", | 20260505 07:36:36.981 | INFO | "huge_shared" : " 27 GB", | 20260505 07:36:36.981 | INFO | "malloc" : " 14 GB", | 20260505 07:36:36.981 | INFO | "pinned" : " 29 GB" | 20260505 07:36:36.981 | INFO | } | 20260505 07:36:36.981 | INFO | Perf report for the last 5.14101 seconds: | 20260505 07:36:36.981 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:36:36.981 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:36.981 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:36.981 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:36.981 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 07:36:36.981 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:36.981 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:36.981 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:36:36.981 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:36.981 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:36.981 | INFO | Uploaded 36GiB to GPU in 2.156s (16.6976GiB/s) | 20260505 07:36:36.981 | INFO | Downloaded 34GiB to GPU in 2.05986s (16.506GiB/s) | 20260505 07:36:42.119 | INFO | Analyzing chunk frames = [1465344,1465856) | 20260505 07:36:42.119 | INFO | MemInfo: { | 20260505 07:36:42.119 | INFO | "gpu" : " 17 GB", | 20260505 07:36:42.119 | INFO | "huge" : "", | 20260505 07:36:42.119 | INFO | "huge_shared" : " 27 GB", | 20260505 07:36:42.119 | INFO | "malloc" : " 14 GB", | 20260505 07:36:42.119 | INFO | "pinned" : " 29 GB" | 20260505 07:36:42.119 | INFO | } | 20260505 07:36:42.119 | INFO | Perf report for the last 5.13769 seconds: | 20260505 07:36:42.119 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:36:42.119 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:42.119 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:42.119 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:42.119 | INFO | ANALYSIS : 64.1%, 3.0, 100.0ms, 0 | 20260505 07:36:42.119 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:42.119 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:42.119 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 07:36:42.119 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:42.119 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:42.119 | INFO | Uploaded 36GiB to GPU in 2.18831s (16.4511GiB/s) | 20260505 07:36:42.119 | INFO | Downloaded 34GiB to GPU in 2.15534s (15.7748GiB/s) | 20260505 07:36:47.258 | INFO | Analyzing chunk frames = [1465856,1466368) | 20260505 07:36:47.258 | INFO | MemInfo: { | 20260505 07:36:47.258 | INFO | "gpu" : " 17 GB", | 20260505 07:36:47.258 | INFO | "huge" : "", | 20260505 07:36:47.258 | INFO | "huge_shared" : " 27 GB", | 20260505 07:36:47.258 | INFO | "malloc" : " 14 GB", | 20260505 07:36:47.258 | INFO | "pinned" : " 29 GB" | 20260505 07:36:47.258 | INFO | } | 20260505 07:36:47.258 | INFO | Perf report for the last 5.13931 seconds: | 20260505 07:36:47.258 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:36:47.258 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:47.258 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:47.258 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:47.258 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 07:36:47.258 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:47.258 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:47.258 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 07:36:47.258 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:47.258 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:47.258 | INFO | Uploaded 36GiB to GPU in 2.16325s (16.6416GiB/s) | 20260505 07:36:47.258 | INFO | Downloaded 34GiB to GPU in 2.04935s (16.5906GiB/s) | 20260505 07:36:47.258 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 07:36:47.258 | INFO | Total mem usage: 15.3 GiB | 20260505 07:36:47.258 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 07:36:47.258 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -5.36%) | 20260505 07:36:47.258 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 07:36:47.258 | INFO | High Water Breakdown: | 20260505 07:36:47.258 | INFO | 60.8% : 8.85 GiB ( 94.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 07:36:47.258 | INFO | 39% : 5.68 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 07:36:47.258 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 07:36:47.258 | INFO | ------------------------------------------ | 20260505 07:36:47.258 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:36:47.259 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 07:36:47.259 | INFO | Total mem usage: 26.5 GiB | 20260505 07:36:47.259 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 07:36:47.259 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 07:36:47.259 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 07:36:47.259 | INFO | High Water Breakdown: | 20260505 07:36:47.259 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:36:47.259 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:36:47.259 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:36:47.259 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:36:47.259 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:36:47.259 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:36:47.259 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 07:36:47.259 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 07:36:47.259 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:36:47.259 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 07:36:47.259 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 07:36:47.259 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 07:36:47.259 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 07:36:47.259 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 07:36:47.259 | INFO | ------------------------------------------ | 20260505 07:36:47.259 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:36:47.259 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 07:36:47.259 | INFO | Total mem usage: 25.6 GiB | 20260505 07:36:47.259 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 07:36:47.259 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 07:36:47.259 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 07:36:47.259 | INFO | High Water Breakdown: | 20260505 07:36:47.259 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 07:36:47.259 | INFO | ------------------------------------------ | 20260505 07:36:47.259 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:36:47.259 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 07:36:47.259 | INFO | Total mem usage: 15.7 GiB | 20260505 07:36:47.259 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 07:36:47.259 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 07:36:47.259 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 07:36:47.259 | INFO | High Water Breakdown: | 20260505 07:36:47.259 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 07:36:47.259 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 07:36:47.259 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 07:36:47.259 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 07:36:47.259 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 07:36:47.259 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 07:36:47.259 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 07:36:47.259 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 07:36:47.259 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 07:36:47.259 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:36:47.259 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 07:36:47.259 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:36:47.259 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 07:36:47.259 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 07:36:47.259 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 07:36:47.259 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 07:36:47.259 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:36:47.259 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 07:36:47.259 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:36:47.259 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:36:47.259 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:36:47.259 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 07:36:47.259 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 07:36:47.259 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:36:47.259 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 07:36:47.259 | INFO | ------------------------------------------ | 20260505 07:36:47.259 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:36:47.259 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 07:36:47.259 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:36:47.259 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:36:47.259 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.53e+01, 1.37e+01 / 2.18e+00 | 20260505 07:36:47.259 | INFO | PerfReport: 0.28%, AggregateStats -- 9.33e-02 / 1.04e-01, 1.03e-02 / 1.02e-02 | 20260505 07:36:47.259 | INFO | PerfReport: 11.62%, Baseline -- 3.82e+00 / 4.44e+00, 6.26e-01 / 2.85e-01 | 20260505 07:36:47.259 | INFO | PerfReport: 3.30%, Histogram -- 1.08e+00 / 1.44e+00, 2.11e-01 / 1.31e-01 | 20260505 07:36:47.259 | INFO | PerfReport: 15.46%, DME -- 4.06e+01 / 4.44e+01, 4.11e+00 / 1.38e+00 | 20260505 07:36:47.259 | INFO | PerfReport: 56.35%, FrameLabeling -- 1.85e+01 / 1.94e+01, 2.03e+00 / 1.70e+00 | 20260505 07:36:47.259 | INFO | PerfReport: 7.60%, PulseAccumulating -- 2.49e+00 / 3.16e+00, 4.41e-01 / 2.02e-01 | 20260505 07:36:47.259 | INFO | PerfReport: 3.95%, Metrics -- 1.30e+00 / 1.71e+00, 3.57e-01 / 1.27e-01 | 20260505 07:36:47.259 | INFO | PerfReport: 1.43%, PulseSerializing -- 4.71e-01 / 5.44e-01, 8.99e-02 / 3.18e-02 | 20260505 07:36:47.259 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 07:36:47.259 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:36:47.259 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:36:47.259 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.27e+01 / 0.00e+00 | 20260505 07:36:47.259 | INFO | PerfReport: 22.36%, Upload -- 2.27e+01 / 0.00e+00, 8.05e+00 / 0.00e+00 | 20260505 07:36:47.260 | INFO | PerfReport: 21.59%, Download -- 2.19e+01 / 0.00e+00, 7.98e+00 / 0.00e+00 | 20260505 07:36:47.260 | INFO | PerfReport: 32.38%, Compute -- 3.28e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 07:36:47.260 | INFO | PerfReport: 1.57%, LockUpload -- 1.59e+00 / 0.00e+00, 7.20e+00 / 0.00e+00 | 20260505 07:36:47.260 | INFO | PerfReport: 0.87%, LockDownload -- 8.78e-01 / 0.00e+00, 2.23e+00 / 0.00e+00 | 20260505 07:36:47.260 | INFO | PerfReport: 21.22%, LockCompute -- 2.15e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260505 07:36:47.260 | INFO | PerfReport: 0.01%, Misc -- 1.46e-02 / 0.00e+00, 2.07e-02 / 0.00e+00 | 20260505 07:36:47.392 | INFO | Starting a round of Baz IO | 20260505 07:36:50.660 | INFO | Populating RTMetrics report | 20260505 07:36:50.661 | INFO | RTMetrics report populated | 20260505 07:36:50.661 | INFO | Opening tmp RTMetrics json file | 20260505 07:36:50.682 | INFO | Serializing RTMetrics to json | 20260505 07:36:50.688 | INFO | Serialization done, swapping tmp file now... | 20260505 07:36:50.705 | INFO | RTMetrics done | 20260505 07:36:50.800 | INFO | Finished a round of Baz IO: Wrote 2.435 GiB in 3.408 seconds | 20260505 07:36:50.801 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 29.45% / 0.03% / 70.52% | 20260505 07:36:50.801 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.43758% / 3.57628e-05% / 9.13938e-05% | 20260505 07:36:50.801 | INFO | Baz IO overall: | 20260505 07:36:50.801 | INFO | BytesWritten : 804553963686 | 20260505 07:36:50.801 | INFO | NumEvents : 381864281890 | 20260505 07:36:52.398 | INFO | Analyzing chunk frames = [1466368,1466880) | 20260505 07:36:52.398 | INFO | MemInfo: { | 20260505 07:36:52.398 | INFO | "gpu" : " 17 GB", | 20260505 07:36:52.398 | INFO | "huge" : "", | 20260505 07:36:52.398 | INFO | "huge_shared" : " 27 GB", | 20260505 07:36:52.398 | INFO | "malloc" : " 14 GB", | 20260505 07:36:52.398 | INFO | "pinned" : " 29 GB" | 20260505 07:36:52.398 | INFO | } | 20260505 07:36:52.398 | INFO | Perf report for the last 5.13973 seconds: | 20260505 07:36:52.398 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:36:52.398 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:52.398 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:52.398 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:52.398 | INFO | ANALYSIS : 65.0%, 2.9, 99.4ms, 0 | 20260505 07:36:52.398 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:52.398 | INFO | RT_METRICS : 46.8%, 1.0, 24.5ms, 0 | 20260505 07:36:52.398 | INFO | PRE_HQ : 63.6%, 1.9, 62.0ms, 0 | 20260505 07:36:52.398 | INFO | BAZWRITER : 65.6%, 4.2, 140.8ms, 0 | 20260505 07:36:52.398 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260505 07:36:52.398 | INFO | Uploaded 36GiB to GPU in 2.21693s (16.2386GiB/s) | 20260505 07:36:52.398 | INFO | Downloaded 38GiB to GPU in 2.32622s (16.3355GiB/s) | 20260505 07:36:57.537 | INFO | Analyzing chunk frames = [1466880,1467392) | 20260505 07:36:57.537 | INFO | MemInfo: { | 20260505 07:36:57.537 | INFO | "gpu" : " 17 GB", | 20260505 07:36:57.537 | INFO | "huge" : "", | 20260505 07:36:57.537 | INFO | "huge_shared" : " 27 GB", | 20260505 07:36:57.537 | INFO | "malloc" : " 14 GB", | 20260505 07:36:57.537 | INFO | "pinned" : " 29 GB" | 20260505 07:36:57.537 | INFO | } | 20260505 07:36:57.537 | INFO | Perf report for the last 5.1389 seconds: | 20260505 07:36:57.537 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:36:57.537 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:57.537 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:57.537 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:57.537 | INFO | ANALYSIS : 64.5%, 3.0, 100.7ms, 0 | 20260505 07:36:57.537 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:57.537 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:57.537 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 07:36:57.537 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:57.537 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:36:57.537 | INFO | Uploaded 36GiB to GPU in 2.19213s (16.4224GiB/s) | 20260505 07:36:57.537 | INFO | Downloaded 34GiB to GPU in 2.05499s (16.5451GiB/s) | 20260505 07:37:02.676 | INFO | Analyzing chunk frames = [1467392,1467904) | 20260505 07:37:02.676 | INFO | MemInfo: { | 20260505 07:37:02.676 | INFO | "gpu" : " 17 GB", | 20260505 07:37:02.676 | INFO | "huge" : "", | 20260505 07:37:02.676 | INFO | "huge_shared" : " 27 GB", | 20260505 07:37:02.676 | INFO | "malloc" : " 14 GB", | 20260505 07:37:02.676 | INFO | "pinned" : " 29 GB" | 20260505 07:37:02.676 | INFO | } | 20260505 07:37:02.676 | INFO | Perf report for the last 5.13956 seconds: | 20260505 07:37:02.676 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:37:02.676 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:02.676 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:02.676 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:02.676 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 07:37:02.676 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:02.676 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:02.676 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:37:02.676 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:02.676 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:02.677 | INFO | Uploaded 36GiB to GPU in 2.14584s (16.7766GiB/s) | 20260505 07:37:02.677 | INFO | Downloaded 34GiB to GPU in 2.05758s (16.5243GiB/s) | 20260505 07:37:07.815 | INFO | Analyzing chunk frames = [1467904,1468416) | 20260505 07:37:07.816 | INFO | MemInfo: { | 20260505 07:37:07.816 | INFO | "gpu" : " 17 GB", | 20260505 07:37:07.816 | INFO | "huge" : "", | 20260505 07:37:07.816 | INFO | "huge_shared" : " 27 GB", | 20260505 07:37:07.816 | INFO | "malloc" : " 14 GB", | 20260505 07:37:07.816 | INFO | "pinned" : " 29 GB" | 20260505 07:37:07.816 | INFO | } | 20260505 07:37:07.816 | INFO | Perf report for the last 5.13927 seconds: | 20260505 07:37:07.816 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:37:07.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:07.816 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:07.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:07.816 | INFO | ANALYSIS : 63.1%, 3.0, 98.0ms, 0 | 20260505 07:37:07.816 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:07.816 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:07.816 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 07:37:07.816 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:07.816 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:07.816 | INFO | Uploaded 36GiB to GPU in 2.12135s (16.9703GiB/s) | 20260505 07:37:07.816 | INFO | Downloaded 34GiB to GPU in 2.03351s (16.7199GiB/s) | 20260505 07:37:12.955 | INFO | Analyzing chunk frames = [1468416,1468928) | 20260505 07:37:12.955 | INFO | MemInfo: { | 20260505 07:37:12.955 | INFO | "gpu" : " 17 GB", | 20260505 07:37:12.955 | INFO | "huge" : "", | 20260505 07:37:12.955 | INFO | "huge_shared" : " 27 GB", | 20260505 07:37:12.955 | INFO | "malloc" : " 14 GB", | 20260505 07:37:12.955 | INFO | "pinned" : " 29 GB" | 20260505 07:37:12.955 | INFO | } | 20260505 07:37:12.955 | INFO | Perf report for the last 5.13905 seconds: | 20260505 07:37:12.955 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:37:12.955 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:12.955 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:12.955 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:12.955 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 07:37:12.955 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:12.955 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:12.955 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:37:12.955 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:12.955 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:12.955 | INFO | Uploaded 36GiB to GPU in 2.19751s (16.3822GiB/s) | 20260505 07:37:12.955 | INFO | Downloaded 34GiB to GPU in 2.05512s (16.5441GiB/s) | 20260505 07:37:18.096 | INFO | Analyzing chunk frames = [1468928,1469440) | 20260505 07:37:18.096 | INFO | MemInfo: { | 20260505 07:37:18.096 | INFO | "gpu" : " 17 GB", | 20260505 07:37:18.096 | INFO | "huge" : "", | 20260505 07:37:18.096 | INFO | "huge_shared" : " 27 GB", | 20260505 07:37:18.096 | INFO | "malloc" : " 14 GB", | 20260505 07:37:18.096 | INFO | "pinned" : " 29 GB" | 20260505 07:37:18.096 | INFO | } | 20260505 07:37:18.096 | INFO | Perf report for the last 5.14128 seconds: | 20260505 07:37:18.096 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:37:18.096 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:18.096 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:18.096 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:18.096 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 07:37:18.096 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:18.096 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:18.096 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 07:37:18.096 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:18.096 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:18.096 | INFO | Uploaded 36GiB to GPU in 2.15998s (16.6668GiB/s) | 20260505 07:37:18.096 | INFO | Downloaded 34GiB to GPU in 2.05671s (16.5313GiB/s) | 20260505 07:37:23.233 | INFO | Analyzing chunk frames = [1469440,1469952) | 20260505 07:37:23.234 | INFO | MemInfo: { | 20260505 07:37:23.234 | INFO | "gpu" : " 17 GB", | 20260505 07:37:23.234 | INFO | "huge" : "", | 20260505 07:37:23.234 | INFO | "huge_shared" : " 27 GB", | 20260505 07:37:23.234 | INFO | "malloc" : " 14 GB", | 20260505 07:37:23.234 | INFO | "pinned" : " 29 GB" | 20260505 07:37:23.234 | INFO | } | 20260505 07:37:23.234 | INFO | Perf report for the last 5.13767 seconds: | 20260505 07:37:23.234 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:37:23.234 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:23.234 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:23.234 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:23.234 | INFO | ANALYSIS : 63.8%, 3.0, 99.5ms, 0 | 20260505 07:37:23.234 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:23.234 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:23.234 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260505 07:37:23.234 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:23.234 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:23.234 | INFO | Uploaded 36GiB to GPU in 2.15552s (16.7013GiB/s) | 20260505 07:37:23.234 | INFO | Downloaded 34GiB to GPU in 2.14171s (15.8752GiB/s) | 20260505 07:37:28.373 | INFO | Analyzing chunk frames = [1469952,1470464) | 20260505 07:37:28.373 | INFO | MemInfo: { | 20260505 07:37:28.373 | INFO | "gpu" : " 17 GB", | 20260505 07:37:28.373 | INFO | "huge" : "", | 20260505 07:37:28.373 | INFO | "huge_shared" : " 27 GB", | 20260505 07:37:28.373 | INFO | "malloc" : " 14 GB", | 20260505 07:37:28.373 | INFO | "pinned" : " 29 GB" | 20260505 07:37:28.373 | INFO | } | 20260505 07:37:28.373 | INFO | Perf report for the last 5.13924 seconds: | 20260505 07:37:28.373 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:37:28.373 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:28.373 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:28.373 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:28.373 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 07:37:28.373 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:28.373 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:28.373 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 07:37:28.373 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:28.373 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:28.373 | INFO | Uploaded 36GiB to GPU in 2.17529s (16.5495GiB/s) | 20260505 07:37:28.373 | INFO | Downloaded 34GiB to GPU in 2.04752s (16.6055GiB/s) | 20260505 07:37:28.507 | INFO | Starting a round of Baz IO | 20260505 07:37:31.798 | INFO | Populating RTMetrics report | 20260505 07:37:31.798 | INFO | RTMetrics report populated | 20260505 07:37:31.798 | INFO | Opening tmp RTMetrics json file | 20260505 07:37:31.809 | INFO | Serializing RTMetrics to json | 20260505 07:37:31.814 | INFO | Serialization done, swapping tmp file now... | 20260505 07:37:31.815 | INFO | RTMetrics done | 20260505 07:37:31.940 | INFO | Finished a round of Baz IO: Wrote 2.437 GiB in 3.433 seconds | 20260505 07:37:31.940 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 29.42% / 0.03% / 70.55% | 20260505 07:37:31.940 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.43198% / 5.16574e-05% / 0.000103315% | 20260505 07:37:31.940 | INFO | Baz IO overall: | 20260505 07:37:31.940 | INFO | BytesWritten : 807170239631 | 20260505 07:37:31.940 | INFO | NumEvents : 383116981318 | 20260505 07:37:33.512 | INFO | Analyzing chunk frames = [1470464,1470976) | 20260505 07:37:33.512 | INFO | MemInfo: { | 20260505 07:37:33.512 | INFO | "gpu" : " 17 GB", | 20260505 07:37:33.512 | INFO | "huge" : "", | 20260505 07:37:33.512 | INFO | "huge_shared" : " 27 GB", | 20260505 07:37:33.512 | INFO | "malloc" : " 14 GB", | 20260505 07:37:33.512 | INFO | "pinned" : " 29 GB" | 20260505 07:37:33.512 | INFO | } | 20260505 07:37:33.512 | INFO | Perf report for the last 5.13925 seconds: | 20260505 07:37:33.512 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:37:33.512 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:33.512 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:33.512 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:33.512 | INFO | ANALYSIS : 65.5%, 2.9, 99.6ms, 0 | 20260505 07:37:33.512 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:33.512 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260505 07:37:33.512 | INFO | PRE_HQ : 64.0%, 1.9, 62.8ms, 0 | 20260505 07:37:33.512 | INFO | BAZWRITER : 66.0%, 4.1, 139.8ms, 0 | 20260505 07:37:33.512 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 07:37:33.512 | INFO | Uploaded 36GiB to GPU in 2.2304s (16.1406GiB/s) | 20260505 07:37:33.512 | INFO | Downloaded 38GiB to GPU in 2.35871s (16.1105GiB/s) | 20260505 07:37:38.651 | INFO | Analyzing chunk frames = [1470976,1471488) | 20260505 07:37:38.651 | INFO | MemInfo: { | 20260505 07:37:38.651 | INFO | "gpu" : " 17 GB", | 20260505 07:37:38.651 | INFO | "huge" : "", | 20260505 07:37:38.651 | INFO | "huge_shared" : " 27 GB", | 20260505 07:37:38.651 | INFO | "malloc" : " 14 GB", | 20260505 07:37:38.651 | INFO | "pinned" : " 29 GB" | 20260505 07:37:38.651 | INFO | } | 20260505 07:37:38.651 | INFO | Perf report for the last 5.13928 seconds: | 20260505 07:37:38.651 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:37:38.651 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:38.651 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:38.652 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:38.652 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 07:37:38.652 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:38.652 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:38.652 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:37:38.652 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:38.652 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:38.652 | INFO | Uploaded 36GiB to GPU in 2.11972s (16.9834GiB/s) | 20260505 07:37:38.652 | INFO | Downloaded 34GiB to GPU in 2.04521s (16.6242GiB/s) | 20260505 07:37:43.791 | INFO | Analyzing chunk frames = [1471488,1472000) | 20260505 07:37:43.791 | INFO | MemInfo: { | 20260505 07:37:43.791 | INFO | "gpu" : " 17 GB", | 20260505 07:37:43.791 | INFO | "huge" : "", | 20260505 07:37:43.791 | INFO | "huge_shared" : " 27 GB", | 20260505 07:37:43.791 | INFO | "malloc" : " 14 GB", | 20260505 07:37:43.791 | INFO | "pinned" : " 29 GB" | 20260505 07:37:43.791 | INFO | } | 20260505 07:37:43.791 | INFO | Perf report for the last 5.13946 seconds: | 20260505 07:37:43.791 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:37:43.791 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:43.791 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:43.791 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:43.791 | INFO | ANALYSIS : 63.2%, 3.0, 98.4ms, 0 | 20260505 07:37:43.791 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:43.791 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:43.791 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 07:37:43.791 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:43.791 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:43.791 | INFO | Uploaded 36GiB to GPU in 2.15001s (16.7441GiB/s) | 20260505 07:37:43.791 | INFO | Downloaded 34GiB to GPU in 2.05601s (16.5369GiB/s) | 20260505 07:37:48.930 | INFO | Analyzing chunk frames = [1472000,1472512) | 20260505 07:37:48.930 | INFO | MemInfo: { | 20260505 07:37:48.930 | INFO | "gpu" : " 17 GB", | 20260505 07:37:48.930 | INFO | "huge" : "", | 20260505 07:37:48.930 | INFO | "huge_shared" : " 27 GB", | 20260505 07:37:48.930 | INFO | "malloc" : " 14 GB", | 20260505 07:37:48.930 | INFO | "pinned" : " 29 GB" | 20260505 07:37:48.930 | INFO | } | 20260505 07:37:48.930 | INFO | Perf report for the last 5.13938 seconds: | 20260505 07:37:48.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:37:48.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:48.930 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:48.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:48.930 | INFO | ANALYSIS : 63.3%, 3.0, 98.3ms, 0 | 20260505 07:37:48.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:48.930 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:48.930 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 07:37:48.930 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:48.930 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:48.931 | INFO | Uploaded 36GiB to GPU in 2.14514s (16.7821GiB/s) | 20260505 07:37:48.931 | INFO | Downloaded 34GiB to GPU in 2.04153s (16.6542GiB/s) | 20260505 07:37:54.069 | INFO | Analyzing chunk frames = [1472512,1473024) | 20260505 07:37:54.070 | INFO | MemInfo: { | 20260505 07:37:54.070 | INFO | "gpu" : " 17 GB", | 20260505 07:37:54.070 | INFO | "huge" : "", | 20260505 07:37:54.070 | INFO | "huge_shared" : " 27 GB", | 20260505 07:37:54.070 | INFO | "malloc" : " 14 GB", | 20260505 07:37:54.070 | INFO | "pinned" : " 29 GB" | 20260505 07:37:54.070 | INFO | } | 20260505 07:37:54.070 | INFO | Perf report for the last 5.13931 seconds: | 20260505 07:37:54.070 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:37:54.070 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:54.070 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:54.070 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:54.070 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 07:37:54.070 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:54.070 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:54.070 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 07:37:54.070 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:54.070 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:54.070 | INFO | Uploaded 36GiB to GPU in 2.16251s (16.6474GiB/s) | 20260505 07:37:54.070 | INFO | Downloaded 34GiB to GPU in 2.05022s (16.5836GiB/s) | 20260505 07:37:59.211 | INFO | Analyzing chunk frames = [1473024,1473536) | 20260505 07:37:59.211 | INFO | MemInfo: { | 20260505 07:37:59.211 | INFO | "gpu" : " 17 GB", | 20260505 07:37:59.211 | INFO | "huge" : "", | 20260505 07:37:59.211 | INFO | "huge_shared" : " 27 GB", | 20260505 07:37:59.211 | INFO | "malloc" : " 14 GB", | 20260505 07:37:59.211 | INFO | "pinned" : " 29 GB" | 20260505 07:37:59.211 | INFO | } | 20260505 07:37:59.211 | INFO | Perf report for the last 5.14106 seconds: | 20260505 07:37:59.211 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:37:59.211 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:59.211 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:59.211 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:59.211 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 07:37:59.211 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:59.211 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:59.211 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 07:37:59.211 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:59.211 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:37:59.211 | INFO | Uploaded 36GiB to GPU in 2.1654s (16.6251GiB/s) | 20260505 07:37:59.211 | INFO | Downloaded 34GiB to GPU in 2.05434s (16.5504GiB/s) | 20260505 07:38:04.348 | INFO | Analyzing chunk frames = [1473536,1474048) | 20260505 07:38:04.348 | INFO | MemInfo: { | 20260505 07:38:04.348 | INFO | "gpu" : " 17 GB", | 20260505 07:38:04.348 | INFO | "huge" : "", | 20260505 07:38:04.348 | INFO | "huge_shared" : " 27 GB", | 20260505 07:38:04.348 | INFO | "malloc" : " 14 GB", | 20260505 07:38:04.348 | INFO | "pinned" : " 29 GB" | 20260505 07:38:04.348 | INFO | } | 20260505 07:38:04.349 | INFO | Perf report for the last 5.1378 seconds: | 20260505 07:38:04.349 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:38:04.349 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:04.349 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:04.349 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:04.349 | INFO | ANALYSIS : 64.1%, 3.0, 100.0ms, 0 | 20260505 07:38:04.349 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:04.349 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:04.349 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 07:38:04.349 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:04.349 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:04.349 | INFO | Uploaded 36GiB to GPU in 2.18149s (16.5025GiB/s) | 20260505 07:38:04.349 | INFO | Downloaded 34GiB to GPU in 2.15142s (15.8035GiB/s) | 20260505 07:38:09.488 | INFO | Analyzing chunk frames = [1474048,1474560) | 20260505 07:38:09.488 | INFO | MemInfo: { | 20260505 07:38:09.488 | INFO | "gpu" : " 17 GB", | 20260505 07:38:09.488 | INFO | "huge" : "", | 20260505 07:38:09.488 | INFO | "huge_shared" : " 27 GB", | 20260505 07:38:09.488 | INFO | "malloc" : " 14 GB", | 20260505 07:38:09.488 | INFO | "pinned" : " 29 GB" | 20260505 07:38:09.488 | INFO | } | 20260505 07:38:09.488 | INFO | Perf report for the last 5.1393 seconds: | 20260505 07:38:09.488 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:38:09.488 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:09.488 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:09.488 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:09.488 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 07:38:09.488 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:09.488 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:09.488 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 07:38:09.488 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:09.488 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:09.488 | INFO | Uploaded 36GiB to GPU in 2.17943s (16.5181GiB/s) | 20260505 07:38:09.488 | INFO | Downloaded 34GiB to GPU in 2.05034s (16.5826GiB/s) | 20260505 07:38:09.488 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 07:38:09.488 | INFO | Total mem usage: 15.3 GiB | 20260505 07:38:09.488 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 07:38:09.488 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.09%) | 20260505 07:38:09.488 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 07:38:09.488 | INFO | High Water Breakdown: | 20260505 07:38:09.488 | INFO | 60.7% : 8.77 GiB ( 93.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 07:38:09.488 | INFO | 39.1% : 5.65 GiB ( 85.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 07:38:09.488 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 07:38:09.488 | INFO | ------------------------------------------ | 20260505 07:38:09.488 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:38:09.488 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 07:38:09.488 | INFO | Total mem usage: 26.5 GiB | 20260505 07:38:09.488 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 07:38:09.488 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 07:38:09.488 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 07:38:09.488 | INFO | High Water Breakdown: | 20260505 07:38:09.488 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:38:09.488 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:38:09.488 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:38:09.488 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:38:09.488 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:38:09.488 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:38:09.488 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 07:38:09.488 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 07:38:09.488 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:38:09.488 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 07:38:09.489 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 07:38:09.489 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 07:38:09.489 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 07:38:09.489 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 07:38:09.489 | INFO | ------------------------------------------ | 20260505 07:38:09.489 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:38:09.489 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 07:38:09.489 | INFO | Total mem usage: 25.6 GiB | 20260505 07:38:09.489 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 07:38:09.489 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 07:38:09.489 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 07:38:09.489 | INFO | High Water Breakdown: | 20260505 07:38:09.489 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 07:38:09.489 | INFO | ------------------------------------------ | 20260505 07:38:09.489 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:38:09.489 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 07:38:09.489 | INFO | Total mem usage: 15.7 GiB | 20260505 07:38:09.489 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 07:38:09.489 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 07:38:09.489 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 07:38:09.489 | INFO | High Water Breakdown: | 20260505 07:38:09.489 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 07:38:09.489 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 07:38:09.489 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 07:38:09.489 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 07:38:09.489 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 07:38:09.489 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 07:38:09.489 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 07:38:09.489 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 07:38:09.489 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 07:38:09.489 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:38:09.489 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 07:38:09.489 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:38:09.489 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 07:38:09.489 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 07:38:09.489 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 07:38:09.489 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 07:38:09.489 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:38:09.489 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 07:38:09.489 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:38:09.489 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:38:09.489 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:38:09.489 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 07:38:09.489 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 07:38:09.489 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:38:09.489 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 07:38:09.489 | INFO | ------------------------------------------ | 20260505 07:38:09.489 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:38:09.489 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 07:38:09.489 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:38:09.489 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:38:09.489 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.49e+01, 1.36e+01 / 2.11e+00 | 20260505 07:38:09.489 | INFO | PerfReport: 0.29%, AggregateStats -- 9.38e-02 / 1.02e-01, 1.12e-02 / 6.19e-03 | 20260505 07:38:09.489 | INFO | PerfReport: 11.62%, Baseline -- 3.81e+00 / 4.37e+00, 6.23e-01 / 3.64e-01 | 20260505 07:38:09.489 | INFO | PerfReport: 3.31%, Histogram -- 1.08e+00 / 1.45e+00, 2.13e-01 / 1.29e-01 | 20260505 07:38:09.489 | INFO | PerfReport: 15.47%, DME -- 4.06e+01 / 4.43e+01, 4.12e+00 / 1.29e+00 | 20260505 07:38:09.489 | INFO | PerfReport: 56.27%, FrameLabeling -- 1.85e+01 / 1.93e+01, 1.98e+00 / 1.70e+00 | 20260505 07:38:09.489 | INFO | PerfReport: 7.63%, PulseAccumulating -- 2.50e+00 / 3.16e+00, 4.46e-01 / 2.34e-01 | 20260505 07:38:09.489 | INFO | PerfReport: 3.98%, Metrics -- 1.30e+00 / 1.71e+00, 3.65e-01 / 1.52e-01 | 20260505 07:38:09.489 | INFO | PerfReport: 1.44%, PulseSerializing -- 4.71e-01 / 5.47e-01, 9.06e-02 / 3.04e-02 | 20260505 07:38:09.489 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 07:38:09.489 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:38:09.489 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:38:09.489 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.26e+01 / 0.00e+00 | 20260505 07:38:09.489 | INFO | PerfReport: 22.29%, Upload -- 2.26e+01 / 0.00e+00, 7.92e+00 / 0.00e+00 | 20260505 07:38:09.489 | INFO | PerfReport: 21.58%, Download -- 2.19e+01 / 0.00e+00, 7.92e+00 / 0.00e+00 | 20260505 07:38:09.489 | INFO | PerfReport: 32.37%, Compute -- 3.28e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 07:38:09.489 | INFO | PerfReport: 1.58%, LockUpload -- 1.60e+00 / 0.00e+00, 7.16e+00 / 0.00e+00 | 20260505 07:38:09.489 | INFO | PerfReport: 0.91%, LockDownload -- 9.20e-01 / 0.00e+00, 2.40e+00 / 0.00e+00 | 20260505 07:38:09.489 | INFO | PerfReport: 21.26%, LockCompute -- 2.15e+01 / 0.00e+00, 1.72e+01 / 0.00e+00 | 20260505 07:38:09.489 | INFO | PerfReport: 0.01%, Misc -- 1.47e-02 / 0.00e+00, 2.01e-02 / 0.00e+00 | 20260505 07:38:09.622 | INFO | Starting a round of Baz IO | 20260505 07:38:12.907 | INFO | Populating RTMetrics report | 20260505 07:38:12.907 | INFO | RTMetrics report populated | 20260505 07:38:12.907 | INFO | Opening tmp RTMetrics json file | 20260505 07:38:12.908 | INFO | Serializing RTMetrics to json | 20260505 07:38:12.909 | INFO | Serialization done, swapping tmp file now... | 20260505 07:38:12.909 | INFO | RTMetrics done | 20260505 07:38:13.049 | INFO | Finished a round of Baz IO: Wrote 2.436 GiB in 3.427 seconds | 20260505 07:38:13.049 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 29.39% / 0.03% / 70.58% | 20260505 07:38:13.049 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.41645% / 3.17891e-05% / 8.34465e-05% | 20260505 07:38:13.049 | INFO | Baz IO overall: | 20260505 07:38:13.049 | INFO | BytesWritten : 809785659041 | 20260505 07:38:13.049 | INFO | NumEvents : 384368676994 | 20260505 07:38:14.627 | INFO | Analyzing chunk frames = [1474560,1475072) | 20260505 07:38:14.627 | INFO | MemInfo: { | 20260505 07:38:14.627 | INFO | "gpu" : " 17 GB", | 20260505 07:38:14.627 | INFO | "huge" : "", | 20260505 07:38:14.627 | INFO | "huge_shared" : " 27 GB", | 20260505 07:38:14.627 | INFO | "malloc" : " 14 GB", | 20260505 07:38:14.627 | INFO | "pinned" : " 29 GB" | 20260505 07:38:14.627 | INFO | } | 20260505 07:38:14.627 | INFO | Perf report for the last 5.13952 seconds: | 20260505 07:38:14.627 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:38:14.627 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:14.627 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:14.627 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:14.627 | INFO | ANALYSIS : 65.4%, 2.9, 99.7ms, 0 | 20260505 07:38:14.627 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:14.627 | INFO | RT_METRICS : 46.2%, 1.0, 24.2ms, 0 | 20260505 07:38:14.627 | INFO | PRE_HQ : 64.0%, 1.9, 62.4ms, 0 | 20260505 07:38:14.627 | INFO | BAZWRITER : 66.0%, 4.1, 140.0ms, 0 | 20260505 07:38:14.627 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 07:38:14.628 | INFO | Uploaded 36GiB to GPU in 2.2341s (16.1139GiB/s) | 20260505 07:38:14.628 | INFO | Downloaded 38GiB to GPU in 2.3453s (16.2026GiB/s) | 20260505 07:38:19.766 | INFO | Analyzing chunk frames = [1475072,1475584) | 20260505 07:38:19.766 | INFO | MemInfo: { | 20260505 07:38:19.766 | INFO | "gpu" : " 17 GB", | 20260505 07:38:19.766 | INFO | "huge" : "", | 20260505 07:38:19.766 | INFO | "huge_shared" : " 27 GB", | 20260505 07:38:19.766 | INFO | "malloc" : " 14 GB", | 20260505 07:38:19.766 | INFO | "pinned" : " 29 GB" | 20260505 07:38:19.766 | INFO | } | 20260505 07:38:19.767 | INFO | Perf report for the last 5.13914 seconds: | 20260505 07:38:19.767 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:38:19.767 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:19.767 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:19.767 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:19.767 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 07:38:19.767 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:19.767 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:19.767 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:38:19.767 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:19.767 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:19.767 | INFO | Uploaded 36GiB to GPU in 2.18561s (16.4714GiB/s) | 20260505 07:38:19.767 | INFO | Downloaded 34GiB to GPU in 2.05483s (16.5463GiB/s) | 20260505 07:38:24.906 | INFO | Analyzing chunk frames = [1475584,1476096) | 20260505 07:38:24.906 | INFO | MemInfo: { | 20260505 07:38:24.906 | INFO | "gpu" : " 17 GB", | 20260505 07:38:24.906 | INFO | "huge" : "", | 20260505 07:38:24.906 | INFO | "huge_shared" : " 27 GB", | 20260505 07:38:24.906 | INFO | "malloc" : " 14 GB", | 20260505 07:38:24.906 | INFO | "pinned" : " 29 GB" | 20260505 07:38:24.906 | INFO | } | 20260505 07:38:24.906 | INFO | Perf report for the last 5.13928 seconds: | 20260505 07:38:24.906 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:38:24.906 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:24.906 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:24.906 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:24.906 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 07:38:24.906 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:24.906 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:24.906 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:38:24.906 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:24.906 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:24.906 | INFO | Uploaded 36GiB to GPU in 2.15401s (16.713GiB/s) | 20260505 07:38:24.906 | INFO | Downloaded 34GiB to GPU in 2.05397s (16.5533GiB/s) | 20260505 07:38:30.045 | INFO | Analyzing chunk frames = [1476096,1476608) | 20260505 07:38:30.045 | INFO | MemInfo: { | 20260505 07:38:30.045 | INFO | "gpu" : " 17 GB", | 20260505 07:38:30.045 | INFO | "huge" : "", | 20260505 07:38:30.045 | INFO | "huge_shared" : " 27 GB", | 20260505 07:38:30.045 | INFO | "malloc" : " 14 GB", | 20260505 07:38:30.045 | INFO | "pinned" : " 29 GB" | 20260505 07:38:30.045 | INFO | } | 20260505 07:38:30.045 | INFO | Perf report for the last 5.13942 seconds: | 20260505 07:38:30.045 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:38:30.045 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:30.045 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:30.045 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:30.045 | INFO | ANALYSIS : 63.3%, 3.0, 98.3ms, 0 | 20260505 07:38:30.045 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:30.045 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:30.045 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 07:38:30.045 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:30.045 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:30.046 | INFO | Uploaded 36GiB to GPU in 2.12888s (16.9103GiB/s) | 20260505 07:38:30.046 | INFO | Downloaded 34GiB to GPU in 2.03132s (16.7379GiB/s) | 20260505 07:38:35.184 | INFO | Analyzing chunk frames = [1476608,1477120) | 20260505 07:38:35.184 | INFO | MemInfo: { | 20260505 07:38:35.184 | INFO | "gpu" : " 17 GB", | 20260505 07:38:35.184 | INFO | "huge" : "", | 20260505 07:38:35.185 | INFO | "huge_shared" : " 27 GB", | 20260505 07:38:35.185 | INFO | "malloc" : " 14 GB", | 20260505 07:38:35.185 | INFO | "pinned" : " 29 GB" | 20260505 07:38:35.185 | INFO | } | 20260505 07:38:35.185 | INFO | Perf report for the last 5.13935 seconds: | 20260505 07:38:35.185 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:38:35.185 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:35.185 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:35.185 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:35.185 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 07:38:35.185 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:35.185 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:35.185 | INFO | PRE_HQ : 29.8%, 1.0, 15.4ms, 0 | 20260505 07:38:35.185 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:35.185 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:35.185 | INFO | Uploaded 36GiB to GPU in 2.20098s (16.3564GiB/s) | 20260505 07:38:35.185 | INFO | Downloaded 34GiB to GPU in 2.05174s (16.5713GiB/s) | 20260505 07:38:40.325 | INFO | Analyzing chunk frames = [1477120,1477632) | 20260505 07:38:40.325 | INFO | MemInfo: { | 20260505 07:38:40.325 | INFO | "gpu" : " 17 GB", | 20260505 07:38:40.325 | INFO | "huge" : "", | 20260505 07:38:40.326 | INFO | "huge_shared" : " 27 GB", | 20260505 07:38:40.326 | INFO | "malloc" : " 14 GB", | 20260505 07:38:40.326 | INFO | "pinned" : " 29 GB" | 20260505 07:38:40.326 | INFO | } | 20260505 07:38:40.326 | INFO | Perf report for the last 5.14099 seconds: | 20260505 07:38:40.326 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:38:40.326 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:40.326 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:40.326 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:40.326 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 07:38:40.326 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:40.326 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:40.326 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 07:38:40.326 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:40.326 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:40.326 | INFO | Uploaded 36GiB to GPU in 2.16058s (16.6622GiB/s) | 20260505 07:38:40.326 | INFO | Downloaded 34GiB to GPU in 2.05706s (16.5285GiB/s) | 20260505 07:38:45.463 | INFO | Analyzing chunk frames = [1477632,1478144) | 20260505 07:38:45.463 | INFO | MemInfo: { | 20260505 07:38:45.463 | INFO | "gpu" : " 17 GB", | 20260505 07:38:45.463 | INFO | "huge" : "", | 20260505 07:38:45.463 | INFO | "huge_shared" : " 27 GB", | 20260505 07:38:45.463 | INFO | "malloc" : " 14 GB", | 20260505 07:38:45.463 | INFO | "pinned" : " 29 GB" | 20260505 07:38:45.463 | INFO | } | 20260505 07:38:45.463 | INFO | Perf report for the last 5.13743 seconds: | 20260505 07:38:45.463 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:38:45.463 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:45.463 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:45.463 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:45.463 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 07:38:45.463 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:45.463 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:45.463 | INFO | PRE_HQ : 31.7%, 1.0, 16.4ms, 0 | 20260505 07:38:45.463 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:45.463 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:45.463 | INFO | Uploaded 36GiB to GPU in 2.1625s (16.6474GiB/s) | 20260505 07:38:45.463 | INFO | Downloaded 34GiB to GPU in 2.13635s (15.915GiB/s) | 20260505 07:38:50.603 | INFO | Analyzing chunk frames = [1478144,1478656) | 20260505 07:38:50.603 | INFO | MemInfo: { | 20260505 07:38:50.603 | INFO | "gpu" : " 17 GB", | 20260505 07:38:50.603 | INFO | "huge" : "", | 20260505 07:38:50.603 | INFO | "huge_shared" : " 27 GB", | 20260505 07:38:50.603 | INFO | "malloc" : " 14 GB", | 20260505 07:38:50.603 | INFO | "pinned" : " 29 GB" | 20260505 07:38:50.603 | INFO | } | 20260505 07:38:50.603 | INFO | Perf report for the last 5.13968 seconds: | 20260505 07:38:50.603 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:38:50.603 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:50.603 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:50.603 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:50.603 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 07:38:50.603 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:50.603 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:50.603 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 07:38:50.603 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:50.603 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:50.603 | INFO | Uploaded 36GiB to GPU in 2.18156s (16.502GiB/s) | 20260505 07:38:50.603 | INFO | Downloaded 34GiB to GPU in 2.04967s (16.588GiB/s) | 20260505 07:38:50.738 | INFO | Starting a round of Baz IO | 20260505 07:38:54.019 | INFO | Populating RTMetrics report | 20260505 07:38:54.019 | INFO | RTMetrics report populated | 20260505 07:38:54.019 | INFO | Opening tmp RTMetrics json file | 20260505 07:38:54.033 | INFO | Serializing RTMetrics to json | 20260505 07:38:54.036 | INFO | Serialization done, swapping tmp file now... | 20260505 07:38:54.038 | INFO | RTMetrics done | 20260505 07:38:54.152 | INFO | Finished a round of Baz IO: Wrote 2.435 GiB in 3.414 seconds | 20260505 07:38:54.152 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 29.36% / 0.03% / 70.61% | 20260505 07:38:54.152 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.39549% / 4.37101e-05% / 9.53674e-05% | 20260505 07:38:54.152 | INFO | Baz IO overall: | 20260505 07:38:54.152 | INFO | BytesWritten : 812399910809 | 20260505 07:38:54.152 | INFO | NumEvents : 385619416573 | 20260505 07:38:55.742 | INFO | Analyzing chunk frames = [1478656,1479168) | 20260505 07:38:55.742 | INFO | MemInfo: { | 20260505 07:38:55.742 | INFO | "gpu" : " 17 GB", | 20260505 07:38:55.742 | INFO | "huge" : "", | 20260505 07:38:55.742 | INFO | "huge_shared" : " 27 GB", | 20260505 07:38:55.742 | INFO | "malloc" : " 14 GB", | 20260505 07:38:55.742 | INFO | "pinned" : " 29 GB" | 20260505 07:38:55.742 | INFO | } | 20260505 07:38:55.742 | INFO | Perf report for the last 5.13966 seconds: | 20260505 07:38:55.742 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:38:55.742 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:55.742 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:55.742 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:55.742 | INFO | ANALYSIS : 65.3%, 2.9, 99.5ms, 0 | 20260505 07:38:55.742 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:38:55.742 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260505 07:38:55.742 | INFO | PRE_HQ : 63.9%, 1.9, 62.1ms, 0 | 20260505 07:38:55.742 | INFO | BAZWRITER : 65.5%, 4.2, 140.2ms, 0 | 20260505 07:38:55.742 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 07:38:55.743 | INFO | Uploaded 36GiB to GPU in 2.21195s (16.2753GiB/s) | 20260505 07:38:55.743 | INFO | Downloaded 38GiB to GPU in 2.34267s (16.2208GiB/s) | 20260505 07:39:00.881 | INFO | Analyzing chunk frames = [1479168,1479680) | 20260505 07:39:00.881 | INFO | MemInfo: { | 20260505 07:39:00.881 | INFO | "gpu" : " 17 GB", | 20260505 07:39:00.881 | INFO | "huge" : "", | 20260505 07:39:00.881 | INFO | "huge_shared" : " 27 GB", | 20260505 07:39:00.881 | INFO | "malloc" : " 14 GB", | 20260505 07:39:00.881 | INFO | "pinned" : " 29 GB" | 20260505 07:39:00.881 | INFO | } | 20260505 07:39:00.881 | INFO | Perf report for the last 5.13891 seconds: | 20260505 07:39:00.881 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:39:00.881 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:00.881 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:00.881 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:00.881 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 07:39:00.881 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:00.881 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:00.881 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:39:00.881 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:00.881 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:00.881 | INFO | Uploaded 36GiB to GPU in 2.18542s (16.4728GiB/s) | 20260505 07:39:00.882 | INFO | Downloaded 34GiB to GPU in 2.05393s (16.5537GiB/s) | 20260505 07:39:06.021 | INFO | Analyzing chunk frames = [1479680,1480192) | 20260505 07:39:06.021 | INFO | MemInfo: { | 20260505 07:39:06.021 | INFO | "gpu" : " 17 GB", | 20260505 07:39:06.021 | INFO | "huge" : "", | 20260505 07:39:06.021 | INFO | "huge_shared" : " 27 GB", | 20260505 07:39:06.021 | INFO | "malloc" : " 14 GB", | 20260505 07:39:06.021 | INFO | "pinned" : " 29 GB" | 20260505 07:39:06.021 | INFO | } | 20260505 07:39:06.021 | INFO | Perf report for the last 5.13993 seconds: | 20260505 07:39:06.021 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:39:06.021 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:06.021 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:06.021 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:06.021 | INFO | ANALYSIS : 63.4%, 3.0, 98.6ms, 0 | 20260505 07:39:06.021 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:06.021 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:06.021 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 07:39:06.021 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:06.021 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:06.021 | INFO | Uploaded 36GiB to GPU in 2.1556s (16.7007GiB/s) | 20260505 07:39:06.022 | INFO | Downloaded 34GiB to GPU in 2.0544s (16.5498GiB/s) | 20260505 07:39:11.160 | INFO | Analyzing chunk frames = [1480192,1480704) | 20260505 07:39:11.160 | INFO | MemInfo: { | 20260505 07:39:11.160 | INFO | "gpu" : " 17 GB", | 20260505 07:39:11.160 | INFO | "huge" : "", | 20260505 07:39:11.160 | INFO | "huge_shared" : " 27 GB", | 20260505 07:39:11.160 | INFO | "malloc" : " 14 GB", | 20260505 07:39:11.160 | INFO | "pinned" : " 29 GB" | 20260505 07:39:11.160 | INFO | } | 20260505 07:39:11.160 | INFO | Perf report for the last 5.13872 seconds: | 20260505 07:39:11.160 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:39:11.160 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:11.160 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:11.160 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:11.160 | INFO | ANALYSIS : 63.1%, 3.0, 98.1ms, 0 | 20260505 07:39:11.160 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:11.160 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:11.160 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:39:11.160 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:11.160 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:11.160 | INFO | Uploaded 36GiB to GPU in 2.15904s (16.6741GiB/s) | 20260505 07:39:11.160 | INFO | Downloaded 34GiB to GPU in 2.04494s (16.6264GiB/s) | 20260505 07:39:16.299 | INFO | Analyzing chunk frames = [1480704,1481216) | 20260505 07:39:16.299 | INFO | MemInfo: { | 20260505 07:39:16.299 | INFO | "gpu" : " 17 GB", | 20260505 07:39:16.299 | INFO | "huge" : "", | 20260505 07:39:16.299 | INFO | "huge_shared" : " 27 GB", | 20260505 07:39:16.299 | INFO | "malloc" : " 14 GB", | 20260505 07:39:16.299 | INFO | "pinned" : " 29 GB" | 20260505 07:39:16.299 | INFO | } | 20260505 07:39:16.299 | INFO | Perf report for the last 5.13944 seconds: | 20260505 07:39:16.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:39:16.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:16.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:16.300 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:16.300 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 07:39:16.300 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:16.300 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:16.300 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 07:39:16.300 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:16.300 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:16.300 | INFO | Uploaded 36GiB to GPU in 2.15218s (16.7272GiB/s) | 20260505 07:39:16.300 | INFO | Downloaded 34GiB to GPU in 2.05074s (16.5794GiB/s) | 20260505 07:39:21.440 | INFO | Analyzing chunk frames = [1481216,1481728) | 20260505 07:39:21.440 | INFO | MemInfo: { | 20260505 07:39:21.440 | INFO | "gpu" : " 17 GB", | 20260505 07:39:21.440 | INFO | "huge" : "", | 20260505 07:39:21.440 | INFO | "huge_shared" : " 27 GB", | 20260505 07:39:21.440 | INFO | "malloc" : " 14 GB", | 20260505 07:39:21.440 | INFO | "pinned" : " 29 GB" | 20260505 07:39:21.440 | INFO | } | 20260505 07:39:21.440 | INFO | Perf report for the last 5.14074 seconds: | 20260505 07:39:21.440 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:39:21.440 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:21.440 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:21.440 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:21.440 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 07:39:21.440 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:21.440 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:21.440 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:39:21.440 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:21.440 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:21.440 | INFO | Uploaded 36GiB to GPU in 2.19382s (16.4097GiB/s) | 20260505 07:39:21.440 | INFO | Downloaded 34GiB to GPU in 2.04921s (16.5917GiB/s) | 20260505 07:39:26.578 | INFO | Analyzing chunk frames = [1481728,1482240) | 20260505 07:39:26.578 | INFO | MemInfo: { | 20260505 07:39:26.578 | INFO | "gpu" : " 17 GB", | 20260505 07:39:26.578 | INFO | "huge" : "", | 20260505 07:39:26.578 | INFO | "huge_shared" : " 27 GB", | 20260505 07:39:26.578 | INFO | "malloc" : " 14 GB", | 20260505 07:39:26.578 | INFO | "pinned" : " 29 GB" | 20260505 07:39:26.578 | INFO | } | 20260505 07:39:26.578 | INFO | Perf report for the last 5.13785 seconds: | 20260505 07:39:26.578 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:39:26.578 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:26.578 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:26.578 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:26.578 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 07:39:26.578 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:26.578 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:26.578 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260505 07:39:26.578 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:26.578 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:26.578 | INFO | Uploaded 36GiB to GPU in 2.16878s (16.5992GiB/s) | 20260505 07:39:26.578 | INFO | Downloaded 34GiB to GPU in 2.14112s (15.8795GiB/s) | 20260505 07:39:31.717 | INFO | Analyzing chunk frames = [1482240,1482752) | 20260505 07:39:31.717 | INFO | MemInfo: { | 20260505 07:39:31.717 | INFO | "gpu" : " 17 GB", | 20260505 07:39:31.717 | INFO | "huge" : "", | 20260505 07:39:31.717 | INFO | "huge_shared" : " 27 GB", | 20260505 07:39:31.717 | INFO | "malloc" : " 14 GB", | 20260505 07:39:31.717 | INFO | "pinned" : " 29 GB" | 20260505 07:39:31.717 | INFO | } | 20260505 07:39:31.717 | INFO | Perf report for the last 5.13934 seconds: | 20260505 07:39:31.717 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:39:31.717 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:31.717 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:31.717 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:31.717 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 07:39:31.717 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:31.718 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:31.718 | INFO | PRE_HQ : 30.5%, 1.0, 15.9ms, 0 | 20260505 07:39:31.718 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:31.718 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:31.718 | INFO | Uploaded 36GiB to GPU in 2.16211s (16.6504GiB/s) | 20260505 07:39:31.718 | INFO | Downloaded 34GiB to GPU in 2.04909s (16.5928GiB/s) | 20260505 07:39:31.718 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 07:39:31.718 | INFO | Total mem usage: 15.3 GiB | 20260505 07:39:31.718 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 07:39:31.718 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.09%) | 20260505 07:39:31.718 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 07:39:31.718 | INFO | High Water Breakdown: | 20260505 07:39:31.718 | INFO | 60.7% : 8.77 GiB ( 93.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 07:39:31.718 | INFO | 39.1% : 5.65 GiB ( 85.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 07:39:31.718 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 07:39:31.718 | INFO | ------------------------------------------ | 20260505 07:39:31.718 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:39:31.718 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 07:39:31.718 | INFO | Total mem usage: 26.5 GiB | 20260505 07:39:31.718 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 07:39:31.718 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 07:39:31.718 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 07:39:31.718 | INFO | High Water Breakdown: | 20260505 07:39:31.718 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:39:31.718 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:39:31.718 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:39:31.718 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:39:31.718 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:39:31.718 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:39:31.718 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 07:39:31.718 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 07:39:31.718 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:39:31.718 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 07:39:31.718 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 07:39:31.718 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 07:39:31.718 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 07:39:31.718 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 07:39:31.718 | INFO | ------------------------------------------ | 20260505 07:39:31.718 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:39:31.718 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 07:39:31.718 | INFO | Total mem usage: 25.6 GiB | 20260505 07:39:31.718 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 07:39:31.718 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 07:39:31.718 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 07:39:31.718 | INFO | High Water Breakdown: | 20260505 07:39:31.718 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 07:39:31.718 | INFO | ------------------------------------------ | 20260505 07:39:31.718 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:39:31.718 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 07:39:31.718 | INFO | Total mem usage: 15.7 GiB | 20260505 07:39:31.718 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 07:39:31.718 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 07:39:31.718 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 07:39:31.718 | INFO | High Water Breakdown: | 20260505 07:39:31.718 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 07:39:31.718 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 07:39:31.718 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 07:39:31.718 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 07:39:31.718 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 07:39:31.718 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 07:39:31.718 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 07:39:31.718 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 07:39:31.718 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 07:39:31.718 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:39:31.718 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 07:39:31.718 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:39:31.718 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 07:39:31.718 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 07:39:31.718 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 07:39:31.718 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 07:39:31.718 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:39:31.718 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 07:39:31.719 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:39:31.719 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:39:31.719 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:39:31.719 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 07:39:31.719 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 07:39:31.719 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:39:31.719 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 07:39:31.719 | INFO | ------------------------------------------ | 20260505 07:39:31.719 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:39:31.719 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 07:39:31.719 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:39:31.719 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:39:31.719 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.48e+01, 1.37e+01 / 1.96e+00 | 20260505 07:39:31.719 | INFO | PerfReport: 0.29%, AggregateStats -- 9.35e-02 / 1.02e-01, 1.04e-02 / 5.72e-03 | 20260505 07:39:31.719 | INFO | PerfReport: 11.64%, Baseline -- 3.82e+00 / 4.41e+00, 6.25e-01 / 2.68e-01 | 20260505 07:39:31.719 | INFO | PerfReport: 3.30%, Histogram -- 1.08e+00 / 1.45e+00, 2.12e-01 / 1.27e-01 | 20260505 07:39:31.719 | INFO | PerfReport: 15.49%, DME -- 4.06e+01 / 4.43e+01, 4.05e+00 / 1.36e+00 | 20260505 07:39:31.719 | INFO | PerfReport: 56.25%, FrameLabeling -- 1.84e+01 / 1.91e+01, 1.98e+00 / 1.34e+00 | 20260505 07:39:31.719 | INFO | PerfReport: 7.63%, PulseAccumulating -- 2.50e+00 / 3.19e+00, 4.46e-01 / 1.65e-01 | 20260505 07:39:31.719 | INFO | PerfReport: 3.97%, Metrics -- 1.30e+00 / 1.69e+00, 3.63e-01 / 1.34e-01 | 20260505 07:39:31.719 | INFO | PerfReport: 1.43%, PulseSerializing -- 4.70e-01 / 5.39e-01, 9.03e-02 / 2.82e-02 | 20260505 07:39:31.719 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 07:39:31.719 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:39:31.719 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:39:31.719 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.26e+01 / 0.00e+00 | 20260505 07:39:31.719 | INFO | PerfReport: 22.34%, Upload -- 2.27e+01 / 0.00e+00, 8.01e+00 / 0.00e+00 | 20260505 07:39:31.719 | INFO | PerfReport: 21.55%, Download -- 2.19e+01 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260505 07:39:31.719 | INFO | PerfReport: 32.33%, Compute -- 3.28e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 07:39:31.719 | INFO | PerfReport: 1.61%, LockUpload -- 1.63e+00 / 0.00e+00, 7.22e+00 / 0.00e+00 | 20260505 07:39:31.719 | INFO | PerfReport: 0.90%, LockDownload -- 9.15e-01 / 0.00e+00, 2.36e+00 / 0.00e+00 | 20260505 07:39:31.719 | INFO | PerfReport: 21.26%, LockCompute -- 2.16e+01 / 0.00e+00, 1.72e+01 / 0.00e+00 | 20260505 07:39:31.719 | INFO | PerfReport: 0.01%, Misc -- 1.46e-02 / 0.00e+00, 1.97e-02 / 0.00e+00 | 20260505 07:39:31.851 | INFO | Starting a round of Baz IO | 20260505 07:39:35.123 | INFO | Populating RTMetrics report | 20260505 07:39:35.123 | INFO | RTMetrics report populated | 20260505 07:39:35.123 | INFO | Opening tmp RTMetrics json file | 20260505 07:39:35.165 | INFO | Serializing RTMetrics to json | 20260505 07:39:35.189 | INFO | Serialization done, swapping tmp file now... | 20260505 07:39:35.191 | INFO | RTMetrics done | 20260505 07:39:35.271 | INFO | Finished a round of Baz IO: Wrote 2.434 GiB in 3.419 seconds | 20260505 07:39:35.271 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 29.33% / 0.03% / 70.64% | 20260505 07:39:35.271 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.38145% / 3.17891e-05% / 5.16574e-05% | 20260505 07:39:35.271 | INFO | Baz IO overall: | 20260505 07:39:35.271 | INFO | BytesWritten : 815013908351 | 20260505 07:39:35.271 | INFO | NumEvents : 386869908097 | 20260505 07:39:36.857 | INFO | Analyzing chunk frames = [1482752,1483264) | 20260505 07:39:36.857 | INFO | MemInfo: { | 20260505 07:39:36.857 | INFO | "gpu" : " 17 GB", | 20260505 07:39:36.857 | INFO | "huge" : "", | 20260505 07:39:36.857 | INFO | "huge_shared" : " 27 GB", | 20260505 07:39:36.857 | INFO | "malloc" : " 14 GB", | 20260505 07:39:36.857 | INFO | "pinned" : " 29 GB" | 20260505 07:39:36.857 | INFO | } | 20260505 07:39:36.857 | INFO | Perf report for the last 5.13978 seconds: | 20260505 07:39:36.857 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:39:36.857 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:36.857 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:36.857 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:36.857 | INFO | ANALYSIS : 65.1%, 2.9, 99.0ms, 0 | 20260505 07:39:36.857 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:36.857 | INFO | RT_METRICS : 47.2%, 1.0, 24.8ms, 0 | 20260505 07:39:36.857 | INFO | PRE_HQ : 63.8%, 1.9, 62.4ms, 0 | 20260505 07:39:36.857 | INFO | BAZWRITER : 65.8%, 4.1, 140.2ms, 0 | 20260505 07:39:36.857 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260505 07:39:36.858 | INFO | Uploaded 36GiB to GPU in 2.21948s (16.2201GiB/s) | 20260505 07:39:36.858 | INFO | Downloaded 38GiB to GPU in 2.32186s (16.3662GiB/s) | 20260505 07:39:41.996 | INFO | Analyzing chunk frames = [1483264,1483776) | 20260505 07:39:41.996 | INFO | MemInfo: { | 20260505 07:39:41.996 | INFO | "gpu" : " 17 GB", | 20260505 07:39:41.996 | INFO | "huge" : "", | 20260505 07:39:41.996 | INFO | "huge_shared" : " 27 GB", | 20260505 07:39:41.996 | INFO | "malloc" : " 14 GB", | 20260505 07:39:41.996 | INFO | "pinned" : " 29 GB" | 20260505 07:39:41.996 | INFO | } | 20260505 07:39:41.996 | INFO | Perf report for the last 5.139 seconds: | 20260505 07:39:41.996 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:39:41.996 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:41.996 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:41.996 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:41.996 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 07:39:41.996 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:41.996 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:41.996 | INFO | PRE_HQ : 30.0%, 1.0, 15.5ms, 0 | 20260505 07:39:41.996 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:41.996 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:41.996 | INFO | Uploaded 36GiB to GPU in 2.18051s (16.5099GiB/s) | 20260505 07:39:41.997 | INFO | Downloaded 34GiB to GPU in 2.0556s (16.5402GiB/s) | 20260505 07:39:47.135 | INFO | Analyzing chunk frames = [1483776,1484288) | 20260505 07:39:47.135 | INFO | MemInfo: { | 20260505 07:39:47.135 | INFO | "gpu" : " 17 GB", | 20260505 07:39:47.135 | INFO | "huge" : "", | 20260505 07:39:47.135 | INFO | "huge_shared" : " 27 GB", | 20260505 07:39:47.135 | INFO | "malloc" : " 14 GB", | 20260505 07:39:47.135 | INFO | "pinned" : " 29 GB" | 20260505 07:39:47.135 | INFO | } | 20260505 07:39:47.135 | INFO | Perf report for the last 5.13923 seconds: | 20260505 07:39:47.135 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:39:47.135 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:47.136 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:47.136 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:47.136 | INFO | ANALYSIS : 63.4%, 3.0, 98.6ms, 0 | 20260505 07:39:47.136 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:47.136 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:47.136 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:39:47.136 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:47.136 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:47.136 | INFO | Uploaded 36GiB to GPU in 2.14844s (16.7564GiB/s) | 20260505 07:39:47.136 | INFO | Downloaded 34GiB to GPU in 2.05805s (16.5205GiB/s) | 20260505 07:39:52.275 | INFO | Analyzing chunk frames = [1484288,1484800) | 20260505 07:39:52.275 | INFO | MemInfo: { | 20260505 07:39:52.275 | INFO | "gpu" : " 17 GB", | 20260505 07:39:52.275 | INFO | "huge" : "", | 20260505 07:39:52.275 | INFO | "huge_shared" : " 27 GB", | 20260505 07:39:52.275 | INFO | "malloc" : " 14 GB", | 20260505 07:39:52.275 | INFO | "pinned" : " 29 GB" | 20260505 07:39:52.275 | INFO | } | 20260505 07:39:52.275 | INFO | Perf report for the last 5.13953 seconds: | 20260505 07:39:52.275 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:39:52.275 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:52.275 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:52.275 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:52.275 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 07:39:52.275 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:52.275 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:52.275 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:39:52.275 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:52.275 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:52.275 | INFO | Uploaded 36GiB to GPU in 2.15319s (16.7194GiB/s) | 20260505 07:39:52.275 | INFO | Downloaded 34GiB to GPU in 2.04188s (16.6513GiB/s) | 20260505 07:39:57.414 | INFO | Analyzing chunk frames = [1484800,1485312) | 20260505 07:39:57.414 | INFO | MemInfo: { | 20260505 07:39:57.414 | INFO | "gpu" : " 17 GB", | 20260505 07:39:57.414 | INFO | "huge" : "", | 20260505 07:39:57.414 | INFO | "huge_shared" : " 27 GB", | 20260505 07:39:57.414 | INFO | "malloc" : " 14 GB", | 20260505 07:39:57.414 | INFO | "pinned" : " 29 GB" | 20260505 07:39:57.414 | INFO | } | 20260505 07:39:57.414 | INFO | Perf report for the last 5.13925 seconds: | 20260505 07:39:57.414 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:39:57.414 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:57.414 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:57.414 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:57.414 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 07:39:57.414 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:57.414 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:57.414 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:39:57.414 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:57.414 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:39:57.415 | INFO | Uploaded 36GiB to GPU in 2.14301s (16.7988GiB/s) | 20260505 07:39:57.415 | INFO | Downloaded 34GiB to GPU in 2.04676s (16.6116GiB/s) | 20260505 07:40:02.555 | INFO | Analyzing chunk frames = [1485312,1485824) | 20260505 07:40:02.555 | INFO | MemInfo: { | 20260505 07:40:02.555 | INFO | "gpu" : " 17 GB", | 20260505 07:40:02.555 | INFO | "huge" : "", | 20260505 07:40:02.555 | INFO | "huge_shared" : " 27 GB", | 20260505 07:40:02.555 | INFO | "malloc" : " 14 GB", | 20260505 07:40:02.555 | INFO | "pinned" : " 29 GB" | 20260505 07:40:02.555 | INFO | } | 20260505 07:40:02.555 | INFO | Perf report for the last 5.14116 seconds: | 20260505 07:40:02.555 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:40:02.555 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:02.556 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:02.556 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:02.556 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 07:40:02.556 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:02.556 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:02.556 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:40:02.556 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:02.556 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:02.556 | INFO | Uploaded 36GiB to GPU in 2.15618s (16.6962GiB/s) | 20260505 07:40:02.556 | INFO | Downloaded 34GiB to GPU in 2.06053s (16.5006GiB/s) | 20260505 07:40:07.693 | INFO | Analyzing chunk frames = [1485824,1486336) | 20260505 07:40:07.693 | INFO | MemInfo: { | 20260505 07:40:07.693 | INFO | "gpu" : " 17 GB", | 20260505 07:40:07.693 | INFO | "huge" : "", | 20260505 07:40:07.693 | INFO | "huge_shared" : " 27 GB", | 20260505 07:40:07.693 | INFO | "malloc" : " 14 GB", | 20260505 07:40:07.693 | INFO | "pinned" : " 29 GB" | 20260505 07:40:07.693 | INFO | } | 20260505 07:40:07.693 | INFO | Perf report for the last 5.13736 seconds: | 20260505 07:40:07.693 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:40:07.693 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:07.693 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:07.693 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:07.693 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 07:40:07.693 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:07.693 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:07.693 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 07:40:07.693 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:07.693 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:07.693 | INFO | Uploaded 36GiB to GPU in 2.1747s (16.554GiB/s) | 20260505 07:40:07.693 | INFO | Downloaded 34GiB to GPU in 2.1524s (15.7963GiB/s) | 20260505 07:40:12.832 | INFO | Analyzing chunk frames = [1486336,1486848) | 20260505 07:40:12.832 | INFO | MemInfo: { | 20260505 07:40:12.832 | INFO | "gpu" : " 17 GB", | 20260505 07:40:12.832 | INFO | "huge" : "", | 20260505 07:40:12.832 | INFO | "huge_shared" : " 27 GB", | 20260505 07:40:12.832 | INFO | "malloc" : " 14 GB", | 20260505 07:40:12.832 | INFO | "pinned" : " 29 GB" | 20260505 07:40:12.833 | INFO | } | 20260505 07:40:12.833 | INFO | Perf report for the last 5.13971 seconds: | 20260505 07:40:12.833 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:40:12.833 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:12.833 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:12.833 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:12.833 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 07:40:12.833 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:12.833 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:12.833 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 07:40:12.833 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:12.833 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:12.833 | INFO | Uploaded 36GiB to GPU in 2.17498s (16.5519GiB/s) | 20260505 07:40:12.833 | INFO | Downloaded 34GiB to GPU in 2.04929s (16.5911GiB/s) | 20260505 07:40:12.968 | INFO | Starting a round of Baz IO | 20260505 07:40:16.237 | INFO | Populating RTMetrics report | 20260505 07:40:16.238 | INFO | RTMetrics report populated | 20260505 07:40:16.238 | INFO | Opening tmp RTMetrics json file | 20260505 07:40:16.238 | INFO | Serializing RTMetrics to json | 20260505 07:40:16.240 | INFO | Serialization done, swapping tmp file now... | 20260505 07:40:16.240 | INFO | RTMetrics done | 20260505 07:40:16.424 | INFO | Finished a round of Baz IO: Wrote 2.437 GiB in 3.456 seconds | 20260505 07:40:16.424 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 29.31% / 0.03% / 70.67% | 20260505 07:40:16.424 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.37682% / 7.15256e-05% / 7.54992e-05% | 20260505 07:40:16.424 | INFO | Baz IO overall: | 20260505 07:40:16.424 | INFO | BytesWritten : 817630149797 | 20260505 07:40:16.424 | INFO | NumEvents : 388121689877 | 20260505 07:40:17.971 | INFO | Analyzing chunk frames = [1486848,1487360) | 20260505 07:40:17.971 | INFO | MemInfo: { | 20260505 07:40:17.971 | INFO | "gpu" : " 17 GB", | 20260505 07:40:17.971 | INFO | "huge" : "", | 20260505 07:40:17.971 | INFO | "huge_shared" : " 27 GB", | 20260505 07:40:17.971 | INFO | "malloc" : " 14 GB", | 20260505 07:40:17.971 | INFO | "pinned" : " 29 GB" | 20260505 07:40:17.972 | INFO | } | 20260505 07:40:17.972 | INFO | Perf report for the last 5.139 seconds: | 20260505 07:40:17.972 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:40:17.972 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:17.972 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:17.972 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:17.972 | INFO | ANALYSIS : 65.1%, 2.9, 99.2ms, 0 | 20260505 07:40:17.972 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:17.972 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 07:40:17.972 | INFO | PRE_HQ : 63.7%, 1.9, 62.3ms, 0 | 20260505 07:40:17.972 | INFO | BAZWRITER : 66.6%, 4.2, 145.1ms, 0 | 20260505 07:40:17.972 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 07:40:17.972 | INFO | Uploaded 36GiB to GPU in 2.21055s (16.2856GiB/s) | 20260505 07:40:17.972 | INFO | Downloaded 38GiB to GPU in 2.31431s (16.4196GiB/s) | 20260505 07:40:23.111 | INFO | Analyzing chunk frames = [1487360,1487872) | 20260505 07:40:23.111 | INFO | MemInfo: { | 20260505 07:40:23.111 | INFO | "gpu" : " 17 GB", | 20260505 07:40:23.111 | INFO | "huge" : "", | 20260505 07:40:23.111 | INFO | "huge_shared" : " 27 GB", | 20260505 07:40:23.111 | INFO | "malloc" : " 14 GB", | 20260505 07:40:23.111 | INFO | "pinned" : " 29 GB" | 20260505 07:40:23.111 | INFO | } | 20260505 07:40:23.111 | INFO | Perf report for the last 5.1393 seconds: | 20260505 07:40:23.111 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:40:23.111 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:23.111 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:23.111 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:23.111 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 07:40:23.111 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:23.111 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:23.111 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:40:23.111 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:23.111 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:23.111 | INFO | Uploaded 36GiB to GPU in 2.11606s (17.0127GiB/s) | 20260505 07:40:23.111 | INFO | Downloaded 34GiB to GPU in 2.04527s (16.6237GiB/s) | 20260505 07:40:28.250 | INFO | Analyzing chunk frames = [1487872,1488384) | 20260505 07:40:28.250 | INFO | MemInfo: { | 20260505 07:40:28.250 | INFO | "gpu" : " 17 GB", | 20260505 07:40:28.250 | INFO | "huge" : "", | 20260505 07:40:28.250 | INFO | "huge_shared" : " 27 GB", | 20260505 07:40:28.250 | INFO | "malloc" : " 14 GB", | 20260505 07:40:28.250 | INFO | "pinned" : " 29 GB" | 20260505 07:40:28.250 | INFO | } | 20260505 07:40:28.250 | INFO | Perf report for the last 5.13935 seconds: | 20260505 07:40:28.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:40:28.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:28.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:28.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:28.250 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 07:40:28.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:28.250 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:28.250 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:40:28.250 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:28.250 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:28.250 | INFO | Uploaded 36GiB to GPU in 2.15033s (16.7416GiB/s) | 20260505 07:40:28.250 | INFO | Downloaded 34GiB to GPU in 2.05862s (16.516GiB/s) | 20260505 07:40:33.389 | INFO | Analyzing chunk frames = [1488384,1488896) | 20260505 07:40:33.389 | INFO | MemInfo: { | 20260505 07:40:33.389 | INFO | "gpu" : " 17 GB", | 20260505 07:40:33.389 | INFO | "huge" : "", | 20260505 07:40:33.389 | INFO | "huge_shared" : " 27 GB", | 20260505 07:40:33.389 | INFO | "malloc" : " 14 GB", | 20260505 07:40:33.389 | INFO | "pinned" : " 29 GB" | 20260505 07:40:33.389 | INFO | } | 20260505 07:40:33.389 | INFO | Perf report for the last 5.13919 seconds: | 20260505 07:40:33.389 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:40:33.389 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:33.389 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:33.389 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:33.389 | INFO | ANALYSIS : 63.2%, 3.0, 98.1ms, 0 | 20260505 07:40:33.389 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:33.390 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:33.390 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 07:40:33.390 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:33.390 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:33.390 | INFO | Uploaded 36GiB to GPU in 2.12219s (16.9636GiB/s) | 20260505 07:40:33.390 | INFO | Downloaded 34GiB to GPU in 2.04124s (16.6566GiB/s) | 20260505 07:40:38.529 | INFO | Analyzing chunk frames = [1488896,1489408) | 20260505 07:40:38.529 | INFO | MemInfo: { | 20260505 07:40:38.529 | INFO | "gpu" : " 17 GB", | 20260505 07:40:38.529 | INFO | "huge" : "", | 20260505 07:40:38.529 | INFO | "huge_shared" : " 27 GB", | 20260505 07:40:38.529 | INFO | "malloc" : " 14 GB", | 20260505 07:40:38.529 | INFO | "pinned" : " 29 GB" | 20260505 07:40:38.529 | INFO | } | 20260505 07:40:38.529 | INFO | Perf report for the last 5.13928 seconds: | 20260505 07:40:38.529 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:40:38.529 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:38.529 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:38.529 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:38.529 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 07:40:38.529 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:38.529 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:38.529 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:40:38.529 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:38.529 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:38.529 | INFO | Uploaded 36GiB to GPU in 2.1997s (16.3658GiB/s) | 20260505 07:40:38.529 | INFO | Downloaded 34GiB to GPU in 2.05573s (16.5391GiB/s) | 20260505 07:40:43.670 | INFO | Analyzing chunk frames = [1489408,1489920) | 20260505 07:40:43.670 | INFO | MemInfo: { | 20260505 07:40:43.670 | INFO | "gpu" : " 17 GB", | 20260505 07:40:43.670 | INFO | "huge" : "", | 20260505 07:40:43.670 | INFO | "huge_shared" : " 27 GB", | 20260505 07:40:43.670 | INFO | "malloc" : " 14 GB", | 20260505 07:40:43.670 | INFO | "pinned" : " 29 GB" | 20260505 07:40:43.670 | INFO | } | 20260505 07:40:43.670 | INFO | Perf report for the last 5.14114 seconds: | 20260505 07:40:43.670 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:40:43.670 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:43.670 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:43.670 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:43.670 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 07:40:43.670 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:43.670 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:43.670 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:40:43.670 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:43.670 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:43.670 | INFO | Uploaded 36GiB to GPU in 2.15204s (16.7283GiB/s) | 20260505 07:40:43.670 | INFO | Downloaded 34GiB to GPU in 2.05702s (16.5288GiB/s) | 20260505 07:40:48.807 | INFO | Analyzing chunk frames = [1489920,1490432) | 20260505 07:40:48.807 | INFO | MemInfo: { | 20260505 07:40:48.807 | INFO | "gpu" : " 17 GB", | 20260505 07:40:48.807 | INFO | "huge" : "", | 20260505 07:40:48.807 | INFO | "huge_shared" : " 27 GB", | 20260505 07:40:48.807 | INFO | "malloc" : " 14 GB", | 20260505 07:40:48.807 | INFO | "pinned" : " 29 GB" | 20260505 07:40:48.807 | INFO | } | 20260505 07:40:48.808 | INFO | Perf report for the last 5.13763 seconds: | 20260505 07:40:48.808 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:40:48.808 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:48.808 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:48.808 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:48.808 | INFO | ANALYSIS : 64.1%, 3.0, 100.0ms, 0 | 20260505 07:40:48.808 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:48.808 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:48.808 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 07:40:48.808 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:48.808 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:48.808 | INFO | Uploaded 36GiB to GPU in 2.19617s (16.3921GiB/s) | 20260505 07:40:48.808 | INFO | Downloaded 34GiB to GPU in 2.15396s (15.7848GiB/s) | 20260505 07:40:53.947 | INFO | Analyzing chunk frames = [1490432,1490944) | 20260505 07:40:53.947 | INFO | MemInfo: { | 20260505 07:40:53.947 | INFO | "gpu" : " 17 GB", | 20260505 07:40:53.947 | INFO | "huge" : "", | 20260505 07:40:53.947 | INFO | "huge_shared" : " 27 GB", | 20260505 07:40:53.947 | INFO | "malloc" : " 14 GB", | 20260505 07:40:53.947 | INFO | "pinned" : " 29 GB" | 20260505 07:40:53.947 | INFO | } | 20260505 07:40:53.947 | INFO | Perf report for the last 5.13928 seconds: | 20260505 07:40:53.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:40:53.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:53.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:53.947 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:53.947 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 07:40:53.947 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:53.947 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:53.947 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:40:53.947 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:53.947 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:53.947 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 07:40:53.947 | INFO | Total mem usage: 15.3 GiB | 20260505 07:40:53.947 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 07:40:53.947 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -5.65%) | 20260505 07:40:53.947 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 07:40:53.947 | INFO | Uploaded 36GiB to GPU in 2.15994s (16.6672GiB/s) | 20260505 07:40:53.947 | INFO | High Water Breakdown: | 20260505 07:40:53.947 | INFO | 60.9% : 8.83 GiB ( 94.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 07:40:53.947 | INFO | 39% : 5.66 GiB ( 85.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 07:40:53.947 | INFO | Downloaded 34GiB to GPU in 2.05145s (16.5736GiB/s) | 20260505 07:40:53.947 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 07:40:53.947 | INFO | ------------------------------------------ | 20260505 07:40:53.947 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:40:53.947 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 07:40:53.947 | INFO | Total mem usage: 26.5 GiB | 20260505 07:40:53.947 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 07:40:53.947 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 07:40:53.947 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 07:40:53.947 | INFO | High Water Breakdown: | 20260505 07:40:53.947 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:40:53.947 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:40:53.947 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:40:53.947 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:40:53.947 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:40:53.947 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:40:53.947 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 07:40:53.947 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 07:40:53.947 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:40:53.947 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 07:40:53.947 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 07:40:53.947 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 07:40:53.947 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 07:40:53.947 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 07:40:53.947 | INFO | ------------------------------------------ | 20260505 07:40:53.948 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:40:53.948 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 07:40:53.948 | INFO | Total mem usage: 25.6 GiB | 20260505 07:40:53.948 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 07:40:53.948 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 07:40:53.948 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 07:40:53.948 | INFO | High Water Breakdown: | 20260505 07:40:53.948 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 07:40:53.948 | INFO | ------------------------------------------ | 20260505 07:40:53.948 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:40:53.948 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 07:40:53.948 | INFO | Total mem usage: 15.7 GiB | 20260505 07:40:53.948 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 07:40:53.948 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 07:40:53.948 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 07:40:53.948 | INFO | High Water Breakdown: | 20260505 07:40:53.948 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 07:40:53.948 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 07:40:53.948 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 07:40:53.948 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 07:40:53.948 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 07:40:53.948 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 07:40:53.948 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 07:40:53.948 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 07:40:53.948 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 07:40:53.948 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:40:53.948 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 07:40:53.948 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:40:53.948 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 07:40:53.948 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 07:40:53.948 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 07:40:53.948 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 07:40:53.948 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:40:53.948 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 07:40:53.948 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:40:53.948 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:40:53.948 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:40:53.948 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 07:40:53.948 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 07:40:53.948 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:40:53.948 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 07:40:53.948 | INFO | ------------------------------------------ | 20260505 07:40:53.948 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:40:53.948 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 07:40:53.948 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:40:53.948 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:40:53.948 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.50e+01, 1.34e+01 / 1.97e+00 | 20260505 07:40:53.948 | INFO | PerfReport: 0.29%, AggregateStats -- 9.36e-02 / 1.02e-01, 1.05e-02 / 8.29e-03 | 20260505 07:40:53.948 | INFO | PerfReport: 11.63%, Baseline -- 3.82e+00 / 4.41e+00, 6.27e-01 / 3.38e-01 | 20260505 07:40:53.948 | INFO | PerfReport: 3.31%, Histogram -- 1.08e+00 / 1.46e+00, 2.11e-01 / 1.30e-01 | 20260505 07:40:53.948 | INFO | PerfReport: 15.46%, DME -- 4.06e+01 / 4.43e+01, 4.13e+00 / 1.31e+00 | 20260505 07:40:53.948 | INFO | PerfReport: 56.30%, FrameLabeling -- 1.85e+01 / 1.93e+01, 2.00e+00 / 1.54e+00 | 20260505 07:40:53.948 | INFO | PerfReport: 7.61%, PulseAccumulating -- 2.50e+00 / 3.13e+00, 4.44e-01 / 2.27e-01 | 20260505 07:40:53.948 | INFO | PerfReport: 3.96%, Metrics -- 1.30e+00 / 1.70e+00, 3.62e-01 / 1.44e-01 | 20260505 07:40:53.948 | INFO | PerfReport: 1.43%, PulseSerializing -- 4.71e-01 / 5.46e-01, 9.07e-02 / 3.45e-02 | 20260505 07:40:53.948 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 07:40:53.948 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:40:53.948 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:40:53.948 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.21e+01 / 0.00e+00 | 20260505 07:40:53.948 | INFO | PerfReport: 22.28%, Upload -- 2.26e+01 / 0.00e+00, 7.92e+00 / 0.00e+00 | 20260505 07:40:53.948 | INFO | PerfReport: 21.58%, Download -- 2.19e+01 / 0.00e+00, 7.93e+00 / 0.00e+00 | 20260505 07:40:53.948 | INFO | PerfReport: 32.40%, Compute -- 3.28e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 07:40:53.948 | INFO | PerfReport: 1.57%, LockUpload -- 1.59e+00 / 0.00e+00, 7.21e+00 / 0.00e+00 | 20260505 07:40:53.948 | INFO | PerfReport: 0.88%, LockDownload -- 8.93e-01 / 0.00e+00, 2.31e+00 / 0.00e+00 | 20260505 07:40:53.948 | INFO | PerfReport: 21.27%, LockCompute -- 2.15e+01 / 0.00e+00, 1.68e+01 / 0.00e+00 | 20260505 07:40:53.948 | INFO | PerfReport: 0.01%, Misc -- 1.47e-02 / 0.00e+00, 2.07e-02 / 0.00e+00 | 20260505 07:40:54.081 | INFO | Starting a round of Baz IO | 20260505 07:40:57.370 | INFO | Populating RTMetrics report | 20260505 07:40:57.370 | INFO | RTMetrics report populated | 20260505 07:40:57.370 | INFO | Opening tmp RTMetrics json file | 20260505 07:40:57.414 | INFO | Serializing RTMetrics to json | 20260505 07:40:57.416 | INFO | Serialization done, swapping tmp file now... | 20260505 07:40:57.416 | INFO | RTMetrics done | 20260505 07:40:57.526 | INFO | Finished a round of Baz IO: Wrote 2.436 GiB in 3.445 seconds | 20260505 07:40:57.526 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 29.28% / 0.03% / 70.69% | 20260505 07:40:57.526 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.3621% / 3.57628e-05% / 8.74201e-05% | 20260505 07:40:57.526 | INFO | Baz IO overall: | 20260505 07:40:57.526 | INFO | BytesWritten : 820246223717 | 20260505 07:40:57.526 | INFO | NumEvents : 389373411091 | 20260505 07:40:59.086 | INFO | Analyzing chunk frames = [1490944,1491456) | 20260505 07:40:59.086 | INFO | MemInfo: { | 20260505 07:40:59.086 | INFO | "gpu" : " 17 GB", | 20260505 07:40:59.086 | INFO | "huge" : "", | 20260505 07:40:59.086 | INFO | "huge_shared" : " 27 GB", | 20260505 07:40:59.086 | INFO | "malloc" : " 14 GB", | 20260505 07:40:59.086 | INFO | "pinned" : " 29 GB" | 20260505 07:40:59.086 | INFO | } | 20260505 07:40:59.086 | INFO | Perf report for the last 5.13966 seconds: | 20260505 07:40:59.086 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:40:59.086 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:59.087 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:59.087 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:59.087 | INFO | ANALYSIS : 65.4%, 2.9, 99.6ms, 0 | 20260505 07:40:59.087 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:40:59.087 | INFO | RT_METRICS : 47.0%, 1.0, 24.6ms, 0 | 20260505 07:40:59.087 | INFO | PRE_HQ : 64.0%, 1.9, 62.4ms, 0 | 20260505 07:40:59.087 | INFO | BAZWRITER : 66.2%, 4.1, 141.1ms, 0 | 20260505 07:40:59.087 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260505 07:40:59.087 | INFO | Uploaded 36GiB to GPU in 2.22235s (16.1991GiB/s) | 20260505 07:40:59.087 | INFO | Downloaded 38GiB to GPU in 2.34733s (16.1886GiB/s) | 20260505 07:41:04.225 | INFO | Analyzing chunk frames = [1491456,1491968) | 20260505 07:41:04.225 | INFO | MemInfo: { | 20260505 07:41:04.225 | INFO | "gpu" : " 17 GB", | 20260505 07:41:04.225 | INFO | "huge" : "", | 20260505 07:41:04.225 | INFO | "huge_shared" : " 27 GB", | 20260505 07:41:04.225 | INFO | "malloc" : " 14 GB", | 20260505 07:41:04.225 | INFO | "pinned" : " 29 GB" | 20260505 07:41:04.225 | INFO | } | 20260505 07:41:04.225 | INFO | Perf report for the last 5.13892 seconds: | 20260505 07:41:04.225 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:41:04.225 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:04.225 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:04.225 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:04.225 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 07:41:04.225 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:04.225 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:04.225 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:41:04.226 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:04.226 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:04.226 | INFO | Uploaded 36GiB to GPU in 2.18015s (16.5126GiB/s) | 20260505 07:41:04.226 | INFO | Downloaded 34GiB to GPU in 2.05588s (16.538GiB/s) | 20260505 07:41:09.365 | INFO | Analyzing chunk frames = [1491968,1492480) | 20260505 07:41:09.365 | INFO | MemInfo: { | 20260505 07:41:09.365 | INFO | "gpu" : " 17 GB", | 20260505 07:41:09.365 | INFO | "huge" : "", | 20260505 07:41:09.365 | INFO | "huge_shared" : " 27 GB", | 20260505 07:41:09.365 | INFO | "malloc" : " 14 GB", | 20260505 07:41:09.365 | INFO | "pinned" : " 29 GB" | 20260505 07:41:09.365 | INFO | } | 20260505 07:41:09.365 | INFO | Perf report for the last 5.13941 seconds: | 20260505 07:41:09.365 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:41:09.365 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:09.365 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:09.365 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:09.365 | INFO | ANALYSIS : 63.2%, 3.0, 98.3ms, 0 | 20260505 07:41:09.365 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:09.365 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:09.365 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:41:09.365 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:09.365 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:09.365 | INFO | Uploaded 36GiB to GPU in 2.14706s (16.7671GiB/s) | 20260505 07:41:09.365 | INFO | Downloaded 34GiB to GPU in 2.05942s (16.5095GiB/s) | 20260505 07:41:14.504 | INFO | Analyzing chunk frames = [1492480,1492992) | 20260505 07:41:14.504 | INFO | MemInfo: { | 20260505 07:41:14.504 | INFO | "gpu" : " 17 GB", | 20260505 07:41:14.504 | INFO | "huge" : "", | 20260505 07:41:14.504 | INFO | "huge_shared" : " 27 GB", | 20260505 07:41:14.504 | INFO | "malloc" : " 14 GB", | 20260505 07:41:14.504 | INFO | "pinned" : " 29 GB" | 20260505 07:41:14.504 | INFO | } | 20260505 07:41:14.505 | INFO | Perf report for the last 5.1397 seconds: | 20260505 07:41:14.505 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:41:14.505 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:14.505 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:14.505 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:14.505 | INFO | ANALYSIS : 63.2%, 3.0, 98.1ms, 0 | 20260505 07:41:14.505 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:14.505 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:14.505 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:41:14.505 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:14.505 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:14.505 | INFO | Uploaded 36GiB to GPU in 2.13261s (16.8807GiB/s) | 20260505 07:41:14.505 | INFO | Downloaded 34GiB to GPU in 2.03659s (16.6945GiB/s) | 20260505 07:41:19.644 | INFO | Analyzing chunk frames = [1492992,1493504) | 20260505 07:41:19.644 | INFO | MemInfo: { | 20260505 07:41:19.644 | INFO | "gpu" : " 17 GB", | 20260505 07:41:19.644 | INFO | "huge" : "", | 20260505 07:41:19.644 | INFO | "huge_shared" : " 27 GB", | 20260505 07:41:19.644 | INFO | "malloc" : " 14 GB", | 20260505 07:41:19.644 | INFO | "pinned" : " 29 GB" | 20260505 07:41:19.644 | INFO | } | 20260505 07:41:19.644 | INFO | Perf report for the last 5.13934 seconds: | 20260505 07:41:19.644 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:41:19.644 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:19.644 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:19.644 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:19.644 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 07:41:19.644 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:19.644 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:19.644 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:41:19.644 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:19.644 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:19.644 | INFO | Uploaded 36GiB to GPU in 2.15593s (16.6981GiB/s) | 20260505 07:41:19.644 | INFO | Downloaded 34GiB to GPU in 2.04838s (16.5985GiB/s) | 20260505 07:41:24.785 | INFO | Analyzing chunk frames = [1493504,1494016) | 20260505 07:41:24.785 | INFO | MemInfo: { | 20260505 07:41:24.785 | INFO | "gpu" : " 17 GB", | 20260505 07:41:24.785 | INFO | "huge" : "", | 20260505 07:41:24.785 | INFO | "huge_shared" : " 27 GB", | 20260505 07:41:24.785 | INFO | "malloc" : " 14 GB", | 20260505 07:41:24.785 | INFO | "pinned" : " 29 GB" | 20260505 07:41:24.785 | INFO | } | 20260505 07:41:24.785 | INFO | Perf report for the last 5.14095 seconds: | 20260505 07:41:24.785 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:41:24.785 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:24.785 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:24.785 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:24.785 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 07:41:24.785 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:24.785 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:24.785 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:41:24.785 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:24.785 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:24.785 | INFO | Uploaded 36GiB to GPU in 2.13505s (16.8614GiB/s) | 20260505 07:41:24.785 | INFO | Downloaded 34GiB to GPU in 2.05362s (16.5561GiB/s) | 20260505 07:41:29.922 | INFO | Analyzing chunk frames = [1494016,1494528) | 20260505 07:41:29.922 | INFO | MemInfo: { | 20260505 07:41:29.922 | INFO | "gpu" : " 17 GB", | 20260505 07:41:29.922 | INFO | "huge" : "", | 20260505 07:41:29.922 | INFO | "huge_shared" : " 27 GB", | 20260505 07:41:29.922 | INFO | "malloc" : " 14 GB", | 20260505 07:41:29.922 | INFO | "pinned" : " 29 GB" | 20260505 07:41:29.922 | INFO | } | 20260505 07:41:29.923 | INFO | Perf report for the last 5.13763 seconds: | 20260505 07:41:29.923 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:41:29.923 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:29.923 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:29.923 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:29.923 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 07:41:29.923 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:29.923 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:29.923 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260505 07:41:29.923 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:29.923 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:29.923 | INFO | Uploaded 36GiB to GPU in 2.1461s (16.7746GiB/s) | 20260505 07:41:29.923 | INFO | Downloaded 34GiB to GPU in 2.13133s (15.9524GiB/s) | 20260505 07:41:35.061 | INFO | Analyzing chunk frames = [1494528,1495040) | 20260505 07:41:35.061 | INFO | MemInfo: { | 20260505 07:41:35.061 | INFO | "gpu" : " 17 GB", | 20260505 07:41:35.061 | INFO | "huge" : "", | 20260505 07:41:35.062 | INFO | "huge_shared" : " 27 GB", | 20260505 07:41:35.062 | INFO | "malloc" : " 14 GB", | 20260505 07:41:35.062 | INFO | "pinned" : " 29 GB" | 20260505 07:41:35.062 | INFO | } | 20260505 07:41:35.062 | INFO | Perf report for the last 5.13913 seconds: | 20260505 07:41:35.062 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:41:35.062 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:35.062 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:35.062 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:35.062 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 07:41:35.062 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:35.062 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:35.062 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 07:41:35.062 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:35.062 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:35.062 | INFO | Uploaded 36GiB to GPU in 2.15656s (16.6932GiB/s) | 20260505 07:41:35.062 | INFO | Downloaded 34GiB to GPU in 2.04948s (16.5895GiB/s) | 20260505 07:41:35.195 | INFO | Starting a round of Baz IO | 20260505 07:41:38.484 | INFO | Populating RTMetrics report | 20260505 07:41:38.484 | INFO | RTMetrics report populated | 20260505 07:41:38.484 | INFO | Opening tmp RTMetrics json file | 20260505 07:41:38.485 | INFO | Serializing RTMetrics to json | 20260505 07:41:38.486 | INFO | Serialization done, swapping tmp file now... | 20260505 07:41:38.486 | INFO | RTMetrics done | 20260505 07:41:38.627 | INFO | Finished a round of Baz IO: Wrote 2.438 GiB in 3.432 seconds | 20260505 07:41:38.627 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 29.25% / 0.03% / 70.72% | 20260505 07:41:38.627 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.35686% / 5.16574e-05% / 5.96046e-05% | 20260505 07:41:38.627 | INFO | Baz IO overall: | 20260505 07:41:38.627 | INFO | BytesWritten : 822864489330 | 20260505 07:41:38.627 | INFO | NumEvents : 390626727580 | 20260505 07:41:40.201 | INFO | Analyzing chunk frames = [1495040,1495552) | 20260505 07:41:40.201 | INFO | MemInfo: { | 20260505 07:41:40.201 | INFO | "gpu" : " 17 GB", | 20260505 07:41:40.201 | INFO | "huge" : "", | 20260505 07:41:40.201 | INFO | "huge_shared" : " 27 GB", | 20260505 07:41:40.201 | INFO | "malloc" : " 14 GB", | 20260505 07:41:40.201 | INFO | "pinned" : " 29 GB" | 20260505 07:41:40.201 | INFO | } | 20260505 07:41:40.201 | INFO | Perf report for the last 5.13944 seconds: | 20260505 07:41:40.201 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:41:40.201 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:40.201 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:40.201 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:40.201 | INFO | ANALYSIS : 65.4%, 2.9, 99.2ms, 0 | 20260505 07:41:40.201 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:40.201 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260505 07:41:40.201 | INFO | PRE_HQ : 64.0%, 1.9, 62.3ms, 0 | 20260505 07:41:40.201 | INFO | BAZWRITER : 66.0%, 4.2, 143.1ms, 0 | 20260505 07:41:40.201 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 07:41:40.201 | INFO | Uploaded 36GiB to GPU in 2.23421s (16.1131GiB/s) | 20260505 07:41:40.202 | INFO | Downloaded 38GiB to GPU in 2.35197s (16.1567GiB/s) | 20260505 07:41:45.340 | INFO | Analyzing chunk frames = [1495552,1496064) | 20260505 07:41:45.340 | INFO | MemInfo: { | 20260505 07:41:45.340 | INFO | "gpu" : " 17 GB", | 20260505 07:41:45.340 | INFO | "huge" : "", | 20260505 07:41:45.340 | INFO | "huge_shared" : " 27 GB", | 20260505 07:41:45.340 | INFO | "malloc" : " 14 GB", | 20260505 07:41:45.340 | INFO | "pinned" : " 29 GB" | 20260505 07:41:45.340 | INFO | } | 20260505 07:41:45.340 | INFO | Perf report for the last 5.1392 seconds: | 20260505 07:41:45.340 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:41:45.340 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:45.340 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:45.340 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:45.340 | INFO | ANALYSIS : 64.4%, 3.0, 100.6ms, 0 | 20260505 07:41:45.340 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:45.340 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:45.340 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:41:45.340 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:45.340 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:45.341 | INFO | Uploaded 36GiB to GPU in 2.18056s (16.5095GiB/s) | 20260505 07:41:45.341 | INFO | Downloaded 34GiB to GPU in 2.05634s (16.5343GiB/s) | 20260505 07:41:50.480 | INFO | Analyzing chunk frames = [1496064,1496576) | 20260505 07:41:50.480 | INFO | MemInfo: { | 20260505 07:41:50.480 | INFO | "gpu" : " 17 GB", | 20260505 07:41:50.480 | INFO | "huge" : "", | 20260505 07:41:50.480 | INFO | "huge_shared" : " 27 GB", | 20260505 07:41:50.480 | INFO | "malloc" : " 14 GB", | 20260505 07:41:50.480 | INFO | "pinned" : " 29 GB" | 20260505 07:41:50.480 | INFO | } | 20260505 07:41:50.480 | INFO | Perf report for the last 5.13934 seconds: | 20260505 07:41:50.480 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:41:50.480 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:50.480 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:50.480 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:50.480 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 07:41:50.480 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:50.480 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:50.480 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:41:50.480 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:50.480 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:50.480 | INFO | Uploaded 36GiB to GPU in 2.15571s (16.6998GiB/s) | 20260505 07:41:50.480 | INFO | Downloaded 34GiB to GPU in 2.06057s (16.5003GiB/s) | 20260505 07:41:55.619 | INFO | Analyzing chunk frames = [1496576,1497088) | 20260505 07:41:55.619 | INFO | MemInfo: { | 20260505 07:41:55.619 | INFO | "gpu" : " 17 GB", | 20260505 07:41:55.619 | INFO | "huge" : "", | 20260505 07:41:55.619 | INFO | "huge_shared" : " 27 GB", | 20260505 07:41:55.619 | INFO | "malloc" : " 14 GB", | 20260505 07:41:55.619 | INFO | "pinned" : " 29 GB" | 20260505 07:41:55.619 | INFO | } | 20260505 07:41:55.619 | INFO | Perf report for the last 5.13925 seconds: | 20260505 07:41:55.619 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:41:55.619 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:55.619 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:55.619 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:55.619 | INFO | ANALYSIS : 63.2%, 3.0, 98.1ms, 0 | 20260505 07:41:55.619 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:55.619 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:55.619 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:41:55.619 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:55.619 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:41:55.620 | INFO | Uploaded 36GiB to GPU in 2.12292s (16.9578GiB/s) | 20260505 07:41:55.620 | INFO | Downloaded 34GiB to GPU in 2.03823s (16.6812GiB/s) | 20260505 07:42:00.758 | INFO | Analyzing chunk frames = [1497088,1497600) | 20260505 07:42:00.758 | INFO | MemInfo: { | 20260505 07:42:00.758 | INFO | "gpu" : " 17 GB", | 20260505 07:42:00.758 | INFO | "huge" : "", | 20260505 07:42:00.758 | INFO | "huge_shared" : " 27 GB", | 20260505 07:42:00.758 | INFO | "malloc" : " 14 GB", | 20260505 07:42:00.758 | INFO | "pinned" : " 29 GB" | 20260505 07:42:00.758 | INFO | } | 20260505 07:42:00.758 | INFO | Perf report for the last 5.13946 seconds: | 20260505 07:42:00.758 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:42:00.758 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:00.758 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:00.758 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:00.758 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 07:42:00.758 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:00.758 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:00.758 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:42:00.759 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:00.759 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:00.759 | INFO | Uploaded 36GiB to GPU in 2.1374s (16.8429GiB/s) | 20260505 07:42:00.759 | INFO | Downloaded 34GiB to GPU in 2.05019s (16.5838GiB/s) | 20260505 07:42:05.899 | INFO | Analyzing chunk frames = [1497600,1498112) | 20260505 07:42:05.899 | INFO | MemInfo: { | 20260505 07:42:05.899 | INFO | "gpu" : " 17 GB", | 20260505 07:42:05.899 | INFO | "huge" : "", | 20260505 07:42:05.899 | INFO | "huge_shared" : " 27 GB", | 20260505 07:42:05.899 | INFO | "malloc" : " 14 GB", | 20260505 07:42:05.899 | INFO | "pinned" : " 29 GB" | 20260505 07:42:05.899 | INFO | } | 20260505 07:42:05.899 | INFO | Perf report for the last 5.1408 seconds: | 20260505 07:42:05.899 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:42:05.899 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:05.899 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:05.899 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:05.899 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 07:42:05.899 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:05.899 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:05.899 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:42:05.899 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:05.899 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:05.899 | INFO | Uploaded 36GiB to GPU in 2.15566s (16.7002GiB/s) | 20260505 07:42:05.900 | INFO | Downloaded 34GiB to GPU in 2.05513s (16.544GiB/s) | 20260505 07:42:11.037 | INFO | Analyzing chunk frames = [1498112,1498624) | 20260505 07:42:11.037 | INFO | MemInfo: { | 20260505 07:42:11.037 | INFO | "gpu" : " 17 GB", | 20260505 07:42:11.037 | INFO | "huge" : "", | 20260505 07:42:11.037 | INFO | "huge_shared" : " 27 GB", | 20260505 07:42:11.037 | INFO | "malloc" : " 14 GB", | 20260505 07:42:11.037 | INFO | "pinned" : " 29 GB" | 20260505 07:42:11.037 | INFO | } | 20260505 07:42:11.037 | INFO | Perf report for the last 5.13781 seconds: | 20260505 07:42:11.037 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:42:11.037 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:11.037 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:11.037 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:11.037 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 07:42:11.037 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:11.037 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:11.037 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260505 07:42:11.037 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:11.037 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:11.037 | INFO | Uploaded 36GiB to GPU in 2.18841s (16.4503GiB/s) | 20260505 07:42:11.037 | INFO | Downloaded 34GiB to GPU in 2.15399s (15.7847GiB/s) | 20260505 07:42:16.176 | INFO | Analyzing chunk frames = [1498624,1499136) | 20260505 07:42:16.176 | INFO | MemInfo: { | 20260505 07:42:16.176 | INFO | "gpu" : " 17 GB", | 20260505 07:42:16.176 | INFO | "huge" : "", | 20260505 07:42:16.176 | INFO | "huge_shared" : " 27 GB", | 20260505 07:42:16.176 | INFO | "malloc" : " 14 GB", | 20260505 07:42:16.176 | INFO | "pinned" : " 29 GB" | 20260505 07:42:16.176 | INFO | } | 20260505 07:42:16.176 | INFO | Perf report for the last 5.1391 seconds: | 20260505 07:42:16.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:42:16.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:16.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:16.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:16.176 | INFO | ANALYSIS : 64.8%, 3.0, 101.1ms, 0 | 20260505 07:42:16.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:16.176 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:16.176 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260505 07:42:16.176 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:16.176 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:16.176 | INFO | Uploaded 36GiB to GPU in 2.21732s (16.2358GiB/s) | 20260505 07:42:16.176 | INFO | Downloaded 34GiB to GPU in 2.04506s (16.6255GiB/s) | 20260505 07:42:16.176 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 07:42:16.176 | INFO | Total mem usage: 15.3 GiB | 20260505 07:42:16.176 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 07:42:16.177 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -5.35%) | 20260505 07:42:16.177 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 07:42:16.177 | INFO | High Water Breakdown: | 20260505 07:42:16.177 | INFO | 60.8% : 8.85 GiB ( 94.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 07:42:16.177 | INFO | 39% : 5.68 GiB ( 85.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 07:42:16.177 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 07:42:16.177 | INFO | ------------------------------------------ | 20260505 07:42:16.177 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:42:16.177 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 07:42:16.177 | INFO | Total mem usage: 26.5 GiB | 20260505 07:42:16.177 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 07:42:16.177 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 07:42:16.177 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 07:42:16.177 | INFO | High Water Breakdown: | 20260505 07:42:16.177 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:42:16.177 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:42:16.177 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:42:16.177 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:42:16.177 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:42:16.177 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:42:16.177 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 07:42:16.177 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 07:42:16.177 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:42:16.177 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 07:42:16.177 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 07:42:16.177 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 07:42:16.177 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 07:42:16.177 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 07:42:16.177 | INFO | ------------------------------------------ | 20260505 07:42:16.177 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:42:16.177 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 07:42:16.177 | INFO | Total mem usage: 25.6 GiB | 20260505 07:42:16.177 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 07:42:16.177 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 07:42:16.177 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 07:42:16.177 | INFO | High Water Breakdown: | 20260505 07:42:16.177 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 07:42:16.177 | INFO | ------------------------------------------ | 20260505 07:42:16.177 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:42:16.177 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 07:42:16.177 | INFO | Total mem usage: 15.7 GiB | 20260505 07:42:16.177 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 07:42:16.177 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 07:42:16.177 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 07:42:16.177 | INFO | High Water Breakdown: | 20260505 07:42:16.177 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 07:42:16.177 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 07:42:16.177 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 07:42:16.177 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 07:42:16.177 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 07:42:16.177 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 07:42:16.177 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 07:42:16.177 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 07:42:16.177 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 07:42:16.177 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:42:16.177 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 07:42:16.177 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:42:16.177 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 07:42:16.177 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 07:42:16.177 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 07:42:16.177 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 07:42:16.177 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:42:16.177 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 07:42:16.177 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:42:16.177 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:42:16.177 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:42:16.177 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 07:42:16.177 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 07:42:16.177 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:42:16.177 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 07:42:16.177 | INFO | ------------------------------------------ | 20260505 07:42:16.177 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:42:16.177 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 07:42:16.177 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:42:16.177 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:42:16.177 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.50e+01, 1.34e+01 / 2.13e+00 | 20260505 07:42:16.177 | INFO | PerfReport: 0.29%, AggregateStats -- 9.35e-02 / 1.02e-01, 1.06e-02 / 6.01e-03 | 20260505 07:42:16.177 | INFO | PerfReport: 11.62%, Baseline -- 3.81e+00 / 4.40e+00, 6.19e-01 / 3.05e-01 | 20260505 07:42:16.177 | INFO | PerfReport: 3.31%, Histogram -- 1.08e+00 / 1.46e+00, 2.11e-01 / 1.31e-01 | 20260505 07:42:16.177 | INFO | PerfReport: 15.49%, DME -- 4.06e+01 / 4.44e+01, 4.13e+00 / 1.38e+00 | 20260505 07:42:16.177 | INFO | PerfReport: 56.28%, FrameLabeling -- 1.84e+01 / 1.93e+01, 1.98e+00 / 1.49e+00 | 20260505 07:42:16.177 | INFO | PerfReport: 7.62%, PulseAccumulating -- 2.50e+00 / 3.16e+00, 4.42e-01 / 2.10e-01 | 20260505 07:42:16.177 | INFO | PerfReport: 3.96%, Metrics -- 1.30e+00 / 1.69e+00, 3.60e-01 / 1.76e-01 | 20260505 07:42:16.177 | INFO | PerfReport: 1.43%, PulseSerializing -- 4.69e-01 / 5.39e-01, 8.96e-02 / 2.65e-02 | 20260505 07:42:16.177 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 07:42:16.177 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:42:16.178 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:42:16.178 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.24e+01 / 0.00e+00 | 20260505 07:42:16.178 | INFO | PerfReport: 22.28%, Upload -- 2.26e+01 / 0.00e+00, 7.94e+00 / 0.00e+00 | 20260505 07:42:16.178 | INFO | PerfReport: 21.59%, Download -- 2.19e+01 / 0.00e+00, 7.93e+00 / 0.00e+00 | 20260505 07:42:16.178 | INFO | PerfReport: 32.35%, Compute -- 3.28e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 07:42:16.178 | INFO | PerfReport: 1.59%, LockUpload -- 1.61e+00 / 0.00e+00, 7.23e+00 / 0.00e+00 | 20260505 07:42:16.178 | INFO | PerfReport: 0.91%, LockDownload -- 9.20e-01 / 0.00e+00, 2.36e+00 / 0.00e+00 | 20260505 07:42:16.178 | INFO | PerfReport: 21.26%, LockCompute -- 2.15e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260505 07:42:16.178 | INFO | PerfReport: 0.01%, Misc -- 1.49e-02 / 0.00e+00, 2.04e-02 / 0.00e+00 | 20260505 07:42:16.311 | INFO | Starting a round of Baz IO | 20260505 07:42:19.598 | INFO | Populating RTMetrics report | 20260505 07:42:19.598 | INFO | RTMetrics report populated | 20260505 07:42:19.598 | INFO | Opening tmp RTMetrics json file | 20260505 07:42:19.624 | INFO | Serializing RTMetrics to json | 20260505 07:42:19.641 | INFO | Serialization done, swapping tmp file now... | 20260505 07:42:19.648 | INFO | RTMetrics done | 20260505 07:42:19.748 | INFO | Finished a round of Baz IO: Wrote 2.437 GiB in 3.437 seconds | 20260505 07:42:19.748 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 29.22% / 0.03% / 70.75% | 20260505 07:42:19.748 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.33968% / 3.57628e-05% / 8.74201e-05% | 20260505 07:42:19.748 | INFO | Baz IO overall: | 20260505 07:42:19.748 | INFO | BytesWritten : 825481294908 | 20260505 07:42:19.748 | INFO | NumEvents : 391878867277 | 20260505 07:42:21.315 | INFO | Analyzing chunk frames = [1499136,1499648) | 20260505 07:42:21.315 | INFO | MemInfo: { | 20260505 07:42:21.315 | INFO | "gpu" : " 17 GB", | 20260505 07:42:21.315 | INFO | "huge" : "", | 20260505 07:42:21.315 | INFO | "huge_shared" : " 27 GB", | 20260505 07:42:21.315 | INFO | "malloc" : " 14 GB", | 20260505 07:42:21.315 | INFO | "pinned" : " 29 GB" | 20260505 07:42:21.315 | INFO | } | 20260505 07:42:21.315 | INFO | Perf report for the last 5.13931 seconds: | 20260505 07:42:21.316 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:42:21.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:21.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:21.316 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:21.316 | INFO | ANALYSIS : 65.4%, 2.9, 99.3ms, 0 | 20260505 07:42:21.316 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:21.316 | INFO | RT_METRICS : 46.8%, 1.0, 24.6ms, 0 | 20260505 07:42:21.316 | INFO | PRE_HQ : 64.0%, 1.9, 62.1ms, 0 | 20260505 07:42:21.316 | INFO | BAZWRITER : 66.2%, 4.2, 142.1ms, 0 | 20260505 07:42:21.316 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 07:42:21.316 | INFO | Uploaded 36GiB to GPU in 2.20683s (16.313GiB/s) | 20260505 07:42:21.316 | INFO | Downloaded 38GiB to GPU in 2.34852s (16.1804GiB/s) | 20260505 07:42:26.455 | INFO | Analyzing chunk frames = [1499648,1500160) | 20260505 07:42:26.455 | INFO | MemInfo: { | 20260505 07:42:26.455 | INFO | "gpu" : " 17 GB", | 20260505 07:42:26.455 | INFO | "huge" : "", | 20260505 07:42:26.455 | INFO | "huge_shared" : " 27 GB", | 20260505 07:42:26.455 | INFO | "malloc" : " 14 GB", | 20260505 07:42:26.455 | INFO | "pinned" : " 29 GB" | 20260505 07:42:26.455 | INFO | } | 20260505 07:42:26.455 | INFO | Perf report for the last 5.13957 seconds: | 20260505 07:42:26.455 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:42:26.455 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:26.455 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:26.455 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:26.455 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 07:42:26.455 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:26.455 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:26.455 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:42:26.455 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:26.455 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:26.455 | INFO | Uploaded 36GiB to GPU in 2.18939s (16.4429GiB/s) | 20260505 07:42:26.455 | INFO | Downloaded 34GiB to GPU in 2.05855s (16.5165GiB/s) | 20260505 07:42:31.594 | INFO | Analyzing chunk frames = [1500160,1500672) | 20260505 07:42:31.594 | INFO | MemInfo: { | 20260505 07:42:31.594 | INFO | "gpu" : " 17 GB", | 20260505 07:42:31.594 | INFO | "huge" : "", | 20260505 07:42:31.594 | INFO | "huge_shared" : " 27 GB", | 20260505 07:42:31.594 | INFO | "malloc" : " 14 GB", | 20260505 07:42:31.594 | INFO | "pinned" : " 29 GB" | 20260505 07:42:31.594 | INFO | } | 20260505 07:42:31.594 | INFO | Perf report for the last 5.13922 seconds: | 20260505 07:42:31.594 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:42:31.594 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:31.594 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:31.594 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:31.594 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 07:42:31.594 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:31.594 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:31.594 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:42:31.594 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:31.594 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:31.595 | INFO | Uploaded 36GiB to GPU in 2.15217s (16.7273GiB/s) | 20260505 07:42:31.595 | INFO | Downloaded 34GiB to GPU in 2.05739s (16.5258GiB/s) | 20260505 07:42:36.734 | INFO | Analyzing chunk frames = [1500672,1501184) | 20260505 07:42:36.734 | INFO | MemInfo: { | 20260505 07:42:36.734 | INFO | "gpu" : " 17 GB", | 20260505 07:42:36.734 | INFO | "huge" : "", | 20260505 07:42:36.734 | INFO | "huge_shared" : " 27 GB", | 20260505 07:42:36.734 | INFO | "malloc" : " 14 GB", | 20260505 07:42:36.734 | INFO | "pinned" : " 29 GB" | 20260505 07:42:36.734 | INFO | } | 20260505 07:42:36.734 | INFO | Perf report for the last 5.13945 seconds: | 20260505 07:42:36.734 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:42:36.734 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:36.734 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:36.734 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:36.734 | INFO | ANALYSIS : 63.1%, 3.0, 98.0ms, 0 | 20260505 07:42:36.734 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:36.734 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:36.734 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:42:36.734 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:36.734 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:36.734 | INFO | Uploaded 36GiB to GPU in 2.1525s (16.7247GiB/s) | 20260505 07:42:36.734 | INFO | Downloaded 34GiB to GPU in 2.04462s (16.629GiB/s) | 20260505 07:42:41.873 | INFO | Analyzing chunk frames = [1501184,1501696) | 20260505 07:42:41.873 | INFO | MemInfo: { | 20260505 07:42:41.873 | INFO | "gpu" : " 17 GB", | 20260505 07:42:41.873 | INFO | "huge" : "", | 20260505 07:42:41.873 | INFO | "huge_shared" : " 27 GB", | 20260505 07:42:41.873 | INFO | "malloc" : " 14 GB", | 20260505 07:42:41.873 | INFO | "pinned" : " 29 GB" | 20260505 07:42:41.873 | INFO | } | 20260505 07:42:41.873 | INFO | Perf report for the last 5.13934 seconds: | 20260505 07:42:41.873 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:42:41.873 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:41.873 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:41.873 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:41.873 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 07:42:41.873 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:41.873 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:41.873 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:42:41.873 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:41.873 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:41.874 | INFO | Uploaded 36GiB to GPU in 2.15798s (16.6823GiB/s) | 20260505 07:42:41.874 | INFO | Downloaded 34GiB to GPU in 2.048s (16.6016GiB/s) | 20260505 07:42:47.014 | INFO | Analyzing chunk frames = [1501696,1502208) | 20260505 07:42:47.014 | INFO | MemInfo: { | 20260505 07:42:47.014 | INFO | "gpu" : " 17 GB", | 20260505 07:42:47.014 | INFO | "huge" : "", | 20260505 07:42:47.014 | INFO | "huge_shared" : " 27 GB", | 20260505 07:42:47.014 | INFO | "malloc" : " 14 GB", | 20260505 07:42:47.014 | INFO | "pinned" : " 29 GB" | 20260505 07:42:47.014 | INFO | } | 20260505 07:42:47.014 | INFO | Perf report for the last 5.14108 seconds: | 20260505 07:42:47.014 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:42:47.014 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:47.014 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:47.014 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:47.014 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 07:42:47.014 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:47.014 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:47.014 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:42:47.014 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:47.014 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:47.014 | INFO | Uploaded 36GiB to GPU in 2.1974s (16.383GiB/s) | 20260505 07:42:47.015 | INFO | Downloaded 34GiB to GPU in 2.04932s (16.5909GiB/s) | 20260505 07:42:52.152 | INFO | Analyzing chunk frames = [1502208,1502720) | 20260505 07:42:52.152 | INFO | MemInfo: { | 20260505 07:42:52.152 | INFO | "gpu" : " 17 GB", | 20260505 07:42:52.152 | INFO | "huge" : "", | 20260505 07:42:52.152 | INFO | "huge_shared" : " 27 GB", | 20260505 07:42:52.152 | INFO | "malloc" : " 14 GB", | 20260505 07:42:52.152 | INFO | "pinned" : " 29 GB" | 20260505 07:42:52.152 | INFO | } | 20260505 07:42:52.152 | INFO | Perf report for the last 5.13758 seconds: | 20260505 07:42:52.152 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:42:52.152 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:52.152 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:52.152 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:52.152 | INFO | ANALYSIS : 64.1%, 3.0, 100.0ms, 0 | 20260505 07:42:52.152 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:52.152 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:52.152 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 07:42:52.152 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:52.152 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:52.152 | INFO | Uploaded 36GiB to GPU in 2.18699s (16.461GiB/s) | 20260505 07:42:52.152 | INFO | Downloaded 34GiB to GPU in 2.14387s (15.8592GiB/s) | 20260505 07:42:57.291 | INFO | Analyzing chunk frames = [1502720,1503232) | 20260505 07:42:57.291 | INFO | MemInfo: { | 20260505 07:42:57.291 | INFO | "gpu" : " 17 GB", | 20260505 07:42:57.291 | INFO | "huge" : "", | 20260505 07:42:57.291 | INFO | "huge_shared" : " 27 GB", | 20260505 07:42:57.291 | INFO | "malloc" : " 14 GB", | 20260505 07:42:57.291 | INFO | "pinned" : " 29 GB" | 20260505 07:42:57.291 | INFO | } | 20260505 07:42:57.291 | INFO | Perf report for the last 5.13944 seconds: | 20260505 07:42:57.291 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:42:57.291 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:57.291 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:57.291 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:57.291 | INFO | ANALYSIS : 64.8%, 3.0, 101.2ms, 0 | 20260505 07:42:57.291 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:57.291 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:57.291 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 07:42:57.291 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:57.291 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:42:57.291 | INFO | Uploaded 36GiB to GPU in 2.22215s (16.2005GiB/s) | 20260505 07:42:57.292 | INFO | Downloaded 34GiB to GPU in 2.04218s (16.6488GiB/s) | 20260505 07:42:57.425 | INFO | Starting a round of Baz IO | 20260505 07:43:00.702 | INFO | Populating RTMetrics report | 20260505 07:43:00.703 | INFO | RTMetrics report populated | 20260505 07:43:00.703 | INFO | Opening tmp RTMetrics json file | 20260505 07:43:00.711 | INFO | Serializing RTMetrics to json | 20260505 07:43:00.714 | INFO | Serialization done, swapping tmp file now... | 20260505 07:43:00.718 | INFO | RTMetrics done | 20260505 07:43:00.833 | INFO | Finished a round of Baz IO: Wrote 2.437 GiB in 3.407 seconds | 20260505 07:43:00.833 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 29.19% / 0.03% / 70.78% | 20260505 07:43:00.833 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.31724% / 4.37101e-05% / 7.54992e-05% | 20260505 07:43:00.833 | INFO | Baz IO overall: | 20260505 07:43:00.833 | INFO | BytesWritten : 828097936431 | 20260505 07:43:00.833 | INFO | NumEvents : 393130258640 | 20260505 07:43:02.430 | INFO | Analyzing chunk frames = [1503232,1503744) | 20260505 07:43:02.430 | INFO | MemInfo: { | 20260505 07:43:02.430 | INFO | "gpu" : " 17 GB", | 20260505 07:43:02.430 | INFO | "huge" : "", | 20260505 07:43:02.430 | INFO | "huge_shared" : " 27 GB", | 20260505 07:43:02.430 | INFO | "malloc" : " 14 GB", | 20260505 07:43:02.430 | INFO | "pinned" : " 29 GB" | 20260505 07:43:02.430 | INFO | } | 20260505 07:43:02.430 | INFO | Perf report for the last 5.13913 seconds: | 20260505 07:43:02.431 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:43:02.431 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:02.431 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:02.431 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:02.431 | INFO | ANALYSIS : 65.2%, 2.9, 99.3ms, 0 | 20260505 07:43:02.431 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:02.431 | INFO | RT_METRICS : 46.2%, 1.0, 24.2ms, 0 | 20260505 07:43:02.431 | INFO | PRE_HQ : 63.9%, 1.9, 62.3ms, 0 | 20260505 07:43:02.431 | INFO | BAZWRITER : 65.6%, 4.2, 140.2ms, 0 | 20260505 07:43:02.431 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260505 07:43:02.431 | INFO | Uploaded 36GiB to GPU in 2.21374s (16.2621GiB/s) | 20260505 07:43:02.431 | INFO | Downloaded 38GiB to GPU in 2.34018s (16.2381GiB/s) | 20260505 07:43:07.570 | INFO | Analyzing chunk frames = [1503744,1504256) | 20260505 07:43:07.570 | INFO | MemInfo: { | 20260505 07:43:07.570 | INFO | "gpu" : " 17 GB", | 20260505 07:43:07.570 | INFO | "huge" : "", | 20260505 07:43:07.570 | INFO | "huge_shared" : " 27 GB", | 20260505 07:43:07.570 | INFO | "malloc" : " 14 GB", | 20260505 07:43:07.570 | INFO | "pinned" : " 29 GB" | 20260505 07:43:07.570 | INFO | } | 20260505 07:43:07.570 | INFO | Perf report for the last 5.13943 seconds: | 20260505 07:43:07.570 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:43:07.570 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:07.570 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:07.570 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:07.570 | INFO | ANALYSIS : 64.3%, 3.0, 100.3ms, 0 | 20260505 07:43:07.570 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:07.570 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:07.570 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:43:07.570 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:07.570 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:07.570 | INFO | Uploaded 36GiB to GPU in 2.18326s (16.4891GiB/s) | 20260505 07:43:07.570 | INFO | Downloaded 34GiB to GPU in 2.05502s (16.5449GiB/s) | 20260505 07:43:12.709 | INFO | Analyzing chunk frames = [1504256,1504768) | 20260505 07:43:12.709 | INFO | MemInfo: { | 20260505 07:43:12.709 | INFO | "gpu" : " 17 GB", | 20260505 07:43:12.709 | INFO | "huge" : "", | 20260505 07:43:12.709 | INFO | "huge_shared" : " 27 GB", | 20260505 07:43:12.709 | INFO | "malloc" : " 14 GB", | 20260505 07:43:12.709 | INFO | "pinned" : " 29 GB" | 20260505 07:43:12.709 | INFO | } | 20260505 07:43:12.709 | INFO | Perf report for the last 5.13942 seconds: | 20260505 07:43:12.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:43:12.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:12.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:12.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:12.709 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 07:43:12.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:12.709 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:12.709 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:43:12.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:12.709 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:12.709 | INFO | Uploaded 36GiB to GPU in 2.15185s (16.7298GiB/s) | 20260505 07:43:12.710 | INFO | Downloaded 34GiB to GPU in 2.05741s (16.5257GiB/s) | 20260505 07:43:17.848 | INFO | Analyzing chunk frames = [1504768,1505280) | 20260505 07:43:17.848 | INFO | MemInfo: { | 20260505 07:43:17.848 | INFO | "gpu" : " 17 GB", | 20260505 07:43:17.848 | INFO | "huge" : "", | 20260505 07:43:17.848 | INFO | "huge_shared" : " 27 GB", | 20260505 07:43:17.848 | INFO | "malloc" : " 14 GB", | 20260505 07:43:17.848 | INFO | "pinned" : " 29 GB" | 20260505 07:43:17.848 | INFO | } | 20260505 07:43:17.848 | INFO | Perf report for the last 5.13906 seconds: | 20260505 07:43:17.848 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:43:17.848 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:17.848 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:17.848 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:17.848 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 07:43:17.848 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:17.848 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:17.848 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:43:17.848 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:17.848 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:17.849 | INFO | Uploaded 36GiB to GPU in 2.15911s (16.6735GiB/s) | 20260505 07:43:17.849 | INFO | Downloaded 34GiB to GPU in 2.04184s (16.6517GiB/s) | 20260505 07:43:22.988 | INFO | Analyzing chunk frames = [1505280,1505792) | 20260505 07:43:22.988 | INFO | MemInfo: { | 20260505 07:43:22.988 | INFO | "gpu" : " 17 GB", | 20260505 07:43:22.988 | INFO | "huge" : "", | 20260505 07:43:22.988 | INFO | "huge_shared" : " 27 GB", | 20260505 07:43:22.988 | INFO | "malloc" : " 14 GB", | 20260505 07:43:22.988 | INFO | "pinned" : " 29 GB" | 20260505 07:43:22.988 | INFO | } | 20260505 07:43:22.988 | INFO | Perf report for the last 5.13975 seconds: | 20260505 07:43:22.988 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:43:22.988 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:22.988 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:22.988 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:22.988 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 07:43:22.988 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:22.988 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:22.988 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:43:22.988 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:22.988 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:22.988 | INFO | Uploaded 36GiB to GPU in 2.19877s (16.3728GiB/s) | 20260505 07:43:22.988 | INFO | Downloaded 34GiB to GPU in 2.05687s (16.53GiB/s) | 20260505 07:43:28.128 | INFO | Analyzing chunk frames = [1505792,1506304) | 20260505 07:43:28.128 | INFO | MemInfo: { | 20260505 07:43:28.128 | INFO | "gpu" : " 17 GB", | 20260505 07:43:28.129 | INFO | "huge" : "", | 20260505 07:43:28.129 | INFO | "huge_shared" : " 27 GB", | 20260505 07:43:28.129 | INFO | "malloc" : " 14 GB", | 20260505 07:43:28.129 | INFO | "pinned" : " 29 GB" | 20260505 07:43:28.129 | INFO | } | 20260505 07:43:28.129 | INFO | Perf report for the last 5.14048 seconds: | 20260505 07:43:28.129 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:43:28.129 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:28.129 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:28.129 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:28.129 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 07:43:28.129 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:28.129 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:28.129 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:43:28.129 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:28.129 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:28.129 | INFO | Uploaded 36GiB to GPU in 2.20315s (16.3402GiB/s) | 20260505 07:43:28.129 | INFO | Downloaded 34GiB to GPU in 2.04862s (16.5966GiB/s) | 20260505 07:43:33.266 | INFO | Analyzing chunk frames = [1506304,1506816) | 20260505 07:43:33.267 | INFO | MemInfo: { | 20260505 07:43:33.267 | INFO | "gpu" : " 17 GB", | 20260505 07:43:33.267 | INFO | "huge" : "", | 20260505 07:43:33.267 | INFO | "huge_shared" : " 27 GB", | 20260505 07:43:33.267 | INFO | "malloc" : " 14 GB", | 20260505 07:43:33.267 | INFO | "pinned" : " 29 GB" | 20260505 07:43:33.267 | INFO | } | 20260505 07:43:33.267 | INFO | Perf report for the last 5.13803 seconds: | 20260505 07:43:33.267 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:43:33.267 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:33.267 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:33.267 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:33.267 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 07:43:33.267 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:33.267 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:33.267 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260505 07:43:33.267 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:33.267 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:33.267 | INFO | Uploaded 36GiB to GPU in 2.18611s (16.4676GiB/s) | 20260505 07:43:33.267 | INFO | Downloaded 34GiB to GPU in 2.1533s (15.7897GiB/s) | 20260505 07:43:38.406 | INFO | Analyzing chunk frames = [1506816,1507328) | 20260505 07:43:38.406 | INFO | MemInfo: { | 20260505 07:43:38.406 | INFO | "gpu" : " 17 GB", | 20260505 07:43:38.406 | INFO | "huge" : "", | 20260505 07:43:38.406 | INFO | "huge_shared" : " 27 GB", | 20260505 07:43:38.406 | INFO | "malloc" : " 14 GB", | 20260505 07:43:38.406 | INFO | "pinned" : " 29 GB" | 20260505 07:43:38.406 | INFO | } | 20260505 07:43:38.406 | INFO | Perf report for the last 5.13908 seconds: | 20260505 07:43:38.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:43:38.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:38.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:38.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:38.406 | INFO | ANALYSIS : 64.8%, 3.0, 101.1ms, 0 | 20260505 07:43:38.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:38.406 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:38.406 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 07:43:38.406 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:38.406 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:38.406 | INFO | Uploaded 36GiB to GPU in 2.22173s (16.2036GiB/s) | 20260505 07:43:38.406 | INFO | Downloaded 34GiB to GPU in 2.05221s (16.5675GiB/s) | 20260505 07:43:38.406 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 07:43:38.406 | INFO | Total mem usage: 15.3 GiB | 20260505 07:43:38.406 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 07:43:38.406 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -5.38%) | 20260505 07:43:38.406 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 07:43:38.406 | INFO | High Water Breakdown: | 20260505 07:43:38.406 | INFO | 60.8% : 8.85 GiB ( 94.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 07:43:38.406 | INFO | 39% : 5.68 GiB ( 85.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 07:43:38.406 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 07:43:38.406 | INFO | ------------------------------------------ | 20260505 07:43:38.406 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:43:38.406 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 07:43:38.406 | INFO | Total mem usage: 26.5 GiB | 20260505 07:43:38.406 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 07:43:38.406 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 07:43:38.406 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 07:43:38.406 | INFO | High Water Breakdown: | 20260505 07:43:38.406 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:43:38.406 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:43:38.406 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:43:38.406 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:43:38.407 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:43:38.407 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:43:38.407 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 07:43:38.407 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 07:43:38.407 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:43:38.407 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 07:43:38.407 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 07:43:38.407 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 07:43:38.407 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 07:43:38.407 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 07:43:38.407 | INFO | ------------------------------------------ | 20260505 07:43:38.407 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:43:38.407 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 07:43:38.407 | INFO | Total mem usage: 25.6 GiB | 20260505 07:43:38.407 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 07:43:38.407 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 07:43:38.407 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 07:43:38.407 | INFO | High Water Breakdown: | 20260505 07:43:38.407 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 07:43:38.407 | INFO | ------------------------------------------ | 20260505 07:43:38.407 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:43:38.407 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 07:43:38.407 | INFO | Total mem usage: 15.7 GiB | 20260505 07:43:38.407 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 07:43:38.407 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 07:43:38.407 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 07:43:38.407 | INFO | High Water Breakdown: | 20260505 07:43:38.407 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 07:43:38.407 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 07:43:38.407 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 07:43:38.407 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 07:43:38.407 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 07:43:38.407 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 07:43:38.407 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 07:43:38.407 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 07:43:38.407 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 07:43:38.407 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:43:38.407 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 07:43:38.407 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:43:38.407 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 07:43:38.407 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 07:43:38.407 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 07:43:38.407 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 07:43:38.407 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:43:38.407 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 07:43:38.407 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:43:38.407 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:43:38.407 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:43:38.407 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 07:43:38.407 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 07:43:38.407 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:43:38.407 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 07:43:38.407 | INFO | ------------------------------------------ | 20260505 07:43:38.407 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:43:38.407 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 07:43:38.407 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:43:38.407 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:43:38.407 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.53e+01, 1.37e+01 / 2.03e+00 | 20260505 07:43:38.407 | INFO | PerfReport: 0.29%, AggregateStats -- 9.37e-02 / 1.04e-01, 1.10e-02 / 5.75e-03 | 20260505 07:43:38.407 | INFO | PerfReport: 11.60%, Baseline -- 3.81e+00 / 4.42e+00, 6.15e-01 / 3.16e-01 | 20260505 07:43:38.407 | INFO | PerfReport: 3.31%, Histogram -- 1.09e+00 / 1.46e+00, 2.12e-01 / 1.22e-01 | 20260505 07:43:38.407 | INFO | PerfReport: 15.49%, DME -- 4.07e+01 / 4.47e+01, 4.16e+00 / 1.23e+00 | 20260505 07:43:38.407 | INFO | PerfReport: 56.44%, FrameLabeling -- 1.85e+01 / 1.92e+01, 2.07e+00 / 1.53e+00 | 20260505 07:43:38.407 | INFO | PerfReport: 7.51%, PulseAccumulating -- 2.47e+00 / 3.16e+00, 4.23e-01 / 1.71e-01 | 20260505 07:43:38.407 | INFO | PerfReport: 3.94%, Metrics -- 1.30e+00 / 1.68e+00, 3.67e-01 / 1.50e-01 | 20260505 07:43:38.407 | INFO | PerfReport: 1.43%, PulseSerializing -- 4.68e-01 / 5.34e-01, 9.08e-02 / 2.80e-02 | 20260505 07:43:38.407 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 07:43:38.407 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:43:38.407 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:43:38.407 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.30e+01 / 0.00e+00 | 20260505 07:43:38.407 | INFO | PerfReport: 22.41%, Upload -- 2.28e+01 / 0.00e+00, 8.07e+00 / 0.00e+00 | 20260505 07:43:38.407 | INFO | PerfReport: 21.52%, Download -- 2.19e+01 / 0.00e+00, 7.97e+00 / 0.00e+00 | 20260505 07:43:38.407 | INFO | PerfReport: 32.33%, Compute -- 3.29e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 07:43:38.407 | INFO | PerfReport: 1.63%, LockUpload -- 1.66e+00 / 0.00e+00, 7.25e+00 / 0.00e+00 | 20260505 07:43:38.407 | INFO | PerfReport: 0.88%, LockDownload -- 8.93e-01 / 0.00e+00, 2.32e+00 / 0.00e+00 | 20260505 07:43:38.407 | INFO | PerfReport: 21.22%, LockCompute -- 2.16e+01 / 0.00e+00, 1.74e+01 / 0.00e+00 | 20260505 07:43:38.407 | INFO | PerfReport: 0.01%, Misc -- 1.48e-02 / 0.00e+00, 2.06e-02 / 0.00e+00 | 20260505 07:43:38.541 | INFO | Starting a round of Baz IO | 20260505 07:43:41.812 | INFO | Populating RTMetrics report | 20260505 07:43:41.812 | INFO | RTMetrics report populated | 20260505 07:43:41.812 | INFO | Opening tmp RTMetrics json file | 20260505 07:43:41.850 | INFO | Serializing RTMetrics to json | 20260505 07:43:41.855 | INFO | Serialization done, swapping tmp file now... | 20260505 07:43:41.857 | INFO | RTMetrics done | 20260505 07:43:41.945 | INFO | Finished a round of Baz IO: Wrote 2.44 GiB in 3.403 seconds | 20260505 07:43:41.945 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 29.16% / 0.03% / 70.81% | 20260505 07:43:41.945 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.29717% / 2.38419e-05% / 5.96046e-05% | 20260505 07:43:41.945 | INFO | Baz IO overall: | 20260505 07:43:41.945 | INFO | BytesWritten : 830717643869 | 20260505 07:43:41.945 | INFO | NumEvents : 394382101992 | 20260505 07:43:43.545 | INFO | Analyzing chunk frames = [1507328,1507840) | 20260505 07:43:43.545 | INFO | MemInfo: { | 20260505 07:43:43.545 | INFO | "gpu" : " 17 GB", | 20260505 07:43:43.545 | INFO | "huge" : "", | 20260505 07:43:43.545 | INFO | "huge_shared" : " 27 GB", | 20260505 07:43:43.545 | INFO | "malloc" : " 14 GB", | 20260505 07:43:43.545 | INFO | "pinned" : " 29 GB" | 20260505 07:43:43.545 | INFO | } | 20260505 07:43:43.545 | INFO | Perf report for the last 5.13942 seconds: | 20260505 07:43:43.545 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:43:43.545 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:43.545 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:43.545 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:43.545 | INFO | ANALYSIS : 65.1%, 2.9, 99.2ms, 0 | 20260505 07:43:43.545 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:43.545 | INFO | RT_METRICS : 46.7%, 1.0, 24.5ms, 0 | 20260505 07:43:43.545 | INFO | PRE_HQ : 63.7%, 1.9, 62.0ms, 0 | 20260505 07:43:43.545 | INFO | BAZWRITER : 65.5%, 4.2, 141.0ms, 0 | 20260505 07:43:43.545 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 07:43:43.545 | INFO | Uploaded 36GiB to GPU in 2.21305s (16.2672GiB/s) | 20260505 07:43:43.545 | INFO | Downloaded 38GiB to GPU in 2.31996s (16.3796GiB/s) | 20260505 07:43:48.685 | INFO | Analyzing chunk frames = [1507840,1508352) | 20260505 07:43:48.685 | INFO | MemInfo: { | 20260505 07:43:48.685 | INFO | "gpu" : " 17 GB", | 20260505 07:43:48.685 | INFO | "huge" : "", | 20260505 07:43:48.685 | INFO | "huge_shared" : " 27 GB", | 20260505 07:43:48.685 | INFO | "malloc" : " 14 GB", | 20260505 07:43:48.685 | INFO | "pinned" : " 29 GB" | 20260505 07:43:48.685 | INFO | } | 20260505 07:43:48.685 | INFO | Perf report for the last 5.13952 seconds: | 20260505 07:43:48.685 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:43:48.685 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:48.685 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:48.685 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:48.685 | INFO | ANALYSIS : 64.3%, 3.0, 100.5ms, 0 | 20260505 07:43:48.685 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:48.685 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:48.685 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:43:48.685 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:48.685 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:48.685 | INFO | Uploaded 36GiB to GPU in 2.18299s (16.4912GiB/s) | 20260505 07:43:48.685 | INFO | Downloaded 34GiB to GPU in 2.06006s (16.5044GiB/s) | 20260505 07:43:53.824 | INFO | Analyzing chunk frames = [1508352,1508864) | 20260505 07:43:53.824 | INFO | MemInfo: { | 20260505 07:43:53.824 | INFO | "gpu" : " 17 GB", | 20260505 07:43:53.824 | INFO | "huge" : "", | 20260505 07:43:53.824 | INFO | "huge_shared" : " 27 GB", | 20260505 07:43:53.824 | INFO | "malloc" : " 14 GB", | 20260505 07:43:53.824 | INFO | "pinned" : " 29 GB" | 20260505 07:43:53.824 | INFO | } | 20260505 07:43:53.824 | INFO | Perf report for the last 5.1395 seconds: | 20260505 07:43:53.824 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:43:53.824 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:53.824 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:53.824 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:53.824 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 07:43:53.824 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:53.824 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:53.824 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:43:53.824 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:53.824 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:53.825 | INFO | Uploaded 36GiB to GPU in 2.14197s (16.807GiB/s) | 20260505 07:43:53.825 | INFO | Downloaded 34GiB to GPU in 2.0575s (16.5249GiB/s) | 20260505 07:43:58.964 | INFO | Analyzing chunk frames = [1508864,1509376) | 20260505 07:43:58.964 | INFO | MemInfo: { | 20260505 07:43:58.964 | INFO | "gpu" : " 17 GB", | 20260505 07:43:58.964 | INFO | "huge" : "", | 20260505 07:43:58.964 | INFO | "huge_shared" : " 27 GB", | 20260505 07:43:58.964 | INFO | "malloc" : " 14 GB", | 20260505 07:43:58.964 | INFO | "pinned" : " 29 GB" | 20260505 07:43:58.964 | INFO | } | 20260505 07:43:58.964 | INFO | Perf report for the last 5.13946 seconds: | 20260505 07:43:58.964 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:43:58.964 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:58.964 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:58.964 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:58.964 | INFO | ANALYSIS : 63.1%, 3.0, 98.1ms, 0 | 20260505 07:43:58.964 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:58.964 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:58.964 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:43:58.964 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:58.964 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:43:58.964 | INFO | Uploaded 36GiB to GPU in 2.14286s (16.8GiB/s) | 20260505 07:43:58.964 | INFO | Downloaded 34GiB to GPU in 2.04173s (16.6526GiB/s) | 20260505 07:44:04.102 | INFO | Analyzing chunk frames = [1509376,1509888) | 20260505 07:44:04.102 | INFO | MemInfo: { | 20260505 07:44:04.103 | INFO | "gpu" : " 17 GB", | 20260505 07:44:04.103 | INFO | "huge" : "", | 20260505 07:44:04.103 | INFO | "huge_shared" : " 27 GB", | 20260505 07:44:04.103 | INFO | "malloc" : " 14 GB", | 20260505 07:44:04.103 | INFO | "pinned" : " 29 GB" | 20260505 07:44:04.103 | INFO | } | 20260505 07:44:04.103 | INFO | Perf report for the last 5.13895 seconds: | 20260505 07:44:04.103 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:44:04.103 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:04.103 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:04.103 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:04.103 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 07:44:04.103 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:04.103 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:04.103 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 07:44:04.103 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:04.103 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:04.103 | INFO | Uploaded 36GiB to GPU in 2.13361s (16.8728GiB/s) | 20260505 07:44:04.103 | INFO | Downloaded 34GiB to GPU in 2.05045s (16.5817GiB/s) | 20260505 07:44:09.244 | INFO | Analyzing chunk frames = [1509888,1510400) | 20260505 07:44:09.244 | INFO | MemInfo: { | 20260505 07:44:09.244 | INFO | "gpu" : " 17 GB", | 20260505 07:44:09.244 | INFO | "huge" : "", | 20260505 07:44:09.244 | INFO | "huge_shared" : " 27 GB", | 20260505 07:44:09.244 | INFO | "malloc" : " 14 GB", | 20260505 07:44:09.244 | INFO | "pinned" : " 29 GB" | 20260505 07:44:09.244 | INFO | } | 20260505 07:44:09.244 | INFO | Perf report for the last 5.14122 seconds: | 20260505 07:44:09.244 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:44:09.244 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:09.244 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:09.244 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:09.244 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 07:44:09.244 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:09.244 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:09.244 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 07:44:09.244 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:09.244 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:09.244 | INFO | Uploaded 36GiB to GPU in 2.15415s (16.7119GiB/s) | 20260505 07:44:09.244 | INFO | Downloaded 34GiB to GPU in 2.05313s (16.5601GiB/s) | 20260505 07:44:14.381 | INFO | Analyzing chunk frames = [1510400,1510912) | 20260505 07:44:14.381 | INFO | MemInfo: { | 20260505 07:44:14.381 | INFO | "gpu" : " 17 GB", | 20260505 07:44:14.381 | INFO | "huge" : "", | 20260505 07:44:14.381 | INFO | "huge_shared" : " 27 GB", | 20260505 07:44:14.381 | INFO | "malloc" : " 14 GB", | 20260505 07:44:14.381 | INFO | "pinned" : " 29 GB" | 20260505 07:44:14.381 | INFO | } | 20260505 07:44:14.382 | INFO | Perf report for the last 5.13763 seconds: | 20260505 07:44:14.382 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:44:14.382 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:14.382 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:14.382 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:14.382 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 07:44:14.382 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:14.382 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:14.382 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260505 07:44:14.382 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:14.382 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:14.382 | INFO | Uploaded 36GiB to GPU in 2.17817s (16.5276GiB/s) | 20260505 07:44:14.382 | INFO | Downloaded 34GiB to GPU in 2.13825s (15.9008GiB/s) | 20260505 07:44:19.520 | INFO | Analyzing chunk frames = [1510912,1511424) | 20260505 07:44:19.521 | INFO | MemInfo: { | 20260505 07:44:19.521 | INFO | "gpu" : " 17 GB", | 20260505 07:44:19.521 | INFO | "huge" : "", | 20260505 07:44:19.521 | INFO | "huge_shared" : " 27 GB", | 20260505 07:44:19.521 | INFO | "malloc" : " 14 GB", | 20260505 07:44:19.521 | INFO | "pinned" : " 29 GB" | 20260505 07:44:19.521 | INFO | } | 20260505 07:44:19.521 | INFO | Perf report for the last 5.1392 seconds: | 20260505 07:44:19.521 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:44:19.521 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:19.521 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:19.521 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:19.521 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 07:44:19.521 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:19.521 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:19.521 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 07:44:19.521 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:19.521 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:19.521 | INFO | Uploaded 36GiB to GPU in 2.15907s (16.6738GiB/s) | 20260505 07:44:19.521 | INFO | Downloaded 34GiB to GPU in 2.05202s (16.569GiB/s) | 20260505 07:44:19.655 | INFO | Starting a round of Baz IO | 20260505 07:44:22.937 | INFO | Populating RTMetrics report | 20260505 07:44:22.937 | INFO | RTMetrics report populated | 20260505 07:44:22.937 | INFO | Opening tmp RTMetrics json file | 20260505 07:44:22.951 | INFO | Serializing RTMetrics to json | 20260505 07:44:22.976 | INFO | Serialization done, swapping tmp file now... | 20260505 07:44:22.978 | INFO | RTMetrics done | 20260505 07:44:23.080 | INFO | Finished a round of Baz IO: Wrote 2.439 GiB in 3.424 seconds | 20260505 07:44:23.080 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 29.14% / 0.03% / 70.84% | 20260505 07:44:23.080 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.29107% / 4.37101e-05% / 8.34465e-05% | 20260505 07:44:23.080 | INFO | Baz IO overall: | 20260505 07:44:23.080 | INFO | BytesWritten : 833336910238 | 20260505 07:44:23.080 | INFO | NumEvents : 395634172929 | 20260505 07:44:24.660 | INFO | Analyzing chunk frames = [1511424,1511936) | 20260505 07:44:24.660 | INFO | MemInfo: { | 20260505 07:44:24.660 | INFO | "gpu" : " 17 GB", | 20260505 07:44:24.660 | INFO | "huge" : "", | 20260505 07:44:24.660 | INFO | "huge_shared" : " 27 GB", | 20260505 07:44:24.660 | INFO | "malloc" : " 14 GB", | 20260505 07:44:24.660 | INFO | "pinned" : " 29 GB" | 20260505 07:44:24.660 | INFO | } | 20260505 07:44:24.660 | INFO | Perf report for the last 5.13919 seconds: | 20260505 07:44:24.660 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:44:24.660 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:24.660 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:24.660 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:24.660 | INFO | ANALYSIS : 65.3%, 2.9, 99.3ms, 0 | 20260505 07:44:24.660 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:24.660 | INFO | RT_METRICS : 46.8%, 1.0, 24.5ms, 0 | 20260505 07:44:24.660 | INFO | PRE_HQ : 64.0%, 1.9, 62.7ms, 0 | 20260505 07:44:24.660 | INFO | BAZWRITER : 66.0%, 4.1, 140.2ms, 0 | 20260505 07:44:24.660 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 07:44:24.660 | INFO | Uploaded 36GiB to GPU in 2.2005s (16.3599GiB/s) | 20260505 07:44:24.660 | INFO | Downloaded 38GiB to GPU in 2.3455s (16.2013GiB/s) | 20260505 07:44:29.799 | INFO | Analyzing chunk frames = [1511936,1512448) | 20260505 07:44:29.799 | INFO | MemInfo: { | 20260505 07:44:29.799 | INFO | "gpu" : " 17 GB", | 20260505 07:44:29.799 | INFO | "huge" : "", | 20260505 07:44:29.799 | INFO | "huge_shared" : " 27 GB", | 20260505 07:44:29.799 | INFO | "malloc" : " 14 GB", | 20260505 07:44:29.799 | INFO | "pinned" : " 29 GB" | 20260505 07:44:29.799 | INFO | } | 20260505 07:44:29.799 | INFO | Perf report for the last 5.13936 seconds: | 20260505 07:44:29.799 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:44:29.799 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:29.799 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:29.799 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:29.799 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 07:44:29.799 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:29.799 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:29.799 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:44:29.799 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:29.799 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:29.800 | INFO | Uploaded 36GiB to GPU in 2.17743s (16.5332GiB/s) | 20260505 07:44:29.800 | INFO | Downloaded 34GiB to GPU in 2.05366s (16.5558GiB/s) | 20260505 07:44:34.939 | INFO | Analyzing chunk frames = [1512448,1512960) | 20260505 07:44:34.939 | INFO | MemInfo: { | 20260505 07:44:34.939 | INFO | "gpu" : " 17 GB", | 20260505 07:44:34.939 | INFO | "huge" : "", | 20260505 07:44:34.939 | INFO | "huge_shared" : " 27 GB", | 20260505 07:44:34.939 | INFO | "malloc" : " 14 GB", | 20260505 07:44:34.939 | INFO | "pinned" : " 29 GB" | 20260505 07:44:34.939 | INFO | } | 20260505 07:44:34.939 | INFO | Perf report for the last 5.13927 seconds: | 20260505 07:44:34.939 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:44:34.939 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:34.939 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:34.939 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:34.939 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 07:44:34.939 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:34.939 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:34.939 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:44:34.939 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:34.939 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:34.939 | INFO | Uploaded 36GiB to GPU in 2.14626s (16.7734GiB/s) | 20260505 07:44:34.939 | INFO | Downloaded 34GiB to GPU in 2.0549s (16.5458GiB/s) | 20260505 07:44:40.078 | INFO | Analyzing chunk frames = [1512960,1513472) | 20260505 07:44:40.078 | INFO | MemInfo: { | 20260505 07:44:40.078 | INFO | "gpu" : " 17 GB", | 20260505 07:44:40.078 | INFO | "huge" : "", | 20260505 07:44:40.078 | INFO | "huge_shared" : " 27 GB", | 20260505 07:44:40.078 | INFO | "malloc" : " 14 GB", | 20260505 07:44:40.078 | INFO | "pinned" : " 29 GB" | 20260505 07:44:40.078 | INFO | } | 20260505 07:44:40.078 | INFO | Perf report for the last 5.13947 seconds: | 20260505 07:44:40.078 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:44:40.078 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:40.078 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:40.078 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:40.078 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 07:44:40.078 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:40.078 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:40.078 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:44:40.078 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:40.078 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:40.078 | INFO | Uploaded 36GiB to GPU in 2.12042s (16.9778GiB/s) | 20260505 07:44:40.078 | INFO | Downloaded 34GiB to GPU in 2.0384s (16.6798GiB/s) | 20260505 07:44:45.217 | INFO | Analyzing chunk frames = [1513472,1513984) | 20260505 07:44:45.217 | INFO | MemInfo: { | 20260505 07:44:45.217 | INFO | "gpu" : " 17 GB", | 20260505 07:44:45.217 | INFO | "huge" : "", | 20260505 07:44:45.217 | INFO | "huge_shared" : " 27 GB", | 20260505 07:44:45.217 | INFO | "malloc" : " 14 GB", | 20260505 07:44:45.217 | INFO | "pinned" : " 29 GB" | 20260505 07:44:45.217 | INFO | } | 20260505 07:44:45.217 | INFO | Perf report for the last 5.13925 seconds: | 20260505 07:44:45.217 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:44:45.217 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:45.217 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:45.217 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:45.217 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 07:44:45.217 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:45.217 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:45.217 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:44:45.217 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:45.217 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:45.218 | INFO | Uploaded 36GiB to GPU in 2.19777s (16.3802GiB/s) | 20260505 07:44:45.218 | INFO | Downloaded 34GiB to GPU in 2.05496s (16.5454GiB/s) | 20260505 07:44:50.358 | INFO | Analyzing chunk frames = [1513984,1514496) | 20260505 07:44:50.358 | INFO | MemInfo: { | 20260505 07:44:50.358 | INFO | "gpu" : " 17 GB", | 20260505 07:44:50.358 | INFO | "huge" : "", | 20260505 07:44:50.358 | INFO | "huge_shared" : " 27 GB", | 20260505 07:44:50.358 | INFO | "malloc" : " 14 GB", | 20260505 07:44:50.358 | INFO | "pinned" : " 29 GB" | 20260505 07:44:50.358 | INFO | } | 20260505 07:44:50.358 | INFO | Perf report for the last 5.1411 seconds: | 20260505 07:44:50.359 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:44:50.359 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:50.359 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:50.359 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:50.359 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 07:44:50.359 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:50.359 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:50.359 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 07:44:50.359 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:50.359 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:50.359 | INFO | Uploaded 36GiB to GPU in 2.13991s (16.8232GiB/s) | 20260505 07:44:50.359 | INFO | Downloaded 34GiB to GPU in 2.05278s (16.5629GiB/s) | 20260505 07:44:55.496 | INFO | Analyzing chunk frames = [1514496,1515008) | 20260505 07:44:55.496 | INFO | MemInfo: { | 20260505 07:44:55.496 | INFO | "gpu" : " 17 GB", | 20260505 07:44:55.496 | INFO | "huge" : "", | 20260505 07:44:55.496 | INFO | "huge_shared" : " 27 GB", | 20260505 07:44:55.496 | INFO | "malloc" : " 14 GB", | 20260505 07:44:55.496 | INFO | "pinned" : " 29 GB" | 20260505 07:44:55.496 | INFO | } | 20260505 07:44:55.496 | INFO | Perf report for the last 5.13751 seconds: | 20260505 07:44:55.496 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:44:55.496 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:55.496 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:55.496 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:55.496 | INFO | ANALYSIS : 64.0%, 3.0, 99.8ms, 0 | 20260505 07:44:55.496 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:55.496 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:55.496 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 07:44:55.496 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:55.496 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:44:55.496 | INFO | Uploaded 36GiB to GPU in 2.17743s (16.5333GiB/s) | 20260505 07:44:55.496 | INFO | Downloaded 34GiB to GPU in 2.15252s (15.7954GiB/s) | 20260505 07:45:00.636 | INFO | Analyzing chunk frames = [1515008,1515520) | 20260505 07:45:00.636 | INFO | MemInfo: { | 20260505 07:45:00.636 | INFO | "gpu" : " 17 GB", | 20260505 07:45:00.636 | INFO | "huge" : "", | 20260505 07:45:00.636 | INFO | "huge_shared" : " 27 GB", | 20260505 07:45:00.636 | INFO | "malloc" : " 14 GB", | 20260505 07:45:00.636 | INFO | "pinned" : " 29 GB" | 20260505 07:45:00.636 | INFO | } | 20260505 07:45:00.636 | INFO | Perf report for the last 5.1402 seconds: | 20260505 07:45:00.636 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:45:00.636 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:00.636 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:00.636 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:00.636 | INFO | ANALYSIS : 64.7%, 3.0, 101.1ms, 0 | 20260505 07:45:00.636 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:00.636 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:00.636 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 07:45:00.636 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:00.636 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:00.636 | INFO | Uploaded 36GiB to GPU in 2.21546s (16.2495GiB/s) | 20260505 07:45:00.637 | INFO | Downloaded 34GiB to GPU in 2.04847s (16.5977GiB/s) | 20260505 07:45:00.637 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 07:45:00.637 | INFO | Total mem usage: 15.3 GiB | 20260505 07:45:00.637 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 07:45:00.637 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.22%) | 20260505 07:45:00.637 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 07:45:00.637 | INFO | High Water Breakdown: | 20260505 07:45:00.637 | INFO | 60.7% : 8.76 GiB ( 93.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 07:45:00.637 | INFO | 39.1% : 5.65 GiB ( 85% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 07:45:00.637 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 07:45:00.637 | INFO | ------------------------------------------ | 20260505 07:45:00.637 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:45:00.637 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 07:45:00.637 | INFO | Total mem usage: 26.5 GiB | 20260505 07:45:00.637 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 07:45:00.637 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 07:45:00.637 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 07:45:00.637 | INFO | High Water Breakdown: | 20260505 07:45:00.637 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:45:00.637 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:45:00.637 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:45:00.637 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:45:00.637 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:45:00.637 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:45:00.637 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 07:45:00.637 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 07:45:00.637 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:45:00.637 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 07:45:00.637 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 07:45:00.637 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 07:45:00.637 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 07:45:00.637 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 07:45:00.637 | INFO | ------------------------------------------ | 20260505 07:45:00.637 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:45:00.637 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 07:45:00.637 | INFO | Total mem usage: 25.6 GiB | 20260505 07:45:00.637 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 07:45:00.637 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 07:45:00.637 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 07:45:00.637 | INFO | High Water Breakdown: | 20260505 07:45:00.637 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 07:45:00.637 | INFO | ------------------------------------------ | 20260505 07:45:00.637 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:45:00.637 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 07:45:00.637 | INFO | Total mem usage: 15.7 GiB | 20260505 07:45:00.637 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 07:45:00.637 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 07:45:00.637 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 07:45:00.637 | INFO | High Water Breakdown: | 20260505 07:45:00.637 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 07:45:00.637 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 07:45:00.637 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 07:45:00.637 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 07:45:00.637 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 07:45:00.637 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 07:45:00.637 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 07:45:00.637 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 07:45:00.637 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 07:45:00.637 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:45:00.637 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 07:45:00.637 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:45:00.637 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 07:45:00.637 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 07:45:00.637 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 07:45:00.637 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 07:45:00.638 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:45:00.638 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 07:45:00.638 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:45:00.638 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:45:00.638 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:45:00.638 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 07:45:00.638 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 07:45:00.638 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:45:00.638 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 07:45:00.638 | INFO | ------------------------------------------ | 20260505 07:45:00.638 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:45:00.638 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 07:45:00.638 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:45:00.638 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:45:00.638 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.51e+01, 1.37e+01 / 2.16e+00 | 20260505 07:45:00.638 | INFO | PerfReport: 0.29%, AggregateStats -- 9.36e-02 / 1.04e-01, 1.08e-02 / 7.14e-03 | 20260505 07:45:00.638 | INFO | PerfReport: 11.64%, Baseline -- 3.82e+00 / 4.42e+00, 6.29e-01 / 3.10e-01 | 20260505 07:45:00.638 | INFO | PerfReport: 3.31%, Histogram -- 1.08e+00 / 1.47e+00, 2.14e-01 / 1.25e-01 | 20260505 07:45:00.638 | INFO | PerfReport: 15.48%, DME -- 4.06e+01 / 4.45e+01, 4.21e+00 / 1.45e+00 | 20260505 07:45:00.638 | INFO | PerfReport: 56.31%, FrameLabeling -- 1.85e+01 / 1.93e+01, 2.00e+00 / 1.45e+00 | 20260505 07:45:00.638 | INFO | PerfReport: 7.60%, PulseAccumulating -- 2.49e+00 / 3.16e+00, 4.44e-01 / 2.16e-01 | 20260505 07:45:00.638 | INFO | PerfReport: 3.95%, Metrics -- 1.30e+00 / 1.68e+00, 3.65e-01 / 1.56e-01 | 20260505 07:45:00.638 | INFO | PerfReport: 1.43%, PulseSerializing -- 4.68e-01 / 5.38e-01, 8.99e-02 / 2.74e-02 | 20260505 07:45:00.638 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 07:45:00.638 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:45:00.638 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:45:00.638 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.25e+01 / 0.00e+00 | 20260505 07:45:00.638 | INFO | PerfReport: 22.27%, Upload -- 2.26e+01 / 0.00e+00, 7.98e+00 / 0.00e+00 | 20260505 07:45:00.638 | INFO | PerfReport: 21.56%, Download -- 2.19e+01 / 0.00e+00, 7.98e+00 / 0.00e+00 | 20260505 07:45:00.638 | INFO | PerfReport: 32.36%, Compute -- 3.28e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 07:45:00.638 | INFO | PerfReport: 1.59%, LockUpload -- 1.62e+00 / 0.00e+00, 7.25e+00 / 0.00e+00 | 20260505 07:45:00.638 | INFO | PerfReport: 0.91%, LockDownload -- 9.23e-01 / 0.00e+00, 2.39e+00 / 0.00e+00 | 20260505 07:45:00.638 | INFO | PerfReport: 21.28%, LockCompute -- 2.16e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260505 07:45:00.638 | INFO | PerfReport: 0.01%, Misc -- 1.47e-02 / 0.00e+00, 2.03e-02 / 0.00e+00 | 20260505 07:45:00.768 | INFO | Starting a round of Baz IO | 20260505 07:45:04.057 | INFO | Populating RTMetrics report | 20260505 07:45:04.057 | INFO | RTMetrics report populated | 20260505 07:45:04.057 | INFO | Opening tmp RTMetrics json file | 20260505 07:45:04.064 | INFO | Serializing RTMetrics to json | 20260505 07:45:04.066 | INFO | Serialization done, swapping tmp file now... | 20260505 07:45:04.066 | INFO | RTMetrics done | 20260505 07:45:04.209 | INFO | Finished a round of Baz IO: Wrote 2.438 GiB in 3.44 seconds | 20260505 07:45:04.209 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 29.11% / 0.03% / 70.86% | 20260505 07:45:04.209 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.28311% / 3.57628e-05% / 6.35783e-05% | 20260505 07:45:04.209 | INFO | Baz IO overall: | 20260505 07:45:04.209 | INFO | BytesWritten : 835954627941 | 20260505 07:45:04.209 | INFO | NumEvents : 396886047647 | 20260505 07:45:05.775 | INFO | Analyzing chunk frames = [1515520,1516032) | 20260505 07:45:05.775 | INFO | MemInfo: { | 20260505 07:45:05.775 | INFO | "gpu" : " 17 GB", | 20260505 07:45:05.775 | INFO | "huge" : "", | 20260505 07:45:05.775 | INFO | "huge_shared" : " 27 GB", | 20260505 07:45:05.775 | INFO | "malloc" : " 14 GB", | 20260505 07:45:05.775 | INFO | "pinned" : " 29 GB" | 20260505 07:45:05.775 | INFO | } | 20260505 07:45:05.775 | INFO | Perf report for the last 5.1385 seconds: | 20260505 07:45:05.775 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:45:05.775 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:05.775 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:05.775 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:05.775 | INFO | ANALYSIS : 65.4%, 2.9, 99.4ms, 0 | 20260505 07:45:05.775 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:05.775 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260505 07:45:05.775 | INFO | PRE_HQ : 64.0%, 1.9, 62.9ms, 0 | 20260505 07:45:05.775 | INFO | BAZWRITER : 66.2%, 4.2, 142.1ms, 0 | 20260505 07:45:05.775 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 07:45:05.775 | INFO | Uploaded 36GiB to GPU in 2.21533s (16.2504GiB/s) | 20260505 07:45:05.775 | INFO | Downloaded 38GiB to GPU in 2.33547s (16.2708GiB/s) | 20260505 07:45:10.914 | INFO | Analyzing chunk frames = [1516032,1516544) | 20260505 07:45:10.914 | INFO | MemInfo: { | 20260505 07:45:10.914 | INFO | "gpu" : " 17 GB", | 20260505 07:45:10.914 | INFO | "huge" : "", | 20260505 07:45:10.914 | INFO | "huge_shared" : " 27 GB", | 20260505 07:45:10.914 | INFO | "malloc" : " 14 GB", | 20260505 07:45:10.914 | INFO | "pinned" : " 29 GB" | 20260505 07:45:10.914 | INFO | } | 20260505 07:45:10.914 | INFO | Perf report for the last 5.13937 seconds: | 20260505 07:45:10.914 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:45:10.914 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:10.914 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:10.914 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:10.914 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 07:45:10.914 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:10.914 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:10.914 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 07:45:10.914 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:10.914 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:10.914 | INFO | Uploaded 36GiB to GPU in 2.18715s (16.4598GiB/s) | 20260505 07:45:10.914 | INFO | Downloaded 34GiB to GPU in 2.05446s (16.5494GiB/s) | 20260505 07:45:16.053 | INFO | Analyzing chunk frames = [1516544,1517056) | 20260505 07:45:16.053 | INFO | MemInfo: { | 20260505 07:45:16.053 | INFO | "gpu" : " 17 GB", | 20260505 07:45:16.053 | INFO | "huge" : "", | 20260505 07:45:16.053 | INFO | "huge_shared" : " 27 GB", | 20260505 07:45:16.053 | INFO | "malloc" : " 14 GB", | 20260505 07:45:16.053 | INFO | "pinned" : " 29 GB" | 20260505 07:45:16.053 | INFO | } | 20260505 07:45:16.053 | INFO | Perf report for the last 5.13918 seconds: | 20260505 07:45:16.053 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:45:16.053 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:16.053 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:16.053 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:16.053 | INFO | ANALYSIS : 64.2%, 3.0, 100.0ms, 0 | 20260505 07:45:16.053 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:16.054 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:16.054 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 07:45:16.054 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:16.054 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:16.054 | INFO | Uploaded 36GiB to GPU in 2.1957s (16.3957GiB/s) | 20260505 07:45:16.054 | INFO | Downloaded 34GiB to GPU in 2.06109s (16.4961GiB/s) | 20260505 07:45:21.193 | INFO | Analyzing chunk frames = [1517056,1517568) | 20260505 07:45:21.193 | INFO | MemInfo: { | 20260505 07:45:21.193 | INFO | "gpu" : " 17 GB", | 20260505 07:45:21.193 | INFO | "huge" : "", | 20260505 07:45:21.193 | INFO | "huge_shared" : " 27 GB", | 20260505 07:45:21.193 | INFO | "malloc" : " 14 GB", | 20260505 07:45:21.193 | INFO | "pinned" : " 29 GB" | 20260505 07:45:21.193 | INFO | } | 20260505 07:45:21.193 | INFO | Perf report for the last 5.13966 seconds: | 20260505 07:45:21.193 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:45:21.193 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:21.193 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:21.193 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:21.193 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 07:45:21.193 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:21.193 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:21.193 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:45:21.193 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:21.193 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:21.193 | INFO | Uploaded 36GiB to GPU in 2.13209s (16.8848GiB/s) | 20260505 07:45:21.193 | INFO | Downloaded 34GiB to GPU in 2.03688s (16.6922GiB/s) | 20260505 07:45:26.332 | INFO | Analyzing chunk frames = [1517568,1518080) | 20260505 07:45:26.332 | INFO | MemInfo: { | 20260505 07:45:26.332 | INFO | "gpu" : " 17 GB", | 20260505 07:45:26.332 | INFO | "huge" : "", | 20260505 07:45:26.332 | INFO | "huge_shared" : " 27 GB", | 20260505 07:45:26.332 | INFO | "malloc" : " 14 GB", | 20260505 07:45:26.332 | INFO | "pinned" : " 29 GB" | 20260505 07:45:26.332 | INFO | } | 20260505 07:45:26.332 | INFO | Perf report for the last 5.13946 seconds: | 20260505 07:45:26.332 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:45:26.332 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:26.332 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:26.332 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:26.332 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 07:45:26.332 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:26.332 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:26.332 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:45:26.332 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:26.332 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:26.333 | INFO | Uploaded 36GiB to GPU in 2.19278s (16.4175GiB/s) | 20260505 07:45:26.333 | INFO | Downloaded 34GiB to GPU in 2.05389s (16.554GiB/s) | 20260505 07:45:31.473 | INFO | Analyzing chunk frames = [1518080,1518592) | 20260505 07:45:31.473 | INFO | MemInfo: { | 20260505 07:45:31.473 | INFO | "gpu" : " 17 GB", | 20260505 07:45:31.473 | INFO | "huge" : "", | 20260505 07:45:31.473 | INFO | "huge_shared" : " 27 GB", | 20260505 07:45:31.473 | INFO | "malloc" : " 14 GB", | 20260505 07:45:31.473 | INFO | "pinned" : " 29 GB" | 20260505 07:45:31.473 | INFO | } | 20260505 07:45:31.473 | INFO | Perf report for the last 5.14063 seconds: | 20260505 07:45:31.473 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:45:31.473 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:31.473 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:31.473 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:31.473 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 07:45:31.473 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:31.473 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:31.473 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 07:45:31.473 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:31.473 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:31.473 | INFO | Uploaded 36GiB to GPU in 2.15835s (16.6794GiB/s) | 20260505 07:45:31.473 | INFO | Downloaded 34GiB to GPU in 2.0579s (16.5217GiB/s) | 20260505 07:45:36.611 | INFO | Analyzing chunk frames = [1518592,1519104) | 20260505 07:45:36.611 | INFO | MemInfo: { | 20260505 07:45:36.611 | INFO | "gpu" : " 17 GB", | 20260505 07:45:36.611 | INFO | "huge" : "", | 20260505 07:45:36.611 | INFO | "huge_shared" : " 27 GB", | 20260505 07:45:36.611 | INFO | "malloc" : " 14 GB", | 20260505 07:45:36.611 | INFO | "pinned" : " 29 GB" | 20260505 07:45:36.611 | INFO | } | 20260505 07:45:36.611 | INFO | Perf report for the last 5.13786 seconds: | 20260505 07:45:36.611 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:45:36.611 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:36.611 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:36.611 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:36.611 | INFO | ANALYSIS : 64.1%, 3.0, 100.0ms, 0 | 20260505 07:45:36.611 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:36.611 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:36.611 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 07:45:36.611 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:36.611 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:36.611 | INFO | Uploaded 36GiB to GPU in 2.19589s (16.3943GiB/s) | 20260505 07:45:36.611 | INFO | Downloaded 34GiB to GPU in 2.149s (15.8213GiB/s) | 20260505 07:45:41.750 | INFO | Analyzing chunk frames = [1519104,1519616) | 20260505 07:45:41.750 | INFO | MemInfo: { | 20260505 07:45:41.750 | INFO | "gpu" : " 17 GB", | 20260505 07:45:41.751 | INFO | "huge" : "", | 20260505 07:45:41.751 | INFO | "huge_shared" : " 27 GB", | 20260505 07:45:41.751 | INFO | "malloc" : " 14 GB", | 20260505 07:45:41.751 | INFO | "pinned" : " 29 GB" | 20260505 07:45:41.751 | INFO | } | 20260505 07:45:41.751 | INFO | Perf report for the last 5.13975 seconds: | 20260505 07:45:41.751 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:45:41.751 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:41.751 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:41.751 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:41.751 | INFO | ANALYSIS : 64.7%, 3.0, 101.1ms, 0 | 20260505 07:45:41.751 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:41.751 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:41.751 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 07:45:41.751 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:41.751 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:41.751 | INFO | Uploaded 36GiB to GPU in 2.21585s (16.2466GiB/s) | 20260505 07:45:41.751 | INFO | Downloaded 34GiB to GPU in 2.04643s (16.6143GiB/s) | 20260505 07:45:41.886 | INFO | Starting a round of Baz IO | 20260505 07:45:45.165 | INFO | Populating RTMetrics report | 20260505 07:45:45.166 | INFO | RTMetrics report populated | 20260505 07:45:45.166 | INFO | Opening tmp RTMetrics json file | 20260505 07:45:45.168 | INFO | Serializing RTMetrics to json | 20260505 07:45:45.171 | INFO | Serialization done, swapping tmp file now... | 20260505 07:45:45.173 | INFO | RTMetrics done | 20260505 07:45:45.314 | INFO | Finished a round of Baz IO: Wrote 2.44 GiB in 3.428 seconds | 20260505 07:45:45.314 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 29.08% / 0.03% / 70.89% | 20260505 07:45:45.314 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.27938% / 3.17891e-05% / 0.000107288% | 20260505 07:45:45.314 | INFO | Baz IO overall: | 20260505 07:45:45.314 | INFO | BytesWritten : 838574367596 | 20260505 07:45:45.314 | INFO | NumEvents : 398139496787 | 20260505 07:45:46.890 | INFO | Analyzing chunk frames = [1519616,1520128) | 20260505 07:45:46.890 | INFO | MemInfo: { | 20260505 07:45:46.890 | INFO | "gpu" : " 17 GB", | 20260505 07:45:46.890 | INFO | "huge" : "", | 20260505 07:45:46.890 | INFO | "huge_shared" : " 27 GB", | 20260505 07:45:46.890 | INFO | "malloc" : " 14 GB", | 20260505 07:45:46.890 | INFO | "pinned" : " 29 GB" | 20260505 07:45:46.890 | INFO | } | 20260505 07:45:46.890 | INFO | Perf report for the last 5.13904 seconds: | 20260505 07:45:46.890 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:45:46.890 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:46.890 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:46.890 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:46.890 | INFO | ANALYSIS : 65.3%, 2.9, 99.4ms, 0 | 20260505 07:45:46.890 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:46.890 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260505 07:45:46.890 | INFO | PRE_HQ : 64.0%, 1.9, 62.5ms, 0 | 20260505 07:45:46.890 | INFO | BAZWRITER : 66.0%, 4.2, 141.8ms, 0 | 20260505 07:45:46.890 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 07:45:46.890 | INFO | Uploaded 36GiB to GPU in 2.20359s (16.337GiB/s) | 20260505 07:45:46.890 | INFO | Downloaded 38GiB to GPU in 2.34927s (16.1752GiB/s) | 20260505 07:45:52.029 | INFO | Analyzing chunk frames = [1520128,1520640) | 20260505 07:45:52.029 | INFO | MemInfo: { | 20260505 07:45:52.029 | INFO | "gpu" : " 17 GB", | 20260505 07:45:52.029 | INFO | "huge" : "", | 20260505 07:45:52.029 | INFO | "huge_shared" : " 27 GB", | 20260505 07:45:52.029 | INFO | "malloc" : " 14 GB", | 20260505 07:45:52.029 | INFO | "pinned" : " 29 GB" | 20260505 07:45:52.029 | INFO | } | 20260505 07:45:52.029 | INFO | Perf report for the last 5.13922 seconds: | 20260505 07:45:52.029 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:45:52.029 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:52.029 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:52.029 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:52.029 | INFO | ANALYSIS : 64.3%, 3.0, 100.5ms, 0 | 20260505 07:45:52.029 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:52.029 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:52.029 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:45:52.029 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:52.029 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:52.029 | INFO | Uploaded 36GiB to GPU in 2.17878s (16.523GiB/s) | 20260505 07:45:52.029 | INFO | Downloaded 34GiB to GPU in 2.0522s (16.5676GiB/s) | 20260505 07:45:57.168 | INFO | Analyzing chunk frames = [1520640,1521152) | 20260505 07:45:57.168 | INFO | MemInfo: { | 20260505 07:45:57.168 | INFO | "gpu" : " 17 GB", | 20260505 07:45:57.168 | INFO | "huge" : "", | 20260505 07:45:57.168 | INFO | "huge_shared" : " 27 GB", | 20260505 07:45:57.168 | INFO | "malloc" : " 14 GB", | 20260505 07:45:57.168 | INFO | "pinned" : " 29 GB" | 20260505 07:45:57.168 | INFO | } | 20260505 07:45:57.168 | INFO | Perf report for the last 5.13927 seconds: | 20260505 07:45:57.168 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:45:57.168 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:57.168 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:57.168 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:57.168 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 07:45:57.168 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:57.168 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:57.168 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:45:57.168 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:57.168 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:45:57.168 | INFO | Uploaded 36GiB to GPU in 2.14249s (16.8028GiB/s) | 20260505 07:45:57.168 | INFO | Downloaded 34GiB to GPU in 2.05611s (16.5361GiB/s) | 20260505 07:46:02.307 | INFO | Analyzing chunk frames = [1521152,1521664) | 20260505 07:46:02.307 | INFO | MemInfo: { | 20260505 07:46:02.307 | INFO | "gpu" : " 17 GB", | 20260505 07:46:02.307 | INFO | "huge" : "", | 20260505 07:46:02.307 | INFO | "huge_shared" : " 27 GB", | 20260505 07:46:02.307 | INFO | "malloc" : " 14 GB", | 20260505 07:46:02.307 | INFO | "pinned" : " 29 GB" | 20260505 07:46:02.307 | INFO | } | 20260505 07:46:02.307 | INFO | Perf report for the last 5.13919 seconds: | 20260505 07:46:02.307 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:46:02.307 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:02.307 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:02.307 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:02.308 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 07:46:02.308 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:02.308 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:02.308 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:46:02.308 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:02.308 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:02.308 | INFO | Uploaded 36GiB to GPU in 2.15604s (16.6973GiB/s) | 20260505 07:46:02.308 | INFO | Downloaded 34GiB to GPU in 2.04209s (16.6496GiB/s) | 20260505 07:46:07.447 | INFO | Analyzing chunk frames = [1521664,1522176) | 20260505 07:46:07.447 | INFO | MemInfo: { | 20260505 07:46:07.447 | INFO | "gpu" : " 17 GB", | 20260505 07:46:07.447 | INFO | "huge" : "", | 20260505 07:46:07.447 | INFO | "huge_shared" : " 27 GB", | 20260505 07:46:07.447 | INFO | "malloc" : " 14 GB", | 20260505 07:46:07.447 | INFO | "pinned" : " 29 GB" | 20260505 07:46:07.447 | INFO | } | 20260505 07:46:07.447 | INFO | Perf report for the last 5.13955 seconds: | 20260505 07:46:07.447 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:46:07.447 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:07.447 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:07.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:07.447 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 07:46:07.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:07.447 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:07.447 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 07:46:07.447 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:07.447 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:07.447 | INFO | Uploaded 36GiB to GPU in 2.2012s (16.3547GiB/s) | 20260505 07:46:07.447 | INFO | Downloaded 34GiB to GPU in 2.06116s (16.4955GiB/s) | 20260505 07:46:12.588 | INFO | Analyzing chunk frames = [1522176,1522688) | 20260505 07:46:12.588 | INFO | MemInfo: { | 20260505 07:46:12.588 | INFO | "gpu" : " 17 GB", | 20260505 07:46:12.588 | INFO | "huge" : "", | 20260505 07:46:12.588 | INFO | "huge_shared" : " 27 GB", | 20260505 07:46:12.588 | INFO | "malloc" : " 14 GB", | 20260505 07:46:12.588 | INFO | "pinned" : " 29 GB" | 20260505 07:46:12.588 | INFO | } | 20260505 07:46:12.588 | INFO | Perf report for the last 5.14109 seconds: | 20260505 07:46:12.588 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:46:12.588 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:12.588 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:12.588 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:12.588 | INFO | ANALYSIS : 63.2%, 3.0, 98.8ms, 0 | 20260505 07:46:12.588 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:12.588 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:12.588 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 07:46:12.588 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:12.588 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:12.588 | INFO | Uploaded 36GiB to GPU in 2.14317s (16.7976GiB/s) | 20260505 07:46:12.588 | INFO | Downloaded 34GiB to GPU in 2.05545s (16.5414GiB/s) | 20260505 07:46:17.725 | INFO | Analyzing chunk frames = [1522688,1523200) | 20260505 07:46:17.725 | INFO | MemInfo: { | 20260505 07:46:17.725 | INFO | "gpu" : " 17 GB", | 20260505 07:46:17.725 | INFO | "huge" : "", | 20260505 07:46:17.725 | INFO | "huge_shared" : " 27 GB", | 20260505 07:46:17.725 | INFO | "malloc" : " 14 GB", | 20260505 07:46:17.725 | INFO | "pinned" : " 29 GB" | 20260505 07:46:17.725 | INFO | } | 20260505 07:46:17.725 | INFO | Perf report for the last 5.13736 seconds: | 20260505 07:46:17.725 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:46:17.725 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:17.725 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:17.726 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:17.726 | INFO | ANALYSIS : 64.1%, 3.0, 100.0ms, 0 | 20260505 07:46:17.726 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:17.726 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:17.726 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 07:46:17.726 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:17.726 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:17.726 | INFO | Uploaded 36GiB to GPU in 2.17728s (16.5344GiB/s) | 20260505 07:46:17.726 | INFO | Downloaded 34GiB to GPU in 2.14073s (15.8824GiB/s) | 20260505 07:46:22.865 | INFO | Analyzing chunk frames = [1523200,1523712) | 20260505 07:46:22.865 | INFO | MemInfo: { | 20260505 07:46:22.865 | INFO | "gpu" : " 17 GB", | 20260505 07:46:22.865 | INFO | "huge" : "", | 20260505 07:46:22.865 | INFO | "huge_shared" : " 27 GB", | 20260505 07:46:22.865 | INFO | "malloc" : " 14 GB", | 20260505 07:46:22.865 | INFO | "pinned" : " 29 GB" | 20260505 07:46:22.865 | INFO | } | 20260505 07:46:22.865 | INFO | Perf report for the last 5.13948 seconds: | 20260505 07:46:22.865 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:46:22.865 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:22.865 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:22.865 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:22.865 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 07:46:22.865 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:22.865 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:22.865 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 07:46:22.865 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:22.865 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:22.865 | INFO | Uploaded 36GiB to GPU in 2.15522s (16.7036GiB/s) | 20260505 07:46:22.865 | INFO | Downloaded 34GiB to GPU in 2.04786s (16.6027GiB/s) | 20260505 07:46:22.865 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 07:46:22.865 | INFO | Total mem usage: 15.3 GiB | 20260505 07:46:22.865 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 07:46:22.865 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.2%) | 20260505 07:46:22.865 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 07:46:22.865 | INFO | High Water Breakdown: | 20260505 07:46:22.865 | INFO | 60.7% : 8.76 GiB ( 93.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 07:46:22.865 | INFO | 39.1% : 5.65 GiB ( 85% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 07:46:22.865 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 07:46:22.865 | INFO | ------------------------------------------ | 20260505 07:46:22.865 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:46:22.865 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 07:46:22.865 | INFO | Total mem usage: 26.5 GiB | 20260505 07:46:22.865 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 07:46:22.865 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 07:46:22.865 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 07:46:22.865 | INFO | High Water Breakdown: | 20260505 07:46:22.865 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:46:22.866 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:46:22.866 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:46:22.866 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:46:22.866 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:46:22.866 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:46:22.866 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 07:46:22.866 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 07:46:22.866 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:46:22.866 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 07:46:22.866 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 07:46:22.866 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 07:46:22.866 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 07:46:22.866 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 07:46:22.866 | INFO | ------------------------------------------ | 20260505 07:46:22.866 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:46:22.866 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 07:46:22.866 | INFO | Total mem usage: 25.6 GiB | 20260505 07:46:22.866 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 07:46:22.866 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 07:46:22.866 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 07:46:22.866 | INFO | High Water Breakdown: | 20260505 07:46:22.866 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 07:46:22.866 | INFO | ------------------------------------------ | 20260505 07:46:22.866 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:46:22.866 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 07:46:22.866 | INFO | Total mem usage: 15.7 GiB | 20260505 07:46:22.866 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 07:46:22.866 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 07:46:22.866 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 07:46:22.866 | INFO | High Water Breakdown: | 20260505 07:46:22.866 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 07:46:22.866 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 07:46:22.866 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 07:46:22.866 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 07:46:22.866 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 07:46:22.866 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 07:46:22.866 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 07:46:22.866 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 07:46:22.866 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 07:46:22.866 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:46:22.866 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 07:46:22.866 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:46:22.866 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 07:46:22.866 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 07:46:22.866 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 07:46:22.866 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 07:46:22.866 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:46:22.866 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 07:46:22.866 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:46:22.866 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:46:22.866 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:46:22.866 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 07:46:22.866 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 07:46:22.866 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:46:22.866 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 07:46:22.866 | INFO | ------------------------------------------ | 20260505 07:46:22.866 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:46:22.866 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 07:46:22.866 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:46:22.866 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:46:22.866 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.53e+01, 1.36e+01 / 2.13e+00 | 20260505 07:46:22.866 | INFO | PerfReport: 0.28%, AggregateStats -- 9.36e-02 / 1.03e-01, 1.06e-02 / 5.37e-03 | 20260505 07:46:22.866 | INFO | PerfReport: 11.58%, Baseline -- 3.81e+00 / 4.39e+00, 6.21e-01 / 3.11e-01 | 20260505 07:46:22.866 | INFO | PerfReport: 3.30%, Histogram -- 1.08e+00 / 1.46e+00, 2.12e-01 / 1.19e-01 | 20260505 07:46:22.866 | INFO | PerfReport: 15.48%, DME -- 4.07e+01 / 4.47e+01, 4.20e+00 / 1.44e+00 | 20260505 07:46:22.866 | INFO | PerfReport: 56.44%, FrameLabeling -- 1.85e+01 / 1.93e+01, 2.06e+00 / 1.38e+00 | 20260505 07:46:22.866 | INFO | PerfReport: 7.55%, PulseAccumulating -- 2.48e+00 / 3.15e+00, 4.36e-01 / 2.22e-01 | 20260505 07:46:22.866 | INFO | PerfReport: 3.94%, Metrics -- 1.29e+00 / 1.68e+00, 3.69e-01 / 1.48e-01 | 20260505 07:46:22.866 | INFO | PerfReport: 1.42%, PulseSerializing -- 4.68e-01 / 5.36e-01, 9.10e-02 / 3.27e-02 | 20260505 07:46:22.866 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 07:46:22.866 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:46:22.866 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:46:22.866 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.28e+01 / 0.00e+00 | 20260505 07:46:22.866 | INFO | PerfReport: 22.33%, Upload -- 2.27e+01 / 0.00e+00, 8.11e+00 / 0.00e+00 | 20260505 07:46:22.866 | INFO | PerfReport: 21.53%, Download -- 2.19e+01 / 0.00e+00, 8.03e+00 / 0.00e+00 | 20260505 07:46:22.866 | INFO | PerfReport: 32.34%, Compute -- 3.29e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 07:46:22.866 | INFO | PerfReport: 1.63%, LockUpload -- 1.66e+00 / 0.00e+00, 7.28e+00 / 0.00e+00 | 20260505 07:46:22.866 | INFO | PerfReport: 0.85%, LockDownload -- 8.64e-01 / 0.00e+00, 2.29e+00 / 0.00e+00 | 20260505 07:46:22.866 | INFO | PerfReport: 21.30%, LockCompute -- 2.16e+01 / 0.00e+00, 1.72e+01 / 0.00e+00 | 20260505 07:46:22.866 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.53e-02 / 0.00e+00 | 20260505 07:46:22.999 | INFO | Starting a round of Baz IO | 20260505 07:46:26.282 | INFO | Populating RTMetrics report | 20260505 07:46:26.283 | INFO | RTMetrics report populated | 20260505 07:46:26.283 | INFO | Opening tmp RTMetrics json file | 20260505 07:46:26.296 | INFO | Serializing RTMetrics to json | 20260505 07:46:26.297 | INFO | Serialization done, swapping tmp file now... | 20260505 07:46:26.298 | INFO | RTMetrics done | 20260505 07:46:26.446 | INFO | Finished a round of Baz IO: Wrote 2.441 GiB in 3.446 seconds | 20260505 07:46:26.446 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 29.06% / 0.03% / 70.92% | 20260505 07:46:26.446 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.26687% / 4.37101e-05% / 7.94729e-05% | 20260505 07:46:26.446 | INFO | Baz IO overall: | 20260505 07:46:26.446 | INFO | BytesWritten : 841195039917 | 20260505 07:46:26.446 | INFO | NumEvents : 399393468220 | 20260505 07:46:28.004 | INFO | Analyzing chunk frames = [1523712,1524224) | 20260505 07:46:28.004 | INFO | MemInfo: { | 20260505 07:46:28.004 | INFO | "gpu" : " 17 GB", | 20260505 07:46:28.004 | INFO | "huge" : "", | 20260505 07:46:28.004 | INFO | "huge_shared" : " 27 GB", | 20260505 07:46:28.004 | INFO | "malloc" : " 14 GB", | 20260505 07:46:28.004 | INFO | "pinned" : " 29 GB" | 20260505 07:46:28.004 | INFO | } | 20260505 07:46:28.004 | INFO | Perf report for the last 5.13927 seconds: | 20260505 07:46:28.004 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:46:28.004 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:28.004 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:28.004 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:28.004 | INFO | ANALYSIS : 65.4%, 2.9, 99.7ms, 0 | 20260505 07:46:28.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:28.004 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260505 07:46:28.004 | INFO | PRE_HQ : 64.1%, 1.9, 62.4ms, 0 | 20260505 07:46:28.004 | INFO | BAZWRITER : 66.3%, 4.2, 142.1ms, 0 | 20260505 07:46:28.004 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 07:46:28.005 | INFO | Uploaded 36GiB to GPU in 2.221s (16.2089GiB/s) | 20260505 07:46:28.005 | INFO | Downloaded 38GiB to GPU in 2.34561s (16.2005GiB/s) | 20260505 07:46:33.143 | INFO | Analyzing chunk frames = [1524224,1524736) | 20260505 07:46:33.144 | INFO | MemInfo: { | 20260505 07:46:33.144 | INFO | "gpu" : " 17 GB", | 20260505 07:46:33.144 | INFO | "huge" : "", | 20260505 07:46:33.144 | INFO | "huge_shared" : " 27 GB", | 20260505 07:46:33.144 | INFO | "malloc" : " 14 GB", | 20260505 07:46:33.144 | INFO | "pinned" : " 29 GB" | 20260505 07:46:33.144 | INFO | } | 20260505 07:46:33.144 | INFO | Perf report for the last 5.13939 seconds: | 20260505 07:46:33.144 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:46:33.144 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:33.144 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:33.144 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:33.144 | INFO | ANALYSIS : 64.3%, 3.0, 100.3ms, 0 | 20260505 07:46:33.144 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:33.144 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:33.144 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:46:33.144 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:33.144 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:33.144 | INFO | Uploaded 36GiB to GPU in 2.18146s (16.5027GiB/s) | 20260505 07:46:33.144 | INFO | Downloaded 34GiB to GPU in 2.05423s (16.5512GiB/s) | 20260505 07:46:38.283 | INFO | Analyzing chunk frames = [1524736,1525248) | 20260505 07:46:38.283 | INFO | MemInfo: { | 20260505 07:46:38.283 | INFO | "gpu" : " 17 GB", | 20260505 07:46:38.283 | INFO | "huge" : "", | 20260505 07:46:38.283 | INFO | "huge_shared" : " 27 GB", | 20260505 07:46:38.284 | INFO | "malloc" : " 14 GB", | 20260505 07:46:38.284 | INFO | "pinned" : " 29 GB" | 20260505 07:46:38.284 | INFO | } | 20260505 07:46:38.284 | INFO | Perf report for the last 5.13992 seconds: | 20260505 07:46:38.284 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:46:38.284 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:38.284 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:38.284 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:38.284 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 07:46:38.284 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:38.284 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:38.284 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 07:46:38.284 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:38.284 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:38.284 | INFO | Uploaded 36GiB to GPU in 2.15063s (16.7393GiB/s) | 20260505 07:46:38.284 | INFO | Downloaded 34GiB to GPU in 2.05709s (16.5282GiB/s) | 20260505 07:46:43.423 | INFO | Analyzing chunk frames = [1525248,1525760) | 20260505 07:46:43.423 | INFO | MemInfo: { | 20260505 07:46:43.423 | INFO | "gpu" : " 17 GB", | 20260505 07:46:43.423 | INFO | "huge" : "", | 20260505 07:46:43.423 | INFO | "huge_shared" : " 27 GB", | 20260505 07:46:43.423 | INFO | "malloc" : " 14 GB", | 20260505 07:46:43.423 | INFO | "pinned" : " 29 GB" | 20260505 07:46:43.423 | INFO | } | 20260505 07:46:43.423 | INFO | Perf report for the last 5.13965 seconds: | 20260505 07:46:43.423 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:46:43.423 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:43.423 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:43.423 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:43.423 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 07:46:43.423 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:43.423 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:43.423 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 07:46:43.423 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:43.423 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:43.423 | INFO | Uploaded 36GiB to GPU in 2.16505s (16.6278GiB/s) | 20260505 07:46:43.424 | INFO | Downloaded 34GiB to GPU in 2.04696s (16.61GiB/s) | 20260505 07:46:48.562 | INFO | Analyzing chunk frames = [1525760,1526272) | 20260505 07:46:48.562 | INFO | MemInfo: { | 20260505 07:46:48.562 | INFO | "gpu" : " 17 GB", | 20260505 07:46:48.562 | INFO | "huge" : "", | 20260505 07:46:48.562 | INFO | "huge_shared" : " 27 GB", | 20260505 07:46:48.562 | INFO | "malloc" : " 14 GB", | 20260505 07:46:48.562 | INFO | "pinned" : " 29 GB" | 20260505 07:46:48.562 | INFO | } | 20260505 07:46:48.562 | INFO | Perf report for the last 5.13905 seconds: | 20260505 07:46:48.562 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:46:48.562 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:48.562 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:48.562 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:48.562 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 07:46:48.562 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:48.562 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:48.562 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 07:46:48.562 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:48.562 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:48.563 | INFO | Uploaded 36GiB to GPU in 2.2035s (16.3376GiB/s) | 20260505 07:46:48.563 | INFO | Downloaded 34GiB to GPU in 2.0515s (16.5732GiB/s) | 20260505 07:46:53.702 | INFO | Analyzing chunk frames = [1526272,1526784) | 20260505 07:46:53.702 | INFO | MemInfo: { | 20260505 07:46:53.702 | INFO | "gpu" : " 17 GB", | 20260505 07:46:53.702 | INFO | "huge" : "", | 20260505 07:46:53.702 | INFO | "huge_shared" : " 27 GB", | 20260505 07:46:53.702 | INFO | "malloc" : " 14 GB", | 20260505 07:46:53.702 | INFO | "pinned" : " 29 GB" | 20260505 07:46:53.702 | INFO | } | 20260505 07:46:53.703 | INFO | Perf report for the last 5.1402 seconds: | 20260505 07:46:53.703 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:46:53.703 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:53.703 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:53.703 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:53.703 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 07:46:53.703 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:53.703 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:53.703 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 07:46:53.703 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:53.703 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:53.703 | INFO | Uploaded 36GiB to GPU in 2.19849s (16.3748GiB/s) | 20260505 07:46:53.703 | INFO | Downloaded 34GiB to GPU in 2.04794s (16.6021GiB/s) | 20260505 07:46:58.840 | INFO | Analyzing chunk frames = [1526784,1527296) | 20260505 07:46:58.840 | INFO | MemInfo: { | 20260505 07:46:58.840 | INFO | "gpu" : " 17 GB", | 20260505 07:46:58.840 | INFO | "huge" : "", | 20260505 07:46:58.841 | INFO | "huge_shared" : " 27 GB", | 20260505 07:46:58.841 | INFO | "malloc" : " 14 GB", | 20260505 07:46:58.841 | INFO | "pinned" : " 29 GB" | 20260505 07:46:58.841 | INFO | } | 20260505 07:46:58.841 | INFO | Perf report for the last 5.13812 seconds: | 20260505 07:46:58.841 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:46:58.841 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:58.841 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:58.841 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:58.841 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 07:46:58.841 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:58.841 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:58.841 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 07:46:58.841 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:58.841 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:46:58.841 | INFO | Uploaded 36GiB to GPU in 2.16669s (16.6152GiB/s) | 20260505 07:46:58.841 | INFO | Downloaded 34GiB to GPU in 2.13666s (15.9127GiB/s) | 20260505 07:47:03.980 | INFO | Analyzing chunk frames = [1527296,1527808) | 20260505 07:47:03.980 | INFO | MemInfo: { | 20260505 07:47:03.980 | INFO | "gpu" : " 17 GB", | 20260505 07:47:03.980 | INFO | "huge" : "", | 20260505 07:47:03.980 | INFO | "huge_shared" : " 27 GB", | 20260505 07:47:03.980 | INFO | "malloc" : " 14 GB", | 20260505 07:47:03.980 | INFO | "pinned" : " 29 GB" | 20260505 07:47:03.980 | INFO | } | 20260505 07:47:03.980 | INFO | Perf report for the last 5.13925 seconds: | 20260505 07:47:03.980 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:47:03.980 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:03.980 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:03.980 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:03.980 | INFO | ANALYSIS : 64.8%, 3.0, 101.2ms, 0 | 20260505 07:47:03.980 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:03.980 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:03.980 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 07:47:03.980 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:03.980 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:03.980 | INFO | Uploaded 36GiB to GPU in 2.22086s (16.21GiB/s) | 20260505 07:47:03.980 | INFO | Downloaded 34GiB to GPU in 2.04188s (16.6514GiB/s) | 20260505 07:47:04.113 | INFO | Starting a round of Baz IO | 20260505 07:47:07.396 | INFO | Populating RTMetrics report | 20260505 07:47:07.397 | INFO | RTMetrics report populated | 20260505 07:47:07.397 | INFO | Opening tmp RTMetrics json file | 20260505 07:47:07.460 | INFO | Serializing RTMetrics to json | 20260505 07:47:07.461 | INFO | Serialization done, swapping tmp file now... | 20260505 07:47:07.461 | INFO | RTMetrics done | 20260505 07:47:07.564 | INFO | Finished a round of Baz IO: Wrote 2.441 GiB in 3.45 seconds | 20260505 07:47:07.564 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 29.03% / 0.03% / 70.94% | 20260505 07:47:07.564 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.25678% / 3.97364e-05% / 6.35783e-05% | 20260505 07:47:07.564 | INFO | Baz IO overall: | 20260505 07:47:07.564 | INFO | BytesWritten : 843815994397 | 20260505 07:47:07.564 | INFO | NumEvents : 400647376052 | 20260505 07:47:09.119 | INFO | Analyzing chunk frames = [1527808,1528320) | 20260505 07:47:09.119 | INFO | MemInfo: { | 20260505 07:47:09.119 | INFO | "gpu" : " 17 GB", | 20260505 07:47:09.119 | INFO | "huge" : "", | 20260505 07:47:09.119 | INFO | "huge_shared" : " 27 GB", | 20260505 07:47:09.119 | INFO | "malloc" : " 14 GB", | 20260505 07:47:09.119 | INFO | "pinned" : " 29 GB" | 20260505 07:47:09.119 | INFO | } | 20260505 07:47:09.119 | INFO | Perf report for the last 5.13908 seconds: | 20260505 07:47:09.119 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:47:09.119 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:09.119 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:09.119 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:09.119 | INFO | ANALYSIS : 65.3%, 2.9, 99.5ms, 0 | 20260505 07:47:09.119 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:09.119 | INFO | RT_METRICS : 47.2%, 1.0, 24.8ms, 0 | 20260505 07:47:09.119 | INFO | PRE_HQ : 63.9%, 1.9, 62.2ms, 0 | 20260505 07:47:09.119 | INFO | BAZWRITER : 66.4%, 4.2, 141.9ms, 0 | 20260505 07:47:09.119 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 07:47:09.119 | INFO | Uploaded 36GiB to GPU in 2.2166s (16.2411GiB/s) | 20260505 07:47:09.119 | INFO | Downloaded 38GiB to GPU in 2.3442s (16.2102GiB/s) | 20260505 07:47:14.259 | INFO | Analyzing chunk frames = [1528320,1528832) | 20260505 07:47:14.259 | INFO | MemInfo: { | 20260505 07:47:14.259 | INFO | "gpu" : " 17 GB", | 20260505 07:47:14.259 | INFO | "huge" : "", | 20260505 07:47:14.259 | INFO | "huge_shared" : " 27 GB", | 20260505 07:47:14.259 | INFO | "malloc" : " 14 GB", | 20260505 07:47:14.259 | INFO | "pinned" : " 29 GB" | 20260505 07:47:14.259 | INFO | } | 20260505 07:47:14.259 | INFO | Perf report for the last 5.13981 seconds: | 20260505 07:47:14.259 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:47:14.259 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:14.259 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:14.259 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:14.259 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 07:47:14.259 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:14.259 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:14.259 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 07:47:14.259 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:14.259 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:14.259 | INFO | Uploaded 36GiB to GPU in 2.18165s (16.5013GiB/s) | 20260505 07:47:14.259 | INFO | Downloaded 34GiB to GPU in 2.05226s (16.5671GiB/s) | 20260505 07:47:19.398 | INFO | Analyzing chunk frames = [1528832,1529344) | 20260505 07:47:19.398 | INFO | MemInfo: { | 20260505 07:47:19.398 | INFO | "gpu" : " 17 GB", | 20260505 07:47:19.398 | INFO | "huge" : "", | 20260505 07:47:19.398 | INFO | "huge_shared" : " 27 GB", | 20260505 07:47:19.399 | INFO | "malloc" : " 14 GB", | 20260505 07:47:19.399 | INFO | "pinned" : " 29 GB" | 20260505 07:47:19.399 | INFO | } | 20260505 07:47:19.399 | INFO | Perf report for the last 5.13994 seconds: | 20260505 07:47:19.399 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:47:19.399 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:19.399 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:19.399 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:19.399 | INFO | ANALYSIS : 64.2%, 3.0, 99.9ms, 0 | 20260505 07:47:19.399 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:19.399 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:19.399 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:47:19.399 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:19.399 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:19.399 | INFO | Uploaded 36GiB to GPU in 2.1997s (16.3659GiB/s) | 20260505 07:47:19.399 | INFO | Downloaded 34GiB to GPU in 2.07124s (16.4153GiB/s) | 20260505 07:47:24.537 | INFO | Analyzing chunk frames = [1529344,1529856) | 20260505 07:47:24.537 | INFO | MemInfo: { | 20260505 07:47:24.537 | INFO | "gpu" : " 17 GB", | 20260505 07:47:24.537 | INFO | "huge" : "", | 20260505 07:47:24.537 | INFO | "huge_shared" : " 27 GB", | 20260505 07:47:24.537 | INFO | "malloc" : " 14 GB", | 20260505 07:47:24.537 | INFO | "pinned" : " 29 GB" | 20260505 07:47:24.537 | INFO | } | 20260505 07:47:24.537 | INFO | Perf report for the last 5.13845 seconds: | 20260505 07:47:24.537 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:47:24.537 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:24.537 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:24.537 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:24.537 | INFO | ANALYSIS : 63.3%, 3.0, 98.3ms, 0 | 20260505 07:47:24.537 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:24.537 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:24.537 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:47:24.537 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:24.537 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:24.537 | INFO | Uploaded 36GiB to GPU in 2.16054s (16.6625GiB/s) | 20260505 07:47:24.538 | INFO | Downloaded 34GiB to GPU in 2.04987s (16.5864GiB/s) | 20260505 07:47:29.677 | INFO | Analyzing chunk frames = [1529856,1530368) | 20260505 07:47:29.677 | INFO | MemInfo: { | 20260505 07:47:29.677 | INFO | "gpu" : " 17 GB", | 20260505 07:47:29.677 | INFO | "huge" : "", | 20260505 07:47:29.677 | INFO | "huge_shared" : " 27 GB", | 20260505 07:47:29.677 | INFO | "malloc" : " 14 GB", | 20260505 07:47:29.677 | INFO | "pinned" : " 29 GB" | 20260505 07:47:29.677 | INFO | } | 20260505 07:47:29.677 | INFO | Perf report for the last 5.13954 seconds: | 20260505 07:47:29.677 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:47:29.677 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:29.677 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:29.677 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:29.677 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 07:47:29.677 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:29.677 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:29.677 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:47:29.677 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:29.677 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:29.677 | INFO | Uploaded 36GiB to GPU in 2.20308s (16.3407GiB/s) | 20260505 07:47:29.677 | INFO | Downloaded 34GiB to GPU in 2.05455s (16.5487GiB/s) | 20260505 07:47:34.818 | INFO | Analyzing chunk frames = [1530368,1530880) | 20260505 07:47:34.818 | INFO | MemInfo: { | 20260505 07:47:34.818 | INFO | "gpu" : " 17 GB", | 20260505 07:47:34.818 | INFO | "huge" : "", | 20260505 07:47:34.818 | INFO | "huge_shared" : " 27 GB", | 20260505 07:47:34.818 | INFO | "malloc" : " 14 GB", | 20260505 07:47:34.818 | INFO | "pinned" : " 29 GB" | 20260505 07:47:34.818 | INFO | } | 20260505 07:47:34.818 | INFO | Perf report for the last 5.14106 seconds: | 20260505 07:47:34.818 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:47:34.818 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:34.818 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:34.818 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:34.818 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 07:47:34.818 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:34.818 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:34.818 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:47:34.818 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:34.818 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:34.818 | INFO | Uploaded 36GiB to GPU in 2.19534s (16.3984GiB/s) | 20260505 07:47:34.818 | INFO | Downloaded 34GiB to GPU in 2.05174s (16.5713GiB/s) | 20260505 07:47:39.955 | INFO | Analyzing chunk frames = [1530880,1531392) | 20260505 07:47:39.955 | INFO | MemInfo: { | 20260505 07:47:39.955 | INFO | "gpu" : " 17 GB", | 20260505 07:47:39.955 | INFO | "huge" : "", | 20260505 07:47:39.955 | INFO | "huge_shared" : " 27 GB", | 20260505 07:47:39.955 | INFO | "malloc" : " 14 GB", | 20260505 07:47:39.955 | INFO | "pinned" : " 29 GB" | 20260505 07:47:39.955 | INFO | } | 20260505 07:47:39.955 | INFO | Perf report for the last 5.13734 seconds: | 20260505 07:47:39.955 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:47:39.955 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:39.955 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:39.955 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:39.955 | INFO | ANALYSIS : 63.8%, 3.0, 99.5ms, 0 | 20260505 07:47:39.955 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:39.955 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:39.955 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 07:47:39.955 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:39.955 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:39.955 | INFO | Uploaded 36GiB to GPU in 2.16825s (16.6032GiB/s) | 20260505 07:47:39.956 | INFO | Downloaded 34GiB to GPU in 2.13936s (15.8926GiB/s) | 20260505 07:47:45.095 | INFO | Analyzing chunk frames = [1531392,1531904) | 20260505 07:47:45.095 | INFO | MemInfo: { | 20260505 07:47:45.095 | INFO | "gpu" : " 17 GB", | 20260505 07:47:45.095 | INFO | "huge" : "", | 20260505 07:47:45.095 | INFO | "huge_shared" : " 27 GB", | 20260505 07:47:45.095 | INFO | "malloc" : " 14 GB", | 20260505 07:47:45.095 | INFO | "pinned" : " 29 GB" | 20260505 07:47:45.095 | INFO | } | 20260505 07:47:45.095 | INFO | Perf report for the last 5.13953 seconds: | 20260505 07:47:45.095 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:47:45.095 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:45.095 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:45.095 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:45.095 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 07:47:45.095 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:45.095 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:45.095 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 07:47:45.095 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:45.095 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:45.095 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 07:47:45.095 | INFO | Total mem usage: 15.3 GiB | 20260505 07:47:45.095 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 07:47:45.095 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -5.95%) | 20260505 07:47:45.095 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 07:47:45.095 | INFO | High Water Breakdown: | 20260505 07:47:45.095 | INFO | 60.7% : 8.77 GiB ( 93.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 07:47:45.095 | INFO | 39.2% : 5.67 GiB ( 85.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 07:47:45.095 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 07:47:45.095 | INFO | ------------------------------------------ | 20260505 07:47:45.095 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:47:45.095 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 07:47:45.095 | INFO | Total mem usage: 26.5 GiB | 20260505 07:47:45.095 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 07:47:45.095 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 07:47:45.095 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 07:47:45.095 | INFO | High Water Breakdown: | 20260505 07:47:45.095 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:47:45.095 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:47:45.095 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:47:45.095 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:47:45.095 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:47:45.095 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:47:45.095 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 07:47:45.095 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 07:47:45.095 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:47:45.095 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 07:47:45.095 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 07:47:45.095 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 07:47:45.095 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 07:47:45.095 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 07:47:45.095 | INFO | ------------------------------------------ | 20260505 07:47:45.095 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:47:45.095 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 07:47:45.095 | INFO | Total mem usage: 25.6 GiB | 20260505 07:47:45.095 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 07:47:45.095 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 07:47:45.095 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 07:47:45.095 | INFO | High Water Breakdown: | 20260505 07:47:45.095 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 07:47:45.095 | INFO | ------------------------------------------ | 20260505 07:47:45.095 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:47:45.095 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 07:47:45.095 | INFO | Total mem usage: 15.7 GiB | 20260505 07:47:45.095 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 07:47:45.095 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 07:47:45.095 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260505 07:47:45.095 | INFO | High Water Breakdown: | 20260505 07:47:45.095 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 07:47:45.095 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 07:47:45.095 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 07:47:45.095 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 07:47:45.095 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 07:47:45.095 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 07:47:45.095 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 07:47:45.095 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 07:47:45.095 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 07:47:45.095 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:47:45.095 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 07:47:45.095 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:47:45.095 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 07:47:45.095 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 07:47:45.095 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 07:47:45.095 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 07:47:45.095 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:47:45.095 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 07:47:45.095 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:47:45.095 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:47:45.095 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:47:45.095 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 07:47:45.095 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 07:47:45.095 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:47:45.095 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 07:47:45.095 | INFO | ------------------------------------------ | 20260505 07:47:45.095 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:47:45.095 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 07:47:45.095 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:47:45.095 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:47:45.095 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.54e+01, 1.36e+01 / 2.04e+00 | 20260505 07:47:45.095 | INFO | PerfReport: 0.28%, AggregateStats -- 9.37e-02 / 1.04e-01, 1.03e-02 / 5.91e-03 | 20260505 07:47:45.095 | INFO | PerfReport: 11.58%, Baseline -- 3.81e+00 / 4.43e+00, 6.14e-01 / 2.96e-01 | 20260505 07:47:45.095 | INFO | PerfReport: 3.31%, Histogram -- 1.09e+00 / 1.48e+00, 2.13e-01 / 1.22e-01 | 20260505 07:47:45.095 | INFO | PerfReport: 15.51%, DME -- 4.08e+01 / 4.48e+01, 4.21e+00 / 1.27e+00 | 20260505 07:47:45.095 | INFO | PerfReport: 56.43%, FrameLabeling -- 1.85e+01 / 1.92e+01, 2.05e+00 / 1.39e+00 | 20260505 07:47:45.095 | INFO | PerfReport: 7.51%, PulseAccumulating -- 2.47e+00 / 3.12e+00, 4.24e-01 / 2.60e-01 | 20260505 07:47:45.095 | INFO | PerfReport: 3.95%, Metrics -- 1.30e+00 / 1.64e+00, 3.70e-01 / 1.82e-01 | 20260505 07:47:45.095 | INFO | PerfReport: 1.42%, PulseSerializing -- 4.67e-01 / 5.26e-01, 9.04e-02 / 2.60e-02 | 20260505 07:47:45.095 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 07:47:45.095 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:47:45.095 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:47:45.095 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.28e+01 / 0.00e+00 | 20260505 07:47:45.095 | INFO | PerfReport: 22.41%, Upload -- 2.28e+01 / 0.00e+00, 8.15e+00 / 0.00e+00 | 20260505 07:47:45.095 | INFO | PerfReport: 21.50%, Download -- 2.19e+01 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260505 07:47:45.095 | INFO | PerfReport: 32.31%, Compute -- 3.29e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 07:47:45.095 | INFO | PerfReport: 1.68%, LockUpload -- 1.70e+00 / 0.00e+00, 7.28e+00 / 0.00e+00 | 20260505 07:47:45.095 | INFO | PerfReport: 0.85%, LockDownload -- 8.65e-01 / 0.00e+00, 2.25e+00 / 0.00e+00 | 20260505 07:47:45.095 | INFO | PerfReport: 21.23%, LockCompute -- 2.16e+01 / 0.00e+00, 1.74e+01 / 0.00e+00 | 20260505 07:47:45.095 | INFO | PerfReport: 0.01%, Misc -- 1.47e-02 / 0.00e+00, 2.39e-02 / 0.00e+00 | 20260505 07:47:45.096 | INFO | Uploaded 36GiB to GPU in 2.18776s (16.4552GiB/s) | 20260505 07:47:45.096 | INFO | Downloaded 34GiB to GPU in 2.05438s (16.55GiB/s) | 20260505 07:47:45.226 | INFO | Starting a round of Baz IO | 20260505 07:47:48.512 | INFO | Populating RTMetrics report | 20260505 07:47:48.513 | INFO | RTMetrics report populated | 20260505 07:47:48.513 | INFO | Opening tmp RTMetrics json file | 20260505 07:47:48.514 | INFO | Serializing RTMetrics to json | 20260505 07:47:48.515 | INFO | Serialization done, swapping tmp file now... | 20260505 07:47:48.515 | INFO | RTMetrics done | 20260505 07:47:48.658 | INFO | Finished a round of Baz IO: Wrote 2.441 GiB in 3.431 seconds | 20260505 07:47:48.658 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 29.00% / 0.03% / 70.97% | 20260505 07:47:48.658 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.23787% / 4.37101e-05% / 7.94729e-05% | 20260505 07:47:48.658 | INFO | Baz IO overall: | 20260505 07:47:48.658 | INFO | BytesWritten : 846437249929 | 20260505 07:47:48.658 | INFO | NumEvents : 401900793729 | 20260505 07:47:50.234 | INFO | Analyzing chunk frames = [1531904,1532416) | 20260505 07:47:50.234 | INFO | MemInfo: { | 20260505 07:47:50.234 | INFO | "gpu" : " 17 GB", | 20260505 07:47:50.234 | INFO | "huge" : "", | 20260505 07:47:50.234 | INFO | "huge_shared" : " 27 GB", | 20260505 07:47:50.234 | INFO | "malloc" : " 14 GB", | 20260505 07:47:50.234 | INFO | "pinned" : " 29 GB" | 20260505 07:47:50.234 | INFO | } | 20260505 07:47:50.234 | INFO | Perf report for the last 5.13912 seconds: | 20260505 07:47:50.234 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:47:50.234 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:50.234 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:50.234 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:50.234 | INFO | ANALYSIS : 65.3%, 2.9, 99.9ms, 0 | 20260505 07:47:50.234 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:50.234 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260505 07:47:50.234 | INFO | PRE_HQ : 63.9%, 1.9, 61.7ms, 0 | 20260505 07:47:50.234 | INFO | BAZWRITER : 66.1%, 4.2, 141.3ms, 0 | 20260505 07:47:50.234 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 07:47:50.234 | INFO | Uploaded 36GiB to GPU in 2.20825s (16.3025GiB/s) | 20260505 07:47:50.234 | INFO | Downloaded 38GiB to GPU in 2.3431s (16.2178GiB/s) | 20260505 07:47:55.373 | INFO | Analyzing chunk frames = [1532416,1532928) | 20260505 07:47:55.373 | INFO | MemInfo: { | 20260505 07:47:55.373 | INFO | "gpu" : " 17 GB", | 20260505 07:47:55.373 | INFO | "huge" : "", | 20260505 07:47:55.373 | INFO | "huge_shared" : " 27 GB", | 20260505 07:47:55.373 | INFO | "malloc" : " 14 GB", | 20260505 07:47:55.373 | INFO | "pinned" : " 29 GB" | 20260505 07:47:55.373 | INFO | } | 20260505 07:47:55.373 | INFO | Perf report for the last 5.1396 seconds: | 20260505 07:47:55.374 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:47:55.374 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:55.374 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:55.374 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:55.374 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 07:47:55.374 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:55.374 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:55.374 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:47:55.374 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:55.374 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:47:55.374 | INFO | Uploaded 36GiB to GPU in 2.17977s (16.5155GiB/s) | 20260505 07:47:55.374 | INFO | Downloaded 34GiB to GPU in 2.04991s (16.5861GiB/s) | 20260505 07:48:00.513 | INFO | Analyzing chunk frames = [1532928,1533440) | 20260505 07:48:00.513 | INFO | MemInfo: { | 20260505 07:48:00.513 | INFO | "gpu" : " 17 GB", | 20260505 07:48:00.513 | INFO | "huge" : "", | 20260505 07:48:00.513 | INFO | "huge_shared" : " 27 GB", | 20260505 07:48:00.513 | INFO | "malloc" : " 14 GB", | 20260505 07:48:00.513 | INFO | "pinned" : " 29 GB" | 20260505 07:48:00.513 | INFO | } | 20260505 07:48:00.513 | INFO | Perf report for the last 5.13913 seconds: | 20260505 07:48:00.513 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:48:00.513 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:00.513 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:00.513 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:00.513 | INFO | ANALYSIS : 64.2%, 3.0, 99.9ms, 0 | 20260505 07:48:00.513 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:00.513 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:00.513 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:48:00.513 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:00.513 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:00.513 | INFO | Uploaded 36GiB to GPU in 2.19388s (16.4092GiB/s) | 20260505 07:48:00.513 | INFO | Downloaded 34GiB to GPU in 2.05939s (16.5097GiB/s) | 20260505 07:48:05.652 | INFO | Analyzing chunk frames = [1533440,1533952) | 20260505 07:48:05.652 | INFO | MemInfo: { | 20260505 07:48:05.652 | INFO | "gpu" : " 17 GB", | 20260505 07:48:05.652 | INFO | "huge" : "", | 20260505 07:48:05.652 | INFO | "huge_shared" : " 27 GB", | 20260505 07:48:05.652 | INFO | "malloc" : " 14 GB", | 20260505 07:48:05.652 | INFO | "pinned" : " 29 GB" | 20260505 07:48:05.652 | INFO | } | 20260505 07:48:05.652 | INFO | Perf report for the last 5.13941 seconds: | 20260505 07:48:05.652 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:48:05.652 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:05.652 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:05.652 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:05.652 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 07:48:05.652 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:05.652 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:05.652 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:48:05.652 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:05.652 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:05.652 | INFO | Uploaded 36GiB to GPU in 2.15436s (16.7103GiB/s) | 20260505 07:48:05.653 | INFO | Downloaded 34GiB to GPU in 2.04271s (16.6446GiB/s) | 20260505 07:48:10.791 | INFO | Analyzing chunk frames = [1533952,1534464) | 20260505 07:48:10.791 | INFO | MemInfo: { | 20260505 07:48:10.791 | INFO | "gpu" : " 17 GB", | 20260505 07:48:10.791 | INFO | "huge" : "", | 20260505 07:48:10.791 | INFO | "huge_shared" : " 27 GB", | 20260505 07:48:10.791 | INFO | "malloc" : " 14 GB", | 20260505 07:48:10.791 | INFO | "pinned" : " 29 GB" | 20260505 07:48:10.791 | INFO | } | 20260505 07:48:10.791 | INFO | Perf report for the last 5.13939 seconds: | 20260505 07:48:10.792 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:48:10.792 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:10.792 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:10.792 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:10.792 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 07:48:10.792 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:10.792 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:10.792 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 07:48:10.792 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:10.792 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:10.792 | INFO | Uploaded 36GiB to GPU in 2.20022s (16.362GiB/s) | 20260505 07:48:10.792 | INFO | Downloaded 34GiB to GPU in 2.05079s (16.579GiB/s) | 20260505 07:48:15.932 | INFO | Analyzing chunk frames = [1534464,1534976) | 20260505 07:48:15.932 | INFO | MemInfo: { | 20260505 07:48:15.932 | INFO | "gpu" : " 17 GB", | 20260505 07:48:15.932 | INFO | "huge" : "", | 20260505 07:48:15.932 | INFO | "huge_shared" : " 27 GB", | 20260505 07:48:15.932 | INFO | "malloc" : " 14 GB", | 20260505 07:48:15.932 | INFO | "pinned" : " 29 GB" | 20260505 07:48:15.932 | INFO | } | 20260505 07:48:15.932 | INFO | Perf report for the last 5.14074 seconds: | 20260505 07:48:15.932 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:48:15.932 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:15.932 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:15.932 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:15.932 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 07:48:15.932 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:15.932 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:15.933 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 07:48:15.933 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:15.933 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:15.933 | INFO | Uploaded 36GiB to GPU in 2.15821s (16.6805GiB/s) | 20260505 07:48:15.933 | INFO | Downloaded 34GiB to GPU in 2.05577s (16.5388GiB/s) | 20260505 07:48:21.070 | INFO | Analyzing chunk frames = [1534976,1535488) | 20260505 07:48:21.070 | INFO | MemInfo: { | 20260505 07:48:21.070 | INFO | "gpu" : " 17 GB", | 20260505 07:48:21.070 | INFO | "huge" : "", | 20260505 07:48:21.070 | INFO | "huge_shared" : " 27 GB", | 20260505 07:48:21.070 | INFO | "malloc" : " 14 GB", | 20260505 07:48:21.070 | INFO | "pinned" : " 29 GB" | 20260505 07:48:21.070 | INFO | } | 20260505 07:48:21.070 | INFO | Perf report for the last 5.13746 seconds: | 20260505 07:48:21.070 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:48:21.070 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:21.070 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:21.070 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:21.070 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 07:48:21.070 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:21.070 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:21.070 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 07:48:21.070 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:21.070 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:21.070 | INFO | Uploaded 36GiB to GPU in 2.18138s (16.5034GiB/s) | 20260505 07:48:21.070 | INFO | Downloaded 34GiB to GPU in 2.13729s (15.908GiB/s) | 20260505 07:48:26.209 | INFO | Analyzing chunk frames = [1535488,1536000) | 20260505 07:48:26.209 | INFO | MemInfo: { | 20260505 07:48:26.209 | INFO | "gpu" : " 17 GB", | 20260505 07:48:26.209 | INFO | "huge" : "", | 20260505 07:48:26.209 | INFO | "huge_shared" : " 27 GB", | 20260505 07:48:26.209 | INFO | "malloc" : " 14 GB", | 20260505 07:48:26.209 | INFO | "pinned" : " 29 GB" | 20260505 07:48:26.209 | INFO | } | 20260505 07:48:26.209 | INFO | Perf report for the last 5.13957 seconds: | 20260505 07:48:26.209 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:48:26.209 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:26.209 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:26.209 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:26.209 | INFO | ANALYSIS : 64.8%, 3.0, 101.2ms, 0 | 20260505 07:48:26.209 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:26.209 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:26.209 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 07:48:26.209 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:26.209 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:26.210 | INFO | Uploaded 36GiB to GPU in 2.21841s (16.2279GiB/s) | 20260505 07:48:26.210 | INFO | Downloaded 34GiB to GPU in 2.03715s (16.69GiB/s) | 20260505 07:48:26.343 | INFO | Starting a round of Baz IO | 20260505 07:48:29.631 | INFO | Populating RTMetrics report | 20260505 07:48:29.631 | INFO | RTMetrics report populated | 20260505 07:48:29.631 | INFO | Opening tmp RTMetrics json file | 20260505 07:48:29.654 | INFO | Serializing RTMetrics to json | 20260505 07:48:29.655 | INFO | Serialization done, swapping tmp file now... | 20260505 07:48:29.656 | INFO | RTMetrics done | 20260505 07:48:29.821 | INFO | Finished a round of Baz IO: Wrote 2.444 GiB in 3.477 seconds | 20260505 07:48:29.821 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 28.98% / 0.03% / 71.00% | 20260505 07:48:29.821 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.23639% / 4.37101e-05% / 9.13938e-05% | 20260505 07:48:29.821 | INFO | Baz IO overall: | 20260505 07:48:29.821 | INFO | BytesWritten : 849061067892 | 20260505 07:48:29.821 | INFO | NumEvents : 403155985605 | 20260505 07:48:31.349 | INFO | Analyzing chunk frames = [1536000,1536512) | 20260505 07:48:31.349 | INFO | MemInfo: { | 20260505 07:48:31.349 | INFO | "gpu" : " 17 GB", | 20260505 07:48:31.349 | INFO | "huge" : "", | 20260505 07:48:31.349 | INFO | "huge_shared" : " 27 GB", | 20260505 07:48:31.349 | INFO | "malloc" : " 14 GB", | 20260505 07:48:31.349 | INFO | "pinned" : " 29 GB" | 20260505 07:48:31.349 | INFO | } | 20260505 07:48:31.349 | INFO | Perf report for the last 5.13992 seconds: | 20260505 07:48:31.349 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:48:31.349 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:31.349 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:31.349 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:31.349 | INFO | ANALYSIS : 65.4%, 2.9, 99.4ms, 0 | 20260505 07:48:31.349 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:31.349 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260505 07:48:31.349 | INFO | PRE_HQ : 64.0%, 1.9, 62.1ms, 0 | 20260505 07:48:31.349 | INFO | BAZWRITER : 66.9%, 4.2, 144.5ms, 0 | 20260505 07:48:31.349 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260505 07:48:31.349 | INFO | Uploaded 36GiB to GPU in 2.23104s (16.136GiB/s) | 20260505 07:48:31.350 | INFO | Downloaded 38GiB to GPU in 2.35865s (16.1109GiB/s) | 20260505 07:48:36.488 | INFO | Analyzing chunk frames = [1536512,1537024) | 20260505 07:48:36.488 | INFO | MemInfo: { | 20260505 07:48:36.488 | INFO | "gpu" : " 17 GB", | 20260505 07:48:36.488 | INFO | "huge" : "", | 20260505 07:48:36.488 | INFO | "huge_shared" : " 27 GB", | 20260505 07:48:36.488 | INFO | "malloc" : " 14 GB", | 20260505 07:48:36.488 | INFO | "pinned" : " 29 GB" | 20260505 07:48:36.488 | INFO | } | 20260505 07:48:36.488 | INFO | Perf report for the last 5.13863 seconds: | 20260505 07:48:36.488 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:48:36.488 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:36.488 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:36.488 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:36.488 | INFO | ANALYSIS : 64.4%, 3.0, 100.6ms, 0 | 20260505 07:48:36.488 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:36.488 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:36.488 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:48:36.488 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:36.488 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:36.488 | INFO | Uploaded 36GiB to GPU in 2.18308s (16.4905GiB/s) | 20260505 07:48:36.488 | INFO | Downloaded 34GiB to GPU in 2.0574s (16.5257GiB/s) | 20260505 07:48:41.627 | INFO | Analyzing chunk frames = [1537024,1537536) | 20260505 07:48:41.627 | INFO | MemInfo: { | 20260505 07:48:41.627 | INFO | "gpu" : " 17 GB", | 20260505 07:48:41.627 | INFO | "huge" : "", | 20260505 07:48:41.627 | INFO | "huge_shared" : " 27 GB", | 20260505 07:48:41.627 | INFO | "malloc" : " 14 GB", | 20260505 07:48:41.627 | INFO | "pinned" : " 29 GB" | 20260505 07:48:41.627 | INFO | } | 20260505 07:48:41.627 | INFO | Perf report for the last 5.13946 seconds: | 20260505 07:48:41.627 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:48:41.627 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:41.627 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:41.627 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:41.627 | INFO | ANALYSIS : 64.2%, 3.0, 99.9ms, 0 | 20260505 07:48:41.627 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:41.627 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:41.627 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:48:41.627 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:41.627 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:41.628 | INFO | Uploaded 36GiB to GPU in 2.19163s (16.4262GiB/s) | 20260505 07:48:41.628 | INFO | Downloaded 34GiB to GPU in 2.06332s (16.4783GiB/s) | 20260505 07:48:46.767 | INFO | Analyzing chunk frames = [1537536,1538048) | 20260505 07:48:46.767 | INFO | MemInfo: { | 20260505 07:48:46.767 | INFO | "gpu" : " 17 GB", | 20260505 07:48:46.767 | INFO | "huge" : "", | 20260505 07:48:46.767 | INFO | "huge_shared" : " 27 GB", | 20260505 07:48:46.767 | INFO | "malloc" : " 14 GB", | 20260505 07:48:46.767 | INFO | "pinned" : " 29 GB" | 20260505 07:48:46.767 | INFO | } | 20260505 07:48:46.767 | INFO | Perf report for the last 5.13962 seconds: | 20260505 07:48:46.767 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:48:46.767 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:46.767 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:46.767 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:46.767 | INFO | ANALYSIS : 63.2%, 3.0, 98.1ms, 0 | 20260505 07:48:46.767 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:46.767 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:46.767 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 07:48:46.767 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:46.767 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:46.767 | INFO | Uploaded 36GiB to GPU in 2.15318s (16.7194GiB/s) | 20260505 07:48:46.767 | INFO | Downloaded 34GiB to GPU in 2.04218s (16.6489GiB/s) | 20260505 07:48:51.906 | INFO | Analyzing chunk frames = [1538048,1538560) | 20260505 07:48:51.906 | INFO | MemInfo: { | 20260505 07:48:51.906 | INFO | "gpu" : " 17 GB", | 20260505 07:48:51.906 | INFO | "huge" : "", | 20260505 07:48:51.906 | INFO | "huge_shared" : " 27 GB", | 20260505 07:48:51.906 | INFO | "malloc" : " 14 GB", | 20260505 07:48:51.906 | INFO | "pinned" : " 29 GB" | 20260505 07:48:51.906 | INFO | } | 20260505 07:48:51.906 | INFO | Perf report for the last 5.13904 seconds: | 20260505 07:48:51.906 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:48:51.906 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:51.906 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:51.906 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:51.906 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 07:48:51.906 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:51.906 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:51.906 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 07:48:51.906 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:51.906 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:51.907 | INFO | Uploaded 36GiB to GPU in 2.19939s (16.3682GiB/s) | 20260505 07:48:51.907 | INFO | Downloaded 34GiB to GPU in 2.05791s (16.5216GiB/s) | 20260505 07:48:57.047 | INFO | Analyzing chunk frames = [1538560,1539072) | 20260505 07:48:57.047 | INFO | MemInfo: { | 20260505 07:48:57.047 | INFO | "gpu" : " 17 GB", | 20260505 07:48:57.047 | INFO | "huge" : "", | 20260505 07:48:57.047 | INFO | "huge_shared" : " 27 GB", | 20260505 07:48:57.047 | INFO | "malloc" : " 14 GB", | 20260505 07:48:57.047 | INFO | "pinned" : " 29 GB" | 20260505 07:48:57.047 | INFO | } | 20260505 07:48:57.047 | INFO | Perf report for the last 5.1411 seconds: | 20260505 07:48:57.047 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:48:57.047 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:57.047 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:57.047 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:57.047 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 07:48:57.047 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:57.047 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:57.047 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 07:48:57.047 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:57.047 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:48:57.048 | INFO | Uploaded 36GiB to GPU in 2.19553s (16.397GiB/s) | 20260505 07:48:57.048 | INFO | Downloaded 34GiB to GPU in 2.04895s (16.5939GiB/s) | 20260505 07:49:02.184 | INFO | Analyzing chunk frames = [1539072,1539584) | 20260505 07:49:02.184 | INFO | MemInfo: { | 20260505 07:49:02.184 | INFO | "gpu" : " 17 GB", | 20260505 07:49:02.184 | INFO | "huge" : "", | 20260505 07:49:02.184 | INFO | "huge_shared" : " 27 GB", | 20260505 07:49:02.184 | INFO | "malloc" : " 14 GB", | 20260505 07:49:02.184 | INFO | "pinned" : " 29 GB" | 20260505 07:49:02.184 | INFO | } | 20260505 07:49:02.185 | INFO | Perf report for the last 5.13754 seconds: | 20260505 07:49:02.185 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:49:02.185 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:02.185 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:02.185 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:02.185 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 07:49:02.185 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:02.185 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:02.185 | INFO | PRE_HQ : 31.7%, 1.0, 16.4ms, 0 | 20260505 07:49:02.185 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:02.185 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:02.185 | INFO | Uploaded 36GiB to GPU in 2.19952s (16.3672GiB/s) | 20260505 07:49:02.185 | INFO | Downloaded 34GiB to GPU in 2.15595s (15.7703GiB/s) | 20260505 07:49:07.325 | INFO | Analyzing chunk frames = [1539584,1540096) | 20260505 07:49:07.325 | INFO | MemInfo: { | 20260505 07:49:07.325 | INFO | "gpu" : " 17 GB", | 20260505 07:49:07.325 | INFO | "huge" : "", | 20260505 07:49:07.325 | INFO | "huge_shared" : " 27 GB", | 20260505 07:49:07.325 | INFO | "malloc" : " 14 GB", | 20260505 07:49:07.325 | INFO | "pinned" : " 29 GB" | 20260505 07:49:07.325 | INFO | } | 20260505 07:49:07.325 | INFO | Perf report for the last 5.14016 seconds: | 20260505 07:49:07.325 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:49:07.325 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:07.325 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:07.325 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:07.325 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 07:49:07.325 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:07.325 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:07.325 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:49:07.325 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:07.325 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:07.325 | INFO | Uploaded 36GiB to GPU in 2.17654s (16.54GiB/s) | 20260505 07:49:07.325 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 07:49:07.325 | INFO | Total mem usage: 15.3 GiB | 20260505 07:49:07.325 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 07:49:07.325 | INFO | Downloaded 34GiB to GPU in 2.05622s (16.5352GiB/s) | 20260505 07:49:07.325 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -5.89%) | 20260505 07:49:07.325 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 07:49:07.325 | INFO | High Water Breakdown: | 20260505 07:49:07.325 | INFO | 60.7% : 8.78 GiB ( 93.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 07:49:07.325 | INFO | 39.2% : 5.67 GiB ( 85.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 07:49:07.325 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 07:49:07.325 | INFO | ------------------------------------------ | 20260505 07:49:07.325 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:49:07.325 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 07:49:07.325 | INFO | Total mem usage: 26.5 GiB | 20260505 07:49:07.325 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 07:49:07.325 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 07:49:07.326 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 07:49:07.326 | INFO | High Water Breakdown: | 20260505 07:49:07.326 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:49:07.326 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:49:07.326 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:49:07.326 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:49:07.326 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:49:07.326 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:49:07.326 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 07:49:07.326 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 07:49:07.326 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:49:07.326 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 07:49:07.326 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 07:49:07.326 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 07:49:07.326 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 07:49:07.326 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 07:49:07.326 | INFO | ------------------------------------------ | 20260505 07:49:07.326 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:49:07.326 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 07:49:07.326 | INFO | Total mem usage: 25.6 GiB | 20260505 07:49:07.326 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 07:49:07.326 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 07:49:07.326 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 07:49:07.326 | INFO | High Water Breakdown: | 20260505 07:49:07.326 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 07:49:07.326 | INFO | ------------------------------------------ | 20260505 07:49:07.326 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:49:07.326 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 07:49:07.326 | INFO | Total mem usage: 15.7 GiB | 20260505 07:49:07.326 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 07:49:07.326 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 07:49:07.326 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 07:49:07.326 | INFO | High Water Breakdown: | 20260505 07:49:07.326 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 07:49:07.326 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 07:49:07.326 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 07:49:07.326 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 07:49:07.326 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 07:49:07.326 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 07:49:07.326 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 07:49:07.326 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 07:49:07.326 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 07:49:07.326 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:49:07.326 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 07:49:07.326 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:49:07.326 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 07:49:07.326 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 07:49:07.326 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 07:49:07.326 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 07:49:07.326 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:49:07.326 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 07:49:07.326 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:49:07.326 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:49:07.326 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:49:07.326 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 07:49:07.326 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 07:49:07.326 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:49:07.326 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 07:49:07.326 | INFO | ------------------------------------------ | 20260505 07:49:07.327 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:49:07.327 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 07:49:07.327 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:49:07.327 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:49:07.327 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.55e+01, 1.37e+01 / 2.02e+00 | 20260505 07:49:07.327 | INFO | PerfReport: 0.28%, AggregateStats -- 9.33e-02 / 1.04e-01, 1.02e-02 / 5.71e-03 | 20260505 07:49:07.327 | INFO | PerfReport: 11.58%, Baseline -- 3.81e+00 / 4.44e+00, 6.17e-01 / 2.67e-01 | 20260505 07:49:07.327 | INFO | PerfReport: 3.30%, Histogram -- 1.08e+00 / 1.47e+00, 2.11e-01 / 1.11e-01 | 20260505 07:49:07.327 | INFO | PerfReport: 15.50%, DME -- 4.08e+01 / 4.49e+01, 4.33e+00 / 1.33e+00 | 20260505 07:49:07.327 | INFO | PerfReport: 56.48%, FrameLabeling -- 1.86e+01 / 1.92e+01, 2.07e+00 / 1.32e+00 | 20260505 07:49:07.327 | INFO | PerfReport: 7.49%, PulseAccumulating -- 2.46e+00 / 3.13e+00, 4.20e-01 / 2.35e-01 | 20260505 07:49:07.327 | INFO | PerfReport: 3.94%, Metrics -- 1.30e+00 / 1.63e+00, 3.73e-01 / 1.85e-01 | 20260505 07:49:07.327 | INFO | PerfReport: 1.42%, PulseSerializing -- 4.67e-01 / 5.26e-01, 9.00e-02 / 2.84e-02 | 20260505 07:49:07.327 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 07:49:07.327 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:49:07.327 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:49:07.327 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.30e+01 / 0.00e+00 | 20260505 07:49:07.327 | INFO | PerfReport: 22.40%, Upload -- 2.28e+01 / 0.00e+00, 8.17e+00 / 0.00e+00 | 20260505 07:49:07.327 | INFO | PerfReport: 21.50%, Download -- 2.19e+01 / 0.00e+00, 8.05e+00 / 0.00e+00 | 20260505 07:49:07.327 | INFO | PerfReport: 32.30%, Compute -- 3.29e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 07:49:07.327 | INFO | PerfReport: 1.70%, LockUpload -- 1.73e+00 / 0.00e+00, 7.31e+00 / 0.00e+00 | 20260505 07:49:07.327 | INFO | PerfReport: 0.84%, LockDownload -- 8.55e-01 / 0.00e+00, 2.27e+00 / 0.00e+00 | 20260505 07:49:07.327 | INFO | PerfReport: 21.25%, LockCompute -- 2.16e+01 / 0.00e+00, 1.75e+01 / 0.00e+00 | 20260505 07:49:07.327 | INFO | PerfReport: 0.01%, Misc -- 1.48e-02 / 0.00e+00, 1.98e-02 / 0.00e+00 | 20260505 07:49:07.456 | INFO | Starting a round of Baz IO | 20260505 07:49:10.745 | INFO | Populating RTMetrics report | 20260505 07:49:10.745 | INFO | RTMetrics report populated | 20260505 07:49:10.745 | INFO | Opening tmp RTMetrics json file | 20260505 07:49:10.751 | INFO | Serializing RTMetrics to json | 20260505 07:49:10.754 | INFO | Serialization done, swapping tmp file now... | 20260505 07:49:10.756 | INFO | RTMetrics done | 20260505 07:49:10.880 | INFO | Finished a round of Baz IO: Wrote 2.442 GiB in 3.424 seconds | 20260505 07:49:10.880 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 28.95% / 0.03% / 71.02% | 20260505 07:49:10.880 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.21603% / 4.76837e-05% / 0.000115236% | 20260505 07:49:10.881 | INFO | Baz IO overall: | 20260505 07:49:10.881 | INFO | BytesWritten : 851682885751 | 20260505 07:49:10.881 | INFO | NumEvents : 404409508349 | 20260505 07:49:12.463 | INFO | Analyzing chunk frames = [1540096,1540608) | 20260505 07:49:12.463 | INFO | MemInfo: { | 20260505 07:49:12.463 | INFO | "gpu" : " 17 GB", | 20260505 07:49:12.463 | INFO | "huge" : "", | 20260505 07:49:12.463 | INFO | "huge_shared" : " 27 GB", | 20260505 07:49:12.463 | INFO | "malloc" : " 14 GB", | 20260505 07:49:12.463 | INFO | "pinned" : " 29 GB" | 20260505 07:49:12.463 | INFO | } | 20260505 07:49:12.463 | INFO | Perf report for the last 5.13846 seconds: | 20260505 07:49:12.463 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:49:12.463 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:12.463 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:12.463 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:12.464 | INFO | ANALYSIS : 65.4%, 2.9, 99.7ms, 0 | 20260505 07:49:12.464 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:12.464 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 07:49:12.464 | INFO | PRE_HQ : 64.0%, 1.9, 62.0ms, 0 | 20260505 07:49:12.464 | INFO | BAZWRITER : 65.9%, 4.1, 140.3ms, 0 | 20260505 07:49:12.464 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 07:49:12.464 | INFO | Uploaded 36GiB to GPU in 2.21338s (16.2647GiB/s) | 20260505 07:49:12.464 | INFO | Downloaded 38GiB to GPU in 2.34958s (16.1731GiB/s) | 20260505 07:49:17.603 | INFO | Analyzing chunk frames = [1540608,1541120) | 20260505 07:49:17.603 | INFO | MemInfo: { | 20260505 07:49:17.603 | INFO | "gpu" : " 17 GB", | 20260505 07:49:17.603 | INFO | "huge" : "", | 20260505 07:49:17.603 | INFO | "huge_shared" : " 27 GB", | 20260505 07:49:17.603 | INFO | "malloc" : " 14 GB", | 20260505 07:49:17.603 | INFO | "pinned" : " 29 GB" | 20260505 07:49:17.603 | INFO | } | 20260505 07:49:17.603 | INFO | Perf report for the last 5.13948 seconds: | 20260505 07:49:17.603 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:49:17.603 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:17.603 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:17.603 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:17.603 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 07:49:17.603 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:17.603 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:17.603 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:49:17.603 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:17.603 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:17.603 | INFO | Uploaded 36GiB to GPU in 2.18747s (16.4574GiB/s) | 20260505 07:49:17.603 | INFO | Downloaded 34GiB to GPU in 2.06099s (16.497GiB/s) | 20260505 07:49:22.743 | INFO | Analyzing chunk frames = [1541120,1541632) | 20260505 07:49:22.743 | INFO | MemInfo: { | 20260505 07:49:22.743 | INFO | "gpu" : " 17 GB", | 20260505 07:49:22.743 | INFO | "huge" : "", | 20260505 07:49:22.743 | INFO | "huge_shared" : " 27 GB", | 20260505 07:49:22.743 | INFO | "malloc" : " 14 GB", | 20260505 07:49:22.743 | INFO | "pinned" : " 29 GB" | 20260505 07:49:22.743 | INFO | } | 20260505 07:49:22.743 | INFO | Perf report for the last 5.13998 seconds: | 20260505 07:49:22.743 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:49:22.743 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:22.743 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:22.743 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:22.743 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 07:49:22.743 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:22.743 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:22.743 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:49:22.743 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:22.743 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:22.743 | INFO | Uploaded 36GiB to GPU in 2.14796s (16.7601GiB/s) | 20260505 07:49:22.743 | INFO | Downloaded 34GiB to GPU in 2.05767s (16.5235GiB/s) | 20260505 07:49:27.881 | INFO | Analyzing chunk frames = [1541632,1542144) | 20260505 07:49:27.881 | INFO | MemInfo: { | 20260505 07:49:27.881 | INFO | "gpu" : " 17 GB", | 20260505 07:49:27.881 | INFO | "huge" : "", | 20260505 07:49:27.881 | INFO | "huge_shared" : " 27 GB", | 20260505 07:49:27.881 | INFO | "malloc" : " 14 GB", | 20260505 07:49:27.881 | INFO | "pinned" : " 29 GB" | 20260505 07:49:27.881 | INFO | } | 20260505 07:49:27.881 | INFO | Perf report for the last 5.13864 seconds: | 20260505 07:49:27.882 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:49:27.882 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:27.882 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:27.882 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:27.882 | INFO | ANALYSIS : 63.1%, 3.0, 98.1ms, 0 | 20260505 07:49:27.882 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:27.882 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:27.882 | INFO | PRE_HQ : 29.8%, 1.0, 15.4ms, 0 | 20260505 07:49:27.882 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:27.882 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:27.882 | INFO | Uploaded 36GiB to GPU in 2.14834s (16.7571GiB/s) | 20260505 07:49:27.882 | INFO | Downloaded 34GiB to GPU in 2.04791s (16.6023GiB/s) | 20260505 07:49:33.021 | INFO | Analyzing chunk frames = [1542144,1542656) | 20260505 07:49:33.021 | INFO | MemInfo: { | 20260505 07:49:33.021 | INFO | "gpu" : " 17 GB", | 20260505 07:49:33.021 | INFO | "huge" : "", | 20260505 07:49:33.021 | INFO | "huge_shared" : " 27 GB", | 20260505 07:49:33.021 | INFO | "malloc" : " 14 GB", | 20260505 07:49:33.021 | INFO | "pinned" : " 29 GB" | 20260505 07:49:33.021 | INFO | } | 20260505 07:49:33.021 | INFO | Perf report for the last 5.13912 seconds: | 20260505 07:49:33.021 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:49:33.021 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:33.021 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:33.021 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:33.021 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 07:49:33.021 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:33.021 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:33.021 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:49:33.021 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:33.021 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:33.021 | INFO | Uploaded 36GiB to GPU in 2.19633s (16.3909GiB/s) | 20260505 07:49:33.021 | INFO | Downloaded 34GiB to GPU in 2.05224s (16.5673GiB/s) | 20260505 07:49:38.162 | INFO | Analyzing chunk frames = [1542656,1543168) | 20260505 07:49:38.162 | INFO | MemInfo: { | 20260505 07:49:38.162 | INFO | "gpu" : " 17 GB", | 20260505 07:49:38.162 | INFO | "huge" : "", | 20260505 07:49:38.162 | INFO | "huge_shared" : " 27 GB", | 20260505 07:49:38.162 | INFO | "malloc" : " 14 GB", | 20260505 07:49:38.162 | INFO | "pinned" : " 29 GB" | 20260505 07:49:38.162 | INFO | } | 20260505 07:49:38.162 | INFO | Perf report for the last 5.14135 seconds: | 20260505 07:49:38.162 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:49:38.162 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:38.162 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:38.162 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:38.162 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 07:49:38.162 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:38.162 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:38.162 | INFO | PRE_HQ : 29.7%, 1.0, 15.5ms, 0 | 20260505 07:49:38.162 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:38.162 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:38.162 | INFO | Uploaded 36GiB to GPU in 2.19123s (16.4291GiB/s) | 20260505 07:49:38.162 | INFO | Downloaded 34GiB to GPU in 2.05157s (16.5727GiB/s) | 20260505 07:49:43.299 | INFO | Analyzing chunk frames = [1543168,1543680) | 20260505 07:49:43.299 | INFO | MemInfo: { | 20260505 07:49:43.299 | INFO | "gpu" : " 17 GB", | 20260505 07:49:43.299 | INFO | "huge" : "", | 20260505 07:49:43.299 | INFO | "huge_shared" : " 27 GB", | 20260505 07:49:43.299 | INFO | "malloc" : " 14 GB", | 20260505 07:49:43.299 | INFO | "pinned" : " 29 GB" | 20260505 07:49:43.299 | INFO | } | 20260505 07:49:43.299 | INFO | Perf report for the last 5.13748 seconds: | 20260505 07:49:43.300 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:49:43.300 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:43.300 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:43.300 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:43.300 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 07:49:43.300 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:43.300 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:43.300 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260505 07:49:43.300 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:43.300 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:43.300 | INFO | Uploaded 36GiB to GPU in 2.18374s (16.4855GiB/s) | 20260505 07:49:43.300 | INFO | Downloaded 34GiB to GPU in 2.15753s (15.7588GiB/s) | 20260505 07:49:48.439 | INFO | Analyzing chunk frames = [1543680,1544192) | 20260505 07:49:48.439 | INFO | MemInfo: { | 20260505 07:49:48.439 | INFO | "gpu" : " 17 GB", | 20260505 07:49:48.439 | INFO | "huge" : "", | 20260505 07:49:48.439 | INFO | "huge_shared" : " 27 GB", | 20260505 07:49:48.439 | INFO | "malloc" : " 14 GB", | 20260505 07:49:48.439 | INFO | "pinned" : " 29 GB" | 20260505 07:49:48.439 | INFO | } | 20260505 07:49:48.439 | INFO | Perf report for the last 5.1395 seconds: | 20260505 07:49:48.439 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:49:48.439 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:48.439 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:48.439 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:48.439 | INFO | ANALYSIS : 64.7%, 3.0, 101.1ms, 0 | 20260505 07:49:48.439 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:48.439 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:48.439 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 07:49:48.439 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:48.439 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:48.439 | INFO | Uploaded 36GiB to GPU in 2.21743s (16.235GiB/s) | 20260505 07:49:48.440 | INFO | Downloaded 34GiB to GPU in 2.05005s (16.585GiB/s) | 20260505 07:49:48.571 | INFO | Starting a round of Baz IO | 20260505 07:49:51.856 | INFO | Populating RTMetrics report | 20260505 07:49:51.857 | INFO | RTMetrics report populated | 20260505 07:49:51.857 | INFO | Opening tmp RTMetrics json file | 20260505 07:49:51.864 | INFO | Serializing RTMetrics to json | 20260505 07:49:51.869 | INFO | Serialization done, swapping tmp file now... | 20260505 07:49:51.873 | INFO | RTMetrics done | 20260505 07:49:51.989 | INFO | Finished a round of Baz IO: Wrote 2.445 GiB in 3.418 seconds | 20260505 07:49:51.989 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 28.92% / 0.03% / 71.05% | 20260505 07:49:51.989 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.21953% / 5.16574e-05% / 6.35783e-05% | 20260505 07:49:51.989 | INFO | Baz IO overall: | 20260505 07:49:51.989 | INFO | BytesWritten : 854307988783 | 20260505 07:49:51.989 | INFO | NumEvents : 405666240586 | 20260505 07:49:53.578 | INFO | Analyzing chunk frames = [1544192,1544704) | 20260505 07:49:53.578 | INFO | MemInfo: { | 20260505 07:49:53.578 | INFO | "gpu" : " 17 GB", | 20260505 07:49:53.578 | INFO | "huge" : "", | 20260505 07:49:53.578 | INFO | "huge_shared" : " 27 GB", | 20260505 07:49:53.578 | INFO | "malloc" : " 14 GB", | 20260505 07:49:53.578 | INFO | "pinned" : " 29 GB" | 20260505 07:49:53.578 | INFO | } | 20260505 07:49:53.578 | INFO | Perf report for the last 5.13916 seconds: | 20260505 07:49:53.578 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:49:53.578 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:53.578 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:53.578 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:53.578 | INFO | ANALYSIS : 65.3%, 2.9, 99.9ms, 0 | 20260505 07:49:53.578 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:53.578 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260505 07:49:53.578 | INFO | PRE_HQ : 64.0%, 1.9, 62.2ms, 0 | 20260505 07:49:53.578 | INFO | BAZWRITER : 65.6%, 4.2, 141.2ms, 0 | 20260505 07:49:53.578 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 07:49:53.579 | INFO | Uploaded 36GiB to GPU in 2.20861s (16.2998GiB/s) | 20260505 07:49:53.579 | INFO | Downloaded 38GiB to GPU in 2.34454s (16.2078GiB/s) | 20260505 07:49:58.718 | INFO | Analyzing chunk frames = [1544704,1545216) | 20260505 07:49:58.718 | INFO | MemInfo: { | 20260505 07:49:58.718 | INFO | "gpu" : " 17 GB", | 20260505 07:49:58.718 | INFO | "huge" : "", | 20260505 07:49:58.718 | INFO | "huge_shared" : " 27 GB", | 20260505 07:49:58.718 | INFO | "malloc" : " 14 GB", | 20260505 07:49:58.718 | INFO | "pinned" : " 29 GB" | 20260505 07:49:58.718 | INFO | } | 20260505 07:49:58.718 | INFO | Perf report for the last 5.13953 seconds: | 20260505 07:49:58.718 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:49:58.718 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:58.718 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:58.718 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:58.718 | INFO | ANALYSIS : 64.3%, 3.0, 100.3ms, 0 | 20260505 07:49:58.718 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:58.718 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:58.718 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 07:49:58.718 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:58.718 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:49:58.718 | INFO | Uploaded 36GiB to GPU in 2.17795s (16.5293GiB/s) | 20260505 07:49:58.718 | INFO | Downloaded 34GiB to GPU in 2.05746s (16.5252GiB/s) | 20260505 07:50:03.857 | INFO | Analyzing chunk frames = [1545216,1545728) | 20260505 07:50:03.857 | INFO | MemInfo: { | 20260505 07:50:03.857 | INFO | "gpu" : " 17 GB", | 20260505 07:50:03.857 | INFO | "huge" : "", | 20260505 07:50:03.857 | INFO | "huge_shared" : " 27 GB", | 20260505 07:50:03.857 | INFO | "malloc" : " 14 GB", | 20260505 07:50:03.857 | INFO | "pinned" : " 29 GB" | 20260505 07:50:03.857 | INFO | } | 20260505 07:50:03.858 | INFO | Perf report for the last 5.13982 seconds: | 20260505 07:50:03.858 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:50:03.858 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:03.858 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:03.858 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:03.858 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 07:50:03.858 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:03.858 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:03.858 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 07:50:03.858 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:03.858 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:03.858 | INFO | Uploaded 36GiB to GPU in 2.14625s (16.7735GiB/s) | 20260505 07:50:03.858 | INFO | Downloaded 34GiB to GPU in 2.06141s (16.4935GiB/s) | 20260505 07:50:08.996 | INFO | Analyzing chunk frames = [1545728,1546240) | 20260505 07:50:08.996 | INFO | MemInfo: { | 20260505 07:50:08.996 | INFO | "gpu" : " 17 GB", | 20260505 07:50:08.996 | INFO | "huge" : "", | 20260505 07:50:08.996 | INFO | "huge_shared" : " 27 GB", | 20260505 07:50:08.996 | INFO | "malloc" : " 14 GB", | 20260505 07:50:08.996 | INFO | "pinned" : " 29 GB" | 20260505 07:50:08.996 | INFO | } | 20260505 07:50:08.996 | INFO | Perf report for the last 5.13895 seconds: | 20260505 07:50:08.996 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:50:08.996 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:08.996 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:08.996 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:08.997 | INFO | ANALYSIS : 63.2%, 3.0, 98.1ms, 0 | 20260505 07:50:08.997 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:08.997 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:08.997 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:50:08.997 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:08.997 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:08.997 | INFO | Uploaded 36GiB to GPU in 2.1515s (16.7325GiB/s) | 20260505 07:50:08.997 | INFO | Downloaded 34GiB to GPU in 2.04499s (16.626GiB/s) | 20260505 07:50:14.136 | INFO | Analyzing chunk frames = [1546240,1546752) | 20260505 07:50:14.136 | INFO | MemInfo: { | 20260505 07:50:14.136 | INFO | "gpu" : " 17 GB", | 20260505 07:50:14.136 | INFO | "huge" : "", | 20260505 07:50:14.136 | INFO | "huge_shared" : " 27 GB", | 20260505 07:50:14.136 | INFO | "malloc" : " 14 GB", | 20260505 07:50:14.136 | INFO | "pinned" : " 29 GB" | 20260505 07:50:14.136 | INFO | } | 20260505 07:50:14.136 | INFO | Perf report for the last 5.1393 seconds: | 20260505 07:50:14.136 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:50:14.136 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:14.136 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:14.136 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:14.136 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 07:50:14.136 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:14.136 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:14.136 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:50:14.136 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:14.136 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:14.136 | INFO | Uploaded 36GiB to GPU in 2.20232s (16.3464GiB/s) | 20260505 07:50:14.136 | INFO | Downloaded 34GiB to GPU in 2.05785s (16.5221GiB/s) | 20260505 07:50:19.276 | INFO | Analyzing chunk frames = [1546752,1547264) | 20260505 07:50:19.276 | INFO | MemInfo: { | 20260505 07:50:19.276 | INFO | "gpu" : " 17 GB", | 20260505 07:50:19.276 | INFO | "huge" : "", | 20260505 07:50:19.276 | INFO | "huge_shared" : " 27 GB", | 20260505 07:50:19.276 | INFO | "malloc" : " 14 GB", | 20260505 07:50:19.277 | INFO | "pinned" : " 29 GB" | 20260505 07:50:19.277 | INFO | } | 20260505 07:50:19.277 | INFO | Perf report for the last 5.14076 seconds: | 20260505 07:50:19.277 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:50:19.277 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:19.277 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:19.277 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:19.277 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 07:50:19.277 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:19.277 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:19.277 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 07:50:19.277 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:19.277 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:19.277 | INFO | Uploaded 36GiB to GPU in 2.19506s (16.4005GiB/s) | 20260505 07:50:19.277 | INFO | Downloaded 34GiB to GPU in 2.05737s (16.5259GiB/s) | 20260505 07:50:24.414 | INFO | Analyzing chunk frames = [1547264,1547776) | 20260505 07:50:24.414 | INFO | MemInfo: { | 20260505 07:50:24.414 | INFO | "gpu" : " 17 GB", | 20260505 07:50:24.414 | INFO | "huge" : "", | 20260505 07:50:24.414 | INFO | "huge_shared" : " 27 GB", | 20260505 07:50:24.414 | INFO | "malloc" : " 14 GB", | 20260505 07:50:24.414 | INFO | "pinned" : " 29 GB" | 20260505 07:50:24.414 | INFO | } | 20260505 07:50:24.414 | INFO | Perf report for the last 5.1378 seconds: | 20260505 07:50:24.414 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:50:24.414 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:24.414 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:24.414 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:24.414 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 07:50:24.414 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:24.414 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:24.415 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260505 07:50:24.415 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:24.415 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:24.415 | INFO | Uploaded 36GiB to GPU in 2.20661s (16.3146GiB/s) | 20260505 07:50:24.415 | INFO | Downloaded 34GiB to GPU in 2.1502s (15.8125GiB/s) | 20260505 07:50:29.554 | INFO | Analyzing chunk frames = [1547776,1548288) | 20260505 07:50:29.554 | INFO | MemInfo: { | 20260505 07:50:29.554 | INFO | "gpu" : " 17 GB", | 20260505 07:50:29.554 | INFO | "huge" : "", | 20260505 07:50:29.554 | INFO | "huge_shared" : " 27 GB", | 20260505 07:50:29.554 | INFO | "malloc" : " 14 GB", | 20260505 07:50:29.554 | INFO | "pinned" : " 29 GB" | 20260505 07:50:29.554 | INFO | } | 20260505 07:50:29.554 | INFO | Perf report for the last 5.13965 seconds: | 20260505 07:50:29.554 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:50:29.554 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:29.554 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:29.554 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:29.554 | INFO | ANALYSIS : 64.7%, 3.0, 101.1ms, 0 | 20260505 07:50:29.554 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:29.554 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:29.554 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 07:50:29.554 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:29.554 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:29.554 | INFO | Uploaded 36GiB to GPU in 2.21756s (16.2341GiB/s) | 20260505 07:50:29.555 | INFO | Downloaded 34GiB to GPU in 2.04661s (16.6128GiB/s) | 20260505 07:50:29.555 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 07:50:29.555 | INFO | Total mem usage: 15.3 GiB | 20260505 07:50:29.555 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 07:50:29.555 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.22%) | 20260505 07:50:29.555 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 07:50:29.555 | INFO | High Water Breakdown: | 20260505 07:50:29.555 | INFO | 60.7% : 8.76 GiB ( 93.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 07:50:29.555 | INFO | 39.1% : 5.65 GiB ( 85% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 07:50:29.555 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 07:50:29.555 | INFO | ------------------------------------------ | 20260505 07:50:29.555 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:50:29.555 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 07:50:29.555 | INFO | Total mem usage: 26.5 GiB | 20260505 07:50:29.555 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 07:50:29.555 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 07:50:29.555 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 07:50:29.555 | INFO | High Water Breakdown: | 20260505 07:50:29.555 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:50:29.555 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:50:29.555 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:50:29.555 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:50:29.555 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:50:29.555 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:50:29.555 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 07:50:29.555 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 07:50:29.555 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:50:29.555 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 07:50:29.555 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 07:50:29.555 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 07:50:29.555 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 07:50:29.555 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 07:50:29.555 | INFO | ------------------------------------------ | 20260505 07:50:29.555 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:50:29.555 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 07:50:29.555 | INFO | Total mem usage: 25.6 GiB | 20260505 07:50:29.555 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 07:50:29.555 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 07:50:29.555 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 07:50:29.555 | INFO | High Water Breakdown: | 20260505 07:50:29.555 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 07:50:29.555 | INFO | ------------------------------------------ | 20260505 07:50:29.555 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:50:29.555 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 07:50:29.555 | INFO | Total mem usage: 15.7 GiB | 20260505 07:50:29.555 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 07:50:29.555 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 07:50:29.555 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 07:50:29.555 | INFO | High Water Breakdown: | 20260505 07:50:29.555 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 07:50:29.555 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 07:50:29.555 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 07:50:29.555 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 07:50:29.555 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 07:50:29.555 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 07:50:29.555 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 07:50:29.555 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 07:50:29.555 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 07:50:29.555 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:50:29.555 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 07:50:29.555 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:50:29.555 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 07:50:29.555 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 07:50:29.555 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 07:50:29.555 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 07:50:29.555 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:50:29.555 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 07:50:29.555 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:50:29.556 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:50:29.556 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:50:29.556 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 07:50:29.556 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 07:50:29.556 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:50:29.556 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 07:50:29.556 | INFO | ------------------------------------------ | 20260505 07:50:29.556 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:50:29.556 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 07:50:29.556 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:50:29.556 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:50:29.556 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.55e+01, 1.37e+01 / 1.81e+00 | 20260505 07:50:29.556 | INFO | PerfReport: 0.29%, AggregateStats -- 9.37e-02 / 1.04e-01, 1.04e-02 / 3.94e-03 | 20260505 07:50:29.556 | INFO | PerfReport: 11.59%, Baseline -- 3.81e+00 / 4.44e+00, 6.22e-01 / 3.20e-01 | 20260505 07:50:29.556 | INFO | PerfReport: 3.31%, Histogram -- 1.09e+00 / 1.48e+00, 2.14e-01 / 1.16e-01 | 20260505 07:50:29.556 | INFO | PerfReport: 15.50%, DME -- 4.07e+01 / 4.48e+01, 4.28e+00 / 1.06e+00 | 20260505 07:50:29.556 | INFO | PerfReport: 56.46%, FrameLabeling -- 1.85e+01 / 1.93e+01, 2.07e+00 / 1.58e+00 | 20260505 07:50:29.556 | INFO | PerfReport: 7.51%, PulseAccumulating -- 2.47e+00 / 3.12e+00, 4.25e-01 / 2.41e-01 | 20260505 07:50:29.556 | INFO | PerfReport: 3.93%, Metrics -- 1.29e+00 / 1.66e+00, 3.69e-01 / 1.76e-01 | 20260505 07:50:29.556 | INFO | PerfReport: 1.42%, PulseSerializing -- 4.66e-01 / 5.31e-01, 9.05e-02 / 2.67e-02 | 20260505 07:50:29.556 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 07:50:29.556 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:50:29.556 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:50:29.556 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.31e+01 / 0.00e+00 | 20260505 07:50:29.556 | INFO | PerfReport: 22.39%, Upload -- 2.28e+01 / 0.00e+00, 8.17e+00 / 0.00e+00 | 20260505 07:50:29.556 | INFO | PerfReport: 21.53%, Download -- 2.19e+01 / 0.00e+00, 8.08e+00 / 0.00e+00 | 20260505 07:50:29.556 | INFO | PerfReport: 32.29%, Compute -- 3.29e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 07:50:29.556 | INFO | PerfReport: 1.66%, LockUpload -- 1.69e+00 / 0.00e+00, 7.30e+00 / 0.00e+00 | 20260505 07:50:29.556 | INFO | PerfReport: 0.89%, LockDownload -- 9.08e-01 / 0.00e+00, 2.35e+00 / 0.00e+00 | 20260505 07:50:29.556 | INFO | PerfReport: 21.22%, LockCompute -- 2.16e+01 / 0.00e+00, 1.73e+01 / 0.00e+00 | 20260505 07:50:29.556 | INFO | PerfReport: 0.01%, Misc -- 1.51e-02 / 0.00e+00, 2.20e-02 / 0.00e+00 | 20260505 07:50:29.688 | INFO | Starting a round of Baz IO | 20260505 07:50:32.974 | INFO | Populating RTMetrics report | 20260505 07:50:32.975 | INFO | RTMetrics report populated | 20260505 07:50:32.975 | INFO | Opening tmp RTMetrics json file | 20260505 07:50:32.985 | INFO | Serializing RTMetrics to json | 20260505 07:50:33.034 | INFO | Serialization done, swapping tmp file now... | 20260505 07:50:33.050 | INFO | RTMetrics done | 20260505 07:50:33.099 | INFO | Finished a round of Baz IO: Wrote 2.444 GiB in 3.411 seconds | 20260505 07:50:33.099 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 28.90% / 0.03% / 71.08% | 20260505 07:50:33.099 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.20253% / 3.17891e-05% / 0.000103315% | 20260505 07:50:33.099 | INFO | Baz IO overall: | 20260505 07:50:33.099 | INFO | BytesWritten : 856932374395 | 20260505 07:50:33.099 | INFO | NumEvents : 406922411262 | 20260505 07:50:34.693 | INFO | Analyzing chunk frames = [1548288,1548800) | 20260505 07:50:34.693 | INFO | MemInfo: { | 20260505 07:50:34.693 | INFO | "gpu" : " 17 GB", | 20260505 07:50:34.693 | INFO | "huge" : "", | 20260505 07:50:34.693 | INFO | "huge_shared" : " 27 GB", | 20260505 07:50:34.693 | INFO | "malloc" : " 14 GB", | 20260505 07:50:34.693 | INFO | "pinned" : " 29 GB" | 20260505 07:50:34.693 | INFO | } | 20260505 07:50:34.693 | INFO | Perf report for the last 5.13908 seconds: | 20260505 07:50:34.693 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:50:34.693 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:34.693 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:34.693 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:34.693 | INFO | ANALYSIS : 65.4%, 2.9, 99.3ms, 0 | 20260505 07:50:34.693 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:34.693 | INFO | RT_METRICS : 47.3%, 1.0, 24.8ms, 0 | 20260505 07:50:34.693 | INFO | PRE_HQ : 63.9%, 1.9, 61.9ms, 0 | 20260505 07:50:34.693 | INFO | BAZWRITER : 65.7%, 4.2, 141.5ms, 0 | 20260505 07:50:34.693 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 07:50:34.693 | INFO | Uploaded 36GiB to GPU in 2.23414s (16.1136GiB/s) | 20260505 07:50:34.693 | INFO | Downloaded 38GiB to GPU in 2.34491s (16.2053GiB/s) | 20260505 07:50:39.832 | INFO | Analyzing chunk frames = [1548800,1549312) | 20260505 07:50:39.832 | INFO | MemInfo: { | 20260505 07:50:39.832 | INFO | "gpu" : " 17 GB", | 20260505 07:50:39.832 | INFO | "huge" : "", | 20260505 07:50:39.832 | INFO | "huge_shared" : " 27 GB", | 20260505 07:50:39.832 | INFO | "malloc" : " 14 GB", | 20260505 07:50:39.832 | INFO | "pinned" : " 29 GB" | 20260505 07:50:39.832 | INFO | } | 20260505 07:50:39.832 | INFO | Perf report for the last 5.13909 seconds: | 20260505 07:50:39.832 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:50:39.832 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:39.832 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:39.832 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:39.832 | INFO | ANALYSIS : 64.3%, 3.0, 100.3ms, 0 | 20260505 07:50:39.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:39.832 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:39.833 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:50:39.833 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:39.833 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:39.833 | INFO | Uploaded 36GiB to GPU in 2.18352s (16.4871GiB/s) | 20260505 07:50:39.833 | INFO | Downloaded 34GiB to GPU in 2.05135s (16.5745GiB/s) | 20260505 07:50:44.972 | INFO | Analyzing chunk frames = [1549312,1549824) | 20260505 07:50:44.972 | INFO | MemInfo: { | 20260505 07:50:44.972 | INFO | "gpu" : " 17 GB", | 20260505 07:50:44.972 | INFO | "huge" : "", | 20260505 07:50:44.972 | INFO | "huge_shared" : " 27 GB", | 20260505 07:50:44.972 | INFO | "malloc" : " 14 GB", | 20260505 07:50:44.972 | INFO | "pinned" : " 29 GB" | 20260505 07:50:44.972 | INFO | } | 20260505 07:50:44.972 | INFO | Perf report for the last 5.13939 seconds: | 20260505 07:50:44.972 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:50:44.972 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:44.972 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:44.972 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:44.972 | INFO | ANALYSIS : 64.3%, 3.0, 100.1ms, 0 | 20260505 07:50:44.972 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:44.972 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:44.972 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:50:44.972 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:44.972 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:44.972 | INFO | Uploaded 36GiB to GPU in 2.19723s (16.3842GiB/s) | 20260505 07:50:44.972 | INFO | Downloaded 34GiB to GPU in 2.05918s (16.5115GiB/s) | 20260505 07:50:50.111 | INFO | Analyzing chunk frames = [1549824,1550336) | 20260505 07:50:50.111 | INFO | MemInfo: { | 20260505 07:50:50.111 | INFO | "gpu" : " 17 GB", | 20260505 07:50:50.111 | INFO | "huge" : "", | 20260505 07:50:50.111 | INFO | "huge_shared" : " 27 GB", | 20260505 07:50:50.111 | INFO | "malloc" : " 14 GB", | 20260505 07:50:50.111 | INFO | "pinned" : " 29 GB" | 20260505 07:50:50.111 | INFO | } | 20260505 07:50:50.111 | INFO | Perf report for the last 5.13958 seconds: | 20260505 07:50:50.111 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:50:50.111 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:50.111 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:50.111 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:50.111 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 07:50:50.111 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:50.111 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:50.111 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:50:50.111 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:50.111 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:50.112 | INFO | Uploaded 36GiB to GPU in 2.16078s (16.6606GiB/s) | 20260505 07:50:50.112 | INFO | Downloaded 34GiB to GPU in 2.04418s (16.6326GiB/s) | 20260505 07:50:55.250 | INFO | Analyzing chunk frames = [1550336,1550848) | 20260505 07:50:55.250 | INFO | MemInfo: { | 20260505 07:50:55.250 | INFO | "gpu" : " 17 GB", | 20260505 07:50:55.250 | INFO | "huge" : "", | 20260505 07:50:55.250 | INFO | "huge_shared" : " 27 GB", | 20260505 07:50:55.250 | INFO | "malloc" : " 14 GB", | 20260505 07:50:55.251 | INFO | "pinned" : " 29 GB" | 20260505 07:50:55.251 | INFO | } | 20260505 07:50:55.251 | INFO | Perf report for the last 5.13946 seconds: | 20260505 07:50:55.251 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:50:55.251 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:55.251 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:55.251 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:55.251 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 07:50:55.251 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:55.251 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:55.251 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:50:55.251 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:55.251 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:50:55.251 | INFO | Uploaded 36GiB to GPU in 2.20054s (16.3596GiB/s) | 20260505 07:50:55.251 | INFO | Downloaded 34GiB to GPU in 2.05962s (16.5079GiB/s) | 20260505 07:51:00.391 | INFO | Analyzing chunk frames = [1550848,1551360) | 20260505 07:51:00.391 | INFO | MemInfo: { | 20260505 07:51:00.391 | INFO | "gpu" : " 17 GB", | 20260505 07:51:00.391 | INFO | "huge" : "", | 20260505 07:51:00.391 | INFO | "huge_shared" : " 27 GB", | 20260505 07:51:00.391 | INFO | "malloc" : " 14 GB", | 20260505 07:51:00.391 | INFO | "pinned" : " 29 GB" | 20260505 07:51:00.391 | INFO | } | 20260505 07:51:00.391 | INFO | Perf report for the last 5.14072 seconds: | 20260505 07:51:00.391 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:51:00.391 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:00.391 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:00.391 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:00.391 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 07:51:00.391 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:00.391 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:00.391 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:51:00.391 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:00.392 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:00.392 | INFO | Uploaded 36GiB to GPU in 2.1973s (16.3838GiB/s) | 20260505 07:51:00.392 | INFO | Downloaded 34GiB to GPU in 2.04687s (16.6107GiB/s) | 20260505 07:51:05.529 | INFO | Analyzing chunk frames = [1551360,1551872) | 20260505 07:51:05.529 | INFO | MemInfo: { | 20260505 07:51:05.529 | INFO | "gpu" : " 17 GB", | 20260505 07:51:05.529 | INFO | "huge" : "", | 20260505 07:51:05.529 | INFO | "huge_shared" : " 27 GB", | 20260505 07:51:05.529 | INFO | "malloc" : " 14 GB", | 20260505 07:51:05.529 | INFO | "pinned" : " 29 GB" | 20260505 07:51:05.529 | INFO | } | 20260505 07:51:05.529 | INFO | Perf report for the last 5.13787 seconds: | 20260505 07:51:05.529 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:51:05.529 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:05.529 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:05.529 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:05.529 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 07:51:05.529 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:05.529 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:05.529 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 07:51:05.529 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:05.529 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:05.530 | INFO | Uploaded 36GiB to GPU in 2.1556s (16.7007GiB/s) | 20260505 07:51:05.530 | INFO | Downloaded 34GiB to GPU in 2.15146s (15.8032GiB/s) | 20260505 07:51:10.668 | INFO | Analyzing chunk frames = [1551872,1552384) | 20260505 07:51:10.668 | INFO | MemInfo: { | 20260505 07:51:10.668 | INFO | "gpu" : " 17 GB", | 20260505 07:51:10.668 | INFO | "huge" : "", | 20260505 07:51:10.668 | INFO | "huge_shared" : " 27 GB", | 20260505 07:51:10.668 | INFO | "malloc" : " 14 GB", | 20260505 07:51:10.668 | INFO | "pinned" : " 29 GB" | 20260505 07:51:10.668 | INFO | } | 20260505 07:51:10.669 | INFO | Perf report for the last 5.13939 seconds: | 20260505 07:51:10.669 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:51:10.669 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:10.669 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:10.669 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:10.669 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 07:51:10.669 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:10.669 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:10.669 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 07:51:10.669 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:10.669 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:10.669 | INFO | Uploaded 36GiB to GPU in 2.18081s (16.5076GiB/s) | 20260505 07:51:10.669 | INFO | Downloaded 34GiB to GPU in 2.0526s (16.5644GiB/s) | 20260505 07:51:10.799 | INFO | Starting a round of Baz IO | 20260505 07:51:14.084 | INFO | Populating RTMetrics report | 20260505 07:51:14.084 | INFO | RTMetrics report populated | 20260505 07:51:14.085 | INFO | Opening tmp RTMetrics json file | 20260505 07:51:14.107 | INFO | Serializing RTMetrics to json | 20260505 07:51:14.108 | INFO | Serialization done, swapping tmp file now... | 20260505 07:51:14.109 | INFO | RTMetrics done | 20260505 07:51:14.228 | INFO | Finished a round of Baz IO: Wrote 2.442 GiB in 3.428 seconds | 20260505 07:51:14.228 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 28.87% / 0.03% / 71.10% | 20260505 07:51:14.228 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.18238% / 5.16574e-05% / 6.35783e-05% | 20260505 07:51:14.228 | INFO | Baz IO overall: | 20260505 07:51:14.228 | INFO | BytesWritten : 859553971769 | 20260505 07:51:14.228 | INFO | NumEvents : 408176187319 | 20260505 07:51:15.808 | INFO | Analyzing chunk frames = [1552384,1552896) | 20260505 07:51:15.808 | INFO | MemInfo: { | 20260505 07:51:15.808 | INFO | "gpu" : " 17 GB", | 20260505 07:51:15.808 | INFO | "huge" : "", | 20260505 07:51:15.808 | INFO | "huge_shared" : " 27 GB", | 20260505 07:51:15.808 | INFO | "malloc" : " 14 GB", | 20260505 07:51:15.808 | INFO | "pinned" : " 29 GB" | 20260505 07:51:15.808 | INFO | } | 20260505 07:51:15.808 | INFO | Perf report for the last 5.1391 seconds: | 20260505 07:51:15.808 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:51:15.808 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:15.808 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:15.808 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:15.808 | INFO | ANALYSIS : 65.3%, 2.9, 98.8ms, 0 | 20260505 07:51:15.808 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:15.808 | INFO | RT_METRICS : 46.6%, 1.0, 24.4ms, 0 | 20260505 07:51:15.808 | INFO | PRE_HQ : 63.9%, 1.9, 62.1ms, 0 | 20260505 07:51:15.808 | INFO | BAZWRITER : 66.0%, 4.2, 142.5ms, 0 | 20260505 07:51:15.808 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260505 07:51:15.808 | INFO | Uploaded 36GiB to GPU in 2.22453s (16.1832GiB/s) | 20260505 07:51:15.808 | INFO | Downloaded 38GiB to GPU in 2.33051s (16.3055GiB/s) | 20260505 07:51:20.947 | INFO | Analyzing chunk frames = [1552896,1553408) | 20260505 07:51:20.947 | INFO | MemInfo: { | 20260505 07:51:20.947 | INFO | "gpu" : " 17 GB", | 20260505 07:51:20.947 | INFO | "huge" : "", | 20260505 07:51:20.947 | INFO | "huge_shared" : " 27 GB", | 20260505 07:51:20.947 | INFO | "malloc" : " 14 GB", | 20260505 07:51:20.947 | INFO | "pinned" : " 29 GB" | 20260505 07:51:20.947 | INFO | } | 20260505 07:51:20.947 | INFO | Perf report for the last 5.13933 seconds: | 20260505 07:51:20.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:51:20.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:20.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:20.947 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:20.947 | INFO | ANALYSIS : 64.4%, 3.0, 100.6ms, 0 | 20260505 07:51:20.947 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:20.947 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:20.947 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 07:51:20.947 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:20.947 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:20.948 | INFO | Uploaded 36GiB to GPU in 2.17991s (16.5145GiB/s) | 20260505 07:51:20.948 | INFO | Downloaded 34GiB to GPU in 2.05453s (16.5488GiB/s) | 20260505 07:51:26.086 | INFO | Analyzing chunk frames = [1553408,1553920) | 20260505 07:51:26.086 | INFO | MemInfo: { | 20260505 07:51:26.086 | INFO | "gpu" : " 17 GB", | 20260505 07:51:26.086 | INFO | "huge" : "", | 20260505 07:51:26.086 | INFO | "huge_shared" : " 27 GB", | 20260505 07:51:26.086 | INFO | "malloc" : " 14 GB", | 20260505 07:51:26.086 | INFO | "pinned" : " 29 GB" | 20260505 07:51:26.086 | INFO | } | 20260505 07:51:26.086 | INFO | Perf report for the last 5.13918 seconds: | 20260505 07:51:26.087 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:51:26.087 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:26.087 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:26.087 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:26.087 | INFO | ANALYSIS : 64.2%, 3.0, 99.9ms, 0 | 20260505 07:51:26.087 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:26.087 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:26.087 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:51:26.087 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:26.087 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:26.087 | INFO | Uploaded 36GiB to GPU in 2.19456s (16.4042GiB/s) | 20260505 07:51:26.087 | INFO | Downloaded 34GiB to GPU in 2.06487s (16.466GiB/s) | 20260505 07:51:31.226 | INFO | Analyzing chunk frames = [1553920,1554432) | 20260505 07:51:31.226 | INFO | MemInfo: { | 20260505 07:51:31.226 | INFO | "gpu" : " 17 GB", | 20260505 07:51:31.226 | INFO | "huge" : "", | 20260505 07:51:31.226 | INFO | "huge_shared" : " 27 GB", | 20260505 07:51:31.226 | INFO | "malloc" : " 14 GB", | 20260505 07:51:31.226 | INFO | "pinned" : " 29 GB" | 20260505 07:51:31.226 | INFO | } | 20260505 07:51:31.226 | INFO | Perf report for the last 5.13932 seconds: | 20260505 07:51:31.226 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:51:31.226 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:31.226 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:31.226 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:31.226 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 07:51:31.226 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:31.226 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:31.226 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:51:31.226 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:31.226 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:31.226 | INFO | Uploaded 36GiB to GPU in 2.156s (16.6976GiB/s) | 20260505 07:51:31.226 | INFO | Downloaded 34GiB to GPU in 2.04026s (16.6646GiB/s) | 20260505 07:51:36.365 | INFO | Analyzing chunk frames = [1554432,1554944) | 20260505 07:51:36.365 | INFO | MemInfo: { | 20260505 07:51:36.365 | INFO | "gpu" : " 17 GB", | 20260505 07:51:36.365 | INFO | "huge" : "", | 20260505 07:51:36.365 | INFO | "huge_shared" : " 27 GB", | 20260505 07:51:36.365 | INFO | "malloc" : " 14 GB", | 20260505 07:51:36.365 | INFO | "pinned" : " 29 GB" | 20260505 07:51:36.365 | INFO | } | 20260505 07:51:36.365 | INFO | Perf report for the last 5.13957 seconds: | 20260505 07:51:36.365 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:51:36.365 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:36.365 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:36.365 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:36.365 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 07:51:36.365 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:36.365 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:36.365 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:51:36.365 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:36.366 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:36.366 | INFO | Uploaded 36GiB to GPU in 2.19882s (16.3725GiB/s) | 20260505 07:51:36.366 | INFO | Downloaded 34GiB to GPU in 2.05227s (16.567GiB/s) | 20260505 07:51:41.506 | INFO | Analyzing chunk frames = [1554944,1555456) | 20260505 07:51:41.506 | INFO | MemInfo: { | 20260505 07:51:41.506 | INFO | "gpu" : " 17 GB", | 20260505 07:51:41.506 | INFO | "huge" : "", | 20260505 07:51:41.506 | INFO | "huge_shared" : " 27 GB", | 20260505 07:51:41.506 | INFO | "malloc" : " 14 GB", | 20260505 07:51:41.506 | INFO | "pinned" : " 29 GB" | 20260505 07:51:41.506 | INFO | } | 20260505 07:51:41.506 | INFO | Perf report for the last 5.141 seconds: | 20260505 07:51:41.506 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:51:41.506 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:41.506 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:41.506 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:41.506 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 07:51:41.506 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:41.506 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:41.506 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:51:41.506 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:41.506 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:41.507 | INFO | Uploaded 36GiB to GPU in 2.19572s (16.3956GiB/s) | 20260505 07:51:41.507 | INFO | Downloaded 34GiB to GPU in 2.04986s (16.5865GiB/s) | 20260505 07:51:46.644 | INFO | Analyzing chunk frames = [1555456,1555968) | 20260505 07:51:46.644 | INFO | MemInfo: { | 20260505 07:51:46.644 | INFO | "gpu" : " 17 GB", | 20260505 07:51:46.644 | INFO | "huge" : "", | 20260505 07:51:46.644 | INFO | "huge_shared" : " 27 GB", | 20260505 07:51:46.644 | INFO | "malloc" : " 14 GB", | 20260505 07:51:46.644 | INFO | "pinned" : " 29 GB" | 20260505 07:51:46.644 | INFO | } | 20260505 07:51:46.644 | INFO | Perf report for the last 5.13748 seconds: | 20260505 07:51:46.644 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:51:46.644 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:46.644 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:46.644 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:46.644 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 07:51:46.644 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:46.644 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:46.644 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260505 07:51:46.644 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:46.644 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:46.644 | INFO | Uploaded 36GiB to GPU in 2.16244s (16.6478GiB/s) | 20260505 07:51:46.644 | INFO | Downloaded 34GiB to GPU in 2.13919s (15.8938GiB/s) | 20260505 07:51:51.783 | INFO | Analyzing chunk frames = [1555968,1556480) | 20260505 07:51:51.783 | INFO | MemInfo: { | 20260505 07:51:51.783 | INFO | "gpu" : " 17 GB", | 20260505 07:51:51.783 | INFO | "huge" : "", | 20260505 07:51:51.783 | INFO | "huge_shared" : " 27 GB", | 20260505 07:51:51.783 | INFO | "malloc" : " 14 GB", | 20260505 07:51:51.783 | INFO | "pinned" : " 29 GB" | 20260505 07:51:51.783 | INFO | } | 20260505 07:51:51.783 | INFO | Perf report for the last 5.13949 seconds: | 20260505 07:51:51.783 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:51:51.783 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:51.783 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:51.783 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:51.783 | INFO | ANALYSIS : 64.8%, 3.0, 101.2ms, 0 | 20260505 07:51:51.783 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:51.783 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:51.783 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 07:51:51.783 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:51.783 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:51.784 | INFO | Uploaded 36GiB to GPU in 2.21957s (16.2194GiB/s) | 20260505 07:51:51.784 | INFO | Downloaded 34GiB to GPU in 2.04424s (16.6321GiB/s) | 20260505 07:51:51.784 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 07:51:51.784 | INFO | Total mem usage: 15.3 GiB | 20260505 07:51:51.784 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 07:51:51.784 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -5.45%) | 20260505 07:51:51.784 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 07:51:51.784 | INFO | High Water Breakdown: | 20260505 07:51:51.784 | INFO | 60.8% : 8.84 GiB ( 94.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 07:51:51.784 | INFO | 39% : 5.67 GiB ( 85.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 07:51:51.784 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 07:51:51.784 | INFO | ------------------------------------------ | 20260505 07:51:51.784 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:51:51.784 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 07:51:51.784 | INFO | Total mem usage: 26.5 GiB | 20260505 07:51:51.784 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 07:51:51.784 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 07:51:51.784 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 07:51:51.784 | INFO | High Water Breakdown: | 20260505 07:51:51.784 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:51:51.784 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:51:51.784 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:51:51.784 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:51:51.784 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:51:51.784 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:51:51.784 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 07:51:51.784 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 07:51:51.784 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:51:51.784 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 07:51:51.784 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 07:51:51.784 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 07:51:51.784 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 07:51:51.784 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 07:51:51.784 | INFO | ------------------------------------------ | 20260505 07:51:51.784 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:51:51.784 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 07:51:51.784 | INFO | Total mem usage: 25.6 GiB | 20260505 07:51:51.784 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 07:51:51.784 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 07:51:51.784 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 07:51:51.784 | INFO | High Water Breakdown: | 20260505 07:51:51.784 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 07:51:51.784 | INFO | ------------------------------------------ | 20260505 07:51:51.784 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:51:51.784 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 07:51:51.785 | INFO | Total mem usage: 15.7 GiB | 20260505 07:51:51.785 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 07:51:51.785 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 07:51:51.785 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 07:51:51.785 | INFO | High Water Breakdown: | 20260505 07:51:51.785 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 07:51:51.785 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 07:51:51.785 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 07:51:51.785 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 07:51:51.785 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 07:51:51.785 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 07:51:51.785 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 07:51:51.785 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 07:51:51.785 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 07:51:51.785 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:51:51.785 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 07:51:51.785 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:51:51.785 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 07:51:51.785 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 07:51:51.785 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 07:51:51.785 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 07:51:51.785 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:51:51.785 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 07:51:51.785 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:51:51.785 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:51:51.785 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:51:51.785 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 07:51:51.785 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 07:51:51.785 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:51:51.785 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 07:51:51.785 | INFO | ------------------------------------------ | 20260505 07:51:51.785 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:51:51.785 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 07:51:51.785 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:51:51.785 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:51:51.785 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.54e+01, 1.38e+01 / 2.06e+00 | 20260505 07:51:51.785 | INFO | PerfReport: 0.29%, AggregateStats -- 9.39e-02 / 1.04e-01, 1.09e-02 / 5.03e-03 | 20260505 07:51:51.785 | INFO | PerfReport: 11.59%, Baseline -- 3.81e+00 / 4.45e+00, 6.22e-01 / 2.99e-01 | 20260505 07:51:51.785 | INFO | PerfReport: 3.31%, Histogram -- 1.09e+00 / 1.48e+00, 2.13e-01 / 1.07e-01 | 20260505 07:51:51.785 | INFO | PerfReport: 15.52%, DME -- 4.08e+01 / 4.48e+01, 4.25e+00 / 1.31e+00 | 20260505 07:51:51.785 | INFO | PerfReport: 56.43%, FrameLabeling -- 1.85e+01 / 1.93e+01, 2.06e+00 / 1.56e+00 | 20260505 07:51:51.785 | INFO | PerfReport: 7.51%, PulseAccumulating -- 2.47e+00 / 3.10e+00, 4.26e-01 / 2.33e-01 | 20260505 07:51:51.785 | INFO | PerfReport: 3.94%, Metrics -- 1.30e+00 / 1.61e+00, 3.72e-01 / 1.88e-01 | 20260505 07:51:51.785 | INFO | PerfReport: 1.42%, PulseSerializing -- 4.65e-01 / 5.24e-01, 8.95e-02 / 2.71e-02 | 20260505 07:51:51.785 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 07:51:51.785 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:51:51.785 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:51:51.785 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.31e+01 / 0.00e+00 | 20260505 07:51:51.785 | INFO | PerfReport: 22.42%, Upload -- 2.28e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260505 07:51:51.785 | INFO | PerfReport: 21.49%, Download -- 2.19e+01 / 0.00e+00, 8.06e+00 / 0.00e+00 | 20260505 07:51:51.785 | INFO | PerfReport: 32.31%, Compute -- 3.29e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 07:51:51.785 | INFO | PerfReport: 1.75%, LockUpload -- 1.78e+00 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260505 07:51:51.785 | INFO | PerfReport: 0.83%, LockDownload -- 8.45e-01 / 0.00e+00, 2.23e+00 / 0.00e+00 | 20260505 07:51:51.785 | INFO | PerfReport: 21.19%, LockCompute -- 2.16e+01 / 0.00e+00, 1.75e+01 / 0.00e+00 | 20260505 07:51:51.785 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 1.91e-02 / 0.00e+00 | 20260505 07:51:51.913 | INFO | Starting a round of Baz IO | 20260505 07:51:55.204 | INFO | Populating RTMetrics report | 20260505 07:51:55.204 | INFO | RTMetrics report populated | 20260505 07:51:55.204 | INFO | Opening tmp RTMetrics json file | 20260505 07:51:55.222 | INFO | Serializing RTMetrics to json | 20260505 07:51:55.228 | INFO | Serialization done, swapping tmp file now... | 20260505 07:51:55.231 | INFO | RTMetrics done | 20260505 07:51:55.347 | INFO | Finished a round of Baz IO: Wrote 2.445 GiB in 3.433 seconds | 20260505 07:51:55.347 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 28.84% / 0.03% / 71.13% | 20260505 07:51:55.347 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.17281% / 1.19209e-05% / 7.94729e-05% | 20260505 07:51:55.347 | INFO | Baz IO overall: | 20260505 07:51:55.347 | INFO | BytesWritten : 862178737800 | 20260505 07:51:55.347 | INFO | NumEvents : 409431822399 | 20260505 07:51:56.922 | INFO | Analyzing chunk frames = [1556480,1556992) | 20260505 07:51:56.922 | INFO | MemInfo: { | 20260505 07:51:56.922 | INFO | "gpu" : " 17 GB", | 20260505 07:51:56.922 | INFO | "huge" : "", | 20260505 07:51:56.922 | INFO | "huge_shared" : " 27 GB", | 20260505 07:51:56.922 | INFO | "malloc" : " 14 GB", | 20260505 07:51:56.922 | INFO | "pinned" : " 29 GB" | 20260505 07:51:56.922 | INFO | } | 20260505 07:51:56.922 | INFO | Perf report for the last 5.13902 seconds: | 20260505 07:51:56.922 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:51:56.922 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:56.922 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:56.922 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:56.922 | INFO | ANALYSIS : 65.4%, 2.9, 99.5ms, 0 | 20260505 07:51:56.922 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:51:56.922 | INFO | RT_METRICS : 46.6%, 1.0, 24.4ms, 0 | 20260505 07:51:56.922 | INFO | PRE_HQ : 63.9%, 1.9, 61.7ms, 0 | 20260505 07:51:56.922 | INFO | BAZWRITER : 66.1%, 4.2, 141.6ms, 0 | 20260505 07:51:56.922 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 07:51:56.923 | INFO | Uploaded 36GiB to GPU in 2.22343s (16.1912GiB/s) | 20260505 07:51:56.923 | INFO | Downloaded 38GiB to GPU in 2.33917s (16.2451GiB/s) | 20260505 07:52:02.062 | INFO | Analyzing chunk frames = [1556992,1557504) | 20260505 07:52:02.062 | INFO | MemInfo: { | 20260505 07:52:02.062 | INFO | "gpu" : " 17 GB", | 20260505 07:52:02.062 | INFO | "huge" : "", | 20260505 07:52:02.062 | INFO | "huge_shared" : " 27 GB", | 20260505 07:52:02.062 | INFO | "malloc" : " 14 GB", | 20260505 07:52:02.062 | INFO | "pinned" : " 29 GB" | 20260505 07:52:02.062 | INFO | } | 20260505 07:52:02.062 | INFO | Perf report for the last 5.1396 seconds: | 20260505 07:52:02.062 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:52:02.062 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:02.062 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:02.062 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:02.062 | INFO | ANALYSIS : 64.5%, 3.0, 100.7ms, 0 | 20260505 07:52:02.062 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:02.062 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:02.062 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 07:52:02.062 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:02.062 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:02.062 | INFO | Uploaded 36GiB to GPU in 2.18617s (16.4672GiB/s) | 20260505 07:52:02.062 | INFO | Downloaded 34GiB to GPU in 2.05974s (16.5069GiB/s) | 20260505 07:52:07.201 | INFO | Analyzing chunk frames = [1557504,1558016) | 20260505 07:52:07.201 | INFO | MemInfo: { | 20260505 07:52:07.201 | INFO | "gpu" : " 17 GB", | 20260505 07:52:07.201 | INFO | "huge" : "", | 20260505 07:52:07.201 | INFO | "huge_shared" : " 27 GB", | 20260505 07:52:07.201 | INFO | "malloc" : " 14 GB", | 20260505 07:52:07.201 | INFO | "pinned" : " 29 GB" | 20260505 07:52:07.201 | INFO | } | 20260505 07:52:07.201 | INFO | Perf report for the last 5.1392 seconds: | 20260505 07:52:07.201 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:52:07.201 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:07.201 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:07.201 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:07.201 | INFO | ANALYSIS : 64.2%, 3.0, 99.9ms, 0 | 20260505 07:52:07.201 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:07.201 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:07.201 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:52:07.201 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:07.201 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:07.202 | INFO | Uploaded 36GiB to GPU in 2.19688s (16.3869GiB/s) | 20260505 07:52:07.202 | INFO | Downloaded 34GiB to GPU in 2.06858s (16.4364GiB/s) | 20260505 07:52:12.340 | INFO | Analyzing chunk frames = [1558016,1558528) | 20260505 07:52:12.340 | INFO | MemInfo: { | 20260505 07:52:12.340 | INFO | "gpu" : " 17 GB", | 20260505 07:52:12.340 | INFO | "huge" : "", | 20260505 07:52:12.340 | INFO | "huge_shared" : " 27 GB", | 20260505 07:52:12.340 | INFO | "malloc" : " 14 GB", | 20260505 07:52:12.340 | INFO | "pinned" : " 29 GB" | 20260505 07:52:12.340 | INFO | } | 20260505 07:52:12.341 | INFO | Perf report for the last 5.13939 seconds: | 20260505 07:52:12.341 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:52:12.341 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:12.341 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:12.341 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:12.341 | INFO | ANALYSIS : 63.2%, 3.0, 98.1ms, 0 | 20260505 07:52:12.341 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:12.341 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:12.341 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:52:12.341 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:12.341 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:12.341 | INFO | Uploaded 36GiB to GPU in 2.15782s (16.6835GiB/s) | 20260505 07:52:12.341 | INFO | Downloaded 34GiB to GPU in 2.04351s (16.638GiB/s) | 20260505 07:52:17.480 | INFO | Analyzing chunk frames = [1558528,1559040) | 20260505 07:52:17.480 | INFO | MemInfo: { | 20260505 07:52:17.480 | INFO | "gpu" : " 17 GB", | 20260505 07:52:17.480 | INFO | "huge" : "", | 20260505 07:52:17.480 | INFO | "huge_shared" : " 27 GB", | 20260505 07:52:17.480 | INFO | "malloc" : " 14 GB", | 20260505 07:52:17.480 | INFO | "pinned" : " 29 GB" | 20260505 07:52:17.480 | INFO | } | 20260505 07:52:17.480 | INFO | Perf report for the last 5.13951 seconds: | 20260505 07:52:17.480 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:52:17.480 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:17.480 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:17.480 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:17.480 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 07:52:17.480 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:17.480 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:17.480 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:52:17.480 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:17.480 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:17.480 | INFO | Uploaded 36GiB to GPU in 2.14438s (16.788GiB/s) | 20260505 07:52:17.480 | INFO | Downloaded 34GiB to GPU in 2.04805s (16.6012GiB/s) | 20260505 07:52:22.621 | INFO | Analyzing chunk frames = [1559040,1559552) | 20260505 07:52:22.621 | INFO | MemInfo: { | 20260505 07:52:22.621 | INFO | "gpu" : " 17 GB", | 20260505 07:52:22.621 | INFO | "huge" : "", | 20260505 07:52:22.621 | INFO | "huge_shared" : " 27 GB", | 20260505 07:52:22.621 | INFO | "malloc" : " 14 GB", | 20260505 07:52:22.621 | INFO | "pinned" : " 29 GB" | 20260505 07:52:22.621 | INFO | } | 20260505 07:52:22.621 | INFO | Perf report for the last 5.14085 seconds: | 20260505 07:52:22.621 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:52:22.621 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:22.621 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:22.621 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:22.621 | INFO | ANALYSIS : 64.1%, 3.0, 100.2ms, 0 | 20260505 07:52:22.621 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:22.621 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:22.621 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 07:52:22.621 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:22.621 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:22.621 | INFO | Uploaded 36GiB to GPU in 2.20127s (16.3542GiB/s) | 20260505 07:52:22.621 | INFO | Downloaded 34GiB to GPU in 2.04993s (16.5859GiB/s) | 20260505 07:52:27.758 | INFO | Analyzing chunk frames = [1559552,1560064) | 20260505 07:52:27.758 | INFO | MemInfo: { | 20260505 07:52:27.758 | INFO | "gpu" : " 17 GB", | 20260505 07:52:27.758 | INFO | "huge" : "", | 20260505 07:52:27.758 | INFO | "huge_shared" : " 27 GB", | 20260505 07:52:27.758 | INFO | "malloc" : " 14 GB", | 20260505 07:52:27.758 | INFO | "pinned" : " 29 GB" | 20260505 07:52:27.758 | INFO | } | 20260505 07:52:27.758 | INFO | Perf report for the last 5.13747 seconds: | 20260505 07:52:27.758 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:52:27.759 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:27.759 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:27.759 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:27.759 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 07:52:27.759 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:27.759 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:27.759 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 07:52:27.759 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:27.759 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:27.759 | INFO | Uploaded 36GiB to GPU in 2.16188s (16.6522GiB/s) | 20260505 07:52:27.759 | INFO | Downloaded 34GiB to GPU in 2.14237s (15.8703GiB/s) | 20260505 07:52:32.898 | INFO | Analyzing chunk frames = [1560064,1560576) | 20260505 07:52:32.898 | INFO | MemInfo: { | 20260505 07:52:32.898 | INFO | "gpu" : " 17 GB", | 20260505 07:52:32.898 | INFO | "huge" : "", | 20260505 07:52:32.898 | INFO | "huge_shared" : " 27 GB", | 20260505 07:52:32.898 | INFO | "malloc" : " 14 GB", | 20260505 07:52:32.898 | INFO | "pinned" : " 29 GB" | 20260505 07:52:32.898 | INFO | } | 20260505 07:52:32.898 | INFO | Perf report for the last 5.1394 seconds: | 20260505 07:52:32.898 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:52:32.898 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:32.898 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:32.898 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:32.898 | INFO | ANALYSIS : 64.1%, 3.0, 100.0ms, 0 | 20260505 07:52:32.898 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:32.898 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:32.898 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 07:52:32.898 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:32.898 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:32.898 | INFO | Uploaded 36GiB to GPU in 2.18388s (16.4844GiB/s) | 20260505 07:52:32.899 | INFO | Downloaded 34GiB to GPU in 2.05649s (16.533GiB/s) | 20260505 07:52:33.032 | INFO | Starting a round of Baz IO | 20260505 07:52:36.312 | INFO | Populating RTMetrics report | 20260505 07:52:36.312 | INFO | RTMetrics report populated | 20260505 07:52:36.312 | INFO | Opening tmp RTMetrics json file | 20260505 07:52:36.318 | INFO | Serializing RTMetrics to json | 20260505 07:52:36.342 | INFO | Serialization done, swapping tmp file now... | 20260505 07:52:36.343 | INFO | RTMetrics done | 20260505 07:52:36.461 | INFO | Finished a round of Baz IO: Wrote 2.444 GiB in 3.428 seconds | 20260505 07:52:36.461 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 28.82% / 0.03% / 71.16% | 20260505 07:52:36.461 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.16027% / 3.97364e-05% / 7.15256e-05% | 20260505 07:52:36.461 | INFO | Baz IO overall: | 20260505 07:52:36.461 | INFO | BytesWritten : 864802989099 | 20260505 07:52:36.461 | INFO | NumEvents : 410686927982 | 20260505 07:52:38.037 | INFO | Analyzing chunk frames = [1560576,1561088) | 20260505 07:52:38.037 | INFO | MemInfo: { | 20260505 07:52:38.037 | INFO | "gpu" : " 17 GB", | 20260505 07:52:38.037 | INFO | "huge" : "", | 20260505 07:52:38.037 | INFO | "huge_shared" : " 27 GB", | 20260505 07:52:38.037 | INFO | "malloc" : " 14 GB", | 20260505 07:52:38.037 | INFO | "pinned" : " 29 GB" | 20260505 07:52:38.037 | INFO | } | 20260505 07:52:38.037 | INFO | Perf report for the last 5.13949 seconds: | 20260505 07:52:38.037 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:52:38.037 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:38.037 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:38.037 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:38.037 | INFO | ANALYSIS : 65.3%, 2.9, 99.2ms, 0 | 20260505 07:52:38.037 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:38.037 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260505 07:52:38.037 | INFO | PRE_HQ : 63.9%, 1.9, 62.2ms, 0 | 20260505 07:52:38.037 | INFO | BAZWRITER : 66.0%, 4.2, 142.0ms, 0 | 20260505 07:52:38.037 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 07:52:38.038 | INFO | Uploaded 36GiB to GPU in 2.21333s (16.2651GiB/s) | 20260505 07:52:38.038 | INFO | Downloaded 38GiB to GPU in 2.34015s (16.2383GiB/s) | 20260505 07:52:43.177 | INFO | Analyzing chunk frames = [1561088,1561600) | 20260505 07:52:43.177 | INFO | MemInfo: { | 20260505 07:52:43.177 | INFO | "gpu" : " 17 GB", | 20260505 07:52:43.177 | INFO | "huge" : "", | 20260505 07:52:43.177 | INFO | "huge_shared" : " 27 GB", | 20260505 07:52:43.177 | INFO | "malloc" : " 14 GB", | 20260505 07:52:43.177 | INFO | "pinned" : " 29 GB" | 20260505 07:52:43.177 | INFO | } | 20260505 07:52:43.177 | INFO | Perf report for the last 5.13941 seconds: | 20260505 07:52:43.177 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:52:43.177 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:43.177 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:43.177 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:43.177 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 07:52:43.177 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:43.177 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:43.177 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:52:43.177 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:43.177 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:43.177 | INFO | Uploaded 36GiB to GPU in 2.17994s (16.5143GiB/s) | 20260505 07:52:43.177 | INFO | Downloaded 34GiB to GPU in 2.05042s (16.582GiB/s) | 20260505 07:52:48.316 | INFO | Analyzing chunk frames = [1561600,1562112) | 20260505 07:52:48.316 | INFO | MemInfo: { | 20260505 07:52:48.316 | INFO | "gpu" : " 17 GB", | 20260505 07:52:48.316 | INFO | "huge" : "", | 20260505 07:52:48.316 | INFO | "huge_shared" : " 27 GB", | 20260505 07:52:48.316 | INFO | "malloc" : " 14 GB", | 20260505 07:52:48.316 | INFO | "pinned" : " 29 GB" | 20260505 07:52:48.316 | INFO | } | 20260505 07:52:48.316 | INFO | Perf report for the last 5.13903 seconds: | 20260505 07:52:48.316 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:52:48.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:48.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:48.316 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:48.316 | INFO | ANALYSIS : 64.2%, 3.0, 99.9ms, 0 | 20260505 07:52:48.316 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:48.316 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:48.316 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 07:52:48.316 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:48.316 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:48.316 | INFO | Uploaded 36GiB to GPU in 2.19652s (16.3896GiB/s) | 20260505 07:52:48.316 | INFO | Downloaded 34GiB to GPU in 2.0648s (16.4665GiB/s) | 20260505 07:52:53.456 | INFO | Analyzing chunk frames = [1562112,1562624) | 20260505 07:52:53.456 | INFO | MemInfo: { | 20260505 07:52:53.456 | INFO | "gpu" : " 17 GB", | 20260505 07:52:53.456 | INFO | "huge" : "", | 20260505 07:52:53.456 | INFO | "huge_shared" : " 27 GB", | 20260505 07:52:53.456 | INFO | "malloc" : " 14 GB", | 20260505 07:52:53.456 | INFO | "pinned" : " 29 GB" | 20260505 07:52:53.456 | INFO | } | 20260505 07:52:53.456 | INFO | Perf report for the last 5.14009 seconds: | 20260505 07:52:53.456 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:52:53.456 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:53.456 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:53.456 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:53.456 | INFO | ANALYSIS : 63.2%, 3.0, 98.1ms, 0 | 20260505 07:52:53.456 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:53.456 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:53.456 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 07:52:53.456 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:53.456 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:53.456 | INFO | Uploaded 36GiB to GPU in 2.15777s (16.6839GiB/s) | 20260505 07:52:53.456 | INFO | Downloaded 34GiB to GPU in 2.04546s (16.6222GiB/s) | 20260505 07:52:58.594 | INFO | Analyzing chunk frames = [1562624,1563136) | 20260505 07:52:58.594 | INFO | MemInfo: { | 20260505 07:52:58.594 | INFO | "gpu" : " 17 GB", | 20260505 07:52:58.594 | INFO | "huge" : "", | 20260505 07:52:58.594 | INFO | "huge_shared" : " 27 GB", | 20260505 07:52:58.594 | INFO | "malloc" : " 14 GB", | 20260505 07:52:58.594 | INFO | "pinned" : " 29 GB" | 20260505 07:52:58.595 | INFO | } | 20260505 07:52:58.595 | INFO | Perf report for the last 5.13876 seconds: | 20260505 07:52:58.595 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:52:58.595 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:58.595 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:58.595 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:58.595 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 07:52:58.595 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:58.595 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:58.595 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:52:58.595 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:58.595 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:52:58.595 | INFO | Uploaded 36GiB to GPU in 2.19772s (16.3806GiB/s) | 20260505 07:52:58.595 | INFO | Downloaded 34GiB to GPU in 2.05602s (16.5368GiB/s) | 20260505 07:53:03.735 | INFO | Analyzing chunk frames = [1563136,1563648) | 20260505 07:53:03.735 | INFO | MemInfo: { | 20260505 07:53:03.735 | INFO | "gpu" : " 17 GB", | 20260505 07:53:03.735 | INFO | "huge" : "", | 20260505 07:53:03.735 | INFO | "huge_shared" : " 27 GB", | 20260505 07:53:03.735 | INFO | "malloc" : " 14 GB", | 20260505 07:53:03.736 | INFO | "pinned" : " 29 GB" | 20260505 07:53:03.736 | INFO | } | 20260505 07:53:03.736 | INFO | Perf report for the last 5.1409 seconds: | 20260505 07:53:03.736 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:53:03.736 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:03.736 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:03.736 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:03.736 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 07:53:03.736 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:03.736 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:03.736 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:53:03.736 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:03.736 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:03.736 | INFO | Uploaded 36GiB to GPU in 2.20487s (16.3275GiB/s) | 20260505 07:53:03.736 | INFO | Downloaded 34GiB to GPU in 2.05128s (16.575GiB/s) | 20260505 07:53:08.873 | INFO | Analyzing chunk frames = [1563648,1564160) | 20260505 07:53:08.873 | INFO | MemInfo: { | 20260505 07:53:08.873 | INFO | "gpu" : " 17 GB", | 20260505 07:53:08.873 | INFO | "huge" : "", | 20260505 07:53:08.873 | INFO | "huge_shared" : " 27 GB", | 20260505 07:53:08.873 | INFO | "malloc" : " 14 GB", | 20260505 07:53:08.873 | INFO | "pinned" : " 29 GB" | 20260505 07:53:08.873 | INFO | } | 20260505 07:53:08.873 | INFO | Perf report for the last 5.13765 seconds: | 20260505 07:53:08.873 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:53:08.873 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:08.873 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:08.873 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:08.873 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 07:53:08.873 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:08.873 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:08.873 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260505 07:53:08.874 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:08.874 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:08.874 | INFO | Uploaded 36GiB to GPU in 2.15068s (16.7389GiB/s) | 20260505 07:53:08.874 | INFO | Downloaded 34GiB to GPU in 2.12299s (16.0152GiB/s) | 20260505 07:53:14.013 | INFO | Analyzing chunk frames = [1564160,1564672) | 20260505 07:53:14.013 | INFO | MemInfo: { | 20260505 07:53:14.013 | INFO | "gpu" : " 17 GB", | 20260505 07:53:14.013 | INFO | "huge" : "", | 20260505 07:53:14.013 | INFO | "huge_shared" : " 27 GB", | 20260505 07:53:14.013 | INFO | "malloc" : " 14 GB", | 20260505 07:53:14.013 | INFO | "pinned" : " 29 GB" | 20260505 07:53:14.013 | INFO | } | 20260505 07:53:14.013 | INFO | Perf report for the last 5.13973 seconds: | 20260505 07:53:14.013 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:53:14.013 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:14.013 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:14.013 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:14.013 | INFO | ANALYSIS : 64.7%, 3.0, 101.1ms, 0 | 20260505 07:53:14.013 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:14.013 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:14.013 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 07:53:14.013 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:14.013 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:14.013 | INFO | Uploaded 36GiB to GPU in 2.21548s (16.2493GiB/s) | 20260505 07:53:14.013 | INFO | Downloaded 34GiB to GPU in 2.0454s (16.6227GiB/s) | 20260505 07:53:14.013 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 07:53:14.013 | INFO | Total mem usage: 15.3 GiB | 20260505 07:53:14.013 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 07:53:14.013 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -5.42%) | 20260505 07:53:14.013 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 07:53:14.013 | INFO | High Water Breakdown: | 20260505 07:53:14.013 | INFO | 60.9% : 8.85 GiB ( 94.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 07:53:14.013 | INFO | 39% : 5.67 GiB ( 85.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 07:53:14.013 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 07:53:14.013 | INFO | ------------------------------------------ | 20260505 07:53:14.013 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:53:14.013 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 07:53:14.013 | INFO | Total mem usage: 26.5 GiB | 20260505 07:53:14.014 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 07:53:14.014 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 07:53:14.014 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 07:53:14.014 | INFO | High Water Breakdown: | 20260505 07:53:14.014 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:53:14.014 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:53:14.014 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:53:14.014 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:53:14.014 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:53:14.014 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:53:14.014 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 07:53:14.014 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 07:53:14.014 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:53:14.014 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 07:53:14.014 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 07:53:14.014 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 07:53:14.014 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 07:53:14.014 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 07:53:14.014 | INFO | ------------------------------------------ | 20260505 07:53:14.014 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:53:14.014 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 07:53:14.014 | INFO | Total mem usage: 25.6 GiB | 20260505 07:53:14.014 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 07:53:14.014 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 07:53:14.014 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 07:53:14.014 | INFO | High Water Breakdown: | 20260505 07:53:14.014 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 07:53:14.014 | INFO | ------------------------------------------ | 20260505 07:53:14.014 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:53:14.014 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 07:53:14.014 | INFO | Total mem usage: 15.7 GiB | 20260505 07:53:14.014 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 07:53:14.014 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 07:53:14.014 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 07:53:14.014 | INFO | High Water Breakdown: | 20260505 07:53:14.014 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 07:53:14.014 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 07:53:14.014 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 07:53:14.014 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 07:53:14.014 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 07:53:14.014 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 07:53:14.014 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 07:53:14.014 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 07:53:14.014 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 07:53:14.014 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:53:14.014 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 07:53:14.014 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:53:14.014 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 07:53:14.014 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 07:53:14.014 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 07:53:14.014 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 07:53:14.014 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:53:14.014 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 07:53:14.014 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:53:14.014 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:53:14.014 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:53:14.014 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 07:53:14.014 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 07:53:14.014 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:53:14.014 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 07:53:14.014 | INFO | ------------------------------------------ | 20260505 07:53:14.014 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:53:14.014 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 07:53:14.014 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:53:14.014 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:53:14.014 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.53e+01, 1.37e+01 / 2.15e+00 | 20260505 07:53:14.014 | INFO | PerfReport: 0.28%, AggregateStats -- 9.35e-02 / 1.04e-01, 1.08e-02 / 5.22e-03 | 20260505 07:53:14.014 | INFO | PerfReport: 11.60%, Baseline -- 3.81e+00 / 4.45e+00, 6.18e-01 / 3.06e-01 | 20260505 07:53:14.014 | INFO | PerfReport: 3.31%, Histogram -- 1.09e+00 / 1.48e+00, 2.12e-01 / 1.17e-01 | 20260505 07:53:14.014 | INFO | PerfReport: 15.53%, DME -- 4.08e+01 / 4.48e+01, 4.30e+00 / 1.35e+00 | 20260505 07:53:14.014 | INFO | PerfReport: 56.39%, FrameLabeling -- 1.85e+01 / 1.92e+01, 2.02e+00 / 1.57e+00 | 20260505 07:53:14.014 | INFO | PerfReport: 7.54%, PulseAccumulating -- 2.47e+00 / 3.15e+00, 4.31e-01 / 2.21e-01 | 20260505 07:53:14.014 | INFO | PerfReport: 3.94%, Metrics -- 1.29e+00 / 1.63e+00, 3.68e-01 / 1.52e-01 | 20260505 07:53:14.014 | INFO | PerfReport: 1.42%, PulseSerializing -- 4.66e-01 / 5.27e-01, 9.07e-02 / 2.47e-02 | 20260505 07:53:14.015 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 07:53:14.015 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:53:14.015 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:53:14.015 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.28e+01 / 0.00e+00 | 20260505 07:53:14.015 | INFO | PerfReport: 22.39%, Upload -- 2.28e+01 / 0.00e+00, 8.10e+00 / 0.00e+00 | 20260505 07:53:14.015 | INFO | PerfReport: 21.50%, Download -- 2.19e+01 / 0.00e+00, 8.00e+00 / 0.00e+00 | 20260505 07:53:14.015 | INFO | PerfReport: 32.29%, Compute -- 3.28e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 07:53:14.015 | INFO | PerfReport: 1.70%, LockUpload -- 1.73e+00 / 0.00e+00, 7.35e+00 / 0.00e+00 | 20260505 07:53:14.015 | INFO | PerfReport: 0.87%, LockDownload -- 8.82e-01 / 0.00e+00, 2.30e+00 / 0.00e+00 | 20260505 07:53:14.015 | INFO | PerfReport: 21.23%, LockCompute -- 2.16e+01 / 0.00e+00, 1.75e+01 / 0.00e+00 | 20260505 07:53:14.015 | INFO | PerfReport: 0.01%, Misc -- 1.50e-02 / 0.00e+00, 3.16e-02 / 0.00e+00 | 20260505 07:53:14.147 | INFO | Starting a round of Baz IO | 20260505 07:53:17.435 | INFO | Populating RTMetrics report | 20260505 07:53:17.435 | INFO | RTMetrics report populated | 20260505 07:53:17.435 | INFO | Opening tmp RTMetrics json file | 20260505 07:53:17.453 | INFO | Serializing RTMetrics to json | 20260505 07:53:17.478 | INFO | Serialization done, swapping tmp file now... | 20260505 07:53:17.479 | INFO | RTMetrics done | 20260505 07:53:17.581 | INFO | Finished a round of Baz IO: Wrote 2.446 GiB in 3.434 seconds | 20260505 07:53:17.581 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 28.79% / 0.03% / 71.18% | 20260505 07:53:17.582 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.15804% / 7.15256e-05% / 4.76837e-05% | 20260505 07:53:17.582 | INFO | Baz IO overall: | 20260505 07:53:17.582 | INFO | BytesWritten : 867428935698 | 20260505 07:53:17.582 | INFO | NumEvents : 411943958390 | 20260505 07:53:19.152 | INFO | Analyzing chunk frames = [1564672,1565184) | 20260505 07:53:19.152 | INFO | MemInfo: { | 20260505 07:53:19.152 | INFO | "gpu" : " 17 GB", | 20260505 07:53:19.152 | INFO | "huge" : "", | 20260505 07:53:19.152 | INFO | "huge_shared" : " 27 GB", | 20260505 07:53:19.152 | INFO | "malloc" : " 14 GB", | 20260505 07:53:19.152 | INFO | "pinned" : " 29 GB" | 20260505 07:53:19.152 | INFO | } | 20260505 07:53:19.152 | INFO | Perf report for the last 5.13898 seconds: | 20260505 07:53:19.152 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:53:19.152 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:19.152 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:19.152 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:19.152 | INFO | ANALYSIS : 65.4%, 2.9, 99.4ms, 0 | 20260505 07:53:19.152 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:19.152 | INFO | RT_METRICS : 46.8%, 1.0, 24.5ms, 0 | 20260505 07:53:19.152 | INFO | PRE_HQ : 64.0%, 1.9, 62.1ms, 0 | 20260505 07:53:19.152 | INFO | BAZWRITER : 66.0%, 4.2, 141.9ms, 0 | 20260505 07:53:19.152 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260505 07:53:19.152 | INFO | Uploaded 36GiB to GPU in 2.20878s (16.2986GiB/s) | 20260505 07:53:19.152 | INFO | Downloaded 38GiB to GPU in 2.34512s (16.2039GiB/s) | 20260505 07:53:24.291 | INFO | Analyzing chunk frames = [1565184,1565696) | 20260505 07:53:24.291 | INFO | MemInfo: { | 20260505 07:53:24.291 | INFO | "gpu" : " 17 GB", | 20260505 07:53:24.291 | INFO | "huge" : "", | 20260505 07:53:24.291 | INFO | "huge_shared" : " 27 GB", | 20260505 07:53:24.291 | INFO | "malloc" : " 14 GB", | 20260505 07:53:24.291 | INFO | "pinned" : " 29 GB" | 20260505 07:53:24.291 | INFO | } | 20260505 07:53:24.291 | INFO | Perf report for the last 5.13947 seconds: | 20260505 07:53:24.292 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:53:24.292 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:24.292 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:24.292 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:24.292 | INFO | ANALYSIS : 64.3%, 3.0, 100.3ms, 0 | 20260505 07:53:24.292 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:24.292 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:24.292 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:53:24.292 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:24.292 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:24.292 | INFO | Uploaded 36GiB to GPU in 2.17801s (16.5289GiB/s) | 20260505 07:53:24.292 | INFO | Downloaded 34GiB to GPU in 2.0532s (16.5595GiB/s) | 20260505 07:53:29.431 | INFO | Analyzing chunk frames = [1565696,1566208) | 20260505 07:53:29.431 | INFO | MemInfo: { | 20260505 07:53:29.431 | INFO | "gpu" : " 17 GB", | 20260505 07:53:29.431 | INFO | "huge" : "", | 20260505 07:53:29.431 | INFO | "huge_shared" : " 27 GB", | 20260505 07:53:29.431 | INFO | "malloc" : " 14 GB", | 20260505 07:53:29.431 | INFO | "pinned" : " 29 GB" | 20260505 07:53:29.431 | INFO | } | 20260505 07:53:29.431 | INFO | Perf report for the last 5.13984 seconds: | 20260505 07:53:29.431 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:53:29.431 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:29.431 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:29.431 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:29.431 | INFO | ANALYSIS : 64.2%, 3.0, 99.9ms, 0 | 20260505 07:53:29.431 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:29.431 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:29.431 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 07:53:29.431 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:29.431 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:29.432 | INFO | Uploaded 36GiB to GPU in 2.18991s (16.439GiB/s) | 20260505 07:53:29.432 | INFO | Downloaded 34GiB to GPU in 2.06509s (16.4642GiB/s) | 20260505 07:53:34.570 | INFO | Analyzing chunk frames = [1566208,1566720) | 20260505 07:53:34.570 | INFO | MemInfo: { | 20260505 07:53:34.570 | INFO | "gpu" : " 17 GB", | 20260505 07:53:34.570 | INFO | "huge" : "", | 20260505 07:53:34.570 | INFO | "huge_shared" : " 27 GB", | 20260505 07:53:34.570 | INFO | "malloc" : " 14 GB", | 20260505 07:53:34.570 | INFO | "pinned" : " 29 GB" | 20260505 07:53:34.570 | INFO | } | 20260505 07:53:34.570 | INFO | Perf report for the last 5.13866 seconds: | 20260505 07:53:34.570 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:53:34.570 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:34.570 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:34.570 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:34.570 | INFO | ANALYSIS : 63.1%, 3.0, 98.0ms, 0 | 20260505 07:53:34.570 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:34.570 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:34.570 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:53:34.570 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:34.570 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:34.570 | INFO | Uploaded 36GiB to GPU in 2.15983s (16.668GiB/s) | 20260505 07:53:34.570 | INFO | Downloaded 34GiB to GPU in 2.04564s (16.6207GiB/s) | 20260505 07:53:39.709 | INFO | Analyzing chunk frames = [1566720,1567232) | 20260505 07:53:39.709 | INFO | MemInfo: { | 20260505 07:53:39.709 | INFO | "gpu" : " 17 GB", | 20260505 07:53:39.710 | INFO | "huge" : "", | 20260505 07:53:39.710 | INFO | "huge_shared" : " 27 GB", | 20260505 07:53:39.710 | INFO | "malloc" : " 14 GB", | 20260505 07:53:39.710 | INFO | "pinned" : " 29 GB" | 20260505 07:53:39.710 | INFO | } | 20260505 07:53:39.710 | INFO | Perf report for the last 5.13976 seconds: | 20260505 07:53:39.710 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:53:39.710 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:39.710 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:39.710 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:39.710 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 07:53:39.710 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:39.710 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:39.710 | INFO | PRE_HQ : 29.8%, 1.0, 15.4ms, 0 | 20260505 07:53:39.710 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:39.710 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:39.710 | INFO | Uploaded 36GiB to GPU in 2.19868s (16.3735GiB/s) | 20260505 07:53:39.710 | INFO | Downloaded 34GiB to GPU in 2.05641s (16.5336GiB/s) | 20260505 07:53:44.850 | INFO | Analyzing chunk frames = [1567232,1567744) | 20260505 07:53:44.850 | INFO | MemInfo: { | 20260505 07:53:44.850 | INFO | "gpu" : " 17 GB", | 20260505 07:53:44.850 | INFO | "huge" : "", | 20260505 07:53:44.850 | INFO | "huge_shared" : " 27 GB", | 20260505 07:53:44.850 | INFO | "malloc" : " 14 GB", | 20260505 07:53:44.850 | INFO | "pinned" : " 29 GB" | 20260505 07:53:44.850 | INFO | } | 20260505 07:53:44.850 | INFO | Perf report for the last 5.14047 seconds: | 20260505 07:53:44.850 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:53:44.850 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:44.850 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:44.850 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:44.850 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 07:53:44.850 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:44.850 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:44.850 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 07:53:44.851 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:44.851 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:44.851 | INFO | Uploaded 36GiB to GPU in 2.19502s (16.4008GiB/s) | 20260505 07:53:44.851 | INFO | Downloaded 34GiB to GPU in 2.05003s (16.5851GiB/s) | 20260505 07:53:49.988 | INFO | Analyzing chunk frames = [1567744,1568256) | 20260505 07:53:49.988 | INFO | MemInfo: { | 20260505 07:53:49.988 | INFO | "gpu" : " 17 GB", | 20260505 07:53:49.988 | INFO | "huge" : "", | 20260505 07:53:49.988 | INFO | "huge_shared" : " 27 GB", | 20260505 07:53:49.988 | INFO | "malloc" : " 14 GB", | 20260505 07:53:49.988 | INFO | "pinned" : " 29 GB" | 20260505 07:53:49.988 | INFO | } | 20260505 07:53:49.988 | INFO | Perf report for the last 5.13776 seconds: | 20260505 07:53:49.988 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:53:49.988 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:49.988 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:49.988 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:49.988 | INFO | ANALYSIS : 64.1%, 3.0, 100.0ms, 0 | 20260505 07:53:49.988 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:49.988 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:49.988 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 07:53:49.988 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:49.988 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:49.988 | INFO | Uploaded 36GiB to GPU in 2.19056s (16.4342GiB/s) | 20260505 07:53:49.988 | INFO | Downloaded 34GiB to GPU in 2.14837s (15.8259GiB/s) | 20260505 07:53:55.127 | INFO | Analyzing chunk frames = [1568256,1568768) | 20260505 07:53:55.127 | INFO | MemInfo: { | 20260505 07:53:55.127 | INFO | "gpu" : " 17 GB", | 20260505 07:53:55.127 | INFO | "huge" : "", | 20260505 07:53:55.127 | INFO | "huge_shared" : " 27 GB", | 20260505 07:53:55.127 | INFO | "malloc" : " 14 GB", | 20260505 07:53:55.127 | INFO | "pinned" : " 29 GB" | 20260505 07:53:55.127 | INFO | } | 20260505 07:53:55.127 | INFO | Perf report for the last 5.13935 seconds: | 20260505 07:53:55.128 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:53:55.128 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:55.128 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:55.128 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:55.128 | INFO | ANALYSIS : 64.7%, 3.0, 101.0ms, 0 | 20260505 07:53:55.128 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:55.128 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:55.128 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 07:53:55.128 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:55.128 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:53:55.128 | INFO | Uploaded 36GiB to GPU in 2.21056s (16.2855GiB/s) | 20260505 07:53:55.128 | INFO | Downloaded 34GiB to GPU in 2.04021s (16.6649GiB/s) | 20260505 07:53:55.259 | INFO | Starting a round of Baz IO | 20260505 07:53:58.551 | INFO | Populating RTMetrics report | 20260505 07:53:58.551 | INFO | RTMetrics report populated | 20260505 07:53:58.551 | INFO | Opening tmp RTMetrics json file | 20260505 07:53:58.605 | INFO | Serializing RTMetrics to json | 20260505 07:53:58.606 | INFO | Serialization done, swapping tmp file now... | 20260505 07:53:58.606 | INFO | RTMetrics done | 20260505 07:53:58.706 | INFO | Finished a round of Baz IO: Wrote 2.445 GiB in 3.447 seconds | 20260505 07:53:58.706 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 28.77% / 0.03% / 71.21% | 20260505 07:53:58.706 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.14088% / 3.97364e-05% / 4.37101e-05% | 20260505 07:53:58.707 | INFO | Baz IO overall: | 20260505 07:53:58.707 | INFO | BytesWritten : 870053802716 | 20260505 07:53:58.707 | INFO | NumEvents : 413200064060 | 20260505 07:54:00.267 | INFO | Analyzing chunk frames = [1568768,1569280) | 20260505 07:54:00.267 | INFO | MemInfo: { | 20260505 07:54:00.267 | INFO | "gpu" : " 17 GB", | 20260505 07:54:00.267 | INFO | "huge" : "", | 20260505 07:54:00.267 | INFO | "huge_shared" : " 27 GB", | 20260505 07:54:00.267 | INFO | "malloc" : " 14 GB", | 20260505 07:54:00.267 | INFO | "pinned" : " 29 GB" | 20260505 07:54:00.267 | INFO | } | 20260505 07:54:00.267 | INFO | Perf report for the last 5.13923 seconds: | 20260505 07:54:00.267 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:54:00.267 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:00.267 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:00.267 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:00.267 | INFO | ANALYSIS : 65.5%, 2.9, 99.5ms, 0 | 20260505 07:54:00.267 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:00.267 | INFO | RT_METRICS : 47.0%, 1.0, 24.6ms, 0 | 20260505 07:54:00.267 | INFO | PRE_HQ : 64.1%, 1.9, 62.3ms, 0 | 20260505 07:54:00.267 | INFO | BAZWRITER : 66.4%, 4.2, 142.9ms, 0 | 20260505 07:54:00.267 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260505 07:54:00.267 | INFO | Uploaded 36GiB to GPU in 2.25114s (15.9919GiB/s) | 20260505 07:54:00.267 | INFO | Downloaded 38GiB to GPU in 2.35541s (16.1331GiB/s) | 20260505 07:54:05.406 | INFO | Analyzing chunk frames = [1569280,1569792) | 20260505 07:54:05.406 | INFO | MemInfo: { | 20260505 07:54:05.406 | INFO | "gpu" : " 17 GB", | 20260505 07:54:05.406 | INFO | "huge" : "", | 20260505 07:54:05.406 | INFO | "huge_shared" : " 27 GB", | 20260505 07:54:05.406 | INFO | "malloc" : " 14 GB", | 20260505 07:54:05.406 | INFO | "pinned" : " 29 GB" | 20260505 07:54:05.406 | INFO | } | 20260505 07:54:05.406 | INFO | Perf report for the last 5.1393 seconds: | 20260505 07:54:05.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:54:05.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:05.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:05.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:05.406 | INFO | ANALYSIS : 64.3%, 3.0, 100.3ms, 0 | 20260505 07:54:05.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:05.406 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:05.406 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:54:05.406 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:05.406 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:05.406 | INFO | Uploaded 36GiB to GPU in 2.1793s (16.5191GiB/s) | 20260505 07:54:05.406 | INFO | Downloaded 34GiB to GPU in 2.05597s (16.5372GiB/s) | 20260505 07:54:10.545 | INFO | Analyzing chunk frames = [1569792,1570304) | 20260505 07:54:10.545 | INFO | MemInfo: { | 20260505 07:54:10.545 | INFO | "gpu" : " 17 GB", | 20260505 07:54:10.545 | INFO | "huge" : "", | 20260505 07:54:10.545 | INFO | "huge_shared" : " 27 GB", | 20260505 07:54:10.545 | INFO | "malloc" : " 14 GB", | 20260505 07:54:10.545 | INFO | "pinned" : " 29 GB" | 20260505 07:54:10.545 | INFO | } | 20260505 07:54:10.545 | INFO | Perf report for the last 5.13945 seconds: | 20260505 07:54:10.546 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:54:10.546 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:10.546 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:10.546 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:10.546 | INFO | ANALYSIS : 64.3%, 3.0, 100.0ms, 0 | 20260505 07:54:10.546 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:10.546 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:10.546 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:54:10.546 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:10.546 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:10.546 | INFO | Uploaded 36GiB to GPU in 2.19997s (16.3639GiB/s) | 20260505 07:54:10.546 | INFO | Downloaded 34GiB to GPU in 2.06422s (16.4712GiB/s) | 20260505 07:54:15.685 | INFO | Analyzing chunk frames = [1570304,1570816) | 20260505 07:54:15.685 | INFO | MemInfo: { | 20260505 07:54:15.685 | INFO | "gpu" : " 17 GB", | 20260505 07:54:15.685 | INFO | "huge" : "", | 20260505 07:54:15.685 | INFO | "huge_shared" : " 27 GB", | 20260505 07:54:15.685 | INFO | "malloc" : " 14 GB", | 20260505 07:54:15.685 | INFO | "pinned" : " 29 GB" | 20260505 07:54:15.685 | INFO | } | 20260505 07:54:15.685 | INFO | Perf report for the last 5.13927 seconds: | 20260505 07:54:15.685 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:54:15.685 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:15.685 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:15.685 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:15.685 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 07:54:15.685 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:15.685 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:15.685 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:54:15.685 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:15.685 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:15.685 | INFO | Uploaded 36GiB to GPU in 2.16305s (16.6432GiB/s) | 20260505 07:54:15.685 | INFO | Downloaded 34GiB to GPU in 2.04867s (16.5961GiB/s) | 20260505 07:54:20.824 | INFO | Analyzing chunk frames = [1570816,1571328) | 20260505 07:54:20.824 | INFO | MemInfo: { | 20260505 07:54:20.824 | INFO | "gpu" : " 17 GB", | 20260505 07:54:20.824 | INFO | "huge" : "", | 20260505 07:54:20.824 | INFO | "huge_shared" : " 27 GB", | 20260505 07:54:20.824 | INFO | "malloc" : " 14 GB", | 20260505 07:54:20.824 | INFO | "pinned" : " 29 GB" | 20260505 07:54:20.824 | INFO | } | 20260505 07:54:20.824 | INFO | Perf report for the last 5.13959 seconds: | 20260505 07:54:20.824 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:54:20.824 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:20.824 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:20.824 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:20.824 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 07:54:20.824 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:20.824 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:20.824 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:54:20.825 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:20.825 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:20.825 | INFO | Uploaded 36GiB to GPU in 2.20262s (16.3442GiB/s) | 20260505 07:54:20.825 | INFO | Downloaded 34GiB to GPU in 2.05356s (16.5566GiB/s) | 20260505 07:54:25.965 | INFO | Analyzing chunk frames = [1571328,1571840) | 20260505 07:54:25.965 | INFO | MemInfo: { | 20260505 07:54:25.965 | INFO | "gpu" : " 17 GB", | 20260505 07:54:25.965 | INFO | "huge" : "", | 20260505 07:54:25.965 | INFO | "huge_shared" : " 27 GB", | 20260505 07:54:25.965 | INFO | "malloc" : " 14 GB", | 20260505 07:54:25.965 | INFO | "pinned" : " 29 GB" | 20260505 07:54:25.965 | INFO | } | 20260505 07:54:25.965 | INFO | Perf report for the last 5.1408 seconds: | 20260505 07:54:25.965 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:54:25.965 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:25.965 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:25.965 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:25.965 | INFO | ANALYSIS : 64.1%, 3.0, 100.0ms, 0 | 20260505 07:54:25.965 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:25.965 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:25.965 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 07:54:25.965 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:25.965 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:25.965 | INFO | Uploaded 36GiB to GPU in 2.1979s (16.3793GiB/s) | 20260505 07:54:25.966 | INFO | Downloaded 34GiB to GPU in 2.05151s (16.5732GiB/s) | 20260505 07:54:31.103 | INFO | Analyzing chunk frames = [1571840,1572352) | 20260505 07:54:31.103 | INFO | MemInfo: { | 20260505 07:54:31.103 | INFO | "gpu" : " 17 GB", | 20260505 07:54:31.103 | INFO | "huge" : "", | 20260505 07:54:31.103 | INFO | "huge_shared" : " 27 GB", | 20260505 07:54:31.103 | INFO | "malloc" : " 14 GB", | 20260505 07:54:31.103 | INFO | "pinned" : " 29 GB" | 20260505 07:54:31.103 | INFO | } | 20260505 07:54:31.103 | INFO | Perf report for the last 5.13781 seconds: | 20260505 07:54:31.103 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:54:31.103 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:31.103 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:31.103 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:31.103 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 07:54:31.103 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:31.103 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:31.103 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260505 07:54:31.103 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:31.103 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:31.103 | INFO | Uploaded 36GiB to GPU in 2.17303s (16.5667GiB/s) | 20260505 07:54:31.103 | INFO | Downloaded 34GiB to GPU in 2.14402s (15.8581GiB/s) | 20260505 07:54:36.242 | INFO | Analyzing chunk frames = [1572352,1572864) | 20260505 07:54:36.242 | INFO | MemInfo: { | 20260505 07:54:36.242 | INFO | "gpu" : " 17 GB", | 20260505 07:54:36.242 | INFO | "huge" : "", | 20260505 07:54:36.242 | INFO | "huge_shared" : " 27 GB", | 20260505 07:54:36.242 | INFO | "malloc" : " 14 GB", | 20260505 07:54:36.242 | INFO | "pinned" : " 29 GB" | 20260505 07:54:36.242 | INFO | } | 20260505 07:54:36.242 | INFO | Perf report for the last 5.13946 seconds: | 20260505 07:54:36.243 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:54:36.243 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:36.243 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:36.243 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:36.243 | INFO | ANALYSIS : 64.8%, 3.0, 101.2ms, 0 | 20260505 07:54:36.243 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:36.243 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:36.243 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 07:54:36.243 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:36.243 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:36.243 | INFO | Uploaded 36GiB to GPU in 2.21716s (16.237GiB/s) | 20260505 07:54:36.243 | INFO | Downloaded 34GiB to GPU in 2.0434s (16.639GiB/s) | 20260505 07:54:36.243 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 07:54:36.243 | INFO | Total mem usage: 15.3 GiB | 20260505 07:54:36.243 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 07:54:36.243 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -5.47%) | 20260505 07:54:36.243 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 07:54:36.243 | INFO | High Water Breakdown: | 20260505 07:54:36.243 | INFO | 60.9% : 8.84 GiB ( 94.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 07:54:36.243 | INFO | 39% : 5.67 GiB ( 85.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 07:54:36.243 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 07:54:36.243 | INFO | ------------------------------------------ | 20260505 07:54:36.244 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:54:36.244 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 07:54:36.244 | INFO | Total mem usage: 26.5 GiB | 20260505 07:54:36.244 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 07:54:36.244 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 07:54:36.244 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 07:54:36.244 | INFO | High Water Breakdown: | 20260505 07:54:36.244 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:54:36.244 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:54:36.244 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:54:36.244 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:54:36.244 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:54:36.244 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:54:36.244 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 07:54:36.244 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 07:54:36.244 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:54:36.244 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 07:54:36.244 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 07:54:36.244 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 07:54:36.244 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 07:54:36.244 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 07:54:36.244 | INFO | ------------------------------------------ | 20260505 07:54:36.244 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:54:36.244 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 07:54:36.244 | INFO | Total mem usage: 25.6 GiB | 20260505 07:54:36.244 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 07:54:36.244 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 07:54:36.244 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 07:54:36.244 | INFO | High Water Breakdown: | 20260505 07:54:36.244 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 07:54:36.244 | INFO | ------------------------------------------ | 20260505 07:54:36.244 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:54:36.244 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 07:54:36.244 | INFO | Total mem usage: 15.7 GiB | 20260505 07:54:36.244 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 07:54:36.244 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 07:54:36.244 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260505 07:54:36.244 | INFO | High Water Breakdown: | 20260505 07:54:36.244 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 07:54:36.244 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 07:54:36.244 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 07:54:36.244 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 07:54:36.244 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 07:54:36.244 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 07:54:36.244 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 07:54:36.244 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 07:54:36.244 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 07:54:36.244 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:54:36.244 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 07:54:36.244 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:54:36.244 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 07:54:36.244 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 07:54:36.244 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 07:54:36.244 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 07:54:36.244 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:54:36.244 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 07:54:36.244 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:54:36.244 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:54:36.244 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:54:36.244 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 07:54:36.244 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 07:54:36.244 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:54:36.244 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 07:54:36.244 | INFO | ------------------------------------------ | 20260505 07:54:36.244 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:54:36.244 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 07:54:36.244 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:54:36.244 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:54:36.244 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.57e+01, 1.37e+01 / 1.81e+00 | 20260505 07:54:36.244 | INFO | PerfReport: 0.28%, AggregateStats -- 9.37e-02 / 1.05e-01, 1.07e-02 / 4.18e-03 | 20260505 07:54:36.244 | INFO | PerfReport: 11.58%, Baseline -- 3.81e+00 / 4.48e+00, 6.20e-01 / 2.43e-01 | 20260505 07:54:36.244 | INFO | PerfReport: 3.31%, Histogram -- 1.09e+00 / 1.50e+00, 2.13e-01 / 9.65e-02 | 20260505 07:54:36.244 | INFO | PerfReport: 15.51%, DME -- 4.08e+01 / 4.50e+01, 4.20e+00 / 1.27e+00 | 20260505 07:54:36.244 | INFO | PerfReport: 56.52%, FrameLabeling -- 1.86e+01 / 1.93e+01, 2.09e+00 / 1.43e+00 | 20260505 07:54:36.244 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.45e+00 / 3.12e+00, 4.15e-01 / 2.14e-01 | 20260505 07:54:36.244 | INFO | PerfReport: 3.93%, Metrics -- 1.29e+00 / 1.61e+00, 3.74e-01 / 1.85e-01 | 20260505 07:54:36.244 | INFO | PerfReport: 1.41%, PulseSerializing -- 4.64e-01 / 5.24e-01, 8.85e-02 / 2.72e-02 | 20260505 07:54:36.245 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 07:54:36.245 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:54:36.245 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:54:36.245 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.32e+01 / 0.00e+00 | 20260505 07:54:36.245 | INFO | PerfReport: 22.44%, Upload -- 2.29e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260505 07:54:36.245 | INFO | PerfReport: 21.48%, Download -- 2.19e+01 / 0.00e+00, 8.11e+00 / 0.00e+00 | 20260505 07:54:36.245 | INFO | PerfReport: 32.29%, Compute -- 3.29e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 07:54:36.245 | INFO | PerfReport: 1.72%, LockUpload -- 1.75e+00 / 0.00e+00, 7.32e+00 / 0.00e+00 | 20260505 07:54:36.245 | INFO | PerfReport: 0.84%, LockDownload -- 8.56e-01 / 0.00e+00, 2.24e+00 / 0.00e+00 | 20260505 07:54:36.245 | INFO | PerfReport: 21.22%, LockCompute -- 2.16e+01 / 0.00e+00, 1.76e+01 / 0.00e+00 | 20260505 07:54:36.245 | INFO | PerfReport: 0.02%, Misc -- 1.53e-02 / 0.00e+00, 2.82e-02 / 0.00e+00 | 20260505 07:54:36.371 | INFO | Starting a round of Baz IO | 20260505 07:54:39.656 | INFO | Populating RTMetrics report | 20260505 07:54:39.656 | INFO | RTMetrics report populated | 20260505 07:54:39.656 | INFO | Opening tmp RTMetrics json file | 20260505 07:54:39.672 | INFO | Serializing RTMetrics to json | 20260505 07:54:39.674 | INFO | Serialization done, swapping tmp file now... | 20260505 07:54:39.674 | INFO | RTMetrics done | 20260505 07:54:39.817 | INFO | Finished a round of Baz IO: Wrote 2.445 GiB in 3.445 seconds | 20260505 07:54:39.817 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 28.74% / 0.03% / 71.23% | 20260505 07:54:39.817 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.12483% / 4.76837e-05% / 6.35783e-05% | 20260505 07:54:39.817 | INFO | Baz IO overall: | 20260505 07:54:39.817 | INFO | BytesWritten : 872678581915 | 20260505 07:54:39.818 | INFO | NumEvents : 414455144634 | 20260505 07:54:41.382 | INFO | Analyzing chunk frames = [1572864,1573376) | 20260505 07:54:41.382 | INFO | MemInfo: { | 20260505 07:54:41.382 | INFO | "gpu" : " 17 GB", | 20260505 07:54:41.382 | INFO | "huge" : "", | 20260505 07:54:41.382 | INFO | "huge_shared" : " 27 GB", | 20260505 07:54:41.382 | INFO | "malloc" : " 14 GB", | 20260505 07:54:41.382 | INFO | "pinned" : " 29 GB" | 20260505 07:54:41.382 | INFO | } | 20260505 07:54:41.382 | INFO | Perf report for the last 5.13945 seconds: | 20260505 07:54:41.382 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:54:41.382 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:41.382 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:41.382 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:41.382 | INFO | ANALYSIS : 65.3%, 2.9, 99.2ms, 0 | 20260505 07:54:41.382 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:41.382 | INFO | RT_METRICS : 46.2%, 1.0, 24.2ms, 0 | 20260505 07:54:41.382 | INFO | PRE_HQ : 64.0%, 1.9, 61.7ms, 0 | 20260505 07:54:41.382 | INFO | BAZWRITER : 66.3%, 4.2, 142.8ms, 0 | 20260505 07:54:41.382 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 07:54:41.382 | INFO | Uploaded 36GiB to GPU in 2.2165s (16.2418GiB/s) | 20260505 07:54:41.382 | INFO | Downloaded 38GiB to GPU in 2.33728s (16.2582GiB/s) | 20260505 07:54:46.521 | INFO | Analyzing chunk frames = [1573376,1573888) | 20260505 07:54:46.521 | INFO | MemInfo: { | 20260505 07:54:46.521 | INFO | "gpu" : " 17 GB", | 20260505 07:54:46.521 | INFO | "huge" : "", | 20260505 07:54:46.521 | INFO | "huge_shared" : " 27 GB", | 20260505 07:54:46.521 | INFO | "malloc" : " 14 GB", | 20260505 07:54:46.521 | INFO | "pinned" : " 29 GB" | 20260505 07:54:46.521 | INFO | } | 20260505 07:54:46.521 | INFO | Perf report for the last 5.13936 seconds: | 20260505 07:54:46.521 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:54:46.521 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:46.521 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:46.521 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:46.521 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 07:54:46.521 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:46.521 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:46.521 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:54:46.521 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:46.521 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:46.521 | INFO | Uploaded 36GiB to GPU in 2.17772s (16.5311GiB/s) | 20260505 07:54:46.522 | INFO | Downloaded 34GiB to GPU in 2.05604s (16.5367GiB/s) | 20260505 07:54:51.660 | INFO | Analyzing chunk frames = [1573888,1574400) | 20260505 07:54:51.660 | INFO | MemInfo: { | 20260505 07:54:51.660 | INFO | "gpu" : " 17 GB", | 20260505 07:54:51.660 | INFO | "huge" : "", | 20260505 07:54:51.660 | INFO | "huge_shared" : " 27 GB", | 20260505 07:54:51.660 | INFO | "malloc" : " 14 GB", | 20260505 07:54:51.660 | INFO | "pinned" : " 29 GB" | 20260505 07:54:51.660 | INFO | } | 20260505 07:54:51.660 | INFO | Perf report for the last 5.139 seconds: | 20260505 07:54:51.660 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:54:51.660 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:51.660 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:51.660 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:51.660 | INFO | ANALYSIS : 64.1%, 3.0, 99.8ms, 0 | 20260505 07:54:51.660 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:51.660 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:51.660 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:54:51.660 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:51.660 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:51.661 | INFO | Uploaded 36GiB to GPU in 2.18929s (16.4437GiB/s) | 20260505 07:54:51.661 | INFO | Downloaded 34GiB to GPU in 2.06315s (16.4796GiB/s) | 20260505 07:54:56.799 | INFO | Analyzing chunk frames = [1574400,1574912) | 20260505 07:54:56.800 | INFO | MemInfo: { | 20260505 07:54:56.800 | INFO | "gpu" : " 17 GB", | 20260505 07:54:56.800 | INFO | "huge" : "", | 20260505 07:54:56.800 | INFO | "huge_shared" : " 27 GB", | 20260505 07:54:56.800 | INFO | "malloc" : " 14 GB", | 20260505 07:54:56.800 | INFO | "pinned" : " 29 GB" | 20260505 07:54:56.800 | INFO | } | 20260505 07:54:56.800 | INFO | Perf report for the last 5.1395 seconds: | 20260505 07:54:56.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:54:56.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:56.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:56.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:56.800 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 07:54:56.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:56.800 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:56.800 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:54:56.800 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:56.800 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:54:56.800 | INFO | Uploaded 36GiB to GPU in 2.15863s (16.6773GiB/s) | 20260505 07:54:56.800 | INFO | Downloaded 34GiB to GPU in 2.04475s (16.6279GiB/s) | 20260505 07:55:01.939 | INFO | Analyzing chunk frames = [1574912,1575424) | 20260505 07:55:01.939 | INFO | MemInfo: { | 20260505 07:55:01.939 | INFO | "gpu" : " 17 GB", | 20260505 07:55:01.939 | INFO | "huge" : "", | 20260505 07:55:01.939 | INFO | "huge_shared" : " 27 GB", | 20260505 07:55:01.939 | INFO | "malloc" : " 14 GB", | 20260505 07:55:01.939 | INFO | "pinned" : " 29 GB" | 20260505 07:55:01.939 | INFO | } | 20260505 07:55:01.939 | INFO | Perf report for the last 5.1393 seconds: | 20260505 07:55:01.939 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:55:01.939 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:01.939 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:01.939 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:01.939 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 07:55:01.939 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:01.939 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:01.939 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:55:01.939 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:01.939 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:01.939 | INFO | Uploaded 36GiB to GPU in 2.19339s (16.4129GiB/s) | 20260505 07:55:01.939 | INFO | Downloaded 34GiB to GPU in 2.05268s (16.5637GiB/s) | 20260505 07:55:07.080 | INFO | Analyzing chunk frames = [1575424,1575936) | 20260505 07:55:07.080 | INFO | MemInfo: { | 20260505 07:55:07.080 | INFO | "gpu" : " 17 GB", | 20260505 07:55:07.080 | INFO | "huge" : "", | 20260505 07:55:07.080 | INFO | "huge_shared" : " 27 GB", | 20260505 07:55:07.080 | INFO | "malloc" : " 14 GB", | 20260505 07:55:07.080 | INFO | "pinned" : " 29 GB" | 20260505 07:55:07.080 | INFO | } | 20260505 07:55:07.080 | INFO | Perf report for the last 5.141 seconds: | 20260505 07:55:07.080 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:55:07.080 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:07.080 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:07.080 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:07.080 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 07:55:07.080 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:07.080 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:07.080 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 07:55:07.080 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:07.080 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:07.080 | INFO | Uploaded 36GiB to GPU in 2.1986s (16.3741GiB/s) | 20260505 07:55:07.081 | INFO | Downloaded 34GiB to GPU in 2.04947s (16.5896GiB/s) | 20260505 07:55:12.218 | INFO | Analyzing chunk frames = [1575936,1576448) | 20260505 07:55:12.218 | INFO | MemInfo: { | 20260505 07:55:12.218 | INFO | "gpu" : " 17 GB", | 20260505 07:55:12.218 | INFO | "huge" : "", | 20260505 07:55:12.218 | INFO | "huge_shared" : " 27 GB", | 20260505 07:55:12.218 | INFO | "malloc" : " 14 GB", | 20260505 07:55:12.218 | INFO | "pinned" : " 29 GB" | 20260505 07:55:12.218 | INFO | } | 20260505 07:55:12.218 | INFO | Perf report for the last 5.13788 seconds: | 20260505 07:55:12.218 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:55:12.218 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:12.218 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:12.218 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:12.218 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 07:55:12.218 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:12.218 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:12.218 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 07:55:12.218 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:12.218 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:12.218 | INFO | Uploaded 36GiB to GPU in 2.17199s (16.5747GiB/s) | 20260505 07:55:12.218 | INFO | Downloaded 34GiB to GPU in 2.15884s (15.7492GiB/s) | 20260505 07:55:17.357 | INFO | Analyzing chunk frames = [1576448,1576960) | 20260505 07:55:17.357 | INFO | MemInfo: { | 20260505 07:55:17.357 | INFO | "gpu" : " 17 GB", | 20260505 07:55:17.357 | INFO | "huge" : "", | 20260505 07:55:17.357 | INFO | "huge_shared" : " 27 GB", | 20260505 07:55:17.357 | INFO | "malloc" : " 14 GB", | 20260505 07:55:17.357 | INFO | "pinned" : " 29 GB" | 20260505 07:55:17.357 | INFO | } | 20260505 07:55:17.357 | INFO | Perf report for the last 5.13915 seconds: | 20260505 07:55:17.357 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:55:17.357 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:17.357 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:17.357 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:17.357 | INFO | ANALYSIS : 64.8%, 3.0, 101.1ms, 0 | 20260505 07:55:17.357 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:17.357 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:17.357 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 07:55:17.357 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:17.357 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:17.357 | INFO | Uploaded 36GiB to GPU in 2.21874s (16.2255GiB/s) | 20260505 07:55:17.357 | INFO | Downloaded 34GiB to GPU in 2.04588s (16.6188GiB/s) | 20260505 07:55:17.487 | INFO | Starting a round of Baz IO | 20260505 07:55:20.772 | INFO | Populating RTMetrics report | 20260505 07:55:20.773 | INFO | RTMetrics report populated | 20260505 07:55:20.773 | INFO | Opening tmp RTMetrics json file | 20260505 07:55:20.773 | INFO | Serializing RTMetrics to json | 20260505 07:55:20.780 | INFO | Serialization done, swapping tmp file now... | 20260505 07:55:20.795 | INFO | RTMetrics done | 20260505 07:55:20.940 | INFO | Finished a round of Baz IO: Wrote 2.444 GiB in 3.453 seconds | 20260505 07:55:20.940 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 28.71% / 0.03% / 71.26% | 20260505 07:55:20.940 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.10761% / 4.76837e-05% / 7.54992e-05% | 20260505 07:55:20.940 | INFO | Baz IO overall: | 20260505 07:55:20.940 | INFO | BytesWritten : 875303155624 | 20260505 07:55:20.940 | INFO | NumEvents : 415709559442 | 20260505 07:55:22.496 | INFO | Analyzing chunk frames = [1576960,1577472) | 20260505 07:55:22.496 | INFO | MemInfo: { | 20260505 07:55:22.496 | INFO | "gpu" : " 17 GB", | 20260505 07:55:22.496 | INFO | "huge" : "", | 20260505 07:55:22.496 | INFO | "huge_shared" : " 27 GB", | 20260505 07:55:22.496 | INFO | "malloc" : " 14 GB", | 20260505 07:55:22.496 | INFO | "pinned" : " 29 GB" | 20260505 07:55:22.496 | INFO | } | 20260505 07:55:22.496 | INFO | Perf report for the last 5.13914 seconds: | 20260505 07:55:22.496 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:55:22.496 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:22.496 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:22.496 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:22.496 | INFO | ANALYSIS : 65.3%, 2.9, 99.2ms, 0 | 20260505 07:55:22.496 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:22.496 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260505 07:55:22.496 | INFO | PRE_HQ : 63.9%, 1.9, 62.1ms, 0 | 20260505 07:55:22.496 | INFO | BAZWRITER : 66.5%, 4.2, 141.8ms, 0 | 20260505 07:55:22.496 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 07:55:22.497 | INFO | Uploaded 36GiB to GPU in 2.21739s (16.2353GiB/s) | 20260505 07:55:22.497 | INFO | Downloaded 38GiB to GPU in 2.3338s (16.2825GiB/s) | 20260505 07:55:27.636 | INFO | Analyzing chunk frames = [1577472,1577984) | 20260505 07:55:27.636 | INFO | MemInfo: { | 20260505 07:55:27.636 | INFO | "gpu" : " 17 GB", | 20260505 07:55:27.636 | INFO | "huge" : "", | 20260505 07:55:27.636 | INFO | "huge_shared" : " 27 GB", | 20260505 07:55:27.636 | INFO | "malloc" : " 14 GB", | 20260505 07:55:27.636 | INFO | "pinned" : " 29 GB" | 20260505 07:55:27.636 | INFO | } | 20260505 07:55:27.636 | INFO | Perf report for the last 5.1396 seconds: | 20260505 07:55:27.636 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:55:27.636 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:27.636 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:27.636 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:27.636 | INFO | ANALYSIS : 64.3%, 3.0, 100.5ms, 0 | 20260505 07:55:27.636 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:27.636 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:27.636 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:55:27.636 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:27.636 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:27.636 | INFO | Uploaded 36GiB to GPU in 2.18675s (16.4628GiB/s) | 20260505 07:55:27.636 | INFO | Downloaded 34GiB to GPU in 2.05217s (16.5678GiB/s) | 20260505 07:55:32.797 | INFO | Analyzing chunk frames = [1577984,1578496) | 20260505 07:55:32.797 | INFO | MemInfo: { | 20260505 07:55:32.797 | INFO | "gpu" : " 17 GB", | 20260505 07:55:32.797 | INFO | "huge" : "", | 20260505 07:55:32.797 | INFO | "huge_shared" : " 27 GB", | 20260505 07:55:32.797 | INFO | "malloc" : " 14 GB", | 20260505 07:55:32.797 | INFO | "pinned" : " 29 GB" | 20260505 07:55:32.797 | INFO | } | 20260505 07:55:32.797 | INFO | Perf report for the last 5.16116 seconds: | 20260505 07:55:32.797 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:55:32.797 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:32.797 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:32.797 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:32.797 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 07:55:32.797 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:32.797 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:32.797 | INFO | PRE_HQ : 30.1%, 1.0, 15.7ms, 0 | 20260505 07:55:32.797 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:32.797 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:32.797 | INFO | Uploaded 36GiB to GPU in 2.19026s (16.4364GiB/s) | 20260505 07:55:32.798 | INFO | Downloaded 34GiB to GPU in 2.06283s (16.4822GiB/s) | 20260505 07:55:37.916 | INFO | Analyzing chunk frames = [1578496,1579008) | 20260505 07:55:37.916 | INFO | MemInfo: { | 20260505 07:55:37.916 | INFO | "gpu" : " 17 GB", | 20260505 07:55:37.916 | INFO | "huge" : "", | 20260505 07:55:37.916 | INFO | "huge_shared" : " 27 GB", | 20260505 07:55:37.916 | INFO | "malloc" : " 14 GB", | 20260505 07:55:37.916 | INFO | "pinned" : " 29 GB" | 20260505 07:55:37.916 | INFO | } | 20260505 07:55:37.916 | INFO | Perf report for the last 5.11881 seconds: | 20260505 07:55:37.916 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:55:37.916 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:37.916 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:37.916 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:37.916 | INFO | ANALYSIS : 63.8%, 3.0, 98.8ms, 0 | 20260505 07:55:37.916 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:37.916 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:37.916 | INFO | PRE_HQ : 30.0%, 1.0, 15.5ms, 0 | 20260505 07:55:37.916 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:37.916 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:37.916 | INFO | Uploaded 36GiB to GPU in 2.16264s (16.6463GiB/s) | 20260505 07:55:37.916 | INFO | Downloaded 34GiB to GPU in 2.06119s (16.4953GiB/s) | 20260505 07:55:43.054 | INFO | Analyzing chunk frames = [1579008,1579520) | 20260505 07:55:43.054 | INFO | MemInfo: { | 20260505 07:55:43.054 | INFO | "gpu" : " 17 GB", | 20260505 07:55:43.054 | INFO | "huge" : "", | 20260505 07:55:43.054 | INFO | "huge_shared" : " 27 GB", | 20260505 07:55:43.054 | INFO | "malloc" : " 14 GB", | 20260505 07:55:43.054 | INFO | "pinned" : " 29 GB" | 20260505 07:55:43.054 | INFO | } | 20260505 07:55:43.054 | INFO | Perf report for the last 5.13797 seconds: | 20260505 07:55:43.054 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:55:43.054 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:43.054 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:43.054 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:43.054 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 07:55:43.054 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:43.054 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:43.054 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 07:55:43.054 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:43.054 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:43.054 | INFO | Uploaded 36GiB to GPU in 2.20752s (16.3079GiB/s) | 20260505 07:55:43.054 | INFO | Downloaded 34GiB to GPU in 2.05486s (16.5461GiB/s) | 20260505 07:55:48.195 | INFO | Analyzing chunk frames = [1579520,1580032) | 20260505 07:55:48.195 | INFO | MemInfo: { | 20260505 07:55:48.195 | INFO | "gpu" : " 17 GB", | 20260505 07:55:48.195 | INFO | "huge" : "", | 20260505 07:55:48.195 | INFO | "huge_shared" : " 27 GB", | 20260505 07:55:48.195 | INFO | "malloc" : " 14 GB", | 20260505 07:55:48.195 | INFO | "pinned" : " 29 GB" | 20260505 07:55:48.195 | INFO | } | 20260505 07:55:48.195 | INFO | Perf report for the last 5.14088 seconds: | 20260505 07:55:48.195 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:55:48.195 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:48.195 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:48.195 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:48.195 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 07:55:48.195 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:48.195 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:48.195 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:55:48.195 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:48.195 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:48.195 | INFO | Uploaded 36GiB to GPU in 2.1921s (16.4226GiB/s) | 20260505 07:55:48.195 | INFO | Downloaded 34GiB to GPU in 2.04299s (16.6423GiB/s) | 20260505 07:55:53.332 | INFO | Analyzing chunk frames = [1580032,1580544) | 20260505 07:55:53.332 | INFO | MemInfo: { | 20260505 07:55:53.332 | INFO | "gpu" : " 17 GB", | 20260505 07:55:53.332 | INFO | "huge" : "", | 20260505 07:55:53.332 | INFO | "huge_shared" : " 27 GB", | 20260505 07:55:53.332 | INFO | "malloc" : " 14 GB", | 20260505 07:55:53.332 | INFO | "pinned" : " 29 GB" | 20260505 07:55:53.332 | INFO | } | 20260505 07:55:53.332 | INFO | Perf report for the last 5.13771 seconds: | 20260505 07:55:53.332 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:55:53.332 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:53.332 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:53.332 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:53.333 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 07:55:53.333 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:53.333 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:53.333 | INFO | PRE_HQ : 31.6%, 1.0, 16.5ms, 0 | 20260505 07:55:53.333 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:53.333 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:53.333 | INFO | Uploaded 36GiB to GPU in 2.13989s (16.8233GiB/s) | 20260505 07:55:53.333 | INFO | Downloaded 34GiB to GPU in 2.14292s (15.8662GiB/s) | 20260505 07:55:58.472 | INFO | Analyzing chunk frames = [1580544,1581056) | 20260505 07:55:58.472 | INFO | MemInfo: { | 20260505 07:55:58.472 | INFO | "gpu" : " 17 GB", | 20260505 07:55:58.472 | INFO | "huge" : "", | 20260505 07:55:58.472 | INFO | "huge_shared" : " 27 GB", | 20260505 07:55:58.472 | INFO | "malloc" : " 14 GB", | 20260505 07:55:58.472 | INFO | "pinned" : " 29 GB" | 20260505 07:55:58.472 | INFO | } | 20260505 07:55:58.472 | INFO | Perf report for the last 5.13983 seconds: | 20260505 07:55:58.472 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:55:58.472 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:58.472 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:58.472 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:58.472 | INFO | ANALYSIS : 64.7%, 3.0, 101.1ms, 0 | 20260505 07:55:58.472 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:58.472 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:58.472 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:55:58.473 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:58.473 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:55:58.473 | INFO | Uploaded 36GiB to GPU in 2.21875s (16.2253GiB/s) | 20260505 07:55:58.473 | INFO | Downloaded 34GiB to GPU in 2.04296s (16.6425GiB/s) | 20260505 07:55:58.473 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 07:55:58.473 | INFO | Total mem usage: 15.3 GiB | 20260505 07:55:58.473 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 07:55:58.473 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -5.47%) | 20260505 07:55:58.473 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 07:55:58.473 | INFO | High Water Breakdown: | 20260505 07:55:58.473 | INFO | 60.9% : 8.84 GiB ( 94.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 07:55:58.473 | INFO | 39% : 5.67 GiB ( 85.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 07:55:58.473 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 07:55:58.473 | INFO | ------------------------------------------ | 20260505 07:55:58.473 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:55:58.473 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 07:55:58.473 | INFO | Total mem usage: 26.5 GiB | 20260505 07:55:58.473 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 07:55:58.473 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 07:55:58.473 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 07:55:58.473 | INFO | High Water Breakdown: | 20260505 07:55:58.473 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:55:58.473 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:55:58.473 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:55:58.473 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:55:58.473 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:55:58.473 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:55:58.473 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 07:55:58.473 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 07:55:58.473 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:55:58.473 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 07:55:58.473 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 07:55:58.473 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 07:55:58.473 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 07:55:58.473 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 07:55:58.473 | INFO | ------------------------------------------ | 20260505 07:55:58.473 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:55:58.473 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 07:55:58.473 | INFO | Total mem usage: 25.6 GiB | 20260505 07:55:58.473 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 07:55:58.473 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 07:55:58.473 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 07:55:58.473 | INFO | High Water Breakdown: | 20260505 07:55:58.473 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 07:55:58.473 | INFO | ------------------------------------------ | 20260505 07:55:58.473 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:55:58.474 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 07:55:58.474 | INFO | Total mem usage: 15.7 GiB | 20260505 07:55:58.474 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 07:55:58.474 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 07:55:58.474 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 07:55:58.474 | INFO | High Water Breakdown: | 20260505 07:55:58.474 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 07:55:58.474 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 07:55:58.474 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 07:55:58.474 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 07:55:58.474 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 07:55:58.474 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 07:55:58.474 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 07:55:58.474 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 07:55:58.474 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 07:55:58.474 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:55:58.474 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 07:55:58.474 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:55:58.474 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 07:55:58.474 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 07:55:58.474 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 07:55:58.474 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 07:55:58.474 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:55:58.474 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 07:55:58.474 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:55:58.474 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:55:58.474 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:55:58.474 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 07:55:58.474 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 07:55:58.474 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:55:58.474 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 07:55:58.474 | INFO | ------------------------------------------ | 20260505 07:55:58.474 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:55:58.474 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 07:55:58.474 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:55:58.474 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:55:58.474 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.54e+01, 1.36e+01 / 2.04e+00 | 20260505 07:55:58.474 | INFO | PerfReport: 0.29%, AggregateStats -- 9.37e-02 / 1.04e-01, 1.11e-02 / 4.61e-03 | 20260505 07:55:58.474 | INFO | PerfReport: 11.59%, Baseline -- 3.81e+00 / 4.46e+00, 6.27e-01 / 3.28e-01 | 20260505 07:55:58.474 | INFO | PerfReport: 3.31%, Histogram -- 1.09e+00 / 1.49e+00, 2.14e-01 / 1.15e-01 | 20260505 07:55:58.474 | INFO | PerfReport: 15.53%, DME -- 4.08e+01 / 4.49e+01, 4.14e+00 / 1.35e+00 | 20260505 07:55:58.474 | INFO | PerfReport: 56.46%, FrameLabeling -- 1.86e+01 / 1.92e+01, 2.06e+00 / 1.52e+00 | 20260505 07:55:58.474 | INFO | PerfReport: 7.47%, PulseAccumulating -- 2.45e+00 / 3.09e+00, 4.18e-01 / 2.64e-01 | 20260505 07:55:58.474 | INFO | PerfReport: 3.94%, Metrics -- 1.29e+00 / 1.60e+00, 3.74e-01 / 1.93e-01 | 20260505 07:55:58.474 | INFO | PerfReport: 1.42%, PulseSerializing -- 4.65e-01 / 5.21e-01, 8.96e-02 / 2.81e-02 | 20260505 07:55:58.474 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 07:55:58.474 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:55:58.474 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:55:58.474 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.32e+01 / 0.00e+00 | 20260505 07:55:58.474 | INFO | PerfReport: 22.40%, Upload -- 2.28e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260505 07:55:58.474 | INFO | PerfReport: 21.48%, Download -- 2.19e+01 / 0.00e+00, 8.07e+00 / 0.00e+00 | 20260505 07:55:58.474 | INFO | PerfReport: 32.27%, Compute -- 3.29e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 07:55:58.474 | INFO | PerfReport: 1.76%, LockUpload -- 1.79e+00 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260505 07:55:58.474 | INFO | PerfReport: 0.88%, LockDownload -- 8.94e-01 / 0.00e+00, 2.36e+00 / 0.00e+00 | 20260505 07:55:58.474 | INFO | PerfReport: 21.20%, LockCompute -- 2.16e+01 / 0.00e+00, 1.76e+01 / 0.00e+00 | 20260505 07:55:58.474 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.75e-02 / 0.00e+00 | 20260505 07:55:58.605 | INFO | Starting a round of Baz IO | 20260505 07:56:01.891 | INFO | Populating RTMetrics report | 20260505 07:56:01.891 | INFO | RTMetrics report populated | 20260505 07:56:01.891 | INFO | Opening tmp RTMetrics json file | 20260505 07:56:01.911 | INFO | Serializing RTMetrics to json | 20260505 07:56:01.937 | INFO | Serialization done, swapping tmp file now... | 20260505 07:56:01.951 | INFO | RTMetrics done | 20260505 07:56:02.025 | INFO | Finished a round of Baz IO: Wrote 2.445 GiB in 3.419 seconds | 20260505 07:56:02.025 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 28.69% / 0.03% / 71.29% | 20260505 07:56:02.025 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.10172% / 5.96046e-05% / 9.53674e-05% | 20260505 07:56:02.025 | INFO | Baz IO overall: | 20260505 07:56:02.025 | INFO | BytesWritten : 877928346800 | 20260505 07:56:02.025 | INFO | NumEvents : 416965228158 | 20260505 07:56:03.611 | INFO | Analyzing chunk frames = [1581056,1581568) | 20260505 07:56:03.611 | INFO | MemInfo: { | 20260505 07:56:03.611 | INFO | "gpu" : " 17 GB", | 20260505 07:56:03.611 | INFO | "huge" : "", | 20260505 07:56:03.611 | INFO | "huge_shared" : " 27 GB", | 20260505 07:56:03.611 | INFO | "malloc" : " 14 GB", | 20260505 07:56:03.611 | INFO | "pinned" : " 29 GB" | 20260505 07:56:03.611 | INFO | } | 20260505 07:56:03.611 | INFO | Perf report for the last 5.13908 seconds: | 20260505 07:56:03.611 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:56:03.611 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:03.611 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:03.611 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:03.611 | INFO | ANALYSIS : 65.4%, 2.9, 99.7ms, 0 | 20260505 07:56:03.611 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:03.611 | INFO | RT_METRICS : 47.0%, 1.0, 24.7ms, 0 | 20260505 07:56:03.611 | INFO | PRE_HQ : 63.9%, 1.9, 61.9ms, 0 | 20260505 07:56:03.612 | INFO | BAZWRITER : 65.9%, 4.2, 141.3ms, 0 | 20260505 07:56:03.612 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260505 07:56:03.612 | INFO | Uploaded 36GiB to GPU in 2.21433s (16.2577GiB/s) | 20260505 07:56:03.612 | INFO | Downloaded 38GiB to GPU in 2.3344s (16.2783GiB/s) | 20260505 07:56:08.750 | INFO | Analyzing chunk frames = [1581568,1582080) | 20260505 07:56:08.750 | INFO | MemInfo: { | 20260505 07:56:08.750 | INFO | "gpu" : " 17 GB", | 20260505 07:56:08.750 | INFO | "huge" : "", | 20260505 07:56:08.750 | INFO | "huge_shared" : " 27 GB", | 20260505 07:56:08.750 | INFO | "malloc" : " 14 GB", | 20260505 07:56:08.750 | INFO | "pinned" : " 29 GB" | 20260505 07:56:08.750 | INFO | } | 20260505 07:56:08.751 | INFO | Perf report for the last 5.13915 seconds: | 20260505 07:56:08.751 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:56:08.751 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:08.751 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:08.751 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:08.751 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 07:56:08.751 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:08.751 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:08.751 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 07:56:08.751 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:08.751 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:08.751 | INFO | Uploaded 36GiB to GPU in 2.18093s (16.5067GiB/s) | 20260505 07:56:08.751 | INFO | Downloaded 34GiB to GPU in 2.05003s (16.5851GiB/s) | 20260505 07:56:13.890 | INFO | Analyzing chunk frames = [1582080,1582592) | 20260505 07:56:13.890 | INFO | MemInfo: { | 20260505 07:56:13.890 | INFO | "gpu" : " 17 GB", | 20260505 07:56:13.890 | INFO | "huge" : "", | 20260505 07:56:13.890 | INFO | "huge_shared" : " 27 GB", | 20260505 07:56:13.890 | INFO | "malloc" : " 14 GB", | 20260505 07:56:13.890 | INFO | "pinned" : " 29 GB" | 20260505 07:56:13.890 | INFO | } | 20260505 07:56:13.890 | INFO | Perf report for the last 5.13982 seconds: | 20260505 07:56:13.890 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:56:13.890 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:13.890 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:13.890 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:13.890 | INFO | ANALYSIS : 64.1%, 3.0, 99.8ms, 0 | 20260505 07:56:13.890 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:13.890 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:13.890 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 07:56:13.890 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:13.890 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:13.891 | INFO | Uploaded 36GiB to GPU in 2.19558s (16.3966GiB/s) | 20260505 07:56:13.891 | INFO | Downloaded 34GiB to GPU in 2.06523s (16.463GiB/s) | 20260505 07:56:19.029 | INFO | Analyzing chunk frames = [1582592,1583104) | 20260505 07:56:19.029 | INFO | MemInfo: { | 20260505 07:56:19.029 | INFO | "gpu" : " 17 GB", | 20260505 07:56:19.029 | INFO | "huge" : "", | 20260505 07:56:19.029 | INFO | "huge_shared" : " 27 GB", | 20260505 07:56:19.029 | INFO | "malloc" : " 14 GB", | 20260505 07:56:19.029 | INFO | "pinned" : " 29 GB" | 20260505 07:56:19.029 | INFO | } | 20260505 07:56:19.029 | INFO | Perf report for the last 5.13872 seconds: | 20260505 07:56:19.029 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:56:19.029 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:19.029 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:19.029 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:19.029 | INFO | ANALYSIS : 63.1%, 3.0, 98.0ms, 0 | 20260505 07:56:19.029 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:19.029 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:19.029 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 07:56:19.029 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:19.029 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:19.030 | INFO | Uploaded 36GiB to GPU in 2.16164s (16.6541GiB/s) | 20260505 07:56:19.030 | INFO | Downloaded 34GiB to GPU in 2.04332s (16.6395GiB/s) | 20260505 07:56:24.168 | INFO | Analyzing chunk frames = [1583104,1583616) | 20260505 07:56:24.168 | INFO | MemInfo: { | 20260505 07:56:24.168 | INFO | "gpu" : " 17 GB", | 20260505 07:56:24.168 | INFO | "huge" : "", | 20260505 07:56:24.168 | INFO | "huge_shared" : " 27 GB", | 20260505 07:56:24.168 | INFO | "malloc" : " 14 GB", | 20260505 07:56:24.168 | INFO | "pinned" : " 29 GB" | 20260505 07:56:24.168 | INFO | } | 20260505 07:56:24.168 | INFO | Perf report for the last 5.13937 seconds: | 20260505 07:56:24.168 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:56:24.168 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:24.168 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:24.168 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:24.168 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 07:56:24.168 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:24.168 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:24.168 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 07:56:24.168 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:24.168 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:24.169 | INFO | Uploaded 36GiB to GPU in 2.20719s (16.3103GiB/s) | 20260505 07:56:24.169 | INFO | Downloaded 34GiB to GPU in 2.0562s (16.5353GiB/s) | 20260505 07:56:29.309 | INFO | Analyzing chunk frames = [1583616,1584128) | 20260505 07:56:29.309 | INFO | MemInfo: { | 20260505 07:56:29.309 | INFO | "gpu" : " 17 GB", | 20260505 07:56:29.309 | INFO | "huge" : "", | 20260505 07:56:29.309 | INFO | "huge_shared" : " 27 GB", | 20260505 07:56:29.310 | INFO | "malloc" : " 14 GB", | 20260505 07:56:29.310 | INFO | "pinned" : " 29 GB" | 20260505 07:56:29.310 | INFO | } | 20260505 07:56:29.310 | INFO | Perf report for the last 5.14105 seconds: | 20260505 07:56:29.310 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:56:29.310 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:29.310 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:29.310 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:29.310 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 07:56:29.310 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:29.310 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:29.310 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 07:56:29.310 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:29.310 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:29.310 | INFO | Uploaded 36GiB to GPU in 2.20287s (16.3423GiB/s) | 20260505 07:56:29.310 | INFO | Downloaded 34GiB to GPU in 2.04752s (16.6054GiB/s) | 20260505 07:56:34.447 | INFO | Analyzing chunk frames = [1584128,1584640) | 20260505 07:56:34.447 | INFO | MemInfo: { | 20260505 07:56:34.447 | INFO | "gpu" : " 17 GB", | 20260505 07:56:34.447 | INFO | "huge" : "", | 20260505 07:56:34.447 | INFO | "huge_shared" : " 27 GB", | 20260505 07:56:34.447 | INFO | "malloc" : " 14 GB", | 20260505 07:56:34.447 | INFO | "pinned" : " 29 GB" | 20260505 07:56:34.447 | INFO | } | 20260505 07:56:34.447 | INFO | Perf report for the last 5.13769 seconds: | 20260505 07:56:34.447 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:56:34.447 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:34.447 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:34.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:34.447 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 07:56:34.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:34.447 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:34.447 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 07:56:34.447 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:34.447 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:34.447 | INFO | Uploaded 36GiB to GPU in 2.20477s (16.3282GiB/s) | 20260505 07:56:34.448 | INFO | Downloaded 34GiB to GPU in 2.14899s (15.8214GiB/s) | 20260505 07:56:39.586 | INFO | Analyzing chunk frames = [1584640,1585152) | 20260505 07:56:39.586 | INFO | MemInfo: { | 20260505 07:56:39.586 | INFO | "gpu" : " 17 GB", | 20260505 07:56:39.586 | INFO | "huge" : "", | 20260505 07:56:39.586 | INFO | "huge_shared" : " 27 GB", | 20260505 07:56:39.586 | INFO | "malloc" : " 14 GB", | 20260505 07:56:39.586 | INFO | "pinned" : " 29 GB" | 20260505 07:56:39.586 | INFO | } | 20260505 07:56:39.586 | INFO | Perf report for the last 5.1392 seconds: | 20260505 07:56:39.587 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:56:39.587 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:39.587 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:39.587 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:39.587 | INFO | ANALYSIS : 64.7%, 3.0, 101.0ms, 0 | 20260505 07:56:39.587 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:39.587 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:39.587 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 07:56:39.587 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:39.587 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:39.587 | INFO | Uploaded 36GiB to GPU in 2.22115s (16.2078GiB/s) | 20260505 07:56:39.587 | INFO | Downloaded 34GiB to GPU in 2.04127s (16.6563GiB/s) | 20260505 07:56:39.717 | INFO | Starting a round of Baz IO | 20260505 07:56:43.001 | INFO | Populating RTMetrics report | 20260505 07:56:43.001 | INFO | RTMetrics report populated | 20260505 07:56:43.001 | INFO | Opening tmp RTMetrics json file | 20260505 07:56:43.205 | INFO | Serializing RTMetrics to json | 20260505 07:56:43.308 | INFO | Serialization done, swapping tmp file now... | 20260505 07:56:43.314 | INFO | RTMetrics done | 20260505 07:56:43.317 | INFO | Finished a round of Baz IO: Wrote 2.442 GiB in 3.6 seconds | 20260505 07:56:43.317 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 28.66% / 0.02% / 71.31% | 20260505 07:56:43.317 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.08754% / 3.17891e-05% / 0.000107288% | 20260505 07:56:43.317 | INFO | Baz IO overall: | 20260505 07:56:43.317 | INFO | BytesWritten : 880550919225 | 20260505 07:56:43.317 | INFO | NumEvents : 418219943864 | 20260505 07:56:44.726 | INFO | Analyzing chunk frames = [1585152,1585664) | 20260505 07:56:44.726 | INFO | MemInfo: { | 20260505 07:56:44.726 | INFO | "gpu" : " 17 GB", | 20260505 07:56:44.726 | INFO | "huge" : "", | 20260505 07:56:44.726 | INFO | "huge_shared" : " 27 GB", | 20260505 07:56:44.726 | INFO | "malloc" : " 14 GB", | 20260505 07:56:44.726 | INFO | "pinned" : " 29 GB" | 20260505 07:56:44.726 | INFO | } | 20260505 07:56:44.726 | INFO | Perf report for the last 5.13935 seconds: | 20260505 07:56:44.726 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:56:44.726 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:44.726 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:44.726 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:44.726 | INFO | ANALYSIS : 65.3%, 2.9, 99.2ms, 0 | 20260505 07:56:44.726 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:44.726 | INFO | RT_METRICS : 52.2%, 1.0, 27.4ms, 0 | 20260505 07:56:44.726 | INFO | PRE_HQ : 64.0%, 1.9, 62.3ms, 0 | 20260505 07:56:44.726 | INFO | BAZWRITER : 69.4%, 4.3, 154.1ms, 0 | 20260505 07:56:44.726 | INFO | BAZ_SAMPLING : 3.4%, 1.0, 1.8ms, 0 | 20260505 07:56:44.727 | INFO | Uploaded 36GiB to GPU in 2.19933s (16.3686GiB/s) | 20260505 07:56:44.727 | INFO | Downloaded 38GiB to GPU in 2.32317s (16.3569GiB/s) | 20260505 07:56:49.865 | INFO | Analyzing chunk frames = [1585664,1586176) | 20260505 07:56:49.865 | INFO | MemInfo: { | 20260505 07:56:49.865 | INFO | "gpu" : " 17 GB", | 20260505 07:56:49.865 | INFO | "huge" : "", | 20260505 07:56:49.865 | INFO | "huge_shared" : " 27 GB", | 20260505 07:56:49.865 | INFO | "malloc" : " 14 GB", | 20260505 07:56:49.865 | INFO | "pinned" : " 29 GB" | 20260505 07:56:49.865 | INFO | } | 20260505 07:56:49.865 | INFO | Perf report for the last 5.13923 seconds: | 20260505 07:56:49.865 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:56:49.865 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:49.865 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:49.865 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:49.865 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 07:56:49.865 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:49.865 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:49.865 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 07:56:49.865 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:49.865 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:49.865 | INFO | Uploaded 36GiB to GPU in 2.185s (16.476GiB/s) | 20260505 07:56:49.866 | INFO | Downloaded 34GiB to GPU in 2.05228s (16.5669GiB/s) | 20260505 07:56:55.004 | INFO | Analyzing chunk frames = [1586176,1586688) | 20260505 07:56:55.004 | INFO | MemInfo: { | 20260505 07:56:55.004 | INFO | "gpu" : " 17 GB", | 20260505 07:56:55.004 | INFO | "huge" : "", | 20260505 07:56:55.004 | INFO | "huge_shared" : " 27 GB", | 20260505 07:56:55.004 | INFO | "malloc" : " 14 GB", | 20260505 07:56:55.004 | INFO | "pinned" : " 29 GB" | 20260505 07:56:55.004 | INFO | } | 20260505 07:56:55.004 | INFO | Perf report for the last 5.13931 seconds: | 20260505 07:56:55.005 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:56:55.005 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:55.005 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:55.005 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:55.005 | INFO | ANALYSIS : 64.2%, 3.0, 99.8ms, 0 | 20260505 07:56:55.005 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:55.005 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:55.005 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:56:55.005 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:55.005 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:56:55.005 | INFO | Uploaded 36GiB to GPU in 2.19517s (16.3997GiB/s) | 20260505 07:56:55.005 | INFO | Downloaded 34GiB to GPU in 2.06236s (16.486GiB/s) | 20260505 07:57:00.144 | INFO | Analyzing chunk frames = [1586688,1587200) | 20260505 07:57:00.144 | INFO | MemInfo: { | 20260505 07:57:00.144 | INFO | "gpu" : " 17 GB", | 20260505 07:57:00.144 | INFO | "huge" : "", | 20260505 07:57:00.144 | INFO | "huge_shared" : " 27 GB", | 20260505 07:57:00.144 | INFO | "malloc" : " 14 GB", | 20260505 07:57:00.144 | INFO | "pinned" : " 29 GB" | 20260505 07:57:00.144 | INFO | } | 20260505 07:57:00.144 | INFO | Perf report for the last 5.13963 seconds: | 20260505 07:57:00.144 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:57:00.144 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:00.144 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:00.144 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:00.144 | INFO | ANALYSIS : 63.1%, 3.0, 98.0ms, 0 | 20260505 07:57:00.144 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:00.144 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:00.144 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 07:57:00.144 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:00.144 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:00.144 | INFO | Uploaded 36GiB to GPU in 2.14783s (16.7611GiB/s) | 20260505 07:57:00.144 | INFO | Downloaded 34GiB to GPU in 2.04983s (16.5867GiB/s) | 20260505 07:57:05.297 | INFO | Analyzing chunk frames = [1587200,1587712) | 20260505 07:57:05.297 | INFO | MemInfo: { | 20260505 07:57:05.297 | INFO | "gpu" : " 17 GB", | 20260505 07:57:05.297 | INFO | "huge" : "", | 20260505 07:57:05.297 | INFO | "huge_shared" : " 27 GB", | 20260505 07:57:05.297 | INFO | "malloc" : " 14 GB", | 20260505 07:57:05.297 | INFO | "pinned" : " 29 GB" | 20260505 07:57:05.297 | INFO | } | 20260505 07:57:05.297 | INFO | Perf report for the last 5.15266 seconds: | 20260505 07:57:05.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:57:05.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:05.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:05.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:05.297 | INFO | ANALYSIS : 63.6%, 3.0, 99.5ms, 0 | 20260505 07:57:05.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:05.297 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:05.297 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 07:57:05.297 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:05.297 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:05.298 | INFO | Uploaded 36GiB to GPU in 2.20153s (16.3523GiB/s) | 20260505 07:57:05.298 | INFO | Downloaded 34GiB to GPU in 2.05828s (16.5186GiB/s) | 20260505 07:57:10.436 | INFO | Analyzing chunk frames = [1587712,1588224) | 20260505 07:57:10.436 | INFO | MemInfo: { | 20260505 07:57:10.436 | INFO | "gpu" : " 17 GB", | 20260505 07:57:10.436 | INFO | "huge" : "", | 20260505 07:57:10.436 | INFO | "huge_shared" : " 27 GB", | 20260505 07:57:10.436 | INFO | "malloc" : " 14 GB", | 20260505 07:57:10.436 | INFO | "pinned" : " 29 GB" | 20260505 07:57:10.436 | INFO | } | 20260505 07:57:10.437 | INFO | Perf report for the last 5.13981 seconds: | 20260505 07:57:10.437 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:57:10.437 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:10.437 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:10.437 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:10.437 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 07:57:10.437 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:10.437 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:10.437 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:57:10.437 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:10.437 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:10.437 | INFO | Uploaded 36GiB to GPU in 2.18961s (16.4413GiB/s) | 20260505 07:57:10.437 | INFO | Downloaded 34GiB to GPU in 2.03861s (16.6781GiB/s) | 20260505 07:57:15.576 | INFO | Analyzing chunk frames = [1588224,1588736) | 20260505 07:57:15.576 | INFO | MemInfo: { | 20260505 07:57:15.576 | INFO | "gpu" : " 17 GB", | 20260505 07:57:15.576 | INFO | "huge" : "", | 20260505 07:57:15.576 | INFO | "huge_shared" : " 27 GB", | 20260505 07:57:15.576 | INFO | "malloc" : " 14 GB", | 20260505 07:57:15.576 | INFO | "pinned" : " 29 GB" | 20260505 07:57:15.576 | INFO | } | 20260505 07:57:15.576 | INFO | Perf report for the last 5.13992 seconds: | 20260505 07:57:15.576 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:57:15.576 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:15.576 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:15.576 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:15.576 | INFO | ANALYSIS : 64.4%, 3.0, 100.6ms, 0 | 20260505 07:57:15.576 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:15.576 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:15.577 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 07:57:15.577 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:15.577 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:15.577 | INFO | Uploaded 36GiB to GPU in 2.20411s (16.3331GiB/s) | 20260505 07:57:15.577 | INFO | Downloaded 34GiB to GPU in 2.14042s (15.8847GiB/s) | 20260505 07:57:20.716 | INFO | Analyzing chunk frames = [1588736,1589248) | 20260505 07:57:20.716 | INFO | MemInfo: { | 20260505 07:57:20.716 | INFO | "gpu" : " 17 GB", | 20260505 07:57:20.716 | INFO | "huge" : "", | 20260505 07:57:20.716 | INFO | "huge_shared" : " 27 GB", | 20260505 07:57:20.716 | INFO | "malloc" : " 14 GB", | 20260505 07:57:20.716 | INFO | "pinned" : " 29 GB" | 20260505 07:57:20.716 | INFO | } | 20260505 07:57:20.716 | INFO | Perf report for the last 5.13953 seconds: | 20260505 07:57:20.716 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:57:20.716 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:20.716 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:20.716 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:20.716 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 07:57:20.716 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:20.716 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:20.716 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:57:20.716 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:20.716 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:20.716 | INFO | Uploaded 36GiB to GPU in 2.20234s (16.3462GiB/s) | 20260505 07:57:20.716 | INFO | Downloaded 34GiB to GPU in 2.04277s (16.6441GiB/s) | 20260505 07:57:20.716 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 07:57:20.716 | INFO | Total mem usage: 15.3 GiB | 20260505 07:57:20.716 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 07:57:20.716 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -5.97%) | 20260505 07:57:20.716 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 07:57:20.716 | INFO | High Water Breakdown: | 20260505 07:57:20.716 | INFO | 60.7% : 8.78 GiB ( 93.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 07:57:20.716 | INFO | 39.2% : 5.67 GiB ( 85.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 07:57:20.716 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 07:57:20.716 | INFO | ------------------------------------------ | 20260505 07:57:20.716 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:57:20.716 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 07:57:20.716 | INFO | Total mem usage: 26.5 GiB | 20260505 07:57:20.716 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 07:57:20.716 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 07:57:20.716 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 07:57:20.717 | INFO | High Water Breakdown: | 20260505 07:57:20.717 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:57:20.717 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:57:20.717 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:57:20.717 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:57:20.717 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:57:20.717 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:57:20.717 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 07:57:20.717 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 07:57:20.717 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:57:20.717 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 07:57:20.717 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 07:57:20.717 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 07:57:20.717 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 07:57:20.717 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 07:57:20.717 | INFO | ------------------------------------------ | 20260505 07:57:20.717 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:57:20.717 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 07:57:20.717 | INFO | Total mem usage: 25.6 GiB | 20260505 07:57:20.717 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 07:57:20.717 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 07:57:20.717 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 07:57:20.717 | INFO | High Water Breakdown: | 20260505 07:57:20.717 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 07:57:20.717 | INFO | ------------------------------------------ | 20260505 07:57:20.717 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:57:20.717 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 07:57:20.717 | INFO | Total mem usage: 15.7 GiB | 20260505 07:57:20.717 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 07:57:20.717 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 07:57:20.717 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 07:57:20.717 | INFO | High Water Breakdown: | 20260505 07:57:20.717 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 07:57:20.717 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 07:57:20.717 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 07:57:20.717 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 07:57:20.717 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 07:57:20.717 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 07:57:20.717 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 07:57:20.717 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 07:57:20.717 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 07:57:20.717 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:57:20.717 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 07:57:20.717 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:57:20.717 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 07:57:20.717 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 07:57:20.717 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 07:57:20.717 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 07:57:20.717 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:57:20.717 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 07:57:20.717 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:57:20.717 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:57:20.717 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:57:20.717 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 07:57:20.717 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 07:57:20.717 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:57:20.717 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 07:57:20.717 | INFO | ------------------------------------------ | 20260505 07:57:20.717 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:57:20.717 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 07:57:20.717 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:57:20.717 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:57:20.717 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.56e+01, 1.33e+01 / 1.83e+00 | 20260505 07:57:20.717 | INFO | PerfReport: 0.28%, AggregateStats -- 9.34e-02 / 1.05e-01, 1.15e-02 / 5.12e-03 | 20260505 07:57:20.717 | INFO | PerfReport: 11.58%, Baseline -- 3.80e+00 / 4.45e+00, 6.20e-01 / 2.90e-01 | 20260505 07:57:20.717 | INFO | PerfReport: 3.30%, Histogram -- 1.08e+00 / 1.48e+00, 2.10e-01 / 1.09e-01 | 20260505 07:57:20.717 | INFO | PerfReport: 15.52%, DME -- 4.07e+01 / 4.50e+01, 4.31e+00 / 1.28e+00 | 20260505 07:57:20.717 | INFO | PerfReport: 56.55%, FrameLabeling -- 1.86e+01 / 1.93e+01, 2.06e+00 / 1.47e+00 | 20260505 07:57:20.717 | INFO | PerfReport: 7.44%, PulseAccumulating -- 2.44e+00 / 3.11e+00, 4.06e-01 / 2.31e-01 | 20260505 07:57:20.717 | INFO | PerfReport: 3.92%, Metrics -- 1.29e+00 / 1.61e+00, 3.72e-01 / 1.91e-01 | 20260505 07:57:20.717 | INFO | PerfReport: 1.41%, PulseSerializing -- 4.62e-01 / 5.23e-01, 8.89e-02 / 3.28e-02 | 20260505 07:57:20.717 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 07:57:20.717 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:57:20.717 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:57:20.717 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.29e+01 / 0.00e+00 | 20260505 07:57:20.717 | INFO | PerfReport: 22.45%, Upload -- 2.29e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260505 07:57:20.717 | INFO | PerfReport: 21.46%, Download -- 2.18e+01 / 0.00e+00, 8.07e+00 / 0.00e+00 | 20260505 07:57:20.717 | INFO | PerfReport: 32.25%, Compute -- 3.28e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260505 07:57:20.717 | INFO | PerfReport: 1.73%, LockUpload -- 1.76e+00 / 0.00e+00, 7.35e+00 / 0.00e+00 | 20260505 07:57:20.717 | INFO | PerfReport: 0.87%, LockDownload -- 8.89e-01 / 0.00e+00, 2.30e+00 / 0.00e+00 | 20260505 07:57:20.717 | INFO | PerfReport: 21.22%, LockCompute -- 2.16e+01 / 0.00e+00, 1.76e+01 / 0.00e+00 | 20260505 07:57:20.717 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.61e-02 / 0.00e+00 | 20260505 07:57:20.849 | INFO | Starting a round of Baz IO | 20260505 07:57:24.151 | INFO | Populating RTMetrics report | 20260505 07:57:24.151 | INFO | RTMetrics report populated | 20260505 07:57:24.151 | INFO | Opening tmp RTMetrics json file | 20260505 07:57:24.172 | INFO | Serializing RTMetrics to json | 20260505 07:57:24.210 | INFO | Serialization done, swapping tmp file now... | 20260505 07:57:24.225 | INFO | RTMetrics done | 20260505 07:57:24.290 | INFO | Finished a round of Baz IO: Wrote 2.442 GiB in 3.44 seconds | 20260505 07:57:24.290 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 28.64% / 0.03% / 71.34% | 20260505 07:57:24.290 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.07504% / 1.98682e-05% / 7.94729e-05% | 20260505 07:57:24.290 | INFO | Baz IO overall: | 20260505 07:57:24.290 | INFO | BytesWritten : 883172492020 | 20260505 07:57:24.290 | INFO | NumEvents : 419474511629 | 20260505 07:57:25.854 | INFO | Analyzing chunk frames = [1589248,1589760) | 20260505 07:57:25.854 | INFO | MemInfo: { | 20260505 07:57:25.854 | INFO | "gpu" : " 17 GB", | 20260505 07:57:25.854 | INFO | "huge" : "", | 20260505 07:57:25.854 | INFO | "huge_shared" : " 27 GB", | 20260505 07:57:25.854 | INFO | "malloc" : " 14 GB", | 20260505 07:57:25.854 | INFO | "pinned" : " 29 GB" | 20260505 07:57:25.854 | INFO | } | 20260505 07:57:25.854 | INFO | Perf report for the last 5.13822 seconds: | 20260505 07:57:25.854 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:57:25.854 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:25.854 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:25.854 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:25.854 | INFO | ANALYSIS : 65.7%, 2.9, 100.2ms, 0 | 20260505 07:57:25.854 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:25.854 | INFO | RT_METRICS : 47.5%, 1.0, 24.9ms, 0 | 20260505 07:57:25.854 | INFO | PRE_HQ : 63.8%, 1.9, 61.7ms, 0 | 20260505 07:57:25.854 | INFO | BAZWRITER : 66.2%, 4.1, 140.2ms, 0 | 20260505 07:57:25.854 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 07:57:25.855 | INFO | Uploaded 36GiB to GPU in 2.22654s (16.1686GiB/s) | 20260505 07:57:25.855 | INFO | Downloaded 38GiB to GPU in 2.33234s (16.2926GiB/s) | 20260505 07:57:30.994 | INFO | Analyzing chunk frames = [1589760,1590272) | 20260505 07:57:30.994 | INFO | MemInfo: { | 20260505 07:57:30.994 | INFO | "gpu" : " 17 GB", | 20260505 07:57:30.994 | INFO | "huge" : "", | 20260505 07:57:30.994 | INFO | "huge_shared" : " 27 GB", | 20260505 07:57:30.994 | INFO | "malloc" : " 14 GB", | 20260505 07:57:30.994 | INFO | "pinned" : " 29 GB" | 20260505 07:57:30.994 | INFO | } | 20260505 07:57:30.994 | INFO | Perf report for the last 5.13993 seconds: | 20260505 07:57:30.994 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:57:30.994 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:30.994 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:30.994 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:30.994 | INFO | ANALYSIS : 64.1%, 3.0, 100.0ms, 0 | 20260505 07:57:30.994 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:30.994 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:30.994 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:57:30.994 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:30.994 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:30.994 | INFO | Uploaded 36GiB to GPU in 2.18043s (16.5105GiB/s) | 20260505 07:57:30.994 | INFO | Downloaded 34GiB to GPU in 2.04912s (16.5925GiB/s) | 20260505 07:57:36.135 | INFO | Analyzing chunk frames = [1590272,1590784) | 20260505 07:57:36.135 | INFO | MemInfo: { | 20260505 07:57:36.135 | INFO | "gpu" : " 17 GB", | 20260505 07:57:36.135 | INFO | "huge" : "", | 20260505 07:57:36.135 | INFO | "huge_shared" : " 27 GB", | 20260505 07:57:36.135 | INFO | "malloc" : " 14 GB", | 20260505 07:57:36.135 | INFO | "pinned" : " 29 GB" | 20260505 07:57:36.135 | INFO | } | 20260505 07:57:36.135 | INFO | Perf report for the last 5.14049 seconds: | 20260505 07:57:36.135 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:57:36.135 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:36.135 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:36.135 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:36.135 | INFO | ANALYSIS : 64.0%, 3.0, 99.6ms, 0 | 20260505 07:57:36.135 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:36.135 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:36.135 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 07:57:36.135 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:36.135 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:36.135 | INFO | Uploaded 36GiB to GPU in 2.18392s (16.4841GiB/s) | 20260505 07:57:36.135 | INFO | Downloaded 34GiB to GPU in 2.06166s (16.4916GiB/s) | 20260505 07:57:41.273 | INFO | Analyzing chunk frames = [1590784,1591296) | 20260505 07:57:41.273 | INFO | MemInfo: { | 20260505 07:57:41.273 | INFO | "gpu" : " 17 GB", | 20260505 07:57:41.273 | INFO | "huge" : "", | 20260505 07:57:41.273 | INFO | "huge_shared" : " 27 GB", | 20260505 07:57:41.273 | INFO | "malloc" : " 14 GB", | 20260505 07:57:41.273 | INFO | "pinned" : " 29 GB" | 20260505 07:57:41.273 | INFO | } | 20260505 07:57:41.273 | INFO | Perf report for the last 5.13845 seconds: | 20260505 07:57:41.273 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:57:41.273 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:41.273 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:41.273 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:41.273 | INFO | ANALYSIS : 63.9%, 3.0, 99.3ms, 0 | 20260505 07:57:41.273 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:41.273 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:41.273 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 07:57:41.273 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:41.273 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:41.273 | INFO | Uploaded 36GiB to GPU in 2.20286s (16.3424GiB/s) | 20260505 07:57:41.273 | INFO | Downloaded 34GiB to GPU in 2.0626s (16.4841GiB/s) | 20260505 07:57:46.413 | INFO | Analyzing chunk frames = [1591296,1591808) | 20260505 07:57:46.413 | INFO | MemInfo: { | 20260505 07:57:46.413 | INFO | "gpu" : " 17 GB", | 20260505 07:57:46.413 | INFO | "huge" : "", | 20260505 07:57:46.413 | INFO | "huge_shared" : " 27 GB", | 20260505 07:57:46.413 | INFO | "malloc" : " 14 GB", | 20260505 07:57:46.413 | INFO | "pinned" : " 29 GB" | 20260505 07:57:46.413 | INFO | } | 20260505 07:57:46.413 | INFO | Perf report for the last 5.13963 seconds: | 20260505 07:57:46.413 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:57:46.413 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:46.413 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:46.413 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:46.413 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 07:57:46.413 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:46.413 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:46.413 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 07:57:46.413 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:46.413 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:46.413 | INFO | Uploaded 36GiB to GPU in 2.20148s (16.3526GiB/s) | 20260505 07:57:46.413 | INFO | Downloaded 34GiB to GPU in 2.0486s (16.5967GiB/s) | 20260505 07:57:51.552 | INFO | Analyzing chunk frames = [1591808,1592320) | 20260505 07:57:51.552 | INFO | MemInfo: { | 20260505 07:57:51.552 | INFO | "gpu" : " 17 GB", | 20260505 07:57:51.552 | INFO | "huge" : "", | 20260505 07:57:51.552 | INFO | "huge_shared" : " 27 GB", | 20260505 07:57:51.552 | INFO | "malloc" : " 14 GB", | 20260505 07:57:51.552 | INFO | "pinned" : " 29 GB" | 20260505 07:57:51.552 | INFO | } | 20260505 07:57:51.552 | INFO | Perf report for the last 5.13969 seconds: | 20260505 07:57:51.552 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:57:51.552 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:51.552 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:51.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:51.553 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 07:57:51.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:51.553 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:51.553 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260505 07:57:51.553 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:51.553 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:51.553 | INFO | Uploaded 36GiB to GPU in 2.19135s (16.4282GiB/s) | 20260505 07:57:51.553 | INFO | Downloaded 34GiB to GPU in 2.04025s (16.6647GiB/s) | 20260505 07:57:56.690 | INFO | Analyzing chunk frames = [1592320,1592832) | 20260505 07:57:56.690 | INFO | MemInfo: { | 20260505 07:57:56.690 | INFO | "gpu" : " 17 GB", | 20260505 07:57:56.690 | INFO | "huge" : "", | 20260505 07:57:56.690 | INFO | "huge_shared" : " 27 GB", | 20260505 07:57:56.690 | INFO | "malloc" : " 14 GB", | 20260505 07:57:56.690 | INFO | "pinned" : " 29 GB" | 20260505 07:57:56.690 | INFO | } | 20260505 07:57:56.690 | INFO | Perf report for the last 5.13793 seconds: | 20260505 07:57:56.690 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:57:56.690 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:56.690 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:56.690 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:56.690 | INFO | ANALYSIS : 64.4%, 3.0, 100.6ms, 0 | 20260505 07:57:56.690 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:56.691 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:56.691 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 07:57:56.691 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:56.691 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:57:56.691 | INFO | Uploaded 36GiB to GPU in 2.20974s (16.2915GiB/s) | 20260505 07:57:56.691 | INFO | Downloaded 34GiB to GPU in 2.14546s (15.8474GiB/s) | 20260505 07:58:01.830 | INFO | Analyzing chunk frames = [1592832,1593344) | 20260505 07:58:01.830 | INFO | MemInfo: { | 20260505 07:58:01.830 | INFO | "gpu" : " 17 GB", | 20260505 07:58:01.830 | INFO | "huge" : "", | 20260505 07:58:01.830 | INFO | "huge_shared" : " 27 GB", | 20260505 07:58:01.830 | INFO | "malloc" : " 14 GB", | 20260505 07:58:01.830 | INFO | "pinned" : " 29 GB" | 20260505 07:58:01.830 | INFO | } | 20260505 07:58:01.830 | INFO | Perf report for the last 5.13935 seconds: | 20260505 07:58:01.830 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:58:01.830 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:01.830 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:01.830 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:01.830 | INFO | ANALYSIS : 64.4%, 3.0, 100.6ms, 0 | 20260505 07:58:01.830 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:01.830 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:01.830 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:58:01.830 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:01.830 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:01.830 | INFO | Uploaded 36GiB to GPU in 2.20451s (16.3302GiB/s) | 20260505 07:58:01.830 | INFO | Downloaded 34GiB to GPU in 2.04635s (16.615GiB/s) | 20260505 07:58:01.963 | INFO | Starting a round of Baz IO | 20260505 07:58:05.263 | INFO | Populating RTMetrics report | 20260505 07:58:05.264 | INFO | RTMetrics report populated | 20260505 07:58:05.264 | INFO | Opening tmp RTMetrics json file | 20260505 07:58:05.264 | INFO | Serializing RTMetrics to json | 20260505 07:58:05.265 | INFO | Serialization done, swapping tmp file now... | 20260505 07:58:05.266 | INFO | RTMetrics done | 20260505 07:58:05.500 | INFO | Finished a round of Baz IO: Wrote 2.442 GiB in 3.536 seconds | 20260505 07:58:05.500 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 28.61% / 0.03% / 71.36% | 20260505 07:58:05.500 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.06464% / 2.78155e-05% / 8.74201e-05% | 20260505 07:58:05.500 | INFO | Baz IO overall: | 20260505 07:58:05.500 | INFO | BytesWritten : 885794109941 | 20260505 07:58:05.500 | INFO | NumEvents : 420728856462 | 20260505 07:58:06.969 | INFO | Analyzing chunk frames = [1593344,1593856) | 20260505 07:58:06.969 | INFO | MemInfo: { | 20260505 07:58:06.969 | INFO | "gpu" : " 17 GB", | 20260505 07:58:06.969 | INFO | "huge" : "", | 20260505 07:58:06.969 | INFO | "huge_shared" : " 27 GB", | 20260505 07:58:06.969 | INFO | "malloc" : " 14 GB", | 20260505 07:58:06.969 | INFO | "pinned" : " 29 GB" | 20260505 07:58:06.969 | INFO | } | 20260505 07:58:06.969 | INFO | Perf report for the last 5.13957 seconds: | 20260505 07:58:06.969 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:58:06.969 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:06.969 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:06.969 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:06.969 | INFO | ANALYSIS : 65.6%, 2.9, 99.9ms, 0 | 20260505 07:58:06.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:06.969 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 07:58:06.969 | INFO | PRE_HQ : 63.8%, 1.9, 61.6ms, 0 | 20260505 07:58:06.969 | INFO | BAZWRITER : 68.1%, 4.2, 147.4ms, 0 | 20260505 07:58:06.969 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 07:58:06.970 | INFO | Uploaded 36GiB to GPU in 2.28085s (15.7836GiB/s) | 20260505 07:58:06.970 | INFO | Downloaded 38GiB to GPU in 2.36084s (16.0959GiB/s) | 20260505 07:58:12.109 | INFO | Analyzing chunk frames = [1593856,1594368) | 20260505 07:58:12.109 | INFO | MemInfo: { | 20260505 07:58:12.109 | INFO | "gpu" : " 17 GB", | 20260505 07:58:12.109 | INFO | "huge" : "", | 20260505 07:58:12.109 | INFO | "huge_shared" : " 27 GB", | 20260505 07:58:12.109 | INFO | "malloc" : " 14 GB", | 20260505 07:58:12.109 | INFO | "pinned" : " 29 GB" | 20260505 07:58:12.109 | INFO | } | 20260505 07:58:12.109 | INFO | Perf report for the last 5.13992 seconds: | 20260505 07:58:12.109 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:58:12.109 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:12.109 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:12.109 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:12.109 | INFO | ANALYSIS : 64.1%, 3.0, 100.2ms, 0 | 20260505 07:58:12.109 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:12.109 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:12.109 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 07:58:12.109 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:12.109 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:12.110 | INFO | Uploaded 36GiB to GPU in 2.18822s (16.4517GiB/s) | 20260505 07:58:12.110 | INFO | Downloaded 34GiB to GPU in 2.04016s (16.6654GiB/s) | 20260505 07:58:17.248 | INFO | Analyzing chunk frames = [1594368,1594880) | 20260505 07:58:17.248 | INFO | MemInfo: { | 20260505 07:58:17.248 | INFO | "gpu" : " 17 GB", | 20260505 07:58:17.248 | INFO | "huge" : "", | 20260505 07:58:17.248 | INFO | "huge_shared" : " 27 GB", | 20260505 07:58:17.248 | INFO | "malloc" : " 14 GB", | 20260505 07:58:17.248 | INFO | "pinned" : " 29 GB" | 20260505 07:58:17.248 | INFO | } | 20260505 07:58:17.248 | INFO | Perf report for the last 5.13905 seconds: | 20260505 07:58:17.248 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:58:17.248 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:17.248 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:17.248 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:17.248 | INFO | ANALYSIS : 63.9%, 3.0, 99.4ms, 0 | 20260505 07:58:17.248 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:17.248 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:17.248 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:58:17.248 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:17.248 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:17.249 | INFO | Uploaded 36GiB to GPU in 2.1829s (16.4918GiB/s) | 20260505 07:58:17.249 | INFO | Downloaded 34GiB to GPU in 2.06533s (16.4622GiB/s) | 20260505 07:58:22.388 | INFO | Analyzing chunk frames = [1594880,1595392) | 20260505 07:58:22.388 | INFO | MemInfo: { | 20260505 07:58:22.388 | INFO | "gpu" : " 17 GB", | 20260505 07:58:22.388 | INFO | "huge" : "", | 20260505 07:58:22.388 | INFO | "huge_shared" : " 27 GB", | 20260505 07:58:22.388 | INFO | "malloc" : " 14 GB", | 20260505 07:58:22.388 | INFO | "pinned" : " 29 GB" | 20260505 07:58:22.388 | INFO | } | 20260505 07:58:22.388 | INFO | Perf report for the last 5.1401 seconds: | 20260505 07:58:22.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:58:22.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:22.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:22.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:22.388 | INFO | ANALYSIS : 63.8%, 3.0, 99.2ms, 0 | 20260505 07:58:22.388 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:22.388 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:22.388 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 07:58:22.388 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:22.389 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:22.389 | INFO | Uploaded 36GiB to GPU in 2.19146s (16.4274GiB/s) | 20260505 07:58:22.389 | INFO | Downloaded 34GiB to GPU in 2.06403s (16.4726GiB/s) | 20260505 07:58:27.528 | INFO | Analyzing chunk frames = [1595392,1595904) | 20260505 07:58:27.528 | INFO | MemInfo: { | 20260505 07:58:27.528 | INFO | "gpu" : " 17 GB", | 20260505 07:58:27.528 | INFO | "huge" : "", | 20260505 07:58:27.528 | INFO | "huge_shared" : " 27 GB", | 20260505 07:58:27.528 | INFO | "malloc" : " 14 GB", | 20260505 07:58:27.528 | INFO | "pinned" : " 29 GB" | 20260505 07:58:27.528 | INFO | } | 20260505 07:58:27.528 | INFO | Perf report for the last 5.13965 seconds: | 20260505 07:58:27.528 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:58:27.528 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:27.528 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:27.528 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:27.528 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 07:58:27.528 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:27.528 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:27.528 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 07:58:27.528 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:27.528 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:27.528 | INFO | Uploaded 36GiB to GPU in 2.17859s (16.5245GiB/s) | 20260505 07:58:27.528 | INFO | Downloaded 34GiB to GPU in 2.05648s (16.5331GiB/s) | 20260505 07:58:32.668 | INFO | Analyzing chunk frames = [1595904,1596416) | 20260505 07:58:32.668 | INFO | MemInfo: { | 20260505 07:58:32.668 | INFO | "gpu" : " 17 GB", | 20260505 07:58:32.668 | INFO | "huge" : "", | 20260505 07:58:32.668 | INFO | "huge_shared" : " 27 GB", | 20260505 07:58:32.668 | INFO | "malloc" : " 14 GB", | 20260505 07:58:32.668 | INFO | "pinned" : " 29 GB" | 20260505 07:58:32.668 | INFO | } | 20260505 07:58:32.668 | INFO | Perf report for the last 5.13988 seconds: | 20260505 07:58:32.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:58:32.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:32.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:32.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:32.668 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 07:58:32.668 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:32.668 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:32.668 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 07:58:32.668 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:32.668 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:32.668 | INFO | Uploaded 36GiB to GPU in 2.19019s (16.4369GiB/s) | 20260505 07:58:32.668 | INFO | Downloaded 34GiB to GPU in 2.05426s (16.551GiB/s) | 20260505 07:58:37.805 | INFO | Analyzing chunk frames = [1596416,1596928) | 20260505 07:58:37.805 | INFO | MemInfo: { | 20260505 07:58:37.805 | INFO | "gpu" : " 17 GB", | 20260505 07:58:37.805 | INFO | "huge" : "", | 20260505 07:58:37.805 | INFO | "huge_shared" : " 27 GB", | 20260505 07:58:37.805 | INFO | "malloc" : " 14 GB", | 20260505 07:58:37.805 | INFO | "pinned" : " 29 GB" | 20260505 07:58:37.805 | INFO | } | 20260505 07:58:37.805 | INFO | Perf report for the last 5.13732 seconds: | 20260505 07:58:37.805 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:58:37.805 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:37.805 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:37.805 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:37.805 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 07:58:37.805 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:37.805 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:37.805 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 07:58:37.805 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:37.805 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:37.806 | INFO | Uploaded 36GiB to GPU in 2.22546s (16.1764GiB/s) | 20260505 07:58:37.806 | INFO | Downloaded 34GiB to GPU in 2.15872s (15.7501GiB/s) | 20260505 07:58:42.944 | INFO | Analyzing chunk frames = [1596928,1597440) | 20260505 07:58:42.944 | INFO | MemInfo: { | 20260505 07:58:42.944 | INFO | "gpu" : " 17 GB", | 20260505 07:58:42.944 | INFO | "huge" : "", | 20260505 07:58:42.944 | INFO | "huge_shared" : " 27 GB", | 20260505 07:58:42.944 | INFO | "malloc" : " 14 GB", | 20260505 07:58:42.944 | INFO | "pinned" : " 29 GB" | 20260505 07:58:42.944 | INFO | } | 20260505 07:58:42.944 | INFO | Perf report for the last 5.13911 seconds: | 20260505 07:58:42.944 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:58:42.944 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:42.944 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:42.944 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:42.945 | INFO | ANALYSIS : 64.6%, 3.0, 100.9ms, 0 | 20260505 07:58:42.945 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:42.945 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:42.945 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 07:58:42.945 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:42.945 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:42.945 | INFO | Uploaded 36GiB to GPU in 2.21401s (16.2601GiB/s) | 20260505 07:58:42.945 | INFO | Downloaded 34GiB to GPU in 2.0507s (16.5797GiB/s) | 20260505 07:58:42.945 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 07:58:42.945 | INFO | Total mem usage: 15.3 GiB | 20260505 07:58:42.945 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 07:58:42.945 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -5.04%) | 20260505 07:58:42.945 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 07:58:42.945 | INFO | High Water Breakdown: | 20260505 07:58:42.945 | INFO | 60.8% : 8.87 GiB ( 94.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 07:58:42.945 | INFO | 39.1% : 5.7 GiB ( 85.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 07:58:42.945 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 07:58:42.945 | INFO | ------------------------------------------ | 20260505 07:58:42.945 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:58:42.945 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 07:58:42.945 | INFO | Total mem usage: 26.5 GiB | 20260505 07:58:42.945 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 07:58:42.945 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 07:58:42.945 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 07:58:42.945 | INFO | High Water Breakdown: | 20260505 07:58:42.945 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:58:42.945 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:58:42.945 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:58:42.945 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:58:42.945 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:58:42.945 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:58:42.945 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 07:58:42.945 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 07:58:42.945 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:58:42.945 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 07:58:42.945 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 07:58:42.945 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 07:58:42.945 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 07:58:42.945 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 07:58:42.945 | INFO | ------------------------------------------ | 20260505 07:58:42.945 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:58:42.945 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 07:58:42.945 | INFO | Total mem usage: 25.6 GiB | 20260505 07:58:42.945 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 07:58:42.945 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 07:58:42.945 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 07:58:42.945 | INFO | High Water Breakdown: | 20260505 07:58:42.945 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 07:58:42.945 | INFO | ------------------------------------------ | 20260505 07:58:42.945 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:58:42.945 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 07:58:42.945 | INFO | Total mem usage: 15.7 GiB | 20260505 07:58:42.945 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 07:58:42.945 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 07:58:42.945 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 07:58:42.945 | INFO | High Water Breakdown: | 20260505 07:58:42.945 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 07:58:42.946 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 07:58:42.946 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 07:58:42.946 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 07:58:42.946 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 07:58:42.946 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 07:58:42.946 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 07:58:42.946 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 07:58:42.946 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 07:58:42.946 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 07:58:42.946 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 07:58:42.946 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 07:58:42.946 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 07:58:42.946 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 07:58:42.946 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 07:58:42.946 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 07:58:42.946 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 07:58:42.946 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 07:58:42.946 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 07:58:42.946 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 07:58:42.946 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 07:58:42.946 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 07:58:42.946 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 07:58:42.946 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 07:58:42.946 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 07:58:42.946 | INFO | ------------------------------------------ | 20260505 07:58:42.946 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 07:58:42.946 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 07:58:42.946 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:58:42.946 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:58:42.946 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.53e+01, 1.37e+01 / 2.21e+00 | 20260505 07:58:42.946 | INFO | PerfReport: 0.28%, AggregateStats -- 9.33e-02 / 1.04e-01, 1.03e-02 / 3.93e-03 | 20260505 07:58:42.946 | INFO | PerfReport: 11.58%, Baseline -- 3.80e+00 / 4.45e+00, 6.22e-01 / 2.65e-01 | 20260505 07:58:42.946 | INFO | PerfReport: 3.31%, Histogram -- 1.08e+00 / 1.48e+00, 2.12e-01 / 1.13e-01 | 20260505 07:58:42.946 | INFO | PerfReport: 15.54%, DME -- 4.08e+01 / 4.48e+01, 4.31e+00 / 1.78e+00 | 20260505 07:58:42.946 | INFO | PerfReport: 56.50%, FrameLabeling -- 1.85e+01 / 1.92e+01, 2.04e+00 / 1.46e+00 | 20260505 07:58:42.946 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.44e+00 / 3.08e+00, 4.14e-01 / 2.47e-01 | 20260505 07:58:42.946 | INFO | PerfReport: 3.93%, Metrics -- 1.29e+00 / 1.59e+00, 3.72e-01 / 2.07e-01 | 20260505 07:58:42.946 | INFO | PerfReport: 1.41%, PulseSerializing -- 4.61e-01 / 5.14e-01, 8.79e-02 / 3.52e-02 | 20260505 07:58:42.946 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 07:58:42.946 | INFO | PerfReport: Gathered 1536 samples. | 20260505 07:58:42.946 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 07:58:42.946 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.32e+01 / 0.00e+00 | 20260505 07:58:42.946 | INFO | PerfReport: 22.52%, Upload -- 2.30e+01 / 0.00e+00, 8.36e+00 / 0.00e+00 | 20260505 07:58:42.946 | INFO | PerfReport: 21.49%, Download -- 2.19e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260505 07:58:42.946 | INFO | PerfReport: 32.18%, Compute -- 3.28e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 07:58:42.946 | INFO | PerfReport: 1.73%, LockUpload -- 1.77e+00 / 0.00e+00, 7.34e+00 / 0.00e+00 | 20260505 07:58:42.946 | INFO | PerfReport: 0.89%, LockDownload -- 9.10e-01 / 0.00e+00, 2.35e+00 / 0.00e+00 | 20260505 07:58:42.946 | INFO | PerfReport: 21.18%, LockCompute -- 2.16e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260505 07:58:42.946 | INFO | PerfReport: 0.01%, Misc -- 1.46e-02 / 0.00e+00, 2.50e-02 / 0.00e+00 | 20260505 07:58:43.079 | INFO | Starting a round of Baz IO | 20260505 07:58:46.362 | INFO | Populating RTMetrics report | 20260505 07:58:46.362 | INFO | RTMetrics report populated | 20260505 07:58:46.362 | INFO | Opening tmp RTMetrics json file | 20260505 07:58:46.366 | INFO | Serializing RTMetrics to json | 20260505 07:58:46.398 | INFO | Serialization done, swapping tmp file now... | 20260505 07:58:46.412 | INFO | RTMetrics done | 20260505 07:58:46.504 | INFO | Finished a round of Baz IO: Wrote 2.441 GiB in 3.424 seconds | 20260505 07:58:46.504 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 28.59% / 0.03% / 71.39% | 20260505 07:58:46.504 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.0487% / 2.38419e-05% / 5.5631e-05% | 20260505 07:58:46.504 | INFO | Baz IO overall: | 20260505 07:58:46.504 | INFO | BytesWritten : 888415603584 | 20260505 07:58:46.504 | INFO | NumEvents : 421982534834 | 20260505 07:58:48.086 | INFO | Analyzing chunk frames = [1597440,1597952) | 20260505 07:58:48.086 | INFO | MemInfo: { | 20260505 07:58:48.086 | INFO | "gpu" : " 17 GB", | 20260505 07:58:48.086 | INFO | "huge" : "", | 20260505 07:58:48.086 | INFO | "huge_shared" : " 27 GB", | 20260505 07:58:48.086 | INFO | "malloc" : " 14 GB", | 20260505 07:58:48.086 | INFO | "pinned" : " 29 GB" | 20260505 07:58:48.086 | INFO | } | 20260505 07:58:48.086 | INFO | Perf report for the last 5.14145 seconds: | 20260505 07:58:48.086 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:58:48.086 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:48.086 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:48.086 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:48.086 | INFO | ANALYSIS : 65.3%, 2.9, 99.6ms, 0 | 20260505 07:58:48.086 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:48.086 | INFO | RT_METRICS : 46.9%, 1.0, 24.6ms, 0 | 20260505 07:58:48.086 | INFO | PRE_HQ : 63.7%, 1.9, 61.9ms, 0 | 20260505 07:58:48.086 | INFO | BAZWRITER : 65.9%, 4.2, 142.2ms, 0 | 20260505 07:58:48.086 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260505 07:58:48.086 | INFO | Uploaded 36GiB to GPU in 2.25551s (15.9609GiB/s) | 20260505 07:58:48.086 | INFO | Downloaded 38GiB to GPU in 2.36425s (16.0727GiB/s) | 20260505 07:58:53.225 | INFO | Analyzing chunk frames = [1597952,1598464) | 20260505 07:58:53.225 | INFO | MemInfo: { | 20260505 07:58:53.225 | INFO | "gpu" : " 17 GB", | 20260505 07:58:53.225 | INFO | "huge" : "", | 20260505 07:58:53.225 | INFO | "huge_shared" : " 27 GB", | 20260505 07:58:53.225 | INFO | "malloc" : " 14 GB", | 20260505 07:58:53.225 | INFO | "pinned" : " 29 GB" | 20260505 07:58:53.225 | INFO | } | 20260505 07:58:53.225 | INFO | Perf report for the last 5.13872 seconds: | 20260505 07:58:53.225 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:58:53.225 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:53.225 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:53.225 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:53.225 | INFO | ANALYSIS : 64.8%, 3.0, 101.1ms, 0 | 20260505 07:58:53.225 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:53.225 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:53.225 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:58:53.225 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:53.225 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:53.225 | INFO | Uploaded 36GiB to GPU in 2.24408s (16.0422GiB/s) | 20260505 07:58:53.225 | INFO | Downloaded 34GiB to GPU in 2.105s (16.152GiB/s) | 20260505 07:58:58.364 | INFO | Analyzing chunk frames = [1598464,1598976) | 20260505 07:58:58.364 | INFO | MemInfo: { | 20260505 07:58:58.364 | INFO | "gpu" : " 17 GB", | 20260505 07:58:58.364 | INFO | "huge" : "", | 20260505 07:58:58.364 | INFO | "huge_shared" : " 27 GB", | 20260505 07:58:58.364 | INFO | "malloc" : " 14 GB", | 20260505 07:58:58.364 | INFO | "pinned" : " 29 GB" | 20260505 07:58:58.364 | INFO | } | 20260505 07:58:58.364 | INFO | Perf report for the last 5.13905 seconds: | 20260505 07:58:58.364 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:58:58.364 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:58.364 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:58.364 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:58.364 | INFO | ANALYSIS : 63.8%, 3.0, 99.3ms, 0 | 20260505 07:58:58.364 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:58.364 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:58.364 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 07:58:58.364 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:58.364 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:58:58.364 | INFO | Uploaded 36GiB to GPU in 2.17553s (16.5477GiB/s) | 20260505 07:58:58.364 | INFO | Downloaded 34GiB to GPU in 2.06403s (16.4726GiB/s) | 20260505 07:59:03.502 | INFO | Analyzing chunk frames = [1598976,1599488) | 20260505 07:59:03.502 | INFO | MemInfo: { | 20260505 07:59:03.502 | INFO | "gpu" : " 17 GB", | 20260505 07:59:03.502 | INFO | "huge" : "", | 20260505 07:59:03.502 | INFO | "huge_shared" : " 27 GB", | 20260505 07:59:03.502 | INFO | "malloc" : " 14 GB", | 20260505 07:59:03.502 | INFO | "pinned" : " 29 GB" | 20260505 07:59:03.502 | INFO | } | 20260505 07:59:03.502 | INFO | Perf report for the last 5.1385 seconds: | 20260505 07:59:03.502 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:59:03.502 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:03.502 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:03.502 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:03.502 | INFO | ANALYSIS : 63.9%, 3.0, 99.2ms, 0 | 20260505 07:59:03.502 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:03.502 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:03.502 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:59:03.502 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:03.502 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:03.502 | INFO | Uploaded 36GiB to GPU in 2.19482s (16.4022GiB/s) | 20260505 07:59:03.503 | INFO | Downloaded 34GiB to GPU in 2.065s (16.4649GiB/s) | 20260505 07:59:08.641 | INFO | Analyzing chunk frames = [1599488,1600000) | 20260505 07:59:08.641 | INFO | MemInfo: { | 20260505 07:59:08.641 | INFO | "gpu" : " 17 GB", | 20260505 07:59:08.641 | INFO | "huge" : "", | 20260505 07:59:08.641 | INFO | "huge_shared" : " 27 GB", | 20260505 07:59:08.641 | INFO | "malloc" : " 14 GB", | 20260505 07:59:08.641 | INFO | "pinned" : " 29 GB" | 20260505 07:59:08.641 | INFO | } | 20260505 07:59:08.641 | INFO | Perf report for the last 5.13861 seconds: | 20260505 07:59:08.641 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:59:08.641 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:08.641 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:08.641 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:08.641 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 07:59:08.641 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:08.641 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:08.641 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 07:59:08.641 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:08.641 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:08.641 | INFO | Uploaded 36GiB to GPU in 2.18179s (16.5002GiB/s) | 20260505 07:59:08.641 | INFO | Downloaded 34GiB to GPU in 2.05806s (16.5204GiB/s) | 20260505 07:59:13.781 | INFO | Analyzing chunk frames = [1600000,1600512) | 20260505 07:59:13.781 | INFO | MemInfo: { | 20260505 07:59:13.781 | INFO | "gpu" : " 17 GB", | 20260505 07:59:13.781 | INFO | "huge" : "", | 20260505 07:59:13.781 | INFO | "huge_shared" : " 27 GB", | 20260505 07:59:13.781 | INFO | "malloc" : " 14 GB", | 20260505 07:59:13.781 | INFO | "pinned" : " 29 GB" | 20260505 07:59:13.781 | INFO | } | 20260505 07:59:13.781 | INFO | Perf report for the last 5.14055 seconds: | 20260505 07:59:13.781 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:59:13.781 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:13.781 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:13.781 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:13.781 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 07:59:13.782 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:13.782 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:13.782 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 07:59:13.782 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:13.782 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:13.782 | INFO | Uploaded 36GiB to GPU in 2.1793s (16.5191GiB/s) | 20260505 07:59:13.782 | INFO | Downloaded 34GiB to GPU in 2.04047s (16.6628GiB/s) | 20260505 07:59:18.921 | INFO | Analyzing chunk frames = [1600512,1601024) | 20260505 07:59:18.921 | INFO | MemInfo: { | 20260505 07:59:18.921 | INFO | "gpu" : " 17 GB", | 20260505 07:59:18.921 | INFO | "huge" : "", | 20260505 07:59:18.921 | INFO | "huge_shared" : " 27 GB", | 20260505 07:59:18.921 | INFO | "malloc" : " 14 GB", | 20260505 07:59:18.921 | INFO | "pinned" : " 29 GB" | 20260505 07:59:18.921 | INFO | } | 20260505 07:59:18.921 | INFO | Perf report for the last 5.14 seconds: | 20260505 07:59:18.921 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:59:18.921 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:18.921 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:18.922 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:18.922 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 07:59:18.922 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:18.922 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:18.922 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 07:59:18.922 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:18.922 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:18.922 | INFO | Uploaded 36GiB to GPU in 2.18502s (16.4758GiB/s) | 20260505 07:59:18.922 | INFO | Downloaded 34GiB to GPU in 2.14919s (15.8199GiB/s) | 20260505 07:59:24.061 | INFO | Analyzing chunk frames = [1601024,1601536) | 20260505 07:59:24.061 | INFO | MemInfo: { | 20260505 07:59:24.061 | INFO | "gpu" : " 17 GB", | 20260505 07:59:24.061 | INFO | "huge" : "", | 20260505 07:59:24.061 | INFO | "huge_shared" : " 27 GB", | 20260505 07:59:24.061 | INFO | "malloc" : " 14 GB", | 20260505 07:59:24.061 | INFO | "pinned" : " 29 GB" | 20260505 07:59:24.061 | INFO | } | 20260505 07:59:24.061 | INFO | Perf report for the last 5.13927 seconds: | 20260505 07:59:24.061 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:59:24.061 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:24.061 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:24.061 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:24.061 | INFO | ANALYSIS : 64.5%, 3.0, 100.6ms, 0 | 20260505 07:59:24.061 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:24.061 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:24.061 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 07:59:24.061 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:24.061 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:24.061 | INFO | Uploaded 36GiB to GPU in 2.19966s (16.3662GiB/s) | 20260505 07:59:24.061 | INFO | Downloaded 34GiB to GPU in 2.03546s (16.7038GiB/s) | 20260505 07:59:24.202 | INFO | Starting a round of Baz IO | 20260505 07:59:27.490 | INFO | Populating RTMetrics report | 20260505 07:59:27.491 | INFO | RTMetrics report populated | 20260505 07:59:27.491 | INFO | Opening tmp RTMetrics json file | 20260505 07:59:27.516 | INFO | Serializing RTMetrics to json | 20260505 07:59:27.540 | INFO | Serialization done, swapping tmp file now... | 20260505 07:59:27.542 | INFO | RTMetrics done | 20260505 07:59:27.644 | INFO | Finished a round of Baz IO: Wrote 2.445 GiB in 3.441 seconds | 20260505 07:59:27.644 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 28.56% / 0.02% / 71.41% | 20260505 07:59:27.644 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.05295% / 2.78155e-05% / 5.96046e-05% | 20260505 07:59:27.644 | INFO | Baz IO overall: | 20260505 07:59:27.644 | INFO | BytesWritten : 891041071217 | 20260505 07:59:27.644 | INFO | NumEvents : 423239245014 | 20260505 07:59:29.199 | INFO | Analyzing chunk frames = [1601536,1602048) | 20260505 07:59:29.199 | INFO | MemInfo: { | 20260505 07:59:29.199 | INFO | "gpu" : " 17 GB", | 20260505 07:59:29.199 | INFO | "huge" : "", | 20260505 07:59:29.199 | INFO | "huge_shared" : " 27 GB", | 20260505 07:59:29.199 | INFO | "malloc" : " 14 GB", | 20260505 07:59:29.199 | INFO | "pinned" : " 29 GB" | 20260505 07:59:29.199 | INFO | } | 20260505 07:59:29.199 | INFO | Perf report for the last 5.13857 seconds: | 20260505 07:59:29.199 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:59:29.199 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:29.199 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:29.199 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:29.199 | INFO | ANALYSIS : 65.6%, 2.9, 100.3ms, 0 | 20260505 07:59:29.199 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:29.199 | INFO | RT_METRICS : 46.9%, 1.0, 24.6ms, 0 | 20260505 07:59:29.199 | INFO | PRE_HQ : 63.8%, 1.9, 61.4ms, 0 | 20260505 07:59:29.199 | INFO | BAZWRITER : 66.3%, 4.1, 141.3ms, 0 | 20260505 07:59:29.199 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 07:59:29.200 | INFO | Uploaded 36GiB to GPU in 2.28254s (15.7719GiB/s) | 20260505 07:59:29.200 | INFO | Downloaded 38GiB to GPU in 2.37263s (16.016GiB/s) | 20260505 07:59:34.339 | INFO | Analyzing chunk frames = [1602048,1602560) | 20260505 07:59:34.339 | INFO | MemInfo: { | 20260505 07:59:34.339 | INFO | "gpu" : " 17 GB", | 20260505 07:59:34.339 | INFO | "huge" : "", | 20260505 07:59:34.339 | INFO | "huge_shared" : " 27 GB", | 20260505 07:59:34.339 | INFO | "malloc" : " 14 GB", | 20260505 07:59:34.339 | INFO | "pinned" : " 29 GB" | 20260505 07:59:34.339 | INFO | } | 20260505 07:59:34.339 | INFO | Perf report for the last 5.13951 seconds: | 20260505 07:59:34.339 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:59:34.339 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:34.339 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:34.339 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:34.339 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 07:59:34.339 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:34.339 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:34.339 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:59:34.339 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:34.339 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:34.339 | INFO | Uploaded 36GiB to GPU in 2.17644s (16.5407GiB/s) | 20260505 07:59:34.339 | INFO | Downloaded 34GiB to GPU in 2.05138s (16.5742GiB/s) | 20260505 07:59:39.478 | INFO | Analyzing chunk frames = [1602560,1603072) | 20260505 07:59:39.478 | INFO | MemInfo: { | 20260505 07:59:39.478 | INFO | "gpu" : " 17 GB", | 20260505 07:59:39.478 | INFO | "huge" : "", | 20260505 07:59:39.478 | INFO | "huge_shared" : " 27 GB", | 20260505 07:59:39.478 | INFO | "malloc" : " 14 GB", | 20260505 07:59:39.478 | INFO | "pinned" : " 29 GB" | 20260505 07:59:39.478 | INFO | } | 20260505 07:59:39.478 | INFO | Perf report for the last 5.13889 seconds: | 20260505 07:59:39.478 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:59:39.478 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:39.478 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:39.478 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:39.478 | INFO | ANALYSIS : 64.1%, 3.0, 99.7ms, 0 | 20260505 07:59:39.478 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:39.478 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:39.478 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:59:39.478 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:39.478 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:39.478 | INFO | Uploaded 36GiB to GPU in 2.19162s (16.4262GiB/s) | 20260505 07:59:39.478 | INFO | Downloaded 34GiB to GPU in 2.06017s (16.5035GiB/s) | 20260505 07:59:44.617 | INFO | Analyzing chunk frames = [1603072,1603584) | 20260505 07:59:44.617 | INFO | MemInfo: { | 20260505 07:59:44.617 | INFO | "gpu" : " 17 GB", | 20260505 07:59:44.617 | INFO | "huge" : "", | 20260505 07:59:44.617 | INFO | "huge_shared" : " 27 GB", | 20260505 07:59:44.617 | INFO | "malloc" : " 14 GB", | 20260505 07:59:44.617 | INFO | "pinned" : " 29 GB" | 20260505 07:59:44.617 | INFO | } | 20260505 07:59:44.617 | INFO | Perf report for the last 5.13922 seconds: | 20260505 07:59:44.617 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:59:44.617 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:44.617 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:44.617 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:44.617 | INFO | ANALYSIS : 63.9%, 3.0, 99.4ms, 0 | 20260505 07:59:44.617 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:44.617 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:44.617 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 07:59:44.617 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:44.617 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:44.617 | INFO | Uploaded 36GiB to GPU in 2.2038s (16.3354GiB/s) | 20260505 07:59:44.617 | INFO | Downloaded 34GiB to GPU in 2.06412s (16.4719GiB/s) | 20260505 07:59:49.756 | INFO | Analyzing chunk frames = [1603584,1604096) | 20260505 07:59:49.756 | INFO | MemInfo: { | 20260505 07:59:49.756 | INFO | "gpu" : " 17 GB", | 20260505 07:59:49.756 | INFO | "huge" : "", | 20260505 07:59:49.757 | INFO | "huge_shared" : " 27 GB", | 20260505 07:59:49.757 | INFO | "malloc" : " 14 GB", | 20260505 07:59:49.757 | INFO | "pinned" : " 29 GB" | 20260505 07:59:49.757 | INFO | } | 20260505 07:59:49.757 | INFO | Perf report for the last 5.13961 seconds: | 20260505 07:59:49.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:59:49.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:49.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:49.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:49.757 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 07:59:49.757 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:49.757 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:49.757 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 07:59:49.757 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:49.757 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:49.757 | INFO | Uploaded 36GiB to GPU in 2.20902s (16.2968GiB/s) | 20260505 07:59:49.757 | INFO | Downloaded 34GiB to GPU in 2.05069s (16.5798GiB/s) | 20260505 07:59:54.897 | INFO | Analyzing chunk frames = [1604096,1604608) | 20260505 07:59:54.897 | INFO | MemInfo: { | 20260505 07:59:54.897 | INFO | "gpu" : " 17 GB", | 20260505 07:59:54.897 | INFO | "huge" : "", | 20260505 07:59:54.897 | INFO | "huge_shared" : " 27 GB", | 20260505 07:59:54.897 | INFO | "malloc" : " 14 GB", | 20260505 07:59:54.897 | INFO | "pinned" : " 29 GB" | 20260505 07:59:54.897 | INFO | } | 20260505 07:59:54.897 | INFO | Perf report for the last 5.14061 seconds: | 20260505 07:59:54.898 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 07:59:54.898 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:54.898 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:54.898 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:54.898 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 07:59:54.898 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:54.898 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:54.898 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 07:59:54.898 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:54.898 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 07:59:54.898 | INFO | Uploaded 36GiB to GPU in 2.19309s (16.4152GiB/s) | 20260505 07:59:54.898 | INFO | Downloaded 34GiB to GPU in 2.04129s (16.6561GiB/s) | 20260505 08:00:00.035 | INFO | Analyzing chunk frames = [1604608,1605120) | 20260505 08:00:00.035 | INFO | MemInfo: { | 20260505 08:00:00.035 | INFO | "gpu" : " 17 GB", | 20260505 08:00:00.035 | INFO | "huge" : "", | 20260505 08:00:00.035 | INFO | "huge_shared" : " 27 GB", | 20260505 08:00:00.035 | INFO | "malloc" : " 14 GB", | 20260505 08:00:00.035 | INFO | "pinned" : " 29 GB" | 20260505 08:00:00.035 | INFO | } | 20260505 08:00:00.035 | INFO | Perf report for the last 5.13808 seconds: | 20260505 08:00:00.035 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:00:00.035 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:00.035 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:00.035 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:00.035 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 08:00:00.035 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:00.035 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:00.035 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 08:00:00.035 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:00.035 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:00.036 | INFO | Uploaded 36GiB to GPU in 2.20631s (16.3169GiB/s) | 20260505 08:00:00.036 | INFO | Downloaded 34GiB to GPU in 2.15618s (15.7686GiB/s) | 20260505 08:00:05.175 | INFO | Analyzing chunk frames = [1605120,1605632) | 20260505 08:00:05.175 | INFO | MemInfo: { | 20260505 08:00:05.175 | INFO | "gpu" : " 17 GB", | 20260505 08:00:05.175 | INFO | "huge" : "", | 20260505 08:00:05.175 | INFO | "huge_shared" : " 27 GB", | 20260505 08:00:05.175 | INFO | "malloc" : " 14 GB", | 20260505 08:00:05.175 | INFO | "pinned" : " 29 GB" | 20260505 08:00:05.175 | INFO | } | 20260505 08:00:05.175 | INFO | Perf report for the last 5.13976 seconds: | 20260505 08:00:05.175 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:00:05.175 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:05.175 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:05.175 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:05.175 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 08:00:05.175 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:05.175 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:05.175 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 08:00:05.175 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:05.175 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:05.175 | INFO | Uploaded 36GiB to GPU in 2.19744s (16.3827GiB/s) | 20260505 08:00:05.175 | INFO | Downloaded 34GiB to GPU in 2.03874s (16.677GiB/s) | 20260505 08:00:05.175 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 08:00:05.175 | INFO | Total mem usage: 15.3 GiB | 20260505 08:00:05.175 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 08:00:05.175 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -5.5%) | 20260505 08:00:05.176 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 08:00:05.176 | INFO | High Water Breakdown: | 20260505 08:00:05.176 | INFO | 60.9% : 8.84 GiB ( 94.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 08:00:05.176 | INFO | 39% : 5.67 GiB ( 85.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 08:00:05.176 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 08:00:05.176 | INFO | ------------------------------------------ | 20260505 08:00:05.176 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:00:05.176 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 08:00:05.176 | INFO | Total mem usage: 26.5 GiB | 20260505 08:00:05.176 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 08:00:05.176 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 08:00:05.176 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 08:00:05.176 | INFO | High Water Breakdown: | 20260505 08:00:05.176 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:00:05.176 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:00:05.176 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:00:05.176 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:00:05.176 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:00:05.176 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:00:05.176 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 08:00:05.176 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 08:00:05.176 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:00:05.176 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 08:00:05.176 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 08:00:05.176 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 08:00:05.176 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 08:00:05.176 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 08:00:05.176 | INFO | ------------------------------------------ | 20260505 08:00:05.176 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:00:05.176 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 08:00:05.176 | INFO | Total mem usage: 25.6 GiB | 20260505 08:00:05.176 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 08:00:05.176 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 08:00:05.176 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 08:00:05.176 | INFO | High Water Breakdown: | 20260505 08:00:05.176 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 08:00:05.176 | INFO | ------------------------------------------ | 20260505 08:00:05.176 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:00:05.176 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 08:00:05.176 | INFO | Total mem usage: 15.7 GiB | 20260505 08:00:05.176 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 08:00:05.176 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 08:00:05.176 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 08:00:05.176 | INFO | High Water Breakdown: | 20260505 08:00:05.176 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 08:00:05.176 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 08:00:05.176 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 08:00:05.176 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 08:00:05.176 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 08:00:05.176 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 08:00:05.176 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 08:00:05.176 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 08:00:05.176 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 08:00:05.176 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:00:05.176 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 08:00:05.176 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:00:05.176 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 08:00:05.176 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 08:00:05.176 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 08:00:05.176 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 08:00:05.176 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:00:05.176 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 08:00:05.176 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:00:05.176 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:00:05.176 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:00:05.176 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 08:00:05.176 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 08:00:05.176 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:00:05.176 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 08:00:05.176 | INFO | ------------------------------------------ | 20260505 08:00:05.176 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:00:05.176 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 08:00:05.176 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:00:05.176 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:00:05.176 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.55e+01, 1.36e+01 / 1.69e+00 | 20260505 08:00:05.176 | INFO | PerfReport: 0.28%, AggregateStats -- 9.33e-02 / 1.04e-01, 1.13e-02 / 3.64e-03 | 20260505 08:00:05.176 | INFO | PerfReport: 11.60%, Baseline -- 3.80e+00 / 4.46e+00, 6.26e-01 / 3.12e-01 | 20260505 08:00:05.176 | INFO | PerfReport: 3.32%, Histogram -- 1.09e+00 / 1.49e+00, 2.14e-01 / 1.05e-01 | 20260505 08:00:05.176 | INFO | PerfReport: 15.51%, DME -- 4.07e+01 / 4.46e+01, 4.21e+00 / 1.25e+00 | 20260505 08:00:05.176 | INFO | PerfReport: 56.47%, FrameLabeling -- 1.85e+01 / 1.96e+01, 2.00e+00 / 1.55e+00 | 20260505 08:00:05.176 | INFO | PerfReport: 7.46%, PulseAccumulating -- 2.44e+00 / 3.09e+00, 4.15e-01 / 2.41e-01 | 20260505 08:00:05.176 | INFO | PerfReport: 3.95%, Metrics -- 1.29e+00 / 1.61e+00, 3.76e-01 / 1.86e-01 | 20260505 08:00:05.176 | INFO | PerfReport: 1.41%, PulseSerializing -- 4.62e-01 / 5.16e-01, 8.91e-02 / 3.34e-02 | 20260505 08:00:05.177 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 08:00:05.177 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:00:05.177 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:00:05.177 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.31e+01 / 0.00e+00 | 20260505 08:00:05.177 | INFO | PerfReport: 22.54%, Upload -- 2.30e+01 / 0.00e+00, 8.29e+00 / 0.00e+00 | 20260505 08:00:05.177 | INFO | PerfReport: 21.54%, Download -- 2.20e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260505 08:00:05.177 | INFO | PerfReport: 32.16%, Compute -- 3.28e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 08:00:05.177 | INFO | PerfReport: 1.75%, LockUpload -- 1.78e+00 / 0.00e+00, 7.42e+00 / 0.00e+00 | 20260505 08:00:05.177 | INFO | PerfReport: 0.90%, LockDownload -- 9.12e-01 / 0.00e+00, 2.31e+00 / 0.00e+00 | 20260505 08:00:05.177 | INFO | PerfReport: 21.10%, LockCompute -- 2.15e+01 / 0.00e+00, 1.78e+01 / 0.00e+00 | 20260505 08:00:05.177 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 1.98e-02 / 0.00e+00 | 20260505 08:00:05.310 | INFO | Starting a round of Baz IO | 20260505 08:00:08.600 | INFO | Populating RTMetrics report | 20260505 08:00:08.600 | INFO | RTMetrics report populated | 20260505 08:00:08.600 | INFO | Opening tmp RTMetrics json file | 20260505 08:00:08.629 | INFO | Serializing RTMetrics to json | 20260505 08:00:08.633 | INFO | Serialization done, swapping tmp file now... | 20260505 08:00:08.636 | INFO | RTMetrics done | 20260505 08:00:08.740 | INFO | Finished a round of Baz IO: Wrote 2.445 GiB in 3.43 seconds | 20260505 08:00:08.740 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 28.54% / 0.02% / 71.44% | 20260505 08:00:08.741 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.04005% / 4.37101e-05% / 7.15256e-05% | 20260505 08:00:08.741 | INFO | Baz IO overall: | 20260505 08:00:08.741 | INFO | BytesWritten : 893665902064 | 20260505 08:00:08.741 | INFO | NumEvents : 424495193275 | 20260505 08:00:10.314 | INFO | Analyzing chunk frames = [1605632,1606144) | 20260505 08:00:10.314 | INFO | MemInfo: { | 20260505 08:00:10.314 | INFO | "gpu" : " 17 GB", | 20260505 08:00:10.314 | INFO | "huge" : "", | 20260505 08:00:10.314 | INFO | "huge_shared" : " 27 GB", | 20260505 08:00:10.314 | INFO | "malloc" : " 14 GB", | 20260505 08:00:10.314 | INFO | "pinned" : " 29 GB" | 20260505 08:00:10.314 | INFO | } | 20260505 08:00:10.315 | INFO | Perf report for the last 5.13947 seconds: | 20260505 08:00:10.315 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:00:10.315 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:10.315 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:10.315 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:10.315 | INFO | ANALYSIS : 65.5%, 2.9, 100.2ms, 0 | 20260505 08:00:10.315 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:10.315 | INFO | RT_METRICS : 46.5%, 1.0, 24.4ms, 0 | 20260505 08:00:10.315 | INFO | PRE_HQ : 63.7%, 1.9, 61.6ms, 0 | 20260505 08:00:10.315 | INFO | BAZWRITER : 66.0%, 4.2, 140.7ms, 0 | 20260505 08:00:10.315 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 08:00:10.315 | INFO | Uploaded 36GiB to GPU in 2.28973s (15.7224GiB/s) | 20260505 08:00:10.315 | INFO | Downloaded 38GiB to GPU in 2.38657s (15.9224GiB/s) | 20260505 08:00:15.453 | INFO | Analyzing chunk frames = [1606144,1606656) | 20260505 08:00:15.453 | INFO | MemInfo: { | 20260505 08:00:15.453 | INFO | "gpu" : " 17 GB", | 20260505 08:00:15.453 | INFO | "huge" : "", | 20260505 08:00:15.453 | INFO | "huge_shared" : " 27 GB", | 20260505 08:00:15.453 | INFO | "malloc" : " 14 GB", | 20260505 08:00:15.453 | INFO | "pinned" : " 29 GB" | 20260505 08:00:15.453 | INFO | } | 20260505 08:00:15.453 | INFO | Perf report for the last 5.13845 seconds: | 20260505 08:00:15.453 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:00:15.453 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:15.453 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:15.453 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:15.453 | INFO | ANALYSIS : 64.7%, 3.0, 101.1ms, 0 | 20260505 08:00:15.453 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:15.453 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:15.453 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:00:15.453 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:15.453 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:15.453 | INFO | Uploaded 36GiB to GPU in 2.24447s (16.0394GiB/s) | 20260505 08:00:15.453 | INFO | Downloaded 34GiB to GPU in 2.11191s (16.0992GiB/s) | 20260505 08:00:20.592 | INFO | Analyzing chunk frames = [1606656,1607168) | 20260505 08:00:20.592 | INFO | MemInfo: { | 20260505 08:00:20.592 | INFO | "gpu" : " 17 GB", | 20260505 08:00:20.592 | INFO | "huge" : "", | 20260505 08:00:20.592 | INFO | "huge_shared" : " 27 GB", | 20260505 08:00:20.592 | INFO | "malloc" : " 14 GB", | 20260505 08:00:20.592 | INFO | "pinned" : " 29 GB" | 20260505 08:00:20.592 | INFO | } | 20260505 08:00:20.592 | INFO | Perf report for the last 5.13942 seconds: | 20260505 08:00:20.592 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:00:20.592 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:20.592 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:20.592 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:20.592 | INFO | ANALYSIS : 63.9%, 3.0, 99.5ms, 0 | 20260505 08:00:20.593 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:20.593 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:20.593 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 08:00:20.593 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:20.593 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:20.593 | INFO | Uploaded 36GiB to GPU in 2.17613s (16.5432GiB/s) | 20260505 08:00:20.593 | INFO | Downloaded 34GiB to GPU in 2.06262s (16.4839GiB/s) | 20260505 08:00:25.733 | INFO | Analyzing chunk frames = [1607168,1607680) | 20260505 08:00:25.733 | INFO | MemInfo: { | 20260505 08:00:25.733 | INFO | "gpu" : " 17 GB", | 20260505 08:00:25.733 | INFO | "huge" : "", | 20260505 08:00:25.733 | INFO | "huge_shared" : " 27 GB", | 20260505 08:00:25.733 | INFO | "malloc" : " 14 GB", | 20260505 08:00:25.733 | INFO | "pinned" : " 29 GB" | 20260505 08:00:25.733 | INFO | } | 20260505 08:00:25.733 | INFO | Perf report for the last 5.14071 seconds: | 20260505 08:00:25.733 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:00:25.733 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:25.733 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:25.733 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:25.733 | INFO | ANALYSIS : 63.9%, 3.0, 99.3ms, 0 | 20260505 08:00:25.733 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:25.733 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:25.733 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 08:00:25.733 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:25.733 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:25.733 | INFO | Uploaded 36GiB to GPU in 2.20227s (16.3467GiB/s) | 20260505 08:00:25.733 | INFO | Downloaded 34GiB to GPU in 2.0601s (16.504GiB/s) | 20260505 08:00:30.872 | INFO | Analyzing chunk frames = [1607680,1608192) | 20260505 08:00:30.872 | INFO | MemInfo: { | 20260505 08:00:30.872 | INFO | "gpu" : " 17 GB", | 20260505 08:00:30.872 | INFO | "huge" : "", | 20260505 08:00:30.872 | INFO | "huge_shared" : " 27 GB", | 20260505 08:00:30.872 | INFO | "malloc" : " 14 GB", | 20260505 08:00:30.872 | INFO | "pinned" : " 29 GB" | 20260505 08:00:30.872 | INFO | } | 20260505 08:00:30.872 | INFO | Perf report for the last 5.13856 seconds: | 20260505 08:00:30.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:00:30.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:30.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:30.872 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:30.872 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 08:00:30.872 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:30.872 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:30.872 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 08:00:30.872 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:30.872 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:30.872 | INFO | Uploaded 36GiB to GPU in 2.18233s (16.4962GiB/s) | 20260505 08:00:30.872 | INFO | Downloaded 34GiB to GPU in 2.05868s (16.5155GiB/s) | 20260505 08:00:36.011 | INFO | Analyzing chunk frames = [1608192,1608704) | 20260505 08:00:36.011 | INFO | MemInfo: { | 20260505 08:00:36.011 | INFO | "gpu" : " 17 GB", | 20260505 08:00:36.011 | INFO | "huge" : "", | 20260505 08:00:36.011 | INFO | "huge_shared" : " 27 GB", | 20260505 08:00:36.011 | INFO | "malloc" : " 14 GB", | 20260505 08:00:36.011 | INFO | "pinned" : " 29 GB" | 20260505 08:00:36.011 | INFO | } | 20260505 08:00:36.011 | INFO | Perf report for the last 5.13948 seconds: | 20260505 08:00:36.011 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:00:36.011 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:36.011 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:36.011 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:36.011 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 08:00:36.011 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:36.011 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:36.011 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 08:00:36.011 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:36.011 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:36.011 | INFO | Uploaded 36GiB to GPU in 2.17896s (16.5216GiB/s) | 20260505 08:00:36.011 | INFO | Downloaded 34GiB to GPU in 2.04454s (16.6297GiB/s) | 20260505 08:00:41.149 | INFO | Analyzing chunk frames = [1608704,1609216) | 20260505 08:00:41.149 | INFO | MemInfo: { | 20260505 08:00:41.149 | INFO | "gpu" : " 17 GB", | 20260505 08:00:41.149 | INFO | "huge" : "", | 20260505 08:00:41.149 | INFO | "huge_shared" : " 27 GB", | 20260505 08:00:41.149 | INFO | "malloc" : " 14 GB", | 20260505 08:00:41.149 | INFO | "pinned" : " 29 GB" | 20260505 08:00:41.149 | INFO | } | 20260505 08:00:41.149 | INFO | Perf report for the last 5.13812 seconds: | 20260505 08:00:41.149 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:00:41.149 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:41.149 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:41.149 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:41.149 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 08:00:41.149 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:41.149 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:41.149 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260505 08:00:41.149 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:41.149 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:41.150 | INFO | Uploaded 36GiB to GPU in 2.20216s (16.3476GiB/s) | 20260505 08:00:41.150 | INFO | Downloaded 34GiB to GPU in 2.15527s (15.7753GiB/s) | 20260505 08:00:46.290 | INFO | Analyzing chunk frames = [1609216,1609728) | 20260505 08:00:46.290 | INFO | MemInfo: { | 20260505 08:00:46.290 | INFO | "gpu" : " 17 GB", | 20260505 08:00:46.290 | INFO | "huge" : "", | 20260505 08:00:46.290 | INFO | "huge_shared" : " 27 GB", | 20260505 08:00:46.290 | INFO | "malloc" : " 14 GB", | 20260505 08:00:46.290 | INFO | "pinned" : " 29 GB" | 20260505 08:00:46.290 | INFO | } | 20260505 08:00:46.290 | INFO | Perf report for the last 5.14044 seconds: | 20260505 08:00:46.290 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:00:46.290 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:46.290 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:46.290 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:46.290 | INFO | ANALYSIS : 64.4%, 3.0, 100.7ms, 0 | 20260505 08:00:46.290 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:46.290 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:46.290 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 08:00:46.290 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:46.290 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:46.290 | INFO | Uploaded 36GiB to GPU in 2.20459s (16.3295GiB/s) | 20260505 08:00:46.290 | INFO | Downloaded 34GiB to GPU in 2.04806s (16.6011GiB/s) | 20260505 08:00:46.424 | INFO | Starting a round of Baz IO | 20260505 08:00:49.727 | INFO | Populating RTMetrics report | 20260505 08:00:49.727 | INFO | RTMetrics report populated | 20260505 08:00:49.727 | INFO | Opening tmp RTMetrics json file | 20260505 08:00:49.768 | INFO | Serializing RTMetrics to json | 20260505 08:00:49.769 | INFO | Serialization done, swapping tmp file now... | 20260505 08:00:49.770 | INFO | RTMetrics done | 20260505 08:00:49.871 | INFO | Finished a round of Baz IO: Wrote 2.447 GiB in 3.447 seconds | 20260505 08:00:49.871 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 28.51% / 0.03% / 71.46% | 20260505 08:00:49.871 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.03231% / 3.97364e-05% / 7.15256e-05% | 20260505 08:00:49.871 | INFO | Baz IO overall: | 20260505 08:00:49.871 | INFO | BytesWritten : 896293020855 | 20260505 08:00:49.871 | INFO | NumEvents : 425752506776 | 20260505 08:00:51.429 | INFO | Analyzing chunk frames = [1609728,1610240) | 20260505 08:00:51.429 | INFO | MemInfo: { | 20260505 08:00:51.429 | INFO | "gpu" : " 17 GB", | 20260505 08:00:51.429 | INFO | "huge" : "", | 20260505 08:00:51.429 | INFO | "huge_shared" : " 27 GB", | 20260505 08:00:51.429 | INFO | "malloc" : " 14 GB", | 20260505 08:00:51.429 | INFO | "pinned" : " 29 GB" | 20260505 08:00:51.429 | INFO | } | 20260505 08:00:51.429 | INFO | Perf report for the last 5.13892 seconds: | 20260505 08:00:51.429 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:00:51.429 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:51.429 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:51.429 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:51.429 | INFO | ANALYSIS : 65.7%, 2.9, 100.4ms, 0 | 20260505 08:00:51.429 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:51.429 | INFO | RT_METRICS : 46.7%, 1.0, 24.5ms, 0 | 20260505 08:00:51.429 | INFO | PRE_HQ : 63.6%, 1.9, 61.6ms, 0 | 20260505 08:00:51.429 | INFO | BAZWRITER : 66.3%, 4.1, 141.1ms, 0 | 20260505 08:00:51.429 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260505 08:00:51.429 | INFO | Uploaded 36GiB to GPU in 2.23969s (16.0736GiB/s) | 20260505 08:00:51.429 | INFO | Downloaded 38GiB to GPU in 2.3345s (16.2776GiB/s) | 20260505 08:00:56.568 | INFO | Analyzing chunk frames = [1610240,1610752) | 20260505 08:00:56.568 | INFO | MemInfo: { | 20260505 08:00:56.568 | INFO | "gpu" : " 17 GB", | 20260505 08:00:56.568 | INFO | "huge" : "", | 20260505 08:00:56.568 | INFO | "huge_shared" : " 27 GB", | 20260505 08:00:56.568 | INFO | "malloc" : " 14 GB", | 20260505 08:00:56.568 | INFO | "pinned" : " 29 GB" | 20260505 08:00:56.568 | INFO | } | 20260505 08:00:56.568 | INFO | Perf report for the last 5.13949 seconds: | 20260505 08:00:56.568 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:00:56.568 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:56.568 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:56.568 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:56.568 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 08:00:56.568 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:56.568 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:56.568 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:00:56.568 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:56.568 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:00:56.568 | INFO | Uploaded 36GiB to GPU in 2.17981s (16.5152GiB/s) | 20260505 08:00:56.569 | INFO | Downloaded 34GiB to GPU in 2.04922s (16.5917GiB/s) | 20260505 08:01:01.709 | INFO | Analyzing chunk frames = [1610752,1611264) | 20260505 08:01:01.709 | INFO | MemInfo: { | 20260505 08:01:01.709 | INFO | "gpu" : " 17 GB", | 20260505 08:01:01.709 | INFO | "huge" : "", | 20260505 08:01:01.709 | INFO | "huge_shared" : " 27 GB", | 20260505 08:01:01.709 | INFO | "malloc" : " 14 GB", | 20260505 08:01:01.709 | INFO | "pinned" : " 29 GB" | 20260505 08:01:01.709 | INFO | } | 20260505 08:01:01.709 | INFO | Perf report for the last 5.14074 seconds: | 20260505 08:01:01.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:01:01.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:01.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:01.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:01.709 | INFO | ANALYSIS : 63.9%, 3.0, 99.5ms, 0 | 20260505 08:01:01.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:01.709 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:01.709 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 08:01:01.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:01.709 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:01.709 | INFO | Uploaded 36GiB to GPU in 2.18115s (16.5051GiB/s) | 20260505 08:01:01.709 | INFO | Downloaded 34GiB to GPU in 2.0643s (16.4705GiB/s) | 20260505 08:01:06.848 | INFO | Analyzing chunk frames = [1611264,1611776) | 20260505 08:01:06.848 | INFO | MemInfo: { | 20260505 08:01:06.848 | INFO | "gpu" : " 17 GB", | 20260505 08:01:06.848 | INFO | "huge" : "", | 20260505 08:01:06.848 | INFO | "huge_shared" : " 27 GB", | 20260505 08:01:06.848 | INFO | "malloc" : " 14 GB", | 20260505 08:01:06.848 | INFO | "pinned" : " 29 GB" | 20260505 08:01:06.848 | INFO | } | 20260505 08:01:06.848 | INFO | Perf report for the last 5.13872 seconds: | 20260505 08:01:06.848 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:01:06.848 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:06.848 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:06.848 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:06.848 | INFO | ANALYSIS : 63.8%, 3.0, 99.1ms, 0 | 20260505 08:01:06.848 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:06.848 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:06.848 | INFO | PRE_HQ : 30.0%, 1.0, 15.5ms, 0 | 20260505 08:01:06.848 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:06.848 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:06.848 | INFO | Uploaded 36GiB to GPU in 2.19298s (16.416GiB/s) | 20260505 08:01:06.848 | INFO | Downloaded 34GiB to GPU in 2.0644s (16.4697GiB/s) | 20260505 08:01:11.987 | INFO | Analyzing chunk frames = [1611776,1612288) | 20260505 08:01:11.987 | INFO | MemInfo: { | 20260505 08:01:11.987 | INFO | "gpu" : " 17 GB", | 20260505 08:01:11.987 | INFO | "huge" : "", | 20260505 08:01:11.987 | INFO | "huge_shared" : " 27 GB", | 20260505 08:01:11.987 | INFO | "malloc" : " 14 GB", | 20260505 08:01:11.987 | INFO | "pinned" : " 29 GB" | 20260505 08:01:11.987 | INFO | } | 20260505 08:01:11.987 | INFO | Perf report for the last 5.13971 seconds: | 20260505 08:01:11.987 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:01:11.987 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:11.988 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:11.988 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:11.988 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 08:01:11.988 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:11.988 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:11.988 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:01:11.988 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:11.988 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:11.988 | INFO | Uploaded 36GiB to GPU in 2.17825s (16.527GiB/s) | 20260505 08:01:11.988 | INFO | Downloaded 34GiB to GPU in 2.05442s (16.5497GiB/s) | 20260505 08:01:17.126 | INFO | Analyzing chunk frames = [1612288,1612800) | 20260505 08:01:17.126 | INFO | MemInfo: { | 20260505 08:01:17.126 | INFO | "gpu" : " 17 GB", | 20260505 08:01:17.126 | INFO | "huge" : "", | 20260505 08:01:17.126 | INFO | "huge_shared" : " 27 GB", | 20260505 08:01:17.126 | INFO | "malloc" : " 14 GB", | 20260505 08:01:17.126 | INFO | "pinned" : " 29 GB" | 20260505 08:01:17.126 | INFO | } | 20260505 08:01:17.126 | INFO | Perf report for the last 5.13824 seconds: | 20260505 08:01:17.126 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:01:17.126 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:17.126 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:17.126 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:17.126 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 08:01:17.126 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:17.126 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:17.126 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:01:17.126 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:17.126 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:17.126 | INFO | Uploaded 36GiB to GPU in 2.16976s (16.5917GiB/s) | 20260505 08:01:17.126 | INFO | Downloaded 34GiB to GPU in 2.04393s (16.6346GiB/s) | 20260505 08:01:22.264 | INFO | Analyzing chunk frames = [1612800,1613312) | 20260505 08:01:22.264 | INFO | MemInfo: { | 20260505 08:01:22.264 | INFO | "gpu" : " 17 GB", | 20260505 08:01:22.264 | INFO | "huge" : "", | 20260505 08:01:22.264 | INFO | "huge_shared" : " 27 GB", | 20260505 08:01:22.264 | INFO | "malloc" : " 14 GB", | 20260505 08:01:22.264 | INFO | "pinned" : " 29 GB" | 20260505 08:01:22.264 | INFO | } | 20260505 08:01:22.264 | INFO | Perf report for the last 5.1387 seconds: | 20260505 08:01:22.264 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:01:22.264 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:22.264 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:22.264 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:22.264 | INFO | ANALYSIS : 64.1%, 3.0, 100.0ms, 0 | 20260505 08:01:22.265 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:22.265 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:22.265 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 08:01:22.265 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:22.265 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:22.265 | INFO | Uploaded 36GiB to GPU in 2.19113s (16.4299GiB/s) | 20260505 08:01:22.265 | INFO | Downloaded 34GiB to GPU in 2.15632s (15.7676GiB/s) | 20260505 08:01:27.405 | INFO | Analyzing chunk frames = [1613312,1613824) | 20260505 08:01:27.405 | INFO | MemInfo: { | 20260505 08:01:27.405 | INFO | "gpu" : " 17 GB", | 20260505 08:01:27.405 | INFO | "huge" : "", | 20260505 08:01:27.405 | INFO | "huge_shared" : " 27 GB", | 20260505 08:01:27.405 | INFO | "malloc" : " 14 GB", | 20260505 08:01:27.405 | INFO | "pinned" : " 29 GB" | 20260505 08:01:27.405 | INFO | } | 20260505 08:01:27.405 | INFO | Perf report for the last 5.14093 seconds: | 20260505 08:01:27.405 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:01:27.405 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:27.405 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:27.405 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:27.405 | INFO | ANALYSIS : 64.5%, 3.0, 100.8ms, 0 | 20260505 08:01:27.405 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:27.405 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:27.405 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 08:01:27.406 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:27.406 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:27.406 | INFO | Uploaded 36GiB to GPU in 2.21458s (16.2559GiB/s) | 20260505 08:01:27.406 | INFO | Downloaded 34GiB to GPU in 2.05344s (16.5576GiB/s) | 20260505 08:01:27.406 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 08:01:27.406 | INFO | Total mem usage: 15.3 GiB | 20260505 08:01:27.406 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 08:01:27.406 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.23%) | 20260505 08:01:27.406 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 08:01:27.406 | INFO | High Water Breakdown: | 20260505 08:01:27.406 | INFO | 60.8% : 8.77 GiB ( 93.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 08:01:27.406 | INFO | 39.1% : 5.64 GiB ( 84.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 08:01:27.406 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 08:01:27.406 | INFO | ------------------------------------------ | 20260505 08:01:27.406 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:01:27.406 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 08:01:27.406 | INFO | Total mem usage: 26.5 GiB | 20260505 08:01:27.406 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 08:01:27.406 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 08:01:27.406 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 08:01:27.406 | INFO | High Water Breakdown: | 20260505 08:01:27.406 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:01:27.406 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:01:27.406 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:01:27.406 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:01:27.406 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:01:27.406 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:01:27.406 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 08:01:27.406 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 08:01:27.406 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:01:27.406 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 08:01:27.406 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 08:01:27.406 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 08:01:27.406 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 08:01:27.406 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 08:01:27.406 | INFO | ------------------------------------------ | 20260505 08:01:27.406 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:01:27.406 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 08:01:27.406 | INFO | Total mem usage: 25.6 GiB | 20260505 08:01:27.406 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 08:01:27.406 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 08:01:27.406 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 08:01:27.406 | INFO | High Water Breakdown: | 20260505 08:01:27.406 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 08:01:27.406 | INFO | ------------------------------------------ | 20260505 08:01:27.406 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:01:27.406 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 08:01:27.406 | INFO | Total mem usage: 15.7 GiB | 20260505 08:01:27.406 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 08:01:27.406 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 08:01:27.406 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 08:01:27.406 | INFO | High Water Breakdown: | 20260505 08:01:27.406 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 08:01:27.406 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 08:01:27.406 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 08:01:27.406 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 08:01:27.406 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 08:01:27.406 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 08:01:27.407 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 08:01:27.407 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 08:01:27.407 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 08:01:27.407 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:01:27.407 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 08:01:27.407 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:01:27.407 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 08:01:27.407 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 08:01:27.407 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 08:01:27.407 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 08:01:27.407 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:01:27.407 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 08:01:27.407 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:01:27.407 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:01:27.407 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:01:27.407 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 08:01:27.407 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 08:01:27.407 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:01:27.407 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 08:01:27.407 | INFO | ------------------------------------------ | 20260505 08:01:27.407 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:01:27.407 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 08:01:27.407 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:01:27.407 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:01:27.407 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.53e+01, 1.36e+01 / 2.19e+00 | 20260505 08:01:27.407 | INFO | PerfReport: 0.28%, AggregateStats -- 9.33e-02 / 1.05e-01, 1.06e-02 / 4.06e-03 | 20260505 08:01:27.407 | INFO | PerfReport: 11.60%, Baseline -- 3.80e+00 / 4.46e+00, 6.20e-01 / 2.66e-01 | 20260505 08:01:27.407 | INFO | PerfReport: 3.31%, Histogram -- 1.08e+00 / 1.49e+00, 2.10e-01 / 1.00e-01 | 20260505 08:01:27.407 | INFO | PerfReport: 15.54%, DME -- 4.07e+01 / 4.46e+01, 4.10e+00 / 1.42e+00 | 20260505 08:01:27.407 | INFO | PerfReport: 56.43%, FrameLabeling -- 1.85e+01 / 1.95e+01, 1.98e+00 / 1.78e+00 | 20260505 08:01:27.407 | INFO | PerfReport: 7.48%, PulseAccumulating -- 2.45e+00 / 3.06e+00, 4.20e-01 / 2.67e-01 | 20260505 08:01:27.407 | INFO | PerfReport: 3.95%, Metrics -- 1.29e+00 / 1.59e+00, 3.73e-01 / 1.96e-01 | 20260505 08:01:27.407 | INFO | PerfReport: 1.41%, PulseSerializing -- 4.62e-01 / 5.16e-01, 8.90e-02 / 3.73e-02 | 20260505 08:01:27.407 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 08:01:27.407 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:01:27.407 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:01:27.407 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.32e+01 / 0.00e+00 | 20260505 08:01:27.407 | INFO | PerfReport: 22.50%, Upload -- 2.29e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260505 08:01:27.407 | INFO | PerfReport: 21.56%, Download -- 2.20e+01 / 0.00e+00, 8.18e+00 / 0.00e+00 | 20260505 08:01:27.407 | INFO | PerfReport: 32.15%, Compute -- 3.28e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 08:01:27.407 | INFO | PerfReport: 1.73%, LockUpload -- 1.76e+00 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260505 08:01:27.407 | INFO | PerfReport: 0.93%, LockDownload -- 9.44e-01 / 0.00e+00, 2.40e+00 / 0.00e+00 | 20260505 08:01:27.407 | INFO | PerfReport: 21.12%, LockCompute -- 2.15e+01 / 0.00e+00, 1.77e+01 / 0.00e+00 | 20260505 08:01:27.407 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.74e-02 / 0.00e+00 | 20260505 08:01:27.546 | INFO | Starting a round of Baz IO | 20260505 08:01:30.828 | INFO | Populating RTMetrics report | 20260505 08:01:30.828 | INFO | RTMetrics report populated | 20260505 08:01:30.828 | INFO | Opening tmp RTMetrics json file | 20260505 08:01:30.838 | INFO | Serializing RTMetrics to json | 20260505 08:01:30.881 | INFO | Serialization done, swapping tmp file now... | 20260505 08:01:30.885 | INFO | RTMetrics done | 20260505 08:01:30.959 | INFO | Finished a round of Baz IO: Wrote 2.447 GiB in 3.413 seconds | 20260505 08:01:30.960 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 28.49% / 0.03% / 71.49% | 20260505 08:01:30.960 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.02049% / 2.38419e-05% / 7.54992e-05% | 20260505 08:01:30.960 | INFO | Baz IO overall: | 20260505 08:01:30.960 | INFO | BytesWritten : 898919961073 | 20260505 08:01:30.960 | INFO | NumEvents : 427009536079 | 20260505 08:01:32.543 | INFO | Analyzing chunk frames = [1613824,1614336) | 20260505 08:01:32.543 | INFO | MemInfo: { | 20260505 08:01:32.543 | INFO | "gpu" : " 17 GB", | 20260505 08:01:32.543 | INFO | "huge" : "", | 20260505 08:01:32.543 | INFO | "huge_shared" : " 27 GB", | 20260505 08:01:32.543 | INFO | "malloc" : " 14 GB", | 20260505 08:01:32.543 | INFO | "pinned" : " 29 GB" | 20260505 08:01:32.543 | INFO | } | 20260505 08:01:32.543 | INFO | Perf report for the last 5.13795 seconds: | 20260505 08:01:32.543 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:01:32.543 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:32.543 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:32.543 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:32.543 | INFO | ANALYSIS : 65.5%, 2.9, 99.9ms, 0 | 20260505 08:01:32.543 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:32.543 | INFO | RT_METRICS : 47.1%, 1.0, 24.7ms, 0 | 20260505 08:01:32.543 | INFO | PRE_HQ : 63.8%, 1.9, 61.9ms, 0 | 20260505 08:01:32.543 | INFO | BAZWRITER : 65.7%, 4.2, 140.4ms, 0 | 20260505 08:01:32.543 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260505 08:01:32.544 | INFO | Uploaded 36GiB to GPU in 2.28789s (15.735GiB/s) | 20260505 08:01:32.544 | INFO | Downloaded 38GiB to GPU in 2.38909s (15.9056GiB/s) | 20260505 08:01:37.684 | INFO | Analyzing chunk frames = [1614336,1614848) | 20260505 08:01:37.684 | INFO | MemInfo: { | 20260505 08:01:37.684 | INFO | "gpu" : " 17 GB", | 20260505 08:01:37.684 | INFO | "huge" : "", | 20260505 08:01:37.684 | INFO | "huge_shared" : " 27 GB", | 20260505 08:01:37.684 | INFO | "malloc" : " 14 GB", | 20260505 08:01:37.684 | INFO | "pinned" : " 29 GB" | 20260505 08:01:37.684 | INFO | } | 20260505 08:01:37.684 | INFO | Perf report for the last 5.14041 seconds: | 20260505 08:01:37.684 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:01:37.684 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:37.684 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:37.684 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:37.684 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 08:01:37.684 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:37.684 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:37.684 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:01:37.684 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:37.684 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:37.684 | INFO | Uploaded 36GiB to GPU in 2.17316s (16.5657GiB/s) | 20260505 08:01:37.684 | INFO | Downloaded 34GiB to GPU in 2.0473s (16.6073GiB/s) | 20260505 08:01:42.822 | INFO | Analyzing chunk frames = [1614848,1615360) | 20260505 08:01:42.823 | INFO | MemInfo: { | 20260505 08:01:42.823 | INFO | "gpu" : " 17 GB", | 20260505 08:01:42.823 | INFO | "huge" : "", | 20260505 08:01:42.823 | INFO | "huge_shared" : " 27 GB", | 20260505 08:01:42.823 | INFO | "malloc" : " 14 GB", | 20260505 08:01:42.823 | INFO | "pinned" : " 29 GB" | 20260505 08:01:42.823 | INFO | } | 20260505 08:01:42.823 | INFO | Perf report for the last 5.13899 seconds: | 20260505 08:01:42.823 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:01:42.823 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:42.823 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:42.823 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:42.823 | INFO | ANALYSIS : 63.8%, 3.0, 99.2ms, 0 | 20260505 08:01:42.823 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:42.823 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:42.823 | INFO | PRE_HQ : 29.8%, 1.0, 15.4ms, 0 | 20260505 08:01:42.823 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:42.823 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:42.823 | INFO | Uploaded 36GiB to GPU in 2.17335s (16.5643GiB/s) | 20260505 08:01:42.823 | INFO | Downloaded 34GiB to GPU in 2.06472s (16.4671GiB/s) | 20260505 08:01:47.961 | INFO | Analyzing chunk frames = [1615360,1615872) | 20260505 08:01:47.961 | INFO | MemInfo: { | 20260505 08:01:47.962 | INFO | "gpu" : " 17 GB", | 20260505 08:01:47.962 | INFO | "huge" : "", | 20260505 08:01:47.962 | INFO | "huge_shared" : " 27 GB", | 20260505 08:01:47.962 | INFO | "malloc" : " 14 GB", | 20260505 08:01:47.962 | INFO | "pinned" : " 29 GB" | 20260505 08:01:47.962 | INFO | } | 20260505 08:01:47.962 | INFO | Perf report for the last 5.13893 seconds: | 20260505 08:01:47.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:01:47.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:47.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:47.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:47.962 | INFO | ANALYSIS : 63.8%, 3.0, 99.1ms, 0 | 20260505 08:01:47.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:47.962 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:47.962 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 08:01:47.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:47.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:47.962 | INFO | Uploaded 36GiB to GPU in 2.19341s (16.4128GiB/s) | 20260505 08:01:47.962 | INFO | Downloaded 34GiB to GPU in 2.06949s (16.4292GiB/s) | 20260505 08:01:53.100 | INFO | Analyzing chunk frames = [1615872,1616384) | 20260505 08:01:53.100 | INFO | MemInfo: { | 20260505 08:01:53.100 | INFO | "gpu" : " 17 GB", | 20260505 08:01:53.100 | INFO | "huge" : "", | 20260505 08:01:53.100 | INFO | "huge_shared" : " 27 GB", | 20260505 08:01:53.100 | INFO | "malloc" : " 14 GB", | 20260505 08:01:53.100 | INFO | "pinned" : " 29 GB" | 20260505 08:01:53.100 | INFO | } | 20260505 08:01:53.100 | INFO | Perf report for the last 5.13849 seconds: | 20260505 08:01:53.100 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:01:53.100 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:53.100 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:53.100 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:53.100 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 08:01:53.100 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:53.100 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:53.100 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 08:01:53.100 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:53.100 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:53.101 | INFO | Uploaded 36GiB to GPU in 2.18765s (16.456GiB/s) | 20260505 08:01:53.101 | INFO | Downloaded 34GiB to GPU in 2.05486s (16.5461GiB/s) | 20260505 08:01:58.241 | INFO | Analyzing chunk frames = [1616384,1616896) | 20260505 08:01:58.241 | INFO | MemInfo: { | 20260505 08:01:58.241 | INFO | "gpu" : " 17 GB", | 20260505 08:01:58.241 | INFO | "huge" : "", | 20260505 08:01:58.241 | INFO | "huge_shared" : " 27 GB", | 20260505 08:01:58.241 | INFO | "malloc" : " 14 GB", | 20260505 08:01:58.241 | INFO | "pinned" : " 29 GB" | 20260505 08:01:58.241 | INFO | } | 20260505 08:01:58.241 | INFO | Perf report for the last 5.14097 seconds: | 20260505 08:01:58.241 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:01:58.241 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:58.241 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:58.241 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:58.241 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 08:01:58.241 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:58.241 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:58.241 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 08:01:58.241 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:58.241 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:01:58.241 | INFO | Uploaded 36GiB to GPU in 2.1971s (16.3852GiB/s) | 20260505 08:01:58.241 | INFO | Downloaded 34GiB to GPU in 2.04813s (16.6005GiB/s) | 20260505 08:02:03.380 | INFO | Analyzing chunk frames = [1616896,1617408) | 20260505 08:02:03.380 | INFO | MemInfo: { | 20260505 08:02:03.380 | INFO | "gpu" : " 17 GB", | 20260505 08:02:03.380 | INFO | "huge" : "", | 20260505 08:02:03.380 | INFO | "huge_shared" : " 27 GB", | 20260505 08:02:03.380 | INFO | "malloc" : " 14 GB", | 20260505 08:02:03.380 | INFO | "pinned" : " 29 GB" | 20260505 08:02:03.380 | INFO | } | 20260505 08:02:03.380 | INFO | Perf report for the last 5.13904 seconds: | 20260505 08:02:03.380 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:02:03.380 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:03.380 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:03.380 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:03.380 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 08:02:03.380 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:03.380 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:03.380 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 08:02:03.380 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:03.380 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:03.380 | INFO | Uploaded 36GiB to GPU in 2.19549s (16.3973GiB/s) | 20260505 08:02:03.380 | INFO | Downloaded 34GiB to GPU in 2.15496s (15.7776GiB/s) | 20260505 08:02:08.518 | INFO | Analyzing chunk frames = [1617408,1617920) | 20260505 08:02:08.518 | INFO | MemInfo: { | 20260505 08:02:08.518 | INFO | "gpu" : " 17 GB", | 20260505 08:02:08.518 | INFO | "huge" : "", | 20260505 08:02:08.518 | INFO | "huge_shared" : " 27 GB", | 20260505 08:02:08.518 | INFO | "malloc" : " 14 GB", | 20260505 08:02:08.518 | INFO | "pinned" : " 29 GB" | 20260505 08:02:08.519 | INFO | } | 20260505 08:02:08.519 | INFO | Perf report for the last 5.13842 seconds: | 20260505 08:02:08.519 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:02:08.519 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:08.519 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:08.519 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:08.519 | INFO | ANALYSIS : 64.5%, 3.0, 100.7ms, 0 | 20260505 08:02:08.519 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:08.519 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:08.519 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 08:02:08.519 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:08.519 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:08.519 | INFO | Uploaded 36GiB to GPU in 2.20042s (16.3605GiB/s) | 20260505 08:02:08.519 | INFO | Downloaded 34GiB to GPU in 2.04567s (16.6205GiB/s) | 20260505 08:02:08.661 | INFO | Starting a round of Baz IO | 20260505 08:02:11.938 | INFO | Populating RTMetrics report | 20260505 08:02:11.938 | INFO | RTMetrics report populated | 20260505 08:02:11.938 | INFO | Opening tmp RTMetrics json file | 20260505 08:02:11.960 | INFO | Serializing RTMetrics to json | 20260505 08:02:11.965 | INFO | Serialization done, swapping tmp file now... | 20260505 08:02:11.968 | INFO | RTMetrics done | 20260505 08:02:12.063 | INFO | Finished a round of Baz IO: Wrote 2.444 GiB in 3.402 seconds | 20260505 08:02:12.063 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 28.46% / 0.03% / 71.51% | 20260505 08:02:12.063 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.00544% / 7.54992e-05% / 5.5631e-05% | 20260505 08:02:12.063 | INFO | Baz IO overall: | 20260505 08:02:12.063 | INFO | BytesWritten : 901544719352 | 20260505 08:02:12.063 | INFO | NumEvents : 428264698552 | 20260505 08:02:13.658 | INFO | Analyzing chunk frames = [1617920,1618432) | 20260505 08:02:13.658 | INFO | MemInfo: { | 20260505 08:02:13.658 | INFO | "gpu" : " 17 GB", | 20260505 08:02:13.658 | INFO | "huge" : "", | 20260505 08:02:13.658 | INFO | "huge_shared" : " 27 GB", | 20260505 08:02:13.658 | INFO | "malloc" : " 14 GB", | 20260505 08:02:13.659 | INFO | "pinned" : " 29 GB" | 20260505 08:02:13.659 | INFO | } | 20260505 08:02:13.659 | INFO | Perf report for the last 5.13996 seconds: | 20260505 08:02:13.659 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:02:13.659 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:13.659 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:13.659 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:13.659 | INFO | ANALYSIS : 65.4%, 2.9, 100.1ms, 0 | 20260505 08:02:13.659 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:13.659 | INFO | RT_METRICS : 46.4%, 1.0, 24.4ms, 0 | 20260505 08:02:13.659 | INFO | PRE_HQ : 63.6%, 1.9, 61.7ms, 0 | 20260505 08:02:13.659 | INFO | BAZWRITER : 65.5%, 4.1, 138.8ms, 0 | 20260505 08:02:13.659 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 08:02:13.659 | INFO | Uploaded 36GiB to GPU in 2.27486s (15.8252GiB/s) | 20260505 08:02:13.659 | INFO | Downloaded 38GiB to GPU in 2.39426s (15.8713GiB/s) | 20260505 08:02:18.799 | INFO | Analyzing chunk frames = [1618432,1618944) | 20260505 08:02:18.799 | INFO | MemInfo: { | 20260505 08:02:18.799 | INFO | "gpu" : " 17 GB", | 20260505 08:02:18.799 | INFO | "huge" : "", | 20260505 08:02:18.799 | INFO | "huge_shared" : " 27 GB", | 20260505 08:02:18.799 | INFO | "malloc" : " 14 GB", | 20260505 08:02:18.799 | INFO | "pinned" : " 29 GB" | 20260505 08:02:18.799 | INFO | } | 20260505 08:02:18.799 | INFO | Perf report for the last 5.14004 seconds: | 20260505 08:02:18.799 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:02:18.799 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:18.799 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:18.799 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:18.799 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 08:02:18.799 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:18.799 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:18.799 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 08:02:18.799 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:18.799 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:18.799 | INFO | Uploaded 36GiB to GPU in 2.17048s (16.5862GiB/s) | 20260505 08:02:18.799 | INFO | Downloaded 34GiB to GPU in 2.03886s (16.6759GiB/s) | 20260505 08:02:23.939 | INFO | Analyzing chunk frames = [1618944,1619456) | 20260505 08:02:23.939 | INFO | MemInfo: { | 20260505 08:02:23.939 | INFO | "gpu" : " 17 GB", | 20260505 08:02:23.939 | INFO | "huge" : "", | 20260505 08:02:23.939 | INFO | "huge_shared" : " 27 GB", | 20260505 08:02:23.939 | INFO | "malloc" : " 14 GB", | 20260505 08:02:23.939 | INFO | "pinned" : " 29 GB" | 20260505 08:02:23.939 | INFO | } | 20260505 08:02:23.939 | INFO | Perf report for the last 5.1401 seconds: | 20260505 08:02:23.939 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:02:23.939 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:23.939 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:23.939 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:23.939 | INFO | ANALYSIS : 63.7%, 3.0, 99.2ms, 0 | 20260505 08:02:23.939 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:23.939 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:23.939 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 08:02:23.939 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:23.939 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:23.939 | INFO | Uploaded 36GiB to GPU in 2.17554s (16.5476GiB/s) | 20260505 08:02:23.939 | INFO | Downloaded 34GiB to GPU in 2.06946s (16.4294GiB/s) | 20260505 08:02:29.076 | INFO | Analyzing chunk frames = [1619456,1619968) | 20260505 08:02:29.076 | INFO | MemInfo: { | 20260505 08:02:29.076 | INFO | "gpu" : " 17 GB", | 20260505 08:02:29.076 | INFO | "huge" : "", | 20260505 08:02:29.076 | INFO | "huge_shared" : " 27 GB", | 20260505 08:02:29.076 | INFO | "malloc" : " 14 GB", | 20260505 08:02:29.076 | INFO | "pinned" : " 29 GB" | 20260505 08:02:29.076 | INFO | } | 20260505 08:02:29.076 | INFO | Perf report for the last 5.13725 seconds: | 20260505 08:02:29.076 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:02:29.076 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:29.076 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:29.076 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:29.076 | INFO | ANALYSIS : 63.8%, 3.0, 99.1ms, 0 | 20260505 08:02:29.076 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:29.076 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:29.076 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 08:02:29.076 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:29.076 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:29.076 | INFO | Uploaded 36GiB to GPU in 2.19161s (16.4263GiB/s) | 20260505 08:02:29.076 | INFO | Downloaded 34GiB to GPU in 2.06783s (16.4423GiB/s) | 20260505 08:02:34.216 | INFO | Analyzing chunk frames = [1619968,1620480) | 20260505 08:02:34.216 | INFO | MemInfo: { | 20260505 08:02:34.216 | INFO | "gpu" : " 17 GB", | 20260505 08:02:34.216 | INFO | "huge" : "", | 20260505 08:02:34.216 | INFO | "huge_shared" : " 27 GB", | 20260505 08:02:34.216 | INFO | "malloc" : " 14 GB", | 20260505 08:02:34.216 | INFO | "pinned" : " 29 GB" | 20260505 08:02:34.216 | INFO | } | 20260505 08:02:34.216 | INFO | Perf report for the last 5.14004 seconds: | 20260505 08:02:34.216 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:02:34.216 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:34.216 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:34.216 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:34.216 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 08:02:34.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:34.216 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:34.216 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 08:02:34.216 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:34.216 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:34.216 | INFO | Uploaded 36GiB to GPU in 2.17139s (16.5792GiB/s) | 20260505 08:02:34.216 | INFO | Downloaded 34GiB to GPU in 2.05491s (16.5457GiB/s) | 20260505 08:02:39.359 | INFO | Analyzing chunk frames = [1620480,1620992) | 20260505 08:02:39.359 | INFO | MemInfo: { | 20260505 08:02:39.359 | INFO | "gpu" : " 17 GB", | 20260505 08:02:39.359 | INFO | "huge" : "", | 20260505 08:02:39.359 | INFO | "huge_shared" : " 27 GB", | 20260505 08:02:39.359 | INFO | "malloc" : " 14 GB", | 20260505 08:02:39.359 | INFO | "pinned" : " 29 GB" | 20260505 08:02:39.359 | INFO | } | 20260505 08:02:39.359 | INFO | Perf report for the last 5.14265 seconds: | 20260505 08:02:39.359 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:02:39.359 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:39.359 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:39.359 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:39.359 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 08:02:39.359 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:39.359 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:39.359 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:02:39.359 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:39.359 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:39.359 | INFO | Uploaded 36GiB to GPU in 2.19528s (16.3988GiB/s) | 20260505 08:02:39.359 | INFO | Downloaded 34GiB to GPU in 2.04788s (16.6026GiB/s) | 20260505 08:02:44.494 | INFO | Analyzing chunk frames = [1620992,1621504) | 20260505 08:02:44.494 | INFO | MemInfo: { | 20260505 08:02:44.494 | INFO | "gpu" : " 17 GB", | 20260505 08:02:44.494 | INFO | "huge" : "", | 20260505 08:02:44.494 | INFO | "huge_shared" : " 27 GB", | 20260505 08:02:44.494 | INFO | "malloc" : " 14 GB", | 20260505 08:02:44.494 | INFO | "pinned" : " 29 GB" | 20260505 08:02:44.494 | INFO | } | 20260505 08:02:44.494 | INFO | Perf report for the last 5.1355 seconds: | 20260505 08:02:44.494 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:02:44.494 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:44.494 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:44.494 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:44.494 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 08:02:44.494 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:44.494 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:44.494 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260505 08:02:44.494 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:44.494 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:44.495 | INFO | Uploaded 36GiB to GPU in 2.22337s (16.1917GiB/s) | 20260505 08:02:44.495 | INFO | Downloaded 34GiB to GPU in 2.15327s (15.79GiB/s) | 20260505 08:02:49.633 | INFO | Analyzing chunk frames = [1621504,1622016) | 20260505 08:02:49.633 | INFO | MemInfo: { | 20260505 08:02:49.633 | INFO | "gpu" : " 17 GB", | 20260505 08:02:49.633 | INFO | "huge" : "", | 20260505 08:02:49.633 | INFO | "huge_shared" : " 27 GB", | 20260505 08:02:49.633 | INFO | "malloc" : " 14 GB", | 20260505 08:02:49.633 | INFO | "pinned" : " 29 GB" | 20260505 08:02:49.633 | INFO | } | 20260505 08:02:49.633 | INFO | Perf report for the last 5.13902 seconds: | 20260505 08:02:49.633 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:02:49.633 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:49.633 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:49.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:49.633 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 08:02:49.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:49.633 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:49.633 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 08:02:49.633 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:49.633 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:49.634 | INFO | Uploaded 36GiB to GPU in 2.20159s (16.3519GiB/s) | 20260505 08:02:49.634 | INFO | Downloaded 34GiB to GPU in 2.04203s (16.6501GiB/s) | 20260505 08:02:49.634 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 08:02:49.634 | INFO | Total mem usage: 15.3 GiB | 20260505 08:02:49.634 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 08:02:49.634 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.1%) | 20260505 08:02:49.634 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 08:02:49.634 | INFO | High Water Breakdown: | 20260505 08:02:49.634 | INFO | 60.8% : 8.79 GiB ( 94% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 08:02:49.634 | INFO | 39% : 5.64 GiB ( 84.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 08:02:49.634 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 08:02:49.634 | INFO | ------------------------------------------ | 20260505 08:02:49.634 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:02:49.634 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 08:02:49.634 | INFO | Total mem usage: 26.5 GiB | 20260505 08:02:49.634 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 08:02:49.634 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 08:02:49.634 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 08:02:49.634 | INFO | High Water Breakdown: | 20260505 08:02:49.634 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:02:49.634 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:02:49.634 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:02:49.634 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:02:49.634 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:02:49.634 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:02:49.634 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 08:02:49.634 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 08:02:49.634 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:02:49.634 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 08:02:49.634 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 08:02:49.634 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 08:02:49.634 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 08:02:49.634 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 08:02:49.634 | INFO | ------------------------------------------ | 20260505 08:02:49.634 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:02:49.634 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 08:02:49.634 | INFO | Total mem usage: 25.6 GiB | 20260505 08:02:49.634 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 08:02:49.634 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 08:02:49.634 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 08:02:49.634 | INFO | High Water Breakdown: | 20260505 08:02:49.634 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 08:02:49.634 | INFO | ------------------------------------------ | 20260505 08:02:49.634 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:02:49.634 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 08:02:49.634 | INFO | Total mem usage: 15.7 GiB | 20260505 08:02:49.634 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 08:02:49.634 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 08:02:49.634 | INFO | Unused Cache: 891 MiB (Waste: 5.55%) | 20260505 08:02:49.634 | INFO | High Water Breakdown: | 20260505 08:02:49.634 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 08:02:49.634 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 08:02:49.634 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 08:02:49.634 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 08:02:49.634 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 08:02:49.634 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 08:02:49.634 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 08:02:49.634 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 08:02:49.634 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 08:02:49.634 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:02:49.634 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 08:02:49.634 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:02:49.634 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 08:02:49.634 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 08:02:49.634 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 08:02:49.634 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 08:02:49.634 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:02:49.634 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 08:02:49.634 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:02:49.635 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:02:49.635 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:02:49.635 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 08:02:49.635 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 08:02:49.635 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:02:49.635 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 08:02:49.635 | INFO | ------------------------------------------ | 20260505 08:02:49.635 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:02:49.635 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 08:02:49.635 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:02:49.635 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:02:49.635 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.48e+01, 1.37e+01 / 1.71e+00 | 20260505 08:02:49.635 | INFO | PerfReport: 0.29%, AggregateStats -- 9.35e-02 / 1.05e-01, 1.12e-02 / 4.28e-03 | 20260505 08:02:49.635 | INFO | PerfReport: 11.61%, Baseline -- 3.80e+00 / 4.45e+00, 6.19e-01 / 2.58e-01 | 20260505 08:02:49.635 | INFO | PerfReport: 3.31%, Histogram -- 1.08e+00 / 1.49e+00, 2.12e-01 / 1.17e-01 | 20260505 08:02:49.635 | INFO | PerfReport: 15.49%, DME -- 4.06e+01 / 4.43e+01, 4.15e+00 / 1.26e+00 | 20260505 08:02:49.635 | INFO | PerfReport: 56.45%, FrameLabeling -- 1.85e+01 / 1.92e+01, 1.97e+00 / 1.46e+00 | 20260505 08:02:49.635 | INFO | PerfReport: 7.49%, PulseAccumulating -- 2.45e+00 / 3.11e+00, 4.25e-01 / 2.09e-01 | 20260505 08:02:49.635 | INFO | PerfReport: 3.95%, Metrics -- 1.29e+00 / 1.58e+00, 3.78e-01 / 1.89e-01 | 20260505 08:02:49.635 | INFO | PerfReport: 1.41%, PulseSerializing -- 4.61e-01 / 5.15e-01, 8.85e-02 / 3.50e-02 | 20260505 08:02:49.635 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 08:02:49.635 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:02:49.635 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:02:49.635 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.30e+01 / 0.00e+00 | 20260505 08:02:49.635 | INFO | PerfReport: 22.52%, Upload -- 2.29e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260505 08:02:49.635 | INFO | PerfReport: 21.59%, Download -- 2.20e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260505 08:02:49.635 | INFO | PerfReport: 32.16%, Compute -- 3.27e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 08:02:49.635 | INFO | PerfReport: 1.68%, LockUpload -- 1.71e+00 / 0.00e+00, 7.30e+00 / 0.00e+00 | 20260505 08:02:49.635 | INFO | PerfReport: 0.94%, LockDownload -- 9.61e-01 / 0.00e+00, 2.37e+00 / 0.00e+00 | 20260505 08:02:49.635 | INFO | PerfReport: 21.09%, LockCompute -- 2.15e+01 / 0.00e+00, 1.76e+01 / 0.00e+00 | 20260505 08:02:49.635 | INFO | PerfReport: 0.01%, Misc -- 1.48e-02 / 0.00e+00, 2.28e-02 / 0.00e+00 | 20260505 08:02:49.775 | INFO | Starting a round of Baz IO | 20260505 08:02:53.048 | INFO | Populating RTMetrics report | 20260505 08:02:53.049 | INFO | RTMetrics report populated | 20260505 08:02:53.049 | INFO | Opening tmp RTMetrics json file | 20260505 08:02:53.061 | INFO | Serializing RTMetrics to json | 20260505 08:02:53.074 | INFO | Serialization done, swapping tmp file now... | 20260505 08:02:53.079 | INFO | RTMetrics done | 20260505 08:02:53.178 | INFO | Finished a round of Baz IO: Wrote 2.446 GiB in 3.403 seconds | 20260505 08:02:53.178 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 28.44% / 0.02% / 71.54% | 20260505 08:02:53.178 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.9977% / 5.5631e-05% / 5.5631e-05% | 20260505 08:02:53.178 | INFO | Baz IO overall: | 20260505 08:02:53.178 | INFO | BytesWritten : 904170639841 | 20260505 08:02:53.178 | INFO | NumEvents : 429520991197 | 20260505 08:02:54.773 | INFO | Analyzing chunk frames = [1622016,1622528) | 20260505 08:02:54.773 | INFO | MemInfo: { | 20260505 08:02:54.773 | INFO | "gpu" : " 17 GB", | 20260505 08:02:54.773 | INFO | "huge" : "", | 20260505 08:02:54.773 | INFO | "huge_shared" : " 27 GB", | 20260505 08:02:54.773 | INFO | "malloc" : " 14 GB", | 20260505 08:02:54.773 | INFO | "pinned" : " 29 GB" | 20260505 08:02:54.773 | INFO | } | 20260505 08:02:54.773 | INFO | Perf report for the last 5.13992 seconds: | 20260505 08:02:54.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:02:54.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:54.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:54.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:54.773 | INFO | ANALYSIS : 65.3%, 2.9, 99.9ms, 0 | 20260505 08:02:54.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:54.773 | INFO | RT_METRICS : 46.4%, 1.0, 24.4ms, 0 | 20260505 08:02:54.773 | INFO | PRE_HQ : 63.6%, 1.9, 61.8ms, 0 | 20260505 08:02:54.773 | INFO | BAZWRITER : 65.5%, 4.1, 139.1ms, 0 | 20260505 08:02:54.773 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 08:02:54.774 | INFO | Uploaded 36GiB to GPU in 2.28148s (15.7792GiB/s) | 20260505 08:02:54.774 | INFO | Downloaded 38GiB to GPU in 2.39592s (15.8603GiB/s) | 20260505 08:02:59.913 | INFO | Analyzing chunk frames = [1622528,1623040) | 20260505 08:02:59.913 | INFO | MemInfo: { | 20260505 08:02:59.913 | INFO | "gpu" : " 17 GB", | 20260505 08:02:59.913 | INFO | "huge" : "", | 20260505 08:02:59.913 | INFO | "huge_shared" : " 27 GB", | 20260505 08:02:59.913 | INFO | "malloc" : " 14 GB", | 20260505 08:02:59.913 | INFO | "pinned" : " 29 GB" | 20260505 08:02:59.913 | INFO | } | 20260505 08:02:59.913 | INFO | Perf report for the last 5.13942 seconds: | 20260505 08:02:59.913 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:02:59.913 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:59.913 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:59.913 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:59.913 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 08:02:59.913 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:59.913 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:59.913 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:02:59.913 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:59.913 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:02:59.913 | INFO | Uploaded 36GiB to GPU in 2.17586s (16.5452GiB/s) | 20260505 08:02:59.913 | INFO | Downloaded 34GiB to GPU in 2.0448s (16.6275GiB/s) | 20260505 08:03:05.052 | INFO | Analyzing chunk frames = [1623040,1623552) | 20260505 08:03:05.052 | INFO | MemInfo: { | 20260505 08:03:05.052 | INFO | "gpu" : " 17 GB", | 20260505 08:03:05.052 | INFO | "huge" : "", | 20260505 08:03:05.052 | INFO | "huge_shared" : " 27 GB", | 20260505 08:03:05.052 | INFO | "malloc" : " 14 GB", | 20260505 08:03:05.052 | INFO | "pinned" : " 29 GB" | 20260505 08:03:05.052 | INFO | } | 20260505 08:03:05.052 | INFO | Perf report for the last 5.13972 seconds: | 20260505 08:03:05.052 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:03:05.052 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:05.052 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:05.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:05.052 | INFO | ANALYSIS : 63.7%, 3.0, 99.2ms, 0 | 20260505 08:03:05.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:05.052 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:05.052 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 08:03:05.053 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:05.053 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:05.053 | INFO | Uploaded 36GiB to GPU in 2.17103s (16.582GiB/s) | 20260505 08:03:05.053 | INFO | Downloaded 34GiB to GPU in 2.06599s (16.457GiB/s) | 20260505 08:03:10.191 | INFO | Analyzing chunk frames = [1623552,1624064) | 20260505 08:03:10.191 | INFO | MemInfo: { | 20260505 08:03:10.191 | INFO | "gpu" : " 17 GB", | 20260505 08:03:10.191 | INFO | "huge" : "", | 20260505 08:03:10.191 | INFO | "huge_shared" : " 27 GB", | 20260505 08:03:10.191 | INFO | "malloc" : " 14 GB", | 20260505 08:03:10.191 | INFO | "pinned" : " 29 GB" | 20260505 08:03:10.191 | INFO | } | 20260505 08:03:10.191 | INFO | Perf report for the last 5.13894 seconds: | 20260505 08:03:10.191 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:03:10.191 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:10.191 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:10.191 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:10.191 | INFO | ANALYSIS : 63.7%, 3.0, 99.0ms, 0 | 20260505 08:03:10.191 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:10.191 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:10.191 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:03:10.191 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:10.192 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:10.192 | INFO | Uploaded 36GiB to GPU in 2.18751s (16.4571GiB/s) | 20260505 08:03:10.192 | INFO | Downloaded 34GiB to GPU in 2.06733s (16.4464GiB/s) | 20260505 08:03:15.331 | INFO | Analyzing chunk frames = [1624064,1624576) | 20260505 08:03:15.331 | INFO | MemInfo: { | 20260505 08:03:15.331 | INFO | "gpu" : " 17 GB", | 20260505 08:03:15.331 | INFO | "huge" : "", | 20260505 08:03:15.331 | INFO | "huge_shared" : " 27 GB", | 20260505 08:03:15.331 | INFO | "malloc" : " 14 GB", | 20260505 08:03:15.331 | INFO | "pinned" : " 29 GB" | 20260505 08:03:15.331 | INFO | } | 20260505 08:03:15.331 | INFO | Perf report for the last 5.13994 seconds: | 20260505 08:03:15.331 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:03:15.331 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:15.331 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:15.331 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:15.331 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 08:03:15.331 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:15.331 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:15.331 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:03:15.331 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:15.331 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:15.332 | INFO | Uploaded 36GiB to GPU in 2.20001s (16.3635GiB/s) | 20260505 08:03:15.332 | INFO | Downloaded 34GiB to GPU in 2.05535s (16.5422GiB/s) | 20260505 08:03:20.470 | INFO | Analyzing chunk frames = [1624576,1625088) | 20260505 08:03:20.470 | INFO | MemInfo: { | 20260505 08:03:20.470 | INFO | "gpu" : " 17 GB", | 20260505 08:03:20.470 | INFO | "huge" : "", | 20260505 08:03:20.470 | INFO | "huge_shared" : " 27 GB", | 20260505 08:03:20.470 | INFO | "malloc" : " 14 GB", | 20260505 08:03:20.470 | INFO | "pinned" : " 29 GB" | 20260505 08:03:20.470 | INFO | } | 20260505 08:03:20.470 | INFO | Perf report for the last 5.13892 seconds: | 20260505 08:03:20.470 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:03:20.470 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:20.470 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:20.470 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:20.470 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 08:03:20.470 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:20.470 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:20.470 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:03:20.470 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:20.470 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:20.470 | INFO | Uploaded 36GiB to GPU in 2.17588s (16.545GiB/s) | 20260505 08:03:20.471 | INFO | Downloaded 34GiB to GPU in 2.04306s (16.6417GiB/s) | 20260505 08:03:25.608 | INFO | Analyzing chunk frames = [1625088,1625600) | 20260505 08:03:25.608 | INFO | MemInfo: { | 20260505 08:03:25.608 | INFO | "gpu" : " 17 GB", | 20260505 08:03:25.608 | INFO | "huge" : "", | 20260505 08:03:25.608 | INFO | "huge_shared" : " 27 GB", | 20260505 08:03:25.608 | INFO | "malloc" : " 14 GB", | 20260505 08:03:25.608 | INFO | "pinned" : " 29 GB" | 20260505 08:03:25.608 | INFO | } | 20260505 08:03:25.608 | INFO | Perf report for the last 5.1382 seconds: | 20260505 08:03:25.608 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:03:25.608 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:25.608 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:25.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:25.609 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 08:03:25.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:25.609 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:25.609 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 08:03:25.609 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:25.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:25.609 | INFO | Uploaded 36GiB to GPU in 2.19698s (16.3861GiB/s) | 20260505 08:03:25.609 | INFO | Downloaded 34GiB to GPU in 2.15675s (15.7644GiB/s) | 20260505 08:03:30.749 | INFO | Analyzing chunk frames = [1625600,1626112) | 20260505 08:03:30.749 | INFO | MemInfo: { | 20260505 08:03:30.749 | INFO | "gpu" : " 17 GB", | 20260505 08:03:30.749 | INFO | "huge" : "", | 20260505 08:03:30.749 | INFO | "huge_shared" : " 27 GB", | 20260505 08:03:30.749 | INFO | "malloc" : " 14 GB", | 20260505 08:03:30.749 | INFO | "pinned" : " 29 GB" | 20260505 08:03:30.749 | INFO | } | 20260505 08:03:30.749 | INFO | Perf report for the last 5.14022 seconds: | 20260505 08:03:30.749 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:03:30.749 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:30.749 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:30.749 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:30.749 | INFO | ANALYSIS : 64.3%, 3.0, 100.5ms, 0 | 20260505 08:03:30.749 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:30.749 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:30.749 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 08:03:30.749 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:30.749 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:30.749 | INFO | Uploaded 36GiB to GPU in 2.19704s (16.3857GiB/s) | 20260505 08:03:30.749 | INFO | Downloaded 34GiB to GPU in 2.04073s (16.6607GiB/s) | 20260505 08:03:30.882 | INFO | Starting a round of Baz IO | 20260505 08:03:34.168 | INFO | Populating RTMetrics report | 20260505 08:03:34.169 | INFO | RTMetrics report populated | 20260505 08:03:34.169 | INFO | Opening tmp RTMetrics json file | 20260505 08:03:34.169 | INFO | Serializing RTMetrics to json | 20260505 08:03:34.171 | INFO | Serialization done, swapping tmp file now... | 20260505 08:03:34.171 | INFO | RTMetrics done | 20260505 08:03:34.339 | INFO | Finished a round of Baz IO: Wrote 2.444 GiB in 3.457 seconds | 20260505 08:03:34.339 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 28.41% / 0.02% / 71.56% | 20260505 08:03:34.339 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.98132% / 3.57628e-05% / 7.94729e-05% | 20260505 08:03:34.339 | INFO | Baz IO overall: | 20260505 08:03:34.339 | INFO | BytesWritten : 906795222031 | 20260505 08:03:34.339 | INFO | NumEvents : 430776437568 | 20260505 08:03:35.888 | INFO | Analyzing chunk frames = [1626112,1626624) | 20260505 08:03:35.888 | INFO | MemInfo: { | 20260505 08:03:35.888 | INFO | "gpu" : " 17 GB", | 20260505 08:03:35.888 | INFO | "huge" : "", | 20260505 08:03:35.888 | INFO | "huge_shared" : " 27 GB", | 20260505 08:03:35.889 | INFO | "malloc" : " 14 GB", | 20260505 08:03:35.889 | INFO | "pinned" : " 29 GB" | 20260505 08:03:35.889 | INFO | } | 20260505 08:03:35.889 | INFO | Perf report for the last 5.13988 seconds: | 20260505 08:03:35.889 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:03:35.889 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:35.889 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:35.889 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:35.889 | INFO | ANALYSIS : 65.4%, 2.9, 100.0ms, 0 | 20260505 08:03:35.889 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:35.889 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260505 08:03:35.889 | INFO | PRE_HQ : 63.7%, 1.9, 61.6ms, 0 | 20260505 08:03:35.889 | INFO | BAZWRITER : 66.5%, 4.2, 141.9ms, 0 | 20260505 08:03:35.889 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 08:03:35.889 | INFO | Uploaded 36GiB to GPU in 2.27277s (15.8397GiB/s) | 20260505 08:03:35.889 | INFO | Downloaded 38GiB to GPU in 2.37607s (15.9928GiB/s) | 20260505 08:03:41.027 | INFO | Analyzing chunk frames = [1626624,1627136) | 20260505 08:03:41.027 | INFO | MemInfo: { | 20260505 08:03:41.027 | INFO | "gpu" : " 17 GB", | 20260505 08:03:41.027 | INFO | "huge" : "", | 20260505 08:03:41.027 | INFO | "huge_shared" : " 27 GB", | 20260505 08:03:41.027 | INFO | "malloc" : " 14 GB", | 20260505 08:03:41.027 | INFO | "pinned" : " 29 GB" | 20260505 08:03:41.027 | INFO | } | 20260505 08:03:41.027 | INFO | Perf report for the last 5.13834 seconds: | 20260505 08:03:41.027 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:03:41.027 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:41.027 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:41.027 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:41.027 | INFO | ANALYSIS : 64.8%, 3.0, 101.1ms, 0 | 20260505 08:03:41.027 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:41.027 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:41.027 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 08:03:41.027 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:41.027 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:41.027 | INFO | Uploaded 36GiB to GPU in 2.24392s (16.0433GiB/s) | 20260505 08:03:41.027 | INFO | Downloaded 34GiB to GPU in 2.11106s (16.1056GiB/s) | 20260505 08:03:46.166 | INFO | Analyzing chunk frames = [1627136,1627648) | 20260505 08:03:46.166 | INFO | MemInfo: { | 20260505 08:03:46.166 | INFO | "gpu" : " 17 GB", | 20260505 08:03:46.166 | INFO | "huge" : "", | 20260505 08:03:46.166 | INFO | "huge_shared" : " 27 GB", | 20260505 08:03:46.166 | INFO | "malloc" : " 14 GB", | 20260505 08:03:46.166 | INFO | "pinned" : " 29 GB" | 20260505 08:03:46.166 | INFO | } | 20260505 08:03:46.166 | INFO | Perf report for the last 5.13888 seconds: | 20260505 08:03:46.166 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:03:46.166 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:46.166 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:46.166 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:46.166 | INFO | ANALYSIS : 63.9%, 3.0, 99.4ms, 0 | 20260505 08:03:46.166 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:46.166 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:46.166 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 08:03:46.166 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:46.166 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:46.166 | INFO | Uploaded 36GiB to GPU in 2.17553s (16.5477GiB/s) | 20260505 08:03:46.166 | INFO | Downloaded 34GiB to GPU in 2.06822s (16.4392GiB/s) | 20260505 08:03:51.307 | INFO | Analyzing chunk frames = [1627648,1628160) | 20260505 08:03:51.307 | INFO | MemInfo: { | 20260505 08:03:51.307 | INFO | "gpu" : " 17 GB", | 20260505 08:03:51.307 | INFO | "huge" : "", | 20260505 08:03:51.307 | INFO | "huge_shared" : " 27 GB", | 20260505 08:03:51.307 | INFO | "malloc" : " 14 GB", | 20260505 08:03:51.307 | INFO | "pinned" : " 29 GB" | 20260505 08:03:51.307 | INFO | } | 20260505 08:03:51.307 | INFO | Perf report for the last 5.14159 seconds: | 20260505 08:03:51.307 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:03:51.307 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:51.307 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:51.308 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:51.308 | INFO | ANALYSIS : 63.8%, 3.0, 99.2ms, 0 | 20260505 08:03:51.308 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:51.308 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:51.308 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 08:03:51.308 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:51.308 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:51.308 | INFO | Uploaded 36GiB to GPU in 2.19663s (16.3887GiB/s) | 20260505 08:03:51.308 | INFO | Downloaded 34GiB to GPU in 2.06496s (16.4652GiB/s) | 20260505 08:03:56.445 | INFO | Analyzing chunk frames = [1628160,1628672) | 20260505 08:03:56.445 | INFO | MemInfo: { | 20260505 08:03:56.445 | INFO | "gpu" : " 17 GB", | 20260505 08:03:56.445 | INFO | "huge" : "", | 20260505 08:03:56.445 | INFO | "huge_shared" : " 27 GB", | 20260505 08:03:56.445 | INFO | "malloc" : " 14 GB", | 20260505 08:03:56.445 | INFO | "pinned" : " 29 GB" | 20260505 08:03:56.445 | INFO | } | 20260505 08:03:56.446 | INFO | Perf report for the last 5.13807 seconds: | 20260505 08:03:56.446 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:03:56.446 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:56.446 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:56.446 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:56.446 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 08:03:56.446 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:56.446 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:56.446 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 08:03:56.446 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:56.446 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:03:56.446 | INFO | Uploaded 36GiB to GPU in 2.18054s (16.5097GiB/s) | 20260505 08:03:56.446 | INFO | Downloaded 34GiB to GPU in 2.05606s (16.5365GiB/s) | 20260505 08:04:01.585 | INFO | Analyzing chunk frames = [1628672,1629184) | 20260505 08:04:01.585 | INFO | MemInfo: { | 20260505 08:04:01.585 | INFO | "gpu" : " 17 GB", | 20260505 08:04:01.585 | INFO | "huge" : "", | 20260505 08:04:01.585 | INFO | "huge_shared" : " 27 GB", | 20260505 08:04:01.585 | INFO | "malloc" : " 14 GB", | 20260505 08:04:01.585 | INFO | "pinned" : " 29 GB" | 20260505 08:04:01.585 | INFO | } | 20260505 08:04:01.585 | INFO | Perf report for the last 5.13938 seconds: | 20260505 08:04:01.585 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:04:01.585 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:01.585 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:01.585 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:01.585 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 08:04:01.585 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:01.585 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:01.585 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 08:04:01.585 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:01.585 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:01.585 | INFO | Uploaded 36GiB to GPU in 2.17179s (16.5762GiB/s) | 20260505 08:04:01.585 | INFO | Downloaded 34GiB to GPU in 2.04912s (16.5925GiB/s) | 20260505 08:04:06.723 | INFO | Analyzing chunk frames = [1629184,1629696) | 20260505 08:04:06.723 | INFO | MemInfo: { | 20260505 08:04:06.723 | INFO | "gpu" : " 17 GB", | 20260505 08:04:06.723 | INFO | "huge" : "", | 20260505 08:04:06.723 | INFO | "huge_shared" : " 27 GB", | 20260505 08:04:06.723 | INFO | "malloc" : " 14 GB", | 20260505 08:04:06.723 | INFO | "pinned" : " 29 GB" | 20260505 08:04:06.723 | INFO | } | 20260505 08:04:06.723 | INFO | Perf report for the last 5.13817 seconds: | 20260505 08:04:06.723 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:04:06.723 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:06.723 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:06.723 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:06.723 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 08:04:06.723 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:06.723 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:06.723 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 08:04:06.723 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:06.723 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:06.724 | INFO | Uploaded 36GiB to GPU in 2.19243s (16.4201GiB/s) | 20260505 08:04:06.724 | INFO | Downloaded 34GiB to GPU in 2.15158s (15.8023GiB/s) | 20260505 08:04:11.863 | INFO | Analyzing chunk frames = [1629696,1630208) | 20260505 08:04:11.863 | INFO | MemInfo: { | 20260505 08:04:11.863 | INFO | "gpu" : " 17 GB", | 20260505 08:04:11.863 | INFO | "huge" : "", | 20260505 08:04:11.863 | INFO | "huge_shared" : " 27 GB", | 20260505 08:04:11.863 | INFO | "malloc" : " 14 GB", | 20260505 08:04:11.863 | INFO | "pinned" : " 29 GB" | 20260505 08:04:11.863 | INFO | } | 20260505 08:04:11.863 | INFO | Perf report for the last 5.14026 seconds: | 20260505 08:04:11.864 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:04:11.864 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:11.864 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:11.864 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:11.864 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 08:04:11.864 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:11.864 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:11.864 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 08:04:11.864 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:11.864 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:11.864 | INFO | Uploaded 36GiB to GPU in 2.19293s (16.4164GiB/s) | 20260505 08:04:11.864 | INFO | Downloaded 34GiB to GPU in 2.04541s (16.6226GiB/s) | 20260505 08:04:11.864 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 08:04:11.864 | INFO | Total mem usage: 15.3 GiB | 20260505 08:04:11.864 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 08:04:11.864 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.26%) | 20260505 08:04:11.864 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 08:04:11.864 | INFO | High Water Breakdown: | 20260505 08:04:11.864 | INFO | 60.8% : 8.77 GiB ( 93.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 08:04:11.864 | INFO | 39.1% : 5.64 GiB ( 84.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 08:04:11.864 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 08:04:11.864 | INFO | ------------------------------------------ | 20260505 08:04:11.864 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:04:11.864 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 08:04:11.864 | INFO | Total mem usage: 26.5 GiB | 20260505 08:04:11.864 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 08:04:11.864 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 08:04:11.864 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 08:04:11.864 | INFO | High Water Breakdown: | 20260505 08:04:11.864 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:04:11.864 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:04:11.864 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:04:11.864 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:04:11.864 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:04:11.864 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:04:11.864 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 08:04:11.864 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 08:04:11.864 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:04:11.864 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 08:04:11.864 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 08:04:11.864 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 08:04:11.864 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 08:04:11.864 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 08:04:11.864 | INFO | ------------------------------------------ | 20260505 08:04:11.864 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:04:11.864 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 08:04:11.864 | INFO | Total mem usage: 25.6 GiB | 20260505 08:04:11.865 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 08:04:11.865 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 08:04:11.865 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 08:04:11.865 | INFO | High Water Breakdown: | 20260505 08:04:11.865 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 08:04:11.865 | INFO | ------------------------------------------ | 20260505 08:04:11.865 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:04:11.865 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 08:04:11.865 | INFO | Total mem usage: 15.7 GiB | 20260505 08:04:11.865 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 08:04:11.865 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 08:04:11.865 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 08:04:11.865 | INFO | High Water Breakdown: | 20260505 08:04:11.865 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 08:04:11.865 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 08:04:11.865 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 08:04:11.865 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 08:04:11.865 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 08:04:11.865 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 08:04:11.865 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 08:04:11.865 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 08:04:11.865 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 08:04:11.865 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:04:11.865 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 08:04:11.865 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:04:11.865 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 08:04:11.865 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 08:04:11.865 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 08:04:11.865 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 08:04:11.865 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:04:11.865 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 08:04:11.865 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:04:11.865 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:04:11.865 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:04:11.865 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 08:04:11.865 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 08:04:11.865 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:04:11.865 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 08:04:11.865 | INFO | ------------------------------------------ | 20260505 08:04:11.865 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:04:11.865 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 08:04:11.865 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:04:11.865 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:04:11.865 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.50e+01, 1.35e+01 / 1.80e+00 | 20260505 08:04:11.865 | INFO | PerfReport: 0.29%, AggregateStats -- 9.33e-02 / 1.04e-01, 1.07e-02 / 4.24e-03 | 20260505 08:04:11.865 | INFO | PerfReport: 11.62%, Baseline -- 3.80e+00 / 4.44e+00, 6.18e-01 / 2.91e-01 | 20260505 08:04:11.865 | INFO | PerfReport: 3.31%, Histogram -- 1.08e+00 / 1.47e+00, 2.10e-01 / 1.17e-01 | 20260505 08:04:11.865 | INFO | PerfReport: 15.50%, DME -- 4.06e+01 / 4.43e+01, 4.04e+00 / 1.19e+00 | 20260505 08:04:11.865 | INFO | PerfReport: 56.43%, FrameLabeling -- 1.85e+01 / 1.94e+01, 1.96e+00 / 1.47e+00 | 20260505 08:04:11.865 | INFO | PerfReport: 7.49%, PulseAccumulating -- 2.45e+00 / 3.10e+00, 4.17e-01 / 2.11e-01 | 20260505 08:04:11.865 | INFO | PerfReport: 3.95%, Metrics -- 1.29e+00 / 1.60e+00, 3.67e-01 / 1.99e-01 | 20260505 08:04:11.865 | INFO | PerfReport: 1.41%, PulseSerializing -- 4.61e-01 / 5.15e-01, 8.82e-02 / 3.30e-02 | 20260505 08:04:11.865 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 08:04:11.865 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:04:11.865 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:04:11.865 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.31e+01 / 0.00e+00 | 20260505 08:04:11.865 | INFO | PerfReport: 22.52%, Upload -- 2.29e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260505 08:04:11.865 | INFO | PerfReport: 21.62%, Download -- 2.20e+01 / 0.00e+00, 8.16e+00 / 0.00e+00 | 20260505 08:04:11.865 | INFO | PerfReport: 32.14%, Compute -- 3.27e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 08:04:11.865 | INFO | PerfReport: 1.70%, LockUpload -- 1.73e+00 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260505 08:04:11.865 | INFO | PerfReport: 0.96%, LockDownload -- 9.74e-01 / 0.00e+00, 2.41e+00 / 0.00e+00 | 20260505 08:04:11.865 | INFO | PerfReport: 21.05%, LockCompute -- 2.14e+01 / 0.00e+00, 1.78e+01 / 0.00e+00 | 20260505 08:04:11.865 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.97e-02 / 0.00e+00 | 20260505 08:04:11.997 | INFO | Starting a round of Baz IO | 20260505 08:04:15.302 | INFO | Populating RTMetrics report | 20260505 08:04:15.302 | INFO | RTMetrics report populated | 20260505 08:04:15.302 | INFO | Opening tmp RTMetrics json file | 20260505 08:04:15.314 | INFO | Serializing RTMetrics to json | 20260505 08:04:15.337 | INFO | Serialization done, swapping tmp file now... | 20260505 08:04:15.353 | INFO | RTMetrics done | 20260505 08:04:15.425 | INFO | Finished a round of Baz IO: Wrote 2.444 GiB in 3.428 seconds | 20260505 08:04:15.425 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 28.39% / 0.02% / 71.59% | 20260505 08:04:15.425 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.9703% / 2.38419e-05% / 5.5631e-05% | 20260505 08:04:15.425 | INFO | Baz IO overall: | 20260505 08:04:15.425 | INFO | BytesWritten : 909419441751 | 20260505 08:04:15.425 | INFO | NumEvents : 432031699037 | 20260505 08:04:17.002 | INFO | Analyzing chunk frames = [1630208,1630720) | 20260505 08:04:17.002 | INFO | MemInfo: { | 20260505 08:04:17.002 | INFO | "gpu" : " 17 GB", | 20260505 08:04:17.002 | INFO | "huge" : "", | 20260505 08:04:17.002 | INFO | "huge_shared" : " 27 GB", | 20260505 08:04:17.002 | INFO | "malloc" : " 14 GB", | 20260505 08:04:17.002 | INFO | "pinned" : " 29 GB" | 20260505 08:04:17.002 | INFO | } | 20260505 08:04:17.002 | INFO | Perf report for the last 5.13896 seconds: | 20260505 08:04:17.002 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:04:17.002 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:17.002 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:17.002 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:17.003 | INFO | ANALYSIS : 65.8%, 2.9, 100.6ms, 0 | 20260505 08:04:17.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:17.003 | INFO | RT_METRICS : 47.0%, 1.0, 24.7ms, 0 | 20260505 08:04:17.003 | INFO | PRE_HQ : 63.8%, 1.9, 62.0ms, 0 | 20260505 08:04:17.003 | INFO | BAZWRITER : 65.9%, 4.1, 138.5ms, 0 | 20260505 08:04:17.003 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 08:04:17.003 | INFO | Uploaded 36GiB to GPU in 2.26575s (15.8888GiB/s) | 20260505 08:04:17.003 | INFO | Downloaded 38GiB to GPU in 2.34558s (16.2007GiB/s) | 20260505 08:04:22.142 | INFO | Analyzing chunk frames = [1630720,1631232) | 20260505 08:04:22.142 | INFO | MemInfo: { | 20260505 08:04:22.142 | INFO | "gpu" : " 17 GB", | 20260505 08:04:22.142 | INFO | "huge" : "", | 20260505 08:04:22.142 | INFO | "huge_shared" : " 27 GB", | 20260505 08:04:22.142 | INFO | "malloc" : " 14 GB", | 20260505 08:04:22.142 | INFO | "pinned" : " 29 GB" | 20260505 08:04:22.142 | INFO | } | 20260505 08:04:22.142 | INFO | Perf report for the last 5.13947 seconds: | 20260505 08:04:22.142 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:04:22.142 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:22.142 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:22.142 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:22.142 | INFO | ANALYSIS : 64.7%, 3.0, 101.0ms, 0 | 20260505 08:04:22.142 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:22.142 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:22.142 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 08:04:22.142 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:22.142 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:22.142 | INFO | Uploaded 36GiB to GPU in 2.24055s (16.0675GiB/s) | 20260505 08:04:22.142 | INFO | Downloaded 34GiB to GPU in 2.11769s (16.0552GiB/s) | 20260505 08:04:27.281 | INFO | Analyzing chunk frames = [1631232,1631744) | 20260505 08:04:27.281 | INFO | MemInfo: { | 20260505 08:04:27.281 | INFO | "gpu" : " 17 GB", | 20260505 08:04:27.281 | INFO | "huge" : "", | 20260505 08:04:27.281 | INFO | "huge_shared" : " 27 GB", | 20260505 08:04:27.281 | INFO | "malloc" : " 14 GB", | 20260505 08:04:27.281 | INFO | "pinned" : " 29 GB" | 20260505 08:04:27.281 | INFO | } | 20260505 08:04:27.281 | INFO | Perf report for the last 5.13933 seconds: | 20260505 08:04:27.281 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:04:27.281 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:27.281 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:27.281 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:27.281 | INFO | ANALYSIS : 63.8%, 3.0, 99.3ms, 0 | 20260505 08:04:27.281 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:27.282 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:27.282 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 08:04:27.282 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:27.282 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:27.282 | INFO | Uploaded 36GiB to GPU in 2.16971s (16.5921GiB/s) | 20260505 08:04:27.282 | INFO | Downloaded 34GiB to GPU in 2.06363s (16.4758GiB/s) | 20260505 08:04:32.420 | INFO | Analyzing chunk frames = [1631744,1632256) | 20260505 08:04:32.420 | INFO | MemInfo: { | 20260505 08:04:32.420 | INFO | "gpu" : " 17 GB", | 20260505 08:04:32.420 | INFO | "huge" : "", | 20260505 08:04:32.420 | INFO | "huge_shared" : " 27 GB", | 20260505 08:04:32.420 | INFO | "malloc" : " 14 GB", | 20260505 08:04:32.421 | INFO | "pinned" : " 29 GB" | 20260505 08:04:32.421 | INFO | } | 20260505 08:04:32.421 | INFO | Perf report for the last 5.13937 seconds: | 20260505 08:04:32.421 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:04:32.421 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:32.421 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:32.421 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:32.421 | INFO | ANALYSIS : 63.8%, 3.0, 99.1ms, 0 | 20260505 08:04:32.421 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:32.421 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:32.421 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:04:32.421 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:32.421 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:32.421 | INFO | Uploaded 36GiB to GPU in 2.19212s (16.4224GiB/s) | 20260505 08:04:32.421 | INFO | Downloaded 34GiB to GPU in 2.06402s (16.4727GiB/s) | 20260505 08:04:37.560 | INFO | Analyzing chunk frames = [1632256,1632768) | 20260505 08:04:37.560 | INFO | MemInfo: { | 20260505 08:04:37.560 | INFO | "gpu" : " 17 GB", | 20260505 08:04:37.560 | INFO | "huge" : "", | 20260505 08:04:37.560 | INFO | "huge_shared" : " 27 GB", | 20260505 08:04:37.560 | INFO | "malloc" : " 14 GB", | 20260505 08:04:37.560 | INFO | "pinned" : " 29 GB" | 20260505 08:04:37.560 | INFO | } | 20260505 08:04:37.560 | INFO | Perf report for the last 5.13938 seconds: | 20260505 08:04:37.560 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:04:37.560 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:37.560 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:37.560 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:37.560 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 08:04:37.560 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:37.560 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:37.560 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 08:04:37.560 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:37.560 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:37.560 | INFO | Uploaded 36GiB to GPU in 2.20442s (16.3308GiB/s) | 20260505 08:04:37.560 | INFO | Downloaded 34GiB to GPU in 2.05194s (16.5697GiB/s) | 20260505 08:04:42.700 | INFO | Analyzing chunk frames = [1632768,1633280) | 20260505 08:04:42.700 | INFO | MemInfo: { | 20260505 08:04:42.700 | INFO | "gpu" : " 17 GB", | 20260505 08:04:42.700 | INFO | "huge" : "", | 20260505 08:04:42.700 | INFO | "huge_shared" : " 27 GB", | 20260505 08:04:42.700 | INFO | "malloc" : " 14 GB", | 20260505 08:04:42.700 | INFO | "pinned" : " 29 GB" | 20260505 08:04:42.700 | INFO | } | 20260505 08:04:42.700 | INFO | Perf report for the last 5.14028 seconds: | 20260505 08:04:42.700 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:04:42.700 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:42.700 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:42.700 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:42.700 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 08:04:42.700 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:42.700 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:42.700 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 08:04:42.700 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:42.700 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:42.700 | INFO | Uploaded 36GiB to GPU in 2.17584s (16.5453GiB/s) | 20260505 08:04:42.700 | INFO | Downloaded 34GiB to GPU in 2.04372s (16.6363GiB/s) | 20260505 08:04:47.838 | INFO | Analyzing chunk frames = [1633280,1633792) | 20260505 08:04:47.838 | INFO | MemInfo: { | 20260505 08:04:47.838 | INFO | "gpu" : " 17 GB", | 20260505 08:04:47.838 | INFO | "huge" : "", | 20260505 08:04:47.838 | INFO | "huge_shared" : " 27 GB", | 20260505 08:04:47.838 | INFO | "malloc" : " 14 GB", | 20260505 08:04:47.838 | INFO | "pinned" : " 29 GB" | 20260505 08:04:47.838 | INFO | } | 20260505 08:04:47.838 | INFO | Perf report for the last 5.13796 seconds: | 20260505 08:04:47.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:04:47.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:47.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:47.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:47.838 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 08:04:47.838 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:47.838 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:47.838 | INFO | PRE_HQ : 31.9%, 1.0, 16.5ms, 0 | 20260505 08:04:47.838 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:47.838 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:47.838 | INFO | Uploaded 36GiB to GPU in 2.21043s (16.2864GiB/s) | 20260505 08:04:47.838 | INFO | Downloaded 34GiB to GPU in 2.16445s (15.7084GiB/s) | 20260505 08:04:52.978 | INFO | Analyzing chunk frames = [1633792,1634304) | 20260505 08:04:52.978 | INFO | MemInfo: { | 20260505 08:04:52.978 | INFO | "gpu" : " 17 GB", | 20260505 08:04:52.978 | INFO | "huge" : "", | 20260505 08:04:52.978 | INFO | "huge_shared" : " 27 GB", | 20260505 08:04:52.978 | INFO | "malloc" : " 14 GB", | 20260505 08:04:52.978 | INFO | "pinned" : " 29 GB" | 20260505 08:04:52.978 | INFO | } | 20260505 08:04:52.978 | INFO | Perf report for the last 5.13953 seconds: | 20260505 08:04:52.978 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:04:52.978 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:52.978 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:52.978 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:52.978 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 08:04:52.978 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:52.978 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:52.978 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 08:04:52.978 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:52.978 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:52.978 | INFO | Uploaded 36GiB to GPU in 2.20411s (16.3332GiB/s) | 20260505 08:04:52.978 | INFO | Downloaded 34GiB to GPU in 2.04924s (16.5915GiB/s) | 20260505 08:04:53.119 | INFO | Starting a round of Baz IO | 20260505 08:04:56.409 | INFO | Populating RTMetrics report | 20260505 08:04:56.409 | INFO | RTMetrics report populated | 20260505 08:04:56.409 | INFO | Opening tmp RTMetrics json file | 20260505 08:04:56.411 | INFO | Serializing RTMetrics to json | 20260505 08:04:56.414 | INFO | Serialization done, swapping tmp file now... | 20260505 08:04:56.415 | INFO | RTMetrics done | 20260505 08:04:56.633 | INFO | Finished a round of Baz IO: Wrote 2.446 GiB in 3.513 seconds | 20260505 08:04:56.633 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 28.36% / 0.02% / 71.61% | 20260505 08:04:56.633 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.95901% / 4.37101e-05% / 0.000103315% | 20260505 08:04:56.633 | INFO | Baz IO overall: | 20260505 08:04:56.633 | INFO | BytesWritten : 912045754615 | 20260505 08:04:56.633 | INFO | NumEvents : 433287566114 | 20260505 08:04:58.117 | INFO | Analyzing chunk frames = [1634304,1634816) | 20260505 08:04:58.117 | INFO | MemInfo: { | 20260505 08:04:58.117 | INFO | "gpu" : " 17 GB", | 20260505 08:04:58.117 | INFO | "huge" : "", | 20260505 08:04:58.117 | INFO | "huge_shared" : " 27 GB", | 20260505 08:04:58.117 | INFO | "malloc" : " 14 GB", | 20260505 08:04:58.117 | INFO | "pinned" : " 29 GB" | 20260505 08:04:58.117 | INFO | } | 20260505 08:04:58.117 | INFO | Perf report for the last 5.13892 seconds: | 20260505 08:04:58.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:04:58.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:58.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:58.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:58.117 | INFO | ANALYSIS : 65.6%, 2.9, 98.7ms, 0 | 20260505 08:04:58.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:04:58.117 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 08:04:58.117 | INFO | PRE_HQ : 63.8%, 1.9, 61.5ms, 0 | 20260505 08:04:58.117 | INFO | BAZWRITER : 67.7%, 4.3, 149.6ms, 0 | 20260505 08:04:58.117 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 08:04:58.117 | INFO | Uploaded 36GiB to GPU in 2.28607s (15.7476GiB/s) | 20260505 08:04:58.117 | INFO | Downloaded 38GiB to GPU in 2.37921s (15.9717GiB/s) | 20260505 08:05:03.257 | INFO | Analyzing chunk frames = [1634816,1635328) | 20260505 08:05:03.257 | INFO | MemInfo: { | 20260505 08:05:03.257 | INFO | "gpu" : " 17 GB", | 20260505 08:05:03.258 | INFO | "huge" : "", | 20260505 08:05:03.258 | INFO | "huge_shared" : " 27 GB", | 20260505 08:05:03.258 | INFO | "malloc" : " 14 GB", | 20260505 08:05:03.258 | INFO | "pinned" : " 29 GB" | 20260505 08:05:03.258 | INFO | } | 20260505 08:05:03.258 | INFO | Perf report for the last 5.14092 seconds: | 20260505 08:05:03.258 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:05:03.258 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:03.258 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:03.258 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:03.258 | INFO | ANALYSIS : 64.1%, 3.0, 100.2ms, 0 | 20260505 08:05:03.258 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:03.258 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:03.258 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:05:03.258 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:03.258 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:03.258 | INFO | Uploaded 36GiB to GPU in 2.18194s (16.499GiB/s) | 20260505 08:05:03.258 | INFO | Downloaded 34GiB to GPU in 2.04515s (16.6247GiB/s) | 20260505 08:05:08.396 | INFO | Analyzing chunk frames = [1635328,1635840) | 20260505 08:05:08.396 | INFO | MemInfo: { | 20260505 08:05:08.396 | INFO | "gpu" : " 17 GB", | 20260505 08:05:08.396 | INFO | "huge" : "", | 20260505 08:05:08.396 | INFO | "huge_shared" : " 27 GB", | 20260505 08:05:08.396 | INFO | "malloc" : " 14 GB", | 20260505 08:05:08.396 | INFO | "pinned" : " 29 GB" | 20260505 08:05:08.396 | INFO | } | 20260505 08:05:08.396 | INFO | Perf report for the last 5.13836 seconds: | 20260505 08:05:08.396 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:05:08.396 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:08.396 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:08.396 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:08.396 | INFO | ANALYSIS : 63.8%, 3.0, 99.2ms, 0 | 20260505 08:05:08.396 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:08.396 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:08.396 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 08:05:08.396 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:08.396 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:08.396 | INFO | Uploaded 36GiB to GPU in 2.18352s (16.4872GiB/s) | 20260505 08:05:08.396 | INFO | Downloaded 34GiB to GPU in 2.07705s (16.3693GiB/s) | 20260505 08:05:13.535 | INFO | Analyzing chunk frames = [1635840,1636352) | 20260505 08:05:13.535 | INFO | MemInfo: { | 20260505 08:05:13.535 | INFO | "gpu" : " 17 GB", | 20260505 08:05:13.535 | INFO | "huge" : "", | 20260505 08:05:13.535 | INFO | "huge_shared" : " 27 GB", | 20260505 08:05:13.535 | INFO | "malloc" : " 14 GB", | 20260505 08:05:13.535 | INFO | "pinned" : " 29 GB" | 20260505 08:05:13.535 | INFO | } | 20260505 08:05:13.535 | INFO | Perf report for the last 5.13951 seconds: | 20260505 08:05:13.535 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:05:13.535 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:13.535 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:13.536 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:13.536 | INFO | ANALYSIS : 63.8%, 3.0, 99.1ms, 0 | 20260505 08:05:13.536 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:13.536 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:13.536 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 08:05:13.536 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:13.536 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:13.536 | INFO | Uploaded 36GiB to GPU in 2.19102s (16.4307GiB/s) | 20260505 08:05:13.536 | INFO | Downloaded 34GiB to GPU in 2.06956s (16.4286GiB/s) | 20260505 08:05:18.674 | INFO | Analyzing chunk frames = [1636352,1636864) | 20260505 08:05:18.674 | INFO | MemInfo: { | 20260505 08:05:18.674 | INFO | "gpu" : " 17 GB", | 20260505 08:05:18.674 | INFO | "huge" : "", | 20260505 08:05:18.674 | INFO | "huge_shared" : " 27 GB", | 20260505 08:05:18.674 | INFO | "malloc" : " 14 GB", | 20260505 08:05:18.674 | INFO | "pinned" : " 29 GB" | 20260505 08:05:18.674 | INFO | } | 20260505 08:05:18.674 | INFO | Perf report for the last 5.13878 seconds: | 20260505 08:05:18.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:05:18.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:18.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:18.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:18.674 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 08:05:18.674 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:18.674 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:18.674 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 08:05:18.674 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:18.674 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:18.674 | INFO | Uploaded 36GiB to GPU in 2.16828s (16.603GiB/s) | 20260505 08:05:18.674 | INFO | Downloaded 34GiB to GPU in 2.05588s (16.5379GiB/s) | 20260505 08:05:23.814 | INFO | Analyzing chunk frames = [1636864,1637376) | 20260505 08:05:23.814 | INFO | MemInfo: { | 20260505 08:05:23.814 | INFO | "gpu" : " 17 GB", | 20260505 08:05:23.814 | INFO | "huge" : "", | 20260505 08:05:23.814 | INFO | "huge_shared" : " 27 GB", | 20260505 08:05:23.814 | INFO | "malloc" : " 14 GB", | 20260505 08:05:23.814 | INFO | "pinned" : " 29 GB" | 20260505 08:05:23.814 | INFO | } | 20260505 08:05:23.814 | INFO | Perf report for the last 5.13979 seconds: | 20260505 08:05:23.814 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:05:23.814 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:23.814 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:23.814 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:23.814 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 08:05:23.814 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:23.814 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:23.814 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 08:05:23.814 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:23.814 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:23.814 | INFO | Uploaded 36GiB to GPU in 2.18926s (16.4439GiB/s) | 20260505 08:05:23.814 | INFO | Downloaded 34GiB to GPU in 2.04625s (16.6158GiB/s) | 20260505 08:05:28.954 | INFO | Analyzing chunk frames = [1637376,1637888) | 20260505 08:05:28.954 | INFO | MemInfo: { | 20260505 08:05:28.954 | INFO | "gpu" : " 17 GB", | 20260505 08:05:28.954 | INFO | "huge" : "", | 20260505 08:05:28.954 | INFO | "huge_shared" : " 27 GB", | 20260505 08:05:28.954 | INFO | "malloc" : " 14 GB", | 20260505 08:05:28.954 | INFO | "pinned" : " 29 GB" | 20260505 08:05:28.954 | INFO | } | 20260505 08:05:28.954 | INFO | Perf report for the last 5.13964 seconds: | 20260505 08:05:28.954 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:05:28.954 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:28.954 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:28.954 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:28.954 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 08:05:28.954 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:28.954 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:28.954 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260505 08:05:28.954 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:28.954 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:28.954 | INFO | Uploaded 36GiB to GPU in 2.20218s (16.3475GiB/s) | 20260505 08:05:28.954 | INFO | Downloaded 34GiB to GPU in 2.15683s (15.7639GiB/s) | 20260505 08:05:34.093 | INFO | Analyzing chunk frames = [1637888,1638400) | 20260505 08:05:34.093 | INFO | MemInfo: { | 20260505 08:05:34.093 | INFO | "gpu" : " 17 GB", | 20260505 08:05:34.093 | INFO | "huge" : "", | 20260505 08:05:34.093 | INFO | "huge_shared" : " 27 GB", | 20260505 08:05:34.093 | INFO | "malloc" : " 14 GB", | 20260505 08:05:34.093 | INFO | "pinned" : " 29 GB" | 20260505 08:05:34.093 | INFO | } | 20260505 08:05:34.093 | INFO | Perf report for the last 5.13892 seconds: | 20260505 08:05:34.093 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:05:34.093 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:34.093 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:34.093 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:34.093 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 08:05:34.093 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:34.093 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:34.093 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 08:05:34.093 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:34.093 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:34.093 | INFO | Uploaded 36GiB to GPU in 2.20041s (16.3606GiB/s) | 20260505 08:05:34.093 | INFO | Downloaded 34GiB to GPU in 2.0377s (16.6855GiB/s) | 20260505 08:05:34.093 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 08:05:34.093 | INFO | Total mem usage: 15.3 GiB | 20260505 08:05:34.093 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 08:05:34.093 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -3.56%) | 20260505 08:05:34.093 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 08:05:34.093 | INFO | High Water Breakdown: | 20260505 08:05:34.093 | INFO | 61.2% : 9.05 GiB ( 96.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 08:05:34.093 | INFO | 38.7% : 5.73 GiB ( 86.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 08:05:34.093 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 08:05:34.093 | INFO | ------------------------------------------ | 20260505 08:05:34.093 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:05:34.093 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 08:05:34.093 | INFO | Total mem usage: 26.5 GiB | 20260505 08:05:34.093 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 08:05:34.093 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 08:05:34.093 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 08:05:34.093 | INFO | High Water Breakdown: | 20260505 08:05:34.093 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:05:34.093 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:05:34.093 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:05:34.093 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:05:34.093 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:05:34.093 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:05:34.093 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 08:05:34.093 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 08:05:34.093 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:05:34.093 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 08:05:34.093 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 08:05:34.093 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 08:05:34.093 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 08:05:34.093 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 08:05:34.093 | INFO | ------------------------------------------ | 20260505 08:05:34.093 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:05:34.093 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 08:05:34.093 | INFO | Total mem usage: 25.6 GiB | 20260505 08:05:34.093 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 08:05:34.093 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 08:05:34.093 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 08:05:34.093 | INFO | High Water Breakdown: | 20260505 08:05:34.093 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 08:05:34.093 | INFO | ------------------------------------------ | 20260505 08:05:34.093 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:05:34.094 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 08:05:34.094 | INFO | Total mem usage: 15.7 GiB | 20260505 08:05:34.094 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 08:05:34.094 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 08:05:34.094 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 08:05:34.094 | INFO | High Water Breakdown: | 20260505 08:05:34.094 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 08:05:34.094 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 08:05:34.094 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 08:05:34.094 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 08:05:34.094 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 08:05:34.094 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 08:05:34.094 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 08:05:34.094 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 08:05:34.094 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 08:05:34.094 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:05:34.094 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 08:05:34.094 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:05:34.094 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 08:05:34.094 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 08:05:34.094 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 08:05:34.094 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 08:05:34.094 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:05:34.094 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 08:05:34.094 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:05:34.094 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:05:34.094 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:05:34.094 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 08:05:34.094 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 08:05:34.094 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:05:34.094 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 08:05:34.094 | INFO | ------------------------------------------ | 20260505 08:05:34.094 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:05:34.094 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 08:05:34.094 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:05:34.094 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:05:34.094 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.56e+01, 1.36e+01 / 1.96e+00 | 20260505 08:05:34.094 | INFO | PerfReport: 0.29%, AggregateStats -- 9.34e-02 / 1.04e-01, 1.04e-02 / 4.11e-03 | 20260505 08:05:34.094 | INFO | PerfReport: 11.61%, Baseline -- 3.80e+00 / 4.46e+00, 6.21e-01 / 2.65e-01 | 20260505 08:05:34.094 | INFO | PerfReport: 3.32%, Histogram -- 1.09e+00 / 1.49e+00, 2.11e-01 / 1.09e-01 | 20260505 08:05:34.094 | INFO | PerfReport: 15.55%, DME -- 4.07e+01 / 4.47e+01, 4.33e+00 / 1.53e+00 | 20260505 08:05:34.094 | INFO | PerfReport: 56.47%, FrameLabeling -- 1.85e+01 / 1.96e+01, 1.98e+00 / 1.61e+00 | 20260505 08:05:34.094 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.44e+00 / 3.06e+00, 4.12e-01 / 2.73e-01 | 20260505 08:05:34.094 | INFO | PerfReport: 3.92%, Metrics -- 1.28e+00 / 1.58e+00, 3.66e-01 / 1.92e-01 | 20260505 08:05:34.094 | INFO | PerfReport: 1.41%, PulseSerializing -- 4.60e-01 / 5.13e-01, 8.87e-02 / 3.51e-02 | 20260505 08:05:34.094 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 08:05:34.094 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:05:34.094 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:05:34.094 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.32e+01 / 0.00e+00 | 20260505 08:05:34.094 | INFO | PerfReport: 22.55%, Upload -- 2.30e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260505 08:05:34.094 | INFO | PerfReport: 21.59%, Download -- 2.20e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260505 08:05:34.094 | INFO | PerfReport: 32.18%, Compute -- 3.28e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 08:05:34.094 | INFO | PerfReport: 1.69%, LockUpload -- 1.72e+00 / 0.00e+00, 7.36e+00 / 0.00e+00 | 20260505 08:05:34.094 | INFO | PerfReport: 0.97%, LockDownload -- 9.90e-01 / 0.00e+00, 2.43e+00 / 0.00e+00 | 20260505 08:05:34.094 | INFO | PerfReport: 21.01%, LockCompute -- 2.14e+01 / 0.00e+00, 1.77e+01 / 0.00e+00 | 20260505 08:05:34.094 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.78e-02 / 0.00e+00 | 20260505 08:05:34.233 | INFO | Starting a round of Baz IO | 20260505 08:05:37.513 | INFO | Populating RTMetrics report | 20260505 08:05:37.514 | INFO | RTMetrics report populated | 20260505 08:05:37.514 | INFO | Opening tmp RTMetrics json file | 20260505 08:05:37.538 | INFO | Serializing RTMetrics to json | 20260505 08:05:37.570 | INFO | Serialization done, swapping tmp file now... | 20260505 08:05:37.572 | INFO | RTMetrics done | 20260505 08:05:37.649 | INFO | Finished a round of Baz IO: Wrote 2.447 GiB in 3.415 seconds | 20260505 08:05:37.649 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 28.34% / 0.02% / 71.64% | 20260505 08:05:37.649 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.94014% / 3.97364e-05% / 9.53674e-05% | 20260505 08:05:37.649 | INFO | Baz IO overall: | 20260505 08:05:37.649 | INFO | BytesWritten : 914673570337 | 20260505 08:05:37.649 | INFO | NumEvents : 434543717657 | 20260505 08:05:39.231 | INFO | Analyzing chunk frames = [1638400,1638912) | 20260505 08:05:39.231 | INFO | MemInfo: { | 20260505 08:05:39.231 | INFO | "gpu" : " 17 GB", | 20260505 08:05:39.232 | INFO | "huge" : "", | 20260505 08:05:39.232 | INFO | "huge_shared" : " 27 GB", | 20260505 08:05:39.232 | INFO | "malloc" : " 14 GB", | 20260505 08:05:39.232 | INFO | "pinned" : " 29 GB" | 20260505 08:05:39.232 | INFO | } | 20260505 08:05:39.232 | INFO | Perf report for the last 5.13896 seconds: | 20260505 08:05:39.232 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:05:39.232 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:39.232 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:39.232 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:39.232 | INFO | ANALYSIS : 65.4%, 2.9, 100.2ms, 0 | 20260505 08:05:39.232 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:39.232 | INFO | RT_METRICS : 46.9%, 1.0, 24.6ms, 0 | 20260505 08:05:39.232 | INFO | PRE_HQ : 63.5%, 1.9, 61.5ms, 0 | 20260505 08:05:39.232 | INFO | BAZWRITER : 65.8%, 4.1, 138.8ms, 0 | 20260505 08:05:39.232 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 08:05:39.232 | INFO | Uploaded 36GiB to GPU in 2.27166s (15.8474GiB/s) | 20260505 08:05:39.232 | INFO | Downloaded 38GiB to GPU in 2.37389s (16.0075GiB/s) | 20260505 08:05:44.372 | INFO | Analyzing chunk frames = [1638912,1639424) | 20260505 08:05:44.372 | INFO | MemInfo: { | 20260505 08:05:44.372 | INFO | "gpu" : " 17 GB", | 20260505 08:05:44.372 | INFO | "huge" : "", | 20260505 08:05:44.372 | INFO | "huge_shared" : " 27 GB", | 20260505 08:05:44.372 | INFO | "malloc" : " 14 GB", | 20260505 08:05:44.372 | INFO | "pinned" : " 29 GB" | 20260505 08:05:44.372 | INFO | } | 20260505 08:05:44.372 | INFO | Perf report for the last 5.14025 seconds: | 20260505 08:05:44.372 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:05:44.372 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:44.372 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:44.372 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:44.372 | INFO | ANALYSIS : 64.8%, 3.0, 101.2ms, 0 | 20260505 08:05:44.372 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:44.372 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:44.372 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:05:44.372 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:44.372 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:44.372 | INFO | Uploaded 36GiB to GPU in 2.24609s (16.0279GiB/s) | 20260505 08:05:44.372 | INFO | Downloaded 34GiB to GPU in 2.08885s (16.2769GiB/s) | 20260505 08:05:49.511 | INFO | Analyzing chunk frames = [1639424,1639936) | 20260505 08:05:49.511 | INFO | MemInfo: { | 20260505 08:05:49.511 | INFO | "gpu" : " 17 GB", | 20260505 08:05:49.511 | INFO | "huge" : "", | 20260505 08:05:49.512 | INFO | "huge_shared" : " 27 GB", | 20260505 08:05:49.512 | INFO | "malloc" : " 14 GB", | 20260505 08:05:49.512 | INFO | "pinned" : " 29 GB" | 20260505 08:05:49.512 | INFO | } | 20260505 08:05:49.512 | INFO | Perf report for the last 5.13974 seconds: | 20260505 08:05:49.512 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:05:49.512 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:49.512 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:49.512 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:49.512 | INFO | ANALYSIS : 63.7%, 3.0, 99.2ms, 0 | 20260505 08:05:49.512 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:49.512 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:49.512 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 08:05:49.512 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:49.512 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:49.512 | INFO | Uploaded 36GiB to GPU in 2.17141s (16.5791GiB/s) | 20260505 08:05:49.512 | INFO | Downloaded 34GiB to GPU in 2.06356s (16.4764GiB/s) | 20260505 08:05:54.649 | INFO | Analyzing chunk frames = [1639936,1640448) | 20260505 08:05:54.649 | INFO | MemInfo: { | 20260505 08:05:54.649 | INFO | "gpu" : " 17 GB", | 20260505 08:05:54.650 | INFO | "huge" : "", | 20260505 08:05:54.650 | INFO | "huge_shared" : " 27 GB", | 20260505 08:05:54.650 | INFO | "malloc" : " 14 GB", | 20260505 08:05:54.650 | INFO | "pinned" : " 29 GB" | 20260505 08:05:54.650 | INFO | } | 20260505 08:05:54.650 | INFO | Perf report for the last 5.13795 seconds: | 20260505 08:05:54.650 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:05:54.650 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:54.650 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:54.650 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:54.650 | INFO | ANALYSIS : 63.8%, 3.0, 99.1ms, 0 | 20260505 08:05:54.650 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:54.650 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:54.650 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:05:54.650 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:54.650 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:54.650 | INFO | Uploaded 36GiB to GPU in 2.19103s (16.4307GiB/s) | 20260505 08:05:54.650 | INFO | Downloaded 34GiB to GPU in 2.06463s (16.4678GiB/s) | 20260505 08:05:59.789 | INFO | Analyzing chunk frames = [1640448,1640960) | 20260505 08:05:59.789 | INFO | MemInfo: { | 20260505 08:05:59.789 | INFO | "gpu" : " 17 GB", | 20260505 08:05:59.789 | INFO | "huge" : "", | 20260505 08:05:59.789 | INFO | "huge_shared" : " 27 GB", | 20260505 08:05:59.789 | INFO | "malloc" : " 14 GB", | 20260505 08:05:59.789 | INFO | "pinned" : " 29 GB" | 20260505 08:05:59.789 | INFO | } | 20260505 08:05:59.789 | INFO | Perf report for the last 5.13921 seconds: | 20260505 08:05:59.789 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:05:59.789 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:59.789 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:59.789 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:59.789 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 08:05:59.789 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:59.789 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:59.789 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 08:05:59.789 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:59.789 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:05:59.789 | INFO | Uploaded 36GiB to GPU in 2.19595s (16.3938GiB/s) | 20260505 08:05:59.789 | INFO | Downloaded 34GiB to GPU in 2.05211s (16.5683GiB/s) | 20260505 08:06:04.930 | INFO | Analyzing chunk frames = [1640960,1641472) | 20260505 08:06:04.930 | INFO | MemInfo: { | 20260505 08:06:04.930 | INFO | "gpu" : " 17 GB", | 20260505 08:06:04.930 | INFO | "huge" : "", | 20260505 08:06:04.930 | INFO | "huge_shared" : " 27 GB", | 20260505 08:06:04.930 | INFO | "malloc" : " 14 GB", | 20260505 08:06:04.930 | INFO | "pinned" : " 29 GB" | 20260505 08:06:04.930 | INFO | } | 20260505 08:06:04.930 | INFO | Perf report for the last 5.14131 seconds: | 20260505 08:06:04.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:06:04.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:04.930 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:04.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:04.930 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 08:06:04.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:04.930 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:04.930 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 08:06:04.930 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:04.930 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:04.930 | INFO | Uploaded 36GiB to GPU in 2.1894s (16.4429GiB/s) | 20260505 08:06:04.930 | INFO | Downloaded 34GiB to GPU in 2.03971s (16.669GiB/s) | 20260505 08:06:10.069 | INFO | Analyzing chunk frames = [1641472,1641984) | 20260505 08:06:10.069 | INFO | MemInfo: { | 20260505 08:06:10.069 | INFO | "gpu" : " 17 GB", | 20260505 08:06:10.069 | INFO | "huge" : "", | 20260505 08:06:10.069 | INFO | "huge_shared" : " 27 GB", | 20260505 08:06:10.069 | INFO | "malloc" : " 14 GB", | 20260505 08:06:10.069 | INFO | "pinned" : " 29 GB" | 20260505 08:06:10.069 | INFO | } | 20260505 08:06:10.069 | INFO | Perf report for the last 5.13891 seconds: | 20260505 08:06:10.069 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:06:10.069 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:10.069 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:10.069 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:10.069 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 08:06:10.069 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:10.069 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:10.069 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260505 08:06:10.069 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:10.069 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:10.069 | INFO | Uploaded 36GiB to GPU in 2.18404s (16.4832GiB/s) | 20260505 08:06:10.069 | INFO | Downloaded 34GiB to GPU in 2.15697s (15.7628GiB/s) | 20260505 08:06:15.208 | INFO | Analyzing chunk frames = [1641984,1642496) | 20260505 08:06:15.208 | INFO | MemInfo: { | 20260505 08:06:15.208 | INFO | "gpu" : " 17 GB", | 20260505 08:06:15.208 | INFO | "huge" : "", | 20260505 08:06:15.208 | INFO | "huge_shared" : " 27 GB", | 20260505 08:06:15.208 | INFO | "malloc" : " 14 GB", | 20260505 08:06:15.208 | INFO | "pinned" : " 29 GB" | 20260505 08:06:15.208 | INFO | } | 20260505 08:06:15.208 | INFO | Perf report for the last 5.13905 seconds: | 20260505 08:06:15.208 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:06:15.208 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:15.208 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:15.208 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:15.208 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 08:06:15.208 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:15.208 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:15.208 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 08:06:15.208 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:15.208 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:15.208 | INFO | Uploaded 36GiB to GPU in 2.19542s (16.3978GiB/s) | 20260505 08:06:15.208 | INFO | Downloaded 34GiB to GPU in 2.03447s (16.712GiB/s) | 20260505 08:06:15.348 | INFO | Starting a round of Baz IO | 20260505 08:06:18.632 | INFO | Populating RTMetrics report | 20260505 08:06:18.632 | INFO | RTMetrics report populated | 20260505 08:06:18.632 | INFO | Opening tmp RTMetrics json file | 20260505 08:06:18.640 | INFO | Serializing RTMetrics to json | 20260505 08:06:18.642 | INFO | Serialization done, swapping tmp file now... | 20260505 08:06:18.643 | INFO | RTMetrics done | 20260505 08:06:18.772 | INFO | Finished a round of Baz IO: Wrote 2.446 GiB in 3.423 seconds | 20260505 08:06:18.772 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 28.32% / 0.02% / 71.66% | 20260505 08:06:18.772 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.93518% / 6.75519e-05% / 7.15256e-05% | 20260505 08:06:18.772 | INFO | Baz IO overall: | 20260505 08:06:18.772 | INFO | BytesWritten : 917300100044 | 20260505 08:06:18.772 | INFO | NumEvents : 435799192007 | 20260505 08:06:20.347 | INFO | Analyzing chunk frames = [1642496,1643008) | 20260505 08:06:20.347 | INFO | MemInfo: { | 20260505 08:06:20.347 | INFO | "gpu" : " 17 GB", | 20260505 08:06:20.347 | INFO | "huge" : "", | 20260505 08:06:20.347 | INFO | "huge_shared" : " 27 GB", | 20260505 08:06:20.347 | INFO | "malloc" : " 14 GB", | 20260505 08:06:20.347 | INFO | "pinned" : " 29 GB" | 20260505 08:06:20.347 | INFO | } | 20260505 08:06:20.347 | INFO | Perf report for the last 5.13878 seconds: | 20260505 08:06:20.347 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:06:20.347 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:20.347 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:20.347 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:20.347 | INFO | ANALYSIS : 65.5%, 2.9, 100.4ms, 0 | 20260505 08:06:20.347 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:20.347 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 08:06:20.347 | INFO | PRE_HQ : 63.7%, 1.9, 61.8ms, 0 | 20260505 08:06:20.347 | INFO | BAZWRITER : 65.9%, 4.1, 139.9ms, 0 | 20260505 08:06:20.347 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 08:06:20.347 | INFO | Uploaded 36GiB to GPU in 2.26413s (15.9001GiB/s) | 20260505 08:06:20.347 | INFO | Downloaded 38GiB to GPU in 2.37222s (16.0187GiB/s) | 20260505 08:06:25.485 | INFO | Analyzing chunk frames = [1643008,1643520) | 20260505 08:06:25.485 | INFO | MemInfo: { | 20260505 08:06:25.485 | INFO | "gpu" : " 17 GB", | 20260505 08:06:25.485 | INFO | "huge" : "", | 20260505 08:06:25.486 | INFO | "huge_shared" : " 27 GB", | 20260505 08:06:25.486 | INFO | "malloc" : " 14 GB", | 20260505 08:06:25.486 | INFO | "pinned" : " 29 GB" | 20260505 08:06:25.486 | INFO | } | 20260505 08:06:25.486 | INFO | Perf report for the last 5.13869 seconds: | 20260505 08:06:25.486 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:06:25.486 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:25.486 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:25.486 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:25.486 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 08:06:25.486 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:25.486 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:25.486 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 08:06:25.486 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:25.486 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:25.486 | INFO | Uploaded 36GiB to GPU in 2.17917s (16.5201GiB/s) | 20260505 08:06:25.486 | INFO | Downloaded 34GiB to GPU in 2.05105s (16.5769GiB/s) | 20260505 08:06:30.625 | INFO | Analyzing chunk frames = [1643520,1644032) | 20260505 08:06:30.625 | INFO | MemInfo: { | 20260505 08:06:30.625 | INFO | "gpu" : " 17 GB", | 20260505 08:06:30.625 | INFO | "huge" : "", | 20260505 08:06:30.625 | INFO | "huge_shared" : " 27 GB", | 20260505 08:06:30.625 | INFO | "malloc" : " 14 GB", | 20260505 08:06:30.625 | INFO | "pinned" : " 29 GB" | 20260505 08:06:30.625 | INFO | } | 20260505 08:06:30.625 | INFO | Perf report for the last 5.13975 seconds: | 20260505 08:06:30.625 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:06:30.625 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:30.625 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:30.625 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:30.625 | INFO | ANALYSIS : 63.8%, 3.0, 99.3ms, 0 | 20260505 08:06:30.625 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:30.625 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:30.625 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:06:30.625 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:30.625 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:30.626 | INFO | Uploaded 36GiB to GPU in 2.17229s (16.5723GiB/s) | 20260505 08:06:30.626 | INFO | Downloaded 34GiB to GPU in 2.06536s (16.462GiB/s) | 20260505 08:06:35.765 | INFO | Analyzing chunk frames = [1644032,1644544) | 20260505 08:06:35.765 | INFO | MemInfo: { | 20260505 08:06:35.765 | INFO | "gpu" : " 17 GB", | 20260505 08:06:35.765 | INFO | "huge" : "", | 20260505 08:06:35.765 | INFO | "huge_shared" : " 27 GB", | 20260505 08:06:35.765 | INFO | "malloc" : " 14 GB", | 20260505 08:06:35.765 | INFO | "pinned" : " 29 GB" | 20260505 08:06:35.765 | INFO | } | 20260505 08:06:35.765 | INFO | Perf report for the last 5.1394 seconds: | 20260505 08:06:35.765 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:06:35.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:35.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:35.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:35.765 | INFO | ANALYSIS : 63.7%, 3.0, 98.9ms, 0 | 20260505 08:06:35.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:35.765 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:35.765 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 08:06:35.765 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:35.765 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:35.765 | INFO | Uploaded 36GiB to GPU in 2.18562s (16.4713GiB/s) | 20260505 08:06:35.765 | INFO | Downloaded 34GiB to GPU in 2.06931s (16.4306GiB/s) | 20260505 08:06:40.904 | INFO | Analyzing chunk frames = [1644544,1645056) | 20260505 08:06:40.904 | INFO | MemInfo: { | 20260505 08:06:40.904 | INFO | "gpu" : " 17 GB", | 20260505 08:06:40.904 | INFO | "huge" : "", | 20260505 08:06:40.904 | INFO | "huge_shared" : " 27 GB", | 20260505 08:06:40.904 | INFO | "malloc" : " 14 GB", | 20260505 08:06:40.904 | INFO | "pinned" : " 29 GB" | 20260505 08:06:40.904 | INFO | } | 20260505 08:06:40.905 | INFO | Perf report for the last 5.13968 seconds: | 20260505 08:06:40.905 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:06:40.905 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:40.905 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:40.905 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:40.905 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 08:06:40.905 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:40.905 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:40.905 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 08:06:40.905 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:40.905 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:40.905 | INFO | Uploaded 36GiB to GPU in 2.19513s (16.3999GiB/s) | 20260505 08:06:40.905 | INFO | Downloaded 34GiB to GPU in 2.05335s (16.5583GiB/s) | 20260505 08:06:46.045 | INFO | Analyzing chunk frames = [1645056,1645568) | 20260505 08:06:46.045 | INFO | MemInfo: { | 20260505 08:06:46.045 | INFO | "gpu" : " 17 GB", | 20260505 08:06:46.045 | INFO | "huge" : "", | 20260505 08:06:46.045 | INFO | "huge_shared" : " 27 GB", | 20260505 08:06:46.045 | INFO | "malloc" : " 14 GB", | 20260505 08:06:46.045 | INFO | "pinned" : " 29 GB" | 20260505 08:06:46.045 | INFO | } | 20260505 08:06:46.045 | INFO | Perf report for the last 5.14028 seconds: | 20260505 08:06:46.045 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:06:46.045 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:46.045 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:46.045 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:46.045 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 08:06:46.045 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:46.045 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:46.045 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 08:06:46.045 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:46.045 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:46.045 | INFO | Uploaded 36GiB to GPU in 2.16389s (16.6367GiB/s) | 20260505 08:06:46.045 | INFO | Downloaded 34GiB to GPU in 2.04389s (16.635GiB/s) | 20260505 08:06:51.183 | INFO | Analyzing chunk frames = [1645568,1646080) | 20260505 08:06:51.183 | INFO | MemInfo: { | 20260505 08:06:51.183 | INFO | "gpu" : " 17 GB", | 20260505 08:06:51.183 | INFO | "huge" : "", | 20260505 08:06:51.183 | INFO | "huge_shared" : " 27 GB", | 20260505 08:06:51.183 | INFO | "malloc" : " 14 GB", | 20260505 08:06:51.183 | INFO | "pinned" : " 29 GB" | 20260505 08:06:51.183 | INFO | } | 20260505 08:06:51.183 | INFO | Perf report for the last 5.13795 seconds: | 20260505 08:06:51.183 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:06:51.183 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:51.183 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:51.183 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:51.183 | INFO | ANALYSIS : 64.0%, 3.0, 99.8ms, 0 | 20260505 08:06:51.183 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:51.183 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:51.183 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260505 08:06:51.183 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:51.183 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:51.183 | INFO | Uploaded 36GiB to GPU in 2.19494s (16.4014GiB/s) | 20260505 08:06:51.183 | INFO | Downloaded 34GiB to GPU in 2.15655s (15.7659GiB/s) | 20260505 08:06:56.322 | INFO | Analyzing chunk frames = [1646080,1646592) | 20260505 08:06:56.322 | INFO | MemInfo: { | 20260505 08:06:56.322 | INFO | "gpu" : " 17 GB", | 20260505 08:06:56.322 | INFO | "huge" : "", | 20260505 08:06:56.322 | INFO | "huge_shared" : " 27 GB", | 20260505 08:06:56.322 | INFO | "malloc" : " 14 GB", | 20260505 08:06:56.322 | INFO | "pinned" : " 29 GB" | 20260505 08:06:56.322 | INFO | } | 20260505 08:06:56.322 | INFO | Perf report for the last 5.13931 seconds: | 20260505 08:06:56.322 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:06:56.322 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:56.322 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:56.322 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:56.322 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 08:06:56.322 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:56.322 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:56.322 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 08:06:56.322 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:56.322 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:06:56.322 | INFO | Uploaded 36GiB to GPU in 2.19233s (16.4209GiB/s) | 20260505 08:06:56.322 | INFO | Downloaded 34GiB to GPU in 2.04432s (16.6315GiB/s) | 20260505 08:06:56.322 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 08:06:56.322 | INFO | Total mem usage: 15.3 GiB | 20260505 08:06:56.322 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 08:06:56.322 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.77%) | 20260505 08:06:56.322 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 08:06:56.322 | INFO | High Water Breakdown: | 20260505 08:06:56.322 | INFO | 60.6% : 8.7 GiB ( 93% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 08:06:56.322 | INFO | 39.3% : 5.63 GiB ( 84.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 08:06:56.322 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 08:06:56.322 | INFO | ------------------------------------------ | 20260505 08:06:56.322 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:06:56.322 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 08:06:56.322 | INFO | Total mem usage: 26.5 GiB | 20260505 08:06:56.323 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 08:06:56.323 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 08:06:56.323 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 08:06:56.323 | INFO | High Water Breakdown: | 20260505 08:06:56.323 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:06:56.323 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:06:56.323 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:06:56.323 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:06:56.323 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:06:56.323 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:06:56.323 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 08:06:56.323 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 08:06:56.323 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:06:56.323 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 08:06:56.323 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 08:06:56.323 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 08:06:56.323 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 08:06:56.323 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 08:06:56.323 | INFO | ------------------------------------------ | 20260505 08:06:56.323 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:06:56.323 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 08:06:56.323 | INFO | Total mem usage: 25.6 GiB | 20260505 08:06:56.323 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 08:06:56.323 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 08:06:56.323 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 08:06:56.323 | INFO | High Water Breakdown: | 20260505 08:06:56.323 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 08:06:56.323 | INFO | ------------------------------------------ | 20260505 08:06:56.323 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:06:56.323 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 08:06:56.323 | INFO | Total mem usage: 15.7 GiB | 20260505 08:06:56.323 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 08:06:56.323 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 08:06:56.323 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 08:06:56.323 | INFO | High Water Breakdown: | 20260505 08:06:56.323 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 08:06:56.323 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 08:06:56.323 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 08:06:56.323 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 08:06:56.323 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 08:06:56.323 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 08:06:56.323 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 08:06:56.323 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 08:06:56.323 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 08:06:56.323 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:06:56.323 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 08:06:56.323 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:06:56.323 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 08:06:56.323 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 08:06:56.323 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 08:06:56.323 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 08:06:56.323 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:06:56.323 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 08:06:56.323 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:06:56.323 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:06:56.323 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:06:56.323 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 08:06:56.323 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 08:06:56.323 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:06:56.323 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 08:06:56.323 | INFO | ------------------------------------------ | 20260505 08:06:56.323 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:06:56.323 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 08:06:56.323 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:06:56.323 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:06:56.323 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.52e+01, 1.37e+01 / 2.02e+00 | 20260505 08:06:56.323 | INFO | PerfReport: 0.28%, AggregateStats -- 9.32e-02 / 1.03e-01, 1.03e-02 / 3.52e-03 | 20260505 08:06:56.323 | INFO | PerfReport: 11.62%, Baseline -- 3.80e+00 / 4.43e+00, 6.27e-01 / 3.31e-01 | 20260505 08:06:56.323 | INFO | PerfReport: 3.31%, Histogram -- 1.08e+00 / 1.48e+00, 2.13e-01 / 1.16e-01 | 20260505 08:06:56.323 | INFO | PerfReport: 15.51%, DME -- 4.06e+01 / 4.45e+01, 4.17e+00 / 1.47e+00 | 20260505 08:06:56.323 | INFO | PerfReport: 56.46%, FrameLabeling -- 1.85e+01 / 1.95e+01, 1.98e+00 / 1.87e+00 | 20260505 08:06:56.323 | INFO | PerfReport: 7.48%, PulseAccumulating -- 2.45e+00 / 3.11e+00, 4.13e-01 / 2.02e-01 | 20260505 08:06:56.323 | INFO | PerfReport: 3.91%, Metrics -- 1.28e+00 / 1.62e+00, 3.63e-01 / 1.89e-01 | 20260505 08:06:56.323 | INFO | PerfReport: 1.41%, PulseSerializing -- 4.61e-01 / 5.11e-01, 8.86e-02 / 3.08e-02 | 20260505 08:06:56.323 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 08:06:56.323 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:06:56.323 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:06:56.323 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.30e+01 / 0.00e+00 | 20260505 08:06:56.323 | INFO | PerfReport: 22.51%, Upload -- 2.29e+01 / 0.00e+00, 8.29e+00 / 0.00e+00 | 20260505 08:06:56.323 | INFO | PerfReport: 21.57%, Download -- 2.20e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260505 08:06:56.323 | INFO | PerfReport: 32.14%, Compute -- 3.27e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 08:06:56.323 | INFO | PerfReport: 1.66%, LockUpload -- 1.69e+00 / 0.00e+00, 7.34e+00 / 0.00e+00 | 20260505 08:06:56.323 | INFO | PerfReport: 1.00%, LockDownload -- 1.01e+00 / 0.00e+00, 2.47e+00 / 0.00e+00 | 20260505 08:06:56.323 | INFO | PerfReport: 21.11%, LockCompute -- 2.15e+01 / 0.00e+00, 1.76e+01 / 0.00e+00 | 20260505 08:06:56.323 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.20e-02 / 0.00e+00 | 20260505 08:06:56.456 | INFO | Starting a round of Baz IO | 20260505 08:06:59.752 | INFO | Populating RTMetrics report | 20260505 08:06:59.752 | INFO | RTMetrics report populated | 20260505 08:06:59.752 | INFO | Opening tmp RTMetrics json file | 20260505 08:06:59.754 | INFO | Serializing RTMetrics to json | 20260505 08:06:59.757 | INFO | Serialization done, swapping tmp file now... | 20260505 08:06:59.758 | INFO | RTMetrics done | 20260505 08:06:59.916 | INFO | Finished a round of Baz IO: Wrote 2.446 GiB in 3.459 seconds | 20260505 08:06:59.916 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 28.29% / 0.02% / 71.68% | 20260505 08:06:59.916 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.92639% / 3.57628e-05% / 9.93411e-05% | 20260505 08:06:59.916 | INFO | Baz IO overall: | 20260505 08:06:59.916 | INFO | BytesWritten : 919926871055 | 20260505 08:06:59.916 | INFO | NumEvents : 437055248522 | 20260505 08:07:01.462 | INFO | Analyzing chunk frames = [1646592,1647104) | 20260505 08:07:01.462 | INFO | MemInfo: { | 20260505 08:07:01.462 | INFO | "gpu" : " 17 GB", | 20260505 08:07:01.462 | INFO | "huge" : "", | 20260505 08:07:01.462 | INFO | "huge_shared" : " 27 GB", | 20260505 08:07:01.462 | INFO | "malloc" : " 14 GB", | 20260505 08:07:01.462 | INFO | "pinned" : " 29 GB" | 20260505 08:07:01.462 | INFO | } | 20260505 08:07:01.462 | INFO | Perf report for the last 5.14005 seconds: | 20260505 08:07:01.462 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:07:01.462 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:01.462 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:01.462 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:01.462 | INFO | ANALYSIS : 65.6%, 2.9, 99.5ms, 0 | 20260505 08:07:01.462 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:01.462 | INFO | RT_METRICS : 46.2%, 1.0, 24.2ms, 0 | 20260505 08:07:01.462 | INFO | PRE_HQ : 64.1%, 1.9, 62.0ms, 0 | 20260505 08:07:01.462 | INFO | BAZWRITER : 66.6%, 4.2, 144.3ms, 0 | 20260505 08:07:01.462 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 08:07:01.462 | INFO | Uploaded 36GiB to GPU in 2.286s (15.7481GiB/s) | 20260505 08:07:01.462 | INFO | Downloaded 38GiB to GPU in 2.33853s (16.2495GiB/s) | 20260505 08:07:06.600 | INFO | Analyzing chunk frames = [1647104,1647616) | 20260505 08:07:06.600 | INFO | MemInfo: { | 20260505 08:07:06.600 | INFO | "gpu" : " 17 GB", | 20260505 08:07:06.601 | INFO | "huge" : "", | 20260505 08:07:06.601 | INFO | "huge_shared" : " 27 GB", | 20260505 08:07:06.601 | INFO | "malloc" : " 14 GB", | 20260505 08:07:06.601 | INFO | "pinned" : " 29 GB" | 20260505 08:07:06.601 | INFO | } | 20260505 08:07:06.601 | INFO | Perf report for the last 5.13846 seconds: | 20260505 08:07:06.601 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:07:06.601 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:06.601 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:06.601 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:06.601 | INFO | ANALYSIS : 65.0%, 3.0, 101.4ms, 0 | 20260505 08:07:06.601 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:06.601 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:06.601 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:07:06.601 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:06.601 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:06.601 | INFO | Uploaded 36GiB to GPU in 2.24215s (16.056GiB/s) | 20260505 08:07:06.601 | INFO | Downloaded 34GiB to GPU in 2.0831s (16.3219GiB/s) | 20260505 08:07:11.740 | INFO | Analyzing chunk frames = [1647616,1648128) | 20260505 08:07:11.740 | INFO | MemInfo: { | 20260505 08:07:11.740 | INFO | "gpu" : " 17 GB", | 20260505 08:07:11.740 | INFO | "huge" : "", | 20260505 08:07:11.740 | INFO | "huge_shared" : " 27 GB", | 20260505 08:07:11.740 | INFO | "malloc" : " 14 GB", | 20260505 08:07:11.740 | INFO | "pinned" : " 29 GB" | 20260505 08:07:11.740 | INFO | } | 20260505 08:07:11.740 | INFO | Perf report for the last 5.13917 seconds: | 20260505 08:07:11.740 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:07:11.740 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:11.740 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:11.740 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:11.740 | INFO | ANALYSIS : 63.8%, 3.0, 99.3ms, 0 | 20260505 08:07:11.740 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:11.740 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:11.740 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 08:07:11.740 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:11.740 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:11.740 | INFO | Uploaded 36GiB to GPU in 2.1754s (16.5487GiB/s) | 20260505 08:07:11.740 | INFO | Downloaded 34GiB to GPU in 2.06673s (16.4511GiB/s) | 20260505 08:07:16.881 | INFO | Analyzing chunk frames = [1648128,1648640) | 20260505 08:07:16.881 | INFO | MemInfo: { | 20260505 08:07:16.881 | INFO | "gpu" : " 17 GB", | 20260505 08:07:16.881 | INFO | "huge" : "", | 20260505 08:07:16.881 | INFO | "huge_shared" : " 27 GB", | 20260505 08:07:16.881 | INFO | "malloc" : " 14 GB", | 20260505 08:07:16.881 | INFO | "pinned" : " 29 GB" | 20260505 08:07:16.881 | INFO | } | 20260505 08:07:16.881 | INFO | Perf report for the last 5.14103 seconds: | 20260505 08:07:16.881 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:07:16.881 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:16.881 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:16.881 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:16.881 | INFO | ANALYSIS : 63.7%, 3.0, 99.0ms, 0 | 20260505 08:07:16.881 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:16.881 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:16.881 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 08:07:16.881 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:16.881 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:16.881 | INFO | Uploaded 36GiB to GPU in 2.18649s (16.4647GiB/s) | 20260505 08:07:16.881 | INFO | Downloaded 34GiB to GPU in 2.06546s (16.4612GiB/s) | 20260505 08:07:22.020 | INFO | Analyzing chunk frames = [1648640,1649152) | 20260505 08:07:22.020 | INFO | MemInfo: { | 20260505 08:07:22.020 | INFO | "gpu" : " 17 GB", | 20260505 08:07:22.020 | INFO | "huge" : "", | 20260505 08:07:22.020 | INFO | "huge_shared" : " 27 GB", | 20260505 08:07:22.020 | INFO | "malloc" : " 14 GB", | 20260505 08:07:22.020 | INFO | "pinned" : " 29 GB" | 20260505 08:07:22.020 | INFO | } | 20260505 08:07:22.020 | INFO | Perf report for the last 5.13929 seconds: | 20260505 08:07:22.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:07:22.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:22.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:22.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:22.020 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 08:07:22.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:22.020 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:22.020 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 08:07:22.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:22.020 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:22.020 | INFO | Uploaded 36GiB to GPU in 2.16668s (16.6153GiB/s) | 20260505 08:07:22.020 | INFO | Downloaded 34GiB to GPU in 2.05941s (16.5096GiB/s) | 20260505 08:07:27.159 | INFO | Analyzing chunk frames = [1649152,1649664) | 20260505 08:07:27.159 | INFO | MemInfo: { | 20260505 08:07:27.159 | INFO | "gpu" : " 17 GB", | 20260505 08:07:27.159 | INFO | "huge" : "", | 20260505 08:07:27.159 | INFO | "huge_shared" : " 27 GB", | 20260505 08:07:27.159 | INFO | "malloc" : " 14 GB", | 20260505 08:07:27.159 | INFO | "pinned" : " 29 GB" | 20260505 08:07:27.159 | INFO | } | 20260505 08:07:27.159 | INFO | Perf report for the last 5.1391 seconds: | 20260505 08:07:27.159 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:07:27.159 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:27.159 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:27.159 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:27.159 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 08:07:27.159 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:27.159 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:27.159 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:07:27.159 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:27.159 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:27.160 | INFO | Uploaded 36GiB to GPU in 2.18263s (16.4939GiB/s) | 20260505 08:07:27.160 | INFO | Downloaded 34GiB to GPU in 2.05363s (16.5561GiB/s) | 20260505 08:07:32.297 | INFO | Analyzing chunk frames = [1649664,1650176) | 20260505 08:07:32.297 | INFO | MemInfo: { | 20260505 08:07:32.297 | INFO | "gpu" : " 17 GB", | 20260505 08:07:32.297 | INFO | "huge" : "", | 20260505 08:07:32.297 | INFO | "huge_shared" : " 27 GB", | 20260505 08:07:32.297 | INFO | "malloc" : " 14 GB", | 20260505 08:07:32.297 | INFO | "pinned" : " 29 GB" | 20260505 08:07:32.297 | INFO | } | 20260505 08:07:32.297 | INFO | Perf report for the last 5.13803 seconds: | 20260505 08:07:32.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:07:32.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:32.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:32.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:32.297 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 08:07:32.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:32.297 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:32.297 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 08:07:32.297 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:32.297 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:32.298 | INFO | Uploaded 36GiB to GPU in 2.21472s (16.2549GiB/s) | 20260505 08:07:32.298 | INFO | Downloaded 34GiB to GPU in 2.15472s (15.7793GiB/s) | 20260505 08:07:37.436 | INFO | Analyzing chunk frames = [1650176,1650688) | 20260505 08:07:37.436 | INFO | MemInfo: { | 20260505 08:07:37.436 | INFO | "gpu" : " 17 GB", | 20260505 08:07:37.436 | INFO | "huge" : "", | 20260505 08:07:37.436 | INFO | "huge_shared" : " 27 GB", | 20260505 08:07:37.436 | INFO | "malloc" : " 14 GB", | 20260505 08:07:37.436 | INFO | "pinned" : " 29 GB" | 20260505 08:07:37.436 | INFO | } | 20260505 08:07:37.436 | INFO | Perf report for the last 5.13919 seconds: | 20260505 08:07:37.436 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:07:37.436 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:37.437 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:37.437 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:37.437 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 08:07:37.437 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:37.437 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:37.437 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 08:07:37.437 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:37.437 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:37.437 | INFO | Uploaded 36GiB to GPU in 2.19725s (16.3841GiB/s) | 20260505 08:07:37.437 | INFO | Downloaded 34GiB to GPU in 2.03791s (16.6837GiB/s) | 20260505 08:07:37.571 | INFO | Starting a round of Baz IO | 20260505 08:07:40.867 | INFO | Populating RTMetrics report | 20260505 08:07:40.867 | INFO | RTMetrics report populated | 20260505 08:07:40.867 | INFO | Opening tmp RTMetrics json file | 20260505 08:07:40.870 | INFO | Serializing RTMetrics to json | 20260505 08:07:40.871 | INFO | Serialization done, swapping tmp file now... | 20260505 08:07:40.871 | INFO | RTMetrics done | 20260505 08:07:41.026 | INFO | Finished a round of Baz IO: Wrote 2.447 GiB in 3.455 seconds | 20260505 08:07:41.026 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 28.27% / 0.02% / 71.71% | 20260505 08:07:41.026 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.92164% / 3.97364e-05% / 8.34465e-05% | 20260505 08:07:41.026 | INFO | Baz IO overall: | 20260505 08:07:41.026 | INFO | BytesWritten : 922554822905 | 20260505 08:07:41.026 | INFO | NumEvents : 438312515823 | 20260505 08:07:42.576 | INFO | Analyzing chunk frames = [1650688,1651200) | 20260505 08:07:42.576 | INFO | MemInfo: { | 20260505 08:07:42.576 | INFO | "gpu" : " 17 GB", | 20260505 08:07:42.576 | INFO | "huge" : "", | 20260505 08:07:42.576 | INFO | "huge_shared" : " 27 GB", | 20260505 08:07:42.576 | INFO | "malloc" : " 14 GB", | 20260505 08:07:42.576 | INFO | "pinned" : " 29 GB" | 20260505 08:07:42.576 | INFO | } | 20260505 08:07:42.576 | INFO | Perf report for the last 5.13944 seconds: | 20260505 08:07:42.576 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:07:42.576 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:42.576 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:42.576 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:42.576 | INFO | ANALYSIS : 65.6%, 2.9, 99.5ms, 0 | 20260505 08:07:42.576 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:42.576 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260505 08:07:42.576 | INFO | PRE_HQ : 64.1%, 1.9, 62.0ms, 0 | 20260505 08:07:42.576 | INFO | BAZWRITER : 66.5%, 4.2, 144.5ms, 0 | 20260505 08:07:42.576 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 08:07:42.577 | INFO | Uploaded 36GiB to GPU in 2.28135s (15.7801GiB/s) | 20260505 08:07:42.577 | INFO | Downloaded 38GiB to GPU in 2.3729s (16.0142GiB/s) | 20260505 08:07:47.716 | INFO | Analyzing chunk frames = [1651200,1651712) | 20260505 08:07:47.716 | INFO | MemInfo: { | 20260505 08:07:47.716 | INFO | "gpu" : " 17 GB", | 20260505 08:07:47.716 | INFO | "huge" : "", | 20260505 08:07:47.716 | INFO | "huge_shared" : " 27 GB", | 20260505 08:07:47.716 | INFO | "malloc" : " 14 GB", | 20260505 08:07:47.716 | INFO | "pinned" : " 29 GB" | 20260505 08:07:47.716 | INFO | } | 20260505 08:07:47.716 | INFO | Perf report for the last 5.14012 seconds: | 20260505 08:07:47.716 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:07:47.716 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:47.716 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:47.716 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:47.716 | INFO | ANALYSIS : 64.9%, 3.0, 101.4ms, 0 | 20260505 08:07:47.716 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:47.716 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:47.716 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 08:07:47.716 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:47.716 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:47.716 | INFO | Uploaded 36GiB to GPU in 2.24262s (16.0526GiB/s) | 20260505 08:07:47.716 | INFO | Downloaded 34GiB to GPU in 2.08346s (16.319GiB/s) | 20260505 08:07:52.856 | INFO | Analyzing chunk frames = [1651712,1652224) | 20260505 08:07:52.856 | INFO | MemInfo: { | 20260505 08:07:52.856 | INFO | "gpu" : " 17 GB", | 20260505 08:07:52.856 | INFO | "huge" : "", | 20260505 08:07:52.856 | INFO | "huge_shared" : " 27 GB", | 20260505 08:07:52.856 | INFO | "malloc" : " 14 GB", | 20260505 08:07:52.856 | INFO | "pinned" : " 29 GB" | 20260505 08:07:52.856 | INFO | } | 20260505 08:07:52.856 | INFO | Perf report for the last 5.13993 seconds: | 20260505 08:07:52.856 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:07:52.856 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:52.856 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:52.856 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:52.856 | INFO | ANALYSIS : 63.7%, 3.0, 99.2ms, 0 | 20260505 08:07:52.856 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:52.856 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:52.856 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 08:07:52.856 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:52.856 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:52.857 | INFO | Uploaded 36GiB to GPU in 2.1779s (16.5297GiB/s) | 20260505 08:07:52.857 | INFO | Downloaded 34GiB to GPU in 2.06992s (16.4257GiB/s) | 20260505 08:07:57.995 | INFO | Analyzing chunk frames = [1652224,1652736) | 20260505 08:07:57.995 | INFO | MemInfo: { | 20260505 08:07:57.995 | INFO | "gpu" : " 17 GB", | 20260505 08:07:57.995 | INFO | "huge" : "", | 20260505 08:07:57.995 | INFO | "huge_shared" : " 27 GB", | 20260505 08:07:57.995 | INFO | "malloc" : " 14 GB", | 20260505 08:07:57.995 | INFO | "pinned" : " 29 GB" | 20260505 08:07:57.995 | INFO | } | 20260505 08:07:57.995 | INFO | Perf report for the last 5.13916 seconds: | 20260505 08:07:57.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:07:57.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:57.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:57.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:57.995 | INFO | ANALYSIS : 63.6%, 3.0, 98.8ms, 0 | 20260505 08:07:57.995 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:57.995 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:57.995 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:07:57.995 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:57.995 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:07:57.996 | INFO | Uploaded 36GiB to GPU in 2.18396s (16.4838GiB/s) | 20260505 08:07:57.996 | INFO | Downloaded 34GiB to GPU in 2.06801s (16.4409GiB/s) | 20260505 08:08:03.133 | INFO | Analyzing chunk frames = [1652736,1653248) | 20260505 08:08:03.133 | INFO | MemInfo: { | 20260505 08:08:03.133 | INFO | "gpu" : " 17 GB", | 20260505 08:08:03.133 | INFO | "huge" : "", | 20260505 08:08:03.133 | INFO | "huge_shared" : " 27 GB", | 20260505 08:08:03.133 | INFO | "malloc" : " 14 GB", | 20260505 08:08:03.133 | INFO | "pinned" : " 29 GB" | 20260505 08:08:03.133 | INFO | } | 20260505 08:08:03.133 | INFO | Perf report for the last 5.13789 seconds: | 20260505 08:08:03.133 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:08:03.133 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:03.133 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:03.133 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:03.133 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 08:08:03.133 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:03.133 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:03.133 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 08:08:03.133 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:03.133 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:03.133 | INFO | Uploaded 36GiB to GPU in 2.18746s (16.4574GiB/s) | 20260505 08:08:03.133 | INFO | Downloaded 34GiB to GPU in 2.05867s (16.5155GiB/s) | 20260505 08:08:08.273 | INFO | Analyzing chunk frames = [1653248,1653760) | 20260505 08:08:08.273 | INFO | MemInfo: { | 20260505 08:08:08.273 | INFO | "gpu" : " 17 GB", | 20260505 08:08:08.273 | INFO | "huge" : "", | 20260505 08:08:08.273 | INFO | "huge_shared" : " 27 GB", | 20260505 08:08:08.273 | INFO | "malloc" : " 14 GB", | 20260505 08:08:08.274 | INFO | "pinned" : " 29 GB" | 20260505 08:08:08.274 | INFO | } | 20260505 08:08:08.274 | INFO | Perf report for the last 5.14046 seconds: | 20260505 08:08:08.274 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:08:08.274 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:08.274 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:08.274 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:08.274 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 08:08:08.274 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:08.274 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:08.274 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:08:08.274 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:08.274 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:08.274 | INFO | Uploaded 36GiB to GPU in 2.18178s (16.5003GiB/s) | 20260505 08:08:08.274 | INFO | Downloaded 34GiB to GPU in 2.04594s (16.6183GiB/s) | 20260505 08:08:13.412 | INFO | Analyzing chunk frames = [1653760,1654272) | 20260505 08:08:13.412 | INFO | MemInfo: { | 20260505 08:08:13.412 | INFO | "gpu" : " 17 GB", | 20260505 08:08:13.412 | INFO | "huge" : "", | 20260505 08:08:13.412 | INFO | "huge_shared" : " 27 GB", | 20260505 08:08:13.412 | INFO | "malloc" : " 14 GB", | 20260505 08:08:13.412 | INFO | "pinned" : " 29 GB" | 20260505 08:08:13.412 | INFO | } | 20260505 08:08:13.412 | INFO | Perf report for the last 5.13851 seconds: | 20260505 08:08:13.412 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:08:13.412 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:13.412 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:13.412 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:13.412 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 08:08:13.412 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:13.412 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:13.412 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 08:08:13.412 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:13.412 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:13.412 | INFO | Uploaded 36GiB to GPU in 2.21711s (16.2374GiB/s) | 20260505 08:08:13.412 | INFO | Downloaded 34GiB to GPU in 2.15489s (15.7781GiB/s) | 20260505 08:08:18.551 | INFO | Analyzing chunk frames = [1654272,1654784) | 20260505 08:08:18.551 | INFO | MemInfo: { | 20260505 08:08:18.551 | INFO | "gpu" : " 17 GB", | 20260505 08:08:18.551 | INFO | "huge" : "", | 20260505 08:08:18.551 | INFO | "huge_shared" : " 27 GB", | 20260505 08:08:18.552 | INFO | "malloc" : " 14 GB", | 20260505 08:08:18.552 | INFO | "pinned" : " 29 GB" | 20260505 08:08:18.552 | INFO | } | 20260505 08:08:18.552 | INFO | Perf report for the last 5.13959 seconds: | 20260505 08:08:18.552 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:08:18.552 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:18.552 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:18.552 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:18.552 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 08:08:18.552 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:18.552 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:18.552 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 08:08:18.552 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:18.552 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:18.552 | INFO | Uploaded 36GiB to GPU in 2.19609s (16.3927GiB/s) | 20260505 08:08:18.552 | INFO | Downloaded 34GiB to GPU in 2.04799s (16.6016GiB/s) | 20260505 08:08:18.552 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 08:08:18.552 | INFO | Total mem usage: 15.3 GiB | 20260505 08:08:18.552 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 08:08:18.552 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -5.46%) | 20260505 08:08:18.552 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 08:08:18.552 | INFO | High Water Breakdown: | 20260505 08:08:18.552 | INFO | 60.9% : 8.85 GiB ( 94.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 08:08:18.552 | INFO | 38.9% : 5.66 GiB ( 85.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 08:08:18.552 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 08:08:18.552 | INFO | ------------------------------------------ | 20260505 08:08:18.552 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:08:18.552 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 08:08:18.552 | INFO | Total mem usage: 26.5 GiB | 20260505 08:08:18.552 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 08:08:18.552 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 08:08:18.552 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 08:08:18.552 | INFO | High Water Breakdown: | 20260505 08:08:18.552 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:08:18.552 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:08:18.552 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:08:18.552 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:08:18.552 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:08:18.552 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:08:18.552 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 08:08:18.552 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 08:08:18.552 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:08:18.552 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 08:08:18.552 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 08:08:18.553 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 08:08:18.553 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 08:08:18.553 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 08:08:18.553 | INFO | ------------------------------------------ | 20260505 08:08:18.553 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:08:18.553 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 08:08:18.553 | INFO | Total mem usage: 25.6 GiB | 20260505 08:08:18.553 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 08:08:18.553 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 08:08:18.553 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 08:08:18.553 | INFO | High Water Breakdown: | 20260505 08:08:18.553 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 08:08:18.553 | INFO | ------------------------------------------ | 20260505 08:08:18.553 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:08:18.553 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 08:08:18.553 | INFO | Total mem usage: 15.7 GiB | 20260505 08:08:18.553 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 08:08:18.553 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 08:08:18.553 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 08:08:18.553 | INFO | High Water Breakdown: | 20260505 08:08:18.553 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 08:08:18.553 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 08:08:18.553 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 08:08:18.553 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 08:08:18.553 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 08:08:18.553 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 08:08:18.553 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 08:08:18.553 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 08:08:18.553 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 08:08:18.553 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:08:18.553 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 08:08:18.553 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:08:18.553 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 08:08:18.553 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 08:08:18.553 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 08:08:18.553 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 08:08:18.553 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:08:18.553 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 08:08:18.553 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:08:18.553 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:08:18.553 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:08:18.553 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 08:08:18.553 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 08:08:18.553 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:08:18.553 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 08:08:18.553 | INFO | ------------------------------------------ | 20260505 08:08:18.553 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:08:18.553 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 08:08:18.553 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:08:18.553 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:08:18.553 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.58e+01, 1.38e+01 / 2.52e+00 | 20260505 08:08:18.553 | INFO | PerfReport: 0.29%, AggregateStats -- 9.34e-02 / 1.04e-01, 1.16e-02 / 5.07e-03 | 20260505 08:08:18.553 | INFO | PerfReport: 11.59%, Baseline -- 3.80e+00 / 4.46e+00, 6.10e-01 / 2.43e-01 | 20260505 08:08:18.553 | INFO | PerfReport: 3.30%, Histogram -- 1.08e+00 / 1.48e+00, 2.11e-01 / 1.11e-01 | 20260505 08:08:18.553 | INFO | PerfReport: 15.53%, DME -- 4.07e+01 / 4.46e+01, 4.16e+00 / 1.22e+00 | 20260505 08:08:18.553 | INFO | PerfReport: 56.52%, FrameLabeling -- 1.85e+01 / 2.00e+01, 2.02e+00 / 2.10e+00 | 20260505 08:08:18.553 | INFO | PerfReport: 7.46%, PulseAccumulating -- 2.44e+00 / 3.09e+00, 4.11e-01 / 2.18e-01 | 20260505 08:08:18.553 | INFO | PerfReport: 3.89%, Metrics -- 1.28e+00 / 1.60e+00, 3.60e-01 / 1.74e-01 | 20260505 08:08:18.553 | INFO | PerfReport: 1.41%, PulseSerializing -- 4.61e-01 / 5.11e-01, 9.00e-02 / 2.94e-02 | 20260505 08:08:18.553 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 08:08:18.553 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:08:18.553 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:08:18.553 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.34e+01 / 0.00e+00 | 20260505 08:08:18.553 | INFO | PerfReport: 22.58%, Upload -- 2.30e+01 / 0.00e+00, 8.34e+00 / 0.00e+00 | 20260505 08:08:18.553 | INFO | PerfReport: 21.59%, Download -- 2.20e+01 / 0.00e+00, 8.15e+00 / 0.00e+00 | 20260505 08:08:18.553 | INFO | PerfReport: 32.17%, Compute -- 3.28e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 08:08:18.553 | INFO | PerfReport: 1.71%, LockUpload -- 1.75e+00 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260505 08:08:18.553 | INFO | PerfReport: 0.94%, LockDownload -- 9.58e-01 / 0.00e+00, 2.29e+00 / 0.00e+00 | 20260505 08:08:18.553 | INFO | PerfReport: 20.99%, LockCompute -- 2.14e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260505 08:08:18.553 | INFO | PerfReport: 0.01%, Misc -- 1.49e-02 / 0.00e+00, 2.78e-02 / 0.00e+00 | 20260505 08:08:18.693 | INFO | Starting a round of Baz IO | 20260505 08:08:21.986 | INFO | Populating RTMetrics report | 20260505 08:08:21.986 | INFO | RTMetrics report populated | 20260505 08:08:21.986 | INFO | Opening tmp RTMetrics json file | 20260505 08:08:22.005 | INFO | Serializing RTMetrics to json | 20260505 08:08:22.021 | INFO | Serialization done, swapping tmp file now... | 20260505 08:08:22.033 | INFO | RTMetrics done | 20260505 08:08:22.127 | INFO | Finished a round of Baz IO: Wrote 2.446 GiB in 3.433 seconds | 20260505 08:08:22.127 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 28.25% / 0.02% / 71.73% | 20260505 08:08:22.127 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.9131% / 5.96046e-05% / 5.96046e-05% | 20260505 08:08:22.127 | INFO | Baz IO overall: | 20260505 08:08:22.127 | INFO | BytesWritten : 925181714510 | 20260505 08:08:22.127 | INFO | NumEvents : 439569234136 | 20260505 08:08:23.691 | INFO | Analyzing chunk frames = [1654784,1655296) | 20260505 08:08:23.691 | INFO | MemInfo: { | 20260505 08:08:23.691 | INFO | "gpu" : " 17 GB", | 20260505 08:08:23.691 | INFO | "huge" : "", | 20260505 08:08:23.691 | INFO | "huge_shared" : " 27 GB", | 20260505 08:08:23.691 | INFO | "malloc" : " 14 GB", | 20260505 08:08:23.691 | INFO | "pinned" : " 29 GB" | 20260505 08:08:23.691 | INFO | } | 20260505 08:08:23.691 | INFO | Perf report for the last 5.13923 seconds: | 20260505 08:08:23.691 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:08:23.691 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:23.691 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:23.691 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:23.691 | INFO | ANALYSIS : 65.7%, 2.9, 100.5ms, 0 | 20260505 08:08:23.691 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:23.691 | INFO | RT_METRICS : 46.6%, 1.0, 24.5ms, 0 | 20260505 08:08:23.691 | INFO | PRE_HQ : 63.4%, 1.9, 61.4ms, 0 | 20260505 08:08:23.691 | INFO | BAZWRITER : 66.1%, 4.1, 140.6ms, 0 | 20260505 08:08:23.691 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 08:08:23.691 | INFO | Uploaded 36GiB to GPU in 2.24394s (16.0432GiB/s) | 20260505 08:08:23.691 | INFO | Downloaded 38GiB to GPU in 2.35246s (16.1533GiB/s) | 20260505 08:08:28.831 | INFO | Analyzing chunk frames = [1655296,1655808) | 20260505 08:08:28.831 | INFO | MemInfo: { | 20260505 08:08:28.831 | INFO | "gpu" : " 17 GB", | 20260505 08:08:28.831 | INFO | "huge" : "", | 20260505 08:08:28.831 | INFO | "huge_shared" : " 27 GB", | 20260505 08:08:28.831 | INFO | "malloc" : " 14 GB", | 20260505 08:08:28.832 | INFO | "pinned" : " 29 GB" | 20260505 08:08:28.832 | INFO | } | 20260505 08:08:28.832 | INFO | Perf report for the last 5.14062 seconds: | 20260505 08:08:28.832 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:08:28.832 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:28.832 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:28.832 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:28.832 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 08:08:28.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:28.832 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:28.832 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:08:28.832 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:28.832 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:28.832 | INFO | Uploaded 36GiB to GPU in 2.17818s (16.5276GiB/s) | 20260505 08:08:28.832 | INFO | Downloaded 34GiB to GPU in 2.05015s (16.5841GiB/s) | 20260505 08:08:33.970 | INFO | Analyzing chunk frames = [1655808,1656320) | 20260505 08:08:33.970 | INFO | MemInfo: { | 20260505 08:08:33.970 | INFO | "gpu" : " 17 GB", | 20260505 08:08:33.970 | INFO | "huge" : "", | 20260505 08:08:33.970 | INFO | "huge_shared" : " 27 GB", | 20260505 08:08:33.970 | INFO | "malloc" : " 14 GB", | 20260505 08:08:33.970 | INFO | "pinned" : " 29 GB" | 20260505 08:08:33.971 | INFO | } | 20260505 08:08:33.971 | INFO | Perf report for the last 5.13898 seconds: | 20260505 08:08:33.971 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:08:33.971 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:33.971 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:33.971 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:33.971 | INFO | ANALYSIS : 63.7%, 3.0, 99.1ms, 0 | 20260505 08:08:33.971 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:33.971 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:33.971 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 08:08:33.971 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:33.971 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:33.971 | INFO | Uploaded 36GiB to GPU in 2.16925s (16.5956GiB/s) | 20260505 08:08:33.971 | INFO | Downloaded 34GiB to GPU in 2.06575s (16.4589GiB/s) | 20260505 08:08:39.109 | INFO | Analyzing chunk frames = [1656320,1656832) | 20260505 08:08:39.109 | INFO | MemInfo: { | 20260505 08:08:39.109 | INFO | "gpu" : " 17 GB", | 20260505 08:08:39.109 | INFO | "huge" : "", | 20260505 08:08:39.109 | INFO | "huge_shared" : " 27 GB", | 20260505 08:08:39.109 | INFO | "malloc" : " 14 GB", | 20260505 08:08:39.109 | INFO | "pinned" : " 29 GB" | 20260505 08:08:39.109 | INFO | } | 20260505 08:08:39.109 | INFO | Perf report for the last 5.13843 seconds: | 20260505 08:08:39.109 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:08:39.109 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:39.109 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:39.109 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:39.109 | INFO | ANALYSIS : 63.8%, 3.0, 99.1ms, 0 | 20260505 08:08:39.109 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:39.109 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:39.109 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 08:08:39.109 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:39.109 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:39.109 | INFO | Uploaded 36GiB to GPU in 2.1943s (16.4061GiB/s) | 20260505 08:08:39.109 | INFO | Downloaded 34GiB to GPU in 2.06707s (16.4484GiB/s) | 20260505 08:08:44.248 | INFO | Analyzing chunk frames = [1656832,1657344) | 20260505 08:08:44.248 | INFO | MemInfo: { | 20260505 08:08:44.248 | INFO | "gpu" : " 17 GB", | 20260505 08:08:44.248 | INFO | "huge" : "", | 20260505 08:08:44.248 | INFO | "huge_shared" : " 27 GB", | 20260505 08:08:44.248 | INFO | "malloc" : " 14 GB", | 20260505 08:08:44.248 | INFO | "pinned" : " 29 GB" | 20260505 08:08:44.248 | INFO | } | 20260505 08:08:44.248 | INFO | Perf report for the last 5.13945 seconds: | 20260505 08:08:44.248 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:08:44.248 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:44.248 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:44.248 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:44.249 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 08:08:44.249 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:44.249 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:44.249 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 08:08:44.249 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:44.249 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:44.249 | INFO | Uploaded 36GiB to GPU in 2.17527s (16.5497GiB/s) | 20260505 08:08:44.249 | INFO | Downloaded 34GiB to GPU in 2.05923s (16.511GiB/s) | 20260505 08:08:49.388 | INFO | Analyzing chunk frames = [1657344,1657856) | 20260505 08:08:49.388 | INFO | MemInfo: { | 20260505 08:08:49.388 | INFO | "gpu" : " 17 GB", | 20260505 08:08:49.388 | INFO | "huge" : "", | 20260505 08:08:49.388 | INFO | "huge_shared" : " 27 GB", | 20260505 08:08:49.388 | INFO | "malloc" : " 14 GB", | 20260505 08:08:49.388 | INFO | "pinned" : " 29 GB" | 20260505 08:08:49.388 | INFO | } | 20260505 08:08:49.388 | INFO | Perf report for the last 5.13967 seconds: | 20260505 08:08:49.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:08:49.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:49.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:49.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:49.388 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 08:08:49.388 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:49.388 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:49.388 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:08:49.388 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:49.388 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:49.388 | INFO | Uploaded 36GiB to GPU in 2.16449s (16.6321GiB/s) | 20260505 08:08:49.388 | INFO | Downloaded 34GiB to GPU in 2.03961s (16.6698GiB/s) | 20260505 08:08:54.527 | INFO | Analyzing chunk frames = [1657856,1658368) | 20260505 08:08:54.527 | INFO | MemInfo: { | 20260505 08:08:54.527 | INFO | "gpu" : " 17 GB", | 20260505 08:08:54.527 | INFO | "huge" : "", | 20260505 08:08:54.527 | INFO | "huge_shared" : " 27 GB", | 20260505 08:08:54.527 | INFO | "malloc" : " 14 GB", | 20260505 08:08:54.527 | INFO | "pinned" : " 29 GB" | 20260505 08:08:54.527 | INFO | } | 20260505 08:08:54.527 | INFO | Perf report for the last 5.13886 seconds: | 20260505 08:08:54.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:08:54.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:54.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:54.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:54.527 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 08:08:54.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:54.527 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:54.527 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 08:08:54.527 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:54.527 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:54.527 | INFO | Uploaded 36GiB to GPU in 2.19258s (16.419GiB/s) | 20260505 08:08:54.527 | INFO | Downloaded 34GiB to GPU in 2.14765s (15.8312GiB/s) | 20260505 08:08:59.667 | INFO | Analyzing chunk frames = [1658368,1658880) | 20260505 08:08:59.667 | INFO | MemInfo: { | 20260505 08:08:59.667 | INFO | "gpu" : " 17 GB", | 20260505 08:08:59.667 | INFO | "huge" : "", | 20260505 08:08:59.667 | INFO | "huge_shared" : " 27 GB", | 20260505 08:08:59.667 | INFO | "malloc" : " 14 GB", | 20260505 08:08:59.667 | INFO | "pinned" : " 29 GB" | 20260505 08:08:59.667 | INFO | } | 20260505 08:08:59.667 | INFO | Perf report for the last 5.14023 seconds: | 20260505 08:08:59.667 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:08:59.667 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:59.667 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:59.667 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:59.667 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 08:08:59.667 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:59.667 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:59.667 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 08:08:59.667 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:59.667 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:08:59.667 | INFO | Uploaded 36GiB to GPU in 2.1917s (16.4256GiB/s) | 20260505 08:08:59.668 | INFO | Downloaded 34GiB to GPU in 2.04494s (16.6264GiB/s) | 20260505 08:08:59.808 | INFO | Starting a round of Baz IO | 20260505 08:09:03.100 | INFO | Populating RTMetrics report | 20260505 08:09:03.100 | INFO | RTMetrics report populated | 20260505 08:09:03.100 | INFO | Opening tmp RTMetrics json file | 20260505 08:09:03.102 | INFO | Serializing RTMetrics to json | 20260505 08:09:03.125 | INFO | Serialization done, swapping tmp file now... | 20260505 08:09:03.130 | INFO | RTMetrics done | 20260505 08:09:03.246 | INFO | Finished a round of Baz IO: Wrote 2.446 GiB in 3.438 seconds | 20260505 08:09:03.246 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 28.22% / 0.02% / 71.75% | 20260505 08:09:03.246 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.90015% / 3.17891e-05% / 5.5631e-05% | 20260505 08:09:03.246 | INFO | Baz IO overall: | 20260505 08:09:03.246 | INFO | BytesWritten : 927807750148 | 20260505 08:09:03.246 | INFO | NumEvents : 440825534088 | 20260505 08:09:04.806 | INFO | Analyzing chunk frames = [1658880,1659392) | 20260505 08:09:04.806 | INFO | MemInfo: { | 20260505 08:09:04.806 | INFO | "gpu" : " 17 GB", | 20260505 08:09:04.806 | INFO | "huge" : "", | 20260505 08:09:04.806 | INFO | "huge_shared" : " 27 GB", | 20260505 08:09:04.806 | INFO | "malloc" : " 14 GB", | 20260505 08:09:04.806 | INFO | "pinned" : " 29 GB" | 20260505 08:09:04.806 | INFO | } | 20260505 08:09:04.806 | INFO | Perf report for the last 5.13878 seconds: | 20260505 08:09:04.806 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:09:04.806 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:04.806 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:04.806 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:04.806 | INFO | ANALYSIS : 65.7%, 2.9, 100.2ms, 0 | 20260505 08:09:04.806 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:04.806 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260505 08:09:04.806 | INFO | PRE_HQ : 63.4%, 1.9, 61.6ms, 0 | 20260505 08:09:04.806 | INFO | BAZWRITER : 66.2%, 4.2, 141.4ms, 0 | 20260505 08:09:04.806 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 08:09:04.806 | INFO | Uploaded 36GiB to GPU in 2.25877s (15.9379GiB/s) | 20260505 08:09:04.806 | INFO | Downloaded 38GiB to GPU in 2.35638s (16.1265GiB/s) | 20260505 08:09:09.945 | INFO | Analyzing chunk frames = [1659392,1659904) | 20260505 08:09:09.945 | INFO | MemInfo: { | 20260505 08:09:09.945 | INFO | "gpu" : " 17 GB", | 20260505 08:09:09.945 | INFO | "huge" : "", | 20260505 08:09:09.946 | INFO | "huge_shared" : " 27 GB", | 20260505 08:09:09.946 | INFO | "malloc" : " 14 GB", | 20260505 08:09:09.946 | INFO | "pinned" : " 29 GB" | 20260505 08:09:09.946 | INFO | } | 20260505 08:09:09.946 | INFO | Perf report for the last 5.13953 seconds: | 20260505 08:09:09.946 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:09:09.946 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:09.946 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:09.946 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:09.946 | INFO | ANALYSIS : 64.9%, 3.0, 101.4ms, 0 | 20260505 08:09:09.946 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:09.946 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:09.946 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:09:09.946 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:09.946 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:09.946 | INFO | Uploaded 36GiB to GPU in 2.24856s (16.0102GiB/s) | 20260505 08:09:09.946 | INFO | Downloaded 34GiB to GPU in 2.0919s (16.2532GiB/s) | 20260505 08:09:15.085 | INFO | Analyzing chunk frames = [1659904,1660416) | 20260505 08:09:15.085 | INFO | MemInfo: { | 20260505 08:09:15.085 | INFO | "gpu" : " 17 GB", | 20260505 08:09:15.085 | INFO | "huge" : "", | 20260505 08:09:15.085 | INFO | "huge_shared" : " 27 GB", | 20260505 08:09:15.085 | INFO | "malloc" : " 14 GB", | 20260505 08:09:15.085 | INFO | "pinned" : " 29 GB" | 20260505 08:09:15.085 | INFO | } | 20260505 08:09:15.085 | INFO | Perf report for the last 5.13915 seconds: | 20260505 08:09:15.085 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:09:15.085 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:15.085 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:15.085 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:15.085 | INFO | ANALYSIS : 63.7%, 3.0, 99.1ms, 0 | 20260505 08:09:15.085 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:15.085 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:15.085 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 08:09:15.085 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:15.085 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:15.085 | INFO | Uploaded 36GiB to GPU in 2.17539s (16.5487GiB/s) | 20260505 08:09:15.085 | INFO | Downloaded 34GiB to GPU in 2.07033s (16.4225GiB/s) | 20260505 08:09:20.223 | INFO | Analyzing chunk frames = [1660416,1660928) | 20260505 08:09:20.223 | INFO | MemInfo: { | 20260505 08:09:20.224 | INFO | "gpu" : " 17 GB", | 20260505 08:09:20.224 | INFO | "huge" : "", | 20260505 08:09:20.224 | INFO | "huge_shared" : " 27 GB", | 20260505 08:09:20.224 | INFO | "malloc" : " 14 GB", | 20260505 08:09:20.224 | INFO | "pinned" : " 29 GB" | 20260505 08:09:20.224 | INFO | } | 20260505 08:09:20.224 | INFO | Perf report for the last 5.13884 seconds: | 20260505 08:09:20.224 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:09:20.224 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:20.224 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:20.224 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:20.224 | INFO | ANALYSIS : 63.7%, 3.0, 99.0ms, 0 | 20260505 08:09:20.224 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:20.224 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:20.224 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 08:09:20.224 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:20.224 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:20.224 | INFO | Uploaded 36GiB to GPU in 2.19191s (16.424GiB/s) | 20260505 08:09:20.224 | INFO | Downloaded 34GiB to GPU in 2.06557s (16.4603GiB/s) | 20260505 08:09:25.363 | INFO | Analyzing chunk frames = [1660928,1661440) | 20260505 08:09:25.363 | INFO | MemInfo: { | 20260505 08:09:25.363 | INFO | "gpu" : " 17 GB", | 20260505 08:09:25.363 | INFO | "huge" : "", | 20260505 08:09:25.363 | INFO | "huge_shared" : " 27 GB", | 20260505 08:09:25.363 | INFO | "malloc" : " 14 GB", | 20260505 08:09:25.363 | INFO | "pinned" : " 29 GB" | 20260505 08:09:25.363 | INFO | } | 20260505 08:09:25.363 | INFO | Perf report for the last 5.13918 seconds: | 20260505 08:09:25.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:09:25.363 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:25.363 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:25.363 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:25.363 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 08:09:25.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:25.363 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:25.363 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 08:09:25.363 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:25.363 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:25.363 | INFO | Uploaded 36GiB to GPU in 2.17499s (16.5518GiB/s) | 20260505 08:09:25.363 | INFO | Downloaded 34GiB to GPU in 2.05619s (16.5355GiB/s) | 20260505 08:09:30.504 | INFO | Analyzing chunk frames = [1661440,1661952) | 20260505 08:09:30.504 | INFO | MemInfo: { | 20260505 08:09:30.504 | INFO | "gpu" : " 17 GB", | 20260505 08:09:30.504 | INFO | "huge" : "", | 20260505 08:09:30.504 | INFO | "huge_shared" : " 27 GB", | 20260505 08:09:30.504 | INFO | "malloc" : " 14 GB", | 20260505 08:09:30.504 | INFO | "pinned" : " 29 GB" | 20260505 08:09:30.504 | INFO | } | 20260505 08:09:30.504 | INFO | Perf report for the last 5.14129 seconds: | 20260505 08:09:30.504 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:09:30.504 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:30.504 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:30.504 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:30.504 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 08:09:30.504 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:30.504 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:30.504 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:09:30.504 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:30.504 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:30.504 | INFO | Uploaded 36GiB to GPU in 2.18628s (16.4664GiB/s) | 20260505 08:09:30.504 | INFO | Downloaded 34GiB to GPU in 2.03943s (16.6713GiB/s) | 20260505 08:09:35.641 | INFO | Analyzing chunk frames = [1661952,1662464) | 20260505 08:09:35.641 | INFO | MemInfo: { | 20260505 08:09:35.641 | INFO | "gpu" : " 17 GB", | 20260505 08:09:35.642 | INFO | "huge" : "", | 20260505 08:09:35.642 | INFO | "huge_shared" : " 27 GB", | 20260505 08:09:35.642 | INFO | "malloc" : " 14 GB", | 20260505 08:09:35.642 | INFO | "pinned" : " 29 GB" | 20260505 08:09:35.642 | INFO | } | 20260505 08:09:35.642 | INFO | Perf report for the last 5.13744 seconds: | 20260505 08:09:35.642 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:09:35.642 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:35.642 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:35.642 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:35.642 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 08:09:35.642 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:35.642 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:35.642 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 08:09:35.642 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:35.642 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:35.642 | INFO | Uploaded 36GiB to GPU in 2.22371s (16.1891GiB/s) | 20260505 08:09:35.642 | INFO | Downloaded 34GiB to GPU in 2.15936s (15.7454GiB/s) | 20260505 08:09:40.782 | INFO | Analyzing chunk frames = [1662464,1662976) | 20260505 08:09:40.782 | INFO | MemInfo: { | 20260505 08:09:40.782 | INFO | "gpu" : " 17 GB", | 20260505 08:09:40.782 | INFO | "huge" : "", | 20260505 08:09:40.782 | INFO | "huge_shared" : " 27 GB", | 20260505 08:09:40.782 | INFO | "malloc" : " 14 GB", | 20260505 08:09:40.782 | INFO | "pinned" : " 29 GB" | 20260505 08:09:40.782 | INFO | } | 20260505 08:09:40.782 | INFO | Perf report for the last 5.14035 seconds: | 20260505 08:09:40.782 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:09:40.782 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:40.782 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:40.782 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:40.782 | INFO | ANALYSIS : 64.3%, 3.0, 100.5ms, 0 | 20260505 08:09:40.782 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:40.782 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:40.782 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 08:09:40.782 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:40.782 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:40.782 | INFO | Uploaded 36GiB to GPU in 2.19657s (16.3892GiB/s) | 20260505 08:09:40.782 | INFO | Downloaded 34GiB to GPU in 2.04446s (16.6303GiB/s) | 20260505 08:09:40.782 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 08:09:40.782 | INFO | Total mem usage: 15.3 GiB | 20260505 08:09:40.782 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 08:09:40.782 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.22%) | 20260505 08:09:40.782 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 08:09:40.782 | INFO | High Water Breakdown: | 20260505 08:09:40.782 | INFO | 60.8% : 8.77 GiB ( 93.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 08:09:40.782 | INFO | 39% : 5.63 GiB ( 84.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 08:09:40.782 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 08:09:40.782 | INFO | ------------------------------------------ | 20260505 08:09:40.782 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:09:40.782 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 08:09:40.782 | INFO | Total mem usage: 26.5 GiB | 20260505 08:09:40.782 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 08:09:40.782 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 08:09:40.782 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 08:09:40.782 | INFO | High Water Breakdown: | 20260505 08:09:40.782 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:09:40.782 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:09:40.782 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:09:40.783 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:09:40.783 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:09:40.783 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:09:40.783 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 08:09:40.783 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 08:09:40.783 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:09:40.783 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 08:09:40.783 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 08:09:40.783 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 08:09:40.783 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 08:09:40.783 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 08:09:40.783 | INFO | ------------------------------------------ | 20260505 08:09:40.783 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:09:40.783 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 08:09:40.783 | INFO | Total mem usage: 25.6 GiB | 20260505 08:09:40.783 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 08:09:40.783 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 08:09:40.783 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 08:09:40.783 | INFO | High Water Breakdown: | 20260505 08:09:40.783 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 08:09:40.783 | INFO | ------------------------------------------ | 20260505 08:09:40.783 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:09:40.783 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 08:09:40.783 | INFO | Total mem usage: 15.7 GiB | 20260505 08:09:40.783 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 08:09:40.783 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 08:09:40.783 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 08:09:40.783 | INFO | High Water Breakdown: | 20260505 08:09:40.783 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 08:09:40.783 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 08:09:40.783 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 08:09:40.783 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 08:09:40.783 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 08:09:40.783 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 08:09:40.783 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 08:09:40.783 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 08:09:40.783 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 08:09:40.783 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:09:40.783 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 08:09:40.783 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:09:40.783 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 08:09:40.783 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 08:09:40.783 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 08:09:40.783 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 08:09:40.783 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:09:40.783 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 08:09:40.783 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:09:40.783 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:09:40.783 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:09:40.783 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 08:09:40.783 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 08:09:40.783 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:09:40.783 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 08:09:40.783 | INFO | ------------------------------------------ | 20260505 08:09:40.783 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:09:40.783 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 08:09:40.783 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:09:40.783 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:09:40.783 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.55e+01, 1.37e+01 / 2.23e+00 | 20260505 08:09:40.783 | INFO | PerfReport: 0.28%, AggregateStats -- 9.30e-02 / 1.03e-01, 1.04e-02 / 4.81e-03 | 20260505 08:09:40.783 | INFO | PerfReport: 11.62%, Baseline -- 3.80e+00 / 4.46e+00, 6.17e-01 / 2.99e-01 | 20260505 08:09:40.783 | INFO | PerfReport: 3.31%, Histogram -- 1.08e+00 / 1.50e+00, 2.09e-01 / 1.05e-01 | 20260505 08:09:40.783 | INFO | PerfReport: 15.55%, DME -- 4.07e+01 / 4.47e+01, 4.32e+00 / 1.70e+00 | 20260505 08:09:40.783 | INFO | PerfReport: 56.48%, FrameLabeling -- 1.85e+01 / 1.96e+01, 1.97e+00 / 1.70e+00 | 20260505 08:09:40.783 | INFO | PerfReport: 7.46%, PulseAccumulating -- 2.44e+00 / 3.06e+00, 4.09e-01 / 2.46e-01 | 20260505 08:09:40.783 | INFO | PerfReport: 3.89%, Metrics -- 1.27e+00 / 1.58e+00, 3.56e-01 / 1.94e-01 | 20260505 08:09:40.783 | INFO | PerfReport: 1.40%, PulseSerializing -- 4.59e-01 / 5.10e-01, 8.84e-02 / 3.70e-02 | 20260505 08:09:40.783 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 08:09:40.783 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:09:40.783 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:09:40.783 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.30e+01 / 0.00e+00 | 20260505 08:09:40.783 | INFO | PerfReport: 22.48%, Upload -- 2.29e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260505 08:09:40.783 | INFO | PerfReport: 21.55%, Download -- 2.19e+01 / 0.00e+00, 8.12e+00 / 0.00e+00 | 20260505 08:09:40.783 | INFO | PerfReport: 32.13%, Compute -- 3.27e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 08:09:40.783 | INFO | PerfReport: 1.66%, LockUpload -- 1.70e+00 / 0.00e+00, 7.36e+00 / 0.00e+00 | 20260505 08:09:40.783 | INFO | PerfReport: 1.00%, LockDownload -- 1.01e+00 / 0.00e+00, 2.48e+00 / 0.00e+00 | 20260505 08:09:40.783 | INFO | PerfReport: 21.15%, LockCompute -- 2.15e+01 / 0.00e+00, 1.78e+01 / 0.00e+00 | 20260505 08:09:40.783 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.99e-02 / 0.00e+00 | 20260505 08:09:40.923 | INFO | Starting a round of Baz IO | 20260505 08:09:44.206 | INFO | Populating RTMetrics report | 20260505 08:09:44.206 | INFO | RTMetrics report populated | 20260505 08:09:44.206 | INFO | Opening tmp RTMetrics json file | 20260505 08:09:44.228 | INFO | Serializing RTMetrics to json | 20260505 08:09:44.254 | INFO | Serialization done, swapping tmp file now... | 20260505 08:09:44.255 | INFO | RTMetrics done | 20260505 08:09:44.358 | INFO | Finished a round of Baz IO: Wrote 2.448 GiB in 3.434 seconds | 20260505 08:09:44.358 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 28.20% / 0.02% / 71.78% | 20260505 08:09:44.358 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.89466% / 2.78155e-05% / 7.15256e-05% | 20260505 08:09:44.358 | INFO | Baz IO overall: | 20260505 08:09:44.358 | INFO | BytesWritten : 930435872275 | 20260505 08:09:44.358 | INFO | NumEvents : 442082805832 | 20260505 08:09:45.921 | INFO | Analyzing chunk frames = [1662976,1663488) | 20260505 08:09:45.921 | INFO | MemInfo: { | 20260505 08:09:45.921 | INFO | "gpu" : " 17 GB", | 20260505 08:09:45.921 | INFO | "huge" : "", | 20260505 08:09:45.921 | INFO | "huge_shared" : " 27 GB", | 20260505 08:09:45.921 | INFO | "malloc" : " 14 GB", | 20260505 08:09:45.921 | INFO | "pinned" : " 29 GB" | 20260505 08:09:45.921 | INFO | } | 20260505 08:09:45.921 | INFO | Perf report for the last 5.13881 seconds: | 20260505 08:09:45.921 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:09:45.921 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:45.921 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:45.921 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:45.921 | INFO | ANALYSIS : 65.5%, 2.9, 99.8ms, 0 | 20260505 08:09:45.921 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:45.921 | INFO | RT_METRICS : 46.9%, 1.0, 24.6ms, 0 | 20260505 08:09:45.921 | INFO | PRE_HQ : 63.8%, 1.9, 61.9ms, 0 | 20260505 08:09:45.921 | INFO | BAZWRITER : 66.1%, 4.1, 141.0ms, 0 | 20260505 08:09:45.921 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 08:09:45.921 | INFO | Uploaded 36GiB to GPU in 2.2691s (15.8653GiB/s) | 20260505 08:09:45.921 | INFO | Downloaded 38GiB to GPU in 2.37073s (16.0288GiB/s) | 20260505 08:09:51.060 | INFO | Analyzing chunk frames = [1663488,1664000) | 20260505 08:09:51.060 | INFO | MemInfo: { | 20260505 08:09:51.060 | INFO | "gpu" : " 17 GB", | 20260505 08:09:51.060 | INFO | "huge" : "", | 20260505 08:09:51.060 | INFO | "huge_shared" : " 27 GB", | 20260505 08:09:51.060 | INFO | "malloc" : " 14 GB", | 20260505 08:09:51.060 | INFO | "pinned" : " 29 GB" | 20260505 08:09:51.060 | INFO | } | 20260505 08:09:51.060 | INFO | Perf report for the last 5.13897 seconds: | 20260505 08:09:51.060 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:09:51.060 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:51.060 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:51.060 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:51.060 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 08:09:51.060 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:51.060 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:51.060 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 08:09:51.060 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:51.060 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:51.060 | INFO | Uploaded 36GiB to GPU in 2.18008s (16.5132GiB/s) | 20260505 08:09:51.060 | INFO | Downloaded 34GiB to GPU in 2.0494s (16.5902GiB/s) | 20260505 08:09:56.199 | INFO | Analyzing chunk frames = [1664000,1664512) | 20260505 08:09:56.199 | INFO | MemInfo: { | 20260505 08:09:56.199 | INFO | "gpu" : " 17 GB", | 20260505 08:09:56.199 | INFO | "huge" : "", | 20260505 08:09:56.199 | INFO | "huge_shared" : " 27 GB", | 20260505 08:09:56.199 | INFO | "malloc" : " 14 GB", | 20260505 08:09:56.199 | INFO | "pinned" : " 29 GB" | 20260505 08:09:56.199 | INFO | } | 20260505 08:09:56.199 | INFO | Perf report for the last 5.13938 seconds: | 20260505 08:09:56.199 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:09:56.199 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:56.199 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:56.199 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:56.199 | INFO | ANALYSIS : 63.6%, 3.0, 99.0ms, 0 | 20260505 08:09:56.199 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:56.199 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:56.199 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 08:09:56.199 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:56.199 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:09:56.199 | INFO | Uploaded 36GiB to GPU in 2.16275s (16.6455GiB/s) | 20260505 08:09:56.199 | INFO | Downloaded 34GiB to GPU in 2.06191s (16.4895GiB/s) | 20260505 08:10:01.339 | INFO | Analyzing chunk frames = [1664512,1665024) | 20260505 08:10:01.339 | INFO | MemInfo: { | 20260505 08:10:01.339 | INFO | "gpu" : " 17 GB", | 20260505 08:10:01.339 | INFO | "huge" : "", | 20260505 08:10:01.339 | INFO | "huge_shared" : " 27 GB", | 20260505 08:10:01.339 | INFO | "malloc" : " 14 GB", | 20260505 08:10:01.339 | INFO | "pinned" : " 29 GB" | 20260505 08:10:01.339 | INFO | } | 20260505 08:10:01.339 | INFO | Perf report for the last 5.13998 seconds: | 20260505 08:10:01.339 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:10:01.339 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:01.339 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:01.339 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:01.339 | INFO | ANALYSIS : 63.7%, 3.0, 99.0ms, 0 | 20260505 08:10:01.339 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:01.339 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:01.339 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 08:10:01.339 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:01.339 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:01.340 | INFO | Uploaded 36GiB to GPU in 2.18629s (16.4663GiB/s) | 20260505 08:10:01.340 | INFO | Downloaded 34GiB to GPU in 2.0644s (16.4697GiB/s) | 20260505 08:10:06.479 | INFO | Analyzing chunk frames = [1665024,1665536) | 20260505 08:10:06.479 | INFO | MemInfo: { | 20260505 08:10:06.479 | INFO | "gpu" : " 17 GB", | 20260505 08:10:06.479 | INFO | "huge" : "", | 20260505 08:10:06.479 | INFO | "huge_shared" : " 27 GB", | 20260505 08:10:06.479 | INFO | "malloc" : " 14 GB", | 20260505 08:10:06.479 | INFO | "pinned" : " 29 GB" | 20260505 08:10:06.479 | INFO | } | 20260505 08:10:06.479 | INFO | Perf report for the last 5.13959 seconds: | 20260505 08:10:06.479 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:10:06.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:06.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:06.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:06.479 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 08:10:06.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:06.479 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:06.479 | INFO | PRE_HQ : 30.1%, 1.0, 15.7ms, 0 | 20260505 08:10:06.479 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:06.479 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:06.479 | INFO | Uploaded 36GiB to GPU in 2.17368s (16.5618GiB/s) | 20260505 08:10:06.479 | INFO | Downloaded 34GiB to GPU in 2.05428s (16.5508GiB/s) | 20260505 08:10:11.618 | INFO | Analyzing chunk frames = [1665536,1666048) | 20260505 08:10:11.618 | INFO | MemInfo: { | 20260505 08:10:11.618 | INFO | "gpu" : " 17 GB", | 20260505 08:10:11.618 | INFO | "huge" : "", | 20260505 08:10:11.618 | INFO | "huge_shared" : " 27 GB", | 20260505 08:10:11.618 | INFO | "malloc" : " 14 GB", | 20260505 08:10:11.618 | INFO | "pinned" : " 29 GB" | 20260505 08:10:11.618 | INFO | } | 20260505 08:10:11.618 | INFO | Perf report for the last 5.13953 seconds: | 20260505 08:10:11.618 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:10:11.618 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:11.618 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:11.618 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:11.618 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 08:10:11.618 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:11.618 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:11.618 | INFO | PRE_HQ : 29.9%, 1.0, 15.6ms, 0 | 20260505 08:10:11.618 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:11.618 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:11.619 | INFO | Uploaded 36GiB to GPU in 2.18224s (16.4968GiB/s) | 20260505 08:10:11.619 | INFO | Downloaded 34GiB to GPU in 2.04396s (16.6344GiB/s) | 20260505 08:10:16.756 | INFO | Analyzing chunk frames = [1666048,1666560) | 20260505 08:10:16.756 | INFO | MemInfo: { | 20260505 08:10:16.756 | INFO | "gpu" : " 17 GB", | 20260505 08:10:16.756 | INFO | "huge" : "", | 20260505 08:10:16.756 | INFO | "huge_shared" : " 27 GB", | 20260505 08:10:16.756 | INFO | "malloc" : " 14 GB", | 20260505 08:10:16.756 | INFO | "pinned" : " 29 GB" | 20260505 08:10:16.756 | INFO | } | 20260505 08:10:16.757 | INFO | Perf report for the last 5.13835 seconds: | 20260505 08:10:16.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:10:16.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:16.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:16.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:16.757 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 08:10:16.757 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:16.757 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:16.757 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 08:10:16.757 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:16.757 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:16.757 | INFO | Uploaded 36GiB to GPU in 2.20651s (16.3154GiB/s) | 20260505 08:10:16.757 | INFO | Downloaded 34GiB to GPU in 2.14735s (15.8335GiB/s) | 20260505 08:10:21.897 | INFO | Analyzing chunk frames = [1666560,1667072) | 20260505 08:10:21.897 | INFO | MemInfo: { | 20260505 08:10:21.897 | INFO | "gpu" : " 17 GB", | 20260505 08:10:21.897 | INFO | "huge" : "", | 20260505 08:10:21.897 | INFO | "huge_shared" : " 27 GB", | 20260505 08:10:21.897 | INFO | "malloc" : " 14 GB", | 20260505 08:10:21.897 | INFO | "pinned" : " 29 GB" | 20260505 08:10:21.897 | INFO | } | 20260505 08:10:21.897 | INFO | Perf report for the last 5.14034 seconds: | 20260505 08:10:21.897 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:10:21.897 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:21.897 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:21.897 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:21.897 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 08:10:21.897 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:21.897 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:21.897 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 08:10:21.897 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:21.897 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:21.897 | INFO | Uploaded 36GiB to GPU in 2.19767s (16.381GiB/s) | 20260505 08:10:21.897 | INFO | Downloaded 34GiB to GPU in 2.04387s (16.6351GiB/s) | 20260505 08:10:22.037 | INFO | Starting a round of Baz IO | 20260505 08:10:25.326 | INFO | Populating RTMetrics report | 20260505 08:10:25.327 | INFO | RTMetrics report populated | 20260505 08:10:25.327 | INFO | Opening tmp RTMetrics json file | 20260505 08:10:25.327 | INFO | Serializing RTMetrics to json | 20260505 08:10:25.329 | INFO | Serialization done, swapping tmp file now... | 20260505 08:10:25.329 | INFO | RTMetrics done | 20260505 08:10:25.477 | INFO | Finished a round of Baz IO: Wrote 2.451 GiB in 3.439 seconds | 20260505 08:10:25.477 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 28.18% / 0.02% / 71.80% | 20260505 08:10:25.477 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.88996% / 3.17891e-05% / 5.5631e-05% | 20260505 08:10:25.477 | INFO | Baz IO overall: | 20260505 08:10:25.477 | INFO | BytesWritten : 933067847426 | 20260505 08:10:25.477 | INFO | NumEvents : 443341745236 | 20260505 08:10:27.036 | INFO | Analyzing chunk frames = [1667072,1667584) | 20260505 08:10:27.036 | INFO | MemInfo: { | 20260505 08:10:27.036 | INFO | "gpu" : " 17 GB", | 20260505 08:10:27.036 | INFO | "huge" : "", | 20260505 08:10:27.036 | INFO | "huge_shared" : " 27 GB", | 20260505 08:10:27.036 | INFO | "malloc" : " 14 GB", | 20260505 08:10:27.036 | INFO | "pinned" : " 29 GB" | 20260505 08:10:27.036 | INFO | } | 20260505 08:10:27.036 | INFO | Perf report for the last 5.13915 seconds: | 20260505 08:10:27.036 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:10:27.036 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:27.036 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:27.036 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:27.036 | INFO | ANALYSIS : 65.6%, 2.9, 100.3ms, 0 | 20260505 08:10:27.036 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:27.036 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260505 08:10:27.036 | INFO | PRE_HQ : 63.4%, 1.9, 61.2ms, 0 | 20260505 08:10:27.036 | INFO | BAZWRITER : 66.2%, 4.2, 142.2ms, 0 | 20260505 08:10:27.036 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 08:10:27.036 | INFO | Uploaded 36GiB to GPU in 2.2339s (16.1153GiB/s) | 20260505 08:10:27.036 | INFO | Downloaded 38GiB to GPU in 2.35207s (16.156GiB/s) | 20260505 08:10:32.175 | INFO | Analyzing chunk frames = [1667584,1668096) | 20260505 08:10:32.175 | INFO | MemInfo: { | 20260505 08:10:32.175 | INFO | "gpu" : " 17 GB", | 20260505 08:10:32.175 | INFO | "huge" : "", | 20260505 08:10:32.175 | INFO | "huge_shared" : " 27 GB", | 20260505 08:10:32.175 | INFO | "malloc" : " 14 GB", | 20260505 08:10:32.175 | INFO | "pinned" : " 29 GB" | 20260505 08:10:32.175 | INFO | } | 20260505 08:10:32.175 | INFO | Perf report for the last 5.13866 seconds: | 20260505 08:10:32.175 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:10:32.175 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:32.175 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:32.175 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:32.175 | INFO | ANALYSIS : 64.9%, 3.0, 101.3ms, 0 | 20260505 08:10:32.175 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:32.175 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:32.175 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:10:32.175 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:32.175 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:32.175 | INFO | Uploaded 36GiB to GPU in 2.24726s (16.0195GiB/s) | 20260505 08:10:32.175 | INFO | Downloaded 34GiB to GPU in 2.09417s (16.2355GiB/s) | 20260505 08:10:37.314 | INFO | Analyzing chunk frames = [1668096,1668608) | 20260505 08:10:37.314 | INFO | MemInfo: { | 20260505 08:10:37.314 | INFO | "gpu" : " 17 GB", | 20260505 08:10:37.314 | INFO | "huge" : "", | 20260505 08:10:37.314 | INFO | "huge_shared" : " 27 GB", | 20260505 08:10:37.314 | INFO | "malloc" : " 14 GB", | 20260505 08:10:37.314 | INFO | "pinned" : " 29 GB" | 20260505 08:10:37.314 | INFO | } | 20260505 08:10:37.314 | INFO | Perf report for the last 5.13919 seconds: | 20260505 08:10:37.314 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:10:37.314 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:37.314 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:37.314 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:37.314 | INFO | ANALYSIS : 63.8%, 3.0, 99.3ms, 0 | 20260505 08:10:37.314 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:37.314 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:37.314 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:10:37.314 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:37.314 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:37.314 | INFO | Uploaded 36GiB to GPU in 2.16957s (16.5931GiB/s) | 20260505 08:10:37.314 | INFO | Downloaded 34GiB to GPU in 2.0635s (16.4769GiB/s) | 20260505 08:10:42.453 | INFO | Analyzing chunk frames = [1668608,1669120) | 20260505 08:10:42.453 | INFO | MemInfo: { | 20260505 08:10:42.453 | INFO | "gpu" : " 17 GB", | 20260505 08:10:42.453 | INFO | "huge" : "", | 20260505 08:10:42.453 | INFO | "huge_shared" : " 27 GB", | 20260505 08:10:42.453 | INFO | "malloc" : " 14 GB", | 20260505 08:10:42.453 | INFO | "pinned" : " 29 GB" | 20260505 08:10:42.453 | INFO | } | 20260505 08:10:42.453 | INFO | Perf report for the last 5.13933 seconds: | 20260505 08:10:42.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:10:42.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:42.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:42.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:42.454 | INFO | ANALYSIS : 63.7%, 3.0, 98.9ms, 0 | 20260505 08:10:42.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:42.454 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:42.454 | INFO | PRE_HQ : 29.8%, 1.0, 15.4ms, 0 | 20260505 08:10:42.454 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:42.454 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:42.454 | INFO | Uploaded 36GiB to GPU in 2.19194s (16.4238GiB/s) | 20260505 08:10:42.454 | INFO | Downloaded 34GiB to GPU in 2.06388s (16.4738GiB/s) | 20260505 08:10:47.593 | INFO | Analyzing chunk frames = [1669120,1669632) | 20260505 08:10:47.593 | INFO | MemInfo: { | 20260505 08:10:47.593 | INFO | "gpu" : " 17 GB", | 20260505 08:10:47.593 | INFO | "huge" : "", | 20260505 08:10:47.593 | INFO | "huge_shared" : " 27 GB", | 20260505 08:10:47.593 | INFO | "malloc" : " 14 GB", | 20260505 08:10:47.593 | INFO | "pinned" : " 29 GB" | 20260505 08:10:47.593 | INFO | } | 20260505 08:10:47.593 | INFO | Perf report for the last 5.1392 seconds: | 20260505 08:10:47.593 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:10:47.593 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:47.593 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:47.593 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:47.593 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 08:10:47.593 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:47.593 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:47.593 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 08:10:47.593 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:47.593 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:47.593 | INFO | Uploaded 36GiB to GPU in 2.18116s (16.505GiB/s) | 20260505 08:10:47.593 | INFO | Downloaded 34GiB to GPU in 2.05742s (16.5256GiB/s) | 20260505 08:10:52.733 | INFO | Analyzing chunk frames = [1669632,1670144) | 20260505 08:10:52.733 | INFO | MemInfo: { | 20260505 08:10:52.733 | INFO | "gpu" : " 17 GB", | 20260505 08:10:52.733 | INFO | "huge" : "", | 20260505 08:10:52.733 | INFO | "huge_shared" : " 27 GB", | 20260505 08:10:52.733 | INFO | "malloc" : " 14 GB", | 20260505 08:10:52.733 | INFO | "pinned" : " 29 GB" | 20260505 08:10:52.733 | INFO | } | 20260505 08:10:52.733 | INFO | Perf report for the last 5.14047 seconds: | 20260505 08:10:52.733 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:10:52.733 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:52.733 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:52.733 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:52.733 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 08:10:52.733 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:52.733 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:52.733 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:10:52.733 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:52.733 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:52.733 | INFO | Uploaded 36GiB to GPU in 2.18391s (16.4842GiB/s) | 20260505 08:10:52.733 | INFO | Downloaded 34GiB to GPU in 2.04819s (16.6GiB/s) | 20260505 08:10:57.872 | INFO | Analyzing chunk frames = [1670144,1670656) | 20260505 08:10:57.872 | INFO | MemInfo: { | 20260505 08:10:57.872 | INFO | "gpu" : " 17 GB", | 20260505 08:10:57.872 | INFO | "huge" : "", | 20260505 08:10:57.872 | INFO | "huge_shared" : " 27 GB", | 20260505 08:10:57.872 | INFO | "malloc" : " 14 GB", | 20260505 08:10:57.872 | INFO | "pinned" : " 29 GB" | 20260505 08:10:57.872 | INFO | } | 20260505 08:10:57.872 | INFO | Perf report for the last 5.13883 seconds: | 20260505 08:10:57.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:10:57.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:57.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:57.872 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:57.872 | INFO | ANALYSIS : 64.1%, 3.0, 100.0ms, 0 | 20260505 08:10:57.872 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:57.872 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:57.872 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 08:10:57.872 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:57.872 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:10:57.872 | INFO | Uploaded 36GiB to GPU in 2.2042s (16.3325GiB/s) | 20260505 08:10:57.872 | INFO | Downloaded 34GiB to GPU in 2.15083s (15.8079GiB/s) | 20260505 08:11:03.010 | INFO | Analyzing chunk frames = [1670656,1671168) | 20260505 08:11:03.010 | INFO | MemInfo: { | 20260505 08:11:03.010 | INFO | "gpu" : " 17 GB", | 20260505 08:11:03.010 | INFO | "huge" : "", | 20260505 08:11:03.010 | INFO | "huge_shared" : " 27 GB", | 20260505 08:11:03.010 | INFO | "malloc" : " 14 GB", | 20260505 08:11:03.010 | INFO | "pinned" : " 29 GB" | 20260505 08:11:03.010 | INFO | } | 20260505 08:11:03.010 | INFO | Perf report for the last 5.13843 seconds: | 20260505 08:11:03.010 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:11:03.010 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:03.010 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:03.010 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:03.010 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 08:11:03.010 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:03.010 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:03.010 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 08:11:03.010 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:03.010 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:03.011 | INFO | Uploaded 36GiB to GPU in 2.19779s (16.3801GiB/s) | 20260505 08:11:03.011 | INFO | Downloaded 34GiB to GPU in 2.04415s (16.6328GiB/s) | 20260505 08:11:03.011 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 08:11:03.011 | INFO | Total mem usage: 15.3 GiB | 20260505 08:11:03.011 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 08:11:03.011 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.25%) | 20260505 08:11:03.011 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 08:11:03.011 | INFO | High Water Breakdown: | 20260505 08:11:03.011 | INFO | 60.8% : 8.77 GiB ( 93.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 08:11:03.011 | INFO | 39.1% : 5.63 GiB ( 84.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 08:11:03.011 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 08:11:03.011 | INFO | ------------------------------------------ | 20260505 08:11:03.011 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:11:03.011 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 08:11:03.011 | INFO | Total mem usage: 26.5 GiB | 20260505 08:11:03.011 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 08:11:03.011 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 08:11:03.011 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 08:11:03.011 | INFO | High Water Breakdown: | 20260505 08:11:03.011 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:11:03.011 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:11:03.011 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:11:03.011 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:11:03.011 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:11:03.011 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:11:03.011 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 08:11:03.011 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 08:11:03.011 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:11:03.011 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 08:11:03.011 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 08:11:03.011 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 08:11:03.011 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 08:11:03.011 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 08:11:03.011 | INFO | ------------------------------------------ | 20260505 08:11:03.011 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:11:03.011 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 08:11:03.011 | INFO | Total mem usage: 25.6 GiB | 20260505 08:11:03.011 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 08:11:03.011 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 08:11:03.011 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 08:11:03.011 | INFO | High Water Breakdown: | 20260505 08:11:03.011 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 08:11:03.011 | INFO | ------------------------------------------ | 20260505 08:11:03.011 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:11:03.012 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 08:11:03.012 | INFO | Total mem usage: 15.7 GiB | 20260505 08:11:03.012 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 08:11:03.012 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 08:11:03.012 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 08:11:03.012 | INFO | High Water Breakdown: | 20260505 08:11:03.012 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 08:11:03.012 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 08:11:03.012 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 08:11:03.012 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 08:11:03.012 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 08:11:03.012 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 08:11:03.012 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 08:11:03.012 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 08:11:03.012 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 08:11:03.012 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:11:03.012 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 08:11:03.012 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:11:03.012 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 08:11:03.012 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 08:11:03.012 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 08:11:03.012 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 08:11:03.012 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:11:03.012 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 08:11:03.012 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:11:03.012 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:11:03.012 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:11:03.012 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 08:11:03.012 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 08:11:03.012 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:11:03.012 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 08:11:03.012 | INFO | ------------------------------------------ | 20260505 08:11:03.012 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:11:03.012 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 08:11:03.012 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:11:03.012 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:11:03.012 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.36e+01 / 2.04e+00 | 20260505 08:11:03.012 | INFO | PerfReport: 0.28%, AggregateStats -- 9.29e-02 / 1.04e-01, 1.01e-02 / 5.37e-03 | 20260505 08:11:03.012 | INFO | PerfReport: 11.62%, Baseline -- 3.80e+00 / 4.45e+00, 6.24e-01 / 2.98e-01 | 20260505 08:11:03.012 | INFO | PerfReport: 3.32%, Histogram -- 1.08e+00 / 1.49e+00, 2.13e-01 / 1.19e-01 | 20260505 08:11:03.012 | INFO | PerfReport: 15.56%, DME -- 4.07e+01 / 4.46e+01, 4.22e+00 / 1.64e+00 | 20260505 08:11:03.012 | INFO | PerfReport: 56.47%, FrameLabeling -- 1.85e+01 / 1.95e+01, 1.97e+00 / 1.71e+00 | 20260505 08:11:03.012 | INFO | PerfReport: 7.46%, PulseAccumulating -- 2.44e+00 / 3.09e+00, 4.08e-01 / 2.41e-01 | 20260505 08:11:03.012 | INFO | PerfReport: 3.88%, Metrics -- 1.27e+00 / 1.58e+00, 3.57e-01 / 1.87e-01 | 20260505 08:11:03.012 | INFO | PerfReport: 1.40%, PulseSerializing -- 4.59e-01 / 5.11e-01, 8.82e-02 / 3.22e-02 | 20260505 08:11:03.012 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 08:11:03.012 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:11:03.012 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:11:03.012 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.30e+01 / 0.00e+00 | 20260505 08:11:03.012 | INFO | PerfReport: 22.50%, Upload -- 2.29e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260505 08:11:03.012 | INFO | PerfReport: 21.56%, Download -- 2.19e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260505 08:11:03.012 | INFO | PerfReport: 32.13%, Compute -- 3.27e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 08:11:03.012 | INFO | PerfReport: 1.66%, LockUpload -- 1.69e+00 / 0.00e+00, 7.32e+00 / 0.00e+00 | 20260505 08:11:03.012 | INFO | PerfReport: 1.02%, LockDownload -- 1.04e+00 / 0.00e+00, 2.51e+00 / 0.00e+00 | 20260505 08:11:03.012 | INFO | PerfReport: 21.12%, LockCompute -- 2.15e+01 / 0.00e+00, 1.78e+01 / 0.00e+00 | 20260505 08:11:03.012 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.13e-02 / 0.00e+00 | 20260505 08:11:03.152 | INFO | Starting a round of Baz IO | 20260505 08:11:06.446 | INFO | Populating RTMetrics report | 20260505 08:11:06.447 | INFO | RTMetrics report populated | 20260505 08:11:06.447 | INFO | Opening tmp RTMetrics json file | 20260505 08:11:06.469 | INFO | Serializing RTMetrics to json | 20260505 08:11:06.471 | INFO | Serialization done, swapping tmp file now... | 20260505 08:11:06.473 | INFO | RTMetrics done | 20260505 08:11:06.663 | INFO | Finished a round of Baz IO: Wrote 2.447 GiB in 3.51 seconds | 20260505 08:11:06.663 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 28.15% / 0.02% / 71.82% | 20260505 08:11:06.663 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.86719% / 1.98682e-05% / 6.35783e-05% | 20260505 08:11:06.663 | INFO | Baz IO overall: | 20260505 08:11:06.663 | INFO | BytesWritten : 935695820822 | 20260505 08:11:06.663 | INFO | NumEvents : 444598040462 | 20260505 08:11:08.150 | INFO | Analyzing chunk frames = [1671168,1671680) | 20260505 08:11:08.150 | INFO | MemInfo: { | 20260505 08:11:08.150 | INFO | "gpu" : " 17 GB", | 20260505 08:11:08.150 | INFO | "huge" : "", | 20260505 08:11:08.150 | INFO | "huge_shared" : " 27 GB", | 20260505 08:11:08.150 | INFO | "malloc" : " 14 GB", | 20260505 08:11:08.150 | INFO | "pinned" : " 29 GB" | 20260505 08:11:08.150 | INFO | } | 20260505 08:11:08.150 | INFO | Perf report for the last 5.13992 seconds: | 20260505 08:11:08.150 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:11:08.150 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:08.150 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:08.150 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:08.150 | INFO | ANALYSIS : 65.7%, 2.9, 100.3ms, 0 | 20260505 08:11:08.150 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:08.150 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260505 08:11:08.150 | INFO | PRE_HQ : 63.5%, 1.9, 61.7ms, 0 | 20260505 08:11:08.150 | INFO | BAZWRITER : 67.6%, 4.2, 144.4ms, 0 | 20260505 08:11:08.150 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 08:11:08.151 | INFO | Uploaded 36GiB to GPU in 2.25727s (15.9485GiB/s) | 20260505 08:11:08.151 | INFO | Downloaded 38GiB to GPU in 2.35352s (16.1461GiB/s) | 20260505 08:11:13.290 | INFO | Analyzing chunk frames = [1671680,1672192) | 20260505 08:11:13.290 | INFO | MemInfo: { | 20260505 08:11:13.290 | INFO | "gpu" : " 17 GB", | 20260505 08:11:13.290 | INFO | "huge" : "", | 20260505 08:11:13.290 | INFO | "huge_shared" : " 27 GB", | 20260505 08:11:13.290 | INFO | "malloc" : " 14 GB", | 20260505 08:11:13.290 | INFO | "pinned" : " 29 GB" | 20260505 08:11:13.290 | INFO | } | 20260505 08:11:13.290 | INFO | Perf report for the last 5.13957 seconds: | 20260505 08:11:13.290 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:11:13.290 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:13.290 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:13.290 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:13.290 | INFO | ANALYSIS : 64.6%, 3.0, 100.9ms, 0 | 20260505 08:11:13.290 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:13.290 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:13.290 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 08:11:13.290 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:13.290 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:13.290 | INFO | Uploaded 36GiB to GPU in 2.24732s (16.0191GiB/s) | 20260505 08:11:13.290 | INFO | Downloaded 34GiB to GPU in 2.10374s (16.1617GiB/s) | 20260505 08:11:18.429 | INFO | Analyzing chunk frames = [1672192,1672704) | 20260505 08:11:18.429 | INFO | MemInfo: { | 20260505 08:11:18.429 | INFO | "gpu" : " 17 GB", | 20260505 08:11:18.429 | INFO | "huge" : "", | 20260505 08:11:18.429 | INFO | "huge_shared" : " 27 GB", | 20260505 08:11:18.429 | INFO | "malloc" : " 14 GB", | 20260505 08:11:18.429 | INFO | "pinned" : " 29 GB" | 20260505 08:11:18.429 | INFO | } | 20260505 08:11:18.429 | INFO | Perf report for the last 5.13931 seconds: | 20260505 08:11:18.429 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:11:18.429 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:18.429 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:18.429 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:18.429 | INFO | ANALYSIS : 63.8%, 3.0, 99.2ms, 0 | 20260505 08:11:18.429 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:18.429 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:18.429 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 08:11:18.429 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:18.429 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:18.429 | INFO | Uploaded 36GiB to GPU in 2.16686s (16.6139GiB/s) | 20260505 08:11:18.430 | INFO | Downloaded 34GiB to GPU in 2.0647s (16.4673GiB/s) | 20260505 08:11:23.568 | INFO | Analyzing chunk frames = [1672704,1673216) | 20260505 08:11:23.568 | INFO | MemInfo: { | 20260505 08:11:23.568 | INFO | "gpu" : " 17 GB", | 20260505 08:11:23.568 | INFO | "huge" : "", | 20260505 08:11:23.568 | INFO | "huge_shared" : " 27 GB", | 20260505 08:11:23.568 | INFO | "malloc" : " 14 GB", | 20260505 08:11:23.568 | INFO | "pinned" : " 29 GB" | 20260505 08:11:23.568 | INFO | } | 20260505 08:11:23.568 | INFO | Perf report for the last 5.13849 seconds: | 20260505 08:11:23.568 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:11:23.568 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:23.568 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:23.568 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:23.568 | INFO | ANALYSIS : 63.6%, 3.0, 98.8ms, 0 | 20260505 08:11:23.568 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:23.568 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:23.568 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 08:11:23.568 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:23.568 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:23.568 | INFO | Uploaded 36GiB to GPU in 2.17938s (16.5185GiB/s) | 20260505 08:11:23.568 | INFO | Downloaded 34GiB to GPU in 2.06505s (16.4645GiB/s) | 20260505 08:11:28.707 | INFO | Analyzing chunk frames = [1673216,1673728) | 20260505 08:11:28.707 | INFO | MemInfo: { | 20260505 08:11:28.707 | INFO | "gpu" : " 17 GB", | 20260505 08:11:28.707 | INFO | "huge" : "", | 20260505 08:11:28.707 | INFO | "huge_shared" : " 27 GB", | 20260505 08:11:28.707 | INFO | "malloc" : " 14 GB", | 20260505 08:11:28.707 | INFO | "pinned" : " 29 GB" | 20260505 08:11:28.707 | INFO | } | 20260505 08:11:28.707 | INFO | Perf report for the last 5.13948 seconds: | 20260505 08:11:28.707 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:11:28.707 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:28.707 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:28.707 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:28.707 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 08:11:28.707 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:28.707 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:28.707 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:11:28.707 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:28.707 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:28.707 | INFO | Uploaded 36GiB to GPU in 2.1806s (16.5092GiB/s) | 20260505 08:11:28.708 | INFO | Downloaded 34GiB to GPU in 2.05262s (16.5642GiB/s) | 20260505 08:11:33.848 | INFO | Analyzing chunk frames = [1673728,1674240) | 20260505 08:11:33.848 | INFO | MemInfo: { | 20260505 08:11:33.848 | INFO | "gpu" : " 17 GB", | 20260505 08:11:33.848 | INFO | "huge" : "", | 20260505 08:11:33.848 | INFO | "huge_shared" : " 27 GB", | 20260505 08:11:33.848 | INFO | "malloc" : " 14 GB", | 20260505 08:11:33.848 | INFO | "pinned" : " 29 GB" | 20260505 08:11:33.848 | INFO | } | 20260505 08:11:33.848 | INFO | Perf report for the last 5.1406 seconds: | 20260505 08:11:33.848 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:11:33.848 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:33.848 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:33.848 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:33.848 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 08:11:33.848 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:33.848 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:33.848 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 08:11:33.848 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:33.848 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:33.848 | INFO | Uploaded 36GiB to GPU in 2.17688s (16.5374GiB/s) | 20260505 08:11:33.848 | INFO | Downloaded 34GiB to GPU in 2.04271s (16.6446GiB/s) | 20260505 08:11:38.986 | INFO | Analyzing chunk frames = [1674240,1674752) | 20260505 08:11:38.986 | INFO | MemInfo: { | 20260505 08:11:38.986 | INFO | "gpu" : " 17 GB", | 20260505 08:11:38.986 | INFO | "huge" : "", | 20260505 08:11:38.986 | INFO | "huge_shared" : " 27 GB", | 20260505 08:11:38.986 | INFO | "malloc" : " 14 GB", | 20260505 08:11:38.987 | INFO | "pinned" : " 29 GB" | 20260505 08:11:38.987 | INFO | } | 20260505 08:11:38.987 | INFO | Perf report for the last 5.13876 seconds: | 20260505 08:11:38.987 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:11:38.987 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:38.987 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:38.987 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:38.987 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 08:11:38.987 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:38.987 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:38.987 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260505 08:11:38.987 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:38.987 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:38.987 | INFO | Uploaded 36GiB to GPU in 2.2011s (16.3554GiB/s) | 20260505 08:11:38.987 | INFO | Downloaded 34GiB to GPU in 2.15798s (15.7555GiB/s) | 20260505 08:11:44.125 | INFO | Analyzing chunk frames = [1674752,1675264) | 20260505 08:11:44.125 | INFO | MemInfo: { | 20260505 08:11:44.125 | INFO | "gpu" : " 17 GB", | 20260505 08:11:44.125 | INFO | "huge" : "", | 20260505 08:11:44.125 | INFO | "huge_shared" : " 27 GB", | 20260505 08:11:44.125 | INFO | "malloc" : " 14 GB", | 20260505 08:11:44.125 | INFO | "pinned" : " 29 GB" | 20260505 08:11:44.125 | INFO | } | 20260505 08:11:44.125 | INFO | Perf report for the last 5.13887 seconds: | 20260505 08:11:44.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:11:44.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:44.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:44.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:44.125 | INFO | ANALYSIS : 64.3%, 3.0, 100.3ms, 0 | 20260505 08:11:44.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:44.126 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:44.126 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 08:11:44.126 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:44.126 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:44.126 | INFO | Uploaded 36GiB to GPU in 2.19883s (16.3723GiB/s) | 20260505 08:11:44.126 | INFO | Downloaded 34GiB to GPU in 2.04798s (16.6017GiB/s) | 20260505 08:11:44.266 | INFO | Starting a round of Baz IO | 20260505 08:11:47.553 | INFO | Populating RTMetrics report | 20260505 08:11:47.554 | INFO | RTMetrics report populated | 20260505 08:11:47.554 | INFO | Opening tmp RTMetrics json file | 20260505 08:11:47.628 | INFO | Serializing RTMetrics to json | 20260505 08:11:47.634 | INFO | Serialization done, swapping tmp file now... | 20260505 08:11:47.636 | INFO | RTMetrics done | 20260505 08:11:47.781 | INFO | Finished a round of Baz IO: Wrote 2.447 GiB in 3.515 seconds | 20260505 08:11:47.781 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 28.13% / 0.02% / 71.85% | 20260505 08:11:47.781 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.85736% / 4.37101e-05% / 6.35783e-05% | 20260505 08:11:47.781 | INFO | Baz IO overall: | 20260505 08:11:47.781 | INFO | BytesWritten : 938323400225 | 20260505 08:11:47.781 | INFO | NumEvents : 445853947372 | 20260505 08:11:49.264 | INFO | Analyzing chunk frames = [1675264,1675776) | 20260505 08:11:49.264 | INFO | MemInfo: { | 20260505 08:11:49.264 | INFO | "gpu" : " 17 GB", | 20260505 08:11:49.264 | INFO | "huge" : "", | 20260505 08:11:49.264 | INFO | "huge_shared" : " 27 GB", | 20260505 08:11:49.264 | INFO | "malloc" : " 14 GB", | 20260505 08:11:49.264 | INFO | "pinned" : " 29 GB" | 20260505 08:11:49.264 | INFO | } | 20260505 08:11:49.264 | INFO | Perf report for the last 5.13901 seconds: | 20260505 08:11:49.264 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:11:49.264 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:49.264 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:49.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:49.265 | INFO | ANALYSIS : 65.6%, 2.9, 100.2ms, 0 | 20260505 08:11:49.265 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:49.265 | INFO | RT_METRICS : 47.5%, 1.0, 24.9ms, 0 | 20260505 08:11:49.265 | INFO | PRE_HQ : 63.6%, 1.9, 61.5ms, 0 | 20260505 08:11:49.265 | INFO | BAZWRITER : 67.7%, 4.2, 144.8ms, 0 | 20260505 08:11:49.265 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260505 08:11:49.265 | INFO | Uploaded 36GiB to GPU in 2.28503s (15.7547GiB/s) | 20260505 08:11:49.265 | INFO | Downloaded 38GiB to GPU in 2.36965s (16.0361GiB/s) | 20260505 08:11:54.404 | INFO | Analyzing chunk frames = [1675776,1676288) | 20260505 08:11:54.404 | INFO | MemInfo: { | 20260505 08:11:54.404 | INFO | "gpu" : " 17 GB", | 20260505 08:11:54.404 | INFO | "huge" : "", | 20260505 08:11:54.405 | INFO | "huge_shared" : " 27 GB", | 20260505 08:11:54.405 | INFO | "malloc" : " 14 GB", | 20260505 08:11:54.405 | INFO | "pinned" : " 29 GB" | 20260505 08:11:54.405 | INFO | } | 20260505 08:11:54.405 | INFO | Perf report for the last 5.14028 seconds: | 20260505 08:11:54.405 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:11:54.405 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:54.405 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:54.405 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:54.405 | INFO | ANALYSIS : 64.8%, 3.0, 101.2ms, 0 | 20260505 08:11:54.405 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:54.405 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:54.405 | INFO | PRE_HQ : 30.5%, 1.0, 16.0ms, 0 | 20260505 08:11:54.405 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:54.405 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:54.405 | INFO | Uploaded 36GiB to GPU in 2.24694s (16.0218GiB/s) | 20260505 08:11:54.405 | INFO | Downloaded 34GiB to GPU in 2.10178s (16.1767GiB/s) | 20260505 08:11:59.543 | INFO | Analyzing chunk frames = [1676288,1676800) | 20260505 08:11:59.543 | INFO | MemInfo: { | 20260505 08:11:59.543 | INFO | "gpu" : " 17 GB", | 20260505 08:11:59.543 | INFO | "huge" : "", | 20260505 08:11:59.543 | INFO | "huge_shared" : " 27 GB", | 20260505 08:11:59.543 | INFO | "malloc" : " 14 GB", | 20260505 08:11:59.543 | INFO | "pinned" : " 29 GB" | 20260505 08:11:59.543 | INFO | } | 20260505 08:11:59.543 | INFO | Perf report for the last 5.13847 seconds: | 20260505 08:11:59.543 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:11:59.543 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:59.543 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:59.543 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:59.543 | INFO | ANALYSIS : 63.7%, 3.0, 99.1ms, 0 | 20260505 08:11:59.543 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:59.543 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:59.543 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:11:59.544 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:59.544 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:11:59.544 | INFO | Uploaded 36GiB to GPU in 2.17366s (16.5619GiB/s) | 20260505 08:11:59.544 | INFO | Downloaded 34GiB to GPU in 2.0655s (16.4609GiB/s) | 20260505 08:12:04.683 | INFO | Analyzing chunk frames = [1676800,1677312) | 20260505 08:12:04.683 | INFO | MemInfo: { | 20260505 08:12:04.683 | INFO | "gpu" : " 17 GB", | 20260505 08:12:04.683 | INFO | "huge" : "", | 20260505 08:12:04.683 | INFO | "huge_shared" : " 27 GB", | 20260505 08:12:04.683 | INFO | "malloc" : " 14 GB", | 20260505 08:12:04.683 | INFO | "pinned" : " 29 GB" | 20260505 08:12:04.683 | INFO | } | 20260505 08:12:04.683 | INFO | Perf report for the last 5.13987 seconds: | 20260505 08:12:04.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:12:04.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:04.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:04.683 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:04.683 | INFO | ANALYSIS : 63.7%, 3.0, 99.0ms, 0 | 20260505 08:12:04.683 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:04.683 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:04.683 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 08:12:04.683 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:04.683 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:04.683 | INFO | Uploaded 36GiB to GPU in 2.18977s (16.4401GiB/s) | 20260505 08:12:04.683 | INFO | Downloaded 34GiB to GPU in 2.06819s (16.4395GiB/s) | 20260505 08:12:09.822 | INFO | Analyzing chunk frames = [1677312,1677824) | 20260505 08:12:09.822 | INFO | MemInfo: { | 20260505 08:12:09.822 | INFO | "gpu" : " 17 GB", | 20260505 08:12:09.822 | INFO | "huge" : "", | 20260505 08:12:09.822 | INFO | "huge_shared" : " 27 GB", | 20260505 08:12:09.822 | INFO | "malloc" : " 14 GB", | 20260505 08:12:09.822 | INFO | "pinned" : " 29 GB" | 20260505 08:12:09.822 | INFO | } | 20260505 08:12:09.822 | INFO | Perf report for the last 5.13919 seconds: | 20260505 08:12:09.822 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:12:09.822 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:09.822 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:09.822 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:09.822 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 08:12:09.822 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:09.822 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:09.822 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:12:09.822 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:09.822 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:09.823 | INFO | Uploaded 36GiB to GPU in 2.17663s (16.5394GiB/s) | 20260505 08:12:09.823 | INFO | Downloaded 34GiB to GPU in 2.06096s (16.4972GiB/s) | 20260505 08:12:14.962 | INFO | Analyzing chunk frames = [1677824,1678336) | 20260505 08:12:14.962 | INFO | MemInfo: { | 20260505 08:12:14.962 | INFO | "gpu" : " 17 GB", | 20260505 08:12:14.962 | INFO | "huge" : "", | 20260505 08:12:14.962 | INFO | "huge_shared" : " 27 GB", | 20260505 08:12:14.962 | INFO | "malloc" : " 14 GB", | 20260505 08:12:14.962 | INFO | "pinned" : " 29 GB" | 20260505 08:12:14.962 | INFO | } | 20260505 08:12:14.962 | INFO | Perf report for the last 5.14003 seconds: | 20260505 08:12:14.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:12:14.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:14.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:14.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:14.962 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 08:12:14.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:14.963 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:14.963 | INFO | PRE_HQ : 29.9%, 1.0, 15.6ms, 0 | 20260505 08:12:14.963 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:14.963 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:14.963 | INFO | Uploaded 36GiB to GPU in 2.19653s (16.3895GiB/s) | 20260505 08:12:14.963 | INFO | Downloaded 34GiB to GPU in 2.04684s (16.611GiB/s) | 20260505 08:12:20.102 | INFO | Analyzing chunk frames = [1678336,1678848) | 20260505 08:12:20.102 | INFO | MemInfo: { | 20260505 08:12:20.102 | INFO | "gpu" : " 17 GB", | 20260505 08:12:20.102 | INFO | "huge" : "", | 20260505 08:12:20.102 | INFO | "huge_shared" : " 27 GB", | 20260505 08:12:20.102 | INFO | "malloc" : " 14 GB", | 20260505 08:12:20.102 | INFO | "pinned" : " 29 GB" | 20260505 08:12:20.102 | INFO | } | 20260505 08:12:20.102 | INFO | Perf report for the last 5.13927 seconds: | 20260505 08:12:20.102 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:12:20.102 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:20.102 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:20.102 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:20.102 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 08:12:20.102 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:20.102 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:20.102 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 08:12:20.102 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:20.102 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:20.102 | INFO | Uploaded 36GiB to GPU in 2.22392s (16.1876GiB/s) | 20260505 08:12:20.102 | INFO | Downloaded 34GiB to GPU in 2.15571s (15.772GiB/s) | 20260505 08:12:25.241 | INFO | Analyzing chunk frames = [1678848,1679360) | 20260505 08:12:25.241 | INFO | MemInfo: { | 20260505 08:12:25.241 | INFO | "gpu" : " 17 GB", | 20260505 08:12:25.241 | INFO | "huge" : "", | 20260505 08:12:25.241 | INFO | "huge_shared" : " 27 GB", | 20260505 08:12:25.241 | INFO | "malloc" : " 14 GB", | 20260505 08:12:25.241 | INFO | "pinned" : " 29 GB" | 20260505 08:12:25.241 | INFO | } | 20260505 08:12:25.241 | INFO | Perf report for the last 5.13943 seconds: | 20260505 08:12:25.241 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:12:25.241 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:25.241 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:25.241 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:25.241 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 08:12:25.241 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:25.241 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:25.241 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 08:12:25.241 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:25.241 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:25.241 | INFO | Uploaded 36GiB to GPU in 2.19185s (16.4245GiB/s) | 20260505 08:12:25.241 | INFO | Downloaded 34GiB to GPU in 2.04347s (16.6384GiB/s) | 20260505 08:12:25.242 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 08:12:25.242 | INFO | Total mem usage: 15.3 GiB | 20260505 08:12:25.242 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 08:12:25.242 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -5.11%) | 20260505 08:12:25.242 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 08:12:25.242 | INFO | High Water Breakdown: | 20260505 08:12:25.242 | INFO | 60.9% : 8.88 GiB ( 94.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 08:12:25.242 | INFO | 39% : 5.68 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 08:12:25.242 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 08:12:25.242 | INFO | ------------------------------------------ | 20260505 08:12:25.242 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:12:25.242 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 08:12:25.242 | INFO | Total mem usage: 26.5 GiB | 20260505 08:12:25.242 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 08:12:25.242 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 08:12:25.242 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 08:12:25.242 | INFO | High Water Breakdown: | 20260505 08:12:25.242 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:12:25.242 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:12:25.242 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:12:25.242 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:12:25.242 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:12:25.242 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:12:25.242 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 08:12:25.242 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 08:12:25.242 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:12:25.242 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 08:12:25.242 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 08:12:25.242 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 08:12:25.242 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 08:12:25.242 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 08:12:25.242 | INFO | ------------------------------------------ | 20260505 08:12:25.242 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:12:25.242 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 08:12:25.242 | INFO | Total mem usage: 25.6 GiB | 20260505 08:12:25.242 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 08:12:25.242 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 08:12:25.242 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 08:12:25.242 | INFO | High Water Breakdown: | 20260505 08:12:25.242 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 08:12:25.242 | INFO | ------------------------------------------ | 20260505 08:12:25.242 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:12:25.242 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 08:12:25.242 | INFO | Total mem usage: 15.7 GiB | 20260505 08:12:25.242 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 08:12:25.242 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 08:12:25.242 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 08:12:25.242 | INFO | High Water Breakdown: | 20260505 08:12:25.242 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 08:12:25.242 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 08:12:25.242 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 08:12:25.242 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 08:12:25.242 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 08:12:25.242 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 08:12:25.242 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 08:12:25.242 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 08:12:25.242 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 08:12:25.242 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:12:25.242 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 08:12:25.242 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:12:25.242 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 08:12:25.242 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 08:12:25.242 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 08:12:25.242 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 08:12:25.242 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:12:25.242 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 08:12:25.242 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:12:25.242 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:12:25.242 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:12:25.242 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 08:12:25.242 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 08:12:25.242 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:12:25.242 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 08:12:25.242 | INFO | ------------------------------------------ | 20260505 08:12:25.242 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:12:25.242 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 08:12:25.242 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:12:25.242 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:12:25.242 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.55e+01, 1.36e+01 / 2.22e+00 | 20260505 08:12:25.242 | INFO | PerfReport: 0.28%, AggregateStats -- 9.29e-02 / 1.04e-01, 1.00e-02 / 4.58e-03 | 20260505 08:12:25.242 | INFO | PerfReport: 11.64%, Baseline -- 3.81e+00 / 4.49e+00, 6.23e-01 / 2.60e-01 | 20260505 08:12:25.242 | INFO | PerfReport: 3.32%, Histogram -- 1.09e+00 / 1.49e+00, 2.15e-01 / 1.11e-01 | 20260505 08:12:25.242 | INFO | PerfReport: 15.53%, DME -- 4.06e+01 / 4.46e+01, 4.19e+00 / 1.85e+00 | 20260505 08:12:25.243 | INFO | PerfReport: 56.49%, FrameLabeling -- 1.85e+01 / 1.98e+01, 1.97e+00 / 1.79e+00 | 20260505 08:12:25.243 | INFO | PerfReport: 7.43%, PulseAccumulating -- 2.43e+00 / 3.02e+00, 4.03e-01 / 2.45e-01 | 20260505 08:12:25.243 | INFO | PerfReport: 3.90%, Metrics -- 1.28e+00 / 1.57e+00, 3.58e-01 / 1.72e-01 | 20260505 08:12:25.243 | INFO | PerfReport: 1.40%, PulseSerializing -- 4.59e-01 / 5.10e-01, 8.81e-02 / 3.72e-02 | 20260505 08:12:25.243 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 08:12:25.243 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:12:25.243 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:12:25.243 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.32e+01 / 0.00e+00 | 20260505 08:12:25.243 | INFO | PerfReport: 22.56%, Upload -- 2.30e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260505 08:12:25.243 | INFO | PerfReport: 21.61%, Download -- 2.20e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260505 08:12:25.243 | INFO | PerfReport: 32.12%, Compute -- 3.27e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 08:12:25.243 | INFO | PerfReport: 1.68%, LockUpload -- 1.71e+00 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260505 08:12:25.243 | INFO | PerfReport: 0.98%, LockDownload -- 9.94e-01 / 0.00e+00, 2.38e+00 / 0.00e+00 | 20260505 08:12:25.243 | INFO | PerfReport: 21.03%, LockCompute -- 2.14e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260505 08:12:25.243 | INFO | PerfReport: 0.01%, Misc -- 1.46e-02 / 0.00e+00, 2.46e-02 / 0.00e+00 | 20260505 08:12:25.382 | INFO | Starting a round of Baz IO | 20260505 08:12:28.655 | INFO | Populating RTMetrics report | 20260505 08:12:28.655 | INFO | RTMetrics report populated | 20260505 08:12:28.655 | INFO | Opening tmp RTMetrics json file | 20260505 08:12:28.658 | INFO | Serializing RTMetrics to json | 20260505 08:12:28.681 | INFO | Serialization done, swapping tmp file now... | 20260505 08:12:28.697 | INFO | RTMetrics done | 20260505 08:12:28.801 | INFO | Finished a round of Baz IO: Wrote 2.449 GiB in 3.419 seconds | 20260505 08:12:28.801 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 28.11% / 0.02% / 71.87% | 20260505 08:12:28.801 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.85664% / 2.38419e-05% / 7.94729e-05% | 20260505 08:12:28.801 | INFO | Baz IO overall: | 20260505 08:12:28.801 | INFO | BytesWritten : 940953403827 | 20260505 08:12:28.801 | INFO | NumEvents : 447111929613 | 20260505 08:12:30.380 | INFO | Analyzing chunk frames = [1679360,1679872) | 20260505 08:12:30.380 | INFO | MemInfo: { | 20260505 08:12:30.380 | INFO | "gpu" : " 17 GB", | 20260505 08:12:30.380 | INFO | "huge" : "", | 20260505 08:12:30.380 | INFO | "huge_shared" : " 27 GB", | 20260505 08:12:30.380 | INFO | "malloc" : " 14 GB", | 20260505 08:12:30.380 | INFO | "pinned" : " 29 GB" | 20260505 08:12:30.380 | INFO | } | 20260505 08:12:30.380 | INFO | Perf report for the last 5.13907 seconds: | 20260505 08:12:30.380 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:12:30.380 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:30.380 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:30.380 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:30.380 | INFO | ANALYSIS : 65.3%, 2.9, 99.9ms, 0 | 20260505 08:12:30.380 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:30.380 | INFO | RT_METRICS : 46.5%, 1.0, 24.4ms, 0 | 20260505 08:12:30.380 | INFO | PRE_HQ : 63.4%, 1.9, 61.3ms, 0 | 20260505 08:12:30.380 | INFO | BAZWRITER : 65.8%, 4.2, 141.2ms, 0 | 20260505 08:12:30.380 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260505 08:12:30.380 | INFO | Uploaded 36GiB to GPU in 2.26194s (15.9155GiB/s) | 20260505 08:12:30.381 | INFO | Downloaded 38GiB to GPU in 2.38988s (15.9004GiB/s) | 20260505 08:12:35.518 | INFO | Analyzing chunk frames = [1679872,1680384) | 20260505 08:12:35.518 | INFO | MemInfo: { | 20260505 08:12:35.518 | INFO | "gpu" : " 17 GB", | 20260505 08:12:35.518 | INFO | "huge" : "", | 20260505 08:12:35.518 | INFO | "huge_shared" : " 27 GB", | 20260505 08:12:35.518 | INFO | "malloc" : " 14 GB", | 20260505 08:12:35.518 | INFO | "pinned" : " 29 GB" | 20260505 08:12:35.518 | INFO | } | 20260505 08:12:35.518 | INFO | Perf report for the last 5.1382 seconds: | 20260505 08:12:35.519 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:12:35.519 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:35.519 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:35.519 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:35.519 | INFO | ANALYSIS : 64.1%, 3.0, 100.0ms, 0 | 20260505 08:12:35.519 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:35.519 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:35.519 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:12:35.519 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:35.519 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:35.519 | INFO | Uploaded 36GiB to GPU in 2.17663s (16.5393GiB/s) | 20260505 08:12:35.519 | INFO | Downloaded 34GiB to GPU in 2.04363s (16.637GiB/s) | 20260505 08:12:40.658 | INFO | Analyzing chunk frames = [1680384,1680896) | 20260505 08:12:40.658 | INFO | MemInfo: { | 20260505 08:12:40.658 | INFO | "gpu" : " 17 GB", | 20260505 08:12:40.658 | INFO | "huge" : "", | 20260505 08:12:40.658 | INFO | "huge_shared" : " 27 GB", | 20260505 08:12:40.658 | INFO | "malloc" : " 14 GB", | 20260505 08:12:40.658 | INFO | "pinned" : " 29 GB" | 20260505 08:12:40.658 | INFO | } | 20260505 08:12:40.658 | INFO | Perf report for the last 5.13979 seconds: | 20260505 08:12:40.658 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:12:40.658 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:40.658 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:40.658 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:40.658 | INFO | ANALYSIS : 63.7%, 3.0, 99.1ms, 0 | 20260505 08:12:40.658 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:40.658 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:40.658 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:12:40.658 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:40.658 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:40.659 | INFO | Uploaded 36GiB to GPU in 2.16787s (16.6062GiB/s) | 20260505 08:12:40.659 | INFO | Downloaded 34GiB to GPU in 2.0643s (16.4705GiB/s) | 20260505 08:12:45.798 | INFO | Analyzing chunk frames = [1680896,1681408) | 20260505 08:12:45.798 | INFO | MemInfo: { | 20260505 08:12:45.798 | INFO | "gpu" : " 17 GB", | 20260505 08:12:45.798 | INFO | "huge" : "", | 20260505 08:12:45.798 | INFO | "huge_shared" : " 27 GB", | 20260505 08:12:45.798 | INFO | "malloc" : " 14 GB", | 20260505 08:12:45.798 | INFO | "pinned" : " 29 GB" | 20260505 08:12:45.798 | INFO | } | 20260505 08:12:45.798 | INFO | Perf report for the last 5.14007 seconds: | 20260505 08:12:45.798 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:12:45.798 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:45.798 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:45.798 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:45.798 | INFO | ANALYSIS : 63.7%, 3.0, 98.9ms, 0 | 20260505 08:12:45.798 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:45.798 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:45.798 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 08:12:45.798 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:45.798 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:45.799 | INFO | Uploaded 36GiB to GPU in 2.18732s (16.4585GiB/s) | 20260505 08:12:45.799 | INFO | Downloaded 34GiB to GPU in 2.06651s (16.4529GiB/s) | 20260505 08:12:50.937 | INFO | Analyzing chunk frames = [1681408,1681920) | 20260505 08:12:50.937 | INFO | MemInfo: { | 20260505 08:12:50.937 | INFO | "gpu" : " 17 GB", | 20260505 08:12:50.937 | INFO | "huge" : "", | 20260505 08:12:50.937 | INFO | "huge_shared" : " 27 GB", | 20260505 08:12:50.937 | INFO | "malloc" : " 14 GB", | 20260505 08:12:50.937 | INFO | "pinned" : " 29 GB" | 20260505 08:12:50.937 | INFO | } | 20260505 08:12:50.938 | INFO | Perf report for the last 5.13921 seconds: | 20260505 08:12:50.938 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:12:50.938 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:50.938 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:50.938 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:50.938 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 08:12:50.938 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:50.938 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:50.938 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:12:50.938 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:50.938 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:50.938 | INFO | Uploaded 36GiB to GPU in 2.18311s (16.4902GiB/s) | 20260505 08:12:50.938 | INFO | Downloaded 34GiB to GPU in 2.05915s (16.5117GiB/s) | 20260505 08:12:56.076 | INFO | Analyzing chunk frames = [1681920,1682432) | 20260505 08:12:56.076 | INFO | MemInfo: { | 20260505 08:12:56.076 | INFO | "gpu" : " 17 GB", | 20260505 08:12:56.076 | INFO | "huge" : "", | 20260505 08:12:56.076 | INFO | "huge_shared" : " 27 GB", | 20260505 08:12:56.076 | INFO | "malloc" : " 14 GB", | 20260505 08:12:56.076 | INFO | "pinned" : " 29 GB" | 20260505 08:12:56.076 | INFO | } | 20260505 08:12:56.076 | INFO | Perf report for the last 5.13895 seconds: | 20260505 08:12:56.077 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:12:56.077 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:56.077 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:56.077 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:56.077 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 08:12:56.077 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:56.077 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:56.077 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:12:56.077 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:56.077 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:12:56.077 | INFO | Uploaded 36GiB to GPU in 2.17712s (16.5356GiB/s) | 20260505 08:12:56.077 | INFO | Downloaded 34GiB to GPU in 2.04277s (16.644GiB/s) | 20260505 08:13:01.215 | INFO | Analyzing chunk frames = [1682432,1682944) | 20260505 08:13:01.215 | INFO | MemInfo: { | 20260505 08:13:01.215 | INFO | "gpu" : " 17 GB", | 20260505 08:13:01.215 | INFO | "huge" : "", | 20260505 08:13:01.215 | INFO | "huge_shared" : " 27 GB", | 20260505 08:13:01.215 | INFO | "malloc" : " 14 GB", | 20260505 08:13:01.215 | INFO | "pinned" : " 29 GB" | 20260505 08:13:01.215 | INFO | } | 20260505 08:13:01.215 | INFO | Perf report for the last 5.13856 seconds: | 20260505 08:13:01.215 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:13:01.215 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:01.215 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:01.215 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:01.215 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 08:13:01.215 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:01.215 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:01.215 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 08:13:01.215 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:01.215 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:01.215 | INFO | Uploaded 36GiB to GPU in 2.20198s (16.3489GiB/s) | 20260505 08:13:01.216 | INFO | Downloaded 34GiB to GPU in 2.14674s (15.838GiB/s) | 20260505 08:13:06.355 | INFO | Analyzing chunk frames = [1682944,1683456) | 20260505 08:13:06.355 | INFO | MemInfo: { | 20260505 08:13:06.355 | INFO | "gpu" : " 17 GB", | 20260505 08:13:06.355 | INFO | "huge" : "", | 20260505 08:13:06.355 | INFO | "huge_shared" : " 27 GB", | 20260505 08:13:06.356 | INFO | "malloc" : " 14 GB", | 20260505 08:13:06.356 | INFO | "pinned" : " 29 GB" | 20260505 08:13:06.356 | INFO | } | 20260505 08:13:06.356 | INFO | Perf report for the last 5.14066 seconds: | 20260505 08:13:06.356 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:13:06.356 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:06.356 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:06.356 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:06.356 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 08:13:06.356 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:06.356 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:06.356 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 08:13:06.356 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:06.356 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:06.356 | INFO | Uploaded 36GiB to GPU in 2.19262s (16.4187GiB/s) | 20260505 08:13:06.356 | INFO | Downloaded 34GiB to GPU in 2.04089s (16.6594GiB/s) | 20260505 08:13:06.496 | INFO | Starting a round of Baz IO | 20260505 08:13:09.778 | INFO | Populating RTMetrics report | 20260505 08:13:09.778 | INFO | RTMetrics report populated | 20260505 08:13:09.778 | INFO | Opening tmp RTMetrics json file | 20260505 08:13:09.792 | INFO | Serializing RTMetrics to json | 20260505 08:13:09.793 | INFO | Serialization done, swapping tmp file now... | 20260505 08:13:09.793 | INFO | RTMetrics done | 20260505 08:13:09.937 | INFO | Finished a round of Baz IO: Wrote 2.449 GiB in 3.441 seconds | 20260505 08:13:09.937 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 28.09% / 0.02% / 71.89% | 20260505 08:13:09.937 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.84924% / 4.37101e-05% / 6.75519e-05% | 20260505 08:13:09.937 | INFO | Baz IO overall: | 20260505 08:13:09.937 | INFO | BytesWritten : 943583351495 | 20260505 08:13:09.937 | INFO | NumEvents : 448369729594 | 20260505 08:13:11.495 | INFO | Analyzing chunk frames = [1683456,1683968) | 20260505 08:13:11.495 | INFO | MemInfo: { | 20260505 08:13:11.495 | INFO | "gpu" : " 17 GB", | 20260505 08:13:11.495 | INFO | "huge" : "", | 20260505 08:13:11.495 | INFO | "huge_shared" : " 27 GB", | 20260505 08:13:11.495 | INFO | "malloc" : " 14 GB", | 20260505 08:13:11.495 | INFO | "pinned" : " 29 GB" | 20260505 08:13:11.495 | INFO | } | 20260505 08:13:11.495 | INFO | Perf report for the last 5.13907 seconds: | 20260505 08:13:11.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:13:11.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:11.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:11.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:11.495 | INFO | ANALYSIS : 65.5%, 2.9, 100.1ms, 0 | 20260505 08:13:11.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:11.495 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 08:13:11.495 | INFO | PRE_HQ : 63.2%, 1.9, 61.5ms, 0 | 20260505 08:13:11.495 | INFO | BAZWRITER : 66.3%, 4.2, 142.2ms, 0 | 20260505 08:13:11.495 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 08:13:11.495 | INFO | Uploaded 36GiB to GPU in 2.23618s (16.0989GiB/s) | 20260505 08:13:11.495 | INFO | Downloaded 38GiB to GPU in 2.34974s (16.172GiB/s) | 20260505 08:13:16.634 | INFO | Analyzing chunk frames = [1683968,1684480) | 20260505 08:13:16.634 | INFO | MemInfo: { | 20260505 08:13:16.634 | INFO | "gpu" : " 17 GB", | 20260505 08:13:16.634 | INFO | "huge" : "", | 20260505 08:13:16.634 | INFO | "huge_shared" : " 27 GB", | 20260505 08:13:16.634 | INFO | "malloc" : " 14 GB", | 20260505 08:13:16.634 | INFO | "pinned" : " 29 GB" | 20260505 08:13:16.634 | INFO | } | 20260505 08:13:16.634 | INFO | Perf report for the last 5.13932 seconds: | 20260505 08:13:16.634 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:13:16.634 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:16.634 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:16.634 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:16.634 | INFO | ANALYSIS : 64.7%, 3.0, 101.1ms, 0 | 20260505 08:13:16.634 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:16.634 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:16.634 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 08:13:16.634 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:16.634 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:16.634 | INFO | Uploaded 36GiB to GPU in 2.24712s (16.0205GiB/s) | 20260505 08:13:16.634 | INFO | Downloaded 34GiB to GPU in 2.10648s (16.1407GiB/s) | 20260505 08:13:21.772 | INFO | Analyzing chunk frames = [1684480,1684992) | 20260505 08:13:21.772 | INFO | MemInfo: { | 20260505 08:13:21.772 | INFO | "gpu" : " 17 GB", | 20260505 08:13:21.772 | INFO | "huge" : "", | 20260505 08:13:21.772 | INFO | "huge_shared" : " 27 GB", | 20260505 08:13:21.772 | INFO | "malloc" : " 14 GB", | 20260505 08:13:21.772 | INFO | "pinned" : " 29 GB" | 20260505 08:13:21.772 | INFO | } | 20260505 08:13:21.773 | INFO | Perf report for the last 5.1386 seconds: | 20260505 08:13:21.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:13:21.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:21.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:21.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:21.773 | INFO | ANALYSIS : 63.7%, 3.0, 99.2ms, 0 | 20260505 08:13:21.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:21.773 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:21.773 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 08:13:21.773 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:21.773 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:21.773 | INFO | Uploaded 36GiB to GPU in 2.17212s (16.5736GiB/s) | 20260505 08:13:21.773 | INFO | Downloaded 34GiB to GPU in 2.06844s (16.4375GiB/s) | 20260505 08:13:26.912 | INFO | Analyzing chunk frames = [1684992,1685504) | 20260505 08:13:26.912 | INFO | MemInfo: { | 20260505 08:13:26.912 | INFO | "gpu" : " 17 GB", | 20260505 08:13:26.912 | INFO | "huge" : "", | 20260505 08:13:26.912 | INFO | "huge_shared" : " 27 GB", | 20260505 08:13:26.912 | INFO | "malloc" : " 14 GB", | 20260505 08:13:26.912 | INFO | "pinned" : " 29 GB" | 20260505 08:13:26.912 | INFO | } | 20260505 08:13:26.912 | INFO | Perf report for the last 5.13965 seconds: | 20260505 08:13:26.913 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:13:26.913 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:26.913 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:26.913 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:26.913 | INFO | ANALYSIS : 63.8%, 3.0, 99.1ms, 0 | 20260505 08:13:26.913 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:26.913 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:26.913 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:13:26.913 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:26.913 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:26.913 | INFO | Uploaded 36GiB to GPU in 2.19229s (16.4212GiB/s) | 20260505 08:13:26.913 | INFO | Downloaded 34GiB to GPU in 2.06716s (16.4477GiB/s) | 20260505 08:13:32.053 | INFO | Analyzing chunk frames = [1685504,1686016) | 20260505 08:13:32.053 | INFO | MemInfo: { | 20260505 08:13:32.053 | INFO | "gpu" : " 17 GB", | 20260505 08:13:32.053 | INFO | "huge" : "", | 20260505 08:13:32.053 | INFO | "huge_shared" : " 27 GB", | 20260505 08:13:32.053 | INFO | "malloc" : " 14 GB", | 20260505 08:13:32.053 | INFO | "pinned" : " 29 GB" | 20260505 08:13:32.053 | INFO | } | 20260505 08:13:32.053 | INFO | Perf report for the last 5.1406 seconds: | 20260505 08:13:32.053 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:13:32.053 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:32.053 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:32.053 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:32.053 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 08:13:32.053 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:32.053 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:32.053 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 08:13:32.053 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:32.053 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:32.053 | INFO | Uploaded 36GiB to GPU in 2.17846s (16.5255GiB/s) | 20260505 08:13:32.053 | INFO | Downloaded 34GiB to GPU in 2.0597s (16.5073GiB/s) | 20260505 08:13:37.192 | INFO | Analyzing chunk frames = [1686016,1686528) | 20260505 08:13:37.192 | INFO | MemInfo: { | 20260505 08:13:37.192 | INFO | "gpu" : " 17 GB", | 20260505 08:13:37.192 | INFO | "huge" : "", | 20260505 08:13:37.192 | INFO | "huge_shared" : " 27 GB", | 20260505 08:13:37.192 | INFO | "malloc" : " 14 GB", | 20260505 08:13:37.192 | INFO | "pinned" : " 29 GB" | 20260505 08:13:37.192 | INFO | } | 20260505 08:13:37.192 | INFO | Perf report for the last 5.13884 seconds: | 20260505 08:13:37.192 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:13:37.192 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:37.192 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:37.192 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:37.192 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 08:13:37.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:37.192 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:37.192 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:13:37.192 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:37.192 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:37.192 | INFO | Uploaded 36GiB to GPU in 2.18354s (16.487GiB/s) | 20260505 08:13:37.192 | INFO | Downloaded 34GiB to GPU in 2.05492s (16.5457GiB/s) | 20260505 08:13:42.330 | INFO | Analyzing chunk frames = [1686528,1687040) | 20260505 08:13:42.330 | INFO | MemInfo: { | 20260505 08:13:42.330 | INFO | "gpu" : " 17 GB", | 20260505 08:13:42.330 | INFO | "huge" : "", | 20260505 08:13:42.330 | INFO | "huge_shared" : " 27 GB", | 20260505 08:13:42.330 | INFO | "malloc" : " 14 GB", | 20260505 08:13:42.330 | INFO | "pinned" : " 29 GB" | 20260505 08:13:42.330 | INFO | } | 20260505 08:13:42.330 | INFO | Perf report for the last 5.13826 seconds: | 20260505 08:13:42.330 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:13:42.330 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:42.330 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:42.330 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:42.330 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 08:13:42.330 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:42.330 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:42.330 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 08:13:42.330 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:42.330 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:42.331 | INFO | Uploaded 36GiB to GPU in 2.16108s (16.6583GiB/s) | 20260505 08:13:42.331 | INFO | Downloaded 34GiB to GPU in 2.14507s (15.8503GiB/s) | 20260505 08:13:47.471 | INFO | Analyzing chunk frames = [1687040,1687552) | 20260505 08:13:47.471 | INFO | MemInfo: { | 20260505 08:13:47.471 | INFO | "gpu" : " 17 GB", | 20260505 08:13:47.471 | INFO | "huge" : "", | 20260505 08:13:47.471 | INFO | "huge_shared" : " 27 GB", | 20260505 08:13:47.471 | INFO | "malloc" : " 14 GB", | 20260505 08:13:47.471 | INFO | "pinned" : " 29 GB" | 20260505 08:13:47.471 | INFO | } | 20260505 08:13:47.471 | INFO | Perf report for the last 5.14074 seconds: | 20260505 08:13:47.471 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:13:47.471 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:47.471 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:47.471 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:47.471 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 08:13:47.471 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:47.471 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:47.471 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 08:13:47.471 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:47.471 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:47.471 | INFO | Uploaded 36GiB to GPU in 2.18654s (16.4644GiB/s) | 20260505 08:13:47.471 | INFO | Downloaded 34GiB to GPU in 2.041s (16.6585GiB/s) | 20260505 08:13:47.471 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 08:13:47.471 | INFO | Total mem usage: 15.3 GiB | 20260505 08:13:47.471 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 08:13:47.471 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -5.53%) | 20260505 08:13:47.471 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 08:13:47.471 | INFO | High Water Breakdown: | 20260505 08:13:47.471 | INFO | 60.9% : 8.85 GiB ( 94.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 08:13:47.471 | INFO | 39% : 5.66 GiB ( 85.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 08:13:47.471 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 08:13:47.471 | INFO | ------------------------------------------ | 20260505 08:13:47.471 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:13:47.471 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 08:13:47.471 | INFO | Total mem usage: 26.5 GiB | 20260505 08:13:47.471 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 08:13:47.471 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 08:13:47.471 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 08:13:47.471 | INFO | High Water Breakdown: | 20260505 08:13:47.471 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:13:47.471 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:13:47.471 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:13:47.471 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:13:47.471 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:13:47.472 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:13:47.472 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 08:13:47.472 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 08:13:47.472 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:13:47.472 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 08:13:47.472 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 08:13:47.472 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 08:13:47.472 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 08:13:47.472 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 08:13:47.472 | INFO | ------------------------------------------ | 20260505 08:13:47.472 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:13:47.472 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 08:13:47.472 | INFO | Total mem usage: 25.6 GiB | 20260505 08:13:47.472 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 08:13:47.472 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 08:13:47.472 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 08:13:47.472 | INFO | High Water Breakdown: | 20260505 08:13:47.472 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 08:13:47.472 | INFO | ------------------------------------------ | 20260505 08:13:47.472 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:13:47.472 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 08:13:47.472 | INFO | Total mem usage: 15.7 GiB | 20260505 08:13:47.472 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 08:13:47.472 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 08:13:47.472 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 08:13:47.472 | INFO | High Water Breakdown: | 20260505 08:13:47.472 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 08:13:47.472 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 08:13:47.472 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 08:13:47.472 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 08:13:47.472 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 08:13:47.472 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 08:13:47.472 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 08:13:47.472 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 08:13:47.472 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 08:13:47.472 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:13:47.472 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 08:13:47.472 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:13:47.472 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 08:13:47.472 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 08:13:47.472 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 08:13:47.472 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 08:13:47.472 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:13:47.472 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 08:13:47.472 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:13:47.472 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:13:47.472 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:13:47.472 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 08:13:47.472 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 08:13:47.472 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:13:47.472 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 08:13:47.472 | INFO | ------------------------------------------ | 20260505 08:13:47.472 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:13:47.472 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 08:13:47.472 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:13:47.472 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:13:47.472 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.50e+01, 1.35e+01 / 2.32e+00 | 20260505 08:13:47.472 | INFO | PerfReport: 0.29%, AggregateStats -- 9.32e-02 / 1.04e-01, 1.12e-02 / 4.49e-03 | 20260505 08:13:47.472 | INFO | PerfReport: 11.63%, Baseline -- 3.80e+00 / 4.45e+00, 6.23e-01 / 2.98e-01 | 20260505 08:13:47.472 | INFO | PerfReport: 3.32%, Histogram -- 1.08e+00 / 1.48e+00, 2.12e-01 / 1.12e-01 | 20260505 08:13:47.472 | INFO | PerfReport: 15.54%, DME -- 4.06e+01 / 4.44e+01, 4.07e+00 / 1.69e+00 | 20260505 08:13:47.472 | INFO | PerfReport: 56.41%, FrameLabeling -- 1.84e+01 / 1.94e+01, 1.93e+00 / 1.62e+00 | 20260505 08:13:47.472 | INFO | PerfReport: 7.50%, PulseAccumulating -- 2.45e+00 / 3.07e+00, 4.14e-01 / 2.23e-01 | 20260505 08:13:47.472 | INFO | PerfReport: 3.92%, Metrics -- 1.28e+00 / 1.58e+00, 3.60e-01 / 1.98e-01 | 20260505 08:13:47.472 | INFO | PerfReport: 1.40%, PulseSerializing -- 4.59e-01 / 5.10e-01, 8.79e-02 / 3.36e-02 | 20260505 08:13:47.472 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 08:13:47.472 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:13:47.472 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:13:47.472 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.31e+01 / 0.00e+00 | 20260505 08:13:47.472 | INFO | PerfReport: 22.48%, Upload -- 2.29e+01 / 0.00e+00, 8.18e+00 / 0.00e+00 | 20260505 08:13:47.472 | INFO | PerfReport: 21.61%, Download -- 2.20e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260505 08:13:47.472 | INFO | PerfReport: 32.14%, Compute -- 3.27e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 08:13:47.472 | INFO | PerfReport: 1.65%, LockUpload -- 1.68e+00 / 0.00e+00, 7.29e+00 / 0.00e+00 | 20260505 08:13:47.472 | INFO | PerfReport: 1.02%, LockDownload -- 1.03e+00 / 0.00e+00, 2.53e+00 / 0.00e+00 | 20260505 08:13:47.472 | INFO | PerfReport: 21.08%, LockCompute -- 2.14e+01 / 0.00e+00, 1.78e+01 / 0.00e+00 | 20260505 08:13:47.472 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 2.89e-02 / 0.00e+00 | 20260505 08:13:47.612 | INFO | Starting a round of Baz IO | 20260505 08:13:50.897 | INFO | Populating RTMetrics report | 20260505 08:13:50.897 | INFO | RTMetrics report populated | 20260505 08:13:50.897 | INFO | Opening tmp RTMetrics json file | 20260505 08:13:50.910 | INFO | Serializing RTMetrics to json | 20260505 08:13:50.958 | INFO | Serialization done, swapping tmp file now... | 20260505 08:13:50.959 | INFO | RTMetrics done | 20260505 08:13:51.045 | INFO | Finished a round of Baz IO: Wrote 2.447 GiB in 3.433 seconds | 20260505 08:13:51.045 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 28.06% / 0.02% / 71.91% | 20260505 08:13:51.045 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.83718% / 1.98682e-05% / 7.94729e-05% | 20260505 08:13:51.045 | INFO | Baz IO overall: | 20260505 08:13:51.045 | INFO | BytesWritten : 946211048134 | 20260505 08:13:51.046 | INFO | NumEvents : 449626423124 | 20260505 08:13:52.609 | INFO | Analyzing chunk frames = [1687552,1688064) | 20260505 08:13:52.609 | INFO | MemInfo: { | 20260505 08:13:52.609 | INFO | "gpu" : " 17 GB", | 20260505 08:13:52.609 | INFO | "huge" : "", | 20260505 08:13:52.609 | INFO | "huge_shared" : " 27 GB", | 20260505 08:13:52.609 | INFO | "malloc" : " 14 GB", | 20260505 08:13:52.609 | INFO | "pinned" : " 29 GB" | 20260505 08:13:52.609 | INFO | } | 20260505 08:13:52.609 | INFO | Perf report for the last 5.1379 seconds: | 20260505 08:13:52.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:13:52.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:52.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:52.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:52.609 | INFO | ANALYSIS : 65.5%, 2.9, 100.2ms, 0 | 20260505 08:13:52.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:52.609 | INFO | RT_METRICS : 47.0%, 1.0, 24.6ms, 0 | 20260505 08:13:52.609 | INFO | PRE_HQ : 63.6%, 1.9, 61.8ms, 0 | 20260505 08:13:52.609 | INFO | BAZWRITER : 66.1%, 4.1, 140.5ms, 0 | 20260505 08:13:52.609 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260505 08:13:52.609 | INFO | Uploaded 36GiB to GPU in 2.27438s (15.8285GiB/s) | 20260505 08:13:52.609 | INFO | Downloaded 38GiB to GPU in 2.369s (16.0405GiB/s) | 20260505 08:13:57.748 | INFO | Analyzing chunk frames = [1688064,1688576) | 20260505 08:13:57.748 | INFO | MemInfo: { | 20260505 08:13:57.748 | INFO | "gpu" : " 17 GB", | 20260505 08:13:57.748 | INFO | "huge" : "", | 20260505 08:13:57.748 | INFO | "huge_shared" : " 27 GB", | 20260505 08:13:57.748 | INFO | "malloc" : " 14 GB", | 20260505 08:13:57.748 | INFO | "pinned" : " 29 GB" | 20260505 08:13:57.748 | INFO | } | 20260505 08:13:57.749 | INFO | Perf report for the last 5.13977 seconds: | 20260505 08:13:57.749 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:13:57.749 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:57.749 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:57.749 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:57.749 | INFO | ANALYSIS : 64.8%, 3.0, 101.2ms, 0 | 20260505 08:13:57.749 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:57.749 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:57.749 | INFO | PRE_HQ : 30.2%, 1.0, 15.8ms, 0 | 20260505 08:13:57.749 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:57.749 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:13:57.749 | INFO | Uploaded 36GiB to GPU in 2.24434s (16.0403GiB/s) | 20260505 08:13:57.749 | INFO | Downloaded 34GiB to GPU in 2.0786s (16.3572GiB/s) | 20260505 08:14:02.889 | INFO | Analyzing chunk frames = [1688576,1689088) | 20260505 08:14:02.889 | INFO | MemInfo: { | 20260505 08:14:02.889 | INFO | "gpu" : " 17 GB", | 20260505 08:14:02.889 | INFO | "huge" : "", | 20260505 08:14:02.889 | INFO | "huge_shared" : " 27 GB", | 20260505 08:14:02.889 | INFO | "malloc" : " 14 GB", | 20260505 08:14:02.889 | INFO | "pinned" : " 29 GB" | 20260505 08:14:02.889 | INFO | } | 20260505 08:14:02.889 | INFO | Perf report for the last 5.14087 seconds: | 20260505 08:14:02.889 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:14:02.889 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:02.889 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:02.889 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:02.890 | INFO | ANALYSIS : 63.7%, 3.0, 99.1ms, 0 | 20260505 08:14:02.890 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:02.890 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:02.890 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 08:14:02.890 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:02.890 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:02.890 | INFO | Uploaded 36GiB to GPU in 2.17043s (16.5866GiB/s) | 20260505 08:14:02.890 | INFO | Downloaded 34GiB to GPU in 2.06809s (16.4403GiB/s) | 20260505 08:14:08.027 | INFO | Analyzing chunk frames = [1689088,1689600) | 20260505 08:14:08.027 | INFO | MemInfo: { | 20260505 08:14:08.027 | INFO | "gpu" : " 17 GB", | 20260505 08:14:08.027 | INFO | "huge" : "", | 20260505 08:14:08.027 | INFO | "huge_shared" : " 27 GB", | 20260505 08:14:08.027 | INFO | "malloc" : " 14 GB", | 20260505 08:14:08.027 | INFO | "pinned" : " 29 GB" | 20260505 08:14:08.027 | INFO | } | 20260505 08:14:08.027 | INFO | Perf report for the last 5.13737 seconds: | 20260505 08:14:08.027 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:14:08.027 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:08.027 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:08.027 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:08.027 | INFO | ANALYSIS : 63.7%, 3.0, 98.9ms, 0 | 20260505 08:14:08.027 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:08.027 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:08.027 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 08:14:08.027 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:08.027 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:08.027 | INFO | Uploaded 36GiB to GPU in 2.1922s (16.4219GiB/s) | 20260505 08:14:08.027 | INFO | Downloaded 34GiB to GPU in 2.06334s (16.4781GiB/s) | 20260505 08:14:13.166 | INFO | Analyzing chunk frames = [1689600,1690112) | 20260505 08:14:13.166 | INFO | MemInfo: { | 20260505 08:14:13.166 | INFO | "gpu" : " 17 GB", | 20260505 08:14:13.166 | INFO | "huge" : "", | 20260505 08:14:13.166 | INFO | "huge_shared" : " 27 GB", | 20260505 08:14:13.166 | INFO | "malloc" : " 14 GB", | 20260505 08:14:13.166 | INFO | "pinned" : " 29 GB" | 20260505 08:14:13.167 | INFO | } | 20260505 08:14:13.167 | INFO | Perf report for the last 5.13975 seconds: | 20260505 08:14:13.167 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:14:13.167 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:13.167 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:13.167 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:13.167 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 08:14:13.167 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:13.167 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:13.167 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:14:13.167 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:13.167 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:13.167 | INFO | Uploaded 36GiB to GPU in 2.18978s (16.44GiB/s) | 20260505 08:14:13.167 | INFO | Downloaded 34GiB to GPU in 2.05906s (16.5124GiB/s) | 20260505 08:14:18.307 | INFO | Analyzing chunk frames = [1690112,1690624) | 20260505 08:14:18.307 | INFO | MemInfo: { | 20260505 08:14:18.307 | INFO | "gpu" : " 17 GB", | 20260505 08:14:18.307 | INFO | "huge" : "", | 20260505 08:14:18.307 | INFO | "huge_shared" : " 27 GB", | 20260505 08:14:18.307 | INFO | "malloc" : " 14 GB", | 20260505 08:14:18.307 | INFO | "pinned" : " 29 GB" | 20260505 08:14:18.307 | INFO | } | 20260505 08:14:18.307 | INFO | Perf report for the last 5.14002 seconds: | 20260505 08:14:18.307 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:14:18.307 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:18.307 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:18.307 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:18.307 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 08:14:18.307 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:18.307 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:18.307 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 08:14:18.307 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:18.307 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:18.307 | INFO | Uploaded 36GiB to GPU in 2.19104s (16.4305GiB/s) | 20260505 08:14:18.307 | INFO | Downloaded 34GiB to GPU in 2.0597s (16.5073GiB/s) | 20260505 08:14:23.445 | INFO | Analyzing chunk frames = [1690624,1691136) | 20260505 08:14:23.445 | INFO | MemInfo: { | 20260505 08:14:23.445 | INFO | "gpu" : " 17 GB", | 20260505 08:14:23.445 | INFO | "huge" : "", | 20260505 08:14:23.445 | INFO | "huge_shared" : " 27 GB", | 20260505 08:14:23.445 | INFO | "malloc" : " 14 GB", | 20260505 08:14:23.445 | INFO | "pinned" : " 29 GB" | 20260505 08:14:23.445 | INFO | } | 20260505 08:14:23.445 | INFO | Perf report for the last 5.13865 seconds: | 20260505 08:14:23.445 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:14:23.445 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:23.445 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:23.445 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:23.445 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 08:14:23.445 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:23.445 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:23.445 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 08:14:23.445 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:23.445 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:23.446 | INFO | Uploaded 36GiB to GPU in 2.20212s (16.3479GiB/s) | 20260505 08:14:23.446 | INFO | Downloaded 34GiB to GPU in 2.15502s (15.7771GiB/s) | 20260505 08:14:28.584 | INFO | Analyzing chunk frames = [1691136,1691648) | 20260505 08:14:28.584 | INFO | MemInfo: { | 20260505 08:14:28.584 | INFO | "gpu" : " 17 GB", | 20260505 08:14:28.584 | INFO | "huge" : "", | 20260505 08:14:28.584 | INFO | "huge_shared" : " 27 GB", | 20260505 08:14:28.584 | INFO | "malloc" : " 14 GB", | 20260505 08:14:28.584 | INFO | "pinned" : " 29 GB" | 20260505 08:14:28.584 | INFO | } | 20260505 08:14:28.584 | INFO | Perf report for the last 5.13862 seconds: | 20260505 08:14:28.584 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:14:28.584 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:28.584 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:28.584 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:28.584 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 08:14:28.584 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:28.584 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:28.584 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 08:14:28.584 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:28.584 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:28.584 | INFO | Uploaded 36GiB to GPU in 2.18917s (16.4446GiB/s) | 20260505 08:14:28.584 | INFO | Downloaded 34GiB to GPU in 2.04231s (16.6478GiB/s) | 20260505 08:14:28.718 | INFO | Starting a round of Baz IO | 20260505 08:14:32.005 | INFO | Populating RTMetrics report | 20260505 08:14:32.005 | INFO | RTMetrics report populated | 20260505 08:14:32.006 | INFO | Opening tmp RTMetrics json file | 20260505 08:14:32.010 | INFO | Serializing RTMetrics to json | 20260505 08:14:32.011 | INFO | Serialization done, swapping tmp file now... | 20260505 08:14:32.012 | INFO | RTMetrics done | 20260505 08:14:32.155 | INFO | Finished a round of Baz IO: Wrote 2.448 GiB in 3.437 seconds | 20260505 08:14:32.155 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 28.04% / 0.02% / 71.94% | 20260505 08:14:32.155 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.82649% / 3.57628e-05% / 7.15256e-05% | 20260505 08:14:32.155 | INFO | Baz IO overall: | 20260505 08:14:32.155 | INFO | BytesWritten : 948839078227 | 20260505 08:14:32.155 | INFO | NumEvents : 450883427806 | 20260505 08:14:33.727 | INFO | Analyzing chunk frames = [1691648,1692160) | 20260505 08:14:33.727 | INFO | MemInfo: { | 20260505 08:14:33.727 | INFO | "gpu" : " 17 GB", | 20260505 08:14:33.727 | INFO | "huge" : "", | 20260505 08:14:33.727 | INFO | "huge_shared" : " 27 GB", | 20260505 08:14:33.727 | INFO | "malloc" : " 14 GB", | 20260505 08:14:33.727 | INFO | "pinned" : " 29 GB" | 20260505 08:14:33.727 | INFO | } | 20260505 08:14:33.727 | INFO | Perf report for the last 5.14352 seconds: | 20260505 08:14:33.727 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:14:33.727 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:33.728 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:33.728 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:33.728 | INFO | ANALYSIS : 65.4%, 2.9, 99.7ms, 0 | 20260505 08:14:33.728 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:33.728 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 08:14:33.728 | INFO | PRE_HQ : 63.9%, 1.9, 62.0ms, 0 | 20260505 08:14:33.728 | INFO | BAZWRITER : 66.0%, 4.2, 141.8ms, 0 | 20260505 08:14:33.728 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 08:14:33.728 | INFO | Uploaded 36GiB to GPU in 2.2506s (15.9958GiB/s) | 20260505 08:14:33.728 | INFO | Downloaded 38GiB to GPU in 2.37397s (16.0069GiB/s) | 20260505 08:14:38.864 | INFO | Analyzing chunk frames = [1692160,1692672) | 20260505 08:14:38.864 | INFO | MemInfo: { | 20260505 08:14:38.864 | INFO | "gpu" : " 17 GB", | 20260505 08:14:38.864 | INFO | "huge" : "", | 20260505 08:14:38.864 | INFO | "huge_shared" : " 27 GB", | 20260505 08:14:38.864 | INFO | "malloc" : " 14 GB", | 20260505 08:14:38.864 | INFO | "pinned" : " 29 GB" | 20260505 08:14:38.864 | INFO | } | 20260505 08:14:38.864 | INFO | Perf report for the last 5.13621 seconds: | 20260505 08:14:38.864 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:14:38.864 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:38.864 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:38.864 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:38.864 | INFO | ANALYSIS : 64.6%, 3.0, 100.8ms, 0 | 20260505 08:14:38.864 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:38.864 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:38.864 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:14:38.864 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:38.864 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:38.864 | INFO | Uploaded 36GiB to GPU in 2.25017s (15.9988GiB/s) | 20260505 08:14:38.864 | INFO | Downloaded 34GiB to GPU in 2.10429s (16.1574GiB/s) | 20260505 08:14:44.003 | INFO | Analyzing chunk frames = [1692672,1693184) | 20260505 08:14:44.003 | INFO | MemInfo: { | 20260505 08:14:44.003 | INFO | "gpu" : " 17 GB", | 20260505 08:14:44.003 | INFO | "huge" : "", | 20260505 08:14:44.003 | INFO | "huge_shared" : " 27 GB", | 20260505 08:14:44.004 | INFO | "malloc" : " 14 GB", | 20260505 08:14:44.004 | INFO | "pinned" : " 29 GB" | 20260505 08:14:44.004 | INFO | } | 20260505 08:14:44.004 | INFO | Perf report for the last 5.13999 seconds: | 20260505 08:14:44.004 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:14:44.004 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:44.004 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:44.004 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:44.004 | INFO | ANALYSIS : 63.7%, 3.0, 99.1ms, 0 | 20260505 08:14:44.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:44.004 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:44.004 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:14:44.004 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:44.004 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:44.004 | INFO | Uploaded 36GiB to GPU in 2.17051s (16.586GiB/s) | 20260505 08:14:44.004 | INFO | Downloaded 34GiB to GPU in 2.06585s (16.4581GiB/s) | 20260505 08:14:49.143 | INFO | Analyzing chunk frames = [1693184,1693696) | 20260505 08:14:49.143 | INFO | MemInfo: { | 20260505 08:14:49.143 | INFO | "gpu" : " 17 GB", | 20260505 08:14:49.143 | INFO | "huge" : "", | 20260505 08:14:49.143 | INFO | "huge_shared" : " 27 GB", | 20260505 08:14:49.143 | INFO | "malloc" : " 14 GB", | 20260505 08:14:49.143 | INFO | "pinned" : " 29 GB" | 20260505 08:14:49.143 | INFO | } | 20260505 08:14:49.143 | INFO | Perf report for the last 5.1394 seconds: | 20260505 08:14:49.143 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:14:49.143 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:49.143 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:49.143 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:49.143 | INFO | ANALYSIS : 63.7%, 3.0, 98.9ms, 0 | 20260505 08:14:49.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:49.143 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:49.143 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:14:49.143 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:49.143 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:49.143 | INFO | Uploaded 36GiB to GPU in 2.19084s (16.4321GiB/s) | 20260505 08:14:49.143 | INFO | Downloaded 34GiB to GPU in 2.06704s (16.4486GiB/s) | 20260505 08:14:54.281 | INFO | Analyzing chunk frames = [1693696,1694208) | 20260505 08:14:54.281 | INFO | MemInfo: { | 20260505 08:14:54.281 | INFO | "gpu" : " 17 GB", | 20260505 08:14:54.281 | INFO | "huge" : "", | 20260505 08:14:54.281 | INFO | "huge_shared" : " 27 GB", | 20260505 08:14:54.281 | INFO | "malloc" : " 14 GB", | 20260505 08:14:54.281 | INFO | "pinned" : " 29 GB" | 20260505 08:14:54.281 | INFO | } | 20260505 08:14:54.281 | INFO | Perf report for the last 5.1384 seconds: | 20260505 08:14:54.281 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:14:54.281 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:54.282 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:54.282 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:54.282 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 08:14:54.282 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:54.282 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:54.282 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:14:54.282 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:54.282 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:54.282 | INFO | Uploaded 36GiB to GPU in 2.18944s (16.4426GiB/s) | 20260505 08:14:54.282 | INFO | Downloaded 34GiB to GPU in 2.05793s (16.5215GiB/s) | 20260505 08:14:59.421 | INFO | Analyzing chunk frames = [1694208,1694720) | 20260505 08:14:59.421 | INFO | MemInfo: { | 20260505 08:14:59.421 | INFO | "gpu" : " 17 GB", | 20260505 08:14:59.421 | INFO | "huge" : "", | 20260505 08:14:59.421 | INFO | "huge_shared" : " 27 GB", | 20260505 08:14:59.422 | INFO | "malloc" : " 14 GB", | 20260505 08:14:59.422 | INFO | "pinned" : " 29 GB" | 20260505 08:14:59.422 | INFO | } | 20260505 08:14:59.422 | INFO | Perf report for the last 5.14017 seconds: | 20260505 08:14:59.422 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:14:59.422 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:59.422 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:59.422 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:59.422 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 08:14:59.422 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:59.422 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:59.422 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 08:14:59.422 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:59.422 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:14:59.422 | INFO | Uploaded 36GiB to GPU in 2.18453s (16.4795GiB/s) | 20260505 08:14:59.422 | INFO | Downloaded 34GiB to GPU in 2.05551s (16.5409GiB/s) | 20260505 08:15:04.560 | INFO | Analyzing chunk frames = [1694720,1695232) | 20260505 08:15:04.560 | INFO | MemInfo: { | 20260505 08:15:04.560 | INFO | "gpu" : " 17 GB", | 20260505 08:15:04.560 | INFO | "huge" : "", | 20260505 08:15:04.560 | INFO | "huge_shared" : " 27 GB", | 20260505 08:15:04.560 | INFO | "malloc" : " 14 GB", | 20260505 08:15:04.560 | INFO | "pinned" : " 29 GB" | 20260505 08:15:04.560 | INFO | } | 20260505 08:15:04.560 | INFO | Perf report for the last 5.13803 seconds: | 20260505 08:15:04.560 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:15:04.560 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:04.560 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:04.560 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:04.560 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 08:15:04.560 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:04.560 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:04.560 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260505 08:15:04.560 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:04.560 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:04.560 | INFO | Uploaded 36GiB to GPU in 2.21295s (16.2679GiB/s) | 20260505 08:15:04.560 | INFO | Downloaded 34GiB to GPU in 2.15535s (15.7747GiB/s) | 20260505 08:15:09.699 | INFO | Analyzing chunk frames = [1695232,1695744) | 20260505 08:15:09.699 | INFO | MemInfo: { | 20260505 08:15:09.699 | INFO | "gpu" : " 17 GB", | 20260505 08:15:09.699 | INFO | "huge" : "", | 20260505 08:15:09.699 | INFO | "huge_shared" : " 27 GB", | 20260505 08:15:09.699 | INFO | "malloc" : " 14 GB", | 20260505 08:15:09.699 | INFO | "pinned" : " 29 GB" | 20260505 08:15:09.699 | INFO | } | 20260505 08:15:09.699 | INFO | Perf report for the last 5.13942 seconds: | 20260505 08:15:09.699 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:15:09.699 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:09.699 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:09.699 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:09.699 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 08:15:09.699 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:09.699 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:09.699 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 08:15:09.699 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:09.699 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:09.699 | INFO | Uploaded 36GiB to GPU in 2.1942s (16.4069GiB/s) | 20260505 08:15:09.699 | INFO | Downloaded 34GiB to GPU in 2.04136s (16.6556GiB/s) | 20260505 08:15:09.700 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 08:15:09.700 | INFO | Total mem usage: 15.3 GiB | 20260505 08:15:09.700 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 08:15:09.700 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.28%) | 20260505 08:15:09.700 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 08:15:09.700 | INFO | High Water Breakdown: | 20260505 08:15:09.700 | INFO | 60.8% : 8.77 GiB ( 93.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 08:15:09.700 | INFO | 39.1% : 5.63 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 08:15:09.700 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 08:15:09.700 | INFO | ------------------------------------------ | 20260505 08:15:09.700 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:15:09.700 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 08:15:09.700 | INFO | Total mem usage: 26.5 GiB | 20260505 08:15:09.700 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 08:15:09.700 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 08:15:09.700 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 08:15:09.700 | INFO | High Water Breakdown: | 20260505 08:15:09.700 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:15:09.700 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:15:09.700 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:15:09.700 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:15:09.700 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:15:09.700 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:15:09.700 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 08:15:09.700 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 08:15:09.700 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:15:09.700 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 08:15:09.700 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 08:15:09.700 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 08:15:09.700 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 08:15:09.700 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 08:15:09.700 | INFO | ------------------------------------------ | 20260505 08:15:09.700 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:15:09.700 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 08:15:09.700 | INFO | Total mem usage: 25.6 GiB | 20260505 08:15:09.700 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 08:15:09.700 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 08:15:09.700 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 08:15:09.700 | INFO | High Water Breakdown: | 20260505 08:15:09.700 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 08:15:09.700 | INFO | ------------------------------------------ | 20260505 08:15:09.700 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:15:09.700 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 08:15:09.700 | INFO | Total mem usage: 15.7 GiB | 20260505 08:15:09.700 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 08:15:09.700 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 08:15:09.700 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 08:15:09.700 | INFO | High Water Breakdown: | 20260505 08:15:09.700 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 08:15:09.700 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 08:15:09.700 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 08:15:09.700 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 08:15:09.700 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 08:15:09.701 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 08:15:09.701 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 08:15:09.701 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 08:15:09.701 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 08:15:09.701 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:15:09.701 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 08:15:09.701 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:15:09.701 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 08:15:09.701 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 08:15:09.701 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 08:15:09.701 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 08:15:09.701 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:15:09.701 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 08:15:09.701 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:15:09.701 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:15:09.701 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:15:09.701 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 08:15:09.701 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 08:15:09.701 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:15:09.701 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 08:15:09.701 | INFO | ------------------------------------------ | 20260505 08:15:09.701 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:15:09.701 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 08:15:09.701 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:15:09.701 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:15:09.701 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.36e+01 / 2.07e+00 | 20260505 08:15:09.701 | INFO | PerfReport: 0.28%, AggregateStats -- 9.31e-02 / 1.05e-01, 1.06e-02 / 6.21e-03 | 20260505 08:15:09.701 | INFO | PerfReport: 11.62%, Baseline -- 3.80e+00 / 4.44e+00, 6.17e-01 / 2.84e-01 | 20260505 08:15:09.701 | INFO | PerfReport: 3.32%, Histogram -- 1.08e+00 / 1.50e+00, 2.13e-01 / 9.68e-02 | 20260505 08:15:09.701 | INFO | PerfReport: 15.51%, DME -- 4.06e+01 / 4.43e+01, 4.00e+00 / 1.39e+00 | 20260505 08:15:09.701 | INFO | PerfReport: 56.51%, FrameLabeling -- 1.85e+01 / 1.98e+01, 1.98e+00 / 2.01e+00 | 20260505 08:15:09.701 | INFO | PerfReport: 7.43%, PulseAccumulating -- 2.43e+00 / 3.03e+00, 4.02e-01 / 2.54e-01 | 20260505 08:15:09.701 | INFO | PerfReport: 3.92%, Metrics -- 1.28e+00 / 1.60e+00, 3.65e-01 / 2.12e-01 | 20260505 08:15:09.701 | INFO | PerfReport: 1.40%, PulseSerializing -- 4.58e-01 / 5.09e-01, 8.79e-02 / 3.60e-02 | 20260505 08:15:09.701 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 08:15:09.701 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:15:09.701 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:15:09.701 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.33e+01 / 0.00e+00 | 20260505 08:15:09.701 | INFO | PerfReport: 22.58%, Upload -- 2.30e+01 / 0.00e+00, 8.31e+00 / 0.00e+00 | 20260505 08:15:09.701 | INFO | PerfReport: 21.63%, Download -- 2.20e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260505 08:15:09.701 | INFO | PerfReport: 32.12%, Compute -- 3.27e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 08:15:09.701 | INFO | PerfReport: 1.68%, LockUpload -- 1.71e+00 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260505 08:15:09.701 | INFO | PerfReport: 1.01%, LockDownload -- 1.02e+00 / 0.00e+00, 2.48e+00 / 0.00e+00 | 20260505 08:15:09.701 | INFO | PerfReport: 20.97%, LockCompute -- 2.13e+01 / 0.00e+00, 1.77e+01 / 0.00e+00 | 20260505 08:15:09.701 | INFO | PerfReport: 0.01%, Misc -- 1.50e-02 / 0.00e+00, 2.62e-02 / 0.00e+00 | 20260505 08:15:09.841 | INFO | Starting a round of Baz IO | 20260505 08:15:13.117 | INFO | Populating RTMetrics report | 20260505 08:15:13.117 | INFO | RTMetrics report populated | 20260505 08:15:13.117 | INFO | Opening tmp RTMetrics json file | 20260505 08:15:13.137 | INFO | Serializing RTMetrics to json | 20260505 08:15:13.189 | INFO | Serialization done, swapping tmp file now... | 20260505 08:15:13.190 | INFO | RTMetrics done | 20260505 08:15:13.266 | INFO | Finished a round of Baz IO: Wrote 2.448 GiB in 3.425 seconds | 20260505 08:15:13.266 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 28.02% / 0.02% / 71.96% | 20260505 08:15:13.266 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.81531% / 4.37101e-05% / 5.16574e-05% | 20260505 08:15:13.266 | INFO | Baz IO overall: | 20260505 08:15:13.266 | INFO | BytesWritten : 951467531245 | 20260505 08:15:13.266 | INFO | NumEvents : 452140460590 | 20260505 08:15:14.839 | INFO | Analyzing chunk frames = [1695744,1696256) | 20260505 08:15:14.839 | INFO | MemInfo: { | 20260505 08:15:14.839 | INFO | "gpu" : " 17 GB", | 20260505 08:15:14.839 | INFO | "huge" : "", | 20260505 08:15:14.839 | INFO | "huge_shared" : " 27 GB", | 20260505 08:15:14.839 | INFO | "malloc" : " 14 GB", | 20260505 08:15:14.839 | INFO | "pinned" : " 29 GB" | 20260505 08:15:14.839 | INFO | } | 20260505 08:15:14.839 | INFO | Perf report for the last 5.14009 seconds: | 20260505 08:15:14.839 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:15:14.839 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:14.839 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:14.839 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:14.839 | INFO | ANALYSIS : 65.4%, 2.9, 99.4ms, 0 | 20260505 08:15:14.839 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:14.839 | INFO | RT_METRICS : 47.3%, 1.0, 24.8ms, 0 | 20260505 08:15:14.839 | INFO | PRE_HQ : 63.8%, 1.9, 61.8ms, 0 | 20260505 08:15:14.839 | INFO | BAZWRITER : 65.8%, 4.2, 141.0ms, 0 | 20260505 08:15:14.839 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 08:15:14.839 | INFO | Uploaded 36GiB to GPU in 2.26398s (15.9012GiB/s) | 20260505 08:15:14.840 | INFO | Downloaded 38GiB to GPU in 2.38117s (15.9585GiB/s) | 20260505 08:15:19.979 | INFO | Analyzing chunk frames = [1696256,1696768) | 20260505 08:15:19.979 | INFO | MemInfo: { | 20260505 08:15:19.979 | INFO | "gpu" : " 17 GB", | 20260505 08:15:19.979 | INFO | "huge" : "", | 20260505 08:15:19.979 | INFO | "huge_shared" : " 27 GB", | 20260505 08:15:19.979 | INFO | "malloc" : " 14 GB", | 20260505 08:15:19.979 | INFO | "pinned" : " 29 GB" | 20260505 08:15:19.979 | INFO | } | 20260505 08:15:19.979 | INFO | Perf report for the last 5.13985 seconds: | 20260505 08:15:19.979 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:15:19.979 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:19.979 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:19.979 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:19.979 | INFO | ANALYSIS : 64.8%, 3.0, 101.2ms, 0 | 20260505 08:15:19.979 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:19.979 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:19.979 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:15:19.979 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:19.979 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:19.979 | INFO | Uploaded 36GiB to GPU in 2.24696s (16.0216GiB/s) | 20260505 08:15:19.979 | INFO | Downloaded 34GiB to GPU in 2.09866s (16.2008GiB/s) | 20260505 08:15:25.118 | INFO | Analyzing chunk frames = [1696768,1697280) | 20260505 08:15:25.118 | INFO | MemInfo: { | 20260505 08:15:25.118 | INFO | "gpu" : " 17 GB", | 20260505 08:15:25.118 | INFO | "huge" : "", | 20260505 08:15:25.118 | INFO | "huge_shared" : " 27 GB", | 20260505 08:15:25.118 | INFO | "malloc" : " 14 GB", | 20260505 08:15:25.119 | INFO | "pinned" : " 29 GB" | 20260505 08:15:25.119 | INFO | } | 20260505 08:15:25.119 | INFO | Perf report for the last 5.13955 seconds: | 20260505 08:15:25.119 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:15:25.119 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:25.119 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:25.119 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:25.119 | INFO | ANALYSIS : 63.6%, 3.0, 99.0ms, 0 | 20260505 08:15:25.119 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:25.119 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:25.119 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:15:25.119 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:25.119 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:25.119 | INFO | Uploaded 36GiB to GPU in 2.16619s (16.6191GiB/s) | 20260505 08:15:25.119 | INFO | Downloaded 34GiB to GPU in 2.0675s (16.445GiB/s) | 20260505 08:15:30.256 | INFO | Analyzing chunk frames = [1697280,1697792) | 20260505 08:15:30.256 | INFO | MemInfo: { | 20260505 08:15:30.256 | INFO | "gpu" : " 17 GB", | 20260505 08:15:30.256 | INFO | "huge" : "", | 20260505 08:15:30.256 | INFO | "huge_shared" : " 27 GB", | 20260505 08:15:30.256 | INFO | "malloc" : " 14 GB", | 20260505 08:15:30.256 | INFO | "pinned" : " 29 GB" | 20260505 08:15:30.256 | INFO | } | 20260505 08:15:30.256 | INFO | Perf report for the last 5.13776 seconds: | 20260505 08:15:30.256 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:15:30.256 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:30.256 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:30.256 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:30.256 | INFO | ANALYSIS : 63.7%, 3.0, 98.9ms, 0 | 20260505 08:15:30.257 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:30.257 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:30.257 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:15:30.257 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:30.257 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:30.257 | INFO | Uploaded 36GiB to GPU in 2.18294s (16.4915GiB/s) | 20260505 08:15:30.257 | INFO | Downloaded 34GiB to GPU in 2.06563s (16.4598GiB/s) | 20260505 08:15:35.396 | INFO | Analyzing chunk frames = [1697792,1698304) | 20260505 08:15:35.396 | INFO | MemInfo: { | 20260505 08:15:35.396 | INFO | "gpu" : " 17 GB", | 20260505 08:15:35.396 | INFO | "huge" : "", | 20260505 08:15:35.396 | INFO | "huge_shared" : " 27 GB", | 20260505 08:15:35.396 | INFO | "malloc" : " 14 GB", | 20260505 08:15:35.396 | INFO | "pinned" : " 29 GB" | 20260505 08:15:35.396 | INFO | } | 20260505 08:15:35.396 | INFO | Perf report for the last 5.13943 seconds: | 20260505 08:15:35.396 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:15:35.396 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:35.396 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:35.396 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:35.396 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 08:15:35.396 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:35.396 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:35.396 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 08:15:35.396 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:35.396 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:35.396 | INFO | Uploaded 36GiB to GPU in 2.18603s (16.4682GiB/s) | 20260505 08:15:35.396 | INFO | Downloaded 34GiB to GPU in 2.06046s (16.5011GiB/s) | 20260505 08:15:40.535 | INFO | Analyzing chunk frames = [1698304,1698816) | 20260505 08:15:40.535 | INFO | MemInfo: { | 20260505 08:15:40.535 | INFO | "gpu" : " 17 GB", | 20260505 08:15:40.535 | INFO | "huge" : "", | 20260505 08:15:40.535 | INFO | "huge_shared" : " 27 GB", | 20260505 08:15:40.535 | INFO | "malloc" : " 14 GB", | 20260505 08:15:40.535 | INFO | "pinned" : " 29 GB" | 20260505 08:15:40.535 | INFO | } | 20260505 08:15:40.535 | INFO | Perf report for the last 5.13946 seconds: | 20260505 08:15:40.535 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:15:40.535 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:40.535 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:40.535 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:40.535 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 08:15:40.535 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:40.535 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:40.535 | INFO | PRE_HQ : 29.8%, 1.0, 15.4ms, 0 | 20260505 08:15:40.535 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:40.536 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:40.536 | INFO | Uploaded 36GiB to GPU in 2.18558s (16.4716GiB/s) | 20260505 08:15:40.536 | INFO | Downloaded 34GiB to GPU in 2.04347s (16.6383GiB/s) | 20260505 08:15:45.675 | INFO | Analyzing chunk frames = [1698816,1699328) | 20260505 08:15:45.675 | INFO | MemInfo: { | 20260505 08:15:45.675 | INFO | "gpu" : " 17 GB", | 20260505 08:15:45.675 | INFO | "huge" : "", | 20260505 08:15:45.675 | INFO | "huge_shared" : " 27 GB", | 20260505 08:15:45.675 | INFO | "malloc" : " 14 GB", | 20260505 08:15:45.675 | INFO | "pinned" : " 29 GB" | 20260505 08:15:45.675 | INFO | } | 20260505 08:15:45.675 | INFO | Perf report for the last 5.13959 seconds: | 20260505 08:15:45.675 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:15:45.675 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:45.675 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:45.675 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:45.675 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 08:15:45.675 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:45.675 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:45.675 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 08:15:45.675 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:45.675 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:45.675 | INFO | Uploaded 36GiB to GPU in 2.22164s (16.2042GiB/s) | 20260505 08:15:45.675 | INFO | Downloaded 34GiB to GPU in 2.15372s (15.7867GiB/s) | 20260505 08:15:50.815 | INFO | Analyzing chunk frames = [1699328,1699840) | 20260505 08:15:50.815 | INFO | MemInfo: { | 20260505 08:15:50.815 | INFO | "gpu" : " 17 GB", | 20260505 08:15:50.815 | INFO | "huge" : "", | 20260505 08:15:50.815 | INFO | "huge_shared" : " 27 GB", | 20260505 08:15:50.815 | INFO | "malloc" : " 14 GB", | 20260505 08:15:50.815 | INFO | "pinned" : " 29 GB" | 20260505 08:15:50.815 | INFO | } | 20260505 08:15:50.815 | INFO | Perf report for the last 5.14003 seconds: | 20260505 08:15:50.815 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:15:50.815 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:50.815 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:50.815 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:50.815 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 08:15:50.815 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:50.815 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:50.815 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 08:15:50.815 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:50.815 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:50.815 | INFO | Uploaded 36GiB to GPU in 2.19514s (16.3999GiB/s) | 20260505 08:15:50.815 | INFO | Downloaded 34GiB to GPU in 2.03802s (16.6828GiB/s) | 20260505 08:15:50.955 | INFO | Starting a round of Baz IO | 20260505 08:15:54.224 | INFO | Populating RTMetrics report | 20260505 08:15:54.224 | INFO | RTMetrics report populated | 20260505 08:15:54.224 | INFO | Opening tmp RTMetrics json file | 20260505 08:15:54.225 | INFO | Serializing RTMetrics to json | 20260505 08:15:54.226 | INFO | Serialization done, swapping tmp file now... | 20260505 08:15:54.227 | INFO | RTMetrics done | 20260505 08:15:54.357 | INFO | Finished a round of Baz IO: Wrote 2.452 GiB in 3.402 seconds | 20260505 08:15:54.357 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 28.00% / 0.02% / 71.98% | 20260505 08:15:54.357 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.80311% / 1.98682e-05% / 7.54992e-05% | 20260505 08:15:54.357 | INFO | Baz IO overall: | 20260505 08:15:54.357 | INFO | BytesWritten : 954100287991 | 20260505 08:15:54.357 | INFO | NumEvents : 453399004477 | 20260505 08:15:55.954 | INFO | Analyzing chunk frames = [1699840,1700352) | 20260505 08:15:55.954 | INFO | MemInfo: { | 20260505 08:15:55.954 | INFO | "gpu" : " 17 GB", | 20260505 08:15:55.954 | INFO | "huge" : "", | 20260505 08:15:55.954 | INFO | "huge_shared" : " 27 GB", | 20260505 08:15:55.954 | INFO | "malloc" : " 14 GB", | 20260505 08:15:55.954 | INFO | "pinned" : " 29 GB" | 20260505 08:15:55.954 | INFO | } | 20260505 08:15:55.954 | INFO | Perf report for the last 5.13939 seconds: | 20260505 08:15:55.954 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:15:55.954 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:55.954 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:55.954 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:55.954 | INFO | ANALYSIS : 65.2%, 2.9, 99.2ms, 0 | 20260505 08:15:55.954 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:15:55.954 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260505 08:15:55.955 | INFO | PRE_HQ : 63.5%, 1.9, 61.5ms, 0 | 20260505 08:15:55.955 | INFO | BAZWRITER : 65.5%, 4.2, 141.8ms, 0 | 20260505 08:15:55.955 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 08:15:55.955 | INFO | Uploaded 36GiB to GPU in 2.25431s (15.9694GiB/s) | 20260505 08:15:55.955 | INFO | Downloaded 38GiB to GPU in 2.38803s (15.9127GiB/s) | 20260505 08:16:01.093 | INFO | Analyzing chunk frames = [1700352,1700864) | 20260505 08:16:01.093 | INFO | MemInfo: { | 20260505 08:16:01.093 | INFO | "gpu" : " 17 GB", | 20260505 08:16:01.093 | INFO | "huge" : "", | 20260505 08:16:01.093 | INFO | "huge_shared" : " 27 GB", | 20260505 08:16:01.093 | INFO | "malloc" : " 14 GB", | 20260505 08:16:01.093 | INFO | "pinned" : " 29 GB" | 20260505 08:16:01.093 | INFO | } | 20260505 08:16:01.093 | INFO | Perf report for the last 5.13838 seconds: | 20260505 08:16:01.093 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:16:01.093 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:01.093 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:01.093 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:01.093 | INFO | ANALYSIS : 64.9%, 3.0, 101.3ms, 0 | 20260505 08:16:01.093 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:01.093 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:01.093 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 08:16:01.093 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:01.093 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:01.093 | INFO | Uploaded 36GiB to GPU in 2.24796s (16.0145GiB/s) | 20260505 08:16:01.093 | INFO | Downloaded 34GiB to GPU in 2.08846s (16.28GiB/s) | 20260505 08:16:06.232 | INFO | Analyzing chunk frames = [1700864,1701376) | 20260505 08:16:06.232 | INFO | MemInfo: { | 20260505 08:16:06.232 | INFO | "gpu" : " 17 GB", | 20260505 08:16:06.232 | INFO | "huge" : "", | 20260505 08:16:06.232 | INFO | "huge_shared" : " 27 GB", | 20260505 08:16:06.232 | INFO | "malloc" : " 14 GB", | 20260505 08:16:06.232 | INFO | "pinned" : " 29 GB" | 20260505 08:16:06.232 | INFO | } | 20260505 08:16:06.233 | INFO | Perf report for the last 5.13971 seconds: | 20260505 08:16:06.233 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:16:06.233 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:06.233 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:06.233 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:06.233 | INFO | ANALYSIS : 63.8%, 3.0, 99.2ms, 0 | 20260505 08:16:06.233 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:06.233 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:06.233 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 08:16:06.233 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:06.233 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:06.233 | INFO | Uploaded 36GiB to GPU in 2.1785s (16.5251GiB/s) | 20260505 08:16:06.233 | INFO | Downloaded 34GiB to GPU in 2.06664s (16.4519GiB/s) | 20260505 08:16:11.371 | INFO | Analyzing chunk frames = [1701376,1701888) | 20260505 08:16:11.371 | INFO | MemInfo: { | 20260505 08:16:11.371 | INFO | "gpu" : " 17 GB", | 20260505 08:16:11.371 | INFO | "huge" : "", | 20260505 08:16:11.371 | INFO | "huge_shared" : " 27 GB", | 20260505 08:16:11.371 | INFO | "malloc" : " 14 GB", | 20260505 08:16:11.371 | INFO | "pinned" : " 29 GB" | 20260505 08:16:11.371 | INFO | } | 20260505 08:16:11.371 | INFO | Perf report for the last 5.1388 seconds: | 20260505 08:16:11.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:16:11.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:11.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:11.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:11.371 | INFO | ANALYSIS : 63.7%, 3.0, 98.9ms, 0 | 20260505 08:16:11.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:11.372 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:11.372 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:16:11.372 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:11.372 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:11.372 | INFO | Uploaded 36GiB to GPU in 2.19061s (16.4338GiB/s) | 20260505 08:16:11.372 | INFO | Downloaded 34GiB to GPU in 2.06366s (16.4756GiB/s) | 20260505 08:16:16.511 | INFO | Analyzing chunk frames = [1701888,1702400) | 20260505 08:16:16.511 | INFO | MemInfo: { | 20260505 08:16:16.511 | INFO | "gpu" : " 17 GB", | 20260505 08:16:16.511 | INFO | "huge" : "", | 20260505 08:16:16.511 | INFO | "huge_shared" : " 27 GB", | 20260505 08:16:16.511 | INFO | "malloc" : " 14 GB", | 20260505 08:16:16.511 | INFO | "pinned" : " 29 GB" | 20260505 08:16:16.511 | INFO | } | 20260505 08:16:16.511 | INFO | Perf report for the last 5.14013 seconds: | 20260505 08:16:16.511 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:16:16.511 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:16.511 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:16.511 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:16.511 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 08:16:16.512 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:16.512 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:16.512 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:16:16.512 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:16.512 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:16.512 | INFO | Uploaded 36GiB to GPU in 2.18428s (16.4814GiB/s) | 20260505 08:16:16.512 | INFO | Downloaded 34GiB to GPU in 2.05607s (16.5364GiB/s) | 20260505 08:16:21.651 | INFO | Analyzing chunk frames = [1702400,1702912) | 20260505 08:16:21.651 | INFO | MemInfo: { | 20260505 08:16:21.651 | INFO | "gpu" : " 17 GB", | 20260505 08:16:21.651 | INFO | "huge" : "", | 20260505 08:16:21.651 | INFO | "huge_shared" : " 27 GB", | 20260505 08:16:21.651 | INFO | "malloc" : " 14 GB", | 20260505 08:16:21.651 | INFO | "pinned" : " 29 GB" | 20260505 08:16:21.651 | INFO | } | 20260505 08:16:21.652 | INFO | Perf report for the last 5.14003 seconds: | 20260505 08:16:21.652 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:16:21.652 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:21.652 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:21.652 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:21.652 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 08:16:21.652 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:21.652 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:21.652 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:16:21.652 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:21.652 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:21.652 | INFO | Uploaded 36GiB to GPU in 2.17819s (16.5275GiB/s) | 20260505 08:16:21.652 | INFO | Downloaded 34GiB to GPU in 2.04356s (16.6377GiB/s) | 20260505 08:16:26.790 | INFO | Analyzing chunk frames = [1702912,1703424) | 20260505 08:16:26.791 | INFO | MemInfo: { | 20260505 08:16:26.791 | INFO | "gpu" : " 17 GB", | 20260505 08:16:26.791 | INFO | "huge" : "", | 20260505 08:16:26.791 | INFO | "huge_shared" : " 27 GB", | 20260505 08:16:26.791 | INFO | "malloc" : " 14 GB", | 20260505 08:16:26.791 | INFO | "pinned" : " 29 GB" | 20260505 08:16:26.791 | INFO | } | 20260505 08:16:26.791 | INFO | Perf report for the last 5.13926 seconds: | 20260505 08:16:26.791 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:16:26.791 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:26.791 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:26.791 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:26.791 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 08:16:26.791 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:26.791 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:26.791 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 08:16:26.791 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:26.791 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:26.791 | INFO | Uploaded 36GiB to GPU in 2.2252s (16.1783GiB/s) | 20260505 08:16:26.791 | INFO | Downloaded 34GiB to GPU in 2.15272s (15.794GiB/s) | 20260505 08:16:31.929 | INFO | Analyzing chunk frames = [1703424,1703936) | 20260505 08:16:31.929 | INFO | MemInfo: { | 20260505 08:16:31.929 | INFO | "gpu" : " 17 GB", | 20260505 08:16:31.929 | INFO | "huge" : "", | 20260505 08:16:31.929 | INFO | "huge_shared" : " 27 GB", | 20260505 08:16:31.929 | INFO | "malloc" : " 14 GB", | 20260505 08:16:31.929 | INFO | "pinned" : " 29 GB" | 20260505 08:16:31.929 | INFO | } | 20260505 08:16:31.929 | INFO | Perf report for the last 5.13853 seconds: | 20260505 08:16:31.929 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:16:31.929 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:31.929 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:31.929 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:31.929 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 08:16:31.929 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:31.929 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:31.929 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 08:16:31.929 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:31.929 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:31.930 | INFO | Uploaded 36GiB to GPU in 2.19114s (16.4298GiB/s) | 20260505 08:16:31.930 | INFO | Downloaded 34GiB to GPU in 2.04412s (16.633GiB/s) | 20260505 08:16:31.930 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 08:16:31.930 | INFO | Total mem usage: 15.3 GiB | 20260505 08:16:31.930 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 08:16:31.930 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -5.54%) | 20260505 08:16:31.930 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 08:16:31.930 | INFO | High Water Breakdown: | 20260505 08:16:31.930 | INFO | 60.9% : 8.85 GiB ( 94.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 08:16:31.930 | INFO | 38.9% : 5.66 GiB ( 85.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 08:16:31.930 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 08:16:31.930 | INFO | ------------------------------------------ | 20260505 08:16:31.930 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:16:31.930 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 08:16:31.930 | INFO | Total mem usage: 26.5 GiB | 20260505 08:16:31.930 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 08:16:31.930 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 08:16:31.930 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 08:16:31.930 | INFO | High Water Breakdown: | 20260505 08:16:31.930 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:16:31.930 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:16:31.930 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:16:31.930 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:16:31.930 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:16:31.930 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:16:31.930 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 08:16:31.930 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 08:16:31.930 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:16:31.930 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 08:16:31.930 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 08:16:31.930 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 08:16:31.930 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 08:16:31.930 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 08:16:31.930 | INFO | ------------------------------------------ | 20260505 08:16:31.930 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:16:31.930 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 08:16:31.930 | INFO | Total mem usage: 25.6 GiB | 20260505 08:16:31.930 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 08:16:31.930 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 08:16:31.930 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 08:16:31.930 | INFO | High Water Breakdown: | 20260505 08:16:31.930 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 08:16:31.931 | INFO | ------------------------------------------ | 20260505 08:16:31.931 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:16:31.931 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 08:16:31.931 | INFO | Total mem usage: 15.7 GiB | 20260505 08:16:31.931 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 08:16:31.931 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 08:16:31.931 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 08:16:31.931 | INFO | High Water Breakdown: | 20260505 08:16:31.931 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 08:16:31.931 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 08:16:31.931 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 08:16:31.931 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 08:16:31.931 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 08:16:31.931 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 08:16:31.931 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 08:16:31.931 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 08:16:31.931 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 08:16:31.931 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:16:31.931 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 08:16:31.931 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:16:31.931 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 08:16:31.931 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 08:16:31.931 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 08:16:31.931 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 08:16:31.931 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:16:31.931 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 08:16:31.931 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:16:31.931 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:16:31.931 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:16:31.931 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 08:16:31.931 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 08:16:31.931 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:16:31.931 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 08:16:31.931 | INFO | ------------------------------------------ | 20260505 08:16:31.931 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:16:31.931 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 08:16:31.931 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:16:31.931 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:16:31.931 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.56e+01, 1.35e+01 / 1.72e+00 | 20260505 08:16:31.931 | INFO | PerfReport: 0.29%, AggregateStats -- 9.33e-02 / 1.05e-01, 1.08e-02 / 6.83e-03 | 20260505 08:16:31.931 | INFO | PerfReport: 11.63%, Baseline -- 3.80e+00 / 4.45e+00, 6.28e-01 / 2.55e-01 | 20260505 08:16:31.931 | INFO | PerfReport: 3.32%, Histogram -- 1.09e+00 / 1.50e+00, 2.14e-01 / 8.22e-02 | 20260505 08:16:31.931 | INFO | PerfReport: 15.50%, DME -- 4.06e+01 / 4.45e+01, 4.12e+00 / 1.13e+00 | 20260505 08:16:31.931 | INFO | PerfReport: 56.49%, FrameLabeling -- 1.85e+01 / 2.00e+01, 1.97e+00 / 1.55e+00 | 20260505 08:16:31.931 | INFO | PerfReport: 7.46%, PulseAccumulating -- 2.44e+00 / 3.05e+00, 4.12e-01 / 2.24e-01 | 20260505 08:16:31.931 | INFO | PerfReport: 3.90%, Metrics -- 1.28e+00 / 1.59e+00, 3.61e-01 / 1.48e-01 | 20260505 08:16:31.931 | INFO | PerfReport: 1.40%, PulseSerializing -- 4.59e-01 / 5.11e-01, 8.82e-02 / 3.13e-02 | 20260505 08:16:31.931 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 08:16:31.931 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:16:31.931 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:16:31.931 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.33e+01 / 0.00e+00 | 20260505 08:16:31.931 | INFO | PerfReport: 22.59%, Upload -- 2.30e+01 / 0.00e+00, 8.30e+00 / 0.00e+00 | 20260505 08:16:31.931 | INFO | PerfReport: 21.64%, Download -- 2.20e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260505 08:16:31.931 | INFO | PerfReport: 32.15%, Compute -- 3.27e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 08:16:31.931 | INFO | PerfReport: 1.69%, LockUpload -- 1.72e+00 / 0.00e+00, 7.32e+00 / 0.00e+00 | 20260505 08:16:31.931 | INFO | PerfReport: 1.00%, LockDownload -- 1.01e+00 / 0.00e+00, 2.47e+00 / 0.00e+00 | 20260505 08:16:31.931 | INFO | PerfReport: 20.93%, LockCompute -- 2.13e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260505 08:16:31.931 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.72e-02 / 0.00e+00 | 20260505 08:16:32.069 | INFO | Starting a round of Baz IO | 20260505 08:16:35.339 | INFO | Populating RTMetrics report | 20260505 08:16:35.339 | INFO | RTMetrics report populated | 20260505 08:16:35.339 | INFO | Opening tmp RTMetrics json file | 20260505 08:16:35.356 | INFO | Serializing RTMetrics to json | 20260505 08:16:35.379 | INFO | Serialization done, swapping tmp file now... | 20260505 08:16:35.380 | INFO | RTMetrics done | 20260505 08:16:35.482 | INFO | Finished a round of Baz IO: Wrote 2.451 GiB in 3.412 seconds | 20260505 08:16:35.482 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 27.97% / 0.02% / 72.00% | 20260505 08:16:35.482 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.79937% / 5.16574e-05% / 9.93411e-05% | 20260505 08:16:35.482 | INFO | Baz IO overall: | 20260505 08:16:35.482 | INFO | BytesWritten : 956731954492 | 20260505 08:16:35.482 | INFO | NumEvents : 454656906464 | 20260505 08:16:37.069 | INFO | Analyzing chunk frames = [1703936,1704448) | 20260505 08:16:37.069 | INFO | MemInfo: { | 20260505 08:16:37.069 | INFO | "gpu" : " 17 GB", | 20260505 08:16:37.069 | INFO | "huge" : "", | 20260505 08:16:37.069 | INFO | "huge_shared" : " 27 GB", | 20260505 08:16:37.069 | INFO | "malloc" : " 14 GB", | 20260505 08:16:37.069 | INFO | "pinned" : " 29 GB" | 20260505 08:16:37.069 | INFO | } | 20260505 08:16:37.069 | INFO | Perf report for the last 5.13982 seconds: | 20260505 08:16:37.069 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:16:37.069 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:37.069 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:37.069 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:37.069 | INFO | ANALYSIS : 65.2%, 2.9, 99.4ms, 0 | 20260505 08:16:37.069 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:37.069 | INFO | RT_METRICS : 46.5%, 1.0, 24.4ms, 0 | 20260505 08:16:37.069 | INFO | PRE_HQ : 63.4%, 1.9, 61.3ms, 0 | 20260505 08:16:37.069 | INFO | BAZWRITER : 65.7%, 4.2, 141.3ms, 0 | 20260505 08:16:37.069 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 08:16:37.069 | INFO | Uploaded 36GiB to GPU in 2.26219s (15.9138GiB/s) | 20260505 08:16:37.069 | INFO | Downloaded 38GiB to GPU in 2.38977s (15.9011GiB/s) | 20260505 08:16:42.208 | INFO | Analyzing chunk frames = [1704448,1704960) | 20260505 08:16:42.208 | INFO | MemInfo: { | 20260505 08:16:42.208 | INFO | "gpu" : " 17 GB", | 20260505 08:16:42.208 | INFO | "huge" : "", | 20260505 08:16:42.208 | INFO | "huge_shared" : " 27 GB", | 20260505 08:16:42.208 | INFO | "malloc" : " 14 GB", | 20260505 08:16:42.208 | INFO | "pinned" : " 29 GB" | 20260505 08:16:42.208 | INFO | } | 20260505 08:16:42.208 | INFO | Perf report for the last 5.13929 seconds: | 20260505 08:16:42.208 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:16:42.208 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:42.208 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:42.209 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:42.209 | INFO | ANALYSIS : 64.8%, 3.0, 101.2ms, 0 | 20260505 08:16:42.209 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:42.209 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:42.209 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 08:16:42.209 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:42.209 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:42.209 | INFO | Uploaded 36GiB to GPU in 2.24575s (16.0303GiB/s) | 20260505 08:16:42.209 | INFO | Downloaded 34GiB to GPU in 2.11022s (16.112GiB/s) | 20260505 08:16:47.346 | INFO | Analyzing chunk frames = [1704960,1705472) | 20260505 08:16:47.346 | INFO | MemInfo: { | 20260505 08:16:47.346 | INFO | "gpu" : " 17 GB", | 20260505 08:16:47.346 | INFO | "huge" : "", | 20260505 08:16:47.346 | INFO | "huge_shared" : " 27 GB", | 20260505 08:16:47.346 | INFO | "malloc" : " 14 GB", | 20260505 08:16:47.346 | INFO | "pinned" : " 29 GB" | 20260505 08:16:47.346 | INFO | } | 20260505 08:16:47.346 | INFO | Perf report for the last 5.13778 seconds: | 20260505 08:16:47.346 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:16:47.346 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:47.346 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:47.346 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:47.346 | INFO | ANALYSIS : 63.7%, 3.0, 99.1ms, 0 | 20260505 08:16:47.346 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:47.346 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:47.346 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 08:16:47.346 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:47.346 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:47.347 | INFO | Uploaded 36GiB to GPU in 2.17398s (16.5595GiB/s) | 20260505 08:16:47.347 | INFO | Downloaded 34GiB to GPU in 2.06952s (16.429GiB/s) | 20260505 08:16:52.486 | INFO | Analyzing chunk frames = [1705472,1705984) | 20260505 08:16:52.486 | INFO | MemInfo: { | 20260505 08:16:52.486 | INFO | "gpu" : " 17 GB", | 20260505 08:16:52.486 | INFO | "huge" : "", | 20260505 08:16:52.486 | INFO | "huge_shared" : " 27 GB", | 20260505 08:16:52.486 | INFO | "malloc" : " 14 GB", | 20260505 08:16:52.486 | INFO | "pinned" : " 29 GB" | 20260505 08:16:52.486 | INFO | } | 20260505 08:16:52.486 | INFO | Perf report for the last 5.14 seconds: | 20260505 08:16:52.486 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:16:52.486 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:52.486 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:52.486 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:52.486 | INFO | ANALYSIS : 63.8%, 3.0, 99.2ms, 0 | 20260505 08:16:52.486 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:52.486 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:52.486 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 08:16:52.486 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:52.486 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:52.486 | INFO | Uploaded 36GiB to GPU in 2.19834s (16.376GiB/s) | 20260505 08:16:52.487 | INFO | Downloaded 34GiB to GPU in 2.07032s (16.4226GiB/s) | 20260505 08:16:57.628 | INFO | Analyzing chunk frames = [1705984,1706496) | 20260505 08:16:57.628 | INFO | MemInfo: { | 20260505 08:16:57.628 | INFO | "gpu" : " 17 GB", | 20260505 08:16:57.628 | INFO | "huge" : "", | 20260505 08:16:57.628 | INFO | "huge_shared" : " 27 GB", | 20260505 08:16:57.628 | INFO | "malloc" : " 14 GB", | 20260505 08:16:57.628 | INFO | "pinned" : " 29 GB" | 20260505 08:16:57.628 | INFO | } | 20260505 08:16:57.628 | INFO | Perf report for the last 5.14174 seconds: | 20260505 08:16:57.628 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:16:57.628 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:57.628 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:57.628 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:57.628 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 08:16:57.628 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:57.628 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:57.628 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:16:57.628 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:57.628 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:16:57.628 | INFO | Uploaded 36GiB to GPU in 2.1754s (16.5487GiB/s) | 20260505 08:16:57.628 | INFO | Downloaded 34GiB to GPU in 2.0579s (16.5217GiB/s) | 20260505 08:17:02.767 | INFO | Analyzing chunk frames = [1706496,1707008) | 20260505 08:17:02.767 | INFO | MemInfo: { | 20260505 08:17:02.767 | INFO | "gpu" : " 17 GB", | 20260505 08:17:02.767 | INFO | "huge" : "", | 20260505 08:17:02.767 | INFO | "huge_shared" : " 27 GB", | 20260505 08:17:02.767 | INFO | "malloc" : " 14 GB", | 20260505 08:17:02.767 | INFO | "pinned" : " 29 GB" | 20260505 08:17:02.767 | INFO | } | 20260505 08:17:02.767 | INFO | Perf report for the last 5.13889 seconds: | 20260505 08:17:02.767 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:17:02.767 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:02.767 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:02.767 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:02.767 | INFO | ANALYSIS : 63.8%, 3.0, 99.5ms, 0 | 20260505 08:17:02.767 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:02.767 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:02.767 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260505 08:17:02.767 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:02.767 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:02.767 | INFO | Uploaded 36GiB to GPU in 2.18996s (16.4386GiB/s) | 20260505 08:17:02.767 | INFO | Downloaded 34GiB to GPU in 2.05924s (16.511GiB/s) | 20260505 08:17:07.904 | INFO | Analyzing chunk frames = [1707008,1707520) | 20260505 08:17:07.904 | INFO | MemInfo: { | 20260505 08:17:07.904 | INFO | "gpu" : " 17 GB", | 20260505 08:17:07.904 | INFO | "huge" : "", | 20260505 08:17:07.904 | INFO | "huge_shared" : " 27 GB", | 20260505 08:17:07.904 | INFO | "malloc" : " 14 GB", | 20260505 08:17:07.904 | INFO | "pinned" : " 29 GB" | 20260505 08:17:07.904 | INFO | } | 20260505 08:17:07.904 | INFO | Perf report for the last 5.13722 seconds: | 20260505 08:17:07.904 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:17:07.904 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:07.904 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:07.904 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:07.904 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 08:17:07.904 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:07.904 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:07.904 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 08:17:07.904 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:07.904 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:07.904 | INFO | Uploaded 36GiB to GPU in 2.21009s (16.2889GiB/s) | 20260505 08:17:07.904 | INFO | Downloaded 34GiB to GPU in 2.15545s (15.774GiB/s) | 20260505 08:17:13.043 | INFO | Analyzing chunk frames = [1707520,1708032) | 20260505 08:17:13.043 | INFO | MemInfo: { | 20260505 08:17:13.043 | INFO | "gpu" : " 17 GB", | 20260505 08:17:13.043 | INFO | "huge" : "", | 20260505 08:17:13.043 | INFO | "huge_shared" : " 27 GB", | 20260505 08:17:13.043 | INFO | "malloc" : " 14 GB", | 20260505 08:17:13.043 | INFO | "pinned" : " 29 GB" | 20260505 08:17:13.043 | INFO | } | 20260505 08:17:13.043 | INFO | Perf report for the last 5.13932 seconds: | 20260505 08:17:13.043 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:17:13.043 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:13.044 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:13.044 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:13.044 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 08:17:13.044 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:13.044 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:13.044 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:17:13.044 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:13.044 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:13.044 | INFO | Uploaded 36GiB to GPU in 2.18786s (16.4545GiB/s) | 20260505 08:17:13.044 | INFO | Downloaded 34GiB to GPU in 2.0432s (16.6406GiB/s) | 20260505 08:17:13.184 | INFO | Starting a round of Baz IO | 20260505 08:17:16.475 | INFO | Populating RTMetrics report | 20260505 08:17:16.475 | INFO | RTMetrics report populated | 20260505 08:17:16.475 | INFO | Opening tmp RTMetrics json file | 20260505 08:17:16.542 | INFO | Serializing RTMetrics to json | 20260505 08:17:16.545 | INFO | Serialization done, swapping tmp file now... | 20260505 08:17:16.548 | INFO | RTMetrics done | 20260505 08:17:16.663 | INFO | Finished a round of Baz IO: Wrote 2.45 GiB in 3.479 seconds | 20260505 08:17:16.663 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 27.95% / 0.02% / 72.03% | 20260505 08:17:16.663 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.79136% / 4.37101e-05% / 8.34465e-05% | 20260505 08:17:16.663 | INFO | Baz IO overall: | 20260505 08:17:16.663 | INFO | BytesWritten : 959362874812 | 20260505 08:17:16.663 | INFO | NumEvents : 455914747883 | 20260505 08:17:18.183 | INFO | Analyzing chunk frames = [1708032,1708544) | 20260505 08:17:18.183 | INFO | MemInfo: { | 20260505 08:17:18.183 | INFO | "gpu" : " 17 GB", | 20260505 08:17:18.183 | INFO | "huge" : "", | 20260505 08:17:18.183 | INFO | "huge_shared" : " 27 GB", | 20260505 08:17:18.183 | INFO | "malloc" : " 14 GB", | 20260505 08:17:18.183 | INFO | "pinned" : " 29 GB" | 20260505 08:17:18.183 | INFO | } | 20260505 08:17:18.183 | INFO | Perf report for the last 5.13973 seconds: | 20260505 08:17:18.183 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:17:18.183 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:18.183 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:18.183 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:18.183 | INFO | ANALYSIS : 65.6%, 2.9, 100.4ms, 0 | 20260505 08:17:18.183 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:18.183 | INFO | RT_METRICS : 47.3%, 1.0, 24.8ms, 0 | 20260505 08:17:18.183 | INFO | PRE_HQ : 63.4%, 1.9, 61.6ms, 0 | 20260505 08:17:18.183 | INFO | BAZWRITER : 67.0%, 4.2, 143.5ms, 0 | 20260505 08:17:18.183 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260505 08:17:18.183 | INFO | Uploaded 36GiB to GPU in 2.22798s (16.1581GiB/s) | 20260505 08:17:18.183 | INFO | Downloaded 38GiB to GPU in 2.35104s (16.1631GiB/s) | 20260505 08:17:23.322 | INFO | Analyzing chunk frames = [1708544,1709056) | 20260505 08:17:23.322 | INFO | MemInfo: { | 20260505 08:17:23.322 | INFO | "gpu" : " 17 GB", | 20260505 08:17:23.322 | INFO | "huge" : "", | 20260505 08:17:23.322 | INFO | "huge_shared" : " 27 GB", | 20260505 08:17:23.322 | INFO | "malloc" : " 14 GB", | 20260505 08:17:23.323 | INFO | "pinned" : " 29 GB" | 20260505 08:17:23.323 | INFO | } | 20260505 08:17:23.323 | INFO | Perf report for the last 5.13941 seconds: | 20260505 08:17:23.323 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:17:23.323 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:23.323 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:23.323 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:23.323 | INFO | ANALYSIS : 64.8%, 3.0, 101.1ms, 0 | 20260505 08:17:23.323 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:23.323 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:23.323 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:17:23.323 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:23.323 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:23.323 | INFO | Uploaded 36GiB to GPU in 2.24781s (16.0156GiB/s) | 20260505 08:17:23.323 | INFO | Downloaded 34GiB to GPU in 2.11042s (16.1105GiB/s) | 20260505 08:17:28.462 | INFO | Analyzing chunk frames = [1709056,1709568) | 20260505 08:17:28.462 | INFO | MemInfo: { | 20260505 08:17:28.462 | INFO | "gpu" : " 17 GB", | 20260505 08:17:28.462 | INFO | "huge" : "", | 20260505 08:17:28.462 | INFO | "huge_shared" : " 27 GB", | 20260505 08:17:28.462 | INFO | "malloc" : " 14 GB", | 20260505 08:17:28.462 | INFO | "pinned" : " 29 GB" | 20260505 08:17:28.462 | INFO | } | 20260505 08:17:28.462 | INFO | Perf report for the last 5.13984 seconds: | 20260505 08:17:28.462 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:17:28.462 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:28.462 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:28.463 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:28.463 | INFO | ANALYSIS : 63.8%, 3.0, 99.2ms, 0 | 20260505 08:17:28.463 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:28.463 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:28.463 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:17:28.463 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:28.463 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:28.463 | INFO | Uploaded 36GiB to GPU in 2.1771s (16.5358GiB/s) | 20260505 08:17:28.463 | INFO | Downloaded 34GiB to GPU in 2.06461s (16.468GiB/s) | 20260505 08:17:33.601 | INFO | Analyzing chunk frames = [1709568,1710080) | 20260505 08:17:33.601 | INFO | MemInfo: { | 20260505 08:17:33.601 | INFO | "gpu" : " 17 GB", | 20260505 08:17:33.602 | INFO | "huge" : "", | 20260505 08:17:33.602 | INFO | "huge_shared" : " 27 GB", | 20260505 08:17:33.602 | INFO | "malloc" : " 14 GB", | 20260505 08:17:33.602 | INFO | "pinned" : " 29 GB" | 20260505 08:17:33.602 | INFO | } | 20260505 08:17:33.602 | INFO | Perf report for the last 5.1391 seconds: | 20260505 08:17:33.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:17:33.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:33.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:33.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:33.602 | INFO | ANALYSIS : 63.6%, 3.0, 98.9ms, 0 | 20260505 08:17:33.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:33.602 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:33.602 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:17:33.602 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:33.602 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:33.602 | INFO | Uploaded 36GiB to GPU in 2.19098s (16.431GiB/s) | 20260505 08:17:33.602 | INFO | Downloaded 34GiB to GPU in 2.06084s (16.4981GiB/s) | 20260505 08:17:38.741 | INFO | Analyzing chunk frames = [1710080,1710592) | 20260505 08:17:38.741 | INFO | MemInfo: { | 20260505 08:17:38.741 | INFO | "gpu" : " 17 GB", | 20260505 08:17:38.741 | INFO | "huge" : "", | 20260505 08:17:38.741 | INFO | "huge_shared" : " 27 GB", | 20260505 08:17:38.741 | INFO | "malloc" : " 14 GB", | 20260505 08:17:38.741 | INFO | "pinned" : " 29 GB" | 20260505 08:17:38.741 | INFO | } | 20260505 08:17:38.741 | INFO | Perf report for the last 5.1398 seconds: | 20260505 08:17:38.741 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:17:38.741 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:38.741 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:38.741 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:38.741 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 08:17:38.741 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:38.742 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:38.742 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:17:38.742 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:38.742 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:38.742 | INFO | Uploaded 36GiB to GPU in 2.18565s (16.4711GiB/s) | 20260505 08:17:38.742 | INFO | Downloaded 34GiB to GPU in 2.0541s (16.5523GiB/s) | 20260505 08:17:43.880 | INFO | Analyzing chunk frames = [1710592,1711104) | 20260505 08:17:43.880 | INFO | MemInfo: { | 20260505 08:17:43.880 | INFO | "gpu" : " 17 GB", | 20260505 08:17:43.880 | INFO | "huge" : "", | 20260505 08:17:43.880 | INFO | "huge_shared" : " 27 GB", | 20260505 08:17:43.880 | INFO | "malloc" : " 14 GB", | 20260505 08:17:43.880 | INFO | "pinned" : " 29 GB" | 20260505 08:17:43.880 | INFO | } | 20260505 08:17:43.880 | INFO | Perf report for the last 5.13895 seconds: | 20260505 08:17:43.880 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:17:43.880 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:43.880 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:43.880 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:43.880 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 08:17:43.880 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:43.880 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:43.880 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:17:43.880 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:43.880 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:43.881 | INFO | Uploaded 36GiB to GPU in 2.18711s (16.4601GiB/s) | 20260505 08:17:43.881 | INFO | Downloaded 34GiB to GPU in 2.05239s (16.5661GiB/s) | 20260505 08:17:49.020 | INFO | Analyzing chunk frames = [1711104,1711616) | 20260505 08:17:49.020 | INFO | MemInfo: { | 20260505 08:17:49.020 | INFO | "gpu" : " 17 GB", | 20260505 08:17:49.020 | INFO | "huge" : "", | 20260505 08:17:49.020 | INFO | "huge_shared" : " 27 GB", | 20260505 08:17:49.020 | INFO | "malloc" : " 14 GB", | 20260505 08:17:49.020 | INFO | "pinned" : " 29 GB" | 20260505 08:17:49.020 | INFO | } | 20260505 08:17:49.020 | INFO | Perf report for the last 5.13961 seconds: | 20260505 08:17:49.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:17:49.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:49.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:49.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:49.020 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 08:17:49.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:49.020 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:49.020 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 08:17:49.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:49.020 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:49.020 | INFO | Uploaded 36GiB to GPU in 2.2009s (16.3569GiB/s) | 20260505 08:17:49.020 | INFO | Downloaded 34GiB to GPU in 2.16012s (15.7399GiB/s) | 20260505 08:17:54.158 | INFO | Analyzing chunk frames = [1711616,1712128) | 20260505 08:17:54.158 | INFO | MemInfo: { | 20260505 08:17:54.158 | INFO | "gpu" : " 17 GB", | 20260505 08:17:54.158 | INFO | "huge" : "", | 20260505 08:17:54.158 | INFO | "huge_shared" : " 27 GB", | 20260505 08:17:54.158 | INFO | "malloc" : " 14 GB", | 20260505 08:17:54.158 | INFO | "pinned" : " 29 GB" | 20260505 08:17:54.158 | INFO | } | 20260505 08:17:54.158 | INFO | Perf report for the last 5.13807 seconds: | 20260505 08:17:54.158 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:17:54.158 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:54.158 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:54.158 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:54.158 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 08:17:54.158 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:54.158 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:54.158 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 08:17:54.158 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:54.158 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:54.159 | INFO | Uploaded 36GiB to GPU in 2.20169s (16.3511GiB/s) | 20260505 08:17:54.159 | INFO | Downloaded 34GiB to GPU in 2.04829s (16.5992GiB/s) | 20260505 08:17:54.159 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 08:17:54.159 | INFO | Total mem usage: 15.3 GiB | 20260505 08:17:54.159 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 08:17:54.159 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -6.03%) | 20260505 08:17:54.159 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 08:17:54.159 | INFO | High Water Breakdown: | 20260505 08:17:54.159 | INFO | 60.7% : 8.78 GiB ( 93.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 08:17:54.159 | INFO | 39.1% : 5.66 GiB ( 85.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 08:17:54.159 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 08:17:54.159 | INFO | ------------------------------------------ | 20260505 08:17:54.159 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:17:54.159 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 08:17:54.159 | INFO | Total mem usage: 26.5 GiB | 20260505 08:17:54.159 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 08:17:54.159 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 08:17:54.159 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 08:17:54.159 | INFO | High Water Breakdown: | 20260505 08:17:54.159 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:17:54.159 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:17:54.159 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:17:54.159 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:17:54.159 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:17:54.159 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:17:54.159 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 08:17:54.159 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 08:17:54.159 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:17:54.159 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 08:17:54.159 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 08:17:54.159 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 08:17:54.159 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 08:17:54.159 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 08:17:54.159 | INFO | ------------------------------------------ | 20260505 08:17:54.159 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:17:54.159 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 08:17:54.159 | INFO | Total mem usage: 25.6 GiB | 20260505 08:17:54.159 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 08:17:54.159 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 08:17:54.159 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 08:17:54.159 | INFO | High Water Breakdown: | 20260505 08:17:54.159 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 08:17:54.159 | INFO | ------------------------------------------ | 20260505 08:17:54.160 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:17:54.160 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 08:17:54.160 | INFO | Total mem usage: 15.7 GiB | 20260505 08:17:54.160 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 08:17:54.160 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 08:17:54.160 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 08:17:54.160 | INFO | High Water Breakdown: | 20260505 08:17:54.160 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 08:17:54.160 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 08:17:54.160 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 08:17:54.160 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 08:17:54.160 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 08:17:54.160 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 08:17:54.160 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 08:17:54.160 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 08:17:54.160 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 08:17:54.160 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:17:54.160 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 08:17:54.160 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:17:54.160 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 08:17:54.160 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 08:17:54.160 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 08:17:54.160 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 08:17:54.160 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:17:54.160 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 08:17:54.160 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:17:54.160 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:17:54.160 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:17:54.160 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 08:17:54.160 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 08:17:54.160 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:17:54.160 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 08:17:54.160 | INFO | ------------------------------------------ | 20260505 08:17:54.160 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:17:54.160 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 08:17:54.160 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:17:54.160 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:17:54.160 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.52e+01, 1.38e+01 / 2.07e+00 | 20260505 08:17:54.160 | INFO | PerfReport: 0.29%, AggregateStats -- 9.32e-02 / 1.03e-01, 1.05e-02 / 6.66e-03 | 20260505 08:17:54.160 | INFO | PerfReport: 11.63%, Baseline -- 3.80e+00 / 4.47e+00, 6.18e-01 / 2.85e-01 | 20260505 08:17:54.160 | INFO | PerfReport: 3.32%, Histogram -- 1.09e+00 / 1.49e+00, 2.12e-01 / 9.54e-02 | 20260505 08:17:54.160 | INFO | PerfReport: 15.52%, DME -- 4.06e+01 / 4.43e+01, 3.89e+00 / 1.49e+00 | 20260505 08:17:54.160 | INFO | PerfReport: 56.51%, FrameLabeling -- 1.85e+01 / 1.98e+01, 1.96e+00 / 2.00e+00 | 20260505 08:17:54.160 | INFO | PerfReport: 7.44%, PulseAccumulating -- 2.43e+00 / 3.02e+00, 4.08e-01 / 2.73e-01 | 20260505 08:17:54.160 | INFO | PerfReport: 3.89%, Metrics -- 1.27e+00 / 1.58e+00, 3.59e-01 / 2.08e-01 | 20260505 08:17:54.160 | INFO | PerfReport: 1.40%, PulseSerializing -- 4.58e-01 / 5.11e-01, 8.74e-02 / 3.73e-02 | 20260505 08:17:54.160 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 08:17:54.160 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:17:54.160 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:17:54.160 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.32e+01 / 0.00e+00 | 20260505 08:17:54.160 | INFO | PerfReport: 22.55%, Upload -- 2.30e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260505 08:17:54.160 | INFO | PerfReport: 21.65%, Download -- 2.20e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260505 08:17:54.160 | INFO | PerfReport: 32.12%, Compute -- 3.27e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 08:17:54.160 | INFO | PerfReport: 1.65%, LockUpload -- 1.68e+00 / 0.00e+00, 7.32e+00 / 0.00e+00 | 20260505 08:17:54.160 | INFO | PerfReport: 1.02%, LockDownload -- 1.04e+00 / 0.00e+00, 2.53e+00 / 0.00e+00 | 20260505 08:17:54.160 | INFO | PerfReport: 20.99%, LockCompute -- 2.14e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260505 08:17:54.160 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.20e-02 / 0.00e+00 | 20260505 08:17:54.299 | INFO | Starting a round of Baz IO | 20260505 08:17:57.576 | INFO | Populating RTMetrics report | 20260505 08:17:57.577 | INFO | RTMetrics report populated | 20260505 08:17:57.577 | INFO | Opening tmp RTMetrics json file | 20260505 08:17:57.583 | INFO | Serializing RTMetrics to json | 20260505 08:17:57.584 | INFO | Serialization done, swapping tmp file now... | 20260505 08:17:57.587 | INFO | RTMetrics done | 20260505 08:17:57.719 | INFO | Finished a round of Baz IO: Wrote 2.45 GiB in 3.42 seconds | 20260505 08:17:57.719 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 27.93% / 0.02% / 72.05% | 20260505 08:17:57.719 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.78097% / 3.57628e-05% / 8.74201e-05% | 20260505 08:17:57.719 | INFO | Baz IO overall: | 20260505 08:17:57.719 | INFO | BytesWritten : 961993488528 | 20260505 08:17:57.719 | INFO | NumEvents : 457173343972 | 20260505 08:17:59.297 | INFO | Analyzing chunk frames = [1712128,1712640) | 20260505 08:17:59.297 | INFO | MemInfo: { | 20260505 08:17:59.297 | INFO | "gpu" : " 17 GB", | 20260505 08:17:59.297 | INFO | "huge" : "", | 20260505 08:17:59.297 | INFO | "huge_shared" : " 27 GB", | 20260505 08:17:59.297 | INFO | "malloc" : " 14 GB", | 20260505 08:17:59.297 | INFO | "pinned" : " 29 GB" | 20260505 08:17:59.297 | INFO | } | 20260505 08:17:59.297 | INFO | Perf report for the last 5.13923 seconds: | 20260505 08:17:59.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:17:59.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:59.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:59.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:59.297 | INFO | ANALYSIS : 65.4%, 2.9, 99.5ms, 0 | 20260505 08:17:59.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:17:59.297 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 08:17:59.298 | INFO | PRE_HQ : 63.6%, 1.9, 61.2ms, 0 | 20260505 08:17:59.298 | INFO | BAZWRITER : 65.8%, 4.2, 142.1ms, 0 | 20260505 08:17:59.298 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 08:17:59.298 | INFO | Uploaded 36GiB to GPU in 2.28062s (15.7852GiB/s) | 20260505 08:17:59.298 | INFO | Downloaded 38GiB to GPU in 2.39212s (15.8855GiB/s) | 20260505 08:18:04.437 | INFO | Analyzing chunk frames = [1712640,1713152) | 20260505 08:18:04.437 | INFO | MemInfo: { | 20260505 08:18:04.437 | INFO | "gpu" : " 17 GB", | 20260505 08:18:04.437 | INFO | "huge" : "", | 20260505 08:18:04.437 | INFO | "huge_shared" : " 27 GB", | 20260505 08:18:04.437 | INFO | "malloc" : " 14 GB", | 20260505 08:18:04.437 | INFO | "pinned" : " 29 GB" | 20260505 08:18:04.437 | INFO | } | 20260505 08:18:04.437 | INFO | Perf report for the last 5.14008 seconds: | 20260505 08:18:04.437 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:18:04.437 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:04.437 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:04.437 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:04.437 | INFO | ANALYSIS : 64.7%, 3.0, 101.1ms, 0 | 20260505 08:18:04.437 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:04.438 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:04.438 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:18:04.438 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:04.438 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:04.438 | INFO | Uploaded 36GiB to GPU in 2.24633s (16.0262GiB/s) | 20260505 08:18:04.438 | INFO | Downloaded 34GiB to GPU in 2.10946s (16.1179GiB/s) | 20260505 08:18:09.577 | INFO | Analyzing chunk frames = [1713152,1713664) | 20260505 08:18:09.577 | INFO | MemInfo: { | 20260505 08:18:09.577 | INFO | "gpu" : " 17 GB", | 20260505 08:18:09.577 | INFO | "huge" : "", | 20260505 08:18:09.577 | INFO | "huge_shared" : " 27 GB", | 20260505 08:18:09.577 | INFO | "malloc" : " 14 GB", | 20260505 08:18:09.577 | INFO | "pinned" : " 29 GB" | 20260505 08:18:09.577 | INFO | } | 20260505 08:18:09.577 | INFO | Perf report for the last 5.13961 seconds: | 20260505 08:18:09.577 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:18:09.577 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:09.577 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:09.577 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:09.577 | INFO | ANALYSIS : 63.7%, 3.0, 99.1ms, 0 | 20260505 08:18:09.577 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:09.577 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:09.577 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:18:09.577 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:09.577 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:09.577 | INFO | Uploaded 36GiB to GPU in 2.17141s (16.5791GiB/s) | 20260505 08:18:09.577 | INFO | Downloaded 34GiB to GPU in 2.06818s (16.4396GiB/s) | 20260505 08:18:14.717 | INFO | Analyzing chunk frames = [1713664,1714176) | 20260505 08:18:14.717 | INFO | MemInfo: { | 20260505 08:18:14.717 | INFO | "gpu" : " 17 GB", | 20260505 08:18:14.717 | INFO | "huge" : "", | 20260505 08:18:14.717 | INFO | "huge_shared" : " 27 GB", | 20260505 08:18:14.717 | INFO | "malloc" : " 14 GB", | 20260505 08:18:14.717 | INFO | "pinned" : " 29 GB" | 20260505 08:18:14.717 | INFO | } | 20260505 08:18:14.717 | INFO | Perf report for the last 5.14012 seconds: | 20260505 08:18:14.717 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:18:14.717 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:14.717 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:14.717 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:14.717 | INFO | ANALYSIS : 63.6%, 3.0, 98.9ms, 0 | 20260505 08:18:14.717 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:14.717 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:14.717 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:18:14.717 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:14.717 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:14.717 | INFO | Uploaded 36GiB to GPU in 2.18786s (16.4544GiB/s) | 20260505 08:18:14.717 | INFO | Downloaded 34GiB to GPU in 2.06879s (16.4347GiB/s) | 20260505 08:18:19.855 | INFO | Analyzing chunk frames = [1714176,1714688) | 20260505 08:18:19.855 | INFO | MemInfo: { | 20260505 08:18:19.855 | INFO | "gpu" : " 17 GB", | 20260505 08:18:19.855 | INFO | "huge" : "", | 20260505 08:18:19.855 | INFO | "huge_shared" : " 27 GB", | 20260505 08:18:19.855 | INFO | "malloc" : " 14 GB", | 20260505 08:18:19.855 | INFO | "pinned" : " 29 GB" | 20260505 08:18:19.855 | INFO | } | 20260505 08:18:19.855 | INFO | Perf report for the last 5.13781 seconds: | 20260505 08:18:19.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:18:19.855 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:19.855 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:19.855 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:19.855 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 08:18:19.855 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:19.855 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:19.855 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 08:18:19.855 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:19.855 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:19.855 | INFO | Uploaded 36GiB to GPU in 2.17558s (16.5473GiB/s) | 20260505 08:18:19.855 | INFO | Downloaded 34GiB to GPU in 2.0566s (16.5322GiB/s) | 20260505 08:18:24.996 | INFO | Analyzing chunk frames = [1714688,1715200) | 20260505 08:18:24.996 | INFO | MemInfo: { | 20260505 08:18:24.996 | INFO | "gpu" : " 17 GB", | 20260505 08:18:24.996 | INFO | "huge" : "", | 20260505 08:18:24.996 | INFO | "huge_shared" : " 27 GB", | 20260505 08:18:24.996 | INFO | "malloc" : " 14 GB", | 20260505 08:18:24.996 | INFO | "pinned" : " 29 GB" | 20260505 08:18:24.996 | INFO | } | 20260505 08:18:24.996 | INFO | Perf report for the last 5.14145 seconds: | 20260505 08:18:24.996 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:18:24.996 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:24.996 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:24.996 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:24.996 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 08:18:24.997 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:24.997 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:24.997 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 08:18:24.997 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:24.997 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:24.997 | INFO | Uploaded 36GiB to GPU in 2.16545s (16.6247GiB/s) | 20260505 08:18:24.997 | INFO | Downloaded 34GiB to GPU in 2.04157s (16.6538GiB/s) | 20260505 08:18:30.134 | INFO | Analyzing chunk frames = [1715200,1715712) | 20260505 08:18:30.134 | INFO | MemInfo: { | 20260505 08:18:30.134 | INFO | "gpu" : " 17 GB", | 20260505 08:18:30.134 | INFO | "huge" : "", | 20260505 08:18:30.134 | INFO | "huge_shared" : " 27 GB", | 20260505 08:18:30.134 | INFO | "malloc" : " 14 GB", | 20260505 08:18:30.134 | INFO | "pinned" : " 29 GB" | 20260505 08:18:30.134 | INFO | } | 20260505 08:18:30.134 | INFO | Perf report for the last 5.13749 seconds: | 20260505 08:18:30.134 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:18:30.134 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:30.134 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:30.134 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:30.134 | INFO | ANALYSIS : 64.1%, 3.0, 100.0ms, 0 | 20260505 08:18:30.134 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:30.134 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:30.134 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 08:18:30.134 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:30.134 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:30.134 | INFO | Uploaded 36GiB to GPU in 2.20547s (16.323GiB/s) | 20260505 08:18:30.134 | INFO | Downloaded 34GiB to GPU in 2.15136s (15.804GiB/s) | 20260505 08:18:35.274 | INFO | Analyzing chunk frames = [1715712,1716224) | 20260505 08:18:35.274 | INFO | MemInfo: { | 20260505 08:18:35.274 | INFO | "gpu" : " 17 GB", | 20260505 08:18:35.274 | INFO | "huge" : "", | 20260505 08:18:35.274 | INFO | "huge_shared" : " 27 GB", | 20260505 08:18:35.274 | INFO | "malloc" : " 14 GB", | 20260505 08:18:35.274 | INFO | "pinned" : " 29 GB" | 20260505 08:18:35.274 | INFO | } | 20260505 08:18:35.275 | INFO | Perf report for the last 5.14057 seconds: | 20260505 08:18:35.275 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:18:35.275 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:35.275 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:35.275 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:35.275 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 08:18:35.275 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:35.275 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:35.275 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 08:18:35.275 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:35.275 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:35.275 | INFO | Uploaded 36GiB to GPU in 2.19834s (16.376GiB/s) | 20260505 08:18:35.275 | INFO | Downloaded 34GiB to GPU in 2.04075s (16.6606GiB/s) | 20260505 08:18:35.415 | INFO | Starting a round of Baz IO | 20260505 08:18:38.687 | INFO | Populating RTMetrics report | 20260505 08:18:38.687 | INFO | RTMetrics report populated | 20260505 08:18:38.687 | INFO | Opening tmp RTMetrics json file | 20260505 08:18:38.739 | INFO | Serializing RTMetrics to json | 20260505 08:18:38.742 | INFO | Serialization done, swapping tmp file now... | 20260505 08:18:38.746 | INFO | RTMetrics done | 20260505 08:18:38.915 | INFO | Finished a round of Baz IO: Wrote 2.449 GiB in 3.5 seconds | 20260505 08:18:38.915 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 27.91% / 0.02% / 72.07% | 20260505 08:18:38.915 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.77363% / 3.97364e-05% / 7.15256e-05% | 20260505 08:18:38.915 | INFO | Baz IO overall: | 20260505 08:18:38.915 | INFO | BytesWritten : 964623205147 | 20260505 08:18:38.915 | INFO | NumEvents : 458431891887 | 20260505 08:18:40.414 | INFO | Analyzing chunk frames = [1716224,1716736) | 20260505 08:18:40.414 | INFO | MemInfo: { | 20260505 08:18:40.414 | INFO | "gpu" : " 17 GB", | 20260505 08:18:40.414 | INFO | "huge" : "", | 20260505 08:18:40.414 | INFO | "huge_shared" : " 27 GB", | 20260505 08:18:40.414 | INFO | "malloc" : " 14 GB", | 20260505 08:18:40.414 | INFO | "pinned" : " 29 GB" | 20260505 08:18:40.414 | INFO | } | 20260505 08:18:40.414 | INFO | Perf report for the last 5.13964 seconds: | 20260505 08:18:40.414 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:18:40.414 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:40.414 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:40.414 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:40.414 | INFO | ANALYSIS : 65.3%, 2.9, 99.5ms, 0 | 20260505 08:18:40.414 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:40.414 | INFO | RT_METRICS : 46.9%, 1.0, 24.6ms, 0 | 20260505 08:18:40.414 | INFO | PRE_HQ : 63.4%, 1.9, 61.5ms, 0 | 20260505 08:18:40.414 | INFO | BAZWRITER : 67.3%, 4.2, 146.4ms, 0 | 20260505 08:18:40.414 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260505 08:18:40.414 | INFO | Uploaded 36GiB to GPU in 2.2615s (15.9186GiB/s) | 20260505 08:18:40.414 | INFO | Downloaded 38GiB to GPU in 2.38137s (15.9572GiB/s) | 20260505 08:18:45.552 | INFO | Analyzing chunk frames = [1716736,1717248) | 20260505 08:18:45.552 | INFO | MemInfo: { | 20260505 08:18:45.552 | INFO | "gpu" : " 17 GB", | 20260505 08:18:45.552 | INFO | "huge" : "", | 20260505 08:18:45.552 | INFO | "huge_shared" : " 27 GB", | 20260505 08:18:45.552 | INFO | "malloc" : " 14 GB", | 20260505 08:18:45.552 | INFO | "pinned" : " 29 GB" | 20260505 08:18:45.552 | INFO | } | 20260505 08:18:45.552 | INFO | Perf report for the last 5.13765 seconds: | 20260505 08:18:45.552 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:18:45.552 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:45.552 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:45.552 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:45.552 | INFO | ANALYSIS : 64.6%, 3.0, 100.8ms, 0 | 20260505 08:18:45.552 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:45.552 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:45.552 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:18:45.552 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:45.552 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:45.552 | INFO | Uploaded 36GiB to GPU in 2.23853s (16.082GiB/s) | 20260505 08:18:45.552 | INFO | Downloaded 34GiB to GPU in 2.1017s (16.1774GiB/s) | 20260505 08:18:50.691 | INFO | Analyzing chunk frames = [1717248,1717760) | 20260505 08:18:50.691 | INFO | MemInfo: { | 20260505 08:18:50.691 | INFO | "gpu" : " 17 GB", | 20260505 08:18:50.691 | INFO | "huge" : "", | 20260505 08:18:50.691 | INFO | "huge_shared" : " 27 GB", | 20260505 08:18:50.691 | INFO | "malloc" : " 14 GB", | 20260505 08:18:50.691 | INFO | "pinned" : " 29 GB" | 20260505 08:18:50.691 | INFO | } | 20260505 08:18:50.691 | INFO | Perf report for the last 5.13966 seconds: | 20260505 08:18:50.691 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:18:50.691 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:50.692 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:50.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:50.692 | INFO | ANALYSIS : 63.7%, 3.0, 99.1ms, 0 | 20260505 08:18:50.692 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:50.692 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:50.692 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:18:50.692 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:50.692 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:50.692 | INFO | Uploaded 36GiB to GPU in 2.16939s (16.5945GiB/s) | 20260505 08:18:50.692 | INFO | Downloaded 34GiB to GPU in 2.06629s (16.4546GiB/s) | 20260505 08:18:55.832 | INFO | Analyzing chunk frames = [1717760,1718272) | 20260505 08:18:55.832 | INFO | MemInfo: { | 20260505 08:18:55.832 | INFO | "gpu" : " 17 GB", | 20260505 08:18:55.832 | INFO | "huge" : "", | 20260505 08:18:55.832 | INFO | "huge_shared" : " 27 GB", | 20260505 08:18:55.832 | INFO | "malloc" : " 14 GB", | 20260505 08:18:55.832 | INFO | "pinned" : " 29 GB" | 20260505 08:18:55.832 | INFO | } | 20260505 08:18:55.832 | INFO | Perf report for the last 5.14029 seconds: | 20260505 08:18:55.832 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:18:55.832 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:55.832 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:55.832 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:55.832 | INFO | ANALYSIS : 63.6%, 3.0, 98.8ms, 0 | 20260505 08:18:55.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:55.832 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:55.832 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260505 08:18:55.832 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:55.832 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:18:55.832 | INFO | Uploaded 36GiB to GPU in 2.18981s (16.4398GiB/s) | 20260505 08:18:55.832 | INFO | Downloaded 34GiB to GPU in 2.06818s (16.4396GiB/s) | 20260505 08:19:00.969 | INFO | Analyzing chunk frames = [1718272,1718784) | 20260505 08:19:00.969 | INFO | MemInfo: { | 20260505 08:19:00.969 | INFO | "gpu" : " 17 GB", | 20260505 08:19:00.969 | INFO | "huge" : "", | 20260505 08:19:00.970 | INFO | "huge_shared" : " 27 GB", | 20260505 08:19:00.970 | INFO | "malloc" : " 14 GB", | 20260505 08:19:00.970 | INFO | "pinned" : " 29 GB" | 20260505 08:19:00.970 | INFO | } | 20260505 08:19:00.970 | INFO | Perf report for the last 5.13797 seconds: | 20260505 08:19:00.970 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:19:00.970 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:00.970 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:00.970 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:00.970 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 08:19:00.970 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:00.970 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:00.970 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 08:19:00.970 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:00.970 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:00.970 | INFO | Uploaded 36GiB to GPU in 2.18496s (16.4762GiB/s) | 20260505 08:19:00.970 | INFO | Downloaded 34GiB to GPU in 2.0549s (16.5458GiB/s) | 20260505 08:19:06.110 | INFO | Analyzing chunk frames = [1718784,1719296) | 20260505 08:19:06.110 | INFO | MemInfo: { | 20260505 08:19:06.110 | INFO | "gpu" : " 17 GB", | 20260505 08:19:06.110 | INFO | "huge" : "", | 20260505 08:19:06.110 | INFO | "huge_shared" : " 27 GB", | 20260505 08:19:06.110 | INFO | "malloc" : " 14 GB", | 20260505 08:19:06.110 | INFO | "pinned" : " 29 GB" | 20260505 08:19:06.110 | INFO | } | 20260505 08:19:06.110 | INFO | Perf report for the last 5.14003 seconds: | 20260505 08:19:06.110 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:19:06.110 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:06.110 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:06.110 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:06.110 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 08:19:06.110 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:06.110 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:06.110 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 08:19:06.110 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:06.110 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:06.110 | INFO | Uploaded 36GiB to GPU in 2.17056s (16.5856GiB/s) | 20260505 08:19:06.110 | INFO | Downloaded 34GiB to GPU in 2.06156s (16.4924GiB/s) | 20260505 08:19:11.249 | INFO | Analyzing chunk frames = [1719296,1719808) | 20260505 08:19:11.249 | INFO | MemInfo: { | 20260505 08:19:11.249 | INFO | "gpu" : " 17 GB", | 20260505 08:19:11.249 | INFO | "huge" : "", | 20260505 08:19:11.249 | INFO | "huge_shared" : " 27 GB", | 20260505 08:19:11.249 | INFO | "malloc" : " 14 GB", | 20260505 08:19:11.249 | INFO | "pinned" : " 29 GB" | 20260505 08:19:11.249 | INFO | } | 20260505 08:19:11.249 | INFO | Perf report for the last 5.13938 seconds: | 20260505 08:19:11.249 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:19:11.249 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:11.249 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:11.249 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:11.249 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 08:19:11.249 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:11.249 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:11.249 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 08:19:11.249 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:11.249 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:11.250 | INFO | Uploaded 36GiB to GPU in 2.19555s (16.3968GiB/s) | 20260505 08:19:11.250 | INFO | Downloaded 34GiB to GPU in 2.14917s (15.8201GiB/s) | 20260505 08:19:16.389 | INFO | Analyzing chunk frames = [1719808,1720320) | 20260505 08:19:16.389 | INFO | MemInfo: { | 20260505 08:19:16.389 | INFO | "gpu" : " 17 GB", | 20260505 08:19:16.389 | INFO | "huge" : "", | 20260505 08:19:16.389 | INFO | "huge_shared" : " 27 GB", | 20260505 08:19:16.389 | INFO | "malloc" : " 14 GB", | 20260505 08:19:16.389 | INFO | "pinned" : " 29 GB" | 20260505 08:19:16.389 | INFO | } | 20260505 08:19:16.389 | INFO | Perf report for the last 5.13983 seconds: | 20260505 08:19:16.389 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:19:16.389 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:16.389 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:16.389 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:16.389 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 08:19:16.389 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:16.389 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:16.389 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 08:19:16.389 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:16.389 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:16.389 | INFO | Uploaded 36GiB to GPU in 2.19683s (16.3873GiB/s) | 20260505 08:19:16.389 | INFO | Downloaded 34GiB to GPU in 2.03673s (16.6935GiB/s) | 20260505 08:19:16.389 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 08:19:16.389 | INFO | Total mem usage: 15.3 GiB | 20260505 08:19:16.389 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 08:19:16.389 | INFO | Recent High Water Marks Sum: 14.7 GiB (Savings: -4.49%) | 20260505 08:19:16.389 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 08:19:16.389 | INFO | High Water Breakdown: | 20260505 08:19:16.389 | INFO | 61% : 8.95 GiB ( 95.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 08:19:16.389 | INFO | 38.8% : 5.69 GiB ( 85.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 08:19:16.389 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 08:19:16.390 | INFO | ------------------------------------------ | 20260505 08:19:16.390 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:19:16.390 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 08:19:16.390 | INFO | Total mem usage: 26.5 GiB | 20260505 08:19:16.390 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 08:19:16.390 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 08:19:16.390 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 08:19:16.390 | INFO | High Water Breakdown: | 20260505 08:19:16.390 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:19:16.390 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:19:16.390 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:19:16.390 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:19:16.390 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:19:16.390 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:19:16.390 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 08:19:16.390 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 08:19:16.390 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:19:16.390 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 08:19:16.390 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 08:19:16.390 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 08:19:16.390 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 08:19:16.390 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 08:19:16.390 | INFO | ------------------------------------------ | 20260505 08:19:16.390 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:19:16.390 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 08:19:16.390 | INFO | Total mem usage: 25.6 GiB | 20260505 08:19:16.390 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 08:19:16.390 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 08:19:16.390 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 08:19:16.390 | INFO | High Water Breakdown: | 20260505 08:19:16.390 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 08:19:16.390 | INFO | ------------------------------------------ | 20260505 08:19:16.390 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:19:16.390 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 08:19:16.390 | INFO | Total mem usage: 15.7 GiB | 20260505 08:19:16.390 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 08:19:16.390 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 08:19:16.390 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 08:19:16.390 | INFO | High Water Breakdown: | 20260505 08:19:16.390 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 08:19:16.390 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 08:19:16.390 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 08:19:16.390 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 08:19:16.390 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 08:19:16.390 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 08:19:16.390 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 08:19:16.390 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 08:19:16.390 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 08:19:16.390 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:19:16.390 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 08:19:16.390 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:19:16.390 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 08:19:16.390 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 08:19:16.390 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 08:19:16.390 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 08:19:16.390 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:19:16.390 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 08:19:16.390 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:19:16.390 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:19:16.390 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:19:16.390 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 08:19:16.390 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 08:19:16.390 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:19:16.390 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 08:19:16.390 | INFO | ------------------------------------------ | 20260505 08:19:16.390 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:19:16.391 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 08:19:16.391 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:19:16.391 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:19:16.391 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.49e+01, 1.36e+01 / 1.70e+00 | 20260505 08:19:16.391 | INFO | PerfReport: 0.28%, AggregateStats -- 9.30e-02 / 1.03e-01, 1.06e-02 / 3.98e-03 | 20260505 08:19:16.391 | INFO | PerfReport: 11.66%, Baseline -- 3.81e+00 / 4.45e+00, 6.30e-01 / 2.75e-01 | 20260505 08:19:16.391 | INFO | PerfReport: 3.32%, Histogram -- 1.09e+00 / 1.49e+00, 2.14e-01 / 1.01e-01 | 20260505 08:19:16.391 | INFO | PerfReport: 15.46%, DME -- 4.04e+01 / 4.40e+01, 3.89e+00 / 1.16e+00 | 20260505 08:19:16.391 | INFO | PerfReport: 56.46%, FrameLabeling -- 1.84e+01 / 1.97e+01, 1.95e+00 / 1.92e+00 | 20260505 08:19:16.391 | INFO | PerfReport: 7.48%, PulseAccumulating -- 2.44e+00 / 3.04e+00, 4.13e-01 / 2.36e-01 | 20260505 08:19:16.391 | INFO | PerfReport: 3.91%, Metrics -- 1.28e+00 / 1.59e+00, 3.65e-01 / 2.03e-01 | 20260505 08:19:16.391 | INFO | PerfReport: 1.41%, PulseSerializing -- 4.59e-01 / 5.12e-01, 8.87e-02 / 3.26e-02 | 20260505 08:19:16.391 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 08:19:16.391 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:19:16.391 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:19:16.391 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.32e+01 / 0.00e+00 | 20260505 08:19:16.391 | INFO | PerfReport: 22.57%, Upload -- 2.29e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260505 08:19:16.391 | INFO | PerfReport: 21.68%, Download -- 2.20e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260505 08:19:16.391 | INFO | PerfReport: 32.13%, Compute -- 3.27e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 08:19:16.391 | INFO | PerfReport: 1.65%, LockUpload -- 1.68e+00 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260505 08:19:16.391 | INFO | PerfReport: 1.01%, LockDownload -- 1.03e+00 / 0.00e+00, 2.46e+00 / 0.00e+00 | 20260505 08:19:16.391 | INFO | PerfReport: 20.95%, LockCompute -- 2.13e+01 / 0.00e+00, 1.77e+01 / 0.00e+00 | 20260505 08:19:16.391 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.73e-02 / 0.00e+00 | 20260505 08:19:16.530 | INFO | Starting a round of Baz IO | 20260505 08:19:19.805 | INFO | Populating RTMetrics report | 20260505 08:19:19.806 | INFO | RTMetrics report populated | 20260505 08:19:19.806 | INFO | Opening tmp RTMetrics json file | 20260505 08:19:19.830 | INFO | Serializing RTMetrics to json | 20260505 08:19:19.838 | INFO | Serialization done, swapping tmp file now... | 20260505 08:19:19.857 | INFO | RTMetrics done | 20260505 08:19:19.957 | INFO | Finished a round of Baz IO: Wrote 2.451 GiB in 3.426 seconds | 20260505 08:19:19.957 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 27.89% / 0.02% / 72.09% | 20260505 08:19:19.957 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.76536% / 2.38419e-05% / 7.94729e-05% | 20260505 08:19:19.957 | INFO | Baz IO overall: | 20260505 08:19:19.957 | INFO | BytesWritten : 967254568046 | 20260505 08:19:19.957 | INFO | NumEvents : 459691232945 | 20260505 08:19:21.527 | INFO | Analyzing chunk frames = [1720320,1720832) | 20260505 08:19:21.527 | INFO | MemInfo: { | 20260505 08:19:21.527 | INFO | "gpu" : " 17 GB", | 20260505 08:19:21.527 | INFO | "huge" : "", | 20260505 08:19:21.527 | INFO | "huge_shared" : " 27 GB", | 20260505 08:19:21.527 | INFO | "malloc" : " 14 GB", | 20260505 08:19:21.527 | INFO | "pinned" : " 29 GB" | 20260505 08:19:21.527 | INFO | } | 20260505 08:19:21.527 | INFO | Perf report for the last 5.13784 seconds: | 20260505 08:19:21.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:19:21.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:21.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:21.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:21.527 | INFO | ANALYSIS : 65.4%, 2.9, 99.9ms, 0 | 20260505 08:19:21.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:21.527 | INFO | RT_METRICS : 46.7%, 1.0, 24.5ms, 0 | 20260505 08:19:21.527 | INFO | PRE_HQ : 63.5%, 1.9, 61.2ms, 0 | 20260505 08:19:21.527 | INFO | BAZWRITER : 66.0%, 4.1, 139.9ms, 0 | 20260505 08:19:21.527 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 08:19:21.527 | INFO | Uploaded 36GiB to GPU in 2.25714s (15.9494GiB/s) | 20260505 08:19:21.527 | INFO | Downloaded 38GiB to GPU in 2.37326s (16.0117GiB/s) | 20260505 08:19:26.666 | INFO | Analyzing chunk frames = [1720832,1721344) | 20260505 08:19:26.666 | INFO | MemInfo: { | 20260505 08:19:26.666 | INFO | "gpu" : " 17 GB", | 20260505 08:19:26.666 | INFO | "huge" : "", | 20260505 08:19:26.666 | INFO | "huge_shared" : " 27 GB", | 20260505 08:19:26.666 | INFO | "malloc" : " 14 GB", | 20260505 08:19:26.666 | INFO | "pinned" : " 29 GB" | 20260505 08:19:26.666 | INFO | } | 20260505 08:19:26.666 | INFO | Perf report for the last 5.13943 seconds: | 20260505 08:19:26.666 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:19:26.666 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:26.666 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:26.666 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:26.666 | INFO | ANALYSIS : 64.7%, 3.0, 101.0ms, 0 | 20260505 08:19:26.666 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:26.666 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:26.666 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:19:26.666 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:26.666 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:26.667 | INFO | Uploaded 36GiB to GPU in 2.24906s (16.0067GiB/s) | 20260505 08:19:26.667 | INFO | Downloaded 34GiB to GPU in 2.10621s (16.1427GiB/s) | 20260505 08:19:31.806 | INFO | Analyzing chunk frames = [1721344,1721856) | 20260505 08:19:31.806 | INFO | MemInfo: { | 20260505 08:19:31.806 | INFO | "gpu" : " 17 GB", | 20260505 08:19:31.806 | INFO | "huge" : "", | 20260505 08:19:31.806 | INFO | "huge_shared" : " 27 GB", | 20260505 08:19:31.806 | INFO | "malloc" : " 14 GB", | 20260505 08:19:31.806 | INFO | "pinned" : " 29 GB" | 20260505 08:19:31.806 | INFO | } | 20260505 08:19:31.806 | INFO | Perf report for the last 5.14013 seconds: | 20260505 08:19:31.806 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:19:31.806 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:31.806 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:31.807 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:31.807 | INFO | ANALYSIS : 63.7%, 3.0, 99.1ms, 0 | 20260505 08:19:31.807 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:31.807 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:31.807 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:19:31.807 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:31.807 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:31.807 | INFO | Uploaded 36GiB to GPU in 2.17223s (16.5728GiB/s) | 20260505 08:19:31.807 | INFO | Downloaded 34GiB to GPU in 2.06937s (16.4301GiB/s) | 20260505 08:19:36.945 | INFO | Analyzing chunk frames = [1721856,1722368) | 20260505 08:19:36.945 | INFO | MemInfo: { | 20260505 08:19:36.945 | INFO | "gpu" : " 17 GB", | 20260505 08:19:36.945 | INFO | "huge" : "", | 20260505 08:19:36.945 | INFO | "huge_shared" : " 27 GB", | 20260505 08:19:36.945 | INFO | "malloc" : " 14 GB", | 20260505 08:19:36.945 | INFO | "pinned" : " 29 GB" | 20260505 08:19:36.945 | INFO | } | 20260505 08:19:36.945 | INFO | Perf report for the last 5.13879 seconds: | 20260505 08:19:36.945 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:19:36.945 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:36.945 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:36.945 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:36.945 | INFO | ANALYSIS : 63.6%, 3.0, 98.9ms, 0 | 20260505 08:19:36.945 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:36.945 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:36.945 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:19:36.945 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:36.945 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:36.946 | INFO | Uploaded 36GiB to GPU in 2.18728s (16.4588GiB/s) | 20260505 08:19:36.946 | INFO | Downloaded 34GiB to GPU in 2.06863s (16.436GiB/s) | 20260505 08:19:42.085 | INFO | Analyzing chunk frames = [1722368,1722880) | 20260505 08:19:42.086 | INFO | MemInfo: { | 20260505 08:19:42.086 | INFO | "gpu" : " 17 GB", | 20260505 08:19:42.086 | INFO | "huge" : "", | 20260505 08:19:42.086 | INFO | "huge_shared" : " 27 GB", | 20260505 08:19:42.086 | INFO | "malloc" : " 14 GB", | 20260505 08:19:42.086 | INFO | "pinned" : " 29 GB" | 20260505 08:19:42.086 | INFO | } | 20260505 08:19:42.086 | INFO | Perf report for the last 5.14046 seconds: | 20260505 08:19:42.086 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:19:42.086 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:42.086 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:42.086 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:42.086 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 08:19:42.086 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:42.086 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:42.086 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 08:19:42.086 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:42.086 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:42.086 | INFO | Uploaded 36GiB to GPU in 2.17455s (16.5551GiB/s) | 20260505 08:19:42.086 | INFO | Downloaded 34GiB to GPU in 2.05752s (16.5247GiB/s) | 20260505 08:19:47.225 | INFO | Analyzing chunk frames = [1722880,1723392) | 20260505 08:19:47.225 | INFO | MemInfo: { | 20260505 08:19:47.225 | INFO | "gpu" : " 17 GB", | 20260505 08:19:47.225 | INFO | "huge" : "", | 20260505 08:19:47.225 | INFO | "huge_shared" : " 27 GB", | 20260505 08:19:47.225 | INFO | "malloc" : " 14 GB", | 20260505 08:19:47.225 | INFO | "pinned" : " 29 GB" | 20260505 08:19:47.226 | INFO | } | 20260505 08:19:47.226 | INFO | Perf report for the last 5.13982 seconds: | 20260505 08:19:47.226 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:19:47.226 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:47.226 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:47.226 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:47.226 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 08:19:47.226 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:47.226 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:47.226 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 08:19:47.226 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:47.226 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:47.226 | INFO | Uploaded 36GiB to GPU in 2.18712s (16.46GiB/s) | 20260505 08:19:47.226 | INFO | Downloaded 34GiB to GPU in 2.05578s (16.5388GiB/s) | 20260505 08:19:52.365 | INFO | Analyzing chunk frames = [1723392,1723904) | 20260505 08:19:52.365 | INFO | MemInfo: { | 20260505 08:19:52.365 | INFO | "gpu" : " 17 GB", | 20260505 08:19:52.365 | INFO | "huge" : "", | 20260505 08:19:52.365 | INFO | "huge_shared" : " 27 GB", | 20260505 08:19:52.365 | INFO | "malloc" : " 14 GB", | 20260505 08:19:52.365 | INFO | "pinned" : " 29 GB" | 20260505 08:19:52.365 | INFO | } | 20260505 08:19:52.365 | INFO | Perf report for the last 5.13925 seconds: | 20260505 08:19:52.365 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:19:52.365 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:52.365 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:52.365 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:52.365 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 08:19:52.365 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:52.365 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:52.365 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 08:19:52.365 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:52.365 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:52.365 | INFO | Uploaded 36GiB to GPU in 2.21419s (16.2587GiB/s) | 20260505 08:19:52.365 | INFO | Downloaded 34GiB to GPU in 2.1514s (15.8037GiB/s) | 20260505 08:19:57.503 | INFO | Analyzing chunk frames = [1723904,1724416) | 20260505 08:19:57.503 | INFO | MemInfo: { | 20260505 08:19:57.503 | INFO | "gpu" : " 17 GB", | 20260505 08:19:57.503 | INFO | "huge" : "", | 20260505 08:19:57.503 | INFO | "huge_shared" : " 27 GB", | 20260505 08:19:57.503 | INFO | "malloc" : " 14 GB", | 20260505 08:19:57.503 | INFO | "pinned" : " 29 GB" | 20260505 08:19:57.503 | INFO | } | 20260505 08:19:57.503 | INFO | Perf report for the last 5.13839 seconds: | 20260505 08:19:57.503 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:19:57.503 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:57.503 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:57.503 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:57.503 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 08:19:57.503 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:57.503 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:57.503 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 08:19:57.503 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:57.503 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:19:57.504 | INFO | Uploaded 36GiB to GPU in 2.19047s (16.4349GiB/s) | 20260505 08:19:57.504 | INFO | Downloaded 34GiB to GPU in 2.03878s (16.6766GiB/s) | 20260505 08:19:57.643 | INFO | Starting a round of Baz IO | 20260505 08:20:00.922 | INFO | Populating RTMetrics report | 20260505 08:20:00.923 | INFO | RTMetrics report populated | 20260505 08:20:00.923 | INFO | Opening tmp RTMetrics json file | 20260505 08:20:00.957 | INFO | Serializing RTMetrics to json | 20260505 08:20:00.962 | INFO | Serialization done, swapping tmp file now... | 20260505 08:20:00.964 | INFO | RTMetrics done | 20260505 08:20:01.103 | INFO | Finished a round of Baz IO: Wrote 2.45 GiB in 3.459 seconds | 20260505 08:20:01.103 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 27.86% / 0.02% / 72.11% | 20260505 08:20:01.103 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.7471% / 2.78155e-05% / 7.94729e-05% | 20260505 08:20:01.103 | INFO | Baz IO overall: | 20260505 08:20:01.103 | INFO | BytesWritten : 969885257985 | 20260505 08:20:01.103 | INFO | NumEvents : 460949603944 | 20260505 08:20:02.642 | INFO | Analyzing chunk frames = [1724416,1724928) | 20260505 08:20:02.642 | INFO | MemInfo: { | 20260505 08:20:02.642 | INFO | "gpu" : " 17 GB", | 20260505 08:20:02.642 | INFO | "huge" : "", | 20260505 08:20:02.642 | INFO | "huge_shared" : " 27 GB", | 20260505 08:20:02.642 | INFO | "malloc" : " 14 GB", | 20260505 08:20:02.642 | INFO | "pinned" : " 29 GB" | 20260505 08:20:02.642 | INFO | } | 20260505 08:20:02.642 | INFO | Perf report for the last 5.13865 seconds: | 20260505 08:20:02.642 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:20:02.642 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:02.642 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:02.642 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:02.642 | INFO | ANALYSIS : 65.4%, 2.9, 99.9ms, 0 | 20260505 08:20:02.642 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:02.642 | INFO | RT_METRICS : 46.6%, 1.0, 24.4ms, 0 | 20260505 08:20:02.642 | INFO | PRE_HQ : 63.5%, 1.9, 61.6ms, 0 | 20260505 08:20:02.642 | INFO | BAZWRITER : 66.6%, 4.1, 141.9ms, 0 | 20260505 08:20:02.642 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 08:20:02.642 | INFO | Uploaded 36GiB to GPU in 2.27676s (15.812GiB/s) | 20260505 08:20:02.642 | INFO | Downloaded 38GiB to GPU in 2.37276s (16.0151GiB/s) | 20260505 08:20:07.782 | INFO | Analyzing chunk frames = [1724928,1725440) | 20260505 08:20:07.782 | INFO | MemInfo: { | 20260505 08:20:07.782 | INFO | "gpu" : " 17 GB", | 20260505 08:20:07.782 | INFO | "huge" : "", | 20260505 08:20:07.782 | INFO | "huge_shared" : " 27 GB", | 20260505 08:20:07.782 | INFO | "malloc" : " 14 GB", | 20260505 08:20:07.782 | INFO | "pinned" : " 29 GB" | 20260505 08:20:07.782 | INFO | } | 20260505 08:20:07.782 | INFO | Perf report for the last 5.14005 seconds: | 20260505 08:20:07.782 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:20:07.782 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:07.782 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:07.782 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:07.782 | INFO | ANALYSIS : 64.8%, 3.0, 101.1ms, 0 | 20260505 08:20:07.782 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:07.782 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:07.782 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 08:20:07.782 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:07.782 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:07.782 | INFO | Uploaded 36GiB to GPU in 2.24409s (16.0421GiB/s) | 20260505 08:20:07.783 | INFO | Downloaded 34GiB to GPU in 2.08993s (16.2685GiB/s) | 20260505 08:20:12.921 | INFO | Analyzing chunk frames = [1725440,1725952) | 20260505 08:20:12.921 | INFO | MemInfo: { | 20260505 08:20:12.921 | INFO | "gpu" : " 17 GB", | 20260505 08:20:12.921 | INFO | "huge" : "", | 20260505 08:20:12.921 | INFO | "huge_shared" : " 27 GB", | 20260505 08:20:12.921 | INFO | "malloc" : " 14 GB", | 20260505 08:20:12.921 | INFO | "pinned" : " 29 GB" | 20260505 08:20:12.921 | INFO | } | 20260505 08:20:12.921 | INFO | Perf report for the last 5.13901 seconds: | 20260505 08:20:12.921 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:20:12.921 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:12.921 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:12.921 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:12.921 | INFO | ANALYSIS : 63.7%, 3.0, 99.1ms, 0 | 20260505 08:20:12.921 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:12.921 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:12.921 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 08:20:12.921 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:12.921 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:12.921 | INFO | Uploaded 36GiB to GPU in 2.16775s (16.6071GiB/s) | 20260505 08:20:12.921 | INFO | Downloaded 34GiB to GPU in 2.06342s (16.4775GiB/s) | 20260505 08:20:18.060 | INFO | Analyzing chunk frames = [1725952,1726464) | 20260505 08:20:18.060 | INFO | MemInfo: { | 20260505 08:20:18.060 | INFO | "gpu" : " 17 GB", | 20260505 08:20:18.060 | INFO | "huge" : "", | 20260505 08:20:18.060 | INFO | "huge_shared" : " 27 GB", | 20260505 08:20:18.060 | INFO | "malloc" : " 14 GB", | 20260505 08:20:18.060 | INFO | "pinned" : " 29 GB" | 20260505 08:20:18.060 | INFO | } | 20260505 08:20:18.060 | INFO | Perf report for the last 5.13938 seconds: | 20260505 08:20:18.060 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:20:18.060 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:18.060 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:18.060 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:18.060 | INFO | ANALYSIS : 63.7%, 3.0, 99.0ms, 0 | 20260505 08:20:18.060 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:18.061 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:18.061 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:20:18.061 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:18.061 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:18.061 | INFO | Uploaded 36GiB to GPU in 2.19336s (16.4132GiB/s) | 20260505 08:20:18.061 | INFO | Downloaded 34GiB to GPU in 2.06706s (16.4485GiB/s) | 20260505 08:20:23.199 | INFO | Analyzing chunk frames = [1726464,1726976) | 20260505 08:20:23.199 | INFO | MemInfo: { | 20260505 08:20:23.199 | INFO | "gpu" : " 17 GB", | 20260505 08:20:23.199 | INFO | "huge" : "", | 20260505 08:20:23.199 | INFO | "huge_shared" : " 27 GB", | 20260505 08:20:23.199 | INFO | "malloc" : " 14 GB", | 20260505 08:20:23.199 | INFO | "pinned" : " 29 GB" | 20260505 08:20:23.199 | INFO | } | 20260505 08:20:23.199 | INFO | Perf report for the last 5.13902 seconds: | 20260505 08:20:23.200 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:20:23.200 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:23.200 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:23.200 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:23.200 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 08:20:23.200 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:23.200 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:23.200 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:20:23.200 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:23.200 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:23.200 | INFO | Uploaded 36GiB to GPU in 2.18839s (16.4505GiB/s) | 20260505 08:20:23.200 | INFO | Downloaded 34GiB to GPU in 2.05627s (16.5348GiB/s) | 20260505 08:20:28.340 | INFO | Analyzing chunk frames = [1726976,1727488) | 20260505 08:20:28.340 | INFO | MemInfo: { | 20260505 08:20:28.340 | INFO | "gpu" : " 17 GB", | 20260505 08:20:28.340 | INFO | "huge" : "", | 20260505 08:20:28.340 | INFO | "huge_shared" : " 27 GB", | 20260505 08:20:28.340 | INFO | "malloc" : " 14 GB", | 20260505 08:20:28.340 | INFO | "pinned" : " 29 GB" | 20260505 08:20:28.340 | INFO | } | 20260505 08:20:28.340 | INFO | Perf report for the last 5.14068 seconds: | 20260505 08:20:28.340 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:20:28.340 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:28.340 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:28.340 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:28.340 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 08:20:28.340 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:28.340 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:28.340 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:20:28.340 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:28.340 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:28.340 | INFO | Uploaded 36GiB to GPU in 2.19246s (16.4199GiB/s) | 20260505 08:20:28.340 | INFO | Downloaded 34GiB to GPU in 2.05508s (16.5444GiB/s) | 20260505 08:20:33.478 | INFO | Analyzing chunk frames = [1727488,1728000) | 20260505 08:20:33.478 | INFO | MemInfo: { | 20260505 08:20:33.478 | INFO | "gpu" : " 17 GB", | 20260505 08:20:33.478 | INFO | "huge" : "", | 20260505 08:20:33.478 | INFO | "huge_shared" : " 27 GB", | 20260505 08:20:33.478 | INFO | "malloc" : " 14 GB", | 20260505 08:20:33.478 | INFO | "pinned" : " 29 GB" | 20260505 08:20:33.478 | INFO | } | 20260505 08:20:33.478 | INFO | Perf report for the last 5.13829 seconds: | 20260505 08:20:33.478 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:20:33.478 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:33.478 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:33.478 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:33.478 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 08:20:33.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:33.479 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:33.479 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 08:20:33.479 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:33.479 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:33.479 | INFO | Uploaded 36GiB to GPU in 2.21404s (16.2599GiB/s) | 20260505 08:20:33.479 | INFO | Downloaded 34GiB to GPU in 2.15294s (15.7923GiB/s) | 20260505 08:20:38.617 | INFO | Analyzing chunk frames = [1728000,1728512) | 20260505 08:20:38.617 | INFO | MemInfo: { | 20260505 08:20:38.617 | INFO | "gpu" : " 17 GB", | 20260505 08:20:38.617 | INFO | "huge" : "", | 20260505 08:20:38.617 | INFO | "huge_shared" : " 27 GB", | 20260505 08:20:38.617 | INFO | "malloc" : " 14 GB", | 20260505 08:20:38.617 | INFO | "pinned" : " 29 GB" | 20260505 08:20:38.617 | INFO | } | 20260505 08:20:38.617 | INFO | Perf report for the last 5.13885 seconds: | 20260505 08:20:38.617 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:20:38.617 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:38.617 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:38.617 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:38.617 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 08:20:38.617 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:38.617 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:38.617 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 08:20:38.617 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:38.617 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:38.618 | INFO | Uploaded 36GiB to GPU in 2.19797s (16.3788GiB/s) | 20260505 08:20:38.618 | INFO | Downloaded 34GiB to GPU in 2.0479s (16.6024GiB/s) | 20260505 08:20:38.618 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 08:20:38.618 | INFO | Total mem usage: 15.3 GiB | 20260505 08:20:38.618 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 08:20:38.618 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -6.03%) | 20260505 08:20:38.618 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 08:20:38.618 | INFO | High Water Breakdown: | 20260505 08:20:38.618 | INFO | 60.7% : 8.78 GiB ( 93.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 08:20:38.618 | INFO | 39.1% : 5.66 GiB ( 85.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 08:20:38.618 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 08:20:38.618 | INFO | ------------------------------------------ | 20260505 08:20:38.618 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:20:38.618 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 08:20:38.618 | INFO | Total mem usage: 26.5 GiB | 20260505 08:20:38.618 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 08:20:38.618 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 08:20:38.618 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 08:20:38.618 | INFO | High Water Breakdown: | 20260505 08:20:38.618 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:20:38.618 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:20:38.618 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:20:38.618 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:20:38.618 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:20:38.618 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:20:38.618 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 08:20:38.618 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 08:20:38.618 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:20:38.618 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 08:20:38.618 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 08:20:38.618 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 08:20:38.618 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 08:20:38.618 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 08:20:38.618 | INFO | ------------------------------------------ | 20260505 08:20:38.618 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:20:38.618 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 08:20:38.618 | INFO | Total mem usage: 25.6 GiB | 20260505 08:20:38.618 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 08:20:38.618 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 08:20:38.618 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 08:20:38.618 | INFO | High Water Breakdown: | 20260505 08:20:38.618 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 08:20:38.618 | INFO | ------------------------------------------ | 20260505 08:20:38.618 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:20:38.618 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 08:20:38.618 | INFO | Total mem usage: 15.7 GiB | 20260505 08:20:38.618 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 08:20:38.618 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 08:20:38.618 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 08:20:38.618 | INFO | High Water Breakdown: | 20260505 08:20:38.618 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 08:20:38.618 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 08:20:38.618 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 08:20:38.618 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 08:20:38.618 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 08:20:38.618 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 08:20:38.618 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 08:20:38.619 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 08:20:38.619 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 08:20:38.619 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:20:38.619 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 08:20:38.619 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:20:38.619 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 08:20:38.619 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 08:20:38.619 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 08:20:38.619 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 08:20:38.619 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:20:38.619 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 08:20:38.619 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:20:38.619 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:20:38.619 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:20:38.619 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 08:20:38.619 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 08:20:38.619 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:20:38.619 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 08:20:38.619 | INFO | ------------------------------------------ | 20260505 08:20:38.619 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:20:38.619 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 08:20:38.619 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:20:38.619 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:20:38.619 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.55e+01, 1.36e+01 / 2.17e+00 | 20260505 08:20:38.619 | INFO | PerfReport: 0.28%, AggregateStats -- 9.28e-02 / 1.03e-01, 1.00e-02 / 3.87e-03 | 20260505 08:20:38.619 | INFO | PerfReport: 11.63%, Baseline -- 3.80e+00 / 4.46e+00, 6.28e-01 / 2.80e-01 | 20260505 08:20:38.619 | INFO | PerfReport: 3.31%, Histogram -- 1.08e+00 / 1.49e+00, 2.12e-01 / 1.13e-01 | 20260505 08:20:38.619 | INFO | PerfReport: 15.54%, DME -- 4.06e+01 / 4.45e+01, 4.16e+00 / 1.65e+00 | 20260505 08:20:38.619 | INFO | PerfReport: 56.53%, FrameLabeling -- 1.85e+01 / 1.98e+01, 1.96e+00 / 1.77e+00 | 20260505 08:20:38.619 | INFO | PerfReport: 7.42%, PulseAccumulating -- 2.43e+00 / 3.02e+00, 3.99e-01 / 2.21e-01 | 20260505 08:20:38.619 | INFO | PerfReport: 3.88%, Metrics -- 1.27e+00 / 1.60e+00, 3.58e-01 / 1.83e-01 | 20260505 08:20:38.619 | INFO | PerfReport: 1.39%, PulseSerializing -- 4.56e-01 / 5.10e-01, 8.65e-02 / 2.99e-02 | 20260505 08:20:38.619 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 08:20:38.619 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:20:38.619 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:20:38.619 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.32e+01 / 0.00e+00 | 20260505 08:20:38.619 | INFO | PerfReport: 22.58%, Upload -- 2.30e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260505 08:20:38.619 | INFO | PerfReport: 21.63%, Download -- 2.20e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260505 08:20:38.619 | INFO | PerfReport: 32.12%, Compute -- 3.27e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 08:20:38.619 | INFO | PerfReport: 1.67%, LockUpload -- 1.70e+00 / 0.00e+00, 7.31e+00 / 0.00e+00 | 20260505 08:20:38.619 | INFO | PerfReport: 1.00%, LockDownload -- 1.02e+00 / 0.00e+00, 2.48e+00 / 0.00e+00 | 20260505 08:20:38.619 | INFO | PerfReport: 20.99%, LockCompute -- 2.14e+01 / 0.00e+00, 1.78e+01 / 0.00e+00 | 20260505 08:20:38.619 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 2.01e-02 / 0.00e+00 | 20260505 08:20:38.758 | INFO | Starting a round of Baz IO | 20260505 08:20:42.050 | INFO | Populating RTMetrics report | 20260505 08:20:42.050 | INFO | RTMetrics report populated | 20260505 08:20:42.050 | INFO | Opening tmp RTMetrics json file | 20260505 08:20:42.060 | INFO | Serializing RTMetrics to json | 20260505 08:20:42.062 | INFO | Serialization done, swapping tmp file now... | 20260505 08:20:42.063 | INFO | RTMetrics done | 20260505 08:20:42.200 | INFO | Finished a round of Baz IO: Wrote 2.45 GiB in 3.442 seconds | 20260505 08:20:42.200 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 27.84% / 0.02% / 72.14% | 20260505 08:20:42.200 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.73658% / 3.57628e-05% / 7.54992e-05% | 20260505 08:20:42.200 | INFO | Baz IO overall: | 20260505 08:20:42.200 | INFO | BytesWritten : 972515625801 | 20260505 08:20:42.200 | INFO | NumEvents : 462207312774 | 20260505 08:20:43.757 | INFO | Analyzing chunk frames = [1728512,1729024) | 20260505 08:20:43.757 | INFO | MemInfo: { | 20260505 08:20:43.757 | INFO | "gpu" : " 17 GB", | 20260505 08:20:43.757 | INFO | "huge" : "", | 20260505 08:20:43.757 | INFO | "huge_shared" : " 27 GB", | 20260505 08:20:43.757 | INFO | "malloc" : " 14 GB", | 20260505 08:20:43.757 | INFO | "pinned" : " 29 GB" | 20260505 08:20:43.757 | INFO | } | 20260505 08:20:43.757 | INFO | Perf report for the last 5.13973 seconds: | 20260505 08:20:43.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:20:43.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:43.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:43.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:43.757 | INFO | ANALYSIS : 65.6%, 2.9, 100.1ms, 0 | 20260505 08:20:43.757 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:43.757 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 08:20:43.757 | INFO | PRE_HQ : 63.3%, 1.9, 61.4ms, 0 | 20260505 08:20:43.757 | INFO | BAZWRITER : 66.3%, 4.2, 141.5ms, 0 | 20260505 08:20:43.757 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 08:20:43.758 | INFO | Uploaded 36GiB to GPU in 2.24688s (16.0222GiB/s) | 20260505 08:20:43.758 | INFO | Downloaded 38GiB to GPU in 2.34966s (16.1726GiB/s) | 20260505 08:20:48.896 | INFO | Analyzing chunk frames = [1729024,1729536) | 20260505 08:20:48.896 | INFO | MemInfo: { | 20260505 08:20:48.896 | INFO | "gpu" : " 17 GB", | 20260505 08:20:48.896 | INFO | "huge" : "", | 20260505 08:20:48.896 | INFO | "huge_shared" : " 27 GB", | 20260505 08:20:48.896 | INFO | "malloc" : " 14 GB", | 20260505 08:20:48.896 | INFO | "pinned" : " 29 GB" | 20260505 08:20:48.896 | INFO | } | 20260505 08:20:48.897 | INFO | Perf report for the last 5.1397 seconds: | 20260505 08:20:48.897 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:20:48.897 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:48.897 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:48.897 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:48.897 | INFO | ANALYSIS : 64.7%, 3.0, 101.1ms, 0 | 20260505 08:20:48.897 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:48.897 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:48.897 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:20:48.897 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:48.897 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:48.897 | INFO | Uploaded 36GiB to GPU in 2.24846s (16.0109GiB/s) | 20260505 08:20:48.897 | INFO | Downloaded 34GiB to GPU in 2.10626s (16.1424GiB/s) | 20260505 08:20:54.036 | INFO | Analyzing chunk frames = [1729536,1730048) | 20260505 08:20:54.036 | INFO | MemInfo: { | 20260505 08:20:54.036 | INFO | "gpu" : " 17 GB", | 20260505 08:20:54.036 | INFO | "huge" : "", | 20260505 08:20:54.036 | INFO | "huge_shared" : " 27 GB", | 20260505 08:20:54.036 | INFO | "malloc" : " 14 GB", | 20260505 08:20:54.036 | INFO | "pinned" : " 29 GB" | 20260505 08:20:54.036 | INFO | } | 20260505 08:20:54.036 | INFO | Perf report for the last 5.13965 seconds: | 20260505 08:20:54.036 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:20:54.036 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:54.036 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:54.036 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:54.036 | INFO | ANALYSIS : 63.8%, 3.0, 99.2ms, 0 | 20260505 08:20:54.036 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:54.036 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:54.036 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:20:54.037 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:54.037 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:54.037 | INFO | Uploaded 36GiB to GPU in 2.17691s (16.5372GiB/s) | 20260505 08:20:54.037 | INFO | Downloaded 34GiB to GPU in 2.06532s (16.4624GiB/s) | 20260505 08:20:59.174 | INFO | Analyzing chunk frames = [1730048,1730560) | 20260505 08:20:59.174 | INFO | MemInfo: { | 20260505 08:20:59.174 | INFO | "gpu" : " 17 GB", | 20260505 08:20:59.174 | INFO | "huge" : "", | 20260505 08:20:59.174 | INFO | "huge_shared" : " 27 GB", | 20260505 08:20:59.174 | INFO | "malloc" : " 14 GB", | 20260505 08:20:59.175 | INFO | "pinned" : " 29 GB" | 20260505 08:20:59.175 | INFO | } | 20260505 08:20:59.175 | INFO | Perf report for the last 5.13837 seconds: | 20260505 08:20:59.175 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:20:59.175 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:59.175 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:59.175 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:59.175 | INFO | ANALYSIS : 63.6%, 3.0, 98.8ms, 0 | 20260505 08:20:59.175 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:59.175 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:59.175 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 08:20:59.175 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:59.175 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:20:59.175 | INFO | Uploaded 36GiB to GPU in 2.17653s (16.5401GiB/s) | 20260505 08:20:59.175 | INFO | Downloaded 34GiB to GPU in 2.06584s (16.4582GiB/s) | 20260505 08:21:04.315 | INFO | Analyzing chunk frames = [1730560,1731072) | 20260505 08:21:04.315 | INFO | MemInfo: { | 20260505 08:21:04.315 | INFO | "gpu" : " 17 GB", | 20260505 08:21:04.315 | INFO | "huge" : "", | 20260505 08:21:04.315 | INFO | "huge_shared" : " 27 GB", | 20260505 08:21:04.315 | INFO | "malloc" : " 14 GB", | 20260505 08:21:04.315 | INFO | "pinned" : " 29 GB" | 20260505 08:21:04.315 | INFO | } | 20260505 08:21:04.315 | INFO | Perf report for the last 5.13998 seconds: | 20260505 08:21:04.315 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:21:04.315 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:04.315 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:04.315 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:04.315 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 08:21:04.315 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:04.315 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:04.315 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 08:21:04.315 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:04.315 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:04.315 | INFO | Uploaded 36GiB to GPU in 2.18293s (16.4916GiB/s) | 20260505 08:21:04.315 | INFO | Downloaded 34GiB to GPU in 2.06022s (16.5031GiB/s) | 20260505 08:21:09.455 | INFO | Analyzing chunk frames = [1731072,1731584) | 20260505 08:21:09.455 | INFO | MemInfo: { | 20260505 08:21:09.455 | INFO | "gpu" : " 17 GB", | 20260505 08:21:09.455 | INFO | "huge" : "", | 20260505 08:21:09.455 | INFO | "huge_shared" : " 27 GB", | 20260505 08:21:09.455 | INFO | "malloc" : " 14 GB", | 20260505 08:21:09.455 | INFO | "pinned" : " 29 GB" | 20260505 08:21:09.455 | INFO | } | 20260505 08:21:09.455 | INFO | Perf report for the last 5.1405 seconds: | 20260505 08:21:09.455 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:21:09.455 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:09.455 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:09.455 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:09.455 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 08:21:09.455 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:09.455 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:09.455 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:21:09.455 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:09.455 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:09.456 | INFO | Uploaded 36GiB to GPU in 2.18103s (16.5059GiB/s) | 20260505 08:21:09.456 | INFO | Downloaded 34GiB to GPU in 2.05468s (16.5476GiB/s) | 20260505 08:21:14.594 | INFO | Analyzing chunk frames = [1731584,1732096) | 20260505 08:21:14.594 | INFO | MemInfo: { | 20260505 08:21:14.594 | INFO | "gpu" : " 17 GB", | 20260505 08:21:14.594 | INFO | "huge" : "", | 20260505 08:21:14.594 | INFO | "huge_shared" : " 27 GB", | 20260505 08:21:14.594 | INFO | "malloc" : " 14 GB", | 20260505 08:21:14.594 | INFO | "pinned" : " 29 GB" | 20260505 08:21:14.594 | INFO | } | 20260505 08:21:14.594 | INFO | Perf report for the last 5.13864 seconds: | 20260505 08:21:14.594 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:21:14.594 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:14.594 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:14.594 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:14.594 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 08:21:14.594 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:14.594 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:14.594 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 08:21:14.594 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:14.594 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:14.594 | INFO | Uploaded 36GiB to GPU in 2.19267s (16.4183GiB/s) | 20260505 08:21:14.594 | INFO | Downloaded 34GiB to GPU in 2.1524s (15.7963GiB/s) | 20260505 08:21:19.732 | INFO | Analyzing chunk frames = [1732096,1732608) | 20260505 08:21:19.732 | INFO | MemInfo: { | 20260505 08:21:19.732 | INFO | "gpu" : " 17 GB", | 20260505 08:21:19.732 | INFO | "huge" : "", | 20260505 08:21:19.732 | INFO | "huge_shared" : " 27 GB", | 20260505 08:21:19.732 | INFO | "malloc" : " 14 GB", | 20260505 08:21:19.732 | INFO | "pinned" : " 29 GB" | 20260505 08:21:19.732 | INFO | } | 20260505 08:21:19.732 | INFO | Perf report for the last 5.1384 seconds: | 20260505 08:21:19.732 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:21:19.732 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:19.732 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:19.732 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:19.732 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 08:21:19.732 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:19.732 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:19.732 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:21:19.732 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:19.733 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:19.733 | INFO | Uploaded 36GiB to GPU in 2.19128s (16.4288GiB/s) | 20260505 08:21:19.733 | INFO | Downloaded 34GiB to GPU in 2.04363s (16.637GiB/s) | 20260505 08:21:19.872 | INFO | Starting a round of Baz IO | 20260505 08:21:23.154 | INFO | Populating RTMetrics report | 20260505 08:21:23.154 | INFO | RTMetrics report populated | 20260505 08:21:23.154 | INFO | Opening tmp RTMetrics json file | 20260505 08:21:23.190 | INFO | Serializing RTMetrics to json | 20260505 08:21:23.216 | INFO | Serialization done, swapping tmp file now... | 20260505 08:21:23.216 | INFO | RTMetrics done | 20260505 08:21:23.283 | INFO | Finished a round of Baz IO: Wrote 2.451 GiB in 3.41 seconds | 20260505 08:21:23.283 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 27.82% / 0.02% / 72.16% | 20260505 08:21:23.283 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.72946% / 3.97364e-05% / 7.54992e-05% | 20260505 08:21:23.283 | INFO | Baz IO overall: | 20260505 08:21:23.283 | INFO | BytesWritten : 975147570200 | 20260505 08:21:23.283 | INFO | NumEvents : 463465863966 | 20260505 08:21:24.872 | INFO | Analyzing chunk frames = [1732608,1733120) | 20260505 08:21:24.872 | INFO | MemInfo: { | 20260505 08:21:24.872 | INFO | "gpu" : " 17 GB", | 20260505 08:21:24.872 | INFO | "huge" : "", | 20260505 08:21:24.872 | INFO | "huge_shared" : " 27 GB", | 20260505 08:21:24.872 | INFO | "malloc" : " 14 GB", | 20260505 08:21:24.872 | INFO | "pinned" : " 29 GB" | 20260505 08:21:24.872 | INFO | } | 20260505 08:21:24.872 | INFO | Perf report for the last 5.13968 seconds: | 20260505 08:21:24.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:21:24.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:24.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:24.872 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:24.872 | INFO | ANALYSIS : 65.4%, 2.9, 100.3ms, 0 | 20260505 08:21:24.872 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:24.872 | INFO | RT_METRICS : 47.1%, 1.0, 24.7ms, 0 | 20260505 08:21:24.872 | INFO | PRE_HQ : 63.5%, 1.9, 61.5ms, 0 | 20260505 08:21:24.872 | INFO | BAZWRITER : 65.6%, 4.1, 139.5ms, 0 | 20260505 08:21:24.872 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 08:21:24.872 | INFO | Uploaded 36GiB to GPU in 2.27685s (15.8113GiB/s) | 20260505 08:21:24.872 | INFO | Downloaded 38GiB to GPU in 2.37042s (16.0309GiB/s) | 20260505 08:21:30.012 | INFO | Analyzing chunk frames = [1733120,1733632) | 20260505 08:21:30.012 | INFO | MemInfo: { | 20260505 08:21:30.012 | INFO | "gpu" : " 17 GB", | 20260505 08:21:30.012 | INFO | "huge" : "", | 20260505 08:21:30.012 | INFO | "huge_shared" : " 27 GB", | 20260505 08:21:30.012 | INFO | "malloc" : " 14 GB", | 20260505 08:21:30.012 | INFO | "pinned" : " 29 GB" | 20260505 08:21:30.012 | INFO | } | 20260505 08:21:30.012 | INFO | Perf report for the last 5.13974 seconds: | 20260505 08:21:30.012 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:21:30.012 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:30.012 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:30.012 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:30.012 | INFO | ANALYSIS : 64.8%, 3.0, 101.1ms, 0 | 20260505 08:21:30.012 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:30.012 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:30.012 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:21:30.012 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:30.012 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:30.012 | INFO | Uploaded 36GiB to GPU in 2.24613s (16.0276GiB/s) | 20260505 08:21:30.012 | INFO | Downloaded 34GiB to GPU in 2.10526s (16.15GiB/s) | 20260505 08:21:35.151 | INFO | Analyzing chunk frames = [1733632,1734144) | 20260505 08:21:35.151 | INFO | MemInfo: { | 20260505 08:21:35.151 | INFO | "gpu" : " 17 GB", | 20260505 08:21:35.151 | INFO | "huge" : "", | 20260505 08:21:35.151 | INFO | "huge_shared" : " 27 GB", | 20260505 08:21:35.151 | INFO | "malloc" : " 14 GB", | 20260505 08:21:35.151 | INFO | "pinned" : " 29 GB" | 20260505 08:21:35.151 | INFO | } | 20260505 08:21:35.151 | INFO | Perf report for the last 5.13913 seconds: | 20260505 08:21:35.151 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:21:35.151 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:35.151 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:35.151 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:35.151 | INFO | ANALYSIS : 63.7%, 3.0, 99.1ms, 0 | 20260505 08:21:35.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:35.151 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:35.151 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:21:35.151 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:35.151 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:35.151 | INFO | Uploaded 36GiB to GPU in 2.17302s (16.5668GiB/s) | 20260505 08:21:35.151 | INFO | Downloaded 34GiB to GPU in 2.06447s (16.4691GiB/s) | 20260505 08:21:40.289 | INFO | Analyzing chunk frames = [1734144,1734656) | 20260505 08:21:40.289 | INFO | MemInfo: { | 20260505 08:21:40.289 | INFO | "gpu" : " 17 GB", | 20260505 08:21:40.289 | INFO | "huge" : "", | 20260505 08:21:40.289 | INFO | "huge_shared" : " 27 GB", | 20260505 08:21:40.289 | INFO | "malloc" : " 14 GB", | 20260505 08:21:40.289 | INFO | "pinned" : " 29 GB" | 20260505 08:21:40.289 | INFO | } | 20260505 08:21:40.289 | INFO | Perf report for the last 5.13855 seconds: | 20260505 08:21:40.289 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:21:40.289 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:40.290 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:40.290 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:40.290 | INFO | ANALYSIS : 63.6%, 3.0, 98.9ms, 0 | 20260505 08:21:40.290 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:40.290 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:40.290 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:21:40.290 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:40.290 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:40.290 | INFO | Uploaded 36GiB to GPU in 2.18406s (16.4831GiB/s) | 20260505 08:21:40.290 | INFO | Downloaded 34GiB to GPU in 2.06874s (16.4352GiB/s) | 20260505 08:21:45.430 | INFO | Analyzing chunk frames = [1734656,1735168) | 20260505 08:21:45.430 | INFO | MemInfo: { | 20260505 08:21:45.430 | INFO | "gpu" : " 17 GB", | 20260505 08:21:45.430 | INFO | "huge" : "", | 20260505 08:21:45.430 | INFO | "huge_shared" : " 27 GB", | 20260505 08:21:45.430 | INFO | "malloc" : " 14 GB", | 20260505 08:21:45.430 | INFO | "pinned" : " 29 GB" | 20260505 08:21:45.430 | INFO | } | 20260505 08:21:45.430 | INFO | Perf report for the last 5.14018 seconds: | 20260505 08:21:45.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:21:45.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:45.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:45.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:45.430 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 08:21:45.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:45.430 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:45.430 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 08:21:45.430 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:45.430 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:45.430 | INFO | Uploaded 36GiB to GPU in 2.17274s (16.569GiB/s) | 20260505 08:21:45.430 | INFO | Downloaded 34GiB to GPU in 2.05462s (16.5481GiB/s) | 20260505 08:21:50.570 | INFO | Analyzing chunk frames = [1735168,1735680) | 20260505 08:21:50.570 | INFO | MemInfo: { | 20260505 08:21:50.570 | INFO | "gpu" : " 17 GB", | 20260505 08:21:50.570 | INFO | "huge" : "", | 20260505 08:21:50.570 | INFO | "huge_shared" : " 27 GB", | 20260505 08:21:50.570 | INFO | "malloc" : " 14 GB", | 20260505 08:21:50.570 | INFO | "pinned" : " 29 GB" | 20260505 08:21:50.570 | INFO | } | 20260505 08:21:50.570 | INFO | Perf report for the last 5.14012 seconds: | 20260505 08:21:50.570 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:21:50.570 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:50.570 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:50.570 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:50.570 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 08:21:50.570 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:50.570 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:50.570 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 08:21:50.570 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:50.570 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:50.570 | INFO | Uploaded 36GiB to GPU in 2.18782s (16.4547GiB/s) | 20260505 08:21:50.570 | INFO | Downloaded 34GiB to GPU in 2.0576s (16.5241GiB/s) | 20260505 08:21:55.707 | INFO | Analyzing chunk frames = [1735680,1736192) | 20260505 08:21:55.707 | INFO | MemInfo: { | 20260505 08:21:55.707 | INFO | "gpu" : " 17 GB", | 20260505 08:21:55.707 | INFO | "huge" : "", | 20260505 08:21:55.707 | INFO | "huge_shared" : " 27 GB", | 20260505 08:21:55.707 | INFO | "malloc" : " 14 GB", | 20260505 08:21:55.707 | INFO | "pinned" : " 29 GB" | 20260505 08:21:55.707 | INFO | } | 20260505 08:21:55.708 | INFO | Perf report for the last 5.13769 seconds: | 20260505 08:21:55.708 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:21:55.708 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:55.708 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:55.708 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:55.708 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 08:21:55.708 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:55.708 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:55.708 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 08:21:55.708 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:55.708 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:21:55.708 | INFO | Uploaded 36GiB to GPU in 2.21747s (16.2348GiB/s) | 20260505 08:21:55.708 | INFO | Downloaded 34GiB to GPU in 2.16214s (15.7252GiB/s) | 20260505 08:22:00.846 | INFO | Analyzing chunk frames = [1736192,1736704) | 20260505 08:22:00.846 | INFO | MemInfo: { | 20260505 08:22:00.846 | INFO | "gpu" : " 17 GB", | 20260505 08:22:00.846 | INFO | "huge" : "", | 20260505 08:22:00.846 | INFO | "huge_shared" : " 27 GB", | 20260505 08:22:00.846 | INFO | "malloc" : " 14 GB", | 20260505 08:22:00.847 | INFO | "pinned" : " 29 GB" | 20260505 08:22:00.847 | INFO | } | 20260505 08:22:00.847 | INFO | Perf report for the last 5.13919 seconds: | 20260505 08:22:00.847 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:22:00.847 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:00.847 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:00.847 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:00.847 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 08:22:00.847 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:00.847 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:00.847 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 08:22:00.847 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:00.847 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:00.847 | INFO | Uploaded 36GiB to GPU in 2.19303s (16.4157GiB/s) | 20260505 08:22:00.847 | INFO | Downloaded 34GiB to GPU in 2.03651s (16.6952GiB/s) | 20260505 08:22:00.847 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 08:22:00.847 | INFO | Total mem usage: 15.3 GiB | 20260505 08:22:00.847 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 08:22:00.847 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.26%) | 20260505 08:22:00.847 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 08:22:00.847 | INFO | High Water Breakdown: | 20260505 08:22:00.847 | INFO | 60.8% : 8.77 GiB ( 93.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 08:22:00.847 | INFO | 39% : 5.63 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 08:22:00.847 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 08:22:00.847 | INFO | ------------------------------------------ | 20260505 08:22:00.847 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:22:00.847 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 08:22:00.847 | INFO | Total mem usage: 26.5 GiB | 20260505 08:22:00.847 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 08:22:00.847 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 08:22:00.847 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 08:22:00.847 | INFO | High Water Breakdown: | 20260505 08:22:00.847 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:22:00.847 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:22:00.847 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:22:00.847 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:22:00.848 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:22:00.848 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:22:00.848 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 08:22:00.848 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 08:22:00.848 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:22:00.848 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 08:22:00.848 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 08:22:00.848 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 08:22:00.848 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 08:22:00.848 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 08:22:00.848 | INFO | ------------------------------------------ | 20260505 08:22:00.848 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:22:00.848 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 08:22:00.848 | INFO | Total mem usage: 25.6 GiB | 20260505 08:22:00.848 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 08:22:00.848 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 08:22:00.848 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 08:22:00.848 | INFO | High Water Breakdown: | 20260505 08:22:00.848 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 08:22:00.848 | INFO | ------------------------------------------ | 20260505 08:22:00.848 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:22:00.848 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 08:22:00.848 | INFO | Total mem usage: 15.7 GiB | 20260505 08:22:00.848 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 08:22:00.848 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 08:22:00.848 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 08:22:00.848 | INFO | High Water Breakdown: | 20260505 08:22:00.848 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 08:22:00.848 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 08:22:00.848 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 08:22:00.848 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 08:22:00.848 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 08:22:00.848 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 08:22:00.848 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 08:22:00.848 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 08:22:00.848 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 08:22:00.848 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:22:00.848 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 08:22:00.848 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:22:00.848 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 08:22:00.848 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 08:22:00.848 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 08:22:00.848 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 08:22:00.848 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:22:00.848 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 08:22:00.848 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:22:00.848 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:22:00.848 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:22:00.848 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 08:22:00.848 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 08:22:00.848 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:22:00.848 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 08:22:00.848 | INFO | ------------------------------------------ | 20260505 08:22:00.848 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:22:00.848 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 08:22:00.848 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:22:00.848 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:22:00.848 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.54e+01, 1.38e+01 / 2.31e+00 | 20260505 08:22:00.848 | INFO | PerfReport: 0.29%, AggregateStats -- 9.33e-02 / 1.03e-01, 1.03e-02 / 4.74e-03 | 20260505 08:22:00.848 | INFO | PerfReport: 11.67%, Baseline -- 3.81e+00 / 4.49e+00, 6.32e-01 / 2.68e-01 | 20260505 08:22:00.848 | INFO | PerfReport: 3.33%, Histogram -- 1.09e+00 / 1.48e+00, 2.14e-01 / 1.20e-01 | 20260505 08:22:00.848 | INFO | PerfReport: 15.53%, DME -- 4.06e+01 / 4.43e+01, 4.09e+00 / 1.73e+00 | 20260505 08:22:00.848 | INFO | PerfReport: 56.43%, FrameLabeling -- 1.84e+01 / 1.99e+01, 1.94e+00 / 1.71e+00 | 20260505 08:22:00.848 | INFO | PerfReport: 7.47%, PulseAccumulating -- 2.44e+00 / 3.02e+00, 4.10e-01 / 2.28e-01 | 20260505 08:22:00.848 | INFO | PerfReport: 3.89%, Metrics -- 1.27e+00 / 1.59e+00, 3.57e-01 / 2.04e-01 | 20260505 08:22:00.848 | INFO | PerfReport: 1.40%, PulseSerializing -- 4.57e-01 / 5.08e-01, 8.69e-02 / 3.02e-02 | 20260505 08:22:00.848 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 08:22:00.848 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:22:00.848 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:22:00.848 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.32e+01 / 0.00e+00 | 20260505 08:22:00.848 | INFO | PerfReport: 22.55%, Upload -- 2.29e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260505 08:22:00.848 | INFO | PerfReport: 21.63%, Download -- 2.20e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260505 08:22:00.848 | INFO | PerfReport: 32.10%, Compute -- 3.27e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 08:22:00.848 | INFO | PerfReport: 1.64%, LockUpload -- 1.67e+00 / 0.00e+00, 7.34e+00 / 0.00e+00 | 20260505 08:22:00.848 | INFO | PerfReport: 1.01%, LockDownload -- 1.03e+00 / 0.00e+00, 2.43e+00 / 0.00e+00 | 20260505 08:22:00.848 | INFO | PerfReport: 21.05%, LockCompute -- 2.14e+01 / 0.00e+00, 1.76e+01 / 0.00e+00 | 20260505 08:22:00.849 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.70e-02 / 0.00e+00 | 20260505 08:22:00.989 | INFO | Starting a round of Baz IO | 20260505 08:22:04.280 | INFO | Populating RTMetrics report | 20260505 08:22:04.280 | INFO | RTMetrics report populated | 20260505 08:22:04.280 | INFO | Opening tmp RTMetrics json file | 20260505 08:22:04.294 | INFO | Serializing RTMetrics to json | 20260505 08:22:04.344 | INFO | Serialization done, swapping tmp file now... | 20260505 08:22:04.367 | INFO | RTMetrics done | 20260505 08:22:04.410 | INFO | Finished a round of Baz IO: Wrote 2.448 GiB in 3.42 seconds | 20260505 08:22:04.410 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 27.80% / 0.02% / 72.18% | 20260505 08:22:04.410 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.71187% / 4.37101e-05% / 8.74201e-05% | 20260505 08:22:04.410 | INFO | Baz IO overall: | 20260505 08:22:04.410 | INFO | BytesWritten : 977775705968 | 20260505 08:22:04.410 | INFO | NumEvents : 464721899646 | 20260505 08:22:05.989 | INFO | Analyzing chunk frames = [1736704,1737216) | 20260505 08:22:05.989 | INFO | MemInfo: { | 20260505 08:22:05.989 | INFO | "gpu" : " 17 GB", | 20260505 08:22:05.989 | INFO | "huge" : "", | 20260505 08:22:05.989 | INFO | "huge_shared" : " 27 GB", | 20260505 08:22:05.989 | INFO | "malloc" : " 14 GB", | 20260505 08:22:05.989 | INFO | "pinned" : " 29 GB" | 20260505 08:22:05.989 | INFO | } | 20260505 08:22:05.989 | INFO | Perf report for the last 5.14205 seconds: | 20260505 08:22:05.989 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:22:05.989 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:05.989 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:05.989 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:05.989 | INFO | ANALYSIS : 65.6%, 2.9, 100.5ms, 0 | 20260505 08:22:05.989 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:05.989 | INFO | RT_METRICS : 47.4%, 1.0, 24.9ms, 0 | 20260505 08:22:05.989 | INFO | PRE_HQ : 63.3%, 1.9, 61.2ms, 0 | 20260505 08:22:05.989 | INFO | BAZWRITER : 65.8%, 4.1, 140.0ms, 0 | 20260505 08:22:05.989 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260505 08:22:05.989 | INFO | Uploaded 36GiB to GPU in 2.2538s (15.973GiB/s) | 20260505 08:22:05.989 | INFO | Downloaded 38GiB to GPU in 2.35237s (16.1539GiB/s) | 20260505 08:22:11.126 | INFO | Analyzing chunk frames = [1737216,1737728) | 20260505 08:22:11.126 | INFO | MemInfo: { | 20260505 08:22:11.126 | INFO | "gpu" : " 17 GB", | 20260505 08:22:11.126 | INFO | "huge" : "", | 20260505 08:22:11.126 | INFO | "huge_shared" : " 27 GB", | 20260505 08:22:11.126 | INFO | "malloc" : " 14 GB", | 20260505 08:22:11.126 | INFO | "pinned" : " 29 GB" | 20260505 08:22:11.126 | INFO | } | 20260505 08:22:11.126 | INFO | Perf report for the last 5.13741 seconds: | 20260505 08:22:11.126 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:22:11.126 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:11.126 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:11.126 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:11.126 | INFO | ANALYSIS : 64.8%, 3.0, 101.1ms, 0 | 20260505 08:22:11.126 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:11.126 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:11.126 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:22:11.126 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:11.126 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:11.127 | INFO | Uploaded 36GiB to GPU in 2.24772s (16.0162GiB/s) | 20260505 08:22:11.127 | INFO | Downloaded 34GiB to GPU in 2.10851s (16.1251GiB/s) | 20260505 08:22:16.265 | INFO | Analyzing chunk frames = [1737728,1738240) | 20260505 08:22:16.265 | INFO | MemInfo: { | 20260505 08:22:16.265 | INFO | "gpu" : " 17 GB", | 20260505 08:22:16.265 | INFO | "huge" : "", | 20260505 08:22:16.265 | INFO | "huge_shared" : " 27 GB", | 20260505 08:22:16.265 | INFO | "malloc" : " 14 GB", | 20260505 08:22:16.265 | INFO | "pinned" : " 29 GB" | 20260505 08:22:16.266 | INFO | } | 20260505 08:22:16.266 | INFO | Perf report for the last 5.13938 seconds: | 20260505 08:22:16.266 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:22:16.266 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:16.266 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:16.266 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:16.266 | INFO | ANALYSIS : 63.6%, 3.0, 98.9ms, 0 | 20260505 08:22:16.266 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:16.266 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:16.266 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 08:22:16.266 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:16.266 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:16.266 | INFO | Uploaded 36GiB to GPU in 2.16794s (16.6056GiB/s) | 20260505 08:22:16.266 | INFO | Downloaded 34GiB to GPU in 2.06766s (16.4437GiB/s) | 20260505 08:22:21.406 | INFO | Analyzing chunk frames = [1738240,1738752) | 20260505 08:22:21.406 | INFO | MemInfo: { | 20260505 08:22:21.406 | INFO | "gpu" : " 17 GB", | 20260505 08:22:21.406 | INFO | "huge" : "", | 20260505 08:22:21.406 | INFO | "huge_shared" : " 27 GB", | 20260505 08:22:21.406 | INFO | "malloc" : " 14 GB", | 20260505 08:22:21.406 | INFO | "pinned" : " 29 GB" | 20260505 08:22:21.406 | INFO | } | 20260505 08:22:21.406 | INFO | Perf report for the last 5.14023 seconds: | 20260505 08:22:21.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:22:21.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:21.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:21.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:21.406 | INFO | ANALYSIS : 63.5%, 3.0, 98.6ms, 0 | 20260505 08:22:21.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:21.406 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:21.406 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:22:21.406 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:21.406 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:21.406 | INFO | Uploaded 36GiB to GPU in 2.17603s (16.5439GiB/s) | 20260505 08:22:21.406 | INFO | Downloaded 34GiB to GPU in 2.06482s (16.4664GiB/s) | 20260505 08:22:26.544 | INFO | Analyzing chunk frames = [1738752,1739264) | 20260505 08:22:26.544 | INFO | MemInfo: { | 20260505 08:22:26.545 | INFO | "gpu" : " 17 GB", | 20260505 08:22:26.545 | INFO | "huge" : "", | 20260505 08:22:26.545 | INFO | "huge_shared" : " 27 GB", | 20260505 08:22:26.545 | INFO | "malloc" : " 14 GB", | 20260505 08:22:26.545 | INFO | "pinned" : " 29 GB" | 20260505 08:22:26.545 | INFO | } | 20260505 08:22:26.545 | INFO | Perf report for the last 5.13896 seconds: | 20260505 08:22:26.545 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:22:26.545 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:26.545 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:26.545 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:26.545 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 08:22:26.545 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:26.545 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:26.545 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:22:26.545 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:26.545 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:26.545 | INFO | Uploaded 36GiB to GPU in 2.16342s (16.6403GiB/s) | 20260505 08:22:26.545 | INFO | Downloaded 34GiB to GPU in 2.05506s (16.5445GiB/s) | 20260505 08:22:31.683 | INFO | Analyzing chunk frames = [1739264,1739776) | 20260505 08:22:31.683 | INFO | MemInfo: { | 20260505 08:22:31.683 | INFO | "gpu" : " 17 GB", | 20260505 08:22:31.683 | INFO | "huge" : "", | 20260505 08:22:31.683 | INFO | "huge_shared" : " 27 GB", | 20260505 08:22:31.683 | INFO | "malloc" : " 14 GB", | 20260505 08:22:31.683 | INFO | "pinned" : " 29 GB" | 20260505 08:22:31.683 | INFO | } | 20260505 08:22:31.683 | INFO | Perf report for the last 5.13859 seconds: | 20260505 08:22:31.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:22:31.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:31.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:31.683 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:31.683 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 08:22:31.683 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:31.684 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:31.684 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:22:31.684 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:31.684 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:31.684 | INFO | Uploaded 36GiB to GPU in 2.18331s (16.4887GiB/s) | 20260505 08:22:31.684 | INFO | Downloaded 34GiB to GPU in 2.05152s (16.573GiB/s) | 20260505 08:22:36.822 | INFO | Analyzing chunk frames = [1739776,1740288) | 20260505 08:22:36.822 | INFO | MemInfo: { | 20260505 08:22:36.822 | INFO | "gpu" : " 17 GB", | 20260505 08:22:36.822 | INFO | "huge" : "", | 20260505 08:22:36.822 | INFO | "huge_shared" : " 27 GB", | 20260505 08:22:36.822 | INFO | "malloc" : " 14 GB", | 20260505 08:22:36.822 | INFO | "pinned" : " 29 GB" | 20260505 08:22:36.822 | INFO | } | 20260505 08:22:36.822 | INFO | Perf report for the last 5.13894 seconds: | 20260505 08:22:36.822 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:22:36.822 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:36.822 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:36.822 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:36.822 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 08:22:36.822 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:36.822 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:36.822 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 08:22:36.823 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:36.823 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:36.823 | INFO | Uploaded 36GiB to GPU in 2.20106s (16.3558GiB/s) | 20260505 08:22:36.823 | INFO | Downloaded 34GiB to GPU in 2.16042s (15.7377GiB/s) | 20260505 08:22:41.963 | INFO | Analyzing chunk frames = [1740288,1740800) | 20260505 08:22:41.963 | INFO | MemInfo: { | 20260505 08:22:41.963 | INFO | "gpu" : " 17 GB", | 20260505 08:22:41.963 | INFO | "huge" : "", | 20260505 08:22:41.963 | INFO | "huge_shared" : " 27 GB", | 20260505 08:22:41.963 | INFO | "malloc" : " 14 GB", | 20260505 08:22:41.963 | INFO | "pinned" : " 29 GB" | 20260505 08:22:41.964 | INFO | } | 20260505 08:22:41.964 | INFO | Perf report for the last 5.14142 seconds: | 20260505 08:22:41.964 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:22:41.964 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:41.964 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:41.964 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:41.964 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 08:22:41.964 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:41.964 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:41.964 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 08:22:41.964 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:41.964 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:41.964 | INFO | Uploaded 36GiB to GPU in 2.19618s (16.3921GiB/s) | 20260505 08:22:41.964 | INFO | Downloaded 34GiB to GPU in 2.03859s (16.6782GiB/s) | 20260505 08:22:42.104 | INFO | Starting a round of Baz IO | 20260505 08:22:45.388 | INFO | Populating RTMetrics report | 20260505 08:22:45.388 | INFO | RTMetrics report populated | 20260505 08:22:45.388 | INFO | Opening tmp RTMetrics json file | 20260505 08:22:45.400 | INFO | Serializing RTMetrics to json | 20260505 08:22:45.424 | INFO | Serialization done, swapping tmp file now... | 20260505 08:22:45.426 | INFO | RTMetrics done | 20260505 08:22:45.533 | INFO | Finished a round of Baz IO: Wrote 2.449 GiB in 3.429 seconds | 20260505 08:22:45.533 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 27.78% / 0.02% / 72.20% | 20260505 08:22:45.533 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.70727% / 3.17891e-05% / 7.54992e-05% | 20260505 08:22:45.533 | INFO | Baz IO overall: | 20260505 08:22:45.533 | INFO | BytesWritten : 980404786702 | 20260505 08:22:45.533 | INFO | NumEvents : 465979051560 | 20260505 08:22:47.103 | INFO | Analyzing chunk frames = [1740800,1741312) | 20260505 08:22:47.103 | INFO | MemInfo: { | 20260505 08:22:47.103 | INFO | "gpu" : " 17 GB", | 20260505 08:22:47.103 | INFO | "huge" : "", | 20260505 08:22:47.103 | INFO | "huge_shared" : " 27 GB", | 20260505 08:22:47.103 | INFO | "malloc" : " 14 GB", | 20260505 08:22:47.103 | INFO | "pinned" : " 29 GB" | 20260505 08:22:47.103 | INFO | } | 20260505 08:22:47.103 | INFO | Perf report for the last 5.13898 seconds: | 20260505 08:22:47.103 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:22:47.103 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:47.103 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:47.103 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:47.103 | INFO | ANALYSIS : 65.5%, 2.9, 100.1ms, 0 | 20260505 08:22:47.103 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:47.103 | INFO | RT_METRICS : 46.5%, 1.0, 24.4ms, 0 | 20260505 08:22:47.103 | INFO | PRE_HQ : 63.6%, 1.9, 61.5ms, 0 | 20260505 08:22:47.103 | INFO | BAZWRITER : 66.0%, 4.1, 140.1ms, 0 | 20260505 08:22:47.103 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 08:22:47.103 | INFO | Uploaded 36GiB to GPU in 2.27749s (15.8069GiB/s) | 20260505 08:22:47.103 | INFO | Downloaded 38GiB to GPU in 2.37068s (16.0291GiB/s) | 20260505 08:22:52.241 | INFO | Analyzing chunk frames = [1741312,1741824) | 20260505 08:22:52.241 | INFO | MemInfo: { | 20260505 08:22:52.241 | INFO | "gpu" : " 17 GB", | 20260505 08:22:52.241 | INFO | "huge" : "", | 20260505 08:22:52.241 | INFO | "huge_shared" : " 27 GB", | 20260505 08:22:52.241 | INFO | "malloc" : " 14 GB", | 20260505 08:22:52.241 | INFO | "pinned" : " 29 GB" | 20260505 08:22:52.241 | INFO | } | 20260505 08:22:52.241 | INFO | Perf report for the last 5.13849 seconds: | 20260505 08:22:52.241 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:22:52.241 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:52.241 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:52.241 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:52.241 | INFO | ANALYSIS : 64.7%, 3.0, 101.1ms, 0 | 20260505 08:22:52.241 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:52.241 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:52.241 | INFO | PRE_HQ : 29.8%, 1.0, 15.4ms, 0 | 20260505 08:22:52.241 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:52.241 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:52.242 | INFO | Uploaded 36GiB to GPU in 2.25127s (15.991GiB/s) | 20260505 08:22:52.242 | INFO | Downloaded 34GiB to GPU in 2.08882s (16.2771GiB/s) | 20260505 08:22:57.380 | INFO | Analyzing chunk frames = [1741824,1742336) | 20260505 08:22:57.380 | INFO | MemInfo: { | 20260505 08:22:57.380 | INFO | "gpu" : " 17 GB", | 20260505 08:22:57.380 | INFO | "huge" : "", | 20260505 08:22:57.380 | INFO | "huge_shared" : " 27 GB", | 20260505 08:22:57.380 | INFO | "malloc" : " 14 GB", | 20260505 08:22:57.380 | INFO | "pinned" : " 29 GB" | 20260505 08:22:57.380 | INFO | } | 20260505 08:22:57.380 | INFO | Perf report for the last 5.13851 seconds: | 20260505 08:22:57.380 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:22:57.380 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:57.380 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:57.380 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:57.380 | INFO | ANALYSIS : 63.6%, 3.0, 99.0ms, 0 | 20260505 08:22:57.380 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:57.380 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:57.380 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 08:22:57.380 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:57.380 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:22:57.380 | INFO | Uploaded 36GiB to GPU in 2.1693s (16.5952GiB/s) | 20260505 08:22:57.380 | INFO | Downloaded 34GiB to GPU in 2.06984s (16.4264GiB/s) | 20260505 08:23:02.519 | INFO | Analyzing chunk frames = [1742336,1742848) | 20260505 08:23:02.519 | INFO | MemInfo: { | 20260505 08:23:02.519 | INFO | "gpu" : " 17 GB", | 20260505 08:23:02.519 | INFO | "huge" : "", | 20260505 08:23:02.519 | INFO | "huge_shared" : " 27 GB", | 20260505 08:23:02.519 | INFO | "malloc" : " 14 GB", | 20260505 08:23:02.519 | INFO | "pinned" : " 29 GB" | 20260505 08:23:02.519 | INFO | } | 20260505 08:23:02.519 | INFO | Perf report for the last 5.13928 seconds: | 20260505 08:23:02.519 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:23:02.519 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:02.519 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:02.519 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:02.519 | INFO | ANALYSIS : 63.7%, 3.0, 98.9ms, 0 | 20260505 08:23:02.519 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:02.519 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:02.519 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 08:23:02.519 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:02.519 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:02.519 | INFO | Uploaded 36GiB to GPU in 2.18831s (16.4511GiB/s) | 20260505 08:23:02.519 | INFO | Downloaded 34GiB to GPU in 2.06893s (16.4336GiB/s) | 20260505 08:23:07.658 | INFO | Analyzing chunk frames = [1742848,1743360) | 20260505 08:23:07.658 | INFO | MemInfo: { | 20260505 08:23:07.658 | INFO | "gpu" : " 17 GB", | 20260505 08:23:07.658 | INFO | "huge" : "", | 20260505 08:23:07.658 | INFO | "huge_shared" : " 27 GB", | 20260505 08:23:07.658 | INFO | "malloc" : " 14 GB", | 20260505 08:23:07.658 | INFO | "pinned" : " 29 GB" | 20260505 08:23:07.658 | INFO | } | 20260505 08:23:07.658 | INFO | Perf report for the last 5.13926 seconds: | 20260505 08:23:07.659 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:23:07.659 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:07.659 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:07.659 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:07.659 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 08:23:07.659 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:07.659 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:07.659 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:23:07.659 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:07.659 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:07.659 | INFO | Uploaded 36GiB to GPU in 2.18347s (16.4875GiB/s) | 20260505 08:23:07.659 | INFO | Downloaded 34GiB to GPU in 2.05858s (16.5162GiB/s) | 20260505 08:23:12.799 | INFO | Analyzing chunk frames = [1743360,1743872) | 20260505 08:23:12.799 | INFO | MemInfo: { | 20260505 08:23:12.799 | INFO | "gpu" : " 17 GB", | 20260505 08:23:12.799 | INFO | "huge" : "", | 20260505 08:23:12.799 | INFO | "huge_shared" : " 27 GB", | 20260505 08:23:12.799 | INFO | "malloc" : " 14 GB", | 20260505 08:23:12.799 | INFO | "pinned" : " 29 GB" | 20260505 08:23:12.799 | INFO | } | 20260505 08:23:12.799 | INFO | Perf report for the last 5.14069 seconds: | 20260505 08:23:12.799 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:23:12.799 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:12.799 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:12.799 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:12.799 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 08:23:12.799 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:12.799 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:12.799 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:23:12.799 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:12.799 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:12.799 | INFO | Uploaded 36GiB to GPU in 2.16709s (16.6121GiB/s) | 20260505 08:23:12.799 | INFO | Downloaded 34GiB to GPU in 2.04606s (16.6173GiB/s) | 20260505 08:23:17.938 | INFO | Analyzing chunk frames = [1743872,1744384) | 20260505 08:23:17.938 | INFO | MemInfo: { | 20260505 08:23:17.938 | INFO | "gpu" : " 17 GB", | 20260505 08:23:17.938 | INFO | "huge" : "", | 20260505 08:23:17.938 | INFO | "huge_shared" : " 27 GB", | 20260505 08:23:17.938 | INFO | "malloc" : " 14 GB", | 20260505 08:23:17.939 | INFO | "pinned" : " 29 GB" | 20260505 08:23:17.939 | INFO | } | 20260505 08:23:17.939 | INFO | Perf report for the last 5.13965 seconds: | 20260505 08:23:17.939 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:23:17.939 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:17.939 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:17.939 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:17.939 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 08:23:17.939 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:17.939 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:17.939 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260505 08:23:17.939 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:17.939 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:17.939 | INFO | Uploaded 36GiB to GPU in 2.18741s (16.4578GiB/s) | 20260505 08:23:17.939 | INFO | Downloaded 34GiB to GPU in 2.15125s (15.8048GiB/s) | 20260505 08:23:23.078 | INFO | Analyzing chunk frames = [1744384,1744896) | 20260505 08:23:23.078 | INFO | MemInfo: { | 20260505 08:23:23.078 | INFO | "gpu" : " 17 GB", | 20260505 08:23:23.078 | INFO | "huge" : "", | 20260505 08:23:23.078 | INFO | "huge_shared" : " 27 GB", | 20260505 08:23:23.078 | INFO | "malloc" : " 14 GB", | 20260505 08:23:23.078 | INFO | "pinned" : " 29 GB" | 20260505 08:23:23.078 | INFO | } | 20260505 08:23:23.078 | INFO | Perf report for the last 5.13944 seconds: | 20260505 08:23:23.078 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:23:23.078 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:23.078 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:23.078 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:23.078 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 08:23:23.078 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:23.078 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:23.078 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 08:23:23.078 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:23.078 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:23.078 | INFO | Uploaded 36GiB to GPU in 2.19754s (16.3819GiB/s) | 20260505 08:23:23.079 | INFO | Downloaded 34GiB to GPU in 2.04342s (16.6388GiB/s) | 20260505 08:23:23.079 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 08:23:23.079 | INFO | Total mem usage: 15.3 GiB | 20260505 08:23:23.079 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 08:23:23.079 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.29%) | 20260505 08:23:23.079 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 08:23:23.079 | INFO | High Water Breakdown: | 20260505 08:23:23.079 | INFO | 60.8% : 8.77 GiB ( 93.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 08:23:23.079 | INFO | 39% : 5.63 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 08:23:23.079 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 08:23:23.079 | INFO | ------------------------------------------ | 20260505 08:23:23.079 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:23:23.079 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 08:23:23.079 | INFO | Total mem usage: 26.5 GiB | 20260505 08:23:23.079 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 08:23:23.079 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 08:23:23.079 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 08:23:23.079 | INFO | High Water Breakdown: | 20260505 08:23:23.079 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:23:23.079 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:23:23.079 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:23:23.079 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:23:23.079 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:23:23.079 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:23:23.079 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 08:23:23.079 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 08:23:23.079 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:23:23.079 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 08:23:23.079 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 08:23:23.079 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 08:23:23.079 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 08:23:23.079 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 08:23:23.079 | INFO | ------------------------------------------ | 20260505 08:23:23.079 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:23:23.079 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 08:23:23.079 | INFO | Total mem usage: 25.6 GiB | 20260505 08:23:23.079 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 08:23:23.079 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 08:23:23.080 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 08:23:23.080 | INFO | High Water Breakdown: | 20260505 08:23:23.080 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 08:23:23.080 | INFO | ------------------------------------------ | 20260505 08:23:23.080 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:23:23.080 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 08:23:23.080 | INFO | Total mem usage: 15.7 GiB | 20260505 08:23:23.080 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 08:23:23.080 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 08:23:23.080 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 08:23:23.080 | INFO | High Water Breakdown: | 20260505 08:23:23.080 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 08:23:23.080 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 08:23:23.080 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 08:23:23.080 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 08:23:23.080 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 08:23:23.080 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 08:23:23.080 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 08:23:23.080 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 08:23:23.080 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 08:23:23.080 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:23:23.080 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 08:23:23.080 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:23:23.080 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 08:23:23.080 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 08:23:23.080 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 08:23:23.080 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 08:23:23.080 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:23:23.080 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 08:23:23.080 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:23:23.080 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:23:23.080 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:23:23.080 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 08:23:23.080 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 08:23:23.080 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:23:23.080 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 08:23:23.080 | INFO | ------------------------------------------ | 20260505 08:23:23.080 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:23:23.080 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 08:23:23.080 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:23:23.080 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:23:23.080 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.54e+01, 1.36e+01 / 2.31e+00 | 20260505 08:23:23.080 | INFO | PerfReport: 0.29%, AggregateStats -- 9.32e-02 / 1.03e-01, 1.07e-02 / 4.25e-03 | 20260505 08:23:23.080 | INFO | PerfReport: 11.65%, Baseline -- 3.80e+00 / 4.48e+00, 6.23e-01 / 2.55e-01 | 20260505 08:23:23.080 | INFO | PerfReport: 3.33%, Histogram -- 1.09e+00 / 1.49e+00, 2.11e-01 / 1.12e-01 | 20260505 08:23:23.080 | INFO | PerfReport: 15.54%, DME -- 4.06e+01 / 4.44e+01, 4.18e+00 / 1.73e+00 | 20260505 08:23:23.080 | INFO | PerfReport: 56.47%, FrameLabeling -- 1.84e+01 / 1.98e+01, 1.93e+00 / 1.94e+00 | 20260505 08:23:23.080 | INFO | PerfReport: 7.46%, PulseAccumulating -- 2.43e+00 / 3.02e+00, 4.07e-01 / 2.49e-01 | 20260505 08:23:23.080 | INFO | PerfReport: 3.87%, Metrics -- 1.26e+00 / 1.57e+00, 3.50e-01 / 1.83e-01 | 20260505 08:23:23.080 | INFO | PerfReport: 1.40%, PulseSerializing -- 4.56e-01 / 5.09e-01, 8.68e-02 / 2.78e-02 | 20260505 08:23:23.080 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 08:23:23.080 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:23:23.080 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:23:23.080 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.31e+01 / 0.00e+00 | 20260505 08:23:23.080 | INFO | PerfReport: 22.54%, Upload -- 2.29e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260505 08:23:23.080 | INFO | PerfReport: 21.63%, Download -- 2.20e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260505 08:23:23.080 | INFO | PerfReport: 32.09%, Compute -- 3.26e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 08:23:23.080 | INFO | PerfReport: 1.67%, LockUpload -- 1.70e+00 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260505 08:23:23.080 | INFO | PerfReport: 1.03%, LockDownload -- 1.05e+00 / 0.00e+00, 2.48e+00 / 0.00e+00 | 20260505 08:23:23.080 | INFO | PerfReport: 21.02%, LockCompute -- 2.14e+01 / 0.00e+00, 1.77e+01 / 0.00e+00 | 20260505 08:23:23.080 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.72e-02 / 0.00e+00 | 20260505 08:23:23.219 | INFO | Starting a round of Baz IO | 20260505 08:23:26.507 | INFO | Populating RTMetrics report | 20260505 08:23:26.507 | INFO | RTMetrics report populated | 20260505 08:23:26.507 | INFO | Opening tmp RTMetrics json file | 20260505 08:23:26.511 | INFO | Serializing RTMetrics to json | 20260505 08:23:26.559 | INFO | Serialization done, swapping tmp file now... | 20260505 08:23:26.611 | INFO | RTMetrics done | 20260505 08:23:26.680 | INFO | Finished a round of Baz IO: Wrote 2.449 GiB in 3.461 seconds | 20260505 08:23:26.680 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 27.76% / 0.02% / 72.22% | 20260505 08:23:26.680 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.69825% / 2.78155e-05% / 3.97364e-05% | 20260505 08:23:26.680 | INFO | Baz IO overall: | 20260505 08:23:26.680 | INFO | BytesWritten : 983033890049 | 20260505 08:23:26.680 | INFO | NumEvents : 467236558977 | 20260505 08:23:28.216 | INFO | Analyzing chunk frames = [1744896,1745408) | 20260505 08:23:28.216 | INFO | MemInfo: { | 20260505 08:23:28.216 | INFO | "gpu" : " 17 GB", | 20260505 08:23:28.216 | INFO | "huge" : "", | 20260505 08:23:28.216 | INFO | "huge_shared" : " 27 GB", | 20260505 08:23:28.216 | INFO | "malloc" : " 14 GB", | 20260505 08:23:28.216 | INFO | "pinned" : " 29 GB" | 20260505 08:23:28.216 | INFO | } | 20260505 08:23:28.216 | INFO | Perf report for the last 5.13767 seconds: | 20260505 08:23:28.216 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:23:28.216 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:28.216 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:28.216 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:28.216 | INFO | ANALYSIS : 65.6%, 2.9, 100.4ms, 0 | 20260505 08:23:28.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:28.216 | INFO | RT_METRICS : 47.7%, 1.0, 25.0ms, 0 | 20260505 08:23:28.216 | INFO | PRE_HQ : 63.4%, 1.9, 61.4ms, 0 | 20260505 08:23:28.216 | INFO | BAZWRITER : 66.7%, 4.2, 142.7ms, 0 | 20260505 08:23:28.216 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260505 08:23:28.216 | INFO | Uploaded 36GiB to GPU in 2.24023s (16.0698GiB/s) | 20260505 08:23:28.216 | INFO | Downloaded 38GiB to GPU in 2.35187s (16.1574GiB/s) | 20260505 08:23:33.355 | INFO | Analyzing chunk frames = [1745408,1745920) | 20260505 08:23:33.355 | INFO | MemInfo: { | 20260505 08:23:33.355 | INFO | "gpu" : " 17 GB", | 20260505 08:23:33.355 | INFO | "huge" : "", | 20260505 08:23:33.355 | INFO | "huge_shared" : " 27 GB", | 20260505 08:23:33.355 | INFO | "malloc" : " 14 GB", | 20260505 08:23:33.355 | INFO | "pinned" : " 29 GB" | 20260505 08:23:33.355 | INFO | } | 20260505 08:23:33.355 | INFO | Perf report for the last 5.13952 seconds: | 20260505 08:23:33.355 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:23:33.355 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:33.355 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:33.355 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:33.355 | INFO | ANALYSIS : 64.8%, 3.0, 101.1ms, 0 | 20260505 08:23:33.355 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:33.355 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:33.355 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 08:23:33.356 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:33.356 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:33.356 | INFO | Uploaded 36GiB to GPU in 2.24513s (16.0347GiB/s) | 20260505 08:23:33.356 | INFO | Downloaded 34GiB to GPU in 2.09385s (16.238GiB/s) | 20260505 08:23:38.494 | INFO | Analyzing chunk frames = [1745920,1746432) | 20260505 08:23:38.494 | INFO | MemInfo: { | 20260505 08:23:38.494 | INFO | "gpu" : " 17 GB", | 20260505 08:23:38.494 | INFO | "huge" : "", | 20260505 08:23:38.494 | INFO | "huge_shared" : " 27 GB", | 20260505 08:23:38.494 | INFO | "malloc" : " 14 GB", | 20260505 08:23:38.494 | INFO | "pinned" : " 29 GB" | 20260505 08:23:38.494 | INFO | } | 20260505 08:23:38.494 | INFO | Perf report for the last 5.13898 seconds: | 20260505 08:23:38.494 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:23:38.494 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:38.494 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:38.494 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:38.494 | INFO | ANALYSIS : 63.6%, 3.0, 99.0ms, 0 | 20260505 08:23:38.494 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:38.494 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:38.494 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:23:38.495 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:38.495 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:38.495 | INFO | Uploaded 36GiB to GPU in 2.17092s (16.5829GiB/s) | 20260505 08:23:38.495 | INFO | Downloaded 34GiB to GPU in 2.06663s (16.4519GiB/s) | 20260505 08:23:43.633 | INFO | Analyzing chunk frames = [1746432,1746944) | 20260505 08:23:43.633 | INFO | MemInfo: { | 20260505 08:23:43.633 | INFO | "gpu" : " 17 GB", | 20260505 08:23:43.633 | INFO | "huge" : "", | 20260505 08:23:43.633 | INFO | "huge_shared" : " 27 GB", | 20260505 08:23:43.633 | INFO | "malloc" : " 14 GB", | 20260505 08:23:43.633 | INFO | "pinned" : " 29 GB" | 20260505 08:23:43.634 | INFO | } | 20260505 08:23:43.634 | INFO | Perf report for the last 5.13915 seconds: | 20260505 08:23:43.634 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:23:43.634 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:43.634 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:43.634 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:43.634 | INFO | ANALYSIS : 63.4%, 3.0, 98.5ms, 0 | 20260505 08:23:43.634 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:43.634 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:43.634 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:23:43.634 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:43.634 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:43.634 | INFO | Uploaded 36GiB to GPU in 2.15866s (16.677GiB/s) | 20260505 08:23:43.634 | INFO | Downloaded 34GiB to GPU in 2.04422s (16.6323GiB/s) | 20260505 08:23:48.773 | INFO | Analyzing chunk frames = [1746944,1747456) | 20260505 08:23:48.773 | INFO | MemInfo: { | 20260505 08:23:48.773 | INFO | "gpu" : " 17 GB", | 20260505 08:23:48.773 | INFO | "huge" : "", | 20260505 08:23:48.773 | INFO | "huge_shared" : " 27 GB", | 20260505 08:23:48.773 | INFO | "malloc" : " 14 GB", | 20260505 08:23:48.773 | INFO | "pinned" : " 29 GB" | 20260505 08:23:48.773 | INFO | } | 20260505 08:23:48.773 | INFO | Perf report for the last 5.13958 seconds: | 20260505 08:23:48.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:23:48.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:48.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:48.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:48.773 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 08:23:48.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:48.773 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:48.773 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 08:23:48.773 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:48.773 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:48.773 | INFO | Uploaded 36GiB to GPU in 2.19184s (16.4246GiB/s) | 20260505 08:23:48.773 | INFO | Downloaded 34GiB to GPU in 2.06093s (16.4974GiB/s) | 20260505 08:23:53.914 | INFO | Analyzing chunk frames = [1747456,1747968) | 20260505 08:23:53.914 | INFO | MemInfo: { | 20260505 08:23:53.914 | INFO | "gpu" : " 17 GB", | 20260505 08:23:53.914 | INFO | "huge" : "", | 20260505 08:23:53.914 | INFO | "huge_shared" : " 27 GB", | 20260505 08:23:53.914 | INFO | "malloc" : " 14 GB", | 20260505 08:23:53.914 | INFO | "pinned" : " 29 GB" | 20260505 08:23:53.914 | INFO | } | 20260505 08:23:53.915 | INFO | Perf report for the last 5.14138 seconds: | 20260505 08:23:53.915 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:23:53.915 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:53.915 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:53.915 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:53.915 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 08:23:53.915 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:53.915 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:53.915 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:23:53.915 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:53.915 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:53.915 | INFO | Uploaded 36GiB to GPU in 2.18256s (16.4944GiB/s) | 20260505 08:23:53.915 | INFO | Downloaded 34GiB to GPU in 2.05074s (16.5794GiB/s) | 20260505 08:23:59.053 | INFO | Analyzing chunk frames = [1747968,1748480) | 20260505 08:23:59.053 | INFO | MemInfo: { | 20260505 08:23:59.053 | INFO | "gpu" : " 17 GB", | 20260505 08:23:59.053 | INFO | "huge" : "", | 20260505 08:23:59.053 | INFO | "huge_shared" : " 27 GB", | 20260505 08:23:59.053 | INFO | "malloc" : " 14 GB", | 20260505 08:23:59.053 | INFO | "pinned" : " 29 GB" | 20260505 08:23:59.053 | INFO | } | 20260505 08:23:59.053 | INFO | Perf report for the last 5.13886 seconds: | 20260505 08:23:59.053 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:23:59.053 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:59.053 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:59.053 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:59.053 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 08:23:59.054 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:59.054 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:59.054 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 08:23:59.054 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:59.054 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:23:59.054 | INFO | Uploaded 36GiB to GPU in 2.22319s (16.1929GiB/s) | 20260505 08:23:59.054 | INFO | Downloaded 34GiB to GPU in 2.1546s (15.7802GiB/s) | 20260505 08:24:04.191 | INFO | Analyzing chunk frames = [1748480,1748992) | 20260505 08:24:04.191 | INFO | MemInfo: { | 20260505 08:24:04.191 | INFO | "gpu" : " 17 GB", | 20260505 08:24:04.191 | INFO | "huge" : "", | 20260505 08:24:04.191 | INFO | "huge_shared" : " 27 GB", | 20260505 08:24:04.191 | INFO | "malloc" : " 14 GB", | 20260505 08:24:04.191 | INFO | "pinned" : " 29 GB" | 20260505 08:24:04.191 | INFO | } | 20260505 08:24:04.191 | INFO | Perf report for the last 5.13758 seconds: | 20260505 08:24:04.191 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:24:04.191 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:04.191 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:04.191 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:04.191 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 08:24:04.191 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:04.191 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:04.191 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 08:24:04.191 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:04.191 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:04.191 | INFO | Uploaded 36GiB to GPU in 2.19201s (16.4233GiB/s) | 20260505 08:24:04.191 | INFO | Downloaded 34GiB to GPU in 2.0387s (16.6773GiB/s) | 20260505 08:24:04.333 | INFO | Starting a round of Baz IO | 20260505 08:24:07.620 | INFO | Populating RTMetrics report | 20260505 08:24:07.620 | INFO | RTMetrics report populated | 20260505 08:24:07.620 | INFO | Opening tmp RTMetrics json file | 20260505 08:24:07.634 | INFO | Serializing RTMetrics to json | 20260505 08:24:07.639 | INFO | Serialization done, swapping tmp file now... | 20260505 08:24:07.651 | INFO | RTMetrics done | 20260505 08:24:07.766 | INFO | Finished a round of Baz IO: Wrote 2.449 GiB in 3.433 seconds | 20260505 08:24:07.766 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 27.73% / 0.02% / 72.24% | 20260505 08:24:07.766 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.69024% / 2.38419e-05% / 8.34465e-05% | 20260505 08:24:07.766 | INFO | Baz IO overall: | 20260505 08:24:07.766 | INFO | BytesWritten : 985663301455 | 20260505 08:24:07.766 | INFO | NumEvents : 468494180023 | 20260505 08:24:09.330 | INFO | Analyzing chunk frames = [1748992,1749504) | 20260505 08:24:09.330 | INFO | MemInfo: { | 20260505 08:24:09.330 | INFO | "gpu" : " 17 GB", | 20260505 08:24:09.330 | INFO | "huge" : "", | 20260505 08:24:09.330 | INFO | "huge_shared" : " 27 GB", | 20260505 08:24:09.330 | INFO | "malloc" : " 14 GB", | 20260505 08:24:09.330 | INFO | "pinned" : " 29 GB" | 20260505 08:24:09.330 | INFO | } | 20260505 08:24:09.331 | INFO | Perf report for the last 5.1395 seconds: | 20260505 08:24:09.331 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:24:09.331 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:09.331 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:09.331 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:09.331 | INFO | ANALYSIS : 65.6%, 2.9, 100.3ms, 0 | 20260505 08:24:09.331 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:09.331 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260505 08:24:09.331 | INFO | PRE_HQ : 63.3%, 1.9, 61.2ms, 0 | 20260505 08:24:09.331 | INFO | BAZWRITER : 66.1%, 4.1, 139.9ms, 0 | 20260505 08:24:09.331 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 08:24:09.331 | INFO | Uploaded 36GiB to GPU in 2.23689s (16.0938GiB/s) | 20260505 08:24:09.331 | INFO | Downloaded 38GiB to GPU in 2.35347s (16.1464GiB/s) | 20260505 08:24:14.471 | INFO | Analyzing chunk frames = [1749504,1750016) | 20260505 08:24:14.471 | INFO | MemInfo: { | 20260505 08:24:14.471 | INFO | "gpu" : " 17 GB", | 20260505 08:24:14.471 | INFO | "huge" : "", | 20260505 08:24:14.471 | INFO | "huge_shared" : " 27 GB", | 20260505 08:24:14.471 | INFO | "malloc" : " 14 GB", | 20260505 08:24:14.471 | INFO | "pinned" : " 29 GB" | 20260505 08:24:14.471 | INFO | } | 20260505 08:24:14.471 | INFO | Perf report for the last 5.14032 seconds: | 20260505 08:24:14.471 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:24:14.471 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:14.471 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:14.471 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:14.471 | INFO | ANALYSIS : 64.7%, 3.0, 101.0ms, 0 | 20260505 08:24:14.471 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:14.471 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:14.471 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260505 08:24:14.471 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:14.471 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:14.471 | INFO | Uploaded 36GiB to GPU in 2.24792s (16.0148GiB/s) | 20260505 08:24:14.471 | INFO | Downloaded 34GiB to GPU in 2.10946s (16.1179GiB/s) | 20260505 08:24:19.610 | INFO | Analyzing chunk frames = [1750016,1750528) | 20260505 08:24:19.610 | INFO | MemInfo: { | 20260505 08:24:19.610 | INFO | "gpu" : " 17 GB", | 20260505 08:24:19.610 | INFO | "huge" : "", | 20260505 08:24:19.610 | INFO | "huge_shared" : " 27 GB", | 20260505 08:24:19.610 | INFO | "malloc" : " 14 GB", | 20260505 08:24:19.610 | INFO | "pinned" : " 29 GB" | 20260505 08:24:19.610 | INFO | } | 20260505 08:24:19.610 | INFO | Perf report for the last 5.13916 seconds: | 20260505 08:24:19.610 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:24:19.610 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:19.610 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:19.610 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:19.610 | INFO | ANALYSIS : 63.6%, 3.0, 99.0ms, 0 | 20260505 08:24:19.610 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:19.610 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:19.610 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:24:19.610 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:19.610 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:19.610 | INFO | Uploaded 36GiB to GPU in 2.1691s (16.5968GiB/s) | 20260505 08:24:19.610 | INFO | Downloaded 34GiB to GPU in 2.06581s (16.4585GiB/s) | 20260505 08:24:24.749 | INFO | Analyzing chunk frames = [1750528,1751040) | 20260505 08:24:24.749 | INFO | MemInfo: { | 20260505 08:24:24.749 | INFO | "gpu" : " 17 GB", | 20260505 08:24:24.749 | INFO | "huge" : "", | 20260505 08:24:24.749 | INFO | "huge_shared" : " 27 GB", | 20260505 08:24:24.749 | INFO | "malloc" : " 14 GB", | 20260505 08:24:24.749 | INFO | "pinned" : " 29 GB" | 20260505 08:24:24.749 | INFO | } | 20260505 08:24:24.749 | INFO | Perf report for the last 5.13925 seconds: | 20260505 08:24:24.749 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:24:24.749 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:24.749 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:24.749 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:24.749 | INFO | ANALYSIS : 63.6%, 3.0, 98.8ms, 0 | 20260505 08:24:24.749 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:24.749 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:24.749 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:24:24.749 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:24.749 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:24.750 | INFO | Uploaded 36GiB to GPU in 2.18422s (16.4819GiB/s) | 20260505 08:24:24.750 | INFO | Downloaded 34GiB to GPU in 2.06649s (16.453GiB/s) | 20260505 08:24:29.889 | INFO | Analyzing chunk frames = [1751040,1751552) | 20260505 08:24:29.889 | INFO | MemInfo: { | 20260505 08:24:29.889 | INFO | "gpu" : " 17 GB", | 20260505 08:24:29.889 | INFO | "huge" : "", | 20260505 08:24:29.889 | INFO | "huge_shared" : " 27 GB", | 20260505 08:24:29.889 | INFO | "malloc" : " 14 GB", | 20260505 08:24:29.889 | INFO | "pinned" : " 29 GB" | 20260505 08:24:29.889 | INFO | } | 20260505 08:24:29.889 | INFO | Perf report for the last 5.13958 seconds: | 20260505 08:24:29.889 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:24:29.889 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:29.889 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:29.889 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:29.889 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 08:24:29.889 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:29.889 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:29.889 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:24:29.889 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:29.889 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:29.889 | INFO | Uploaded 36GiB to GPU in 2.18812s (16.4525GiB/s) | 20260505 08:24:29.889 | INFO | Downloaded 34GiB to GPU in 2.06035s (16.502GiB/s) | 20260505 08:24:35.029 | INFO | Analyzing chunk frames = [1751552,1752064) | 20260505 08:24:35.029 | INFO | MemInfo: { | 20260505 08:24:35.029 | INFO | "gpu" : " 17 GB", | 20260505 08:24:35.029 | INFO | "huge" : "", | 20260505 08:24:35.029 | INFO | "huge_shared" : " 27 GB", | 20260505 08:24:35.029 | INFO | "malloc" : " 14 GB", | 20260505 08:24:35.029 | INFO | "pinned" : " 29 GB" | 20260505 08:24:35.029 | INFO | } | 20260505 08:24:35.029 | INFO | Perf report for the last 5.14016 seconds: | 20260505 08:24:35.029 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:24:35.029 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:35.029 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:35.029 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:35.029 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 08:24:35.029 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:35.029 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:35.029 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 08:24:35.029 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:35.029 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:35.029 | INFO | Uploaded 36GiB to GPU in 2.19027s (16.4364GiB/s) | 20260505 08:24:35.029 | INFO | Downloaded 34GiB to GPU in 2.05802s (16.5208GiB/s) | 20260505 08:24:40.167 | INFO | Analyzing chunk frames = [1752064,1752576) | 20260505 08:24:40.167 | INFO | MemInfo: { | 20260505 08:24:40.167 | INFO | "gpu" : " 17 GB", | 20260505 08:24:40.167 | INFO | "huge" : "", | 20260505 08:24:40.167 | INFO | "huge_shared" : " 27 GB", | 20260505 08:24:40.167 | INFO | "malloc" : " 14 GB", | 20260505 08:24:40.167 | INFO | "pinned" : " 29 GB" | 20260505 08:24:40.167 | INFO | } | 20260505 08:24:40.167 | INFO | Perf report for the last 5.13824 seconds: | 20260505 08:24:40.167 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:24:40.167 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:40.167 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:40.167 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:40.167 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 08:24:40.167 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:40.167 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:40.167 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 08:24:40.167 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:40.167 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:40.168 | INFO | Uploaded 36GiB to GPU in 2.20292s (16.3419GiB/s) | 20260505 08:24:40.168 | INFO | Downloaded 34GiB to GPU in 2.15054s (15.81GiB/s) | 20260505 08:24:45.306 | INFO | Analyzing chunk frames = [1752576,1753088) | 20260505 08:24:45.306 | INFO | MemInfo: { | 20260505 08:24:45.306 | INFO | "gpu" : " 17 GB", | 20260505 08:24:45.306 | INFO | "huge" : "", | 20260505 08:24:45.306 | INFO | "huge_shared" : " 27 GB", | 20260505 08:24:45.306 | INFO | "malloc" : " 14 GB", | 20260505 08:24:45.306 | INFO | "pinned" : " 29 GB" | 20260505 08:24:45.306 | INFO | } | 20260505 08:24:45.306 | INFO | Perf report for the last 5.13872 seconds: | 20260505 08:24:45.306 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:24:45.306 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:45.306 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:45.306 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:45.306 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 08:24:45.306 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:45.306 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:45.306 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 08:24:45.306 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:45.306 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:45.306 | INFO | Uploaded 36GiB to GPU in 2.1977s (16.3808GiB/s) | 20260505 08:24:45.306 | INFO | Downloaded 34GiB to GPU in 2.04417s (16.6327GiB/s) | 20260505 08:24:45.307 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 08:24:45.307 | INFO | Total mem usage: 15.3 GiB | 20260505 08:24:45.307 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 08:24:45.307 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.29%) | 20260505 08:24:45.307 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 08:24:45.307 | INFO | High Water Breakdown: | 20260505 08:24:45.307 | INFO | 60.8% : 8.77 GiB ( 93.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 08:24:45.307 | INFO | 39% : 5.63 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 08:24:45.307 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 08:24:45.307 | INFO | ------------------------------------------ | 20260505 08:24:45.307 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:24:45.307 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 08:24:45.307 | INFO | Total mem usage: 26.5 GiB | 20260505 08:24:45.307 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 08:24:45.307 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 08:24:45.307 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 08:24:45.307 | INFO | High Water Breakdown: | 20260505 08:24:45.307 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:24:45.307 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:24:45.307 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:24:45.307 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:24:45.307 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:24:45.307 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:24:45.307 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 08:24:45.307 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 08:24:45.307 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:24:45.307 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 08:24:45.307 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 08:24:45.307 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 08:24:45.307 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 08:24:45.307 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 08:24:45.307 | INFO | ------------------------------------------ | 20260505 08:24:45.307 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:24:45.307 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 08:24:45.307 | INFO | Total mem usage: 25.6 GiB | 20260505 08:24:45.307 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 08:24:45.307 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 08:24:45.307 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 08:24:45.307 | INFO | High Water Breakdown: | 20260505 08:24:45.307 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 08:24:45.307 | INFO | ------------------------------------------ | 20260505 08:24:45.307 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:24:45.307 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 08:24:45.307 | INFO | Total mem usage: 15.7 GiB | 20260505 08:24:45.307 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 08:24:45.307 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 08:24:45.307 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 08:24:45.307 | INFO | High Water Breakdown: | 20260505 08:24:45.307 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 08:24:45.307 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 08:24:45.307 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 08:24:45.307 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 08:24:45.307 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 08:24:45.307 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 08:24:45.307 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 08:24:45.307 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 08:24:45.307 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 08:24:45.307 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:24:45.307 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 08:24:45.307 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:24:45.307 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 08:24:45.307 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 08:24:45.307 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 08:24:45.307 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 08:24:45.307 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:24:45.307 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 08:24:45.307 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:24:45.307 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:24:45.307 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:24:45.307 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 08:24:45.307 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 08:24:45.307 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:24:45.307 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 08:24:45.307 | INFO | ------------------------------------------ | 20260505 08:24:45.307 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:24:45.307 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 08:24:45.307 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:24:45.307 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:24:45.307 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.55e+01, 1.36e+01 / 2.09e+00 | 20260505 08:24:45.307 | INFO | PerfReport: 0.28%, AggregateStats -- 9.30e-02 / 1.03e-01, 1.01e-02 / 4.38e-03 | 20260505 08:24:45.307 | INFO | PerfReport: 11.64%, Baseline -- 3.80e+00 / 4.48e+00, 6.27e-01 / 2.74e-01 | 20260505 08:24:45.307 | INFO | PerfReport: 3.32%, Histogram -- 1.08e+00 / 1.50e+00, 2.14e-01 / 1.03e-01 | 20260505 08:24:45.307 | INFO | PerfReport: 15.55%, DME -- 4.06e+01 / 4.45e+01, 4.12e+00 / 1.85e+00 | 20260505 08:24:45.307 | INFO | PerfReport: 56.48%, FrameLabeling -- 1.84e+01 / 1.99e+01, 1.96e+00 / 1.56e+00 | 20260505 08:24:45.307 | INFO | PerfReport: 7.44%, PulseAccumulating -- 2.43e+00 / 3.01e+00, 4.02e-01 / 2.84e-01 | 20260505 08:24:45.307 | INFO | PerfReport: 3.89%, Metrics -- 1.27e+00 / 1.57e+00, 3.56e-01 / 1.51e-01 | 20260505 08:24:45.307 | INFO | PerfReport: 1.40%, PulseSerializing -- 4.56e-01 / 5.09e-01, 8.71e-02 / 2.90e-02 | 20260505 08:24:45.307 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 08:24:45.307 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:24:45.307 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:24:45.307 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.32e+01 / 0.00e+00 | 20260505 08:24:45.307 | INFO | PerfReport: 22.53%, Upload -- 2.29e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260505 08:24:45.308 | INFO | PerfReport: 21.60%, Download -- 2.20e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260505 08:24:45.308 | INFO | PerfReport: 32.08%, Compute -- 3.27e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 08:24:45.308 | INFO | PerfReport: 1.68%, LockUpload -- 1.71e+00 / 0.00e+00, 7.32e+00 / 0.00e+00 | 20260505 08:24:45.308 | INFO | PerfReport: 1.02%, LockDownload -- 1.04e+00 / 0.00e+00, 2.52e+00 / 0.00e+00 | 20260505 08:24:45.308 | INFO | PerfReport: 21.06%, LockCompute -- 2.14e+01 / 0.00e+00, 1.78e+01 / 0.00e+00 | 20260505 08:24:45.308 | INFO | PerfReport: 0.01%, Misc -- 1.46e-02 / 0.00e+00, 2.11e-02 / 0.00e+00 | 20260505 08:24:45.446 | INFO | Starting a round of Baz IO | 20260505 08:24:48.728 | INFO | Populating RTMetrics report | 20260505 08:24:48.728 | INFO | RTMetrics report populated | 20260505 08:24:48.728 | INFO | Opening tmp RTMetrics json file | 20260505 08:24:48.747 | INFO | Serializing RTMetrics to json | 20260505 08:24:48.751 | INFO | Serialization done, swapping tmp file now... | 20260505 08:24:48.751 | INFO | RTMetrics done | 20260505 08:24:48.877 | INFO | Finished a round of Baz IO: Wrote 2.449 GiB in 3.43 seconds | 20260505 08:24:48.877 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 27.71% / 0.02% / 72.27% | 20260505 08:24:48.877 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.67873% / 3.97364e-05% / 8.74201e-05% | 20260505 08:24:48.877 | INFO | Baz IO overall: | 20260505 08:24:48.877 | INFO | BytesWritten : 988293046852 | 20260505 08:24:48.877 | INFO | NumEvents : 469751782605 | 20260505 08:24:50.446 | INFO | Analyzing chunk frames = [1753088,1753600) | 20260505 08:24:50.446 | INFO | MemInfo: { | 20260505 08:24:50.446 | INFO | "gpu" : " 17 GB", | 20260505 08:24:50.446 | INFO | "huge" : "", | 20260505 08:24:50.446 | INFO | "huge_shared" : " 27 GB", | 20260505 08:24:50.446 | INFO | "malloc" : " 14 GB", | 20260505 08:24:50.446 | INFO | "pinned" : " 29 GB" | 20260505 08:24:50.446 | INFO | } | 20260505 08:24:50.446 | INFO | Perf report for the last 5.13981 seconds: | 20260505 08:24:50.446 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:24:50.446 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:50.446 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:50.446 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:50.446 | INFO | ANALYSIS : 65.5%, 2.9, 100.2ms, 0 | 20260505 08:24:50.446 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:50.446 | INFO | RT_METRICS : 46.2%, 1.0, 24.2ms, 0 | 20260505 08:24:50.446 | INFO | PRE_HQ : 63.5%, 1.9, 61.1ms, 0 | 20260505 08:24:50.446 | INFO | BAZWRITER : 66.0%, 4.1, 140.7ms, 0 | 20260505 08:24:50.446 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 08:24:50.446 | INFO | Uploaded 36GiB to GPU in 2.27615s (15.8162GiB/s) | 20260505 08:24:50.446 | INFO | Downloaded 38GiB to GPU in 2.37219s (16.0189GiB/s) | 20260505 08:24:55.586 | INFO | Analyzing chunk frames = [1753600,1754112) | 20260505 08:24:55.586 | INFO | MemInfo: { | 20260505 08:24:55.586 | INFO | "gpu" : " 17 GB", | 20260505 08:24:55.586 | INFO | "huge" : "", | 20260505 08:24:55.586 | INFO | "huge_shared" : " 27 GB", | 20260505 08:24:55.586 | INFO | "malloc" : " 14 GB", | 20260505 08:24:55.587 | INFO | "pinned" : " 29 GB" | 20260505 08:24:55.587 | INFO | } | 20260505 08:24:55.587 | INFO | Perf report for the last 5.14091 seconds: | 20260505 08:24:55.587 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:24:55.587 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:55.587 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:55.587 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:55.587 | INFO | ANALYSIS : 64.6%, 3.0, 100.8ms, 0 | 20260505 08:24:55.587 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:55.587 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:55.587 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:24:55.587 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:55.587 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:24:55.587 | INFO | Uploaded 36GiB to GPU in 2.23986s (16.0724GiB/s) | 20260505 08:24:55.587 | INFO | Downloaded 34GiB to GPU in 2.10778s (16.1307GiB/s) | 20260505 08:25:00.725 | INFO | Analyzing chunk frames = [1754112,1754624) | 20260505 08:25:00.725 | INFO | MemInfo: { | 20260505 08:25:00.725 | INFO | "gpu" : " 17 GB", | 20260505 08:25:00.725 | INFO | "huge" : "", | 20260505 08:25:00.725 | INFO | "huge_shared" : " 27 GB", | 20260505 08:25:00.726 | INFO | "malloc" : " 14 GB", | 20260505 08:25:00.726 | INFO | "pinned" : " 29 GB" | 20260505 08:25:00.726 | INFO | } | 20260505 08:25:00.726 | INFO | Perf report for the last 5.13881 seconds: | 20260505 08:25:00.726 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:25:00.726 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:00.726 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:00.726 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:00.726 | INFO | ANALYSIS : 63.6%, 3.0, 99.0ms, 0 | 20260505 08:25:00.726 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:00.726 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:00.726 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:25:00.726 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:00.726 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:00.726 | INFO | Uploaded 36GiB to GPU in 2.16375s (16.6378GiB/s) | 20260505 08:25:00.726 | INFO | Downloaded 34GiB to GPU in 2.06563s (16.4599GiB/s) | 20260505 08:25:05.863 | INFO | Analyzing chunk frames = [1754624,1755136) | 20260505 08:25:05.863 | INFO | MemInfo: { | 20260505 08:25:05.863 | INFO | "gpu" : " 17 GB", | 20260505 08:25:05.863 | INFO | "huge" : "", | 20260505 08:25:05.863 | INFO | "huge_shared" : " 27 GB", | 20260505 08:25:05.863 | INFO | "malloc" : " 14 GB", | 20260505 08:25:05.863 | INFO | "pinned" : " 29 GB" | 20260505 08:25:05.863 | INFO | } | 20260505 08:25:05.863 | INFO | Perf report for the last 5.13756 seconds: | 20260505 08:25:05.863 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:25:05.863 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:05.863 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:05.863 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:05.863 | INFO | ANALYSIS : 63.5%, 3.0, 98.6ms, 0 | 20260505 08:25:05.863 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:05.863 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:05.863 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:25:05.863 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:05.863 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:05.864 | INFO | Uploaded 36GiB to GPU in 2.16596s (16.6208GiB/s) | 20260505 08:25:05.864 | INFO | Downloaded 34GiB to GPU in 2.06784s (16.4423GiB/s) | 20260505 08:25:11.002 | INFO | Analyzing chunk frames = [1755136,1755648) | 20260505 08:25:11.002 | INFO | MemInfo: { | 20260505 08:25:11.002 | INFO | "gpu" : " 17 GB", | 20260505 08:25:11.003 | INFO | "huge" : "", | 20260505 08:25:11.003 | INFO | "huge_shared" : " 27 GB", | 20260505 08:25:11.003 | INFO | "malloc" : " 14 GB", | 20260505 08:25:11.003 | INFO | "pinned" : " 29 GB" | 20260505 08:25:11.003 | INFO | } | 20260505 08:25:11.003 | INFO | Perf report for the last 5.13959 seconds: | 20260505 08:25:11.003 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:25:11.003 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:11.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:11.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:11.003 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 08:25:11.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:11.003 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:11.003 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:25:11.003 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:11.003 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:11.003 | INFO | Uploaded 36GiB to GPU in 2.1816s (16.5016GiB/s) | 20260505 08:25:11.003 | INFO | Downloaded 34GiB to GPU in 2.05441s (16.5498GiB/s) | 20260505 08:25:16.143 | INFO | Analyzing chunk frames = [1755648,1756160) | 20260505 08:25:16.143 | INFO | MemInfo: { | 20260505 08:25:16.143 | INFO | "gpu" : " 17 GB", | 20260505 08:25:16.143 | INFO | "huge" : "", | 20260505 08:25:16.143 | INFO | "huge_shared" : " 27 GB", | 20260505 08:25:16.143 | INFO | "malloc" : " 14 GB", | 20260505 08:25:16.143 | INFO | "pinned" : " 29 GB" | 20260505 08:25:16.147 | INFO | } | 20260505 08:25:16.147 | INFO | Perf report for the last 5.14422 seconds: | 20260505 08:25:16.147 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:25:16.147 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:16.147 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:16.147 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:16.147 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 08:25:16.147 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:16.147 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:16.147 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:25:16.147 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:16.147 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:16.147 | INFO | Uploaded 36GiB to GPU in 2.18385s (16.4846GiB/s) | 20260505 08:25:16.147 | INFO | Downloaded 34GiB to GPU in 2.05564s (16.5398GiB/s) | 20260505 08:25:21.281 | INFO | Analyzing chunk frames = [1756160,1756672) | 20260505 08:25:21.281 | INFO | MemInfo: { | 20260505 08:25:21.281 | INFO | "gpu" : " 17 GB", | 20260505 08:25:21.281 | INFO | "huge" : "", | 20260505 08:25:21.281 | INFO | "huge_shared" : " 27 GB", | 20260505 08:25:21.281 | INFO | "malloc" : " 14 GB", | 20260505 08:25:21.281 | INFO | "pinned" : " 29 GB" | 20260505 08:25:21.281 | INFO | } | 20260505 08:25:21.281 | INFO | Perf report for the last 5.13447 seconds: | 20260505 08:25:21.281 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:25:21.281 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:21.281 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:21.281 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:21.282 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 08:25:21.282 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:21.282 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:21.282 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 08:25:21.282 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:21.282 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:21.282 | INFO | Uploaded 36GiB to GPU in 2.16178s (16.653GiB/s) | 20260505 08:25:21.282 | INFO | Downloaded 34GiB to GPU in 2.14832s (15.8263GiB/s) | 20260505 08:25:26.421 | INFO | Analyzing chunk frames = [1756672,1757184) | 20260505 08:25:26.421 | INFO | MemInfo: { | 20260505 08:25:26.421 | INFO | "gpu" : " 17 GB", | 20260505 08:25:26.421 | INFO | "huge" : "", | 20260505 08:25:26.421 | INFO | "huge_shared" : " 27 GB", | 20260505 08:25:26.422 | INFO | "malloc" : " 14 GB", | 20260505 08:25:26.422 | INFO | "pinned" : " 29 GB" | 20260505 08:25:26.422 | INFO | } | 20260505 08:25:26.422 | INFO | Perf report for the last 5.14022 seconds: | 20260505 08:25:26.422 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:25:26.422 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:26.422 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:26.422 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:26.422 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 08:25:26.422 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:26.422 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:26.422 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 08:25:26.422 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:26.422 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:26.422 | INFO | Uploaded 36GiB to GPU in 2.20281s (16.3428GiB/s) | 20260505 08:25:26.422 | INFO | Downloaded 34GiB to GPU in 2.04482s (16.6274GiB/s) | 20260505 08:25:26.563 | INFO | Starting a round of Baz IO | 20260505 08:25:29.847 | INFO | Populating RTMetrics report | 20260505 08:25:29.847 | INFO | RTMetrics report populated | 20260505 08:25:29.847 | INFO | Opening tmp RTMetrics json file | 20260505 08:25:29.850 | INFO | Serializing RTMetrics to json | 20260505 08:25:29.851 | INFO | Serialization done, swapping tmp file now... | 20260505 08:25:29.852 | INFO | RTMetrics done | 20260505 08:25:29.988 | INFO | Finished a round of Baz IO: Wrote 2.452 GiB in 3.425 seconds | 20260505 08:25:29.988 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 27.69% / 0.02% / 72.29% | 20260505 08:25:29.988 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.67113% / 3.17891e-05% / 5.5631e-05% | 20260505 08:25:29.988 | INFO | Baz IO overall: | 20260505 08:25:29.988 | INFO | BytesWritten : 990925383217 | 20260505 08:25:29.988 | INFO | NumEvents : 471010462778 | 20260505 08:25:31.561 | INFO | Analyzing chunk frames = [1757184,1757696) | 20260505 08:25:31.561 | INFO | MemInfo: { | 20260505 08:25:31.561 | INFO | "gpu" : " 17 GB", | 20260505 08:25:31.561 | INFO | "huge" : "", | 20260505 08:25:31.561 | INFO | "huge_shared" : " 27 GB", | 20260505 08:25:31.562 | INFO | "malloc" : " 14 GB", | 20260505 08:25:31.562 | INFO | "pinned" : " 29 GB" | 20260505 08:25:31.562 | INFO | } | 20260505 08:25:31.562 | INFO | Perf report for the last 5.1399 seconds: | 20260505 08:25:31.562 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:25:31.562 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:31.562 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:31.562 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:31.562 | INFO | ANALYSIS : 65.5%, 2.9, 100.0ms, 0 | 20260505 08:25:31.562 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:31.562 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260505 08:25:31.562 | INFO | PRE_HQ : 63.5%, 1.9, 61.6ms, 0 | 20260505 08:25:31.562 | INFO | BAZWRITER : 65.9%, 4.1, 139.5ms, 0 | 20260505 08:25:31.562 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 08:25:31.562 | INFO | Uploaded 36GiB to GPU in 2.2707s (15.8541GiB/s) | 20260505 08:25:31.562 | INFO | Downloaded 38GiB to GPU in 2.36907s (16.04GiB/s) | 20260505 08:25:36.700 | INFO | Analyzing chunk frames = [1757696,1758208) | 20260505 08:25:36.700 | INFO | MemInfo: { | 20260505 08:25:36.700 | INFO | "gpu" : " 17 GB", | 20260505 08:25:36.700 | INFO | "huge" : "", | 20260505 08:25:36.700 | INFO | "huge_shared" : " 27 GB", | 20260505 08:25:36.700 | INFO | "malloc" : " 14 GB", | 20260505 08:25:36.700 | INFO | "pinned" : " 29 GB" | 20260505 08:25:36.700 | INFO | } | 20260505 08:25:36.700 | INFO | Perf report for the last 5.13825 seconds: | 20260505 08:25:36.700 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:25:36.700 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:36.700 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:36.700 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:36.700 | INFO | ANALYSIS : 64.5%, 3.0, 100.7ms, 0 | 20260505 08:25:36.700 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:36.700 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:36.700 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:25:36.700 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:36.700 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:36.700 | INFO | Uploaded 36GiB to GPU in 2.22303s (16.1941GiB/s) | 20260505 08:25:36.700 | INFO | Downloaded 34GiB to GPU in 2.08892s (16.2763GiB/s) | 20260505 08:25:41.839 | INFO | Analyzing chunk frames = [1758208,1758720) | 20260505 08:25:41.839 | INFO | MemInfo: { | 20260505 08:25:41.839 | INFO | "gpu" : " 17 GB", | 20260505 08:25:41.839 | INFO | "huge" : "", | 20260505 08:25:41.839 | INFO | "huge_shared" : " 27 GB", | 20260505 08:25:41.839 | INFO | "malloc" : " 14 GB", | 20260505 08:25:41.839 | INFO | "pinned" : " 29 GB" | 20260505 08:25:41.839 | INFO | } | 20260505 08:25:41.839 | INFO | Perf report for the last 5.13916 seconds: | 20260505 08:25:41.839 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:25:41.839 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:41.839 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:41.839 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:41.839 | INFO | ANALYSIS : 63.6%, 3.0, 99.0ms, 0 | 20260505 08:25:41.839 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:41.839 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:41.839 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 08:25:41.839 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:41.839 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:41.839 | INFO | Uploaded 36GiB to GPU in 2.17104s (16.5819GiB/s) | 20260505 08:25:41.839 | INFO | Downloaded 34GiB to GPU in 2.06766s (16.4437GiB/s) | 20260505 08:25:46.978 | INFO | Analyzing chunk frames = [1758720,1759232) | 20260505 08:25:46.979 | INFO | MemInfo: { | 20260505 08:25:46.979 | INFO | "gpu" : " 17 GB", | 20260505 08:25:46.979 | INFO | "huge" : "", | 20260505 08:25:46.979 | INFO | "huge_shared" : " 27 GB", | 20260505 08:25:46.979 | INFO | "malloc" : " 14 GB", | 20260505 08:25:46.979 | INFO | "pinned" : " 29 GB" | 20260505 08:25:46.979 | INFO | } | 20260505 08:25:46.979 | INFO | Perf report for the last 5.1397 seconds: | 20260505 08:25:46.979 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:25:46.979 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:46.979 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:46.979 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:46.979 | INFO | ANALYSIS : 63.6%, 3.0, 98.8ms, 0 | 20260505 08:25:46.979 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:46.979 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:46.979 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 08:25:46.979 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:46.979 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:46.979 | INFO | Uploaded 36GiB to GPU in 2.18303s (16.4909GiB/s) | 20260505 08:25:46.979 | INFO | Downloaded 34GiB to GPU in 2.06972s (16.4274GiB/s) | 20260505 08:25:52.118 | INFO | Analyzing chunk frames = [1759232,1759744) | 20260505 08:25:52.118 | INFO | MemInfo: { | 20260505 08:25:52.118 | INFO | "gpu" : " 17 GB", | 20260505 08:25:52.118 | INFO | "huge" : "", | 20260505 08:25:52.118 | INFO | "huge_shared" : " 27 GB", | 20260505 08:25:52.118 | INFO | "malloc" : " 14 GB", | 20260505 08:25:52.118 | INFO | "pinned" : " 29 GB" | 20260505 08:25:52.118 | INFO | } | 20260505 08:25:52.118 | INFO | Perf report for the last 5.13911 seconds: | 20260505 08:25:52.118 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:25:52.118 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:52.118 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:52.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:52.118 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 08:25:52.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:52.118 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:52.118 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:25:52.118 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:52.118 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:52.118 | INFO | Uploaded 36GiB to GPU in 2.18368s (16.4859GiB/s) | 20260505 08:25:52.118 | INFO | Downloaded 34GiB to GPU in 2.0553s (16.5426GiB/s) | 20260505 08:25:57.257 | INFO | Analyzing chunk frames = [1759744,1760256) | 20260505 08:25:57.257 | INFO | MemInfo: { | 20260505 08:25:57.257 | INFO | "gpu" : " 17 GB", | 20260505 08:25:57.257 | INFO | "huge" : "", | 20260505 08:25:57.257 | INFO | "huge_shared" : " 27 GB", | 20260505 08:25:57.257 | INFO | "malloc" : " 14 GB", | 20260505 08:25:57.257 | INFO | "pinned" : " 29 GB" | 20260505 08:25:57.258 | INFO | } | 20260505 08:25:57.258 | INFO | Perf report for the last 5.13967 seconds: | 20260505 08:25:57.258 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:25:57.258 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:57.258 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:57.258 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:57.258 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 08:25:57.258 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:57.258 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:57.258 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 08:25:57.258 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:57.258 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:25:57.258 | INFO | Uploaded 36GiB to GPU in 2.16689s (16.6137GiB/s) | 20260505 08:25:57.258 | INFO | Downloaded 34GiB to GPU in 2.04278s (16.644GiB/s) | 20260505 08:26:02.397 | INFO | Analyzing chunk frames = [1760256,1760768) | 20260505 08:26:02.397 | INFO | MemInfo: { | 20260505 08:26:02.397 | INFO | "gpu" : " 17 GB", | 20260505 08:26:02.397 | INFO | "huge" : "", | 20260505 08:26:02.397 | INFO | "huge_shared" : " 27 GB", | 20260505 08:26:02.397 | INFO | "malloc" : " 14 GB", | 20260505 08:26:02.397 | INFO | "pinned" : " 29 GB" | 20260505 08:26:02.397 | INFO | } | 20260505 08:26:02.398 | INFO | Perf report for the last 5.13992 seconds: | 20260505 08:26:02.398 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:26:02.398 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:02.398 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:02.398 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:02.398 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 08:26:02.398 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:02.398 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:02.398 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260505 08:26:02.398 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:02.398 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:02.398 | INFO | Uploaded 36GiB to GPU in 2.2075s (16.308GiB/s) | 20260505 08:26:02.398 | INFO | Downloaded 34GiB to GPU in 2.14829s (15.8265GiB/s) | 20260505 08:26:07.538 | INFO | Analyzing chunk frames = [1760768,1761280) | 20260505 08:26:07.538 | INFO | MemInfo: { | 20260505 08:26:07.538 | INFO | "gpu" : " 17 GB", | 20260505 08:26:07.538 | INFO | "huge" : "", | 20260505 08:26:07.538 | INFO | "huge_shared" : " 27 GB", | 20260505 08:26:07.538 | INFO | "malloc" : " 14 GB", | 20260505 08:26:07.538 | INFO | "pinned" : " 29 GB" | 20260505 08:26:07.538 | INFO | } | 20260505 08:26:07.538 | INFO | Perf report for the last 5.14052 seconds: | 20260505 08:26:07.538 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:26:07.538 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:07.538 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:07.538 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:07.538 | INFO | ANALYSIS : 64.1%, 3.0, 100.2ms, 0 | 20260505 08:26:07.538 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:07.538 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:07.538 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 08:26:07.538 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:07.538 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:07.538 | INFO | Uploaded 36GiB to GPU in 2.19529s (16.3988GiB/s) | 20260505 08:26:07.538 | INFO | Downloaded 34GiB to GPU in 2.04554s (16.6215GiB/s) | 20260505 08:26:07.539 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 08:26:07.539 | INFO | Total mem usage: 15.3 GiB | 20260505 08:26:07.539 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 08:26:07.539 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.31%) | 20260505 08:26:07.539 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 08:26:07.539 | INFO | High Water Breakdown: | 20260505 08:26:07.539 | INFO | 60.8% : 8.77 GiB ( 93.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 08:26:07.539 | INFO | 39% : 5.63 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 08:26:07.539 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 08:26:07.539 | INFO | ------------------------------------------ | 20260505 08:26:07.539 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:26:07.539 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 08:26:07.539 | INFO | Total mem usage: 26.5 GiB | 20260505 08:26:07.539 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 08:26:07.539 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 08:26:07.539 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 08:26:07.539 | INFO | High Water Breakdown: | 20260505 08:26:07.539 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:26:07.539 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:26:07.539 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:26:07.539 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:26:07.539 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:26:07.539 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:26:07.539 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 08:26:07.539 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 08:26:07.539 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:26:07.539 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 08:26:07.539 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 08:26:07.539 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 08:26:07.539 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 08:26:07.539 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 08:26:07.539 | INFO | ------------------------------------------ | 20260505 08:26:07.539 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:26:07.539 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 08:26:07.539 | INFO | Total mem usage: 25.6 GiB | 20260505 08:26:07.539 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 08:26:07.539 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 08:26:07.539 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 08:26:07.539 | INFO | High Water Breakdown: | 20260505 08:26:07.539 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 08:26:07.539 | INFO | ------------------------------------------ | 20260505 08:26:07.539 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:26:07.539 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 08:26:07.539 | INFO | Total mem usage: 15.7 GiB | 20260505 08:26:07.539 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 08:26:07.539 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 08:26:07.539 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 08:26:07.539 | INFO | High Water Breakdown: | 20260505 08:26:07.539 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 08:26:07.539 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 08:26:07.539 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 08:26:07.539 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 08:26:07.539 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 08:26:07.539 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 08:26:07.539 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 08:26:07.539 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 08:26:07.539 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 08:26:07.539 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:26:07.539 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 08:26:07.539 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:26:07.539 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 08:26:07.539 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 08:26:07.539 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 08:26:07.539 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 08:26:07.539 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:26:07.539 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 08:26:07.539 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:26:07.539 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:26:07.539 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:26:07.539 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 08:26:07.539 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 08:26:07.539 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:26:07.539 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 08:26:07.539 | INFO | ------------------------------------------ | 20260505 08:26:07.539 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:26:07.539 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 08:26:07.539 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:26:07.539 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:26:07.539 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.55e+01, 1.37e+01 / 2.17e+00 | 20260505 08:26:07.539 | INFO | PerfReport: 0.28%, AggregateStats -- 9.28e-02 / 1.03e-01, 1.01e-02 / 3.78e-03 | 20260505 08:26:07.539 | INFO | PerfReport: 11.66%, Baseline -- 3.81e+00 / 4.47e+00, 6.26e-01 / 2.39e-01 | 20260505 08:26:07.539 | INFO | PerfReport: 3.33%, Histogram -- 1.09e+00 / 1.50e+00, 2.14e-01 / 1.04e-01 | 20260505 08:26:07.539 | INFO | PerfReport: 15.54%, DME -- 4.06e+01 / 4.44e+01, 4.25e+00 / 1.67e+00 | 20260505 08:26:07.539 | INFO | PerfReport: 56.44%, FrameLabeling -- 1.84e+01 / 1.98e+01, 1.92e+00 / 1.67e+00 | 20260505 08:26:07.539 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.43e+00 / 3.05e+00, 4.02e-01 / 2.33e-01 | 20260505 08:26:07.539 | INFO | PerfReport: 3.89%, Metrics -- 1.27e+00 / 1.61e+00, 3.57e-01 / 1.81e-01 | 20260505 08:26:07.539 | INFO | PerfReport: 1.40%, PulseSerializing -- 4.56e-01 / 5.09e-01, 8.74e-02 / 2.86e-02 | 20260505 08:26:07.539 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 08:26:07.539 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:26:07.539 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:26:07.539 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.31e+01 / 0.00e+00 | 20260505 08:26:07.539 | INFO | PerfReport: 22.53%, Upload -- 2.29e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260505 08:26:07.539 | INFO | PerfReport: 21.65%, Download -- 2.20e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260505 08:26:07.539 | INFO | PerfReport: 32.11%, Compute -- 3.26e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 08:26:07.539 | INFO | PerfReport: 1.65%, LockUpload -- 1.67e+00 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260505 08:26:07.539 | INFO | PerfReport: 1.04%, LockDownload -- 1.06e+00 / 0.00e+00, 2.51e+00 / 0.00e+00 | 20260505 08:26:07.539 | INFO | PerfReport: 21.02%, LockCompute -- 2.14e+01 / 0.00e+00, 1.77e+01 / 0.00e+00 | 20260505 08:26:07.539 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.97e-02 / 0.00e+00 | 20260505 08:26:07.678 | INFO | Starting a round of Baz IO | 20260505 08:26:10.941 | INFO | Populating RTMetrics report | 20260505 08:26:10.941 | INFO | RTMetrics report populated | 20260505 08:26:10.941 | INFO | Opening tmp RTMetrics json file | 20260505 08:26:10.951 | INFO | Serializing RTMetrics to json | 20260505 08:26:10.972 | INFO | Serialization done, swapping tmp file now... | 20260505 08:26:10.973 | INFO | RTMetrics done | 20260505 08:26:11.092 | INFO | Finished a round of Baz IO: Wrote 2.449 GiB in 3.414 seconds | 20260505 08:26:11.092 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 27.67% / 0.02% / 72.31% | 20260505 08:26:11.092 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.65328% / 4.37101e-05% / 0.000115236% | 20260505 08:26:11.092 | INFO | Baz IO overall: | 20260505 08:26:11.092 | INFO | BytesWritten : 993554663178 | 20260505 08:26:11.092 | INFO | NumEvents : 472267138663 | 20260505 08:26:12.678 | INFO | Analyzing chunk frames = [1761280,1761792) | 20260505 08:26:12.678 | INFO | MemInfo: { | 20260505 08:26:12.678 | INFO | "gpu" : " 17 GB", | 20260505 08:26:12.678 | INFO | "huge" : "", | 20260505 08:26:12.678 | INFO | "huge_shared" : " 27 GB", | 20260505 08:26:12.678 | INFO | "malloc" : " 14 GB", | 20260505 08:26:12.678 | INFO | "pinned" : " 29 GB" | 20260505 08:26:12.678 | INFO | } | 20260505 08:26:12.678 | INFO | Perf report for the last 5.13969 seconds: | 20260505 08:26:12.678 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:26:12.678 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:12.678 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:12.678 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:12.678 | INFO | ANALYSIS : 65.1%, 2.9, 99.6ms, 0 | 20260505 08:26:12.678 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:12.678 | INFO | RT_METRICS : 46.5%, 1.0, 24.4ms, 0 | 20260505 08:26:12.678 | INFO | PRE_HQ : 63.3%, 1.9, 61.5ms, 0 | 20260505 08:26:12.678 | INFO | BAZWRITER : 65.8%, 4.2, 140.5ms, 0 | 20260505 08:26:12.678 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 08:26:12.678 | INFO | Uploaded 36GiB to GPU in 2.24212s (16.0563GiB/s) | 20260505 08:26:12.678 | INFO | Downloaded 38GiB to GPU in 2.37839s (15.9772GiB/s) | 20260505 08:26:17.814 | INFO | Analyzing chunk frames = [1761792,1762304) | 20260505 08:26:17.814 | INFO | MemInfo: { | 20260505 08:26:17.814 | INFO | "gpu" : " 17 GB", | 20260505 08:26:17.814 | INFO | "huge" : "", | 20260505 08:26:17.814 | INFO | "huge_shared" : " 27 GB", | 20260505 08:26:17.814 | INFO | "malloc" : " 14 GB", | 20260505 08:26:17.814 | INFO | "pinned" : " 29 GB" | 20260505 08:26:17.814 | INFO | } | 20260505 08:26:17.814 | INFO | Perf report for the last 5.13646 seconds: | 20260505 08:26:17.814 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:26:17.814 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:17.814 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:17.814 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:17.814 | INFO | ANALYSIS : 64.5%, 3.0, 100.7ms, 0 | 20260505 08:26:17.814 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:17.814 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:17.814 | INFO | PRE_HQ : 30.1%, 1.0, 15.8ms, 0 | 20260505 08:26:17.814 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:17.814 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:17.814 | INFO | Uploaded 36GiB to GPU in 2.21827s (16.2288GiB/s) | 20260505 08:26:17.814 | INFO | Downloaded 34GiB to GPU in 2.08454s (16.3105GiB/s) | 20260505 08:26:22.953 | INFO | Analyzing chunk frames = [1762304,1762816) | 20260505 08:26:22.953 | INFO | MemInfo: { | 20260505 08:26:22.953 | INFO | "gpu" : " 17 GB", | 20260505 08:26:22.953 | INFO | "huge" : "", | 20260505 08:26:22.953 | INFO | "huge_shared" : " 27 GB", | 20260505 08:26:22.954 | INFO | "malloc" : " 14 GB", | 20260505 08:26:22.954 | INFO | "pinned" : " 29 GB" | 20260505 08:26:22.954 | INFO | } | 20260505 08:26:22.954 | INFO | Perf report for the last 5.13945 seconds: | 20260505 08:26:22.954 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:26:22.954 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:22.954 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:22.954 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:22.954 | INFO | ANALYSIS : 63.6%, 3.0, 99.0ms, 0 | 20260505 08:26:22.954 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:22.954 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:22.954 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 08:26:22.954 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:22.954 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:22.954 | INFO | Uploaded 36GiB to GPU in 2.17376s (16.5611GiB/s) | 20260505 08:26:22.954 | INFO | Downloaded 34GiB to GPU in 2.06682s (16.4504GiB/s) | 20260505 08:26:28.096 | INFO | Analyzing chunk frames = [1762816,1763328) | 20260505 08:26:28.096 | INFO | MemInfo: { | 20260505 08:26:28.096 | INFO | "gpu" : " 17 GB", | 20260505 08:26:28.096 | INFO | "huge" : "", | 20260505 08:26:28.096 | INFO | "huge_shared" : " 27 GB", | 20260505 08:26:28.097 | INFO | "malloc" : " 14 GB", | 20260505 08:26:28.097 | INFO | "pinned" : " 29 GB" | 20260505 08:26:28.097 | INFO | } | 20260505 08:26:28.097 | INFO | Perf report for the last 5.14303 seconds: | 20260505 08:26:28.097 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:26:28.097 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:28.097 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:28.097 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:28.097 | INFO | ANALYSIS : 63.7%, 3.0, 99.0ms, 0 | 20260505 08:26:28.097 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:28.097 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:28.097 | INFO | PRE_HQ : 30.3%, 1.0, 15.8ms, 0 | 20260505 08:26:28.097 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:28.097 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:28.097 | INFO | Uploaded 36GiB to GPU in 2.18821s (16.4518GiB/s) | 20260505 08:26:28.097 | INFO | Downloaded 34GiB to GPU in 2.06147s (16.493GiB/s) | 20260505 08:26:33.233 | INFO | Analyzing chunk frames = [1763328,1763840) | 20260505 08:26:33.233 | INFO | MemInfo: { | 20260505 08:26:33.233 | INFO | "gpu" : " 17 GB", | 20260505 08:26:33.233 | INFO | "huge" : "", | 20260505 08:26:33.233 | INFO | "huge_shared" : " 27 GB", | 20260505 08:26:33.233 | INFO | "malloc" : " 14 GB", | 20260505 08:26:33.233 | INFO | "pinned" : " 29 GB" | 20260505 08:26:33.233 | INFO | } | 20260505 08:26:33.233 | INFO | Perf report for the last 5.13654 seconds: | 20260505 08:26:33.233 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:26:33.233 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:33.233 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:33.233 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:33.233 | INFO | ANALYSIS : 63.2%, 3.0, 97.7ms, 0 | 20260505 08:26:33.233 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:33.233 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:33.233 | INFO | PRE_HQ : 30.4%, 1.0, 15.6ms, 0 | 20260505 08:26:33.233 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:33.233 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:33.233 | INFO | Uploaded 36GiB to GPU in 2.16149s (16.6552GiB/s) | 20260505 08:26:33.233 | INFO | Downloaded 34GiB to GPU in 2.05752s (16.5248GiB/s) | 20260505 08:26:38.373 | INFO | Analyzing chunk frames = [1763840,1764352) | 20260505 08:26:38.373 | INFO | MemInfo: { | 20260505 08:26:38.373 | INFO | "gpu" : " 17 GB", | 20260505 08:26:38.373 | INFO | "huge" : "", | 20260505 08:26:38.373 | INFO | "huge_shared" : " 27 GB", | 20260505 08:26:38.373 | INFO | "malloc" : " 14 GB", | 20260505 08:26:38.373 | INFO | "pinned" : " 29 GB" | 20260505 08:26:38.373 | INFO | } | 20260505 08:26:38.373 | INFO | Perf report for the last 5.13957 seconds: | 20260505 08:26:38.373 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:26:38.373 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:38.373 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:38.373 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:38.373 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 08:26:38.373 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:38.373 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:38.373 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:26:38.373 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:38.373 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:38.373 | INFO | Uploaded 36GiB to GPU in 2.18619s (16.467GiB/s) | 20260505 08:26:38.373 | INFO | Downloaded 34GiB to GPU in 2.05541s (16.5417GiB/s) | 20260505 08:26:43.511 | INFO | Analyzing chunk frames = [1764352,1764864) | 20260505 08:26:43.511 | INFO | MemInfo: { | 20260505 08:26:43.512 | INFO | "gpu" : " 17 GB", | 20260505 08:26:43.512 | INFO | "huge" : "", | 20260505 08:26:43.512 | INFO | "huge_shared" : " 27 GB", | 20260505 08:26:43.512 | INFO | "malloc" : " 14 GB", | 20260505 08:26:43.512 | INFO | "pinned" : " 29 GB" | 20260505 08:26:43.512 | INFO | } | 20260505 08:26:43.512 | INFO | Perf report for the last 5.13883 seconds: | 20260505 08:26:43.512 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:26:43.512 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:43.512 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:43.512 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:43.512 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 08:26:43.512 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:43.512 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:43.512 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260505 08:26:43.512 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:43.512 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:43.512 | INFO | Uploaded 36GiB to GPU in 2.21997s (16.2165GiB/s) | 20260505 08:26:43.512 | INFO | Downloaded 34GiB to GPU in 2.14379s (15.8598GiB/s) | 20260505 08:26:48.650 | INFO | Analyzing chunk frames = [1764864,1765376) | 20260505 08:26:48.650 | INFO | MemInfo: { | 20260505 08:26:48.650 | INFO | "gpu" : " 17 GB", | 20260505 08:26:48.650 | INFO | "huge" : "", | 20260505 08:26:48.650 | INFO | "huge_shared" : " 27 GB", | 20260505 08:26:48.650 | INFO | "malloc" : " 14 GB", | 20260505 08:26:48.650 | INFO | "pinned" : " 29 GB" | 20260505 08:26:48.650 | INFO | } | 20260505 08:26:48.650 | INFO | Perf report for the last 5.13854 seconds: | 20260505 08:26:48.650 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:26:48.650 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:48.650 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:48.650 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:48.650 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 08:26:48.650 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:48.650 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:48.650 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 08:26:48.650 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:48.650 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:48.650 | INFO | Uploaded 36GiB to GPU in 2.19785s (16.3796GiB/s) | 20260505 08:26:48.651 | INFO | Downloaded 34GiB to GPU in 2.04825s (16.5995GiB/s) | 20260505 08:26:48.790 | INFO | Starting a round of Baz IO | 20260505 08:26:52.054 | INFO | Populating RTMetrics report | 20260505 08:26:52.055 | INFO | RTMetrics report populated | 20260505 08:26:52.055 | INFO | Opening tmp RTMetrics json file | 20260505 08:26:52.141 | INFO | Serializing RTMetrics to json | 20260505 08:26:52.142 | INFO | Serialization done, swapping tmp file now... | 20260505 08:26:52.142 | INFO | RTMetrics done | 20260505 08:26:52.214 | INFO | Finished a round of Baz IO: Wrote 2.448 GiB in 3.424 seconds | 20260505 08:26:52.214 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 27.65% / 0.02% / 72.33% | 20260505 08:26:52.214 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.64613% / 2.38419e-05% / 7.94729e-05% | 20260505 08:26:52.214 | INFO | Baz IO overall: | 20260505 08:26:52.214 | INFO | BytesWritten : 996183042885 | 20260505 08:26:52.214 | INFO | NumEvents : 473523778360 | 20260505 08:26:53.789 | INFO | Analyzing chunk frames = [1765376,1765888) | 20260505 08:26:53.789 | INFO | MemInfo: { | 20260505 08:26:53.789 | INFO | "gpu" : " 17 GB", | 20260505 08:26:53.789 | INFO | "huge" : "", | 20260505 08:26:53.789 | INFO | "huge_shared" : " 27 GB", | 20260505 08:26:53.789 | INFO | "malloc" : " 14 GB", | 20260505 08:26:53.790 | INFO | "pinned" : " 29 GB" | 20260505 08:26:53.790 | INFO | } | 20260505 08:26:53.790 | INFO | Perf report for the last 5.13968 seconds: | 20260505 08:26:53.790 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:26:53.790 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:53.790 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:53.790 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:53.790 | INFO | ANALYSIS : 65.1%, 2.9, 99.6ms, 0 | 20260505 08:26:53.790 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:53.790 | INFO | RT_METRICS : 47.5%, 1.0, 24.9ms, 0 | 20260505 08:26:53.790 | INFO | PRE_HQ : 63.3%, 1.9, 61.3ms, 0 | 20260505 08:26:53.790 | INFO | BAZWRITER : 65.9%, 4.2, 141.1ms, 0 | 20260505 08:26:53.790 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 08:26:53.790 | INFO | Uploaded 36GiB to GPU in 2.2497s (16.0021GiB/s) | 20260505 08:26:53.790 | INFO | Downloaded 38GiB to GPU in 2.38115s (15.9587GiB/s) | 20260505 08:26:58.929 | INFO | Analyzing chunk frames = [1765888,1766400) | 20260505 08:26:58.929 | INFO | MemInfo: { | 20260505 08:26:58.929 | INFO | "gpu" : " 17 GB", | 20260505 08:26:58.929 | INFO | "huge" : "", | 20260505 08:26:58.929 | INFO | "huge_shared" : " 27 GB", | 20260505 08:26:58.929 | INFO | "malloc" : " 14 GB", | 20260505 08:26:58.929 | INFO | "pinned" : " 29 GB" | 20260505 08:26:58.929 | INFO | } | 20260505 08:26:58.929 | INFO | Perf report for the last 5.13917 seconds: | 20260505 08:26:58.929 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:26:58.929 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:58.929 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:58.929 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:58.929 | INFO | ANALYSIS : 64.6%, 3.0, 100.9ms, 0 | 20260505 08:26:58.929 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:58.929 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:58.929 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:26:58.929 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:58.929 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:26:58.929 | INFO | Uploaded 36GiB to GPU in 2.25114s (15.9919GiB/s) | 20260505 08:26:58.929 | INFO | Downloaded 34GiB to GPU in 2.09751s (16.2097GiB/s) | 20260505 08:27:04.070 | INFO | Analyzing chunk frames = [1766400,1766912) | 20260505 08:27:04.070 | INFO | MemInfo: { | 20260505 08:27:04.070 | INFO | "gpu" : " 17 GB", | 20260505 08:27:04.070 | INFO | "huge" : "", | 20260505 08:27:04.070 | INFO | "huge_shared" : " 27 GB", | 20260505 08:27:04.070 | INFO | "malloc" : " 14 GB", | 20260505 08:27:04.070 | INFO | "pinned" : " 29 GB" | 20260505 08:27:04.070 | INFO | } | 20260505 08:27:04.070 | INFO | Perf report for the last 5.14092 seconds: | 20260505 08:27:04.070 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:27:04.070 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:04.070 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:04.070 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:04.070 | INFO | ANALYSIS : 63.7%, 3.0, 99.1ms, 0 | 20260505 08:27:04.070 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:04.070 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:04.070 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:27:04.070 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:04.070 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:04.070 | INFO | Uploaded 36GiB to GPU in 2.17168s (16.577GiB/s) | 20260505 08:27:04.070 | INFO | Downloaded 34GiB to GPU in 2.06668s (16.4515GiB/s) | 20260505 08:27:09.208 | INFO | Analyzing chunk frames = [1766912,1767424) | 20260505 08:27:09.208 | INFO | MemInfo: { | 20260505 08:27:09.208 | INFO | "gpu" : " 17 GB", | 20260505 08:27:09.208 | INFO | "huge" : "", | 20260505 08:27:09.208 | INFO | "huge_shared" : " 27 GB", | 20260505 08:27:09.208 | INFO | "malloc" : " 14 GB", | 20260505 08:27:09.208 | INFO | "pinned" : " 29 GB" | 20260505 08:27:09.208 | INFO | } | 20260505 08:27:09.208 | INFO | Perf report for the last 5.13824 seconds: | 20260505 08:27:09.208 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:27:09.208 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:09.208 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:09.208 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:09.208 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260505 08:27:09.208 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:09.208 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:09.208 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:27:09.208 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:09.208 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:09.209 | INFO | Uploaded 36GiB to GPU in 2.17844s (16.5256GiB/s) | 20260505 08:27:09.209 | INFO | Downloaded 34GiB to GPU in 2.06401s (16.4728GiB/s) | 20260505 08:27:14.347 | INFO | Analyzing chunk frames = [1767424,1767936) | 20260505 08:27:14.347 | INFO | MemInfo: { | 20260505 08:27:14.347 | INFO | "gpu" : " 17 GB", | 20260505 08:27:14.347 | INFO | "huge" : "", | 20260505 08:27:14.347 | INFO | "huge_shared" : " 27 GB", | 20260505 08:27:14.347 | INFO | "malloc" : " 14 GB", | 20260505 08:27:14.347 | INFO | "pinned" : " 29 GB" | 20260505 08:27:14.347 | INFO | } | 20260505 08:27:14.347 | INFO | Perf report for the last 5.13879 seconds: | 20260505 08:27:14.347 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:27:14.347 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:14.347 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:14.347 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:14.347 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 08:27:14.347 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:14.347 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:14.347 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:27:14.347 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:14.347 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:14.347 | INFO | Uploaded 36GiB to GPU in 2.1709s (16.583GiB/s) | 20260505 08:27:14.347 | INFO | Downloaded 34GiB to GPU in 2.05959s (16.5082GiB/s) | 20260505 08:27:19.487 | INFO | Analyzing chunk frames = [1767936,1768448) | 20260505 08:27:19.487 | INFO | MemInfo: { | 20260505 08:27:19.487 | INFO | "gpu" : " 17 GB", | 20260505 08:27:19.487 | INFO | "huge" : "", | 20260505 08:27:19.487 | INFO | "huge_shared" : " 27 GB", | 20260505 08:27:19.487 | INFO | "malloc" : " 14 GB", | 20260505 08:27:19.488 | INFO | "pinned" : " 29 GB" | 20260505 08:27:19.488 | INFO | } | 20260505 08:27:19.488 | INFO | Perf report for the last 5.14063 seconds: | 20260505 08:27:19.488 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:27:19.488 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:19.488 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:19.488 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:19.488 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 08:27:19.488 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:19.488 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:19.488 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:27:19.488 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:19.488 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:19.488 | INFO | Uploaded 36GiB to GPU in 2.18329s (16.4889GiB/s) | 20260505 08:27:19.488 | INFO | Downloaded 34GiB to GPU in 2.05344s (16.5576GiB/s) | 20260505 08:27:24.626 | INFO | Analyzing chunk frames = [1768448,1768960) | 20260505 08:27:24.626 | INFO | MemInfo: { | 20260505 08:27:24.626 | INFO | "gpu" : " 17 GB", | 20260505 08:27:24.626 | INFO | "huge" : "", | 20260505 08:27:24.626 | INFO | "huge_shared" : " 27 GB", | 20260505 08:27:24.626 | INFO | "malloc" : " 14 GB", | 20260505 08:27:24.627 | INFO | "pinned" : " 29 GB" | 20260505 08:27:24.627 | INFO | } | 20260505 08:27:24.627 | INFO | Perf report for the last 5.13885 seconds: | 20260505 08:27:24.627 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:27:24.627 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:24.627 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:24.627 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:24.627 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 08:27:24.627 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:24.627 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:24.627 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 08:27:24.627 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:24.627 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:24.627 | INFO | Uploaded 36GiB to GPU in 2.21428s (16.2581GiB/s) | 20260505 08:27:24.627 | INFO | Downloaded 34GiB to GPU in 2.15707s (15.7621GiB/s) | 20260505 08:27:29.765 | INFO | Analyzing chunk frames = [1768960,1769472) | 20260505 08:27:29.765 | INFO | MemInfo: { | 20260505 08:27:29.765 | INFO | "gpu" : " 17 GB", | 20260505 08:27:29.765 | INFO | "huge" : "", | 20260505 08:27:29.765 | INFO | "huge_shared" : " 27 GB", | 20260505 08:27:29.765 | INFO | "malloc" : " 14 GB", | 20260505 08:27:29.765 | INFO | "pinned" : " 29 GB" | 20260505 08:27:29.765 | INFO | } | 20260505 08:27:29.765 | INFO | Perf report for the last 5.13842 seconds: | 20260505 08:27:29.765 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:27:29.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:29.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:29.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:29.765 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 08:27:29.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:29.765 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:29.765 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 08:27:29.765 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:29.765 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:29.765 | INFO | Uploaded 36GiB to GPU in 2.19779s (16.3801GiB/s) | 20260505 08:27:29.765 | INFO | Downloaded 34GiB to GPU in 2.03889s (16.6757GiB/s) | 20260505 08:27:29.765 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 08:27:29.765 | INFO | Total mem usage: 15.3 GiB | 20260505 08:27:29.766 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 08:27:29.766 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.35%) | 20260505 08:27:29.766 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 08:27:29.766 | INFO | High Water Breakdown: | 20260505 08:27:29.766 | INFO | 60.8% : 8.77 GiB ( 93.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 08:27:29.766 | INFO | 39% : 5.63 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 08:27:29.766 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 08:27:29.766 | INFO | ------------------------------------------ | 20260505 08:27:29.766 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:27:29.766 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 08:27:29.766 | INFO | Total mem usage: 26.5 GiB | 20260505 08:27:29.766 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 08:27:29.766 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 08:27:29.766 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 08:27:29.766 | INFO | High Water Breakdown: | 20260505 08:27:29.766 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:27:29.766 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:27:29.766 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:27:29.766 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:27:29.766 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:27:29.766 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:27:29.766 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 08:27:29.766 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 08:27:29.766 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:27:29.766 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 08:27:29.766 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 08:27:29.766 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 08:27:29.766 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 08:27:29.766 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 08:27:29.766 | INFO | ------------------------------------------ | 20260505 08:27:29.766 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:27:29.766 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 08:27:29.766 | INFO | Total mem usage: 25.6 GiB | 20260505 08:27:29.766 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 08:27:29.766 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 08:27:29.766 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 08:27:29.766 | INFO | High Water Breakdown: | 20260505 08:27:29.766 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 08:27:29.766 | INFO | ------------------------------------------ | 20260505 08:27:29.766 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:27:29.766 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 08:27:29.766 | INFO | Total mem usage: 15.7 GiB | 20260505 08:27:29.767 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 08:27:29.767 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 08:27:29.767 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 08:27:29.767 | INFO | High Water Breakdown: | 20260505 08:27:29.767 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 08:27:29.767 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 08:27:29.767 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 08:27:29.768 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 08:27:29.768 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 08:27:29.768 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 08:27:29.768 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 08:27:29.768 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 08:27:29.768 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 08:27:29.768 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:27:29.768 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 08:27:29.768 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:27:29.768 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 08:27:29.768 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 08:27:29.768 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 08:27:29.768 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 08:27:29.768 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:27:29.768 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 08:27:29.768 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:27:29.768 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:27:29.768 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:27:29.768 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 08:27:29.768 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 08:27:29.768 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:27:29.768 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 08:27:29.768 | INFO | ------------------------------------------ | 20260505 08:27:29.768 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:27:29.768 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 08:27:29.768 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:27:29.768 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:27:29.768 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.51e+01, 1.37e+01 / 1.57e+00 | 20260505 08:27:29.768 | INFO | PerfReport: 0.29%, AggregateStats -- 9.33e-02 / 1.04e-01, 1.06e-02 / 4.32e-03 | 20260505 08:27:29.768 | INFO | PerfReport: 11.64%, Baseline -- 3.80e+00 / 4.44e+00, 6.24e-01 / 3.08e-01 | 20260505 08:27:29.768 | INFO | PerfReport: 3.32%, Histogram -- 1.08e+00 / 1.48e+00, 2.12e-01 / 1.05e-01 | 20260505 08:27:29.768 | INFO | PerfReport: 15.49%, DME -- 4.04e+01 / 4.40e+01, 3.94e+00 / 1.26e+00 | 20260505 08:27:29.768 | INFO | PerfReport: 56.47%, FrameLabeling -- 1.84e+01 / 1.99e+01, 1.94e+00 / 1.74e+00 | 20260505 08:27:29.768 | INFO | PerfReport: 7.49%, PulseAccumulating -- 2.44e+00 / 3.02e+00, 4.14e-01 / 1.98e-01 | 20260505 08:27:29.768 | INFO | PerfReport: 3.91%, Metrics -- 1.27e+00 / 1.60e+00, 3.63e-01 / 1.83e-01 | 20260505 08:27:29.768 | INFO | PerfReport: 1.40%, PulseSerializing -- 4.55e-01 / 5.11e-01, 8.64e-02 / 3.03e-02 | 20260505 08:27:29.768 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 08:27:29.768 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:27:29.768 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:27:29.768 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.30e+01 / 0.00e+00 | 20260505 08:27:29.768 | INFO | PerfReport: 22.55%, Upload -- 2.29e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260505 08:27:29.768 | INFO | PerfReport: 21.66%, Download -- 2.20e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260505 08:27:29.768 | INFO | PerfReport: 32.08%, Compute -- 3.26e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 08:27:29.768 | INFO | PerfReport: 1.69%, LockUpload -- 1.72e+00 / 0.00e+00, 7.43e+00 / 0.00e+00 | 20260505 08:27:29.768 | INFO | PerfReport: 1.06%, LockDownload -- 1.08e+00 / 0.00e+00, 2.61e+00 / 0.00e+00 | 20260505 08:27:29.768 | INFO | PerfReport: 20.95%, LockCompute -- 2.13e+01 / 0.00e+00, 1.77e+01 / 0.00e+00 | 20260505 08:27:29.768 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.58e-02 / 0.00e+00 | 20260505 08:27:29.905 | INFO | Starting a round of Baz IO | 20260505 08:27:33.188 | INFO | Populating RTMetrics report | 20260505 08:27:33.188 | INFO | RTMetrics report populated | 20260505 08:27:33.188 | INFO | Opening tmp RTMetrics json file | 20260505 08:27:33.201 | INFO | Serializing RTMetrics to json | 20260505 08:27:33.251 | INFO | Serialization done, swapping tmp file now... | 20260505 08:27:33.253 | INFO | RTMetrics done | 20260505 08:27:33.313 | INFO | Finished a round of Baz IO: Wrote 2.45 GiB in 3.407 seconds | 20260505 08:27:33.313 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 27.63% / 0.02% / 72.35% | 20260505 08:27:33.313 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.64763% / 5.16574e-05% / 5.96046e-05% | 20260505 08:27:33.313 | INFO | Baz IO overall: | 20260505 08:27:33.313 | INFO | BytesWritten : 998813426820 | 20260505 08:27:33.313 | INFO | NumEvents : 474781829522 | 20260505 08:27:34.904 | INFO | Analyzing chunk frames = [1769472,1769984) | 20260505 08:27:34.904 | INFO | MemInfo: { | 20260505 08:27:34.904 | INFO | "gpu" : " 17 GB", | 20260505 08:27:34.904 | INFO | "huge" : "", | 20260505 08:27:34.904 | INFO | "huge_shared" : " 27 GB", | 20260505 08:27:34.904 | INFO | "malloc" : " 14 GB", | 20260505 08:27:34.904 | INFO | "pinned" : " 29 GB" | 20260505 08:27:34.904 | INFO | } | 20260505 08:27:34.904 | INFO | Perf report for the last 5.13942 seconds: | 20260505 08:27:34.905 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:27:34.905 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:34.905 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:34.905 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:34.905 | INFO | ANALYSIS : 65.5%, 2.9, 100.2ms, 0 | 20260505 08:27:34.905 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:34.905 | INFO | RT_METRICS : 47.0%, 1.0, 24.6ms, 0 | 20260505 08:27:34.905 | INFO | PRE_HQ : 63.4%, 1.9, 61.1ms, 0 | 20260505 08:27:34.905 | INFO | BAZWRITER : 65.4%, 4.2, 139.6ms, 0 | 20260505 08:27:34.905 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 08:27:34.905 | INFO | Uploaded 36GiB to GPU in 2.28647s (15.7448GiB/s) | 20260505 08:27:34.905 | INFO | Downloaded 38GiB to GPU in 2.37263s (16.016GiB/s) | 20260505 08:27:40.047 | INFO | Analyzing chunk frames = [1769984,1770496) | 20260505 08:27:40.047 | INFO | MemInfo: { | 20260505 08:27:40.047 | INFO | "gpu" : " 17 GB", | 20260505 08:27:40.047 | INFO | "huge" : "", | 20260505 08:27:40.047 | INFO | "huge_shared" : " 27 GB", | 20260505 08:27:40.047 | INFO | "malloc" : " 14 GB", | 20260505 08:27:40.047 | INFO | "pinned" : " 29 GB" | 20260505 08:27:40.047 | INFO | } | 20260505 08:27:40.047 | INFO | Perf report for the last 5.14244 seconds: | 20260505 08:27:40.047 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:27:40.047 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:40.047 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:40.047 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:40.047 | INFO | ANALYSIS : 64.6%, 3.0, 101.0ms, 0 | 20260505 08:27:40.047 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:40.047 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:40.047 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:27:40.047 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:40.047 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:40.047 | INFO | Uploaded 36GiB to GPU in 2.24766s (16.0167GiB/s) | 20260505 08:27:40.047 | INFO | Downloaded 34GiB to GPU in 2.09931s (16.1958GiB/s) | 20260505 08:27:45.183 | INFO | Analyzing chunk frames = [1770496,1771008) | 20260505 08:27:45.183 | INFO | MemInfo: { | 20260505 08:27:45.183 | INFO | "gpu" : " 17 GB", | 20260505 08:27:45.183 | INFO | "huge" : "", | 20260505 08:27:45.183 | INFO | "huge_shared" : " 27 GB", | 20260505 08:27:45.183 | INFO | "malloc" : " 14 GB", | 20260505 08:27:45.183 | INFO | "pinned" : " 29 GB" | 20260505 08:27:45.183 | INFO | } | 20260505 08:27:45.183 | INFO | Perf report for the last 5.13627 seconds: | 20260505 08:27:45.183 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:27:45.183 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:45.183 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:45.183 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:45.183 | INFO | ANALYSIS : 63.7%, 3.0, 99.0ms, 0 | 20260505 08:27:45.183 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:45.183 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:45.183 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:27:45.183 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:45.183 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:45.183 | INFO | Uploaded 36GiB to GPU in 2.1725s (16.5708GiB/s) | 20260505 08:27:45.184 | INFO | Downloaded 34GiB to GPU in 2.06741s (16.4457GiB/s) | 20260505 08:27:50.323 | INFO | Analyzing chunk frames = [1771008,1771520) | 20260505 08:27:50.323 | INFO | MemInfo: { | 20260505 08:27:50.323 | INFO | "gpu" : " 17 GB", | 20260505 08:27:50.323 | INFO | "huge" : "", | 20260505 08:27:50.323 | INFO | "huge_shared" : " 27 GB", | 20260505 08:27:50.323 | INFO | "malloc" : " 14 GB", | 20260505 08:27:50.323 | INFO | "pinned" : " 29 GB" | 20260505 08:27:50.323 | INFO | } | 20260505 08:27:50.323 | INFO | Perf report for the last 5.1397 seconds: | 20260505 08:27:50.323 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:27:50.323 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:50.323 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:50.323 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:50.323 | INFO | ANALYSIS : 63.5%, 3.0, 98.6ms, 0 | 20260505 08:27:50.323 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:50.323 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:50.323 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:27:50.323 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:50.323 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:50.323 | INFO | Uploaded 36GiB to GPU in 2.17506s (16.5513GiB/s) | 20260505 08:27:50.323 | INFO | Downloaded 34GiB to GPU in 2.06545s (16.4613GiB/s) | 20260505 08:27:55.461 | INFO | Analyzing chunk frames = [1771520,1772032) | 20260505 08:27:55.461 | INFO | MemInfo: { | 20260505 08:27:55.461 | INFO | "gpu" : " 17 GB", | 20260505 08:27:55.461 | INFO | "huge" : "", | 20260505 08:27:55.461 | INFO | "huge_shared" : " 27 GB", | 20260505 08:27:55.462 | INFO | "malloc" : " 14 GB", | 20260505 08:27:55.462 | INFO | "pinned" : " 29 GB" | 20260505 08:27:55.462 | INFO | } | 20260505 08:27:55.462 | INFO | Perf report for the last 5.13883 seconds: | 20260505 08:27:55.462 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:27:55.462 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:55.462 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:55.462 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:55.462 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 08:27:55.462 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:55.462 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:55.462 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:27:55.462 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:55.462 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:27:55.462 | INFO | Uploaded 36GiB to GPU in 2.18831s (16.4511GiB/s) | 20260505 08:27:55.462 | INFO | Downloaded 34GiB to GPU in 2.05674s (16.531GiB/s) | 20260505 08:28:00.602 | INFO | Analyzing chunk frames = [1772032,1772544) | 20260505 08:28:00.602 | INFO | MemInfo: { | 20260505 08:28:00.602 | INFO | "gpu" : " 17 GB", | 20260505 08:28:00.602 | INFO | "huge" : "", | 20260505 08:28:00.602 | INFO | "huge_shared" : " 27 GB", | 20260505 08:28:00.602 | INFO | "malloc" : " 14 GB", | 20260505 08:28:00.602 | INFO | "pinned" : " 29 GB" | 20260505 08:28:00.602 | INFO | } | 20260505 08:28:00.602 | INFO | Perf report for the last 5.14065 seconds: | 20260505 08:28:00.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:28:00.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:00.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:00.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:00.602 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 08:28:00.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:00.602 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:00.603 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:28:00.603 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:00.603 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:00.603 | INFO | Uploaded 36GiB to GPU in 2.18575s (16.4703GiB/s) | 20260505 08:28:00.603 | INFO | Downloaded 34GiB to GPU in 2.05443s (16.5496GiB/s) | 20260505 08:28:05.740 | INFO | Analyzing chunk frames = [1772544,1773056) | 20260505 08:28:05.740 | INFO | MemInfo: { | 20260505 08:28:05.740 | INFO | "gpu" : " 17 GB", | 20260505 08:28:05.740 | INFO | "huge" : "", | 20260505 08:28:05.740 | INFO | "huge_shared" : " 27 GB", | 20260505 08:28:05.740 | INFO | "malloc" : " 14 GB", | 20260505 08:28:05.740 | INFO | "pinned" : " 29 GB" | 20260505 08:28:05.740 | INFO | } | 20260505 08:28:05.740 | INFO | Perf report for the last 5.13791 seconds: | 20260505 08:28:05.740 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:28:05.740 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:05.740 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:05.740 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:05.740 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 08:28:05.740 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:05.740 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:05.740 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 08:28:05.741 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:05.741 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:05.741 | INFO | Uploaded 36GiB to GPU in 2.19971s (16.3658GiB/s) | 20260505 08:28:05.741 | INFO | Downloaded 34GiB to GPU in 2.15224s (15.7975GiB/s) | 20260505 08:28:10.881 | INFO | Analyzing chunk frames = [1773056,1773568) | 20260505 08:28:10.881 | INFO | MemInfo: { | 20260505 08:28:10.881 | INFO | "gpu" : " 17 GB", | 20260505 08:28:10.881 | INFO | "huge" : "", | 20260505 08:28:10.881 | INFO | "huge_shared" : " 27 GB", | 20260505 08:28:10.881 | INFO | "malloc" : " 14 GB", | 20260505 08:28:10.881 | INFO | "pinned" : " 29 GB" | 20260505 08:28:10.881 | INFO | } | 20260505 08:28:10.881 | INFO | Perf report for the last 5.14079 seconds: | 20260505 08:28:10.881 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:28:10.881 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:10.881 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:10.881 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:10.881 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 08:28:10.881 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:10.881 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:10.881 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 08:28:10.881 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:10.881 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:10.881 | INFO | Uploaded 36GiB to GPU in 2.19487s (16.4019GiB/s) | 20260505 08:28:10.881 | INFO | Downloaded 34GiB to GPU in 2.04698s (16.6098GiB/s) | 20260505 08:28:11.022 | INFO | Starting a round of Baz IO | 20260505 08:28:14.307 | INFO | Populating RTMetrics report | 20260505 08:28:14.308 | INFO | RTMetrics report populated | 20260505 08:28:14.308 | INFO | Opening tmp RTMetrics json file | 20260505 08:28:14.402 | INFO | Serializing RTMetrics to json | 20260505 08:28:14.436 | INFO | Serialization done, swapping tmp file now... | 20260505 08:28:14.436 | INFO | RTMetrics done | 20260505 08:28:14.453 | INFO | Finished a round of Baz IO: Wrote 2.451 GiB in 3.431 seconds | 20260505 08:28:14.453 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 27.61% / 0.02% / 72.37% | 20260505 08:28:14.453 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.64223% / 1.98682e-05% / 8.34465e-05% | 20260505 08:28:14.453 | INFO | Baz IO overall: | 20260505 08:28:14.453 | INFO | BytesWritten : 1001444912106 | 20260505 08:28:14.453 | INFO | NumEvents : 476040830063 | 20260505 08:28:16.020 | INFO | Analyzing chunk frames = [1773568,1774080) | 20260505 08:28:16.020 | INFO | MemInfo: { | 20260505 08:28:16.020 | INFO | "gpu" : " 17 GB", | 20260505 08:28:16.020 | INFO | "huge" : "", | 20260505 08:28:16.020 | INFO | "huge_shared" : " 27 GB", | 20260505 08:28:16.020 | INFO | "malloc" : " 14 GB", | 20260505 08:28:16.020 | INFO | "pinned" : " 29 GB" | 20260505 08:28:16.020 | INFO | } | 20260505 08:28:16.020 | INFO | Perf report for the last 5.13873 seconds: | 20260505 08:28:16.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:28:16.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:16.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:16.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:16.020 | INFO | ANALYSIS : 65.5%, 2.9, 100.1ms, 0 | 20260505 08:28:16.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:16.020 | INFO | RT_METRICS : 48.3%, 1.0, 25.3ms, 0 | 20260505 08:28:16.020 | INFO | PRE_HQ : 63.2%, 1.9, 61.5ms, 0 | 20260505 08:28:16.020 | INFO | BAZWRITER : 66.1%, 4.1, 140.0ms, 0 | 20260505 08:28:16.020 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260505 08:28:16.020 | INFO | Uploaded 36GiB to GPU in 2.23036s (16.1409GiB/s) | 20260505 08:28:16.020 | INFO | Downloaded 38GiB to GPU in 2.35023s (16.1686GiB/s) | 20260505 08:28:21.159 | INFO | Analyzing chunk frames = [1774080,1774592) | 20260505 08:28:21.159 | INFO | MemInfo: { | 20260505 08:28:21.159 | INFO | "gpu" : " 17 GB", | 20260505 08:28:21.159 | INFO | "huge" : "", | 20260505 08:28:21.159 | INFO | "huge_shared" : " 27 GB", | 20260505 08:28:21.159 | INFO | "malloc" : " 14 GB", | 20260505 08:28:21.159 | INFO | "pinned" : " 29 GB" | 20260505 08:28:21.159 | INFO | } | 20260505 08:28:21.159 | INFO | Perf report for the last 5.13941 seconds: | 20260505 08:28:21.159 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:28:21.159 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:21.159 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:21.159 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:21.159 | INFO | ANALYSIS : 64.6%, 3.0, 100.8ms, 0 | 20260505 08:28:21.159 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:21.159 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:21.159 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:28:21.159 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:21.159 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:21.160 | INFO | Uploaded 36GiB to GPU in 2.25227s (15.9839GiB/s) | 20260505 08:28:21.160 | INFO | Downloaded 34GiB to GPU in 2.09714s (16.2125GiB/s) | 20260505 08:28:26.298 | INFO | Analyzing chunk frames = [1774592,1775104) | 20260505 08:28:26.298 | INFO | MemInfo: { | 20260505 08:28:26.298 | INFO | "gpu" : " 17 GB", | 20260505 08:28:26.298 | INFO | "huge" : "", | 20260505 08:28:26.298 | INFO | "huge_shared" : " 27 GB", | 20260505 08:28:26.298 | INFO | "malloc" : " 14 GB", | 20260505 08:28:26.298 | INFO | "pinned" : " 29 GB" | 20260505 08:28:26.298 | INFO | } | 20260505 08:28:26.298 | INFO | Perf report for the last 5.13892 seconds: | 20260505 08:28:26.298 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:28:26.298 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:26.298 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:26.298 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:26.298 | INFO | ANALYSIS : 63.7%, 3.0, 99.1ms, 0 | 20260505 08:28:26.298 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:26.298 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:26.298 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:28:26.298 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:26.298 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:26.299 | INFO | Uploaded 36GiB to GPU in 2.17331s (16.5646GiB/s) | 20260505 08:28:26.299 | INFO | Downloaded 34GiB to GPU in 2.06633s (16.4543GiB/s) | 20260505 08:28:31.437 | INFO | Analyzing chunk frames = [1775104,1775616) | 20260505 08:28:31.437 | INFO | MemInfo: { | 20260505 08:28:31.437 | INFO | "gpu" : " 17 GB", | 20260505 08:28:31.437 | INFO | "huge" : "", | 20260505 08:28:31.437 | INFO | "huge_shared" : " 27 GB", | 20260505 08:28:31.437 | INFO | "malloc" : " 14 GB", | 20260505 08:28:31.437 | INFO | "pinned" : " 29 GB" | 20260505 08:28:31.437 | INFO | } | 20260505 08:28:31.437 | INFO | Perf report for the last 5.13893 seconds: | 20260505 08:28:31.437 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:28:31.437 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:31.437 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:31.437 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:31.437 | INFO | ANALYSIS : 63.6%, 3.0, 98.8ms, 0 | 20260505 08:28:31.437 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:31.437 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:31.437 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:28:31.438 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:31.438 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:31.438 | INFO | Uploaded 36GiB to GPU in 2.18845s (16.45GiB/s) | 20260505 08:28:31.438 | INFO | Downloaded 34GiB to GPU in 2.06275s (16.4829GiB/s) | 20260505 08:28:36.577 | INFO | Analyzing chunk frames = [1775616,1776128) | 20260505 08:28:36.577 | INFO | MemInfo: { | 20260505 08:28:36.577 | INFO | "gpu" : " 17 GB", | 20260505 08:28:36.577 | INFO | "huge" : "", | 20260505 08:28:36.577 | INFO | "huge_shared" : " 27 GB", | 20260505 08:28:36.577 | INFO | "malloc" : " 14 GB", | 20260505 08:28:36.577 | INFO | "pinned" : " 29 GB" | 20260505 08:28:36.577 | INFO | } | 20260505 08:28:36.577 | INFO | Perf report for the last 5.13967 seconds: | 20260505 08:28:36.577 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:28:36.577 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:36.577 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:36.577 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:36.577 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 08:28:36.577 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:36.577 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:36.577 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 08:28:36.577 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:36.577 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:36.577 | INFO | Uploaded 36GiB to GPU in 2.19068s (16.4332GiB/s) | 20260505 08:28:36.577 | INFO | Downloaded 34GiB to GPU in 2.05822s (16.5192GiB/s) | 20260505 08:28:41.717 | INFO | Analyzing chunk frames = [1776128,1776640) | 20260505 08:28:41.717 | INFO | MemInfo: { | 20260505 08:28:41.717 | INFO | "gpu" : " 17 GB", | 20260505 08:28:41.717 | INFO | "huge" : "", | 20260505 08:28:41.717 | INFO | "huge_shared" : " 27 GB", | 20260505 08:28:41.717 | INFO | "malloc" : " 14 GB", | 20260505 08:28:41.718 | INFO | "pinned" : " 29 GB" | 20260505 08:28:41.718 | INFO | } | 20260505 08:28:41.718 | INFO | Perf report for the last 5.1408 seconds: | 20260505 08:28:41.718 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:28:41.718 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:41.718 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:41.718 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:41.718 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 08:28:41.718 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:41.718 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:41.718 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 08:28:41.718 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:41.718 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:41.718 | INFO | Uploaded 36GiB to GPU in 2.19056s (16.4341GiB/s) | 20260505 08:28:41.718 | INFO | Downloaded 34GiB to GPU in 2.05621s (16.5352GiB/s) | 20260505 08:28:46.855 | INFO | Analyzing chunk frames = [1776640,1777152) | 20260505 08:28:46.855 | INFO | MemInfo: { | 20260505 08:28:46.855 | INFO | "gpu" : " 17 GB", | 20260505 08:28:46.855 | INFO | "huge" : "", | 20260505 08:28:46.855 | INFO | "huge_shared" : " 27 GB", | 20260505 08:28:46.856 | INFO | "malloc" : " 14 GB", | 20260505 08:28:46.856 | INFO | "pinned" : " 29 GB" | 20260505 08:28:46.856 | INFO | } | 20260505 08:28:46.856 | INFO | Perf report for the last 5.13813 seconds: | 20260505 08:28:46.856 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:28:46.856 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:46.856 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:46.856 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:46.856 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 08:28:46.856 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:46.856 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:46.856 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 08:28:46.856 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:46.856 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:46.856 | INFO | Uploaded 36GiB to GPU in 2.21432s (16.2578GiB/s) | 20260505 08:28:46.856 | INFO | Downloaded 34GiB to GPU in 2.13969s (15.8902GiB/s) | 20260505 08:28:51.995 | INFO | Analyzing chunk frames = [1777152,1777664) | 20260505 08:28:51.995 | INFO | MemInfo: { | 20260505 08:28:51.995 | INFO | "gpu" : " 17 GB", | 20260505 08:28:51.995 | INFO | "huge" : "", | 20260505 08:28:51.995 | INFO | "huge_shared" : " 27 GB", | 20260505 08:28:51.995 | INFO | "malloc" : " 14 GB", | 20260505 08:28:51.995 | INFO | "pinned" : " 29 GB" | 20260505 08:28:51.996 | INFO | } | 20260505 08:28:51.996 | INFO | Perf report for the last 5.13988 seconds: | 20260505 08:28:51.996 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:28:51.996 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:51.996 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:51.996 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:51.996 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 08:28:51.996 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:51.996 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:51.996 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:28:51.996 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:51.996 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:51.996 | INFO | Uploaded 36GiB to GPU in 2.19567s (16.3959GiB/s) | 20260505 08:28:51.996 | INFO | Downloaded 34GiB to GPU in 2.0357s (16.7019GiB/s) | 20260505 08:28:51.996 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 08:28:51.996 | INFO | Total mem usage: 15.3 GiB | 20260505 08:28:51.996 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 08:28:51.996 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.36%) | 20260505 08:28:51.996 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 08:28:51.996 | INFO | High Water Breakdown: | 20260505 08:28:51.996 | INFO | 60.8% : 8.77 GiB ( 93.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 08:28:51.996 | INFO | 39% : 5.62 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 08:28:51.996 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 08:28:51.996 | INFO | ------------------------------------------ | 20260505 08:28:51.996 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:28:51.996 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 08:28:51.996 | INFO | Total mem usage: 26.5 GiB | 20260505 08:28:51.997 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 08:28:51.997 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 08:28:51.997 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 08:28:51.997 | INFO | High Water Breakdown: | 20260505 08:28:51.997 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:28:51.997 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:28:51.997 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:28:51.997 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:28:51.997 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:28:51.997 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:28:51.997 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 08:28:51.997 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 08:28:51.997 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:28:51.997 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 08:28:51.997 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 08:28:51.997 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 08:28:51.997 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 08:28:51.997 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 08:28:51.997 | INFO | ------------------------------------------ | 20260505 08:28:51.997 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:28:51.997 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 08:28:51.997 | INFO | Total mem usage: 25.6 GiB | 20260505 08:28:51.997 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 08:28:51.997 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 08:28:51.997 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 08:28:51.997 | INFO | High Water Breakdown: | 20260505 08:28:51.997 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 08:28:51.997 | INFO | ------------------------------------------ | 20260505 08:28:51.997 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:28:51.997 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 08:28:51.997 | INFO | Total mem usage: 15.7 GiB | 20260505 08:28:51.997 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 08:28:51.997 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 08:28:51.997 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 08:28:51.997 | INFO | High Water Breakdown: | 20260505 08:28:51.997 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 08:28:51.997 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 08:28:51.997 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 08:28:51.997 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 08:28:51.997 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 08:28:51.997 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 08:28:51.997 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 08:28:51.997 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 08:28:51.997 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 08:28:51.997 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:28:51.997 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 08:28:51.997 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:28:51.997 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 08:28:51.997 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 08:28:51.997 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 08:28:51.997 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 08:28:51.997 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:28:51.997 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 08:28:51.997 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:28:51.997 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:28:51.997 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:28:51.997 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 08:28:51.997 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 08:28:51.997 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:28:51.997 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 08:28:51.997 | INFO | ------------------------------------------ | 20260505 08:28:51.997 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:28:51.997 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 08:28:51.997 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:28:51.997 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:28:51.997 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.56e+01, 1.37e+01 / 2.01e+00 | 20260505 08:28:51.997 | INFO | PerfReport: 0.29%, AggregateStats -- 9.31e-02 / 1.03e-01, 1.02e-02 / 5.13e-03 | 20260505 08:28:51.997 | INFO | PerfReport: 11.66%, Baseline -- 3.80e+00 / 4.46e+00, 6.26e-01 / 2.78e-01 | 20260505 08:28:51.997 | INFO | PerfReport: 3.32%, Histogram -- 1.08e+00 / 1.49e+00, 2.12e-01 / 1.05e-01 | 20260505 08:28:51.997 | INFO | PerfReport: 15.56%, DME -- 4.06e+01 / 4.45e+01, 4.21e+00 / 1.78e+00 | 20260505 08:28:51.997 | INFO | PerfReport: 56.45%, FrameLabeling -- 1.84e+01 / 1.99e+01, 1.92e+00 / 1.84e+00 | 20260505 08:28:51.997 | INFO | PerfReport: 7.44%, PulseAccumulating -- 2.43e+00 / 3.02e+00, 4.03e-01 / 2.35e-01 | 20260505 08:28:51.997 | INFO | PerfReport: 3.89%, Metrics -- 1.27e+00 / 1.60e+00, 3.59e-01 / 1.76e-01 | 20260505 08:28:51.998 | INFO | PerfReport: 1.40%, PulseSerializing -- 4.56e-01 / 5.10e-01, 8.71e-02 / 2.95e-02 | 20260505 08:28:51.998 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 08:28:51.998 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:28:51.998 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:28:51.998 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.31e+01 / 0.00e+00 | 20260505 08:28:51.998 | INFO | PerfReport: 22.58%, Upload -- 2.30e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260505 08:28:51.998 | INFO | PerfReport: 21.62%, Download -- 2.20e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260505 08:28:51.998 | INFO | PerfReport: 32.09%, Compute -- 3.26e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 08:28:51.998 | INFO | PerfReport: 1.67%, LockUpload -- 1.70e+00 / 0.00e+00, 7.31e+00 / 0.00e+00 | 20260505 08:28:51.998 | INFO | PerfReport: 1.00%, LockDownload -- 1.02e+00 / 0.00e+00, 2.46e+00 / 0.00e+00 | 20260505 08:28:51.998 | INFO | PerfReport: 21.02%, LockCompute -- 2.14e+01 / 0.00e+00, 1.78e+01 / 0.00e+00 | 20260505 08:28:51.998 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.65e-02 / 0.00e+00 | 20260505 08:28:52.136 | INFO | Starting a round of Baz IO | 20260505 08:28:55.425 | INFO | Populating RTMetrics report | 20260505 08:28:55.425 | INFO | RTMetrics report populated | 20260505 08:28:55.425 | INFO | Opening tmp RTMetrics json file | 20260505 08:28:55.426 | INFO | Serializing RTMetrics to json | 20260505 08:28:55.427 | INFO | Serialization done, swapping tmp file now... | 20260505 08:28:55.428 | INFO | RTMetrics done | 20260505 08:28:55.617 | INFO | Finished a round of Baz IO: Wrote 2.451 GiB in 3.481 seconds | 20260505 08:28:55.617 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 27.59% / 0.02% / 72.39% | 20260505 08:28:55.617 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.63539% / 2.78155e-05% / 8.74201e-05% | 20260505 08:28:55.617 | INFO | Baz IO overall: | 20260505 08:28:55.618 | INFO | BytesWritten : 1004076205822 | 20260505 08:28:55.618 | INFO | NumEvents : 477300157751 | 20260505 08:28:57.134 | INFO | Analyzing chunk frames = [1777664,1778176) | 20260505 08:28:57.134 | INFO | MemInfo: { | 20260505 08:28:57.134 | INFO | "gpu" : " 17 GB", | 20260505 08:28:57.134 | INFO | "huge" : "", | 20260505 08:28:57.134 | INFO | "huge_shared" : " 27 GB", | 20260505 08:28:57.134 | INFO | "malloc" : " 14 GB", | 20260505 08:28:57.134 | INFO | "pinned" : " 29 GB" | 20260505 08:28:57.134 | INFO | } | 20260505 08:28:57.134 | INFO | Perf report for the last 5.13857 seconds: | 20260505 08:28:57.134 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:28:57.134 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:57.134 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:57.134 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:57.134 | INFO | ANALYSIS : 65.6%, 2.9, 100.3ms, 0 | 20260505 08:28:57.135 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:28:57.135 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260505 08:28:57.135 | INFO | PRE_HQ : 62.8%, 1.9, 61.0ms, 0 | 20260505 08:28:57.135 | INFO | BAZWRITER : 67.1%, 4.2, 143.8ms, 0 | 20260505 08:28:57.135 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 08:28:57.135 | INFO | Uploaded 36GiB to GPU in 2.24929s (16.005GiB/s) | 20260505 08:28:57.135 | INFO | Downloaded 38GiB to GPU in 2.33536s (16.2716GiB/s) | 20260505 08:29:02.273 | INFO | Analyzing chunk frames = [1778176,1778688) | 20260505 08:29:02.273 | INFO | MemInfo: { | 20260505 08:29:02.273 | INFO | "gpu" : " 17 GB", | 20260505 08:29:02.273 | INFO | "huge" : "", | 20260505 08:29:02.273 | INFO | "huge_shared" : " 27 GB", | 20260505 08:29:02.273 | INFO | "malloc" : " 14 GB", | 20260505 08:29:02.273 | INFO | "pinned" : " 29 GB" | 20260505 08:29:02.273 | INFO | } | 20260505 08:29:02.274 | INFO | Perf report for the last 5.13937 seconds: | 20260505 08:29:02.274 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:29:02.274 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:02.274 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:02.274 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:02.274 | INFO | ANALYSIS : 64.5%, 3.0, 100.7ms, 0 | 20260505 08:29:02.274 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:02.274 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:02.274 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:29:02.274 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:02.274 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:02.274 | INFO | Uploaded 36GiB to GPU in 2.23164s (16.1316GiB/s) | 20260505 08:29:02.274 | INFO | Downloaded 34GiB to GPU in 2.09912s (16.1972GiB/s) | 20260505 08:29:07.413 | INFO | Analyzing chunk frames = [1778688,1779200) | 20260505 08:29:07.413 | INFO | MemInfo: { | 20260505 08:29:07.413 | INFO | "gpu" : " 17 GB", | 20260505 08:29:07.413 | INFO | "huge" : "", | 20260505 08:29:07.413 | INFO | "huge_shared" : " 27 GB", | 20260505 08:29:07.413 | INFO | "malloc" : " 14 GB", | 20260505 08:29:07.413 | INFO | "pinned" : " 29 GB" | 20260505 08:29:07.413 | INFO | } | 20260505 08:29:07.413 | INFO | Perf report for the last 5.13942 seconds: | 20260505 08:29:07.413 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:29:07.413 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:07.413 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:07.413 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:07.413 | INFO | ANALYSIS : 63.7%, 3.0, 99.1ms, 0 | 20260505 08:29:07.413 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:07.413 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:07.413 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:29:07.413 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:07.413 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:07.413 | INFO | Uploaded 36GiB to GPU in 2.17758s (16.5321GiB/s) | 20260505 08:29:07.414 | INFO | Downloaded 34GiB to GPU in 2.06623s (16.4551GiB/s) | 20260505 08:29:12.553 | INFO | Analyzing chunk frames = [1779200,1779712) | 20260505 08:29:12.553 | INFO | MemInfo: { | 20260505 08:29:12.553 | INFO | "gpu" : " 17 GB", | 20260505 08:29:12.553 | INFO | "huge" : "", | 20260505 08:29:12.553 | INFO | "huge_shared" : " 27 GB", | 20260505 08:29:12.553 | INFO | "malloc" : " 14 GB", | 20260505 08:29:12.553 | INFO | "pinned" : " 29 GB" | 20260505 08:29:12.553 | INFO | } | 20260505 08:29:12.553 | INFO | Perf report for the last 5.14027 seconds: | 20260505 08:29:12.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:29:12.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:12.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:12.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:12.553 | INFO | ANALYSIS : 63.6%, 3.0, 98.8ms, 0 | 20260505 08:29:12.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:12.553 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:12.553 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 08:29:12.553 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:12.554 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:12.554 | INFO | Uploaded 36GiB to GPU in 2.18834s (16.4508GiB/s) | 20260505 08:29:12.554 | INFO | Downloaded 34GiB to GPU in 2.0655s (16.4609GiB/s) | 20260505 08:29:17.692 | INFO | Analyzing chunk frames = [1779712,1780224) | 20260505 08:29:17.692 | INFO | MemInfo: { | 20260505 08:29:17.692 | INFO | "gpu" : " 17 GB", | 20260505 08:29:17.692 | INFO | "huge" : "", | 20260505 08:29:17.692 | INFO | "huge_shared" : " 27 GB", | 20260505 08:29:17.692 | INFO | "malloc" : " 14 GB", | 20260505 08:29:17.692 | INFO | "pinned" : " 29 GB" | 20260505 08:29:17.692 | INFO | } | 20260505 08:29:17.692 | INFO | Perf report for the last 5.13896 seconds: | 20260505 08:29:17.692 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:29:17.692 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:17.692 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:17.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:17.692 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 08:29:17.692 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:17.692 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:17.692 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 08:29:17.692 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:17.692 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:17.693 | INFO | Uploaded 36GiB to GPU in 2.18959s (16.4414GiB/s) | 20260505 08:29:17.693 | INFO | Downloaded 34GiB to GPU in 2.06015s (16.5037GiB/s) | 20260505 08:29:22.835 | INFO | Analyzing chunk frames = [1780224,1780736) | 20260505 08:29:22.835 | INFO | MemInfo: { | 20260505 08:29:22.835 | INFO | "gpu" : " 17 GB", | 20260505 08:29:22.835 | INFO | "huge" : "", | 20260505 08:29:22.835 | INFO | "huge_shared" : " 27 GB", | 20260505 08:29:22.835 | INFO | "malloc" : " 14 GB", | 20260505 08:29:22.835 | INFO | "pinned" : " 29 GB" | 20260505 08:29:22.835 | INFO | } | 20260505 08:29:22.835 | INFO | Perf report for the last 5.14261 seconds: | 20260505 08:29:22.835 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:29:22.835 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:22.835 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:22.835 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:22.835 | INFO | ANALYSIS : 63.5%, 3.0, 99.3ms, 0 | 20260505 08:29:22.835 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:22.835 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:22.835 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 08:29:22.835 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:22.835 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:22.835 | INFO | Uploaded 36GiB to GPU in 2.18352s (16.4872GiB/s) | 20260505 08:29:22.835 | INFO | Downloaded 34GiB to GPU in 2.05031s (16.5829GiB/s) | 20260505 08:29:27.970 | INFO | Analyzing chunk frames = [1780736,1781248) | 20260505 08:29:27.970 | INFO | MemInfo: { | 20260505 08:29:27.970 | INFO | "gpu" : " 17 GB", | 20260505 08:29:27.970 | INFO | "huge" : "", | 20260505 08:29:27.970 | INFO | "huge_shared" : " 27 GB", | 20260505 08:29:27.970 | INFO | "malloc" : " 14 GB", | 20260505 08:29:27.970 | INFO | "pinned" : " 29 GB" | 20260505 08:29:27.970 | INFO | } | 20260505 08:29:27.970 | INFO | Perf report for the last 5.13511 seconds: | 20260505 08:29:27.970 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:29:27.970 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:27.970 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:27.970 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:27.970 | INFO | ANALYSIS : 64.2%, 3.0, 100.1ms, 0 | 20260505 08:29:27.970 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:27.970 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:27.970 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 08:29:27.970 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:27.970 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:27.970 | INFO | Uploaded 36GiB to GPU in 2.21437s (16.2575GiB/s) | 20260505 08:29:27.971 | INFO | Downloaded 34GiB to GPU in 2.14453s (15.8543GiB/s) | 20260505 08:29:33.109 | INFO | Analyzing chunk frames = [1781248,1781760) | 20260505 08:29:33.109 | INFO | MemInfo: { | 20260505 08:29:33.109 | INFO | "gpu" : " 17 GB", | 20260505 08:29:33.109 | INFO | "huge" : "", | 20260505 08:29:33.109 | INFO | "huge_shared" : " 27 GB", | 20260505 08:29:33.109 | INFO | "malloc" : " 14 GB", | 20260505 08:29:33.109 | INFO | "pinned" : " 29 GB" | 20260505 08:29:33.109 | INFO | } | 20260505 08:29:33.109 | INFO | Perf report for the last 5.13936 seconds: | 20260505 08:29:33.109 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:29:33.109 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:33.109 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:33.109 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:33.110 | INFO | ANALYSIS : 64.1%, 3.0, 100.0ms, 0 | 20260505 08:29:33.110 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:33.110 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:33.110 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 08:29:33.110 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:33.110 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:33.110 | INFO | Uploaded 36GiB to GPU in 2.17538s (16.5488GiB/s) | 20260505 08:29:33.110 | INFO | Downloaded 34GiB to GPU in 2.03601s (16.6994GiB/s) | 20260505 08:29:33.250 | INFO | Starting a round of Baz IO | 20260505 08:29:36.531 | INFO | Populating RTMetrics report | 20260505 08:29:36.532 | INFO | RTMetrics report populated | 20260505 08:29:36.532 | INFO | Opening tmp RTMetrics json file | 20260505 08:29:36.574 | INFO | Serializing RTMetrics to json | 20260505 08:29:36.575 | INFO | Serialization done, swapping tmp file now... | 20260505 08:29:36.576 | INFO | RTMetrics done | 20260505 08:29:36.682 | INFO | Finished a round of Baz IO: Wrote 2.448 GiB in 3.431 seconds | 20260505 08:29:36.682 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 27.57% / 0.02% / 72.41% | 20260505 08:29:36.682 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.61683% / 2.78155e-05% / 7.94729e-05% | 20260505 08:29:36.682 | INFO | Baz IO overall: | 20260505 08:29:36.682 | INFO | BytesWritten : 1006704564578 | 20260505 08:29:36.682 | INFO | NumEvents : 478557102248 | 20260505 08:29:38.249 | INFO | Analyzing chunk frames = [1781760,1782272) | 20260505 08:29:38.249 | INFO | MemInfo: { | 20260505 08:29:38.249 | INFO | "gpu" : " 17 GB", | 20260505 08:29:38.249 | INFO | "huge" : "", | 20260505 08:29:38.249 | INFO | "huge_shared" : " 27 GB", | 20260505 08:29:38.249 | INFO | "malloc" : " 14 GB", | 20260505 08:29:38.249 | INFO | "pinned" : " 29 GB" | 20260505 08:29:38.249 | INFO | } | 20260505 08:29:38.249 | INFO | Perf report for the last 5.13995 seconds: | 20260505 08:29:38.249 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:29:38.249 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:38.249 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:38.249 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:38.249 | INFO | ANALYSIS : 65.4%, 2.9, 100.4ms, 0 | 20260505 08:29:38.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:38.250 | INFO | RT_METRICS : 46.6%, 1.0, 24.4ms, 0 | 20260505 08:29:38.250 | INFO | PRE_HQ : 63.4%, 1.9, 61.5ms, 0 | 20260505 08:29:38.250 | INFO | BAZWRITER : 66.1%, 4.1, 140.2ms, 0 | 20260505 08:29:38.250 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 08:29:38.250 | INFO | Uploaded 36GiB to GPU in 2.23545s (16.1041GiB/s) | 20260505 08:29:38.250 | INFO | Downloaded 38GiB to GPU in 2.3467s (16.1929GiB/s) | 20260505 08:29:43.388 | INFO | Analyzing chunk frames = [1782272,1782784) | 20260505 08:29:43.388 | INFO | MemInfo: { | 20260505 08:29:43.388 | INFO | "gpu" : " 17 GB", | 20260505 08:29:43.388 | INFO | "huge" : "", | 20260505 08:29:43.388 | INFO | "huge_shared" : " 27 GB", | 20260505 08:29:43.388 | INFO | "malloc" : " 14 GB", | 20260505 08:29:43.388 | INFO | "pinned" : " 29 GB" | 20260505 08:29:43.388 | INFO | } | 20260505 08:29:43.389 | INFO | Perf report for the last 5.13909 seconds: | 20260505 08:29:43.389 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:29:43.389 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:43.389 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:43.389 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:43.389 | INFO | ANALYSIS : 64.7%, 3.0, 101.0ms, 0 | 20260505 08:29:43.389 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:43.389 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:43.389 | INFO | PRE_HQ : 30.1%, 1.0, 15.7ms, 0 | 20260505 08:29:43.389 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:43.389 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:43.389 | INFO | Uploaded 36GiB to GPU in 2.24743s (16.0183GiB/s) | 20260505 08:29:43.389 | INFO | Downloaded 34GiB to GPU in 2.11087s (16.1071GiB/s) | 20260505 08:29:48.528 | INFO | Analyzing chunk frames = [1782784,1783296) | 20260505 08:29:48.528 | INFO | MemInfo: { | 20260505 08:29:48.528 | INFO | "gpu" : " 17 GB", | 20260505 08:29:48.528 | INFO | "huge" : "", | 20260505 08:29:48.528 | INFO | "huge_shared" : " 27 GB", | 20260505 08:29:48.528 | INFO | "malloc" : " 14 GB", | 20260505 08:29:48.528 | INFO | "pinned" : " 29 GB" | 20260505 08:29:48.528 | INFO | } | 20260505 08:29:48.528 | INFO | Perf report for the last 5.13952 seconds: | 20260505 08:29:48.528 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:29:48.528 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:48.528 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:48.528 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:48.528 | INFO | ANALYSIS : 63.7%, 3.0, 99.1ms, 0 | 20260505 08:29:48.528 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:48.528 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:48.528 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 08:29:48.528 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:48.528 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:48.528 | INFO | Uploaded 36GiB to GPU in 2.16995s (16.5902GiB/s) | 20260505 08:29:48.528 | INFO | Downloaded 34GiB to GPU in 2.06715s (16.4478GiB/s) | 20260505 08:29:53.668 | INFO | Analyzing chunk frames = [1783296,1783808) | 20260505 08:29:53.668 | INFO | MemInfo: { | 20260505 08:29:53.668 | INFO | "gpu" : " 17 GB", | 20260505 08:29:53.668 | INFO | "huge" : "", | 20260505 08:29:53.668 | INFO | "huge_shared" : " 27 GB", | 20260505 08:29:53.668 | INFO | "malloc" : " 14 GB", | 20260505 08:29:53.668 | INFO | "pinned" : " 29 GB" | 20260505 08:29:53.668 | INFO | } | 20260505 08:29:53.668 | INFO | Perf report for the last 5.14019 seconds: | 20260505 08:29:53.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:29:53.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:53.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:53.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:53.668 | INFO | ANALYSIS : 63.6%, 3.0, 98.8ms, 0 | 20260505 08:29:53.668 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:53.668 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:53.668 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 08:29:53.668 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:53.668 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:53.669 | INFO | Uploaded 36GiB to GPU in 2.18027s (16.5117GiB/s) | 20260505 08:29:53.669 | INFO | Downloaded 34GiB to GPU in 2.06202s (16.4887GiB/s) | 20260505 08:29:58.807 | INFO | Analyzing chunk frames = [1783808,1784320) | 20260505 08:29:58.807 | INFO | MemInfo: { | 20260505 08:29:58.807 | INFO | "gpu" : " 17 GB", | 20260505 08:29:58.807 | INFO | "huge" : "", | 20260505 08:29:58.807 | INFO | "huge_shared" : " 27 GB", | 20260505 08:29:58.807 | INFO | "malloc" : " 14 GB", | 20260505 08:29:58.807 | INFO | "pinned" : " 29 GB" | 20260505 08:29:58.807 | INFO | } | 20260505 08:29:58.807 | INFO | Perf report for the last 5.13835 seconds: | 20260505 08:29:58.807 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:29:58.807 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:58.807 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:58.807 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:58.807 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 08:29:58.807 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:58.807 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:58.807 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:29:58.807 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:58.807 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:29:58.807 | INFO | Uploaded 36GiB to GPU in 2.16341s (16.6404GiB/s) | 20260505 08:29:58.807 | INFO | Downloaded 34GiB to GPU in 2.05737s (16.5259GiB/s) | 20260505 08:30:03.946 | INFO | Analyzing chunk frames = [1784320,1784832) | 20260505 08:30:03.946 | INFO | MemInfo: { | 20260505 08:30:03.946 | INFO | "gpu" : " 17 GB", | 20260505 08:30:03.946 | INFO | "huge" : "", | 20260505 08:30:03.946 | INFO | "huge_shared" : " 27 GB", | 20260505 08:30:03.946 | INFO | "malloc" : " 14 GB", | 20260505 08:30:03.946 | INFO | "pinned" : " 29 GB" | 20260505 08:30:03.946 | INFO | } | 20260505 08:30:03.947 | INFO | Perf report for the last 5.13991 seconds: | 20260505 08:30:03.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:30:03.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:03.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:03.947 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:03.947 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 08:30:03.947 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:03.947 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:03.947 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 08:30:03.947 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:03.947 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:03.947 | INFO | Uploaded 36GiB to GPU in 2.19325s (16.414GiB/s) | 20260505 08:30:03.947 | INFO | Downloaded 34GiB to GPU in 2.06164s (16.4917GiB/s) | 20260505 08:30:09.084 | INFO | Analyzing chunk frames = [1784832,1785344) | 20260505 08:30:09.084 | INFO | MemInfo: { | 20260505 08:30:09.084 | INFO | "gpu" : " 17 GB", | 20260505 08:30:09.084 | INFO | "huge" : "", | 20260505 08:30:09.084 | INFO | "huge_shared" : " 27 GB", | 20260505 08:30:09.084 | INFO | "malloc" : " 14 GB", | 20260505 08:30:09.084 | INFO | "pinned" : " 29 GB" | 20260505 08:30:09.084 | INFO | } | 20260505 08:30:09.084 | INFO | Perf report for the last 5.1379 seconds: | 20260505 08:30:09.085 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:30:09.085 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:09.085 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:09.085 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:09.085 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 08:30:09.085 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:09.085 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:09.085 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260505 08:30:09.085 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:09.085 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:09.085 | INFO | Uploaded 36GiB to GPU in 2.21557s (16.2486GiB/s) | 20260505 08:30:09.085 | INFO | Downloaded 34GiB to GPU in 2.15351s (15.7882GiB/s) | 20260505 08:30:14.224 | INFO | Analyzing chunk frames = [1785344,1785856) | 20260505 08:30:14.224 | INFO | MemInfo: { | 20260505 08:30:14.224 | INFO | "gpu" : " 17 GB", | 20260505 08:30:14.224 | INFO | "huge" : "", | 20260505 08:30:14.224 | INFO | "huge_shared" : " 27 GB", | 20260505 08:30:14.224 | INFO | "malloc" : " 14 GB", | 20260505 08:30:14.224 | INFO | "pinned" : " 29 GB" | 20260505 08:30:14.224 | INFO | } | 20260505 08:30:14.224 | INFO | Perf report for the last 5.13953 seconds: | 20260505 08:30:14.224 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:30:14.224 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:14.224 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:14.224 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:14.224 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 08:30:14.224 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:14.224 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:14.224 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 08:30:14.224 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:14.224 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:14.224 | INFO | Uploaded 36GiB to GPU in 2.1937s (16.4106GiB/s) | 20260505 08:30:14.224 | INFO | Downloaded 34GiB to GPU in 2.04002s (16.6665GiB/s) | 20260505 08:30:14.224 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 08:30:14.224 | INFO | Total mem usage: 15.3 GiB | 20260505 08:30:14.224 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 08:30:14.224 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.07%) | 20260505 08:30:14.224 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 08:30:14.224 | INFO | High Water Breakdown: | 20260505 08:30:14.224 | INFO | 60.8% : 8.78 GiB ( 93.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 08:30:14.224 | INFO | 39.1% : 5.65 GiB ( 85% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 08:30:14.224 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 08:30:14.224 | INFO | ------------------------------------------ | 20260505 08:30:14.224 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:30:14.225 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 08:30:14.225 | INFO | Total mem usage: 26.5 GiB | 20260505 08:30:14.225 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 08:30:14.225 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 08:30:14.225 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 08:30:14.225 | INFO | High Water Breakdown: | 20260505 08:30:14.225 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:30:14.225 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:30:14.225 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:30:14.225 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:30:14.225 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:30:14.225 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:30:14.225 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 08:30:14.225 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 08:30:14.225 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:30:14.225 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 08:30:14.225 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 08:30:14.225 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 08:30:14.225 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 08:30:14.225 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 08:30:14.225 | INFO | ------------------------------------------ | 20260505 08:30:14.225 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:30:14.225 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 08:30:14.225 | INFO | Total mem usage: 25.6 GiB | 20260505 08:30:14.225 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 08:30:14.225 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 08:30:14.225 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 08:30:14.225 | INFO | High Water Breakdown: | 20260505 08:30:14.225 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 08:30:14.225 | INFO | ------------------------------------------ | 20260505 08:30:14.225 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:30:14.225 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 08:30:14.225 | INFO | Total mem usage: 15.7 GiB | 20260505 08:30:14.225 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 08:30:14.225 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 08:30:14.225 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 08:30:14.225 | INFO | High Water Breakdown: | 20260505 08:30:14.225 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 08:30:14.225 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 08:30:14.225 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 08:30:14.225 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 08:30:14.225 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 08:30:14.225 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 08:30:14.225 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 08:30:14.225 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 08:30:14.225 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 08:30:14.225 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:30:14.225 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 08:30:14.225 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:30:14.225 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 08:30:14.225 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 08:30:14.225 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 08:30:14.225 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 08:30:14.225 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:30:14.225 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 08:30:14.225 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:30:14.225 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:30:14.225 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:30:14.225 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 08:30:14.225 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 08:30:14.225 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:30:14.225 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 08:30:14.225 | INFO | ------------------------------------------ | 20260505 08:30:14.225 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:30:14.225 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 08:30:14.225 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:30:14.225 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:30:14.225 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.55e+01, 1.35e+01 / 2.20e+00 | 20260505 08:30:14.225 | INFO | PerfReport: 0.29%, AggregateStats -- 9.30e-02 / 1.03e-01, 1.04e-02 / 4.66e-03 | 20260505 08:30:14.225 | INFO | PerfReport: 11.63%, Baseline -- 3.79e+00 / 4.46e+00, 6.18e-01 / 3.17e-01 | 20260505 08:30:14.225 | INFO | PerfReport: 3.32%, Histogram -- 1.08e+00 / 1.50e+00, 2.13e-01 / 1.03e-01 | 20260505 08:30:14.225 | INFO | PerfReport: 15.54%, DME -- 4.06e+01 / 4.44e+01, 4.19e+00 / 1.85e+00 | 20260505 08:30:14.225 | INFO | PerfReport: 56.48%, FrameLabeling -- 1.84e+01 / 1.99e+01, 1.94e+00 / 1.67e+00 | 20260505 08:30:14.225 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.43e+00 / 2.99e+00, 4.06e-01 / 2.76e-01 | 20260505 08:30:14.225 | INFO | PerfReport: 3.89%, Metrics -- 1.27e+00 / 1.59e+00, 3.55e-01 / 1.38e-01 | 20260505 08:30:14.225 | INFO | PerfReport: 1.40%, PulseSerializing -- 4.55e-01 / 5.13e-01, 8.67e-02 / 2.77e-02 | 20260505 08:30:14.225 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 08:30:14.225 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:30:14.225 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:30:14.225 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.32e+01 / 0.00e+00 | 20260505 08:30:14.225 | INFO | PerfReport: 22.53%, Upload -- 2.29e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260505 08:30:14.225 | INFO | PerfReport: 21.60%, Download -- 2.20e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260505 08:30:14.226 | INFO | PerfReport: 32.05%, Compute -- 3.26e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 08:30:14.226 | INFO | PerfReport: 1.68%, LockUpload -- 1.71e+00 / 0.00e+00, 7.34e+00 / 0.00e+00 | 20260505 08:30:14.226 | INFO | PerfReport: 1.05%, LockDownload -- 1.07e+00 / 0.00e+00, 2.53e+00 / 0.00e+00 | 20260505 08:30:14.226 | INFO | PerfReport: 21.07%, LockCompute -- 2.14e+01 / 0.00e+00, 1.78e+01 / 0.00e+00 | 20260505 08:30:14.226 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.74e-02 / 0.00e+00 | 20260505 08:30:14.365 | INFO | Starting a round of Baz IO | 20260505 08:30:17.650 | INFO | Populating RTMetrics report | 20260505 08:30:17.651 | INFO | RTMetrics report populated | 20260505 08:30:17.651 | INFO | Opening tmp RTMetrics json file | 20260505 08:30:17.660 | INFO | Serializing RTMetrics to json | 20260505 08:30:17.662 | INFO | Serialization done, swapping tmp file now... | 20260505 08:30:17.662 | INFO | RTMetrics done | 20260505 08:30:17.828 | INFO | Finished a round of Baz IO: Wrote 2.451 GiB in 3.462 seconds | 20260505 08:30:17.828 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 27.55% / 0.02% / 72.43% | 20260505 08:30:17.828 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.61014% / 3.17891e-05% / 7.54992e-05% | 20260505 08:30:17.828 | INFO | Baz IO overall: | 20260505 08:30:17.828 | INFO | BytesWritten : 1009336096722 | 20260505 08:30:17.828 | INFO | NumEvents : 479815733449 | 20260505 08:30:19.364 | INFO | Analyzing chunk frames = [1785856,1786368) | 20260505 08:30:19.364 | INFO | MemInfo: { | 20260505 08:30:19.364 | INFO | "gpu" : " 17 GB", | 20260505 08:30:19.364 | INFO | "huge" : "", | 20260505 08:30:19.364 | INFO | "huge_shared" : " 27 GB", | 20260505 08:30:19.364 | INFO | "malloc" : " 14 GB", | 20260505 08:30:19.364 | INFO | "pinned" : " 29 GB" | 20260505 08:30:19.364 | INFO | } | 20260505 08:30:19.364 | INFO | Perf report for the last 5.13967 seconds: | 20260505 08:30:19.364 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:30:19.364 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:19.364 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:19.364 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:19.364 | INFO | ANALYSIS : 65.5%, 2.9, 100.2ms, 0 | 20260505 08:30:19.364 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:19.364 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260505 08:30:19.364 | INFO | PRE_HQ : 63.3%, 1.9, 61.1ms, 0 | 20260505 08:30:19.364 | INFO | BAZWRITER : 66.7%, 4.2, 142.3ms, 0 | 20260505 08:30:19.364 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 08:30:19.364 | INFO | Uploaded 36GiB to GPU in 2.237s (16.093GiB/s) | 20260505 08:30:19.364 | INFO | Downloaded 38GiB to GPU in 2.35054s (16.1665GiB/s) | 20260505 08:30:24.503 | INFO | Analyzing chunk frames = [1786368,1786880) | 20260505 08:30:24.503 | INFO | MemInfo: { | 20260505 08:30:24.503 | INFO | "gpu" : " 17 GB", | 20260505 08:30:24.503 | INFO | "huge" : "", | 20260505 08:30:24.503 | INFO | "huge_shared" : " 27 GB", | 20260505 08:30:24.503 | INFO | "malloc" : " 14 GB", | 20260505 08:30:24.503 | INFO | "pinned" : " 29 GB" | 20260505 08:30:24.503 | INFO | } | 20260505 08:30:24.503 | INFO | Perf report for the last 5.13965 seconds: | 20260505 08:30:24.503 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:30:24.503 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:24.503 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:24.503 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:24.503 | INFO | ANALYSIS : 64.5%, 3.0, 100.7ms, 0 | 20260505 08:30:24.503 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:24.503 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:24.503 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:30:24.503 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:24.503 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:24.504 | INFO | Uploaded 36GiB to GPU in 2.223s (16.1943GiB/s) | 20260505 08:30:24.504 | INFO | Downloaded 34GiB to GPU in 2.09198s (16.2526GiB/s) | 20260505 08:30:29.643 | INFO | Analyzing chunk frames = [1786880,1787392) | 20260505 08:30:29.643 | INFO | MemInfo: { | 20260505 08:30:29.643 | INFO | "gpu" : " 17 GB", | 20260505 08:30:29.643 | INFO | "huge" : "", | 20260505 08:30:29.643 | INFO | "huge_shared" : " 27 GB", | 20260505 08:30:29.643 | INFO | "malloc" : " 14 GB", | 20260505 08:30:29.643 | INFO | "pinned" : " 29 GB" | 20260505 08:30:29.643 | INFO | } | 20260505 08:30:29.643 | INFO | Perf report for the last 5.13927 seconds: | 20260505 08:30:29.643 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:30:29.643 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:29.643 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:29.643 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:29.643 | INFO | ANALYSIS : 63.6%, 3.0, 99.0ms, 0 | 20260505 08:30:29.643 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:29.643 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:29.643 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 08:30:29.643 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:29.643 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:29.643 | INFO | Uploaded 36GiB to GPU in 2.16804s (16.6049GiB/s) | 20260505 08:30:29.643 | INFO | Downloaded 34GiB to GPU in 2.06385s (16.4741GiB/s) | 20260505 08:30:34.783 | INFO | Analyzing chunk frames = [1787392,1787904) | 20260505 08:30:34.783 | INFO | MemInfo: { | 20260505 08:30:34.783 | INFO | "gpu" : " 17 GB", | 20260505 08:30:34.783 | INFO | "huge" : "", | 20260505 08:30:34.783 | INFO | "huge_shared" : " 27 GB", | 20260505 08:30:34.783 | INFO | "malloc" : " 14 GB", | 20260505 08:30:34.783 | INFO | "pinned" : " 29 GB" | 20260505 08:30:34.783 | INFO | } | 20260505 08:30:34.783 | INFO | Perf report for the last 5.14006 seconds: | 20260505 08:30:34.783 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:30:34.783 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:34.783 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:34.783 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:34.783 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260505 08:30:34.783 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:34.783 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:34.783 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:30:34.783 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:34.783 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:34.783 | INFO | Uploaded 36GiB to GPU in 2.17849s (16.5252GiB/s) | 20260505 08:30:34.783 | INFO | Downloaded 34GiB to GPU in 2.06287s (16.4819GiB/s) | 20260505 08:30:39.920 | INFO | Analyzing chunk frames = [1787904,1788416) | 20260505 08:30:39.920 | INFO | MemInfo: { | 20260505 08:30:39.920 | INFO | "gpu" : " 17 GB", | 20260505 08:30:39.920 | INFO | "huge" : "", | 20260505 08:30:39.920 | INFO | "huge_shared" : " 27 GB", | 20260505 08:30:39.920 | INFO | "malloc" : " 14 GB", | 20260505 08:30:39.921 | INFO | "pinned" : " 29 GB" | 20260505 08:30:39.921 | INFO | } | 20260505 08:30:39.921 | INFO | Perf report for the last 5.13795 seconds: | 20260505 08:30:39.921 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:30:39.921 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:39.921 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:39.921 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:39.921 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 08:30:39.921 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:39.921 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:39.921 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 08:30:39.921 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:39.921 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:39.921 | INFO | Uploaded 36GiB to GPU in 2.16106s (16.6585GiB/s) | 20260505 08:30:39.921 | INFO | Downloaded 34GiB to GPU in 2.05774s (16.523GiB/s) | 20260505 08:30:45.061 | INFO | Analyzing chunk frames = [1788416,1788928) | 20260505 08:30:45.061 | INFO | MemInfo: { | 20260505 08:30:45.061 | INFO | "gpu" : " 17 GB", | 20260505 08:30:45.061 | INFO | "huge" : "", | 20260505 08:30:45.061 | INFO | "huge_shared" : " 27 GB", | 20260505 08:30:45.061 | INFO | "malloc" : " 14 GB", | 20260505 08:30:45.061 | INFO | "pinned" : " 29 GB" | 20260505 08:30:45.061 | INFO | } | 20260505 08:30:45.061 | INFO | Perf report for the last 5.14046 seconds: | 20260505 08:30:45.061 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:30:45.061 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:45.061 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:45.061 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:45.061 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 08:30:45.061 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:45.061 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:45.061 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 08:30:45.061 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:45.061 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:45.062 | INFO | Uploaded 36GiB to GPU in 2.18573s (16.4704GiB/s) | 20260505 08:30:45.062 | INFO | Downloaded 34GiB to GPU in 2.05713s (16.5279GiB/s) | 20260505 08:30:50.200 | INFO | Analyzing chunk frames = [1788928,1789440) | 20260505 08:30:50.200 | INFO | MemInfo: { | 20260505 08:30:50.200 | INFO | "gpu" : " 17 GB", | 20260505 08:30:50.200 | INFO | "huge" : "", | 20260505 08:30:50.200 | INFO | "huge_shared" : " 27 GB", | 20260505 08:30:50.200 | INFO | "malloc" : " 14 GB", | 20260505 08:30:50.200 | INFO | "pinned" : " 29 GB" | 20260505 08:30:50.200 | INFO | } | 20260505 08:30:50.200 | INFO | Perf report for the last 5.13918 seconds: | 20260505 08:30:50.200 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:30:50.200 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:50.200 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:50.200 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:50.201 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 08:30:50.201 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:50.201 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:50.201 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 08:30:50.201 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:50.201 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:50.201 | INFO | Uploaded 36GiB to GPU in 2.20308s (16.3407GiB/s) | 20260505 08:30:50.201 | INFO | Downloaded 34GiB to GPU in 2.15764s (15.758GiB/s) | 20260505 08:30:55.339 | INFO | Analyzing chunk frames = [1789440,1789952) | 20260505 08:30:55.339 | INFO | MemInfo: { | 20260505 08:30:55.339 | INFO | "gpu" : " 17 GB", | 20260505 08:30:55.339 | INFO | "huge" : "", | 20260505 08:30:55.339 | INFO | "huge_shared" : " 27 GB", | 20260505 08:30:55.339 | INFO | "malloc" : " 14 GB", | 20260505 08:30:55.339 | INFO | "pinned" : " 29 GB" | 20260505 08:30:55.339 | INFO | } | 20260505 08:30:55.339 | INFO | Perf report for the last 5.13828 seconds: | 20260505 08:30:55.339 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:30:55.339 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:55.339 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:55.339 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:55.339 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 08:30:55.339 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:55.339 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:55.339 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 08:30:55.339 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:55.339 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:30:55.339 | INFO | Uploaded 36GiB to GPU in 2.19601s (16.3933GiB/s) | 20260505 08:30:55.339 | INFO | Downloaded 34GiB to GPU in 2.04144s (16.6549GiB/s) | 20260505 08:30:55.479 | INFO | Starting a round of Baz IO | 20260505 08:30:58.750 | INFO | Populating RTMetrics report | 20260505 08:30:58.750 | INFO | RTMetrics report populated | 20260505 08:30:58.750 | INFO | Opening tmp RTMetrics json file | 20260505 08:30:58.758 | INFO | Serializing RTMetrics to json | 20260505 08:30:58.760 | INFO | Serialization done, swapping tmp file now... | 20260505 08:30:58.760 | INFO | RTMetrics done | 20260505 08:30:58.957 | INFO | Finished a round of Baz IO: Wrote 2.453 GiB in 3.478 seconds | 20260505 08:30:58.957 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 27.53% / 0.02% / 72.45% | 20260505 08:30:58.957 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.60238% / 3.97364e-05% / 7.15256e-05% | 20260505 08:30:58.957 | INFO | Baz IO overall: | 20260505 08:30:58.957 | INFO | BytesWritten : 1011969553101 | 20260505 08:30:58.957 | INFO | NumEvents : 481075366419 | 20260505 08:31:00.478 | INFO | Analyzing chunk frames = [1789952,1790464) | 20260505 08:31:00.478 | INFO | MemInfo: { | 20260505 08:31:00.478 | INFO | "gpu" : " 17 GB", | 20260505 08:31:00.478 | INFO | "huge" : "", | 20260505 08:31:00.478 | INFO | "huge_shared" : " 27 GB", | 20260505 08:31:00.478 | INFO | "malloc" : " 14 GB", | 20260505 08:31:00.478 | INFO | "pinned" : " 29 GB" | 20260505 08:31:00.478 | INFO | } | 20260505 08:31:00.478 | INFO | Perf report for the last 5.13929 seconds: | 20260505 08:31:00.478 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:31:00.478 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:00.478 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:00.478 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:00.478 | INFO | ANALYSIS : 65.2%, 2.9, 99.1ms, 0 | 20260505 08:31:00.478 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:00.478 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 08:31:00.478 | INFO | PRE_HQ : 63.6%, 1.8, 60.9ms, 0 | 20260505 08:31:00.478 | INFO | BAZWRITER : 67.0%, 4.2, 146.2ms, 0 | 20260505 08:31:00.478 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 08:31:00.478 | INFO | Uploaded 36GiB to GPU in 2.27746s (15.8071GiB/s) | 20260505 08:31:00.478 | INFO | Downloaded 38GiB to GPU in 2.39157s (15.8891GiB/s) | 20260505 08:31:05.618 | INFO | Analyzing chunk frames = [1790464,1790976) | 20260505 08:31:05.618 | INFO | MemInfo: { | 20260505 08:31:05.618 | INFO | "gpu" : " 17 GB", | 20260505 08:31:05.618 | INFO | "huge" : "", | 20260505 08:31:05.618 | INFO | "huge_shared" : " 27 GB", | 20260505 08:31:05.618 | INFO | "malloc" : " 14 GB", | 20260505 08:31:05.618 | INFO | "pinned" : " 29 GB" | 20260505 08:31:05.618 | INFO | } | 20260505 08:31:05.618 | INFO | Perf report for the last 5.14056 seconds: | 20260505 08:31:05.619 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:31:05.619 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:05.619 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:05.619 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:05.619 | INFO | ANALYSIS : 64.8%, 3.0, 101.2ms, 0 | 20260505 08:31:05.619 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:05.619 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:05.619 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 08:31:05.619 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:05.619 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:05.619 | INFO | Uploaded 36GiB to GPU in 2.25109s (15.9923GiB/s) | 20260505 08:31:05.619 | INFO | Downloaded 34GiB to GPU in 2.10074s (16.1847GiB/s) | 20260505 08:31:10.757 | INFO | Analyzing chunk frames = [1790976,1791488) | 20260505 08:31:10.757 | INFO | MemInfo: { | 20260505 08:31:10.757 | INFO | "gpu" : " 17 GB", | 20260505 08:31:10.757 | INFO | "huge" : "", | 20260505 08:31:10.757 | INFO | "huge_shared" : " 27 GB", | 20260505 08:31:10.758 | INFO | "malloc" : " 14 GB", | 20260505 08:31:10.758 | INFO | "pinned" : " 29 GB" | 20260505 08:31:10.758 | INFO | } | 20260505 08:31:10.758 | INFO | Perf report for the last 5.13909 seconds: | 20260505 08:31:10.758 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:31:10.758 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:10.758 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:10.758 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:10.758 | INFO | ANALYSIS : 63.7%, 3.0, 99.1ms, 0 | 20260505 08:31:10.758 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:10.758 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:10.758 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:31:10.758 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:10.758 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:10.758 | INFO | Uploaded 36GiB to GPU in 2.16993s (16.5904GiB/s) | 20260505 08:31:10.758 | INFO | Downloaded 34GiB to GPU in 2.0663s (16.4546GiB/s) | 20260505 08:31:15.896 | INFO | Analyzing chunk frames = [1791488,1792000) | 20260505 08:31:15.896 | INFO | MemInfo: { | 20260505 08:31:15.896 | INFO | "gpu" : " 17 GB", | 20260505 08:31:15.896 | INFO | "huge" : "", | 20260505 08:31:15.896 | INFO | "huge_shared" : " 27 GB", | 20260505 08:31:15.896 | INFO | "malloc" : " 14 GB", | 20260505 08:31:15.896 | INFO | "pinned" : " 29 GB" | 20260505 08:31:15.896 | INFO | } | 20260505 08:31:15.896 | INFO | Perf report for the last 5.13842 seconds: | 20260505 08:31:15.896 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:31:15.896 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:15.896 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:15.896 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:15.896 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260505 08:31:15.896 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:15.896 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:15.896 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:31:15.896 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:15.896 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:15.896 | INFO | Uploaded 36GiB to GPU in 2.17745s (16.5331GiB/s) | 20260505 08:31:15.896 | INFO | Downloaded 34GiB to GPU in 2.06717s (16.4476GiB/s) | 20260505 08:31:21.036 | INFO | Analyzing chunk frames = [1792000,1792512) | 20260505 08:31:21.036 | INFO | MemInfo: { | 20260505 08:31:21.036 | INFO | "gpu" : " 17 GB", | 20260505 08:31:21.036 | INFO | "huge" : "", | 20260505 08:31:21.036 | INFO | "huge_shared" : " 27 GB", | 20260505 08:31:21.036 | INFO | "malloc" : " 14 GB", | 20260505 08:31:21.036 | INFO | "pinned" : " 29 GB" | 20260505 08:31:21.036 | INFO | } | 20260505 08:31:21.036 | INFO | Perf report for the last 5.13973 seconds: | 20260505 08:31:21.036 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:31:21.036 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:21.036 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:21.036 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:21.036 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 08:31:21.036 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:21.036 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:21.036 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 08:31:21.036 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:21.036 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:21.036 | INFO | Uploaded 36GiB to GPU in 2.18984s (16.4396GiB/s) | 20260505 08:31:21.036 | INFO | Downloaded 34GiB to GPU in 2.06196s (16.4892GiB/s) | 20260505 08:31:26.179 | INFO | Analyzing chunk frames = [1792512,1793024) | 20260505 08:31:26.179 | INFO | MemInfo: { | 20260505 08:31:26.179 | INFO | "gpu" : " 17 GB", | 20260505 08:31:26.179 | INFO | "huge" : "", | 20260505 08:31:26.179 | INFO | "huge_shared" : " 27 GB", | 20260505 08:31:26.179 | INFO | "malloc" : " 14 GB", | 20260505 08:31:26.179 | INFO | "pinned" : " 29 GB" | 20260505 08:31:26.179 | INFO | } | 20260505 08:31:26.179 | INFO | Perf report for the last 5.14338 seconds: | 20260505 08:31:26.179 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:31:26.179 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:26.179 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:26.179 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:26.179 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 08:31:26.179 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:26.179 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:26.179 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 08:31:26.179 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:26.179 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:26.179 | INFO | Uploaded 36GiB to GPU in 2.19121s (16.4293GiB/s) | 20260505 08:31:26.179 | INFO | Downloaded 34GiB to GPU in 2.05785s (16.5221GiB/s) | 20260505 08:31:31.314 | INFO | Analyzing chunk frames = [1793024,1793536) | 20260505 08:31:31.314 | INFO | MemInfo: { | 20260505 08:31:31.314 | INFO | "gpu" : " 17 GB", | 20260505 08:31:31.314 | INFO | "huge" : "", | 20260505 08:31:31.314 | INFO | "huge_shared" : " 27 GB", | 20260505 08:31:31.314 | INFO | "malloc" : " 14 GB", | 20260505 08:31:31.314 | INFO | "pinned" : " 29 GB" | 20260505 08:31:31.314 | INFO | } | 20260505 08:31:31.314 | INFO | Perf report for the last 5.13505 seconds: | 20260505 08:31:31.314 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:31:31.314 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:31.314 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:31.314 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:31.314 | INFO | ANALYSIS : 64.1%, 3.0, 100.0ms, 0 | 20260505 08:31:31.314 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:31.314 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:31.314 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 08:31:31.314 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:31.314 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:31.314 | INFO | Uploaded 36GiB to GPU in 2.21434s (16.2576GiB/s) | 20260505 08:31:31.315 | INFO | Downloaded 34GiB to GPU in 2.14944s (15.818GiB/s) | 20260505 08:31:36.454 | INFO | Analyzing chunk frames = [1793536,1794048) | 20260505 08:31:36.454 | INFO | MemInfo: { | 20260505 08:31:36.454 | INFO | "gpu" : " 17 GB", | 20260505 08:31:36.454 | INFO | "huge" : "", | 20260505 08:31:36.454 | INFO | "huge_shared" : " 27 GB", | 20260505 08:31:36.454 | INFO | "malloc" : " 14 GB", | 20260505 08:31:36.454 | INFO | "pinned" : " 29 GB" | 20260505 08:31:36.454 | INFO | } | 20260505 08:31:36.454 | INFO | Perf report for the last 5.13955 seconds: | 20260505 08:31:36.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:31:36.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:36.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:36.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:36.454 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 08:31:36.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:36.454 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:36.454 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:31:36.454 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:36.454 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:36.454 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 08:31:36.454 | INFO | Uploaded 36GiB to GPU in 2.19392s (16.409GiB/s) | 20260505 08:31:36.454 | INFO | Total mem usage: 15.3 GiB | 20260505 08:31:36.454 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 08:31:36.454 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -5.64%) | 20260505 08:31:36.454 | INFO | Downloaded 34GiB to GPU in 2.03871s (16.6772GiB/s) | 20260505 08:31:36.454 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 08:31:36.454 | INFO | High Water Breakdown: | 20260505 08:31:36.454 | INFO | 60.9% : 8.84 GiB ( 94.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 08:31:36.454 | INFO | 38.9% : 5.65 GiB ( 85% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 08:31:36.454 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 08:31:36.454 | INFO | ------------------------------------------ | 20260505 08:31:36.454 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:31:36.454 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 08:31:36.454 | INFO | Total mem usage: 26.5 GiB | 20260505 08:31:36.454 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 08:31:36.454 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 08:31:36.454 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 08:31:36.454 | INFO | High Water Breakdown: | 20260505 08:31:36.454 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:31:36.455 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:31:36.455 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:31:36.455 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:31:36.455 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:31:36.455 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:31:36.455 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 08:31:36.455 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 08:31:36.455 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:31:36.455 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 08:31:36.455 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 08:31:36.455 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 08:31:36.455 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 08:31:36.455 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 08:31:36.455 | INFO | ------------------------------------------ | 20260505 08:31:36.455 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:31:36.455 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 08:31:36.455 | INFO | Total mem usage: 25.6 GiB | 20260505 08:31:36.455 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 08:31:36.455 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 08:31:36.455 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 08:31:36.455 | INFO | High Water Breakdown: | 20260505 08:31:36.455 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 08:31:36.455 | INFO | ------------------------------------------ | 20260505 08:31:36.455 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:31:36.455 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 08:31:36.455 | INFO | Total mem usage: 15.7 GiB | 20260505 08:31:36.455 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 08:31:36.455 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 08:31:36.455 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 08:31:36.455 | INFO | High Water Breakdown: | 20260505 08:31:36.455 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 08:31:36.455 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 08:31:36.455 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 08:31:36.455 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 08:31:36.455 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 08:31:36.455 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 08:31:36.455 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 08:31:36.455 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 08:31:36.455 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 08:31:36.455 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:31:36.455 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 08:31:36.455 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:31:36.455 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 08:31:36.455 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 08:31:36.455 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 08:31:36.455 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 08:31:36.455 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:31:36.455 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 08:31:36.455 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:31:36.455 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:31:36.455 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:31:36.455 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 08:31:36.455 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 08:31:36.455 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:31:36.455 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 08:31:36.455 | INFO | ------------------------------------------ | 20260505 08:31:36.455 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:31:36.455 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 08:31:36.455 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:31:36.455 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:31:36.455 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.53e+01, 1.36e+01 / 2.02e+00 | 20260505 08:31:36.456 | INFO | PerfReport: 0.29%, AggregateStats -- 9.31e-02 / 1.04e-01, 1.02e-02 / 4.81e-03 | 20260505 08:31:36.456 | INFO | PerfReport: 11.64%, Baseline -- 3.80e+00 / 4.46e+00, 6.18e-01 / 2.87e-01 | 20260505 08:31:36.456 | INFO | PerfReport: 3.32%, Histogram -- 1.08e+00 / 1.48e+00, 2.09e-01 / 1.12e-01 | 20260505 08:31:36.456 | INFO | PerfReport: 15.52%, DME -- 4.05e+01 / 4.43e+01, 4.09e+00 / 1.52e+00 | 20260505 08:31:36.456 | INFO | PerfReport: 56.46%, FrameLabeling -- 1.84e+01 / 1.99e+01, 1.93e+00 / 1.69e+00 | 20260505 08:31:36.456 | INFO | PerfReport: 7.48%, PulseAccumulating -- 2.44e+00 / 3.00e+00, 4.10e-01 / 2.24e-01 | 20260505 08:31:36.456 | INFO | PerfReport: 3.90%, Metrics -- 1.27e+00 / 1.60e+00, 3.61e-01 / 1.84e-01 | 20260505 08:31:36.456 | INFO | PerfReport: 1.39%, PulseSerializing -- 4.54e-01 / 5.09e-01, 8.56e-02 / 3.05e-02 | 20260505 08:31:36.456 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 08:31:36.456 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:31:36.456 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:31:36.456 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.31e+01 / 0.00e+00 | 20260505 08:31:36.456 | INFO | PerfReport: 22.56%, Upload -- 2.29e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260505 08:31:36.456 | INFO | PerfReport: 21.66%, Download -- 2.20e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260505 08:31:36.456 | INFO | PerfReport: 32.11%, Compute -- 3.26e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 08:31:36.456 | INFO | PerfReport: 1.68%, LockUpload -- 1.71e+00 / 0.00e+00, 7.34e+00 / 0.00e+00 | 20260505 08:31:36.456 | INFO | PerfReport: 1.04%, LockDownload -- 1.06e+00 / 0.00e+00, 2.53e+00 / 0.00e+00 | 20260505 08:31:36.456 | INFO | PerfReport: 20.94%, LockCompute -- 2.13e+01 / 0.00e+00, 1.77e+01 / 0.00e+00 | 20260505 08:31:36.456 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.90e-02 / 0.00e+00 | 20260505 08:31:36.594 | INFO | Starting a round of Baz IO | 20260505 08:31:39.871 | INFO | Populating RTMetrics report | 20260505 08:31:39.871 | INFO | RTMetrics report populated | 20260505 08:31:39.871 | INFO | Opening tmp RTMetrics json file | 20260505 08:31:39.882 | INFO | Serializing RTMetrics to json | 20260505 08:31:39.905 | INFO | Serialization done, swapping tmp file now... | 20260505 08:31:39.905 | INFO | RTMetrics done | 20260505 08:31:40.013 | INFO | Finished a round of Baz IO: Wrote 2.451 GiB in 3.418 seconds | 20260505 08:31:40.013 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 27.51% / 0.02% / 72.47% | 20260505 08:31:40.013 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.59067% / 4.37101e-05% / 5.5631e-05% | 20260505 08:31:40.013 | INFO | Baz IO overall: | 20260505 08:31:40.013 | INFO | BytesWritten : 1014601237176 | 20260505 08:31:40.013 | INFO | NumEvents : 482333683336 | 20260505 08:31:41.594 | INFO | Analyzing chunk frames = [1794048,1794560) | 20260505 08:31:41.594 | INFO | MemInfo: { | 20260505 08:31:41.594 | INFO | "gpu" : " 17 GB", | 20260505 08:31:41.594 | INFO | "huge" : "", | 20260505 08:31:41.594 | INFO | "huge_shared" : " 27 GB", | 20260505 08:31:41.594 | INFO | "malloc" : " 14 GB", | 20260505 08:31:41.594 | INFO | "pinned" : " 29 GB" | 20260505 08:31:41.594 | INFO | } | 20260505 08:31:41.594 | INFO | Perf report for the last 5.1402 seconds: | 20260505 08:31:41.594 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:31:41.594 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:41.594 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:41.594 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:41.594 | INFO | ANALYSIS : 65.3%, 2.9, 100.0ms, 0 | 20260505 08:31:41.594 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:41.594 | INFO | RT_METRICS : 46.6%, 1.0, 24.4ms, 0 | 20260505 08:31:41.594 | INFO | PRE_HQ : 63.3%, 1.9, 61.2ms, 0 | 20260505 08:31:41.594 | INFO | BAZWRITER : 65.8%, 4.2, 142.1ms, 0 | 20260505 08:31:41.594 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 08:31:41.594 | INFO | Uploaded 36GiB to GPU in 2.26593s (15.8875GiB/s) | 20260505 08:31:41.594 | INFO | Downloaded 38GiB to GPU in 2.37119s (16.0257GiB/s) | 20260505 08:31:46.737 | INFO | Analyzing chunk frames = [1794560,1795072) | 20260505 08:31:46.737 | INFO | MemInfo: { | 20260505 08:31:46.737 | INFO | "gpu" : " 17 GB", | 20260505 08:31:46.737 | INFO | "huge" : "", | 20260505 08:31:46.737 | INFO | "huge_shared" : " 27 GB", | 20260505 08:31:46.737 | INFO | "malloc" : " 14 GB", | 20260505 08:31:46.737 | INFO | "pinned" : " 29 GB" | 20260505 08:31:46.737 | INFO | } | 20260505 08:31:46.737 | INFO | Perf report for the last 5.1434 seconds: | 20260505 08:31:46.738 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:31:46.738 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:46.738 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:46.738 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:46.738 | INFO | ANALYSIS : 64.6%, 3.0, 100.9ms, 0 | 20260505 08:31:46.738 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:46.738 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:46.738 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:31:46.738 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:46.738 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:46.738 | INFO | Uploaded 36GiB to GPU in 2.24531s (16.0334GiB/s) | 20260505 08:31:46.738 | INFO | Downloaded 34GiB to GPU in 2.094s (16.2368GiB/s) | 20260505 08:31:51.873 | INFO | Analyzing chunk frames = [1795072,1795584) | 20260505 08:31:51.873 | INFO | MemInfo: { | 20260505 08:31:51.873 | INFO | "gpu" : " 17 GB", | 20260505 08:31:51.873 | INFO | "huge" : "", | 20260505 08:31:51.873 | INFO | "huge_shared" : " 27 GB", | 20260505 08:31:51.873 | INFO | "malloc" : " 14 GB", | 20260505 08:31:51.873 | INFO | "pinned" : " 29 GB" | 20260505 08:31:51.873 | INFO | } | 20260505 08:31:51.873 | INFO | Perf report for the last 5.13546 seconds: | 20260505 08:31:51.873 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:31:51.873 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:51.873 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:51.873 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:51.873 | INFO | ANALYSIS : 63.6%, 3.0, 98.9ms, 0 | 20260505 08:31:51.873 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:51.873 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:51.873 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:31:51.873 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:51.873 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:51.873 | INFO | Uploaded 36GiB to GPU in 2.16196s (16.6515GiB/s) | 20260505 08:31:51.873 | INFO | Downloaded 34GiB to GPU in 2.0661s (16.4561GiB/s) | 20260505 08:31:57.011 | INFO | Analyzing chunk frames = [1795584,1796096) | 20260505 08:31:57.011 | INFO | MemInfo: { | 20260505 08:31:57.011 | INFO | "gpu" : " 17 GB", | 20260505 08:31:57.011 | INFO | "huge" : "", | 20260505 08:31:57.011 | INFO | "huge_shared" : " 27 GB", | 20260505 08:31:57.011 | INFO | "malloc" : " 14 GB", | 20260505 08:31:57.011 | INFO | "pinned" : " 29 GB" | 20260505 08:31:57.011 | INFO | } | 20260505 08:31:57.012 | INFO | Perf report for the last 5.1387 seconds: | 20260505 08:31:57.012 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:31:57.012 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:57.012 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:57.012 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:57.012 | INFO | ANALYSIS : 63.6%, 3.0, 98.7ms, 0 | 20260505 08:31:57.012 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:57.012 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:57.012 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 08:31:57.012 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:57.012 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:31:57.012 | INFO | Uploaded 36GiB to GPU in 2.18288s (16.492GiB/s) | 20260505 08:31:57.012 | INFO | Downloaded 34GiB to GPU in 2.06838s (16.438GiB/s) | 20260505 08:32:02.150 | INFO | Analyzing chunk frames = [1796096,1796608) | 20260505 08:32:02.150 | INFO | MemInfo: { | 20260505 08:32:02.150 | INFO | "gpu" : " 17 GB", | 20260505 08:32:02.150 | INFO | "huge" : "", | 20260505 08:32:02.150 | INFO | "huge_shared" : " 27 GB", | 20260505 08:32:02.150 | INFO | "malloc" : " 14 GB", | 20260505 08:32:02.150 | INFO | "pinned" : " 29 GB" | 20260505 08:32:02.150 | INFO | } | 20260505 08:32:02.150 | INFO | Perf report for the last 5.13885 seconds: | 20260505 08:32:02.150 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:32:02.150 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:02.150 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:02.150 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:02.150 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 08:32:02.150 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:02.150 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:02.151 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 08:32:02.151 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:02.151 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:02.151 | INFO | Uploaded 36GiB to GPU in 2.17907s (16.5208GiB/s) | 20260505 08:32:02.151 | INFO | Downloaded 34GiB to GPU in 2.05926s (16.5108GiB/s) | 20260505 08:32:07.291 | INFO | Analyzing chunk frames = [1796608,1797120) | 20260505 08:32:07.291 | INFO | MemInfo: { | 20260505 08:32:07.291 | INFO | "gpu" : " 17 GB", | 20260505 08:32:07.291 | INFO | "huge" : "", | 20260505 08:32:07.291 | INFO | "huge_shared" : " 27 GB", | 20260505 08:32:07.291 | INFO | "malloc" : " 14 GB", | 20260505 08:32:07.291 | INFO | "pinned" : " 29 GB" | 20260505 08:32:07.291 | INFO | } | 20260505 08:32:07.291 | INFO | Perf report for the last 5.14058 seconds: | 20260505 08:32:07.291 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:32:07.291 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:07.291 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:07.291 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:07.291 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 08:32:07.291 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:07.291 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:07.291 | INFO | PRE_HQ : 29.7%, 1.0, 15.5ms, 0 | 20260505 08:32:07.291 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:07.291 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:07.291 | INFO | Uploaded 36GiB to GPU in 2.18466s (16.4785GiB/s) | 20260505 08:32:07.291 | INFO | Downloaded 34GiB to GPU in 2.05199s (16.5693GiB/s) | 20260505 08:32:12.429 | INFO | Analyzing chunk frames = [1797120,1797632) | 20260505 08:32:12.429 | INFO | MemInfo: { | 20260505 08:32:12.429 | INFO | "gpu" : " 17 GB", | 20260505 08:32:12.429 | INFO | "huge" : "", | 20260505 08:32:12.429 | INFO | "huge_shared" : " 27 GB", | 20260505 08:32:12.429 | INFO | "malloc" : " 14 GB", | 20260505 08:32:12.429 | INFO | "pinned" : " 29 GB" | 20260505 08:32:12.429 | INFO | } | 20260505 08:32:12.429 | INFO | Perf report for the last 5.13786 seconds: | 20260505 08:32:12.429 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:32:12.429 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:12.429 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:12.429 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:12.429 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 08:32:12.429 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:12.429 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:12.429 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 08:32:12.429 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:12.429 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:12.429 | INFO | Uploaded 36GiB to GPU in 2.17409s (16.5586GiB/s) | 20260505 08:32:12.429 | INFO | Downloaded 34GiB to GPU in 2.15288s (15.7928GiB/s) | 20260505 08:32:17.569 | INFO | Analyzing chunk frames = [1797632,1798144) | 20260505 08:32:17.569 | INFO | MemInfo: { | 20260505 08:32:17.569 | INFO | "gpu" : " 17 GB", | 20260505 08:32:17.569 | INFO | "huge" : "", | 20260505 08:32:17.569 | INFO | "huge_shared" : " 27 GB", | 20260505 08:32:17.569 | INFO | "malloc" : " 14 GB", | 20260505 08:32:17.569 | INFO | "pinned" : " 29 GB" | 20260505 08:32:17.569 | INFO | } | 20260505 08:32:17.569 | INFO | Perf report for the last 5.14029 seconds: | 20260505 08:32:17.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:32:17.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:17.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:17.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:17.569 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 08:32:17.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:17.569 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:17.569 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 08:32:17.569 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:17.569 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:17.570 | INFO | Uploaded 36GiB to GPU in 2.19093s (16.4314GiB/s) | 20260505 08:32:17.570 | INFO | Downloaded 34GiB to GPU in 2.04085s (16.6597GiB/s) | 20260505 08:32:17.709 | INFO | Starting a round of Baz IO | 20260505 08:32:20.985 | INFO | Populating RTMetrics report | 20260505 08:32:20.985 | INFO | RTMetrics report populated | 20260505 08:32:20.985 | INFO | Opening tmp RTMetrics json file | 20260505 08:32:20.992 | INFO | Serializing RTMetrics to json | 20260505 08:32:20.993 | INFO | Serialization done, swapping tmp file now... | 20260505 08:32:20.993 | INFO | RTMetrics done | 20260505 08:32:21.118 | INFO | Finished a round of Baz IO: Wrote 2.449 GiB in 3.409 seconds | 20260505 08:32:21.119 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 27.49% / 0.02% / 72.49% | 20260505 08:32:21.119 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.58236% / 1.98682e-05% / 5.5631e-05% | 20260505 08:32:21.119 | INFO | Baz IO overall: | 20260505 08:32:21.119 | INFO | BytesWritten : 1017231213974 | 20260505 08:32:21.119 | INFO | NumEvents : 483590956215 | 20260505 08:32:22.708 | INFO | Analyzing chunk frames = [1798144,1798656) | 20260505 08:32:22.708 | INFO | MemInfo: { | 20260505 08:32:22.708 | INFO | "gpu" : " 17 GB", | 20260505 08:32:22.708 | INFO | "huge" : "", | 20260505 08:32:22.708 | INFO | "huge_shared" : " 27 GB", | 20260505 08:32:22.708 | INFO | "malloc" : " 14 GB", | 20260505 08:32:22.708 | INFO | "pinned" : " 29 GB" | 20260505 08:32:22.708 | INFO | } | 20260505 08:32:22.708 | INFO | Perf report for the last 5.13916 seconds: | 20260505 08:32:22.708 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:32:22.708 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:22.708 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:22.708 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:22.708 | INFO | ANALYSIS : 65.3%, 2.9, 100.0ms, 0 | 20260505 08:32:22.708 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:22.708 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260505 08:32:22.708 | INFO | PRE_HQ : 63.4%, 1.9, 61.0ms, 0 | 20260505 08:32:22.709 | INFO | BAZWRITER : 65.6%, 4.1, 139.5ms, 0 | 20260505 08:32:22.709 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 08:32:22.709 | INFO | Uploaded 36GiB to GPU in 2.26037s (15.9266GiB/s) | 20260505 08:32:22.709 | INFO | Downloaded 38GiB to GPU in 2.36883s (16.0417GiB/s) | 20260505 08:32:27.848 | INFO | Analyzing chunk frames = [1798656,1799168) | 20260505 08:32:27.848 | INFO | MemInfo: { | 20260505 08:32:27.848 | INFO | "gpu" : " 17 GB", | 20260505 08:32:27.848 | INFO | "huge" : "", | 20260505 08:32:27.848 | INFO | "huge_shared" : " 27 GB", | 20260505 08:32:27.848 | INFO | "malloc" : " 14 GB", | 20260505 08:32:27.848 | INFO | "pinned" : " 29 GB" | 20260505 08:32:27.848 | INFO | } | 20260505 08:32:27.848 | INFO | Perf report for the last 5.13981 seconds: | 20260505 08:32:27.848 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:32:27.848 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:27.848 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:27.848 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:27.848 | INFO | ANALYSIS : 64.6%, 3.0, 100.8ms, 0 | 20260505 08:32:27.848 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:27.848 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:27.848 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 08:32:27.848 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:27.848 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:27.848 | INFO | Uploaded 36GiB to GPU in 2.24697s (16.0216GiB/s) | 20260505 08:32:27.849 | INFO | Downloaded 34GiB to GPU in 2.09598s (16.2215GiB/s) | 20260505 08:32:32.986 | INFO | Analyzing chunk frames = [1799168,1799680) | 20260505 08:32:32.986 | INFO | MemInfo: { | 20260505 08:32:32.986 | INFO | "gpu" : " 17 GB", | 20260505 08:32:32.986 | INFO | "huge" : "", | 20260505 08:32:32.986 | INFO | "huge_shared" : " 27 GB", | 20260505 08:32:32.986 | INFO | "malloc" : " 14 GB", | 20260505 08:32:32.986 | INFO | "pinned" : " 29 GB" | 20260505 08:32:32.986 | INFO | } | 20260505 08:32:32.986 | INFO | Perf report for the last 5.13765 seconds: | 20260505 08:32:32.986 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:32:32.986 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:32.986 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:32.986 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:32.986 | INFO | ANALYSIS : 63.6%, 3.0, 99.0ms, 0 | 20260505 08:32:32.986 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:32.986 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:32.986 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:32:32.986 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:32.986 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:32.986 | INFO | Uploaded 36GiB to GPU in 2.16526s (16.6262GiB/s) | 20260505 08:32:32.986 | INFO | Downloaded 34GiB to GPU in 2.06603s (16.4567GiB/s) | 20260505 08:32:38.126 | INFO | Analyzing chunk frames = [1799680,1800192) | 20260505 08:32:38.126 | INFO | MemInfo: { | 20260505 08:32:38.126 | INFO | "gpu" : " 17 GB", | 20260505 08:32:38.126 | INFO | "huge" : "", | 20260505 08:32:38.126 | INFO | "huge_shared" : " 27 GB", | 20260505 08:32:38.126 | INFO | "malloc" : " 14 GB", | 20260505 08:32:38.126 | INFO | "pinned" : " 29 GB" | 20260505 08:32:38.126 | INFO | } | 20260505 08:32:38.126 | INFO | Perf report for the last 5.14023 seconds: | 20260505 08:32:38.126 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:32:38.126 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:38.126 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:38.126 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:38.126 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260505 08:32:38.126 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:38.126 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:38.126 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:32:38.126 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:38.126 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:38.126 | INFO | Uploaded 36GiB to GPU in 2.17597s (16.5444GiB/s) | 20260505 08:32:38.126 | INFO | Downloaded 34GiB to GPU in 2.06548s (16.4611GiB/s) | 20260505 08:32:43.265 | INFO | Analyzing chunk frames = [1800192,1800704) | 20260505 08:32:43.265 | INFO | MemInfo: { | 20260505 08:32:43.265 | INFO | "gpu" : " 17 GB", | 20260505 08:32:43.265 | INFO | "huge" : "", | 20260505 08:32:43.265 | INFO | "huge_shared" : " 27 GB", | 20260505 08:32:43.265 | INFO | "malloc" : " 14 GB", | 20260505 08:32:43.265 | INFO | "pinned" : " 29 GB" | 20260505 08:32:43.265 | INFO | } | 20260505 08:32:43.265 | INFO | Perf report for the last 5.13912 seconds: | 20260505 08:32:43.265 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:32:43.265 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:43.265 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:43.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:43.265 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 08:32:43.265 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:43.265 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:43.266 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 08:32:43.266 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:43.266 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:43.266 | INFO | Uploaded 36GiB to GPU in 2.18846s (16.4499GiB/s) | 20260505 08:32:43.266 | INFO | Downloaded 34GiB to GPU in 2.0573s (16.5265GiB/s) | 20260505 08:32:48.405 | INFO | Analyzing chunk frames = [1800704,1801216) | 20260505 08:32:48.405 | INFO | MemInfo: { | 20260505 08:32:48.405 | INFO | "gpu" : " 17 GB", | 20260505 08:32:48.405 | INFO | "huge" : "", | 20260505 08:32:48.405 | INFO | "huge_shared" : " 27 GB", | 20260505 08:32:48.405 | INFO | "malloc" : " 14 GB", | 20260505 08:32:48.405 | INFO | "pinned" : " 29 GB" | 20260505 08:32:48.405 | INFO | } | 20260505 08:32:48.405 | INFO | Perf report for the last 5.14017 seconds: | 20260505 08:32:48.405 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:32:48.405 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:48.405 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:48.405 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:48.405 | INFO | ANALYSIS : 64.4%, 3.0, 100.6ms, 0 | 20260505 08:32:48.405 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:48.406 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:48.406 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:32:48.406 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:48.406 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:48.406 | INFO | Uploaded 36GiB to GPU in 2.25585s (15.9585GiB/s) | 20260505 08:32:48.406 | INFO | Downloaded 34GiB to GPU in 2.11298s (16.091GiB/s) | 20260505 08:32:53.543 | INFO | Analyzing chunk frames = [1801216,1801728) | 20260505 08:32:53.543 | INFO | MemInfo: { | 20260505 08:32:53.543 | INFO | "gpu" : " 17 GB", | 20260505 08:32:53.543 | INFO | "huge" : "", | 20260505 08:32:53.543 | INFO | "huge_shared" : " 27 GB", | 20260505 08:32:53.543 | INFO | "malloc" : " 14 GB", | 20260505 08:32:53.543 | INFO | "pinned" : " 29 GB" | 20260505 08:32:53.543 | INFO | } | 20260505 08:32:53.543 | INFO | Perf report for the last 5.13801 seconds: | 20260505 08:32:53.544 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:32:53.544 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:53.544 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:53.544 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:53.544 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 08:32:53.544 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:53.544 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:53.544 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 08:32:53.544 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:53.544 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:53.544 | INFO | Uploaded 36GiB to GPU in 2.20231s (16.3465GiB/s) | 20260505 08:32:53.544 | INFO | Downloaded 34GiB to GPU in 2.13573s (15.9196GiB/s) | 20260505 08:32:58.683 | INFO | Analyzing chunk frames = [1801728,1802240) | 20260505 08:32:58.683 | INFO | MemInfo: { | 20260505 08:32:58.683 | INFO | "gpu" : " 17 GB", | 20260505 08:32:58.683 | INFO | "huge" : "", | 20260505 08:32:58.683 | INFO | "huge_shared" : " 27 GB", | 20260505 08:32:58.683 | INFO | "malloc" : " 14 GB", | 20260505 08:32:58.683 | INFO | "pinned" : " 29 GB" | 20260505 08:32:58.683 | INFO | } | 20260505 08:32:58.683 | INFO | Perf report for the last 5.14 seconds: | 20260505 08:32:58.684 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:32:58.684 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:58.684 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:58.684 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:58.684 | INFO | ANALYSIS : 64.1%, 3.0, 100.2ms, 0 | 20260505 08:32:58.684 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:58.684 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:58.684 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:32:58.684 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:58.684 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:32:58.684 | INFO | Uploaded 36GiB to GPU in 2.19569s (16.3958GiB/s) | 20260505 08:32:58.684 | INFO | Downloaded 34GiB to GPU in 2.0395s (16.6708GiB/s) | 20260505 08:32:58.684 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 08:32:58.684 | INFO | Total mem usage: 15.3 GiB | 20260505 08:32:58.684 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 08:32:58.684 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -5.64%) | 20260505 08:32:58.684 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 08:32:58.684 | INFO | High Water Breakdown: | 20260505 08:32:58.684 | INFO | 60.9% : 8.84 GiB ( 94.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 08:32:58.684 | INFO | 38.9% : 5.65 GiB ( 85% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 08:32:58.684 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 08:32:58.684 | INFO | ------------------------------------------ | 20260505 08:32:58.684 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:32:58.684 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 08:32:58.684 | INFO | Total mem usage: 26.5 GiB | 20260505 08:32:58.684 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 08:32:58.684 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 08:32:58.684 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 08:32:58.684 | INFO | High Water Breakdown: | 20260505 08:32:58.684 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:32:58.684 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:32:58.684 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:32:58.684 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:32:58.684 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:32:58.684 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:32:58.684 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 08:32:58.684 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 08:32:58.684 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:32:58.684 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 08:32:58.684 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 08:32:58.684 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 08:32:58.684 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 08:32:58.684 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 08:32:58.684 | INFO | ------------------------------------------ | 20260505 08:32:58.684 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:32:58.685 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 08:32:58.685 | INFO | Total mem usage: 25.6 GiB | 20260505 08:32:58.685 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 08:32:58.685 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 08:32:58.685 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 08:32:58.685 | INFO | High Water Breakdown: | 20260505 08:32:58.685 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 08:32:58.685 | INFO | ------------------------------------------ | 20260505 08:32:58.685 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:32:58.685 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 08:32:58.685 | INFO | Total mem usage: 15.7 GiB | 20260505 08:32:58.685 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 08:32:58.685 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 08:32:58.685 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 08:32:58.685 | INFO | High Water Breakdown: | 20260505 08:32:58.685 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 08:32:58.685 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 08:32:58.685 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 08:32:58.685 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 08:32:58.685 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 08:32:58.685 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 08:32:58.685 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 08:32:58.685 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 08:32:58.685 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 08:32:58.685 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:32:58.685 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 08:32:58.685 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:32:58.685 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 08:32:58.685 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 08:32:58.685 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 08:32:58.685 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 08:32:58.685 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:32:58.685 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 08:32:58.685 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:32:58.685 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:32:58.685 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:32:58.685 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 08:32:58.685 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 08:32:58.685 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:32:58.685 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 08:32:58.685 | INFO | ------------------------------------------ | 20260505 08:32:58.685 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:32:58.685 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 08:32:58.685 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:32:58.685 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:32:58.685 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.55e+01, 1.37e+01 / 2.03e+00 | 20260505 08:32:58.685 | INFO | PerfReport: 0.29%, AggregateStats -- 9.31e-02 / 1.04e-01, 1.04e-02 / 4.61e-03 | 20260505 08:32:58.685 | INFO | PerfReport: 11.66%, Baseline -- 3.80e+00 / 4.45e+00, 6.24e-01 / 2.52e-01 | 20260505 08:32:58.685 | INFO | PerfReport: 3.33%, Histogram -- 1.09e+00 / 1.50e+00, 2.15e-01 / 8.62e-02 | 20260505 08:32:58.685 | INFO | PerfReport: 15.53%, DME -- 4.05e+01 / 4.42e+01, 4.12e+00 / 1.62e+00 | 20260505 08:32:58.685 | INFO | PerfReport: 56.43%, FrameLabeling -- 1.84e+01 / 2.02e+01, 1.92e+00 / 1.88e+00 | 20260505 08:32:58.685 | INFO | PerfReport: 7.47%, PulseAccumulating -- 2.44e+00 / 2.99e+00, 4.07e-01 / 2.53e-01 | 20260505 08:32:58.685 | INFO | PerfReport: 3.90%, Metrics -- 1.27e+00 / 1.58e+00, 3.59e-01 / 2.02e-01 | 20260505 08:32:58.685 | INFO | PerfReport: 1.39%, PulseSerializing -- 4.53e-01 / 5.04e-01, 8.57e-02 / 3.64e-02 | 20260505 08:32:58.685 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 08:32:58.685 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:32:58.685 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:32:58.685 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.33e+01 / 0.00e+00 | 20260505 08:32:58.685 | INFO | PerfReport: 22.58%, Upload -- 2.30e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260505 08:32:58.685 | INFO | PerfReport: 21.67%, Download -- 2.20e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260505 08:32:58.685 | INFO | PerfReport: 32.07%, Compute -- 3.26e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 08:32:58.685 | INFO | PerfReport: 1.72%, LockUpload -- 1.75e+00 / 0.00e+00, 7.36e+00 / 0.00e+00 | 20260505 08:32:58.685 | INFO | PerfReport: 1.04%, LockDownload -- 1.05e+00 / 0.00e+00, 2.50e+00 / 0.00e+00 | 20260505 08:32:58.685 | INFO | PerfReport: 20.92%, LockCompute -- 2.13e+01 / 0.00e+00, 1.78e+01 / 0.00e+00 | 20260505 08:32:58.685 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 2.13e-02 / 0.00e+00 | 20260505 08:32:58.823 | INFO | Starting a round of Baz IO | 20260505 08:33:02.102 | INFO | Populating RTMetrics report | 20260505 08:33:02.102 | INFO | RTMetrics report populated | 20260505 08:33:02.102 | INFO | Opening tmp RTMetrics json file | 20260505 08:33:02.116 | INFO | Serializing RTMetrics to json | 20260505 08:33:02.139 | INFO | Serialization done, swapping tmp file now... | 20260505 08:33:02.164 | INFO | RTMetrics done | 20260505 08:33:02.240 | INFO | Finished a round of Baz IO: Wrote 2.45 GiB in 3.417 seconds | 20260505 08:33:02.240 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 27.47% / 0.02% / 72.51% | 20260505 08:33:02.240 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.57646% / 3.17891e-05% / 8.34465e-05% | 20260505 08:33:02.240 | INFO | Baz IO overall: | 20260505 08:33:02.240 | INFO | BytesWritten : 1019861545105 | 20260505 08:33:02.241 | INFO | NumEvents : 484849457484 | 20260505 08:33:03.822 | INFO | Analyzing chunk frames = [1802240,1802752) | 20260505 08:33:03.822 | INFO | MemInfo: { | 20260505 08:33:03.822 | INFO | "gpu" : " 17 GB", | 20260505 08:33:03.822 | INFO | "huge" : "", | 20260505 08:33:03.822 | INFO | "huge_shared" : " 27 GB", | 20260505 08:33:03.822 | INFO | "malloc" : " 14 GB", | 20260505 08:33:03.822 | INFO | "pinned" : " 29 GB" | 20260505 08:33:03.822 | INFO | } | 20260505 08:33:03.822 | INFO | Perf report for the last 5.13904 seconds: | 20260505 08:33:03.822 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:33:03.822 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:03.822 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:03.822 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:03.822 | INFO | ANALYSIS : 65.4%, 2.9, 99.9ms, 0 | 20260505 08:33:03.823 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:03.823 | INFO | RT_METRICS : 47.1%, 1.0, 24.7ms, 0 | 20260505 08:33:03.823 | INFO | PRE_HQ : 63.1%, 1.9, 61.3ms, 0 | 20260505 08:33:03.823 | INFO | BAZWRITER : 65.7%, 4.1, 139.7ms, 0 | 20260505 08:33:03.823 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 08:33:03.823 | INFO | Uploaded 36GiB to GPU in 2.21865s (16.2261GiB/s) | 20260505 08:33:03.823 | INFO | Downloaded 38GiB to GPU in 2.34773s (16.1858GiB/s) | 20260505 08:33:08.961 | INFO | Analyzing chunk frames = [1802752,1803264) | 20260505 08:33:08.961 | INFO | MemInfo: { | 20260505 08:33:08.961 | INFO | "gpu" : " 17 GB", | 20260505 08:33:08.962 | INFO | "huge" : "", | 20260505 08:33:08.962 | INFO | "huge_shared" : " 27 GB", | 20260505 08:33:08.962 | INFO | "malloc" : " 14 GB", | 20260505 08:33:08.962 | INFO | "pinned" : " 29 GB" | 20260505 08:33:08.962 | INFO | } | 20260505 08:33:08.962 | INFO | Perf report for the last 5.13922 seconds: | 20260505 08:33:08.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:33:08.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:08.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:08.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:08.962 | INFO | ANALYSIS : 64.7%, 3.0, 101.1ms, 0 | 20260505 08:33:08.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:08.962 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:08.962 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:33:08.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:08.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:08.962 | INFO | Uploaded 36GiB to GPU in 2.24574s (16.0304GiB/s) | 20260505 08:33:08.962 | INFO | Downloaded 34GiB to GPU in 2.11185s (16.0997GiB/s) | 20260505 08:33:14.101 | INFO | Analyzing chunk frames = [1803264,1803776) | 20260505 08:33:14.101 | INFO | MemInfo: { | 20260505 08:33:14.101 | INFO | "gpu" : " 17 GB", | 20260505 08:33:14.101 | INFO | "huge" : "", | 20260505 08:33:14.101 | INFO | "huge_shared" : " 27 GB", | 20260505 08:33:14.101 | INFO | "malloc" : " 14 GB", | 20260505 08:33:14.101 | INFO | "pinned" : " 29 GB" | 20260505 08:33:14.101 | INFO | } | 20260505 08:33:14.101 | INFO | Perf report for the last 5.1393 seconds: | 20260505 08:33:14.101 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:33:14.101 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:14.101 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:14.101 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:14.101 | INFO | ANALYSIS : 63.6%, 3.0, 98.9ms, 0 | 20260505 08:33:14.101 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:14.101 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:14.101 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:33:14.101 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:14.101 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:14.101 | INFO | Uploaded 36GiB to GPU in 2.1689s (16.5982GiB/s) | 20260505 08:33:14.101 | INFO | Downloaded 34GiB to GPU in 2.06608s (16.4563GiB/s) | 20260505 08:33:19.240 | INFO | Analyzing chunk frames = [1803776,1804288) | 20260505 08:33:19.240 | INFO | MemInfo: { | 20260505 08:33:19.240 | INFO | "gpu" : " 17 GB", | 20260505 08:33:19.240 | INFO | "huge" : "", | 20260505 08:33:19.240 | INFO | "huge_shared" : " 27 GB", | 20260505 08:33:19.240 | INFO | "malloc" : " 14 GB", | 20260505 08:33:19.241 | INFO | "pinned" : " 29 GB" | 20260505 08:33:19.241 | INFO | } | 20260505 08:33:19.241 | INFO | Perf report for the last 5.13957 seconds: | 20260505 08:33:19.241 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:33:19.241 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:19.241 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:19.241 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:19.241 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260505 08:33:19.241 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:19.241 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:19.241 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:33:19.241 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:19.241 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:19.241 | INFO | Uploaded 36GiB to GPU in 2.17839s (16.5259GiB/s) | 20260505 08:33:19.241 | INFO | Downloaded 34GiB to GPU in 2.06641s (16.4537GiB/s) | 20260505 08:33:24.380 | INFO | Analyzing chunk frames = [1804288,1804800) | 20260505 08:33:24.380 | INFO | MemInfo: { | 20260505 08:33:24.380 | INFO | "gpu" : " 17 GB", | 20260505 08:33:24.380 | INFO | "huge" : "", | 20260505 08:33:24.380 | INFO | "huge_shared" : " 27 GB", | 20260505 08:33:24.380 | INFO | "malloc" : " 14 GB", | 20260505 08:33:24.380 | INFO | "pinned" : " 29 GB" | 20260505 08:33:24.380 | INFO | } | 20260505 08:33:24.380 | INFO | Perf report for the last 5.13927 seconds: | 20260505 08:33:24.380 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:33:24.380 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:24.380 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:24.380 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:24.380 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 08:33:24.380 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:24.380 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:24.380 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:33:24.380 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:24.380 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:24.380 | INFO | Uploaded 36GiB to GPU in 2.1853s (16.4737GiB/s) | 20260505 08:33:24.380 | INFO | Downloaded 34GiB to GPU in 2.06263s (16.4838GiB/s) | 20260505 08:33:29.520 | INFO | Analyzing chunk frames = [1804800,1805312) | 20260505 08:33:29.520 | INFO | MemInfo: { | 20260505 08:33:29.520 | INFO | "gpu" : " 17 GB", | 20260505 08:33:29.520 | INFO | "huge" : "", | 20260505 08:33:29.520 | INFO | "huge_shared" : " 27 GB", | 20260505 08:33:29.520 | INFO | "malloc" : " 14 GB", | 20260505 08:33:29.520 | INFO | "pinned" : " 29 GB" | 20260505 08:33:29.521 | INFO | } | 20260505 08:33:29.521 | INFO | Perf report for the last 5.14065 seconds: | 20260505 08:33:29.521 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:33:29.521 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:29.521 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:29.521 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:29.521 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 08:33:29.521 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:29.521 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:29.521 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 08:33:29.521 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:29.521 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:29.521 | INFO | Uploaded 36GiB to GPU in 2.17799s (16.529GiB/s) | 20260505 08:33:29.521 | INFO | Downloaded 34GiB to GPU in 2.05811s (16.52GiB/s) | 20260505 08:33:34.658 | INFO | Analyzing chunk frames = [1805312,1805824) | 20260505 08:33:34.658 | INFO | MemInfo: { | 20260505 08:33:34.658 | INFO | "gpu" : " 17 GB", | 20260505 08:33:34.658 | INFO | "huge" : "", | 20260505 08:33:34.659 | INFO | "huge_shared" : " 27 GB", | 20260505 08:33:34.659 | INFO | "malloc" : " 14 GB", | 20260505 08:33:34.659 | INFO | "pinned" : " 29 GB" | 20260505 08:33:34.659 | INFO | } | 20260505 08:33:34.659 | INFO | Perf report for the last 5.13802 seconds: | 20260505 08:33:34.659 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:33:34.659 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:34.659 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:34.659 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:34.659 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 08:33:34.659 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:34.659 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:34.659 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 08:33:34.659 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:34.659 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:34.659 | INFO | Uploaded 36GiB to GPU in 2.18039s (16.5108GiB/s) | 20260505 08:33:34.659 | INFO | Downloaded 34GiB to GPU in 2.15146s (15.8032GiB/s) | 20260505 08:33:39.798 | INFO | Analyzing chunk frames = [1805824,1806336) | 20260505 08:33:39.798 | INFO | MemInfo: { | 20260505 08:33:39.798 | INFO | "gpu" : " 17 GB", | 20260505 08:33:39.798 | INFO | "huge" : "", | 20260505 08:33:39.798 | INFO | "huge_shared" : " 27 GB", | 20260505 08:33:39.798 | INFO | "malloc" : " 14 GB", | 20260505 08:33:39.798 | INFO | "pinned" : " 29 GB" | 20260505 08:33:39.798 | INFO | } | 20260505 08:33:39.798 | INFO | Perf report for the last 5.13975 seconds: | 20260505 08:33:39.798 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:33:39.798 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:39.798 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:39.798 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:39.798 | INFO | ANALYSIS : 64.1%, 3.0, 100.2ms, 0 | 20260505 08:33:39.798 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:39.798 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:39.798 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 08:33:39.798 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:39.798 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:39.799 | INFO | Uploaded 36GiB to GPU in 2.19629s (16.3912GiB/s) | 20260505 08:33:39.799 | INFO | Downloaded 34GiB to GPU in 2.05136s (16.5744GiB/s) | 20260505 08:33:39.939 | INFO | Starting a round of Baz IO | 20260505 08:33:43.227 | INFO | Populating RTMetrics report | 20260505 08:33:43.228 | INFO | RTMetrics report populated | 20260505 08:33:43.228 | INFO | Opening tmp RTMetrics json file | 20260505 08:33:43.270 | INFO | Serializing RTMetrics to json | 20260505 08:33:43.289 | INFO | Serialization done, swapping tmp file now... | 20260505 08:33:43.295 | INFO | RTMetrics done | 20260505 08:33:43.351 | INFO | Finished a round of Baz IO: Wrote 2.45 GiB in 3.412 seconds | 20260505 08:33:43.351 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 27.45% / 0.02% / 72.53% | 20260505 08:33:43.352 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.57535% / 2.38419e-05% / 7.15256e-05% | 20260505 08:33:43.352 | INFO | Baz IO overall: | 20260505 08:33:43.352 | INFO | BytesWritten : 1022492599481 | 20260505 08:33:43.352 | INFO | NumEvents : 486109291240 | 20260505 08:33:44.939 | INFO | Analyzing chunk frames = [1806336,1806848) | 20260505 08:33:44.939 | INFO | MemInfo: { | 20260505 08:33:44.939 | INFO | "gpu" : " 17 GB", | 20260505 08:33:44.939 | INFO | "huge" : "", | 20260505 08:33:44.939 | INFO | "huge_shared" : " 27 GB", | 20260505 08:33:44.939 | INFO | "malloc" : " 14 GB", | 20260505 08:33:44.939 | INFO | "pinned" : " 29 GB" | 20260505 08:33:44.939 | INFO | } | 20260505 08:33:44.939 | INFO | Perf report for the last 5.14036 seconds: | 20260505 08:33:44.939 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:33:44.939 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:44.939 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:44.939 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:44.939 | INFO | ANALYSIS : 65.6%, 2.9, 100.6ms, 0 | 20260505 08:33:44.939 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:44.939 | INFO | RT_METRICS : 47.0%, 1.0, 24.7ms, 0 | 20260505 08:33:44.939 | INFO | PRE_HQ : 62.6%, 1.9, 61.0ms, 0 | 20260505 08:33:44.939 | INFO | BAZWRITER : 65.7%, 4.1, 139.1ms, 0 | 20260505 08:33:44.939 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 08:33:44.939 | INFO | Uploaded 36GiB to GPU in 2.19818s (16.3772GiB/s) | 20260505 08:33:44.939 | INFO | Downloaded 38GiB to GPU in 2.32084s (16.3734GiB/s) | 20260505 08:33:50.077 | INFO | Analyzing chunk frames = [1806848,1807360) | 20260505 08:33:50.077 | INFO | MemInfo: { | 20260505 08:33:50.077 | INFO | "gpu" : " 17 GB", | 20260505 08:33:50.077 | INFO | "huge" : "", | 20260505 08:33:50.077 | INFO | "huge_shared" : " 27 GB", | 20260505 08:33:50.077 | INFO | "malloc" : " 14 GB", | 20260505 08:33:50.077 | INFO | "pinned" : " 29 GB" | 20260505 08:33:50.077 | INFO | } | 20260505 08:33:50.077 | INFO | Perf report for the last 5.13809 seconds: | 20260505 08:33:50.077 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:33:50.077 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:50.077 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:50.077 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:50.077 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 08:33:50.077 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:50.077 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:50.077 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:33:50.077 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:50.077 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:50.077 | INFO | Uploaded 36GiB to GPU in 2.18876s (16.4476GiB/s) | 20260505 08:33:50.077 | INFO | Downloaded 34GiB to GPU in 2.08527s (16.3049GiB/s) | 20260505 08:33:55.216 | INFO | Analyzing chunk frames = [1807360,1807872) | 20260505 08:33:55.216 | INFO | MemInfo: { | 20260505 08:33:55.216 | INFO | "gpu" : " 17 GB", | 20260505 08:33:55.217 | INFO | "huge" : "", | 20260505 08:33:55.217 | INFO | "huge_shared" : " 27 GB", | 20260505 08:33:55.217 | INFO | "malloc" : " 14 GB", | 20260505 08:33:55.217 | INFO | "pinned" : " 29 GB" | 20260505 08:33:55.217 | INFO | } | 20260505 08:33:55.217 | INFO | Perf report for the last 5.13988 seconds: | 20260505 08:33:55.217 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:33:55.217 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:55.217 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:55.217 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:55.217 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 08:33:55.217 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:55.217 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:55.217 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:33:55.217 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:55.217 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:33:55.217 | INFO | Uploaded 36GiB to GPU in 2.16909s (16.5968GiB/s) | 20260505 08:33:55.217 | INFO | Downloaded 34GiB to GPU in 2.07053s (16.4209GiB/s) | 20260505 08:34:00.356 | INFO | Analyzing chunk frames = [1807872,1808384) | 20260505 08:34:00.356 | INFO | MemInfo: { | 20260505 08:34:00.356 | INFO | "gpu" : " 17 GB", | 20260505 08:34:00.356 | INFO | "huge" : "", | 20260505 08:34:00.356 | INFO | "huge_shared" : " 27 GB", | 20260505 08:34:00.356 | INFO | "malloc" : " 14 GB", | 20260505 08:34:00.356 | INFO | "pinned" : " 29 GB" | 20260505 08:34:00.356 | INFO | } | 20260505 08:34:00.356 | INFO | Perf report for the last 5.13933 seconds: | 20260505 08:34:00.356 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:34:00.356 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:00.356 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:00.356 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:00.356 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260505 08:34:00.356 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:00.356 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:00.356 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:34:00.356 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:00.356 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:00.356 | INFO | Uploaded 36GiB to GPU in 2.18121s (16.5046GiB/s) | 20260505 08:34:00.356 | INFO | Downloaded 34GiB to GPU in 2.06259s (16.4841GiB/s) | 20260505 08:34:05.494 | INFO | Analyzing chunk frames = [1808384,1808896) | 20260505 08:34:05.494 | INFO | MemInfo: { | 20260505 08:34:05.494 | INFO | "gpu" : " 17 GB", | 20260505 08:34:05.494 | INFO | "huge" : "", | 20260505 08:34:05.494 | INFO | "huge_shared" : " 27 GB", | 20260505 08:34:05.494 | INFO | "malloc" : " 14 GB", | 20260505 08:34:05.494 | INFO | "pinned" : " 29 GB" | 20260505 08:34:05.494 | INFO | } | 20260505 08:34:05.494 | INFO | Perf report for the last 5.13804 seconds: | 20260505 08:34:05.494 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:34:05.494 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:05.494 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:05.494 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:05.494 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 08:34:05.494 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:05.494 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:05.494 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 08:34:05.494 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:05.494 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:05.494 | INFO | Uploaded 36GiB to GPU in 2.18695s (16.4613GiB/s) | 20260505 08:34:05.494 | INFO | Downloaded 34GiB to GPU in 2.05903s (16.5126GiB/s) | 20260505 08:34:10.634 | INFO | Analyzing chunk frames = [1808896,1809408) | 20260505 08:34:10.634 | INFO | MemInfo: { | 20260505 08:34:10.634 | INFO | "gpu" : " 17 GB", | 20260505 08:34:10.634 | INFO | "huge" : "", | 20260505 08:34:10.634 | INFO | "huge_shared" : " 27 GB", | 20260505 08:34:10.634 | INFO | "malloc" : " 14 GB", | 20260505 08:34:10.634 | INFO | "pinned" : " 29 GB" | 20260505 08:34:10.634 | INFO | } | 20260505 08:34:10.634 | INFO | Perf report for the last 5.14022 seconds: | 20260505 08:34:10.634 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:34:10.634 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:10.634 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:10.634 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:10.634 | INFO | ANALYSIS : 63.5%, 3.0, 98.2ms, 0 | 20260505 08:34:10.634 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:10.634 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:10.634 | INFO | PRE_HQ : 29.4%, 1.0, 15.1ms, 0 | 20260505 08:34:10.634 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:10.634 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:10.635 | INFO | Uploaded 36GiB to GPU in 2.1769s (16.5373GiB/s) | 20260505 08:34:10.635 | INFO | Downloaded 34GiB to GPU in 2.04082s (16.66GiB/s) | 20260505 08:34:15.774 | INFO | Analyzing chunk frames = [1809408,1809920) | 20260505 08:34:15.774 | INFO | MemInfo: { | 20260505 08:34:15.775 | INFO | "gpu" : " 17 GB", | 20260505 08:34:15.775 | INFO | "huge" : "", | 20260505 08:34:15.775 | INFO | "huge_shared" : " 27 GB", | 20260505 08:34:15.775 | INFO | "malloc" : " 14 GB", | 20260505 08:34:15.775 | INFO | "pinned" : " 29 GB" | 20260505 08:34:15.775 | INFO | } | 20260505 08:34:15.775 | INFO | Perf report for the last 5.14032 seconds: | 20260505 08:34:15.775 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:34:15.775 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:15.775 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:15.775 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:15.775 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 08:34:15.775 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:15.775 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:15.775 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 08:34:15.775 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:15.775 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:15.775 | INFO | Uploaded 36GiB to GPU in 2.1807s (16.5085GiB/s) | 20260505 08:34:15.775 | INFO | Downloaded 34GiB to GPU in 2.15576s (15.7717GiB/s) | 20260505 08:34:20.912 | INFO | Analyzing chunk frames = [1809920,1810432) | 20260505 08:34:20.912 | INFO | MemInfo: { | 20260505 08:34:20.912 | INFO | "gpu" : " 17 GB", | 20260505 08:34:20.912 | INFO | "huge" : "", | 20260505 08:34:20.912 | INFO | "huge_shared" : " 27 GB", | 20260505 08:34:20.912 | INFO | "malloc" : " 14 GB", | 20260505 08:34:20.912 | INFO | "pinned" : " 29 GB" | 20260505 08:34:20.912 | INFO | } | 20260505 08:34:20.913 | INFO | Perf report for the last 5.13787 seconds: | 20260505 08:34:20.913 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:34:20.913 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:20.913 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:20.913 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:20.913 | INFO | ANALYSIS : 64.1%, 3.0, 100.0ms, 0 | 20260505 08:34:20.913 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:20.913 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:20.913 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 08:34:20.913 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:20.913 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:20.913 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 08:34:20.913 | INFO | Total mem usage: 15.3 GiB | 20260505 08:34:20.913 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 08:34:20.913 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.38%) | 20260505 08:34:20.913 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 08:34:20.913 | INFO | High Water Breakdown: | 20260505 08:34:20.913 | INFO | 60.8% : 8.77 GiB ( 93.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 08:34:20.913 | INFO | 39% : 5.62 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 08:34:20.913 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 08:34:20.913 | INFO | ------------------------------------------ | 20260505 08:34:20.913 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:34:20.913 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 08:34:20.913 | INFO | Total mem usage: 26.5 GiB | 20260505 08:34:20.913 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 08:34:20.913 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 08:34:20.913 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 08:34:20.913 | INFO | High Water Breakdown: | 20260505 08:34:20.913 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:34:20.913 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:34:20.913 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:34:20.913 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:34:20.913 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:34:20.913 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:34:20.913 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 08:34:20.913 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 08:34:20.913 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:34:20.913 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 08:34:20.913 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 08:34:20.913 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 08:34:20.913 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 08:34:20.913 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 08:34:20.913 | INFO | ------------------------------------------ | 20260505 08:34:20.913 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:34:20.913 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 08:34:20.913 | INFO | Total mem usage: 25.6 GiB | 20260505 08:34:20.913 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 08:34:20.913 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 08:34:20.913 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 08:34:20.913 | INFO | High Water Breakdown: | 20260505 08:34:20.913 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 08:34:20.913 | INFO | ------------------------------------------ | 20260505 08:34:20.913 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:34:20.913 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 08:34:20.913 | INFO | Total mem usage: 15.7 GiB | 20260505 08:34:20.913 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 08:34:20.913 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 08:34:20.913 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 08:34:20.913 | INFO | High Water Breakdown: | 20260505 08:34:20.913 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 08:34:20.913 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 08:34:20.913 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 08:34:20.913 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 08:34:20.913 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 08:34:20.913 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 08:34:20.913 | INFO | Uploaded 36GiB to GPU in 2.17185s (16.5757GiB/s) | 20260505 08:34:20.913 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 08:34:20.913 | INFO | Downloaded 34GiB to GPU in 2.04001s (16.6666GiB/s) | 20260505 08:34:20.913 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 08:34:20.913 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 08:34:20.913 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:34:20.913 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 08:34:20.913 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:34:20.913 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 08:34:20.913 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 08:34:20.913 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 08:34:20.913 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 08:34:20.913 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:34:20.913 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 08:34:20.914 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:34:20.914 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:34:20.914 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:34:20.914 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 08:34:20.914 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 08:34:20.914 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:34:20.914 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 08:34:20.914 | INFO | ------------------------------------------ | 20260505 08:34:20.914 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:34:20.914 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 08:34:20.914 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:34:20.914 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:34:20.914 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.53e+01, 1.35e+01 / 1.98e+00 | 20260505 08:34:20.914 | INFO | PerfReport: 0.29%, AggregateStats -- 9.31e-02 / 1.04e-01, 1.04e-02 / 4.74e-03 | 20260505 08:34:20.914 | INFO | PerfReport: 11.65%, Baseline -- 3.80e+00 / 4.53e+00, 6.26e-01 / 2.25e-01 | 20260505 08:34:20.914 | INFO | PerfReport: 3.33%, Histogram -- 1.09e+00 / 1.50e+00, 2.14e-01 / 9.65e-02 | 20260505 08:34:20.914 | INFO | PerfReport: 15.53%, DME -- 4.06e+01 / 4.44e+01, 4.14e+00 / 1.41e+00 | 20260505 08:34:20.914 | INFO | PerfReport: 56.42%, FrameLabeling -- 1.84e+01 / 1.97e+01, 1.91e+00 / 1.72e+00 | 20260505 08:34:20.914 | INFO | PerfReport: 7.49%, PulseAccumulating -- 2.44e+00 / 3.03e+00, 4.13e-01 / 2.21e-01 | 20260505 08:34:20.914 | INFO | PerfReport: 3.91%, Metrics -- 1.28e+00 / 1.58e+00, 3.66e-01 / 1.86e-01 | 20260505 08:34:20.914 | INFO | PerfReport: 1.39%, PulseSerializing -- 4.53e-01 / 5.03e-01, 8.59e-02 / 2.89e-02 | 20260505 08:34:20.914 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 08:34:20.914 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:34:20.914 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:34:20.914 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.31e+01 / 0.00e+00 | 20260505 08:34:20.914 | INFO | PerfReport: 22.43%, Upload -- 2.28e+01 / 0.00e+00, 8.14e+00 / 0.00e+00 | 20260505 08:34:20.914 | INFO | PerfReport: 21.63%, Download -- 2.20e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260505 08:34:20.914 | INFO | PerfReport: 32.12%, Compute -- 3.26e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 08:34:20.914 | INFO | PerfReport: 1.66%, LockUpload -- 1.69e+00 / 0.00e+00, 7.35e+00 / 0.00e+00 | 20260505 08:34:20.914 | INFO | PerfReport: 1.06%, LockDownload -- 1.08e+00 / 0.00e+00, 2.58e+00 / 0.00e+00 | 20260505 08:34:20.914 | INFO | PerfReport: 21.08%, LockCompute -- 2.14e+01 / 0.00e+00, 1.77e+01 / 0.00e+00 | 20260505 08:34:20.914 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.72e-02 / 0.00e+00 | 20260505 08:34:21.053 | INFO | Starting a round of Baz IO | 20260505 08:34:24.330 | INFO | Populating RTMetrics report | 20260505 08:34:24.331 | INFO | RTMetrics report populated | 20260505 08:34:24.331 | INFO | Opening tmp RTMetrics json file | 20260505 08:34:24.338 | INFO | Serializing RTMetrics to json | 20260505 08:34:24.340 | INFO | Serialization done, swapping tmp file now... | 20260505 08:34:24.343 | INFO | RTMetrics done | 20260505 08:34:24.473 | INFO | Finished a round of Baz IO: Wrote 2.451 GiB in 3.42 seconds | 20260505 08:34:24.473 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 27.43% / 0.02% / 72.55% | 20260505 08:34:24.473 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.5604% / 3.97364e-05% / 6.35783e-05% | 20260505 08:34:24.473 | INFO | Baz IO overall: | 20260505 08:34:24.473 | INFO | BytesWritten : 1025124579309 | 20260505 08:34:24.473 | INFO | NumEvents : 487368087615 | 20260505 08:34:26.052 | INFO | Analyzing chunk frames = [1810432,1810944) | 20260505 08:34:26.052 | INFO | MemInfo: { | 20260505 08:34:26.052 | INFO | "gpu" : " 17 GB", | 20260505 08:34:26.052 | INFO | "huge" : "", | 20260505 08:34:26.052 | INFO | "huge_shared" : " 27 GB", | 20260505 08:34:26.052 | INFO | "malloc" : " 14 GB", | 20260505 08:34:26.052 | INFO | "pinned" : " 29 GB" | 20260505 08:34:26.052 | INFO | } | 20260505 08:34:26.052 | INFO | Perf report for the last 5.13922 seconds: | 20260505 08:34:26.052 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:34:26.052 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:26.052 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:26.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:26.052 | INFO | ANALYSIS : 65.3%, 2.9, 100.3ms, 0 | 20260505 08:34:26.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:26.052 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260505 08:34:26.052 | INFO | PRE_HQ : 63.5%, 1.9, 61.5ms, 0 | 20260505 08:34:26.052 | INFO | BAZWRITER : 65.9%, 4.1, 140.1ms, 0 | 20260505 08:34:26.052 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 08:34:26.052 | INFO | Uploaded 36GiB to GPU in 2.25369s (15.9738GiB/s) | 20260505 08:34:26.052 | INFO | Downloaded 38GiB to GPU in 2.36047s (16.0985GiB/s) | 20260505 08:34:31.194 | INFO | Analyzing chunk frames = [1810944,1811456) | 20260505 08:34:31.194 | INFO | MemInfo: { | 20260505 08:34:31.194 | INFO | "gpu" : " 17 GB", | 20260505 08:34:31.194 | INFO | "huge" : "", | 20260505 08:34:31.194 | INFO | "huge_shared" : " 27 GB", | 20260505 08:34:31.194 | INFO | "malloc" : " 14 GB", | 20260505 08:34:31.194 | INFO | "pinned" : " 29 GB" | 20260505 08:34:31.194 | INFO | } | 20260505 08:34:31.194 | INFO | Perf report for the last 5.1426 seconds: | 20260505 08:34:31.194 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:34:31.194 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:31.194 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:31.194 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:31.194 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 08:34:31.194 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:31.194 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:31.194 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 08:34:31.194 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:31.194 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:31.195 | INFO | Uploaded 36GiB to GPU in 2.17445s (16.5559GiB/s) | 20260505 08:34:31.195 | INFO | Downloaded 34GiB to GPU in 2.08251s (16.3265GiB/s) | 20260505 08:34:36.331 | INFO | Analyzing chunk frames = [1811456,1811968) | 20260505 08:34:36.331 | INFO | MemInfo: { | 20260505 08:34:36.331 | INFO | "gpu" : " 17 GB", | 20260505 08:34:36.331 | INFO | "huge" : "", | 20260505 08:34:36.331 | INFO | "huge_shared" : " 27 GB", | 20260505 08:34:36.331 | INFO | "malloc" : " 14 GB", | 20260505 08:34:36.331 | INFO | "pinned" : " 29 GB" | 20260505 08:34:36.331 | INFO | } | 20260505 08:34:36.331 | INFO | Perf report for the last 5.13702 seconds: | 20260505 08:34:36.331 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:34:36.331 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:36.331 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:36.331 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:36.331 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260505 08:34:36.331 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:36.331 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:36.331 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 08:34:36.331 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:36.331 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:36.332 | INFO | Uploaded 36GiB to GPU in 2.16668s (16.6153GiB/s) | 20260505 08:34:36.332 | INFO | Downloaded 34GiB to GPU in 2.06508s (16.4643GiB/s) | 20260505 08:34:41.469 | INFO | Analyzing chunk frames = [1811968,1812480) | 20260505 08:34:41.469 | INFO | MemInfo: { | 20260505 08:34:41.469 | INFO | "gpu" : " 17 GB", | 20260505 08:34:41.469 | INFO | "huge" : "", | 20260505 08:34:41.470 | INFO | "huge_shared" : " 27 GB", | 20260505 08:34:41.470 | INFO | "malloc" : " 14 GB", | 20260505 08:34:41.470 | INFO | "pinned" : " 29 GB" | 20260505 08:34:41.470 | INFO | } | 20260505 08:34:41.470 | INFO | Perf report for the last 5.13814 seconds: | 20260505 08:34:41.470 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:34:41.470 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:41.470 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:41.470 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:41.470 | INFO | ANALYSIS : 63.6%, 3.0, 98.7ms, 0 | 20260505 08:34:41.470 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:41.470 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:41.470 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260505 08:34:41.470 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:41.470 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:41.470 | INFO | Uploaded 36GiB to GPU in 2.18341s (16.488GiB/s) | 20260505 08:34:41.470 | INFO | Downloaded 34GiB to GPU in 2.06613s (16.4559GiB/s) | 20260505 08:34:46.609 | INFO | Analyzing chunk frames = [1812480,1812992) | 20260505 08:34:46.609 | INFO | MemInfo: { | 20260505 08:34:46.609 | INFO | "gpu" : " 17 GB", | 20260505 08:34:46.609 | INFO | "huge" : "", | 20260505 08:34:46.609 | INFO | "huge_shared" : " 27 GB", | 20260505 08:34:46.609 | INFO | "malloc" : " 14 GB", | 20260505 08:34:46.609 | INFO | "pinned" : " 29 GB" | 20260505 08:34:46.609 | INFO | } | 20260505 08:34:46.609 | INFO | Perf report for the last 5.13962 seconds: | 20260505 08:34:46.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:34:46.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:46.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:46.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:46.609 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 08:34:46.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:46.609 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:46.609 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 08:34:46.609 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:46.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:46.609 | INFO | Uploaded 36GiB to GPU in 2.1862s (16.4669GiB/s) | 20260505 08:34:46.609 | INFO | Downloaded 34GiB to GPU in 2.05624s (16.5351GiB/s) | 20260505 08:34:51.749 | INFO | Analyzing chunk frames = [1812992,1813504) | 20260505 08:34:51.749 | INFO | MemInfo: { | 20260505 08:34:51.749 | INFO | "gpu" : " 17 GB", | 20260505 08:34:51.749 | INFO | "huge" : "", | 20260505 08:34:51.749 | INFO | "huge_shared" : " 27 GB", | 20260505 08:34:51.749 | INFO | "malloc" : " 14 GB", | 20260505 08:34:51.749 | INFO | "pinned" : " 29 GB" | 20260505 08:34:51.749 | INFO | } | 20260505 08:34:51.749 | INFO | Perf report for the last 5.14026 seconds: | 20260505 08:34:51.750 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:34:51.750 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:51.750 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:51.750 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:51.750 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 08:34:51.750 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:51.750 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:51.750 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 08:34:51.750 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:51.750 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:51.750 | INFO | Uploaded 36GiB to GPU in 2.18777s (16.4551GiB/s) | 20260505 08:34:51.750 | INFO | Downloaded 34GiB to GPU in 2.0537s (16.5555GiB/s) | 20260505 08:34:56.892 | INFO | Analyzing chunk frames = [1813504,1814016) | 20260505 08:34:56.892 | INFO | MemInfo: { | 20260505 08:34:56.892 | INFO | "gpu" : " 17 GB", | 20260505 08:34:56.892 | INFO | "huge" : "", | 20260505 08:34:56.892 | INFO | "huge_shared" : " 27 GB", | 20260505 08:34:56.892 | INFO | "malloc" : " 14 GB", | 20260505 08:34:56.892 | INFO | "pinned" : " 29 GB" | 20260505 08:34:56.892 | INFO | } | 20260505 08:34:56.892 | INFO | Perf report for the last 5.14227 seconds: | 20260505 08:34:56.892 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:34:56.892 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:56.892 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:56.892 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:56.892 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 08:34:56.892 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:56.892 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:56.892 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 08:34:56.892 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:56.892 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:34:56.892 | INFO | Uploaded 36GiB to GPU in 2.21395s (16.2606GiB/s) | 20260505 08:34:56.892 | INFO | Downloaded 34GiB to GPU in 2.1492s (15.8198GiB/s) | 20260505 08:35:02.028 | INFO | Analyzing chunk frames = [1814016,1814528) | 20260505 08:35:02.028 | INFO | MemInfo: { | 20260505 08:35:02.028 | INFO | "gpu" : " 17 GB", | 20260505 08:35:02.028 | INFO | "huge" : "", | 20260505 08:35:02.028 | INFO | "huge_shared" : " 27 GB", | 20260505 08:35:02.028 | INFO | "malloc" : " 14 GB", | 20260505 08:35:02.028 | INFO | "pinned" : " 29 GB" | 20260505 08:35:02.028 | INFO | } | 20260505 08:35:02.028 | INFO | Perf report for the last 5.13609 seconds: | 20260505 08:35:02.028 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:35:02.028 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:02.028 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:02.028 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:02.028 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 08:35:02.028 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:02.028 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:02.028 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 08:35:02.028 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:02.028 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:02.028 | INFO | Uploaded 36GiB to GPU in 2.19514s (16.3999GiB/s) | 20260505 08:35:02.028 | INFO | Downloaded 34GiB to GPU in 2.02671s (16.776GiB/s) | 20260505 08:35:02.168 | INFO | Starting a round of Baz IO | 20260505 08:35:05.437 | INFO | Populating RTMetrics report | 20260505 08:35:05.437 | INFO | RTMetrics report populated | 20260505 08:35:05.437 | INFO | Opening tmp RTMetrics json file | 20260505 08:35:05.446 | INFO | Serializing RTMetrics to json | 20260505 08:35:05.449 | INFO | Serialization done, swapping tmp file now... | 20260505 08:35:05.451 | INFO | RTMetrics done | 20260505 08:35:05.575 | INFO | Finished a round of Baz IO: Wrote 2.454 GiB in 3.407 seconds | 20260505 08:35:05.575 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 27.41% / 0.02% / 72.57% | 20260505 08:35:05.575 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.55309% / 2.38419e-05% / 6.35783e-05% | 20260505 08:35:05.575 | INFO | Baz IO overall: | 20260505 08:35:05.575 | INFO | BytesWritten : 1027759252532 | 20260505 08:35:05.575 | INFO | NumEvents : 488628382339 | 20260505 08:35:07.167 | INFO | Analyzing chunk frames = [1814528,1815040) | 20260505 08:35:07.167 | INFO | MemInfo: { | 20260505 08:35:07.167 | INFO | "gpu" : " 17 GB", | 20260505 08:35:07.167 | INFO | "huge" : "", | 20260505 08:35:07.167 | INFO | "huge_shared" : " 27 GB", | 20260505 08:35:07.167 | INFO | "malloc" : " 14 GB", | 20260505 08:35:07.167 | INFO | "pinned" : " 29 GB" | 20260505 08:35:07.167 | INFO | } | 20260505 08:35:07.167 | INFO | Perf report for the last 5.13932 seconds: | 20260505 08:35:07.167 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:35:07.167 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:07.167 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:07.167 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:07.167 | INFO | ANALYSIS : 65.2%, 2.9, 99.5ms, 0 | 20260505 08:35:07.167 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:07.167 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 08:35:07.167 | INFO | PRE_HQ : 63.4%, 1.9, 61.3ms, 0 | 20260505 08:35:07.167 | INFO | BAZWRITER : 65.6%, 4.2, 141.4ms, 0 | 20260505 08:35:07.167 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 08:35:07.167 | INFO | Uploaded 36GiB to GPU in 2.25349s (15.9753GiB/s) | 20260505 08:35:07.167 | INFO | Downloaded 38GiB to GPU in 2.38574s (15.9279GiB/s) | 20260505 08:35:12.305 | INFO | Analyzing chunk frames = [1815040,1815552) | 20260505 08:35:12.305 | INFO | MemInfo: { | 20260505 08:35:12.305 | INFO | "gpu" : " 17 GB", | 20260505 08:35:12.305 | INFO | "huge" : "", | 20260505 08:35:12.305 | INFO | "huge_shared" : " 27 GB", | 20260505 08:35:12.305 | INFO | "malloc" : " 14 GB", | 20260505 08:35:12.305 | INFO | "pinned" : " 29 GB" | 20260505 08:35:12.305 | INFO | } | 20260505 08:35:12.305 | INFO | Perf report for the last 5.13802 seconds: | 20260505 08:35:12.305 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:35:12.305 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:12.305 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:12.305 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:12.305 | INFO | ANALYSIS : 64.4%, 3.0, 100.6ms, 0 | 20260505 08:35:12.305 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:12.305 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:12.305 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:35:12.305 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:12.305 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:12.306 | INFO | Uploaded 36GiB to GPU in 2.21556s (16.2487GiB/s) | 20260505 08:35:12.306 | INFO | Downloaded 34GiB to GPU in 2.08782s (16.2849GiB/s) | 20260505 08:35:17.445 | INFO | Analyzing chunk frames = [1815552,1816064) | 20260505 08:35:17.445 | INFO | MemInfo: { | 20260505 08:35:17.445 | INFO | "gpu" : " 17 GB", | 20260505 08:35:17.445 | INFO | "huge" : "", | 20260505 08:35:17.445 | INFO | "huge_shared" : " 27 GB", | 20260505 08:35:17.445 | INFO | "malloc" : " 14 GB", | 20260505 08:35:17.445 | INFO | "pinned" : " 29 GB" | 20260505 08:35:17.445 | INFO | } | 20260505 08:35:17.445 | INFO | Perf report for the last 5.13972 seconds: | 20260505 08:35:17.445 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:35:17.445 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:17.445 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:17.445 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:17.445 | INFO | ANALYSIS : 63.6%, 3.0, 99.0ms, 0 | 20260505 08:35:17.445 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:17.445 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:17.445 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:35:17.445 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:17.445 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:17.445 | INFO | Uploaded 36GiB to GPU in 2.17127s (16.5801GiB/s) | 20260505 08:35:17.445 | INFO | Downloaded 34GiB to GPU in 2.06694s (16.4495GiB/s) | 20260505 08:35:22.584 | INFO | Analyzing chunk frames = [1816064,1816576) | 20260505 08:35:22.584 | INFO | MemInfo: { | 20260505 08:35:22.584 | INFO | "gpu" : " 17 GB", | 20260505 08:35:22.584 | INFO | "huge" : "", | 20260505 08:35:22.584 | INFO | "huge_shared" : " 27 GB", | 20260505 08:35:22.584 | INFO | "malloc" : " 14 GB", | 20260505 08:35:22.584 | INFO | "pinned" : " 29 GB" | 20260505 08:35:22.584 | INFO | } | 20260505 08:35:22.584 | INFO | Perf report for the last 5.13949 seconds: | 20260505 08:35:22.584 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:35:22.584 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:22.584 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:22.584 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:22.584 | INFO | ANALYSIS : 63.6%, 3.0, 98.8ms, 0 | 20260505 08:35:22.584 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:22.584 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:22.584 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:35:22.584 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:22.584 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:22.585 | INFO | Uploaded 36GiB to GPU in 2.18949s (16.4422GiB/s) | 20260505 08:35:22.585 | INFO | Downloaded 34GiB to GPU in 2.06521s (16.4632GiB/s) | 20260505 08:35:27.725 | INFO | Analyzing chunk frames = [1816576,1817088) | 20260505 08:35:27.725 | INFO | MemInfo: { | 20260505 08:35:27.725 | INFO | "gpu" : " 17 GB", | 20260505 08:35:27.725 | INFO | "huge" : "", | 20260505 08:35:27.725 | INFO | "huge_shared" : " 27 GB", | 20260505 08:35:27.725 | INFO | "malloc" : " 14 GB", | 20260505 08:35:27.725 | INFO | "pinned" : " 29 GB" | 20260505 08:35:27.725 | INFO | } | 20260505 08:35:27.725 | INFO | Perf report for the last 5.14026 seconds: | 20260505 08:35:27.725 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:35:27.725 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:27.725 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:27.725 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:27.725 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 08:35:27.725 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:27.725 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:27.725 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 08:35:27.725 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:27.725 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:27.725 | INFO | Uploaded 36GiB to GPU in 2.18983s (16.4396GiB/s) | 20260505 08:35:27.725 | INFO | Downloaded 34GiB to GPU in 2.05936s (16.51GiB/s) | 20260505 08:35:32.864 | INFO | Analyzing chunk frames = [1817088,1817600) | 20260505 08:35:32.864 | INFO | MemInfo: { | 20260505 08:35:32.864 | INFO | "gpu" : " 17 GB", | 20260505 08:35:32.864 | INFO | "huge" : "", | 20260505 08:35:32.865 | INFO | "huge_shared" : " 27 GB", | 20260505 08:35:32.865 | INFO | "malloc" : " 14 GB", | 20260505 08:35:32.865 | INFO | "pinned" : " 29 GB" | 20260505 08:35:32.865 | INFO | } | 20260505 08:35:32.865 | INFO | Perf report for the last 5.14003 seconds: | 20260505 08:35:32.865 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:35:32.865 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:32.865 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:32.865 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:32.865 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 08:35:32.865 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:32.865 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:32.865 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 08:35:32.865 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:32.865 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:32.865 | INFO | Uploaded 36GiB to GPU in 2.19251s (16.4196GiB/s) | 20260505 08:35:32.865 | INFO | Downloaded 34GiB to GPU in 2.05843s (16.5174GiB/s) | 20260505 08:35:38.003 | INFO | Analyzing chunk frames = [1817600,1818112) | 20260505 08:35:38.003 | INFO | MemInfo: { | 20260505 08:35:38.003 | INFO | "gpu" : " 17 GB", | 20260505 08:35:38.003 | INFO | "huge" : "", | 20260505 08:35:38.003 | INFO | "huge_shared" : " 27 GB", | 20260505 08:35:38.003 | INFO | "malloc" : " 14 GB", | 20260505 08:35:38.003 | INFO | "pinned" : " 29 GB" | 20260505 08:35:38.003 | INFO | } | 20260505 08:35:38.003 | INFO | Perf report for the last 5.1384 seconds: | 20260505 08:35:38.003 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:35:38.003 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:38.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:38.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:38.003 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 08:35:38.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:38.003 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:38.003 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 08:35:38.003 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:38.003 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:38.004 | INFO | Uploaded 36GiB to GPU in 2.18459s (16.4791GiB/s) | 20260505 08:35:38.004 | INFO | Downloaded 34GiB to GPU in 2.14879s (15.8228GiB/s) | 20260505 08:35:43.142 | INFO | Analyzing chunk frames = [1818112,1818624) | 20260505 08:35:43.142 | INFO | MemInfo: { | 20260505 08:35:43.142 | INFO | "gpu" : " 17 GB", | 20260505 08:35:43.142 | INFO | "huge" : "", | 20260505 08:35:43.142 | INFO | "huge_shared" : " 27 GB", | 20260505 08:35:43.142 | INFO | "malloc" : " 14 GB", | 20260505 08:35:43.142 | INFO | "pinned" : " 29 GB" | 20260505 08:35:43.142 | INFO | } | 20260505 08:35:43.142 | INFO | Perf report for the last 5.1388 seconds: | 20260505 08:35:43.142 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:35:43.142 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:43.142 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:43.142 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:43.142 | INFO | ANALYSIS : 64.1%, 3.0, 100.0ms, 0 | 20260505 08:35:43.142 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:43.142 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:43.142 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:35:43.142 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:43.142 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:43.142 | INFO | Uploaded 36GiB to GPU in 2.18065s (16.5089GiB/s) | 20260505 08:35:43.142 | INFO | Downloaded 34GiB to GPU in 2.02991s (16.7495GiB/s) | 20260505 08:35:43.142 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 08:35:43.142 | INFO | Total mem usage: 15.3 GiB | 20260505 08:35:43.142 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 08:35:43.142 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.39%) | 20260505 08:35:43.142 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 08:35:43.143 | INFO | High Water Breakdown: | 20260505 08:35:43.143 | INFO | 60.8% : 8.76 GiB ( 93.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 08:35:43.143 | INFO | 39% : 5.62 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 08:35:43.143 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 08:35:43.143 | INFO | ------------------------------------------ | 20260505 08:35:43.143 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:35:43.143 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 08:35:43.143 | INFO | Total mem usage: 26.5 GiB | 20260505 08:35:43.143 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 08:35:43.143 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 08:35:43.143 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 08:35:43.143 | INFO | High Water Breakdown: | 20260505 08:35:43.143 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:35:43.143 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:35:43.143 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:35:43.143 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:35:43.143 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:35:43.143 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:35:43.143 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 08:35:43.143 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 08:35:43.143 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:35:43.143 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 08:35:43.143 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 08:35:43.143 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 08:35:43.143 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 08:35:43.143 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 08:35:43.143 | INFO | ------------------------------------------ | 20260505 08:35:43.143 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:35:43.143 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 08:35:43.143 | INFO | Total mem usage: 25.6 GiB | 20260505 08:35:43.143 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 08:35:43.143 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 08:35:43.143 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 08:35:43.143 | INFO | High Water Breakdown: | 20260505 08:35:43.143 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 08:35:43.143 | INFO | ------------------------------------------ | 20260505 08:35:43.143 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:35:43.143 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 08:35:43.143 | INFO | Total mem usage: 15.7 GiB | 20260505 08:35:43.143 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 08:35:43.143 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 08:35:43.143 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 08:35:43.143 | INFO | High Water Breakdown: | 20260505 08:35:43.143 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 08:35:43.143 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 08:35:43.143 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 08:35:43.143 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 08:35:43.143 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 08:35:43.143 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 08:35:43.143 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 08:35:43.143 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 08:35:43.143 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 08:35:43.143 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:35:43.143 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 08:35:43.143 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:35:43.143 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 08:35:43.143 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 08:35:43.143 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 08:35:43.143 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 08:35:43.143 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:35:43.143 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 08:35:43.143 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:35:43.143 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:35:43.143 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:35:43.143 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 08:35:43.143 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 08:35:43.143 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:35:43.143 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 08:35:43.143 | INFO | ------------------------------------------ | 20260505 08:35:43.143 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:35:43.144 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 08:35:43.144 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:35:43.144 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:35:43.144 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.51e+01, 1.36e+01 / 1.80e+00 | 20260505 08:35:43.144 | INFO | PerfReport: 0.29%, AggregateStats -- 9.31e-02 / 1.03e-01, 1.02e-02 / 5.68e-03 | 20260505 08:35:43.144 | INFO | PerfReport: 11.66%, Baseline -- 3.80e+00 / 4.44e+00, 6.19e-01 / 3.10e-01 | 20260505 08:35:43.144 | INFO | PerfReport: 3.32%, Histogram -- 1.08e+00 / 1.46e+00, 2.09e-01 / 1.30e-01 | 20260505 08:35:43.144 | INFO | PerfReport: 15.52%, DME -- 4.05e+01 / 4.41e+01, 4.06e+00 / 1.47e+00 | 20260505 08:35:43.144 | INFO | PerfReport: 56.42%, FrameLabeling -- 1.84e+01 / 1.98e+01, 1.91e+00 / 1.75e+00 | 20260505 08:35:43.144 | INFO | PerfReport: 7.50%, PulseAccumulating -- 2.44e+00 / 3.05e+00, 4.17e-01 / 2.28e-01 | 20260505 08:35:43.144 | INFO | PerfReport: 3.90%, Metrics -- 1.27e+00 / 1.62e+00, 3.62e-01 / 1.68e-01 | 20260505 08:35:43.144 | INFO | PerfReport: 1.39%, PulseSerializing -- 4.54e-01 / 5.13e-01, 8.71e-02 / 2.84e-02 | 20260505 08:35:43.144 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 08:35:43.144 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:35:43.144 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:35:43.144 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.30e+01 / 0.00e+00 | 20260505 08:35:43.144 | INFO | PerfReport: 22.51%, Upload -- 2.29e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260505 08:35:43.144 | INFO | PerfReport: 21.63%, Download -- 2.20e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260505 08:35:43.144 | INFO | PerfReport: 32.07%, Compute -- 3.26e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 08:35:43.144 | INFO | PerfReport: 1.69%, LockUpload -- 1.71e+00 / 0.00e+00, 7.45e+00 / 0.00e+00 | 20260505 08:35:43.144 | INFO | PerfReport: 1.08%, LockDownload -- 1.10e+00 / 0.00e+00, 2.62e+00 / 0.00e+00 | 20260505 08:35:43.144 | INFO | PerfReport: 21.01%, LockCompute -- 2.13e+01 / 0.00e+00, 1.77e+01 / 0.00e+00 | 20260505 08:35:43.144 | INFO | PerfReport: 0.02%, Misc -- 1.69e-02 / 0.00e+00, 4.71e-02 / 0.00e+00 | 20260505 08:35:43.281 | INFO | Starting a round of Baz IO | 20260505 08:35:46.563 | INFO | Populating RTMetrics report | 20260505 08:35:46.564 | INFO | RTMetrics report populated | 20260505 08:35:46.564 | INFO | Opening tmp RTMetrics json file | 20260505 08:35:46.571 | INFO | Serializing RTMetrics to json | 20260505 08:35:46.572 | INFO | Serialization done, swapping tmp file now... | 20260505 08:35:46.572 | INFO | RTMetrics done | 20260505 08:35:46.721 | INFO | Finished a round of Baz IO: Wrote 2.45 GiB in 3.439 seconds | 20260505 08:35:46.721 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 27.39% / 0.02% / 72.59% | 20260505 08:35:46.721 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.53083% / 2.38419e-05% / 6.35783e-05% | 20260505 08:35:46.721 | INFO | Baz IO overall: | 20260505 08:35:46.721 | INFO | BytesWritten : 1030389472294 | 20260505 08:35:46.721 | INFO | NumEvents : 489885653522 | 20260505 08:35:48.281 | INFO | Analyzing chunk frames = [1818624,1819136) | 20260505 08:35:48.281 | INFO | MemInfo: { | 20260505 08:35:48.281 | INFO | "gpu" : " 17 GB", | 20260505 08:35:48.281 | INFO | "huge" : "", | 20260505 08:35:48.281 | INFO | "huge_shared" : " 27 GB", | 20260505 08:35:48.281 | INFO | "malloc" : " 14 GB", | 20260505 08:35:48.281 | INFO | "pinned" : " 29 GB" | 20260505 08:35:48.281 | INFO | } | 20260505 08:35:48.281 | INFO | Perf report for the last 5.13912 seconds: | 20260505 08:35:48.281 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:35:48.281 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:48.281 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:48.281 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:48.281 | INFO | ANALYSIS : 65.4%, 2.9, 100.1ms, 0 | 20260505 08:35:48.281 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:48.281 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260505 08:35:48.281 | INFO | PRE_HQ : 63.1%, 1.9, 61.2ms, 0 | 20260505 08:35:48.281 | INFO | BAZWRITER : 66.3%, 4.1, 140.9ms, 0 | 20260505 08:35:48.281 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 08:35:48.281 | INFO | Uploaded 36GiB to GPU in 2.21973s (16.2182GiB/s) | 20260505 08:35:48.282 | INFO | Downloaded 38GiB to GPU in 2.3473s (16.1888GiB/s) | 20260505 08:35:53.421 | INFO | Analyzing chunk frames = [1819136,1819648) | 20260505 08:35:53.421 | INFO | MemInfo: { | 20260505 08:35:53.421 | INFO | "gpu" : " 17 GB", | 20260505 08:35:53.421 | INFO | "huge" : "", | 20260505 08:35:53.421 | INFO | "huge_shared" : " 27 GB", | 20260505 08:35:53.421 | INFO | "malloc" : " 14 GB", | 20260505 08:35:53.421 | INFO | "pinned" : " 29 GB" | 20260505 08:35:53.421 | INFO | } | 20260505 08:35:53.421 | INFO | Perf report for the last 5.13987 seconds: | 20260505 08:35:53.421 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:35:53.421 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:53.421 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:53.421 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:53.421 | INFO | ANALYSIS : 64.6%, 3.0, 100.8ms, 0 | 20260505 08:35:53.421 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:53.421 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:53.421 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 08:35:53.421 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:53.421 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:53.421 | INFO | Uploaded 36GiB to GPU in 2.24736s (16.0188GiB/s) | 20260505 08:35:53.421 | INFO | Downloaded 34GiB to GPU in 2.08979s (16.2696GiB/s) | 20260505 08:35:58.560 | INFO | Analyzing chunk frames = [1819648,1820160) | 20260505 08:35:58.560 | INFO | MemInfo: { | 20260505 08:35:58.560 | INFO | "gpu" : " 17 GB", | 20260505 08:35:58.560 | INFO | "huge" : "", | 20260505 08:35:58.560 | INFO | "huge_shared" : " 27 GB", | 20260505 08:35:58.560 | INFO | "malloc" : " 14 GB", | 20260505 08:35:58.560 | INFO | "pinned" : " 29 GB" | 20260505 08:35:58.560 | INFO | } | 20260505 08:35:58.560 | INFO | Perf report for the last 5.13898 seconds: | 20260505 08:35:58.560 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:35:58.560 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:58.560 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:58.560 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:58.560 | INFO | ANALYSIS : 63.7%, 3.0, 99.1ms, 0 | 20260505 08:35:58.560 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:58.560 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:58.560 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:35:58.560 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:58.560 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:35:58.560 | INFO | Uploaded 36GiB to GPU in 2.16979s (16.5915GiB/s) | 20260505 08:35:58.561 | INFO | Downloaded 34GiB to GPU in 2.06595s (16.4573GiB/s) | 20260505 08:36:03.699 | INFO | Analyzing chunk frames = [1820160,1820672) | 20260505 08:36:03.699 | INFO | MemInfo: { | 20260505 08:36:03.699 | INFO | "gpu" : " 17 GB", | 20260505 08:36:03.699 | INFO | "huge" : "", | 20260505 08:36:03.699 | INFO | "huge_shared" : " 27 GB", | 20260505 08:36:03.699 | INFO | "malloc" : " 14 GB", | 20260505 08:36:03.699 | INFO | "pinned" : " 29 GB" | 20260505 08:36:03.699 | INFO | } | 20260505 08:36:03.699 | INFO | Perf report for the last 5.13934 seconds: | 20260505 08:36:03.700 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:36:03.700 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:03.700 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:03.700 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:03.700 | INFO | ANALYSIS : 63.6%, 3.0, 98.8ms, 0 | 20260505 08:36:03.700 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:03.700 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:03.700 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:36:03.700 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:03.700 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:03.700 | INFO | Uploaded 36GiB to GPU in 2.17826s (16.527GiB/s) | 20260505 08:36:03.700 | INFO | Downloaded 34GiB to GPU in 2.07029s (16.4228GiB/s) | 20260505 08:36:08.840 | INFO | Analyzing chunk frames = [1820672,1821184) | 20260505 08:36:08.840 | INFO | MemInfo: { | 20260505 08:36:08.840 | INFO | "gpu" : " 17 GB", | 20260505 08:36:08.840 | INFO | "huge" : "", | 20260505 08:36:08.840 | INFO | "huge_shared" : " 27 GB", | 20260505 08:36:08.840 | INFO | "malloc" : " 14 GB", | 20260505 08:36:08.840 | INFO | "pinned" : " 29 GB" | 20260505 08:36:08.840 | INFO | } | 20260505 08:36:08.840 | INFO | Perf report for the last 5.14047 seconds: | 20260505 08:36:08.840 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:36:08.840 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:08.840 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:08.840 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:08.840 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 08:36:08.840 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:08.840 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:08.840 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:36:08.840 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:08.840 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:08.840 | INFO | Uploaded 36GiB to GPU in 2.17749s (16.5328GiB/s) | 20260505 08:36:08.840 | INFO | Downloaded 34GiB to GPU in 2.05927s (16.5107GiB/s) | 20260505 08:36:13.979 | INFO | Analyzing chunk frames = [1821184,1821696) | 20260505 08:36:13.979 | INFO | MemInfo: { | 20260505 08:36:13.979 | INFO | "gpu" : " 17 GB", | 20260505 08:36:13.979 | INFO | "huge" : "", | 20260505 08:36:13.979 | INFO | "huge_shared" : " 27 GB", | 20260505 08:36:13.980 | INFO | "malloc" : " 14 GB", | 20260505 08:36:13.980 | INFO | "pinned" : " 29 GB" | 20260505 08:36:13.980 | INFO | } | 20260505 08:36:13.981 | INFO | Perf report for the last 5.14068 seconds: | 20260505 08:36:13.981 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:36:13.981 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:13.981 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:13.981 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:13.981 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 08:36:13.981 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:13.981 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:13.981 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 08:36:13.981 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:13.981 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:13.981 | INFO | Uploaded 36GiB to GPU in 2.19158s (16.4265GiB/s) | 20260505 08:36:13.981 | INFO | Downloaded 34GiB to GPU in 2.05768s (16.5235GiB/s) | 20260505 08:36:19.117 | INFO | Analyzing chunk frames = [1821696,1822208) | 20260505 08:36:19.117 | INFO | MemInfo: { | 20260505 08:36:19.117 | INFO | "gpu" : " 17 GB", | 20260505 08:36:19.117 | INFO | "huge" : "", | 20260505 08:36:19.117 | INFO | "huge_shared" : " 27 GB", | 20260505 08:36:19.117 | INFO | "malloc" : " 14 GB", | 20260505 08:36:19.117 | INFO | "pinned" : " 29 GB" | 20260505 08:36:19.117 | INFO | } | 20260505 08:36:19.117 | INFO | Perf report for the last 5.13674 seconds: | 20260505 08:36:19.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:36:19.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:19.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:19.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:19.117 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 08:36:19.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:19.117 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:19.117 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 08:36:19.117 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:19.117 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:19.117 | INFO | Uploaded 36GiB to GPU in 2.19161s (16.4263GiB/s) | 20260505 08:36:19.118 | INFO | Downloaded 34GiB to GPU in 2.16856s (15.6786GiB/s) | 20260505 08:36:24.256 | INFO | Analyzing chunk frames = [1822208,1822720) | 20260505 08:36:24.256 | INFO | MemInfo: { | 20260505 08:36:24.256 | INFO | "gpu" : " 17 GB", | 20260505 08:36:24.257 | INFO | "huge" : "", | 20260505 08:36:24.257 | INFO | "huge_shared" : " 27 GB", | 20260505 08:36:24.257 | INFO | "malloc" : " 14 GB", | 20260505 08:36:24.257 | INFO | "pinned" : " 29 GB" | 20260505 08:36:24.257 | INFO | } | 20260505 08:36:24.257 | INFO | Perf report for the last 5.13933 seconds: | 20260505 08:36:24.257 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:36:24.257 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:24.257 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:24.257 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:24.257 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 08:36:24.257 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:24.257 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:24.257 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:36:24.257 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:24.257 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:24.257 | INFO | Uploaded 36GiB to GPU in 2.17783s (16.5302GiB/s) | 20260505 08:36:24.257 | INFO | Downloaded 34GiB to GPU in 2.0416s (16.6536GiB/s) | 20260505 08:36:24.396 | INFO | Starting a round of Baz IO | 20260505 08:36:27.666 | INFO | Populating RTMetrics report | 20260505 08:36:27.667 | INFO | RTMetrics report populated | 20260505 08:36:27.667 | INFO | Opening tmp RTMetrics json file | 20260505 08:36:27.702 | INFO | Serializing RTMetrics to json | 20260505 08:36:27.741 | INFO | Serialization done, swapping tmp file now... | 20260505 08:36:27.782 | INFO | RTMetrics done | 20260505 08:36:27.804 | INFO | Finished a round of Baz IO: Wrote 2.453 GiB in 3.407 seconds | 20260505 08:36:27.804 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 27.37% / 0.02% / 72.61% | 20260505 08:36:27.804 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.53711% / 3.57628e-05% / 7.54992e-05% | 20260505 08:36:27.804 | INFO | Baz IO overall: | 20260505 08:36:27.804 | INFO | BytesWritten : 1033023403326 | 20260505 08:36:27.804 | INFO | NumEvents : 491145914037 | 20260505 08:36:29.396 | INFO | Analyzing chunk frames = [1822720,1823232) | 20260505 08:36:29.396 | INFO | MemInfo: { | 20260505 08:36:29.396 | INFO | "gpu" : " 17 GB", | 20260505 08:36:29.396 | INFO | "huge" : "", | 20260505 08:36:29.396 | INFO | "huge_shared" : " 27 GB", | 20260505 08:36:29.396 | INFO | "malloc" : " 14 GB", | 20260505 08:36:29.396 | INFO | "pinned" : " 29 GB" | 20260505 08:36:29.396 | INFO | } | 20260505 08:36:29.396 | INFO | Perf report for the last 5.1395 seconds: | 20260505 08:36:29.396 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:36:29.396 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:29.396 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:29.396 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:29.396 | INFO | ANALYSIS : 65.2%, 2.9, 99.3ms, 0 | 20260505 08:36:29.396 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:29.396 | INFO | RT_METRICS : 48.0%, 1.0, 25.2ms, 0 | 20260505 08:36:29.396 | INFO | PRE_HQ : 63.5%, 1.9, 61.3ms, 0 | 20260505 08:36:29.396 | INFO | BAZWRITER : 65.6%, 4.2, 140.8ms, 0 | 20260505 08:36:29.396 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 08:36:29.396 | INFO | Uploaded 36GiB to GPU in 2.26625s (15.8853GiB/s) | 20260505 08:36:29.396 | INFO | Downloaded 38GiB to GPU in 2.38766s (15.9152GiB/s) | 20260505 08:36:34.536 | INFO | Analyzing chunk frames = [1823232,1823744) | 20260505 08:36:34.536 | INFO | MemInfo: { | 20260505 08:36:34.536 | INFO | "gpu" : " 17 GB", | 20260505 08:36:34.536 | INFO | "huge" : "", | 20260505 08:36:34.536 | INFO | "huge_shared" : " 27 GB", | 20260505 08:36:34.536 | INFO | "malloc" : " 14 GB", | 20260505 08:36:34.536 | INFO | "pinned" : " 29 GB" | 20260505 08:36:34.536 | INFO | } | 20260505 08:36:34.536 | INFO | Perf report for the last 5.13979 seconds: | 20260505 08:36:34.536 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:36:34.536 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:34.536 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:34.536 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:34.536 | INFO | ANALYSIS : 64.7%, 3.0, 101.0ms, 0 | 20260505 08:36:34.536 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:34.536 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:34.536 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:36:34.536 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:34.536 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:34.536 | INFO | Uploaded 36GiB to GPU in 2.24979s (16.0015GiB/s) | 20260505 08:36:34.536 | INFO | Downloaded 34GiB to GPU in 2.10024s (16.1886GiB/s) | 20260505 08:36:39.676 | INFO | Analyzing chunk frames = [1823744,1824256) | 20260505 08:36:39.676 | INFO | MemInfo: { | 20260505 08:36:39.676 | INFO | "gpu" : " 17 GB", | 20260505 08:36:39.676 | INFO | "huge" : "", | 20260505 08:36:39.676 | INFO | "huge_shared" : " 27 GB", | 20260505 08:36:39.676 | INFO | "malloc" : " 14 GB", | 20260505 08:36:39.676 | INFO | "pinned" : " 29 GB" | 20260505 08:36:39.676 | INFO | } | 20260505 08:36:39.676 | INFO | Perf report for the last 5.1401 seconds: | 20260505 08:36:39.676 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:36:39.676 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:39.676 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:39.676 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:39.676 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 08:36:39.676 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:39.676 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:39.676 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:36:39.676 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:39.676 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:39.677 | INFO | Uploaded 36GiB to GPU in 2.16698s (16.613GiB/s) | 20260505 08:36:39.677 | INFO | Downloaded 34GiB to GPU in 2.06842s (16.4377GiB/s) | 20260505 08:36:44.815 | INFO | Analyzing chunk frames = [1824256,1824768) | 20260505 08:36:44.815 | INFO | MemInfo: { | 20260505 08:36:44.815 | INFO | "gpu" : " 17 GB", | 20260505 08:36:44.815 | INFO | "huge" : "", | 20260505 08:36:44.815 | INFO | "huge_shared" : " 27 GB", | 20260505 08:36:44.815 | INFO | "malloc" : " 14 GB", | 20260505 08:36:44.815 | INFO | "pinned" : " 29 GB" | 20260505 08:36:44.815 | INFO | } | 20260505 08:36:44.815 | INFO | Perf report for the last 5.13864 seconds: | 20260505 08:36:44.815 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:36:44.815 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:44.815 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:44.815 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:44.815 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260505 08:36:44.815 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:44.815 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:44.815 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:36:44.815 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:44.815 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:44.815 | INFO | Uploaded 36GiB to GPU in 2.16193s (16.6518GiB/s) | 20260505 08:36:44.815 | INFO | Downloaded 34GiB to GPU in 2.06436s (16.47GiB/s) | 20260505 08:36:49.954 | INFO | Analyzing chunk frames = [1824768,1825280) | 20260505 08:36:49.954 | INFO | MemInfo: { | 20260505 08:36:49.954 | INFO | "gpu" : " 17 GB", | 20260505 08:36:49.954 | INFO | "huge" : "", | 20260505 08:36:49.954 | INFO | "huge_shared" : " 27 GB", | 20260505 08:36:49.954 | INFO | "malloc" : " 14 GB", | 20260505 08:36:49.954 | INFO | "pinned" : " 29 GB" | 20260505 08:36:49.954 | INFO | } | 20260505 08:36:49.954 | INFO | Perf report for the last 5.13893 seconds: | 20260505 08:36:49.954 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:36:49.954 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:49.954 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:49.954 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:49.954 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 08:36:49.954 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:49.954 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:49.954 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 08:36:49.954 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:49.954 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:49.954 | INFO | Uploaded 36GiB to GPU in 2.18155s (16.502GiB/s) | 20260505 08:36:49.954 | INFO | Downloaded 34GiB to GPU in 2.06s (16.5048GiB/s) | 20260505 08:36:55.094 | INFO | Analyzing chunk frames = [1825280,1825792) | 20260505 08:36:55.094 | INFO | MemInfo: { | 20260505 08:36:55.094 | INFO | "gpu" : " 17 GB", | 20260505 08:36:55.094 | INFO | "huge" : "", | 20260505 08:36:55.094 | INFO | "huge_shared" : " 27 GB", | 20260505 08:36:55.094 | INFO | "malloc" : " 14 GB", | 20260505 08:36:55.094 | INFO | "pinned" : " 29 GB" | 20260505 08:36:55.094 | INFO | } | 20260505 08:36:55.094 | INFO | Perf report for the last 5.14085 seconds: | 20260505 08:36:55.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:36:55.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:55.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:55.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:55.099 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 08:36:55.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:55.099 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:55.099 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 08:36:55.099 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:55.099 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:36:55.099 | INFO | Uploaded 36GiB to GPU in 2.18742s (16.4577GiB/s) | 20260505 08:36:55.099 | INFO | Downloaded 34GiB to GPU in 2.056s (16.5369GiB/s) | 20260505 08:37:00.232 | INFO | Analyzing chunk frames = [1825792,1826304) | 20260505 08:37:00.232 | INFO | MemInfo: { | 20260505 08:37:00.232 | INFO | "gpu" : " 17 GB", | 20260505 08:37:00.232 | INFO | "huge" : "", | 20260505 08:37:00.232 | INFO | "huge_shared" : " 27 GB", | 20260505 08:37:00.232 | INFO | "malloc" : " 14 GB", | 20260505 08:37:00.232 | INFO | "pinned" : " 29 GB" | 20260505 08:37:00.232 | INFO | } | 20260505 08:37:00.232 | INFO | Perf report for the last 5.1375 seconds: | 20260505 08:37:00.232 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:37:00.232 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:00.232 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:00.232 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:00.232 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 08:37:00.232 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:00.232 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:00.232 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 08:37:00.232 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:00.232 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:00.232 | INFO | Uploaded 36GiB to GPU in 2.19607s (16.393GiB/s) | 20260505 08:37:00.232 | INFO | Downloaded 34GiB to GPU in 2.13661s (15.913GiB/s) | 20260505 08:37:05.372 | INFO | Analyzing chunk frames = [1826304,1826816) | 20260505 08:37:05.372 | INFO | MemInfo: { | 20260505 08:37:05.372 | INFO | "gpu" : " 17 GB", | 20260505 08:37:05.372 | INFO | "huge" : "", | 20260505 08:37:05.372 | INFO | "huge_shared" : " 27 GB", | 20260505 08:37:05.372 | INFO | "malloc" : " 14 GB", | 20260505 08:37:05.372 | INFO | "pinned" : " 29 GB" | 20260505 08:37:05.372 | INFO | } | 20260505 08:37:05.372 | INFO | Perf report for the last 5.14014 seconds: | 20260505 08:37:05.372 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:37:05.372 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:05.372 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:05.372 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:05.372 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 08:37:05.372 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:05.372 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:05.372 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 08:37:05.372 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:05.372 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:05.372 | INFO | Uploaded 36GiB to GPU in 2.19687s (16.387GiB/s) | 20260505 08:37:05.372 | INFO | Downloaded 34GiB to GPU in 2.04242s (16.647GiB/s) | 20260505 08:37:05.373 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 08:37:05.373 | INFO | Total mem usage: 15.3 GiB | 20260505 08:37:05.373 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 08:37:05.373 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.39%) | 20260505 08:37:05.373 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 08:37:05.373 | INFO | High Water Breakdown: | 20260505 08:37:05.373 | INFO | 60.8% : 8.77 GiB ( 93.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 08:37:05.373 | INFO | 39% : 5.62 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 08:37:05.373 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 08:37:05.373 | INFO | ------------------------------------------ | 20260505 08:37:05.373 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:37:05.373 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 08:37:05.373 | INFO | Total mem usage: 26.5 GiB | 20260505 08:37:05.373 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 08:37:05.373 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 08:37:05.373 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 08:37:05.373 | INFO | High Water Breakdown: | 20260505 08:37:05.373 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:37:05.373 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:37:05.373 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:37:05.373 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:37:05.373 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:37:05.373 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:37:05.373 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 08:37:05.373 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 08:37:05.373 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:37:05.373 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 08:37:05.373 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 08:37:05.373 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 08:37:05.373 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 08:37:05.373 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 08:37:05.373 | INFO | ------------------------------------------ | 20260505 08:37:05.373 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:37:05.373 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 08:37:05.373 | INFO | Total mem usage: 25.6 GiB | 20260505 08:37:05.373 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 08:37:05.373 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 08:37:05.373 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 08:37:05.373 | INFO | High Water Breakdown: | 20260505 08:37:05.373 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 08:37:05.373 | INFO | ------------------------------------------ | 20260505 08:37:05.373 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:37:05.373 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 08:37:05.373 | INFO | Total mem usage: 15.7 GiB | 20260505 08:37:05.373 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 08:37:05.373 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 08:37:05.373 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 08:37:05.373 | INFO | High Water Breakdown: | 20260505 08:37:05.373 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 08:37:05.373 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 08:37:05.373 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 08:37:05.373 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 08:37:05.373 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 08:37:05.373 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 08:37:05.373 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 08:37:05.373 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 08:37:05.373 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 08:37:05.373 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:37:05.373 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 08:37:05.373 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:37:05.373 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 08:37:05.373 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 08:37:05.373 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 08:37:05.373 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 08:37:05.373 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:37:05.373 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 08:37:05.373 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:37:05.373 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:37:05.373 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:37:05.373 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 08:37:05.373 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 08:37:05.373 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:37:05.373 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 08:37:05.373 | INFO | ------------------------------------------ | 20260505 08:37:05.373 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:37:05.374 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 08:37:05.374 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:37:05.374 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:37:05.374 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.54e+01, 1.36e+01 / 2.03e+00 | 20260505 08:37:05.374 | INFO | PerfReport: 0.29%, AggregateStats -- 9.31e-02 / 1.03e-01, 1.08e-02 / 5.29e-03 | 20260505 08:37:05.374 | INFO | PerfReport: 11.65%, Baseline -- 3.80e+00 / 4.45e+00, 6.21e-01 / 3.00e-01 | 20260505 08:37:05.374 | INFO | PerfReport: 3.32%, Histogram -- 1.08e+00 / 1.49e+00, 2.12e-01 / 1.05e-01 | 20260505 08:37:05.374 | INFO | PerfReport: 15.54%, DME -- 4.05e+01 / 4.43e+01, 4.08e+00 / 1.72e+00 | 20260505 08:37:05.374 | INFO | PerfReport: 56.48%, FrameLabeling -- 1.84e+01 / 1.99e+01, 1.90e+00 / 2.03e+00 | 20260505 08:37:05.374 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.43e+00 / 3.01e+00, 4.04e-01 / 2.49e-01 | 20260505 08:37:05.374 | INFO | PerfReport: 3.88%, Metrics -- 1.27e+00 / 1.59e+00, 3.58e-01 / 1.78e-01 | 20260505 08:37:05.374 | INFO | PerfReport: 1.39%, PulseSerializing -- 4.53e-01 / 5.09e-01, 8.65e-02 / 2.71e-02 | 20260505 08:37:05.374 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 08:37:05.374 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:37:05.374 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:37:05.374 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.29e+01 / 0.00e+00 | 20260505 08:37:05.374 | INFO | PerfReport: 22.52%, Upload -- 2.29e+01 / 0.00e+00, 8.14e+00 / 0.00e+00 | 20260505 08:37:05.374 | INFO | PerfReport: 21.66%, Download -- 2.20e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260505 08:37:05.374 | INFO | PerfReport: 32.09%, Compute -- 3.26e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 08:37:05.374 | INFO | PerfReport: 1.68%, LockUpload -- 1.71e+00 / 0.00e+00, 7.29e+00 / 0.00e+00 | 20260505 08:37:05.374 | INFO | PerfReport: 1.07%, LockDownload -- 1.09e+00 / 0.00e+00, 2.63e+00 / 0.00e+00 | 20260505 08:37:05.374 | INFO | PerfReport: 20.96%, LockCompute -- 2.13e+01 / 0.00e+00, 1.77e+01 / 0.00e+00 | 20260505 08:37:05.374 | INFO | PerfReport: 0.01%, Misc -- 1.46e-02 / 0.00e+00, 2.53e-02 / 0.00e+00 | 20260505 08:37:05.512 | INFO | Starting a round of Baz IO | 20260505 08:37:08.794 | INFO | Populating RTMetrics report | 20260505 08:37:08.794 | INFO | RTMetrics report populated | 20260505 08:37:08.794 | INFO | Opening tmp RTMetrics json file | 20260505 08:37:08.813 | INFO | Serializing RTMetrics to json | 20260505 08:37:08.814 | INFO | Serialization done, swapping tmp file now... | 20260505 08:37:08.814 | INFO | RTMetrics done | 20260505 08:37:08.967 | INFO | Finished a round of Baz IO: Wrote 2.451 GiB in 3.455 seconds | 20260505 08:37:08.967 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 27.35% / 0.02% / 72.63% | 20260505 08:37:08.967 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.52335% / 3.57628e-05% / 8.74201e-05% | 20260505 08:37:08.967 | INFO | Baz IO overall: | 20260505 08:37:08.967 | INFO | BytesWritten : 1035655383746 | 20260505 08:37:08.967 | INFO | NumEvents : 492404563838 | 20260505 08:37:10.511 | INFO | Analyzing chunk frames = [1826816,1827328) | 20260505 08:37:10.511 | INFO | MemInfo: { | 20260505 08:37:10.511 | INFO | "gpu" : " 17 GB", | 20260505 08:37:10.511 | INFO | "huge" : "", | 20260505 08:37:10.511 | INFO | "huge_shared" : " 27 GB", | 20260505 08:37:10.511 | INFO | "malloc" : " 14 GB", | 20260505 08:37:10.511 | INFO | "pinned" : " 29 GB" | 20260505 08:37:10.511 | INFO | } | 20260505 08:37:10.512 | INFO | Perf report for the last 5.13934 seconds: | 20260505 08:37:10.512 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:37:10.512 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:10.512 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:10.512 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:10.512 | INFO | ANALYSIS : 65.4%, 2.9, 100.1ms, 0 | 20260505 08:37:10.512 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:10.512 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260505 08:37:10.512 | INFO | PRE_HQ : 63.1%, 1.9, 61.2ms, 0 | 20260505 08:37:10.512 | INFO | BAZWRITER : 66.5%, 4.2, 142.5ms, 0 | 20260505 08:37:10.512 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 08:37:10.512 | INFO | Uploaded 36GiB to GPU in 2.2326s (16.1247GiB/s) | 20260505 08:37:10.512 | INFO | Downloaded 38GiB to GPU in 2.3508s (16.1647GiB/s) | 20260505 08:37:15.651 | INFO | Analyzing chunk frames = [1827328,1827840) | 20260505 08:37:15.651 | INFO | MemInfo: { | 20260505 08:37:15.651 | INFO | "gpu" : " 17 GB", | 20260505 08:37:15.651 | INFO | "huge" : "", | 20260505 08:37:15.651 | INFO | "huge_shared" : " 27 GB", | 20260505 08:37:15.651 | INFO | "malloc" : " 14 GB", | 20260505 08:37:15.651 | INFO | "pinned" : " 29 GB" | 20260505 08:37:15.651 | INFO | } | 20260505 08:37:15.651 | INFO | Perf report for the last 5.13968 seconds: | 20260505 08:37:15.651 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:37:15.651 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:15.651 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:15.651 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:15.651 | INFO | ANALYSIS : 64.4%, 3.0, 100.6ms, 0 | 20260505 08:37:15.651 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:15.651 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:15.651 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 08:37:15.651 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:15.651 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:15.652 | INFO | Uploaded 36GiB to GPU in 2.21757s (16.234GiB/s) | 20260505 08:37:15.652 | INFO | Downloaded 34GiB to GPU in 2.08425s (16.3129GiB/s) | 20260505 08:37:20.790 | INFO | Analyzing chunk frames = [1827840,1828352) | 20260505 08:37:20.790 | INFO | MemInfo: { | 20260505 08:37:20.790 | INFO | "gpu" : " 17 GB", | 20260505 08:37:20.790 | INFO | "huge" : "", | 20260505 08:37:20.790 | INFO | "huge_shared" : " 27 GB", | 20260505 08:37:20.790 | INFO | "malloc" : " 14 GB", | 20260505 08:37:20.790 | INFO | "pinned" : " 29 GB" | 20260505 08:37:20.790 | INFO | } | 20260505 08:37:20.790 | INFO | Perf report for the last 5.13893 seconds: | 20260505 08:37:20.790 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:37:20.790 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:20.790 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:20.790 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:20.790 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 08:37:20.790 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:20.790 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:20.790 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:37:20.790 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:20.790 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:20.790 | INFO | Uploaded 36GiB to GPU in 2.16888s (16.5985GiB/s) | 20260505 08:37:20.790 | INFO | Downloaded 34GiB to GPU in 2.06543s (16.4615GiB/s) | 20260505 08:37:25.929 | INFO | Analyzing chunk frames = [1828352,1828864) | 20260505 08:37:25.929 | INFO | MemInfo: { | 20260505 08:37:25.929 | INFO | "gpu" : " 17 GB", | 20260505 08:37:25.929 | INFO | "huge" : "", | 20260505 08:37:25.929 | INFO | "huge_shared" : " 27 GB", | 20260505 08:37:25.929 | INFO | "malloc" : " 14 GB", | 20260505 08:37:25.929 | INFO | "pinned" : " 29 GB" | 20260505 08:37:25.929 | INFO | } | 20260505 08:37:25.929 | INFO | Perf report for the last 5.13918 seconds: | 20260505 08:37:25.929 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:37:25.929 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:25.929 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:25.929 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:25.929 | INFO | ANALYSIS : 63.5%, 3.0, 98.6ms, 0 | 20260505 08:37:25.929 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:25.929 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:25.929 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260505 08:37:25.930 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:25.930 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:25.930 | INFO | Uploaded 36GiB to GPU in 2.17437s (16.5565GiB/s) | 20260505 08:37:25.930 | INFO | Downloaded 34GiB to GPU in 2.06297s (16.4811GiB/s) | 20260505 08:37:31.068 | INFO | Analyzing chunk frames = [1828864,1829376) | 20260505 08:37:31.068 | INFO | MemInfo: { | 20260505 08:37:31.068 | INFO | "gpu" : " 17 GB", | 20260505 08:37:31.068 | INFO | "huge" : "", | 20260505 08:37:31.068 | INFO | "huge_shared" : " 27 GB", | 20260505 08:37:31.068 | INFO | "malloc" : " 14 GB", | 20260505 08:37:31.068 | INFO | "pinned" : " 29 GB" | 20260505 08:37:31.068 | INFO | } | 20260505 08:37:31.068 | INFO | Perf report for the last 5.13883 seconds: | 20260505 08:37:31.068 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:37:31.068 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:31.068 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:31.068 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:31.068 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 08:37:31.068 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:31.068 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:31.068 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 08:37:31.068 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:31.068 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:31.068 | INFO | Uploaded 36GiB to GPU in 2.16385s (16.637GiB/s) | 20260505 08:37:31.068 | INFO | Downloaded 34GiB to GPU in 2.05685s (16.5301GiB/s) | 20260505 08:37:36.208 | INFO | Analyzing chunk frames = [1829376,1829888) | 20260505 08:37:36.208 | INFO | MemInfo: { | 20260505 08:37:36.208 | INFO | "gpu" : " 17 GB", | 20260505 08:37:36.208 | INFO | "huge" : "", | 20260505 08:37:36.208 | INFO | "huge_shared" : " 27 GB", | 20260505 08:37:36.208 | INFO | "malloc" : " 14 GB", | 20260505 08:37:36.208 | INFO | "pinned" : " 29 GB" | 20260505 08:37:36.208 | INFO | } | 20260505 08:37:36.208 | INFO | Perf report for the last 5.13975 seconds: | 20260505 08:37:36.208 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:37:36.208 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:36.208 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:36.208 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:36.208 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 08:37:36.208 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:36.208 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:36.208 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 08:37:36.208 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:36.208 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:36.208 | INFO | Uploaded 36GiB to GPU in 2.18957s (16.4416GiB/s) | 20260505 08:37:36.208 | INFO | Downloaded 34GiB to GPU in 2.05277s (16.563GiB/s) | 20260505 08:37:41.348 | INFO | Analyzing chunk frames = [1829888,1830400) | 20260505 08:37:41.348 | INFO | MemInfo: { | 20260505 08:37:41.348 | INFO | "gpu" : " 17 GB", | 20260505 08:37:41.348 | INFO | "huge" : "", | 20260505 08:37:41.348 | INFO | "huge_shared" : " 27 GB", | 20260505 08:37:41.348 | INFO | "malloc" : " 14 GB", | 20260505 08:37:41.348 | INFO | "pinned" : " 29 GB" | 20260505 08:37:41.348 | INFO | } | 20260505 08:37:41.348 | INFO | Perf report for the last 5.13996 seconds: | 20260505 08:37:41.348 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:37:41.348 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:41.348 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:41.348 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:41.348 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 08:37:41.348 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:41.348 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:41.348 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260505 08:37:41.348 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:41.348 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:41.348 | INFO | Uploaded 36GiB to GPU in 2.20969s (16.2919GiB/s) | 20260505 08:37:41.348 | INFO | Downloaded 34GiB to GPU in 2.14261s (15.8685GiB/s) | 20260505 08:37:46.488 | INFO | Analyzing chunk frames = [1830400,1830912) | 20260505 08:37:46.488 | INFO | MemInfo: { | 20260505 08:37:46.488 | INFO | "gpu" : " 17 GB", | 20260505 08:37:46.488 | INFO | "huge" : "", | 20260505 08:37:46.488 | INFO | "huge_shared" : " 27 GB", | 20260505 08:37:46.488 | INFO | "malloc" : " 14 GB", | 20260505 08:37:46.488 | INFO | "pinned" : " 29 GB" | 20260505 08:37:46.488 | INFO | } | 20260505 08:37:46.488 | INFO | Perf report for the last 5.14012 seconds: | 20260505 08:37:46.488 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:37:46.488 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:46.488 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:46.488 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:46.488 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 08:37:46.488 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:46.488 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:46.488 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 08:37:46.488 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:46.488 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:46.488 | INFO | Uploaded 36GiB to GPU in 2.17114s (16.5811GiB/s) | 20260505 08:37:46.488 | INFO | Downloaded 34GiB to GPU in 2.0346s (16.7109GiB/s) | 20260505 08:37:46.627 | INFO | Starting a round of Baz IO | 20260505 08:37:49.913 | INFO | Populating RTMetrics report | 20260505 08:37:49.913 | INFO | RTMetrics report populated | 20260505 08:37:49.913 | INFO | Opening tmp RTMetrics json file | 20260505 08:37:49.919 | INFO | Serializing RTMetrics to json | 20260505 08:37:49.920 | INFO | Serialization done, swapping tmp file now... | 20260505 08:37:49.921 | INFO | RTMetrics done | 20260505 08:37:50.038 | INFO | Finished a round of Baz IO: Wrote 2.45 GiB in 3.411 seconds | 20260505 08:37:50.038 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 27.33% / 0.02% / 72.65% | 20260505 08:37:50.038 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.51064% / 2.38419e-05% / 7.54992e-05% | 20260505 08:37:50.038 | INFO | Baz IO overall: | 20260505 08:37:50.038 | INFO | BytesWritten : 1038285880824 | 20260505 08:37:50.038 | INFO | NumEvents : 493662003988 | 20260505 08:37:51.626 | INFO | Analyzing chunk frames = [1830912,1831424) | 20260505 08:37:51.626 | INFO | MemInfo: { | 20260505 08:37:51.626 | INFO | "gpu" : " 17 GB", | 20260505 08:37:51.626 | INFO | "huge" : "", | 20260505 08:37:51.626 | INFO | "huge_shared" : " 27 GB", | 20260505 08:37:51.626 | INFO | "malloc" : " 14 GB", | 20260505 08:37:51.626 | INFO | "pinned" : " 29 GB" | 20260505 08:37:51.626 | INFO | } | 20260505 08:37:51.626 | INFO | Perf report for the last 5.13789 seconds: | 20260505 08:37:51.626 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:37:51.626 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:51.626 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:51.626 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:51.626 | INFO | ANALYSIS : 65.5%, 2.9, 100.3ms, 0 | 20260505 08:37:51.626 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:51.626 | INFO | RT_METRICS : 45.9%, 1.0, 24.0ms, 0 | 20260505 08:37:51.626 | INFO | PRE_HQ : 63.0%, 1.9, 61.0ms, 0 | 20260505 08:37:51.626 | INFO | BAZWRITER : 65.7%, 4.1, 139.2ms, 0 | 20260505 08:37:51.626 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 08:37:51.626 | INFO | Uploaded 36GiB to GPU in 2.20277s (16.3431GiB/s) | 20260505 08:37:51.626 | INFO | Downloaded 38GiB to GPU in 2.33303s (16.2879GiB/s) | 20260505 08:37:56.764 | INFO | Analyzing chunk frames = [1831424,1831936) | 20260505 08:37:56.764 | INFO | MemInfo: { | 20260505 08:37:56.764 | INFO | "gpu" : " 17 GB", | 20260505 08:37:56.764 | INFO | "huge" : "", | 20260505 08:37:56.765 | INFO | "huge_shared" : " 27 GB", | 20260505 08:37:56.765 | INFO | "malloc" : " 14 GB", | 20260505 08:37:56.765 | INFO | "pinned" : " 29 GB" | 20260505 08:37:56.765 | INFO | } | 20260505 08:37:56.765 | INFO | Perf report for the last 5.13861 seconds: | 20260505 08:37:56.765 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:37:56.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:56.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:56.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:56.765 | INFO | ANALYSIS : 64.5%, 3.0, 100.7ms, 0 | 20260505 08:37:56.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:56.765 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:56.765 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 08:37:56.765 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:56.765 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:37:56.765 | INFO | Uploaded 36GiB to GPU in 2.22292s (16.195GiB/s) | 20260505 08:37:56.765 | INFO | Downloaded 34GiB to GPU in 2.09182s (16.2538GiB/s) | 20260505 08:38:01.904 | INFO | Analyzing chunk frames = [1831936,1832448) | 20260505 08:38:01.904 | INFO | MemInfo: { | 20260505 08:38:01.904 | INFO | "gpu" : " 17 GB", | 20260505 08:38:01.904 | INFO | "huge" : "", | 20260505 08:38:01.904 | INFO | "huge_shared" : " 27 GB", | 20260505 08:38:01.904 | INFO | "malloc" : " 14 GB", | 20260505 08:38:01.904 | INFO | "pinned" : " 29 GB" | 20260505 08:38:01.904 | INFO | } | 20260505 08:38:01.905 | INFO | Perf report for the last 5.13992 seconds: | 20260505 08:38:01.905 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:38:01.905 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:01.905 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:01.905 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:01.905 | INFO | ANALYSIS : 63.6%, 3.0, 99.0ms, 0 | 20260505 08:38:01.905 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:01.905 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:01.905 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:38:01.905 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:01.905 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:01.905 | INFO | Uploaded 36GiB to GPU in 2.17712s (16.5356GiB/s) | 20260505 08:38:01.905 | INFO | Downloaded 34GiB to GPU in 2.06626s (16.4548GiB/s) | 20260505 08:38:07.044 | INFO | Analyzing chunk frames = [1832448,1832960) | 20260505 08:38:07.044 | INFO | MemInfo: { | 20260505 08:38:07.044 | INFO | "gpu" : " 17 GB", | 20260505 08:38:07.044 | INFO | "huge" : "", | 20260505 08:38:07.044 | INFO | "huge_shared" : " 27 GB", | 20260505 08:38:07.044 | INFO | "malloc" : " 14 GB", | 20260505 08:38:07.044 | INFO | "pinned" : " 29 GB" | 20260505 08:38:07.044 | INFO | } | 20260505 08:38:07.044 | INFO | Perf report for the last 5.13977 seconds: | 20260505 08:38:07.044 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:38:07.044 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:07.044 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:07.044 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:07.044 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260505 08:38:07.045 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:07.045 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:07.045 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:38:07.045 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:07.045 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:07.045 | INFO | Uploaded 36GiB to GPU in 2.17685s (16.5377GiB/s) | 20260505 08:38:07.045 | INFO | Downloaded 34GiB to GPU in 2.0664s (16.4537GiB/s) | 20260505 08:38:12.183 | INFO | Analyzing chunk frames = [1832960,1833472) | 20260505 08:38:12.183 | INFO | MemInfo: { | 20260505 08:38:12.183 | INFO | "gpu" : " 17 GB", | 20260505 08:38:12.183 | INFO | "huge" : "", | 20260505 08:38:12.183 | INFO | "huge_shared" : " 27 GB", | 20260505 08:38:12.183 | INFO | "malloc" : " 14 GB", | 20260505 08:38:12.183 | INFO | "pinned" : " 29 GB" | 20260505 08:38:12.183 | INFO | } | 20260505 08:38:12.183 | INFO | Perf report for the last 5.13879 seconds: | 20260505 08:38:12.183 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:38:12.183 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:12.183 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:12.183 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:12.183 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 08:38:12.183 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:12.183 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:12.183 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 08:38:12.183 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:12.183 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:12.184 | INFO | Uploaded 36GiB to GPU in 2.16309s (16.6429GiB/s) | 20260505 08:38:12.184 | INFO | Downloaded 34GiB to GPU in 2.059s (16.5128GiB/s) | 20260505 08:38:17.323 | INFO | Analyzing chunk frames = [1833472,1833984) | 20260505 08:38:17.323 | INFO | MemInfo: { | 20260505 08:38:17.323 | INFO | "gpu" : " 17 GB", | 20260505 08:38:17.323 | INFO | "huge" : "", | 20260505 08:38:17.323 | INFO | "huge_shared" : " 27 GB", | 20260505 08:38:17.323 | INFO | "malloc" : " 14 GB", | 20260505 08:38:17.323 | INFO | "pinned" : " 29 GB" | 20260505 08:38:17.323 | INFO | } | 20260505 08:38:17.323 | INFO | Perf report for the last 5.14019 seconds: | 20260505 08:38:17.323 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:38:17.323 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:17.323 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:17.324 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:17.324 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 08:38:17.324 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:17.324 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:17.324 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 08:38:17.324 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:17.324 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:17.324 | INFO | Uploaded 36GiB to GPU in 2.19082s (16.4322GiB/s) | 20260505 08:38:17.324 | INFO | Downloaded 34GiB to GPU in 2.05905s (16.5125GiB/s) | 20260505 08:38:22.463 | INFO | Analyzing chunk frames = [1833984,1834496) | 20260505 08:38:22.463 | INFO | MemInfo: { | 20260505 08:38:22.463 | INFO | "gpu" : " 17 GB", | 20260505 08:38:22.463 | INFO | "huge" : "", | 20260505 08:38:22.463 | INFO | "huge_shared" : " 27 GB", | 20260505 08:38:22.463 | INFO | "malloc" : " 14 GB", | 20260505 08:38:22.463 | INFO | "pinned" : " 29 GB" | 20260505 08:38:22.463 | INFO | } | 20260505 08:38:22.463 | INFO | Perf report for the last 5.13998 seconds: | 20260505 08:38:22.463 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:38:22.463 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:22.463 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:22.463 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:22.463 | INFO | ANALYSIS : 64.1%, 3.0, 100.2ms, 0 | 20260505 08:38:22.463 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:22.463 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:22.463 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 08:38:22.463 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:22.463 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:22.464 | INFO | Uploaded 36GiB to GPU in 2.22322s (16.1927GiB/s) | 20260505 08:38:22.464 | INFO | Downloaded 34GiB to GPU in 2.14889s (15.8221GiB/s) | 20260505 08:38:27.601 | INFO | Analyzing chunk frames = [1834496,1835008) | 20260505 08:38:27.601 | INFO | MemInfo: { | 20260505 08:38:27.601 | INFO | "gpu" : " 17 GB", | 20260505 08:38:27.601 | INFO | "huge" : "", | 20260505 08:38:27.601 | INFO | "huge_shared" : " 27 GB", | 20260505 08:38:27.601 | INFO | "malloc" : " 14 GB", | 20260505 08:38:27.601 | INFO | "pinned" : " 29 GB" | 20260505 08:38:27.602 | INFO | } | 20260505 08:38:27.602 | INFO | Perf report for the last 5.1384 seconds: | 20260505 08:38:27.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:38:27.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:27.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:27.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:27.602 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 08:38:27.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:27.602 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:27.602 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 08:38:27.602 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:27.602 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:27.602 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 08:38:27.602 | INFO | Total mem usage: 15.3 GiB | 20260505 08:38:27.602 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 08:38:27.602 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.36%) | 20260505 08:38:27.602 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 08:38:27.602 | INFO | High Water Breakdown: | 20260505 08:38:27.602 | INFO | 60.9% : 8.77 GiB ( 93.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 08:38:27.602 | INFO | 39% : 5.62 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 08:38:27.602 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 08:38:27.602 | INFO | ------------------------------------------ | 20260505 08:38:27.602 | INFO | Uploaded 36GiB to GPU in 2.17562s (16.547GiB/s) | 20260505 08:38:27.602 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:38:27.602 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 08:38:27.602 | INFO | Total mem usage: 26.5 GiB | 20260505 08:38:27.602 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 08:38:27.602 | INFO | Downloaded 34GiB to GPU in 2.02731s (16.771GiB/s) | 20260505 08:38:27.602 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 08:38:27.602 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 08:38:27.602 | INFO | High Water Breakdown: | 20260505 08:38:27.602 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:38:27.602 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:38:27.602 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:38:27.602 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:38:27.602 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:38:27.602 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:38:27.602 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 08:38:27.602 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 08:38:27.602 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:38:27.602 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 08:38:27.603 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 08:38:27.603 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 08:38:27.603 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 08:38:27.603 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 08:38:27.603 | INFO | ------------------------------------------ | 20260505 08:38:27.603 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:38:27.603 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 08:38:27.603 | INFO | Total mem usage: 25.6 GiB | 20260505 08:38:27.603 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 08:38:27.603 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 08:38:27.603 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 08:38:27.603 | INFO | High Water Breakdown: | 20260505 08:38:27.603 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 08:38:27.603 | INFO | ------------------------------------------ | 20260505 08:38:27.603 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:38:27.603 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 08:38:27.603 | INFO | Total mem usage: 15.7 GiB | 20260505 08:38:27.603 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 08:38:27.603 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 08:38:27.603 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 08:38:27.603 | INFO | High Water Breakdown: | 20260505 08:38:27.603 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 08:38:27.603 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 08:38:27.603 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 08:38:27.603 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 08:38:27.603 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 08:38:27.603 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 08:38:27.603 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 08:38:27.603 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 08:38:27.603 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 08:38:27.603 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:38:27.603 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 08:38:27.603 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:38:27.603 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 08:38:27.603 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 08:38:27.603 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 08:38:27.603 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 08:38:27.603 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:38:27.603 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 08:38:27.603 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:38:27.603 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:38:27.603 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:38:27.603 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 08:38:27.603 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 08:38:27.603 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:38:27.603 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 08:38:27.603 | INFO | ------------------------------------------ | 20260505 08:38:27.603 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:38:27.603 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 08:38:27.603 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:38:27.603 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:38:27.603 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.54e+01, 1.34e+01 / 2.48e+00 | 20260505 08:38:27.603 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 1.03e-01, 1.10e-02 / 4.36e-03 | 20260505 08:38:27.603 | INFO | PerfReport: 11.65%, Baseline -- 3.80e+00 / 4.48e+00, 6.19e-01 / 2.68e-01 | 20260505 08:38:27.603 | INFO | PerfReport: 3.31%, Histogram -- 1.08e+00 / 1.46e+00, 2.08e-01 / 1.21e-01 | 20260505 08:38:27.603 | INFO | PerfReport: 15.56%, DME -- 4.05e+01 / 4.44e+01, 4.13e+00 / 2.21e+00 | 20260505 08:38:27.603 | INFO | PerfReport: 56.43%, FrameLabeling -- 1.84e+01 / 1.98e+01, 1.91e+00 / 1.93e+00 | 20260505 08:38:27.603 | INFO | PerfReport: 7.48%, PulseAccumulating -- 2.44e+00 / 3.01e+00, 4.09e-01 / 2.29e-01 | 20260505 08:38:27.603 | INFO | PerfReport: 3.89%, Metrics -- 1.27e+00 / 1.61e+00, 3.58e-01 / 1.96e-01 | 20260505 08:38:27.603 | INFO | PerfReport: 1.39%, PulseSerializing -- 4.53e-01 / 5.14e-01, 8.65e-02 / 2.65e-02 | 20260505 08:38:27.603 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 08:38:27.603 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:38:27.603 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:38:27.603 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.29e+01 / 0.00e+00 | 20260505 08:38:27.603 | INFO | PerfReport: 22.46%, Upload -- 2.28e+01 / 0.00e+00, 8.16e+00 / 0.00e+00 | 20260505 08:38:27.603 | INFO | PerfReport: 21.59%, Download -- 2.19e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260505 08:38:27.603 | INFO | PerfReport: 32.04%, Compute -- 3.26e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 08:38:27.603 | INFO | PerfReport: 1.69%, LockUpload -- 1.72e+00 / 0.00e+00, 7.42e+00 / 0.00e+00 | 20260505 08:38:27.603 | INFO | PerfReport: 1.11%, LockDownload -- 1.13e+00 / 0.00e+00, 2.69e+00 / 0.00e+00 | 20260505 08:38:27.603 | INFO | PerfReport: 21.09%, LockCompute -- 2.14e+01 / 0.00e+00, 1.77e+01 / 0.00e+00 | 20260505 08:38:27.603 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.64e-02 / 0.00e+00 | 20260505 08:38:27.741 | INFO | Starting a round of Baz IO | 20260505 08:38:31.029 | INFO | Populating RTMetrics report | 20260505 08:38:31.029 | INFO | RTMetrics report populated | 20260505 08:38:31.029 | INFO | Opening tmp RTMetrics json file | 20260505 08:38:31.035 | INFO | Serializing RTMetrics to json | 20260505 08:38:31.038 | INFO | Serialization done, swapping tmp file now... | 20260505 08:38:31.040 | INFO | RTMetrics done | 20260505 08:38:31.168 | INFO | Finished a round of Baz IO: Wrote 2.448 GiB in 3.426 seconds | 20260505 08:38:31.168 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 27.31% / 0.02% / 72.67% | 20260505 08:38:31.168 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.49992% / 3.57628e-05% / 6.75519e-05% | 20260505 08:38:31.168 | INFO | Baz IO overall: | 20260505 08:38:31.168 | INFO | BytesWritten : 1040913944760 | 20260505 08:38:31.168 | INFO | NumEvents : 494917960970 | 20260505 08:38:32.740 | INFO | Analyzing chunk frames = [1835008,1835520) | 20260505 08:38:32.740 | INFO | MemInfo: { | 20260505 08:38:32.740 | INFO | "gpu" : " 17 GB", | 20260505 08:38:32.740 | INFO | "huge" : "", | 20260505 08:38:32.740 | INFO | "huge_shared" : " 27 GB", | 20260505 08:38:32.740 | INFO | "malloc" : " 14 GB", | 20260505 08:38:32.740 | INFO | "pinned" : " 29 GB" | 20260505 08:38:32.740 | INFO | } | 20260505 08:38:32.740 | INFO | Perf report for the last 5.13847 seconds: | 20260505 08:38:32.740 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:38:32.740 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:32.740 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:32.740 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:32.740 | INFO | ANALYSIS : 65.6%, 2.9, 100.4ms, 0 | 20260505 08:38:32.740 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:32.740 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260505 08:38:32.740 | INFO | PRE_HQ : 63.3%, 1.9, 61.2ms, 0 | 20260505 08:38:32.740 | INFO | BAZWRITER : 65.8%, 4.1, 140.1ms, 0 | 20260505 08:38:32.740 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 08:38:32.741 | INFO | Uploaded 36GiB to GPU in 2.24218s (16.0558GiB/s) | 20260505 08:38:32.741 | INFO | Downloaded 38GiB to GPU in 2.35315s (16.1486GiB/s) | 20260505 08:38:37.879 | INFO | Analyzing chunk frames = [1835520,1836032) | 20260505 08:38:37.879 | INFO | MemInfo: { | 20260505 08:38:37.879 | INFO | "gpu" : " 17 GB", | 20260505 08:38:37.879 | INFO | "huge" : "", | 20260505 08:38:37.879 | INFO | "huge_shared" : " 27 GB", | 20260505 08:38:37.879 | INFO | "malloc" : " 14 GB", | 20260505 08:38:37.879 | INFO | "pinned" : " 29 GB" | 20260505 08:38:37.880 | INFO | } | 20260505 08:38:37.880 | INFO | Perf report for the last 5.13948 seconds: | 20260505 08:38:37.880 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:38:37.880 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:37.880 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:37.880 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:37.880 | INFO | ANALYSIS : 64.7%, 3.0, 101.1ms, 0 | 20260505 08:38:37.880 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:37.880 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:37.880 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:38:37.880 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:37.880 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:37.881 | INFO | Uploaded 36GiB to GPU in 2.24913s (16.0062GiB/s) | 20260505 08:38:37.881 | INFO | Downloaded 34GiB to GPU in 2.10753s (16.1326GiB/s) | 20260505 08:38:43.019 | INFO | Analyzing chunk frames = [1836032,1836544) | 20260505 08:38:43.019 | INFO | MemInfo: { | 20260505 08:38:43.020 | INFO | "gpu" : " 17 GB", | 20260505 08:38:43.020 | INFO | "huge" : "", | 20260505 08:38:43.020 | INFO | "huge_shared" : " 27 GB", | 20260505 08:38:43.020 | INFO | "malloc" : " 14 GB", | 20260505 08:38:43.020 | INFO | "pinned" : " 29 GB" | 20260505 08:38:43.020 | INFO | } | 20260505 08:38:43.020 | INFO | Perf report for the last 5.13986 seconds: | 20260505 08:38:43.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:38:43.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:43.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:43.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:43.020 | INFO | ANALYSIS : 63.6%, 3.0, 99.1ms, 0 | 20260505 08:38:43.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:43.020 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:43.020 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:38:43.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:43.020 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:43.020 | INFO | Uploaded 36GiB to GPU in 2.17291s (16.5676GiB/s) | 20260505 08:38:43.020 | INFO | Downloaded 34GiB to GPU in 2.06721s (16.4473GiB/s) | 20260505 08:38:48.159 | INFO | Analyzing chunk frames = [1836544,1837056) | 20260505 08:38:48.159 | INFO | MemInfo: { | 20260505 08:38:48.159 | INFO | "gpu" : " 17 GB", | 20260505 08:38:48.159 | INFO | "huge" : "", | 20260505 08:38:48.159 | INFO | "huge_shared" : " 27 GB", | 20260505 08:38:48.159 | INFO | "malloc" : " 14 GB", | 20260505 08:38:48.159 | INFO | "pinned" : " 29 GB" | 20260505 08:38:48.159 | INFO | } | 20260505 08:38:48.159 | INFO | Perf report for the last 5.13943 seconds: | 20260505 08:38:48.159 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:38:48.159 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:48.159 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:48.159 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:48.159 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260505 08:38:48.159 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:48.159 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:48.159 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 08:38:48.159 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:48.159 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:48.159 | INFO | Uploaded 36GiB to GPU in 2.16775s (16.6071GiB/s) | 20260505 08:38:48.159 | INFO | Downloaded 34GiB to GPU in 2.06398s (16.4731GiB/s) | 20260505 08:38:53.299 | INFO | Analyzing chunk frames = [1837056,1837568) | 20260505 08:38:53.299 | INFO | MemInfo: { | 20260505 08:38:53.299 | INFO | "gpu" : " 17 GB", | 20260505 08:38:53.299 | INFO | "huge" : "", | 20260505 08:38:53.299 | INFO | "huge_shared" : " 27 GB", | 20260505 08:38:53.299 | INFO | "malloc" : " 14 GB", | 20260505 08:38:53.299 | INFO | "pinned" : " 29 GB" | 20260505 08:38:53.299 | INFO | } | 20260505 08:38:53.299 | INFO | Perf report for the last 5.13979 seconds: | 20260505 08:38:53.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:38:53.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:53.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:53.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:53.299 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 08:38:53.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:53.299 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:53.299 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:38:53.299 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:53.299 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:53.299 | INFO | Uploaded 36GiB to GPU in 2.16013s (16.6656GiB/s) | 20260505 08:38:53.299 | INFO | Downloaded 34GiB to GPU in 2.05664s (16.5318GiB/s) | 20260505 08:38:58.439 | INFO | Analyzing chunk frames = [1837568,1838080) | 20260505 08:38:58.439 | INFO | MemInfo: { | 20260505 08:38:58.439 | INFO | "gpu" : " 17 GB", | 20260505 08:38:58.439 | INFO | "huge" : "", | 20260505 08:38:58.439 | INFO | "huge_shared" : " 27 GB", | 20260505 08:38:58.439 | INFO | "malloc" : " 14 GB", | 20260505 08:38:58.439 | INFO | "pinned" : " 29 GB" | 20260505 08:38:58.439 | INFO | } | 20260505 08:38:58.439 | INFO | Perf report for the last 5.14019 seconds: | 20260505 08:38:58.439 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:38:58.439 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:58.439 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:58.439 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:58.439 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 08:38:58.439 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:58.439 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:58.439 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 08:38:58.439 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:58.439 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:38:58.439 | INFO | Uploaded 36GiB to GPU in 2.16975s (16.5917GiB/s) | 20260505 08:38:58.440 | INFO | Downloaded 34GiB to GPU in 2.05744s (16.5254GiB/s) | 20260505 08:39:03.576 | INFO | Analyzing chunk frames = [1838080,1838592) | 20260505 08:39:03.576 | INFO | MemInfo: { | 20260505 08:39:03.576 | INFO | "gpu" : " 17 GB", | 20260505 08:39:03.576 | INFO | "huge" : "", | 20260505 08:39:03.576 | INFO | "huge_shared" : " 27 GB", | 20260505 08:39:03.577 | INFO | "malloc" : " 14 GB", | 20260505 08:39:03.577 | INFO | "pinned" : " 29 GB" | 20260505 08:39:03.577 | INFO | } | 20260505 08:39:03.577 | INFO | Perf report for the last 5.13761 seconds: | 20260505 08:39:03.577 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:39:03.577 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:03.577 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:03.577 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:03.577 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 08:39:03.577 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:03.577 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:03.577 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 08:39:03.577 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:03.577 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:03.577 | INFO | Uploaded 36GiB to GPU in 2.19324s (16.414GiB/s) | 20260505 08:39:03.577 | INFO | Downloaded 34GiB to GPU in 2.15787s (15.7563GiB/s) | 20260505 08:39:08.716 | INFO | Analyzing chunk frames = [1838592,1839104) | 20260505 08:39:08.716 | INFO | MemInfo: { | 20260505 08:39:08.716 | INFO | "gpu" : " 17 GB", | 20260505 08:39:08.716 | INFO | "huge" : "", | 20260505 08:39:08.716 | INFO | "huge_shared" : " 27 GB", | 20260505 08:39:08.716 | INFO | "malloc" : " 14 GB", | 20260505 08:39:08.716 | INFO | "pinned" : " 29 GB" | 20260505 08:39:08.716 | INFO | } | 20260505 08:39:08.716 | INFO | Perf report for the last 5.13914 seconds: | 20260505 08:39:08.716 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:39:08.716 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:08.716 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:08.716 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:08.716 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 08:39:08.716 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:08.716 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:08.716 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 08:39:08.716 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:08.716 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:08.716 | INFO | Uploaded 36GiB to GPU in 2.19404s (16.4081GiB/s) | 20260505 08:39:08.716 | INFO | Downloaded 34GiB to GPU in 2.04642s (16.6143GiB/s) | 20260505 08:39:08.856 | INFO | Starting a round of Baz IO | 20260505 08:39:12.144 | INFO | Populating RTMetrics report | 20260505 08:39:12.144 | INFO | RTMetrics report populated | 20260505 08:39:12.144 | INFO | Opening tmp RTMetrics json file | 20260505 08:39:12.190 | INFO | Serializing RTMetrics to json | 20260505 08:39:12.191 | INFO | Serialization done, swapping tmp file now... | 20260505 08:39:12.192 | INFO | RTMetrics done | 20260505 08:39:12.286 | INFO | Finished a round of Baz IO: Wrote 2.449 GiB in 3.429 seconds | 20260505 08:39:12.286 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 27.29% / 0.02% / 72.69% | 20260505 08:39:12.286 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.49901% / 2.38419e-05% / 9.13938e-05% | 20260505 08:39:12.286 | INFO | Baz IO overall: | 20260505 08:39:12.286 | INFO | BytesWritten : 1043543947326 | 20260505 08:39:12.286 | INFO | NumEvents : 496175661911 | 20260505 08:39:13.856 | INFO | Analyzing chunk frames = [1839104,1839616) | 20260505 08:39:13.856 | INFO | MemInfo: { | 20260505 08:39:13.856 | INFO | "gpu" : " 17 GB", | 20260505 08:39:13.856 | INFO | "huge" : "", | 20260505 08:39:13.856 | INFO | "huge_shared" : " 27 GB", | 20260505 08:39:13.856 | INFO | "malloc" : " 14 GB", | 20260505 08:39:13.856 | INFO | "pinned" : " 29 GB" | 20260505 08:39:13.856 | INFO | } | 20260505 08:39:13.856 | INFO | Perf report for the last 5.1406 seconds: | 20260505 08:39:13.856 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:39:13.856 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:13.856 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:13.856 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:13.856 | INFO | ANALYSIS : 65.5%, 2.9, 100.3ms, 0 | 20260505 08:39:13.856 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:13.856 | INFO | RT_METRICS : 46.7%, 1.0, 24.5ms, 0 | 20260505 08:39:13.856 | INFO | PRE_HQ : 63.5%, 1.9, 61.1ms, 0 | 20260505 08:39:13.856 | INFO | BAZWRITER : 65.8%, 4.2, 140.7ms, 0 | 20260505 08:39:13.857 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 08:39:13.857 | INFO | Uploaded 36GiB to GPU in 2.24388s (16.0436GiB/s) | 20260505 08:39:13.857 | INFO | Downloaded 38GiB to GPU in 2.35023s (16.1686GiB/s) | 20260505 08:39:18.997 | INFO | Analyzing chunk frames = [1839616,1840128) | 20260505 08:39:18.997 | INFO | MemInfo: { | 20260505 08:39:18.997 | INFO | "gpu" : " 17 GB", | 20260505 08:39:18.997 | INFO | "huge" : "", | 20260505 08:39:18.998 | INFO | "huge_shared" : " 27 GB", | 20260505 08:39:18.998 | INFO | "malloc" : " 14 GB", | 20260505 08:39:18.998 | INFO | "pinned" : " 29 GB" | 20260505 08:39:18.998 | INFO | } | 20260505 08:39:18.998 | INFO | Perf report for the last 5.14136 seconds: | 20260505 08:39:18.998 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:39:18.998 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:18.998 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:18.998 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:18.998 | INFO | ANALYSIS : 64.4%, 3.0, 100.6ms, 0 | 20260505 08:39:18.998 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:18.998 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:18.998 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 08:39:18.998 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:18.998 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:18.998 | INFO | Uploaded 36GiB to GPU in 2.23114s (16.1353GiB/s) | 20260505 08:39:18.998 | INFO | Downloaded 34GiB to GPU in 2.09791s (16.2066GiB/s) | 20260505 08:39:24.134 | INFO | Analyzing chunk frames = [1840128,1840640) | 20260505 08:39:24.134 | INFO | MemInfo: { | 20260505 08:39:24.134 | INFO | "gpu" : " 17 GB", | 20260505 08:39:24.134 | INFO | "huge" : "", | 20260505 08:39:24.134 | INFO | "huge_shared" : " 27 GB", | 20260505 08:39:24.134 | INFO | "malloc" : " 14 GB", | 20260505 08:39:24.134 | INFO | "pinned" : " 29 GB" | 20260505 08:39:24.134 | INFO | } | 20260505 08:39:24.134 | INFO | Perf report for the last 5.13646 seconds: | 20260505 08:39:24.134 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:39:24.134 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:24.134 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:24.134 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:24.134 | INFO | ANALYSIS : 63.6%, 3.0, 98.9ms, 0 | 20260505 08:39:24.134 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:24.134 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:24.134 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 08:39:24.134 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:24.135 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:24.135 | INFO | Uploaded 36GiB to GPU in 2.16888s (16.5984GiB/s) | 20260505 08:39:24.135 | INFO | Downloaded 34GiB to GPU in 2.06643s (16.4535GiB/s) | 20260505 08:39:29.273 | INFO | Analyzing chunk frames = [1840640,1841152) | 20260505 08:39:29.273 | INFO | MemInfo: { | 20260505 08:39:29.273 | INFO | "gpu" : " 17 GB", | 20260505 08:39:29.273 | INFO | "huge" : "", | 20260505 08:39:29.273 | INFO | "huge_shared" : " 27 GB", | 20260505 08:39:29.273 | INFO | "malloc" : " 14 GB", | 20260505 08:39:29.273 | INFO | "pinned" : " 29 GB" | 20260505 08:39:29.273 | INFO | } | 20260505 08:39:29.273 | INFO | Perf report for the last 5.13916 seconds: | 20260505 08:39:29.273 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:39:29.273 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:29.273 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:29.273 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:29.273 | INFO | ANALYSIS : 63.4%, 3.0, 98.6ms, 0 | 20260505 08:39:29.273 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:29.273 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:29.273 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 08:39:29.273 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:29.273 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:29.274 | INFO | Uploaded 36GiB to GPU in 2.1577s (16.6844GiB/s) | 20260505 08:39:29.274 | INFO | Downloaded 34GiB to GPU in 2.06239s (16.4857GiB/s) | 20260505 08:39:34.413 | INFO | Analyzing chunk frames = [1841152,1841664) | 20260505 08:39:34.413 | INFO | MemInfo: { | 20260505 08:39:34.413 | INFO | "gpu" : " 17 GB", | 20260505 08:39:34.413 | INFO | "huge" : "", | 20260505 08:39:34.413 | INFO | "huge_shared" : " 27 GB", | 20260505 08:39:34.413 | INFO | "malloc" : " 14 GB", | 20260505 08:39:34.413 | INFO | "pinned" : " 29 GB" | 20260505 08:39:34.413 | INFO | } | 20260505 08:39:34.413 | INFO | Perf report for the last 5.13936 seconds: | 20260505 08:39:34.413 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:39:34.413 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:34.413 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:34.413 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:34.413 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 08:39:34.413 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:34.413 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:34.413 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 08:39:34.413 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:34.413 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:34.413 | INFO | Uploaded 36GiB to GPU in 2.17001s (16.5898GiB/s) | 20260505 08:39:34.413 | INFO | Downloaded 34GiB to GPU in 2.05769s (16.5234GiB/s) | 20260505 08:39:39.553 | INFO | Analyzing chunk frames = [1841664,1842176) | 20260505 08:39:39.553 | INFO | MemInfo: { | 20260505 08:39:39.553 | INFO | "gpu" : " 17 GB", | 20260505 08:39:39.553 | INFO | "huge" : "", | 20260505 08:39:39.553 | INFO | "huge_shared" : " 27 GB", | 20260505 08:39:39.553 | INFO | "malloc" : " 14 GB", | 20260505 08:39:39.553 | INFO | "pinned" : " 29 GB" | 20260505 08:39:39.553 | INFO | } | 20260505 08:39:39.553 | INFO | Perf report for the last 5.14049 seconds: | 20260505 08:39:39.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:39:39.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:39.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:39.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:39.553 | INFO | ANALYSIS : 64.1%, 3.0, 100.2ms, 0 | 20260505 08:39:39.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:39.553 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:39.553 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 08:39:39.553 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:39.553 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:39.554 | INFO | Uploaded 36GiB to GPU in 2.24889s (16.0079GiB/s) | 20260505 08:39:39.554 | INFO | Downloaded 34GiB to GPU in 2.1096s (16.1168GiB/s) | 20260505 08:39:44.691 | INFO | Analyzing chunk frames = [1842176,1842688) | 20260505 08:39:44.691 | INFO | MemInfo: { | 20260505 08:39:44.691 | INFO | "gpu" : " 17 GB", | 20260505 08:39:44.691 | INFO | "huge" : "", | 20260505 08:39:44.691 | INFO | "huge_shared" : " 27 GB", | 20260505 08:39:44.691 | INFO | "malloc" : " 14 GB", | 20260505 08:39:44.691 | INFO | "pinned" : " 29 GB" | 20260505 08:39:44.691 | INFO | } | 20260505 08:39:44.691 | INFO | Perf report for the last 5.13753 seconds: | 20260505 08:39:44.691 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:39:44.691 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:44.691 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:44.691 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:44.691 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 08:39:44.691 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:44.691 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:44.691 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 08:39:44.691 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:44.691 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:44.691 | INFO | Uploaded 36GiB to GPU in 2.178s (16.5289GiB/s) | 20260505 08:39:44.691 | INFO | Downloaded 34GiB to GPU in 2.14449s (15.8546GiB/s) | 20260505 08:39:49.830 | INFO | Analyzing chunk frames = [1842688,1843200) | 20260505 08:39:49.830 | INFO | MemInfo: { | 20260505 08:39:49.830 | INFO | "gpu" : " 17 GB", | 20260505 08:39:49.830 | INFO | "huge" : "", | 20260505 08:39:49.831 | INFO | "huge_shared" : " 27 GB", | 20260505 08:39:49.831 | INFO | "malloc" : " 14 GB", | 20260505 08:39:49.831 | INFO | "pinned" : " 29 GB" | 20260505 08:39:49.831 | INFO | } | 20260505 08:39:49.831 | INFO | Perf report for the last 5.13989 seconds: | 20260505 08:39:49.831 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:39:49.831 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:49.831 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:49.831 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:49.831 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 08:39:49.831 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:49.831 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:49.831 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 08:39:49.831 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:49.831 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:49.831 | INFO | Uploaded 36GiB to GPU in 2.19652s (16.3896GiB/s) | 20260505 08:39:49.831 | INFO | Downloaded 34GiB to GPU in 2.04192s (16.651GiB/s) | 20260505 08:39:49.831 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 08:39:49.831 | INFO | Total mem usage: 15.3 GiB | 20260505 08:39:49.831 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 08:39:49.831 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.39%) | 20260505 08:39:49.831 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 08:39:49.831 | INFO | High Water Breakdown: | 20260505 08:39:49.831 | INFO | 60.9% : 8.77 GiB ( 93.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 08:39:49.831 | INFO | 39% : 5.62 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 08:39:49.831 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 08:39:49.831 | INFO | ------------------------------------------ | 20260505 08:39:49.831 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:39:49.831 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 08:39:49.831 | INFO | Total mem usage: 26.5 GiB | 20260505 08:39:49.831 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 08:39:49.831 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 08:39:49.831 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 08:39:49.831 | INFO | High Water Breakdown: | 20260505 08:39:49.831 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:39:49.831 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:39:49.831 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:39:49.831 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:39:49.831 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:39:49.831 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:39:49.832 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 08:39:49.832 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 08:39:49.832 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:39:49.832 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 08:39:49.832 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 08:39:49.832 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 08:39:49.832 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 08:39:49.832 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 08:39:49.832 | INFO | ------------------------------------------ | 20260505 08:39:49.832 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:39:49.832 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 08:39:49.832 | INFO | Total mem usage: 25.6 GiB | 20260505 08:39:49.832 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 08:39:49.832 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 08:39:49.832 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 08:39:49.832 | INFO | High Water Breakdown: | 20260505 08:39:49.832 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 08:39:49.832 | INFO | ------------------------------------------ | 20260505 08:39:49.832 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:39:49.832 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 08:39:49.832 | INFO | Total mem usage: 15.7 GiB | 20260505 08:39:49.832 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 08:39:49.832 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 08:39:49.832 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 08:39:49.832 | INFO | High Water Breakdown: | 20260505 08:39:49.832 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 08:39:49.832 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 08:39:49.832 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 08:39:49.832 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 08:39:49.832 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 08:39:49.832 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 08:39:49.832 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 08:39:49.832 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 08:39:49.832 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 08:39:49.832 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:39:49.832 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 08:39:49.832 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:39:49.832 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 08:39:49.832 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 08:39:49.832 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 08:39:49.832 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 08:39:49.832 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:39:49.832 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 08:39:49.832 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:39:49.832 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:39:49.832 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:39:49.832 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 08:39:49.832 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 08:39:49.832 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:39:49.832 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 08:39:49.832 | INFO | ------------------------------------------ | 20260505 08:39:49.832 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:39:49.832 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 08:39:49.832 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:39:49.832 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:39:49.832 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.56e+01, 1.35e+01 / 1.98e+00 | 20260505 08:39:49.832 | INFO | PerfReport: 0.29%, AggregateStats -- 9.32e-02 / 1.03e-01, 1.04e-02 / 4.56e-03 | 20260505 08:39:49.832 | INFO | PerfReport: 11.67%, Baseline -- 3.80e+00 / 4.48e+00, 6.26e-01 / 2.45e-01 | 20260505 08:39:49.832 | INFO | PerfReport: 3.32%, Histogram -- 1.08e+00 / 1.49e+00, 2.11e-01 / 1.08e-01 | 20260505 08:39:49.832 | INFO | PerfReport: 15.54%, DME -- 4.05e+01 / 4.42e+01, 4.00e+00 / 1.68e+00 | 20260505 08:39:49.832 | INFO | PerfReport: 56.44%, FrameLabeling -- 1.84e+01 / 2.03e+01, 1.90e+00 / 2.12e+00 | 20260505 08:39:49.832 | INFO | PerfReport: 7.48%, PulseAccumulating -- 2.44e+00 / 2.97e+00, 4.09e-01 / 2.63e-01 | 20260505 08:39:49.832 | INFO | PerfReport: 3.88%, Metrics -- 1.27e+00 / 1.57e+00, 3.56e-01 / 1.83e-01 | 20260505 08:39:49.832 | INFO | PerfReport: 1.39%, PulseSerializing -- 4.52e-01 / 5.02e-01, 8.56e-02 / 3.32e-02 | 20260505 08:39:49.832 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 08:39:49.832 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:39:49.832 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:39:49.832 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.33e+01 / 0.00e+00 | 20260505 08:39:49.832 | INFO | PerfReport: 22.50%, Upload -- 2.29e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260505 08:39:49.832 | INFO | PerfReport: 21.67%, Download -- 2.20e+01 / 0.00e+00, 8.29e+00 / 0.00e+00 | 20260505 08:39:49.832 | INFO | PerfReport: 32.05%, Compute -- 3.26e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 08:39:49.832 | INFO | PerfReport: 1.70%, LockUpload -- 1.73e+00 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260505 08:39:49.832 | INFO | PerfReport: 1.07%, LockDownload -- 1.08e+00 / 0.00e+00, 2.60e+00 / 0.00e+00 | 20260505 08:39:49.832 | INFO | PerfReport: 20.99%, LockCompute -- 2.13e+01 / 0.00e+00, 1.78e+01 / 0.00e+00 | 20260505 08:39:49.832 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.59e-02 / 0.00e+00 | 20260505 08:39:49.970 | INFO | Starting a round of Baz IO | 20260505 08:39:53.237 | INFO | Populating RTMetrics report | 20260505 08:39:53.238 | INFO | RTMetrics report populated | 20260505 08:39:53.238 | INFO | Opening tmp RTMetrics json file | 20260505 08:39:53.266 | INFO | Serializing RTMetrics to json | 20260505 08:39:53.291 | INFO | Serialization done, swapping tmp file now... | 20260505 08:39:53.291 | INFO | RTMetrics done | 20260505 08:39:53.388 | INFO | Finished a round of Baz IO: Wrote 2.45 GiB in 3.418 seconds | 20260505 08:39:53.388 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 27.27% / 0.02% / 72.71% | 20260505 08:39:53.388 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.48559% / 3.97364e-05% / 0.000115236% | 20260505 08:39:53.388 | INFO | Baz IO overall: | 20260505 08:39:53.388 | INFO | BytesWritten : 1046175135004 | 20260505 08:39:53.388 | INFO | NumEvents : 497433839906 | 20260505 08:39:54.972 | INFO | Analyzing chunk frames = [1843200,1843712) | 20260505 08:39:54.972 | INFO | MemInfo: { | 20260505 08:39:54.972 | INFO | "gpu" : " 17 GB", | 20260505 08:39:54.972 | INFO | "huge" : "", | 20260505 08:39:54.972 | INFO | "huge_shared" : " 27 GB", | 20260505 08:39:54.972 | INFO | "malloc" : " 14 GB", | 20260505 08:39:54.972 | INFO | "pinned" : " 29 GB" | 20260505 08:39:54.972 | INFO | } | 20260505 08:39:54.972 | INFO | Perf report for the last 5.14095 seconds: | 20260505 08:39:54.972 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:39:54.972 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:54.972 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:54.972 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:54.972 | INFO | ANALYSIS : 65.1%, 2.9, 98.7ms, 0 | 20260505 08:39:54.972 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:39:54.972 | INFO | RT_METRICS : 46.8%, 1.0, 24.5ms, 0 | 20260505 08:39:54.972 | INFO | PRE_HQ : 63.4%, 1.9, 61.2ms, 0 | 20260505 08:39:54.972 | INFO | BAZWRITER : 65.8%, 4.2, 143.4ms, 0 | 20260505 08:39:54.972 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260505 08:39:54.972 | INFO | Uploaded 36GiB to GPU in 2.25774s (15.9452GiB/s) | 20260505 08:39:54.972 | INFO | Downloaded 38GiB to GPU in 2.38922s (15.9048GiB/s) | 20260505 08:40:00.110 | INFO | Analyzing chunk frames = [1843712,1844224) | 20260505 08:40:00.110 | INFO | MemInfo: { | 20260505 08:40:00.110 | INFO | "gpu" : " 17 GB", | 20260505 08:40:00.110 | INFO | "huge" : "", | 20260505 08:40:00.110 | INFO | "huge_shared" : " 27 GB", | 20260505 08:40:00.110 | INFO | "malloc" : " 14 GB", | 20260505 08:40:00.110 | INFO | "pinned" : " 29 GB" | 20260505 08:40:00.110 | INFO | } | 20260505 08:40:00.110 | INFO | Perf report for the last 5.13803 seconds: | 20260505 08:40:00.110 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:40:00.110 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:00.110 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:00.110 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:00.110 | INFO | ANALYSIS : 64.5%, 3.0, 100.7ms, 0 | 20260505 08:40:00.110 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:00.110 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:00.110 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:40:00.110 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:00.110 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:00.110 | INFO | Uploaded 36GiB to GPU in 2.2253s (16.1776GiB/s) | 20260505 08:40:00.110 | INFO | Downloaded 34GiB to GPU in 2.0952s (16.2276GiB/s) | 20260505 08:40:05.249 | INFO | Analyzing chunk frames = [1844224,1844736) | 20260505 08:40:05.249 | INFO | MemInfo: { | 20260505 08:40:05.249 | INFO | "gpu" : " 17 GB", | 20260505 08:40:05.249 | INFO | "huge" : "", | 20260505 08:40:05.249 | INFO | "huge_shared" : " 27 GB", | 20260505 08:40:05.249 | INFO | "malloc" : " 14 GB", | 20260505 08:40:05.249 | INFO | "pinned" : " 29 GB" | 20260505 08:40:05.249 | INFO | } | 20260505 08:40:05.249 | INFO | Perf report for the last 5.13902 seconds: | 20260505 08:40:05.249 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:40:05.249 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:05.249 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:05.249 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:05.249 | INFO | ANALYSIS : 63.6%, 3.0, 98.9ms, 0 | 20260505 08:40:05.249 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:05.249 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:05.249 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:40:05.249 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:05.249 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:05.249 | INFO | Uploaded 36GiB to GPU in 2.17127s (16.5801GiB/s) | 20260505 08:40:05.249 | INFO | Downloaded 34GiB to GPU in 2.06794s (16.4415GiB/s) | 20260505 08:40:10.388 | INFO | Analyzing chunk frames = [1844736,1845248) | 20260505 08:40:10.388 | INFO | MemInfo: { | 20260505 08:40:10.388 | INFO | "gpu" : " 17 GB", | 20260505 08:40:10.388 | INFO | "huge" : "", | 20260505 08:40:10.388 | INFO | "huge_shared" : " 27 GB", | 20260505 08:40:10.388 | INFO | "malloc" : " 14 GB", | 20260505 08:40:10.388 | INFO | "pinned" : " 29 GB" | 20260505 08:40:10.388 | INFO | } | 20260505 08:40:10.388 | INFO | Perf report for the last 5.13914 seconds: | 20260505 08:40:10.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:40:10.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:10.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:10.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:10.388 | INFO | ANALYSIS : 63.5%, 3.0, 98.6ms, 0 | 20260505 08:40:10.388 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:10.388 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:10.388 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 08:40:10.388 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:10.388 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:10.388 | INFO | Uploaded 36GiB to GPU in 2.16166s (16.6539GiB/s) | 20260505 08:40:10.388 | INFO | Downloaded 34GiB to GPU in 2.06063s (16.4998GiB/s) | 20260505 08:40:15.527 | INFO | Analyzing chunk frames = [1845248,1845760) | 20260505 08:40:15.527 | INFO | MemInfo: { | 20260505 08:40:15.527 | INFO | "gpu" : " 17 GB", | 20260505 08:40:15.527 | INFO | "huge" : "", | 20260505 08:40:15.527 | INFO | "huge_shared" : " 27 GB", | 20260505 08:40:15.527 | INFO | "malloc" : " 14 GB", | 20260505 08:40:15.527 | INFO | "pinned" : " 29 GB" | 20260505 08:40:15.527 | INFO | } | 20260505 08:40:15.527 | INFO | Perf report for the last 5.13924 seconds: | 20260505 08:40:15.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:40:15.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:15.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:15.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:15.527 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 08:40:15.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:15.527 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:15.527 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:40:15.527 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:15.527 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:15.527 | INFO | Uploaded 36GiB to GPU in 2.16271s (16.6458GiB/s) | 20260505 08:40:15.527 | INFO | Downloaded 34GiB to GPU in 2.05703s (16.5287GiB/s) | 20260505 08:40:20.667 | INFO | Analyzing chunk frames = [1845760,1846272) | 20260505 08:40:20.668 | INFO | MemInfo: { | 20260505 08:40:20.668 | INFO | "gpu" : " 17 GB", | 20260505 08:40:20.668 | INFO | "huge" : "", | 20260505 08:40:20.668 | INFO | "huge_shared" : " 27 GB", | 20260505 08:40:20.668 | INFO | "malloc" : " 14 GB", | 20260505 08:40:20.668 | INFO | "pinned" : " 29 GB" | 20260505 08:40:20.668 | INFO | } | 20260505 08:40:20.668 | INFO | Perf report for the last 5.14053 seconds: | 20260505 08:40:20.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:40:20.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:20.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:20.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:20.668 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 08:40:20.668 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:20.668 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:20.668 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:40:20.668 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:20.668 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:20.668 | INFO | Uploaded 36GiB to GPU in 2.18187s (16.4996GiB/s) | 20260505 08:40:20.668 | INFO | Downloaded 34GiB to GPU in 2.05228s (16.5669GiB/s) | 20260505 08:40:25.807 | INFO | Analyzing chunk frames = [1846272,1846784) | 20260505 08:40:25.807 | INFO | MemInfo: { | 20260505 08:40:25.807 | INFO | "gpu" : " 17 GB", | 20260505 08:40:25.807 | INFO | "huge" : "", | 20260505 08:40:25.807 | INFO | "huge_shared" : " 27 GB", | 20260505 08:40:25.807 | INFO | "malloc" : " 14 GB", | 20260505 08:40:25.807 | INFO | "pinned" : " 29 GB" | 20260505 08:40:25.807 | INFO | } | 20260505 08:40:25.807 | INFO | Perf report for the last 5.1392 seconds: | 20260505 08:40:25.807 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:40:25.807 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:25.807 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:25.807 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:25.807 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 08:40:25.807 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:25.807 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:25.807 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 08:40:25.807 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:25.807 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:25.807 | INFO | Uploaded 36GiB to GPU in 2.17736s (16.5338GiB/s) | 20260505 08:40:25.807 | INFO | Downloaded 34GiB to GPU in 2.15556s (15.7732GiB/s) | 20260505 08:40:30.948 | INFO | Analyzing chunk frames = [1846784,1847296) | 20260505 08:40:30.948 | INFO | MemInfo: { | 20260505 08:40:30.948 | INFO | "gpu" : " 17 GB", | 20260505 08:40:30.948 | INFO | "huge" : "", | 20260505 08:40:30.948 | INFO | "huge_shared" : " 27 GB", | 20260505 08:40:30.948 | INFO | "malloc" : " 14 GB", | 20260505 08:40:30.948 | INFO | "pinned" : " 29 GB" | 20260505 08:40:30.949 | INFO | } | 20260505 08:40:30.949 | INFO | Perf report for the last 5.14168 seconds: | 20260505 08:40:30.949 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:40:30.949 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:30.949 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:30.949 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:30.949 | INFO | ANALYSIS : 64.0%, 3.0, 100.1ms, 0 | 20260505 08:40:30.949 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:30.949 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:30.949 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 08:40:30.949 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:30.949 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:30.949 | INFO | Uploaded 36GiB to GPU in 2.19122s (16.4292GiB/s) | 20260505 08:40:30.949 | INFO | Downloaded 34GiB to GPU in 2.04164s (16.6533GiB/s) | 20260505 08:40:31.086 | INFO | Starting a round of Baz IO | 20260505 08:40:34.368 | INFO | Populating RTMetrics report | 20260505 08:40:34.368 | INFO | RTMetrics report populated | 20260505 08:40:34.368 | INFO | Opening tmp RTMetrics json file | 20260505 08:40:34.413 | INFO | Serializing RTMetrics to json | 20260505 08:40:34.414 | INFO | Serialization done, swapping tmp file now... | 20260505 08:40:34.414 | INFO | RTMetrics done | 20260505 08:40:34.524 | INFO | Finished a round of Baz IO: Wrote 2.449 GiB in 3.437 seconds | 20260505 08:40:34.524 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 27.25% / 0.02% / 72.73% | 20260505 08:40:34.524 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.47668% / 3.57628e-05% / 8.74201e-05% | 20260505 08:40:34.524 | INFO | Baz IO overall: | 20260505 08:40:34.524 | INFO | BytesWritten : 1048804635900 | 20260505 08:40:34.524 | INFO | NumEvents : 498691143686 | 20260505 08:40:36.085 | INFO | Analyzing chunk frames = [1847296,1847808) | 20260505 08:40:36.085 | INFO | MemInfo: { | 20260505 08:40:36.085 | INFO | "gpu" : " 17 GB", | 20260505 08:40:36.085 | INFO | "huge" : "", | 20260505 08:40:36.085 | INFO | "huge_shared" : " 27 GB", | 20260505 08:40:36.085 | INFO | "malloc" : " 14 GB", | 20260505 08:40:36.085 | INFO | "pinned" : " 29 GB" | 20260505 08:40:36.085 | INFO | } | 20260505 08:40:36.085 | INFO | Perf report for the last 5.13687 seconds: | 20260505 08:40:36.085 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:40:36.085 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:36.085 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:36.086 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:36.086 | INFO | ANALYSIS : 65.4%, 2.9, 99.9ms, 0 | 20260505 08:40:36.086 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:36.086 | INFO | RT_METRICS : 47.0%, 1.0, 24.6ms, 0 | 20260505 08:40:36.086 | INFO | PRE_HQ : 63.2%, 1.9, 61.5ms, 0 | 20260505 08:40:36.086 | INFO | BAZWRITER : 66.2%, 4.2, 141.4ms, 0 | 20260505 08:40:36.086 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260505 08:40:36.086 | INFO | Uploaded 36GiB to GPU in 2.23582s (16.1015GiB/s) | 20260505 08:40:36.086 | INFO | Downloaded 38GiB to GPU in 2.35299s (16.1497GiB/s) | 20260505 08:40:41.224 | INFO | Analyzing chunk frames = [1847808,1848320) | 20260505 08:40:41.224 | INFO | MemInfo: { | 20260505 08:40:41.224 | INFO | "gpu" : " 17 GB", | 20260505 08:40:41.224 | INFO | "huge" : "", | 20260505 08:40:41.224 | INFO | "huge_shared" : " 27 GB", | 20260505 08:40:41.224 | INFO | "malloc" : " 14 GB", | 20260505 08:40:41.224 | INFO | "pinned" : " 29 GB" | 20260505 08:40:41.224 | INFO | } | 20260505 08:40:41.224 | INFO | Perf report for the last 5.139 seconds: | 20260505 08:40:41.225 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:40:41.225 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:41.225 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:41.225 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:41.225 | INFO | ANALYSIS : 64.5%, 3.0, 100.7ms, 0 | 20260505 08:40:41.225 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:41.225 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:41.225 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:40:41.225 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:41.225 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:41.225 | INFO | Uploaded 36GiB to GPU in 2.21674s (16.2401GiB/s) | 20260505 08:40:41.225 | INFO | Downloaded 34GiB to GPU in 2.08708s (16.2907GiB/s) | 20260505 08:40:46.364 | INFO | Analyzing chunk frames = [1848320,1848832) | 20260505 08:40:46.364 | INFO | MemInfo: { | 20260505 08:40:46.364 | INFO | "gpu" : " 17 GB", | 20260505 08:40:46.364 | INFO | "huge" : "", | 20260505 08:40:46.364 | INFO | "huge_shared" : " 27 GB", | 20260505 08:40:46.364 | INFO | "malloc" : " 14 GB", | 20260505 08:40:46.364 | INFO | "pinned" : " 29 GB" | 20260505 08:40:46.364 | INFO | } | 20260505 08:40:46.364 | INFO | Perf report for the last 5.13972 seconds: | 20260505 08:40:46.364 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:40:46.364 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:46.364 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:46.364 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:46.364 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260505 08:40:46.364 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:46.364 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:46.364 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:40:46.364 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:46.364 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:46.364 | INFO | Uploaded 36GiB to GPU in 2.16883s (16.5988GiB/s) | 20260505 08:40:46.365 | INFO | Downloaded 34GiB to GPU in 2.06375s (16.4748GiB/s) | 20260505 08:40:51.503 | INFO | Analyzing chunk frames = [1848832,1849344) | 20260505 08:40:51.503 | INFO | MemInfo: { | 20260505 08:40:51.503 | INFO | "gpu" : " 17 GB", | 20260505 08:40:51.503 | INFO | "huge" : "", | 20260505 08:40:51.503 | INFO | "huge_shared" : " 27 GB", | 20260505 08:40:51.503 | INFO | "malloc" : " 14 GB", | 20260505 08:40:51.503 | INFO | "pinned" : " 29 GB" | 20260505 08:40:51.503 | INFO | } | 20260505 08:40:51.503 | INFO | Perf report for the last 5.1388 seconds: | 20260505 08:40:51.503 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:40:51.503 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:51.503 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:51.503 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:51.503 | INFO | ANALYSIS : 63.6%, 3.0, 98.8ms, 0 | 20260505 08:40:51.503 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:51.503 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:51.503 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:40:51.503 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:51.503 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:51.503 | INFO | Uploaded 36GiB to GPU in 2.18019s (16.5123GiB/s) | 20260505 08:40:51.503 | INFO | Downloaded 34GiB to GPU in 2.06091s (16.4976GiB/s) | 20260505 08:40:56.642 | INFO | Analyzing chunk frames = [1849344,1849856) | 20260505 08:40:56.642 | INFO | MemInfo: { | 20260505 08:40:56.642 | INFO | "gpu" : " 17 GB", | 20260505 08:40:56.642 | INFO | "huge" : "", | 20260505 08:40:56.642 | INFO | "huge_shared" : " 27 GB", | 20260505 08:40:56.643 | INFO | "malloc" : " 14 GB", | 20260505 08:40:56.643 | INFO | "pinned" : " 29 GB" | 20260505 08:40:56.643 | INFO | } | 20260505 08:40:56.643 | INFO | Perf report for the last 5.13952 seconds: | 20260505 08:40:56.643 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:40:56.643 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:56.643 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:56.643 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:56.643 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 08:40:56.643 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:56.643 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:56.643 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 08:40:56.643 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:56.643 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:40:56.643 | INFO | Uploaded 36GiB to GPU in 2.17174s (16.5766GiB/s) | 20260505 08:40:56.643 | INFO | Downloaded 34GiB to GPU in 2.0562s (16.5354GiB/s) | 20260505 08:41:01.782 | INFO | Analyzing chunk frames = [1849856,1850368) | 20260505 08:41:01.782 | INFO | MemInfo: { | 20260505 08:41:01.782 | INFO | "gpu" : " 17 GB", | 20260505 08:41:01.782 | INFO | "huge" : "", | 20260505 08:41:01.782 | INFO | "huge_shared" : " 27 GB", | 20260505 08:41:01.782 | INFO | "malloc" : " 14 GB", | 20260505 08:41:01.782 | INFO | "pinned" : " 29 GB" | 20260505 08:41:01.782 | INFO | } | 20260505 08:41:01.782 | INFO | Perf report for the last 5.13984 seconds: | 20260505 08:41:01.782 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:41:01.782 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:01.782 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:01.782 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:01.782 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 08:41:01.782 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:01.782 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:01.783 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 08:41:01.783 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:01.783 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:01.783 | INFO | Uploaded 36GiB to GPU in 2.19411s (16.4076GiB/s) | 20260505 08:41:01.783 | INFO | Downloaded 34GiB to GPU in 2.05558s (16.5403GiB/s) | 20260505 08:41:06.922 | INFO | Analyzing chunk frames = [1850368,1850880) | 20260505 08:41:06.922 | INFO | MemInfo: { | 20260505 08:41:06.922 | INFO | "gpu" : " 17 GB", | 20260505 08:41:06.922 | INFO | "huge" : "", | 20260505 08:41:06.922 | INFO | "huge_shared" : " 27 GB", | 20260505 08:41:06.922 | INFO | "malloc" : " 14 GB", | 20260505 08:41:06.922 | INFO | "pinned" : " 29 GB" | 20260505 08:41:06.922 | INFO | } | 20260505 08:41:06.922 | INFO | Perf report for the last 5.13928 seconds: | 20260505 08:41:06.922 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:41:06.922 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:06.922 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:06.922 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:06.922 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 08:41:06.922 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:06.922 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:06.922 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260505 08:41:06.922 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:06.922 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:06.922 | INFO | Uploaded 36GiB to GPU in 2.21725s (16.2363GiB/s) | 20260505 08:41:06.922 | INFO | Downloaded 34GiB to GPU in 2.15453s (15.7807GiB/s) | 20260505 08:41:12.060 | INFO | Analyzing chunk frames = [1850880,1851392) | 20260505 08:41:12.060 | INFO | MemInfo: { | 20260505 08:41:12.060 | INFO | "gpu" : " 17 GB", | 20260505 08:41:12.060 | INFO | "huge" : "", | 20260505 08:41:12.061 | INFO | "huge_shared" : " 27 GB", | 20260505 08:41:12.061 | INFO | "malloc" : " 14 GB", | 20260505 08:41:12.061 | INFO | "pinned" : " 29 GB" | 20260505 08:41:12.061 | INFO | } | 20260505 08:41:12.061 | INFO | Perf report for the last 5.13889 seconds: | 20260505 08:41:12.061 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:41:12.061 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:12.061 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:12.061 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:12.061 | INFO | ANALYSIS : 64.1%, 3.0, 100.0ms, 0 | 20260505 08:41:12.061 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:12.061 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:12.061 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:41:12.061 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:12.061 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:12.061 | INFO | Uploaded 36GiB to GPU in 2.19374s (16.4103GiB/s) | 20260505 08:41:12.061 | INFO | Downloaded 34GiB to GPU in 2.0374s (16.688GiB/s) | 20260505 08:41:12.061 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 08:41:12.061 | INFO | Total mem usage: 15.3 GiB | 20260505 08:41:12.061 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 08:41:12.061 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -5.62%) | 20260505 08:41:12.061 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 08:41:12.061 | INFO | High Water Breakdown: | 20260505 08:41:12.061 | INFO | 61% : 8.85 GiB ( 94.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 08:41:12.061 | INFO | 38.9% : 5.64 GiB ( 84.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 08:41:12.061 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 08:41:12.061 | INFO | ------------------------------------------ | 20260505 08:41:12.061 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:41:12.061 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 08:41:12.061 | INFO | Total mem usage: 26.5 GiB | 20260505 08:41:12.061 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 08:41:12.061 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 08:41:12.061 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 08:41:12.061 | INFO | High Water Breakdown: | 20260505 08:41:12.061 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:41:12.061 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:41:12.061 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:41:12.061 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:41:12.061 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:41:12.061 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:41:12.061 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 08:41:12.061 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 08:41:12.061 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:41:12.061 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 08:41:12.061 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 08:41:12.061 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 08:41:12.061 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 08:41:12.061 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 08:41:12.061 | INFO | ------------------------------------------ | 20260505 08:41:12.061 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:41:12.061 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 08:41:12.061 | INFO | Total mem usage: 25.6 GiB | 20260505 08:41:12.061 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 08:41:12.061 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 08:41:12.061 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 08:41:12.061 | INFO | High Water Breakdown: | 20260505 08:41:12.061 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 08:41:12.061 | INFO | ------------------------------------------ | 20260505 08:41:12.061 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:41:12.061 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 08:41:12.061 | INFO | Total mem usage: 15.7 GiB | 20260505 08:41:12.061 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 08:41:12.061 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 08:41:12.061 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 08:41:12.061 | INFO | High Water Breakdown: | 20260505 08:41:12.061 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 08:41:12.061 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 08:41:12.061 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 08:41:12.061 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 08:41:12.061 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 08:41:12.061 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 08:41:12.061 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 08:41:12.061 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 08:41:12.061 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 08:41:12.061 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:41:12.061 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 08:41:12.062 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:41:12.062 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 08:41:12.062 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 08:41:12.062 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 08:41:12.062 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 08:41:12.062 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:41:12.062 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 08:41:12.062 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:41:12.062 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:41:12.062 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:41:12.062 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 08:41:12.062 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 08:41:12.062 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:41:12.062 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 08:41:12.062 | INFO | ------------------------------------------ | 20260505 08:41:12.062 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:41:12.062 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 08:41:12.062 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:41:12.062 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:41:12.062 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.50e+01, 1.37e+01 / 1.63e+00 | 20260505 08:41:12.062 | INFO | PerfReport: 0.29%, AggregateStats -- 9.32e-02 / 1.04e-01, 1.04e-02 / 4.66e-03 | 20260505 08:41:12.062 | INFO | PerfReport: 11.67%, Baseline -- 3.80e+00 / 4.48e+00, 6.20e-01 / 2.47e-01 | 20260505 08:41:12.062 | INFO | PerfReport: 3.32%, Histogram -- 1.08e+00 / 1.47e+00, 2.10e-01 / 1.04e-01 | 20260505 08:41:12.062 | INFO | PerfReport: 15.50%, DME -- 4.04e+01 / 4.40e+01, 4.00e+00 / 1.48e+00 | 20260505 08:41:12.062 | INFO | PerfReport: 56.44%, FrameLabeling -- 1.84e+01 / 1.98e+01, 1.90e+00 / 1.77e+00 | 20260505 08:41:12.062 | INFO | PerfReport: 7.50%, PulseAccumulating -- 2.44e+00 / 3.00e+00, 4.12e-01 / 2.54e-01 | 20260505 08:41:12.062 | INFO | PerfReport: 3.90%, Metrics -- 1.27e+00 / 1.58e+00, 3.56e-01 / 1.72e-01 | 20260505 08:41:12.062 | INFO | PerfReport: 1.39%, PulseSerializing -- 4.52e-01 / 5.07e-01, 8.56e-02 / 2.55e-02 | 20260505 08:41:12.062 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 08:41:12.062 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:41:12.062 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:41:12.062 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.31e+01 / 0.00e+00 | 20260505 08:41:12.062 | INFO | PerfReport: 22.53%, Upload -- 2.29e+01 / 0.00e+00, 8.16e+00 / 0.00e+00 | 20260505 08:41:12.062 | INFO | PerfReport: 21.68%, Download -- 2.20e+01 / 0.00e+00, 8.17e+00 / 0.00e+00 | 20260505 08:41:12.062 | INFO | PerfReport: 32.10%, Compute -- 3.26e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 08:41:12.062 | INFO | PerfReport: 1.69%, LockUpload -- 1.71e+00 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260505 08:41:12.062 | INFO | PerfReport: 1.09%, LockDownload -- 1.11e+00 / 0.00e+00, 2.66e+00 / 0.00e+00 | 20260505 08:41:12.062 | INFO | PerfReport: 20.89%, LockCompute -- 2.12e+01 / 0.00e+00, 1.77e+01 / 0.00e+00 | 20260505 08:41:12.062 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.71e-02 / 0.00e+00 | 20260505 08:41:12.199 | INFO | Starting a round of Baz IO | 20260505 08:41:15.477 | INFO | Populating RTMetrics report | 20260505 08:41:15.478 | INFO | RTMetrics report populated | 20260505 08:41:15.478 | INFO | Opening tmp RTMetrics json file | 20260505 08:41:15.481 | INFO | Serializing RTMetrics to json | 20260505 08:41:15.483 | INFO | Serialization done, swapping tmp file now... | 20260505 08:41:15.483 | INFO | RTMetrics done | 20260505 08:41:15.666 | INFO | Finished a round of Baz IO: Wrote 2.452 GiB in 3.466 seconds | 20260505 08:41:15.666 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 27.23% / 0.02% / 72.75% | 20260505 08:41:15.666 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.47506% / 1.98682e-05% / 0.000107288% | 20260505 08:41:15.666 | INFO | Baz IO overall: | 20260505 08:41:15.666 | INFO | BytesWritten : 1051437628415 | 20260505 08:41:15.666 | INFO | NumEvents : 499951169505 | 20260505 08:41:17.200 | INFO | Analyzing chunk frames = [1851392,1851904) | 20260505 08:41:17.200 | INFO | MemInfo: { | 20260505 08:41:17.200 | INFO | "gpu" : " 17 GB", | 20260505 08:41:17.200 | INFO | "huge" : "", | 20260505 08:41:17.200 | INFO | "huge_shared" : " 27 GB", | 20260505 08:41:17.200 | INFO | "malloc" : " 14 GB", | 20260505 08:41:17.200 | INFO | "pinned" : " 29 GB" | 20260505 08:41:17.200 | INFO | } | 20260505 08:41:17.200 | INFO | Perf report for the last 5.1396 seconds: | 20260505 08:41:17.200 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:41:17.200 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:17.200 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:17.200 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:17.200 | INFO | ANALYSIS : 65.3%, 2.9, 99.7ms, 0 | 20260505 08:41:17.200 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:17.200 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 08:41:17.200 | INFO | PRE_HQ : 63.4%, 1.9, 61.3ms, 0 | 20260505 08:41:17.200 | INFO | BAZWRITER : 66.8%, 4.2, 144.6ms, 0 | 20260505 08:41:17.200 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 08:41:17.200 | INFO | Uploaded 36GiB to GPU in 2.26849s (15.8696GiB/s) | 20260505 08:41:17.200 | INFO | Downloaded 38GiB to GPU in 2.37239s (16.0176GiB/s) | 20260505 08:41:22.338 | INFO | Analyzing chunk frames = [1851904,1852416) | 20260505 08:41:22.338 | INFO | MemInfo: { | 20260505 08:41:22.338 | INFO | "gpu" : " 17 GB", | 20260505 08:41:22.338 | INFO | "huge" : "", | 20260505 08:41:22.339 | INFO | "huge_shared" : " 27 GB", | 20260505 08:41:22.339 | INFO | "malloc" : " 14 GB", | 20260505 08:41:22.339 | INFO | "pinned" : " 29 GB" | 20260505 08:41:22.339 | INFO | } | 20260505 08:41:22.339 | INFO | Perf report for the last 5.13832 seconds: | 20260505 08:41:22.339 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:41:22.339 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:22.339 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:22.339 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:22.339 | INFO | ANALYSIS : 64.7%, 3.0, 101.1ms, 0 | 20260505 08:41:22.339 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:22.339 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:22.339 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 08:41:22.339 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:22.339 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:22.339 | INFO | Uploaded 36GiB to GPU in 2.24954s (16.0032GiB/s) | 20260505 08:41:22.339 | INFO | Downloaded 34GiB to GPU in 2.09769s (16.2083GiB/s) | 20260505 08:41:27.479 | INFO | Analyzing chunk frames = [1852416,1852928) | 20260505 08:41:27.479 | INFO | MemInfo: { | 20260505 08:41:27.479 | INFO | "gpu" : " 17 GB", | 20260505 08:41:27.479 | INFO | "huge" : "", | 20260505 08:41:27.479 | INFO | "huge_shared" : " 27 GB", | 20260505 08:41:27.479 | INFO | "malloc" : " 14 GB", | 20260505 08:41:27.479 | INFO | "pinned" : " 29 GB" | 20260505 08:41:27.479 | INFO | } | 20260505 08:41:27.479 | INFO | Perf report for the last 5.14012 seconds: | 20260505 08:41:27.479 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:41:27.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:27.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:27.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:27.479 | INFO | ANALYSIS : 63.6%, 3.0, 99.1ms, 0 | 20260505 08:41:27.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:27.479 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:27.479 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:41:27.479 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:27.479 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:27.479 | INFO | Uploaded 36GiB to GPU in 2.18032s (16.5114GiB/s) | 20260505 08:41:27.479 | INFO | Downloaded 34GiB to GPU in 2.06576s (16.4588GiB/s) | 20260505 08:41:32.620 | INFO | Analyzing chunk frames = [1852928,1853440) | 20260505 08:41:32.620 | INFO | MemInfo: { | 20260505 08:41:32.620 | INFO | "gpu" : " 17 GB", | 20260505 08:41:32.620 | INFO | "huge" : "", | 20260505 08:41:32.620 | INFO | "huge_shared" : " 27 GB", | 20260505 08:41:32.620 | INFO | "malloc" : " 14 GB", | 20260505 08:41:32.620 | INFO | "pinned" : " 29 GB" | 20260505 08:41:32.620 | INFO | } | 20260505 08:41:32.620 | INFO | Perf report for the last 5.14137 seconds: | 20260505 08:41:32.620 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:41:32.620 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:32.620 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:32.620 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:32.620 | INFO | ANALYSIS : 63.5%, 3.0, 98.6ms, 0 | 20260505 08:41:32.620 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:32.620 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:32.620 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:41:32.620 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:32.620 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:32.620 | INFO | Uploaded 36GiB to GPU in 2.1701s (16.5891GiB/s) | 20260505 08:41:32.620 | INFO | Downloaded 34GiB to GPU in 2.06531s (16.4624GiB/s) | 20260505 08:41:37.756 | INFO | Analyzing chunk frames = [1853440,1853952) | 20260505 08:41:37.757 | INFO | MemInfo: { | 20260505 08:41:37.757 | INFO | "gpu" : " 17 GB", | 20260505 08:41:37.757 | INFO | "huge" : "", | 20260505 08:41:37.757 | INFO | "huge_shared" : " 27 GB", | 20260505 08:41:37.757 | INFO | "malloc" : " 14 GB", | 20260505 08:41:37.757 | INFO | "pinned" : " 29 GB" | 20260505 08:41:37.757 | INFO | } | 20260505 08:41:37.757 | INFO | Perf report for the last 5.13653 seconds: | 20260505 08:41:37.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:41:37.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:37.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:37.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:37.757 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 08:41:37.757 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:37.757 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:37.757 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 08:41:37.757 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:37.757 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:37.757 | INFO | Uploaded 36GiB to GPU in 2.16544s (16.6248GiB/s) | 20260505 08:41:37.757 | INFO | Downloaded 34GiB to GPU in 2.0578s (16.5225GiB/s) | 20260505 08:41:42.897 | INFO | Analyzing chunk frames = [1853952,1854464) | 20260505 08:41:42.897 | INFO | MemInfo: { | 20260505 08:41:42.897 | INFO | "gpu" : " 17 GB", | 20260505 08:41:42.897 | INFO | "huge" : "", | 20260505 08:41:42.897 | INFO | "huge_shared" : " 27 GB", | 20260505 08:41:42.897 | INFO | "malloc" : " 14 GB", | 20260505 08:41:42.897 | INFO | "pinned" : " 29 GB" | 20260505 08:41:42.897 | INFO | } | 20260505 08:41:42.897 | INFO | Perf report for the last 5.14062 seconds: | 20260505 08:41:42.897 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:41:42.897 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:42.897 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:42.897 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:42.897 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 08:41:42.897 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:42.897 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:42.897 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:41:42.897 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:42.897 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:42.898 | INFO | Uploaded 36GiB to GPU in 2.16825s (16.6033GiB/s) | 20260505 08:41:42.898 | INFO | Downloaded 34GiB to GPU in 2.05749s (16.525GiB/s) | 20260505 08:41:48.036 | INFO | Analyzing chunk frames = [1854464,1854976) | 20260505 08:41:48.036 | INFO | MemInfo: { | 20260505 08:41:48.036 | INFO | "gpu" : " 17 GB", | 20260505 08:41:48.036 | INFO | "huge" : "", | 20260505 08:41:48.036 | INFO | "huge_shared" : " 27 GB", | 20260505 08:41:48.036 | INFO | "malloc" : " 14 GB", | 20260505 08:41:48.036 | INFO | "pinned" : " 29 GB" | 20260505 08:41:48.036 | INFO | } | 20260505 08:41:48.036 | INFO | Perf report for the last 5.13899 seconds: | 20260505 08:41:48.036 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:41:48.036 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:48.036 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:48.036 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:48.036 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 08:41:48.036 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:48.036 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:48.036 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260505 08:41:48.036 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:48.036 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:48.037 | INFO | Uploaded 36GiB to GPU in 2.22602s (16.1723GiB/s) | 20260505 08:41:48.037 | INFO | Downloaded 34GiB to GPU in 2.15457s (15.7804GiB/s) | 20260505 08:41:53.176 | INFO | Analyzing chunk frames = [1854976,1855488) | 20260505 08:41:53.176 | INFO | MemInfo: { | 20260505 08:41:53.176 | INFO | "gpu" : " 17 GB", | 20260505 08:41:53.176 | INFO | "huge" : "", | 20260505 08:41:53.176 | INFO | "huge_shared" : " 27 GB", | 20260505 08:41:53.176 | INFO | "malloc" : " 14 GB", | 20260505 08:41:53.176 | INFO | "pinned" : " 29 GB" | 20260505 08:41:53.176 | INFO | } | 20260505 08:41:53.176 | INFO | Perf report for the last 5.13981 seconds: | 20260505 08:41:53.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:41:53.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:53.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:53.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:53.176 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 08:41:53.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:53.176 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:53.176 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 08:41:53.176 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:53.176 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:53.176 | INFO | Uploaded 36GiB to GPU in 2.17045s (16.5865GiB/s) | 20260505 08:41:53.176 | INFO | Downloaded 34GiB to GPU in 2.02343s (16.8032GiB/s) | 20260505 08:41:53.316 | INFO | Starting a round of Baz IO | 20260505 08:41:56.604 | INFO | Populating RTMetrics report | 20260505 08:41:56.604 | INFO | RTMetrics report populated | 20260505 08:41:56.604 | INFO | Opening tmp RTMetrics json file | 20260505 08:41:56.647 | INFO | Serializing RTMetrics to json | 20260505 08:41:56.694 | INFO | Serialization done, swapping tmp file now... | 20260505 08:41:56.720 | INFO | RTMetrics done | 20260505 08:41:56.736 | INFO | Finished a round of Baz IO: Wrote 2.449 GiB in 3.42 seconds | 20260505 08:41:56.736 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 27.21% / 0.02% / 72.77% | 20260505 08:41:56.736 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.46103% / 2.78155e-05% / 9.13938e-05% | 20260505 08:41:56.736 | INFO | Baz IO overall: | 20260505 08:41:56.736 | INFO | BytesWritten : 1054067244612 | 20260505 08:41:56.737 | INFO | NumEvents : 501208630118 | 20260505 08:41:58.314 | INFO | Analyzing chunk frames = [1855488,1856000) | 20260505 08:41:58.314 | INFO | MemInfo: { | 20260505 08:41:58.314 | INFO | "gpu" : " 17 GB", | 20260505 08:41:58.314 | INFO | "huge" : "", | 20260505 08:41:58.314 | INFO | "huge_shared" : " 27 GB", | 20260505 08:41:58.314 | INFO | "malloc" : " 14 GB", | 20260505 08:41:58.314 | INFO | "pinned" : " 29 GB" | 20260505 08:41:58.314 | INFO | } | 20260505 08:41:58.314 | INFO | Perf report for the last 5.13801 seconds: | 20260505 08:41:58.314 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:41:58.314 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:58.314 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:58.314 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:58.314 | INFO | ANALYSIS : 65.6%, 2.9, 100.5ms, 0 | 20260505 08:41:58.314 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:41:58.314 | INFO | RT_METRICS : 47.9%, 1.0, 25.1ms, 0 | 20260505 08:41:58.314 | INFO | PRE_HQ : 63.3%, 1.9, 60.9ms, 0 | 20260505 08:41:58.314 | INFO | BAZWRITER : 65.9%, 4.1, 140.4ms, 0 | 20260505 08:41:58.314 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 08:41:58.314 | INFO | Uploaded 36GiB to GPU in 2.24414s (16.0418GiB/s) | 20260505 08:41:58.314 | INFO | Downloaded 38GiB to GPU in 2.35268s (16.1518GiB/s) | 20260505 08:42:03.453 | INFO | Analyzing chunk frames = [1856000,1856512) | 20260505 08:42:03.453 | INFO | MemInfo: { | 20260505 08:42:03.453 | INFO | "gpu" : " 17 GB", | 20260505 08:42:03.453 | INFO | "huge" : "", | 20260505 08:42:03.453 | INFO | "huge_shared" : " 27 GB", | 20260505 08:42:03.454 | INFO | "malloc" : " 14 GB", | 20260505 08:42:03.454 | INFO | "pinned" : " 29 GB" | 20260505 08:42:03.454 | INFO | } | 20260505 08:42:03.454 | INFO | Perf report for the last 5.13942 seconds: | 20260505 08:42:03.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:42:03.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:03.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:03.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:03.454 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 08:42:03.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:03.454 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:03.454 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:42:03.454 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:03.454 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:03.454 | INFO | Uploaded 36GiB to GPU in 2.19807s (16.378GiB/s) | 20260505 08:42:03.454 | INFO | Downloaded 34GiB to GPU in 2.09179s (16.2541GiB/s) | 20260505 08:42:08.593 | INFO | Analyzing chunk frames = [1856512,1857024) | 20260505 08:42:08.593 | INFO | MemInfo: { | 20260505 08:42:08.593 | INFO | "gpu" : " 17 GB", | 20260505 08:42:08.593 | INFO | "huge" : "", | 20260505 08:42:08.593 | INFO | "huge_shared" : " 27 GB", | 20260505 08:42:08.593 | INFO | "malloc" : " 14 GB", | 20260505 08:42:08.593 | INFO | "pinned" : " 29 GB" | 20260505 08:42:08.593 | INFO | } | 20260505 08:42:08.593 | INFO | Perf report for the last 5.1393 seconds: | 20260505 08:42:08.593 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:42:08.593 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:08.593 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:08.593 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:08.593 | INFO | ANALYSIS : 63.6%, 3.0, 99.0ms, 0 | 20260505 08:42:08.593 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:08.593 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:08.593 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:42:08.593 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:08.593 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:08.593 | INFO | Uploaded 36GiB to GPU in 2.16895s (16.5979GiB/s) | 20260505 08:42:08.593 | INFO | Downloaded 34GiB to GPU in 2.06737s (16.446GiB/s) | 20260505 08:42:13.732 | INFO | Analyzing chunk frames = [1857024,1857536) | 20260505 08:42:13.732 | INFO | MemInfo: { | 20260505 08:42:13.732 | INFO | "gpu" : " 17 GB", | 20260505 08:42:13.732 | INFO | "huge" : "", | 20260505 08:42:13.732 | INFO | "huge_shared" : " 27 GB", | 20260505 08:42:13.732 | INFO | "malloc" : " 14 GB", | 20260505 08:42:13.732 | INFO | "pinned" : " 29 GB" | 20260505 08:42:13.732 | INFO | } | 20260505 08:42:13.732 | INFO | Perf report for the last 5.13942 seconds: | 20260505 08:42:13.732 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:42:13.732 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:13.732 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:13.732 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:13.732 | INFO | ANALYSIS : 63.4%, 3.0, 98.5ms, 0 | 20260505 08:42:13.732 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:13.732 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:13.732 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:42:13.732 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:13.732 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:13.733 | INFO | Uploaded 36GiB to GPU in 2.16217s (16.6499GiB/s) | 20260505 08:42:13.733 | INFO | Downloaded 34GiB to GPU in 2.06677s (16.4508GiB/s) | 20260505 08:42:18.872 | INFO | Analyzing chunk frames = [1857536,1858048) | 20260505 08:42:18.872 | INFO | MemInfo: { | 20260505 08:42:18.872 | INFO | "gpu" : " 17 GB", | 20260505 08:42:18.872 | INFO | "huge" : "", | 20260505 08:42:18.872 | INFO | "huge_shared" : " 27 GB", | 20260505 08:42:18.872 | INFO | "malloc" : " 14 GB", | 20260505 08:42:18.872 | INFO | "pinned" : " 29 GB" | 20260505 08:42:18.872 | INFO | } | 20260505 08:42:18.872 | INFO | Perf report for the last 5.13997 seconds: | 20260505 08:42:18.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:42:18.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:18.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:18.872 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:18.872 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 08:42:18.872 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:18.872 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:18.872 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:42:18.872 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:18.872 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:18.873 | INFO | Uploaded 36GiB to GPU in 2.15826s (16.6801GiB/s) | 20260505 08:42:18.873 | INFO | Downloaded 34GiB to GPU in 2.056s (16.5369GiB/s) | 20260505 08:42:24.012 | INFO | Analyzing chunk frames = [1858048,1858560) | 20260505 08:42:24.012 | INFO | MemInfo: { | 20260505 08:42:24.012 | INFO | "gpu" : " 17 GB", | 20260505 08:42:24.012 | INFO | "huge" : "", | 20260505 08:42:24.012 | INFO | "huge_shared" : " 27 GB", | 20260505 08:42:24.012 | INFO | "malloc" : " 14 GB", | 20260505 08:42:24.012 | INFO | "pinned" : " 29 GB" | 20260505 08:42:24.012 | INFO | } | 20260505 08:42:24.012 | INFO | Perf report for the last 5.13975 seconds: | 20260505 08:42:24.012 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:42:24.012 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:24.012 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:24.012 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:24.012 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 08:42:24.012 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:24.012 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:24.012 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:42:24.012 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:24.012 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:24.012 | INFO | Uploaded 36GiB to GPU in 2.16334s (16.6409GiB/s) | 20260505 08:42:24.012 | INFO | Downloaded 34GiB to GPU in 2.06067s (16.4995GiB/s) | 20260505 08:42:29.151 | INFO | Analyzing chunk frames = [1858560,1859072) | 20260505 08:42:29.151 | INFO | MemInfo: { | 20260505 08:42:29.151 | INFO | "gpu" : " 17 GB", | 20260505 08:42:29.151 | INFO | "huge" : "", | 20260505 08:42:29.151 | INFO | "huge_shared" : " 27 GB", | 20260505 08:42:29.151 | INFO | "malloc" : " 14 GB", | 20260505 08:42:29.151 | INFO | "pinned" : " 29 GB" | 20260505 08:42:29.151 | INFO | } | 20260505 08:42:29.151 | INFO | Perf report for the last 5.13924 seconds: | 20260505 08:42:29.151 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:42:29.151 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:29.151 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:29.151 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:29.151 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 08:42:29.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:29.151 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:29.151 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260505 08:42:29.151 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:29.151 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:29.152 | INFO | Uploaded 36GiB to GPU in 2.20871s (16.2991GiB/s) | 20260505 08:42:29.152 | INFO | Downloaded 34GiB to GPU in 2.15526s (15.7754GiB/s) | 20260505 08:42:34.289 | INFO | Analyzing chunk frames = [1859072,1859584) | 20260505 08:42:34.289 | INFO | MemInfo: { | 20260505 08:42:34.289 | INFO | "gpu" : " 17 GB", | 20260505 08:42:34.289 | INFO | "huge" : "", | 20260505 08:42:34.289 | INFO | "huge_shared" : " 27 GB", | 20260505 08:42:34.289 | INFO | "malloc" : " 14 GB", | 20260505 08:42:34.289 | INFO | "pinned" : " 29 GB" | 20260505 08:42:34.289 | INFO | } | 20260505 08:42:34.290 | INFO | Perf report for the last 5.13817 seconds: | 20260505 08:42:34.290 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:42:34.290 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:34.290 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:34.290 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:34.290 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 08:42:34.290 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:34.290 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:34.290 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260505 08:42:34.290 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:34.290 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:34.290 | INFO | Uploaded 36GiB to GPU in 2.19488s (16.4018GiB/s) | 20260505 08:42:34.290 | INFO | Downloaded 34GiB to GPU in 2.04661s (16.6129GiB/s) | 20260505 08:42:34.290 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 08:42:34.290 | INFO | Total mem usage: 15.3 GiB | 20260505 08:42:34.290 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 08:42:34.290 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -5.56%) | 20260505 08:42:34.290 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 08:42:34.290 | INFO | High Water Breakdown: | 20260505 08:42:34.290 | INFO | 61% : 8.86 GiB ( 94.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 08:42:34.290 | INFO | 38.9% : 5.64 GiB ( 84.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 08:42:34.290 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 08:42:34.290 | INFO | ------------------------------------------ | 20260505 08:42:34.290 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:42:34.290 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 08:42:34.290 | INFO | Total mem usage: 26.5 GiB | 20260505 08:42:34.290 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 08:42:34.290 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 08:42:34.290 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 08:42:34.290 | INFO | High Water Breakdown: | 20260505 08:42:34.290 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:42:34.290 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:42:34.290 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:42:34.290 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:42:34.290 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:42:34.290 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:42:34.290 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 08:42:34.290 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 08:42:34.290 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:42:34.290 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 08:42:34.290 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 08:42:34.290 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 08:42:34.290 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 08:42:34.290 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 08:42:34.290 | INFO | ------------------------------------------ | 20260505 08:42:34.290 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:42:34.290 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 08:42:34.290 | INFO | Total mem usage: 25.6 GiB | 20260505 08:42:34.290 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 08:42:34.290 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 08:42:34.290 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 08:42:34.290 | INFO | High Water Breakdown: | 20260505 08:42:34.290 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 08:42:34.290 | INFO | ------------------------------------------ | 20260505 08:42:34.290 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:42:34.290 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 08:42:34.290 | INFO | Total mem usage: 15.7 GiB | 20260505 08:42:34.290 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 08:42:34.290 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 08:42:34.290 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 08:42:34.290 | INFO | High Water Breakdown: | 20260505 08:42:34.290 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 08:42:34.290 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 08:42:34.290 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 08:42:34.290 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 08:42:34.290 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 08:42:34.290 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 08:42:34.290 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 08:42:34.290 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 08:42:34.290 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 08:42:34.290 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:42:34.290 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 08:42:34.290 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:42:34.290 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 08:42:34.290 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 08:42:34.290 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 08:42:34.291 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 08:42:34.291 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:42:34.291 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 08:42:34.291 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:42:34.291 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:42:34.291 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:42:34.291 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 08:42:34.291 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 08:42:34.291 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:42:34.291 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 08:42:34.291 | INFO | ------------------------------------------ | 20260505 08:42:34.291 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:42:34.291 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 08:42:34.291 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:42:34.291 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:42:34.291 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.55e+01, 1.34e+01 / 1.96e+00 | 20260505 08:42:34.291 | INFO | PerfReport: 0.29%, AggregateStats -- 9.34e-02 / 1.04e-01, 1.13e-02 / 4.46e-03 | 20260505 08:42:34.291 | INFO | PerfReport: 11.65%, Baseline -- 3.80e+00 / 4.45e+00, 6.18e-01 / 2.85e-01 | 20260505 08:42:34.291 | INFO | PerfReport: 3.31%, Histogram -- 1.08e+00 / 1.47e+00, 2.09e-01 / 1.13e-01 | 20260505 08:42:34.291 | INFO | PerfReport: 15.54%, DME -- 4.05e+01 / 4.44e+01, 4.18e+00 / 1.72e+00 | 20260505 08:42:34.291 | INFO | PerfReport: 56.47%, FrameLabeling -- 1.84e+01 / 1.99e+01, 1.91e+00 / 1.74e+00 | 20260505 08:42:34.291 | INFO | PerfReport: 7.47%, PulseAccumulating -- 2.43e+00 / 3.02e+00, 4.10e-01 / 2.59e-01 | 20260505 08:42:34.291 | INFO | PerfReport: 3.88%, Metrics -- 1.26e+00 / 1.59e+00, 3.56e-01 / 1.66e-01 | 20260505 08:42:34.291 | INFO | PerfReport: 1.39%, PulseSerializing -- 4.51e-01 / 5.08e-01, 8.49e-02 / 2.48e-02 | 20260505 08:42:34.291 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 08:42:34.291 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:42:34.291 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:42:34.291 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.30e+01 / 0.00e+00 | 20260505 08:42:34.291 | INFO | PerfReport: 22.49%, Upload -- 2.29e+01 / 0.00e+00, 8.17e+00 / 0.00e+00 | 20260505 08:42:34.291 | INFO | PerfReport: 21.65%, Download -- 2.20e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260505 08:42:34.291 | INFO | PerfReport: 32.07%, Compute -- 3.26e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 08:42:34.291 | INFO | PerfReport: 1.67%, LockUpload -- 1.69e+00 / 0.00e+00, 7.38e+00 / 0.00e+00 | 20260505 08:42:34.291 | INFO | PerfReport: 1.09%, LockDownload -- 1.11e+00 / 0.00e+00, 2.64e+00 / 0.00e+00 | 20260505 08:42:34.291 | INFO | PerfReport: 21.02%, LockCompute -- 2.14e+01 / 0.00e+00, 1.77e+01 / 0.00e+00 | 20260505 08:42:34.291 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.87e-02 / 0.00e+00 | 20260505 08:42:34.431 | INFO | Starting a round of Baz IO | 20260505 08:42:37.704 | INFO | Populating RTMetrics report | 20260505 08:42:37.705 | INFO | RTMetrics report populated | 20260505 08:42:37.705 | INFO | Opening tmp RTMetrics json file | 20260505 08:42:37.708 | INFO | Serializing RTMetrics to json | 20260505 08:42:37.710 | INFO | Serialization done, swapping tmp file now... | 20260505 08:42:37.713 | INFO | RTMetrics done | 20260505 08:42:37.836 | INFO | Finished a round of Baz IO: Wrote 2.45 GiB in 3.405 seconds | 20260505 08:42:37.837 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 27.19% / 0.02% / 72.79% | 20260505 08:42:37.837 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.4496% / 1.58946e-05% / 9.93411e-05% | 20260505 08:42:37.837 | INFO | Baz IO overall: | 20260505 08:42:37.837 | INFO | BytesWritten : 1056697474967 | 20260505 08:42:37.837 | INFO | NumEvents : 502465454946 | 20260505 08:42:39.429 | INFO | Analyzing chunk frames = [1859584,1860096) | 20260505 08:42:39.429 | INFO | MemInfo: { | 20260505 08:42:39.429 | INFO | "gpu" : " 17 GB", | 20260505 08:42:39.429 | INFO | "huge" : "", | 20260505 08:42:39.429 | INFO | "huge_shared" : " 27 GB", | 20260505 08:42:39.429 | INFO | "malloc" : " 14 GB", | 20260505 08:42:39.429 | INFO | "pinned" : " 29 GB" | 20260505 08:42:39.429 | INFO | } | 20260505 08:42:39.429 | INFO | Perf report for the last 5.13972 seconds: | 20260505 08:42:39.429 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:42:39.429 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:39.429 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:39.429 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:39.429 | INFO | ANALYSIS : 65.2%, 2.9, 99.6ms, 0 | 20260505 08:42:39.429 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:39.429 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 08:42:39.429 | INFO | PRE_HQ : 63.4%, 1.9, 61.4ms, 0 | 20260505 08:42:39.429 | INFO | BAZWRITER : 65.6%, 4.2, 140.7ms, 0 | 20260505 08:42:39.429 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 08:42:39.429 | INFO | Uploaded 36GiB to GPU in 2.26492s (15.8946GiB/s) | 20260505 08:42:39.429 | INFO | Downloaded 38GiB to GPU in 2.38868s (15.9084GiB/s) | 20260505 08:42:44.570 | INFO | Analyzing chunk frames = [1860096,1860608) | 20260505 08:42:44.570 | INFO | MemInfo: { | 20260505 08:42:44.570 | INFO | "gpu" : " 17 GB", | 20260505 08:42:44.570 | INFO | "huge" : "", | 20260505 08:42:44.570 | INFO | "huge_shared" : " 27 GB", | 20260505 08:42:44.570 | INFO | "malloc" : " 14 GB", | 20260505 08:42:44.570 | INFO | "pinned" : " 29 GB" | 20260505 08:42:44.570 | INFO | } | 20260505 08:42:44.570 | INFO | Perf report for the last 5.14115 seconds: | 20260505 08:42:44.570 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:42:44.570 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:44.570 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:44.570 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:44.571 | INFO | ANALYSIS : 64.4%, 3.0, 100.6ms, 0 | 20260505 08:42:44.571 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:44.571 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:44.571 | INFO | PRE_HQ : 29.9%, 1.0, 15.8ms, 0 | 20260505 08:42:44.571 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:44.571 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:44.571 | INFO | Uploaded 36GiB to GPU in 2.22585s (16.1736GiB/s) | 20260505 08:42:44.571 | INFO | Downloaded 34GiB to GPU in 2.09539s (16.2261GiB/s) | 20260505 08:42:49.708 | INFO | Analyzing chunk frames = [1860608,1861120) | 20260505 08:42:49.708 | INFO | MemInfo: { | 20260505 08:42:49.708 | INFO | "gpu" : " 17 GB", | 20260505 08:42:49.708 | INFO | "huge" : "", | 20260505 08:42:49.708 | INFO | "huge_shared" : " 27 GB", | 20260505 08:42:49.708 | INFO | "malloc" : " 14 GB", | 20260505 08:42:49.708 | INFO | "pinned" : " 29 GB" | 20260505 08:42:49.708 | INFO | } | 20260505 08:42:49.708 | INFO | Perf report for the last 5.13793 seconds: | 20260505 08:42:49.708 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:42:49.708 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:49.708 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:49.708 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:49.708 | INFO | ANALYSIS : 63.6%, 3.0, 98.9ms, 0 | 20260505 08:42:49.708 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:49.708 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:49.709 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:42:49.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:49.709 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:49.709 | INFO | Uploaded 36GiB to GPU in 2.16143s (16.6556GiB/s) | 20260505 08:42:49.709 | INFO | Downloaded 34GiB to GPU in 2.06468s (16.4674GiB/s) | 20260505 08:42:54.846 | INFO | Analyzing chunk frames = [1861120,1861632) | 20260505 08:42:54.846 | INFO | MemInfo: { | 20260505 08:42:54.846 | INFO | "gpu" : " 17 GB", | 20260505 08:42:54.846 | INFO | "huge" : "", | 20260505 08:42:54.847 | INFO | "huge_shared" : " 27 GB", | 20260505 08:42:54.847 | INFO | "malloc" : " 14 GB", | 20260505 08:42:54.847 | INFO | "pinned" : " 29 GB" | 20260505 08:42:54.847 | INFO | } | 20260505 08:42:54.847 | INFO | Perf report for the last 5.13873 seconds: | 20260505 08:42:54.847 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:42:54.847 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:54.847 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:54.847 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:54.847 | INFO | ANALYSIS : 63.4%, 3.0, 98.4ms, 0 | 20260505 08:42:54.847 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:54.847 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:54.847 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 08:42:54.847 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:54.847 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:54.847 | INFO | Uploaded 36GiB to GPU in 2.16544s (16.6248GiB/s) | 20260505 08:42:54.847 | INFO | Downloaded 34GiB to GPU in 2.06756s (16.4445GiB/s) | 20260505 08:42:59.987 | INFO | Analyzing chunk frames = [1861632,1862144) | 20260505 08:42:59.987 | INFO | MemInfo: { | 20260505 08:42:59.987 | INFO | "gpu" : " 17 GB", | 20260505 08:42:59.987 | INFO | "huge" : "", | 20260505 08:42:59.987 | INFO | "huge_shared" : " 27 GB", | 20260505 08:42:59.987 | INFO | "malloc" : " 14 GB", | 20260505 08:42:59.987 | INFO | "pinned" : " 29 GB" | 20260505 08:42:59.987 | INFO | } | 20260505 08:42:59.987 | INFO | Perf report for the last 5.13996 seconds: | 20260505 08:42:59.987 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:42:59.987 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:59.987 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:59.987 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:59.987 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 08:42:59.987 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:59.987 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:59.987 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 08:42:59.987 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:59.987 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:42:59.987 | INFO | Uploaded 36GiB to GPU in 2.18084s (16.5074GiB/s) | 20260505 08:42:59.987 | INFO | Downloaded 34GiB to GPU in 2.06014s (16.5037GiB/s) | 20260505 08:43:05.127 | INFO | Analyzing chunk frames = [1862144,1862656) | 20260505 08:43:05.127 | INFO | MemInfo: { | 20260505 08:43:05.127 | INFO | "gpu" : " 17 GB", | 20260505 08:43:05.127 | INFO | "huge" : "", | 20260505 08:43:05.127 | INFO | "huge_shared" : " 27 GB", | 20260505 08:43:05.127 | INFO | "malloc" : " 14 GB", | 20260505 08:43:05.127 | INFO | "pinned" : " 29 GB" | 20260505 08:43:05.127 | INFO | } | 20260505 08:43:05.127 | INFO | Perf report for the last 5.14013 seconds: | 20260505 08:43:05.127 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:43:05.127 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:05.127 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:05.127 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:05.127 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 08:43:05.127 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:05.127 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:05.127 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:43:05.127 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:05.127 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:05.127 | INFO | Uploaded 36GiB to GPU in 2.16225s (16.6494GiB/s) | 20260505 08:43:05.127 | INFO | Downloaded 34GiB to GPU in 2.06271s (16.4832GiB/s) | 20260505 08:43:10.265 | INFO | Analyzing chunk frames = [1862656,1863168) | 20260505 08:43:10.265 | INFO | MemInfo: { | 20260505 08:43:10.265 | INFO | "gpu" : " 17 GB", | 20260505 08:43:10.265 | INFO | "huge" : "", | 20260505 08:43:10.265 | INFO | "huge_shared" : " 27 GB", | 20260505 08:43:10.265 | INFO | "malloc" : " 14 GB", | 20260505 08:43:10.265 | INFO | "pinned" : " 29 GB" | 20260505 08:43:10.265 | INFO | } | 20260505 08:43:10.265 | INFO | Perf report for the last 5.1379 seconds: | 20260505 08:43:10.265 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:43:10.265 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:10.265 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:10.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:10.265 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 08:43:10.265 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:10.265 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:10.265 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260505 08:43:10.265 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:10.265 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:10.265 | INFO | Uploaded 36GiB to GPU in 2.19455s (16.4043GiB/s) | 20260505 08:43:10.265 | INFO | Downloaded 34GiB to GPU in 2.15357s (15.7877GiB/s) | 20260505 08:43:15.404 | INFO | Analyzing chunk frames = [1863168,1863680) | 20260505 08:43:15.404 | INFO | MemInfo: { | 20260505 08:43:15.404 | INFO | "gpu" : " 17 GB", | 20260505 08:43:15.404 | INFO | "huge" : "", | 20260505 08:43:15.404 | INFO | "huge_shared" : " 27 GB", | 20260505 08:43:15.404 | INFO | "malloc" : " 14 GB", | 20260505 08:43:15.404 | INFO | "pinned" : " 29 GB" | 20260505 08:43:15.404 | INFO | } | 20260505 08:43:15.404 | INFO | Perf report for the last 5.13903 seconds: | 20260505 08:43:15.404 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:43:15.404 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:15.404 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:15.404 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:15.404 | INFO | ANALYSIS : 64.1%, 3.0, 100.0ms, 0 | 20260505 08:43:15.404 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:15.404 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:15.404 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 08:43:15.404 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:15.404 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:15.404 | INFO | Uploaded 36GiB to GPU in 2.19646s (16.39GiB/s) | 20260505 08:43:15.404 | INFO | Downloaded 34GiB to GPU in 2.03748s (16.6873GiB/s) | 20260505 08:43:15.559 | INFO | Starting a round of Baz IO | 20260505 08:43:18.822 | INFO | Populating RTMetrics report | 20260505 08:43:18.823 | INFO | RTMetrics report populated | 20260505 08:43:18.823 | INFO | Opening tmp RTMetrics json file | 20260505 08:43:18.825 | INFO | Serializing RTMetrics to json | 20260505 08:43:18.829 | INFO | Serialization done, swapping tmp file now... | 20260505 08:43:18.831 | INFO | RTMetrics done | 20260505 08:43:18.969 | INFO | Finished a round of Baz IO: Wrote 2.452 GiB in 3.41 seconds | 20260505 08:43:18.969 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 27.17% / 0.02% / 72.81% | 20260505 08:43:18.969 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.4412% / 2.38419e-05% / 9.93411e-05% | 20260505 08:43:18.969 | INFO | Baz IO overall: | 20260505 08:43:18.969 | INFO | BytesWritten : 1059330150481 | 20260505 08:43:18.969 | INFO | NumEvents : 503723408531 | 20260505 08:43:20.544 | INFO | Analyzing chunk frames = [1863680,1864192) | 20260505 08:43:20.544 | INFO | MemInfo: { | 20260505 08:43:20.544 | INFO | "gpu" : " 17 GB", | 20260505 08:43:20.544 | INFO | "huge" : "", | 20260505 08:43:20.544 | INFO | "huge_shared" : " 27 GB", | 20260505 08:43:20.544 | INFO | "malloc" : " 14 GB", | 20260505 08:43:20.544 | INFO | "pinned" : " 29 GB" | 20260505 08:43:20.544 | INFO | } | 20260505 08:43:20.544 | INFO | Perf report for the last 5.13967 seconds: | 20260505 08:43:20.544 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:43:20.544 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:20.544 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:20.544 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:20.544 | INFO | ANALYSIS : 65.4%, 2.9, 99.9ms, 0 | 20260505 08:43:20.544 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:20.544 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260505 08:43:20.544 | INFO | PRE_HQ : 63.2%, 1.9, 61.3ms, 0 | 20260505 08:43:20.544 | INFO | BAZWRITER : 66.0%, 4.2, 141.2ms, 0 | 20260505 08:43:20.544 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 08:43:20.544 | INFO | Uploaded 36GiB to GPU in 2.22152s (16.2051GiB/s) | 20260505 08:43:20.544 | INFO | Downloaded 38GiB to GPU in 2.34256s (16.2215GiB/s) | 20260505 08:43:25.683 | INFO | Analyzing chunk frames = [1864192,1864704) | 20260505 08:43:25.683 | INFO | MemInfo: { | 20260505 08:43:25.683 | INFO | "gpu" : " 17 GB", | 20260505 08:43:25.683 | INFO | "huge" : "", | 20260505 08:43:25.683 | INFO | "huge_shared" : " 27 GB", | 20260505 08:43:25.683 | INFO | "malloc" : " 14 GB", | 20260505 08:43:25.683 | INFO | "pinned" : " 29 GB" | 20260505 08:43:25.683 | INFO | } | 20260505 08:43:25.683 | INFO | Perf report for the last 5.13893 seconds: | 20260505 08:43:25.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:43:25.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:25.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:25.683 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:25.683 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 08:43:25.683 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:25.683 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:25.683 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 08:43:25.683 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:25.683 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:25.683 | INFO | Uploaded 36GiB to GPU in 2.16243s (16.6479GiB/s) | 20260505 08:43:25.683 | INFO | Downloaded 34GiB to GPU in 2.08438s (16.3118GiB/s) | 20260505 08:43:30.822 | INFO | Analyzing chunk frames = [1864704,1865216) | 20260505 08:43:30.822 | INFO | MemInfo: { | 20260505 08:43:30.822 | INFO | "gpu" : " 17 GB", | 20260505 08:43:30.822 | INFO | "huge" : "", | 20260505 08:43:30.822 | INFO | "huge_shared" : " 27 GB", | 20260505 08:43:30.822 | INFO | "malloc" : " 14 GB", | 20260505 08:43:30.822 | INFO | "pinned" : " 29 GB" | 20260505 08:43:30.822 | INFO | } | 20260505 08:43:30.822 | INFO | Perf report for the last 5.13952 seconds: | 20260505 08:43:30.822 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:43:30.822 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:30.822 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:30.822 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:30.822 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 08:43:30.822 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:30.822 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:30.822 | INFO | PRE_HQ : 29.8%, 1.0, 15.4ms, 0 | 20260505 08:43:30.822 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:30.822 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:30.823 | INFO | Uploaded 36GiB to GPU in 2.16427s (16.6338GiB/s) | 20260505 08:43:30.823 | INFO | Downloaded 34GiB to GPU in 2.06503s (16.4647GiB/s) | 20260505 08:43:35.962 | INFO | Analyzing chunk frames = [1865216,1865728) | 20260505 08:43:35.962 | INFO | MemInfo: { | 20260505 08:43:35.962 | INFO | "gpu" : " 17 GB", | 20260505 08:43:35.962 | INFO | "huge" : "", | 20260505 08:43:35.962 | INFO | "huge_shared" : " 27 GB", | 20260505 08:43:35.962 | INFO | "malloc" : " 14 GB", | 20260505 08:43:35.962 | INFO | "pinned" : " 29 GB" | 20260505 08:43:35.962 | INFO | } | 20260505 08:43:35.962 | INFO | Perf report for the last 5.13973 seconds: | 20260505 08:43:35.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:43:35.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:35.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:35.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:35.962 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260505 08:43:35.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:35.962 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:35.962 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 08:43:35.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:35.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:35.962 | INFO | Uploaded 36GiB to GPU in 2.1733s (16.5647GiB/s) | 20260505 08:43:35.962 | INFO | Downloaded 34GiB to GPU in 2.06829s (16.4387GiB/s) | 20260505 08:43:41.101 | INFO | Analyzing chunk frames = [1865728,1866240) | 20260505 08:43:41.101 | INFO | MemInfo: { | 20260505 08:43:41.101 | INFO | "gpu" : " 17 GB", | 20260505 08:43:41.101 | INFO | "huge" : "", | 20260505 08:43:41.101 | INFO | "huge_shared" : " 27 GB", | 20260505 08:43:41.101 | INFO | "malloc" : " 14 GB", | 20260505 08:43:41.101 | INFO | "pinned" : " 29 GB" | 20260505 08:43:41.101 | INFO | } | 20260505 08:43:41.101 | INFO | Perf report for the last 5.13934 seconds: | 20260505 08:43:41.101 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:43:41.101 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:41.101 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:41.101 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:41.101 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 08:43:41.101 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:41.101 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:41.101 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 08:43:41.102 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:41.102 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:41.102 | INFO | Uploaded 36GiB to GPU in 2.16382s (16.6373GiB/s) | 20260505 08:43:41.102 | INFO | Downloaded 34GiB to GPU in 2.06011s (16.504GiB/s) | 20260505 08:43:46.241 | INFO | Analyzing chunk frames = [1866240,1866752) | 20260505 08:43:46.241 | INFO | MemInfo: { | 20260505 08:43:46.241 | INFO | "gpu" : " 17 GB", | 20260505 08:43:46.241 | INFO | "huge" : "", | 20260505 08:43:46.241 | INFO | "huge_shared" : " 27 GB", | 20260505 08:43:46.241 | INFO | "malloc" : " 14 GB", | 20260505 08:43:46.241 | INFO | "pinned" : " 29 GB" | 20260505 08:43:46.241 | INFO | } | 20260505 08:43:46.241 | INFO | Perf report for the last 5.13976 seconds: | 20260505 08:43:46.241 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:43:46.241 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:46.241 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:46.241 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:46.241 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 08:43:46.241 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:46.241 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:46.241 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:43:46.241 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:46.241 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:46.241 | INFO | Uploaded 36GiB to GPU in 2.1803s (16.5115GiB/s) | 20260505 08:43:46.241 | INFO | Downloaded 34GiB to GPU in 2.05706s (16.5285GiB/s) | 20260505 08:43:51.380 | INFO | Analyzing chunk frames = [1866752,1867264) | 20260505 08:43:51.380 | INFO | MemInfo: { | 20260505 08:43:51.380 | INFO | "gpu" : " 17 GB", | 20260505 08:43:51.380 | INFO | "huge" : "", | 20260505 08:43:51.380 | INFO | "huge_shared" : " 27 GB", | 20260505 08:43:51.380 | INFO | "malloc" : " 14 GB", | 20260505 08:43:51.380 | INFO | "pinned" : " 29 GB" | 20260505 08:43:51.380 | INFO | } | 20260505 08:43:51.380 | INFO | Perf report for the last 5.13866 seconds: | 20260505 08:43:51.380 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:43:51.380 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:51.380 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:51.380 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:51.380 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 08:43:51.380 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:51.380 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:51.380 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 08:43:51.380 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:51.380 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:51.380 | INFO | Uploaded 36GiB to GPU in 2.21111s (16.2814GiB/s) | 20260505 08:43:51.380 | INFO | Downloaded 34GiB to GPU in 2.15559s (15.773GiB/s) | 20260505 08:43:56.520 | INFO | Analyzing chunk frames = [1867264,1867776) | 20260505 08:43:56.520 | INFO | MemInfo: { | 20260505 08:43:56.520 | INFO | "gpu" : " 17 GB", | 20260505 08:43:56.520 | INFO | "huge" : "", | 20260505 08:43:56.520 | INFO | "huge_shared" : " 27 GB", | 20260505 08:43:56.520 | INFO | "malloc" : " 14 GB", | 20260505 08:43:56.520 | INFO | "pinned" : " 29 GB" | 20260505 08:43:56.520 | INFO | } | 20260505 08:43:56.520 | INFO | Perf report for the last 5.14027 seconds: | 20260505 08:43:56.520 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:43:56.520 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:56.520 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:56.520 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:56.520 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 08:43:56.520 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:56.520 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:56.520 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 08:43:56.520 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:56.520 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:43:56.520 | INFO | Uploaded 36GiB to GPU in 2.17198s (16.5747GiB/s) | 20260505 08:43:56.520 | INFO | Downloaded 34GiB to GPU in 2.04048s (16.6627GiB/s) | 20260505 08:43:56.520 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 08:43:56.520 | INFO | Total mem usage: 15.3 GiB | 20260505 08:43:56.520 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 08:43:56.520 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -5.6%) | 20260505 08:43:56.520 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 08:43:56.520 | INFO | High Water Breakdown: | 20260505 08:43:56.520 | INFO | 61% : 8.85 GiB ( 94.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 08:43:56.520 | INFO | 38.9% : 5.64 GiB ( 84.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 08:43:56.520 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 08:43:56.520 | INFO | ------------------------------------------ | 20260505 08:43:56.520 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:43:56.520 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 08:43:56.521 | INFO | Total mem usage: 26.5 GiB | 20260505 08:43:56.521 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 08:43:56.521 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 08:43:56.521 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 08:43:56.521 | INFO | High Water Breakdown: | 20260505 08:43:56.521 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:43:56.521 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:43:56.521 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:43:56.521 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:43:56.521 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:43:56.521 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:43:56.521 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 08:43:56.521 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 08:43:56.521 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:43:56.521 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 08:43:56.521 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 08:43:56.521 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 08:43:56.521 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 08:43:56.521 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 08:43:56.521 | INFO | ------------------------------------------ | 20260505 08:43:56.521 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:43:56.521 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 08:43:56.521 | INFO | Total mem usage: 25.6 GiB | 20260505 08:43:56.521 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 08:43:56.521 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 08:43:56.521 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 08:43:56.521 | INFO | High Water Breakdown: | 20260505 08:43:56.521 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 08:43:56.521 | INFO | ------------------------------------------ | 20260505 08:43:56.521 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:43:56.521 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 08:43:56.521 | INFO | Total mem usage: 15.7 GiB | 20260505 08:43:56.521 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 08:43:56.521 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 08:43:56.521 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 08:43:56.521 | INFO | High Water Breakdown: | 20260505 08:43:56.521 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 08:43:56.521 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 08:43:56.521 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 08:43:56.521 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 08:43:56.521 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 08:43:56.521 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 08:43:56.521 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 08:43:56.521 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 08:43:56.521 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 08:43:56.521 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:43:56.521 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 08:43:56.521 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:43:56.521 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 08:43:56.521 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 08:43:56.521 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 08:43:56.521 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 08:43:56.521 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:43:56.521 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 08:43:56.521 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:43:56.521 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:43:56.521 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:43:56.521 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 08:43:56.521 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 08:43:56.521 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:43:56.521 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 08:43:56.521 | INFO | ------------------------------------------ | 20260505 08:43:56.521 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:43:56.521 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 08:43:56.521 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:43:56.521 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:43:56.521 | INFO | PerfReport: Full Execution -- 3.25e+01 / 7.52e+01, 1.37e+01 / 1.49e+00 | 20260505 08:43:56.521 | INFO | PerfReport: 0.29%, AggregateStats -- 9.35e-02 / 1.04e-01, 1.18e-02 / 5.49e-03 | 20260505 08:43:56.521 | INFO | PerfReport: 11.68%, Baseline -- 3.80e+00 / 4.45e+00, 6.25e-01 / 2.83e-01 | 20260505 08:43:56.521 | INFO | PerfReport: 3.32%, Histogram -- 1.08e+00 / 1.46e+00, 2.09e-01 / 1.17e-01 | 20260505 08:43:56.521 | INFO | PerfReport: 15.52%, DME -- 4.04e+01 / 4.42e+01, 3.97e+00 / 1.39e+00 | 20260505 08:43:56.521 | INFO | PerfReport: 56.41%, FrameLabeling -- 1.84e+01 / 1.99e+01, 1.89e+00 / 1.68e+00 | 20260505 08:43:56.521 | INFO | PerfReport: 7.51%, PulseAccumulating -- 2.44e+00 / 3.02e+00, 4.22e-01 / 2.51e-01 | 20260505 08:43:56.521 | INFO | PerfReport: 3.89%, Metrics -- 1.27e+00 / 1.60e+00, 3.62e-01 / 1.63e-01 | 20260505 08:43:56.521 | INFO | PerfReport: 1.39%, PulseSerializing -- 4.52e-01 / 5.08e-01, 8.55e-02 / 2.17e-02 | 20260505 08:43:56.521 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 08:43:56.521 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:43:56.521 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:43:56.521 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.29e+01 / 0.00e+00 | 20260505 08:43:56.521 | INFO | PerfReport: 22.44%, Upload -- 2.28e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260505 08:43:56.521 | INFO | PerfReport: 21.68%, Download -- 2.20e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260505 08:43:56.521 | INFO | PerfReport: 32.06%, Compute -- 3.25e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 08:43:56.521 | INFO | PerfReport: 1.67%, LockUpload -- 1.70e+00 / 0.00e+00, 7.43e+00 / 0.00e+00 | 20260505 08:43:56.521 | INFO | PerfReport: 1.17%, LockDownload -- 1.18e+00 / 0.00e+00, 2.77e+00 / 0.00e+00 | 20260505 08:43:56.521 | INFO | PerfReport: 20.97%, LockCompute -- 2.13e+01 / 0.00e+00, 1.78e+01 / 0.00e+00 | 20260505 08:43:56.521 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 2.02e-02 / 0.00e+00 | 20260505 08:43:56.657 | INFO | Starting a round of Baz IO | 20260505 08:43:59.941 | INFO | Populating RTMetrics report | 20260505 08:43:59.941 | INFO | RTMetrics report populated | 20260505 08:43:59.941 | INFO | Opening tmp RTMetrics json file | 20260505 08:43:59.952 | INFO | Serializing RTMetrics to json | 20260505 08:43:59.956 | INFO | Serialization done, swapping tmp file now... | 20260505 08:43:59.956 | INFO | RTMetrics done | 20260505 08:44:00.165 | INFO | Finished a round of Baz IO: Wrote 2.448 GiB in 3.508 seconds | 20260505 08:44:00.165 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 27.15% / 0.02% / 72.83% | 20260505 08:44:00.165 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.43167% / 1.58946e-05% / 8.74201e-05% | 20260505 08:44:00.165 | INFO | Baz IO overall: | 20260505 08:44:00.165 | INFO | BytesWritten : 1061958694364 | 20260505 08:44:00.165 | INFO | NumEvents : 504979467040 | 20260505 08:44:01.659 | INFO | Analyzing chunk frames = [1867776,1868288) | 20260505 08:44:01.659 | INFO | MemInfo: { | 20260505 08:44:01.659 | INFO | "gpu" : " 17 GB", | 20260505 08:44:01.659 | INFO | "huge" : "", | 20260505 08:44:01.659 | INFO | "huge_shared" : " 27 GB", | 20260505 08:44:01.659 | INFO | "malloc" : " 14 GB", | 20260505 08:44:01.659 | INFO | "pinned" : " 29 GB" | 20260505 08:44:01.659 | INFO | } | 20260505 08:44:01.659 | INFO | Perf report for the last 5.13886 seconds: | 20260505 08:44:01.659 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:44:01.659 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:01.659 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:01.659 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:01.659 | INFO | ANALYSIS : 65.4%, 2.9, 100.0ms, 0 | 20260505 08:44:01.659 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:01.659 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260505 08:44:01.659 | INFO | PRE_HQ : 62.8%, 1.9, 60.9ms, 0 | 20260505 08:44:01.659 | INFO | BAZWRITER : 67.6%, 4.2, 144.2ms, 0 | 20260505 08:44:01.659 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 08:44:01.659 | INFO | Uploaded 36GiB to GPU in 2.20363s (16.3367GiB/s) | 20260505 08:44:01.659 | INFO | Downloaded 38GiB to GPU in 2.33539s (16.2713GiB/s) | 20260505 08:44:06.798 | INFO | Analyzing chunk frames = [1868288,1868800) | 20260505 08:44:06.798 | INFO | MemInfo: { | 20260505 08:44:06.798 | INFO | "gpu" : " 17 GB", | 20260505 08:44:06.798 | INFO | "huge" : "", | 20260505 08:44:06.798 | INFO | "huge_shared" : " 27 GB", | 20260505 08:44:06.798 | INFO | "malloc" : " 14 GB", | 20260505 08:44:06.798 | INFO | "pinned" : " 29 GB" | 20260505 08:44:06.798 | INFO | } | 20260505 08:44:06.798 | INFO | Perf report for the last 5.13861 seconds: | 20260505 08:44:06.798 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:44:06.798 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:06.798 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:06.798 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:06.798 | INFO | ANALYSIS : 64.6%, 3.0, 100.8ms, 0 | 20260505 08:44:06.798 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:06.798 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:06.798 | INFO | PRE_HQ : 29.8%, 1.0, 15.4ms, 0 | 20260505 08:44:06.798 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:06.798 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:06.798 | INFO | Uploaded 36GiB to GPU in 2.25128s (15.9909GiB/s) | 20260505 08:44:06.798 | INFO | Downloaded 34GiB to GPU in 2.10573s (16.1465GiB/s) | 20260505 08:44:11.937 | INFO | Analyzing chunk frames = [1868800,1869312) | 20260505 08:44:11.937 | INFO | MemInfo: { | 20260505 08:44:11.937 | INFO | "gpu" : " 17 GB", | 20260505 08:44:11.938 | INFO | "huge" : "", | 20260505 08:44:11.938 | INFO | "huge_shared" : " 27 GB", | 20260505 08:44:11.938 | INFO | "malloc" : " 14 GB", | 20260505 08:44:11.938 | INFO | "pinned" : " 29 GB" | 20260505 08:44:11.938 | INFO | } | 20260505 08:44:11.938 | INFO | Perf report for the last 5.13997 seconds: | 20260505 08:44:11.938 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:44:11.938 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:11.938 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:11.938 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:11.938 | INFO | ANALYSIS : 63.7%, 3.0, 99.1ms, 0 | 20260505 08:44:11.938 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:11.938 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:11.938 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:44:11.938 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:11.938 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:11.938 | INFO | Uploaded 36GiB to GPU in 2.18376s (16.4854GiB/s) | 20260505 08:44:11.938 | INFO | Downloaded 34GiB to GPU in 2.08082s (16.3397GiB/s) | 20260505 08:44:17.077 | INFO | Analyzing chunk frames = [1869312,1869824) | 20260505 08:44:17.077 | INFO | MemInfo: { | 20260505 08:44:17.077 | INFO | "gpu" : " 17 GB", | 20260505 08:44:17.077 | INFO | "huge" : "", | 20260505 08:44:17.077 | INFO | "huge_shared" : " 27 GB", | 20260505 08:44:17.077 | INFO | "malloc" : " 14 GB", | 20260505 08:44:17.077 | INFO | "pinned" : " 29 GB" | 20260505 08:44:17.077 | INFO | } | 20260505 08:44:17.077 | INFO | Perf report for the last 5.13914 seconds: | 20260505 08:44:17.077 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:44:17.077 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:17.077 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:17.077 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:17.077 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260505 08:44:17.077 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:17.077 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:17.077 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:44:17.077 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:17.077 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:17.077 | INFO | Uploaded 36GiB to GPU in 2.18005s (16.5134GiB/s) | 20260505 08:44:17.077 | INFO | Downloaded 34GiB to GPU in 2.06389s (16.4738GiB/s) | 20260505 08:44:22.217 | INFO | Analyzing chunk frames = [1869824,1870336) | 20260505 08:44:22.217 | INFO | MemInfo: { | 20260505 08:44:22.217 | INFO | "gpu" : " 17 GB", | 20260505 08:44:22.217 | INFO | "huge" : "", | 20260505 08:44:22.217 | INFO | "huge_shared" : " 27 GB", | 20260505 08:44:22.218 | INFO | "malloc" : " 14 GB", | 20260505 08:44:22.218 | INFO | "pinned" : " 29 GB" | 20260505 08:44:22.218 | INFO | } | 20260505 08:44:22.218 | INFO | Perf report for the last 5.14083 seconds: | 20260505 08:44:22.218 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:44:22.218 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:22.218 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:22.218 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:22.218 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 08:44:22.218 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:22.218 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:22.218 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 08:44:22.218 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:22.218 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:22.218 | INFO | Uploaded 36GiB to GPU in 2.16899s (16.5976GiB/s) | 20260505 08:44:22.218 | INFO | Downloaded 34GiB to GPU in 2.0606s (16.5GiB/s) | 20260505 08:44:27.356 | INFO | Analyzing chunk frames = [1870336,1870848) | 20260505 08:44:27.356 | INFO | MemInfo: { | 20260505 08:44:27.356 | INFO | "gpu" : " 17 GB", | 20260505 08:44:27.356 | INFO | "huge" : "", | 20260505 08:44:27.356 | INFO | "huge_shared" : " 27 GB", | 20260505 08:44:27.356 | INFO | "malloc" : " 14 GB", | 20260505 08:44:27.356 | INFO | "pinned" : " 29 GB" | 20260505 08:44:27.356 | INFO | } | 20260505 08:44:27.356 | INFO | Perf report for the last 5.13824 seconds: | 20260505 08:44:27.356 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:44:27.356 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:27.356 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:27.356 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:27.356 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 08:44:27.356 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:27.356 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:27.356 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:44:27.356 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:27.356 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:27.356 | INFO | Uploaded 36GiB to GPU in 2.24725s (16.0196GiB/s) | 20260505 08:44:27.356 | INFO | Downloaded 34GiB to GPU in 2.11416s (16.082GiB/s) | 20260505 08:44:32.494 | INFO | Analyzing chunk frames = [1870848,1871360) | 20260505 08:44:32.494 | INFO | MemInfo: { | 20260505 08:44:32.494 | INFO | "gpu" : " 17 GB", | 20260505 08:44:32.494 | INFO | "huge" : "", | 20260505 08:44:32.494 | INFO | "huge_shared" : " 27 GB", | 20260505 08:44:32.494 | INFO | "malloc" : " 14 GB", | 20260505 08:44:32.494 | INFO | "pinned" : " 29 GB" | 20260505 08:44:32.495 | INFO | } | 20260505 08:44:32.495 | INFO | Perf report for the last 5.13868 seconds: | 20260505 08:44:32.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:44:32.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:32.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:32.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:32.495 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 08:44:32.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:32.495 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:32.495 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 08:44:32.495 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:32.495 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:32.495 | INFO | Uploaded 36GiB to GPU in 2.20562s (16.3219GiB/s) | 20260505 08:44:32.495 | INFO | Downloaded 34GiB to GPU in 2.14549s (15.8472GiB/s) | 20260505 08:44:37.634 | INFO | Analyzing chunk frames = [1871360,1871872) | 20260505 08:44:37.634 | INFO | MemInfo: { | 20260505 08:44:37.634 | INFO | "gpu" : " 17 GB", | 20260505 08:44:37.634 | INFO | "huge" : "", | 20260505 08:44:37.635 | INFO | "huge_shared" : " 27 GB", | 20260505 08:44:37.635 | INFO | "malloc" : " 14 GB", | 20260505 08:44:37.635 | INFO | "pinned" : " 29 GB" | 20260505 08:44:37.635 | INFO | } | 20260505 08:44:37.635 | INFO | Perf report for the last 5.14007 seconds: | 20260505 08:44:37.635 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:44:37.635 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:37.635 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:37.635 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:37.635 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 08:44:37.635 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:37.635 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:37.635 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 08:44:37.635 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:37.635 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:37.635 | INFO | Uploaded 36GiB to GPU in 2.17753s (16.5325GiB/s) | 20260505 08:44:37.635 | INFO | Downloaded 34GiB to GPU in 2.0328s (16.7257GiB/s) | 20260505 08:44:37.773 | INFO | Starting a round of Baz IO | 20260505 08:44:41.054 | INFO | Populating RTMetrics report | 20260505 08:44:41.055 | INFO | RTMetrics report populated | 20260505 08:44:41.055 | INFO | Opening tmp RTMetrics json file | 20260505 08:44:41.094 | INFO | Serializing RTMetrics to json | 20260505 08:44:41.110 | INFO | Serialization done, swapping tmp file now... | 20260505 08:44:41.118 | INFO | RTMetrics done | 20260505 08:44:41.195 | INFO | Finished a round of Baz IO: Wrote 2.448 GiB in 3.422 seconds | 20260505 08:44:41.195 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 27.14% / 0.02% / 72.85% | 20260505 08:44:41.195 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.42832% / 2.78155e-05% / 9.93411e-05% | 20260505 08:44:41.195 | INFO | Baz IO overall: | 20260505 08:44:41.195 | INFO | BytesWritten : 1064587079429 | 20260505 08:44:41.195 | INFO | NumEvents : 506236142676 | 20260505 08:44:42.774 | INFO | Analyzing chunk frames = [1871872,1872384) | 20260505 08:44:42.774 | INFO | MemInfo: { | 20260505 08:44:42.774 | INFO | "gpu" : " 17 GB", | 20260505 08:44:42.774 | INFO | "huge" : "", | 20260505 08:44:42.774 | INFO | "huge_shared" : " 27 GB", | 20260505 08:44:42.774 | INFO | "malloc" : " 14 GB", | 20260505 08:44:42.774 | INFO | "pinned" : " 29 GB" | 20260505 08:44:42.774 | INFO | } | 20260505 08:44:42.774 | INFO | Perf report for the last 5.13922 seconds: | 20260505 08:44:42.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:44:42.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:42.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:42.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:42.774 | INFO | ANALYSIS : 65.4%, 2.9, 100.3ms, 0 | 20260505 08:44:42.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:42.774 | INFO | RT_METRICS : 47.0%, 1.0, 24.6ms, 0 | 20260505 08:44:42.774 | INFO | PRE_HQ : 63.0%, 1.9, 60.6ms, 0 | 20260505 08:44:42.774 | INFO | BAZWRITER : 65.9%, 4.1, 139.2ms, 0 | 20260505 08:44:42.774 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.5ms, 0 | 20260505 08:44:42.774 | INFO | Uploaded 36GiB to GPU in 2.24529s (16.0336GiB/s) | 20260505 08:44:42.774 | INFO | Downloaded 38GiB to GPU in 2.34744s (16.1878GiB/s) | 20260505 08:44:47.912 | INFO | Analyzing chunk frames = [1872384,1872896) | 20260505 08:44:47.912 | INFO | MemInfo: { | 20260505 08:44:47.912 | INFO | "gpu" : " 17 GB", | 20260505 08:44:47.912 | INFO | "huge" : "", | 20260505 08:44:47.912 | INFO | "huge_shared" : " 27 GB", | 20260505 08:44:47.912 | INFO | "malloc" : " 14 GB", | 20260505 08:44:47.912 | INFO | "pinned" : " 29 GB" | 20260505 08:44:47.912 | INFO | } | 20260505 08:44:47.912 | INFO | Perf report for the last 5.13835 seconds: | 20260505 08:44:47.912 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:44:47.912 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:47.912 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:47.912 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:47.912 | INFO | ANALYSIS : 64.5%, 3.0, 100.6ms, 0 | 20260505 08:44:47.912 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:47.912 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:47.912 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:44:47.912 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:47.912 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:47.912 | INFO | Uploaded 36GiB to GPU in 2.22055s (16.2122GiB/s) | 20260505 08:44:47.913 | INFO | Downloaded 34GiB to GPU in 2.08994s (16.2684GiB/s) | 20260505 08:44:53.052 | INFO | Analyzing chunk frames = [1872896,1873408) | 20260505 08:44:53.052 | INFO | MemInfo: { | 20260505 08:44:53.052 | INFO | "gpu" : " 17 GB", | 20260505 08:44:53.052 | INFO | "huge" : "", | 20260505 08:44:53.052 | INFO | "huge_shared" : " 27 GB", | 20260505 08:44:53.052 | INFO | "malloc" : " 14 GB", | 20260505 08:44:53.052 | INFO | "pinned" : " 29 GB" | 20260505 08:44:53.052 | INFO | } | 20260505 08:44:53.052 | INFO | Perf report for the last 5.13978 seconds: | 20260505 08:44:53.052 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:44:53.052 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:53.052 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:53.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:53.052 | INFO | ANALYSIS : 63.6%, 3.0, 99.0ms, 0 | 20260505 08:44:53.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:53.052 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:53.052 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:44:53.052 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:53.052 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:53.052 | INFO | Uploaded 36GiB to GPU in 2.17432s (16.5569GiB/s) | 20260505 08:44:53.052 | INFO | Downloaded 34GiB to GPU in 2.07067s (16.4198GiB/s) | 20260505 08:44:58.192 | INFO | Analyzing chunk frames = [1873408,1873920) | 20260505 08:44:58.192 | INFO | MemInfo: { | 20260505 08:44:58.192 | INFO | "gpu" : " 17 GB", | 20260505 08:44:58.194 | INFO | "huge" : "", | 20260505 08:44:58.194 | INFO | "huge_shared" : " 27 GB", | 20260505 08:44:58.194 | INFO | "malloc" : " 14 GB", | 20260505 08:44:58.194 | INFO | "pinned" : " 29 GB" | 20260505 08:44:58.194 | INFO | } | 20260505 08:44:58.194 | INFO | Perf report for the last 5.14213 seconds: | 20260505 08:44:58.194 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:44:58.194 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:58.194 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:58.194 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:58.194 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260505 08:44:58.194 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:58.194 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:58.194 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 08:44:58.194 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:58.194 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:44:58.194 | INFO | Uploaded 36GiB to GPU in 2.18144s (16.5029GiB/s) | 20260505 08:44:58.194 | INFO | Downloaded 34GiB to GPU in 2.06593s (16.4574GiB/s) | 20260505 08:45:03.331 | INFO | Analyzing chunk frames = [1873920,1874432) | 20260505 08:45:03.331 | INFO | MemInfo: { | 20260505 08:45:03.331 | INFO | "gpu" : " 17 GB", | 20260505 08:45:03.331 | INFO | "huge" : "", | 20260505 08:45:03.331 | INFO | "huge_shared" : " 27 GB", | 20260505 08:45:03.331 | INFO | "malloc" : " 14 GB", | 20260505 08:45:03.331 | INFO | "pinned" : " 29 GB" | 20260505 08:45:03.331 | INFO | } | 20260505 08:45:03.331 | INFO | Perf report for the last 5.13727 seconds: | 20260505 08:45:03.331 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:45:03.331 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:03.331 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:03.332 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:03.332 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 08:45:03.332 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:03.332 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:03.332 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 08:45:03.332 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:03.332 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:03.332 | INFO | Uploaded 36GiB to GPU in 2.16876s (16.5993GiB/s) | 20260505 08:45:03.332 | INFO | Downloaded 34GiB to GPU in 2.06147s (16.4931GiB/s) | 20260505 08:45:08.471 | INFO | Analyzing chunk frames = [1874432,1874944) | 20260505 08:45:08.471 | INFO | MemInfo: { | 20260505 08:45:08.471 | INFO | "gpu" : " 17 GB", | 20260505 08:45:08.471 | INFO | "huge" : "", | 20260505 08:45:08.471 | INFO | "huge_shared" : " 27 GB", | 20260505 08:45:08.471 | INFO | "malloc" : " 14 GB", | 20260505 08:45:08.471 | INFO | "pinned" : " 29 GB" | 20260505 08:45:08.471 | INFO | } | 20260505 08:45:08.471 | INFO | Perf report for the last 5.13983 seconds: | 20260505 08:45:08.471 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:45:08.471 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:08.471 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:08.471 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:08.471 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 08:45:08.471 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:08.471 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:08.471 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 08:45:08.471 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:08.471 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:08.471 | INFO | Uploaded 36GiB to GPU in 2.16215s (16.6501GiB/s) | 20260505 08:45:08.472 | INFO | Downloaded 34GiB to GPU in 2.05793s (16.5214GiB/s) | 20260505 08:45:13.610 | INFO | Analyzing chunk frames = [1874944,1875456) | 20260505 08:45:13.610 | INFO | MemInfo: { | 20260505 08:45:13.610 | INFO | "gpu" : " 17 GB", | 20260505 08:45:13.610 | INFO | "huge" : "", | 20260505 08:45:13.610 | INFO | "huge_shared" : " 27 GB", | 20260505 08:45:13.610 | INFO | "malloc" : " 14 GB", | 20260505 08:45:13.610 | INFO | "pinned" : " 29 GB" | 20260505 08:45:13.610 | INFO | } | 20260505 08:45:13.610 | INFO | Perf report for the last 5.1392 seconds: | 20260505 08:45:13.611 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:45:13.611 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:13.611 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:13.611 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:13.611 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 08:45:13.611 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:13.611 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:13.611 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 08:45:13.611 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:13.611 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:13.611 | INFO | Uploaded 36GiB to GPU in 2.18877s (16.4476GiB/s) | 20260505 08:45:13.611 | INFO | Downloaded 34GiB to GPU in 2.13748s (15.9066GiB/s) | 20260505 08:45:18.749 | INFO | Analyzing chunk frames = [1875456,1875968) | 20260505 08:45:18.749 | INFO | MemInfo: { | 20260505 08:45:18.749 | INFO | "gpu" : " 17 GB", | 20260505 08:45:18.749 | INFO | "huge" : "", | 20260505 08:45:18.749 | INFO | "huge_shared" : " 27 GB", | 20260505 08:45:18.749 | INFO | "malloc" : " 14 GB", | 20260505 08:45:18.749 | INFO | "pinned" : " 29 GB" | 20260505 08:45:18.749 | INFO | } | 20260505 08:45:18.749 | INFO | Perf report for the last 5.13872 seconds: | 20260505 08:45:18.749 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:45:18.749 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:18.749 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:18.749 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:18.749 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 08:45:18.749 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:18.749 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:18.749 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 08:45:18.749 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:18.749 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:18.750 | INFO | Uploaded 36GiB to GPU in 2.16943s (16.5942GiB/s) | 20260505 08:45:18.750 | INFO | Downloaded 34GiB to GPU in 2.0288s (16.7586GiB/s) | 20260505 08:45:18.750 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 08:45:18.750 | INFO | Total mem usage: 15.3 GiB | 20260505 08:45:18.750 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 08:45:18.750 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -5.34%) | 20260505 08:45:18.750 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 08:45:18.750 | INFO | High Water Breakdown: | 20260505 08:45:18.750 | INFO | 60.9% : 8.86 GiB ( 94.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 08:45:18.750 | INFO | 38.9% : 5.67 GiB ( 85.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 08:45:18.750 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 08:45:18.750 | INFO | ------------------------------------------ | 20260505 08:45:18.750 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:45:18.750 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 08:45:18.750 | INFO | Total mem usage: 26.5 GiB | 20260505 08:45:18.750 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 08:45:18.750 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 08:45:18.750 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 08:45:18.750 | INFO | High Water Breakdown: | 20260505 08:45:18.750 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:45:18.750 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:45:18.750 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:45:18.750 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:45:18.750 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:45:18.750 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:45:18.750 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 08:45:18.750 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 08:45:18.750 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:45:18.750 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 08:45:18.750 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 08:45:18.750 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 08:45:18.750 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 08:45:18.750 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 08:45:18.750 | INFO | ------------------------------------------ | 20260505 08:45:18.750 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:45:18.750 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 08:45:18.750 | INFO | Total mem usage: 25.6 GiB | 20260505 08:45:18.750 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 08:45:18.750 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 08:45:18.750 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 08:45:18.750 | INFO | High Water Breakdown: | 20260505 08:45:18.750 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 08:45:18.750 | INFO | ------------------------------------------ | 20260505 08:45:18.750 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:45:18.750 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 08:45:18.750 | INFO | Total mem usage: 15.7 GiB | 20260505 08:45:18.750 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 08:45:18.750 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 08:45:18.750 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 08:45:18.750 | INFO | High Water Breakdown: | 20260505 08:45:18.750 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 08:45:18.750 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 08:45:18.750 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 08:45:18.750 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 08:45:18.750 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 08:45:18.750 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 08:45:18.750 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 08:45:18.750 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 08:45:18.750 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 08:45:18.750 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:45:18.750 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 08:45:18.750 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:45:18.750 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 08:45:18.751 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 08:45:18.751 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 08:45:18.751 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 08:45:18.751 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:45:18.751 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 08:45:18.751 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:45:18.751 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:45:18.751 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:45:18.751 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 08:45:18.751 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 08:45:18.751 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:45:18.751 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 08:45:18.751 | INFO | ------------------------------------------ | 20260505 08:45:18.751 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:45:18.751 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 08:45:18.751 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:45:18.751 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:45:18.751 | INFO | PerfReport: Full Execution -- 3.25e+01 / 7.56e+01, 1.37e+01 / 2.50e+00 | 20260505 08:45:18.751 | INFO | PerfReport: 0.29%, AggregateStats -- 9.32e-02 / 1.04e-01, 1.10e-02 / 5.49e-03 | 20260505 08:45:18.751 | INFO | PerfReport: 11.66%, Baseline -- 3.80e+00 / 4.50e+00, 6.21e-01 / 2.36e-01 | 20260505 08:45:18.751 | INFO | PerfReport: 3.31%, Histogram -- 1.08e+00 / 1.48e+00, 2.09e-01 / 1.05e-01 | 20260505 08:45:18.751 | INFO | PerfReport: 15.53%, DME -- 4.04e+01 / 4.41e+01, 4.08e+00 / 2.14e+00 | 20260505 08:45:18.751 | INFO | PerfReport: 56.42%, FrameLabeling -- 1.84e+01 / 2.03e+01, 1.87e+00 / 2.22e+00 | 20260505 08:45:18.751 | INFO | PerfReport: 7.51%, PulseAccumulating -- 2.44e+00 / 2.96e+00, 4.17e-01 / 2.78e-01 | 20260505 08:45:18.751 | INFO | PerfReport: 3.90%, Metrics -- 1.27e+00 / 1.58e+00, 3.61e-01 / 2.15e-01 | 20260505 08:45:18.751 | INFO | PerfReport: 1.39%, PulseSerializing -- 4.51e-01 / 5.06e-01, 8.53e-02 / 3.13e-02 | 20260505 08:45:18.751 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 08:45:18.751 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:45:18.751 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:45:18.751 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.33e+01 / 0.00e+00 | 20260505 08:45:18.751 | INFO | PerfReport: 22.50%, Upload -- 2.29e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260505 08:45:18.751 | INFO | PerfReport: 21.65%, Download -- 2.20e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260505 08:45:18.751 | INFO | PerfReport: 32.02%, Compute -- 3.25e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 08:45:18.751 | INFO | PerfReport: 1.78%, LockUpload -- 1.81e+00 / 0.00e+00, 7.47e+00 / 0.00e+00 | 20260505 08:45:18.751 | INFO | PerfReport: 1.07%, LockDownload -- 1.08e+00 / 0.00e+00, 2.65e+00 / 0.00e+00 | 20260505 08:45:18.751 | INFO | PerfReport: 20.98%, LockCompute -- 2.13e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260505 08:45:18.751 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.14e-02 / 0.00e+00 | 20260505 08:45:18.888 | INFO | Starting a round of Baz IO | 20260505 08:45:22.167 | INFO | Populating RTMetrics report | 20260505 08:45:22.168 | INFO | RTMetrics report populated | 20260505 08:45:22.168 | INFO | Opening tmp RTMetrics json file | 20260505 08:45:22.195 | INFO | Serializing RTMetrics to json | 20260505 08:45:22.244 | INFO | Serialization done, swapping tmp file now... | 20260505 08:45:22.244 | INFO | RTMetrics done | 20260505 08:45:22.322 | INFO | Finished a round of Baz IO: Wrote 2.449 GiB in 3.433 seconds | 20260505 08:45:22.322 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 27.12% / 0.02% / 72.86% | 20260505 08:45:22.322 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.42895% / 3.17891e-05% / 0.000107288% | 20260505 08:45:22.322 | INFO | Baz IO overall: | 20260505 08:45:22.322 | INFO | BytesWritten : 1067217177711 | 20260505 08:45:22.322 | INFO | NumEvents : 507494557163 | 20260505 08:45:23.888 | INFO | Analyzing chunk frames = [1875968,1876480) | 20260505 08:45:23.888 | INFO | MemInfo: { | 20260505 08:45:23.888 | INFO | "gpu" : " 17 GB", | 20260505 08:45:23.888 | INFO | "huge" : "", | 20260505 08:45:23.888 | INFO | "huge_shared" : " 27 GB", | 20260505 08:45:23.888 | INFO | "malloc" : " 14 GB", | 20260505 08:45:23.888 | INFO | "pinned" : " 29 GB" | 20260505 08:45:23.888 | INFO | } | 20260505 08:45:23.888 | INFO | Perf report for the last 5.13895 seconds: | 20260505 08:45:23.888 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:45:23.888 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:23.888 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:23.888 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:23.888 | INFO | ANALYSIS : 65.4%, 2.9, 100.3ms, 0 | 20260505 08:45:23.888 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:23.888 | INFO | RT_METRICS : 47.2%, 1.0, 24.8ms, 0 | 20260505 08:45:23.888 | INFO | PRE_HQ : 63.1%, 1.9, 60.6ms, 0 | 20260505 08:45:23.888 | INFO | BAZWRITER : 66.1%, 4.1, 140.1ms, 0 | 20260505 08:45:23.888 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260505 08:45:23.889 | INFO | Uploaded 36GiB to GPU in 2.23161s (16.1319GiB/s) | 20260505 08:45:23.889 | INFO | Downloaded 38GiB to GPU in 2.34848s (16.1807GiB/s) | 20260505 08:45:29.028 | INFO | Analyzing chunk frames = [1876480,1876992) | 20260505 08:45:29.028 | INFO | MemInfo: { | 20260505 08:45:29.028 | INFO | "gpu" : " 17 GB", | 20260505 08:45:29.028 | INFO | "huge" : "", | 20260505 08:45:29.028 | INFO | "huge_shared" : " 27 GB", | 20260505 08:45:29.028 | INFO | "malloc" : " 14 GB", | 20260505 08:45:29.028 | INFO | "pinned" : " 29 GB" | 20260505 08:45:29.028 | INFO | } | 20260505 08:45:29.028 | INFO | Perf report for the last 5.13984 seconds: | 20260505 08:45:29.028 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:45:29.028 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:29.028 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:29.028 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:29.028 | INFO | ANALYSIS : 64.4%, 3.0, 100.6ms, 0 | 20260505 08:45:29.028 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:29.028 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:29.028 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 08:45:29.028 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:29.028 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:29.028 | INFO | Uploaded 36GiB to GPU in 2.24019s (16.07GiB/s) | 20260505 08:45:29.028 | INFO | Downloaded 34GiB to GPU in 2.09097s (16.2604GiB/s) | 20260505 08:45:34.168 | INFO | Analyzing chunk frames = [1876992,1877504) | 20260505 08:45:34.168 | INFO | MemInfo: { | 20260505 08:45:34.168 | INFO | "gpu" : " 17 GB", | 20260505 08:45:34.168 | INFO | "huge" : "", | 20260505 08:45:34.168 | INFO | "huge_shared" : " 27 GB", | 20260505 08:45:34.168 | INFO | "malloc" : " 14 GB", | 20260505 08:45:34.168 | INFO | "pinned" : " 29 GB" | 20260505 08:45:34.168 | INFO | } | 20260505 08:45:34.168 | INFO | Perf report for the last 5.13987 seconds: | 20260505 08:45:34.168 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:45:34.168 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:34.168 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:34.168 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:34.168 | INFO | ANALYSIS : 63.6%, 3.0, 99.0ms, 0 | 20260505 08:45:34.168 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:34.168 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:34.168 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 08:45:34.168 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:34.168 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:34.168 | INFO | Uploaded 36GiB to GPU in 2.1758s (16.5456GiB/s) | 20260505 08:45:34.168 | INFO | Downloaded 34GiB to GPU in 2.06697s (16.4492GiB/s) | 20260505 08:45:39.308 | INFO | Analyzing chunk frames = [1877504,1878016) | 20260505 08:45:39.308 | INFO | MemInfo: { | 20260505 08:45:39.308 | INFO | "gpu" : " 17 GB", | 20260505 08:45:39.308 | INFO | "huge" : "", | 20260505 08:45:39.308 | INFO | "huge_shared" : " 27 GB", | 20260505 08:45:39.308 | INFO | "malloc" : " 14 GB", | 20260505 08:45:39.308 | INFO | "pinned" : " 29 GB" | 20260505 08:45:39.308 | INFO | } | 20260505 08:45:39.308 | INFO | Perf report for the last 5.13965 seconds: | 20260505 08:45:39.308 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:45:39.308 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:39.308 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:39.308 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:39.308 | INFO | ANALYSIS : 63.4%, 3.0, 98.6ms, 0 | 20260505 08:45:39.308 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:39.308 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:39.308 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:45:39.308 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:39.308 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:39.308 | INFO | Uploaded 36GiB to GPU in 2.16552s (16.6242GiB/s) | 20260505 08:45:39.308 | INFO | Downloaded 34GiB to GPU in 2.06171s (16.4912GiB/s) | 20260505 08:45:44.446 | INFO | Analyzing chunk frames = [1878016,1878528) | 20260505 08:45:44.446 | INFO | MemInfo: { | 20260505 08:45:44.446 | INFO | "gpu" : " 17 GB", | 20260505 08:45:44.446 | INFO | "huge" : "", | 20260505 08:45:44.446 | INFO | "huge_shared" : " 27 GB", | 20260505 08:45:44.446 | INFO | "malloc" : " 14 GB", | 20260505 08:45:44.446 | INFO | "pinned" : " 29 GB" | 20260505 08:45:44.446 | INFO | } | 20260505 08:45:44.446 | INFO | Perf report for the last 5.13814 seconds: | 20260505 08:45:44.446 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:45:44.446 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:44.446 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:44.446 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:44.446 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 08:45:44.446 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:44.446 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:44.446 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 08:45:44.446 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:44.446 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:44.446 | INFO | Uploaded 36GiB to GPU in 2.23521s (16.1059GiB/s) | 20260505 08:45:44.446 | INFO | Downloaded 34GiB to GPU in 2.11039s (16.1108GiB/s) | 20260505 08:45:49.586 | INFO | Analyzing chunk frames = [1878528,1879040) | 20260505 08:45:49.586 | INFO | MemInfo: { | 20260505 08:45:49.586 | INFO | "gpu" : " 17 GB", | 20260505 08:45:49.586 | INFO | "huge" : "", | 20260505 08:45:49.586 | INFO | "huge_shared" : " 27 GB", | 20260505 08:45:49.586 | INFO | "malloc" : " 14 GB", | 20260505 08:45:49.586 | INFO | "pinned" : " 29 GB" | 20260505 08:45:49.586 | INFO | } | 20260505 08:45:49.586 | INFO | Perf report for the last 5.13993 seconds: | 20260505 08:45:49.586 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:45:49.586 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:49.586 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:49.586 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:49.586 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 08:45:49.586 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:49.586 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:49.586 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 08:45:49.586 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:49.586 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:49.586 | INFO | Uploaded 36GiB to GPU in 2.24655s (16.0246GiB/s) | 20260505 08:45:49.586 | INFO | Downloaded 34GiB to GPU in 2.11043s (16.1105GiB/s) | 20260505 08:45:54.724 | INFO | Analyzing chunk frames = [1879040,1879552) | 20260505 08:45:54.724 | INFO | MemInfo: { | 20260505 08:45:54.724 | INFO | "gpu" : " 17 GB", | 20260505 08:45:54.724 | INFO | "huge" : "", | 20260505 08:45:54.724 | INFO | "huge_shared" : " 27 GB", | 20260505 08:45:54.724 | INFO | "malloc" : " 14 GB", | 20260505 08:45:54.724 | INFO | "pinned" : " 29 GB" | 20260505 08:45:54.724 | INFO | } | 20260505 08:45:54.724 | INFO | Perf report for the last 5.13868 seconds: | 20260505 08:45:54.725 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:45:54.725 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:54.725 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:54.725 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:54.725 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 08:45:54.725 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:54.725 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:54.725 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260505 08:45:54.725 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:54.725 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:54.725 | INFO | Uploaded 36GiB to GPU in 2.21228s (16.2728GiB/s) | 20260505 08:45:54.725 | INFO | Downloaded 34GiB to GPU in 2.14371s (15.8604GiB/s) | 20260505 08:45:59.863 | INFO | Analyzing chunk frames = [1879552,1880064) | 20260505 08:45:59.863 | INFO | MemInfo: { | 20260505 08:45:59.863 | INFO | "gpu" : " 17 GB", | 20260505 08:45:59.863 | INFO | "huge" : "", | 20260505 08:45:59.863 | INFO | "huge_shared" : " 27 GB", | 20260505 08:45:59.863 | INFO | "malloc" : " 14 GB", | 20260505 08:45:59.863 | INFO | "pinned" : " 29 GB" | 20260505 08:45:59.863 | INFO | } | 20260505 08:45:59.863 | INFO | Perf report for the last 5.13869 seconds: | 20260505 08:45:59.863 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:45:59.863 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:59.863 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:59.863 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:59.863 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 08:45:59.863 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:59.863 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:59.863 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:45:59.863 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:59.863 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:45:59.863 | INFO | Uploaded 36GiB to GPU in 2.16776s (16.607GiB/s) | 20260505 08:45:59.863 | INFO | Downloaded 34GiB to GPU in 2.02998s (16.7489GiB/s) | 20260505 08:46:00.003 | INFO | Starting a round of Baz IO | 20260505 08:46:03.280 | INFO | Populating RTMetrics report | 20260505 08:46:03.280 | INFO | RTMetrics report populated | 20260505 08:46:03.280 | INFO | Opening tmp RTMetrics json file | 20260505 08:46:03.308 | INFO | Serializing RTMetrics to json | 20260505 08:46:03.353 | INFO | Serialization done, swapping tmp file now... | 20260505 08:46:03.355 | INFO | RTMetrics done | 20260505 08:46:03.423 | INFO | Finished a round of Baz IO: Wrote 2.45 GiB in 3.419 seconds | 20260505 08:46:03.423 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 27.10% / 0.02% / 72.88% | 20260505 08:46:03.423 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.42336% / 2.38419e-05% / 9.93411e-05% | 20260505 08:46:03.423 | INFO | Baz IO overall: | 20260505 08:46:03.424 | INFO | BytesWritten : 1069848343257 | 20260505 08:46:03.424 | INFO | NumEvents : 508753914563 | 20260505 08:46:05.002 | INFO | Analyzing chunk frames = [1880064,1880576) | 20260505 08:46:05.002 | INFO | MemInfo: { | 20260505 08:46:05.002 | INFO | "gpu" : " 17 GB", | 20260505 08:46:05.002 | INFO | "huge" : "", | 20260505 08:46:05.002 | INFO | "huge_shared" : " 27 GB", | 20260505 08:46:05.003 | INFO | "malloc" : " 14 GB", | 20260505 08:46:05.003 | INFO | "pinned" : " 29 GB" | 20260505 08:46:05.003 | INFO | } | 20260505 08:46:05.003 | INFO | Perf report for the last 5.13956 seconds: | 20260505 08:46:05.003 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:46:05.003 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:05.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:05.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:05.003 | INFO | ANALYSIS : 65.3%, 2.9, 100.0ms, 0 | 20260505 08:46:05.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:05.003 | INFO | RT_METRICS : 47.2%, 1.0, 24.7ms, 0 | 20260505 08:46:05.003 | INFO | PRE_HQ : 63.3%, 1.8, 60.7ms, 0 | 20260505 08:46:05.003 | INFO | BAZWRITER : 65.9%, 4.2, 140.8ms, 0 | 20260505 08:46:05.003 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 08:46:05.003 | INFO | Uploaded 36GiB to GPU in 2.28398s (15.7619GiB/s) | 20260505 08:46:05.003 | INFO | Downloaded 38GiB to GPU in 2.3736s (16.0094GiB/s) | 20260505 08:46:10.144 | INFO | Analyzing chunk frames = [1880576,1881088) | 20260505 08:46:10.144 | INFO | MemInfo: { | 20260505 08:46:10.144 | INFO | "gpu" : " 17 GB", | 20260505 08:46:10.144 | INFO | "huge" : "", | 20260505 08:46:10.144 | INFO | "huge_shared" : " 27 GB", | 20260505 08:46:10.144 | INFO | "malloc" : " 14 GB", | 20260505 08:46:10.144 | INFO | "pinned" : " 29 GB" | 20260505 08:46:10.144 | INFO | } | 20260505 08:46:10.144 | INFO | Perf report for the last 5.14122 seconds: | 20260505 08:46:10.144 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:46:10.144 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:10.144 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:10.144 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:10.144 | INFO | ANALYSIS : 64.4%, 3.0, 100.6ms, 0 | 20260505 08:46:10.144 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:10.144 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:10.144 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 08:46:10.144 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:10.144 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:10.144 | INFO | Uploaded 36GiB to GPU in 2.22483s (16.181GiB/s) | 20260505 08:46:10.144 | INFO | Downloaded 34GiB to GPU in 2.08443s (16.3114GiB/s) | 20260505 08:46:15.283 | INFO | Analyzing chunk frames = [1881088,1881600) | 20260505 08:46:15.283 | INFO | MemInfo: { | 20260505 08:46:15.283 | INFO | "gpu" : " 17 GB", | 20260505 08:46:15.283 | INFO | "huge" : "", | 20260505 08:46:15.283 | INFO | "huge_shared" : " 27 GB", | 20260505 08:46:15.283 | INFO | "malloc" : " 14 GB", | 20260505 08:46:15.283 | INFO | "pinned" : " 29 GB" | 20260505 08:46:15.283 | INFO | } | 20260505 08:46:15.283 | INFO | Perf report for the last 5.13886 seconds: | 20260505 08:46:15.283 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:46:15.283 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:15.283 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:15.283 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:15.283 | INFO | ANALYSIS : 63.6%, 3.0, 99.0ms, 0 | 20260505 08:46:15.283 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:15.283 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:15.283 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:46:15.283 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:15.283 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:15.283 | INFO | Uploaded 36GiB to GPU in 2.17305s (16.5665GiB/s) | 20260505 08:46:15.283 | INFO | Downloaded 34GiB to GPU in 2.06817s (16.4396GiB/s) | 20260505 08:46:20.421 | INFO | Analyzing chunk frames = [1881600,1882112) | 20260505 08:46:20.421 | INFO | MemInfo: { | 20260505 08:46:20.421 | INFO | "gpu" : " 17 GB", | 20260505 08:46:20.421 | INFO | "huge" : "", | 20260505 08:46:20.421 | INFO | "huge_shared" : " 27 GB", | 20260505 08:46:20.421 | INFO | "malloc" : " 14 GB", | 20260505 08:46:20.421 | INFO | "pinned" : " 29 GB" | 20260505 08:46:20.421 | INFO | } | 20260505 08:46:20.421 | INFO | Perf report for the last 5.13858 seconds: | 20260505 08:46:20.421 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:46:20.421 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:20.421 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:20.421 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:20.421 | INFO | ANALYSIS : 63.5%, 3.0, 98.6ms, 0 | 20260505 08:46:20.421 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:20.421 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:20.422 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:46:20.422 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:20.422 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:20.422 | INFO | Uploaded 36GiB to GPU in 2.1677s (16.6075GiB/s) | 20260505 08:46:20.422 | INFO | Downloaded 34GiB to GPU in 2.06692s (16.4496GiB/s) | 20260505 08:46:25.560 | INFO | Analyzing chunk frames = [1882112,1882624) | 20260505 08:46:25.560 | INFO | MemInfo: { | 20260505 08:46:25.560 | INFO | "gpu" : " 17 GB", | 20260505 08:46:25.560 | INFO | "huge" : "", | 20260505 08:46:25.560 | INFO | "huge_shared" : " 27 GB", | 20260505 08:46:25.560 | INFO | "malloc" : " 14 GB", | 20260505 08:46:25.560 | INFO | "pinned" : " 29 GB" | 20260505 08:46:25.560 | INFO | } | 20260505 08:46:25.560 | INFO | Perf report for the last 5.13877 seconds: | 20260505 08:46:25.560 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:46:25.560 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:25.560 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:25.560 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:25.560 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 08:46:25.560 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:25.560 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:25.560 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260505 08:46:25.560 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:25.560 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:25.560 | INFO | Uploaded 36GiB to GPU in 2.17314s (16.5659GiB/s) | 20260505 08:46:25.560 | INFO | Downloaded 34GiB to GPU in 2.0639s (16.4737GiB/s) | 20260505 08:46:30.700 | INFO | Analyzing chunk frames = [1882624,1883136) | 20260505 08:46:30.701 | INFO | MemInfo: { | 20260505 08:46:30.701 | INFO | "gpu" : " 17 GB", | 20260505 08:46:30.701 | INFO | "huge" : "", | 20260505 08:46:30.701 | INFO | "huge_shared" : " 27 GB", | 20260505 08:46:30.701 | INFO | "malloc" : " 14 GB", | 20260505 08:46:30.701 | INFO | "pinned" : " 29 GB" | 20260505 08:46:30.701 | INFO | } | 20260505 08:46:30.701 | INFO | Perf report for the last 5.14043 seconds: | 20260505 08:46:30.701 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:46:30.701 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:30.701 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:30.701 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:30.701 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 08:46:30.701 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:30.701 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:30.701 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:46:30.701 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:30.701 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:30.701 | INFO | Uploaded 36GiB to GPU in 2.17335s (16.5643GiB/s) | 20260505 08:46:30.701 | INFO | Downloaded 34GiB to GPU in 2.05419s (16.5515GiB/s) | 20260505 08:46:35.839 | INFO | Analyzing chunk frames = [1883136,1883648) | 20260505 08:46:35.839 | INFO | MemInfo: { | 20260505 08:46:35.839 | INFO | "gpu" : " 17 GB", | 20260505 08:46:35.839 | INFO | "huge" : "", | 20260505 08:46:35.839 | INFO | "huge_shared" : " 27 GB", | 20260505 08:46:35.839 | INFO | "malloc" : " 14 GB", | 20260505 08:46:35.839 | INFO | "pinned" : " 29 GB" | 20260505 08:46:35.839 | INFO | } | 20260505 08:46:35.839 | INFO | Perf report for the last 5.13812 seconds: | 20260505 08:46:35.839 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:46:35.839 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:35.839 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:35.839 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:35.839 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 08:46:35.839 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:35.839 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:35.839 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 08:46:35.839 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:35.839 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:35.839 | INFO | Uploaded 36GiB to GPU in 2.22015s (16.2151GiB/s) | 20260505 08:46:35.839 | INFO | Downloaded 34GiB to GPU in 2.14759s (15.8317GiB/s) | 20260505 08:46:40.978 | INFO | Analyzing chunk frames = [1883648,1884160) | 20260505 08:46:40.978 | INFO | MemInfo: { | 20260505 08:46:40.978 | INFO | "gpu" : " 17 GB", | 20260505 08:46:40.978 | INFO | "huge" : "", | 20260505 08:46:40.978 | INFO | "huge_shared" : " 27 GB", | 20260505 08:46:40.978 | INFO | "malloc" : " 14 GB", | 20260505 08:46:40.978 | INFO | "pinned" : " 29 GB" | 20260505 08:46:40.978 | INFO | } | 20260505 08:46:40.978 | INFO | Perf report for the last 5.13932 seconds: | 20260505 08:46:40.978 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:46:40.978 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:40.978 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:40.978 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:40.978 | INFO | ANALYSIS : 64.1%, 3.0, 100.0ms, 0 | 20260505 08:46:40.978 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:40.978 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:40.978 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:46:40.978 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:40.978 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:40.978 | INFO | Uploaded 36GiB to GPU in 2.19017s (16.4371GiB/s) | 20260505 08:46:40.979 | INFO | Downloaded 34GiB to GPU in 2.0367s (16.6937GiB/s) | 20260505 08:46:40.979 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 08:46:40.979 | INFO | Total mem usage: 15.3 GiB | 20260505 08:46:40.979 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 08:46:40.979 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.34%) | 20260505 08:46:40.979 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 08:46:40.979 | INFO | High Water Breakdown: | 20260505 08:46:40.979 | INFO | 60.9% : 8.78 GiB ( 93.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 08:46:40.979 | INFO | 39% : 5.62 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 08:46:40.979 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 08:46:40.979 | INFO | ------------------------------------------ | 20260505 08:46:40.979 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:46:40.979 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 08:46:40.979 | INFO | Total mem usage: 26.5 GiB | 20260505 08:46:40.979 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 08:46:40.979 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 08:46:40.979 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 08:46:40.979 | INFO | High Water Breakdown: | 20260505 08:46:40.979 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:46:40.979 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:46:40.979 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:46:40.979 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:46:40.979 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:46:40.979 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:46:40.979 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 08:46:40.979 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 08:46:40.979 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:46:40.979 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 08:46:40.979 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 08:46:40.979 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 08:46:40.979 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 08:46:40.979 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 08:46:40.979 | INFO | ------------------------------------------ | 20260505 08:46:40.979 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:46:40.979 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 08:46:40.979 | INFO | Total mem usage: 25.6 GiB | 20260505 08:46:40.979 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 08:46:40.979 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 08:46:40.979 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 08:46:40.979 | INFO | High Water Breakdown: | 20260505 08:46:40.979 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 08:46:40.979 | INFO | ------------------------------------------ | 20260505 08:46:40.979 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:46:40.979 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 08:46:40.979 | INFO | Total mem usage: 15.7 GiB | 20260505 08:46:40.979 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 08:46:40.979 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 08:46:40.979 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 08:46:40.979 | INFO | High Water Breakdown: | 20260505 08:46:40.979 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 08:46:40.979 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 08:46:40.979 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 08:46:40.979 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 08:46:40.979 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 08:46:40.979 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 08:46:40.979 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 08:46:40.979 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 08:46:40.979 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 08:46:40.979 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:46:40.979 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 08:46:40.979 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:46:40.979 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 08:46:40.979 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 08:46:40.979 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 08:46:40.979 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 08:46:40.979 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:46:40.979 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 08:46:40.979 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:46:40.979 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:46:40.979 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:46:40.979 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 08:46:40.980 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 08:46:40.980 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:46:40.980 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 08:46:40.980 | INFO | ------------------------------------------ | 20260505 08:46:40.980 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:46:40.980 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 08:46:40.980 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:46:40.980 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:46:40.980 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.58e+01, 1.35e+01 / 2.28e+00 | 20260505 08:46:40.980 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 1.03e-01, 1.01e-02 / 3.87e-03 | 20260505 08:46:40.980 | INFO | PerfReport: 11.65%, Baseline -- 3.79e+00 / 4.43e+00, 6.14e-01 / 3.24e-01 | 20260505 08:46:40.980 | INFO | PerfReport: 3.31%, Histogram -- 1.08e+00 / 1.47e+00, 2.06e-01 / 1.20e-01 | 20260505 08:46:40.980 | INFO | PerfReport: 15.52%, DME -- 4.04e+01 / 4.41e+01, 4.14e+00 / 1.90e+00 | 20260505 08:46:40.980 | INFO | PerfReport: 56.48%, FrameLabeling -- 1.84e+01 / 2.06e+01, 1.89e+00 / 1.90e+00 | 20260505 08:46:40.980 | INFO | PerfReport: 7.47%, PulseAccumulating -- 2.43e+00 / 2.93e+00, 4.09e-01 / 3.02e-01 | 20260505 08:46:40.980 | INFO | PerfReport: 3.90%, Metrics -- 1.27e+00 / 1.58e+00, 3.61e-01 / 1.88e-01 | 20260505 08:46:40.980 | INFO | PerfReport: 1.39%, PulseSerializing -- 4.52e-01 / 5.03e-01, 8.57e-02 / 3.81e-02 | 20260505 08:46:40.980 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 08:46:40.980 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:46:40.980 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:46:40.980 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.31e+01 / 0.00e+00 | 20260505 08:46:40.980 | INFO | PerfReport: 22.58%, Upload -- 2.30e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260505 08:46:40.980 | INFO | PerfReport: 21.67%, Download -- 2.20e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260505 08:46:40.980 | INFO | PerfReport: 32.01%, Compute -- 3.26e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 08:46:40.980 | INFO | PerfReport: 1.76%, LockUpload -- 1.79e+00 / 0.00e+00, 7.45e+00 / 0.00e+00 | 20260505 08:46:40.980 | INFO | PerfReport: 1.06%, LockDownload -- 1.08e+00 / 0.00e+00, 2.59e+00 / 0.00e+00 | 20260505 08:46:40.980 | INFO | PerfReport: 20.89%, LockCompute -- 2.13e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260505 08:46:40.980 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.16e-02 / 0.00e+00 | 20260505 08:46:41.117 | INFO | Starting a round of Baz IO | 20260505 08:46:44.387 | INFO | Populating RTMetrics report | 20260505 08:46:44.387 | INFO | RTMetrics report populated | 20260505 08:46:44.387 | INFO | Opening tmp RTMetrics json file | 20260505 08:46:44.400 | INFO | Serializing RTMetrics to json | 20260505 08:46:44.403 | INFO | Serialization done, swapping tmp file now... | 20260505 08:46:44.406 | INFO | RTMetrics done | 20260505 08:46:44.521 | INFO | Finished a round of Baz IO: Wrote 2.448 GiB in 3.404 seconds | 20260505 08:46:44.521 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 27.08% / 0.02% / 72.90% | 20260505 08:46:44.521 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.40947% / 3.17891e-05% / 9.93411e-05% | 20260505 08:46:44.521 | INFO | Baz IO overall: | 20260505 08:46:44.521 | INFO | BytesWritten : 1072476575440 | 20260505 08:46:44.521 | INFO | NumEvents : 510010968644 | 20260505 08:46:46.118 | INFO | Analyzing chunk frames = [1884160,1884672) | 20260505 08:46:46.118 | INFO | MemInfo: { | 20260505 08:46:46.118 | INFO | "gpu" : " 17 GB", | 20260505 08:46:46.118 | INFO | "huge" : "", | 20260505 08:46:46.118 | INFO | "huge_shared" : " 27 GB", | 20260505 08:46:46.118 | INFO | "malloc" : " 14 GB", | 20260505 08:46:46.118 | INFO | "pinned" : " 29 GB" | 20260505 08:46:46.118 | INFO | } | 20260505 08:46:46.118 | INFO | Perf report for the last 5.13998 seconds: | 20260505 08:46:46.118 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:46:46.118 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:46.118 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:46.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:46.118 | INFO | ANALYSIS : 65.2%, 2.9, 99.5ms, 0 | 20260505 08:46:46.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:46.118 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260505 08:46:46.118 | INFO | PRE_HQ : 63.3%, 1.9, 60.9ms, 0 | 20260505 08:46:46.118 | INFO | BAZWRITER : 65.5%, 4.2, 140.8ms, 0 | 20260505 08:46:46.118 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 08:46:46.118 | INFO | Uploaded 36GiB to GPU in 2.26069s (15.9243GiB/s) | 20260505 08:46:46.119 | INFO | Downloaded 38GiB to GPU in 2.38647s (15.9231GiB/s) | 20260505 08:46:51.258 | INFO | Analyzing chunk frames = [1884672,1885184) | 20260505 08:46:51.258 | INFO | MemInfo: { | 20260505 08:46:51.258 | INFO | "gpu" : " 17 GB", | 20260505 08:46:51.258 | INFO | "huge" : "", | 20260505 08:46:51.258 | INFO | "huge_shared" : " 27 GB", | 20260505 08:46:51.258 | INFO | "malloc" : " 14 GB", | 20260505 08:46:51.258 | INFO | "pinned" : " 29 GB" | 20260505 08:46:51.258 | INFO | } | 20260505 08:46:51.259 | INFO | Perf report for the last 5.14059 seconds: | 20260505 08:46:51.259 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:46:51.259 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:51.259 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:51.259 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:51.259 | INFO | ANALYSIS : 64.4%, 3.0, 100.6ms, 0 | 20260505 08:46:51.259 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:51.259 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:51.259 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 08:46:51.259 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:51.259 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:51.259 | INFO | Uploaded 36GiB to GPU in 2.2191s (16.2228GiB/s) | 20260505 08:46:51.259 | INFO | Downloaded 34GiB to GPU in 2.09224s (16.2505GiB/s) | 20260505 08:46:56.396 | INFO | Analyzing chunk frames = [1885184,1885696) | 20260505 08:46:56.396 | INFO | MemInfo: { | 20260505 08:46:56.396 | INFO | "gpu" : " 17 GB", | 20260505 08:46:56.396 | INFO | "huge" : "", | 20260505 08:46:56.396 | INFO | "huge_shared" : " 27 GB", | 20260505 08:46:56.396 | INFO | "malloc" : " 14 GB", | 20260505 08:46:56.396 | INFO | "pinned" : " 29 GB" | 20260505 08:46:56.397 | INFO | } | 20260505 08:46:56.397 | INFO | Perf report for the last 5.13804 seconds: | 20260505 08:46:56.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:46:56.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:56.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:56.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:56.397 | INFO | ANALYSIS : 63.6%, 3.0, 98.9ms, 0 | 20260505 08:46:56.397 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:56.397 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:56.397 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:46:56.397 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:56.397 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:46:56.397 | INFO | Uploaded 36GiB to GPU in 2.17304s (16.5667GiB/s) | 20260505 08:46:56.397 | INFO | Downloaded 34GiB to GPU in 2.06968s (16.4277GiB/s) | 20260505 08:47:01.535 | INFO | Analyzing chunk frames = [1885696,1886208) | 20260505 08:47:01.535 | INFO | MemInfo: { | 20260505 08:47:01.535 | INFO | "gpu" : " 17 GB", | 20260505 08:47:01.535 | INFO | "huge" : "", | 20260505 08:47:01.535 | INFO | "huge_shared" : " 27 GB", | 20260505 08:47:01.535 | INFO | "malloc" : " 14 GB", | 20260505 08:47:01.536 | INFO | "pinned" : " 29 GB" | 20260505 08:47:01.536 | INFO | } | 20260505 08:47:01.536 | INFO | Perf report for the last 5.139 seconds: | 20260505 08:47:01.536 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:47:01.536 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:01.536 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:01.536 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:01.536 | INFO | ANALYSIS : 63.5%, 3.0, 98.6ms, 0 | 20260505 08:47:01.536 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:01.536 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:01.536 | INFO | PRE_HQ : 29.8%, 1.0, 15.4ms, 0 | 20260505 08:47:01.536 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:01.536 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:01.536 | INFO | Uploaded 36GiB to GPU in 2.16844s (16.6018GiB/s) | 20260505 08:47:01.536 | INFO | Downloaded 34GiB to GPU in 2.06323s (16.479GiB/s) | 20260505 08:47:06.674 | INFO | Analyzing chunk frames = [1886208,1886720) | 20260505 08:47:06.674 | INFO | MemInfo: { | 20260505 08:47:06.674 | INFO | "gpu" : " 17 GB", | 20260505 08:47:06.674 | INFO | "huge" : "", | 20260505 08:47:06.674 | INFO | "huge_shared" : " 27 GB", | 20260505 08:47:06.674 | INFO | "malloc" : " 14 GB", | 20260505 08:47:06.674 | INFO | "pinned" : " 29 GB" | 20260505 08:47:06.674 | INFO | } | 20260505 08:47:06.675 | INFO | Perf report for the last 5.13881 seconds: | 20260505 08:47:06.675 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:47:06.675 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:06.675 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:06.675 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:06.675 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 08:47:06.675 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:06.675 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:06.675 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 08:47:06.675 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:06.675 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:06.675 | INFO | Uploaded 36GiB to GPU in 2.23326s (16.1199GiB/s) | 20260505 08:47:06.675 | INFO | Downloaded 34GiB to GPU in 2.10882s (16.1227GiB/s) | 20260505 08:47:11.814 | INFO | Analyzing chunk frames = [1886720,1887232) | 20260505 08:47:11.814 | INFO | MemInfo: { | 20260505 08:47:11.814 | INFO | "gpu" : " 17 GB", | 20260505 08:47:11.814 | INFO | "huge" : "", | 20260505 08:47:11.814 | INFO | "huge_shared" : " 27 GB", | 20260505 08:47:11.814 | INFO | "malloc" : " 14 GB", | 20260505 08:47:11.814 | INFO | "pinned" : " 29 GB" | 20260505 08:47:11.814 | INFO | } | 20260505 08:47:11.814 | INFO | Perf report for the last 5.13989 seconds: | 20260505 08:47:11.815 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:47:11.815 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:11.815 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:11.815 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:11.815 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 08:47:11.815 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:11.815 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:11.815 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 08:47:11.815 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:11.815 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:11.815 | INFO | Uploaded 36GiB to GPU in 2.16582s (16.6219GiB/s) | 20260505 08:47:11.815 | INFO | Downloaded 34GiB to GPU in 2.06275s (16.4829GiB/s) | 20260505 08:47:16.953 | INFO | Analyzing chunk frames = [1887232,1887744) | 20260505 08:47:16.953 | INFO | MemInfo: { | 20260505 08:47:16.953 | INFO | "gpu" : " 17 GB", | 20260505 08:47:16.954 | INFO | "huge" : "", | 20260505 08:47:16.954 | INFO | "huge_shared" : " 27 GB", | 20260505 08:47:16.954 | INFO | "malloc" : " 14 GB", | 20260505 08:47:16.954 | INFO | "pinned" : " 29 GB" | 20260505 08:47:16.954 | INFO | } | 20260505 08:47:16.954 | INFO | Perf report for the last 5.1393 seconds: | 20260505 08:47:16.954 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:47:16.954 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:16.954 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:16.954 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:16.954 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 08:47:16.954 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:16.954 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:16.954 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 08:47:16.954 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:16.954 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:16.954 | INFO | Uploaded 36GiB to GPU in 2.21718s (16.2369GiB/s) | 20260505 08:47:16.954 | INFO | Downloaded 34GiB to GPU in 2.15193s (15.7998GiB/s) | 20260505 08:47:22.094 | INFO | Analyzing chunk frames = [1887744,1888256) | 20260505 08:47:22.094 | INFO | MemInfo: { | 20260505 08:47:22.094 | INFO | "gpu" : " 17 GB", | 20260505 08:47:22.094 | INFO | "huge" : "", | 20260505 08:47:22.094 | INFO | "huge_shared" : " 27 GB", | 20260505 08:47:22.094 | INFO | "malloc" : " 14 GB", | 20260505 08:47:22.094 | INFO | "pinned" : " 29 GB" | 20260505 08:47:22.094 | INFO | } | 20260505 08:47:22.094 | INFO | Perf report for the last 5.14042 seconds: | 20260505 08:47:22.094 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:47:22.094 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:22.094 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:22.094 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:22.094 | INFO | ANALYSIS : 64.1%, 3.0, 100.2ms, 0 | 20260505 08:47:22.094 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:22.094 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:22.094 | INFO | PRE_HQ : 30.3%, 1.0, 15.8ms, 0 | 20260505 08:47:22.094 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:22.094 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:22.094 | INFO | Uploaded 36GiB to GPU in 2.19659s (16.389GiB/s) | 20260505 08:47:22.094 | INFO | Downloaded 34GiB to GPU in 2.03724s (16.6892GiB/s) | 20260505 08:47:22.230 | INFO | Starting a round of Baz IO | 20260505 08:47:25.497 | INFO | Populating RTMetrics report | 20260505 08:47:25.498 | INFO | RTMetrics report populated | 20260505 08:47:25.498 | INFO | Opening tmp RTMetrics json file | 20260505 08:47:25.539 | INFO | Serializing RTMetrics to json | 20260505 08:47:25.563 | INFO | Serialization done, swapping tmp file now... | 20260505 08:47:25.591 | INFO | RTMetrics done | 20260505 08:47:25.619 | INFO | Finished a round of Baz IO: Wrote 2.446 GiB in 3.389 seconds | 20260505 08:47:25.619 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 27.06% / 0.02% / 72.92% | 20260505 08:47:25.619 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.39368% / 2.38419e-05% / 7.54992e-05% | 20260505 08:47:25.619 | INFO | Baz IO overall: | 20260505 08:47:25.619 | INFO | BytesWritten : 1075102714015 | 20260505 08:47:25.619 | INFO | NumEvents : 511266458179 | 20260505 08:47:27.233 | INFO | Analyzing chunk frames = [1888256,1888768) | 20260505 08:47:27.233 | INFO | MemInfo: { | 20260505 08:47:27.233 | INFO | "gpu" : " 17 GB", | 20260505 08:47:27.233 | INFO | "huge" : "", | 20260505 08:47:27.233 | INFO | "huge_shared" : " 27 GB", | 20260505 08:47:27.233 | INFO | "malloc" : " 14 GB", | 20260505 08:47:27.233 | INFO | "pinned" : " 29 GB" | 20260505 08:47:27.233 | INFO | } | 20260505 08:47:27.233 | INFO | Perf report for the last 5.139 seconds: | 20260505 08:47:27.233 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:47:27.233 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:27.233 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:27.233 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:27.233 | INFO | ANALYSIS : 65.1%, 2.9, 99.3ms, 0 | 20260505 08:47:27.233 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:27.233 | INFO | RT_METRICS : 47.7%, 1.0, 25.0ms, 0 | 20260505 08:47:27.233 | INFO | PRE_HQ : 63.4%, 1.9, 61.4ms, 0 | 20260505 08:47:27.233 | INFO | BAZWRITER : 65.0%, 4.2, 140.1ms, 0 | 20260505 08:47:27.233 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 08:47:27.234 | INFO | Uploaded 36GiB to GPU in 2.25908s (15.9357GiB/s) | 20260505 08:47:27.234 | INFO | Downloaded 38GiB to GPU in 2.38071s (15.9616GiB/s) | 20260505 08:47:32.371 | INFO | Analyzing chunk frames = [1888768,1889280) | 20260505 08:47:32.371 | INFO | MemInfo: { | 20260505 08:47:32.371 | INFO | "gpu" : " 17 GB", | 20260505 08:47:32.371 | INFO | "huge" : "", | 20260505 08:47:32.371 | INFO | "huge_shared" : " 27 GB", | 20260505 08:47:32.371 | INFO | "malloc" : " 14 GB", | 20260505 08:47:32.371 | INFO | "pinned" : " 29 GB" | 20260505 08:47:32.372 | INFO | } | 20260505 08:47:32.372 | INFO | Perf report for the last 5.13832 seconds: | 20260505 08:47:32.372 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:47:32.372 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:32.372 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:32.372 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:32.372 | INFO | ANALYSIS : 64.4%, 3.0, 100.6ms, 0 | 20260505 08:47:32.372 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:32.372 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:32.372 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:47:32.372 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:32.372 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:32.372 | INFO | Uploaded 36GiB to GPU in 2.22617s (16.1713GiB/s) | 20260505 08:47:32.372 | INFO | Downloaded 34GiB to GPU in 2.08322s (16.3209GiB/s) | 20260505 08:47:37.512 | INFO | Analyzing chunk frames = [1889280,1889792) | 20260505 08:47:37.512 | INFO | MemInfo: { | 20260505 08:47:37.512 | INFO | "gpu" : " 17 GB", | 20260505 08:47:37.512 | INFO | "huge" : "", | 20260505 08:47:37.512 | INFO | "huge_shared" : " 27 GB", | 20260505 08:47:37.512 | INFO | "malloc" : " 14 GB", | 20260505 08:47:37.512 | INFO | "pinned" : " 29 GB" | 20260505 08:47:37.512 | INFO | } | 20260505 08:47:37.512 | INFO | Perf report for the last 5.14012 seconds: | 20260505 08:47:37.512 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:47:37.512 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:37.512 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:37.512 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:37.512 | INFO | ANALYSIS : 63.6%, 3.0, 99.0ms, 0 | 20260505 08:47:37.512 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:37.512 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:37.512 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:47:37.512 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:37.512 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:37.512 | INFO | Uploaded 36GiB to GPU in 2.18035s (16.5111GiB/s) | 20260505 08:47:37.512 | INFO | Downloaded 34GiB to GPU in 2.0702s (16.4236GiB/s) | 20260505 08:47:42.650 | INFO | Analyzing chunk frames = [1889792,1890304) | 20260505 08:47:42.650 | INFO | MemInfo: { | 20260505 08:47:42.650 | INFO | "gpu" : " 17 GB", | 20260505 08:47:42.650 | INFO | "huge" : "", | 20260505 08:47:42.650 | INFO | "huge_shared" : " 27 GB", | 20260505 08:47:42.650 | INFO | "malloc" : " 14 GB", | 20260505 08:47:42.650 | INFO | "pinned" : " 29 GB" | 20260505 08:47:42.650 | INFO | } | 20260505 08:47:42.650 | INFO | Perf report for the last 5.13844 seconds: | 20260505 08:47:42.650 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:47:42.650 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:42.650 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:42.650 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:42.650 | INFO | ANALYSIS : 63.6%, 3.0, 98.7ms, 0 | 20260505 08:47:42.650 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:42.650 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:42.650 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:47:42.650 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:42.650 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:42.650 | INFO | Uploaded 36GiB to GPU in 2.17394s (16.5598GiB/s) | 20260505 08:47:42.650 | INFO | Downloaded 34GiB to GPU in 2.06719s (16.4474GiB/s) | 20260505 08:47:47.789 | INFO | Analyzing chunk frames = [1890304,1890816) | 20260505 08:47:47.789 | INFO | MemInfo: { | 20260505 08:47:47.789 | INFO | "gpu" : " 17 GB", | 20260505 08:47:47.789 | INFO | "huge" : "", | 20260505 08:47:47.789 | INFO | "huge_shared" : " 27 GB", | 20260505 08:47:47.789 | INFO | "malloc" : " 14 GB", | 20260505 08:47:47.790 | INFO | "pinned" : " 29 GB" | 20260505 08:47:47.790 | INFO | } | 20260505 08:47:47.790 | INFO | Perf report for the last 5.13957 seconds: | 20260505 08:47:47.790 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:47:47.790 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:47.790 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:47.790 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:47.790 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 08:47:47.790 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:47.790 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:47.790 | INFO | PRE_HQ : 30.0%, 1.0, 15.5ms, 0 | 20260505 08:47:47.790 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:47.790 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:47.790 | INFO | Uploaded 36GiB to GPU in 2.18948s (16.4423GiB/s) | 20260505 08:47:47.790 | INFO | Downloaded 34GiB to GPU in 2.05883s (16.5142GiB/s) | 20260505 08:47:52.932 | INFO | Analyzing chunk frames = [1890816,1891328) | 20260505 08:47:52.932 | INFO | MemInfo: { | 20260505 08:47:52.932 | INFO | "gpu" : " 17 GB", | 20260505 08:47:52.932 | INFO | "huge" : "", | 20260505 08:47:52.932 | INFO | "huge_shared" : " 27 GB", | 20260505 08:47:52.932 | INFO | "malloc" : " 14 GB", | 20260505 08:47:52.932 | INFO | "pinned" : " 29 GB" | 20260505 08:47:52.932 | INFO | } | 20260505 08:47:52.933 | INFO | Perf report for the last 5.1429 seconds: | 20260505 08:47:52.933 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:47:52.933 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:52.933 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:52.933 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:52.933 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 08:47:52.933 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:52.933 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:52.933 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 08:47:52.933 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:52.933 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:52.933 | INFO | Uploaded 36GiB to GPU in 2.19093s (16.4313GiB/s) | 20260505 08:47:52.933 | INFO | Downloaded 34GiB to GPU in 2.05708s (16.5283GiB/s) | 20260505 08:47:58.069 | INFO | Analyzing chunk frames = [1891328,1891840) | 20260505 08:47:58.069 | INFO | MemInfo: { | 20260505 08:47:58.069 | INFO | "gpu" : " 17 GB", | 20260505 08:47:58.069 | INFO | "huge" : "", | 20260505 08:47:58.069 | INFO | "huge_shared" : " 27 GB", | 20260505 08:47:58.069 | INFO | "malloc" : " 14 GB", | 20260505 08:47:58.069 | INFO | "pinned" : " 29 GB" | 20260505 08:47:58.069 | INFO | } | 20260505 08:47:58.069 | INFO | Perf report for the last 5.13608 seconds: | 20260505 08:47:58.069 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:47:58.069 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:58.069 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:58.069 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:58.069 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 08:47:58.069 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:58.069 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:58.069 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260505 08:47:58.069 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:58.069 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:47:58.069 | INFO | Uploaded 36GiB to GPU in 2.16785s (16.6063GiB/s) | 20260505 08:47:58.069 | INFO | Downloaded 34GiB to GPU in 2.15807s (15.7548GiB/s) | 20260505 08:48:03.208 | INFO | Analyzing chunk frames = [1891840,1892352) | 20260505 08:48:03.209 | INFO | MemInfo: { | 20260505 08:48:03.209 | INFO | "gpu" : " 17 GB", | 20260505 08:48:03.209 | INFO | "huge" : "", | 20260505 08:48:03.209 | INFO | "huge_shared" : " 27 GB", | 20260505 08:48:03.209 | INFO | "malloc" : " 14 GB", | 20260505 08:48:03.209 | INFO | "pinned" : " 29 GB" | 20260505 08:48:03.209 | INFO | } | 20260505 08:48:03.209 | INFO | Perf report for the last 5.14007 seconds: | 20260505 08:48:03.209 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:48:03.209 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:03.209 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:03.209 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:03.209 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 08:48:03.209 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:03.209 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:03.209 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 08:48:03.209 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:03.209 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:03.209 | INFO | Uploaded 36GiB to GPU in 2.17861s (16.5243GiB/s) | 20260505 08:48:03.209 | INFO | Downloaded 34GiB to GPU in 2.03912s (16.6738GiB/s) | 20260505 08:48:03.209 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 08:48:03.209 | INFO | Total mem usage: 15.3 GiB | 20260505 08:48:03.209 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 08:48:03.209 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.35%) | 20260505 08:48:03.209 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 08:48:03.209 | INFO | High Water Breakdown: | 20260505 08:48:03.209 | INFO | 60.9% : 8.78 GiB ( 93.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 08:48:03.209 | INFO | 39% : 5.61 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 08:48:03.209 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 08:48:03.209 | INFO | ------------------------------------------ | 20260505 08:48:03.209 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:48:03.209 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 08:48:03.209 | INFO | Total mem usage: 26.5 GiB | 20260505 08:48:03.209 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 08:48:03.209 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 08:48:03.209 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 08:48:03.209 | INFO | High Water Breakdown: | 20260505 08:48:03.209 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:48:03.209 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:48:03.209 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:48:03.209 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:48:03.209 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:48:03.210 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:48:03.210 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 08:48:03.210 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 08:48:03.210 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:48:03.210 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 08:48:03.210 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 08:48:03.210 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 08:48:03.210 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 08:48:03.210 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 08:48:03.210 | INFO | ------------------------------------------ | 20260505 08:48:03.210 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:48:03.210 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 08:48:03.210 | INFO | Total mem usage: 25.6 GiB | 20260505 08:48:03.210 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 08:48:03.210 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 08:48:03.210 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 08:48:03.210 | INFO | High Water Breakdown: | 20260505 08:48:03.210 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 08:48:03.210 | INFO | ------------------------------------------ | 20260505 08:48:03.210 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:48:03.210 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 08:48:03.210 | INFO | Total mem usage: 15.7 GiB | 20260505 08:48:03.210 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 08:48:03.210 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 08:48:03.210 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 08:48:03.210 | INFO | High Water Breakdown: | 20260505 08:48:03.210 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 08:48:03.210 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 08:48:03.210 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 08:48:03.210 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 08:48:03.210 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 08:48:03.210 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 08:48:03.210 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 08:48:03.210 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 08:48:03.210 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 08:48:03.210 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:48:03.210 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 08:48:03.210 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:48:03.210 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 08:48:03.210 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 08:48:03.210 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 08:48:03.210 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 08:48:03.210 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:48:03.210 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 08:48:03.210 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:48:03.210 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:48:03.210 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:48:03.210 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 08:48:03.210 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 08:48:03.210 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:48:03.210 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 08:48:03.210 | INFO | ------------------------------------------ | 20260505 08:48:03.210 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:48:03.210 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 08:48:03.210 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:48:03.210 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:48:03.210 | INFO | PerfReport: Full Execution -- 3.25e+01 / 7.53e+01, 1.37e+01 / 1.60e+00 | 20260505 08:48:03.210 | INFO | PerfReport: 0.29%, AggregateStats -- 9.35e-02 / 1.04e-01, 1.09e-02 / 3.36e-03 | 20260505 08:48:03.210 | INFO | PerfReport: 11.65%, Baseline -- 3.79e+00 / 4.42e+00, 6.18e-01 / 3.37e-01 | 20260505 08:48:03.210 | INFO | PerfReport: 3.32%, Histogram -- 1.08e+00 / 1.48e+00, 2.10e-01 / 1.07e-01 | 20260505 08:48:03.210 | INFO | PerfReport: 15.48%, DME -- 4.03e+01 / 4.39e+01, 3.94e+00 / 1.19e+00 | 20260505 08:48:03.210 | INFO | PerfReport: 56.50%, FrameLabeling -- 1.84e+01 / 2.03e+01, 1.91e+00 / 1.72e+00 | 20260505 08:48:03.210 | INFO | PerfReport: 7.47%, PulseAccumulating -- 2.43e+00 / 2.98e+00, 4.14e-01 / 2.53e-01 | 20260505 08:48:03.210 | INFO | PerfReport: 3.90%, Metrics -- 1.27e+00 / 1.60e+00, 3.68e-01 / 1.68e-01 | 20260505 08:48:03.210 | INFO | PerfReport: 1.38%, PulseSerializing -- 4.50e-01 / 5.09e-01, 8.48e-02 / 3.21e-02 | 20260505 08:48:03.210 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 08:48:03.210 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:48:03.210 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:48:03.210 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.32e+01 / 0.00e+00 | 20260505 08:48:03.210 | INFO | PerfReport: 22.57%, Upload -- 2.29e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260505 08:48:03.210 | INFO | PerfReport: 21.73%, Download -- 2.21e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260505 08:48:03.210 | INFO | PerfReport: 32.06%, Compute -- 3.26e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 08:48:03.211 | INFO | PerfReport: 1.76%, LockUpload -- 1.78e+00 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260505 08:48:03.211 | INFO | PerfReport: 1.07%, LockDownload -- 1.09e+00 / 0.00e+00, 2.65e+00 / 0.00e+00 | 20260505 08:48:03.211 | INFO | PerfReport: 20.79%, LockCompute -- 2.11e+01 / 0.00e+00, 1.78e+01 / 0.00e+00 | 20260505 08:48:03.211 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.96e-02 / 0.00e+00 | 20260505 08:48:03.345 | INFO | Starting a round of Baz IO | 20260505 08:48:06.615 | INFO | Populating RTMetrics report | 20260505 08:48:06.615 | INFO | RTMetrics report populated | 20260505 08:48:06.615 | INFO | Opening tmp RTMetrics json file | 20260505 08:48:06.621 | INFO | Serializing RTMetrics to json | 20260505 08:48:06.622 | INFO | Serialization done, swapping tmp file now... | 20260505 08:48:06.623 | INFO | RTMetrics done | 20260505 08:48:06.739 | INFO | Finished a round of Baz IO: Wrote 2.446 GiB in 3.394 seconds | 20260505 08:48:06.739 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 27.04% / 0.02% / 72.94% | 20260505 08:48:06.739 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.38715% / 2.78155e-05% / 0.000103315% | 20260505 08:48:06.739 | INFO | Baz IO overall: | 20260505 08:48:06.739 | INFO | BytesWritten : 1077729109371 | 20260505 08:48:06.739 | INFO | NumEvents : 512522262696 | 20260505 08:48:08.348 | INFO | Analyzing chunk frames = [1892352,1892864) | 20260505 08:48:08.348 | INFO | MemInfo: { | 20260505 08:48:08.348 | INFO | "gpu" : " 17 GB", | 20260505 08:48:08.348 | INFO | "huge" : "", | 20260505 08:48:08.348 | INFO | "huge_shared" : " 27 GB", | 20260505 08:48:08.348 | INFO | "malloc" : " 14 GB", | 20260505 08:48:08.348 | INFO | "pinned" : " 29 GB" | 20260505 08:48:08.348 | INFO | } | 20260505 08:48:08.348 | INFO | Perf report for the last 5.13887 seconds: | 20260505 08:48:08.348 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:48:08.348 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:08.348 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:08.348 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:08.348 | INFO | ANALYSIS : 65.1%, 2.9, 99.1ms, 0 | 20260505 08:48:08.348 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:08.348 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260505 08:48:08.348 | INFO | PRE_HQ : 63.4%, 1.9, 61.4ms, 0 | 20260505 08:48:08.348 | INFO | BAZWRITER : 65.4%, 4.2, 141.8ms, 0 | 20260505 08:48:08.348 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 08:48:08.348 | INFO | Uploaded 36GiB to GPU in 2.26154s (15.9184GiB/s) | 20260505 08:48:08.348 | INFO | Downloaded 38GiB to GPU in 2.38213s (15.9521GiB/s) | 20260505 08:48:13.487 | INFO | Analyzing chunk frames = [1892864,1893376) | 20260505 08:48:13.487 | INFO | MemInfo: { | 20260505 08:48:13.487 | INFO | "gpu" : " 17 GB", | 20260505 08:48:13.487 | INFO | "huge" : "", | 20260505 08:48:13.487 | INFO | "huge_shared" : " 27 GB", | 20260505 08:48:13.487 | INFO | "malloc" : " 14 GB", | 20260505 08:48:13.487 | INFO | "pinned" : " 29 GB" | 20260505 08:48:13.487 | INFO | } | 20260505 08:48:13.487 | INFO | Perf report for the last 5.13931 seconds: | 20260505 08:48:13.487 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:48:13.487 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:13.487 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:13.487 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:13.487 | INFO | ANALYSIS : 64.5%, 3.0, 100.7ms, 0 | 20260505 08:48:13.487 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:13.487 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:13.487 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 08:48:13.487 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:13.487 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:13.487 | INFO | Uploaded 36GiB to GPU in 2.2264s (16.1696GiB/s) | 20260505 08:48:13.487 | INFO | Downloaded 34GiB to GPU in 2.07866s (16.3567GiB/s) | 20260505 08:48:18.625 | INFO | Analyzing chunk frames = [1893376,1893888) | 20260505 08:48:18.625 | INFO | MemInfo: { | 20260505 08:48:18.625 | INFO | "gpu" : " 17 GB", | 20260505 08:48:18.625 | INFO | "huge" : "", | 20260505 08:48:18.625 | INFO | "huge_shared" : " 27 GB", | 20260505 08:48:18.626 | INFO | "malloc" : " 14 GB", | 20260505 08:48:18.626 | INFO | "pinned" : " 29 GB" | 20260505 08:48:18.626 | INFO | } | 20260505 08:48:18.626 | INFO | Perf report for the last 5.13869 seconds: | 20260505 08:48:18.626 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:48:18.626 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:18.626 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:18.626 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:18.626 | INFO | ANALYSIS : 63.6%, 3.0, 99.0ms, 0 | 20260505 08:48:18.626 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:18.626 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:18.626 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 08:48:18.626 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:18.626 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:18.626 | INFO | Uploaded 36GiB to GPU in 2.17288s (16.5678GiB/s) | 20260505 08:48:18.626 | INFO | Downloaded 34GiB to GPU in 2.06698s (16.4491GiB/s) | 20260505 08:48:23.766 | INFO | Analyzing chunk frames = [1893888,1894400) | 20260505 08:48:23.766 | INFO | MemInfo: { | 20260505 08:48:23.766 | INFO | "gpu" : " 17 GB", | 20260505 08:48:23.766 | INFO | "huge" : "", | 20260505 08:48:23.766 | INFO | "huge_shared" : " 27 GB", | 20260505 08:48:23.766 | INFO | "malloc" : " 14 GB", | 20260505 08:48:23.766 | INFO | "pinned" : " 29 GB" | 20260505 08:48:23.766 | INFO | } | 20260505 08:48:23.766 | INFO | Perf report for the last 5.14021 seconds: | 20260505 08:48:23.766 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:48:23.766 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:23.766 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:23.766 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:23.766 | INFO | ANALYSIS : 63.5%, 3.0, 98.6ms, 0 | 20260505 08:48:23.766 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:23.766 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:23.766 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 08:48:23.766 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:23.766 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:23.766 | INFO | Uploaded 36GiB to GPU in 2.16647s (16.6169GiB/s) | 20260505 08:48:23.766 | INFO | Downloaded 34GiB to GPU in 2.06334s (16.4781GiB/s) | 20260505 08:48:28.906 | INFO | Analyzing chunk frames = [1894400,1894912) | 20260505 08:48:28.906 | INFO | MemInfo: { | 20260505 08:48:28.906 | INFO | "gpu" : " 17 GB", | 20260505 08:48:28.906 | INFO | "huge" : "", | 20260505 08:48:28.906 | INFO | "huge_shared" : " 27 GB", | 20260505 08:48:28.906 | INFO | "malloc" : " 14 GB", | 20260505 08:48:28.906 | INFO | "pinned" : " 29 GB" | 20260505 08:48:28.906 | INFO | } | 20260505 08:48:28.906 | INFO | Perf report for the last 5.14054 seconds: | 20260505 08:48:28.906 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:48:28.906 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:28.906 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:28.906 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:28.907 | INFO | ANALYSIS : 63.2%, 3.0, 98.8ms, 0 | 20260505 08:48:28.907 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:28.907 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:28.907 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:48:28.907 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:28.907 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:28.907 | INFO | Uploaded 36GiB to GPU in 2.17316s (16.5657GiB/s) | 20260505 08:48:28.907 | INFO | Downloaded 34GiB to GPU in 2.05932s (16.5103GiB/s) | 20260505 08:48:34.045 | INFO | Analyzing chunk frames = [1894912,1895424) | 20260505 08:48:34.045 | INFO | MemInfo: { | 20260505 08:48:34.045 | INFO | "gpu" : " 17 GB", | 20260505 08:48:34.045 | INFO | "huge" : "", | 20260505 08:48:34.045 | INFO | "huge_shared" : " 27 GB", | 20260505 08:48:34.045 | INFO | "malloc" : " 14 GB", | 20260505 08:48:34.045 | INFO | "pinned" : " 29 GB" | 20260505 08:48:34.045 | INFO | } | 20260505 08:48:34.045 | INFO | Perf report for the last 5.13883 seconds: | 20260505 08:48:34.045 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:48:34.045 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:34.045 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:34.045 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:34.045 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 08:48:34.045 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:34.045 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:34.045 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 08:48:34.046 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:34.046 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:34.046 | INFO | Uploaded 36GiB to GPU in 2.16207s (16.6507GiB/s) | 20260505 08:48:34.046 | INFO | Downloaded 34GiB to GPU in 2.0609s (16.4976GiB/s) | 20260505 08:48:39.183 | INFO | Analyzing chunk frames = [1895424,1895936) | 20260505 08:48:39.183 | INFO | MemInfo: { | 20260505 08:48:39.183 | INFO | "gpu" : " 17 GB", | 20260505 08:48:39.183 | INFO | "huge" : "", | 20260505 08:48:39.183 | INFO | "huge_shared" : " 27 GB", | 20260505 08:48:39.183 | INFO | "malloc" : " 14 GB", | 20260505 08:48:39.183 | INFO | "pinned" : " 29 GB" | 20260505 08:48:39.183 | INFO | } | 20260505 08:48:39.183 | INFO | Perf report for the last 5.13773 seconds: | 20260505 08:48:39.183 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:48:39.183 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:39.183 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:39.183 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:39.183 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 08:48:39.183 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:39.183 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:39.183 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260505 08:48:39.183 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:39.183 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:39.183 | INFO | Uploaded 36GiB to GPU in 2.19717s (16.3847GiB/s) | 20260505 08:48:39.183 | INFO | Downloaded 34GiB to GPU in 2.138s (15.9027GiB/s) | 20260505 08:48:44.323 | INFO | Analyzing chunk frames = [1895936,1896448) | 20260505 08:48:44.323 | INFO | MemInfo: { | 20260505 08:48:44.323 | INFO | "gpu" : " 17 GB", | 20260505 08:48:44.323 | INFO | "huge" : "", | 20260505 08:48:44.323 | INFO | "huge_shared" : " 27 GB", | 20260505 08:48:44.323 | INFO | "malloc" : " 14 GB", | 20260505 08:48:44.323 | INFO | "pinned" : " 29 GB" | 20260505 08:48:44.323 | INFO | } | 20260505 08:48:44.323 | INFO | Perf report for the last 5.14005 seconds: | 20260505 08:48:44.323 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:48:44.323 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:44.323 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:44.323 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:44.323 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 08:48:44.323 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:44.323 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:44.323 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 08:48:44.323 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:44.323 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:44.324 | INFO | Uploaded 36GiB to GPU in 2.1711s (16.5814GiB/s) | 20260505 08:48:44.324 | INFO | Downloaded 34GiB to GPU in 2.02864s (16.76GiB/s) | 20260505 08:48:44.462 | INFO | Starting a round of Baz IO | 20260505 08:48:47.742 | INFO | Populating RTMetrics report | 20260505 08:48:47.742 | INFO | RTMetrics report populated | 20260505 08:48:47.742 | INFO | Opening tmp RTMetrics json file | 20260505 08:48:47.749 | INFO | Serializing RTMetrics to json | 20260505 08:48:47.752 | INFO | Serialization done, swapping tmp file now... | 20260505 08:48:47.756 | INFO | RTMetrics done | 20260505 08:48:47.881 | INFO | Finished a round of Baz IO: Wrote 2.447 GiB in 3.419 seconds | 20260505 08:48:47.881 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 27.02% / 0.02% / 72.96% | 20260505 08:48:47.881 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.38367% / 2.38419e-05% / 5.96046e-05% | 20260505 08:48:47.881 | INFO | Baz IO overall: | 20260505 08:48:47.881 | INFO | BytesWritten : 1080356526087 | 20260505 08:48:47.881 | INFO | NumEvents : 513778862713 | 20260505 08:48:49.462 | INFO | Analyzing chunk frames = [1896448,1896960) | 20260505 08:48:49.462 | INFO | MemInfo: { | 20260505 08:48:49.462 | INFO | "gpu" : " 17 GB", | 20260505 08:48:49.462 | INFO | "huge" : "", | 20260505 08:48:49.462 | INFO | "huge_shared" : " 27 GB", | 20260505 08:48:49.462 | INFO | "malloc" : " 14 GB", | 20260505 08:48:49.462 | INFO | "pinned" : " 29 GB" | 20260505 08:48:49.462 | INFO | } | 20260505 08:48:49.462 | INFO | Perf report for the last 5.13892 seconds: | 20260505 08:48:49.462 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:48:49.462 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:49.462 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:49.462 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:49.462 | INFO | ANALYSIS : 65.4%, 2.9, 100.2ms, 0 | 20260505 08:48:49.462 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:49.462 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260505 08:48:49.462 | INFO | PRE_HQ : 63.2%, 1.9, 61.0ms, 0 | 20260505 08:48:49.462 | INFO | BAZWRITER : 65.8%, 4.1, 140.3ms, 0 | 20260505 08:48:49.462 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 08:48:49.462 | INFO | Uploaded 36GiB to GPU in 2.23041s (16.1405GiB/s) | 20260505 08:48:49.462 | INFO | Downloaded 38GiB to GPU in 2.34887s (16.178GiB/s) | 20260505 08:48:54.601 | INFO | Analyzing chunk frames = [1896960,1897472) | 20260505 08:48:54.601 | INFO | MemInfo: { | 20260505 08:48:54.601 | INFO | "gpu" : " 17 GB", | 20260505 08:48:54.601 | INFO | "huge" : "", | 20260505 08:48:54.601 | INFO | "huge_shared" : " 27 GB", | 20260505 08:48:54.601 | INFO | "malloc" : " 14 GB", | 20260505 08:48:54.601 | INFO | "pinned" : " 29 GB" | 20260505 08:48:54.601 | INFO | } | 20260505 08:48:54.601 | INFO | Perf report for the last 5.13904 seconds: | 20260505 08:48:54.601 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:48:54.601 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:54.601 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:54.601 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:54.601 | INFO | ANALYSIS : 64.4%, 3.0, 100.6ms, 0 | 20260505 08:48:54.601 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:54.601 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:54.601 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 08:48:54.601 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:54.601 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:54.601 | INFO | Uploaded 36GiB to GPU in 2.22668s (16.1676GiB/s) | 20260505 08:48:54.601 | INFO | Downloaded 34GiB to GPU in 2.09033s (16.2654GiB/s) | 20260505 08:48:59.741 | INFO | Analyzing chunk frames = [1897472,1897984) | 20260505 08:48:59.741 | INFO | MemInfo: { | 20260505 08:48:59.741 | INFO | "gpu" : " 17 GB", | 20260505 08:48:59.741 | INFO | "huge" : "", | 20260505 08:48:59.741 | INFO | "huge_shared" : " 27 GB", | 20260505 08:48:59.741 | INFO | "malloc" : " 14 GB", | 20260505 08:48:59.741 | INFO | "pinned" : " 29 GB" | 20260505 08:48:59.741 | INFO | } | 20260505 08:48:59.741 | INFO | Perf report for the last 5.13968 seconds: | 20260505 08:48:59.741 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:48:59.741 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:59.741 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:59.741 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:59.741 | INFO | ANALYSIS : 63.6%, 3.0, 98.9ms, 0 | 20260505 08:48:59.741 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:59.741 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:59.741 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:48:59.741 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:59.741 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:48:59.742 | INFO | Uploaded 36GiB to GPU in 2.16617s (16.6192GiB/s) | 20260505 08:48:59.742 | INFO | Downloaded 34GiB to GPU in 2.06648s (16.4531GiB/s) | 20260505 08:49:04.881 | INFO | Analyzing chunk frames = [1897984,1898496) | 20260505 08:49:04.881 | INFO | MemInfo: { | 20260505 08:49:04.881 | INFO | "gpu" : " 17 GB", | 20260505 08:49:04.881 | INFO | "huge" : "", | 20260505 08:49:04.881 | INFO | "huge_shared" : " 27 GB", | 20260505 08:49:04.881 | INFO | "malloc" : " 14 GB", | 20260505 08:49:04.881 | INFO | "pinned" : " 29 GB" | 20260505 08:49:04.881 | INFO | } | 20260505 08:49:04.881 | INFO | Perf report for the last 5.14024 seconds: | 20260505 08:49:04.881 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:49:04.881 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:04.881 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:04.881 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:04.881 | INFO | ANALYSIS : 63.4%, 3.0, 98.6ms, 0 | 20260505 08:49:04.881 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:04.881 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:04.881 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:49:04.881 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:04.881 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:04.882 | INFO | Uploaded 36GiB to GPU in 2.1696s (16.593GiB/s) | 20260505 08:49:04.882 | INFO | Downloaded 34GiB to GPU in 2.06531s (16.4624GiB/s) | 20260505 08:49:10.020 | INFO | Analyzing chunk frames = [1898496,1899008) | 20260505 08:49:10.020 | INFO | MemInfo: { | 20260505 08:49:10.020 | INFO | "gpu" : " 17 GB", | 20260505 08:49:10.020 | INFO | "huge" : "", | 20260505 08:49:10.020 | INFO | "huge_shared" : " 27 GB", | 20260505 08:49:10.020 | INFO | "malloc" : " 14 GB", | 20260505 08:49:10.020 | INFO | "pinned" : " 29 GB" | 20260505 08:49:10.020 | INFO | } | 20260505 08:49:10.020 | INFO | Perf report for the last 5.13914 seconds: | 20260505 08:49:10.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:49:10.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:10.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:10.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:10.020 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 08:49:10.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:10.020 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:10.020 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:49:10.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:10.020 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:10.021 | INFO | Uploaded 36GiB to GPU in 2.15911s (16.6735GiB/s) | 20260505 08:49:10.021 | INFO | Downloaded 34GiB to GPU in 2.06049s (16.501GiB/s) | 20260505 08:49:15.159 | INFO | Analyzing chunk frames = [1899008,1899520) | 20260505 08:49:15.159 | INFO | MemInfo: { | 20260505 08:49:15.159 | INFO | "gpu" : " 17 GB", | 20260505 08:49:15.160 | INFO | "huge" : "", | 20260505 08:49:15.160 | INFO | "huge_shared" : " 27 GB", | 20260505 08:49:15.160 | INFO | "malloc" : " 14 GB", | 20260505 08:49:15.160 | INFO | "pinned" : " 29 GB" | 20260505 08:49:15.160 | INFO | } | 20260505 08:49:15.160 | INFO | Perf report for the last 5.13951 seconds: | 20260505 08:49:15.160 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:49:15.160 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:15.160 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:15.160 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:15.160 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 08:49:15.160 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:15.160 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:15.160 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 08:49:15.160 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:15.160 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:15.160 | INFO | Uploaded 36GiB to GPU in 2.15724s (16.688GiB/s) | 20260505 08:49:15.160 | INFO | Downloaded 34GiB to GPU in 2.0576s (16.5241GiB/s) | 20260505 08:49:20.298 | INFO | Analyzing chunk frames = [1899520,1900032) | 20260505 08:49:20.298 | INFO | MemInfo: { | 20260505 08:49:20.298 | INFO | "gpu" : " 17 GB", | 20260505 08:49:20.298 | INFO | "huge" : "", | 20260505 08:49:20.298 | INFO | "huge_shared" : " 27 GB", | 20260505 08:49:20.298 | INFO | "malloc" : " 14 GB", | 20260505 08:49:20.298 | INFO | "pinned" : " 29 GB" | 20260505 08:49:20.298 | INFO | } | 20260505 08:49:20.298 | INFO | Perf report for the last 5.13807 seconds: | 20260505 08:49:20.298 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:49:20.298 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:20.298 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:20.298 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:20.298 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 08:49:20.298 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:20.298 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:20.298 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 08:49:20.298 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:20.298 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:20.298 | INFO | Uploaded 36GiB to GPU in 2.21277s (16.2692GiB/s) | 20260505 08:49:20.298 | INFO | Downloaded 34GiB to GPU in 2.1471s (15.8353GiB/s) | 20260505 08:49:25.437 | INFO | Analyzing chunk frames = [1900032,1900544) | 20260505 08:49:25.437 | INFO | MemInfo: { | 20260505 08:49:25.437 | INFO | "gpu" : " 17 GB", | 20260505 08:49:25.437 | INFO | "huge" : "", | 20260505 08:49:25.437 | INFO | "huge_shared" : " 27 GB", | 20260505 08:49:25.437 | INFO | "malloc" : " 14 GB", | 20260505 08:49:25.437 | INFO | "pinned" : " 29 GB" | 20260505 08:49:25.437 | INFO | } | 20260505 08:49:25.437 | INFO | Perf report for the last 5.13919 seconds: | 20260505 08:49:25.437 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:49:25.437 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:25.437 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:25.437 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:25.437 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 08:49:25.437 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:25.437 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:25.437 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:49:25.437 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:25.437 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:25.437 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 08:49:25.437 | INFO | Total mem usage: 15.3 GiB | 20260505 08:49:25.437 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 08:49:25.437 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -5.57%) | 20260505 08:49:25.437 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 08:49:25.437 | INFO | High Water Breakdown: | 20260505 08:49:25.437 | INFO | 61% : 8.86 GiB ( 94.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 08:49:25.437 | INFO | 38.8% : 5.64 GiB ( 84.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 08:49:25.437 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 08:49:25.437 | INFO | ------------------------------------------ | 20260505 08:49:25.437 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:49:25.437 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 08:49:25.437 | INFO | Total mem usage: 26.5 GiB | 20260505 08:49:25.437 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 08:49:25.437 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 08:49:25.437 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 08:49:25.437 | INFO | High Water Breakdown: | 20260505 08:49:25.437 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:49:25.437 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:49:25.437 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:49:25.437 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:49:25.437 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:49:25.437 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:49:25.437 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 08:49:25.437 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 08:49:25.437 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:49:25.437 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 08:49:25.437 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 08:49:25.437 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 08:49:25.437 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 08:49:25.437 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 08:49:25.437 | INFO | ------------------------------------------ | 20260505 08:49:25.437 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:49:25.437 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 08:49:25.437 | INFO | Total mem usage: 25.6 GiB | 20260505 08:49:25.437 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 08:49:25.437 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 08:49:25.437 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 08:49:25.437 | INFO | High Water Breakdown: | 20260505 08:49:25.437 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 08:49:25.437 | INFO | ------------------------------------------ | 20260505 08:49:25.437 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:49:25.437 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 08:49:25.437 | INFO | Total mem usage: 15.7 GiB | 20260505 08:49:25.437 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 08:49:25.437 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 08:49:25.437 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 08:49:25.437 | INFO | High Water Breakdown: | 20260505 08:49:25.437 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 08:49:25.437 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 08:49:25.437 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 08:49:25.437 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 08:49:25.437 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 08:49:25.437 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 08:49:25.437 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 08:49:25.437 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 08:49:25.437 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 08:49:25.437 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:49:25.437 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 08:49:25.437 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:49:25.437 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 08:49:25.437 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 08:49:25.437 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 08:49:25.437 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 08:49:25.437 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:49:25.437 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 08:49:25.437 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:49:25.437 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:49:25.437 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:49:25.437 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 08:49:25.437 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 08:49:25.437 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:49:25.437 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 08:49:25.437 | INFO | ------------------------------------------ | 20260505 08:49:25.437 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:49:25.437 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 08:49:25.437 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:49:25.437 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:49:25.437 | INFO | PerfReport: Full Execution -- 3.25e+01 / 7.52e+01, 1.34e+01 / 1.66e+00 | 20260505 08:49:25.437 | INFO | PerfReport: 0.29%, AggregateStats -- 9.32e-02 / 1.04e-01, 1.09e-02 / 4.21e-03 | 20260505 08:49:25.437 | INFO | PerfReport: 11.66%, Baseline -- 3.79e+00 / 4.44e+00, 6.22e-01 / 2.31e-01 | 20260505 08:49:25.437 | INFO | PerfReport: 3.31%, Histogram -- 1.08e+00 / 1.45e+00, 2.07e-01 / 1.00e-01 | 20260505 08:49:25.437 | INFO | PerfReport: 15.51%, DME -- 4.04e+01 / 4.42e+01, 4.09e+00 / 1.63e+00 | 20260505 08:49:25.437 | INFO | PerfReport: 56.43%, FrameLabeling -- 1.84e+01 / 1.99e+01, 1.88e+00 / 1.64e+00 | 20260505 08:49:25.437 | INFO | PerfReport: 7.52%, PulseAccumulating -- 2.45e+00 / 3.01e+00, 4.21e-01 / 2.36e-01 | 20260505 08:49:25.437 | INFO | PerfReport: 3.89%, Metrics -- 1.26e+00 / 1.58e+00, 3.58e-01 / 1.55e-01 | 20260505 08:49:25.437 | INFO | PerfReport: 1.38%, PulseSerializing -- 4.50e-01 / 5.08e-01, 8.47e-02 / 2.19e-02 | 20260505 08:49:25.437 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 08:49:25.437 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:49:25.437 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:49:25.437 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.28e+01 / 0.00e+00 | 20260505 08:49:25.437 | INFO | PerfReport: 22.47%, Upload -- 2.28e+01 / 0.00e+00, 8.18e+00 / 0.00e+00 | 20260505 08:49:25.437 | INFO | PerfReport: 21.64%, Download -- 2.20e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260505 08:49:25.437 | INFO | PerfReport: 32.05%, Compute -- 3.25e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 08:49:25.437 | INFO | PerfReport: 1.73%, LockUpload -- 1.75e+00 / 0.00e+00, 7.49e+00 / 0.00e+00 | 20260505 08:49:25.437 | INFO | PerfReport: 1.11%, LockDownload -- 1.13e+00 / 0.00e+00, 2.71e+00 / 0.00e+00 | 20260505 08:49:25.437 | INFO | PerfReport: 20.99%, LockCompute -- 2.13e+01 / 0.00e+00, 1.77e+01 / 0.00e+00 | 20260505 08:49:25.437 | INFO | PerfReport: 0.01%, Misc -- 1.46e-02 / 0.00e+00, 2.72e-02 / 0.00e+00 | 20260505 08:49:25.438 | INFO | Uploaded 36GiB to GPU in 2.1788s (16.5228GiB/s) | 20260505 08:49:25.438 | INFO | Downloaded 34GiB to GPU in 2.03446s (16.7121GiB/s) | 20260505 08:49:25.576 | INFO | Starting a round of Baz IO | 20260505 08:49:28.847 | INFO | Populating RTMetrics report | 20260505 08:49:28.847 | INFO | RTMetrics report populated | 20260505 08:49:28.849 | INFO | Opening tmp RTMetrics json file | 20260505 08:49:28.853 | INFO | Serializing RTMetrics to json | 20260505 08:49:28.874 | INFO | Serialization done, swapping tmp file now... | 20260505 08:49:28.874 | INFO | RTMetrics done | 20260505 08:49:28.993 | INFO | Finished a round of Baz IO: Wrote 2.447 GiB in 3.416 seconds | 20260505 08:49:28.993 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 27.00% / 0.02% / 72.98% | 20260505 08:49:28.993 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.37375% / 1.98682e-05% / 0.000111262% | 20260505 08:49:28.993 | INFO | Baz IO overall: | 20260505 08:49:28.993 | INFO | BytesWritten : 1082983481193 | 20260505 08:49:28.993 | INFO | NumEvents : 515035059487 | 20260505 08:49:30.577 | INFO | Analyzing chunk frames = [1900544,1901056) | 20260505 08:49:30.577 | INFO | MemInfo: { | 20260505 08:49:30.577 | INFO | "gpu" : " 17 GB", | 20260505 08:49:30.577 | INFO | "huge" : "", | 20260505 08:49:30.577 | INFO | "huge_shared" : " 27 GB", | 20260505 08:49:30.577 | INFO | "malloc" : " 14 GB", | 20260505 08:49:30.577 | INFO | "pinned" : " 29 GB" | 20260505 08:49:30.577 | INFO | } | 20260505 08:49:30.577 | INFO | Perf report for the last 5.14037 seconds: | 20260505 08:49:30.577 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:49:30.577 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:30.577 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:30.577 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:30.577 | INFO | ANALYSIS : 65.2%, 2.9, 99.6ms, 0 | 20260505 08:49:30.577 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:30.577 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260505 08:49:30.577 | INFO | PRE_HQ : 63.2%, 1.9, 60.8ms, 0 | 20260505 08:49:30.577 | INFO | BAZWRITER : 65.8%, 4.2, 141.5ms, 0 | 20260505 08:49:30.577 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260505 08:49:30.578 | INFO | Uploaded 36GiB to GPU in 2.27278s (15.8397GiB/s) | 20260505 08:49:30.578 | INFO | Downloaded 38GiB to GPU in 2.3852s (15.9316GiB/s) | 20260505 08:49:35.716 | INFO | Analyzing chunk frames = [1901056,1901568) | 20260505 08:49:35.716 | INFO | MemInfo: { | 20260505 08:49:35.716 | INFO | "gpu" : " 17 GB", | 20260505 08:49:35.716 | INFO | "huge" : "", | 20260505 08:49:35.716 | INFO | "huge_shared" : " 27 GB", | 20260505 08:49:35.716 | INFO | "malloc" : " 14 GB", | 20260505 08:49:35.716 | INFO | "pinned" : " 29 GB" | 20260505 08:49:35.716 | INFO | } | 20260505 08:49:35.716 | INFO | Perf report for the last 5.13885 seconds: | 20260505 08:49:35.716 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:49:35.716 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:35.716 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:35.716 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:35.716 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 08:49:35.717 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:35.717 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:35.717 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 08:49:35.717 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:35.717 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:35.717 | INFO | Uploaded 36GiB to GPU in 2.20515s (16.3254GiB/s) | 20260505 08:49:35.717 | INFO | Downloaded 34GiB to GPU in 2.09218s (16.251GiB/s) | 20260505 08:49:40.856 | INFO | Analyzing chunk frames = [1901568,1902080) | 20260505 08:49:40.856 | INFO | MemInfo: { | 20260505 08:49:40.856 | INFO | "gpu" : " 17 GB", | 20260505 08:49:40.856 | INFO | "huge" : "", | 20260505 08:49:40.856 | INFO | "huge_shared" : " 27 GB", | 20260505 08:49:40.856 | INFO | "malloc" : " 14 GB", | 20260505 08:49:40.856 | INFO | "pinned" : " 29 GB" | 20260505 08:49:40.856 | INFO | } | 20260505 08:49:40.856 | INFO | Perf report for the last 5.1395 seconds: | 20260505 08:49:40.856 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:49:40.856 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:40.856 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:40.856 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:40.856 | INFO | ANALYSIS : 63.6%, 3.0, 98.9ms, 0 | 20260505 08:49:40.856 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:40.856 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:40.856 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:49:40.856 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:40.856 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:40.856 | INFO | Uploaded 36GiB to GPU in 2.17443s (16.556GiB/s) | 20260505 08:49:40.856 | INFO | Downloaded 34GiB to GPU in 2.07221s (16.4076GiB/s) | 20260505 08:49:45.995 | INFO | Analyzing chunk frames = [1902080,1902592) | 20260505 08:49:45.995 | INFO | MemInfo: { | 20260505 08:49:45.995 | INFO | "gpu" : " 17 GB", | 20260505 08:49:45.995 | INFO | "huge" : "", | 20260505 08:49:45.995 | INFO | "huge_shared" : " 27 GB", | 20260505 08:49:45.995 | INFO | "malloc" : " 14 GB", | 20260505 08:49:45.995 | INFO | "pinned" : " 29 GB" | 20260505 08:49:45.995 | INFO | } | 20260505 08:49:45.995 | INFO | Perf report for the last 5.13965 seconds: | 20260505 08:49:45.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:49:45.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:45.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:45.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:45.995 | INFO | ANALYSIS : 63.5%, 3.0, 98.6ms, 0 | 20260505 08:49:45.995 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:45.995 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:45.996 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:49:45.996 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:45.996 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:45.996 | INFO | Uploaded 36GiB to GPU in 2.17342s (16.5638GiB/s) | 20260505 08:49:45.996 | INFO | Downloaded 34GiB to GPU in 2.06637s (16.454GiB/s) | 20260505 08:49:51.134 | INFO | Analyzing chunk frames = [1902592,1903104) | 20260505 08:49:51.134 | INFO | MemInfo: { | 20260505 08:49:51.134 | INFO | "gpu" : " 17 GB", | 20260505 08:49:51.134 | INFO | "huge" : "", | 20260505 08:49:51.134 | INFO | "huge_shared" : " 27 GB", | 20260505 08:49:51.134 | INFO | "malloc" : " 14 GB", | 20260505 08:49:51.135 | INFO | "pinned" : " 29 GB" | 20260505 08:49:51.135 | INFO | } | 20260505 08:49:51.135 | INFO | Perf report for the last 5.13918 seconds: | 20260505 08:49:51.135 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:49:51.135 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:51.135 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:51.135 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:51.135 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 08:49:51.135 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:51.135 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:51.135 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 08:49:51.135 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:51.135 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:51.135 | INFO | Uploaded 36GiB to GPU in 2.16364s (16.6386GiB/s) | 20260505 08:49:51.135 | INFO | Downloaded 34GiB to GPU in 2.0627s (16.4832GiB/s) | 20260505 08:49:56.274 | INFO | Analyzing chunk frames = [1903104,1903616) | 20260505 08:49:56.274 | INFO | MemInfo: { | 20260505 08:49:56.274 | INFO | "gpu" : " 17 GB", | 20260505 08:49:56.274 | INFO | "huge" : "", | 20260505 08:49:56.274 | INFO | "huge_shared" : " 27 GB", | 20260505 08:49:56.274 | INFO | "malloc" : " 14 GB", | 20260505 08:49:56.274 | INFO | "pinned" : " 29 GB" | 20260505 08:49:56.274 | INFO | } | 20260505 08:49:56.274 | INFO | Perf report for the last 5.13951 seconds: | 20260505 08:49:56.274 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:49:56.274 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:56.274 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:56.274 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:56.274 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 08:49:56.274 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:56.274 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:56.274 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:49:56.274 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:56.274 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:49:56.274 | INFO | Uploaded 36GiB to GPU in 2.24535s (16.0331GiB/s) | 20260505 08:49:56.274 | INFO | Downloaded 34GiB to GPU in 2.11151s (16.1022GiB/s) | 20260505 08:50:01.413 | INFO | Analyzing chunk frames = [1903616,1904128) | 20260505 08:50:01.413 | INFO | MemInfo: { | 20260505 08:50:01.413 | INFO | "gpu" : " 17 GB", | 20260505 08:50:01.413 | INFO | "huge" : "", | 20260505 08:50:01.413 | INFO | "huge_shared" : " 27 GB", | 20260505 08:50:01.413 | INFO | "malloc" : " 14 GB", | 20260505 08:50:01.413 | INFO | "pinned" : " 29 GB" | 20260505 08:50:01.413 | INFO | } | 20260505 08:50:01.413 | INFO | Perf report for the last 5.13919 seconds: | 20260505 08:50:01.413 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:50:01.413 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:01.413 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:01.413 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:01.413 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 08:50:01.413 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:01.413 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:01.413 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 08:50:01.413 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:01.413 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:01.414 | INFO | Uploaded 36GiB to GPU in 2.1886s (16.4488GiB/s) | 20260505 08:50:01.414 | INFO | Downloaded 34GiB to GPU in 2.15006s (15.8135GiB/s) | 20260505 08:50:06.553 | INFO | Analyzing chunk frames = [1904128,1904640) | 20260505 08:50:06.553 | INFO | MemInfo: { | 20260505 08:50:06.553 | INFO | "gpu" : " 17 GB", | 20260505 08:50:06.553 | INFO | "huge" : "", | 20260505 08:50:06.553 | INFO | "huge_shared" : " 27 GB", | 20260505 08:50:06.553 | INFO | "malloc" : " 14 GB", | 20260505 08:50:06.553 | INFO | "pinned" : " 29 GB" | 20260505 08:50:06.553 | INFO | } | 20260505 08:50:06.553 | INFO | Perf report for the last 5.13961 seconds: | 20260505 08:50:06.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:50:06.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:06.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:06.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:06.553 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 08:50:06.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:06.553 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:06.553 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 08:50:06.553 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:06.553 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:06.553 | INFO | Uploaded 36GiB to GPU in 2.19862s (16.3739GiB/s) | 20260505 08:50:06.553 | INFO | Downloaded 34GiB to GPU in 2.04521s (16.6242GiB/s) | 20260505 08:50:06.690 | INFO | Starting a round of Baz IO | 20260505 08:50:09.960 | INFO | Populating RTMetrics report | 20260505 08:50:09.960 | INFO | RTMetrics report populated | 20260505 08:50:09.960 | INFO | Opening tmp RTMetrics json file | 20260505 08:50:09.969 | INFO | Serializing RTMetrics to json | 20260505 08:50:09.970 | INFO | Serialization done, swapping tmp file now... | 20260505 08:50:09.971 | INFO | RTMetrics done | 20260505 08:50:10.134 | INFO | Finished a round of Baz IO: Wrote 2.448 GiB in 3.444 seconds | 20260505 08:50:10.134 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 26.99% / 0.02% / 73.00% | 20260505 08:50:10.134 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.37304% / 3.17891e-05% / 6.35783e-05% | 20260505 08:50:10.134 | INFO | Baz IO overall: | 20260505 08:50:10.135 | INFO | BytesWritten : 1085612231370 | 20260505 08:50:10.135 | INFO | NumEvents : 516292803475 | 20260505 08:50:11.691 | INFO | Analyzing chunk frames = [1904640,1905152) | 20260505 08:50:11.691 | INFO | MemInfo: { | 20260505 08:50:11.691 | INFO | "gpu" : " 17 GB", | 20260505 08:50:11.691 | INFO | "huge" : "", | 20260505 08:50:11.691 | INFO | "huge_shared" : " 27 GB", | 20260505 08:50:11.691 | INFO | "malloc" : " 14 GB", | 20260505 08:50:11.691 | INFO | "pinned" : " 29 GB" | 20260505 08:50:11.691 | INFO | } | 20260505 08:50:11.691 | INFO | Perf report for the last 5.13764 seconds: | 20260505 08:50:11.691 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:50:11.691 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:11.691 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:11.691 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:11.691 | INFO | ANALYSIS : 65.2%, 2.9, 99.5ms, 0 | 20260505 08:50:11.691 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:11.691 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260505 08:50:11.691 | INFO | PRE_HQ : 63.3%, 1.9, 61.0ms, 0 | 20260505 08:50:11.691 | INFO | BAZWRITER : 66.3%, 4.2, 143.0ms, 0 | 20260505 08:50:11.691 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 08:50:11.691 | INFO | Uploaded 36GiB to GPU in 2.27572s (15.8192GiB/s) | 20260505 08:50:11.691 | INFO | Downloaded 38GiB to GPU in 2.38349s (15.943GiB/s) | 20260505 08:50:16.831 | INFO | Analyzing chunk frames = [1905152,1905664) | 20260505 08:50:16.831 | INFO | MemInfo: { | 20260505 08:50:16.831 | INFO | "gpu" : " 17 GB", | 20260505 08:50:16.831 | INFO | "huge" : "", | 20260505 08:50:16.831 | INFO | "huge_shared" : " 27 GB", | 20260505 08:50:16.831 | INFO | "malloc" : " 14 GB", | 20260505 08:50:16.831 | INFO | "pinned" : " 29 GB" | 20260505 08:50:16.831 | INFO | } | 20260505 08:50:16.832 | INFO | Perf report for the last 5.14094 seconds: | 20260505 08:50:16.832 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:50:16.832 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:16.832 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:16.832 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:16.832 | INFO | ANALYSIS : 64.3%, 3.0, 100.5ms, 0 | 20260505 08:50:16.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:16.832 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:16.832 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:50:16.832 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:16.832 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:16.832 | INFO | Uploaded 36GiB to GPU in 2.20794s (16.3048GiB/s) | 20260505 08:50:16.832 | INFO | Downloaded 34GiB to GPU in 2.08725s (16.2894GiB/s) | 20260505 08:50:21.971 | INFO | Analyzing chunk frames = [1905664,1906176) | 20260505 08:50:21.971 | INFO | MemInfo: { | 20260505 08:50:21.971 | INFO | "gpu" : " 17 GB", | 20260505 08:50:21.971 | INFO | "huge" : "", | 20260505 08:50:21.971 | INFO | "huge_shared" : " 27 GB", | 20260505 08:50:21.971 | INFO | "malloc" : " 14 GB", | 20260505 08:50:21.971 | INFO | "pinned" : " 29 GB" | 20260505 08:50:21.971 | INFO | } | 20260505 08:50:21.971 | INFO | Perf report for the last 5.13919 seconds: | 20260505 08:50:21.971 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:50:21.971 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:21.971 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:21.971 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:21.971 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 08:50:21.971 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:21.971 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:21.971 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:50:21.971 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:21.971 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:21.971 | INFO | Uploaded 36GiB to GPU in 2.17051s (16.5859GiB/s) | 20260505 08:50:21.971 | INFO | Downloaded 34GiB to GPU in 2.0699s (16.4259GiB/s) | 20260505 08:50:27.110 | INFO | Analyzing chunk frames = [1906176,1906688) | 20260505 08:50:27.110 | INFO | MemInfo: { | 20260505 08:50:27.110 | INFO | "gpu" : " 17 GB", | 20260505 08:50:27.110 | INFO | "huge" : "", | 20260505 08:50:27.110 | INFO | "huge_shared" : " 27 GB", | 20260505 08:50:27.110 | INFO | "malloc" : " 14 GB", | 20260505 08:50:27.110 | INFO | "pinned" : " 29 GB" | 20260505 08:50:27.110 | INFO | } | 20260505 08:50:27.110 | INFO | Perf report for the last 5.13911 seconds: | 20260505 08:50:27.110 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:50:27.110 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:27.110 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:27.110 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:27.110 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260505 08:50:27.110 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:27.110 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:27.110 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:50:27.110 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:27.110 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:27.110 | INFO | Uploaded 36GiB to GPU in 2.18146s (16.5027GiB/s) | 20260505 08:50:27.110 | INFO | Downloaded 34GiB to GPU in 2.06839s (16.4379GiB/s) | 20260505 08:50:32.249 | INFO | Analyzing chunk frames = [1906688,1907200) | 20260505 08:50:32.249 | INFO | MemInfo: { | 20260505 08:50:32.249 | INFO | "gpu" : " 17 GB", | 20260505 08:50:32.249 | INFO | "huge" : "", | 20260505 08:50:32.249 | INFO | "huge_shared" : " 27 GB", | 20260505 08:50:32.249 | INFO | "malloc" : " 14 GB", | 20260505 08:50:32.249 | INFO | "pinned" : " 29 GB" | 20260505 08:50:32.249 | INFO | } | 20260505 08:50:32.249 | INFO | Perf report for the last 5.13949 seconds: | 20260505 08:50:32.249 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:50:32.249 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:32.249 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:32.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:32.250 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 08:50:32.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:32.250 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:32.250 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:50:32.250 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:32.250 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:32.250 | INFO | Uploaded 36GiB to GPU in 2.16207s (16.6507GiB/s) | 20260505 08:50:32.250 | INFO | Downloaded 34GiB to GPU in 2.06068s (16.4994GiB/s) | 20260505 08:50:37.389 | INFO | Analyzing chunk frames = [1907200,1907712) | 20260505 08:50:37.389 | INFO | MemInfo: { | 20260505 08:50:37.389 | INFO | "gpu" : " 17 GB", | 20260505 08:50:37.389 | INFO | "huge" : "", | 20260505 08:50:37.389 | INFO | "huge_shared" : " 27 GB", | 20260505 08:50:37.389 | INFO | "malloc" : " 14 GB", | 20260505 08:50:37.389 | INFO | "pinned" : " 29 GB" | 20260505 08:50:37.389 | INFO | } | 20260505 08:50:37.389 | INFO | Perf report for the last 5.13948 seconds: | 20260505 08:50:37.389 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:50:37.389 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:37.389 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:37.389 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:37.389 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 08:50:37.389 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:37.389 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:37.389 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 08:50:37.389 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:37.389 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:37.389 | INFO | Uploaded 36GiB to GPU in 2.19129s (16.4287GiB/s) | 20260505 08:50:37.389 | INFO | Downloaded 34GiB to GPU in 2.06137s (16.4939GiB/s) | 20260505 08:50:42.527 | INFO | Analyzing chunk frames = [1907712,1908224) | 20260505 08:50:42.527 | INFO | MemInfo: { | 20260505 08:50:42.527 | INFO | "gpu" : " 17 GB", | 20260505 08:50:42.527 | INFO | "huge" : "", | 20260505 08:50:42.527 | INFO | "huge_shared" : " 27 GB", | 20260505 08:50:42.527 | INFO | "malloc" : " 14 GB", | 20260505 08:50:42.528 | INFO | "pinned" : " 29 GB" | 20260505 08:50:42.528 | INFO | } | 20260505 08:50:42.528 | INFO | Perf report for the last 5.13875 seconds: | 20260505 08:50:42.528 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:50:42.528 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:42.528 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:42.528 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:42.528 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 08:50:42.528 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:42.528 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:42.528 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 08:50:42.528 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:42.528 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:42.528 | INFO | Uploaded 36GiB to GPU in 2.21932s (16.2212GiB/s) | 20260505 08:50:42.528 | INFO | Downloaded 34GiB to GPU in 2.14609s (15.8428GiB/s) | 20260505 08:50:47.667 | INFO | Analyzing chunk frames = [1908224,1908736) | 20260505 08:50:47.667 | INFO | MemInfo: { | 20260505 08:50:47.667 | INFO | "gpu" : " 17 GB", | 20260505 08:50:47.667 | INFO | "huge" : "", | 20260505 08:50:47.667 | INFO | "huge_shared" : " 27 GB", | 20260505 08:50:47.667 | INFO | "malloc" : " 14 GB", | 20260505 08:50:47.667 | INFO | "pinned" : " 29 GB" | 20260505 08:50:47.667 | INFO | } | 20260505 08:50:47.667 | INFO | Perf report for the last 5.13922 seconds: | 20260505 08:50:47.667 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:50:47.667 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:47.667 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:47.667 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:47.667 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 08:50:47.667 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:47.667 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:47.667 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 08:50:47.667 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:47.667 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:47.667 | INFO | Uploaded 36GiB to GPU in 2.17885s (16.5224GiB/s) | 20260505 08:50:47.667 | INFO | Downloaded 34GiB to GPU in 2.03518s (16.7061GiB/s) | 20260505 08:50:47.667 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 08:50:47.667 | INFO | Total mem usage: 15.3 GiB | 20260505 08:50:47.667 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 08:50:47.667 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -5.76%) | 20260505 08:50:47.667 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 08:50:47.667 | INFO | High Water Breakdown: | 20260505 08:50:47.667 | INFO | 61% : 8.83 GiB ( 94.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 08:50:47.667 | INFO | 38.9% : 5.64 GiB ( 84.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 08:50:47.667 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 08:50:47.667 | INFO | ------------------------------------------ | 20260505 08:50:47.667 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:50:47.668 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 08:50:47.668 | INFO | Total mem usage: 26.5 GiB | 20260505 08:50:47.668 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 08:50:47.668 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 08:50:47.668 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 08:50:47.668 | INFO | High Water Breakdown: | 20260505 08:50:47.668 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:50:47.668 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:50:47.668 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:50:47.668 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:50:47.668 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:50:47.668 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:50:47.668 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 08:50:47.668 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 08:50:47.668 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:50:47.668 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 08:50:47.668 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 08:50:47.668 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 08:50:47.668 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 08:50:47.668 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 08:50:47.668 | INFO | ------------------------------------------ | 20260505 08:50:47.668 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:50:47.668 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 08:50:47.668 | INFO | Total mem usage: 25.6 GiB | 20260505 08:50:47.668 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 08:50:47.668 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 08:50:47.668 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 08:50:47.668 | INFO | High Water Breakdown: | 20260505 08:50:47.668 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 08:50:47.668 | INFO | ------------------------------------------ | 20260505 08:50:47.668 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:50:47.668 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 08:50:47.668 | INFO | Total mem usage: 15.7 GiB | 20260505 08:50:47.668 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 08:50:47.668 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 08:50:47.668 | INFO | Unused Cache: 880 MiB (Waste: 5.48%) | 20260505 08:50:47.668 | INFO | High Water Breakdown: | 20260505 08:50:47.668 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 08:50:47.668 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 08:50:47.668 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 08:50:47.668 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 08:50:47.668 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 08:50:47.668 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 08:50:47.668 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 08:50:47.668 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 08:50:47.668 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 08:50:47.668 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:50:47.668 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 08:50:47.668 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:50:47.668 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 08:50:47.668 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 08:50:47.668 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 08:50:47.668 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 08:50:47.668 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:50:47.668 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 08:50:47.668 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:50:47.668 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:50:47.668 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:50:47.668 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 08:50:47.668 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 08:50:47.669 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:50:47.669 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 08:50:47.669 | INFO | ------------------------------------------ | 20260505 08:50:47.669 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:50:47.669 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 08:50:47.669 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:50:47.669 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:50:47.669 | INFO | PerfReport: Full Execution -- 3.25e+01 / 7.51e+01, 1.34e+01 / 1.61e+00 | 20260505 08:50:47.669 | INFO | PerfReport: 0.29%, AggregateStats -- 9.32e-02 / 1.05e-01, 1.11e-02 / 4.42e-03 | 20260505 08:50:47.669 | INFO | PerfReport: 11.66%, Baseline -- 3.79e+00 / 4.42e+00, 6.22e-01 / 3.18e-01 | 20260505 08:50:47.669 | INFO | PerfReport: 3.32%, Histogram -- 1.08e+00 / 1.46e+00, 2.08e-01 / 1.18e-01 | 20260505 08:50:47.669 | INFO | PerfReport: 15.49%, DME -- 4.03e+01 / 4.38e+01, 3.87e+00 / 1.19e+00 | 20260505 08:50:47.669 | INFO | PerfReport: 56.46%, FrameLabeling -- 1.84e+01 / 2.02e+01, 1.88e+00 / 1.87e+00 | 20260505 08:50:47.669 | INFO | PerfReport: 7.49%, PulseAccumulating -- 2.44e+00 / 3.00e+00, 4.18e-01 / 2.57e-01 | 20260505 08:50:47.669 | INFO | PerfReport: 3.91%, Metrics -- 1.27e+00 / 1.61e+00, 3.69e-01 / 1.78e-01 | 20260505 08:50:47.669 | INFO | PerfReport: 1.38%, PulseSerializing -- 4.49e-01 / 5.12e-01, 8.48e-02 / 2.62e-02 | 20260505 08:50:47.669 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 08:50:47.669 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:50:47.669 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:50:47.669 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.32e+01 / 0.00e+00 | 20260505 08:50:47.669 | INFO | PerfReport: 22.57%, Upload -- 2.29e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260505 08:50:47.669 | INFO | PerfReport: 21.73%, Download -- 2.21e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260505 08:50:47.669 | INFO | PerfReport: 32.04%, Compute -- 3.25e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 08:50:47.669 | INFO | PerfReport: 1.74%, LockUpload -- 1.77e+00 / 0.00e+00, 7.44e+00 / 0.00e+00 | 20260505 08:50:47.669 | INFO | PerfReport: 1.05%, LockDownload -- 1.06e+00 / 0.00e+00, 2.56e+00 / 0.00e+00 | 20260505 08:50:47.669 | INFO | PerfReport: 20.84%, LockCompute -- 2.12e+01 / 0.00e+00, 1.78e+01 / 0.00e+00 | 20260505 08:50:47.669 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 2.64e-02 / 0.00e+00 | 20260505 08:50:47.802 | INFO | Starting a round of Baz IO | 20260505 08:50:51.074 | INFO | Populating RTMetrics report | 20260505 08:50:51.075 | INFO | RTMetrics report populated | 20260505 08:50:51.075 | INFO | Opening tmp RTMetrics json file | 20260505 08:50:51.079 | INFO | Serializing RTMetrics to json | 20260505 08:50:51.102 | INFO | Serialization done, swapping tmp file now... | 20260505 08:50:51.103 | INFO | RTMetrics done | 20260505 08:50:51.215 | INFO | Finished a round of Baz IO: Wrote 2.446 GiB in 3.412 seconds | 20260505 08:50:51.215 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 26.97% / 0.02% / 73.01% | 20260505 08:50:51.215 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.3584% / 3.57628e-05% / 5.16574e-05% | 20260505 08:50:51.215 | INFO | Baz IO overall: | 20260505 08:50:51.215 | INFO | BytesWritten : 1088238538236 | 20260505 08:50:51.215 | INFO | NumEvents : 517549066778 | 20260505 08:50:52.806 | INFO | Analyzing chunk frames = [1908736,1909248) | 20260505 08:50:52.806 | INFO | MemInfo: { | 20260505 08:50:52.806 | INFO | "gpu" : " 17 GB", | 20260505 08:50:52.806 | INFO | "huge" : "", | 20260505 08:50:52.806 | INFO | "huge_shared" : " 27 GB", | 20260505 08:50:52.807 | INFO | "malloc" : " 14 GB", | 20260505 08:50:52.807 | INFO | "pinned" : " 29 GB" | 20260505 08:50:52.807 | INFO | } | 20260505 08:50:52.807 | INFO | Perf report for the last 5.13975 seconds: | 20260505 08:50:52.807 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:50:52.807 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:52.807 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:52.807 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:52.807 | INFO | ANALYSIS : 65.2%, 2.9, 99.6ms, 0 | 20260505 08:50:52.807 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:52.807 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260505 08:50:52.807 | INFO | PRE_HQ : 63.3%, 1.9, 60.8ms, 0 | 20260505 08:50:52.807 | INFO | BAZWRITER : 65.7%, 4.2, 140.1ms, 0 | 20260505 08:50:52.807 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 08:50:52.807 | INFO | Uploaded 36GiB to GPU in 2.27122s (15.8505GiB/s) | 20260505 08:50:52.807 | INFO | Downloaded 38GiB to GPU in 2.38771s (15.9148GiB/s) | 20260505 08:50:57.945 | INFO | Analyzing chunk frames = [1909248,1909760) | 20260505 08:50:57.945 | INFO | MemInfo: { | 20260505 08:50:57.945 | INFO | "gpu" : " 17 GB", | 20260505 08:50:57.945 | INFO | "huge" : "", | 20260505 08:50:57.945 | INFO | "huge_shared" : " 27 GB", | 20260505 08:50:57.945 | INFO | "malloc" : " 14 GB", | 20260505 08:50:57.945 | INFO | "pinned" : " 29 GB" | 20260505 08:50:57.945 | INFO | } | 20260505 08:50:57.945 | INFO | Perf report for the last 5.13847 seconds: | 20260505 08:50:57.945 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:50:57.945 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:57.945 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:57.945 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:57.945 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 08:50:57.945 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:57.945 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:57.945 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 08:50:57.945 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:57.945 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:50:57.945 | INFO | Uploaded 36GiB to GPU in 2.21256s (16.2707GiB/s) | 20260505 08:50:57.946 | INFO | Downloaded 34GiB to GPU in 2.07767s (16.3645GiB/s) | 20260505 08:51:03.085 | INFO | Analyzing chunk frames = [1909760,1910272) | 20260505 08:51:03.085 | INFO | MemInfo: { | 20260505 08:51:03.085 | INFO | "gpu" : " 17 GB", | 20260505 08:51:03.085 | INFO | "huge" : "", | 20260505 08:51:03.085 | INFO | "huge_shared" : " 27 GB", | 20260505 08:51:03.085 | INFO | "malloc" : " 14 GB", | 20260505 08:51:03.085 | INFO | "pinned" : " 29 GB" | 20260505 08:51:03.085 | INFO | } | 20260505 08:51:03.085 | INFO | Perf report for the last 5.14021 seconds: | 20260505 08:51:03.085 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:51:03.085 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:03.085 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:03.085 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:03.085 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260505 08:51:03.085 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:03.085 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:03.085 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:51:03.086 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:03.086 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:03.086 | INFO | Uploaded 36GiB to GPU in 2.171s (16.5822GiB/s) | 20260505 08:51:03.086 | INFO | Downloaded 34GiB to GPU in 2.06349s (16.4769GiB/s) | 20260505 08:51:08.225 | INFO | Analyzing chunk frames = [1910272,1910784) | 20260505 08:51:08.225 | INFO | MemInfo: { | 20260505 08:51:08.225 | INFO | "gpu" : " 17 GB", | 20260505 08:51:08.225 | INFO | "huge" : "", | 20260505 08:51:08.225 | INFO | "huge_shared" : " 27 GB", | 20260505 08:51:08.225 | INFO | "malloc" : " 14 GB", | 20260505 08:51:08.225 | INFO | "pinned" : " 29 GB" | 20260505 08:51:08.225 | INFO | } | 20260505 08:51:08.225 | INFO | Perf report for the last 5.13953 seconds: | 20260505 08:51:08.225 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:51:08.225 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:08.225 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:08.225 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:08.225 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260505 08:51:08.225 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:08.225 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:08.225 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:51:08.225 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:08.225 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:08.225 | INFO | Uploaded 36GiB to GPU in 2.1795s (16.5176GiB/s) | 20260505 08:51:08.225 | INFO | Downloaded 34GiB to GPU in 2.06434s (16.4702GiB/s) | 20260505 08:51:13.363 | INFO | Analyzing chunk frames = [1910784,1911296) | 20260505 08:51:13.363 | INFO | MemInfo: { | 20260505 08:51:13.363 | INFO | "gpu" : " 17 GB", | 20260505 08:51:13.363 | INFO | "huge" : "", | 20260505 08:51:13.363 | INFO | "huge_shared" : " 27 GB", | 20260505 08:51:13.363 | INFO | "malloc" : " 14 GB", | 20260505 08:51:13.363 | INFO | "pinned" : " 29 GB" | 20260505 08:51:13.363 | INFO | } | 20260505 08:51:13.364 | INFO | Perf report for the last 5.13858 seconds: | 20260505 08:51:13.364 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:51:13.364 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:13.364 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:13.364 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:13.364 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 08:51:13.364 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:13.364 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:13.364 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 08:51:13.364 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:13.364 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:13.364 | INFO | Uploaded 36GiB to GPU in 2.17118s (16.5809GiB/s) | 20260505 08:51:13.364 | INFO | Downloaded 34GiB to GPU in 2.06124s (16.4949GiB/s) | 20260505 08:51:18.505 | INFO | Analyzing chunk frames = [1911296,1911808) | 20260505 08:51:18.505 | INFO | MemInfo: { | 20260505 08:51:18.505 | INFO | "gpu" : " 17 GB", | 20260505 08:51:18.505 | INFO | "huge" : "", | 20260505 08:51:18.505 | INFO | "huge_shared" : " 27 GB", | 20260505 08:51:18.505 | INFO | "malloc" : " 14 GB", | 20260505 08:51:18.505 | INFO | "pinned" : " 29 GB" | 20260505 08:51:18.505 | INFO | } | 20260505 08:51:18.505 | INFO | Perf report for the last 5.14134 seconds: | 20260505 08:51:18.505 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:51:18.505 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:18.505 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:18.505 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:18.505 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 08:51:18.505 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:18.505 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:18.505 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:51:18.505 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:18.505 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:18.505 | INFO | Uploaded 36GiB to GPU in 2.18649s (16.4647GiB/s) | 20260505 08:51:18.505 | INFO | Downloaded 34GiB to GPU in 2.05929s (16.5105GiB/s) | 20260505 08:51:23.642 | INFO | Analyzing chunk frames = [1911808,1912320) | 20260505 08:51:23.642 | INFO | MemInfo: { | 20260505 08:51:23.642 | INFO | "gpu" : " 17 GB", | 20260505 08:51:23.643 | INFO | "huge" : "", | 20260505 08:51:23.643 | INFO | "huge_shared" : " 27 GB", | 20260505 08:51:23.643 | INFO | "malloc" : " 14 GB", | 20260505 08:51:23.643 | INFO | "pinned" : " 29 GB" | 20260505 08:51:23.643 | INFO | } | 20260505 08:51:23.643 | INFO | Perf report for the last 5.13784 seconds: | 20260505 08:51:23.643 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:51:23.643 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:23.643 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:23.643 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:23.643 | INFO | ANALYSIS : 63.8%, 3.0, 99.5ms, 0 | 20260505 08:51:23.643 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:23.643 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:23.643 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 08:51:23.643 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:23.643 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:23.643 | INFO | Uploaded 36GiB to GPU in 2.1906s (16.4338GiB/s) | 20260505 08:51:23.643 | INFO | Downloaded 34GiB to GPU in 2.14982s (15.8153GiB/s) | 20260505 08:51:28.782 | INFO | Analyzing chunk frames = [1912320,1912832) | 20260505 08:51:28.782 | INFO | MemInfo: { | 20260505 08:51:28.782 | INFO | "gpu" : " 17 GB", | 20260505 08:51:28.782 | INFO | "huge" : "", | 20260505 08:51:28.782 | INFO | "huge_shared" : " 27 GB", | 20260505 08:51:28.782 | INFO | "malloc" : " 14 GB", | 20260505 08:51:28.782 | INFO | "pinned" : " 29 GB" | 20260505 08:51:28.782 | INFO | } | 20260505 08:51:28.782 | INFO | Perf report for the last 5.13951 seconds: | 20260505 08:51:28.782 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:51:28.782 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:28.782 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:28.782 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:28.782 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 08:51:28.782 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:28.782 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:28.782 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 08:51:28.782 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:28.782 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:28.783 | INFO | Uploaded 36GiB to GPU in 2.16894s (16.598GiB/s) | 20260505 08:51:28.783 | INFO | Downloaded 34GiB to GPU in 2.02813s (16.7642GiB/s) | 20260505 08:51:28.920 | INFO | Starting a round of Baz IO | 20260505 08:51:32.198 | INFO | Populating RTMetrics report | 20260505 08:51:32.198 | INFO | RTMetrics report populated | 20260505 08:51:32.198 | INFO | Opening tmp RTMetrics json file | 20260505 08:51:32.212 | INFO | Serializing RTMetrics to json | 20260505 08:51:32.215 | INFO | Serialization done, swapping tmp file now... | 20260505 08:51:32.217 | INFO | RTMetrics done | 20260505 08:51:32.350 | INFO | Finished a round of Baz IO: Wrote 2.446 GiB in 3.43 seconds | 20260505 08:51:32.350 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 26.95% / 0.02% / 73.03% | 20260505 08:51:32.350 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.34972% / 4.37101e-05% / 5.96046e-05% | 20260505 08:51:32.351 | INFO | Baz IO overall: | 20260505 08:51:32.351 | INFO | BytesWritten : 1090864452484 | 20260505 08:51:32.351 | INFO | NumEvents : 518805025211 | 20260505 08:51:33.921 | INFO | Analyzing chunk frames = [1912832,1913344) | 20260505 08:51:33.921 | INFO | MemInfo: { | 20260505 08:51:33.921 | INFO | "gpu" : " 17 GB", | 20260505 08:51:33.921 | INFO | "huge" : "", | 20260505 08:51:33.921 | INFO | "huge_shared" : " 27 GB", | 20260505 08:51:33.921 | INFO | "malloc" : " 14 GB", | 20260505 08:51:33.921 | INFO | "pinned" : " 29 GB" | 20260505 08:51:33.921 | INFO | } | 20260505 08:51:33.921 | INFO | Perf report for the last 5.13852 seconds: | 20260505 08:51:33.921 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:51:33.921 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:33.921 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:33.921 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:33.921 | INFO | ANALYSIS : 65.3%, 2.9, 99.6ms, 0 | 20260505 08:51:33.921 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:33.921 | INFO | RT_METRICS : 46.2%, 1.0, 24.2ms, 0 | 20260505 08:51:33.921 | INFO | PRE_HQ : 63.5%, 1.9, 61.1ms, 0 | 20260505 08:51:33.921 | INFO | BAZWRITER : 66.1%, 4.2, 141.3ms, 0 | 20260505 08:51:33.921 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 08:51:33.921 | INFO | Uploaded 36GiB to GPU in 2.27253s (15.8414GiB/s) | 20260505 08:51:33.921 | INFO | Downloaded 38GiB to GPU in 2.37345s (16.0105GiB/s) | 20260505 08:51:39.061 | INFO | Analyzing chunk frames = [1913344,1913856) | 20260505 08:51:39.061 | INFO | MemInfo: { | 20260505 08:51:39.061 | INFO | "gpu" : " 17 GB", | 20260505 08:51:39.061 | INFO | "huge" : "", | 20260505 08:51:39.061 | INFO | "huge_shared" : " 27 GB", | 20260505 08:51:39.061 | INFO | "malloc" : " 14 GB", | 20260505 08:51:39.061 | INFO | "pinned" : " 29 GB" | 20260505 08:51:39.061 | INFO | } | 20260505 08:51:39.061 | INFO | Perf report for the last 5.13991 seconds: | 20260505 08:51:39.061 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:51:39.061 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:39.061 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:39.061 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:39.061 | INFO | ANALYSIS : 64.5%, 3.0, 100.8ms, 0 | 20260505 08:51:39.061 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:39.061 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:39.061 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 08:51:39.061 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:39.061 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:39.061 | INFO | Uploaded 36GiB to GPU in 2.24019s (16.0701GiB/s) | 20260505 08:51:39.061 | INFO | Downloaded 34GiB to GPU in 2.09371s (16.2391GiB/s) | 20260505 08:51:44.199 | INFO | Analyzing chunk frames = [1913856,1914368) | 20260505 08:51:44.199 | INFO | MemInfo: { | 20260505 08:51:44.199 | INFO | "gpu" : " 17 GB", | 20260505 08:51:44.199 | INFO | "huge" : "", | 20260505 08:51:44.199 | INFO | "huge_shared" : " 27 GB", | 20260505 08:51:44.199 | INFO | "malloc" : " 14 GB", | 20260505 08:51:44.199 | INFO | "pinned" : " 29 GB" | 20260505 08:51:44.199 | INFO | } | 20260505 08:51:44.199 | INFO | Perf report for the last 5.13864 seconds: | 20260505 08:51:44.199 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:51:44.199 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:44.199 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:44.199 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:44.199 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 08:51:44.199 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:44.199 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:44.200 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:51:44.200 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:44.200 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:44.200 | INFO | Uploaded 36GiB to GPU in 2.16943s (16.5942GiB/s) | 20260505 08:51:44.200 | INFO | Downloaded 34GiB to GPU in 2.06539s (16.4618GiB/s) | 20260505 08:51:49.338 | INFO | Analyzing chunk frames = [1914368,1914880) | 20260505 08:51:49.338 | INFO | MemInfo: { | 20260505 08:51:49.338 | INFO | "gpu" : " 17 GB", | 20260505 08:51:49.338 | INFO | "huge" : "", | 20260505 08:51:49.338 | INFO | "huge_shared" : " 27 GB", | 20260505 08:51:49.339 | INFO | "malloc" : " 14 GB", | 20260505 08:51:49.339 | INFO | "pinned" : " 29 GB" | 20260505 08:51:49.339 | INFO | } | 20260505 08:51:49.339 | INFO | Perf report for the last 5.13939 seconds: | 20260505 08:51:49.339 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:51:49.339 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:49.339 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:49.339 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:49.339 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260505 08:51:49.339 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:49.339 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:49.339 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 08:51:49.339 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:49.339 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:49.339 | INFO | Uploaded 36GiB to GPU in 2.17236s (16.5719GiB/s) | 20260505 08:51:49.339 | INFO | Downloaded 34GiB to GPU in 2.06248s (16.485GiB/s) | 20260505 08:51:54.479 | INFO | Analyzing chunk frames = [1914880,1915392) | 20260505 08:51:54.479 | INFO | MemInfo: { | 20260505 08:51:54.479 | INFO | "gpu" : " 17 GB", | 20260505 08:51:54.479 | INFO | "huge" : "", | 20260505 08:51:54.479 | INFO | "huge_shared" : " 27 GB", | 20260505 08:51:54.479 | INFO | "malloc" : " 14 GB", | 20260505 08:51:54.479 | INFO | "pinned" : " 29 GB" | 20260505 08:51:54.479 | INFO | } | 20260505 08:51:54.479 | INFO | Perf report for the last 5.14037 seconds: | 20260505 08:51:54.479 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:51:54.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:54.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:54.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:54.479 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 08:51:54.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:54.479 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:54.479 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:51:54.479 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:54.479 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:54.479 | INFO | Uploaded 36GiB to GPU in 2.16203s (16.651GiB/s) | 20260505 08:51:54.480 | INFO | Downloaded 34GiB to GPU in 2.06101s (16.4968GiB/s) | 20260505 08:51:59.619 | INFO | Analyzing chunk frames = [1915392,1915904) | 20260505 08:51:59.619 | INFO | MemInfo: { | 20260505 08:51:59.619 | INFO | "gpu" : " 17 GB", | 20260505 08:51:59.619 | INFO | "huge" : "", | 20260505 08:51:59.619 | INFO | "huge_shared" : " 27 GB", | 20260505 08:51:59.620 | INFO | "malloc" : " 14 GB", | 20260505 08:51:59.620 | INFO | "pinned" : " 29 GB" | 20260505 08:51:59.620 | INFO | } | 20260505 08:51:59.620 | INFO | Perf report for the last 5.14049 seconds: | 20260505 08:51:59.620 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:51:59.620 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:59.620 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:59.620 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:59.620 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 08:51:59.620 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:59.620 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:59.620 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:51:59.620 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:59.620 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:51:59.620 | INFO | Uploaded 36GiB to GPU in 2.16709s (16.6122GiB/s) | 20260505 08:51:59.620 | INFO | Downloaded 34GiB to GPU in 2.05553s (16.5407GiB/s) | 20260505 08:52:04.756 | INFO | Analyzing chunk frames = [1915904,1916416) | 20260505 08:52:04.756 | INFO | MemInfo: { | 20260505 08:52:04.756 | INFO | "gpu" : " 17 GB", | 20260505 08:52:04.756 | INFO | "huge" : "", | 20260505 08:52:04.756 | INFO | "huge_shared" : " 27 GB", | 20260505 08:52:04.757 | INFO | "malloc" : " 14 GB", | 20260505 08:52:04.757 | INFO | "pinned" : " 29 GB" | 20260505 08:52:04.757 | INFO | } | 20260505 08:52:04.757 | INFO | Perf report for the last 5.13706 seconds: | 20260505 08:52:04.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:52:04.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:04.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:04.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:04.757 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 08:52:04.757 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:04.757 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:04.757 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 08:52:04.757 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:04.757 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:04.757 | INFO | Uploaded 36GiB to GPU in 2.21759s (16.2338GiB/s) | 20260505 08:52:04.757 | INFO | Downloaded 34GiB to GPU in 2.15081s (15.808GiB/s) | 20260505 08:52:09.896 | INFO | Analyzing chunk frames = [1916416,1916928) | 20260505 08:52:09.896 | INFO | MemInfo: { | 20260505 08:52:09.896 | INFO | "gpu" : " 17 GB", | 20260505 08:52:09.897 | INFO | "huge" : "", | 20260505 08:52:09.897 | INFO | "huge_shared" : " 27 GB", | 20260505 08:52:09.897 | INFO | "malloc" : " 14 GB", | 20260505 08:52:09.897 | INFO | "pinned" : " 29 GB" | 20260505 08:52:09.897 | INFO | } | 20260505 08:52:09.897 | INFO | Perf report for the last 5.13995 seconds: | 20260505 08:52:09.897 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:52:09.897 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:09.897 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:09.897 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:09.897 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 08:52:09.897 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:09.897 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:09.897 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:52:09.897 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:09.897 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:09.897 | INFO | Uploaded 36GiB to GPU in 2.19974s (16.3655GiB/s) | 20260505 08:52:09.897 | INFO | Downloaded 34GiB to GPU in 2.04443s (16.6305GiB/s) | 20260505 08:52:09.897 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 08:52:09.897 | INFO | Total mem usage: 15.3 GiB | 20260505 08:52:09.897 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 08:52:09.897 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.33%) | 20260505 08:52:09.897 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 08:52:09.897 | INFO | High Water Breakdown: | 20260505 08:52:09.897 | INFO | 60.9% : 8.78 GiB ( 93.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 08:52:09.897 | INFO | 38.9% : 5.61 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 08:52:09.897 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 08:52:09.897 | INFO | ------------------------------------------ | 20260505 08:52:09.897 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:52:09.897 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 08:52:09.897 | INFO | Total mem usage: 26.5 GiB | 20260505 08:52:09.897 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 08:52:09.897 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 08:52:09.897 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 08:52:09.897 | INFO | High Water Breakdown: | 20260505 08:52:09.897 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:52:09.897 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:52:09.897 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:52:09.897 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:52:09.897 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:52:09.897 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:52:09.897 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 08:52:09.897 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 08:52:09.897 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:52:09.897 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 08:52:09.897 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 08:52:09.897 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 08:52:09.897 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 08:52:09.897 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 08:52:09.897 | INFO | ------------------------------------------ | 20260505 08:52:09.897 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:52:09.897 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 08:52:09.897 | INFO | Total mem usage: 25.6 GiB | 20260505 08:52:09.897 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 08:52:09.897 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 08:52:09.897 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 08:52:09.897 | INFO | High Water Breakdown: | 20260505 08:52:09.897 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 08:52:09.897 | INFO | ------------------------------------------ | 20260505 08:52:09.897 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:52:09.898 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 08:52:09.898 | INFO | Total mem usage: 15.7 GiB | 20260505 08:52:09.898 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 08:52:09.898 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 08:52:09.898 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 08:52:09.898 | INFO | High Water Breakdown: | 20260505 08:52:09.898 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 08:52:09.898 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 08:52:09.898 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 08:52:09.898 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 08:52:09.898 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 08:52:09.898 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 08:52:09.898 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 08:52:09.898 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 08:52:09.898 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 08:52:09.898 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:52:09.898 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 08:52:09.898 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:52:09.898 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 08:52:09.898 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 08:52:09.898 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 08:52:09.898 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 08:52:09.898 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:52:09.898 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 08:52:09.898 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:52:09.898 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:52:09.898 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:52:09.898 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 08:52:09.898 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 08:52:09.898 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:52:09.898 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 08:52:09.898 | INFO | ------------------------------------------ | 20260505 08:52:09.898 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:52:09.898 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 08:52:09.898 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:52:09.898 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:52:09.898 | INFO | PerfReport: Full Execution -- 3.25e+01 / 7.48e+01, 1.33e+01 / 1.63e+00 | 20260505 08:52:09.898 | INFO | PerfReport: 0.29%, AggregateStats -- 9.35e-02 / 1.04e-01, 1.10e-02 / 4.43e-03 | 20260505 08:52:09.898 | INFO | PerfReport: 11.66%, Baseline -- 3.79e+00 / 4.40e+00, 6.20e-01 / 3.15e-01 | 20260505 08:52:09.898 | INFO | PerfReport: 3.32%, Histogram -- 1.08e+00 / 1.45e+00, 2.08e-01 / 1.20e-01 | 20260505 08:52:09.898 | INFO | PerfReport: 15.49%, DME -- 4.03e+01 / 4.38e+01, 3.94e+00 / 1.40e+00 | 20260505 08:52:09.898 | INFO | PerfReport: 56.45%, FrameLabeling -- 1.84e+01 / 1.99e+01, 1.88e+00 / 1.89e+00 | 20260505 08:52:09.898 | INFO | PerfReport: 7.51%, PulseAccumulating -- 2.44e+00 / 3.02e+00, 4.19e-01 / 2.22e-01 | 20260505 08:52:09.898 | INFO | PerfReport: 3.90%, Metrics -- 1.27e+00 / 1.62e+00, 3.63e-01 / 1.74e-01 | 20260505 08:52:09.898 | INFO | PerfReport: 1.38%, PulseSerializing -- 4.50e-01 / 5.05e-01, 8.53e-02 / 2.24e-02 | 20260505 08:52:09.898 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 08:52:09.898 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:52:09.898 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:52:09.899 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.29e+01 / 0.00e+00 | 20260505 08:52:09.899 | INFO | PerfReport: 22.56%, Upload -- 2.29e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260505 08:52:09.899 | INFO | PerfReport: 21.69%, Download -- 2.20e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260505 08:52:09.899 | INFO | PerfReport: 32.04%, Compute -- 3.25e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260505 08:52:09.899 | INFO | PerfReport: 1.73%, LockUpload -- 1.76e+00 / 0.00e+00, 7.45e+00 / 0.00e+00 | 20260505 08:52:09.899 | INFO | PerfReport: 1.09%, LockDownload -- 1.10e+00 / 0.00e+00, 2.65e+00 / 0.00e+00 | 20260505 08:52:09.899 | INFO | PerfReport: 20.88%, LockCompute -- 2.12e+01 / 0.00e+00, 1.75e+01 / 0.00e+00 | 20260505 08:52:09.899 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.96e-02 / 0.00e+00 | 20260505 08:52:10.036 | INFO | Starting a round of Baz IO | 20260505 08:52:13.312 | INFO | Populating RTMetrics report | 20260505 08:52:13.312 | INFO | RTMetrics report populated | 20260505 08:52:13.312 | INFO | Opening tmp RTMetrics json file | 20260505 08:52:13.318 | INFO | Serializing RTMetrics to json | 20260505 08:52:13.336 | INFO | Serialization done, swapping tmp file now... | 20260505 08:52:13.337 | INFO | RTMetrics done | 20260505 08:52:13.443 | INFO | Finished a round of Baz IO: Wrote 2.447 GiB in 3.407 seconds | 20260505 08:52:13.443 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 26.93% / 0.02% / 73.05% | 20260505 08:52:13.443 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.34102% / 1.19209e-05% / 8.34465e-05% | 20260505 08:52:13.443 | INFO | Baz IO overall: | 20260505 08:52:13.443 | INFO | BytesWritten : 1093492000535 | 20260505 08:52:13.443 | INFO | NumEvents : 520062237990 | 20260505 08:52:15.036 | INFO | Analyzing chunk frames = [1916928,1917440) | 20260505 08:52:15.036 | INFO | MemInfo: { | 20260505 08:52:15.036 | INFO | "gpu" : " 17 GB", | 20260505 08:52:15.036 | INFO | "huge" : "", | 20260505 08:52:15.036 | INFO | "huge_shared" : " 27 GB", | 20260505 08:52:15.036 | INFO | "malloc" : " 14 GB", | 20260505 08:52:15.036 | INFO | "pinned" : " 29 GB" | 20260505 08:52:15.036 | INFO | } | 20260505 08:52:15.036 | INFO | Perf report for the last 5.1391 seconds: | 20260505 08:52:15.036 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:52:15.036 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:15.036 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:15.036 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:15.036 | INFO | ANALYSIS : 65.3%, 2.9, 100.0ms, 0 | 20260505 08:52:15.036 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:15.036 | INFO | RT_METRICS : 46.2%, 1.0, 24.3ms, 0 | 20260505 08:52:15.036 | INFO | PRE_HQ : 63.3%, 1.8, 60.6ms, 0 | 20260505 08:52:15.036 | INFO | BAZWRITER : 65.4%, 4.2, 140.0ms, 0 | 20260505 08:52:15.036 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 08:52:15.036 | INFO | Uploaded 36GiB to GPU in 2.2818s (15.777GiB/s) | 20260505 08:52:15.036 | INFO | Downloaded 38GiB to GPU in 2.37408s (16.0062GiB/s) | 20260505 08:52:20.175 | INFO | Analyzing chunk frames = [1917440,1917952) | 20260505 08:52:20.175 | INFO | MemInfo: { | 20260505 08:52:20.175 | INFO | "gpu" : " 17 GB", | 20260505 08:52:20.175 | INFO | "huge" : "", | 20260505 08:52:20.175 | INFO | "huge_shared" : " 27 GB", | 20260505 08:52:20.175 | INFO | "malloc" : " 14 GB", | 20260505 08:52:20.175 | INFO | "pinned" : " 29 GB" | 20260505 08:52:20.175 | INFO | } | 20260505 08:52:20.175 | INFO | Perf report for the last 5.13905 seconds: | 20260505 08:52:20.175 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:52:20.175 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:20.175 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:20.175 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:20.175 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 08:52:20.175 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:20.175 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:20.175 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:52:20.175 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:20.175 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:20.175 | INFO | Uploaded 36GiB to GPU in 2.22137s (16.2062GiB/s) | 20260505 08:52:20.175 | INFO | Downloaded 34GiB to GPU in 2.07035s (16.4224GiB/s) | 20260505 08:52:25.314 | INFO | Analyzing chunk frames = [1917952,1918464) | 20260505 08:52:25.314 | INFO | MemInfo: { | 20260505 08:52:25.314 | INFO | "gpu" : " 17 GB", | 20260505 08:52:25.314 | INFO | "huge" : "", | 20260505 08:52:25.314 | INFO | "huge_shared" : " 27 GB", | 20260505 08:52:25.314 | INFO | "malloc" : " 14 GB", | 20260505 08:52:25.314 | INFO | "pinned" : " 29 GB" | 20260505 08:52:25.314 | INFO | } | 20260505 08:52:25.314 | INFO | Perf report for the last 5.13912 seconds: | 20260505 08:52:25.314 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:52:25.314 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:25.314 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:25.314 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:25.314 | INFO | ANALYSIS : 63.6%, 3.0, 98.9ms, 0 | 20260505 08:52:25.314 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:25.314 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:25.314 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:52:25.314 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:25.314 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:25.314 | INFO | Uploaded 36GiB to GPU in 2.17034s (16.5873GiB/s) | 20260505 08:52:25.314 | INFO | Downloaded 34GiB to GPU in 2.06408s (16.4723GiB/s) | 20260505 08:52:30.454 | INFO | Analyzing chunk frames = [1918464,1918976) | 20260505 08:52:30.454 | INFO | MemInfo: { | 20260505 08:52:30.454 | INFO | "gpu" : " 17 GB", | 20260505 08:52:30.454 | INFO | "huge" : "", | 20260505 08:52:30.454 | INFO | "huge_shared" : " 27 GB", | 20260505 08:52:30.454 | INFO | "malloc" : " 14 GB", | 20260505 08:52:30.454 | INFO | "pinned" : " 29 GB" | 20260505 08:52:30.454 | INFO | } | 20260505 08:52:30.454 | INFO | Perf report for the last 5.14042 seconds: | 20260505 08:52:30.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:52:30.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:30.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:30.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:30.454 | INFO | ANALYSIS : 63.4%, 3.0, 98.6ms, 0 | 20260505 08:52:30.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:30.455 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:30.455 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:52:30.455 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:30.455 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:30.455 | INFO | Uploaded 36GiB to GPU in 2.16502s (16.628GiB/s) | 20260505 08:52:30.455 | INFO | Downloaded 34GiB to GPU in 2.06683s (16.4503GiB/s) | 20260505 08:52:35.594 | INFO | Analyzing chunk frames = [1918976,1919488) | 20260505 08:52:35.594 | INFO | MemInfo: { | 20260505 08:52:35.594 | INFO | "gpu" : " 17 GB", | 20260505 08:52:35.594 | INFO | "huge" : "", | 20260505 08:52:35.594 | INFO | "huge_shared" : " 27 GB", | 20260505 08:52:35.594 | INFO | "malloc" : " 14 GB", | 20260505 08:52:35.594 | INFO | "pinned" : " 29 GB" | 20260505 08:52:35.594 | INFO | } | 20260505 08:52:35.594 | INFO | Perf report for the last 5.13975 seconds: | 20260505 08:52:35.594 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:52:35.594 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:35.594 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:35.594 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:35.594 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 08:52:35.594 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:35.594 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:35.594 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:52:35.594 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:35.594 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:35.594 | INFO | Uploaded 36GiB to GPU in 2.16255s (16.647GiB/s) | 20260505 08:52:35.595 | INFO | Downloaded 34GiB to GPU in 2.0611s (16.496GiB/s) | 20260505 08:52:40.733 | INFO | Analyzing chunk frames = [1919488,1920000) | 20260505 08:52:40.733 | INFO | MemInfo: { | 20260505 08:52:40.733 | INFO | "gpu" : " 17 GB", | 20260505 08:52:40.733 | INFO | "huge" : "", | 20260505 08:52:40.734 | INFO | "huge_shared" : " 27 GB", | 20260505 08:52:40.734 | INFO | "malloc" : " 14 GB", | 20260505 08:52:40.734 | INFO | "pinned" : " 29 GB" | 20260505 08:52:40.734 | INFO | } | 20260505 08:52:40.734 | INFO | Perf report for the last 5.13956 seconds: | 20260505 08:52:40.734 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:52:40.734 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:40.734 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:40.734 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:40.734 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 08:52:40.734 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:40.734 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:40.734 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 08:52:40.734 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:40.734 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:40.734 | INFO | Uploaded 36GiB to GPU in 2.22972s (16.1455GiB/s) | 20260505 08:52:40.734 | INFO | Downloaded 34GiB to GPU in 2.10956s (16.1171GiB/s) | 20260505 08:52:45.872 | INFO | Analyzing chunk frames = [1920000,1920512) | 20260505 08:52:45.872 | INFO | MemInfo: { | 20260505 08:52:45.872 | INFO | "gpu" : " 17 GB", | 20260505 08:52:45.872 | INFO | "huge" : "", | 20260505 08:52:45.872 | INFO | "huge_shared" : " 27 GB", | 20260505 08:52:45.872 | INFO | "malloc" : " 14 GB", | 20260505 08:52:45.872 | INFO | "pinned" : " 29 GB" | 20260505 08:52:45.872 | INFO | } | 20260505 08:52:45.872 | INFO | Perf report for the last 5.13817 seconds: | 20260505 08:52:45.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:52:45.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:45.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:45.872 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:45.872 | INFO | ANALYSIS : 63.8%, 3.0, 99.5ms, 0 | 20260505 08:52:45.872 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:45.872 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:45.872 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 08:52:45.872 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:45.872 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:45.872 | INFO | Uploaded 36GiB to GPU in 2.17718s (16.5352GiB/s) | 20260505 08:52:45.872 | INFO | Downloaded 34GiB to GPU in 2.15813s (15.7543GiB/s) | 20260505 08:52:51.010 | INFO | Analyzing chunk frames = [1920512,1921024) | 20260505 08:52:51.010 | INFO | MemInfo: { | 20260505 08:52:51.010 | INFO | "gpu" : " 17 GB", | 20260505 08:52:51.010 | INFO | "huge" : "", | 20260505 08:52:51.010 | INFO | "huge_shared" : " 27 GB", | 20260505 08:52:51.010 | INFO | "malloc" : " 14 GB", | 20260505 08:52:51.011 | INFO | "pinned" : " 29 GB" | 20260505 08:52:51.011 | INFO | } | 20260505 08:52:51.011 | INFO | Perf report for the last 5.13872 seconds: | 20260505 08:52:51.011 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:52:51.011 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:51.011 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:51.011 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:51.011 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 08:52:51.011 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:51.011 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:51.011 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 08:52:51.011 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:51.011 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:51.011 | INFO | Uploaded 36GiB to GPU in 2.17467s (16.5543GiB/s) | 20260505 08:52:51.011 | INFO | Downloaded 34GiB to GPU in 2.03154s (16.7361GiB/s) | 20260505 08:52:51.150 | INFO | Starting a round of Baz IO | 20260505 08:52:54.418 | INFO | Populating RTMetrics report | 20260505 08:52:54.418 | INFO | RTMetrics report populated | 20260505 08:52:54.418 | INFO | Opening tmp RTMetrics json file | 20260505 08:52:54.451 | INFO | Serializing RTMetrics to json | 20260505 08:52:54.452 | INFO | Serialization done, swapping tmp file now... | 20260505 08:52:54.452 | INFO | RTMetrics done | 20260505 08:52:54.556 | INFO | Finished a round of Baz IO: Wrote 2.446 GiB in 3.405 seconds | 20260505 08:52:54.556 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 26.91% / 0.02% / 73.07% | 20260505 08:52:54.556 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.33184% / 2.78155e-05% / 7.15256e-05% | 20260505 08:52:54.556 | INFO | Baz IO overall: | 20260505 08:52:54.556 | INFO | BytesWritten : 1096118245366 | 20260505 08:52:54.556 | INFO | NumEvents : 521318547783 | 20260505 08:52:56.150 | INFO | Analyzing chunk frames = [1921024,1921536) | 20260505 08:52:56.150 | INFO | MemInfo: { | 20260505 08:52:56.150 | INFO | "gpu" : " 17 GB", | 20260505 08:52:56.150 | INFO | "huge" : "", | 20260505 08:52:56.150 | INFO | "huge_shared" : " 27 GB", | 20260505 08:52:56.150 | INFO | "malloc" : " 14 GB", | 20260505 08:52:56.150 | INFO | "pinned" : " 29 GB" | 20260505 08:52:56.150 | INFO | } | 20260505 08:52:56.150 | INFO | Perf report for the last 5.13936 seconds: | 20260505 08:52:56.150 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:52:56.150 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:56.150 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:56.150 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:56.150 | INFO | ANALYSIS : 65.2%, 2.9, 99.4ms, 0 | 20260505 08:52:56.150 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:52:56.150 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260505 08:52:56.150 | INFO | PRE_HQ : 63.2%, 1.8, 60.5ms, 0 | 20260505 08:52:56.150 | INFO | BAZWRITER : 65.6%, 4.2, 141.3ms, 0 | 20260505 08:52:56.150 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 08:52:56.150 | INFO | Uploaded 36GiB to GPU in 2.27075s (15.8538GiB/s) | 20260505 08:52:56.150 | INFO | Downloaded 38GiB to GPU in 2.38954s (15.9026GiB/s) | 20260505 08:53:01.289 | INFO | Analyzing chunk frames = [1921536,1922048) | 20260505 08:53:01.289 | INFO | MemInfo: { | 20260505 08:53:01.289 | INFO | "gpu" : " 17 GB", | 20260505 08:53:01.289 | INFO | "huge" : "", | 20260505 08:53:01.289 | INFO | "huge_shared" : " 27 GB", | 20260505 08:53:01.289 | INFO | "malloc" : " 14 GB", | 20260505 08:53:01.289 | INFO | "pinned" : " 29 GB" | 20260505 08:53:01.289 | INFO | } | 20260505 08:53:01.290 | INFO | Perf report for the last 5.1395 seconds: | 20260505 08:53:01.290 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:53:01.290 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:01.290 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:01.290 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:01.290 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 08:53:01.290 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:01.290 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:01.290 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 08:53:01.290 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:01.290 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:01.290 | INFO | Uploaded 36GiB to GPU in 2.22793s (16.1585GiB/s) | 20260505 08:53:01.290 | INFO | Downloaded 34GiB to GPU in 2.08617s (16.2978GiB/s) | 20260505 08:53:06.430 | INFO | Analyzing chunk frames = [1922048,1922560) | 20260505 08:53:06.430 | INFO | MemInfo: { | 20260505 08:53:06.430 | INFO | "gpu" : " 17 GB", | 20260505 08:53:06.430 | INFO | "huge" : "", | 20260505 08:53:06.430 | INFO | "huge_shared" : " 27 GB", | 20260505 08:53:06.430 | INFO | "malloc" : " 14 GB", | 20260505 08:53:06.430 | INFO | "pinned" : " 29 GB" | 20260505 08:53:06.430 | INFO | } | 20260505 08:53:06.430 | INFO | Perf report for the last 5.14026 seconds: | 20260505 08:53:06.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:53:06.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:06.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:06.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:06.430 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 08:53:06.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:06.430 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:06.430 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:53:06.430 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:06.430 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:06.430 | INFO | Uploaded 36GiB to GPU in 2.16661s (16.6158GiB/s) | 20260505 08:53:06.430 | INFO | Downloaded 34GiB to GPU in 2.06886s (16.4342GiB/s) | 20260505 08:53:11.570 | INFO | Analyzing chunk frames = [1922560,1923072) | 20260505 08:53:11.570 | INFO | MemInfo: { | 20260505 08:53:11.570 | INFO | "gpu" : " 17 GB", | 20260505 08:53:11.570 | INFO | "huge" : "", | 20260505 08:53:11.570 | INFO | "huge_shared" : " 27 GB", | 20260505 08:53:11.570 | INFO | "malloc" : " 14 GB", | 20260505 08:53:11.570 | INFO | "pinned" : " 29 GB" | 20260505 08:53:11.570 | INFO | } | 20260505 08:53:11.570 | INFO | Perf report for the last 5.14005 seconds: | 20260505 08:53:11.570 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:53:11.570 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:11.570 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:11.570 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:11.570 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 08:53:11.570 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:11.570 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:11.570 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 08:53:11.570 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:11.570 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:11.570 | INFO | Uploaded 36GiB to GPU in 2.15792s (16.6827GiB/s) | 20260505 08:53:11.570 | INFO | Downloaded 34GiB to GPU in 2.06804s (16.4407GiB/s) | 20260505 08:53:16.708 | INFO | Analyzing chunk frames = [1923072,1923584) | 20260505 08:53:16.708 | INFO | MemInfo: { | 20260505 08:53:16.708 | INFO | "gpu" : " 17 GB", | 20260505 08:53:16.708 | INFO | "huge" : "", | 20260505 08:53:16.708 | INFO | "huge_shared" : " 27 GB", | 20260505 08:53:16.708 | INFO | "malloc" : " 14 GB", | 20260505 08:53:16.708 | INFO | "pinned" : " 29 GB" | 20260505 08:53:16.708 | INFO | } | 20260505 08:53:16.708 | INFO | Perf report for the last 5.13778 seconds: | 20260505 08:53:16.708 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:53:16.708 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:16.708 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:16.708 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:16.708 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 08:53:16.708 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:16.708 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:16.708 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:53:16.708 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:16.708 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:16.708 | INFO | Uploaded 36GiB to GPU in 2.23684s (16.0941GiB/s) | 20260505 08:53:16.708 | INFO | Downloaded 34GiB to GPU in 2.11078s (16.1078GiB/s) | 20260505 08:53:21.847 | INFO | Analyzing chunk frames = [1923584,1924096) | 20260505 08:53:21.847 | INFO | MemInfo: { | 20260505 08:53:21.847 | INFO | "gpu" : " 17 GB", | 20260505 08:53:21.847 | INFO | "huge" : "", | 20260505 08:53:21.847 | INFO | "huge_shared" : " 27 GB", | 20260505 08:53:21.847 | INFO | "malloc" : " 14 GB", | 20260505 08:53:21.847 | INFO | "pinned" : " 29 GB" | 20260505 08:53:21.847 | INFO | } | 20260505 08:53:21.848 | INFO | Perf report for the last 5.13987 seconds: | 20260505 08:53:21.848 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:53:21.848 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:21.848 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:21.848 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:21.848 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 08:53:21.848 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:21.848 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:21.848 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:53:21.848 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:21.848 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:21.848 | INFO | Uploaded 36GiB to GPU in 2.16528s (16.626GiB/s) | 20260505 08:53:21.848 | INFO | Downloaded 34GiB to GPU in 2.0625s (16.4848GiB/s) | 20260505 08:53:26.986 | INFO | Analyzing chunk frames = [1924096,1924608) | 20260505 08:53:26.986 | INFO | MemInfo: { | 20260505 08:53:26.986 | INFO | "gpu" : " 17 GB", | 20260505 08:53:26.986 | INFO | "huge" : "", | 20260505 08:53:26.987 | INFO | "huge_shared" : " 27 GB", | 20260505 08:53:26.987 | INFO | "malloc" : " 14 GB", | 20260505 08:53:26.987 | INFO | "pinned" : " 29 GB" | 20260505 08:53:26.987 | INFO | } | 20260505 08:53:26.987 | INFO | Perf report for the last 5.13911 seconds: | 20260505 08:53:26.987 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:53:26.987 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:26.987 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:26.987 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:26.987 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 08:53:26.987 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:26.987 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:26.987 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 08:53:26.987 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:26.987 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:26.987 | INFO | Uploaded 36GiB to GPU in 2.19639s (16.3905GiB/s) | 20260505 08:53:26.987 | INFO | Downloaded 34GiB to GPU in 2.15179s (15.8008GiB/s) | 20260505 08:53:32.126 | INFO | Analyzing chunk frames = [1924608,1925120) | 20260505 08:53:32.126 | INFO | MemInfo: { | 20260505 08:53:32.126 | INFO | "gpu" : " 17 GB", | 20260505 08:53:32.126 | INFO | "huge" : "", | 20260505 08:53:32.126 | INFO | "huge_shared" : " 27 GB", | 20260505 08:53:32.126 | INFO | "malloc" : " 14 GB", | 20260505 08:53:32.126 | INFO | "pinned" : " 29 GB" | 20260505 08:53:32.126 | INFO | } | 20260505 08:53:32.126 | INFO | Perf report for the last 5.13957 seconds: | 20260505 08:53:32.126 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:53:32.126 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:32.126 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:32.126 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:32.126 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 08:53:32.126 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:32.126 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:32.126 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 08:53:32.126 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:32.126 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:32.127 | INFO | Uploaded 36GiB to GPU in 2.1772s (16.535GiB/s) | 20260505 08:53:32.127 | INFO | Downloaded 34GiB to GPU in 2.03515s (16.7064GiB/s) | 20260505 08:53:32.127 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 08:53:32.127 | INFO | Total mem usage: 15.3 GiB | 20260505 08:53:32.127 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 08:53:32.127 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -5.6%) | 20260505 08:53:32.127 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 08:53:32.127 | INFO | High Water Breakdown: | 20260505 08:53:32.127 | INFO | 61% : 8.86 GiB ( 94.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 08:53:32.127 | INFO | 38.8% : 5.64 GiB ( 84.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 08:53:32.127 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 08:53:32.127 | INFO | ------------------------------------------ | 20260505 08:53:32.127 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:53:32.127 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 08:53:32.127 | INFO | Total mem usage: 26.5 GiB | 20260505 08:53:32.127 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 08:53:32.127 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 08:53:32.127 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 08:53:32.127 | INFO | High Water Breakdown: | 20260505 08:53:32.127 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:53:32.127 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:53:32.127 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:53:32.127 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:53:32.127 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:53:32.127 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:53:32.127 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 08:53:32.127 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 08:53:32.127 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:53:32.127 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 08:53:32.127 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 08:53:32.127 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 08:53:32.127 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 08:53:32.127 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 08:53:32.127 | INFO | ------------------------------------------ | 20260505 08:53:32.127 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:53:32.127 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 08:53:32.127 | INFO | Total mem usage: 25.6 GiB | 20260505 08:53:32.127 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 08:53:32.127 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 08:53:32.127 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 08:53:32.127 | INFO | High Water Breakdown: | 20260505 08:53:32.127 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 08:53:32.127 | INFO | ------------------------------------------ | 20260505 08:53:32.127 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:53:32.127 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 08:53:32.127 | INFO | Total mem usage: 15.7 GiB | 20260505 08:53:32.127 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 08:53:32.127 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 08:53:32.127 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 08:53:32.127 | INFO | High Water Breakdown: | 20260505 08:53:32.127 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 08:53:32.127 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 08:53:32.127 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 08:53:32.128 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 08:53:32.128 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 08:53:32.128 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 08:53:32.128 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 08:53:32.128 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 08:53:32.128 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 08:53:32.128 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:53:32.128 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 08:53:32.128 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:53:32.128 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 08:53:32.128 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 08:53:32.128 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 08:53:32.128 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 08:53:32.128 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:53:32.128 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 08:53:32.128 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:53:32.128 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:53:32.128 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:53:32.128 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 08:53:32.128 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 08:53:32.128 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:53:32.128 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 08:53:32.128 | INFO | ------------------------------------------ | 20260505 08:53:32.128 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:53:32.128 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 08:53:32.128 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:53:32.128 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:53:32.128 | INFO | PerfReport: Full Execution -- 3.25e+01 / 7.53e+01, 1.37e+01 / 1.65e+00 | 20260505 08:53:32.128 | INFO | PerfReport: 0.29%, AggregateStats -- 9.31e-02 / 1.05e-01, 1.04e-02 / 3.06e-03 | 20260505 08:53:32.128 | INFO | PerfReport: 11.69%, Baseline -- 3.80e+00 / 4.45e+00, 6.26e-01 / 2.72e-01 | 20260505 08:53:32.128 | INFO | PerfReport: 3.31%, Histogram -- 1.08e+00 / 1.46e+00, 2.09e-01 / 1.12e-01 | 20260505 08:53:32.128 | INFO | PerfReport: 15.49%, DME -- 4.03e+01 / 4.37e+01, 3.85e+00 / 1.38e+00 | 20260505 08:53:32.128 | INFO | PerfReport: 56.45%, FrameLabeling -- 1.84e+01 / 2.05e+01, 1.87e+00 / 1.67e+00 | 20260505 08:53:32.128 | INFO | PerfReport: 7.50%, PulseAccumulating -- 2.44e+00 / 2.96e+00, 4.19e-01 / 2.81e-01 | 20260505 08:53:32.128 | INFO | PerfReport: 3.89%, Metrics -- 1.27e+00 / 1.60e+00, 3.63e-01 / 1.57e-01 | 20260505 08:53:32.128 | INFO | PerfReport: 1.38%, PulseSerializing -- 4.48e-01 / 5.07e-01, 8.42e-02 / 2.90e-02 | 20260505 08:53:32.128 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 08:53:32.128 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:53:32.128 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:53:32.128 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.33e+01 / 0.00e+00 | 20260505 08:53:32.128 | INFO | PerfReport: 22.56%, Upload -- 2.29e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260505 08:53:32.128 | INFO | PerfReport: 21.74%, Download -- 2.21e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260505 08:53:32.128 | INFO | PerfReport: 32.01%, Compute -- 3.25e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 08:53:32.128 | INFO | PerfReport: 1.78%, LockUpload -- 1.81e+00 / 0.00e+00, 7.55e+00 / 0.00e+00 | 20260505 08:53:32.128 | INFO | PerfReport: 1.12%, LockDownload -- 1.13e+00 / 0.00e+00, 2.71e+00 / 0.00e+00 | 20260505 08:53:32.128 | INFO | PerfReport: 20.78%, LockCompute -- 2.11e+01 / 0.00e+00, 1.77e+01 / 0.00e+00 | 20260505 08:53:32.128 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.36e-02 / 0.00e+00 | 20260505 08:53:32.263 | INFO | Starting a round of Baz IO | 20260505 08:53:35.538 | INFO | Populating RTMetrics report | 20260505 08:53:35.538 | INFO | RTMetrics report populated | 20260505 08:53:35.538 | INFO | Opening tmp RTMetrics json file | 20260505 08:53:35.547 | INFO | Serializing RTMetrics to json | 20260505 08:53:35.596 | INFO | Serialization done, swapping tmp file now... | 20260505 08:53:35.620 | INFO | RTMetrics done | 20260505 08:53:35.661 | INFO | Finished a round of Baz IO: Wrote 2.446 GiB in 3.397 seconds | 20260505 08:53:35.661 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 26.89% / 0.02% / 73.09% | 20260505 08:53:35.661 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.32644% / 1.98682e-05% / 8.74201e-05% | 20260505 08:53:35.661 | INFO | Baz IO overall: | 20260505 08:53:35.661 | INFO | BytesWritten : 1098744105724 | 20260505 08:53:35.661 | INFO | NumEvents : 522574446948 | 20260505 08:53:37.265 | INFO | Analyzing chunk frames = [1925120,1925632) | 20260505 08:53:37.265 | INFO | MemInfo: { | 20260505 08:53:37.265 | INFO | "gpu" : " 17 GB", | 20260505 08:53:37.265 | INFO | "huge" : "", | 20260505 08:53:37.265 | INFO | "huge_shared" : " 27 GB", | 20260505 08:53:37.265 | INFO | "malloc" : " 14 GB", | 20260505 08:53:37.265 | INFO | "pinned" : " 29 GB" | 20260505 08:53:37.265 | INFO | } | 20260505 08:53:37.265 | INFO | Perf report for the last 5.13885 seconds: | 20260505 08:53:37.265 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:53:37.265 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:37.265 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:37.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:37.265 | INFO | ANALYSIS : 65.2%, 2.9, 99.3ms, 0 | 20260505 08:53:37.265 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:37.265 | INFO | RT_METRICS : 47.5%, 1.0, 24.9ms, 0 | 20260505 08:53:37.265 | INFO | PRE_HQ : 63.3%, 1.9, 61.1ms, 0 | 20260505 08:53:37.265 | INFO | BAZWRITER : 65.4%, 4.2, 141.1ms, 0 | 20260505 08:53:37.265 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260505 08:53:37.265 | INFO | Uploaded 36GiB to GPU in 2.2846s (15.7577GiB/s) | 20260505 08:53:37.265 | INFO | Downloaded 38GiB to GPU in 2.38757s (15.9158GiB/s) | 20260505 08:53:42.405 | INFO | Analyzing chunk frames = [1925632,1926144) | 20260505 08:53:42.405 | INFO | MemInfo: { | 20260505 08:53:42.405 | INFO | "gpu" : " 17 GB", | 20260505 08:53:42.405 | INFO | "huge" : "", | 20260505 08:53:42.405 | INFO | "huge_shared" : " 27 GB", | 20260505 08:53:42.405 | INFO | "malloc" : " 14 GB", | 20260505 08:53:42.405 | INFO | "pinned" : " 29 GB" | 20260505 08:53:42.405 | INFO | } | 20260505 08:53:42.405 | INFO | Perf report for the last 5.13975 seconds: | 20260505 08:53:42.405 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:53:42.405 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:42.405 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:42.405 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:42.405 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 08:53:42.405 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:42.405 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:42.405 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 08:53:42.405 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:42.405 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:42.405 | INFO | Uploaded 36GiB to GPU in 2.22374s (16.1889GiB/s) | 20260505 08:53:42.405 | INFO | Downloaded 34GiB to GPU in 2.08221s (16.3288GiB/s) | 20260505 08:53:47.544 | INFO | Analyzing chunk frames = [1926144,1926656) | 20260505 08:53:47.544 | INFO | MemInfo: { | 20260505 08:53:47.544 | INFO | "gpu" : " 17 GB", | 20260505 08:53:47.544 | INFO | "huge" : "", | 20260505 08:53:47.544 | INFO | "huge_shared" : " 27 GB", | 20260505 08:53:47.544 | INFO | "malloc" : " 14 GB", | 20260505 08:53:47.544 | INFO | "pinned" : " 29 GB" | 20260505 08:53:47.544 | INFO | } | 20260505 08:53:47.545 | INFO | Perf report for the last 5.13967 seconds: | 20260505 08:53:47.545 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:53:47.545 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:47.545 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:47.545 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:47.545 | INFO | ANALYSIS : 63.6%, 3.0, 99.0ms, 0 | 20260505 08:53:47.545 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:47.545 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:47.545 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:53:47.545 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:47.545 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:47.545 | INFO | Uploaded 36GiB to GPU in 2.17263s (16.5698GiB/s) | 20260505 08:53:47.545 | INFO | Downloaded 34GiB to GPU in 2.06948s (16.4293GiB/s) | 20260505 08:53:52.683 | INFO | Analyzing chunk frames = [1926656,1927168) | 20260505 08:53:52.683 | INFO | MemInfo: { | 20260505 08:53:52.683 | INFO | "gpu" : " 17 GB", | 20260505 08:53:52.683 | INFO | "huge" : "", | 20260505 08:53:52.683 | INFO | "huge_shared" : " 27 GB", | 20260505 08:53:52.683 | INFO | "malloc" : " 14 GB", | 20260505 08:53:52.683 | INFO | "pinned" : " 29 GB" | 20260505 08:53:52.683 | INFO | } | 20260505 08:53:52.683 | INFO | Perf report for the last 5.13878 seconds: | 20260505 08:53:52.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:53:52.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:52.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:52.683 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:52.683 | INFO | ANALYSIS : 63.4%, 3.0, 98.6ms, 0 | 20260505 08:53:52.683 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:52.683 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:52.683 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:53:52.683 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:52.683 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:52.684 | INFO | Uploaded 36GiB to GPU in 2.16628s (16.6183GiB/s) | 20260505 08:53:52.684 | INFO | Downloaded 34GiB to GPU in 2.06041s (16.5016GiB/s) | 20260505 08:53:57.822 | INFO | Analyzing chunk frames = [1927168,1927680) | 20260505 08:53:57.822 | INFO | MemInfo: { | 20260505 08:53:57.822 | INFO | "gpu" : " 17 GB", | 20260505 08:53:57.822 | INFO | "huge" : "", | 20260505 08:53:57.822 | INFO | "huge_shared" : " 27 GB", | 20260505 08:53:57.822 | INFO | "malloc" : " 14 GB", | 20260505 08:53:57.822 | INFO | "pinned" : " 29 GB" | 20260505 08:53:57.822 | INFO | } | 20260505 08:53:57.822 | INFO | Perf report for the last 5.13856 seconds: | 20260505 08:53:57.822 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:53:57.822 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:57.822 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:57.822 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:57.822 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 08:53:57.822 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:57.822 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:57.822 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 08:53:57.822 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:57.822 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:53:57.822 | INFO | Uploaded 36GiB to GPU in 2.16046s (16.6631GiB/s) | 20260505 08:53:57.822 | INFO | Downloaded 34GiB to GPU in 2.06005s (16.5045GiB/s) | 20260505 08:54:02.962 | INFO | Analyzing chunk frames = [1927680,1928192) | 20260505 08:54:02.962 | INFO | MemInfo: { | 20260505 08:54:02.962 | INFO | "gpu" : " 17 GB", | 20260505 08:54:02.962 | INFO | "huge" : "", | 20260505 08:54:02.962 | INFO | "huge_shared" : " 27 GB", | 20260505 08:54:02.962 | INFO | "malloc" : " 14 GB", | 20260505 08:54:02.962 | INFO | "pinned" : " 29 GB" | 20260505 08:54:02.962 | INFO | } | 20260505 08:54:02.962 | INFO | Perf report for the last 5.14028 seconds: | 20260505 08:54:02.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:54:02.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:02.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:02.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:02.962 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 08:54:02.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:02.962 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:02.962 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:54:02.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:02.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:02.963 | INFO | Uploaded 36GiB to GPU in 2.1679s (16.6059GiB/s) | 20260505 08:54:02.963 | INFO | Downloaded 34GiB to GPU in 2.06532s (16.4624GiB/s) | 20260505 08:54:08.101 | INFO | Analyzing chunk frames = [1928192,1928704) | 20260505 08:54:08.101 | INFO | MemInfo: { | 20260505 08:54:08.101 | INFO | "gpu" : " 17 GB", | 20260505 08:54:08.101 | INFO | "huge" : "", | 20260505 08:54:08.101 | INFO | "huge_shared" : " 27 GB", | 20260505 08:54:08.101 | INFO | "malloc" : " 14 GB", | 20260505 08:54:08.101 | INFO | "pinned" : " 29 GB" | 20260505 08:54:08.101 | INFO | } | 20260505 08:54:08.102 | INFO | Perf report for the last 5.13928 seconds: | 20260505 08:54:08.102 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:54:08.102 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:08.102 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:08.102 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:08.102 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 08:54:08.102 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:08.102 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:08.102 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 08:54:08.102 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:08.102 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:08.102 | INFO | Uploaded 36GiB to GPU in 2.18385s (16.4846GiB/s) | 20260505 08:54:08.102 | INFO | Downloaded 34GiB to GPU in 2.14249s (15.8694GiB/s) | 20260505 08:54:13.240 | INFO | Analyzing chunk frames = [1928704,1929216) | 20260505 08:54:13.240 | INFO | MemInfo: { | 20260505 08:54:13.240 | INFO | "gpu" : " 17 GB", | 20260505 08:54:13.240 | INFO | "huge" : "", | 20260505 08:54:13.240 | INFO | "huge_shared" : " 27 GB", | 20260505 08:54:13.240 | INFO | "malloc" : " 14 GB", | 20260505 08:54:13.241 | INFO | "pinned" : " 29 GB" | 20260505 08:54:13.241 | INFO | } | 20260505 08:54:13.241 | INFO | Perf report for the last 5.1392 seconds: | 20260505 08:54:13.241 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:54:13.241 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:13.241 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:13.241 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:13.241 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 08:54:13.241 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:13.241 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:13.241 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 08:54:13.241 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:13.241 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:13.241 | INFO | Uploaded 36GiB to GPU in 2.19933s (16.3686GiB/s) | 20260505 08:54:13.241 | INFO | Downloaded 34GiB to GPU in 2.04394s (16.6345GiB/s) | 20260505 08:54:13.377 | INFO | Starting a round of Baz IO | 20260505 08:54:16.650 | INFO | Populating RTMetrics report | 20260505 08:54:16.650 | INFO | RTMetrics report populated | 20260505 08:54:16.650 | INFO | Opening tmp RTMetrics json file | 20260505 08:54:16.657 | INFO | Serializing RTMetrics to json | 20260505 08:54:16.661 | INFO | Serialization done, swapping tmp file now... | 20260505 08:54:16.661 | INFO | RTMetrics done | 20260505 08:54:16.892 | INFO | Finished a round of Baz IO: Wrote 2.447 GiB in 3.515 seconds | 20260505 08:54:16.892 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 26.88% / 0.02% / 73.11% | 20260505 08:54:16.892 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.3232% / 2.78155e-05% / 5.5631e-05% | 20260505 08:54:16.893 | INFO | Baz IO overall: | 20260505 08:54:16.893 | INFO | BytesWritten : 1101371044175 | 20260505 08:54:16.893 | INFO | NumEvents : 523830904970 | 20260505 08:54:18.380 | INFO | Analyzing chunk frames = [1929216,1929728) | 20260505 08:54:18.380 | INFO | MemInfo: { | 20260505 08:54:18.380 | INFO | "gpu" : " 17 GB", | 20260505 08:54:18.380 | INFO | "huge" : "", | 20260505 08:54:18.380 | INFO | "huge_shared" : " 27 GB", | 20260505 08:54:18.380 | INFO | "malloc" : " 14 GB", | 20260505 08:54:18.380 | INFO | "pinned" : " 29 GB" | 20260505 08:54:18.380 | INFO | } | 20260505 08:54:18.380 | INFO | Perf report for the last 5.13924 seconds: | 20260505 08:54:18.380 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:54:18.380 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:18.380 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:18.380 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:18.380 | INFO | ANALYSIS : 65.2%, 2.9, 99.2ms, 0 | 20260505 08:54:18.380 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:18.380 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 08:54:18.380 | INFO | PRE_HQ : 63.4%, 1.9, 61.0ms, 0 | 20260505 08:54:18.380 | INFO | BAZWRITER : 67.7%, 4.2, 147.3ms, 0 | 20260505 08:54:18.380 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 08:54:18.380 | INFO | Uploaded 36GiB to GPU in 2.27667s (15.8126GiB/s) | 20260505 08:54:18.380 | INFO | Downloaded 38GiB to GPU in 2.38517s (15.9317GiB/s) | 20260505 08:54:23.519 | INFO | Analyzing chunk frames = [1929728,1930240) | 20260505 08:54:23.519 | INFO | MemInfo: { | 20260505 08:54:23.519 | INFO | "gpu" : " 17 GB", | 20260505 08:54:23.519 | INFO | "huge" : "", | 20260505 08:54:23.519 | INFO | "huge_shared" : " 27 GB", | 20260505 08:54:23.519 | INFO | "malloc" : " 14 GB", | 20260505 08:54:23.519 | INFO | "pinned" : " 29 GB" | 20260505 08:54:23.519 | INFO | } | 20260505 08:54:23.519 | INFO | Perf report for the last 5.13887 seconds: | 20260505 08:54:23.519 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:54:23.519 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:23.519 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:23.519 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:23.519 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 08:54:23.519 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:23.519 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:23.519 | INFO | PRE_HQ : 29.8%, 1.0, 15.6ms, 0 | 20260505 08:54:23.519 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:23.519 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:23.519 | INFO | Uploaded 36GiB to GPU in 2.21118s (16.2809GiB/s) | 20260505 08:54:23.519 | INFO | Downloaded 34GiB to GPU in 2.06952s (16.429GiB/s) | 20260505 08:54:28.658 | INFO | Analyzing chunk frames = [1930240,1930752) | 20260505 08:54:28.658 | INFO | MemInfo: { | 20260505 08:54:28.658 | INFO | "gpu" : " 17 GB", | 20260505 08:54:28.658 | INFO | "huge" : "", | 20260505 08:54:28.658 | INFO | "huge_shared" : " 27 GB", | 20260505 08:54:28.658 | INFO | "malloc" : " 14 GB", | 20260505 08:54:28.658 | INFO | "pinned" : " 29 GB" | 20260505 08:54:28.658 | INFO | } | 20260505 08:54:28.658 | INFO | Perf report for the last 5.13923 seconds: | 20260505 08:54:28.658 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:54:28.658 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:28.658 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:28.659 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:28.659 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260505 08:54:28.659 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:28.659 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:28.659 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 08:54:28.659 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:28.659 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:28.659 | INFO | Uploaded 36GiB to GPU in 2.1716s (16.5776GiB/s) | 20260505 08:54:28.659 | INFO | Downloaded 34GiB to GPU in 2.06999s (16.4252GiB/s) | 20260505 08:54:33.797 | INFO | Analyzing chunk frames = [1930752,1931264) | 20260505 08:54:33.797 | INFO | MemInfo: { | 20260505 08:54:33.797 | INFO | "gpu" : " 17 GB", | 20260505 08:54:33.797 | INFO | "huge" : "", | 20260505 08:54:33.797 | INFO | "huge_shared" : " 27 GB", | 20260505 08:54:33.797 | INFO | "malloc" : " 14 GB", | 20260505 08:54:33.797 | INFO | "pinned" : " 29 GB" | 20260505 08:54:33.797 | INFO | } | 20260505 08:54:33.797 | INFO | Perf report for the last 5.1393 seconds: | 20260505 08:54:33.798 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:54:33.798 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:33.798 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:33.798 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:33.798 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260505 08:54:33.798 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:33.798 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:33.798 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 08:54:33.798 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:33.798 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:33.798 | INFO | Uploaded 36GiB to GPU in 2.18307s (16.4905GiB/s) | 20260505 08:54:33.798 | INFO | Downloaded 34GiB to GPU in 2.06527s (16.4627GiB/s) | 20260505 08:54:38.937 | INFO | Analyzing chunk frames = [1931264,1931776) | 20260505 08:54:38.937 | INFO | MemInfo: { | 20260505 08:54:38.937 | INFO | "gpu" : " 17 GB", | 20260505 08:54:38.937 | INFO | "huge" : "", | 20260505 08:54:38.937 | INFO | "huge_shared" : " 27 GB", | 20260505 08:54:38.937 | INFO | "malloc" : " 14 GB", | 20260505 08:54:38.937 | INFO | "pinned" : " 29 GB" | 20260505 08:54:38.937 | INFO | } | 20260505 08:54:38.937 | INFO | Perf report for the last 5.13953 seconds: | 20260505 08:54:38.937 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:54:38.937 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:38.937 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:38.937 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:38.937 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 08:54:38.937 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:38.937 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:38.937 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 08:54:38.937 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:38.937 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:38.938 | INFO | Uploaded 36GiB to GPU in 2.23651s (16.0965GiB/s) | 20260505 08:54:38.938 | INFO | Downloaded 34GiB to GPU in 2.10963s (16.1166GiB/s) | 20260505 08:54:44.078 | INFO | Analyzing chunk frames = [1931776,1932288) | 20260505 08:54:44.078 | INFO | MemInfo: { | 20260505 08:54:44.078 | INFO | "gpu" : " 17 GB", | 20260505 08:54:44.078 | INFO | "huge" : "", | 20260505 08:54:44.078 | INFO | "huge_shared" : " 27 GB", | 20260505 08:54:44.078 | INFO | "malloc" : " 14 GB", | 20260505 08:54:44.078 | INFO | "pinned" : " 29 GB" | 20260505 08:54:44.078 | INFO | } | 20260505 08:54:44.078 | INFO | Perf report for the last 5.14111 seconds: | 20260505 08:54:44.078 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:54:44.078 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:44.078 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:44.078 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:44.078 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 08:54:44.078 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:44.078 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:44.078 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 08:54:44.078 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:44.078 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:44.078 | INFO | Uploaded 36GiB to GPU in 2.16964s (16.5926GiB/s) | 20260505 08:54:44.078 | INFO | Downloaded 34GiB to GPU in 2.05724s (16.527GiB/s) | 20260505 08:54:49.215 | INFO | Analyzing chunk frames = [1932288,1932800) | 20260505 08:54:49.215 | INFO | MemInfo: { | 20260505 08:54:49.215 | INFO | "gpu" : " 17 GB", | 20260505 08:54:49.215 | INFO | "huge" : "", | 20260505 08:54:49.215 | INFO | "huge_shared" : " 27 GB", | 20260505 08:54:49.215 | INFO | "malloc" : " 14 GB", | 20260505 08:54:49.215 | INFO | "pinned" : " 29 GB" | 20260505 08:54:49.215 | INFO | } | 20260505 08:54:49.216 | INFO | Perf report for the last 5.13744 seconds: | 20260505 08:54:49.216 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:54:49.216 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:49.216 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:49.216 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:49.216 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 08:54:49.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:49.216 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:49.216 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 08:54:49.216 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:49.216 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:49.216 | INFO | Uploaded 36GiB to GPU in 2.19114s (16.4298GiB/s) | 20260505 08:54:49.216 | INFO | Downloaded 34GiB to GPU in 2.15801s (15.7553GiB/s) | 20260505 08:54:54.355 | INFO | Analyzing chunk frames = [1932800,1933312) | 20260505 08:54:54.355 | INFO | MemInfo: { | 20260505 08:54:54.355 | INFO | "gpu" : " 17 GB", | 20260505 08:54:54.355 | INFO | "huge" : "", | 20260505 08:54:54.355 | INFO | "huge_shared" : " 27 GB", | 20260505 08:54:54.355 | INFO | "malloc" : " 14 GB", | 20260505 08:54:54.355 | INFO | "pinned" : " 29 GB" | 20260505 08:54:54.355 | INFO | } | 20260505 08:54:54.355 | INFO | Perf report for the last 5.13985 seconds: | 20260505 08:54:54.355 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:54:54.355 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:54.355 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:54.355 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:54.355 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 08:54:54.356 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:54.356 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:54.356 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 08:54:54.356 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:54.356 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:54.356 | INFO | Uploaded 36GiB to GPU in 2.19752s (16.3821GiB/s) | 20260505 08:54:54.356 | INFO | Downloaded 34GiB to GPU in 2.04115s (16.6573GiB/s) | 20260505 08:54:54.356 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 08:54:54.356 | INFO | Total mem usage: 15.3 GiB | 20260505 08:54:54.356 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 08:54:54.356 | INFO | Recent High Water Marks Sum: 14.7 GiB (Savings: -4.55%) | 20260505 08:54:54.356 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 08:54:54.356 | INFO | High Water Breakdown: | 20260505 08:54:54.356 | INFO | 61.1% : 8.95 GiB ( 95.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 08:54:54.356 | INFO | 38.8% : 5.69 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 08:54:54.356 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 08:54:54.356 | INFO | ------------------------------------------ | 20260505 08:54:54.356 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:54:54.356 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 08:54:54.356 | INFO | Total mem usage: 26.5 GiB | 20260505 08:54:54.356 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 08:54:54.356 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260505 08:54:54.356 | INFO | Unused Cache: 1.6 GiB (Waste: 6.05%) | 20260505 08:54:54.356 | INFO | High Water Breakdown: | 20260505 08:54:54.356 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:54:54.356 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:54:54.356 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:54:54.356 | INFO | 2% : 523 MiB ( 99.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:54:54.356 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:54:54.356 | INFO | 0.557% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:54:54.356 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 08:54:54.356 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 08:54:54.356 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:54:54.356 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 08:54:54.356 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 08:54:54.356 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 08:54:54.356 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 08:54:54.356 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 08:54:54.356 | INFO | ------------------------------------------ | 20260505 08:54:54.356 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:54:54.356 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 08:54:54.356 | INFO | Total mem usage: 25.6 GiB | 20260505 08:54:54.356 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 08:54:54.356 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 08:54:54.356 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 08:54:54.356 | INFO | High Water Breakdown: | 20260505 08:54:54.356 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 08:54:54.356 | INFO | ------------------------------------------ | 20260505 08:54:54.356 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:54:54.356 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 08:54:54.356 | INFO | Total mem usage: 15.7 GiB | 20260505 08:54:54.356 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 08:54:54.357 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 08:54:54.357 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 08:54:54.357 | INFO | High Water Breakdown: | 20260505 08:54:54.357 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 08:54:54.357 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 08:54:54.357 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 08:54:54.357 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 08:54:54.357 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 08:54:54.357 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 08:54:54.357 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 08:54:54.357 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 08:54:54.357 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 08:54:54.357 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:54:54.357 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 08:54:54.357 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:54:54.357 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 08:54:54.357 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 08:54:54.357 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 08:54:54.357 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 08:54:54.357 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:54:54.357 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 08:54:54.357 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:54:54.357 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:54:54.357 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:54:54.357 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 08:54:54.357 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 08:54:54.357 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:54:54.357 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 08:54:54.357 | INFO | ------------------------------------------ | 20260505 08:54:54.357 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:54:54.357 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 08:54:54.357 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:54:54.357 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:54:54.357 | INFO | PerfReport: Full Execution -- 3.25e+01 / 7.51e+01, 1.35e+01 / 1.63e+00 | 20260505 08:54:54.357 | INFO | PerfReport: 0.29%, AggregateStats -- 9.35e-02 / 1.06e-01, 1.10e-02 / 6.42e-03 | 20260505 08:54:54.357 | INFO | PerfReport: 11.67%, Baseline -- 3.79e+00 / 4.45e+00, 6.18e-01 / 2.67e-01 | 20260505 08:54:54.357 | INFO | PerfReport: 3.32%, Histogram -- 1.08e+00 / 1.48e+00, 2.08e-01 / 1.18e-01 | 20260505 08:54:54.357 | INFO | PerfReport: 15.45%, DME -- 4.02e+01 / 4.38e+01, 3.79e+00 / 1.06e+00 | 20260505 08:54:54.357 | INFO | PerfReport: 56.53%, FrameLabeling -- 1.84e+01 / 2.02e+01, 1.89e+00 / 2.06e+00 | 20260505 08:54:54.357 | INFO | PerfReport: 7.47%, PulseAccumulating -- 2.43e+00 / 3.02e+00, 4.14e-01 / 2.56e-01 | 20260505 08:54:54.357 | INFO | PerfReport: 3.90%, Metrics -- 1.27e+00 / 1.60e+00, 3.67e-01 / 1.90e-01 | 20260505 08:54:54.357 | INFO | PerfReport: 1.38%, PulseSerializing -- 4.49e-01 / 5.05e-01, 8.49e-02 / 2.50e-02 | 20260505 08:54:54.357 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 08:54:54.357 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:54:54.357 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:54:54.357 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.32e+01 / 0.00e+00 | 20260505 08:54:54.357 | INFO | PerfReport: 22.58%, Upload -- 2.29e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260505 08:54:54.357 | INFO | PerfReport: 21.73%, Download -- 2.21e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260505 08:54:54.357 | INFO | PerfReport: 32.05%, Compute -- 3.25e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 08:54:54.357 | INFO | PerfReport: 1.76%, LockUpload -- 1.79e+00 / 0.00e+00, 7.47e+00 / 0.00e+00 | 20260505 08:54:54.357 | INFO | PerfReport: 1.06%, LockDownload -- 1.08e+00 / 0.00e+00, 2.58e+00 / 0.00e+00 | 20260505 08:54:54.357 | INFO | PerfReport: 20.80%, LockCompute -- 2.11e+01 / 0.00e+00, 1.77e+01 / 0.00e+00 | 20260505 08:54:54.357 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.78e-02 / 0.00e+00 | 20260505 08:54:54.494 | INFO | Starting a round of Baz IO | 20260505 08:54:57.771 | INFO | Populating RTMetrics report | 20260505 08:54:57.771 | INFO | RTMetrics report populated | 20260505 08:54:57.771 | INFO | Opening tmp RTMetrics json file | 20260505 08:54:57.785 | INFO | Serializing RTMetrics to json | 20260505 08:54:57.812 | INFO | Serialization done, swapping tmp file now... | 20260505 08:54:57.815 | INFO | RTMetrics done | 20260505 08:54:57.900 | INFO | Finished a round of Baz IO: Wrote 2.446 GiB in 3.405 seconds | 20260505 08:54:57.900 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 26.86% / 0.02% / 73.12% | 20260505 08:54:57.900 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.31206% / 2.78155e-05% / 9.53674e-05% | 20260505 08:54:57.900 | INFO | Baz IO overall: | 20260505 08:54:57.900 | INFO | BytesWritten : 1103997452276 | 20260505 08:54:57.900 | INFO | NumEvents : 525086790083 | 20260505 08:54:59.494 | INFO | Analyzing chunk frames = [1933312,1933824) | 20260505 08:54:59.494 | INFO | MemInfo: { | 20260505 08:54:59.494 | INFO | "gpu" : " 17 GB", | 20260505 08:54:59.494 | INFO | "huge" : "", | 20260505 08:54:59.494 | INFO | "huge_shared" : " 27 GB", | 20260505 08:54:59.494 | INFO | "malloc" : " 14 GB", | 20260505 08:54:59.494 | INFO | "pinned" : " 29 GB" | 20260505 08:54:59.494 | INFO | } | 20260505 08:54:59.494 | INFO | Perf report for the last 5.13869 seconds: | 20260505 08:54:59.494 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:54:59.494 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:59.494 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:59.494 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:59.494 | INFO | ANALYSIS : 65.3%, 2.9, 100.1ms, 0 | 20260505 08:54:59.494 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:54:59.494 | INFO | RT_METRICS : 46.8%, 1.0, 24.5ms, 0 | 20260505 08:54:59.494 | INFO | PRE_HQ : 63.0%, 1.9, 60.8ms, 0 | 20260505 08:54:59.494 | INFO | BAZWRITER : 65.6%, 4.1, 139.3ms, 0 | 20260505 08:54:59.494 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 08:54:59.494 | INFO | Uploaded 36GiB to GPU in 2.21467s (16.2552GiB/s) | 20260505 08:54:59.495 | INFO | Downloaded 38GiB to GPU in 2.34552s (16.2011GiB/s) | 20260505 08:55:04.633 | INFO | Analyzing chunk frames = [1933824,1934336) | 20260505 08:55:04.633 | INFO | MemInfo: { | 20260505 08:55:04.633 | INFO | "gpu" : " 17 GB", | 20260505 08:55:04.633 | INFO | "huge" : "", | 20260505 08:55:04.633 | INFO | "huge_shared" : " 27 GB", | 20260505 08:55:04.633 | INFO | "malloc" : " 14 GB", | 20260505 08:55:04.633 | INFO | "pinned" : " 29 GB" | 20260505 08:55:04.633 | INFO | } | 20260505 08:55:04.633 | INFO | Perf report for the last 5.13897 seconds: | 20260505 08:55:04.633 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:55:04.633 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:04.633 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:04.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:04.633 | INFO | ANALYSIS : 64.4%, 3.0, 100.6ms, 0 | 20260505 08:55:04.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:04.633 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:04.633 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 08:55:04.633 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:04.633 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:04.633 | INFO | Uploaded 36GiB to GPU in 2.21445s (16.2569GiB/s) | 20260505 08:55:04.634 | INFO | Downloaded 34GiB to GPU in 2.08913s (16.2747GiB/s) | 20260505 08:55:09.773 | INFO | Analyzing chunk frames = [1934336,1934848) | 20260505 08:55:09.773 | INFO | MemInfo: { | 20260505 08:55:09.773 | INFO | "gpu" : " 17 GB", | 20260505 08:55:09.773 | INFO | "huge" : "", | 20260505 08:55:09.774 | INFO | "huge_shared" : " 27 GB", | 20260505 08:55:09.774 | INFO | "malloc" : " 14 GB", | 20260505 08:55:09.774 | INFO | "pinned" : " 29 GB" | 20260505 08:55:09.774 | INFO | } | 20260505 08:55:09.774 | INFO | Perf report for the last 5.14049 seconds: | 20260505 08:55:09.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:55:09.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:09.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:09.774 | INFO | SAVE_TRACE : 0.1%, 1.0, 0.1ms, 0 | 20260505 08:55:09.774 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260505 08:55:09.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:09.774 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:09.774 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 08:55:09.774 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:09.774 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:09.774 | INFO | Uploaded 36GiB to GPU in 2.16393s (16.6364GiB/s) | 20260505 08:55:09.774 | INFO | Downloaded 34GiB to GPU in 2.06815s (16.4398GiB/s) | 20260505 08:55:14.913 | INFO | Analyzing chunk frames = [1934848,1935360) | 20260505 08:55:14.913 | INFO | MemInfo: { | 20260505 08:55:14.913 | INFO | "gpu" : " 17 GB", | 20260505 08:55:14.913 | INFO | "huge" : "", | 20260505 08:55:14.913 | INFO | "huge_shared" : " 27 GB", | 20260505 08:55:14.913 | INFO | "malloc" : " 14 GB", | 20260505 08:55:14.913 | INFO | "pinned" : " 29 GB" | 20260505 08:55:14.913 | INFO | } | 20260505 08:55:14.913 | INFO | Perf report for the last 5.13954 seconds: | 20260505 08:55:14.913 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:55:14.913 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:14.913 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:14.913 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:14.913 | INFO | ANALYSIS : 63.4%, 3.0, 98.6ms, 0 | 20260505 08:55:14.913 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:14.913 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:14.913 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260505 08:55:14.913 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:14.913 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:14.913 | INFO | Uploaded 36GiB to GPU in 2.16038s (16.6637GiB/s) | 20260505 08:55:14.913 | INFO | Downloaded 34GiB to GPU in 2.07113s (16.4162GiB/s) | 20260505 08:55:20.053 | INFO | Analyzing chunk frames = [1935360,1935872) | 20260505 08:55:20.053 | INFO | MemInfo: { | 20260505 08:55:20.053 | INFO | "gpu" : " 17 GB", | 20260505 08:55:20.053 | INFO | "huge" : "", | 20260505 08:55:20.053 | INFO | "huge_shared" : " 27 GB", | 20260505 08:55:20.053 | INFO | "malloc" : " 14 GB", | 20260505 08:55:20.053 | INFO | "pinned" : " 29 GB" | 20260505 08:55:20.053 | INFO | } | 20260505 08:55:20.054 | INFO | Perf report for the last 5.14036 seconds: | 20260505 08:55:20.054 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:55:20.054 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:20.054 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:20.054 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:20.054 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 08:55:20.054 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:20.054 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:20.054 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:55:20.054 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:20.054 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:20.054 | INFO | Uploaded 36GiB to GPU in 2.2319s (16.1297GiB/s) | 20260505 08:55:20.054 | INFO | Downloaded 34GiB to GPU in 2.11035s (16.111GiB/s) | 20260505 08:55:25.192 | INFO | Analyzing chunk frames = [1935872,1936384) | 20260505 08:55:25.192 | INFO | MemInfo: { | 20260505 08:55:25.192 | INFO | "gpu" : " 17 GB", | 20260505 08:55:25.192 | INFO | "huge" : "", | 20260505 08:55:25.192 | INFO | "huge_shared" : " 27 GB", | 20260505 08:55:25.192 | INFO | "malloc" : " 14 GB", | 20260505 08:55:25.192 | INFO | "pinned" : " 29 GB" | 20260505 08:55:25.192 | INFO | } | 20260505 08:55:25.192 | INFO | Perf report for the last 5.13884 seconds: | 20260505 08:55:25.192 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:55:25.192 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:25.192 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:25.192 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:25.192 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 08:55:25.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:25.192 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:25.193 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 08:55:25.193 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:25.193 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:25.193 | INFO | Uploaded 36GiB to GPU in 2.16832s (16.6027GiB/s) | 20260505 08:55:25.193 | INFO | Downloaded 34GiB to GPU in 2.0578s (16.5225GiB/s) | 20260505 08:55:30.330 | INFO | Analyzing chunk frames = [1936384,1936896) | 20260505 08:55:30.330 | INFO | MemInfo: { | 20260505 08:55:30.330 | INFO | "gpu" : " 17 GB", | 20260505 08:55:30.330 | INFO | "huge" : "", | 20260505 08:55:30.330 | INFO | "huge_shared" : " 27 GB", | 20260505 08:55:30.330 | INFO | "malloc" : " 14 GB", | 20260505 08:55:30.330 | INFO | "pinned" : " 29 GB" | 20260505 08:55:30.330 | INFO | } | 20260505 08:55:30.330 | INFO | Perf report for the last 5.13759 seconds: | 20260505 08:55:30.330 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:55:30.330 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:30.330 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:30.330 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:30.330 | INFO | ANALYSIS : 63.9%, 3.0, 98.7ms, 0 | 20260505 08:55:30.330 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:30.330 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:30.330 | INFO | PRE_HQ : 31.0%, 1.0, 16.0ms, 0 | 20260505 08:55:30.330 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:30.330 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:30.330 | INFO | Uploaded 36GiB to GPU in 2.18234s (16.496GiB/s) | 20260505 08:55:30.330 | INFO | Downloaded 34GiB to GPU in 2.15015s (15.8129GiB/s) | 20260505 08:55:35.470 | INFO | Analyzing chunk frames = [1936896,1937408) | 20260505 08:55:35.470 | INFO | MemInfo: { | 20260505 08:55:35.470 | INFO | "gpu" : " 17 GB", | 20260505 08:55:35.470 | INFO | "huge" : "", | 20260505 08:55:35.470 | INFO | "huge_shared" : " 27 GB", | 20260505 08:55:35.470 | INFO | "malloc" : " 14 GB", | 20260505 08:55:35.470 | INFO | "pinned" : " 29 GB" | 20260505 08:55:35.470 | INFO | } | 20260505 08:55:35.470 | INFO | Perf report for the last 5.14032 seconds: | 20260505 08:55:35.470 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:55:35.470 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:35.470 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:35.470 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:35.470 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 08:55:35.470 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:35.470 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:35.470 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:55:35.470 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:35.470 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:35.471 | INFO | Uploaded 36GiB to GPU in 2.18279s (16.4926GiB/s) | 20260505 08:55:35.471 | INFO | Downloaded 34GiB to GPU in 2.03333s (16.7213GiB/s) | 20260505 08:55:35.609 | INFO | Starting a round of Baz IO | 20260505 08:55:38.891 | INFO | Populating RTMetrics report | 20260505 08:55:38.892 | INFO | RTMetrics report populated | 20260505 08:55:38.892 | INFO | Opening tmp RTMetrics json file | 20260505 08:55:38.909 | INFO | Serializing RTMetrics to json | 20260505 08:55:38.910 | INFO | Serialization done, swapping tmp file now... | 20260505 08:55:38.910 | INFO | RTMetrics done | 20260505 08:55:39.034 | INFO | Finished a round of Baz IO: Wrote 2.448 GiB in 3.425 seconds | 20260505 08:55:39.034 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 26.84% / 0.02% / 73.14% | 20260505 08:55:39.034 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.31148% / 2.78155e-05% / 5.96046e-05% | 20260505 08:55:39.034 | INFO | Baz IO overall: | 20260505 08:55:39.034 | INFO | BytesWritten : 1106625992689 | 20260505 08:55:39.034 | INFO | NumEvents : 526344579351 | 20260505 08:55:40.610 | INFO | Analyzing chunk frames = [1937408,1937920) | 20260505 08:55:40.610 | INFO | MemInfo: { | 20260505 08:55:40.610 | INFO | "gpu" : " 17 GB", | 20260505 08:55:40.610 | INFO | "huge" : "", | 20260505 08:55:40.610 | INFO | "huge_shared" : " 27 GB", | 20260505 08:55:40.610 | INFO | "malloc" : " 14 GB", | 20260505 08:55:40.610 | INFO | "pinned" : " 29 GB" | 20260505 08:55:40.610 | INFO | } | 20260505 08:55:40.610 | INFO | Perf report for the last 5.13975 seconds: | 20260505 08:55:40.610 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:55:40.610 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:40.610 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:40.610 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:40.610 | INFO | ANALYSIS : 65.4%, 2.9, 100.3ms, 0 | 20260505 08:55:40.610 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:40.610 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 08:55:40.610 | INFO | PRE_HQ : 63.1%, 1.8, 60.5ms, 0 | 20260505 08:55:40.610 | INFO | BAZWRITER : 65.9%, 4.1, 140.0ms, 0 | 20260505 08:55:40.610 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 08:55:40.610 | INFO | Uploaded 36GiB to GPU in 2.23113s (16.1353GiB/s) | 20260505 08:55:40.610 | INFO | Downloaded 38GiB to GPU in 2.3524s (16.1537GiB/s) | 20260505 08:55:45.749 | INFO | Analyzing chunk frames = [1937920,1938432) | 20260505 08:55:45.749 | INFO | MemInfo: { | 20260505 08:55:45.749 | INFO | "gpu" : " 17 GB", | 20260505 08:55:45.749 | INFO | "huge" : "", | 20260505 08:55:45.749 | INFO | "huge_shared" : " 27 GB", | 20260505 08:55:45.749 | INFO | "malloc" : " 14 GB", | 20260505 08:55:45.749 | INFO | "pinned" : " 29 GB" | 20260505 08:55:45.749 | INFO | } | 20260505 08:55:45.749 | INFO | Perf report for the last 5.13897 seconds: | 20260505 08:55:45.749 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:55:45.749 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:45.749 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:45.749 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:45.749 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 08:55:45.749 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:45.749 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:45.749 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 08:55:45.749 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:45.749 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:45.749 | INFO | Uploaded 36GiB to GPU in 2.18179s (16.5002GiB/s) | 20260505 08:55:45.749 | INFO | Downloaded 34GiB to GPU in 2.09003s (16.2677GiB/s) | 20260505 08:55:50.889 | INFO | Analyzing chunk frames = [1938432,1938944) | 20260505 08:55:50.889 | INFO | MemInfo: { | 20260505 08:55:50.889 | INFO | "gpu" : " 17 GB", | 20260505 08:55:50.889 | INFO | "huge" : "", | 20260505 08:55:50.889 | INFO | "huge_shared" : " 27 GB", | 20260505 08:55:50.889 | INFO | "malloc" : " 14 GB", | 20260505 08:55:50.889 | INFO | "pinned" : " 29 GB" | 20260505 08:55:50.889 | INFO | } | 20260505 08:55:50.889 | INFO | Perf report for the last 5.13983 seconds: | 20260505 08:55:50.889 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:55:50.889 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:50.889 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:50.889 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:50.889 | INFO | ANALYSIS : 63.6%, 3.0, 98.9ms, 0 | 20260505 08:55:50.889 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:50.889 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:50.889 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 08:55:50.889 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:50.889 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:50.889 | INFO | Uploaded 36GiB to GPU in 2.16648s (16.6168GiB/s) | 20260505 08:55:50.889 | INFO | Downloaded 34GiB to GPU in 2.07035s (16.4224GiB/s) | 20260505 08:55:56.027 | INFO | Analyzing chunk frames = [1938944,1939456) | 20260505 08:55:56.027 | INFO | MemInfo: { | 20260505 08:55:56.027 | INFO | "gpu" : " 17 GB", | 20260505 08:55:56.027 | INFO | "huge" : "", | 20260505 08:55:56.027 | INFO | "huge_shared" : " 27 GB", | 20260505 08:55:56.027 | INFO | "malloc" : " 14 GB", | 20260505 08:55:56.027 | INFO | "pinned" : " 29 GB" | 20260505 08:55:56.027 | INFO | } | 20260505 08:55:56.027 | INFO | Perf report for the last 5.13846 seconds: | 20260505 08:55:56.027 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:55:56.027 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:56.027 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:56.027 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:56.027 | INFO | ANALYSIS : 63.3%, 3.0, 98.3ms, 0 | 20260505 08:55:56.027 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:56.027 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:56.027 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:55:56.028 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:56.028 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:55:56.028 | INFO | Uploaded 36GiB to GPU in 2.1558s (16.6991GiB/s) | 20260505 08:55:56.028 | INFO | Downloaded 34GiB to GPU in 2.06725s (16.447GiB/s) | 20260505 08:56:01.166 | INFO | Analyzing chunk frames = [1939456,1939968) | 20260505 08:56:01.166 | INFO | MemInfo: { | 20260505 08:56:01.166 | INFO | "gpu" : " 17 GB", | 20260505 08:56:01.166 | INFO | "huge" : "", | 20260505 08:56:01.166 | INFO | "huge_shared" : " 27 GB", | 20260505 08:56:01.167 | INFO | "malloc" : " 14 GB", | 20260505 08:56:01.167 | INFO | "pinned" : " 29 GB" | 20260505 08:56:01.167 | INFO | } | 20260505 08:56:01.167 | INFO | Perf report for the last 5.13933 seconds: | 20260505 08:56:01.167 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:56:01.167 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:01.167 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:01.167 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:01.167 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 08:56:01.167 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:01.167 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:01.167 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 08:56:01.167 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:01.167 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:01.167 | INFO | Uploaded 36GiB to GPU in 2.23205s (16.1287GiB/s) | 20260505 08:56:01.167 | INFO | Downloaded 34GiB to GPU in 2.11159s (16.1016GiB/s) | 20260505 08:56:06.306 | INFO | Analyzing chunk frames = [1939968,1940480) | 20260505 08:56:06.306 | INFO | MemInfo: { | 20260505 08:56:06.306 | INFO | "gpu" : " 17 GB", | 20260505 08:56:06.306 | INFO | "huge" : "", | 20260505 08:56:06.307 | INFO | "huge_shared" : " 27 GB", | 20260505 08:56:06.307 | INFO | "malloc" : " 14 GB", | 20260505 08:56:06.307 | INFO | "pinned" : " 29 GB" | 20260505 08:56:06.307 | INFO | } | 20260505 08:56:06.307 | INFO | Perf report for the last 5.13989 seconds: | 20260505 08:56:06.307 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:56:06.307 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:06.307 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:06.307 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:06.307 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 08:56:06.307 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:06.307 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:06.307 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 08:56:06.307 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:06.307 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:06.307 | INFO | Uploaded 36GiB to GPU in 2.162s (16.6513GiB/s) | 20260505 08:56:06.307 | INFO | Downloaded 34GiB to GPU in 2.05937s (16.5099GiB/s) | 20260505 08:56:11.446 | INFO | Analyzing chunk frames = [1940480,1940992) | 20260505 08:56:11.446 | INFO | MemInfo: { | 20260505 08:56:11.446 | INFO | "gpu" : " 17 GB", | 20260505 08:56:11.446 | INFO | "huge" : "", | 20260505 08:56:11.446 | INFO | "huge_shared" : " 27 GB", | 20260505 08:56:11.446 | INFO | "malloc" : " 14 GB", | 20260505 08:56:11.446 | INFO | "pinned" : " 29 GB" | 20260505 08:56:11.446 | INFO | } | 20260505 08:56:11.446 | INFO | Perf report for the last 5.13925 seconds: | 20260505 08:56:11.446 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:56:11.446 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:11.446 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:11.446 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:11.446 | INFO | ANALYSIS : 64.1%, 3.0, 100.0ms, 0 | 20260505 08:56:11.446 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:11.446 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:11.446 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 08:56:11.446 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:11.446 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:11.446 | INFO | Uploaded 36GiB to GPU in 2.21344s (16.2643GiB/s) | 20260505 08:56:11.446 | INFO | Downloaded 34GiB to GPU in 2.1474s (15.8331GiB/s) | 20260505 08:56:16.585 | INFO | Analyzing chunk frames = [1940992,1941504) | 20260505 08:56:16.585 | INFO | MemInfo: { | 20260505 08:56:16.585 | INFO | "gpu" : " 17 GB", | 20260505 08:56:16.585 | INFO | "huge" : "", | 20260505 08:56:16.585 | INFO | "huge_shared" : " 27 GB", | 20260505 08:56:16.585 | INFO | "malloc" : " 14 GB", | 20260505 08:56:16.585 | INFO | "pinned" : " 29 GB" | 20260505 08:56:16.585 | INFO | } | 20260505 08:56:16.585 | INFO | Perf report for the last 5.13941 seconds: | 20260505 08:56:16.585 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:56:16.585 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:16.585 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:16.585 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:16.585 | INFO | ANALYSIS : 64.1%, 3.0, 100.0ms, 0 | 20260505 08:56:16.585 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:16.585 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:16.585 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 08:56:16.585 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:16.585 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:16.586 | INFO | Uploaded 36GiB to GPU in 2.17529s (16.5495GiB/s) | 20260505 08:56:16.586 | INFO | Downloaded 34GiB to GPU in 2.0239s (16.7993GiB/s) | 20260505 08:56:16.586 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 08:56:16.586 | INFO | Total mem usage: 15.3 GiB | 20260505 08:56:16.586 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 08:56:16.586 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.35%) | 20260505 08:56:16.586 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 08:56:16.586 | INFO | High Water Breakdown: | 20260505 08:56:16.586 | INFO | 60.9% : 8.78 GiB ( 93.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 08:56:16.586 | INFO | 38.9% : 5.61 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 08:56:16.586 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 08:56:16.586 | INFO | ------------------------------------------ | 20260505 08:56:16.586 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:56:16.586 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 08:56:16.586 | INFO | Total mem usage: 26.5 GiB | 20260505 08:56:16.586 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 08:56:16.586 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 08:56:16.586 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 08:56:16.586 | INFO | High Water Breakdown: | 20260505 08:56:16.586 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:56:16.586 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:56:16.586 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:56:16.586 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:56:16.586 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:56:16.586 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:56:16.586 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 08:56:16.586 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 08:56:16.586 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:56:16.586 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 08:56:16.586 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 08:56:16.586 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 08:56:16.586 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 08:56:16.586 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 08:56:16.586 | INFO | ------------------------------------------ | 20260505 08:56:16.586 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:56:16.586 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 08:56:16.586 | INFO | Total mem usage: 25.6 GiB | 20260505 08:56:16.586 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 08:56:16.586 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 08:56:16.586 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 08:56:16.586 | INFO | High Water Breakdown: | 20260505 08:56:16.586 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 08:56:16.586 | INFO | ------------------------------------------ | 20260505 08:56:16.586 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:56:16.586 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 08:56:16.586 | INFO | Total mem usage: 15.7 GiB | 20260505 08:56:16.586 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 08:56:16.586 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 08:56:16.586 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 08:56:16.586 | INFO | High Water Breakdown: | 20260505 08:56:16.586 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 08:56:16.586 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 08:56:16.586 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 08:56:16.586 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 08:56:16.586 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 08:56:16.586 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 08:56:16.586 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 08:56:16.586 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 08:56:16.586 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 08:56:16.586 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:56:16.586 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 08:56:16.586 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:56:16.586 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 08:56:16.586 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 08:56:16.586 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 08:56:16.586 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 08:56:16.586 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:56:16.586 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 08:56:16.586 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:56:16.586 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:56:16.586 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:56:16.586 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 08:56:16.586 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 08:56:16.586 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:56:16.586 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 08:56:16.586 | INFO | ------------------------------------------ | 20260505 08:56:16.586 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:56:16.586 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 08:56:16.586 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:56:16.586 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:56:16.586 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.57e+01, 1.37e+01 / 2.43e+00 | 20260505 08:56:16.586 | INFO | PerfReport: 0.29%, AggregateStats -- 9.32e-02 / 1.05e-01, 1.12e-02 / 8.85e-03 | 20260505 08:56:16.586 | INFO | PerfReport: 11.65%, Baseline -- 3.79e+00 / 4.43e+00, 6.18e-01 / 3.20e-01 | 20260505 08:56:16.586 | INFO | PerfReport: 3.31%, Histogram -- 1.08e+00 / 1.47e+00, 2.10e-01 / 1.09e-01 | 20260505 08:56:16.586 | INFO | PerfReport: 15.49%, DME -- 4.03e+01 / 4.40e+01, 4.13e+00 / 2.07e+00 | 20260505 08:56:16.586 | INFO | PerfReport: 56.55%, FrameLabeling -- 1.84e+01 / 2.06e+01, 1.92e+00 / 1.98e+00 | 20260505 08:56:16.586 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.43e+00 / 2.97e+00, 4.12e-01 / 2.67e-01 | 20260505 08:56:16.586 | INFO | PerfReport: 3.88%, Metrics -- 1.26e+00 / 1.59e+00, 3.62e-01 / 1.77e-01 | 20260505 08:56:16.586 | INFO | PerfReport: 1.38%, PulseSerializing -- 4.48e-01 / 5.04e-01, 8.45e-02 / 2.64e-02 | 20260505 08:56:16.587 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 08:56:16.587 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:56:16.587 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:56:16.587 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.33e+01 / 0.00e+00 | 20260505 08:56:16.587 | INFO | PerfReport: 22.44%, Upload -- 2.28e+01 / 0.00e+00, 8.15e+00 / 0.00e+00 | 20260505 08:56:16.587 | INFO | PerfReport: 21.68%, Download -- 2.20e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260505 08:56:16.587 | INFO | PerfReport: 32.02%, Compute -- 3.26e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 08:56:16.587 | INFO | PerfReport: 1.73%, LockUpload -- 1.76e+00 / 0.00e+00, 7.50e+00 / 0.00e+00 | 20260505 08:56:16.587 | INFO | PerfReport: 1.14%, LockDownload -- 1.15e+00 / 0.00e+00, 2.78e+00 / 0.00e+00 | 20260505 08:56:16.587 | INFO | PerfReport: 20.98%, LockCompute -- 2.13e+01 / 0.00e+00, 1.78e+01 / 0.00e+00 | 20260505 08:56:16.587 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.16e-02 / 0.00e+00 | 20260505 08:56:16.725 | INFO | Starting a round of Baz IO | 20260505 08:56:19.995 | INFO | Populating RTMetrics report | 20260505 08:56:19.996 | INFO | RTMetrics report populated | 20260505 08:56:19.996 | INFO | Opening tmp RTMetrics json file | 20260505 08:56:20.004 | INFO | Serializing RTMetrics to json | 20260505 08:56:20.005 | INFO | Serialization done, swapping tmp file now... | 20260505 08:56:20.005 | INFO | RTMetrics done | 20260505 08:56:20.131 | INFO | Finished a round of Baz IO: Wrote 2.449 GiB in 3.406 seconds | 20260505 08:56:20.131 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 26.82% / 0.02% / 73.16% | 20260505 08:56:20.131 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.30353% / 2.78155e-05% / 8.34465e-05% | 20260505 08:56:20.131 | INFO | Baz IO overall: | 20260505 08:56:20.131 | INFO | BytesWritten : 1109255321020 | 20260505 08:56:20.131 | INFO | NumEvents : 527602822181 | 20260505 08:56:21.725 | INFO | Analyzing chunk frames = [1941504,1942016) | 20260505 08:56:21.725 | INFO | MemInfo: { | 20260505 08:56:21.725 | INFO | "gpu" : " 17 GB", | 20260505 08:56:21.725 | INFO | "huge" : "", | 20260505 08:56:21.725 | INFO | "huge_shared" : " 27 GB", | 20260505 08:56:21.725 | INFO | "malloc" : " 14 GB", | 20260505 08:56:21.725 | INFO | "pinned" : " 29 GB" | 20260505 08:56:21.725 | INFO | } | 20260505 08:56:21.725 | INFO | Perf report for the last 5.1394 seconds: | 20260505 08:56:21.725 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:56:21.725 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:21.725 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:21.725 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:21.725 | INFO | ANALYSIS : 65.2%, 2.9, 99.7ms, 0 | 20260505 08:56:21.725 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:21.725 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260505 08:56:21.725 | INFO | PRE_HQ : 63.4%, 1.9, 61.0ms, 0 | 20260505 08:56:21.725 | INFO | BAZWRITER : 65.6%, 4.2, 141.2ms, 0 | 20260505 08:56:21.725 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 08:56:21.725 | INFO | Uploaded 36GiB to GPU in 2.2735s (15.8346GiB/s) | 20260505 08:56:21.725 | INFO | Downloaded 38GiB to GPU in 2.38606s (15.9258GiB/s) | 20260505 08:56:26.863 | INFO | Analyzing chunk frames = [1942016,1942528) | 20260505 08:56:26.863 | INFO | MemInfo: { | 20260505 08:56:26.863 | INFO | "gpu" : " 17 GB", | 20260505 08:56:26.863 | INFO | "huge" : "", | 20260505 08:56:26.863 | INFO | "huge_shared" : " 27 GB", | 20260505 08:56:26.863 | INFO | "malloc" : " 14 GB", | 20260505 08:56:26.864 | INFO | "pinned" : " 29 GB" | 20260505 08:56:26.864 | INFO | } | 20260505 08:56:26.864 | INFO | Perf report for the last 5.13902 seconds: | 20260505 08:56:26.864 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:56:26.864 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:26.864 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:26.864 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:26.864 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 08:56:26.864 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:26.864 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:26.864 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260505 08:56:26.864 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:26.864 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:26.864 | INFO | Uploaded 36GiB to GPU in 2.18905s (16.4455GiB/s) | 20260505 08:56:26.864 | INFO | Downloaded 34GiB to GPU in 2.08643s (16.2958GiB/s) | 20260505 08:56:32.003 | INFO | Analyzing chunk frames = [1942528,1943040) | 20260505 08:56:32.003 | INFO | MemInfo: { | 20260505 08:56:32.003 | INFO | "gpu" : " 17 GB", | 20260505 08:56:32.003 | INFO | "huge" : "", | 20260505 08:56:32.003 | INFO | "huge_shared" : " 27 GB", | 20260505 08:56:32.003 | INFO | "malloc" : " 14 GB", | 20260505 08:56:32.003 | INFO | "pinned" : " 29 GB" | 20260505 08:56:32.003 | INFO | } | 20260505 08:56:32.003 | INFO | Perf report for the last 5.13942 seconds: | 20260505 08:56:32.003 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:56:32.003 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:32.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:32.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:32.003 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 08:56:32.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:32.003 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:32.003 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:56:32.003 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:32.003 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:32.003 | INFO | Uploaded 36GiB to GPU in 2.1651s (16.6274GiB/s) | 20260505 08:56:32.003 | INFO | Downloaded 34GiB to GPU in 2.06398s (16.473GiB/s) | 20260505 08:56:37.142 | INFO | Analyzing chunk frames = [1943040,1943552) | 20260505 08:56:37.142 | INFO | MemInfo: { | 20260505 08:56:37.142 | INFO | "gpu" : " 17 GB", | 20260505 08:56:37.142 | INFO | "huge" : "", | 20260505 08:56:37.142 | INFO | "huge_shared" : " 27 GB", | 20260505 08:56:37.142 | INFO | "malloc" : " 14 GB", | 20260505 08:56:37.142 | INFO | "pinned" : " 29 GB" | 20260505 08:56:37.142 | INFO | } | 20260505 08:56:37.142 | INFO | Perf report for the last 5.13861 seconds: | 20260505 08:56:37.142 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:56:37.142 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:37.142 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:37.142 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:37.142 | INFO | ANALYSIS : 63.4%, 3.0, 98.5ms, 0 | 20260505 08:56:37.142 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:37.142 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:37.142 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:56:37.142 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:37.142 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:37.142 | INFO | Uploaded 36GiB to GPU in 2.16163s (16.6541GiB/s) | 20260505 08:56:37.142 | INFO | Downloaded 34GiB to GPU in 2.06418s (16.4714GiB/s) | 20260505 08:56:42.280 | INFO | Analyzing chunk frames = [1943552,1944064) | 20260505 08:56:42.280 | INFO | MemInfo: { | 20260505 08:56:42.280 | INFO | "gpu" : " 17 GB", | 20260505 08:56:42.280 | INFO | "huge" : "", | 20260505 08:56:42.281 | INFO | "huge_shared" : " 27 GB", | 20260505 08:56:42.281 | INFO | "malloc" : " 14 GB", | 20260505 08:56:42.281 | INFO | "pinned" : " 29 GB" | 20260505 08:56:42.281 | INFO | } | 20260505 08:56:42.281 | INFO | Perf report for the last 5.13904 seconds: | 20260505 08:56:42.281 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:56:42.281 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:42.281 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:42.281 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:42.281 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 08:56:42.281 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:42.281 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:42.281 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:56:42.281 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:42.281 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:42.281 | INFO | Uploaded 36GiB to GPU in 2.16182s (16.6526GiB/s) | 20260505 08:56:42.281 | INFO | Downloaded 34GiB to GPU in 2.0541s (16.5523GiB/s) | 20260505 08:56:47.421 | INFO | Analyzing chunk frames = [1944064,1944576) | 20260505 08:56:47.421 | INFO | MemInfo: { | 20260505 08:56:47.421 | INFO | "gpu" : " 17 GB", | 20260505 08:56:47.421 | INFO | "huge" : "", | 20260505 08:56:47.422 | INFO | "huge_shared" : " 27 GB", | 20260505 08:56:47.422 | INFO | "malloc" : " 14 GB", | 20260505 08:56:47.422 | INFO | "pinned" : " 29 GB" | 20260505 08:56:47.422 | INFO | } | 20260505 08:56:47.422 | INFO | Perf report for the last 5.14077 seconds: | 20260505 08:56:47.422 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:56:47.422 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:47.422 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:47.422 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:47.422 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 08:56:47.422 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:47.422 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:47.422 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 08:56:47.422 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:47.422 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:47.422 | INFO | Uploaded 36GiB to GPU in 2.19111s (16.43GiB/s) | 20260505 08:56:47.422 | INFO | Downloaded 34GiB to GPU in 2.06576s (16.4588GiB/s) | 20260505 08:56:52.560 | INFO | Analyzing chunk frames = [1944576,1945088) | 20260505 08:56:52.560 | INFO | MemInfo: { | 20260505 08:56:52.560 | INFO | "gpu" : " 17 GB", | 20260505 08:56:52.560 | INFO | "huge" : "", | 20260505 08:56:52.560 | INFO | "huge_shared" : " 27 GB", | 20260505 08:56:52.560 | INFO | "malloc" : " 14 GB", | 20260505 08:56:52.560 | INFO | "pinned" : " 29 GB" | 20260505 08:56:52.560 | INFO | } | 20260505 08:56:52.561 | INFO | Perf report for the last 5.13892 seconds: | 20260505 08:56:52.561 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:56:52.561 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:52.561 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:52.561 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:52.561 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 08:56:52.561 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:52.561 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:52.561 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 08:56:52.561 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:52.561 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:52.561 | INFO | Uploaded 36GiB to GPU in 2.19603s (16.3932GiB/s) | 20260505 08:56:52.561 | INFO | Downloaded 34GiB to GPU in 2.16245s (15.7229GiB/s) | 20260505 08:56:57.700 | INFO | Analyzing chunk frames = [1945088,1945600) | 20260505 08:56:57.700 | INFO | MemInfo: { | 20260505 08:56:57.700 | INFO | "gpu" : " 17 GB", | 20260505 08:56:57.700 | INFO | "huge" : "", | 20260505 08:56:57.700 | INFO | "huge_shared" : " 27 GB", | 20260505 08:56:57.700 | INFO | "malloc" : " 14 GB", | 20260505 08:56:57.700 | INFO | "pinned" : " 29 GB" | 20260505 08:56:57.700 | INFO | } | 20260505 08:56:57.700 | INFO | Perf report for the last 5.13987 seconds: | 20260505 08:56:57.700 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:56:57.700 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:57.700 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:57.701 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:57.701 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 08:56:57.701 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:57.701 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:57.701 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:56:57.701 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:57.701 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:56:57.701 | INFO | Uploaded 36GiB to GPU in 2.17993s (16.5143GiB/s) | 20260505 08:56:57.701 | INFO | Downloaded 34GiB to GPU in 2.02599s (16.7819GiB/s) | 20260505 08:56:57.839 | INFO | Starting a round of Baz IO | 20260505 08:57:01.121 | INFO | Populating RTMetrics report | 20260505 08:57:01.121 | INFO | RTMetrics report populated | 20260505 08:57:01.121 | INFO | Opening tmp RTMetrics json file | 20260505 08:57:01.124 | INFO | Serializing RTMetrics to json | 20260505 08:57:01.126 | INFO | Serialization done, swapping tmp file now... | 20260505 08:57:01.126 | INFO | RTMetrics done | 20260505 08:57:01.250 | INFO | Finished a round of Baz IO: Wrote 2.447 GiB in 3.41 seconds | 20260505 08:57:01.250 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 26.80% / 0.02% / 73.18% | 20260505 08:57:01.250 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.2924% / 2.38419e-05% / 9.93411e-05% | 20260505 08:57:01.250 | INFO | Baz IO overall: | 20260505 08:57:01.250 | INFO | BytesWritten : 1111882500866 | 20260505 08:57:01.250 | INFO | NumEvents : 528859279392 | 20260505 08:57:02.839 | INFO | Analyzing chunk frames = [1945600,1946112) | 20260505 08:57:02.839 | INFO | MemInfo: { | 20260505 08:57:02.839 | INFO | "gpu" : " 17 GB", | 20260505 08:57:02.839 | INFO | "huge" : "", | 20260505 08:57:02.839 | INFO | "huge_shared" : " 27 GB", | 20260505 08:57:02.839 | INFO | "malloc" : " 14 GB", | 20260505 08:57:02.839 | INFO | "pinned" : " 29 GB" | 20260505 08:57:02.839 | INFO | } | 20260505 08:57:02.839 | INFO | Perf report for the last 5.13857 seconds: | 20260505 08:57:02.839 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:57:02.839 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:02.839 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:02.839 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:02.839 | INFO | ANALYSIS : 65.4%, 2.9, 100.3ms, 0 | 20260505 08:57:02.839 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:02.839 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260505 08:57:02.839 | INFO | PRE_HQ : 63.1%, 1.9, 60.9ms, 0 | 20260505 08:57:02.839 | INFO | BAZWRITER : 65.7%, 4.1, 139.7ms, 0 | 20260505 08:57:02.839 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 08:57:02.839 | INFO | Uploaded 36GiB to GPU in 2.24096s (16.0646GiB/s) | 20260505 08:57:02.839 | INFO | Downloaded 38GiB to GPU in 2.34763s (16.1865GiB/s) | 20260505 08:57:07.977 | INFO | Analyzing chunk frames = [1946112,1946624) | 20260505 08:57:07.977 | INFO | MemInfo: { | 20260505 08:57:07.977 | INFO | "gpu" : " 17 GB", | 20260505 08:57:07.977 | INFO | "huge" : "", | 20260505 08:57:07.977 | INFO | "huge_shared" : " 27 GB", | 20260505 08:57:07.977 | INFO | "malloc" : " 14 GB", | 20260505 08:57:07.977 | INFO | "pinned" : " 29 GB" | 20260505 08:57:07.977 | INFO | } | 20260505 08:57:07.977 | INFO | Perf report for the last 5.13841 seconds: | 20260505 08:57:07.978 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:57:07.978 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:07.978 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:07.978 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:07.978 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 08:57:07.978 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:07.978 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:07.978 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 08:57:07.978 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:07.978 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:07.978 | INFO | Uploaded 36GiB to GPU in 2.21149s (16.2786GiB/s) | 20260505 08:57:07.978 | INFO | Downloaded 34GiB to GPU in 2.08108s (16.3376GiB/s) | 20260505 08:57:13.117 | INFO | Analyzing chunk frames = [1946624,1947136) | 20260505 08:57:13.117 | INFO | MemInfo: { | 20260505 08:57:13.117 | INFO | "gpu" : " 17 GB", | 20260505 08:57:13.117 | INFO | "huge" : "", | 20260505 08:57:13.117 | INFO | "huge_shared" : " 27 GB", | 20260505 08:57:13.117 | INFO | "malloc" : " 14 GB", | 20260505 08:57:13.117 | INFO | "pinned" : " 29 GB" | 20260505 08:57:13.117 | INFO | } | 20260505 08:57:13.117 | INFO | Perf report for the last 5.1396 seconds: | 20260505 08:57:13.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:57:13.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:13.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:13.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:13.117 | INFO | ANALYSIS : 63.6%, 3.0, 99.0ms, 0 | 20260505 08:57:13.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:13.117 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:13.117 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 08:57:13.117 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:13.117 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:13.117 | INFO | Uploaded 36GiB to GPU in 2.1692s (16.596GiB/s) | 20260505 08:57:13.117 | INFO | Downloaded 34GiB to GPU in 2.06951s (16.429GiB/s) | 20260505 08:57:18.256 | INFO | Analyzing chunk frames = [1947136,1947648) | 20260505 08:57:18.256 | INFO | MemInfo: { | 20260505 08:57:18.256 | INFO | "gpu" : " 17 GB", | 20260505 08:57:18.256 | INFO | "huge" : "", | 20260505 08:57:18.256 | INFO | "huge_shared" : " 27 GB", | 20260505 08:57:18.257 | INFO | "malloc" : " 14 GB", | 20260505 08:57:18.257 | INFO | "pinned" : " 29 GB" | 20260505 08:57:18.257 | INFO | } | 20260505 08:57:18.257 | INFO | Perf report for the last 5.13954 seconds: | 20260505 08:57:18.257 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:57:18.257 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:18.257 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:18.257 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:18.257 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260505 08:57:18.257 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:18.257 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:18.257 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:57:18.257 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:18.257 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:18.257 | INFO | Uploaded 36GiB to GPU in 2.16377s (16.6376GiB/s) | 20260505 08:57:18.257 | INFO | Downloaded 34GiB to GPU in 2.06486s (16.466GiB/s) | 20260505 08:57:23.397 | INFO | Analyzing chunk frames = [1947648,1948160) | 20260505 08:57:23.397 | INFO | MemInfo: { | 20260505 08:57:23.397 | INFO | "gpu" : " 17 GB", | 20260505 08:57:23.397 | INFO | "huge" : "", | 20260505 08:57:23.397 | INFO | "huge_shared" : " 27 GB", | 20260505 08:57:23.397 | INFO | "malloc" : " 14 GB", | 20260505 08:57:23.397 | INFO | "pinned" : " 29 GB" | 20260505 08:57:23.397 | INFO | } | 20260505 08:57:23.397 | INFO | Perf report for the last 5.14028 seconds: | 20260505 08:57:23.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:57:23.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:23.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:23.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:23.397 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 08:57:23.397 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:23.397 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:23.397 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:57:23.397 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:23.397 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:23.397 | INFO | Uploaded 36GiB to GPU in 2.23244s (16.1259GiB/s) | 20260505 08:57:23.397 | INFO | Downloaded 34GiB to GPU in 2.11302s (16.0907GiB/s) | 20260505 08:57:28.537 | INFO | Analyzing chunk frames = [1948160,1948672) | 20260505 08:57:28.537 | INFO | MemInfo: { | 20260505 08:57:28.537 | INFO | "gpu" : " 17 GB", | 20260505 08:57:28.537 | INFO | "huge" : "", | 20260505 08:57:28.537 | INFO | "huge_shared" : " 27 GB", | 20260505 08:57:28.537 | INFO | "malloc" : " 14 GB", | 20260505 08:57:28.537 | INFO | "pinned" : " 29 GB" | 20260505 08:57:28.537 | INFO | } | 20260505 08:57:28.537 | INFO | Perf report for the last 5.14025 seconds: | 20260505 08:57:28.537 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:57:28.537 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:28.537 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:28.537 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:28.537 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 08:57:28.537 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:28.537 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:28.537 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 08:57:28.537 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:28.537 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:28.537 | INFO | Uploaded 36GiB to GPU in 2.16875s (16.5994GiB/s) | 20260505 08:57:28.537 | INFO | Downloaded 34GiB to GPU in 2.05757s (16.5243GiB/s) | 20260505 08:57:33.677 | INFO | Analyzing chunk frames = [1948672,1949184) | 20260505 08:57:33.677 | INFO | MemInfo: { | 20260505 08:57:33.677 | INFO | "gpu" : " 17 GB", | 20260505 08:57:33.677 | INFO | "huge" : "", | 20260505 08:57:33.677 | INFO | "huge_shared" : " 27 GB", | 20260505 08:57:33.677 | INFO | "malloc" : " 14 GB", | 20260505 08:57:33.677 | INFO | "pinned" : " 29 GB" | 20260505 08:57:33.677 | INFO | } | 20260505 08:57:33.677 | INFO | Perf report for the last 5.14029 seconds: | 20260505 08:57:33.677 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:57:33.677 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:33.677 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:33.678 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:33.678 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 08:57:33.678 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:33.678 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:33.678 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 08:57:33.678 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:33.678 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:33.678 | INFO | Uploaded 36GiB to GPU in 2.21832s (16.2285GiB/s) | 20260505 08:57:33.678 | INFO | Downloaded 34GiB to GPU in 2.14519s (15.8494GiB/s) | 20260505 08:57:38.814 | INFO | Analyzing chunk frames = [1949184,1949696) | 20260505 08:57:38.814 | INFO | MemInfo: { | 20260505 08:57:38.814 | INFO | "gpu" : " 17 GB", | 20260505 08:57:38.814 | INFO | "huge" : "", | 20260505 08:57:38.814 | INFO | "huge_shared" : " 27 GB", | 20260505 08:57:38.814 | INFO | "malloc" : " 14 GB", | 20260505 08:57:38.814 | INFO | "pinned" : " 29 GB" | 20260505 08:57:38.814 | INFO | } | 20260505 08:57:38.814 | INFO | Perf report for the last 5.13656 seconds: | 20260505 08:57:38.814 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:57:38.814 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:38.814 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:38.814 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:38.814 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 08:57:38.814 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:38.814 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:38.814 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 08:57:38.814 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:38.814 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:38.814 | INFO | Uploaded 36GiB to GPU in 2.17195s (16.5749GiB/s) | 20260505 08:57:38.815 | INFO | Downloaded 34GiB to GPU in 2.02356s (16.8021GiB/s) | 20260505 08:57:38.815 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 08:57:38.815 | INFO | Total mem usage: 15.3 GiB | 20260505 08:57:38.815 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 08:57:38.815 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -5.6%) | 20260505 08:57:38.815 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 08:57:38.815 | INFO | High Water Breakdown: | 20260505 08:57:38.815 | INFO | 61% : 8.86 GiB ( 94.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 08:57:38.815 | INFO | 38.8% : 5.64 GiB ( 84.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 08:57:38.815 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 08:57:38.815 | INFO | ------------------------------------------ | 20260505 08:57:38.815 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:57:38.815 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 08:57:38.815 | INFO | Total mem usage: 26.5 GiB | 20260505 08:57:38.815 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 08:57:38.815 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 08:57:38.815 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 08:57:38.815 | INFO | High Water Breakdown: | 20260505 08:57:38.815 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:57:38.815 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:57:38.815 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:57:38.815 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:57:38.815 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:57:38.815 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:57:38.815 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 08:57:38.815 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 08:57:38.815 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:57:38.815 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 08:57:38.815 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 08:57:38.815 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 08:57:38.815 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 08:57:38.815 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 08:57:38.815 | INFO | ------------------------------------------ | 20260505 08:57:38.815 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:57:38.815 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 08:57:38.815 | INFO | Total mem usage: 25.6 GiB | 20260505 08:57:38.815 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 08:57:38.815 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 08:57:38.815 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 08:57:38.815 | INFO | High Water Breakdown: | 20260505 08:57:38.816 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 08:57:38.816 | INFO | ------------------------------------------ | 20260505 08:57:38.816 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:57:38.816 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 08:57:38.816 | INFO | Total mem usage: 15.7 GiB | 20260505 08:57:38.816 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 08:57:38.816 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 08:57:38.816 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 08:57:38.816 | INFO | High Water Breakdown: | 20260505 08:57:38.816 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 08:57:38.816 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 08:57:38.816 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 08:57:38.816 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 08:57:38.816 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 08:57:38.816 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 08:57:38.816 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 08:57:38.816 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 08:57:38.816 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 08:57:38.816 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:57:38.816 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 08:57:38.816 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:57:38.816 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 08:57:38.816 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 08:57:38.816 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 08:57:38.816 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 08:57:38.816 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:57:38.816 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 08:57:38.816 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:57:38.816 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:57:38.816 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:57:38.816 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 08:57:38.816 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 08:57:38.816 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:57:38.816 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 08:57:38.816 | INFO | ------------------------------------------ | 20260505 08:57:38.816 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:57:38.816 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 08:57:38.816 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:57:38.816 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:57:38.816 | INFO | PerfReport: Full Execution -- 3.25e+01 / 7.53e+01, 1.37e+01 / 1.83e+00 | 20260505 08:57:38.816 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 1.04e-01, 1.05e-02 / 6.13e-03 | 20260505 08:57:38.816 | INFO | PerfReport: 11.68%, Baseline -- 3.80e+00 / 4.43e+00, 6.16e-01 / 2.71e-01 | 20260505 08:57:38.816 | INFO | PerfReport: 3.31%, Histogram -- 1.08e+00 / 1.46e+00, 2.07e-01 / 1.11e-01 | 20260505 08:57:38.816 | INFO | PerfReport: 15.50%, DME -- 4.03e+01 / 4.40e+01, 3.99e+00 / 1.65e+00 | 20260505 08:57:38.816 | INFO | PerfReport: 56.45%, FrameLabeling -- 1.84e+01 / 2.02e+01, 1.87e+00 / 1.73e+00 | 20260505 08:57:38.816 | INFO | PerfReport: 7.51%, PulseAccumulating -- 2.44e+00 / 3.02e+00, 4.23e-01 / 2.23e-01 | 20260505 08:57:38.816 | INFO | PerfReport: 3.89%, Metrics -- 1.27e+00 / 1.59e+00, 3.61e-01 / 1.63e-01 | 20260505 08:57:38.816 | INFO | PerfReport: 1.38%, PulseSerializing -- 4.48e-01 / 5.04e-01, 8.42e-02 / 2.48e-02 | 20260505 08:57:38.816 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 08:57:38.816 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:57:38.816 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:57:38.816 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.32e+01 / 0.00e+00 | 20260505 08:57:38.816 | INFO | PerfReport: 22.50%, Upload -- 2.28e+01 / 0.00e+00, 8.14e+00 / 0.00e+00 | 20260505 08:57:38.816 | INFO | PerfReport: 21.68%, Download -- 2.20e+01 / 0.00e+00, 8.17e+00 / 0.00e+00 | 20260505 08:57:38.816 | INFO | PerfReport: 32.03%, Compute -- 3.25e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 08:57:38.816 | INFO | PerfReport: 1.74%, LockUpload -- 1.76e+00 / 0.00e+00, 7.52e+00 / 0.00e+00 | 20260505 08:57:38.816 | INFO | PerfReport: 1.11%, LockDownload -- 1.13e+00 / 0.00e+00, 2.67e+00 / 0.00e+00 | 20260505 08:57:38.816 | INFO | PerfReport: 20.94%, LockCompute -- 2.13e+01 / 0.00e+00, 1.77e+01 / 0.00e+00 | 20260505 08:57:38.816 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.00e-02 / 0.00e+00 | 20260505 08:57:38.954 | INFO | Starting a round of Baz IO | 20260505 08:57:42.218 | INFO | Populating RTMetrics report | 20260505 08:57:42.218 | INFO | RTMetrics report populated | 20260505 08:57:42.218 | INFO | Opening tmp RTMetrics json file | 20260505 08:57:42.232 | INFO | Serializing RTMetrics to json | 20260505 08:57:42.233 | INFO | Serialization done, swapping tmp file now... | 20260505 08:57:42.233 | INFO | RTMetrics done | 20260505 08:57:42.385 | INFO | Finished a round of Baz IO: Wrote 2.445 GiB in 3.43 seconds | 20260505 08:57:42.385 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 26.79% / 0.02% / 73.20% | 20260505 08:57:42.385 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.28227% / 4.37101e-05% / 9.13938e-05% | 20260505 08:57:42.385 | INFO | Baz IO overall: | 20260505 08:57:42.385 | INFO | BytesWritten : 1114508006893 | 20260505 08:57:42.385 | INFO | NumEvents : 530114370966 | 20260505 08:57:43.953 | INFO | Analyzing chunk frames = [1949696,1950208) | 20260505 08:57:43.953 | INFO | MemInfo: { | 20260505 08:57:43.953 | INFO | "gpu" : " 17 GB", | 20260505 08:57:43.953 | INFO | "huge" : "", | 20260505 08:57:43.953 | INFO | "huge_shared" : " 27 GB", | 20260505 08:57:43.953 | INFO | "malloc" : " 14 GB", | 20260505 08:57:43.953 | INFO | "pinned" : " 29 GB" | 20260505 08:57:43.953 | INFO | } | 20260505 08:57:43.953 | INFO | Perf report for the last 5.13932 seconds: | 20260505 08:57:43.953 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:57:43.953 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:43.953 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:43.954 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:43.954 | INFO | ANALYSIS : 65.1%, 2.9, 99.6ms, 0 | 20260505 08:57:43.954 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:43.954 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 08:57:43.954 | INFO | PRE_HQ : 63.3%, 1.9, 61.0ms, 0 | 20260505 08:57:43.954 | INFO | BAZWRITER : 65.9%, 4.2, 141.5ms, 0 | 20260505 08:57:43.954 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 08:57:43.954 | INFO | Uploaded 36GiB to GPU in 2.26118s (15.9209GiB/s) | 20260505 08:57:43.954 | INFO | Downloaded 38GiB to GPU in 2.3861s (15.9256GiB/s) | 20260505 08:57:49.094 | INFO | Analyzing chunk frames = [1950208,1950720) | 20260505 08:57:49.094 | INFO | MemInfo: { | 20260505 08:57:49.094 | INFO | "gpu" : " 17 GB", | 20260505 08:57:49.094 | INFO | "huge" : "", | 20260505 08:57:49.094 | INFO | "huge_shared" : " 27 GB", | 20260505 08:57:49.094 | INFO | "malloc" : " 14 GB", | 20260505 08:57:49.094 | INFO | "pinned" : " 29 GB" | 20260505 08:57:49.094 | INFO | } | 20260505 08:57:49.094 | INFO | Perf report for the last 5.14076 seconds: | 20260505 08:57:49.094 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:57:49.094 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:49.094 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:49.094 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:49.094 | INFO | ANALYSIS : 64.4%, 3.0, 100.6ms, 0 | 20260505 08:57:49.094 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:49.094 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:49.094 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:57:49.094 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:49.094 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:49.095 | INFO | Uploaded 36GiB to GPU in 2.23567s (16.1026GiB/s) | 20260505 08:57:49.095 | INFO | Downloaded 34GiB to GPU in 2.09468s (16.2316GiB/s) | 20260505 08:57:54.232 | INFO | Analyzing chunk frames = [1950720,1951232) | 20260505 08:57:54.232 | INFO | MemInfo: { | 20260505 08:57:54.232 | INFO | "gpu" : " 17 GB", | 20260505 08:57:54.232 | INFO | "huge" : "", | 20260505 08:57:54.232 | INFO | "huge_shared" : " 27 GB", | 20260505 08:57:54.232 | INFO | "malloc" : " 14 GB", | 20260505 08:57:54.232 | INFO | "pinned" : " 29 GB" | 20260505 08:57:54.232 | INFO | } | 20260505 08:57:54.232 | INFO | Perf report for the last 5.1379 seconds: | 20260505 08:57:54.232 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:57:54.232 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:54.232 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:54.232 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:54.232 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 08:57:54.232 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:54.232 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:54.232 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260505 08:57:54.232 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:54.232 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:54.232 | INFO | Uploaded 36GiB to GPU in 2.16911s (16.5966GiB/s) | 20260505 08:57:54.232 | INFO | Downloaded 34GiB to GPU in 2.06998s (16.4253GiB/s) | 20260505 08:57:59.371 | INFO | Analyzing chunk frames = [1951232,1951744) | 20260505 08:57:59.371 | INFO | MemInfo: { | 20260505 08:57:59.371 | INFO | "gpu" : " 17 GB", | 20260505 08:57:59.371 | INFO | "huge" : "", | 20260505 08:57:59.371 | INFO | "huge_shared" : " 27 GB", | 20260505 08:57:59.371 | INFO | "malloc" : " 14 GB", | 20260505 08:57:59.371 | INFO | "pinned" : " 29 GB" | 20260505 08:57:59.371 | INFO | } | 20260505 08:57:59.371 | INFO | Perf report for the last 5.13909 seconds: | 20260505 08:57:59.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:57:59.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:59.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:59.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:59.371 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260505 08:57:59.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:59.371 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:59.371 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:57:59.371 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:59.371 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:57:59.372 | INFO | Uploaded 36GiB to GPU in 2.16168s (16.6537GiB/s) | 20260505 08:57:59.372 | INFO | Downloaded 34GiB to GPU in 2.07091s (16.4179GiB/s) | 20260505 08:58:04.512 | INFO | Analyzing chunk frames = [1951744,1952256) | 20260505 08:58:04.512 | INFO | MemInfo: { | 20260505 08:58:04.512 | INFO | "gpu" : " 17 GB", | 20260505 08:58:04.512 | INFO | "huge" : "", | 20260505 08:58:04.512 | INFO | "huge_shared" : " 27 GB", | 20260505 08:58:04.512 | INFO | "malloc" : " 14 GB", | 20260505 08:58:04.512 | INFO | "pinned" : " 29 GB" | 20260505 08:58:04.512 | INFO | } | 20260505 08:58:04.512 | INFO | Perf report for the last 5.14056 seconds: | 20260505 08:58:04.512 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:58:04.512 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:04.512 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:04.512 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:04.512 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 08:58:04.512 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:04.512 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:04.512 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:58:04.512 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:04.512 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:04.512 | INFO | Uploaded 36GiB to GPU in 2.23055s (16.1395GiB/s) | 20260505 08:58:04.512 | INFO | Downloaded 34GiB to GPU in 2.1137s (16.0855GiB/s) | 20260505 08:58:09.651 | INFO | Analyzing chunk frames = [1952256,1952768) | 20260505 08:58:09.651 | INFO | MemInfo: { | 20260505 08:58:09.651 | INFO | "gpu" : " 17 GB", | 20260505 08:58:09.651 | INFO | "huge" : "", | 20260505 08:58:09.651 | INFO | "huge_shared" : " 27 GB", | 20260505 08:58:09.651 | INFO | "malloc" : " 14 GB", | 20260505 08:58:09.651 | INFO | "pinned" : " 29 GB" | 20260505 08:58:09.651 | INFO | } | 20260505 08:58:09.651 | INFO | Perf report for the last 5.13923 seconds: | 20260505 08:58:09.651 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:58:09.651 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:09.651 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:09.651 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:09.651 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 08:58:09.651 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:09.651 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:09.651 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 08:58:09.651 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:09.651 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:09.651 | INFO | Uploaded 36GiB to GPU in 2.1623s (16.6489GiB/s) | 20260505 08:58:09.651 | INFO | Downloaded 34GiB to GPU in 2.05871s (16.5152GiB/s) | 20260505 08:58:14.789 | INFO | Analyzing chunk frames = [1952768,1953280) | 20260505 08:58:14.789 | INFO | MemInfo: { | 20260505 08:58:14.789 | INFO | "gpu" : " 17 GB", | 20260505 08:58:14.789 | INFO | "huge" : "", | 20260505 08:58:14.789 | INFO | "huge_shared" : " 27 GB", | 20260505 08:58:14.789 | INFO | "malloc" : " 14 GB", | 20260505 08:58:14.789 | INFO | "pinned" : " 29 GB" | 20260505 08:58:14.789 | INFO | } | 20260505 08:58:14.789 | INFO | Perf report for the last 5.13846 seconds: | 20260505 08:58:14.789 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:58:14.789 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:14.789 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:14.789 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:14.789 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 08:58:14.789 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:14.789 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:14.789 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260505 08:58:14.789 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:14.790 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:14.790 | INFO | Uploaded 36GiB to GPU in 2.1807s (16.5084GiB/s) | 20260505 08:58:14.790 | INFO | Downloaded 34GiB to GPU in 2.15387s (15.7855GiB/s) | 20260505 08:58:19.928 | INFO | Analyzing chunk frames = [1953280,1953792) | 20260505 08:58:19.928 | INFO | MemInfo: { | 20260505 08:58:19.928 | INFO | "gpu" : " 17 GB", | 20260505 08:58:19.928 | INFO | "huge" : "", | 20260505 08:58:19.928 | INFO | "huge_shared" : " 27 GB", | 20260505 08:58:19.928 | INFO | "malloc" : " 14 GB", | 20260505 08:58:19.928 | INFO | "pinned" : " 29 GB" | 20260505 08:58:19.928 | INFO | } | 20260505 08:58:19.928 | INFO | Perf report for the last 5.13883 seconds: | 20260505 08:58:19.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:58:19.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:19.928 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:19.928 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:19.928 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 08:58:19.928 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:19.928 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:19.928 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 08:58:19.928 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:19.928 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:19.929 | INFO | Uploaded 36GiB to GPU in 2.1807s (16.5085GiB/s) | 20260505 08:58:19.929 | INFO | Downloaded 34GiB to GPU in 2.02626s (16.7797GiB/s) | 20260505 08:58:20.067 | INFO | Starting a round of Baz IO | 20260505 08:58:23.338 | INFO | Populating RTMetrics report | 20260505 08:58:23.338 | INFO | RTMetrics report populated | 20260505 08:58:23.338 | INFO | Opening tmp RTMetrics json file | 20260505 08:58:23.339 | INFO | Serializing RTMetrics to json | 20260505 08:58:23.342 | INFO | Serialization done, swapping tmp file now... | 20260505 08:58:23.342 | INFO | RTMetrics done | 20260505 08:58:23.483 | INFO | Finished a round of Baz IO: Wrote 2.446 GiB in 3.416 seconds | 20260505 08:58:23.484 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 26.77% / 0.02% / 73.21% | 20260505 08:58:23.484 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.27631% / 2.78155e-05% / 6.75519e-05% | 20260505 08:58:23.484 | INFO | Baz IO overall: | 20260505 08:58:23.484 | INFO | BytesWritten : 1117134398892 | 20260505 08:58:23.484 | INFO | NumEvents : 531369822078 | 20260505 08:58:25.068 | INFO | Analyzing chunk frames = [1953792,1954304) | 20260505 08:58:25.068 | INFO | MemInfo: { | 20260505 08:58:25.068 | INFO | "gpu" : " 17 GB", | 20260505 08:58:25.068 | INFO | "huge" : "", | 20260505 08:58:25.068 | INFO | "huge_shared" : " 27 GB", | 20260505 08:58:25.068 | INFO | "malloc" : " 14 GB", | 20260505 08:58:25.068 | INFO | "pinned" : " 29 GB" | 20260505 08:58:25.068 | INFO | } | 20260505 08:58:25.068 | INFO | Perf report for the last 5.13993 seconds: | 20260505 08:58:25.068 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:58:25.068 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:25.068 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:25.068 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:25.068 | INFO | ANALYSIS : 65.2%, 2.9, 99.7ms, 0 | 20260505 08:58:25.068 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:25.068 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 08:58:25.068 | INFO | PRE_HQ : 63.5%, 1.9, 61.1ms, 0 | 20260505 08:58:25.068 | INFO | BAZWRITER : 65.8%, 4.2, 141.1ms, 0 | 20260505 08:58:25.068 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 08:58:25.068 | INFO | Uploaded 36GiB to GPU in 2.27768s (15.8056GiB/s) | 20260505 08:58:25.068 | INFO | Downloaded 38GiB to GPU in 2.39029s (15.8976GiB/s) | 20260505 08:58:30.207 | INFO | Analyzing chunk frames = [1954304,1954816) | 20260505 08:58:30.207 | INFO | MemInfo: { | 20260505 08:58:30.207 | INFO | "gpu" : " 17 GB", | 20260505 08:58:30.207 | INFO | "huge" : "", | 20260505 08:58:30.207 | INFO | "huge_shared" : " 27 GB", | 20260505 08:58:30.207 | INFO | "malloc" : " 14 GB", | 20260505 08:58:30.207 | INFO | "pinned" : " 29 GB" | 20260505 08:58:30.207 | INFO | } | 20260505 08:58:30.207 | INFO | Perf report for the last 5.13879 seconds: | 20260505 08:58:30.207 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:58:30.207 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:30.207 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:30.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:30.207 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 08:58:30.207 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:30.207 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:30.207 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 08:58:30.207 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:30.207 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:30.207 | INFO | Uploaded 36GiB to GPU in 2.19842s (16.3754GiB/s) | 20260505 08:58:30.207 | INFO | Downloaded 34GiB to GPU in 2.08884s (16.277GiB/s) | 20260505 08:58:35.347 | INFO | Analyzing chunk frames = [1954816,1955328) | 20260505 08:58:35.347 | INFO | MemInfo: { | 20260505 08:58:35.347 | INFO | "gpu" : " 17 GB", | 20260505 08:58:35.347 | INFO | "huge" : "", | 20260505 08:58:35.347 | INFO | "huge_shared" : " 27 GB", | 20260505 08:58:35.347 | INFO | "malloc" : " 14 GB", | 20260505 08:58:35.347 | INFO | "pinned" : " 29 GB" | 20260505 08:58:35.347 | INFO | } | 20260505 08:58:35.347 | INFO | Perf report for the last 5.13978 seconds: | 20260505 08:58:35.347 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:58:35.347 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:35.347 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:35.347 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:35.347 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 08:58:35.347 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:35.347 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:35.347 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 08:58:35.347 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:35.347 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:35.347 | INFO | Uploaded 36GiB to GPU in 2.16394s (16.6363GiB/s) | 20260505 08:58:35.347 | INFO | Downloaded 34GiB to GPU in 2.06738s (16.4459GiB/s) | 20260505 08:58:40.486 | INFO | Analyzing chunk frames = [1955328,1955840) | 20260505 08:58:40.486 | INFO | MemInfo: { | 20260505 08:58:40.486 | INFO | "gpu" : " 17 GB", | 20260505 08:58:40.486 | INFO | "huge" : "", | 20260505 08:58:40.486 | INFO | "huge_shared" : " 27 GB", | 20260505 08:58:40.486 | INFO | "malloc" : " 14 GB", | 20260505 08:58:40.486 | INFO | "pinned" : " 29 GB" | 20260505 08:58:40.486 | INFO | } | 20260505 08:58:40.486 | INFO | Perf report for the last 5.13907 seconds: | 20260505 08:58:40.486 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:58:40.486 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:40.486 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:40.486 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:40.486 | INFO | ANALYSIS : 63.4%, 3.0, 98.5ms, 0 | 20260505 08:58:40.486 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:40.486 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:40.486 | INFO | PRE_HQ : 30.0%, 1.0, 15.5ms, 0 | 20260505 08:58:40.486 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:40.486 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:40.486 | INFO | Uploaded 36GiB to GPU in 2.16155s (16.6548GiB/s) | 20260505 08:58:40.486 | INFO | Downloaded 34GiB to GPU in 2.05887s (16.5139GiB/s) | 20260505 08:58:45.626 | INFO | Analyzing chunk frames = [1955840,1956352) | 20260505 08:58:45.626 | INFO | MemInfo: { | 20260505 08:58:45.626 | INFO | "gpu" : " 17 GB", | 20260505 08:58:45.626 | INFO | "huge" : "", | 20260505 08:58:45.626 | INFO | "huge_shared" : " 27 GB", | 20260505 08:58:45.626 | INFO | "malloc" : " 14 GB", | 20260505 08:58:45.626 | INFO | "pinned" : " 29 GB" | 20260505 08:58:45.626 | INFO | } | 20260505 08:58:45.626 | INFO | Perf report for the last 5.14001 seconds: | 20260505 08:58:45.626 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:58:45.626 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:45.626 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:45.626 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:45.626 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 08:58:45.626 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:45.626 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:45.626 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 08:58:45.626 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:45.626 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:45.626 | INFO | Uploaded 36GiB to GPU in 2.16824s (16.6033GiB/s) | 20260505 08:58:45.626 | INFO | Downloaded 34GiB to GPU in 2.05953s (16.5086GiB/s) | 20260505 08:58:50.767 | INFO | Analyzing chunk frames = [1956352,1956864) | 20260505 08:58:50.767 | INFO | MemInfo: { | 20260505 08:58:50.767 | INFO | "gpu" : " 17 GB", | 20260505 08:58:50.767 | INFO | "huge" : "", | 20260505 08:58:50.767 | INFO | "huge_shared" : " 27 GB", | 20260505 08:58:50.767 | INFO | "malloc" : " 14 GB", | 20260505 08:58:50.767 | INFO | "pinned" : " 29 GB" | 20260505 08:58:50.767 | INFO | } | 20260505 08:58:50.767 | INFO | Perf report for the last 5.14093 seconds: | 20260505 08:58:50.767 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:58:50.767 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:50.767 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:50.767 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:50.767 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 08:58:50.767 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:50.767 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:50.767 | INFO | PRE_HQ : 29.7%, 1.0, 15.5ms, 0 | 20260505 08:58:50.767 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:50.767 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:50.767 | INFO | Uploaded 36GiB to GPU in 2.17528s (16.5496GiB/s) | 20260505 08:58:50.767 | INFO | Downloaded 34GiB to GPU in 2.06395s (16.4733GiB/s) | 20260505 08:58:55.904 | INFO | Analyzing chunk frames = [1956864,1957376) | 20260505 08:58:55.904 | INFO | MemInfo: { | 20260505 08:58:55.904 | INFO | "gpu" : " 17 GB", | 20260505 08:58:55.904 | INFO | "huge" : "", | 20260505 08:58:55.904 | INFO | "huge_shared" : " 27 GB", | 20260505 08:58:55.904 | INFO | "malloc" : " 14 GB", | 20260505 08:58:55.904 | INFO | "pinned" : " 29 GB" | 20260505 08:58:55.904 | INFO | } | 20260505 08:58:55.904 | INFO | Perf report for the last 5.13707 seconds: | 20260505 08:58:55.904 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:58:55.904 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:55.904 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:55.904 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:55.904 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 08:58:55.904 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:55.904 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:55.904 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260505 08:58:55.904 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:55.904 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:58:55.904 | INFO | Uploaded 36GiB to GPU in 2.1951s (16.4002GiB/s) | 20260505 08:58:55.904 | INFO | Downloaded 34GiB to GPU in 2.15536s (15.7746GiB/s) | 20260505 08:59:01.043 | INFO | Analyzing chunk frames = [1957376,1957888) | 20260505 08:59:01.043 | INFO | MemInfo: { | 20260505 08:59:01.043 | INFO | "gpu" : " 17 GB", | 20260505 08:59:01.043 | INFO | "huge" : "", | 20260505 08:59:01.043 | INFO | "huge_shared" : " 27 GB", | 20260505 08:59:01.043 | INFO | "malloc" : " 14 GB", | 20260505 08:59:01.043 | INFO | "pinned" : " 29 GB" | 20260505 08:59:01.043 | INFO | } | 20260505 08:59:01.043 | INFO | Perf report for the last 5.13946 seconds: | 20260505 08:59:01.043 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:59:01.043 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:01.043 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:01.043 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:01.043 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 08:59:01.043 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:01.043 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:01.043 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 08:59:01.043 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:01.043 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:01.044 | INFO | Uploaded 36GiB to GPU in 2.18016s (16.5126GiB/s) | 20260505 08:59:01.044 | INFO | Downloaded 34GiB to GPU in 2.03039s (16.7455GiB/s) | 20260505 08:59:01.044 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 08:59:01.044 | INFO | Total mem usage: 15.3 GiB | 20260505 08:59:01.044 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 08:59:01.044 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.24%) | 20260505 08:59:01.044 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 08:59:01.044 | INFO | High Water Breakdown: | 20260505 08:59:01.044 | INFO | 61% : 8.79 GiB ( 94% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 08:59:01.044 | INFO | 38.9% : 5.61 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 08:59:01.044 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 08:59:01.044 | INFO | ------------------------------------------ | 20260505 08:59:01.044 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:59:01.044 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 08:59:01.044 | INFO | Total mem usage: 26.5 GiB | 20260505 08:59:01.044 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 08:59:01.044 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 08:59:01.044 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 08:59:01.044 | INFO | High Water Breakdown: | 20260505 08:59:01.044 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:59:01.044 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:59:01.044 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:59:01.044 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:59:01.044 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:59:01.044 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:59:01.044 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 08:59:01.044 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 08:59:01.044 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:59:01.044 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 08:59:01.044 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 08:59:01.044 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 08:59:01.044 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 08:59:01.044 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 08:59:01.044 | INFO | ------------------------------------------ | 20260505 08:59:01.044 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:59:01.044 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 08:59:01.044 | INFO | Total mem usage: 25.6 GiB | 20260505 08:59:01.044 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 08:59:01.044 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 08:59:01.044 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 08:59:01.044 | INFO | High Water Breakdown: | 20260505 08:59:01.044 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 08:59:01.044 | INFO | ------------------------------------------ | 20260505 08:59:01.044 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:59:01.044 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 08:59:01.044 | INFO | Total mem usage: 15.7 GiB | 20260505 08:59:01.044 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 08:59:01.044 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 08:59:01.044 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 08:59:01.044 | INFO | High Water Breakdown: | 20260505 08:59:01.044 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 08:59:01.044 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 08:59:01.044 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 08:59:01.044 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 08:59:01.044 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 08:59:01.044 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 08:59:01.044 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 08:59:01.044 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 08:59:01.044 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 08:59:01.044 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 08:59:01.044 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 08:59:01.044 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 08:59:01.044 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 08:59:01.044 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 08:59:01.044 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 08:59:01.044 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 08:59:01.044 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 08:59:01.044 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 08:59:01.044 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 08:59:01.044 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 08:59:01.044 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 08:59:01.044 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 08:59:01.044 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 08:59:01.044 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 08:59:01.044 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 08:59:01.044 | INFO | ------------------------------------------ | 20260505 08:59:01.044 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 08:59:01.045 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 08:59:01.045 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:59:01.045 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:59:01.045 | INFO | PerfReport: Full Execution -- 3.25e+01 / 7.52e+01, 1.37e+01 / 1.74e+00 | 20260505 08:59:01.045 | INFO | PerfReport: 0.29%, AggregateStats -- 9.35e-02 / 1.04e-01, 1.11e-02 / 4.61e-03 | 20260505 08:59:01.045 | INFO | PerfReport: 11.69%, Baseline -- 3.80e+00 / 4.44e+00, 6.27e-01 / 3.09e-01 | 20260505 08:59:01.045 | INFO | PerfReport: 3.32%, Histogram -- 1.08e+00 / 1.47e+00, 2.09e-01 / 1.06e-01 | 20260505 08:59:01.045 | INFO | PerfReport: 15.45%, DME -- 4.02e+01 / 4.37e+01, 3.88e+00 / 8.93e-01 | 20260505 08:59:01.045 | INFO | PerfReport: 56.49%, FrameLabeling -- 1.84e+01 / 2.03e+01, 1.89e+00 / 1.99e+00 | 20260505 08:59:01.045 | INFO | PerfReport: 7.50%, PulseAccumulating -- 2.44e+00 / 2.99e+00, 4.23e-01 / 2.49e-01 | 20260505 08:59:01.045 | INFO | PerfReport: 3.89%, Metrics -- 1.27e+00 / 1.62e+00, 3.67e-01 / 1.59e-01 | 20260505 08:59:01.045 | INFO | PerfReport: 1.38%, PulseSerializing -- 4.49e-01 / 5.05e-01, 8.53e-02 / 2.42e-02 | 20260505 08:59:01.045 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 08:59:01.045 | INFO | PerfReport: Gathered 1536 samples. | 20260505 08:59:01.045 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 08:59:01.045 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.31e+01 / 0.00e+00 | 20260505 08:59:01.045 | INFO | PerfReport: 22.51%, Upload -- 2.29e+01 / 0.00e+00, 8.18e+00 / 0.00e+00 | 20260505 08:59:01.045 | INFO | PerfReport: 21.73%, Download -- 2.21e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260505 08:59:01.045 | INFO | PerfReport: 32.06%, Compute -- 3.25e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 08:59:01.045 | INFO | PerfReport: 1.73%, LockUpload -- 1.75e+00 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260505 08:59:01.045 | INFO | PerfReport: 1.09%, LockDownload -- 1.11e+00 / 0.00e+00, 2.65e+00 / 0.00e+00 | 20260505 08:59:01.045 | INFO | PerfReport: 20.86%, LockCompute -- 2.12e+01 / 0.00e+00, 1.77e+01 / 0.00e+00 | 20260505 08:59:01.045 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 2.25e-02 / 0.00e+00 | 20260505 08:59:01.181 | INFO | Starting a round of Baz IO | 20260505 08:59:04.463 | INFO | Populating RTMetrics report | 20260505 08:59:04.463 | INFO | RTMetrics report populated | 20260505 08:59:04.463 | INFO | Opening tmp RTMetrics json file | 20260505 08:59:04.472 | INFO | Serializing RTMetrics to json | 20260505 08:59:04.474 | INFO | Serialization done, swapping tmp file now... | 20260505 08:59:04.474 | INFO | RTMetrics done | 20260505 08:59:04.598 | INFO | Finished a round of Baz IO: Wrote 2.447 GiB in 3.417 seconds | 20260505 08:59:04.598 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 26.75% / 0.02% / 73.23% | 20260505 08:59:04.598 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.26881% / 2.38419e-05% / 5.96046e-05% | 20260505 08:59:04.598 | INFO | Baz IO overall: | 20260505 08:59:04.598 | INFO | BytesWritten : 1119761396846 | 20260505 08:59:04.598 | INFO | NumEvents : 532625537634 | 20260505 08:59:06.184 | INFO | Analyzing chunk frames = [1957888,1958400) | 20260505 08:59:06.184 | INFO | MemInfo: { | 20260505 08:59:06.184 | INFO | "gpu" : " 17 GB", | 20260505 08:59:06.184 | INFO | "huge" : "", | 20260505 08:59:06.184 | INFO | "huge_shared" : " 27 GB", | 20260505 08:59:06.184 | INFO | "malloc" : " 14 GB", | 20260505 08:59:06.184 | INFO | "pinned" : " 29 GB" | 20260505 08:59:06.184 | INFO | } | 20260505 08:59:06.184 | INFO | Perf report for the last 5.14043 seconds: | 20260505 08:59:06.184 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:59:06.184 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:06.184 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:06.184 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:06.184 | INFO | ANALYSIS : 65.4%, 2.9, 100.4ms, 0 | 20260505 08:59:06.184 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:06.184 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 08:59:06.184 | INFO | PRE_HQ : 63.0%, 1.9, 60.6ms, 0 | 20260505 08:59:06.184 | INFO | BAZWRITER : 65.8%, 4.1, 139.1ms, 0 | 20260505 08:59:06.184 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 08:59:06.185 | INFO | Uploaded 36GiB to GPU in 2.23586s (16.1012GiB/s) | 20260505 08:59:06.185 | INFO | Downloaded 38GiB to GPU in 2.35076s (16.165GiB/s) | 20260505 08:59:11.322 | INFO | Analyzing chunk frames = [1958400,1958912) | 20260505 08:59:11.322 | INFO | MemInfo: { | 20260505 08:59:11.322 | INFO | "gpu" : " 17 GB", | 20260505 08:59:11.322 | INFO | "huge" : "", | 20260505 08:59:11.322 | INFO | "huge_shared" : " 27 GB", | 20260505 08:59:11.322 | INFO | "malloc" : " 14 GB", | 20260505 08:59:11.322 | INFO | "pinned" : " 29 GB" | 20260505 08:59:11.322 | INFO | } | 20260505 08:59:11.322 | INFO | Perf report for the last 5.13856 seconds: | 20260505 08:59:11.322 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:59:11.322 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:11.322 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:11.322 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:11.322 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 08:59:11.322 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:11.322 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:11.322 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 08:59:11.322 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:11.323 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:11.323 | INFO | Uploaded 36GiB to GPU in 2.17091s (16.5829GiB/s) | 20260505 08:59:11.323 | INFO | Downloaded 34GiB to GPU in 2.10561s (16.1474GiB/s) | 20260505 08:59:16.461 | INFO | Analyzing chunk frames = [1958912,1959424) | 20260505 08:59:16.461 | INFO | MemInfo: { | 20260505 08:59:16.461 | INFO | "gpu" : " 17 GB", | 20260505 08:59:16.461 | INFO | "huge" : "", | 20260505 08:59:16.461 | INFO | "huge_shared" : " 27 GB", | 20260505 08:59:16.461 | INFO | "malloc" : " 14 GB", | 20260505 08:59:16.461 | INFO | "pinned" : " 29 GB" | 20260505 08:59:16.461 | INFO | } | 20260505 08:59:16.461 | INFO | Perf report for the last 5.13894 seconds: | 20260505 08:59:16.461 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:59:16.461 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:16.461 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:16.461 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:16.461 | INFO | ANALYSIS : 63.6%, 3.0, 99.0ms, 0 | 20260505 08:59:16.461 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:16.461 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:16.461 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 08:59:16.461 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:16.461 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:16.462 | INFO | Uploaded 36GiB to GPU in 2.17233s (16.5721GiB/s) | 20260505 08:59:16.462 | INFO | Downloaded 34GiB to GPU in 2.0705s (16.4212GiB/s) | 20260505 08:59:21.601 | INFO | Analyzing chunk frames = [1959424,1959936) | 20260505 08:59:21.601 | INFO | MemInfo: { | 20260505 08:59:21.601 | INFO | "gpu" : " 17 GB", | 20260505 08:59:21.601 | INFO | "huge" : "", | 20260505 08:59:21.601 | INFO | "huge_shared" : " 27 GB", | 20260505 08:59:21.601 | INFO | "malloc" : " 14 GB", | 20260505 08:59:21.601 | INFO | "pinned" : " 29 GB" | 20260505 08:59:21.601 | INFO | } | 20260505 08:59:21.601 | INFO | Perf report for the last 5.13935 seconds: | 20260505 08:59:21.601 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:59:21.601 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:21.601 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:21.601 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:21.601 | INFO | ANALYSIS : 63.4%, 3.0, 98.6ms, 0 | 20260505 08:59:21.601 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:21.601 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:21.601 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 08:59:21.601 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:21.601 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:21.601 | INFO | Uploaded 36GiB to GPU in 2.16835s (16.6025GiB/s) | 20260505 08:59:21.601 | INFO | Downloaded 34GiB to GPU in 2.06601s (16.4568GiB/s) | 20260505 08:59:26.741 | INFO | Analyzing chunk frames = [1959936,1960448) | 20260505 08:59:26.741 | INFO | MemInfo: { | 20260505 08:59:26.741 | INFO | "gpu" : " 17 GB", | 20260505 08:59:26.741 | INFO | "huge" : "", | 20260505 08:59:26.741 | INFO | "huge_shared" : " 27 GB", | 20260505 08:59:26.741 | INFO | "malloc" : " 14 GB", | 20260505 08:59:26.741 | INFO | "pinned" : " 29 GB" | 20260505 08:59:26.741 | INFO | } | 20260505 08:59:26.741 | INFO | Perf report for the last 5.13994 seconds: | 20260505 08:59:26.741 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:59:26.741 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:26.741 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:26.741 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:26.741 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 08:59:26.741 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:26.741 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:26.741 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 08:59:26.741 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:26.741 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:26.741 | INFO | Uploaded 36GiB to GPU in 2.16799s (16.6052GiB/s) | 20260505 08:59:26.741 | INFO | Downloaded 34GiB to GPU in 2.05942s (16.5095GiB/s) | 20260505 08:59:31.881 | INFO | Analyzing chunk frames = [1960448,1960960) | 20260505 08:59:31.881 | INFO | MemInfo: { | 20260505 08:59:31.881 | INFO | "gpu" : " 17 GB", | 20260505 08:59:31.881 | INFO | "huge" : "", | 20260505 08:59:31.881 | INFO | "huge_shared" : " 27 GB", | 20260505 08:59:31.881 | INFO | "malloc" : " 14 GB", | 20260505 08:59:31.881 | INFO | "pinned" : " 29 GB" | 20260505 08:59:31.881 | INFO | } | 20260505 08:59:31.881 | INFO | Perf report for the last 5.14008 seconds: | 20260505 08:59:31.881 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:59:31.881 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:31.881 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:31.881 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:31.881 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 08:59:31.881 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:31.881 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:31.881 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 08:59:31.881 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:31.881 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:31.881 | INFO | Uploaded 36GiB to GPU in 2.1898s (16.4399GiB/s) | 20260505 08:59:31.881 | INFO | Downloaded 34GiB to GPU in 2.06624s (16.455GiB/s) | 20260505 08:59:37.019 | INFO | Analyzing chunk frames = [1960960,1961472) | 20260505 08:59:37.019 | INFO | MemInfo: { | 20260505 08:59:37.019 | INFO | "gpu" : " 17 GB", | 20260505 08:59:37.019 | INFO | "huge" : "", | 20260505 08:59:37.019 | INFO | "huge_shared" : " 27 GB", | 20260505 08:59:37.019 | INFO | "malloc" : " 14 GB", | 20260505 08:59:37.019 | INFO | "pinned" : " 29 GB" | 20260505 08:59:37.019 | INFO | } | 20260505 08:59:37.019 | INFO | Perf report for the last 5.13834 seconds: | 20260505 08:59:37.019 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:59:37.019 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:37.019 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:37.019 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:37.019 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 08:59:37.019 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:37.019 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:37.019 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 08:59:37.019 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:37.019 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:37.020 | INFO | Uploaded 36GiB to GPU in 2.20608s (16.3185GiB/s) | 20260505 08:59:37.020 | INFO | Downloaded 34GiB to GPU in 2.15245s (15.7959GiB/s) | 20260505 08:59:42.159 | INFO | Analyzing chunk frames = [1961472,1961984) | 20260505 08:59:42.159 | INFO | MemInfo: { | 20260505 08:59:42.159 | INFO | "gpu" : " 17 GB", | 20260505 08:59:42.159 | INFO | "huge" : "", | 20260505 08:59:42.159 | INFO | "huge_shared" : " 27 GB", | 20260505 08:59:42.159 | INFO | "malloc" : " 14 GB", | 20260505 08:59:42.159 | INFO | "pinned" : " 29 GB" | 20260505 08:59:42.159 | INFO | } | 20260505 08:59:42.159 | INFO | Perf report for the last 5.14008 seconds: | 20260505 08:59:42.159 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:59:42.159 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:42.159 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:42.159 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:42.159 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 08:59:42.159 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:42.159 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:42.159 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 08:59:42.159 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:42.159 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:42.159 | INFO | Uploaded 36GiB to GPU in 2.17262s (16.5699GiB/s) | 20260505 08:59:42.159 | INFO | Downloaded 34GiB to GPU in 2.023s (16.8067GiB/s) | 20260505 08:59:42.297 | INFO | Starting a round of Baz IO | 20260505 08:59:45.567 | INFO | Populating RTMetrics report | 20260505 08:59:45.567 | INFO | RTMetrics report populated | 20260505 08:59:45.567 | INFO | Opening tmp RTMetrics json file | 20260505 08:59:45.588 | INFO | Serializing RTMetrics to json | 20260505 08:59:45.593 | INFO | Serialization done, swapping tmp file now... | 20260505 08:59:45.594 | INFO | RTMetrics done | 20260505 08:59:45.784 | INFO | Finished a round of Baz IO: Wrote 2.447 GiB in 3.487 seconds | 20260505 08:59:45.784 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 26.73% / 0.02% / 73.25% | 20260505 08:59:45.784 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.26429% / 1.58946e-05% / 6.35783e-05% | 20260505 08:59:45.784 | INFO | Baz IO overall: | 20260505 08:59:45.784 | INFO | BytesWritten : 1122389171995 | 20260505 08:59:45.784 | INFO | NumEvents : 533881772094 | 20260505 08:59:47.297 | INFO | Analyzing chunk frames = [1961984,1962496) | 20260505 08:59:47.297 | INFO | MemInfo: { | 20260505 08:59:47.297 | INFO | "gpu" : " 17 GB", | 20260505 08:59:47.297 | INFO | "huge" : "", | 20260505 08:59:47.297 | INFO | "huge_shared" : " 27 GB", | 20260505 08:59:47.297 | INFO | "malloc" : " 14 GB", | 20260505 08:59:47.297 | INFO | "pinned" : " 29 GB" | 20260505 08:59:47.297 | INFO | } | 20260505 08:59:47.297 | INFO | Perf report for the last 5.1381 seconds: | 20260505 08:59:47.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:59:47.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:47.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:47.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:47.297 | INFO | ANALYSIS : 65.2%, 2.9, 99.0ms, 0 | 20260505 08:59:47.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:47.297 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260505 08:59:47.297 | INFO | PRE_HQ : 63.2%, 1.8, 60.6ms, 0 | 20260505 08:59:47.297 | INFO | BAZWRITER : 67.2%, 4.3, 147.2ms, 0 | 20260505 08:59:47.297 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 08:59:47.298 | INFO | Uploaded 36GiB to GPU in 2.29187s (15.7077GiB/s) | 20260505 08:59:47.298 | INFO | Downloaded 38GiB to GPU in 2.39449s (15.8698GiB/s) | 20260505 08:59:52.436 | INFO | Analyzing chunk frames = [1962496,1963008) | 20260505 08:59:52.436 | INFO | MemInfo: { | 20260505 08:59:52.436 | INFO | "gpu" : " 17 GB", | 20260505 08:59:52.436 | INFO | "huge" : "", | 20260505 08:59:52.436 | INFO | "huge_shared" : " 27 GB", | 20260505 08:59:52.436 | INFO | "malloc" : " 14 GB", | 20260505 08:59:52.436 | INFO | "pinned" : " 29 GB" | 20260505 08:59:52.436 | INFO | } | 20260505 08:59:52.436 | INFO | Perf report for the last 5.13913 seconds: | 20260505 08:59:52.436 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:59:52.436 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:52.436 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:52.437 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:52.437 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 08:59:52.437 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:52.437 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:52.437 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 08:59:52.437 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:52.437 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:52.437 | INFO | Uploaded 36GiB to GPU in 2.2055s (16.3229GiB/s) | 20260505 08:59:52.437 | INFO | Downloaded 34GiB to GPU in 2.11516s (16.0744GiB/s) | 20260505 08:59:57.577 | INFO | Analyzing chunk frames = [1963008,1963520) | 20260505 08:59:57.577 | INFO | MemInfo: { | 20260505 08:59:57.577 | INFO | "gpu" : " 17 GB", | 20260505 08:59:57.577 | INFO | "huge" : "", | 20260505 08:59:57.577 | INFO | "huge_shared" : " 27 GB", | 20260505 08:59:57.577 | INFO | "malloc" : " 14 GB", | 20260505 08:59:57.577 | INFO | "pinned" : " 29 GB" | 20260505 08:59:57.577 | INFO | } | 20260505 08:59:57.577 | INFO | Perf report for the last 5.14017 seconds: | 20260505 08:59:57.577 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 08:59:57.577 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:57.577 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:57.577 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:57.577 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 08:59:57.577 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:57.577 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:57.577 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 08:59:57.577 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:57.577 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 08:59:57.577 | INFO | Uploaded 36GiB to GPU in 2.16877s (16.5993GiB/s) | 20260505 08:59:57.577 | INFO | Downloaded 34GiB to GPU in 2.06952s (16.4289GiB/s) | 20260505 09:00:02.716 | INFO | Analyzing chunk frames = [1963520,1964032) | 20260505 09:00:02.716 | INFO | MemInfo: { | 20260505 09:00:02.716 | INFO | "gpu" : " 17 GB", | 20260505 09:00:02.716 | INFO | "huge" : "", | 20260505 09:00:02.716 | INFO | "huge_shared" : " 27 GB", | 20260505 09:00:02.716 | INFO | "malloc" : " 14 GB", | 20260505 09:00:02.716 | INFO | "pinned" : " 29 GB" | 20260505 09:00:02.716 | INFO | } | 20260505 09:00:02.716 | INFO | Perf report for the last 5.13953 seconds: | 20260505 09:00:02.716 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:00:02.716 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:02.716 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:02.716 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:02.716 | INFO | ANALYSIS : 63.4%, 3.0, 98.5ms, 0 | 20260505 09:00:02.716 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:02.716 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:02.716 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:00:02.716 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:02.716 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:02.716 | INFO | Uploaded 36GiB to GPU in 2.16728s (16.6107GiB/s) | 20260505 09:00:02.716 | INFO | Downloaded 34GiB to GPU in 2.06589s (16.4578GiB/s) | 20260505 09:00:07.855 | INFO | Analyzing chunk frames = [1964032,1964544) | 20260505 09:00:07.855 | INFO | MemInfo: { | 20260505 09:00:07.855 | INFO | "gpu" : " 17 GB", | 20260505 09:00:07.855 | INFO | "huge" : "", | 20260505 09:00:07.855 | INFO | "huge_shared" : " 27 GB", | 20260505 09:00:07.855 | INFO | "malloc" : " 14 GB", | 20260505 09:00:07.855 | INFO | "pinned" : " 29 GB" | 20260505 09:00:07.855 | INFO | } | 20260505 09:00:07.855 | INFO | Perf report for the last 5.1391 seconds: | 20260505 09:00:07.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:00:07.855 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:07.855 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:07.855 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:07.855 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 09:00:07.855 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:07.855 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:07.855 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:00:07.855 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:07.855 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:07.856 | INFO | Uploaded 36GiB to GPU in 2.16237s (16.6484GiB/s) | 20260505 09:00:07.856 | INFO | Downloaded 34GiB to GPU in 2.0598s (16.5065GiB/s) | 20260505 09:00:12.995 | INFO | Analyzing chunk frames = [1964544,1965056) | 20260505 09:00:12.995 | INFO | MemInfo: { | 20260505 09:00:12.995 | INFO | "gpu" : " 17 GB", | 20260505 09:00:12.995 | INFO | "huge" : "", | 20260505 09:00:12.995 | INFO | "huge_shared" : " 27 GB", | 20260505 09:00:12.995 | INFO | "malloc" : " 14 GB", | 20260505 09:00:12.995 | INFO | "pinned" : " 29 GB" | 20260505 09:00:12.995 | INFO | } | 20260505 09:00:12.995 | INFO | Perf report for the last 5.14011 seconds: | 20260505 09:00:12.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:00:12.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:12.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:12.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:12.995 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 09:00:12.995 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:12.996 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:12.996 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 09:00:12.996 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:12.996 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:12.996 | INFO | Uploaded 36GiB to GPU in 2.15944s (16.671GiB/s) | 20260505 09:00:12.996 | INFO | Downloaded 34GiB to GPU in 2.06335s (16.4781GiB/s) | 20260505 09:00:18.134 | INFO | Analyzing chunk frames = [1965056,1965568) | 20260505 09:00:18.134 | INFO | MemInfo: { | 20260505 09:00:18.134 | INFO | "gpu" : " 17 GB", | 20260505 09:00:18.134 | INFO | "huge" : "", | 20260505 09:00:18.134 | INFO | "huge_shared" : " 27 GB", | 20260505 09:00:18.135 | INFO | "malloc" : " 14 GB", | 20260505 09:00:18.135 | INFO | "pinned" : " 29 GB" | 20260505 09:00:18.135 | INFO | } | 20260505 09:00:18.135 | INFO | Perf report for the last 5.13916 seconds: | 20260505 09:00:18.135 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:00:18.135 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:18.135 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:18.135 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:18.135 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 09:00:18.135 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:18.135 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:18.135 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 09:00:18.135 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:18.135 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:18.135 | INFO | Uploaded 36GiB to GPU in 2.18722s (16.4593GiB/s) | 20260505 09:00:18.135 | INFO | Downloaded 34GiB to GPU in 2.15382s (15.7859GiB/s) | 20260505 09:00:23.274 | INFO | Analyzing chunk frames = [1965568,1966080) | 20260505 09:00:23.274 | INFO | MemInfo: { | 20260505 09:00:23.274 | INFO | "gpu" : " 17 GB", | 20260505 09:00:23.274 | INFO | "huge" : "", | 20260505 09:00:23.274 | INFO | "huge_shared" : " 27 GB", | 20260505 09:00:23.274 | INFO | "malloc" : " 14 GB", | 20260505 09:00:23.274 | INFO | "pinned" : " 29 GB" | 20260505 09:00:23.274 | INFO | } | 20260505 09:00:23.274 | INFO | Perf report for the last 5.13923 seconds: | 20260505 09:00:23.274 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:00:23.274 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:23.274 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:23.274 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:23.274 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 09:00:23.274 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:23.274 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:23.274 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 09:00:23.274 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:23.274 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:23.274 | INFO | Uploaded 36GiB to GPU in 2.20184s (16.35GiB/s) | 20260505 09:00:23.274 | INFO | Downloaded 34GiB to GPU in 2.04659s (16.613GiB/s) | 20260505 09:00:23.274 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 09:00:23.274 | INFO | Total mem usage: 15.3 GiB | 20260505 09:00:23.274 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 09:00:23.274 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -5.29%) | 20260505 09:00:23.274 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 09:00:23.274 | INFO | High Water Breakdown: | 20260505 09:00:23.274 | INFO | 61% : 8.87 GiB ( 94.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 09:00:23.274 | INFO | 38.9% : 5.66 GiB ( 85.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 09:00:23.274 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 09:00:23.274 | INFO | ------------------------------------------ | 20260505 09:00:23.274 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:00:23.274 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 09:00:23.274 | INFO | Total mem usage: 26.5 GiB | 20260505 09:00:23.274 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 09:00:23.274 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 09:00:23.274 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 09:00:23.274 | INFO | High Water Breakdown: | 20260505 09:00:23.274 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:00:23.274 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:00:23.274 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:00:23.274 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:00:23.274 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:00:23.274 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:00:23.274 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 09:00:23.274 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 09:00:23.274 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:00:23.274 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 09:00:23.274 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 09:00:23.275 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 09:00:23.275 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 09:00:23.275 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 09:00:23.275 | INFO | ------------------------------------------ | 20260505 09:00:23.275 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:00:23.275 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 09:00:23.275 | INFO | Total mem usage: 25.6 GiB | 20260505 09:00:23.275 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 09:00:23.275 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 09:00:23.275 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 09:00:23.275 | INFO | High Water Breakdown: | 20260505 09:00:23.275 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 09:00:23.275 | INFO | ------------------------------------------ | 20260505 09:00:23.275 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:00:23.275 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 09:00:23.275 | INFO | Total mem usage: 15.7 GiB | 20260505 09:00:23.275 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 09:00:23.275 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 09:00:23.275 | INFO | Unused Cache: 880 MiB (Waste: 5.48%) | 20260505 09:00:23.275 | INFO | High Water Breakdown: | 20260505 09:00:23.275 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 09:00:23.275 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 09:00:23.275 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 09:00:23.275 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 09:00:23.275 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 09:00:23.275 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 09:00:23.275 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 09:00:23.275 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 09:00:23.275 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 09:00:23.275 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:00:23.275 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 09:00:23.275 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:00:23.275 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 09:00:23.275 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 09:00:23.275 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 09:00:23.275 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 09:00:23.275 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:00:23.275 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 09:00:23.275 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:00:23.275 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:00:23.275 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:00:23.275 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 09:00:23.275 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 09:00:23.275 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:00:23.275 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 09:00:23.275 | INFO | ------------------------------------------ | 20260505 09:00:23.275 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:00:23.275 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 09:00:23.275 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:00:23.275 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:00:23.275 | INFO | PerfReport: Full Execution -- 3.25e+01 / 7.49e+01, 1.35e+01 / 1.84e+00 | 20260505 09:00:23.275 | INFO | PerfReport: 0.29%, AggregateStats -- 9.34e-02 / 1.07e-01, 1.13e-02 / 1.66e-02 | 20260505 09:00:23.275 | INFO | PerfReport: 11.65%, Baseline -- 3.79e+00 / 4.39e+00, 6.06e-01 / 3.58e-01 | 20260505 09:00:23.275 | INFO | PerfReport: 3.32%, Histogram -- 1.08e+00 / 1.47e+00, 2.04e-01 / 1.11e-01 | 20260505 09:00:23.275 | INFO | PerfReport: 15.48%, DME -- 4.03e+01 / 4.39e+01, 3.92e+00 / 1.55e+00 | 20260505 09:00:23.275 | INFO | PerfReport: 56.44%, FrameLabeling -- 1.83e+01 / 1.99e+01, 1.83e+00 / 1.82e+00 | 20260505 09:00:23.275 | INFO | PerfReport: 7.54%, PulseAccumulating -- 2.45e+00 / 3.00e+00, 4.24e-01 / 2.83e-01 | 20260505 09:00:23.275 | INFO | PerfReport: 3.90%, Metrics -- 1.27e+00 / 1.60e+00, 3.59e-01 / 1.84e-01 | 20260505 09:00:23.275 | INFO | PerfReport: 1.38%, PulseSerializing -- 4.48e-01 / 5.07e-01, 8.32e-02 / 3.14e-02 | 20260505 09:00:23.275 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 09:00:23.275 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:00:23.275 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:00:23.275 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.27e+01 / 0.00e+00 | 20260505 09:00:23.275 | INFO | PerfReport: 22.49%, Upload -- 2.28e+01 / 0.00e+00, 7.98e+00 / 0.00e+00 | 20260505 09:00:23.275 | INFO | PerfReport: 21.74%, Download -- 2.20e+01 / 0.00e+00, 8.12e+00 / 0.00e+00 | 20260505 09:00:23.275 | INFO | PerfReport: 32.05%, Compute -- 3.25e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 09:00:23.275 | INFO | PerfReport: 1.69%, LockUpload -- 1.71e+00 / 0.00e+00, 7.48e+00 / 0.00e+00 | 20260505 09:00:23.275 | INFO | PerfReport: 1.14%, LockDownload -- 1.16e+00 / 0.00e+00, 2.70e+00 / 0.00e+00 | 20260505 09:00:23.275 | INFO | PerfReport: 20.88%, LockCompute -- 2.12e+01 / 0.00e+00, 1.75e+01 / 0.00e+00 | 20260505 09:00:23.275 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.97e-02 / 0.00e+00 | 20260505 09:00:23.409 | INFO | Starting a round of Baz IO | 20260505 09:00:26.687 | INFO | Populating RTMetrics report | 20260505 09:00:26.687 | INFO | RTMetrics report populated | 20260505 09:00:26.687 | INFO | Opening tmp RTMetrics json file | 20260505 09:00:26.707 | INFO | Serializing RTMetrics to json | 20260505 09:00:26.732 | INFO | Serialization done, swapping tmp file now... | 20260505 09:00:26.732 | INFO | RTMetrics done | 20260505 09:00:26.838 | INFO | Finished a round of Baz IO: Wrote 2.449 GiB in 3.429 seconds | 20260505 09:00:26.838 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 26.72% / 0.02% / 73.27% | 20260505 09:00:26.838 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.26144% / 1.98682e-05% / 7.54992e-05% | 20260505 09:00:26.838 | INFO | Baz IO overall: | 20260505 09:00:26.839 | INFO | BytesWritten : 1125019112108 | 20260505 09:00:26.839 | INFO | NumEvents : 535139492697 | 20260505 09:00:28.413 | INFO | Analyzing chunk frames = [1966080,1966592) | 20260505 09:00:28.413 | INFO | MemInfo: { | 20260505 09:00:28.413 | INFO | "gpu" : " 17 GB", | 20260505 09:00:28.413 | INFO | "huge" : "", | 20260505 09:00:28.413 | INFO | "huge_shared" : " 27 GB", | 20260505 09:00:28.413 | INFO | "malloc" : " 14 GB", | 20260505 09:00:28.413 | INFO | "pinned" : " 29 GB" | 20260505 09:00:28.413 | INFO | } | 20260505 09:00:28.413 | INFO | Perf report for the last 5.13888 seconds: | 20260505 09:00:28.413 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:00:28.413 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:28.413 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:28.413 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:28.413 | INFO | ANALYSIS : 65.3%, 2.9, 100.0ms, 0 | 20260505 09:00:28.413 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:28.413 | INFO | RT_METRICS : 46.6%, 1.0, 24.4ms, 0 | 20260505 09:00:28.413 | INFO | PRE_HQ : 63.1%, 1.8, 60.6ms, 0 | 20260505 09:00:28.413 | INFO | BAZWRITER : 66.0%, 4.1, 140.7ms, 0 | 20260505 09:00:28.413 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260505 09:00:28.413 | INFO | Uploaded 36GiB to GPU in 2.28372s (15.7638GiB/s) | 20260505 09:00:28.413 | INFO | Downloaded 38GiB to GPU in 2.37622s (15.9918GiB/s) | 20260505 09:00:33.551 | INFO | Analyzing chunk frames = [1966592,1967104) | 20260505 09:00:33.551 | INFO | MemInfo: { | 20260505 09:00:33.551 | INFO | "gpu" : " 17 GB", | 20260505 09:00:33.552 | INFO | "huge" : "", | 20260505 09:00:33.552 | INFO | "huge_shared" : " 27 GB", | 20260505 09:00:33.552 | INFO | "malloc" : " 14 GB", | 20260505 09:00:33.552 | INFO | "pinned" : " 29 GB" | 20260505 09:00:33.552 | INFO | } | 20260505 09:00:33.552 | INFO | Perf report for the last 5.13877 seconds: | 20260505 09:00:33.552 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:00:33.552 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:33.552 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:33.552 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:33.552 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 09:00:33.552 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:33.552 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:33.552 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260505 09:00:33.552 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:33.552 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:33.552 | INFO | Uploaded 36GiB to GPU in 2.21927s (16.2216GiB/s) | 20260505 09:00:33.552 | INFO | Downloaded 34GiB to GPU in 2.07865s (16.3567GiB/s) | 20260505 09:00:38.691 | INFO | Analyzing chunk frames = [1967104,1967616) | 20260505 09:00:38.691 | INFO | MemInfo: { | 20260505 09:00:38.691 | INFO | "gpu" : " 17 GB", | 20260505 09:00:38.691 | INFO | "huge" : "", | 20260505 09:00:38.691 | INFO | "huge_shared" : " 27 GB", | 20260505 09:00:38.691 | INFO | "malloc" : " 14 GB", | 20260505 09:00:38.691 | INFO | "pinned" : " 29 GB" | 20260505 09:00:38.691 | INFO | } | 20260505 09:00:38.691 | INFO | Perf report for the last 5.13969 seconds: | 20260505 09:00:38.691 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:00:38.691 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:38.691 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:38.691 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:38.691 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 09:00:38.691 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:38.691 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:38.691 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 09:00:38.691 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:38.691 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:38.691 | INFO | Uploaded 36GiB to GPU in 2.17169s (16.577GiB/s) | 20260505 09:00:38.692 | INFO | Downloaded 34GiB to GPU in 2.06812s (16.44GiB/s) | 20260505 09:00:43.834 | INFO | Analyzing chunk frames = [1967616,1968128) | 20260505 09:00:43.834 | INFO | MemInfo: { | 20260505 09:00:43.834 | INFO | "gpu" : " 17 GB", | 20260505 09:00:43.834 | INFO | "huge" : "", | 20260505 09:00:43.834 | INFO | "huge_shared" : " 27 GB", | 20260505 09:00:43.834 | INFO | "malloc" : " 14 GB", | 20260505 09:00:43.835 | INFO | "pinned" : " 29 GB" | 20260505 09:00:43.835 | INFO | } | 20260505 09:00:43.835 | INFO | Perf report for the last 5.14328 seconds: | 20260505 09:00:43.835 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:00:43.835 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:43.835 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:43.835 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:43.835 | INFO | ANALYSIS : 63.4%, 3.0, 98.5ms, 0 | 20260505 09:00:43.835 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:43.835 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:43.835 | INFO | PRE_HQ : 29.5%, 1.0, 15.4ms, 0 | 20260505 09:00:43.835 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:43.835 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:43.835 | INFO | Uploaded 36GiB to GPU in 2.17913s (16.5204GiB/s) | 20260505 09:00:43.835 | INFO | Downloaded 34GiB to GPU in 2.06307s (16.4803GiB/s) | 20260505 09:00:48.969 | INFO | Analyzing chunk frames = [1968128,1968640) | 20260505 09:00:48.969 | INFO | MemInfo: { | 20260505 09:00:48.969 | INFO | "gpu" : " 17 GB", | 20260505 09:00:48.969 | INFO | "huge" : "", | 20260505 09:00:48.970 | INFO | "huge_shared" : " 27 GB", | 20260505 09:00:48.970 | INFO | "malloc" : " 14 GB", | 20260505 09:00:48.970 | INFO | "pinned" : " 29 GB" | 20260505 09:00:48.970 | INFO | } | 20260505 09:00:48.970 | INFO | Perf report for the last 5.135 seconds: | 20260505 09:00:48.970 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:00:48.970 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:48.970 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:48.970 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:48.970 | INFO | ANALYSIS : 63.8%, 3.0, 99.5ms, 0 | 20260505 09:00:48.970 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:48.970 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:48.970 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 09:00:48.970 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:48.970 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:48.970 | INFO | Uploaded 36GiB to GPU in 2.23363s (16.1173GiB/s) | 20260505 09:00:48.970 | INFO | Downloaded 34GiB to GPU in 2.11212s (16.0976GiB/s) | 20260505 09:00:54.109 | INFO | Analyzing chunk frames = [1968640,1969152) | 20260505 09:00:54.109 | INFO | MemInfo: { | 20260505 09:00:54.109 | INFO | "gpu" : " 17 GB", | 20260505 09:00:54.110 | INFO | "huge" : "", | 20260505 09:00:54.110 | INFO | "huge_shared" : " 27 GB", | 20260505 09:00:54.110 | INFO | "malloc" : " 14 GB", | 20260505 09:00:54.110 | INFO | "pinned" : " 29 GB" | 20260505 09:00:54.110 | INFO | } | 20260505 09:00:54.110 | INFO | Perf report for the last 5.14004 seconds: | 20260505 09:00:54.110 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:00:54.110 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:54.110 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:54.110 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:54.110 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 09:00:54.110 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:54.110 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:54.110 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:00:54.110 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:54.110 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:54.110 | INFO | Uploaded 36GiB to GPU in 2.18515s (16.4748GiB/s) | 20260505 09:00:54.110 | INFO | Downloaded 34GiB to GPU in 2.06153s (16.4926GiB/s) | 20260505 09:00:59.249 | INFO | Analyzing chunk frames = [1969152,1969664) | 20260505 09:00:59.249 | INFO | MemInfo: { | 20260505 09:00:59.249 | INFO | "gpu" : " 17 GB", | 20260505 09:00:59.249 | INFO | "huge" : "", | 20260505 09:00:59.249 | INFO | "huge_shared" : " 27 GB", | 20260505 09:00:59.249 | INFO | "malloc" : " 14 GB", | 20260505 09:00:59.249 | INFO | "pinned" : " 29 GB" | 20260505 09:00:59.249 | INFO | } | 20260505 09:00:59.249 | INFO | Perf report for the last 5.1396 seconds: | 20260505 09:00:59.249 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:00:59.249 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:59.249 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:59.249 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:59.249 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 09:00:59.249 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:59.249 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:59.249 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 09:00:59.249 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:59.249 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:00:59.249 | INFO | Uploaded 36GiB to GPU in 2.21565s (16.2481GiB/s) | 20260505 09:00:59.250 | INFO | Downloaded 34GiB to GPU in 2.14489s (15.8516GiB/s) | 20260505 09:01:04.389 | INFO | Analyzing chunk frames = [1969664,1970176) | 20260505 09:01:04.389 | INFO | MemInfo: { | 20260505 09:01:04.389 | INFO | "gpu" : " 17 GB", | 20260505 09:01:04.389 | INFO | "huge" : "", | 20260505 09:01:04.389 | INFO | "huge_shared" : " 27 GB", | 20260505 09:01:04.389 | INFO | "malloc" : " 14 GB", | 20260505 09:01:04.389 | INFO | "pinned" : " 29 GB" | 20260505 09:01:04.389 | INFO | } | 20260505 09:01:04.389 | INFO | Perf report for the last 5.13992 seconds: | 20260505 09:01:04.389 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:01:04.389 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:04.389 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:04.389 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:04.389 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 09:01:04.389 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:04.389 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:04.389 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 09:01:04.389 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:04.389 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:04.390 | INFO | Uploaded 36GiB to GPU in 2.20261s (16.3443GiB/s) | 20260505 09:01:04.390 | INFO | Downloaded 34GiB to GPU in 2.04305s (16.6418GiB/s) | 20260505 09:01:04.524 | INFO | Starting a round of Baz IO | 20260505 09:01:07.798 | INFO | Populating RTMetrics report | 20260505 09:01:07.798 | INFO | RTMetrics report populated | 20260505 09:01:07.798 | INFO | Opening tmp RTMetrics json file | 20260505 09:01:07.871 | INFO | Serializing RTMetrics to json | 20260505 09:01:07.918 | INFO | Serialization done, swapping tmp file now... | 20260505 09:01:07.925 | INFO | RTMetrics done | 20260505 09:01:07.963 | INFO | Finished a round of Baz IO: Wrote 2.449 GiB in 3.439 seconds | 20260505 09:01:07.963 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 26.70% / 0.02% / 73.28% | 20260505 09:01:07.963 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.25091% / 2.38419e-05% / 7.94729e-05% | 20260505 09:01:07.963 | INFO | Baz IO overall: | 20260505 09:01:07.963 | INFO | BytesWritten : 1127648457747 | 20260505 09:01:07.963 | INFO | NumEvents : 536397037464 | 20260505 09:01:09.527 | INFO | Analyzing chunk frames = [1970176,1970688) | 20260505 09:01:09.527 | INFO | MemInfo: { | 20260505 09:01:09.527 | INFO | "gpu" : " 17 GB", | 20260505 09:01:09.527 | INFO | "huge" : "", | 20260505 09:01:09.527 | INFO | "huge_shared" : " 27 GB", | 20260505 09:01:09.527 | INFO | "malloc" : " 14 GB", | 20260505 09:01:09.527 | INFO | "pinned" : " 29 GB" | 20260505 09:01:09.527 | INFO | } | 20260505 09:01:09.527 | INFO | Perf report for the last 5.13797 seconds: | 20260505 09:01:09.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:01:09.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:09.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:09.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:09.527 | INFO | ANALYSIS : 65.2%, 2.9, 99.6ms, 0 | 20260505 09:01:09.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:09.527 | INFO | RT_METRICS : 48.2%, 1.0, 25.3ms, 0 | 20260505 09:01:09.527 | INFO | PRE_HQ : 63.2%, 1.9, 60.9ms, 0 | 20260505 09:01:09.527 | INFO | BAZWRITER : 66.2%, 4.2, 141.8ms, 0 | 20260505 09:01:09.527 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 09:01:09.527 | INFO | Uploaded 36GiB to GPU in 2.25554s (15.9607GiB/s) | 20260505 09:01:09.528 | INFO | Downloaded 38GiB to GPU in 2.36737s (16.0516GiB/s) | 20260505 09:01:14.667 | INFO | Analyzing chunk frames = [1970688,1971200) | 20260505 09:01:14.667 | INFO | MemInfo: { | 20260505 09:01:14.667 | INFO | "gpu" : " 17 GB", | 20260505 09:01:14.667 | INFO | "huge" : "", | 20260505 09:01:14.667 | INFO | "huge_shared" : " 27 GB", | 20260505 09:01:14.667 | INFO | "malloc" : " 14 GB", | 20260505 09:01:14.667 | INFO | "pinned" : " 29 GB" | 20260505 09:01:14.667 | INFO | } | 20260505 09:01:14.667 | INFO | Perf report for the last 5.14004 seconds: | 20260505 09:01:14.667 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:01:14.667 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:14.667 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:14.667 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:14.667 | INFO | ANALYSIS : 64.6%, 3.0, 100.9ms, 0 | 20260505 09:01:14.667 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:14.667 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:14.667 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 09:01:14.667 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:14.667 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:14.667 | INFO | Uploaded 36GiB to GPU in 2.25144s (15.9898GiB/s) | 20260505 09:01:14.667 | INFO | Downloaded 34GiB to GPU in 2.1059s (16.1451GiB/s) | 20260505 09:01:19.806 | INFO | Analyzing chunk frames = [1971200,1971712) | 20260505 09:01:19.806 | INFO | MemInfo: { | 20260505 09:01:19.806 | INFO | "gpu" : " 17 GB", | 20260505 09:01:19.806 | INFO | "huge" : "", | 20260505 09:01:19.806 | INFO | "huge_shared" : " 27 GB", | 20260505 09:01:19.806 | INFO | "malloc" : " 14 GB", | 20260505 09:01:19.806 | INFO | "pinned" : " 29 GB" | 20260505 09:01:19.806 | INFO | } | 20260505 09:01:19.806 | INFO | Perf report for the last 5.13877 seconds: | 20260505 09:01:19.806 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:01:19.806 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:19.806 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:19.806 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:19.806 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 09:01:19.806 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:19.806 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:19.806 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 09:01:19.806 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:19.806 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:19.806 | INFO | Uploaded 36GiB to GPU in 2.1636s (16.6389GiB/s) | 20260505 09:01:19.806 | INFO | Downloaded 34GiB to GPU in 2.06562s (16.4599GiB/s) | 20260505 09:01:24.945 | INFO | Analyzing chunk frames = [1971712,1972224) | 20260505 09:01:24.945 | INFO | MemInfo: { | 20260505 09:01:24.945 | INFO | "gpu" : " 17 GB", | 20260505 09:01:24.945 | INFO | "huge" : "", | 20260505 09:01:24.945 | INFO | "huge_shared" : " 27 GB", | 20260505 09:01:24.945 | INFO | "malloc" : " 14 GB", | 20260505 09:01:24.945 | INFO | "pinned" : " 29 GB" | 20260505 09:01:24.945 | INFO | } | 20260505 09:01:24.945 | INFO | Perf report for the last 5.13874 seconds: | 20260505 09:01:24.945 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:01:24.945 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:24.945 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:24.945 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:24.945 | INFO | ANALYSIS : 63.5%, 3.0, 98.6ms, 0 | 20260505 09:01:24.945 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:24.945 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:24.945 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 09:01:24.945 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:24.945 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:24.945 | INFO | Uploaded 36GiB to GPU in 2.17013s (16.5889GiB/s) | 20260505 09:01:24.945 | INFO | Downloaded 34GiB to GPU in 2.06796s (16.4413GiB/s) | 20260505 09:01:30.084 | INFO | Analyzing chunk frames = [1972224,1972736) | 20260505 09:01:30.084 | INFO | MemInfo: { | 20260505 09:01:30.084 | INFO | "gpu" : " 17 GB", | 20260505 09:01:30.084 | INFO | "huge" : "", | 20260505 09:01:30.084 | INFO | "huge_shared" : " 27 GB", | 20260505 09:01:30.084 | INFO | "malloc" : " 14 GB", | 20260505 09:01:30.084 | INFO | "pinned" : " 29 GB" | 20260505 09:01:30.084 | INFO | } | 20260505 09:01:30.084 | INFO | Perf report for the last 5.13967 seconds: | 20260505 09:01:30.085 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:01:30.085 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:30.085 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:30.085 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:30.085 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 09:01:30.085 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:30.085 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:30.085 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 09:01:30.085 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:30.085 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:30.085 | INFO | Uploaded 36GiB to GPU in 2.23638s (16.0974GiB/s) | 20260505 09:01:30.085 | INFO | Downloaded 34GiB to GPU in 2.107s (16.1367GiB/s) | 20260505 09:01:35.225 | INFO | Analyzing chunk frames = [1972736,1973248) | 20260505 09:01:35.225 | INFO | MemInfo: { | 20260505 09:01:35.225 | INFO | "gpu" : " 17 GB", | 20260505 09:01:35.225 | INFO | "huge" : "", | 20260505 09:01:35.225 | INFO | "huge_shared" : " 27 GB", | 20260505 09:01:35.225 | INFO | "malloc" : " 14 GB", | 20260505 09:01:35.225 | INFO | "pinned" : " 29 GB" | 20260505 09:01:35.225 | INFO | } | 20260505 09:01:35.225 | INFO | Perf report for the last 5.14095 seconds: | 20260505 09:01:35.226 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:01:35.226 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:35.226 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:35.226 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:35.226 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 09:01:35.226 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:35.226 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:35.226 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 09:01:35.226 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:35.226 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:35.226 | INFO | Uploaded 36GiB to GPU in 2.23963s (16.0741GiB/s) | 20260505 09:01:35.226 | INFO | Downloaded 34GiB to GPU in 2.11071s (16.1083GiB/s) | 20260505 09:01:40.363 | INFO | Analyzing chunk frames = [1973248,1973760) | 20260505 09:01:40.363 | INFO | MemInfo: { | 20260505 09:01:40.363 | INFO | "gpu" : " 17 GB", | 20260505 09:01:40.363 | INFO | "huge" : "", | 20260505 09:01:40.363 | INFO | "huge_shared" : " 27 GB", | 20260505 09:01:40.363 | INFO | "malloc" : " 14 GB", | 20260505 09:01:40.363 | INFO | "pinned" : " 29 GB" | 20260505 09:01:40.364 | INFO | } | 20260505 09:01:40.364 | INFO | Perf report for the last 5.13816 seconds: | 20260505 09:01:40.364 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:01:40.364 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:40.364 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:40.364 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:40.364 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 09:01:40.364 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:40.364 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:40.364 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 09:01:40.364 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:40.364 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:40.364 | INFO | Uploaded 36GiB to GPU in 2.17695s (16.5369GiB/s) | 20260505 09:01:40.364 | INFO | Downloaded 34GiB to GPU in 2.15713s (15.7617GiB/s) | 20260505 09:01:45.503 | INFO | Analyzing chunk frames = [1973760,1974272) | 20260505 09:01:45.503 | INFO | MemInfo: { | 20260505 09:01:45.503 | INFO | "gpu" : " 17 GB", | 20260505 09:01:45.503 | INFO | "huge" : "", | 20260505 09:01:45.503 | INFO | "huge_shared" : " 27 GB", | 20260505 09:01:45.503 | INFO | "malloc" : " 14 GB", | 20260505 09:01:45.503 | INFO | "pinned" : " 29 GB" | 20260505 09:01:45.503 | INFO | } | 20260505 09:01:45.503 | INFO | Perf report for the last 5.13925 seconds: | 20260505 09:01:45.503 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:01:45.503 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:45.503 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:45.503 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:45.503 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 09:01:45.503 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:45.503 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:45.503 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 09:01:45.503 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:45.503 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:45.503 | INFO | Uploaded 36GiB to GPU in 2.16582s (16.6219GiB/s) | 20260505 09:01:45.503 | INFO | Downloaded 34GiB to GPU in 2.02923s (16.7551GiB/s) | 20260505 09:01:45.503 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 09:01:45.503 | INFO | Total mem usage: 15.3 GiB | 20260505 09:01:45.503 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 09:01:45.503 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.36%) | 20260505 09:01:45.503 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 09:01:45.503 | INFO | High Water Breakdown: | 20260505 09:01:45.503 | INFO | 60.9% : 8.78 GiB ( 93.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 09:01:45.503 | INFO | 38.9% : 5.61 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 09:01:45.504 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 09:01:45.504 | INFO | ------------------------------------------ | 20260505 09:01:45.504 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:01:45.504 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 09:01:45.504 | INFO | Total mem usage: 26.5 GiB | 20260505 09:01:45.504 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 09:01:45.504 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 09:01:45.504 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 09:01:45.504 | INFO | High Water Breakdown: | 20260505 09:01:45.504 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:01:45.504 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:01:45.504 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:01:45.504 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:01:45.504 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:01:45.504 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:01:45.504 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 09:01:45.504 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 09:01:45.504 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:01:45.504 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 09:01:45.504 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 09:01:45.504 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 09:01:45.504 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 09:01:45.504 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 09:01:45.504 | INFO | ------------------------------------------ | 20260505 09:01:45.504 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:01:45.504 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 09:01:45.504 | INFO | Total mem usage: 25.6 GiB | 20260505 09:01:45.504 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 09:01:45.504 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 09:01:45.504 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 09:01:45.504 | INFO | High Water Breakdown: | 20260505 09:01:45.504 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 09:01:45.504 | INFO | ------------------------------------------ | 20260505 09:01:45.504 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:01:45.504 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 09:01:45.504 | INFO | Total mem usage: 15.7 GiB | 20260505 09:01:45.504 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 09:01:45.504 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 09:01:45.504 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 09:01:45.504 | INFO | High Water Breakdown: | 20260505 09:01:45.504 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 09:01:45.504 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 09:01:45.504 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 09:01:45.504 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 09:01:45.504 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 09:01:45.504 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 09:01:45.504 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 09:01:45.504 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 09:01:45.504 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 09:01:45.504 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:01:45.504 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 09:01:45.504 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:01:45.504 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 09:01:45.504 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 09:01:45.504 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 09:01:45.504 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 09:01:45.504 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:01:45.504 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 09:01:45.504 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:01:45.504 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:01:45.504 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:01:45.504 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 09:01:45.504 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 09:01:45.504 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:01:45.504 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 09:01:45.504 | INFO | ------------------------------------------ | 20260505 09:01:45.504 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:01:45.504 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 09:01:45.504 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:01:45.504 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:01:45.504 | INFO | PerfReport: Full Execution -- 3.25e+01 / 7.58e+01, 1.37e+01 / 1.82e+00 | 20260505 09:01:45.505 | INFO | PerfReport: 0.29%, AggregateStats -- 9.33e-02 / 1.05e-01, 1.08e-02 / 4.81e-03 | 20260505 09:01:45.505 | INFO | PerfReport: 11.67%, Baseline -- 3.80e+00 / 4.41e+00, 6.23e-01 / 3.30e-01 | 20260505 09:01:45.505 | INFO | PerfReport: 3.32%, Histogram -- 1.08e+00 / 1.47e+00, 2.10e-01 / 1.01e-01 | 20260505 09:01:45.505 | INFO | PerfReport: 15.48%, DME -- 4.03e+01 / 4.38e+01, 4.02e+00 / 1.54e+00 | 20260505 09:01:45.505 | INFO | PerfReport: 56.55%, FrameLabeling -- 1.84e+01 / 2.10e+01, 1.89e+00 / 2.03e+00 | 20260505 09:01:45.505 | INFO | PerfReport: 7.44%, PulseAccumulating -- 2.42e+00 / 2.95e+00, 4.04e-01 / 2.29e-01 | 20260505 09:01:45.505 | INFO | PerfReport: 3.87%, Metrics -- 1.26e+00 / 1.56e+00, 3.60e-01 / 1.89e-01 | 20260505 09:01:45.505 | INFO | PerfReport: 1.38%, PulseSerializing -- 4.48e-01 / 5.01e-01, 8.48e-02 / 2.38e-02 | 20260505 09:01:45.505 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 09:01:45.505 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:01:45.505 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:01:45.505 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.34e+01 / 0.00e+00 | 20260505 09:01:45.505 | INFO | PerfReport: 22.65%, Upload -- 2.30e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260505 09:01:45.505 | INFO | PerfReport: 21.76%, Download -- 2.21e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260505 09:01:45.505 | INFO | PerfReport: 32.02%, Compute -- 3.25e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 09:01:45.505 | INFO | PerfReport: 1.80%, LockUpload -- 1.83e+00 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260505 09:01:45.505 | INFO | PerfReport: 1.03%, LockDownload -- 1.05e+00 / 0.00e+00, 2.53e+00 / 0.00e+00 | 20260505 09:01:45.505 | INFO | PerfReport: 20.74%, LockCompute -- 2.11e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260505 09:01:45.505 | INFO | PerfReport: 0.01%, Misc -- 1.49e-02 / 0.00e+00, 2.72e-02 / 0.00e+00 | 20260505 09:01:45.639 | INFO | Starting a round of Baz IO | 20260505 09:01:48.916 | INFO | Populating RTMetrics report | 20260505 09:01:48.916 | INFO | RTMetrics report populated | 20260505 09:01:48.916 | INFO | Opening tmp RTMetrics json file | 20260505 09:01:48.928 | INFO | Serializing RTMetrics to json | 20260505 09:01:48.932 | INFO | Serialization done, swapping tmp file now... | 20260505 09:01:48.935 | INFO | RTMetrics done | 20260505 09:01:49.068 | INFO | Finished a round of Baz IO: Wrote 2.447 GiB in 3.428 seconds | 20260505 09:01:49.068 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 26.68% / 0.02% / 73.30% | 20260505 09:01:49.068 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.24065% / 3.57628e-05% / 6.35783e-05% | 20260505 09:01:49.068 | INFO | Baz IO overall: | 20260505 09:01:49.068 | INFO | BytesWritten : 1130275754876 | 20260505 09:01:49.068 | INFO | NumEvents : 537653480806 | 20260505 09:01:50.642 | INFO | Analyzing chunk frames = [1974272,1974784) | 20260505 09:01:50.642 | INFO | MemInfo: { | 20260505 09:01:50.642 | INFO | "gpu" : " 17 GB", | 20260505 09:01:50.642 | INFO | "huge" : "", | 20260505 09:01:50.642 | INFO | "huge_shared" : " 27 GB", | 20260505 09:01:50.642 | INFO | "malloc" : " 14 GB", | 20260505 09:01:50.642 | INFO | "pinned" : " 29 GB" | 20260505 09:01:50.642 | INFO | } | 20260505 09:01:50.642 | INFO | Perf report for the last 5.13903 seconds: | 20260505 09:01:50.642 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:01:50.642 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:50.642 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:50.642 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:50.642 | INFO | ANALYSIS : 65.3%, 2.9, 99.5ms, 0 | 20260505 09:01:50.642 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:50.642 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260505 09:01:50.642 | INFO | PRE_HQ : 63.2%, 1.9, 61.0ms, 0 | 20260505 09:01:50.642 | INFO | BAZWRITER : 66.0%, 4.2, 141.1ms, 0 | 20260505 09:01:50.642 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 09:01:50.642 | INFO | Uploaded 36GiB to GPU in 2.28022s (15.788GiB/s) | 20260505 09:01:50.642 | INFO | Downloaded 38GiB to GPU in 2.37134s (16.0247GiB/s) | 20260505 09:01:55.781 | INFO | Analyzing chunk frames = [1974784,1975296) | 20260505 09:01:55.781 | INFO | MemInfo: { | 20260505 09:01:55.781 | INFO | "gpu" : " 17 GB", | 20260505 09:01:55.781 | INFO | "huge" : "", | 20260505 09:01:55.781 | INFO | "huge_shared" : " 27 GB", | 20260505 09:01:55.781 | INFO | "malloc" : " 14 GB", | 20260505 09:01:55.781 | INFO | "pinned" : " 29 GB" | 20260505 09:01:55.781 | INFO | } | 20260505 09:01:55.781 | INFO | Perf report for the last 5.13957 seconds: | 20260505 09:01:55.782 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:01:55.782 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:55.782 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:55.782 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:55.782 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 09:01:55.782 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:55.782 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:55.782 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 09:01:55.782 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:55.782 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:01:55.782 | INFO | Uploaded 36GiB to GPU in 2.2246s (16.1827GiB/s) | 20260505 09:01:55.782 | INFO | Downloaded 34GiB to GPU in 2.08791s (16.2842GiB/s) | 20260505 09:02:00.920 | INFO | Analyzing chunk frames = [1975296,1975808) | 20260505 09:02:00.920 | INFO | MemInfo: { | 20260505 09:02:00.920 | INFO | "gpu" : " 17 GB", | 20260505 09:02:00.920 | INFO | "huge" : "", | 20260505 09:02:00.920 | INFO | "huge_shared" : " 27 GB", | 20260505 09:02:00.920 | INFO | "malloc" : " 14 GB", | 20260505 09:02:00.920 | INFO | "pinned" : " 29 GB" | 20260505 09:02:00.920 | INFO | } | 20260505 09:02:00.921 | INFO | Perf report for the last 5.13907 seconds: | 20260505 09:02:00.921 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:02:00.921 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:00.921 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:00.921 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:00.921 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260505 09:02:00.921 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:00.921 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:00.921 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 09:02:00.921 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:00.921 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:00.921 | INFO | Uploaded 36GiB to GPU in 2.17628s (16.542GiB/s) | 20260505 09:02:00.921 | INFO | Downloaded 34GiB to GPU in 2.06855s (16.4366GiB/s) | 20260505 09:02:06.060 | INFO | Analyzing chunk frames = [1975808,1976320) | 20260505 09:02:06.060 | INFO | MemInfo: { | 20260505 09:02:06.060 | INFO | "gpu" : " 17 GB", | 20260505 09:02:06.060 | INFO | "huge" : "", | 20260505 09:02:06.060 | INFO | "huge_shared" : " 27 GB", | 20260505 09:02:06.060 | INFO | "malloc" : " 14 GB", | 20260505 09:02:06.060 | INFO | "pinned" : " 29 GB" | 20260505 09:02:06.060 | INFO | } | 20260505 09:02:06.060 | INFO | Perf report for the last 5.13951 seconds: | 20260505 09:02:06.060 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:02:06.060 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:06.060 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:06.060 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:06.060 | INFO | ANALYSIS : 63.5%, 3.0, 98.6ms, 0 | 20260505 09:02:06.060 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:06.060 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:06.060 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 09:02:06.060 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:06.060 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:06.060 | INFO | Uploaded 36GiB to GPU in 2.17747s (16.5329GiB/s) | 20260505 09:02:06.060 | INFO | Downloaded 34GiB to GPU in 2.06567s (16.4595GiB/s) | 20260505 09:02:11.200 | INFO | Analyzing chunk frames = [1976320,1976832) | 20260505 09:02:11.200 | INFO | MemInfo: { | 20260505 09:02:11.200 | INFO | "gpu" : " 17 GB", | 20260505 09:02:11.200 | INFO | "huge" : "", | 20260505 09:02:11.200 | INFO | "huge_shared" : " 27 GB", | 20260505 09:02:11.200 | INFO | "malloc" : " 14 GB", | 20260505 09:02:11.200 | INFO | "pinned" : " 29 GB" | 20260505 09:02:11.200 | INFO | } | 20260505 09:02:11.200 | INFO | Perf report for the last 5.13958 seconds: | 20260505 09:02:11.200 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:02:11.200 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:11.200 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:11.200 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:11.200 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 09:02:11.200 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:11.200 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:11.200 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 09:02:11.200 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:11.200 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:11.200 | INFO | Uploaded 36GiB to GPU in 2.23432s (16.1123GiB/s) | 20260505 09:02:11.200 | INFO | Downloaded 34GiB to GPU in 2.10935s (16.1187GiB/s) | 20260505 09:02:16.340 | INFO | Analyzing chunk frames = [1976832,1977344) | 20260505 09:02:16.340 | INFO | MemInfo: { | 20260505 09:02:16.340 | INFO | "gpu" : " 17 GB", | 20260505 09:02:16.340 | INFO | "huge" : "", | 20260505 09:02:16.340 | INFO | "huge_shared" : " 27 GB", | 20260505 09:02:16.340 | INFO | "malloc" : " 14 GB", | 20260505 09:02:16.340 | INFO | "pinned" : " 29 GB" | 20260505 09:02:16.340 | INFO | } | 20260505 09:02:16.341 | INFO | Perf report for the last 5.14084 seconds: | 20260505 09:02:16.341 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:02:16.341 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:16.341 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:16.341 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:16.341 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 09:02:16.341 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:16.341 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:16.341 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:02:16.341 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:16.341 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:16.341 | INFO | Uploaded 36GiB to GPU in 2.17175s (16.5765GiB/s) | 20260505 09:02:16.341 | INFO | Downloaded 34GiB to GPU in 2.06044s (16.5013GiB/s) | 20260505 09:02:21.478 | INFO | Analyzing chunk frames = [1977344,1977856) | 20260505 09:02:21.478 | INFO | MemInfo: { | 20260505 09:02:21.478 | INFO | "gpu" : " 17 GB", | 20260505 09:02:21.478 | INFO | "huge" : "", | 20260505 09:02:21.478 | INFO | "huge_shared" : " 27 GB", | 20260505 09:02:21.478 | INFO | "malloc" : " 14 GB", | 20260505 09:02:21.478 | INFO | "pinned" : " 29 GB" | 20260505 09:02:21.478 | INFO | } | 20260505 09:02:21.478 | INFO | Perf report for the last 5.13768 seconds: | 20260505 09:02:21.478 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:02:21.478 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:21.478 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:21.478 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:21.478 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 09:02:21.478 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:21.478 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:21.478 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 09:02:21.478 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:21.478 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:21.479 | INFO | Uploaded 36GiB to GPU in 2.1845s (16.4798GiB/s) | 20260505 09:02:21.479 | INFO | Downloaded 34GiB to GPU in 2.15013s (15.813GiB/s) | 20260505 09:02:26.617 | INFO | Analyzing chunk frames = [1977856,1978368) | 20260505 09:02:26.617 | INFO | MemInfo: { | 20260505 09:02:26.617 | INFO | "gpu" : " 17 GB", | 20260505 09:02:26.617 | INFO | "huge" : "", | 20260505 09:02:26.617 | INFO | "huge_shared" : " 27 GB", | 20260505 09:02:26.617 | INFO | "malloc" : " 14 GB", | 20260505 09:02:26.617 | INFO | "pinned" : " 29 GB" | 20260505 09:02:26.617 | INFO | } | 20260505 09:02:26.618 | INFO | Perf report for the last 5.13939 seconds: | 20260505 09:02:26.618 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:02:26.618 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:26.618 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:26.618 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:26.618 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 09:02:26.618 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:26.618 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:26.618 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 09:02:26.618 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:26.618 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:26.618 | INFO | Uploaded 36GiB to GPU in 2.20028s (16.3615GiB/s) | 20260505 09:02:26.618 | INFO | Downloaded 34GiB to GPU in 2.04178s (16.6521GiB/s) | 20260505 09:02:26.755 | INFO | Starting a round of Baz IO | 20260505 09:02:30.035 | INFO | Populating RTMetrics report | 20260505 09:02:30.035 | INFO | RTMetrics report populated | 20260505 09:02:30.035 | INFO | Opening tmp RTMetrics json file | 20260505 09:02:30.063 | INFO | Serializing RTMetrics to json | 20260505 09:02:30.084 | INFO | Serialization done, swapping tmp file now... | 20260505 09:02:30.085 | INFO | RTMetrics done | 20260505 09:02:30.209 | INFO | Finished a round of Baz IO: Wrote 2.446 GiB in 3.454 seconds | 20260505 09:02:30.209 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 26.66% / 0.02% / 73.32% | 20260505 09:02:30.209 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.23348% / 2.38419e-05% / 5.16574e-05% | 20260505 09:02:30.209 | INFO | Baz IO overall: | 20260505 09:02:30.209 | INFO | BytesWritten : 1132901990820 | 20260505 09:02:30.209 | INFO | NumEvents : 538909889397 | 20260505 09:02:31.757 | INFO | Analyzing chunk frames = [1978368,1978880) | 20260505 09:02:31.757 | INFO | MemInfo: { | 20260505 09:02:31.757 | INFO | "gpu" : " 17 GB", | 20260505 09:02:31.757 | INFO | "huge" : "", | 20260505 09:02:31.757 | INFO | "huge_shared" : " 27 GB", | 20260505 09:02:31.757 | INFO | "malloc" : " 14 GB", | 20260505 09:02:31.757 | INFO | "pinned" : " 29 GB" | 20260505 09:02:31.757 | INFO | } | 20260505 09:02:31.757 | INFO | Perf report for the last 5.13918 seconds: | 20260505 09:02:31.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:02:31.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:31.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:31.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:31.757 | INFO | ANALYSIS : 65.4%, 2.9, 100.0ms, 0 | 20260505 09:02:31.757 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:31.757 | INFO | RT_METRICS : 47.0%, 1.0, 24.6ms, 0 | 20260505 09:02:31.757 | INFO | PRE_HQ : 63.5%, 1.8, 60.9ms, 0 | 20260505 09:02:31.757 | INFO | BAZWRITER : 66.5%, 4.2, 142.0ms, 0 | 20260505 09:02:31.757 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260505 09:02:31.757 | INFO | Uploaded 36GiB to GPU in 2.27858s (15.7993GiB/s) | 20260505 09:02:31.757 | INFO | Downloaded 38GiB to GPU in 2.37685s (15.9876GiB/s) | 20260505 09:02:36.896 | INFO | Analyzing chunk frames = [1978880,1979392) | 20260505 09:02:36.896 | INFO | MemInfo: { | 20260505 09:02:36.896 | INFO | "gpu" : " 17 GB", | 20260505 09:02:36.896 | INFO | "huge" : "", | 20260505 09:02:36.896 | INFO | "huge_shared" : " 27 GB", | 20260505 09:02:36.896 | INFO | "malloc" : " 14 GB", | 20260505 09:02:36.897 | INFO | "pinned" : " 29 GB" | 20260505 09:02:36.897 | INFO | } | 20260505 09:02:36.897 | INFO | Perf report for the last 5.13993 seconds: | 20260505 09:02:36.897 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:02:36.897 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:36.897 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:36.897 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:36.897 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 09:02:36.897 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:36.897 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:36.897 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 09:02:36.897 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:36.897 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:36.897 | INFO | Uploaded 36GiB to GPU in 2.20698s (16.3119GiB/s) | 20260505 09:02:36.897 | INFO | Downloaded 34GiB to GPU in 2.08503s (16.3067GiB/s) | 20260505 09:02:42.039 | INFO | Analyzing chunk frames = [1979392,1979904) | 20260505 09:02:42.039 | INFO | MemInfo: { | 20260505 09:02:42.039 | INFO | "gpu" : " 17 GB", | 20260505 09:02:42.039 | INFO | "huge" : "", | 20260505 09:02:42.039 | INFO | "huge_shared" : " 27 GB", | 20260505 09:02:42.039 | INFO | "malloc" : " 14 GB", | 20260505 09:02:42.039 | INFO | "pinned" : " 29 GB" | 20260505 09:02:42.039 | INFO | } | 20260505 09:02:42.039 | INFO | Perf report for the last 5.14196 seconds: | 20260505 09:02:42.039 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:02:42.039 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:42.039 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:42.039 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:42.039 | INFO | ANALYSIS : 63.4%, 3.0, 98.8ms, 0 | 20260505 09:02:42.039 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:42.039 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:42.039 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:02:42.039 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:42.039 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:42.039 | INFO | Uploaded 36GiB to GPU in 2.16575s (16.6224GiB/s) | 20260505 09:02:42.039 | INFO | Downloaded 34GiB to GPU in 2.06685s (16.4502GiB/s) | 20260505 09:02:47.174 | INFO | Analyzing chunk frames = [1979904,1980416) | 20260505 09:02:47.174 | INFO | MemInfo: { | 20260505 09:02:47.174 | INFO | "gpu" : " 17 GB", | 20260505 09:02:47.174 | INFO | "huge" : "", | 20260505 09:02:47.174 | INFO | "huge_shared" : " 27 GB", | 20260505 09:02:47.174 | INFO | "malloc" : " 14 GB", | 20260505 09:02:47.174 | INFO | "pinned" : " 29 GB" | 20260505 09:02:47.174 | INFO | } | 20260505 09:02:47.175 | INFO | Perf report for the last 5.13584 seconds: | 20260505 09:02:47.175 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:02:47.175 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:47.175 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:47.175 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:47.175 | INFO | ANALYSIS : 63.4%, 3.0, 98.4ms, 0 | 20260505 09:02:47.175 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:47.175 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:47.175 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 09:02:47.175 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:47.175 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:47.175 | INFO | Uploaded 36GiB to GPU in 2.16561s (16.6235GiB/s) | 20260505 09:02:47.175 | INFO | Downloaded 34GiB to GPU in 2.07223s (16.4075GiB/s) | 20260505 09:02:52.314 | INFO | Analyzing chunk frames = [1980416,1980928) | 20260505 09:02:52.314 | INFO | MemInfo: { | 20260505 09:02:52.314 | INFO | "gpu" : " 17 GB", | 20260505 09:02:52.314 | INFO | "huge" : "", | 20260505 09:02:52.314 | INFO | "huge_shared" : " 27 GB", | 20260505 09:02:52.314 | INFO | "malloc" : " 14 GB", | 20260505 09:02:52.314 | INFO | "pinned" : " 29 GB" | 20260505 09:02:52.314 | INFO | } | 20260505 09:02:52.314 | INFO | Perf report for the last 5.13953 seconds: | 20260505 09:02:52.314 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:02:52.314 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:52.314 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:52.314 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:52.314 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 09:02:52.314 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:52.314 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:52.314 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 09:02:52.314 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:52.314 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:52.314 | INFO | Uploaded 36GiB to GPU in 2.23677s (16.0946GiB/s) | 20260505 09:02:52.314 | INFO | Downloaded 34GiB to GPU in 2.11253s (16.0944GiB/s) | 20260505 09:02:57.454 | INFO | Analyzing chunk frames = [1980928,1981440) | 20260505 09:02:57.454 | INFO | MemInfo: { | 20260505 09:02:57.454 | INFO | "gpu" : " 17 GB", | 20260505 09:02:57.454 | INFO | "huge" : "", | 20260505 09:02:57.454 | INFO | "huge_shared" : " 27 GB", | 20260505 09:02:57.454 | INFO | "malloc" : " 14 GB", | 20260505 09:02:57.454 | INFO | "pinned" : " 29 GB" | 20260505 09:02:57.454 | INFO | } | 20260505 09:02:57.454 | INFO | Perf report for the last 5.14036 seconds: | 20260505 09:02:57.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:02:57.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:57.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:57.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:57.454 | INFO | ANALYSIS : 64.1%, 3.0, 100.2ms, 0 | 20260505 09:02:57.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:57.454 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:57.455 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:02:57.455 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:57.455 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:02:57.455 | INFO | Uploaded 36GiB to GPU in 2.24817s (16.013GiB/s) | 20260505 09:02:57.455 | INFO | Downloaded 34GiB to GPU in 2.11798s (16.053GiB/s) | 20260505 09:03:02.593 | INFO | Analyzing chunk frames = [1981440,1981952) | 20260505 09:03:02.593 | INFO | MemInfo: { | 20260505 09:03:02.593 | INFO | "gpu" : " 17 GB", | 20260505 09:03:02.593 | INFO | "huge" : "", | 20260505 09:03:02.593 | INFO | "huge_shared" : " 27 GB", | 20260505 09:03:02.593 | INFO | "malloc" : " 14 GB", | 20260505 09:03:02.593 | INFO | "pinned" : " 29 GB" | 20260505 09:03:02.593 | INFO | } | 20260505 09:03:02.593 | INFO | Perf report for the last 5.13835 seconds: | 20260505 09:03:02.593 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:03:02.593 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:02.593 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:02.593 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:02.593 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 09:03:02.593 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:02.593 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:02.593 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 09:03:02.593 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:02.593 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:02.593 | INFO | Uploaded 36GiB to GPU in 2.23083s (16.1375GiB/s) | 20260505 09:03:02.593 | INFO | Downloaded 34GiB to GPU in 2.14954s (15.8173GiB/s) | 20260505 09:03:07.732 | INFO | Analyzing chunk frames = [1981952,1982464) | 20260505 09:03:07.732 | INFO | MemInfo: { | 20260505 09:03:07.732 | INFO | "gpu" : " 17 GB", | 20260505 09:03:07.732 | INFO | "huge" : "", | 20260505 09:03:07.732 | INFO | "huge_shared" : " 27 GB", | 20260505 09:03:07.732 | INFO | "malloc" : " 14 GB", | 20260505 09:03:07.732 | INFO | "pinned" : " 29 GB" | 20260505 09:03:07.732 | INFO | } | 20260505 09:03:07.732 | INFO | Perf report for the last 5.13946 seconds: | 20260505 09:03:07.732 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:03:07.732 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:07.732 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:07.732 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:07.732 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 09:03:07.732 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:07.732 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:07.732 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 09:03:07.732 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:07.732 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:07.733 | INFO | Uploaded 36GiB to GPU in 2.17776s (16.5308GiB/s) | 20260505 09:03:07.733 | INFO | Downloaded 34GiB to GPU in 2.03181s (16.7339GiB/s) | 20260505 09:03:07.733 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 09:03:07.733 | INFO | Total mem usage: 15.3 GiB | 20260505 09:03:07.733 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 09:03:07.733 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.18%) | 20260505 09:03:07.733 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 09:03:07.733 | INFO | High Water Breakdown: | 20260505 09:03:07.733 | INFO | 60.8% : 8.78 GiB ( 93.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 09:03:07.733 | INFO | 39% : 5.64 GiB ( 84.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 09:03:07.733 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 09:03:07.733 | INFO | ------------------------------------------ | 20260505 09:03:07.733 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:03:07.733 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 09:03:07.733 | INFO | Total mem usage: 26.5 GiB | 20260505 09:03:07.733 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 09:03:07.733 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 09:03:07.733 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 09:03:07.733 | INFO | High Water Breakdown: | 20260505 09:03:07.733 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:03:07.733 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:03:07.733 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:03:07.733 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:03:07.733 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:03:07.733 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:03:07.733 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 09:03:07.733 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 09:03:07.733 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:03:07.733 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 09:03:07.733 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 09:03:07.733 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 09:03:07.733 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 09:03:07.733 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 09:03:07.733 | INFO | ------------------------------------------ | 20260505 09:03:07.733 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:03:07.733 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 09:03:07.733 | INFO | Total mem usage: 25.6 GiB | 20260505 09:03:07.733 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 09:03:07.733 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 09:03:07.733 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 09:03:07.733 | INFO | High Water Breakdown: | 20260505 09:03:07.733 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 09:03:07.733 | INFO | ------------------------------------------ | 20260505 09:03:07.733 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:03:07.733 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 09:03:07.733 | INFO | Total mem usage: 15.7 GiB | 20260505 09:03:07.733 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 09:03:07.733 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 09:03:07.733 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 09:03:07.733 | INFO | High Water Breakdown: | 20260505 09:03:07.733 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 09:03:07.733 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 09:03:07.733 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 09:03:07.733 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 09:03:07.733 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 09:03:07.733 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 09:03:07.733 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 09:03:07.733 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 09:03:07.733 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 09:03:07.733 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:03:07.733 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 09:03:07.733 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:03:07.733 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 09:03:07.733 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 09:03:07.734 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 09:03:07.734 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 09:03:07.734 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:03:07.734 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 09:03:07.734 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:03:07.734 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:03:07.734 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:03:07.734 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 09:03:07.734 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 09:03:07.734 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:03:07.734 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 09:03:07.734 | INFO | ------------------------------------------ | 20260505 09:03:07.734 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:03:07.734 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 09:03:07.734 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:03:07.734 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:03:07.734 | INFO | PerfReport: Full Execution -- 3.25e+01 / 7.58e+01, 1.36e+01 / 2.03e+00 | 20260505 09:03:07.734 | INFO | PerfReport: 0.29%, AggregateStats -- 9.31e-02 / 1.04e-01, 1.03e-02 / 4.49e-03 | 20260505 09:03:07.734 | INFO | PerfReport: 11.64%, Baseline -- 3.79e+00 / 4.41e+00, 6.17e-01 / 3.44e-01 | 20260505 09:03:07.734 | INFO | PerfReport: 3.31%, Histogram -- 1.08e+00 / 1.47e+00, 2.09e-01 / 1.16e-01 | 20260505 09:03:07.734 | INFO | PerfReport: 15.50%, DME -- 4.04e+01 / 4.40e+01, 4.02e+00 / 1.86e+00 | 20260505 09:03:07.734 | INFO | PerfReport: 56.59%, FrameLabeling -- 1.84e+01 / 2.09e+01, 1.92e+00 / 2.18e+00 | 20260505 09:03:07.734 | INFO | PerfReport: 7.41%, PulseAccumulating -- 2.41e+00 / 2.93e+00, 4.03e-01 / 2.92e-01 | 20260505 09:03:07.734 | INFO | PerfReport: 3.87%, Metrics -- 1.26e+00 / 1.56e+00, 3.67e-01 / 2.36e-01 | 20260505 09:03:07.734 | INFO | PerfReport: 1.37%, PulseSerializing -- 4.47e-01 / 4.97e-01, 8.49e-02 / 4.26e-02 | 20260505 09:03:07.734 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 09:03:07.734 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:03:07.734 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:03:07.734 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.35e+01 / 0.00e+00 | 20260505 09:03:07.734 | INFO | PerfReport: 22.65%, Upload -- 2.30e+01 / 0.00e+00, 8.37e+00 / 0.00e+00 | 20260505 09:03:07.734 | INFO | PerfReport: 21.76%, Download -- 2.21e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260505 09:03:07.734 | INFO | PerfReport: 32.02%, Compute -- 3.25e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 09:03:07.734 | INFO | PerfReport: 1.79%, LockUpload -- 1.82e+00 / 0.00e+00, 7.45e+00 / 0.00e+00 | 20260505 09:03:07.734 | INFO | PerfReport: 1.05%, LockDownload -- 1.07e+00 / 0.00e+00, 2.57e+00 / 0.00e+00 | 20260505 09:03:07.734 | INFO | PerfReport: 20.73%, LockCompute -- 2.11e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260505 09:03:07.734 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.85e-02 / 0.00e+00 | 20260505 09:03:07.870 | INFO | Starting a round of Baz IO | 20260505 09:03:11.151 | INFO | Populating RTMetrics report | 20260505 09:03:11.152 | INFO | RTMetrics report populated | 20260505 09:03:11.152 | INFO | Opening tmp RTMetrics json file | 20260505 09:03:11.160 | INFO | Serializing RTMetrics to json | 20260505 09:03:11.166 | INFO | Serialization done, swapping tmp file now... | 20260505 09:03:11.180 | INFO | RTMetrics done | 20260505 09:03:11.300 | INFO | Finished a round of Baz IO: Wrote 2.443 GiB in 3.429 seconds | 20260505 09:03:11.300 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 26.65% / 0.02% / 73.34% | 20260505 09:03:11.300 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.21503% / 1.98682e-05% / 9.53674e-05% | 20260505 09:03:11.300 | INFO | Baz IO overall: | 20260505 09:03:11.300 | INFO | BytesWritten : 1135524807966 | 20260505 09:03:11.300 | INFO | NumEvents : 540163946021 | 20260505 09:03:12.871 | INFO | Analyzing chunk frames = [1982464,1982976) | 20260505 09:03:12.871 | INFO | MemInfo: { | 20260505 09:03:12.871 | INFO | "gpu" : " 17 GB", | 20260505 09:03:12.871 | INFO | "huge" : "", | 20260505 09:03:12.871 | INFO | "huge_shared" : " 27 GB", | 20260505 09:03:12.871 | INFO | "malloc" : " 14 GB", | 20260505 09:03:12.871 | INFO | "pinned" : " 29 GB" | 20260505 09:03:12.871 | INFO | } | 20260505 09:03:12.871 | INFO | Perf report for the last 5.13873 seconds: | 20260505 09:03:12.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:03:12.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:12.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:12.872 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:12.872 | INFO | ANALYSIS : 65.4%, 2.9, 99.9ms, 0 | 20260505 09:03:12.872 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:12.872 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260505 09:03:12.872 | INFO | PRE_HQ : 63.1%, 1.9, 60.7ms, 0 | 20260505 09:03:12.872 | INFO | BAZWRITER : 66.0%, 4.1, 139.7ms, 0 | 20260505 09:03:12.872 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 09:03:12.872 | INFO | Uploaded 36GiB to GPU in 2.25157s (15.9889GiB/s) | 20260505 09:03:12.872 | INFO | Downloaded 38GiB to GPU in 2.34636s (16.1953GiB/s) | 20260505 09:03:18.011 | INFO | Analyzing chunk frames = [1982976,1983488) | 20260505 09:03:18.011 | INFO | MemInfo: { | 20260505 09:03:18.011 | INFO | "gpu" : " 17 GB", | 20260505 09:03:18.011 | INFO | "huge" : "", | 20260505 09:03:18.011 | INFO | "huge_shared" : " 27 GB", | 20260505 09:03:18.011 | INFO | "malloc" : " 14 GB", | 20260505 09:03:18.011 | INFO | "pinned" : " 29 GB" | 20260505 09:03:18.011 | INFO | } | 20260505 09:03:18.011 | INFO | Perf report for the last 5.14019 seconds: | 20260505 09:03:18.011 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:03:18.011 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:18.011 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:18.011 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:18.011 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 09:03:18.011 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:18.011 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:18.011 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 09:03:18.011 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:18.011 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:18.012 | INFO | Uploaded 36GiB to GPU in 2.185s (16.476GiB/s) | 20260505 09:03:18.012 | INFO | Downloaded 34GiB to GPU in 2.09501s (16.229GiB/s) | 20260505 09:03:23.150 | INFO | Analyzing chunk frames = [1983488,1984000) | 20260505 09:03:23.150 | INFO | MemInfo: { | 20260505 09:03:23.150 | INFO | "gpu" : " 17 GB", | 20260505 09:03:23.150 | INFO | "huge" : "", | 20260505 09:03:23.150 | INFO | "huge_shared" : " 27 GB", | 20260505 09:03:23.150 | INFO | "malloc" : " 14 GB", | 20260505 09:03:23.150 | INFO | "pinned" : " 29 GB" | 20260505 09:03:23.150 | INFO | } | 20260505 09:03:23.150 | INFO | Perf report for the last 5.13914 seconds: | 20260505 09:03:23.150 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:03:23.150 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:23.150 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:23.150 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:23.150 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 09:03:23.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:23.151 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:23.151 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:03:23.151 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:23.151 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:23.151 | INFO | Uploaded 36GiB to GPU in 2.16427s (16.6338GiB/s) | 20260505 09:03:23.151 | INFO | Downloaded 34GiB to GPU in 2.0664s (16.4537GiB/s) | 20260505 09:03:28.289 | INFO | Analyzing chunk frames = [1984000,1984512) | 20260505 09:03:28.289 | INFO | MemInfo: { | 20260505 09:03:28.289 | INFO | "gpu" : " 17 GB", | 20260505 09:03:28.289 | INFO | "huge" : "", | 20260505 09:03:28.289 | INFO | "huge_shared" : " 27 GB", | 20260505 09:03:28.289 | INFO | "malloc" : " 14 GB", | 20260505 09:03:28.289 | INFO | "pinned" : " 29 GB" | 20260505 09:03:28.289 | INFO | } | 20260505 09:03:28.289 | INFO | Perf report for the last 5.13903 seconds: | 20260505 09:03:28.289 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:03:28.289 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:28.289 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:28.290 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:28.290 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 09:03:28.290 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:28.290 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:28.290 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 09:03:28.290 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:28.290 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:28.290 | INFO | Uploaded 36GiB to GPU in 2.14983s (16.7455GiB/s) | 20260505 09:03:28.290 | INFO | Downloaded 34GiB to GPU in 2.06852s (16.4368GiB/s) | 20260505 09:03:33.429 | INFO | Analyzing chunk frames = [1984512,1985024) | 20260505 09:03:33.429 | INFO | MemInfo: { | 20260505 09:03:33.429 | INFO | "gpu" : " 17 GB", | 20260505 09:03:33.429 | INFO | "huge" : "", | 20260505 09:03:33.429 | INFO | "huge_shared" : " 27 GB", | 20260505 09:03:33.429 | INFO | "malloc" : " 14 GB", | 20260505 09:03:33.429 | INFO | "pinned" : " 29 GB" | 20260505 09:03:33.429 | INFO | } | 20260505 09:03:33.429 | INFO | Perf report for the last 5.13933 seconds: | 20260505 09:03:33.429 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:03:33.429 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:33.429 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:33.429 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:33.429 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 09:03:33.429 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:33.429 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:33.429 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 09:03:33.429 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:33.429 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:33.429 | INFO | Uploaded 36GiB to GPU in 2.16973s (16.5919GiB/s) | 20260505 09:03:33.429 | INFO | Downloaded 34GiB to GPU in 2.05595s (16.5374GiB/s) | 20260505 09:03:38.568 | INFO | Analyzing chunk frames = [1985024,1985536) | 20260505 09:03:38.568 | INFO | MemInfo: { | 20260505 09:03:38.568 | INFO | "gpu" : " 17 GB", | 20260505 09:03:38.568 | INFO | "huge" : "", | 20260505 09:03:38.568 | INFO | "huge_shared" : " 27 GB", | 20260505 09:03:38.568 | INFO | "malloc" : " 14 GB", | 20260505 09:03:38.568 | INFO | "pinned" : " 29 GB" | 20260505 09:03:38.568 | INFO | } | 20260505 09:03:38.569 | INFO | Perf report for the last 5.13974 seconds: | 20260505 09:03:38.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:03:38.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:38.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:38.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:38.569 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 09:03:38.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:38.569 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:38.569 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 09:03:38.569 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:38.569 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:38.569 | INFO | Uploaded 36GiB to GPU in 2.16098s (16.6591GiB/s) | 20260505 09:03:38.569 | INFO | Downloaded 34GiB to GPU in 2.05691s (16.5297GiB/s) | 20260505 09:03:43.708 | INFO | Analyzing chunk frames = [1985536,1986048) | 20260505 09:03:43.708 | INFO | MemInfo: { | 20260505 09:03:43.708 | INFO | "gpu" : " 17 GB", | 20260505 09:03:43.708 | INFO | "huge" : "", | 20260505 09:03:43.708 | INFO | "huge_shared" : " 27 GB", | 20260505 09:03:43.708 | INFO | "malloc" : " 14 GB", | 20260505 09:03:43.708 | INFO | "pinned" : " 29 GB" | 20260505 09:03:43.708 | INFO | } | 20260505 09:03:43.708 | INFO | Perf report for the last 5.1391 seconds: | 20260505 09:03:43.708 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:03:43.708 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:43.708 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:43.708 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:43.708 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 09:03:43.708 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:43.708 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:43.708 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 09:03:43.708 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:43.708 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:43.708 | INFO | Uploaded 36GiB to GPU in 2.17297s (16.5672GiB/s) | 20260505 09:03:43.708 | INFO | Downloaded 34GiB to GPU in 2.15239s (15.7964GiB/s) | 20260505 09:03:48.848 | INFO | Analyzing chunk frames = [1986048,1986560) | 20260505 09:03:48.848 | INFO | MemInfo: { | 20260505 09:03:48.848 | INFO | "gpu" : " 17 GB", | 20260505 09:03:48.848 | INFO | "huge" : "", | 20260505 09:03:48.848 | INFO | "huge_shared" : " 27 GB", | 20260505 09:03:48.848 | INFO | "malloc" : " 14 GB", | 20260505 09:03:48.848 | INFO | "pinned" : " 29 GB" | 20260505 09:03:48.848 | INFO | } | 20260505 09:03:48.848 | INFO | Perf report for the last 5.14015 seconds: | 20260505 09:03:48.848 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:03:48.848 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:48.848 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:48.848 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:48.848 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 09:03:48.848 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:48.848 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:48.848 | INFO | PRE_HQ : 29.9%, 1.0, 15.6ms, 0 | 20260505 09:03:48.848 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:48.848 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:48.848 | INFO | Uploaded 36GiB to GPU in 2.18034s (16.5112GiB/s) | 20260505 09:03:48.848 | INFO | Downloaded 34GiB to GPU in 2.02324s (16.8047GiB/s) | 20260505 09:03:48.985 | INFO | Starting a round of Baz IO | 20260505 09:03:52.253 | INFO | Populating RTMetrics report | 20260505 09:03:52.253 | INFO | RTMetrics report populated | 20260505 09:03:52.253 | INFO | Opening tmp RTMetrics json file | 20260505 09:03:52.254 | INFO | Serializing RTMetrics to json | 20260505 09:03:52.255 | INFO | Serialization done, swapping tmp file now... | 20260505 09:03:52.256 | INFO | RTMetrics done | 20260505 09:03:52.400 | INFO | Finished a round of Baz IO: Wrote 2.442 GiB in 3.415 seconds | 20260505 09:03:52.400 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 26.63% / 0.02% / 73.35% | 20260505 09:03:52.400 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.20753% / 2.78155e-05% / 6.75519e-05% | 20260505 09:03:52.400 | INFO | Baz IO overall: | 20260505 09:03:52.400 | INFO | BytesWritten : 1138146692509 | 20260505 09:03:52.401 | INFO | NumEvents : 541417382099 | 20260505 09:03:53.988 | INFO | Analyzing chunk frames = [1986560,1987072) | 20260505 09:03:53.988 | INFO | MemInfo: { | 20260505 09:03:53.988 | INFO | "gpu" : " 17 GB", | 20260505 09:03:53.988 | INFO | "huge" : "", | 20260505 09:03:53.988 | INFO | "huge_shared" : " 27 GB", | 20260505 09:03:53.988 | INFO | "malloc" : " 14 GB", | 20260505 09:03:53.988 | INFO | "pinned" : " 29 GB" | 20260505 09:03:53.988 | INFO | } | 20260505 09:03:53.988 | INFO | Perf report for the last 5.14052 seconds: | 20260505 09:03:53.988 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:03:53.988 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:53.988 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:53.988 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:53.988 | INFO | ANALYSIS : 65.1%, 2.9, 99.6ms, 0 | 20260505 09:03:53.989 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:53.989 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260505 09:03:53.989 | INFO | PRE_HQ : 63.0%, 1.8, 60.4ms, 0 | 20260505 09:03:53.989 | INFO | BAZWRITER : 65.8%, 4.2, 141.2ms, 0 | 20260505 09:03:53.989 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 09:03:53.989 | INFO | Uploaded 36GiB to GPU in 2.26898s (15.8662GiB/s) | 20260505 09:03:53.989 | INFO | Downloaded 38GiB to GPU in 2.38569s (15.9283GiB/s) | 20260505 09:03:59.126 | INFO | Analyzing chunk frames = [1987072,1987584) | 20260505 09:03:59.126 | INFO | MemInfo: { | 20260505 09:03:59.126 | INFO | "gpu" : " 17 GB", | 20260505 09:03:59.126 | INFO | "huge" : "", | 20260505 09:03:59.126 | INFO | "huge_shared" : " 27 GB", | 20260505 09:03:59.126 | INFO | "malloc" : " 14 GB", | 20260505 09:03:59.126 | INFO | "pinned" : " 29 GB" | 20260505 09:03:59.126 | INFO | } | 20260505 09:03:59.126 | INFO | Perf report for the last 5.13776 seconds: | 20260505 09:03:59.126 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:03:59.126 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:59.126 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:59.126 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:59.126 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 09:03:59.126 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:59.126 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:59.126 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:03:59.126 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:59.126 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:03:59.127 | INFO | Uploaded 36GiB to GPU in 2.21816s (16.2297GiB/s) | 20260505 09:03:59.127 | INFO | Downloaded 34GiB to GPU in 2.11637s (16.0653GiB/s) | 20260505 09:04:04.265 | INFO | Analyzing chunk frames = [1987584,1988096) | 20260505 09:04:04.265 | INFO | MemInfo: { | 20260505 09:04:04.265 | INFO | "gpu" : " 17 GB", | 20260505 09:04:04.265 | INFO | "huge" : "", | 20260505 09:04:04.265 | INFO | "huge_shared" : " 27 GB", | 20260505 09:04:04.265 | INFO | "malloc" : " 14 GB", | 20260505 09:04:04.265 | INFO | "pinned" : " 29 GB" | 20260505 09:04:04.265 | INFO | } | 20260505 09:04:04.265 | INFO | Perf report for the last 5.13904 seconds: | 20260505 09:04:04.265 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:04:04.265 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:04.265 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:04.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:04.265 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260505 09:04:04.265 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:04.265 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:04.265 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 09:04:04.265 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:04.265 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:04.265 | INFO | Uploaded 36GiB to GPU in 2.1576s (16.6852GiB/s) | 20260505 09:04:04.266 | INFO | Downloaded 34GiB to GPU in 2.0649s (16.4657GiB/s) | 20260505 09:04:09.405 | INFO | Analyzing chunk frames = [1988096,1988608) | 20260505 09:04:09.405 | INFO | MemInfo: { | 20260505 09:04:09.405 | INFO | "gpu" : " 17 GB", | 20260505 09:04:09.405 | INFO | "huge" : "", | 20260505 09:04:09.405 | INFO | "huge_shared" : " 27 GB", | 20260505 09:04:09.405 | INFO | "malloc" : " 14 GB", | 20260505 09:04:09.405 | INFO | "pinned" : " 29 GB" | 20260505 09:04:09.405 | INFO | } | 20260505 09:04:09.405 | INFO | Perf report for the last 5.13973 seconds: | 20260505 09:04:09.405 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:04:09.405 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:09.405 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:09.405 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:09.405 | INFO | ANALYSIS : 63.5%, 3.0, 98.6ms, 0 | 20260505 09:04:09.405 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:09.405 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:09.405 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:04:09.405 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:09.405 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:09.405 | INFO | Uploaded 36GiB to GPU in 2.18186s (16.4997GiB/s) | 20260505 09:04:09.405 | INFO | Downloaded 34GiB to GPU in 2.06628s (16.4547GiB/s) | 20260505 09:04:14.544 | INFO | Analyzing chunk frames = [1988608,1989120) | 20260505 09:04:14.544 | INFO | MemInfo: { | 20260505 09:04:14.544 | INFO | "gpu" : " 17 GB", | 20260505 09:04:14.544 | INFO | "huge" : "", | 20260505 09:04:14.544 | INFO | "huge_shared" : " 27 GB", | 20260505 09:04:14.544 | INFO | "malloc" : " 14 GB", | 20260505 09:04:14.544 | INFO | "pinned" : " 29 GB" | 20260505 09:04:14.544 | INFO | } | 20260505 09:04:14.544 | INFO | Perf report for the last 5.13864 seconds: | 20260505 09:04:14.544 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:04:14.544 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:14.544 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:14.544 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:14.544 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 09:04:14.544 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:14.544 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:14.544 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 09:04:14.544 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:14.544 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:14.544 | INFO | Uploaded 36GiB to GPU in 2.17031s (16.5875GiB/s) | 20260505 09:04:14.544 | INFO | Downloaded 34GiB to GPU in 2.05658s (16.5323GiB/s) | 20260505 09:04:19.684 | INFO | Analyzing chunk frames = [1989120,1989632) | 20260505 09:04:19.684 | INFO | MemInfo: { | 20260505 09:04:19.684 | INFO | "gpu" : " 17 GB", | 20260505 09:04:19.684 | INFO | "huge" : "", | 20260505 09:04:19.684 | INFO | "huge_shared" : " 27 GB", | 20260505 09:04:19.684 | INFO | "malloc" : " 14 GB", | 20260505 09:04:19.684 | INFO | "pinned" : " 29 GB" | 20260505 09:04:19.684 | INFO | } | 20260505 09:04:19.684 | INFO | Perf report for the last 5.14038 seconds: | 20260505 09:04:19.684 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:04:19.684 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:19.684 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:19.684 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:19.684 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 09:04:19.684 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:19.684 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:19.684 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:04:19.684 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:19.684 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:19.684 | INFO | Uploaded 36GiB to GPU in 2.17209s (16.5739GiB/s) | 20260505 09:04:19.685 | INFO | Downloaded 34GiB to GPU in 2.05709s (16.5282GiB/s) | 20260505 09:04:24.823 | INFO | Analyzing chunk frames = [1989632,1990144) | 20260505 09:04:24.823 | INFO | MemInfo: { | 20260505 09:04:24.823 | INFO | "gpu" : " 17 GB", | 20260505 09:04:24.823 | INFO | "huge" : "", | 20260505 09:04:24.823 | INFO | "huge_shared" : " 27 GB", | 20260505 09:04:24.823 | INFO | "malloc" : " 14 GB", | 20260505 09:04:24.823 | INFO | "pinned" : " 29 GB" | 20260505 09:04:24.823 | INFO | } | 20260505 09:04:24.823 | INFO | Perf report for the last 5.13945 seconds: | 20260505 09:04:24.824 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:04:24.824 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:24.824 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:24.824 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:24.824 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 09:04:24.824 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:24.824 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:24.824 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 09:04:24.824 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:24.824 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:24.824 | INFO | Uploaded 36GiB to GPU in 2.21621s (16.2439GiB/s) | 20260505 09:04:24.824 | INFO | Downloaded 34GiB to GPU in 2.15706s (15.7622GiB/s) | 20260505 09:04:29.963 | INFO | Analyzing chunk frames = [1990144,1990656) | 20260505 09:04:29.963 | INFO | MemInfo: { | 20260505 09:04:29.963 | INFO | "gpu" : " 17 GB", | 20260505 09:04:29.963 | INFO | "huge" : "", | 20260505 09:04:29.963 | INFO | "huge_shared" : " 27 GB", | 20260505 09:04:29.963 | INFO | "malloc" : " 14 GB", | 20260505 09:04:29.963 | INFO | "pinned" : " 29 GB" | 20260505 09:04:29.963 | INFO | } | 20260505 09:04:29.963 | INFO | Perf report for the last 5.13923 seconds: | 20260505 09:04:29.963 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:04:29.963 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:29.963 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:29.963 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:29.963 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 09:04:29.963 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:29.963 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:29.963 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:04:29.963 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:29.963 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:29.963 | INFO | Uploaded 36GiB to GPU in 2.16791s (16.6059GiB/s) | 20260505 09:04:29.963 | INFO | Downloaded 34GiB to GPU in 2.02394s (16.799GiB/s) | 20260505 09:04:29.963 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 09:04:29.963 | INFO | Total mem usage: 15.3 GiB | 20260505 09:04:29.963 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 09:04:29.963 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.38%) | 20260505 09:04:29.963 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 09:04:29.963 | INFO | High Water Breakdown: | 20260505 09:04:29.963 | INFO | 60.9% : 8.78 GiB ( 93.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 09:04:29.963 | INFO | 38.9% : 5.61 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 09:04:29.963 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 09:04:29.963 | INFO | ------------------------------------------ | 20260505 09:04:29.963 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:04:29.963 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 09:04:29.963 | INFO | Total mem usage: 26.5 GiB | 20260505 09:04:29.963 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 09:04:29.963 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 09:04:29.963 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 09:04:29.964 | INFO | High Water Breakdown: | 20260505 09:04:29.964 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:04:29.964 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:04:29.964 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:04:29.964 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:04:29.964 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:04:29.964 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:04:29.964 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 09:04:29.964 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 09:04:29.964 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:04:29.964 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 09:04:29.964 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 09:04:29.964 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 09:04:29.964 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 09:04:29.964 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 09:04:29.964 | INFO | ------------------------------------------ | 20260505 09:04:29.964 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:04:29.964 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 09:04:29.964 | INFO | Total mem usage: 25.6 GiB | 20260505 09:04:29.964 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 09:04:29.964 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 09:04:29.964 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 09:04:29.964 | INFO | High Water Breakdown: | 20260505 09:04:29.964 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 09:04:29.964 | INFO | ------------------------------------------ | 20260505 09:04:29.964 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:04:29.964 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 09:04:29.964 | INFO | Total mem usage: 15.7 GiB | 20260505 09:04:29.964 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 09:04:29.964 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 09:04:29.964 | INFO | Unused Cache: 880 MiB (Waste: 5.48%) | 20260505 09:04:29.964 | INFO | High Water Breakdown: | 20260505 09:04:29.964 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 09:04:29.964 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 09:04:29.964 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 09:04:29.964 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 09:04:29.964 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 09:04:29.964 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 09:04:29.964 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 09:04:29.964 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 09:04:29.964 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 09:04:29.964 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:04:29.964 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 09:04:29.964 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:04:29.964 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 09:04:29.964 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 09:04:29.964 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 09:04:29.964 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 09:04:29.964 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:04:29.964 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 09:04:29.964 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:04:29.964 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:04:29.964 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:04:29.964 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 09:04:29.964 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 09:04:29.964 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:04:29.964 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 09:04:29.964 | INFO | ------------------------------------------ | 20260505 09:04:29.964 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:04:29.964 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 09:04:29.964 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:04:29.964 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:04:29.964 | INFO | PerfReport: Full Execution -- 3.25e+01 / 7.48e+01, 1.36e+01 / 1.65e+00 | 20260505 09:04:29.964 | INFO | PerfReport: 0.29%, AggregateStats -- 9.33e-02 / 1.04e-01, 1.11e-02 / 4.99e-03 | 20260505 09:04:29.964 | INFO | PerfReport: 11.67%, Baseline -- 3.79e+00 / 4.40e+00, 6.19e-01 / 3.25e-01 | 20260505 09:04:29.964 | INFO | PerfReport: 3.31%, Histogram -- 1.08e+00 / 1.45e+00, 2.07e-01 / 1.23e-01 | 20260505 09:04:29.964 | INFO | PerfReport: 15.51%, DME -- 4.03e+01 / 4.39e+01, 3.98e+00 / 1.65e+00 | 20260505 09:04:29.964 | INFO | PerfReport: 56.45%, FrameLabeling -- 1.84e+01 / 1.98e+01, 1.85e+00 / 1.74e+00 | 20260505 09:04:29.964 | INFO | PerfReport: 7.52%, PulseAccumulating -- 2.44e+00 / 3.02e+00, 4.28e-01 / 2.59e-01 | 20260505 09:04:29.964 | INFO | PerfReport: 3.88%, Metrics -- 1.26e+00 / 1.61e+00, 3.61e-01 / 1.81e-01 | 20260505 09:04:29.964 | INFO | PerfReport: 1.37%, PulseSerializing -- 4.47e-01 / 5.04e-01, 8.40e-02 / 3.28e-02 | 20260505 09:04:29.964 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 09:04:29.964 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:04:29.964 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:04:29.965 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.29e+01 / 0.00e+00 | 20260505 09:04:29.965 | INFO | PerfReport: 22.46%, Upload -- 2.28e+01 / 0.00e+00, 8.12e+00 / 0.00e+00 | 20260505 09:04:29.965 | INFO | PerfReport: 21.69%, Download -- 2.20e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260505 09:04:29.965 | INFO | PerfReport: 32.06%, Compute -- 3.25e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 09:04:29.965 | INFO | PerfReport: 1.72%, LockUpload -- 1.74e+00 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260505 09:04:29.965 | INFO | PerfReport: 1.12%, LockDownload -- 1.14e+00 / 0.00e+00, 2.72e+00 / 0.00e+00 | 20260505 09:04:29.965 | INFO | PerfReport: 20.93%, LockCompute -- 2.12e+01 / 0.00e+00, 1.75e+01 / 0.00e+00 | 20260505 09:04:29.965 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.03e-02 / 0.00e+00 | 20260505 09:04:30.099 | INFO | Starting a round of Baz IO | 20260505 09:04:33.368 | INFO | Populating RTMetrics report | 20260505 09:04:33.368 | INFO | RTMetrics report populated | 20260505 09:04:33.368 | INFO | Opening tmp RTMetrics json file | 20260505 09:04:33.389 | INFO | Serializing RTMetrics to json | 20260505 09:04:33.414 | INFO | Serialization done, swapping tmp file now... | 20260505 09:04:33.416 | INFO | RTMetrics done | 20260505 09:04:33.521 | INFO | Finished a round of Baz IO: Wrote 2.445 GiB in 3.421 seconds | 20260505 09:04:33.521 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 26.61% / 0.02% / 73.37% | 20260505 09:04:33.521 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.21155% / 1.58946e-05% / 8.74201e-05% | 20260505 09:04:33.521 | INFO | Baz IO overall: | 20260505 09:04:33.521 | INFO | BytesWritten : 1140772420437 | 20260505 09:04:33.521 | INFO | NumEvents : 542674099803 | 20260505 09:04:35.101 | INFO | Analyzing chunk frames = [1990656,1991168) | 20260505 09:04:35.101 | INFO | MemInfo: { | 20260505 09:04:35.101 | INFO | "gpu" : " 17 GB", | 20260505 09:04:35.101 | INFO | "huge" : "", | 20260505 09:04:35.101 | INFO | "huge_shared" : " 27 GB", | 20260505 09:04:35.101 | INFO | "malloc" : " 14 GB", | 20260505 09:04:35.101 | INFO | "pinned" : " 29 GB" | 20260505 09:04:35.101 | INFO | } | 20260505 09:04:35.101 | INFO | Perf report for the last 5.1383 seconds: | 20260505 09:04:35.101 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:04:35.101 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:35.101 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:35.101 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:35.101 | INFO | ANALYSIS : 65.1%, 2.9, 99.6ms, 0 | 20260505 09:04:35.101 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:35.101 | INFO | RT_METRICS : 46.7%, 1.0, 24.5ms, 0 | 20260505 09:04:35.101 | INFO | PRE_HQ : 63.1%, 1.8, 60.2ms, 0 | 20260505 09:04:35.101 | INFO | BAZWRITER : 65.9%, 4.2, 142.0ms, 0 | 20260505 09:04:35.101 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260505 09:04:35.101 | INFO | Uploaded 36GiB to GPU in 2.27889s (15.7972GiB/s) | 20260505 09:04:35.101 | INFO | Downloaded 38GiB to GPU in 2.38665s (15.9219GiB/s) | 20260505 09:04:40.240 | INFO | Analyzing chunk frames = [1991168,1991680) | 20260505 09:04:40.240 | INFO | MemInfo: { | 20260505 09:04:40.240 | INFO | "gpu" : " 17 GB", | 20260505 09:04:40.240 | INFO | "huge" : "", | 20260505 09:04:40.240 | INFO | "huge_shared" : " 27 GB", | 20260505 09:04:40.240 | INFO | "malloc" : " 14 GB", | 20260505 09:04:40.240 | INFO | "pinned" : " 29 GB" | 20260505 09:04:40.240 | INFO | } | 20260505 09:04:40.240 | INFO | Perf report for the last 5.13941 seconds: | 20260505 09:04:40.240 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:04:40.240 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:40.240 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:40.240 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:40.241 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 09:04:40.241 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:40.241 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:40.241 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 09:04:40.241 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:40.241 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:40.241 | INFO | Uploaded 36GiB to GPU in 2.20626s (16.3172GiB/s) | 20260505 09:04:40.241 | INFO | Downloaded 34GiB to GPU in 2.08876s (16.2776GiB/s) | 20260505 09:04:45.379 | INFO | Analyzing chunk frames = [1991680,1992192) | 20260505 09:04:45.379 | INFO | MemInfo: { | 20260505 09:04:45.379 | INFO | "gpu" : " 17 GB", | 20260505 09:04:45.379 | INFO | "huge" : "", | 20260505 09:04:45.379 | INFO | "huge_shared" : " 27 GB", | 20260505 09:04:45.379 | INFO | "malloc" : " 14 GB", | 20260505 09:04:45.380 | INFO | "pinned" : " 29 GB" | 20260505 09:04:45.380 | INFO | } | 20260505 09:04:45.380 | INFO | Perf report for the last 5.13923 seconds: | 20260505 09:04:45.380 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:04:45.380 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:45.380 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:45.380 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:45.380 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260505 09:04:45.380 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:45.380 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:45.380 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 09:04:45.380 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:45.380 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:45.380 | INFO | Uploaded 36GiB to GPU in 2.16444s (16.6325GiB/s) | 20260505 09:04:45.380 | INFO | Downloaded 34GiB to GPU in 2.06862s (16.4361GiB/s) | 20260505 09:04:50.519 | INFO | Analyzing chunk frames = [1992192,1992704) | 20260505 09:04:50.519 | INFO | MemInfo: { | 20260505 09:04:50.519 | INFO | "gpu" : " 17 GB", | 20260505 09:04:50.519 | INFO | "huge" : "", | 20260505 09:04:50.519 | INFO | "huge_shared" : " 27 GB", | 20260505 09:04:50.519 | INFO | "malloc" : " 14 GB", | 20260505 09:04:50.519 | INFO | "pinned" : " 29 GB" | 20260505 09:04:50.519 | INFO | } | 20260505 09:04:50.519 | INFO | Perf report for the last 5.13953 seconds: | 20260505 09:04:50.519 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:04:50.519 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:50.519 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:50.519 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:50.519 | INFO | ANALYSIS : 63.5%, 3.0, 98.6ms, 0 | 20260505 09:04:50.519 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:50.519 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:50.519 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:04:50.519 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:50.519 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:50.519 | INFO | Uploaded 36GiB to GPU in 2.1698s (16.5914GiB/s) | 20260505 09:04:50.520 | INFO | Downloaded 34GiB to GPU in 2.0691s (16.4322GiB/s) | 20260505 09:04:55.660 | INFO | Analyzing chunk frames = [1992704,1993216) | 20260505 09:04:55.660 | INFO | MemInfo: { | 20260505 09:04:55.660 | INFO | "gpu" : " 17 GB", | 20260505 09:04:55.660 | INFO | "huge" : "", | 20260505 09:04:55.660 | INFO | "huge_shared" : " 27 GB", | 20260505 09:04:55.660 | INFO | "malloc" : " 14 GB", | 20260505 09:04:55.660 | INFO | "pinned" : " 29 GB" | 20260505 09:04:55.660 | INFO | } | 20260505 09:04:55.660 | INFO | Perf report for the last 5.14058 seconds: | 20260505 09:04:55.660 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:04:55.660 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:55.660 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:55.660 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:55.660 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 09:04:55.660 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:55.660 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:55.660 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:04:55.660 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:55.660 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:04:55.660 | INFO | Uploaded 36GiB to GPU in 2.16935s (16.5948GiB/s) | 20260505 09:04:55.660 | INFO | Downloaded 34GiB to GPU in 2.06069s (16.4993GiB/s) | 20260505 09:05:00.799 | INFO | Analyzing chunk frames = [1993216,1993728) | 20260505 09:05:00.799 | INFO | MemInfo: { | 20260505 09:05:00.799 | INFO | "gpu" : " 17 GB", | 20260505 09:05:00.799 | INFO | "huge" : "", | 20260505 09:05:00.799 | INFO | "huge_shared" : " 27 GB", | 20260505 09:05:00.799 | INFO | "malloc" : " 14 GB", | 20260505 09:05:00.799 | INFO | "pinned" : " 29 GB" | 20260505 09:05:00.799 | INFO | } | 20260505 09:05:00.799 | INFO | Perf report for the last 5.13947 seconds: | 20260505 09:05:00.799 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:05:00.799 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:00.799 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:00.799 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:00.799 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 09:05:00.799 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:00.799 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:00.799 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:05:00.799 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:00.799 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:00.799 | INFO | Uploaded 36GiB to GPU in 2.16204s (16.651GiB/s) | 20260505 09:05:00.799 | INFO | Downloaded 34GiB to GPU in 2.05837s (16.5179GiB/s) | 20260505 09:05:05.937 | INFO | Analyzing chunk frames = [1993728,1994240) | 20260505 09:05:05.937 | INFO | MemInfo: { | 20260505 09:05:05.937 | INFO | "gpu" : " 17 GB", | 20260505 09:05:05.937 | INFO | "huge" : "", | 20260505 09:05:05.937 | INFO | "huge_shared" : " 27 GB", | 20260505 09:05:05.937 | INFO | "malloc" : " 14 GB", | 20260505 09:05:05.937 | INFO | "pinned" : " 29 GB" | 20260505 09:05:05.937 | INFO | } | 20260505 09:05:05.937 | INFO | Perf report for the last 5.13785 seconds: | 20260505 09:05:05.937 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:05:05.937 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:05.937 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:05.937 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:05.937 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 09:05:05.937 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:05.937 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:05.937 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 09:05:05.937 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:05.937 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:05.938 | INFO | Uploaded 36GiB to GPU in 2.19274s (16.4178GiB/s) | 20260505 09:05:05.938 | INFO | Downloaded 34GiB to GPU in 2.15539s (15.7744GiB/s) | 20260505 09:05:11.077 | INFO | Analyzing chunk frames = [1994240,1994752) | 20260505 09:05:11.077 | INFO | MemInfo: { | 20260505 09:05:11.077 | INFO | "gpu" : " 17 GB", | 20260505 09:05:11.077 | INFO | "huge" : "", | 20260505 09:05:11.077 | INFO | "huge_shared" : " 27 GB", | 20260505 09:05:11.077 | INFO | "malloc" : " 14 GB", | 20260505 09:05:11.077 | INFO | "pinned" : " 29 GB" | 20260505 09:05:11.077 | INFO | } | 20260505 09:05:11.077 | INFO | Perf report for the last 5.1402 seconds: | 20260505 09:05:11.077 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:05:11.077 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:11.077 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:11.077 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:11.077 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 09:05:11.077 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:11.077 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:11.077 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:05:11.077 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:11.077 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:11.078 | INFO | Uploaded 36GiB to GPU in 2.19602s (16.3933GiB/s) | 20260505 09:05:11.078 | INFO | Downloaded 34GiB to GPU in 2.04107s (16.6579GiB/s) | 20260505 09:05:11.211 | INFO | Starting a round of Baz IO | 20260505 09:05:14.473 | INFO | Populating RTMetrics report | 20260505 09:05:14.473 | INFO | RTMetrics report populated | 20260505 09:05:14.473 | INFO | Opening tmp RTMetrics json file | 20260505 09:05:14.475 | INFO | Serializing RTMetrics to json | 20260505 09:05:14.477 | INFO | Serialization done, swapping tmp file now... | 20260505 09:05:14.479 | INFO | RTMetrics done | 20260505 09:05:14.640 | INFO | Finished a round of Baz IO: Wrote 2.443 GiB in 3.428 seconds | 20260505 09:05:14.640 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 26.60% / 0.02% / 73.39% | 20260505 09:05:14.640 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.19613% / 1.58946e-05% / 9.93411e-05% | 20260505 09:05:14.640 | INFO | Baz IO overall: | 20260505 09:05:14.641 | INFO | BytesWritten : 1143395695247 | 20260505 09:05:14.641 | INFO | NumEvents : 543928164132 | 20260505 09:05:16.216 | INFO | Analyzing chunk frames = [1994752,1995264) | 20260505 09:05:16.216 | INFO | MemInfo: { | 20260505 09:05:16.216 | INFO | "gpu" : " 17 GB", | 20260505 09:05:16.216 | INFO | "huge" : "", | 20260505 09:05:16.216 | INFO | "huge_shared" : " 27 GB", | 20260505 09:05:16.216 | INFO | "malloc" : " 14 GB", | 20260505 09:05:16.216 | INFO | "pinned" : " 29 GB" | 20260505 09:05:16.216 | INFO | } | 20260505 09:05:16.216 | INFO | Perf report for the last 5.1387 seconds: | 20260505 09:05:16.216 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:05:16.216 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:16.216 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:16.216 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:16.216 | INFO | ANALYSIS : 64.9%, 2.9, 98.8ms, 0 | 20260505 09:05:16.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:16.216 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 09:05:16.216 | INFO | PRE_HQ : 63.3%, 1.9, 61.1ms, 0 | 20260505 09:05:16.216 | INFO | BAZWRITER : 66.0%, 4.2, 142.9ms, 0 | 20260505 09:05:16.216 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 09:05:16.216 | INFO | Uploaded 36GiB to GPU in 2.25077s (15.9946GiB/s) | 20260505 09:05:16.216 | INFO | Downloaded 38GiB to GPU in 2.37594s (15.9936GiB/s) | 20260505 09:05:21.355 | INFO | Analyzing chunk frames = [1995264,1995776) | 20260505 09:05:21.355 | INFO | MemInfo: { | 20260505 09:05:21.355 | INFO | "gpu" : " 17 GB", | 20260505 09:05:21.355 | INFO | "huge" : "", | 20260505 09:05:21.355 | INFO | "huge_shared" : " 27 GB", | 20260505 09:05:21.355 | INFO | "malloc" : " 14 GB", | 20260505 09:05:21.355 | INFO | "pinned" : " 29 GB" | 20260505 09:05:21.355 | INFO | } | 20260505 09:05:21.355 | INFO | Perf report for the last 5.13932 seconds: | 20260505 09:05:21.356 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:05:21.356 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:21.356 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:21.356 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:21.356 | INFO | ANALYSIS : 64.3%, 3.0, 100.5ms, 0 | 20260505 09:05:21.356 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:21.356 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:21.356 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 09:05:21.356 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:21.356 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:21.356 | INFO | Uploaded 36GiB to GPU in 2.20427s (16.3319GiB/s) | 20260505 09:05:21.356 | INFO | Downloaded 34GiB to GPU in 2.08145s (16.3348GiB/s) | 20260505 09:05:26.495 | INFO | Analyzing chunk frames = [1995776,1996288) | 20260505 09:05:26.495 | INFO | MemInfo: { | 20260505 09:05:26.495 | INFO | "gpu" : " 17 GB", | 20260505 09:05:26.495 | INFO | "huge" : "", | 20260505 09:05:26.495 | INFO | "huge_shared" : " 27 GB", | 20260505 09:05:26.495 | INFO | "malloc" : " 14 GB", | 20260505 09:05:26.495 | INFO | "pinned" : " 29 GB" | 20260505 09:05:26.495 | INFO | } | 20260505 09:05:26.495 | INFO | Perf report for the last 5.13969 seconds: | 20260505 09:05:26.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:05:26.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:26.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:26.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:26.495 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260505 09:05:26.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:26.495 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:26.495 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 09:05:26.495 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:26.495 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:26.495 | INFO | Uploaded 36GiB to GPU in 2.16878s (16.5992GiB/s) | 20260505 09:05:26.495 | INFO | Downloaded 34GiB to GPU in 2.06744s (16.4455GiB/s) | 20260505 09:05:31.634 | INFO | Analyzing chunk frames = [1996288,1996800) | 20260505 09:05:31.634 | INFO | MemInfo: { | 20260505 09:05:31.634 | INFO | "gpu" : " 17 GB", | 20260505 09:05:31.634 | INFO | "huge" : "", | 20260505 09:05:31.634 | INFO | "huge_shared" : " 27 GB", | 20260505 09:05:31.634 | INFO | "malloc" : " 14 GB", | 20260505 09:05:31.634 | INFO | "pinned" : " 29 GB" | 20260505 09:05:31.634 | INFO | } | 20260505 09:05:31.634 | INFO | Perf report for the last 5.13932 seconds: | 20260505 09:05:31.634 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:05:31.634 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:31.634 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:31.634 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:31.634 | INFO | ANALYSIS : 63.4%, 3.0, 98.5ms, 0 | 20260505 09:05:31.634 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:31.634 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:31.634 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 09:05:31.635 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:31.635 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:31.635 | INFO | Uploaded 36GiB to GPU in 2.1674s (16.6097GiB/s) | 20260505 09:05:31.635 | INFO | Downloaded 34GiB to GPU in 2.0657s (16.4593GiB/s) | 20260505 09:05:36.774 | INFO | Analyzing chunk frames = [1996800,1997312) | 20260505 09:05:36.774 | INFO | MemInfo: { | 20260505 09:05:36.774 | INFO | "gpu" : " 17 GB", | 20260505 09:05:36.774 | INFO | "huge" : "", | 20260505 09:05:36.774 | INFO | "huge_shared" : " 27 GB", | 20260505 09:05:36.774 | INFO | "malloc" : " 14 GB", | 20260505 09:05:36.774 | INFO | "pinned" : " 29 GB" | 20260505 09:05:36.774 | INFO | } | 20260505 09:05:36.774 | INFO | Perf report for the last 5.13934 seconds: | 20260505 09:05:36.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:05:36.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:36.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:36.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:36.774 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 09:05:36.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:36.774 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:36.774 | INFO | PRE_HQ : 29.8%, 1.0, 15.4ms, 0 | 20260505 09:05:36.774 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:36.774 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:36.774 | INFO | Uploaded 36GiB to GPU in 2.23374s (16.1165GiB/s) | 20260505 09:05:36.774 | INFO | Downloaded 34GiB to GPU in 2.1124s (16.0955GiB/s) | 20260505 09:05:41.914 | INFO | Analyzing chunk frames = [1997312,1997824) | 20260505 09:05:41.914 | INFO | MemInfo: { | 20260505 09:05:41.914 | INFO | "gpu" : " 17 GB", | 20260505 09:05:41.914 | INFO | "huge" : "", | 20260505 09:05:41.914 | INFO | "huge_shared" : " 27 GB", | 20260505 09:05:41.914 | INFO | "malloc" : " 14 GB", | 20260505 09:05:41.914 | INFO | "pinned" : " 29 GB" | 20260505 09:05:41.914 | INFO | } | 20260505 09:05:41.914 | INFO | Perf report for the last 5.14057 seconds: | 20260505 09:05:41.915 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:05:41.915 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:41.915 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:41.915 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:41.915 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 09:05:41.915 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:41.915 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:41.915 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:05:41.915 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:41.915 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:41.915 | INFO | Uploaded 36GiB to GPU in 2.16234s (16.6486GiB/s) | 20260505 09:05:41.915 | INFO | Downloaded 34GiB to GPU in 2.05907s (16.5123GiB/s) | 20260505 09:05:47.052 | INFO | Analyzing chunk frames = [1997824,1998336) | 20260505 09:05:47.052 | INFO | MemInfo: { | 20260505 09:05:47.052 | INFO | "gpu" : " 17 GB", | 20260505 09:05:47.052 | INFO | "huge" : "", | 20260505 09:05:47.052 | INFO | "huge_shared" : " 27 GB", | 20260505 09:05:47.052 | INFO | "malloc" : " 14 GB", | 20260505 09:05:47.052 | INFO | "pinned" : " 29 GB" | 20260505 09:05:47.052 | INFO | } | 20260505 09:05:47.052 | INFO | Perf report for the last 5.13801 seconds: | 20260505 09:05:47.053 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:05:47.053 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:47.053 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:47.053 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:47.053 | INFO | ANALYSIS : 64.0%, 3.0, 99.8ms, 0 | 20260505 09:05:47.053 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:47.053 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:47.053 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 09:05:47.053 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:47.053 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:47.053 | INFO | Uploaded 36GiB to GPU in 2.20597s (16.3194GiB/s) | 20260505 09:05:47.053 | INFO | Downloaded 34GiB to GPU in 2.14215s (15.8719GiB/s) | 20260505 09:05:52.191 | INFO | Analyzing chunk frames = [1998336,1998848) | 20260505 09:05:52.191 | INFO | MemInfo: { | 20260505 09:05:52.191 | INFO | "gpu" : " 17 GB", | 20260505 09:05:52.191 | INFO | "huge" : "", | 20260505 09:05:52.191 | INFO | "huge_shared" : " 27 GB", | 20260505 09:05:52.191 | INFO | "malloc" : " 14 GB", | 20260505 09:05:52.191 | INFO | "pinned" : " 29 GB" | 20260505 09:05:52.191 | INFO | } | 20260505 09:05:52.191 | INFO | Perf report for the last 5.13843 seconds: | 20260505 09:05:52.191 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:05:52.191 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:52.191 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:52.191 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:52.191 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 09:05:52.191 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:52.191 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:52.191 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 09:05:52.191 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:52.191 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:52.191 | INFO | Uploaded 36GiB to GPU in 2.16484s (16.6294GiB/s) | 20260505 09:05:52.191 | INFO | Downloaded 34GiB to GPU in 2.02504s (16.7898GiB/s) | 20260505 09:05:52.191 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 09:05:52.191 | INFO | Total mem usage: 15.3 GiB | 20260505 09:05:52.191 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 09:05:52.191 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.41%) | 20260505 09:05:52.191 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 09:05:52.191 | INFO | High Water Breakdown: | 20260505 09:05:52.191 | INFO | 60.9% : 8.77 GiB ( 93.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 09:05:52.191 | INFO | 39% : 5.61 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 09:05:52.191 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 09:05:52.191 | INFO | ------------------------------------------ | 20260505 09:05:52.191 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:05:52.192 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 09:05:52.192 | INFO | Total mem usage: 26.5 GiB | 20260505 09:05:52.192 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 09:05:52.192 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 09:05:52.192 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 09:05:52.192 | INFO | High Water Breakdown: | 20260505 09:05:52.192 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:05:52.192 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:05:52.192 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:05:52.192 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:05:52.192 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:05:52.192 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:05:52.192 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 09:05:52.192 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 09:05:52.192 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:05:52.192 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 09:05:52.192 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 09:05:52.192 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 09:05:52.192 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 09:05:52.192 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 09:05:52.192 | INFO | ------------------------------------------ | 20260505 09:05:52.192 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:05:52.192 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 09:05:52.192 | INFO | Total mem usage: 25.6 GiB | 20260505 09:05:52.192 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 09:05:52.192 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 09:05:52.192 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 09:05:52.192 | INFO | High Water Breakdown: | 20260505 09:05:52.192 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 09:05:52.192 | INFO | ------------------------------------------ | 20260505 09:05:52.192 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:05:52.192 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 09:05:52.192 | INFO | Total mem usage: 15.7 GiB | 20260505 09:05:52.192 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 09:05:52.192 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 09:05:52.192 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 09:05:52.192 | INFO | High Water Breakdown: | 20260505 09:05:52.192 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 09:05:52.192 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 09:05:52.192 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 09:05:52.192 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 09:05:52.192 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 09:05:52.192 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 09:05:52.192 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 09:05:52.192 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 09:05:52.192 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 09:05:52.192 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:05:52.192 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 09:05:52.192 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:05:52.192 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 09:05:52.192 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 09:05:52.192 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 09:05:52.192 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 09:05:52.192 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:05:52.192 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 09:05:52.192 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:05:52.192 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:05:52.192 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:05:52.192 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 09:05:52.192 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 09:05:52.192 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:05:52.192 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 09:05:52.192 | INFO | ------------------------------------------ | 20260505 09:05:52.192 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:05:52.192 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 09:05:52.192 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:05:52.192 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:05:52.192 | INFO | PerfReport: Full Execution -- 3.25e+01 / 7.51e+01, 1.35e+01 / 1.74e+00 | 20260505 09:05:52.192 | INFO | PerfReport: 0.29%, AggregateStats -- 9.34e-02 / 1.05e-01, 1.11e-02 / 3.58e-03 | 20260505 09:05:52.192 | INFO | PerfReport: 11.67%, Baseline -- 3.79e+00 / 4.42e+00, 6.22e-01 / 2.98e-01 | 20260505 09:05:52.192 | INFO | PerfReport: 3.31%, Histogram -- 1.08e+00 / 1.46e+00, 2.08e-01 / 1.05e-01 | 20260505 09:05:52.192 | INFO | PerfReport: 15.45%, DME -- 4.02e+01 / 4.38e+01, 3.99e+00 / 1.10e+00 | 20260505 09:05:52.192 | INFO | PerfReport: 56.52%, FrameLabeling -- 1.84e+01 / 2.02e+01, 1.90e+00 / 1.96e+00 | 20260505 09:05:52.192 | INFO | PerfReport: 7.50%, PulseAccumulating -- 2.44e+00 / 3.01e+00, 4.23e-01 / 2.49e-01 | 20260505 09:05:52.192 | INFO | PerfReport: 3.88%, Metrics -- 1.26e+00 / 1.61e+00, 3.67e-01 / 1.66e-01 | 20260505 09:05:52.193 | INFO | PerfReport: 1.37%, PulseSerializing -- 4.47e-01 / 5.06e-01, 8.42e-02 / 2.62e-02 | 20260505 09:05:52.193 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 09:05:52.193 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:05:52.193 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:05:52.193 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.33e+01 / 0.00e+00 | 20260505 09:05:52.193 | INFO | PerfReport: 22.53%, Upload -- 2.29e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260505 09:05:52.193 | INFO | PerfReport: 21.74%, Download -- 2.20e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260505 09:05:52.193 | INFO | PerfReport: 32.06%, Compute -- 3.25e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 09:05:52.193 | INFO | PerfReport: 1.74%, LockUpload -- 1.76e+00 / 0.00e+00, 7.45e+00 / 0.00e+00 | 20260505 09:05:52.193 | INFO | PerfReport: 1.08%, LockDownload -- 1.10e+00 / 0.00e+00, 2.63e+00 / 0.00e+00 | 20260505 09:05:52.193 | INFO | PerfReport: 20.85%, LockCompute -- 2.11e+01 / 0.00e+00, 1.78e+01 / 0.00e+00 | 20260505 09:05:52.193 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.78e-02 / 0.00e+00 | 20260505 09:05:52.329 | INFO | Starting a round of Baz IO | 20260505 09:05:55.612 | INFO | Populating RTMetrics report | 20260505 09:05:55.612 | INFO | RTMetrics report populated | 20260505 09:05:55.612 | INFO | Opening tmp RTMetrics json file | 20260505 09:05:55.629 | INFO | Serializing RTMetrics to json | 20260505 09:05:55.677 | INFO | Serialization done, swapping tmp file now... | 20260505 09:05:55.693 | INFO | RTMetrics done | 20260505 09:05:55.742 | INFO | Finished a round of Baz IO: Wrote 2.443 GiB in 3.413 seconds | 20260505 09:05:55.742 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 26.58% / 0.02% / 73.40% | 20260505 09:05:55.742 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.18659% / 7.94729e-06% / 7.15256e-05% | 20260505 09:05:55.742 | INFO | Baz IO overall: | 20260505 09:05:55.742 | INFO | BytesWritten : 1146018340690 | 20260505 09:05:55.742 | INFO | NumEvents : 545181932448 | 20260505 09:05:57.330 | INFO | Analyzing chunk frames = [1998848,1999360) | 20260505 09:05:57.330 | INFO | MemInfo: { | 20260505 09:05:57.330 | INFO | "gpu" : " 17 GB", | 20260505 09:05:57.330 | INFO | "huge" : "", | 20260505 09:05:57.330 | INFO | "huge_shared" : " 27 GB", | 20260505 09:05:57.330 | INFO | "malloc" : " 14 GB", | 20260505 09:05:57.330 | INFO | "pinned" : " 29 GB" | 20260505 09:05:57.330 | INFO | } | 20260505 09:05:57.330 | INFO | Perf report for the last 5.13964 seconds: | 20260505 09:05:57.330 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:05:57.330 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:57.330 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:57.330 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:57.331 | INFO | ANALYSIS : 65.4%, 2.9, 100.3ms, 0 | 20260505 09:05:57.331 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:05:57.331 | INFO | RT_METRICS : 47.3%, 1.0, 24.8ms, 0 | 20260505 09:05:57.331 | INFO | PRE_HQ : 62.8%, 1.9, 60.6ms, 0 | 20260505 09:05:57.331 | INFO | BAZWRITER : 65.7%, 4.1, 139.1ms, 0 | 20260505 09:05:57.331 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260505 09:05:57.331 | INFO | Uploaded 36GiB to GPU in 2.22263s (16.197GiB/s) | 20260505 09:05:57.331 | INFO | Downloaded 38GiB to GPU in 2.33025s (16.3073GiB/s) | 20260505 09:06:02.470 | INFO | Analyzing chunk frames = [1999360,1999872) | 20260505 09:06:02.470 | INFO | MemInfo: { | 20260505 09:06:02.470 | INFO | "gpu" : " 17 GB", | 20260505 09:06:02.470 | INFO | "huge" : "", | 20260505 09:06:02.470 | INFO | "huge_shared" : " 27 GB", | 20260505 09:06:02.470 | INFO | "malloc" : " 14 GB", | 20260505 09:06:02.470 | INFO | "pinned" : " 29 GB" | 20260505 09:06:02.471 | INFO | } | 20260505 09:06:02.471 | INFO | Perf report for the last 5.14017 seconds: | 20260505 09:06:02.471 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:06:02.471 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:02.471 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:02.471 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:02.471 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 09:06:02.471 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:02.471 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:02.471 | INFO | PRE_HQ : 29.6%, 1.0, 15.5ms, 0 | 20260505 09:06:02.471 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:02.471 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:02.471 | INFO | Uploaded 36GiB to GPU in 2.18448s (16.4799GiB/s) | 20260505 09:06:02.471 | INFO | Downloaded 34GiB to GPU in 2.08528s (16.3047GiB/s) | 20260505 09:06:07.610 | INFO | Analyzing chunk frames = [1999872,2000384) | 20260505 09:06:07.610 | INFO | MemInfo: { | 20260505 09:06:07.610 | INFO | "gpu" : " 17 GB", | 20260505 09:06:07.610 | INFO | "huge" : "", | 20260505 09:06:07.610 | INFO | "huge_shared" : " 27 GB", | 20260505 09:06:07.610 | INFO | "malloc" : " 14 GB", | 20260505 09:06:07.610 | INFO | "pinned" : " 29 GB" | 20260505 09:06:07.610 | INFO | } | 20260505 09:06:07.610 | INFO | Perf report for the last 5.13945 seconds: | 20260505 09:06:07.610 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:06:07.610 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:07.610 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:07.610 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:07.610 | INFO | ANALYSIS : 63.6%, 3.0, 99.0ms, 0 | 20260505 09:06:07.610 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:07.610 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:07.610 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 09:06:07.610 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:07.610 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:07.610 | INFO | Uploaded 36GiB to GPU in 2.17524s (16.5499GiB/s) | 20260505 09:06:07.611 | INFO | Downloaded 34GiB to GPU in 2.06531s (16.4624GiB/s) | 20260505 09:06:12.750 | INFO | Analyzing chunk frames = [2000384,2000896) | 20260505 09:06:12.750 | INFO | MemInfo: { | 20260505 09:06:12.750 | INFO | "gpu" : " 17 GB", | 20260505 09:06:12.750 | INFO | "huge" : "", | 20260505 09:06:12.750 | INFO | "huge_shared" : " 27 GB", | 20260505 09:06:12.750 | INFO | "malloc" : " 14 GB", | 20260505 09:06:12.750 | INFO | "pinned" : " 29 GB" | 20260505 09:06:12.750 | INFO | } | 20260505 09:06:12.750 | INFO | Perf report for the last 5.13965 seconds: | 20260505 09:06:12.750 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:06:12.750 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:12.750 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:12.750 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:12.750 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 09:06:12.750 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:12.750 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:12.750 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:06:12.750 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:12.750 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:12.750 | INFO | Uploaded 36GiB to GPU in 2.15982s (16.668GiB/s) | 20260505 09:06:12.750 | INFO | Downloaded 34GiB to GPU in 2.07093s (16.4177GiB/s) | 20260505 09:06:17.888 | INFO | Analyzing chunk frames = [2000896,2001408) | 20260505 09:06:17.888 | INFO | MemInfo: { | 20260505 09:06:17.888 | INFO | "gpu" : " 17 GB", | 20260505 09:06:17.888 | INFO | "huge" : "", | 20260505 09:06:17.888 | INFO | "huge_shared" : " 27 GB", | 20260505 09:06:17.888 | INFO | "malloc" : " 14 GB", | 20260505 09:06:17.888 | INFO | "pinned" : " 29 GB" | 20260505 09:06:17.888 | INFO | } | 20260505 09:06:17.888 | INFO | Perf report for the last 5.13813 seconds: | 20260505 09:06:17.888 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:06:17.888 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:17.888 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:17.888 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:17.888 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 09:06:17.888 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:17.888 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:17.888 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 09:06:17.888 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:17.888 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:17.888 | INFO | Uploaded 36GiB to GPU in 2.23474s (16.1093GiB/s) | 20260505 09:06:17.888 | INFO | Downloaded 34GiB to GPU in 2.11039s (16.1108GiB/s) | 20260505 09:06:23.028 | INFO | Analyzing chunk frames = [2001408,2001920) | 20260505 09:06:23.028 | INFO | MemInfo: { | 20260505 09:06:23.028 | INFO | "gpu" : " 17 GB", | 20260505 09:06:23.028 | INFO | "huge" : "", | 20260505 09:06:23.028 | INFO | "huge_shared" : " 27 GB", | 20260505 09:06:23.028 | INFO | "malloc" : " 14 GB", | 20260505 09:06:23.028 | INFO | "pinned" : " 29 GB" | 20260505 09:06:23.028 | INFO | } | 20260505 09:06:23.028 | INFO | Perf report for the last 5.14038 seconds: | 20260505 09:06:23.028 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:06:23.028 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:23.028 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:23.029 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:23.029 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 09:06:23.029 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:23.029 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:23.029 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 09:06:23.029 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:23.029 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:23.029 | INFO | Uploaded 36GiB to GPU in 2.19029s (16.4362GiB/s) | 20260505 09:06:23.029 | INFO | Downloaded 34GiB to GPU in 2.06205s (16.4885GiB/s) | 20260505 09:06:28.167 | INFO | Analyzing chunk frames = [2001920,2002432) | 20260505 09:06:28.167 | INFO | MemInfo: { | 20260505 09:06:28.167 | INFO | "gpu" : " 17 GB", | 20260505 09:06:28.167 | INFO | "huge" : "", | 20260505 09:06:28.167 | INFO | "huge_shared" : " 27 GB", | 20260505 09:06:28.167 | INFO | "malloc" : " 14 GB", | 20260505 09:06:28.167 | INFO | "pinned" : " 29 GB" | 20260505 09:06:28.167 | INFO | } | 20260505 09:06:28.167 | INFO | Perf report for the last 5.13888 seconds: | 20260505 09:06:28.167 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:06:28.167 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:28.167 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:28.167 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:28.167 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 09:06:28.167 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:28.167 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:28.167 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 09:06:28.167 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:28.167 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:28.168 | INFO | Uploaded 36GiB to GPU in 2.18021s (16.5121GiB/s) | 20260505 09:06:28.168 | INFO | Downloaded 34GiB to GPU in 2.14485s (15.8519GiB/s) | 20260505 09:06:33.307 | INFO | Analyzing chunk frames = [2002432,2002944) | 20260505 09:06:33.307 | INFO | MemInfo: { | 20260505 09:06:33.307 | INFO | "gpu" : " 17 GB", | 20260505 09:06:33.307 | INFO | "huge" : "", | 20260505 09:06:33.307 | INFO | "huge_shared" : " 27 GB", | 20260505 09:06:33.307 | INFO | "malloc" : " 14 GB", | 20260505 09:06:33.308 | INFO | "pinned" : " 29 GB" | 20260505 09:06:33.308 | INFO | } | 20260505 09:06:33.308 | INFO | Perf report for the last 5.14044 seconds: | 20260505 09:06:33.308 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:06:33.308 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:33.308 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:33.308 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:33.308 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 09:06:33.308 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:33.308 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:33.308 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 09:06:33.308 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:33.308 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:33.308 | INFO | Uploaded 36GiB to GPU in 2.20246s (16.3453GiB/s) | 20260505 09:06:33.308 | INFO | Downloaded 34GiB to GPU in 2.04749s (16.6057GiB/s) | 20260505 09:06:33.443 | INFO | Starting a round of Baz IO | 20260505 09:06:36.709 | INFO | Populating RTMetrics report | 20260505 09:06:36.710 | INFO | RTMetrics report populated | 20260505 09:06:36.710 | INFO | Opening tmp RTMetrics json file | 20260505 09:06:36.729 | INFO | Serializing RTMetrics to json | 20260505 09:06:36.730 | INFO | Serialization done, swapping tmp file now... | 20260505 09:06:36.731 | INFO | RTMetrics done | 20260505 09:06:36.878 | INFO | Finished a round of Baz IO: Wrote 2.447 GiB in 3.434 seconds | 20260505 09:06:36.878 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 26.56% / 0.02% / 73.42% | 20260505 09:06:36.878 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.19074% / 2.38419e-05% / 6.75519e-05% | 20260505 09:06:36.878 | INFO | Baz IO overall: | 20260505 09:06:36.878 | INFO | BytesWritten : 1148646225360 | 20260505 09:06:36.878 | INFO | NumEvents : 546439307683 | 20260505 09:06:38.446 | INFO | Analyzing chunk frames = [2002944,2003456) | 20260505 09:06:38.446 | INFO | MemInfo: { | 20260505 09:06:38.446 | INFO | "gpu" : " 17 GB", | 20260505 09:06:38.446 | INFO | "huge" : "", | 20260505 09:06:38.446 | INFO | "huge_shared" : " 27 GB", | 20260505 09:06:38.446 | INFO | "malloc" : " 14 GB", | 20260505 09:06:38.446 | INFO | "pinned" : " 29 GB" | 20260505 09:06:38.446 | INFO | } | 20260505 09:06:38.446 | INFO | Perf report for the last 5.13829 seconds: | 20260505 09:06:38.446 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:06:38.446 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:38.446 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:38.446 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:38.446 | INFO | ANALYSIS : 65.1%, 2.9, 99.3ms, 0 | 20260505 09:06:38.446 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:38.446 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260505 09:06:38.446 | INFO | PRE_HQ : 63.2%, 1.8, 60.3ms, 0 | 20260505 09:06:38.446 | INFO | BAZWRITER : 66.1%, 4.2, 142.8ms, 0 | 20260505 09:06:38.446 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260505 09:06:38.446 | INFO | Uploaded 36GiB to GPU in 2.26571s (15.889GiB/s) | 20260505 09:06:38.446 | INFO | Downloaded 38GiB to GPU in 2.38496s (15.9332GiB/s) | 20260505 09:06:43.585 | INFO | Analyzing chunk frames = [2003456,2003968) | 20260505 09:06:43.585 | INFO | MemInfo: { | 20260505 09:06:43.585 | INFO | "gpu" : " 17 GB", | 20260505 09:06:43.585 | INFO | "huge" : "", | 20260505 09:06:43.585 | INFO | "huge_shared" : " 27 GB", | 20260505 09:06:43.585 | INFO | "malloc" : " 14 GB", | 20260505 09:06:43.585 | INFO | "pinned" : " 29 GB" | 20260505 09:06:43.585 | INFO | } | 20260505 09:06:43.585 | INFO | Perf report for the last 5.13904 seconds: | 20260505 09:06:43.585 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:06:43.585 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:43.585 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:43.585 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:43.585 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 09:06:43.585 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:43.585 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:43.585 | INFO | PRE_HQ : 29.8%, 1.0, 15.7ms, 0 | 20260505 09:06:43.585 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:43.585 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:43.585 | INFO | Uploaded 36GiB to GPU in 2.20737s (16.309GiB/s) | 20260505 09:06:43.585 | INFO | Downloaded 34GiB to GPU in 2.08776s (16.2854GiB/s) | 20260505 09:06:48.725 | INFO | Analyzing chunk frames = [2003968,2004480) | 20260505 09:06:48.725 | INFO | MemInfo: { | 20260505 09:06:48.725 | INFO | "gpu" : " 17 GB", | 20260505 09:06:48.725 | INFO | "huge" : "", | 20260505 09:06:48.725 | INFO | "huge_shared" : " 27 GB", | 20260505 09:06:48.725 | INFO | "malloc" : " 14 GB", | 20260505 09:06:48.725 | INFO | "pinned" : " 29 GB" | 20260505 09:06:48.725 | INFO | } | 20260505 09:06:48.725 | INFO | Perf report for the last 5.13993 seconds: | 20260505 09:06:48.725 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:06:48.725 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:48.725 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:48.725 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:48.725 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260505 09:06:48.725 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:48.725 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:48.725 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 09:06:48.725 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:48.725 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:48.726 | INFO | Uploaded 36GiB to GPU in 2.16961s (16.5928GiB/s) | 20260505 09:06:48.726 | INFO | Downloaded 34GiB to GPU in 2.06626s (16.4548GiB/s) | 20260505 09:06:53.864 | INFO | Analyzing chunk frames = [2004480,2004992) | 20260505 09:06:53.864 | INFO | MemInfo: { | 20260505 09:06:53.864 | INFO | "gpu" : " 17 GB", | 20260505 09:06:53.864 | INFO | "huge" : "", | 20260505 09:06:53.864 | INFO | "huge_shared" : " 27 GB", | 20260505 09:06:53.864 | INFO | "malloc" : " 14 GB", | 20260505 09:06:53.864 | INFO | "pinned" : " 29 GB" | 20260505 09:06:53.864 | INFO | } | 20260505 09:06:53.864 | INFO | Perf report for the last 5.13906 seconds: | 20260505 09:06:53.864 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:06:53.864 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:53.864 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:53.864 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:53.864 | INFO | ANALYSIS : 63.3%, 3.0, 98.3ms, 0 | 20260505 09:06:53.864 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:53.864 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:53.864 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 09:06:53.864 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:53.864 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:53.864 | INFO | Uploaded 36GiB to GPU in 2.15965s (16.6694GiB/s) | 20260505 09:06:53.864 | INFO | Downloaded 34GiB to GPU in 2.06672s (16.4512GiB/s) | 20260505 09:06:59.003 | INFO | Analyzing chunk frames = [2004992,2005504) | 20260505 09:06:59.003 | INFO | MemInfo: { | 20260505 09:06:59.003 | INFO | "gpu" : " 17 GB", | 20260505 09:06:59.003 | INFO | "huge" : "", | 20260505 09:06:59.003 | INFO | "huge_shared" : " 27 GB", | 20260505 09:06:59.004 | INFO | "malloc" : " 14 GB", | 20260505 09:06:59.004 | INFO | "pinned" : " 29 GB" | 20260505 09:06:59.004 | INFO | } | 20260505 09:06:59.004 | INFO | Perf report for the last 5.13956 seconds: | 20260505 09:06:59.004 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:06:59.004 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:59.004 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:59.004 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:59.004 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 09:06:59.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:59.004 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:59.004 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:06:59.004 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:59.004 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:06:59.004 | INFO | Uploaded 36GiB to GPU in 2.23632s (16.0979GiB/s) | 20260505 09:06:59.004 | INFO | Downloaded 34GiB to GPU in 2.1101s (16.113GiB/s) | 20260505 09:07:04.144 | INFO | Analyzing chunk frames = [2005504,2006016) | 20260505 09:07:04.144 | INFO | MemInfo: { | 20260505 09:07:04.144 | INFO | "gpu" : " 17 GB", | 20260505 09:07:04.144 | INFO | "huge" : "", | 20260505 09:07:04.144 | INFO | "huge_shared" : " 27 GB", | 20260505 09:07:04.144 | INFO | "malloc" : " 14 GB", | 20260505 09:07:04.144 | INFO | "pinned" : " 29 GB" | 20260505 09:07:04.144 | INFO | } | 20260505 09:07:04.144 | INFO | Perf report for the last 5.14044 seconds: | 20260505 09:07:04.144 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:07:04.144 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:04.144 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:04.144 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:04.144 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 09:07:04.144 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:04.144 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:04.144 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 09:07:04.144 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:04.144 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:04.144 | INFO | Uploaded 36GiB to GPU in 2.1638s (16.6374GiB/s) | 20260505 09:07:04.144 | INFO | Downloaded 34GiB to GPU in 2.05985s (16.5061GiB/s) | 20260505 09:07:09.282 | INFO | Analyzing chunk frames = [2006016,2006528) | 20260505 09:07:09.282 | INFO | MemInfo: { | 20260505 09:07:09.282 | INFO | "gpu" : " 17 GB", | 20260505 09:07:09.282 | INFO | "huge" : "", | 20260505 09:07:09.282 | INFO | "huge_shared" : " 27 GB", | 20260505 09:07:09.282 | INFO | "malloc" : " 14 GB", | 20260505 09:07:09.283 | INFO | "pinned" : " 29 GB" | 20260505 09:07:09.283 | INFO | } | 20260505 09:07:09.283 | INFO | Perf report for the last 5.1385 seconds: | 20260505 09:07:09.283 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:07:09.283 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:09.283 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:09.283 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:09.283 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 09:07:09.283 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:09.283 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:09.283 | INFO | PRE_HQ : 30.5%, 1.0, 15.9ms, 0 | 20260505 09:07:09.283 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:09.283 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:09.283 | INFO | Uploaded 36GiB to GPU in 2.21063s (16.2849GiB/s) | 20260505 09:07:09.283 | INFO | Downloaded 34GiB to GPU in 2.14298s (15.8658GiB/s) | 20260505 09:07:14.424 | INFO | Analyzing chunk frames = [2006528,2007040) | 20260505 09:07:14.424 | INFO | MemInfo: { | 20260505 09:07:14.424 | INFO | "gpu" : " 17 GB", | 20260505 09:07:14.424 | INFO | "huge" : "", | 20260505 09:07:14.424 | INFO | "huge_shared" : " 27 GB", | 20260505 09:07:14.424 | INFO | "malloc" : " 14 GB", | 20260505 09:07:14.424 | INFO | "pinned" : " 29 GB" | 20260505 09:07:14.424 | INFO | } | 20260505 09:07:14.424 | INFO | Perf report for the last 5.14174 seconds: | 20260505 09:07:14.424 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:07:14.424 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:14.424 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:14.424 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:14.424 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 09:07:14.424 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:14.424 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:14.424 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 09:07:14.424 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:14.424 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:14.425 | INFO | Uploaded 36GiB to GPU in 2.20055s (16.3595GiB/s) | 20260505 09:07:14.425 | INFO | Downloaded 34GiB to GPU in 2.04671s (16.612GiB/s) | 20260505 09:07:14.425 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 09:07:14.425 | INFO | Total mem usage: 15.3 GiB | 20260505 09:07:14.425 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 09:07:14.425 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -5.68%) | 20260505 09:07:14.425 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 09:07:14.425 | INFO | High Water Breakdown: | 20260505 09:07:14.425 | INFO | 61% : 8.85 GiB ( 94.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 09:07:14.425 | INFO | 38.8% : 5.63 GiB ( 84.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 09:07:14.425 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 09:07:14.425 | INFO | ------------------------------------------ | 20260505 09:07:14.425 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:07:14.425 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 09:07:14.425 | INFO | Total mem usage: 26.5 GiB | 20260505 09:07:14.425 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 09:07:14.425 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 09:07:14.425 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 09:07:14.425 | INFO | High Water Breakdown: | 20260505 09:07:14.425 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:07:14.425 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:07:14.425 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:07:14.425 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:07:14.425 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:07:14.425 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:07:14.425 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 09:07:14.425 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 09:07:14.425 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:07:14.425 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 09:07:14.425 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 09:07:14.425 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 09:07:14.425 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 09:07:14.425 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 09:07:14.425 | INFO | ------------------------------------------ | 20260505 09:07:14.425 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:07:14.425 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 09:07:14.425 | INFO | Total mem usage: 25.6 GiB | 20260505 09:07:14.425 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 09:07:14.425 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 09:07:14.425 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 09:07:14.425 | INFO | High Water Breakdown: | 20260505 09:07:14.425 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 09:07:14.425 | INFO | ------------------------------------------ | 20260505 09:07:14.425 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:07:14.425 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 09:07:14.425 | INFO | Total mem usage: 15.7 GiB | 20260505 09:07:14.425 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 09:07:14.425 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 09:07:14.425 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 09:07:14.425 | INFO | High Water Breakdown: | 20260505 09:07:14.425 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 09:07:14.425 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 09:07:14.425 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 09:07:14.425 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 09:07:14.425 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 09:07:14.425 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 09:07:14.425 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 09:07:14.425 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 09:07:14.425 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 09:07:14.425 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:07:14.425 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 09:07:14.425 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:07:14.425 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 09:07:14.425 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 09:07:14.425 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 09:07:14.425 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 09:07:14.425 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:07:14.425 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 09:07:14.425 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:07:14.425 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:07:14.425 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:07:14.425 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 09:07:14.425 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 09:07:14.425 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:07:14.425 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 09:07:14.425 | INFO | ------------------------------------------ | 20260505 09:07:14.425 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:07:14.426 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 09:07:14.426 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:07:14.426 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:07:14.426 | INFO | PerfReport: Full Execution -- 3.25e+01 / 7.56e+01, 1.33e+01 / 1.95e+00 | 20260505 09:07:14.426 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 1.03e-01, 1.00e-02 / 4.70e-03 | 20260505 09:07:14.426 | INFO | PerfReport: 11.65%, Baseline -- 3.79e+00 / 4.42e+00, 6.12e-01 / 3.43e-01 | 20260505 09:07:14.426 | INFO | PerfReport: 3.31%, Histogram -- 1.08e+00 / 1.45e+00, 2.05e-01 / 1.25e-01 | 20260505 09:07:14.426 | INFO | PerfReport: 15.47%, DME -- 4.03e+01 / 4.38e+01, 4.09e+00 / 1.61e+00 | 20260505 09:07:14.426 | INFO | PerfReport: 56.57%, FrameLabeling -- 1.84e+01 / 2.08e+01, 1.90e+00 / 2.13e+00 | 20260505 09:07:14.426 | INFO | PerfReport: 7.47%, PulseAccumulating -- 2.43e+00 / 2.98e+00, 4.12e-01 / 2.76e-01 | 20260505 09:07:14.426 | INFO | PerfReport: 3.86%, Metrics -- 1.26e+00 / 1.58e+00, 3.58e-01 / 2.01e-01 | 20260505 09:07:14.426 | INFO | PerfReport: 1.37%, PulseSerializing -- 4.46e-01 / 5.05e-01, 8.40e-02 / 3.34e-02 | 20260505 09:07:14.426 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 09:07:14.426 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:07:14.426 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:07:14.426 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.32e+01 / 0.00e+00 | 20260505 09:07:14.426 | INFO | PerfReport: 22.54%, Upload -- 2.29e+01 / 0.00e+00, 8.16e+00 / 0.00e+00 | 20260505 09:07:14.426 | INFO | PerfReport: 21.72%, Download -- 2.21e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260505 09:07:14.426 | INFO | PerfReport: 32.02%, Compute -- 3.25e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260505 09:07:14.426 | INFO | PerfReport: 1.78%, LockUpload -- 1.81e+00 / 0.00e+00, 7.51e+00 / 0.00e+00 | 20260505 09:07:14.426 | INFO | PerfReport: 1.07%, LockDownload -- 1.09e+00 / 0.00e+00, 2.61e+00 / 0.00e+00 | 20260505 09:07:14.426 | INFO | PerfReport: 20.86%, LockCompute -- 2.12e+01 / 0.00e+00, 1.77e+01 / 0.00e+00 | 20260505 09:07:14.426 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.81e-02 / 0.00e+00 | 20260505 09:07:14.561 | INFO | Starting a round of Baz IO | 20260505 09:07:17.839 | INFO | Populating RTMetrics report | 20260505 09:07:17.839 | INFO | RTMetrics report populated | 20260505 09:07:17.839 | INFO | Opening tmp RTMetrics json file | 20260505 09:07:17.860 | INFO | Serializing RTMetrics to json | 20260505 09:07:17.892 | INFO | Serialization done, swapping tmp file now... | 20260505 09:07:17.911 | INFO | RTMetrics done | 20260505 09:07:17.986 | INFO | Finished a round of Baz IO: Wrote 2.446 GiB in 3.425 seconds | 20260505 09:07:17.986 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 26.55% / 0.02% / 73.44% | 20260505 09:07:17.986 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.18357% / 3.57628e-05% / 7.54992e-05% | 20260505 09:07:17.986 | INFO | Baz IO overall: | 20260505 09:07:17.986 | INFO | BytesWritten : 1151272088367 | 20260505 09:07:17.986 | INFO | NumEvents : 547695355826 | 20260505 09:07:19.561 | INFO | Analyzing chunk frames = [2007040,2007552) | 20260505 09:07:19.561 | INFO | MemInfo: { | 20260505 09:07:19.561 | INFO | "gpu" : " 17 GB", | 20260505 09:07:19.561 | INFO | "huge" : "", | 20260505 09:07:19.561 | INFO | "huge_shared" : " 27 GB", | 20260505 09:07:19.561 | INFO | "malloc" : " 14 GB", | 20260505 09:07:19.561 | INFO | "pinned" : " 29 GB" | 20260505 09:07:19.561 | INFO | } | 20260505 09:07:19.561 | INFO | Perf report for the last 5.13691 seconds: | 20260505 09:07:19.561 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:07:19.561 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:19.561 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:19.561 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:19.561 | INFO | ANALYSIS : 65.3%, 2.9, 99.8ms, 0 | 20260505 09:07:19.561 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:19.561 | INFO | RT_METRICS : 47.2%, 1.0, 24.7ms, 0 | 20260505 09:07:19.561 | INFO | PRE_HQ : 63.0%, 1.9, 60.8ms, 0 | 20260505 09:07:19.561 | INFO | BAZWRITER : 66.0%, 4.2, 142.0ms, 0 | 20260505 09:07:19.561 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 09:07:19.562 | INFO | Uploaded 36GiB to GPU in 2.23837s (16.0832GiB/s) | 20260505 09:07:19.562 | INFO | Downloaded 38GiB to GPU in 2.34877s (16.1787GiB/s) | 20260505 09:07:24.700 | INFO | Analyzing chunk frames = [2007552,2008064) | 20260505 09:07:24.700 | INFO | MemInfo: { | 20260505 09:07:24.700 | INFO | "gpu" : " 17 GB", | 20260505 09:07:24.700 | INFO | "huge" : "", | 20260505 09:07:24.700 | INFO | "huge_shared" : " 27 GB", | 20260505 09:07:24.700 | INFO | "malloc" : " 14 GB", | 20260505 09:07:24.700 | INFO | "pinned" : " 29 GB" | 20260505 09:07:24.700 | INFO | } | 20260505 09:07:24.700 | INFO | Perf report for the last 5.1388 seconds: | 20260505 09:07:24.700 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:07:24.700 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:24.700 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:24.700 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:24.700 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 09:07:24.700 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:24.700 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:24.700 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 09:07:24.700 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:24.700 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:24.700 | INFO | Uploaded 36GiB to GPU in 2.24224s (16.0554GiB/s) | 20260505 09:07:24.700 | INFO | Downloaded 34GiB to GPU in 2.11063s (16.1089GiB/s) | 20260505 09:07:29.839 | INFO | Analyzing chunk frames = [2008064,2008576) | 20260505 09:07:29.839 | INFO | MemInfo: { | 20260505 09:07:29.839 | INFO | "gpu" : " 17 GB", | 20260505 09:07:29.839 | INFO | "huge" : "", | 20260505 09:07:29.839 | INFO | "huge_shared" : " 27 GB", | 20260505 09:07:29.839 | INFO | "malloc" : " 14 GB", | 20260505 09:07:29.839 | INFO | "pinned" : " 29 GB" | 20260505 09:07:29.839 | INFO | } | 20260505 09:07:29.839 | INFO | Perf report for the last 5.13911 seconds: | 20260505 09:07:29.839 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:07:29.839 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:29.839 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:29.839 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:29.839 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 09:07:29.839 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:29.839 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:29.839 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 09:07:29.839 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:29.839 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:29.839 | INFO | Uploaded 36GiB to GPU in 2.17095s (16.5826GiB/s) | 20260505 09:07:29.839 | INFO | Downloaded 34GiB to GPU in 2.06924s (16.4312GiB/s) | 20260505 09:07:34.980 | INFO | Analyzing chunk frames = [2008576,2009088) | 20260505 09:07:34.980 | INFO | MemInfo: { | 20260505 09:07:34.980 | INFO | "gpu" : " 17 GB", | 20260505 09:07:34.980 | INFO | "huge" : "", | 20260505 09:07:34.980 | INFO | "huge_shared" : " 27 GB", | 20260505 09:07:34.980 | INFO | "malloc" : " 14 GB", | 20260505 09:07:34.980 | INFO | "pinned" : " 29 GB" | 20260505 09:07:34.980 | INFO | } | 20260505 09:07:34.980 | INFO | Perf report for the last 5.1411 seconds: | 20260505 09:07:34.980 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:07:34.980 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:34.980 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:34.980 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:34.980 | INFO | ANALYSIS : 63.6%, 3.0, 98.8ms, 0 | 20260505 09:07:34.980 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:34.980 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:34.980 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:07:34.980 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:34.980 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:34.981 | INFO | Uploaded 36GiB to GPU in 2.18955s (16.4417GiB/s) | 20260505 09:07:34.981 | INFO | Downloaded 34GiB to GPU in 2.06882s (16.4345GiB/s) | 20260505 09:07:40.119 | INFO | Analyzing chunk frames = [2009088,2009600) | 20260505 09:07:40.119 | INFO | MemInfo: { | 20260505 09:07:40.119 | INFO | "gpu" : " 17 GB", | 20260505 09:07:40.119 | INFO | "huge" : "", | 20260505 09:07:40.119 | INFO | "huge_shared" : " 27 GB", | 20260505 09:07:40.119 | INFO | "malloc" : " 14 GB", | 20260505 09:07:40.119 | INFO | "pinned" : " 29 GB" | 20260505 09:07:40.119 | INFO | } | 20260505 09:07:40.119 | INFO | Perf report for the last 5.13886 seconds: | 20260505 09:07:40.119 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:07:40.119 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:40.119 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:40.119 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:40.119 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 09:07:40.119 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:40.119 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:40.119 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 09:07:40.119 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:40.120 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:40.120 | INFO | Uploaded 36GiB to GPU in 2.23423s (16.1129GiB/s) | 20260505 09:07:40.120 | INFO | Downloaded 34GiB to GPU in 2.11368s (16.0857GiB/s) | 20260505 09:07:45.261 | INFO | Analyzing chunk frames = [2009600,2010112) | 20260505 09:07:45.261 | INFO | MemInfo: { | 20260505 09:07:45.261 | INFO | "gpu" : " 17 GB", | 20260505 09:07:45.261 | INFO | "huge" : "", | 20260505 09:07:45.261 | INFO | "huge_shared" : " 27 GB", | 20260505 09:07:45.261 | INFO | "malloc" : " 14 GB", | 20260505 09:07:45.261 | INFO | "pinned" : " 29 GB" | 20260505 09:07:45.261 | INFO | } | 20260505 09:07:45.261 | INFO | Perf report for the last 5.1415 seconds: | 20260505 09:07:45.261 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:07:45.261 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:45.261 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:45.261 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:45.261 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 09:07:45.261 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:45.261 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:45.261 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:07:45.261 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:45.261 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:45.261 | INFO | Uploaded 36GiB to GPU in 2.16795s (16.6056GiB/s) | 20260505 09:07:45.261 | INFO | Downloaded 34GiB to GPU in 2.06062s (16.4999GiB/s) | 20260505 09:07:50.397 | INFO | Analyzing chunk frames = [2010112,2010624) | 20260505 09:07:50.397 | INFO | MemInfo: { | 20260505 09:07:50.397 | INFO | "gpu" : " 17 GB", | 20260505 09:07:50.397 | INFO | "huge" : "", | 20260505 09:07:50.397 | INFO | "huge_shared" : " 27 GB", | 20260505 09:07:50.397 | INFO | "malloc" : " 14 GB", | 20260505 09:07:50.397 | INFO | "pinned" : " 29 GB" | 20260505 09:07:50.397 | INFO | } | 20260505 09:07:50.397 | INFO | Perf report for the last 5.13613 seconds: | 20260505 09:07:50.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:07:50.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:50.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:50.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:50.397 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 09:07:50.397 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:50.397 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:50.397 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260505 09:07:50.397 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:50.397 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:50.397 | INFO | Uploaded 36GiB to GPU in 2.21924s (16.2217GiB/s) | 20260505 09:07:50.397 | INFO | Downloaded 34GiB to GPU in 2.13688s (15.911GiB/s) | 20260505 09:07:55.536 | INFO | Analyzing chunk frames = [2010624,2011136) | 20260505 09:07:55.536 | INFO | MemInfo: { | 20260505 09:07:55.536 | INFO | "gpu" : " 17 GB", | 20260505 09:07:55.536 | INFO | "huge" : "", | 20260505 09:07:55.536 | INFO | "huge_shared" : " 27 GB", | 20260505 09:07:55.536 | INFO | "malloc" : " 14 GB", | 20260505 09:07:55.536 | INFO | "pinned" : " 29 GB" | 20260505 09:07:55.536 | INFO | } | 20260505 09:07:55.536 | INFO | Perf report for the last 5.13904 seconds: | 20260505 09:07:55.536 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:07:55.536 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:55.536 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:55.536 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:55.536 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 09:07:55.536 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:55.536 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:55.536 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 09:07:55.536 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:55.536 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:07:55.536 | INFO | Uploaded 36GiB to GPU in 2.20166s (16.3513GiB/s) | 20260505 09:07:55.536 | INFO | Downloaded 34GiB to GPU in 2.04656s (16.6132GiB/s) | 20260505 09:07:55.673 | INFO | Starting a round of Baz IO | 20260505 09:07:58.953 | INFO | Populating RTMetrics report | 20260505 09:07:58.953 | INFO | RTMetrics report populated | 20260505 09:07:58.953 | INFO | Opening tmp RTMetrics json file | 20260505 09:07:58.954 | INFO | Serializing RTMetrics to json | 20260505 09:07:58.955 | INFO | Serialization done, swapping tmp file now... | 20260505 09:07:58.955 | INFO | RTMetrics done | 20260505 09:07:59.128 | INFO | Finished a round of Baz IO: Wrote 2.444 GiB in 3.455 seconds | 20260505 09:07:59.128 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 26.53% / 0.02% / 73.45% | 20260505 09:07:59.128 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.17114% / 1.58946e-05% / 7.15256e-05% | 20260505 09:07:59.128 | INFO | Baz IO overall: | 20260505 09:07:59.128 | INFO | BytesWritten : 1153896374885 | 20260505 09:07:59.128 | INFO | NumEvents : 548950268879 | 20260505 09:08:00.675 | INFO | Analyzing chunk frames = [2011136,2011648) | 20260505 09:08:00.675 | INFO | MemInfo: { | 20260505 09:08:00.675 | INFO | "gpu" : " 17 GB", | 20260505 09:08:00.675 | INFO | "huge" : "", | 20260505 09:08:00.675 | INFO | "huge_shared" : " 27 GB", | 20260505 09:08:00.675 | INFO | "malloc" : " 14 GB", | 20260505 09:08:00.675 | INFO | "pinned" : " 29 GB" | 20260505 09:08:00.675 | INFO | } | 20260505 09:08:00.676 | INFO | Perf report for the last 5.13962 seconds: | 20260505 09:08:00.676 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:08:00.676 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:00.676 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:00.676 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:00.676 | INFO | ANALYSIS : 65.2%, 2.9, 99.3ms, 0 | 20260505 09:08:00.676 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:00.676 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 09:08:00.676 | INFO | PRE_HQ : 63.3%, 1.9, 61.4ms, 0 | 20260505 09:08:00.676 | INFO | BAZWRITER : 66.5%, 4.2, 144.4ms, 0 | 20260505 09:08:00.676 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 09:08:00.676 | INFO | Uploaded 36GiB to GPU in 2.28788s (15.7351GiB/s) | 20260505 09:08:00.676 | INFO | Downloaded 38GiB to GPU in 2.38813s (15.9121GiB/s) | 20260505 09:08:05.815 | INFO | Analyzing chunk frames = [2011648,2012160) | 20260505 09:08:05.815 | INFO | MemInfo: { | 20260505 09:08:05.815 | INFO | "gpu" : " 17 GB", | 20260505 09:08:05.815 | INFO | "huge" : "", | 20260505 09:08:05.815 | INFO | "huge_shared" : " 27 GB", | 20260505 09:08:05.815 | INFO | "malloc" : " 14 GB", | 20260505 09:08:05.815 | INFO | "pinned" : " 29 GB" | 20260505 09:08:05.815 | INFO | } | 20260505 09:08:05.815 | INFO | Perf report for the last 5.13926 seconds: | 20260505 09:08:05.815 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:08:05.815 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:05.815 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:05.815 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:05.815 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 09:08:05.815 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:05.815 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:05.815 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 09:08:05.815 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:05.815 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:05.815 | INFO | Uploaded 36GiB to GPU in 2.21716s (16.237GiB/s) | 20260505 09:08:05.815 | INFO | Downloaded 34GiB to GPU in 2.07733s (16.3672GiB/s) | 20260505 09:08:10.956 | INFO | Analyzing chunk frames = [2012160,2012672) | 20260505 09:08:10.956 | INFO | MemInfo: { | 20260505 09:08:10.956 | INFO | "gpu" : " 17 GB", | 20260505 09:08:10.956 | INFO | "huge" : "", | 20260505 09:08:10.956 | INFO | "huge_shared" : " 27 GB", | 20260505 09:08:10.956 | INFO | "malloc" : " 14 GB", | 20260505 09:08:10.956 | INFO | "pinned" : " 29 GB" | 20260505 09:08:10.956 | INFO | } | 20260505 09:08:10.956 | INFO | Perf report for the last 5.14138 seconds: | 20260505 09:08:10.956 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:08:10.956 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:10.956 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:10.956 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:10.956 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260505 09:08:10.956 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:10.956 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:10.956 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 09:08:10.957 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:10.957 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:10.957 | INFO | Uploaded 36GiB to GPU in 2.1736s (16.5624GiB/s) | 20260505 09:08:10.957 | INFO | Downloaded 34GiB to GPU in 2.06787s (16.4421GiB/s) | 20260505 09:08:16.094 | INFO | Analyzing chunk frames = [2012672,2013184) | 20260505 09:08:16.094 | INFO | MemInfo: { | 20260505 09:08:16.094 | INFO | "gpu" : " 17 GB", | 20260505 09:08:16.094 | INFO | "huge" : "", | 20260505 09:08:16.094 | INFO | "huge_shared" : " 27 GB", | 20260505 09:08:16.094 | INFO | "malloc" : " 14 GB", | 20260505 09:08:16.094 | INFO | "pinned" : " 29 GB" | 20260505 09:08:16.094 | INFO | } | 20260505 09:08:16.094 | INFO | Perf report for the last 5.13826 seconds: | 20260505 09:08:16.095 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:08:16.095 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:16.095 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:16.095 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:16.095 | INFO | ANALYSIS : 63.4%, 3.0, 98.5ms, 0 | 20260505 09:08:16.095 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:16.095 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:16.095 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 09:08:16.095 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:16.095 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:16.095 | INFO | Uploaded 36GiB to GPU in 2.16253s (16.6471GiB/s) | 20260505 09:08:16.095 | INFO | Downloaded 34GiB to GPU in 2.06934s (16.4304GiB/s) | 20260505 09:08:21.232 | INFO | Analyzing chunk frames = [2013184,2013696) | 20260505 09:08:21.232 | INFO | MemInfo: { | 20260505 09:08:21.232 | INFO | "gpu" : " 17 GB", | 20260505 09:08:21.232 | INFO | "huge" : "", | 20260505 09:08:21.232 | INFO | "huge_shared" : " 27 GB", | 20260505 09:08:21.232 | INFO | "malloc" : " 14 GB", | 20260505 09:08:21.232 | INFO | "pinned" : " 29 GB" | 20260505 09:08:21.233 | INFO | } | 20260505 09:08:21.233 | INFO | Perf report for the last 5.13825 seconds: | 20260505 09:08:21.233 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:08:21.233 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:21.233 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:21.233 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:21.233 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 09:08:21.233 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:21.233 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:21.233 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 09:08:21.233 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:21.233 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:21.233 | INFO | Uploaded 36GiB to GPU in 2.23067s (16.1386GiB/s) | 20260505 09:08:21.233 | INFO | Downloaded 34GiB to GPU in 2.11267s (16.0934GiB/s) | 20260505 09:08:26.373 | INFO | Analyzing chunk frames = [2013696,2014208) | 20260505 09:08:26.373 | INFO | MemInfo: { | 20260505 09:08:26.373 | INFO | "gpu" : " 17 GB", | 20260505 09:08:26.373 | INFO | "huge" : "", | 20260505 09:08:26.373 | INFO | "huge_shared" : " 27 GB", | 20260505 09:08:26.373 | INFO | "malloc" : " 14 GB", | 20260505 09:08:26.373 | INFO | "pinned" : " 29 GB" | 20260505 09:08:26.373 | INFO | } | 20260505 09:08:26.373 | INFO | Perf report for the last 5.14009 seconds: | 20260505 09:08:26.373 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:08:26.373 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:26.373 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:26.373 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:26.373 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 09:08:26.373 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:26.373 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:26.373 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:08:26.373 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:26.373 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:26.373 | INFO | Uploaded 36GiB to GPU in 2.16979s (16.5915GiB/s) | 20260505 09:08:26.373 | INFO | Downloaded 34GiB to GPU in 2.06164s (16.4918GiB/s) | 20260505 09:08:31.512 | INFO | Analyzing chunk frames = [2014208,2014720) | 20260505 09:08:31.512 | INFO | MemInfo: { | 20260505 09:08:31.512 | INFO | "gpu" : " 17 GB", | 20260505 09:08:31.512 | INFO | "huge" : "", | 20260505 09:08:31.512 | INFO | "huge_shared" : " 27 GB", | 20260505 09:08:31.512 | INFO | "malloc" : " 14 GB", | 20260505 09:08:31.512 | INFO | "pinned" : " 29 GB" | 20260505 09:08:31.512 | INFO | } | 20260505 09:08:31.512 | INFO | Perf report for the last 5.1393 seconds: | 20260505 09:08:31.512 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:08:31.512 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:31.512 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:31.512 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:31.512 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 09:08:31.512 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:31.512 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:31.512 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 09:08:31.512 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:31.512 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:31.513 | INFO | Uploaded 36GiB to GPU in 2.18781s (16.4548GiB/s) | 20260505 09:08:31.513 | INFO | Downloaded 34GiB to GPU in 2.13696s (15.9105GiB/s) | 20260505 09:08:36.650 | INFO | Analyzing chunk frames = [2014720,2015232) | 20260505 09:08:36.650 | INFO | MemInfo: { | 20260505 09:08:36.650 | INFO | "gpu" : " 17 GB", | 20260505 09:08:36.650 | INFO | "huge" : "", | 20260505 09:08:36.650 | INFO | "huge_shared" : " 27 GB", | 20260505 09:08:36.650 | INFO | "malloc" : " 14 GB", | 20260505 09:08:36.650 | INFO | "pinned" : " 29 GB" | 20260505 09:08:36.650 | INFO | } | 20260505 09:08:36.650 | INFO | Perf report for the last 5.13819 seconds: | 20260505 09:08:36.651 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:08:36.651 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:36.651 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:36.651 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:36.651 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 09:08:36.651 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:36.651 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:36.651 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 09:08:36.651 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:36.651 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:36.651 | INFO | Uploaded 36GiB to GPU in 2.202s (16.3487GiB/s) | 20260505 09:08:36.651 | INFO | Downloaded 34GiB to GPU in 2.04982s (16.5868GiB/s) | 20260505 09:08:36.651 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 09:08:36.651 | INFO | Total mem usage: 15.3 GiB | 20260505 09:08:36.651 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 09:08:36.651 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -5.69%) | 20260505 09:08:36.651 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 09:08:36.651 | INFO | High Water Breakdown: | 20260505 09:08:36.651 | INFO | 61% : 8.85 GiB ( 94.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 09:08:36.651 | INFO | 38.9% : 5.63 GiB ( 84.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 09:08:36.651 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 09:08:36.651 | INFO | ------------------------------------------ | 20260505 09:08:36.651 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:08:36.651 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 09:08:36.651 | INFO | Total mem usage: 26.5 GiB | 20260505 09:08:36.651 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 09:08:36.651 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 09:08:36.651 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 09:08:36.651 | INFO | High Water Breakdown: | 20260505 09:08:36.651 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:08:36.651 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:08:36.651 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:08:36.651 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:08:36.651 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:08:36.651 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:08:36.651 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 09:08:36.651 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 09:08:36.651 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:08:36.651 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 09:08:36.651 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 09:08:36.651 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 09:08:36.651 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 09:08:36.651 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 09:08:36.651 | INFO | ------------------------------------------ | 20260505 09:08:36.651 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:08:36.651 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 09:08:36.651 | INFO | Total mem usage: 25.6 GiB | 20260505 09:08:36.651 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 09:08:36.652 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 09:08:36.652 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 09:08:36.652 | INFO | High Water Breakdown: | 20260505 09:08:36.652 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 09:08:36.652 | INFO | ------------------------------------------ | 20260505 09:08:36.652 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:08:36.652 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 09:08:36.652 | INFO | Total mem usage: 15.7 GiB | 20260505 09:08:36.652 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 09:08:36.652 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 09:08:36.652 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 09:08:36.652 | INFO | High Water Breakdown: | 20260505 09:08:36.652 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 09:08:36.652 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 09:08:36.652 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 09:08:36.652 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 09:08:36.652 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 09:08:36.652 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 09:08:36.652 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 09:08:36.652 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 09:08:36.652 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 09:08:36.652 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:08:36.652 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 09:08:36.652 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:08:36.652 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 09:08:36.652 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 09:08:36.652 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 09:08:36.652 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 09:08:36.652 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:08:36.652 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 09:08:36.652 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:08:36.652 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:08:36.652 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:08:36.652 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 09:08:36.652 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 09:08:36.652 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:08:36.652 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 09:08:36.652 | INFO | ------------------------------------------ | 20260505 09:08:36.652 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:08:36.652 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 09:08:36.652 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:08:36.652 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:08:36.652 | INFO | PerfReport: Full Execution -- 3.25e+01 / 7.56e+01, 1.37e+01 / 2.08e+00 | 20260505 09:08:36.652 | INFO | PerfReport: 0.29%, AggregateStats -- 9.32e-02 / 1.04e-01, 1.11e-02 / 3.96e-03 | 20260505 09:08:36.652 | INFO | PerfReport: 11.64%, Baseline -- 3.79e+00 / 4.42e+00, 6.19e-01 / 3.10e-01 | 20260505 09:08:36.652 | INFO | PerfReport: 3.32%, Histogram -- 1.08e+00 / 1.48e+00, 2.10e-01 / 9.02e-02 | 20260505 09:08:36.652 | INFO | PerfReport: 15.49%, DME -- 4.03e+01 / 4.39e+01, 4.15e+00 / 1.58e+00 | 20260505 09:08:36.652 | INFO | PerfReport: 56.59%, FrameLabeling -- 1.84e+01 / 2.06e+01, 1.91e+00 / 2.19e+00 | 20260505 09:08:36.652 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.42e+00 / 2.97e+00, 4.07e-01 / 2.52e-01 | 20260505 09:08:36.652 | INFO | PerfReport: 3.86%, Metrics -- 1.25e+00 / 1.55e+00, 3.60e-01 / 1.52e-01 | 20260505 09:08:36.652 | INFO | PerfReport: 1.37%, PulseSerializing -- 4.46e-01 / 4.96e-01, 8.40e-02 / 2.78e-02 | 20260505 09:08:36.652 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 09:08:36.652 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:08:36.652 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:08:36.652 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.35e+01 / 0.00e+00 | 20260505 09:08:36.652 | INFO | PerfReport: 22.62%, Upload -- 2.30e+01 / 0.00e+00, 8.29e+00 / 0.00e+00 | 20260505 09:08:36.652 | INFO | PerfReport: 21.74%, Download -- 2.21e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260505 09:08:36.652 | INFO | PerfReport: 32.02%, Compute -- 3.25e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 09:08:36.652 | INFO | PerfReport: 1.77%, LockUpload -- 1.80e+00 / 0.00e+00, 7.42e+00 / 0.00e+00 | 20260505 09:08:36.652 | INFO | PerfReport: 1.03%, LockDownload -- 1.05e+00 / 0.00e+00, 2.52e+00 / 0.00e+00 | 20260505 09:08:36.652 | INFO | PerfReport: 20.79%, LockCompute -- 2.11e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260505 09:08:36.652 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.69e-02 / 0.00e+00 | 20260505 09:08:36.788 | INFO | Starting a round of Baz IO | 20260505 09:08:40.068 | INFO | Populating RTMetrics report | 20260505 09:08:40.068 | INFO | RTMetrics report populated | 20260505 09:08:40.068 | INFO | Opening tmp RTMetrics json file | 20260505 09:08:40.073 | INFO | Serializing RTMetrics to json | 20260505 09:08:40.077 | INFO | Serialization done, swapping tmp file now... | 20260505 09:08:40.079 | INFO | RTMetrics done | 20260505 09:08:40.207 | INFO | Finished a round of Baz IO: Wrote 2.44 GiB in 3.418 seconds | 20260505 09:08:40.207 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 26.51% / 0.02% / 73.47% | 20260505 09:08:40.207 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.15212% / 1.98682e-05% / 0.000127157% | 20260505 09:08:40.207 | INFO | Baz IO overall: | 20260505 09:08:40.207 | INFO | BytesWritten : 1156515928018 | 20260505 09:08:40.207 | INFO | NumEvents : 550201593030 | 20260505 09:08:41.790 | INFO | Analyzing chunk frames = [2015232,2015744) | 20260505 09:08:41.790 | INFO | MemInfo: { | 20260505 09:08:41.790 | INFO | "gpu" : " 17 GB", | 20260505 09:08:41.790 | INFO | "huge" : "", | 20260505 09:08:41.790 | INFO | "huge_shared" : " 27 GB", | 20260505 09:08:41.790 | INFO | "malloc" : " 14 GB", | 20260505 09:08:41.790 | INFO | "pinned" : " 29 GB" | 20260505 09:08:41.790 | INFO | } | 20260505 09:08:41.790 | INFO | Perf report for the last 5.13994 seconds: | 20260505 09:08:41.790 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:08:41.790 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:41.790 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:41.790 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:41.790 | INFO | ANALYSIS : 65.4%, 2.9, 100.3ms, 0 | 20260505 09:08:41.790 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:41.790 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260505 09:08:41.791 | INFO | PRE_HQ : 63.0%, 1.8, 60.4ms, 0 | 20260505 09:08:41.791 | INFO | BAZWRITER : 65.8%, 4.1, 139.8ms, 0 | 20260505 09:08:41.791 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 09:08:41.791 | INFO | Uploaded 36GiB to GPU in 2.24047s (16.068GiB/s) | 20260505 09:08:41.791 | INFO | Downloaded 38GiB to GPU in 2.3486s (16.1799GiB/s) | 20260505 09:08:46.931 | INFO | Analyzing chunk frames = [2015744,2016256) | 20260505 09:08:46.931 | INFO | MemInfo: { | 20260505 09:08:46.931 | INFO | "gpu" : " 17 GB", | 20260505 09:08:46.931 | INFO | "huge" : "", | 20260505 09:08:46.931 | INFO | "huge_shared" : " 27 GB", | 20260505 09:08:46.931 | INFO | "malloc" : " 14 GB", | 20260505 09:08:46.931 | INFO | "pinned" : " 29 GB" | 20260505 09:08:46.931 | INFO | } | 20260505 09:08:46.931 | INFO | Perf report for the last 5.14094 seconds: | 20260505 09:08:46.931 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:08:46.931 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:46.931 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:46.931 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:46.931 | INFO | ANALYSIS : 64.3%, 3.0, 100.5ms, 0 | 20260505 09:08:46.931 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:46.932 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:46.932 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:08:46.932 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:46.932 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:46.932 | INFO | Uploaded 36GiB to GPU in 2.21752s (16.2343GiB/s) | 20260505 09:08:46.932 | INFO | Downloaded 34GiB to GPU in 2.08603s (16.2989GiB/s) | 20260505 09:08:52.070 | INFO | Analyzing chunk frames = [2016256,2016768) | 20260505 09:08:52.070 | INFO | MemInfo: { | 20260505 09:08:52.070 | INFO | "gpu" : " 17 GB", | 20260505 09:08:52.070 | INFO | "huge" : "", | 20260505 09:08:52.070 | INFO | "huge_shared" : " 27 GB", | 20260505 09:08:52.070 | INFO | "malloc" : " 14 GB", | 20260505 09:08:52.070 | INFO | "pinned" : " 29 GB" | 20260505 09:08:52.070 | INFO | } | 20260505 09:08:52.070 | INFO | Perf report for the last 5.13897 seconds: | 20260505 09:08:52.070 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:08:52.070 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:52.070 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:52.070 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:52.070 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260505 09:08:52.070 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:52.070 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:52.070 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:08:52.070 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:52.070 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:52.070 | INFO | Uploaded 36GiB to GPU in 2.17136s (16.5795GiB/s) | 20260505 09:08:52.071 | INFO | Downloaded 34GiB to GPU in 2.0686s (16.4363GiB/s) | 20260505 09:08:57.208 | INFO | Analyzing chunk frames = [2016768,2017280) | 20260505 09:08:57.208 | INFO | MemInfo: { | 20260505 09:08:57.208 | INFO | "gpu" : " 17 GB", | 20260505 09:08:57.208 | INFO | "huge" : "", | 20260505 09:08:57.208 | INFO | "huge_shared" : " 27 GB", | 20260505 09:08:57.208 | INFO | "malloc" : " 14 GB", | 20260505 09:08:57.208 | INFO | "pinned" : " 29 GB" | 20260505 09:08:57.208 | INFO | } | 20260505 09:08:57.208 | INFO | Perf report for the last 5.13773 seconds: | 20260505 09:08:57.208 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:08:57.208 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:57.208 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:57.208 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:57.208 | INFO | ANALYSIS : 63.5%, 3.0, 98.6ms, 0 | 20260505 09:08:57.208 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:57.208 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:57.208 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:08:57.208 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:57.208 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:08:57.208 | INFO | Uploaded 36GiB to GPU in 2.16843s (16.6019GiB/s) | 20260505 09:08:57.208 | INFO | Downloaded 34GiB to GPU in 2.06803s (16.4408GiB/s) | 20260505 09:09:02.347 | INFO | Analyzing chunk frames = [2017280,2017792) | 20260505 09:09:02.347 | INFO | MemInfo: { | 20260505 09:09:02.347 | INFO | "gpu" : " 17 GB", | 20260505 09:09:02.347 | INFO | "huge" : "", | 20260505 09:09:02.347 | INFO | "huge_shared" : " 27 GB", | 20260505 09:09:02.347 | INFO | "malloc" : " 14 GB", | 20260505 09:09:02.347 | INFO | "pinned" : " 29 GB" | 20260505 09:09:02.347 | INFO | } | 20260505 09:09:02.347 | INFO | Perf report for the last 5.13904 seconds: | 20260505 09:09:02.347 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:09:02.347 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:02.347 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:02.347 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:02.347 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 09:09:02.347 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:02.347 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:02.347 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 09:09:02.347 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:02.347 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:02.347 | INFO | Uploaded 36GiB to GPU in 2.23605s (16.0998GiB/s) | 20260505 09:09:02.347 | INFO | Downloaded 34GiB to GPU in 2.10748s (16.133GiB/s) | 20260505 09:09:07.487 | INFO | Analyzing chunk frames = [2017792,2018304) | 20260505 09:09:07.487 | INFO | MemInfo: { | 20260505 09:09:07.487 | INFO | "gpu" : " 17 GB", | 20260505 09:09:07.487 | INFO | "huge" : "", | 20260505 09:09:07.487 | INFO | "huge_shared" : " 27 GB", | 20260505 09:09:07.487 | INFO | "malloc" : " 14 GB", | 20260505 09:09:07.487 | INFO | "pinned" : " 29 GB" | 20260505 09:09:07.487 | INFO | } | 20260505 09:09:07.487 | INFO | Perf report for the last 5.14045 seconds: | 20260505 09:09:07.488 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:09:07.488 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:07.488 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:07.488 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:07.488 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 09:09:07.488 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:07.488 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:07.488 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 09:09:07.488 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:07.488 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:07.488 | INFO | Uploaded 36GiB to GPU in 2.16495s (16.6286GiB/s) | 20260505 09:09:07.488 | INFO | Downloaded 34GiB to GPU in 2.05939s (16.5097GiB/s) | 20260505 09:09:12.627 | INFO | Analyzing chunk frames = [2018304,2018816) | 20260505 09:09:12.627 | INFO | MemInfo: { | 20260505 09:09:12.627 | INFO | "gpu" : " 17 GB", | 20260505 09:09:12.627 | INFO | "huge" : "", | 20260505 09:09:12.627 | INFO | "huge_shared" : " 27 GB", | 20260505 09:09:12.627 | INFO | "malloc" : " 14 GB", | 20260505 09:09:12.627 | INFO | "pinned" : " 29 GB" | 20260505 09:09:12.627 | INFO | } | 20260505 09:09:12.627 | INFO | Perf report for the last 5.13936 seconds: | 20260505 09:09:12.627 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:09:12.627 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:12.627 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:12.627 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:12.627 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 09:09:12.627 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:12.627 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:12.627 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 09:09:12.627 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:12.627 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:12.627 | INFO | Uploaded 36GiB to GPU in 2.20052s (16.3597GiB/s) | 20260505 09:09:12.627 | INFO | Downloaded 34GiB to GPU in 2.15381s (15.786GiB/s) | 20260505 09:09:17.766 | INFO | Analyzing chunk frames = [2018816,2019328) | 20260505 09:09:17.766 | INFO | MemInfo: { | 20260505 09:09:17.766 | INFO | "gpu" : " 17 GB", | 20260505 09:09:17.766 | INFO | "huge" : "", | 20260505 09:09:17.766 | INFO | "huge_shared" : " 27 GB", | 20260505 09:09:17.766 | INFO | "malloc" : " 14 GB", | 20260505 09:09:17.766 | INFO | "pinned" : " 29 GB" | 20260505 09:09:17.766 | INFO | } | 20260505 09:09:17.766 | INFO | Perf report for the last 5.13935 seconds: | 20260505 09:09:17.766 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:09:17.766 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:17.766 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:17.766 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:17.766 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 09:09:17.766 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:17.766 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:17.766 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 09:09:17.766 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:17.766 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:17.766 | INFO | Uploaded 36GiB to GPU in 2.1854s (16.473GiB/s) | 20260505 09:09:17.767 | INFO | Downloaded 34GiB to GPU in 2.05549s (16.541GiB/s) | 20260505 09:09:17.902 | INFO | Starting a round of Baz IO | 20260505 09:09:21.168 | INFO | Populating RTMetrics report | 20260505 09:09:21.168 | INFO | RTMetrics report populated | 20260505 09:09:21.168 | INFO | Opening tmp RTMetrics json file | 20260505 09:09:21.201 | INFO | Serializing RTMetrics to json | 20260505 09:09:21.202 | INFO | Serialization done, swapping tmp file now... | 20260505 09:09:21.203 | INFO | RTMetrics done | 20260505 09:09:21.310 | INFO | Finished a round of Baz IO: Wrote 2.445 GiB in 3.408 seconds | 20260505 09:09:21.310 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 26.50% / 0.02% / 73.49% | 20260505 09:09:21.310 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.15451% / 2.78155e-05% / 0.000103315% | 20260505 09:09:21.310 | INFO | Baz IO overall: | 20260505 09:09:21.310 | INFO | BytesWritten : 1159140932509 | 20260505 09:09:21.310 | INFO | NumEvents : 551456604269 | 20260505 09:09:22.905 | INFO | Analyzing chunk frames = [2019328,2019840) | 20260505 09:09:22.905 | INFO | MemInfo: { | 20260505 09:09:22.905 | INFO | "gpu" : " 17 GB", | 20260505 09:09:22.905 | INFO | "huge" : "", | 20260505 09:09:22.905 | INFO | "huge_shared" : " 27 GB", | 20260505 09:09:22.905 | INFO | "malloc" : " 14 GB", | 20260505 09:09:22.905 | INFO | "pinned" : " 29 GB" | 20260505 09:09:22.905 | INFO | } | 20260505 09:09:22.905 | INFO | Perf report for the last 5.13912 seconds: | 20260505 09:09:22.905 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:09:22.905 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:22.905 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:22.905 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:22.905 | INFO | ANALYSIS : 65.1%, 2.9, 99.5ms, 0 | 20260505 09:09:22.905 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:22.905 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260505 09:09:22.905 | INFO | PRE_HQ : 63.2%, 1.8, 60.6ms, 0 | 20260505 09:09:22.905 | INFO | BAZWRITER : 65.6%, 4.2, 140.3ms, 0 | 20260505 09:09:22.905 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 09:09:22.906 | INFO | Uploaded 36GiB to GPU in 2.28098s (15.7827GiB/s) | 20260505 09:09:22.906 | INFO | Downloaded 38GiB to GPU in 2.39017s (15.8984GiB/s) | 20260505 09:09:28.045 | INFO | Analyzing chunk frames = [2019840,2020352) | 20260505 09:09:28.045 | INFO | MemInfo: { | 20260505 09:09:28.045 | INFO | "gpu" : " 17 GB", | 20260505 09:09:28.045 | INFO | "huge" : "", | 20260505 09:09:28.045 | INFO | "huge_shared" : " 27 GB", | 20260505 09:09:28.045 | INFO | "malloc" : " 14 GB", | 20260505 09:09:28.045 | INFO | "pinned" : " 29 GB" | 20260505 09:09:28.045 | INFO | } | 20260505 09:09:28.045 | INFO | Perf report for the last 5.1394 seconds: | 20260505 09:09:28.045 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:09:28.045 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:28.045 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:28.045 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:28.045 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 09:09:28.045 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:28.045 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:28.045 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:09:28.045 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:28.045 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:28.045 | INFO | Uploaded 36GiB to GPU in 2.23683s (16.0942GiB/s) | 20260505 09:09:28.045 | INFO | Downloaded 34GiB to GPU in 2.11341s (16.0878GiB/s) | 20260505 09:09:33.183 | INFO | Analyzing chunk frames = [2020352,2020864) | 20260505 09:09:33.183 | INFO | MemInfo: { | 20260505 09:09:33.183 | INFO | "gpu" : " 17 GB", | 20260505 09:09:33.183 | INFO | "huge" : "", | 20260505 09:09:33.183 | INFO | "huge_shared" : " 27 GB", | 20260505 09:09:33.183 | INFO | "malloc" : " 14 GB", | 20260505 09:09:33.183 | INFO | "pinned" : " 29 GB" | 20260505 09:09:33.183 | INFO | } | 20260505 09:09:33.184 | INFO | Perf report for the last 5.13881 seconds: | 20260505 09:09:33.184 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:09:33.184 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:33.184 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:33.184 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:33.184 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260505 09:09:33.184 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:33.184 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:33.184 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 09:09:33.184 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:33.184 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:33.184 | INFO | Uploaded 36GiB to GPU in 2.16372s (16.638GiB/s) | 20260505 09:09:33.184 | INFO | Downloaded 34GiB to GPU in 2.06747s (16.4452GiB/s) | 20260505 09:09:38.323 | INFO | Analyzing chunk frames = [2020864,2021376) | 20260505 09:09:38.323 | INFO | MemInfo: { | 20260505 09:09:38.323 | INFO | "gpu" : " 17 GB", | 20260505 09:09:38.323 | INFO | "huge" : "", | 20260505 09:09:38.323 | INFO | "huge_shared" : " 27 GB", | 20260505 09:09:38.323 | INFO | "malloc" : " 14 GB", | 20260505 09:09:38.323 | INFO | "pinned" : " 29 GB" | 20260505 09:09:38.323 | INFO | } | 20260505 09:09:38.323 | INFO | Perf report for the last 5.13907 seconds: | 20260505 09:09:38.323 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:09:38.323 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:38.323 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:38.323 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:38.323 | INFO | ANALYSIS : 63.4%, 3.0, 98.6ms, 0 | 20260505 09:09:38.323 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:38.323 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:38.323 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 09:09:38.323 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:38.323 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:38.323 | INFO | Uploaded 36GiB to GPU in 2.17084s (16.5835GiB/s) | 20260505 09:09:38.323 | INFO | Downloaded 34GiB to GPU in 2.07139s (16.4141GiB/s) | 20260505 09:09:43.463 | INFO | Analyzing chunk frames = [2021376,2021888) | 20260505 09:09:43.463 | INFO | MemInfo: { | 20260505 09:09:43.463 | INFO | "gpu" : " 17 GB", | 20260505 09:09:43.463 | INFO | "huge" : "", | 20260505 09:09:43.463 | INFO | "huge_shared" : " 27 GB", | 20260505 09:09:43.463 | INFO | "malloc" : " 14 GB", | 20260505 09:09:43.463 | INFO | "pinned" : " 29 GB" | 20260505 09:09:43.463 | INFO | } | 20260505 09:09:43.463 | INFO | Perf report for the last 5.14039 seconds: | 20260505 09:09:43.463 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:09:43.463 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:43.463 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:43.463 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:43.463 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 09:09:43.463 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:43.463 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:43.463 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:09:43.463 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:43.463 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:43.463 | INFO | Uploaded 36GiB to GPU in 2.17084s (16.5834GiB/s) | 20260505 09:09:43.463 | INFO | Downloaded 34GiB to GPU in 2.05943s (16.5094GiB/s) | 20260505 09:09:48.603 | INFO | Analyzing chunk frames = [2021888,2022400) | 20260505 09:09:48.603 | INFO | MemInfo: { | 20260505 09:09:48.603 | INFO | "gpu" : " 17 GB", | 20260505 09:09:48.603 | INFO | "huge" : "", | 20260505 09:09:48.603 | INFO | "huge_shared" : " 27 GB", | 20260505 09:09:48.603 | INFO | "malloc" : " 14 GB", | 20260505 09:09:48.603 | INFO | "pinned" : " 29 GB" | 20260505 09:09:48.603 | INFO | } | 20260505 09:09:48.603 | INFO | Perf report for the last 5.13964 seconds: | 20260505 09:09:48.603 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:09:48.603 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:48.603 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:48.603 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:48.603 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 09:09:48.603 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:48.603 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:48.603 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 09:09:48.603 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:48.603 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:48.603 | INFO | Uploaded 36GiB to GPU in 2.16309s (16.6429GiB/s) | 20260505 09:09:48.603 | INFO | Downloaded 34GiB to GPU in 2.05905s (16.5125GiB/s) | 20260505 09:09:53.741 | INFO | Analyzing chunk frames = [2022400,2022912) | 20260505 09:09:53.741 | INFO | MemInfo: { | 20260505 09:09:53.741 | INFO | "gpu" : " 17 GB", | 20260505 09:09:53.741 | INFO | "huge" : "", | 20260505 09:09:53.741 | INFO | "huge_shared" : " 27 GB", | 20260505 09:09:53.741 | INFO | "malloc" : " 14 GB", | 20260505 09:09:53.741 | INFO | "pinned" : " 29 GB" | 20260505 09:09:53.741 | INFO | } | 20260505 09:09:53.741 | INFO | Perf report for the last 5.13813 seconds: | 20260505 09:09:53.741 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:09:53.741 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:53.741 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:53.741 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:53.741 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 09:09:53.741 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:53.741 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:53.741 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260505 09:09:53.741 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:53.741 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:53.741 | INFO | Uploaded 36GiB to GPU in 2.20208s (16.3482GiB/s) | 20260505 09:09:53.741 | INFO | Downloaded 34GiB to GPU in 2.13166s (15.95GiB/s) | 20260505 09:09:58.881 | INFO | Analyzing chunk frames = [2022912,2023424) | 20260505 09:09:58.881 | INFO | MemInfo: { | 20260505 09:09:58.881 | INFO | "gpu" : " 17 GB", | 20260505 09:09:58.881 | INFO | "huge" : "", | 20260505 09:09:58.881 | INFO | "huge_shared" : " 27 GB", | 20260505 09:09:58.881 | INFO | "malloc" : " 14 GB", | 20260505 09:09:58.881 | INFO | "pinned" : " 29 GB" | 20260505 09:09:58.881 | INFO | } | 20260505 09:09:58.881 | INFO | Perf report for the last 5.14016 seconds: | 20260505 09:09:58.881 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:09:58.881 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:58.881 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:58.881 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:58.881 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 09:09:58.881 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:58.881 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:58.881 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 09:09:58.881 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:58.881 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:09:58.881 | INFO | Uploaded 36GiB to GPU in 2.18328s (16.489GiB/s) | 20260505 09:09:58.881 | INFO | Downloaded 34GiB to GPU in 2.05232s (16.5666GiB/s) | 20260505 09:09:58.882 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 09:09:58.882 | INFO | Total mem usage: 15.3 GiB | 20260505 09:09:58.882 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 09:09:58.882 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.42%) | 20260505 09:09:58.882 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 09:09:58.882 | INFO | High Water Breakdown: | 20260505 09:09:58.882 | INFO | 60.9% : 8.77 GiB ( 93.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 09:09:58.882 | INFO | 38.9% : 5.61 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 09:09:58.882 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 09:09:58.882 | INFO | ------------------------------------------ | 20260505 09:09:58.882 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:09:58.882 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 09:09:58.882 | INFO | Total mem usage: 26.5 GiB | 20260505 09:09:58.882 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 09:09:58.882 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 09:09:58.882 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 09:09:58.882 | INFO | High Water Breakdown: | 20260505 09:09:58.882 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:09:58.882 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:09:58.882 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:09:58.882 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:09:58.882 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:09:58.882 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:09:58.882 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 09:09:58.882 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 09:09:58.882 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:09:58.882 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 09:09:58.882 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 09:09:58.882 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 09:09:58.882 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 09:09:58.882 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 09:09:58.882 | INFO | ------------------------------------------ | 20260505 09:09:58.882 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:09:58.882 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 09:09:58.882 | INFO | Total mem usage: 25.6 GiB | 20260505 09:09:58.882 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 09:09:58.882 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 09:09:58.882 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 09:09:58.882 | INFO | High Water Breakdown: | 20260505 09:09:58.882 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 09:09:58.882 | INFO | ------------------------------------------ | 20260505 09:09:58.882 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:09:58.882 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 09:09:58.882 | INFO | Total mem usage: 15.7 GiB | 20260505 09:09:58.882 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 09:09:58.882 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 09:09:58.882 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 09:09:58.882 | INFO | High Water Breakdown: | 20260505 09:09:58.882 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 09:09:58.882 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 09:09:58.882 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 09:09:58.883 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 09:09:58.883 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 09:09:58.883 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 09:09:58.883 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 09:09:58.883 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 09:09:58.883 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 09:09:58.883 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:09:58.883 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 09:09:58.883 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:09:58.883 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 09:09:58.883 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 09:09:58.883 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 09:09:58.883 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 09:09:58.883 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:09:58.883 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 09:09:58.883 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:09:58.883 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:09:58.883 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:09:58.883 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 09:09:58.883 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 09:09:58.883 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:09:58.883 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 09:09:58.883 | INFO | ------------------------------------------ | 20260505 09:09:58.883 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:09:58.883 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 09:09:58.883 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:09:58.883 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:09:58.883 | INFO | PerfReport: Full Execution -- 3.25e+01 / 7.53e+01, 1.37e+01 / 2.15e+00 | 20260505 09:09:58.883 | INFO | PerfReport: 0.29%, AggregateStats -- 9.33e-02 / 1.07e-01, 1.16e-02 / 1.47e-02 | 20260505 09:09:58.883 | INFO | PerfReport: 11.65%, Baseline -- 3.79e+00 / 4.41e+00, 6.16e-01 / 3.04e-01 | 20260505 09:09:58.883 | INFO | PerfReport: 3.31%, Histogram -- 1.07e+00 / 1.46e+00, 2.08e-01 / 1.04e-01 | 20260505 09:09:58.883 | INFO | PerfReport: 15.50%, DME -- 4.03e+01 / 4.40e+01, 4.12e+00 / 1.87e+00 | 20260505 09:09:58.883 | INFO | PerfReport: 56.51%, FrameLabeling -- 1.84e+01 / 2.03e+01, 1.87e+00 / 1.98e+00 | 20260505 09:09:58.883 | INFO | PerfReport: 7.50%, PulseAccumulating -- 2.44e+00 / 2.97e+00, 4.22e-01 / 2.78e-01 | 20260505 09:09:58.883 | INFO | PerfReport: 3.87%, Metrics -- 1.26e+00 / 1.56e+00, 3.60e-01 / 1.57e-01 | 20260505 09:09:58.883 | INFO | PerfReport: 1.37%, PulseSerializing -- 4.46e-01 / 4.97e-01, 8.34e-02 / 2.67e-02 | 20260505 09:09:58.883 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 09:09:58.883 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:09:58.883 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:09:58.883 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.31e+01 / 0.00e+00 | 20260505 09:09:58.883 | INFO | PerfReport: 22.54%, Upload -- 2.29e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260505 09:09:58.883 | INFO | PerfReport: 21.73%, Download -- 2.21e+01 / 0.00e+00, 8.15e+00 / 0.00e+00 | 20260505 09:09:58.883 | INFO | PerfReport: 32.01%, Compute -- 3.25e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 09:09:58.883 | INFO | PerfReport: 1.74%, LockUpload -- 1.77e+00 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260505 09:09:58.883 | INFO | PerfReport: 1.07%, LockDownload -- 1.09e+00 / 0.00e+00, 2.63e+00 / 0.00e+00 | 20260505 09:09:58.883 | INFO | PerfReport: 20.89%, LockCompute -- 2.12e+01 / 0.00e+00, 1.77e+01 / 0.00e+00 | 20260505 09:09:58.883 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.91e-02 / 0.00e+00 | 20260505 09:09:59.016 | INFO | Starting a round of Baz IO | 20260505 09:10:02.291 | INFO | Populating RTMetrics report | 20260505 09:10:02.291 | INFO | RTMetrics report populated | 20260505 09:10:02.291 | INFO | Opening tmp RTMetrics json file | 20260505 09:10:02.309 | INFO | Serializing RTMetrics to json | 20260505 09:10:02.312 | INFO | Serialization done, swapping tmp file now... | 20260505 09:10:02.315 | INFO | RTMetrics done | 20260505 09:10:02.450 | INFO | Finished a round of Baz IO: Wrote 2.444 GiB in 3.434 seconds | 20260505 09:10:02.450 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 26.48% / 0.02% / 73.50% | 20260505 09:10:02.450 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.14963% / 1.98682e-05% / 8.74201e-05% | 20260505 09:10:02.450 | INFO | Baz IO overall: | 20260505 09:10:02.450 | INFO | BytesWritten : 1161765479617 | 20260505 09:10:02.450 | INFO | NumEvents : 552711472477 | 20260505 09:10:04.020 | INFO | Analyzing chunk frames = [2023424,2023936) | 20260505 09:10:04.020 | INFO | MemInfo: { | 20260505 09:10:04.020 | INFO | "gpu" : " 17 GB", | 20260505 09:10:04.020 | INFO | "huge" : "", | 20260505 09:10:04.020 | INFO | "huge_shared" : " 27 GB", | 20260505 09:10:04.020 | INFO | "malloc" : " 14 GB", | 20260505 09:10:04.020 | INFO | "pinned" : " 29 GB" | 20260505 09:10:04.020 | INFO | } | 20260505 09:10:04.020 | INFO | Perf report for the last 5.1387 seconds: | 20260505 09:10:04.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:10:04.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:04.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:04.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:04.020 | INFO | ANALYSIS : 65.2%, 2.9, 99.5ms, 0 | 20260505 09:10:04.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:04.020 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260505 09:10:04.020 | INFO | PRE_HQ : 63.2%, 1.9, 60.8ms, 0 | 20260505 09:10:04.020 | INFO | BAZWRITER : 66.1%, 4.2, 142.0ms, 0 | 20260505 09:10:04.020 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 09:10:04.020 | INFO | Uploaded 36GiB to GPU in 2.26741s (15.8772GiB/s) | 20260505 09:10:04.020 | INFO | Downloaded 38GiB to GPU in 2.36637s (16.0583GiB/s) | 20260505 09:10:09.159 | INFO | Analyzing chunk frames = [2023936,2024448) | 20260505 09:10:09.159 | INFO | MemInfo: { | 20260505 09:10:09.159 | INFO | "gpu" : " 17 GB", | 20260505 09:10:09.159 | INFO | "huge" : "", | 20260505 09:10:09.159 | INFO | "huge_shared" : " 27 GB", | 20260505 09:10:09.159 | INFO | "malloc" : " 14 GB", | 20260505 09:10:09.159 | INFO | "pinned" : " 29 GB" | 20260505 09:10:09.159 | INFO | } | 20260505 09:10:09.159 | INFO | Perf report for the last 5.13911 seconds: | 20260505 09:10:09.163 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:10:09.163 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:09.163 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:09.163 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:09.163 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 09:10:09.163 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:09.163 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:09.163 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 09:10:09.163 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:09.163 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:09.163 | INFO | Uploaded 36GiB to GPU in 2.20362s (16.3368GiB/s) | 20260505 09:10:09.163 | INFO | Downloaded 34GiB to GPU in 2.08805s (16.2832GiB/s) | 20260505 09:10:14.299 | INFO | Analyzing chunk frames = [2024448,2024960) | 20260505 09:10:14.299 | INFO | MemInfo: { | 20260505 09:10:14.299 | INFO | "gpu" : " 17 GB", | 20260505 09:10:14.299 | INFO | "huge" : "", | 20260505 09:10:14.299 | INFO | "huge_shared" : " 27 GB", | 20260505 09:10:14.302 | INFO | "malloc" : " 14 GB", | 20260505 09:10:14.302 | INFO | "pinned" : " 29 GB" | 20260505 09:10:14.302 | INFO | } | 20260505 09:10:14.302 | INFO | Perf report for the last 5.14312 seconds: | 20260505 09:10:14.302 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:10:14.302 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:14.302 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:14.302 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:14.302 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 09:10:14.302 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:14.302 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:14.302 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:10:14.302 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:14.302 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:14.302 | INFO | Uploaded 36GiB to GPU in 2.15659s (16.693GiB/s) | 20260505 09:10:14.302 | INFO | Downloaded 34GiB to GPU in 2.06766s (16.4437GiB/s) | 20260505 09:10:19.439 | INFO | Analyzing chunk frames = [2024960,2025472) | 20260505 09:10:19.439 | INFO | MemInfo: { | 20260505 09:10:19.439 | INFO | "gpu" : " 17 GB", | 20260505 09:10:19.439 | INFO | "huge" : "", | 20260505 09:10:19.439 | INFO | "huge_shared" : " 27 GB", | 20260505 09:10:19.439 | INFO | "malloc" : " 14 GB", | 20260505 09:10:19.439 | INFO | "pinned" : " 29 GB" | 20260505 09:10:19.439 | INFO | } | 20260505 09:10:19.439 | INFO | Perf report for the last 5.1368 seconds: | 20260505 09:10:19.439 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:10:19.439 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:19.439 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:19.439 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:19.439 | INFO | ANALYSIS : 63.3%, 3.0, 98.3ms, 0 | 20260505 09:10:19.439 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:19.439 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:19.439 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:10:19.439 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:19.439 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:19.439 | INFO | Uploaded 36GiB to GPU in 2.15646s (16.694GiB/s) | 20260505 09:10:19.439 | INFO | Downloaded 34GiB to GPU in 2.06974s (16.4272GiB/s) | 20260505 09:10:24.581 | INFO | Analyzing chunk frames = [2025472,2025984) | 20260505 09:10:24.581 | INFO | MemInfo: { | 20260505 09:10:24.581 | INFO | "gpu" : " 17 GB", | 20260505 09:10:24.581 | INFO | "huge" : "", | 20260505 09:10:24.581 | INFO | "huge_shared" : " 27 GB", | 20260505 09:10:24.581 | INFO | "malloc" : " 14 GB", | 20260505 09:10:24.581 | INFO | "pinned" : " 29 GB" | 20260505 09:10:24.581 | INFO | } | 20260505 09:10:24.581 | INFO | Perf report for the last 5.14217 seconds: | 20260505 09:10:24.581 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:10:24.581 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:24.581 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:24.581 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:24.581 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 09:10:24.581 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:24.581 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:24.581 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 09:10:24.581 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:24.581 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:24.581 | INFO | Uploaded 36GiB to GPU in 2.23536s (16.1048GiB/s) | 20260505 09:10:24.581 | INFO | Downloaded 34GiB to GPU in 2.11103s (16.1059GiB/s) | 20260505 09:10:29.717 | INFO | Analyzing chunk frames = [2025984,2026496) | 20260505 09:10:29.717 | INFO | MemInfo: { | 20260505 09:10:29.717 | INFO | "gpu" : " 17 GB", | 20260505 09:10:29.717 | INFO | "huge" : "", | 20260505 09:10:29.717 | INFO | "huge_shared" : " 27 GB", | 20260505 09:10:29.717 | INFO | "malloc" : " 14 GB", | 20260505 09:10:29.717 | INFO | "pinned" : " 29 GB" | 20260505 09:10:29.717 | INFO | } | 20260505 09:10:29.717 | INFO | Perf report for the last 5.13627 seconds: | 20260505 09:10:29.717 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:10:29.717 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:29.717 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:29.717 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:29.718 | INFO | ANALYSIS : 64.1%, 3.0, 100.0ms, 0 | 20260505 09:10:29.718 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:29.718 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:29.718 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 09:10:29.718 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:29.718 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:29.718 | INFO | Uploaded 36GiB to GPU in 2.23707s (16.0925GiB/s) | 20260505 09:10:29.718 | INFO | Downloaded 34GiB to GPU in 2.10926s (16.1194GiB/s) | 20260505 09:10:34.855 | INFO | Analyzing chunk frames = [2026496,2027008) | 20260505 09:10:34.855 | INFO | MemInfo: { | 20260505 09:10:34.855 | INFO | "gpu" : " 17 GB", | 20260505 09:10:34.855 | INFO | "huge" : "", | 20260505 09:10:34.855 | INFO | "huge_shared" : " 27 GB", | 20260505 09:10:34.855 | INFO | "malloc" : " 14 GB", | 20260505 09:10:34.855 | INFO | "pinned" : " 29 GB" | 20260505 09:10:34.855 | INFO | } | 20260505 09:10:34.855 | INFO | Perf report for the last 5.1378 seconds: | 20260505 09:10:34.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:10:34.855 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:34.855 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:34.855 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:34.855 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 09:10:34.855 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:34.855 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:34.855 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 09:10:34.855 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:34.855 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:34.856 | INFO | Uploaded 36GiB to GPU in 2.19946s (16.3676GiB/s) | 20260505 09:10:34.856 | INFO | Downloaded 34GiB to GPU in 2.14539s (15.848GiB/s) | 20260505 09:10:39.995 | INFO | Analyzing chunk frames = [2027008,2027520) | 20260505 09:10:39.995 | INFO | MemInfo: { | 20260505 09:10:39.995 | INFO | "gpu" : " 17 GB", | 20260505 09:10:39.995 | INFO | "huge" : "", | 20260505 09:10:39.995 | INFO | "huge_shared" : " 27 GB", | 20260505 09:10:39.995 | INFO | "malloc" : " 14 GB", | 20260505 09:10:39.995 | INFO | "pinned" : " 29 GB" | 20260505 09:10:39.995 | INFO | } | 20260505 09:10:39.995 | INFO | Perf report for the last 5.14014 seconds: | 20260505 09:10:39.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:10:39.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:39.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:39.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:39.995 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 09:10:39.995 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:39.995 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:39.995 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:10:39.996 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:39.996 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:39.996 | INFO | Uploaded 36GiB to GPU in 2.20695s (16.3121GiB/s) | 20260505 09:10:39.996 | INFO | Downloaded 34GiB to GPU in 2.04355s (16.6377GiB/s) | 20260505 09:10:40.143 | INFO | Starting a round of Baz IO | 20260505 09:10:43.400 | INFO | Populating RTMetrics report | 20260505 09:10:43.400 | INFO | RTMetrics report populated | 20260505 09:10:43.400 | INFO | Opening tmp RTMetrics json file | 20260505 09:10:43.431 | INFO | Serializing RTMetrics to json | 20260505 09:10:43.451 | INFO | Serialization done, swapping tmp file now... | 20260505 09:10:43.453 | INFO | RTMetrics done | 20260505 09:10:43.549 | INFO | Finished a round of Baz IO: Wrote 2.443 GiB in 3.405 seconds | 20260505 09:10:43.549 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 26.46% / 0.02% / 73.52% | 20260505 09:10:43.549 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.13955% / 1.98682e-05% / 0.000103315% | 20260505 09:10:43.549 | INFO | Baz IO overall: | 20260505 09:10:43.549 | INFO | BytesWritten : 1164388641784 | 20260505 09:10:43.549 | INFO | NumEvents : 553965367865 | 20260505 09:10:45.135 | INFO | Analyzing chunk frames = [2027520,2028032) | 20260505 09:10:45.135 | INFO | MemInfo: { | 20260505 09:10:45.135 | INFO | "gpu" : " 17 GB", | 20260505 09:10:45.135 | INFO | "huge" : "", | 20260505 09:10:45.135 | INFO | "huge_shared" : " 27 GB", | 20260505 09:10:45.135 | INFO | "malloc" : " 14 GB", | 20260505 09:10:45.135 | INFO | "pinned" : " 29 GB" | 20260505 09:10:45.135 | INFO | } | 20260505 09:10:45.135 | INFO | Perf report for the last 5.13946 seconds: | 20260505 09:10:45.135 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:10:45.135 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:45.135 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:45.135 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:45.135 | INFO | ANALYSIS : 65.1%, 2.9, 99.1ms, 0 | 20260505 09:10:45.135 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:45.135 | INFO | RT_METRICS : 46.8%, 1.0, 24.6ms, 0 | 20260505 09:10:45.135 | INFO | PRE_HQ : 63.3%, 1.8, 60.7ms, 0 | 20260505 09:10:45.135 | INFO | BAZWRITER : 65.6%, 4.2, 142.5ms, 0 | 20260505 09:10:45.135 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 09:10:45.135 | INFO | Uploaded 36GiB to GPU in 2.32177s (15.5054GiB/s) | 20260505 09:10:45.135 | INFO | Downloaded 38GiB to GPU in 2.40544s (15.7975GiB/s) | 20260505 09:10:50.274 | INFO | Analyzing chunk frames = [2028032,2028544) | 20260505 09:10:50.274 | INFO | MemInfo: { | 20260505 09:10:50.274 | INFO | "gpu" : " 17 GB", | 20260505 09:10:50.274 | INFO | "huge" : "", | 20260505 09:10:50.274 | INFO | "huge_shared" : " 27 GB", | 20260505 09:10:50.274 | INFO | "malloc" : " 14 GB", | 20260505 09:10:50.274 | INFO | "pinned" : " 29 GB" | 20260505 09:10:50.274 | INFO | } | 20260505 09:10:50.274 | INFO | Perf report for the last 5.13963 seconds: | 20260505 09:10:50.274 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:10:50.274 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:50.274 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:50.274 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:50.274 | INFO | ANALYSIS : 64.3%, 3.0, 100.3ms, 0 | 20260505 09:10:50.274 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:50.274 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:50.274 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:10:50.275 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:50.275 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:50.275 | INFO | Uploaded 36GiB to GPU in 2.20008s (16.3631GiB/s) | 20260505 09:10:50.275 | INFO | Downloaded 34GiB to GPU in 2.09286s (16.2457GiB/s) | 20260505 09:10:55.419 | INFO | Analyzing chunk frames = [2028544,2029056) | 20260505 09:10:55.419 | INFO | MemInfo: { | 20260505 09:10:55.419 | INFO | "gpu" : " 17 GB", | 20260505 09:10:55.419 | INFO | "huge" : "", | 20260505 09:10:55.419 | INFO | "huge_shared" : " 27 GB", | 20260505 09:10:55.419 | INFO | "malloc" : " 14 GB", | 20260505 09:10:55.419 | INFO | "pinned" : " 29 GB" | 20260505 09:10:55.419 | INFO | } | 20260505 09:10:55.419 | INFO | Perf report for the last 5.14451 seconds: | 20260505 09:10:55.419 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:10:55.419 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:55.419 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:55.419 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:55.419 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 09:10:55.419 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:55.419 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:55.419 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 09:10:55.419 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:55.419 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:10:55.419 | INFO | Uploaded 36GiB to GPU in 2.16736s (16.6101GiB/s) | 20260505 09:10:55.419 | INFO | Downloaded 34GiB to GPU in 2.06974s (16.4272GiB/s) | 20260505 09:11:00.553 | INFO | Analyzing chunk frames = [2029056,2029568) | 20260505 09:11:00.553 | INFO | MemInfo: { | 20260505 09:11:00.553 | INFO | "gpu" : " 17 GB", | 20260505 09:11:00.553 | INFO | "huge" : "", | 20260505 09:11:00.553 | INFO | "huge_shared" : " 27 GB", | 20260505 09:11:00.553 | INFO | "malloc" : " 14 GB", | 20260505 09:11:00.553 | INFO | "pinned" : " 29 GB" | 20260505 09:11:00.553 | INFO | } | 20260505 09:11:00.553 | INFO | Perf report for the last 5.13395 seconds: | 20260505 09:11:00.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:11:00.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:00.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:00.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:00.553 | INFO | ANALYSIS : 63.4%, 3.0, 98.4ms, 0 | 20260505 09:11:00.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:00.553 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:00.553 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:11:00.553 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:00.553 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:00.553 | INFO | Uploaded 36GiB to GPU in 2.17143s (16.5789GiB/s) | 20260505 09:11:00.553 | INFO | Downloaded 34GiB to GPU in 2.07223s (16.4075GiB/s) | 20260505 09:11:05.692 | INFO | Analyzing chunk frames = [2029568,2030080) | 20260505 09:11:05.692 | INFO | MemInfo: { | 20260505 09:11:05.692 | INFO | "gpu" : " 17 GB", | 20260505 09:11:05.692 | INFO | "huge" : "", | 20260505 09:11:05.692 | INFO | "huge_shared" : " 27 GB", | 20260505 09:11:05.692 | INFO | "malloc" : " 14 GB", | 20260505 09:11:05.692 | INFO | "pinned" : " 29 GB" | 20260505 09:11:05.692 | INFO | } | 20260505 09:11:05.692 | INFO | Perf report for the last 5.13896 seconds: | 20260505 09:11:05.692 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:11:05.692 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:05.692 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:05.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:05.692 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 09:11:05.692 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:05.692 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:05.692 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 09:11:05.692 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:05.692 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:05.692 | INFO | Uploaded 36GiB to GPU in 2.23343s (16.1187GiB/s) | 20260505 09:11:05.692 | INFO | Downloaded 34GiB to GPU in 2.10917s (16.1201GiB/s) | 20260505 09:11:10.832 | INFO | Analyzing chunk frames = [2030080,2030592) | 20260505 09:11:10.832 | INFO | MemInfo: { | 20260505 09:11:10.832 | INFO | "gpu" : " 17 GB", | 20260505 09:11:10.832 | INFO | "huge" : "", | 20260505 09:11:10.832 | INFO | "huge_shared" : " 27 GB", | 20260505 09:11:10.832 | INFO | "malloc" : " 14 GB", | 20260505 09:11:10.832 | INFO | "pinned" : " 29 GB" | 20260505 09:11:10.832 | INFO | } | 20260505 09:11:10.832 | INFO | Perf report for the last 5.13991 seconds: | 20260505 09:11:10.832 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:11:10.832 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:10.832 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:10.832 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:10.832 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 09:11:10.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:10.832 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:10.832 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 09:11:10.832 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:10.832 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:10.832 | INFO | Uploaded 36GiB to GPU in 2.17215s (16.5734GiB/s) | 20260505 09:11:10.832 | INFO | Downloaded 34GiB to GPU in 2.0612s (16.4953GiB/s) | 20260505 09:11:15.970 | INFO | Analyzing chunk frames = [2030592,2031104) | 20260505 09:11:15.970 | INFO | MemInfo: { | 20260505 09:11:15.970 | INFO | "gpu" : " 17 GB", | 20260505 09:11:15.970 | INFO | "huge" : "", | 20260505 09:11:15.970 | INFO | "huge_shared" : " 27 GB", | 20260505 09:11:15.970 | INFO | "malloc" : " 14 GB", | 20260505 09:11:15.970 | INFO | "pinned" : " 29 GB" | 20260505 09:11:15.970 | INFO | } | 20260505 09:11:15.970 | INFO | Perf report for the last 5.13841 seconds: | 20260505 09:11:15.970 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:11:15.970 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:15.970 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:15.970 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:15.970 | INFO | ANALYSIS : 64.0%, 3.0, 99.8ms, 0 | 20260505 09:11:15.970 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:15.970 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:15.970 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 09:11:15.971 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:15.971 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:15.971 | INFO | Uploaded 36GiB to GPU in 2.20411s (16.3331GiB/s) | 20260505 09:11:15.971 | INFO | Downloaded 34GiB to GPU in 2.14123s (15.8788GiB/s) | 20260505 09:11:21.110 | INFO | Analyzing chunk frames = [2031104,2031616) | 20260505 09:11:21.110 | INFO | MemInfo: { | 20260505 09:11:21.110 | INFO | "gpu" : " 17 GB", | 20260505 09:11:21.110 | INFO | "huge" : "", | 20260505 09:11:21.110 | INFO | "huge_shared" : " 27 GB", | 20260505 09:11:21.110 | INFO | "malloc" : " 14 GB", | 20260505 09:11:21.110 | INFO | "pinned" : " 29 GB" | 20260505 09:11:21.110 | INFO | } | 20260505 09:11:21.110 | INFO | Perf report for the last 5.14021 seconds: | 20260505 09:11:21.110 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:11:21.110 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:21.110 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:21.110 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:21.110 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 09:11:21.110 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:21.110 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:21.110 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 09:11:21.110 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:21.110 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:21.111 | INFO | Uploaded 36GiB to GPU in 2.17365s (16.562GiB/s) | 20260505 09:11:21.111 | INFO | Downloaded 34GiB to GPU in 2.03271s (16.7265GiB/s) | 20260505 09:11:21.111 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 09:11:21.111 | INFO | Total mem usage: 15.3 GiB | 20260505 09:11:21.111 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 09:11:21.111 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -5.71%) | 20260505 09:11:21.111 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 09:11:21.111 | INFO | High Water Breakdown: | 20260505 09:11:21.111 | INFO | 61% : 8.85 GiB ( 94.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 09:11:21.111 | INFO | 38.8% : 5.63 GiB ( 84.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 09:11:21.111 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 09:11:21.111 | INFO | ------------------------------------------ | 20260505 09:11:21.111 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:11:21.111 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 09:11:21.111 | INFO | Total mem usage: 26.5 GiB | 20260505 09:11:21.111 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 09:11:21.111 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 09:11:21.111 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 09:11:21.111 | INFO | High Water Breakdown: | 20260505 09:11:21.111 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:11:21.111 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:11:21.111 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:11:21.111 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:11:21.111 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:11:21.111 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:11:21.111 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 09:11:21.111 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 09:11:21.111 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:11:21.111 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 09:11:21.111 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 09:11:21.111 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 09:11:21.111 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 09:11:21.111 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 09:11:21.111 | INFO | ------------------------------------------ | 20260505 09:11:21.111 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:11:21.111 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 09:11:21.111 | INFO | Total mem usage: 25.6 GiB | 20260505 09:11:21.111 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 09:11:21.111 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 09:11:21.111 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 09:11:21.111 | INFO | High Water Breakdown: | 20260505 09:11:21.111 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 09:11:21.111 | INFO | ------------------------------------------ | 20260505 09:11:21.111 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:11:21.111 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 09:11:21.111 | INFO | Total mem usage: 15.7 GiB | 20260505 09:11:21.111 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 09:11:21.111 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 09:11:21.111 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 09:11:21.111 | INFO | High Water Breakdown: | 20260505 09:11:21.111 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 09:11:21.111 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 09:11:21.111 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 09:11:21.111 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 09:11:21.111 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 09:11:21.111 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 09:11:21.111 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 09:11:21.111 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 09:11:21.111 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 09:11:21.111 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:11:21.111 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 09:11:21.111 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:11:21.111 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 09:11:21.111 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 09:11:21.111 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 09:11:21.111 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 09:11:21.112 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:11:21.112 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 09:11:21.112 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:11:21.112 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:11:21.112 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:11:21.112 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 09:11:21.112 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 09:11:21.112 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:11:21.112 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 09:11:21.112 | INFO | ------------------------------------------ | 20260505 09:11:21.112 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:11:21.112 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 09:11:21.112 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:11:21.112 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:11:21.112 | INFO | PerfReport: Full Execution -- 3.25e+01 / 7.58e+01, 1.38e+01 / 1.95e+00 | 20260505 09:11:21.112 | INFO | PerfReport: 0.29%, AggregateStats -- 9.32e-02 / 1.03e-01, 1.07e-02 / 4.42e-03 | 20260505 09:11:21.112 | INFO | PerfReport: 11.64%, Baseline -- 3.78e+00 / 4.38e+00, 6.18e-01 / 3.58e-01 | 20260505 09:11:21.112 | INFO | PerfReport: 3.31%, Histogram -- 1.07e+00 / 1.45e+00, 2.08e-01 / 1.22e-01 | 20260505 09:11:21.112 | INFO | PerfReport: 15.46%, DME -- 4.02e+01 / 4.38e+01, 4.11e+00 / 1.43e+00 | 20260505 09:11:21.112 | INFO | PerfReport: 56.61%, FrameLabeling -- 1.84e+01 / 2.10e+01, 1.91e+00 / 1.92e+00 | 20260505 09:11:21.112 | INFO | PerfReport: 7.46%, PulseAccumulating -- 2.42e+00 / 2.98e+00, 4.16e-01 / 2.65e-01 | 20260505 09:11:21.112 | INFO | PerfReport: 3.86%, Metrics -- 1.26e+00 / 1.59e+00, 3.65e-01 / 1.57e-01 | 20260505 09:11:21.112 | INFO | PerfReport: 1.37%, PulseSerializing -- 4.45e-01 / 4.96e-01, 8.38e-02 / 2.29e-02 | 20260505 09:11:21.112 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 09:11:21.112 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:11:21.112 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:11:21.112 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.34e+01 / 0.00e+00 | 20260505 09:11:21.112 | INFO | PerfReport: 22.65%, Upload -- 2.30e+01 / 0.00e+00, 8.31e+00 / 0.00e+00 | 20260505 09:11:21.112 | INFO | PerfReport: 21.80%, Download -- 2.21e+01 / 0.00e+00, 8.30e+00 / 0.00e+00 | 20260505 09:11:21.112 | INFO | PerfReport: 32.02%, Compute -- 3.25e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 09:11:21.112 | INFO | PerfReport: 1.75%, LockUpload -- 1.78e+00 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260505 09:11:21.112 | INFO | PerfReport: 1.08%, LockDownload -- 1.10e+00 / 0.00e+00, 2.59e+00 / 0.00e+00 | 20260505 09:11:21.112 | INFO | PerfReport: 20.68%, LockCompute -- 2.10e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260505 09:11:21.112 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.75e-02 / 0.00e+00 | 20260505 09:11:21.248 | INFO | Starting a round of Baz IO | 20260505 09:11:24.516 | INFO | Populating RTMetrics report | 20260505 09:11:24.516 | INFO | RTMetrics report populated | 20260505 09:11:24.516 | INFO | Opening tmp RTMetrics json file | 20260505 09:11:24.523 | INFO | Serializing RTMetrics to json | 20260505 09:11:24.524 | INFO | Serialization done, swapping tmp file now... | 20260505 09:11:24.525 | INFO | RTMetrics done | 20260505 09:11:24.663 | INFO | Finished a round of Baz IO: Wrote 2.443 GiB in 3.415 seconds | 20260505 09:11:24.663 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 26.45% / 0.02% / 73.54% | 20260505 09:11:24.663 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.13391% / 1.19209e-05% / 4.37101e-05% | 20260505 09:11:24.664 | INFO | Baz IO overall: | 20260505 09:11:24.664 | INFO | BytesWritten : 1167011808440 | 20260505 09:11:24.664 | INFO | NumEvents : 555219002225 | 20260505 09:11:26.250 | INFO | Analyzing chunk frames = [2031616,2032128) | 20260505 09:11:26.250 | INFO | MemInfo: { | 20260505 09:11:26.250 | INFO | "gpu" : " 17 GB", | 20260505 09:11:26.250 | INFO | "huge" : "", | 20260505 09:11:26.250 | INFO | "huge_shared" : " 27 GB", | 20260505 09:11:26.250 | INFO | "malloc" : " 14 GB", | 20260505 09:11:26.250 | INFO | "pinned" : " 29 GB" | 20260505 09:11:26.250 | INFO | } | 20260505 09:11:26.250 | INFO | Perf report for the last 5.13937 seconds: | 20260505 09:11:26.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:11:26.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:26.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:26.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:26.250 | INFO | ANALYSIS : 65.1%, 2.9, 99.5ms, 0 | 20260505 09:11:26.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:26.250 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260505 09:11:26.250 | INFO | PRE_HQ : 63.3%, 1.8, 60.8ms, 0 | 20260505 09:11:26.250 | INFO | BAZWRITER : 65.7%, 4.2, 141.2ms, 0 | 20260505 09:11:26.250 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 09:11:26.250 | INFO | Uploaded 36GiB to GPU in 2.26472s (15.896GiB/s) | 20260505 09:11:26.250 | INFO | Downloaded 38GiB to GPU in 2.38625s (15.9246GiB/s) | 20260505 09:11:31.389 | INFO | Analyzing chunk frames = [2032128,2032640) | 20260505 09:11:31.389 | INFO | MemInfo: { | 20260505 09:11:31.389 | INFO | "gpu" : " 17 GB", | 20260505 09:11:31.389 | INFO | "huge" : "", | 20260505 09:11:31.389 | INFO | "huge_shared" : " 27 GB", | 20260505 09:11:31.389 | INFO | "malloc" : " 14 GB", | 20260505 09:11:31.389 | INFO | "pinned" : " 29 GB" | 20260505 09:11:31.389 | INFO | } | 20260505 09:11:31.389 | INFO | Perf report for the last 5.1391 seconds: | 20260505 09:11:31.389 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:11:31.389 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:31.389 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:31.389 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:31.389 | INFO | ANALYSIS : 64.5%, 3.0, 100.7ms, 0 | 20260505 09:11:31.389 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:31.389 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:31.389 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 09:11:31.389 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:31.389 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:31.389 | INFO | Uploaded 36GiB to GPU in 2.21264s (16.2701GiB/s) | 20260505 09:11:31.389 | INFO | Downloaded 34GiB to GPU in 2.09347s (16.241GiB/s) | 20260505 09:11:36.529 | INFO | Analyzing chunk frames = [2032640,2033152) | 20260505 09:11:36.529 | INFO | MemInfo: { | 20260505 09:11:36.529 | INFO | "gpu" : " 17 GB", | 20260505 09:11:36.529 | INFO | "huge" : "", | 20260505 09:11:36.529 | INFO | "huge_shared" : " 27 GB", | 20260505 09:11:36.529 | INFO | "malloc" : " 14 GB", | 20260505 09:11:36.529 | INFO | "pinned" : " 29 GB" | 20260505 09:11:36.529 | INFO | } | 20260505 09:11:36.529 | INFO | Perf report for the last 5.14048 seconds: | 20260505 09:11:36.529 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:11:36.529 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:36.529 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:36.529 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:36.529 | INFO | ANALYSIS : 63.4%, 3.0, 98.6ms, 0 | 20260505 09:11:36.529 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:36.529 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:36.529 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:11:36.529 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:36.530 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:36.530 | INFO | Uploaded 36GiB to GPU in 2.16295s (16.6439GiB/s) | 20260505 09:11:36.530 | INFO | Downloaded 34GiB to GPU in 2.06482s (16.4663GiB/s) | 20260505 09:11:41.667 | INFO | Analyzing chunk frames = [2033152,2033664) | 20260505 09:11:41.667 | INFO | MemInfo: { | 20260505 09:11:41.667 | INFO | "gpu" : " 17 GB", | 20260505 09:11:41.667 | INFO | "huge" : "", | 20260505 09:11:41.667 | INFO | "huge_shared" : " 27 GB", | 20260505 09:11:41.667 | INFO | "malloc" : " 14 GB", | 20260505 09:11:41.667 | INFO | "pinned" : " 29 GB" | 20260505 09:11:41.667 | INFO | } | 20260505 09:11:41.667 | INFO | Perf report for the last 5.13802 seconds: | 20260505 09:11:41.667 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:11:41.667 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:41.667 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:41.667 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:41.667 | INFO | ANALYSIS : 63.4%, 3.0, 98.4ms, 0 | 20260505 09:11:41.667 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:41.667 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:41.667 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 09:11:41.668 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:41.668 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:41.668 | INFO | Uploaded 36GiB to GPU in 2.16026s (16.6647GiB/s) | 20260505 09:11:41.668 | INFO | Downloaded 34GiB to GPU in 2.06851s (16.4369GiB/s) | 20260505 09:11:46.807 | INFO | Analyzing chunk frames = [2033664,2034176) | 20260505 09:11:46.807 | INFO | MemInfo: { | 20260505 09:11:46.807 | INFO | "gpu" : " 17 GB", | 20260505 09:11:46.807 | INFO | "huge" : "", | 20260505 09:11:46.807 | INFO | "huge_shared" : " 27 GB", | 20260505 09:11:46.807 | INFO | "malloc" : " 14 GB", | 20260505 09:11:46.807 | INFO | "pinned" : " 29 GB" | 20260505 09:11:46.807 | INFO | } | 20260505 09:11:46.807 | INFO | Perf report for the last 5.13928 seconds: | 20260505 09:11:46.807 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:11:46.807 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:46.807 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:46.807 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:46.807 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 09:11:46.807 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:46.807 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:46.807 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 09:11:46.807 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:46.807 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:46.807 | INFO | Uploaded 36GiB to GPU in 2.23827s (16.0839GiB/s) | 20260505 09:11:46.807 | INFO | Downloaded 34GiB to GPU in 2.09472s (16.2313GiB/s) | 20260505 09:11:51.947 | INFO | Analyzing chunk frames = [2034176,2034688) | 20260505 09:11:51.947 | INFO | MemInfo: { | 20260505 09:11:51.947 | INFO | "gpu" : " 17 GB", | 20260505 09:11:51.947 | INFO | "huge" : "", | 20260505 09:11:51.947 | INFO | "huge_shared" : " 27 GB", | 20260505 09:11:51.947 | INFO | "malloc" : " 14 GB", | 20260505 09:11:51.947 | INFO | "pinned" : " 29 GB" | 20260505 09:11:51.947 | INFO | } | 20260505 09:11:51.947 | INFO | Perf report for the last 5.14029 seconds: | 20260505 09:11:51.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:11:51.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:51.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:51.947 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:51.947 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 09:11:51.947 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:51.947 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:51.947 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 09:11:51.947 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:51.947 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:51.947 | INFO | Uploaded 36GiB to GPU in 2.1621s (16.6504GiB/s) | 20260505 09:11:51.947 | INFO | Downloaded 34GiB to GPU in 2.05833s (16.5182GiB/s) | 20260505 09:11:57.087 | INFO | Analyzing chunk frames = [2034688,2035200) | 20260505 09:11:57.087 | INFO | MemInfo: { | 20260505 09:11:57.087 | INFO | "gpu" : " 17 GB", | 20260505 09:11:57.087 | INFO | "huge" : "", | 20260505 09:11:57.087 | INFO | "huge_shared" : " 27 GB", | 20260505 09:11:57.087 | INFO | "malloc" : " 14 GB", | 20260505 09:11:57.087 | INFO | "pinned" : " 29 GB" | 20260505 09:11:57.087 | INFO | } | 20260505 09:11:57.087 | INFO | Perf report for the last 5.13981 seconds: | 20260505 09:11:57.087 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:11:57.087 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:57.087 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:57.087 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:57.087 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 09:11:57.087 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:57.087 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:57.087 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 09:11:57.087 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:57.087 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:11:57.087 | INFO | Uploaded 36GiB to GPU in 2.19084s (16.432GiB/s) | 20260505 09:11:57.087 | INFO | Downloaded 34GiB to GPU in 2.16443s (15.7086GiB/s) | 20260505 09:12:02.224 | INFO | Analyzing chunk frames = [2035200,2035712) | 20260505 09:12:02.224 | INFO | MemInfo: { | 20260505 09:12:02.224 | INFO | "gpu" : " 17 GB", | 20260505 09:12:02.224 | INFO | "huge" : "", | 20260505 09:12:02.224 | INFO | "huge_shared" : " 27 GB", | 20260505 09:12:02.224 | INFO | "malloc" : " 14 GB", | 20260505 09:12:02.224 | INFO | "pinned" : " 29 GB" | 20260505 09:12:02.225 | INFO | } | 20260505 09:12:02.225 | INFO | Perf report for the last 5.13773 seconds: | 20260505 09:12:02.225 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:12:02.225 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:02.225 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:02.225 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:02.225 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 09:12:02.225 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:02.225 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:02.225 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 09:12:02.225 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:02.225 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:02.225 | INFO | Uploaded 36GiB to GPU in 2.17434s (16.5567GiB/s) | 20260505 09:12:02.225 | INFO | Downloaded 34GiB to GPU in 2.02982s (16.7502GiB/s) | 20260505 09:12:02.360 | INFO | Starting a round of Baz IO | 20260505 09:12:05.622 | INFO | Populating RTMetrics report | 20260505 09:12:05.622 | INFO | RTMetrics report populated | 20260505 09:12:05.622 | INFO | Opening tmp RTMetrics json file | 20260505 09:12:05.625 | INFO | Serializing RTMetrics to json | 20260505 09:12:05.628 | INFO | Serialization done, swapping tmp file now... | 20260505 09:12:05.630 | INFO | RTMetrics done | 20260505 09:12:05.785 | INFO | Finished a round of Baz IO: Wrote 2.446 GiB in 3.425 seconds | 20260505 09:12:05.785 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 26.43% / 0.02% / 73.55% | 20260505 09:12:05.785 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.13578% / 2.38419e-05% / 6.75519e-05% | 20260505 09:12:05.785 | INFO | Baz IO overall: | 20260505 09:12:05.785 | INFO | BytesWritten : 1169638403388 | 20260505 09:12:05.785 | INFO | NumEvents : 556475255486 | 20260505 09:12:07.365 | INFO | Analyzing chunk frames = [2035712,2036224) | 20260505 09:12:07.365 | INFO | MemInfo: { | 20260505 09:12:07.365 | INFO | "gpu" : " 17 GB", | 20260505 09:12:07.365 | INFO | "huge" : "", | 20260505 09:12:07.365 | INFO | "huge_shared" : " 27 GB", | 20260505 09:12:07.365 | INFO | "malloc" : " 14 GB", | 20260505 09:12:07.365 | INFO | "pinned" : " 29 GB" | 20260505 09:12:07.365 | INFO | } | 20260505 09:12:07.365 | INFO | Perf report for the last 5.14017 seconds: | 20260505 09:12:07.365 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:12:07.365 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:07.365 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:07.365 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:07.365 | INFO | ANALYSIS : 64.9%, 2.9, 98.7ms, 0 | 20260505 09:12:07.365 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:07.365 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260505 09:12:07.365 | INFO | PRE_HQ : 63.2%, 1.8, 60.5ms, 0 | 20260505 09:12:07.365 | INFO | BAZWRITER : 65.9%, 4.2, 143.7ms, 0 | 20260505 09:12:07.365 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 09:12:07.365 | INFO | Uploaded 36GiB to GPU in 2.26141s (15.9193GiB/s) | 20260505 09:12:07.365 | INFO | Downloaded 38GiB to GPU in 2.38698s (15.9197GiB/s) | 20260505 09:12:12.504 | INFO | Analyzing chunk frames = [2036224,2036736) | 20260505 09:12:12.504 | INFO | MemInfo: { | 20260505 09:12:12.504 | INFO | "gpu" : " 17 GB", | 20260505 09:12:12.504 | INFO | "huge" : "", | 20260505 09:12:12.504 | INFO | "huge_shared" : " 27 GB", | 20260505 09:12:12.504 | INFO | "malloc" : " 14 GB", | 20260505 09:12:12.504 | INFO | "pinned" : " 29 GB" | 20260505 09:12:12.504 | INFO | } | 20260505 09:12:12.505 | INFO | Perf report for the last 5.13974 seconds: | 20260505 09:12:12.505 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:12:12.505 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:12.505 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:12.505 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:12.505 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 09:12:12.505 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:12.505 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:12.505 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 09:12:12.505 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:12.505 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:12.505 | INFO | Uploaded 36GiB to GPU in 2.19965s (16.3662GiB/s) | 20260505 09:12:12.505 | INFO | Downloaded 34GiB to GPU in 2.12141s (16.0271GiB/s) | 20260505 09:12:17.643 | INFO | Analyzing chunk frames = [2036736,2037248) | 20260505 09:12:17.643 | INFO | MemInfo: { | 20260505 09:12:17.643 | INFO | "gpu" : " 17 GB", | 20260505 09:12:17.643 | INFO | "huge" : "", | 20260505 09:12:17.643 | INFO | "huge_shared" : " 27 GB", | 20260505 09:12:17.643 | INFO | "malloc" : " 14 GB", | 20260505 09:12:17.643 | INFO | "pinned" : " 29 GB" | 20260505 09:12:17.643 | INFO | } | 20260505 09:12:17.643 | INFO | Perf report for the last 5.13886 seconds: | 20260505 09:12:17.643 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:12:17.643 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:17.643 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:17.643 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:17.643 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 09:12:17.643 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:17.644 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:17.644 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:12:17.644 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:17.644 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:17.644 | INFO | Uploaded 36GiB to GPU in 2.15321s (16.7192GiB/s) | 20260505 09:12:17.644 | INFO | Downloaded 34GiB to GPU in 2.06985s (16.4263GiB/s) | 20260505 09:12:22.783 | INFO | Analyzing chunk frames = [2037248,2037760) | 20260505 09:12:22.783 | INFO | MemInfo: { | 20260505 09:12:22.783 | INFO | "gpu" : " 17 GB", | 20260505 09:12:22.783 | INFO | "huge" : "", | 20260505 09:12:22.783 | INFO | "huge_shared" : " 27 GB", | 20260505 09:12:22.783 | INFO | "malloc" : " 14 GB", | 20260505 09:12:22.783 | INFO | "pinned" : " 29 GB" | 20260505 09:12:22.783 | INFO | } | 20260505 09:12:22.783 | INFO | Perf report for the last 5.13962 seconds: | 20260505 09:12:22.783 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:12:22.783 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:22.783 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:22.783 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:22.783 | INFO | ANALYSIS : 63.4%, 3.0, 98.4ms, 0 | 20260505 09:12:22.783 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:22.783 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:22.783 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:12:22.783 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:22.783 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:22.783 | INFO | Uploaded 36GiB to GPU in 2.15864s (16.6772GiB/s) | 20260505 09:12:22.783 | INFO | Downloaded 34GiB to GPU in 2.07142s (16.4139GiB/s) | 20260505 09:12:27.921 | INFO | Analyzing chunk frames = [2037760,2038272) | 20260505 09:12:27.921 | INFO | MemInfo: { | 20260505 09:12:27.921 | INFO | "gpu" : " 17 GB", | 20260505 09:12:27.921 | INFO | "huge" : "", | 20260505 09:12:27.921 | INFO | "huge_shared" : " 27 GB", | 20260505 09:12:27.921 | INFO | "malloc" : " 14 GB", | 20260505 09:12:27.921 | INFO | "pinned" : " 29 GB" | 20260505 09:12:27.921 | INFO | } | 20260505 09:12:27.921 | INFO | Perf report for the last 5.13818 seconds: | 20260505 09:12:27.921 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:12:27.921 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:27.921 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:27.921 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:27.921 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 09:12:27.921 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:27.921 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:27.921 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 09:12:27.922 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:27.922 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:27.922 | INFO | Uploaded 36GiB to GPU in 2.23622s (16.0986GiB/s) | 20260505 09:12:27.922 | INFO | Downloaded 34GiB to GPU in 2.10413s (16.1587GiB/s) | 20260505 09:12:33.062 | INFO | Analyzing chunk frames = [2038272,2038784) | 20260505 09:12:33.062 | INFO | MemInfo: { | 20260505 09:12:33.062 | INFO | "gpu" : " 17 GB", | 20260505 09:12:33.062 | INFO | "huge" : "", | 20260505 09:12:33.062 | INFO | "huge_shared" : " 27 GB", | 20260505 09:12:33.062 | INFO | "malloc" : " 14 GB", | 20260505 09:12:33.062 | INFO | "pinned" : " 29 GB" | 20260505 09:12:33.062 | INFO | } | 20260505 09:12:33.062 | INFO | Perf report for the last 5.14095 seconds: | 20260505 09:12:33.062 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:12:33.062 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:33.062 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:33.062 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:33.062 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 09:12:33.062 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:33.062 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:33.062 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 09:12:33.062 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:33.062 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:33.062 | INFO | Uploaded 36GiB to GPU in 2.24521s (16.0342GiB/s) | 20260505 09:12:33.063 | INFO | Downloaded 34GiB to GPU in 2.10783s (16.1303GiB/s) | 20260505 09:12:38.202 | INFO | Analyzing chunk frames = [2038784,2039296) | 20260505 09:12:38.202 | INFO | MemInfo: { | 20260505 09:12:38.202 | INFO | "gpu" : " 17 GB", | 20260505 09:12:38.202 | INFO | "huge" : "", | 20260505 09:12:38.202 | INFO | "huge_shared" : " 27 GB", | 20260505 09:12:38.202 | INFO | "malloc" : " 14 GB", | 20260505 09:12:38.202 | INFO | "pinned" : " 29 GB" | 20260505 09:12:38.202 | INFO | } | 20260505 09:12:38.202 | INFO | Perf report for the last 5.14005 seconds: | 20260505 09:12:38.202 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:12:38.202 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:38.202 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:38.202 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:38.202 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 09:12:38.202 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:38.202 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:38.202 | INFO | PRE_HQ : 30.5%, 1.0, 15.9ms, 0 | 20260505 09:12:38.202 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:38.202 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:38.203 | INFO | Uploaded 36GiB to GPU in 2.19909s (16.3704GiB/s) | 20260505 09:12:38.203 | INFO | Downloaded 34GiB to GPU in 2.13023s (15.9607GiB/s) | 20260505 09:12:43.342 | INFO | Analyzing chunk frames = [2039296,2039808) | 20260505 09:12:43.342 | INFO | MemInfo: { | 20260505 09:12:43.342 | INFO | "gpu" : " 17 GB", | 20260505 09:12:43.342 | INFO | "huge" : "", | 20260505 09:12:43.343 | INFO | "huge_shared" : " 27 GB", | 20260505 09:12:43.343 | INFO | "malloc" : " 14 GB", | 20260505 09:12:43.343 | INFO | "pinned" : " 29 GB" | 20260505 09:12:43.343 | INFO | } | 20260505 09:12:43.343 | INFO | Perf report for the last 5.14049 seconds: | 20260505 09:12:43.343 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:12:43.343 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:43.343 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:43.343 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:43.343 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 09:12:43.343 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:43.343 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:43.343 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 09:12:43.343 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:43.343 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:43.343 | INFO | Uploaded 36GiB to GPU in 2.18191s (16.4993GiB/s) | 20260505 09:12:43.343 | INFO | Downloaded 34GiB to GPU in 2.05407s (16.5525GiB/s) | 20260505 09:12:43.343 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 09:12:43.343 | INFO | Total mem usage: 15.3 GiB | 20260505 09:12:43.343 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 09:12:43.343 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -4.92%) | 20260505 09:12:43.343 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 09:12:43.343 | INFO | High Water Breakdown: | 20260505 09:12:43.343 | INFO | 61.1% : 8.93 GiB ( 95.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 09:12:43.343 | INFO | 38.7% : 5.66 GiB ( 85.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 09:12:43.343 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 09:12:43.343 | INFO | ------------------------------------------ | 20260505 09:12:43.344 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:12:43.344 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 09:12:43.344 | INFO | Total mem usage: 26.5 GiB | 20260505 09:12:43.344 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 09:12:43.344 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 09:12:43.344 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 09:12:43.344 | INFO | High Water Breakdown: | 20260505 09:12:43.344 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:12:43.344 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:12:43.344 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:12:43.344 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:12:43.344 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:12:43.344 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:12:43.344 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 09:12:43.344 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 09:12:43.344 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:12:43.344 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 09:12:43.344 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 09:12:43.344 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 09:12:43.344 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 09:12:43.344 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 09:12:43.344 | INFO | ------------------------------------------ | 20260505 09:12:43.344 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:12:43.344 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 09:12:43.344 | INFO | Total mem usage: 25.6 GiB | 20260505 09:12:43.344 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 09:12:43.344 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 09:12:43.344 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 09:12:43.344 | INFO | High Water Breakdown: | 20260505 09:12:43.344 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 09:12:43.344 | INFO | ------------------------------------------ | 20260505 09:12:43.344 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:12:43.344 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 09:12:43.344 | INFO | Total mem usage: 15.7 GiB | 20260505 09:12:43.344 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 09:12:43.344 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 09:12:43.344 | INFO | Unused Cache: 880 MiB (Waste: 5.48%) | 20260505 09:12:43.344 | INFO | High Water Breakdown: | 20260505 09:12:43.344 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 09:12:43.344 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 09:12:43.344 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 09:12:43.344 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 09:12:43.344 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 09:12:43.344 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 09:12:43.344 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 09:12:43.344 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 09:12:43.344 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 09:12:43.344 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:12:43.344 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 09:12:43.344 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:12:43.344 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 09:12:43.344 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 09:12:43.344 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 09:12:43.344 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 09:12:43.344 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:12:43.344 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 09:12:43.344 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:12:43.344 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:12:43.344 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:12:43.344 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 09:12:43.344 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 09:12:43.344 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:12:43.344 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 09:12:43.344 | INFO | ------------------------------------------ | 20260505 09:12:43.344 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:12:43.344 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 09:12:43.344 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:12:43.344 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:12:43.344 | INFO | PerfReport: Full Execution -- 3.25e+01 / 7.60e+01, 1.35e+01 / 1.72e+00 | 20260505 09:12:43.344 | INFO | PerfReport: 0.29%, AggregateStats -- 9.33e-02 / 1.05e-01, 1.14e-02 / 4.84e-03 | 20260505 09:12:43.344 | INFO | PerfReport: 11.64%, Baseline -- 3.79e+00 / 4.43e+00, 6.20e-01 / 2.96e-01 | 20260505 09:12:43.344 | INFO | PerfReport: 3.31%, Histogram -- 1.08e+00 / 1.46e+00, 2.08e-01 / 1.18e-01 | 20260505 09:12:43.344 | INFO | PerfReport: 15.50%, DME -- 4.03e+01 / 4.39e+01, 4.03e+00 / 1.32e+00 | 20260505 09:12:43.344 | INFO | PerfReport: 56.59%, FrameLabeling -- 1.84e+01 / 2.10e+01, 1.90e+00 / 1.83e+00 | 20260505 09:12:43.344 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.42e+00 / 2.99e+00, 4.10e-01 / 2.23e-01 | 20260505 09:12:43.344 | INFO | PerfReport: 3.85%, Metrics -- 1.25e+00 / 1.55e+00, 3.59e-01 / 1.79e-01 | 20260505 09:12:43.344 | INFO | PerfReport: 1.37%, PulseSerializing -- 4.45e-01 / 4.90e-01, 8.37e-02 / 2.78e-02 | 20260505 09:12:43.344 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 09:12:43.344 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:12:43.344 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:12:43.344 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.36e+01 / 0.00e+00 | 20260505 09:12:43.344 | INFO | PerfReport: 22.58%, Upload -- 2.29e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260505 09:12:43.344 | INFO | PerfReport: 21.81%, Download -- 2.21e+01 / 0.00e+00, 8.29e+00 / 0.00e+00 | 20260505 09:12:43.344 | INFO | PerfReport: 32.04%, Compute -- 3.25e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 09:12:43.345 | INFO | PerfReport: 1.76%, LockUpload -- 1.79e+00 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260505 09:12:43.345 | INFO | PerfReport: 1.08%, LockDownload -- 1.10e+00 / 0.00e+00, 2.64e+00 / 0.00e+00 | 20260505 09:12:43.345 | INFO | PerfReport: 20.70%, LockCompute -- 2.10e+01 / 0.00e+00, 1.78e+01 / 0.00e+00 | 20260505 09:12:43.345 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.91e-02 / 0.00e+00 | 20260505 09:12:43.477 | INFO | Starting a round of Baz IO | 20260505 09:12:46.753 | INFO | Populating RTMetrics report | 20260505 09:12:46.753 | INFO | RTMetrics report populated | 20260505 09:12:46.753 | INFO | Opening tmp RTMetrics json file | 20260505 09:12:46.767 | INFO | Serializing RTMetrics to json | 20260505 09:12:46.768 | INFO | Serialization done, swapping tmp file now... | 20260505 09:12:46.769 | INFO | RTMetrics done | 20260505 09:12:46.889 | INFO | Finished a round of Baz IO: Wrote 2.444 GiB in 3.412 seconds | 20260505 09:12:46.889 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 26.42% / 0.02% / 73.57% | 20260505 09:12:46.889 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.12837% / 2.78155e-05% / 6.75519e-05% | 20260505 09:12:46.889 | INFO | Baz IO overall: | 20260505 09:12:46.889 | INFO | BytesWritten : 1172263088612 | 20260505 09:12:46.889 | INFO | NumEvents : 557730073056 | 20260505 09:12:48.480 | INFO | Analyzing chunk frames = [2039808,2040320) | 20260505 09:12:48.480 | INFO | MemInfo: { | 20260505 09:12:48.480 | INFO | "gpu" : " 17 GB", | 20260505 09:12:48.480 | INFO | "huge" : "", | 20260505 09:12:48.480 | INFO | "huge_shared" : " 27 GB", | 20260505 09:12:48.480 | INFO | "malloc" : " 14 GB", | 20260505 09:12:48.480 | INFO | "pinned" : " 29 GB" | 20260505 09:12:48.480 | INFO | } | 20260505 09:12:48.480 | INFO | Perf report for the last 5.13747 seconds: | 20260505 09:12:48.480 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:12:48.480 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:48.480 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:48.480 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:48.480 | INFO | ANALYSIS : 65.2%, 2.9, 99.8ms, 0 | 20260505 09:12:48.480 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:48.480 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 09:12:48.480 | INFO | PRE_HQ : 62.9%, 1.8, 60.3ms, 0 | 20260505 09:12:48.480 | INFO | BAZWRITER : 65.7%, 4.2, 140.6ms, 0 | 20260505 09:12:48.480 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260505 09:12:48.481 | INFO | Uploaded 36GiB to GPU in 2.21705s (16.2378GiB/s) | 20260505 09:12:48.481 | INFO | Downloaded 38GiB to GPU in 2.35042s (16.1673GiB/s) | 20260505 09:12:53.618 | INFO | Analyzing chunk frames = [2040320,2040832) | 20260505 09:12:53.618 | INFO | MemInfo: { | 20260505 09:12:53.618 | INFO | "gpu" : " 17 GB", | 20260505 09:12:53.618 | INFO | "huge" : "", | 20260505 09:12:53.618 | INFO | "huge_shared" : " 27 GB", | 20260505 09:12:53.618 | INFO | "malloc" : " 14 GB", | 20260505 09:12:53.619 | INFO | "pinned" : " 29 GB" | 20260505 09:12:53.619 | INFO | } | 20260505 09:12:53.619 | INFO | Perf report for the last 5.13848 seconds: | 20260505 09:12:53.619 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:12:53.619 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:53.619 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:53.619 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:53.619 | INFO | ANALYSIS : 64.3%, 3.0, 100.3ms, 0 | 20260505 09:12:53.619 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:53.619 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:53.619 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:12:53.619 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:53.619 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:53.619 | INFO | Uploaded 36GiB to GPU in 2.19849s (16.3749GiB/s) | 20260505 09:12:53.619 | INFO | Downloaded 34GiB to GPU in 2.10718s (16.1353GiB/s) | 20260505 09:12:58.758 | INFO | Analyzing chunk frames = [2040832,2041344) | 20260505 09:12:58.758 | INFO | MemInfo: { | 20260505 09:12:58.758 | INFO | "gpu" : " 17 GB", | 20260505 09:12:58.758 | INFO | "huge" : "", | 20260505 09:12:58.758 | INFO | "huge_shared" : " 27 GB", | 20260505 09:12:58.758 | INFO | "malloc" : " 14 GB", | 20260505 09:12:58.758 | INFO | "pinned" : " 29 GB" | 20260505 09:12:58.758 | INFO | } | 20260505 09:12:58.758 | INFO | Perf report for the last 5.13904 seconds: | 20260505 09:12:58.758 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:12:58.758 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:58.758 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:58.758 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:58.758 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 09:12:58.758 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:58.758 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:58.758 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 09:12:58.758 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:58.758 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:12:58.758 | INFO | Uploaded 36GiB to GPU in 2.16196s (16.6515GiB/s) | 20260505 09:12:58.758 | INFO | Downloaded 34GiB to GPU in 2.06584s (16.4582GiB/s) | 20260505 09:13:03.897 | INFO | Analyzing chunk frames = [2041344,2041856) | 20260505 09:13:03.897 | INFO | MemInfo: { | 20260505 09:13:03.897 | INFO | "gpu" : " 17 GB", | 20260505 09:13:03.897 | INFO | "huge" : "", | 20260505 09:13:03.897 | INFO | "huge_shared" : " 27 GB", | 20260505 09:13:03.897 | INFO | "malloc" : " 14 GB", | 20260505 09:13:03.897 | INFO | "pinned" : " 29 GB" | 20260505 09:13:03.897 | INFO | } | 20260505 09:13:03.897 | INFO | Perf report for the last 5.13928 seconds: | 20260505 09:13:03.897 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:13:03.897 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:03.897 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:03.897 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:03.897 | INFO | ANALYSIS : 63.2%, 3.0, 98.3ms, 0 | 20260505 09:13:03.897 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:03.897 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:03.897 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 09:13:03.897 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:03.897 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:03.897 | INFO | Uploaded 36GiB to GPU in 2.16223s (16.6495GiB/s) | 20260505 09:13:03.897 | INFO | Downloaded 34GiB to GPU in 2.07272s (16.4036GiB/s) | 20260505 09:13:09.039 | INFO | Analyzing chunk frames = [2041856,2042368) | 20260505 09:13:09.039 | INFO | MemInfo: { | 20260505 09:13:09.039 | INFO | "gpu" : " 17 GB", | 20260505 09:13:09.039 | INFO | "huge" : "", | 20260505 09:13:09.039 | INFO | "huge_shared" : " 27 GB", | 20260505 09:13:09.039 | INFO | "malloc" : " 14 GB", | 20260505 09:13:09.039 | INFO | "pinned" : " 29 GB" | 20260505 09:13:09.039 | INFO | } | 20260505 09:13:09.039 | INFO | Perf report for the last 5.14213 seconds: | 20260505 09:13:09.039 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:13:09.039 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:09.039 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:09.039 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:09.039 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 09:13:09.039 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:09.039 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:09.039 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 09:13:09.039 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:09.039 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:09.039 | INFO | Uploaded 36GiB to GPU in 2.16792s (16.6058GiB/s) | 20260505 09:13:09.040 | INFO | Downloaded 34GiB to GPU in 2.05847s (16.5171GiB/s) | 20260505 09:13:14.176 | INFO | Analyzing chunk frames = [2042368,2042880) | 20260505 09:13:14.176 | INFO | MemInfo: { | 20260505 09:13:14.176 | INFO | "gpu" : " 17 GB", | 20260505 09:13:14.176 | INFO | "huge" : "", | 20260505 09:13:14.176 | INFO | "huge_shared" : " 27 GB", | 20260505 09:13:14.176 | INFO | "malloc" : " 14 GB", | 20260505 09:13:14.176 | INFO | "pinned" : " 29 GB" | 20260505 09:13:14.177 | INFO | } | 20260505 09:13:14.177 | INFO | Perf report for the last 5.13754 seconds: | 20260505 09:13:14.177 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:13:14.177 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:14.177 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:14.177 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:14.177 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 09:13:14.177 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:14.177 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:14.177 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 09:13:14.177 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:14.177 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:14.177 | INFO | Uploaded 36GiB to GPU in 2.24051s (16.0678GiB/s) | 20260505 09:13:14.177 | INFO | Downloaded 34GiB to GPU in 2.09701s (16.2136GiB/s) | 20260505 09:13:19.315 | INFO | Analyzing chunk frames = [2042880,2043392) | 20260505 09:13:19.315 | INFO | MemInfo: { | 20260505 09:13:19.315 | INFO | "gpu" : " 17 GB", | 20260505 09:13:19.315 | INFO | "huge" : "", | 20260505 09:13:19.315 | INFO | "huge_shared" : " 27 GB", | 20260505 09:13:19.315 | INFO | "malloc" : " 14 GB", | 20260505 09:13:19.315 | INFO | "pinned" : " 29 GB" | 20260505 09:13:19.315 | INFO | } | 20260505 09:13:19.315 | INFO | Perf report for the last 5.13814 seconds: | 20260505 09:13:19.315 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:13:19.315 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:19.315 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:19.315 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:19.315 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 09:13:19.315 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:19.315 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:19.315 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260505 09:13:19.315 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:19.315 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:19.315 | INFO | Uploaded 36GiB to GPU in 2.20062s (16.359GiB/s) | 20260505 09:13:19.315 | INFO | Downloaded 34GiB to GPU in 2.1443s (15.856GiB/s) | 20260505 09:13:24.455 | INFO | Analyzing chunk frames = [2043392,2043904) | 20260505 09:13:24.455 | INFO | MemInfo: { | 20260505 09:13:24.455 | INFO | "gpu" : " 17 GB", | 20260505 09:13:24.455 | INFO | "huge" : "", | 20260505 09:13:24.455 | INFO | "huge_shared" : " 27 GB", | 20260505 09:13:24.455 | INFO | "malloc" : " 14 GB", | 20260505 09:13:24.455 | INFO | "pinned" : " 29 GB" | 20260505 09:13:24.455 | INFO | } | 20260505 09:13:24.455 | INFO | Perf report for the last 5.13996 seconds: | 20260505 09:13:24.455 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:13:24.455 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:24.455 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:24.455 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:24.455 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 09:13:24.455 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:24.455 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:24.455 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 09:13:24.455 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:24.455 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:24.455 | INFO | Uploaded 36GiB to GPU in 2.19038s (16.4355GiB/s) | 20260505 09:13:24.455 | INFO | Downloaded 34GiB to GPU in 2.05758s (16.5242GiB/s) | 20260505 09:13:24.589 | INFO | Starting a round of Baz IO | 20260505 09:13:27.866 | INFO | Populating RTMetrics report | 20260505 09:13:27.866 | INFO | RTMetrics report populated | 20260505 09:13:27.866 | INFO | Opening tmp RTMetrics json file | 20260505 09:13:27.872 | INFO | Serializing RTMetrics to json | 20260505 09:13:27.875 | INFO | Serialization done, swapping tmp file now... | 20260505 09:13:27.876 | INFO | RTMetrics done | 20260505 09:13:28.015 | INFO | Finished a round of Baz IO: Wrote 2.443 GiB in 3.425 seconds | 20260505 09:13:28.015 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 26.40% / 0.02% / 73.58% | 20260505 09:13:28.015 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.11809% / 2.78155e-05% / 0.000115236% | 20260505 09:13:28.015 | INFO | Baz IO overall: | 20260505 09:13:28.015 | INFO | BytesWritten : 1174885839465 | 20260505 09:13:28.015 | INFO | NumEvents : 558983810344 | 20260505 09:13:29.594 | INFO | Analyzing chunk frames = [2043904,2044416) | 20260505 09:13:29.594 | INFO | MemInfo: { | 20260505 09:13:29.594 | INFO | "gpu" : " 17 GB", | 20260505 09:13:29.594 | INFO | "huge" : "", | 20260505 09:13:29.594 | INFO | "huge_shared" : " 27 GB", | 20260505 09:13:29.594 | INFO | "malloc" : " 14 GB", | 20260505 09:13:29.594 | INFO | "pinned" : " 29 GB" | 20260505 09:13:29.594 | INFO | } | 20260505 09:13:29.594 | INFO | Perf report for the last 5.13961 seconds: | 20260505 09:13:29.594 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:13:29.594 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:29.594 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:29.594 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:29.595 | INFO | ANALYSIS : 65.2%, 2.9, 99.9ms, 0 | 20260505 09:13:29.595 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:29.595 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260505 09:13:29.595 | INFO | PRE_HQ : 62.9%, 1.8, 60.1ms, 0 | 20260505 09:13:29.595 | INFO | BAZWRITER : 66.0%, 4.2, 140.8ms, 0 | 20260505 09:13:29.595 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260505 09:13:29.595 | INFO | Uploaded 36GiB to GPU in 2.21692s (16.2388GiB/s) | 20260505 09:13:29.595 | INFO | Downloaded 38GiB to GPU in 2.35376s (16.1444GiB/s) | 20260505 09:13:34.732 | INFO | Analyzing chunk frames = [2044416,2044928) | 20260505 09:13:34.732 | INFO | MemInfo: { | 20260505 09:13:34.732 | INFO | "gpu" : " 17 GB", | 20260505 09:13:34.732 | INFO | "huge" : "", | 20260505 09:13:34.732 | INFO | "huge_shared" : " 27 GB", | 20260505 09:13:34.732 | INFO | "malloc" : " 14 GB", | 20260505 09:13:34.732 | INFO | "pinned" : " 29 GB" | 20260505 09:13:34.733 | INFO | } | 20260505 09:13:34.733 | INFO | Perf report for the last 5.13829 seconds: | 20260505 09:13:34.733 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:13:34.733 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:34.733 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:34.733 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:34.733 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 09:13:34.733 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:34.733 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:34.733 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 09:13:34.733 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:34.733 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:34.733 | INFO | Uploaded 36GiB to GPU in 2.19011s (16.4375GiB/s) | 20260505 09:13:34.733 | INFO | Downloaded 34GiB to GPU in 2.10861s (16.1244GiB/s) | 20260505 09:13:39.871 | INFO | Analyzing chunk frames = [2044928,2045440) | 20260505 09:13:39.871 | INFO | MemInfo: { | 20260505 09:13:39.871 | INFO | "gpu" : " 17 GB", | 20260505 09:13:39.871 | INFO | "huge" : "", | 20260505 09:13:39.871 | INFO | "huge_shared" : " 27 GB", | 20260505 09:13:39.871 | INFO | "malloc" : " 14 GB", | 20260505 09:13:39.871 | INFO | "pinned" : " 29 GB" | 20260505 09:13:39.871 | INFO | } | 20260505 09:13:39.872 | INFO | Perf report for the last 5.13873 seconds: | 20260505 09:13:39.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:13:39.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:39.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:39.872 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:39.872 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 09:13:39.872 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:39.872 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:39.872 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 09:13:39.872 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:39.872 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:39.872 | INFO | Uploaded 36GiB to GPU in 2.17096s (16.5825GiB/s) | 20260505 09:13:39.872 | INFO | Downloaded 34GiB to GPU in 2.07024s (16.4232GiB/s) | 20260505 09:13:45.012 | INFO | Analyzing chunk frames = [2045440,2045952) | 20260505 09:13:45.012 | INFO | MemInfo: { | 20260505 09:13:45.012 | INFO | "gpu" : " 17 GB", | 20260505 09:13:45.012 | INFO | "huge" : "", | 20260505 09:13:45.012 | INFO | "huge_shared" : " 27 GB", | 20260505 09:13:45.012 | INFO | "malloc" : " 14 GB", | 20260505 09:13:45.012 | INFO | "pinned" : " 29 GB" | 20260505 09:13:45.012 | INFO | } | 20260505 09:13:45.012 | INFO | Perf report for the last 5.14089 seconds: | 20260505 09:13:45.012 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:13:45.012 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:45.012 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:45.012 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:45.013 | INFO | ANALYSIS : 63.2%, 3.0, 98.3ms, 0 | 20260505 09:13:45.013 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:45.013 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:45.013 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:13:45.013 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:45.013 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:45.013 | INFO | Uploaded 36GiB to GPU in 2.14621s (16.7737GiB/s) | 20260505 09:13:45.013 | INFO | Downloaded 34GiB to GPU in 2.07147s (16.4135GiB/s) | 20260505 09:13:50.152 | INFO | Analyzing chunk frames = [2045952,2046464) | 20260505 09:13:50.152 | INFO | MemInfo: { | 20260505 09:13:50.152 | INFO | "gpu" : " 17 GB", | 20260505 09:13:50.152 | INFO | "huge" : "", | 20260505 09:13:50.152 | INFO | "huge_shared" : " 27 GB", | 20260505 09:13:50.152 | INFO | "malloc" : " 14 GB", | 20260505 09:13:50.152 | INFO | "pinned" : " 29 GB" | 20260505 09:13:50.152 | INFO | } | 20260505 09:13:50.152 | INFO | Perf report for the last 5.13989 seconds: | 20260505 09:13:50.153 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:13:50.153 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:50.153 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:50.153 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:50.153 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 09:13:50.153 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:50.153 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:50.153 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 09:13:50.153 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:50.153 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:50.153 | INFO | Uploaded 36GiB to GPU in 2.23603s (16.1GiB/s) | 20260505 09:13:50.153 | INFO | Downloaded 34GiB to GPU in 2.09228s (16.2502GiB/s) | 20260505 09:13:55.292 | INFO | Analyzing chunk frames = [2046464,2046976) | 20260505 09:13:55.292 | INFO | MemInfo: { | 20260505 09:13:55.292 | INFO | "gpu" : " 17 GB", | 20260505 09:13:55.292 | INFO | "huge" : "", | 20260505 09:13:55.292 | INFO | "huge_shared" : " 27 GB", | 20260505 09:13:55.292 | INFO | "malloc" : " 14 GB", | 20260505 09:13:55.292 | INFO | "pinned" : " 29 GB" | 20260505 09:13:55.292 | INFO | } | 20260505 09:13:55.292 | INFO | Perf report for the last 5.14002 seconds: | 20260505 09:13:55.292 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:13:55.292 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:55.292 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:55.292 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:55.292 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 09:13:55.292 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:55.292 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:55.292 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 09:13:55.292 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:55.293 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:13:55.293 | INFO | Uploaded 36GiB to GPU in 2.16105s (16.6586GiB/s) | 20260505 09:13:55.293 | INFO | Downloaded 34GiB to GPU in 2.06076s (16.4988GiB/s) | 20260505 09:14:00.429 | INFO | Analyzing chunk frames = [2046976,2047488) | 20260505 09:14:00.429 | INFO | MemInfo: { | 20260505 09:14:00.429 | INFO | "gpu" : " 17 GB", | 20260505 09:14:00.429 | INFO | "huge" : "", | 20260505 09:14:00.429 | INFO | "huge_shared" : " 27 GB", | 20260505 09:14:00.429 | INFO | "malloc" : " 14 GB", | 20260505 09:14:00.429 | INFO | "pinned" : " 29 GB" | 20260505 09:14:00.429 | INFO | } | 20260505 09:14:00.430 | INFO | Perf report for the last 5.13715 seconds: | 20260505 09:14:00.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:14:00.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:00.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:00.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:00.430 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 09:14:00.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:00.430 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:00.430 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 09:14:00.430 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:00.430 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:00.430 | INFO | Uploaded 36GiB to GPU in 2.19076s (16.4327GiB/s) | 20260505 09:14:00.430 | INFO | Downloaded 34GiB to GPU in 2.15622s (15.7683GiB/s) | 20260505 09:14:05.569 | INFO | Analyzing chunk frames = [2047488,2048000) | 20260505 09:14:05.569 | INFO | MemInfo: { | 20260505 09:14:05.569 | INFO | "gpu" : " 17 GB", | 20260505 09:14:05.569 | INFO | "huge" : "", | 20260505 09:14:05.569 | INFO | "huge_shared" : " 27 GB", | 20260505 09:14:05.569 | INFO | "malloc" : " 14 GB", | 20260505 09:14:05.569 | INFO | "pinned" : " 29 GB" | 20260505 09:14:05.569 | INFO | } | 20260505 09:14:05.569 | INFO | Perf report for the last 5.13958 seconds: | 20260505 09:14:05.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:14:05.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:05.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:05.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:05.569 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 09:14:05.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:05.569 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:05.569 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 09:14:05.569 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:05.569 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:05.569 | INFO | Uploaded 36GiB to GPU in 2.17595s (16.5445GiB/s) | 20260505 09:14:05.569 | INFO | Downloaded 34GiB to GPU in 2.02725s (16.7715GiB/s) | 20260505 09:14:05.570 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 09:14:05.570 | INFO | Total mem usage: 15.3 GiB | 20260505 09:14:05.570 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 09:14:05.570 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.49%) | 20260505 09:14:05.570 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 09:14:05.570 | INFO | High Water Breakdown: | 20260505 09:14:05.570 | INFO | 60.9% : 8.77 GiB ( 93.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 09:14:05.570 | INFO | 38.9% : 5.61 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 09:14:05.570 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 09:14:05.570 | INFO | ------------------------------------------ | 20260505 09:14:05.570 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:14:05.570 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 09:14:05.570 | INFO | Total mem usage: 26.5 GiB | 20260505 09:14:05.570 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 09:14:05.570 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 09:14:05.570 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 09:14:05.570 | INFO | High Water Breakdown: | 20260505 09:14:05.570 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:14:05.570 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:14:05.570 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:14:05.570 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:14:05.570 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:14:05.570 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:14:05.570 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 09:14:05.570 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 09:14:05.570 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:14:05.570 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 09:14:05.570 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 09:14:05.570 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 09:14:05.570 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 09:14:05.570 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 09:14:05.570 | INFO | ------------------------------------------ | 20260505 09:14:05.570 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:14:05.570 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 09:14:05.570 | INFO | Total mem usage: 25.6 GiB | 20260505 09:14:05.570 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 09:14:05.570 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 09:14:05.570 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 09:14:05.570 | INFO | High Water Breakdown: | 20260505 09:14:05.570 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 09:14:05.570 | INFO | ------------------------------------------ | 20260505 09:14:05.570 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:14:05.570 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 09:14:05.570 | INFO | Total mem usage: 15.7 GiB | 20260505 09:14:05.570 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 09:14:05.570 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 09:14:05.570 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 09:14:05.570 | INFO | High Water Breakdown: | 20260505 09:14:05.570 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 09:14:05.570 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 09:14:05.570 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 09:14:05.570 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 09:14:05.570 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 09:14:05.570 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 09:14:05.570 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 09:14:05.570 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 09:14:05.570 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 09:14:05.570 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:14:05.570 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 09:14:05.571 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:14:05.571 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 09:14:05.571 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 09:14:05.571 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 09:14:05.571 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 09:14:05.571 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:14:05.571 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 09:14:05.571 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:14:05.571 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:14:05.571 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:14:05.571 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 09:14:05.571 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 09:14:05.571 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:14:05.571 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 09:14:05.571 | INFO | ------------------------------------------ | 20260505 09:14:05.571 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:14:05.571 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 09:14:05.571 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:14:05.571 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:14:05.571 | INFO | PerfReport: Full Execution -- 3.25e+01 / 7.59e+01, 1.36e+01 / 2.12e+00 | 20260505 09:14:05.571 | INFO | PerfReport: 0.29%, AggregateStats -- 9.31e-02 / 1.04e-01, 1.08e-02 / 6.45e-03 | 20260505 09:14:05.571 | INFO | PerfReport: 11.65%, Baseline -- 3.79e+00 / 4.39e+00, 6.11e-01 / 3.28e-01 | 20260505 09:14:05.571 | INFO | PerfReport: 3.31%, Histogram -- 1.07e+00 / 1.47e+00, 2.05e-01 / 1.05e-01 | 20260505 09:14:05.571 | INFO | PerfReport: 15.53%, DME -- 4.04e+01 / 4.42e+01, 4.08e+00 / 2.20e+00 | 20260505 09:14:05.571 | INFO | PerfReport: 56.52%, FrameLabeling -- 1.84e+01 / 2.07e+01, 1.86e+00 / 1.87e+00 | 20260505 09:14:05.571 | INFO | PerfReport: 7.48%, PulseAccumulating -- 2.43e+00 / 2.99e+00, 4.09e-01 / 2.42e-01 | 20260505 09:14:05.571 | INFO | PerfReport: 3.86%, Metrics -- 1.25e+00 / 1.57e+00, 3.56e-01 / 1.68e-01 | 20260505 09:14:05.571 | INFO | PerfReport: 1.37%, PulseSerializing -- 4.44e-01 / 4.95e-01, 8.24e-02 / 2.69e-02 | 20260505 09:14:05.571 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 09:14:05.571 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:14:05.571 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:14:05.571 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.30e+01 / 0.00e+00 | 20260505 09:14:05.571 | INFO | PerfReport: 22.47%, Upload -- 2.28e+01 / 0.00e+00, 8.06e+00 / 0.00e+00 | 20260505 09:14:05.571 | INFO | PerfReport: 21.75%, Download -- 2.21e+01 / 0.00e+00, 8.18e+00 / 0.00e+00 | 20260505 09:14:05.571 | INFO | PerfReport: 32.02%, Compute -- 3.25e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 09:14:05.571 | INFO | PerfReport: 1.73%, LockUpload -- 1.75e+00 / 0.00e+00, 7.44e+00 / 0.00e+00 | 20260505 09:14:05.571 | INFO | PerfReport: 1.14%, LockDownload -- 1.16e+00 / 0.00e+00, 2.82e+00 / 0.00e+00 | 20260505 09:14:05.571 | INFO | PerfReport: 20.87%, LockCompute -- 2.12e+01 / 0.00e+00, 1.77e+01 / 0.00e+00 | 20260505 09:14:05.571 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.22e-02 / 0.00e+00 | 20260505 09:14:05.706 | INFO | Starting a round of Baz IO | 20260505 09:14:08.989 | INFO | Populating RTMetrics report | 20260505 09:14:08.989 | INFO | RTMetrics report populated | 20260505 09:14:08.989 | INFO | Opening tmp RTMetrics json file | 20260505 09:14:09.035 | INFO | Serializing RTMetrics to json | 20260505 09:14:09.036 | INFO | Serialization done, swapping tmp file now... | 20260505 09:14:09.037 | INFO | RTMetrics done | 20260505 09:14:09.132 | INFO | Finished a round of Baz IO: Wrote 2.444 GiB in 3.425 seconds | 20260505 09:14:09.132 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 26.38% / 0.02% / 73.60% | 20260505 09:14:09.132 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.11367% / 2.78155e-05% / 6.75519e-05% | 20260505 09:14:09.132 | INFO | Baz IO overall: | 20260505 09:14:09.132 | INFO | BytesWritten : 1177510391134 | 20260505 09:14:09.132 | INFO | NumEvents : 560239224332 | 20260505 09:14:10.709 | INFO | Analyzing chunk frames = [2048000,2048512) | 20260505 09:14:10.709 | INFO | MemInfo: { | 20260505 09:14:10.709 | INFO | "gpu" : " 17 GB", | 20260505 09:14:10.709 | INFO | "huge" : "", | 20260505 09:14:10.709 | INFO | "huge_shared" : " 27 GB", | 20260505 09:14:10.709 | INFO | "malloc" : " 14 GB", | 20260505 09:14:10.709 | INFO | "pinned" : " 29 GB" | 20260505 09:14:10.709 | INFO | } | 20260505 09:14:10.709 | INFO | Perf report for the last 5.14023 seconds: | 20260505 09:14:10.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:14:10.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:10.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:10.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:10.709 | INFO | ANALYSIS : 65.4%, 2.9, 100.0ms, 0 | 20260505 09:14:10.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:10.709 | INFO | RT_METRICS : 46.9%, 1.0, 24.6ms, 0 | 20260505 09:14:10.709 | INFO | PRE_HQ : 62.9%, 1.9, 60.6ms, 0 | 20260505 09:14:10.710 | INFO | BAZWRITER : 66.0%, 4.2, 140.9ms, 0 | 20260505 09:14:10.710 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 09:14:10.710 | INFO | Uploaded 36GiB to GPU in 2.2142s (16.2587GiB/s) | 20260505 09:14:10.710 | INFO | Downloaded 38GiB to GPU in 2.33564s (16.2696GiB/s) | 20260505 09:14:15.847 | INFO | Analyzing chunk frames = [2048512,2049024) | 20260505 09:14:15.847 | INFO | MemInfo: { | 20260505 09:14:15.847 | INFO | "gpu" : " 17 GB", | 20260505 09:14:15.847 | INFO | "huge" : "", | 20260505 09:14:15.847 | INFO | "huge_shared" : " 27 GB", | 20260505 09:14:15.847 | INFO | "malloc" : " 14 GB", | 20260505 09:14:15.847 | INFO | "pinned" : " 29 GB" | 20260505 09:14:15.847 | INFO | } | 20260505 09:14:15.847 | INFO | Perf report for the last 5.1381 seconds: | 20260505 09:14:15.847 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:14:15.847 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:15.848 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:15.848 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:15.848 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 09:14:15.848 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:15.848 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:15.848 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 09:14:15.848 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:15.848 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:15.848 | INFO | Uploaded 36GiB to GPU in 2.20441s (16.3309GiB/s) | 20260505 09:14:15.848 | INFO | Downloaded 34GiB to GPU in 2.12006s (16.0372GiB/s) | 20260505 09:14:20.987 | INFO | Analyzing chunk frames = [2049024,2049536) | 20260505 09:14:20.987 | INFO | MemInfo: { | 20260505 09:14:20.987 | INFO | "gpu" : " 17 GB", | 20260505 09:14:20.987 | INFO | "huge" : "", | 20260505 09:14:20.987 | INFO | "huge_shared" : " 27 GB", | 20260505 09:14:20.987 | INFO | "malloc" : " 14 GB", | 20260505 09:14:20.987 | INFO | "pinned" : " 29 GB" | 20260505 09:14:20.987 | INFO | } | 20260505 09:14:20.987 | INFO | Perf report for the last 5.13975 seconds: | 20260505 09:14:20.987 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:14:20.987 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:20.987 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:20.987 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:20.987 | INFO | ANALYSIS : 63.4%, 3.0, 98.6ms, 0 | 20260505 09:14:20.987 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:20.987 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:20.987 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 09:14:20.987 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:20.987 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:20.988 | INFO | Uploaded 36GiB to GPU in 2.16124s (16.6571GiB/s) | 20260505 09:14:20.988 | INFO | Downloaded 34GiB to GPU in 2.06159s (16.4921GiB/s) | 20260505 09:14:26.126 | INFO | Analyzing chunk frames = [2049536,2050048) | 20260505 09:14:26.126 | INFO | MemInfo: { | 20260505 09:14:26.126 | INFO | "gpu" : " 17 GB", | 20260505 09:14:26.126 | INFO | "huge" : "", | 20260505 09:14:26.126 | INFO | "huge_shared" : " 27 GB", | 20260505 09:14:26.126 | INFO | "malloc" : " 14 GB", | 20260505 09:14:26.126 | INFO | "pinned" : " 29 GB" | 20260505 09:14:26.126 | INFO | } | 20260505 09:14:26.127 | INFO | Perf report for the last 5.13942 seconds: | 20260505 09:14:26.127 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:14:26.127 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:26.127 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:26.127 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:26.127 | INFO | ANALYSIS : 63.3%, 3.0, 98.3ms, 0 | 20260505 09:14:26.127 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:26.127 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:26.127 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 09:14:26.127 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:26.127 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:26.127 | INFO | Uploaded 36GiB to GPU in 2.1491s (16.7512GiB/s) | 20260505 09:14:26.127 | INFO | Downloaded 34GiB to GPU in 2.07043s (16.4217GiB/s) | 20260505 09:14:31.266 | INFO | Analyzing chunk frames = [2050048,2050560) | 20260505 09:14:31.266 | INFO | MemInfo: { | 20260505 09:14:31.266 | INFO | "gpu" : " 17 GB", | 20260505 09:14:31.266 | INFO | "huge" : "", | 20260505 09:14:31.266 | INFO | "huge_shared" : " 27 GB", | 20260505 09:14:31.266 | INFO | "malloc" : " 14 GB", | 20260505 09:14:31.266 | INFO | "pinned" : " 29 GB" | 20260505 09:14:31.266 | INFO | } | 20260505 09:14:31.266 | INFO | Perf report for the last 5.13948 seconds: | 20260505 09:14:31.266 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:14:31.266 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:31.266 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:31.266 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:31.266 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 09:14:31.266 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:31.266 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:31.266 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 09:14:31.267 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:31.267 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:31.267 | INFO | Uploaded 36GiB to GPU in 2.23815s (16.0847GiB/s) | 20260505 09:14:31.267 | INFO | Downloaded 34GiB to GPU in 2.10189s (16.1759GiB/s) | 20260505 09:14:36.406 | INFO | Analyzing chunk frames = [2050560,2051072) | 20260505 09:14:36.406 | INFO | MemInfo: { | 20260505 09:14:36.406 | INFO | "gpu" : " 17 GB", | 20260505 09:14:36.406 | INFO | "huge" : "", | 20260505 09:14:36.406 | INFO | "huge_shared" : " 27 GB", | 20260505 09:14:36.406 | INFO | "malloc" : " 14 GB", | 20260505 09:14:36.406 | INFO | "pinned" : " 29 GB" | 20260505 09:14:36.406 | INFO | } | 20260505 09:14:36.407 | INFO | Perf report for the last 5.14097 seconds: | 20260505 09:14:36.407 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:14:36.407 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:36.407 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:36.407 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:36.407 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 09:14:36.407 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:36.407 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:36.407 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 09:14:36.407 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:36.407 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:36.407 | INFO | Uploaded 36GiB to GPU in 2.16567s (16.6231GiB/s) | 20260505 09:14:36.408 | INFO | Downloaded 34GiB to GPU in 2.06034s (16.5022GiB/s) | 20260505 09:14:41.544 | INFO | Analyzing chunk frames = [2051072,2051584) | 20260505 09:14:41.544 | INFO | MemInfo: { | 20260505 09:14:41.544 | INFO | "gpu" : " 17 GB", | 20260505 09:14:41.544 | INFO | "huge" : "", | 20260505 09:14:41.544 | INFO | "huge_shared" : " 27 GB", | 20260505 09:14:41.544 | INFO | "malloc" : " 14 GB", | 20260505 09:14:41.544 | INFO | "pinned" : " 29 GB" | 20260505 09:14:41.544 | INFO | } | 20260505 09:14:41.544 | INFO | Perf report for the last 5.13713 seconds: | 20260505 09:14:41.544 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:14:41.544 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:41.544 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:41.544 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:41.544 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 09:14:41.544 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:41.544 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:41.544 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 09:14:41.544 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:41.544 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:41.545 | INFO | Uploaded 36GiB to GPU in 2.1997s (16.3659GiB/s) | 20260505 09:14:41.545 | INFO | Downloaded 34GiB to GPU in 2.13807s (15.9022GiB/s) | 20260505 09:14:46.683 | INFO | Analyzing chunk frames = [2051584,2052096) | 20260505 09:14:46.683 | INFO | MemInfo: { | 20260505 09:14:46.683 | INFO | "gpu" : " 17 GB", | 20260505 09:14:46.683 | INFO | "huge" : "", | 20260505 09:14:46.683 | INFO | "huge_shared" : " 27 GB", | 20260505 09:14:46.683 | INFO | "malloc" : " 14 GB", | 20260505 09:14:46.683 | INFO | "pinned" : " 29 GB" | 20260505 09:14:46.683 | INFO | } | 20260505 09:14:46.683 | INFO | Perf report for the last 5.13893 seconds: | 20260505 09:14:46.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:14:46.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:46.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:46.683 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:46.684 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 09:14:46.684 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:46.684 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:46.684 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 09:14:46.684 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:46.684 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:46.684 | INFO | Uploaded 36GiB to GPU in 2.17302s (16.5668GiB/s) | 20260505 09:14:46.684 | INFO | Downloaded 34GiB to GPU in 2.02752s (16.7693GiB/s) | 20260505 09:14:46.821 | INFO | Starting a round of Baz IO | 20260505 09:14:50.099 | INFO | Populating RTMetrics report | 20260505 09:14:50.099 | INFO | RTMetrics report populated | 20260505 09:14:50.099 | INFO | Opening tmp RTMetrics json file | 20260505 09:14:50.108 | INFO | Serializing RTMetrics to json | 20260505 09:14:50.124 | INFO | Serialization done, swapping tmp file now... | 20260505 09:14:50.124 | INFO | RTMetrics done | 20260505 09:14:50.251 | INFO | Finished a round of Baz IO: Wrote 2.444 GiB in 3.43 seconds | 20260505 09:14:50.251 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 26.37% / 0.02% / 73.62% | 20260505 09:14:50.251 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.1058% / 1.58946e-05% / 7.15256e-05% | 20260505 09:14:50.251 | INFO | Baz IO overall: | 20260505 09:14:50.251 | INFO | BytesWritten : 1180134237429 | 20260505 09:14:50.251 | INFO | NumEvents : 561494334941 | 20260505 09:14:51.822 | INFO | Analyzing chunk frames = [2052096,2052608) | 20260505 09:14:51.822 | INFO | MemInfo: { | 20260505 09:14:51.822 | INFO | "gpu" : " 17 GB", | 20260505 09:14:51.822 | INFO | "huge" : "", | 20260505 09:14:51.822 | INFO | "huge_shared" : " 27 GB", | 20260505 09:14:51.822 | INFO | "malloc" : " 14 GB", | 20260505 09:14:51.822 | INFO | "pinned" : " 29 GB" | 20260505 09:14:51.822 | INFO | } | 20260505 09:14:51.822 | INFO | Perf report for the last 5.1392 seconds: | 20260505 09:14:51.822 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:14:51.822 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:51.822 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:51.822 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:51.823 | INFO | ANALYSIS : 65.3%, 2.9, 100.1ms, 0 | 20260505 09:14:51.823 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:51.823 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260505 09:14:51.823 | INFO | PRE_HQ : 63.0%, 1.8, 60.3ms, 0 | 20260505 09:14:51.823 | INFO | BAZWRITER : 66.1%, 4.2, 141.3ms, 0 | 20260505 09:14:51.823 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 09:14:51.823 | INFO | Uploaded 36GiB to GPU in 2.2291s (16.15GiB/s) | 20260505 09:14:51.823 | INFO | Downloaded 38GiB to GPU in 2.34731s (16.1887GiB/s) | 20260505 09:14:56.962 | INFO | Analyzing chunk frames = [2052608,2053120) | 20260505 09:14:56.962 | INFO | MemInfo: { | 20260505 09:14:56.962 | INFO | "gpu" : " 17 GB", | 20260505 09:14:56.962 | INFO | "huge" : "", | 20260505 09:14:56.962 | INFO | "huge_shared" : " 27 GB", | 20260505 09:14:56.962 | INFO | "malloc" : " 14 GB", | 20260505 09:14:56.962 | INFO | "pinned" : " 29 GB" | 20260505 09:14:56.962 | INFO | } | 20260505 09:14:56.962 | INFO | Perf report for the last 5.13998 seconds: | 20260505 09:14:56.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:14:56.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:56.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:56.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:56.962 | INFO | ANALYSIS : 64.3%, 3.0, 100.3ms, 0 | 20260505 09:14:56.963 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:56.963 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:56.963 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:14:56.963 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:56.963 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:14:56.963 | INFO | Uploaded 36GiB to GPU in 2.20696s (16.312GiB/s) | 20260505 09:14:56.963 | INFO | Downloaded 34GiB to GPU in 2.11424s (16.0814GiB/s) | 20260505 09:15:02.102 | INFO | Analyzing chunk frames = [2053120,2053632) | 20260505 09:15:02.102 | INFO | MemInfo: { | 20260505 09:15:02.102 | INFO | "gpu" : " 17 GB", | 20260505 09:15:02.102 | INFO | "huge" : "", | 20260505 09:15:02.102 | INFO | "huge_shared" : " 27 GB", | 20260505 09:15:02.102 | INFO | "malloc" : " 14 GB", | 20260505 09:15:02.102 | INFO | "pinned" : " 29 GB" | 20260505 09:15:02.102 | INFO | } | 20260505 09:15:02.102 | INFO | Perf report for the last 5.13955 seconds: | 20260505 09:15:02.102 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:15:02.102 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:02.102 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:02.102 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:02.102 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 09:15:02.102 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:02.102 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:02.102 | INFO | PRE_HQ : 29.8%, 1.0, 15.4ms, 0 | 20260505 09:15:02.102 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:02.102 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:02.102 | INFO | Uploaded 36GiB to GPU in 2.15297s (16.7211GiB/s) | 20260505 09:15:02.102 | INFO | Downloaded 34GiB to GPU in 2.06956s (16.4286GiB/s) | 20260505 09:15:07.241 | INFO | Analyzing chunk frames = [2053632,2054144) | 20260505 09:15:07.241 | INFO | MemInfo: { | 20260505 09:15:07.241 | INFO | "gpu" : " 17 GB", | 20260505 09:15:07.241 | INFO | "huge" : "", | 20260505 09:15:07.241 | INFO | "huge_shared" : " 27 GB", | 20260505 09:15:07.241 | INFO | "malloc" : " 14 GB", | 20260505 09:15:07.241 | INFO | "pinned" : " 29 GB" | 20260505 09:15:07.241 | INFO | } | 20260505 09:15:07.242 | INFO | Perf report for the last 5.1395 seconds: | 20260505 09:15:07.245 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:15:07.245 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:07.245 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:07.245 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:07.245 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 09:15:07.245 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:07.245 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:07.245 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 09:15:07.245 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:07.245 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:07.245 | INFO | Uploaded 36GiB to GPU in 2.16158s (16.6545GiB/s) | 20260505 09:15:07.245 | INFO | Downloaded 34GiB to GPU in 2.06602s (16.4568GiB/s) | 20260505 09:15:12.381 | INFO | Analyzing chunk frames = [2054144,2054656) | 20260505 09:15:12.381 | INFO | MemInfo: { | 20260505 09:15:12.381 | INFO | "gpu" : " 17 GB", | 20260505 09:15:12.381 | INFO | "huge" : "", | 20260505 09:15:12.381 | INFO | "huge_shared" : " 27 GB", | 20260505 09:15:12.381 | INFO | "malloc" : " 14 GB", | 20260505 09:15:12.381 | INFO | "pinned" : " 29 GB" | 20260505 09:15:12.381 | INFO | } | 20260505 09:15:12.381 | INFO | Perf report for the last 5.13934 seconds: | 20260505 09:15:12.381 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:15:12.381 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:12.381 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:12.381 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:12.381 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 09:15:12.381 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:12.381 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:12.381 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 09:15:12.381 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:12.381 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:12.381 | INFO | Uploaded 36GiB to GPU in 2.23337s (16.1191GiB/s) | 20260505 09:15:12.381 | INFO | Downloaded 34GiB to GPU in 2.10695s (16.1371GiB/s) | 20260505 09:15:17.520 | INFO | Analyzing chunk frames = [2054656,2055168) | 20260505 09:15:17.520 | INFO | MemInfo: { | 20260505 09:15:17.520 | INFO | "gpu" : " 17 GB", | 20260505 09:15:17.520 | INFO | "huge" : "", | 20260505 09:15:17.520 | INFO | "huge_shared" : " 27 GB", | 20260505 09:15:17.520 | INFO | "malloc" : " 14 GB", | 20260505 09:15:17.524 | INFO | "pinned" : " 29 GB" | 20260505 09:15:17.524 | INFO | } | 20260505 09:15:17.524 | INFO | Perf report for the last 5.14271 seconds: | 20260505 09:15:17.524 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:15:17.524 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:17.524 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:17.524 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:17.524 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 09:15:17.524 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:17.524 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:17.524 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 09:15:17.524 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:17.524 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:17.524 | INFO | Uploaded 36GiB to GPU in 2.16064s (16.6617GiB/s) | 20260505 09:15:17.524 | INFO | Downloaded 34GiB to GPU in 2.05773s (16.5231GiB/s) | 20260505 09:15:22.659 | INFO | Analyzing chunk frames = [2055168,2055680) | 20260505 09:15:22.659 | INFO | MemInfo: { | 20260505 09:15:22.659 | INFO | "gpu" : " 17 GB", | 20260505 09:15:22.659 | INFO | "huge" : "", | 20260505 09:15:22.659 | INFO | "huge_shared" : " 27 GB", | 20260505 09:15:22.659 | INFO | "malloc" : " 14 GB", | 20260505 09:15:22.659 | INFO | "pinned" : " 29 GB" | 20260505 09:15:22.660 | INFO | } | 20260505 09:15:22.660 | INFO | Perf report for the last 5.13621 seconds: | 20260505 09:15:22.660 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:15:22.660 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:22.660 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:22.660 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:22.660 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 09:15:22.660 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:22.660 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:22.660 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 09:15:22.660 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:22.660 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:22.660 | INFO | Uploaded 36GiB to GPU in 2.16273s (16.6456GiB/s) | 20260505 09:15:22.660 | INFO | Downloaded 34GiB to GPU in 2.16024s (15.739GiB/s) | 20260505 09:15:27.799 | INFO | Analyzing chunk frames = [2055680,2056192) | 20260505 09:15:27.799 | INFO | MemInfo: { | 20260505 09:15:27.799 | INFO | "gpu" : " 17 GB", | 20260505 09:15:27.799 | INFO | "huge" : "", | 20260505 09:15:27.799 | INFO | "huge_shared" : " 27 GB", | 20260505 09:15:27.799 | INFO | "malloc" : " 14 GB", | 20260505 09:15:27.799 | INFO | "pinned" : " 29 GB" | 20260505 09:15:27.799 | INFO | } | 20260505 09:15:27.799 | INFO | Perf report for the last 5.13937 seconds: | 20260505 09:15:27.799 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:15:27.799 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:27.799 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:27.799 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:27.799 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 09:15:27.799 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:27.799 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:27.799 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 09:15:27.799 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:27.799 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:27.799 | INFO | Uploaded 36GiB to GPU in 2.18477s (16.4777GiB/s) | 20260505 09:15:27.799 | INFO | Downloaded 34GiB to GPU in 2.05554s (16.5407GiB/s) | 20260505 09:15:27.800 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 09:15:27.800 | INFO | Total mem usage: 15.3 GiB | 20260505 09:15:27.800 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 09:15:27.800 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.51%) | 20260505 09:15:27.800 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 09:15:27.800 | INFO | High Water Breakdown: | 20260505 09:15:27.800 | INFO | 60.9% : 8.76 GiB ( 93.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 09:15:27.800 | INFO | 39% : 5.61 GiB ( 84.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 09:15:27.800 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 09:15:27.800 | INFO | ------------------------------------------ | 20260505 09:15:27.800 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:15:27.800 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 09:15:27.800 | INFO | Total mem usage: 26.5 GiB | 20260505 09:15:27.800 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 09:15:27.800 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 09:15:27.800 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 09:15:27.800 | INFO | High Water Breakdown: | 20260505 09:15:27.800 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:15:27.800 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:15:27.800 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:15:27.800 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:15:27.800 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:15:27.800 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:15:27.800 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 09:15:27.800 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 09:15:27.800 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:15:27.800 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 09:15:27.800 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 09:15:27.800 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 09:15:27.800 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 09:15:27.800 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 09:15:27.800 | INFO | ------------------------------------------ | 20260505 09:15:27.800 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:15:27.800 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 09:15:27.800 | INFO | Total mem usage: 25.6 GiB | 20260505 09:15:27.800 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 09:15:27.800 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 09:15:27.800 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 09:15:27.800 | INFO | High Water Breakdown: | 20260505 09:15:27.800 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 09:15:27.800 | INFO | ------------------------------------------ | 20260505 09:15:27.800 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:15:27.800 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 09:15:27.800 | INFO | Total mem usage: 15.7 GiB | 20260505 09:15:27.800 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 09:15:27.800 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 09:15:27.800 | INFO | Unused Cache: 880 MiB (Waste: 5.48%) | 20260505 09:15:27.800 | INFO | High Water Breakdown: | 20260505 09:15:27.800 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 09:15:27.800 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 09:15:27.800 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 09:15:27.800 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 09:15:27.800 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 09:15:27.800 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 09:15:27.800 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 09:15:27.800 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 09:15:27.800 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 09:15:27.800 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:15:27.800 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 09:15:27.800 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:15:27.800 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 09:15:27.800 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 09:15:27.801 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 09:15:27.801 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 09:15:27.801 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:15:27.801 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 09:15:27.801 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:15:27.801 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:15:27.801 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:15:27.801 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 09:15:27.801 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 09:15:27.801 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:15:27.801 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 09:15:27.801 | INFO | ------------------------------------------ | 20260505 09:15:27.801 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:15:27.801 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 09:15:27.801 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:15:27.801 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:15:27.801 | INFO | PerfReport: Full Execution -- 3.25e+01 / 7.59e+01, 1.35e+01 / 2.15e+00 | 20260505 09:15:27.801 | INFO | PerfReport: 0.29%, AggregateStats -- 9.35e-02 / 1.06e-01, 1.11e-02 / 1.39e-02 | 20260505 09:15:27.801 | INFO | PerfReport: 11.66%, Baseline -- 3.79e+00 / 4.45e+00, 6.16e-01 / 2.92e-01 | 20260505 09:15:27.801 | INFO | PerfReport: 3.32%, Histogram -- 1.08e+00 / 1.48e+00, 2.09e-01 / 1.06e-01 | 20260505 09:15:27.801 | INFO | PerfReport: 15.51%, DME -- 4.03e+01 / 4.42e+01, 4.14e+00 / 2.07e+00 | 20260505 09:15:27.801 | INFO | PerfReport: 56.54%, FrameLabeling -- 1.84e+01 / 2.06e+01, 1.89e+00 / 1.81e+00 | 20260505 09:15:27.801 | INFO | PerfReport: 7.47%, PulseAccumulating -- 2.43e+00 / 2.97e+00, 4.13e-01 / 2.52e-01 | 20260505 09:15:27.801 | INFO | PerfReport: 3.85%, Metrics -- 1.25e+00 / 1.56e+00, 3.57e-01 / 1.90e-01 | 20260505 09:15:27.801 | INFO | PerfReport: 1.37%, PulseSerializing -- 4.45e-01 / 4.97e-01, 8.36e-02 / 3.05e-02 | 20260505 09:15:27.801 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 09:15:27.801 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:15:27.801 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:15:27.801 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.32e+01 / 0.00e+00 | 20260505 09:15:27.801 | INFO | PerfReport: 22.45%, Upload -- 2.28e+01 / 0.00e+00, 8.13e+00 / 0.00e+00 | 20260505 09:15:27.801 | INFO | PerfReport: 21.75%, Download -- 2.21e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260505 09:15:27.801 | INFO | PerfReport: 32.04%, Compute -- 3.25e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 09:15:27.801 | INFO | PerfReport: 1.71%, LockUpload -- 1.74e+00 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260505 09:15:27.801 | INFO | PerfReport: 1.13%, LockDownload -- 1.15e+00 / 0.00e+00, 2.75e+00 / 0.00e+00 | 20260505 09:15:27.801 | INFO | PerfReport: 20.90%, LockCompute -- 2.12e+01 / 0.00e+00, 1.78e+01 / 0.00e+00 | 20260505 09:15:27.801 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.60e-02 / 0.00e+00 | 20260505 09:15:27.929 | INFO | Starting a round of Baz IO | 20260505 09:15:31.198 | INFO | Populating RTMetrics report | 20260505 09:15:31.198 | INFO | RTMetrics report populated | 20260505 09:15:31.198 | INFO | Opening tmp RTMetrics json file | 20260505 09:15:31.217 | INFO | Serializing RTMetrics to json | 20260505 09:15:31.219 | INFO | Serialization done, swapping tmp file now... | 20260505 09:15:31.222 | INFO | RTMetrics done | 20260505 09:15:31.364 | INFO | Finished a round of Baz IO: Wrote 2.443 GiB in 3.434 seconds | 20260505 09:15:31.364 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 26.35% / 0.02% / 73.63% | 20260505 09:15:31.364 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.09701% / 3.97364e-05% / 6.75519e-05% | 20260505 09:15:31.364 | INFO | Baz IO overall: | 20260505 09:15:31.364 | INFO | BytesWritten : 1182757759557 | 20260505 09:15:31.364 | INFO | NumEvents : 562749036663 | 20260505 09:15:32.938 | INFO | Analyzing chunk frames = [2056192,2056704) | 20260505 09:15:32.938 | INFO | MemInfo: { | 20260505 09:15:32.938 | INFO | "gpu" : " 17 GB", | 20260505 09:15:32.938 | INFO | "huge" : "", | 20260505 09:15:32.938 | INFO | "huge_shared" : " 27 GB", | 20260505 09:15:32.938 | INFO | "malloc" : " 14 GB", | 20260505 09:15:32.938 | INFO | "pinned" : " 29 GB" | 20260505 09:15:32.938 | INFO | } | 20260505 09:15:32.938 | INFO | Perf report for the last 5.1389 seconds: | 20260505 09:15:32.938 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:15:32.938 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:32.938 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:32.938 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:32.938 | INFO | ANALYSIS : 65.0%, 2.9, 98.6ms, 0 | 20260505 09:15:32.938 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:32.938 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260505 09:15:32.938 | INFO | PRE_HQ : 63.2%, 1.9, 60.9ms, 0 | 20260505 09:15:32.938 | INFO | BAZWRITER : 66.1%, 4.2, 144.4ms, 0 | 20260505 09:15:32.938 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 09:15:32.938 | INFO | Uploaded 36GiB to GPU in 2.25466s (15.9669GiB/s) | 20260505 09:15:32.938 | INFO | Downloaded 38GiB to GPU in 2.3796s (15.9691GiB/s) | 20260505 09:15:38.077 | INFO | Analyzing chunk frames = [2056704,2057216) | 20260505 09:15:38.077 | INFO | MemInfo: { | 20260505 09:15:38.077 | INFO | "gpu" : " 17 GB", | 20260505 09:15:38.077 | INFO | "huge" : "", | 20260505 09:15:38.077 | INFO | "huge_shared" : " 27 GB", | 20260505 09:15:38.077 | INFO | "malloc" : " 14 GB", | 20260505 09:15:38.078 | INFO | "pinned" : " 29 GB" | 20260505 09:15:38.078 | INFO | } | 20260505 09:15:38.078 | INFO | Perf report for the last 5.13947 seconds: | 20260505 09:15:38.078 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:15:38.078 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:38.078 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:38.078 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:38.078 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 09:15:38.078 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:38.078 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:38.078 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 09:15:38.078 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:38.078 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:38.078 | INFO | Uploaded 36GiB to GPU in 2.18805s (16.453GiB/s) | 20260505 09:15:38.078 | INFO | Downloaded 34GiB to GPU in 2.10741s (16.1335GiB/s) | 20260505 09:15:43.217 | INFO | Analyzing chunk frames = [2057216,2057728) | 20260505 09:15:43.217 | INFO | MemInfo: { | 20260505 09:15:43.217 | INFO | "gpu" : " 17 GB", | 20260505 09:15:43.217 | INFO | "huge" : "", | 20260505 09:15:43.217 | INFO | "huge_shared" : " 27 GB", | 20260505 09:15:43.217 | INFO | "malloc" : " 14 GB", | 20260505 09:15:43.217 | INFO | "pinned" : " 29 GB" | 20260505 09:15:43.217 | INFO | } | 20260505 09:15:43.218 | INFO | Perf report for the last 5.13993 seconds: | 20260505 09:15:43.218 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:15:43.218 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:43.218 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:43.218 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:43.218 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 09:15:43.218 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:43.218 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:43.218 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 09:15:43.218 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:43.218 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:43.218 | INFO | Uploaded 36GiB to GPU in 2.14858s (16.7553GiB/s) | 20260505 09:15:43.218 | INFO | Downloaded 34GiB to GPU in 2.06741s (16.4457GiB/s) | 20260505 09:15:48.357 | INFO | Analyzing chunk frames = [2057728,2058240) | 20260505 09:15:48.357 | INFO | MemInfo: { | 20260505 09:15:48.357 | INFO | "gpu" : " 17 GB", | 20260505 09:15:48.357 | INFO | "huge" : "", | 20260505 09:15:48.357 | INFO | "huge_shared" : " 27 GB", | 20260505 09:15:48.357 | INFO | "malloc" : " 14 GB", | 20260505 09:15:48.357 | INFO | "pinned" : " 29 GB" | 20260505 09:15:48.357 | INFO | } | 20260505 09:15:48.357 | INFO | Perf report for the last 5.13932 seconds: | 20260505 09:15:48.357 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:15:48.357 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:48.357 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:48.357 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:48.357 | INFO | ANALYSIS : 63.3%, 3.0, 98.3ms, 0 | 20260505 09:15:48.357 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:48.357 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:48.357 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:15:48.357 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:48.357 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:48.357 | INFO | Uploaded 36GiB to GPU in 2.15811s (16.6813GiB/s) | 20260505 09:15:48.357 | INFO | Downloaded 34GiB to GPU in 2.0685s (16.437GiB/s) | 20260505 09:15:53.495 | INFO | Analyzing chunk frames = [2058240,2058752) | 20260505 09:15:53.495 | INFO | MemInfo: { | 20260505 09:15:53.495 | INFO | "gpu" : " 17 GB", | 20260505 09:15:53.495 | INFO | "huge" : "", | 20260505 09:15:53.495 | INFO | "huge_shared" : " 27 GB", | 20260505 09:15:53.495 | INFO | "malloc" : " 14 GB", | 20260505 09:15:53.495 | INFO | "pinned" : " 29 GB" | 20260505 09:15:53.495 | INFO | } | 20260505 09:15:53.495 | INFO | Perf report for the last 5.13822 seconds: | 20260505 09:15:53.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:15:53.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:53.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:53.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:53.495 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 09:15:53.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:53.495 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:53.495 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:15:53.495 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:53.495 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:53.495 | INFO | Uploaded 36GiB to GPU in 2.23415s (16.1135GiB/s) | 20260505 09:15:53.495 | INFO | Downloaded 34GiB to GPU in 2.09842s (16.2026GiB/s) | 20260505 09:15:58.634 | INFO | Analyzing chunk frames = [2058752,2059264) | 20260505 09:15:58.634 | INFO | MemInfo: { | 20260505 09:15:58.634 | INFO | "gpu" : " 17 GB", | 20260505 09:15:58.634 | INFO | "huge" : "", | 20260505 09:15:58.634 | INFO | "huge_shared" : " 27 GB", | 20260505 09:15:58.634 | INFO | "malloc" : " 14 GB", | 20260505 09:15:58.635 | INFO | "pinned" : " 29 GB" | 20260505 09:15:58.635 | INFO | } | 20260505 09:15:58.635 | INFO | Perf report for the last 5.13967 seconds: | 20260505 09:15:58.635 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:15:58.635 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:58.635 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:58.635 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:58.635 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 09:15:58.635 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:58.635 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:58.635 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:15:58.635 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:58.635 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:15:58.635 | INFO | Uploaded 36GiB to GPU in 2.15761s (16.6851GiB/s) | 20260505 09:15:58.635 | INFO | Downloaded 34GiB to GPU in 2.05776s (16.5229GiB/s) | 20260505 09:16:03.775 | INFO | Analyzing chunk frames = [2059264,2059776) | 20260505 09:16:03.775 | INFO | MemInfo: { | 20260505 09:16:03.775 | INFO | "gpu" : " 17 GB", | 20260505 09:16:03.775 | INFO | "huge" : "", | 20260505 09:16:03.775 | INFO | "huge_shared" : " 27 GB", | 20260505 09:16:03.775 | INFO | "malloc" : " 14 GB", | 20260505 09:16:03.775 | INFO | "pinned" : " 29 GB" | 20260505 09:16:03.775 | INFO | } | 20260505 09:16:03.775 | INFO | Perf report for the last 5.14019 seconds: | 20260505 09:16:03.775 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:16:03.775 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:03.775 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:03.775 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:03.775 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 09:16:03.775 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:03.775 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:03.775 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 09:16:03.775 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:03.775 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:03.775 | INFO | Uploaded 36GiB to GPU in 2.18511s (16.4751GiB/s) | 20260505 09:16:03.775 | INFO | Downloaded 34GiB to GPU in 2.14805s (15.8283GiB/s) | 20260505 09:16:08.914 | INFO | Analyzing chunk frames = [2059776,2060288) | 20260505 09:16:08.914 | INFO | MemInfo: { | 20260505 09:16:08.914 | INFO | "gpu" : " 17 GB", | 20260505 09:16:08.915 | INFO | "huge" : "", | 20260505 09:16:08.915 | INFO | "huge_shared" : " 27 GB", | 20260505 09:16:08.915 | INFO | "malloc" : " 14 GB", | 20260505 09:16:08.915 | INFO | "pinned" : " 29 GB" | 20260505 09:16:08.915 | INFO | } | 20260505 09:16:08.915 | INFO | Perf report for the last 5.13975 seconds: | 20260505 09:16:08.915 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:16:08.915 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:08.915 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:08.915 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:08.915 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 09:16:08.915 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:08.915 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:08.915 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 09:16:08.915 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:08.915 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:08.915 | INFO | Uploaded 36GiB to GPU in 2.18381s (16.4849GiB/s) | 20260505 09:16:08.915 | INFO | Downloaded 34GiB to GPU in 2.05406s (16.5526GiB/s) | 20260505 09:16:09.045 | INFO | Starting a round of Baz IO | 20260505 09:16:12.314 | INFO | Populating RTMetrics report | 20260505 09:16:12.314 | INFO | RTMetrics report populated | 20260505 09:16:12.314 | INFO | Opening tmp RTMetrics json file | 20260505 09:16:12.338 | INFO | Serializing RTMetrics to json | 20260505 09:16:12.375 | INFO | Serialization done, swapping tmp file now... | 20260505 09:16:12.402 | INFO | RTMetrics done | 20260505 09:16:12.484 | INFO | Finished a round of Baz IO: Wrote 2.444 GiB in 3.438 seconds | 20260505 09:16:12.484 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 26.34% / 0.02% / 73.65% | 20260505 09:16:12.484 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.09314% / 3.17891e-05% / 6.75519e-05% | 20260505 09:16:12.484 | INFO | Baz IO overall: | 20260505 09:16:12.484 | INFO | BytesWritten : 1185382428876 | 20260505 09:16:12.484 | INFO | NumEvents : 564004553631 | 20260505 09:16:14.053 | INFO | Analyzing chunk frames = [2060288,2060800) | 20260505 09:16:14.053 | INFO | MemInfo: { | 20260505 09:16:14.053 | INFO | "gpu" : " 17 GB", | 20260505 09:16:14.053 | INFO | "huge" : "", | 20260505 09:16:14.053 | INFO | "huge_shared" : " 27 GB", | 20260505 09:16:14.053 | INFO | "malloc" : " 14 GB", | 20260505 09:16:14.053 | INFO | "pinned" : " 29 GB" | 20260505 09:16:14.053 | INFO | } | 20260505 09:16:14.053 | INFO | Perf report for the last 5.13841 seconds: | 20260505 09:16:14.053 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:16:14.053 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:14.053 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:14.053 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:14.053 | INFO | ANALYSIS : 65.0%, 2.9, 99.3ms, 0 | 20260505 09:16:14.053 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:14.053 | INFO | RT_METRICS : 47.6%, 1.0, 25.0ms, 0 | 20260505 09:16:14.053 | INFO | PRE_HQ : 63.0%, 1.8, 60.4ms, 0 | 20260505 09:16:14.053 | INFO | BAZWRITER : 66.2%, 4.2, 141.8ms, 0 | 20260505 09:16:14.053 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 09:16:14.054 | INFO | Uploaded 36GiB to GPU in 2.25117s (15.9917GiB/s) | 20260505 09:16:14.054 | INFO | Downloaded 38GiB to GPU in 2.36215s (16.087GiB/s) | 20260505 09:16:19.192 | INFO | Analyzing chunk frames = [2060800,2061312) | 20260505 09:16:19.192 | INFO | MemInfo: { | 20260505 09:16:19.192 | INFO | "gpu" : " 17 GB", | 20260505 09:16:19.192 | INFO | "huge" : "", | 20260505 09:16:19.192 | INFO | "huge_shared" : " 27 GB", | 20260505 09:16:19.192 | INFO | "malloc" : " 14 GB", | 20260505 09:16:19.192 | INFO | "pinned" : " 29 GB" | 20260505 09:16:19.192 | INFO | } | 20260505 09:16:19.193 | INFO | Perf report for the last 5.13948 seconds: | 20260505 09:16:19.193 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:16:19.193 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:19.193 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:19.193 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:19.193 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 09:16:19.193 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:19.193 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:19.193 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:16:19.193 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:19.193 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:19.193 | INFO | Uploaded 36GiB to GPU in 2.19993s (16.3642GiB/s) | 20260505 09:16:19.193 | INFO | Downloaded 34GiB to GPU in 2.10834s (16.1264GiB/s) | 20260505 09:16:24.331 | INFO | Analyzing chunk frames = [2061312,2061824) | 20260505 09:16:24.331 | INFO | MemInfo: { | 20260505 09:16:24.331 | INFO | "gpu" : " 17 GB", | 20260505 09:16:24.331 | INFO | "huge" : "", | 20260505 09:16:24.331 | INFO | "huge_shared" : " 27 GB", | 20260505 09:16:24.331 | INFO | "malloc" : " 14 GB", | 20260505 09:16:24.332 | INFO | "pinned" : " 29 GB" | 20260505 09:16:24.332 | INFO | } | 20260505 09:16:24.332 | INFO | Perf report for the last 5.13907 seconds: | 20260505 09:16:24.332 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:16:24.332 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:24.332 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:24.332 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:24.332 | INFO | ANALYSIS : 63.4%, 3.0, 98.6ms, 0 | 20260505 09:16:24.332 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:24.332 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:24.332 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 09:16:24.332 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:24.332 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:24.332 | INFO | Uploaded 36GiB to GPU in 2.16192s (16.6519GiB/s) | 20260505 09:16:24.332 | INFO | Downloaded 34GiB to GPU in 2.07161s (16.4123GiB/s) | 20260505 09:16:29.472 | INFO | Analyzing chunk frames = [2061824,2062336) | 20260505 09:16:29.472 | INFO | MemInfo: { | 20260505 09:16:29.472 | INFO | "gpu" : " 17 GB", | 20260505 09:16:29.472 | INFO | "huge" : "", | 20260505 09:16:29.472 | INFO | "huge_shared" : " 27 GB", | 20260505 09:16:29.472 | INFO | "malloc" : " 14 GB", | 20260505 09:16:29.472 | INFO | "pinned" : " 29 GB" | 20260505 09:16:29.472 | INFO | } | 20260505 09:16:29.472 | INFO | Perf report for the last 5.14036 seconds: | 20260505 09:16:29.472 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:16:29.472 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:29.472 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:29.472 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:29.472 | INFO | ANALYSIS : 63.3%, 3.0, 98.3ms, 0 | 20260505 09:16:29.472 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:29.472 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:29.472 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 09:16:29.472 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:29.472 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:29.472 | INFO | Uploaded 36GiB to GPU in 2.15625s (16.6956GiB/s) | 20260505 09:16:29.472 | INFO | Downloaded 34GiB to GPU in 2.06938s (16.43GiB/s) | 20260505 09:16:34.611 | INFO | Analyzing chunk frames = [2062336,2062848) | 20260505 09:16:34.611 | INFO | MemInfo: { | 20260505 09:16:34.611 | INFO | "gpu" : " 17 GB", | 20260505 09:16:34.611 | INFO | "huge" : "", | 20260505 09:16:34.611 | INFO | "huge_shared" : " 27 GB", | 20260505 09:16:34.611 | INFO | "malloc" : " 14 GB", | 20260505 09:16:34.611 | INFO | "pinned" : " 29 GB" | 20260505 09:16:34.611 | INFO | } | 20260505 09:16:34.611 | INFO | Perf report for the last 5.13898 seconds: | 20260505 09:16:34.611 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:16:34.611 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:34.611 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:34.611 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:34.611 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 09:16:34.611 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:34.611 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:34.611 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 09:16:34.611 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:34.611 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:34.611 | INFO | Uploaded 36GiB to GPU in 2.23794s (16.0862GiB/s) | 20260505 09:16:34.611 | INFO | Downloaded 34GiB to GPU in 2.09531s (16.2267GiB/s) | 20260505 09:16:39.751 | INFO | Analyzing chunk frames = [2062848,2063360) | 20260505 09:16:39.751 | INFO | MemInfo: { | 20260505 09:16:39.751 | INFO | "gpu" : " 17 GB", | 20260505 09:16:39.751 | INFO | "huge" : "", | 20260505 09:16:39.751 | INFO | "huge_shared" : " 27 GB", | 20260505 09:16:39.751 | INFO | "malloc" : " 14 GB", | 20260505 09:16:39.751 | INFO | "pinned" : " 29 GB" | 20260505 09:16:39.751 | INFO | } | 20260505 09:16:39.752 | INFO | Perf report for the last 5.14039 seconds: | 20260505 09:16:39.752 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:16:39.752 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:39.752 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:39.752 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:39.752 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 09:16:39.752 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:39.752 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:39.752 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 09:16:39.752 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:39.752 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:39.752 | INFO | Uploaded 36GiB to GPU in 2.19577s (16.3951GiB/s) | 20260505 09:16:39.752 | INFO | Downloaded 34GiB to GPU in 2.0973s (16.2113GiB/s) | 20260505 09:16:44.890 | INFO | Analyzing chunk frames = [2063360,2063872) | 20260505 09:16:44.890 | INFO | MemInfo: { | 20260505 09:16:44.890 | INFO | "gpu" : " 17 GB", | 20260505 09:16:44.890 | INFO | "huge" : "", | 20260505 09:16:44.890 | INFO | "huge_shared" : " 27 GB", | 20260505 09:16:44.890 | INFO | "malloc" : " 14 GB", | 20260505 09:16:44.890 | INFO | "pinned" : " 29 GB" | 20260505 09:16:44.890 | INFO | } | 20260505 09:16:44.890 | INFO | Perf report for the last 5.1384 seconds: | 20260505 09:16:44.890 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:16:44.890 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:44.890 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:44.890 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:44.890 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 09:16:44.890 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:44.890 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:44.890 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 09:16:44.890 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:44.890 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:44.890 | INFO | Uploaded 36GiB to GPU in 2.17325s (16.565GiB/s) | 20260505 09:16:44.890 | INFO | Downloaded 34GiB to GPU in 2.13911s (15.8945GiB/s) | 20260505 09:16:50.028 | INFO | Analyzing chunk frames = [2063872,2064384) | 20260505 09:16:50.028 | INFO | MemInfo: { | 20260505 09:16:50.028 | INFO | "gpu" : " 17 GB", | 20260505 09:16:50.028 | INFO | "huge" : "", | 20260505 09:16:50.028 | INFO | "huge_shared" : " 27 GB", | 20260505 09:16:50.028 | INFO | "malloc" : " 14 GB", | 20260505 09:16:50.028 | INFO | "pinned" : " 29 GB" | 20260505 09:16:50.028 | INFO | } | 20260505 09:16:50.028 | INFO | Perf report for the last 5.13836 seconds: | 20260505 09:16:50.028 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:16:50.028 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:50.028 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:50.028 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:50.028 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 09:16:50.028 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:50.028 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:50.028 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 09:16:50.028 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:50.028 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:50.029 | INFO | Uploaded 36GiB to GPU in 2.1802s (16.5123GiB/s) | 20260505 09:16:50.029 | INFO | Downloaded 34GiB to GPU in 2.05644s (16.5334GiB/s) | 20260505 09:16:50.029 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 09:16:50.029 | INFO | Total mem usage: 15.3 GiB | 20260505 09:16:50.029 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 09:16:50.029 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -4.97%) | 20260505 09:16:50.029 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 09:16:50.029 | INFO | High Water Breakdown: | 20260505 09:16:50.029 | INFO | 61.1% : 8.93 GiB ( 95.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 09:16:50.029 | INFO | 38.7% : 5.66 GiB ( 85.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 09:16:50.029 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 09:16:50.029 | INFO | ------------------------------------------ | 20260505 09:16:50.029 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:16:50.029 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 09:16:50.029 | INFO | Total mem usage: 26.5 GiB | 20260505 09:16:50.029 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 09:16:50.029 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 09:16:50.029 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 09:16:50.029 | INFO | High Water Breakdown: | 20260505 09:16:50.029 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:16:50.029 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:16:50.029 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:16:50.029 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:16:50.029 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:16:50.029 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:16:50.029 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 09:16:50.029 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 09:16:50.029 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:16:50.029 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 09:16:50.029 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 09:16:50.029 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 09:16:50.029 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 09:16:50.029 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 09:16:50.029 | INFO | ------------------------------------------ | 20260505 09:16:50.029 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:16:50.029 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 09:16:50.029 | INFO | Total mem usage: 25.6 GiB | 20260505 09:16:50.029 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 09:16:50.029 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 09:16:50.029 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 09:16:50.029 | INFO | High Water Breakdown: | 20260505 09:16:50.029 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 09:16:50.029 | INFO | ------------------------------------------ | 20260505 09:16:50.029 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:16:50.029 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 09:16:50.029 | INFO | Total mem usage: 15.7 GiB | 20260505 09:16:50.030 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 09:16:50.030 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 09:16:50.030 | INFO | Unused Cache: 880 MiB (Waste: 5.48%) | 20260505 09:16:50.030 | INFO | High Water Breakdown: | 20260505 09:16:50.030 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 09:16:50.030 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 09:16:50.030 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 09:16:50.030 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 09:16:50.030 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 09:16:50.030 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 09:16:50.030 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 09:16:50.030 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 09:16:50.030 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 09:16:50.030 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:16:50.030 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 09:16:50.030 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:16:50.030 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 09:16:50.030 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 09:16:50.030 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 09:16:50.030 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 09:16:50.030 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:16:50.030 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 09:16:50.030 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:16:50.030 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:16:50.030 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:16:50.030 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 09:16:50.030 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 09:16:50.030 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:16:50.030 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 09:16:50.030 | INFO | ------------------------------------------ | 20260505 09:16:50.030 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:16:50.030 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 09:16:50.030 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:16:50.030 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:16:50.030 | INFO | PerfReport: Full Execution -- 3.25e+01 / 7.58e+01, 1.36e+01 / 1.99e+00 | 20260505 09:16:50.030 | INFO | PerfReport: 0.29%, AggregateStats -- 9.31e-02 / 1.05e-01, 1.05e-02 / 5.72e-03 | 20260505 09:16:50.030 | INFO | PerfReport: 11.65%, Baseline -- 3.78e+00 / 4.36e+00, 6.02e-01 / 3.24e-01 | 20260505 09:16:50.030 | INFO | PerfReport: 3.31%, Histogram -- 1.08e+00 / 1.45e+00, 2.04e-01 / 1.07e-01 | 20260505 09:16:50.030 | INFO | PerfReport: 15.50%, DME -- 4.03e+01 / 4.38e+01, 4.01e+00 / 1.21e+00 | 20260505 09:16:50.030 | INFO | PerfReport: 56.53%, FrameLabeling -- 1.84e+01 / 2.10e+01, 1.82e+00 / 1.94e+00 | 20260505 09:16:50.030 | INFO | PerfReport: 7.50%, PulseAccumulating -- 2.44e+00 / 3.01e+00, 4.16e-01 / 2.22e-01 | 20260505 09:16:50.030 | INFO | PerfReport: 3.85%, Metrics -- 1.25e+00 / 1.59e+00, 3.53e-01 / 1.80e-01 | 20260505 09:16:50.030 | INFO | PerfReport: 1.37%, PulseSerializing -- 4.44e-01 / 4.93e-01, 8.22e-02 / 2.73e-02 | 20260505 09:16:50.030 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 09:16:50.030 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:16:50.030 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:16:50.030 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.33e+01 / 0.00e+00 | 20260505 09:16:50.030 | INFO | PerfReport: 22.52%, Upload -- 2.28e+01 / 0.00e+00, 7.97e+00 / 0.00e+00 | 20260505 09:16:50.030 | INFO | PerfReport: 21.82%, Download -- 2.21e+01 / 0.00e+00, 8.14e+00 / 0.00e+00 | 20260505 09:16:50.030 | INFO | PerfReport: 32.03%, Compute -- 3.25e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 09:16:50.030 | INFO | PerfReport: 1.75%, LockUpload -- 1.78e+00 / 0.00e+00, 7.45e+00 / 0.00e+00 | 20260505 09:16:50.030 | INFO | PerfReport: 1.15%, LockDownload -- 1.16e+00 / 0.00e+00, 2.74e+00 / 0.00e+00 | 20260505 09:16:50.030 | INFO | PerfReport: 20.72%, LockCompute -- 2.10e+01 / 0.00e+00, 1.77e+01 / 0.00e+00 | 20260505 09:16:50.030 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.83e-02 / 0.00e+00 | 20260505 09:16:50.163 | INFO | Starting a round of Baz IO | 20260505 09:16:53.439 | INFO | Populating RTMetrics report | 20260505 09:16:53.439 | INFO | RTMetrics report populated | 20260505 09:16:53.439 | INFO | Opening tmp RTMetrics json file | 20260505 09:16:53.455 | INFO | Serializing RTMetrics to json | 20260505 09:16:53.457 | INFO | Serialization done, swapping tmp file now... | 20260505 09:16:53.460 | INFO | RTMetrics done | 20260505 09:16:53.606 | INFO | Finished a round of Baz IO: Wrote 2.445 GiB in 3.443 seconds | 20260505 09:16:53.606 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 26.32% / 0.02% / 73.66% | 20260505 09:16:53.606 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.08723% / 3.17891e-05% / 7.54992e-05% | 20260505 09:16:53.606 | INFO | Baz IO overall: | 20260505 09:16:53.606 | INFO | BytesWritten : 1188007476630 | 20260505 09:16:53.606 | INFO | NumEvents : 565260639120 | 20260505 09:16:55.167 | INFO | Analyzing chunk frames = [2064384,2064896) | 20260505 09:16:55.167 | INFO | MemInfo: { | 20260505 09:16:55.167 | INFO | "gpu" : " 17 GB", | 20260505 09:16:55.167 | INFO | "huge" : "", | 20260505 09:16:55.167 | INFO | "huge_shared" : " 27 GB", | 20260505 09:16:55.167 | INFO | "malloc" : " 14 GB", | 20260505 09:16:55.167 | INFO | "pinned" : " 29 GB" | 20260505 09:16:55.167 | INFO | } | 20260505 09:16:55.167 | INFO | Perf report for the last 5.13915 seconds: | 20260505 09:16:55.167 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:16:55.167 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:55.167 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:55.167 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:55.167 | INFO | ANALYSIS : 65.2%, 2.9, 100.1ms, 0 | 20260505 09:16:55.167 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:16:55.167 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 09:16:55.167 | INFO | PRE_HQ : 62.8%, 1.8, 60.2ms, 0 | 20260505 09:16:55.167 | INFO | BAZWRITER : 66.3%, 4.1, 141.3ms, 0 | 20260505 09:16:55.167 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 09:16:55.168 | INFO | Uploaded 36GiB to GPU in 2.23258s (16.1249GiB/s) | 20260505 09:16:55.168 | INFO | Downloaded 38GiB to GPU in 2.34538s (16.202GiB/s) | 20260505 09:17:00.307 | INFO | Analyzing chunk frames = [2064896,2065408) | 20260505 09:17:00.307 | INFO | MemInfo: { | 20260505 09:17:00.307 | INFO | "gpu" : " 17 GB", | 20260505 09:17:00.309 | INFO | "huge" : "", | 20260505 09:17:00.309 | INFO | "huge_shared" : " 27 GB", | 20260505 09:17:00.309 | INFO | "malloc" : " 14 GB", | 20260505 09:17:00.309 | INFO | "pinned" : " 29 GB" | 20260505 09:17:00.309 | INFO | } | 20260505 09:17:00.310 | INFO | Perf report for the last 5.14204 seconds: | 20260505 09:17:00.310 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:17:00.310 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:00.310 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:00.310 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:00.310 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 09:17:00.310 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:00.310 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:00.310 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:17:00.310 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:00.310 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:00.310 | INFO | Uploaded 36GiB to GPU in 2.18897s (16.4461GiB/s) | 20260505 09:17:00.310 | INFO | Downloaded 34GiB to GPU in 2.10588s (16.1453GiB/s) | 20260505 09:17:05.446 | INFO | Analyzing chunk frames = [2065408,2065920) | 20260505 09:17:05.446 | INFO | MemInfo: { | 20260505 09:17:05.446 | INFO | "gpu" : " 17 GB", | 20260505 09:17:05.446 | INFO | "huge" : "", | 20260505 09:17:05.446 | INFO | "huge_shared" : " 27 GB", | 20260505 09:17:05.446 | INFO | "malloc" : " 14 GB", | 20260505 09:17:05.446 | INFO | "pinned" : " 29 GB" | 20260505 09:17:05.446 | INFO | } | 20260505 09:17:05.446 | INFO | Perf report for the last 5.13631 seconds: | 20260505 09:17:05.446 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:17:05.446 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:05.446 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:05.446 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:05.446 | INFO | ANALYSIS : 63.4%, 3.0, 98.5ms, 0 | 20260505 09:17:05.446 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:05.446 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:05.446 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 09:17:05.446 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:05.446 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:05.446 | INFO | Uploaded 36GiB to GPU in 2.15708s (16.6892GiB/s) | 20260505 09:17:05.446 | INFO | Downloaded 34GiB to GPU in 2.06703s (16.4487GiB/s) | 20260505 09:17:10.586 | INFO | Analyzing chunk frames = [2065920,2066432) | 20260505 09:17:10.586 | INFO | MemInfo: { | 20260505 09:17:10.586 | INFO | "gpu" : " 17 GB", | 20260505 09:17:10.586 | INFO | "huge" : "", | 20260505 09:17:10.586 | INFO | "huge_shared" : " 27 GB", | 20260505 09:17:10.586 | INFO | "malloc" : " 14 GB", | 20260505 09:17:10.586 | INFO | "pinned" : " 29 GB" | 20260505 09:17:10.586 | INFO | } | 20260505 09:17:10.587 | INFO | Perf report for the last 5.14091 seconds: | 20260505 09:17:10.587 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:17:10.587 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:10.587 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:10.587 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:10.587 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 09:17:10.587 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:10.587 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:10.587 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 09:17:10.587 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:10.587 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:10.587 | INFO | Uploaded 36GiB to GPU in 2.16747s (16.6092GiB/s) | 20260505 09:17:10.587 | INFO | Downloaded 34GiB to GPU in 2.06742s (16.4456GiB/s) | 20260505 09:17:15.726 | INFO | Analyzing chunk frames = [2066432,2066944) | 20260505 09:17:15.726 | INFO | MemInfo: { | 20260505 09:17:15.726 | INFO | "gpu" : " 17 GB", | 20260505 09:17:15.726 | INFO | "huge" : "", | 20260505 09:17:15.726 | INFO | "huge_shared" : " 27 GB", | 20260505 09:17:15.726 | INFO | "malloc" : " 14 GB", | 20260505 09:17:15.726 | INFO | "pinned" : " 29 GB" | 20260505 09:17:15.726 | INFO | } | 20260505 09:17:15.726 | INFO | Perf report for the last 5.13949 seconds: | 20260505 09:17:15.726 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:17:15.726 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:15.726 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:15.726 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:15.726 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 09:17:15.726 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:15.726 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:15.726 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260505 09:17:15.726 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:15.726 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:15.727 | INFO | Uploaded 36GiB to GPU in 2.23902s (16.0784GiB/s) | 20260505 09:17:15.727 | INFO | Downloaded 34GiB to GPU in 2.1011s (16.182GiB/s) | 20260505 09:17:20.865 | INFO | Analyzing chunk frames = [2066944,2067456) | 20260505 09:17:20.865 | INFO | MemInfo: { | 20260505 09:17:20.865 | INFO | "gpu" : " 17 GB", | 20260505 09:17:20.865 | INFO | "huge" : "", | 20260505 09:17:20.865 | INFO | "huge_shared" : " 27 GB", | 20260505 09:17:20.865 | INFO | "malloc" : " 14 GB", | 20260505 09:17:20.865 | INFO | "pinned" : " 29 GB" | 20260505 09:17:20.865 | INFO | } | 20260505 09:17:20.865 | INFO | Perf report for the last 5.13895 seconds: | 20260505 09:17:20.865 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:17:20.865 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:20.865 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:20.865 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:20.865 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 09:17:20.865 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:20.865 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:20.865 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 09:17:20.865 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:20.865 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:20.866 | INFO | Uploaded 36GiB to GPU in 2.23906s (16.0781GiB/s) | 20260505 09:17:20.866 | INFO | Downloaded 34GiB to GPU in 2.09522s (16.2274GiB/s) | 20260505 09:17:26.003 | INFO | Analyzing chunk frames = [2067456,2067968) | 20260505 09:17:26.003 | INFO | MemInfo: { | 20260505 09:17:26.003 | INFO | "gpu" : " 17 GB", | 20260505 09:17:26.003 | INFO | "huge" : "", | 20260505 09:17:26.003 | INFO | "huge_shared" : " 27 GB", | 20260505 09:17:26.003 | INFO | "malloc" : " 14 GB", | 20260505 09:17:26.003 | INFO | "pinned" : " 29 GB" | 20260505 09:17:26.003 | INFO | } | 20260505 09:17:26.003 | INFO | Perf report for the last 5.13802 seconds: | 20260505 09:17:26.003 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:17:26.003 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:26.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:26.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:26.003 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 09:17:26.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:26.003 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:26.003 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 09:17:26.003 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:26.003 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:26.004 | INFO | Uploaded 36GiB to GPU in 2.20338s (16.3385GiB/s) | 20260505 09:17:26.004 | INFO | Downloaded 34GiB to GPU in 2.1336s (15.9355GiB/s) | 20260505 09:17:31.142 | INFO | Analyzing chunk frames = [2067968,2068480) | 20260505 09:17:31.142 | INFO | MemInfo: { | 20260505 09:17:31.142 | INFO | "gpu" : " 17 GB", | 20260505 09:17:31.142 | INFO | "huge" : "", | 20260505 09:17:31.142 | INFO | "huge_shared" : " 27 GB", | 20260505 09:17:31.142 | INFO | "malloc" : " 14 GB", | 20260505 09:17:31.142 | INFO | "pinned" : " 29 GB" | 20260505 09:17:31.142 | INFO | } | 20260505 09:17:31.143 | INFO | Perf report for the last 5.13925 seconds: | 20260505 09:17:31.143 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:17:31.143 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:31.143 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:31.143 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:31.143 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 09:17:31.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:31.143 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:31.143 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 09:17:31.143 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:31.143 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:31.143 | INFO | Uploaded 36GiB to GPU in 2.2084s (16.3014GiB/s) | 20260505 09:17:31.143 | INFO | Downloaded 34GiB to GPU in 2.05474s (16.5471GiB/s) | 20260505 09:17:31.279 | INFO | Starting a round of Baz IO | 20260505 09:17:34.551 | INFO | Populating RTMetrics report | 20260505 09:17:34.552 | INFO | RTMetrics report populated | 20260505 09:17:34.552 | INFO | Opening tmp RTMetrics json file | 20260505 09:17:34.565 | INFO | Serializing RTMetrics to json | 20260505 09:17:34.586 | INFO | Serialization done, swapping tmp file now... | 20260505 09:17:34.602 | INFO | RTMetrics done | 20260505 09:17:34.717 | INFO | Finished a round of Baz IO: Wrote 2.444 GiB in 3.437 seconds | 20260505 09:17:34.717 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 26.30% / 0.02% / 73.68% | 20260505 09:17:34.717 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.0818% / 2.38419e-05% / 7.94729e-05% | 20260505 09:17:34.717 | INFO | Baz IO overall: | 20260505 09:17:34.717 | INFO | BytesWritten : 1190632205771 | 20260505 09:17:34.717 | INFO | NumEvents : 566516248467 | 20260505 09:17:36.283 | INFO | Analyzing chunk frames = [2068480,2068992) | 20260505 09:17:36.283 | INFO | MemInfo: { | 20260505 09:17:36.283 | INFO | "gpu" : " 17 GB", | 20260505 09:17:36.283 | INFO | "huge" : "", | 20260505 09:17:36.283 | INFO | "huge_shared" : " 27 GB", | 20260505 09:17:36.283 | INFO | "malloc" : " 14 GB", | 20260505 09:17:36.283 | INFO | "pinned" : " 29 GB" | 20260505 09:17:36.283 | INFO | } | 20260505 09:17:36.284 | INFO | Perf report for the last 5.14097 seconds: | 20260505 09:17:36.284 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:17:36.284 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:36.284 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:36.284 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:36.284 | INFO | ANALYSIS : 65.2%, 2.9, 99.9ms, 0 | 20260505 09:17:36.284 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:36.284 | INFO | RT_METRICS : 46.6%, 1.0, 24.5ms, 0 | 20260505 09:17:36.284 | INFO | PRE_HQ : 63.1%, 1.8, 60.2ms, 0 | 20260505 09:17:36.284 | INFO | BAZWRITER : 66.2%, 4.2, 141.7ms, 0 | 20260505 09:17:36.284 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 09:17:36.284 | INFO | Uploaded 36GiB to GPU in 2.26006s (15.9288GiB/s) | 20260505 09:17:36.284 | INFO | Downloaded 38GiB to GPU in 2.37196s (16.0205GiB/s) | 20260505 09:17:41.422 | INFO | Analyzing chunk frames = [2068992,2069504) | 20260505 09:17:41.422 | INFO | MemInfo: { | 20260505 09:17:41.422 | INFO | "gpu" : " 17 GB", | 20260505 09:17:41.422 | INFO | "huge" : "", | 20260505 09:17:41.422 | INFO | "huge_shared" : " 27 GB", | 20260505 09:17:41.422 | INFO | "malloc" : " 14 GB", | 20260505 09:17:41.422 | INFO | "pinned" : " 29 GB" | 20260505 09:17:41.423 | INFO | } | 20260505 09:17:41.423 | INFO | Perf report for the last 5.13906 seconds: | 20260505 09:17:41.423 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:17:41.423 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:41.423 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:41.423 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:41.423 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 09:17:41.423 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:41.423 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:41.423 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 09:17:41.423 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:41.423 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:41.423 | INFO | Uploaded 36GiB to GPU in 2.19579s (16.395GiB/s) | 20260505 09:17:41.423 | INFO | Downloaded 34GiB to GPU in 2.1075s (16.1328GiB/s) | 20260505 09:17:46.562 | INFO | Analyzing chunk frames = [2069504,2070016) | 20260505 09:17:46.562 | INFO | MemInfo: { | 20260505 09:17:46.562 | INFO | "gpu" : " 17 GB", | 20260505 09:17:46.562 | INFO | "huge" : "", | 20260505 09:17:46.562 | INFO | "huge_shared" : " 27 GB", | 20260505 09:17:46.562 | INFO | "malloc" : " 14 GB", | 20260505 09:17:46.562 | INFO | "pinned" : " 29 GB" | 20260505 09:17:46.562 | INFO | } | 20260505 09:17:46.562 | INFO | Perf report for the last 5.13985 seconds: | 20260505 09:17:46.562 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:17:46.562 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:46.562 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:46.563 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:46.563 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 09:17:46.563 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:46.563 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:46.563 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 09:17:46.563 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:46.563 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:46.563 | INFO | Uploaded 36GiB to GPU in 2.16479s (16.6298GiB/s) | 20260505 09:17:46.563 | INFO | Downloaded 34GiB to GPU in 2.0683s (16.4386GiB/s) | 20260505 09:17:51.700 | INFO | Analyzing chunk frames = [2070016,2070528) | 20260505 09:17:51.700 | INFO | MemInfo: { | 20260505 09:17:51.700 | INFO | "gpu" : " 17 GB", | 20260505 09:17:51.700 | INFO | "huge" : "", | 20260505 09:17:51.700 | INFO | "huge_shared" : " 27 GB", | 20260505 09:17:51.700 | INFO | "malloc" : " 14 GB", | 20260505 09:17:51.700 | INFO | "pinned" : " 29 GB" | 20260505 09:17:51.700 | INFO | } | 20260505 09:17:51.700 | INFO | Perf report for the last 5.13787 seconds: | 20260505 09:17:51.700 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:17:51.700 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:51.700 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:51.701 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:51.701 | INFO | ANALYSIS : 63.2%, 3.0, 98.1ms, 0 | 20260505 09:17:51.701 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:51.701 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:51.701 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 09:17:51.701 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:51.701 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:51.701 | INFO | Uploaded 36GiB to GPU in 2.15239s (16.7256GiB/s) | 20260505 09:17:51.701 | INFO | Downloaded 34GiB to GPU in 2.07165s (16.412GiB/s) | 20260505 09:17:56.840 | INFO | Analyzing chunk frames = [2070528,2071040) | 20260505 09:17:56.840 | INFO | MemInfo: { | 20260505 09:17:56.840 | INFO | "gpu" : " 17 GB", | 20260505 09:17:56.840 | INFO | "huge" : "", | 20260505 09:17:56.840 | INFO | "huge_shared" : " 27 GB", | 20260505 09:17:56.840 | INFO | "malloc" : " 14 GB", | 20260505 09:17:56.840 | INFO | "pinned" : " 29 GB" | 20260505 09:17:56.840 | INFO | } | 20260505 09:17:56.840 | INFO | Perf report for the last 5.13997 seconds: | 20260505 09:17:56.840 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:17:56.840 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:56.840 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:56.840 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:56.840 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 09:17:56.840 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:56.840 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:56.840 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 09:17:56.840 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:56.840 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:17:56.841 | INFO | Uploaded 36GiB to GPU in 2.2335s (16.1182GiB/s) | 20260505 09:17:56.841 | INFO | Downloaded 34GiB to GPU in 2.10308s (16.1668GiB/s) | 20260505 09:18:01.980 | INFO | Analyzing chunk frames = [2071040,2071552) | 20260505 09:18:01.980 | INFO | MemInfo: { | 20260505 09:18:01.980 | INFO | "gpu" : " 17 GB", | 20260505 09:18:01.980 | INFO | "huge" : "", | 20260505 09:18:01.980 | INFO | "huge_shared" : " 27 GB", | 20260505 09:18:01.980 | INFO | "malloc" : " 14 GB", | 20260505 09:18:01.980 | INFO | "pinned" : " 29 GB" | 20260505 09:18:01.980 | INFO | } | 20260505 09:18:01.980 | INFO | Perf report for the last 5.13935 seconds: | 20260505 09:18:01.980 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:18:01.980 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:01.980 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:01.980 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:01.980 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 09:18:01.980 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:01.980 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:01.980 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 09:18:01.980 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:01.980 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:01.980 | INFO | Uploaded 36GiB to GPU in 2.16359s (16.639GiB/s) | 20260505 09:18:01.980 | INFO | Downloaded 34GiB to GPU in 2.05758s (16.5243GiB/s) | 20260505 09:18:07.118 | INFO | Analyzing chunk frames = [2071552,2072064) | 20260505 09:18:07.118 | INFO | MemInfo: { | 20260505 09:18:07.118 | INFO | "gpu" : " 17 GB", | 20260505 09:18:07.118 | INFO | "huge" : "", | 20260505 09:18:07.118 | INFO | "huge_shared" : " 27 GB", | 20260505 09:18:07.118 | INFO | "malloc" : " 14 GB", | 20260505 09:18:07.118 | INFO | "pinned" : " 29 GB" | 20260505 09:18:07.118 | INFO | } | 20260505 09:18:07.118 | INFO | Perf report for the last 5.13856 seconds: | 20260505 09:18:07.118 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:18:07.118 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:07.118 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:07.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:07.118 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 09:18:07.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:07.118 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:07.118 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 09:18:07.118 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:07.118 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:07.119 | INFO | Uploaded 36GiB to GPU in 2.20227s (16.3468GiB/s) | 20260505 09:18:07.119 | INFO | Downloaded 34GiB to GPU in 2.14073s (15.8824GiB/s) | 20260505 09:18:12.258 | INFO | Analyzing chunk frames = [2072064,2072576) | 20260505 09:18:12.258 | INFO | MemInfo: { | 20260505 09:18:12.258 | INFO | "gpu" : " 17 GB", | 20260505 09:18:12.258 | INFO | "huge" : "", | 20260505 09:18:12.258 | INFO | "huge_shared" : " 27 GB", | 20260505 09:18:12.258 | INFO | "malloc" : " 14 GB", | 20260505 09:18:12.258 | INFO | "pinned" : " 29 GB" | 20260505 09:18:12.258 | INFO | } | 20260505 09:18:12.258 | INFO | Perf report for the last 5.13999 seconds: | 20260505 09:18:12.258 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:18:12.258 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:12.258 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:12.258 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:12.258 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 09:18:12.258 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:12.258 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:12.258 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 09:18:12.258 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:12.258 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:12.259 | INFO | Uploaded 36GiB to GPU in 2.20164s (16.3514GiB/s) | 20260505 09:18:12.259 | INFO | Downloaded 34GiB to GPU in 2.04777s (16.6034GiB/s) | 20260505 09:18:12.259 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 09:18:12.259 | INFO | Total mem usage: 15.3 GiB | 20260505 09:18:12.259 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 09:18:12.259 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.54%) | 20260505 09:18:12.259 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 09:18:12.259 | INFO | High Water Breakdown: | 20260505 09:18:12.259 | INFO | 60.9% : 8.76 GiB ( 93.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 09:18:12.259 | INFO | 39% : 5.6 GiB ( 84.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 09:18:12.259 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 09:18:12.259 | INFO | ------------------------------------------ | 20260505 09:18:12.259 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:18:12.259 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 09:18:12.259 | INFO | Total mem usage: 26.5 GiB | 20260505 09:18:12.259 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 09:18:12.259 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 09:18:12.259 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 09:18:12.259 | INFO | High Water Breakdown: | 20260505 09:18:12.259 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:18:12.259 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:18:12.259 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:18:12.259 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:18:12.259 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:18:12.259 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:18:12.259 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 09:18:12.259 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 09:18:12.259 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:18:12.259 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 09:18:12.259 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 09:18:12.259 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 09:18:12.259 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 09:18:12.259 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 09:18:12.259 | INFO | ------------------------------------------ | 20260505 09:18:12.259 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:18:12.259 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 09:18:12.259 | INFO | Total mem usage: 25.6 GiB | 20260505 09:18:12.259 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 09:18:12.259 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 09:18:12.259 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 09:18:12.259 | INFO | High Water Breakdown: | 20260505 09:18:12.259 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 09:18:12.259 | INFO | ------------------------------------------ | 20260505 09:18:12.259 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:18:12.259 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 09:18:12.259 | INFO | Total mem usage: 15.7 GiB | 20260505 09:18:12.259 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 09:18:12.259 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 09:18:12.259 | INFO | Unused Cache: 880 MiB (Waste: 5.48%) | 20260505 09:18:12.259 | INFO | High Water Breakdown: | 20260505 09:18:12.259 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 09:18:12.259 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 09:18:12.259 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 09:18:12.259 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 09:18:12.259 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 09:18:12.259 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 09:18:12.259 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 09:18:12.260 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 09:18:12.260 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 09:18:12.260 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:18:12.260 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 09:18:12.260 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:18:12.260 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 09:18:12.260 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 09:18:12.260 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 09:18:12.260 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 09:18:12.260 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:18:12.260 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 09:18:12.260 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:18:12.260 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:18:12.260 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:18:12.260 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 09:18:12.260 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 09:18:12.260 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:18:12.260 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 09:18:12.260 | INFO | ------------------------------------------ | 20260505 09:18:12.260 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:18:12.260 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 09:18:12.260 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:18:12.260 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:18:12.260 | INFO | PerfReport: Full Execution -- 3.25e+01 / 7.62e+01, 1.35e+01 / 2.20e+00 | 20260505 09:18:12.260 | INFO | PerfReport: 0.29%, AggregateStats -- 9.33e-02 / 1.04e-01, 1.11e-02 / 5.67e-03 | 20260505 09:18:12.260 | INFO | PerfReport: 11.65%, Baseline -- 3.78e+00 / 4.36e+00, 6.19e-01 / 3.66e-01 | 20260505 09:18:12.260 | INFO | PerfReport: 3.31%, Histogram -- 1.08e+00 / 1.46e+00, 2.08e-01 / 1.12e-01 | 20260505 09:18:12.260 | INFO | PerfReport: 15.52%, DME -- 4.03e+01 / 4.41e+01, 4.07e+00 / 2.10e+00 | 20260505 09:18:12.260 | INFO | PerfReport: 56.58%, FrameLabeling -- 1.84e+01 / 2.11e+01, 1.88e+00 / 1.85e+00 | 20260505 09:18:12.260 | INFO | PerfReport: 7.43%, PulseAccumulating -- 2.41e+00 / 2.99e+00, 4.06e-01 / 1.99e-01 | 20260505 09:18:12.260 | INFO | PerfReport: 3.85%, Metrics -- 1.25e+00 / 1.58e+00, 3.59e-01 / 2.02e-01 | 20260505 09:18:12.260 | INFO | PerfReport: 1.37%, PulseSerializing -- 4.45e-01 / 4.91e-01, 8.32e-02 / 2.98e-02 | 20260505 09:18:12.260 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 09:18:12.260 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:18:12.260 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:18:12.260 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.34e+01 / 0.00e+00 | 20260505 09:18:12.260 | INFO | PerfReport: 22.57%, Upload -- 2.29e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260505 09:18:12.260 | INFO | PerfReport: 21.76%, Download -- 2.21e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260505 09:18:12.260 | INFO | PerfReport: 31.99%, Compute -- 3.25e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 09:18:12.260 | INFO | PerfReport: 1.75%, LockUpload -- 1.78e+00 / 0.00e+00, 7.42e+00 / 0.00e+00 | 20260505 09:18:12.260 | INFO | PerfReport: 1.10%, LockDownload -- 1.12e+00 / 0.00e+00, 2.67e+00 / 0.00e+00 | 20260505 09:18:12.260 | INFO | PerfReport: 20.81%, LockCompute -- 2.11e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260505 09:18:12.260 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.23e-02 / 0.00e+00 | 20260505 09:18:12.389 | INFO | Starting a round of Baz IO | 20260505 09:18:15.651 | INFO | Populating RTMetrics report | 20260505 09:18:15.652 | INFO | RTMetrics report populated | 20260505 09:18:15.652 | INFO | Opening tmp RTMetrics json file | 20260505 09:18:15.656 | INFO | Serializing RTMetrics to json | 20260505 09:18:15.658 | INFO | Serialization done, swapping tmp file now... | 20260505 09:18:15.658 | INFO | RTMetrics done | 20260505 09:18:15.799 | INFO | Finished a round of Baz IO: Wrote 2.441 GiB in 3.41 seconds | 20260505 09:18:15.799 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 26.29% / 0.02% / 73.70% | 20260505 09:18:15.799 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.06409% / 1.98682e-05% / 9.93411e-05% | 20260505 09:18:15.799 | INFO | Baz IO overall: | 20260505 09:18:15.799 | INFO | BytesWritten : 1193252817742 | 20260505 09:18:15.799 | INFO | NumEvents : 567768539493 | 20260505 09:18:17.398 | INFO | Analyzing chunk frames = [2072576,2073088) | 20260505 09:18:17.398 | INFO | MemInfo: { | 20260505 09:18:17.398 | INFO | "gpu" : " 17 GB", | 20260505 09:18:17.398 | INFO | "huge" : "", | 20260505 09:18:17.398 | INFO | "huge_shared" : " 27 GB", | 20260505 09:18:17.398 | INFO | "malloc" : " 14 GB", | 20260505 09:18:17.398 | INFO | "pinned" : " 29 GB" | 20260505 09:18:17.398 | INFO | } | 20260505 09:18:17.398 | INFO | Perf report for the last 5.14016 seconds: | 20260505 09:18:17.399 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:18:17.399 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:17.399 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:17.399 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:17.399 | INFO | ANALYSIS : 64.9%, 2.9, 98.7ms, 0 | 20260505 09:18:17.399 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:17.399 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 09:18:17.399 | INFO | PRE_HQ : 63.0%, 1.8, 60.5ms, 0 | 20260505 09:18:17.399 | INFO | BAZWRITER : 65.6%, 4.3, 143.6ms, 0 | 20260505 09:18:17.399 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 09:18:17.399 | INFO | Uploaded 36GiB to GPU in 2.24741s (16.0185GiB/s) | 20260505 09:18:17.399 | INFO | Downloaded 38GiB to GPU in 2.3741s (16.006GiB/s) | 20260505 09:18:22.537 | INFO | Analyzing chunk frames = [2073088,2073600) | 20260505 09:18:22.537 | INFO | MemInfo: { | 20260505 09:18:22.537 | INFO | "gpu" : " 17 GB", | 20260505 09:18:22.537 | INFO | "huge" : "", | 20260505 09:18:22.537 | INFO | "huge_shared" : " 27 GB", | 20260505 09:18:22.537 | INFO | "malloc" : " 14 GB", | 20260505 09:18:22.537 | INFO | "pinned" : " 29 GB" | 20260505 09:18:22.537 | INFO | } | 20260505 09:18:22.537 | INFO | Perf report for the last 5.13865 seconds: | 20260505 09:18:22.537 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:18:22.537 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:22.537 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:22.537 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:22.537 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 09:18:22.537 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:22.537 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:22.537 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 09:18:22.537 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:22.537 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:22.537 | INFO | Uploaded 36GiB to GPU in 2.21582s (16.2468GiB/s) | 20260505 09:18:22.537 | INFO | Downloaded 34GiB to GPU in 2.11438s (16.0804GiB/s) | 20260505 09:18:27.676 | INFO | Analyzing chunk frames = [2073600,2074112) | 20260505 09:18:27.676 | INFO | MemInfo: { | 20260505 09:18:27.676 | INFO | "gpu" : " 17 GB", | 20260505 09:18:27.676 | INFO | "huge" : "", | 20260505 09:18:27.676 | INFO | "huge_shared" : " 27 GB", | 20260505 09:18:27.676 | INFO | "malloc" : " 14 GB", | 20260505 09:18:27.676 | INFO | "pinned" : " 29 GB" | 20260505 09:18:27.676 | INFO | } | 20260505 09:18:27.676 | INFO | Perf report for the last 5.13934 seconds: | 20260505 09:18:27.677 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:18:27.677 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:27.677 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:27.677 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:27.677 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 09:18:27.677 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:27.677 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:27.677 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260505 09:18:27.677 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:27.677 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:27.677 | INFO | Uploaded 36GiB to GPU in 2.1543s (16.7108GiB/s) | 20260505 09:18:27.677 | INFO | Downloaded 34GiB to GPU in 2.06966s (16.4278GiB/s) | 20260505 09:18:32.816 | INFO | Analyzing chunk frames = [2074112,2074624) | 20260505 09:18:32.816 | INFO | MemInfo: { | 20260505 09:18:32.816 | INFO | "gpu" : " 17 GB", | 20260505 09:18:32.816 | INFO | "huge" : "", | 20260505 09:18:32.816 | INFO | "huge_shared" : " 27 GB", | 20260505 09:18:32.816 | INFO | "malloc" : " 14 GB", | 20260505 09:18:32.816 | INFO | "pinned" : " 29 GB" | 20260505 09:18:32.816 | INFO | } | 20260505 09:18:32.816 | INFO | Perf report for the last 5.13942 seconds: | 20260505 09:18:32.816 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:18:32.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:32.816 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:32.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:32.816 | INFO | ANALYSIS : 63.3%, 3.0, 98.3ms, 0 | 20260505 09:18:32.816 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:32.816 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:32.816 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:18:32.816 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:32.816 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:32.816 | INFO | Uploaded 36GiB to GPU in 2.16178s (16.6529GiB/s) | 20260505 09:18:32.816 | INFO | Downloaded 34GiB to GPU in 2.07472s (16.3878GiB/s) | 20260505 09:18:37.954 | INFO | Analyzing chunk frames = [2074624,2075136) | 20260505 09:18:37.954 | INFO | MemInfo: { | 20260505 09:18:37.954 | INFO | "gpu" : " 17 GB", | 20260505 09:18:37.954 | INFO | "huge" : "", | 20260505 09:18:37.954 | INFO | "huge_shared" : " 27 GB", | 20260505 09:18:37.954 | INFO | "malloc" : " 14 GB", | 20260505 09:18:37.954 | INFO | "pinned" : " 29 GB" | 20260505 09:18:37.954 | INFO | } | 20260505 09:18:37.954 | INFO | Perf report for the last 5.13861 seconds: | 20260505 09:18:37.954 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:18:37.954 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:37.955 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:37.955 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:37.955 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 09:18:37.955 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:37.955 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:37.955 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:18:37.955 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:37.955 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:37.955 | INFO | Uploaded 36GiB to GPU in 2.23755s (16.089GiB/s) | 20260505 09:18:37.955 | INFO | Downloaded 34GiB to GPU in 2.1013s (16.1805GiB/s) | 20260505 09:18:43.094 | INFO | Analyzing chunk frames = [2075136,2075648) | 20260505 09:18:43.095 | INFO | MemInfo: { | 20260505 09:18:43.095 | INFO | "gpu" : " 17 GB", | 20260505 09:18:43.095 | INFO | "huge" : "", | 20260505 09:18:43.095 | INFO | "huge_shared" : " 27 GB", | 20260505 09:18:43.095 | INFO | "malloc" : " 14 GB", | 20260505 09:18:43.095 | INFO | "pinned" : " 29 GB" | 20260505 09:18:43.095 | INFO | } | 20260505 09:18:43.095 | INFO | Perf report for the last 5.14012 seconds: | 20260505 09:18:43.095 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:18:43.095 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:43.095 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:43.095 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:43.095 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 09:18:43.095 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:43.095 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:43.095 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 09:18:43.095 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:43.095 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:43.095 | INFO | Uploaded 36GiB to GPU in 2.16461s (16.6312GiB/s) | 20260505 09:18:43.095 | INFO | Downloaded 34GiB to GPU in 2.06238s (16.4858GiB/s) | 20260505 09:18:48.234 | INFO | Analyzing chunk frames = [2075648,2076160) | 20260505 09:18:48.234 | INFO | MemInfo: { | 20260505 09:18:48.234 | INFO | "gpu" : " 17 GB", | 20260505 09:18:48.234 | INFO | "huge" : "", | 20260505 09:18:48.234 | INFO | "huge_shared" : " 27 GB", | 20260505 09:18:48.234 | INFO | "malloc" : " 14 GB", | 20260505 09:18:48.234 | INFO | "pinned" : " 29 GB" | 20260505 09:18:48.234 | INFO | } | 20260505 09:18:48.234 | INFO | Perf report for the last 5.13979 seconds: | 20260505 09:18:48.234 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:18:48.234 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:48.234 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:48.234 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:48.234 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 09:18:48.234 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:48.234 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:48.235 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 09:18:48.235 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:48.235 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:48.235 | INFO | Uploaded 36GiB to GPU in 2.18789s (16.4542GiB/s) | 20260505 09:18:48.235 | INFO | Downloaded 34GiB to GPU in 2.14406s (15.8578GiB/s) | 20260505 09:18:53.375 | INFO | Analyzing chunk frames = [2076160,2076672) | 20260505 09:18:53.375 | INFO | MemInfo: { | 20260505 09:18:53.375 | INFO | "gpu" : " 17 GB", | 20260505 09:18:53.375 | INFO | "huge" : "", | 20260505 09:18:53.375 | INFO | "huge_shared" : " 27 GB", | 20260505 09:18:53.375 | INFO | "malloc" : " 14 GB", | 20260505 09:18:53.375 | INFO | "pinned" : " 29 GB" | 20260505 09:18:53.375 | INFO | } | 20260505 09:18:53.375 | INFO | Perf report for the last 5.14075 seconds: | 20260505 09:18:53.375 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:18:53.375 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:53.375 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:53.375 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:53.375 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 09:18:53.375 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:53.375 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:53.375 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 09:18:53.375 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:53.375 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:53.376 | INFO | Uploaded 36GiB to GPU in 2.2038s (16.3354GiB/s) | 20260505 09:18:53.376 | INFO | Downloaded 34GiB to GPU in 2.04663s (16.6127GiB/s) | 20260505 09:18:53.509 | INFO | Starting a round of Baz IO | 20260505 09:18:56.784 | INFO | Populating RTMetrics report | 20260505 09:18:56.784 | INFO | RTMetrics report populated | 20260505 09:18:56.784 | INFO | Opening tmp RTMetrics json file | 20260505 09:18:56.796 | INFO | Serializing RTMetrics to json | 20260505 09:18:56.800 | INFO | Serialization done, swapping tmp file now... | 20260505 09:18:56.803 | INFO | RTMetrics done | 20260505 09:18:56.922 | INFO | Finished a round of Baz IO: Wrote 2.44 GiB in 3.413 seconds | 20260505 09:18:56.922 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 26.27% / 0.02% / 73.71% | 20260505 09:18:56.923 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.05763% / 1.98682e-05% / 9.53674e-05% | 20260505 09:18:56.923 | INFO | Baz IO overall: | 20260505 09:18:56.923 | INFO | BytesWritten : 1195873079486 | 20260505 09:18:56.923 | INFO | NumEvents : 569020725869 | 20260505 09:18:58.513 | INFO | Analyzing chunk frames = [2076672,2077184) | 20260505 09:18:58.513 | INFO | MemInfo: { | 20260505 09:18:58.513 | INFO | "gpu" : " 17 GB", | 20260505 09:18:58.513 | INFO | "huge" : "", | 20260505 09:18:58.513 | INFO | "huge_shared" : " 27 GB", | 20260505 09:18:58.513 | INFO | "malloc" : " 14 GB", | 20260505 09:18:58.513 | INFO | "pinned" : " 29 GB" | 20260505 09:18:58.513 | INFO | } | 20260505 09:18:58.513 | INFO | Perf report for the last 5.13774 seconds: | 20260505 09:18:58.514 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:18:58.514 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:58.514 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:58.514 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:58.514 | INFO | ANALYSIS : 65.2%, 2.9, 100.3ms, 0 | 20260505 09:18:58.514 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:18:58.514 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260505 09:18:58.514 | INFO | PRE_HQ : 62.9%, 1.8, 60.4ms, 0 | 20260505 09:18:58.514 | INFO | BAZWRITER : 65.7%, 4.1, 139.1ms, 0 | 20260505 09:18:58.514 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 09:18:58.515 | INFO | Uploaded 36GiB to GPU in 2.22072s (16.2109GiB/s) | 20260505 09:18:58.515 | INFO | Downloaded 38GiB to GPU in 2.34275s (16.2202GiB/s) | 20260505 09:19:03.652 | INFO | Analyzing chunk frames = [2077184,2077696) | 20260505 09:19:03.652 | INFO | MemInfo: { | 20260505 09:19:03.652 | INFO | "gpu" : " 17 GB", | 20260505 09:19:03.652 | INFO | "huge" : "", | 20260505 09:19:03.652 | INFO | "huge_shared" : " 27 GB", | 20260505 09:19:03.652 | INFO | "malloc" : " 14 GB", | 20260505 09:19:03.652 | INFO | "pinned" : " 29 GB" | 20260505 09:19:03.652 | INFO | } | 20260505 09:19:03.652 | INFO | Perf report for the last 5.13926 seconds: | 20260505 09:19:03.652 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:19:03.652 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:03.652 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:03.652 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:03.652 | INFO | ANALYSIS : 64.5%, 3.0, 100.7ms, 0 | 20260505 09:19:03.652 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:03.652 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:03.652 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 09:19:03.652 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:03.652 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:03.652 | INFO | Uploaded 36GiB to GPU in 2.21247s (16.2714GiB/s) | 20260505 09:19:03.652 | INFO | Downloaded 34GiB to GPU in 2.1017s (16.1774GiB/s) | 20260505 09:19:08.791 | INFO | Analyzing chunk frames = [2077696,2078208) | 20260505 09:19:08.791 | INFO | MemInfo: { | 20260505 09:19:08.791 | INFO | "gpu" : " 17 GB", | 20260505 09:19:08.791 | INFO | "huge" : "", | 20260505 09:19:08.791 | INFO | "huge_shared" : " 27 GB", | 20260505 09:19:08.791 | INFO | "malloc" : " 14 GB", | 20260505 09:19:08.791 | INFO | "pinned" : " 29 GB" | 20260505 09:19:08.791 | INFO | } | 20260505 09:19:08.791 | INFO | Perf report for the last 5.13887 seconds: | 20260505 09:19:08.791 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:19:08.791 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:08.791 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:08.791 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:08.791 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 09:19:08.791 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:08.791 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:08.791 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 09:19:08.791 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:08.791 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:08.791 | INFO | Uploaded 36GiB to GPU in 2.15004s (16.7439GiB/s) | 20260505 09:19:08.792 | INFO | Downloaded 34GiB to GPU in 2.0664s (16.4537GiB/s) | 20260505 09:19:13.931 | INFO | Analyzing chunk frames = [2078208,2078720) | 20260505 09:19:13.931 | INFO | MemInfo: { | 20260505 09:19:13.931 | INFO | "gpu" : " 17 GB", | 20260505 09:19:13.931 | INFO | "huge" : "", | 20260505 09:19:13.931 | INFO | "huge_shared" : " 27 GB", | 20260505 09:19:13.931 | INFO | "malloc" : " 14 GB", | 20260505 09:19:13.931 | INFO | "pinned" : " 29 GB" | 20260505 09:19:13.931 | INFO | } | 20260505 09:19:13.931 | INFO | Perf report for the last 5.1398 seconds: | 20260505 09:19:13.931 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:19:13.931 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:13.931 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:13.931 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:13.931 | INFO | ANALYSIS : 63.3%, 3.0, 98.3ms, 0 | 20260505 09:19:13.931 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:13.931 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:13.931 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:19:13.931 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:13.931 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:13.931 | INFO | Uploaded 36GiB to GPU in 2.15033s (16.7416GiB/s) | 20260505 09:19:13.931 | INFO | Downloaded 34GiB to GPU in 2.06854s (16.4367GiB/s) | 20260505 09:19:19.069 | INFO | Analyzing chunk frames = [2078720,2079232) | 20260505 09:19:19.069 | INFO | MemInfo: { | 20260505 09:19:19.069 | INFO | "gpu" : " 17 GB", | 20260505 09:19:19.069 | INFO | "huge" : "", | 20260505 09:19:19.069 | INFO | "huge_shared" : " 27 GB", | 20260505 09:19:19.069 | INFO | "malloc" : " 14 GB", | 20260505 09:19:19.069 | INFO | "pinned" : " 29 GB" | 20260505 09:19:19.069 | INFO | } | 20260505 09:19:19.069 | INFO | Perf report for the last 5.13852 seconds: | 20260505 09:19:19.069 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:19:19.069 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:19.069 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:19.069 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:19.069 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 09:19:19.069 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:19.069 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:19.069 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:19:19.069 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:19.070 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:19.070 | INFO | Uploaded 36GiB to GPU in 2.23539s (16.1046GiB/s) | 20260505 09:19:19.070 | INFO | Downloaded 34GiB to GPU in 2.09813s (16.2049GiB/s) | 20260505 09:19:24.210 | INFO | Analyzing chunk frames = [2079232,2079744) | 20260505 09:19:24.210 | INFO | MemInfo: { | 20260505 09:19:24.210 | INFO | "gpu" : " 17 GB", | 20260505 09:19:24.210 | INFO | "huge" : "", | 20260505 09:19:24.210 | INFO | "huge_shared" : " 27 GB", | 20260505 09:19:24.210 | INFO | "malloc" : " 14 GB", | 20260505 09:19:24.210 | INFO | "pinned" : " 29 GB" | 20260505 09:19:24.210 | INFO | } | 20260505 09:19:24.210 | INFO | Perf report for the last 5.14067 seconds: | 20260505 09:19:24.210 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:19:24.210 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:24.210 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:24.210 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:24.210 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 09:19:24.210 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:24.210 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:24.210 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:19:24.210 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:24.210 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:24.210 | INFO | Uploaded 36GiB to GPU in 2.15939s (16.6714GiB/s) | 20260505 09:19:24.211 | INFO | Downloaded 34GiB to GPU in 2.05845s (16.5173GiB/s) | 20260505 09:19:29.351 | INFO | Analyzing chunk frames = [2079744,2080256) | 20260505 09:19:29.351 | INFO | MemInfo: { | 20260505 09:19:29.351 | INFO | "gpu" : " 17 GB", | 20260505 09:19:29.351 | INFO | "huge" : "", | 20260505 09:19:29.351 | INFO | "huge_shared" : " 27 GB", | 20260505 09:19:29.351 | INFO | "malloc" : " 14 GB", | 20260505 09:19:29.351 | INFO | "pinned" : " 29 GB" | 20260505 09:19:29.351 | INFO | } | 20260505 09:19:29.351 | INFO | Perf report for the last 5.14126 seconds: | 20260505 09:19:29.351 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:19:29.351 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:29.351 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:29.351 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:29.351 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 09:19:29.351 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:29.351 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:29.351 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 09:19:29.351 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:29.351 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:29.352 | INFO | Uploaded 36GiB to GPU in 2.18133s (16.5037GiB/s) | 20260505 09:19:29.352 | INFO | Downloaded 34GiB to GPU in 2.14046s (15.8844GiB/s) | 20260505 09:19:34.491 | INFO | Analyzing chunk frames = [2080256,2080768) | 20260505 09:19:34.491 | INFO | MemInfo: { | 20260505 09:19:34.491 | INFO | "gpu" : " 17 GB", | 20260505 09:19:34.491 | INFO | "huge" : "", | 20260505 09:19:34.491 | INFO | "huge_shared" : " 27 GB", | 20260505 09:19:34.491 | INFO | "malloc" : " 14 GB", | 20260505 09:19:34.491 | INFO | "pinned" : " 29 GB" | 20260505 09:19:34.491 | INFO | } | 20260505 09:19:34.491 | INFO | Perf report for the last 5.13944 seconds: | 20260505 09:19:34.491 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:19:34.491 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:34.491 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:34.491 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:34.491 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 09:19:34.491 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:34.491 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:34.491 | INFO | PRE_HQ : 30.5%, 1.0, 15.9ms, 0 | 20260505 09:19:34.491 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:34.491 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:34.491 | INFO | Uploaded 36GiB to GPU in 2.1798s (16.5153GiB/s) | 20260505 09:19:34.491 | INFO | Downloaded 34GiB to GPU in 2.05391s (16.5538GiB/s) | 20260505 09:19:34.491 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 09:19:34.491 | INFO | Total mem usage: 15.3 GiB | 20260505 09:19:34.491 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 09:19:34.491 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -5%) | 20260505 09:19:34.491 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 09:19:34.491 | INFO | High Water Breakdown: | 20260505 09:19:34.491 | INFO | 61.1% : 8.92 GiB ( 95.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 09:19:34.491 | INFO | 38.7% : 5.65 GiB ( 85.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 09:19:34.491 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 09:19:34.491 | INFO | ------------------------------------------ | 20260505 09:19:34.491 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:19:34.491 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 09:19:34.491 | INFO | Total mem usage: 26.5 GiB | 20260505 09:19:34.491 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 09:19:34.491 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 09:19:34.491 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 09:19:34.491 | INFO | High Water Breakdown: | 20260505 09:19:34.491 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:19:34.491 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:19:34.491 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:19:34.491 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:19:34.491 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:19:34.491 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:19:34.491 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 09:19:34.491 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 09:19:34.491 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:19:34.492 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 09:19:34.492 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 09:19:34.492 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 09:19:34.492 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 09:19:34.492 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 09:19:34.492 | INFO | ------------------------------------------ | 20260505 09:19:34.492 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:19:34.492 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 09:19:34.492 | INFO | Total mem usage: 25.6 GiB | 20260505 09:19:34.492 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 09:19:34.492 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 09:19:34.492 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 09:19:34.492 | INFO | High Water Breakdown: | 20260505 09:19:34.492 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 09:19:34.492 | INFO | ------------------------------------------ | 20260505 09:19:34.492 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:19:34.492 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 09:19:34.492 | INFO | Total mem usage: 15.7 GiB | 20260505 09:19:34.492 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 09:19:34.492 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 09:19:34.492 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 09:19:34.492 | INFO | High Water Breakdown: | 20260505 09:19:34.492 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 09:19:34.492 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 09:19:34.492 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 09:19:34.492 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 09:19:34.492 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 09:19:34.492 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 09:19:34.492 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 09:19:34.492 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 09:19:34.492 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 09:19:34.492 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:19:34.492 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 09:19:34.492 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:19:34.492 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 09:19:34.492 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 09:19:34.492 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 09:19:34.492 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 09:19:34.492 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:19:34.492 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 09:19:34.492 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:19:34.492 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:19:34.492 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:19:34.492 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 09:19:34.492 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 09:19:34.492 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:19:34.492 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 09:19:34.492 | INFO | ------------------------------------------ | 20260505 09:19:34.492 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:19:34.492 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 09:19:34.492 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:19:34.492 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:19:34.492 | INFO | PerfReport: Full Execution -- 3.25e+01 / 7.59e+01, 1.36e+01 / 2.36e+00 | 20260505 09:19:34.492 | INFO | PerfReport: 0.29%, AggregateStats -- 9.34e-02 / 1.04e-01, 1.21e-02 / 2.94e-03 | 20260505 09:19:34.492 | INFO | PerfReport: 11.66%, Baseline -- 3.78e+00 / 4.39e+00, 6.16e-01 / 3.20e-01 | 20260505 09:19:34.492 | INFO | PerfReport: 3.31%, Histogram -- 1.08e+00 / 1.45e+00, 2.08e-01 / 1.26e-01 | 20260505 09:19:34.492 | INFO | PerfReport: 15.58%, DME -- 4.04e+01 / 4.44e+01, 4.19e+00 / 2.04e+00 | 20260505 09:19:34.492 | INFO | PerfReport: 56.50%, FrameLabeling -- 1.83e+01 / 2.04e+01, 1.86e+00 / 1.53e+00 | 20260505 09:19:34.492 | INFO | PerfReport: 7.46%, PulseAccumulating -- 2.42e+00 / 3.00e+00, 4.18e-01 / 2.34e-01 | 20260505 09:19:34.492 | INFO | PerfReport: 3.83%, Metrics -- 1.24e+00 / 1.57e+00, 3.57e-01 / 1.87e-01 | 20260505 09:19:34.492 | INFO | PerfReport: 1.37%, PulseSerializing -- 4.43e-01 / 4.93e-01, 8.38e-02 / 2.62e-02 | 20260505 09:19:34.492 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 09:19:34.492 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:19:34.492 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:19:34.492 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.33e+01 / 0.00e+00 | 20260505 09:19:34.492 | INFO | PerfReport: 22.51%, Upload -- 2.28e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260505 09:19:34.492 | INFO | PerfReport: 21.78%, Download -- 2.21e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260505 09:19:34.492 | INFO | PerfReport: 32.01%, Compute -- 3.25e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 09:19:34.492 | INFO | PerfReport: 1.74%, LockUpload -- 1.76e+00 / 0.00e+00, 7.35e+00 / 0.00e+00 | 20260505 09:19:34.492 | INFO | PerfReport: 1.14%, LockDownload -- 1.16e+00 / 0.00e+00, 2.76e+00 / 0.00e+00 | 20260505 09:19:34.492 | INFO | PerfReport: 20.81%, LockCompute -- 2.11e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260505 09:19:34.492 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.86e-02 / 0.00e+00 | 20260505 09:19:34.623 | INFO | Starting a round of Baz IO | 20260505 09:19:37.884 | INFO | Populating RTMetrics report | 20260505 09:19:37.884 | INFO | RTMetrics report populated | 20260505 09:19:37.884 | INFO | Opening tmp RTMetrics json file | 20260505 09:19:37.916 | INFO | Serializing RTMetrics to json | 20260505 09:19:37.957 | INFO | Serialization done, swapping tmp file now... | 20260505 09:19:37.985 | INFO | RTMetrics done | 20260505 09:19:38.047 | INFO | Finished a round of Baz IO: Wrote 2.442 GiB in 3.424 seconds | 20260505 09:19:38.047 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 26.26% / 0.02% / 73.73% | 20260505 09:19:38.047 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.05337% / 2.78155e-05% / 7.94729e-05% | 20260505 09:19:38.047 | INFO | Baz IO overall: | 20260505 09:19:38.047 | INFO | BytesWritten : 1198495496818 | 20260505 09:19:38.047 | INFO | NumEvents : 570274164874 | 20260505 09:19:39.629 | INFO | Analyzing chunk frames = [2080768,2081280) | 20260505 09:19:39.629 | INFO | MemInfo: { | 20260505 09:19:39.629 | INFO | "gpu" : " 17 GB", | 20260505 09:19:39.629 | INFO | "huge" : "", | 20260505 09:19:39.629 | INFO | "huge_shared" : " 27 GB", | 20260505 09:19:39.629 | INFO | "malloc" : " 14 GB", | 20260505 09:19:39.629 | INFO | "pinned" : " 29 GB" | 20260505 09:19:39.629 | INFO | } | 20260505 09:19:39.629 | INFO | Perf report for the last 5.13866 seconds: | 20260505 09:19:39.630 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:19:39.630 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:39.630 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:39.630 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:39.630 | INFO | ANALYSIS : 64.9%, 2.9, 99.2ms, 0 | 20260505 09:19:39.630 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:39.630 | INFO | RT_METRICS : 47.7%, 1.0, 25.0ms, 0 | 20260505 09:19:39.630 | INFO | PRE_HQ : 63.1%, 1.8, 60.4ms, 0 | 20260505 09:19:39.630 | INFO | BAZWRITER : 65.9%, 4.2, 141.7ms, 0 | 20260505 09:19:39.630 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 09:19:39.630 | INFO | Uploaded 36GiB to GPU in 2.24851s (16.0106GiB/s) | 20260505 09:19:39.630 | INFO | Downloaded 38GiB to GPU in 2.38167s (15.9552GiB/s) | 20260505 09:19:44.765 | INFO | Analyzing chunk frames = [2081280,2081792) | 20260505 09:19:44.765 | INFO | MemInfo: { | 20260505 09:19:44.765 | INFO | "gpu" : " 17 GB", | 20260505 09:19:44.765 | INFO | "huge" : "", | 20260505 09:19:44.765 | INFO | "huge_shared" : " 27 GB", | 20260505 09:19:44.765 | INFO | "malloc" : " 14 GB", | 20260505 09:19:44.765 | INFO | "pinned" : " 29 GB" | 20260505 09:19:44.765 | INFO | } | 20260505 09:19:44.766 | INFO | Perf report for the last 5.1362 seconds: | 20260505 09:19:44.766 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:19:44.766 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:44.766 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:44.766 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:44.766 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 09:19:44.766 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:44.766 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:44.766 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 09:19:44.766 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:44.766 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:44.766 | INFO | Uploaded 36GiB to GPU in 2.16431s (16.6334GiB/s) | 20260505 09:19:44.766 | INFO | Downloaded 34GiB to GPU in 2.09886s (16.1993GiB/s) | 20260505 09:19:49.905 | INFO | Analyzing chunk frames = [2081792,2082304) | 20260505 09:19:49.905 | INFO | MemInfo: { | 20260505 09:19:49.905 | INFO | "gpu" : " 17 GB", | 20260505 09:19:49.905 | INFO | "huge" : "", | 20260505 09:19:49.905 | INFO | "huge_shared" : " 27 GB", | 20260505 09:19:49.905 | INFO | "malloc" : " 14 GB", | 20260505 09:19:49.905 | INFO | "pinned" : " 29 GB" | 20260505 09:19:49.905 | INFO | } | 20260505 09:19:49.905 | INFO | Perf report for the last 5.13941 seconds: | 20260505 09:19:49.905 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:19:49.905 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:49.905 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:49.905 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:49.905 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 09:19:49.905 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:49.905 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:49.905 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:19:49.905 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:49.905 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:49.905 | INFO | Uploaded 36GiB to GPU in 2.1704s (16.5868GiB/s) | 20260505 09:19:49.905 | INFO | Downloaded 34GiB to GPU in 2.07322s (16.3996GiB/s) | 20260505 09:19:55.044 | INFO | Analyzing chunk frames = [2082304,2082816) | 20260505 09:19:55.044 | INFO | MemInfo: { | 20260505 09:19:55.044 | INFO | "gpu" : " 17 GB", | 20260505 09:19:55.044 | INFO | "huge" : "", | 20260505 09:19:55.044 | INFO | "huge_shared" : " 27 GB", | 20260505 09:19:55.044 | INFO | "malloc" : " 14 GB", | 20260505 09:19:55.044 | INFO | "pinned" : " 29 GB" | 20260505 09:19:55.044 | INFO | } | 20260505 09:19:55.045 | INFO | Perf report for the last 5.13953 seconds: | 20260505 09:19:55.045 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:19:55.045 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:55.045 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:55.045 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:55.045 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 09:19:55.045 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:55.045 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:55.045 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:19:55.045 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:55.045 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:19:55.045 | INFO | Uploaded 36GiB to GPU in 2.14964s (16.747GiB/s) | 20260505 09:19:55.045 | INFO | Downloaded 34GiB to GPU in 2.06765s (16.4438GiB/s) | 20260505 09:20:00.184 | INFO | Analyzing chunk frames = [2082816,2083328) | 20260505 09:20:00.184 | INFO | MemInfo: { | 20260505 09:20:00.184 | INFO | "gpu" : " 17 GB", | 20260505 09:20:00.184 | INFO | "huge" : "", | 20260505 09:20:00.184 | INFO | "huge_shared" : " 27 GB", | 20260505 09:20:00.184 | INFO | "malloc" : " 14 GB", | 20260505 09:20:00.184 | INFO | "pinned" : " 29 GB" | 20260505 09:20:00.184 | INFO | } | 20260505 09:20:00.184 | INFO | Perf report for the last 5.13953 seconds: | 20260505 09:20:00.184 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:20:00.184 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:00.184 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:00.184 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:00.184 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 09:20:00.184 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:00.184 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:00.184 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 09:20:00.184 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:00.184 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:00.185 | INFO | Uploaded 36GiB to GPU in 2.23463s (16.1101GiB/s) | 20260505 09:20:00.185 | INFO | Downloaded 34GiB to GPU in 2.10831s (16.1267GiB/s) | 20260505 09:20:05.324 | INFO | Analyzing chunk frames = [2083328,2083840) | 20260505 09:20:05.324 | INFO | MemInfo: { | 20260505 09:20:05.324 | INFO | "gpu" : " 17 GB", | 20260505 09:20:05.324 | INFO | "huge" : "", | 20260505 09:20:05.324 | INFO | "huge_shared" : " 27 GB", | 20260505 09:20:05.324 | INFO | "malloc" : " 14 GB", | 20260505 09:20:05.325 | INFO | "pinned" : " 29 GB" | 20260505 09:20:05.325 | INFO | } | 20260505 09:20:05.325 | INFO | Perf report for the last 5.14047 seconds: | 20260505 09:20:05.325 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:20:05.325 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:05.325 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:05.325 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:05.325 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 09:20:05.325 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:05.325 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:05.325 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:20:05.325 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:05.325 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:05.325 | INFO | Uploaded 36GiB to GPU in 2.16639s (16.6175GiB/s) | 20260505 09:20:05.325 | INFO | Downloaded 34GiB to GPU in 2.05931s (16.5104GiB/s) | 20260505 09:20:10.464 | INFO | Analyzing chunk frames = [2083840,2084352) | 20260505 09:20:10.464 | INFO | MemInfo: { | 20260505 09:20:10.464 | INFO | "gpu" : " 17 GB", | 20260505 09:20:10.464 | INFO | "huge" : "", | 20260505 09:20:10.464 | INFO | "huge_shared" : " 27 GB", | 20260505 09:20:10.464 | INFO | "malloc" : " 14 GB", | 20260505 09:20:10.464 | INFO | "pinned" : " 29 GB" | 20260505 09:20:10.464 | INFO | } | 20260505 09:20:10.464 | INFO | Perf report for the last 5.13917 seconds: | 20260505 09:20:10.464 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:20:10.464 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:10.464 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:10.464 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:10.464 | INFO | ANALYSIS : 63.7%, 3.0, 99.1ms, 0 | 20260505 09:20:10.464 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:10.464 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:10.464 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 09:20:10.464 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:10.464 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:10.464 | INFO | Uploaded 36GiB to GPU in 2.18835s (16.4508GiB/s) | 20260505 09:20:10.464 | INFO | Downloaded 34GiB to GPU in 2.15802s (15.7552GiB/s) | 20260505 09:20:15.602 | INFO | Analyzing chunk frames = [2084352,2084864) | 20260505 09:20:15.602 | INFO | MemInfo: { | 20260505 09:20:15.602 | INFO | "gpu" : " 17 GB", | 20260505 09:20:15.602 | INFO | "huge" : "", | 20260505 09:20:15.602 | INFO | "huge_shared" : " 27 GB", | 20260505 09:20:15.602 | INFO | "malloc" : " 14 GB", | 20260505 09:20:15.602 | INFO | "pinned" : " 29 GB" | 20260505 09:20:15.602 | INFO | } | 20260505 09:20:15.602 | INFO | Perf report for the last 5.13826 seconds: | 20260505 09:20:15.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:20:15.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:15.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:15.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:15.602 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 09:20:15.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:15.602 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:15.602 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 09:20:15.602 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:15.602 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:15.602 | INFO | Uploaded 36GiB to GPU in 2.18045s (16.5103GiB/s) | 20260505 09:20:15.602 | INFO | Downloaded 34GiB to GPU in 2.05011s (16.5844GiB/s) | 20260505 09:20:15.738 | INFO | Starting a round of Baz IO | 20260505 09:20:19.002 | INFO | Populating RTMetrics report | 20260505 09:20:19.003 | INFO | RTMetrics report populated | 20260505 09:20:19.003 | INFO | Opening tmp RTMetrics json file | 20260505 09:20:19.014 | INFO | Serializing RTMetrics to json | 20260505 09:20:19.039 | INFO | Serialization done, swapping tmp file now... | 20260505 09:20:19.052 | INFO | RTMetrics done | 20260505 09:20:19.127 | INFO | Finished a round of Baz IO: Wrote 2.442 GiB in 3.389 seconds | 20260505 09:20:19.127 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 26.24% / 0.02% / 73.74% | 20260505 09:20:19.127 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.04589% / 1.19209e-05% / 5.5631e-05% | 20260505 09:20:19.127 | INFO | Baz IO overall: | 20260505 09:20:19.127 | INFO | BytesWritten : 1201117316516 | 20260505 09:20:19.127 | INFO | NumEvents : 571527213893 | 20260505 09:20:20.741 | INFO | Analyzing chunk frames = [2084864,2085376) | 20260505 09:20:20.741 | INFO | MemInfo: { | 20260505 09:20:20.741 | INFO | "gpu" : " 17 GB", | 20260505 09:20:20.741 | INFO | "huge" : "", | 20260505 09:20:20.741 | INFO | "huge_shared" : " 27 GB", | 20260505 09:20:20.741 | INFO | "malloc" : " 14 GB", | 20260505 09:20:20.741 | INFO | "pinned" : " 29 GB" | 20260505 09:20:20.741 | INFO | } | 20260505 09:20:20.741 | INFO | Perf report for the last 5.13921 seconds: | 20260505 09:20:20.741 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:20:20.741 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:20.741 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:20.741 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:20.741 | INFO | ANALYSIS : 65.0%, 2.9, 99.5ms, 0 | 20260505 09:20:20.741 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:20.741 | INFO | RT_METRICS : 46.7%, 1.0, 24.5ms, 0 | 20260505 09:20:20.742 | INFO | PRE_HQ : 63.2%, 1.8, 60.3ms, 0 | 20260505 09:20:20.742 | INFO | BAZWRITER : 65.2%, 4.1, 138.9ms, 0 | 20260505 09:20:20.742 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 09:20:20.742 | INFO | Uploaded 36GiB to GPU in 2.26112s (15.9213GiB/s) | 20260505 09:20:20.742 | INFO | Downloaded 38GiB to GPU in 2.38468s (15.9351GiB/s) | 20260505 09:20:25.881 | INFO | Analyzing chunk frames = [2085376,2085888) | 20260505 09:20:25.881 | INFO | MemInfo: { | 20260505 09:20:25.881 | INFO | "gpu" : " 17 GB", | 20260505 09:20:25.881 | INFO | "huge" : "", | 20260505 09:20:25.881 | INFO | "huge_shared" : " 27 GB", | 20260505 09:20:25.881 | INFO | "malloc" : " 14 GB", | 20260505 09:20:25.882 | INFO | "pinned" : " 29 GB" | 20260505 09:20:25.882 | INFO | } | 20260505 09:20:25.882 | INFO | Perf report for the last 5.14026 seconds: | 20260505 09:20:25.882 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:20:25.882 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:25.882 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:25.882 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:25.882 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 09:20:25.882 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:25.882 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:25.882 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:20:25.882 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:25.882 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:25.882 | INFO | Uploaded 36GiB to GPU in 2.20732s (16.3093GiB/s) | 20260505 09:20:25.882 | INFO | Downloaded 34GiB to GPU in 2.11708s (16.0598GiB/s) | 20260505 09:20:31.020 | INFO | Analyzing chunk frames = [2085888,2086400) | 20260505 09:20:31.020 | INFO | MemInfo: { | 20260505 09:20:31.020 | INFO | "gpu" : " 17 GB", | 20260505 09:20:31.020 | INFO | "huge" : "", | 20260505 09:20:31.020 | INFO | "huge_shared" : " 27 GB", | 20260505 09:20:31.020 | INFO | "malloc" : " 14 GB", | 20260505 09:20:31.020 | INFO | "pinned" : " 29 GB" | 20260505 09:20:31.020 | INFO | } | 20260505 09:20:31.020 | INFO | Perf report for the last 5.1382 seconds: | 20260505 09:20:31.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:20:31.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:31.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:31.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:31.020 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 09:20:31.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:31.020 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:31.020 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260505 09:20:31.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:31.020 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:31.020 | INFO | Uploaded 36GiB to GPU in 2.1611s (16.6582GiB/s) | 20260505 09:20:31.020 | INFO | Downloaded 34GiB to GPU in 2.06964s (16.428GiB/s) | 20260505 09:20:36.159 | INFO | Analyzing chunk frames = [2086400,2086912) | 20260505 09:20:36.159 | INFO | MemInfo: { | 20260505 09:20:36.159 | INFO | "gpu" : " 17 GB", | 20260505 09:20:36.159 | INFO | "huge" : "", | 20260505 09:20:36.159 | INFO | "huge_shared" : " 27 GB", | 20260505 09:20:36.159 | INFO | "malloc" : " 14 GB", | 20260505 09:20:36.159 | INFO | "pinned" : " 29 GB" | 20260505 09:20:36.159 | INFO | } | 20260505 09:20:36.159 | INFO | Perf report for the last 5.13925 seconds: | 20260505 09:20:36.159 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:20:36.159 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:36.159 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:36.159 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:36.159 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 09:20:36.159 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:36.159 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:36.159 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 09:20:36.159 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:36.159 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:36.159 | INFO | Uploaded 36GiB to GPU in 2.14968s (16.7467GiB/s) | 20260505 09:20:36.159 | INFO | Downloaded 34GiB to GPU in 2.07252s (16.4051GiB/s) | 20260505 09:20:41.299 | INFO | Analyzing chunk frames = [2086912,2087424) | 20260505 09:20:41.299 | INFO | MemInfo: { | 20260505 09:20:41.299 | INFO | "gpu" : " 17 GB", | 20260505 09:20:41.299 | INFO | "huge" : "", | 20260505 09:20:41.299 | INFO | "huge_shared" : " 27 GB", | 20260505 09:20:41.300 | INFO | "malloc" : " 14 GB", | 20260505 09:20:41.300 | INFO | "pinned" : " 29 GB" | 20260505 09:20:41.300 | INFO | } | 20260505 09:20:41.300 | INFO | Perf report for the last 5.14051 seconds: | 20260505 09:20:41.300 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:20:41.300 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:41.300 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:41.300 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:41.300 | INFO | ANALYSIS : 63.9%, 3.0, 99.9ms, 0 | 20260505 09:20:41.300 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:41.300 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:41.300 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:20:41.300 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:41.300 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:41.300 | INFO | Uploaded 36GiB to GPU in 2.23855s (16.0818GiB/s) | 20260505 09:20:41.300 | INFO | Downloaded 34GiB to GPU in 2.09874s (16.2002GiB/s) | 20260505 09:20:46.439 | INFO | Analyzing chunk frames = [2087424,2087936) | 20260505 09:20:46.439 | INFO | MemInfo: { | 20260505 09:20:46.439 | INFO | "gpu" : " 17 GB", | 20260505 09:20:46.439 | INFO | "huge" : "", | 20260505 09:20:46.439 | INFO | "huge_shared" : " 27 GB", | 20260505 09:20:46.440 | INFO | "malloc" : " 14 GB", | 20260505 09:20:46.440 | INFO | "pinned" : " 29 GB" | 20260505 09:20:46.440 | INFO | } | 20260505 09:20:46.440 | INFO | Perf report for the last 5.14007 seconds: | 20260505 09:20:46.440 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:20:46.440 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:46.440 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:46.440 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:46.440 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 09:20:46.440 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:46.440 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:46.440 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:20:46.440 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:46.440 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:46.440 | INFO | Uploaded 36GiB to GPU in 2.24099s (16.0643GiB/s) | 20260505 09:20:46.440 | INFO | Downloaded 34GiB to GPU in 2.09855s (16.2017GiB/s) | 20260505 09:20:51.577 | INFO | Analyzing chunk frames = [2087936,2088448) | 20260505 09:20:51.577 | INFO | MemInfo: { | 20260505 09:20:51.577 | INFO | "gpu" : " 17 GB", | 20260505 09:20:51.577 | INFO | "huge" : "", | 20260505 09:20:51.577 | INFO | "huge_shared" : " 27 GB", | 20260505 09:20:51.578 | INFO | "malloc" : " 14 GB", | 20260505 09:20:51.578 | INFO | "pinned" : " 29 GB" | 20260505 09:20:51.578 | INFO | } | 20260505 09:20:51.578 | INFO | Perf report for the last 5.13788 seconds: | 20260505 09:20:51.578 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:20:51.578 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:51.578 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:51.578 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:51.578 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 09:20:51.578 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:51.578 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:51.578 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 09:20:51.578 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:51.578 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:51.578 | INFO | Uploaded 36GiB to GPU in 2.19227s (16.4213GiB/s) | 20260505 09:20:51.578 | INFO | Downloaded 34GiB to GPU in 2.13307s (15.9395GiB/s) | 20260505 09:20:56.717 | INFO | Analyzing chunk frames = [2088448,2088960) | 20260505 09:20:56.717 | INFO | MemInfo: { | 20260505 09:20:56.717 | INFO | "gpu" : " 17 GB", | 20260505 09:20:56.717 | INFO | "huge" : "", | 20260505 09:20:56.717 | INFO | "huge_shared" : " 27 GB", | 20260505 09:20:56.717 | INFO | "malloc" : " 14 GB", | 20260505 09:20:56.717 | INFO | "pinned" : " 29 GB" | 20260505 09:20:56.717 | INFO | } | 20260505 09:20:56.717 | INFO | Perf report for the last 5.13926 seconds: | 20260505 09:20:56.717 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:20:56.717 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:56.717 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:56.717 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:56.717 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 09:20:56.717 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:56.717 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:56.717 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:20:56.717 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:56.717 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:20:56.717 | INFO | Uploaded 36GiB to GPU in 2.18116s (16.505GiB/s) | 20260505 09:20:56.717 | INFO | Downloaded 34GiB to GPU in 2.05206s (16.5687GiB/s) | 20260505 09:20:56.717 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 09:20:56.717 | INFO | Total mem usage: 15.3 GiB | 20260505 09:20:56.717 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 09:20:56.717 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.57%) | 20260505 09:20:56.717 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 09:20:56.717 | INFO | High Water Breakdown: | 20260505 09:20:56.717 | INFO | 60.9% : 8.76 GiB ( 93.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 09:20:56.717 | INFO | 38.9% : 5.6 GiB ( 84.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 09:20:56.717 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 09:20:56.717 | INFO | ------------------------------------------ | 20260505 09:20:56.717 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:20:56.717 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 09:20:56.717 | INFO | Total mem usage: 26.5 GiB | 20260505 09:20:56.717 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 09:20:56.717 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 09:20:56.717 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 09:20:56.717 | INFO | High Water Breakdown: | 20260505 09:20:56.717 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:20:56.717 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:20:56.717 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:20:56.717 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:20:56.718 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:20:56.718 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:20:56.718 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 09:20:56.718 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 09:20:56.718 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:20:56.718 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 09:20:56.718 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 09:20:56.718 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 09:20:56.718 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 09:20:56.718 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 09:20:56.718 | INFO | ------------------------------------------ | 20260505 09:20:56.718 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:20:56.718 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 09:20:56.718 | INFO | Total mem usage: 25.6 GiB | 20260505 09:20:56.718 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 09:20:56.718 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 09:20:56.718 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 09:20:56.718 | INFO | High Water Breakdown: | 20260505 09:20:56.718 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 09:20:56.718 | INFO | ------------------------------------------ | 20260505 09:20:56.718 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:20:56.718 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 09:20:56.718 | INFO | Total mem usage: 15.7 GiB | 20260505 09:20:56.718 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 09:20:56.718 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 09:20:56.718 | INFO | Unused Cache: 880 MiB (Waste: 5.48%) | 20260505 09:20:56.718 | INFO | High Water Breakdown: | 20260505 09:20:56.718 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 09:20:56.718 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 09:20:56.718 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 09:20:56.718 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 09:20:56.718 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 09:20:56.718 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 09:20:56.718 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 09:20:56.718 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 09:20:56.718 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 09:20:56.718 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:20:56.718 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 09:20:56.718 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:20:56.718 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 09:20:56.718 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 09:20:56.718 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 09:20:56.718 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 09:20:56.718 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:20:56.718 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 09:20:56.718 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:20:56.718 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:20:56.718 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:20:56.718 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 09:20:56.718 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 09:20:56.718 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:20:56.718 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 09:20:56.718 | INFO | ------------------------------------------ | 20260505 09:20:56.718 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:20:56.718 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 09:20:56.718 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:20:56.718 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:20:56.718 | INFO | PerfReport: Full Execution -- 3.25e+01 / 7.60e+01, 1.36e+01 / 2.29e+00 | 20260505 09:20:56.718 | INFO | PerfReport: 0.29%, AggregateStats -- 9.33e-02 / 1.03e-01, 1.09e-02 / 4.31e-03 | 20260505 09:20:56.718 | INFO | PerfReport: 11.65%, Baseline -- 3.79e+00 / 4.41e+00, 6.14e-01 / 2.98e-01 | 20260505 09:20:56.718 | INFO | PerfReport: 3.32%, Histogram -- 1.08e+00 / 1.48e+00, 2.09e-01 / 9.31e-02 | 20260505 09:20:56.718 | INFO | PerfReport: 15.55%, DME -- 4.04e+01 / 4.43e+01, 4.17e+00 / 1.53e+00 | 20260505 09:20:56.718 | INFO | PerfReport: 56.54%, FrameLabeling -- 1.84e+01 / 2.07e+01, 1.89e+00 / 1.99e+00 | 20260505 09:20:56.718 | INFO | PerfReport: 7.46%, PulseAccumulating -- 2.42e+00 / 3.01e+00, 4.19e-01 / 2.06e-01 | 20260505 09:20:56.718 | INFO | PerfReport: 3.83%, Metrics -- 1.24e+00 / 1.55e+00, 3.59e-01 / 1.81e-01 | 20260505 09:20:56.718 | INFO | PerfReport: 1.36%, PulseSerializing -- 4.43e-01 / 4.92e-01, 8.30e-02 / 3.36e-02 | 20260505 09:20:56.718 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 09:20:56.718 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:20:56.718 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:20:56.718 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.36e+01 / 0.00e+00 | 20260505 09:20:56.718 | INFO | PerfReport: 22.54%, Upload -- 2.29e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260505 09:20:56.718 | INFO | PerfReport: 21.83%, Download -- 2.22e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260505 09:20:56.718 | INFO | PerfReport: 32.04%, Compute -- 3.25e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 09:20:56.718 | INFO | PerfReport: 1.73%, LockUpload -- 1.76e+00 / 0.00e+00, 7.36e+00 / 0.00e+00 | 20260505 09:20:56.718 | INFO | PerfReport: 1.12%, LockDownload -- 1.13e+00 / 0.00e+00, 2.68e+00 / 0.00e+00 | 20260505 09:20:56.718 | INFO | PerfReport: 20.72%, LockCompute -- 2.10e+01 / 0.00e+00, 1.77e+01 / 0.00e+00 | 20260505 09:20:56.718 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.88e-02 / 0.00e+00 | 20260505 09:20:56.853 | INFO | Starting a round of Baz IO | 20260505 09:21:00.128 | INFO | Populating RTMetrics report | 20260505 09:21:00.128 | INFO | RTMetrics report populated | 20260505 09:21:00.128 | INFO | Opening tmp RTMetrics json file | 20260505 09:21:00.163 | INFO | Serializing RTMetrics to json | 20260505 09:21:00.212 | INFO | Serialization done, swapping tmp file now... | 20260505 09:21:00.236 | INFO | RTMetrics done | 20260505 09:21:00.278 | INFO | Finished a round of Baz IO: Wrote 2.443 GiB in 3.424 seconds | 20260505 09:21:00.278 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 26.23% / 0.02% / 73.76% | 20260505 09:21:00.278 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.04237% / 1.58946e-05% / 7.54992e-05% | 20260505 09:21:00.278 | INFO | Baz IO overall: | 20260505 09:21:00.278 | INFO | BytesWritten : 1203740058770 | 20260505 09:21:00.278 | INFO | NumEvents : 572781077433 | 20260505 09:21:01.856 | INFO | Analyzing chunk frames = [2088960,2089472) | 20260505 09:21:01.856 | INFO | MemInfo: { | 20260505 09:21:01.856 | INFO | "gpu" : " 17 GB", | 20260505 09:21:01.856 | INFO | "huge" : "", | 20260505 09:21:01.856 | INFO | "huge_shared" : " 27 GB", | 20260505 09:21:01.856 | INFO | "malloc" : " 14 GB", | 20260505 09:21:01.856 | INFO | "pinned" : " 29 GB" | 20260505 09:21:01.856 | INFO | } | 20260505 09:21:01.856 | INFO | Perf report for the last 5.13949 seconds: | 20260505 09:21:01.856 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:21:01.856 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:01.856 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:01.856 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:01.856 | INFO | ANALYSIS : 65.2%, 2.9, 99.7ms, 0 | 20260505 09:21:01.856 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:01.856 | INFO | RT_METRICS : 47.9%, 1.0, 25.1ms, 0 | 20260505 09:21:01.857 | INFO | PRE_HQ : 63.0%, 1.8, 60.4ms, 0 | 20260505 09:21:01.857 | INFO | BAZWRITER : 66.0%, 4.1, 140.7ms, 0 | 20260505 09:21:01.857 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 09:21:01.857 | INFO | Uploaded 36GiB to GPU in 2.21271s (16.2696GiB/s) | 20260505 09:21:01.857 | INFO | Downloaded 38GiB to GPU in 2.347s (16.1909GiB/s) | 20260505 09:21:06.995 | INFO | Analyzing chunk frames = [2089472,2089984) | 20260505 09:21:06.995 | INFO | MemInfo: { | 20260505 09:21:06.995 | INFO | "gpu" : " 17 GB", | 20260505 09:21:06.995 | INFO | "huge" : "", | 20260505 09:21:06.995 | INFO | "huge_shared" : " 27 GB", | 20260505 09:21:06.995 | INFO | "malloc" : " 14 GB", | 20260505 09:21:06.995 | INFO | "pinned" : " 29 GB" | 20260505 09:21:06.995 | INFO | } | 20260505 09:21:06.995 | INFO | Perf report for the last 5.139 seconds: | 20260505 09:21:06.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:21:06.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:06.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:06.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:06.995 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 09:21:06.995 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:06.995 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:06.996 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260505 09:21:06.996 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:06.996 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:06.996 | INFO | Uploaded 36GiB to GPU in 2.23379s (16.1161GiB/s) | 20260505 09:21:06.996 | INFO | Downloaded 34GiB to GPU in 2.12097s (16.0304GiB/s) | 20260505 09:21:12.135 | INFO | Analyzing chunk frames = [2089984,2090496) | 20260505 09:21:12.135 | INFO | MemInfo: { | 20260505 09:21:12.135 | INFO | "gpu" : " 17 GB", | 20260505 09:21:12.135 | INFO | "huge" : "", | 20260505 09:21:12.135 | INFO | "huge_shared" : " 27 GB", | 20260505 09:21:12.135 | INFO | "malloc" : " 14 GB", | 20260505 09:21:12.135 | INFO | "pinned" : " 29 GB" | 20260505 09:21:12.135 | INFO | } | 20260505 09:21:12.135 | INFO | Perf report for the last 5.13949 seconds: | 20260505 09:21:12.135 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:21:12.135 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:12.135 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:12.135 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:12.135 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260505 09:21:12.135 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:12.135 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:12.135 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 09:21:12.135 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:12.135 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:12.135 | INFO | Uploaded 36GiB to GPU in 2.16905s (16.5971GiB/s) | 20260505 09:21:12.135 | INFO | Downloaded 34GiB to GPU in 2.07126s (16.4151GiB/s) | 20260505 09:21:17.274 | INFO | Analyzing chunk frames = [2090496,2091008) | 20260505 09:21:17.274 | INFO | MemInfo: { | 20260505 09:21:17.275 | INFO | "gpu" : " 17 GB", | 20260505 09:21:17.275 | INFO | "huge" : "", | 20260505 09:21:17.275 | INFO | "huge_shared" : " 27 GB", | 20260505 09:21:17.275 | INFO | "malloc" : " 14 GB", | 20260505 09:21:17.275 | INFO | "pinned" : " 29 GB" | 20260505 09:21:17.275 | INFO | } | 20260505 09:21:17.275 | INFO | Perf report for the last 5.13985 seconds: | 20260505 09:21:17.275 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:21:17.275 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:17.275 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:17.275 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:17.275 | INFO | ANALYSIS : 63.2%, 3.0, 98.3ms, 0 | 20260505 09:21:17.275 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:17.275 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:17.275 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:21:17.275 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:17.275 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:17.275 | INFO | Uploaded 36GiB to GPU in 2.15486s (16.7064GiB/s) | 20260505 09:21:17.275 | INFO | Downloaded 34GiB to GPU in 2.07019s (16.4236GiB/s) | 20260505 09:21:22.414 | INFO | Analyzing chunk frames = [2091008,2091520) | 20260505 09:21:22.414 | INFO | MemInfo: { | 20260505 09:21:22.414 | INFO | "gpu" : " 17 GB", | 20260505 09:21:22.414 | INFO | "huge" : "", | 20260505 09:21:22.414 | INFO | "huge_shared" : " 27 GB", | 20260505 09:21:22.414 | INFO | "malloc" : " 14 GB", | 20260505 09:21:22.414 | INFO | "pinned" : " 29 GB" | 20260505 09:21:22.414 | INFO | } | 20260505 09:21:22.414 | INFO | Perf report for the last 5.13944 seconds: | 20260505 09:21:22.414 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:21:22.414 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:22.414 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:22.414 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:22.414 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 09:21:22.414 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:22.414 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:22.414 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 09:21:22.414 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:22.414 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:22.415 | INFO | Uploaded 36GiB to GPU in 2.23267s (16.1242GiB/s) | 20260505 09:21:22.415 | INFO | Downloaded 34GiB to GPU in 2.10437s (16.1569GiB/s) | 20260505 09:21:27.555 | INFO | Analyzing chunk frames = [2091520,2092032) | 20260505 09:21:27.555 | INFO | MemInfo: { | 20260505 09:21:27.555 | INFO | "gpu" : " 17 GB", | 20260505 09:21:27.555 | INFO | "huge" : "", | 20260505 09:21:27.555 | INFO | "huge_shared" : " 27 GB", | 20260505 09:21:27.556 | INFO | "malloc" : " 14 GB", | 20260505 09:21:27.556 | INFO | "pinned" : " 29 GB" | 20260505 09:21:27.556 | INFO | } | 20260505 09:21:27.556 | INFO | Perf report for the last 5.14151 seconds: | 20260505 09:21:27.556 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:21:27.556 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:27.556 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:27.556 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:27.556 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 09:21:27.556 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:27.556 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:27.556 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 09:21:27.556 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:27.556 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:27.556 | INFO | Uploaded 36GiB to GPU in 2.16605s (16.6201GiB/s) | 20260505 09:21:27.556 | INFO | Downloaded 34GiB to GPU in 2.058s (16.5209GiB/s) | 20260505 09:21:32.692 | INFO | Analyzing chunk frames = [2092032,2092544) | 20260505 09:21:32.692 | INFO | MemInfo: { | 20260505 09:21:32.692 | INFO | "gpu" : " 17 GB", | 20260505 09:21:32.692 | INFO | "huge" : "", | 20260505 09:21:32.692 | INFO | "huge_shared" : " 27 GB", | 20260505 09:21:32.693 | INFO | "malloc" : " 14 GB", | 20260505 09:21:32.693 | INFO | "pinned" : " 29 GB" | 20260505 09:21:32.693 | INFO | } | 20260505 09:21:32.693 | INFO | Perf report for the last 5.13698 seconds: | 20260505 09:21:32.693 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:21:32.693 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:32.693 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:32.693 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:32.693 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 09:21:32.693 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:32.693 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:32.693 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260505 09:21:32.693 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:32.693 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:32.693 | INFO | Uploaded 36GiB to GPU in 2.19981s (16.3651GiB/s) | 20260505 09:21:32.693 | INFO | Downloaded 34GiB to GPU in 2.14458s (15.8539GiB/s) | 20260505 09:21:37.831 | INFO | Analyzing chunk frames = [2092544,2093056) | 20260505 09:21:37.831 | INFO | MemInfo: { | 20260505 09:21:37.831 | INFO | "gpu" : " 17 GB", | 20260505 09:21:37.831 | INFO | "huge" : "", | 20260505 09:21:37.831 | INFO | "huge_shared" : " 27 GB", | 20260505 09:21:37.831 | INFO | "malloc" : " 14 GB", | 20260505 09:21:37.831 | INFO | "pinned" : " 29 GB" | 20260505 09:21:37.831 | INFO | } | 20260505 09:21:37.832 | INFO | Perf report for the last 5.1389 seconds: | 20260505 09:21:37.832 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:21:37.832 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:37.832 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:37.832 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:37.832 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 09:21:37.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:37.832 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:37.832 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 09:21:37.832 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:37.832 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:37.832 | INFO | Uploaded 36GiB to GPU in 2.18221s (16.497GiB/s) | 20260505 09:21:37.832 | INFO | Downloaded 34GiB to GPU in 2.04996s (16.5857GiB/s) | 20260505 09:21:37.963 | INFO | Starting a round of Baz IO | 20260505 09:21:41.235 | INFO | Populating RTMetrics report | 20260505 09:21:41.235 | INFO | RTMetrics report populated | 20260505 09:21:41.235 | INFO | Opening tmp RTMetrics json file | 20260505 09:21:41.250 | INFO | Serializing RTMetrics to json | 20260505 09:21:41.276 | INFO | Serialization done, swapping tmp file now... | 20260505 09:21:41.276 | INFO | RTMetrics done | 20260505 09:21:41.372 | INFO | Finished a round of Baz IO: Wrote 2.444 GiB in 3.408 seconds | 20260505 09:21:41.372 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 26.21% / 0.02% / 73.77% | 20260505 09:21:41.372 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.03856% / 1.98682e-05% / 6.75519e-05% | 20260505 09:21:41.372 | INFO | Baz IO overall: | 20260505 09:21:41.372 | INFO | BytesWritten : 1206363961206 | 20260505 09:21:41.372 | INFO | NumEvents : 574035633823 | 20260505 09:21:42.971 | INFO | Analyzing chunk frames = [2093056,2093568) | 20260505 09:21:42.971 | INFO | MemInfo: { | 20260505 09:21:42.971 | INFO | "gpu" : " 17 GB", | 20260505 09:21:42.971 | INFO | "huge" : "", | 20260505 09:21:42.971 | INFO | "huge_shared" : " 27 GB", | 20260505 09:21:42.971 | INFO | "malloc" : " 14 GB", | 20260505 09:21:42.971 | INFO | "pinned" : " 29 GB" | 20260505 09:21:42.971 | INFO | } | 20260505 09:21:42.971 | INFO | Perf report for the last 5.13961 seconds: | 20260505 09:21:42.971 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:21:42.971 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:42.971 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:42.971 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:42.971 | INFO | ANALYSIS : 65.1%, 2.9, 100.0ms, 0 | 20260505 09:21:42.971 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:42.971 | INFO | RT_METRICS : 46.7%, 1.0, 24.5ms, 0 | 20260505 09:21:42.971 | INFO | PRE_HQ : 63.0%, 1.9, 60.6ms, 0 | 20260505 09:21:42.971 | INFO | BAZWRITER : 65.6%, 4.1, 139.6ms, 0 | 20260505 09:21:42.971 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 09:21:42.972 | INFO | Uploaded 36GiB to GPU in 2.24461s (16.0384GiB/s) | 20260505 09:21:42.972 | INFO | Downloaded 38GiB to GPU in 2.36112s (16.094GiB/s) | 20260505 09:21:48.110 | INFO | Analyzing chunk frames = [2093568,2094080) | 20260505 09:21:48.110 | INFO | MemInfo: { | 20260505 09:21:48.110 | INFO | "gpu" : " 17 GB", | 20260505 09:21:48.110 | INFO | "huge" : "", | 20260505 09:21:48.110 | INFO | "huge_shared" : " 27 GB", | 20260505 09:21:48.110 | INFO | "malloc" : " 14 GB", | 20260505 09:21:48.110 | INFO | "pinned" : " 29 GB" | 20260505 09:21:48.110 | INFO | } | 20260505 09:21:48.111 | INFO | Perf report for the last 5.13925 seconds: | 20260505 09:21:48.111 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:21:48.111 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:48.111 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:48.111 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:48.111 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 09:21:48.111 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:48.111 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:48.111 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 09:21:48.111 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:48.111 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:48.111 | INFO | Uploaded 36GiB to GPU in 2.20708s (16.3111GiB/s) | 20260505 09:21:48.111 | INFO | Downloaded 34GiB to GPU in 2.11591s (16.0688GiB/s) | 20260505 09:21:53.250 | INFO | Analyzing chunk frames = [2094080,2094592) | 20260505 09:21:53.250 | INFO | MemInfo: { | 20260505 09:21:53.250 | INFO | "gpu" : " 17 GB", | 20260505 09:21:53.250 | INFO | "huge" : "", | 20260505 09:21:53.250 | INFO | "huge_shared" : " 27 GB", | 20260505 09:21:53.250 | INFO | "malloc" : " 14 GB", | 20260505 09:21:53.250 | INFO | "pinned" : " 29 GB" | 20260505 09:21:53.250 | INFO | } | 20260505 09:21:53.250 | INFO | Perf report for the last 5.13932 seconds: | 20260505 09:21:53.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:21:53.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:53.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:53.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:53.250 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 09:21:53.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:53.250 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:53.250 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 09:21:53.250 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:53.250 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:53.250 | INFO | Uploaded 36GiB to GPU in 2.15061s (16.7394GiB/s) | 20260505 09:21:53.250 | INFO | Downloaded 34GiB to GPU in 2.06573s (16.4591GiB/s) | 20260505 09:21:58.389 | INFO | Analyzing chunk frames = [2094592,2095104) | 20260505 09:21:58.389 | INFO | MemInfo: { | 20260505 09:21:58.389 | INFO | "gpu" : " 17 GB", | 20260505 09:21:58.390 | INFO | "huge" : "", | 20260505 09:21:58.390 | INFO | "huge_shared" : " 27 GB", | 20260505 09:21:58.390 | INFO | "malloc" : " 14 GB", | 20260505 09:21:58.390 | INFO | "pinned" : " 29 GB" | 20260505 09:21:58.390 | INFO | } | 20260505 09:21:58.390 | INFO | Perf report for the last 5.13986 seconds: | 20260505 09:21:58.390 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:21:58.390 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:58.390 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:58.390 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:58.390 | INFO | ANALYSIS : 63.3%, 3.0, 98.3ms, 0 | 20260505 09:21:58.390 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:58.390 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:58.390 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 09:21:58.390 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:58.390 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:21:58.390 | INFO | Uploaded 36GiB to GPU in 2.14314s (16.7978GiB/s) | 20260505 09:21:58.390 | INFO | Downloaded 34GiB to GPU in 2.06977s (16.427GiB/s) | 20260505 09:22:03.528 | INFO | Analyzing chunk frames = [2095104,2095616) | 20260505 09:22:03.528 | INFO | MemInfo: { | 20260505 09:22:03.528 | INFO | "gpu" : " 17 GB", | 20260505 09:22:03.528 | INFO | "huge" : "", | 20260505 09:22:03.528 | INFO | "huge_shared" : " 27 GB", | 20260505 09:22:03.529 | INFO | "malloc" : " 14 GB", | 20260505 09:22:03.529 | INFO | "pinned" : " 29 GB" | 20260505 09:22:03.529 | INFO | } | 20260505 09:22:03.529 | INFO | Perf report for the last 5.139 seconds: | 20260505 09:22:03.529 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:22:03.529 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:03.529 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:03.529 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:03.529 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 09:22:03.529 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:03.529 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:03.529 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:22:03.529 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:03.529 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:03.529 | INFO | Uploaded 36GiB to GPU in 2.23266s (16.1242GiB/s) | 20260505 09:22:03.529 | INFO | Downloaded 34GiB to GPU in 2.10586s (16.1455GiB/s) | 20260505 09:22:08.670 | INFO | Analyzing chunk frames = [2095616,2096128) | 20260505 09:22:08.670 | INFO | MemInfo: { | 20260505 09:22:08.670 | INFO | "gpu" : " 17 GB", | 20260505 09:22:08.670 | INFO | "huge" : "", | 20260505 09:22:08.670 | INFO | "huge_shared" : " 27 GB", | 20260505 09:22:08.670 | INFO | "malloc" : " 14 GB", | 20260505 09:22:08.670 | INFO | "pinned" : " 29 GB" | 20260505 09:22:08.670 | INFO | } | 20260505 09:22:08.670 | INFO | Perf report for the last 5.1412 seconds: | 20260505 09:22:08.670 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:22:08.670 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:08.670 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:08.670 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:08.670 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 09:22:08.670 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:08.670 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:08.670 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 09:22:08.670 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:08.670 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:08.670 | INFO | Uploaded 36GiB to GPU in 2.16407s (16.6353GiB/s) | 20260505 09:22:08.670 | INFO | Downloaded 34GiB to GPU in 2.05975s (16.5069GiB/s) | 20260505 09:22:13.807 | INFO | Analyzing chunk frames = [2096128,2096640) | 20260505 09:22:13.807 | INFO | MemInfo: { | 20260505 09:22:13.807 | INFO | "gpu" : " 17 GB", | 20260505 09:22:13.807 | INFO | "huge" : "", | 20260505 09:22:13.807 | INFO | "huge_shared" : " 27 GB", | 20260505 09:22:13.807 | INFO | "malloc" : " 14 GB", | 20260505 09:22:13.807 | INFO | "pinned" : " 29 GB" | 20260505 09:22:13.807 | INFO | } | 20260505 09:22:13.807 | INFO | Perf report for the last 5.13745 seconds: | 20260505 09:22:13.807 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:22:13.807 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:13.807 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:13.807 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:13.807 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 09:22:13.807 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:13.807 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:13.808 | INFO | PRE_HQ : 30.5%, 1.0, 15.9ms, 0 | 20260505 09:22:13.808 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:13.808 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:13.808 | INFO | Uploaded 36GiB to GPU in 2.20184s (16.35GiB/s) | 20260505 09:22:13.808 | INFO | Downloaded 34GiB to GPU in 2.13025s (15.9606GiB/s) | 20260505 09:22:18.946 | INFO | Analyzing chunk frames = [2096640,2097152) | 20260505 09:22:18.946 | INFO | MemInfo: { | 20260505 09:22:18.946 | INFO | "gpu" : " 17 GB", | 20260505 09:22:18.946 | INFO | "huge" : "", | 20260505 09:22:18.946 | INFO | "huge_shared" : " 27 GB", | 20260505 09:22:18.947 | INFO | "malloc" : " 14 GB", | 20260505 09:22:18.947 | INFO | "pinned" : " 29 GB" | 20260505 09:22:18.947 | INFO | } | 20260505 09:22:18.947 | INFO | Perf report for the last 5.13934 seconds: | 20260505 09:22:18.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:22:18.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:18.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:18.947 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:18.947 | INFO | ANALYSIS : 64.1%, 3.0, 100.0ms, 0 | 20260505 09:22:18.947 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:18.947 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:18.947 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 09:22:18.947 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:18.947 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:18.947 | INFO | Uploaded 36GiB to GPU in 2.20167s (16.3512GiB/s) | 20260505 09:22:18.947 | INFO | Downloaded 34GiB to GPU in 2.04898s (16.5936GiB/s) | 20260505 09:22:18.947 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 09:22:18.947 | INFO | Total mem usage: 15.3 GiB | 20260505 09:22:18.947 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 09:22:18.947 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -7.01%) | 20260505 09:22:18.947 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 09:22:18.947 | INFO | High Water Breakdown: | 20260505 09:22:18.947 | INFO | 60.8% : 8.7 GiB ( 93% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 09:22:18.947 | INFO | 39.1% : 5.6 GiB ( 84.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 09:22:18.947 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 09:22:18.947 | INFO | ------------------------------------------ | 20260505 09:22:18.947 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:22:18.947 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 09:22:18.947 | INFO | Total mem usage: 26.5 GiB | 20260505 09:22:18.947 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 09:22:18.947 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 09:22:18.947 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 09:22:18.947 | INFO | High Water Breakdown: | 20260505 09:22:18.947 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:22:18.947 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:22:18.947 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:22:18.947 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:22:18.947 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:22:18.947 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:22:18.947 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 09:22:18.947 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 09:22:18.948 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:22:18.948 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 09:22:18.948 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 09:22:18.948 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 09:22:18.948 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 09:22:18.948 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 09:22:18.948 | INFO | ------------------------------------------ | 20260505 09:22:18.948 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:22:18.948 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 09:22:18.948 | INFO | Total mem usage: 25.6 GiB | 20260505 09:22:18.948 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 09:22:18.948 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 09:22:18.948 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 09:22:18.948 | INFO | High Water Breakdown: | 20260505 09:22:18.948 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 09:22:18.948 | INFO | ------------------------------------------ | 20260505 09:22:18.948 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:22:18.948 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 09:22:18.948 | INFO | Total mem usage: 15.7 GiB | 20260505 09:22:18.948 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 09:22:18.948 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 09:22:18.948 | INFO | Unused Cache: 880 MiB (Waste: 5.48%) | 20260505 09:22:18.948 | INFO | High Water Breakdown: | 20260505 09:22:18.948 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 09:22:18.948 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 09:22:18.948 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 09:22:18.948 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 09:22:18.948 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 09:22:18.948 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 09:22:18.948 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 09:22:18.948 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 09:22:18.948 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 09:22:18.948 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:22:18.948 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 09:22:18.948 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:22:18.948 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 09:22:18.948 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 09:22:18.948 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 09:22:18.948 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 09:22:18.948 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:22:18.948 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 09:22:18.948 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:22:18.948 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:22:18.948 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:22:18.948 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 09:22:18.948 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 09:22:18.948 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:22:18.948 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 09:22:18.948 | INFO | ------------------------------------------ | 20260505 09:22:18.948 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:22:18.948 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 09:22:18.948 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:22:18.948 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:22:18.948 | INFO | PerfReport: Full Execution -- 3.25e+01 / 7.57e+01, 1.39e+01 / 2.17e+00 | 20260505 09:22:18.948 | INFO | PerfReport: 0.29%, AggregateStats -- 9.33e-02 / 1.05e-01, 1.13e-02 / 4.83e-03 | 20260505 09:22:18.948 | INFO | PerfReport: 11.66%, Baseline -- 3.79e+00 / 4.39e+00, 6.20e-01 / 3.01e-01 | 20260505 09:22:18.948 | INFO | PerfReport: 3.31%, Histogram -- 1.08e+00 / 1.46e+00, 2.09e-01 / 1.02e-01 | 20260505 09:22:18.948 | INFO | PerfReport: 15.51%, DME -- 4.03e+01 / 4.40e+01, 4.21e+00 / 1.78e+00 | 20260505 09:22:18.948 | INFO | PerfReport: 56.57%, FrameLabeling -- 1.84e+01 / 2.07e+01, 1.89e+00 / 2.13e+00 | 20260505 09:22:18.948 | INFO | PerfReport: 7.46%, PulseAccumulating -- 2.42e+00 / 2.96e+00, 4.12e-01 / 2.32e-01 | 20260505 09:22:18.948 | INFO | PerfReport: 3.83%, Metrics -- 1.24e+00 / 1.57e+00, 3.52e-01 / 1.53e-01 | 20260505 09:22:18.948 | INFO | PerfReport: 1.37%, PulseSerializing -- 4.43e-01 / 4.97e-01, 8.26e-02 / 2.75e-02 | 20260505 09:22:18.948 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 09:22:18.948 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:22:18.948 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:22:18.948 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.32e+01 / 0.00e+00 | 20260505 09:22:18.948 | INFO | PerfReport: 22.51%, Upload -- 2.28e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260505 09:22:18.948 | INFO | PerfReport: 21.76%, Download -- 2.21e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260505 09:22:18.948 | INFO | PerfReport: 32.01%, Compute -- 3.25e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 09:22:18.948 | INFO | PerfReport: 1.73%, LockUpload -- 1.75e+00 / 0.00e+00, 7.36e+00 / 0.00e+00 | 20260505 09:22:18.948 | INFO | PerfReport: 1.14%, LockDownload -- 1.15e+00 / 0.00e+00, 2.79e+00 / 0.00e+00 | 20260505 09:22:18.948 | INFO | PerfReport: 20.84%, LockCompute -- 2.12e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260505 09:22:18.948 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.71e-02 / 0.00e+00 | 20260505 09:22:19.081 | INFO | Starting a round of Baz IO | 20260505 09:22:22.362 | INFO | Populating RTMetrics report | 20260505 09:22:22.362 | INFO | RTMetrics report populated | 20260505 09:22:22.362 | INFO | Opening tmp RTMetrics json file | 20260505 09:22:22.377 | INFO | Serializing RTMetrics to json | 20260505 09:22:22.378 | INFO | Serialization done, swapping tmp file now... | 20260505 09:22:22.379 | INFO | RTMetrics done | 20260505 09:22:22.508 | INFO | Finished a round of Baz IO: Wrote 2.442 GiB in 3.427 seconds | 20260505 09:22:22.508 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 26.19% / 0.02% / 73.79% | 20260505 09:22:22.508 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.02924% / 7.94729e-06% / 8.34465e-05% | 20260505 09:22:22.508 | INFO | Baz IO overall: | 20260505 09:22:22.508 | INFO | BytesWritten : 1208986492450 | 20260505 09:22:22.508 | INFO | NumEvents : 575289083656 | 20260505 09:22:24.085 | INFO | Analyzing chunk frames = [2097152,2097664) | 20260505 09:22:24.085 | INFO | MemInfo: { | 20260505 09:22:24.085 | INFO | "gpu" : " 17 GB", | 20260505 09:22:24.085 | INFO | "huge" : "", | 20260505 09:22:24.085 | INFO | "huge_shared" : " 27 GB", | 20260505 09:22:24.085 | INFO | "malloc" : " 14 GB", | 20260505 09:22:24.085 | INFO | "pinned" : " 29 GB" | 20260505 09:22:24.085 | INFO | } | 20260505 09:22:24.085 | INFO | Perf report for the last 5.13855 seconds: | 20260505 09:22:24.085 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:22:24.085 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:24.085 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:24.085 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:24.085 | INFO | ANALYSIS : 65.3%, 2.9, 100.3ms, 0 | 20260505 09:22:24.085 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:24.085 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 09:22:24.086 | INFO | PRE_HQ : 62.7%, 1.8, 60.0ms, 0 | 20260505 09:22:24.086 | INFO | BAZWRITER : 66.0%, 4.1, 140.5ms, 0 | 20260505 09:22:24.086 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 09:22:24.086 | INFO | Uploaded 36GiB to GPU in 2.19321s (16.4143GiB/s) | 20260505 09:22:24.086 | INFO | Downloaded 38GiB to GPU in 2.34155s (16.2286GiB/s) | 20260505 09:22:29.224 | INFO | Analyzing chunk frames = [2097664,2098176) | 20260505 09:22:29.224 | INFO | MemInfo: { | 20260505 09:22:29.224 | INFO | "gpu" : " 17 GB", | 20260505 09:22:29.224 | INFO | "huge" : "", | 20260505 09:22:29.224 | INFO | "huge_shared" : " 27 GB", | 20260505 09:22:29.224 | INFO | "malloc" : " 14 GB", | 20260505 09:22:29.224 | INFO | "pinned" : " 29 GB" | 20260505 09:22:29.224 | INFO | } | 20260505 09:22:29.224 | INFO | Perf report for the last 5.13918 seconds: | 20260505 09:22:29.225 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:22:29.225 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:29.225 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:29.225 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:29.225 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 09:22:29.225 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:29.225 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:29.225 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 09:22:29.225 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:29.225 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:29.225 | INFO | Uploaded 36GiB to GPU in 2.23206s (16.1286GiB/s) | 20260505 09:22:29.225 | INFO | Downloaded 34GiB to GPU in 2.11956s (16.0411GiB/s) | 20260505 09:22:34.364 | INFO | Analyzing chunk frames = [2098176,2098688) | 20260505 09:22:34.364 | INFO | MemInfo: { | 20260505 09:22:34.364 | INFO | "gpu" : " 17 GB", | 20260505 09:22:34.364 | INFO | "huge" : "", | 20260505 09:22:34.364 | INFO | "huge_shared" : " 27 GB", | 20260505 09:22:34.364 | INFO | "malloc" : " 14 GB", | 20260505 09:22:34.364 | INFO | "pinned" : " 29 GB" | 20260505 09:22:34.364 | INFO | } | 20260505 09:22:34.365 | INFO | Perf report for the last 5.14002 seconds: | 20260505 09:22:34.365 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:22:34.365 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:34.365 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:34.365 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:34.365 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 09:22:34.365 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:34.365 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:34.365 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:22:34.365 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:34.365 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:34.365 | INFO | Uploaded 36GiB to GPU in 2.15176s (16.7305GiB/s) | 20260505 09:22:34.365 | INFO | Downloaded 34GiB to GPU in 2.06553s (16.4607GiB/s) | 20260505 09:22:39.507 | INFO | Analyzing chunk frames = [2098688,2099200) | 20260505 09:22:39.507 | INFO | MemInfo: { | 20260505 09:22:39.507 | INFO | "gpu" : " 17 GB", | 20260505 09:22:39.507 | INFO | "huge" : "", | 20260505 09:22:39.507 | INFO | "huge_shared" : " 27 GB", | 20260505 09:22:39.507 | INFO | "malloc" : " 14 GB", | 20260505 09:22:39.507 | INFO | "pinned" : " 29 GB" | 20260505 09:22:39.507 | INFO | } | 20260505 09:22:39.507 | INFO | Perf report for the last 5.14251 seconds: | 20260505 09:22:39.507 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:22:39.507 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:39.507 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:39.507 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:39.507 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 09:22:39.507 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:39.507 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:39.507 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:22:39.507 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:39.507 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:39.507 | INFO | Uploaded 36GiB to GPU in 2.15849s (16.6784GiB/s) | 20260505 09:22:39.508 | INFO | Downloaded 34GiB to GPU in 2.06565s (16.4597GiB/s) | 20260505 09:22:44.645 | INFO | Analyzing chunk frames = [2099200,2099712) | 20260505 09:22:44.645 | INFO | MemInfo: { | 20260505 09:22:44.645 | INFO | "gpu" : " 17 GB", | 20260505 09:22:44.645 | INFO | "huge" : "", | 20260505 09:22:44.645 | INFO | "huge_shared" : " 27 GB", | 20260505 09:22:44.645 | INFO | "malloc" : " 14 GB", | 20260505 09:22:44.645 | INFO | "pinned" : " 29 GB" | 20260505 09:22:44.645 | INFO | } | 20260505 09:22:44.645 | INFO | Perf report for the last 5.13815 seconds: | 20260505 09:22:44.645 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:22:44.645 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:44.645 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:44.645 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:44.645 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 09:22:44.645 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:44.645 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:44.645 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:22:44.645 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:44.645 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:44.645 | INFO | Uploaded 36GiB to GPU in 2.23549s (16.1038GiB/s) | 20260505 09:22:44.646 | INFO | Downloaded 34GiB to GPU in 2.08907s (16.2752GiB/s) | 20260505 09:22:49.783 | INFO | Analyzing chunk frames = [2099712,2100224) | 20260505 09:22:49.783 | INFO | MemInfo: { | 20260505 09:22:49.783 | INFO | "gpu" : " 17 GB", | 20260505 09:22:49.783 | INFO | "huge" : "", | 20260505 09:22:49.783 | INFO | "huge_shared" : " 27 GB", | 20260505 09:22:49.783 | INFO | "malloc" : " 14 GB", | 20260505 09:22:49.783 | INFO | "pinned" : " 29 GB" | 20260505 09:22:49.783 | INFO | } | 20260505 09:22:49.783 | INFO | Perf report for the last 5.13771 seconds: | 20260505 09:22:49.783 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:22:49.783 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:49.783 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:49.783 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:49.783 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 09:22:49.783 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:49.783 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:49.783 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:22:49.783 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:49.783 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:49.783 | INFO | Uploaded 36GiB to GPU in 2.2408s (16.0657GiB/s) | 20260505 09:22:49.783 | INFO | Downloaded 34GiB to GPU in 2.10033s (16.1879GiB/s) | 20260505 09:22:54.923 | INFO | Analyzing chunk frames = [2100224,2100736) | 20260505 09:22:54.923 | INFO | MemInfo: { | 20260505 09:22:54.923 | INFO | "gpu" : " 17 GB", | 20260505 09:22:54.923 | INFO | "huge" : "", | 20260505 09:22:54.923 | INFO | "huge_shared" : " 27 GB", | 20260505 09:22:54.923 | INFO | "malloc" : " 14 GB", | 20260505 09:22:54.923 | INFO | "pinned" : " 29 GB" | 20260505 09:22:54.923 | INFO | } | 20260505 09:22:54.923 | INFO | Perf report for the last 5.1399 seconds: | 20260505 09:22:54.923 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:22:54.923 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:54.923 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:54.923 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:54.923 | INFO | ANALYSIS : 64.1%, 3.0, 100.0ms, 0 | 20260505 09:22:54.923 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:54.923 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:54.923 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 09:22:54.923 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:54.923 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:22:54.924 | INFO | Uploaded 36GiB to GPU in 2.19988s (16.3646GiB/s) | 20260505 09:22:54.924 | INFO | Downloaded 34GiB to GPU in 2.13642s (15.9145GiB/s) | 20260505 09:23:00.062 | INFO | Analyzing chunk frames = [2100736,2101248) | 20260505 09:23:00.062 | INFO | MemInfo: { | 20260505 09:23:00.062 | INFO | "gpu" : " 17 GB", | 20260505 09:23:00.062 | INFO | "huge" : "", | 20260505 09:23:00.062 | INFO | "huge_shared" : " 27 GB", | 20260505 09:23:00.062 | INFO | "malloc" : " 14 GB", | 20260505 09:23:00.062 | INFO | "pinned" : " 29 GB" | 20260505 09:23:00.062 | INFO | } | 20260505 09:23:00.062 | INFO | Perf report for the last 5.13944 seconds: | 20260505 09:23:00.062 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:23:00.062 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:00.062 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:00.062 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:00.062 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 09:23:00.062 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:00.062 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:00.062 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 09:23:00.062 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:00.062 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:00.063 | INFO | Uploaded 36GiB to GPU in 2.18877s (16.4476GiB/s) | 20260505 09:23:00.063 | INFO | Downloaded 34GiB to GPU in 2.04932s (16.5909GiB/s) | 20260505 09:23:00.192 | INFO | Starting a round of Baz IO | 20260505 09:23:03.456 | INFO | Populating RTMetrics report | 20260505 09:23:03.457 | INFO | RTMetrics report populated | 20260505 09:23:03.457 | INFO | Opening tmp RTMetrics json file | 20260505 09:23:03.461 | INFO | Serializing RTMetrics to json | 20260505 09:23:03.486 | INFO | Serialization done, swapping tmp file now... | 20260505 09:23:03.512 | INFO | RTMetrics done | 20260505 09:23:03.588 | INFO | Finished a round of Baz IO: Wrote 2.444 GiB in 3.395 seconds | 20260505 09:23:03.588 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 26.18% / 0.02% / 73.81% | 20260505 09:23:03.588 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.02811% / 1.58946e-05% / 7.54992e-05% | 20260505 09:23:03.588 | INFO | Baz IO overall: | 20260505 09:23:03.588 | INFO | BytesWritten : 1211610670957 | 20260505 09:23:03.588 | INFO | NumEvents : 576543769538 | 20260505 09:23:05.201 | INFO | Analyzing chunk frames = [2101248,2101760) | 20260505 09:23:05.201 | INFO | MemInfo: { | 20260505 09:23:05.201 | INFO | "gpu" : " 17 GB", | 20260505 09:23:05.201 | INFO | "huge" : "", | 20260505 09:23:05.201 | INFO | "huge_shared" : " 27 GB", | 20260505 09:23:05.201 | INFO | "malloc" : " 14 GB", | 20260505 09:23:05.201 | INFO | "pinned" : " 29 GB" | 20260505 09:23:05.201 | INFO | } | 20260505 09:23:05.201 | INFO | Perf report for the last 5.13912 seconds: | 20260505 09:23:05.201 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:23:05.201 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:05.201 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:05.201 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:05.202 | INFO | ANALYSIS : 64.9%, 2.9, 99.5ms, 0 | 20260505 09:23:05.202 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:05.202 | INFO | RT_METRICS : 46.8%, 1.0, 24.5ms, 0 | 20260505 09:23:05.202 | INFO | PRE_HQ : 63.0%, 1.8, 60.2ms, 0 | 20260505 09:23:05.202 | INFO | BAZWRITER : 65.3%, 4.1, 138.6ms, 0 | 20260505 09:23:05.202 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 09:23:05.202 | INFO | Uploaded 36GiB to GPU in 2.28327s (15.7668GiB/s) | 20260505 09:23:05.202 | INFO | Downloaded 38GiB to GPU in 2.38154s (15.956GiB/s) | 20260505 09:23:10.340 | INFO | Analyzing chunk frames = [2101760,2102272) | 20260505 09:23:10.340 | INFO | MemInfo: { | 20260505 09:23:10.340 | INFO | "gpu" : " 17 GB", | 20260505 09:23:10.340 | INFO | "huge" : "", | 20260505 09:23:10.340 | INFO | "huge_shared" : " 27 GB", | 20260505 09:23:10.340 | INFO | "malloc" : " 14 GB", | 20260505 09:23:10.340 | INFO | "pinned" : " 29 GB" | 20260505 09:23:10.340 | INFO | } | 20260505 09:23:10.340 | INFO | Perf report for the last 5.13852 seconds: | 20260505 09:23:10.340 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:23:10.340 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:10.340 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:10.340 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:10.340 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 09:23:10.340 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:10.340 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:10.340 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 09:23:10.340 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:10.340 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:10.340 | INFO | Uploaded 36GiB to GPU in 2.22916s (16.1496GiB/s) | 20260505 09:23:10.340 | INFO | Downloaded 34GiB to GPU in 2.12147s (16.0266GiB/s) | 20260505 09:23:15.480 | INFO | Analyzing chunk frames = [2102272,2102784) | 20260505 09:23:15.480 | INFO | MemInfo: { | 20260505 09:23:15.480 | INFO | "gpu" : " 17 GB", | 20260505 09:23:15.480 | INFO | "huge" : "", | 20260505 09:23:15.480 | INFO | "huge_shared" : " 27 GB", | 20260505 09:23:15.480 | INFO | "malloc" : " 14 GB", | 20260505 09:23:15.480 | INFO | "pinned" : " 29 GB" | 20260505 09:23:15.480 | INFO | } | 20260505 09:23:15.480 | INFO | Perf report for the last 5.13985 seconds: | 20260505 09:23:15.480 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:23:15.480 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:15.480 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:15.480 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:15.480 | INFO | ANALYSIS : 63.4%, 3.0, 98.6ms, 0 | 20260505 09:23:15.480 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:15.480 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:15.480 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 09:23:15.480 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:15.480 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:15.480 | INFO | Uploaded 36GiB to GPU in 2.16402s (16.6357GiB/s) | 20260505 09:23:15.480 | INFO | Downloaded 34GiB to GPU in 2.06818s (16.4396GiB/s) | 20260505 09:23:20.623 | INFO | Analyzing chunk frames = [2102784,2103296) | 20260505 09:23:20.623 | INFO | MemInfo: { | 20260505 09:23:20.623 | INFO | "gpu" : " 17 GB", | 20260505 09:23:20.623 | INFO | "huge" : "", | 20260505 09:23:20.623 | INFO | "huge_shared" : " 27 GB", | 20260505 09:23:20.623 | INFO | "malloc" : " 14 GB", | 20260505 09:23:20.623 | INFO | "pinned" : " 29 GB" | 20260505 09:23:20.623 | INFO | } | 20260505 09:23:20.623 | INFO | Perf report for the last 5.14291 seconds: | 20260505 09:23:20.623 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:23:20.623 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:20.623 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:20.623 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:20.623 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 09:23:20.623 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:20.623 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:20.623 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:23:20.623 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:20.623 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:20.623 | INFO | Uploaded 36GiB to GPU in 2.16545s (16.6247GiB/s) | 20260505 09:23:20.623 | INFO | Downloaded 34GiB to GPU in 2.07095s (16.4176GiB/s) | 20260505 09:23:25.758 | INFO | Analyzing chunk frames = [2103296,2103808) | 20260505 09:23:25.758 | INFO | MemInfo: { | 20260505 09:23:25.758 | INFO | "gpu" : " 17 GB", | 20260505 09:23:25.758 | INFO | "huge" : "", | 20260505 09:23:25.758 | INFO | "huge_shared" : " 27 GB", | 20260505 09:23:25.758 | INFO | "malloc" : " 14 GB", | 20260505 09:23:25.758 | INFO | "pinned" : " 29 GB" | 20260505 09:23:25.758 | INFO | } | 20260505 09:23:25.758 | INFO | Perf report for the last 5.13499 seconds: | 20260505 09:23:25.758 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:23:25.758 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:25.758 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:25.758 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:25.758 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 09:23:25.758 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:25.758 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:25.758 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:23:25.758 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:25.758 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:25.758 | INFO | Uploaded 36GiB to GPU in 2.23651s (16.0965GiB/s) | 20260505 09:23:25.758 | INFO | Downloaded 34GiB to GPU in 2.10251s (16.1711GiB/s) | 20260505 09:23:30.898 | INFO | Analyzing chunk frames = [2103808,2104320) | 20260505 09:23:30.898 | INFO | MemInfo: { | 20260505 09:23:30.898 | INFO | "gpu" : " 17 GB", | 20260505 09:23:30.898 | INFO | "huge" : "", | 20260505 09:23:30.898 | INFO | "huge_shared" : " 27 GB", | 20260505 09:23:30.898 | INFO | "malloc" : " 14 GB", | 20260505 09:23:30.898 | INFO | "pinned" : " 29 GB" | 20260505 09:23:30.898 | INFO | } | 20260505 09:23:30.898 | INFO | Perf report for the last 5.14004 seconds: | 20260505 09:23:30.898 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:23:30.898 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:30.898 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:30.898 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:30.898 | INFO | ANALYSIS : 64.1%, 3.0, 100.0ms, 0 | 20260505 09:23:30.898 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:30.898 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:30.898 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 09:23:30.898 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:30.898 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:30.898 | INFO | Uploaded 36GiB to GPU in 2.24114s (16.0632GiB/s) | 20260505 09:23:30.898 | INFO | Downloaded 34GiB to GPU in 2.09996s (16.1908GiB/s) | 20260505 09:23:36.037 | INFO | Analyzing chunk frames = [2104320,2104832) | 20260505 09:23:36.037 | INFO | MemInfo: { | 20260505 09:23:36.037 | INFO | "gpu" : " 17 GB", | 20260505 09:23:36.037 | INFO | "huge" : "", | 20260505 09:23:36.037 | INFO | "huge_shared" : " 27 GB", | 20260505 09:23:36.038 | INFO | "malloc" : " 14 GB", | 20260505 09:23:36.038 | INFO | "pinned" : " 29 GB" | 20260505 09:23:36.038 | INFO | } | 20260505 09:23:36.038 | INFO | Perf report for the last 5.13977 seconds: | 20260505 09:23:36.038 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:23:36.038 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:36.038 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:36.038 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:36.038 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 09:23:36.038 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:36.038 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:36.038 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 09:23:36.038 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:36.038 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:36.038 | INFO | Uploaded 36GiB to GPU in 2.19765s (16.3811GiB/s) | 20260505 09:23:36.038 | INFO | Downloaded 34GiB to GPU in 2.13804s (15.9024GiB/s) | 20260505 09:23:41.176 | INFO | Analyzing chunk frames = [2104832,2105344) | 20260505 09:23:41.176 | INFO | MemInfo: { | 20260505 09:23:41.176 | INFO | "gpu" : " 17 GB", | 20260505 09:23:41.176 | INFO | "huge" : "", | 20260505 09:23:41.176 | INFO | "huge_shared" : " 27 GB", | 20260505 09:23:41.176 | INFO | "malloc" : " 14 GB", | 20260505 09:23:41.176 | INFO | "pinned" : " 29 GB" | 20260505 09:23:41.176 | INFO | } | 20260505 09:23:41.176 | INFO | Perf report for the last 5.13833 seconds: | 20260505 09:23:41.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:23:41.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:41.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:41.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:41.176 | INFO | ANALYSIS : 64.7%, 3.0, 101.0ms, 0 | 20260505 09:23:41.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:41.176 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:41.176 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 09:23:41.176 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:41.176 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:41.176 | INFO | Uploaded 36GiB to GPU in 2.2487s (16.0092GiB/s) | 20260505 09:23:41.176 | INFO | Downloaded 34GiB to GPU in 2.08384s (16.316GiB/s) | 20260505 09:23:41.177 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 09:23:41.177 | INFO | Total mem usage: 15.3 GiB | 20260505 09:23:41.177 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 09:23:41.177 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.56%) | 20260505 09:23:41.177 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 09:23:41.177 | INFO | High Water Breakdown: | 20260505 09:23:41.177 | INFO | 60.9% : 8.76 GiB ( 93.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 09:23:41.177 | INFO | 38.9% : 5.6 GiB ( 84.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 09:23:41.177 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 09:23:41.177 | INFO | ------------------------------------------ | 20260505 09:23:41.177 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:23:41.177 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 09:23:41.177 | INFO | Total mem usage: 26.5 GiB | 20260505 09:23:41.177 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 09:23:41.177 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 09:23:41.177 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 09:23:41.177 | INFO | High Water Breakdown: | 20260505 09:23:41.177 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:23:41.177 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:23:41.177 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:23:41.177 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:23:41.177 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:23:41.177 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:23:41.177 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 09:23:41.177 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 09:23:41.177 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:23:41.177 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 09:23:41.177 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 09:23:41.177 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 09:23:41.177 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 09:23:41.177 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 09:23:41.177 | INFO | ------------------------------------------ | 20260505 09:23:41.177 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:23:41.177 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 09:23:41.177 | INFO | Total mem usage: 25.6 GiB | 20260505 09:23:41.177 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 09:23:41.177 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 09:23:41.177 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 09:23:41.177 | INFO | High Water Breakdown: | 20260505 09:23:41.177 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 09:23:41.177 | INFO | ------------------------------------------ | 20260505 09:23:41.177 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:23:41.177 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 09:23:41.177 | INFO | Total mem usage: 15.7 GiB | 20260505 09:23:41.177 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 09:23:41.177 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 09:23:41.177 | INFO | Unused Cache: 880 MiB (Waste: 5.48%) | 20260505 09:23:41.177 | INFO | High Water Breakdown: | 20260505 09:23:41.177 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 09:23:41.177 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 09:23:41.177 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 09:23:41.177 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 09:23:41.177 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 09:23:41.177 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 09:23:41.177 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 09:23:41.177 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 09:23:41.177 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 09:23:41.177 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:23:41.177 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 09:23:41.177 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:23:41.177 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 09:23:41.177 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 09:23:41.177 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 09:23:41.177 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 09:23:41.177 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:23:41.177 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 09:23:41.177 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:23:41.177 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:23:41.177 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:23:41.177 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 09:23:41.177 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 09:23:41.177 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:23:41.177 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 09:23:41.177 | INFO | ------------------------------------------ | 20260505 09:23:41.177 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:23:41.178 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 09:23:41.178 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:23:41.178 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:23:41.178 | INFO | PerfReport: Full Execution -- 3.25e+01 / 7.67e+01, 1.38e+01 / 2.42e+00 | 20260505 09:23:41.178 | INFO | PerfReport: 0.29%, AggregateStats -- 9.33e-02 / 1.05e-01, 1.03e-02 / 4.94e-03 | 20260505 09:23:41.178 | INFO | PerfReport: 11.64%, Baseline -- 3.78e+00 / 4.44e+00, 6.09e-01 / 2.94e-01 | 20260505 09:23:41.178 | INFO | PerfReport: 3.32%, Histogram -- 1.08e+00 / 1.48e+00, 2.08e-01 / 1.09e-01 | 20260505 09:23:41.178 | INFO | PerfReport: 15.49%, DME -- 4.03e+01 / 4.40e+01, 4.20e+00 / 1.77e+00 | 20260505 09:23:41.178 | INFO | PerfReport: 56.69%, FrameLabeling -- 1.84e+01 / 2.17e+01, 1.90e+00 / 2.40e+00 | 20260505 09:23:41.178 | INFO | PerfReport: 7.37%, PulseAccumulating -- 2.40e+00 / 2.91e+00, 3.95e-01 / 2.28e-01 | 20260505 09:23:41.178 | INFO | PerfReport: 3.84%, Metrics -- 1.25e+00 / 1.55e+00, 3.64e-01 / 1.84e-01 | 20260505 09:23:41.178 | INFO | PerfReport: 1.36%, PulseSerializing -- 4.43e-01 / 4.90e-01, 8.42e-02 / 2.78e-02 | 20260505 09:23:41.178 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 09:23:41.178 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:23:41.178 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:23:41.178 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.37e+01 / 0.00e+00 | 20260505 09:23:41.178 | INFO | PerfReport: 22.64%, Upload -- 2.30e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260505 09:23:41.178 | INFO | PerfReport: 21.79%, Download -- 2.22e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260505 09:23:41.178 | INFO | PerfReport: 31.98%, Compute -- 3.25e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 09:23:41.178 | INFO | PerfReport: 1.84%, LockUpload -- 1.87e+00 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260505 09:23:41.178 | INFO | PerfReport: 1.07%, LockDownload -- 1.09e+00 / 0.00e+00, 2.64e+00 / 0.00e+00 | 20260505 09:23:41.178 | INFO | PerfReport: 20.67%, LockCompute -- 2.10e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260505 09:23:41.178 | INFO | PerfReport: 0.01%, Misc -- 1.46e-02 / 0.00e+00, 2.49e-02 / 0.00e+00 | 20260505 09:23:41.310 | INFO | Starting a round of Baz IO | 20260505 09:23:44.602 | INFO | Populating RTMetrics report | 20260505 09:23:44.602 | INFO | RTMetrics report populated | 20260505 09:23:44.602 | INFO | Opening tmp RTMetrics json file | 20260505 09:23:44.608 | INFO | Serializing RTMetrics to json | 20260505 09:23:44.610 | INFO | Serialization done, swapping tmp file now... | 20260505 09:23:44.610 | INFO | RTMetrics done | 20260505 09:23:44.736 | INFO | Finished a round of Baz IO: Wrote 2.443 GiB in 3.426 seconds | 20260505 09:23:44.736 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 26.16% / 0.02% / 73.82% | 20260505 09:23:44.736 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.01759% / 1.98682e-05% / 7.94729e-05% | 20260505 09:23:44.736 | INFO | Baz IO overall: | 20260505 09:23:44.736 | INFO | BytesWritten : 1214233716497 | 20260505 09:23:44.736 | INFO | NumEvents : 577797384037 | 20260505 09:23:46.318 | INFO | Analyzing chunk frames = [2105344,2105856) | 20260505 09:23:46.318 | INFO | MemInfo: { | 20260505 09:23:46.318 | INFO | "gpu" : " 17 GB", | 20260505 09:23:46.318 | INFO | "huge" : "", | 20260505 09:23:46.318 | INFO | "huge_shared" : " 27 GB", | 20260505 09:23:46.318 | INFO | "malloc" : " 14 GB", | 20260505 09:23:46.318 | INFO | "pinned" : " 29 GB" | 20260505 09:23:46.318 | INFO | } | 20260505 09:23:46.318 | INFO | Perf report for the last 5.14202 seconds: | 20260505 09:23:46.318 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:23:46.318 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:46.318 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:46.318 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:46.318 | INFO | ANALYSIS : 65.5%, 2.9, 100.2ms, 0 | 20260505 09:23:46.318 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:46.318 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260505 09:23:46.318 | INFO | PRE_HQ : 62.6%, 1.9, 60.5ms, 0 | 20260505 09:23:46.318 | INFO | BAZWRITER : 65.9%, 4.1, 140.4ms, 0 | 20260505 09:23:46.318 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 09:23:46.318 | INFO | Uploaded 36GiB to GPU in 2.23098s (16.1364GiB/s) | 20260505 09:23:46.318 | INFO | Downloaded 38GiB to GPU in 2.33585s (16.2682GiB/s) | 20260505 09:23:51.455 | INFO | Analyzing chunk frames = [2105856,2106368) | 20260505 09:23:51.455 | INFO | MemInfo: { | 20260505 09:23:51.455 | INFO | "gpu" : " 17 GB", | 20260505 09:23:51.455 | INFO | "huge" : "", | 20260505 09:23:51.455 | INFO | "huge_shared" : " 27 GB", | 20260505 09:23:51.455 | INFO | "malloc" : " 14 GB", | 20260505 09:23:51.455 | INFO | "pinned" : " 29 GB" | 20260505 09:23:51.455 | INFO | } | 20260505 09:23:51.455 | INFO | Perf report for the last 5.13733 seconds: | 20260505 09:23:51.455 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:23:51.455 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:51.455 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:51.455 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:51.455 | INFO | ANALYSIS : 64.5%, 3.0, 100.7ms, 0 | 20260505 09:23:51.455 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:51.455 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:51.455 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 09:23:51.455 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:51.455 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:51.456 | INFO | Uploaded 36GiB to GPU in 2.21485s (16.2539GiB/s) | 20260505 09:23:51.456 | INFO | Downloaded 34GiB to GPU in 2.10525s (16.1501GiB/s) | 20260505 09:23:56.594 | INFO | Analyzing chunk frames = [2106368,2106880) | 20260505 09:23:56.594 | INFO | MemInfo: { | 20260505 09:23:56.594 | INFO | "gpu" : " 17 GB", | 20260505 09:23:56.594 | INFO | "huge" : "", | 20260505 09:23:56.594 | INFO | "huge_shared" : " 27 GB", | 20260505 09:23:56.594 | INFO | "malloc" : " 14 GB", | 20260505 09:23:56.594 | INFO | "pinned" : " 29 GB" | 20260505 09:23:56.594 | INFO | } | 20260505 09:23:56.595 | INFO | Perf report for the last 5.13915 seconds: | 20260505 09:23:56.595 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:23:56.595 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:56.595 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:56.595 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:56.595 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 09:23:56.595 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:56.595 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:56.595 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 09:23:56.595 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:56.595 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:23:56.595 | INFO | Uploaded 36GiB to GPU in 2.1703s (16.5876GiB/s) | 20260505 09:23:56.595 | INFO | Downloaded 34GiB to GPU in 2.06795s (16.4414GiB/s) | 20260505 09:24:01.733 | INFO | Analyzing chunk frames = [2106880,2107392) | 20260505 09:24:01.733 | INFO | MemInfo: { | 20260505 09:24:01.733 | INFO | "gpu" : " 17 GB", | 20260505 09:24:01.733 | INFO | "huge" : "", | 20260505 09:24:01.733 | INFO | "huge_shared" : " 27 GB", | 20260505 09:24:01.733 | INFO | "malloc" : " 14 GB", | 20260505 09:24:01.733 | INFO | "pinned" : " 29 GB" | 20260505 09:24:01.733 | INFO | } | 20260505 09:24:01.733 | INFO | Perf report for the last 5.13845 seconds: | 20260505 09:24:01.733 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:24:01.733 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:01.733 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:01.733 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:01.733 | INFO | ANALYSIS : 63.3%, 3.0, 98.3ms, 0 | 20260505 09:24:01.733 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:01.733 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:01.733 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:24:01.733 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:01.733 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:01.733 | INFO | Uploaded 36GiB to GPU in 2.16647s (16.6169GiB/s) | 20260505 09:24:01.733 | INFO | Downloaded 34GiB to GPU in 2.07366s (16.3962GiB/s) | 20260505 09:24:06.874 | INFO | Analyzing chunk frames = [2107392,2107904) | 20260505 09:24:06.874 | INFO | MemInfo: { | 20260505 09:24:06.874 | INFO | "gpu" : " 17 GB", | 20260505 09:24:06.874 | INFO | "huge" : "", | 20260505 09:24:06.874 | INFO | "huge_shared" : " 27 GB", | 20260505 09:24:06.874 | INFO | "malloc" : " 14 GB", | 20260505 09:24:06.874 | INFO | "pinned" : " 29 GB" | 20260505 09:24:06.874 | INFO | } | 20260505 09:24:06.874 | INFO | Perf report for the last 5.14112 seconds: | 20260505 09:24:06.874 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:24:06.874 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:06.874 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:06.874 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:06.874 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 09:24:06.874 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:06.874 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:06.874 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:24:06.874 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:06.874 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:06.874 | INFO | Uploaded 36GiB to GPU in 2.23675s (16.0948GiB/s) | 20260505 09:24:06.874 | INFO | Downloaded 34GiB to GPU in 2.09695s (16.2141GiB/s) | 20260505 09:24:12.014 | INFO | Analyzing chunk frames = [2107904,2108416) | 20260505 09:24:12.014 | INFO | MemInfo: { | 20260505 09:24:12.014 | INFO | "gpu" : " 17 GB", | 20260505 09:24:12.014 | INFO | "huge" : "", | 20260505 09:24:12.014 | INFO | "huge_shared" : " 27 GB", | 20260505 09:24:12.014 | INFO | "malloc" : " 14 GB", | 20260505 09:24:12.014 | INFO | "pinned" : " 29 GB" | 20260505 09:24:12.014 | INFO | } | 20260505 09:24:12.014 | INFO | Perf report for the last 5.13983 seconds: | 20260505 09:24:12.014 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:24:12.014 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:12.014 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:12.014 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:12.014 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 09:24:12.014 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:12.014 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:12.014 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:24:12.014 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:12.014 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:12.014 | INFO | Uploaded 36GiB to GPU in 2.23514s (16.1064GiB/s) | 20260505 09:24:12.014 | INFO | Downloaded 34GiB to GPU in 2.09708s (16.213GiB/s) | 20260505 09:24:17.151 | INFO | Analyzing chunk frames = [2108416,2108928) | 20260505 09:24:17.151 | INFO | MemInfo: { | 20260505 09:24:17.151 | INFO | "gpu" : " 17 GB", | 20260505 09:24:17.151 | INFO | "huge" : "", | 20260505 09:24:17.151 | INFO | "huge_shared" : " 27 GB", | 20260505 09:24:17.151 | INFO | "malloc" : " 14 GB", | 20260505 09:24:17.151 | INFO | "pinned" : " 29 GB" | 20260505 09:24:17.151 | INFO | } | 20260505 09:24:17.151 | INFO | Perf report for the last 5.13723 seconds: | 20260505 09:24:17.151 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:24:17.151 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:17.151 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:17.151 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:17.151 | INFO | ANALYSIS : 64.0%, 3.0, 99.8ms, 0 | 20260505 09:24:17.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:17.151 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:17.151 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 09:24:17.151 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:17.151 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:17.151 | INFO | Uploaded 36GiB to GPU in 2.20279s (16.3429GiB/s) | 20260505 09:24:17.152 | INFO | Downloaded 34GiB to GPU in 2.145s (15.8508GiB/s) | 20260505 09:24:22.293 | INFO | Analyzing chunk frames = [2108928,2109440) | 20260505 09:24:22.293 | INFO | MemInfo: { | 20260505 09:24:22.293 | INFO | "gpu" : " 17 GB", | 20260505 09:24:22.293 | INFO | "huge" : "", | 20260505 09:24:22.293 | INFO | "huge_shared" : " 27 GB", | 20260505 09:24:22.293 | INFO | "malloc" : " 14 GB", | 20260505 09:24:22.293 | INFO | "pinned" : " 29 GB" | 20260505 09:24:22.293 | INFO | } | 20260505 09:24:22.293 | INFO | Perf report for the last 5.14225 seconds: | 20260505 09:24:22.293 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:24:22.293 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:22.294 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:22.294 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:22.294 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 09:24:22.294 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:22.294 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:22.294 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 09:24:22.294 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:22.294 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:22.294 | INFO | Uploaded 36GiB to GPU in 2.18475s (16.4778GiB/s) | 20260505 09:24:22.294 | INFO | Downloaded 34GiB to GPU in 2.05108s (16.5766GiB/s) | 20260505 09:24:22.425 | INFO | Starting a round of Baz IO | 20260505 09:24:25.705 | INFO | Populating RTMetrics report | 20260505 09:24:25.705 | INFO | RTMetrics report populated | 20260505 09:24:25.705 | INFO | Opening tmp RTMetrics json file | 20260505 09:24:25.732 | INFO | Serializing RTMetrics to json | 20260505 09:24:25.760 | INFO | Serialization done, swapping tmp file now... | 20260505 09:24:25.765 | INFO | RTMetrics done | 20260505 09:24:25.874 | INFO | Finished a round of Baz IO: Wrote 2.443 GiB in 3.448 seconds | 20260505 09:24:25.874 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 26.15% / 0.02% / 73.84% | 20260505 09:24:25.874 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.01409% / 2.38419e-05% / 7.15256e-05% | 20260505 09:24:25.874 | INFO | Baz IO overall: | 20260505 09:24:25.874 | INFO | BytesWritten : 1216856827381 | 20260505 09:24:25.874 | INFO | NumEvents : 579052040977 | 20260505 09:24:27.430 | INFO | Analyzing chunk frames = [2109440,2109952) | 20260505 09:24:27.430 | INFO | MemInfo: { | 20260505 09:24:27.430 | INFO | "gpu" : " 17 GB", | 20260505 09:24:27.430 | INFO | "huge" : "", | 20260505 09:24:27.430 | INFO | "huge_shared" : " 27 GB", | 20260505 09:24:27.430 | INFO | "malloc" : " 14 GB", | 20260505 09:24:27.430 | INFO | "pinned" : " 29 GB" | 20260505 09:24:27.430 | INFO | } | 20260505 09:24:27.430 | INFO | Perf report for the last 5.13702 seconds: | 20260505 09:24:27.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:24:27.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:27.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:27.431 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:27.431 | INFO | ANALYSIS : 65.3%, 2.9, 99.9ms, 0 | 20260505 09:24:27.431 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:27.431 | INFO | RT_METRICS : 46.9%, 1.0, 24.6ms, 0 | 20260505 09:24:27.431 | INFO | PRE_HQ : 62.9%, 1.8, 60.2ms, 0 | 20260505 09:24:27.431 | INFO | BAZWRITER : 66.4%, 4.1, 140.7ms, 0 | 20260505 09:24:27.431 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260505 09:24:27.431 | INFO | Uploaded 36GiB to GPU in 2.22704s (16.1649GiB/s) | 20260505 09:24:27.431 | INFO | Downloaded 38GiB to GPU in 2.34256s (16.2216GiB/s) | 20260505 09:24:32.575 | INFO | Analyzing chunk frames = [2109952,2110464) | 20260505 09:24:32.575 | INFO | MemInfo: { | 20260505 09:24:32.575 | INFO | "gpu" : " 17 GB", | 20260505 09:24:32.575 | INFO | "huge" : "", | 20260505 09:24:32.575 | INFO | "huge_shared" : " 27 GB", | 20260505 09:24:32.575 | INFO | "malloc" : " 14 GB", | 20260505 09:24:32.575 | INFO | "pinned" : " 29 GB" | 20260505 09:24:32.575 | INFO | } | 20260505 09:24:32.575 | INFO | Perf report for the last 5.14469 seconds: | 20260505 09:24:32.575 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:24:32.575 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:32.575 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:32.575 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:32.575 | INFO | ANALYSIS : 64.4%, 3.0, 100.6ms, 0 | 20260505 09:24:32.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:32.575 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:32.575 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:24:32.575 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:32.575 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:32.576 | INFO | Uploaded 36GiB to GPU in 2.19749s (16.3823GiB/s) | 20260505 09:24:32.576 | INFO | Downloaded 34GiB to GPU in 2.1085s (16.1252GiB/s) | 20260505 09:24:37.710 | INFO | Analyzing chunk frames = [2110464,2110976) | 20260505 09:24:37.710 | INFO | MemInfo: { | 20260505 09:24:37.710 | INFO | "gpu" : " 17 GB", | 20260505 09:24:37.710 | INFO | "huge" : "", | 20260505 09:24:37.710 | INFO | "huge_shared" : " 27 GB", | 20260505 09:24:37.710 | INFO | "malloc" : " 14 GB", | 20260505 09:24:37.710 | INFO | "pinned" : " 29 GB" | 20260505 09:24:37.710 | INFO | } | 20260505 09:24:37.710 | INFO | Perf report for the last 5.13454 seconds: | 20260505 09:24:37.710 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:24:37.710 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:37.710 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:37.710 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:37.710 | INFO | ANALYSIS : 63.2%, 3.0, 98.3ms, 0 | 20260505 09:24:37.710 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:37.710 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:37.710 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:24:37.710 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:37.710 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:37.710 | INFO | Uploaded 36GiB to GPU in 2.15875s (16.6763GiB/s) | 20260505 09:24:37.710 | INFO | Downloaded 34GiB to GPU in 2.06178s (16.4906GiB/s) | 20260505 09:24:42.848 | INFO | Analyzing chunk frames = [2110976,2111488) | 20260505 09:24:42.848 | INFO | MemInfo: { | 20260505 09:24:42.848 | INFO | "gpu" : " 17 GB", | 20260505 09:24:42.848 | INFO | "huge" : "", | 20260505 09:24:42.848 | INFO | "huge_shared" : " 27 GB", | 20260505 09:24:42.848 | INFO | "malloc" : " 14 GB", | 20260505 09:24:42.848 | INFO | "pinned" : " 29 GB" | 20260505 09:24:42.848 | INFO | } | 20260505 09:24:42.848 | INFO | Perf report for the last 5.13853 seconds: | 20260505 09:24:42.848 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:24:42.848 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:42.848 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:42.848 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:42.848 | INFO | ANALYSIS : 63.2%, 3.0, 98.1ms, 0 | 20260505 09:24:42.848 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:42.848 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:42.848 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 09:24:42.848 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:42.848 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:42.849 | INFO | Uploaded 36GiB to GPU in 2.16037s (16.6638GiB/s) | 20260505 09:24:42.849 | INFO | Downloaded 34GiB to GPU in 2.07055s (16.4207GiB/s) | 20260505 09:24:47.987 | INFO | Analyzing chunk frames = [2111488,2112000) | 20260505 09:24:47.987 | INFO | MemInfo: { | 20260505 09:24:47.987 | INFO | "gpu" : " 17 GB", | 20260505 09:24:47.987 | INFO | "huge" : "", | 20260505 09:24:47.987 | INFO | "huge_shared" : " 27 GB", | 20260505 09:24:47.987 | INFO | "malloc" : " 14 GB", | 20260505 09:24:47.987 | INFO | "pinned" : " 29 GB" | 20260505 09:24:47.987 | INFO | } | 20260505 09:24:47.987 | INFO | Perf report for the last 5.13886 seconds: | 20260505 09:24:47.987 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:24:47.987 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:47.987 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:47.987 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:47.987 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 09:24:47.987 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:47.987 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:47.987 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 09:24:47.987 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:47.987 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:47.988 | INFO | Uploaded 36GiB to GPU in 2.23493s (16.1079GiB/s) | 20260505 09:24:47.988 | INFO | Downloaded 34GiB to GPU in 2.10824s (16.1272GiB/s) | 20260505 09:24:53.127 | INFO | Analyzing chunk frames = [2112000,2112512) | 20260505 09:24:53.127 | INFO | MemInfo: { | 20260505 09:24:53.127 | INFO | "gpu" : " 17 GB", | 20260505 09:24:53.127 | INFO | "huge" : "", | 20260505 09:24:53.127 | INFO | "huge_shared" : " 27 GB", | 20260505 09:24:53.127 | INFO | "malloc" : " 14 GB", | 20260505 09:24:53.127 | INFO | "pinned" : " 29 GB" | 20260505 09:24:53.127 | INFO | } | 20260505 09:24:53.127 | INFO | Perf report for the last 5.14 seconds: | 20260505 09:24:53.127 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:24:53.127 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:53.127 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:53.127 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:53.127 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 09:24:53.127 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:53.127 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:53.127 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 09:24:53.127 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:53.127 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:53.127 | INFO | Uploaded 36GiB to GPU in 2.23652s (16.0964GiB/s) | 20260505 09:24:53.127 | INFO | Downloaded 34GiB to GPU in 2.0967s (16.216GiB/s) | 20260505 09:24:58.265 | INFO | Analyzing chunk frames = [2112512,2113024) | 20260505 09:24:58.265 | INFO | MemInfo: { | 20260505 09:24:58.265 | INFO | "gpu" : " 17 GB", | 20260505 09:24:58.265 | INFO | "huge" : "", | 20260505 09:24:58.265 | INFO | "huge_shared" : " 27 GB", | 20260505 09:24:58.265 | INFO | "malloc" : " 14 GB", | 20260505 09:24:58.265 | INFO | "pinned" : " 29 GB" | 20260505 09:24:58.265 | INFO | } | 20260505 09:24:58.266 | INFO | Perf report for the last 5.13837 seconds: | 20260505 09:24:58.266 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:24:58.266 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:58.266 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:58.266 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:58.266 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 09:24:58.266 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:58.266 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:58.266 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 09:24:58.266 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:58.266 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:24:58.266 | INFO | Uploaded 36GiB to GPU in 2.20826s (16.3024GiB/s) | 20260505 09:24:58.266 | INFO | Downloaded 34GiB to GPU in 2.15561s (15.7728GiB/s) | 20260505 09:25:03.406 | INFO | Analyzing chunk frames = [2113024,2113536) | 20260505 09:25:03.406 | INFO | MemInfo: { | 20260505 09:25:03.406 | INFO | "gpu" : " 17 GB", | 20260505 09:25:03.406 | INFO | "huge" : "", | 20260505 09:25:03.406 | INFO | "huge_shared" : " 27 GB", | 20260505 09:25:03.406 | INFO | "malloc" : " 14 GB", | 20260505 09:25:03.406 | INFO | "pinned" : " 29 GB" | 20260505 09:25:03.406 | INFO | } | 20260505 09:25:03.406 | INFO | Perf report for the last 5.14059 seconds: | 20260505 09:25:03.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:25:03.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:03.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:03.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:03.406 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 09:25:03.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:03.406 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:03.406 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:25:03.406 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:03.406 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:03.406 | INFO | Uploaded 36GiB to GPU in 2.20489s (16.3273GiB/s) | 20260505 09:25:03.407 | INFO | Downloaded 34GiB to GPU in 2.05758s (16.5243GiB/s) | 20260505 09:25:03.407 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 09:25:03.407 | INFO | Total mem usage: 15.3 GiB | 20260505 09:25:03.407 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 09:25:03.407 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.25%) | 20260505 09:25:03.407 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 09:25:03.407 | INFO | High Water Breakdown: | 20260505 09:25:03.407 | INFO | 60.9% : 8.78 GiB ( 93.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 09:25:03.407 | INFO | 39% : 5.63 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 09:25:03.407 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 09:25:03.407 | INFO | ------------------------------------------ | 20260505 09:25:03.407 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:25:03.407 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 09:25:03.407 | INFO | Total mem usage: 26.5 GiB | 20260505 09:25:03.407 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 09:25:03.407 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 09:25:03.407 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 09:25:03.407 | INFO | High Water Breakdown: | 20260505 09:25:03.407 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:25:03.407 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:25:03.407 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:25:03.407 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:25:03.407 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:25:03.407 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:25:03.407 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 09:25:03.407 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 09:25:03.407 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:25:03.407 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 09:25:03.407 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 09:25:03.407 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 09:25:03.407 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 09:25:03.407 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 09:25:03.407 | INFO | ------------------------------------------ | 20260505 09:25:03.407 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:25:03.407 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 09:25:03.407 | INFO | Total mem usage: 25.6 GiB | 20260505 09:25:03.407 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 09:25:03.407 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 09:25:03.407 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 09:25:03.407 | INFO | High Water Breakdown: | 20260505 09:25:03.407 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 09:25:03.407 | INFO | ------------------------------------------ | 20260505 09:25:03.407 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:25:03.407 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 09:25:03.407 | INFO | Total mem usage: 15.7 GiB | 20260505 09:25:03.407 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 09:25:03.407 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 09:25:03.407 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 09:25:03.407 | INFO | High Water Breakdown: | 20260505 09:25:03.407 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 09:25:03.407 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 09:25:03.407 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 09:25:03.407 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 09:25:03.407 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 09:25:03.407 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 09:25:03.407 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 09:25:03.407 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 09:25:03.407 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 09:25:03.407 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:25:03.407 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 09:25:03.407 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:25:03.407 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 09:25:03.407 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 09:25:03.407 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 09:25:03.407 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 09:25:03.407 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:25:03.407 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 09:25:03.407 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:25:03.408 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:25:03.408 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:25:03.408 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 09:25:03.408 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 09:25:03.408 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:25:03.408 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 09:25:03.408 | INFO | ------------------------------------------ | 20260505 09:25:03.408 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:25:03.408 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 09:25:03.408 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:25:03.408 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:25:03.408 | INFO | PerfReport: Full Execution -- 3.25e+01 / 7.69e+01, 1.38e+01 / 2.34e+00 | 20260505 09:25:03.408 | INFO | PerfReport: 0.29%, AggregateStats -- 9.33e-02 / 1.03e-01, 1.14e-02 / 3.57e-03 | 20260505 09:25:03.408 | INFO | PerfReport: 11.63%, Baseline -- 3.78e+00 / 4.44e+00, 6.06e-01 / 2.72e-01 | 20260505 09:25:03.408 | INFO | PerfReport: 3.30%, Histogram -- 1.07e+00 / 1.48e+00, 2.07e-01 / 1.06e-01 | 20260505 09:25:03.408 | INFO | PerfReport: 15.65%, DME -- 4.07e+01 / 4.52e+01, 4.33e+00 / 1.93e+00 | 20260505 09:25:03.408 | INFO | PerfReport: 56.52%, FrameLabeling -- 1.84e+01 / 2.08e+01, 1.87e+00 / 2.00e+00 | 20260505 09:25:03.408 | INFO | PerfReport: 7.41%, PulseAccumulating -- 2.41e+00 / 2.95e+00, 4.09e-01 / 2.49e-01 | 20260505 09:25:03.408 | INFO | PerfReport: 3.83%, Metrics -- 1.25e+00 / 1.52e+00, 3.59e-01 / 2.06e-01 | 20260505 09:25:03.408 | INFO | PerfReport: 1.36%, PulseSerializing -- 4.42e-01 / 4.89e-01, 8.33e-02 / 3.39e-02 | 20260505 09:25:03.408 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 09:25:03.408 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:25:03.408 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:25:03.408 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.38e+01 / 0.00e+00 | 20260505 09:25:03.408 | INFO | PerfReport: 22.58%, Upload -- 2.30e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260505 09:25:03.408 | INFO | PerfReport: 21.75%, Download -- 2.21e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260505 09:25:03.408 | INFO | PerfReport: 31.98%, Compute -- 3.25e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 09:25:03.408 | INFO | PerfReport: 1.86%, LockUpload -- 1.89e+00 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260505 09:25:03.408 | INFO | PerfReport: 1.08%, LockDownload -- 1.10e+00 / 0.00e+00, 2.65e+00 / 0.00e+00 | 20260505 09:25:03.408 | INFO | PerfReport: 20.74%, LockCompute -- 2.11e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260505 09:25:03.408 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.41e-02 / 0.00e+00 | 20260505 09:25:03.536 | INFO | Starting a round of Baz IO | 20260505 09:25:06.800 | INFO | Populating RTMetrics report | 20260505 09:25:06.801 | INFO | RTMetrics report populated | 20260505 09:25:06.801 | INFO | Opening tmp RTMetrics json file | 20260505 09:25:06.854 | INFO | Serializing RTMetrics to json | 20260505 09:25:06.858 | INFO | Serialization done, swapping tmp file now... | 20260505 09:25:06.860 | INFO | RTMetrics done | 20260505 09:25:06.970 | INFO | Finished a round of Baz IO: Wrote 2.44 GiB in 3.433 seconds | 20260505 09:25:06.970 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 26.13% / 0.02% / 73.85% | 20260505 09:25:06.970 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.0027% / 7.94729e-06% / 4.76837e-05% | 20260505 09:25:06.970 | INFO | Baz IO overall: | 20260505 09:25:06.970 | INFO | BytesWritten : 1219476356974 | 20260505 09:25:06.970 | INFO | NumEvents : 580304009965 | 20260505 09:25:08.545 | INFO | Analyzing chunk frames = [2113536,2114048) | 20260505 09:25:08.545 | INFO | MemInfo: { | 20260505 09:25:08.545 | INFO | "gpu" : " 17 GB", | 20260505 09:25:08.545 | INFO | "huge" : "", | 20260505 09:25:08.545 | INFO | "huge_shared" : " 27 GB", | 20260505 09:25:08.545 | INFO | "malloc" : " 14 GB", | 20260505 09:25:08.545 | INFO | "pinned" : " 29 GB" | 20260505 09:25:08.545 | INFO | } | 20260505 09:25:08.545 | INFO | Perf report for the last 5.13912 seconds: | 20260505 09:25:08.548 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:25:08.548 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:08.548 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:08.548 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:08.548 | INFO | ANALYSIS : 64.9%, 2.9, 99.1ms, 0 | 20260505 09:25:08.548 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:08.548 | INFO | RT_METRICS : 46.9%, 1.0, 24.6ms, 0 | 20260505 09:25:08.548 | INFO | PRE_HQ : 63.2%, 1.9, 60.7ms, 0 | 20260505 09:25:08.548 | INFO | BAZWRITER : 66.1%, 4.2, 142.3ms, 0 | 20260505 09:25:08.548 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260505 09:25:08.548 | INFO | Uploaded 36GiB to GPU in 2.25025s (15.9983GiB/s) | 20260505 09:25:08.548 | INFO | Downloaded 38GiB to GPU in 2.37865s (15.9755GiB/s) | 20260505 09:25:13.686 | INFO | Analyzing chunk frames = [2114048,2114560) | 20260505 09:25:13.686 | INFO | MemInfo: { | 20260505 09:25:13.686 | INFO | "gpu" : " 17 GB", | 20260505 09:25:13.686 | INFO | "huge" : "", | 20260505 09:25:13.686 | INFO | "huge_shared" : " 27 GB", | 20260505 09:25:13.686 | INFO | "malloc" : " 14 GB", | 20260505 09:25:13.686 | INFO | "pinned" : " 29 GB" | 20260505 09:25:13.686 | INFO | } | 20260505 09:25:13.686 | INFO | Perf report for the last 5.14092 seconds: | 20260505 09:25:13.686 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:25:13.686 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:13.686 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:13.686 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:13.686 | INFO | ANALYSIS : 64.5%, 3.0, 100.7ms, 0 | 20260505 09:25:13.686 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:13.686 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:13.686 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 09:25:13.686 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:13.686 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:13.687 | INFO | Uploaded 36GiB to GPU in 2.22138s (16.2061GiB/s) | 20260505 09:25:13.687 | INFO | Downloaded 34GiB to GPU in 2.10588s (16.1453GiB/s) | 20260505 09:25:18.824 | INFO | Analyzing chunk frames = [2114560,2115072) | 20260505 09:25:18.824 | INFO | MemInfo: { | 20260505 09:25:18.824 | INFO | "gpu" : " 17 GB", | 20260505 09:25:18.824 | INFO | "huge" : "", | 20260505 09:25:18.824 | INFO | "huge_shared" : " 27 GB", | 20260505 09:25:18.824 | INFO | "malloc" : " 14 GB", | 20260505 09:25:18.824 | INFO | "pinned" : " 29 GB" | 20260505 09:25:18.824 | INFO | } | 20260505 09:25:18.824 | INFO | Perf report for the last 5.13788 seconds: | 20260505 09:25:18.824 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:25:18.824 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:18.824 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:18.824 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:18.824 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 09:25:18.824 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:18.824 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:18.824 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 09:25:18.824 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:18.824 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:18.824 | INFO | Uploaded 36GiB to GPU in 2.14654s (16.7711GiB/s) | 20260505 09:25:18.824 | INFO | Downloaded 34GiB to GPU in 2.06591s (16.4577GiB/s) | 20260505 09:25:23.962 | INFO | Analyzing chunk frames = [2115072,2115584) | 20260505 09:25:23.962 | INFO | MemInfo: { | 20260505 09:25:23.962 | INFO | "gpu" : " 17 GB", | 20260505 09:25:23.963 | INFO | "huge" : "", | 20260505 09:25:23.963 | INFO | "huge_shared" : " 27 GB", | 20260505 09:25:23.963 | INFO | "malloc" : " 14 GB", | 20260505 09:25:23.963 | INFO | "pinned" : " 29 GB" | 20260505 09:25:23.963 | INFO | } | 20260505 09:25:23.963 | INFO | Perf report for the last 5.13857 seconds: | 20260505 09:25:23.963 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:25:23.963 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:23.963 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:23.963 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:23.963 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 09:25:23.963 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:23.963 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:23.963 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 09:25:23.963 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:23.963 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:23.963 | INFO | Uploaded 36GiB to GPU in 2.15403s (16.7128GiB/s) | 20260505 09:25:23.963 | INFO | Downloaded 34GiB to GPU in 2.0737s (16.3958GiB/s) | 20260505 09:25:29.102 | INFO | Analyzing chunk frames = [2115584,2116096) | 20260505 09:25:29.102 | INFO | MemInfo: { | 20260505 09:25:29.102 | INFO | "gpu" : " 17 GB", | 20260505 09:25:29.102 | INFO | "huge" : "", | 20260505 09:25:29.102 | INFO | "huge_shared" : " 27 GB", | 20260505 09:25:29.102 | INFO | "malloc" : " 14 GB", | 20260505 09:25:29.102 | INFO | "pinned" : " 29 GB" | 20260505 09:25:29.102 | INFO | } | 20260505 09:25:29.102 | INFO | Perf report for the last 5.1395 seconds: | 20260505 09:25:29.102 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:25:29.102 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:29.102 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:29.102 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:29.102 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 09:25:29.102 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:29.102 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:29.102 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 09:25:29.102 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:29.102 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:29.103 | INFO | Uploaded 36GiB to GPU in 2.23666s (16.0955GiB/s) | 20260505 09:25:29.103 | INFO | Downloaded 34GiB to GPU in 2.10653s (16.1403GiB/s) | 20260505 09:25:34.242 | INFO | Analyzing chunk frames = [2116096,2116608) | 20260505 09:25:34.242 | INFO | MemInfo: { | 20260505 09:25:34.242 | INFO | "gpu" : " 17 GB", | 20260505 09:25:34.242 | INFO | "huge" : "", | 20260505 09:25:34.242 | INFO | "huge_shared" : " 27 GB", | 20260505 09:25:34.242 | INFO | "malloc" : " 14 GB", | 20260505 09:25:34.242 | INFO | "pinned" : " 29 GB" | 20260505 09:25:34.242 | INFO | } | 20260505 09:25:34.242 | INFO | Perf report for the last 5.13963 seconds: | 20260505 09:25:34.242 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:25:34.242 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:34.242 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:34.242 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:34.242 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 09:25:34.242 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:34.242 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:34.242 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:25:34.242 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:34.242 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:34.242 | INFO | Uploaded 36GiB to GPU in 2.15992s (16.6673GiB/s) | 20260505 09:25:34.242 | INFO | Downloaded 34GiB to GPU in 2.05983s (16.5062GiB/s) | 20260505 09:25:39.381 | INFO | Analyzing chunk frames = [2116608,2117120) | 20260505 09:25:39.381 | INFO | MemInfo: { | 20260505 09:25:39.381 | INFO | "gpu" : " 17 GB", | 20260505 09:25:39.381 | INFO | "huge" : "", | 20260505 09:25:39.381 | INFO | "huge_shared" : " 27 GB", | 20260505 09:25:39.381 | INFO | "malloc" : " 14 GB", | 20260505 09:25:39.381 | INFO | "pinned" : " 29 GB" | 20260505 09:25:39.381 | INFO | } | 20260505 09:25:39.381 | INFO | Perf report for the last 5.13901 seconds: | 20260505 09:25:39.381 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:25:39.381 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:39.381 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:39.381 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:39.381 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 09:25:39.381 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:39.381 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:39.381 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 09:25:39.381 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:39.381 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:39.381 | INFO | Uploaded 36GiB to GPU in 2.17951s (16.5175GiB/s) | 20260505 09:25:39.381 | INFO | Downloaded 34GiB to GPU in 2.13545s (15.9217GiB/s) | 20260505 09:25:44.523 | INFO | Analyzing chunk frames = [2117120,2117632) | 20260505 09:25:44.523 | INFO | MemInfo: { | 20260505 09:25:44.523 | INFO | "gpu" : " 17 GB", | 20260505 09:25:44.523 | INFO | "huge" : "", | 20260505 09:25:44.523 | INFO | "huge_shared" : " 27 GB", | 20260505 09:25:44.523 | INFO | "malloc" : " 14 GB", | 20260505 09:25:44.523 | INFO | "pinned" : " 29 GB" | 20260505 09:25:44.523 | INFO | } | 20260505 09:25:44.523 | INFO | Perf report for the last 5.14235 seconds: | 20260505 09:25:44.523 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:25:44.523 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:44.523 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:44.523 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:44.523 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 09:25:44.523 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:44.523 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:44.523 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 09:25:44.523 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:44.523 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:44.523 | INFO | Uploaded 36GiB to GPU in 2.18857s (16.4491GiB/s) | 20260505 09:25:44.523 | INFO | Downloaded 34GiB to GPU in 2.05499s (16.5451GiB/s) | 20260505 09:25:44.653 | INFO | Starting a round of Baz IO | 20260505 09:25:47.933 | INFO | Populating RTMetrics report | 20260505 09:25:47.933 | INFO | RTMetrics report populated | 20260505 09:25:47.933 | INFO | Opening tmp RTMetrics json file | 20260505 09:25:47.950 | INFO | Serializing RTMetrics to json | 20260505 09:25:47.975 | INFO | Serialization done, swapping tmp file now... | 20260505 09:25:47.976 | INFO | RTMetrics done | 20260505 09:25:48.080 | INFO | Finished a round of Baz IO: Wrote 2.441 GiB in 3.426 seconds | 20260505 09:25:48.080 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 26.12% / 0.01% / 73.87% | 20260505 09:25:48.080 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.99565% / 1.19209e-05% / 7.15256e-05% | 20260505 09:25:48.080 | INFO | Baz IO overall: | 20260505 09:25:48.080 | INFO | BytesWritten : 1222097250823 | 20260505 09:25:48.080 | INFO | NumEvents : 581557010099 | 20260505 09:25:49.662 | INFO | Analyzing chunk frames = [2117632,2118144) | 20260505 09:25:49.662 | INFO | MemInfo: { | 20260505 09:25:49.662 | INFO | "gpu" : " 17 GB", | 20260505 09:25:49.662 | INFO | "huge" : "", | 20260505 09:25:49.662 | INFO | "huge_shared" : " 27 GB", | 20260505 09:25:49.662 | INFO | "malloc" : " 14 GB", | 20260505 09:25:49.662 | INFO | "pinned" : " 29 GB" | 20260505 09:25:49.662 | INFO | } | 20260505 09:25:49.662 | INFO | Perf report for the last 5.13887 seconds: | 20260505 09:25:49.662 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:25:49.662 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:49.662 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:49.662 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:49.662 | INFO | ANALYSIS : 65.2%, 2.9, 100.1ms, 0 | 20260505 09:25:49.662 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:49.662 | INFO | RT_METRICS : 46.5%, 1.0, 24.4ms, 0 | 20260505 09:25:49.662 | INFO | PRE_HQ : 63.0%, 1.9, 60.6ms, 0 | 20260505 09:25:49.662 | INFO | BAZWRITER : 66.0%, 4.1, 140.4ms, 0 | 20260505 09:25:49.662 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260505 09:25:49.662 | INFO | Uploaded 36GiB to GPU in 2.22833s (16.1556GiB/s) | 20260505 09:25:49.662 | INFO | Downloaded 38GiB to GPU in 2.33647s (16.2639GiB/s) | 20260505 09:25:54.799 | INFO | Analyzing chunk frames = [2118144,2118656) | 20260505 09:25:54.799 | INFO | MemInfo: { | 20260505 09:25:54.799 | INFO | "gpu" : " 17 GB", | 20260505 09:25:54.799 | INFO | "huge" : "", | 20260505 09:25:54.799 | INFO | "huge_shared" : " 27 GB", | 20260505 09:25:54.799 | INFO | "malloc" : " 14 GB", | 20260505 09:25:54.799 | INFO | "pinned" : " 29 GB" | 20260505 09:25:54.799 | INFO | } | 20260505 09:25:54.799 | INFO | Perf report for the last 5.13697 seconds: | 20260505 09:25:54.799 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:25:54.799 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:54.799 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:54.799 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:54.799 | INFO | ANALYSIS : 64.6%, 3.0, 100.8ms, 0 | 20260505 09:25:54.799 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:54.799 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:54.799 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:25:54.799 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:54.799 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:54.799 | INFO | Uploaded 36GiB to GPU in 2.22103s (16.2087GiB/s) | 20260505 09:25:54.799 | INFO | Downloaded 34GiB to GPU in 2.10324s (16.1655GiB/s) | 20260505 09:25:59.938 | INFO | Analyzing chunk frames = [2118656,2119168) | 20260505 09:25:59.938 | INFO | MemInfo: { | 20260505 09:25:59.938 | INFO | "gpu" : " 17 GB", | 20260505 09:25:59.938 | INFO | "huge" : "", | 20260505 09:25:59.938 | INFO | "huge_shared" : " 27 GB", | 20260505 09:25:59.938 | INFO | "malloc" : " 14 GB", | 20260505 09:25:59.938 | INFO | "pinned" : " 29 GB" | 20260505 09:25:59.938 | INFO | } | 20260505 09:25:59.938 | INFO | Perf report for the last 5.13866 seconds: | 20260505 09:25:59.938 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:25:59.938 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:59.938 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:59.938 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:59.938 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 09:25:59.938 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:59.938 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:59.938 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 09:25:59.938 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:59.938 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:25:59.938 | INFO | Uploaded 36GiB to GPU in 2.15472s (16.7075GiB/s) | 20260505 09:25:59.938 | INFO | Downloaded 34GiB to GPU in 2.06763s (16.444GiB/s) | 20260505 09:26:05.077 | INFO | Analyzing chunk frames = [2119168,2119680) | 20260505 09:26:05.077 | INFO | MemInfo: { | 20260505 09:26:05.077 | INFO | "gpu" : " 17 GB", | 20260505 09:26:05.077 | INFO | "huge" : "", | 20260505 09:26:05.077 | INFO | "huge_shared" : " 27 GB", | 20260505 09:26:05.077 | INFO | "malloc" : " 14 GB", | 20260505 09:26:05.077 | INFO | "pinned" : " 29 GB" | 20260505 09:26:05.077 | INFO | } | 20260505 09:26:05.078 | INFO | Perf report for the last 5.13977 seconds: | 20260505 09:26:05.078 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:26:05.078 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:05.078 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:05.078 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:05.078 | INFO | ANALYSIS : 63.1%, 3.0, 98.1ms, 0 | 20260505 09:26:05.078 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:05.078 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:05.078 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:26:05.078 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:05.078 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:05.078 | INFO | Uploaded 36GiB to GPU in 2.15872s (16.6766GiB/s) | 20260505 09:26:05.078 | INFO | Downloaded 34GiB to GPU in 2.06896s (16.4333GiB/s) | 20260505 09:26:10.217 | INFO | Analyzing chunk frames = [2119680,2120192) | 20260505 09:26:10.217 | INFO | MemInfo: { | 20260505 09:26:10.217 | INFO | "gpu" : " 17 GB", | 20260505 09:26:10.217 | INFO | "huge" : "", | 20260505 09:26:10.217 | INFO | "huge_shared" : " 27 GB", | 20260505 09:26:10.217 | INFO | "malloc" : " 14 GB", | 20260505 09:26:10.217 | INFO | "pinned" : " 29 GB" | 20260505 09:26:10.217 | INFO | } | 20260505 09:26:10.217 | INFO | Perf report for the last 5.13905 seconds: | 20260505 09:26:10.217 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:26:10.217 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:10.217 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:10.217 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:10.217 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 09:26:10.217 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:10.217 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:10.217 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 09:26:10.217 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:10.217 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:10.217 | INFO | Uploaded 36GiB to GPU in 2.23344s (16.1186GiB/s) | 20260505 09:26:10.217 | INFO | Downloaded 34GiB to GPU in 2.10384s (16.1609GiB/s) | 20260505 09:26:15.357 | INFO | Analyzing chunk frames = [2120192,2120704) | 20260505 09:26:15.357 | INFO | MemInfo: { | 20260505 09:26:15.357 | INFO | "gpu" : " 17 GB", | 20260505 09:26:15.357 | INFO | "huge" : "", | 20260505 09:26:15.357 | INFO | "huge_shared" : " 27 GB", | 20260505 09:26:15.357 | INFO | "malloc" : " 14 GB", | 20260505 09:26:15.357 | INFO | "pinned" : " 29 GB" | 20260505 09:26:15.357 | INFO | } | 20260505 09:26:15.357 | INFO | Perf report for the last 5.14074 seconds: | 20260505 09:26:15.357 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:26:15.357 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:15.357 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:15.357 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:15.357 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 09:26:15.357 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:15.357 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:15.357 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:26:15.357 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:15.357 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:15.358 | INFO | Uploaded 36GiB to GPU in 2.2373s (16.0908GiB/s) | 20260505 09:26:15.358 | INFO | Downloaded 34GiB to GPU in 2.10236s (16.1723GiB/s) | 20260505 09:26:20.498 | INFO | Analyzing chunk frames = [2120704,2121216) | 20260505 09:26:20.498 | INFO | MemInfo: { | 20260505 09:26:20.498 | INFO | "gpu" : " 17 GB", | 20260505 09:26:20.498 | INFO | "huge" : "", | 20260505 09:26:20.498 | INFO | "huge_shared" : " 27 GB", | 20260505 09:26:20.498 | INFO | "malloc" : " 14 GB", | 20260505 09:26:20.498 | INFO | "pinned" : " 29 GB" | 20260505 09:26:20.498 | INFO | } | 20260505 09:26:20.498 | INFO | Perf report for the last 5.14085 seconds: | 20260505 09:26:20.498 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:26:20.498 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:20.498 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:20.498 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:20.498 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 09:26:20.498 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:20.498 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:20.498 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260505 09:26:20.498 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:20.498 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:20.498 | INFO | Uploaded 36GiB to GPU in 2.2207s (16.2111GiB/s) | 20260505 09:26:20.498 | INFO | Downloaded 34GiB to GPU in 2.13456s (15.9284GiB/s) | 20260505 09:26:25.636 | INFO | Analyzing chunk frames = [2121216,2121728) | 20260505 09:26:25.636 | INFO | MemInfo: { | 20260505 09:26:25.636 | INFO | "gpu" : " 17 GB", | 20260505 09:26:25.636 | INFO | "huge" : "", | 20260505 09:26:25.636 | INFO | "huge_shared" : " 27 GB", | 20260505 09:26:25.636 | INFO | "malloc" : " 14 GB", | 20260505 09:26:25.636 | INFO | "pinned" : " 29 GB" | 20260505 09:26:25.636 | INFO | } | 20260505 09:26:25.636 | INFO | Perf report for the last 5.13743 seconds: | 20260505 09:26:25.636 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:26:25.636 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:25.636 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:25.636 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:25.636 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 09:26:25.636 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:25.636 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:25.636 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:26:25.636 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:25.636 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:25.636 | INFO | Uploaded 36GiB to GPU in 2.17534s (16.5492GiB/s) | 20260505 09:26:25.636 | INFO | Downloaded 34GiB to GPU in 2.05146s (16.5735GiB/s) | 20260505 09:26:25.636 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 09:26:25.636 | INFO | Total mem usage: 15.3 GiB | 20260505 09:26:25.636 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 09:26:25.636 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.45%) | 20260505 09:26:25.636 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 09:26:25.636 | INFO | High Water Breakdown: | 20260505 09:26:25.636 | INFO | 60.9% : 8.76 GiB ( 93.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 09:26:25.636 | INFO | 39% : 5.61 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 09:26:25.636 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 09:26:25.636 | INFO | ------------------------------------------ | 20260505 09:26:25.636 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:26:25.636 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 09:26:25.636 | INFO | Total mem usage: 26.5 GiB | 20260505 09:26:25.636 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 09:26:25.636 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 09:26:25.636 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 09:26:25.636 | INFO | High Water Breakdown: | 20260505 09:26:25.636 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:26:25.636 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:26:25.636 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:26:25.636 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:26:25.636 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:26:25.636 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:26:25.636 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 09:26:25.636 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 09:26:25.636 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:26:25.636 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 09:26:25.636 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 09:26:25.636 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 09:26:25.636 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 09:26:25.636 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 09:26:25.637 | INFO | ------------------------------------------ | 20260505 09:26:25.637 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:26:25.637 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 09:26:25.637 | INFO | Total mem usage: 25.6 GiB | 20260505 09:26:25.637 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 09:26:25.637 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 09:26:25.637 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 09:26:25.637 | INFO | High Water Breakdown: | 20260505 09:26:25.637 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 09:26:25.637 | INFO | ------------------------------------------ | 20260505 09:26:25.637 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:26:25.637 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 09:26:25.637 | INFO | Total mem usage: 15.7 GiB | 20260505 09:26:25.637 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 09:26:25.637 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 09:26:25.637 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 09:26:25.637 | INFO | High Water Breakdown: | 20260505 09:26:25.637 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 09:26:25.637 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 09:26:25.637 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 09:26:25.637 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 09:26:25.637 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 09:26:25.637 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 09:26:25.637 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 09:26:25.637 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 09:26:25.637 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 09:26:25.637 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:26:25.637 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 09:26:25.637 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:26:25.637 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 09:26:25.637 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 09:26:25.637 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 09:26:25.637 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 09:26:25.637 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:26:25.637 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 09:26:25.637 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:26:25.637 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:26:25.637 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:26:25.637 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 09:26:25.637 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 09:26:25.637 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:26:25.637 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 09:26:25.637 | INFO | ------------------------------------------ | 20260505 09:26:25.637 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:26:25.637 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 09:26:25.637 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:26:25.637 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:26:25.637 | INFO | PerfReport: Full Execution -- 3.25e+01 / 7.65e+01, 1.36e+01 / 2.87e+00 | 20260505 09:26:25.637 | INFO | PerfReport: 0.29%, AggregateStats -- 9.34e-02 / 1.03e-01, 1.16e-02 / 6.65e-03 | 20260505 09:26:25.637 | INFO | PerfReport: 11.64%, Baseline -- 3.78e+00 / 4.38e+00, 6.08e-01 / 3.16e-01 | 20260505 09:26:25.637 | INFO | PerfReport: 3.30%, Histogram -- 1.07e+00 / 1.46e+00, 2.06e-01 / 1.14e-01 | 20260505 09:26:25.637 | INFO | PerfReport: 15.60%, DME -- 4.05e+01 / 4.49e+01, 4.45e+00 / 2.30e+00 | 20260505 09:26:25.637 | INFO | PerfReport: 56.50%, FrameLabeling -- 1.83e+01 / 2.06e+01, 1.85e+00 / 2.23e+00 | 20260505 09:26:25.637 | INFO | PerfReport: 7.46%, PulseAccumulating -- 2.42e+00 / 2.99e+00, 4.20e-01 / 2.44e-01 | 20260505 09:26:25.637 | INFO | PerfReport: 3.84%, Metrics -- 1.25e+00 / 1.56e+00, 3.61e-01 / 1.79e-01 | 20260505 09:26:25.637 | INFO | PerfReport: 1.36%, PulseSerializing -- 4.42e-01 / 4.91e-01, 8.30e-02 / 3.57e-02 | 20260505 09:26:25.637 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 09:26:25.637 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:26:25.637 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:26:25.637 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.36e+01 / 0.00e+00 | 20260505 09:26:25.637 | INFO | PerfReport: 22.55%, Upload -- 2.29e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260505 09:26:25.637 | INFO | PerfReport: 21.77%, Download -- 2.21e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260505 09:26:25.637 | INFO | PerfReport: 31.99%, Compute -- 3.25e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 09:26:25.637 | INFO | PerfReport: 1.82%, LockUpload -- 1.85e+00 / 0.00e+00, 7.43e+00 / 0.00e+00 | 20260505 09:26:25.637 | INFO | PerfReport: 1.11%, LockDownload -- 1.13e+00 / 0.00e+00, 2.73e+00 / 0.00e+00 | 20260505 09:26:25.637 | INFO | PerfReport: 20.73%, LockCompute -- 2.10e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260505 09:26:25.637 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.06e-02 / 0.00e+00 | 20260505 09:26:25.768 | INFO | Starting a round of Baz IO | 20260505 09:26:29.045 | INFO | Populating RTMetrics report | 20260505 09:26:29.045 | INFO | RTMetrics report populated | 20260505 09:26:29.045 | INFO | Opening tmp RTMetrics json file | 20260505 09:26:29.064 | INFO | Serializing RTMetrics to json | 20260505 09:26:29.067 | INFO | Serialization done, swapping tmp file now... | 20260505 09:26:29.069 | INFO | RTMetrics done | 20260505 09:26:29.194 | INFO | Finished a round of Baz IO: Wrote 2.442 GiB in 3.425 seconds | 20260505 09:26:29.194 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 26.10% / 0.01% / 73.88% | 20260505 09:26:29.194 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.99311% / 1.19209e-05% / 8.34465e-05% | 20260505 09:26:29.194 | INFO | Baz IO overall: | 20260505 09:26:29.194 | INFO | BytesWritten : 1224719140235 | 20260505 09:26:29.194 | INFO | NumEvents : 582810677732 | 20260505 09:26:30.775 | INFO | Analyzing chunk frames = [2121728,2122240) | 20260505 09:26:30.775 | INFO | MemInfo: { | 20260505 09:26:30.775 | INFO | "gpu" : " 17 GB", | 20260505 09:26:30.775 | INFO | "huge" : "", | 20260505 09:26:30.775 | INFO | "huge_shared" : " 27 GB", | 20260505 09:26:30.775 | INFO | "malloc" : " 14 GB", | 20260505 09:26:30.775 | INFO | "pinned" : " 29 GB" | 20260505 09:26:30.775 | INFO | } | 20260505 09:26:30.775 | INFO | Perf report for the last 5.13928 seconds: | 20260505 09:26:30.775 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:26:30.775 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:30.775 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:30.775 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:30.775 | INFO | ANALYSIS : 65.2%, 2.9, 99.8ms, 0 | 20260505 09:26:30.775 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:30.775 | INFO | RT_METRICS : 46.2%, 1.0, 24.2ms, 0 | 20260505 09:26:30.775 | INFO | PRE_HQ : 62.9%, 1.8, 60.2ms, 0 | 20260505 09:26:30.775 | INFO | BAZWRITER : 65.9%, 4.2, 141.4ms, 0 | 20260505 09:26:30.775 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 09:26:30.775 | INFO | Uploaded 36GiB to GPU in 2.2238s (16.1885GiB/s) | 20260505 09:26:30.775 | INFO | Downloaded 38GiB to GPU in 2.35358s (16.1456GiB/s) | 20260505 09:26:35.913 | INFO | Analyzing chunk frames = [2122240,2122752) | 20260505 09:26:35.913 | INFO | MemInfo: { | 20260505 09:26:35.913 | INFO | "gpu" : " 17 GB", | 20260505 09:26:35.913 | INFO | "huge" : "", | 20260505 09:26:35.913 | INFO | "huge_shared" : " 27 GB", | 20260505 09:26:35.913 | INFO | "malloc" : " 14 GB", | 20260505 09:26:35.913 | INFO | "pinned" : " 29 GB" | 20260505 09:26:35.913 | INFO | } | 20260505 09:26:35.913 | INFO | Perf report for the last 5.13815 seconds: | 20260505 09:26:35.913 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:26:35.913 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:35.913 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:35.913 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:35.913 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 09:26:35.913 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:35.913 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:35.913 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 09:26:35.913 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:35.913 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:35.913 | INFO | Uploaded 36GiB to GPU in 2.23456s (16.1105GiB/s) | 20260505 09:26:35.913 | INFO | Downloaded 34GiB to GPU in 2.11684s (16.0617GiB/s) | 20260505 09:26:41.052 | INFO | Analyzing chunk frames = [2122752,2123264) | 20260505 09:26:41.052 | INFO | MemInfo: { | 20260505 09:26:41.052 | INFO | "gpu" : " 17 GB", | 20260505 09:26:41.052 | INFO | "huge" : "", | 20260505 09:26:41.052 | INFO | "huge_shared" : " 27 GB", | 20260505 09:26:41.052 | INFO | "malloc" : " 14 GB", | 20260505 09:26:41.052 | INFO | "pinned" : " 29 GB" | 20260505 09:26:41.052 | INFO | } | 20260505 09:26:41.052 | INFO | Perf report for the last 5.13918 seconds: | 20260505 09:26:41.052 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:26:41.052 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:41.052 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:41.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:41.052 | INFO | ANALYSIS : 63.4%, 3.0, 98.6ms, 0 | 20260505 09:26:41.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:41.052 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:41.052 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 09:26:41.052 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:41.052 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:41.052 | INFO | Uploaded 36GiB to GPU in 2.17224s (16.5728GiB/s) | 20260505 09:26:41.053 | INFO | Downloaded 34GiB to GPU in 2.06922s (16.4314GiB/s) | 20260505 09:26:46.192 | INFO | Analyzing chunk frames = [2123264,2123776) | 20260505 09:26:46.192 | INFO | MemInfo: { | 20260505 09:26:46.192 | INFO | "gpu" : " 17 GB", | 20260505 09:26:46.192 | INFO | "huge" : "", | 20260505 09:26:46.192 | INFO | "huge_shared" : " 27 GB", | 20260505 09:26:46.192 | INFO | "malloc" : " 14 GB", | 20260505 09:26:46.192 | INFO | "pinned" : " 29 GB" | 20260505 09:26:46.192 | INFO | } | 20260505 09:26:46.193 | INFO | Perf report for the last 5.14021 seconds: | 20260505 09:26:46.193 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:26:46.193 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:46.193 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:46.193 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:46.193 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 09:26:46.193 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:46.193 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:46.193 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 09:26:46.193 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:46.193 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:46.193 | INFO | Uploaded 36GiB to GPU in 2.16436s (16.6331GiB/s) | 20260505 09:26:46.193 | INFO | Downloaded 34GiB to GPU in 2.07195s (16.4096GiB/s) | 20260505 09:26:51.331 | INFO | Analyzing chunk frames = [2123776,2124288) | 20260505 09:26:51.331 | INFO | MemInfo: { | 20260505 09:26:51.331 | INFO | "gpu" : " 17 GB", | 20260505 09:26:51.331 | INFO | "huge" : "", | 20260505 09:26:51.331 | INFO | "huge_shared" : " 27 GB", | 20260505 09:26:51.331 | INFO | "malloc" : " 14 GB", | 20260505 09:26:51.331 | INFO | "pinned" : " 29 GB" | 20260505 09:26:51.331 | INFO | } | 20260505 09:26:51.331 | INFO | Perf report for the last 5.13892 seconds: | 20260505 09:26:51.331 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:26:51.331 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:51.331 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:51.331 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:51.331 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 09:26:51.332 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:51.332 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:51.332 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 09:26:51.332 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:51.332 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:51.332 | INFO | Uploaded 36GiB to GPU in 2.23891s (16.0793GiB/s) | 20260505 09:26:51.332 | INFO | Downloaded 34GiB to GPU in 2.09792s (16.2065GiB/s) | 20260505 09:26:56.472 | INFO | Analyzing chunk frames = [2124288,2124800) | 20260505 09:26:56.472 | INFO | MemInfo: { | 20260505 09:26:56.472 | INFO | "gpu" : " 17 GB", | 20260505 09:26:56.472 | INFO | "huge" : "", | 20260505 09:26:56.472 | INFO | "huge_shared" : " 27 GB", | 20260505 09:26:56.472 | INFO | "malloc" : " 14 GB", | 20260505 09:26:56.472 | INFO | "pinned" : " 29 GB" | 20260505 09:26:56.472 | INFO | } | 20260505 09:26:56.472 | INFO | Perf report for the last 5.1404 seconds: | 20260505 09:26:56.472 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:26:56.472 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:56.472 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:56.472 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:56.472 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 09:26:56.472 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:56.472 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:56.472 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 09:26:56.472 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:56.472 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:26:56.472 | INFO | Uploaded 36GiB to GPU in 2.24156s (16.0602GiB/s) | 20260505 09:26:56.472 | INFO | Downloaded 34GiB to GPU in 2.09805s (16.2055GiB/s) | 20260505 09:27:01.611 | INFO | Analyzing chunk frames = [2124800,2125312) | 20260505 09:27:01.611 | INFO | MemInfo: { | 20260505 09:27:01.611 | INFO | "gpu" : " 17 GB", | 20260505 09:27:01.611 | INFO | "huge" : "", | 20260505 09:27:01.611 | INFO | "huge_shared" : " 27 GB", | 20260505 09:27:01.611 | INFO | "malloc" : " 14 GB", | 20260505 09:27:01.611 | INFO | "pinned" : " 29 GB" | 20260505 09:27:01.611 | INFO | } | 20260505 09:27:01.611 | INFO | Perf report for the last 5.13938 seconds: | 20260505 09:27:01.611 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:27:01.611 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:01.611 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:01.611 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:01.611 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 09:27:01.611 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:01.611 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:01.611 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 09:27:01.611 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:01.611 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:01.611 | INFO | Uploaded 36GiB to GPU in 2.20616s (16.318GiB/s) | 20260505 09:27:01.611 | INFO | Downloaded 34GiB to GPU in 2.13962s (15.8907GiB/s) | 20260505 09:27:06.750 | INFO | Analyzing chunk frames = [2125312,2125824) | 20260505 09:27:06.750 | INFO | MemInfo: { | 20260505 09:27:06.750 | INFO | "gpu" : " 17 GB", | 20260505 09:27:06.750 | INFO | "huge" : "", | 20260505 09:27:06.750 | INFO | "huge_shared" : " 27 GB", | 20260505 09:27:06.750 | INFO | "malloc" : " 14 GB", | 20260505 09:27:06.750 | INFO | "pinned" : " 29 GB" | 20260505 09:27:06.750 | INFO | } | 20260505 09:27:06.750 | INFO | Perf report for the last 5.1387 seconds: | 20260505 09:27:06.750 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:27:06.750 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:06.750 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:06.750 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:06.750 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 09:27:06.750 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:06.750 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:06.750 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 09:27:06.750 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:06.750 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:06.750 | INFO | Uploaded 36GiB to GPU in 2.18558s (16.4716GiB/s) | 20260505 09:27:06.750 | INFO | Downloaded 34GiB to GPU in 2.05176s (16.5711GiB/s) | 20260505 09:27:06.886 | INFO | Starting a round of Baz IO | 20260505 09:27:10.146 | INFO | Populating RTMetrics report | 20260505 09:27:10.146 | INFO | RTMetrics report populated | 20260505 09:27:10.146 | INFO | Opening tmp RTMetrics json file | 20260505 09:27:10.159 | INFO | Serializing RTMetrics to json | 20260505 09:27:10.167 | INFO | Serialization done, swapping tmp file now... | 20260505 09:27:10.186 | INFO | RTMetrics done | 20260505 09:27:10.283 | INFO | Finished a round of Baz IO: Wrote 2.439 GiB in 3.397 seconds | 20260505 09:27:10.284 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 26.09% / 0.02% / 73.90% | 20260505 09:27:10.284 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.98203% / 1.98682e-05% / 0.000103315% | 20260505 09:27:10.284 | INFO | Baz IO overall: | 20260505 09:27:10.284 | INFO | BytesWritten : 1227338038325 | 20260505 09:27:10.284 | INFO | NumEvents : 584062266953 | 20260505 09:27:11.889 | INFO | Analyzing chunk frames = [2125824,2126336) | 20260505 09:27:11.889 | INFO | MemInfo: { | 20260505 09:27:11.889 | INFO | "gpu" : " 17 GB", | 20260505 09:27:11.889 | INFO | "huge" : "", | 20260505 09:27:11.889 | INFO | "huge_shared" : " 27 GB", | 20260505 09:27:11.889 | INFO | "malloc" : " 14 GB", | 20260505 09:27:11.889 | INFO | "pinned" : " 29 GB" | 20260505 09:27:11.889 | INFO | } | 20260505 09:27:11.889 | INFO | Perf report for the last 5.13942 seconds: | 20260505 09:27:11.889 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:27:11.889 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:11.889 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:11.889 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:11.889 | INFO | ANALYSIS : 64.9%, 2.9, 99.0ms, 0 | 20260505 09:27:11.889 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:11.889 | INFO | RT_METRICS : 46.5%, 1.0, 24.4ms, 0 | 20260505 09:27:11.889 | INFO | PRE_HQ : 63.3%, 1.8, 60.5ms, 0 | 20260505 09:27:11.889 | INFO | BAZWRITER : 65.4%, 4.2, 141.2ms, 0 | 20260505 09:27:11.889 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 09:27:11.890 | INFO | Uploaded 36GiB to GPU in 2.26053s (15.9255GiB/s) | 20260505 09:27:11.890 | INFO | Downloaded 38GiB to GPU in 2.38878s (15.9077GiB/s) | 20260505 09:27:17.028 | INFO | Analyzing chunk frames = [2126336,2126848) | 20260505 09:27:17.028 | INFO | MemInfo: { | 20260505 09:27:17.028 | INFO | "gpu" : " 17 GB", | 20260505 09:27:17.028 | INFO | "huge" : "", | 20260505 09:27:17.028 | INFO | "huge_shared" : " 27 GB", | 20260505 09:27:17.028 | INFO | "malloc" : " 14 GB", | 20260505 09:27:17.028 | INFO | "pinned" : " 29 GB" | 20260505 09:27:17.028 | INFO | } | 20260505 09:27:17.028 | INFO | Perf report for the last 5.13856 seconds: | 20260505 09:27:17.028 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:27:17.028 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:17.028 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:17.028 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:17.028 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 09:27:17.028 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:17.028 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:17.028 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 09:27:17.028 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:17.028 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:17.028 | INFO | Uploaded 36GiB to GPU in 2.19361s (16.4113GiB/s) | 20260505 09:27:17.028 | INFO | Downloaded 34GiB to GPU in 2.10584s (16.1456GiB/s) | 20260505 09:27:22.168 | INFO | Analyzing chunk frames = [2126848,2127360) | 20260505 09:27:22.168 | INFO | MemInfo: { | 20260505 09:27:22.168 | INFO | "gpu" : " 17 GB", | 20260505 09:27:22.168 | INFO | "huge" : "", | 20260505 09:27:22.168 | INFO | "huge_shared" : " 27 GB", | 20260505 09:27:22.168 | INFO | "malloc" : " 14 GB", | 20260505 09:27:22.168 | INFO | "pinned" : " 29 GB" | 20260505 09:27:22.168 | INFO | } | 20260505 09:27:22.168 | INFO | Perf report for the last 5.14001 seconds: | 20260505 09:27:22.168 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:27:22.168 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:22.168 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:22.168 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:22.168 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 09:27:22.168 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:22.168 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:22.168 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 09:27:22.168 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:22.168 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:22.168 | INFO | Uploaded 36GiB to GPU in 2.15816s (16.6809GiB/s) | 20260505 09:27:22.168 | INFO | Downloaded 34GiB to GPU in 2.06996s (16.4254GiB/s) | 20260505 09:27:27.307 | INFO | Analyzing chunk frames = [2127360,2127872) | 20260505 09:27:27.307 | INFO | MemInfo: { | 20260505 09:27:27.307 | INFO | "gpu" : " 17 GB", | 20260505 09:27:27.307 | INFO | "huge" : "", | 20260505 09:27:27.307 | INFO | "huge_shared" : " 27 GB", | 20260505 09:27:27.307 | INFO | "malloc" : " 14 GB", | 20260505 09:27:27.307 | INFO | "pinned" : " 29 GB" | 20260505 09:27:27.307 | INFO | } | 20260505 09:27:27.307 | INFO | Perf report for the last 5.1387 seconds: | 20260505 09:27:27.307 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:27:27.307 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:27.307 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:27.307 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:27.307 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 09:27:27.307 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:27.307 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:27.307 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 09:27:27.307 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:27.307 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:27.307 | INFO | Uploaded 36GiB to GPU in 2.15819s (16.6806GiB/s) | 20260505 09:27:27.307 | INFO | Downloaded 34GiB to GPU in 2.07177s (16.4111GiB/s) | 20260505 09:27:32.446 | INFO | Analyzing chunk frames = [2127872,2128384) | 20260505 09:27:32.446 | INFO | MemInfo: { | 20260505 09:27:32.446 | INFO | "gpu" : " 17 GB", | 20260505 09:27:32.446 | INFO | "huge" : "", | 20260505 09:27:32.446 | INFO | "huge_shared" : " 27 GB", | 20260505 09:27:32.446 | INFO | "malloc" : " 14 GB", | 20260505 09:27:32.446 | INFO | "pinned" : " 29 GB" | 20260505 09:27:32.446 | INFO | } | 20260505 09:27:32.446 | INFO | Perf report for the last 5.13954 seconds: | 20260505 09:27:32.446 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:27:32.446 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:32.446 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:32.446 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:32.446 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 09:27:32.446 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:32.446 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:32.446 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 09:27:32.446 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:32.446 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:32.446 | INFO | Uploaded 36GiB to GPU in 2.23549s (16.1039GiB/s) | 20260505 09:27:32.447 | INFO | Downloaded 34GiB to GPU in 2.10955s (16.1172GiB/s) | 20260505 09:27:37.586 | INFO | Analyzing chunk frames = [2128384,2128896) | 20260505 09:27:37.586 | INFO | MemInfo: { | 20260505 09:27:37.586 | INFO | "gpu" : " 17 GB", | 20260505 09:27:37.586 | INFO | "huge" : "", | 20260505 09:27:37.586 | INFO | "huge_shared" : " 27 GB", | 20260505 09:27:37.586 | INFO | "malloc" : " 14 GB", | 20260505 09:27:37.586 | INFO | "pinned" : " 29 GB" | 20260505 09:27:37.586 | INFO | } | 20260505 09:27:37.586 | INFO | Perf report for the last 5.14014 seconds: | 20260505 09:27:37.586 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:27:37.586 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:37.586 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:37.586 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:37.586 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 09:27:37.586 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:37.586 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:37.586 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 09:27:37.586 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:37.586 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:37.587 | INFO | Uploaded 36GiB to GPU in 2.23881s (16.08GiB/s) | 20260505 09:27:37.587 | INFO | Downloaded 34GiB to GPU in 2.10496s (16.1523GiB/s) | 20260505 09:27:42.725 | INFO | Analyzing chunk frames = [2128896,2129408) | 20260505 09:27:42.725 | INFO | MemInfo: { | 20260505 09:27:42.725 | INFO | "gpu" : " 17 GB", | 20260505 09:27:42.725 | INFO | "huge" : "", | 20260505 09:27:42.725 | INFO | "huge_shared" : " 27 GB", | 20260505 09:27:42.725 | INFO | "malloc" : " 14 GB", | 20260505 09:27:42.725 | INFO | "pinned" : " 29 GB" | 20260505 09:27:42.725 | INFO | } | 20260505 09:27:42.725 | INFO | Perf report for the last 5.13828 seconds: | 20260505 09:27:42.725 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:27:42.725 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:42.725 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:42.725 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:42.725 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 09:27:42.725 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:42.725 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:42.725 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 09:27:42.725 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:42.725 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:42.725 | INFO | Uploaded 36GiB to GPU in 2.19976s (16.3654GiB/s) | 20260505 09:27:42.725 | INFO | Downloaded 34GiB to GPU in 2.13352s (15.9361GiB/s) | 20260505 09:27:47.864 | INFO | Analyzing chunk frames = [2129408,2129920) | 20260505 09:27:47.864 | INFO | MemInfo: { | 20260505 09:27:47.864 | INFO | "gpu" : " 17 GB", | 20260505 09:27:47.864 | INFO | "huge" : "", | 20260505 09:27:47.864 | INFO | "huge_shared" : " 27 GB", | 20260505 09:27:47.864 | INFO | "malloc" : " 14 GB", | 20260505 09:27:47.864 | INFO | "pinned" : " 29 GB" | 20260505 09:27:47.864 | INFO | } | 20260505 09:27:47.864 | INFO | Perf report for the last 5.13954 seconds: | 20260505 09:27:47.864 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:27:47.864 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:47.864 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:47.864 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:47.864 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 09:27:47.864 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:47.864 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:47.864 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 09:27:47.864 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:47.864 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:47.864 | INFO | Uploaded 36GiB to GPU in 2.20442s (16.3309GiB/s) | 20260505 09:27:47.865 | INFO | Downloaded 34GiB to GPU in 2.05157s (16.5727GiB/s) | 20260505 09:27:47.865 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 09:27:47.865 | INFO | Total mem usage: 15.3 GiB | 20260505 09:27:47.865 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 09:27:47.865 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -5.82%) | 20260505 09:27:47.865 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 09:27:47.865 | INFO | High Water Breakdown: | 20260505 09:27:47.865 | INFO | 61% : 8.84 GiB ( 94.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 09:27:47.865 | INFO | 38.8% : 5.63 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 09:27:47.865 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 09:27:47.865 | INFO | ------------------------------------------ | 20260505 09:27:47.865 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:27:47.865 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 09:27:47.865 | INFO | Total mem usage: 26.5 GiB | 20260505 09:27:47.865 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 09:27:47.865 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 09:27:47.865 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 09:27:47.865 | INFO | High Water Breakdown: | 20260505 09:27:47.865 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:27:47.865 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:27:47.865 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:27:47.865 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:27:47.865 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:27:47.865 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:27:47.865 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 09:27:47.865 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 09:27:47.865 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:27:47.865 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 09:27:47.865 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 09:27:47.865 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 09:27:47.865 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 09:27:47.865 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 09:27:47.865 | INFO | ------------------------------------------ | 20260505 09:27:47.865 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:27:47.865 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 09:27:47.865 | INFO | Total mem usage: 25.6 GiB | 20260505 09:27:47.865 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 09:27:47.865 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 09:27:47.865 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 09:27:47.865 | INFO | High Water Breakdown: | 20260505 09:27:47.865 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 09:27:47.865 | INFO | ------------------------------------------ | 20260505 09:27:47.865 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:27:47.865 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 09:27:47.865 | INFO | Total mem usage: 15.7 GiB | 20260505 09:27:47.865 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 09:27:47.865 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 09:27:47.865 | INFO | Unused Cache: 880 MiB (Waste: 5.48%) | 20260505 09:27:47.865 | INFO | High Water Breakdown: | 20260505 09:27:47.866 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 09:27:47.866 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 09:27:47.866 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 09:27:47.866 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 09:27:47.866 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 09:27:47.866 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 09:27:47.866 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 09:27:47.866 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 09:27:47.866 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 09:27:47.866 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:27:47.866 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 09:27:47.866 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:27:47.866 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 09:27:47.866 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 09:27:47.866 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 09:27:47.866 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 09:27:47.866 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:27:47.866 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 09:27:47.866 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:27:47.866 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:27:47.866 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:27:47.866 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 09:27:47.866 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 09:27:47.866 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:27:47.866 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 09:27:47.866 | INFO | ------------------------------------------ | 20260505 09:27:47.866 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:27:47.866 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 09:27:47.866 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:27:47.866 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:27:47.866 | INFO | PerfReport: Full Execution -- 3.25e+01 / 7.67e+01, 1.37e+01 / 2.38e+00 | 20260505 09:27:47.866 | INFO | PerfReport: 0.29%, AggregateStats -- 9.34e-02 / 1.04e-01, 1.09e-02 / 4.24e-03 | 20260505 09:27:47.866 | INFO | PerfReport: 11.63%, Baseline -- 3.78e+00 / 4.41e+00, 6.14e-01 / 2.93e-01 | 20260505 09:27:47.866 | INFO | PerfReport: 3.32%, Histogram -- 1.08e+00 / 1.47e+00, 2.10e-01 / 1.00e-01 | 20260505 09:27:47.866 | INFO | PerfReport: 15.59%, DME -- 4.05e+01 / 4.47e+01, 4.43e+00 / 2.02e+00 | 20260505 09:27:47.866 | INFO | PerfReport: 56.63%, FrameLabeling -- 1.84e+01 / 2.11e+01, 1.92e+00 / 2.39e+00 | 20260505 09:27:47.866 | INFO | PerfReport: 7.37%, PulseAccumulating -- 2.40e+00 / 2.95e+00, 3.96e-01 / 2.61e-01 | 20260505 09:27:47.866 | INFO | PerfReport: 3.82%, Metrics -- 1.24e+00 / 1.53e+00, 3.61e-01 / 1.94e-01 | 20260505 09:27:47.866 | INFO | PerfReport: 1.36%, PulseSerializing -- 4.41e-01 / 4.84e-01, 8.28e-02 / 2.74e-02 | 20260505 09:27:47.866 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 09:27:47.866 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:27:47.866 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:27:47.866 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.39e+01 / 0.00e+00 | 20260505 09:27:47.866 | INFO | PerfReport: 22.63%, Upload -- 2.30e+01 / 0.00e+00, 8.31e+00 / 0.00e+00 | 20260505 09:27:47.866 | INFO | PerfReport: 21.81%, Download -- 2.22e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260505 09:27:47.866 | INFO | PerfReport: 32.00%, Compute -- 3.25e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 09:27:47.866 | INFO | PerfReport: 1.81%, LockUpload -- 1.84e+00 / 0.00e+00, 7.42e+00 / 0.00e+00 | 20260505 09:27:47.866 | INFO | PerfReport: 1.07%, LockDownload -- 1.09e+00 / 0.00e+00, 2.67e+00 / 0.00e+00 | 20260505 09:27:47.866 | INFO | PerfReport: 20.66%, LockCompute -- 2.10e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260505 09:27:47.866 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 2.34e-02 / 0.00e+00 | 20260505 09:27:47.995 | INFO | Starting a round of Baz IO | 20260505 09:27:51.271 | INFO | Populating RTMetrics report | 20260505 09:27:51.271 | INFO | RTMetrics report populated | 20260505 09:27:51.271 | INFO | Opening tmp RTMetrics json file | 20260505 09:27:51.313 | INFO | Serializing RTMetrics to json | 20260505 09:27:51.346 | INFO | Serialization done, swapping tmp file now... | 20260505 09:27:51.347 | INFO | RTMetrics done | 20260505 09:27:51.411 | INFO | Finished a round of Baz IO: Wrote 2.44 GiB in 3.415 seconds | 20260505 09:27:51.411 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 26.07% / 0.02% / 73.91% | 20260505 09:27:51.411 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.97503% / 3.17891e-05% / 6.75519e-05% | 20260505 09:27:51.411 | INFO | Baz IO overall: | 20260505 09:27:51.411 | INFO | BytesWritten : 1229957766328 | 20260505 09:27:51.411 | INFO | NumEvents : 585314564036 | 20260505 09:27:53.003 | INFO | Analyzing chunk frames = [2129920,2130432) | 20260505 09:27:53.003 | INFO | MemInfo: { | 20260505 09:27:53.003 | INFO | "gpu" : " 17 GB", | 20260505 09:27:53.003 | INFO | "huge" : "", | 20260505 09:27:53.003 | INFO | "huge_shared" : " 27 GB", | 20260505 09:27:53.003 | INFO | "malloc" : " 14 GB", | 20260505 09:27:53.003 | INFO | "pinned" : " 29 GB" | 20260505 09:27:53.003 | INFO | } | 20260505 09:27:53.003 | INFO | Perf report for the last 5.13921 seconds: | 20260505 09:27:53.004 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:27:53.004 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:53.004 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:53.004 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:53.004 | INFO | ANALYSIS : 65.2%, 2.9, 99.9ms, 0 | 20260505 09:27:53.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:53.004 | INFO | RT_METRICS : 47.4%, 1.0, 24.9ms, 0 | 20260505 09:27:53.004 | INFO | PRE_HQ : 63.1%, 1.8, 60.6ms, 0 | 20260505 09:27:53.004 | INFO | BAZWRITER : 65.8%, 4.1, 138.3ms, 0 | 20260505 09:27:53.004 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 09:27:53.004 | INFO | Uploaded 36GiB to GPU in 2.20939s (16.2941GiB/s) | 20260505 09:27:53.004 | INFO | Downloaded 38GiB to GPU in 2.33819s (16.2519GiB/s) | 20260505 09:27:58.143 | INFO | Analyzing chunk frames = [2130432,2130944) | 20260505 09:27:58.143 | INFO | MemInfo: { | 20260505 09:27:58.143 | INFO | "gpu" : " 17 GB", | 20260505 09:27:58.143 | INFO | "huge" : "", | 20260505 09:27:58.143 | INFO | "huge_shared" : " 27 GB", | 20260505 09:27:58.143 | INFO | "malloc" : " 14 GB", | 20260505 09:27:58.144 | INFO | "pinned" : " 29 GB" | 20260505 09:27:58.144 | INFO | } | 20260505 09:27:58.144 | INFO | Perf report for the last 5.14013 seconds: | 20260505 09:27:58.144 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:27:58.144 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:58.144 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:58.144 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:58.144 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 09:27:58.144 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:58.144 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:58.144 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 09:27:58.144 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:58.144 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:27:58.144 | INFO | Uploaded 36GiB to GPU in 2.24663s (16.024GiB/s) | 20260505 09:27:58.144 | INFO | Downloaded 34GiB to GPU in 2.1216s (16.0257GiB/s) | 20260505 09:28:03.283 | INFO | Analyzing chunk frames = [2130944,2131456) | 20260505 09:28:03.283 | INFO | MemInfo: { | 20260505 09:28:03.283 | INFO | "gpu" : " 17 GB", | 20260505 09:28:03.283 | INFO | "huge" : "", | 20260505 09:28:03.283 | INFO | "huge_shared" : " 27 GB", | 20260505 09:28:03.283 | INFO | "malloc" : " 14 GB", | 20260505 09:28:03.283 | INFO | "pinned" : " 29 GB" | 20260505 09:28:03.283 | INFO | } | 20260505 09:28:03.283 | INFO | Perf report for the last 5.13911 seconds: | 20260505 09:28:03.283 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:28:03.283 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:03.283 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:03.283 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:03.283 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 09:28:03.283 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:03.283 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:03.283 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 09:28:03.283 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:03.283 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:03.283 | INFO | Uploaded 36GiB to GPU in 2.15774s (16.6841GiB/s) | 20260505 09:28:03.283 | INFO | Downloaded 34GiB to GPU in 2.06972s (16.4274GiB/s) | 20260505 09:28:08.421 | INFO | Analyzing chunk frames = [2131456,2131968) | 20260505 09:28:08.421 | INFO | MemInfo: { | 20260505 09:28:08.421 | INFO | "gpu" : " 17 GB", | 20260505 09:28:08.421 | INFO | "huge" : "", | 20260505 09:28:08.421 | INFO | "huge_shared" : " 27 GB", | 20260505 09:28:08.421 | INFO | "malloc" : " 14 GB", | 20260505 09:28:08.422 | INFO | "pinned" : " 29 GB" | 20260505 09:28:08.422 | INFO | } | 20260505 09:28:08.422 | INFO | Perf report for the last 5.13893 seconds: | 20260505 09:28:08.422 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:28:08.422 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:08.422 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:08.422 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:08.422 | INFO | ANALYSIS : 63.3%, 3.0, 98.3ms, 0 | 20260505 09:28:08.422 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:08.422 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:08.422 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:28:08.422 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:08.422 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:08.422 | INFO | Uploaded 36GiB to GPU in 2.16278s (16.6452GiB/s) | 20260505 09:28:08.422 | INFO | Downloaded 34GiB to GPU in 2.07655s (16.3733GiB/s) | 20260505 09:28:13.562 | INFO | Analyzing chunk frames = [2131968,2132480) | 20260505 09:28:13.562 | INFO | MemInfo: { | 20260505 09:28:13.562 | INFO | "gpu" : " 17 GB", | 20260505 09:28:13.562 | INFO | "huge" : "", | 20260505 09:28:13.562 | INFO | "huge_shared" : " 27 GB", | 20260505 09:28:13.562 | INFO | "malloc" : " 14 GB", | 20260505 09:28:13.562 | INFO | "pinned" : " 29 GB" | 20260505 09:28:13.562 | INFO | } | 20260505 09:28:13.562 | INFO | Perf report for the last 5.14012 seconds: | 20260505 09:28:13.562 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:28:13.562 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:13.562 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:13.562 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:13.562 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 09:28:13.562 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:13.562 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:13.562 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:28:13.562 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:13.562 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:13.562 | INFO | Uploaded 36GiB to GPU in 2.23398s (16.1147GiB/s) | 20260505 09:28:13.562 | INFO | Downloaded 34GiB to GPU in 2.09782s (16.2073GiB/s) | 20260505 09:28:18.705 | INFO | Analyzing chunk frames = [2132480,2132992) | 20260505 09:28:18.705 | INFO | MemInfo: { | 20260505 09:28:18.705 | INFO | "gpu" : " 17 GB", | 20260505 09:28:18.705 | INFO | "huge" : "", | 20260505 09:28:18.705 | INFO | "huge_shared" : " 27 GB", | 20260505 09:28:18.705 | INFO | "malloc" : " 14 GB", | 20260505 09:28:18.705 | INFO | "pinned" : " 29 GB" | 20260505 09:28:18.705 | INFO | } | 20260505 09:28:18.705 | INFO | Perf report for the last 5.14334 seconds: | 20260505 09:28:18.705 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:28:18.705 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:18.705 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:18.705 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:18.705 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 09:28:18.705 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:18.705 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:18.705 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 09:28:18.705 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:18.705 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:18.705 | INFO | Uploaded 36GiB to GPU in 2.23638s (16.0974GiB/s) | 20260505 09:28:18.706 | INFO | Downloaded 34GiB to GPU in 2.10154s (16.1786GiB/s) | 20260505 09:28:23.840 | INFO | Analyzing chunk frames = [2132992,2133504) | 20260505 09:28:23.840 | INFO | MemInfo: { | 20260505 09:28:23.840 | INFO | "gpu" : " 17 GB", | 20260505 09:28:23.840 | INFO | "huge" : "", | 20260505 09:28:23.840 | INFO | "huge_shared" : " 27 GB", | 20260505 09:28:23.840 | INFO | "malloc" : " 14 GB", | 20260505 09:28:23.840 | INFO | "pinned" : " 29 GB" | 20260505 09:28:23.840 | INFO | } | 20260505 09:28:23.840 | INFO | Perf report for the last 5.13485 seconds: | 20260505 09:28:23.840 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:28:23.840 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:23.840 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:23.840 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:23.840 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 09:28:23.840 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:23.840 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:23.840 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 09:28:23.840 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:23.840 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:23.840 | INFO | Uploaded 36GiB to GPU in 2.19574s (16.3954GiB/s) | 20260505 09:28:23.840 | INFO | Downloaded 34GiB to GPU in 2.13205s (15.9471GiB/s) | 20260505 09:28:28.979 | INFO | Analyzing chunk frames = [2133504,2134016) | 20260505 09:28:28.979 | INFO | MemInfo: { | 20260505 09:28:28.979 | INFO | "gpu" : " 17 GB", | 20260505 09:28:28.979 | INFO | "huge" : "", | 20260505 09:28:28.980 | INFO | "huge_shared" : " 27 GB", | 20260505 09:28:28.980 | INFO | "malloc" : " 14 GB", | 20260505 09:28:28.980 | INFO | "pinned" : " 29 GB" | 20260505 09:28:28.980 | INFO | } | 20260505 09:28:28.980 | INFO | Perf report for the last 5.13955 seconds: | 20260505 09:28:28.980 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:28:28.980 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:28.980 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:28.980 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:28.980 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 09:28:28.980 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:28.980 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:28.980 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 09:28:28.980 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:28.980 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:28.980 | INFO | Uploaded 36GiB to GPU in 2.18275s (16.493GiB/s) | 20260505 09:28:28.980 | INFO | Downloaded 34GiB to GPU in 2.05429s (16.5507GiB/s) | 20260505 09:28:29.110 | INFO | Starting a round of Baz IO | 20260505 09:28:32.388 | INFO | Populating RTMetrics report | 20260505 09:28:32.389 | INFO | RTMetrics report populated | 20260505 09:28:32.389 | INFO | Opening tmp RTMetrics json file | 20260505 09:28:32.416 | INFO | Serializing RTMetrics to json | 20260505 09:28:32.440 | INFO | Serialization done, swapping tmp file now... | 20260505 09:28:32.440 | INFO | RTMetrics done | 20260505 09:28:32.519 | INFO | Finished a round of Baz IO: Wrote 2.44 GiB in 3.409 seconds | 20260505 09:28:32.519 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 26.06% / 0.02% / 73.93% | 20260505 09:28:32.519 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.96844% / 3.57628e-05% / 9.13938e-05% | 20260505 09:28:32.519 | INFO | Baz IO overall: | 20260505 09:28:32.519 | INFO | BytesWritten : 1232577658203 | 20260505 09:28:32.519 | INFO | NumEvents : 586567050622 | 20260505 09:28:34.118 | INFO | Analyzing chunk frames = [2134016,2134528) | 20260505 09:28:34.118 | INFO | MemInfo: { | 20260505 09:28:34.118 | INFO | "gpu" : " 17 GB", | 20260505 09:28:34.118 | INFO | "huge" : "", | 20260505 09:28:34.118 | INFO | "huge_shared" : " 27 GB", | 20260505 09:28:34.118 | INFO | "malloc" : " 14 GB", | 20260505 09:28:34.118 | INFO | "pinned" : " 29 GB" | 20260505 09:28:34.118 | INFO | } | 20260505 09:28:34.118 | INFO | Perf report for the last 5.13858 seconds: | 20260505 09:28:34.118 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:28:34.118 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:34.118 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:34.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:34.118 | INFO | ANALYSIS : 65.2%, 2.9, 100.5ms, 0 | 20260505 09:28:34.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:34.118 | INFO | RT_METRICS : 46.7%, 1.0, 24.5ms, 0 | 20260505 09:28:34.118 | INFO | PRE_HQ : 62.8%, 1.8, 60.2ms, 0 | 20260505 09:28:34.118 | INFO | BAZWRITER : 65.7%, 4.1, 137.6ms, 0 | 20260505 09:28:34.118 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260505 09:28:34.118 | INFO | Uploaded 36GiB to GPU in 2.22369s (16.1893GiB/s) | 20260505 09:28:34.119 | INFO | Downloaded 38GiB to GPU in 2.33447s (16.2778GiB/s) | 20260505 09:28:39.257 | INFO | Analyzing chunk frames = [2134528,2135040) | 20260505 09:28:39.257 | INFO | MemInfo: { | 20260505 09:28:39.257 | INFO | "gpu" : " 17 GB", | 20260505 09:28:39.257 | INFO | "huge" : "", | 20260505 09:28:39.257 | INFO | "huge_shared" : " 27 GB", | 20260505 09:28:39.257 | INFO | "malloc" : " 14 GB", | 20260505 09:28:39.257 | INFO | "pinned" : " 29 GB" | 20260505 09:28:39.257 | INFO | } | 20260505 09:28:39.257 | INFO | Perf report for the last 5.1391 seconds: | 20260505 09:28:39.257 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:28:39.257 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:39.257 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:39.257 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:39.257 | INFO | ANALYSIS : 64.3%, 3.0, 100.3ms, 0 | 20260505 09:28:39.257 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:39.257 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:39.257 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 09:28:39.257 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:39.257 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:39.258 | INFO | Uploaded 36GiB to GPU in 2.21643s (16.2424GiB/s) | 20260505 09:28:39.258 | INFO | Downloaded 34GiB to GPU in 2.121s (16.0302GiB/s) | 20260505 09:28:44.399 | INFO | Analyzing chunk frames = [2135040,2135552) | 20260505 09:28:44.399 | INFO | MemInfo: { | 20260505 09:28:44.399 | INFO | "gpu" : " 17 GB", | 20260505 09:28:44.399 | INFO | "huge" : "", | 20260505 09:28:44.399 | INFO | "huge_shared" : " 27 GB", | 20260505 09:28:44.399 | INFO | "malloc" : " 14 GB", | 20260505 09:28:44.399 | INFO | "pinned" : " 29 GB" | 20260505 09:28:44.399 | INFO | } | 20260505 09:28:44.399 | INFO | Perf report for the last 5.14191 seconds: | 20260505 09:28:44.399 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:28:44.399 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:44.399 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:44.399 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:44.399 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 09:28:44.399 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:44.399 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:44.399 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 09:28:44.399 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:44.399 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:44.399 | INFO | Uploaded 36GiB to GPU in 2.16042s (16.6634GiB/s) | 20260505 09:28:44.400 | INFO | Downloaded 34GiB to GPU in 2.07079s (16.4189GiB/s) | 20260505 09:28:49.537 | INFO | Analyzing chunk frames = [2135552,2136064) | 20260505 09:28:49.537 | INFO | MemInfo: { | 20260505 09:28:49.537 | INFO | "gpu" : " 17 GB", | 20260505 09:28:49.537 | INFO | "huge" : "", | 20260505 09:28:49.537 | INFO | "huge_shared" : " 27 GB", | 20260505 09:28:49.537 | INFO | "malloc" : " 14 GB", | 20260505 09:28:49.537 | INFO | "pinned" : " 29 GB" | 20260505 09:28:49.537 | INFO | } | 20260505 09:28:49.537 | INFO | Perf report for the last 5.138 seconds: | 20260505 09:28:49.537 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:28:49.537 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:49.537 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:49.537 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:49.537 | INFO | ANALYSIS : 63.2%, 3.0, 98.1ms, 0 | 20260505 09:28:49.537 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:49.537 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:49.537 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 09:28:49.537 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:49.537 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:49.537 | INFO | Uploaded 36GiB to GPU in 2.1596s (16.6698GiB/s) | 20260505 09:28:49.537 | INFO | Downloaded 34GiB to GPU in 2.0688s (16.4347GiB/s) | 20260505 09:28:54.675 | INFO | Analyzing chunk frames = [2136064,2136576) | 20260505 09:28:54.675 | INFO | MemInfo: { | 20260505 09:28:54.675 | INFO | "gpu" : " 17 GB", | 20260505 09:28:54.675 | INFO | "huge" : "", | 20260505 09:28:54.675 | INFO | "huge_shared" : " 27 GB", | 20260505 09:28:54.675 | INFO | "malloc" : " 14 GB", | 20260505 09:28:54.675 | INFO | "pinned" : " 29 GB" | 20260505 09:28:54.675 | INFO | } | 20260505 09:28:54.675 | INFO | Perf report for the last 5.13819 seconds: | 20260505 09:28:54.675 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:28:54.675 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:54.675 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:54.675 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:54.676 | INFO | ANALYSIS : 63.8%, 3.0, 99.5ms, 0 | 20260505 09:28:54.676 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:54.676 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:54.676 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 09:28:54.676 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:54.676 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:54.676 | INFO | Uploaded 36GiB to GPU in 2.23371s (16.1167GiB/s) | 20260505 09:28:54.676 | INFO | Downloaded 34GiB to GPU in 2.1107s (16.1084GiB/s) | 20260505 09:28:59.816 | INFO | Analyzing chunk frames = [2136576,2137088) | 20260505 09:28:59.816 | INFO | MemInfo: { | 20260505 09:28:59.816 | INFO | "gpu" : " 17 GB", | 20260505 09:28:59.816 | INFO | "huge" : "", | 20260505 09:28:59.816 | INFO | "huge_shared" : " 27 GB", | 20260505 09:28:59.816 | INFO | "malloc" : " 14 GB", | 20260505 09:28:59.816 | INFO | "pinned" : " 29 GB" | 20260505 09:28:59.816 | INFO | } | 20260505 09:28:59.816 | INFO | Perf report for the last 5.14049 seconds: | 20260505 09:28:59.816 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:28:59.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:59.816 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:59.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:59.816 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 09:28:59.816 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:59.816 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:59.816 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 09:28:59.816 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:59.816 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:28:59.816 | INFO | Uploaded 36GiB to GPU in 2.15749s (16.6861GiB/s) | 20260505 09:28:59.816 | INFO | Downloaded 34GiB to GPU in 2.05873s (16.515GiB/s) | 20260505 09:29:04.954 | INFO | Analyzing chunk frames = [2137088,2137600) | 20260505 09:29:04.954 | INFO | MemInfo: { | 20260505 09:29:04.954 | INFO | "gpu" : " 17 GB", | 20260505 09:29:04.954 | INFO | "huge" : "", | 20260505 09:29:04.954 | INFO | "huge_shared" : " 27 GB", | 20260505 09:29:04.954 | INFO | "malloc" : " 14 GB", | 20260505 09:29:04.954 | INFO | "pinned" : " 29 GB" | 20260505 09:29:04.954 | INFO | } | 20260505 09:29:04.954 | INFO | Perf report for the last 5.13826 seconds: | 20260505 09:29:04.954 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:29:04.954 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:04.954 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:04.954 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:04.954 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 09:29:04.954 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:04.954 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:04.954 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 09:29:04.954 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:04.954 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:04.954 | INFO | Uploaded 36GiB to GPU in 2.19596s (16.3938GiB/s) | 20260505 09:29:04.954 | INFO | Downloaded 34GiB to GPU in 2.13239s (15.9445GiB/s) | 20260505 09:29:10.094 | INFO | Analyzing chunk frames = [2137600,2138112) | 20260505 09:29:10.094 | INFO | MemInfo: { | 20260505 09:29:10.094 | INFO | "gpu" : " 17 GB", | 20260505 09:29:10.094 | INFO | "huge" : "", | 20260505 09:29:10.094 | INFO | "huge_shared" : " 27 GB", | 20260505 09:29:10.094 | INFO | "malloc" : " 14 GB", | 20260505 09:29:10.094 | INFO | "pinned" : " 29 GB" | 20260505 09:29:10.094 | INFO | } | 20260505 09:29:10.094 | INFO | Perf report for the last 5.13973 seconds: | 20260505 09:29:10.094 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:29:10.094 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:10.094 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:10.094 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:10.094 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 09:29:10.094 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:10.094 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:10.094 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:29:10.094 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:10.094 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:10.094 | INFO | Uploaded 36GiB to GPU in 2.20567s (16.3216GiB/s) | 20260505 09:29:10.094 | INFO | Downloaded 34GiB to GPU in 2.05488s (16.546GiB/s) | 20260505 09:29:10.094 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 09:29:10.094 | INFO | Total mem usage: 15.3 GiB | 20260505 09:29:10.094 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 09:29:10.094 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.56%) | 20260505 09:29:10.094 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 09:29:10.094 | INFO | High Water Breakdown: | 20260505 09:29:10.094 | INFO | 60.9% : 8.76 GiB ( 93.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 09:29:10.094 | INFO | 38.9% : 5.6 GiB ( 84.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 09:29:10.094 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 09:29:10.094 | INFO | ------------------------------------------ | 20260505 09:29:10.094 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:29:10.094 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 09:29:10.094 | INFO | Total mem usage: 26.5 GiB | 20260505 09:29:10.094 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 09:29:10.094 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 09:29:10.094 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 09:29:10.094 | INFO | High Water Breakdown: | 20260505 09:29:10.094 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:29:10.094 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:29:10.094 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:29:10.094 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:29:10.094 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:29:10.094 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:29:10.095 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 09:29:10.095 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 09:29:10.095 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:29:10.095 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 09:29:10.095 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 09:29:10.095 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 09:29:10.095 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 09:29:10.095 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 09:29:10.095 | INFO | ------------------------------------------ | 20260505 09:29:10.095 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:29:10.095 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 09:29:10.095 | INFO | Total mem usage: 25.6 GiB | 20260505 09:29:10.095 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 09:29:10.095 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 09:29:10.095 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 09:29:10.095 | INFO | High Water Breakdown: | 20260505 09:29:10.095 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 09:29:10.095 | INFO | ------------------------------------------ | 20260505 09:29:10.095 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:29:10.095 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 09:29:10.095 | INFO | Total mem usage: 15.7 GiB | 20260505 09:29:10.095 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 09:29:10.095 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 09:29:10.095 | INFO | Unused Cache: 880 MiB (Waste: 5.48%) | 20260505 09:29:10.095 | INFO | High Water Breakdown: | 20260505 09:29:10.095 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 09:29:10.095 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 09:29:10.095 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 09:29:10.095 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 09:29:10.095 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 09:29:10.095 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 09:29:10.095 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 09:29:10.095 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 09:29:10.095 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 09:29:10.095 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:29:10.095 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 09:29:10.095 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:29:10.095 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 09:29:10.095 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 09:29:10.095 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 09:29:10.095 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 09:29:10.095 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:29:10.095 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 09:29:10.095 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:29:10.095 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:29:10.095 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:29:10.095 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 09:29:10.095 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 09:29:10.095 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:29:10.095 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 09:29:10.095 | INFO | ------------------------------------------ | 20260505 09:29:10.095 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:29:10.095 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 09:29:10.095 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:29:10.095 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:29:10.095 | INFO | PerfReport: Full Execution -- 3.24e+01 / 7.61e+01, 1.37e+01 / 2.29e+00 | 20260505 09:29:10.095 | INFO | PerfReport: 0.29%, AggregateStats -- 9.32e-02 / 1.04e-01, 1.07e-02 / 3.23e-03 | 20260505 09:29:10.095 | INFO | PerfReport: 11.65%, Baseline -- 3.78e+00 / 4.38e+00, 6.09e-01 / 3.23e-01 | 20260505 09:29:10.095 | INFO | PerfReport: 3.31%, Histogram -- 1.07e+00 / 1.45e+00, 2.06e-01 / 1.20e-01 | 20260505 09:29:10.095 | INFO | PerfReport: 15.53%, DME -- 4.03e+01 / 4.41e+01, 4.26e+00 / 1.85e+00 | 20260505 09:29:10.095 | INFO | PerfReport: 56.62%, FrameLabeling -- 1.84e+01 / 2.10e+01, 1.86e+00 / 2.32e+00 | 20260505 09:29:10.095 | INFO | PerfReport: 7.41%, PulseAccumulating -- 2.41e+00 / 2.95e+00, 3.99e-01 / 2.30e-01 | 20260505 09:29:10.095 | INFO | PerfReport: 3.82%, Metrics -- 1.24e+00 / 1.56e+00, 3.52e-01 / 1.81e-01 | 20260505 09:29:10.095 | INFO | PerfReport: 1.36%, PulseSerializing -- 4.42e-01 / 4.91e-01, 8.30e-02 / 2.86e-02 | 20260505 09:29:10.095 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 09:29:10.095 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:29:10.095 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:29:10.095 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.33e+01 / 0.00e+00 | 20260505 09:29:10.095 | INFO | PerfReport: 22.55%, Upload -- 2.29e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260505 09:29:10.095 | INFO | PerfReport: 21.76%, Download -- 2.21e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260505 09:29:10.095 | INFO | PerfReport: 31.95%, Compute -- 3.25e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 09:29:10.095 | INFO | PerfReport: 1.76%, LockUpload -- 1.79e+00 / 0.00e+00, 7.36e+00 / 0.00e+00 | 20260505 09:29:10.095 | INFO | PerfReport: 1.13%, LockDownload -- 1.14e+00 / 0.00e+00, 2.75e+00 / 0.00e+00 | 20260505 09:29:10.095 | INFO | PerfReport: 20.84%, LockCompute -- 2.12e+01 / 0.00e+00, 1.77e+01 / 0.00e+00 | 20260505 09:29:10.095 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.08e-02 / 0.00e+00 | 20260505 09:29:10.225 | INFO | Starting a round of Baz IO | 20260505 09:29:13.494 | INFO | Populating RTMetrics report | 20260505 09:29:13.494 | INFO | RTMetrics report populated | 20260505 09:29:13.494 | INFO | Opening tmp RTMetrics json file | 20260505 09:29:13.531 | INFO | Serializing RTMetrics to json | 20260505 09:29:13.578 | INFO | Serialization done, swapping tmp file now... | 20260505 09:29:13.603 | INFO | RTMetrics done | 20260505 09:29:13.620 | INFO | Finished a round of Baz IO: Wrote 2.438 GiB in 3.395 seconds | 20260505 09:29:13.621 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 26.05% / 0.01% / 73.94% | 20260505 09:29:13.621 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.95984% / 2.78155e-05% / 8.34465e-05% | 20260505 09:29:13.621 | INFO | Baz IO overall: | 20260505 09:29:13.621 | INFO | BytesWritten : 1235195300849 | 20260505 09:29:13.621 | INFO | NumEvents : 587817850720 | 20260505 09:29:15.233 | INFO | Analyzing chunk frames = [2138112,2138624) | 20260505 09:29:15.233 | INFO | MemInfo: { | 20260505 09:29:15.233 | INFO | "gpu" : " 17 GB", | 20260505 09:29:15.233 | INFO | "huge" : "", | 20260505 09:29:15.233 | INFO | "huge_shared" : " 27 GB", | 20260505 09:29:15.233 | INFO | "malloc" : " 14 GB", | 20260505 09:29:15.233 | INFO | "pinned" : " 29 GB" | 20260505 09:29:15.233 | INFO | } | 20260505 09:29:15.233 | INFO | Perf report for the last 5.13885 seconds: | 20260505 09:29:15.233 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:29:15.233 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:15.233 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:15.233 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:15.233 | INFO | ANALYSIS : 65.0%, 2.9, 99.9ms, 0 | 20260505 09:29:15.233 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:15.233 | INFO | RT_METRICS : 47.8%, 1.0, 25.1ms, 0 | 20260505 09:29:15.233 | INFO | PRE_HQ : 63.1%, 1.8, 60.2ms, 0 | 20260505 09:29:15.233 | INFO | BAZWRITER : 65.4%, 4.1, 137.7ms, 0 | 20260505 09:29:15.233 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 09:29:15.233 | INFO | Uploaded 36GiB to GPU in 2.25266s (15.9811GiB/s) | 20260505 09:29:15.233 | INFO | Downloaded 38GiB to GPU in 2.36019s (16.1004GiB/s) | 20260505 09:29:20.373 | INFO | Analyzing chunk frames = [2138624,2139136) | 20260505 09:29:20.373 | INFO | MemInfo: { | 20260505 09:29:20.373 | INFO | "gpu" : " 17 GB", | 20260505 09:29:20.373 | INFO | "huge" : "", | 20260505 09:29:20.373 | INFO | "huge_shared" : " 27 GB", | 20260505 09:29:20.373 | INFO | "malloc" : " 14 GB", | 20260505 09:29:20.373 | INFO | "pinned" : " 29 GB" | 20260505 09:29:20.373 | INFO | } | 20260505 09:29:20.373 | INFO | Perf report for the last 5.14061 seconds: | 20260505 09:29:20.373 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:29:20.373 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:20.373 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:20.373 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:20.373 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 09:29:20.374 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:20.374 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:20.374 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 09:29:20.374 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:20.374 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:20.374 | INFO | Uploaded 36GiB to GPU in 2.19417s (16.4071GiB/s) | 20260505 09:29:20.374 | INFO | Downloaded 34GiB to GPU in 2.11585s (16.0692GiB/s) | 20260505 09:29:25.512 | INFO | Analyzing chunk frames = [2139136,2139648) | 20260505 09:29:25.512 | INFO | MemInfo: { | 20260505 09:29:25.512 | INFO | "gpu" : " 17 GB", | 20260505 09:29:25.512 | INFO | "huge" : "", | 20260505 09:29:25.512 | INFO | "huge_shared" : " 27 GB", | 20260505 09:29:25.512 | INFO | "malloc" : " 14 GB", | 20260505 09:29:25.512 | INFO | "pinned" : " 29 GB" | 20260505 09:29:25.512 | INFO | } | 20260505 09:29:25.513 | INFO | Perf report for the last 5.1391 seconds: | 20260505 09:29:25.513 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:29:25.513 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:25.513 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:25.513 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:25.513 | INFO | ANALYSIS : 63.2%, 3.0, 98.4ms, 0 | 20260505 09:29:25.513 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:25.513 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:25.513 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 09:29:25.513 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:25.513 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:25.513 | INFO | Uploaded 36GiB to GPU in 2.15075s (16.7383GiB/s) | 20260505 09:29:25.513 | INFO | Downloaded 34GiB to GPU in 2.0667s (16.4513GiB/s) | 20260505 09:29:30.652 | INFO | Analyzing chunk frames = [2139648,2140160) | 20260505 09:29:30.652 | INFO | MemInfo: { | 20260505 09:29:30.652 | INFO | "gpu" : " 17 GB", | 20260505 09:29:30.652 | INFO | "huge" : "", | 20260505 09:29:30.652 | INFO | "huge_shared" : " 27 GB", | 20260505 09:29:30.652 | INFO | "malloc" : " 14 GB", | 20260505 09:29:30.652 | INFO | "pinned" : " 29 GB" | 20260505 09:29:30.652 | INFO | } | 20260505 09:29:30.652 | INFO | Perf report for the last 5.13921 seconds: | 20260505 09:29:30.652 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:29:30.652 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:30.652 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:30.652 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:30.652 | INFO | ANALYSIS : 63.2%, 3.0, 98.1ms, 0 | 20260505 09:29:30.652 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:30.652 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:30.652 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:29:30.652 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:30.652 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:30.652 | INFO | Uploaded 36GiB to GPU in 2.1564s (16.6945GiB/s) | 20260505 09:29:30.652 | INFO | Downloaded 34GiB to GPU in 2.06906s (16.4326GiB/s) | 20260505 09:29:35.790 | INFO | Analyzing chunk frames = [2140160,2140672) | 20260505 09:29:35.790 | INFO | MemInfo: { | 20260505 09:29:35.790 | INFO | "gpu" : " 17 GB", | 20260505 09:29:35.790 | INFO | "huge" : "", | 20260505 09:29:35.790 | INFO | "huge_shared" : " 27 GB", | 20260505 09:29:35.790 | INFO | "malloc" : " 14 GB", | 20260505 09:29:35.790 | INFO | "pinned" : " 29 GB" | 20260505 09:29:35.790 | INFO | } | 20260505 09:29:35.790 | INFO | Perf report for the last 5.1387 seconds: | 20260505 09:29:35.790 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:29:35.790 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:35.791 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:35.791 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:35.791 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 09:29:35.791 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:35.791 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:35.791 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 09:29:35.791 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:35.791 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:35.791 | INFO | Uploaded 36GiB to GPU in 2.23933s (16.0762GiB/s) | 20260505 09:29:35.791 | INFO | Downloaded 34GiB to GPU in 2.10117s (16.1815GiB/s) | 20260505 09:29:40.930 | INFO | Analyzing chunk frames = [2140672,2141184) | 20260505 09:29:40.930 | INFO | MemInfo: { | 20260505 09:29:40.930 | INFO | "gpu" : " 17 GB", | 20260505 09:29:40.930 | INFO | "huge" : "", | 20260505 09:29:40.930 | INFO | "huge_shared" : " 27 GB", | 20260505 09:29:40.930 | INFO | "malloc" : " 14 GB", | 20260505 09:29:40.930 | INFO | "pinned" : " 29 GB" | 20260505 09:29:40.930 | INFO | } | 20260505 09:29:40.930 | INFO | Perf report for the last 5.13988 seconds: | 20260505 09:29:40.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:29:40.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:40.930 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:40.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:40.930 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 09:29:40.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:40.930 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:40.930 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 09:29:40.930 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:40.930 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:40.931 | INFO | Uploaded 36GiB to GPU in 2.16192s (16.6519GiB/s) | 20260505 09:29:40.931 | INFO | Downloaded 34GiB to GPU in 2.05967s (16.5075GiB/s) | 20260505 09:29:46.068 | INFO | Analyzing chunk frames = [2141184,2141696) | 20260505 09:29:46.068 | INFO | MemInfo: { | 20260505 09:29:46.068 | INFO | "gpu" : " 17 GB", | 20260505 09:29:46.068 | INFO | "huge" : "", | 20260505 09:29:46.068 | INFO | "huge_shared" : " 27 GB", | 20260505 09:29:46.068 | INFO | "malloc" : " 14 GB", | 20260505 09:29:46.068 | INFO | "pinned" : " 29 GB" | 20260505 09:29:46.068 | INFO | } | 20260505 09:29:46.068 | INFO | Perf report for the last 5.1381 seconds: | 20260505 09:29:46.068 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:29:46.068 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:46.068 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:46.069 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:46.069 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 09:29:46.069 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:46.069 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:46.069 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 09:29:46.069 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:46.069 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:46.069 | INFO | Uploaded 36GiB to GPU in 2.2111s (16.2815GiB/s) | 20260505 09:29:46.069 | INFO | Downloaded 34GiB to GPU in 2.14234s (15.8705GiB/s) | 20260505 09:29:51.209 | INFO | Analyzing chunk frames = [2141696,2142208) | 20260505 09:29:51.209 | INFO | MemInfo: { | 20260505 09:29:51.209 | INFO | "gpu" : " 17 GB", | 20260505 09:29:51.209 | INFO | "huge" : "", | 20260505 09:29:51.209 | INFO | "huge_shared" : " 27 GB", | 20260505 09:29:51.209 | INFO | "malloc" : " 14 GB", | 20260505 09:29:51.209 | INFO | "pinned" : " 29 GB" | 20260505 09:29:51.209 | INFO | } | 20260505 09:29:51.209 | INFO | Perf report for the last 5.14069 seconds: | 20260505 09:29:51.209 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:29:51.209 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:51.209 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:51.209 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:51.209 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 09:29:51.209 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:51.209 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:51.209 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 09:29:51.209 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:51.209 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:51.209 | INFO | Uploaded 36GiB to GPU in 2.18103s (16.506GiB/s) | 20260505 09:29:51.209 | INFO | Downloaded 34GiB to GPU in 2.05397s (16.5533GiB/s) | 20260505 09:29:51.340 | INFO | Starting a round of Baz IO | 20260505 09:29:54.611 | INFO | Populating RTMetrics report | 20260505 09:29:54.611 | INFO | RTMetrics report populated | 20260505 09:29:54.611 | INFO | Opening tmp RTMetrics json file | 20260505 09:29:54.612 | INFO | Serializing RTMetrics to json | 20260505 09:29:54.613 | INFO | Serialization done, swapping tmp file now... | 20260505 09:29:54.614 | INFO | RTMetrics done | 20260505 09:29:54.753 | INFO | Finished a round of Baz IO: Wrote 2.436 GiB in 3.413 seconds | 20260505 09:29:54.753 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 26.03% / 0.01% / 73.95% | 20260505 09:29:54.753 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.94876% / 1.98682e-05% / 9.13938e-05% | 20260505 09:29:54.753 | INFO | Baz IO overall: | 20260505 09:29:54.753 | INFO | BytesWritten : 1237811353435 | 20260505 09:29:54.753 | INFO | NumEvents : 589067721902 | 20260505 09:29:56.350 | INFO | Analyzing chunk frames = [2142208,2142720) | 20260505 09:29:56.350 | INFO | MemInfo: { | 20260505 09:29:56.350 | INFO | "gpu" : " 17 GB", | 20260505 09:29:56.350 | INFO | "huge" : "", | 20260505 09:29:56.350 | INFO | "huge_shared" : " 27 GB", | 20260505 09:29:56.350 | INFO | "malloc" : " 14 GB", | 20260505 09:29:56.350 | INFO | "pinned" : " 29 GB" | 20260505 09:29:56.350 | INFO | } | 20260505 09:29:56.350 | INFO | Perf report for the last 5.14097 seconds: | 20260505 09:29:56.350 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:29:56.350 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:56.350 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:56.350 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:56.350 | INFO | ANALYSIS : 65.0%, 2.9, 99.5ms, 0 | 20260505 09:29:56.350 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:29:56.350 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260505 09:29:56.350 | INFO | PRE_HQ : 63.0%, 1.8, 60.5ms, 0 | 20260505 09:29:56.350 | INFO | BAZWRITER : 65.7%, 4.1, 139.5ms, 0 | 20260505 09:29:56.350 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 09:29:56.350 | INFO | Uploaded 36GiB to GPU in 2.25054s (15.9962GiB/s) | 20260505 09:29:56.350 | INFO | Downloaded 38GiB to GPU in 2.35609s (16.1284GiB/s) | 20260505 09:30:01.488 | INFO | Analyzing chunk frames = [2142720,2143232) | 20260505 09:30:01.488 | INFO | MemInfo: { | 20260505 09:30:01.488 | INFO | "gpu" : " 17 GB", | 20260505 09:30:01.488 | INFO | "huge" : "", | 20260505 09:30:01.489 | INFO | "huge_shared" : " 27 GB", | 20260505 09:30:01.489 | INFO | "malloc" : " 14 GB", | 20260505 09:30:01.489 | INFO | "pinned" : " 29 GB" | 20260505 09:30:01.489 | INFO | } | 20260505 09:30:01.489 | INFO | Perf report for the last 5.13842 seconds: | 20260505 09:30:01.489 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:30:01.489 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:01.489 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:01.489 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:01.489 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 09:30:01.489 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:01.489 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:01.489 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:30:01.489 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:01.489 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:01.489 | INFO | Uploaded 36GiB to GPU in 2.17937s (16.5186GiB/s) | 20260505 09:30:01.489 | INFO | Downloaded 34GiB to GPU in 2.10179s (16.1767GiB/s) | 20260505 09:30:06.626 | INFO | Analyzing chunk frames = [2143232,2143744) | 20260505 09:30:06.626 | INFO | MemInfo: { | 20260505 09:30:06.626 | INFO | "gpu" : " 17 GB", | 20260505 09:30:06.626 | INFO | "huge" : "", | 20260505 09:30:06.626 | INFO | "huge_shared" : " 27 GB", | 20260505 09:30:06.626 | INFO | "malloc" : " 14 GB", | 20260505 09:30:06.626 | INFO | "pinned" : " 29 GB" | 20260505 09:30:06.626 | INFO | } | 20260505 09:30:06.626 | INFO | Perf report for the last 5.13765 seconds: | 20260505 09:30:06.626 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:30:06.626 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:06.626 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:06.626 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:06.626 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 09:30:06.626 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:06.626 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:06.626 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 09:30:06.626 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:06.626 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:06.626 | INFO | Uploaded 36GiB to GPU in 2.14234s (16.804GiB/s) | 20260505 09:30:06.627 | INFO | Downloaded 34GiB to GPU in 2.06588s (16.4579GiB/s) | 20260505 09:30:11.766 | INFO | Analyzing chunk frames = [2143744,2144256) | 20260505 09:30:11.766 | INFO | MemInfo: { | 20260505 09:30:11.766 | INFO | "gpu" : " 17 GB", | 20260505 09:30:11.766 | INFO | "huge" : "", | 20260505 09:30:11.766 | INFO | "huge_shared" : " 27 GB", | 20260505 09:30:11.766 | INFO | "malloc" : " 14 GB", | 20260505 09:30:11.766 | INFO | "pinned" : " 29 GB" | 20260505 09:30:11.766 | INFO | } | 20260505 09:30:11.766 | INFO | Perf report for the last 5.14004 seconds: | 20260505 09:30:11.766 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:30:11.766 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:11.766 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:11.766 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:11.766 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 09:30:11.766 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:11.766 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:11.766 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 09:30:11.766 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:11.766 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:11.767 | INFO | Uploaded 36GiB to GPU in 2.14051s (16.8184GiB/s) | 20260505 09:30:11.767 | INFO | Downloaded 34GiB to GPU in 2.06936s (16.4302GiB/s) | 20260505 09:30:16.905 | INFO | Analyzing chunk frames = [2144256,2144768) | 20260505 09:30:16.905 | INFO | MemInfo: { | 20260505 09:30:16.905 | INFO | "gpu" : " 17 GB", | 20260505 09:30:16.905 | INFO | "huge" : "", | 20260505 09:30:16.905 | INFO | "huge_shared" : " 27 GB", | 20260505 09:30:16.905 | INFO | "malloc" : " 14 GB", | 20260505 09:30:16.905 | INFO | "pinned" : " 29 GB" | 20260505 09:30:16.905 | INFO | } | 20260505 09:30:16.905 | INFO | Perf report for the last 5.13904 seconds: | 20260505 09:30:16.905 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:30:16.905 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:16.905 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:16.905 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:16.905 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 09:30:16.905 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:16.905 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:16.905 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:30:16.905 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:16.905 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:16.906 | INFO | Uploaded 36GiB to GPU in 2.23654s (16.0963GiB/s) | 20260505 09:30:16.906 | INFO | Downloaded 34GiB to GPU in 2.09914s (16.1971GiB/s) | 20260505 09:30:22.046 | INFO | Analyzing chunk frames = [2144768,2145280) | 20260505 09:30:22.046 | INFO | MemInfo: { | 20260505 09:30:22.046 | INFO | "gpu" : " 17 GB", | 20260505 09:30:22.046 | INFO | "huge" : "", | 20260505 09:30:22.046 | INFO | "huge_shared" : " 27 GB", | 20260505 09:30:22.046 | INFO | "malloc" : " 14 GB", | 20260505 09:30:22.046 | INFO | "pinned" : " 29 GB" | 20260505 09:30:22.046 | INFO | } | 20260505 09:30:22.046 | INFO | Perf report for the last 5.14051 seconds: | 20260505 09:30:22.046 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:30:22.046 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:22.046 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:22.046 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:22.046 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 09:30:22.046 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:22.046 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:22.046 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:30:22.046 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:22.046 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:22.046 | INFO | Uploaded 36GiB to GPU in 2.23823s (16.0842GiB/s) | 20260505 09:30:22.046 | INFO | Downloaded 34GiB to GPU in 2.10385s (16.1609GiB/s) | 20260505 09:30:27.184 | INFO | Analyzing chunk frames = [2145280,2145792) | 20260505 09:30:27.184 | INFO | MemInfo: { | 20260505 09:30:27.184 | INFO | "gpu" : " 17 GB", | 20260505 09:30:27.184 | INFO | "huge" : "", | 20260505 09:30:27.184 | INFO | "huge_shared" : " 27 GB", | 20260505 09:30:27.184 | INFO | "malloc" : " 14 GB", | 20260505 09:30:27.184 | INFO | "pinned" : " 29 GB" | 20260505 09:30:27.184 | INFO | } | 20260505 09:30:27.184 | INFO | Perf report for the last 5.13806 seconds: | 20260505 09:30:27.184 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:30:27.184 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:27.184 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:27.184 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:27.184 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 09:30:27.184 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:27.184 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:27.184 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 09:30:27.184 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:27.184 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:27.184 | INFO | Uploaded 36GiB to GPU in 2.20118s (16.3548GiB/s) | 20260505 09:30:27.184 | INFO | Downloaded 34GiB to GPU in 2.12545s (15.9966GiB/s) | 20260505 09:30:32.324 | INFO | Analyzing chunk frames = [2145792,2146304) | 20260505 09:30:32.324 | INFO | MemInfo: { | 20260505 09:30:32.324 | INFO | "gpu" : " 17 GB", | 20260505 09:30:32.324 | INFO | "huge" : "", | 20260505 09:30:32.324 | INFO | "huge_shared" : " 27 GB", | 20260505 09:30:32.324 | INFO | "malloc" : " 14 GB", | 20260505 09:30:32.324 | INFO | "pinned" : " 29 GB" | 20260505 09:30:32.324 | INFO | } | 20260505 09:30:32.324 | INFO | Perf report for the last 5.13983 seconds: | 20260505 09:30:32.324 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:30:32.324 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:32.324 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:32.324 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:32.324 | INFO | ANALYSIS : 64.7%, 3.0, 101.0ms, 0 | 20260505 09:30:32.324 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:32.324 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:32.324 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 09:30:32.324 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:32.324 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:32.324 | INFO | Uploaded 36GiB to GPU in 2.24834s (16.0118GiB/s) | 20260505 09:30:32.324 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 09:30:32.324 | INFO | Total mem usage: 15.3 GiB | 20260505 09:30:32.324 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 09:30:32.324 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.6%) | 20260505 09:30:32.324 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 09:30:32.324 | INFO | High Water Breakdown: | 20260505 09:30:32.324 | INFO | 60.9% : 8.76 GiB ( 93.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 09:30:32.324 | INFO | 38.9% : 5.6 GiB ( 84.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 09:30:32.324 | INFO | Downloaded 34GiB to GPU in 2.08187s (16.3315GiB/s) | 20260505 09:30:32.324 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 09:30:32.324 | INFO | ------------------------------------------ | 20260505 09:30:32.324 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:30:32.324 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 09:30:32.324 | INFO | Total mem usage: 26.5 GiB | 20260505 09:30:32.324 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 09:30:32.324 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 09:30:32.324 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 09:30:32.324 | INFO | High Water Breakdown: | 20260505 09:30:32.324 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:30:32.324 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:30:32.324 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:30:32.324 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:30:32.324 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:30:32.324 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:30:32.324 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 09:30:32.324 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 09:30:32.324 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:30:32.324 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 09:30:32.324 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 09:30:32.324 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 09:30:32.324 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 09:30:32.324 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 09:30:32.324 | INFO | ------------------------------------------ | 20260505 09:30:32.324 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:30:32.324 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 09:30:32.324 | INFO | Total mem usage: 25.6 GiB | 20260505 09:30:32.324 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 09:30:32.324 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 09:30:32.324 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 09:30:32.324 | INFO | High Water Breakdown: | 20260505 09:30:32.324 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 09:30:32.324 | INFO | ------------------------------------------ | 20260505 09:30:32.325 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:30:32.325 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 09:30:32.325 | INFO | Total mem usage: 15.7 GiB | 20260505 09:30:32.325 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 09:30:32.325 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 09:30:32.325 | INFO | Unused Cache: 880 MiB (Waste: 5.48%) | 20260505 09:30:32.325 | INFO | High Water Breakdown: | 20260505 09:30:32.325 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 09:30:32.325 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 09:30:32.325 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 09:30:32.325 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 09:30:32.325 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 09:30:32.325 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 09:30:32.325 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 09:30:32.325 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 09:30:32.325 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 09:30:32.325 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:30:32.325 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 09:30:32.325 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:30:32.325 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 09:30:32.325 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 09:30:32.325 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 09:30:32.325 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 09:30:32.325 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:30:32.325 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 09:30:32.325 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:30:32.325 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:30:32.325 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:30:32.325 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 09:30:32.325 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 09:30:32.325 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:30:32.325 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 09:30:32.325 | INFO | ------------------------------------------ | 20260505 09:30:32.325 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:30:32.325 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 09:30:32.325 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:30:32.325 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:30:32.325 | INFO | PerfReport: Full Execution -- 3.25e+01 / 7.68e+01, 1.37e+01 / 2.51e+00 | 20260505 09:30:32.325 | INFO | PerfReport: 0.29%, AggregateStats -- 9.34e-02 / 1.04e-01, 1.09e-02 / 3.62e-03 | 20260505 09:30:32.325 | INFO | PerfReport: 11.64%, Baseline -- 3.78e+00 / 4.39e+00, 6.11e-01 / 3.16e-01 | 20260505 09:30:32.325 | INFO | PerfReport: 3.31%, Histogram -- 1.08e+00 / 1.46e+00, 2.08e-01 / 1.20e-01 | 20260505 09:30:32.325 | INFO | PerfReport: 15.57%, DME -- 4.05e+01 / 4.46e+01, 4.40e+00 / 1.83e+00 | 20260505 09:30:32.325 | INFO | PerfReport: 56.64%, FrameLabeling -- 1.84e+01 / 2.12e+01, 1.92e+00 / 2.30e+00 | 20260505 09:30:32.325 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.40e+00 / 2.99e+00, 4.04e-01 / 1.88e-01 | 20260505 09:30:32.325 | INFO | PerfReport: 3.81%, Metrics -- 1.24e+00 / 1.57e+00, 3.59e-01 / 1.73e-01 | 20260505 09:30:32.325 | INFO | PerfReport: 1.35%, PulseSerializing -- 4.40e-01 / 4.90e-01, 8.24e-02 / 2.72e-02 | 20260505 09:30:32.325 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 09:30:32.325 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:30:32.325 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:30:32.325 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.36e+01 / 0.00e+00 | 20260505 09:30:32.325 | INFO | PerfReport: 22.55%, Upload -- 2.29e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260505 09:30:32.325 | INFO | PerfReport: 21.77%, Download -- 2.21e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260505 09:30:32.325 | INFO | PerfReport: 32.01%, Compute -- 3.25e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 09:30:32.325 | INFO | PerfReport: 1.79%, LockUpload -- 1.82e+00 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260505 09:30:32.325 | INFO | PerfReport: 1.10%, LockDownload -- 1.12e+00 / 0.00e+00, 2.72e+00 / 0.00e+00 | 20260505 09:30:32.325 | INFO | PerfReport: 20.76%, LockCompute -- 2.11e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260505 09:30:32.325 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.07e-02 / 0.00e+00 | 20260505 09:30:32.454 | INFO | Starting a round of Baz IO | 20260505 09:30:35.719 | INFO | Populating RTMetrics report | 20260505 09:30:35.719 | INFO | RTMetrics report populated | 20260505 09:30:35.719 | INFO | Opening tmp RTMetrics json file | 20260505 09:30:35.732 | INFO | Serializing RTMetrics to json | 20260505 09:30:35.756 | INFO | Serialization done, swapping tmp file now... | 20260505 09:30:35.775 | INFO | RTMetrics done | 20260505 09:30:35.861 | INFO | Finished a round of Baz IO: Wrote 2.436 GiB in 3.406 seconds | 20260505 09:30:35.861 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 26.02% / 0.01% / 73.97% | 20260505 09:30:35.861 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.94209% / 2.38419e-05% / 0.000103315% | 20260505 09:30:35.861 | INFO | Baz IO overall: | 20260505 09:30:35.861 | INFO | BytesWritten : 1240427310270 | 20260505 09:30:35.861 | INFO | NumEvents : 590317333924 | 20260505 09:30:37.463 | INFO | Analyzing chunk frames = [2146304,2146816) | 20260505 09:30:37.463 | INFO | MemInfo: { | 20260505 09:30:37.463 | INFO | "gpu" : " 17 GB", | 20260505 09:30:37.463 | INFO | "huge" : "", | 20260505 09:30:37.463 | INFO | "huge_shared" : " 27 GB", | 20260505 09:30:37.463 | INFO | "malloc" : " 14 GB", | 20260505 09:30:37.463 | INFO | "pinned" : " 29 GB" | 20260505 09:30:37.463 | INFO | } | 20260505 09:30:37.463 | INFO | Perf report for the last 5.13968 seconds: | 20260505 09:30:37.463 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:30:37.463 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:37.463 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:37.464 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:37.464 | INFO | ANALYSIS : 64.8%, 2.9, 99.3ms, 0 | 20260505 09:30:37.464 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:37.464 | INFO | RT_METRICS : 47.0%, 1.0, 24.7ms, 0 | 20260505 09:30:37.464 | INFO | PRE_HQ : 62.9%, 1.8, 60.4ms, 0 | 20260505 09:30:37.464 | INFO | BAZWRITER : 65.6%, 4.1, 139.6ms, 0 | 20260505 09:30:37.464 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 09:30:37.464 | INFO | Uploaded 36GiB to GPU in 2.24336s (16.0474GiB/s) | 20260505 09:30:37.464 | INFO | Downloaded 38GiB to GPU in 2.37573s (15.9951GiB/s) | 20260505 09:30:42.602 | INFO | Analyzing chunk frames = [2146816,2147328) | 20260505 09:30:42.602 | INFO | MemInfo: { | 20260505 09:30:42.602 | INFO | "gpu" : " 17 GB", | 20260505 09:30:42.602 | INFO | "huge" : "", | 20260505 09:30:42.602 | INFO | "huge_shared" : " 27 GB", | 20260505 09:30:42.602 | INFO | "malloc" : " 14 GB", | 20260505 09:30:42.602 | INFO | "pinned" : " 29 GB" | 20260505 09:30:42.602 | INFO | } | 20260505 09:30:42.602 | INFO | Perf report for the last 5.13856 seconds: | 20260505 09:30:42.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:30:42.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:42.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:42.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:42.602 | INFO | ANALYSIS : 64.5%, 3.0, 100.7ms, 0 | 20260505 09:30:42.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:42.602 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:42.602 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:30:42.602 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:42.602 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:42.602 | INFO | Uploaded 36GiB to GPU in 2.23845s (16.0825GiB/s) | 20260505 09:30:42.602 | INFO | Downloaded 34GiB to GPU in 2.10566s (16.147GiB/s) | 20260505 09:30:47.741 | INFO | Analyzing chunk frames = [2147328,2147840) | 20260505 09:30:47.741 | INFO | MemInfo: { | 20260505 09:30:47.741 | INFO | "gpu" : " 17 GB", | 20260505 09:30:47.741 | INFO | "huge" : "", | 20260505 09:30:47.741 | INFO | "huge_shared" : " 27 GB", | 20260505 09:30:47.741 | INFO | "malloc" : " 14 GB", | 20260505 09:30:47.741 | INFO | "pinned" : " 29 GB" | 20260505 09:30:47.741 | INFO | } | 20260505 09:30:47.741 | INFO | Perf report for the last 5.13901 seconds: | 20260505 09:30:47.741 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:30:47.741 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:47.741 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:47.741 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:47.741 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 09:30:47.741 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:47.741 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:47.741 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 09:30:47.741 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:47.741 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:47.741 | INFO | Uploaded 36GiB to GPU in 2.15512s (16.7044GiB/s) | 20260505 09:30:47.741 | INFO | Downloaded 34GiB to GPU in 2.06703s (16.4488GiB/s) | 20260505 09:30:52.881 | INFO | Analyzing chunk frames = [2147840,2148352) | 20260505 09:30:52.881 | INFO | MemInfo: { | 20260505 09:30:52.881 | INFO | "gpu" : " 17 GB", | 20260505 09:30:52.881 | INFO | "huge" : "", | 20260505 09:30:52.881 | INFO | "huge_shared" : " 27 GB", | 20260505 09:30:52.881 | INFO | "malloc" : " 14 GB", | 20260505 09:30:52.881 | INFO | "pinned" : " 29 GB" | 20260505 09:30:52.881 | INFO | } | 20260505 09:30:52.881 | INFO | Perf report for the last 5.1399 seconds: | 20260505 09:30:52.881 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:30:52.881 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:52.881 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:52.881 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:52.881 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 09:30:52.881 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:52.881 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:52.881 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 09:30:52.881 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:52.881 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:52.881 | INFO | Uploaded 36GiB to GPU in 2.1619s (16.652GiB/s) | 20260505 09:30:52.881 | INFO | Downloaded 34GiB to GPU in 2.07289s (16.4022GiB/s) | 20260505 09:30:58.020 | INFO | Analyzing chunk frames = [2148352,2148864) | 20260505 09:30:58.020 | INFO | MemInfo: { | 20260505 09:30:58.020 | INFO | "gpu" : " 17 GB", | 20260505 09:30:58.020 | INFO | "huge" : "", | 20260505 09:30:58.020 | INFO | "huge_shared" : " 27 GB", | 20260505 09:30:58.020 | INFO | "malloc" : " 14 GB", | 20260505 09:30:58.020 | INFO | "pinned" : " 29 GB" | 20260505 09:30:58.020 | INFO | } | 20260505 09:30:58.020 | INFO | Perf report for the last 5.13906 seconds: | 20260505 09:30:58.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:30:58.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:58.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:58.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:58.020 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 09:30:58.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:58.020 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:58.020 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:30:58.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:58.020 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:30:58.020 | INFO | Uploaded 36GiB to GPU in 2.23287s (16.1227GiB/s) | 20260505 09:30:58.020 | INFO | Downloaded 34GiB to GPU in 2.10418s (16.1583GiB/s) | 20260505 09:31:03.160 | INFO | Analyzing chunk frames = [2148864,2149376) | 20260505 09:31:03.160 | INFO | MemInfo: { | 20260505 09:31:03.160 | INFO | "gpu" : " 17 GB", | 20260505 09:31:03.160 | INFO | "huge" : "", | 20260505 09:31:03.160 | INFO | "huge_shared" : " 27 GB", | 20260505 09:31:03.160 | INFO | "malloc" : " 14 GB", | 20260505 09:31:03.160 | INFO | "pinned" : " 29 GB" | 20260505 09:31:03.160 | INFO | } | 20260505 09:31:03.160 | INFO | Perf report for the last 5.14032 seconds: | 20260505 09:31:03.160 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:31:03.160 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:03.160 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:03.160 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:03.160 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 09:31:03.160 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:03.160 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:03.161 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 09:31:03.161 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:03.161 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:03.161 | INFO | Uploaded 36GiB to GPU in 2.23969s (16.0737GiB/s) | 20260505 09:31:03.161 | INFO | Downloaded 34GiB to GPU in 2.10379s (16.1613GiB/s) | 20260505 09:31:08.300 | INFO | Analyzing chunk frames = [2149376,2149888) | 20260505 09:31:08.300 | INFO | MemInfo: { | 20260505 09:31:08.300 | INFO | "gpu" : " 17 GB", | 20260505 09:31:08.300 | INFO | "huge" : "", | 20260505 09:31:08.300 | INFO | "huge_shared" : " 27 GB", | 20260505 09:31:08.300 | INFO | "malloc" : " 14 GB", | 20260505 09:31:08.300 | INFO | "pinned" : " 29 GB" | 20260505 09:31:08.300 | INFO | } | 20260505 09:31:08.300 | INFO | Perf report for the last 5.13933 seconds: | 20260505 09:31:08.300 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:31:08.300 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:08.300 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:08.300 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:08.300 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 09:31:08.300 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:08.300 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:08.300 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260505 09:31:08.300 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:08.300 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:08.300 | INFO | Uploaded 36GiB to GPU in 2.20235s (16.3462GiB/s) | 20260505 09:31:08.300 | INFO | Downloaded 34GiB to GPU in 2.12587s (15.9934GiB/s) | 20260505 09:31:13.439 | INFO | Analyzing chunk frames = [2149888,2150400) | 20260505 09:31:13.439 | INFO | MemInfo: { | 20260505 09:31:13.439 | INFO | "gpu" : " 17 GB", | 20260505 09:31:13.439 | INFO | "huge" : "", | 20260505 09:31:13.439 | INFO | "huge_shared" : " 27 GB", | 20260505 09:31:13.439 | INFO | "malloc" : " 14 GB", | 20260505 09:31:13.439 | INFO | "pinned" : " 29 GB" | 20260505 09:31:13.439 | INFO | } | 20260505 09:31:13.439 | INFO | Perf report for the last 5.13904 seconds: | 20260505 09:31:13.439 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:31:13.439 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:13.439 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:13.439 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:13.439 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 09:31:13.439 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:13.439 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:13.439 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 09:31:13.439 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:13.439 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:13.439 | INFO | Uploaded 36GiB to GPU in 2.18723s (16.4591GiB/s) | 20260505 09:31:13.439 | INFO | Downloaded 34GiB to GPU in 2.0561s (16.5362GiB/s) | 20260505 09:31:13.571 | INFO | Starting a round of Baz IO | 20260505 09:31:16.845 | INFO | Populating RTMetrics report | 20260505 09:31:16.845 | INFO | RTMetrics report populated | 20260505 09:31:16.845 | INFO | Opening tmp RTMetrics json file | 20260505 09:31:16.880 | INFO | Serializing RTMetrics to json | 20260505 09:31:16.956 | INFO | Serialization done, swapping tmp file now... | 20260505 09:31:16.960 | INFO | RTMetrics done | 20260505 09:31:17.088 | INFO | Finished a round of Baz IO: Wrote 2.437 GiB in 3.516 seconds | 20260505 09:31:17.088 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 26.00% / 0.01% / 73.98% | 20260505 09:31:17.088 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.93701% / 3.57628e-05% / 8.34465e-05% | 20260505 09:31:17.088 | INFO | Baz IO overall: | 20260505 09:31:17.088 | INFO | BytesWritten : 1243044296595 | 20260505 09:31:17.088 | INFO | NumEvents : 591567532106 | 20260505 09:31:18.577 | INFO | Analyzing chunk frames = [2150400,2150912) | 20260505 09:31:18.577 | INFO | MemInfo: { | 20260505 09:31:18.577 | INFO | "gpu" : " 17 GB", | 20260505 09:31:18.577 | INFO | "huge" : "", | 20260505 09:31:18.577 | INFO | "huge_shared" : " 27 GB", | 20260505 09:31:18.577 | INFO | "malloc" : " 14 GB", | 20260505 09:31:18.577 | INFO | "pinned" : " 29 GB" | 20260505 09:31:18.577 | INFO | } | 20260505 09:31:18.577 | INFO | Perf report for the last 5.1381 seconds: | 20260505 09:31:18.577 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:31:18.577 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:18.577 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:18.577 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:18.577 | INFO | ANALYSIS : 65.2%, 2.9, 99.6ms, 0 | 20260505 09:31:18.577 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:18.577 | INFO | RT_METRICS : 48.2%, 1.0, 25.3ms, 0 | 20260505 09:31:18.577 | INFO | PRE_HQ : 62.9%, 1.9, 60.6ms, 0 | 20260505 09:31:18.577 | INFO | BAZWRITER : 67.7%, 4.1, 142.8ms, 0 | 20260505 09:31:18.577 | INFO | BAZ_SAMPLING : 2.6%, 1.0, 1.4ms, 0 | 20260505 09:31:18.577 | INFO | Uploaded 36GiB to GPU in 2.20833s (16.3019GiB/s) | 20260505 09:31:18.577 | INFO | Downloaded 38GiB to GPU in 2.34099s (16.2325GiB/s) | 20260505 09:31:23.717 | INFO | Analyzing chunk frames = [2150912,2151424) | 20260505 09:31:23.717 | INFO | MemInfo: { | 20260505 09:31:23.717 | INFO | "gpu" : " 17 GB", | 20260505 09:31:23.717 | INFO | "huge" : "", | 20260505 09:31:23.717 | INFO | "huge_shared" : " 27 GB", | 20260505 09:31:23.717 | INFO | "malloc" : " 14 GB", | 20260505 09:31:23.717 | INFO | "pinned" : " 29 GB" | 20260505 09:31:23.717 | INFO | } | 20260505 09:31:23.717 | INFO | Perf report for the last 5.13978 seconds: | 20260505 09:31:23.717 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:31:23.717 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:23.717 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:23.717 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:23.717 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 09:31:23.717 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:23.717 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:23.717 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 09:31:23.717 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:23.717 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:23.718 | INFO | Uploaded 36GiB to GPU in 2.21238s (16.2721GiB/s) | 20260505 09:31:23.718 | INFO | Downloaded 34GiB to GPU in 2.12226s (16.0207GiB/s) | 20260505 09:31:28.858 | INFO | Analyzing chunk frames = [2151424,2151936) | 20260505 09:31:28.858 | INFO | MemInfo: { | 20260505 09:31:28.858 | INFO | "gpu" : " 17 GB", | 20260505 09:31:28.858 | INFO | "huge" : "", | 20260505 09:31:28.858 | INFO | "huge_shared" : " 27 GB", | 20260505 09:31:28.858 | INFO | "malloc" : " 14 GB", | 20260505 09:31:28.858 | INFO | "pinned" : " 29 GB" | 20260505 09:31:28.858 | INFO | } | 20260505 09:31:28.858 | INFO | Perf report for the last 5.14126 seconds: | 20260505 09:31:28.858 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:31:28.858 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:28.858 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:28.858 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:28.858 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 09:31:28.858 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:28.858 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:28.858 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 09:31:28.858 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:28.858 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:28.858 | INFO | Uploaded 36GiB to GPU in 2.15531s (16.703GiB/s) | 20260505 09:31:28.858 | INFO | Downloaded 34GiB to GPU in 2.07185s (16.4104GiB/s) | 20260505 09:31:33.997 | INFO | Analyzing chunk frames = [2151936,2152448) | 20260505 09:31:33.997 | INFO | MemInfo: { | 20260505 09:31:33.997 | INFO | "gpu" : " 17 GB", | 20260505 09:31:33.997 | INFO | "huge" : "", | 20260505 09:31:33.997 | INFO | "huge_shared" : " 27 GB", | 20260505 09:31:33.997 | INFO | "malloc" : " 14 GB", | 20260505 09:31:33.997 | INFO | "pinned" : " 29 GB" | 20260505 09:31:33.997 | INFO | } | 20260505 09:31:33.997 | INFO | Perf report for the last 5.13904 seconds: | 20260505 09:31:33.997 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:31:33.997 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:33.997 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:33.997 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:33.997 | INFO | ANALYSIS : 63.1%, 3.0, 98.0ms, 0 | 20260505 09:31:33.997 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:33.997 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:33.997 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 09:31:33.997 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:33.997 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:33.997 | INFO | Uploaded 36GiB to GPU in 2.16125s (16.657GiB/s) | 20260505 09:31:33.997 | INFO | Downloaded 34GiB to GPU in 2.07218s (16.4079GiB/s) | 20260505 09:31:39.136 | INFO | Analyzing chunk frames = [2152448,2152960) | 20260505 09:31:39.136 | INFO | MemInfo: { | 20260505 09:31:39.136 | INFO | "gpu" : " 17 GB", | 20260505 09:31:39.136 | INFO | "huge" : "", | 20260505 09:31:39.136 | INFO | "huge_shared" : " 27 GB", | 20260505 09:31:39.136 | INFO | "malloc" : " 14 GB", | 20260505 09:31:39.136 | INFO | "pinned" : " 29 GB" | 20260505 09:31:39.136 | INFO | } | 20260505 09:31:39.136 | INFO | Perf report for the last 5.13937 seconds: | 20260505 09:31:39.136 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:31:39.136 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:39.136 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:39.136 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:39.136 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 09:31:39.136 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:39.137 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:39.137 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:31:39.137 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:39.137 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:39.137 | INFO | Uploaded 36GiB to GPU in 2.23483s (16.1086GiB/s) | 20260505 09:31:39.137 | INFO | Downloaded 34GiB to GPU in 2.10747s (16.1331GiB/s) | 20260505 09:31:44.274 | INFO | Analyzing chunk frames = [2152960,2153472) | 20260505 09:31:44.274 | INFO | MemInfo: { | 20260505 09:31:44.274 | INFO | "gpu" : " 17 GB", | 20260505 09:31:44.274 | INFO | "huge" : "", | 20260505 09:31:44.274 | INFO | "huge_shared" : " 27 GB", | 20260505 09:31:44.274 | INFO | "malloc" : " 14 GB", | 20260505 09:31:44.274 | INFO | "pinned" : " 29 GB" | 20260505 09:31:44.274 | INFO | } | 20260505 09:31:44.274 | INFO | Perf report for the last 5.13804 seconds: | 20260505 09:31:44.274 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:31:44.274 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:44.275 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:44.275 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:44.275 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 09:31:44.275 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:44.275 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:44.275 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 09:31:44.275 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:44.275 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:44.275 | INFO | Uploaded 36GiB to GPU in 2.2345s (16.111GiB/s) | 20260505 09:31:44.275 | INFO | Downloaded 34GiB to GPU in 2.10568s (16.1468GiB/s) | 20260505 09:31:49.414 | INFO | Analyzing chunk frames = [2153472,2153984) | 20260505 09:31:49.414 | INFO | MemInfo: { | 20260505 09:31:49.414 | INFO | "gpu" : " 17 GB", | 20260505 09:31:49.414 | INFO | "huge" : "", | 20260505 09:31:49.414 | INFO | "huge_shared" : " 27 GB", | 20260505 09:31:49.414 | INFO | "malloc" : " 14 GB", | 20260505 09:31:49.414 | INFO | "pinned" : " 29 GB" | 20260505 09:31:49.414 | INFO | } | 20260505 09:31:49.414 | INFO | Perf report for the last 5.13954 seconds: | 20260505 09:31:49.414 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:31:49.414 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:49.414 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:49.414 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:49.414 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 09:31:49.414 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:49.414 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:49.414 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 09:31:49.414 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:49.414 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:49.414 | INFO | Uploaded 36GiB to GPU in 2.20153s (16.3522GiB/s) | 20260505 09:31:49.414 | INFO | Downloaded 34GiB to GPU in 2.14405s (15.8578GiB/s) | 20260505 09:31:54.554 | INFO | Analyzing chunk frames = [2153984,2154496) | 20260505 09:31:54.554 | INFO | MemInfo: { | 20260505 09:31:54.554 | INFO | "gpu" : " 17 GB", | 20260505 09:31:54.554 | INFO | "huge" : "", | 20260505 09:31:54.554 | INFO | "huge_shared" : " 27 GB", | 20260505 09:31:54.554 | INFO | "malloc" : " 14 GB", | 20260505 09:31:54.554 | INFO | "pinned" : " 29 GB" | 20260505 09:31:54.554 | INFO | } | 20260505 09:31:54.554 | INFO | Perf report for the last 5.1404 seconds: | 20260505 09:31:54.554 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:31:54.554 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:54.554 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:54.554 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:54.554 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 09:31:54.554 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:54.555 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:54.555 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:31:54.555 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:54.555 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:54.555 | INFO | Uploaded 36GiB to GPU in 2.18165s (16.5013GiB/s) | 20260505 09:31:54.555 | INFO | Downloaded 34GiB to GPU in 2.05792s (16.5215GiB/s) | 20260505 09:31:54.555 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 09:31:54.555 | INFO | Total mem usage: 15.3 GiB | 20260505 09:31:54.555 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 09:31:54.555 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.19%) | 20260505 09:31:54.555 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 09:31:54.555 | INFO | High Water Breakdown: | 20260505 09:31:54.555 | INFO | 60.9% : 8.79 GiB ( 94% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 09:31:54.555 | INFO | 39% : 5.62 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 09:31:54.555 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 09:31:54.555 | INFO | ------------------------------------------ | 20260505 09:31:54.555 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:31:54.555 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 09:31:54.555 | INFO | Total mem usage: 26.5 GiB | 20260505 09:31:54.555 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 09:31:54.555 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 09:31:54.555 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 09:31:54.555 | INFO | High Water Breakdown: | 20260505 09:31:54.556 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:31:54.556 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:31:54.556 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:31:54.556 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:31:54.556 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:31:54.556 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:31:54.556 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 09:31:54.556 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 09:31:54.556 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:31:54.556 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 09:31:54.556 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 09:31:54.556 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 09:31:54.556 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 09:31:54.556 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 09:31:54.556 | INFO | ------------------------------------------ | 20260505 09:31:54.556 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:31:54.556 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 09:31:54.556 | INFO | Total mem usage: 25.6 GiB | 20260505 09:31:54.556 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 09:31:54.556 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 09:31:54.556 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 09:31:54.556 | INFO | High Water Breakdown: | 20260505 09:31:54.556 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 09:31:54.556 | INFO | ------------------------------------------ | 20260505 09:31:54.556 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:31:54.556 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 09:31:54.556 | INFO | Total mem usage: 15.7 GiB | 20260505 09:31:54.556 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 09:31:54.556 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 09:31:54.556 | INFO | Unused Cache: 880 MiB (Waste: 5.48%) | 20260505 09:31:54.556 | INFO | High Water Breakdown: | 20260505 09:31:54.556 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 09:31:54.556 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 09:31:54.556 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 09:31:54.556 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 09:31:54.556 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 09:31:54.556 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 09:31:54.556 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 09:31:54.556 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 09:31:54.556 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 09:31:54.556 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:31:54.556 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 09:31:54.556 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:31:54.556 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 09:31:54.556 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 09:31:54.556 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 09:31:54.556 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 09:31:54.556 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:31:54.556 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 09:31:54.556 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:31:54.556 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:31:54.556 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:31:54.556 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 09:31:54.556 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 09:31:54.556 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:31:54.556 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 09:31:54.556 | INFO | ------------------------------------------ | 20260505 09:31:54.556 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:31:54.556 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 09:31:54.556 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:31:54.556 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:31:54.556 | INFO | PerfReport: Full Execution -- 3.25e+01 / 7.65e+01, 1.37e+01 / 2.38e+00 | 20260505 09:31:54.556 | INFO | PerfReport: 0.29%, AggregateStats -- 9.35e-02 / 1.06e-01, 1.16e-02 / 1.25e-02 | 20260505 09:31:54.556 | INFO | PerfReport: 11.63%, Baseline -- 3.78e+00 / 4.39e+00, 6.10e-01 / 3.13e-01 | 20260505 09:31:54.556 | INFO | PerfReport: 3.31%, Histogram -- 1.07e+00 / 1.46e+00, 2.07e-01 / 1.12e-01 | 20260505 09:31:54.556 | INFO | PerfReport: 15.56%, DME -- 4.04e+01 / 4.44e+01, 4.30e+00 / 2.14e+00 | 20260505 09:31:54.556 | INFO | PerfReport: 56.63%, FrameLabeling -- 1.84e+01 / 2.12e+01, 1.89e+00 / 1.76e+00 | 20260505 09:31:54.556 | INFO | PerfReport: 7.40%, PulseAccumulating -- 2.40e+00 / 2.94e+00, 4.05e-01 / 2.15e-01 | 20260505 09:31:54.556 | INFO | PerfReport: 3.82%, Metrics -- 1.24e+00 / 1.54e+00, 3.59e-01 / 1.78e-01 | 20260505 09:31:54.556 | INFO | PerfReport: 1.36%, PulseSerializing -- 4.40e-01 / 4.86e-01, 8.28e-02 / 2.76e-02 | 20260505 09:31:54.556 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 09:31:54.556 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:31:54.556 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:31:54.556 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.35e+01 / 0.00e+00 | 20260505 09:31:54.556 | INFO | PerfReport: 22.60%, Upload -- 2.30e+01 / 0.00e+00, 8.29e+00 / 0.00e+00 | 20260505 09:31:54.556 | INFO | PerfReport: 21.82%, Download -- 2.22e+01 / 0.00e+00, 8.31e+00 / 0.00e+00 | 20260505 09:31:54.556 | INFO | PerfReport: 31.98%, Compute -- 3.25e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 09:31:54.556 | INFO | PerfReport: 1.81%, LockUpload -- 1.84e+00 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260505 09:31:54.556 | INFO | PerfReport: 1.10%, LockDownload -- 1.12e+00 / 0.00e+00, 2.73e+00 / 0.00e+00 | 20260505 09:31:54.556 | INFO | PerfReport: 20.67%, LockCompute -- 2.10e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260505 09:31:54.556 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.52e-02 / 0.00e+00 | 20260505 09:31:54.684 | INFO | Starting a round of Baz IO | 20260505 09:31:57.955 | INFO | Populating RTMetrics report | 20260505 09:31:57.956 | INFO | RTMetrics report populated | 20260505 09:31:57.956 | INFO | Opening tmp RTMetrics json file | 20260505 09:31:58.003 | INFO | Serializing RTMetrics to json | 20260505 09:31:58.013 | INFO | Serialization done, swapping tmp file now... | 20260505 09:31:58.034 | INFO | RTMetrics done | 20260505 09:31:58.093 | INFO | Finished a round of Baz IO: Wrote 2.437 GiB in 3.409 seconds | 20260505 09:31:58.093 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.99% / 0.01% / 74.00% | 20260505 09:31:58.093 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.93239% / 1.19209e-05% / 8.34465e-05% | 20260505 09:31:58.094 | INFO | Baz IO overall: | 20260505 09:31:58.094 | INFO | BytesWritten : 1245661363851 | 20260505 09:31:58.094 | INFO | NumEvents : 592817946443 | 20260505 09:31:59.691 | INFO | Analyzing chunk frames = [2154496,2155008) | 20260505 09:31:59.691 | INFO | MemInfo: { | 20260505 09:31:59.691 | INFO | "gpu" : " 17 GB", | 20260505 09:31:59.691 | INFO | "huge" : "", | 20260505 09:31:59.691 | INFO | "huge_shared" : " 27 GB", | 20260505 09:31:59.691 | INFO | "malloc" : " 14 GB", | 20260505 09:31:59.691 | INFO | "pinned" : " 29 GB" | 20260505 09:31:59.691 | INFO | } | 20260505 09:31:59.691 | INFO | Perf report for the last 5.13706 seconds: | 20260505 09:31:59.692 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:31:59.692 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:59.692 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:59.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:59.692 | INFO | ANALYSIS : 65.1%, 2.9, 99.7ms, 0 | 20260505 09:31:59.692 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:31:59.692 | INFO | RT_METRICS : 47.2%, 1.0, 24.7ms, 0 | 20260505 09:31:59.692 | INFO | PRE_HQ : 63.0%, 1.8, 60.2ms, 0 | 20260505 09:31:59.692 | INFO | BAZWRITER : 65.6%, 4.2, 140.2ms, 0 | 20260505 09:31:59.692 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260505 09:31:59.692 | INFO | Uploaded 36GiB to GPU in 2.24956s (16.0032GiB/s) | 20260505 09:31:59.692 | INFO | Downloaded 38GiB to GPU in 2.35132s (16.1612GiB/s) | 20260505 09:32:04.831 | INFO | Analyzing chunk frames = [2155008,2155520) | 20260505 09:32:04.831 | INFO | MemInfo: { | 20260505 09:32:04.831 | INFO | "gpu" : " 17 GB", | 20260505 09:32:04.831 | INFO | "huge" : "", | 20260505 09:32:04.831 | INFO | "huge_shared" : " 27 GB", | 20260505 09:32:04.831 | INFO | "malloc" : " 14 GB", | 20260505 09:32:04.831 | INFO | "pinned" : " 29 GB" | 20260505 09:32:04.831 | INFO | } | 20260505 09:32:04.831 | INFO | Perf report for the last 5.1398 seconds: | 20260505 09:32:04.831 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:32:04.831 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:04.831 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:04.831 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:04.831 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 09:32:04.831 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:04.831 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:04.831 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 09:32:04.831 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:04.831 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:04.832 | INFO | Uploaded 36GiB to GPU in 2.20866s (16.2995GiB/s) | 20260505 09:32:04.832 | INFO | Downloaded 34GiB to GPU in 2.0895s (16.2718GiB/s) | 20260505 09:32:09.972 | INFO | Analyzing chunk frames = [2155520,2156032) | 20260505 09:32:09.972 | INFO | MemInfo: { | 20260505 09:32:09.972 | INFO | "gpu" : " 17 GB", | 20260505 09:32:09.972 | INFO | "huge" : "", | 20260505 09:32:09.972 | INFO | "huge_shared" : " 27 GB", | 20260505 09:32:09.972 | INFO | "malloc" : " 14 GB", | 20260505 09:32:09.972 | INFO | "pinned" : " 29 GB" | 20260505 09:32:09.972 | INFO | } | 20260505 09:32:09.972 | INFO | Perf report for the last 5.14049 seconds: | 20260505 09:32:09.972 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:32:09.972 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:09.972 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:09.972 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:09.972 | INFO | ANALYSIS : 63.2%, 3.0, 98.4ms, 0 | 20260505 09:32:09.972 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:09.972 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:09.972 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 09:32:09.972 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:09.972 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:09.972 | INFO | Uploaded 36GiB to GPU in 2.15067s (16.739GiB/s) | 20260505 09:32:09.972 | INFO | Downloaded 34GiB to GPU in 2.06766s (16.4437GiB/s) | 20260505 09:32:15.111 | INFO | Analyzing chunk frames = [2156032,2156544) | 20260505 09:32:15.111 | INFO | MemInfo: { | 20260505 09:32:15.111 | INFO | "gpu" : " 17 GB", | 20260505 09:32:15.112 | INFO | "huge" : "", | 20260505 09:32:15.112 | INFO | "huge_shared" : " 27 GB", | 20260505 09:32:15.112 | INFO | "malloc" : " 14 GB", | 20260505 09:32:15.112 | INFO | "pinned" : " 29 GB" | 20260505 09:32:15.112 | INFO | } | 20260505 09:32:15.112 | INFO | Perf report for the last 5.13985 seconds: | 20260505 09:32:15.112 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:32:15.112 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:15.112 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:15.112 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:15.112 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 09:32:15.112 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:15.112 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:15.112 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:32:15.112 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:15.112 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:15.112 | INFO | Uploaded 36GiB to GPU in 2.15826s (16.6801GiB/s) | 20260505 09:32:15.112 | INFO | Downloaded 34GiB to GPU in 2.07735s (16.367GiB/s) | 20260505 09:32:20.249 | INFO | Analyzing chunk frames = [2156544,2157056) | 20260505 09:32:20.249 | INFO | MemInfo: { | 20260505 09:32:20.249 | INFO | "gpu" : " 17 GB", | 20260505 09:32:20.249 | INFO | "huge" : "", | 20260505 09:32:20.249 | INFO | "huge_shared" : " 27 GB", | 20260505 09:32:20.249 | INFO | "malloc" : " 14 GB", | 20260505 09:32:20.249 | INFO | "pinned" : " 29 GB" | 20260505 09:32:20.249 | INFO | } | 20260505 09:32:20.249 | INFO | Perf report for the last 5.13763 seconds: | 20260505 09:32:20.249 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:32:20.249 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:20.249 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:20.249 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:20.249 | INFO | ANALYSIS : 63.8%, 3.0, 99.5ms, 0 | 20260505 09:32:20.249 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:20.249 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:20.249 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:32:20.249 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:20.249 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:20.250 | INFO | Uploaded 36GiB to GPU in 2.23542s (16.1044GiB/s) | 20260505 09:32:20.250 | INFO | Downloaded 34GiB to GPU in 2.10452s (16.1557GiB/s) | 20260505 09:32:25.390 | INFO | Analyzing chunk frames = [2157056,2157568) | 20260505 09:32:25.390 | INFO | MemInfo: { | 20260505 09:32:25.390 | INFO | "gpu" : " 17 GB", | 20260505 09:32:25.390 | INFO | "huge" : "", | 20260505 09:32:25.390 | INFO | "huge_shared" : " 27 GB", | 20260505 09:32:25.390 | INFO | "malloc" : " 14 GB", | 20260505 09:32:25.390 | INFO | "pinned" : " 29 GB" | 20260505 09:32:25.390 | INFO | } | 20260505 09:32:25.390 | INFO | Perf report for the last 5.14062 seconds: | 20260505 09:32:25.390 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:32:25.390 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:25.390 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:25.390 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:25.390 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 09:32:25.390 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:25.390 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:25.390 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 09:32:25.390 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:25.390 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:25.390 | INFO | Uploaded 36GiB to GPU in 2.23725s (16.0911GiB/s) | 20260505 09:32:25.390 | INFO | Downloaded 34GiB to GPU in 2.10482s (16.1534GiB/s) | 20260505 09:32:30.528 | INFO | Analyzing chunk frames = [2157568,2158080) | 20260505 09:32:30.528 | INFO | MemInfo: { | 20260505 09:32:30.528 | INFO | "gpu" : " 17 GB", | 20260505 09:32:30.528 | INFO | "huge" : "", | 20260505 09:32:30.528 | INFO | "huge_shared" : " 27 GB", | 20260505 09:32:30.528 | INFO | "malloc" : " 14 GB", | 20260505 09:32:30.528 | INFO | "pinned" : " 29 GB" | 20260505 09:32:30.528 | INFO | } | 20260505 09:32:30.528 | INFO | Perf report for the last 5.13817 seconds: | 20260505 09:32:30.528 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:32:30.528 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:30.528 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:30.528 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:30.528 | INFO | ANALYSIS : 63.8%, 3.0, 99.5ms, 0 | 20260505 09:32:30.528 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:30.528 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:30.528 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 09:32:30.528 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:30.528 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:30.528 | INFO | Uploaded 36GiB to GPU in 2.19909s (16.3704GiB/s) | 20260505 09:32:30.528 | INFO | Downloaded 34GiB to GPU in 2.13724s (15.9084GiB/s) | 20260505 09:32:35.667 | INFO | Analyzing chunk frames = [2158080,2158592) | 20260505 09:32:35.667 | INFO | MemInfo: { | 20260505 09:32:35.667 | INFO | "gpu" : " 17 GB", | 20260505 09:32:35.667 | INFO | "huge" : "", | 20260505 09:32:35.667 | INFO | "huge_shared" : " 27 GB", | 20260505 09:32:35.668 | INFO | "malloc" : " 14 GB", | 20260505 09:32:35.668 | INFO | "pinned" : " 29 GB" | 20260505 09:32:35.668 | INFO | } | 20260505 09:32:35.668 | INFO | Perf report for the last 5.13958 seconds: | 20260505 09:32:35.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:32:35.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:35.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:35.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:35.668 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 09:32:35.668 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:35.668 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:35.668 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:32:35.668 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:35.668 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:35.668 | INFO | Uploaded 36GiB to GPU in 2.20492s (16.3271GiB/s) | 20260505 09:32:35.668 | INFO | Downloaded 34GiB to GPU in 2.05009s (16.5847GiB/s) | 20260505 09:32:35.797 | INFO | Starting a round of Baz IO | 20260505 09:32:39.060 | INFO | Populating RTMetrics report | 20260505 09:32:39.061 | INFO | RTMetrics report populated | 20260505 09:32:39.061 | INFO | Opening tmp RTMetrics json file | 20260505 09:32:39.072 | INFO | Serializing RTMetrics to json | 20260505 09:32:39.112 | INFO | Serialization done, swapping tmp file now... | 20260505 09:32:39.122 | INFO | RTMetrics done | 20260505 09:32:39.228 | INFO | Finished a round of Baz IO: Wrote 2.436 GiB in 3.43 seconds | 20260505 09:32:39.228 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.97% / 0.01% / 74.01% | 20260505 09:32:39.228 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.924% / 2.38419e-05% / 9.13938e-05% | 20260505 09:32:39.228 | INFO | Baz IO overall: | 20260505 09:32:39.228 | INFO | BytesWritten : 1248276769053 | 20260505 09:32:39.228 | INFO | NumEvents : 594066987604 | 20260505 09:32:40.808 | INFO | Analyzing chunk frames = [2158592,2159104) | 20260505 09:32:40.808 | INFO | MemInfo: { | 20260505 09:32:40.808 | INFO | "gpu" : " 17 GB", | 20260505 09:32:40.808 | INFO | "huge" : "", | 20260505 09:32:40.808 | INFO | "huge_shared" : " 27 GB", | 20260505 09:32:40.808 | INFO | "malloc" : " 14 GB", | 20260505 09:32:40.808 | INFO | "pinned" : " 29 GB" | 20260505 09:32:40.808 | INFO | } | 20260505 09:32:40.808 | INFO | Perf report for the last 5.14006 seconds: | 20260505 09:32:40.808 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:32:40.808 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:40.808 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:40.808 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:40.808 | INFO | ANALYSIS : 64.9%, 2.9, 98.7ms, 0 | 20260505 09:32:40.808 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:40.808 | INFO | RT_METRICS : 46.8%, 1.0, 24.6ms, 0 | 20260505 09:32:40.808 | INFO | PRE_HQ : 63.0%, 1.8, 60.0ms, 0 | 20260505 09:32:40.808 | INFO | BAZWRITER : 66.0%, 4.2, 144.0ms, 0 | 20260505 09:32:40.808 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.9ms, 0 | 20260505 09:32:40.808 | INFO | Uploaded 36GiB to GPU in 2.27087s (15.853GiB/s) | 20260505 09:32:40.808 | INFO | Downloaded 38GiB to GPU in 2.38253s (15.9494GiB/s) | 20260505 09:32:45.948 | INFO | Analyzing chunk frames = [2159104,2159616) | 20260505 09:32:45.948 | INFO | MemInfo: { | 20260505 09:32:45.948 | INFO | "gpu" : " 17 GB", | 20260505 09:32:45.948 | INFO | "huge" : "", | 20260505 09:32:45.948 | INFO | "huge_shared" : " 27 GB", | 20260505 09:32:45.948 | INFO | "malloc" : " 14 GB", | 20260505 09:32:45.948 | INFO | "pinned" : " 29 GB" | 20260505 09:32:45.948 | INFO | } | 20260505 09:32:45.948 | INFO | Perf report for the last 5.14022 seconds: | 20260505 09:32:45.948 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:32:45.948 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:45.948 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:45.948 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:45.948 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 09:32:45.948 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:45.948 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:45.948 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 09:32:45.948 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:45.948 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:45.948 | INFO | Uploaded 36GiB to GPU in 2.17885s (16.5224GiB/s) | 20260505 09:32:45.948 | INFO | Downloaded 34GiB to GPU in 2.10153s (16.1787GiB/s) | 20260505 09:32:51.087 | INFO | Analyzing chunk frames = [2159616,2160128) | 20260505 09:32:51.087 | INFO | MemInfo: { | 20260505 09:32:51.087 | INFO | "gpu" : " 17 GB", | 20260505 09:32:51.087 | INFO | "huge" : "", | 20260505 09:32:51.087 | INFO | "huge_shared" : " 27 GB", | 20260505 09:32:51.087 | INFO | "malloc" : " 14 GB", | 20260505 09:32:51.087 | INFO | "pinned" : " 29 GB" | 20260505 09:32:51.087 | INFO | } | 20260505 09:32:51.087 | INFO | Perf report for the last 5.13935 seconds: | 20260505 09:32:51.087 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:32:51.087 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:51.087 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:51.087 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:51.087 | INFO | ANALYSIS : 63.0%, 3.0, 98.0ms, 0 | 20260505 09:32:51.087 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:51.087 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:51.087 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 09:32:51.088 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:51.088 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:51.088 | INFO | Uploaded 36GiB to GPU in 2.14362s (16.794GiB/s) | 20260505 09:32:51.088 | INFO | Downloaded 34GiB to GPU in 2.05528s (16.5428GiB/s) | 20260505 09:32:56.225 | INFO | Analyzing chunk frames = [2160128,2160640) | 20260505 09:32:56.225 | INFO | MemInfo: { | 20260505 09:32:56.225 | INFO | "gpu" : " 17 GB", | 20260505 09:32:56.225 | INFO | "huge" : "", | 20260505 09:32:56.225 | INFO | "huge_shared" : " 27 GB", | 20260505 09:32:56.225 | INFO | "malloc" : " 14 GB", | 20260505 09:32:56.225 | INFO | "pinned" : " 29 GB" | 20260505 09:32:56.225 | INFO | } | 20260505 09:32:56.225 | INFO | Perf report for the last 5.13751 seconds: | 20260505 09:32:56.225 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:32:56.225 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:56.225 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:56.225 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:56.225 | INFO | ANALYSIS : 63.3%, 3.0, 98.3ms, 0 | 20260505 09:32:56.225 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:56.225 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:56.225 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:32:56.225 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:56.225 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:32:56.225 | INFO | Uploaded 36GiB to GPU in 2.16026s (16.6646GiB/s) | 20260505 09:32:56.225 | INFO | Downloaded 34GiB to GPU in 2.07502s (16.3854GiB/s) | 20260505 09:33:01.364 | INFO | Analyzing chunk frames = [2160640,2161152) | 20260505 09:33:01.364 | INFO | MemInfo: { | 20260505 09:33:01.364 | INFO | "gpu" : " 17 GB", | 20260505 09:33:01.364 | INFO | "huge" : "", | 20260505 09:33:01.364 | INFO | "huge_shared" : " 27 GB", | 20260505 09:33:01.364 | INFO | "malloc" : " 14 GB", | 20260505 09:33:01.364 | INFO | "pinned" : " 29 GB" | 20260505 09:33:01.364 | INFO | } | 20260505 09:33:01.364 | INFO | Perf report for the last 5.13926 seconds: | 20260505 09:33:01.364 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:33:01.364 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:01.364 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:01.364 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:01.364 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 09:33:01.364 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:01.364 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:01.364 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:33:01.364 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:01.364 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:01.364 | INFO | Uploaded 36GiB to GPU in 2.23564s (16.1028GiB/s) | 20260505 09:33:01.364 | INFO | Downloaded 34GiB to GPU in 2.10476s (16.1538GiB/s) | 20260505 09:33:06.506 | INFO | Analyzing chunk frames = [2161152,2161664) | 20260505 09:33:06.506 | INFO | MemInfo: { | 20260505 09:33:06.506 | INFO | "gpu" : " 17 GB", | 20260505 09:33:06.506 | INFO | "huge" : "", | 20260505 09:33:06.506 | INFO | "huge_shared" : " 27 GB", | 20260505 09:33:06.506 | INFO | "malloc" : " 14 GB", | 20260505 09:33:06.506 | INFO | "pinned" : " 29 GB" | 20260505 09:33:06.506 | INFO | } | 20260505 09:33:06.506 | INFO | Perf report for the last 5.14224 seconds: | 20260505 09:33:06.506 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:33:06.506 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:06.506 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:06.506 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:06.506 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 09:33:06.507 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:06.507 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:06.507 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 09:33:06.507 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:06.507 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:06.507 | INFO | Uploaded 36GiB to GPU in 2.16134s (16.6563GiB/s) | 20260505 09:33:06.507 | INFO | Downloaded 34GiB to GPU in 2.05681s (16.5305GiB/s) | 20260505 09:33:11.643 | INFO | Analyzing chunk frames = [2161664,2162176) | 20260505 09:33:11.643 | INFO | MemInfo: { | 20260505 09:33:11.643 | INFO | "gpu" : " 17 GB", | 20260505 09:33:11.643 | INFO | "huge" : "", | 20260505 09:33:11.643 | INFO | "huge_shared" : " 27 GB", | 20260505 09:33:11.643 | INFO | "malloc" : " 14 GB", | 20260505 09:33:11.643 | INFO | "pinned" : " 29 GB" | 20260505 09:33:11.643 | INFO | } | 20260505 09:33:11.643 | INFO | Perf report for the last 5.13667 seconds: | 20260505 09:33:11.643 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:33:11.643 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:11.643 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:11.643 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:11.643 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 09:33:11.643 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:11.643 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:11.643 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 09:33:11.643 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:11.643 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:11.644 | INFO | Uploaded 36GiB to GPU in 2.18201s (16.4986GiB/s) | 20260505 09:33:11.644 | INFO | Downloaded 34GiB to GPU in 2.14703s (15.8359GiB/s) | 20260505 09:33:16.783 | INFO | Analyzing chunk frames = [2162176,2162688) | 20260505 09:33:16.783 | INFO | MemInfo: { | 20260505 09:33:16.783 | INFO | "gpu" : " 17 GB", | 20260505 09:33:16.783 | INFO | "huge" : "", | 20260505 09:33:16.783 | INFO | "huge_shared" : " 27 GB", | 20260505 09:33:16.783 | INFO | "malloc" : " 14 GB", | 20260505 09:33:16.783 | INFO | "pinned" : " 29 GB" | 20260505 09:33:16.783 | INFO | } | 20260505 09:33:16.783 | INFO | Perf report for the last 5.1399 seconds: | 20260505 09:33:16.783 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:33:16.783 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:16.783 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:16.783 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:16.783 | INFO | ANALYSIS : 63.9%, 3.0, 99.9ms, 0 | 20260505 09:33:16.783 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:16.783 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:16.783 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:33:16.783 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:16.783 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:16.783 | INFO | Uploaded 36GiB to GPU in 2.18243s (16.4954GiB/s) | 20260505 09:33:16.783 | INFO | Downloaded 34GiB to GPU in 2.05834s (16.5182GiB/s) | 20260505 09:33:16.784 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 09:33:16.784 | INFO | Total mem usage: 15.3 GiB | 20260505 09:33:16.784 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 09:33:16.784 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -5.8%) | 20260505 09:33:16.784 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 09:33:16.784 | INFO | High Water Breakdown: | 20260505 09:33:16.784 | INFO | 61% : 8.84 GiB ( 94.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 09:33:16.784 | INFO | 38.8% : 5.62 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 09:33:16.784 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 09:33:16.784 | INFO | ------------------------------------------ | 20260505 09:33:16.784 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:33:16.784 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 09:33:16.784 | INFO | Total mem usage: 26.5 GiB | 20260505 09:33:16.784 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 09:33:16.784 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 09:33:16.784 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 09:33:16.784 | INFO | High Water Breakdown: | 20260505 09:33:16.784 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:33:16.784 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:33:16.784 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:33:16.784 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:33:16.784 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:33:16.784 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:33:16.784 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 09:33:16.784 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 09:33:16.784 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:33:16.784 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 09:33:16.784 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 09:33:16.784 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 09:33:16.784 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 09:33:16.784 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 09:33:16.784 | INFO | ------------------------------------------ | 20260505 09:33:16.784 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:33:16.784 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 09:33:16.784 | INFO | Total mem usage: 25.6 GiB | 20260505 09:33:16.784 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 09:33:16.784 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 09:33:16.784 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 09:33:16.784 | INFO | High Water Breakdown: | 20260505 09:33:16.784 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 09:33:16.784 | INFO | ------------------------------------------ | 20260505 09:33:16.785 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:33:16.785 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 09:33:16.785 | INFO | Total mem usage: 15.7 GiB | 20260505 09:33:16.785 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 09:33:16.785 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 09:33:16.785 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 09:33:16.785 | INFO | High Water Breakdown: | 20260505 09:33:16.785 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 09:33:16.785 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 09:33:16.785 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 09:33:16.785 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 09:33:16.785 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 09:33:16.785 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 09:33:16.785 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 09:33:16.785 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 09:33:16.785 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 09:33:16.785 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:33:16.785 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 09:33:16.785 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:33:16.785 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 09:33:16.785 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 09:33:16.785 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 09:33:16.785 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 09:33:16.785 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:33:16.785 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 09:33:16.785 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:33:16.785 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:33:16.785 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:33:16.785 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 09:33:16.785 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 09:33:16.785 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:33:16.785 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 09:33:16.785 | INFO | ------------------------------------------ | 20260505 09:33:16.785 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:33:16.785 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 09:33:16.785 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:33:16.785 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:33:16.785 | INFO | PerfReport: Full Execution -- 3.25e+01 / 7.63e+01, 1.38e+01 / 2.20e+00 | 20260505 09:33:16.785 | INFO | PerfReport: 0.29%, AggregateStats -- 9.31e-02 / 1.02e-01, 1.14e-02 / 4.69e-03 | 20260505 09:33:16.785 | INFO | PerfReport: 11.67%, Baseline -- 3.79e+00 / 4.39e+00, 6.17e-01 / 3.39e-01 | 20260505 09:33:16.785 | INFO | PerfReport: 3.30%, Histogram -- 1.07e+00 / 1.45e+00, 2.06e-01 / 1.22e-01 | 20260505 09:33:16.785 | INFO | PerfReport: 15.54%, DME -- 4.03e+01 / 4.43e+01, 4.17e+00 / 1.73e+00 | 20260505 09:33:16.785 | INFO | PerfReport: 56.57%, FrameLabeling -- 1.84e+01 / 2.09e+01, 1.88e+00 / 1.66e+00 | 20260505 09:33:16.785 | INFO | PerfReport: 7.44%, PulseAccumulating -- 2.41e+00 / 2.99e+00, 4.14e-01 / 1.80e-01 | 20260505 09:33:16.785 | INFO | PerfReport: 3.84%, Metrics -- 1.24e+00 / 1.58e+00, 3.61e-01 / 2.26e-01 | 20260505 09:33:16.785 | INFO | PerfReport: 1.36%, PulseSerializing -- 4.40e-01 / 4.90e-01, 8.25e-02 / 2.63e-02 | 20260505 09:33:16.785 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 09:33:16.785 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:33:16.785 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:33:16.785 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.35e+01 / 0.00e+00 | 20260505 09:33:16.785 | INFO | PerfReport: 22.58%, Upload -- 2.29e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260505 09:33:16.785 | INFO | PerfReport: 21.81%, Download -- 2.21e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260505 09:33:16.785 | INFO | PerfReport: 32.01%, Compute -- 3.25e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 09:33:16.785 | INFO | PerfReport: 1.80%, LockUpload -- 1.82e+00 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260505 09:33:16.785 | INFO | PerfReport: 1.12%, LockDownload -- 1.14e+00 / 0.00e+00, 2.72e+00 / 0.00e+00 | 20260505 09:33:16.785 | INFO | PerfReport: 20.66%, LockCompute -- 2.09e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260505 09:33:16.785 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 1.97e-02 / 0.00e+00 | 20260505 09:33:16.912 | INFO | Starting a round of Baz IO | 20260505 09:33:20.193 | INFO | Populating RTMetrics report | 20260505 09:33:20.193 | INFO | RTMetrics report populated | 20260505 09:33:20.193 | INFO | Opening tmp RTMetrics json file | 20260505 09:33:20.197 | INFO | Serializing RTMetrics to json | 20260505 09:33:20.204 | INFO | Serialization done, swapping tmp file now... | 20260505 09:33:20.219 | INFO | RTMetrics done | 20260505 09:33:20.334 | INFO | Finished a round of Baz IO: Wrote 2.438 GiB in 3.422 seconds | 20260505 09:33:20.335 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.96% / 0.01% / 74.03% | 20260505 09:33:20.335 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.9229% / 1.98682e-05% / 7.54992e-05% | 20260505 09:33:20.335 | INFO | Baz IO overall: | 20260505 09:33:20.335 | INFO | BytesWritten : 1250894062278 | 20260505 09:33:20.335 | INFO | NumEvents : 595317420221 | 20260505 09:33:21.922 | INFO | Analyzing chunk frames = [2162688,2163200) | 20260505 09:33:21.922 | INFO | MemInfo: { | 20260505 09:33:21.922 | INFO | "gpu" : " 17 GB", | 20260505 09:33:21.922 | INFO | "huge" : "", | 20260505 09:33:21.922 | INFO | "huge_shared" : " 27 GB", | 20260505 09:33:21.922 | INFO | "malloc" : " 14 GB", | 20260505 09:33:21.922 | INFO | "pinned" : " 29 GB" | 20260505 09:33:21.922 | INFO | } | 20260505 09:33:21.922 | INFO | Perf report for the last 5.13945 seconds: | 20260505 09:33:21.923 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:33:21.923 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:21.923 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:21.923 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:21.923 | INFO | ANALYSIS : 65.2%, 2.9, 99.9ms, 0 | 20260505 09:33:21.923 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:21.923 | INFO | RT_METRICS : 46.2%, 1.0, 24.3ms, 0 | 20260505 09:33:21.923 | INFO | PRE_HQ : 62.9%, 1.8, 60.2ms, 0 | 20260505 09:33:21.923 | INFO | BAZWRITER : 65.9%, 4.2, 141.2ms, 0 | 20260505 09:33:21.923 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 09:33:21.923 | INFO | Uploaded 36GiB to GPU in 2.24119s (16.0629GiB/s) | 20260505 09:33:21.923 | INFO | Downloaded 38GiB to GPU in 2.34133s (16.2301GiB/s) | 20260505 09:33:27.062 | INFO | Analyzing chunk frames = [2163200,2163712) | 20260505 09:33:27.062 | INFO | MemInfo: { | 20260505 09:33:27.062 | INFO | "gpu" : " 17 GB", | 20260505 09:33:27.062 | INFO | "huge" : "", | 20260505 09:33:27.062 | INFO | "huge_shared" : " 27 GB", | 20260505 09:33:27.062 | INFO | "malloc" : " 14 GB", | 20260505 09:33:27.062 | INFO | "pinned" : " 29 GB" | 20260505 09:33:27.062 | INFO | } | 20260505 09:33:27.062 | INFO | Perf report for the last 5.13952 seconds: | 20260505 09:33:27.062 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:33:27.062 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:27.062 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:27.062 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:27.062 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 09:33:27.062 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:27.062 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:27.062 | INFO | PRE_HQ : 29.3%, 1.0, 15.3ms, 0 | 20260505 09:33:27.062 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:27.062 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:27.062 | INFO | Uploaded 36GiB to GPU in 2.19987s (16.3646GiB/s) | 20260505 09:33:27.062 | INFO | Downloaded 34GiB to GPU in 2.10184s (16.1763GiB/s) | 20260505 09:33:32.200 | INFO | Analyzing chunk frames = [2163712,2164224) | 20260505 09:33:32.200 | INFO | MemInfo: { | 20260505 09:33:32.200 | INFO | "gpu" : " 17 GB", | 20260505 09:33:32.200 | INFO | "huge" : "", | 20260505 09:33:32.200 | INFO | "huge_shared" : " 27 GB", | 20260505 09:33:32.200 | INFO | "malloc" : " 14 GB", | 20260505 09:33:32.200 | INFO | "pinned" : " 29 GB" | 20260505 09:33:32.200 | INFO | } | 20260505 09:33:32.200 | INFO | Perf report for the last 5.1383 seconds: | 20260505 09:33:32.200 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:33:32.200 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:32.200 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:32.200 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:32.200 | INFO | ANALYSIS : 63.0%, 3.0, 98.1ms, 0 | 20260505 09:33:32.200 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:32.200 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:32.200 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:33:32.200 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:32.200 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:32.201 | INFO | Uploaded 36GiB to GPU in 2.14511s (16.7824GiB/s) | 20260505 09:33:32.201 | INFO | Downloaded 34GiB to GPU in 2.05767s (16.5236GiB/s) | 20260505 09:33:37.339 | INFO | Analyzing chunk frames = [2164224,2164736) | 20260505 09:33:37.339 | INFO | MemInfo: { | 20260505 09:33:37.339 | INFO | "gpu" : " 17 GB", | 20260505 09:33:37.339 | INFO | "huge" : "", | 20260505 09:33:37.339 | INFO | "huge_shared" : " 27 GB", | 20260505 09:33:37.339 | INFO | "malloc" : " 14 GB", | 20260505 09:33:37.339 | INFO | "pinned" : " 29 GB" | 20260505 09:33:37.339 | INFO | } | 20260505 09:33:37.339 | INFO | Perf report for the last 5.13898 seconds: | 20260505 09:33:37.339 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:33:37.339 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:37.339 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:37.339 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:37.339 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 09:33:37.339 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:37.339 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:37.339 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:33:37.339 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:37.339 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:37.340 | INFO | Uploaded 36GiB to GPU in 2.1608s (16.6605GiB/s) | 20260505 09:33:37.340 | INFO | Downloaded 34GiB to GPU in 2.07431s (16.391GiB/s) | 20260505 09:33:42.479 | INFO | Analyzing chunk frames = [2164736,2165248) | 20260505 09:33:42.479 | INFO | MemInfo: { | 20260505 09:33:42.479 | INFO | "gpu" : " 17 GB", | 20260505 09:33:42.479 | INFO | "huge" : "", | 20260505 09:33:42.479 | INFO | "huge_shared" : " 27 GB", | 20260505 09:33:42.479 | INFO | "malloc" : " 14 GB", | 20260505 09:33:42.479 | INFO | "pinned" : " 29 GB" | 20260505 09:33:42.479 | INFO | } | 20260505 09:33:42.479 | INFO | Perf report for the last 5.13993 seconds: | 20260505 09:33:42.479 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:33:42.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:42.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:42.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:42.479 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 09:33:42.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:42.479 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:42.479 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 09:33:42.479 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:42.479 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:42.480 | INFO | Uploaded 36GiB to GPU in 2.23988s (16.0723GiB/s) | 20260505 09:33:42.480 | INFO | Downloaded 34GiB to GPU in 2.09675s (16.2156GiB/s) | 20260505 09:33:47.620 | INFO | Analyzing chunk frames = [2165248,2165760) | 20260505 09:33:47.620 | INFO | MemInfo: { | 20260505 09:33:47.620 | INFO | "gpu" : " 17 GB", | 20260505 09:33:47.620 | INFO | "huge" : "", | 20260505 09:33:47.620 | INFO | "huge_shared" : " 27 GB", | 20260505 09:33:47.620 | INFO | "malloc" : " 14 GB", | 20260505 09:33:47.620 | INFO | "pinned" : " 29 GB" | 20260505 09:33:47.620 | INFO | } | 20260505 09:33:47.620 | INFO | Perf report for the last 5.14057 seconds: | 20260505 09:33:47.620 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:33:47.620 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:47.620 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:47.620 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:47.620 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 09:33:47.620 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:47.620 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:47.620 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 09:33:47.620 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:47.620 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:47.620 | INFO | Uploaded 36GiB to GPU in 2.15851s (16.6782GiB/s) | 20260505 09:33:47.620 | INFO | Downloaded 34GiB to GPU in 2.05592s (16.5376GiB/s) | 20260505 09:33:52.758 | INFO | Analyzing chunk frames = [2165760,2166272) | 20260505 09:33:52.758 | INFO | MemInfo: { | 20260505 09:33:52.758 | INFO | "gpu" : " 17 GB", | 20260505 09:33:52.758 | INFO | "huge" : "", | 20260505 09:33:52.758 | INFO | "huge_shared" : " 27 GB", | 20260505 09:33:52.758 | INFO | "malloc" : " 14 GB", | 20260505 09:33:52.758 | INFO | "pinned" : " 29 GB" | 20260505 09:33:52.758 | INFO | } | 20260505 09:33:52.758 | INFO | Perf report for the last 5.13836 seconds: | 20260505 09:33:52.758 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:33:52.758 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:52.758 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:52.758 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:52.758 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 09:33:52.758 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:52.758 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:52.758 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 09:33:52.758 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:52.758 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:52.758 | INFO | Uploaded 36GiB to GPU in 2.19674s (16.3879GiB/s) | 20260505 09:33:52.759 | INFO | Downloaded 34GiB to GPU in 2.14363s (15.861GiB/s) | 20260505 09:33:57.897 | INFO | Analyzing chunk frames = [2166272,2166784) | 20260505 09:33:57.897 | INFO | MemInfo: { | 20260505 09:33:57.897 | INFO | "gpu" : " 17 GB", | 20260505 09:33:57.897 | INFO | "huge" : "", | 20260505 09:33:57.897 | INFO | "huge_shared" : " 27 GB", | 20260505 09:33:57.897 | INFO | "malloc" : " 14 GB", | 20260505 09:33:57.897 | INFO | "pinned" : " 29 GB" | 20260505 09:33:57.897 | INFO | } | 20260505 09:33:57.898 | INFO | Perf report for the last 5.13933 seconds: | 20260505 09:33:57.898 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:33:57.898 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:57.898 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:57.898 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:57.898 | INFO | ANALYSIS : 64.7%, 3.0, 101.1ms, 0 | 20260505 09:33:57.898 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:57.898 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:57.898 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 09:33:57.898 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:57.898 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:33:57.898 | INFO | Uploaded 36GiB to GPU in 2.24511s (16.0349GiB/s) | 20260505 09:33:57.898 | INFO | Downloaded 34GiB to GPU in 2.08765s (16.2862GiB/s) | 20260505 09:33:58.027 | INFO | Starting a round of Baz IO | 20260505 09:34:01.300 | INFO | Populating RTMetrics report | 20260505 09:34:01.300 | INFO | RTMetrics report populated | 20260505 09:34:01.300 | INFO | Opening tmp RTMetrics json file | 20260505 09:34:01.310 | INFO | Serializing RTMetrics to json | 20260505 09:34:01.322 | INFO | Serialization done, swapping tmp file now... | 20260505 09:34:01.337 | INFO | RTMetrics done | 20260505 09:34:01.445 | INFO | Finished a round of Baz IO: Wrote 2.437 GiB in 3.417 seconds | 20260505 09:34:01.445 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.94% / 0.01% / 74.04% | 20260505 09:34:01.445 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.91824% / 2.38419e-05% / 8.74201e-05% | 20260505 09:34:01.445 | INFO | Baz IO overall: | 20260505 09:34:01.445 | INFO | BytesWritten : 1253510558399 | 20260505 09:34:01.445 | INFO | NumEvents : 596567386602 | 20260505 09:34:03.037 | INFO | Analyzing chunk frames = [2166784,2167296) | 20260505 09:34:03.038 | INFO | MemInfo: { | 20260505 09:34:03.038 | INFO | "gpu" : " 17 GB", | 20260505 09:34:03.038 | INFO | "huge" : "", | 20260505 09:34:03.038 | INFO | "huge_shared" : " 27 GB", | 20260505 09:34:03.038 | INFO | "malloc" : " 14 GB", | 20260505 09:34:03.038 | INFO | "pinned" : " 29 GB" | 20260505 09:34:03.038 | INFO | } | 20260505 09:34:03.038 | INFO | Perf report for the last 5.14022 seconds: | 20260505 09:34:03.038 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:34:03.038 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:03.038 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:03.038 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:03.038 | INFO | ANALYSIS : 65.0%, 2.9, 99.8ms, 0 | 20260505 09:34:03.038 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:03.038 | INFO | RT_METRICS : 46.5%, 1.0, 24.4ms, 0 | 20260505 09:34:03.038 | INFO | PRE_HQ : 63.0%, 1.8, 60.2ms, 0 | 20260505 09:34:03.038 | INFO | BAZWRITER : 65.8%, 4.2, 140.5ms, 0 | 20260505 09:34:03.038 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 09:34:03.038 | INFO | Uploaded 36GiB to GPU in 2.25364s (15.9741GiB/s) | 20260505 09:34:03.038 | INFO | Downloaded 38GiB to GPU in 2.35944s (16.1056GiB/s) | 20260505 09:34:08.176 | INFO | Analyzing chunk frames = [2167296,2167808) | 20260505 09:34:08.176 | INFO | MemInfo: { | 20260505 09:34:08.176 | INFO | "gpu" : " 17 GB", | 20260505 09:34:08.176 | INFO | "huge" : "", | 20260505 09:34:08.176 | INFO | "huge_shared" : " 27 GB", | 20260505 09:34:08.176 | INFO | "malloc" : " 14 GB", | 20260505 09:34:08.176 | INFO | "pinned" : " 29 GB" | 20260505 09:34:08.176 | INFO | } | 20260505 09:34:08.176 | INFO | Perf report for the last 5.13838 seconds: | 20260505 09:34:08.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:34:08.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:08.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:08.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:08.176 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 09:34:08.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:08.176 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:08.176 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:34:08.176 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:08.176 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:08.176 | INFO | Uploaded 36GiB to GPU in 2.17761s (16.5318GiB/s) | 20260505 09:34:08.177 | INFO | Downloaded 34GiB to GPU in 2.10059s (16.1859GiB/s) | 20260505 09:34:13.315 | INFO | Analyzing chunk frames = [2167808,2168320) | 20260505 09:34:13.315 | INFO | MemInfo: { | 20260505 09:34:13.315 | INFO | "gpu" : " 17 GB", | 20260505 09:34:13.315 | INFO | "huge" : "", | 20260505 09:34:13.315 | INFO | "huge_shared" : " 27 GB", | 20260505 09:34:13.315 | INFO | "malloc" : " 14 GB", | 20260505 09:34:13.315 | INFO | "pinned" : " 29 GB" | 20260505 09:34:13.315 | INFO | } | 20260505 09:34:13.315 | INFO | Perf report for the last 5.13875 seconds: | 20260505 09:34:13.315 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:34:13.315 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:13.315 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:13.315 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:13.315 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 09:34:13.315 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:13.315 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:13.315 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:34:13.315 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:13.315 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:13.315 | INFO | Uploaded 36GiB to GPU in 2.14463s (16.7861GiB/s) | 20260505 09:34:13.315 | INFO | Downloaded 34GiB to GPU in 2.06491s (16.4656GiB/s) | 20260505 09:34:18.456 | INFO | Analyzing chunk frames = [2168320,2168832) | 20260505 09:34:18.456 | INFO | MemInfo: { | 20260505 09:34:18.456 | INFO | "gpu" : " 17 GB", | 20260505 09:34:18.456 | INFO | "huge" : "", | 20260505 09:34:18.456 | INFO | "huge_shared" : " 27 GB", | 20260505 09:34:18.456 | INFO | "malloc" : " 14 GB", | 20260505 09:34:18.456 | INFO | "pinned" : " 29 GB" | 20260505 09:34:18.456 | INFO | } | 20260505 09:34:18.456 | INFO | Perf report for the last 5.14088 seconds: | 20260505 09:34:18.456 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:34:18.456 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:18.456 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:18.456 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:18.456 | INFO | ANALYSIS : 62.9%, 3.0, 97.8ms, 0 | 20260505 09:34:18.456 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:18.456 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:18.456 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 09:34:18.456 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:18.456 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:18.456 | INFO | Uploaded 36GiB to GPU in 2.16334s (16.6409GiB/s) | 20260505 09:34:18.456 | INFO | Downloaded 34GiB to GPU in 2.07994s (16.3466GiB/s) | 20260505 09:34:23.594 | INFO | Analyzing chunk frames = [2168832,2169344) | 20260505 09:34:23.594 | INFO | MemInfo: { | 20260505 09:34:23.594 | INFO | "gpu" : " 17 GB", | 20260505 09:34:23.594 | INFO | "huge" : "", | 20260505 09:34:23.594 | INFO | "huge_shared" : " 27 GB", | 20260505 09:34:23.595 | INFO | "malloc" : " 14 GB", | 20260505 09:34:23.595 | INFO | "pinned" : " 29 GB" | 20260505 09:34:23.595 | INFO | } | 20260505 09:34:23.595 | INFO | Perf report for the last 5.13873 seconds: | 20260505 09:34:23.595 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:34:23.595 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:23.595 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:23.595 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:23.595 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 09:34:23.595 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:23.595 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:23.595 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 09:34:23.595 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:23.595 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:23.595 | INFO | Uploaded 36GiB to GPU in 2.23597s (16.1004GiB/s) | 20260505 09:34:23.595 | INFO | Downloaded 34GiB to GPU in 2.10408s (16.1591GiB/s) | 20260505 09:34:28.734 | INFO | Analyzing chunk frames = [2169344,2169856) | 20260505 09:34:28.734 | INFO | MemInfo: { | 20260505 09:34:28.734 | INFO | "gpu" : " 17 GB", | 20260505 09:34:28.734 | INFO | "huge" : "", | 20260505 09:34:28.734 | INFO | "huge_shared" : " 27 GB", | 20260505 09:34:28.734 | INFO | "malloc" : " 14 GB", | 20260505 09:34:28.734 | INFO | "pinned" : " 29 GB" | 20260505 09:34:28.734 | INFO | } | 20260505 09:34:28.734 | INFO | Perf report for the last 5.13985 seconds: | 20260505 09:34:28.734 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:34:28.734 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:28.734 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:28.735 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:28.735 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 09:34:28.735 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:28.735 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:28.735 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 09:34:28.735 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:28.735 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:28.735 | INFO | Uploaded 36GiB to GPU in 2.20315s (16.3403GiB/s) | 20260505 09:34:28.735 | INFO | Downloaded 34GiB to GPU in 2.09673s (16.2157GiB/s) | 20260505 09:34:33.873 | INFO | Analyzing chunk frames = [2169856,2170368) | 20260505 09:34:33.873 | INFO | MemInfo: { | 20260505 09:34:33.873 | INFO | "gpu" : " 17 GB", | 20260505 09:34:33.873 | INFO | "huge" : "", | 20260505 09:34:33.873 | INFO | "huge_shared" : " 27 GB", | 20260505 09:34:33.873 | INFO | "malloc" : " 14 GB", | 20260505 09:34:33.873 | INFO | "pinned" : " 29 GB" | 20260505 09:34:33.873 | INFO | } | 20260505 09:34:33.873 | INFO | Perf report for the last 5.13849 seconds: | 20260505 09:34:33.873 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:34:33.873 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:33.873 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:33.873 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:33.873 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 09:34:33.873 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:33.873 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:33.873 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 09:34:33.873 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:33.873 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:33.873 | INFO | Uploaded 36GiB to GPU in 2.18809s (16.4527GiB/s) | 20260505 09:34:33.873 | INFO | Downloaded 34GiB to GPU in 2.16926s (15.6736GiB/s) | 20260505 09:34:39.012 | INFO | Analyzing chunk frames = [2170368,2170880) | 20260505 09:34:39.012 | INFO | MemInfo: { | 20260505 09:34:39.012 | INFO | "gpu" : " 17 GB", | 20260505 09:34:39.012 | INFO | "huge" : "", | 20260505 09:34:39.012 | INFO | "huge_shared" : " 27 GB", | 20260505 09:34:39.012 | INFO | "malloc" : " 14 GB", | 20260505 09:34:39.012 | INFO | "pinned" : " 29 GB" | 20260505 09:34:39.012 | INFO | } | 20260505 09:34:39.012 | INFO | Perf report for the last 5.13926 seconds: | 20260505 09:34:39.012 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:34:39.012 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:39.012 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:39.012 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:39.012 | INFO | ANALYSIS : 64.7%, 3.0, 101.0ms, 0 | 20260505 09:34:39.012 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:39.012 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:39.012 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 09:34:39.012 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:39.012 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:39.013 | INFO | Uploaded 36GiB to GPU in 2.24856s (16.0102GiB/s) | 20260505 09:34:39.013 | INFO | Downloaded 34GiB to GPU in 2.08319s (16.3211GiB/s) | 20260505 09:34:39.013 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 09:34:39.013 | INFO | Total mem usage: 15.3 GiB | 20260505 09:34:39.013 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 09:34:39.013 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.55%) | 20260505 09:34:39.013 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 09:34:39.013 | INFO | High Water Breakdown: | 20260505 09:34:39.013 | INFO | 61% : 8.77 GiB ( 93.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 09:34:39.013 | INFO | 38.9% : 5.6 GiB ( 84.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 09:34:39.013 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 09:34:39.013 | INFO | ------------------------------------------ | 20260505 09:34:39.013 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:34:39.013 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 09:34:39.013 | INFO | Total mem usage: 26.5 GiB | 20260505 09:34:39.013 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 09:34:39.013 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 09:34:39.013 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 09:34:39.013 | INFO | High Water Breakdown: | 20260505 09:34:39.013 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:34:39.013 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:34:39.013 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:34:39.013 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:34:39.013 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:34:39.013 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:34:39.013 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 09:34:39.013 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 09:34:39.013 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:34:39.013 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 09:34:39.013 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 09:34:39.013 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 09:34:39.013 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 09:34:39.013 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 09:34:39.013 | INFO | ------------------------------------------ | 20260505 09:34:39.013 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:34:39.013 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 09:34:39.013 | INFO | Total mem usage: 25.6 GiB | 20260505 09:34:39.013 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 09:34:39.013 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 09:34:39.013 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 09:34:39.013 | INFO | High Water Breakdown: | 20260505 09:34:39.013 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 09:34:39.013 | INFO | ------------------------------------------ | 20260505 09:34:39.013 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:34:39.013 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 09:34:39.013 | INFO | Total mem usage: 15.7 GiB | 20260505 09:34:39.013 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 09:34:39.013 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 09:34:39.013 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 09:34:39.013 | INFO | High Water Breakdown: | 20260505 09:34:39.013 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 09:34:39.013 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 09:34:39.013 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 09:34:39.014 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 09:34:39.014 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 09:34:39.014 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 09:34:39.014 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 09:34:39.014 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 09:34:39.014 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 09:34:39.014 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:34:39.014 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 09:34:39.014 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:34:39.014 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 09:34:39.014 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 09:34:39.014 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 09:34:39.014 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 09:34:39.014 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:34:39.014 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 09:34:39.014 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:34:39.014 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:34:39.014 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:34:39.014 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 09:34:39.014 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 09:34:39.014 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:34:39.014 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 09:34:39.014 | INFO | ------------------------------------------ | 20260505 09:34:39.014 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:34:39.014 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 09:34:39.014 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:34:39.014 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:34:39.014 | INFO | PerfReport: Full Execution -- 3.25e+01 / 7.75e+01, 1.39e+01 / 2.33e+00 | 20260505 09:34:39.014 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 1.03e-01, 1.04e-02 / 6.43e-03 | 20260505 09:34:39.014 | INFO | PerfReport: 11.63%, Baseline -- 3.78e+00 / 4.42e+00, 6.16e-01 / 2.75e-01 | 20260505 09:34:39.014 | INFO | PerfReport: 3.30%, Histogram -- 1.07e+00 / 1.46e+00, 2.10e-01 / 1.13e-01 | 20260505 09:34:39.014 | INFO | PerfReport: 15.67%, DME -- 4.07e+01 / 4.52e+01, 4.67e+00 / 2.40e+00 | 20260505 09:34:39.014 | INFO | PerfReport: 56.56%, FrameLabeling -- 1.84e+01 / 2.13e+01, 1.88e+00 / 1.20e+00 | 20260505 09:34:39.014 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.40e+00 / 2.98e+00, 4.16e-01 / 2.46e-01 | 20260505 09:34:39.014 | INFO | PerfReport: 3.81%, Metrics -- 1.24e+00 / 1.53e+00, 3.61e-01 / 2.18e-01 | 20260505 09:34:39.014 | INFO | PerfReport: 1.35%, PulseSerializing -- 4.39e-01 / 4.88e-01, 8.30e-02 / 3.04e-02 | 20260505 09:34:39.014 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 09:34:39.014 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:34:39.014 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:34:39.014 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.40e+01 / 0.00e+00 | 20260505 09:34:39.014 | INFO | PerfReport: 22.56%, Upload -- 2.29e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260505 09:34:39.014 | INFO | PerfReport: 21.80%, Download -- 2.21e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260505 09:34:39.014 | INFO | PerfReport: 32.00%, Compute -- 3.25e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 09:34:39.014 | INFO | PerfReport: 1.87%, LockUpload -- 1.90e+00 / 0.00e+00, 7.45e+00 / 0.00e+00 | 20260505 09:34:39.014 | INFO | PerfReport: 1.09%, LockDownload -- 1.11e+00 / 0.00e+00, 2.70e+00 / 0.00e+00 | 20260505 09:34:39.014 | INFO | PerfReport: 20.65%, LockCompute -- 2.10e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260505 09:34:39.014 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.71e-02 / 0.00e+00 | 20260505 09:34:39.140 | INFO | Starting a round of Baz IO | 20260505 09:34:42.424 | INFO | Populating RTMetrics report | 20260505 09:34:42.425 | INFO | RTMetrics report populated | 20260505 09:34:42.425 | INFO | Opening tmp RTMetrics json file | 20260505 09:34:42.445 | INFO | Serializing RTMetrics to json | 20260505 09:34:42.448 | INFO | Serialization done, swapping tmp file now... | 20260505 09:34:42.448 | INFO | RTMetrics done | 20260505 09:34:42.577 | INFO | Finished a round of Baz IO: Wrote 2.435 GiB in 3.436 seconds | 20260505 09:34:42.577 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.93% / 0.01% / 74.06% | 20260505 09:34:42.577 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.90451% / 1.58946e-05% / 9.53674e-05% | 20260505 09:34:42.577 | INFO | Baz IO overall: | 20260505 09:34:42.577 | INFO | BytesWritten : 1256124649222 | 20260505 09:34:42.577 | INFO | NumEvents : 597815745829 | 20260505 09:34:44.151 | INFO | Analyzing chunk frames = [2170880,2171392) | 20260505 09:34:44.151 | INFO | MemInfo: { | 20260505 09:34:44.151 | INFO | "gpu" : " 17 GB", | 20260505 09:34:44.151 | INFO | "huge" : "", | 20260505 09:34:44.151 | INFO | "huge_shared" : " 27 GB", | 20260505 09:34:44.152 | INFO | "malloc" : " 14 GB", | 20260505 09:34:44.152 | INFO | "pinned" : " 29 GB" | 20260505 09:34:44.152 | INFO | } | 20260505 09:34:44.152 | INFO | Perf report for the last 5.13945 seconds: | 20260505 09:34:44.152 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:34:44.152 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:44.152 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:44.152 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:44.152 | INFO | ANALYSIS : 65.2%, 2.9, 99.6ms, 0 | 20260505 09:34:44.152 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:44.152 | INFO | RT_METRICS : 46.4%, 1.0, 24.4ms, 0 | 20260505 09:34:44.152 | INFO | PRE_HQ : 62.9%, 1.9, 60.8ms, 0 | 20260505 09:34:44.152 | INFO | BAZWRITER : 66.1%, 4.2, 141.4ms, 0 | 20260505 09:34:44.152 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 09:34:44.152 | INFO | Uploaded 36GiB to GPU in 2.22597s (16.1727GiB/s) | 20260505 09:34:44.152 | INFO | Downloaded 38GiB to GPU in 2.32596s (16.3373GiB/s) | 20260505 09:34:49.290 | INFO | Analyzing chunk frames = [2171392,2171904) | 20260505 09:34:49.290 | INFO | MemInfo: { | 20260505 09:34:49.290 | INFO | "gpu" : " 17 GB", | 20260505 09:34:49.290 | INFO | "huge" : "", | 20260505 09:34:49.290 | INFO | "huge_shared" : " 27 GB", | 20260505 09:34:49.290 | INFO | "malloc" : " 14 GB", | 20260505 09:34:49.290 | INFO | "pinned" : " 29 GB" | 20260505 09:34:49.291 | INFO | } | 20260505 09:34:49.291 | INFO | Perf report for the last 5.13889 seconds: | 20260505 09:34:49.291 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:34:49.291 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:49.291 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:49.291 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:49.291 | INFO | ANALYSIS : 64.5%, 3.0, 100.7ms, 0 | 20260505 09:34:49.291 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:49.291 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:49.291 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 09:34:49.291 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:49.291 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:49.291 | INFO | Uploaded 36GiB to GPU in 2.22276s (16.1961GiB/s) | 20260505 09:34:49.291 | INFO | Downloaded 34GiB to GPU in 2.10341s (16.1642GiB/s) | 20260505 09:34:54.430 | INFO | Analyzing chunk frames = [2171904,2172416) | 20260505 09:34:54.430 | INFO | MemInfo: { | 20260505 09:34:54.430 | INFO | "gpu" : " 17 GB", | 20260505 09:34:54.430 | INFO | "huge" : "", | 20260505 09:34:54.430 | INFO | "huge_shared" : " 27 GB", | 20260505 09:34:54.430 | INFO | "malloc" : " 14 GB", | 20260505 09:34:54.430 | INFO | "pinned" : " 29 GB" | 20260505 09:34:54.430 | INFO | } | 20260505 09:34:54.430 | INFO | Perf report for the last 5.13972 seconds: | 20260505 09:34:54.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:34:54.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:54.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:54.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:54.430 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 09:34:54.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:54.430 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:54.430 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 09:34:54.431 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:54.431 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:54.431 | INFO | Uploaded 36GiB to GPU in 2.15061s (16.7394GiB/s) | 20260505 09:34:54.431 | INFO | Downloaded 34GiB to GPU in 2.06878s (16.4348GiB/s) | 20260505 09:34:59.569 | INFO | Analyzing chunk frames = [2172416,2172928) | 20260505 09:34:59.569 | INFO | MemInfo: { | 20260505 09:34:59.569 | INFO | "gpu" : " 17 GB", | 20260505 09:34:59.569 | INFO | "huge" : "", | 20260505 09:34:59.569 | INFO | "huge_shared" : " 27 GB", | 20260505 09:34:59.569 | INFO | "malloc" : " 14 GB", | 20260505 09:34:59.569 | INFO | "pinned" : " 29 GB" | 20260505 09:34:59.571 | INFO | } | 20260505 09:34:59.571 | INFO | Perf report for the last 5.14038 seconds: | 20260505 09:34:59.571 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:34:59.571 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:59.571 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:59.571 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:59.571 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 09:34:59.571 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:59.571 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:59.571 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:34:59.571 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:59.571 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:34:59.571 | INFO | Uploaded 36GiB to GPU in 2.1656s (16.6236GiB/s) | 20260505 09:34:59.571 | INFO | Downloaded 34GiB to GPU in 2.07567s (16.3802GiB/s) | 20260505 09:35:04.709 | INFO | Analyzing chunk frames = [2172928,2173440) | 20260505 09:35:04.709 | INFO | MemInfo: { | 20260505 09:35:04.709 | INFO | "gpu" : " 17 GB", | 20260505 09:35:04.709 | INFO | "huge" : "", | 20260505 09:35:04.709 | INFO | "huge_shared" : " 27 GB", | 20260505 09:35:04.709 | INFO | "malloc" : " 14 GB", | 20260505 09:35:04.709 | INFO | "pinned" : " 29 GB" | 20260505 09:35:04.709 | INFO | } | 20260505 09:35:04.709 | INFO | Perf report for the last 5.13811 seconds: | 20260505 09:35:04.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:35:04.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:04.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:04.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:04.709 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 09:35:04.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:04.709 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:04.709 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 09:35:04.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:04.709 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:04.709 | INFO | Uploaded 36GiB to GPU in 2.2256s (16.1754GiB/s) | 20260505 09:35:04.709 | INFO | Downloaded 34GiB to GPU in 2.10326s (16.1654GiB/s) | 20260505 09:35:09.848 | INFO | Analyzing chunk frames = [2173440,2173952) | 20260505 09:35:09.848 | INFO | MemInfo: { | 20260505 09:35:09.848 | INFO | "gpu" : " 17 GB", | 20260505 09:35:09.848 | INFO | "huge" : "", | 20260505 09:35:09.848 | INFO | "huge_shared" : " 27 GB", | 20260505 09:35:09.848 | INFO | "malloc" : " 14 GB", | 20260505 09:35:09.848 | INFO | "pinned" : " 29 GB" | 20260505 09:35:09.848 | INFO | } | 20260505 09:35:09.849 | INFO | Perf report for the last 5.1397 seconds: | 20260505 09:35:09.849 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:35:09.849 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:09.849 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:09.849 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:09.849 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 09:35:09.849 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:09.849 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:09.849 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:35:09.849 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:09.849 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:09.849 | INFO | Uploaded 36GiB to GPU in 2.2367s (16.0952GiB/s) | 20260505 09:35:09.849 | INFO | Downloaded 34GiB to GPU in 2.10281s (16.1688GiB/s) | 20260505 09:35:14.987 | INFO | Analyzing chunk frames = [2173952,2174464) | 20260505 09:35:14.987 | INFO | MemInfo: { | 20260505 09:35:14.987 | INFO | "gpu" : " 17 GB", | 20260505 09:35:14.987 | INFO | "huge" : "", | 20260505 09:35:14.987 | INFO | "huge_shared" : " 27 GB", | 20260505 09:35:14.987 | INFO | "malloc" : " 14 GB", | 20260505 09:35:14.987 | INFO | "pinned" : " 29 GB" | 20260505 09:35:14.987 | INFO | } | 20260505 09:35:14.987 | INFO | Perf report for the last 5.13881 seconds: | 20260505 09:35:14.987 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:35:14.987 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:14.987 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:14.987 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:14.987 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 09:35:14.987 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:14.987 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:14.987 | INFO | PRE_HQ : 30.5%, 1.0, 15.9ms, 0 | 20260505 09:35:14.988 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:14.988 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:14.988 | INFO | Uploaded 36GiB to GPU in 2.1861s (16.4677GiB/s) | 20260505 09:35:14.988 | INFO | Downloaded 34GiB to GPU in 2.1416s (15.876GiB/s) | 20260505 09:35:20.127 | INFO | Analyzing chunk frames = [2174464,2174976) | 20260505 09:35:20.127 | INFO | MemInfo: { | 20260505 09:35:20.127 | INFO | "gpu" : " 17 GB", | 20260505 09:35:20.127 | INFO | "huge" : "", | 20260505 09:35:20.127 | INFO | "huge_shared" : " 27 GB", | 20260505 09:35:20.127 | INFO | "malloc" : " 14 GB", | 20260505 09:35:20.127 | INFO | "pinned" : " 29 GB" | 20260505 09:35:20.127 | INFO | } | 20260505 09:35:20.127 | INFO | Perf report for the last 5.13992 seconds: | 20260505 09:35:20.127 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:35:20.127 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:20.127 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:20.127 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:20.127 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 09:35:20.127 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:20.127 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:20.127 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:35:20.127 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:20.127 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:20.128 | INFO | Uploaded 36GiB to GPU in 2.18318s (16.4897GiB/s) | 20260505 09:35:20.128 | INFO | Downloaded 34GiB to GPU in 2.05495s (16.5454GiB/s) | 20260505 09:35:20.257 | INFO | Starting a round of Baz IO | 20260505 09:35:23.540 | INFO | Populating RTMetrics report | 20260505 09:35:23.540 | INFO | RTMetrics report populated | 20260505 09:35:23.540 | INFO | Opening tmp RTMetrics json file | 20260505 09:35:23.541 | INFO | Serializing RTMetrics to json | 20260505 09:35:23.542 | INFO | Serialization done, swapping tmp file now... | 20260505 09:35:23.543 | INFO | RTMetrics done | 20260505 09:35:23.706 | INFO | Finished a round of Baz IO: Wrote 2.436 GiB in 3.448 seconds | 20260505 09:35:23.706 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.92% / 0.01% / 74.07% | 20260505 09:35:23.706 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.90204% / 7.94729e-06% / 0.000111262% | 20260505 09:35:23.706 | INFO | Baz IO overall: | 20260505 09:35:23.706 | INFO | BytesWritten : 1258739906414 | 20260505 09:35:23.706 | INFO | NumEvents : 599064765618 | 20260505 09:35:25.266 | INFO | Analyzing chunk frames = [2174976,2175488) | 20260505 09:35:25.266 | INFO | MemInfo: { | 20260505 09:35:25.266 | INFO | "gpu" : " 17 GB", | 20260505 09:35:25.266 | INFO | "huge" : "", | 20260505 09:35:25.266 | INFO | "huge_shared" : " 27 GB", | 20260505 09:35:25.266 | INFO | "malloc" : " 14 GB", | 20260505 09:35:25.266 | INFO | "pinned" : " 29 GB" | 20260505 09:35:25.266 | INFO | } | 20260505 09:35:25.266 | INFO | Perf report for the last 5.13898 seconds: | 20260505 09:35:25.266 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:35:25.266 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:25.266 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:25.266 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:25.266 | INFO | ANALYSIS : 65.3%, 2.9, 100.3ms, 0 | 20260505 09:35:25.266 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:25.266 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260505 09:35:25.266 | INFO | PRE_HQ : 62.7%, 1.8, 60.0ms, 0 | 20260505 09:35:25.266 | INFO | BAZWRITER : 66.2%, 4.2, 141.3ms, 0 | 20260505 09:35:25.266 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 09:35:25.267 | INFO | Uploaded 36GiB to GPU in 2.21568s (16.2478GiB/s) | 20260505 09:35:25.267 | INFO | Downloaded 38GiB to GPU in 2.32661s (16.3328GiB/s) | 20260505 09:35:30.406 | INFO | Analyzing chunk frames = [2175488,2176000) | 20260505 09:35:30.406 | INFO | MemInfo: { | 20260505 09:35:30.406 | INFO | "gpu" : " 17 GB", | 20260505 09:35:30.406 | INFO | "huge" : "", | 20260505 09:35:30.406 | INFO | "huge_shared" : " 27 GB", | 20260505 09:35:30.406 | INFO | "malloc" : " 14 GB", | 20260505 09:35:30.406 | INFO | "pinned" : " 29 GB" | 20260505 09:35:30.406 | INFO | } | 20260505 09:35:30.406 | INFO | Perf report for the last 5.13951 seconds: | 20260505 09:35:30.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:35:30.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:30.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:30.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:30.406 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 09:35:30.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:30.406 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:30.406 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 09:35:30.406 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:30.406 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:30.406 | INFO | Uploaded 36GiB to GPU in 2.21579s (16.2471GiB/s) | 20260505 09:35:30.406 | INFO | Downloaded 34GiB to GPU in 2.1175s (16.0567GiB/s) | 20260505 09:35:35.545 | INFO | Analyzing chunk frames = [2176000,2176512) | 20260505 09:35:35.545 | INFO | MemInfo: { | 20260505 09:35:35.545 | INFO | "gpu" : " 17 GB", | 20260505 09:35:35.545 | INFO | "huge" : "", | 20260505 09:35:35.546 | INFO | "huge_shared" : " 27 GB", | 20260505 09:35:35.546 | INFO | "malloc" : " 14 GB", | 20260505 09:35:35.546 | INFO | "pinned" : " 29 GB" | 20260505 09:35:35.546 | INFO | } | 20260505 09:35:35.546 | INFO | Perf report for the last 5.13978 seconds: | 20260505 09:35:35.546 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:35:35.546 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:35.546 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:35.546 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:35.546 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 09:35:35.546 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:35.546 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:35.546 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 09:35:35.546 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:35.546 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:35.546 | INFO | Uploaded 36GiB to GPU in 2.14691s (16.7683GiB/s) | 20260505 09:35:35.546 | INFO | Downloaded 34GiB to GPU in 2.06577s (16.4587GiB/s) | 20260505 09:35:40.684 | INFO | Analyzing chunk frames = [2176512,2177024) | 20260505 09:35:40.684 | INFO | MemInfo: { | 20260505 09:35:40.684 | INFO | "gpu" : " 17 GB", | 20260505 09:35:40.684 | INFO | "huge" : "", | 20260505 09:35:40.684 | INFO | "huge_shared" : " 27 GB", | 20260505 09:35:40.684 | INFO | "malloc" : " 14 GB", | 20260505 09:35:40.684 | INFO | "pinned" : " 29 GB" | 20260505 09:35:40.685 | INFO | } | 20260505 09:35:40.685 | INFO | Perf report for the last 5.13893 seconds: | 20260505 09:35:40.685 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:35:40.685 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:40.685 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:40.685 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:40.685 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 09:35:40.685 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:40.685 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:40.685 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 09:35:40.685 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:40.685 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:40.685 | INFO | Uploaded 36GiB to GPU in 2.15792s (16.6827GiB/s) | 20260505 09:35:40.685 | INFO | Downloaded 34GiB to GPU in 2.07092s (16.4178GiB/s) | 20260505 09:35:45.824 | INFO | Analyzing chunk frames = [2177024,2177536) | 20260505 09:35:45.824 | INFO | MemInfo: { | 20260505 09:35:45.824 | INFO | "gpu" : " 17 GB", | 20260505 09:35:45.824 | INFO | "huge" : "", | 20260505 09:35:45.824 | INFO | "huge_shared" : " 27 GB", | 20260505 09:35:45.824 | INFO | "malloc" : " 14 GB", | 20260505 09:35:45.824 | INFO | "pinned" : " 29 GB" | 20260505 09:35:45.824 | INFO | } | 20260505 09:35:45.824 | INFO | Perf report for the last 5.13973 seconds: | 20260505 09:35:45.824 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:35:45.824 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:45.824 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:45.824 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:45.824 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 09:35:45.824 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:45.824 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:45.824 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:35:45.824 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:45.824 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:45.825 | INFO | Uploaded 36GiB to GPU in 2.2389s (16.0794GiB/s) | 20260505 09:35:45.825 | INFO | Downloaded 34GiB to GPU in 2.10954s (16.1172GiB/s) | 20260505 09:35:50.963 | INFO | Analyzing chunk frames = [2177536,2178048) | 20260505 09:35:50.964 | INFO | MemInfo: { | 20260505 09:35:50.964 | INFO | "gpu" : " 17 GB", | 20260505 09:35:50.964 | INFO | "huge" : "", | 20260505 09:35:50.964 | INFO | "huge_shared" : " 27 GB", | 20260505 09:35:50.964 | INFO | "malloc" : " 14 GB", | 20260505 09:35:50.964 | INFO | "pinned" : " 29 GB" | 20260505 09:35:50.964 | INFO | } | 20260505 09:35:50.964 | INFO | Perf report for the last 5.13948 seconds: | 20260505 09:35:50.964 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:35:50.964 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:50.964 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:50.964 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:50.964 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 09:35:50.964 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:50.964 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:50.964 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:35:50.964 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:50.964 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:50.964 | INFO | Uploaded 36GiB to GPU in 2.23735s (16.0905GiB/s) | 20260505 09:35:50.964 | INFO | Downloaded 34GiB to GPU in 2.10145s (16.1793GiB/s) | 20260505 09:35:56.102 | INFO | Analyzing chunk frames = [2178048,2178560) | 20260505 09:35:56.102 | INFO | MemInfo: { | 20260505 09:35:56.102 | INFO | "gpu" : " 17 GB", | 20260505 09:35:56.102 | INFO | "huge" : "", | 20260505 09:35:56.102 | INFO | "huge_shared" : " 27 GB", | 20260505 09:35:56.102 | INFO | "malloc" : " 14 GB", | 20260505 09:35:56.102 | INFO | "pinned" : " 29 GB" | 20260505 09:35:56.102 | INFO | } | 20260505 09:35:56.102 | INFO | Perf report for the last 5.1382 seconds: | 20260505 09:35:56.102 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:35:56.102 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:56.102 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:56.102 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:56.102 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 09:35:56.102 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:56.102 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:56.102 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 09:35:56.102 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:56.102 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:35:56.102 | INFO | Uploaded 36GiB to GPU in 2.19889s (16.3719GiB/s) | 20260505 09:35:56.102 | INFO | Downloaded 34GiB to GPU in 2.13191s (15.9482GiB/s) | 20260505 09:36:01.241 | INFO | Analyzing chunk frames = [2178560,2179072) | 20260505 09:36:01.241 | INFO | MemInfo: { | 20260505 09:36:01.241 | INFO | "gpu" : " 17 GB", | 20260505 09:36:01.241 | INFO | "huge" : "", | 20260505 09:36:01.241 | INFO | "huge_shared" : " 27 GB", | 20260505 09:36:01.241 | INFO | "malloc" : " 14 GB", | 20260505 09:36:01.241 | INFO | "pinned" : " 29 GB" | 20260505 09:36:01.241 | INFO | } | 20260505 09:36:01.241 | INFO | Perf report for the last 5.13938 seconds: | 20260505 09:36:01.241 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:36:01.241 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:01.241 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:01.241 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:01.241 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 09:36:01.241 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:01.242 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:01.242 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 09:36:01.242 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:01.242 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:01.242 | INFO | Uploaded 36GiB to GPU in 2.1863s (16.4662GiB/s) | 20260505 09:36:01.242 | INFO | Downloaded 34GiB to GPU in 2.05169s (16.5717GiB/s) | 20260505 09:36:01.242 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 09:36:01.242 | INFO | Total mem usage: 15.3 GiB | 20260505 09:36:01.242 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 09:36:01.242 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.58%) | 20260505 09:36:01.242 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 09:36:01.242 | INFO | High Water Breakdown: | 20260505 09:36:01.242 | INFO | 60.9% : 8.76 GiB ( 93.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 09:36:01.242 | INFO | 38.9% : 5.6 GiB ( 84.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 09:36:01.242 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 09:36:01.242 | INFO | ------------------------------------------ | 20260505 09:36:01.242 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:36:01.242 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 09:36:01.242 | INFO | Total mem usage: 26.5 GiB | 20260505 09:36:01.242 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 09:36:01.242 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 09:36:01.242 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 09:36:01.242 | INFO | High Water Breakdown: | 20260505 09:36:01.242 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:36:01.242 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:36:01.242 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:36:01.242 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:36:01.242 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:36:01.242 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:36:01.242 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 09:36:01.242 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 09:36:01.242 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:36:01.242 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 09:36:01.242 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 09:36:01.242 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 09:36:01.242 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 09:36:01.242 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 09:36:01.242 | INFO | ------------------------------------------ | 20260505 09:36:01.242 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:36:01.242 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 09:36:01.242 | INFO | Total mem usage: 25.6 GiB | 20260505 09:36:01.242 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 09:36:01.242 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 09:36:01.242 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 09:36:01.242 | INFO | High Water Breakdown: | 20260505 09:36:01.242 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 09:36:01.242 | INFO | ------------------------------------------ | 20260505 09:36:01.242 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:36:01.243 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 09:36:01.243 | INFO | Total mem usage: 15.7 GiB | 20260505 09:36:01.243 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 09:36:01.243 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 09:36:01.243 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 09:36:01.243 | INFO | High Water Breakdown: | 20260505 09:36:01.243 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 09:36:01.243 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 09:36:01.243 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 09:36:01.243 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 09:36:01.243 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 09:36:01.243 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 09:36:01.243 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 09:36:01.243 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 09:36:01.243 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 09:36:01.243 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:36:01.243 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 09:36:01.243 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:36:01.243 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 09:36:01.243 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 09:36:01.243 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 09:36:01.243 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 09:36:01.243 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:36:01.243 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 09:36:01.243 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:36:01.243 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:36:01.243 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:36:01.243 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 09:36:01.243 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 09:36:01.243 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:36:01.243 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 09:36:01.243 | INFO | ------------------------------------------ | 20260505 09:36:01.243 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:36:01.243 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 09:36:01.243 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:36:01.243 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:36:01.243 | INFO | PerfReport: Full Execution -- 3.25e+01 / 7.68e+01, 1.38e+01 / 2.21e+00 | 20260505 09:36:01.243 | INFO | PerfReport: 0.29%, AggregateStats -- 9.32e-02 / 1.04e-01, 1.04e-02 / 6.90e-03 | 20260505 09:36:01.243 | INFO | PerfReport: 11.64%, Baseline -- 3.78e+00 / 4.40e+00, 6.08e-01 / 3.25e-01 | 20260505 09:36:01.243 | INFO | PerfReport: 3.32%, Histogram -- 1.08e+00 / 1.48e+00, 2.08e-01 / 1.20e-01 | 20260505 09:36:01.243 | INFO | PerfReport: 15.58%, DME -- 4.05e+01 / 4.46e+01, 4.35e+00 / 1.86e+00 | 20260505 09:36:01.243 | INFO | PerfReport: 56.60%, FrameLabeling -- 1.84e+01 / 2.13e+01, 1.88e+00 / 1.98e+00 | 20260505 09:36:01.243 | INFO | PerfReport: 7.40%, PulseAccumulating -- 2.40e+00 / 2.94e+00, 4.00e-01 / 2.39e-01 | 20260505 09:36:01.243 | INFO | PerfReport: 3.82%, Metrics -- 1.24e+00 / 1.54e+00, 3.58e-01 / 1.97e-01 | 20260505 09:36:01.243 | INFO | PerfReport: 1.35%, PulseSerializing -- 4.39e-01 / 4.86e-01, 8.27e-02 / 3.08e-02 | 20260505 09:36:01.243 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 09:36:01.243 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:36:01.243 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:36:01.243 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.36e+01 / 0.00e+00 | 20260505 09:36:01.243 | INFO | PerfReport: 22.56%, Upload -- 2.29e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260505 09:36:01.243 | INFO | PerfReport: 21.76%, Download -- 2.21e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260505 09:36:01.243 | INFO | PerfReport: 31.97%, Compute -- 3.25e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 09:36:01.243 | INFO | PerfReport: 1.84%, LockUpload -- 1.87e+00 / 0.00e+00, 7.42e+00 / 0.00e+00 | 20260505 09:36:01.243 | INFO | PerfReport: 1.08%, LockDownload -- 1.10e+00 / 0.00e+00, 2.66e+00 / 0.00e+00 | 20260505 09:36:01.243 | INFO | PerfReport: 20.77%, LockCompute -- 2.11e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260505 09:36:01.243 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.87e-02 / 0.00e+00 | 20260505 09:36:01.370 | INFO | Starting a round of Baz IO | 20260505 09:36:04.646 | INFO | Populating RTMetrics report | 20260505 09:36:04.646 | INFO | RTMetrics report populated | 20260505 09:36:04.646 | INFO | Opening tmp RTMetrics json file | 20260505 09:36:04.673 | INFO | Serializing RTMetrics to json | 20260505 09:36:04.722 | INFO | Serialization done, swapping tmp file now... | 20260505 09:36:04.748 | INFO | RTMetrics done | 20260505 09:36:04.781 | INFO | Finished a round of Baz IO: Wrote 2.437 GiB in 3.41 seconds | 20260505 09:36:04.781 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.90% / 0.01% / 74.08% | 20260505 09:36:04.781 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.90328% / 1.58946e-05% / 8.74201e-05% | 20260505 09:36:04.781 | INFO | Baz IO overall: | 20260505 09:36:04.781 | INFO | BytesWritten : 1261356797315 | 20260505 09:36:04.781 | INFO | NumEvents : 600315200749 | 20260505 09:36:06.381 | INFO | Analyzing chunk frames = [2179072,2179584) | 20260505 09:36:06.381 | INFO | MemInfo: { | 20260505 09:36:06.381 | INFO | "gpu" : " 17 GB", | 20260505 09:36:06.381 | INFO | "huge" : "", | 20260505 09:36:06.381 | INFO | "huge_shared" : " 27 GB", | 20260505 09:36:06.381 | INFO | "malloc" : " 14 GB", | 20260505 09:36:06.381 | INFO | "pinned" : " 29 GB" | 20260505 09:36:06.381 | INFO | } | 20260505 09:36:06.381 | INFO | Perf report for the last 5.13964 seconds: | 20260505 09:36:06.381 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:36:06.381 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:06.381 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:06.381 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:06.381 | INFO | ANALYSIS : 65.1%, 2.9, 99.7ms, 0 | 20260505 09:36:06.381 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:06.381 | INFO | RT_METRICS : 47.6%, 1.0, 25.0ms, 0 | 20260505 09:36:06.381 | INFO | PRE_HQ : 62.7%, 1.8, 60.1ms, 0 | 20260505 09:36:06.381 | INFO | BAZWRITER : 65.7%, 4.1, 139.9ms, 0 | 20260505 09:36:06.381 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 09:36:06.381 | INFO | Uploaded 36GiB to GPU in 2.22317s (16.1931GiB/s) | 20260505 09:36:06.381 | INFO | Downloaded 38GiB to GPU in 2.33512s (16.2733GiB/s) | 20260505 09:36:11.520 | INFO | Analyzing chunk frames = [2179584,2180096) | 20260505 09:36:11.520 | INFO | MemInfo: { | 20260505 09:36:11.520 | INFO | "gpu" : " 17 GB", | 20260505 09:36:11.520 | INFO | "huge" : "", | 20260505 09:36:11.520 | INFO | "huge_shared" : " 27 GB", | 20260505 09:36:11.520 | INFO | "malloc" : " 14 GB", | 20260505 09:36:11.520 | INFO | "pinned" : " 29 GB" | 20260505 09:36:11.520 | INFO | } | 20260505 09:36:11.521 | INFO | Perf report for the last 5.13942 seconds: | 20260505 09:36:11.521 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:36:11.521 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:11.521 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:11.521 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:11.521 | INFO | ANALYSIS : 64.5%, 3.0, 100.6ms, 0 | 20260505 09:36:11.521 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:11.521 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:11.521 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:36:11.521 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:11.521 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:11.521 | INFO | Uploaded 36GiB to GPU in 2.21138s (16.2794GiB/s) | 20260505 09:36:11.521 | INFO | Downloaded 34GiB to GPU in 2.10535s (16.1494GiB/s) | 20260505 09:36:16.660 | INFO | Analyzing chunk frames = [2180096,2180608) | 20260505 09:36:16.660 | INFO | MemInfo: { | 20260505 09:36:16.660 | INFO | "gpu" : " 17 GB", | 20260505 09:36:16.660 | INFO | "huge" : "", | 20260505 09:36:16.660 | INFO | "huge_shared" : " 27 GB", | 20260505 09:36:16.660 | INFO | "malloc" : " 14 GB", | 20260505 09:36:16.660 | INFO | "pinned" : " 29 GB" | 20260505 09:36:16.660 | INFO | } | 20260505 09:36:16.660 | INFO | Perf report for the last 5.13994 seconds: | 20260505 09:36:16.660 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:36:16.660 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:16.660 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:16.660 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:16.661 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 09:36:16.661 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:16.661 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:16.661 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 09:36:16.661 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:16.661 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:16.661 | INFO | Uploaded 36GiB to GPU in 2.15866s (16.677GiB/s) | 20260505 09:36:16.661 | INFO | Downloaded 34GiB to GPU in 2.07076s (16.4191GiB/s) | 20260505 09:36:21.799 | INFO | Analyzing chunk frames = [2180608,2181120) | 20260505 09:36:21.799 | INFO | MemInfo: { | 20260505 09:36:21.799 | INFO | "gpu" : " 17 GB", | 20260505 09:36:21.799 | INFO | "huge" : "", | 20260505 09:36:21.799 | INFO | "huge_shared" : " 27 GB", | 20260505 09:36:21.799 | INFO | "malloc" : " 14 GB", | 20260505 09:36:21.799 | INFO | "pinned" : " 29 GB" | 20260505 09:36:21.799 | INFO | } | 20260505 09:36:21.799 | INFO | Perf report for the last 5.13846 seconds: | 20260505 09:36:21.799 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:36:21.799 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:21.799 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:21.799 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:21.799 | INFO | ANALYSIS : 63.1%, 3.0, 98.1ms, 0 | 20260505 09:36:21.799 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:21.799 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:21.799 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 09:36:21.799 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:21.799 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:21.799 | INFO | Uploaded 36GiB to GPU in 2.14315s (16.7977GiB/s) | 20260505 09:36:21.799 | INFO | Downloaded 34GiB to GPU in 2.06858s (16.4364GiB/s) | 20260505 09:36:26.938 | INFO | Analyzing chunk frames = [2181120,2181632) | 20260505 09:36:26.938 | INFO | MemInfo: { | 20260505 09:36:26.938 | INFO | "gpu" : " 17 GB", | 20260505 09:36:26.938 | INFO | "huge" : "", | 20260505 09:36:26.938 | INFO | "huge_shared" : " 27 GB", | 20260505 09:36:26.938 | INFO | "malloc" : " 14 GB", | 20260505 09:36:26.938 | INFO | "pinned" : " 29 GB" | 20260505 09:36:26.939 | INFO | } | 20260505 09:36:26.939 | INFO | Perf report for the last 5.13962 seconds: | 20260505 09:36:26.939 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:36:26.939 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:26.939 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:26.939 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:26.939 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 09:36:26.939 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:26.939 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:26.939 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:36:26.939 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:26.939 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:26.939 | INFO | Uploaded 36GiB to GPU in 2.23404s (16.1143GiB/s) | 20260505 09:36:26.939 | INFO | Downloaded 34GiB to GPU in 2.10378s (16.1614GiB/s) | 20260505 09:36:32.079 | INFO | Analyzing chunk frames = [2181632,2182144) | 20260505 09:36:32.079 | INFO | MemInfo: { | 20260505 09:36:32.079 | INFO | "gpu" : " 17 GB", | 20260505 09:36:32.079 | INFO | "huge" : "", | 20260505 09:36:32.079 | INFO | "huge_shared" : " 27 GB", | 20260505 09:36:32.079 | INFO | "malloc" : " 14 GB", | 20260505 09:36:32.079 | INFO | "pinned" : " 29 GB" | 20260505 09:36:32.079 | INFO | } | 20260505 09:36:32.079 | INFO | Perf report for the last 5.14022 seconds: | 20260505 09:36:32.079 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:36:32.079 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:32.079 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:32.079 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:32.079 | INFO | ANALYSIS : 64.1%, 3.0, 100.2ms, 0 | 20260505 09:36:32.079 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:32.079 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:32.079 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:36:32.079 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:32.079 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:32.079 | INFO | Uploaded 36GiB to GPU in 2.24063s (16.0669GiB/s) | 20260505 09:36:32.079 | INFO | Downloaded 34GiB to GPU in 2.10261s (16.1704GiB/s) | 20260505 09:36:37.217 | INFO | Analyzing chunk frames = [2182144,2182656) | 20260505 09:36:37.217 | INFO | MemInfo: { | 20260505 09:36:37.217 | INFO | "gpu" : " 17 GB", | 20260505 09:36:37.217 | INFO | "huge" : "", | 20260505 09:36:37.217 | INFO | "huge_shared" : " 27 GB", | 20260505 09:36:37.217 | INFO | "malloc" : " 14 GB", | 20260505 09:36:37.217 | INFO | "pinned" : " 29 GB" | 20260505 09:36:37.217 | INFO | } | 20260505 09:36:37.217 | INFO | Perf report for the last 5.13803 seconds: | 20260505 09:36:37.217 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:36:37.217 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:37.217 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:37.217 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:37.217 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 09:36:37.217 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:37.217 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:37.217 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 09:36:37.217 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:37.217 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:37.217 | INFO | Uploaded 36GiB to GPU in 2.20205s (16.3484GiB/s) | 20260505 09:36:37.217 | INFO | Downloaded 34GiB to GPU in 2.13151s (15.9511GiB/s) | 20260505 09:36:42.356 | INFO | Analyzing chunk frames = [2182656,2183168) | 20260505 09:36:42.356 | INFO | MemInfo: { | 20260505 09:36:42.356 | INFO | "gpu" : " 17 GB", | 20260505 09:36:42.356 | INFO | "huge" : "", | 20260505 09:36:42.356 | INFO | "huge_shared" : " 27 GB", | 20260505 09:36:42.356 | INFO | "malloc" : " 14 GB", | 20260505 09:36:42.356 | INFO | "pinned" : " 29 GB" | 20260505 09:36:42.357 | INFO | } | 20260505 09:36:42.357 | INFO | Perf report for the last 5.1397 seconds: | 20260505 09:36:42.357 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:36:42.357 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:42.357 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:42.357 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:42.357 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 09:36:42.357 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:42.357 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:42.357 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:36:42.357 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:42.357 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:42.357 | INFO | Uploaded 36GiB to GPU in 2.18178s (16.5003GiB/s) | 20260505 09:36:42.357 | INFO | Downloaded 34GiB to GPU in 2.05659s (16.5322GiB/s) | 20260505 09:36:42.488 | INFO | Starting a round of Baz IO | 20260505 09:36:45.748 | INFO | Populating RTMetrics report | 20260505 09:36:45.748 | INFO | RTMetrics report populated | 20260505 09:36:45.748 | INFO | Opening tmp RTMetrics json file | 20260505 09:36:45.761 | INFO | Serializing RTMetrics to json | 20260505 09:36:45.812 | INFO | Serialization done, swapping tmp file now... | 20260505 09:36:45.820 | INFO | RTMetrics done | 20260505 09:36:45.878 | INFO | Finished a round of Baz IO: Wrote 2.437 GiB in 3.389 seconds | 20260505 09:36:45.878 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.89% / 0.01% / 74.10% | 20260505 09:36:45.878 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.89521% / 2.38419e-05% / 0.000103315% | 20260505 09:36:45.878 | INFO | Baz IO overall: | 20260505 09:36:45.878 | INFO | BytesWritten : 1263973034758 | 20260505 09:36:45.878 | INFO | NumEvents : 601565018073 | 20260505 09:36:47.496 | INFO | Analyzing chunk frames = [2183168,2183680) | 20260505 09:36:47.496 | INFO | MemInfo: { | 20260505 09:36:47.496 | INFO | "gpu" : " 17 GB", | 20260505 09:36:47.496 | INFO | "huge" : "", | 20260505 09:36:47.496 | INFO | "huge_shared" : " 27 GB", | 20260505 09:36:47.496 | INFO | "malloc" : " 14 GB", | 20260505 09:36:47.496 | INFO | "pinned" : " 29 GB" | 20260505 09:36:47.496 | INFO | } | 20260505 09:36:47.496 | INFO | Perf report for the last 5.13915 seconds: | 20260505 09:36:47.496 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:36:47.496 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:47.496 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:47.496 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:47.496 | INFO | ANALYSIS : 64.9%, 2.9, 99.2ms, 0 | 20260505 09:36:47.496 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:47.496 | INFO | RT_METRICS : 47.1%, 1.0, 24.7ms, 0 | 20260505 09:36:47.496 | INFO | PRE_HQ : 62.9%, 1.8, 60.2ms, 0 | 20260505 09:36:47.496 | INFO | BAZWRITER : 65.2%, 4.2, 140.7ms, 0 | 20260505 09:36:47.496 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 09:36:47.496 | INFO | Uploaded 36GiB to GPU in 2.28725s (15.7394GiB/s) | 20260505 09:36:47.496 | INFO | Downloaded 38GiB to GPU in 2.38362s (15.9421GiB/s) | 20260505 09:36:52.635 | INFO | Analyzing chunk frames = [2183680,2184192) | 20260505 09:36:52.635 | INFO | MemInfo: { | 20260505 09:36:52.635 | INFO | "gpu" : " 17 GB", | 20260505 09:36:52.635 | INFO | "huge" : "", | 20260505 09:36:52.635 | INFO | "huge_shared" : " 27 GB", | 20260505 09:36:52.635 | INFO | "malloc" : " 14 GB", | 20260505 09:36:52.636 | INFO | "pinned" : " 29 GB" | 20260505 09:36:52.636 | INFO | } | 20260505 09:36:52.636 | INFO | Perf report for the last 5.13992 seconds: | 20260505 09:36:52.636 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:36:52.636 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:52.636 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:52.636 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:52.636 | INFO | ANALYSIS : 64.4%, 3.0, 100.6ms, 0 | 20260505 09:36:52.636 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:52.636 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:52.636 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:36:52.636 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:52.636 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:52.636 | INFO | Uploaded 36GiB to GPU in 2.21571s (16.2476GiB/s) | 20260505 09:36:52.636 | INFO | Downloaded 34GiB to GPU in 2.1032s (16.1658GiB/s) | 20260505 09:36:57.775 | INFO | Analyzing chunk frames = [2184192,2184704) | 20260505 09:36:57.775 | INFO | MemInfo: { | 20260505 09:36:57.775 | INFO | "gpu" : " 17 GB", | 20260505 09:36:57.775 | INFO | "huge" : "", | 20260505 09:36:57.775 | INFO | "huge_shared" : " 27 GB", | 20260505 09:36:57.775 | INFO | "malloc" : " 14 GB", | 20260505 09:36:57.775 | INFO | "pinned" : " 29 GB" | 20260505 09:36:57.775 | INFO | } | 20260505 09:36:57.775 | INFO | Perf report for the last 5.1391 seconds: | 20260505 09:36:57.775 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:36:57.775 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:57.775 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:57.775 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:57.775 | INFO | ANALYSIS : 63.2%, 3.0, 98.4ms, 0 | 20260505 09:36:57.775 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:57.775 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:57.775 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:36:57.775 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:57.775 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:36:57.775 | INFO | Uploaded 36GiB to GPU in 2.15299s (16.7209GiB/s) | 20260505 09:36:57.775 | INFO | Downloaded 34GiB to GPU in 2.06996s (16.4254GiB/s) | 20260505 09:37:02.913 | INFO | Analyzing chunk frames = [2184704,2185216) | 20260505 09:37:02.913 | INFO | MemInfo: { | 20260505 09:37:02.913 | INFO | "gpu" : " 17 GB", | 20260505 09:37:02.913 | INFO | "huge" : "", | 20260505 09:37:02.913 | INFO | "huge_shared" : " 27 GB", | 20260505 09:37:02.913 | INFO | "malloc" : " 14 GB", | 20260505 09:37:02.913 | INFO | "pinned" : " 29 GB" | 20260505 09:37:02.913 | INFO | } | 20260505 09:37:02.914 | INFO | Perf report for the last 5.13873 seconds: | 20260505 09:37:02.914 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:37:02.914 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:02.914 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:02.914 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:02.914 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 09:37:02.914 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:02.914 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:02.914 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:37:02.914 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:02.914 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:02.914 | INFO | Uploaded 36GiB to GPU in 2.15454s (16.7089GiB/s) | 20260505 09:37:02.914 | INFO | Downloaded 34GiB to GPU in 2.0714s (16.414GiB/s) | 20260505 09:37:08.053 | INFO | Analyzing chunk frames = [2185216,2185728) | 20260505 09:37:08.053 | INFO | MemInfo: { | 20260505 09:37:08.053 | INFO | "gpu" : " 17 GB", | 20260505 09:37:08.053 | INFO | "huge" : "", | 20260505 09:37:08.053 | INFO | "huge_shared" : " 27 GB", | 20260505 09:37:08.053 | INFO | "malloc" : " 14 GB", | 20260505 09:37:08.053 | INFO | "pinned" : " 29 GB" | 20260505 09:37:08.053 | INFO | } | 20260505 09:37:08.053 | INFO | Perf report for the last 5.13967 seconds: | 20260505 09:37:08.053 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:37:08.053 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:08.053 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:08.053 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:08.053 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 09:37:08.053 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:08.053 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:08.053 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 09:37:08.053 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:08.053 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:08.053 | INFO | Uploaded 36GiB to GPU in 2.23382s (16.1159GiB/s) | 20260505 09:37:08.053 | INFO | Downloaded 34GiB to GPU in 2.11047s (16.1102GiB/s) | 20260505 09:37:13.194 | INFO | Analyzing chunk frames = [2185728,2186240) | 20260505 09:37:13.194 | INFO | MemInfo: { | 20260505 09:37:13.194 | INFO | "gpu" : " 17 GB", | 20260505 09:37:13.194 | INFO | "huge" : "", | 20260505 09:37:13.194 | INFO | "huge_shared" : " 27 GB", | 20260505 09:37:13.194 | INFO | "malloc" : " 14 GB", | 20260505 09:37:13.194 | INFO | "pinned" : " 29 GB" | 20260505 09:37:13.194 | INFO | } | 20260505 09:37:13.194 | INFO | Perf report for the last 5.14067 seconds: | 20260505 09:37:13.194 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:37:13.194 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:13.194 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:13.194 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:13.194 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 09:37:13.194 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:13.194 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:13.194 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 09:37:13.194 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:13.194 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:13.194 | INFO | Uploaded 36GiB to GPU in 2.19028s (16.4362GiB/s) | 20260505 09:37:13.194 | INFO | Downloaded 34GiB to GPU in 2.09778s (16.2076GiB/s) | 20260505 09:37:18.332 | INFO | Analyzing chunk frames = [2186240,2186752) | 20260505 09:37:18.332 | INFO | MemInfo: { | 20260505 09:37:18.332 | INFO | "gpu" : " 17 GB", | 20260505 09:37:18.332 | INFO | "huge" : "", | 20260505 09:37:18.332 | INFO | "huge_shared" : " 27 GB", | 20260505 09:37:18.332 | INFO | "malloc" : " 14 GB", | 20260505 09:37:18.332 | INFO | "pinned" : " 29 GB" | 20260505 09:37:18.332 | INFO | } | 20260505 09:37:18.332 | INFO | Perf report for the last 5.1386 seconds: | 20260505 09:37:18.332 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:37:18.333 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:18.333 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:18.333 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:18.333 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 09:37:18.333 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:18.333 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:18.333 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 09:37:18.333 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:18.333 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:18.333 | INFO | Uploaded 36GiB to GPU in 2.15173s (16.7308GiB/s) | 20260505 09:37:18.333 | INFO | Downloaded 34GiB to GPU in 2.14303s (15.8654GiB/s) | 20260505 09:37:23.471 | INFO | Analyzing chunk frames = [2186752,2187264) | 20260505 09:37:23.471 | INFO | MemInfo: { | 20260505 09:37:23.471 | INFO | "gpu" : " 17 GB", | 20260505 09:37:23.471 | INFO | "huge" : "", | 20260505 09:37:23.471 | INFO | "huge_shared" : " 27 GB", | 20260505 09:37:23.471 | INFO | "malloc" : " 14 GB", | 20260505 09:37:23.471 | INFO | "pinned" : " 29 GB" | 20260505 09:37:23.471 | INFO | } | 20260505 09:37:23.471 | INFO | Perf report for the last 5.13884 seconds: | 20260505 09:37:23.472 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:37:23.472 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:23.472 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:23.472 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:23.472 | INFO | ANALYSIS : 64.7%, 3.0, 101.0ms, 0 | 20260505 09:37:23.472 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:23.472 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:23.472 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 09:37:23.472 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:23.472 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:23.472 | INFO | Uploaded 36GiB to GPU in 2.24495s (16.036GiB/s) | 20260505 09:37:23.472 | INFO | Downloaded 34GiB to GPU in 2.08803s (16.2833GiB/s) | 20260505 09:37:23.472 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 09:37:23.472 | INFO | Total mem usage: 15.3 GiB | 20260505 09:37:23.472 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 09:37:23.472 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.45%) | 20260505 09:37:23.472 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 09:37:23.472 | INFO | High Water Breakdown: | 20260505 09:37:23.472 | INFO | 61% : 8.78 GiB ( 93.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 09:37:23.472 | INFO | 38.9% : 5.6 GiB ( 84.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 09:37:23.472 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 09:37:23.472 | INFO | ------------------------------------------ | 20260505 09:37:23.472 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:37:23.472 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 09:37:23.472 | INFO | Total mem usage: 26.5 GiB | 20260505 09:37:23.472 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 09:37:23.472 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 09:37:23.472 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 09:37:23.472 | INFO | High Water Breakdown: | 20260505 09:37:23.472 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:37:23.472 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:37:23.472 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:37:23.472 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:37:23.472 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:37:23.472 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:37:23.472 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 09:37:23.472 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 09:37:23.472 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:37:23.472 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 09:37:23.472 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 09:37:23.473 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 09:37:23.473 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 09:37:23.473 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 09:37:23.473 | INFO | ------------------------------------------ | 20260505 09:37:23.473 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:37:23.473 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 09:37:23.473 | INFO | Total mem usage: 25.6 GiB | 20260505 09:37:23.473 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 09:37:23.473 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 09:37:23.473 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 09:37:23.473 | INFO | High Water Breakdown: | 20260505 09:37:23.473 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 09:37:23.473 | INFO | ------------------------------------------ | 20260505 09:37:23.473 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:37:23.473 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 09:37:23.473 | INFO | Total mem usage: 15.7 GiB | 20260505 09:37:23.473 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 09:37:23.473 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 09:37:23.473 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 09:37:23.473 | INFO | High Water Breakdown: | 20260505 09:37:23.473 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 09:37:23.473 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 09:37:23.473 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 09:37:23.473 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 09:37:23.473 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 09:37:23.473 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 09:37:23.473 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 09:37:23.473 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 09:37:23.473 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 09:37:23.473 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:37:23.473 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 09:37:23.473 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:37:23.473 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 09:37:23.473 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 09:37:23.473 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 09:37:23.473 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 09:37:23.473 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:37:23.473 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 09:37:23.473 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:37:23.473 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:37:23.473 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:37:23.473 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 09:37:23.473 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 09:37:23.473 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:37:23.473 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 09:37:23.473 | INFO | ------------------------------------------ | 20260505 09:37:23.473 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:37:23.473 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 09:37:23.473 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:37:23.473 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:37:23.473 | INFO | PerfReport: Full Execution -- 3.25e+01 / 7.70e+01, 1.39e+01 / 2.41e+00 | 20260505 09:37:23.473 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 1.02e-01, 1.02e-02 / 4.41e-03 | 20260505 09:37:23.473 | INFO | PerfReport: 11.63%, Baseline -- 3.78e+00 / 4.41e+00, 6.14e-01 / 2.93e-01 | 20260505 09:37:23.473 | INFO | PerfReport: 3.31%, Histogram -- 1.07e+00 / 1.48e+00, 2.10e-01 / 1.16e-01 | 20260505 09:37:23.473 | INFO | PerfReport: 15.66%, DME -- 4.07e+01 / 4.49e+01, 4.46e+00 / 2.24e+00 | 20260505 09:37:23.473 | INFO | PerfReport: 56.57%, FrameLabeling -- 1.84e+01 / 2.12e+01, 1.87e+00 / 1.85e+00 | 20260505 09:37:23.473 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.39e+00 / 2.93e+00, 4.06e-01 / 2.41e-01 | 20260505 09:37:23.473 | INFO | PerfReport: 3.81%, Metrics -- 1.24e+00 / 1.53e+00, 3.57e-01 / 1.65e-01 | 20260505 09:37:23.473 | INFO | PerfReport: 1.36%, PulseSerializing -- 4.40e-01 / 4.83e-01, 8.30e-02 / 2.97e-02 | 20260505 09:37:23.473 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 09:37:23.473 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:37:23.473 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:37:23.473 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.39e+01 / 0.00e+00 | 20260505 09:37:23.473 | INFO | PerfReport: 22.58%, Upload -- 2.29e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260505 09:37:23.473 | INFO | PerfReport: 21.82%, Download -- 2.22e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260505 09:37:23.473 | INFO | PerfReport: 31.97%, Compute -- 3.25e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 09:37:23.473 | INFO | PerfReport: 1.85%, LockUpload -- 1.88e+00 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260505 09:37:23.473 | INFO | PerfReport: 1.15%, LockDownload -- 1.17e+00 / 0.00e+00, 2.76e+00 / 0.00e+00 | 20260505 09:37:23.473 | INFO | PerfReport: 20.62%, LockCompute -- 2.09e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260505 09:37:23.473 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.98e-02 / 0.00e+00 | 20260505 09:37:23.603 | INFO | Starting a round of Baz IO | 20260505 09:37:26.879 | INFO | Populating RTMetrics report | 20260505 09:37:26.880 | INFO | RTMetrics report populated | 20260505 09:37:26.880 | INFO | Opening tmp RTMetrics json file | 20260505 09:37:26.892 | INFO | Serializing RTMetrics to json | 20260505 09:37:26.894 | INFO | Serialization done, swapping tmp file now... | 20260505 09:37:26.894 | INFO | RTMetrics done | 20260505 09:37:27.033 | INFO | Finished a round of Baz IO: Wrote 2.439 GiB in 3.429 seconds | 20260505 09:37:27.033 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.87% / 0.01% / 74.11% | 20260505 09:37:27.033 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.89385% / 1.58946e-05% / 8.34465e-05% | 20260505 09:37:27.033 | INFO | Baz IO overall: | 20260505 09:37:27.033 | INFO | BytesWritten : 1266591471850 | 20260505 09:37:27.033 | INFO | NumEvents : 602816499570 | 20260505 09:37:28.611 | INFO | Analyzing chunk frames = [2187264,2187776) | 20260505 09:37:28.611 | INFO | MemInfo: { | 20260505 09:37:28.611 | INFO | "gpu" : " 17 GB", | 20260505 09:37:28.611 | INFO | "huge" : "", | 20260505 09:37:28.611 | INFO | "huge_shared" : " 27 GB", | 20260505 09:37:28.611 | INFO | "malloc" : " 14 GB", | 20260505 09:37:28.611 | INFO | "pinned" : " 29 GB" | 20260505 09:37:28.611 | INFO | } | 20260505 09:37:28.611 | INFO | Perf report for the last 5.13985 seconds: | 20260505 09:37:28.611 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:37:28.611 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:28.611 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:28.611 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:28.611 | INFO | ANALYSIS : 65.2%, 2.9, 99.9ms, 0 | 20260505 09:37:28.611 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:28.612 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260505 09:37:28.612 | INFO | PRE_HQ : 62.9%, 1.8, 59.8ms, 0 | 20260505 09:37:28.612 | INFO | BAZWRITER : 66.1%, 4.1, 140.8ms, 0 | 20260505 09:37:28.612 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 09:37:28.612 | INFO | Uploaded 36GiB to GPU in 2.21317s (16.2662GiB/s) | 20260505 09:37:28.612 | INFO | Downloaded 38GiB to GPU in 2.34599s (16.1979GiB/s) | 20260505 09:37:33.750 | INFO | Analyzing chunk frames = [2187776,2188288) | 20260505 09:37:33.750 | INFO | MemInfo: { | 20260505 09:37:33.750 | INFO | "gpu" : " 17 GB", | 20260505 09:37:33.750 | INFO | "huge" : "", | 20260505 09:37:33.750 | INFO | "huge_shared" : " 27 GB", | 20260505 09:37:33.750 | INFO | "malloc" : " 14 GB", | 20260505 09:37:33.750 | INFO | "pinned" : " 29 GB" | 20260505 09:37:33.750 | INFO | } | 20260505 09:37:33.750 | INFO | Perf report for the last 5.13897 seconds: | 20260505 09:37:33.750 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:37:33.750 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:33.750 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:33.750 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:33.750 | INFO | ANALYSIS : 64.5%, 3.0, 100.7ms, 0 | 20260505 09:37:33.750 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:33.750 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:33.750 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 09:37:33.750 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:33.750 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:33.750 | INFO | Uploaded 36GiB to GPU in 2.2239s (16.1878GiB/s) | 20260505 09:37:33.751 | INFO | Downloaded 34GiB to GPU in 2.10445s (16.1562GiB/s) | 20260505 09:37:38.889 | INFO | Analyzing chunk frames = [2188288,2188800) | 20260505 09:37:38.889 | INFO | MemInfo: { | 20260505 09:37:38.889 | INFO | "gpu" : " 17 GB", | 20260505 09:37:38.889 | INFO | "huge" : "", | 20260505 09:37:38.889 | INFO | "huge_shared" : " 27 GB", | 20260505 09:37:38.889 | INFO | "malloc" : " 14 GB", | 20260505 09:37:38.889 | INFO | "pinned" : " 29 GB" | 20260505 09:37:38.889 | INFO | } | 20260505 09:37:38.889 | INFO | Perf report for the last 5.13875 seconds: | 20260505 09:37:38.889 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:37:38.889 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:38.889 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:38.889 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:38.889 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 09:37:38.889 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:38.889 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:38.889 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 09:37:38.889 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:38.889 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:38.889 | INFO | Uploaded 36GiB to GPU in 2.15521s (16.7037GiB/s) | 20260505 09:37:38.889 | INFO | Downloaded 34GiB to GPU in 2.06853s (16.4368GiB/s) | 20260505 09:37:44.031 | INFO | Analyzing chunk frames = [2188800,2189312) | 20260505 09:37:44.031 | INFO | MemInfo: { | 20260505 09:37:44.031 | INFO | "gpu" : " 17 GB", | 20260505 09:37:44.031 | INFO | "huge" : "", | 20260505 09:37:44.031 | INFO | "huge_shared" : " 27 GB", | 20260505 09:37:44.031 | INFO | "malloc" : " 14 GB", | 20260505 09:37:44.031 | INFO | "pinned" : " 29 GB" | 20260505 09:37:44.031 | INFO | } | 20260505 09:37:44.031 | INFO | Perf report for the last 5.14222 seconds: | 20260505 09:37:44.031 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:37:44.031 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:44.031 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:44.031 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:44.031 | INFO | ANALYSIS : 63.2%, 3.0, 98.3ms, 0 | 20260505 09:37:44.031 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:44.031 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:44.031 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 09:37:44.031 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:44.031 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:44.031 | INFO | Uploaded 36GiB to GPU in 2.16694s (16.6133GiB/s) | 20260505 09:37:44.032 | INFO | Downloaded 34GiB to GPU in 2.07933s (16.3514GiB/s) | 20260505 09:37:49.169 | INFO | Analyzing chunk frames = [2189312,2189824) | 20260505 09:37:49.169 | INFO | MemInfo: { | 20260505 09:37:49.169 | INFO | "gpu" : " 17 GB", | 20260505 09:37:49.169 | INFO | "huge" : "", | 20260505 09:37:49.169 | INFO | "huge_shared" : " 27 GB", | 20260505 09:37:49.169 | INFO | "malloc" : " 14 GB", | 20260505 09:37:49.169 | INFO | "pinned" : " 29 GB" | 20260505 09:37:49.169 | INFO | } | 20260505 09:37:49.169 | INFO | Perf report for the last 5.13768 seconds: | 20260505 09:37:49.169 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:37:49.169 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:49.169 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:49.169 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:49.169 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 09:37:49.169 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:49.169 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:49.169 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 09:37:49.169 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:49.169 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:49.169 | INFO | Uploaded 36GiB to GPU in 2.2387s (16.0807GiB/s) | 20260505 09:37:49.169 | INFO | Downloaded 34GiB to GPU in 2.10767s (16.1315GiB/s) | 20260505 09:37:54.308 | INFO | Analyzing chunk frames = [2189824,2190336) | 20260505 09:37:54.308 | INFO | MemInfo: { | 20260505 09:37:54.308 | INFO | "gpu" : " 17 GB", | 20260505 09:37:54.308 | INFO | "huge" : "", | 20260505 09:37:54.308 | INFO | "huge_shared" : " 27 GB", | 20260505 09:37:54.308 | INFO | "malloc" : " 14 GB", | 20260505 09:37:54.308 | INFO | "pinned" : " 29 GB" | 20260505 09:37:54.308 | INFO | } | 20260505 09:37:54.308 | INFO | Perf report for the last 5.13929 seconds: | 20260505 09:37:54.308 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:37:54.308 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:54.308 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:54.308 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:54.308 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 09:37:54.308 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:54.308 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:54.308 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:37:54.308 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:54.308 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:54.308 | INFO | Uploaded 36GiB to GPU in 2.23008s (16.1429GiB/s) | 20260505 09:37:54.308 | INFO | Downloaded 34GiB to GPU in 2.09526s (16.2271GiB/s) | 20260505 09:37:59.447 | INFO | Analyzing chunk frames = [2190336,2190848) | 20260505 09:37:59.447 | INFO | MemInfo: { | 20260505 09:37:59.447 | INFO | "gpu" : " 17 GB", | 20260505 09:37:59.447 | INFO | "huge" : "", | 20260505 09:37:59.447 | INFO | "huge_shared" : " 27 GB", | 20260505 09:37:59.447 | INFO | "malloc" : " 14 GB", | 20260505 09:37:59.447 | INFO | "pinned" : " 29 GB" | 20260505 09:37:59.447 | INFO | } | 20260505 09:37:59.447 | INFO | Perf report for the last 5.1388 seconds: | 20260505 09:37:59.447 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:37:59.447 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:59.447 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:59.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:59.447 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 09:37:59.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:59.447 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:59.447 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 09:37:59.447 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:59.447 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:37:59.447 | INFO | Uploaded 36GiB to GPU in 2.20472s (16.3286GiB/s) | 20260505 09:37:59.447 | INFO | Downloaded 34GiB to GPU in 2.14948s (15.8178GiB/s) | 20260505 09:38:04.586 | INFO | Analyzing chunk frames = [2190848,2191360) | 20260505 09:38:04.586 | INFO | MemInfo: { | 20260505 09:38:04.586 | INFO | "gpu" : " 17 GB", | 20260505 09:38:04.586 | INFO | "huge" : "", | 20260505 09:38:04.586 | INFO | "huge_shared" : " 27 GB", | 20260505 09:38:04.586 | INFO | "malloc" : " 14 GB", | 20260505 09:38:04.587 | INFO | "pinned" : " 29 GB" | 20260505 09:38:04.587 | INFO | } | 20260505 09:38:04.587 | INFO | Perf report for the last 5.13969 seconds: | 20260505 09:38:04.587 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:38:04.587 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:04.587 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:04.587 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:04.587 | INFO | ANALYSIS : 63.9%, 3.0, 99.9ms, 0 | 20260505 09:38:04.587 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:04.587 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:04.587 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:38:04.587 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:04.587 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:04.587 | INFO | Uploaded 36GiB to GPU in 2.18777s (16.4551GiB/s) | 20260505 09:38:04.587 | INFO | Downloaded 34GiB to GPU in 2.05522s (16.5433GiB/s) | 20260505 09:38:04.718 | INFO | Starting a round of Baz IO | 20260505 09:38:07.999 | INFO | Populating RTMetrics report | 20260505 09:38:07.999 | INFO | RTMetrics report populated | 20260505 09:38:07.999 | INFO | Opening tmp RTMetrics json file | 20260505 09:38:08.000 | INFO | Serializing RTMetrics to json | 20260505 09:38:08.001 | INFO | Serialization done, swapping tmp file now... | 20260505 09:38:08.001 | INFO | RTMetrics done | 20260505 09:38:08.139 | INFO | Finished a round of Baz IO: Wrote 2.44 GiB in 3.42 seconds | 20260505 09:38:08.139 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.86% / 0.01% / 74.13% | 20260505 09:38:08.139 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.89253% / 0% / 6.75519e-05% | 20260505 09:38:08.139 | INFO | Baz IO overall: | 20260505 09:38:08.139 | INFO | BytesWritten : 1269211393078 | 20260505 09:38:08.139 | INFO | NumEvents : 604068844817 | 20260505 09:38:09.724 | INFO | Analyzing chunk frames = [2191360,2191872) | 20260505 09:38:09.724 | INFO | MemInfo: { | 20260505 09:38:09.724 | INFO | "gpu" : " 17 GB", | 20260505 09:38:09.724 | INFO | "huge" : "", | 20260505 09:38:09.724 | INFO | "huge_shared" : " 27 GB", | 20260505 09:38:09.724 | INFO | "malloc" : " 14 GB", | 20260505 09:38:09.724 | INFO | "pinned" : " 29 GB" | 20260505 09:38:09.724 | INFO | } | 20260505 09:38:09.724 | INFO | Perf report for the last 5.13771 seconds: | 20260505 09:38:09.724 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:38:09.724 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:09.724 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:09.724 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:09.724 | INFO | ANALYSIS : 65.3%, 2.9, 100.2ms, 0 | 20260505 09:38:09.724 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:09.725 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260505 09:38:09.725 | INFO | PRE_HQ : 62.6%, 1.8, 60.0ms, 0 | 20260505 09:38:09.725 | INFO | BAZWRITER : 65.9%, 4.1, 139.8ms, 0 | 20260505 09:38:09.725 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 09:38:09.725 | INFO | Uploaded 36GiB to GPU in 2.22495s (16.1801GiB/s) | 20260505 09:38:09.725 | INFO | Downloaded 38GiB to GPU in 2.3255s (16.3405GiB/s) | 20260505 09:38:14.864 | INFO | Analyzing chunk frames = [2191872,2192384) | 20260505 09:38:14.864 | INFO | MemInfo: { | 20260505 09:38:14.864 | INFO | "gpu" : " 17 GB", | 20260505 09:38:14.864 | INFO | "huge" : "", | 20260505 09:38:14.864 | INFO | "huge_shared" : " 27 GB", | 20260505 09:38:14.864 | INFO | "malloc" : " 14 GB", | 20260505 09:38:14.864 | INFO | "pinned" : " 29 GB" | 20260505 09:38:14.864 | INFO | } | 20260505 09:38:14.864 | INFO | Perf report for the last 5.13985 seconds: | 20260505 09:38:14.864 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:38:14.864 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:14.864 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:14.864 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:14.864 | INFO | ANALYSIS : 64.3%, 3.0, 100.5ms, 0 | 20260505 09:38:14.864 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:14.864 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:14.864 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:38:14.864 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:14.864 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:14.865 | INFO | Uploaded 36GiB to GPU in 2.18303s (16.4909GiB/s) | 20260505 09:38:14.865 | INFO | Downloaded 34GiB to GPU in 2.10126s (16.1808GiB/s) | 20260505 09:38:20.004 | INFO | Analyzing chunk frames = [2192384,2192896) | 20260505 09:38:20.004 | INFO | MemInfo: { | 20260505 09:38:20.004 | INFO | "gpu" : " 17 GB", | 20260505 09:38:20.004 | INFO | "huge" : "", | 20260505 09:38:20.004 | INFO | "huge_shared" : " 27 GB", | 20260505 09:38:20.004 | INFO | "malloc" : " 14 GB", | 20260505 09:38:20.004 | INFO | "pinned" : " 29 GB" | 20260505 09:38:20.004 | INFO | } | 20260505 09:38:20.004 | INFO | Perf report for the last 5.13982 seconds: | 20260505 09:38:20.004 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:38:20.004 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:20.004 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:20.004 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:20.004 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 09:38:20.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:20.004 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:20.004 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:38:20.004 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:20.004 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:20.005 | INFO | Uploaded 36GiB to GPU in 2.15694s (16.6903GiB/s) | 20260505 09:38:20.005 | INFO | Downloaded 34GiB to GPU in 2.06879s (16.4347GiB/s) | 20260505 09:38:25.144 | INFO | Analyzing chunk frames = [2192896,2193408) | 20260505 09:38:25.144 | INFO | MemInfo: { | 20260505 09:38:25.144 | INFO | "gpu" : " 17 GB", | 20260505 09:38:25.144 | INFO | "huge" : "", | 20260505 09:38:25.144 | INFO | "huge_shared" : " 27 GB", | 20260505 09:38:25.144 | INFO | "malloc" : " 14 GB", | 20260505 09:38:25.144 | INFO | "pinned" : " 29 GB" | 20260505 09:38:25.144 | INFO | } | 20260505 09:38:25.144 | INFO | Perf report for the last 5.13986 seconds: | 20260505 09:38:25.144 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:38:25.144 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:25.144 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:25.144 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:25.144 | INFO | ANALYSIS : 63.1%, 3.0, 98.1ms, 0 | 20260505 09:38:25.144 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:25.144 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:25.144 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 09:38:25.144 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:25.144 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:25.144 | INFO | Uploaded 36GiB to GPU in 2.17258s (16.5702GiB/s) | 20260505 09:38:25.144 | INFO | Downloaded 34GiB to GPU in 2.07418s (16.392GiB/s) | 20260505 09:38:30.282 | INFO | Analyzing chunk frames = [2193408,2193920) | 20260505 09:38:30.282 | INFO | MemInfo: { | 20260505 09:38:30.282 | INFO | "gpu" : " 17 GB", | 20260505 09:38:30.282 | INFO | "huge" : "", | 20260505 09:38:30.282 | INFO | "huge_shared" : " 27 GB", | 20260505 09:38:30.282 | INFO | "malloc" : " 14 GB", | 20260505 09:38:30.282 | INFO | "pinned" : " 29 GB" | 20260505 09:38:30.282 | INFO | } | 20260505 09:38:30.282 | INFO | Perf report for the last 5.13835 seconds: | 20260505 09:38:30.282 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:38:30.282 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:30.282 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:30.282 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:30.282 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 09:38:30.282 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:30.282 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:30.282 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 09:38:30.282 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:30.282 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:30.283 | INFO | Uploaded 36GiB to GPU in 2.23711s (16.0922GiB/s) | 20260505 09:38:30.283 | INFO | Downloaded 34GiB to GPU in 2.10085s (16.1839GiB/s) | 20260505 09:38:35.423 | INFO | Analyzing chunk frames = [2193920,2194432) | 20260505 09:38:35.423 | INFO | MemInfo: { | 20260505 09:38:35.423 | INFO | "gpu" : " 17 GB", | 20260505 09:38:35.423 | INFO | "huge" : "", | 20260505 09:38:35.423 | INFO | "huge_shared" : " 27 GB", | 20260505 09:38:35.423 | INFO | "malloc" : " 14 GB", | 20260505 09:38:35.423 | INFO | "pinned" : " 29 GB" | 20260505 09:38:35.423 | INFO | } | 20260505 09:38:35.423 | INFO | Perf report for the last 5.14052 seconds: | 20260505 09:38:35.423 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:38:35.423 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:35.423 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:35.423 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:35.423 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 09:38:35.423 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:35.423 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:35.423 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 09:38:35.423 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:35.423 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:35.423 | INFO | Uploaded 36GiB to GPU in 2.23902s (16.0785GiB/s) | 20260505 09:38:35.423 | INFO | Downloaded 34GiB to GPU in 2.09707s (16.2131GiB/s) | 20260505 09:38:40.561 | INFO | Analyzing chunk frames = [2194432,2194944) | 20260505 09:38:40.561 | INFO | MemInfo: { | 20260505 09:38:40.561 | INFO | "gpu" : " 17 GB", | 20260505 09:38:40.561 | INFO | "huge" : "", | 20260505 09:38:40.561 | INFO | "huge_shared" : " 27 GB", | 20260505 09:38:40.561 | INFO | "malloc" : " 14 GB", | 20260505 09:38:40.561 | INFO | "pinned" : " 29 GB" | 20260505 09:38:40.561 | INFO | } | 20260505 09:38:40.561 | INFO | Perf report for the last 5.13832 seconds: | 20260505 09:38:40.561 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:38:40.561 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:40.561 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:40.561 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:40.561 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 09:38:40.561 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:40.561 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:40.561 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 09:38:40.561 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:40.561 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:40.561 | INFO | Uploaded 36GiB to GPU in 2.20621s (16.3176GiB/s) | 20260505 09:38:40.562 | INFO | Downloaded 34GiB to GPU in 2.14795s (15.8291GiB/s) | 20260505 09:38:45.700 | INFO | Analyzing chunk frames = [2194944,2195456) | 20260505 09:38:45.700 | INFO | MemInfo: { | 20260505 09:38:45.700 | INFO | "gpu" : " 17 GB", | 20260505 09:38:45.700 | INFO | "huge" : "", | 20260505 09:38:45.700 | INFO | "huge_shared" : " 27 GB", | 20260505 09:38:45.700 | INFO | "malloc" : " 14 GB", | 20260505 09:38:45.701 | INFO | "pinned" : " 29 GB" | 20260505 09:38:45.701 | INFO | } | 20260505 09:38:45.701 | INFO | Perf report for the last 5.13949 seconds: | 20260505 09:38:45.701 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:38:45.701 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:45.701 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:45.701 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:45.701 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 09:38:45.701 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:45.701 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:45.701 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 09:38:45.701 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:45.701 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:45.701 | INFO | Uploaded 36GiB to GPU in 2.18301s (16.491GiB/s) | 20260505 09:38:45.701 | INFO | Downloaded 34GiB to GPU in 2.0523s (16.5668GiB/s) | 20260505 09:38:45.701 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 09:38:45.701 | INFO | Total mem usage: 15.3 GiB | 20260505 09:38:45.701 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 09:38:45.701 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -7.07%) | 20260505 09:38:45.701 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 09:38:45.701 | INFO | High Water Breakdown: | 20260505 09:38:45.701 | INFO | 60.8% : 8.7 GiB ( 93% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 09:38:45.701 | INFO | 39.1% : 5.6 GiB ( 84.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 09:38:45.701 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 09:38:45.701 | INFO | ------------------------------------------ | 20260505 09:38:45.701 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:38:45.701 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 09:38:45.701 | INFO | Total mem usage: 26.5 GiB | 20260505 09:38:45.701 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 09:38:45.701 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 09:38:45.701 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 09:38:45.701 | INFO | High Water Breakdown: | 20260505 09:38:45.701 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:38:45.701 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:38:45.701 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:38:45.701 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:38:45.701 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:38:45.701 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:38:45.701 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 09:38:45.701 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 09:38:45.702 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:38:45.702 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 09:38:45.702 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 09:38:45.702 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 09:38:45.702 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 09:38:45.702 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 09:38:45.702 | INFO | ------------------------------------------ | 20260505 09:38:45.702 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:38:45.702 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 09:38:45.702 | INFO | Total mem usage: 25.6 GiB | 20260505 09:38:45.702 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 09:38:45.702 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 09:38:45.702 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 09:38:45.702 | INFO | High Water Breakdown: | 20260505 09:38:45.702 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 09:38:45.702 | INFO | ------------------------------------------ | 20260505 09:38:45.702 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:38:45.702 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 09:38:45.702 | INFO | Total mem usage: 15.7 GiB | 20260505 09:38:45.702 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 09:38:45.702 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 09:38:45.702 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 09:38:45.702 | INFO | High Water Breakdown: | 20260505 09:38:45.702 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 09:38:45.702 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 09:38:45.702 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 09:38:45.702 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 09:38:45.702 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 09:38:45.702 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 09:38:45.702 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 09:38:45.702 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 09:38:45.702 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 09:38:45.702 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:38:45.702 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 09:38:45.702 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:38:45.702 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 09:38:45.702 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 09:38:45.702 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 09:38:45.702 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 09:38:45.702 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:38:45.702 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 09:38:45.702 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:38:45.702 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:38:45.702 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:38:45.702 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 09:38:45.702 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 09:38:45.702 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:38:45.702 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 09:38:45.702 | INFO | ------------------------------------------ | 20260505 09:38:45.702 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:38:45.702 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 09:38:45.702 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:38:45.702 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:38:45.702 | INFO | PerfReport: Full Execution -- 3.25e+01 / 7.70e+01, 1.40e+01 / 2.81e+00 | 20260505 09:38:45.702 | INFO | PerfReport: 0.29%, AggregateStats -- 9.32e-02 / 1.04e-01, 1.15e-02 / 4.35e-03 | 20260505 09:38:45.702 | INFO | PerfReport: 11.63%, Baseline -- 3.77e+00 / 4.42e+00, 5.99e-01 / 2.64e-01 | 20260505 09:38:45.702 | INFO | PerfReport: 3.31%, Histogram -- 1.07e+00 / 1.48e+00, 2.07e-01 / 1.02e-01 | 20260505 09:38:45.702 | INFO | PerfReport: 15.66%, DME -- 4.07e+01 / 4.52e+01, 4.42e+00 / 2.24e+00 | 20260505 09:38:45.702 | INFO | PerfReport: 56.55%, FrameLabeling -- 1.84e+01 / 2.09e+01, 1.85e+00 / 2.17e+00 | 20260505 09:38:45.702 | INFO | PerfReport: 7.41%, PulseAccumulating -- 2.40e+00 / 2.96e+00, 4.12e-01 / 2.24e-01 | 20260505 09:38:45.702 | INFO | PerfReport: 3.81%, Metrics -- 1.24e+00 / 1.52e+00, 3.53e-01 / 1.85e-01 | 20260505 09:38:45.702 | INFO | PerfReport: 1.35%, PulseSerializing -- 4.39e-01 / 4.88e-01, 8.18e-02 / 2.75e-02 | 20260505 09:38:45.702 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 09:38:45.702 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:38:45.702 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:38:45.702 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.38e+01 / 0.00e+00 | 20260505 09:38:45.702 | INFO | PerfReport: 22.58%, Upload -- 2.29e+01 / 0.00e+00, 8.15e+00 / 0.00e+00 | 20260505 09:38:45.702 | INFO | PerfReport: 21.78%, Download -- 2.21e+01 / 0.00e+00, 8.17e+00 / 0.00e+00 | 20260505 09:38:45.702 | INFO | PerfReport: 31.97%, Compute -- 3.25e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260505 09:38:45.702 | INFO | PerfReport: 1.86%, LockUpload -- 1.89e+00 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260505 09:38:45.702 | INFO | PerfReport: 1.09%, LockDownload -- 1.11e+00 / 0.00e+00, 2.72e+00 / 0.00e+00 | 20260505 09:38:45.702 | INFO | PerfReport: 20.71%, LockCompute -- 2.10e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260505 09:38:45.702 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.72e-02 / 0.00e+00 | 20260505 09:38:45.834 | INFO | Starting a round of Baz IO | 20260505 09:38:49.104 | INFO | Populating RTMetrics report | 20260505 09:38:49.104 | INFO | RTMetrics report populated | 20260505 09:38:49.104 | INFO | Opening tmp RTMetrics json file | 20260505 09:38:49.171 | INFO | Serializing RTMetrics to json | 20260505 09:38:49.176 | INFO | Serialization done, swapping tmp file now... | 20260505 09:38:49.178 | INFO | RTMetrics done | 20260505 09:38:49.246 | INFO | Finished a round of Baz IO: Wrote 2.436 GiB in 3.412 seconds | 20260505 09:38:49.246 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.84% / 0.01% / 74.14% | 20260505 09:38:49.246 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.87538% / 1.98682e-05% / 7.15256e-05% | 20260505 09:38:49.246 | INFO | Baz IO overall: | 20260505 09:38:49.246 | INFO | BytesWritten : 1271826499846 | 20260505 09:38:49.246 | INFO | NumEvents : 605317490073 | 20260505 09:38:50.840 | INFO | Analyzing chunk frames = [2195456,2195968) | 20260505 09:38:50.840 | INFO | MemInfo: { | 20260505 09:38:50.840 | INFO | "gpu" : " 17 GB", | 20260505 09:38:50.840 | INFO | "huge" : "", | 20260505 09:38:50.840 | INFO | "huge_shared" : " 27 GB", | 20260505 09:38:50.840 | INFO | "malloc" : " 14 GB", | 20260505 09:38:50.840 | INFO | "pinned" : " 29 GB" | 20260505 09:38:50.840 | INFO | } | 20260505 09:38:50.840 | INFO | Perf report for the last 5.13928 seconds: | 20260505 09:38:50.840 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:38:50.840 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:50.840 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:50.840 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:50.840 | INFO | ANALYSIS : 65.1%, 2.9, 99.6ms, 0 | 20260505 09:38:50.840 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:50.840 | INFO | RT_METRICS : 47.3%, 1.0, 24.8ms, 0 | 20260505 09:38:50.840 | INFO | PRE_HQ : 62.8%, 1.9, 60.3ms, 0 | 20260505 09:38:50.840 | INFO | BAZWRITER : 65.7%, 4.2, 140.7ms, 0 | 20260505 09:38:50.840 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260505 09:38:50.840 | INFO | Uploaded 36GiB to GPU in 2.2222s (16.2002GiB/s) | 20260505 09:38:50.840 | INFO | Downloaded 38GiB to GPU in 2.34388s (16.2124GiB/s) | 20260505 09:38:55.979 | INFO | Analyzing chunk frames = [2195968,2196480) | 20260505 09:38:55.979 | INFO | MemInfo: { | 20260505 09:38:55.979 | INFO | "gpu" : " 17 GB", | 20260505 09:38:55.979 | INFO | "huge" : "", | 20260505 09:38:55.979 | INFO | "huge_shared" : " 27 GB", | 20260505 09:38:55.979 | INFO | "malloc" : " 14 GB", | 20260505 09:38:55.979 | INFO | "pinned" : " 29 GB" | 20260505 09:38:55.980 | INFO | } | 20260505 09:38:55.980 | INFO | Perf report for the last 5.13967 seconds: | 20260505 09:38:55.980 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:38:55.980 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:55.980 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:55.980 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:55.980 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 09:38:55.980 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:55.980 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:55.980 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 09:38:55.980 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:55.980 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:38:55.980 | INFO | Uploaded 36GiB to GPU in 2.21457s (16.256GiB/s) | 20260505 09:38:55.980 | INFO | Downloaded 34GiB to GPU in 2.10382s (16.1611GiB/s) | 20260505 09:39:01.120 | INFO | Analyzing chunk frames = [2196480,2196992) | 20260505 09:39:01.120 | INFO | MemInfo: { | 20260505 09:39:01.120 | INFO | "gpu" : " 17 GB", | 20260505 09:39:01.120 | INFO | "huge" : "", | 20260505 09:39:01.120 | INFO | "huge_shared" : " 27 GB", | 20260505 09:39:01.120 | INFO | "malloc" : " 14 GB", | 20260505 09:39:01.120 | INFO | "pinned" : " 29 GB" | 20260505 09:39:01.120 | INFO | } | 20260505 09:39:01.120 | INFO | Perf report for the last 5.14057 seconds: | 20260505 09:39:01.120 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:39:01.120 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:01.120 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:01.120 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:01.120 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 09:39:01.120 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:01.120 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:01.120 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 09:39:01.120 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:01.120 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:01.121 | INFO | Uploaded 36GiB to GPU in 2.15244s (16.7252GiB/s) | 20260505 09:39:01.121 | INFO | Downloaded 34GiB to GPU in 2.06863s (16.436GiB/s) | 20260505 09:39:06.260 | INFO | Analyzing chunk frames = [2196992,2197504) | 20260505 09:39:06.260 | INFO | MemInfo: { | 20260505 09:39:06.260 | INFO | "gpu" : " 17 GB", | 20260505 09:39:06.260 | INFO | "huge" : "", | 20260505 09:39:06.260 | INFO | "huge_shared" : " 27 GB", | 20260505 09:39:06.260 | INFO | "malloc" : " 14 GB", | 20260505 09:39:06.260 | INFO | "pinned" : " 29 GB" | 20260505 09:39:06.260 | INFO | } | 20260505 09:39:06.260 | INFO | Perf report for the last 5.1397 seconds: | 20260505 09:39:06.260 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:39:06.260 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:06.260 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:06.260 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:06.260 | INFO | ANALYSIS : 63.2%, 3.0, 98.1ms, 0 | 20260505 09:39:06.260 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:06.260 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:06.260 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 09:39:06.260 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:06.260 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:06.260 | INFO | Uploaded 36GiB to GPU in 2.16719s (16.6114GiB/s) | 20260505 09:39:06.260 | INFO | Downloaded 34GiB to GPU in 2.07138s (16.4141GiB/s) | 20260505 09:39:11.397 | INFO | Analyzing chunk frames = [2197504,2198016) | 20260505 09:39:11.397 | INFO | MemInfo: { | 20260505 09:39:11.397 | INFO | "gpu" : " 17 GB", | 20260505 09:39:11.397 | INFO | "huge" : "", | 20260505 09:39:11.397 | INFO | "huge_shared" : " 27 GB", | 20260505 09:39:11.397 | INFO | "malloc" : " 14 GB", | 20260505 09:39:11.397 | INFO | "pinned" : " 29 GB" | 20260505 09:39:11.397 | INFO | } | 20260505 09:39:11.397 | INFO | Perf report for the last 5.13748 seconds: | 20260505 09:39:11.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:39:11.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:11.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:11.398 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:11.398 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 09:39:11.398 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:11.398 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:11.398 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 09:39:11.398 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:11.398 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:11.398 | INFO | Uploaded 36GiB to GPU in 2.23667s (16.0953GiB/s) | 20260505 09:39:11.398 | INFO | Downloaded 34GiB to GPU in 2.10579s (16.146GiB/s) | 20260505 09:39:16.537 | INFO | Analyzing chunk frames = [2198016,2198528) | 20260505 09:39:16.537 | INFO | MemInfo: { | 20260505 09:39:16.537 | INFO | "gpu" : " 17 GB", | 20260505 09:39:16.537 | INFO | "huge" : "", | 20260505 09:39:16.537 | INFO | "huge_shared" : " 27 GB", | 20260505 09:39:16.537 | INFO | "malloc" : " 14 GB", | 20260505 09:39:16.537 | INFO | "pinned" : " 29 GB" | 20260505 09:39:16.537 | INFO | } | 20260505 09:39:16.537 | INFO | Perf report for the last 5.13976 seconds: | 20260505 09:39:16.537 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:39:16.537 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:16.537 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:16.537 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:16.537 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 09:39:16.537 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:16.537 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:16.537 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 09:39:16.537 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:16.537 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:16.538 | INFO | Uploaded 36GiB to GPU in 2.23788s (16.0866GiB/s) | 20260505 09:39:16.538 | INFO | Downloaded 34GiB to GPU in 2.10103s (16.1826GiB/s) | 20260505 09:39:21.676 | INFO | Analyzing chunk frames = [2198528,2199040) | 20260505 09:39:21.676 | INFO | MemInfo: { | 20260505 09:39:21.676 | INFO | "gpu" : " 17 GB", | 20260505 09:39:21.676 | INFO | "huge" : "", | 20260505 09:39:21.676 | INFO | "huge_shared" : " 27 GB", | 20260505 09:39:21.676 | INFO | "malloc" : " 14 GB", | 20260505 09:39:21.676 | INFO | "pinned" : " 29 GB" | 20260505 09:39:21.676 | INFO | } | 20260505 09:39:21.676 | INFO | Perf report for the last 5.13881 seconds: | 20260505 09:39:21.676 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:39:21.676 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:21.676 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:21.676 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:21.676 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 09:39:21.676 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:21.676 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:21.676 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 09:39:21.676 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:21.676 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:21.676 | INFO | Uploaded 36GiB to GPU in 2.19185s (16.4245GiB/s) | 20260505 09:39:21.676 | INFO | Downloaded 34GiB to GPU in 2.11524s (16.0738GiB/s) | 20260505 09:39:26.817 | INFO | Analyzing chunk frames = [2199040,2199552) | 20260505 09:39:26.817 | INFO | MemInfo: { | 20260505 09:39:26.817 | INFO | "gpu" : " 17 GB", | 20260505 09:39:26.817 | INFO | "huge" : "", | 20260505 09:39:26.817 | INFO | "huge_shared" : " 27 GB", | 20260505 09:39:26.817 | INFO | "malloc" : " 14 GB", | 20260505 09:39:26.817 | INFO | "pinned" : " 29 GB" | 20260505 09:39:26.817 | INFO | } | 20260505 09:39:26.818 | INFO | Perf report for the last 5.14148 seconds: | 20260505 09:39:26.818 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:39:26.818 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:26.818 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:26.818 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:26.818 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 09:39:26.818 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:26.818 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:26.818 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 09:39:26.818 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:26.818 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:26.818 | INFO | Uploaded 36GiB to GPU in 2.18196s (16.4989GiB/s) | 20260505 09:39:26.818 | INFO | Downloaded 34GiB to GPU in 2.0543s (16.5506GiB/s) | 20260505 09:39:26.951 | INFO | Starting a round of Baz IO | 20260505 09:39:30.224 | INFO | Populating RTMetrics report | 20260505 09:39:30.225 | INFO | RTMetrics report populated | 20260505 09:39:30.225 | INFO | Opening tmp RTMetrics json file | 20260505 09:39:30.243 | INFO | Serializing RTMetrics to json | 20260505 09:39:30.265 | INFO | Serialization done, swapping tmp file now... | 20260505 09:39:30.270 | INFO | RTMetrics done | 20260505 09:39:30.370 | INFO | Finished a round of Baz IO: Wrote 2.439 GiB in 3.419 seconds | 20260505 09:39:30.370 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.83% / 0.01% / 74.16% | 20260505 09:39:30.370 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.87536% / 2.38419e-05% / 6.35783e-05% | 20260505 09:39:30.370 | INFO | Baz IO overall: | 20260505 09:39:30.370 | INFO | BytesWritten : 1274444892748 | 20260505 09:39:30.370 | INFO | NumEvents : 606568480423 | 20260505 09:39:31.954 | INFO | Analyzing chunk frames = [2199552,2200064) | 20260505 09:39:31.954 | INFO | MemInfo: { | 20260505 09:39:31.954 | INFO | "gpu" : " 17 GB", | 20260505 09:39:31.954 | INFO | "huge" : "", | 20260505 09:39:31.954 | INFO | "huge_shared" : " 27 GB", | 20260505 09:39:31.954 | INFO | "malloc" : " 14 GB", | 20260505 09:39:31.954 | INFO | "pinned" : " 29 GB" | 20260505 09:39:31.954 | INFO | } | 20260505 09:39:31.955 | INFO | Perf report for the last 5.13694 seconds: | 20260505 09:39:31.955 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:39:31.955 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:31.955 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:31.955 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:31.955 | INFO | ANALYSIS : 65.2%, 2.9, 99.8ms, 0 | 20260505 09:39:31.955 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:31.955 | INFO | RT_METRICS : 46.7%, 1.0, 24.5ms, 0 | 20260505 09:39:31.955 | INFO | PRE_HQ : 62.9%, 1.9, 60.5ms, 0 | 20260505 09:39:31.955 | INFO | BAZWRITER : 65.9%, 4.1, 139.8ms, 0 | 20260505 09:39:31.955 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 09:39:31.955 | INFO | Uploaded 36GiB to GPU in 2.25544s (15.9614GiB/s) | 20260505 09:39:31.955 | INFO | Downloaded 38GiB to GPU in 2.34856s (16.1801GiB/s) | 20260505 09:39:37.094 | INFO | Analyzing chunk frames = [2200064,2200576) | 20260505 09:39:37.094 | INFO | MemInfo: { | 20260505 09:39:37.094 | INFO | "gpu" : " 17 GB", | 20260505 09:39:37.094 | INFO | "huge" : "", | 20260505 09:39:37.094 | INFO | "huge_shared" : " 27 GB", | 20260505 09:39:37.094 | INFO | "malloc" : " 14 GB", | 20260505 09:39:37.094 | INFO | "pinned" : " 29 GB" | 20260505 09:39:37.094 | INFO | } | 20260505 09:39:37.095 | INFO | Perf report for the last 5.14001 seconds: | 20260505 09:39:37.095 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:39:37.095 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:37.095 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:37.095 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:37.095 | INFO | ANALYSIS : 64.5%, 3.0, 100.6ms, 0 | 20260505 09:39:37.095 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:37.095 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:37.095 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:39:37.095 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:37.095 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:37.095 | INFO | Uploaded 36GiB to GPU in 2.22335s (16.1918GiB/s) | 20260505 09:39:37.095 | INFO | Downloaded 34GiB to GPU in 2.10656s (16.14GiB/s) | 20260505 09:39:42.234 | INFO | Analyzing chunk frames = [2200576,2201088) | 20260505 09:39:42.234 | INFO | MemInfo: { | 20260505 09:39:42.234 | INFO | "gpu" : " 17 GB", | 20260505 09:39:42.234 | INFO | "huge" : "", | 20260505 09:39:42.234 | INFO | "huge_shared" : " 27 GB", | 20260505 09:39:42.234 | INFO | "malloc" : " 14 GB", | 20260505 09:39:42.234 | INFO | "pinned" : " 29 GB" | 20260505 09:39:42.234 | INFO | } | 20260505 09:39:42.234 | INFO | Perf report for the last 5.13947 seconds: | 20260505 09:39:42.234 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:39:42.234 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:42.234 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:42.234 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:42.234 | INFO | ANALYSIS : 63.0%, 3.0, 98.1ms, 0 | 20260505 09:39:42.234 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:42.234 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:42.234 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:39:42.234 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:42.234 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:42.234 | INFO | Uploaded 36GiB to GPU in 2.15264s (16.7237GiB/s) | 20260505 09:39:42.235 | INFO | Downloaded 34GiB to GPU in 2.0556s (16.5402GiB/s) | 20260505 09:39:47.373 | INFO | Analyzing chunk frames = [2201088,2201600) | 20260505 09:39:47.373 | INFO | MemInfo: { | 20260505 09:39:47.373 | INFO | "gpu" : " 17 GB", | 20260505 09:39:47.373 | INFO | "huge" : "", | 20260505 09:39:47.373 | INFO | "huge_shared" : " 27 GB", | 20260505 09:39:47.373 | INFO | "malloc" : " 14 GB", | 20260505 09:39:47.373 | INFO | "pinned" : " 29 GB" | 20260505 09:39:47.373 | INFO | } | 20260505 09:39:47.373 | INFO | Perf report for the last 5.13872 seconds: | 20260505 09:39:47.373 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:39:47.373 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:47.373 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:47.373 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:47.373 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 09:39:47.373 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:47.373 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:47.373 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 09:39:47.373 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:47.373 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:47.373 | INFO | Uploaded 36GiB to GPU in 2.17802s (16.5288GiB/s) | 20260505 09:39:47.373 | INFO | Downloaded 34GiB to GPU in 2.07485s (16.3868GiB/s) | 20260505 09:39:52.512 | INFO | Analyzing chunk frames = [2201600,2202112) | 20260505 09:39:52.512 | INFO | MemInfo: { | 20260505 09:39:52.512 | INFO | "gpu" : " 17 GB", | 20260505 09:39:52.512 | INFO | "huge" : "", | 20260505 09:39:52.512 | INFO | "huge_shared" : " 27 GB", | 20260505 09:39:52.512 | INFO | "malloc" : " 14 GB", | 20260505 09:39:52.512 | INFO | "pinned" : " 29 GB" | 20260505 09:39:52.512 | INFO | } | 20260505 09:39:52.512 | INFO | Perf report for the last 5.13928 seconds: | 20260505 09:39:52.512 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:39:52.512 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:52.512 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:52.512 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:52.512 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 09:39:52.512 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:52.512 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:52.512 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 09:39:52.512 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:52.512 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:52.512 | INFO | Uploaded 36GiB to GPU in 2.23794s (16.0862GiB/s) | 20260505 09:39:52.512 | INFO | Downloaded 34GiB to GPU in 2.10089s (16.1836GiB/s) | 20260505 09:39:57.652 | INFO | Analyzing chunk frames = [2202112,2202624) | 20260505 09:39:57.652 | INFO | MemInfo: { | 20260505 09:39:57.652 | INFO | "gpu" : " 17 GB", | 20260505 09:39:57.652 | INFO | "huge" : "", | 20260505 09:39:57.652 | INFO | "huge_shared" : " 27 GB", | 20260505 09:39:57.652 | INFO | "malloc" : " 14 GB", | 20260505 09:39:57.652 | INFO | "pinned" : " 29 GB" | 20260505 09:39:57.652 | INFO | } | 20260505 09:39:57.652 | INFO | Perf report for the last 5.14023 seconds: | 20260505 09:39:57.652 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:39:57.652 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:57.652 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:57.652 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:57.652 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 09:39:57.653 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:57.653 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:57.653 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 09:39:57.653 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:57.653 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:39:57.653 | INFO | Uploaded 36GiB to GPU in 2.22233s (16.1992GiB/s) | 20260505 09:39:57.653 | INFO | Downloaded 34GiB to GPU in 2.10258s (16.1706GiB/s) | 20260505 09:40:02.791 | INFO | Analyzing chunk frames = [2202624,2203136) | 20260505 09:40:02.791 | INFO | MemInfo: { | 20260505 09:40:02.791 | INFO | "gpu" : " 17 GB", | 20260505 09:40:02.791 | INFO | "huge" : "", | 20260505 09:40:02.791 | INFO | "huge_shared" : " 27 GB", | 20260505 09:40:02.791 | INFO | "malloc" : " 14 GB", | 20260505 09:40:02.791 | INFO | "pinned" : " 29 GB" | 20260505 09:40:02.791 | INFO | } | 20260505 09:40:02.791 | INFO | Perf report for the last 5.13863 seconds: | 20260505 09:40:02.791 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:40:02.791 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:02.791 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:02.791 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:02.791 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 09:40:02.791 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:02.791 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:02.791 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260505 09:40:02.791 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:02.791 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:02.791 | INFO | Uploaded 36GiB to GPU in 2.18687s (16.4619GiB/s) | 20260505 09:40:02.791 | INFO | Downloaded 34GiB to GPU in 2.1298s (15.964GiB/s) | 20260505 09:40:07.930 | INFO | Analyzing chunk frames = [2203136,2203648) | 20260505 09:40:07.930 | INFO | MemInfo: { | 20260505 09:40:07.930 | INFO | "gpu" : " 17 GB", | 20260505 09:40:07.930 | INFO | "huge" : "", | 20260505 09:40:07.930 | INFO | "huge_shared" : " 27 GB", | 20260505 09:40:07.930 | INFO | "malloc" : " 14 GB", | 20260505 09:40:07.930 | INFO | "pinned" : " 29 GB" | 20260505 09:40:07.930 | INFO | } | 20260505 09:40:07.930 | INFO | Perf report for the last 5.13956 seconds: | 20260505 09:40:07.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:40:07.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:07.931 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:07.931 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:07.931 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 09:40:07.931 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:07.931 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:07.931 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 09:40:07.931 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:07.931 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:07.931 | INFO | Uploaded 36GiB to GPU in 2.18467s (16.4785GiB/s) | 20260505 09:40:07.931 | INFO | Downloaded 34GiB to GPU in 2.05572s (16.5392GiB/s) | 20260505 09:40:07.931 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 09:40:07.931 | INFO | Total mem usage: 15.3 GiB | 20260505 09:40:07.931 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 09:40:07.931 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.54%) | 20260505 09:40:07.931 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 09:40:07.931 | INFO | High Water Breakdown: | 20260505 09:40:07.931 | INFO | 61% : 8.77 GiB ( 93.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 09:40:07.931 | INFO | 38.9% : 5.6 GiB ( 84.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 09:40:07.931 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 09:40:07.931 | INFO | ------------------------------------------ | 20260505 09:40:07.931 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:40:07.931 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 09:40:07.931 | INFO | Total mem usage: 26.5 GiB | 20260505 09:40:07.931 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 09:40:07.931 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 09:40:07.931 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 09:40:07.931 | INFO | High Water Breakdown: | 20260505 09:40:07.931 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:40:07.931 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:40:07.931 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:40:07.931 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:40:07.931 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:40:07.931 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:40:07.931 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 09:40:07.931 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 09:40:07.931 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:40:07.931 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 09:40:07.931 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 09:40:07.931 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 09:40:07.931 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 09:40:07.931 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 09:40:07.931 | INFO | ------------------------------------------ | 20260505 09:40:07.931 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:40:07.931 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 09:40:07.931 | INFO | Total mem usage: 25.6 GiB | 20260505 09:40:07.931 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 09:40:07.931 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 09:40:07.931 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 09:40:07.931 | INFO | High Water Breakdown: | 20260505 09:40:07.931 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 09:40:07.931 | INFO | ------------------------------------------ | 20260505 09:40:07.931 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:40:07.931 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 09:40:07.931 | INFO | Total mem usage: 15.7 GiB | 20260505 09:40:07.931 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 09:40:07.931 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 09:40:07.931 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 09:40:07.931 | INFO | High Water Breakdown: | 20260505 09:40:07.931 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 09:40:07.931 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 09:40:07.931 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 09:40:07.931 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 09:40:07.931 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 09:40:07.931 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 09:40:07.931 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 09:40:07.932 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 09:40:07.932 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 09:40:07.932 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:40:07.932 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 09:40:07.932 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:40:07.932 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 09:40:07.932 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 09:40:07.932 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 09:40:07.932 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 09:40:07.932 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:40:07.932 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 09:40:07.932 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:40:07.932 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:40:07.932 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:40:07.932 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 09:40:07.932 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 09:40:07.932 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:40:07.932 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 09:40:07.932 | INFO | ------------------------------------------ | 20260505 09:40:07.932 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:40:07.932 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 09:40:07.932 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:40:07.932 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:40:07.932 | INFO | PerfReport: Full Execution -- 3.24e+01 / 7.70e+01, 1.39e+01 / 2.41e+00 | 20260505 09:40:07.932 | INFO | PerfReport: 0.29%, AggregateStats -- 9.32e-02 / 1.04e-01, 1.01e-02 / 4.03e-03 | 20260505 09:40:07.932 | INFO | PerfReport: 11.65%, Baseline -- 3.78e+00 / 4.41e+00, 6.10e-01 / 2.72e-01 | 20260505 09:40:07.932 | INFO | PerfReport: 3.30%, Histogram -- 1.07e+00 / 1.45e+00, 2.05e-01 / 1.19e-01 | 20260505 09:40:07.932 | INFO | PerfReport: 15.66%, DME -- 4.07e+01 / 4.51e+01, 4.45e+00 / 2.27e+00 | 20260505 09:40:07.932 | INFO | PerfReport: 56.50%, FrameLabeling -- 1.83e+01 / 2.09e+01, 1.84e+00 / 1.93e+00 | 20260505 09:40:07.932 | INFO | PerfReport: 7.43%, PulseAccumulating -- 2.41e+00 / 2.97e+00, 4.16e-01 / 1.84e-01 | 20260505 09:40:07.932 | INFO | PerfReport: 3.81%, Metrics -- 1.24e+00 / 1.55e+00, 3.54e-01 / 1.80e-01 | 20260505 09:40:07.932 | INFO | PerfReport: 1.35%, PulseSerializing -- 4.39e-01 / 4.91e-01, 8.28e-02 / 2.88e-02 | 20260505 09:40:07.932 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 09:40:07.932 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:40:07.932 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:40:07.932 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.39e+01 / 0.00e+00 | 20260505 09:40:07.932 | INFO | PerfReport: 22.61%, Upload -- 2.29e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260505 09:40:07.932 | INFO | PerfReport: 21.77%, Download -- 2.21e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260505 09:40:07.932 | INFO | PerfReport: 31.97%, Compute -- 3.24e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 09:40:07.932 | INFO | PerfReport: 1.82%, LockUpload -- 1.85e+00 / 0.00e+00, 7.43e+00 / 0.00e+00 | 20260505 09:40:07.932 | INFO | PerfReport: 1.13%, LockDownload -- 1.14e+00 / 0.00e+00, 2.71e+00 / 0.00e+00 | 20260505 09:40:07.932 | INFO | PerfReport: 20.69%, LockCompute -- 2.10e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260505 09:40:07.932 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.69e-02 / 0.00e+00 | 20260505 09:40:08.062 | INFO | Starting a round of Baz IO | 20260505 09:40:11.338 | INFO | Populating RTMetrics report | 20260505 09:40:11.338 | INFO | RTMetrics report populated | 20260505 09:40:11.338 | INFO | Opening tmp RTMetrics json file | 20260505 09:40:11.398 | INFO | Serializing RTMetrics to json | 20260505 09:40:11.399 | INFO | Serialization done, swapping tmp file now... | 20260505 09:40:11.400 | INFO | RTMetrics done | 20260505 09:40:11.476 | INFO | Finished a round of Baz IO: Wrote 2.437 GiB in 3.414 seconds | 20260505 09:40:11.476 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.82% / 0.01% / 74.17% | 20260505 09:40:11.476 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.86752% / 1.98682e-05% / 7.94729e-05% | 20260505 09:40:11.476 | INFO | Baz IO overall: | 20260505 09:40:11.477 | INFO | BytesWritten : 1277061225285 | 20260505 09:40:11.477 | INFO | NumEvents : 607817736069 | 20260505 09:40:13.070 | INFO | Analyzing chunk frames = [2203648,2204160) | 20260505 09:40:13.070 | INFO | MemInfo: { | 20260505 09:40:13.070 | INFO | "gpu" : " 17 GB", | 20260505 09:40:13.070 | INFO | "huge" : "", | 20260505 09:40:13.070 | INFO | "huge_shared" : " 27 GB", | 20260505 09:40:13.070 | INFO | "malloc" : " 14 GB", | 20260505 09:40:13.070 | INFO | "pinned" : " 29 GB" | 20260505 09:40:13.070 | INFO | } | 20260505 09:40:13.071 | INFO | Perf report for the last 5.14003 seconds: | 20260505 09:40:13.071 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:40:13.071 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:13.071 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:13.071 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:13.071 | INFO | ANALYSIS : 65.2%, 2.9, 99.9ms, 0 | 20260505 09:40:13.071 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:13.071 | INFO | RT_METRICS : 46.9%, 1.0, 24.6ms, 0 | 20260505 09:40:13.071 | INFO | PRE_HQ : 62.7%, 1.8, 60.1ms, 0 | 20260505 09:40:13.071 | INFO | BAZWRITER : 65.7%, 4.1, 139.0ms, 0 | 20260505 09:40:13.071 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 09:40:13.071 | INFO | Uploaded 36GiB to GPU in 2.25413s (15.9707GiB/s) | 20260505 09:40:13.071 | INFO | Downloaded 38GiB to GPU in 2.34689s (16.1917GiB/s) | 20260505 09:40:18.210 | INFO | Analyzing chunk frames = [2204160,2204672) | 20260505 09:40:18.210 | INFO | MemInfo: { | 20260505 09:40:18.210 | INFO | "gpu" : " 17 GB", | 20260505 09:40:18.210 | INFO | "huge" : "", | 20260505 09:40:18.210 | INFO | "huge_shared" : " 27 GB", | 20260505 09:40:18.210 | INFO | "malloc" : " 14 GB", | 20260505 09:40:18.210 | INFO | "pinned" : " 29 GB" | 20260505 09:40:18.210 | INFO | } | 20260505 09:40:18.210 | INFO | Perf report for the last 5.13943 seconds: | 20260505 09:40:18.210 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:40:18.210 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:18.210 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:18.210 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:18.210 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 09:40:18.210 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:18.210 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:18.210 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260505 09:40:18.210 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:18.210 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:18.211 | INFO | Uploaded 36GiB to GPU in 2.17947s (16.5177GiB/s) | 20260505 09:40:18.211 | INFO | Downloaded 34GiB to GPU in 2.0995s (16.1943GiB/s) | 20260505 09:40:23.349 | INFO | Analyzing chunk frames = [2204672,2205184) | 20260505 09:40:23.349 | INFO | MemInfo: { | 20260505 09:40:23.349 | INFO | "gpu" : " 17 GB", | 20260505 09:40:23.349 | INFO | "huge" : "", | 20260505 09:40:23.349 | INFO | "huge_shared" : " 27 GB", | 20260505 09:40:23.349 | INFO | "malloc" : " 14 GB", | 20260505 09:40:23.349 | INFO | "pinned" : " 29 GB" | 20260505 09:40:23.349 | INFO | } | 20260505 09:40:23.349 | INFO | Perf report for the last 5.13921 seconds: | 20260505 09:40:23.349 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:40:23.349 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:23.349 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:23.349 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:23.349 | INFO | ANALYSIS : 63.0%, 3.0, 98.0ms, 0 | 20260505 09:40:23.349 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:23.349 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:23.349 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 09:40:23.349 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:23.349 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:23.349 | INFO | Uploaded 36GiB to GPU in 2.14513s (16.7822GiB/s) | 20260505 09:40:23.349 | INFO | Downloaded 34GiB to GPU in 2.05432s (16.5505GiB/s) | 20260505 09:40:28.487 | INFO | Analyzing chunk frames = [2205184,2205696) | 20260505 09:40:28.487 | INFO | MemInfo: { | 20260505 09:40:28.487 | INFO | "gpu" : " 17 GB", | 20260505 09:40:28.487 | INFO | "huge" : "", | 20260505 09:40:28.487 | INFO | "huge_shared" : " 27 GB", | 20260505 09:40:28.487 | INFO | "malloc" : " 14 GB", | 20260505 09:40:28.487 | INFO | "pinned" : " 29 GB" | 20260505 09:40:28.487 | INFO | } | 20260505 09:40:28.487 | INFO | Perf report for the last 5.13765 seconds: | 20260505 09:40:28.487 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:40:28.487 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:28.487 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:28.487 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:28.487 | INFO | ANALYSIS : 63.2%, 3.0, 98.1ms, 0 | 20260505 09:40:28.487 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:28.487 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:28.487 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:40:28.487 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:28.487 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:28.487 | INFO | Uploaded 36GiB to GPU in 2.15557s (16.7009GiB/s) | 20260505 09:40:28.487 | INFO | Downloaded 34GiB to GPU in 2.06833s (16.4384GiB/s) | 20260505 09:40:33.626 | INFO | Analyzing chunk frames = [2205696,2206208) | 20260505 09:40:33.626 | INFO | MemInfo: { | 20260505 09:40:33.626 | INFO | "gpu" : " 17 GB", | 20260505 09:40:33.626 | INFO | "huge" : "", | 20260505 09:40:33.626 | INFO | "huge_shared" : " 27 GB", | 20260505 09:40:33.626 | INFO | "malloc" : " 14 GB", | 20260505 09:40:33.626 | INFO | "pinned" : " 29 GB" | 20260505 09:40:33.626 | INFO | } | 20260505 09:40:33.626 | INFO | Perf report for the last 5.13944 seconds: | 20260505 09:40:33.626 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:40:33.626 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:33.627 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:33.627 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:33.627 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 09:40:33.627 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:33.627 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:33.627 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:40:33.627 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:33.627 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:33.627 | INFO | Uploaded 36GiB to GPU in 2.23785s (16.0869GiB/s) | 20260505 09:40:33.627 | INFO | Downloaded 34GiB to GPU in 2.0992s (16.1967GiB/s) | 20260505 09:40:38.771 | INFO | Analyzing chunk frames = [2206208,2206720) | 20260505 09:40:38.771 | INFO | MemInfo: { | 20260505 09:40:38.771 | INFO | "gpu" : " 17 GB", | 20260505 09:40:38.771 | INFO | "huge" : "", | 20260505 09:40:38.771 | INFO | "huge_shared" : " 27 GB", | 20260505 09:40:38.771 | INFO | "malloc" : " 14 GB", | 20260505 09:40:38.771 | INFO | "pinned" : " 29 GB" | 20260505 09:40:38.771 | INFO | } | 20260505 09:40:38.771 | INFO | Perf report for the last 5.14431 seconds: | 20260505 09:40:38.771 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:40:38.771 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:38.771 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:38.771 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:38.771 | INFO | ANALYSIS : 63.9%, 3.0, 99.9ms, 0 | 20260505 09:40:38.771 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:38.771 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:38.771 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:40:38.771 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:38.771 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:38.771 | INFO | Uploaded 36GiB to GPU in 2.23823s (16.0841GiB/s) | 20260505 09:40:38.771 | INFO | Downloaded 34GiB to GPU in 2.09494s (16.2296GiB/s) | 20260505 09:40:43.906 | INFO | Analyzing chunk frames = [2206720,2207232) | 20260505 09:40:43.906 | INFO | MemInfo: { | 20260505 09:40:43.906 | INFO | "gpu" : " 17 GB", | 20260505 09:40:43.906 | INFO | "huge" : "", | 20260505 09:40:43.906 | INFO | "huge_shared" : " 27 GB", | 20260505 09:40:43.906 | INFO | "malloc" : " 14 GB", | 20260505 09:40:43.906 | INFO | "pinned" : " 29 GB" | 20260505 09:40:43.906 | INFO | } | 20260505 09:40:43.906 | INFO | Perf report for the last 5.13537 seconds: | 20260505 09:40:43.906 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:40:43.906 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:43.906 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:43.906 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:43.906 | INFO | ANALYSIS : 63.8%, 3.0, 99.5ms, 0 | 20260505 09:40:43.906 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:43.906 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:43.906 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 09:40:43.906 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:43.906 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:43.906 | INFO | Uploaded 36GiB to GPU in 2.18258s (16.4942GiB/s) | 20260505 09:40:43.906 | INFO | Downloaded 34GiB to GPU in 2.15236s (15.7966GiB/s) | 20260505 09:40:49.044 | INFO | Analyzing chunk frames = [2207232,2207744) | 20260505 09:40:49.044 | INFO | MemInfo: { | 20260505 09:40:49.044 | INFO | "gpu" : " 17 GB", | 20260505 09:40:49.044 | INFO | "huge" : "", | 20260505 09:40:49.044 | INFO | "huge_shared" : " 27 GB", | 20260505 09:40:49.044 | INFO | "malloc" : " 14 GB", | 20260505 09:40:49.044 | INFO | "pinned" : " 29 GB" | 20260505 09:40:49.045 | INFO | } | 20260505 09:40:49.045 | INFO | Perf report for the last 5.13855 seconds: | 20260505 09:40:49.045 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:40:49.045 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:49.045 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:49.045 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:49.045 | INFO | ANALYSIS : 64.7%, 3.0, 101.1ms, 0 | 20260505 09:40:49.045 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:49.045 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:49.045 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 09:40:49.045 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:49.045 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:49.045 | INFO | Uploaded 36GiB to GPU in 2.24985s (16.0011GiB/s) | 20260505 09:40:49.045 | INFO | Downloaded 34GiB to GPU in 2.08644s (16.2957GiB/s) | 20260505 09:40:49.174 | INFO | Starting a round of Baz IO | 20260505 09:40:52.461 | INFO | Populating RTMetrics report | 20260505 09:40:52.461 | INFO | RTMetrics report populated | 20260505 09:40:52.461 | INFO | Opening tmp RTMetrics json file | 20260505 09:40:52.492 | INFO | Serializing RTMetrics to json | 20260505 09:40:52.493 | INFO | Serialization done, swapping tmp file now... | 20260505 09:40:52.494 | INFO | RTMetrics done | 20260505 09:40:52.618 | INFO | Finished a round of Baz IO: Wrote 2.438 GiB in 3.444 seconds | 20260505 09:40:52.618 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.80% / 0.01% / 74.18% | 20260505 09:40:52.618 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.86484% / 1.58946e-05% / 6.75519e-05% | 20260505 09:40:52.618 | INFO | Baz IO overall: | 20260505 09:40:52.619 | INFO | BytesWritten : 1279679297895 | 20260505 09:40:52.619 | INFO | NumEvents : 609068419033 | 20260505 09:40:54.184 | INFO | Analyzing chunk frames = [2207744,2208256) | 20260505 09:40:54.184 | INFO | MemInfo: { | 20260505 09:40:54.184 | INFO | "gpu" : " 17 GB", | 20260505 09:40:54.184 | INFO | "huge" : "", | 20260505 09:40:54.184 | INFO | "huge_shared" : " 27 GB", | 20260505 09:40:54.184 | INFO | "malloc" : " 14 GB", | 20260505 09:40:54.184 | INFO | "pinned" : " 29 GB" | 20260505 09:40:54.184 | INFO | } | 20260505 09:40:54.184 | INFO | Perf report for the last 5.1395 seconds: | 20260505 09:40:54.184 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:40:54.184 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:54.184 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:54.184 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:54.184 | INFO | ANALYSIS : 65.3%, 2.9, 100.3ms, 0 | 20260505 09:40:54.184 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:54.184 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260505 09:40:54.184 | INFO | PRE_HQ : 62.5%, 1.8, 60.0ms, 0 | 20260505 09:40:54.184 | INFO | BAZWRITER : 66.3%, 4.2, 142.2ms, 0 | 20260505 09:40:54.184 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.5ms, 0 | 20260505 09:40:54.184 | INFO | Uploaded 36GiB to GPU in 2.23037s (16.1408GiB/s) | 20260505 09:40:54.185 | INFO | Downloaded 38GiB to GPU in 2.3172s (16.3991GiB/s) | 20260505 09:40:59.324 | INFO | Analyzing chunk frames = [2208256,2208768) | 20260505 09:40:59.324 | INFO | MemInfo: { | 20260505 09:40:59.324 | INFO | "gpu" : " 17 GB", | 20260505 09:40:59.324 | INFO | "huge" : "", | 20260505 09:40:59.324 | INFO | "huge_shared" : " 27 GB", | 20260505 09:40:59.324 | INFO | "malloc" : " 14 GB", | 20260505 09:40:59.324 | INFO | "pinned" : " 29 GB" | 20260505 09:40:59.324 | INFO | } | 20260505 09:40:59.324 | INFO | Perf report for the last 5.13961 seconds: | 20260505 09:40:59.324 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:40:59.324 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:59.324 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:59.324 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:59.324 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 09:40:59.324 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:59.324 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:59.324 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 09:40:59.324 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:59.324 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:40:59.324 | INFO | Uploaded 36GiB to GPU in 2.18076s (16.508GiB/s) | 20260505 09:40:59.324 | INFO | Downloaded 34GiB to GPU in 2.09986s (16.1916GiB/s) | 20260505 09:41:04.463 | INFO | Analyzing chunk frames = [2208768,2209280) | 20260505 09:41:04.463 | INFO | MemInfo: { | 20260505 09:41:04.463 | INFO | "gpu" : " 17 GB", | 20260505 09:41:04.463 | INFO | "huge" : "", | 20260505 09:41:04.463 | INFO | "huge_shared" : " 27 GB", | 20260505 09:41:04.463 | INFO | "malloc" : " 14 GB", | 20260505 09:41:04.463 | INFO | "pinned" : " 29 GB" | 20260505 09:41:04.463 | INFO | } | 20260505 09:41:04.463 | INFO | Perf report for the last 5.13935 seconds: | 20260505 09:41:04.463 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:41:04.463 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:04.463 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:04.463 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:04.463 | INFO | ANALYSIS : 63.2%, 3.0, 98.4ms, 0 | 20260505 09:41:04.463 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:04.463 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:04.463 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:41:04.463 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:04.463 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:04.464 | INFO | Uploaded 36GiB to GPU in 2.15042s (16.7409GiB/s) | 20260505 09:41:04.464 | INFO | Downloaded 34GiB to GPU in 2.06104s (16.4965GiB/s) | 20260505 09:41:09.602 | INFO | Analyzing chunk frames = [2209280,2209792) | 20260505 09:41:09.602 | INFO | MemInfo: { | 20260505 09:41:09.602 | INFO | "gpu" : " 17 GB", | 20260505 09:41:09.602 | INFO | "huge" : "", | 20260505 09:41:09.602 | INFO | "huge_shared" : " 27 GB", | 20260505 09:41:09.602 | INFO | "malloc" : " 14 GB", | 20260505 09:41:09.602 | INFO | "pinned" : " 29 GB" | 20260505 09:41:09.602 | INFO | } | 20260505 09:41:09.602 | INFO | Perf report for the last 5.13901 seconds: | 20260505 09:41:09.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:41:09.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:09.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:09.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:09.602 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 09:41:09.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:09.602 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:09.602 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 09:41:09.602 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:09.602 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:09.602 | INFO | Uploaded 36GiB to GPU in 2.16119s (16.6575GiB/s) | 20260505 09:41:09.602 | INFO | Downloaded 34GiB to GPU in 2.07169s (16.4117GiB/s) | 20260505 09:41:14.742 | INFO | Analyzing chunk frames = [2209792,2210304) | 20260505 09:41:14.742 | INFO | MemInfo: { | 20260505 09:41:14.742 | INFO | "gpu" : " 17 GB", | 20260505 09:41:14.742 | INFO | "huge" : "", | 20260505 09:41:14.742 | INFO | "huge_shared" : " 27 GB", | 20260505 09:41:14.742 | INFO | "malloc" : " 14 GB", | 20260505 09:41:14.742 | INFO | "pinned" : " 29 GB" | 20260505 09:41:14.742 | INFO | } | 20260505 09:41:14.742 | INFO | Perf report for the last 5.1402 seconds: | 20260505 09:41:14.742 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:41:14.742 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:14.742 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:14.742 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:14.742 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 09:41:14.742 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:14.742 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:14.742 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 09:41:14.742 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:14.742 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:14.743 | INFO | Uploaded 36GiB to GPU in 2.2378s (16.0872GiB/s) | 20260505 09:41:14.743 | INFO | Downloaded 34GiB to GPU in 2.09407s (16.2363GiB/s) | 20260505 09:41:19.881 | INFO | Analyzing chunk frames = [2210304,2210816) | 20260505 09:41:19.881 | INFO | MemInfo: { | 20260505 09:41:19.881 | INFO | "gpu" : " 17 GB", | 20260505 09:41:19.881 | INFO | "huge" : "", | 20260505 09:41:19.882 | INFO | "huge_shared" : " 27 GB", | 20260505 09:41:19.882 | INFO | "malloc" : " 14 GB", | 20260505 09:41:19.882 | INFO | "pinned" : " 29 GB" | 20260505 09:41:19.882 | INFO | } | 20260505 09:41:19.882 | INFO | Perf report for the last 5.13926 seconds: | 20260505 09:41:19.882 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:41:19.882 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:19.882 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:19.882 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:19.882 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 09:41:19.882 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:19.882 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:19.882 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 09:41:19.882 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:19.882 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:19.882 | INFO | Uploaded 36GiB to GPU in 2.23795s (16.0861GiB/s) | 20260505 09:41:19.882 | INFO | Downloaded 34GiB to GPU in 2.0994s (16.1951GiB/s) | 20260505 09:41:25.021 | INFO | Analyzing chunk frames = [2210816,2211328) | 20260505 09:41:25.021 | INFO | MemInfo: { | 20260505 09:41:25.021 | INFO | "gpu" : " 17 GB", | 20260505 09:41:25.021 | INFO | "huge" : "", | 20260505 09:41:25.021 | INFO | "huge_shared" : " 27 GB", | 20260505 09:41:25.021 | INFO | "malloc" : " 14 GB", | 20260505 09:41:25.021 | INFO | "pinned" : " 29 GB" | 20260505 09:41:25.021 | INFO | } | 20260505 09:41:25.021 | INFO | Perf report for the last 5.13903 seconds: | 20260505 09:41:25.021 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:41:25.021 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:25.021 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:25.021 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:25.021 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 09:41:25.021 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:25.021 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:25.021 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 09:41:25.021 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:25.021 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:25.021 | INFO | Uploaded 36GiB to GPU in 2.18486s (16.477GiB/s) | 20260505 09:41:25.021 | INFO | Downloaded 34GiB to GPU in 2.14394s (15.8587GiB/s) | 20260505 09:41:30.159 | INFO | Analyzing chunk frames = [2211328,2211840) | 20260505 09:41:30.159 | INFO | MemInfo: { | 20260505 09:41:30.159 | INFO | "gpu" : " 17 GB", | 20260505 09:41:30.159 | INFO | "huge" : "", | 20260505 09:41:30.159 | INFO | "huge_shared" : " 27 GB", | 20260505 09:41:30.159 | INFO | "malloc" : " 14 GB", | 20260505 09:41:30.159 | INFO | "pinned" : " 29 GB" | 20260505 09:41:30.159 | INFO | } | 20260505 09:41:30.160 | INFO | Perf report for the last 5.13891 seconds: | 20260505 09:41:30.160 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:41:30.160 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:30.160 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:30.160 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:30.160 | INFO | ANALYSIS : 64.7%, 3.0, 101.0ms, 0 | 20260505 09:41:30.160 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:30.160 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:30.160 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:41:30.160 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:30.160 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:30.160 | INFO | Uploaded 36GiB to GPU in 2.25014s (15.999GiB/s) | 20260505 09:41:30.160 | INFO | Downloaded 34GiB to GPU in 2.08769s (16.2859GiB/s) | 20260505 09:41:30.160 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 09:41:30.160 | INFO | Total mem usage: 15.3 GiB | 20260505 09:41:30.160 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 09:41:30.160 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.54%) | 20260505 09:41:30.160 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 09:41:30.160 | INFO | High Water Breakdown: | 20260505 09:41:30.160 | INFO | 61% : 8.77 GiB ( 93.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 09:41:30.160 | INFO | 38.9% : 5.59 GiB ( 84.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 09:41:30.160 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 09:41:30.160 | INFO | ------------------------------------------ | 20260505 09:41:30.160 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:41:30.160 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 09:41:30.160 | INFO | Total mem usage: 26.5 GiB | 20260505 09:41:30.160 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 09:41:30.160 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 09:41:30.160 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 09:41:30.160 | INFO | High Water Breakdown: | 20260505 09:41:30.160 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:41:30.160 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:41:30.160 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:41:30.160 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:41:30.160 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:41:30.160 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:41:30.161 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 09:41:30.161 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 09:41:30.161 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:41:30.161 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 09:41:30.161 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 09:41:30.161 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 09:41:30.161 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 09:41:30.161 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 09:41:30.161 | INFO | ------------------------------------------ | 20260505 09:41:30.161 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:41:30.161 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 09:41:30.161 | INFO | Total mem usage: 25.6 GiB | 20260505 09:41:30.161 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 09:41:30.161 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 09:41:30.161 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 09:41:30.161 | INFO | High Water Breakdown: | 20260505 09:41:30.161 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 09:41:30.161 | INFO | ------------------------------------------ | 20260505 09:41:30.161 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:41:30.161 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 09:41:30.161 | INFO | Total mem usage: 15.7 GiB | 20260505 09:41:30.161 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 09:41:30.161 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 09:41:30.161 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 09:41:30.161 | INFO | High Water Breakdown: | 20260505 09:41:30.161 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 09:41:30.161 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 09:41:30.161 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 09:41:30.161 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 09:41:30.161 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 09:41:30.161 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 09:41:30.161 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 09:41:30.161 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 09:41:30.161 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 09:41:30.161 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:41:30.161 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 09:41:30.161 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:41:30.161 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 09:41:30.161 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 09:41:30.161 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 09:41:30.161 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 09:41:30.161 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:41:30.161 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 09:41:30.161 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:41:30.161 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:41:30.161 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:41:30.161 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 09:41:30.161 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 09:41:30.161 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:41:30.161 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 09:41:30.161 | INFO | ------------------------------------------ | 20260505 09:41:30.161 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:41:30.161 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 09:41:30.161 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:41:30.161 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:41:30.161 | INFO | PerfReport: Full Execution -- 3.25e+01 / 7.75e+01, 1.39e+01 / 2.51e+00 | 20260505 09:41:30.161 | INFO | PerfReport: 0.29%, AggregateStats -- 9.34e-02 / 1.02e-01, 1.17e-02 / 3.65e-03 | 20260505 09:41:30.161 | INFO | PerfReport: 11.64%, Baseline -- 3.78e+00 / 4.43e+00, 6.16e-01 / 3.07e-01 | 20260505 09:41:30.161 | INFO | PerfReport: 3.31%, Histogram -- 1.08e+00 / 1.49e+00, 2.11e-01 / 1.04e-01 | 20260505 09:41:30.161 | INFO | PerfReport: 15.66%, DME -- 4.07e+01 / 4.51e+01, 4.61e+00 / 2.52e+00 | 20260505 09:41:30.161 | INFO | PerfReport: 56.60%, FrameLabeling -- 1.84e+01 / 2.15e+01, 1.89e+00 / 2.19e+00 | 20260505 09:41:30.161 | INFO | PerfReport: 7.35%, PulseAccumulating -- 2.39e+00 / 2.95e+00, 4.05e-01 / 2.54e-01 | 20260505 09:41:30.161 | INFO | PerfReport: 3.80%, Metrics -- 1.24e+00 / 1.52e+00, 3.59e-01 / 2.16e-01 | 20260505 09:41:30.161 | INFO | PerfReport: 1.35%, PulseSerializing -- 4.39e-01 / 4.82e-01, 8.29e-02 / 3.78e-02 | 20260505 09:41:30.161 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 09:41:30.161 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:41:30.161 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:41:30.161 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.42e+01 / 0.00e+00 | 20260505 09:41:30.161 | INFO | PerfReport: 22.58%, Upload -- 2.30e+01 / 0.00e+00, 8.31e+00 / 0.00e+00 | 20260505 09:41:30.161 | INFO | PerfReport: 21.75%, Download -- 2.21e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260505 09:41:30.161 | INFO | PerfReport: 31.96%, Compute -- 3.25e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 09:41:30.161 | INFO | PerfReport: 1.93%, LockUpload -- 1.97e+00 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260505 09:41:30.161 | INFO | PerfReport: 1.08%, LockDownload -- 1.10e+00 / 0.00e+00, 2.69e+00 / 0.00e+00 | 20260505 09:41:30.161 | INFO | PerfReport: 20.68%, LockCompute -- 2.10e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260505 09:41:30.161 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.06e-02 / 0.00e+00 | 20260505 09:41:30.292 | INFO | Starting a round of Baz IO | 20260505 09:41:33.572 | INFO | Populating RTMetrics report | 20260505 09:41:33.572 | INFO | RTMetrics report populated | 20260505 09:41:33.572 | INFO | Opening tmp RTMetrics json file | 20260505 09:41:33.574 | INFO | Serializing RTMetrics to json | 20260505 09:41:33.598 | INFO | Serialization done, swapping tmp file now... | 20260505 09:41:33.600 | INFO | RTMetrics done | 20260505 09:41:33.734 | INFO | Finished a round of Baz IO: Wrote 2.441 GiB in 3.441 seconds | 20260505 09:41:33.734 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.79% / 0.01% / 74.20% | 20260505 09:41:33.734 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.86533% / 3.17891e-05% / 5.16574e-05% | 20260505 09:41:33.734 | INFO | Baz IO overall: | 20260505 09:41:33.734 | INFO | BytesWritten : 1282300093673 | 20260505 09:41:33.734 | INFO | NumEvents : 610321150076 | 20260505 09:41:35.299 | INFO | Analyzing chunk frames = [2211840,2212352) | 20260505 09:41:35.299 | INFO | MemInfo: { | 20260505 09:41:35.299 | INFO | "gpu" : " 17 GB", | 20260505 09:41:35.299 | INFO | "huge" : "", | 20260505 09:41:35.299 | INFO | "huge_shared" : " 27 GB", | 20260505 09:41:35.299 | INFO | "malloc" : " 14 GB", | 20260505 09:41:35.299 | INFO | "pinned" : " 29 GB" | 20260505 09:41:35.299 | INFO | } | 20260505 09:41:35.299 | INFO | Perf report for the last 5.13941 seconds: | 20260505 09:41:35.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:41:35.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:35.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:35.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:35.299 | INFO | ANALYSIS : 65.3%, 2.9, 100.3ms, 0 | 20260505 09:41:35.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:35.299 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260505 09:41:35.299 | INFO | PRE_HQ : 62.4%, 1.8, 59.9ms, 0 | 20260505 09:41:35.299 | INFO | BAZWRITER : 66.3%, 4.2, 142.2ms, 0 | 20260505 09:41:35.299 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 09:41:35.299 | INFO | Uploaded 36GiB to GPU in 2.22672s (16.1673GiB/s) | 20260505 09:41:35.300 | INFO | Downloaded 38GiB to GPU in 2.33699s (16.2603GiB/s) | 20260505 09:41:40.438 | INFO | Analyzing chunk frames = [2212352,2212864) | 20260505 09:41:40.438 | INFO | MemInfo: { | 20260505 09:41:40.438 | INFO | "gpu" : " 17 GB", | 20260505 09:41:40.438 | INFO | "huge" : "", | 20260505 09:41:40.438 | INFO | "huge_shared" : " 27 GB", | 20260505 09:41:40.438 | INFO | "malloc" : " 14 GB", | 20260505 09:41:40.438 | INFO | "pinned" : " 29 GB" | 20260505 09:41:40.438 | INFO | } | 20260505 09:41:40.438 | INFO | Perf report for the last 5.13921 seconds: | 20260505 09:41:40.438 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:41:40.438 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:40.438 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:40.438 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:40.438 | INFO | ANALYSIS : 64.5%, 3.0, 100.7ms, 0 | 20260505 09:41:40.438 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:40.438 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:40.438 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 09:41:40.438 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:40.438 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:40.438 | INFO | Uploaded 36GiB to GPU in 2.2332s (16.1203GiB/s) | 20260505 09:41:40.439 | INFO | Downloaded 34GiB to GPU in 2.10623s (16.1426GiB/s) | 20260505 09:41:45.578 | INFO | Analyzing chunk frames = [2212864,2213376) | 20260505 09:41:45.578 | INFO | MemInfo: { | 20260505 09:41:45.578 | INFO | "gpu" : " 17 GB", | 20260505 09:41:45.578 | INFO | "huge" : "", | 20260505 09:41:45.578 | INFO | "huge_shared" : " 27 GB", | 20260505 09:41:45.578 | INFO | "malloc" : " 14 GB", | 20260505 09:41:45.578 | INFO | "pinned" : " 29 GB" | 20260505 09:41:45.578 | INFO | } | 20260505 09:41:45.578 | INFO | Perf report for the last 5.1397 seconds: | 20260505 09:41:45.578 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:41:45.578 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:45.578 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:45.578 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:45.578 | INFO | ANALYSIS : 63.4%, 3.0, 98.6ms, 0 | 20260505 09:41:45.578 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:45.578 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:45.578 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:41:45.578 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:45.578 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:45.578 | INFO | Uploaded 36GiB to GPU in 2.16659s (16.616GiB/s) | 20260505 09:41:45.578 | INFO | Downloaded 34GiB to GPU in 2.06959s (16.4284GiB/s) | 20260505 09:41:50.718 | INFO | Analyzing chunk frames = [2213376,2213888) | 20260505 09:41:50.718 | INFO | MemInfo: { | 20260505 09:41:50.718 | INFO | "gpu" : " 17 GB", | 20260505 09:41:50.718 | INFO | "huge" : "", | 20260505 09:41:50.718 | INFO | "huge_shared" : " 27 GB", | 20260505 09:41:50.718 | INFO | "malloc" : " 14 GB", | 20260505 09:41:50.718 | INFO | "pinned" : " 29 GB" | 20260505 09:41:50.718 | INFO | } | 20260505 09:41:50.718 | INFO | Perf report for the last 5.13993 seconds: | 20260505 09:41:50.718 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:41:50.718 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:50.718 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:50.718 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:50.718 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 09:41:50.718 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:50.718 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:50.718 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 09:41:50.718 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:50.718 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:50.718 | INFO | Uploaded 36GiB to GPU in 2.15896s (16.6747GiB/s) | 20260505 09:41:50.718 | INFO | Downloaded 34GiB to GPU in 2.07175s (16.4113GiB/s) | 20260505 09:41:55.856 | INFO | Analyzing chunk frames = [2213888,2214400) | 20260505 09:41:55.856 | INFO | MemInfo: { | 20260505 09:41:55.856 | INFO | "gpu" : " 17 GB", | 20260505 09:41:55.856 | INFO | "huge" : "", | 20260505 09:41:55.856 | INFO | "huge_shared" : " 27 GB", | 20260505 09:41:55.856 | INFO | "malloc" : " 14 GB", | 20260505 09:41:55.856 | INFO | "pinned" : " 29 GB" | 20260505 09:41:55.856 | INFO | } | 20260505 09:41:55.856 | INFO | Perf report for the last 5.1385 seconds: | 20260505 09:41:55.856 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:41:55.856 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:55.857 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:55.857 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:55.857 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 09:41:55.857 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:55.857 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:55.857 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260505 09:41:55.857 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:55.857 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:41:55.857 | INFO | Uploaded 36GiB to GPU in 2.23811s (16.085GiB/s) | 20260505 09:41:55.857 | INFO | Downloaded 34GiB to GPU in 2.10237s (16.1722GiB/s) | 20260505 09:42:00.996 | INFO | Analyzing chunk frames = [2214400,2214912) | 20260505 09:42:00.996 | INFO | MemInfo: { | 20260505 09:42:00.996 | INFO | "gpu" : " 17 GB", | 20260505 09:42:00.996 | INFO | "huge" : "", | 20260505 09:42:00.996 | INFO | "huge_shared" : " 27 GB", | 20260505 09:42:00.996 | INFO | "malloc" : " 14 GB", | 20260505 09:42:00.996 | INFO | "pinned" : " 29 GB" | 20260505 09:42:00.996 | INFO | } | 20260505 09:42:00.996 | INFO | Perf report for the last 5.14006 seconds: | 20260505 09:42:00.997 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:42:00.997 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:00.997 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:00.997 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:00.997 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 09:42:00.997 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:00.997 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:00.997 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 09:42:00.997 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:00.997 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:00.997 | INFO | Uploaded 36GiB to GPU in 2.23174s (16.1309GiB/s) | 20260505 09:42:00.997 | INFO | Downloaded 34GiB to GPU in 2.09273s (16.2467GiB/s) | 20260505 09:42:06.135 | INFO | Analyzing chunk frames = [2214912,2215424) | 20260505 09:42:06.135 | INFO | MemInfo: { | 20260505 09:42:06.135 | INFO | "gpu" : " 17 GB", | 20260505 09:42:06.135 | INFO | "huge" : "", | 20260505 09:42:06.135 | INFO | "huge_shared" : " 27 GB", | 20260505 09:42:06.135 | INFO | "malloc" : " 14 GB", | 20260505 09:42:06.135 | INFO | "pinned" : " 29 GB" | 20260505 09:42:06.135 | INFO | } | 20260505 09:42:06.135 | INFO | Perf report for the last 5.13833 seconds: | 20260505 09:42:06.135 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:42:06.135 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:06.135 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:06.135 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:06.135 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 09:42:06.135 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:06.135 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:06.135 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 09:42:06.135 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:06.135 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:06.135 | INFO | Uploaded 36GiB to GPU in 2.17708s (16.5359GiB/s) | 20260505 09:42:06.135 | INFO | Downloaded 34GiB to GPU in 2.14431s (15.8559GiB/s) | 20260505 09:42:11.274 | INFO | Analyzing chunk frames = [2215424,2215936) | 20260505 09:42:11.274 | INFO | MemInfo: { | 20260505 09:42:11.274 | INFO | "gpu" : " 17 GB", | 20260505 09:42:11.274 | INFO | "huge" : "", | 20260505 09:42:11.274 | INFO | "huge_shared" : " 27 GB", | 20260505 09:42:11.274 | INFO | "malloc" : " 14 GB", | 20260505 09:42:11.274 | INFO | "pinned" : " 29 GB" | 20260505 09:42:11.274 | INFO | } | 20260505 09:42:11.274 | INFO | Perf report for the last 5.13935 seconds: | 20260505 09:42:11.274 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:42:11.274 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:11.274 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:11.274 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:11.274 | INFO | ANALYSIS : 64.6%, 3.0, 100.9ms, 0 | 20260505 09:42:11.274 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:11.274 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:11.274 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 09:42:11.274 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:11.274 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:11.275 | INFO | Uploaded 36GiB to GPU in 2.24891s (16.0078GiB/s) | 20260505 09:42:11.275 | INFO | Downloaded 34GiB to GPU in 2.08768s (16.286GiB/s) | 20260505 09:42:11.403 | INFO | Starting a round of Baz IO | 20260505 09:42:14.665 | INFO | Populating RTMetrics report | 20260505 09:42:14.665 | INFO | RTMetrics report populated | 20260505 09:42:14.665 | INFO | Opening tmp RTMetrics json file | 20260505 09:42:14.668 | INFO | Serializing RTMetrics to json | 20260505 09:42:14.717 | INFO | Serialization done, swapping tmp file now... | 20260505 09:42:14.771 | INFO | RTMetrics done | 20260505 09:42:14.846 | INFO | Finished a round of Baz IO: Wrote 2.438 GiB in 3.443 seconds | 20260505 09:42:14.846 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.78% / 0.01% / 74.21% | 20260505 09:42:14.846 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.85386% / 1.98682e-05% / 7.54992e-05% | 20260505 09:42:14.846 | INFO | Baz IO overall: | 20260505 09:42:14.846 | INFO | BytesWritten : 1284918035680 | 20260505 09:42:14.846 | INFO | NumEvents : 611571583141 | 20260505 09:42:16.414 | INFO | Analyzing chunk frames = [2215936,2216448) | 20260505 09:42:16.414 | INFO | MemInfo: { | 20260505 09:42:16.414 | INFO | "gpu" : " 17 GB", | 20260505 09:42:16.414 | INFO | "huge" : "", | 20260505 09:42:16.414 | INFO | "huge_shared" : " 27 GB", | 20260505 09:42:16.414 | INFO | "malloc" : " 14 GB", | 20260505 09:42:16.414 | INFO | "pinned" : " 29 GB" | 20260505 09:42:16.414 | INFO | } | 20260505 09:42:16.414 | INFO | Perf report for the last 5.13983 seconds: | 20260505 09:42:16.414 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:42:16.414 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:16.414 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:16.414 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:16.414 | INFO | ANALYSIS : 64.8%, 2.9, 98.6ms, 0 | 20260505 09:42:16.414 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:16.414 | INFO | RT_METRICS : 47.8%, 1.0, 25.1ms, 0 | 20260505 09:42:16.414 | INFO | PRE_HQ : 63.1%, 1.8, 60.3ms, 0 | 20260505 09:42:16.414 | INFO | BAZWRITER : 66.3%, 4.2, 143.4ms, 0 | 20260505 09:42:16.414 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260505 09:42:16.414 | INFO | Uploaded 36GiB to GPU in 2.24887s (16.008GiB/s) | 20260505 09:42:16.414 | INFO | Downloaded 38GiB to GPU in 2.37206s (16.0199GiB/s) | 20260505 09:42:21.553 | INFO | Analyzing chunk frames = [2216448,2216960) | 20260505 09:42:21.553 | INFO | MemInfo: { | 20260505 09:42:21.553 | INFO | "gpu" : " 17 GB", | 20260505 09:42:21.553 | INFO | "huge" : "", | 20260505 09:42:21.553 | INFO | "huge_shared" : " 27 GB", | 20260505 09:42:21.553 | INFO | "malloc" : " 14 GB", | 20260505 09:42:21.553 | INFO | "pinned" : " 29 GB" | 20260505 09:42:21.553 | INFO | } | 20260505 09:42:21.553 | INFO | Perf report for the last 5.13944 seconds: | 20260505 09:42:21.554 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:42:21.554 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:21.554 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:21.554 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:21.554 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 09:42:21.554 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:21.554 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:21.554 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 09:42:21.554 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:21.554 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:21.554 | INFO | Uploaded 36GiB to GPU in 2.19299s (16.4159GiB/s) | 20260505 09:42:21.554 | INFO | Downloaded 34GiB to GPU in 2.11429s (16.081GiB/s) | 20260505 09:42:26.692 | INFO | Analyzing chunk frames = [2216960,2217472) | 20260505 09:42:26.692 | INFO | MemInfo: { | 20260505 09:42:26.692 | INFO | "gpu" : " 17 GB", | 20260505 09:42:26.692 | INFO | "huge" : "", | 20260505 09:42:26.692 | INFO | "huge_shared" : " 27 GB", | 20260505 09:42:26.692 | INFO | "malloc" : " 14 GB", | 20260505 09:42:26.692 | INFO | "pinned" : " 29 GB" | 20260505 09:42:26.692 | INFO | } | 20260505 09:42:26.692 | INFO | Perf report for the last 5.13853 seconds: | 20260505 09:42:26.692 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:42:26.692 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:26.692 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:26.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:26.692 | INFO | ANALYSIS : 63.1%, 3.0, 98.2ms, 0 | 20260505 09:42:26.692 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:26.692 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:26.692 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 09:42:26.692 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:26.692 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:26.692 | INFO | Uploaded 36GiB to GPU in 2.13689s (16.8469GiB/s) | 20260505 09:42:26.692 | INFO | Downloaded 34GiB to GPU in 2.06254s (16.4846GiB/s) | 20260505 09:42:31.833 | INFO | Analyzing chunk frames = [2217472,2217984) | 20260505 09:42:31.833 | INFO | MemInfo: { | 20260505 09:42:31.833 | INFO | "gpu" : " 17 GB", | 20260505 09:42:31.833 | INFO | "huge" : "", | 20260505 09:42:31.833 | INFO | "huge_shared" : " 27 GB", | 20260505 09:42:31.833 | INFO | "malloc" : " 14 GB", | 20260505 09:42:31.833 | INFO | "pinned" : " 29 GB" | 20260505 09:42:31.833 | INFO | } | 20260505 09:42:31.833 | INFO | Perf report for the last 5.14089 seconds: | 20260505 09:42:31.833 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:42:31.833 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:31.833 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:31.833 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:31.833 | INFO | ANALYSIS : 63.3%, 3.0, 98.3ms, 0 | 20260505 09:42:31.833 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:31.833 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:31.833 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 09:42:31.833 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:31.833 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:31.833 | INFO | Uploaded 36GiB to GPU in 2.16735s (16.6101GiB/s) | 20260505 09:42:31.833 | INFO | Downloaded 34GiB to GPU in 2.07567s (16.3802GiB/s) | 20260505 09:42:36.974 | INFO | Analyzing chunk frames = [2217984,2218496) | 20260505 09:42:36.974 | INFO | MemInfo: { | 20260505 09:42:36.974 | INFO | "gpu" : " 17 GB", | 20260505 09:42:36.974 | INFO | "huge" : "", | 20260505 09:42:36.974 | INFO | "huge_shared" : " 27 GB", | 20260505 09:42:36.974 | INFO | "malloc" : " 14 GB", | 20260505 09:42:36.974 | INFO | "pinned" : " 29 GB" | 20260505 09:42:36.974 | INFO | } | 20260505 09:42:36.974 | INFO | Perf report for the last 5.14119 seconds: | 20260505 09:42:36.974 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:42:36.974 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:36.974 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:36.974 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:36.974 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 09:42:36.974 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:36.974 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:36.974 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 09:42:36.974 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:36.974 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:36.974 | INFO | Uploaded 36GiB to GPU in 2.23848s (16.0823GiB/s) | 20260505 09:42:36.974 | INFO | Downloaded 34GiB to GPU in 2.09664s (16.2164GiB/s) | 20260505 09:42:42.111 | INFO | Analyzing chunk frames = [2218496,2219008) | 20260505 09:42:42.111 | INFO | MemInfo: { | 20260505 09:42:42.111 | INFO | "gpu" : " 17 GB", | 20260505 09:42:42.111 | INFO | "huge" : "", | 20260505 09:42:42.111 | INFO | "huge_shared" : " 27 GB", | 20260505 09:42:42.111 | INFO | "malloc" : " 14 GB", | 20260505 09:42:42.111 | INFO | "pinned" : " 29 GB" | 20260505 09:42:42.111 | INFO | } | 20260505 09:42:42.111 | INFO | Perf report for the last 5.13671 seconds: | 20260505 09:42:42.111 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:42:42.111 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:42.111 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:42.111 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:42.111 | INFO | ANALYSIS : 64.1%, 3.0, 100.0ms, 0 | 20260505 09:42:42.111 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:42.111 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:42.111 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 09:42:42.111 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:42.111 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:42.111 | INFO | Uploaded 36GiB to GPU in 2.23727s (16.0911GiB/s) | 20260505 09:42:42.111 | INFO | Downloaded 34GiB to GPU in 2.09094s (16.2606GiB/s) | 20260505 09:42:47.249 | INFO | Analyzing chunk frames = [2219008,2219520) | 20260505 09:42:47.249 | INFO | MemInfo: { | 20260505 09:42:47.249 | INFO | "gpu" : " 17 GB", | 20260505 09:42:47.249 | INFO | "huge" : "", | 20260505 09:42:47.249 | INFO | "huge_shared" : " 27 GB", | 20260505 09:42:47.249 | INFO | "malloc" : " 14 GB", | 20260505 09:42:47.249 | INFO | "pinned" : " 29 GB" | 20260505 09:42:47.249 | INFO | } | 20260505 09:42:47.249 | INFO | Perf report for the last 5.13857 seconds: | 20260505 09:42:47.249 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:42:47.249 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:47.249 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:47.249 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:47.249 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 09:42:47.249 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:47.249 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:47.249 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 09:42:47.249 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:47.249 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:47.250 | INFO | Uploaded 36GiB to GPU in 2.14304s (16.7985GiB/s) | 20260505 09:42:47.250 | INFO | Downloaded 34GiB to GPU in 2.15568s (15.7723GiB/s) | 20260505 09:42:52.389 | INFO | Analyzing chunk frames = [2219520,2220032) | 20260505 09:42:52.389 | INFO | MemInfo: { | 20260505 09:42:52.389 | INFO | "gpu" : " 17 GB", | 20260505 09:42:52.389 | INFO | "huge" : "", | 20260505 09:42:52.389 | INFO | "huge_shared" : " 27 GB", | 20260505 09:42:52.389 | INFO | "malloc" : " 14 GB", | 20260505 09:42:52.389 | INFO | "pinned" : " 29 GB" | 20260505 09:42:52.389 | INFO | } | 20260505 09:42:52.389 | INFO | Perf report for the last 5.13985 seconds: | 20260505 09:42:52.389 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:42:52.389 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:52.389 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:52.389 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:52.389 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 09:42:52.389 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:52.389 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:52.389 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 09:42:52.389 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:52.389 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:52.389 | INFO | Uploaded 36GiB to GPU in 2.19108s (16.4303GiB/s) | 20260505 09:42:52.389 | INFO | Downloaded 34GiB to GPU in 2.04996s (16.5857GiB/s) | 20260505 09:42:52.390 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 09:42:52.390 | INFO | Total mem usage: 15.3 GiB | 20260505 09:42:52.390 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 09:42:52.390 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -5.76%) | 20260505 09:42:52.390 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 09:42:52.390 | INFO | High Water Breakdown: | 20260505 09:42:52.390 | INFO | 61.1% : 8.85 GiB ( 94.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 09:42:52.390 | INFO | 38.8% : 5.62 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 09:42:52.390 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 09:42:52.390 | INFO | ------------------------------------------ | 20260505 09:42:52.390 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:42:52.390 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 09:42:52.390 | INFO | Total mem usage: 26.5 GiB | 20260505 09:42:52.390 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 09:42:52.390 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 09:42:52.390 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 09:42:52.390 | INFO | High Water Breakdown: | 20260505 09:42:52.390 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:42:52.390 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:42:52.390 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:42:52.390 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:42:52.390 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:42:52.390 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:42:52.390 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 09:42:52.390 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 09:42:52.390 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:42:52.390 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 09:42:52.390 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 09:42:52.390 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 09:42:52.390 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 09:42:52.390 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 09:42:52.390 | INFO | ------------------------------------------ | 20260505 09:42:52.390 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:42:52.390 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 09:42:52.390 | INFO | Total mem usage: 25.6 GiB | 20260505 09:42:52.390 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 09:42:52.390 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 09:42:52.390 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 09:42:52.390 | INFO | High Water Breakdown: | 20260505 09:42:52.390 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 09:42:52.390 | INFO | ------------------------------------------ | 20260505 09:42:52.390 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:42:52.390 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 09:42:52.390 | INFO | Total mem usage: 15.7 GiB | 20260505 09:42:52.390 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 09:42:52.390 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 09:42:52.390 | INFO | Unused Cache: 880 MiB (Waste: 5.48%) | 20260505 09:42:52.390 | INFO | High Water Breakdown: | 20260505 09:42:52.390 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 09:42:52.390 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 09:42:52.390 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 09:42:52.390 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 09:42:52.390 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 09:42:52.390 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 09:42:52.390 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 09:42:52.390 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 09:42:52.390 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 09:42:52.390 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:42:52.390 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 09:42:52.390 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:42:52.390 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 09:42:52.390 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 09:42:52.390 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 09:42:52.390 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 09:42:52.390 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:42:52.390 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 09:42:52.390 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:42:52.390 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:42:52.390 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:42:52.390 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 09:42:52.390 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 09:42:52.390 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:42:52.390 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 09:42:52.391 | INFO | ------------------------------------------ | 20260505 09:42:52.391 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:42:52.391 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 09:42:52.391 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:42:52.391 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:42:52.391 | INFO | PerfReport: Full Execution -- 3.25e+01 / 7.69e+01, 1.38e+01 / 2.37e+00 | 20260505 09:42:52.391 | INFO | PerfReport: 0.29%, AggregateStats -- 9.35e-02 / 1.02e-01, 1.16e-02 / 4.31e-03 | 20260505 09:42:52.391 | INFO | PerfReport: 11.65%, Baseline -- 3.78e+00 / 4.45e+00, 6.11e-01 / 3.23e-01 | 20260505 09:42:52.391 | INFO | PerfReport: 3.32%, Histogram -- 1.08e+00 / 1.48e+00, 2.09e-01 / 1.06e-01 | 20260505 09:42:52.391 | INFO | PerfReport: 15.56%, DME -- 4.04e+01 / 4.44e+01, 4.34e+00 / 1.92e+00 | 20260505 09:42:52.391 | INFO | PerfReport: 56.61%, FrameLabeling -- 1.84e+01 / 2.14e+01, 1.88e+00 / 2.02e+00 | 20260505 09:42:52.391 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.40e+00 / 2.95e+00, 4.08e-01 / 2.36e-01 | 20260505 09:42:52.391 | INFO | PerfReport: 3.83%, Metrics -- 1.24e+00 / 1.53e+00, 3.63e-01 / 2.13e-01 | 20260505 09:42:52.391 | INFO | PerfReport: 1.35%, PulseSerializing -- 4.40e-01 / 4.84e-01, 8.35e-02 / 2.91e-02 | 20260505 09:42:52.391 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 09:42:52.391 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:42:52.391 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:42:52.391 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.38e+01 / 0.00e+00 | 20260505 09:42:52.391 | INFO | PerfReport: 22.60%, Upload -- 2.29e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260505 09:42:52.391 | INFO | PerfReport: 21.82%, Download -- 2.22e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260505 09:42:52.391 | INFO | PerfReport: 32.00%, Compute -- 3.25e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 09:42:52.391 | INFO | PerfReport: 1.87%, LockUpload -- 1.90e+00 / 0.00e+00, 7.45e+00 / 0.00e+00 | 20260505 09:42:52.391 | INFO | PerfReport: 1.10%, LockDownload -- 1.12e+00 / 0.00e+00, 2.75e+00 / 0.00e+00 | 20260505 09:42:52.391 | INFO | PerfReport: 20.59%, LockCompute -- 2.09e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260505 09:42:52.391 | INFO | PerfReport: 0.01%, Misc -- 1.50e-02 / 0.00e+00, 2.68e-02 / 0.00e+00 | 20260505 09:42:52.518 | INFO | Starting a round of Baz IO | 20260505 09:42:55.780 | INFO | Populating RTMetrics report | 20260505 09:42:55.780 | INFO | RTMetrics report populated | 20260505 09:42:55.780 | INFO | Opening tmp RTMetrics json file | 20260505 09:42:55.820 | INFO | Serializing RTMetrics to json | 20260505 09:42:55.865 | INFO | Serialization done, swapping tmp file now... | 20260505 09:42:55.875 | INFO | RTMetrics done | 20260505 09:42:55.913 | INFO | Finished a round of Baz IO: Wrote 2.437 GiB in 3.394 seconds | 20260505 09:42:55.913 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.76% / 0.01% / 74.22% | 20260505 09:42:55.913 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.83508% / 1.98682e-05% / 7.94729e-05% | 20260505 09:42:55.913 | INFO | Baz IO overall: | 20260505 09:42:55.913 | INFO | BytesWritten : 1287534799342 | 20260505 09:42:55.913 | INFO | NumEvents : 612820640496 | 20260505 09:42:57.528 | INFO | Analyzing chunk frames = [2220032,2220544) | 20260505 09:42:57.528 | INFO | MemInfo: { | 20260505 09:42:57.528 | INFO | "gpu" : " 17 GB", | 20260505 09:42:57.528 | INFO | "huge" : "", | 20260505 09:42:57.528 | INFO | "huge_shared" : " 27 GB", | 20260505 09:42:57.529 | INFO | "malloc" : " 14 GB", | 20260505 09:42:57.529 | INFO | "pinned" : " 29 GB" | 20260505 09:42:57.529 | INFO | } | 20260505 09:42:57.529 | INFO | Perf report for the last 5.13943 seconds: | 20260505 09:42:57.529 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:42:57.529 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:57.529 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:57.529 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:57.529 | INFO | ANALYSIS : 64.9%, 2.9, 99.3ms, 0 | 20260505 09:42:57.529 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:42:57.529 | INFO | RT_METRICS : 47.5%, 1.0, 24.9ms, 0 | 20260505 09:42:57.529 | INFO | PRE_HQ : 63.3%, 1.8, 60.3ms, 0 | 20260505 09:42:57.529 | INFO | BAZWRITER : 65.2%, 4.2, 139.8ms, 0 | 20260505 09:42:57.529 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260505 09:42:57.529 | INFO | Uploaded 36GiB to GPU in 2.26291s (15.9087GiB/s) | 20260505 09:42:57.529 | INFO | Downloaded 38GiB to GPU in 2.3683s (16.0452GiB/s) | 20260505 09:43:02.667 | INFO | Analyzing chunk frames = [2220544,2221056) | 20260505 09:43:02.667 | INFO | MemInfo: { | 20260505 09:43:02.667 | INFO | "gpu" : " 17 GB", | 20260505 09:43:02.667 | INFO | "huge" : "", | 20260505 09:43:02.667 | INFO | "huge_shared" : " 27 GB", | 20260505 09:43:02.667 | INFO | "malloc" : " 14 GB", | 20260505 09:43:02.667 | INFO | "pinned" : " 29 GB" | 20260505 09:43:02.667 | INFO | } | 20260505 09:43:02.667 | INFO | Perf report for the last 5.13876 seconds: | 20260505 09:43:02.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:43:02.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:02.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:02.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:02.668 | INFO | ANALYSIS : 64.5%, 3.0, 100.8ms, 0 | 20260505 09:43:02.668 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:02.668 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:02.668 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 09:43:02.668 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:02.668 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:02.668 | INFO | Uploaded 36GiB to GPU in 2.22458s (16.1828GiB/s) | 20260505 09:43:02.668 | INFO | Downloaded 34GiB to GPU in 2.10744s (16.1333GiB/s) | 20260505 09:43:07.807 | INFO | Analyzing chunk frames = [2221056,2221568) | 20260505 09:43:07.807 | INFO | MemInfo: { | 20260505 09:43:07.807 | INFO | "gpu" : " 17 GB", | 20260505 09:43:07.807 | INFO | "huge" : "", | 20260505 09:43:07.807 | INFO | "huge_shared" : " 27 GB", | 20260505 09:43:07.807 | INFO | "malloc" : " 14 GB", | 20260505 09:43:07.807 | INFO | "pinned" : " 29 GB" | 20260505 09:43:07.807 | INFO | } | 20260505 09:43:07.807 | INFO | Perf report for the last 5.13975 seconds: | 20260505 09:43:07.807 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:43:07.807 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:07.807 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:07.807 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:07.807 | INFO | ANALYSIS : 63.2%, 3.0, 98.3ms, 0 | 20260505 09:43:07.807 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:07.807 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:07.807 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:43:07.807 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:07.807 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:07.808 | INFO | Uploaded 36GiB to GPU in 2.14937s (16.7491GiB/s) | 20260505 09:43:07.808 | INFO | Downloaded 34GiB to GPU in 2.0601s (16.504GiB/s) | 20260505 09:43:12.946 | INFO | Analyzing chunk frames = [2221568,2222080) | 20260505 09:43:12.946 | INFO | MemInfo: { | 20260505 09:43:12.946 | INFO | "gpu" : " 17 GB", | 20260505 09:43:12.946 | INFO | "huge" : "", | 20260505 09:43:12.946 | INFO | "huge_shared" : " 27 GB", | 20260505 09:43:12.946 | INFO | "malloc" : " 14 GB", | 20260505 09:43:12.946 | INFO | "pinned" : " 29 GB" | 20260505 09:43:12.946 | INFO | } | 20260505 09:43:12.947 | INFO | Perf report for the last 5.13946 seconds: | 20260505 09:43:12.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:43:12.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:12.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:12.947 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:12.947 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 09:43:12.947 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:12.947 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:12.947 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:43:12.947 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:12.947 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:12.947 | INFO | Uploaded 36GiB to GPU in 2.17016s (16.5886GiB/s) | 20260505 09:43:12.947 | INFO | Downloaded 34GiB to GPU in 2.07903s (16.3538GiB/s) | 20260505 09:43:18.085 | INFO | Analyzing chunk frames = [2222080,2222592) | 20260505 09:43:18.085 | INFO | MemInfo: { | 20260505 09:43:18.085 | INFO | "gpu" : " 17 GB", | 20260505 09:43:18.085 | INFO | "huge" : "", | 20260505 09:43:18.085 | INFO | "huge_shared" : " 27 GB", | 20260505 09:43:18.085 | INFO | "malloc" : " 14 GB", | 20260505 09:43:18.085 | INFO | "pinned" : " 29 GB" | 20260505 09:43:18.086 | INFO | } | 20260505 09:43:18.086 | INFO | Perf report for the last 5.13893 seconds: | 20260505 09:43:18.086 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:43:18.086 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:18.086 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:18.086 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:18.086 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 09:43:18.086 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:18.086 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:18.086 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 09:43:18.086 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:18.086 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:18.086 | INFO | Uploaded 36GiB to GPU in 2.23896s (16.0789GiB/s) | 20260505 09:43:18.086 | INFO | Downloaded 34GiB to GPU in 2.10849s (16.1253GiB/s) | 20260505 09:43:23.226 | INFO | Analyzing chunk frames = [2222592,2223104) | 20260505 09:43:23.226 | INFO | MemInfo: { | 20260505 09:43:23.226 | INFO | "gpu" : " 17 GB", | 20260505 09:43:23.226 | INFO | "huge" : "", | 20260505 09:43:23.226 | INFO | "huge_shared" : " 27 GB", | 20260505 09:43:23.226 | INFO | "malloc" : " 14 GB", | 20260505 09:43:23.226 | INFO | "pinned" : " 29 GB" | 20260505 09:43:23.226 | INFO | } | 20260505 09:43:23.226 | INFO | Perf report for the last 5.1405 seconds: | 20260505 09:43:23.226 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:43:23.226 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:23.226 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:23.226 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:23.226 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 09:43:23.226 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:23.226 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:23.226 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 09:43:23.226 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:23.226 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:23.226 | INFO | Uploaded 36GiB to GPU in 2.23651s (16.0965GiB/s) | 20260505 09:43:23.226 | INFO | Downloaded 34GiB to GPU in 2.09551s (16.2251GiB/s) | 20260505 09:43:28.364 | INFO | Analyzing chunk frames = [2223104,2223616) | 20260505 09:43:28.364 | INFO | MemInfo: { | 20260505 09:43:28.364 | INFO | "gpu" : " 17 GB", | 20260505 09:43:28.364 | INFO | "huge" : "", | 20260505 09:43:28.364 | INFO | "huge_shared" : " 27 GB", | 20260505 09:43:28.364 | INFO | "malloc" : " 14 GB", | 20260505 09:43:28.364 | INFO | "pinned" : " 29 GB" | 20260505 09:43:28.364 | INFO | } | 20260505 09:43:28.365 | INFO | Perf report for the last 5.13857 seconds: | 20260505 09:43:28.365 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:43:28.365 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:28.365 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:28.365 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:28.365 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 09:43:28.365 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:28.365 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:28.365 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 09:43:28.365 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:28.365 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:28.365 | INFO | Uploaded 36GiB to GPU in 2.17099s (16.5823GiB/s) | 20260505 09:43:28.365 | INFO | Downloaded 34GiB to GPU in 2.14181s (15.8745GiB/s) | 20260505 09:43:33.503 | INFO | Analyzing chunk frames = [2223616,2224128) | 20260505 09:43:33.503 | INFO | MemInfo: { | 20260505 09:43:33.503 | INFO | "gpu" : " 17 GB", | 20260505 09:43:33.503 | INFO | "huge" : "", | 20260505 09:43:33.503 | INFO | "huge_shared" : " 27 GB", | 20260505 09:43:33.503 | INFO | "malloc" : " 14 GB", | 20260505 09:43:33.504 | INFO | "pinned" : " 29 GB" | 20260505 09:43:33.504 | INFO | } | 20260505 09:43:33.504 | INFO | Perf report for the last 5.13892 seconds: | 20260505 09:43:33.504 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:43:33.504 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:33.504 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:33.504 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:33.504 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 09:43:33.504 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:33.504 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:33.504 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 09:43:33.504 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:33.504 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:33.504 | INFO | Uploaded 36GiB to GPU in 2.18135s (16.5035GiB/s) | 20260505 09:43:33.504 | INFO | Downloaded 34GiB to GPU in 2.05443s (16.5496GiB/s) | 20260505 09:43:33.635 | INFO | Starting a round of Baz IO | 20260505 09:43:36.912 | INFO | Populating RTMetrics report | 20260505 09:43:36.913 | INFO | RTMetrics report populated | 20260505 09:43:36.913 | INFO | Opening tmp RTMetrics json file | 20260505 09:43:36.926 | INFO | Serializing RTMetrics to json | 20260505 09:43:36.928 | INFO | Serialization done, swapping tmp file now... | 20260505 09:43:36.929 | INFO | RTMetrics done | 20260505 09:43:37.067 | INFO | Finished a round of Baz IO: Wrote 2.436 GiB in 3.431 seconds | 20260505 09:43:37.067 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.75% / 0.01% / 74.24% | 20260505 09:43:37.067 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.82692% / 1.19209e-05% / 9.93411e-05% | 20260505 09:43:37.067 | INFO | Baz IO overall: | 20260505 09:43:37.067 | INFO | BytesWritten : 1290150163582 | 20260505 09:43:37.067 | INFO | NumEvents : 614068979701 | 20260505 09:43:38.642 | INFO | Analyzing chunk frames = [2224128,2224640) | 20260505 09:43:38.642 | INFO | MemInfo: { | 20260505 09:43:38.642 | INFO | "gpu" : " 17 GB", | 20260505 09:43:38.642 | INFO | "huge" : "", | 20260505 09:43:38.643 | INFO | "huge_shared" : " 27 GB", | 20260505 09:43:38.643 | INFO | "malloc" : " 14 GB", | 20260505 09:43:38.643 | INFO | "pinned" : " 29 GB" | 20260505 09:43:38.643 | INFO | } | 20260505 09:43:38.643 | INFO | Perf report for the last 5.13897 seconds: | 20260505 09:43:38.643 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:43:38.643 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:38.643 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:38.643 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:38.643 | INFO | ANALYSIS : 65.2%, 2.9, 99.8ms, 0 | 20260505 09:43:38.643 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:38.643 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260505 09:43:38.643 | INFO | PRE_HQ : 62.7%, 1.8, 60.1ms, 0 | 20260505 09:43:38.643 | INFO | BAZWRITER : 66.1%, 4.2, 141.7ms, 0 | 20260505 09:43:38.643 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 09:43:38.643 | INFO | Uploaded 36GiB to GPU in 2.18364s (16.4862GiB/s) | 20260505 09:43:38.643 | INFO | Downloaded 38GiB to GPU in 2.33435s (16.2786GiB/s) | 20260505 09:43:43.782 | INFO | Analyzing chunk frames = [2224640,2225152) | 20260505 09:43:43.782 | INFO | MemInfo: { | 20260505 09:43:43.782 | INFO | "gpu" : " 17 GB", | 20260505 09:43:43.782 | INFO | "huge" : "", | 20260505 09:43:43.782 | INFO | "huge_shared" : " 27 GB", | 20260505 09:43:43.782 | INFO | "malloc" : " 14 GB", | 20260505 09:43:43.782 | INFO | "pinned" : " 29 GB" | 20260505 09:43:43.782 | INFO | } | 20260505 09:43:43.782 | INFO | Perf report for the last 5.13984 seconds: | 20260505 09:43:43.782 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:43:43.782 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:43.782 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:43.782 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:43.782 | INFO | ANALYSIS : 64.6%, 3.0, 100.8ms, 0 | 20260505 09:43:43.783 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:43.783 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:43.783 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 09:43:43.783 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:43.783 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:43.783 | INFO | Uploaded 36GiB to GPU in 2.24109s (16.0636GiB/s) | 20260505 09:43:43.783 | INFO | Downloaded 34GiB to GPU in 2.10492s (16.1526GiB/s) | 20260505 09:43:48.924 | INFO | Analyzing chunk frames = [2225152,2225664) | 20260505 09:43:48.924 | INFO | MemInfo: { | 20260505 09:43:48.924 | INFO | "gpu" : " 17 GB", | 20260505 09:43:48.924 | INFO | "huge" : "", | 20260505 09:43:48.924 | INFO | "huge_shared" : " 27 GB", | 20260505 09:43:48.924 | INFO | "malloc" : " 14 GB", | 20260505 09:43:48.924 | INFO | "pinned" : " 29 GB" | 20260505 09:43:48.924 | INFO | } | 20260505 09:43:48.924 | INFO | Perf report for the last 5.14133 seconds: | 20260505 09:43:48.924 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:43:48.924 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:48.924 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:48.924 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:48.924 | INFO | ANALYSIS : 63.1%, 3.0, 98.2ms, 0 | 20260505 09:43:48.924 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:48.924 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:48.924 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 09:43:48.924 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:48.924 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:48.924 | INFO | Uploaded 36GiB to GPU in 2.15325s (16.7189GiB/s) | 20260505 09:43:48.924 | INFO | Downloaded 34GiB to GPU in 2.05956s (16.5084GiB/s) | 20260505 09:43:54.062 | INFO | Analyzing chunk frames = [2225664,2226176) | 20260505 09:43:54.062 | INFO | MemInfo: { | 20260505 09:43:54.062 | INFO | "gpu" : " 17 GB", | 20260505 09:43:54.062 | INFO | "huge" : "", | 20260505 09:43:54.062 | INFO | "huge_shared" : " 27 GB", | 20260505 09:43:54.062 | INFO | "malloc" : " 14 GB", | 20260505 09:43:54.062 | INFO | "pinned" : " 29 GB" | 20260505 09:43:54.062 | INFO | } | 20260505 09:43:54.062 | INFO | Perf report for the last 5.13838 seconds: | 20260505 09:43:54.062 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:43:54.062 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:54.062 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:54.062 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:54.062 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 09:43:54.062 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:54.062 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:54.062 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 09:43:54.062 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:54.062 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:54.063 | INFO | Uploaded 36GiB to GPU in 2.16853s (16.6011GiB/s) | 20260505 09:43:54.063 | INFO | Downloaded 34GiB to GPU in 2.07438s (16.3905GiB/s) | 20260505 09:43:59.200 | INFO | Analyzing chunk frames = [2226176,2226688) | 20260505 09:43:59.200 | INFO | MemInfo: { | 20260505 09:43:59.200 | INFO | "gpu" : " 17 GB", | 20260505 09:43:59.200 | INFO | "huge" : "", | 20260505 09:43:59.200 | INFO | "huge_shared" : " 27 GB", | 20260505 09:43:59.200 | INFO | "malloc" : " 14 GB", | 20260505 09:43:59.200 | INFO | "pinned" : " 29 GB" | 20260505 09:43:59.201 | INFO | } | 20260505 09:43:59.201 | INFO | Perf report for the last 5.13838 seconds: | 20260505 09:43:59.201 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:43:59.201 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:59.201 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:59.201 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:59.201 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 09:43:59.201 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:59.201 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:59.201 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 09:43:59.201 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:59.201 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:43:59.201 | INFO | Uploaded 36GiB to GPU in 2.23766s (16.0882GiB/s) | 20260505 09:43:59.201 | INFO | Downloaded 34GiB to GPU in 2.10339s (16.1643GiB/s) | 20260505 09:44:04.340 | INFO | Analyzing chunk frames = [2226688,2227200) | 20260505 09:44:04.340 | INFO | MemInfo: { | 20260505 09:44:04.340 | INFO | "gpu" : " 17 GB", | 20260505 09:44:04.340 | INFO | "huge" : "", | 20260505 09:44:04.340 | INFO | "huge_shared" : " 27 GB", | 20260505 09:44:04.340 | INFO | "malloc" : " 14 GB", | 20260505 09:44:04.340 | INFO | "pinned" : " 29 GB" | 20260505 09:44:04.340 | INFO | } | 20260505 09:44:04.340 | INFO | Perf report for the last 5.13962 seconds: | 20260505 09:44:04.340 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:44:04.340 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:04.340 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:04.340 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:04.340 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 09:44:04.340 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:04.340 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:04.340 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 09:44:04.340 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:04.340 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:04.341 | INFO | Uploaded 36GiB to GPU in 2.16995s (16.5902GiB/s) | 20260505 09:44:04.341 | INFO | Downloaded 34GiB to GPU in 2.05702s (16.5288GiB/s) | 20260505 09:44:09.479 | INFO | Analyzing chunk frames = [2227200,2227712) | 20260505 09:44:09.479 | INFO | MemInfo: { | 20260505 09:44:09.479 | INFO | "gpu" : " 17 GB", | 20260505 09:44:09.479 | INFO | "huge" : "", | 20260505 09:44:09.479 | INFO | "huge_shared" : " 27 GB", | 20260505 09:44:09.479 | INFO | "malloc" : " 14 GB", | 20260505 09:44:09.479 | INFO | "pinned" : " 29 GB" | 20260505 09:44:09.479 | INFO | } | 20260505 09:44:09.479 | INFO | Perf report for the last 5.13901 seconds: | 20260505 09:44:09.479 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:44:09.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:09.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:09.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:09.479 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 09:44:09.480 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:09.480 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:09.480 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 09:44:09.480 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:09.480 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:09.480 | INFO | Uploaded 36GiB to GPU in 2.19833s (16.3761GiB/s) | 20260505 09:44:09.480 | INFO | Downloaded 34GiB to GPU in 2.12185s (16.0238GiB/s) | 20260505 09:44:14.619 | INFO | Analyzing chunk frames = [2227712,2228224) | 20260505 09:44:14.619 | INFO | MemInfo: { | 20260505 09:44:14.619 | INFO | "gpu" : " 17 GB", | 20260505 09:44:14.619 | INFO | "huge" : "", | 20260505 09:44:14.619 | INFO | "huge_shared" : " 27 GB", | 20260505 09:44:14.619 | INFO | "malloc" : " 14 GB", | 20260505 09:44:14.619 | INFO | "pinned" : " 29 GB" | 20260505 09:44:14.619 | INFO | } | 20260505 09:44:14.619 | INFO | Perf report for the last 5.13995 seconds: | 20260505 09:44:14.619 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:44:14.619 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:14.619 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:14.619 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:14.619 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 09:44:14.619 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:14.619 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:14.619 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:44:14.619 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:14.619 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:14.620 | INFO | Uploaded 36GiB to GPU in 2.17928s (16.5192GiB/s) | 20260505 09:44:14.620 | INFO | Downloaded 34GiB to GPU in 2.04872s (16.5957GiB/s) | 20260505 09:44:14.620 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 09:44:14.620 | INFO | Total mem usage: 15.3 GiB | 20260505 09:44:14.620 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 09:44:14.620 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.42%) | 20260505 09:44:14.620 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 09:44:14.620 | INFO | High Water Breakdown: | 20260505 09:44:14.620 | INFO | 61% : 8.79 GiB ( 94% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 09:44:14.620 | INFO | 38.8% : 5.59 GiB ( 84.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 09:44:14.620 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 09:44:14.620 | INFO | ------------------------------------------ | 20260505 09:44:14.620 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:44:14.620 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 09:44:14.620 | INFO | Total mem usage: 26.5 GiB | 20260505 09:44:14.620 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 09:44:14.620 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 09:44:14.620 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 09:44:14.620 | INFO | High Water Breakdown: | 20260505 09:44:14.620 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:44:14.620 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:44:14.620 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:44:14.620 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:44:14.620 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:44:14.620 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:44:14.620 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 09:44:14.620 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 09:44:14.620 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:44:14.620 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 09:44:14.620 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 09:44:14.620 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 09:44:14.620 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 09:44:14.620 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 09:44:14.620 | INFO | ------------------------------------------ | 20260505 09:44:14.620 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:44:14.620 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 09:44:14.620 | INFO | Total mem usage: 25.6 GiB | 20260505 09:44:14.620 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 09:44:14.620 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 09:44:14.620 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 09:44:14.620 | INFO | High Water Breakdown: | 20260505 09:44:14.620 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 09:44:14.620 | INFO | ------------------------------------------ | 20260505 09:44:14.620 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:44:14.620 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 09:44:14.620 | INFO | Total mem usage: 15.7 GiB | 20260505 09:44:14.620 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 09:44:14.620 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 09:44:14.620 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 09:44:14.620 | INFO | High Water Breakdown: | 20260505 09:44:14.620 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 09:44:14.620 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 09:44:14.621 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 09:44:14.621 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 09:44:14.621 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 09:44:14.621 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 09:44:14.621 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 09:44:14.621 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 09:44:14.621 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 09:44:14.621 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:44:14.621 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 09:44:14.621 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:44:14.621 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 09:44:14.621 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 09:44:14.621 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 09:44:14.621 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 09:44:14.621 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:44:14.621 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 09:44:14.621 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:44:14.621 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:44:14.621 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:44:14.621 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 09:44:14.621 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 09:44:14.621 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:44:14.621 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 09:44:14.621 | INFO | ------------------------------------------ | 20260505 09:44:14.621 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:44:14.621 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 09:44:14.621 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:44:14.621 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:44:14.621 | INFO | PerfReport: Full Execution -- 3.25e+01 / 7.70e+01, 1.38e+01 / 2.78e+00 | 20260505 09:44:14.621 | INFO | PerfReport: 0.29%, AggregateStats -- 9.32e-02 / 1.04e-01, 1.06e-02 / 4.65e-03 | 20260505 09:44:14.621 | INFO | PerfReport: 11.65%, Baseline -- 3.78e+00 / 4.42e+00, 6.11e-01 / 2.92e-01 | 20260505 09:44:14.621 | INFO | PerfReport: 3.30%, Histogram -- 1.07e+00 / 1.47e+00, 2.07e-01 / 1.12e-01 | 20260505 09:44:14.621 | INFO | PerfReport: 15.64%, DME -- 4.06e+01 / 4.52e+01, 4.43e+00 / 2.40e+00 | 20260505 09:44:14.621 | INFO | PerfReport: 56.54%, FrameLabeling -- 1.83e+01 / 2.07e+01, 1.86e+00 / 2.00e+00 | 20260505 09:44:14.621 | INFO | PerfReport: 7.44%, PulseAccumulating -- 2.41e+00 / 3.03e+00, 4.19e-01 / 1.90e-01 | 20260505 09:44:14.621 | INFO | PerfReport: 3.80%, Metrics -- 1.23e+00 / 1.57e+00, 3.55e-01 / 1.88e-01 | 20260505 09:44:14.621 | INFO | PerfReport: 1.35%, PulseSerializing -- 4.38e-01 / 4.87e-01, 8.21e-02 / 2.77e-02 | 20260505 09:44:14.621 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 09:44:14.621 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:44:14.621 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:44:14.621 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.37e+01 / 0.00e+00 | 20260505 09:44:14.621 | INFO | PerfReport: 22.57%, Upload -- 2.29e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260505 09:44:14.621 | INFO | PerfReport: 21.77%, Download -- 2.21e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260505 09:44:14.621 | INFO | PerfReport: 31.99%, Compute -- 3.25e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 09:44:14.621 | INFO | PerfReport: 1.86%, LockUpload -- 1.88e+00 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260505 09:44:14.621 | INFO | PerfReport: 1.10%, LockDownload -- 1.12e+00 / 0.00e+00, 2.73e+00 / 0.00e+00 | 20260505 09:44:14.621 | INFO | PerfReport: 20.70%, LockCompute -- 2.10e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260505 09:44:14.621 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.66e-02 / 0.00e+00 | 20260505 09:44:14.749 | INFO | Starting a round of Baz IO | 20260505 09:44:18.014 | INFO | Populating RTMetrics report | 20260505 09:44:18.014 | INFO | RTMetrics report populated | 20260505 09:44:18.014 | INFO | Opening tmp RTMetrics json file | 20260505 09:44:18.015 | INFO | Serializing RTMetrics to json | 20260505 09:44:18.016 | INFO | Serialization done, swapping tmp file now... | 20260505 09:44:18.017 | INFO | RTMetrics done | 20260505 09:44:18.167 | INFO | Finished a round of Baz IO: Wrote 2.435 GiB in 3.418 seconds | 20260505 09:44:18.167 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.73% / 0.01% / 74.25% | 20260505 09:44:18.168 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.82398% / 3.57628e-05% / 5.16574e-05% | 20260505 09:44:18.168 | INFO | Baz IO overall: | 20260505 09:44:18.168 | INFO | BytesWritten : 1292764301953 | 20260505 09:44:18.168 | INFO | NumEvents : 615316916392 | 20260505 09:44:19.758 | INFO | Analyzing chunk frames = [2228224,2228736) | 20260505 09:44:19.758 | INFO | MemInfo: { | 20260505 09:44:19.758 | INFO | "gpu" : " 17 GB", | 20260505 09:44:19.758 | INFO | "huge" : "", | 20260505 09:44:19.758 | INFO | "huge_shared" : " 27 GB", | 20260505 09:44:19.758 | INFO | "malloc" : " 14 GB", | 20260505 09:44:19.758 | INFO | "pinned" : " 29 GB" | 20260505 09:44:19.758 | INFO | } | 20260505 09:44:19.758 | INFO | Perf report for the last 5.13851 seconds: | 20260505 09:44:19.758 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:44:19.758 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:19.758 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:19.758 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:19.758 | INFO | ANALYSIS : 64.9%, 2.9, 99.1ms, 0 | 20260505 09:44:19.758 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:19.758 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260505 09:44:19.758 | INFO | PRE_HQ : 62.9%, 1.8, 59.8ms, 0 | 20260505 09:44:19.758 | INFO | BAZWRITER : 65.8%, 4.3, 144.0ms, 0 | 20260505 09:44:19.758 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 09:44:19.758 | INFO | Uploaded 36GiB to GPU in 2.26425s (15.8993GiB/s) | 20260505 09:44:19.758 | INFO | Downloaded 38GiB to GPU in 2.37906s (15.9727GiB/s) | 20260505 09:44:24.897 | INFO | Analyzing chunk frames = [2228736,2229248) | 20260505 09:44:24.897 | INFO | MemInfo: { | 20260505 09:44:24.897 | INFO | "gpu" : " 17 GB", | 20260505 09:44:24.897 | INFO | "huge" : "", | 20260505 09:44:24.897 | INFO | "huge_shared" : " 27 GB", | 20260505 09:44:24.897 | INFO | "malloc" : " 14 GB", | 20260505 09:44:24.897 | INFO | "pinned" : " 29 GB" | 20260505 09:44:24.897 | INFO | } | 20260505 09:44:24.898 | INFO | Perf report for the last 5.13989 seconds: | 20260505 09:44:24.898 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:44:24.898 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:24.898 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:24.898 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:24.898 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 09:44:24.898 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:24.898 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:24.898 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 09:44:24.898 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:24.898 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:24.898 | INFO | Uploaded 36GiB to GPU in 2.18187s (16.4996GiB/s) | 20260505 09:44:24.898 | INFO | Downloaded 34GiB to GPU in 2.0974s (16.2105GiB/s) | 20260505 09:44:30.037 | INFO | Analyzing chunk frames = [2229248,2229760) | 20260505 09:44:30.037 | INFO | MemInfo: { | 20260505 09:44:30.037 | INFO | "gpu" : " 17 GB", | 20260505 09:44:30.037 | INFO | "huge" : "", | 20260505 09:44:30.037 | INFO | "huge_shared" : " 27 GB", | 20260505 09:44:30.037 | INFO | "malloc" : " 14 GB", | 20260505 09:44:30.037 | INFO | "pinned" : " 29 GB" | 20260505 09:44:30.037 | INFO | } | 20260505 09:44:30.037 | INFO | Perf report for the last 5.13975 seconds: | 20260505 09:44:30.037 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:44:30.037 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:30.037 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:30.037 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:30.037 | INFO | ANALYSIS : 63.2%, 3.0, 98.4ms, 0 | 20260505 09:44:30.037 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:30.038 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:30.038 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 09:44:30.038 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:30.038 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:30.038 | INFO | Uploaded 36GiB to GPU in 2.15266s (16.7235GiB/s) | 20260505 09:44:30.038 | INFO | Downloaded 34GiB to GPU in 2.06409s (16.4721GiB/s) | 20260505 09:44:35.176 | INFO | Analyzing chunk frames = [2229760,2230272) | 20260505 09:44:35.176 | INFO | MemInfo: { | 20260505 09:44:35.176 | INFO | "gpu" : " 17 GB", | 20260505 09:44:35.176 | INFO | "huge" : "", | 20260505 09:44:35.176 | INFO | "huge_shared" : " 27 GB", | 20260505 09:44:35.176 | INFO | "malloc" : " 14 GB", | 20260505 09:44:35.176 | INFO | "pinned" : " 29 GB" | 20260505 09:44:35.176 | INFO | } | 20260505 09:44:35.176 | INFO | Perf report for the last 5.13875 seconds: | 20260505 09:44:35.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:44:35.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:35.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:35.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:35.176 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 09:44:35.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:35.176 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:35.176 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:44:35.176 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:35.176 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:35.176 | INFO | Uploaded 36GiB to GPU in 2.17758s (16.5321GiB/s) | 20260505 09:44:35.176 | INFO | Downloaded 34GiB to GPU in 2.07552s (16.3814GiB/s) | 20260505 09:44:40.315 | INFO | Analyzing chunk frames = [2230272,2230784) | 20260505 09:44:40.315 | INFO | MemInfo: { | 20260505 09:44:40.315 | INFO | "gpu" : " 17 GB", | 20260505 09:44:40.315 | INFO | "huge" : "", | 20260505 09:44:40.315 | INFO | "huge_shared" : " 27 GB", | 20260505 09:44:40.315 | INFO | "malloc" : " 14 GB", | 20260505 09:44:40.315 | INFO | "pinned" : " 29 GB" | 20260505 09:44:40.315 | INFO | } | 20260505 09:44:40.315 | INFO | Perf report for the last 5.13871 seconds: | 20260505 09:44:40.315 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:44:40.315 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:40.315 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:40.315 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:40.315 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 09:44:40.315 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:40.315 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:40.315 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 09:44:40.315 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:40.315 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:40.315 | INFO | Uploaded 36GiB to GPU in 2.23896s (16.0789GiB/s) | 20260505 09:44:40.315 | INFO | Downloaded 34GiB to GPU in 2.10328s (16.1652GiB/s) | 20260505 09:44:45.455 | INFO | Analyzing chunk frames = [2230784,2231296) | 20260505 09:44:45.455 | INFO | MemInfo: { | 20260505 09:44:45.455 | INFO | "gpu" : " 17 GB", | 20260505 09:44:45.455 | INFO | "huge" : "", | 20260505 09:44:45.455 | INFO | "huge_shared" : " 27 GB", | 20260505 09:44:45.455 | INFO | "malloc" : " 14 GB", | 20260505 09:44:45.455 | INFO | "pinned" : " 29 GB" | 20260505 09:44:45.455 | INFO | } | 20260505 09:44:45.455 | INFO | Perf report for the last 5.14019 seconds: | 20260505 09:44:45.455 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:44:45.455 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:45.455 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:45.455 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:45.455 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 09:44:45.455 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:45.455 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:45.455 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 09:44:45.455 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:45.455 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:45.455 | INFO | Uploaded 36GiB to GPU in 2.2403s (16.0693GiB/s) | 20260505 09:44:45.455 | INFO | Downloaded 34GiB to GPU in 2.09557s (16.2247GiB/s) | 20260505 09:44:50.595 | INFO | Analyzing chunk frames = [2231296,2231808) | 20260505 09:44:50.595 | INFO | MemInfo: { | 20260505 09:44:50.595 | INFO | "gpu" : " 17 GB", | 20260505 09:44:50.595 | INFO | "huge" : "", | 20260505 09:44:50.595 | INFO | "huge_shared" : " 27 GB", | 20260505 09:44:50.595 | INFO | "malloc" : " 14 GB", | 20260505 09:44:50.595 | INFO | "pinned" : " 29 GB" | 20260505 09:44:50.595 | INFO | } | 20260505 09:44:50.595 | INFO | Perf report for the last 5.13975 seconds: | 20260505 09:44:50.595 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:44:50.595 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:50.595 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:50.595 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:50.595 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 09:44:50.595 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:50.595 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:50.595 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 09:44:50.595 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:50.595 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:50.595 | INFO | Uploaded 36GiB to GPU in 2.19497s (16.4012GiB/s) | 20260505 09:44:50.595 | INFO | Downloaded 34GiB to GPU in 2.13969s (15.8901GiB/s) | 20260505 09:44:55.734 | INFO | Analyzing chunk frames = [2231808,2232320) | 20260505 09:44:55.734 | INFO | MemInfo: { | 20260505 09:44:55.734 | INFO | "gpu" : " 17 GB", | 20260505 09:44:55.734 | INFO | "huge" : "", | 20260505 09:44:55.734 | INFO | "huge_shared" : " 27 GB", | 20260505 09:44:55.734 | INFO | "malloc" : " 14 GB", | 20260505 09:44:55.734 | INFO | "pinned" : " 29 GB" | 20260505 09:44:55.734 | INFO | } | 20260505 09:44:55.734 | INFO | Perf report for the last 5.13925 seconds: | 20260505 09:44:55.734 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:44:55.734 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:55.734 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:55.734 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:55.734 | INFO | ANALYSIS : 64.7%, 3.0, 101.1ms, 0 | 20260505 09:44:55.734 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:55.734 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:55.734 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 09:44:55.734 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:55.734 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:44:55.734 | INFO | Uploaded 36GiB to GPU in 2.24909s (16.0065GiB/s) | 20260505 09:44:55.734 | INFO | Downloaded 34GiB to GPU in 2.07994s (16.3466GiB/s) | 20260505 09:44:55.866 | INFO | Starting a round of Baz IO | 20260505 09:44:59.139 | INFO | Populating RTMetrics report | 20260505 09:44:59.139 | INFO | RTMetrics report populated | 20260505 09:44:59.139 | INFO | Opening tmp RTMetrics json file | 20260505 09:44:59.159 | INFO | Serializing RTMetrics to json | 20260505 09:44:59.187 | INFO | Serialization done, swapping tmp file now... | 20260505 09:44:59.188 | INFO | RTMetrics done | 20260505 09:44:59.295 | INFO | Finished a round of Baz IO: Wrote 2.435 GiB in 3.429 seconds | 20260505 09:44:59.295 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.72% / 0.01% / 74.27% | 20260505 09:44:59.295 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.81825% / 1.58946e-05% / 9.13938e-05% | 20260505 09:44:59.295 | INFO | Baz IO overall: | 20260505 09:44:59.295 | INFO | BytesWritten : 1295378558386 | 20260505 09:44:59.295 | INFO | NumEvents : 616565125934 | 20260505 09:45:00.873 | INFO | Analyzing chunk frames = [2232320,2232832) | 20260505 09:45:00.873 | INFO | MemInfo: { | 20260505 09:45:00.873 | INFO | "gpu" : " 17 GB", | 20260505 09:45:00.873 | INFO | "huge" : "", | 20260505 09:45:00.873 | INFO | "huge_shared" : " 27 GB", | 20260505 09:45:00.873 | INFO | "malloc" : " 14 GB", | 20260505 09:45:00.873 | INFO | "pinned" : " 29 GB" | 20260505 09:45:00.873 | INFO | } | 20260505 09:45:00.873 | INFO | Perf report for the last 5.13907 seconds: | 20260505 09:45:00.873 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:45:00.873 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:00.873 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:00.873 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:00.873 | INFO | ANALYSIS : 65.1%, 2.9, 99.1ms, 0 | 20260505 09:45:00.873 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:00.873 | INFO | RT_METRICS : 46.9%, 1.0, 24.6ms, 0 | 20260505 09:45:00.873 | INFO | PRE_HQ : 62.9%, 1.9, 60.8ms, 0 | 20260505 09:45:00.873 | INFO | BAZWRITER : 66.0%, 4.2, 141.4ms, 0 | 20260505 09:45:00.873 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260505 09:45:00.873 | INFO | Uploaded 36GiB to GPU in 2.28773s (15.7361GiB/s) | 20260505 09:45:00.873 | INFO | Downloaded 38GiB to GPU in 2.36452s (16.0709GiB/s) | 20260505 09:45:06.012 | INFO | Analyzing chunk frames = [2232832,2233344) | 20260505 09:45:06.012 | INFO | MemInfo: { | 20260505 09:45:06.012 | INFO | "gpu" : " 17 GB", | 20260505 09:45:06.012 | INFO | "huge" : "", | 20260505 09:45:06.012 | INFO | "huge_shared" : " 27 GB", | 20260505 09:45:06.012 | INFO | "malloc" : " 14 GB", | 20260505 09:45:06.012 | INFO | "pinned" : " 29 GB" | 20260505 09:45:06.012 | INFO | } | 20260505 09:45:06.012 | INFO | Perf report for the last 5.13898 seconds: | 20260505 09:45:06.012 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:45:06.012 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:06.012 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:06.012 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:06.012 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 09:45:06.012 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:06.012 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:06.012 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 09:45:06.012 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:06.012 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:06.012 | INFO | Uploaded 36GiB to GPU in 2.17794s (16.5294GiB/s) | 20260505 09:45:06.012 | INFO | Downloaded 34GiB to GPU in 2.09897s (16.1984GiB/s) | 20260505 09:45:11.151 | INFO | Analyzing chunk frames = [2233344,2233856) | 20260505 09:45:11.151 | INFO | MemInfo: { | 20260505 09:45:11.151 | INFO | "gpu" : " 17 GB", | 20260505 09:45:11.151 | INFO | "huge" : "", | 20260505 09:45:11.151 | INFO | "huge_shared" : " 27 GB", | 20260505 09:45:11.151 | INFO | "malloc" : " 14 GB", | 20260505 09:45:11.151 | INFO | "pinned" : " 29 GB" | 20260505 09:45:11.151 | INFO | } | 20260505 09:45:11.151 | INFO | Perf report for the last 5.13913 seconds: | 20260505 09:45:11.151 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:45:11.151 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:11.151 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:11.151 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:11.151 | INFO | ANALYSIS : 63.2%, 3.0, 98.3ms, 0 | 20260505 09:45:11.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:11.151 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:11.151 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 09:45:11.151 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:11.151 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:11.152 | INFO | Uploaded 36GiB to GPU in 2.14436s (16.7882GiB/s) | 20260505 09:45:11.152 | INFO | Downloaded 34GiB to GPU in 2.06498s (16.4651GiB/s) | 20260505 09:45:16.291 | INFO | Analyzing chunk frames = [2233856,2234368) | 20260505 09:45:16.291 | INFO | MemInfo: { | 20260505 09:45:16.291 | INFO | "gpu" : " 17 GB", | 20260505 09:45:16.291 | INFO | "huge" : "", | 20260505 09:45:16.291 | INFO | "huge_shared" : " 27 GB", | 20260505 09:45:16.291 | INFO | "malloc" : " 14 GB", | 20260505 09:45:16.291 | INFO | "pinned" : " 29 GB" | 20260505 09:45:16.291 | INFO | } | 20260505 09:45:16.291 | INFO | Perf report for the last 5.13943 seconds: | 20260505 09:45:16.291 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:45:16.291 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:16.291 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:16.291 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:16.291 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 09:45:16.291 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:16.291 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:16.291 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 09:45:16.291 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:16.291 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:16.291 | INFO | Uploaded 36GiB to GPU in 2.16087s (16.6599GiB/s) | 20260505 09:45:16.291 | INFO | Downloaded 34GiB to GPU in 2.07074s (16.4192GiB/s) | 20260505 09:45:21.430 | INFO | Analyzing chunk frames = [2234368,2234880) | 20260505 09:45:21.430 | INFO | MemInfo: { | 20260505 09:45:21.430 | INFO | "gpu" : " 17 GB", | 20260505 09:45:21.430 | INFO | "huge" : "", | 20260505 09:45:21.430 | INFO | "huge_shared" : " 27 GB", | 20260505 09:45:21.430 | INFO | "malloc" : " 14 GB", | 20260505 09:45:21.430 | INFO | "pinned" : " 29 GB" | 20260505 09:45:21.430 | INFO | } | 20260505 09:45:21.430 | INFO | Perf report for the last 5.13912 seconds: | 20260505 09:45:21.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:45:21.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:21.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:21.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:21.430 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 09:45:21.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:21.430 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:21.430 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 09:45:21.430 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:21.430 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:21.430 | INFO | Uploaded 36GiB to GPU in 2.23768s (16.0881GiB/s) | 20260505 09:45:21.430 | INFO | Downloaded 34GiB to GPU in 2.10029s (16.1883GiB/s) | 20260505 09:45:26.571 | INFO | Analyzing chunk frames = [2234880,2235392) | 20260505 09:45:26.571 | INFO | MemInfo: { | 20260505 09:45:26.571 | INFO | "gpu" : " 17 GB", | 20260505 09:45:26.571 | INFO | "huge" : "", | 20260505 09:45:26.571 | INFO | "huge_shared" : " 27 GB", | 20260505 09:45:26.571 | INFO | "malloc" : " 14 GB", | 20260505 09:45:26.571 | INFO | "pinned" : " 29 GB" | 20260505 09:45:26.571 | INFO | } | 20260505 09:45:26.571 | INFO | Perf report for the last 5.14091 seconds: | 20260505 09:45:26.571 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:45:26.571 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:26.571 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:26.571 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:26.571 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 09:45:26.571 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:26.571 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:26.571 | INFO | PRE_HQ : 29.7%, 1.0, 15.5ms, 0 | 20260505 09:45:26.571 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:26.571 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:26.571 | INFO | Uploaded 36GiB to GPU in 2.24008s (16.0708GiB/s) | 20260505 09:45:26.571 | INFO | Downloaded 34GiB to GPU in 2.09934s (16.1956GiB/s) | 20260505 09:45:31.709 | INFO | Analyzing chunk frames = [2235392,2235904) | 20260505 09:45:31.709 | INFO | MemInfo: { | 20260505 09:45:31.709 | INFO | "gpu" : " 17 GB", | 20260505 09:45:31.709 | INFO | "huge" : "", | 20260505 09:45:31.709 | INFO | "huge_shared" : " 27 GB", | 20260505 09:45:31.709 | INFO | "malloc" : " 14 GB", | 20260505 09:45:31.709 | INFO | "pinned" : " 29 GB" | 20260505 09:45:31.709 | INFO | } | 20260505 09:45:31.709 | INFO | Perf report for the last 5.13857 seconds: | 20260505 09:45:31.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:45:31.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:31.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:31.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:31.709 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 09:45:31.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:31.709 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:31.709 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 09:45:31.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:31.709 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:31.710 | INFO | Uploaded 36GiB to GPU in 2.20404s (16.3337GiB/s) | 20260505 09:45:31.710 | INFO | Downloaded 34GiB to GPU in 2.12291s (16.0157GiB/s) | 20260505 09:45:36.848 | INFO | Analyzing chunk frames = [2235904,2236416) | 20260505 09:45:36.848 | INFO | MemInfo: { | 20260505 09:45:36.848 | INFO | "gpu" : " 17 GB", | 20260505 09:45:36.848 | INFO | "huge" : "", | 20260505 09:45:36.848 | INFO | "huge_shared" : " 27 GB", | 20260505 09:45:36.848 | INFO | "malloc" : " 14 GB", | 20260505 09:45:36.848 | INFO | "pinned" : " 29 GB" | 20260505 09:45:36.848 | INFO | } | 20260505 09:45:36.848 | INFO | Perf report for the last 5.13866 seconds: | 20260505 09:45:36.848 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:45:36.848 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:36.848 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:36.848 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:36.848 | INFO | ANALYSIS : 64.6%, 3.0, 100.9ms, 0 | 20260505 09:45:36.848 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:36.848 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:36.848 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 09:45:36.848 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:36.848 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:36.848 | INFO | Uploaded 36GiB to GPU in 2.24964s (16.0025GiB/s) | 20260505 09:45:36.848 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 09:45:36.848 | INFO | Total mem usage: 15.3 GiB | 20260505 09:45:36.848 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 09:45:36.848 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -5.77%) | 20260505 09:45:36.848 | INFO | Downloaded 34GiB to GPU in 2.08221s (16.3288GiB/s) | 20260505 09:45:36.848 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 09:45:36.848 | INFO | High Water Breakdown: | 20260505 09:45:36.848 | INFO | 61.1% : 8.85 GiB ( 94.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 09:45:36.848 | INFO | 38.8% : 5.62 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 09:45:36.848 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 09:45:36.849 | INFO | ------------------------------------------ | 20260505 09:45:36.849 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:45:36.849 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 09:45:36.849 | INFO | Total mem usage: 26.5 GiB | 20260505 09:45:36.849 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 09:45:36.849 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 09:45:36.849 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 09:45:36.849 | INFO | High Water Breakdown: | 20260505 09:45:36.849 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:45:36.849 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:45:36.849 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:45:36.849 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:45:36.849 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:45:36.849 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:45:36.849 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 09:45:36.849 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 09:45:36.849 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:45:36.849 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 09:45:36.849 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 09:45:36.849 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 09:45:36.849 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 09:45:36.849 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 09:45:36.849 | INFO | ------------------------------------------ | 20260505 09:45:36.849 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:45:36.849 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 09:45:36.849 | INFO | Total mem usage: 25.6 GiB | 20260505 09:45:36.849 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 09:45:36.849 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 09:45:36.849 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 09:45:36.849 | INFO | High Water Breakdown: | 20260505 09:45:36.849 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 09:45:36.849 | INFO | ------------------------------------------ | 20260505 09:45:36.849 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:45:36.849 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 09:45:36.849 | INFO | Total mem usage: 15.7 GiB | 20260505 09:45:36.849 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 09:45:36.849 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 09:45:36.849 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 09:45:36.849 | INFO | High Water Breakdown: | 20260505 09:45:36.849 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 09:45:36.849 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 09:45:36.849 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 09:45:36.849 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 09:45:36.849 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 09:45:36.849 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 09:45:36.849 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 09:45:36.849 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 09:45:36.849 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 09:45:36.849 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:45:36.849 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 09:45:36.849 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:45:36.849 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 09:45:36.849 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 09:45:36.849 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 09:45:36.849 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 09:45:36.849 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:45:36.849 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 09:45:36.849 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:45:36.849 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:45:36.849 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:45:36.849 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 09:45:36.849 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 09:45:36.849 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:45:36.849 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 09:45:36.849 | INFO | ------------------------------------------ | 20260505 09:45:36.849 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:45:36.849 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 09:45:36.849 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:45:36.849 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:45:36.849 | INFO | PerfReport: Full Execution -- 3.25e+01 / 7.76e+01, 1.39e+01 / 2.09e+00 | 20260505 09:45:36.849 | INFO | PerfReport: 0.29%, AggregateStats -- 9.31e-02 / 1.03e-01, 1.04e-02 / 3.20e-03 | 20260505 09:45:36.849 | INFO | PerfReport: 11.63%, Baseline -- 3.78e+00 / 4.44e+00, 6.08e-01 / 2.38e-01 | 20260505 09:45:36.849 | INFO | PerfReport: 3.31%, Histogram -- 1.08e+00 / 1.48e+00, 2.07e-01 / 9.19e-02 | 20260505 09:45:36.849 | INFO | PerfReport: 15.62%, DME -- 4.06e+01 / 4.50e+01, 4.55e+00 / 2.20e+00 | 20260505 09:45:36.849 | INFO | PerfReport: 56.69%, FrameLabeling -- 1.84e+01 / 2.15e+01, 1.91e+00 / 2.42e+00 | 20260505 09:45:36.849 | INFO | PerfReport: 7.32%, PulseAccumulating -- 2.38e+00 / 2.98e+00, 3.98e-01 / 1.81e-01 | 20260505 09:45:36.849 | INFO | PerfReport: 3.80%, Metrics -- 1.23e+00 / 1.52e+00, 3.59e-01 / 1.96e-01 | 20260505 09:45:36.849 | INFO | PerfReport: 1.34%, PulseSerializing -- 4.37e-01 / 4.79e-01, 8.18e-02 / 2.61e-02 | 20260505 09:45:36.849 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 09:45:36.849 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:45:36.849 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:45:36.849 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.40e+01 / 0.00e+00 | 20260505 09:45:36.849 | INFO | PerfReport: 22.68%, Upload -- 2.30e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260505 09:45:36.849 | INFO | PerfReport: 21.81%, Download -- 2.22e+01 / 0.00e+00, 8.11e+00 / 0.00e+00 | 20260505 09:45:36.849 | INFO | PerfReport: 32.00%, Compute -- 3.25e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 09:45:36.849 | INFO | PerfReport: 1.91%, LockUpload -- 1.94e+00 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260505 09:45:36.849 | INFO | PerfReport: 1.04%, LockDownload -- 1.06e+00 / 0.00e+00, 2.56e+00 / 0.00e+00 | 20260505 09:45:36.849 | INFO | PerfReport: 20.55%, LockCompute -- 2.09e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260505 09:45:36.849 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.08e-02 / 0.00e+00 | 20260505 09:45:36.980 | INFO | Starting a round of Baz IO | 20260505 09:45:40.261 | INFO | Populating RTMetrics report | 20260505 09:45:40.262 | INFO | RTMetrics report populated | 20260505 09:45:40.262 | INFO | Opening tmp RTMetrics json file | 20260505 09:45:40.291 | INFO | Serializing RTMetrics to json | 20260505 09:45:40.324 | INFO | Serialization done, swapping tmp file now... | 20260505 09:45:40.326 | INFO | RTMetrics done | 20260505 09:45:40.396 | INFO | Finished a round of Baz IO: Wrote 2.432 GiB in 3.415 seconds | 20260505 09:45:40.396 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.71% / 0.01% / 74.28% | 20260505 09:45:40.396 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.81154% / 2.78155e-05% / 7.94729e-05% | 20260505 09:45:40.396 | INFO | Baz IO overall: | 20260505 09:45:40.396 | INFO | BytesWritten : 1297990320518 | 20260505 09:45:40.396 | INFO | NumEvents : 617811984261 | 20260505 09:45:41.988 | INFO | Analyzing chunk frames = [2236416,2236928) | 20260505 09:45:41.988 | INFO | MemInfo: { | 20260505 09:45:41.988 | INFO | "gpu" : " 17 GB", | 20260505 09:45:41.988 | INFO | "huge" : "", | 20260505 09:45:41.988 | INFO | "huge_shared" : " 27 GB", | 20260505 09:45:41.988 | INFO | "malloc" : " 14 GB", | 20260505 09:45:41.988 | INFO | "pinned" : " 29 GB" | 20260505 09:45:41.988 | INFO | } | 20260505 09:45:41.988 | INFO | Perf report for the last 5.13955 seconds: | 20260505 09:45:41.988 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:45:41.988 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:41.988 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:41.988 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:41.988 | INFO | ANALYSIS : 65.3%, 2.9, 100.2ms, 0 | 20260505 09:45:41.988 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:41.988 | INFO | RT_METRICS : 46.9%, 1.0, 24.6ms, 0 | 20260505 09:45:41.988 | INFO | PRE_HQ : 62.4%, 1.9, 60.0ms, 0 | 20260505 09:45:41.988 | INFO | BAZWRITER : 65.8%, 4.1, 139.4ms, 0 | 20260505 09:45:41.988 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 09:45:41.988 | INFO | Uploaded 36GiB to GPU in 2.25193s (15.9863GiB/s) | 20260505 09:45:41.988 | INFO | Downloaded 38GiB to GPU in 2.3269s (16.3308GiB/s) | 20260505 09:45:47.127 | INFO | Analyzing chunk frames = [2236928,2237440) | 20260505 09:45:47.127 | INFO | MemInfo: { | 20260505 09:45:47.127 | INFO | "gpu" : " 17 GB", | 20260505 09:45:47.127 | INFO | "huge" : "", | 20260505 09:45:47.127 | INFO | "huge_shared" : " 27 GB", | 20260505 09:45:47.127 | INFO | "malloc" : " 14 GB", | 20260505 09:45:47.127 | INFO | "pinned" : " 29 GB" | 20260505 09:45:47.127 | INFO | } | 20260505 09:45:47.127 | INFO | Perf report for the last 5.13917 seconds: | 20260505 09:45:47.127 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:45:47.127 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:47.127 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:47.127 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:47.127 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 09:45:47.127 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:47.127 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:47.127 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260505 09:45:47.127 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:47.127 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:47.127 | INFO | Uploaded 36GiB to GPU in 2.18711s (16.46GiB/s) | 20260505 09:45:47.127 | INFO | Downloaded 34GiB to GPU in 2.09602s (16.2212GiB/s) | 20260505 09:45:52.266 | INFO | Analyzing chunk frames = [2237440,2237952) | 20260505 09:45:52.266 | INFO | MemInfo: { | 20260505 09:45:52.266 | INFO | "gpu" : " 17 GB", | 20260505 09:45:52.266 | INFO | "huge" : "", | 20260505 09:45:52.266 | INFO | "huge_shared" : " 27 GB", | 20260505 09:45:52.266 | INFO | "malloc" : " 14 GB", | 20260505 09:45:52.266 | INFO | "pinned" : " 29 GB" | 20260505 09:45:52.266 | INFO | } | 20260505 09:45:52.266 | INFO | Perf report for the last 5.13896 seconds: | 20260505 09:45:52.266 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:45:52.266 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:52.266 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:52.266 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:52.266 | INFO | ANALYSIS : 63.2%, 3.0, 98.3ms, 0 | 20260505 09:45:52.266 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:52.266 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:52.266 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 09:45:52.266 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:52.266 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:52.266 | INFO | Uploaded 36GiB to GPU in 2.14502s (16.7831GiB/s) | 20260505 09:45:52.266 | INFO | Downloaded 34GiB to GPU in 2.06047s (16.5011GiB/s) | 20260505 09:45:57.405 | INFO | Analyzing chunk frames = [2237952,2238464) | 20260505 09:45:57.405 | INFO | MemInfo: { | 20260505 09:45:57.405 | INFO | "gpu" : " 17 GB", | 20260505 09:45:57.405 | INFO | "huge" : "", | 20260505 09:45:57.405 | INFO | "huge_shared" : " 27 GB", | 20260505 09:45:57.405 | INFO | "malloc" : " 14 GB", | 20260505 09:45:57.405 | INFO | "pinned" : " 29 GB" | 20260505 09:45:57.405 | INFO | } | 20260505 09:45:57.405 | INFO | Perf report for the last 5.13936 seconds: | 20260505 09:45:57.405 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:45:57.405 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:57.405 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:57.405 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:57.405 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 09:45:57.405 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:57.405 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:57.405 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:45:57.405 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:57.405 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:45:57.405 | INFO | Uploaded 36GiB to GPU in 2.15949s (16.6706GiB/s) | 20260505 09:45:57.405 | INFO | Downloaded 34GiB to GPU in 2.07252s (16.4051GiB/s) | 20260505 09:46:02.546 | INFO | Analyzing chunk frames = [2238464,2238976) | 20260505 09:46:02.546 | INFO | MemInfo: { | 20260505 09:46:02.546 | INFO | "gpu" : " 17 GB", | 20260505 09:46:02.546 | INFO | "huge" : "", | 20260505 09:46:02.546 | INFO | "huge_shared" : " 27 GB", | 20260505 09:46:02.546 | INFO | "malloc" : " 14 GB", | 20260505 09:46:02.546 | INFO | "pinned" : " 29 GB" | 20260505 09:46:02.546 | INFO | } | 20260505 09:46:02.546 | INFO | Perf report for the last 5.1406 seconds: | 20260505 09:46:02.546 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:46:02.546 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:02.546 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:02.546 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:02.546 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 09:46:02.546 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:02.546 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:02.546 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:46:02.546 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:02.546 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:02.546 | INFO | Uploaded 36GiB to GPU in 2.23861s (16.0814GiB/s) | 20260505 09:46:02.546 | INFO | Downloaded 34GiB to GPU in 2.10789s (16.1299GiB/s) | 20260505 09:46:07.686 | INFO | Analyzing chunk frames = [2238976,2239488) | 20260505 09:46:07.686 | INFO | MemInfo: { | 20260505 09:46:07.686 | INFO | "gpu" : " 17 GB", | 20260505 09:46:07.686 | INFO | "huge" : "", | 20260505 09:46:07.686 | INFO | "huge_shared" : " 27 GB", | 20260505 09:46:07.686 | INFO | "malloc" : " 14 GB", | 20260505 09:46:07.686 | INFO | "pinned" : " 29 GB" | 20260505 09:46:07.686 | INFO | } | 20260505 09:46:07.686 | INFO | Perf report for the last 5.14004 seconds: | 20260505 09:46:07.686 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:46:07.686 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:07.686 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:07.686 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:07.686 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 09:46:07.686 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:07.686 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:07.686 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 09:46:07.686 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:07.686 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:07.686 | INFO | Uploaded 36GiB to GPU in 2.2391s (16.0779GiB/s) | 20260505 09:46:07.686 | INFO | Downloaded 34GiB to GPU in 2.09921s (16.1966GiB/s) | 20260505 09:46:12.824 | INFO | Analyzing chunk frames = [2239488,2240000) | 20260505 09:46:12.824 | INFO | MemInfo: { | 20260505 09:46:12.824 | INFO | "gpu" : " 17 GB", | 20260505 09:46:12.824 | INFO | "huge" : "", | 20260505 09:46:12.824 | INFO | "huge_shared" : " 27 GB", | 20260505 09:46:12.824 | INFO | "malloc" : " 14 GB", | 20260505 09:46:12.824 | INFO | "pinned" : " 29 GB" | 20260505 09:46:12.824 | INFO | } | 20260505 09:46:12.824 | INFO | Perf report for the last 5.1379 seconds: | 20260505 09:46:12.824 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:46:12.824 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:12.824 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:12.824 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:12.824 | INFO | ANALYSIS : 64.0%, 3.0, 99.8ms, 0 | 20260505 09:46:12.824 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:12.824 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:12.824 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260505 09:46:12.824 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:12.824 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:12.824 | INFO | Uploaded 36GiB to GPU in 2.21216s (16.2737GiB/s) | 20260505 09:46:12.824 | INFO | Downloaded 34GiB to GPU in 2.10569s (16.1467GiB/s) | 20260505 09:46:17.962 | INFO | Analyzing chunk frames = [2240000,2240512) | 20260505 09:46:17.962 | INFO | MemInfo: { | 20260505 09:46:17.962 | INFO | "gpu" : " 17 GB", | 20260505 09:46:17.962 | INFO | "huge" : "", | 20260505 09:46:17.962 | INFO | "huge_shared" : " 27 GB", | 20260505 09:46:17.962 | INFO | "malloc" : " 14 GB", | 20260505 09:46:17.962 | INFO | "pinned" : " 29 GB" | 20260505 09:46:17.962 | INFO | } | 20260505 09:46:17.962 | INFO | Perf report for the last 5.13866 seconds: | 20260505 09:46:17.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:46:17.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:17.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:17.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:17.962 | INFO | ANALYSIS : 64.7%, 3.0, 101.0ms, 0 | 20260505 09:46:17.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:17.963 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:17.963 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 09:46:17.963 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:17.963 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:17.963 | INFO | Uploaded 36GiB to GPU in 2.25405s (15.9712GiB/s) | 20260505 09:46:17.963 | INFO | Downloaded 34GiB to GPU in 2.08336s (16.3198GiB/s) | 20260505 09:46:18.097 | INFO | Starting a round of Baz IO | 20260505 09:46:21.378 | INFO | Populating RTMetrics report | 20260505 09:46:21.379 | INFO | RTMetrics report populated | 20260505 09:46:21.379 | INFO | Opening tmp RTMetrics json file | 20260505 09:46:21.390 | INFO | Serializing RTMetrics to json | 20260505 09:46:21.392 | INFO | Serialization done, swapping tmp file now... | 20260505 09:46:21.394 | INFO | RTMetrics done | 20260505 09:46:21.524 | INFO | Finished a round of Baz IO: Wrote 2.433 GiB in 3.427 seconds | 20260505 09:46:21.524 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.69% / 0.01% / 74.29% | 20260505 09:46:21.524 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.80767% / 2.38419e-05% / 4.37101e-05% | 20260505 09:46:21.524 | INFO | Baz IO overall: | 20260505 09:46:21.525 | INFO | BytesWritten : 1300602496295 | 20260505 09:46:21.525 | INFO | NumEvents : 619059784335 | 20260505 09:46:23.102 | INFO | Analyzing chunk frames = [2240512,2241024) | 20260505 09:46:23.102 | INFO | MemInfo: { | 20260505 09:46:23.102 | INFO | "gpu" : " 17 GB", | 20260505 09:46:23.102 | INFO | "huge" : "", | 20260505 09:46:23.102 | INFO | "huge_shared" : " 27 GB", | 20260505 09:46:23.102 | INFO | "malloc" : " 14 GB", | 20260505 09:46:23.102 | INFO | "pinned" : " 29 GB" | 20260505 09:46:23.102 | INFO | } | 20260505 09:46:23.102 | INFO | Perf report for the last 5.13954 seconds: | 20260505 09:46:23.102 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:46:23.102 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:23.102 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:23.102 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:23.102 | INFO | ANALYSIS : 65.3%, 2.9, 100.0ms, 0 | 20260505 09:46:23.102 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:23.102 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 09:46:23.102 | INFO | PRE_HQ : 62.4%, 1.9, 60.4ms, 0 | 20260505 09:46:23.102 | INFO | BAZWRITER : 66.0%, 4.2, 142.9ms, 0 | 20260505 09:46:23.102 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 09:46:23.102 | INFO | Uploaded 36GiB to GPU in 2.22457s (16.1829GiB/s) | 20260505 09:46:23.102 | INFO | Downloaded 38GiB to GPU in 2.35894s (16.1089GiB/s) | 20260505 09:46:28.241 | INFO | Analyzing chunk frames = [2241024,2241536) | 20260505 09:46:28.241 | INFO | MemInfo: { | 20260505 09:46:28.241 | INFO | "gpu" : " 17 GB", | 20260505 09:46:28.241 | INFO | "huge" : "", | 20260505 09:46:28.241 | INFO | "huge_shared" : " 27 GB", | 20260505 09:46:28.241 | INFO | "malloc" : " 14 GB", | 20260505 09:46:28.241 | INFO | "pinned" : " 29 GB" | 20260505 09:46:28.241 | INFO | } | 20260505 09:46:28.241 | INFO | Perf report for the last 5.13936 seconds: | 20260505 09:46:28.241 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:46:28.241 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:28.241 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:28.241 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:28.241 | INFO | ANALYSIS : 64.6%, 3.0, 100.8ms, 0 | 20260505 09:46:28.241 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:28.241 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:28.241 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:46:28.241 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:28.241 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:28.242 | INFO | Uploaded 36GiB to GPU in 2.22913s (16.1498GiB/s) | 20260505 09:46:28.242 | INFO | Downloaded 34GiB to GPU in 2.10417s (16.1584GiB/s) | 20260505 09:46:33.381 | INFO | Analyzing chunk frames = [2241536,2242048) | 20260505 09:46:33.381 | INFO | MemInfo: { | 20260505 09:46:33.381 | INFO | "gpu" : " 17 GB", | 20260505 09:46:33.381 | INFO | "huge" : "", | 20260505 09:46:33.381 | INFO | "huge_shared" : " 27 GB", | 20260505 09:46:33.381 | INFO | "malloc" : " 14 GB", | 20260505 09:46:33.381 | INFO | "pinned" : " 29 GB" | 20260505 09:46:33.381 | INFO | } | 20260505 09:46:33.381 | INFO | Perf report for the last 5.1398 seconds: | 20260505 09:46:33.381 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:46:33.381 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:33.381 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:33.381 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:33.381 | INFO | ANALYSIS : 63.2%, 3.0, 98.4ms, 0 | 20260505 09:46:33.381 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:33.381 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:33.381 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260505 09:46:33.381 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:33.381 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:33.381 | INFO | Uploaded 36GiB to GPU in 2.15252s (16.7246GiB/s) | 20260505 09:46:33.381 | INFO | Downloaded 34GiB to GPU in 2.07011s (16.4243GiB/s) | 20260505 09:46:38.520 | INFO | Analyzing chunk frames = [2242048,2242560) | 20260505 09:46:38.520 | INFO | MemInfo: { | 20260505 09:46:38.520 | INFO | "gpu" : " 17 GB", | 20260505 09:46:38.520 | INFO | "huge" : "", | 20260505 09:46:38.520 | INFO | "huge_shared" : " 27 GB", | 20260505 09:46:38.520 | INFO | "malloc" : " 14 GB", | 20260505 09:46:38.520 | INFO | "pinned" : " 29 GB" | 20260505 09:46:38.520 | INFO | } | 20260505 09:46:38.520 | INFO | Perf report for the last 5.13922 seconds: | 20260505 09:46:38.520 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:46:38.520 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:38.520 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:38.520 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:38.520 | INFO | ANALYSIS : 63.3%, 3.0, 98.3ms, 0 | 20260505 09:46:38.520 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:38.520 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:38.521 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 09:46:38.521 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:38.521 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:38.521 | INFO | Uploaded 36GiB to GPU in 2.1619s (16.652GiB/s) | 20260505 09:46:38.521 | INFO | Downloaded 34GiB to GPU in 2.07534s (16.3828GiB/s) | 20260505 09:46:43.660 | INFO | Analyzing chunk frames = [2242560,2243072) | 20260505 09:46:43.660 | INFO | MemInfo: { | 20260505 09:46:43.660 | INFO | "gpu" : " 17 GB", | 20260505 09:46:43.660 | INFO | "huge" : "", | 20260505 09:46:43.660 | INFO | "huge_shared" : " 27 GB", | 20260505 09:46:43.660 | INFO | "malloc" : " 14 GB", | 20260505 09:46:43.660 | INFO | "pinned" : " 29 GB" | 20260505 09:46:43.660 | INFO | } | 20260505 09:46:43.660 | INFO | Perf report for the last 5.13936 seconds: | 20260505 09:46:43.660 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:46:43.660 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:43.660 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:43.660 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:43.660 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 09:46:43.660 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:43.660 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:43.660 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 09:46:43.660 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:43.660 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:43.660 | INFO | Uploaded 36GiB to GPU in 2.23671s (16.0951GiB/s) | 20260505 09:46:43.660 | INFO | Downloaded 34GiB to GPU in 2.10397s (16.1599GiB/s) | 20260505 09:46:48.800 | INFO | Analyzing chunk frames = [2243072,2243584) | 20260505 09:46:48.800 | INFO | MemInfo: { | 20260505 09:46:48.800 | INFO | "gpu" : " 17 GB", | 20260505 09:46:48.800 | INFO | "huge" : "", | 20260505 09:46:48.800 | INFO | "huge_shared" : " 27 GB", | 20260505 09:46:48.800 | INFO | "malloc" : " 14 GB", | 20260505 09:46:48.800 | INFO | "pinned" : " 29 GB" | 20260505 09:46:48.800 | INFO | } | 20260505 09:46:48.800 | INFO | Perf report for the last 5.14009 seconds: | 20260505 09:46:48.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:46:48.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:48.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:48.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:48.800 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 09:46:48.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:48.800 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:48.800 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 09:46:48.800 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:48.800 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:48.800 | INFO | Uploaded 36GiB to GPU in 2.2362s (16.0987GiB/s) | 20260505 09:46:48.800 | INFO | Downloaded 34GiB to GPU in 2.09632s (16.2189GiB/s) | 20260505 09:46:53.938 | INFO | Analyzing chunk frames = [2243584,2244096) | 20260505 09:46:53.938 | INFO | MemInfo: { | 20260505 09:46:53.938 | INFO | "gpu" : " 17 GB", | 20260505 09:46:53.938 | INFO | "huge" : "", | 20260505 09:46:53.938 | INFO | "huge_shared" : " 27 GB", | 20260505 09:46:53.938 | INFO | "malloc" : " 14 GB", | 20260505 09:46:53.938 | INFO | "pinned" : " 29 GB" | 20260505 09:46:53.938 | INFO | } | 20260505 09:46:53.938 | INFO | Perf report for the last 5.1384 seconds: | 20260505 09:46:53.938 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:46:53.938 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:53.938 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:53.938 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:53.938 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 09:46:53.938 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:53.938 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:53.938 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 09:46:53.938 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:53.938 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:53.939 | INFO | Uploaded 36GiB to GPU in 2.1789s (16.5221GiB/s) | 20260505 09:46:53.939 | INFO | Downloaded 34GiB to GPU in 2.13162s (15.9503GiB/s) | 20260505 09:46:59.077 | INFO | Analyzing chunk frames = [2244096,2244608) | 20260505 09:46:59.077 | INFO | MemInfo: { | 20260505 09:46:59.077 | INFO | "gpu" : " 17 GB", | 20260505 09:46:59.077 | INFO | "huge" : "", | 20260505 09:46:59.077 | INFO | "huge_shared" : " 27 GB", | 20260505 09:46:59.077 | INFO | "malloc" : " 14 GB", | 20260505 09:46:59.077 | INFO | "pinned" : " 29 GB" | 20260505 09:46:59.077 | INFO | } | 20260505 09:46:59.078 | INFO | Perf report for the last 5.13926 seconds: | 20260505 09:46:59.078 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:46:59.078 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:59.078 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:59.078 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:59.078 | INFO | ANALYSIS : 64.7%, 3.0, 101.0ms, 0 | 20260505 09:46:59.078 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:59.078 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:59.078 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 09:46:59.078 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:59.078 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:46:59.078 | INFO | Uploaded 36GiB to GPU in 2.24682s (16.0226GiB/s) | 20260505 09:46:59.078 | INFO | Downloaded 34GiB to GPU in 2.08165s (16.3332GiB/s) | 20260505 09:46:59.078 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 09:46:59.078 | INFO | Total mem usage: 15.3 GiB | 20260505 09:46:59.078 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 09:46:59.078 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.54%) | 20260505 09:46:59.078 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 09:46:59.078 | INFO | High Water Breakdown: | 20260505 09:46:59.078 | INFO | 61% : 8.77 GiB ( 93.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 09:46:59.078 | INFO | 38.9% : 5.59 GiB ( 84.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 09:46:59.078 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 09:46:59.078 | INFO | ------------------------------------------ | 20260505 09:46:59.078 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:46:59.078 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 09:46:59.078 | INFO | Total mem usage: 26.5 GiB | 20260505 09:46:59.078 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 09:46:59.078 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 09:46:59.078 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 09:46:59.078 | INFO | High Water Breakdown: | 20260505 09:46:59.078 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:46:59.078 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:46:59.078 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:46:59.078 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:46:59.078 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:46:59.078 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:46:59.078 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 09:46:59.078 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 09:46:59.078 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:46:59.078 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 09:46:59.078 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 09:46:59.078 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 09:46:59.078 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 09:46:59.078 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 09:46:59.078 | INFO | ------------------------------------------ | 20260505 09:46:59.078 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:46:59.078 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 09:46:59.078 | INFO | Total mem usage: 25.6 GiB | 20260505 09:46:59.078 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 09:46:59.078 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 09:46:59.078 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 09:46:59.078 | INFO | High Water Breakdown: | 20260505 09:46:59.078 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 09:46:59.078 | INFO | ------------------------------------------ | 20260505 09:46:59.078 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:46:59.078 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 09:46:59.078 | INFO | Total mem usage: 15.7 GiB | 20260505 09:46:59.078 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 09:46:59.078 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 09:46:59.078 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 09:46:59.078 | INFO | High Water Breakdown: | 20260505 09:46:59.078 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 09:46:59.078 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 09:46:59.078 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 09:46:59.078 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 09:46:59.078 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 09:46:59.078 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 09:46:59.078 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 09:46:59.078 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 09:46:59.078 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 09:46:59.078 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:46:59.078 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 09:46:59.078 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:46:59.078 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 09:46:59.078 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 09:46:59.078 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 09:46:59.078 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 09:46:59.078 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:46:59.078 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 09:46:59.078 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:46:59.078 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:46:59.078 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:46:59.078 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 09:46:59.078 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 09:46:59.078 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:46:59.078 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 09:46:59.078 | INFO | ------------------------------------------ | 20260505 09:46:59.078 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:46:59.079 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 09:46:59.079 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:46:59.079 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:46:59.079 | INFO | PerfReport: Full Execution -- 3.25e+01 / 7.77e+01, 1.39e+01 / 2.30e+00 | 20260505 09:46:59.079 | INFO | PerfReport: 0.29%, AggregateStats -- 9.36e-02 / 1.02e-01, 1.12e-02 / 3.38e-03 | 20260505 09:46:59.079 | INFO | PerfReport: 11.64%, Baseline -- 3.78e+00 / 4.47e+00, 6.14e-01 / 2.49e-01 | 20260505 09:46:59.079 | INFO | PerfReport: 3.32%, Histogram -- 1.08e+00 / 1.48e+00, 2.11e-01 / 1.07e-01 | 20260505 09:46:59.079 | INFO | PerfReport: 15.67%, DME -- 4.07e+01 / 4.53e+01, 4.56e+00 / 1.62e+00 | 20260505 09:46:59.079 | INFO | PerfReport: 56.62%, FrameLabeling -- 1.84e+01 / 2.14e+01, 1.89e+00 / 1.69e+00 | 20260505 09:46:59.080 | INFO | PerfReport: 7.33%, PulseAccumulating -- 2.38e+00 / 2.97e+00, 4.00e-01 / 2.19e-01 | 20260505 09:46:59.080 | INFO | PerfReport: 3.80%, Metrics -- 1.24e+00 / 1.51e+00, 3.61e-01 / 1.41e-01 | 20260505 09:46:59.080 | INFO | PerfReport: 1.35%, PulseSerializing -- 4.38e-01 / 4.81e-01, 8.26e-02 / 2.99e-02 | 20260505 09:46:59.080 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 09:46:59.080 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:46:59.080 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:46:59.080 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.42e+01 / 0.00e+00 | 20260505 09:46:59.080 | INFO | PerfReport: 22.62%, Upload -- 2.30e+01 / 0.00e+00, 8.32e+00 / 0.00e+00 | 20260505 09:46:59.080 | INFO | PerfReport: 21.74%, Download -- 2.21e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260505 09:46:59.080 | INFO | PerfReport: 31.96%, Compute -- 3.25e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 09:46:59.080 | INFO | PerfReport: 1.98%, LockUpload -- 2.01e+00 / 0.00e+00, 7.55e+00 / 0.00e+00 | 20260505 09:46:59.080 | INFO | PerfReport: 1.03%, LockDownload -- 1.05e+00 / 0.00e+00, 2.59e+00 / 0.00e+00 | 20260505 09:46:59.080 | INFO | PerfReport: 20.65%, LockCompute -- 2.10e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260505 09:46:59.080 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.93e-02 / 0.00e+00 | 20260505 09:46:59.209 | INFO | Starting a round of Baz IO | 20260505 09:47:02.470 | INFO | Populating RTMetrics report | 20260505 09:47:02.470 | INFO | RTMetrics report populated | 20260505 09:47:02.470 | INFO | Opening tmp RTMetrics json file | 20260505 09:47:02.516 | INFO | Serializing RTMetrics to json | 20260505 09:47:02.517 | INFO | Serialization done, swapping tmp file now... | 20260505 09:47:02.517 | INFO | RTMetrics done | 20260505 09:47:02.627 | INFO | Finished a round of Baz IO: Wrote 2.432 GiB in 3.418 seconds | 20260505 09:47:02.627 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.68% / 0.01% / 74.31% | 20260505 09:47:02.627 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.79897% / 1.98682e-05% / 6.75519e-05% | 20260505 09:47:02.627 | INFO | Baz IO overall: | 20260505 09:47:02.627 | INFO | BytesWritten : 1303213965399 | 20260505 09:47:02.627 | INFO | NumEvents : 620306876790 | 20260505 09:47:04.217 | INFO | Analyzing chunk frames = [2244608,2245120) | 20260505 09:47:04.217 | INFO | MemInfo: { | 20260505 09:47:04.217 | INFO | "gpu" : " 17 GB", | 20260505 09:47:04.217 | INFO | "huge" : "", | 20260505 09:47:04.217 | INFO | "huge_shared" : " 27 GB", | 20260505 09:47:04.217 | INFO | "malloc" : " 14 GB", | 20260505 09:47:04.217 | INFO | "pinned" : " 29 GB" | 20260505 09:47:04.217 | INFO | } | 20260505 09:47:04.217 | INFO | Perf report for the last 5.13915 seconds: | 20260505 09:47:04.217 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:47:04.217 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:04.217 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:04.217 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:04.217 | INFO | ANALYSIS : 64.9%, 2.9, 99.0ms, 0 | 20260505 09:47:04.217 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:04.217 | INFO | RT_METRICS : 46.5%, 1.0, 24.4ms, 0 | 20260505 09:47:04.217 | INFO | PRE_HQ : 62.7%, 1.8, 59.7ms, 0 | 20260505 09:47:04.217 | INFO | BAZWRITER : 65.8%, 4.2, 142.1ms, 0 | 20260505 09:47:04.217 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260505 09:47:04.217 | INFO | Uploaded 36GiB to GPU in 2.29776s (15.6675GiB/s) | 20260505 09:47:04.217 | INFO | Downloaded 38GiB to GPU in 2.39144s (15.89GiB/s) | 20260505 09:47:09.357 | INFO | Analyzing chunk frames = [2245120,2245632) | 20260505 09:47:09.357 | INFO | MemInfo: { | 20260505 09:47:09.357 | INFO | "gpu" : " 17 GB", | 20260505 09:47:09.357 | INFO | "huge" : "", | 20260505 09:47:09.357 | INFO | "huge_shared" : " 27 GB", | 20260505 09:47:09.357 | INFO | "malloc" : " 14 GB", | 20260505 09:47:09.357 | INFO | "pinned" : " 29 GB" | 20260505 09:47:09.357 | INFO | } | 20260505 09:47:09.357 | INFO | Perf report for the last 5.14028 seconds: | 20260505 09:47:09.357 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:47:09.357 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:09.357 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:09.357 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:09.357 | INFO | ANALYSIS : 64.5%, 3.0, 100.7ms, 0 | 20260505 09:47:09.357 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:09.357 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:09.357 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 09:47:09.357 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:09.357 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:09.357 | INFO | Uploaded 36GiB to GPU in 2.21733s (16.2357GiB/s) | 20260505 09:47:09.357 | INFO | Downloaded 34GiB to GPU in 2.10631s (16.142GiB/s) | 20260505 09:47:14.496 | INFO | Analyzing chunk frames = [2245632,2246144) | 20260505 09:47:14.496 | INFO | MemInfo: { | 20260505 09:47:14.496 | INFO | "gpu" : " 17 GB", | 20260505 09:47:14.496 | INFO | "huge" : "", | 20260505 09:47:14.496 | INFO | "huge_shared" : " 27 GB", | 20260505 09:47:14.496 | INFO | "malloc" : " 14 GB", | 20260505 09:47:14.496 | INFO | "pinned" : " 29 GB" | 20260505 09:47:14.496 | INFO | } | 20260505 09:47:14.496 | INFO | Perf report for the last 5.13868 seconds: | 20260505 09:47:14.496 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:47:14.496 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:14.496 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:14.496 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:14.496 | INFO | ANALYSIS : 63.2%, 3.0, 98.4ms, 0 | 20260505 09:47:14.496 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:14.496 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:14.496 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:47:14.496 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:14.496 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:14.496 | INFO | Uploaded 36GiB to GPU in 2.15303s (16.7206GiB/s) | 20260505 09:47:14.496 | INFO | Downloaded 34GiB to GPU in 2.06949s (16.4292GiB/s) | 20260505 09:47:19.635 | INFO | Analyzing chunk frames = [2246144,2246656) | 20260505 09:47:19.635 | INFO | MemInfo: { | 20260505 09:47:19.635 | INFO | "gpu" : " 17 GB", | 20260505 09:47:19.635 | INFO | "huge" : "", | 20260505 09:47:19.635 | INFO | "huge_shared" : " 27 GB", | 20260505 09:47:19.635 | INFO | "malloc" : " 14 GB", | 20260505 09:47:19.635 | INFO | "pinned" : " 29 GB" | 20260505 09:47:19.635 | INFO | } | 20260505 09:47:19.635 | INFO | Perf report for the last 5.13975 seconds: | 20260505 09:47:19.635 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:47:19.635 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:19.635 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:19.636 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:19.636 | INFO | ANALYSIS : 63.2%, 3.0, 98.3ms, 0 | 20260505 09:47:19.636 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:19.636 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:19.636 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:47:19.636 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:19.636 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:19.636 | INFO | Uploaded 36GiB to GPU in 2.16605s (16.6201GiB/s) | 20260505 09:47:19.636 | INFO | Downloaded 34GiB to GPU in 2.07023s (16.4233GiB/s) | 20260505 09:47:24.775 | INFO | Analyzing chunk frames = [2246656,2247168) | 20260505 09:47:24.775 | INFO | MemInfo: { | 20260505 09:47:24.775 | INFO | "gpu" : " 17 GB", | 20260505 09:47:24.775 | INFO | "huge" : "", | 20260505 09:47:24.775 | INFO | "huge_shared" : " 27 GB", | 20260505 09:47:24.775 | INFO | "malloc" : " 14 GB", | 20260505 09:47:24.775 | INFO | "pinned" : " 29 GB" | 20260505 09:47:24.775 | INFO | } | 20260505 09:47:24.775 | INFO | Perf report for the last 5.14003 seconds: | 20260505 09:47:24.776 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:47:24.776 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:24.776 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:24.776 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:24.776 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 09:47:24.776 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:24.776 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:24.776 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 09:47:24.776 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:24.776 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:24.776 | INFO | Uploaded 36GiB to GPU in 2.23621s (16.0986GiB/s) | 20260505 09:47:24.776 | INFO | Downloaded 34GiB to GPU in 2.11017s (16.1124GiB/s) | 20260505 09:47:29.917 | INFO | Analyzing chunk frames = [2247168,2247680) | 20260505 09:47:29.917 | INFO | MemInfo: { | 20260505 09:47:29.917 | INFO | "gpu" : " 17 GB", | 20260505 09:47:29.917 | INFO | "huge" : "", | 20260505 09:47:29.917 | INFO | "huge_shared" : " 27 GB", | 20260505 09:47:29.917 | INFO | "malloc" : " 14 GB", | 20260505 09:47:29.917 | INFO | "pinned" : " 29 GB" | 20260505 09:47:29.917 | INFO | } | 20260505 09:47:29.917 | INFO | Perf report for the last 5.14119 seconds: | 20260505 09:47:29.917 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:47:29.917 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:29.917 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:29.917 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:29.917 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 09:47:29.917 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:29.917 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:29.917 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:47:29.917 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:29.917 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:29.917 | INFO | Uploaded 36GiB to GPU in 2.23052s (16.1397GiB/s) | 20260505 09:47:29.917 | INFO | Downloaded 34GiB to GPU in 2.09937s (16.1953GiB/s) | 20260505 09:47:35.053 | INFO | Analyzing chunk frames = [2247680,2248192) | 20260505 09:47:35.053 | INFO | MemInfo: { | 20260505 09:47:35.053 | INFO | "gpu" : " 17 GB", | 20260505 09:47:35.053 | INFO | "huge" : "", | 20260505 09:47:35.053 | INFO | "huge_shared" : " 27 GB", | 20260505 09:47:35.053 | INFO | "malloc" : " 14 GB", | 20260505 09:47:35.053 | INFO | "pinned" : " 29 GB" | 20260505 09:47:35.053 | INFO | } | 20260505 09:47:35.053 | INFO | Perf report for the last 5.13644 seconds: | 20260505 09:47:35.053 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:47:35.053 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:35.053 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:35.053 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:35.053 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 09:47:35.053 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:35.053 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:35.053 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 09:47:35.053 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:35.053 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:35.053 | INFO | Uploaded 36GiB to GPU in 2.17452s (16.5554GiB/s) | 20260505 09:47:35.053 | INFO | Downloaded 34GiB to GPU in 2.1043s (16.1574GiB/s) | 20260505 09:47:40.192 | INFO | Analyzing chunk frames = [2248192,2248704) | 20260505 09:47:40.192 | INFO | MemInfo: { | 20260505 09:47:40.192 | INFO | "gpu" : " 17 GB", | 20260505 09:47:40.192 | INFO | "huge" : "", | 20260505 09:47:40.193 | INFO | "huge_shared" : " 27 GB", | 20260505 09:47:40.193 | INFO | "malloc" : " 14 GB", | 20260505 09:47:40.193 | INFO | "pinned" : " 29 GB" | 20260505 09:47:40.193 | INFO | } | 20260505 09:47:40.193 | INFO | Perf report for the last 5.13947 seconds: | 20260505 09:47:40.193 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:47:40.193 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:40.193 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:40.193 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:40.193 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 09:47:40.193 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:40.193 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:40.193 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 09:47:40.193 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:40.193 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:40.193 | INFO | Uploaded 36GiB to GPU in 2.17848s (16.5253GiB/s) | 20260505 09:47:40.193 | INFO | Downloaded 34GiB to GPU in 2.05169s (16.5717GiB/s) | 20260505 09:47:40.324 | INFO | Starting a round of Baz IO | 20260505 09:47:43.597 | INFO | Populating RTMetrics report | 20260505 09:47:43.598 | INFO | RTMetrics report populated | 20260505 09:47:43.598 | INFO | Opening tmp RTMetrics json file | 20260505 09:47:43.605 | INFO | Serializing RTMetrics to json | 20260505 09:47:43.630 | INFO | Serialization done, swapping tmp file now... | 20260505 09:47:43.633 | INFO | RTMetrics done | 20260505 09:47:43.731 | INFO | Finished a round of Baz IO: Wrote 2.432 GiB in 3.406 seconds | 20260505 09:47:43.731 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.67% / 0.01% / 74.32% | 20260505 09:47:43.731 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.79727% / 3.97364e-05% / 9.13938e-05% | 20260505 09:47:43.731 | INFO | Baz IO overall: | 20260505 09:47:43.731 | INFO | BytesWritten : 1305825668362 | 20260505 09:47:43.731 | INFO | NumEvents : 621554464082 | 20260505 09:47:45.335 | INFO | Analyzing chunk frames = [2248704,2249216) | 20260505 09:47:45.335 | INFO | MemInfo: { | 20260505 09:47:45.335 | INFO | "gpu" : " 17 GB", | 20260505 09:47:45.335 | INFO | "huge" : "", | 20260505 09:47:45.335 | INFO | "huge_shared" : " 27 GB", | 20260505 09:47:45.335 | INFO | "malloc" : " 14 GB", | 20260505 09:47:45.335 | INFO | "pinned" : " 29 GB" | 20260505 09:47:45.335 | INFO | } | 20260505 09:47:45.335 | INFO | Perf report for the last 5.14229 seconds: | 20260505 09:47:45.335 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:47:45.335 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:45.335 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:45.335 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:45.335 | INFO | ANALYSIS : 65.1%, 2.9, 100.1ms, 0 | 20260505 09:47:45.335 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:45.335 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260505 09:47:45.335 | INFO | PRE_HQ : 62.6%, 1.8, 59.8ms, 0 | 20260505 09:47:45.335 | INFO | BAZWRITER : 65.5%, 4.1, 139.6ms, 0 | 20260505 09:47:45.335 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 09:47:45.335 | INFO | Uploaded 36GiB to GPU in 2.22954s (16.1469GiB/s) | 20260505 09:47:45.335 | INFO | Downloaded 38GiB to GPU in 2.34259s (16.2214GiB/s) | 20260505 09:47:50.471 | INFO | Analyzing chunk frames = [2249216,2249728) | 20260505 09:47:50.471 | INFO | MemInfo: { | 20260505 09:47:50.471 | INFO | "gpu" : " 17 GB", | 20260505 09:47:50.471 | INFO | "huge" : "", | 20260505 09:47:50.471 | INFO | "huge_shared" : " 27 GB", | 20260505 09:47:50.471 | INFO | "malloc" : " 14 GB", | 20260505 09:47:50.471 | INFO | "pinned" : " 29 GB" | 20260505 09:47:50.471 | INFO | } | 20260505 09:47:50.471 | INFO | Perf report for the last 5.13574 seconds: | 20260505 09:47:50.471 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:47:50.471 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:50.471 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:50.471 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:50.471 | INFO | ANALYSIS : 64.4%, 3.0, 100.4ms, 0 | 20260505 09:47:50.471 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:50.471 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:50.471 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:47:50.471 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:50.471 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:50.471 | INFO | Uploaded 36GiB to GPU in 2.17543s (16.5485GiB/s) | 20260505 09:47:50.471 | INFO | Downloaded 34GiB to GPU in 2.09202s (16.2523GiB/s) | 20260505 09:47:55.610 | INFO | Analyzing chunk frames = [2249728,2250240) | 20260505 09:47:55.610 | INFO | MemInfo: { | 20260505 09:47:55.610 | INFO | "gpu" : " 17 GB", | 20260505 09:47:55.610 | INFO | "huge" : "", | 20260505 09:47:55.610 | INFO | "huge_shared" : " 27 GB", | 20260505 09:47:55.610 | INFO | "malloc" : " 14 GB", | 20260505 09:47:55.610 | INFO | "pinned" : " 29 GB" | 20260505 09:47:55.610 | INFO | } | 20260505 09:47:55.610 | INFO | Perf report for the last 5.13953 seconds: | 20260505 09:47:55.610 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:47:55.610 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:55.610 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:55.610 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:55.610 | INFO | ANALYSIS : 63.1%, 3.0, 98.2ms, 0 | 20260505 09:47:55.610 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:55.610 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:55.610 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:47:55.610 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:55.610 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:47:55.610 | INFO | Uploaded 36GiB to GPU in 2.14979s (16.7458GiB/s) | 20260505 09:47:55.611 | INFO | Downloaded 34GiB to GPU in 2.05987s (16.5059GiB/s) | 20260505 09:48:00.750 | INFO | Analyzing chunk frames = [2250240,2250752) | 20260505 09:48:00.750 | INFO | MemInfo: { | 20260505 09:48:00.750 | INFO | "gpu" : " 17 GB", | 20260505 09:48:00.750 | INFO | "huge" : "", | 20260505 09:48:00.750 | INFO | "huge_shared" : " 27 GB", | 20260505 09:48:00.750 | INFO | "malloc" : " 14 GB", | 20260505 09:48:00.750 | INFO | "pinned" : " 29 GB" | 20260505 09:48:00.750 | INFO | } | 20260505 09:48:00.751 | INFO | Perf report for the last 5.1403 seconds: | 20260505 09:48:00.751 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:48:00.751 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:00.751 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:00.751 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:00.751 | INFO | ANALYSIS : 63.1%, 3.0, 98.1ms, 0 | 20260505 09:48:00.751 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:00.751 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:00.751 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:48:00.751 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:00.751 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:00.751 | INFO | Uploaded 36GiB to GPU in 2.17887s (16.5223GiB/s) | 20260505 09:48:00.751 | INFO | Downloaded 34GiB to GPU in 2.0707s (16.4195GiB/s) | 20260505 09:48:05.889 | INFO | Analyzing chunk frames = [2250752,2251264) | 20260505 09:48:05.889 | INFO | MemInfo: { | 20260505 09:48:05.889 | INFO | "gpu" : " 17 GB", | 20260505 09:48:05.889 | INFO | "huge" : "", | 20260505 09:48:05.889 | INFO | "huge_shared" : " 27 GB", | 20260505 09:48:05.889 | INFO | "malloc" : " 14 GB", | 20260505 09:48:05.889 | INFO | "pinned" : " 29 GB" | 20260505 09:48:05.889 | INFO | } | 20260505 09:48:05.889 | INFO | Perf report for the last 5.13892 seconds: | 20260505 09:48:05.889 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:48:05.889 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:05.889 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:05.889 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:05.889 | INFO | ANALYSIS : 63.8%, 3.0, 99.5ms, 0 | 20260505 09:48:05.889 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:05.889 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:05.890 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:48:05.890 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:05.890 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:05.890 | INFO | Uploaded 36GiB to GPU in 2.23543s (16.1043GiB/s) | 20260505 09:48:05.890 | INFO | Downloaded 34GiB to GPU in 2.10753s (16.1326GiB/s) | 20260505 09:48:11.029 | INFO | Analyzing chunk frames = [2251264,2251776) | 20260505 09:48:11.029 | INFO | MemInfo: { | 20260505 09:48:11.029 | INFO | "gpu" : " 17 GB", | 20260505 09:48:11.029 | INFO | "huge" : "", | 20260505 09:48:11.029 | INFO | "huge_shared" : " 27 GB", | 20260505 09:48:11.029 | INFO | "malloc" : " 14 GB", | 20260505 09:48:11.029 | INFO | "pinned" : " 29 GB" | 20260505 09:48:11.029 | INFO | } | 20260505 09:48:11.029 | INFO | Perf report for the last 5.13921 seconds: | 20260505 09:48:11.029 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:48:11.029 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:11.029 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:11.029 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:11.029 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 09:48:11.029 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:11.029 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:11.029 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 09:48:11.029 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:11.029 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:11.029 | INFO | Uploaded 36GiB to GPU in 2.2322s (16.1276GiB/s) | 20260505 09:48:11.029 | INFO | Downloaded 34GiB to GPU in 2.09746s (16.2101GiB/s) | 20260505 09:48:16.167 | INFO | Analyzing chunk frames = [2251776,2252288) | 20260505 09:48:16.167 | INFO | MemInfo: { | 20260505 09:48:16.167 | INFO | "gpu" : " 17 GB", | 20260505 09:48:16.167 | INFO | "huge" : "", | 20260505 09:48:16.167 | INFO | "huge_shared" : " 27 GB", | 20260505 09:48:16.167 | INFO | "malloc" : " 14 GB", | 20260505 09:48:16.167 | INFO | "pinned" : " 29 GB" | 20260505 09:48:16.167 | INFO | } | 20260505 09:48:16.167 | INFO | Perf report for the last 5.13875 seconds: | 20260505 09:48:16.167 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:48:16.167 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:16.167 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:16.167 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:16.167 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 09:48:16.168 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:16.168 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:16.168 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 09:48:16.168 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:16.168 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:16.168 | INFO | Uploaded 36GiB to GPU in 2.20051s (16.3599GiB/s) | 20260505 09:48:16.168 | INFO | Downloaded 34GiB to GPU in 2.14186s (15.874GiB/s) | 20260505 09:48:21.308 | INFO | Analyzing chunk frames = [2252288,2252800) | 20260505 09:48:21.308 | INFO | MemInfo: { | 20260505 09:48:21.308 | INFO | "gpu" : " 17 GB", | 20260505 09:48:21.308 | INFO | "huge" : "", | 20260505 09:48:21.308 | INFO | "huge_shared" : " 27 GB", | 20260505 09:48:21.308 | INFO | "malloc" : " 14 GB", | 20260505 09:48:21.308 | INFO | "pinned" : " 29 GB" | 20260505 09:48:21.308 | INFO | } | 20260505 09:48:21.308 | INFO | Perf report for the last 5.14054 seconds: | 20260505 09:48:21.308 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:48:21.308 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:21.308 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:21.308 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:21.308 | INFO | ANALYSIS : 64.7%, 3.0, 101.1ms, 0 | 20260505 09:48:21.308 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:21.308 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:21.308 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 09:48:21.308 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:21.308 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:21.308 | INFO | Uploaded 36GiB to GPU in 2.24852s (16.0105GiB/s) | 20260505 09:48:21.308 | INFO | Downloaded 34GiB to GPU in 2.0826s (16.3258GiB/s) | 20260505 09:48:21.308 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 09:48:21.308 | INFO | Total mem usage: 15.3 GiB | 20260505 09:48:21.308 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 09:48:21.308 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.46%) | 20260505 09:48:21.308 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 09:48:21.308 | INFO | High Water Breakdown: | 20260505 09:48:21.308 | INFO | 61% : 8.78 GiB ( 93.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 09:48:21.308 | INFO | 38.9% : 5.6 GiB ( 84.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 09:48:21.308 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 09:48:21.308 | INFO | ------------------------------------------ | 20260505 09:48:21.308 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:48:21.308 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 09:48:21.308 | INFO | Total mem usage: 26.5 GiB | 20260505 09:48:21.308 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 09:48:21.308 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 09:48:21.308 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 09:48:21.308 | INFO | High Water Breakdown: | 20260505 09:48:21.308 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:48:21.308 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:48:21.308 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:48:21.308 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:48:21.308 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:48:21.309 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:48:21.309 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 09:48:21.309 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 09:48:21.309 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:48:21.309 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 09:48:21.309 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 09:48:21.309 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 09:48:21.309 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 09:48:21.309 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 09:48:21.309 | INFO | ------------------------------------------ | 20260505 09:48:21.309 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:48:21.309 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 09:48:21.309 | INFO | Total mem usage: 25.6 GiB | 20260505 09:48:21.309 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 09:48:21.309 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 09:48:21.309 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 09:48:21.309 | INFO | High Water Breakdown: | 20260505 09:48:21.309 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 09:48:21.309 | INFO | ------------------------------------------ | 20260505 09:48:21.309 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:48:21.309 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 09:48:21.309 | INFO | Total mem usage: 15.7 GiB | 20260505 09:48:21.309 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 09:48:21.309 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 09:48:21.309 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 09:48:21.309 | INFO | High Water Breakdown: | 20260505 09:48:21.309 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 09:48:21.309 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 09:48:21.309 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 09:48:21.309 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 09:48:21.309 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 09:48:21.309 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 09:48:21.309 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 09:48:21.309 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 09:48:21.309 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 09:48:21.309 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:48:21.309 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 09:48:21.309 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:48:21.309 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 09:48:21.309 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 09:48:21.309 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 09:48:21.309 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 09:48:21.309 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:48:21.309 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 09:48:21.309 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:48:21.309 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:48:21.309 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:48:21.309 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 09:48:21.309 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 09:48:21.309 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:48:21.309 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 09:48:21.309 | INFO | ------------------------------------------ | 20260505 09:48:21.309 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:48:21.309 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 09:48:21.309 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:48:21.309 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:48:21.309 | INFO | PerfReport: Full Execution -- 3.25e+01 / 7.71e+01, 1.38e+01 / 2.62e+00 | 20260505 09:48:21.309 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 1.02e-01, 1.04e-02 / 4.14e-03 | 20260505 09:48:21.309 | INFO | PerfReport: 11.64%, Baseline -- 3.78e+00 / 4.43e+00, 6.10e-01 / 2.72e-01 | 20260505 09:48:21.309 | INFO | PerfReport: 3.30%, Histogram -- 1.07e+00 / 1.47e+00, 2.06e-01 / 1.22e-01 | 20260505 09:48:21.309 | INFO | PerfReport: 15.63%, DME -- 4.06e+01 / 4.50e+01, 4.52e+00 / 2.30e+00 | 20260505 09:48:21.309 | INFO | PerfReport: 56.60%, FrameLabeling -- 1.84e+01 / 2.12e+01, 1.87e+00 / 2.19e+00 | 20260505 09:48:21.309 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.40e+00 / 2.95e+00, 4.14e-01 / 2.41e-01 | 20260505 09:48:21.309 | INFO | PerfReport: 3.81%, Metrics -- 1.24e+00 / 1.52e+00, 3.61e-01 / 1.79e-01 | 20260505 09:48:21.309 | INFO | PerfReport: 1.35%, PulseSerializing -- 4.37e-01 / 4.81e-01, 8.23e-02 / 2.54e-02 | 20260505 09:48:21.309 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 09:48:21.309 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:48:21.309 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:48:21.309 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.40e+01 / 0.00e+00 | 20260505 09:48:21.309 | INFO | PerfReport: 22.63%, Upload -- 2.30e+01 / 0.00e+00, 8.30e+00 / 0.00e+00 | 20260505 09:48:21.309 | INFO | PerfReport: 21.80%, Download -- 2.21e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260505 09:48:21.309 | INFO | PerfReport: 31.96%, Compute -- 3.25e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 09:48:21.309 | INFO | PerfReport: 1.86%, LockUpload -- 1.89e+00 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260505 09:48:21.309 | INFO | PerfReport: 1.09%, LockDownload -- 1.11e+00 / 0.00e+00, 2.65e+00 / 0.00e+00 | 20260505 09:48:21.309 | INFO | PerfReport: 20.64%, LockCompute -- 2.10e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260505 09:48:21.309 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.13e-02 / 0.00e+00 | 20260505 09:48:21.437 | INFO | Starting a round of Baz IO | 20260505 09:48:24.717 | INFO | Populating RTMetrics report | 20260505 09:48:24.717 | INFO | RTMetrics report populated | 20260505 09:48:24.717 | INFO | Opening tmp RTMetrics json file | 20260505 09:48:24.718 | INFO | Serializing RTMetrics to json | 20260505 09:48:24.720 | INFO | Serialization done, swapping tmp file now... | 20260505 09:48:24.725 | INFO | RTMetrics done | 20260505 09:48:24.894 | INFO | Finished a round of Baz IO: Wrote 2.431 GiB in 3.456 seconds | 20260505 09:48:24.894 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.65% / 0.01% / 74.33% | 20260505 09:48:24.894 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.7914% / 3.17891e-05% / 6.75519e-05% | 20260505 09:48:24.894 | INFO | Baz IO overall: | 20260505 09:48:24.894 | INFO | BytesWritten : 1308436404478 | 20260505 09:48:24.894 | INFO | NumEvents : 622801476515 | 20260505 09:48:26.446 | INFO | Analyzing chunk frames = [2252800,2253312) | 20260505 09:48:26.446 | INFO | MemInfo: { | 20260505 09:48:26.446 | INFO | "gpu" : " 17 GB", | 20260505 09:48:26.446 | INFO | "huge" : "", | 20260505 09:48:26.446 | INFO | "huge_shared" : " 27 GB", | 20260505 09:48:26.446 | INFO | "malloc" : " 14 GB", | 20260505 09:48:26.446 | INFO | "pinned" : " 29 GB" | 20260505 09:48:26.446 | INFO | } | 20260505 09:48:26.446 | INFO | Perf report for the last 5.13833 seconds: | 20260505 09:48:26.446 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:48:26.446 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:26.446 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:26.446 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:26.446 | INFO | ANALYSIS : 65.2%, 2.9, 99.9ms, 0 | 20260505 09:48:26.446 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:26.446 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260505 09:48:26.446 | INFO | PRE_HQ : 62.8%, 1.8, 60.0ms, 0 | 20260505 09:48:26.446 | INFO | BAZWRITER : 66.6%, 4.1, 141.4ms, 0 | 20260505 09:48:26.446 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 09:48:26.447 | INFO | Uploaded 36GiB to GPU in 2.23672s (16.095GiB/s) | 20260505 09:48:26.447 | INFO | Downloaded 38GiB to GPU in 2.34026s (16.2375GiB/s) | 20260505 09:48:31.586 | INFO | Analyzing chunk frames = [2253312,2253824) | 20260505 09:48:31.586 | INFO | MemInfo: { | 20260505 09:48:31.586 | INFO | "gpu" : " 17 GB", | 20260505 09:48:31.586 | INFO | "huge" : "", | 20260505 09:48:31.586 | INFO | "huge_shared" : " 27 GB", | 20260505 09:48:31.586 | INFO | "malloc" : " 14 GB", | 20260505 09:48:31.586 | INFO | "pinned" : " 29 GB" | 20260505 09:48:31.586 | INFO | } | 20260505 09:48:31.586 | INFO | Perf report for the last 5.13972 seconds: | 20260505 09:48:31.586 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:48:31.586 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:31.586 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:31.586 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:31.586 | INFO | ANALYSIS : 64.4%, 3.0, 100.6ms, 0 | 20260505 09:48:31.586 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:31.586 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:31.586 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:48:31.586 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:31.586 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:31.586 | INFO | Uploaded 36GiB to GPU in 2.2293s (16.1485GiB/s) | 20260505 09:48:31.586 | INFO | Downloaded 34GiB to GPU in 2.10646s (16.1408GiB/s) | 20260505 09:48:36.726 | INFO | Analyzing chunk frames = [2253824,2254336) | 20260505 09:48:36.726 | INFO | MemInfo: { | 20260505 09:48:36.726 | INFO | "gpu" : " 17 GB", | 20260505 09:48:36.726 | INFO | "huge" : "", | 20260505 09:48:36.726 | INFO | "huge_shared" : " 27 GB", | 20260505 09:48:36.726 | INFO | "malloc" : " 14 GB", | 20260505 09:48:36.726 | INFO | "pinned" : " 29 GB" | 20260505 09:48:36.726 | INFO | } | 20260505 09:48:36.726 | INFO | Perf report for the last 5.14005 seconds: | 20260505 09:48:36.726 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:48:36.726 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:36.726 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:36.726 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:36.726 | INFO | ANALYSIS : 63.1%, 3.0, 98.2ms, 0 | 20260505 09:48:36.726 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:36.726 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:36.726 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:48:36.726 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:36.726 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:36.726 | INFO | Uploaded 36GiB to GPU in 2.14243s (16.8033GiB/s) | 20260505 09:48:36.727 | INFO | Downloaded 34GiB to GPU in 2.05881s (16.5144GiB/s) | 20260505 09:48:41.867 | INFO | Analyzing chunk frames = [2254336,2254848) | 20260505 09:48:41.867 | INFO | MemInfo: { | 20260505 09:48:41.867 | INFO | "gpu" : " 17 GB", | 20260505 09:48:41.867 | INFO | "huge" : "", | 20260505 09:48:41.867 | INFO | "huge_shared" : " 27 GB", | 20260505 09:48:41.867 | INFO | "malloc" : " 14 GB", | 20260505 09:48:41.867 | INFO | "pinned" : " 29 GB" | 20260505 09:48:41.867 | INFO | } | 20260505 09:48:41.867 | INFO | Perf report for the last 5.1408 seconds: | 20260505 09:48:41.867 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:48:41.867 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:41.867 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:41.867 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:41.867 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 09:48:41.867 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:41.867 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:41.867 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 09:48:41.867 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:41.867 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:41.867 | INFO | Uploaded 36GiB to GPU in 2.17964s (16.5165GiB/s) | 20260505 09:48:41.867 | INFO | Downloaded 34GiB to GPU in 2.08214s (16.3293GiB/s) | 20260505 09:48:47.005 | INFO | Analyzing chunk frames = [2254848,2255360) | 20260505 09:48:47.005 | INFO | MemInfo: { | 20260505 09:48:47.005 | INFO | "gpu" : " 17 GB", | 20260505 09:48:47.005 | INFO | "huge" : "", | 20260505 09:48:47.005 | INFO | "huge_shared" : " 27 GB", | 20260505 09:48:47.005 | INFO | "malloc" : " 14 GB", | 20260505 09:48:47.005 | INFO | "pinned" : " 29 GB" | 20260505 09:48:47.005 | INFO | } | 20260505 09:48:47.005 | INFO | Perf report for the last 5.13785 seconds: | 20260505 09:48:47.005 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:48:47.005 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:47.005 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:47.005 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:47.005 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 09:48:47.005 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:47.005 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:47.005 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 09:48:47.005 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:47.005 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:47.005 | INFO | Uploaded 36GiB to GPU in 2.22981s (16.1449GiB/s) | 20260505 09:48:47.005 | INFO | Downloaded 34GiB to GPU in 2.10329s (16.1651GiB/s) | 20260505 09:48:52.143 | INFO | Analyzing chunk frames = [2255360,2255872) | 20260505 09:48:52.143 | INFO | MemInfo: { | 20260505 09:48:52.143 | INFO | "gpu" : " 17 GB", | 20260505 09:48:52.144 | INFO | "huge" : "", | 20260505 09:48:52.144 | INFO | "huge_shared" : " 27 GB", | 20260505 09:48:52.144 | INFO | "malloc" : " 14 GB", | 20260505 09:48:52.144 | INFO | "pinned" : " 29 GB" | 20260505 09:48:52.144 | INFO | } | 20260505 09:48:52.144 | INFO | Perf report for the last 5.13891 seconds: | 20260505 09:48:52.144 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:48:52.144 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:52.144 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:52.144 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:52.144 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 09:48:52.144 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:52.144 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:52.144 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 09:48:52.144 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:52.144 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:52.144 | INFO | Uploaded 36GiB to GPU in 2.23705s (16.0926GiB/s) | 20260505 09:48:52.144 | INFO | Downloaded 34GiB to GPU in 2.10012s (16.1896GiB/s) | 20260505 09:48:57.284 | INFO | Analyzing chunk frames = [2255872,2256384) | 20260505 09:48:57.284 | INFO | MemInfo: { | 20260505 09:48:57.284 | INFO | "gpu" : " 17 GB", | 20260505 09:48:57.284 | INFO | "huge" : "", | 20260505 09:48:57.284 | INFO | "huge_shared" : " 27 GB", | 20260505 09:48:57.284 | INFO | "malloc" : " 14 GB", | 20260505 09:48:57.284 | INFO | "pinned" : " 29 GB" | 20260505 09:48:57.284 | INFO | } | 20260505 09:48:57.284 | INFO | Perf report for the last 5.14013 seconds: | 20260505 09:48:57.284 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:48:57.284 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:57.284 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:57.284 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:57.284 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 09:48:57.284 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:57.284 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:57.284 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 09:48:57.284 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:57.284 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:48:57.284 | INFO | Uploaded 36GiB to GPU in 2.14298s (16.799GiB/s) | 20260505 09:48:57.284 | INFO | Downloaded 34GiB to GPU in 2.15622s (15.7684GiB/s) | 20260505 09:49:02.422 | INFO | Analyzing chunk frames = [2256384,2256896) | 20260505 09:49:02.422 | INFO | MemInfo: { | 20260505 09:49:02.422 | INFO | "gpu" : " 17 GB", | 20260505 09:49:02.422 | INFO | "huge" : "", | 20260505 09:49:02.423 | INFO | "huge_shared" : " 27 GB", | 20260505 09:49:02.423 | INFO | "malloc" : " 14 GB", | 20260505 09:49:02.423 | INFO | "pinned" : " 29 GB" | 20260505 09:49:02.423 | INFO | } | 20260505 09:49:02.423 | INFO | Perf report for the last 5.13874 seconds: | 20260505 09:49:02.423 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:49:02.423 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:02.423 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:02.423 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:02.423 | INFO | ANALYSIS : 64.5%, 3.0, 100.7ms, 0 | 20260505 09:49:02.423 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:02.423 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:02.423 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 09:49:02.423 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:02.423 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:02.423 | INFO | Uploaded 36GiB to GPU in 2.24579s (16.03GiB/s) | 20260505 09:49:02.423 | INFO | Downloaded 34GiB to GPU in 2.09899s (16.1982GiB/s) | 20260505 09:49:02.554 | INFO | Starting a round of Baz IO | 20260505 09:49:05.835 | INFO | Populating RTMetrics report | 20260505 09:49:05.835 | INFO | RTMetrics report populated | 20260505 09:49:05.835 | INFO | Opening tmp RTMetrics json file | 20260505 09:49:05.857 | INFO | Serializing RTMetrics to json | 20260505 09:49:05.882 | INFO | Serialization done, swapping tmp file now... | 20260505 09:49:05.884 | INFO | RTMetrics done | 20260505 09:49:05.985 | INFO | Finished a round of Baz IO: Wrote 2.43 GiB in 3.431 seconds | 20260505 09:49:05.985 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.64% / 0.01% / 74.35% | 20260505 09:49:05.986 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.7845% / 7.94729e-06% / 7.54992e-05% | 20260505 09:49:05.986 | INFO | Baz IO overall: | 20260505 09:49:05.986 | INFO | BytesWritten : 1311045813346 | 20260505 09:49:05.986 | INFO | NumEvents : 624047434122 | 20260505 09:49:07.561 | INFO | Analyzing chunk frames = [2256896,2257408) | 20260505 09:49:07.561 | INFO | MemInfo: { | 20260505 09:49:07.561 | INFO | "gpu" : " 17 GB", | 20260505 09:49:07.561 | INFO | "huge" : "", | 20260505 09:49:07.561 | INFO | "huge_shared" : " 27 GB", | 20260505 09:49:07.561 | INFO | "malloc" : " 14 GB", | 20260505 09:49:07.561 | INFO | "pinned" : " 29 GB" | 20260505 09:49:07.561 | INFO | } | 20260505 09:49:07.562 | INFO | Perf report for the last 5.13885 seconds: | 20260505 09:49:07.562 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:49:07.562 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:07.562 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:07.562 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:07.562 | INFO | ANALYSIS : 65.3%, 2.9, 99.7ms, 0 | 20260505 09:49:07.562 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:07.562 | INFO | RT_METRICS : 46.9%, 1.0, 24.6ms, 0 | 20260505 09:49:07.562 | INFO | PRE_HQ : 62.2%, 1.9, 60.1ms, 0 | 20260505 09:49:07.562 | INFO | BAZWRITER : 66.1%, 4.2, 140.9ms, 0 | 20260505 09:49:07.562 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260505 09:49:07.562 | INFO | Uploaded 36GiB to GPU in 2.25131s (15.9907GiB/s) | 20260505 09:49:07.562 | INFO | Downloaded 38GiB to GPU in 2.32109s (16.3716GiB/s) | 20260505 09:49:12.703 | INFO | Analyzing chunk frames = [2257408,2257920) | 20260505 09:49:12.703 | INFO | MemInfo: { | 20260505 09:49:12.703 | INFO | "gpu" : " 17 GB", | 20260505 09:49:12.703 | INFO | "huge" : "", | 20260505 09:49:12.703 | INFO | "huge_shared" : " 27 GB", | 20260505 09:49:12.703 | INFO | "malloc" : " 14 GB", | 20260505 09:49:12.703 | INFO | "pinned" : " 29 GB" | 20260505 09:49:12.703 | INFO | } | 20260505 09:49:12.703 | INFO | Perf report for the last 5.14184 seconds: | 20260505 09:49:12.703 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:49:12.703 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:12.703 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:12.703 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:12.703 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 09:49:12.703 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:12.703 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:12.703 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:49:12.703 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:12.704 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:12.704 | INFO | Uploaded 36GiB to GPU in 2.17673s (16.5386GiB/s) | 20260505 09:49:12.704 | INFO | Downloaded 34GiB to GPU in 2.10019s (16.189GiB/s) | 20260505 09:49:17.840 | INFO | Analyzing chunk frames = [2257920,2258432) | 20260505 09:49:17.840 | INFO | MemInfo: { | 20260505 09:49:17.840 | INFO | "gpu" : " 17 GB", | 20260505 09:49:17.840 | INFO | "huge" : "", | 20260505 09:49:17.840 | INFO | "huge_shared" : " 27 GB", | 20260505 09:49:17.841 | INFO | "malloc" : " 14 GB", | 20260505 09:49:17.841 | INFO | "pinned" : " 29 GB" | 20260505 09:49:17.841 | INFO | } | 20260505 09:49:17.841 | INFO | Perf report for the last 5.13735 seconds: | 20260505 09:49:17.841 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:49:17.841 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:17.841 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:17.841 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:17.841 | INFO | ANALYSIS : 63.1%, 3.0, 98.2ms, 0 | 20260505 09:49:17.841 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:17.841 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:17.841 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:49:17.841 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:17.841 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:17.841 | INFO | Uploaded 36GiB to GPU in 2.16308s (16.6429GiB/s) | 20260505 09:49:17.841 | INFO | Downloaded 34GiB to GPU in 2.06362s (16.4759GiB/s) | 20260505 09:49:22.979 | INFO | Analyzing chunk frames = [2258432,2258944) | 20260505 09:49:22.979 | INFO | MemInfo: { | 20260505 09:49:22.979 | INFO | "gpu" : " 17 GB", | 20260505 09:49:22.979 | INFO | "huge" : "", | 20260505 09:49:22.979 | INFO | "huge_shared" : " 27 GB", | 20260505 09:49:22.979 | INFO | "malloc" : " 14 GB", | 20260505 09:49:22.979 | INFO | "pinned" : " 29 GB" | 20260505 09:49:22.979 | INFO | } | 20260505 09:49:22.979 | INFO | Perf report for the last 5.13867 seconds: | 20260505 09:49:22.979 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:49:22.979 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:22.979 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:22.979 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:22.979 | INFO | ANALYSIS : 63.1%, 3.0, 98.1ms, 0 | 20260505 09:49:22.979 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:22.980 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:22.980 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 09:49:22.980 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:22.980 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:22.980 | INFO | Uploaded 36GiB to GPU in 2.16701s (16.6127GiB/s) | 20260505 09:49:22.980 | INFO | Downloaded 34GiB to GPU in 2.07354s (16.3971GiB/s) | 20260505 09:49:28.118 | INFO | Analyzing chunk frames = [2258944,2259456) | 20260505 09:49:28.118 | INFO | MemInfo: { | 20260505 09:49:28.118 | INFO | "gpu" : " 17 GB", | 20260505 09:49:28.119 | INFO | "huge" : "", | 20260505 09:49:28.119 | INFO | "huge_shared" : " 27 GB", | 20260505 09:49:28.119 | INFO | "malloc" : " 14 GB", | 20260505 09:49:28.119 | INFO | "pinned" : " 29 GB" | 20260505 09:49:28.119 | INFO | } | 20260505 09:49:28.119 | INFO | Perf report for the last 5.1393 seconds: | 20260505 09:49:28.119 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:49:28.119 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:28.119 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:28.119 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:28.119 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 09:49:28.119 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:28.119 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:28.119 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 09:49:28.119 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:28.119 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:28.119 | INFO | Uploaded 36GiB to GPU in 2.23817s (16.0846GiB/s) | 20260505 09:49:28.119 | INFO | Downloaded 34GiB to GPU in 2.10453s (16.1557GiB/s) | 20260505 09:49:33.258 | INFO | Analyzing chunk frames = [2259456,2259968) | 20260505 09:49:33.258 | INFO | MemInfo: { | 20260505 09:49:33.258 | INFO | "gpu" : " 17 GB", | 20260505 09:49:33.258 | INFO | "huge" : "", | 20260505 09:49:33.258 | INFO | "huge_shared" : " 27 GB", | 20260505 09:49:33.258 | INFO | "malloc" : " 14 GB", | 20260505 09:49:33.258 | INFO | "pinned" : " 29 GB" | 20260505 09:49:33.258 | INFO | } | 20260505 09:49:33.258 | INFO | Perf report for the last 5.13974 seconds: | 20260505 09:49:33.258 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:49:33.258 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:33.258 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:33.259 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:33.259 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 09:49:33.259 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:33.259 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:33.259 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 09:49:33.259 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:33.259 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:33.259 | INFO | Uploaded 36GiB to GPU in 2.23747s (16.0896GiB/s) | 20260505 09:49:33.259 | INFO | Downloaded 34GiB to GPU in 2.10004s (16.1902GiB/s) | 20260505 09:49:38.397 | INFO | Analyzing chunk frames = [2259968,2260480) | 20260505 09:49:38.397 | INFO | MemInfo: { | 20260505 09:49:38.397 | INFO | "gpu" : " 17 GB", | 20260505 09:49:38.397 | INFO | "huge" : "", | 20260505 09:49:38.397 | INFO | "huge_shared" : " 27 GB", | 20260505 09:49:38.397 | INFO | "malloc" : " 14 GB", | 20260505 09:49:38.398 | INFO | "pinned" : " 29 GB" | 20260505 09:49:38.398 | INFO | } | 20260505 09:49:38.398 | INFO | Perf report for the last 5.13917 seconds: | 20260505 09:49:38.398 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:49:38.398 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:38.398 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:38.398 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:38.398 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 09:49:38.398 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:38.398 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:38.398 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 09:49:38.398 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:38.398 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:38.398 | INFO | Uploaded 36GiB to GPU in 2.22094s (16.2094GiB/s) | 20260505 09:49:38.398 | INFO | Downloaded 34GiB to GPU in 2.1388s (15.8967GiB/s) | 20260505 09:49:43.537 | INFO | Analyzing chunk frames = [2260480,2260992) | 20260505 09:49:43.537 | INFO | MemInfo: { | 20260505 09:49:43.537 | INFO | "gpu" : " 17 GB", | 20260505 09:49:43.537 | INFO | "huge" : "", | 20260505 09:49:43.537 | INFO | "huge_shared" : " 27 GB", | 20260505 09:49:43.537 | INFO | "malloc" : " 14 GB", | 20260505 09:49:43.537 | INFO | "pinned" : " 29 GB" | 20260505 09:49:43.537 | INFO | } | 20260505 09:49:43.537 | INFO | Perf report for the last 5.13947 seconds: | 20260505 09:49:43.537 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:49:43.537 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:43.537 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:43.537 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:43.537 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 09:49:43.537 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:43.537 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:43.537 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:49:43.537 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:43.537 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:43.537 | INFO | Uploaded 36GiB to GPU in 2.18233s (16.4961GiB/s) | 20260505 09:49:43.537 | INFO | Downloaded 34GiB to GPU in 2.04434s (16.6313GiB/s) | 20260505 09:49:43.538 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 09:49:43.538 | INFO | Total mem usage: 15.3 GiB | 20260505 09:49:43.538 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 09:49:43.538 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.5%) | 20260505 09:49:43.538 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 09:49:43.538 | INFO | High Water Breakdown: | 20260505 09:49:43.538 | INFO | 61% : 8.78 GiB ( 93.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 09:49:43.538 | INFO | 38.9% : 5.59 GiB ( 84.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 09:49:43.538 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 09:49:43.538 | INFO | ------------------------------------------ | 20260505 09:49:43.538 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:49:43.538 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 09:49:43.538 | INFO | Total mem usage: 26.5 GiB | 20260505 09:49:43.538 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 09:49:43.538 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 09:49:43.538 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 09:49:43.538 | INFO | High Water Breakdown: | 20260505 09:49:43.538 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:49:43.538 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:49:43.538 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:49:43.538 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:49:43.538 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:49:43.538 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:49:43.538 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 09:49:43.538 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 09:49:43.538 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:49:43.538 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 09:49:43.538 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 09:49:43.538 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 09:49:43.538 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 09:49:43.538 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 09:49:43.538 | INFO | ------------------------------------------ | 20260505 09:49:43.538 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:49:43.538 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 09:49:43.538 | INFO | Total mem usage: 25.6 GiB | 20260505 09:49:43.538 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 09:49:43.538 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 09:49:43.538 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 09:49:43.538 | INFO | High Water Breakdown: | 20260505 09:49:43.538 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 09:49:43.538 | INFO | ------------------------------------------ | 20260505 09:49:43.538 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:49:43.538 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 09:49:43.538 | INFO | Total mem usage: 15.7 GiB | 20260505 09:49:43.538 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 09:49:43.538 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 09:49:43.538 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 09:49:43.538 | INFO | High Water Breakdown: | 20260505 09:49:43.538 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 09:49:43.538 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 09:49:43.538 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 09:49:43.538 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 09:49:43.538 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 09:49:43.538 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 09:49:43.538 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 09:49:43.538 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 09:49:43.538 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 09:49:43.538 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:49:43.538 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 09:49:43.538 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:49:43.538 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 09:49:43.538 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 09:49:43.538 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 09:49:43.538 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 09:49:43.538 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:49:43.538 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 09:49:43.538 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:49:43.538 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:49:43.538 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:49:43.538 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 09:49:43.538 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 09:49:43.538 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:49:43.538 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 09:49:43.538 | INFO | ------------------------------------------ | 20260505 09:49:43.538 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:49:43.539 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 09:49:43.539 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:49:43.539 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:49:43.539 | INFO | PerfReport: Full Execution -- 3.25e+01 / 7.72e+01, 1.39e+01 / 2.33e+00 | 20260505 09:49:43.539 | INFO | PerfReport: 0.29%, AggregateStats -- 9.30e-02 / 1.02e-01, 1.04e-02 / 4.47e-03 | 20260505 09:49:43.539 | INFO | PerfReport: 11.64%, Baseline -- 3.78e+00 / 4.42e+00, 6.06e-01 / 3.19e-01 | 20260505 09:49:43.539 | INFO | PerfReport: 3.31%, Histogram -- 1.07e+00 / 1.49e+00, 2.07e-01 / 1.04e-01 | 20260505 09:49:43.539 | INFO | PerfReport: 15.64%, DME -- 4.06e+01 / 4.51e+01, 4.53e+00 / 2.35e+00 | 20260505 09:49:43.539 | INFO | PerfReport: 56.54%, FrameLabeling -- 1.84e+01 / 2.12e+01, 1.85e+00 / 1.99e+00 | 20260505 09:49:43.539 | INFO | PerfReport: 7.43%, PulseAccumulating -- 2.41e+00 / 2.93e+00, 4.20e-01 / 2.14e-01 | 20260505 09:49:43.539 | INFO | PerfReport: 3.81%, Metrics -- 1.24e+00 / 1.53e+00, 3.57e-01 / 1.99e-01 | 20260505 09:49:43.539 | INFO | PerfReport: 1.34%, PulseSerializing -- 4.36e-01 / 4.87e-01, 8.20e-02 / 3.03e-02 | 20260505 09:49:43.539 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 09:49:43.539 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:49:43.539 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:49:43.539 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.38e+01 / 0.00e+00 | 20260505 09:49:43.539 | INFO | PerfReport: 22.63%, Upload -- 2.30e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260505 09:49:43.539 | INFO | PerfReport: 21.81%, Download -- 2.21e+01 / 0.00e+00, 8.15e+00 / 0.00e+00 | 20260505 09:49:43.539 | INFO | PerfReport: 31.99%, Compute -- 3.25e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 09:49:43.539 | INFO | PerfReport: 1.88%, LockUpload -- 1.91e+00 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260505 09:49:43.539 | INFO | PerfReport: 1.08%, LockDownload -- 1.10e+00 / 0.00e+00, 2.61e+00 / 0.00e+00 | 20260505 09:49:43.539 | INFO | PerfReport: 20.59%, LockCompute -- 2.09e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260505 09:49:43.539 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.68e-02 / 0.00e+00 | 20260505 09:49:43.670 | INFO | Starting a round of Baz IO | 20260505 09:49:46.950 | INFO | Populating RTMetrics report | 20260505 09:49:46.951 | INFO | RTMetrics report populated | 20260505 09:49:46.951 | INFO | Opening tmp RTMetrics json file | 20260505 09:49:46.990 | INFO | Serializing RTMetrics to json | 20260505 09:49:47.025 | INFO | Serialization done, swapping tmp file now... | 20260505 09:49:47.027 | INFO | RTMetrics done | 20260505 09:49:47.105 | INFO | Finished a round of Baz IO: Wrote 2.43 GiB in 3.435 seconds | 20260505 09:49:47.105 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.63% / 0.01% / 74.36% | 20260505 09:49:47.105 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.77767% / 1.19209e-05% / 5.16574e-05% | 20260505 09:49:47.106 | INFO | Baz IO overall: | 20260505 09:49:47.106 | INFO | BytesWritten : 1313655406242 | 20260505 09:49:47.106 | INFO | NumEvents : 625293415744 | 20260505 09:49:48.677 | INFO | Analyzing chunk frames = [2260992,2261504) | 20260505 09:49:48.677 | INFO | MemInfo: { | 20260505 09:49:48.677 | INFO | "gpu" : " 17 GB", | 20260505 09:49:48.677 | INFO | "huge" : "", | 20260505 09:49:48.677 | INFO | "huge_shared" : " 27 GB", | 20260505 09:49:48.677 | INFO | "malloc" : " 14 GB", | 20260505 09:49:48.677 | INFO | "pinned" : " 29 GB" | 20260505 09:49:48.677 | INFO | } | 20260505 09:49:48.677 | INFO | Perf report for the last 5.13959 seconds: | 20260505 09:49:48.677 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:49:48.677 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:48.677 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:48.677 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:48.677 | INFO | ANALYSIS : 65.3%, 2.9, 99.9ms, 0 | 20260505 09:49:48.677 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:48.677 | INFO | RT_METRICS : 47.2%, 1.0, 24.7ms, 0 | 20260505 09:49:48.677 | INFO | PRE_HQ : 62.4%, 1.8, 59.9ms, 0 | 20260505 09:49:48.677 | INFO | BAZWRITER : 66.1%, 4.2, 141.4ms, 0 | 20260505 09:49:48.677 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 09:49:48.677 | INFO | Uploaded 36GiB to GPU in 2.23346s (16.1185GiB/s) | 20260505 09:49:48.677 | INFO | Downloaded 38GiB to GPU in 2.34233s (16.2232GiB/s) | 20260505 09:49:53.816 | INFO | Analyzing chunk frames = [2261504,2262016) | 20260505 09:49:53.816 | INFO | MemInfo: { | 20260505 09:49:53.816 | INFO | "gpu" : " 17 GB", | 20260505 09:49:53.816 | INFO | "huge" : "", | 20260505 09:49:53.816 | INFO | "huge_shared" : " 27 GB", | 20260505 09:49:53.816 | INFO | "malloc" : " 14 GB", | 20260505 09:49:53.816 | INFO | "pinned" : " 29 GB" | 20260505 09:49:53.816 | INFO | } | 20260505 09:49:53.816 | INFO | Perf report for the last 5.13958 seconds: | 20260505 09:49:53.816 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:49:53.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:53.816 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:53.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:53.816 | INFO | ANALYSIS : 64.3%, 3.0, 100.3ms, 0 | 20260505 09:49:53.816 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:53.816 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:53.816 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 09:49:53.816 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:53.816 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:53.817 | INFO | Uploaded 36GiB to GPU in 2.17671s (16.5388GiB/s) | 20260505 09:49:53.817 | INFO | Downloaded 34GiB to GPU in 2.09584s (16.2226GiB/s) | 20260505 09:49:58.955 | INFO | Analyzing chunk frames = [2262016,2262528) | 20260505 09:49:58.955 | INFO | MemInfo: { | 20260505 09:49:58.955 | INFO | "gpu" : " 17 GB", | 20260505 09:49:58.955 | INFO | "huge" : "", | 20260505 09:49:58.955 | INFO | "huge_shared" : " 27 GB", | 20260505 09:49:58.955 | INFO | "malloc" : " 14 GB", | 20260505 09:49:58.955 | INFO | "pinned" : " 29 GB" | 20260505 09:49:58.955 | INFO | } | 20260505 09:49:58.955 | INFO | Perf report for the last 5.13869 seconds: | 20260505 09:49:58.955 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:49:58.955 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:58.955 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:58.955 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:58.955 | INFO | ANALYSIS : 63.0%, 3.0, 98.1ms, 0 | 20260505 09:49:58.955 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:58.955 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:58.955 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:49:58.955 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:58.955 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:49:58.955 | INFO | Uploaded 36GiB to GPU in 2.14682s (16.769GiB/s) | 20260505 09:49:58.955 | INFO | Downloaded 34GiB to GPU in 2.05869s (16.5154GiB/s) | 20260505 09:50:04.093 | INFO | Analyzing chunk frames = [2262528,2263040) | 20260505 09:50:04.093 | INFO | MemInfo: { | 20260505 09:50:04.093 | INFO | "gpu" : " 17 GB", | 20260505 09:50:04.094 | INFO | "huge" : "", | 20260505 09:50:04.094 | INFO | "huge_shared" : " 27 GB", | 20260505 09:50:04.094 | INFO | "malloc" : " 14 GB", | 20260505 09:50:04.094 | INFO | "pinned" : " 29 GB" | 20260505 09:50:04.094 | INFO | } | 20260505 09:50:04.094 | INFO | Perf report for the last 5.13873 seconds: | 20260505 09:50:04.094 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:50:04.094 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:04.094 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:04.094 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:04.094 | INFO | ANALYSIS : 63.2%, 3.0, 98.1ms, 0 | 20260505 09:50:04.094 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:04.094 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:04.094 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 09:50:04.094 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:04.094 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:04.094 | INFO | Uploaded 36GiB to GPU in 2.15849s (16.6783GiB/s) | 20260505 09:50:04.094 | INFO | Downloaded 34GiB to GPU in 2.06944s (16.4296GiB/s) | 20260505 09:50:09.233 | INFO | Analyzing chunk frames = [2263040,2263552) | 20260505 09:50:09.233 | INFO | MemInfo: { | 20260505 09:50:09.233 | INFO | "gpu" : " 17 GB", | 20260505 09:50:09.233 | INFO | "huge" : "", | 20260505 09:50:09.233 | INFO | "huge_shared" : " 27 GB", | 20260505 09:50:09.233 | INFO | "malloc" : " 14 GB", | 20260505 09:50:09.233 | INFO | "pinned" : " 29 GB" | 20260505 09:50:09.233 | INFO | } | 20260505 09:50:09.233 | INFO | Perf report for the last 5.13953 seconds: | 20260505 09:50:09.233 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:50:09.233 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:09.233 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:09.233 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:09.233 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 09:50:09.233 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:09.233 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:09.233 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 09:50:09.233 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:09.233 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:09.234 | INFO | Uploaded 36GiB to GPU in 2.23445s (16.1113GiB/s) | 20260505 09:50:09.234 | INFO | Downloaded 34GiB to GPU in 2.1066s (16.1397GiB/s) | 20260505 09:50:14.374 | INFO | Analyzing chunk frames = [2263552,2264064) | 20260505 09:50:14.374 | INFO | MemInfo: { | 20260505 09:50:14.374 | INFO | "gpu" : " 17 GB", | 20260505 09:50:14.374 | INFO | "huge" : "", | 20260505 09:50:14.374 | INFO | "huge_shared" : " 27 GB", | 20260505 09:50:14.374 | INFO | "malloc" : " 14 GB", | 20260505 09:50:14.374 | INFO | "pinned" : " 29 GB" | 20260505 09:50:14.374 | INFO | } | 20260505 09:50:14.374 | INFO | Perf report for the last 5.14113 seconds: | 20260505 09:50:14.374 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:50:14.374 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:14.375 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:14.375 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:14.375 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 09:50:14.375 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:14.375 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:14.375 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 09:50:14.375 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:14.375 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:14.375 | INFO | Uploaded 36GiB to GPU in 2.23633s (16.0978GiB/s) | 20260505 09:50:14.375 | INFO | Downloaded 34GiB to GPU in 2.10118s (16.1814GiB/s) | 20260505 09:50:19.513 | INFO | Analyzing chunk frames = [2264064,2264576) | 20260505 09:50:19.513 | INFO | MemInfo: { | 20260505 09:50:19.513 | INFO | "gpu" : " 17 GB", | 20260505 09:50:19.513 | INFO | "huge" : "", | 20260505 09:50:19.513 | INFO | "huge_shared" : " 27 GB", | 20260505 09:50:19.513 | INFO | "malloc" : " 14 GB", | 20260505 09:50:19.513 | INFO | "pinned" : " 29 GB" | 20260505 09:50:19.513 | INFO | } | 20260505 09:50:19.514 | INFO | Perf report for the last 5.13903 seconds: | 20260505 09:50:19.514 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:50:19.514 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:19.514 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:19.514 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:19.514 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 09:50:19.514 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:19.514 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:19.514 | INFO | PRE_HQ : 30.5%, 1.0, 15.9ms, 0 | 20260505 09:50:19.514 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:19.514 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:19.514 | INFO | Uploaded 36GiB to GPU in 2.20435s (16.3313GiB/s) | 20260505 09:50:19.514 | INFO | Downloaded 34GiB to GPU in 2.11816s (16.0517GiB/s) | 20260505 09:50:24.652 | INFO | Analyzing chunk frames = [2264576,2265088) | 20260505 09:50:24.652 | INFO | MemInfo: { | 20260505 09:50:24.652 | INFO | "gpu" : " 17 GB", | 20260505 09:50:24.652 | INFO | "huge" : "", | 20260505 09:50:24.652 | INFO | "huge_shared" : " 27 GB", | 20260505 09:50:24.652 | INFO | "malloc" : " 14 GB", | 20260505 09:50:24.652 | INFO | "pinned" : " 29 GB" | 20260505 09:50:24.652 | INFO | } | 20260505 09:50:24.652 | INFO | Perf report for the last 5.13875 seconds: | 20260505 09:50:24.652 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:50:24.652 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:24.652 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:24.652 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:24.652 | INFO | ANALYSIS : 64.8%, 3.0, 101.1ms, 0 | 20260505 09:50:24.652 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:24.652 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:24.652 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 09:50:24.652 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:24.652 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:24.653 | INFO | Uploaded 36GiB to GPU in 2.24762s (16.0169GiB/s) | 20260505 09:50:24.653 | INFO | Downloaded 34GiB to GPU in 2.0859s (16.2999GiB/s) | 20260505 09:50:24.786 | INFO | Starting a round of Baz IO | 20260505 09:50:28.066 | INFO | Populating RTMetrics report | 20260505 09:50:28.067 | INFO | RTMetrics report populated | 20260505 09:50:28.067 | INFO | Opening tmp RTMetrics json file | 20260505 09:50:28.085 | INFO | Serializing RTMetrics to json | 20260505 09:50:28.109 | INFO | Serialization done, swapping tmp file now... | 20260505 09:50:28.114 | INFO | RTMetrics done | 20260505 09:50:28.211 | INFO | Finished a round of Baz IO: Wrote 2.431 GiB in 3.425 seconds | 20260505 09:50:28.211 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.61% / 0.01% / 74.37% | 20260505 09:50:28.211 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.7765% / 2.78155e-05% / 7.54992e-05% | 20260505 09:50:28.211 | INFO | Baz IO overall: | 20260505 09:50:28.211 | INFO | BytesWritten : 1316266117754 | 20260505 09:50:28.211 | INFO | NumEvents : 626540420526 | 20260505 09:50:29.790 | INFO | Analyzing chunk frames = [2265088,2265600) | 20260505 09:50:29.790 | INFO | MemInfo: { | 20260505 09:50:29.790 | INFO | "gpu" : " 17 GB", | 20260505 09:50:29.790 | INFO | "huge" : "", | 20260505 09:50:29.790 | INFO | "huge_shared" : " 27 GB", | 20260505 09:50:29.790 | INFO | "malloc" : " 14 GB", | 20260505 09:50:29.791 | INFO | "pinned" : " 29 GB" | 20260505 09:50:29.791 | INFO | } | 20260505 09:50:29.791 | INFO | Perf report for the last 5.13832 seconds: | 20260505 09:50:29.791 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:50:29.791 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:29.791 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:29.791 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:29.791 | INFO | ANALYSIS : 65.3%, 2.9, 100.0ms, 0 | 20260505 09:50:29.791 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:29.791 | INFO | RT_METRICS : 46.5%, 1.0, 24.4ms, 0 | 20260505 09:50:29.791 | INFO | PRE_HQ : 61.8%, 1.9, 59.9ms, 0 | 20260505 09:50:29.791 | INFO | BAZWRITER : 65.9%, 4.1, 140.4ms, 0 | 20260505 09:50:29.791 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260505 09:50:29.791 | INFO | Uploaded 36GiB to GPU in 2.21849s (16.2272GiB/s) | 20260505 09:50:29.791 | INFO | Downloaded 38GiB to GPU in 2.30933s (16.455GiB/s) | 20260505 09:50:34.930 | INFO | Analyzing chunk frames = [2265600,2266112) | 20260505 09:50:34.930 | INFO | MemInfo: { | 20260505 09:50:34.930 | INFO | "gpu" : " 17 GB", | 20260505 09:50:34.930 | INFO | "huge" : "", | 20260505 09:50:34.930 | INFO | "huge_shared" : " 27 GB", | 20260505 09:50:34.930 | INFO | "malloc" : " 14 GB", | 20260505 09:50:34.930 | INFO | "pinned" : " 29 GB" | 20260505 09:50:34.930 | INFO | } | 20260505 09:50:34.930 | INFO | Perf report for the last 5.13978 seconds: | 20260505 09:50:34.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:50:34.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:34.930 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:34.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:34.930 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 09:50:34.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:34.930 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:34.931 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 09:50:34.931 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:34.931 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:34.931 | INFO | Uploaded 36GiB to GPU in 2.1762s (16.5426GiB/s) | 20260505 09:50:34.931 | INFO | Downloaded 34GiB to GPU in 2.10163s (16.1779GiB/s) | 20260505 09:50:40.070 | INFO | Analyzing chunk frames = [2266112,2266624) | 20260505 09:50:40.070 | INFO | MemInfo: { | 20260505 09:50:40.070 | INFO | "gpu" : " 17 GB", | 20260505 09:50:40.070 | INFO | "huge" : "", | 20260505 09:50:40.070 | INFO | "huge_shared" : " 27 GB", | 20260505 09:50:40.070 | INFO | "malloc" : " 14 GB", | 20260505 09:50:40.070 | INFO | "pinned" : " 29 GB" | 20260505 09:50:40.070 | INFO | } | 20260505 09:50:40.070 | INFO | Perf report for the last 5.14004 seconds: | 20260505 09:50:40.070 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:50:40.070 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:40.071 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:40.071 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:40.071 | INFO | ANALYSIS : 63.1%, 3.0, 98.1ms, 0 | 20260505 09:50:40.071 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:40.071 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:40.071 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:50:40.071 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:40.071 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:40.071 | INFO | Uploaded 36GiB to GPU in 2.14728s (16.7654GiB/s) | 20260505 09:50:40.071 | INFO | Downloaded 34GiB to GPU in 2.0561s (16.5362GiB/s) | 20260505 09:50:45.208 | INFO | Analyzing chunk frames = [2266624,2267136) | 20260505 09:50:45.208 | INFO | MemInfo: { | 20260505 09:50:45.208 | INFO | "gpu" : " 17 GB", | 20260505 09:50:45.208 | INFO | "huge" : "", | 20260505 09:50:45.208 | INFO | "huge_shared" : " 27 GB", | 20260505 09:50:45.209 | INFO | "malloc" : " 14 GB", | 20260505 09:50:45.209 | INFO | "pinned" : " 29 GB" | 20260505 09:50:45.209 | INFO | } | 20260505 09:50:45.209 | INFO | Perf report for the last 5.13815 seconds: | 20260505 09:50:45.209 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:50:45.209 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:45.209 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:45.209 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:45.209 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 09:50:45.209 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:45.209 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:45.209 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 09:50:45.209 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:45.209 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:45.209 | INFO | Uploaded 36GiB to GPU in 2.17132s (16.5798GiB/s) | 20260505 09:50:45.209 | INFO | Downloaded 34GiB to GPU in 2.0744s (16.3903GiB/s) | 20260505 09:50:50.349 | INFO | Analyzing chunk frames = [2267136,2267648) | 20260505 09:50:50.349 | INFO | MemInfo: { | 20260505 09:50:50.349 | INFO | "gpu" : " 17 GB", | 20260505 09:50:50.349 | INFO | "huge" : "", | 20260505 09:50:50.349 | INFO | "huge_shared" : " 27 GB", | 20260505 09:50:50.349 | INFO | "malloc" : " 14 GB", | 20260505 09:50:50.349 | INFO | "pinned" : " 29 GB" | 20260505 09:50:50.349 | INFO | } | 20260505 09:50:50.349 | INFO | Perf report for the last 5.14049 seconds: | 20260505 09:50:50.349 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:50:50.349 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:50.349 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:50.349 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:50.349 | INFO | ANALYSIS : 63.7%, 3.0, 99.6ms, 0 | 20260505 09:50:50.349 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:50.349 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:50.349 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 09:50:50.349 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:50.349 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:50.349 | INFO | Uploaded 36GiB to GPU in 2.23431s (16.1124GiB/s) | 20260505 09:50:50.349 | INFO | Downloaded 34GiB to GPU in 2.10971s (16.116GiB/s) | 20260505 09:50:55.490 | INFO | Analyzing chunk frames = [2267648,2268160) | 20260505 09:50:55.490 | INFO | MemInfo: { | 20260505 09:50:55.490 | INFO | "gpu" : " 17 GB", | 20260505 09:50:55.490 | INFO | "huge" : "", | 20260505 09:50:55.490 | INFO | "huge_shared" : " 27 GB", | 20260505 09:50:55.490 | INFO | "malloc" : " 14 GB", | 20260505 09:50:55.490 | INFO | "pinned" : " 29 GB" | 20260505 09:50:55.490 | INFO | } | 20260505 09:50:55.490 | INFO | Perf report for the last 5.14089 seconds: | 20260505 09:50:55.490 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:50:55.490 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:55.490 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:55.490 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:55.490 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 09:50:55.490 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:55.490 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:55.490 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 09:50:55.490 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:55.490 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:50:55.491 | INFO | Uploaded 36GiB to GPU in 2.23633s (16.0978GiB/s) | 20260505 09:50:55.491 | INFO | Downloaded 34GiB to GPU in 2.09751s (16.2097GiB/s) | 20260505 09:51:00.627 | INFO | Analyzing chunk frames = [2268160,2268672) | 20260505 09:51:00.627 | INFO | MemInfo: { | 20260505 09:51:00.627 | INFO | "gpu" : " 17 GB", | 20260505 09:51:00.627 | INFO | "huge" : "", | 20260505 09:51:00.627 | INFO | "huge_shared" : " 27 GB", | 20260505 09:51:00.627 | INFO | "malloc" : " 14 GB", | 20260505 09:51:00.627 | INFO | "pinned" : " 29 GB" | 20260505 09:51:00.627 | INFO | } | 20260505 09:51:00.627 | INFO | Perf report for the last 5.1371 seconds: | 20260505 09:51:00.627 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:51:00.627 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:00.627 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:00.627 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:00.627 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 09:51:00.627 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:00.627 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:00.627 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 09:51:00.627 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:00.627 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:00.627 | INFO | Uploaded 36GiB to GPU in 2.21768s (16.2332GiB/s) | 20260505 09:51:00.627 | INFO | Downloaded 34GiB to GPU in 2.12938s (15.9671GiB/s) | 20260505 09:51:05.766 | INFO | Analyzing chunk frames = [2268672,2269184) | 20260505 09:51:05.766 | INFO | MemInfo: { | 20260505 09:51:05.766 | INFO | "gpu" : " 17 GB", | 20260505 09:51:05.766 | INFO | "huge" : "", | 20260505 09:51:05.766 | INFO | "huge_shared" : " 27 GB", | 20260505 09:51:05.766 | INFO | "malloc" : " 14 GB", | 20260505 09:51:05.766 | INFO | "pinned" : " 29 GB" | 20260505 09:51:05.766 | INFO | } | 20260505 09:51:05.766 | INFO | Perf report for the last 5.13928 seconds: | 20260505 09:51:05.766 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:51:05.766 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:05.766 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:05.766 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:05.766 | INFO | ANALYSIS : 64.6%, 3.0, 100.9ms, 0 | 20260505 09:51:05.766 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:05.767 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:05.767 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:51:05.767 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:05.767 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:05.767 | INFO | Uploaded 36GiB to GPU in 2.24655s (16.0246GiB/s) | 20260505 09:51:05.767 | INFO | Downloaded 34GiB to GPU in 2.0892s (16.2742GiB/s) | 20260505 09:51:05.767 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 09:51:05.767 | INFO | Total mem usage: 15.3 GiB | 20260505 09:51:05.767 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 09:51:05.767 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.58%) | 20260505 09:51:05.767 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 09:51:05.767 | INFO | High Water Breakdown: | 20260505 09:51:05.767 | INFO | 61% : 8.77 GiB ( 93.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 09:51:05.767 | INFO | 38.9% : 5.59 GiB ( 84.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 09:51:05.767 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 09:51:05.767 | INFO | ------------------------------------------ | 20260505 09:51:05.767 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:51:05.767 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 09:51:05.767 | INFO | Total mem usage: 26.5 GiB | 20260505 09:51:05.767 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 09:51:05.767 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 09:51:05.767 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 09:51:05.767 | INFO | High Water Breakdown: | 20260505 09:51:05.767 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:51:05.767 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:51:05.767 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:51:05.767 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:51:05.767 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:51:05.767 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:51:05.767 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 09:51:05.767 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 09:51:05.767 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:51:05.767 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 09:51:05.767 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 09:51:05.767 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 09:51:05.767 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 09:51:05.767 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 09:51:05.767 | INFO | ------------------------------------------ | 20260505 09:51:05.767 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:51:05.767 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 09:51:05.767 | INFO | Total mem usage: 25.6 GiB | 20260505 09:51:05.767 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 09:51:05.767 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 09:51:05.767 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 09:51:05.767 | INFO | High Water Breakdown: | 20260505 09:51:05.767 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 09:51:05.767 | INFO | ------------------------------------------ | 20260505 09:51:05.767 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:51:05.767 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 09:51:05.767 | INFO | Total mem usage: 15.7 GiB | 20260505 09:51:05.767 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 09:51:05.767 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 09:51:05.767 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 09:51:05.767 | INFO | High Water Breakdown: | 20260505 09:51:05.767 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 09:51:05.767 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 09:51:05.767 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 09:51:05.767 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 09:51:05.767 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 09:51:05.767 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 09:51:05.767 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 09:51:05.767 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 09:51:05.767 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 09:51:05.767 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:51:05.767 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 09:51:05.767 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:51:05.767 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 09:51:05.767 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 09:51:05.767 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 09:51:05.767 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 09:51:05.768 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:51:05.768 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 09:51:05.768 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:51:05.768 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:51:05.768 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:51:05.768 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 09:51:05.768 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 09:51:05.768 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:51:05.768 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 09:51:05.768 | INFO | ------------------------------------------ | 20260505 09:51:05.768 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:51:05.768 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 09:51:05.768 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:51:05.768 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:51:05.768 | INFO | PerfReport: Full Execution -- 3.25e+01 / 7.78e+01, 1.38e+01 / 2.96e+00 | 20260505 09:51:05.768 | INFO | PerfReport: 0.29%, AggregateStats -- 9.31e-02 / 1.03e-01, 1.04e-02 / 3.61e-03 | 20260505 09:51:05.768 | INFO | PerfReport: 11.65%, Baseline -- 3.79e+00 / 4.49e+00, 6.17e-01 / 2.23e-01 | 20260505 09:51:05.768 | INFO | PerfReport: 3.31%, Histogram -- 1.08e+00 / 1.49e+00, 2.09e-01 / 8.30e-02 | 20260505 09:51:05.768 | INFO | PerfReport: 15.65%, DME -- 4.07e+01 / 4.53e+01, 4.70e+00 / 2.30e+00 | 20260505 09:51:05.768 | INFO | PerfReport: 56.66%, FrameLabeling -- 1.84e+01 / 2.14e+01, 1.91e+00 / 1.99e+00 | 20260505 09:51:05.768 | INFO | PerfReport: 7.31%, PulseAccumulating -- 2.38e+00 / 2.96e+00, 3.94e-01 / 2.22e-01 | 20260505 09:51:05.768 | INFO | PerfReport: 3.78%, Metrics -- 1.23e+00 / 1.50e+00, 3.56e-01 / 1.72e-01 | 20260505 09:51:05.768 | INFO | PerfReport: 1.34%, PulseSerializing -- 4.37e-01 / 4.85e-01, 8.21e-02 / 2.93e-02 | 20260505 09:51:05.768 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 09:51:05.768 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:51:05.768 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:51:05.768 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.44e+01 / 0.00e+00 | 20260505 09:51:05.768 | INFO | PerfReport: 22.59%, Upload -- 2.30e+01 / 0.00e+00, 8.33e+00 / 0.00e+00 | 20260505 09:51:05.768 | INFO | PerfReport: 21.73%, Download -- 2.21e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260505 09:51:05.768 | INFO | PerfReport: 31.98%, Compute -- 3.25e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 09:51:05.768 | INFO | PerfReport: 1.93%, LockUpload -- 1.97e+00 / 0.00e+00, 7.52e+00 / 0.00e+00 | 20260505 09:51:05.768 | INFO | PerfReport: 1.07%, LockDownload -- 1.09e+00 / 0.00e+00, 2.65e+00 / 0.00e+00 | 20260505 09:51:05.768 | INFO | PerfReport: 20.67%, LockCompute -- 2.10e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260505 09:51:05.768 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.44e-02 / 0.00e+00 | 20260505 09:51:05.898 | INFO | Starting a round of Baz IO | 20260505 09:51:09.179 | INFO | Populating RTMetrics report | 20260505 09:51:09.180 | INFO | RTMetrics report populated | 20260505 09:51:09.180 | INFO | Opening tmp RTMetrics json file | 20260505 09:51:09.202 | INFO | Serializing RTMetrics to json | 20260505 09:51:09.204 | INFO | Serialization done, swapping tmp file now... | 20260505 09:51:09.204 | INFO | RTMetrics done | 20260505 09:51:09.377 | INFO | Finished a round of Baz IO: Wrote 2.432 GiB in 3.478 seconds | 20260505 09:51:09.377 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.60% / 0.01% / 74.39% | 20260505 09:51:09.377 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.7699% / 7.94729e-06% / 6.35783e-05% | 20260505 09:51:09.377 | INFO | Baz IO overall: | 20260505 09:51:09.377 | INFO | BytesWritten : 1318877369257 | 20260505 09:51:09.377 | INFO | NumEvents : 627787738690 | 20260505 09:51:10.906 | INFO | Analyzing chunk frames = [2269184,2269696) | 20260505 09:51:10.906 | INFO | MemInfo: { | 20260505 09:51:10.906 | INFO | "gpu" : " 17 GB", | 20260505 09:51:10.906 | INFO | "huge" : "", | 20260505 09:51:10.906 | INFO | "huge_shared" : " 27 GB", | 20260505 09:51:10.906 | INFO | "malloc" : " 14 GB", | 20260505 09:51:10.906 | INFO | "pinned" : " 29 GB" | 20260505 09:51:10.906 | INFO | } | 20260505 09:51:10.906 | INFO | Perf report for the last 5.13919 seconds: | 20260505 09:51:10.906 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:51:10.906 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:10.906 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:10.906 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:10.906 | INFO | ANALYSIS : 65.3%, 2.9, 99.8ms, 0 | 20260505 09:51:10.906 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:10.906 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260505 09:51:10.906 | INFO | PRE_HQ : 62.6%, 1.8, 59.9ms, 0 | 20260505 09:51:10.906 | INFO | BAZWRITER : 67.0%, 4.2, 144.1ms, 0 | 20260505 09:51:10.906 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260505 09:51:10.906 | INFO | Uploaded 36GiB to GPU in 2.22957s (16.1466GiB/s) | 20260505 09:51:10.906 | INFO | Downloaded 38GiB to GPU in 2.33525s (16.2723GiB/s) | 20260505 09:51:16.044 | INFO | Analyzing chunk frames = [2269696,2270208) | 20260505 09:51:16.044 | INFO | MemInfo: { | 20260505 09:51:16.044 | INFO | "gpu" : " 17 GB", | 20260505 09:51:16.044 | INFO | "huge" : "", | 20260505 09:51:16.045 | INFO | "huge_shared" : " 27 GB", | 20260505 09:51:16.045 | INFO | "malloc" : " 14 GB", | 20260505 09:51:16.045 | INFO | "pinned" : " 29 GB" | 20260505 09:51:16.045 | INFO | } | 20260505 09:51:16.045 | INFO | Perf report for the last 5.13897 seconds: | 20260505 09:51:16.045 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:51:16.045 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:16.045 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:16.045 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:16.045 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 09:51:16.045 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:16.045 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:16.045 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 09:51:16.045 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:16.045 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:16.045 | INFO | Uploaded 36GiB to GPU in 2.17959s (16.5169GiB/s) | 20260505 09:51:16.045 | INFO | Downloaded 34GiB to GPU in 2.09702s (16.2135GiB/s) | 20260505 09:51:21.184 | INFO | Analyzing chunk frames = [2270208,2270720) | 20260505 09:51:21.184 | INFO | MemInfo: { | 20260505 09:51:21.184 | INFO | "gpu" : " 17 GB", | 20260505 09:51:21.184 | INFO | "huge" : "", | 20260505 09:51:21.184 | INFO | "huge_shared" : " 27 GB", | 20260505 09:51:21.184 | INFO | "malloc" : " 14 GB", | 20260505 09:51:21.184 | INFO | "pinned" : " 29 GB" | 20260505 09:51:21.184 | INFO | } | 20260505 09:51:21.184 | INFO | Perf report for the last 5.13984 seconds: | 20260505 09:51:21.184 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:51:21.184 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:21.184 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:21.184 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:21.185 | INFO | ANALYSIS : 63.2%, 3.0, 98.4ms, 0 | 20260505 09:51:21.185 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:21.185 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:21.185 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 09:51:21.185 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:21.185 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:21.185 | INFO | Uploaded 36GiB to GPU in 2.15144s (16.733GiB/s) | 20260505 09:51:21.185 | INFO | Downloaded 34GiB to GPU in 2.06883s (16.4344GiB/s) | 20260505 09:51:26.325 | INFO | Analyzing chunk frames = [2270720,2271232) | 20260505 09:51:26.325 | INFO | MemInfo: { | 20260505 09:51:26.325 | INFO | "gpu" : " 17 GB", | 20260505 09:51:26.325 | INFO | "huge" : "", | 20260505 09:51:26.325 | INFO | "huge_shared" : " 27 GB", | 20260505 09:51:26.325 | INFO | "malloc" : " 14 GB", | 20260505 09:51:26.325 | INFO | "pinned" : " 29 GB" | 20260505 09:51:26.325 | INFO | } | 20260505 09:51:26.325 | INFO | Perf report for the last 5.14072 seconds: | 20260505 09:51:26.325 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:51:26.325 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:26.325 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:26.325 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:26.325 | INFO | ANALYSIS : 63.1%, 3.0, 98.1ms, 0 | 20260505 09:51:26.325 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:26.325 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:26.325 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 09:51:26.325 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:26.325 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:26.325 | INFO | Uploaded 36GiB to GPU in 2.17628s (16.542GiB/s) | 20260505 09:51:26.326 | INFO | Downloaded 34GiB to GPU in 2.07616s (16.3764GiB/s) | 20260505 09:51:31.464 | INFO | Analyzing chunk frames = [2271232,2271744) | 20260505 09:51:31.464 | INFO | MemInfo: { | 20260505 09:51:31.464 | INFO | "gpu" : " 17 GB", | 20260505 09:51:31.464 | INFO | "huge" : "", | 20260505 09:51:31.464 | INFO | "huge_shared" : " 27 GB", | 20260505 09:51:31.464 | INFO | "malloc" : " 14 GB", | 20260505 09:51:31.464 | INFO | "pinned" : " 29 GB" | 20260505 09:51:31.464 | INFO | } | 20260505 09:51:31.464 | INFO | Perf report for the last 5.13895 seconds: | 20260505 09:51:31.464 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:51:31.464 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:31.464 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:31.464 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:31.464 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 09:51:31.464 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:31.464 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:31.464 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:51:31.464 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:31.464 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:31.464 | INFO | Uploaded 36GiB to GPU in 2.23473s (16.1093GiB/s) | 20260505 09:51:31.464 | INFO | Downloaded 34GiB to GPU in 2.10727s (16.1346GiB/s) | 20260505 09:51:36.602 | INFO | Analyzing chunk frames = [2271744,2272256) | 20260505 09:51:36.602 | INFO | MemInfo: { | 20260505 09:51:36.602 | INFO | "gpu" : " 17 GB", | 20260505 09:51:36.602 | INFO | "huge" : "", | 20260505 09:51:36.602 | INFO | "huge_shared" : " 27 GB", | 20260505 09:51:36.603 | INFO | "malloc" : " 14 GB", | 20260505 09:51:36.603 | INFO | "pinned" : " 29 GB" | 20260505 09:51:36.603 | INFO | } | 20260505 09:51:36.603 | INFO | Perf report for the last 5.13844 seconds: | 20260505 09:51:36.603 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:51:36.603 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:36.603 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:36.603 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:36.603 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 09:51:36.603 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:36.603 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:36.603 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 09:51:36.603 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:36.603 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:36.603 | INFO | Uploaded 36GiB to GPU in 2.23516s (16.1062GiB/s) | 20260505 09:51:36.603 | INFO | Downloaded 34GiB to GPU in 2.10363s (16.1625GiB/s) | 20260505 09:51:41.741 | INFO | Analyzing chunk frames = [2272256,2272768) | 20260505 09:51:41.741 | INFO | MemInfo: { | 20260505 09:51:41.741 | INFO | "gpu" : " 17 GB", | 20260505 09:51:41.741 | INFO | "huge" : "", | 20260505 09:51:41.741 | INFO | "huge_shared" : " 27 GB", | 20260505 09:51:41.742 | INFO | "malloc" : " 14 GB", | 20260505 09:51:41.742 | INFO | "pinned" : " 29 GB" | 20260505 09:51:41.742 | INFO | } | 20260505 09:51:41.742 | INFO | Perf report for the last 5.13895 seconds: | 20260505 09:51:41.742 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:51:41.742 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:41.742 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:41.742 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:41.742 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 09:51:41.742 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:41.742 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:41.742 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260505 09:51:41.742 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:41.742 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:41.742 | INFO | Uploaded 36GiB to GPU in 2.15176s (16.7305GiB/s) | 20260505 09:51:41.742 | INFO | Downloaded 34GiB to GPU in 2.14788s (15.8296GiB/s) | 20260505 09:51:46.881 | INFO | Analyzing chunk frames = [2272768,2273280) | 20260505 09:51:46.881 | INFO | MemInfo: { | 20260505 09:51:46.881 | INFO | "gpu" : " 17 GB", | 20260505 09:51:46.881 | INFO | "huge" : "", | 20260505 09:51:46.881 | INFO | "huge_shared" : " 27 GB", | 20260505 09:51:46.881 | INFO | "malloc" : " 14 GB", | 20260505 09:51:46.881 | INFO | "pinned" : " 29 GB" | 20260505 09:51:46.881 | INFO | } | 20260505 09:51:46.882 | INFO | Perf report for the last 5.13994 seconds: | 20260505 09:51:46.882 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:51:46.882 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:46.882 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:46.882 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:46.882 | INFO | ANALYSIS : 64.7%, 3.0, 101.0ms, 0 | 20260505 09:51:46.882 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:46.882 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:46.882 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 09:51:46.882 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:46.882 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:46.882 | INFO | Uploaded 36GiB to GPU in 2.24711s (16.0206GiB/s) | 20260505 09:51:46.882 | INFO | Downloaded 34GiB to GPU in 2.09873s (16.2003GiB/s) | 20260505 09:51:47.015 | INFO | Starting a round of Baz IO | 20260505 09:51:50.282 | INFO | Populating RTMetrics report | 20260505 09:51:50.282 | INFO | RTMetrics report populated | 20260505 09:51:50.282 | INFO | Opening tmp RTMetrics json file | 20260505 09:51:50.303 | INFO | Serializing RTMetrics to json | 20260505 09:51:50.306 | INFO | Serialization done, swapping tmp file now... | 20260505 09:51:50.308 | INFO | RTMetrics done | 20260505 09:51:50.447 | INFO | Finished a round of Baz IO: Wrote 2.432 GiB in 3.432 seconds | 20260505 09:51:50.447 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.59% / 0.01% / 74.40% | 20260505 09:51:50.447 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.76904% / 1.58946e-05% / 6.75519e-05% | 20260505 09:51:50.447 | INFO | Baz IO overall: | 20260505 09:51:50.447 | INFO | BytesWritten : 1321488572565 | 20260505 09:51:50.447 | INFO | NumEvents : 629035114095 | 20260505 09:51:52.020 | INFO | Analyzing chunk frames = [2273280,2273792) | 20260505 09:51:52.020 | INFO | MemInfo: { | 20260505 09:51:52.020 | INFO | "gpu" : " 17 GB", | 20260505 09:51:52.020 | INFO | "huge" : "", | 20260505 09:51:52.021 | INFO | "huge_shared" : " 27 GB", | 20260505 09:51:52.021 | INFO | "malloc" : " 14 GB", | 20260505 09:51:52.021 | INFO | "pinned" : " 29 GB" | 20260505 09:51:52.021 | INFO | } | 20260505 09:51:52.021 | INFO | Perf report for the last 5.13906 seconds: | 20260505 09:51:52.021 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:51:52.021 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:52.021 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:52.021 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:52.021 | INFO | ANALYSIS : 65.0%, 2.9, 99.4ms, 0 | 20260505 09:51:52.021 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:52.021 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260505 09:51:52.021 | INFO | PRE_HQ : 62.8%, 1.8, 60.1ms, 0 | 20260505 09:51:52.021 | INFO | BAZWRITER : 66.1%, 4.2, 143.0ms, 0 | 20260505 09:51:52.021 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 09:51:52.021 | INFO | Uploaded 36GiB to GPU in 2.29078s (15.7152GiB/s) | 20260505 09:51:52.021 | INFO | Downloaded 38GiB to GPU in 2.37357s (16.0096GiB/s) | 20260505 09:51:57.160 | INFO | Analyzing chunk frames = [2273792,2274304) | 20260505 09:51:57.160 | INFO | MemInfo: { | 20260505 09:51:57.160 | INFO | "gpu" : " 17 GB", | 20260505 09:51:57.160 | INFO | "huge" : "", | 20260505 09:51:57.160 | INFO | "huge_shared" : " 27 GB", | 20260505 09:51:57.160 | INFO | "malloc" : " 14 GB", | 20260505 09:51:57.160 | INFO | "pinned" : " 29 GB" | 20260505 09:51:57.160 | INFO | } | 20260505 09:51:57.161 | INFO | Perf report for the last 5.13991 seconds: | 20260505 09:51:57.161 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:51:57.161 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:57.161 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:57.161 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:57.161 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 09:51:57.161 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:57.161 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:57.161 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 09:51:57.161 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:57.161 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:51:57.161 | INFO | Uploaded 36GiB to GPU in 2.18324s (16.4893GiB/s) | 20260505 09:51:57.161 | INFO | Downloaded 34GiB to GPU in 2.0937s (16.2392GiB/s) | 20260505 09:52:02.299 | INFO | Analyzing chunk frames = [2274304,2274816) | 20260505 09:52:02.299 | INFO | MemInfo: { | 20260505 09:52:02.299 | INFO | "gpu" : " 17 GB", | 20260505 09:52:02.299 | INFO | "huge" : "", | 20260505 09:52:02.299 | INFO | "huge_shared" : " 27 GB", | 20260505 09:52:02.299 | INFO | "malloc" : " 14 GB", | 20260505 09:52:02.299 | INFO | "pinned" : " 29 GB" | 20260505 09:52:02.299 | INFO | } | 20260505 09:52:02.299 | INFO | Perf report for the last 5.1383 seconds: | 20260505 09:52:02.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:52:02.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:02.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:02.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:02.299 | INFO | ANALYSIS : 63.1%, 3.0, 98.1ms, 0 | 20260505 09:52:02.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:02.299 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:02.299 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 09:52:02.299 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:02.299 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:02.299 | INFO | Uploaded 36GiB to GPU in 2.14815s (16.7586GiB/s) | 20260505 09:52:02.299 | INFO | Downloaded 34GiB to GPU in 2.05305s (16.5608GiB/s) | 20260505 09:52:07.440 | INFO | Analyzing chunk frames = [2274816,2275328) | 20260505 09:52:07.440 | INFO | MemInfo: { | 20260505 09:52:07.440 | INFO | "gpu" : " 17 GB", | 20260505 09:52:07.440 | INFO | "huge" : "", | 20260505 09:52:07.440 | INFO | "huge_shared" : " 27 GB", | 20260505 09:52:07.440 | INFO | "malloc" : " 14 GB", | 20260505 09:52:07.440 | INFO | "pinned" : " 29 GB" | 20260505 09:52:07.440 | INFO | } | 20260505 09:52:07.440 | INFO | Perf report for the last 5.14135 seconds: | 20260505 09:52:07.440 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:52:07.440 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:07.440 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:07.440 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:07.440 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 09:52:07.440 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:07.440 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:07.440 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 09:52:07.440 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:07.440 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:07.441 | INFO | Uploaded 36GiB to GPU in 2.17961s (16.5167GiB/s) | 20260505 09:52:07.441 | INFO | Downloaded 34GiB to GPU in 2.07334s (16.3987GiB/s) | 20260505 09:52:12.577 | INFO | Analyzing chunk frames = [2275328,2275840) | 20260505 09:52:12.577 | INFO | MemInfo: { | 20260505 09:52:12.577 | INFO | "gpu" : " 17 GB", | 20260505 09:52:12.577 | INFO | "huge" : "", | 20260505 09:52:12.577 | INFO | "huge_shared" : " 27 GB", | 20260505 09:52:12.577 | INFO | "malloc" : " 14 GB", | 20260505 09:52:12.577 | INFO | "pinned" : " 29 GB" | 20260505 09:52:12.577 | INFO | } | 20260505 09:52:12.577 | INFO | Perf report for the last 5.13724 seconds: | 20260505 09:52:12.577 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:52:12.577 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:12.578 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:12.578 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:12.578 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 09:52:12.578 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:12.578 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:12.578 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 09:52:12.578 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:12.578 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:12.578 | INFO | Uploaded 36GiB to GPU in 2.23678s (16.0946GiB/s) | 20260505 09:52:12.578 | INFO | Downloaded 34GiB to GPU in 2.09061s (16.2632GiB/s) | 20260505 09:52:17.718 | INFO | Analyzing chunk frames = [2275840,2276352) | 20260505 09:52:17.718 | INFO | MemInfo: { | 20260505 09:52:17.718 | INFO | "gpu" : " 17 GB", | 20260505 09:52:17.718 | INFO | "huge" : "", | 20260505 09:52:17.718 | INFO | "huge_shared" : " 27 GB", | 20260505 09:52:17.718 | INFO | "malloc" : " 14 GB", | 20260505 09:52:17.718 | INFO | "pinned" : " 29 GB" | 20260505 09:52:17.718 | INFO | } | 20260505 09:52:17.718 | INFO | Perf report for the last 5.14042 seconds: | 20260505 09:52:17.718 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:52:17.718 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:17.718 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:17.718 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:17.718 | INFO | ANALYSIS : 63.9%, 3.0, 99.9ms, 0 | 20260505 09:52:17.718 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:17.718 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:17.718 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:52:17.718 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:17.718 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:17.718 | INFO | Uploaded 36GiB to GPU in 2.23713s (16.092GiB/s) | 20260505 09:52:17.718 | INFO | Downloaded 34GiB to GPU in 2.099s (16.1982GiB/s) | 20260505 09:52:22.856 | INFO | Analyzing chunk frames = [2276352,2276864) | 20260505 09:52:22.857 | INFO | MemInfo: { | 20260505 09:52:22.857 | INFO | "gpu" : " 17 GB", | 20260505 09:52:22.857 | INFO | "huge" : "", | 20260505 09:52:22.857 | INFO | "huge_shared" : " 27 GB", | 20260505 09:52:22.857 | INFO | "malloc" : " 14 GB", | 20260505 09:52:22.857 | INFO | "pinned" : " 29 GB" | 20260505 09:52:22.857 | INFO | } | 20260505 09:52:22.857 | INFO | Perf report for the last 5.13891 seconds: | 20260505 09:52:22.857 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:52:22.857 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:22.857 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:22.857 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:22.857 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 09:52:22.857 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:22.857 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:22.857 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 09:52:22.857 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:22.857 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:22.857 | INFO | Uploaded 36GiB to GPU in 2.21015s (16.2885GiB/s) | 20260505 09:52:22.857 | INFO | Downloaded 34GiB to GPU in 2.11274s (16.0929GiB/s) | 20260505 09:52:27.996 | INFO | Analyzing chunk frames = [2276864,2277376) | 20260505 09:52:27.996 | INFO | MemInfo: { | 20260505 09:52:27.996 | INFO | "gpu" : " 17 GB", | 20260505 09:52:27.996 | INFO | "huge" : "", | 20260505 09:52:27.996 | INFO | "huge_shared" : " 27 GB", | 20260505 09:52:27.996 | INFO | "malloc" : " 14 GB", | 20260505 09:52:27.996 | INFO | "pinned" : " 29 GB" | 20260505 09:52:27.996 | INFO | } | 20260505 09:52:27.996 | INFO | Perf report for the last 5.13915 seconds: | 20260505 09:52:27.996 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:52:27.996 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:27.996 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:27.996 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:27.996 | INFO | ANALYSIS : 64.6%, 3.0, 100.8ms, 0 | 20260505 09:52:27.996 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:27.996 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:27.996 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 09:52:27.996 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:27.996 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:27.996 | INFO | Uploaded 36GiB to GPU in 2.24982s (16.0013GiB/s) | 20260505 09:52:27.996 | INFO | Downloaded 34GiB to GPU in 2.087s (16.2913GiB/s) | 20260505 09:52:27.997 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 09:52:27.997 | INFO | Total mem usage: 15.3 GiB | 20260505 09:52:27.997 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 09:52:27.997 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -5.59%) | 20260505 09:52:27.997 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 09:52:27.997 | INFO | High Water Breakdown: | 20260505 09:52:27.997 | INFO | 61.1% : 8.86 GiB ( 94.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 09:52:27.997 | INFO | 38.8% : 5.63 GiB ( 84.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 09:52:27.997 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 09:52:27.997 | INFO | ------------------------------------------ | 20260505 09:52:27.997 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:52:27.997 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 09:52:27.997 | INFO | Total mem usage: 26.5 GiB | 20260505 09:52:27.997 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 09:52:27.997 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 09:52:27.997 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 09:52:27.997 | INFO | High Water Breakdown: | 20260505 09:52:27.997 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:52:27.997 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:52:27.997 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:52:27.997 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:52:27.997 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:52:27.997 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:52:27.997 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 09:52:27.997 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 09:52:27.997 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:52:27.997 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 09:52:27.997 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 09:52:27.997 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 09:52:27.997 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 09:52:27.997 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 09:52:27.997 | INFO | ------------------------------------------ | 20260505 09:52:27.997 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:52:27.997 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 09:52:27.997 | INFO | Total mem usage: 25.6 GiB | 20260505 09:52:27.997 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 09:52:27.997 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 09:52:27.997 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 09:52:27.997 | INFO | High Water Breakdown: | 20260505 09:52:27.997 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 09:52:27.997 | INFO | ------------------------------------------ | 20260505 09:52:27.997 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:52:27.997 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 09:52:27.997 | INFO | Total mem usage: 15.7 GiB | 20260505 09:52:27.997 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 09:52:27.997 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 09:52:27.997 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 09:52:27.997 | INFO | High Water Breakdown: | 20260505 09:52:27.997 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 09:52:27.997 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 09:52:27.997 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 09:52:27.997 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 09:52:27.997 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 09:52:27.997 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 09:52:27.997 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 09:52:27.997 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 09:52:27.997 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 09:52:27.997 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:52:27.997 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 09:52:27.997 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:52:27.997 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 09:52:27.997 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 09:52:27.997 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 09:52:27.997 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 09:52:27.997 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:52:27.997 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 09:52:27.997 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:52:27.997 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:52:27.997 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:52:27.997 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 09:52:27.997 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 09:52:27.997 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:52:27.997 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 09:52:27.997 | INFO | ------------------------------------------ | 20260505 09:52:27.997 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:52:27.997 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 09:52:27.997 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:52:27.997 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:52:27.997 | INFO | PerfReport: Full Execution -- 3.25e+01 / 7.76e+01, 1.37e+01 / 2.63e+00 | 20260505 09:52:27.997 | INFO | PerfReport: 0.29%, AggregateStats -- 9.33e-02 / 1.02e-01, 1.05e-02 / 3.83e-03 | 20260505 09:52:27.997 | INFO | PerfReport: 11.65%, Baseline -- 3.78e+00 / 4.44e+00, 6.12e-01 / 2.74e-01 | 20260505 09:52:27.997 | INFO | PerfReport: 3.32%, Histogram -- 1.08e+00 / 1.48e+00, 2.09e-01 / 1.04e-01 | 20260505 09:52:27.997 | INFO | PerfReport: 15.65%, DME -- 4.07e+01 / 4.51e+01, 4.60e+00 / 2.44e+00 | 20260505 09:52:27.997 | INFO | PerfReport: 56.62%, FrameLabeling -- 1.84e+01 / 2.15e+01, 1.86e+00 / 2.18e+00 | 20260505 09:52:27.997 | INFO | PerfReport: 7.34%, PulseAccumulating -- 2.38e+00 / 2.98e+00, 4.00e-01 / 1.93e-01 | 20260505 09:52:27.997 | INFO | PerfReport: 3.78%, Metrics -- 1.23e+00 / 1.52e+00, 3.55e-01 / 2.22e-01 | 20260505 09:52:27.997 | INFO | PerfReport: 1.34%, PulseSerializing -- 4.37e-01 / 4.81e-01, 8.23e-02 / 3.26e-02 | 20260505 09:52:27.997 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 09:52:27.997 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:52:27.997 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:52:27.997 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260505 09:52:27.997 | INFO | PerfReport: 22.65%, Upload -- 2.30e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260505 09:52:27.998 | INFO | PerfReport: 21.80%, Download -- 2.21e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260505 09:52:27.998 | INFO | PerfReport: 31.97%, Compute -- 3.25e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 09:52:27.998 | INFO | PerfReport: 1.92%, LockUpload -- 1.95e+00 / 0.00e+00, 7.47e+00 / 0.00e+00 | 20260505 09:52:27.998 | INFO | PerfReport: 1.07%, LockDownload -- 1.09e+00 / 0.00e+00, 2.62e+00 / 0.00e+00 | 20260505 09:52:27.998 | INFO | PerfReport: 20.57%, LockCompute -- 2.09e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260505 09:52:27.998 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.63e-02 / 0.00e+00 | 20260505 09:52:28.125 | INFO | Starting a round of Baz IO | 20260505 09:52:31.387 | INFO | Populating RTMetrics report | 20260505 09:52:31.388 | INFO | RTMetrics report populated | 20260505 09:52:31.388 | INFO | Opening tmp RTMetrics json file | 20260505 09:52:31.411 | INFO | Serializing RTMetrics to json | 20260505 09:52:31.412 | INFO | Serialization done, swapping tmp file now... | 20260505 09:52:31.412 | INFO | RTMetrics done | 20260505 09:52:31.529 | INFO | Finished a round of Baz IO: Wrote 2.43 GiB in 3.403 seconds | 20260505 09:52:31.529 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.58% / 0.01% / 74.41% | 20260505 09:52:31.529 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.75931% / 7.94729e-06% / 4.37101e-05% | 20260505 09:52:31.529 | INFO | Baz IO overall: | 20260505 09:52:31.529 | INFO | BytesWritten : 1324097752270 | 20260505 09:52:31.529 | INFO | NumEvents : 630280600121 | 20260505 09:52:33.135 | INFO | Analyzing chunk frames = [2277376,2277888) | 20260505 09:52:33.135 | INFO | MemInfo: { | 20260505 09:52:33.135 | INFO | "gpu" : " 17 GB", | 20260505 09:52:33.135 | INFO | "huge" : "", | 20260505 09:52:33.135 | INFO | "huge_shared" : " 27 GB", | 20260505 09:52:33.135 | INFO | "malloc" : " 14 GB", | 20260505 09:52:33.135 | INFO | "pinned" : " 29 GB" | 20260505 09:52:33.135 | INFO | } | 20260505 09:52:33.135 | INFO | Perf report for the last 5.13906 seconds: | 20260505 09:52:33.135 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:52:33.135 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:33.135 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:33.135 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:33.135 | INFO | ANALYSIS : 64.9%, 2.9, 99.2ms, 0 | 20260505 09:52:33.135 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:33.135 | INFO | RT_METRICS : 46.2%, 1.0, 24.2ms, 0 | 20260505 09:52:33.135 | INFO | PRE_HQ : 63.0%, 1.8, 60.3ms, 0 | 20260505 09:52:33.135 | INFO | BAZWRITER : 65.5%, 4.2, 141.8ms, 0 | 20260505 09:52:33.135 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260505 09:52:33.135 | INFO | Uploaded 36GiB to GPU in 2.25862s (15.9389GiB/s) | 20260505 09:52:33.136 | INFO | Downloaded 38GiB to GPU in 2.37311s (16.0128GiB/s) | 20260505 09:52:38.274 | INFO | Analyzing chunk frames = [2277888,2278400) | 20260505 09:52:38.274 | INFO | MemInfo: { | 20260505 09:52:38.274 | INFO | "gpu" : " 17 GB", | 20260505 09:52:38.274 | INFO | "huge" : "", | 20260505 09:52:38.274 | INFO | "huge_shared" : " 27 GB", | 20260505 09:52:38.274 | INFO | "malloc" : " 14 GB", | 20260505 09:52:38.274 | INFO | "pinned" : " 29 GB" | 20260505 09:52:38.274 | INFO | } | 20260505 09:52:38.274 | INFO | Perf report for the last 5.13944 seconds: | 20260505 09:52:38.274 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:52:38.275 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:38.275 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:38.275 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:38.275 | INFO | ANALYSIS : 64.5%, 3.0, 100.7ms, 0 | 20260505 09:52:38.275 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:38.275 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:38.275 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 09:52:38.275 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:38.275 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:38.275 | INFO | Uploaded 36GiB to GPU in 2.21974s (16.2181GiB/s) | 20260505 09:52:38.275 | INFO | Downloaded 34GiB to GPU in 2.10459s (16.1551GiB/s) | 20260505 09:52:43.414 | INFO | Analyzing chunk frames = [2278400,2278912) | 20260505 09:52:43.414 | INFO | MemInfo: { | 20260505 09:52:43.414 | INFO | "gpu" : " 17 GB", | 20260505 09:52:43.414 | INFO | "huge" : "", | 20260505 09:52:43.414 | INFO | "huge_shared" : " 27 GB", | 20260505 09:52:43.414 | INFO | "malloc" : " 14 GB", | 20260505 09:52:43.414 | INFO | "pinned" : " 29 GB" | 20260505 09:52:43.414 | INFO | } | 20260505 09:52:43.414 | INFO | Perf report for the last 5.13989 seconds: | 20260505 09:52:43.414 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:52:43.414 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:43.414 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:43.414 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:43.414 | INFO | ANALYSIS : 63.2%, 3.0, 98.4ms, 0 | 20260505 09:52:43.414 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:43.414 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:43.414 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 09:52:43.415 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:43.415 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:43.415 | INFO | Uploaded 36GiB to GPU in 2.15586s (16.6987GiB/s) | 20260505 09:52:43.415 | INFO | Downloaded 34GiB to GPU in 2.06494s (16.4654GiB/s) | 20260505 09:52:48.553 | INFO | Analyzing chunk frames = [2278912,2279424) | 20260505 09:52:48.553 | INFO | MemInfo: { | 20260505 09:52:48.554 | INFO | "gpu" : " 17 GB", | 20260505 09:52:48.554 | INFO | "huge" : "", | 20260505 09:52:48.554 | INFO | "huge_shared" : " 27 GB", | 20260505 09:52:48.554 | INFO | "malloc" : " 14 GB", | 20260505 09:52:48.554 | INFO | "pinned" : " 29 GB" | 20260505 09:52:48.554 | INFO | } | 20260505 09:52:48.554 | INFO | Perf report for the last 5.13942 seconds: | 20260505 09:52:48.554 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:52:48.554 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:48.554 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:48.554 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:48.554 | INFO | ANALYSIS : 63.1%, 3.0, 98.0ms, 0 | 20260505 09:52:48.554 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:48.554 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:48.554 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 09:52:48.554 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:48.554 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:48.554 | INFO | Uploaded 36GiB to GPU in 2.15785s (16.6833GiB/s) | 20260505 09:52:48.554 | INFO | Downloaded 34GiB to GPU in 2.07589s (16.3785GiB/s) | 20260505 09:52:53.692 | INFO | Analyzing chunk frames = [2279424,2279936) | 20260505 09:52:53.692 | INFO | MemInfo: { | 20260505 09:52:53.692 | INFO | "gpu" : " 17 GB", | 20260505 09:52:53.692 | INFO | "huge" : "", | 20260505 09:52:53.692 | INFO | "huge_shared" : " 27 GB", | 20260505 09:52:53.692 | INFO | "malloc" : " 14 GB", | 20260505 09:52:53.692 | INFO | "pinned" : " 29 GB" | 20260505 09:52:53.692 | INFO | } | 20260505 09:52:53.692 | INFO | Perf report for the last 5.13828 seconds: | 20260505 09:52:53.692 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:52:53.692 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:53.692 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:53.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:53.692 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 09:52:53.692 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:53.692 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:53.692 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:52:53.692 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:53.692 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:53.692 | INFO | Uploaded 36GiB to GPU in 2.23383s (16.1158GiB/s) | 20260505 09:52:53.692 | INFO | Downloaded 34GiB to GPU in 2.10655s (16.1402GiB/s) | 20260505 09:52:58.833 | INFO | Analyzing chunk frames = [2279936,2280448) | 20260505 09:52:58.833 | INFO | MemInfo: { | 20260505 09:52:58.833 | INFO | "gpu" : " 17 GB", | 20260505 09:52:58.833 | INFO | "huge" : "", | 20260505 09:52:58.833 | INFO | "huge_shared" : " 27 GB", | 20260505 09:52:58.833 | INFO | "malloc" : " 14 GB", | 20260505 09:52:58.833 | INFO | "pinned" : " 29 GB" | 20260505 09:52:58.833 | INFO | } | 20260505 09:52:58.833 | INFO | Perf report for the last 5.14087 seconds: | 20260505 09:52:58.833 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:52:58.833 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:58.833 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:58.833 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:58.833 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 09:52:58.833 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:58.833 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:58.833 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 09:52:58.833 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:58.833 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:52:58.833 | INFO | Uploaded 36GiB to GPU in 2.19252s (16.4195GiB/s) | 20260505 09:52:58.833 | INFO | Downloaded 34GiB to GPU in 2.09373s (16.239GiB/s) | 20260505 09:53:03.971 | INFO | Analyzing chunk frames = [2280448,2280960) | 20260505 09:53:03.971 | INFO | MemInfo: { | 20260505 09:53:03.971 | INFO | "gpu" : " 17 GB", | 20260505 09:53:03.971 | INFO | "huge" : "", | 20260505 09:53:03.971 | INFO | "huge_shared" : " 27 GB", | 20260505 09:53:03.971 | INFO | "malloc" : " 14 GB", | 20260505 09:53:03.971 | INFO | "pinned" : " 29 GB" | 20260505 09:53:03.971 | INFO | } | 20260505 09:53:03.971 | INFO | Perf report for the last 5.13825 seconds: | 20260505 09:53:03.971 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:53:03.971 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:03.971 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:03.971 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:03.971 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 09:53:03.971 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:03.971 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:03.971 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 09:53:03.971 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:03.971 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:03.972 | INFO | Uploaded 36GiB to GPU in 2.19046s (16.4349GiB/s) | 20260505 09:53:03.972 | INFO | Downloaded 34GiB to GPU in 2.10978s (16.1154GiB/s) | 20260505 09:53:09.110 | INFO | Analyzing chunk frames = [2280960,2281472) | 20260505 09:53:09.110 | INFO | MemInfo: { | 20260505 09:53:09.110 | INFO | "gpu" : " 17 GB", | 20260505 09:53:09.110 | INFO | "huge" : "", | 20260505 09:53:09.110 | INFO | "huge_shared" : " 27 GB", | 20260505 09:53:09.110 | INFO | "malloc" : " 14 GB", | 20260505 09:53:09.110 | INFO | "pinned" : " 29 GB" | 20260505 09:53:09.110 | INFO | } | 20260505 09:53:09.110 | INFO | Perf report for the last 5.13871 seconds: | 20260505 09:53:09.110 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:53:09.110 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:09.110 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:09.110 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:09.110 | INFO | ANALYSIS : 64.0%, 3.0, 98.9ms, 0 | 20260505 09:53:09.110 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:09.110 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:09.110 | INFO | PRE_HQ : 29.3%, 1.0, 15.1ms, 0 | 20260505 09:53:09.110 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:09.110 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:09.110 | INFO | Uploaded 36GiB to GPU in 2.19437s (16.4056GiB/s) | 20260505 09:53:09.110 | INFO | Downloaded 34GiB to GPU in 2.05614s (16.5358GiB/s) | 20260505 09:53:09.243 | INFO | Starting a round of Baz IO | 20260505 09:53:12.518 | INFO | Populating RTMetrics report | 20260505 09:53:12.519 | INFO | RTMetrics report populated | 20260505 09:53:12.519 | INFO | Opening tmp RTMetrics json file | 20260505 09:53:12.538 | INFO | Serializing RTMetrics to json | 20260505 09:53:12.587 | INFO | Serialization done, swapping tmp file now... | 20260505 09:53:12.591 | INFO | RTMetrics done | 20260505 09:53:12.672 | INFO | Finished a round of Baz IO: Wrote 2.43 GiB in 3.428 seconds | 20260505 09:53:12.672 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.56% / 0.01% / 74.42% | 20260505 09:53:12.672 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.75215% / 1.98682e-05% / 7.54992e-05% | 20260505 09:53:12.672 | INFO | Baz IO overall: | 20260505 09:53:12.672 | INFO | BytesWritten : 1326707189035 | 20260505 09:53:12.672 | INFO | NumEvents : 631526242500 | 20260505 09:53:14.249 | INFO | Analyzing chunk frames = [2281472,2281984) | 20260505 09:53:14.249 | INFO | MemInfo: { | 20260505 09:53:14.249 | INFO | "gpu" : " 17 GB", | 20260505 09:53:14.249 | INFO | "huge" : "", | 20260505 09:53:14.249 | INFO | "huge_shared" : " 27 GB", | 20260505 09:53:14.249 | INFO | "malloc" : " 14 GB", | 20260505 09:53:14.249 | INFO | "pinned" : " 29 GB" | 20260505 09:53:14.250 | INFO | } | 20260505 09:53:14.250 | INFO | Perf report for the last 5.13958 seconds: | 20260505 09:53:14.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:53:14.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:14.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:14.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:14.250 | INFO | ANALYSIS : 65.2%, 2.9, 100.0ms, 0 | 20260505 09:53:14.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:14.250 | INFO | RT_METRICS : 46.9%, 1.0, 24.6ms, 0 | 20260505 09:53:14.250 | INFO | PRE_HQ : 61.8%, 1.9, 59.5ms, 0 | 20260505 09:53:14.250 | INFO | BAZWRITER : 65.9%, 4.1, 140.6ms, 0 | 20260505 09:53:14.250 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260505 09:53:14.250 | INFO | Uploaded 36GiB to GPU in 2.19581s (16.3948GiB/s) | 20260505 09:53:14.250 | INFO | Downloaded 38GiB to GPU in 2.30705s (16.4713GiB/s) | 20260505 09:53:19.390 | INFO | Analyzing chunk frames = [2281984,2282496) | 20260505 09:53:19.390 | INFO | MemInfo: { | 20260505 09:53:19.390 | INFO | "gpu" : " 17 GB", | 20260505 09:53:19.390 | INFO | "huge" : "", | 20260505 09:53:19.390 | INFO | "huge_shared" : " 27 GB", | 20260505 09:53:19.390 | INFO | "malloc" : " 14 GB", | 20260505 09:53:19.390 | INFO | "pinned" : " 29 GB" | 20260505 09:53:19.390 | INFO | } | 20260505 09:53:19.390 | INFO | Perf report for the last 5.1403 seconds: | 20260505 09:53:19.390 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:53:19.390 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:19.390 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:19.390 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:19.390 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 09:53:19.390 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:19.390 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:19.390 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 09:53:19.390 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:19.390 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:19.390 | INFO | Uploaded 36GiB to GPU in 2.18022s (16.5121GiB/s) | 20260505 09:53:19.390 | INFO | Downloaded 34GiB to GPU in 2.0992s (16.1967GiB/s) | 20260505 09:53:24.528 | INFO | Analyzing chunk frames = [2282496,2283008) | 20260505 09:53:24.528 | INFO | MemInfo: { | 20260505 09:53:24.528 | INFO | "gpu" : " 17 GB", | 20260505 09:53:24.528 | INFO | "huge" : "", | 20260505 09:53:24.528 | INFO | "huge_shared" : " 27 GB", | 20260505 09:53:24.529 | INFO | "malloc" : " 14 GB", | 20260505 09:53:24.529 | INFO | "pinned" : " 29 GB" | 20260505 09:53:24.529 | INFO | } | 20260505 09:53:24.529 | INFO | Perf report for the last 5.13878 seconds: | 20260505 09:53:24.529 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:53:24.529 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:24.529 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:24.529 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:24.529 | INFO | ANALYSIS : 63.1%, 3.0, 98.2ms, 0 | 20260505 09:53:24.529 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:24.529 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:24.529 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 09:53:24.529 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:24.529 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:24.529 | INFO | Uploaded 36GiB to GPU in 2.14921s (16.7503GiB/s) | 20260505 09:53:24.529 | INFO | Downloaded 34GiB to GPU in 2.06088s (16.4978GiB/s) | 20260505 09:53:29.667 | INFO | Analyzing chunk frames = [2283008,2283520) | 20260505 09:53:29.667 | INFO | MemInfo: { | 20260505 09:53:29.667 | INFO | "gpu" : " 17 GB", | 20260505 09:53:29.667 | INFO | "huge" : "", | 20260505 09:53:29.668 | INFO | "huge_shared" : " 27 GB", | 20260505 09:53:29.668 | INFO | "malloc" : " 14 GB", | 20260505 09:53:29.668 | INFO | "pinned" : " 29 GB" | 20260505 09:53:29.668 | INFO | } | 20260505 09:53:29.668 | INFO | Perf report for the last 5.13897 seconds: | 20260505 09:53:29.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:53:29.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:29.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:29.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:29.668 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 09:53:29.668 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:29.668 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:29.668 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 09:53:29.668 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:29.668 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:29.668 | INFO | Uploaded 36GiB to GPU in 2.16112s (16.6581GiB/s) | 20260505 09:53:29.668 | INFO | Downloaded 34GiB to GPU in 2.07165s (16.412GiB/s) | 20260505 09:53:34.807 | INFO | Analyzing chunk frames = [2283520,2284032) | 20260505 09:53:34.807 | INFO | MemInfo: { | 20260505 09:53:34.807 | INFO | "gpu" : " 17 GB", | 20260505 09:53:34.807 | INFO | "huge" : "", | 20260505 09:53:34.807 | INFO | "huge_shared" : " 27 GB", | 20260505 09:53:34.807 | INFO | "malloc" : " 14 GB", | 20260505 09:53:34.807 | INFO | "pinned" : " 29 GB" | 20260505 09:53:34.807 | INFO | } | 20260505 09:53:34.807 | INFO | Perf report for the last 5.13975 seconds: | 20260505 09:53:34.807 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:53:34.807 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:34.807 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:34.807 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:34.807 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 09:53:34.807 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:34.808 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:34.808 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 09:53:34.808 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:34.808 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:34.808 | INFO | Uploaded 36GiB to GPU in 2.23543s (16.1043GiB/s) | 20260505 09:53:34.808 | INFO | Downloaded 34GiB to GPU in 2.10468s (16.1545GiB/s) | 20260505 09:53:39.947 | INFO | Analyzing chunk frames = [2284032,2284544) | 20260505 09:53:39.947 | INFO | MemInfo: { | 20260505 09:53:39.947 | INFO | "gpu" : " 17 GB", | 20260505 09:53:39.947 | INFO | "huge" : "", | 20260505 09:53:39.947 | INFO | "huge_shared" : " 27 GB", | 20260505 09:53:39.948 | INFO | "malloc" : " 14 GB", | 20260505 09:53:39.948 | INFO | "pinned" : " 29 GB" | 20260505 09:53:39.948 | INFO | } | 20260505 09:53:39.948 | INFO | Perf report for the last 5.14017 seconds: | 20260505 09:53:39.948 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:53:39.948 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:39.948 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:39.948 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:39.948 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 09:53:39.948 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:39.948 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:39.948 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 09:53:39.948 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:39.948 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:39.948 | INFO | Uploaded 36GiB to GPU in 2.23933s (16.0762GiB/s) | 20260505 09:53:39.948 | INFO | Downloaded 34GiB to GPU in 2.09948s (16.1945GiB/s) | 20260505 09:53:45.086 | INFO | Analyzing chunk frames = [2284544,2285056) | 20260505 09:53:45.086 | INFO | MemInfo: { | 20260505 09:53:45.086 | INFO | "gpu" : " 17 GB", | 20260505 09:53:45.086 | INFO | "huge" : "", | 20260505 09:53:45.086 | INFO | "huge_shared" : " 27 GB", | 20260505 09:53:45.086 | INFO | "malloc" : " 14 GB", | 20260505 09:53:45.086 | INFO | "pinned" : " 29 GB" | 20260505 09:53:45.086 | INFO | } | 20260505 09:53:45.086 | INFO | Perf report for the last 5.13852 seconds: | 20260505 09:53:45.086 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:53:45.086 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:45.086 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:45.086 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:45.086 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 09:53:45.086 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:45.086 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:45.086 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260505 09:53:45.086 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:45.086 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:45.086 | INFO | Uploaded 36GiB to GPU in 2.20544s (16.3233GiB/s) | 20260505 09:53:45.086 | INFO | Downloaded 34GiB to GPU in 2.10999s (16.1138GiB/s) | 20260505 09:53:50.225 | INFO | Analyzing chunk frames = [2285056,2285568) | 20260505 09:53:50.225 | INFO | MemInfo: { | 20260505 09:53:50.225 | INFO | "gpu" : " 17 GB", | 20260505 09:53:50.225 | INFO | "huge" : "", | 20260505 09:53:50.225 | INFO | "huge_shared" : " 27 GB", | 20260505 09:53:50.225 | INFO | "malloc" : " 14 GB", | 20260505 09:53:50.225 | INFO | "pinned" : " 29 GB" | 20260505 09:53:50.225 | INFO | } | 20260505 09:53:50.225 | INFO | Perf report for the last 5.13875 seconds: | 20260505 09:53:50.225 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:53:50.225 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:50.225 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:50.225 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:50.225 | INFO | ANALYSIS : 64.7%, 3.0, 101.0ms, 0 | 20260505 09:53:50.225 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:50.225 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:50.225 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 09:53:50.225 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:50.225 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:50.225 | INFO | Uploaded 36GiB to GPU in 2.24734s (16.0189GiB/s) | 20260505 09:53:50.225 | INFO | Downloaded 34GiB to GPU in 2.08095s (16.3387GiB/s) | 20260505 09:53:50.225 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 09:53:50.225 | INFO | Total mem usage: 15.3 GiB | 20260505 09:53:50.225 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 09:53:50.225 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.38%) | 20260505 09:53:50.225 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 09:53:50.225 | INFO | High Water Breakdown: | 20260505 09:53:50.225 | INFO | 61% : 8.79 GiB ( 94% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 09:53:50.226 | INFO | 38.8% : 5.59 GiB ( 84.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 09:53:50.226 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 09:53:50.226 | INFO | ------------------------------------------ | 20260505 09:53:50.226 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:53:50.226 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 09:53:50.226 | INFO | Total mem usage: 26.5 GiB | 20260505 09:53:50.226 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 09:53:50.226 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 09:53:50.226 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 09:53:50.226 | INFO | High Water Breakdown: | 20260505 09:53:50.226 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:53:50.226 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:53:50.226 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:53:50.226 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:53:50.226 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:53:50.226 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:53:50.226 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 09:53:50.226 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 09:53:50.226 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:53:50.226 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 09:53:50.226 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 09:53:50.226 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 09:53:50.226 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 09:53:50.226 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 09:53:50.226 | INFO | ------------------------------------------ | 20260505 09:53:50.226 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:53:50.226 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 09:53:50.226 | INFO | Total mem usage: 25.6 GiB | 20260505 09:53:50.226 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 09:53:50.226 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 09:53:50.226 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 09:53:50.226 | INFO | High Water Breakdown: | 20260505 09:53:50.226 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 09:53:50.226 | INFO | ------------------------------------------ | 20260505 09:53:50.226 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:53:50.226 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 09:53:50.226 | INFO | Total mem usage: 15.7 GiB | 20260505 09:53:50.226 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 09:53:50.226 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 09:53:50.226 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 09:53:50.226 | INFO | High Water Breakdown: | 20260505 09:53:50.226 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 09:53:50.226 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 09:53:50.226 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 09:53:50.226 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 09:53:50.226 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 09:53:50.226 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 09:53:50.226 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 09:53:50.226 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 09:53:50.226 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 09:53:50.226 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:53:50.226 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 09:53:50.226 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:53:50.226 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 09:53:50.226 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 09:53:50.226 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 09:53:50.226 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 09:53:50.226 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:53:50.226 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 09:53:50.226 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:53:50.226 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:53:50.226 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:53:50.226 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 09:53:50.226 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 09:53:50.226 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:53:50.226 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 09:53:50.226 | INFO | ------------------------------------------ | 20260505 09:53:50.226 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:53:50.226 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 09:53:50.226 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:53:50.226 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:53:50.226 | INFO | PerfReport: Full Execution -- 3.25e+01 / 7.74e+01, 1.38e+01 / 2.52e+00 | 20260505 09:53:50.226 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 1.03e-01, 1.03e-02 / 2.34e-03 | 20260505 09:53:50.226 | INFO | PerfReport: 11.64%, Baseline -- 3.78e+00 / 4.46e+00, 6.06e-01 / 2.25e-01 | 20260505 09:53:50.226 | INFO | PerfReport: 3.31%, Histogram -- 1.07e+00 / 1.49e+00, 2.07e-01 / 7.78e-02 | 20260505 09:53:50.226 | INFO | PerfReport: 15.64%, DME -- 4.06e+01 / 4.51e+01, 4.46e+00 / 1.93e+00 | 20260505 09:53:50.226 | INFO | PerfReport: 56.63%, FrameLabeling -- 1.84e+01 / 2.12e+01, 1.86e+00 / 2.09e+00 | 20260505 09:53:50.226 | INFO | PerfReport: 7.36%, PulseAccumulating -- 2.39e+00 / 2.99e+00, 4.07e-01 / 1.78e-01 | 20260505 09:53:50.226 | INFO | PerfReport: 3.79%, Metrics -- 1.23e+00 / 1.53e+00, 3.60e-01 / 1.56e-01 | 20260505 09:53:50.226 | INFO | PerfReport: 1.34%, PulseSerializing -- 4.35e-01 / 4.84e-01, 8.19e-02 / 2.69e-02 | 20260505 09:53:50.226 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 09:53:50.226 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:53:50.226 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:53:50.226 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260505 09:53:50.226 | INFO | PerfReport: 22.58%, Upload -- 2.29e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260505 09:53:50.226 | INFO | PerfReport: 21.75%, Download -- 2.21e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260505 09:53:50.226 | INFO | PerfReport: 31.99%, Compute -- 3.25e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 09:53:50.226 | INFO | PerfReport: 1.93%, LockUpload -- 1.96e+00 / 0.00e+00, 7.51e+00 / 0.00e+00 | 20260505 09:53:50.226 | INFO | PerfReport: 1.05%, LockDownload -- 1.07e+00 / 0.00e+00, 2.61e+00 / 0.00e+00 | 20260505 09:53:50.227 | INFO | PerfReport: 20.69%, LockCompute -- 2.10e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260505 09:53:50.227 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.53e-02 / 0.00e+00 | 20260505 09:53:50.356 | INFO | Starting a round of Baz IO | 20260505 09:53:53.631 | INFO | Populating RTMetrics report | 20260505 09:53:53.632 | INFO | RTMetrics report populated | 20260505 09:53:53.632 | INFO | Opening tmp RTMetrics json file | 20260505 09:53:53.665 | INFO | Serializing RTMetrics to json | 20260505 09:53:53.668 | INFO | Serialization done, swapping tmp file now... | 20260505 09:53:53.669 | INFO | RTMetrics done | 20260505 09:53:53.787 | INFO | Finished a round of Baz IO: Wrote 2.429 GiB in 3.431 seconds | 20260505 09:53:53.787 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.55% / 0.01% / 74.44% | 20260505 09:53:53.787 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.7464% / 1.19209e-05% / 6.75519e-05% | 20260505 09:53:53.787 | INFO | Baz IO overall: | 20260505 09:53:53.787 | INFO | BytesWritten : 1329315790672 | 20260505 09:53:53.787 | INFO | NumEvents : 632771481709 | 20260505 09:53:55.364 | INFO | Analyzing chunk frames = [2285568,2286080) | 20260505 09:53:55.364 | INFO | MemInfo: { | 20260505 09:53:55.364 | INFO | "gpu" : " 17 GB", | 20260505 09:53:55.364 | INFO | "huge" : "", | 20260505 09:53:55.364 | INFO | "huge_shared" : " 27 GB", | 20260505 09:53:55.364 | INFO | "malloc" : " 14 GB", | 20260505 09:53:55.364 | INFO | "pinned" : " 29 GB" | 20260505 09:53:55.364 | INFO | } | 20260505 09:53:55.364 | INFO | Perf report for the last 5.13906 seconds: | 20260505 09:53:55.364 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:53:55.364 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:55.364 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:55.364 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:55.364 | INFO | ANALYSIS : 65.2%, 2.9, 99.8ms, 0 | 20260505 09:53:55.364 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:53:55.364 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260505 09:53:55.364 | INFO | PRE_HQ : 62.6%, 1.8, 59.9ms, 0 | 20260505 09:53:55.364 | INFO | BAZWRITER : 65.8%, 4.2, 141.7ms, 0 | 20260505 09:53:55.364 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 09:53:55.364 | INFO | Uploaded 36GiB to GPU in 2.25799s (15.9434GiB/s) | 20260505 09:53:55.364 | INFO | Downloaded 38GiB to GPU in 2.35131s (16.1612GiB/s) | 20260505 09:54:00.503 | INFO | Analyzing chunk frames = [2286080,2286592) | 20260505 09:54:00.503 | INFO | MemInfo: { | 20260505 09:54:00.503 | INFO | "gpu" : " 17 GB", | 20260505 09:54:00.503 | INFO | "huge" : "", | 20260505 09:54:00.504 | INFO | "huge_shared" : " 27 GB", | 20260505 09:54:00.504 | INFO | "malloc" : " 14 GB", | 20260505 09:54:00.504 | INFO | "pinned" : " 29 GB" | 20260505 09:54:00.504 | INFO | } | 20260505 09:54:00.504 | INFO | Perf report for the last 5.13976 seconds: | 20260505 09:54:00.504 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:54:00.504 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:00.504 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:00.504 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:00.504 | INFO | ANALYSIS : 64.4%, 3.0, 100.6ms, 0 | 20260505 09:54:00.504 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:00.504 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:00.504 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 09:54:00.504 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:00.504 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:00.504 | INFO | Uploaded 36GiB to GPU in 2.21492s (16.2534GiB/s) | 20260505 09:54:00.504 | INFO | Downloaded 34GiB to GPU in 2.10658s (16.1399GiB/s) | 20260505 09:54:05.643 | INFO | Analyzing chunk frames = [2286592,2287104) | 20260505 09:54:05.643 | INFO | MemInfo: { | 20260505 09:54:05.643 | INFO | "gpu" : " 17 GB", | 20260505 09:54:05.643 | INFO | "huge" : "", | 20260505 09:54:05.643 | INFO | "huge_shared" : " 27 GB", | 20260505 09:54:05.644 | INFO | "malloc" : " 14 GB", | 20260505 09:54:05.644 | INFO | "pinned" : " 29 GB" | 20260505 09:54:05.644 | INFO | } | 20260505 09:54:05.644 | INFO | Perf report for the last 5.13993 seconds: | 20260505 09:54:05.644 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:54:05.644 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:05.644 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:05.644 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:05.644 | INFO | ANALYSIS : 63.1%, 3.0, 98.2ms, 0 | 20260505 09:54:05.644 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:05.644 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:05.644 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 09:54:05.644 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:05.644 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:05.644 | INFO | Uploaded 36GiB to GPU in 2.1617s (16.6535GiB/s) | 20260505 09:54:05.644 | INFO | Downloaded 34GiB to GPU in 2.06219s (16.4873GiB/s) | 20260505 09:54:10.782 | INFO | Analyzing chunk frames = [2287104,2287616) | 20260505 09:54:10.782 | INFO | MemInfo: { | 20260505 09:54:10.782 | INFO | "gpu" : " 17 GB", | 20260505 09:54:10.782 | INFO | "huge" : "", | 20260505 09:54:10.782 | INFO | "huge_shared" : " 27 GB", | 20260505 09:54:10.782 | INFO | "malloc" : " 14 GB", | 20260505 09:54:10.782 | INFO | "pinned" : " 29 GB" | 20260505 09:54:10.782 | INFO | } | 20260505 09:54:10.782 | INFO | Perf report for the last 5.1385 seconds: | 20260505 09:54:10.782 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:54:10.782 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:10.782 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:10.782 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:10.782 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 09:54:10.782 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:10.782 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:10.782 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 09:54:10.782 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:10.782 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:10.782 | INFO | Uploaded 36GiB to GPU in 2.17389s (16.5602GiB/s) | 20260505 09:54:10.782 | INFO | Downloaded 34GiB to GPU in 2.07097s (16.4175GiB/s) | 20260505 09:54:15.922 | INFO | Analyzing chunk frames = [2287616,2288128) | 20260505 09:54:15.922 | INFO | MemInfo: { | 20260505 09:54:15.922 | INFO | "gpu" : " 17 GB", | 20260505 09:54:15.922 | INFO | "huge" : "", | 20260505 09:54:15.922 | INFO | "huge_shared" : " 27 GB", | 20260505 09:54:15.922 | INFO | "malloc" : " 14 GB", | 20260505 09:54:15.922 | INFO | "pinned" : " 29 GB" | 20260505 09:54:15.922 | INFO | } | 20260505 09:54:15.922 | INFO | Perf report for the last 5.13961 seconds: | 20260505 09:54:15.922 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:54:15.922 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:15.922 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:15.922 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:15.922 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 09:54:15.922 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:15.922 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:15.922 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 09:54:15.922 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:15.922 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:15.922 | INFO | Uploaded 36GiB to GPU in 2.24007s (16.0709GiB/s) | 20260505 09:54:15.922 | INFO | Downloaded 34GiB to GPU in 2.10351s (16.1635GiB/s) | 20260505 09:54:21.061 | INFO | Analyzing chunk frames = [2288128,2288640) | 20260505 09:54:21.061 | INFO | MemInfo: { | 20260505 09:54:21.061 | INFO | "gpu" : " 17 GB", | 20260505 09:54:21.061 | INFO | "huge" : "", | 20260505 09:54:21.061 | INFO | "huge_shared" : " 27 GB", | 20260505 09:54:21.061 | INFO | "malloc" : " 14 GB", | 20260505 09:54:21.061 | INFO | "pinned" : " 29 GB" | 20260505 09:54:21.061 | INFO | } | 20260505 09:54:21.061 | INFO | Perf report for the last 5.13949 seconds: | 20260505 09:54:21.061 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:54:21.061 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:21.061 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:21.061 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:21.061 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 09:54:21.061 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:21.061 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:21.061 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 09:54:21.061 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:21.061 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:21.062 | INFO | Uploaded 36GiB to GPU in 2.23942s (16.0756GiB/s) | 20260505 09:54:21.062 | INFO | Downloaded 34GiB to GPU in 2.10123s (16.181GiB/s) | 20260505 09:54:26.200 | INFO | Analyzing chunk frames = [2288640,2289152) | 20260505 09:54:26.200 | INFO | MemInfo: { | 20260505 09:54:26.200 | INFO | "gpu" : " 17 GB", | 20260505 09:54:26.200 | INFO | "huge" : "", | 20260505 09:54:26.200 | INFO | "huge_shared" : " 27 GB", | 20260505 09:54:26.200 | INFO | "malloc" : " 14 GB", | 20260505 09:54:26.201 | INFO | "pinned" : " 29 GB" | 20260505 09:54:26.201 | INFO | } | 20260505 09:54:26.201 | INFO | Perf report for the last 5.13929 seconds: | 20260505 09:54:26.201 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:54:26.201 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:26.201 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:26.201 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:26.201 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 09:54:26.201 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:26.201 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:26.201 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 09:54:26.201 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:26.201 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:26.201 | INFO | Uploaded 36GiB to GPU in 2.1795s (16.5175GiB/s) | 20260505 09:54:26.201 | INFO | Downloaded 34GiB to GPU in 2.14547s (15.8473GiB/s) | 20260505 09:54:31.340 | INFO | Analyzing chunk frames = [2289152,2289664) | 20260505 09:54:31.340 | INFO | MemInfo: { | 20260505 09:54:31.340 | INFO | "gpu" : " 17 GB", | 20260505 09:54:31.340 | INFO | "huge" : "", | 20260505 09:54:31.340 | INFO | "huge_shared" : " 27 GB", | 20260505 09:54:31.340 | INFO | "malloc" : " 14 GB", | 20260505 09:54:31.340 | INFO | "pinned" : " 29 GB" | 20260505 09:54:31.340 | INFO | } | 20260505 09:54:31.340 | INFO | Perf report for the last 5.13987 seconds: | 20260505 09:54:31.341 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:54:31.341 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:31.341 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:31.341 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:31.341 | INFO | ANALYSIS : 64.7%, 3.0, 101.0ms, 0 | 20260505 09:54:31.341 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:31.341 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:31.341 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 09:54:31.341 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:31.341 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:31.341 | INFO | Uploaded 36GiB to GPU in 2.25145s (15.9897GiB/s) | 20260505 09:54:31.341 | INFO | Downloaded 34GiB to GPU in 2.09335s (16.2419GiB/s) | 20260505 09:54:31.472 | INFO | Starting a round of Baz IO | 20260505 09:54:34.738 | INFO | Populating RTMetrics report | 20260505 09:54:34.738 | INFO | RTMetrics report populated | 20260505 09:54:34.738 | INFO | Opening tmp RTMetrics json file | 20260505 09:54:34.760 | INFO | Serializing RTMetrics to json | 20260505 09:54:34.773 | INFO | Serialization done, swapping tmp file now... | 20260505 09:54:34.773 | INFO | RTMetrics done | 20260505 09:54:34.884 | INFO | Finished a round of Baz IO: Wrote 2.43 GiB in 3.412 seconds | 20260505 09:54:34.884 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.54% / 0.01% / 74.45% | 20260505 09:54:34.884 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.74239% / 2.38419e-05% / 5.96046e-05% | 20260505 09:54:34.884 | INFO | Baz IO overall: | 20260505 09:54:34.884 | INFO | BytesWritten : 1331924858570 | 20260505 09:54:34.884 | INFO | NumEvents : 634017130989 | 20260505 09:54:36.480 | INFO | Analyzing chunk frames = [2289664,2290176) | 20260505 09:54:36.480 | INFO | MemInfo: { | 20260505 09:54:36.480 | INFO | "gpu" : " 17 GB", | 20260505 09:54:36.480 | INFO | "huge" : "", | 20260505 09:54:36.480 | INFO | "huge_shared" : " 27 GB", | 20260505 09:54:36.480 | INFO | "malloc" : " 14 GB", | 20260505 09:54:36.480 | INFO | "pinned" : " 29 GB" | 20260505 09:54:36.480 | INFO | } | 20260505 09:54:36.480 | INFO | Perf report for the last 5.13932 seconds: | 20260505 09:54:36.480 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:54:36.480 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:36.480 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:36.480 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:36.480 | INFO | ANALYSIS : 65.0%, 2.9, 99.5ms, 0 | 20260505 09:54:36.480 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:36.480 | INFO | RT_METRICS : 46.5%, 1.0, 24.4ms, 0 | 20260505 09:54:36.480 | INFO | PRE_HQ : 62.9%, 1.8, 59.8ms, 0 | 20260505 09:54:36.480 | INFO | BAZWRITER : 65.5%, 4.2, 139.9ms, 0 | 20260505 09:54:36.480 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 09:54:36.480 | INFO | Uploaded 36GiB to GPU in 2.25528s (15.9625GiB/s) | 20260505 09:54:36.480 | INFO | Downloaded 38GiB to GPU in 2.36053s (16.0981GiB/s) | 20260505 09:54:41.618 | INFO | Analyzing chunk frames = [2290176,2290688) | 20260505 09:54:41.618 | INFO | MemInfo: { | 20260505 09:54:41.618 | INFO | "gpu" : " 17 GB", | 20260505 09:54:41.618 | INFO | "huge" : "", | 20260505 09:54:41.618 | INFO | "huge_shared" : " 27 GB", | 20260505 09:54:41.618 | INFO | "malloc" : " 14 GB", | 20260505 09:54:41.618 | INFO | "pinned" : " 29 GB" | 20260505 09:54:41.618 | INFO | } | 20260505 09:54:41.618 | INFO | Perf report for the last 5.13816 seconds: | 20260505 09:54:41.618 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:54:41.618 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:41.618 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:41.618 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:41.618 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 09:54:41.618 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:41.618 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:41.618 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:54:41.618 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:41.618 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:41.618 | INFO | Uploaded 36GiB to GPU in 2.17593s (16.5447GiB/s) | 20260505 09:54:41.618 | INFO | Downloaded 34GiB to GPU in 2.09856s (16.2016GiB/s) | 20260505 09:54:46.758 | INFO | Analyzing chunk frames = [2290688,2291200) | 20260505 09:54:46.758 | INFO | MemInfo: { | 20260505 09:54:46.758 | INFO | "gpu" : " 17 GB", | 20260505 09:54:46.758 | INFO | "huge" : "", | 20260505 09:54:46.758 | INFO | "huge_shared" : " 27 GB", | 20260505 09:54:46.758 | INFO | "malloc" : " 14 GB", | 20260505 09:54:46.758 | INFO | "pinned" : " 29 GB" | 20260505 09:54:46.758 | INFO | } | 20260505 09:54:46.758 | INFO | Perf report for the last 5.1405 seconds: | 20260505 09:54:46.758 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:54:46.758 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:46.759 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:46.759 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:46.759 | INFO | ANALYSIS : 63.2%, 3.0, 98.3ms, 0 | 20260505 09:54:46.759 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:46.759 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:46.759 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:54:46.759 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:46.759 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:46.759 | INFO | Uploaded 36GiB to GPU in 2.14135s (16.8118GiB/s) | 20260505 09:54:46.759 | INFO | Downloaded 34GiB to GPU in 2.06624s (16.455GiB/s) | 20260505 09:54:51.898 | INFO | Analyzing chunk frames = [2291200,2291712) | 20260505 09:54:51.898 | INFO | MemInfo: { | 20260505 09:54:51.898 | INFO | "gpu" : " 17 GB", | 20260505 09:54:51.898 | INFO | "huge" : "", | 20260505 09:54:51.898 | INFO | "huge_shared" : " 27 GB", | 20260505 09:54:51.898 | INFO | "malloc" : " 14 GB", | 20260505 09:54:51.898 | INFO | "pinned" : " 29 GB" | 20260505 09:54:51.898 | INFO | } | 20260505 09:54:51.898 | INFO | Perf report for the last 5.13955 seconds: | 20260505 09:54:51.898 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:54:51.898 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:51.898 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:51.898 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:51.898 | INFO | ANALYSIS : 63.1%, 3.0, 98.0ms, 0 | 20260505 09:54:51.898 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:51.898 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:51.898 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:54:51.898 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:51.898 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:51.898 | INFO | Uploaded 36GiB to GPU in 2.15586s (16.6986GiB/s) | 20260505 09:54:51.898 | INFO | Downloaded 34GiB to GPU in 2.07239s (16.4062GiB/s) | 20260505 09:54:57.037 | INFO | Analyzing chunk frames = [2291712,2292224) | 20260505 09:54:57.037 | INFO | MemInfo: { | 20260505 09:54:57.037 | INFO | "gpu" : " 17 GB", | 20260505 09:54:57.037 | INFO | "huge" : "", | 20260505 09:54:57.037 | INFO | "huge_shared" : " 27 GB", | 20260505 09:54:57.037 | INFO | "malloc" : " 14 GB", | 20260505 09:54:57.037 | INFO | "pinned" : " 29 GB" | 20260505 09:54:57.037 | INFO | } | 20260505 09:54:57.037 | INFO | Perf report for the last 5.13909 seconds: | 20260505 09:54:57.037 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:54:57.037 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:57.037 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:57.037 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:57.037 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 09:54:57.037 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:57.037 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:57.037 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 09:54:57.037 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:57.037 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:54:57.037 | INFO | Uploaded 36GiB to GPU in 2.23537s (16.1047GiB/s) | 20260505 09:54:57.037 | INFO | Downloaded 34GiB to GPU in 2.10488s (16.1529GiB/s) | 20260505 09:55:02.178 | INFO | Analyzing chunk frames = [2292224,2292736) | 20260505 09:55:02.178 | INFO | MemInfo: { | 20260505 09:55:02.178 | INFO | "gpu" : " 17 GB", | 20260505 09:55:02.178 | INFO | "huge" : "", | 20260505 09:55:02.178 | INFO | "huge_shared" : " 27 GB", | 20260505 09:55:02.178 | INFO | "malloc" : " 14 GB", | 20260505 09:55:02.178 | INFO | "pinned" : " 29 GB" | 20260505 09:55:02.178 | INFO | } | 20260505 09:55:02.178 | INFO | Perf report for the last 5.14069 seconds: | 20260505 09:55:02.178 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:55:02.178 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:02.178 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:02.178 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:02.178 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 09:55:02.178 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:02.178 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:02.178 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:55:02.178 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:02.178 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:02.178 | INFO | Uploaded 36GiB to GPU in 2.23478s (16.1089GiB/s) | 20260505 09:55:02.178 | INFO | Downloaded 34GiB to GPU in 2.0993s (16.1959GiB/s) | 20260505 09:55:07.315 | INFO | Analyzing chunk frames = [2292736,2293248) | 20260505 09:55:07.315 | INFO | MemInfo: { | 20260505 09:55:07.315 | INFO | "gpu" : " 17 GB", | 20260505 09:55:07.315 | INFO | "huge" : "", | 20260505 09:55:07.315 | INFO | "huge_shared" : " 27 GB", | 20260505 09:55:07.315 | INFO | "malloc" : " 14 GB", | 20260505 09:55:07.315 | INFO | "pinned" : " 29 GB" | 20260505 09:55:07.315 | INFO | } | 20260505 09:55:07.316 | INFO | Perf report for the last 5.13762 seconds: | 20260505 09:55:07.316 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:55:07.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:07.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:07.316 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:07.316 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 09:55:07.316 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:07.316 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:07.316 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 09:55:07.316 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:07.316 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:07.316 | INFO | Uploaded 36GiB to GPU in 2.20444s (16.3307GiB/s) | 20260505 09:55:07.316 | INFO | Downloaded 34GiB to GPU in 2.12524s (15.9982GiB/s) | 20260505 09:55:12.455 | INFO | Analyzing chunk frames = [2293248,2293760) | 20260505 09:55:12.455 | INFO | MemInfo: { | 20260505 09:55:12.455 | INFO | "gpu" : " 17 GB", | 20260505 09:55:12.455 | INFO | "huge" : "", | 20260505 09:55:12.455 | INFO | "huge_shared" : " 27 GB", | 20260505 09:55:12.455 | INFO | "malloc" : " 14 GB", | 20260505 09:55:12.455 | INFO | "pinned" : " 29 GB" | 20260505 09:55:12.455 | INFO | } | 20260505 09:55:12.455 | INFO | Perf report for the last 5.13963 seconds: | 20260505 09:55:12.455 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:55:12.455 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:12.455 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:12.455 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:12.455 | INFO | ANALYSIS : 64.6%, 3.0, 100.9ms, 0 | 20260505 09:55:12.455 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:12.455 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:12.455 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:55:12.455 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:12.455 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:12.455 | INFO | Uploaded 36GiB to GPU in 2.24549s (16.0322GiB/s) | 20260505 09:55:12.455 | INFO | Downloaded 34GiB to GPU in 2.08977s (16.2697GiB/s) | 20260505 09:55:12.456 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 09:55:12.456 | INFO | Total mem usage: 15.3 GiB | 20260505 09:55:12.456 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 09:55:12.456 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.51%) | 20260505 09:55:12.456 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 09:55:12.456 | INFO | High Water Breakdown: | 20260505 09:55:12.456 | INFO | 61% : 8.78 GiB ( 93.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 09:55:12.456 | INFO | 38.9% : 5.59 GiB ( 84.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 09:55:12.456 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 09:55:12.456 | INFO | ------------------------------------------ | 20260505 09:55:12.456 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:55:12.456 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 09:55:12.456 | INFO | Total mem usage: 26.5 GiB | 20260505 09:55:12.456 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 09:55:12.456 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 09:55:12.456 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 09:55:12.456 | INFO | High Water Breakdown: | 20260505 09:55:12.456 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:55:12.456 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:55:12.456 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:55:12.456 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:55:12.456 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:55:12.456 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:55:12.456 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 09:55:12.456 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 09:55:12.456 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:55:12.456 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 09:55:12.456 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 09:55:12.456 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 09:55:12.456 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 09:55:12.456 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 09:55:12.456 | INFO | ------------------------------------------ | 20260505 09:55:12.456 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:55:12.456 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 09:55:12.456 | INFO | Total mem usage: 25.6 GiB | 20260505 09:55:12.456 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 09:55:12.456 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 09:55:12.456 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 09:55:12.456 | INFO | High Water Breakdown: | 20260505 09:55:12.456 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 09:55:12.456 | INFO | ------------------------------------------ | 20260505 09:55:12.456 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:55:12.456 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 09:55:12.456 | INFO | Total mem usage: 15.7 GiB | 20260505 09:55:12.456 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 09:55:12.456 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 09:55:12.456 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 09:55:12.456 | INFO | High Water Breakdown: | 20260505 09:55:12.456 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 09:55:12.456 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 09:55:12.456 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 09:55:12.456 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 09:55:12.456 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 09:55:12.456 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 09:55:12.456 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 09:55:12.456 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 09:55:12.456 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 09:55:12.456 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:55:12.456 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 09:55:12.456 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:55:12.456 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 09:55:12.456 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 09:55:12.456 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 09:55:12.456 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 09:55:12.456 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:55:12.456 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 09:55:12.456 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:55:12.456 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:55:12.456 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:55:12.456 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 09:55:12.456 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 09:55:12.456 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:55:12.456 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 09:55:12.456 | INFO | ------------------------------------------ | 20260505 09:55:12.457 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:55:12.457 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 09:55:12.457 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:55:12.457 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:55:12.457 | INFO | PerfReport: Full Execution -- 3.25e+01 / 7.76e+01, 1.37e+01 / 2.35e+00 | 20260505 09:55:12.457 | INFO | PerfReport: 0.29%, AggregateStats -- 9.32e-02 / 1.03e-01, 9.93e-03 / 3.41e-03 | 20260505 09:55:12.457 | INFO | PerfReport: 11.65%, Baseline -- 3.78e+00 / 4.43e+00, 6.15e-01 / 2.93e-01 | 20260505 09:55:12.457 | INFO | PerfReport: 3.31%, Histogram -- 1.07e+00 / 1.47e+00, 2.08e-01 / 1.11e-01 | 20260505 09:55:12.457 | INFO | PerfReport: 15.64%, DME -- 4.06e+01 / 4.50e+01, 4.57e+00 / 2.06e+00 | 20260505 09:55:12.457 | INFO | PerfReport: 56.66%, FrameLabeling -- 1.84e+01 / 2.16e+01, 1.89e+00 / 1.96e+00 | 20260505 09:55:12.457 | INFO | PerfReport: 7.31%, PulseAccumulating -- 2.38e+00 / 2.96e+00, 3.99e-01 / 2.27e-01 | 20260505 09:55:12.457 | INFO | PerfReport: 3.80%, Metrics -- 1.23e+00 / 1.53e+00, 3.60e-01 / 1.98e-01 | 20260505 09:55:12.457 | INFO | PerfReport: 1.34%, PulseSerializing -- 4.36e-01 / 4.79e-01, 8.24e-02 / 3.39e-02 | 20260505 09:55:12.457 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 09:55:12.457 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:55:12.457 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:55:12.457 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.39e+01 / 0.00e+00 | 20260505 09:55:12.457 | INFO | PerfReport: 22.66%, Upload -- 2.30e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260505 09:55:12.457 | INFO | PerfReport: 21.82%, Download -- 2.22e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260505 09:55:12.457 | INFO | PerfReport: 31.97%, Compute -- 3.25e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 09:55:12.457 | INFO | PerfReport: 1.90%, LockUpload -- 1.93e+00 / 0.00e+00, 7.44e+00 / 0.00e+00 | 20260505 09:55:12.457 | INFO | PerfReport: 1.07%, LockDownload -- 1.08e+00 / 0.00e+00, 2.62e+00 / 0.00e+00 | 20260505 09:55:12.457 | INFO | PerfReport: 20.57%, LockCompute -- 2.09e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260505 09:55:12.457 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.68e-02 / 0.00e+00 | 20260505 09:55:12.584 | INFO | Starting a round of Baz IO | 20260505 09:55:15.869 | INFO | Populating RTMetrics report | 20260505 09:55:15.869 | INFO | RTMetrics report populated | 20260505 09:55:15.869 | INFO | Opening tmp RTMetrics json file | 20260505 09:55:15.912 | INFO | Serializing RTMetrics to json | 20260505 09:55:15.936 | INFO | Serialization done, swapping tmp file now... | 20260505 09:55:15.943 | INFO | RTMetrics done | 20260505 09:55:16.005 | INFO | Finished a round of Baz IO: Wrote 2.428 GiB in 3.42 seconds | 20260505 09:55:16.005 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.52% / 0.01% / 74.46% | 20260505 09:55:16.005 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.7319% / 3.57628e-05% / 6.75519e-05% | 20260505 09:55:16.005 | INFO | Baz IO overall: | 20260505 09:55:16.005 | INFO | BytesWritten : 1334531472575 | 20260505 09:55:16.005 | INFO | NumEvents : 635260755273 | 20260505 09:55:17.594 | INFO | Analyzing chunk frames = [2293760,2294272) | 20260505 09:55:17.594 | INFO | MemInfo: { | 20260505 09:55:17.594 | INFO | "gpu" : " 17 GB", | 20260505 09:55:17.594 | INFO | "huge" : "", | 20260505 09:55:17.594 | INFO | "huge_shared" : " 27 GB", | 20260505 09:55:17.594 | INFO | "malloc" : " 14 GB", | 20260505 09:55:17.594 | INFO | "pinned" : " 29 GB" | 20260505 09:55:17.594 | INFO | } | 20260505 09:55:17.594 | INFO | Perf report for the last 5.13901 seconds: | 20260505 09:55:17.594 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:55:17.594 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:17.594 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:17.594 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:17.594 | INFO | ANALYSIS : 65.3%, 2.9, 100.3ms, 0 | 20260505 09:55:17.594 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:17.594 | INFO | RT_METRICS : 47.1%, 1.0, 24.7ms, 0 | 20260505 09:55:17.594 | INFO | PRE_HQ : 62.6%, 1.8, 59.8ms, 0 | 20260505 09:55:17.594 | INFO | BAZWRITER : 65.9%, 4.2, 140.5ms, 0 | 20260505 09:55:17.594 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 09:55:17.595 | INFO | Uploaded 36GiB to GPU in 2.21455s (16.2561GiB/s) | 20260505 09:55:17.595 | INFO | Downloaded 38GiB to GPU in 2.32618s (16.3358GiB/s) | 20260505 09:55:22.733 | INFO | Analyzing chunk frames = [2294272,2294784) | 20260505 09:55:22.733 | INFO | MemInfo: { | 20260505 09:55:22.733 | INFO | "gpu" : " 17 GB", | 20260505 09:55:22.733 | INFO | "huge" : "", | 20260505 09:55:22.733 | INFO | "huge_shared" : " 27 GB", | 20260505 09:55:22.733 | INFO | "malloc" : " 14 GB", | 20260505 09:55:22.733 | INFO | "pinned" : " 29 GB" | 20260505 09:55:22.733 | INFO | } | 20260505 09:55:22.733 | INFO | Perf report for the last 5.13883 seconds: | 20260505 09:55:22.733 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:55:22.733 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:22.733 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:22.733 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:22.733 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 09:55:22.733 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:22.733 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:22.733 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 09:55:22.733 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:22.733 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:22.733 | INFO | Uploaded 36GiB to GPU in 2.17607s (16.5436GiB/s) | 20260505 09:55:22.733 | INFO | Downloaded 34GiB to GPU in 2.10072s (16.1849GiB/s) | 20260505 09:55:27.872 | INFO | Analyzing chunk frames = [2294784,2295296) | 20260505 09:55:27.872 | INFO | MemInfo: { | 20260505 09:55:27.872 | INFO | "gpu" : " 17 GB", | 20260505 09:55:27.872 | INFO | "huge" : "", | 20260505 09:55:27.872 | INFO | "huge_shared" : " 27 GB", | 20260505 09:55:27.872 | INFO | "malloc" : " 14 GB", | 20260505 09:55:27.872 | INFO | "pinned" : " 29 GB" | 20260505 09:55:27.872 | INFO | } | 20260505 09:55:27.872 | INFO | Perf report for the last 5.13946 seconds: | 20260505 09:55:27.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:55:27.873 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:27.873 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:27.873 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:27.873 | INFO | ANALYSIS : 63.2%, 3.0, 98.3ms, 0 | 20260505 09:55:27.873 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:27.873 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:27.873 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 09:55:27.873 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:27.873 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:27.873 | INFO | Uploaded 36GiB to GPU in 2.14051s (16.8184GiB/s) | 20260505 09:55:27.873 | INFO | Downloaded 34GiB to GPU in 2.06894s (16.4336GiB/s) | 20260505 09:55:33.012 | INFO | Analyzing chunk frames = [2295296,2295808) | 20260505 09:55:33.012 | INFO | MemInfo: { | 20260505 09:55:33.012 | INFO | "gpu" : " 17 GB", | 20260505 09:55:33.012 | INFO | "huge" : "", | 20260505 09:55:33.012 | INFO | "huge_shared" : " 27 GB", | 20260505 09:55:33.012 | INFO | "malloc" : " 14 GB", | 20260505 09:55:33.013 | INFO | "pinned" : " 29 GB" | 20260505 09:55:33.013 | INFO | } | 20260505 09:55:33.013 | INFO | Perf report for the last 5.14018 seconds: | 20260505 09:55:33.013 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:55:33.013 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:33.013 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:33.013 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:33.013 | INFO | ANALYSIS : 63.1%, 3.0, 98.1ms, 0 | 20260505 09:55:33.013 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:33.013 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:33.013 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 09:55:33.013 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:33.013 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:33.013 | INFO | Uploaded 36GiB to GPU in 2.1699s (16.5906GiB/s) | 20260505 09:55:33.013 | INFO | Downloaded 34GiB to GPU in 2.07538s (16.3826GiB/s) | 20260505 09:55:38.154 | INFO | Analyzing chunk frames = [2295808,2296320) | 20260505 09:55:38.154 | INFO | MemInfo: { | 20260505 09:55:38.154 | INFO | "gpu" : " 17 GB", | 20260505 09:55:38.154 | INFO | "huge" : "", | 20260505 09:55:38.154 | INFO | "huge_shared" : " 27 GB", | 20260505 09:55:38.154 | INFO | "malloc" : " 14 GB", | 20260505 09:55:38.154 | INFO | "pinned" : " 29 GB" | 20260505 09:55:38.154 | INFO | } | 20260505 09:55:38.154 | INFO | Perf report for the last 5.14179 seconds: | 20260505 09:55:38.154 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:55:38.154 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:38.154 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:38.154 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:38.155 | INFO | ANALYSIS : 63.7%, 3.0, 99.6ms, 0 | 20260505 09:55:38.155 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:38.155 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:38.155 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:55:38.155 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:38.155 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:38.155 | INFO | Uploaded 36GiB to GPU in 2.23483s (16.1086GiB/s) | 20260505 09:55:38.155 | INFO | Downloaded 34GiB to GPU in 2.10673s (16.1387GiB/s) | 20260505 09:55:43.292 | INFO | Analyzing chunk frames = [2296320,2296832) | 20260505 09:55:43.292 | INFO | MemInfo: { | 20260505 09:55:43.292 | INFO | "gpu" : " 17 GB", | 20260505 09:55:43.292 | INFO | "huge" : "", | 20260505 09:55:43.293 | INFO | "huge_shared" : " 27 GB", | 20260505 09:55:43.293 | INFO | "malloc" : " 14 GB", | 20260505 09:55:43.293 | INFO | "pinned" : " 29 GB" | 20260505 09:55:43.293 | INFO | } | 20260505 09:55:43.293 | INFO | Perf report for the last 5.13819 seconds: | 20260505 09:55:43.293 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:55:43.293 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:43.293 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:43.293 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:43.293 | INFO | ANALYSIS : 63.8%, 3.0, 99.5ms, 0 | 20260505 09:55:43.293 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:43.293 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:43.293 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 09:55:43.293 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:43.293 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:43.293 | INFO | Uploaded 36GiB to GPU in 2.2277s (16.1602GiB/s) | 20260505 09:55:43.293 | INFO | Downloaded 34GiB to GPU in 2.09899s (16.1982GiB/s) | 20260505 09:55:48.430 | INFO | Analyzing chunk frames = [2296832,2297344) | 20260505 09:55:48.430 | INFO | MemInfo: { | 20260505 09:55:48.430 | INFO | "gpu" : " 17 GB", | 20260505 09:55:48.430 | INFO | "huge" : "", | 20260505 09:55:48.430 | INFO | "huge_shared" : " 27 GB", | 20260505 09:55:48.430 | INFO | "malloc" : " 14 GB", | 20260505 09:55:48.430 | INFO | "pinned" : " 29 GB" | 20260505 09:55:48.430 | INFO | } | 20260505 09:55:48.430 | INFO | Perf report for the last 5.13721 seconds: | 20260505 09:55:48.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:55:48.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:48.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:48.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:48.430 | INFO | ANALYSIS : 63.8%, 3.0, 99.5ms, 0 | 20260505 09:55:48.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:48.430 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:48.430 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 09:55:48.430 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:48.430 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:48.430 | INFO | Uploaded 36GiB to GPU in 2.18632s (16.466GiB/s) | 20260505 09:55:48.430 | INFO | Downloaded 34GiB to GPU in 2.1458s (15.8449GiB/s) | 20260505 09:55:53.569 | INFO | Analyzing chunk frames = [2297344,2297856) | 20260505 09:55:53.569 | INFO | MemInfo: { | 20260505 09:55:53.569 | INFO | "gpu" : " 17 GB", | 20260505 09:55:53.569 | INFO | "huge" : "", | 20260505 09:55:53.569 | INFO | "huge_shared" : " 27 GB", | 20260505 09:55:53.569 | INFO | "malloc" : " 14 GB", | 20260505 09:55:53.569 | INFO | "pinned" : " 29 GB" | 20260505 09:55:53.569 | INFO | } | 20260505 09:55:53.569 | INFO | Perf report for the last 5.13931 seconds: | 20260505 09:55:53.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:55:53.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:53.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:53.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:53.569 | INFO | ANALYSIS : 64.7%, 3.0, 101.0ms, 0 | 20260505 09:55:53.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:53.569 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:53.569 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 09:55:53.569 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:53.569 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:53.570 | INFO | Uploaded 36GiB to GPU in 2.24723s (16.0197GiB/s) | 20260505 09:55:53.570 | INFO | Downloaded 34GiB to GPU in 2.08923s (16.2739GiB/s) | 20260505 09:55:53.698 | INFO | Starting a round of Baz IO | 20260505 09:55:56.964 | INFO | Populating RTMetrics report | 20260505 09:55:56.964 | INFO | RTMetrics report populated | 20260505 09:55:56.964 | INFO | Opening tmp RTMetrics json file | 20260505 09:55:56.968 | INFO | Serializing RTMetrics to json | 20260505 09:55:56.974 | INFO | Serialization done, swapping tmp file now... | 20260505 09:55:56.985 | INFO | RTMetrics done | 20260505 09:55:57.085 | INFO | Finished a round of Baz IO: Wrote 2.429 GiB in 3.386 seconds | 20260505 09:55:57.085 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.51% / 0.01% / 74.48% | 20260505 09:55:57.085 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.72842% / 7.94729e-06% / 5.5631e-05% | 20260505 09:55:57.085 | INFO | Baz IO overall: | 20260505 09:55:57.085 | INFO | BytesWritten : 1337139126946 | 20260505 09:55:57.085 | INFO | NumEvents : 636505624989 | 20260505 09:55:58.709 | INFO | Analyzing chunk frames = [2297856,2298368) | 20260505 09:55:58.709 | INFO | MemInfo: { | 20260505 09:55:58.709 | INFO | "gpu" : " 17 GB", | 20260505 09:55:58.709 | INFO | "huge" : "", | 20260505 09:55:58.709 | INFO | "huge_shared" : " 27 GB", | 20260505 09:55:58.709 | INFO | "malloc" : " 14 GB", | 20260505 09:55:58.709 | INFO | "pinned" : " 29 GB" | 20260505 09:55:58.709 | INFO | } | 20260505 09:55:58.709 | INFO | Perf report for the last 5.13961 seconds: | 20260505 09:55:58.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:55:58.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:58.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:58.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:58.709 | INFO | ANALYSIS : 64.9%, 2.9, 99.5ms, 0 | 20260505 09:55:58.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:55:58.709 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 09:55:58.709 | INFO | PRE_HQ : 62.8%, 1.8, 59.6ms, 0 | 20260505 09:55:58.709 | INFO | BAZWRITER : 65.2%, 4.2, 139.6ms, 0 | 20260505 09:55:58.709 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 09:55:58.709 | INFO | Uploaded 36GiB to GPU in 2.26163s (15.9177GiB/s) | 20260505 09:55:58.709 | INFO | Downloaded 38GiB to GPU in 2.37199s (16.0203GiB/s) | 20260505 09:56:03.848 | INFO | Analyzing chunk frames = [2298368,2298880) | 20260505 09:56:03.848 | INFO | MemInfo: { | 20260505 09:56:03.848 | INFO | "gpu" : " 17 GB", | 20260505 09:56:03.848 | INFO | "huge" : "", | 20260505 09:56:03.848 | INFO | "huge_shared" : " 27 GB", | 20260505 09:56:03.848 | INFO | "malloc" : " 14 GB", | 20260505 09:56:03.848 | INFO | "pinned" : " 29 GB" | 20260505 09:56:03.849 | INFO | } | 20260505 09:56:03.849 | INFO | Perf report for the last 5.13969 seconds: | 20260505 09:56:03.849 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:56:03.849 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:03.849 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:03.849 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:03.849 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 09:56:03.849 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:03.849 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:03.849 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:56:03.849 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:03.849 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:03.849 | INFO | Uploaded 36GiB to GPU in 2.18514s (16.4749GiB/s) | 20260505 09:56:03.849 | INFO | Downloaded 34GiB to GPU in 2.1034s (16.1643GiB/s) | 20260505 09:56:08.988 | INFO | Analyzing chunk frames = [2298880,2299392) | 20260505 09:56:08.988 | INFO | MemInfo: { | 20260505 09:56:08.988 | INFO | "gpu" : " 17 GB", | 20260505 09:56:08.988 | INFO | "huge" : "", | 20260505 09:56:08.988 | INFO | "huge_shared" : " 27 GB", | 20260505 09:56:08.988 | INFO | "malloc" : " 14 GB", | 20260505 09:56:08.988 | INFO | "pinned" : " 29 GB" | 20260505 09:56:08.988 | INFO | } | 20260505 09:56:08.988 | INFO | Perf report for the last 5.1393 seconds: | 20260505 09:56:08.988 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:56:08.988 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:08.988 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:08.988 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:08.988 | INFO | ANALYSIS : 63.1%, 3.0, 98.2ms, 0 | 20260505 09:56:08.988 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:08.988 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:08.988 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 09:56:08.988 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:08.988 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:08.988 | INFO | Uploaded 36GiB to GPU in 2.14233s (16.8041GiB/s) | 20260505 09:56:08.988 | INFO | Downloaded 34GiB to GPU in 2.06216s (16.4876GiB/s) | 20260505 09:56:14.128 | INFO | Analyzing chunk frames = [2299392,2299904) | 20260505 09:56:14.128 | INFO | MemInfo: { | 20260505 09:56:14.128 | INFO | "gpu" : " 17 GB", | 20260505 09:56:14.128 | INFO | "huge" : "", | 20260505 09:56:14.128 | INFO | "huge_shared" : " 27 GB", | 20260505 09:56:14.128 | INFO | "malloc" : " 14 GB", | 20260505 09:56:14.128 | INFO | "pinned" : " 29 GB" | 20260505 09:56:14.128 | INFO | } | 20260505 09:56:14.129 | INFO | Perf report for the last 5.14066 seconds: | 20260505 09:56:14.129 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:56:14.129 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:14.129 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:14.129 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:14.129 | INFO | ANALYSIS : 63.1%, 3.0, 98.1ms, 0 | 20260505 09:56:14.129 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:14.129 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:14.129 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 09:56:14.129 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:14.129 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:14.129 | INFO | Uploaded 36GiB to GPU in 2.16511s (16.6273GiB/s) | 20260505 09:56:14.129 | INFO | Downloaded 34GiB to GPU in 2.07353s (16.3972GiB/s) | 20260505 09:56:19.267 | INFO | Analyzing chunk frames = [2299904,2300416) | 20260505 09:56:19.267 | INFO | MemInfo: { | 20260505 09:56:19.267 | INFO | "gpu" : " 17 GB", | 20260505 09:56:19.267 | INFO | "huge" : "", | 20260505 09:56:19.267 | INFO | "huge_shared" : " 27 GB", | 20260505 09:56:19.267 | INFO | "malloc" : " 14 GB", | 20260505 09:56:19.267 | INFO | "pinned" : " 29 GB" | 20260505 09:56:19.267 | INFO | } | 20260505 09:56:19.267 | INFO | Perf report for the last 5.13852 seconds: | 20260505 09:56:19.267 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:56:19.267 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:19.267 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:19.267 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:19.267 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 09:56:19.267 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:19.267 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:19.267 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 09:56:19.267 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:19.267 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:19.267 | INFO | Uploaded 36GiB to GPU in 2.23356s (16.1178GiB/s) | 20260505 09:56:19.267 | INFO | Downloaded 34GiB to GPU in 2.11006s (16.1133GiB/s) | 20260505 09:56:24.407 | INFO | Analyzing chunk frames = [2300416,2300928) | 20260505 09:56:24.407 | INFO | MemInfo: { | 20260505 09:56:24.407 | INFO | "gpu" : " 17 GB", | 20260505 09:56:24.407 | INFO | "huge" : "", | 20260505 09:56:24.407 | INFO | "huge_shared" : " 27 GB", | 20260505 09:56:24.407 | INFO | "malloc" : " 14 GB", | 20260505 09:56:24.407 | INFO | "pinned" : " 29 GB" | 20260505 09:56:24.407 | INFO | } | 20260505 09:56:24.407 | INFO | Perf report for the last 5.14012 seconds: | 20260505 09:56:24.407 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:56:24.407 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:24.407 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:24.407 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:24.407 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 09:56:24.407 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:24.407 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:24.407 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 09:56:24.407 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:24.407 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:24.408 | INFO | Uploaded 36GiB to GPU in 2.23426s (16.1127GiB/s) | 20260505 09:56:24.408 | INFO | Downloaded 34GiB to GPU in 2.1001s (16.1897GiB/s) | 20260505 09:56:29.545 | INFO | Analyzing chunk frames = [2300928,2301440) | 20260505 09:56:29.545 | INFO | MemInfo: { | 20260505 09:56:29.545 | INFO | "gpu" : " 17 GB", | 20260505 09:56:29.545 | INFO | "huge" : "", | 20260505 09:56:29.545 | INFO | "huge_shared" : " 27 GB", | 20260505 09:56:29.545 | INFO | "malloc" : " 14 GB", | 20260505 09:56:29.545 | INFO | "pinned" : " 29 GB" | 20260505 09:56:29.545 | INFO | } | 20260505 09:56:29.545 | INFO | Perf report for the last 5.13767 seconds: | 20260505 09:56:29.545 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:56:29.545 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:29.545 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:29.545 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:29.545 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 09:56:29.545 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:29.545 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:29.545 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 09:56:29.545 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:29.545 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:29.545 | INFO | Uploaded 36GiB to GPU in 2.20317s (16.3401GiB/s) | 20260505 09:56:29.545 | INFO | Downloaded 34GiB to GPU in 2.12582s (15.9938GiB/s) | 20260505 09:56:34.684 | INFO | Analyzing chunk frames = [2301440,2301952) | 20260505 09:56:34.684 | INFO | MemInfo: { | 20260505 09:56:34.684 | INFO | "gpu" : " 17 GB", | 20260505 09:56:34.684 | INFO | "huge" : "", | 20260505 09:56:34.684 | INFO | "huge_shared" : " 27 GB", | 20260505 09:56:34.684 | INFO | "malloc" : " 14 GB", | 20260505 09:56:34.684 | INFO | "pinned" : " 29 GB" | 20260505 09:56:34.684 | INFO | } | 20260505 09:56:34.685 | INFO | Perf report for the last 5.13962 seconds: | 20260505 09:56:34.685 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:56:34.685 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:34.685 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:34.685 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:34.685 | INFO | ANALYSIS : 64.6%, 3.0, 100.8ms, 0 | 20260505 09:56:34.685 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:34.685 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:34.685 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 09:56:34.685 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:34.685 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:34.685 | INFO | Uploaded 36GiB to GPU in 2.24486s (16.0366GiB/s) | 20260505 09:56:34.685 | INFO | Downloaded 34GiB to GPU in 2.09205s (16.252GiB/s) | 20260505 09:56:34.685 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 09:56:34.685 | INFO | Total mem usage: 15.3 GiB | 20260505 09:56:34.685 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 09:56:34.685 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -5.76%) | 20260505 09:56:34.685 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 09:56:34.685 | INFO | High Water Breakdown: | 20260505 09:56:34.685 | INFO | 61.1% : 8.86 GiB ( 94.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 09:56:34.685 | INFO | 38.8% : 5.62 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 09:56:34.685 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 09:56:34.685 | INFO | ------------------------------------------ | 20260505 09:56:34.685 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:56:34.685 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 09:56:34.685 | INFO | Total mem usage: 26.5 GiB | 20260505 09:56:34.685 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 09:56:34.685 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 09:56:34.685 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 09:56:34.686 | INFO | High Water Breakdown: | 20260505 09:56:34.686 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:56:34.686 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:56:34.686 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:56:34.686 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:56:34.686 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:56:34.686 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:56:34.686 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 09:56:34.686 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 09:56:34.686 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:56:34.686 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 09:56:34.686 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 09:56:34.686 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 09:56:34.686 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 09:56:34.686 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 09:56:34.686 | INFO | ------------------------------------------ | 20260505 09:56:34.686 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:56:34.686 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 09:56:34.686 | INFO | Total mem usage: 25.6 GiB | 20260505 09:56:34.686 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 09:56:34.686 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 09:56:34.686 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 09:56:34.686 | INFO | High Water Breakdown: | 20260505 09:56:34.686 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 09:56:34.686 | INFO | ------------------------------------------ | 20260505 09:56:34.686 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:56:34.686 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 09:56:34.686 | INFO | Total mem usage: 15.7 GiB | 20260505 09:56:34.686 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 09:56:34.686 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 09:56:34.686 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 09:56:34.686 | INFO | High Water Breakdown: | 20260505 09:56:34.686 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 09:56:34.686 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 09:56:34.686 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 09:56:34.686 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 09:56:34.686 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 09:56:34.686 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 09:56:34.686 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 09:56:34.686 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 09:56:34.686 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 09:56:34.686 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:56:34.686 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 09:56:34.686 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:56:34.686 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 09:56:34.686 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 09:56:34.686 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 09:56:34.686 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 09:56:34.686 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:56:34.686 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 09:56:34.686 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:56:34.686 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:56:34.686 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:56:34.686 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 09:56:34.686 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 09:56:34.686 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:56:34.686 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 09:56:34.686 | INFO | ------------------------------------------ | 20260505 09:56:34.686 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:56:34.686 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 09:56:34.686 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:56:34.686 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:56:34.686 | INFO | PerfReport: Full Execution -- 3.24e+01 / 7.73e+01, 1.38e+01 / 2.28e+00 | 20260505 09:56:34.686 | INFO | PerfReport: 0.29%, AggregateStats -- 9.32e-02 / 1.02e-01, 1.09e-02 / 4.53e-03 | 20260505 09:56:34.686 | INFO | PerfReport: 11.66%, Baseline -- 3.78e+00 / 4.45e+00, 6.14e-01 / 2.61e-01 | 20260505 09:56:34.686 | INFO | PerfReport: 3.32%, Histogram -- 1.08e+00 / 1.49e+00, 2.11e-01 / 9.72e-02 | 20260505 09:56:34.686 | INFO | PerfReport: 15.62%, DME -- 4.05e+01 / 4.48e+01, 4.40e+00 / 1.98e+00 | 20260505 09:56:34.686 | INFO | PerfReport: 56.62%, FrameLabeling -- 1.84e+01 / 2.15e+01, 1.87e+00 / 2.19e+00 | 20260505 09:56:34.686 | INFO | PerfReport: 7.36%, PulseAccumulating -- 2.39e+00 / 2.97e+00, 4.09e-01 / 2.14e-01 | 20260505 09:56:34.686 | INFO | PerfReport: 3.79%, Metrics -- 1.23e+00 / 1.52e+00, 3.58e-01 / 1.41e-01 | 20260505 09:56:34.686 | INFO | PerfReport: 1.34%, PulseSerializing -- 4.35e-01 / 4.82e-01, 8.17e-02 / 3.74e-02 | 20260505 09:56:34.686 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 09:56:34.686 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:56:34.686 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:56:34.686 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.39e+01 / 0.00e+00 | 20260505 09:56:34.686 | INFO | PerfReport: 22.60%, Upload -- 2.30e+01 / 0.00e+00, 8.33e+00 / 0.00e+00 | 20260505 09:56:34.686 | INFO | PerfReport: 21.82%, Download -- 2.22e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260505 09:56:34.686 | INFO | PerfReport: 31.93%, Compute -- 3.24e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 09:56:34.686 | INFO | PerfReport: 1.89%, LockUpload -- 1.92e+00 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260505 09:56:34.686 | INFO | PerfReport: 1.11%, LockDownload -- 1.13e+00 / 0.00e+00, 2.70e+00 / 0.00e+00 | 20260505 09:56:34.686 | INFO | PerfReport: 20.64%, LockCompute -- 2.10e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260505 09:56:34.686 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.13e-02 / 0.00e+00 | 20260505 09:56:34.816 | INFO | Starting a round of Baz IO | 20260505 09:56:38.094 | INFO | Populating RTMetrics report | 20260505 09:56:38.094 | INFO | RTMetrics report populated | 20260505 09:56:38.094 | INFO | Opening tmp RTMetrics json file | 20260505 09:56:38.098 | INFO | Serializing RTMetrics to json | 20260505 09:56:38.101 | INFO | Serialization done, swapping tmp file now... | 20260505 09:56:38.103 | INFO | RTMetrics done | 20260505 09:56:38.234 | INFO | Finished a round of Baz IO: Wrote 2.427 GiB in 3.417 seconds | 20260505 09:56:38.234 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.50% / 0.01% / 74.49% | 20260505 09:56:38.234 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.7194% / 2.78155e-05% / 7.94729e-05% | 20260505 09:56:38.234 | INFO | Baz IO overall: | 20260505 09:56:38.234 | INFO | BytesWritten : 1339744849158 | 20260505 09:56:38.234 | INFO | NumEvents : 637749459542 | 20260505 09:56:39.824 | INFO | Analyzing chunk frames = [2301952,2302464) | 20260505 09:56:39.824 | INFO | MemInfo: { | 20260505 09:56:39.824 | INFO | "gpu" : " 17 GB", | 20260505 09:56:39.824 | INFO | "huge" : "", | 20260505 09:56:39.824 | INFO | "huge_shared" : " 27 GB", | 20260505 09:56:39.824 | INFO | "malloc" : " 14 GB", | 20260505 09:56:39.824 | INFO | "pinned" : " 29 GB" | 20260505 09:56:39.824 | INFO | } | 20260505 09:56:39.824 | INFO | Perf report for the last 5.13958 seconds: | 20260505 09:56:39.824 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:56:39.824 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:39.824 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:39.824 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:39.824 | INFO | ANALYSIS : 65.2%, 2.9, 100.0ms, 0 | 20260505 09:56:39.824 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:39.824 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260505 09:56:39.824 | INFO | PRE_HQ : 62.6%, 1.8, 59.7ms, 0 | 20260505 09:56:39.824 | INFO | BAZWRITER : 65.8%, 4.1, 139.5ms, 0 | 20260505 09:56:39.824 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 09:56:39.824 | INFO | Uploaded 36GiB to GPU in 2.25518s (15.9632GiB/s) | 20260505 09:56:39.824 | INFO | Downloaded 38GiB to GPU in 2.34434s (16.2092GiB/s) | 20260505 09:56:44.963 | INFO | Analyzing chunk frames = [2302464,2302976) | 20260505 09:56:44.963 | INFO | MemInfo: { | 20260505 09:56:44.963 | INFO | "gpu" : " 17 GB", | 20260505 09:56:44.963 | INFO | "huge" : "", | 20260505 09:56:44.963 | INFO | "huge_shared" : " 27 GB", | 20260505 09:56:44.963 | INFO | "malloc" : " 14 GB", | 20260505 09:56:44.963 | INFO | "pinned" : " 29 GB" | 20260505 09:56:44.963 | INFO | } | 20260505 09:56:44.963 | INFO | Perf report for the last 5.13911 seconds: | 20260505 09:56:44.963 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:56:44.963 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:44.963 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:44.963 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:44.963 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 09:56:44.963 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:44.963 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:44.963 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 09:56:44.963 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:44.963 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:44.964 | INFO | Uploaded 36GiB to GPU in 2.18403s (16.4833GiB/s) | 20260505 09:56:44.964 | INFO | Downloaded 34GiB to GPU in 2.10177s (16.1768GiB/s) | 20260505 09:56:50.103 | INFO | Analyzing chunk frames = [2302976,2303488) | 20260505 09:56:50.103 | INFO | MemInfo: { | 20260505 09:56:50.103 | INFO | "gpu" : " 17 GB", | 20260505 09:56:50.103 | INFO | "huge" : "", | 20260505 09:56:50.103 | INFO | "huge_shared" : " 27 GB", | 20260505 09:56:50.103 | INFO | "malloc" : " 14 GB", | 20260505 09:56:50.103 | INFO | "pinned" : " 29 GB" | 20260505 09:56:50.103 | INFO | } | 20260505 09:56:50.103 | INFO | Perf report for the last 5.14012 seconds: | 20260505 09:56:50.103 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:56:50.103 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:50.103 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:50.103 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:50.103 | INFO | ANALYSIS : 63.2%, 3.0, 98.4ms, 0 | 20260505 09:56:50.103 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:50.103 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:50.104 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 09:56:50.104 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:50.104 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:50.104 | INFO | Uploaded 36GiB to GPU in 2.14993s (16.7448GiB/s) | 20260505 09:56:50.104 | INFO | Downloaded 34GiB to GPU in 2.07074s (16.4192GiB/s) | 20260505 09:56:55.241 | INFO | Analyzing chunk frames = [2303488,2304000) | 20260505 09:56:55.242 | INFO | MemInfo: { | 20260505 09:56:55.242 | INFO | "gpu" : " 17 GB", | 20260505 09:56:55.242 | INFO | "huge" : "", | 20260505 09:56:55.242 | INFO | "huge_shared" : " 27 GB", | 20260505 09:56:55.242 | INFO | "malloc" : " 14 GB", | 20260505 09:56:55.242 | INFO | "pinned" : " 29 GB" | 20260505 09:56:55.242 | INFO | } | 20260505 09:56:55.242 | INFO | Perf report for the last 5.13831 seconds: | 20260505 09:56:55.242 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:56:55.242 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:55.242 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:55.242 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:55.242 | INFO | ANALYSIS : 63.1%, 3.0, 98.1ms, 0 | 20260505 09:56:55.242 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:55.242 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:55.242 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 09:56:55.242 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:55.242 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:56:55.242 | INFO | Uploaded 36GiB to GPU in 2.167s (16.6128GiB/s) | 20260505 09:56:55.242 | INFO | Downloaded 34GiB to GPU in 2.0711s (16.4164GiB/s) | 20260505 09:57:00.381 | INFO | Analyzing chunk frames = [2304000,2304512) | 20260505 09:57:00.381 | INFO | MemInfo: { | 20260505 09:57:00.381 | INFO | "gpu" : " 17 GB", | 20260505 09:57:00.381 | INFO | "huge" : "", | 20260505 09:57:00.381 | INFO | "huge_shared" : " 27 GB", | 20260505 09:57:00.381 | INFO | "malloc" : " 14 GB", | 20260505 09:57:00.381 | INFO | "pinned" : " 29 GB" | 20260505 09:57:00.381 | INFO | } | 20260505 09:57:00.381 | INFO | Perf report for the last 5.13964 seconds: | 20260505 09:57:00.381 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:57:00.381 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:00.381 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:00.381 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:00.381 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 09:57:00.381 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:00.381 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:00.381 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:57:00.381 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:00.381 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:00.382 | INFO | Uploaded 36GiB to GPU in 2.23399s (16.1147GiB/s) | 20260505 09:57:00.382 | INFO | Downloaded 34GiB to GPU in 2.11292s (16.0914GiB/s) | 20260505 09:57:05.521 | INFO | Analyzing chunk frames = [2304512,2305024) | 20260505 09:57:05.521 | INFO | MemInfo: { | 20260505 09:57:05.521 | INFO | "gpu" : " 17 GB", | 20260505 09:57:05.521 | INFO | "huge" : "", | 20260505 09:57:05.521 | INFO | "huge_shared" : " 27 GB", | 20260505 09:57:05.521 | INFO | "malloc" : " 14 GB", | 20260505 09:57:05.522 | INFO | "pinned" : " 29 GB" | 20260505 09:57:05.522 | INFO | } | 20260505 09:57:05.522 | INFO | Perf report for the last 5.14022 seconds: | 20260505 09:57:05.522 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:57:05.522 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:05.522 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:05.522 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:05.522 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 09:57:05.522 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:05.522 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:05.522 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:57:05.522 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:05.522 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:05.522 | INFO | Uploaded 36GiB to GPU in 2.23771s (16.0879GiB/s) | 20260505 09:57:05.522 | INFO | Downloaded 34GiB to GPU in 2.10363s (16.1625GiB/s) | 20260505 09:57:10.660 | INFO | Analyzing chunk frames = [2305024,2305536) | 20260505 09:57:10.660 | INFO | MemInfo: { | 20260505 09:57:10.660 | INFO | "gpu" : " 17 GB", | 20260505 09:57:10.660 | INFO | "huge" : "", | 20260505 09:57:10.660 | INFO | "huge_shared" : " 27 GB", | 20260505 09:57:10.660 | INFO | "malloc" : " 14 GB", | 20260505 09:57:10.660 | INFO | "pinned" : " 29 GB" | 20260505 09:57:10.660 | INFO | } | 20260505 09:57:10.660 | INFO | Perf report for the last 5.13866 seconds: | 20260505 09:57:10.660 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:57:10.660 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:10.660 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:10.660 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:10.660 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 09:57:10.660 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:10.660 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:10.660 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 09:57:10.660 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:10.660 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:10.661 | INFO | Uploaded 36GiB to GPU in 2.17418s (16.5579GiB/s) | 20260505 09:57:10.661 | INFO | Downloaded 34GiB to GPU in 2.12946s (15.9665GiB/s) | 20260505 09:57:15.800 | INFO | Analyzing chunk frames = [2305536,2306048) | 20260505 09:57:15.800 | INFO | MemInfo: { | 20260505 09:57:15.800 | INFO | "gpu" : " 17 GB", | 20260505 09:57:15.800 | INFO | "huge" : "", | 20260505 09:57:15.800 | INFO | "huge_shared" : " 27 GB", | 20260505 09:57:15.800 | INFO | "malloc" : " 14 GB", | 20260505 09:57:15.800 | INFO | "pinned" : " 29 GB" | 20260505 09:57:15.800 | INFO | } | 20260505 09:57:15.800 | INFO | Perf report for the last 5.13964 seconds: | 20260505 09:57:15.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:57:15.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:15.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:15.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:15.800 | INFO | ANALYSIS : 64.7%, 3.0, 101.1ms, 0 | 20260505 09:57:15.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:15.800 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:15.800 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 09:57:15.800 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:15.800 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:15.800 | INFO | Uploaded 36GiB to GPU in 2.24777s (16.0159GiB/s) | 20260505 09:57:15.800 | INFO | Downloaded 34GiB to GPU in 2.08504s (16.3067GiB/s) | 20260505 09:57:15.933 | INFO | Starting a round of Baz IO | 20260505 09:57:19.217 | INFO | Populating RTMetrics report | 20260505 09:57:19.218 | INFO | RTMetrics report populated | 20260505 09:57:19.218 | INFO | Opening tmp RTMetrics json file | 20260505 09:57:19.238 | INFO | Serializing RTMetrics to json | 20260505 09:57:19.247 | INFO | Serialization done, swapping tmp file now... | 20260505 09:57:19.262 | INFO | RTMetrics done | 20260505 09:57:19.359 | INFO | Finished a round of Baz IO: Wrote 2.427 GiB in 3.426 seconds | 20260505 09:57:19.359 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.49% / 0.01% / 74.50% | 20260505 09:57:19.359 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.71602% / 3.97364e-06% / 7.94729e-05% | 20260505 09:57:19.359 | INFO | Baz IO overall: | 20260505 09:57:19.359 | INFO | BytesWritten : 1342350875185 | 20260505 09:57:19.359 | INFO | NumEvents : 638993712836 | 20260505 09:57:20.938 | INFO | Analyzing chunk frames = [2306048,2306560) | 20260505 09:57:20.938 | INFO | MemInfo: { | 20260505 09:57:20.938 | INFO | "gpu" : " 17 GB", | 20260505 09:57:20.938 | INFO | "huge" : "", | 20260505 09:57:20.938 | INFO | "huge_shared" : " 27 GB", | 20260505 09:57:20.938 | INFO | "malloc" : " 14 GB", | 20260505 09:57:20.938 | INFO | "pinned" : " 29 GB" | 20260505 09:57:20.938 | INFO | } | 20260505 09:57:20.938 | INFO | Perf report for the last 5.13835 seconds: | 20260505 09:57:20.938 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:57:20.938 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:20.938 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:20.938 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:20.938 | INFO | ANALYSIS : 65.4%, 2.9, 100.2ms, 0 | 20260505 09:57:20.938 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:20.938 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260505 09:57:20.938 | INFO | PRE_HQ : 61.8%, 1.9, 59.6ms, 0 | 20260505 09:57:20.938 | INFO | BAZWRITER : 65.9%, 4.2, 140.9ms, 0 | 20260505 09:57:20.938 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 09:57:20.939 | INFO | Uploaded 36GiB to GPU in 2.2025s (16.345GiB/s) | 20260505 09:57:20.939 | INFO | Downloaded 38GiB to GPU in 2.32092s (16.3729GiB/s) | 20260505 09:57:26.077 | INFO | Analyzing chunk frames = [2306560,2307072) | 20260505 09:57:26.077 | INFO | MemInfo: { | 20260505 09:57:26.077 | INFO | "gpu" : " 17 GB", | 20260505 09:57:26.077 | INFO | "huge" : "", | 20260505 09:57:26.077 | INFO | "huge_shared" : " 27 GB", | 20260505 09:57:26.077 | INFO | "malloc" : " 14 GB", | 20260505 09:57:26.077 | INFO | "pinned" : " 29 GB" | 20260505 09:57:26.077 | INFO | } | 20260505 09:57:26.077 | INFO | Perf report for the last 5.13909 seconds: | 20260505 09:57:26.077 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:57:26.077 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:26.077 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:26.077 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:26.077 | INFO | ANALYSIS : 64.4%, 3.0, 100.6ms, 0 | 20260505 09:57:26.077 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:26.077 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:26.077 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 09:57:26.077 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:26.077 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:26.078 | INFO | Uploaded 36GiB to GPU in 2.21828s (16.2288GiB/s) | 20260505 09:57:26.078 | INFO | Downloaded 34GiB to GPU in 2.10624s (16.1425GiB/s) | 20260505 09:57:31.217 | INFO | Analyzing chunk frames = [2307072,2307584) | 20260505 09:57:31.217 | INFO | MemInfo: { | 20260505 09:57:31.217 | INFO | "gpu" : " 17 GB", | 20260505 09:57:31.217 | INFO | "huge" : "", | 20260505 09:57:31.217 | INFO | "huge_shared" : " 27 GB", | 20260505 09:57:31.217 | INFO | "malloc" : " 14 GB", | 20260505 09:57:31.217 | INFO | "pinned" : " 29 GB" | 20260505 09:57:31.217 | INFO | } | 20260505 09:57:31.217 | INFO | Perf report for the last 5.14002 seconds: | 20260505 09:57:31.217 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:57:31.217 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:31.217 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:31.217 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:31.217 | INFO | ANALYSIS : 63.0%, 3.0, 98.1ms, 0 | 20260505 09:57:31.218 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:31.218 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:31.218 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 09:57:31.218 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:31.218 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:31.218 | INFO | Uploaded 36GiB to GPU in 2.14954s (16.7478GiB/s) | 20260505 09:57:31.218 | INFO | Downloaded 34GiB to GPU in 2.05955s (16.5085GiB/s) | 20260505 09:57:36.356 | INFO | Analyzing chunk frames = [2307584,2308096) | 20260505 09:57:36.356 | INFO | MemInfo: { | 20260505 09:57:36.356 | INFO | "gpu" : " 17 GB", | 20260505 09:57:36.356 | INFO | "huge" : "", | 20260505 09:57:36.356 | INFO | "huge_shared" : " 27 GB", | 20260505 09:57:36.356 | INFO | "malloc" : " 14 GB", | 20260505 09:57:36.356 | INFO | "pinned" : " 29 GB" | 20260505 09:57:36.356 | INFO | } | 20260505 09:57:36.357 | INFO | Perf report for the last 5.13914 seconds: | 20260505 09:57:36.357 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:57:36.357 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:36.357 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:36.357 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:36.357 | INFO | ANALYSIS : 63.1%, 3.0, 98.0ms, 0 | 20260505 09:57:36.357 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:36.357 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:36.357 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 09:57:36.357 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:36.357 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:36.357 | INFO | Uploaded 36GiB to GPU in 2.14203s (16.8065GiB/s) | 20260505 09:57:36.357 | INFO | Downloaded 34GiB to GPU in 2.07303s (16.4011GiB/s) | 20260505 09:57:41.496 | INFO | Analyzing chunk frames = [2308096,2308608) | 20260505 09:57:41.496 | INFO | MemInfo: { | 20260505 09:57:41.496 | INFO | "gpu" : " 17 GB", | 20260505 09:57:41.496 | INFO | "huge" : "", | 20260505 09:57:41.496 | INFO | "huge_shared" : " 27 GB", | 20260505 09:57:41.496 | INFO | "malloc" : " 14 GB", | 20260505 09:57:41.496 | INFO | "pinned" : " 29 GB" | 20260505 09:57:41.496 | INFO | } | 20260505 09:57:41.496 | INFO | Perf report for the last 5.13953 seconds: | 20260505 09:57:41.496 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:57:41.496 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:41.496 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:41.496 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:41.496 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 09:57:41.496 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:41.496 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:41.496 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 09:57:41.496 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:41.496 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:41.496 | INFO | Uploaded 36GiB to GPU in 2.23679s (16.0945GiB/s) | 20260505 09:57:41.496 | INFO | Downloaded 34GiB to GPU in 2.10523s (16.1503GiB/s) | 20260505 09:57:46.637 | INFO | Analyzing chunk frames = [2308608,2309120) | 20260505 09:57:46.637 | INFO | MemInfo: { | 20260505 09:57:46.637 | INFO | "gpu" : " 17 GB", | 20260505 09:57:46.637 | INFO | "huge" : "", | 20260505 09:57:46.637 | INFO | "huge_shared" : " 27 GB", | 20260505 09:57:46.637 | INFO | "malloc" : " 14 GB", | 20260505 09:57:46.637 | INFO | "pinned" : " 29 GB" | 20260505 09:57:46.637 | INFO | } | 20260505 09:57:46.637 | INFO | Perf report for the last 5.14077 seconds: | 20260505 09:57:46.637 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:57:46.637 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:46.637 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:46.637 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:46.637 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 09:57:46.637 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:46.637 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:46.637 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 09:57:46.637 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:46.637 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:46.637 | INFO | Uploaded 36GiB to GPU in 2.2327s (16.124GiB/s) | 20260505 09:57:46.637 | INFO | Downloaded 34GiB to GPU in 2.09933s (16.1957GiB/s) | 20260505 09:57:51.776 | INFO | Analyzing chunk frames = [2309120,2309632) | 20260505 09:57:51.776 | INFO | MemInfo: { | 20260505 09:57:51.776 | INFO | "gpu" : " 17 GB", | 20260505 09:57:51.776 | INFO | "huge" : "", | 20260505 09:57:51.776 | INFO | "huge_shared" : " 27 GB", | 20260505 09:57:51.776 | INFO | "malloc" : " 14 GB", | 20260505 09:57:51.776 | INFO | "pinned" : " 29 GB" | 20260505 09:57:51.776 | INFO | } | 20260505 09:57:51.776 | INFO | Perf report for the last 5.13936 seconds: | 20260505 09:57:51.776 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:57:51.776 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:51.776 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:51.776 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:51.776 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 09:57:51.776 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:51.776 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:51.776 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 09:57:51.776 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:51.776 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:51.776 | INFO | Uploaded 36GiB to GPU in 2.16448s (16.6322GiB/s) | 20260505 09:57:51.777 | INFO | Downloaded 34GiB to GPU in 2.12275s (16.017GiB/s) | 20260505 09:57:56.914 | INFO | Analyzing chunk frames = [2309632,2310144) | 20260505 09:57:56.914 | INFO | MemInfo: { | 20260505 09:57:56.914 | INFO | "gpu" : " 17 GB", | 20260505 09:57:56.914 | INFO | "huge" : "", | 20260505 09:57:56.914 | INFO | "huge_shared" : " 27 GB", | 20260505 09:57:56.914 | INFO | "malloc" : " 14 GB", | 20260505 09:57:56.914 | INFO | "pinned" : " 29 GB" | 20260505 09:57:56.914 | INFO | } | 20260505 09:57:56.914 | INFO | Perf report for the last 5.13816 seconds: | 20260505 09:57:56.914 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:57:56.914 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:56.914 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:56.914 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:56.914 | INFO | ANALYSIS : 64.6%, 3.0, 100.9ms, 0 | 20260505 09:57:56.914 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:56.914 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:56.914 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 09:57:56.915 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:56.915 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:57:56.915 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 09:57:56.915 | INFO | Total mem usage: 15.3 GiB | 20260505 09:57:56.915 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 09:57:56.915 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.51%) | 20260505 09:57:56.915 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 09:57:56.915 | INFO | High Water Breakdown: | 20260505 09:57:56.915 | INFO | 61% : 8.78 GiB ( 93.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 09:57:56.915 | INFO | 38.9% : 5.59 GiB ( 84.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 09:57:56.915 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 09:57:56.915 | INFO | ------------------------------------------ | 20260505 09:57:56.915 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:57:56.915 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 09:57:56.915 | INFO | Total mem usage: 26.5 GiB | 20260505 09:57:56.915 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 09:57:56.915 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 09:57:56.915 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 09:57:56.915 | INFO | High Water Breakdown: | 20260505 09:57:56.915 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:57:56.915 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:57:56.915 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:57:56.915 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:57:56.915 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:57:56.915 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:57:56.915 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 09:57:56.915 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 09:57:56.915 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:57:56.915 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 09:57:56.915 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 09:57:56.915 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 09:57:56.915 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 09:57:56.915 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 09:57:56.915 | INFO | ------------------------------------------ | 20260505 09:57:56.915 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:57:56.915 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 09:57:56.915 | INFO | Total mem usage: 25.6 GiB | 20260505 09:57:56.915 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 09:57:56.915 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 09:57:56.915 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 09:57:56.915 | INFO | High Water Breakdown: | 20260505 09:57:56.915 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 09:57:56.915 | INFO | ------------------------------------------ | 20260505 09:57:56.915 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:57:56.915 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 09:57:56.915 | INFO | Total mem usage: 15.7 GiB | 20260505 09:57:56.915 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 09:57:56.915 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 09:57:56.915 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 09:57:56.915 | INFO | High Water Breakdown: | 20260505 09:57:56.915 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 09:57:56.915 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 09:57:56.915 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 09:57:56.915 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 09:57:56.915 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 09:57:56.915 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 09:57:56.915 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 09:57:56.915 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 09:57:56.915 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 09:57:56.915 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:57:56.915 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 09:57:56.915 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:57:56.915 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 09:57:56.915 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 09:57:56.915 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 09:57:56.915 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 09:57:56.915 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:57:56.915 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 09:57:56.915 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:57:56.915 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:57:56.915 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:57:56.915 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 09:57:56.915 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 09:57:56.915 | INFO | Uploaded 36GiB to GPU in 2.24818s (16.0129GiB/s) | 20260505 09:57:56.915 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:57:56.916 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 09:57:56.916 | INFO | Downloaded 34GiB to GPU in 2.08791s (16.2842GiB/s) | 20260505 09:57:56.916 | INFO | ------------------------------------------ | 20260505 09:57:56.916 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:57:56.916 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 09:57:56.916 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:57:56.916 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:57:56.916 | INFO | PerfReport: Full Execution -- 3.25e+01 / 7.76e+01, 1.39e+01 / 2.41e+00 | 20260505 09:57:56.916 | INFO | PerfReport: 0.29%, AggregateStats -- 9.31e-02 / 1.02e-01, 1.05e-02 / 4.08e-03 | 20260505 09:57:56.916 | INFO | PerfReport: 11.65%, Baseline -- 3.78e+00 / 4.49e+00, 6.09e-01 / 2.05e-01 | 20260505 09:57:56.916 | INFO | PerfReport: 3.31%, Histogram -- 1.08e+00 / 1.48e+00, 2.07e-01 / 8.89e-02 | 20260505 09:57:56.916 | INFO | PerfReport: 15.66%, DME -- 4.07e+01 / 4.51e+01, 4.57e+00 / 2.26e+00 | 20260505 09:57:56.916 | INFO | PerfReport: 56.65%, FrameLabeling -- 1.84e+01 / 2.15e+01, 1.84e+00 / 1.43e+00 | 20260505 09:57:56.916 | INFO | PerfReport: 7.32%, PulseAccumulating -- 2.38e+00 / 2.96e+00, 4.03e-01 / 1.80e-01 | 20260505 09:57:56.916 | INFO | PerfReport: 3.78%, Metrics -- 1.23e+00 / 1.50e+00, 3.53e-01 / 1.65e-01 | 20260505 09:57:56.916 | INFO | PerfReport: 1.34%, PulseSerializing -- 4.35e-01 / 4.75e-01, 8.20e-02 / 2.35e-02 | 20260505 09:57:56.916 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 09:57:56.916 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:57:56.916 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:57:56.916 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.42e+01 / 0.00e+00 | 20260505 09:57:56.916 | INFO | PerfReport: 22.58%, Upload -- 2.29e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260505 09:57:56.916 | INFO | PerfReport: 21.78%, Download -- 2.21e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260505 09:57:56.916 | INFO | PerfReport: 31.96%, Compute -- 3.25e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 09:57:56.916 | INFO | PerfReport: 1.90%, LockUpload -- 1.93e+00 / 0.00e+00, 7.47e+00 / 0.00e+00 | 20260505 09:57:56.916 | INFO | PerfReport: 1.08%, LockDownload -- 1.09e+00 / 0.00e+00, 2.64e+00 / 0.00e+00 | 20260505 09:57:56.916 | INFO | PerfReport: 20.68%, LockCompute -- 2.10e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260505 09:57:56.916 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.83e-02 / 0.00e+00 | 20260505 09:57:57.047 | INFO | Starting a round of Baz IO | 20260505 09:58:00.329 | INFO | Populating RTMetrics report | 20260505 09:58:00.329 | INFO | RTMetrics report populated | 20260505 09:58:00.329 | INFO | Opening tmp RTMetrics json file | 20260505 09:58:00.330 | INFO | Serializing RTMetrics to json | 20260505 09:58:00.331 | INFO | Serialization done, swapping tmp file now... | 20260505 09:58:00.331 | INFO | RTMetrics done | 20260505 09:58:00.511 | INFO | Finished a round of Baz IO: Wrote 2.427 GiB in 3.464 seconds | 20260505 09:58:00.511 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.47% / 0.01% / 74.51% | 20260505 09:58:00.511 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.70898% / 2.78155e-05% / 7.54992e-05% | 20260505 09:58:00.512 | INFO | Baz IO overall: | 20260505 09:58:00.512 | INFO | BytesWritten : 1344956385851 | 20260505 09:58:00.512 | INFO | NumEvents : 640237537970 | 20260505 09:58:02.053 | INFO | Analyzing chunk frames = [2310144,2310656) | 20260505 09:58:02.053 | INFO | MemInfo: { | 20260505 09:58:02.053 | INFO | "gpu" : " 17 GB", | 20260505 09:58:02.053 | INFO | "huge" : "", | 20260505 09:58:02.053 | INFO | "huge_shared" : " 27 GB", | 20260505 09:58:02.053 | INFO | "malloc" : " 14 GB", | 20260505 09:58:02.053 | INFO | "pinned" : " 29 GB" | 20260505 09:58:02.053 | INFO | } | 20260505 09:58:02.053 | INFO | Perf report for the last 5.13849 seconds: | 20260505 09:58:02.053 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:58:02.053 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:02.053 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:02.053 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:02.053 | INFO | ANALYSIS : 65.3%, 2.9, 100.1ms, 0 | 20260505 09:58:02.053 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:02.053 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260505 09:58:02.053 | INFO | PRE_HQ : 62.3%, 1.8, 59.6ms, 0 | 20260505 09:58:02.053 | INFO | BAZWRITER : 66.7%, 4.2, 142.7ms, 0 | 20260505 09:58:02.053 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 09:58:02.053 | INFO | Uploaded 36GiB to GPU in 2.25374s (15.9735GiB/s) | 20260505 09:58:02.053 | INFO | Downloaded 38GiB to GPU in 2.33111s (16.3013GiB/s) | 20260505 09:58:07.192 | INFO | Analyzing chunk frames = [2310656,2311168) | 20260505 09:58:07.192 | INFO | MemInfo: { | 20260505 09:58:07.192 | INFO | "gpu" : " 17 GB", | 20260505 09:58:07.192 | INFO | "huge" : "", | 20260505 09:58:07.192 | INFO | "huge_shared" : " 27 GB", | 20260505 09:58:07.192 | INFO | "malloc" : " 14 GB", | 20260505 09:58:07.192 | INFO | "pinned" : " 29 GB" | 20260505 09:58:07.192 | INFO | } | 20260505 09:58:07.192 | INFO | Perf report for the last 5.13944 seconds: | 20260505 09:58:07.192 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:58:07.192 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:07.192 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:07.192 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:07.192 | INFO | ANALYSIS : 64.3%, 3.0, 100.5ms, 0 | 20260505 09:58:07.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:07.192 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:07.192 | INFO | PRE_HQ : 29.3%, 1.0, 15.3ms, 0 | 20260505 09:58:07.192 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:07.192 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:07.193 | INFO | Uploaded 36GiB to GPU in 2.18421s (16.4819GiB/s) | 20260505 09:58:07.193 | INFO | Downloaded 34GiB to GPU in 2.09793s (16.2065GiB/s) | 20260505 09:58:12.331 | INFO | Analyzing chunk frames = [2311168,2311680) | 20260505 09:58:12.331 | INFO | MemInfo: { | 20260505 09:58:12.331 | INFO | "gpu" : " 17 GB", | 20260505 09:58:12.331 | INFO | "huge" : "", | 20260505 09:58:12.331 | INFO | "huge_shared" : " 27 GB", | 20260505 09:58:12.331 | INFO | "malloc" : " 14 GB", | 20260505 09:58:12.331 | INFO | "pinned" : " 29 GB" | 20260505 09:58:12.331 | INFO | } | 20260505 09:58:12.331 | INFO | Perf report for the last 5.13889 seconds: | 20260505 09:58:12.331 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:58:12.331 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:12.331 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:12.331 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:12.331 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 09:58:12.331 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:12.331 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:12.332 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 09:58:12.332 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:12.332 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:12.332 | INFO | Uploaded 36GiB to GPU in 2.15409s (16.7124GiB/s) | 20260505 09:58:12.332 | INFO | Downloaded 34GiB to GPU in 2.06722s (16.4472GiB/s) | 20260505 09:58:17.471 | INFO | Analyzing chunk frames = [2311680,2312192) | 20260505 09:58:17.471 | INFO | MemInfo: { | 20260505 09:58:17.471 | INFO | "gpu" : " 17 GB", | 20260505 09:58:17.471 | INFO | "huge" : "", | 20260505 09:58:17.471 | INFO | "huge_shared" : " 27 GB", | 20260505 09:58:17.471 | INFO | "malloc" : " 14 GB", | 20260505 09:58:17.471 | INFO | "pinned" : " 29 GB" | 20260505 09:58:17.471 | INFO | } | 20260505 09:58:17.471 | INFO | Perf report for the last 5.14012 seconds: | 20260505 09:58:17.471 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:58:17.471 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:17.471 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:17.471 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:17.471 | INFO | ANALYSIS : 63.3%, 3.0, 98.3ms, 0 | 20260505 09:58:17.471 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:17.471 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:17.471 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 09:58:17.471 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:17.471 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:17.472 | INFO | Uploaded 36GiB to GPU in 2.18401s (16.4834GiB/s) | 20260505 09:58:17.472 | INFO | Downloaded 34GiB to GPU in 2.08009s (16.3454GiB/s) | 20260505 09:58:22.611 | INFO | Analyzing chunk frames = [2312192,2312704) | 20260505 09:58:22.611 | INFO | MemInfo: { | 20260505 09:58:22.611 | INFO | "gpu" : " 17 GB", | 20260505 09:58:22.611 | INFO | "huge" : "", | 20260505 09:58:22.611 | INFO | "huge_shared" : " 27 GB", | 20260505 09:58:22.611 | INFO | "malloc" : " 14 GB", | 20260505 09:58:22.611 | INFO | "pinned" : " 29 GB" | 20260505 09:58:22.611 | INFO | } | 20260505 09:58:22.611 | INFO | Perf report for the last 5.13944 seconds: | 20260505 09:58:22.611 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:58:22.611 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:22.611 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:22.611 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:22.611 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 09:58:22.611 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:22.611 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:22.611 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 09:58:22.611 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:22.611 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:22.611 | INFO | Uploaded 36GiB to GPU in 2.23478s (16.1089GiB/s) | 20260505 09:58:22.611 | INFO | Downloaded 34GiB to GPU in 2.10968s (16.1162GiB/s) | 20260505 09:58:27.751 | INFO | Analyzing chunk frames = [2312704,2313216) | 20260505 09:58:27.751 | INFO | MemInfo: { | 20260505 09:58:27.751 | INFO | "gpu" : " 17 GB", | 20260505 09:58:27.751 | INFO | "huge" : "", | 20260505 09:58:27.751 | INFO | "huge_shared" : " 27 GB", | 20260505 09:58:27.751 | INFO | "malloc" : " 14 GB", | 20260505 09:58:27.751 | INFO | "pinned" : " 29 GB" | 20260505 09:58:27.751 | INFO | } | 20260505 09:58:27.751 | INFO | Perf report for the last 5.14013 seconds: | 20260505 09:58:27.751 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:58:27.751 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:27.751 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:27.751 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:27.751 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 09:58:27.751 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:27.751 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:27.751 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 09:58:27.751 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:27.751 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:27.751 | INFO | Uploaded 36GiB to GPU in 2.23173s (16.131GiB/s) | 20260505 09:58:27.751 | INFO | Downloaded 34GiB to GPU in 2.09763s (16.2088GiB/s) | 20260505 09:58:32.889 | INFO | Analyzing chunk frames = [2313216,2313728) | 20260505 09:58:32.889 | INFO | MemInfo: { | 20260505 09:58:32.889 | INFO | "gpu" : " 17 GB", | 20260505 09:58:32.889 | INFO | "huge" : "", | 20260505 09:58:32.889 | INFO | "huge_shared" : " 27 GB", | 20260505 09:58:32.889 | INFO | "malloc" : " 14 GB", | 20260505 09:58:32.889 | INFO | "pinned" : " 29 GB" | 20260505 09:58:32.889 | INFO | } | 20260505 09:58:32.890 | INFO | Perf report for the last 5.1385 seconds: | 20260505 09:58:32.890 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:58:32.890 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:32.890 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:32.890 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:32.890 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 09:58:32.890 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:32.890 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:32.890 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 09:58:32.890 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:32.890 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:32.890 | INFO | Uploaded 36GiB to GPU in 2.17072s (16.5844GiB/s) | 20260505 09:58:32.890 | INFO | Downloaded 34GiB to GPU in 2.15479s (15.7788GiB/s) | 20260505 09:58:38.029 | INFO | Analyzing chunk frames = [2313728,2314240) | 20260505 09:58:38.029 | INFO | MemInfo: { | 20260505 09:58:38.029 | INFO | "gpu" : " 17 GB", | 20260505 09:58:38.029 | INFO | "huge" : "", | 20260505 09:58:38.029 | INFO | "huge_shared" : " 27 GB", | 20260505 09:58:38.029 | INFO | "malloc" : " 14 GB", | 20260505 09:58:38.029 | INFO | "pinned" : " 29 GB" | 20260505 09:58:38.029 | INFO | } | 20260505 09:58:38.029 | INFO | Perf report for the last 5.1392 seconds: | 20260505 09:58:38.029 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:58:38.029 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:38.029 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:38.029 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:38.029 | INFO | ANALYSIS : 64.6%, 3.0, 100.8ms, 0 | 20260505 09:58:38.029 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:38.029 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:38.029 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 09:58:38.029 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:38.029 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:38.029 | INFO | Uploaded 36GiB to GPU in 2.24593s (16.029GiB/s) | 20260505 09:58:38.029 | INFO | Downloaded 34GiB to GPU in 2.0955s (16.2253GiB/s) | 20260505 09:58:38.162 | INFO | Starting a round of Baz IO | 20260505 09:58:41.445 | INFO | Populating RTMetrics report | 20260505 09:58:41.445 | INFO | RTMetrics report populated | 20260505 09:58:41.445 | INFO | Opening tmp RTMetrics json file | 20260505 09:58:41.446 | INFO | Serializing RTMetrics to json | 20260505 09:58:41.448 | INFO | Serialization done, swapping tmp file now... | 20260505 09:58:41.450 | INFO | RTMetrics done | 20260505 09:58:41.591 | INFO | Finished a round of Baz IO: Wrote 2.426 GiB in 3.428 seconds | 20260505 09:58:41.591 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.46% / 0.01% / 74.53% | 20260505 09:58:41.591 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.70623% / 3.17891e-05% / 0.000103315% | 20260505 09:58:41.591 | INFO | Baz IO overall: | 20260505 09:58:41.591 | INFO | BytesWritten : 1347561790832 | 20260505 09:58:41.591 | INFO | NumEvents : 641481096060 | 20260505 09:58:43.168 | INFO | Analyzing chunk frames = [2314240,2314752) | 20260505 09:58:43.168 | INFO | MemInfo: { | 20260505 09:58:43.168 | INFO | "gpu" : " 17 GB", | 20260505 09:58:43.168 | INFO | "huge" : "", | 20260505 09:58:43.168 | INFO | "huge_shared" : " 27 GB", | 20260505 09:58:43.168 | INFO | "malloc" : " 14 GB", | 20260505 09:58:43.168 | INFO | "pinned" : " 29 GB" | 20260505 09:58:43.168 | INFO | } | 20260505 09:58:43.168 | INFO | Perf report for the last 5.13906 seconds: | 20260505 09:58:43.168 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:58:43.168 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:43.168 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:43.168 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:43.168 | INFO | ANALYSIS : 65.3%, 2.9, 100.5ms, 0 | 20260505 09:58:43.168 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:43.168 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260505 09:58:43.168 | INFO | PRE_HQ : 62.3%, 1.8, 59.6ms, 0 | 20260505 09:58:43.168 | INFO | BAZWRITER : 66.0%, 4.1, 140.0ms, 0 | 20260505 09:58:43.168 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 09:58:43.168 | INFO | Uploaded 36GiB to GPU in 2.22617s (16.1713GiB/s) | 20260505 09:58:43.168 | INFO | Downloaded 38GiB to GPU in 2.32213s (16.3643GiB/s) | 20260505 09:58:48.307 | INFO | Analyzing chunk frames = [2314752,2315264) | 20260505 09:58:48.307 | INFO | MemInfo: { | 20260505 09:58:48.307 | INFO | "gpu" : " 17 GB", | 20260505 09:58:48.307 | INFO | "huge" : "", | 20260505 09:58:48.307 | INFO | "huge_shared" : " 27 GB", | 20260505 09:58:48.307 | INFO | "malloc" : " 14 GB", | 20260505 09:58:48.308 | INFO | "pinned" : " 29 GB" | 20260505 09:58:48.308 | INFO | } | 20260505 09:58:48.308 | INFO | Perf report for the last 5.13982 seconds: | 20260505 09:58:48.308 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:58:48.308 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:48.308 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:48.308 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:48.308 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 09:58:48.308 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:48.308 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:48.308 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 09:58:48.308 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:48.308 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:48.308 | INFO | Uploaded 36GiB to GPU in 2.17737s (16.5337GiB/s) | 20260505 09:58:48.308 | INFO | Downloaded 34GiB to GPU in 2.09919s (16.1968GiB/s) | 20260505 09:58:53.447 | INFO | Analyzing chunk frames = [2315264,2315776) | 20260505 09:58:53.447 | INFO | MemInfo: { | 20260505 09:58:53.447 | INFO | "gpu" : " 17 GB", | 20260505 09:58:53.447 | INFO | "huge" : "", | 20260505 09:58:53.447 | INFO | "huge_shared" : " 27 GB", | 20260505 09:58:53.447 | INFO | "malloc" : " 14 GB", | 20260505 09:58:53.447 | INFO | "pinned" : " 29 GB" | 20260505 09:58:53.447 | INFO | } | 20260505 09:58:53.447 | INFO | Perf report for the last 5.1394 seconds: | 20260505 09:58:53.447 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:58:53.447 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:53.447 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:53.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:53.447 | INFO | ANALYSIS : 63.2%, 3.0, 98.3ms, 0 | 20260505 09:58:53.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:53.447 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:53.447 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 09:58:53.447 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:53.447 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:53.447 | INFO | Uploaded 36GiB to GPU in 2.14765s (16.7625GiB/s) | 20260505 09:58:53.447 | INFO | Downloaded 34GiB to GPU in 2.06916s (16.4318GiB/s) | 20260505 09:58:58.585 | INFO | Analyzing chunk frames = [2315776,2316288) | 20260505 09:58:58.585 | INFO | MemInfo: { | 20260505 09:58:58.585 | INFO | "gpu" : " 17 GB", | 20260505 09:58:58.585 | INFO | "huge" : "", | 20260505 09:58:58.585 | INFO | "huge_shared" : " 27 GB", | 20260505 09:58:58.585 | INFO | "malloc" : " 14 GB", | 20260505 09:58:58.585 | INFO | "pinned" : " 29 GB" | 20260505 09:58:58.585 | INFO | } | 20260505 09:58:58.585 | INFO | Perf report for the last 5.13815 seconds: | 20260505 09:58:58.585 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:58:58.585 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:58.585 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:58.585 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:58.585 | INFO | ANALYSIS : 63.1%, 3.0, 98.0ms, 0 | 20260505 09:58:58.585 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:58.585 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:58.585 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 09:58:58.585 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:58.585 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:58:58.586 | INFO | Uploaded 36GiB to GPU in 2.15725s (16.6879GiB/s) | 20260505 09:58:58.586 | INFO | Downloaded 34GiB to GPU in 2.07374s (16.3955GiB/s) | 20260505 09:59:03.725 | INFO | Analyzing chunk frames = [2316288,2316800) | 20260505 09:59:03.725 | INFO | MemInfo: { | 20260505 09:59:03.725 | INFO | "gpu" : " 17 GB", | 20260505 09:59:03.725 | INFO | "huge" : "", | 20260505 09:59:03.725 | INFO | "huge_shared" : " 27 GB", | 20260505 09:59:03.725 | INFO | "malloc" : " 14 GB", | 20260505 09:59:03.725 | INFO | "pinned" : " 29 GB" | 20260505 09:59:03.725 | INFO | } | 20260505 09:59:03.725 | INFO | Perf report for the last 5.1402 seconds: | 20260505 09:59:03.725 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:59:03.725 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:03.725 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:03.725 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:03.725 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 09:59:03.725 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:03.725 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:03.725 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 09:59:03.725 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:03.725 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:03.726 | INFO | Uploaded 36GiB to GPU in 2.23458s (16.1104GiB/s) | 20260505 09:59:03.726 | INFO | Downloaded 34GiB to GPU in 2.1143s (16.081GiB/s) | 20260505 09:59:08.866 | INFO | Analyzing chunk frames = [2316800,2317312) | 20260505 09:59:08.866 | INFO | MemInfo: { | 20260505 09:59:08.866 | INFO | "gpu" : " 17 GB", | 20260505 09:59:08.866 | INFO | "huge" : "", | 20260505 09:59:08.866 | INFO | "huge_shared" : " 27 GB", | 20260505 09:59:08.866 | INFO | "malloc" : " 14 GB", | 20260505 09:59:08.866 | INFO | "pinned" : " 29 GB" | 20260505 09:59:08.866 | INFO | } | 20260505 09:59:08.866 | INFO | Perf report for the last 5.14057 seconds: | 20260505 09:59:08.866 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:59:08.866 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:08.866 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:08.866 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:08.866 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 09:59:08.866 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:08.866 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:08.866 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 09:59:08.866 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:08.866 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:08.866 | INFO | Uploaded 36GiB to GPU in 2.23441s (16.1117GiB/s) | 20260505 09:59:08.866 | INFO | Downloaded 34GiB to GPU in 2.1011s (16.182GiB/s) | 20260505 09:59:14.005 | INFO | Analyzing chunk frames = [2317312,2317824) | 20260505 09:59:14.005 | INFO | MemInfo: { | 20260505 09:59:14.005 | INFO | "gpu" : " 17 GB", | 20260505 09:59:14.005 | INFO | "huge" : "", | 20260505 09:59:14.005 | INFO | "huge_shared" : " 27 GB", | 20260505 09:59:14.007 | INFO | "malloc" : " 14 GB", | 20260505 09:59:14.007 | INFO | "pinned" : " 29 GB" | 20260505 09:59:14.007 | INFO | } | 20260505 09:59:14.007 | INFO | Perf report for the last 5.14083 seconds: | 20260505 09:59:14.007 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:59:14.007 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:14.007 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:14.007 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:14.007 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 09:59:14.007 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:14.007 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:14.007 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 09:59:14.007 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:14.007 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:14.007 | INFO | Uploaded 36GiB to GPU in 2.19964s (16.3663GiB/s) | 20260505 09:59:14.007 | INFO | Downloaded 34GiB to GPU in 2.12529s (15.9978GiB/s) | 20260505 09:59:19.144 | INFO | Analyzing chunk frames = [2317824,2318336) | 20260505 09:59:19.144 | INFO | MemInfo: { | 20260505 09:59:19.144 | INFO | "gpu" : " 17 GB", | 20260505 09:59:19.144 | INFO | "huge" : "", | 20260505 09:59:19.144 | INFO | "huge_shared" : " 27 GB", | 20260505 09:59:19.144 | INFO | "malloc" : " 14 GB", | 20260505 09:59:19.144 | INFO | "pinned" : " 29 GB" | 20260505 09:59:19.144 | INFO | } | 20260505 09:59:19.144 | INFO | Perf report for the last 5.13751 seconds: | 20260505 09:59:19.144 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:59:19.144 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:19.144 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:19.144 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:19.144 | INFO | ANALYSIS : 64.6%, 3.0, 100.8ms, 0 | 20260505 09:59:19.144 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:19.144 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:19.144 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 09:59:19.145 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:19.145 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:19.145 | INFO | Uploaded 36GiB to GPU in 2.24613s (16.0276GiB/s) | 20260505 09:59:19.145 | INFO | Downloaded 34GiB to GPU in 2.08765s (16.2863GiB/s) | 20260505 09:59:19.145 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 09:59:19.145 | INFO | Total mem usage: 15.3 GiB | 20260505 09:59:19.145 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 09:59:19.145 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -7.04%) | 20260505 09:59:19.145 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 09:59:19.145 | INFO | High Water Breakdown: | 20260505 09:59:19.145 | INFO | 60.8% : 8.71 GiB ( 93.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 09:59:19.145 | INFO | 39% : 5.59 GiB ( 84.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 09:59:19.145 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 09:59:19.145 | INFO | ------------------------------------------ | 20260505 09:59:19.145 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:59:19.145 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 09:59:19.145 | INFO | Total mem usage: 26.5 GiB | 20260505 09:59:19.145 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 09:59:19.145 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 09:59:19.145 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 09:59:19.145 | INFO | High Water Breakdown: | 20260505 09:59:19.145 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:59:19.145 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:59:19.145 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:59:19.145 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:59:19.145 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:59:19.145 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:59:19.145 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 09:59:19.145 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 09:59:19.145 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:59:19.145 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 09:59:19.145 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 09:59:19.145 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 09:59:19.145 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 09:59:19.145 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 09:59:19.145 | INFO | ------------------------------------------ | 20260505 09:59:19.145 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:59:19.145 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 09:59:19.145 | INFO | Total mem usage: 25.6 GiB | 20260505 09:59:19.145 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 09:59:19.145 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 09:59:19.145 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 09:59:19.145 | INFO | High Water Breakdown: | 20260505 09:59:19.145 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 09:59:19.145 | INFO | ------------------------------------------ | 20260505 09:59:19.145 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:59:19.145 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 09:59:19.145 | INFO | Total mem usage: 15.7 GiB | 20260505 09:59:19.146 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 09:59:19.146 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 09:59:19.146 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 09:59:19.146 | INFO | High Water Breakdown: | 20260505 09:59:19.146 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 09:59:19.146 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 09:59:19.146 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 09:59:19.146 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 09:59:19.146 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 09:59:19.146 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 09:59:19.146 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 09:59:19.146 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 09:59:19.146 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 09:59:19.146 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 09:59:19.146 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 09:59:19.146 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 09:59:19.146 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 09:59:19.146 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 09:59:19.146 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 09:59:19.146 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 09:59:19.146 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 09:59:19.146 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 09:59:19.146 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 09:59:19.146 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 09:59:19.146 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 09:59:19.146 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 09:59:19.146 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 09:59:19.146 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 09:59:19.146 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 09:59:19.146 | INFO | ------------------------------------------ | 20260505 09:59:19.146 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 09:59:19.146 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 09:59:19.146 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:59:19.146 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:59:19.146 | INFO | PerfReport: Full Execution -- 3.25e+01 / 7.75e+01, 1.40e+01 / 2.28e+00 | 20260505 09:59:19.146 | INFO | PerfReport: 0.29%, AggregateStats -- 9.34e-02 / 1.04e-01, 1.17e-02 / 6.10e-03 | 20260505 09:59:19.146 | INFO | PerfReport: 11.64%, Baseline -- 3.78e+00 / 4.47e+00, 6.14e-01 / 2.01e-01 | 20260505 09:59:19.146 | INFO | PerfReport: 3.31%, Histogram -- 1.07e+00 / 1.51e+00, 2.11e-01 / 8.59e-02 | 20260505 09:59:19.146 | INFO | PerfReport: 15.66%, DME -- 4.07e+01 / 4.52e+01, 4.65e+00 / 2.29e+00 | 20260505 09:59:19.146 | INFO | PerfReport: 56.68%, FrameLabeling -- 1.84e+01 / 2.14e+01, 1.90e+00 / 2.04e+00 | 20260505 09:59:19.146 | INFO | PerfReport: 7.29%, PulseAccumulating -- 2.37e+00 / 2.92e+00, 3.94e-01 / 2.27e-01 | 20260505 09:59:19.146 | INFO | PerfReport: 3.79%, Metrics -- 1.23e+00 / 1.49e+00, 3.60e-01 / 2.17e-01 | 20260505 09:59:19.146 | INFO | PerfReport: 1.34%, PulseSerializing -- 4.36e-01 / 4.81e-01, 8.20e-02 / 3.14e-02 | 20260505 09:59:19.146 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 09:59:19.146 | INFO | PerfReport: Gathered 1536 samples. | 20260505 09:59:19.146 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 09:59:19.146 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.42e+01 / 0.00e+00 | 20260505 09:59:19.146 | INFO | PerfReport: 22.59%, Upload -- 2.30e+01 / 0.00e+00, 8.36e+00 / 0.00e+00 | 20260505 09:59:19.146 | INFO | PerfReport: 21.79%, Download -- 2.22e+01 / 0.00e+00, 8.29e+00 / 0.00e+00 | 20260505 09:59:19.146 | INFO | PerfReport: 31.96%, Compute -- 3.25e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260505 09:59:19.146 | INFO | PerfReport: 1.92%, LockUpload -- 1.96e+00 / 0.00e+00, 7.47e+00 / 0.00e+00 | 20260505 09:59:19.146 | INFO | PerfReport: 1.07%, LockDownload -- 1.09e+00 / 0.00e+00, 2.67e+00 / 0.00e+00 | 20260505 09:59:19.146 | INFO | PerfReport: 20.66%, LockCompute -- 2.10e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260505 09:59:19.146 | INFO | PerfReport: 0.02%, Misc -- 1.58e-02 / 0.00e+00, 4.08e-02 / 0.00e+00 | 20260505 09:59:19.275 | INFO | Starting a round of Baz IO | 20260505 09:59:22.553 | INFO | Populating RTMetrics report | 20260505 09:59:22.553 | INFO | RTMetrics report populated | 20260505 09:59:22.553 | INFO | Opening tmp RTMetrics json file | 20260505 09:59:22.555 | INFO | Serializing RTMetrics to json | 20260505 09:59:22.557 | INFO | Serialization done, swapping tmp file now... | 20260505 09:59:22.559 | INFO | RTMetrics done | 20260505 09:59:22.785 | INFO | Finished a round of Baz IO: Wrote 2.428 GiB in 3.509 seconds | 20260505 09:59:22.785 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.45% / 0.01% / 74.54% | 20260505 09:59:22.785 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.70227% / 1.58946e-05% / 0.000111262% | 20260505 09:59:22.785 | INFO | Baz IO overall: | 20260505 09:59:22.785 | INFO | BytesWritten : 1350168876208 | 20260505 09:59:22.785 | INFO | NumEvents : 642725691791 | 20260505 09:59:24.282 | INFO | Analyzing chunk frames = [2318336,2318848) | 20260505 09:59:24.282 | INFO | MemInfo: { | 20260505 09:59:24.282 | INFO | "gpu" : " 17 GB", | 20260505 09:59:24.282 | INFO | "huge" : "", | 20260505 09:59:24.282 | INFO | "huge_shared" : " 27 GB", | 20260505 09:59:24.282 | INFO | "malloc" : " 14 GB", | 20260505 09:59:24.282 | INFO | "pinned" : " 29 GB" | 20260505 09:59:24.282 | INFO | } | 20260505 09:59:24.282 | INFO | Perf report for the last 5.13808 seconds: | 20260505 09:59:24.283 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:59:24.283 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:24.283 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:24.283 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:24.283 | INFO | ANALYSIS : 65.2%, 2.9, 99.9ms, 0 | 20260505 09:59:24.283 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:24.283 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260505 09:59:24.283 | INFO | PRE_HQ : 62.7%, 1.8, 59.6ms, 0 | 20260505 09:59:24.283 | INFO | BAZWRITER : 67.6%, 4.1, 143.5ms, 0 | 20260505 09:59:24.283 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 09:59:24.283 | INFO | Uploaded 36GiB to GPU in 2.23518s (16.1061GiB/s) | 20260505 09:59:24.283 | INFO | Downloaded 38GiB to GPU in 2.34319s (16.2172GiB/s) | 20260505 09:59:29.422 | INFO | Analyzing chunk frames = [2318848,2319360) | 20260505 09:59:29.422 | INFO | MemInfo: { | 20260505 09:59:29.422 | INFO | "gpu" : " 17 GB", | 20260505 09:59:29.422 | INFO | "huge" : "", | 20260505 09:59:29.422 | INFO | "huge_shared" : " 27 GB", | 20260505 09:59:29.422 | INFO | "malloc" : " 14 GB", | 20260505 09:59:29.422 | INFO | "pinned" : " 29 GB" | 20260505 09:59:29.423 | INFO | } | 20260505 09:59:29.423 | INFO | Perf report for the last 5.14013 seconds: | 20260505 09:59:29.423 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:59:29.423 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:29.423 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:29.423 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:29.423 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 09:59:29.423 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:29.423 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:29.423 | INFO | PRE_HQ : 29.4%, 1.0, 15.5ms, 0 | 20260505 09:59:29.423 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:29.423 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:29.423 | INFO | Uploaded 36GiB to GPU in 2.17805s (16.5286GiB/s) | 20260505 09:59:29.423 | INFO | Downloaded 34GiB to GPU in 2.10001s (16.1904GiB/s) | 20260505 09:59:34.566 | INFO | Analyzing chunk frames = [2319360,2319872) | 20260505 09:59:34.566 | INFO | MemInfo: { | 20260505 09:59:34.566 | INFO | "gpu" : " 17 GB", | 20260505 09:59:34.566 | INFO | "huge" : "", | 20260505 09:59:34.566 | INFO | "huge_shared" : " 27 GB", | 20260505 09:59:34.566 | INFO | "malloc" : " 14 GB", | 20260505 09:59:34.566 | INFO | "pinned" : " 29 GB" | 20260505 09:59:34.566 | INFO | } | 20260505 09:59:34.566 | INFO | Perf report for the last 5.14375 seconds: | 20260505 09:59:34.566 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:59:34.566 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:34.566 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:34.566 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:34.566 | INFO | ANALYSIS : 63.1%, 3.0, 98.2ms, 0 | 20260505 09:59:34.566 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:34.566 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:34.567 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 09:59:34.567 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:34.567 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:34.567 | INFO | Uploaded 36GiB to GPU in 2.14652s (16.7713GiB/s) | 20260505 09:59:34.567 | INFO | Downloaded 34GiB to GPU in 2.06448s (16.4691GiB/s) | 20260505 09:59:39.700 | INFO | Analyzing chunk frames = [2319872,2320384) | 20260505 09:59:39.700 | INFO | MemInfo: { | 20260505 09:59:39.700 | INFO | "gpu" : " 17 GB", | 20260505 09:59:39.700 | INFO | "huge" : "", | 20260505 09:59:39.700 | INFO | "huge_shared" : " 27 GB", | 20260505 09:59:39.700 | INFO | "malloc" : " 14 GB", | 20260505 09:59:39.700 | INFO | "pinned" : " 29 GB" | 20260505 09:59:39.700 | INFO | } | 20260505 09:59:39.700 | INFO | Perf report for the last 5.13377 seconds: | 20260505 09:59:39.700 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:59:39.700 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:39.700 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:39.700 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:39.700 | INFO | ANALYSIS : 63.2%, 3.0, 98.1ms, 0 | 20260505 09:59:39.700 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:39.700 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:39.700 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 09:59:39.700 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:39.700 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:39.700 | INFO | Uploaded 36GiB to GPU in 2.15908s (16.6737GiB/s) | 20260505 09:59:39.701 | INFO | Downloaded 34GiB to GPU in 2.07445s (16.3899GiB/s) | 20260505 09:59:44.840 | INFO | Analyzing chunk frames = [2320384,2320896) | 20260505 09:59:44.840 | INFO | MemInfo: { | 20260505 09:59:44.840 | INFO | "gpu" : " 17 GB", | 20260505 09:59:44.840 | INFO | "huge" : "", | 20260505 09:59:44.840 | INFO | "huge_shared" : " 27 GB", | 20260505 09:59:44.840 | INFO | "malloc" : " 14 GB", | 20260505 09:59:44.840 | INFO | "pinned" : " 29 GB" | 20260505 09:59:44.840 | INFO | } | 20260505 09:59:44.840 | INFO | Perf report for the last 5.13981 seconds: | 20260505 09:59:44.840 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:59:44.840 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:44.840 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:44.840 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:44.840 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 09:59:44.840 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:44.840 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:44.840 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 09:59:44.840 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:44.840 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:44.840 | INFO | Uploaded 36GiB to GPU in 2.23482s (16.1087GiB/s) | 20260505 09:59:44.840 | INFO | Downloaded 34GiB to GPU in 2.10576s (16.1462GiB/s) | 20260505 09:59:49.981 | INFO | Analyzing chunk frames = [2320896,2321408) | 20260505 09:59:49.981 | INFO | MemInfo: { | 20260505 09:59:49.981 | INFO | "gpu" : " 17 GB", | 20260505 09:59:49.981 | INFO | "huge" : "", | 20260505 09:59:49.981 | INFO | "huge_shared" : " 27 GB", | 20260505 09:59:49.981 | INFO | "malloc" : " 14 GB", | 20260505 09:59:49.981 | INFO | "pinned" : " 29 GB" | 20260505 09:59:49.981 | INFO | } | 20260505 09:59:49.981 | INFO | Perf report for the last 5.14084 seconds: | 20260505 09:59:49.981 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:59:49.981 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:49.981 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:49.981 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:49.981 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 09:59:49.981 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:49.981 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:49.981 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 09:59:49.981 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:49.981 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:49.981 | INFO | Uploaded 36GiB to GPU in 2.23401s (16.1146GiB/s) | 20260505 09:59:49.981 | INFO | Downloaded 34GiB to GPU in 2.09309s (16.2439GiB/s) | 20260505 09:59:55.120 | INFO | Analyzing chunk frames = [2321408,2321920) | 20260505 09:59:55.120 | INFO | MemInfo: { | 20260505 09:59:55.120 | INFO | "gpu" : " 17 GB", | 20260505 09:59:55.120 | INFO | "huge" : "", | 20260505 09:59:55.120 | INFO | "huge_shared" : " 27 GB", | 20260505 09:59:55.120 | INFO | "malloc" : " 14 GB", | 20260505 09:59:55.120 | INFO | "pinned" : " 29 GB" | 20260505 09:59:55.120 | INFO | } | 20260505 09:59:55.120 | INFO | Perf report for the last 5.13899 seconds: | 20260505 09:59:55.120 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 09:59:55.120 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:55.120 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:55.120 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:55.120 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 09:59:55.120 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:55.120 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:55.120 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 09:59:55.120 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:55.120 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 09:59:55.120 | INFO | Uploaded 36GiB to GPU in 2.19311s (16.415GiB/s) | 20260505 09:59:55.120 | INFO | Downloaded 34GiB to GPU in 2.15743s (15.7595GiB/s) | 20260505 10:00:00.257 | INFO | Analyzing chunk frames = [2321920,2322432) | 20260505 10:00:00.257 | INFO | MemInfo: { | 20260505 10:00:00.257 | INFO | "gpu" : " 17 GB", | 20260505 10:00:00.257 | INFO | "huge" : "", | 20260505 10:00:00.257 | INFO | "huge_shared" : " 27 GB", | 20260505 10:00:00.257 | INFO | "malloc" : " 14 GB", | 20260505 10:00:00.257 | INFO | "pinned" : " 29 GB" | 20260505 10:00:00.258 | INFO | } | 20260505 10:00:00.258 | INFO | Perf report for the last 5.13773 seconds: | 20260505 10:00:00.258 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:00:00.258 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:00.258 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:00.258 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:00.258 | INFO | ANALYSIS : 64.6%, 3.0, 100.9ms, 0 | 20260505 10:00:00.258 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:00.258 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:00.258 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 10:00:00.258 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:00.258 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:00.258 | INFO | Uploaded 36GiB to GPU in 2.2496s (16.0029GiB/s) | 20260505 10:00:00.258 | INFO | Downloaded 34GiB to GPU in 2.092s (16.2524GiB/s) | 20260505 10:00:00.386 | INFO | Starting a round of Baz IO | 20260505 10:00:03.650 | INFO | Populating RTMetrics report | 20260505 10:00:03.650 | INFO | RTMetrics report populated | 20260505 10:00:03.650 | INFO | Opening tmp RTMetrics json file | 20260505 10:00:03.668 | INFO | Serializing RTMetrics to json | 20260505 10:00:03.670 | INFO | Serialization done, swapping tmp file now... | 20260505 10:00:03.672 | INFO | RTMetrics done | 20260505 10:00:03.852 | INFO | Finished a round of Baz IO: Wrote 2.428 GiB in 3.465 seconds | 20260505 10:00:03.852 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.44% / 0.01% / 74.55% | 20260505 10:00:03.852 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.70021% / 1.58946e-05% / 8.74201e-05% | 20260505 10:00:03.852 | INFO | Baz IO overall: | 20260505 10:00:03.852 | INFO | BytesWritten : 1352776199048 | 20260505 10:00:03.852 | INFO | NumEvents : 643970602094 | 20260505 10:00:05.397 | INFO | Analyzing chunk frames = [2322432,2322944) | 20260505 10:00:05.397 | INFO | MemInfo: { | 20260505 10:00:05.397 | INFO | "gpu" : " 17 GB", | 20260505 10:00:05.397 | INFO | "huge" : "", | 20260505 10:00:05.397 | INFO | "huge_shared" : " 27 GB", | 20260505 10:00:05.397 | INFO | "malloc" : " 14 GB", | 20260505 10:00:05.397 | INFO | "pinned" : " 29 GB" | 20260505 10:00:05.397 | INFO | } | 20260505 10:00:05.397 | INFO | Perf report for the last 5.13966 seconds: | 20260505 10:00:05.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:00:05.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:05.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:05.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:05.397 | INFO | ANALYSIS : 64.9%, 2.9, 98.8ms, 0 | 20260505 10:00:05.397 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:05.397 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260505 10:00:05.397 | INFO | PRE_HQ : 62.9%, 1.8, 59.9ms, 0 | 20260505 10:00:05.397 | INFO | BAZWRITER : 66.7%, 4.3, 146.6ms, 0 | 20260505 10:00:05.397 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 10:00:05.398 | INFO | Uploaded 36GiB to GPU in 2.27259s (15.841GiB/s) | 20260505 10:00:05.398 | INFO | Downloaded 38GiB to GPU in 2.38048s (15.9631GiB/s) | 20260505 10:00:10.537 | INFO | Analyzing chunk frames = [2322944,2323456) | 20260505 10:00:10.537 | INFO | MemInfo: { | 20260505 10:00:10.537 | INFO | "gpu" : " 17 GB", | 20260505 10:00:10.537 | INFO | "huge" : "", | 20260505 10:00:10.537 | INFO | "huge_shared" : " 27 GB", | 20260505 10:00:10.537 | INFO | "malloc" : " 14 GB", | 20260505 10:00:10.537 | INFO | "pinned" : " 29 GB" | 20260505 10:00:10.537 | INFO | } | 20260505 10:00:10.537 | INFO | Perf report for the last 5.14014 seconds: | 20260505 10:00:10.537 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:00:10.537 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:10.537 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:10.537 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:10.537 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 10:00:10.538 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:10.538 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:10.538 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 10:00:10.538 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:10.538 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:10.538 | INFO | Uploaded 36GiB to GPU in 2.17211s (16.5737GiB/s) | 20260505 10:00:10.538 | INFO | Downloaded 34GiB to GPU in 2.10379s (16.1613GiB/s) | 20260505 10:00:15.676 | INFO | Analyzing chunk frames = [2323456,2323968) | 20260505 10:00:15.676 | INFO | MemInfo: { | 20260505 10:00:15.676 | INFO | "gpu" : " 17 GB", | 20260505 10:00:15.676 | INFO | "huge" : "", | 20260505 10:00:15.676 | INFO | "huge_shared" : " 27 GB", | 20260505 10:00:15.676 | INFO | "malloc" : " 14 GB", | 20260505 10:00:15.676 | INFO | "pinned" : " 29 GB" | 20260505 10:00:15.676 | INFO | } | 20260505 10:00:15.677 | INFO | Perf report for the last 5.13914 seconds: | 20260505 10:00:15.677 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:00:15.677 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:15.677 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:15.677 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:15.677 | INFO | ANALYSIS : 63.0%, 3.0, 98.0ms, 0 | 20260505 10:00:15.677 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:15.677 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:15.677 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 10:00:15.677 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:15.677 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:15.677 | INFO | Uploaded 36GiB to GPU in 2.15052s (16.7401GiB/s) | 20260505 10:00:15.677 | INFO | Downloaded 34GiB to GPU in 2.05579s (16.5386GiB/s) | 20260505 10:00:20.816 | INFO | Analyzing chunk frames = [2323968,2324480) | 20260505 10:00:20.816 | INFO | MemInfo: { | 20260505 10:00:20.816 | INFO | "gpu" : " 17 GB", | 20260505 10:00:20.816 | INFO | "huge" : "", | 20260505 10:00:20.816 | INFO | "huge_shared" : " 27 GB", | 20260505 10:00:20.816 | INFO | "malloc" : " 14 GB", | 20260505 10:00:20.816 | INFO | "pinned" : " 29 GB" | 20260505 10:00:20.816 | INFO | } | 20260505 10:00:20.816 | INFO | Perf report for the last 5.13972 seconds: | 20260505 10:00:20.816 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:00:20.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:20.816 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:20.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:20.816 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 10:00:20.816 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:20.816 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:20.816 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 10:00:20.816 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:20.816 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:20.817 | INFO | Uploaded 36GiB to GPU in 2.17992s (16.5143GiB/s) | 20260505 10:00:20.817 | INFO | Downloaded 34GiB to GPU in 2.07625s (16.3756GiB/s) | 20260505 10:00:25.955 | INFO | Analyzing chunk frames = [2324480,2324992) | 20260505 10:00:25.955 | INFO | MemInfo: { | 20260505 10:00:25.955 | INFO | "gpu" : " 17 GB", | 20260505 10:00:25.955 | INFO | "huge" : "", | 20260505 10:00:25.956 | INFO | "huge_shared" : " 27 GB", | 20260505 10:00:25.956 | INFO | "malloc" : " 14 GB", | 20260505 10:00:25.956 | INFO | "pinned" : " 29 GB" | 20260505 10:00:25.956 | INFO | } | 20260505 10:00:25.956 | INFO | Perf report for the last 5.13949 seconds: | 20260505 10:00:25.956 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:00:25.956 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:25.956 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:25.956 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:25.956 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 10:00:25.956 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:25.956 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:25.956 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 10:00:25.956 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:25.956 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:25.956 | INFO | Uploaded 36GiB to GPU in 2.23998s (16.0716GiB/s) | 20260505 10:00:25.956 | INFO | Downloaded 34GiB to GPU in 2.10872s (16.1235GiB/s) | 20260505 10:00:31.095 | INFO | Analyzing chunk frames = [2324992,2325504) | 20260505 10:00:31.095 | INFO | MemInfo: { | 20260505 10:00:31.095 | INFO | "gpu" : " 17 GB", | 20260505 10:00:31.095 | INFO | "huge" : "", | 20260505 10:00:31.095 | INFO | "huge_shared" : " 27 GB", | 20260505 10:00:31.095 | INFO | "malloc" : " 14 GB", | 20260505 10:00:31.095 | INFO | "pinned" : " 29 GB" | 20260505 10:00:31.095 | INFO | } | 20260505 10:00:31.096 | INFO | Perf report for the last 5.13974 seconds: | 20260505 10:00:31.096 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:00:31.096 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:31.096 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:31.096 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:31.096 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 10:00:31.096 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:31.096 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:31.096 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:00:31.096 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:31.096 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:31.096 | INFO | Uploaded 36GiB to GPU in 2.23747s (16.0896GiB/s) | 20260505 10:00:31.096 | INFO | Downloaded 34GiB to GPU in 2.09947s (16.1946GiB/s) | 20260505 10:00:36.233 | INFO | Analyzing chunk frames = [2325504,2326016) | 20260505 10:00:36.233 | INFO | MemInfo: { | 20260505 10:00:36.233 | INFO | "gpu" : " 17 GB", | 20260505 10:00:36.233 | INFO | "huge" : "", | 20260505 10:00:36.233 | INFO | "huge_shared" : " 27 GB", | 20260505 10:00:36.233 | INFO | "malloc" : " 14 GB", | 20260505 10:00:36.234 | INFO | "pinned" : " 29 GB" | 20260505 10:00:36.234 | INFO | } | 20260505 10:00:36.234 | INFO | Perf report for the last 5.13808 seconds: | 20260505 10:00:36.234 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:00:36.234 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:36.234 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:36.234 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:36.234 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 10:00:36.234 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:36.234 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:36.234 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 10:00:36.234 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:36.234 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:36.234 | INFO | Uploaded 36GiB to GPU in 2.21135s (16.2796GiB/s) | 20260505 10:00:36.234 | INFO | Downloaded 34GiB to GPU in 2.12721s (15.9834GiB/s) | 20260505 10:00:41.373 | INFO | Analyzing chunk frames = [2326016,2326528) | 20260505 10:00:41.373 | INFO | MemInfo: { | 20260505 10:00:41.373 | INFO | "gpu" : " 17 GB", | 20260505 10:00:41.373 | INFO | "huge" : "", | 20260505 10:00:41.373 | INFO | "huge_shared" : " 27 GB", | 20260505 10:00:41.373 | INFO | "malloc" : " 14 GB", | 20260505 10:00:41.373 | INFO | "pinned" : " 29 GB" | 20260505 10:00:41.373 | INFO | } | 20260505 10:00:41.373 | INFO | Perf report for the last 5.13934 seconds: | 20260505 10:00:41.373 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:00:41.373 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:41.373 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:41.373 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:41.373 | INFO | ANALYSIS : 64.7%, 3.0, 101.0ms, 0 | 20260505 10:00:41.373 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:41.373 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:41.373 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 10:00:41.373 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:41.373 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:41.373 | INFO | Uploaded 36GiB to GPU in 2.2498s (16.0014GiB/s) | 20260505 10:00:41.373 | INFO | Downloaded 34GiB to GPU in 2.09541s (16.226GiB/s) | 20260505 10:00:41.373 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 10:00:41.373 | INFO | Total mem usage: 15.3 GiB | 20260505 10:00:41.373 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 10:00:41.373 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -4.92%) | 20260505 10:00:41.373 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 10:00:41.373 | INFO | High Water Breakdown: | 20260505 10:00:41.374 | INFO | 61.2% : 8.95 GiB ( 95.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 10:00:41.374 | INFO | 38.6% : 5.64 GiB ( 84.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 10:00:41.374 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 10:00:41.374 | INFO | ------------------------------------------ | 20260505 10:00:41.374 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:00:41.374 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 10:00:41.374 | INFO | Total mem usage: 26.5 GiB | 20260505 10:00:41.374 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 10:00:41.374 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 10:00:41.374 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 10:00:41.374 | INFO | High Water Breakdown: | 20260505 10:00:41.374 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:00:41.374 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:00:41.374 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:00:41.374 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:00:41.374 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:00:41.374 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:00:41.374 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 10:00:41.374 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 10:00:41.374 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:00:41.374 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 10:00:41.374 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 10:00:41.374 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 10:00:41.374 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 10:00:41.374 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 10:00:41.374 | INFO | ------------------------------------------ | 20260505 10:00:41.374 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:00:41.374 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 10:00:41.374 | INFO | Total mem usage: 25.6 GiB | 20260505 10:00:41.374 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 10:00:41.374 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 10:00:41.374 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 10:00:41.374 | INFO | High Water Breakdown: | 20260505 10:00:41.374 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 10:00:41.374 | INFO | ------------------------------------------ | 20260505 10:00:41.374 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:00:41.374 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 10:00:41.374 | INFO | Total mem usage: 15.7 GiB | 20260505 10:00:41.374 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 10:00:41.374 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 10:00:41.374 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 10:00:41.374 | INFO | High Water Breakdown: | 20260505 10:00:41.374 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 10:00:41.374 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 10:00:41.374 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 10:00:41.374 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 10:00:41.374 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 10:00:41.374 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 10:00:41.374 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 10:00:41.374 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 10:00:41.374 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 10:00:41.374 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:00:41.374 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 10:00:41.374 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:00:41.374 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 10:00:41.374 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 10:00:41.374 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 10:00:41.374 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 10:00:41.374 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:00:41.374 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 10:00:41.374 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:00:41.374 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:00:41.374 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:00:41.374 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 10:00:41.374 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 10:00:41.374 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:00:41.374 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 10:00:41.374 | INFO | ------------------------------------------ | 20260505 10:00:41.374 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:00:41.374 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 10:00:41.374 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:00:41.374 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:00:41.374 | INFO | PerfReport: Full Execution -- 3.24e+01 / 7.76e+01, 1.37e+01 / 2.37e+00 | 20260505 10:00:41.374 | INFO | PerfReport: 0.29%, AggregateStats -- 9.31e-02 / 1.04e-01, 1.03e-02 / 5.27e-03 | 20260505 10:00:41.374 | INFO | PerfReport: 11.64%, Baseline -- 3.78e+00 / 4.42e+00, 6.12e-01 / 2.86e-01 | 20260505 10:00:41.374 | INFO | PerfReport: 3.31%, Histogram -- 1.07e+00 / 1.49e+00, 2.09e-01 / 8.96e-02 | 20260505 10:00:41.374 | INFO | PerfReport: 15.64%, DME -- 4.06e+01 / 4.49e+01, 4.42e+00 / 2.11e+00 | 20260505 10:00:41.374 | INFO | PerfReport: 56.67%, FrameLabeling -- 1.84e+01 / 2.17e+01, 1.88e+00 / 2.18e+00 | 20260505 10:00:41.374 | INFO | PerfReport: 7.32%, PulseAccumulating -- 2.37e+00 / 2.95e+00, 4.00e-01 / 2.16e-01 | 20260505 10:00:41.374 | INFO | PerfReport: 3.79%, Metrics -- 1.23e+00 / 1.52e+00, 3.63e-01 / 1.73e-01 | 20260505 10:00:41.374 | INFO | PerfReport: 1.34%, PulseSerializing -- 4.35e-01 / 4.76e-01, 8.22e-02 / 2.88e-02 | 20260505 10:00:41.374 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 10:00:41.374 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:00:41.375 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:00:41.375 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260505 10:00:41.375 | INFO | PerfReport: 22.66%, Upload -- 2.30e+01 / 0.00e+00, 8.31e+00 / 0.00e+00 | 20260505 10:00:41.375 | INFO | PerfReport: 21.85%, Download -- 2.22e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260505 10:00:41.375 | INFO | PerfReport: 31.96%, Compute -- 3.25e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 10:00:41.375 | INFO | PerfReport: 1.90%, LockUpload -- 1.93e+00 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260505 10:00:41.375 | INFO | PerfReport: 1.11%, LockDownload -- 1.13e+00 / 0.00e+00, 2.70e+00 / 0.00e+00 | 20260505 10:00:41.375 | INFO | PerfReport: 20.51%, LockCompute -- 2.08e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260505 10:00:41.375 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.90e-02 / 0.00e+00 | 20260505 10:00:41.517 | INFO | Starting a round of Baz IO | 20260505 10:00:44.782 | INFO | Populating RTMetrics report | 20260505 10:00:44.783 | INFO | RTMetrics report populated | 20260505 10:00:44.783 | INFO | Opening tmp RTMetrics json file | 20260505 10:00:44.828 | INFO | Serializing RTMetrics to json | 20260505 10:00:44.832 | INFO | Serialization done, swapping tmp file now... | 20260505 10:00:44.833 | INFO | RTMetrics done | 20260505 10:00:44.934 | INFO | Finished a round of Baz IO: Wrote 2.425 GiB in 3.416 seconds | 20260505 10:00:44.934 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.43% / 0.01% / 74.56% | 20260505 10:00:44.934 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.68601% / 1.58946e-05% / 5.16574e-05% | 20260505 10:00:44.934 | INFO | Baz IO overall: | 20260505 10:00:44.934 | INFO | BytesWritten : 1355380214934 | 20260505 10:00:44.934 | INFO | NumEvents : 645213094212 | 20260505 10:00:46.515 | INFO | Analyzing chunk frames = [2326528,2327040) | 20260505 10:00:46.515 | INFO | MemInfo: { | 20260505 10:00:46.515 | INFO | "gpu" : " 17 GB", | 20260505 10:00:46.515 | INFO | "huge" : "", | 20260505 10:00:46.515 | INFO | "huge_shared" : " 27 GB", | 20260505 10:00:46.515 | INFO | "malloc" : " 14 GB", | 20260505 10:00:46.515 | INFO | "pinned" : " 29 GB" | 20260505 10:00:46.515 | INFO | } | 20260505 10:00:46.515 | INFO | Perf report for the last 5.14211 seconds: | 20260505 10:00:46.515 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:00:46.515 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:46.515 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:46.515 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:46.515 | INFO | ANALYSIS : 65.2%, 2.9, 99.8ms, 0 | 20260505 10:00:46.515 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:46.515 | INFO | RT_METRICS : 46.7%, 1.0, 24.5ms, 0 | 20260505 10:00:46.515 | INFO | PRE_HQ : 62.6%, 1.8, 60.0ms, 0 | 20260505 10:00:46.515 | INFO | BAZWRITER : 66.1%, 4.2, 141.5ms, 0 | 20260505 10:00:46.515 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260505 10:00:46.515 | INFO | Uploaded 36GiB to GPU in 2.22215s (16.2005GiB/s) | 20260505 10:00:46.515 | INFO | Downloaded 38GiB to GPU in 2.34176s (16.2271GiB/s) | 20260505 10:00:51.651 | INFO | Analyzing chunk frames = [2327040,2327552) | 20260505 10:00:51.651 | INFO | MemInfo: { | 20260505 10:00:51.651 | INFO | "gpu" : " 17 GB", | 20260505 10:00:51.651 | INFO | "huge" : "", | 20260505 10:00:51.651 | INFO | "huge_shared" : " 27 GB", | 20260505 10:00:51.651 | INFO | "malloc" : " 14 GB", | 20260505 10:00:51.651 | INFO | "pinned" : " 29 GB" | 20260505 10:00:51.651 | INFO | } | 20260505 10:00:51.651 | INFO | Perf report for the last 5.13628 seconds: | 20260505 10:00:51.651 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:00:51.651 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:51.651 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:51.651 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:51.651 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 10:00:51.651 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:51.652 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:51.652 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:00:51.652 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:51.652 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:51.652 | INFO | Uploaded 36GiB to GPU in 2.17607s (16.5436GiB/s) | 20260505 10:00:51.652 | INFO | Downloaded 34GiB to GPU in 2.10192s (16.1757GiB/s) | 20260505 10:00:56.791 | INFO | Analyzing chunk frames = [2327552,2328064) | 20260505 10:00:56.791 | INFO | MemInfo: { | 20260505 10:00:56.791 | INFO | "gpu" : " 17 GB", | 20260505 10:00:56.791 | INFO | "huge" : "", | 20260505 10:00:56.791 | INFO | "huge_shared" : " 27 GB", | 20260505 10:00:56.791 | INFO | "malloc" : " 14 GB", | 20260505 10:00:56.791 | INFO | "pinned" : " 29 GB" | 20260505 10:00:56.791 | INFO | } | 20260505 10:00:56.791 | INFO | Perf report for the last 5.13987 seconds: | 20260505 10:00:56.791 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:00:56.791 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:56.791 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:56.791 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:56.791 | INFO | ANALYSIS : 63.0%, 3.0, 98.0ms, 0 | 20260505 10:00:56.791 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:56.791 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:56.791 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 10:00:56.791 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:56.792 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:00:56.792 | INFO | Uploaded 36GiB to GPU in 2.14473s (16.7853GiB/s) | 20260505 10:00:56.792 | INFO | Downloaded 34GiB to GPU in 2.05993s (16.5054GiB/s) | 20260505 10:01:01.931 | INFO | Analyzing chunk frames = [2328064,2328576) | 20260505 10:01:01.931 | INFO | MemInfo: { | 20260505 10:01:01.931 | INFO | "gpu" : " 17 GB", | 20260505 10:01:01.931 | INFO | "huge" : "", | 20260505 10:01:01.931 | INFO | "huge_shared" : " 27 GB", | 20260505 10:01:01.931 | INFO | "malloc" : " 14 GB", | 20260505 10:01:01.931 | INFO | "pinned" : " 29 GB" | 20260505 10:01:01.931 | INFO | } | 20260505 10:01:01.931 | INFO | Perf report for the last 5.14002 seconds: | 20260505 10:01:01.931 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:01:01.931 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:01.931 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:01.931 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:01.931 | INFO | ANALYSIS : 63.1%, 3.0, 98.0ms, 0 | 20260505 10:01:01.931 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:01.931 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:01.931 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 10:01:01.931 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:01.931 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:01.932 | INFO | Uploaded 36GiB to GPU in 2.1658s (16.622GiB/s) | 20260505 10:01:01.932 | INFO | Downloaded 34GiB to GPU in 2.07487s (16.3866GiB/s) | 20260505 10:01:07.070 | INFO | Analyzing chunk frames = [2328576,2329088) | 20260505 10:01:07.070 | INFO | MemInfo: { | 20260505 10:01:07.070 | INFO | "gpu" : " 17 GB", | 20260505 10:01:07.070 | INFO | "huge" : "", | 20260505 10:01:07.070 | INFO | "huge_shared" : " 27 GB", | 20260505 10:01:07.070 | INFO | "malloc" : " 14 GB", | 20260505 10:01:07.070 | INFO | "pinned" : " 29 GB" | 20260505 10:01:07.070 | INFO | } | 20260505 10:01:07.070 | INFO | Perf report for the last 5.13864 seconds: | 20260505 10:01:07.070 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:01:07.070 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:07.070 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:07.070 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:07.070 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 10:01:07.070 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:07.070 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:07.070 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:01:07.070 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:07.070 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:07.070 | INFO | Uploaded 36GiB to GPU in 2.2293s (16.1486GiB/s) | 20260505 10:01:07.070 | INFO | Downloaded 34GiB to GPU in 2.10839s (16.1261GiB/s) | 20260505 10:01:12.209 | INFO | Analyzing chunk frames = [2329088,2329600) | 20260505 10:01:12.209 | INFO | MemInfo: { | 20260505 10:01:12.209 | INFO | "gpu" : " 17 GB", | 20260505 10:01:12.209 | INFO | "huge" : "", | 20260505 10:01:12.209 | INFO | "huge_shared" : " 27 GB", | 20260505 10:01:12.209 | INFO | "malloc" : " 14 GB", | 20260505 10:01:12.209 | INFO | "pinned" : " 29 GB" | 20260505 10:01:12.209 | INFO | } | 20260505 10:01:12.209 | INFO | Perf report for the last 5.13945 seconds: | 20260505 10:01:12.209 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:01:12.209 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:12.209 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:12.209 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:12.209 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 10:01:12.210 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:12.210 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:12.210 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:01:12.210 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:12.210 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:12.210 | INFO | Uploaded 36GiB to GPU in 2.18621s (16.4669GiB/s) | 20260505 10:01:12.210 | INFO | Downloaded 34GiB to GPU in 2.09928s (16.196GiB/s) | 20260505 10:01:17.348 | INFO | Analyzing chunk frames = [2329600,2330112) | 20260505 10:01:17.348 | INFO | MemInfo: { | 20260505 10:01:17.348 | INFO | "gpu" : " 17 GB", | 20260505 10:01:17.348 | INFO | "huge" : "", | 20260505 10:01:17.348 | INFO | "huge_shared" : " 27 GB", | 20260505 10:01:17.348 | INFO | "malloc" : " 14 GB", | 20260505 10:01:17.348 | INFO | "pinned" : " 29 GB" | 20260505 10:01:17.348 | INFO | } | 20260505 10:01:17.348 | INFO | Perf report for the last 5.13856 seconds: | 20260505 10:01:17.348 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:01:17.348 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:17.348 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:17.348 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:17.348 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 10:01:17.348 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:17.348 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:17.348 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 10:01:17.348 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:17.348 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:17.348 | INFO | Uploaded 36GiB to GPU in 2.20872s (16.299GiB/s) | 20260505 10:01:17.348 | INFO | Downloaded 34GiB to GPU in 2.13661s (15.913GiB/s) | 20260505 10:01:22.488 | INFO | Analyzing chunk frames = [2330112,2330624) | 20260505 10:01:22.488 | INFO | MemInfo: { | 20260505 10:01:22.488 | INFO | "gpu" : " 17 GB", | 20260505 10:01:22.488 | INFO | "huge" : "", | 20260505 10:01:22.488 | INFO | "huge_shared" : " 27 GB", | 20260505 10:01:22.489 | INFO | "malloc" : " 14 GB", | 20260505 10:01:22.489 | INFO | "pinned" : " 29 GB" | 20260505 10:01:22.489 | INFO | } | 20260505 10:01:22.489 | INFO | Perf report for the last 5.14068 seconds: | 20260505 10:01:22.489 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:01:22.489 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:22.489 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:22.489 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:22.489 | INFO | ANALYSIS : 64.7%, 3.0, 101.0ms, 0 | 20260505 10:01:22.489 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:22.489 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:22.489 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 10:01:22.489 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:22.489 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:22.489 | INFO | Uploaded 36GiB to GPU in 2.24718s (16.0201GiB/s) | 20260505 10:01:22.489 | INFO | Downloaded 34GiB to GPU in 2.08356s (16.3182GiB/s) | 20260505 10:01:22.621 | INFO | Starting a round of Baz IO | 20260505 10:01:25.897 | INFO | Populating RTMetrics report | 20260505 10:01:25.897 | INFO | RTMetrics report populated | 20260505 10:01:25.897 | INFO | Opening tmp RTMetrics json file | 20260505 10:01:25.915 | INFO | Serializing RTMetrics to json | 20260505 10:01:25.951 | INFO | Serialization done, swapping tmp file now... | 20260505 10:01:25.977 | INFO | RTMetrics done | 20260505 10:01:26.049 | INFO | Finished a round of Baz IO: Wrote 2.428 GiB in 3.428 seconds | 20260505 10:01:26.049 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.41% / 0.01% / 74.57% | 20260505 10:01:26.049 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.68758% / 7.94729e-06% / 4.76837e-05% | 20260505 10:01:26.049 | INFO | Baz IO overall: | 20260505 10:01:26.049 | INFO | BytesWritten : 1357987085228 | 20260505 10:01:26.049 | INFO | NumEvents : 646457477591 | 20260505 10:01:27.628 | INFO | Analyzing chunk frames = [2330624,2331136) | 20260505 10:01:27.628 | INFO | MemInfo: { | 20260505 10:01:27.628 | INFO | "gpu" : " 17 GB", | 20260505 10:01:27.628 | INFO | "huge" : "", | 20260505 10:01:27.628 | INFO | "huge_shared" : " 27 GB", | 20260505 10:01:27.628 | INFO | "malloc" : " 14 GB", | 20260505 10:01:27.628 | INFO | "pinned" : " 29 GB" | 20260505 10:01:27.628 | INFO | } | 20260505 10:01:27.628 | INFO | Perf report for the last 5.13911 seconds: | 20260505 10:01:27.628 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:01:27.628 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:27.628 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:27.628 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:27.628 | INFO | ANALYSIS : 65.2%, 2.9, 99.8ms, 0 | 20260505 10:01:27.628 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:27.628 | INFO | RT_METRICS : 47.2%, 1.0, 24.8ms, 0 | 20260505 10:01:27.628 | INFO | PRE_HQ : 62.6%, 1.8, 59.8ms, 0 | 20260505 10:01:27.628 | INFO | BAZWRITER : 66.0%, 4.1, 140.2ms, 0 | 20260505 10:01:27.628 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260505 10:01:27.628 | INFO | Uploaded 36GiB to GPU in 2.25533s (15.9622GiB/s) | 20260505 10:01:27.628 | INFO | Downloaded 38GiB to GPU in 2.35136s (16.1609GiB/s) | 20260505 10:01:32.767 | INFO | Analyzing chunk frames = [2331136,2331648) | 20260505 10:01:32.767 | INFO | MemInfo: { | 20260505 10:01:32.767 | INFO | "gpu" : " 17 GB", | 20260505 10:01:32.767 | INFO | "huge" : "", | 20260505 10:01:32.767 | INFO | "huge_shared" : " 27 GB", | 20260505 10:01:32.767 | INFO | "malloc" : " 14 GB", | 20260505 10:01:32.767 | INFO | "pinned" : " 29 GB" | 20260505 10:01:32.767 | INFO | } | 20260505 10:01:32.767 | INFO | Perf report for the last 5.13932 seconds: | 20260505 10:01:32.767 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:01:32.767 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:32.767 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:32.767 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:32.767 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 10:01:32.767 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:32.767 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:32.767 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:01:32.767 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:32.767 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:32.767 | INFO | Uploaded 36GiB to GPU in 2.19401s (16.4083GiB/s) | 20260505 10:01:32.767 | INFO | Downloaded 34GiB to GPU in 2.10177s (16.1768GiB/s) | 20260505 10:01:37.905 | INFO | Analyzing chunk frames = [2331648,2332160) | 20260505 10:01:37.905 | INFO | MemInfo: { | 20260505 10:01:37.905 | INFO | "gpu" : " 17 GB", | 20260505 10:01:37.905 | INFO | "huge" : "", | 20260505 10:01:37.905 | INFO | "huge_shared" : " 27 GB", | 20260505 10:01:37.905 | INFO | "malloc" : " 14 GB", | 20260505 10:01:37.905 | INFO | "pinned" : " 29 GB" | 20260505 10:01:37.905 | INFO | } | 20260505 10:01:37.905 | INFO | Perf report for the last 5.13799 seconds: | 20260505 10:01:37.905 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:01:37.905 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:37.905 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:37.905 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:37.905 | INFO | ANALYSIS : 63.0%, 3.0, 98.0ms, 0 | 20260505 10:01:37.905 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:37.905 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:37.905 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:01:37.905 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:37.905 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:37.905 | INFO | Uploaded 36GiB to GPU in 2.14946s (16.7484GiB/s) | 20260505 10:01:37.905 | INFO | Downloaded 34GiB to GPU in 2.0567s (16.5313GiB/s) | 20260505 10:01:43.045 | INFO | Analyzing chunk frames = [2332160,2332672) | 20260505 10:01:43.045 | INFO | MemInfo: { | 20260505 10:01:43.045 | INFO | "gpu" : " 17 GB", | 20260505 10:01:43.045 | INFO | "huge" : "", | 20260505 10:01:43.045 | INFO | "huge_shared" : " 27 GB", | 20260505 10:01:43.045 | INFO | "malloc" : " 14 GB", | 20260505 10:01:43.045 | INFO | "pinned" : " 29 GB" | 20260505 10:01:43.045 | INFO | } | 20260505 10:01:43.045 | INFO | Perf report for the last 5.13959 seconds: | 20260505 10:01:43.045 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:01:43.045 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:43.045 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:43.045 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:43.045 | INFO | ANALYSIS : 63.1%, 3.0, 98.1ms, 0 | 20260505 10:01:43.045 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:43.045 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:43.045 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:01:43.045 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:43.045 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:43.045 | INFO | Uploaded 36GiB to GPU in 2.16623s (16.6187GiB/s) | 20260505 10:01:43.045 | INFO | Downloaded 34GiB to GPU in 2.07468s (16.3881GiB/s) | 20260505 10:01:48.184 | INFO | Analyzing chunk frames = [2332672,2333184) | 20260505 10:01:48.184 | INFO | MemInfo: { | 20260505 10:01:48.184 | INFO | "gpu" : " 17 GB", | 20260505 10:01:48.184 | INFO | "huge" : "", | 20260505 10:01:48.184 | INFO | "huge_shared" : " 27 GB", | 20260505 10:01:48.184 | INFO | "malloc" : " 14 GB", | 20260505 10:01:48.184 | INFO | "pinned" : " 29 GB" | 20260505 10:01:48.184 | INFO | } | 20260505 10:01:48.185 | INFO | Perf report for the last 5.13998 seconds: | 20260505 10:01:48.185 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:01:48.185 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:48.185 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:48.185 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:48.185 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 10:01:48.185 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:48.185 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:48.185 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 10:01:48.185 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:48.185 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:48.185 | INFO | Uploaded 36GiB to GPU in 2.23702s (16.0929GiB/s) | 20260505 10:01:48.185 | INFO | Downloaded 34GiB to GPU in 2.1082s (16.1275GiB/s) | 20260505 10:01:53.324 | INFO | Analyzing chunk frames = [2333184,2333696) | 20260505 10:01:53.324 | INFO | MemInfo: { | 20260505 10:01:53.324 | INFO | "gpu" : " 17 GB", | 20260505 10:01:53.324 | INFO | "huge" : "", | 20260505 10:01:53.324 | INFO | "huge_shared" : " 27 GB", | 20260505 10:01:53.324 | INFO | "malloc" : " 14 GB", | 20260505 10:01:53.324 | INFO | "pinned" : " 29 GB" | 20260505 10:01:53.324 | INFO | } | 20260505 10:01:53.324 | INFO | Perf report for the last 5.1393 seconds: | 20260505 10:01:53.324 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:01:53.324 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:53.324 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:53.324 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:53.324 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 10:01:53.324 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:53.324 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:53.324 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:01:53.324 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:53.324 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:53.324 | INFO | Uploaded 36GiB to GPU in 2.23549s (16.1039GiB/s) | 20260505 10:01:53.324 | INFO | Downloaded 34GiB to GPU in 2.09884s (16.1994GiB/s) | 20260505 10:01:58.465 | INFO | Analyzing chunk frames = [2333696,2334208) | 20260505 10:01:58.465 | INFO | MemInfo: { | 20260505 10:01:58.465 | INFO | "gpu" : " 17 GB", | 20260505 10:01:58.465 | INFO | "huge" : "", | 20260505 10:01:58.465 | INFO | "huge_shared" : " 27 GB", | 20260505 10:01:58.465 | INFO | "malloc" : " 14 GB", | 20260505 10:01:58.465 | INFO | "pinned" : " 29 GB" | 20260505 10:01:58.465 | INFO | } | 20260505 10:01:58.465 | INFO | Perf report for the last 5.14084 seconds: | 20260505 10:01:58.465 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:01:58.465 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:58.465 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:58.465 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:58.465 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 10:01:58.465 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:58.465 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:58.465 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 10:01:58.465 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:58.465 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:01:58.465 | INFO | Uploaded 36GiB to GPU in 2.21015s (16.2885GiB/s) | 20260505 10:01:58.465 | INFO | Downloaded 34GiB to GPU in 2.11588s (16.069GiB/s) | 20260505 10:02:03.603 | INFO | Analyzing chunk frames = [2334208,2334720) | 20260505 10:02:03.603 | INFO | MemInfo: { | 20260505 10:02:03.603 | INFO | "gpu" : " 17 GB", | 20260505 10:02:03.603 | INFO | "huge" : "", | 20260505 10:02:03.603 | INFO | "huge_shared" : " 27 GB", | 20260505 10:02:03.603 | INFO | "malloc" : " 14 GB", | 20260505 10:02:03.603 | INFO | "pinned" : " 29 GB" | 20260505 10:02:03.603 | INFO | } | 20260505 10:02:03.603 | INFO | Perf report for the last 5.13831 seconds: | 20260505 10:02:03.603 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:02:03.603 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:03.603 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:03.603 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:03.603 | INFO | ANALYSIS : 64.5%, 3.0, 100.8ms, 0 | 20260505 10:02:03.603 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:03.603 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:03.603 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 10:02:03.603 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:03.603 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:03.604 | INFO | Uploaded 36GiB to GPU in 2.24946s (16.0039GiB/s) | 20260505 10:02:03.604 | INFO | Downloaded 34GiB to GPU in 2.09001s (16.2678GiB/s) | 20260505 10:02:03.604 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 10:02:03.604 | INFO | Total mem usage: 15.3 GiB | 20260505 10:02:03.604 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 10:02:03.604 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.5%) | 20260505 10:02:03.604 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 10:02:03.604 | INFO | High Water Breakdown: | 20260505 10:02:03.604 | INFO | 61% : 8.78 GiB ( 93.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 10:02:03.604 | INFO | 38.8% : 5.59 GiB ( 84.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 10:02:03.604 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 10:02:03.604 | INFO | ------------------------------------------ | 20260505 10:02:03.604 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:02:03.604 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 10:02:03.604 | INFO | Total mem usage: 26.5 GiB | 20260505 10:02:03.604 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 10:02:03.604 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 10:02:03.604 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 10:02:03.604 | INFO | High Water Breakdown: | 20260505 10:02:03.604 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:02:03.604 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:02:03.604 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:02:03.604 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:02:03.604 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:02:03.604 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:02:03.604 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 10:02:03.604 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 10:02:03.604 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:02:03.604 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 10:02:03.604 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 10:02:03.604 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 10:02:03.604 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 10:02:03.604 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 10:02:03.604 | INFO | ------------------------------------------ | 20260505 10:02:03.604 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:02:03.604 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 10:02:03.604 | INFO | Total mem usage: 25.6 GiB | 20260505 10:02:03.604 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 10:02:03.604 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 10:02:03.604 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 10:02:03.604 | INFO | High Water Breakdown: | 20260505 10:02:03.604 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 10:02:03.604 | INFO | ------------------------------------------ | 20260505 10:02:03.604 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:02:03.604 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 10:02:03.604 | INFO | Total mem usage: 15.7 GiB | 20260505 10:02:03.604 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 10:02:03.604 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 10:02:03.604 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 10:02:03.604 | INFO | High Water Breakdown: | 20260505 10:02:03.604 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 10:02:03.604 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 10:02:03.604 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 10:02:03.604 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 10:02:03.604 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 10:02:03.604 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 10:02:03.605 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 10:02:03.605 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 10:02:03.605 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 10:02:03.605 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:02:03.605 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 10:02:03.605 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:02:03.605 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 10:02:03.605 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 10:02:03.605 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 10:02:03.605 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 10:02:03.605 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:02:03.605 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 10:02:03.605 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:02:03.605 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:02:03.605 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:02:03.605 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 10:02:03.605 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 10:02:03.605 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:02:03.605 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 10:02:03.605 | INFO | ------------------------------------------ | 20260505 10:02:03.605 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:02:03.605 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 10:02:03.605 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:02:03.605 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:02:03.605 | INFO | PerfReport: Full Execution -- 3.24e+01 / 7.76e+01, 1.40e+01 / 2.31e+00 | 20260505 10:02:03.605 | INFO | PerfReport: 0.29%, AggregateStats -- 9.31e-02 / 1.02e-01, 1.05e-02 / 4.28e-03 | 20260505 10:02:03.605 | INFO | PerfReport: 11.66%, Baseline -- 3.78e+00 / 4.43e+00, 6.15e-01 / 2.81e-01 | 20260505 10:02:03.605 | INFO | PerfReport: 3.31%, Histogram -- 1.07e+00 / 1.47e+00, 2.10e-01 / 1.12e-01 | 20260505 10:02:03.605 | INFO | PerfReport: 15.67%, DME -- 4.07e+01 / 4.51e+01, 4.61e+00 / 2.44e+00 | 20260505 10:02:03.605 | INFO | PerfReport: 56.60%, FrameLabeling -- 1.84e+01 / 2.16e+01, 1.87e+00 / 2.30e+00 | 20260505 10:02:03.605 | INFO | PerfReport: 7.35%, PulseAccumulating -- 2.38e+00 / 2.91e+00, 4.03e-01 / 2.21e-01 | 20260505 10:02:03.605 | INFO | PerfReport: 3.78%, Metrics -- 1.23e+00 / 1.51e+00, 3.53e-01 / 1.99e-01 | 20260505 10:02:03.605 | INFO | PerfReport: 1.34%, PulseSerializing -- 4.34e-01 / 4.81e-01, 8.10e-02 / 3.11e-02 | 20260505 10:02:03.605 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 10:02:03.605 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:02:03.605 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:02:03.605 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260505 10:02:03.605 | INFO | PerfReport: 22.62%, Upload -- 2.30e+01 / 0.00e+00, 8.32e+00 / 0.00e+00 | 20260505 10:02:03.605 | INFO | PerfReport: 21.80%, Download -- 2.21e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260505 10:02:03.605 | INFO | PerfReport: 31.94%, Compute -- 3.24e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260505 10:02:03.605 | INFO | PerfReport: 1.88%, LockUpload -- 1.91e+00 / 0.00e+00, 7.43e+00 / 0.00e+00 | 20260505 10:02:03.605 | INFO | PerfReport: 1.15%, LockDownload -- 1.17e+00 / 0.00e+00, 2.76e+00 / 0.00e+00 | 20260505 10:02:03.605 | INFO | PerfReport: 20.59%, LockCompute -- 2.09e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260505 10:02:03.605 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.93e-02 / 0.00e+00 | 20260505 10:02:03.736 | INFO | Starting a round of Baz IO | 20260505 10:02:07.012 | INFO | Populating RTMetrics report | 20260505 10:02:07.012 | INFO | RTMetrics report populated | 20260505 10:02:07.012 | INFO | Opening tmp RTMetrics json file | 20260505 10:02:07.029 | INFO | Serializing RTMetrics to json | 20260505 10:02:07.039 | INFO | Serialization done, swapping tmp file now... | 20260505 10:02:07.057 | INFO | RTMetrics done | 20260505 10:02:07.141 | INFO | Finished a round of Baz IO: Wrote 2.426 GiB in 3.405 seconds | 20260505 10:02:07.141 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.40% / 0.01% / 74.59% | 20260505 10:02:07.141 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.67865% / 4.37101e-05% / 8.74201e-05% | 20260505 10:02:07.141 | INFO | Baz IO overall: | 20260505 10:02:07.141 | INFO | BytesWritten : 1360591556979 | 20260505 10:02:07.142 | INFO | NumEvents : 647700407307 | 20260505 10:02:08.742 | INFO | Analyzing chunk frames = [2334720,2335232) | 20260505 10:02:08.742 | INFO | MemInfo: { | 20260505 10:02:08.742 | INFO | "gpu" : " 17 GB", | 20260505 10:02:08.742 | INFO | "huge" : "", | 20260505 10:02:08.742 | INFO | "huge_shared" : " 27 GB", | 20260505 10:02:08.742 | INFO | "malloc" : " 14 GB", | 20260505 10:02:08.742 | INFO | "pinned" : " 29 GB" | 20260505 10:02:08.742 | INFO | } | 20260505 10:02:08.742 | INFO | Perf report for the last 5.13892 seconds: | 20260505 10:02:08.742 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:02:08.742 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:08.742 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:08.742 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:08.742 | INFO | ANALYSIS : 65.2%, 2.9, 99.8ms, 0 | 20260505 10:02:08.742 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:08.742 | INFO | RT_METRICS : 46.6%, 1.0, 24.4ms, 0 | 20260505 10:02:08.742 | INFO | PRE_HQ : 62.6%, 1.8, 60.0ms, 0 | 20260505 10:02:08.742 | INFO | BAZWRITER : 65.6%, 4.2, 140.4ms, 0 | 20260505 10:02:08.742 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 10:02:08.743 | INFO | Uploaded 36GiB to GPU in 2.25412s (15.9708GiB/s) | 20260505 10:02:08.743 | INFO | Downloaded 38GiB to GPU in 2.3536s (16.1455GiB/s) | 20260505 10:02:13.881 | INFO | Analyzing chunk frames = [2335232,2335744) | 20260505 10:02:13.881 | INFO | MemInfo: { | 20260505 10:02:13.881 | INFO | "gpu" : " 17 GB", | 20260505 10:02:13.881 | INFO | "huge" : "", | 20260505 10:02:13.881 | INFO | "huge_shared" : " 27 GB", | 20260505 10:02:13.881 | INFO | "malloc" : " 14 GB", | 20260505 10:02:13.881 | INFO | "pinned" : " 29 GB" | 20260505 10:02:13.881 | INFO | } | 20260505 10:02:13.881 | INFO | Perf report for the last 5.13916 seconds: | 20260505 10:02:13.881 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:02:13.881 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:13.881 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:13.881 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:13.881 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 10:02:13.881 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:13.881 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:13.881 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:02:13.881 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:13.881 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:13.881 | INFO | Uploaded 36GiB to GPU in 2.1753s (16.5495GiB/s) | 20260505 10:02:13.882 | INFO | Downloaded 34GiB to GPU in 2.09971s (16.1927GiB/s) | 20260505 10:02:19.020 | INFO | Analyzing chunk frames = [2335744,2336256) | 20260505 10:02:19.020 | INFO | MemInfo: { | 20260505 10:02:19.020 | INFO | "gpu" : " 17 GB", | 20260505 10:02:19.020 | INFO | "huge" : "", | 20260505 10:02:19.020 | INFO | "huge_shared" : " 27 GB", | 20260505 10:02:19.020 | INFO | "malloc" : " 14 GB", | 20260505 10:02:19.020 | INFO | "pinned" : " 29 GB" | 20260505 10:02:19.020 | INFO | } | 20260505 10:02:19.020 | INFO | Perf report for the last 5.13853 seconds: | 20260505 10:02:19.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:02:19.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:19.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:19.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:19.020 | INFO | ANALYSIS : 63.1%, 3.0, 98.1ms, 0 | 20260505 10:02:19.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:19.020 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:19.020 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 10:02:19.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:19.020 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:19.020 | INFO | Uploaded 36GiB to GPU in 2.1409s (16.8154GiB/s) | 20260505 10:02:19.020 | INFO | Downloaded 34GiB to GPU in 2.05958s (16.5082GiB/s) | 20260505 10:02:24.160 | INFO | Analyzing chunk frames = [2336256,2336768) | 20260505 10:02:24.160 | INFO | MemInfo: { | 20260505 10:02:24.160 | INFO | "gpu" : " 17 GB", | 20260505 10:02:24.160 | INFO | "huge" : "", | 20260505 10:02:24.160 | INFO | "huge_shared" : " 27 GB", | 20260505 10:02:24.160 | INFO | "malloc" : " 14 GB", | 20260505 10:02:24.160 | INFO | "pinned" : " 29 GB" | 20260505 10:02:24.160 | INFO | } | 20260505 10:02:24.160 | INFO | Perf report for the last 5.13996 seconds: | 20260505 10:02:24.160 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:02:24.160 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:24.160 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:24.160 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:24.160 | INFO | ANALYSIS : 63.0%, 3.0, 98.0ms, 0 | 20260505 10:02:24.160 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:24.160 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:24.160 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:02:24.160 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:24.160 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:24.160 | INFO | Uploaded 36GiB to GPU in 2.16974s (16.5918GiB/s) | 20260505 10:02:24.160 | INFO | Downloaded 34GiB to GPU in 2.07513s (16.3845GiB/s) | 20260505 10:02:29.300 | INFO | Analyzing chunk frames = [2336768,2337280) | 20260505 10:02:29.300 | INFO | MemInfo: { | 20260505 10:02:29.300 | INFO | "gpu" : " 17 GB", | 20260505 10:02:29.300 | INFO | "huge" : "", | 20260505 10:02:29.300 | INFO | "huge_shared" : " 27 GB", | 20260505 10:02:29.300 | INFO | "malloc" : " 14 GB", | 20260505 10:02:29.300 | INFO | "pinned" : " 29 GB" | 20260505 10:02:29.300 | INFO | } | 20260505 10:02:29.300 | INFO | Perf report for the last 5.13994 seconds: | 20260505 10:02:29.300 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:02:29.300 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:29.300 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:29.300 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:29.300 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 10:02:29.300 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:29.300 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:29.300 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:02:29.300 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:29.300 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:29.300 | INFO | Uploaded 36GiB to GPU in 2.2348s (16.1088GiB/s) | 20260505 10:02:29.300 | INFO | Downloaded 34GiB to GPU in 2.10169s (16.1774GiB/s) | 20260505 10:02:34.440 | INFO | Analyzing chunk frames = [2337280,2337792) | 20260505 10:02:34.440 | INFO | MemInfo: { | 20260505 10:02:34.440 | INFO | "gpu" : " 17 GB", | 20260505 10:02:34.440 | INFO | "huge" : "", | 20260505 10:02:34.440 | INFO | "huge_shared" : " 27 GB", | 20260505 10:02:34.440 | INFO | "malloc" : " 14 GB", | 20260505 10:02:34.440 | INFO | "pinned" : " 29 GB" | 20260505 10:02:34.440 | INFO | } | 20260505 10:02:34.440 | INFO | Perf report for the last 5.14045 seconds: | 20260505 10:02:34.440 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:02:34.440 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:34.440 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:34.440 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:34.440 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 10:02:34.440 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:34.440 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:34.440 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 10:02:34.440 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:34.441 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:34.441 | INFO | Uploaded 36GiB to GPU in 2.23251s (16.1254GiB/s) | 20260505 10:02:34.441 | INFO | Downloaded 34GiB to GPU in 2.09739s (16.2106GiB/s) | 20260505 10:02:39.578 | INFO | Analyzing chunk frames = [2337792,2338304) | 20260505 10:02:39.578 | INFO | MemInfo: { | 20260505 10:02:39.578 | INFO | "gpu" : " 17 GB", | 20260505 10:02:39.578 | INFO | "huge" : "", | 20260505 10:02:39.578 | INFO | "huge_shared" : " 27 GB", | 20260505 10:02:39.578 | INFO | "malloc" : " 14 GB", | 20260505 10:02:39.578 | INFO | "pinned" : " 29 GB" | 20260505 10:02:39.578 | INFO | } | 20260505 10:02:39.578 | INFO | Perf report for the last 5.13789 seconds: | 20260505 10:02:39.578 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:02:39.578 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:39.578 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:39.578 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:39.578 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 10:02:39.578 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:39.578 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:39.578 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 10:02:39.578 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:39.578 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:39.579 | INFO | Uploaded 36GiB to GPU in 2.17977s (16.5155GiB/s) | 20260505 10:02:39.579 | INFO | Downloaded 34GiB to GPU in 2.14261s (15.8685GiB/s) | 20260505 10:02:44.717 | INFO | Analyzing chunk frames = [2338304,2338816) | 20260505 10:02:44.717 | INFO | MemInfo: { | 20260505 10:02:44.717 | INFO | "gpu" : " 17 GB", | 20260505 10:02:44.717 | INFO | "huge" : "", | 20260505 10:02:44.717 | INFO | "huge_shared" : " 27 GB", | 20260505 10:02:44.717 | INFO | "malloc" : " 14 GB", | 20260505 10:02:44.717 | INFO | "pinned" : " 29 GB" | 20260505 10:02:44.717 | INFO | } | 20260505 10:02:44.718 | INFO | Perf report for the last 5.13937 seconds: | 20260505 10:02:44.718 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:02:44.718 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:44.718 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:44.718 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:44.718 | INFO | ANALYSIS : 64.6%, 3.0, 100.9ms, 0 | 20260505 10:02:44.718 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:44.718 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:44.718 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 10:02:44.718 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:44.718 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:44.718 | INFO | Uploaded 36GiB to GPU in 2.24774s (16.0161GiB/s) | 20260505 10:02:44.718 | INFO | Downloaded 34GiB to GPU in 2.09346s (16.2411GiB/s) | 20260505 10:02:44.849 | INFO | Starting a round of Baz IO | 20260505 10:02:48.133 | INFO | Populating RTMetrics report | 20260505 10:02:48.133 | INFO | RTMetrics report populated | 20260505 10:02:48.133 | INFO | Opening tmp RTMetrics json file | 20260505 10:02:48.135 | INFO | Serializing RTMetrics to json | 20260505 10:02:48.160 | INFO | Serialization done, swapping tmp file now... | 20260505 10:02:48.164 | INFO | RTMetrics done | 20260505 10:02:48.271 | INFO | Finished a round of Baz IO: Wrote 2.428 GiB in 3.422 seconds | 20260505 10:02:48.271 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.39% / 0.01% / 74.60% | 20260505 10:02:48.271 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.67886% / 1.58946e-05% / 4.37101e-05% | 20260505 10:02:48.271 | INFO | Baz IO overall: | 20260505 10:02:48.271 | INFO | BytesWritten : 1363198618731 | 20260505 10:02:48.271 | INFO | NumEvents : 648945125161 | 20260505 10:02:49.856 | INFO | Analyzing chunk frames = [2338816,2339328) | 20260505 10:02:49.856 | INFO | MemInfo: { | 20260505 10:02:49.856 | INFO | "gpu" : " 17 GB", | 20260505 10:02:49.856 | INFO | "huge" : "", | 20260505 10:02:49.856 | INFO | "huge_shared" : " 27 GB", | 20260505 10:02:49.856 | INFO | "malloc" : " 14 GB", | 20260505 10:02:49.856 | INFO | "pinned" : " 29 GB" | 20260505 10:02:49.856 | INFO | } | 20260505 10:02:49.856 | INFO | Perf report for the last 5.13848 seconds: | 20260505 10:02:49.856 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:02:49.856 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:49.856 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:49.856 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:49.856 | INFO | ANALYSIS : 65.3%, 2.9, 100.1ms, 0 | 20260505 10:02:49.856 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:49.856 | INFO | RT_METRICS : 46.2%, 1.0, 24.2ms, 0 | 20260505 10:02:49.856 | INFO | PRE_HQ : 62.0%, 1.9, 59.9ms, 0 | 20260505 10:02:49.856 | INFO | BAZWRITER : 65.7%, 4.2, 140.7ms, 0 | 20260505 10:02:49.856 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 10:02:49.856 | INFO | Uploaded 36GiB to GPU in 2.2265s (16.1689GiB/s) | 20260505 10:02:49.857 | INFO | Downloaded 38GiB to GPU in 2.30553s (16.4821GiB/s) | 20260505 10:02:54.996 | INFO | Analyzing chunk frames = [2339328,2339840) | 20260505 10:02:54.996 | INFO | MemInfo: { | 20260505 10:02:54.996 | INFO | "gpu" : " 17 GB", | 20260505 10:02:54.996 | INFO | "huge" : "", | 20260505 10:02:54.996 | INFO | "huge_shared" : " 27 GB", | 20260505 10:02:54.996 | INFO | "malloc" : " 14 GB", | 20260505 10:02:54.996 | INFO | "pinned" : " 29 GB" | 20260505 10:02:54.996 | INFO | } | 20260505 10:02:54.996 | INFO | Perf report for the last 5.14017 seconds: | 20260505 10:02:54.996 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:02:54.996 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:54.996 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:54.996 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:54.996 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 10:02:54.996 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:54.996 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:54.996 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 10:02:54.996 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:54.996 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:02:54.997 | INFO | Uploaded 36GiB to GPU in 2.17019s (16.5884GiB/s) | 20260505 10:02:54.997 | INFO | Downloaded 34GiB to GPU in 2.09947s (16.1945GiB/s) | 20260505 10:03:00.136 | INFO | Analyzing chunk frames = [2339840,2340352) | 20260505 10:03:00.136 | INFO | MemInfo: { | 20260505 10:03:00.136 | INFO | "gpu" : " 17 GB", | 20260505 10:03:00.136 | INFO | "huge" : "", | 20260505 10:03:00.136 | INFO | "huge_shared" : " 27 GB", | 20260505 10:03:00.136 | INFO | "malloc" : " 14 GB", | 20260505 10:03:00.136 | INFO | "pinned" : " 29 GB" | 20260505 10:03:00.136 | INFO | } | 20260505 10:03:00.137 | INFO | Perf report for the last 5.14048 seconds: | 20260505 10:03:00.137 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:03:00.137 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:00.137 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:00.137 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:00.137 | INFO | ANALYSIS : 63.0%, 3.0, 98.0ms, 0 | 20260505 10:03:00.137 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:00.137 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:00.137 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:03:00.137 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:00.137 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:00.137 | INFO | Uploaded 36GiB to GPU in 2.13573s (16.856GiB/s) | 20260505 10:03:00.137 | INFO | Downloaded 34GiB to GPU in 2.05866s (16.5156GiB/s) | 20260505 10:03:05.275 | INFO | Analyzing chunk frames = [2340352,2340864) | 20260505 10:03:05.275 | INFO | MemInfo: { | 20260505 10:03:05.275 | INFO | "gpu" : " 17 GB", | 20260505 10:03:05.275 | INFO | "huge" : "", | 20260505 10:03:05.275 | INFO | "huge_shared" : " 27 GB", | 20260505 10:03:05.275 | INFO | "malloc" : " 14 GB", | 20260505 10:03:05.275 | INFO | "pinned" : " 29 GB" | 20260505 10:03:05.275 | INFO | } | 20260505 10:03:05.275 | INFO | Perf report for the last 5.13833 seconds: | 20260505 10:03:05.275 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:03:05.275 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:05.275 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:05.275 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:05.275 | INFO | ANALYSIS : 63.1%, 3.0, 98.0ms, 0 | 20260505 10:03:05.275 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:05.275 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:05.275 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 10:03:05.275 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:05.275 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:05.275 | INFO | Uploaded 36GiB to GPU in 2.15625s (16.6957GiB/s) | 20260505 10:03:05.275 | INFO | Downloaded 34GiB to GPU in 2.07618s (16.3762GiB/s) | 20260505 10:03:10.414 | INFO | Analyzing chunk frames = [2340864,2341376) | 20260505 10:03:10.414 | INFO | MemInfo: { | 20260505 10:03:10.414 | INFO | "gpu" : " 17 GB", | 20260505 10:03:10.414 | INFO | "huge" : "", | 20260505 10:03:10.414 | INFO | "huge_shared" : " 27 GB", | 20260505 10:03:10.414 | INFO | "malloc" : " 14 GB", | 20260505 10:03:10.414 | INFO | "pinned" : " 29 GB" | 20260505 10:03:10.414 | INFO | } | 20260505 10:03:10.414 | INFO | Perf report for the last 5.13878 seconds: | 20260505 10:03:10.414 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:03:10.414 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:10.414 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:10.414 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:10.414 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 10:03:10.414 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:10.414 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:10.414 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 10:03:10.414 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:10.414 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:10.414 | INFO | Uploaded 36GiB to GPU in 2.23599s (16.1003GiB/s) | 20260505 10:03:10.414 | INFO | Downloaded 34GiB to GPU in 2.1104s (16.1107GiB/s) | 20260505 10:03:15.554 | INFO | Analyzing chunk frames = [2341376,2341888) | 20260505 10:03:15.554 | INFO | MemInfo: { | 20260505 10:03:15.554 | INFO | "gpu" : " 17 GB", | 20260505 10:03:15.554 | INFO | "huge" : "", | 20260505 10:03:15.554 | INFO | "huge_shared" : " 27 GB", | 20260505 10:03:15.554 | INFO | "malloc" : " 14 GB", | 20260505 10:03:15.554 | INFO | "pinned" : " 29 GB" | 20260505 10:03:15.554 | INFO | } | 20260505 10:03:15.554 | INFO | Perf report for the last 5.14011 seconds: | 20260505 10:03:15.554 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:03:15.554 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:15.554 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:15.554 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:15.554 | INFO | ANALYSIS : 63.9%, 3.0, 99.9ms, 0 | 20260505 10:03:15.554 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:15.554 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:15.554 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:03:15.554 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:15.554 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:15.554 | INFO | Uploaded 36GiB to GPU in 2.2338s (16.116GiB/s) | 20260505 10:03:15.554 | INFO | Downloaded 34GiB to GPU in 2.10997s (16.114GiB/s) | 20260505 10:03:20.693 | INFO | Analyzing chunk frames = [2341888,2342400) | 20260505 10:03:20.693 | INFO | MemInfo: { | 20260505 10:03:20.693 | INFO | "gpu" : " 17 GB", | 20260505 10:03:20.693 | INFO | "huge" : "", | 20260505 10:03:20.693 | INFO | "huge_shared" : " 27 GB", | 20260505 10:03:20.693 | INFO | "malloc" : " 14 GB", | 20260505 10:03:20.693 | INFO | "pinned" : " 29 GB" | 20260505 10:03:20.693 | INFO | } | 20260505 10:03:20.693 | INFO | Perf report for the last 5.13907 seconds: | 20260505 10:03:20.693 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:03:20.693 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:20.693 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:20.693 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:20.693 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 10:03:20.693 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:20.693 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:20.693 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260505 10:03:20.693 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:20.693 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:20.693 | INFO | Uploaded 36GiB to GPU in 2.16483s (16.6294GiB/s) | 20260505 10:03:20.693 | INFO | Downloaded 34GiB to GPU in 2.13003s (15.9622GiB/s) | 20260505 10:03:25.831 | INFO | Analyzing chunk frames = [2342400,2342912) | 20260505 10:03:25.831 | INFO | MemInfo: { | 20260505 10:03:25.831 | INFO | "gpu" : " 17 GB", | 20260505 10:03:25.831 | INFO | "huge" : "", | 20260505 10:03:25.831 | INFO | "huge_shared" : " 27 GB", | 20260505 10:03:25.831 | INFO | "malloc" : " 14 GB", | 20260505 10:03:25.831 | INFO | "pinned" : " 29 GB" | 20260505 10:03:25.831 | INFO | } | 20260505 10:03:25.831 | INFO | Perf report for the last 5.13841 seconds: | 20260505 10:03:25.832 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:03:25.832 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:25.832 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:25.832 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:25.832 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 10:03:25.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:25.832 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:25.832 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 10:03:25.832 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:25.832 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:25.832 | INFO | Uploaded 36GiB to GPU in 2.18799s (16.4534GiB/s) | 20260505 10:03:25.832 | INFO | Downloaded 34GiB to GPU in 2.0607s (16.4993GiB/s) | 20260505 10:03:25.832 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 10:03:25.832 | INFO | Total mem usage: 15.3 GiB | 20260505 10:03:25.832 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 10:03:25.832 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.5%) | 20260505 10:03:25.832 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 10:03:25.832 | INFO | High Water Breakdown: | 20260505 10:03:25.832 | INFO | 61% : 8.78 GiB ( 93.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 10:03:25.832 | INFO | 38.8% : 5.59 GiB ( 84.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 10:03:25.832 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 10:03:25.832 | INFO | ------------------------------------------ | 20260505 10:03:25.832 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:03:25.832 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 10:03:25.832 | INFO | Total mem usage: 26.5 GiB | 20260505 10:03:25.832 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 10:03:25.832 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 10:03:25.832 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 10:03:25.832 | INFO | High Water Breakdown: | 20260505 10:03:25.832 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:03:25.832 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:03:25.832 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:03:25.832 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:03:25.832 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:03:25.832 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:03:25.833 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 10:03:25.833 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 10:03:25.833 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:03:25.833 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 10:03:25.833 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 10:03:25.833 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 10:03:25.833 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 10:03:25.833 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 10:03:25.833 | INFO | ------------------------------------------ | 20260505 10:03:25.833 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:03:25.833 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 10:03:25.833 | INFO | Total mem usage: 25.6 GiB | 20260505 10:03:25.833 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 10:03:25.833 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 10:03:25.833 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 10:03:25.833 | INFO | High Water Breakdown: | 20260505 10:03:25.833 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 10:03:25.833 | INFO | ------------------------------------------ | 20260505 10:03:25.833 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:03:25.833 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 10:03:25.833 | INFO | Total mem usage: 15.7 GiB | 20260505 10:03:25.833 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 10:03:25.833 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 10:03:25.833 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 10:03:25.833 | INFO | High Water Breakdown: | 20260505 10:03:25.833 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 10:03:25.833 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 10:03:25.833 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 10:03:25.833 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 10:03:25.833 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 10:03:25.833 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 10:03:25.833 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 10:03:25.833 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 10:03:25.833 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 10:03:25.833 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:03:25.833 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 10:03:25.833 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:03:25.833 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 10:03:25.833 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 10:03:25.833 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 10:03:25.833 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 10:03:25.833 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:03:25.833 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 10:03:25.833 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:03:25.833 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:03:25.833 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:03:25.833 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 10:03:25.833 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 10:03:25.833 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:03:25.833 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 10:03:25.833 | INFO | ------------------------------------------ | 20260505 10:03:25.833 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:03:25.833 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 10:03:25.833 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:03:25.833 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:03:25.833 | INFO | PerfReport: Full Execution -- 3.24e+01 / 7.75e+01, 1.40e+01 / 2.40e+00 | 20260505 10:03:25.833 | INFO | PerfReport: 0.30%, AggregateStats -- 9.58e-02 / 1.73e-01, 1.12e-01 / 4.76e-01 | 20260505 10:03:25.833 | INFO | PerfReport: 11.64%, Baseline -- 3.78e+00 / 4.45e+00, 6.14e-01 / 3.00e-01 | 20260505 10:03:25.833 | INFO | PerfReport: 3.32%, Histogram -- 1.08e+00 / 1.50e+00, 2.10e-01 / 8.17e-02 | 20260505 10:03:25.833 | INFO | PerfReport: 15.66%, DME -- 4.07e+01 / 4.52e+01, 4.69e+00 / 2.31e+00 | 20260505 10:03:25.833 | INFO | PerfReport: 56.63%, FrameLabeling -- 1.84e+01 / 2.12e+01, 1.87e+00 / 1.56e+00 | 20260505 10:03:25.833 | INFO | PerfReport: 7.33%, PulseAccumulating -- 2.38e+00 / 2.95e+00, 4.04e-01 / 2.14e-01 | 20260505 10:03:25.833 | INFO | PerfReport: 3.78%, Metrics -- 1.23e+00 / 1.52e+00, 3.58e-01 / 1.94e-01 | 20260505 10:03:25.833 | INFO | PerfReport: 1.34%, PulseSerializing -- 4.34e-01 / 4.82e-01, 8.14e-02 / 3.70e-02 | 20260505 10:03:25.833 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 10:03:25.833 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:03:25.833 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:03:25.833 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260505 10:03:25.833 | INFO | PerfReport: 22.54%, Upload -- 2.29e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260505 10:03:25.833 | INFO | PerfReport: 21.79%, Download -- 2.21e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260505 10:03:25.833 | INFO | PerfReport: 31.97%, Compute -- 3.25e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260505 10:03:25.833 | INFO | PerfReport: 1.86%, LockUpload -- 1.88e+00 / 0.00e+00, 7.44e+00 / 0.00e+00 | 20260505 10:03:25.833 | INFO | PerfReport: 1.11%, LockDownload -- 1.13e+00 / 0.00e+00, 2.69e+00 / 0.00e+00 | 20260505 10:03:25.833 | INFO | PerfReport: 20.71%, LockCompute -- 2.10e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260505 10:03:25.833 | INFO | PerfReport: 0.01%, Misc -- 1.50e-02 / 0.00e+00, 2.92e-02 / 0.00e+00 | 20260505 10:03:25.964 | INFO | Starting a round of Baz IO | 20260505 10:03:29.243 | INFO | Populating RTMetrics report | 20260505 10:03:29.243 | INFO | RTMetrics report populated | 20260505 10:03:29.243 | INFO | Opening tmp RTMetrics json file | 20260505 10:03:29.248 | INFO | Serializing RTMetrics to json | 20260505 10:03:29.249 | INFO | Serialization done, swapping tmp file now... | 20260505 10:03:29.250 | INFO | RTMetrics done | 20260505 10:03:29.388 | INFO | Finished a round of Baz IO: Wrote 2.426 GiB in 3.424 seconds | 20260505 10:03:29.388 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.38% / 0.01% / 74.61% | 20260505 10:03:29.388 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.66964% / 7.94729e-06% / 9.53674e-05% | 20260505 10:03:29.388 | INFO | Baz IO overall: | 20260505 10:03:29.388 | INFO | BytesWritten : 1365803196795 | 20260505 10:03:29.389 | INFO | NumEvents : 650188276230 | 20260505 10:03:30.971 | INFO | Analyzing chunk frames = [2342912,2343424) | 20260505 10:03:30.971 | INFO | MemInfo: { | 20260505 10:03:30.971 | INFO | "gpu" : " 17 GB", | 20260505 10:03:30.971 | INFO | "huge" : "", | 20260505 10:03:30.971 | INFO | "huge_shared" : " 27 GB", | 20260505 10:03:30.971 | INFO | "malloc" : " 14 GB", | 20260505 10:03:30.971 | INFO | "pinned" : " 29 GB" | 20260505 10:03:30.971 | INFO | } | 20260505 10:03:30.971 | INFO | Perf report for the last 5.1391 seconds: | 20260505 10:03:30.971 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:03:30.971 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:30.971 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:30.971 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:30.971 | INFO | ANALYSIS : 65.2%, 2.9, 100.3ms, 0 | 20260505 10:03:30.971 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:30.971 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260505 10:03:30.971 | INFO | PRE_HQ : 62.4%, 1.8, 59.9ms, 0 | 20260505 10:03:30.971 | INFO | BAZWRITER : 65.9%, 4.1, 140.4ms, 0 | 20260505 10:03:30.971 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 10:03:30.971 | INFO | Uploaded 36GiB to GPU in 2.21744s (16.2349GiB/s) | 20260505 10:03:30.971 | INFO | Downloaded 38GiB to GPU in 2.3379s (16.2539GiB/s) | 20260505 10:03:36.111 | INFO | Analyzing chunk frames = [2343424,2343936) | 20260505 10:03:36.111 | INFO | MemInfo: { | 20260505 10:03:36.111 | INFO | "gpu" : " 17 GB", | 20260505 10:03:36.111 | INFO | "huge" : "", | 20260505 10:03:36.111 | INFO | "huge_shared" : " 27 GB", | 20260505 10:03:36.111 | INFO | "malloc" : " 14 GB", | 20260505 10:03:36.111 | INFO | "pinned" : " 29 GB" | 20260505 10:03:36.111 | INFO | } | 20260505 10:03:36.111 | INFO | Perf report for the last 5.14073 seconds: | 20260505 10:03:36.111 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:03:36.111 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:36.111 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:36.111 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:36.111 | INFO | ANALYSIS : 64.4%, 3.0, 100.6ms, 0 | 20260505 10:03:36.111 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:36.111 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:36.111 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:03:36.111 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:36.112 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:36.112 | INFO | Uploaded 36GiB to GPU in 2.17904s (16.521GiB/s) | 20260505 10:03:36.112 | INFO | Downloaded 34GiB to GPU in 2.10274s (16.1694GiB/s) | 20260505 10:03:41.251 | INFO | Analyzing chunk frames = [2343936,2344448) | 20260505 10:03:41.251 | INFO | MemInfo: { | 20260505 10:03:41.251 | INFO | "gpu" : " 17 GB", | 20260505 10:03:41.251 | INFO | "huge" : "", | 20260505 10:03:41.251 | INFO | "huge_shared" : " 27 GB", | 20260505 10:03:41.251 | INFO | "malloc" : " 14 GB", | 20260505 10:03:41.251 | INFO | "pinned" : " 29 GB" | 20260505 10:03:41.251 | INFO | } | 20260505 10:03:41.251 | INFO | Perf report for the last 5.13994 seconds: | 20260505 10:03:41.251 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:03:41.251 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:41.251 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:41.251 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:41.251 | INFO | ANALYSIS : 63.0%, 3.0, 98.1ms, 0 | 20260505 10:03:41.251 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:41.251 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:41.251 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:03:41.251 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:41.251 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:41.252 | INFO | Uploaded 36GiB to GPU in 2.14036s (16.8196GiB/s) | 20260505 10:03:41.252 | INFO | Downloaded 34GiB to GPU in 2.05902s (16.5127GiB/s) | 20260505 10:03:46.390 | INFO | Analyzing chunk frames = [2344448,2344960) | 20260505 10:03:46.390 | INFO | MemInfo: { | 20260505 10:03:46.390 | INFO | "gpu" : " 17 GB", | 20260505 10:03:46.390 | INFO | "huge" : "", | 20260505 10:03:46.390 | INFO | "huge_shared" : " 27 GB", | 20260505 10:03:46.390 | INFO | "malloc" : " 14 GB", | 20260505 10:03:46.390 | INFO | "pinned" : " 29 GB" | 20260505 10:03:46.390 | INFO | } | 20260505 10:03:46.390 | INFO | Perf report for the last 5.13846 seconds: | 20260505 10:03:46.390 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:03:46.390 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:46.390 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:46.390 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:46.390 | INFO | ANALYSIS : 63.2%, 3.0, 98.1ms, 0 | 20260505 10:03:46.390 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:46.390 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:46.390 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:03:46.390 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:46.390 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:46.390 | INFO | Uploaded 36GiB to GPU in 2.17071s (16.5845GiB/s) | 20260505 10:03:46.390 | INFO | Downloaded 34GiB to GPU in 2.07455s (16.3891GiB/s) | 20260505 10:03:51.528 | INFO | Analyzing chunk frames = [2344960,2345472) | 20260505 10:03:51.528 | INFO | MemInfo: { | 20260505 10:03:51.528 | INFO | "gpu" : " 17 GB", | 20260505 10:03:51.528 | INFO | "huge" : "", | 20260505 10:03:51.528 | INFO | "huge_shared" : " 27 GB", | 20260505 10:03:51.528 | INFO | "malloc" : " 14 GB", | 20260505 10:03:51.528 | INFO | "pinned" : " 29 GB" | 20260505 10:03:51.528 | INFO | } | 20260505 10:03:51.529 | INFO | Perf report for the last 5.13872 seconds: | 20260505 10:03:51.529 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:03:51.529 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:51.529 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:51.529 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:51.529 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 10:03:51.529 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:51.529 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:51.529 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:03:51.529 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:51.529 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:51.529 | INFO | Uploaded 36GiB to GPU in 2.22892s (16.1513GiB/s) | 20260505 10:03:51.529 | INFO | Downloaded 34GiB to GPU in 2.10707s (16.1362GiB/s) | 20260505 10:03:56.669 | INFO | Analyzing chunk frames = [2345472,2345984) | 20260505 10:03:56.669 | INFO | MemInfo: { | 20260505 10:03:56.669 | INFO | "gpu" : " 17 GB", | 20260505 10:03:56.669 | INFO | "huge" : "", | 20260505 10:03:56.669 | INFO | "huge_shared" : " 27 GB", | 20260505 10:03:56.669 | INFO | "malloc" : " 14 GB", | 20260505 10:03:56.669 | INFO | "pinned" : " 29 GB" | 20260505 10:03:56.669 | INFO | } | 20260505 10:03:56.669 | INFO | Perf report for the last 5.14023 seconds: | 20260505 10:03:56.669 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:03:56.669 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:56.669 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:56.669 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:56.669 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 10:03:56.669 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:56.669 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:56.669 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 10:03:56.669 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:56.669 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:03:56.669 | INFO | Uploaded 36GiB to GPU in 2.23819s (16.0844GiB/s) | 20260505 10:03:56.669 | INFO | Downloaded 34GiB to GPU in 2.10639s (16.1414GiB/s) | 20260505 10:04:01.807 | INFO | Analyzing chunk frames = [2345984,2346496) | 20260505 10:04:01.807 | INFO | MemInfo: { | 20260505 10:04:01.807 | INFO | "gpu" : " 17 GB", | 20260505 10:04:01.807 | INFO | "huge" : "", | 20260505 10:04:01.807 | INFO | "huge_shared" : " 27 GB", | 20260505 10:04:01.807 | INFO | "malloc" : " 14 GB", | 20260505 10:04:01.807 | INFO | "pinned" : " 29 GB" | 20260505 10:04:01.807 | INFO | } | 20260505 10:04:01.807 | INFO | Perf report for the last 5.13872 seconds: | 20260505 10:04:01.807 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:04:01.807 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:01.808 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:01.808 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:01.808 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 10:04:01.808 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:01.808 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:01.808 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 10:04:01.808 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:01.808 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:01.808 | INFO | Uploaded 36GiB to GPU in 2.18772s (16.4555GiB/s) | 20260505 10:04:01.808 | INFO | Downloaded 34GiB to GPU in 2.12323s (16.0133GiB/s) | 20260505 10:04:06.947 | INFO | Analyzing chunk frames = [2346496,2347008) | 20260505 10:04:06.947 | INFO | MemInfo: { | 20260505 10:04:06.947 | INFO | "gpu" : " 17 GB", | 20260505 10:04:06.947 | INFO | "huge" : "", | 20260505 10:04:06.947 | INFO | "huge_shared" : " 27 GB", | 20260505 10:04:06.947 | INFO | "malloc" : " 14 GB", | 20260505 10:04:06.947 | INFO | "pinned" : " 29 GB" | 20260505 10:04:06.947 | INFO | } | 20260505 10:04:06.947 | INFO | Perf report for the last 5.13957 seconds: | 20260505 10:04:06.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:04:06.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:06.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:06.947 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:06.947 | INFO | ANALYSIS : 64.6%, 3.0, 100.9ms, 0 | 20260505 10:04:06.947 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:06.947 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:06.947 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 10:04:06.947 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:06.947 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:06.948 | INFO | Uploaded 36GiB to GPU in 2.24708s (16.0208GiB/s) | 20260505 10:04:06.948 | INFO | Downloaded 34GiB to GPU in 2.09392s (16.2375GiB/s) | 20260505 10:04:07.084 | INFO | Starting a round of Baz IO | 20260505 10:04:10.346 | INFO | Populating RTMetrics report | 20260505 10:04:10.346 | INFO | RTMetrics report populated | 20260505 10:04:10.346 | INFO | Opening tmp RTMetrics json file | 20260505 10:04:10.354 | INFO | Serializing RTMetrics to json | 20260505 10:04:10.385 | INFO | Serialization done, swapping tmp file now... | 20260505 10:04:10.395 | INFO | RTMetrics done | 20260505 10:04:10.503 | INFO | Finished a round of Baz IO: Wrote 2.425 GiB in 3.419 seconds | 20260505 10:04:10.503 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.36% / 0.01% / 74.62% | 20260505 10:04:10.503 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.66367% / 1.58946e-05% / 5.5631e-05% | 20260505 10:04:10.503 | INFO | Baz IO overall: | 20260505 10:04:10.503 | INFO | BytesWritten : 1368406776908 | 20260505 10:04:10.503 | INFO | NumEvents : 651430579351 | 20260505 10:04:12.088 | INFO | Analyzing chunk frames = [2347008,2347520) | 20260505 10:04:12.088 | INFO | MemInfo: { | 20260505 10:04:12.088 | INFO | "gpu" : " 17 GB", | 20260505 10:04:12.088 | INFO | "huge" : "", | 20260505 10:04:12.088 | INFO | "huge_shared" : " 27 GB", | 20260505 10:04:12.088 | INFO | "malloc" : " 14 GB", | 20260505 10:04:12.088 | INFO | "pinned" : " 29 GB" | 20260505 10:04:12.088 | INFO | } | 20260505 10:04:12.088 | INFO | Perf report for the last 5.141 seconds: | 20260505 10:04:12.088 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:04:12.088 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:12.088 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:12.088 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:12.088 | INFO | ANALYSIS : 65.0%, 2.9, 99.5ms, 0 | 20260505 10:04:12.088 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:12.088 | INFO | RT_METRICS : 46.5%, 1.0, 24.4ms, 0 | 20260505 10:04:12.088 | INFO | PRE_HQ : 62.6%, 1.8, 59.8ms, 0 | 20260505 10:04:12.088 | INFO | BAZWRITER : 65.9%, 4.2, 141.7ms, 0 | 20260505 10:04:12.088 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 10:04:12.088 | INFO | Uploaded 36GiB to GPU in 2.29898s (15.6591GiB/s) | 20260505 10:04:12.088 | INFO | Downloaded 38GiB to GPU in 2.38609s (15.9257GiB/s) | 20260505 10:04:17.225 | INFO | Analyzing chunk frames = [2347520,2348032) | 20260505 10:04:17.225 | INFO | MemInfo: { | 20260505 10:04:17.225 | INFO | "gpu" : " 17 GB", | 20260505 10:04:17.225 | INFO | "huge" : "", | 20260505 10:04:17.225 | INFO | "huge_shared" : " 27 GB", | 20260505 10:04:17.226 | INFO | "malloc" : " 14 GB", | 20260505 10:04:17.226 | INFO | "pinned" : " 29 GB" | 20260505 10:04:17.226 | INFO | } | 20260505 10:04:17.226 | INFO | Perf report for the last 5.13754 seconds: | 20260505 10:04:17.226 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:04:17.226 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:17.226 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:17.226 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:17.226 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 10:04:17.226 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:17.226 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:17.226 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:04:17.226 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:17.226 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:17.226 | INFO | Uploaded 36GiB to GPU in 2.17462s (16.5546GiB/s) | 20260505 10:04:17.226 | INFO | Downloaded 34GiB to GPU in 2.09024s (16.2661GiB/s) | 20260505 10:04:22.364 | INFO | Analyzing chunk frames = [2348032,2348544) | 20260505 10:04:22.364 | INFO | MemInfo: { | 20260505 10:04:22.364 | INFO | "gpu" : " 17 GB", | 20260505 10:04:22.364 | INFO | "huge" : "", | 20260505 10:04:22.364 | INFO | "huge_shared" : " 27 GB", | 20260505 10:04:22.364 | INFO | "malloc" : " 14 GB", | 20260505 10:04:22.365 | INFO | "pinned" : " 29 GB" | 20260505 10:04:22.365 | INFO | } | 20260505 10:04:22.365 | INFO | Perf report for the last 5.13909 seconds: | 20260505 10:04:22.365 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:04:22.365 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:22.365 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:22.365 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:22.365 | INFO | ANALYSIS : 63.1%, 3.0, 98.1ms, 0 | 20260505 10:04:22.365 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:22.365 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:22.365 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:04:22.365 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:22.365 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:22.365 | INFO | Uploaded 36GiB to GPU in 2.15232s (16.7262GiB/s) | 20260505 10:04:22.365 | INFO | Downloaded 34GiB to GPU in 2.05842s (16.5175GiB/s) | 20260505 10:04:27.504 | INFO | Analyzing chunk frames = [2348544,2349056) | 20260505 10:04:27.504 | INFO | MemInfo: { | 20260505 10:04:27.504 | INFO | "gpu" : " 17 GB", | 20260505 10:04:27.504 | INFO | "huge" : "", | 20260505 10:04:27.504 | INFO | "huge_shared" : " 27 GB", | 20260505 10:04:27.504 | INFO | "malloc" : " 14 GB", | 20260505 10:04:27.504 | INFO | "pinned" : " 29 GB" | 20260505 10:04:27.504 | INFO | } | 20260505 10:04:27.504 | INFO | Perf report for the last 5.13964 seconds: | 20260505 10:04:27.504 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:04:27.504 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:27.504 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:27.504 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:27.504 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 10:04:27.504 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:27.504 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:27.505 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:04:27.505 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:27.505 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:27.505 | INFO | Uploaded 36GiB to GPU in 2.18394s (16.484GiB/s) | 20260505 10:04:27.505 | INFO | Downloaded 34GiB to GPU in 2.07328s (16.3991GiB/s) | 20260505 10:04:32.644 | INFO | Analyzing chunk frames = [2349056,2349568) | 20260505 10:04:32.644 | INFO | MemInfo: { | 20260505 10:04:32.644 | INFO | "gpu" : " 17 GB", | 20260505 10:04:32.644 | INFO | "huge" : "", | 20260505 10:04:32.644 | INFO | "huge_shared" : " 27 GB", | 20260505 10:04:32.644 | INFO | "malloc" : " 14 GB", | 20260505 10:04:32.645 | INFO | "pinned" : " 29 GB" | 20260505 10:04:32.645 | INFO | } | 20260505 10:04:32.645 | INFO | Perf report for the last 5.14023 seconds: | 20260505 10:04:32.645 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:04:32.645 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:32.645 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:32.645 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:32.645 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 10:04:32.645 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:32.645 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:32.645 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 10:04:32.645 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:32.645 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:32.645 | INFO | Uploaded 36GiB to GPU in 2.23576s (16.1019GiB/s) | 20260505 10:04:32.645 | INFO | Downloaded 34GiB to GPU in 2.10583s (16.1456GiB/s) | 20260505 10:04:37.784 | INFO | Analyzing chunk frames = [2349568,2350080) | 20260505 10:04:37.784 | INFO | MemInfo: { | 20260505 10:04:37.784 | INFO | "gpu" : " 17 GB", | 20260505 10:04:37.784 | INFO | "huge" : "", | 20260505 10:04:37.784 | INFO | "huge_shared" : " 27 GB", | 20260505 10:04:37.784 | INFO | "malloc" : " 14 GB", | 20260505 10:04:37.784 | INFO | "pinned" : " 29 GB" | 20260505 10:04:37.784 | INFO | } | 20260505 10:04:37.784 | INFO | Perf report for the last 5.13925 seconds: | 20260505 10:04:37.784 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:04:37.784 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:37.784 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:37.784 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:37.784 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 10:04:37.784 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:37.784 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:37.784 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:04:37.784 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:37.784 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:37.784 | INFO | Uploaded 36GiB to GPU in 2.23936s (16.076GiB/s) | 20260505 10:04:37.784 | INFO | Downloaded 34GiB to GPU in 2.10394s (16.1601GiB/s) | 20260505 10:04:42.922 | INFO | Analyzing chunk frames = [2350080,2350592) | 20260505 10:04:42.922 | INFO | MemInfo: { | 20260505 10:04:42.922 | INFO | "gpu" : " 17 GB", | 20260505 10:04:42.922 | INFO | "huge" : "", | 20260505 10:04:42.922 | INFO | "huge_shared" : " 27 GB", | 20260505 10:04:42.922 | INFO | "malloc" : " 14 GB", | 20260505 10:04:42.922 | INFO | "pinned" : " 29 GB" | 20260505 10:04:42.922 | INFO | } | 20260505 10:04:42.922 | INFO | Perf report for the last 5.13834 seconds: | 20260505 10:04:42.922 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:04:42.922 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:42.922 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:42.922 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:42.922 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 10:04:42.922 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:42.922 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:42.922 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 10:04:42.922 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:42.922 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:42.923 | INFO | Uploaded 36GiB to GPU in 2.1562s (16.696GiB/s) | 20260505 10:04:42.923 | INFO | Downloaded 34GiB to GPU in 2.14758s (15.8318GiB/s) | 20260505 10:04:48.061 | INFO | Analyzing chunk frames = [2350592,2351104) | 20260505 10:04:48.061 | INFO | MemInfo: { | 20260505 10:04:48.061 | INFO | "gpu" : " 17 GB", | 20260505 10:04:48.061 | INFO | "huge" : "", | 20260505 10:04:48.061 | INFO | "huge_shared" : " 27 GB", | 20260505 10:04:48.061 | INFO | "malloc" : " 14 GB", | 20260505 10:04:48.061 | INFO | "pinned" : " 29 GB" | 20260505 10:04:48.062 | INFO | } | 20260505 10:04:48.062 | INFO | Perf report for the last 5.13947 seconds: | 20260505 10:04:48.062 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:04:48.062 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:48.062 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:48.062 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:48.062 | INFO | ANALYSIS : 64.6%, 3.0, 100.9ms, 0 | 20260505 10:04:48.062 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:48.062 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:48.062 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 10:04:48.062 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:48.062 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:48.062 | INFO | Uploaded 36GiB to GPU in 2.24684s (16.0225GiB/s) | 20260505 10:04:48.062 | INFO | Downloaded 34GiB to GPU in 2.09165s (16.2551GiB/s) | 20260505 10:04:48.062 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 10:04:48.062 | INFO | Total mem usage: 15.3 GiB | 20260505 10:04:48.062 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 10:04:48.062 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.51%) | 20260505 10:04:48.062 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 10:04:48.062 | INFO | High Water Breakdown: | 20260505 10:04:48.062 | INFO | 61% : 8.78 GiB ( 93.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 10:04:48.062 | INFO | 38.8% : 5.59 GiB ( 84.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 10:04:48.062 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 10:04:48.062 | INFO | ------------------------------------------ | 20260505 10:04:48.062 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:04:48.062 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 10:04:48.062 | INFO | Total mem usage: 26.5 GiB | 20260505 10:04:48.062 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 10:04:48.062 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 10:04:48.062 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 10:04:48.062 | INFO | High Water Breakdown: | 20260505 10:04:48.062 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:04:48.062 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:04:48.062 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:04:48.062 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:04:48.062 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:04:48.063 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:04:48.063 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 10:04:48.063 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 10:04:48.063 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:04:48.063 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 10:04:48.063 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 10:04:48.063 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 10:04:48.063 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 10:04:48.063 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 10:04:48.063 | INFO | ------------------------------------------ | 20260505 10:04:48.063 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:04:48.063 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 10:04:48.063 | INFO | Total mem usage: 25.6 GiB | 20260505 10:04:48.063 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 10:04:48.063 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 10:04:48.063 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 10:04:48.063 | INFO | High Water Breakdown: | 20260505 10:04:48.063 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 10:04:48.063 | INFO | ------------------------------------------ | 20260505 10:04:48.063 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:04:48.063 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 10:04:48.063 | INFO | Total mem usage: 15.7 GiB | 20260505 10:04:48.063 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 10:04:48.063 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 10:04:48.063 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 10:04:48.063 | INFO | High Water Breakdown: | 20260505 10:04:48.063 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 10:04:48.063 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 10:04:48.063 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 10:04:48.063 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 10:04:48.063 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 10:04:48.063 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 10:04:48.063 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 10:04:48.063 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 10:04:48.063 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 10:04:48.063 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:04:48.063 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 10:04:48.063 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:04:48.063 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 10:04:48.063 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 10:04:48.063 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 10:04:48.063 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 10:04:48.063 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:04:48.063 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 10:04:48.063 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:04:48.063 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:04:48.063 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:04:48.063 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 10:04:48.063 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 10:04:48.063 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:04:48.063 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 10:04:48.063 | INFO | ------------------------------------------ | 20260505 10:04:48.063 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:04:48.063 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 10:04:48.063 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:04:48.063 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:04:48.063 | INFO | PerfReport: Full Execution -- 3.24e+01 / 7.74e+01, 1.37e+01 / 2.26e+00 | 20260505 10:04:48.063 | INFO | PerfReport: 0.29%, AggregateStats -- 9.32e-02 / 1.04e-01, 1.18e-02 / 6.43e-03 | 20260505 10:04:48.063 | INFO | PerfReport: 11.67%, Baseline -- 3.78e+00 / 4.42e+00, 6.12e-01 / 2.89e-01 | 20260505 10:04:48.063 | INFO | PerfReport: 3.32%, Histogram -- 1.08e+00 / 1.48e+00, 2.08e-01 / 1.07e-01 | 20260505 10:04:48.063 | INFO | PerfReport: 15.64%, DME -- 4.06e+01 / 4.48e+01, 4.50e+00 / 2.27e+00 | 20260505 10:04:48.063 | INFO | PerfReport: 56.60%, FrameLabeling -- 1.84e+01 / 2.16e+01, 1.82e+00 / 1.89e+00 | 20260505 10:04:48.063 | INFO | PerfReport: 7.36%, PulseAccumulating -- 2.39e+00 / 2.96e+00, 4.06e-01 / 1.81e-01 | 20260505 10:04:48.063 | INFO | PerfReport: 3.79%, Metrics -- 1.23e+00 / 1.52e+00, 3.53e-01 / 2.07e-01 | 20260505 10:04:48.063 | INFO | PerfReport: 1.34%, PulseSerializing -- 4.35e-01 / 4.77e-01, 8.16e-02 / 3.29e-02 | 20260505 10:04:48.064 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 10:04:48.064 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:04:48.064 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:04:48.064 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260505 10:04:48.064 | INFO | PerfReport: 22.62%, Upload -- 2.30e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260505 10:04:48.064 | INFO | PerfReport: 21.83%, Download -- 2.22e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260505 10:04:48.064 | INFO | PerfReport: 31.92%, Compute -- 3.24e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 10:04:48.064 | INFO | PerfReport: 1.92%, LockUpload -- 1.95e+00 / 0.00e+00, 7.51e+00 / 0.00e+00 | 20260505 10:04:48.064 | INFO | PerfReport: 1.12%, LockDownload -- 1.14e+00 / 0.00e+00, 2.76e+00 / 0.00e+00 | 20260505 10:04:48.064 | INFO | PerfReport: 20.57%, LockCompute -- 2.09e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260505 10:04:48.064 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 2.02e-02 / 0.00e+00 | 20260505 10:04:48.197 | INFO | Starting a round of Baz IO | 20260505 10:04:51.472 | INFO | Populating RTMetrics report | 20260505 10:04:51.472 | INFO | RTMetrics report populated | 20260505 10:04:51.472 | INFO | Opening tmp RTMetrics json file | 20260505 10:04:51.490 | INFO | Serializing RTMetrics to json | 20260505 10:04:51.491 | INFO | Serialization done, swapping tmp file now... | 20260505 10:04:51.492 | INFO | RTMetrics done | 20260505 10:04:51.614 | INFO | Finished a round of Baz IO: Wrote 2.427 GiB in 3.417 seconds | 20260505 10:04:51.614 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.35% / 0.01% / 74.64% | 20260505 10:04:51.614 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.66083% / 7.94729e-06% / 8.34465e-05% | 20260505 10:04:51.614 | INFO | Baz IO overall: | 20260505 10:04:51.614 | INFO | BytesWritten : 1371012233692 | 20260505 10:04:51.614 | INFO | NumEvents : 652674114151 | 20260505 10:04:53.201 | INFO | Analyzing chunk frames = [2351104,2351616) | 20260505 10:04:53.201 | INFO | MemInfo: { | 20260505 10:04:53.201 | INFO | "gpu" : " 17 GB", | 20260505 10:04:53.201 | INFO | "huge" : "", | 20260505 10:04:53.201 | INFO | "huge_shared" : " 27 GB", | 20260505 10:04:53.201 | INFO | "malloc" : " 14 GB", | 20260505 10:04:53.201 | INFO | "pinned" : " 29 GB" | 20260505 10:04:53.201 | INFO | } | 20260505 10:04:53.201 | INFO | Perf report for the last 5.13947 seconds: | 20260505 10:04:53.201 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:04:53.201 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:53.201 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:53.201 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:53.201 | INFO | ANALYSIS : 65.2%, 2.9, 99.7ms, 0 | 20260505 10:04:53.201 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:53.201 | INFO | RT_METRICS : 46.0%, 1.0, 24.2ms, 0 | 20260505 10:04:53.201 | INFO | PRE_HQ : 61.8%, 1.9, 60.1ms, 0 | 20260505 10:04:53.201 | INFO | BAZWRITER : 65.8%, 4.2, 140.8ms, 0 | 20260505 10:04:53.201 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 10:04:53.202 | INFO | Uploaded 36GiB to GPU in 2.18s (16.5138GiB/s) | 20260505 10:04:53.202 | INFO | Downloaded 38GiB to GPU in 2.31412s (16.4209GiB/s) | 20260505 10:04:58.341 | INFO | Analyzing chunk frames = [2351616,2352128) | 20260505 10:04:58.341 | INFO | MemInfo: { | 20260505 10:04:58.341 | INFO | "gpu" : " 17 GB", | 20260505 10:04:58.341 | INFO | "huge" : "", | 20260505 10:04:58.341 | INFO | "huge_shared" : " 27 GB", | 20260505 10:04:58.341 | INFO | "malloc" : " 14 GB", | 20260505 10:04:58.341 | INFO | "pinned" : " 29 GB" | 20260505 10:04:58.341 | INFO | } | 20260505 10:04:58.341 | INFO | Perf report for the last 5.13965 seconds: | 20260505 10:04:58.341 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:04:58.341 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:58.341 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:58.341 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:58.341 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 10:04:58.341 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:58.341 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:58.341 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:04:58.341 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:58.341 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:04:58.341 | INFO | Uploaded 36GiB to GPU in 2.17046s (16.5863GiB/s) | 20260505 10:04:58.341 | INFO | Downloaded 34GiB to GPU in 2.09197s (16.2526GiB/s) | 20260505 10:05:03.479 | INFO | Analyzing chunk frames = [2352128,2352640) | 20260505 10:05:03.479 | INFO | MemInfo: { | 20260505 10:05:03.479 | INFO | "gpu" : " 17 GB", | 20260505 10:05:03.479 | INFO | "huge" : "", | 20260505 10:05:03.479 | INFO | "huge_shared" : " 27 GB", | 20260505 10:05:03.479 | INFO | "malloc" : " 14 GB", | 20260505 10:05:03.479 | INFO | "pinned" : " 29 GB" | 20260505 10:05:03.479 | INFO | } | 20260505 10:05:03.479 | INFO | Perf report for the last 5.13829 seconds: | 20260505 10:05:03.479 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:05:03.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:03.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:03.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:03.479 | INFO | ANALYSIS : 63.1%, 3.0, 98.1ms, 0 | 20260505 10:05:03.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:03.479 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:03.479 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 10:05:03.479 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:03.479 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:03.479 | INFO | Uploaded 36GiB to GPU in 2.15593s (16.6981GiB/s) | 20260505 10:05:03.479 | INFO | Downloaded 34GiB to GPU in 2.06264s (16.4837GiB/s) | 20260505 10:05:08.618 | INFO | Analyzing chunk frames = [2352640,2353152) | 20260505 10:05:08.618 | INFO | MemInfo: { | 20260505 10:05:08.618 | INFO | "gpu" : " 17 GB", | 20260505 10:05:08.618 | INFO | "huge" : "", | 20260505 10:05:08.618 | INFO | "huge_shared" : " 27 GB", | 20260505 10:05:08.618 | INFO | "malloc" : " 14 GB", | 20260505 10:05:08.618 | INFO | "pinned" : " 29 GB" | 20260505 10:05:08.618 | INFO | } | 20260505 10:05:08.619 | INFO | Perf report for the last 5.13935 seconds: | 20260505 10:05:08.619 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:05:08.619 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:08.619 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:08.619 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:08.619 | INFO | ANALYSIS : 63.1%, 3.0, 98.1ms, 0 | 20260505 10:05:08.619 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:08.619 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:08.619 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:05:08.619 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:08.619 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:08.619 | INFO | Uploaded 36GiB to GPU in 2.16227s (16.6492GiB/s) | 20260505 10:05:08.619 | INFO | Downloaded 34GiB to GPU in 2.07021s (16.4234GiB/s) | 20260505 10:05:13.758 | INFO | Analyzing chunk frames = [2353152,2353664) | 20260505 10:05:13.758 | INFO | MemInfo: { | 20260505 10:05:13.758 | INFO | "gpu" : " 17 GB", | 20260505 10:05:13.758 | INFO | "huge" : "", | 20260505 10:05:13.758 | INFO | "huge_shared" : " 27 GB", | 20260505 10:05:13.759 | INFO | "malloc" : " 14 GB", | 20260505 10:05:13.759 | INFO | "pinned" : " 29 GB" | 20260505 10:05:13.759 | INFO | } | 20260505 10:05:13.759 | INFO | Perf report for the last 5.14015 seconds: | 20260505 10:05:13.759 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:05:13.759 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:13.759 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:13.759 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:13.759 | INFO | ANALYSIS : 63.7%, 3.0, 99.6ms, 0 | 20260505 10:05:13.759 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:13.759 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:13.759 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 10:05:13.759 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:13.759 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:13.759 | INFO | Uploaded 36GiB to GPU in 2.23346s (16.1185GiB/s) | 20260505 10:05:13.759 | INFO | Downloaded 34GiB to GPU in 2.1165s (16.0642GiB/s) | 20260505 10:05:18.898 | INFO | Analyzing chunk frames = [2353664,2354176) | 20260505 10:05:18.898 | INFO | MemInfo: { | 20260505 10:05:18.898 | INFO | "gpu" : " 17 GB", | 20260505 10:05:18.898 | INFO | "huge" : "", | 20260505 10:05:18.898 | INFO | "huge_shared" : " 27 GB", | 20260505 10:05:18.898 | INFO | "malloc" : " 14 GB", | 20260505 10:05:18.899 | INFO | "pinned" : " 29 GB" | 20260505 10:05:18.899 | INFO | } | 20260505 10:05:18.899 | INFO | Perf report for the last 5.13992 seconds: | 20260505 10:05:18.899 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:05:18.899 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:18.899 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:18.899 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:18.899 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 10:05:18.899 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:18.899 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:18.899 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 10:05:18.899 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:18.899 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:18.899 | INFO | Uploaded 36GiB to GPU in 2.23786s (16.0868GiB/s) | 20260505 10:05:18.899 | INFO | Downloaded 34GiB to GPU in 2.10415s (16.1585GiB/s) | 20260505 10:05:24.056 | INFO | Analyzing chunk frames = [2354176,2354688) | 20260505 10:05:24.056 | INFO | MemInfo: { | 20260505 10:05:24.056 | INFO | "gpu" : " 17 GB", | 20260505 10:05:24.056 | INFO | "huge" : "", | 20260505 10:05:24.056 | INFO | "huge_shared" : " 27 GB", | 20260505 10:05:24.056 | INFO | "malloc" : " 14 GB", | 20260505 10:05:24.056 | INFO | "pinned" : " 29 GB" | 20260505 10:05:24.056 | INFO | } | 20260505 10:05:24.056 | INFO | Perf report for the last 5.15726 seconds: | 20260505 10:05:24.056 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:05:24.056 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:24.056 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:24.056 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:24.056 | INFO | ANALYSIS : 63.6%, 3.0, 99.7ms, 0 | 20260505 10:05:24.056 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:24.056 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:24.056 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260505 10:05:24.056 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:24.056 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:24.056 | INFO | Uploaded 36GiB to GPU in 2.16919s (16.596GiB/s) | 20260505 10:05:24.056 | INFO | Downloaded 34GiB to GPU in 2.1482s (15.8272GiB/s) | 20260505 10:05:29.285 | INFO | Analyzing chunk frames = [2354688,2355200) | 20260505 10:05:29.285 | INFO | MemInfo: { | 20260505 10:05:29.285 | INFO | "gpu" : " 17 GB", | 20260505 10:05:29.285 | INFO | "huge" : "", | 20260505 10:05:29.285 | INFO | "huge_shared" : " 27 GB", | 20260505 10:05:29.285 | INFO | "malloc" : " 14 GB", | 20260505 10:05:29.286 | INFO | "pinned" : " 29 GB" | 20260505 10:05:29.286 | INFO | } | 20260505 10:05:29.286 | INFO | Perf report for the last 5.22979 seconds: | 20260505 10:05:29.286 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:05:29.286 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:29.286 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:29.286 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:29.286 | INFO | ANALYSIS : 62.8%, 3.0, 99.8ms, 0 | 20260505 10:05:29.286 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:29.286 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:29.286 | INFO | PRE_HQ : 30.0%, 1.0, 15.8ms, 0 | 20260505 10:05:29.286 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:29.286 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:29.286 | INFO | Uploaded 36GiB to GPU in 2.12188s (16.9661GiB/s) | 20260505 10:05:29.286 | INFO | Downloaded 34GiB to GPU in 2.10243s (16.1718GiB/s) | 20260505 10:05:29.420 | INFO | Starting a round of Baz IO | 20260505 10:05:32.686 | INFO | Populating RTMetrics report | 20260505 10:05:32.687 | INFO | RTMetrics report populated | 20260505 10:05:32.687 | INFO | Opening tmp RTMetrics json file | 20260505 10:05:32.698 | INFO | Serializing RTMetrics to json | 20260505 10:05:32.701 | INFO | Serialization done, swapping tmp file now... | 20260505 10:05:32.703 | INFO | RTMetrics done | 20260505 10:05:32.843 | INFO | Finished a round of Baz IO: Wrote 2.425 GiB in 3.422 seconds | 20260505 10:05:32.843 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.34% / 0.01% / 74.65% | 20260505 10:05:32.843 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.65495% / 7.94729e-06% / 8.34465e-05% | 20260505 10:05:32.843 | INFO | Baz IO overall: | 20260505 10:05:32.843 | INFO | BytesWritten : 1373616069829 | 20260505 10:05:32.843 | INFO | NumEvents : 653916600520 | 20260505 10:05:34.316 | INFO | Analyzing chunk frames = [2355200,2355712) | 20260505 10:05:34.316 | INFO | MemInfo: { | 20260505 10:05:34.316 | INFO | "gpu" : " 17 GB", | 20260505 10:05:34.316 | INFO | "huge" : "", | 20260505 10:05:34.316 | INFO | "huge_shared" : " 27 GB", | 20260505 10:05:34.316 | INFO | "malloc" : " 14 GB", | 20260505 10:05:34.316 | INFO | "pinned" : " 29 GB" | 20260505 10:05:34.316 | INFO | } | 20260505 10:05:34.316 | INFO | Perf report for the last 5.03049 seconds: | 20260505 10:05:34.316 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:05:34.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:34.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:34.316 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:34.316 | INFO | ANALYSIS : 66.4%, 2.9, 98.7ms, 0 | 20260505 10:05:34.316 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:34.316 | INFO | RT_METRICS : 47.9%, 1.0, 24.6ms, 0 | 20260505 10:05:34.316 | INFO | PRE_HQ : 64.5%, 1.9, 61.1ms, 0 | 20260505 10:05:34.316 | INFO | BAZWRITER : 67.3%, 4.2, 143.8ms, 0 | 20260505 10:05:34.316 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 10:05:34.317 | INFO | Uploaded 36GiB to GPU in 2.28866s (15.7297GiB/s) | 20260505 10:05:34.317 | INFO | Downloaded 38GiB to GPU in 2.39895s (15.8403GiB/s) | 20260505 10:05:39.455 | INFO | Analyzing chunk frames = [2355712,2356224) | 20260505 10:05:39.455 | INFO | MemInfo: { | 20260505 10:05:39.455 | INFO | "gpu" : " 17 GB", | 20260505 10:05:39.455 | INFO | "huge" : "", | 20260505 10:05:39.455 | INFO | "huge_shared" : " 27 GB", | 20260505 10:05:39.455 | INFO | "malloc" : " 14 GB", | 20260505 10:05:39.455 | INFO | "pinned" : " 29 GB" | 20260505 10:05:39.455 | INFO | } | 20260505 10:05:39.455 | INFO | Perf report for the last 5.13895 seconds: | 20260505 10:05:39.455 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:05:39.455 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:39.455 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:39.455 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:39.455 | INFO | ANALYSIS : 64.5%, 3.0, 100.6ms, 0 | 20260505 10:05:39.455 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:39.455 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:39.455 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:05:39.455 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:39.455 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:39.456 | INFO | Uploaded 36GiB to GPU in 2.22471s (16.1819GiB/s) | 20260505 10:05:39.456 | INFO | Downloaded 34GiB to GPU in 2.10452s (16.1557GiB/s) | 20260505 10:05:44.595 | INFO | Analyzing chunk frames = [2356224,2356736) | 20260505 10:05:44.595 | INFO | MemInfo: { | 20260505 10:05:44.595 | INFO | "gpu" : " 17 GB", | 20260505 10:05:44.595 | INFO | "huge" : "", | 20260505 10:05:44.595 | INFO | "huge_shared" : " 27 GB", | 20260505 10:05:44.595 | INFO | "malloc" : " 14 GB", | 20260505 10:05:44.595 | INFO | "pinned" : " 29 GB" | 20260505 10:05:44.595 | INFO | } | 20260505 10:05:44.595 | INFO | Perf report for the last 5.1403 seconds: | 20260505 10:05:44.595 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:05:44.595 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:44.595 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:44.595 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:44.595 | INFO | ANALYSIS : 63.2%, 3.0, 98.4ms, 0 | 20260505 10:05:44.595 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:44.595 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:44.596 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:05:44.596 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:44.596 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:44.596 | INFO | Uploaded 36GiB to GPU in 2.14908s (16.7514GiB/s) | 20260505 10:05:44.596 | INFO | Downloaded 34GiB to GPU in 2.06414s (16.4718GiB/s) | 20260505 10:05:49.735 | INFO | Analyzing chunk frames = [2356736,2357248) | 20260505 10:05:49.735 | INFO | MemInfo: { | 20260505 10:05:49.735 | INFO | "gpu" : " 17 GB", | 20260505 10:05:49.735 | INFO | "huge" : "", | 20260505 10:05:49.735 | INFO | "huge_shared" : " 27 GB", | 20260505 10:05:49.735 | INFO | "malloc" : " 14 GB", | 20260505 10:05:49.735 | INFO | "pinned" : " 29 GB" | 20260505 10:05:49.735 | INFO | } | 20260505 10:05:49.736 | INFO | Perf report for the last 5.14035 seconds: | 20260505 10:05:49.736 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:05:49.736 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:49.736 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:49.736 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:49.736 | INFO | ANALYSIS : 63.2%, 3.0, 98.1ms, 0 | 20260505 10:05:49.736 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:49.736 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:49.736 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 10:05:49.736 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:49.736 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:49.736 | INFO | Uploaded 36GiB to GPU in 2.17795s (16.5293GiB/s) | 20260505 10:05:49.736 | INFO | Downloaded 34GiB to GPU in 2.07809s (16.3612GiB/s) | 20260505 10:05:54.873 | INFO | Analyzing chunk frames = [2357248,2357760) | 20260505 10:05:54.873 | INFO | MemInfo: { | 20260505 10:05:54.873 | INFO | "gpu" : " 17 GB", | 20260505 10:05:54.873 | INFO | "huge" : "", | 20260505 10:05:54.873 | INFO | "huge_shared" : " 27 GB", | 20260505 10:05:54.873 | INFO | "malloc" : " 14 GB", | 20260505 10:05:54.873 | INFO | "pinned" : " 29 GB" | 20260505 10:05:54.873 | INFO | } | 20260505 10:05:54.873 | INFO | Perf report for the last 5.13702 seconds: | 20260505 10:05:54.873 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:05:54.873 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:54.873 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:54.873 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:54.873 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 10:05:54.873 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:54.873 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:54.873 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:05:54.873 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:54.873 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:05:54.873 | INFO | Uploaded 36GiB to GPU in 2.2324s (16.1261GiB/s) | 20260505 10:05:54.874 | INFO | Downloaded 34GiB to GPU in 2.10558s (16.1476GiB/s) | 20260505 10:06:00.014 | INFO | Analyzing chunk frames = [2357760,2358272) | 20260505 10:06:00.014 | INFO | MemInfo: { | 20260505 10:06:00.014 | INFO | "gpu" : " 17 GB", | 20260505 10:06:00.014 | INFO | "huge" : "", | 20260505 10:06:00.014 | INFO | "huge_shared" : " 27 GB", | 20260505 10:06:00.014 | INFO | "malloc" : " 14 GB", | 20260505 10:06:00.014 | INFO | "pinned" : " 29 GB" | 20260505 10:06:00.014 | INFO | } | 20260505 10:06:00.014 | INFO | Perf report for the last 5.14121 seconds: | 20260505 10:06:00.014 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:06:00.014 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:00.014 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:00.014 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:00.014 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 10:06:00.014 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:00.014 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:00.014 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:06:00.014 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:00.014 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:00.014 | INFO | Uploaded 36GiB to GPU in 2.23206s (16.1286GiB/s) | 20260505 10:06:00.014 | INFO | Downloaded 34GiB to GPU in 2.0933s (16.2423GiB/s) | 20260505 10:06:05.152 | INFO | Analyzing chunk frames = [2358272,2358784) | 20260505 10:06:05.152 | INFO | MemInfo: { | 20260505 10:06:05.152 | INFO | "gpu" : " 17 GB", | 20260505 10:06:05.152 | INFO | "huge" : "", | 20260505 10:06:05.152 | INFO | "huge_shared" : " 27 GB", | 20260505 10:06:05.152 | INFO | "malloc" : " 14 GB", | 20260505 10:06:05.152 | INFO | "pinned" : " 29 GB" | 20260505 10:06:05.152 | INFO | } | 20260505 10:06:05.152 | INFO | Perf report for the last 5.13813 seconds: | 20260505 10:06:05.152 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:06:05.152 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:05.152 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:05.152 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:05.152 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 10:06:05.152 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:05.152 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:05.152 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 10:06:05.153 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:05.153 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:05.153 | INFO | Uploaded 36GiB to GPU in 2.19222s (16.4217GiB/s) | 20260505 10:06:05.153 | INFO | Downloaded 34GiB to GPU in 2.14979s (15.8155GiB/s) | 20260505 10:06:10.291 | INFO | Analyzing chunk frames = [2358784,2359296) | 20260505 10:06:10.291 | INFO | MemInfo: { | 20260505 10:06:10.291 | INFO | "gpu" : " 17 GB", | 20260505 10:06:10.291 | INFO | "huge" : "", | 20260505 10:06:10.291 | INFO | "huge_shared" : " 27 GB", | 20260505 10:06:10.291 | INFO | "malloc" : " 14 GB", | 20260505 10:06:10.291 | INFO | "pinned" : " 29 GB" | 20260505 10:06:10.291 | INFO | } | 20260505 10:06:10.291 | INFO | Perf report for the last 5.13856 seconds: | 20260505 10:06:10.291 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:06:10.291 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:10.291 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:10.291 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:10.291 | INFO | ANALYSIS : 64.4%, 3.0, 100.6ms, 0 | 20260505 10:06:10.291 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:10.291 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:10.291 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 10:06:10.291 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:10.291 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:10.291 | INFO | Uploaded 36GiB to GPU in 2.24811s (16.0135GiB/s) | 20260505 10:06:10.291 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 10:06:10.291 | INFO | Total mem usage: 15.3 GiB | 20260505 10:06:10.291 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 10:06:10.291 | INFO | Downloaded 34GiB to GPU in 2.09378s (16.2386GiB/s) | 20260505 10:06:10.291 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -5.76%) | 20260505 10:06:10.291 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 10:06:10.292 | INFO | High Water Breakdown: | 20260505 10:06:10.292 | INFO | 61.1% : 8.86 GiB ( 94.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 10:06:10.292 | INFO | 38.7% : 5.62 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 10:06:10.292 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 10:06:10.292 | INFO | ------------------------------------------ | 20260505 10:06:10.292 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:06:10.292 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 10:06:10.292 | INFO | Total mem usage: 26.5 GiB | 20260505 10:06:10.292 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 10:06:10.292 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 10:06:10.292 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 10:06:10.292 | INFO | High Water Breakdown: | 20260505 10:06:10.292 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:06:10.292 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:06:10.292 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:06:10.292 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:06:10.292 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:06:10.292 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:06:10.292 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 10:06:10.292 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 10:06:10.292 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:06:10.292 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 10:06:10.292 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 10:06:10.292 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 10:06:10.292 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 10:06:10.292 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 10:06:10.292 | INFO | ------------------------------------------ | 20260505 10:06:10.292 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:06:10.292 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 10:06:10.292 | INFO | Total mem usage: 25.6 GiB | 20260505 10:06:10.292 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 10:06:10.292 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 10:06:10.292 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 10:06:10.292 | INFO | High Water Breakdown: | 20260505 10:06:10.292 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 10:06:10.292 | INFO | ------------------------------------------ | 20260505 10:06:10.292 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:06:10.292 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 10:06:10.292 | INFO | Total mem usage: 15.7 GiB | 20260505 10:06:10.292 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 10:06:10.292 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 10:06:10.292 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 10:06:10.292 | INFO | High Water Breakdown: | 20260505 10:06:10.292 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 10:06:10.292 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 10:06:10.292 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 10:06:10.292 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 10:06:10.292 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 10:06:10.292 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 10:06:10.292 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 10:06:10.292 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 10:06:10.292 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 10:06:10.292 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:06:10.292 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 10:06:10.292 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:06:10.292 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 10:06:10.292 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 10:06:10.292 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 10:06:10.292 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 10:06:10.292 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:06:10.292 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 10:06:10.292 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:06:10.292 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:06:10.292 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:06:10.292 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 10:06:10.292 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 10:06:10.292 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:06:10.292 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 10:06:10.293 | INFO | ------------------------------------------ | 20260505 10:06:10.293 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:06:10.293 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 10:06:10.293 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:06:10.293 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:06:10.293 | INFO | PerfReport: Full Execution -- 3.24e+01 / 7.72e+01, 1.39e+01 / 2.36e+00 | 20260505 10:06:10.293 | INFO | PerfReport: 0.29%, AggregateStats -- 9.35e-02 / 1.04e-01, 1.16e-02 / 6.84e-03 | 20260505 10:06:10.293 | INFO | PerfReport: 11.65%, Baseline -- 3.78e+00 / 4.45e+00, 6.06e-01 / 2.56e-01 | 20260505 10:06:10.293 | INFO | PerfReport: 3.31%, Histogram -- 1.07e+00 / 1.50e+00, 2.09e-01 / 9.53e-02 | 20260505 10:06:10.293 | INFO | PerfReport: 15.69%, DME -- 4.07e+01 / 4.52e+01, 4.66e+00 / 2.63e+00 | 20260505 10:06:10.293 | INFO | PerfReport: 56.58%, FrameLabeling -- 1.83e+01 / 2.10e+01, 1.82e+00 / 2.19e+00 | 20260505 10:06:10.293 | INFO | PerfReport: 7.35%, PulseAccumulating -- 2.38e+00 / 2.91e+00, 4.04e-01 / 2.28e-01 | 20260505 10:06:10.293 | INFO | PerfReport: 3.79%, Metrics -- 1.23e+00 / 1.52e+00, 3.56e-01 / 2.16e-01 | 20260505 10:06:10.293 | INFO | PerfReport: 1.34%, PulseSerializing -- 4.34e-01 / 4.80e-01, 8.07e-02 / 3.89e-02 | 20260505 10:06:10.293 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 10:06:10.293 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:06:10.293 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:06:10.293 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.42e+01 / 0.00e+00 | 20260505 10:06:10.293 | INFO | PerfReport: 22.58%, Upload -- 2.29e+01 / 0.00e+00, 8.17e+00 / 0.00e+00 | 20260505 10:06:10.293 | INFO | PerfReport: 21.89%, Download -- 2.22e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260505 10:06:10.293 | INFO | PerfReport: 31.97%, Compute -- 3.24e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 10:06:10.293 | INFO | PerfReport: 1.89%, LockUpload -- 1.91e+00 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260505 10:06:10.293 | INFO | PerfReport: 1.18%, LockDownload -- 1.19e+00 / 0.00e+00, 2.85e+00 / 0.00e+00 | 20260505 10:06:10.293 | INFO | PerfReport: 20.49%, LockCompute -- 2.08e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260505 10:06:10.293 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.64e-02 / 0.00e+00 | 20260505 10:06:10.424 | INFO | Starting a round of Baz IO | 20260505 10:06:13.698 | INFO | Populating RTMetrics report | 20260505 10:06:13.698 | INFO | RTMetrics report populated | 20260505 10:06:13.698 | INFO | Opening tmp RTMetrics json file | 20260505 10:06:13.718 | INFO | Serializing RTMetrics to json | 20260505 10:06:13.722 | INFO | Serialization done, swapping tmp file now... | 20260505 10:06:13.723 | INFO | RTMetrics done | 20260505 10:06:13.853 | INFO | Finished a round of Baz IO: Wrote 2.426 GiB in 3.429 seconds | 20260505 10:06:13.853 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.33% / 0.01% / 74.66% | 20260505 10:06:13.853 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.64997% / 7.94729e-06% / 6.35783e-05% | 20260505 10:06:13.853 | INFO | Baz IO overall: | 20260505 10:06:13.853 | INFO | BytesWritten : 1376220544359 | 20260505 10:06:13.853 | INFO | NumEvents : 655159493807 | 20260505 10:06:15.430 | INFO | Analyzing chunk frames = [2359296,2359808) | 20260505 10:06:15.430 | INFO | MemInfo: { | 20260505 10:06:15.430 | INFO | "gpu" : " 17 GB", | 20260505 10:06:15.430 | INFO | "huge" : "", | 20260505 10:06:15.430 | INFO | "huge_shared" : " 27 GB", | 20260505 10:06:15.430 | INFO | "malloc" : " 14 GB", | 20260505 10:06:15.430 | INFO | "pinned" : " 29 GB" | 20260505 10:06:15.430 | INFO | } | 20260505 10:06:15.430 | INFO | Perf report for the last 5.13889 seconds: | 20260505 10:06:15.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:06:15.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:15.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:15.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:15.430 | INFO | ANALYSIS : 65.2%, 2.9, 99.8ms, 0 | 20260505 10:06:15.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:15.430 | INFO | RT_METRICS : 46.2%, 1.0, 24.2ms, 0 | 20260505 10:06:15.430 | INFO | PRE_HQ : 62.7%, 1.8, 59.8ms, 0 | 20260505 10:06:15.430 | INFO | BAZWRITER : 66.0%, 4.2, 141.3ms, 0 | 20260505 10:06:15.430 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 10:06:15.430 | INFO | Uploaded 36GiB to GPU in 2.247s (16.0213GiB/s) | 20260505 10:06:15.430 | INFO | Downloaded 38GiB to GPU in 2.35251s (16.153GiB/s) | 20260505 10:06:20.570 | INFO | Analyzing chunk frames = [2359808,2360320) | 20260505 10:06:20.570 | INFO | MemInfo: { | 20260505 10:06:20.570 | INFO | "gpu" : " 17 GB", | 20260505 10:06:20.570 | INFO | "huge" : "", | 20260505 10:06:20.570 | INFO | "huge_shared" : " 27 GB", | 20260505 10:06:20.570 | INFO | "malloc" : " 14 GB", | 20260505 10:06:20.570 | INFO | "pinned" : " 29 GB" | 20260505 10:06:20.570 | INFO | } | 20260505 10:06:20.570 | INFO | Perf report for the last 5.1405 seconds: | 20260505 10:06:20.570 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:06:20.570 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:20.570 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:20.570 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:20.570 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 10:06:20.570 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:20.570 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:20.570 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:06:20.570 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:20.570 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:20.570 | INFO | Uploaded 36GiB to GPU in 2.18002s (16.5136GiB/s) | 20260505 10:06:20.570 | INFO | Downloaded 34GiB to GPU in 2.10187s (16.1761GiB/s) | 20260505 10:06:25.710 | INFO | Analyzing chunk frames = [2360320,2360832) | 20260505 10:06:25.710 | INFO | MemInfo: { | 20260505 10:06:25.710 | INFO | "gpu" : " 17 GB", | 20260505 10:06:25.710 | INFO | "huge" : "", | 20260505 10:06:25.710 | INFO | "huge_shared" : " 27 GB", | 20260505 10:06:25.710 | INFO | "malloc" : " 14 GB", | 20260505 10:06:25.710 | INFO | "pinned" : " 29 GB" | 20260505 10:06:25.710 | INFO | } | 20260505 10:06:25.710 | INFO | Perf report for the last 5.14002 seconds: | 20260505 10:06:25.710 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:06:25.710 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:25.710 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:25.710 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:25.710 | INFO | ANALYSIS : 63.1%, 3.0, 98.1ms, 0 | 20260505 10:06:25.710 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:25.710 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:25.710 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:06:25.711 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:25.711 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:25.711 | INFO | Uploaded 36GiB to GPU in 2.15224s (16.7267GiB/s) | 20260505 10:06:25.711 | INFO | Downloaded 34GiB to GPU in 2.05963s (16.5078GiB/s) | 20260505 10:06:30.848 | INFO | Analyzing chunk frames = [2360832,2361344) | 20260505 10:06:30.848 | INFO | MemInfo: { | 20260505 10:06:30.848 | INFO | "gpu" : " 17 GB", | 20260505 10:06:30.848 | INFO | "huge" : "", | 20260505 10:06:30.848 | INFO | "huge_shared" : " 27 GB", | 20260505 10:06:30.848 | INFO | "malloc" : " 14 GB", | 20260505 10:06:30.848 | INFO | "pinned" : " 29 GB" | 20260505 10:06:30.848 | INFO | } | 20260505 10:06:30.848 | INFO | Perf report for the last 5.13813 seconds: | 20260505 10:06:30.849 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:06:30.849 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:30.849 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:30.849 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:30.849 | INFO | ANALYSIS : 63.1%, 3.0, 98.1ms, 0 | 20260505 10:06:30.849 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:30.849 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:30.849 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 10:06:30.849 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:30.849 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:30.849 | INFO | Uploaded 36GiB to GPU in 2.17443s (16.5561GiB/s) | 20260505 10:06:30.849 | INFO | Downloaded 34GiB to GPU in 2.07601s (16.3776GiB/s) | 20260505 10:06:35.988 | INFO | Analyzing chunk frames = [2361344,2361856) | 20260505 10:06:35.988 | INFO | MemInfo: { | 20260505 10:06:35.988 | INFO | "gpu" : " 17 GB", | 20260505 10:06:35.988 | INFO | "huge" : "", | 20260505 10:06:35.988 | INFO | "huge_shared" : " 27 GB", | 20260505 10:06:35.988 | INFO | "malloc" : " 14 GB", | 20260505 10:06:35.988 | INFO | "pinned" : " 29 GB" | 20260505 10:06:35.988 | INFO | } | 20260505 10:06:35.988 | INFO | Perf report for the last 5.13961 seconds: | 20260505 10:06:35.988 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:06:35.988 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:35.988 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:35.988 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:35.988 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 10:06:35.988 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:35.988 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:35.988 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 10:06:35.988 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:35.988 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:35.988 | INFO | Uploaded 36GiB to GPU in 2.23542s (16.1043GiB/s) | 20260505 10:06:35.989 | INFO | Downloaded 34GiB to GPU in 2.11391s (16.084GiB/s) | 20260505 10:06:41.128 | INFO | Analyzing chunk frames = [2361856,2362368) | 20260505 10:06:41.128 | INFO | MemInfo: { | 20260505 10:06:41.128 | INFO | "gpu" : " 17 GB", | 20260505 10:06:41.128 | INFO | "huge" : "", | 20260505 10:06:41.128 | INFO | "huge_shared" : " 27 GB", | 20260505 10:06:41.128 | INFO | "malloc" : " 14 GB", | 20260505 10:06:41.128 | INFO | "pinned" : " 29 GB" | 20260505 10:06:41.128 | INFO | } | 20260505 10:06:41.128 | INFO | Perf report for the last 5.14033 seconds: | 20260505 10:06:41.129 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:06:41.129 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:41.129 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:41.129 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:41.129 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 10:06:41.129 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:41.129 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:41.129 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 10:06:41.129 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:41.129 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:41.129 | INFO | Uploaded 36GiB to GPU in 2.23602s (16.1001GiB/s) | 20260505 10:06:41.129 | INFO | Downloaded 34GiB to GPU in 2.10064s (16.1856GiB/s) | 20260505 10:06:46.267 | INFO | Analyzing chunk frames = [2362368,2362880) | 20260505 10:06:46.267 | INFO | MemInfo: { | 20260505 10:06:46.267 | INFO | "gpu" : " 17 GB", | 20260505 10:06:46.267 | INFO | "huge" : "", | 20260505 10:06:46.267 | INFO | "huge_shared" : " 27 GB", | 20260505 10:06:46.267 | INFO | "malloc" : " 14 GB", | 20260505 10:06:46.267 | INFO | "pinned" : " 29 GB" | 20260505 10:06:46.267 | INFO | } | 20260505 10:06:46.267 | INFO | Perf report for the last 5.13849 seconds: | 20260505 10:06:46.267 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:06:46.267 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:46.267 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:46.267 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:46.267 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 10:06:46.267 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:46.267 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:46.267 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 10:06:46.267 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:46.267 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:46.267 | INFO | Uploaded 36GiB to GPU in 2.16563s (16.6234GiB/s) | 20260505 10:06:46.267 | INFO | Downloaded 34GiB to GPU in 2.14662s (15.8388GiB/s) | 20260505 10:06:51.406 | INFO | Analyzing chunk frames = [2362880,2363392) | 20260505 10:06:51.406 | INFO | MemInfo: { | 20260505 10:06:51.406 | INFO | "gpu" : " 17 GB", | 20260505 10:06:51.406 | INFO | "huge" : "", | 20260505 10:06:51.406 | INFO | "huge_shared" : " 27 GB", | 20260505 10:06:51.406 | INFO | "malloc" : " 14 GB", | 20260505 10:06:51.406 | INFO | "pinned" : " 29 GB" | 20260505 10:06:51.406 | INFO | } | 20260505 10:06:51.406 | INFO | Perf report for the last 5.13897 seconds: | 20260505 10:06:51.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:06:51.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:51.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:51.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:51.406 | INFO | ANALYSIS : 64.6%, 3.0, 100.9ms, 0 | 20260505 10:06:51.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:51.406 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:51.406 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 10:06:51.406 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:51.406 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:51.406 | INFO | Uploaded 36GiB to GPU in 2.24862s (16.0098GiB/s) | 20260505 10:06:51.406 | INFO | Downloaded 34GiB to GPU in 2.09551s (16.2252GiB/s) | 20260505 10:06:51.559 | INFO | Starting a round of Baz IO | 20260505 10:06:54.812 | INFO | Populating RTMetrics report | 20260505 10:06:54.812 | INFO | RTMetrics report populated | 20260505 10:06:54.812 | INFO | Opening tmp RTMetrics json file | 20260505 10:06:54.813 | INFO | Serializing RTMetrics to json | 20260505 10:06:54.815 | INFO | Serialization done, swapping tmp file now... | 20260505 10:06:54.815 | INFO | RTMetrics done | 20260505 10:06:54.958 | INFO | Finished a round of Baz IO: Wrote 2.425 GiB in 3.398 seconds | 20260505 10:06:54.958 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.32% / 0.01% / 74.67% | 20260505 10:06:54.958 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.64567% / 1.98682e-05% / 9.13938e-05% | 20260505 10:06:54.958 | INFO | Baz IO overall: | 20260505 10:06:54.958 | INFO | BytesWritten : 1378824718090 | 20260505 10:06:54.958 | INFO | NumEvents : 656402133600 | 20260505 10:06:56.641 | INFO | Analyzing chunk frames = [2363392,2363904) | 20260505 10:06:56.641 | INFO | MemInfo: { | 20260505 10:06:56.641 | INFO | "gpu" : " 17 GB", | 20260505 10:06:56.641 | INFO | "huge" : "", | 20260505 10:06:56.641 | INFO | "huge_shared" : " 27 GB", | 20260505 10:06:56.641 | INFO | "malloc" : " 14 GB", | 20260505 10:06:56.641 | INFO | "pinned" : " 29 GB" | 20260505 10:06:56.641 | INFO | } | 20260505 10:06:56.641 | INFO | Perf report for the last 5.23516 seconds: | 20260505 10:06:56.641 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:06:56.641 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:56.641 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:56.641 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:56.641 | INFO | ANALYSIS : 64.0%, 2.9, 99.8ms, 0 | 20260505 10:06:56.641 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:06:56.641 | INFO | RT_METRICS : 44.9%, 1.0, 24.0ms, 0 | 20260505 10:06:56.641 | INFO | PRE_HQ : 61.4%, 1.8, 59.9ms, 0 | 20260505 10:06:56.641 | INFO | BAZWRITER : 64.9%, 4.2, 141.6ms, 0 | 20260505 10:06:56.641 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 10:06:56.641 | INFO | Uploaded 36GiB to GPU in 2.25735s (15.9479GiB/s) | 20260505 10:06:56.641 | INFO | Downloaded 38GiB to GPU in 2.35924s (16.1069GiB/s) | 20260505 10:07:01.787 | INFO | Analyzing chunk frames = [2363904,2364416) | 20260505 10:07:01.787 | INFO | MemInfo: { | 20260505 10:07:01.787 | INFO | "gpu" : " 17 GB", | 20260505 10:07:01.787 | INFO | "huge" : "", | 20260505 10:07:01.787 | INFO | "huge_shared" : " 27 GB", | 20260505 10:07:01.787 | INFO | "malloc" : " 14 GB", | 20260505 10:07:01.787 | INFO | "pinned" : " 29 GB" | 20260505 10:07:01.787 | INFO | } | 20260505 10:07:01.787 | INFO | Perf report for the last 5.14606 seconds: | 20260505 10:07:01.787 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:07:01.787 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:01.787 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:01.787 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:01.787 | INFO | ANALYSIS : 62.5%, 3.0, 97.7ms, 0 | 20260505 10:07:01.787 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:01.787 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:01.787 | INFO | PRE_HQ : 29.5%, 1.0, 15.4ms, 0 | 20260505 10:07:01.787 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:01.787 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:01.787 | INFO | Uploaded 36GiB to GPU in 2.03325s (17.7056GiB/s) | 20260505 10:07:01.787 | INFO | Downloaded 34GiB to GPU in 1.99979s (17.0018GiB/s) | 20260505 10:07:06.926 | INFO | Analyzing chunk frames = [2364416,2364928) | 20260505 10:07:06.926 | INFO | MemInfo: { | 20260505 10:07:06.926 | INFO | "gpu" : " 17 GB", | 20260505 10:07:06.926 | INFO | "huge" : "", | 20260505 10:07:06.926 | INFO | "huge_shared" : " 27 GB", | 20260505 10:07:06.926 | INFO | "malloc" : " 14 GB", | 20260505 10:07:06.926 | INFO | "pinned" : " 29 GB" | 20260505 10:07:06.926 | INFO | } | 20260505 10:07:06.926 | INFO | Perf report for the last 5.13909 seconds: | 20260505 10:07:06.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:07:06.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:06.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:06.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:06.926 | INFO | ANALYSIS : 63.9%, 3.0, 99.4ms, 0 | 20260505 10:07:06.926 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:06.926 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:06.926 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 10:07:06.926 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:06.926 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:06.926 | INFO | Uploaded 36GiB to GPU in 2.22835s (16.1555GiB/s) | 20260505 10:07:06.926 | INFO | Downloaded 34GiB to GPU in 2.12585s (15.9936GiB/s) | 20260505 10:07:12.071 | INFO | Analyzing chunk frames = [2364928,2365440) | 20260505 10:07:12.071 | INFO | MemInfo: { | 20260505 10:07:12.071 | INFO | "gpu" : " 17 GB", | 20260505 10:07:12.071 | INFO | "huge" : "", | 20260505 10:07:12.071 | INFO | "huge_shared" : " 27 GB", | 20260505 10:07:12.071 | INFO | "malloc" : " 14 GB", | 20260505 10:07:12.071 | INFO | "pinned" : " 29 GB" | 20260505 10:07:12.071 | INFO | } | 20260505 10:07:12.071 | INFO | Perf report for the last 5.1446 seconds: | 20260505 10:07:12.071 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:07:12.071 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:12.071 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:12.071 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:12.071 | INFO | ANALYSIS : 63.1%, 3.0, 98.1ms, 0 | 20260505 10:07:12.071 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:12.071 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:12.071 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 10:07:12.071 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:12.071 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:12.071 | INFO | Uploaded 36GiB to GPU in 2.16534s (16.6256GiB/s) | 20260505 10:07:12.071 | INFO | Downloaded 34GiB to GPU in 2.07452s (16.3893GiB/s) | 20260505 10:07:17.204 | INFO | Analyzing chunk frames = [2365440,2365952) | 20260505 10:07:17.204 | INFO | MemInfo: { | 20260505 10:07:17.204 | INFO | "gpu" : " 17 GB", | 20260505 10:07:17.204 | INFO | "huge" : "", | 20260505 10:07:17.204 | INFO | "huge_shared" : " 27 GB", | 20260505 10:07:17.204 | INFO | "malloc" : " 14 GB", | 20260505 10:07:17.204 | INFO | "pinned" : " 29 GB" | 20260505 10:07:17.204 | INFO | } | 20260505 10:07:17.204 | INFO | Perf report for the last 5.13345 seconds: | 20260505 10:07:17.204 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:07:17.204 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:17.204 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:17.204 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:17.204 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 10:07:17.204 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:17.204 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:17.204 | INFO | PRE_HQ : 30.0%, 1.0, 15.5ms, 0 | 20260505 10:07:17.204 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:17.204 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:17.204 | INFO | Uploaded 36GiB to GPU in 2.21981s (16.2176GiB/s) | 20260505 10:07:17.205 | INFO | Downloaded 34GiB to GPU in 2.12197s (16.0228GiB/s) | 20260505 10:07:22.341 | INFO | Analyzing chunk frames = [2365952,2366464) | 20260505 10:07:22.341 | INFO | MemInfo: { | 20260505 10:07:22.341 | INFO | "gpu" : " 17 GB", | 20260505 10:07:22.341 | INFO | "huge" : "", | 20260505 10:07:22.341 | INFO | "huge_shared" : " 27 GB", | 20260505 10:07:22.341 | INFO | "malloc" : " 14 GB", | 20260505 10:07:22.341 | INFO | "pinned" : " 29 GB" | 20260505 10:07:22.341 | INFO | } | 20260505 10:07:22.341 | INFO | Perf report for the last 5.13666 seconds: | 20260505 10:07:22.341 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:07:22.341 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:22.341 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:22.341 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:22.341 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 10:07:22.341 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:22.341 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:22.341 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:07:22.341 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:22.341 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:22.341 | INFO | Uploaded 36GiB to GPU in 2.18989s (16.4392GiB/s) | 20260505 10:07:22.341 | INFO | Downloaded 34GiB to GPU in 2.06345s (16.4772GiB/s) | 20260505 10:07:27.490 | INFO | Analyzing chunk frames = [2366464,2366976) | 20260505 10:07:27.490 | INFO | MemInfo: { | 20260505 10:07:27.490 | INFO | "gpu" : " 17 GB", | 20260505 10:07:27.490 | INFO | "huge" : "", | 20260505 10:07:27.490 | INFO | "huge_shared" : " 27 GB", | 20260505 10:07:27.490 | INFO | "malloc" : " 14 GB", | 20260505 10:07:27.490 | INFO | "pinned" : " 29 GB" | 20260505 10:07:27.490 | INFO | } | 20260505 10:07:27.490 | INFO | Perf report for the last 5.1489 seconds: | 20260505 10:07:27.490 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:07:27.490 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:27.490 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:27.490 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:27.490 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 10:07:27.490 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:27.490 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:27.490 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260505 10:07:27.490 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:27.490 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:27.490 | INFO | Uploaded 36GiB to GPU in 2.19344s (16.4126GiB/s) | 20260505 10:07:27.490 | INFO | Downloaded 34GiB to GPU in 2.11105s (16.1057GiB/s) | 20260505 10:07:32.624 | INFO | Analyzing chunk frames = [2366976,2367488) | 20260505 10:07:32.624 | INFO | MemInfo: { | 20260505 10:07:32.624 | INFO | "gpu" : " 17 GB", | 20260505 10:07:32.624 | INFO | "huge" : "", | 20260505 10:07:32.624 | INFO | "huge_shared" : " 27 GB", | 20260505 10:07:32.624 | INFO | "malloc" : " 14 GB", | 20260505 10:07:32.624 | INFO | "pinned" : " 29 GB" | 20260505 10:07:32.624 | INFO | } | 20260505 10:07:32.624 | INFO | Perf report for the last 5.13408 seconds: | 20260505 10:07:32.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:07:32.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:32.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:32.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:32.624 | INFO | ANALYSIS : 64.5%, 3.0, 100.6ms, 0 | 20260505 10:07:32.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:32.624 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:32.624 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 10:07:32.624 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:32.624 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:32.624 | INFO | Uploaded 36GiB to GPU in 2.24533s (16.0333GiB/s) | 20260505 10:07:32.624 | INFO | Downloaded 34GiB to GPU in 2.0936s (16.24GiB/s) | 20260505 10:07:32.624 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 10:07:32.624 | INFO | Total mem usage: 15.3 GiB | 20260505 10:07:32.624 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 10:07:32.624 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.51%) | 20260505 10:07:32.624 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 10:07:32.624 | INFO | High Water Breakdown: | 20260505 10:07:32.624 | INFO | 61% : 8.78 GiB ( 93.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 10:07:32.624 | INFO | 38.8% : 5.59 GiB ( 84.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 10:07:32.624 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 10:07:32.624 | INFO | ------------------------------------------ | 20260505 10:07:32.624 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:07:32.624 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 10:07:32.624 | INFO | Total mem usage: 26.5 GiB | 20260505 10:07:32.624 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 10:07:32.624 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 10:07:32.624 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 10:07:32.624 | INFO | High Water Breakdown: | 20260505 10:07:32.624 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:07:32.624 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:07:32.624 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:07:32.625 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:07:32.625 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:07:32.625 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:07:32.625 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 10:07:32.625 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 10:07:32.625 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:07:32.625 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 10:07:32.625 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 10:07:32.625 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 10:07:32.625 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 10:07:32.625 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 10:07:32.625 | INFO | ------------------------------------------ | 20260505 10:07:32.625 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:07:32.625 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 10:07:32.625 | INFO | Total mem usage: 25.6 GiB | 20260505 10:07:32.625 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 10:07:32.625 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 10:07:32.625 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 10:07:32.625 | INFO | High Water Breakdown: | 20260505 10:07:32.625 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 10:07:32.625 | INFO | ------------------------------------------ | 20260505 10:07:32.625 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:07:32.625 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 10:07:32.625 | INFO | Total mem usage: 15.7 GiB | 20260505 10:07:32.625 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 10:07:32.625 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 10:07:32.625 | INFO | Unused Cache: 889 MiB (Waste: 5.54%) | 20260505 10:07:32.625 | INFO | High Water Breakdown: | 20260505 10:07:32.625 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 10:07:32.625 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 10:07:32.625 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 10:07:32.625 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 10:07:32.625 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 10:07:32.625 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 10:07:32.625 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 10:07:32.625 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 10:07:32.625 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 10:07:32.625 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:07:32.625 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 10:07:32.625 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:07:32.625 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 10:07:32.625 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 10:07:32.625 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 10:07:32.625 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 10:07:32.625 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:07:32.625 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 10:07:32.625 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:07:32.625 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:07:32.625 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:07:32.625 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 10:07:32.625 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 10:07:32.625 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:07:32.625 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 10:07:32.625 | INFO | ------------------------------------------ | 20260505 10:07:32.625 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:07:32.625 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 10:07:32.625 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:07:32.625 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:07:32.625 | INFO | PerfReport: Full Execution -- 3.24e+01 / 7.70e+01, 1.39e+01 / 2.73e+00 | 20260505 10:07:32.625 | INFO | PerfReport: 0.29%, AggregateStats -- 9.31e-02 / 1.04e-01, 1.07e-02 / 6.07e-03 | 20260505 10:07:32.625 | INFO | PerfReport: 11.65%, Baseline -- 3.78e+00 / 4.45e+00, 6.12e-01 / 2.55e-01 | 20260505 10:07:32.625 | INFO | PerfReport: 3.32%, Histogram -- 1.08e+00 / 1.47e+00, 2.10e-01 / 1.07e-01 | 20260505 10:07:32.625 | INFO | PerfReport: 15.63%, DME -- 4.05e+01 / 4.46e+01, 4.40e+00 / 2.28e+00 | 20260505 10:07:32.625 | INFO | PerfReport: 56.62%, FrameLabeling -- 1.84e+01 / 2.15e+01, 1.84e+00 / 1.99e+00 | 20260505 10:07:32.625 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.39e+00 / 2.89e+00, 4.11e-01 / 2.73e-01 | 20260505 10:07:32.625 | INFO | PerfReport: 3.78%, Metrics -- 1.23e+00 / 1.52e+00, 3.55e-01 / 1.93e-01 | 20260505 10:07:32.625 | INFO | PerfReport: 1.34%, PulseSerializing -- 4.35e-01 / 4.79e-01, 8.24e-02 / 2.88e-02 | 20260505 10:07:32.625 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 10:07:32.625 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:07:32.625 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:07:32.625 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.38e+01 / 0.00e+00 | 20260505 10:07:32.625 | INFO | PerfReport: 22.59%, Upload -- 2.29e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260505 10:07:32.625 | INFO | PerfReport: 21.84%, Download -- 2.21e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260505 10:07:32.625 | INFO | PerfReport: 31.99%, Compute -- 3.24e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 10:07:32.626 | INFO | PerfReport: 1.83%, LockUpload -- 1.85e+00 / 0.00e+00, 7.45e+00 / 0.00e+00 | 20260505 10:07:32.626 | INFO | PerfReport: 1.11%, LockDownload -- 1.13e+00 / 0.00e+00, 2.71e+00 / 0.00e+00 | 20260505 10:07:32.626 | INFO | PerfReport: 20.62%, LockCompute -- 2.09e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260505 10:07:32.626 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 2.10e-02 / 0.00e+00 | 20260505 10:07:32.753 | INFO | Starting a round of Baz IO | 20260505 10:07:36.022 | INFO | Populating RTMetrics report | 20260505 10:07:36.022 | INFO | RTMetrics report populated | 20260505 10:07:36.022 | INFO | Opening tmp RTMetrics json file | 20260505 10:07:36.069 | INFO | Serializing RTMetrics to json | 20260505 10:07:36.102 | INFO | Serialization done, swapping tmp file now... | 20260505 10:07:36.126 | INFO | RTMetrics done | 20260505 10:07:36.156 | INFO | Finished a round of Baz IO: Wrote 2.425 GiB in 3.403 seconds | 20260505 10:07:36.157 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.30% / 0.01% / 74.68% | 20260505 10:07:36.157 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.64175% / 3.97364e-06% / 5.16574e-05% | 20260505 10:07:36.157 | INFO | Baz IO overall: | 20260505 10:07:36.157 | INFO | BytesWritten : 1381428032071 | 20260505 10:07:36.157 | INFO | NumEvents : 657644655657 | 20260505 10:07:37.767 | INFO | Analyzing chunk frames = [2367488,2368000) | 20260505 10:07:37.767 | INFO | MemInfo: { | 20260505 10:07:37.767 | INFO | "gpu" : " 17 GB", | 20260505 10:07:37.767 | INFO | "huge" : "", | 20260505 10:07:37.767 | INFO | "huge_shared" : " 27 GB", | 20260505 10:07:37.767 | INFO | "malloc" : " 14 GB", | 20260505 10:07:37.767 | INFO | "pinned" : " 29 GB" | 20260505 10:07:37.767 | INFO | } | 20260505 10:07:37.767 | INFO | Perf report for the last 5.143 seconds: | 20260505 10:07:37.767 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:07:37.767 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:37.767 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:37.767 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:37.767 | INFO | ANALYSIS : 65.0%, 2.9, 99.9ms, 0 | 20260505 10:07:37.767 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:37.767 | INFO | RT_METRICS : 47.5%, 1.0, 24.9ms, 0 | 20260505 10:07:37.767 | INFO | PRE_HQ : 62.8%, 1.8, 59.6ms, 0 | 20260505 10:07:37.767 | INFO | BAZWRITER : 65.4%, 4.2, 141.2ms, 0 | 20260505 10:07:37.767 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 10:07:37.767 | INFO | Uploaded 36GiB to GPU in 2.26093s (15.9226GiB/s) | 20260505 10:07:37.767 | INFO | Downloaded 38GiB to GPU in 2.35604s (16.1287GiB/s) | 20260505 10:07:42.899 | INFO | Analyzing chunk frames = [2368000,2368512) | 20260505 10:07:42.899 | INFO | MemInfo: { | 20260505 10:07:42.899 | INFO | "gpu" : " 17 GB", | 20260505 10:07:42.899 | INFO | "huge" : "", | 20260505 10:07:42.899 | INFO | "huge_shared" : " 27 GB", | 20260505 10:07:42.899 | INFO | "malloc" : " 14 GB", | 20260505 10:07:42.899 | INFO | "pinned" : " 29 GB" | 20260505 10:07:42.899 | INFO | } | 20260505 10:07:42.899 | INFO | Perf report for the last 5.13247 seconds: | 20260505 10:07:42.899 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:07:42.899 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:42.899 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:42.899 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:42.899 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 10:07:42.899 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:42.899 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:42.899 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:07:42.899 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:42.899 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:42.900 | INFO | Uploaded 36GiB to GPU in 2.20364s (16.3366GiB/s) | 20260505 10:07:42.900 | INFO | Downloaded 34GiB to GPU in 2.10676s (16.1385GiB/s) | 20260505 10:07:48.046 | INFO | Analyzing chunk frames = [2368512,2369024) | 20260505 10:07:48.046 | INFO | MemInfo: { | 20260505 10:07:48.046 | INFO | "gpu" : " 17 GB", | 20260505 10:07:48.046 | INFO | "huge" : "", | 20260505 10:07:48.046 | INFO | "huge_shared" : " 27 GB", | 20260505 10:07:48.046 | INFO | "malloc" : " 14 GB", | 20260505 10:07:48.046 | INFO | "pinned" : " 29 GB" | 20260505 10:07:48.046 | INFO | } | 20260505 10:07:48.046 | INFO | Perf report for the last 5.14683 seconds: | 20260505 10:07:48.046 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:07:48.046 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:48.046 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:48.046 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:48.046 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 10:07:48.046 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:48.046 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:48.047 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:07:48.047 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:48.047 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:48.047 | INFO | Uploaded 36GiB to GPU in 2.16594s (16.6209GiB/s) | 20260505 10:07:48.047 | INFO | Downloaded 34GiB to GPU in 2.07107s (16.4166GiB/s) | 20260505 10:07:53.186 | INFO | Analyzing chunk frames = [2369024,2369536) | 20260505 10:07:53.186 | INFO | MemInfo: { | 20260505 10:07:53.186 | INFO | "gpu" : " 17 GB", | 20260505 10:07:53.186 | INFO | "huge" : "", | 20260505 10:07:53.186 | INFO | "huge_shared" : " 27 GB", | 20260505 10:07:53.186 | INFO | "malloc" : " 14 GB", | 20260505 10:07:53.186 | INFO | "pinned" : " 29 GB" | 20260505 10:07:53.186 | INFO | } | 20260505 10:07:53.186 | INFO | Perf report for the last 5.13954 seconds: | 20260505 10:07:53.186 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:07:53.186 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:53.186 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:53.186 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:53.186 | INFO | ANALYSIS : 63.0%, 3.0, 97.9ms, 0 | 20260505 10:07:53.186 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:53.186 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:53.186 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:07:53.186 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:53.186 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:53.186 | INFO | Uploaded 36GiB to GPU in 2.15284s (16.7221GiB/s) | 20260505 10:07:53.186 | INFO | Downloaded 34GiB to GPU in 2.07376s (16.3954GiB/s) | 20260505 10:07:58.321 | INFO | Analyzing chunk frames = [2369536,2370048) | 20260505 10:07:58.321 | INFO | MemInfo: { | 20260505 10:07:58.321 | INFO | "gpu" : " 17 GB", | 20260505 10:07:58.321 | INFO | "huge" : "", | 20260505 10:07:58.321 | INFO | "huge_shared" : " 27 GB", | 20260505 10:07:58.321 | INFO | "malloc" : " 14 GB", | 20260505 10:07:58.321 | INFO | "pinned" : " 29 GB" | 20260505 10:07:58.321 | INFO | } | 20260505 10:07:58.321 | INFO | Perf report for the last 5.1355 seconds: | 20260505 10:07:58.321 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:07:58.321 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:58.322 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:58.322 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:58.322 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 10:07:58.322 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:58.322 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:58.322 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 10:07:58.322 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:58.322 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:07:58.322 | INFO | Uploaded 36GiB to GPU in 2.1589s (16.6751GiB/s) | 20260505 10:07:58.322 | INFO | Downloaded 34GiB to GPU in 2.12527s (15.998GiB/s) | 20260505 10:08:03.454 | INFO | Analyzing chunk frames = [2370048,2370560) | 20260505 10:08:03.454 | INFO | MemInfo: { | 20260505 10:08:03.454 | INFO | "gpu" : " 17 GB", | 20260505 10:08:03.454 | INFO | "huge" : "", | 20260505 10:08:03.454 | INFO | "huge_shared" : " 27 GB", | 20260505 10:08:03.454 | INFO | "malloc" : " 14 GB", | 20260505 10:08:03.454 | INFO | "pinned" : " 29 GB" | 20260505 10:08:03.454 | INFO | } | 20260505 10:08:03.454 | INFO | Perf report for the last 5.13258 seconds: | 20260505 10:08:03.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:08:03.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:03.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:03.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:03.454 | INFO | ANALYSIS : 64.1%, 3.0, 100.0ms, 0 | 20260505 10:08:03.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:03.454 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:03.454 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 10:08:03.454 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:03.454 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:03.454 | INFO | Uploaded 36GiB to GPU in 2.23393s (16.1151GiB/s) | 20260505 10:08:03.454 | INFO | Downloaded 34GiB to GPU in 2.10587s (16.1454GiB/s) | 20260505 10:08:08.592 | INFO | Analyzing chunk frames = [2370560,2371072) | 20260505 10:08:08.592 | INFO | MemInfo: { | 20260505 10:08:08.592 | INFO | "gpu" : " 17 GB", | 20260505 10:08:08.592 | INFO | "huge" : "", | 20260505 10:08:08.592 | INFO | "huge_shared" : " 27 GB", | 20260505 10:08:08.592 | INFO | "malloc" : " 14 GB", | 20260505 10:08:08.592 | INFO | "pinned" : " 29 GB" | 20260505 10:08:08.593 | INFO | } | 20260505 10:08:08.593 | INFO | Perf report for the last 5.13868 seconds: | 20260505 10:08:08.593 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:08:08.593 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:08.593 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:08.593 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:08.593 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 10:08:08.593 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:08.593 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:08.593 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260505 10:08:08.593 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:08.593 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:08.593 | INFO | Uploaded 36GiB to GPU in 2.21251s (16.2711GiB/s) | 20260505 10:08:08.593 | INFO | Downloaded 34GiB to GPU in 2.15101s (15.8065GiB/s) | 20260505 10:08:13.743 | INFO | Analyzing chunk frames = [2371072,2371584) | 20260505 10:08:13.743 | INFO | MemInfo: { | 20260505 10:08:13.743 | INFO | "gpu" : " 17 GB", | 20260505 10:08:13.743 | INFO | "huge" : "", | 20260505 10:08:13.743 | INFO | "huge_shared" : " 27 GB", | 20260505 10:08:13.743 | INFO | "malloc" : " 14 GB", | 20260505 10:08:13.743 | INFO | "pinned" : " 29 GB" | 20260505 10:08:13.743 | INFO | } | 20260505 10:08:13.743 | INFO | Perf report for the last 5.15037 seconds: | 20260505 10:08:13.743 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:08:13.743 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:13.743 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:13.743 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:13.743 | INFO | ANALYSIS : 64.6%, 3.0, 101.0ms, 0 | 20260505 10:08:13.743 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:13.743 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:13.743 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 10:08:13.743 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:13.743 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:13.743 | INFO | Uploaded 36GiB to GPU in 2.24545s (16.0324GiB/s) | 20260505 10:08:13.743 | INFO | Downloaded 34GiB to GPU in 2.09126s (16.2582GiB/s) | 20260505 10:08:13.886 | INFO | Starting a round of Baz IO | 20260505 10:08:17.146 | INFO | Populating RTMetrics report | 20260505 10:08:17.146 | INFO | RTMetrics report populated | 20260505 10:08:17.146 | INFO | Opening tmp RTMetrics json file | 20260505 10:08:17.163 | INFO | Serializing RTMetrics to json | 20260505 10:08:17.188 | INFO | Serialization done, swapping tmp file now... | 20260505 10:08:17.190 | INFO | RTMetrics done | 20260505 10:08:17.296 | INFO | Finished a round of Baz IO: Wrote 2.423 GiB in 3.409 seconds | 20260505 10:08:17.296 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.29% / 0.01% / 74.70% | 20260505 10:08:17.296 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.63456% / 1.58946e-05% / 5.96046e-05% | 20260505 10:08:17.296 | INFO | Baz IO overall: | 20260505 10:08:17.296 | INFO | BytesWritten : 1384029843806 | 20260505 10:08:17.296 | INFO | NumEvents : 658886379199 | 20260505 10:08:18.884 | INFO | Analyzing chunk frames = [2371584,2372096) | 20260505 10:08:18.884 | INFO | MemInfo: { | 20260505 10:08:18.884 | INFO | "gpu" : " 17 GB", | 20260505 10:08:18.884 | INFO | "huge" : "", | 20260505 10:08:18.884 | INFO | "huge_shared" : " 27 GB", | 20260505 10:08:18.884 | INFO | "malloc" : " 14 GB", | 20260505 10:08:18.884 | INFO | "pinned" : " 29 GB" | 20260505 10:08:18.885 | INFO | } | 20260505 10:08:18.885 | INFO | Perf report for the last 5.14158 seconds: | 20260505 10:08:18.885 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:08:18.885 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:18.885 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:18.885 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:18.885 | INFO | ANALYSIS : 65.1%, 2.9, 99.6ms, 0 | 20260505 10:08:18.885 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:18.885 | INFO | RT_METRICS : 46.5%, 1.0, 24.4ms, 0 | 20260505 10:08:18.885 | INFO | PRE_HQ : 61.8%, 1.9, 60.0ms, 0 | 20260505 10:08:18.885 | INFO | BAZWRITER : 65.7%, 4.2, 141.8ms, 0 | 20260505 10:08:18.885 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 10:08:18.885 | INFO | Uploaded 36GiB to GPU in 2.2341s (16.1139GiB/s) | 20260505 10:08:18.885 | INFO | Downloaded 38GiB to GPU in 2.29604s (16.5503GiB/s) | 20260505 10:08:24.019 | INFO | Analyzing chunk frames = [2372096,2372608) | 20260505 10:08:24.019 | INFO | MemInfo: { | 20260505 10:08:24.019 | INFO | "gpu" : " 17 GB", | 20260505 10:08:24.019 | INFO | "huge" : "", | 20260505 10:08:24.019 | INFO | "huge_shared" : " 27 GB", | 20260505 10:08:24.019 | INFO | "malloc" : " 14 GB", | 20260505 10:08:24.019 | INFO | "pinned" : " 29 GB" | 20260505 10:08:24.019 | INFO | } | 20260505 10:08:24.019 | INFO | Perf report for the last 5.13412 seconds: | 20260505 10:08:24.019 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:08:24.019 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:24.019 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:24.019 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:24.019 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 10:08:24.019 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:24.019 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:24.019 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:08:24.019 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:24.019 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:24.019 | INFO | Uploaded 36GiB to GPU in 2.17546s (16.5482GiB/s) | 20260505 10:08:24.019 | INFO | Downloaded 34GiB to GPU in 2.08736s (16.2885GiB/s) | 20260505 10:08:29.150 | INFO | Analyzing chunk frames = [2372608,2373120) | 20260505 10:08:29.150 | INFO | MemInfo: { | 20260505 10:08:29.150 | INFO | "gpu" : " 17 GB", | 20260505 10:08:29.150 | INFO | "huge" : "", | 20260505 10:08:29.150 | INFO | "huge_shared" : " 27 GB", | 20260505 10:08:29.150 | INFO | "malloc" : " 14 GB", | 20260505 10:08:29.150 | INFO | "pinned" : " 29 GB" | 20260505 10:08:29.151 | INFO | } | 20260505 10:08:29.151 | INFO | Perf report for the last 5.13188 seconds: | 20260505 10:08:29.151 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:08:29.151 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:29.151 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:29.151 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:29.151 | INFO | ANALYSIS : 63.4%, 3.0, 98.5ms, 0 | 20260505 10:08:29.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:29.151 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:29.151 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:08:29.151 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:29.151 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:29.151 | INFO | Uploaded 36GiB to GPU in 2.16663s (16.6157GiB/s) | 20260505 10:08:29.151 | INFO | Downloaded 34GiB to GPU in 2.07015s (16.4239GiB/s) | 20260505 10:08:34.294 | INFO | Analyzing chunk frames = [2373120,2373632) | 20260505 10:08:34.294 | INFO | MemInfo: { | 20260505 10:08:34.294 | INFO | "gpu" : " 17 GB", | 20260505 10:08:34.294 | INFO | "huge" : "", | 20260505 10:08:34.294 | INFO | "huge_shared" : " 27 GB", | 20260505 10:08:34.294 | INFO | "malloc" : " 14 GB", | 20260505 10:08:34.294 | INFO | "pinned" : " 29 GB" | 20260505 10:08:34.295 | INFO | } | 20260505 10:08:34.295 | INFO | Perf report for the last 5.14402 seconds: | 20260505 10:08:34.295 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:08:34.295 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:34.295 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:34.295 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:34.295 | INFO | ANALYSIS : 63.0%, 3.0, 98.0ms, 0 | 20260505 10:08:34.295 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:34.295 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:34.295 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:08:34.295 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:34.295 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:34.295 | INFO | Uploaded 36GiB to GPU in 2.15181s (16.7301GiB/s) | 20260505 10:08:34.295 | INFO | Downloaded 34GiB to GPU in 2.07115s (16.416GiB/s) | 20260505 10:08:39.436 | INFO | Analyzing chunk frames = [2373632,2374144) | 20260505 10:08:39.436 | INFO | MemInfo: { | 20260505 10:08:39.436 | INFO | "gpu" : " 17 GB", | 20260505 10:08:39.436 | INFO | "huge" : "", | 20260505 10:08:39.436 | INFO | "huge_shared" : " 27 GB", | 20260505 10:08:39.436 | INFO | "malloc" : " 14 GB", | 20260505 10:08:39.436 | INFO | "pinned" : " 29 GB" | 20260505 10:08:39.436 | INFO | } | 20260505 10:08:39.436 | INFO | Perf report for the last 5.14135 seconds: | 20260505 10:08:39.436 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:08:39.436 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:39.436 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:39.436 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:39.436 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 10:08:39.436 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:39.436 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:39.436 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 10:08:39.436 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:39.436 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:39.436 | INFO | Uploaded 36GiB to GPU in 2.16184s (16.6525GiB/s) | 20260505 10:08:39.436 | INFO | Downloaded 34GiB to GPU in 2.12752s (15.981GiB/s) | 20260505 10:08:44.577 | INFO | Analyzing chunk frames = [2374144,2374656) | 20260505 10:08:44.577 | INFO | MemInfo: { | 20260505 10:08:44.577 | INFO | "gpu" : " 17 GB", | 20260505 10:08:44.577 | INFO | "huge" : "", | 20260505 10:08:44.577 | INFO | "huge_shared" : " 27 GB", | 20260505 10:08:44.577 | INFO | "malloc" : " 14 GB", | 20260505 10:08:44.577 | INFO | "pinned" : " 29 GB" | 20260505 10:08:44.577 | INFO | } | 20260505 10:08:44.577 | INFO | Perf report for the last 5.14096 seconds: | 20260505 10:08:44.577 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:08:44.577 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:44.577 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:44.577 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:44.577 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 10:08:44.577 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:44.577 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:44.577 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:08:44.577 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:44.577 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:44.577 | INFO | Uploaded 36GiB to GPU in 2.21968s (16.2185GiB/s) | 20260505 10:08:44.577 | INFO | Downloaded 34GiB to GPU in 2.09923s (16.1965GiB/s) | 20260505 10:08:49.711 | INFO | Analyzing chunk frames = [2374656,2375168) | 20260505 10:08:49.711 | INFO | MemInfo: { | 20260505 10:08:49.711 | INFO | "gpu" : " 17 GB", | 20260505 10:08:49.711 | INFO | "huge" : "", | 20260505 10:08:49.711 | INFO | "huge_shared" : " 27 GB", | 20260505 10:08:49.711 | INFO | "malloc" : " 14 GB", | 20260505 10:08:49.711 | INFO | "pinned" : " 29 GB" | 20260505 10:08:49.711 | INFO | } | 20260505 10:08:49.711 | INFO | Perf report for the last 5.13411 seconds: | 20260505 10:08:49.711 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:08:49.711 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:49.711 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:49.711 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:49.711 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 10:08:49.711 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:49.711 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:49.711 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 10:08:49.711 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:49.711 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:49.711 | INFO | Uploaded 36GiB to GPU in 2.18416s (16.4823GiB/s) | 20260505 10:08:49.711 | INFO | Downloaded 34GiB to GPU in 2.13377s (15.9342GiB/s) | 20260505 10:08:54.851 | INFO | Analyzing chunk frames = [2375168,2375680) | 20260505 10:08:54.851 | INFO | MemInfo: { | 20260505 10:08:54.851 | INFO | "gpu" : " 17 GB", | 20260505 10:08:54.851 | INFO | "huge" : "", | 20260505 10:08:54.851 | INFO | "huge_shared" : " 27 GB", | 20260505 10:08:54.851 | INFO | "malloc" : " 14 GB", | 20260505 10:08:54.851 | INFO | "pinned" : " 29 GB" | 20260505 10:08:54.851 | INFO | } | 20260505 10:08:54.851 | INFO | Perf report for the last 5.14029 seconds: | 20260505 10:08:54.852 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:08:54.852 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:54.852 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:54.852 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:54.852 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 10:08:54.852 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:54.852 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:54.852 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 10:08:54.852 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:54.852 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:54.852 | INFO | Uploaded 36GiB to GPU in 2.17581s (16.5456GiB/s) | 20260505 10:08:54.852 | INFO | Downloaded 34GiB to GPU in 2.05396s (16.5534GiB/s) | 20260505 10:08:54.852 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 10:08:54.852 | INFO | Total mem usage: 15.3 GiB | 20260505 10:08:54.852 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 10:08:54.852 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -5.75%) | 20260505 10:08:54.852 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 10:08:54.852 | INFO | High Water Breakdown: | 20260505 10:08:54.852 | INFO | 61.1% : 8.86 GiB ( 94.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 10:08:54.852 | INFO | 38.7% : 5.62 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 10:08:54.852 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 10:08:54.852 | INFO | ------------------------------------------ | 20260505 10:08:54.852 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:08:54.852 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 10:08:54.852 | INFO | Total mem usage: 26.5 GiB | 20260505 10:08:54.852 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 10:08:54.852 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 10:08:54.852 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 10:08:54.852 | INFO | High Water Breakdown: | 20260505 10:08:54.852 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:08:54.852 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:08:54.853 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:08:54.853 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:08:54.853 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:08:54.853 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:08:54.853 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 10:08:54.853 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 10:08:54.853 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:08:54.853 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 10:08:54.853 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 10:08:54.853 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 10:08:54.853 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 10:08:54.853 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 10:08:54.853 | INFO | ------------------------------------------ | 20260505 10:08:54.853 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:08:54.853 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 10:08:54.853 | INFO | Total mem usage: 25.6 GiB | 20260505 10:08:54.853 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 10:08:54.853 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 10:08:54.853 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 10:08:54.853 | INFO | High Water Breakdown: | 20260505 10:08:54.853 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 10:08:54.853 | INFO | ------------------------------------------ | 20260505 10:08:54.853 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:08:54.853 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 10:08:54.853 | INFO | Total mem usage: 15.7 GiB | 20260505 10:08:54.853 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 10:08:54.853 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 10:08:54.853 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 10:08:54.853 | INFO | High Water Breakdown: | 20260505 10:08:54.853 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 10:08:54.853 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 10:08:54.853 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 10:08:54.853 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 10:08:54.853 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 10:08:54.853 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 10:08:54.853 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 10:08:54.853 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 10:08:54.853 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 10:08:54.853 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:08:54.853 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 10:08:54.853 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:08:54.853 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 10:08:54.853 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 10:08:54.853 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 10:08:54.853 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 10:08:54.853 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:08:54.853 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 10:08:54.853 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:08:54.853 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:08:54.853 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:08:54.853 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 10:08:54.853 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 10:08:54.853 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:08:54.853 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 10:08:54.853 | INFO | ------------------------------------------ | 20260505 10:08:54.853 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:08:54.853 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 10:08:54.853 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:08:54.853 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:08:54.853 | INFO | PerfReport: Full Execution -- 3.24e+01 / 7.73e+01, 1.39e+01 / 2.22e+00 | 20260505 10:08:54.853 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 1.01e-01, 1.03e-02 / 3.80e-03 | 20260505 10:08:54.853 | INFO | PerfReport: 11.65%, Baseline -- 3.77e+00 / 4.45e+00, 6.04e-01 / 2.17e-01 | 20260505 10:08:54.853 | INFO | PerfReport: 3.30%, Histogram -- 1.07e+00 / 1.48e+00, 2.05e-01 / 1.02e-01 | 20260505 10:08:54.853 | INFO | PerfReport: 15.81%, DME -- 4.10e+01 / 4.59e+01, 4.51e+00 / 2.15e+00 | 20260505 10:08:54.853 | INFO | PerfReport: 56.44%, FrameLabeling -- 1.83e+01 / 2.04e+01, 1.79e+00 / 1.93e+00 | 20260505 10:08:54.853 | INFO | PerfReport: 7.41%, PulseAccumulating -- 2.40e+00 / 2.99e+00, 4.23e-01 / 1.98e-01 | 20260505 10:08:54.853 | INFO | PerfReport: 3.76%, Metrics -- 1.22e+00 / 1.54e+00, 3.46e-01 / 1.84e-01 | 20260505 10:08:54.853 | INFO | PerfReport: 1.34%, PulseSerializing -- 4.33e-01 / 4.85e-01, 8.09e-02 / 2.76e-02 | 20260505 10:08:54.853 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 10:08:54.853 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:08:54.853 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:08:54.853 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.42e+01 / 0.00e+00 | 20260505 10:08:54.853 | INFO | PerfReport: 22.53%, Upload -- 2.29e+01 / 0.00e+00, 8.17e+00 / 0.00e+00 | 20260505 10:08:54.853 | INFO | PerfReport: 21.83%, Download -- 2.21e+01 / 0.00e+00, 8.18e+00 / 0.00e+00 | 20260505 10:08:54.853 | INFO | PerfReport: 31.93%, Compute -- 3.24e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 10:08:54.853 | INFO | PerfReport: 1.88%, LockUpload -- 1.91e+00 / 0.00e+00, 7.49e+00 / 0.00e+00 | 20260505 10:08:54.853 | INFO | PerfReport: 1.18%, LockDownload -- 1.20e+00 / 0.00e+00, 2.72e+00 / 0.00e+00 | 20260505 10:08:54.853 | INFO | PerfReport: 20.64%, LockCompute -- 2.09e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260505 10:08:54.853 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.65e-02 / 0.00e+00 | 20260505 10:08:54.986 | INFO | Starting a round of Baz IO | 20260505 10:08:58.257 | INFO | Populating RTMetrics report | 20260505 10:08:58.258 | INFO | RTMetrics report populated | 20260505 10:08:58.258 | INFO | Opening tmp RTMetrics json file | 20260505 10:08:58.285 | INFO | Serializing RTMetrics to json | 20260505 10:08:58.309 | INFO | Serialization done, swapping tmp file now... | 20260505 10:08:58.325 | INFO | RTMetrics done | 20260505 10:08:58.407 | INFO | Finished a round of Baz IO: Wrote 2.421 GiB in 3.42 seconds | 20260505 10:08:58.407 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.28% / 0.01% / 74.71% | 20260505 10:08:58.407 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.62657% / 3.97364e-06% / 6.35783e-05% | 20260505 10:08:58.407 | INFO | Baz IO overall: | 20260505 10:08:58.407 | INFO | BytesWritten : 1386629742457 | 20260505 10:08:58.407 | INFO | NumEvents : 660126629230 | 20260505 10:08:59.998 | INFO | Analyzing chunk frames = [2375680,2376192) | 20260505 10:08:59.998 | INFO | MemInfo: { | 20260505 10:08:59.998 | INFO | "gpu" : " 17 GB", | 20260505 10:08:59.998 | INFO | "huge" : "", | 20260505 10:08:59.998 | INFO | "huge_shared" : " 27 GB", | 20260505 10:08:59.998 | INFO | "malloc" : " 14 GB", | 20260505 10:08:59.998 | INFO | "pinned" : " 29 GB" | 20260505 10:08:59.998 | INFO | } | 20260505 10:08:59.998 | INFO | Perf report for the last 5.14681 seconds: | 20260505 10:08:59.998 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:08:59.998 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:59.998 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:59.998 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:59.998 | INFO | ANALYSIS : 65.0%, 2.9, 100.0ms, 0 | 20260505 10:08:59.998 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:08:59.998 | INFO | RT_METRICS : 46.8%, 1.0, 24.6ms, 0 | 20260505 10:08:59.998 | INFO | PRE_HQ : 62.3%, 1.8, 59.5ms, 0 | 20260505 10:08:59.998 | INFO | BAZWRITER : 65.8%, 4.2, 140.5ms, 0 | 20260505 10:08:59.999 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 10:08:59.999 | INFO | Uploaded 36GiB to GPU in 2.25672s (15.9524GiB/s) | 20260505 10:08:59.999 | INFO | Downloaded 38GiB to GPU in 2.35645s (16.1259GiB/s) | 20260505 10:09:05.134 | INFO | Analyzing chunk frames = [2376192,2376704) | 20260505 10:09:05.134 | INFO | MemInfo: { | 20260505 10:09:05.134 | INFO | "gpu" : " 17 GB", | 20260505 10:09:05.134 | INFO | "huge" : "", | 20260505 10:09:05.134 | INFO | "huge_shared" : " 27 GB", | 20260505 10:09:05.134 | INFO | "malloc" : " 14 GB", | 20260505 10:09:05.134 | INFO | "pinned" : " 29 GB" | 20260505 10:09:05.134 | INFO | } | 20260505 10:09:05.134 | INFO | Perf report for the last 5.13599 seconds: | 20260505 10:09:05.134 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:09:05.134 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:05.134 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:05.134 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:05.134 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 10:09:05.134 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:05.134 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:05.134 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 10:09:05.134 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:05.134 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:05.134 | INFO | Uploaded 36GiB to GPU in 2.23778s (16.0873GiB/s) | 20260505 10:09:05.135 | INFO | Downloaded 34GiB to GPU in 2.08343s (16.3192GiB/s) | 20260505 10:09:10.270 | INFO | Analyzing chunk frames = [2376704,2377216) | 20260505 10:09:10.270 | INFO | MemInfo: { | 20260505 10:09:10.270 | INFO | "gpu" : " 17 GB", | 20260505 10:09:10.270 | INFO | "huge" : "", | 20260505 10:09:10.270 | INFO | "huge_shared" : " 27 GB", | 20260505 10:09:10.270 | INFO | "malloc" : " 14 GB", | 20260505 10:09:10.270 | INFO | "pinned" : " 29 GB" | 20260505 10:09:10.270 | INFO | } | 20260505 10:09:10.270 | INFO | Perf report for the last 5.1355 seconds: | 20260505 10:09:10.270 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:09:10.270 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:10.270 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:10.270 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:10.270 | INFO | ANALYSIS : 63.0%, 3.0, 98.0ms, 0 | 20260505 10:09:10.270 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:10.270 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:10.270 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:09:10.270 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:10.270 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:10.270 | INFO | Uploaded 36GiB to GPU in 2.14963s (16.7471GiB/s) | 20260505 10:09:10.270 | INFO | Downloaded 34GiB to GPU in 2.05901s (16.5128GiB/s) | 20260505 10:09:15.404 | INFO | Analyzing chunk frames = [2377216,2377728) | 20260505 10:09:15.404 | INFO | MemInfo: { | 20260505 10:09:15.404 | INFO | "gpu" : " 17 GB", | 20260505 10:09:15.404 | INFO | "huge" : "", | 20260505 10:09:15.404 | INFO | "huge_shared" : " 27 GB", | 20260505 10:09:15.404 | INFO | "malloc" : " 14 GB", | 20260505 10:09:15.404 | INFO | "pinned" : " 29 GB" | 20260505 10:09:15.404 | INFO | } | 20260505 10:09:15.405 | INFO | Perf report for the last 5.13481 seconds: | 20260505 10:09:15.405 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:09:15.405 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:15.405 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:15.405 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:15.405 | INFO | ANALYSIS : 63.2%, 3.0, 98.1ms, 0 | 20260505 10:09:15.405 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:15.405 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:15.405 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 10:09:15.405 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:15.405 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:15.405 | INFO | Uploaded 36GiB to GPU in 2.16473s (16.6303GiB/s) | 20260505 10:09:15.405 | INFO | Downloaded 34GiB to GPU in 2.07487s (16.3866GiB/s) | 20260505 10:09:20.546 | INFO | Analyzing chunk frames = [2377728,2378240) | 20260505 10:09:20.546 | INFO | MemInfo: { | 20260505 10:09:20.546 | INFO | "gpu" : " 17 GB", | 20260505 10:09:20.546 | INFO | "huge" : "", | 20260505 10:09:20.546 | INFO | "huge_shared" : " 27 GB", | 20260505 10:09:20.546 | INFO | "malloc" : " 14 GB", | 20260505 10:09:20.546 | INFO | "pinned" : " 29 GB" | 20260505 10:09:20.546 | INFO | } | 20260505 10:09:20.547 | INFO | Perf report for the last 5.14209 seconds: | 20260505 10:09:20.547 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:09:20.547 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:20.547 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:20.547 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:20.547 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 10:09:20.547 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:20.547 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:20.547 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:09:20.547 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:20.547 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:20.547 | INFO | Uploaded 36GiB to GPU in 2.17068s (16.5846GiB/s) | 20260505 10:09:20.547 | INFO | Downloaded 34GiB to GPU in 2.05967s (16.5075GiB/s) | 20260505 10:09:25.690 | INFO | Analyzing chunk frames = [2378240,2378752) | 20260505 10:09:25.690 | INFO | MemInfo: { | 20260505 10:09:25.690 | INFO | "gpu" : " 17 GB", | 20260505 10:09:25.690 | INFO | "huge" : "", | 20260505 10:09:25.690 | INFO | "huge_shared" : " 27 GB", | 20260505 10:09:25.690 | INFO | "malloc" : " 14 GB", | 20260505 10:09:25.690 | INFO | "pinned" : " 29 GB" | 20260505 10:09:25.690 | INFO | } | 20260505 10:09:25.690 | INFO | Perf report for the last 5.14382 seconds: | 20260505 10:09:25.690 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:09:25.690 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:25.690 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:25.690 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:25.691 | INFO | ANALYSIS : 63.5%, 3.0, 99.3ms, 0 | 20260505 10:09:25.691 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:25.691 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:25.691 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:09:25.691 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:25.691 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:25.691 | INFO | Uploaded 36GiB to GPU in 2.22284s (16.1955GiB/s) | 20260505 10:09:25.691 | INFO | Downloaded 34GiB to GPU in 2.1011s (16.182GiB/s) | 20260505 10:09:30.839 | INFO | Analyzing chunk frames = [2378752,2379264) | 20260505 10:09:30.839 | INFO | MemInfo: { | 20260505 10:09:30.839 | INFO | "gpu" : " 17 GB", | 20260505 10:09:30.839 | INFO | "huge" : "", | 20260505 10:09:30.839 | INFO | "huge_shared" : " 27 GB", | 20260505 10:09:30.839 | INFO | "malloc" : " 14 GB", | 20260505 10:09:30.839 | INFO | "pinned" : " 29 GB" | 20260505 10:09:30.839 | INFO | } | 20260505 10:09:30.839 | INFO | Perf report for the last 5.14834 seconds: | 20260505 10:09:30.839 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:09:30.839 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:30.839 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:30.839 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:30.839 | INFO | ANALYSIS : 63.7%, 3.0, 99.7ms, 0 | 20260505 10:09:30.839 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:30.839 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:30.839 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260505 10:09:30.839 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:30.839 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:30.839 | INFO | Uploaded 36GiB to GPU in 2.18814s (16.4523GiB/s) | 20260505 10:09:30.839 | INFO | Downloaded 34GiB to GPU in 2.11149s (16.1024GiB/s) | 20260505 10:09:35.971 | INFO | Analyzing chunk frames = [2379264,2379776) | 20260505 10:09:35.971 | INFO | MemInfo: { | 20260505 10:09:35.971 | INFO | "gpu" : " 17 GB", | 20260505 10:09:35.971 | INFO | "huge" : "", | 20260505 10:09:35.971 | INFO | "huge_shared" : " 27 GB", | 20260505 10:09:35.971 | INFO | "malloc" : " 14 GB", | 20260505 10:09:35.971 | INFO | "pinned" : " 29 GB" | 20260505 10:09:35.971 | INFO | } | 20260505 10:09:35.971 | INFO | Perf report for the last 5.13222 seconds: | 20260505 10:09:35.971 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:09:35.971 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:35.971 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:35.971 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:35.971 | INFO | ANALYSIS : 64.2%, 3.0, 100.1ms, 0 | 20260505 10:09:35.971 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:35.971 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:35.971 | INFO | PRE_HQ : 29.8%, 1.0, 15.4ms, 0 | 20260505 10:09:35.971 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:35.971 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:35.971 | INFO | Uploaded 36GiB to GPU in 2.22109s (16.2083GiB/s) | 20260505 10:09:35.971 | INFO | Downloaded 34GiB to GPU in 2.09014s (16.2669GiB/s) | 20260505 10:09:36.101 | INFO | Starting a round of Baz IO | 20260505 10:09:39.361 | INFO | Populating RTMetrics report | 20260505 10:09:39.362 | INFO | RTMetrics report populated | 20260505 10:09:39.362 | INFO | Opening tmp RTMetrics json file | 20260505 10:09:39.378 | INFO | Serializing RTMetrics to json | 20260505 10:09:39.426 | INFO | Serialization done, swapping tmp file now... | 20260505 10:09:39.428 | INFO | RTMetrics done | 20260505 10:09:39.495 | INFO | Finished a round of Baz IO: Wrote 2.422 GiB in 3.393 seconds | 20260505 10:09:39.495 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.27% / 0.01% / 74.72% | 20260505 10:09:39.495 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.62458% / 7.94729e-06% / 5.96046e-05% | 20260505 10:09:39.495 | INFO | Baz IO overall: | 20260505 10:09:39.495 | INFO | BytesWritten : 1389230280055 | 20260505 10:09:39.495 | INFO | NumEvents : 661367511826 | 20260505 10:09:41.102 | INFO | Analyzing chunk frames = [2379776,2380288) | 20260505 10:09:41.102 | INFO | MemInfo: { | 20260505 10:09:41.102 | INFO | "gpu" : " 17 GB", | 20260505 10:09:41.102 | INFO | "huge" : "", | 20260505 10:09:41.102 | INFO | "huge_shared" : " 27 GB", | 20260505 10:09:41.102 | INFO | "malloc" : " 14 GB", | 20260505 10:09:41.102 | INFO | "pinned" : " 29 GB" | 20260505 10:09:41.102 | INFO | } | 20260505 10:09:41.102 | INFO | Perf report for the last 5.13056 seconds: | 20260505 10:09:41.102 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:09:41.102 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:41.102 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:41.102 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:41.102 | INFO | ANALYSIS : 64.9%, 2.9, 99.7ms, 0 | 20260505 10:09:41.102 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:41.102 | INFO | RT_METRICS : 47.0%, 1.0, 24.6ms, 0 | 20260505 10:09:41.102 | INFO | PRE_HQ : 62.9%, 1.8, 59.7ms, 0 | 20260505 10:09:41.102 | INFO | BAZWRITER : 65.5%, 4.2, 139.9ms, 0 | 20260505 10:09:41.102 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 10:09:41.102 | INFO | Uploaded 36GiB to GPU in 2.26247s (15.9118GiB/s) | 20260505 10:09:41.102 | INFO | Downloaded 38GiB to GPU in 2.37552s (15.9965GiB/s) | 20260505 10:09:46.245 | INFO | Analyzing chunk frames = [2380288,2380800) | 20260505 10:09:46.245 | INFO | MemInfo: { | 20260505 10:09:46.245 | INFO | "gpu" : " 17 GB", | 20260505 10:09:46.245 | INFO | "huge" : "", | 20260505 10:09:46.245 | INFO | "huge_shared" : " 27 GB", | 20260505 10:09:46.245 | INFO | "malloc" : " 14 GB", | 20260505 10:09:46.245 | INFO | "pinned" : " 29 GB" | 20260505 10:09:46.245 | INFO | } | 20260505 10:09:46.245 | INFO | Perf report for the last 5.14324 seconds: | 20260505 10:09:46.245 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:09:46.245 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:46.245 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:46.245 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:46.245 | INFO | ANALYSIS : 64.4%, 3.0, 100.6ms, 0 | 20260505 10:09:46.245 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:46.245 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:46.245 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:09:46.245 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:46.245 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:46.245 | INFO | Uploaded 36GiB to GPU in 2.23172s (16.131GiB/s) | 20260505 10:09:46.245 | INFO | Downloaded 34GiB to GPU in 2.10306s (16.1669GiB/s) | 20260505 10:09:51.386 | INFO | Analyzing chunk frames = [2380800,2381312) | 20260505 10:09:51.386 | INFO | MemInfo: { | 20260505 10:09:51.386 | INFO | "gpu" : " 17 GB", | 20260505 10:09:51.386 | INFO | "huge" : "", | 20260505 10:09:51.386 | INFO | "huge_shared" : " 27 GB", | 20260505 10:09:51.386 | INFO | "malloc" : " 14 GB", | 20260505 10:09:51.387 | INFO | "pinned" : " 29 GB" | 20260505 10:09:51.387 | INFO | } | 20260505 10:09:51.387 | INFO | Perf report for the last 5.14181 seconds: | 20260505 10:09:51.387 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:09:51.387 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:51.387 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:51.387 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:51.387 | INFO | ANALYSIS : 63.1%, 3.0, 98.3ms, 0 | 20260505 10:09:51.387 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:51.387 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:51.387 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:09:51.387 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:51.387 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:51.387 | INFO | Uploaded 36GiB to GPU in 2.14853s (16.7556GiB/s) | 20260505 10:09:51.387 | INFO | Downloaded 34GiB to GPU in 2.0679s (16.4418GiB/s) | 20260505 10:09:56.525 | INFO | Analyzing chunk frames = [2381312,2381824) | 20260505 10:09:56.525 | INFO | MemInfo: { | 20260505 10:09:56.525 | INFO | "gpu" : " 17 GB", | 20260505 10:09:56.525 | INFO | "huge" : "", | 20260505 10:09:56.525 | INFO | "huge_shared" : " 27 GB", | 20260505 10:09:56.525 | INFO | "malloc" : " 14 GB", | 20260505 10:09:56.525 | INFO | "pinned" : " 29 GB" | 20260505 10:09:56.525 | INFO | } | 20260505 10:09:56.525 | INFO | Perf report for the last 5.13839 seconds: | 20260505 10:09:56.525 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:09:56.525 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:56.525 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:56.525 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:56.525 | INFO | ANALYSIS : 63.1%, 3.0, 98.1ms, 0 | 20260505 10:09:56.525 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:56.525 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:56.525 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:09:56.525 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:56.525 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:09:56.525 | INFO | Uploaded 36GiB to GPU in 2.16764s (16.6079GiB/s) | 20260505 10:09:56.526 | INFO | Downloaded 34GiB to GPU in 2.07737s (16.3669GiB/s) | 20260505 10:10:01.659 | INFO | Analyzing chunk frames = [2381824,2382336) | 20260505 10:10:01.659 | INFO | MemInfo: { | 20260505 10:10:01.659 | INFO | "gpu" : " 17 GB", | 20260505 10:10:01.659 | INFO | "huge" : "", | 20260505 10:10:01.659 | INFO | "huge_shared" : " 27 GB", | 20260505 10:10:01.659 | INFO | "malloc" : " 14 GB", | 20260505 10:10:01.659 | INFO | "pinned" : " 29 GB" | 20260505 10:10:01.659 | INFO | } | 20260505 10:10:01.659 | INFO | Perf report for the last 5.13433 seconds: | 20260505 10:10:01.660 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:10:01.660 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:01.660 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:01.660 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:01.660 | INFO | ANALYSIS : 63.8%, 3.0, 99.5ms, 0 | 20260505 10:10:01.660 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:01.660 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:01.660 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260505 10:10:01.660 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:01.660 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:01.660 | INFO | Uploaded 36GiB to GPU in 2.23481s (16.1087GiB/s) | 20260505 10:10:01.660 | INFO | Downloaded 34GiB to GPU in 2.11169s (16.1008GiB/s) | 20260505 10:10:06.808 | INFO | Analyzing chunk frames = [2382336,2382848) | 20260505 10:10:06.808 | INFO | MemInfo: { | 20260505 10:10:06.808 | INFO | "gpu" : " 17 GB", | 20260505 10:10:06.808 | INFO | "huge" : "", | 20260505 10:10:06.808 | INFO | "huge_shared" : " 27 GB", | 20260505 10:10:06.808 | INFO | "malloc" : " 14 GB", | 20260505 10:10:06.808 | INFO | "pinned" : " 29 GB" | 20260505 10:10:06.808 | INFO | } | 20260505 10:10:06.808 | INFO | Perf report for the last 5.14877 seconds: | 20260505 10:10:06.808 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:10:06.808 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:06.808 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:06.808 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:06.808 | INFO | ANALYSIS : 63.1%, 3.0, 98.8ms, 0 | 20260505 10:10:06.808 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:06.808 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:06.808 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 10:10:06.808 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:06.808 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:06.809 | INFO | Uploaded 36GiB to GPU in 2.16853s (16.6011GiB/s) | 20260505 10:10:06.809 | INFO | Downloaded 34GiB to GPU in 2.06488s (16.4658GiB/s) | 20260505 10:10:11.949 | INFO | Analyzing chunk frames = [2382848,2383360) | 20260505 10:10:11.949 | INFO | MemInfo: { | 20260505 10:10:11.949 | INFO | "gpu" : " 17 GB", | 20260505 10:10:11.949 | INFO | "huge" : "", | 20260505 10:10:11.949 | INFO | "huge_shared" : " 27 GB", | 20260505 10:10:11.949 | INFO | "malloc" : " 14 GB", | 20260505 10:10:11.949 | INFO | "pinned" : " 29 GB" | 20260505 10:10:11.949 | INFO | } | 20260505 10:10:11.949 | INFO | Perf report for the last 5.14099 seconds: | 20260505 10:10:11.949 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:10:11.949 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:11.949 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:11.949 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:11.949 | INFO | ANALYSIS : 63.7%, 3.0, 99.6ms, 0 | 20260505 10:10:11.949 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:11.949 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:11.949 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 10:10:11.950 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:11.950 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:11.950 | INFO | Uploaded 36GiB to GPU in 2.19466s (16.4035GiB/s) | 20260505 10:10:11.950 | INFO | Downloaded 34GiB to GPU in 2.15458s (15.7804GiB/s) | 20260505 10:10:17.087 | INFO | Analyzing chunk frames = [2383360,2383872) | 20260505 10:10:17.087 | INFO | MemInfo: { | 20260505 10:10:17.087 | INFO | "gpu" : " 17 GB", | 20260505 10:10:17.087 | INFO | "huge" : "", | 20260505 10:10:17.087 | INFO | "huge_shared" : " 27 GB", | 20260505 10:10:17.087 | INFO | "malloc" : " 14 GB", | 20260505 10:10:17.087 | INFO | "pinned" : " 29 GB" | 20260505 10:10:17.087 | INFO | } | 20260505 10:10:17.087 | INFO | Perf report for the last 5.13816 seconds: | 20260505 10:10:17.088 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:10:17.088 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:17.088 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:17.088 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:17.088 | INFO | ANALYSIS : 64.5%, 3.0, 100.6ms, 0 | 20260505 10:10:17.088 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:17.088 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:17.088 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 10:10:17.088 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:17.088 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:17.088 | INFO | Uploaded 36GiB to GPU in 2.24835s (16.0117GiB/s) | 20260505 10:10:17.088 | INFO | Downloaded 34GiB to GPU in 2.09013s (16.2669GiB/s) | 20260505 10:10:17.088 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 10:10:17.088 | INFO | Total mem usage: 15.3 GiB | 20260505 10:10:17.088 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 10:10:17.088 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.09%) | 20260505 10:10:17.088 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 10:10:17.088 | INFO | High Water Breakdown: | 20260505 10:10:17.088 | INFO | 61.1% : 8.83 GiB ( 94.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 10:10:17.088 | INFO | 38.7% : 5.59 GiB ( 84.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 10:10:17.088 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 10:10:17.088 | INFO | ------------------------------------------ | 20260505 10:10:17.088 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:10:17.088 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 10:10:17.088 | INFO | Total mem usage: 26.5 GiB | 20260505 10:10:17.088 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 10:10:17.088 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 10:10:17.088 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 10:10:17.088 | INFO | High Water Breakdown: | 20260505 10:10:17.088 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:10:17.088 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:10:17.088 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:10:17.088 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:10:17.088 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:10:17.088 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:10:17.088 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 10:10:17.088 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 10:10:17.088 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:10:17.088 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 10:10:17.088 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 10:10:17.088 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 10:10:17.088 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 10:10:17.089 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 10:10:17.089 | INFO | ------------------------------------------ | 20260505 10:10:17.089 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:10:17.089 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 10:10:17.089 | INFO | Total mem usage: 25.6 GiB | 20260505 10:10:17.089 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 10:10:17.089 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 10:10:17.089 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 10:10:17.089 | INFO | High Water Breakdown: | 20260505 10:10:17.089 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 10:10:17.089 | INFO | ------------------------------------------ | 20260505 10:10:17.089 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:10:17.089 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 10:10:17.089 | INFO | Total mem usage: 15.7 GiB | 20260505 10:10:17.089 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 10:10:17.089 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 10:10:17.089 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 10:10:17.089 | INFO | High Water Breakdown: | 20260505 10:10:17.089 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 10:10:17.089 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 10:10:17.089 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 10:10:17.089 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 10:10:17.089 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 10:10:17.089 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 10:10:17.089 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 10:10:17.089 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 10:10:17.089 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 10:10:17.089 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:10:17.089 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 10:10:17.089 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:10:17.089 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 10:10:17.089 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 10:10:17.089 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 10:10:17.089 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 10:10:17.089 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:10:17.089 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 10:10:17.089 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:10:17.089 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:10:17.089 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:10:17.089 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 10:10:17.089 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 10:10:17.089 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:10:17.089 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 10:10:17.089 | INFO | ------------------------------------------ | 20260505 10:10:17.089 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:10:17.089 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 10:10:17.089 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:10:17.089 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:10:17.089 | INFO | PerfReport: Full Execution -- 3.23e+01 / 7.68e+01, 1.37e+01 / 2.21e+00 | 20260505 10:10:17.089 | INFO | PerfReport: 0.29%, AggregateStats -- 9.30e-02 / 1.03e-01, 1.05e-02 / 3.34e-03 | 20260505 10:10:17.089 | INFO | PerfReport: 11.65%, Baseline -- 3.77e+00 / 4.41e+00, 6.04e-01 / 2.53e-01 | 20260505 10:10:17.089 | INFO | PerfReport: 3.31%, Histogram -- 1.07e+00 / 1.48e+00, 2.06e-01 / 9.35e-02 | 20260505 10:10:17.089 | INFO | PerfReport: 15.63%, DME -- 4.04e+01 / 4.47e+01, 4.23e+00 / 1.89e+00 | 20260505 10:10:17.089 | INFO | PerfReport: 56.55%, FrameLabeling -- 1.83e+01 / 2.12e+01, 1.80e+00 / 1.91e+00 | 20260505 10:10:17.089 | INFO | PerfReport: 7.44%, PulseAccumulating -- 2.41e+00 / 2.89e+00, 4.15e-01 / 1.83e-01 | 20260505 10:10:17.089 | INFO | PerfReport: 3.78%, Metrics -- 1.22e+00 / 1.54e+00, 3.48e-01 / 1.76e-01 | 20260505 10:10:17.089 | INFO | PerfReport: 1.34%, PulseSerializing -- 4.33e-01 / 4.78e-01, 8.04e-02 / 2.74e-02 | 20260505 10:10:17.089 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 10:10:17.089 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:10:17.089 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:10:17.089 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.37e+01 / 0.00e+00 | 20260505 10:10:17.089 | INFO | PerfReport: 22.65%, Upload -- 2.30e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260505 10:10:17.089 | INFO | PerfReport: 21.82%, Download -- 2.21e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260505 10:10:17.089 | INFO | PerfReport: 31.89%, Compute -- 3.23e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 10:10:17.089 | INFO | PerfReport: 1.89%, LockUpload -- 1.91e+00 / 0.00e+00, 7.50e+00 / 0.00e+00 | 20260505 10:10:17.089 | INFO | PerfReport: 1.14%, LockDownload -- 1.16e+00 / 0.00e+00, 2.74e+00 / 0.00e+00 | 20260505 10:10:17.089 | INFO | PerfReport: 20.59%, LockCompute -- 2.09e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260505 10:10:17.089 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 2.05e-02 / 0.00e+00 | 20260505 10:10:17.220 | INFO | Starting a round of Baz IO | 20260505 10:10:20.495 | INFO | Populating RTMetrics report | 20260505 10:10:20.496 | INFO | RTMetrics report populated | 20260505 10:10:20.496 | INFO | Opening tmp RTMetrics json file | 20260505 10:10:20.496 | INFO | Serializing RTMetrics to json | 20260505 10:10:20.497 | INFO | Serialization done, swapping tmp file now... | 20260505 10:10:20.498 | INFO | RTMetrics done | 20260505 10:10:20.650 | INFO | Finished a round of Baz IO: Wrote 2.422 GiB in 3.429 seconds | 20260505 10:10:20.650 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.26% / 0.01% / 74.73% | 20260505 10:10:20.650 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.61933% / 1.19209e-05% / 5.5631e-05% | 20260505 10:10:20.650 | INFO | Baz IO overall: | 20260505 10:10:20.650 | INFO | BytesWritten : 1391830995280 | 20260505 10:10:20.650 | INFO | NumEvents : 662608774137 | 20260505 10:10:22.215 | INFO | Analyzing chunk frames = [2383872,2384384) | 20260505 10:10:22.215 | INFO | MemInfo: { | 20260505 10:10:22.215 | INFO | "gpu" : " 17 GB", | 20260505 10:10:22.215 | INFO | "huge" : "", | 20260505 10:10:22.215 | INFO | "huge_shared" : " 27 GB", | 20260505 10:10:22.215 | INFO | "malloc" : " 14 GB", | 20260505 10:10:22.215 | INFO | "pinned" : " 29 GB" | 20260505 10:10:22.215 | INFO | } | 20260505 10:10:22.215 | INFO | Perf report for the last 5.1281 seconds: | 20260505 10:10:22.216 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:10:22.216 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:22.216 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:22.216 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:22.216 | INFO | ANALYSIS : 65.3%, 2.9, 100.1ms, 0 | 20260505 10:10:22.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:22.216 | INFO | RT_METRICS : 45.7%, 1.0, 23.9ms, 0 | 20260505 10:10:22.216 | INFO | PRE_HQ : 62.1%, 1.9, 59.8ms, 0 | 20260505 10:10:22.216 | INFO | BAZWRITER : 66.2%, 4.1, 140.5ms, 0 | 20260505 10:10:22.216 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 10:10:22.216 | INFO | Uploaded 36GiB to GPU in 2.22122s (16.2073GiB/s) | 20260505 10:10:22.216 | INFO | Downloaded 38GiB to GPU in 2.32856s (16.3191GiB/s) | 20260505 10:10:27.355 | INFO | Analyzing chunk frames = [2384384,2384896) | 20260505 10:10:27.355 | INFO | MemInfo: { | 20260505 10:10:27.355 | INFO | "gpu" : " 17 GB", | 20260505 10:10:27.355 | INFO | "huge" : "", | 20260505 10:10:27.355 | INFO | "huge_shared" : " 27 GB", | 20260505 10:10:27.355 | INFO | "malloc" : " 14 GB", | 20260505 10:10:27.355 | INFO | "pinned" : " 29 GB" | 20260505 10:10:27.355 | INFO | } | 20260505 10:10:27.355 | INFO | Perf report for the last 5.13946 seconds: | 20260505 10:10:27.355 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:10:27.355 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:27.355 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:27.355 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:27.355 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 10:10:27.355 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:27.355 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:27.355 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:10:27.355 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:27.355 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:27.355 | INFO | Uploaded 36GiB to GPU in 2.21652s (16.2416GiB/s) | 20260505 10:10:27.355 | INFO | Downloaded 34GiB to GPU in 2.08109s (16.3376GiB/s) | 20260505 10:10:32.503 | INFO | Analyzing chunk frames = [2384896,2385408) | 20260505 10:10:32.503 | INFO | MemInfo: { | 20260505 10:10:32.503 | INFO | "gpu" : " 17 GB", | 20260505 10:10:32.503 | INFO | "huge" : "", | 20260505 10:10:32.503 | INFO | "huge_shared" : " 27 GB", | 20260505 10:10:32.503 | INFO | "malloc" : " 14 GB", | 20260505 10:10:32.503 | INFO | "pinned" : " 29 GB" | 20260505 10:10:32.503 | INFO | } | 20260505 10:10:32.503 | INFO | Perf report for the last 5.14816 seconds: | 20260505 10:10:32.503 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:10:32.503 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:32.503 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:32.503 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:32.503 | INFO | ANALYSIS : 62.9%, 3.0, 98.0ms, 0 | 20260505 10:10:32.503 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:32.503 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:32.503 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 10:10:32.503 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:32.503 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:32.504 | INFO | Uploaded 36GiB to GPU in 2.14934s (16.7493GiB/s) | 20260505 10:10:32.504 | INFO | Downloaded 34GiB to GPU in 2.06054s (16.5005GiB/s) | 20260505 10:10:37.640 | INFO | Analyzing chunk frames = [2385408,2385920) | 20260505 10:10:37.640 | INFO | MemInfo: { | 20260505 10:10:37.640 | INFO | "gpu" : " 17 GB", | 20260505 10:10:37.640 | INFO | "huge" : "", | 20260505 10:10:37.640 | INFO | "huge_shared" : " 27 GB", | 20260505 10:10:37.640 | INFO | "malloc" : " 14 GB", | 20260505 10:10:37.640 | INFO | "pinned" : " 29 GB" | 20260505 10:10:37.640 | INFO | } | 20260505 10:10:37.641 | INFO | Perf report for the last 5.13738 seconds: | 20260505 10:10:37.641 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:10:37.641 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:37.641 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:37.641 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:37.641 | INFO | ANALYSIS : 63.9%, 3.0, 99.2ms, 0 | 20260505 10:10:37.641 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:37.641 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:37.641 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 10:10:37.641 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:37.641 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:37.641 | INFO | Uploaded 36GiB to GPU in 2.22509s (16.1791GiB/s) | 20260505 10:10:37.641 | INFO | Downloaded 34GiB to GPU in 2.11726s (16.0585GiB/s) | 20260505 10:10:42.783 | INFO | Analyzing chunk frames = [2385920,2386432) | 20260505 10:10:42.783 | INFO | MemInfo: { | 20260505 10:10:42.783 | INFO | "gpu" : " 17 GB", | 20260505 10:10:42.783 | INFO | "huge" : "", | 20260505 10:10:42.783 | INFO | "huge_shared" : " 27 GB", | 20260505 10:10:42.783 | INFO | "malloc" : " 14 GB", | 20260505 10:10:42.783 | INFO | "pinned" : " 29 GB" | 20260505 10:10:42.783 | INFO | } | 20260505 10:10:42.783 | INFO | Perf report for the last 5.1426 seconds: | 20260505 10:10:42.783 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:10:42.783 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:42.783 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:42.783 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:42.783 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 10:10:42.783 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:42.783 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:42.783 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 10:10:42.783 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:42.783 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:42.783 | INFO | Uploaded 36GiB to GPU in 2.23716s (16.0918GiB/s) | 20260505 10:10:42.783 | INFO | Downloaded 34GiB to GPU in 2.10482s (16.1534GiB/s) | 20260505 10:10:47.916 | INFO | Analyzing chunk frames = [2386432,2386944) | 20260505 10:10:47.916 | INFO | MemInfo: { | 20260505 10:10:47.916 | INFO | "gpu" : " 17 GB", | 20260505 10:10:47.916 | INFO | "huge" : "", | 20260505 10:10:47.916 | INFO | "huge_shared" : " 27 GB", | 20260505 10:10:47.916 | INFO | "malloc" : " 14 GB", | 20260505 10:10:47.916 | INFO | "pinned" : " 29 GB" | 20260505 10:10:47.916 | INFO | } | 20260505 10:10:47.916 | INFO | Perf report for the last 5.13257 seconds: | 20260505 10:10:47.916 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:10:47.916 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:47.916 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:47.916 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:47.916 | INFO | ANALYSIS : 64.1%, 3.0, 100.0ms, 0 | 20260505 10:10:47.916 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:47.916 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:47.916 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:10:47.916 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:47.916 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:47.916 | INFO | Uploaded 36GiB to GPU in 2.23912s (16.0777GiB/s) | 20260505 10:10:47.916 | INFO | Downloaded 34GiB to GPU in 2.09894s (16.1986GiB/s) | 20260505 10:10:53.059 | INFO | Analyzing chunk frames = [2386944,2387456) | 20260505 10:10:53.059 | INFO | MemInfo: { | 20260505 10:10:53.059 | INFO | "gpu" : " 17 GB", | 20260505 10:10:53.059 | INFO | "huge" : "", | 20260505 10:10:53.059 | INFO | "huge_shared" : " 27 GB", | 20260505 10:10:53.059 | INFO | "malloc" : " 14 GB", | 20260505 10:10:53.059 | INFO | "pinned" : " 29 GB" | 20260505 10:10:53.059 | INFO | } | 20260505 10:10:53.059 | INFO | Perf report for the last 5.14351 seconds: | 20260505 10:10:53.059 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:10:53.059 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:53.059 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:53.059 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:53.059 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 10:10:53.059 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:53.059 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:53.059 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 10:10:53.059 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:53.059 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:53.060 | INFO | Uploaded 36GiB to GPU in 2.20488s (16.3274GiB/s) | 20260505 10:10:53.060 | INFO | Downloaded 34GiB to GPU in 2.13022s (15.9608GiB/s) | 20260505 10:10:58.200 | INFO | Analyzing chunk frames = [2387456,2387968) | 20260505 10:10:58.200 | INFO | MemInfo: { | 20260505 10:10:58.200 | INFO | "gpu" : " 17 GB", | 20260505 10:10:58.200 | INFO | "huge" : "", | 20260505 10:10:58.200 | INFO | "huge_shared" : " 27 GB", | 20260505 10:10:58.200 | INFO | "malloc" : " 14 GB", | 20260505 10:10:58.200 | INFO | "pinned" : " 29 GB" | 20260505 10:10:58.200 | INFO | } | 20260505 10:10:58.200 | INFO | Perf report for the last 5.14105 seconds: | 20260505 10:10:58.200 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:10:58.200 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:58.200 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:58.200 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:58.200 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 10:10:58.200 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:58.200 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:58.200 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 10:10:58.200 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:58.200 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:10:58.201 | INFO | Uploaded 36GiB to GPU in 2.23713s (16.0921GiB/s) | 20260505 10:10:58.201 | INFO | Downloaded 34GiB to GPU in 2.08516s (16.3057GiB/s) | 20260505 10:10:58.335 | INFO | Starting a round of Baz IO | 20260505 10:11:01.591 | INFO | Populating RTMetrics report | 20260505 10:11:01.591 | INFO | RTMetrics report populated | 20260505 10:11:01.591 | INFO | Opening tmp RTMetrics json file | 20260505 10:11:01.600 | INFO | Serializing RTMetrics to json | 20260505 10:11:01.642 | INFO | Serialization done, swapping tmp file now... | 20260505 10:11:01.645 | INFO | RTMetrics done | 20260505 10:11:01.787 | INFO | Finished a round of Baz IO: Wrote 2.421 GiB in 3.451 seconds | 20260505 10:11:01.787 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.25% / 0.01% / 74.74% | 20260505 10:11:01.787 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.61075% / 1.19209e-05% / 7.54992e-05% | 20260505 10:11:01.787 | INFO | Baz IO overall: | 20260505 10:11:01.787 | INFO | BytesWritten : 1394430079769 | 20260505 10:11:01.787 | INFO | NumEvents : 663848804099 | 20260505 10:11:03.338 | INFO | Analyzing chunk frames = [2387968,2388480) | 20260505 10:11:03.338 | INFO | MemInfo: { | 20260505 10:11:03.338 | INFO | "gpu" : " 17 GB", | 20260505 10:11:03.338 | INFO | "huge" : "", | 20260505 10:11:03.338 | INFO | "huge_shared" : " 27 GB", | 20260505 10:11:03.338 | INFO | "malloc" : " 14 GB", | 20260505 10:11:03.338 | INFO | "pinned" : " 29 GB" | 20260505 10:11:03.338 | INFO | } | 20260505 10:11:03.338 | INFO | Perf report for the last 5.13775 seconds: | 20260505 10:11:03.338 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:11:03.338 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:03.338 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:03.338 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:03.338 | INFO | ANALYSIS : 64.9%, 2.9, 99.1ms, 0 | 20260505 10:11:03.338 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:03.338 | INFO | RT_METRICS : 46.7%, 1.0, 24.5ms, 0 | 20260505 10:11:03.338 | INFO | PRE_HQ : 62.7%, 1.8, 59.7ms, 0 | 20260505 10:11:03.338 | INFO | BAZWRITER : 66.5%, 4.2, 143.8ms, 0 | 20260505 10:11:03.338 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 10:11:03.338 | INFO | Uploaded 36GiB to GPU in 2.29874s (15.6608GiB/s) | 20260505 10:11:03.338 | INFO | Downloaded 38GiB to GPU in 2.39379s (15.8744GiB/s) | 20260505 10:11:08.473 | INFO | Analyzing chunk frames = [2388480,2388992) | 20260505 10:11:08.473 | INFO | MemInfo: { | 20260505 10:11:08.473 | INFO | "gpu" : " 17 GB", | 20260505 10:11:08.473 | INFO | "huge" : "", | 20260505 10:11:08.473 | INFO | "huge_shared" : " 27 GB", | 20260505 10:11:08.473 | INFO | "malloc" : " 14 GB", | 20260505 10:11:08.473 | INFO | "pinned" : " 29 GB" | 20260505 10:11:08.473 | INFO | } | 20260505 10:11:08.473 | INFO | Perf report for the last 5.13475 seconds: | 20260505 10:11:08.473 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:11:08.473 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:08.473 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:08.473 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:08.473 | INFO | ANALYSIS : 64.3%, 3.0, 100.3ms, 0 | 20260505 10:11:08.473 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:08.473 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:08.473 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:11:08.473 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:08.473 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:08.473 | INFO | Uploaded 36GiB to GPU in 2.17887s (16.5224GiB/s) | 20260505 10:11:08.473 | INFO | Downloaded 34GiB to GPU in 2.09294s (16.2451GiB/s) | 20260505 10:11:13.609 | INFO | Analyzing chunk frames = [2388992,2389504) | 20260505 10:11:13.609 | INFO | MemInfo: { | 20260505 10:11:13.609 | INFO | "gpu" : " 17 GB", | 20260505 10:11:13.609 | INFO | "huge" : "", | 20260505 10:11:13.609 | INFO | "huge_shared" : " 27 GB", | 20260505 10:11:13.609 | INFO | "malloc" : " 14 GB", | 20260505 10:11:13.609 | INFO | "pinned" : " 29 GB" | 20260505 10:11:13.609 | INFO | } | 20260505 10:11:13.609 | INFO | Perf report for the last 5.13642 seconds: | 20260505 10:11:13.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:11:13.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:13.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:13.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:13.609 | INFO | ANALYSIS : 63.2%, 3.0, 98.3ms, 0 | 20260505 10:11:13.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:13.609 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:13.609 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 10:11:13.609 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:13.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:13.609 | INFO | Uploaded 36GiB to GPU in 2.14611s (16.7745GiB/s) | 20260505 10:11:13.609 | INFO | Downloaded 34GiB to GPU in 2.06483s (16.4662GiB/s) | 20260505 10:11:18.760 | INFO | Analyzing chunk frames = [2389504,2390016) | 20260505 10:11:18.760 | INFO | MemInfo: { | 20260505 10:11:18.760 | INFO | "gpu" : " 17 GB", | 20260505 10:11:18.760 | INFO | "huge" : "", | 20260505 10:11:18.760 | INFO | "huge_shared" : " 27 GB", | 20260505 10:11:18.760 | INFO | "malloc" : " 14 GB", | 20260505 10:11:18.760 | INFO | "pinned" : " 29 GB" | 20260505 10:11:18.760 | INFO | } | 20260505 10:11:18.760 | INFO | Perf report for the last 5.15075 seconds: | 20260505 10:11:18.760 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:11:18.760 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:18.760 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:18.760 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:18.760 | INFO | ANALYSIS : 63.1%, 3.0, 98.2ms, 0 | 20260505 10:11:18.760 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:18.760 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:18.760 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:11:18.760 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:18.760 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:18.760 | INFO | Uploaded 36GiB to GPU in 2.16216s (16.65GiB/s) | 20260505 10:11:18.760 | INFO | Downloaded 34GiB to GPU in 2.07349s (16.3975GiB/s) | 20260505 10:11:23.900 | INFO | Analyzing chunk frames = [2390016,2390528) | 20260505 10:11:23.900 | INFO | MemInfo: { | 20260505 10:11:23.900 | INFO | "gpu" : " 17 GB", | 20260505 10:11:23.900 | INFO | "huge" : "", | 20260505 10:11:23.900 | INFO | "huge_shared" : " 27 GB", | 20260505 10:11:23.901 | INFO | "malloc" : " 14 GB", | 20260505 10:11:23.901 | INFO | "pinned" : " 29 GB" | 20260505 10:11:23.901 | INFO | } | 20260505 10:11:23.901 | INFO | Perf report for the last 5.14064 seconds: | 20260505 10:11:23.901 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:11:23.901 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:23.901 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:23.901 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:23.901 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 10:11:23.901 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:23.901 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:23.901 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:11:23.901 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:23.901 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:23.901 | INFO | Uploaded 36GiB to GPU in 2.16614s (16.6194GiB/s) | 20260505 10:11:23.901 | INFO | Downloaded 34GiB to GPU in 2.12868s (15.9724GiB/s) | 20260505 10:11:29.034 | INFO | Analyzing chunk frames = [2390528,2391040) | 20260505 10:11:29.034 | INFO | MemInfo: { | 20260505 10:11:29.034 | INFO | "gpu" : " 17 GB", | 20260505 10:11:29.034 | INFO | "huge" : "", | 20260505 10:11:29.034 | INFO | "huge_shared" : " 27 GB", | 20260505 10:11:29.034 | INFO | "malloc" : " 14 GB", | 20260505 10:11:29.034 | INFO | "pinned" : " 29 GB" | 20260505 10:11:29.034 | INFO | } | 20260505 10:11:29.034 | INFO | Perf report for the last 5.13301 seconds: | 20260505 10:11:29.034 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:11:29.034 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:29.034 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:29.034 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:29.034 | INFO | ANALYSIS : 63.8%, 3.0, 99.4ms, 0 | 20260505 10:11:29.034 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:29.034 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:29.034 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:11:29.034 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:29.034 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:29.034 | INFO | Uploaded 36GiB to GPU in 2.22344s (16.1912GiB/s) | 20260505 10:11:29.034 | INFO | Downloaded 34GiB to GPU in 2.10083s (16.1841GiB/s) | 20260505 10:11:34.165 | INFO | Analyzing chunk frames = [2391040,2391552) | 20260505 10:11:34.165 | INFO | MemInfo: { | 20260505 10:11:34.165 | INFO | "gpu" : " 17 GB", | 20260505 10:11:34.165 | INFO | "huge" : "", | 20260505 10:11:34.165 | INFO | "huge_shared" : " 27 GB", | 20260505 10:11:34.165 | INFO | "malloc" : " 14 GB", | 20260505 10:11:34.165 | INFO | "pinned" : " 29 GB" | 20260505 10:11:34.165 | INFO | } | 20260505 10:11:34.165 | INFO | Perf report for the last 5.13095 seconds: | 20260505 10:11:34.165 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:11:34.165 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:34.165 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:34.165 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:34.165 | INFO | ANALYSIS : 63.8%, 3.0, 99.5ms, 0 | 20260505 10:11:34.165 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:34.165 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:34.165 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 10:11:34.165 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:34.165 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:34.165 | INFO | Uploaded 36GiB to GPU in 2.19789s (16.3793GiB/s) | 20260505 10:11:34.165 | INFO | Downloaded 34GiB to GPU in 2.11827s (16.0508GiB/s) | 20260505 10:11:39.308 | INFO | Analyzing chunk frames = [2391552,2392064) | 20260505 10:11:39.308 | INFO | MemInfo: { | 20260505 10:11:39.308 | INFO | "gpu" : " 17 GB", | 20260505 10:11:39.308 | INFO | "huge" : "", | 20260505 10:11:39.308 | INFO | "huge_shared" : " 27 GB", | 20260505 10:11:39.308 | INFO | "malloc" : " 14 GB", | 20260505 10:11:39.308 | INFO | "pinned" : " 29 GB" | 20260505 10:11:39.308 | INFO | } | 20260505 10:11:39.308 | INFO | Perf report for the last 5.14317 seconds: | 20260505 10:11:39.308 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:11:39.308 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:39.308 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:39.308 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:39.308 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 10:11:39.308 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:39.308 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:39.308 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:11:39.308 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:39.308 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:39.308 | INFO | Uploaded 36GiB to GPU in 2.17532s (16.5493GiB/s) | 20260505 10:11:39.308 | INFO | Downloaded 34GiB to GPU in 2.03469s (16.7102GiB/s) | 20260505 10:11:39.308 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 10:11:39.308 | INFO | Total mem usage: 15.3 GiB | 20260505 10:11:39.308 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 10:11:39.308 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -5.52%) | 20260505 10:11:39.308 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 10:11:39.308 | INFO | High Water Breakdown: | 20260505 10:11:39.308 | INFO | 61% : 8.87 GiB ( 94.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 10:11:39.309 | INFO | 38.8% : 5.64 GiB ( 84.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 10:11:39.309 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 10:11:39.309 | INFO | ------------------------------------------ | 20260505 10:11:39.309 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:11:39.309 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 10:11:39.309 | INFO | Total mem usage: 26.5 GiB | 20260505 10:11:39.309 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 10:11:39.309 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 10:11:39.309 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 10:11:39.309 | INFO | High Water Breakdown: | 20260505 10:11:39.309 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:11:39.309 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:11:39.309 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:11:39.309 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:11:39.309 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:11:39.309 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:11:39.309 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 10:11:39.309 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 10:11:39.309 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:11:39.309 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 10:11:39.309 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 10:11:39.309 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 10:11:39.309 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 10:11:39.309 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 10:11:39.309 | INFO | ------------------------------------------ | 20260505 10:11:39.309 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:11:39.309 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 10:11:39.309 | INFO | Total mem usage: 25.6 GiB | 20260505 10:11:39.309 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 10:11:39.309 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 10:11:39.309 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 10:11:39.309 | INFO | High Water Breakdown: | 20260505 10:11:39.309 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 10:11:39.309 | INFO | ------------------------------------------ | 20260505 10:11:39.309 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:11:39.309 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 10:11:39.309 | INFO | Total mem usage: 15.7 GiB | 20260505 10:11:39.309 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 10:11:39.309 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 10:11:39.309 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 10:11:39.309 | INFO | High Water Breakdown: | 20260505 10:11:39.309 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 10:11:39.309 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 10:11:39.309 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 10:11:39.309 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 10:11:39.309 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 10:11:39.309 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 10:11:39.309 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 10:11:39.309 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 10:11:39.309 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 10:11:39.309 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:11:39.309 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 10:11:39.309 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:11:39.309 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 10:11:39.309 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 10:11:39.309 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 10:11:39.309 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 10:11:39.309 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:11:39.309 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 10:11:39.309 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:11:39.309 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:11:39.309 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:11:39.309 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 10:11:39.309 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 10:11:39.309 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:11:39.309 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 10:11:39.309 | INFO | ------------------------------------------ | 20260505 10:11:39.309 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:11:39.309 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 10:11:39.309 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:11:39.309 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:11:39.309 | INFO | PerfReport: Full Execution -- 3.24e+01 / 7.69e+01, 1.39e+01 / 2.35e+00 | 20260505 10:11:39.309 | INFO | PerfReport: 0.29%, AggregateStats -- 9.34e-02 / 1.03e-01, 1.07e-02 / 4.67e-03 | 20260505 10:11:39.309 | INFO | PerfReport: 11.67%, Baseline -- 3.78e+00 / 4.46e+00, 6.13e-01 / 2.59e-01 | 20260505 10:11:39.309 | INFO | PerfReport: 3.32%, Histogram -- 1.08e+00 / 1.47e+00, 2.08e-01 / 1.11e-01 | 20260505 10:11:39.309 | INFO | PerfReport: 15.63%, DME -- 4.05e+01 / 4.46e+01, 4.30e+00 / 2.30e+00 | 20260505 10:11:39.309 | INFO | PerfReport: 56.60%, FrameLabeling -- 1.83e+01 / 2.14e+01, 1.84e+00 / 2.01e+00 | 20260505 10:11:39.309 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.39e+00 / 2.92e+00, 4.07e-01 / 2.12e-01 | 20260505 10:11:39.309 | INFO | PerfReport: 3.78%, Metrics -- 1.23e+00 / 1.52e+00, 3.54e-01 / 1.69e-01 | 20260505 10:11:39.309 | INFO | PerfReport: 1.34%, PulseSerializing -- 4.33e-01 / 4.81e-01, 8.14e-02 / 3.04e-02 | 20260505 10:11:39.309 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 10:11:39.309 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:11:39.309 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:11:39.309 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260505 10:11:39.309 | INFO | PerfReport: 22.64%, Upload -- 2.30e+01 / 0.00e+00, 8.29e+00 / 0.00e+00 | 20260505 10:11:39.309 | INFO | PerfReport: 21.82%, Download -- 2.21e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260505 10:11:39.309 | INFO | PerfReport: 31.93%, Compute -- 3.24e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 10:11:39.309 | INFO | PerfReport: 1.93%, LockUpload -- 1.96e+00 / 0.00e+00, 7.53e+00 / 0.00e+00 | 20260505 10:11:39.309 | INFO | PerfReport: 1.14%, LockDownload -- 1.16e+00 / 0.00e+00, 2.75e+00 / 0.00e+00 | 20260505 10:11:39.309 | INFO | PerfReport: 20.52%, LockCompute -- 2.08e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260505 10:11:39.309 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.40e-02 / 0.00e+00 | 20260505 10:11:39.441 | INFO | Starting a round of Baz IO | 20260505 10:11:42.716 | INFO | Populating RTMetrics report | 20260505 10:11:42.717 | INFO | RTMetrics report populated | 20260505 10:11:42.717 | INFO | Opening tmp RTMetrics json file | 20260505 10:11:42.721 | INFO | Serializing RTMetrics to json | 20260505 10:11:42.722 | INFO | Serialization done, swapping tmp file now... | 20260505 10:11:42.722 | INFO | RTMetrics done | 20260505 10:11:42.866 | INFO | Finished a round of Baz IO: Wrote 2.421 GiB in 3.424 seconds | 20260505 10:11:42.866 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.23% / 0.01% / 74.75% | 20260505 10:11:42.866 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.60776% / 3.97364e-05% / 5.16574e-05% | 20260505 10:11:42.866 | INFO | Baz IO overall: | 20260505 10:11:42.866 | INFO | BytesWritten : 1397030108657 | 20260505 10:11:42.866 | INFO | NumEvents : 665089915836 | 20260505 10:11:44.455 | INFO | Analyzing chunk frames = [2392064,2392576) | 20260505 10:11:44.455 | INFO | MemInfo: { | 20260505 10:11:44.455 | INFO | "gpu" : " 17 GB", | 20260505 10:11:44.455 | INFO | "huge" : "", | 20260505 10:11:44.455 | INFO | "huge_shared" : " 27 GB", | 20260505 10:11:44.455 | INFO | "malloc" : " 14 GB", | 20260505 10:11:44.455 | INFO | "pinned" : " 29 GB" | 20260505 10:11:44.455 | INFO | } | 20260505 10:11:44.455 | INFO | Perf report for the last 5.14737 seconds: | 20260505 10:11:44.455 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:11:44.455 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:44.455 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:44.455 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:44.455 | INFO | ANALYSIS : 65.1%, 2.9, 100.1ms, 0 | 20260505 10:11:44.455 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:44.455 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260505 10:11:44.455 | INFO | PRE_HQ : 61.7%, 1.9, 59.4ms, 0 | 20260505 10:11:44.455 | INFO | BAZWRITER : 65.9%, 4.2, 140.9ms, 0 | 20260505 10:11:44.456 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 10:11:44.456 | INFO | Uploaded 36GiB to GPU in 2.18978s (16.44GiB/s) | 20260505 10:11:44.456 | INFO | Downloaded 38GiB to GPU in 2.31115s (16.442GiB/s) | 20260505 10:11:49.595 | INFO | Analyzing chunk frames = [2392576,2393088) | 20260505 10:11:49.595 | INFO | MemInfo: { | 20260505 10:11:49.595 | INFO | "gpu" : " 17 GB", | 20260505 10:11:49.595 | INFO | "huge" : "", | 20260505 10:11:49.595 | INFO | "huge_shared" : " 27 GB", | 20260505 10:11:49.595 | INFO | "malloc" : " 14 GB", | 20260505 10:11:49.595 | INFO | "pinned" : " 29 GB" | 20260505 10:11:49.596 | INFO | } | 20260505 10:11:49.596 | INFO | Perf report for the last 5.14038 seconds: | 20260505 10:11:49.596 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:11:49.596 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:49.596 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:49.596 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:49.596 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 10:11:49.596 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:49.596 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:49.596 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:11:49.596 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:49.596 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:49.596 | INFO | Uploaded 36GiB to GPU in 2.20738s (16.309GiB/s) | 20260505 10:11:49.596 | INFO | Downloaded 34GiB to GPU in 2.08607s (16.2986GiB/s) | 20260505 10:11:54.729 | INFO | Analyzing chunk frames = [2393088,2393600) | 20260505 10:11:54.729 | INFO | MemInfo: { | 20260505 10:11:54.729 | INFO | "gpu" : " 17 GB", | 20260505 10:11:54.729 | INFO | "huge" : "", | 20260505 10:11:54.729 | INFO | "huge_shared" : " 27 GB", | 20260505 10:11:54.729 | INFO | "malloc" : " 14 GB", | 20260505 10:11:54.729 | INFO | "pinned" : " 29 GB" | 20260505 10:11:54.729 | INFO | } | 20260505 10:11:54.729 | INFO | Perf report for the last 5.13313 seconds: | 20260505 10:11:54.729 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:11:54.729 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:54.729 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:54.729 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:54.729 | INFO | ANALYSIS : 63.1%, 3.0, 98.0ms, 0 | 20260505 10:11:54.729 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:54.729 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:54.729 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:11:54.729 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:54.729 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:54.729 | INFO | Uploaded 36GiB to GPU in 2.14903s (16.7518GiB/s) | 20260505 10:11:54.729 | INFO | Downloaded 34GiB to GPU in 2.05576s (16.5389GiB/s) | 20260505 10:11:59.864 | INFO | Analyzing chunk frames = [2393600,2394112) | 20260505 10:11:59.864 | INFO | MemInfo: { | 20260505 10:11:59.864 | INFO | "gpu" : " 17 GB", | 20260505 10:11:59.864 | INFO | "huge" : "", | 20260505 10:11:59.864 | INFO | "huge_shared" : " 27 GB", | 20260505 10:11:59.864 | INFO | "malloc" : " 14 GB", | 20260505 10:11:59.864 | INFO | "pinned" : " 29 GB" | 20260505 10:11:59.864 | INFO | } | 20260505 10:11:59.864 | INFO | Perf report for the last 5.13516 seconds: | 20260505 10:11:59.864 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:11:59.864 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:59.864 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:59.864 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:59.864 | INFO | ANALYSIS : 63.2%, 3.0, 98.1ms, 0 | 20260505 10:11:59.864 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:59.864 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:59.864 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:11:59.864 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:59.864 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:11:59.864 | INFO | Uploaded 36GiB to GPU in 2.16458s (16.6314GiB/s) | 20260505 10:11:59.864 | INFO | Downloaded 34GiB to GPU in 2.06989s (16.426GiB/s) | 20260505 10:12:05.012 | INFO | Analyzing chunk frames = [2394112,2394624) | 20260505 10:12:05.012 | INFO | MemInfo: { | 20260505 10:12:05.012 | INFO | "gpu" : " 17 GB", | 20260505 10:12:05.012 | INFO | "huge" : "", | 20260505 10:12:05.012 | INFO | "huge_shared" : " 27 GB", | 20260505 10:12:05.012 | INFO | "malloc" : " 14 GB", | 20260505 10:12:05.012 | INFO | "pinned" : " 29 GB" | 20260505 10:12:05.012 | INFO | } | 20260505 10:12:05.012 | INFO | Perf report for the last 5.14834 seconds: | 20260505 10:12:05.012 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:12:05.012 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:05.012 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:05.012 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:05.012 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 10:12:05.012 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:05.012 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:05.012 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:12:05.012 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:05.012 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:05.013 | INFO | Uploaded 36GiB to GPU in 2.23484s (16.1085GiB/s) | 20260505 10:12:05.013 | INFO | Downloaded 34GiB to GPU in 2.11564s (16.0708GiB/s) | 20260505 10:12:10.156 | INFO | Analyzing chunk frames = [2394624,2395136) | 20260505 10:12:10.156 | INFO | MemInfo: { | 20260505 10:12:10.156 | INFO | "gpu" : " 17 GB", | 20260505 10:12:10.156 | INFO | "huge" : "", | 20260505 10:12:10.156 | INFO | "huge_shared" : " 27 GB", | 20260505 10:12:10.156 | INFO | "malloc" : " 14 GB", | 20260505 10:12:10.156 | INFO | "pinned" : " 29 GB" | 20260505 10:12:10.156 | INFO | } | 20260505 10:12:10.156 | INFO | Perf report for the last 5.14346 seconds: | 20260505 10:12:10.156 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:12:10.156 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:10.156 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:10.156 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:10.156 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 10:12:10.156 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:10.156 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:10.156 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 10:12:10.156 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:10.156 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:10.156 | INFO | Uploaded 36GiB to GPU in 2.19598s (16.3936GiB/s) | 20260505 10:12:10.156 | INFO | Downloaded 34GiB to GPU in 2.10443s (16.1564GiB/s) | 20260505 10:12:15.281 | INFO | Analyzing chunk frames = [2395136,2395648) | 20260505 10:12:15.281 | INFO | MemInfo: { | 20260505 10:12:15.281 | INFO | "gpu" : " 17 GB", | 20260505 10:12:15.281 | INFO | "huge" : "", | 20260505 10:12:15.281 | INFO | "huge_shared" : " 27 GB", | 20260505 10:12:15.281 | INFO | "malloc" : " 14 GB", | 20260505 10:12:15.281 | INFO | "pinned" : " 29 GB" | 20260505 10:12:15.281 | INFO | } | 20260505 10:12:15.281 | INFO | Perf report for the last 5.12531 seconds: | 20260505 10:12:15.281 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:12:15.281 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:15.281 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:15.281 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:15.281 | INFO | ANALYSIS : 64.0%, 3.0, 99.7ms, 0 | 20260505 10:12:15.281 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:15.281 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:15.281 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 10:12:15.281 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:15.281 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:15.281 | INFO | Uploaded 36GiB to GPU in 2.18108s (16.5056GiB/s) | 20260505 10:12:15.281 | INFO | Downloaded 34GiB to GPU in 2.11147s (16.1025GiB/s) | 20260505 10:12:20.423 | INFO | Analyzing chunk frames = [2395648,2396160) | 20260505 10:12:20.423 | INFO | MemInfo: { | 20260505 10:12:20.423 | INFO | "gpu" : " 17 GB", | 20260505 10:12:20.423 | INFO | "huge" : "", | 20260505 10:12:20.423 | INFO | "huge_shared" : " 27 GB", | 20260505 10:12:20.423 | INFO | "malloc" : " 14 GB", | 20260505 10:12:20.423 | INFO | "pinned" : " 29 GB" | 20260505 10:12:20.423 | INFO | } | 20260505 10:12:20.424 | INFO | Perf report for the last 5.14236 seconds: | 20260505 10:12:20.424 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:12:20.424 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:20.424 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:20.424 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:20.424 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 10:12:20.424 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:20.424 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:20.424 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 10:12:20.424 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:20.424 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:20.424 | INFO | Uploaded 36GiB to GPU in 2.19341s (16.4128GiB/s) | 20260505 10:12:20.424 | INFO | Downloaded 34GiB to GPU in 2.03735s (16.6883GiB/s) | 20260505 10:12:20.557 | INFO | Starting a round of Baz IO | 20260505 10:12:23.831 | INFO | Populating RTMetrics report | 20260505 10:12:23.832 | INFO | RTMetrics report populated | 20260505 10:12:23.832 | INFO | Opening tmp RTMetrics json file | 20260505 10:12:23.847 | INFO | Serializing RTMetrics to json | 20260505 10:12:23.877 | INFO | Serialization done, swapping tmp file now... | 20260505 10:12:23.897 | INFO | RTMetrics done | 20260505 10:12:24.010 | INFO | Finished a round of Baz IO: Wrote 2.419 GiB in 3.452 seconds | 20260505 10:12:24.010 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.22% / 0.01% / 74.77% | 20260505 10:12:24.010 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.59973% / 1.98682e-05% / 6.75519e-05% | 20260505 10:12:24.010 | INFO | Baz IO overall: | 20260505 10:12:24.010 | INFO | BytesWritten : 1399627464998 | 20260505 10:12:24.010 | INFO | NumEvents : 666328787864 | 20260505 10:12:25.570 | INFO | Analyzing chunk frames = [2396160,2396672) | 20260505 10:12:25.570 | INFO | MemInfo: { | 20260505 10:12:25.570 | INFO | "gpu" : " 17 GB", | 20260505 10:12:25.570 | INFO | "huge" : "", | 20260505 10:12:25.570 | INFO | "huge_shared" : " 27 GB", | 20260505 10:12:25.570 | INFO | "malloc" : " 14 GB", | 20260505 10:12:25.570 | INFO | "pinned" : " 29 GB" | 20260505 10:12:25.570 | INFO | } | 20260505 10:12:25.570 | INFO | Perf report for the last 5.14662 seconds: | 20260505 10:12:25.570 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:12:25.570 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:25.570 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:25.570 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:25.570 | INFO | ANALYSIS : 65.1%, 2.9, 99.9ms, 0 | 20260505 10:12:25.570 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:25.570 | INFO | RT_METRICS : 46.8%, 1.0, 24.6ms, 0 | 20260505 10:12:25.570 | INFO | PRE_HQ : 62.1%, 1.8, 59.4ms, 0 | 20260505 10:12:25.570 | INFO | BAZWRITER : 66.4%, 4.1, 141.6ms, 0 | 20260505 10:12:25.570 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 10:12:25.570 | INFO | Uploaded 36GiB to GPU in 2.23026s (16.1417GiB/s) | 20260505 10:12:25.570 | INFO | Downloaded 38GiB to GPU in 2.34717s (16.1897GiB/s) | 20260505 10:12:30.708 | INFO | Analyzing chunk frames = [2396672,2397184) | 20260505 10:12:30.708 | INFO | MemInfo: { | 20260505 10:12:30.708 | INFO | "gpu" : " 17 GB", | 20260505 10:12:30.708 | INFO | "huge" : "", | 20260505 10:12:30.708 | INFO | "huge_shared" : " 27 GB", | 20260505 10:12:30.708 | INFO | "malloc" : " 14 GB", | 20260505 10:12:30.708 | INFO | "pinned" : " 29 GB" | 20260505 10:12:30.708 | INFO | } | 20260505 10:12:30.708 | INFO | Perf report for the last 5.13833 seconds: | 20260505 10:12:30.708 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:12:30.708 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:30.708 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:30.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:30.709 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 10:12:30.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:30.709 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:30.709 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:12:30.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:30.709 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:30.709 | INFO | Uploaded 36GiB to GPU in 2.20422s (16.3323GiB/s) | 20260505 10:12:30.709 | INFO | Downloaded 34GiB to GPU in 2.08916s (16.2745GiB/s) | 20260505 10:12:35.846 | INFO | Analyzing chunk frames = [2397184,2397696) | 20260505 10:12:35.846 | INFO | MemInfo: { | 20260505 10:12:35.846 | INFO | "gpu" : " 17 GB", | 20260505 10:12:35.846 | INFO | "huge" : "", | 20260505 10:12:35.846 | INFO | "huge_shared" : " 27 GB", | 20260505 10:12:35.846 | INFO | "malloc" : " 14 GB", | 20260505 10:12:35.846 | INFO | "pinned" : " 29 GB" | 20260505 10:12:35.846 | INFO | } | 20260505 10:12:35.846 | INFO | Perf report for the last 5.13789 seconds: | 20260505 10:12:35.846 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:12:35.846 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:35.846 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:35.846 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:35.846 | INFO | ANALYSIS : 63.0%, 3.0, 98.0ms, 0 | 20260505 10:12:35.846 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:35.846 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:35.846 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 10:12:35.846 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:35.846 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:35.847 | INFO | Uploaded 36GiB to GPU in 2.15063s (16.7393GiB/s) | 20260505 10:12:35.847 | INFO | Downloaded 34GiB to GPU in 2.06299s (16.4809GiB/s) | 20260505 10:12:40.982 | INFO | Analyzing chunk frames = [2397696,2398208) | 20260505 10:12:40.982 | INFO | MemInfo: { | 20260505 10:12:40.982 | INFO | "gpu" : " 17 GB", | 20260505 10:12:40.982 | INFO | "huge" : "", | 20260505 10:12:40.982 | INFO | "huge_shared" : " 27 GB", | 20260505 10:12:40.982 | INFO | "malloc" : " 14 GB", | 20260505 10:12:40.982 | INFO | "pinned" : " 29 GB" | 20260505 10:12:40.982 | INFO | } | 20260505 10:12:40.982 | INFO | Perf report for the last 5.13582 seconds: | 20260505 10:12:40.982 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:12:40.982 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:40.982 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:40.982 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:40.982 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 10:12:40.982 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:40.982 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:40.982 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 10:12:40.982 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:40.982 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:40.982 | INFO | Uploaded 36GiB to GPU in 2.16452s (16.6319GiB/s) | 20260505 10:12:40.983 | INFO | Downloaded 34GiB to GPU in 2.08058s (16.3416GiB/s) | 20260505 10:12:46.123 | INFO | Analyzing chunk frames = [2398208,2398720) | 20260505 10:12:46.123 | INFO | MemInfo: { | 20260505 10:12:46.123 | INFO | "gpu" : " 17 GB", | 20260505 10:12:46.123 | INFO | "huge" : "", | 20260505 10:12:46.123 | INFO | "huge_shared" : " 27 GB", | 20260505 10:12:46.123 | INFO | "malloc" : " 14 GB", | 20260505 10:12:46.123 | INFO | "pinned" : " 29 GB" | 20260505 10:12:46.123 | INFO | } | 20260505 10:12:46.123 | INFO | Perf report for the last 5.14057 seconds: | 20260505 10:12:46.123 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:12:46.123 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:46.123 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:46.123 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:46.123 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 10:12:46.123 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:46.123 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:46.123 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:12:46.123 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:46.123 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:46.123 | INFO | Uploaded 36GiB to GPU in 2.23116s (16.1351GiB/s) | 20260505 10:12:46.123 | INFO | Downloaded 34GiB to GPU in 2.11364s (16.086GiB/s) | 20260505 10:12:51.264 | INFO | Analyzing chunk frames = [2398720,2399232) | 20260505 10:12:51.264 | INFO | MemInfo: { | 20260505 10:12:51.264 | INFO | "gpu" : " 17 GB", | 20260505 10:12:51.264 | INFO | "huge" : "", | 20260505 10:12:51.264 | INFO | "huge_shared" : " 27 GB", | 20260505 10:12:51.264 | INFO | "malloc" : " 14 GB", | 20260505 10:12:51.264 | INFO | "pinned" : " 29 GB" | 20260505 10:12:51.264 | INFO | } | 20260505 10:12:51.264 | INFO | Perf report for the last 5.14094 seconds: | 20260505 10:12:51.264 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:12:51.264 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:51.264 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:51.264 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:51.264 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 10:12:51.264 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:51.264 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:51.264 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:12:51.264 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:51.264 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:51.264 | INFO | Uploaded 36GiB to GPU in 2.23035s (16.141GiB/s) | 20260505 10:12:51.264 | INFO | Downloaded 34GiB to GPU in 2.1086s (16.1245GiB/s) | 20260505 10:12:56.408 | INFO | Analyzing chunk frames = [2399232,2399744) | 20260505 10:12:56.408 | INFO | MemInfo: { | 20260505 10:12:56.408 | INFO | "gpu" : " 17 GB", | 20260505 10:12:56.408 | INFO | "huge" : "", | 20260505 10:12:56.408 | INFO | "huge_shared" : " 27 GB", | 20260505 10:12:56.408 | INFO | "malloc" : " 14 GB", | 20260505 10:12:56.408 | INFO | "pinned" : " 29 GB" | 20260505 10:12:56.408 | INFO | } | 20260505 10:12:56.408 | INFO | Perf report for the last 5.14397 seconds: | 20260505 10:12:56.408 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:12:56.408 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:56.408 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:56.408 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:56.408 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 10:12:56.408 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:56.408 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:56.408 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 10:12:56.408 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:56.408 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:12:56.408 | INFO | Uploaded 36GiB to GPU in 2.16097s (16.6592GiB/s) | 20260505 10:12:56.408 | INFO | Downloaded 34GiB to GPU in 2.11972s (16.0398GiB/s) | 20260505 10:13:01.543 | INFO | Analyzing chunk frames = [2399744,2400256) | 20260505 10:13:01.543 | INFO | MemInfo: { | 20260505 10:13:01.543 | INFO | "gpu" : " 17 GB", | 20260505 10:13:01.543 | INFO | "huge" : "", | 20260505 10:13:01.543 | INFO | "huge_shared" : " 27 GB", | 20260505 10:13:01.543 | INFO | "malloc" : " 14 GB", | 20260505 10:13:01.543 | INFO | "pinned" : " 29 GB" | 20260505 10:13:01.543 | INFO | } | 20260505 10:13:01.543 | INFO | Perf report for the last 5.13562 seconds: | 20260505 10:13:01.543 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:13:01.543 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:01.543 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:01.543 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:01.543 | INFO | ANALYSIS : 64.5%, 3.0, 100.7ms, 0 | 20260505 10:13:01.543 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:01.543 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:01.543 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 10:13:01.543 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:01.544 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:01.544 | INFO | Uploaded 36GiB to GPU in 2.24511s (16.0349GiB/s) | 20260505 10:13:01.544 | INFO | Downloaded 34GiB to GPU in 2.09587s (16.2224GiB/s) | 20260505 10:13:01.544 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 10:13:01.544 | INFO | Total mem usage: 15.3 GiB | 20260505 10:13:01.544 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 10:13:01.544 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.54%) | 20260505 10:13:01.544 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 10:13:01.544 | INFO | High Water Breakdown: | 20260505 10:13:01.544 | INFO | 61% : 8.77 GiB ( 93.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 10:13:01.544 | INFO | 38.9% : 5.59 GiB ( 84.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 10:13:01.544 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 10:13:01.544 | INFO | ------------------------------------------ | 20260505 10:13:01.544 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:13:01.544 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 10:13:01.544 | INFO | Total mem usage: 26.5 GiB | 20260505 10:13:01.544 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 10:13:01.544 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 10:13:01.544 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 10:13:01.544 | INFO | High Water Breakdown: | 20260505 10:13:01.544 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:13:01.544 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:13:01.544 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:13:01.544 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:13:01.544 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:13:01.544 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:13:01.544 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 10:13:01.544 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 10:13:01.544 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:13:01.544 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 10:13:01.544 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 10:13:01.544 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 10:13:01.544 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 10:13:01.544 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 10:13:01.544 | INFO | ------------------------------------------ | 20260505 10:13:01.544 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:13:01.544 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 10:13:01.544 | INFO | Total mem usage: 25.6 GiB | 20260505 10:13:01.544 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 10:13:01.544 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 10:13:01.544 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 10:13:01.544 | INFO | High Water Breakdown: | 20260505 10:13:01.544 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 10:13:01.544 | INFO | ------------------------------------------ | 20260505 10:13:01.544 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:13:01.544 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 10:13:01.544 | INFO | Total mem usage: 15.7 GiB | 20260505 10:13:01.544 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 10:13:01.544 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 10:13:01.544 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 10:13:01.544 | INFO | High Water Breakdown: | 20260505 10:13:01.544 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 10:13:01.544 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 10:13:01.544 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 10:13:01.544 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 10:13:01.544 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 10:13:01.545 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 10:13:01.545 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 10:13:01.545 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 10:13:01.545 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 10:13:01.545 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:13:01.545 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 10:13:01.545 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:13:01.545 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 10:13:01.545 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 10:13:01.545 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 10:13:01.545 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 10:13:01.545 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:13:01.545 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 10:13:01.545 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:13:01.545 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:13:01.545 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:13:01.545 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 10:13:01.545 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 10:13:01.545 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:13:01.545 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 10:13:01.545 | INFO | ------------------------------------------ | 20260505 10:13:01.545 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:13:01.545 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 10:13:01.545 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:13:01.545 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:13:01.545 | INFO | PerfReport: Full Execution -- 3.24e+01 / 7.73e+01, 1.39e+01 / 2.15e+00 | 20260505 10:13:01.545 | INFO | PerfReport: 0.29%, AggregateStats -- 9.30e-02 / 1.03e-01, 1.04e-02 / 4.70e-03 | 20260505 10:13:01.545 | INFO | PerfReport: 11.67%, Baseline -- 3.78e+00 / 4.49e+00, 6.08e-01 / 1.77e-01 | 20260505 10:13:01.545 | INFO | PerfReport: 3.32%, Histogram -- 1.07e+00 / 1.50e+00, 2.08e-01 / 8.68e-02 | 20260505 10:13:01.545 | INFO | PerfReport: 15.66%, DME -- 4.06e+01 / 4.52e+01, 4.47e+00 / 1.91e+00 | 20260505 10:13:01.545 | INFO | PerfReport: 56.58%, FrameLabeling -- 1.83e+01 / 2.11e+01, 1.83e+00 / 1.68e+00 | 20260505 10:13:01.545 | INFO | PerfReport: 7.36%, PulseAccumulating -- 2.39e+00 / 2.95e+00, 4.05e-01 / 1.73e-01 | 20260505 10:13:01.545 | INFO | PerfReport: 3.78%, Metrics -- 1.23e+00 / 1.53e+00, 3.55e-01 / 1.51e-01 | 20260505 10:13:01.545 | INFO | PerfReport: 1.33%, PulseSerializing -- 4.32e-01 / 4.78e-01, 8.01e-02 / 2.72e-02 | 20260505 10:13:01.545 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 10:13:01.545 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:13:01.545 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:13:01.545 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.40e+01 / 0.00e+00 | 20260505 10:13:01.545 | INFO | PerfReport: 22.55%, Upload -- 2.29e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260505 10:13:01.545 | INFO | PerfReport: 21.76%, Download -- 2.21e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260505 10:13:01.545 | INFO | PerfReport: 31.95%, Compute -- 3.24e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 10:13:01.545 | INFO | PerfReport: 1.88%, LockUpload -- 1.91e+00 / 0.00e+00, 7.52e+00 / 0.00e+00 | 20260505 10:13:01.545 | INFO | PerfReport: 1.12%, LockDownload -- 1.14e+00 / 0.00e+00, 2.71e+00 / 0.00e+00 | 20260505 10:13:01.545 | INFO | PerfReport: 20.72%, LockCompute -- 2.10e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260505 10:13:01.545 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.17e-02 / 0.00e+00 | 20260505 10:13:01.673 | INFO | Starting a round of Baz IO | 20260505 10:13:04.934 | INFO | Populating RTMetrics report | 20260505 10:13:04.934 | INFO | RTMetrics report populated | 20260505 10:13:04.934 | INFO | Opening tmp RTMetrics json file | 20260505 10:13:04.941 | INFO | Serializing RTMetrics to json | 20260505 10:13:04.977 | INFO | Serialization done, swapping tmp file now... | 20260505 10:13:05.036 | INFO | RTMetrics done | 20260505 10:13:05.119 | INFO | Finished a round of Baz IO: Wrote 2.421 GiB in 3.446 seconds | 20260505 10:13:05.119 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.21% / 0.01% / 74.78% | 20260505 10:13:05.119 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.59901% / 7.94729e-06% / 6.35783e-05% | 20260505 10:13:05.119 | INFO | Baz IO overall: | 20260505 10:13:05.119 | INFO | BytesWritten : 1402227354845 | 20260505 10:13:05.119 | INFO | NumEvents : 667569686993 | 20260505 10:13:06.677 | INFO | Analyzing chunk frames = [2400256,2400768) | 20260505 10:13:06.677 | INFO | MemInfo: { | 20260505 10:13:06.677 | INFO | "gpu" : " 17 GB", | 20260505 10:13:06.677 | INFO | "huge" : "", | 20260505 10:13:06.677 | INFO | "huge_shared" : " 27 GB", | 20260505 10:13:06.677 | INFO | "malloc" : " 14 GB", | 20260505 10:13:06.677 | INFO | "pinned" : " 29 GB" | 20260505 10:13:06.677 | INFO | } | 20260505 10:13:06.677 | INFO | Perf report for the last 5.13405 seconds: | 20260505 10:13:06.677 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:13:06.677 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:06.677 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:06.677 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:06.677 | INFO | ANALYSIS : 64.9%, 2.9, 99.2ms, 0 | 20260505 10:13:06.677 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:06.677 | INFO | RT_METRICS : 47.7%, 1.0, 25.0ms, 0 | 20260505 10:13:06.677 | INFO | PRE_HQ : 63.1%, 1.8, 59.8ms, 0 | 20260505 10:13:06.677 | INFO | BAZWRITER : 66.4%, 4.2, 142.5ms, 0 | 20260505 10:13:06.678 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260505 10:13:06.678 | INFO | Uploaded 36GiB to GPU in 2.27602s (15.8171GiB/s) | 20260505 10:13:06.678 | INFO | Downloaded 38GiB to GPU in 2.37611s (15.9925GiB/s) | 20260505 10:13:11.825 | INFO | Analyzing chunk frames = [2400768,2401280) | 20260505 10:13:11.825 | INFO | MemInfo: { | 20260505 10:13:11.825 | INFO | "gpu" : " 17 GB", | 20260505 10:13:11.825 | INFO | "huge" : "", | 20260505 10:13:11.825 | INFO | "huge_shared" : " 27 GB", | 20260505 10:13:11.825 | INFO | "malloc" : " 14 GB", | 20260505 10:13:11.825 | INFO | "pinned" : " 29 GB" | 20260505 10:13:11.825 | INFO | } | 20260505 10:13:11.825 | INFO | Perf report for the last 5.14729 seconds: | 20260505 10:13:11.825 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:13:11.825 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:11.825 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:11.825 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:11.825 | INFO | ANALYSIS : 64.2%, 3.0, 100.4ms, 0 | 20260505 10:13:11.825 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:11.825 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:11.825 | INFO | PRE_HQ : 29.5%, 1.0, 15.4ms, 0 | 20260505 10:13:11.825 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:11.825 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:11.825 | INFO | Uploaded 36GiB to GPU in 2.19413s (16.4074GiB/s) | 20260505 10:13:11.825 | INFO | Downloaded 34GiB to GPU in 2.10573s (16.1464GiB/s) | 20260505 10:13:16.961 | INFO | Analyzing chunk frames = [2401280,2401792) | 20260505 10:13:16.961 | INFO | MemInfo: { | 20260505 10:13:16.961 | INFO | "gpu" : " 17 GB", | 20260505 10:13:16.961 | INFO | "huge" : "", | 20260505 10:13:16.961 | INFO | "huge_shared" : " 27 GB", | 20260505 10:13:16.961 | INFO | "malloc" : " 14 GB", | 20260505 10:13:16.961 | INFO | "pinned" : " 29 GB" | 20260505 10:13:16.961 | INFO | } | 20260505 10:13:16.961 | INFO | Perf report for the last 5.13671 seconds: | 20260505 10:13:16.961 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:13:16.961 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:16.961 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:16.961 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:16.961 | INFO | ANALYSIS : 63.1%, 3.0, 98.1ms, 0 | 20260505 10:13:16.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:16.962 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:16.962 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 10:13:16.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:16.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:16.962 | INFO | Uploaded 36GiB to GPU in 2.16986s (16.5909GiB/s) | 20260505 10:13:16.962 | INFO | Downloaded 34GiB to GPU in 2.06114s (16.4957GiB/s) | 20260505 10:13:22.101 | INFO | Analyzing chunk frames = [2401792,2402304) | 20260505 10:13:22.101 | INFO | MemInfo: { | 20260505 10:13:22.101 | INFO | "gpu" : " 17 GB", | 20260505 10:13:22.101 | INFO | "huge" : "", | 20260505 10:13:22.101 | INFO | "huge_shared" : " 27 GB", | 20260505 10:13:22.101 | INFO | "malloc" : " 14 GB", | 20260505 10:13:22.101 | INFO | "pinned" : " 29 GB" | 20260505 10:13:22.101 | INFO | } | 20260505 10:13:22.101 | INFO | Perf report for the last 5.13946 seconds: | 20260505 10:13:22.101 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:13:22.101 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:22.101 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:22.101 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:22.101 | INFO | ANALYSIS : 63.0%, 3.0, 97.9ms, 0 | 20260505 10:13:22.101 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:22.101 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:22.101 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:13:22.101 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:22.101 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:22.101 | INFO | Uploaded 36GiB to GPU in 2.16603s (16.6203GiB/s) | 20260505 10:13:22.101 | INFO | Downloaded 34GiB to GPU in 2.07217s (16.4079GiB/s) | 20260505 10:13:27.232 | INFO | Analyzing chunk frames = [2402304,2402816) | 20260505 10:13:27.232 | INFO | MemInfo: { | 20260505 10:13:27.232 | INFO | "gpu" : " 17 GB", | 20260505 10:13:27.232 | INFO | "huge" : "", | 20260505 10:13:27.232 | INFO | "huge_shared" : " 27 GB", | 20260505 10:13:27.232 | INFO | "malloc" : " 14 GB", | 20260505 10:13:27.232 | INFO | "pinned" : " 29 GB" | 20260505 10:13:27.232 | INFO | } | 20260505 10:13:27.232 | INFO | Perf report for the last 5.13151 seconds: | 20260505 10:13:27.232 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:13:27.232 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:27.232 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:27.233 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:27.233 | INFO | ANALYSIS : 63.9%, 3.0, 99.6ms, 0 | 20260505 10:13:27.233 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:27.233 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:27.233 | INFO | PRE_HQ : 29.8%, 1.0, 15.4ms, 0 | 20260505 10:13:27.233 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:27.233 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:27.233 | INFO | Uploaded 36GiB to GPU in 2.23838s (16.083GiB/s) | 20260505 10:13:27.233 | INFO | Downloaded 34GiB to GPU in 2.10917s (16.1201GiB/s) | 20260505 10:13:32.373 | INFO | Analyzing chunk frames = [2402816,2403328) | 20260505 10:13:32.373 | INFO | MemInfo: { | 20260505 10:13:32.373 | INFO | "gpu" : " 17 GB", | 20260505 10:13:32.373 | INFO | "huge" : "", | 20260505 10:13:32.373 | INFO | "huge_shared" : " 27 GB", | 20260505 10:13:32.373 | INFO | "malloc" : " 14 GB", | 20260505 10:13:32.373 | INFO | "pinned" : " 29 GB" | 20260505 10:13:32.373 | INFO | } | 20260505 10:13:32.373 | INFO | Perf report for the last 5.14082 seconds: | 20260505 10:13:32.373 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:13:32.373 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:32.373 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:32.373 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:32.373 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 10:13:32.373 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:32.373 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:32.373 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 10:13:32.373 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:32.373 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:32.373 | INFO | Uploaded 36GiB to GPU in 2.16412s (16.6349GiB/s) | 20260505 10:13:32.374 | INFO | Downloaded 34GiB to GPU in 2.06392s (16.4735GiB/s) | 20260505 10:13:37.522 | INFO | Analyzing chunk frames = [2403328,2403840) | 20260505 10:13:37.522 | INFO | MemInfo: { | 20260505 10:13:37.522 | INFO | "gpu" : " 17 GB", | 20260505 10:13:37.522 | INFO | "huge" : "", | 20260505 10:13:37.522 | INFO | "huge_shared" : " 27 GB", | 20260505 10:13:37.522 | INFO | "malloc" : " 14 GB", | 20260505 10:13:37.522 | INFO | "pinned" : " 29 GB" | 20260505 10:13:37.522 | INFO | } | 20260505 10:13:37.522 | INFO | Perf report for the last 5.1489 seconds: | 20260505 10:13:37.522 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:13:37.522 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:37.522 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:37.522 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:37.522 | INFO | ANALYSIS : 63.2%, 3.0, 98.9ms, 0 | 20260505 10:13:37.522 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:37.522 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:37.522 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 10:13:37.522 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:37.522 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:37.522 | INFO | Uploaded 36GiB to GPU in 2.15964s (16.6695GiB/s) | 20260505 10:13:37.523 | INFO | Downloaded 34GiB to GPU in 2.1539s (15.7853GiB/s) | 20260505 10:13:42.666 | INFO | Analyzing chunk frames = [2403840,2404352) | 20260505 10:13:42.666 | INFO | MemInfo: { | 20260505 10:13:42.666 | INFO | "gpu" : " 17 GB", | 20260505 10:13:42.666 | INFO | "huge" : "", | 20260505 10:13:42.666 | INFO | "huge_shared" : " 27 GB", | 20260505 10:13:42.666 | INFO | "malloc" : " 14 GB", | 20260505 10:13:42.666 | INFO | "pinned" : " 29 GB" | 20260505 10:13:42.666 | INFO | } | 20260505 10:13:42.666 | INFO | Perf report for the last 5.14351 seconds: | 20260505 10:13:42.666 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:13:42.666 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:42.666 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:42.666 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:42.666 | INFO | ANALYSIS : 63.9%, 3.0, 99.9ms, 0 | 20260505 10:13:42.666 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:42.666 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:42.666 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:13:42.666 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:42.666 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:42.666 | INFO | Uploaded 36GiB to GPU in 2.22174s (16.2035GiB/s) | 20260505 10:13:42.666 | INFO | Downloaded 34GiB to GPU in 2.09073s (16.2623GiB/s) | 20260505 10:13:42.806 | INFO | Starting a round of Baz IO | 20260505 10:13:46.071 | INFO | Populating RTMetrics report | 20260505 10:13:46.071 | INFO | RTMetrics report populated | 20260505 10:13:46.071 | INFO | Opening tmp RTMetrics json file | 20260505 10:13:46.089 | INFO | Serializing RTMetrics to json | 20260505 10:13:46.115 | INFO | Serialization done, swapping tmp file now... | 20260505 10:13:46.115 | INFO | RTMetrics done | 20260505 10:13:46.225 | INFO | Finished a round of Baz IO: Wrote 2.42 GiB in 3.419 seconds | 20260505 10:13:46.225 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.20% / 0.01% / 74.79% | 20260505 10:13:46.225 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.5918% / 7.94729e-06% / 5.5631e-05% | 20260505 10:13:46.225 | INFO | Baz IO overall: | 20260505 10:13:46.225 | INFO | BytesWritten : 1404826120766 | 20260505 10:13:46.225 | INFO | NumEvents : 668809730512 | 20260505 10:13:47.792 | INFO | Analyzing chunk frames = [2404352,2404864) | 20260505 10:13:47.792 | INFO | MemInfo: { | 20260505 10:13:47.792 | INFO | "gpu" : " 17 GB", | 20260505 10:13:47.792 | INFO | "huge" : "", | 20260505 10:13:47.792 | INFO | "huge_shared" : " 27 GB", | 20260505 10:13:47.792 | INFO | "malloc" : " 14 GB", | 20260505 10:13:47.792 | INFO | "pinned" : " 29 GB" | 20260505 10:13:47.792 | INFO | } | 20260505 10:13:47.792 | INFO | Perf report for the last 5.12612 seconds: | 20260505 10:13:47.792 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:13:47.792 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:47.792 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:47.792 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:47.792 | INFO | ANALYSIS : 65.3%, 2.9, 99.8ms, 0 | 20260505 10:13:47.792 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:47.792 | INFO | RT_METRICS : 46.7%, 1.0, 24.4ms, 0 | 20260505 10:13:47.792 | INFO | PRE_HQ : 62.0%, 1.9, 59.7ms, 0 | 20260505 10:13:47.792 | INFO | BAZWRITER : 66.1%, 4.2, 140.7ms, 0 | 20260505 10:13:47.792 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 10:13:47.792 | INFO | Uploaded 36GiB to GPU in 2.22992s (16.1441GiB/s) | 20260505 10:13:47.792 | INFO | Downloaded 38GiB to GPU in 2.29705s (16.543GiB/s) | 20260505 10:13:52.929 | INFO | Analyzing chunk frames = [2404864,2405376) | 20260505 10:13:52.929 | INFO | MemInfo: { | 20260505 10:13:52.929 | INFO | "gpu" : " 17 GB", | 20260505 10:13:52.929 | INFO | "huge" : "", | 20260505 10:13:52.929 | INFO | "huge_shared" : " 27 GB", | 20260505 10:13:52.930 | INFO | "malloc" : " 14 GB", | 20260505 10:13:52.930 | INFO | "pinned" : " 29 GB" | 20260505 10:13:52.930 | INFO | } | 20260505 10:13:52.930 | INFO | Perf report for the last 5.13769 seconds: | 20260505 10:13:52.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:13:52.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:52.930 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:52.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:52.930 | INFO | ANALYSIS : 64.4%, 3.0, 100.6ms, 0 | 20260505 10:13:52.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:52.930 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:52.930 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:13:52.930 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:52.930 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:52.930 | INFO | Uploaded 36GiB to GPU in 2.24672s (16.0233GiB/s) | 20260505 10:13:52.930 | INFO | Downloaded 34GiB to GPU in 2.10136s (16.18GiB/s) | 20260505 10:13:58.077 | INFO | Analyzing chunk frames = [2405376,2405888) | 20260505 10:13:58.077 | INFO | MemInfo: { | 20260505 10:13:58.077 | INFO | "gpu" : " 17 GB", | 20260505 10:13:58.077 | INFO | "huge" : "", | 20260505 10:13:58.077 | INFO | "huge_shared" : " 27 GB", | 20260505 10:13:58.077 | INFO | "malloc" : " 14 GB", | 20260505 10:13:58.077 | INFO | "pinned" : " 29 GB" | 20260505 10:13:58.077 | INFO | } | 20260505 10:13:58.077 | INFO | Perf report for the last 5.14716 seconds: | 20260505 10:13:58.077 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:13:58.077 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:58.077 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:58.077 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:58.077 | INFO | ANALYSIS : 63.0%, 3.0, 98.1ms, 0 | 20260505 10:13:58.077 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:58.077 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:58.077 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:13:58.077 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:58.077 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:13:58.077 | INFO | Uploaded 36GiB to GPU in 2.14466s (16.7859GiB/s) | 20260505 10:13:58.077 | INFO | Downloaded 34GiB to GPU in 2.06543s (16.4615GiB/s) | 20260505 10:14:03.215 | INFO | Analyzing chunk frames = [2405888,2406400) | 20260505 10:14:03.215 | INFO | MemInfo: { | 20260505 10:14:03.215 | INFO | "gpu" : " 17 GB", | 20260505 10:14:03.215 | INFO | "huge" : "", | 20260505 10:14:03.215 | INFO | "huge_shared" : " 27 GB", | 20260505 10:14:03.215 | INFO | "malloc" : " 14 GB", | 20260505 10:14:03.215 | INFO | "pinned" : " 29 GB" | 20260505 10:14:03.215 | INFO | } | 20260505 10:14:03.215 | INFO | Perf report for the last 5.1387 seconds: | 20260505 10:14:03.215 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:14:03.215 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:03.215 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:03.215 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:03.215 | INFO | ANALYSIS : 63.9%, 3.0, 99.2ms, 0 | 20260505 10:14:03.215 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:03.216 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:03.216 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:14:03.216 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:03.216 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:03.216 | INFO | Uploaded 36GiB to GPU in 2.22437s (16.1843GiB/s) | 20260505 10:14:03.216 | INFO | Downloaded 34GiB to GPU in 2.12234s (16.0201GiB/s) | 20260505 10:14:08.358 | INFO | Analyzing chunk frames = [2406400,2406912) | 20260505 10:14:08.358 | INFO | MemInfo: { | 20260505 10:14:08.358 | INFO | "gpu" : " 17 GB", | 20260505 10:14:08.358 | INFO | "huge" : "", | 20260505 10:14:08.358 | INFO | "huge_shared" : " 27 GB", | 20260505 10:14:08.358 | INFO | "malloc" : " 14 GB", | 20260505 10:14:08.358 | INFO | "pinned" : " 29 GB" | 20260505 10:14:08.358 | INFO | } | 20260505 10:14:08.358 | INFO | Perf report for the last 5.14231 seconds: | 20260505 10:14:08.358 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:14:08.358 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:08.358 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:08.358 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:08.358 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 10:14:08.358 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:08.358 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:08.358 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:14:08.358 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:08.358 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:08.358 | INFO | Uploaded 36GiB to GPU in 2.23494s (16.1078GiB/s) | 20260505 10:14:08.358 | INFO | Downloaded 34GiB to GPU in 2.10744s (16.1333GiB/s) | 20260505 10:14:13.489 | INFO | Analyzing chunk frames = [2406912,2407424) | 20260505 10:14:13.489 | INFO | MemInfo: { | 20260505 10:14:13.489 | INFO | "gpu" : " 17 GB", | 20260505 10:14:13.489 | INFO | "huge" : "", | 20260505 10:14:13.489 | INFO | "huge_shared" : " 27 GB", | 20260505 10:14:13.489 | INFO | "malloc" : " 14 GB", | 20260505 10:14:13.489 | INFO | "pinned" : " 29 GB" | 20260505 10:14:13.489 | INFO | } | 20260505 10:14:13.489 | INFO | Perf report for the last 5.13102 seconds: | 20260505 10:14:13.489 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:14:13.489 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:13.489 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:13.489 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:13.489 | INFO | ANALYSIS : 64.1%, 3.0, 99.9ms, 0 | 20260505 10:14:13.489 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:13.489 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:13.489 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:14:13.489 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:13.489 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:13.489 | INFO | Uploaded 36GiB to GPU in 2.23677s (16.0947GiB/s) | 20260505 10:14:13.489 | INFO | Downloaded 34GiB to GPU in 2.10003s (16.1903GiB/s) | 20260505 10:14:18.631 | INFO | Analyzing chunk frames = [2407424,2407936) | 20260505 10:14:18.631 | INFO | MemInfo: { | 20260505 10:14:18.631 | INFO | "gpu" : " 17 GB", | 20260505 10:14:18.631 | INFO | "huge" : "", | 20260505 10:14:18.631 | INFO | "huge_shared" : " 27 GB", | 20260505 10:14:18.631 | INFO | "malloc" : " 14 GB", | 20260505 10:14:18.631 | INFO | "pinned" : " 29 GB" | 20260505 10:14:18.631 | INFO | } | 20260505 10:14:18.631 | INFO | Perf report for the last 5.1422 seconds: | 20260505 10:14:18.631 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:14:18.631 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:18.631 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:18.631 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:18.631 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 10:14:18.631 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:18.631 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:18.631 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 10:14:18.631 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:18.631 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:18.631 | INFO | Uploaded 36GiB to GPU in 2.207s (16.3117GiB/s) | 20260505 10:14:18.631 | INFO | Downloaded 34GiB to GPU in 2.14639s (15.8405GiB/s) | 20260505 10:14:23.776 | INFO | Analyzing chunk frames = [2407936,2408448) | 20260505 10:14:23.776 | INFO | MemInfo: { | 20260505 10:14:23.776 | INFO | "gpu" : " 17 GB", | 20260505 10:14:23.776 | INFO | "huge" : "", | 20260505 10:14:23.776 | INFO | "huge_shared" : " 27 GB", | 20260505 10:14:23.776 | INFO | "malloc" : " 14 GB", | 20260505 10:14:23.776 | INFO | "pinned" : " 29 GB" | 20260505 10:14:23.776 | INFO | } | 20260505 10:14:23.776 | INFO | Perf report for the last 5.14493 seconds: | 20260505 10:14:23.776 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:14:23.776 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:23.776 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:23.776 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:23.776 | INFO | ANALYSIS : 64.5%, 3.0, 100.9ms, 0 | 20260505 10:14:23.776 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:23.776 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:23.776 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 10:14:23.776 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:23.776 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:23.776 | INFO | Uploaded 36GiB to GPU in 2.24987s (16.0009GiB/s) | 20260505 10:14:23.776 | INFO | Downloaded 34GiB to GPU in 2.08579s (16.3008GiB/s) | 20260505 10:14:23.776 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 10:14:23.776 | INFO | Total mem usage: 15.3 GiB | 20260505 10:14:23.776 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 10:14:23.776 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.23%) | 20260505 10:14:23.776 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 10:14:23.776 | INFO | High Water Breakdown: | 20260505 10:14:23.776 | INFO | 61% : 8.79 GiB ( 94% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 10:14:23.776 | INFO | 38.9% : 5.61 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 10:14:23.776 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 10:14:23.777 | INFO | ------------------------------------------ | 20260505 10:14:23.777 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:14:23.777 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 10:14:23.777 | INFO | Total mem usage: 26.5 GiB | 20260505 10:14:23.777 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 10:14:23.777 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 10:14:23.777 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 10:14:23.777 | INFO | High Water Breakdown: | 20260505 10:14:23.777 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:14:23.777 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:14:23.777 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:14:23.777 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:14:23.777 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:14:23.777 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:14:23.777 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 10:14:23.777 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 10:14:23.777 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:14:23.777 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 10:14:23.777 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 10:14:23.777 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 10:14:23.777 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 10:14:23.777 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 10:14:23.777 | INFO | ------------------------------------------ | 20260505 10:14:23.777 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:14:23.777 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 10:14:23.777 | INFO | Total mem usage: 25.6 GiB | 20260505 10:14:23.777 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 10:14:23.777 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 10:14:23.777 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 10:14:23.777 | INFO | High Water Breakdown: | 20260505 10:14:23.777 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 10:14:23.777 | INFO | ------------------------------------------ | 20260505 10:14:23.777 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:14:23.777 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 10:14:23.777 | INFO | Total mem usage: 15.7 GiB | 20260505 10:14:23.777 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 10:14:23.777 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 10:14:23.777 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 10:14:23.777 | INFO | High Water Breakdown: | 20260505 10:14:23.777 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 10:14:23.777 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 10:14:23.777 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 10:14:23.777 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 10:14:23.777 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 10:14:23.777 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 10:14:23.777 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 10:14:23.777 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 10:14:23.777 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 10:14:23.777 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:14:23.777 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 10:14:23.777 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:14:23.777 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 10:14:23.777 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 10:14:23.777 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 10:14:23.777 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 10:14:23.777 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:14:23.777 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 10:14:23.777 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:14:23.777 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:14:23.777 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:14:23.777 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 10:14:23.777 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 10:14:23.777 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:14:23.777 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 10:14:23.777 | INFO | ------------------------------------------ | 20260505 10:14:23.777 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:14:23.777 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 10:14:23.777 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:14:23.777 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:14:23.777 | INFO | PerfReport: Full Execution -- 3.24e+01 / 7.71e+01, 1.38e+01 / 2.21e+00 | 20260505 10:14:23.777 | INFO | PerfReport: 0.29%, AggregateStats -- 9.34e-02 / 1.04e-01, 1.07e-02 / 5.90e-03 | 20260505 10:14:23.777 | INFO | PerfReport: 11.68%, Baseline -- 3.78e+00 / 4.47e+00, 6.13e-01 / 2.19e-01 | 20260505 10:14:23.777 | INFO | PerfReport: 3.33%, Histogram -- 1.08e+00 / 1.49e+00, 2.10e-01 / 7.93e-02 | 20260505 10:14:23.777 | INFO | PerfReport: 15.55%, DME -- 4.03e+01 / 4.42e+01, 4.30e+00 / 2.05e+00 | 20260505 10:14:23.777 | INFO | PerfReport: 56.76%, FrameLabeling -- 1.84e+01 / 2.19e+01, 1.90e+00 / 2.20e+00 | 20260505 10:14:23.777 | INFO | PerfReport: 7.30%, PulseAccumulating -- 2.37e+00 / 2.92e+00, 3.91e-01 / 2.10e-01 | 20260505 10:14:23.777 | INFO | PerfReport: 3.76%, Metrics -- 1.22e+00 / 1.50e+00, 3.50e-01 / 1.72e-01 | 20260505 10:14:23.777 | INFO | PerfReport: 1.33%, PulseSerializing -- 4.31e-01 / 4.80e-01, 8.06e-02 / 3.15e-02 | 20260505 10:14:23.777 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 10:14:23.777 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:14:23.777 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:14:23.777 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260505 10:14:23.777 | INFO | PerfReport: 22.69%, Upload -- 2.30e+01 / 0.00e+00, 8.36e+00 / 0.00e+00 | 20260505 10:14:23.777 | INFO | PerfReport: 21.86%, Download -- 2.22e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260505 10:14:23.777 | INFO | PerfReport: 31.95%, Compute -- 3.24e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 10:14:23.777 | INFO | PerfReport: 1.89%, LockUpload -- 1.91e+00 / 0.00e+00, 7.26e+00 / 0.00e+00 | 20260505 10:14:23.777 | INFO | PerfReport: 1.07%, LockDownload -- 1.09e+00 / 0.00e+00, 2.56e+00 / 0.00e+00 | 20260505 10:14:23.777 | INFO | PerfReport: 20.53%, LockCompute -- 2.08e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260505 10:14:23.777 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.59e-02 / 0.00e+00 | 20260505 10:14:23.916 | INFO | Starting a round of Baz IO | 20260505 10:14:27.174 | INFO | Populating RTMetrics report | 20260505 10:14:27.174 | INFO | RTMetrics report populated | 20260505 10:14:27.174 | INFO | Opening tmp RTMetrics json file | 20260505 10:14:27.223 | INFO | Serializing RTMetrics to json | 20260505 10:14:27.225 | INFO | Serialization done, swapping tmp file now... | 20260505 10:14:27.227 | INFO | RTMetrics done | 20260505 10:14:27.339 | INFO | Finished a round of Baz IO: Wrote 2.418 GiB in 3.423 seconds | 20260505 10:14:27.339 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.19% / 0.01% / 74.80% | 20260505 10:14:27.339 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.58329% / 1.19209e-05% / 0.000111262% | 20260505 10:14:27.339 | INFO | Baz IO overall: | 20260505 10:14:27.339 | INFO | BytesWritten : 1407422480822 | 20260505 10:14:27.339 | INFO | NumEvents : 670047885622 | 20260505 10:14:28.914 | INFO | Analyzing chunk frames = [2408448,2408960) | 20260505 10:14:28.914 | INFO | MemInfo: { | 20260505 10:14:28.914 | INFO | "gpu" : " 17 GB", | 20260505 10:14:28.914 | INFO | "huge" : "", | 20260505 10:14:28.914 | INFO | "huge_shared" : " 27 GB", | 20260505 10:14:28.914 | INFO | "malloc" : " 14 GB", | 20260505 10:14:28.914 | INFO | "pinned" : " 29 GB" | 20260505 10:14:28.914 | INFO | } | 20260505 10:14:28.914 | INFO | Perf report for the last 5.13836 seconds: | 20260505 10:14:28.914 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:14:28.914 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:28.914 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:28.914 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:28.914 | INFO | ANALYSIS : 65.0%, 2.9, 99.3ms, 0 | 20260505 10:14:28.914 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:28.914 | INFO | RT_METRICS : 46.6%, 1.0, 24.5ms, 0 | 20260505 10:14:28.914 | INFO | PRE_HQ : 61.6%, 1.9, 59.7ms, 0 | 20260505 10:14:28.914 | INFO | BAZWRITER : 66.0%, 4.2, 141.6ms, 0 | 20260505 10:14:28.914 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 10:14:28.914 | INFO | Uploaded 36GiB to GPU in 2.2022s (16.3473GiB/s) | 20260505 10:14:28.915 | INFO | Downloaded 38GiB to GPU in 2.3033s (16.498GiB/s) | 20260505 10:14:34.052 | INFO | Analyzing chunk frames = [2408960,2409472) | 20260505 10:14:34.052 | INFO | MemInfo: { | 20260505 10:14:34.052 | INFO | "gpu" : " 17 GB", | 20260505 10:14:34.052 | INFO | "huge" : "", | 20260505 10:14:34.052 | INFO | "huge_shared" : " 27 GB", | 20260505 10:14:34.052 | INFO | "malloc" : " 14 GB", | 20260505 10:14:34.052 | INFO | "pinned" : " 29 GB" | 20260505 10:14:34.052 | INFO | } | 20260505 10:14:34.052 | INFO | Perf report for the last 5.13795 seconds: | 20260505 10:14:34.052 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:14:34.052 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:34.052 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:34.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:34.052 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 10:14:34.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:34.052 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:34.052 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:14:34.052 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:34.053 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:34.053 | INFO | Uploaded 36GiB to GPU in 2.20717s (16.3105GiB/s) | 20260505 10:14:34.053 | INFO | Downloaded 34GiB to GPU in 2.11278s (16.0926GiB/s) | 20260505 10:14:39.183 | INFO | Analyzing chunk frames = [2409472,2409984) | 20260505 10:14:39.183 | INFO | MemInfo: { | 20260505 10:14:39.183 | INFO | "gpu" : " 17 GB", | 20260505 10:14:39.183 | INFO | "huge" : "", | 20260505 10:14:39.183 | INFO | "huge_shared" : " 27 GB", | 20260505 10:14:39.184 | INFO | "malloc" : " 14 GB", | 20260505 10:14:39.184 | INFO | "pinned" : " 29 GB" | 20260505 10:14:39.184 | INFO | } | 20260505 10:14:39.184 | INFO | Perf report for the last 5.1313 seconds: | 20260505 10:14:39.184 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:14:39.184 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:39.184 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:39.184 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:39.184 | INFO | ANALYSIS : 63.3%, 3.0, 98.3ms, 0 | 20260505 10:14:39.184 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:39.184 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:39.184 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 10:14:39.184 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:39.184 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:39.184 | INFO | Uploaded 36GiB to GPU in 2.15403s (16.7128GiB/s) | 20260505 10:14:39.184 | INFO | Downloaded 34GiB to GPU in 2.06999s (16.4252GiB/s) | 20260505 10:14:44.332 | INFO | Analyzing chunk frames = [2409984,2410496) | 20260505 10:14:44.332 | INFO | MemInfo: { | 20260505 10:14:44.332 | INFO | "gpu" : " 17 GB", | 20260505 10:14:44.332 | INFO | "huge" : "", | 20260505 10:14:44.332 | INFO | "huge_shared" : " 27 GB", | 20260505 10:14:44.332 | INFO | "malloc" : " 14 GB", | 20260505 10:14:44.332 | INFO | "pinned" : " 29 GB" | 20260505 10:14:44.332 | INFO | } | 20260505 10:14:44.332 | INFO | Perf report for the last 5.14868 seconds: | 20260505 10:14:44.332 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:14:44.332 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:44.332 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:44.332 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:44.332 | INFO | ANALYSIS : 63.2%, 3.0, 98.3ms, 0 | 20260505 10:14:44.332 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:44.332 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:44.332 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:14:44.332 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:44.332 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:44.333 | INFO | Uploaded 36GiB to GPU in 2.17015s (16.5887GiB/s) | 20260505 10:14:44.333 | INFO | Downloaded 34GiB to GPU in 2.07103s (16.4169GiB/s) | 20260505 10:14:49.470 | INFO | Analyzing chunk frames = [2410496,2411008) | 20260505 10:14:49.470 | INFO | MemInfo: { | 20260505 10:14:49.470 | INFO | "gpu" : " 17 GB", | 20260505 10:14:49.470 | INFO | "huge" : "", | 20260505 10:14:49.470 | INFO | "huge_shared" : " 27 GB", | 20260505 10:14:49.470 | INFO | "malloc" : " 14 GB", | 20260505 10:14:49.470 | INFO | "pinned" : " 29 GB" | 20260505 10:14:49.470 | INFO | } | 20260505 10:14:49.470 | INFO | Perf report for the last 5.13771 seconds: | 20260505 10:14:49.470 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:14:49.470 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:49.470 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:49.470 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:49.470 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 10:14:49.470 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:49.470 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:49.470 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 10:14:49.470 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:49.470 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:49.470 | INFO | Uploaded 36GiB to GPU in 2.1594s (16.6713GiB/s) | 20260505 10:14:49.470 | INFO | Downloaded 34GiB to GPU in 2.12176s (16.0244GiB/s) | 20260505 10:14:54.615 | INFO | Analyzing chunk frames = [2411008,2411520) | 20260505 10:14:54.615 | INFO | MemInfo: { | 20260505 10:14:54.615 | INFO | "gpu" : " 17 GB", | 20260505 10:14:54.615 | INFO | "huge" : "", | 20260505 10:14:54.615 | INFO | "huge_shared" : " 27 GB", | 20260505 10:14:54.615 | INFO | "malloc" : " 14 GB", | 20260505 10:14:54.615 | INFO | "pinned" : " 29 GB" | 20260505 10:14:54.615 | INFO | } | 20260505 10:14:54.615 | INFO | Perf report for the last 5.14473 seconds: | 20260505 10:14:54.615 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:14:54.615 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:54.615 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:54.615 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:54.615 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 10:14:54.615 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:54.615 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:54.615 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:14:54.615 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:54.615 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:54.615 | INFO | Uploaded 36GiB to GPU in 2.21492s (16.2534GiB/s) | 20260505 10:14:54.615 | INFO | Downloaded 34GiB to GPU in 2.1006s (16.1858GiB/s) | 20260505 10:14:59.744 | INFO | Analyzing chunk frames = [2411520,2412032) | 20260505 10:14:59.744 | INFO | MemInfo: { | 20260505 10:14:59.744 | INFO | "gpu" : " 17 GB", | 20260505 10:14:59.744 | INFO | "huge" : "", | 20260505 10:14:59.744 | INFO | "huge_shared" : " 27 GB", | 20260505 10:14:59.744 | INFO | "malloc" : " 14 GB", | 20260505 10:14:59.744 | INFO | "pinned" : " 29 GB" | 20260505 10:14:59.744 | INFO | } | 20260505 10:14:59.744 | INFO | Perf report for the last 5.12972 seconds: | 20260505 10:14:59.744 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:14:59.745 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:59.745 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:59.745 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:59.745 | INFO | ANALYSIS : 63.8%, 3.0, 99.4ms, 0 | 20260505 10:14:59.745 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:59.745 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:59.745 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260505 10:14:59.745 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:59.745 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:14:59.745 | INFO | Uploaded 36GiB to GPU in 2.21026s (16.2877GiB/s) | 20260505 10:14:59.745 | INFO | Downloaded 34GiB to GPU in 2.12448s (16.0039GiB/s) | 20260505 10:15:04.886 | INFO | Analyzing chunk frames = [2412032,2412544) | 20260505 10:15:04.886 | INFO | MemInfo: { | 20260505 10:15:04.886 | INFO | "gpu" : " 17 GB", | 20260505 10:15:04.886 | INFO | "huge" : "", | 20260505 10:15:04.886 | INFO | "huge_shared" : " 27 GB", | 20260505 10:15:04.886 | INFO | "malloc" : " 14 GB", | 20260505 10:15:04.886 | INFO | "pinned" : " 29 GB" | 20260505 10:15:04.886 | INFO | } | 20260505 10:15:04.886 | INFO | Perf report for the last 5.14136 seconds: | 20260505 10:15:04.886 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:15:04.886 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:04.886 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:04.886 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:04.886 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 10:15:04.886 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:04.886 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:04.886 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 10:15:04.886 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:04.886 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:04.886 | INFO | Uploaded 36GiB to GPU in 2.18716s (16.4597GiB/s) | 20260505 10:15:04.886 | INFO | Downloaded 34GiB to GPU in 2.0295s (16.7529GiB/s) | 20260505 10:15:05.027 | INFO | Starting a round of Baz IO | 20260505 10:15:08.294 | INFO | Populating RTMetrics report | 20260505 10:15:08.294 | INFO | RTMetrics report populated | 20260505 10:15:08.294 | INFO | Opening tmp RTMetrics json file | 20260505 10:15:08.313 | INFO | Serializing RTMetrics to json | 20260505 10:15:08.362 | INFO | Serialization done, swapping tmp file now... | 20260505 10:15:08.365 | INFO | RTMetrics done | 20260505 10:15:08.429 | INFO | Finished a round of Baz IO: Wrote 2.42 GiB in 3.401 seconds | 20260505 10:15:08.429 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.18% / 0.01% / 74.81% | 20260505 10:15:08.429 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.58301% / 2.78155e-05% / 7.54992e-05% | 20260505 10:15:08.429 | INFO | Baz IO overall: | 20260505 10:15:08.429 | INFO | BytesWritten : 1410021025063 | 20260505 10:15:08.429 | INFO | NumEvents : 671287837622 | 20260505 10:15:10.026 | INFO | Analyzing chunk frames = [2412544,2413056) | 20260505 10:15:10.026 | INFO | MemInfo: { | 20260505 10:15:10.026 | INFO | "gpu" : " 17 GB", | 20260505 10:15:10.026 | INFO | "huge" : "", | 20260505 10:15:10.026 | INFO | "huge_shared" : " 27 GB", | 20260505 10:15:10.026 | INFO | "malloc" : " 14 GB", | 20260505 10:15:10.027 | INFO | "pinned" : " 29 GB" | 20260505 10:15:10.027 | INFO | } | 20260505 10:15:10.027 | INFO | Perf report for the last 5.1407 seconds: | 20260505 10:15:10.027 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:15:10.027 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:10.027 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:10.027 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:10.027 | INFO | ANALYSIS : 65.2%, 2.9, 99.8ms, 0 | 20260505 10:15:10.027 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:10.027 | INFO | RT_METRICS : 46.9%, 1.0, 24.6ms, 0 | 20260505 10:15:10.027 | INFO | PRE_HQ : 61.7%, 1.9, 59.6ms, 0 | 20260505 10:15:10.027 | INFO | BAZWRITER : 65.5%, 4.2, 140.0ms, 0 | 20260505 10:15:10.027 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260505 10:15:10.027 | INFO | Uploaded 36GiB to GPU in 2.20054s (16.3596GiB/s) | 20260505 10:15:10.027 | INFO | Downloaded 38GiB to GPU in 2.31103s (16.4429GiB/s) | 20260505 10:15:15.171 | INFO | Analyzing chunk frames = [2413056,2413568) | 20260505 10:15:15.171 | INFO | MemInfo: { | 20260505 10:15:15.171 | INFO | "gpu" : " 17 GB", | 20260505 10:15:15.171 | INFO | "huge" : "", | 20260505 10:15:15.171 | INFO | "huge_shared" : " 27 GB", | 20260505 10:15:15.171 | INFO | "malloc" : " 14 GB", | 20260505 10:15:15.171 | INFO | "pinned" : " 29 GB" | 20260505 10:15:15.171 | INFO | } | 20260505 10:15:15.172 | INFO | Perf report for the last 5.14496 seconds: | 20260505 10:15:15.172 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:15:15.172 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:15.172 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:15.172 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:15.172 | INFO | ANALYSIS : 64.4%, 3.0, 100.7ms, 0 | 20260505 10:15:15.172 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:15.172 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:15.172 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:15:15.172 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:15.172 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:15.172 | INFO | Uploaded 36GiB to GPU in 2.23848s (16.0823GiB/s) | 20260505 10:15:15.172 | INFO | Downloaded 34GiB to GPU in 2.08274s (16.3247GiB/s) | 20260505 10:15:20.309 | INFO | Analyzing chunk frames = [2413568,2414080) | 20260505 10:15:20.309 | INFO | MemInfo: { | 20260505 10:15:20.309 | INFO | "gpu" : " 17 GB", | 20260505 10:15:20.309 | INFO | "huge" : "", | 20260505 10:15:20.309 | INFO | "huge_shared" : " 27 GB", | 20260505 10:15:20.309 | INFO | "malloc" : " 14 GB", | 20260505 10:15:20.309 | INFO | "pinned" : " 29 GB" | 20260505 10:15:20.309 | INFO | } | 20260505 10:15:20.309 | INFO | Perf report for the last 5.13754 seconds: | 20260505 10:15:20.309 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:15:20.309 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:20.309 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:20.309 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:20.309 | INFO | ANALYSIS : 62.9%, 3.0, 97.9ms, 0 | 20260505 10:15:20.309 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:20.309 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:20.309 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 10:15:20.309 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:20.309 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:20.309 | INFO | Uploaded 36GiB to GPU in 2.16323s (16.6418GiB/s) | 20260505 10:15:20.309 | INFO | Downloaded 34GiB to GPU in 2.05952s (16.5087GiB/s) | 20260505 10:15:25.435 | INFO | Analyzing chunk frames = [2414080,2414592) | 20260505 10:15:25.435 | INFO | MemInfo: { | 20260505 10:15:25.435 | INFO | "gpu" : " 17 GB", | 20260505 10:15:25.435 | INFO | "huge" : "", | 20260505 10:15:25.435 | INFO | "huge_shared" : " 27 GB", | 20260505 10:15:25.435 | INFO | "malloc" : " 14 GB", | 20260505 10:15:25.435 | INFO | "pinned" : " 29 GB" | 20260505 10:15:25.435 | INFO | } | 20260505 10:15:25.435 | INFO | Perf report for the last 5.12615 seconds: | 20260505 10:15:25.435 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:15:25.435 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:25.435 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:25.435 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:25.435 | INFO | ANALYSIS : 63.2%, 3.0, 98.0ms, 0 | 20260505 10:15:25.435 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:25.435 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:25.435 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:15:25.435 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:25.435 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:25.435 | INFO | Uploaded 36GiB to GPU in 2.16901s (16.5974GiB/s) | 20260505 10:15:25.435 | INFO | Downloaded 34GiB to GPU in 2.07212s (16.4083GiB/s) | 20260505 10:15:30.583 | INFO | Analyzing chunk frames = [2414592,2415104) | 20260505 10:15:30.583 | INFO | MemInfo: { | 20260505 10:15:30.583 | INFO | "gpu" : " 17 GB", | 20260505 10:15:30.583 | INFO | "huge" : "", | 20260505 10:15:30.583 | INFO | "huge_shared" : " 27 GB", | 20260505 10:15:30.583 | INFO | "malloc" : " 14 GB", | 20260505 10:15:30.583 | INFO | "pinned" : " 29 GB" | 20260505 10:15:30.583 | INFO | } | 20260505 10:15:30.583 | INFO | Perf report for the last 5.14769 seconds: | 20260505 10:15:30.583 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:15:30.583 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:30.583 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:30.583 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:30.583 | INFO | ANALYSIS : 63.7%, 3.0, 99.6ms, 0 | 20260505 10:15:30.583 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:30.583 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:30.583 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:15:30.583 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:30.583 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:30.583 | INFO | Uploaded 36GiB to GPU in 2.23714s (16.092GiB/s) | 20260505 10:15:30.583 | INFO | Downloaded 34GiB to GPU in 2.10477s (16.1538GiB/s) | 20260505 10:15:35.727 | INFO | Analyzing chunk frames = [2415104,2415616) | 20260505 10:15:35.727 | INFO | MemInfo: { | 20260505 10:15:35.727 | INFO | "gpu" : " 17 GB", | 20260505 10:15:35.727 | INFO | "huge" : "", | 20260505 10:15:35.727 | INFO | "huge_shared" : " 27 GB", | 20260505 10:15:35.727 | INFO | "malloc" : " 14 GB", | 20260505 10:15:35.727 | INFO | "pinned" : " 29 GB" | 20260505 10:15:35.727 | INFO | } | 20260505 10:15:35.727 | INFO | Perf report for the last 5.14401 seconds: | 20260505 10:15:35.727 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:15:35.727 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:35.727 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:35.727 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:35.727 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 10:15:35.727 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:35.727 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:35.727 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 10:15:35.727 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:35.727 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:35.727 | INFO | Uploaded 36GiB to GPU in 2.20628s (16.317GiB/s) | 20260505 10:15:35.727 | INFO | Downloaded 34GiB to GPU in 2.09983s (16.1918GiB/s) | 20260505 10:15:40.862 | INFO | Analyzing chunk frames = [2415616,2416128) | 20260505 10:15:40.862 | INFO | MemInfo: { | 20260505 10:15:40.862 | INFO | "gpu" : " 17 GB", | 20260505 10:15:40.862 | INFO | "huge" : "", | 20260505 10:15:40.862 | INFO | "huge_shared" : " 27 GB", | 20260505 10:15:40.862 | INFO | "malloc" : " 14 GB", | 20260505 10:15:40.862 | INFO | "pinned" : " 29 GB" | 20260505 10:15:40.862 | INFO | } | 20260505 10:15:40.862 | INFO | Perf report for the last 5.13499 seconds: | 20260505 10:15:40.862 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:15:40.862 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:40.862 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:40.862 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:40.862 | INFO | ANALYSIS : 63.9%, 3.0, 99.6ms, 0 | 20260505 10:15:40.862 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:40.862 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:40.862 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 10:15:40.862 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:40.862 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:40.863 | INFO | Uploaded 36GiB to GPU in 2.17506s (16.5512GiB/s) | 20260505 10:15:40.863 | INFO | Downloaded 34GiB to GPU in 2.14646s (15.84GiB/s) | 20260505 10:15:45.994 | INFO | Analyzing chunk frames = [2416128,2416640) | 20260505 10:15:45.994 | INFO | MemInfo: { | 20260505 10:15:45.994 | INFO | "gpu" : " 17 GB", | 20260505 10:15:45.994 | INFO | "huge" : "", | 20260505 10:15:45.994 | INFO | "huge_shared" : " 27 GB", | 20260505 10:15:45.994 | INFO | "malloc" : " 14 GB", | 20260505 10:15:45.994 | INFO | "pinned" : " 29 GB" | 20260505 10:15:45.994 | INFO | } | 20260505 10:15:45.994 | INFO | Perf report for the last 5.13241 seconds: | 20260505 10:15:45.994 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:15:45.994 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:45.994 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:45.994 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:45.994 | INFO | ANALYSIS : 63.9%, 3.0, 99.6ms, 0 | 20260505 10:15:45.994 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:45.994 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:45.994 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 10:15:45.994 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:45.995 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:45.995 | INFO | Uploaded 36GiB to GPU in 2.18214s (16.4976GiB/s) | 20260505 10:15:45.995 | INFO | Downloaded 34GiB to GPU in 2.05809s (16.5202GiB/s) | 20260505 10:15:45.995 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 10:15:45.995 | INFO | Total mem usage: 15.3 GiB | 20260505 10:15:45.995 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 10:15:45.995 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.31%) | 20260505 10:15:45.995 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 10:15:45.995 | INFO | High Water Breakdown: | 20260505 10:15:45.995 | INFO | 60.9% : 8.78 GiB ( 93.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 10:15:45.995 | INFO | 38.9% : 5.61 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 10:15:45.995 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 10:15:45.995 | INFO | ------------------------------------------ | 20260505 10:15:45.995 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:15:45.995 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 10:15:45.995 | INFO | Total mem usage: 26.5 GiB | 20260505 10:15:45.995 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 10:15:45.995 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 10:15:45.995 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 10:15:45.995 | INFO | High Water Breakdown: | 20260505 10:15:45.995 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:15:45.995 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:15:45.995 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:15:45.995 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:15:45.995 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:15:45.995 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:15:45.995 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 10:15:45.995 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 10:15:45.995 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:15:45.995 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 10:15:45.995 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 10:15:45.995 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 10:15:45.995 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 10:15:45.995 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 10:15:45.995 | INFO | ------------------------------------------ | 20260505 10:15:45.995 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:15:45.995 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 10:15:45.995 | INFO | Total mem usage: 25.6 GiB | 20260505 10:15:45.995 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 10:15:45.995 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 10:15:45.995 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 10:15:45.995 | INFO | High Water Breakdown: | 20260505 10:15:45.995 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 10:15:45.995 | INFO | ------------------------------------------ | 20260505 10:15:45.995 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:15:45.995 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 10:15:45.995 | INFO | Total mem usage: 15.7 GiB | 20260505 10:15:45.995 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 10:15:45.995 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 10:15:45.995 | INFO | Unused Cache: 880 MiB (Waste: 5.48%) | 20260505 10:15:45.995 | INFO | High Water Breakdown: | 20260505 10:15:45.995 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 10:15:45.995 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 10:15:45.995 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 10:15:45.996 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 10:15:45.996 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 10:15:45.996 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 10:15:45.996 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 10:15:45.996 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 10:15:45.996 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 10:15:45.996 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:15:45.996 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 10:15:45.996 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:15:45.996 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 10:15:45.996 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 10:15:45.996 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 10:15:45.996 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 10:15:45.996 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:15:45.996 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 10:15:45.996 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:15:45.996 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:15:45.996 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:15:45.996 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 10:15:45.996 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 10:15:45.996 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:15:45.996 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 10:15:45.996 | INFO | ------------------------------------------ | 20260505 10:15:45.996 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:15:45.996 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 10:15:45.996 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:15:45.996 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:15:45.996 | INFO | PerfReport: Full Execution -- 3.24e+01 / 7.66e+01, 1.37e+01 / 1.87e+00 | 20260505 10:15:45.996 | INFO | PerfReport: 0.29%, AggregateStats -- 9.33e-02 / 1.03e-01, 1.10e-02 / 3.73e-03 | 20260505 10:15:45.996 | INFO | PerfReport: 11.68%, Baseline -- 3.78e+00 / 4.49e+00, 6.10e-01 / 2.30e-01 | 20260505 10:15:45.996 | INFO | PerfReport: 3.32%, Histogram -- 1.08e+00 / 1.50e+00, 2.08e-01 / 8.87e-02 | 20260505 10:15:45.996 | INFO | PerfReport: 15.66%, DME -- 4.06e+01 / 4.49e+01, 4.37e+00 / 1.87e+00 | 20260505 10:15:45.996 | INFO | PerfReport: 56.56%, FrameLabeling -- 1.83e+01 / 2.07e+01, 1.82e+00 / 2.12e+00 | 20260505 10:15:45.996 | INFO | PerfReport: 7.39%, PulseAccumulating -- 2.39e+00 / 2.91e+00, 4.10e-01 / 2.27e-01 | 20260505 10:15:45.996 | INFO | PerfReport: 3.76%, Metrics -- 1.22e+00 / 1.53e+00, 3.47e-01 / 1.78e-01 | 20260505 10:15:45.996 | INFO | PerfReport: 1.33%, PulseSerializing -- 4.32e-01 / 4.81e-01, 8.14e-02 / 3.03e-02 | 20260505 10:15:45.996 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 10:15:45.996 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:15:45.996 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:15:45.996 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.40e+01 / 0.00e+00 | 20260505 10:15:45.996 | INFO | PerfReport: 22.54%, Upload -- 2.28e+01 / 0.00e+00, 8.16e+00 / 0.00e+00 | 20260505 10:15:45.996 | INFO | PerfReport: 21.76%, Download -- 2.21e+01 / 0.00e+00, 8.15e+00 / 0.00e+00 | 20260505 10:15:45.996 | INFO | PerfReport: 31.96%, Compute -- 3.24e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 10:15:45.996 | INFO | PerfReport: 1.90%, LockUpload -- 1.92e+00 / 0.00e+00, 7.60e+00 / 0.00e+00 | 20260505 10:15:45.996 | INFO | PerfReport: 1.12%, LockDownload -- 1.13e+00 / 0.00e+00, 2.63e+00 / 0.00e+00 | 20260505 10:15:45.996 | INFO | PerfReport: 20.70%, LockCompute -- 2.10e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260505 10:15:45.996 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.67e-02 / 0.00e+00 | 20260505 10:15:46.131 | INFO | Starting a round of Baz IO | 20260505 10:15:49.404 | INFO | Populating RTMetrics report | 20260505 10:15:49.404 | INFO | RTMetrics report populated | 20260505 10:15:49.404 | INFO | Opening tmp RTMetrics json file | 20260505 10:15:49.407 | INFO | Serializing RTMetrics to json | 20260505 10:15:49.409 | INFO | Serialization done, swapping tmp file now... | 20260505 10:15:49.412 | INFO | RTMetrics done | 20260505 10:15:49.542 | INFO | Finished a round of Baz IO: Wrote 2.419 GiB in 3.41 seconds | 20260505 10:15:49.542 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.17% / 0.01% / 74.82% | 20260505 10:15:49.542 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.57367% / 3.57628e-05% / 7.54992e-05% | 20260505 10:15:49.542 | INFO | Baz IO overall: | 20260505 10:15:49.542 | INFO | BytesWritten : 1412618210344 | 20260505 10:15:49.542 | INFO | NumEvents : 672526848158 | 20260505 10:15:51.143 | INFO | Analyzing chunk frames = [2416640,2417152) | 20260505 10:15:51.143 | INFO | MemInfo: { | 20260505 10:15:51.143 | INFO | "gpu" : " 17 GB", | 20260505 10:15:51.143 | INFO | "huge" : "", | 20260505 10:15:51.144 | INFO | "huge_shared" : " 27 GB", | 20260505 10:15:51.144 | INFO | "malloc" : " 14 GB", | 20260505 10:15:51.144 | INFO | "pinned" : " 29 GB" | 20260505 10:15:51.144 | INFO | } | 20260505 10:15:51.144 | INFO | Perf report for the last 5.14918 seconds: | 20260505 10:15:51.144 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:15:51.144 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:51.144 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:51.144 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:51.144 | INFO | ANALYSIS : 65.1%, 2.9, 100.1ms, 0 | 20260505 10:15:51.144 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:51.144 | INFO | RT_METRICS : 45.6%, 1.0, 24.0ms, 0 | 20260505 10:15:51.144 | INFO | PRE_HQ : 62.5%, 1.8, 59.5ms, 0 | 20260505 10:15:51.144 | INFO | BAZWRITER : 65.5%, 4.2, 140.2ms, 0 | 20260505 10:15:51.144 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 10:15:51.144 | INFO | Uploaded 36GiB to GPU in 2.24307s (16.0494GiB/s) | 20260505 10:15:51.144 | INFO | Downloaded 38GiB to GPU in 2.35177s (16.1581GiB/s) | 20260505 10:15:56.284 | INFO | Analyzing chunk frames = [2417152,2417664) | 20260505 10:15:56.284 | INFO | MemInfo: { | 20260505 10:15:56.284 | INFO | "gpu" : " 17 GB", | 20260505 10:15:56.284 | INFO | "huge" : "", | 20260505 10:15:56.284 | INFO | "huge_shared" : " 27 GB", | 20260505 10:15:56.284 | INFO | "malloc" : " 14 GB", | 20260505 10:15:56.284 | INFO | "pinned" : " 29 GB" | 20260505 10:15:56.284 | INFO | } | 20260505 10:15:56.284 | INFO | Perf report for the last 5.14054 seconds: | 20260505 10:15:56.284 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:15:56.284 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:56.284 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:56.284 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:56.284 | INFO | ANALYSIS : 64.3%, 3.0, 100.5ms, 0 | 20260505 10:15:56.284 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:56.284 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:56.284 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:15:56.284 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:56.284 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:15:56.284 | INFO | Uploaded 36GiB to GPU in 2.21043s (16.2864GiB/s) | 20260505 10:15:56.284 | INFO | Downloaded 34GiB to GPU in 2.08538s (16.304GiB/s) | 20260505 10:16:01.420 | INFO | Analyzing chunk frames = [2417664,2418176) | 20260505 10:16:01.420 | INFO | MemInfo: { | 20260505 10:16:01.420 | INFO | "gpu" : " 17 GB", | 20260505 10:16:01.420 | INFO | "huge" : "", | 20260505 10:16:01.420 | INFO | "huge_shared" : " 27 GB", | 20260505 10:16:01.420 | INFO | "malloc" : " 14 GB", | 20260505 10:16:01.420 | INFO | "pinned" : " 29 GB" | 20260505 10:16:01.420 | INFO | } | 20260505 10:16:01.420 | INFO | Perf report for the last 5.13608 seconds: | 20260505 10:16:01.420 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:16:01.420 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:01.420 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:01.420 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:01.420 | INFO | ANALYSIS : 64.0%, 3.0, 99.5ms, 0 | 20260505 10:16:01.420 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:01.420 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:01.420 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 10:16:01.420 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:01.420 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:01.420 | INFO | Uploaded 36GiB to GPU in 2.23049s (16.14GiB/s) | 20260505 10:16:01.421 | INFO | Downloaded 34GiB to GPU in 2.12237s (16.0198GiB/s) | 20260505 10:16:06.558 | INFO | Analyzing chunk frames = [2418176,2418688) | 20260505 10:16:06.558 | INFO | MemInfo: { | 20260505 10:16:06.558 | INFO | "gpu" : " 17 GB", | 20260505 10:16:06.558 | INFO | "huge" : "", | 20260505 10:16:06.558 | INFO | "huge_shared" : " 27 GB", | 20260505 10:16:06.558 | INFO | "malloc" : " 14 GB", | 20260505 10:16:06.558 | INFO | "pinned" : " 29 GB" | 20260505 10:16:06.558 | INFO | } | 20260505 10:16:06.558 | INFO | Perf report for the last 5.13822 seconds: | 20260505 10:16:06.558 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:16:06.558 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:06.558 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:06.558 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:06.559 | INFO | ANALYSIS : 63.1%, 3.0, 98.1ms, 0 | 20260505 10:16:06.559 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:06.559 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:06.559 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:16:06.559 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:06.559 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:06.559 | INFO | Uploaded 36GiB to GPU in 2.16904s (16.5972GiB/s) | 20260505 10:16:06.559 | INFO | Downloaded 34GiB to GPU in 2.07369s (16.3959GiB/s) | 20260505 10:16:11.693 | INFO | Analyzing chunk frames = [2418688,2419200) | 20260505 10:16:11.693 | INFO | MemInfo: { | 20260505 10:16:11.693 | INFO | "gpu" : " 17 GB", | 20260505 10:16:11.693 | INFO | "huge" : "", | 20260505 10:16:11.693 | INFO | "huge_shared" : " 27 GB", | 20260505 10:16:11.693 | INFO | "malloc" : " 14 GB", | 20260505 10:16:11.693 | INFO | "pinned" : " 29 GB" | 20260505 10:16:11.693 | INFO | } | 20260505 10:16:11.693 | INFO | Perf report for the last 5.13466 seconds: | 20260505 10:16:11.693 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:16:11.693 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:11.693 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:11.693 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:11.693 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 10:16:11.693 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:11.693 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:11.693 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 10:16:11.693 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:11.693 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:11.693 | INFO | Uploaded 36GiB to GPU in 2.23845s (16.0826GiB/s) | 20260505 10:16:11.693 | INFO | Downloaded 34GiB to GPU in 2.10517s (16.1507GiB/s) | 20260505 10:16:16.836 | INFO | Analyzing chunk frames = [2419200,2419712) | 20260505 10:16:16.836 | INFO | MemInfo: { | 20260505 10:16:16.836 | INFO | "gpu" : " 17 GB", | 20260505 10:16:16.836 | INFO | "huge" : "", | 20260505 10:16:16.836 | INFO | "huge_shared" : " 27 GB", | 20260505 10:16:16.836 | INFO | "malloc" : " 14 GB", | 20260505 10:16:16.836 | INFO | "pinned" : " 29 GB" | 20260505 10:16:16.836 | INFO | } | 20260505 10:16:16.836 | INFO | Perf report for the last 5.14254 seconds: | 20260505 10:16:16.836 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:16:16.836 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:16.836 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:16.836 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:16.836 | INFO | ANALYSIS : 63.2%, 3.0, 98.8ms, 0 | 20260505 10:16:16.836 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:16.836 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:16.836 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 10:16:16.836 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:16.836 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:16.836 | INFO | Uploaded 36GiB to GPU in 2.17212s (16.5737GiB/s) | 20260505 10:16:16.836 | INFO | Downloaded 34GiB to GPU in 2.06454s (16.4686GiB/s) | 20260505 10:16:21.982 | INFO | Analyzing chunk frames = [2419712,2420224) | 20260505 10:16:21.982 | INFO | MemInfo: { | 20260505 10:16:21.982 | INFO | "gpu" : " 17 GB", | 20260505 10:16:21.982 | INFO | "huge" : "", | 20260505 10:16:21.982 | INFO | "huge_shared" : " 27 GB", | 20260505 10:16:21.982 | INFO | "malloc" : " 14 GB", | 20260505 10:16:21.982 | INFO | "pinned" : " 29 GB" | 20260505 10:16:21.982 | INFO | } | 20260505 10:16:21.982 | INFO | Perf report for the last 5.14615 seconds: | 20260505 10:16:21.982 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:16:21.982 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:21.982 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:21.982 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:21.982 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 10:16:21.982 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:21.982 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:21.982 | INFO | PRE_HQ : 30.3%, 1.0, 15.8ms, 0 | 20260505 10:16:21.982 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:21.982 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:21.982 | INFO | Uploaded 36GiB to GPU in 2.17091s (16.5829GiB/s) | 20260505 10:16:21.982 | INFO | Downloaded 34GiB to GPU in 2.14396s (15.8585GiB/s) | 20260505 10:16:27.124 | INFO | Analyzing chunk frames = [2420224,2420736) | 20260505 10:16:27.125 | INFO | MemInfo: { | 20260505 10:16:27.125 | INFO | "gpu" : " 17 GB", | 20260505 10:16:27.125 | INFO | "huge" : "", | 20260505 10:16:27.125 | INFO | "huge_shared" : " 27 GB", | 20260505 10:16:27.125 | INFO | "malloc" : " 14 GB", | 20260505 10:16:27.125 | INFO | "pinned" : " 29 GB" | 20260505 10:16:27.125 | INFO | } | 20260505 10:16:27.125 | INFO | Perf report for the last 5.14279 seconds: | 20260505 10:16:27.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:16:27.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:27.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:27.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:27.125 | INFO | ANALYSIS : 64.1%, 3.0, 100.2ms, 0 | 20260505 10:16:27.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:27.125 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:27.125 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 10:16:27.125 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:27.125 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:27.125 | INFO | Uploaded 36GiB to GPU in 2.21713s (16.2372GiB/s) | 20260505 10:16:27.125 | INFO | Downloaded 34GiB to GPU in 2.09425s (16.235GiB/s) | 20260505 10:16:27.260 | INFO | Starting a round of Baz IO | 20260505 10:16:30.534 | INFO | Populating RTMetrics report | 20260505 10:16:30.534 | INFO | RTMetrics report populated | 20260505 10:16:30.534 | INFO | Opening tmp RTMetrics json file | 20260505 10:16:30.553 | INFO | Serializing RTMetrics to json | 20260505 10:16:30.603 | INFO | Serialization done, swapping tmp file now... | 20260505 10:16:30.605 | INFO | RTMetrics done | 20260505 10:16:30.673 | INFO | Finished a round of Baz IO: Wrote 2.421 GiB in 3.413 seconds | 20260505 10:16:30.673 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.15% / 0.01% / 74.83% | 20260505 10:16:30.673 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.57534% / 3.97364e-06% / 6.35783e-05% | 20260505 10:16:30.673 | INFO | Baz IO overall: | 20260505 10:16:30.673 | INFO | BytesWritten : 1415217677378 | 20260505 10:16:30.673 | INFO | NumEvents : 673767481744 | 20260505 10:16:32.257 | INFO | Analyzing chunk frames = [2420736,2421248) | 20260505 10:16:32.257 | INFO | MemInfo: { | 20260505 10:16:32.257 | INFO | "gpu" : " 17 GB", | 20260505 10:16:32.257 | INFO | "huge" : "", | 20260505 10:16:32.257 | INFO | "huge_shared" : " 27 GB", | 20260505 10:16:32.257 | INFO | "malloc" : " 14 GB", | 20260505 10:16:32.257 | INFO | "pinned" : " 29 GB" | 20260505 10:16:32.257 | INFO | } | 20260505 10:16:32.257 | INFO | Perf report for the last 5.13255 seconds: | 20260505 10:16:32.257 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:16:32.257 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:32.257 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:32.257 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:32.257 | INFO | ANALYSIS : 65.3%, 2.9, 100.0ms, 0 | 20260505 10:16:32.257 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:32.257 | INFO | RT_METRICS : 47.2%, 1.0, 24.7ms, 0 | 20260505 10:16:32.257 | INFO | PRE_HQ : 62.0%, 1.9, 59.8ms, 0 | 20260505 10:16:32.257 | INFO | BAZWRITER : 65.8%, 4.1, 139.9ms, 0 | 20260505 10:16:32.257 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 10:16:32.257 | INFO | Uploaded 36GiB to GPU in 2.18833s (16.4509GiB/s) | 20260505 10:16:32.257 | INFO | Downloaded 38GiB to GPU in 2.31797s (16.3937GiB/s) | 20260505 10:16:37.392 | INFO | Analyzing chunk frames = [2421248,2421760) | 20260505 10:16:37.392 | INFO | MemInfo: { | 20260505 10:16:37.392 | INFO | "gpu" : " 17 GB", | 20260505 10:16:37.392 | INFO | "huge" : "", | 20260505 10:16:37.392 | INFO | "huge_shared" : " 27 GB", | 20260505 10:16:37.392 | INFO | "malloc" : " 14 GB", | 20260505 10:16:37.392 | INFO | "pinned" : " 29 GB" | 20260505 10:16:37.392 | INFO | } | 20260505 10:16:37.392 | INFO | Perf report for the last 5.13509 seconds: | 20260505 10:16:37.392 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:16:37.392 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:37.392 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:37.392 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:37.392 | INFO | ANALYSIS : 64.2%, 3.0, 100.1ms, 0 | 20260505 10:16:37.392 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:37.392 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:37.392 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:16:37.392 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:37.392 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:37.393 | INFO | Uploaded 36GiB to GPU in 2.17567s (16.5466GiB/s) | 20260505 10:16:37.393 | INFO | Downloaded 34GiB to GPU in 2.09538s (16.2262GiB/s) | 20260505 10:16:42.534 | INFO | Analyzing chunk frames = [2421760,2422272) | 20260505 10:16:42.534 | INFO | MemInfo: { | 20260505 10:16:42.534 | INFO | "gpu" : " 17 GB", | 20260505 10:16:42.534 | INFO | "huge" : "", | 20260505 10:16:42.534 | INFO | "huge_shared" : " 27 GB", | 20260505 10:16:42.534 | INFO | "malloc" : " 14 GB", | 20260505 10:16:42.534 | INFO | "pinned" : " 29 GB" | 20260505 10:16:42.534 | INFO | } | 20260505 10:16:42.534 | INFO | Perf report for the last 5.14131 seconds: | 20260505 10:16:42.534 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:16:42.534 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:42.534 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:42.534 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:42.534 | INFO | ANALYSIS : 63.0%, 3.0, 98.1ms, 0 | 20260505 10:16:42.534 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:42.534 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:42.534 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:16:42.534 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:42.534 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:42.534 | INFO | Uploaded 36GiB to GPU in 2.15579s (16.6992GiB/s) | 20260505 10:16:42.534 | INFO | Downloaded 34GiB to GPU in 2.05962s (16.5079GiB/s) | 20260505 10:16:47.685 | INFO | Analyzing chunk frames = [2422272,2422784) | 20260505 10:16:47.685 | INFO | MemInfo: { | 20260505 10:16:47.685 | INFO | "gpu" : " 17 GB", | 20260505 10:16:47.685 | INFO | "huge" : "", | 20260505 10:16:47.685 | INFO | "huge_shared" : " 27 GB", | 20260505 10:16:47.685 | INFO | "malloc" : " 14 GB", | 20260505 10:16:47.685 | INFO | "pinned" : " 29 GB" | 20260505 10:16:47.685 | INFO | } | 20260505 10:16:47.685 | INFO | Perf report for the last 5.15155 seconds: | 20260505 10:16:47.685 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:16:47.685 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:47.685 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:47.685 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:47.685 | INFO | ANALYSIS : 62.9%, 3.0, 98.0ms, 0 | 20260505 10:16:47.685 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:47.685 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:47.685 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:16:47.685 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:47.685 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:47.685 | INFO | Uploaded 36GiB to GPU in 2.17047s (16.5863GiB/s) | 20260505 10:16:47.686 | INFO | Downloaded 34GiB to GPU in 2.07488s (16.3865GiB/s) | 20260505 10:16:52.814 | INFO | Analyzing chunk frames = [2422784,2423296) | 20260505 10:16:52.814 | INFO | MemInfo: { | 20260505 10:16:52.814 | INFO | "gpu" : " 17 GB", | 20260505 10:16:52.815 | INFO | "huge" : "", | 20260505 10:16:52.815 | INFO | "huge_shared" : " 27 GB", | 20260505 10:16:52.815 | INFO | "malloc" : " 14 GB", | 20260505 10:16:52.815 | INFO | "pinned" : " 29 GB" | 20260505 10:16:52.815 | INFO | } | 20260505 10:16:52.815 | INFO | Perf report for the last 5.12942 seconds: | 20260505 10:16:52.815 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:16:52.815 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:52.815 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:52.815 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:52.815 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 10:16:52.815 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:52.815 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:52.815 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:16:52.815 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:52.815 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:52.815 | INFO | Uploaded 36GiB to GPU in 2.22819s (16.1566GiB/s) | 20260505 10:16:52.815 | INFO | Downloaded 34GiB to GPU in 2.10471s (16.1542GiB/s) | 20260505 10:16:57.947 | INFO | Analyzing chunk frames = [2423296,2423808) | 20260505 10:16:57.947 | INFO | MemInfo: { | 20260505 10:16:57.947 | INFO | "gpu" : " 17 GB", | 20260505 10:16:57.947 | INFO | "huge" : "", | 20260505 10:16:57.947 | INFO | "huge_shared" : " 27 GB", | 20260505 10:16:57.947 | INFO | "malloc" : " 14 GB", | 20260505 10:16:57.947 | INFO | "pinned" : " 29 GB" | 20260505 10:16:57.947 | INFO | } | 20260505 10:16:57.947 | INFO | Perf report for the last 5.13235 seconds: | 20260505 10:16:57.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:16:57.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:57.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:57.947 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:57.947 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 10:16:57.947 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:57.947 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:57.947 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:16:57.947 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:57.947 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:16:57.947 | INFO | Uploaded 36GiB to GPU in 2.23827s (16.0838GiB/s) | 20260505 10:16:57.947 | INFO | Downloaded 34GiB to GPU in 2.10127s (16.1807GiB/s) | 20260505 10:17:03.097 | INFO | Analyzing chunk frames = [2423808,2424320) | 20260505 10:17:03.097 | INFO | MemInfo: { | 20260505 10:17:03.097 | INFO | "gpu" : " 17 GB", | 20260505 10:17:03.097 | INFO | "huge" : "", | 20260505 10:17:03.097 | INFO | "huge_shared" : " 27 GB", | 20260505 10:17:03.097 | INFO | "malloc" : " 14 GB", | 20260505 10:17:03.097 | INFO | "pinned" : " 29 GB" | 20260505 10:17:03.097 | INFO | } | 20260505 10:17:03.097 | INFO | Perf report for the last 5.14999 seconds: | 20260505 10:17:03.097 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:17:03.097 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:03.097 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:03.097 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:03.097 | INFO | ANALYSIS : 63.7%, 3.0, 99.7ms, 0 | 20260505 10:17:03.097 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:03.097 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:03.097 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 10:17:03.097 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:03.097 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:03.097 | INFO | Uploaded 36GiB to GPU in 2.16537s (16.6253GiB/s) | 20260505 10:17:03.097 | INFO | Downloaded 34GiB to GPU in 2.13716s (15.909GiB/s) | 20260505 10:17:08.237 | INFO | Analyzing chunk frames = [2424320,2424832) | 20260505 10:17:08.237 | INFO | MemInfo: { | 20260505 10:17:08.237 | INFO | "gpu" : " 17 GB", | 20260505 10:17:08.237 | INFO | "huge" : "", | 20260505 10:17:08.237 | INFO | "huge_shared" : " 27 GB", | 20260505 10:17:08.237 | INFO | "malloc" : " 14 GB", | 20260505 10:17:08.237 | INFO | "pinned" : " 29 GB" | 20260505 10:17:08.237 | INFO | } | 20260505 10:17:08.237 | INFO | Perf report for the last 5.14001 seconds: | 20260505 10:17:08.237 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:17:08.237 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:08.237 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:08.237 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:08.237 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 10:17:08.237 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:08.237 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:08.237 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 10:17:08.237 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:08.237 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:08.237 | INFO | Uploaded 36GiB to GPU in 2.22284s (16.1955GiB/s) | 20260505 10:17:08.237 | INFO | Downloaded 34GiB to GPU in 2.09336s (16.2419GiB/s) | 20260505 10:17:08.237 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 10:17:08.237 | INFO | Total mem usage: 15.3 GiB | 20260505 10:17:08.237 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 10:17:08.237 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.51%) | 20260505 10:17:08.237 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 10:17:08.237 | INFO | High Water Breakdown: | 20260505 10:17:08.237 | INFO | 61% : 8.78 GiB ( 93.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 10:17:08.237 | INFO | 38.8% : 5.59 GiB ( 84.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 10:17:08.237 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 10:17:08.237 | INFO | ------------------------------------------ | 20260505 10:17:08.237 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:17:08.237 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 10:17:08.237 | INFO | Total mem usage: 26.5 GiB | 20260505 10:17:08.237 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 10:17:08.237 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 10:17:08.237 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 10:17:08.238 | INFO | High Water Breakdown: | 20260505 10:17:08.238 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:17:08.238 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:17:08.238 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:17:08.238 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:17:08.238 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:17:08.238 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:17:08.238 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 10:17:08.238 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 10:17:08.238 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:17:08.238 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 10:17:08.238 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 10:17:08.238 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 10:17:08.238 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 10:17:08.238 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 10:17:08.238 | INFO | ------------------------------------------ | 20260505 10:17:08.238 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:17:08.238 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 10:17:08.238 | INFO | Total mem usage: 25.6 GiB | 20260505 10:17:08.238 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 10:17:08.238 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 10:17:08.238 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 10:17:08.238 | INFO | High Water Breakdown: | 20260505 10:17:08.238 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 10:17:08.238 | INFO | ------------------------------------------ | 20260505 10:17:08.238 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:17:08.238 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 10:17:08.238 | INFO | Total mem usage: 15.7 GiB | 20260505 10:17:08.238 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 10:17:08.238 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 10:17:08.238 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 10:17:08.238 | INFO | High Water Breakdown: | 20260505 10:17:08.238 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 10:17:08.238 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 10:17:08.238 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 10:17:08.238 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 10:17:08.238 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 10:17:08.238 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 10:17:08.238 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 10:17:08.238 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 10:17:08.238 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 10:17:08.238 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:17:08.238 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 10:17:08.238 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:17:08.238 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 10:17:08.238 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 10:17:08.238 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 10:17:08.238 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 10:17:08.238 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:17:08.238 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 10:17:08.238 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:17:08.238 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:17:08.238 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:17:08.238 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 10:17:08.238 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 10:17:08.238 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:17:08.238 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 10:17:08.238 | INFO | ------------------------------------------ | 20260505 10:17:08.238 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:17:08.238 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 10:17:08.238 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:17:08.238 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:17:08.238 | INFO | PerfReport: Full Execution -- 3.24e+01 / 7.75e+01, 1.37e+01 / 2.23e+00 | 20260505 10:17:08.238 | INFO | PerfReport: 0.29%, AggregateStats -- 9.31e-02 / 1.04e-01, 1.06e-02 / 1.12e-02 | 20260505 10:17:08.238 | INFO | PerfReport: 11.67%, Baseline -- 3.78e+00 / 4.46e+00, 6.07e-01 / 2.24e-01 | 20260505 10:17:08.238 | INFO | PerfReport: 3.31%, Histogram -- 1.07e+00 / 1.48e+00, 2.07e-01 / 1.05e-01 | 20260505 10:17:08.238 | INFO | PerfReport: 15.66%, DME -- 4.06e+01 / 4.49e+01, 4.50e+00 / 2.57e+00 | 20260505 10:17:08.238 | INFO | PerfReport: 56.64%, FrameLabeling -- 1.84e+01 / 2.17e+01, 1.84e+00 / 1.91e+00 | 20260505 10:17:08.238 | INFO | PerfReport: 7.33%, PulseAccumulating -- 2.38e+00 / 2.88e+00, 3.96e-01 / 2.31e-01 | 20260505 10:17:08.238 | INFO | PerfReport: 3.77%, Metrics -- 1.22e+00 / 1.49e+00, 3.50e-01 / 2.10e-01 | 20260505 10:17:08.238 | INFO | PerfReport: 1.33%, PulseSerializing -- 4.30e-01 / 4.72e-01, 7.93e-02 / 3.63e-02 | 20260505 10:17:08.238 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 10:17:08.238 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:17:08.238 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:17:08.238 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260505 10:17:08.238 | INFO | PerfReport: 22.57%, Upload -- 2.29e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260505 10:17:08.238 | INFO | PerfReport: 21.82%, Download -- 2.22e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260505 10:17:08.238 | INFO | PerfReport: 31.92%, Compute -- 3.24e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 10:17:08.238 | INFO | PerfReport: 1.91%, LockUpload -- 1.94e+00 / 0.00e+00, 7.53e+00 / 0.00e+00 | 20260505 10:17:08.238 | INFO | PerfReport: 1.13%, LockDownload -- 1.15e+00 / 0.00e+00, 2.77e+00 / 0.00e+00 | 20260505 10:17:08.238 | INFO | PerfReport: 20.62%, LockCompute -- 2.09e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260505 10:17:08.238 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.86e-02 / 0.00e+00 | 20260505 10:17:08.377 | INFO | Starting a round of Baz IO | 20260505 10:17:11.640 | INFO | Populating RTMetrics report | 20260505 10:17:11.641 | INFO | RTMetrics report populated | 20260505 10:17:11.641 | INFO | Opening tmp RTMetrics json file | 20260505 10:17:11.653 | INFO | Serializing RTMetrics to json | 20260505 10:17:11.678 | INFO | Serialization done, swapping tmp file now... | 20260505 10:17:11.680 | INFO | RTMetrics done | 20260505 10:17:11.799 | INFO | Finished a round of Baz IO: Wrote 2.418 GiB in 3.422 seconds | 20260505 10:17:11.799 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.14% / 0.01% / 74.85% | 20260505 10:17:11.799 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.56506% / 7.94729e-06% / 0.000103315% | 20260505 10:17:11.799 | INFO | Baz IO overall: | 20260505 10:17:11.799 | INFO | BytesWritten : 1417813747026 | 20260505 10:17:11.799 | INFO | NumEvents : 675005626005 | 20260505 10:17:13.368 | INFO | Analyzing chunk frames = [2424832,2425344) | 20260505 10:17:13.368 | INFO | MemInfo: { | 20260505 10:17:13.368 | INFO | "gpu" : " 17 GB", | 20260505 10:17:13.368 | INFO | "huge" : "", | 20260505 10:17:13.368 | INFO | "huge_shared" : " 27 GB", | 20260505 10:17:13.368 | INFO | "malloc" : " 14 GB", | 20260505 10:17:13.368 | INFO | "pinned" : " 29 GB" | 20260505 10:17:13.368 | INFO | } | 20260505 10:17:13.368 | INFO | Perf report for the last 5.13121 seconds: | 20260505 10:17:13.368 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:17:13.368 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:13.368 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:13.368 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:13.368 | INFO | ANALYSIS : 65.2%, 2.9, 99.9ms, 0 | 20260505 10:17:13.368 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:13.368 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260505 10:17:13.368 | INFO | PRE_HQ : 61.5%, 1.9, 59.2ms, 0 | 20260505 10:17:13.368 | INFO | BAZWRITER : 66.0%, 4.2, 141.2ms, 0 | 20260505 10:17:13.368 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 10:17:13.368 | INFO | Uploaded 36GiB to GPU in 2.20041s (16.3606GiB/s) | 20260505 10:17:13.368 | INFO | Downloaded 38GiB to GPU in 2.30777s (16.4661GiB/s) | 20260505 10:17:18.503 | INFO | Analyzing chunk frames = [2425344,2425856) | 20260505 10:17:18.503 | INFO | MemInfo: { | 20260505 10:17:18.503 | INFO | "gpu" : " 17 GB", | 20260505 10:17:18.503 | INFO | "huge" : "", | 20260505 10:17:18.503 | INFO | "huge_shared" : " 27 GB", | 20260505 10:17:18.503 | INFO | "malloc" : " 14 GB", | 20260505 10:17:18.503 | INFO | "pinned" : " 29 GB" | 20260505 10:17:18.503 | INFO | } | 20260505 10:17:18.503 | INFO | Perf report for the last 5.13466 seconds: | 20260505 10:17:18.503 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:17:18.503 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:18.503 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:18.503 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:18.503 | INFO | ANALYSIS : 64.2%, 3.0, 100.1ms, 0 | 20260505 10:17:18.503 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:18.503 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:18.503 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:17:18.503 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:18.503 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:18.503 | INFO | Uploaded 36GiB to GPU in 2.20252s (16.3449GiB/s) | 20260505 10:17:18.503 | INFO | Downloaded 34GiB to GPU in 2.11681s (16.0619GiB/s) | 20260505 10:17:23.646 | INFO | Analyzing chunk frames = [2425856,2426368) | 20260505 10:17:23.646 | INFO | MemInfo: { | 20260505 10:17:23.646 | INFO | "gpu" : " 17 GB", | 20260505 10:17:23.646 | INFO | "huge" : "", | 20260505 10:17:23.646 | INFO | "huge_shared" : " 27 GB", | 20260505 10:17:23.646 | INFO | "malloc" : " 14 GB", | 20260505 10:17:23.646 | INFO | "pinned" : " 29 GB" | 20260505 10:17:23.646 | INFO | } | 20260505 10:17:23.646 | INFO | Perf report for the last 5.14296 seconds: | 20260505 10:17:23.646 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:17:23.646 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:23.646 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:23.646 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:23.646 | INFO | ANALYSIS : 63.2%, 3.0, 98.4ms, 0 | 20260505 10:17:23.646 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:23.646 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:23.646 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:17:23.646 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:23.646 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:23.646 | INFO | Uploaded 36GiB to GPU in 2.1677s (16.6075GiB/s) | 20260505 10:17:23.646 | INFO | Downloaded 34GiB to GPU in 2.0737s (16.3958GiB/s) | 20260505 10:17:28.791 | INFO | Analyzing chunk frames = [2426368,2426880) | 20260505 10:17:28.791 | INFO | MemInfo: { | 20260505 10:17:28.791 | INFO | "gpu" : " 17 GB", | 20260505 10:17:28.791 | INFO | "huge" : "", | 20260505 10:17:28.791 | INFO | "huge_shared" : " 27 GB", | 20260505 10:17:28.791 | INFO | "malloc" : " 14 GB", | 20260505 10:17:28.791 | INFO | "pinned" : " 29 GB" | 20260505 10:17:28.791 | INFO | } | 20260505 10:17:28.791 | INFO | Perf report for the last 5.14529 seconds: | 20260505 10:17:28.791 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:17:28.791 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:28.791 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:28.791 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:28.791 | INFO | ANALYSIS : 63.1%, 3.0, 98.2ms, 0 | 20260505 10:17:28.791 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:28.791 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:28.791 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 10:17:28.791 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:28.791 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:28.791 | INFO | Uploaded 36GiB to GPU in 2.18217s (16.4974GiB/s) | 20260505 10:17:28.791 | INFO | Downloaded 34GiB to GPU in 2.08122s (16.3366GiB/s) | 20260505 10:17:33.936 | INFO | Analyzing chunk frames = [2426880,2427392) | 20260505 10:17:33.936 | INFO | MemInfo: { | 20260505 10:17:33.936 | INFO | "gpu" : " 17 GB", | 20260505 10:17:33.936 | INFO | "huge" : "", | 20260505 10:17:33.936 | INFO | "huge_shared" : " 27 GB", | 20260505 10:17:33.936 | INFO | "malloc" : " 14 GB", | 20260505 10:17:33.936 | INFO | "pinned" : " 29 GB" | 20260505 10:17:33.936 | INFO | } | 20260505 10:17:33.936 | INFO | Perf report for the last 5.14485 seconds: | 20260505 10:17:33.936 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:17:33.936 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:33.936 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:33.936 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:33.936 | INFO | ANALYSIS : 63.2%, 3.0, 98.8ms, 0 | 20260505 10:17:33.936 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:33.936 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:33.936 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:17:33.936 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:33.936 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:33.936 | INFO | Uploaded 36GiB to GPU in 2.11739s (17.0021GiB/s) | 20260505 10:17:33.936 | INFO | Downloaded 34GiB to GPU in 2.126s (15.9925GiB/s) | 20260505 10:17:39.068 | INFO | Analyzing chunk frames = [2427392,2427904) | 20260505 10:17:39.068 | INFO | MemInfo: { | 20260505 10:17:39.068 | INFO | "gpu" : " 17 GB", | 20260505 10:17:39.068 | INFO | "huge" : "", | 20260505 10:17:39.068 | INFO | "huge_shared" : " 27 GB", | 20260505 10:17:39.068 | INFO | "malloc" : " 14 GB", | 20260505 10:17:39.068 | INFO | "pinned" : " 29 GB" | 20260505 10:17:39.068 | INFO | } | 20260505 10:17:39.068 | INFO | Perf report for the last 5.13228 seconds: | 20260505 10:17:39.068 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:17:39.068 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:39.068 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:39.068 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:39.068 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 10:17:39.068 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:39.068 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:39.068 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:17:39.068 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:39.068 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:39.069 | INFO | Uploaded 36GiB to GPU in 2.1872s (16.4594GiB/s) | 20260505 10:17:39.069 | INFO | Downloaded 34GiB to GPU in 2.09694s (16.2141GiB/s) | 20260505 10:17:44.203 | INFO | Analyzing chunk frames = [2427904,2428416) | 20260505 10:17:44.203 | INFO | MemInfo: { | 20260505 10:17:44.203 | INFO | "gpu" : " 17 GB", | 20260505 10:17:44.203 | INFO | "huge" : "", | 20260505 10:17:44.203 | INFO | "huge_shared" : " 27 GB", | 20260505 10:17:44.203 | INFO | "malloc" : " 14 GB", | 20260505 10:17:44.203 | INFO | "pinned" : " 29 GB" | 20260505 10:17:44.203 | INFO | } | 20260505 10:17:44.203 | INFO | Perf report for the last 5.13441 seconds: | 20260505 10:17:44.203 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:17:44.203 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:44.203 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:44.203 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:44.203 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 10:17:44.203 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:44.203 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:44.203 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 10:17:44.203 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:44.203 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:44.203 | INFO | Uploaded 36GiB to GPU in 2.1609s (16.6598GiB/s) | 20260505 10:17:44.203 | INFO | Downloaded 34GiB to GPU in 2.14693s (15.8366GiB/s) | 20260505 10:17:49.352 | INFO | Analyzing chunk frames = [2428416,2428928) | 20260505 10:17:49.352 | INFO | MemInfo: { | 20260505 10:17:49.352 | INFO | "gpu" : " 17 GB", | 20260505 10:17:49.352 | INFO | "huge" : "", | 20260505 10:17:49.352 | INFO | "huge_shared" : " 27 GB", | 20260505 10:17:49.352 | INFO | "malloc" : " 14 GB", | 20260505 10:17:49.352 | INFO | "pinned" : " 29 GB" | 20260505 10:17:49.352 | INFO | } | 20260505 10:17:49.352 | INFO | Perf report for the last 5.1495 seconds: | 20260505 10:17:49.352 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:17:49.352 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:49.352 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:49.352 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:49.352 | INFO | ANALYSIS : 63.7%, 3.0, 99.6ms, 0 | 20260505 10:17:49.352 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:49.352 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:49.352 | INFO | PRE_HQ : 29.1%, 1.0, 15.2ms, 0 | 20260505 10:17:49.352 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:49.352 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:49.353 | INFO | Uploaded 36GiB to GPU in 2.18208s (16.498GiB/s) | 20260505 10:17:49.353 | INFO | Downloaded 34GiB to GPU in 2.05984s (16.5061GiB/s) | 20260505 10:17:49.489 | INFO | Starting a round of Baz IO | 20260505 10:17:52.752 | INFO | Populating RTMetrics report | 20260505 10:17:52.752 | INFO | RTMetrics report populated | 20260505 10:17:52.752 | INFO | Opening tmp RTMetrics json file | 20260505 10:17:52.765 | INFO | Serializing RTMetrics to json | 20260505 10:17:52.767 | INFO | Serialization done, swapping tmp file now... | 20260505 10:17:52.767 | INFO | RTMetrics done | 20260505 10:17:52.889 | INFO | Finished a round of Baz IO: Wrote 2.417 GiB in 3.399 seconds | 20260505 10:17:52.889 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.13% / 0.01% / 74.86% | 20260505 10:17:52.889 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.55796% / 1.58946e-05% / 8.74201e-05% | 20260505 10:17:52.889 | INFO | Baz IO overall: | 20260505 10:17:52.889 | INFO | BytesWritten : 1420408672778 | 20260505 10:17:52.889 | INFO | NumEvents : 676242808611 | 20260505 10:17:54.484 | INFO | Analyzing chunk frames = [2428928,2429440) | 20260505 10:17:54.484 | INFO | MemInfo: { | 20260505 10:17:54.484 | INFO | "gpu" : " 17 GB", | 20260505 10:17:54.484 | INFO | "huge" : "", | 20260505 10:17:54.484 | INFO | "huge_shared" : " 27 GB", | 20260505 10:17:54.484 | INFO | "malloc" : " 14 GB", | 20260505 10:17:54.484 | INFO | "pinned" : " 29 GB" | 20260505 10:17:54.484 | INFO | } | 20260505 10:17:54.484 | INFO | Perf report for the last 5.13141 seconds: | 20260505 10:17:54.484 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:17:54.484 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:54.484 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:54.484 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:54.484 | INFO | ANALYSIS : 65.1%, 2.9, 99.8ms, 0 | 20260505 10:17:54.484 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:54.484 | INFO | RT_METRICS : 45.9%, 1.0, 24.0ms, 0 | 20260505 10:17:54.484 | INFO | PRE_HQ : 61.6%, 1.9, 59.4ms, 0 | 20260505 10:17:54.484 | INFO | BAZWRITER : 65.6%, 4.2, 139.9ms, 0 | 20260505 10:17:54.484 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 10:17:54.484 | INFO | Uploaded 36GiB to GPU in 2.21562s (16.2483GiB/s) | 20260505 10:17:54.484 | INFO | Downloaded 38GiB to GPU in 2.30604s (16.4785GiB/s) | 20260505 10:17:59.631 | INFO | Analyzing chunk frames = [2429440,2429952) | 20260505 10:17:59.631 | INFO | MemInfo: { | 20260505 10:17:59.631 | INFO | "gpu" : " 17 GB", | 20260505 10:17:59.631 | INFO | "huge" : "", | 20260505 10:17:59.631 | INFO | "huge_shared" : " 27 GB", | 20260505 10:17:59.631 | INFO | "malloc" : " 14 GB", | 20260505 10:17:59.631 | INFO | "pinned" : " 29 GB" | 20260505 10:17:59.631 | INFO | } | 20260505 10:17:59.631 | INFO | Perf report for the last 5.1474 seconds: | 20260505 10:17:59.631 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:17:59.631 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:59.631 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:59.631 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:59.631 | INFO | ANALYSIS : 64.4%, 3.0, 100.7ms, 0 | 20260505 10:17:59.631 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:59.631 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:59.631 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 10:17:59.631 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:59.631 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:17:59.631 | INFO | Uploaded 36GiB to GPU in 2.22861s (16.1536GiB/s) | 20260505 10:17:59.631 | INFO | Downloaded 34GiB to GPU in 2.11181s (16.0999GiB/s) | 20260505 10:18:04.761 | INFO | Analyzing chunk frames = [2429952,2430464) | 20260505 10:18:04.761 | INFO | MemInfo: { | 20260505 10:18:04.761 | INFO | "gpu" : " 17 GB", | 20260505 10:18:04.762 | INFO | "huge" : "", | 20260505 10:18:04.762 | INFO | "huge_shared" : " 27 GB", | 20260505 10:18:04.762 | INFO | "malloc" : " 14 GB", | 20260505 10:18:04.762 | INFO | "pinned" : " 29 GB" | 20260505 10:18:04.762 | INFO | } | 20260505 10:18:04.762 | INFO | Perf report for the last 5.13047 seconds: | 20260505 10:18:04.762 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:18:04.762 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:04.762 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:04.762 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:04.762 | INFO | ANALYSIS : 63.0%, 3.0, 97.9ms, 0 | 20260505 10:18:04.762 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:04.762 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:04.762 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:18:04.762 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:04.762 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:04.762 | INFO | Uploaded 36GiB to GPU in 2.142s (16.8067GiB/s) | 20260505 10:18:04.762 | INFO | Downloaded 34GiB to GPU in 2.0623s (16.4865GiB/s) | 20260505 10:18:09.905 | INFO | Analyzing chunk frames = [2430464,2430976) | 20260505 10:18:09.905 | INFO | MemInfo: { | 20260505 10:18:09.905 | INFO | "gpu" : " 17 GB", | 20260505 10:18:09.905 | INFO | "huge" : "", | 20260505 10:18:09.905 | INFO | "huge_shared" : " 27 GB", | 20260505 10:18:09.905 | INFO | "malloc" : " 14 GB", | 20260505 10:18:09.905 | INFO | "pinned" : " 29 GB" | 20260505 10:18:09.905 | INFO | } | 20260505 10:18:09.905 | INFO | Perf report for the last 5.14387 seconds: | 20260505 10:18:09.905 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:18:09.905 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:09.905 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:09.905 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:09.906 | INFO | ANALYSIS : 63.0%, 3.0, 98.0ms, 0 | 20260505 10:18:09.906 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:09.906 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:09.906 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 10:18:09.906 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:09.906 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:09.906 | INFO | Uploaded 36GiB to GPU in 2.16554s (16.6241GiB/s) | 20260505 10:18:09.906 | INFO | Downloaded 34GiB to GPU in 2.07241s (16.406GiB/s) | 20260505 10:18:15.052 | INFO | Analyzing chunk frames = [2430976,2431488) | 20260505 10:18:15.052 | INFO | MemInfo: { | 20260505 10:18:15.052 | INFO | "gpu" : " 17 GB", | 20260505 10:18:15.052 | INFO | "huge" : "", | 20260505 10:18:15.052 | INFO | "huge_shared" : " 27 GB", | 20260505 10:18:15.052 | INFO | "malloc" : " 14 GB", | 20260505 10:18:15.052 | INFO | "pinned" : " 29 GB" | 20260505 10:18:15.052 | INFO | } | 20260505 10:18:15.052 | INFO | Perf report for the last 5.1462 seconds: | 20260505 10:18:15.052 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:18:15.052 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:15.052 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:15.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:15.052 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 10:18:15.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:15.052 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:15.052 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 10:18:15.052 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:15.052 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:15.052 | INFO | Uploaded 36GiB to GPU in 2.15878s (16.6761GiB/s) | 20260505 10:18:15.052 | INFO | Downloaded 34GiB to GPU in 2.12896s (15.9702GiB/s) | 20260505 10:18:20.188 | INFO | Analyzing chunk frames = [2431488,2432000) | 20260505 10:18:20.188 | INFO | MemInfo: { | 20260505 10:18:20.188 | INFO | "gpu" : " 17 GB", | 20260505 10:18:20.188 | INFO | "huge" : "", | 20260505 10:18:20.188 | INFO | "huge_shared" : " 27 GB", | 20260505 10:18:20.188 | INFO | "malloc" : " 14 GB", | 20260505 10:18:20.188 | INFO | "pinned" : " 29 GB" | 20260505 10:18:20.188 | INFO | } | 20260505 10:18:20.188 | INFO | Perf report for the last 5.13654 seconds: | 20260505 10:18:20.188 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:18:20.188 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:20.188 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:20.188 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:20.188 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 10:18:20.188 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:20.188 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:20.188 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 10:18:20.188 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:20.188 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:20.188 | INFO | Uploaded 36GiB to GPU in 2.15995s (16.667GiB/s) | 20260505 10:18:20.189 | INFO | Downloaded 34GiB to GPU in 2.12057s (16.0334GiB/s) | 20260505 10:18:25.317 | INFO | Analyzing chunk frames = [2432000,2432512) | 20260505 10:18:25.317 | INFO | MemInfo: { | 20260505 10:18:25.317 | INFO | "gpu" : " 17 GB", | 20260505 10:18:25.317 | INFO | "huge" : "", | 20260505 10:18:25.317 | INFO | "huge_shared" : " 27 GB", | 20260505 10:18:25.317 | INFO | "malloc" : " 14 GB", | 20260505 10:18:25.317 | INFO | "pinned" : " 29 GB" | 20260505 10:18:25.317 | INFO | } | 20260505 10:18:25.317 | INFO | Perf report for the last 5.12861 seconds: | 20260505 10:18:25.317 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:18:25.317 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:25.317 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:25.317 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:25.317 | INFO | ANALYSIS : 63.6%, 3.0, 99.1ms, 0 | 20260505 10:18:25.317 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:25.317 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:25.317 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 10:18:25.317 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:25.317 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:25.317 | INFO | Uploaded 36GiB to GPU in 2.18416s (16.4823GiB/s) | 20260505 10:18:25.317 | INFO | Downloaded 34GiB to GPU in 2.12759s (15.9805GiB/s) | 20260505 10:18:30.455 | INFO | Analyzing chunk frames = [2432512,2433024) | 20260505 10:18:30.455 | INFO | MemInfo: { | 20260505 10:18:30.455 | INFO | "gpu" : " 17 GB", | 20260505 10:18:30.455 | INFO | "huge" : "", | 20260505 10:18:30.455 | INFO | "huge_shared" : " 27 GB", | 20260505 10:18:30.455 | INFO | "malloc" : " 14 GB", | 20260505 10:18:30.455 | INFO | "pinned" : " 29 GB" | 20260505 10:18:30.455 | INFO | } | 20260505 10:18:30.455 | INFO | Perf report for the last 5.13786 seconds: | 20260505 10:18:30.455 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:18:30.455 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:30.455 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:30.455 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:30.455 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 10:18:30.455 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:30.455 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:30.455 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 10:18:30.455 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:30.455 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:30.455 | INFO | Uploaded 36GiB to GPU in 2.17264s (16.5697GiB/s) | 20260505 10:18:30.455 | INFO | Downloaded 34GiB to GPU in 2.0265s (16.7777GiB/s) | 20260505 10:18:30.455 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 10:18:30.455 | INFO | Total mem usage: 15.3 GiB | 20260505 10:18:30.455 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 10:18:30.455 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.54%) | 20260505 10:18:30.455 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 10:18:30.455 | INFO | High Water Breakdown: | 20260505 10:18:30.455 | INFO | 61% : 8.78 GiB ( 93.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 10:18:30.455 | INFO | 38.8% : 5.59 GiB ( 84.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 10:18:30.455 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 10:18:30.455 | INFO | ------------------------------------------ | 20260505 10:18:30.455 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:18:30.455 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 10:18:30.455 | INFO | Total mem usage: 26.5 GiB | 20260505 10:18:30.455 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 10:18:30.455 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 10:18:30.455 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 10:18:30.455 | INFO | High Water Breakdown: | 20260505 10:18:30.455 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:18:30.455 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:18:30.455 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:18:30.455 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:18:30.455 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:18:30.455 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:18:30.455 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 10:18:30.455 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 10:18:30.455 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:18:30.455 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 10:18:30.455 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 10:18:30.455 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 10:18:30.455 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 10:18:30.455 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 10:18:30.455 | INFO | ------------------------------------------ | 20260505 10:18:30.455 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:18:30.455 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 10:18:30.455 | INFO | Total mem usage: 25.6 GiB | 20260505 10:18:30.455 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 10:18:30.455 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 10:18:30.455 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 10:18:30.455 | INFO | High Water Breakdown: | 20260505 10:18:30.455 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 10:18:30.455 | INFO | ------------------------------------------ | 20260505 10:18:30.455 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:18:30.455 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 10:18:30.455 | INFO | Total mem usage: 15.7 GiB | 20260505 10:18:30.455 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 10:18:30.455 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 10:18:30.455 | INFO | Unused Cache: 880 MiB (Waste: 5.48%) | 20260505 10:18:30.455 | INFO | High Water Breakdown: | 20260505 10:18:30.455 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 10:18:30.455 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 10:18:30.455 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 10:18:30.456 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 10:18:30.456 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 10:18:30.456 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 10:18:30.456 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 10:18:30.456 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 10:18:30.456 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 10:18:30.456 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:18:30.456 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 10:18:30.456 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:18:30.456 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 10:18:30.456 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 10:18:30.456 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 10:18:30.456 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 10:18:30.456 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:18:30.456 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 10:18:30.456 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:18:30.456 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:18:30.456 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:18:30.456 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 10:18:30.456 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 10:18:30.456 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:18:30.456 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 10:18:30.456 | INFO | ------------------------------------------ | 20260505 10:18:30.456 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:18:30.456 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 10:18:30.456 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:18:30.456 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:18:30.456 | INFO | PerfReport: Full Execution -- 3.23e+01 / 7.64e+01, 1.37e+01 / 2.29e+00 | 20260505 10:18:30.456 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 1.02e-01, 1.03e-02 / 6.73e-03 | 20260505 10:18:30.456 | INFO | PerfReport: 11.71%, Baseline -- 3.78e+00 / 4.49e+00, 6.09e-01 / 2.41e-01 | 20260505 10:18:30.456 | INFO | PerfReport: 3.31%, Histogram -- 1.07e+00 / 1.49e+00, 2.07e-01 / 8.94e-02 | 20260505 10:18:30.456 | INFO | PerfReport: 15.81%, DME -- 4.09e+01 / 4.55e+01, 4.50e+00 / 2.54e+00 | 20260505 10:18:30.456 | INFO | PerfReport: 56.33%, FrameLabeling -- 1.82e+01 / 1.98e+01, 1.68e+00 / 1.90e+00 | 20260505 10:18:30.456 | INFO | PerfReport: 7.46%, PulseAccumulating -- 2.41e+00 / 2.96e+00, 4.29e-01 / 2.25e-01 | 20260505 10:18:30.456 | INFO | PerfReport: 3.75%, Metrics -- 1.21e+00 / 1.54e+00, 3.40e-01 / 1.79e-01 | 20260505 10:18:30.456 | INFO | PerfReport: 1.33%, PulseSerializing -- 4.30e-01 / 4.83e-01, 7.89e-02 / 3.35e-02 | 20260505 10:18:30.456 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 10:18:30.456 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:18:30.456 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:18:30.456 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.43e+01 / 0.00e+00 | 20260505 10:18:30.456 | INFO | PerfReport: 22.40%, Upload -- 2.27e+01 / 0.00e+00, 8.01e+00 / 0.00e+00 | 20260505 10:18:30.456 | INFO | PerfReport: 21.85%, Download -- 2.21e+01 / 0.00e+00, 8.14e+00 / 0.00e+00 | 20260505 10:18:30.456 | INFO | PerfReport: 31.92%, Compute -- 3.23e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 10:18:30.456 | INFO | PerfReport: 1.83%, LockUpload -- 1.85e+00 / 0.00e+00, 7.55e+00 / 0.00e+00 | 20260505 10:18:30.456 | INFO | PerfReport: 1.28%, LockDownload -- 1.29e+00 / 0.00e+00, 2.86e+00 / 0.00e+00 | 20260505 10:18:30.456 | INFO | PerfReport: 20.71%, LockCompute -- 2.10e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260505 10:18:30.456 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.64e-02 / 0.00e+00 | 20260505 10:18:30.586 | INFO | Starting a round of Baz IO | 20260505 10:18:33.860 | INFO | Populating RTMetrics report | 20260505 10:18:33.860 | INFO | RTMetrics report populated | 20260505 10:18:33.860 | INFO | Opening tmp RTMetrics json file | 20260505 10:18:33.888 | INFO | Serializing RTMetrics to json | 20260505 10:18:33.891 | INFO | Serialization done, swapping tmp file now... | 20260505 10:18:33.893 | INFO | RTMetrics done | 20260505 10:18:34.014 | INFO | Finished a round of Baz IO: Wrote 2.418 GiB in 3.427 seconds | 20260505 10:18:34.014 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.12% / 0.01% / 74.87% | 20260505 10:18:34.014 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.55599% / 1.19209e-05% / 5.5631e-05% | 20260505 10:18:34.014 | INFO | Baz IO overall: | 20260505 10:18:34.014 | INFO | BytesWritten : 1423004495312 | 20260505 10:18:34.014 | INFO | NumEvents : 677480942844 | 20260505 10:18:35.606 | INFO | Analyzing chunk frames = [2433024,2433536) | 20260505 10:18:35.606 | INFO | MemInfo: { | 20260505 10:18:35.606 | INFO | "gpu" : " 17 GB", | 20260505 10:18:35.606 | INFO | "huge" : "", | 20260505 10:18:35.606 | INFO | "huge_shared" : " 27 GB", | 20260505 10:18:35.606 | INFO | "malloc" : " 14 GB", | 20260505 10:18:35.606 | INFO | "pinned" : " 29 GB" | 20260505 10:18:35.606 | INFO | } | 20260505 10:18:35.606 | INFO | Perf report for the last 5.15092 seconds: | 20260505 10:18:35.606 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:18:35.606 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:35.606 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:35.606 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:35.606 | INFO | ANALYSIS : 65.0%, 2.9, 99.7ms, 0 | 20260505 10:18:35.606 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:35.606 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260505 10:18:35.606 | INFO | PRE_HQ : 62.2%, 1.8, 59.4ms, 0 | 20260505 10:18:35.606 | INFO | BAZWRITER : 65.8%, 4.2, 141.4ms, 0 | 20260505 10:18:35.606 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 10:18:35.606 | INFO | Uploaded 36GiB to GPU in 2.20801s (16.3042GiB/s) | 20260505 10:18:35.606 | INFO | Downloaded 38GiB to GPU in 2.33673s (16.262GiB/s) | 20260505 10:18:40.746 | INFO | Analyzing chunk frames = [2433536,2434048) | 20260505 10:18:40.746 | INFO | MemInfo: { | 20260505 10:18:40.746 | INFO | "gpu" : " 17 GB", | 20260505 10:18:40.746 | INFO | "huge" : "", | 20260505 10:18:40.746 | INFO | "huge_shared" : " 27 GB", | 20260505 10:18:40.746 | INFO | "malloc" : " 14 GB", | 20260505 10:18:40.746 | INFO | "pinned" : " 29 GB" | 20260505 10:18:40.746 | INFO | } | 20260505 10:18:40.746 | INFO | Perf report for the last 5.14009 seconds: | 20260505 10:18:40.746 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:18:40.746 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:40.746 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:40.746 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:40.746 | INFO | ANALYSIS : 64.3%, 3.0, 100.5ms, 0 | 20260505 10:18:40.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:40.746 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:40.746 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:18:40.746 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:40.746 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:40.746 | INFO | Uploaded 36GiB to GPU in 2.23318s (16.1205GiB/s) | 20260505 10:18:40.746 | INFO | Downloaded 34GiB to GPU in 2.07433s (16.3909GiB/s) | 20260505 10:18:45.888 | INFO | Analyzing chunk frames = [2434048,2434560) | 20260505 10:18:45.888 | INFO | MemInfo: { | 20260505 10:18:45.888 | INFO | "gpu" : " 17 GB", | 20260505 10:18:45.888 | INFO | "huge" : "", | 20260505 10:18:45.888 | INFO | "huge_shared" : " 27 GB", | 20260505 10:18:45.888 | INFO | "malloc" : " 14 GB", | 20260505 10:18:45.888 | INFO | "pinned" : " 29 GB" | 20260505 10:18:45.888 | INFO | } | 20260505 10:18:45.888 | INFO | Perf report for the last 5.14233 seconds: | 20260505 10:18:45.888 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:18:45.888 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:45.888 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:45.888 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:45.888 | INFO | ANALYSIS : 63.8%, 3.0, 99.4ms, 0 | 20260505 10:18:45.888 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:45.888 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:45.888 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 10:18:45.888 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:45.888 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:45.888 | INFO | Uploaded 36GiB to GPU in 2.22814s (16.157GiB/s) | 20260505 10:18:45.888 | INFO | Downloaded 34GiB to GPU in 2.12716s (15.9838GiB/s) | 20260505 10:18:51.016 | INFO | Analyzing chunk frames = [2434560,2435072) | 20260505 10:18:51.016 | INFO | MemInfo: { | 20260505 10:18:51.016 | INFO | "gpu" : " 17 GB", | 20260505 10:18:51.016 | INFO | "huge" : "", | 20260505 10:18:51.016 | INFO | "huge_shared" : " 27 GB", | 20260505 10:18:51.016 | INFO | "malloc" : " 14 GB", | 20260505 10:18:51.016 | INFO | "pinned" : " 29 GB" | 20260505 10:18:51.016 | INFO | } | 20260505 10:18:51.016 | INFO | Perf report for the last 5.12839 seconds: | 20260505 10:18:51.017 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:18:51.017 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:51.017 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:51.017 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:51.017 | INFO | ANALYSIS : 63.2%, 3.0, 98.0ms, 0 | 20260505 10:18:51.017 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:51.017 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:51.017 | INFO | PRE_HQ : 29.1%, 1.0, 15.0ms, 0 | 20260505 10:18:51.017 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:51.017 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:51.017 | INFO | Uploaded 36GiB to GPU in 2.16923s (16.5957GiB/s) | 20260505 10:18:51.017 | INFO | Downloaded 34GiB to GPU in 2.07902s (16.3538GiB/s) | 20260505 10:18:56.157 | INFO | Analyzing chunk frames = [2435072,2435584) | 20260505 10:18:56.157 | INFO | MemInfo: { | 20260505 10:18:56.157 | INFO | "gpu" : " 17 GB", | 20260505 10:18:56.157 | INFO | "huge" : "", | 20260505 10:18:56.157 | INFO | "huge_shared" : " 27 GB", | 20260505 10:18:56.157 | INFO | "malloc" : " 14 GB", | 20260505 10:18:56.157 | INFO | "pinned" : " 29 GB" | 20260505 10:18:56.157 | INFO | } | 20260505 10:18:56.157 | INFO | Perf report for the last 5.14044 seconds: | 20260505 10:18:56.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:18:56.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:56.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:56.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:56.157 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 10:18:56.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:56.157 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:56.157 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:18:56.157 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:56.157 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:18:56.157 | INFO | Uploaded 36GiB to GPU in 2.23326s (16.1199GiB/s) | 20260505 10:18:56.157 | INFO | Downloaded 34GiB to GPU in 2.10031s (16.1881GiB/s) | 20260505 10:19:01.299 | INFO | Analyzing chunk frames = [2435584,2436096) | 20260505 10:19:01.299 | INFO | MemInfo: { | 20260505 10:19:01.299 | INFO | "gpu" : " 17 GB", | 20260505 10:19:01.299 | INFO | "huge" : "", | 20260505 10:19:01.299 | INFO | "huge_shared" : " 27 GB", | 20260505 10:19:01.299 | INFO | "malloc" : " 14 GB", | 20260505 10:19:01.299 | INFO | "pinned" : " 29 GB" | 20260505 10:19:01.299 | INFO | } | 20260505 10:19:01.299 | INFO | Perf report for the last 5.14222 seconds: | 20260505 10:19:01.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:19:01.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:01.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:01.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:01.299 | INFO | ANALYSIS : 63.5%, 3.0, 99.3ms, 0 | 20260505 10:19:01.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:01.299 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:01.299 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 10:19:01.299 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:01.299 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:01.299 | INFO | Uploaded 36GiB to GPU in 2.20693s (16.3123GiB/s) | 20260505 10:19:01.299 | INFO | Downloaded 34GiB to GPU in 2.10409s (16.159GiB/s) | 20260505 10:19:06.437 | INFO | Analyzing chunk frames = [2436096,2436608) | 20260505 10:19:06.437 | INFO | MemInfo: { | 20260505 10:19:06.437 | INFO | "gpu" : " 17 GB", | 20260505 10:19:06.437 | INFO | "huge" : "", | 20260505 10:19:06.437 | INFO | "huge_shared" : " 27 GB", | 20260505 10:19:06.437 | INFO | "malloc" : " 14 GB", | 20260505 10:19:06.437 | INFO | "pinned" : " 29 GB" | 20260505 10:19:06.437 | INFO | } | 20260505 10:19:06.437 | INFO | Perf report for the last 5.13808 seconds: | 20260505 10:19:06.437 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:19:06.437 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:06.437 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:06.437 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:06.437 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 10:19:06.437 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:06.437 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:06.437 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 10:19:06.437 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:06.437 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:06.437 | INFO | Uploaded 36GiB to GPU in 2.19148s (16.4272GiB/s) | 20260505 10:19:06.438 | INFO | Downloaded 34GiB to GPU in 2.15183s (15.8005GiB/s) | 20260505 10:19:11.574 | INFO | Analyzing chunk frames = [2436608,2437120) | 20260505 10:19:11.574 | INFO | MemInfo: { | 20260505 10:19:11.574 | INFO | "gpu" : " 17 GB", | 20260505 10:19:11.574 | INFO | "huge" : "", | 20260505 10:19:11.574 | INFO | "huge_shared" : " 27 GB", | 20260505 10:19:11.574 | INFO | "malloc" : " 14 GB", | 20260505 10:19:11.574 | INFO | "pinned" : " 29 GB" | 20260505 10:19:11.574 | INFO | } | 20260505 10:19:11.574 | INFO | Perf report for the last 5.13637 seconds: | 20260505 10:19:11.574 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:19:11.574 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:11.574 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:11.574 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:11.574 | INFO | ANALYSIS : 64.7%, 3.0, 100.9ms, 0 | 20260505 10:19:11.574 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:11.574 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:11.574 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 10:19:11.574 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:11.574 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:11.574 | INFO | Uploaded 36GiB to GPU in 2.24725s (16.0196GiB/s) | 20260505 10:19:11.574 | INFO | Downloaded 34GiB to GPU in 2.09597s (16.2216GiB/s) | 20260505 10:19:11.704 | INFO | Starting a round of Baz IO | 20260505 10:19:14.979 | INFO | Populating RTMetrics report | 20260505 10:19:14.979 | INFO | RTMetrics report populated | 20260505 10:19:14.979 | INFO | Opening tmp RTMetrics json file | 20260505 10:19:14.996 | INFO | Serializing RTMetrics to json | 20260505 10:19:15.010 | INFO | Serialization done, swapping tmp file now... | 20260505 10:19:15.027 | INFO | RTMetrics done | 20260505 10:19:15.123 | INFO | Finished a round of Baz IO: Wrote 2.418 GiB in 3.419 seconds | 20260505 10:19:15.123 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.11% / 0.01% / 74.88% | 20260505 10:19:15.123 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.55371% / 1.98682e-05% / 5.96046e-05% | 20260505 10:19:15.123 | INFO | Baz IO overall: | 20260505 10:19:15.123 | INFO | BytesWritten : 1425600543187 | 20260505 10:19:15.123 | INFO | NumEvents : 678719318491 | 20260505 10:19:16.710 | INFO | Analyzing chunk frames = [2437120,2437632) | 20260505 10:19:16.710 | INFO | MemInfo: { | 20260505 10:19:16.710 | INFO | "gpu" : " 17 GB", | 20260505 10:19:16.710 | INFO | "huge" : "", | 20260505 10:19:16.710 | INFO | "huge_shared" : " 27 GB", | 20260505 10:19:16.710 | INFO | "malloc" : " 14 GB", | 20260505 10:19:16.710 | INFO | "pinned" : " 29 GB" | 20260505 10:19:16.710 | INFO | } | 20260505 10:19:16.710 | INFO | Perf report for the last 5.13641 seconds: | 20260505 10:19:16.710 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:19:16.710 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:16.710 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:16.710 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:16.710 | INFO | ANALYSIS : 65.2%, 2.9, 99.8ms, 0 | 20260505 10:19:16.710 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:16.710 | INFO | RT_METRICS : 46.5%, 1.0, 24.4ms, 0 | 20260505 10:19:16.710 | INFO | PRE_HQ : 62.4%, 1.8, 59.3ms, 0 | 20260505 10:19:16.710 | INFO | BAZWRITER : 65.9%, 4.2, 141.3ms, 0 | 20260505 10:19:16.710 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260505 10:19:16.710 | INFO | Uploaded 36GiB to GPU in 2.2087s (16.2992GiB/s) | 20260505 10:19:16.710 | INFO | Downloaded 38GiB to GPU in 2.33639s (16.2644GiB/s) | 20260505 10:19:21.862 | INFO | Analyzing chunk frames = [2437632,2438144) | 20260505 10:19:21.862 | INFO | MemInfo: { | 20260505 10:19:21.862 | INFO | "gpu" : " 17 GB", | 20260505 10:19:21.862 | INFO | "huge" : "", | 20260505 10:19:21.862 | INFO | "huge_shared" : " 27 GB", | 20260505 10:19:21.862 | INFO | "malloc" : " 14 GB", | 20260505 10:19:21.863 | INFO | "pinned" : " 29 GB" | 20260505 10:19:21.863 | INFO | } | 20260505 10:19:21.863 | INFO | Perf report for the last 5.15245 seconds: | 20260505 10:19:21.863 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:19:21.863 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:21.863 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:21.863 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:21.863 | INFO | ANALYSIS : 64.1%, 3.0, 100.3ms, 0 | 20260505 10:19:21.863 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:21.863 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:21.863 | INFO | PRE_HQ : 29.2%, 1.0, 15.3ms, 0 | 20260505 10:19:21.863 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:21.863 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:21.863 | INFO | Uploaded 36GiB to GPU in 2.18016s (16.5126GiB/s) | 20260505 10:19:21.863 | INFO | Downloaded 34GiB to GPU in 2.10054s (16.1863GiB/s) | 20260505 10:19:26.999 | INFO | Analyzing chunk frames = [2438144,2438656) | 20260505 10:19:26.999 | INFO | MemInfo: { | 20260505 10:19:26.999 | INFO | "gpu" : " 17 GB", | 20260505 10:19:26.999 | INFO | "huge" : "", | 20260505 10:19:26.999 | INFO | "huge_shared" : " 27 GB", | 20260505 10:19:26.999 | INFO | "malloc" : " 14 GB", | 20260505 10:19:26.999 | INFO | "pinned" : " 29 GB" | 20260505 10:19:26.999 | INFO | } | 20260505 10:19:26.999 | INFO | Perf report for the last 5.1363 seconds: | 20260505 10:19:26.999 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:19:26.999 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:26.999 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:26.999 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:26.999 | INFO | ANALYSIS : 63.7%, 3.0, 99.0ms, 0 | 20260505 10:19:26.999 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:26.999 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:26.999 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 10:19:26.999 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:26.999 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:26.999 | INFO | Uploaded 36GiB to GPU in 2.22513s (16.1788GiB/s) | 20260505 10:19:26.999 | INFO | Downloaded 34GiB to GPU in 2.12306s (16.0146GiB/s) | 20260505 10:19:32.130 | INFO | Analyzing chunk frames = [2438656,2439168) | 20260505 10:19:32.130 | INFO | MemInfo: { | 20260505 10:19:32.130 | INFO | "gpu" : " 17 GB", | 20260505 10:19:32.130 | INFO | "huge" : "", | 20260505 10:19:32.130 | INFO | "huge_shared" : " 27 GB", | 20260505 10:19:32.130 | INFO | "malloc" : " 14 GB", | 20260505 10:19:32.130 | INFO | "pinned" : " 29 GB" | 20260505 10:19:32.130 | INFO | } | 20260505 10:19:32.130 | INFO | Perf report for the last 5.13096 seconds: | 20260505 10:19:32.130 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:19:32.130 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:32.130 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:32.130 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:32.130 | INFO | ANALYSIS : 63.1%, 3.0, 97.9ms, 0 | 20260505 10:19:32.130 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:32.130 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:32.130 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 10:19:32.130 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:32.130 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:32.130 | INFO | Uploaded 36GiB to GPU in 2.14985s (16.7453GiB/s) | 20260505 10:19:32.130 | INFO | Downloaded 34GiB to GPU in 2.07468s (16.3881GiB/s) | 20260505 10:19:37.264 | INFO | Analyzing chunk frames = [2439168,2439680) | 20260505 10:19:37.264 | INFO | MemInfo: { | 20260505 10:19:37.264 | INFO | "gpu" : " 17 GB", | 20260505 10:19:37.264 | INFO | "huge" : "", | 20260505 10:19:37.264 | INFO | "huge_shared" : " 27 GB", | 20260505 10:19:37.264 | INFO | "malloc" : " 14 GB", | 20260505 10:19:37.264 | INFO | "pinned" : " 29 GB" | 20260505 10:19:37.264 | INFO | } | 20260505 10:19:37.264 | INFO | Perf report for the last 5.13393 seconds: | 20260505 10:19:37.264 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:19:37.264 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:37.264 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:37.264 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:37.264 | INFO | ANALYSIS : 63.1%, 3.0, 98.3ms, 0 | 20260505 10:19:37.264 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:37.264 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:37.264 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 10:19:37.264 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:37.264 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:37.264 | INFO | Uploaded 36GiB to GPU in 2.16468s (16.6306GiB/s) | 20260505 10:19:37.264 | INFO | Downloaded 34GiB to GPU in 2.06158s (16.4922GiB/s) | 20260505 10:19:42.405 | INFO | Analyzing chunk frames = [2439680,2440192) | 20260505 10:19:42.405 | INFO | MemInfo: { | 20260505 10:19:42.405 | INFO | "gpu" : " 17 GB", | 20260505 10:19:42.405 | INFO | "huge" : "", | 20260505 10:19:42.405 | INFO | "huge_shared" : " 27 GB", | 20260505 10:19:42.405 | INFO | "malloc" : " 14 GB", | 20260505 10:19:42.405 | INFO | "pinned" : " 29 GB" | 20260505 10:19:42.405 | INFO | } | 20260505 10:19:42.405 | INFO | Perf report for the last 5.14117 seconds: | 20260505 10:19:42.405 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:19:42.405 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:42.405 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:42.405 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:42.405 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 10:19:42.405 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:42.405 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:42.405 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 10:19:42.405 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:42.405 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:42.405 | INFO | Uploaded 36GiB to GPU in 2.17937s (16.5185GiB/s) | 20260505 10:19:42.405 | INFO | Downloaded 34GiB to GPU in 2.06416s (16.4716GiB/s) | 20260505 10:19:47.552 | INFO | Analyzing chunk frames = [2440192,2440704) | 20260505 10:19:47.552 | INFO | MemInfo: { | 20260505 10:19:47.552 | INFO | "gpu" : " 17 GB", | 20260505 10:19:47.552 | INFO | "huge" : "", | 20260505 10:19:47.552 | INFO | "huge_shared" : " 27 GB", | 20260505 10:19:47.552 | INFO | "malloc" : " 14 GB", | 20260505 10:19:47.552 | INFO | "pinned" : " 29 GB" | 20260505 10:19:47.552 | INFO | } | 20260505 10:19:47.552 | INFO | Perf report for the last 5.14669 seconds: | 20260505 10:19:47.552 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:19:47.552 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:47.552 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:47.552 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:47.552 | INFO | ANALYSIS : 63.9%, 3.0, 99.9ms, 0 | 20260505 10:19:47.552 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:47.552 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:47.552 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260505 10:19:47.552 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:47.552 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:47.552 | INFO | Uploaded 36GiB to GPU in 2.2129s (16.2682GiB/s) | 20260505 10:19:47.552 | INFO | Downloaded 34GiB to GPU in 2.14164s (15.8757GiB/s) | 20260505 10:19:52.699 | INFO | Analyzing chunk frames = [2440704,2441216) | 20260505 10:19:52.699 | INFO | MemInfo: { | 20260505 10:19:52.699 | INFO | "gpu" : " 17 GB", | 20260505 10:19:52.699 | INFO | "huge" : "", | 20260505 10:19:52.699 | INFO | "huge_shared" : " 27 GB", | 20260505 10:19:52.699 | INFO | "malloc" : " 14 GB", | 20260505 10:19:52.699 | INFO | "pinned" : " 29 GB" | 20260505 10:19:52.699 | INFO | } | 20260505 10:19:52.699 | INFO | Perf report for the last 5.14755 seconds: | 20260505 10:19:52.699 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:19:52.699 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:52.699 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:52.699 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:52.699 | INFO | ANALYSIS : 64.1%, 3.0, 100.2ms, 0 | 20260505 10:19:52.699 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:52.699 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:52.699 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 10:19:52.699 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:52.699 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:52.699 | INFO | Uploaded 36GiB to GPU in 2.22105s (16.2085GiB/s) | 20260505 10:19:52.700 | INFO | Downloaded 34GiB to GPU in 2.09097s (16.2604GiB/s) | 20260505 10:19:52.700 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 10:19:52.700 | INFO | Total mem usage: 15.3 GiB | 20260505 10:19:52.700 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 10:19:52.700 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.57%) | 20260505 10:19:52.700 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 10:19:52.700 | INFO | High Water Breakdown: | 20260505 10:19:52.700 | INFO | 61% : 8.78 GiB ( 93.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 10:19:52.700 | INFO | 38.8% : 5.59 GiB ( 84.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 10:19:52.700 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 10:19:52.700 | INFO | ------------------------------------------ | 20260505 10:19:52.700 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:19:52.700 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 10:19:52.700 | INFO | Total mem usage: 26.5 GiB | 20260505 10:19:52.700 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 10:19:52.700 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 10:19:52.700 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 10:19:52.700 | INFO | High Water Breakdown: | 20260505 10:19:52.700 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:19:52.700 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:19:52.700 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:19:52.700 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:19:52.700 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:19:52.700 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:19:52.700 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 10:19:52.700 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 10:19:52.700 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:19:52.700 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 10:19:52.700 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 10:19:52.700 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 10:19:52.700 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 10:19:52.700 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 10:19:52.700 | INFO | ------------------------------------------ | 20260505 10:19:52.700 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:19:52.700 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 10:19:52.700 | INFO | Total mem usage: 25.6 GiB | 20260505 10:19:52.700 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 10:19:52.700 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 10:19:52.700 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 10:19:52.700 | INFO | High Water Breakdown: | 20260505 10:19:52.700 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 10:19:52.700 | INFO | ------------------------------------------ | 20260505 10:19:52.700 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:19:52.700 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 10:19:52.700 | INFO | Total mem usage: 15.7 GiB | 20260505 10:19:52.700 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 10:19:52.700 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 10:19:52.700 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 10:19:52.700 | INFO | High Water Breakdown: | 20260505 10:19:52.700 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 10:19:52.700 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 10:19:52.700 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 10:19:52.700 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 10:19:52.700 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 10:19:52.700 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 10:19:52.700 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 10:19:52.700 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 10:19:52.700 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 10:19:52.700 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:19:52.700 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 10:19:52.700 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:19:52.700 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 10:19:52.700 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 10:19:52.700 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 10:19:52.700 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 10:19:52.700 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:19:52.700 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 10:19:52.700 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:19:52.700 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:19:52.700 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:19:52.700 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 10:19:52.700 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 10:19:52.700 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:19:52.700 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 10:19:52.700 | INFO | ------------------------------------------ | 20260505 10:19:52.700 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:19:52.700 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 10:19:52.700 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:19:52.700 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:19:52.700 | INFO | PerfReport: Full Execution -- 3.24e+01 / 7.77e+01, 1.40e+01 / 2.09e+00 | 20260505 10:19:52.700 | INFO | PerfReport: 0.29%, AggregateStats -- 9.33e-02 / 1.02e-01, 1.10e-02 / 3.36e-03 | 20260505 10:19:52.700 | INFO | PerfReport: 11.67%, Baseline -- 3.78e+00 / 4.46e+00, 6.06e-01 / 2.34e-01 | 20260505 10:19:52.700 | INFO | PerfReport: 3.31%, Histogram -- 1.07e+00 / 1.48e+00, 2.06e-01 / 9.58e-02 | 20260505 10:19:52.700 | INFO | PerfReport: 15.66%, DME -- 4.06e+01 / 4.49e+01, 4.37e+00 / 2.16e+00 | 20260505 10:19:52.700 | INFO | PerfReport: 56.57%, FrameLabeling -- 1.83e+01 / 2.18e+01, 1.82e+00 / 1.61e+00 | 20260505 10:19:52.700 | INFO | PerfReport: 7.39%, PulseAccumulating -- 2.39e+00 / 2.93e+00, 4.14e-01 / 1.61e-01 | 20260505 10:19:52.700 | INFO | PerfReport: 3.77%, Metrics -- 1.22e+00 / 1.51e+00, 3.53e-01 / 1.77e-01 | 20260505 10:19:52.700 | INFO | PerfReport: 1.33%, PulseSerializing -- 4.31e-01 / 4.77e-01, 8.11e-02 / 3.44e-02 | 20260505 10:19:52.700 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 10:19:52.700 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:19:52.700 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:19:52.700 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.39e+01 / 0.00e+00 | 20260505 10:19:52.700 | INFO | PerfReport: 22.62%, Upload -- 2.30e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260505 10:19:52.700 | INFO | PerfReport: 21.85%, Download -- 2.22e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260505 10:19:52.700 | INFO | PerfReport: 31.91%, Compute -- 3.24e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 10:19:52.700 | INFO | PerfReport: 1.94%, LockUpload -- 1.97e+00 / 0.00e+00, 7.52e+00 / 0.00e+00 | 20260505 10:19:52.700 | INFO | PerfReport: 1.12%, LockDownload -- 1.13e+00 / 0.00e+00, 2.74e+00 / 0.00e+00 | 20260505 10:19:52.700 | INFO | PerfReport: 20.55%, LockCompute -- 2.09e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260505 10:19:52.700 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.75e-02 / 0.00e+00 | 20260505 10:19:52.839 | INFO | Starting a round of Baz IO | 20260505 10:19:56.103 | INFO | Populating RTMetrics report | 20260505 10:19:56.103 | INFO | RTMetrics report populated | 20260505 10:19:56.103 | INFO | Opening tmp RTMetrics json file | 20260505 10:19:56.136 | INFO | Serializing RTMetrics to json | 20260505 10:19:56.175 | INFO | Serialization done, swapping tmp file now... | 20260505 10:19:56.177 | INFO | RTMetrics done | 20260505 10:19:56.247 | INFO | Finished a round of Baz IO: Wrote 2.418 GiB in 3.408 seconds | 20260505 10:19:56.248 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.10% / 0.01% / 74.89% | 20260505 10:19:56.248 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.55036% / 1.98682e-05% / 7.94729e-05% | 20260505 10:19:56.248 | INFO | Baz IO overall: | 20260505 10:19:56.248 | INFO | BytesWritten : 1428196667494 | 20260505 10:19:56.248 | INFO | NumEvents : 679958008762 | 20260505 10:19:57.832 | INFO | Analyzing chunk frames = [2441216,2441728) | 20260505 10:19:57.832 | INFO | MemInfo: { | 20260505 10:19:57.832 | INFO | "gpu" : " 17 GB", | 20260505 10:19:57.832 | INFO | "huge" : "", | 20260505 10:19:57.832 | INFO | "huge_shared" : " 27 GB", | 20260505 10:19:57.832 | INFO | "malloc" : " 14 GB", | 20260505 10:19:57.832 | INFO | "pinned" : " 29 GB" | 20260505 10:19:57.832 | INFO | } | 20260505 10:19:57.832 | INFO | Perf report for the last 5.13247 seconds: | 20260505 10:19:57.832 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:19:57.832 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:57.832 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:57.832 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:57.832 | INFO | ANALYSIS : 65.2%, 2.9, 99.7ms, 0 | 20260505 10:19:57.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:19:57.832 | INFO | RT_METRICS : 47.1%, 1.0, 24.7ms, 0 | 20260505 10:19:57.832 | INFO | PRE_HQ : 61.7%, 1.9, 59.5ms, 0 | 20260505 10:19:57.832 | INFO | BAZWRITER : 65.7%, 4.2, 140.8ms, 0 | 20260505 10:19:57.832 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260505 10:19:57.832 | INFO | Uploaded 36GiB to GPU in 2.20391s (16.3346GiB/s) | 20260505 10:19:57.832 | INFO | Downloaded 38GiB to GPU in 2.31183s (16.4372GiB/s) | 20260505 10:20:02.968 | INFO | Analyzing chunk frames = [2441728,2442240) | 20260505 10:20:02.968 | INFO | MemInfo: { | 20260505 10:20:02.968 | INFO | "gpu" : " 17 GB", | 20260505 10:20:02.968 | INFO | "huge" : "", | 20260505 10:20:02.968 | INFO | "huge_shared" : " 27 GB", | 20260505 10:20:02.968 | INFO | "malloc" : " 14 GB", | 20260505 10:20:02.968 | INFO | "pinned" : " 29 GB" | 20260505 10:20:02.968 | INFO | } | 20260505 10:20:02.968 | INFO | Perf report for the last 5.13675 seconds: | 20260505 10:20:02.968 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:20:02.968 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:02.968 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:02.968 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:02.968 | INFO | ANALYSIS : 64.3%, 3.0, 100.3ms, 0 | 20260505 10:20:02.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:02.969 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:02.969 | INFO | PRE_HQ : 29.8%, 1.0, 15.4ms, 0 | 20260505 10:20:02.969 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:02.969 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:02.969 | INFO | Uploaded 36GiB to GPU in 2.18433s (16.481GiB/s) | 20260505 10:20:02.969 | INFO | Downloaded 34GiB to GPU in 2.1022s (16.1735GiB/s) | 20260505 10:20:08.108 | INFO | Analyzing chunk frames = [2442240,2442752) | 20260505 10:20:08.108 | INFO | MemInfo: { | 20260505 10:20:08.108 | INFO | "gpu" : " 17 GB", | 20260505 10:20:08.108 | INFO | "huge" : "", | 20260505 10:20:08.108 | INFO | "huge_shared" : " 27 GB", | 20260505 10:20:08.108 | INFO | "malloc" : " 14 GB", | 20260505 10:20:08.108 | INFO | "pinned" : " 29 GB" | 20260505 10:20:08.108 | INFO | } | 20260505 10:20:08.108 | INFO | Perf report for the last 5.13967 seconds: | 20260505 10:20:08.108 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:20:08.108 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:08.108 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:08.108 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:08.108 | INFO | ANALYSIS : 63.0%, 3.0, 98.0ms, 0 | 20260505 10:20:08.108 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:08.108 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:08.108 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 10:20:08.108 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:08.108 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:08.108 | INFO | Uploaded 36GiB to GPU in 2.15238s (16.7257GiB/s) | 20260505 10:20:08.109 | INFO | Downloaded 34GiB to GPU in 2.05953s (16.5086GiB/s) | 20260505 10:20:13.256 | INFO | Analyzing chunk frames = [2442752,2443264) | 20260505 10:20:13.256 | INFO | MemInfo: { | 20260505 10:20:13.256 | INFO | "gpu" : " 17 GB", | 20260505 10:20:13.256 | INFO | "huge" : "", | 20260505 10:20:13.256 | INFO | "huge_shared" : " 27 GB", | 20260505 10:20:13.256 | INFO | "malloc" : " 14 GB", | 20260505 10:20:13.256 | INFO | "pinned" : " 29 GB" | 20260505 10:20:13.256 | INFO | } | 20260505 10:20:13.256 | INFO | Perf report for the last 5.14753 seconds: | 20260505 10:20:13.256 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:20:13.256 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:13.256 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:13.256 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:13.256 | INFO | ANALYSIS : 63.0%, 3.0, 98.0ms, 0 | 20260505 10:20:13.256 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:13.256 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:13.256 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 10:20:13.256 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:13.256 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:13.256 | INFO | Uploaded 36GiB to GPU in 2.16304s (16.6433GiB/s) | 20260505 10:20:13.256 | INFO | Downloaded 34GiB to GPU in 2.07689s (16.3706GiB/s) | 20260505 10:20:18.388 | INFO | Analyzing chunk frames = [2443264,2443776) | 20260505 10:20:18.388 | INFO | MemInfo: { | 20260505 10:20:18.388 | INFO | "gpu" : " 17 GB", | 20260505 10:20:18.388 | INFO | "huge" : "", | 20260505 10:20:18.388 | INFO | "huge_shared" : " 27 GB", | 20260505 10:20:18.388 | INFO | "malloc" : " 14 GB", | 20260505 10:20:18.388 | INFO | "pinned" : " 29 GB" | 20260505 10:20:18.388 | INFO | } | 20260505 10:20:18.388 | INFO | Perf report for the last 5.13223 seconds: | 20260505 10:20:18.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:20:18.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:18.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:18.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:18.388 | INFO | ANALYSIS : 63.5%, 3.0, 99.0ms, 0 | 20260505 10:20:18.388 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:18.388 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:18.388 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 10:20:18.388 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:18.388 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:18.389 | INFO | Uploaded 36GiB to GPU in 2.22084s (16.2101GiB/s) | 20260505 10:20:18.389 | INFO | Downloaded 34GiB to GPU in 2.1218s (16.0241GiB/s) | 20260505 10:20:23.522 | INFO | Analyzing chunk frames = [2443776,2444288) | 20260505 10:20:23.522 | INFO | MemInfo: { | 20260505 10:20:23.522 | INFO | "gpu" : " 17 GB", | 20260505 10:20:23.522 | INFO | "huge" : "", | 20260505 10:20:23.522 | INFO | "huge_shared" : " 27 GB", | 20260505 10:20:23.522 | INFO | "malloc" : " 14 GB", | 20260505 10:20:23.522 | INFO | "pinned" : " 29 GB" | 20260505 10:20:23.522 | INFO | } | 20260505 10:20:23.522 | INFO | Perf report for the last 5.13388 seconds: | 20260505 10:20:23.522 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:20:23.522 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:23.522 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:23.522 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:23.522 | INFO | ANALYSIS : 64.0%, 3.0, 99.8ms, 0 | 20260505 10:20:23.522 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:23.522 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:23.522 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 10:20:23.522 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:23.522 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:23.522 | INFO | Uploaded 36GiB to GPU in 2.23717s (16.0918GiB/s) | 20260505 10:20:23.522 | INFO | Downloaded 34GiB to GPU in 2.10199s (16.1752GiB/s) | 20260505 10:20:28.666 | INFO | Analyzing chunk frames = [2444288,2444800) | 20260505 10:20:28.666 | INFO | MemInfo: { | 20260505 10:20:28.666 | INFO | "gpu" : " 17 GB", | 20260505 10:20:28.666 | INFO | "huge" : "", | 20260505 10:20:28.666 | INFO | "huge_shared" : " 27 GB", | 20260505 10:20:28.666 | INFO | "malloc" : " 14 GB", | 20260505 10:20:28.666 | INFO | "pinned" : " 29 GB" | 20260505 10:20:28.666 | INFO | } | 20260505 10:20:28.666 | INFO | Perf report for the last 5.1439 seconds: | 20260505 10:20:28.666 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:20:28.666 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:28.666 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:28.666 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:28.666 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 10:20:28.666 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:28.666 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:28.666 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 10:20:28.666 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:28.666 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:28.666 | INFO | Uploaded 36GiB to GPU in 2.19882s (16.3724GiB/s) | 20260505 10:20:28.666 | INFO | Downloaded 34GiB to GPU in 2.15025s (15.8121GiB/s) | 20260505 10:20:33.810 | INFO | Analyzing chunk frames = [2444800,2445312) | 20260505 10:20:33.810 | INFO | MemInfo: { | 20260505 10:20:33.810 | INFO | "gpu" : " 17 GB", | 20260505 10:20:33.810 | INFO | "huge" : "", | 20260505 10:20:33.810 | INFO | "huge_shared" : " 27 GB", | 20260505 10:20:33.810 | INFO | "malloc" : " 14 GB", | 20260505 10:20:33.810 | INFO | "pinned" : " 29 GB" | 20260505 10:20:33.810 | INFO | } | 20260505 10:20:33.810 | INFO | Perf report for the last 5.14438 seconds: | 20260505 10:20:33.810 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:20:33.810 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:33.810 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:33.810 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:33.810 | INFO | ANALYSIS : 64.0%, 3.0, 100.1ms, 0 | 20260505 10:20:33.810 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:33.810 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:33.810 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 10:20:33.810 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:33.810 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:33.810 | INFO | Uploaded 36GiB to GPU in 2.2213s (16.2067GiB/s) | 20260505 10:20:33.810 | INFO | Downloaded 34GiB to GPU in 2.09338s (16.2416GiB/s) | 20260505 10:20:33.951 | INFO | Starting a round of Baz IO | 20260505 10:20:37.219 | INFO | Populating RTMetrics report | 20260505 10:20:37.219 | INFO | RTMetrics report populated | 20260505 10:20:37.219 | INFO | Opening tmp RTMetrics json file | 20260505 10:20:37.223 | INFO | Serializing RTMetrics to json | 20260505 10:20:37.246 | INFO | Serialization done, swapping tmp file now... | 20260505 10:20:37.248 | INFO | RTMetrics done | 20260505 10:20:37.379 | INFO | Finished a round of Baz IO: Wrote 2.415 GiB in 3.427 seconds | 20260505 10:20:37.379 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.09% / 0.01% / 74.90% | 20260505 10:20:37.379 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.54068% / 7.94729e-06% / 5.16574e-05% | 20260505 10:20:37.379 | INFO | Baz IO overall: | 20260505 10:20:37.379 | INFO | BytesWritten : 1430790165359 | 20260505 10:20:37.379 | INFO | NumEvents : 681194845761 | 20260505 10:20:38.951 | INFO | Analyzing chunk frames = [2445312,2445824) | 20260505 10:20:38.951 | INFO | MemInfo: { | 20260505 10:20:38.951 | INFO | "gpu" : " 17 GB", | 20260505 10:20:38.951 | INFO | "huge" : "", | 20260505 10:20:38.951 | INFO | "huge_shared" : " 27 GB", | 20260505 10:20:38.951 | INFO | "malloc" : " 14 GB", | 20260505 10:20:38.951 | INFO | "pinned" : " 29 GB" | 20260505 10:20:38.951 | INFO | } | 20260505 10:20:38.951 | INFO | Perf report for the last 5.14077 seconds: | 20260505 10:20:38.951 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:20:38.951 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:38.951 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:38.951 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:38.951 | INFO | ANALYSIS : 65.2%, 2.9, 99.9ms, 0 | 20260505 10:20:38.951 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:38.951 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260505 10:20:38.951 | INFO | PRE_HQ : 61.8%, 1.9, 59.5ms, 0 | 20260505 10:20:38.951 | INFO | BAZWRITER : 65.9%, 4.2, 142.6ms, 0 | 20260505 10:20:38.951 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 10:20:38.951 | INFO | Uploaded 36GiB to GPU in 2.20497s (16.3267GiB/s) | 20260505 10:20:38.951 | INFO | Downloaded 38GiB to GPU in 2.31704s (16.4003GiB/s) | 20260505 10:20:44.075 | INFO | Analyzing chunk frames = [2445824,2446336) | 20260505 10:20:44.075 | INFO | MemInfo: { | 20260505 10:20:44.075 | INFO | "gpu" : " 17 GB", | 20260505 10:20:44.075 | INFO | "huge" : "", | 20260505 10:20:44.075 | INFO | "huge_shared" : " 27 GB", | 20260505 10:20:44.075 | INFO | "malloc" : " 14 GB", | 20260505 10:20:44.075 | INFO | "pinned" : " 29 GB" | 20260505 10:20:44.075 | INFO | } | 20260505 10:20:44.075 | INFO | Perf report for the last 5.12403 seconds: | 20260505 10:20:44.075 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:20:44.075 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:44.075 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:44.075 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:44.075 | INFO | ANALYSIS : 64.4%, 3.0, 100.2ms, 0 | 20260505 10:20:44.075 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:44.075 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:44.075 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:20:44.075 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:44.075 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:44.075 | INFO | Uploaded 36GiB to GPU in 2.1757s (16.5464GiB/s) | 20260505 10:20:44.075 | INFO | Downloaded 34GiB to GPU in 2.09546s (16.2256GiB/s) | 20260505 10:20:49.218 | INFO | Analyzing chunk frames = [2446336,2446848) | 20260505 10:20:49.218 | INFO | MemInfo: { | 20260505 10:20:49.218 | INFO | "gpu" : " 17 GB", | 20260505 10:20:49.218 | INFO | "huge" : "", | 20260505 10:20:49.218 | INFO | "huge_shared" : " 27 GB", | 20260505 10:20:49.218 | INFO | "malloc" : " 14 GB", | 20260505 10:20:49.218 | INFO | "pinned" : " 29 GB" | 20260505 10:20:49.218 | INFO | } | 20260505 10:20:49.218 | INFO | Perf report for the last 5.14311 seconds: | 20260505 10:20:49.218 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:20:49.218 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:49.218 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:49.218 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:49.218 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 10:20:49.218 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:49.218 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:49.218 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:20:49.218 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:49.218 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:49.218 | INFO | Uploaded 36GiB to GPU in 2.17395s (16.5597GiB/s) | 20260505 10:20:49.218 | INFO | Downloaded 34GiB to GPU in 2.07213s (16.4082GiB/s) | 20260505 10:20:54.361 | INFO | Analyzing chunk frames = [2446848,2447360) | 20260505 10:20:54.361 | INFO | MemInfo: { | 20260505 10:20:54.361 | INFO | "gpu" : " 17 GB", | 20260505 10:20:54.361 | INFO | "huge" : "", | 20260505 10:20:54.361 | INFO | "huge_shared" : " 27 GB", | 20260505 10:20:54.361 | INFO | "malloc" : " 14 GB", | 20260505 10:20:54.361 | INFO | "pinned" : " 29 GB" | 20260505 10:20:54.361 | INFO | } | 20260505 10:20:54.361 | INFO | Perf report for the last 5.14313 seconds: | 20260505 10:20:54.361 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:20:54.361 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:54.361 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:54.361 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:54.361 | INFO | ANALYSIS : 63.0%, 3.0, 97.9ms, 0 | 20260505 10:20:54.361 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:54.361 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:54.361 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:20:54.361 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:54.361 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:54.361 | INFO | Uploaded 36GiB to GPU in 2.16544s (16.6248GiB/s) | 20260505 10:20:54.362 | INFO | Downloaded 34GiB to GPU in 2.07206s (16.4088GiB/s) | 20260505 10:20:59.508 | INFO | Analyzing chunk frames = [2447360,2447872) | 20260505 10:20:59.508 | INFO | MemInfo: { | 20260505 10:20:59.508 | INFO | "gpu" : " 17 GB", | 20260505 10:20:59.508 | INFO | "huge" : "", | 20260505 10:20:59.508 | INFO | "huge_shared" : " 27 GB", | 20260505 10:20:59.508 | INFO | "malloc" : " 14 GB", | 20260505 10:20:59.508 | INFO | "pinned" : " 29 GB" | 20260505 10:20:59.508 | INFO | } | 20260505 10:20:59.508 | INFO | Perf report for the last 5.14644 seconds: | 20260505 10:20:59.508 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:20:59.508 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:59.508 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:59.508 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:59.508 | INFO | ANALYSIS : 63.3%, 3.0, 99.0ms, 0 | 20260505 10:20:59.508 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:59.508 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:59.508 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 10:20:59.508 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:59.508 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:20:59.508 | INFO | Uploaded 36GiB to GPU in 2.22708s (16.1646GiB/s) | 20260505 10:20:59.508 | INFO | Downloaded 34GiB to GPU in 2.10518s (16.1506GiB/s) | 20260505 10:21:04.642 | INFO | Analyzing chunk frames = [2447872,2448384) | 20260505 10:21:04.642 | INFO | MemInfo: { | 20260505 10:21:04.642 | INFO | "gpu" : " 17 GB", | 20260505 10:21:04.642 | INFO | "huge" : "", | 20260505 10:21:04.642 | INFO | "huge_shared" : " 27 GB", | 20260505 10:21:04.642 | INFO | "malloc" : " 14 GB", | 20260505 10:21:04.642 | INFO | "pinned" : " 29 GB" | 20260505 10:21:04.642 | INFO | } | 20260505 10:21:04.642 | INFO | Perf report for the last 5.13457 seconds: | 20260505 10:21:04.642 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:21:04.642 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:04.642 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:04.642 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:04.642 | INFO | ANALYSIS : 63.8%, 3.0, 99.5ms, 0 | 20260505 10:21:04.642 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:04.642 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:04.642 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:21:04.642 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:04.642 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:04.642 | INFO | Uploaded 36GiB to GPU in 2.21735s (16.2356GiB/s) | 20260505 10:21:04.642 | INFO | Downloaded 34GiB to GPU in 2.10076s (16.1846GiB/s) | 20260505 10:21:09.778 | INFO | Analyzing chunk frames = [2448384,2448896) | 20260505 10:21:09.778 | INFO | MemInfo: { | 20260505 10:21:09.778 | INFO | "gpu" : " 17 GB", | 20260505 10:21:09.778 | INFO | "huge" : "", | 20260505 10:21:09.778 | INFO | "huge_shared" : " 27 GB", | 20260505 10:21:09.778 | INFO | "malloc" : " 14 GB", | 20260505 10:21:09.778 | INFO | "pinned" : " 29 GB" | 20260505 10:21:09.778 | INFO | } | 20260505 10:21:09.778 | INFO | Perf report for the last 5.13608 seconds: | 20260505 10:21:09.778 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:21:09.778 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:09.778 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:09.778 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:09.778 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 10:21:09.778 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:09.778 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:09.778 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 10:21:09.778 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:09.778 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:09.779 | INFO | Uploaded 36GiB to GPU in 2.17431s (16.557GiB/s) | 20260505 10:21:09.779 | INFO | Downloaded 34GiB to GPU in 2.13785s (15.9038GiB/s) | 20260505 10:21:14.922 | INFO | Analyzing chunk frames = [2448896,2449408) | 20260505 10:21:14.922 | INFO | MemInfo: { | 20260505 10:21:14.922 | INFO | "gpu" : " 17 GB", | 20260505 10:21:14.922 | INFO | "huge" : "", | 20260505 10:21:14.922 | INFO | "huge_shared" : " 27 GB", | 20260505 10:21:14.922 | INFO | "malloc" : " 14 GB", | 20260505 10:21:14.922 | INFO | "pinned" : " 29 GB" | 20260505 10:21:14.922 | INFO | } | 20260505 10:21:14.922 | INFO | Perf report for the last 5.14356 seconds: | 20260505 10:21:14.922 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:21:14.922 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:14.922 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:14.922 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:14.922 | INFO | ANALYSIS : 64.5%, 3.0, 100.9ms, 0 | 20260505 10:21:14.922 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:14.922 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:14.922 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 10:21:14.922 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:14.922 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:14.922 | INFO | Uploaded 36GiB to GPU in 2.2448s (16.0371GiB/s) | 20260505 10:21:14.922 | INFO | Downloaded 34GiB to GPU in 2.09284s (16.2459GiB/s) | 20260505 10:21:14.922 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 10:21:14.922 | INFO | Total mem usage: 15.3 GiB | 20260505 10:21:14.922 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 10:21:14.922 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.61%) | 20260505 10:21:14.922 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 10:21:14.922 | INFO | High Water Breakdown: | 20260505 10:21:14.922 | INFO | 61% : 8.77 GiB ( 93.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 10:21:14.922 | INFO | 38.8% : 5.58 GiB ( 84% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 10:21:14.923 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 10:21:14.923 | INFO | ------------------------------------------ | 20260505 10:21:14.923 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:21:14.923 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 10:21:14.923 | INFO | Total mem usage: 26.5 GiB | 20260505 10:21:14.923 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 10:21:14.923 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 10:21:14.923 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 10:21:14.923 | INFO | High Water Breakdown: | 20260505 10:21:14.923 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:21:14.923 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:21:14.923 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:21:14.923 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:21:14.923 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:21:14.923 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:21:14.923 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 10:21:14.923 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 10:21:14.923 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:21:14.923 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 10:21:14.923 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 10:21:14.923 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 10:21:14.923 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 10:21:14.923 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 10:21:14.923 | INFO | ------------------------------------------ | 20260505 10:21:14.923 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:21:14.923 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 10:21:14.923 | INFO | Total mem usage: 25.6 GiB | 20260505 10:21:14.923 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 10:21:14.923 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 10:21:14.923 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 10:21:14.923 | INFO | High Water Breakdown: | 20260505 10:21:14.923 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 10:21:14.923 | INFO | ------------------------------------------ | 20260505 10:21:14.923 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:21:14.923 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 10:21:14.923 | INFO | Total mem usage: 15.7 GiB | 20260505 10:21:14.923 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 10:21:14.923 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 10:21:14.923 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 10:21:14.923 | INFO | High Water Breakdown: | 20260505 10:21:14.923 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 10:21:14.923 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 10:21:14.923 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 10:21:14.923 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 10:21:14.923 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 10:21:14.924 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 10:21:14.924 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 10:21:14.924 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 10:21:14.924 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 10:21:14.924 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:21:14.924 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 10:21:14.924 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:21:14.924 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 10:21:14.924 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 10:21:14.924 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 10:21:14.924 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 10:21:14.924 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:21:14.924 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 10:21:14.924 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:21:14.924 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:21:14.924 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:21:14.924 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 10:21:14.924 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 10:21:14.924 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:21:14.924 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 10:21:14.924 | INFO | ------------------------------------------ | 20260505 10:21:14.924 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:21:14.924 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 10:21:14.924 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:21:14.924 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:21:14.924 | INFO | PerfReport: Full Execution -- 3.24e+01 / 7.73e+01, 1.39e+01 / 2.22e+00 | 20260505 10:21:14.924 | INFO | PerfReport: 0.29%, AggregateStats -- 9.32e-02 / 1.02e-01, 1.08e-02 / 6.16e-03 | 20260505 10:21:14.924 | INFO | PerfReport: 11.69%, Baseline -- 3.79e+00 / 4.52e+00, 6.16e-01 / 1.80e-01 | 20260505 10:21:14.924 | INFO | PerfReport: 3.32%, Histogram -- 1.08e+00 / 1.49e+00, 2.11e-01 / 8.62e-02 | 20260505 10:21:14.924 | INFO | PerfReport: 15.64%, DME -- 4.05e+01 / 4.48e+01, 4.43e+00 / 2.26e+00 | 20260505 10:21:14.924 | INFO | PerfReport: 56.61%, FrameLabeling -- 1.83e+01 / 2.15e+01, 1.85e+00 / 2.38e+00 | 20260505 10:21:14.924 | INFO | PerfReport: 7.36%, PulseAccumulating -- 2.38e+00 / 2.88e+00, 4.10e-01 / 2.45e-01 | 20260505 10:21:14.924 | INFO | PerfReport: 3.76%, Metrics -- 1.22e+00 / 1.51e+00, 3.51e-01 / 1.65e-01 | 20260505 10:21:14.924 | INFO | PerfReport: 1.33%, PulseSerializing -- 4.30e-01 / 4.78e-01, 7.94e-02 / 2.68e-02 | 20260505 10:21:14.924 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 10:21:14.924 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:21:14.924 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:21:14.924 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260505 10:21:14.924 | INFO | PerfReport: 22.58%, Upload -- 2.29e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260505 10:21:14.925 | INFO | PerfReport: 21.84%, Download -- 2.21e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260505 10:21:14.925 | INFO | PerfReport: 31.96%, Compute -- 3.24e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 10:21:14.925 | INFO | PerfReport: 1.86%, LockUpload -- 1.89e+00 / 0.00e+00, 7.45e+00 / 0.00e+00 | 20260505 10:21:14.925 | INFO | PerfReport: 1.13%, LockDownload -- 1.15e+00 / 0.00e+00, 2.64e+00 / 0.00e+00 | 20260505 10:21:14.925 | INFO | PerfReport: 20.62%, LockCompute -- 2.09e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260505 10:21:14.925 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.95e-02 / 0.00e+00 | 20260505 10:21:15.060 | INFO | Starting a round of Baz IO | 20260505 10:21:18.319 | INFO | Populating RTMetrics report | 20260505 10:21:18.319 | INFO | RTMetrics report populated | 20260505 10:21:18.319 | INFO | Opening tmp RTMetrics json file | 20260505 10:21:18.321 | INFO | Serializing RTMetrics to json | 20260505 10:21:18.325 | INFO | Serialization done, swapping tmp file now... | 20260505 10:21:18.326 | INFO | RTMetrics done | 20260505 10:21:18.490 | INFO | Finished a round of Baz IO: Wrote 2.418 GiB in 3.429 seconds | 20260505 10:21:18.490 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.08% / 0.01% / 74.91% | 20260505 10:21:18.490 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.5389% / 1.19209e-05% / 7.54992e-05% | 20260505 10:21:18.490 | INFO | Baz IO overall: | 20260505 10:21:18.490 | INFO | BytesWritten : 1433386477563 | 20260505 10:21:18.490 | INFO | NumEvents : 682433747547 | 20260505 10:21:20.058 | INFO | Analyzing chunk frames = [2449408,2449920) | 20260505 10:21:20.058 | INFO | MemInfo: { | 20260505 10:21:20.058 | INFO | "gpu" : " 17 GB", | 20260505 10:21:20.058 | INFO | "huge" : "", | 20260505 10:21:20.058 | INFO | "huge_shared" : " 27 GB", | 20260505 10:21:20.058 | INFO | "malloc" : " 14 GB", | 20260505 10:21:20.058 | INFO | "pinned" : " 29 GB" | 20260505 10:21:20.058 | INFO | } | 20260505 10:21:20.058 | INFO | Perf report for the last 5.13621 seconds: | 20260505 10:21:20.058 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:21:20.058 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:20.058 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:20.058 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:20.058 | INFO | ANALYSIS : 65.0%, 2.9, 99.6ms, 0 | 20260505 10:21:20.058 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:20.058 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260505 10:21:20.058 | INFO | PRE_HQ : 61.6%, 1.9, 59.4ms, 0 | 20260505 10:21:20.058 | INFO | BAZWRITER : 65.9%, 4.2, 141.7ms, 0 | 20260505 10:21:20.058 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 10:21:20.058 | INFO | Uploaded 36GiB to GPU in 2.2193s (16.2213GiB/s) | 20260505 10:21:20.058 | INFO | Downloaded 38GiB to GPU in 2.29811s (16.5353GiB/s) | 20260505 10:21:25.210 | INFO | Analyzing chunk frames = [2449920,2450432) | 20260505 10:21:25.210 | INFO | MemInfo: { | 20260505 10:21:25.210 | INFO | "gpu" : " 17 GB", | 20260505 10:21:25.210 | INFO | "huge" : "", | 20260505 10:21:25.210 | INFO | "huge_shared" : " 27 GB", | 20260505 10:21:25.210 | INFO | "malloc" : " 14 GB", | 20260505 10:21:25.210 | INFO | "pinned" : " 29 GB" | 20260505 10:21:25.210 | INFO | } | 20260505 10:21:25.211 | INFO | Perf report for the last 5.15238 seconds: | 20260505 10:21:25.211 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:21:25.211 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:25.211 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:25.211 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:25.211 | INFO | ANALYSIS : 64.1%, 3.0, 100.3ms, 0 | 20260505 10:21:25.211 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:25.211 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:25.211 | INFO | PRE_HQ : 29.1%, 1.0, 15.2ms, 0 | 20260505 10:21:25.211 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:25.211 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:25.211 | INFO | Uploaded 36GiB to GPU in 2.1965s (16.3897GiB/s) | 20260505 10:21:25.211 | INFO | Downloaded 34GiB to GPU in 2.10227s (16.173GiB/s) | 20260505 10:21:30.342 | INFO | Analyzing chunk frames = [2450432,2450944) | 20260505 10:21:30.342 | INFO | MemInfo: { | 20260505 10:21:30.342 | INFO | "gpu" : " 17 GB", | 20260505 10:21:30.342 | INFO | "huge" : "", | 20260505 10:21:30.342 | INFO | "huge_shared" : " 27 GB", | 20260505 10:21:30.342 | INFO | "malloc" : " 14 GB", | 20260505 10:21:30.342 | INFO | "pinned" : " 29 GB" | 20260505 10:21:30.342 | INFO | } | 20260505 10:21:30.342 | INFO | Perf report for the last 5.13124 seconds: | 20260505 10:21:30.342 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:21:30.342 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:30.342 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:30.342 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:30.342 | INFO | ANALYSIS : 63.0%, 3.0, 97.9ms, 0 | 20260505 10:21:30.342 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:30.342 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:30.342 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 10:21:30.342 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:30.342 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:30.342 | INFO | Uploaded 36GiB to GPU in 2.15513s (16.7043GiB/s) | 20260505 10:21:30.342 | INFO | Downloaded 34GiB to GPU in 2.06275s (16.4829GiB/s) | 20260505 10:21:35.475 | INFO | Analyzing chunk frames = [2450944,2451456) | 20260505 10:21:35.475 | INFO | MemInfo: { | 20260505 10:21:35.475 | INFO | "gpu" : " 17 GB", | 20260505 10:21:35.475 | INFO | "huge" : "", | 20260505 10:21:35.475 | INFO | "huge_shared" : " 27 GB", | 20260505 10:21:35.475 | INFO | "malloc" : " 14 GB", | 20260505 10:21:35.475 | INFO | "pinned" : " 29 GB" | 20260505 10:21:35.475 | INFO | } | 20260505 10:21:35.475 | INFO | Perf report for the last 5.1331 seconds: | 20260505 10:21:35.475 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:21:35.475 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:35.475 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:35.475 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:35.475 | INFO | ANALYSIS : 63.2%, 3.0, 98.0ms, 0 | 20260505 10:21:35.475 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:35.475 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:35.475 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:21:35.475 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:35.475 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:35.475 | INFO | Uploaded 36GiB to GPU in 2.17197s (16.5748GiB/s) | 20260505 10:21:35.475 | INFO | Downloaded 34GiB to GPU in 2.07324s (16.3994GiB/s) | 20260505 10:21:40.621 | INFO | Analyzing chunk frames = [2451456,2451968) | 20260505 10:21:40.621 | INFO | MemInfo: { | 20260505 10:21:40.621 | INFO | "gpu" : " 17 GB", | 20260505 10:21:40.621 | INFO | "huge" : "", | 20260505 10:21:40.622 | INFO | "huge_shared" : " 27 GB", | 20260505 10:21:40.622 | INFO | "malloc" : " 14 GB", | 20260505 10:21:40.622 | INFO | "pinned" : " 29 GB" | 20260505 10:21:40.622 | INFO | } | 20260505 10:21:40.622 | INFO | Perf report for the last 5.14672 seconds: | 20260505 10:21:40.622 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:21:40.622 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:40.622 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:40.622 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:40.622 | INFO | ANALYSIS : 63.3%, 3.0, 99.0ms, 0 | 20260505 10:21:40.622 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:40.622 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:40.622 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 10:21:40.622 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:40.622 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:40.622 | INFO | Uploaded 36GiB to GPU in 2.15108s (16.7358GiB/s) | 20260505 10:21:40.622 | INFO | Downloaded 34GiB to GPU in 2.13058s (15.9581GiB/s) | 20260505 10:21:45.762 | INFO | Analyzing chunk frames = [2451968,2452480) | 20260505 10:21:45.762 | INFO | MemInfo: { | 20260505 10:21:45.762 | INFO | "gpu" : " 17 GB", | 20260505 10:21:45.762 | INFO | "huge" : "", | 20260505 10:21:45.762 | INFO | "huge_shared" : " 27 GB", | 20260505 10:21:45.762 | INFO | "malloc" : " 14 GB", | 20260505 10:21:45.762 | INFO | "pinned" : " 29 GB" | 20260505 10:21:45.762 | INFO | } | 20260505 10:21:45.762 | INFO | Perf report for the last 5.14082 seconds: | 20260505 10:21:45.762 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:21:45.762 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:45.762 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:45.762 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:45.762 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 10:21:45.762 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:45.763 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:45.763 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:21:45.763 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:45.763 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:45.763 | INFO | Uploaded 36GiB to GPU in 2.18578s (16.4701GiB/s) | 20260505 10:21:45.763 | INFO | Downloaded 34GiB to GPU in 2.11772s (16.055GiB/s) | 20260505 10:21:50.899 | INFO | Analyzing chunk frames = [2452480,2452992) | 20260505 10:21:50.899 | INFO | MemInfo: { | 20260505 10:21:50.899 | INFO | "gpu" : " 17 GB", | 20260505 10:21:50.899 | INFO | "huge" : "", | 20260505 10:21:50.899 | INFO | "huge_shared" : " 27 GB", | 20260505 10:21:50.899 | INFO | "malloc" : " 14 GB", | 20260505 10:21:50.899 | INFO | "pinned" : " 29 GB" | 20260505 10:21:50.899 | INFO | } | 20260505 10:21:50.899 | INFO | Perf report for the last 5.13679 seconds: | 20260505 10:21:50.899 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:21:50.899 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:50.899 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:50.899 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:50.899 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 10:21:50.899 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:50.899 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:50.899 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 10:21:50.899 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:50.899 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:50.899 | INFO | Uploaded 36GiB to GPU in 2.16574s (16.6225GiB/s) | 20260505 10:21:50.900 | INFO | Downloaded 34GiB to GPU in 2.15037s (15.8112GiB/s) | 20260505 10:21:56.029 | INFO | Analyzing chunk frames = [2452992,2453504) | 20260505 10:21:56.029 | INFO | MemInfo: { | 20260505 10:21:56.029 | INFO | "gpu" : " 17 GB", | 20260505 10:21:56.029 | INFO | "huge" : "", | 20260505 10:21:56.029 | INFO | "huge_shared" : " 27 GB", | 20260505 10:21:56.029 | INFO | "malloc" : " 14 GB", | 20260505 10:21:56.029 | INFO | "pinned" : " 29 GB" | 20260505 10:21:56.029 | INFO | } | 20260505 10:21:56.029 | INFO | Perf report for the last 5.13009 seconds: | 20260505 10:21:56.029 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:21:56.029 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:56.029 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:56.029 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:56.029 | INFO | ANALYSIS : 64.6%, 3.0, 100.8ms, 0 | 20260505 10:21:56.029 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:56.029 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:56.029 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 10:21:56.030 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:56.030 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:21:56.030 | INFO | Uploaded 36GiB to GPU in 2.24405s (16.0424GiB/s) | 20260505 10:21:56.030 | INFO | Downloaded 34GiB to GPU in 2.09507s (16.2286GiB/s) | 20260505 10:21:56.162 | INFO | Starting a round of Baz IO | 20260505 10:21:59.432 | INFO | Populating RTMetrics report | 20260505 10:21:59.432 | INFO | RTMetrics report populated | 20260505 10:21:59.432 | INFO | Opening tmp RTMetrics json file | 20260505 10:21:59.552 | INFO | Serializing RTMetrics to json | 20260505 10:21:59.554 | INFO | Serialization done, swapping tmp file now... | 20260505 10:21:59.555 | INFO | RTMetrics done | 20260505 10:21:59.673 | INFO | Finished a round of Baz IO: Wrote 2.416 GiB in 3.511 seconds | 20260505 10:21:59.673 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.07% / 0.01% / 74.92% | 20260505 10:21:59.673 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.53085% / 7.94729e-06% / 6.35783e-05% | 20260505 10:21:59.674 | INFO | Baz IO overall: | 20260505 10:21:59.674 | INFO | BytesWritten : 1435980511885 | 20260505 10:21:59.674 | INFO | NumEvents : 683670794143 | 20260505 10:22:01.176 | INFO | Analyzing chunk frames = [2453504,2454016) | 20260505 10:22:01.176 | INFO | MemInfo: { | 20260505 10:22:01.176 | INFO | "gpu" : " 17 GB", | 20260505 10:22:01.176 | INFO | "huge" : "", | 20260505 10:22:01.176 | INFO | "huge_shared" : " 27 GB", | 20260505 10:22:01.176 | INFO | "malloc" : " 14 GB", | 20260505 10:22:01.176 | INFO | "pinned" : " 29 GB" | 20260505 10:22:01.176 | INFO | } | 20260505 10:22:01.176 | INFO | Perf report for the last 5.14636 seconds: | 20260505 10:22:01.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:22:01.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:01.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:01.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:01.176 | INFO | ANALYSIS : 65.0%, 2.9, 99.4ms, 0 | 20260505 10:22:01.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:01.176 | INFO | RT_METRICS : 48.3%, 1.0, 25.3ms, 0 | 20260505 10:22:01.176 | INFO | PRE_HQ : 62.6%, 1.8, 59.7ms, 0 | 20260505 10:22:01.176 | INFO | BAZWRITER : 67.5%, 4.2, 144.5ms, 0 | 20260505 10:22:01.176 | INFO | BAZ_SAMPLING : 2.1%, 1.0, 1.1ms, 0 | 20260505 10:22:01.176 | INFO | Uploaded 36GiB to GPU in 2.22257s (16.1975GiB/s) | 20260505 10:22:01.176 | INFO | Downloaded 38GiB to GPU in 2.34345s (16.2154GiB/s) | 20260505 10:22:06.317 | INFO | Analyzing chunk frames = [2454016,2454528) | 20260505 10:22:06.317 | INFO | MemInfo: { | 20260505 10:22:06.317 | INFO | "gpu" : " 17 GB", | 20260505 10:22:06.317 | INFO | "huge" : "", | 20260505 10:22:06.317 | INFO | "huge_shared" : " 27 GB", | 20260505 10:22:06.317 | INFO | "malloc" : " 14 GB", | 20260505 10:22:06.317 | INFO | "pinned" : " 29 GB" | 20260505 10:22:06.317 | INFO | } | 20260505 10:22:06.317 | INFO | Perf report for the last 5.14172 seconds: | 20260505 10:22:06.317 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:22:06.317 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:06.317 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:06.318 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:06.318 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 10:22:06.318 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:06.318 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:06.318 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:22:06.318 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:06.318 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:06.318 | INFO | Uploaded 36GiB to GPU in 2.23492s (16.108GiB/s) | 20260505 10:22:06.318 | INFO | Downloaded 34GiB to GPU in 2.08474s (16.309GiB/s) | 20260505 10:22:11.458 | INFO | Analyzing chunk frames = [2454528,2455040) | 20260505 10:22:11.458 | INFO | MemInfo: { | 20260505 10:22:11.458 | INFO | "gpu" : " 17 GB", | 20260505 10:22:11.458 | INFO | "huge" : "", | 20260505 10:22:11.458 | INFO | "huge_shared" : " 27 GB", | 20260505 10:22:11.458 | INFO | "malloc" : " 14 GB", | 20260505 10:22:11.458 | INFO | "pinned" : " 29 GB" | 20260505 10:22:11.458 | INFO | } | 20260505 10:22:11.458 | INFO | Perf report for the last 5.14041 seconds: | 20260505 10:22:11.458 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:22:11.458 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:11.458 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:11.458 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:11.458 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260505 10:22:11.458 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:11.458 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:11.458 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:22:11.458 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:11.458 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:11.458 | INFO | Uploaded 36GiB to GPU in 2.14912s (16.751GiB/s) | 20260505 10:22:11.458 | INFO | Downloaded 34GiB to GPU in 2.06386s (16.474GiB/s) | 20260505 10:22:16.594 | INFO | Analyzing chunk frames = [2455040,2455552) | 20260505 10:22:16.594 | INFO | MemInfo: { | 20260505 10:22:16.594 | INFO | "gpu" : " 17 GB", | 20260505 10:22:16.594 | INFO | "huge" : "", | 20260505 10:22:16.594 | INFO | "huge_shared" : " 27 GB", | 20260505 10:22:16.594 | INFO | "malloc" : " 14 GB", | 20260505 10:22:16.594 | INFO | "pinned" : " 29 GB" | 20260505 10:22:16.594 | INFO | } | 20260505 10:22:16.594 | INFO | Perf report for the last 5.13588 seconds: | 20260505 10:22:16.594 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:22:16.594 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:16.594 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:16.594 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:16.594 | INFO | ANALYSIS : 63.1%, 3.0, 98.0ms, 0 | 20260505 10:22:16.594 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:16.594 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:16.594 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 10:22:16.594 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:16.594 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:16.594 | INFO | Uploaded 36GiB to GPU in 2.17526s (16.5498GiB/s) | 20260505 10:22:16.594 | INFO | Downloaded 34GiB to GPU in 2.07622s (16.3759GiB/s) | 20260505 10:22:21.727 | INFO | Analyzing chunk frames = [2455552,2456064) | 20260505 10:22:21.727 | INFO | MemInfo: { | 20260505 10:22:21.727 | INFO | "gpu" : " 17 GB", | 20260505 10:22:21.727 | INFO | "huge" : "", | 20260505 10:22:21.727 | INFO | "huge_shared" : " 27 GB", | 20260505 10:22:21.727 | INFO | "malloc" : " 14 GB", | 20260505 10:22:21.727 | INFO | "pinned" : " 29 GB" | 20260505 10:22:21.727 | INFO | } | 20260505 10:22:21.727 | INFO | Perf report for the last 5.13369 seconds: | 20260505 10:22:21.727 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:22:21.727 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:21.728 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:21.728 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:21.728 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 10:22:21.728 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:21.728 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:21.728 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 10:22:21.728 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:21.728 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:21.728 | INFO | Uploaded 36GiB to GPU in 2.23512s (16.1065GiB/s) | 20260505 10:22:21.728 | INFO | Downloaded 34GiB to GPU in 2.11802s (16.0527GiB/s) | 20260505 10:22:26.874 | INFO | Analyzing chunk frames = [2456064,2456576) | 20260505 10:22:26.874 | INFO | MemInfo: { | 20260505 10:22:26.874 | INFO | "gpu" : " 17 GB", | 20260505 10:22:26.874 | INFO | "huge" : "", | 20260505 10:22:26.874 | INFO | "huge_shared" : " 27 GB", | 20260505 10:22:26.874 | INFO | "malloc" : " 14 GB", | 20260505 10:22:26.874 | INFO | "pinned" : " 29 GB" | 20260505 10:22:26.874 | INFO | } | 20260505 10:22:26.874 | INFO | Perf report for the last 5.1464 seconds: | 20260505 10:22:26.874 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:22:26.874 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:26.874 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:26.874 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:26.874 | INFO | ANALYSIS : 63.7%, 3.0, 99.6ms, 0 | 20260505 10:22:26.874 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:26.874 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:26.874 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 10:22:26.874 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:26.874 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:26.874 | INFO | Uploaded 36GiB to GPU in 2.22257s (16.1974GiB/s) | 20260505 10:22:26.874 | INFO | Downloaded 34GiB to GPU in 2.10327s (16.1653GiB/s) | 20260505 10:22:32.018 | INFO | Analyzing chunk frames = [2456576,2457088) | 20260505 10:22:32.018 | INFO | MemInfo: { | 20260505 10:22:32.018 | INFO | "gpu" : " 17 GB", | 20260505 10:22:32.018 | INFO | "huge" : "", | 20260505 10:22:32.018 | INFO | "huge_shared" : " 27 GB", | 20260505 10:22:32.018 | INFO | "malloc" : " 14 GB", | 20260505 10:22:32.018 | INFO | "pinned" : " 29 GB" | 20260505 10:22:32.018 | INFO | } | 20260505 10:22:32.018 | INFO | Perf report for the last 5.14413 seconds: | 20260505 10:22:32.018 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:22:32.018 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:32.018 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:32.018 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:32.018 | INFO | ANALYSIS : 63.4%, 3.0, 99.2ms, 0 | 20260505 10:22:32.018 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:32.018 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:32.018 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 10:22:32.018 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:32.018 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:32.018 | INFO | Uploaded 36GiB to GPU in 2.18017s (16.5125GiB/s) | 20260505 10:22:32.018 | INFO | Downloaded 34GiB to GPU in 2.15847s (15.7519GiB/s) | 20260505 10:22:37.153 | INFO | Analyzing chunk frames = [2457088,2457600) | 20260505 10:22:37.153 | INFO | MemInfo: { | 20260505 10:22:37.153 | INFO | "gpu" : " 17 GB", | 20260505 10:22:37.153 | INFO | "huge" : "", | 20260505 10:22:37.153 | INFO | "huge_shared" : " 27 GB", | 20260505 10:22:37.153 | INFO | "malloc" : " 14 GB", | 20260505 10:22:37.153 | INFO | "pinned" : " 29 GB" | 20260505 10:22:37.153 | INFO | } | 20260505 10:22:37.153 | INFO | Perf report for the last 5.13511 seconds: | 20260505 10:22:37.153 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:22:37.153 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:37.153 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:37.153 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:37.153 | INFO | ANALYSIS : 64.4%, 3.0, 100.4ms, 0 | 20260505 10:22:37.153 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:37.153 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:37.153 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 10:22:37.153 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:37.153 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:37.153 | INFO | Uploaded 36GiB to GPU in 2.24335s (16.0475GiB/s) | 20260505 10:22:37.154 | INFO | Downloaded 34GiB to GPU in 2.09935s (16.1955GiB/s) | 20260505 10:22:37.154 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 10:22:37.154 | INFO | Total mem usage: 15.3 GiB | 20260505 10:22:37.154 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 10:22:37.154 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -5.48%) | 20260505 10:22:37.154 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 10:22:37.154 | INFO | High Water Breakdown: | 20260505 10:22:37.154 | INFO | 61.1% : 8.88 GiB ( 94.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 10:22:37.154 | INFO | 38.8% : 5.63 GiB ( 84.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 10:22:37.154 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 10:22:37.154 | INFO | ------------------------------------------ | 20260505 10:22:37.154 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:22:37.154 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 10:22:37.154 | INFO | Total mem usage: 26.5 GiB | 20260505 10:22:37.154 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 10:22:37.154 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 10:22:37.154 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 10:22:37.154 | INFO | High Water Breakdown: | 20260505 10:22:37.154 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:22:37.154 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:22:37.154 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:22:37.154 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:22:37.154 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:22:37.154 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:22:37.154 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 10:22:37.154 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 10:22:37.154 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:22:37.154 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 10:22:37.154 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 10:22:37.154 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 10:22:37.154 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 10:22:37.154 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 10:22:37.154 | INFO | ------------------------------------------ | 20260505 10:22:37.154 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:22:37.154 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 10:22:37.154 | INFO | Total mem usage: 25.6 GiB | 20260505 10:22:37.154 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 10:22:37.154 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 10:22:37.154 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 10:22:37.154 | INFO | High Water Breakdown: | 20260505 10:22:37.154 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 10:22:37.154 | INFO | ------------------------------------------ | 20260505 10:22:37.154 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:22:37.154 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 10:22:37.154 | INFO | Total mem usage: 15.7 GiB | 20260505 10:22:37.154 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 10:22:37.154 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 10:22:37.154 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 10:22:37.154 | INFO | High Water Breakdown: | 20260505 10:22:37.154 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 10:22:37.154 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 10:22:37.154 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 10:22:37.154 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 10:22:37.154 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 10:22:37.154 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 10:22:37.154 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 10:22:37.154 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 10:22:37.154 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 10:22:37.154 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:22:37.154 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 10:22:37.154 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:22:37.154 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 10:22:37.154 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 10:22:37.154 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 10:22:37.154 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 10:22:37.154 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:22:37.154 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 10:22:37.154 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:22:37.154 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:22:37.154 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:22:37.154 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 10:22:37.154 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 10:22:37.155 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:22:37.155 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 10:22:37.155 | INFO | ------------------------------------------ | 20260505 10:22:37.155 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:22:37.155 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 10:22:37.155 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:22:37.155 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:22:37.155 | INFO | PerfReport: Full Execution -- 3.23e+01 / 7.71e+01, 1.37e+01 / 2.32e+00 | 20260505 10:22:37.155 | INFO | PerfReport: 0.29%, AggregateStats -- 9.31e-02 / 1.02e-01, 1.04e-02 / 4.05e-03 | 20260505 10:22:37.155 | INFO | PerfReport: 11.69%, Baseline -- 3.78e+00 / 4.49e+00, 6.06e-01 / 2.09e-01 | 20260505 10:22:37.155 | INFO | PerfReport: 3.32%, Histogram -- 1.07e+00 / 1.51e+00, 2.07e-01 / 6.47e-02 | 20260505 10:22:37.155 | INFO | PerfReport: 15.78%, DME -- 4.08e+01 / 4.52e+01, 4.60e+00 / 2.32e+00 | 20260505 10:22:37.155 | INFO | PerfReport: 56.43%, FrameLabeling -- 1.82e+01 / 2.09e+01, 1.74e+00 / 2.31e+00 | 20260505 10:22:37.155 | INFO | PerfReport: 7.40%, PulseAccumulating -- 2.39e+00 / 2.92e+00, 4.19e-01 / 2.67e-01 | 20260505 10:22:37.155 | INFO | PerfReport: 3.76%, Metrics -- 1.22e+00 / 1.48e+00, 3.48e-01 / 2.34e-01 | 20260505 10:22:37.155 | INFO | PerfReport: 1.33%, PulseSerializing -- 4.30e-01 / 4.72e-01, 7.98e-02 / 4.61e-02 | 20260505 10:22:37.155 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 10:22:37.155 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:22:37.155 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:22:37.155 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.44e+01 / 0.00e+00 | 20260505 10:22:37.155 | INFO | PerfReport: 22.58%, Upload -- 2.29e+01 / 0.00e+00, 8.15e+00 / 0.00e+00 | 20260505 10:22:37.155 | INFO | PerfReport: 21.89%, Download -- 2.22e+01 / 0.00e+00, 8.17e+00 / 0.00e+00 | 20260505 10:22:37.155 | INFO | PerfReport: 31.90%, Compute -- 3.23e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 10:22:37.155 | INFO | PerfReport: 1.88%, LockUpload -- 1.91e+00 / 0.00e+00, 7.50e+00 / 0.00e+00 | 20260505 10:22:37.155 | INFO | PerfReport: 1.22%, LockDownload -- 1.23e+00 / 0.00e+00, 2.79e+00 / 0.00e+00 | 20260505 10:22:37.155 | INFO | PerfReport: 20.52%, LockCompute -- 2.08e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260505 10:22:37.155 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.73e-02 / 0.00e+00 | 20260505 10:22:37.286 | INFO | Starting a round of Baz IO | 20260505 10:22:40.546 | INFO | Populating RTMetrics report | 20260505 10:22:40.547 | INFO | RTMetrics report populated | 20260505 10:22:40.547 | INFO | Opening tmp RTMetrics json file | 20260505 10:22:40.562 | INFO | Serializing RTMetrics to json | 20260505 10:22:40.571 | INFO | Serialization done, swapping tmp file now... | 20260505 10:22:40.589 | INFO | RTMetrics done | 20260505 10:22:40.693 | INFO | Finished a round of Baz IO: Wrote 2.417 GiB in 3.407 seconds | 20260505 10:22:40.693 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.06% / 0.01% / 74.93% | 20260505 10:22:40.693 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.53071% / 1.19209e-05% / 9.93411e-05% | 20260505 10:22:40.693 | INFO | Baz IO overall: | 20260505 10:22:40.693 | INFO | BytesWritten : 1438575835077 | 20260505 10:22:40.693 | INFO | NumEvents : 684908912809 | 20260505 10:22:42.289 | INFO | Analyzing chunk frames = [2457600,2458112) | 20260505 10:22:42.289 | INFO | MemInfo: { | 20260505 10:22:42.289 | INFO | "gpu" : " 17 GB", | 20260505 10:22:42.289 | INFO | "huge" : "", | 20260505 10:22:42.289 | INFO | "huge_shared" : " 27 GB", | 20260505 10:22:42.289 | INFO | "malloc" : " 14 GB", | 20260505 10:22:42.289 | INFO | "pinned" : " 29 GB" | 20260505 10:22:42.289 | INFO | } | 20260505 10:22:42.290 | INFO | Perf report for the last 5.13633 seconds: | 20260505 10:22:42.290 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:22:42.290 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:42.290 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:42.290 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:42.290 | INFO | ANALYSIS : 64.9%, 2.9, 99.3ms, 0 | 20260505 10:22:42.290 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:42.290 | INFO | RT_METRICS : 46.6%, 1.0, 24.4ms, 0 | 20260505 10:22:42.290 | INFO | PRE_HQ : 63.0%, 1.8, 59.7ms, 0 | 20260505 10:22:42.290 | INFO | BAZWRITER : 65.6%, 4.2, 140.3ms, 0 | 20260505 10:22:42.290 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 10:22:42.290 | INFO | Uploaded 36GiB to GPU in 2.24608s (16.0279GiB/s) | 20260505 10:22:42.290 | INFO | Downloaded 38GiB to GPU in 2.38554s (15.9293GiB/s) | 20260505 10:22:47.434 | INFO | Analyzing chunk frames = [2458112,2458624) | 20260505 10:22:47.434 | INFO | MemInfo: { | 20260505 10:22:47.434 | INFO | "gpu" : " 17 GB", | 20260505 10:22:47.434 | INFO | "huge" : "", | 20260505 10:22:47.434 | INFO | "huge_shared" : " 27 GB", | 20260505 10:22:47.434 | INFO | "malloc" : " 14 GB", | 20260505 10:22:47.434 | INFO | "pinned" : " 29 GB" | 20260505 10:22:47.434 | INFO | } | 20260505 10:22:47.434 | INFO | Perf report for the last 5.14476 seconds: | 20260505 10:22:47.434 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:22:47.434 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:47.434 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:47.434 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:47.434 | INFO | ANALYSIS : 64.1%, 3.0, 100.2ms, 0 | 20260505 10:22:47.434 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:47.434 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:47.434 | INFO | PRE_HQ : 29.3%, 1.0, 15.3ms, 0 | 20260505 10:22:47.434 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:47.434 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:47.435 | INFO | Uploaded 36GiB to GPU in 2.17746s (16.533GiB/s) | 20260505 10:22:47.435 | INFO | Downloaded 34GiB to GPU in 2.09259s (16.2478GiB/s) | 20260505 10:22:52.572 | INFO | Analyzing chunk frames = [2458624,2459136) | 20260505 10:22:52.572 | INFO | MemInfo: { | 20260505 10:22:52.572 | INFO | "gpu" : " 17 GB", | 20260505 10:22:52.572 | INFO | "huge" : "", | 20260505 10:22:52.572 | INFO | "huge_shared" : " 27 GB", | 20260505 10:22:52.572 | INFO | "malloc" : " 14 GB", | 20260505 10:22:52.572 | INFO | "pinned" : " 29 GB" | 20260505 10:22:52.572 | INFO | } | 20260505 10:22:52.572 | INFO | Perf report for the last 5.13784 seconds: | 20260505 10:22:52.572 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:22:52.572 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:52.572 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:52.572 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:52.572 | INFO | ANALYSIS : 63.8%, 3.0, 99.3ms, 0 | 20260505 10:22:52.572 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:52.572 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:52.572 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:22:52.572 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:52.572 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:52.572 | INFO | Uploaded 36GiB to GPU in 2.22648s (16.169GiB/s) | 20260505 10:22:52.572 | INFO | Downloaded 34GiB to GPU in 2.12625s (15.9906GiB/s) | 20260505 10:22:57.705 | INFO | Analyzing chunk frames = [2459136,2459648) | 20260505 10:22:57.705 | INFO | MemInfo: { | 20260505 10:22:57.705 | INFO | "gpu" : " 17 GB", | 20260505 10:22:57.705 | INFO | "huge" : "", | 20260505 10:22:57.705 | INFO | "huge_shared" : " 27 GB", | 20260505 10:22:57.705 | INFO | "malloc" : " 14 GB", | 20260505 10:22:57.705 | INFO | "pinned" : " 29 GB" | 20260505 10:22:57.705 | INFO | } | 20260505 10:22:57.705 | INFO | Perf report for the last 5.13319 seconds: | 20260505 10:22:57.705 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:22:57.705 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:57.705 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:57.705 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:57.705 | INFO | ANALYSIS : 63.1%, 3.0, 97.9ms, 0 | 20260505 10:22:57.705 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:57.705 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:57.705 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:22:57.705 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:57.705 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:22:57.706 | INFO | Uploaded 36GiB to GPU in 2.16244s (16.6479GiB/s) | 20260505 10:22:57.706 | INFO | Downloaded 34GiB to GPU in 2.07694s (16.3702GiB/s) | 20260505 10:23:02.851 | INFO | Analyzing chunk frames = [2459648,2460160) | 20260505 10:23:02.851 | INFO | MemInfo: { | 20260505 10:23:02.851 | INFO | "gpu" : " 17 GB", | 20260505 10:23:02.851 | INFO | "huge" : "", | 20260505 10:23:02.851 | INFO | "huge_shared" : " 27 GB", | 20260505 10:23:02.851 | INFO | "malloc" : " 14 GB", | 20260505 10:23:02.851 | INFO | "pinned" : " 29 GB" | 20260505 10:23:02.851 | INFO | } | 20260505 10:23:02.851 | INFO | Perf report for the last 5.14559 seconds: | 20260505 10:23:02.851 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:23:02.851 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:02.851 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:02.851 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:02.851 | INFO | ANALYSIS : 63.6%, 3.0, 99.5ms, 0 | 20260505 10:23:02.851 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:02.851 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:02.851 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 10:23:02.851 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:02.851 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:02.851 | INFO | Uploaded 36GiB to GPU in 2.23882s (16.0799GiB/s) | 20260505 10:23:02.851 | INFO | Downloaded 34GiB to GPU in 2.11064s (16.1089GiB/s) | 20260505 10:23:07.983 | INFO | Analyzing chunk frames = [2460160,2460672) | 20260505 10:23:07.983 | INFO | MemInfo: { | 20260505 10:23:07.983 | INFO | "gpu" : " 17 GB", | 20260505 10:23:07.983 | INFO | "huge" : "", | 20260505 10:23:07.983 | INFO | "huge_shared" : " 27 GB", | 20260505 10:23:07.983 | INFO | "malloc" : " 14 GB", | 20260505 10:23:07.983 | INFO | "pinned" : " 29 GB" | 20260505 10:23:07.983 | INFO | } | 20260505 10:23:07.983 | INFO | Perf report for the last 5.13239 seconds: | 20260505 10:23:07.983 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:23:07.983 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:07.983 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:07.983 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:07.983 | INFO | ANALYSIS : 63.9%, 3.0, 99.6ms, 0 | 20260505 10:23:07.983 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:07.983 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:07.983 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 10:23:07.983 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:07.983 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:07.984 | INFO | Uploaded 36GiB to GPU in 2.23264s (16.1244GiB/s) | 20260505 10:23:07.984 | INFO | Downloaded 34GiB to GPU in 2.1006s (16.1859GiB/s) | 20260505 10:23:13.131 | INFO | Analyzing chunk frames = [2460672,2461184) | 20260505 10:23:13.131 | INFO | MemInfo: { | 20260505 10:23:13.131 | INFO | "gpu" : " 17 GB", | 20260505 10:23:13.131 | INFO | "huge" : "", | 20260505 10:23:13.131 | INFO | "huge_shared" : " 27 GB", | 20260505 10:23:13.131 | INFO | "malloc" : " 14 GB", | 20260505 10:23:13.131 | INFO | "pinned" : " 29 GB" | 20260505 10:23:13.131 | INFO | } | 20260505 10:23:13.131 | INFO | Perf report for the last 5.14753 seconds: | 20260505 10:23:13.131 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:23:13.131 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:13.131 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:13.131 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:13.131 | INFO | ANALYSIS : 63.6%, 3.0, 99.5ms, 0 | 20260505 10:23:13.131 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:13.131 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:13.131 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 10:23:13.131 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:13.131 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:13.131 | INFO | Uploaded 36GiB to GPU in 2.19978s (16.3653GiB/s) | 20260505 10:23:13.131 | INFO | Downloaded 34GiB to GPU in 2.15753s (15.7588GiB/s) | 20260505 10:23:18.275 | INFO | Analyzing chunk frames = [2461184,2461696) | 20260505 10:23:18.275 | INFO | MemInfo: { | 20260505 10:23:18.275 | INFO | "gpu" : " 17 GB", | 20260505 10:23:18.275 | INFO | "huge" : "", | 20260505 10:23:18.275 | INFO | "huge_shared" : " 27 GB", | 20260505 10:23:18.275 | INFO | "malloc" : " 14 GB", | 20260505 10:23:18.275 | INFO | "pinned" : " 29 GB" | 20260505 10:23:18.275 | INFO | } | 20260505 10:23:18.275 | INFO | Perf report for the last 5.14403 seconds: | 20260505 10:23:18.275 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:23:18.275 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:18.275 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:18.275 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:18.275 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 10:23:18.275 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:18.275 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:18.275 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 10:23:18.275 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:18.275 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:18.275 | INFO | Uploaded 36GiB to GPU in 2.22038s (16.2135GiB/s) | 20260505 10:23:18.275 | INFO | Downloaded 34GiB to GPU in 2.08207s (16.3299GiB/s) | 20260505 10:23:18.413 | INFO | Starting a round of Baz IO | 20260505 10:23:21.676 | INFO | Populating RTMetrics report | 20260505 10:23:21.676 | INFO | RTMetrics report populated | 20260505 10:23:21.676 | INFO | Opening tmp RTMetrics json file | 20260505 10:23:21.721 | INFO | Serializing RTMetrics to json | 20260505 10:23:21.725 | INFO | Serialization done, swapping tmp file now... | 20260505 10:23:21.726 | INFO | RTMetrics done | 20260505 10:23:21.818 | INFO | Finished a round of Baz IO: Wrote 2.417 GiB in 3.404 seconds | 20260505 10:23:21.818 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.04% / 0.01% / 74.94% | 20260505 10:23:21.818 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.52546% / 1.58946e-05% / 8.34465e-05% | 20260505 10:23:21.818 | INFO | Baz IO overall: | 20260505 10:23:21.818 | INFO | BytesWritten : 1441171445237 | 20260505 10:23:21.818 | INFO | NumEvents : 686147039190 | 20260505 10:23:23.403 | INFO | Analyzing chunk frames = [2461696,2462208) | 20260505 10:23:23.403 | INFO | MemInfo: { | 20260505 10:23:23.403 | INFO | "gpu" : " 17 GB", | 20260505 10:23:23.403 | INFO | "huge" : "", | 20260505 10:23:23.403 | INFO | "huge_shared" : " 27 GB", | 20260505 10:23:23.403 | INFO | "malloc" : " 14 GB", | 20260505 10:23:23.403 | INFO | "pinned" : " 29 GB" | 20260505 10:23:23.403 | INFO | } | 20260505 10:23:23.403 | INFO | Perf report for the last 5.12809 seconds: | 20260505 10:23:23.403 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:23:23.403 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:23.403 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:23.403 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:23.403 | INFO | ANALYSIS : 65.2%, 2.9, 99.7ms, 0 | 20260505 10:23:23.403 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:23.403 | INFO | RT_METRICS : 46.6%, 1.0, 24.4ms, 0 | 20260505 10:23:23.403 | INFO | PRE_HQ : 61.8%, 1.9, 59.5ms, 0 | 20260505 10:23:23.403 | INFO | BAZWRITER : 65.7%, 4.1, 139.5ms, 0 | 20260505 10:23:23.403 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260505 10:23:23.403 | INFO | Uploaded 36GiB to GPU in 2.19424s (16.4066GiB/s) | 20260505 10:23:23.403 | INFO | Downloaded 38GiB to GPU in 2.30777s (16.4661GiB/s) | 20260505 10:23:28.537 | INFO | Analyzing chunk frames = [2462208,2462720) | 20260505 10:23:28.537 | INFO | MemInfo: { | 20260505 10:23:28.537 | INFO | "gpu" : " 17 GB", | 20260505 10:23:28.537 | INFO | "huge" : "", | 20260505 10:23:28.537 | INFO | "huge_shared" : " 27 GB", | 20260505 10:23:28.537 | INFO | "malloc" : " 14 GB", | 20260505 10:23:28.537 | INFO | "pinned" : " 29 GB" | 20260505 10:23:28.537 | INFO | } | 20260505 10:23:28.537 | INFO | Perf report for the last 5.13394 seconds: | 20260505 10:23:28.537 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:23:28.537 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:28.537 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:28.537 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:28.537 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 10:23:28.537 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:28.537 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:28.537 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:23:28.537 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:28.537 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:28.537 | INFO | Uploaded 36GiB to GPU in 2.22542s (16.1767GiB/s) | 20260505 10:23:28.537 | INFO | Downloaded 34GiB to GPU in 2.10607s (16.1438GiB/s) | 20260505 10:23:33.683 | INFO | Analyzing chunk frames = [2462720,2463232) | 20260505 10:23:33.683 | INFO | MemInfo: { | 20260505 10:23:33.683 | INFO | "gpu" : " 17 GB", | 20260505 10:23:33.683 | INFO | "huge" : "", | 20260505 10:23:33.683 | INFO | "huge_shared" : " 27 GB", | 20260505 10:23:33.683 | INFO | "malloc" : " 14 GB", | 20260505 10:23:33.683 | INFO | "pinned" : " 29 GB" | 20260505 10:23:33.683 | INFO | } | 20260505 10:23:33.683 | INFO | Perf report for the last 5.1457 seconds: | 20260505 10:23:33.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:23:33.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:33.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:33.683 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:33.683 | INFO | ANALYSIS : 63.1%, 3.0, 98.3ms, 0 | 20260505 10:23:33.683 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:33.683 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:33.683 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:23:33.683 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:33.683 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:33.683 | INFO | Uploaded 36GiB to GPU in 2.15242s (16.7254GiB/s) | 20260505 10:23:33.683 | INFO | Downloaded 34GiB to GPU in 2.07345s (16.3978GiB/s) | 20260505 10:23:38.826 | INFO | Analyzing chunk frames = [2463232,2463744) | 20260505 10:23:38.826 | INFO | MemInfo: { | 20260505 10:23:38.826 | INFO | "gpu" : " 17 GB", | 20260505 10:23:38.826 | INFO | "huge" : "", | 20260505 10:23:38.826 | INFO | "huge_shared" : " 27 GB", | 20260505 10:23:38.826 | INFO | "malloc" : " 14 GB", | 20260505 10:23:38.826 | INFO | "pinned" : " 29 GB" | 20260505 10:23:38.826 | INFO | } | 20260505 10:23:38.826 | INFO | Perf report for the last 5.14315 seconds: | 20260505 10:23:38.826 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:23:38.826 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:38.826 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:38.826 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:38.826 | INFO | ANALYSIS : 62.9%, 3.0, 97.8ms, 0 | 20260505 10:23:38.826 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:38.826 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:38.826 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 10:23:38.826 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:38.826 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:38.826 | INFO | Uploaded 36GiB to GPU in 2.14925s (16.75GiB/s) | 20260505 10:23:38.826 | INFO | Downloaded 34GiB to GPU in 2.08008s (16.3455GiB/s) | 20260505 10:23:43.968 | INFO | Analyzing chunk frames = [2463744,2464256) | 20260505 10:23:43.968 | INFO | MemInfo: { | 20260505 10:23:43.968 | INFO | "gpu" : " 17 GB", | 20260505 10:23:43.968 | INFO | "huge" : "", | 20260505 10:23:43.968 | INFO | "huge_shared" : " 27 GB", | 20260505 10:23:43.968 | INFO | "malloc" : " 14 GB", | 20260505 10:23:43.968 | INFO | "pinned" : " 29 GB" | 20260505 10:23:43.968 | INFO | } | 20260505 10:23:43.968 | INFO | Perf report for the last 5.14194 seconds: | 20260505 10:23:43.968 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:23:43.968 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:43.968 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:43.968 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:43.968 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 10:23:43.968 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:43.968 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:43.968 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:23:43.968 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:43.968 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:43.968 | INFO | Uploaded 36GiB to GPU in 2.15056s (16.7398GiB/s) | 20260505 10:23:43.968 | INFO | Downloaded 34GiB to GPU in 2.12753s (15.9809GiB/s) | 20260505 10:23:49.100 | INFO | Analyzing chunk frames = [2464256,2464768) | 20260505 10:23:49.100 | INFO | MemInfo: { | 20260505 10:23:49.100 | INFO | "gpu" : " 17 GB", | 20260505 10:23:49.100 | INFO | "huge" : "", | 20260505 10:23:49.100 | INFO | "huge_shared" : " 27 GB", | 20260505 10:23:49.100 | INFO | "malloc" : " 14 GB", | 20260505 10:23:49.100 | INFO | "pinned" : " 29 GB" | 20260505 10:23:49.100 | INFO | } | 20260505 10:23:49.100 | INFO | Perf report for the last 5.13205 seconds: | 20260505 10:23:49.100 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:23:49.100 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:49.100 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:49.100 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:49.100 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 10:23:49.100 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:49.100 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:49.100 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 10:23:49.100 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:49.100 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:49.100 | INFO | Uploaded 36GiB to GPU in 2.20638s (16.3163GiB/s) | 20260505 10:23:49.100 | INFO | Downloaded 34GiB to GPU in 2.10459s (16.1552GiB/s) | 20260505 10:23:54.241 | INFO | Analyzing chunk frames = [2464768,2465280) | 20260505 10:23:54.241 | INFO | MemInfo: { | 20260505 10:23:54.241 | INFO | "gpu" : " 17 GB", | 20260505 10:23:54.241 | INFO | "huge" : "", | 20260505 10:23:54.241 | INFO | "huge_shared" : " 27 GB", | 20260505 10:23:54.241 | INFO | "malloc" : " 14 GB", | 20260505 10:23:54.241 | INFO | "pinned" : " 29 GB" | 20260505 10:23:54.241 | INFO | } | 20260505 10:23:54.241 | INFO | Perf report for the last 5.1408 seconds: | 20260505 10:23:54.241 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:23:54.241 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:54.241 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:54.241 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:54.241 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 10:23:54.241 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:54.241 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:54.241 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 10:23:54.241 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:54.241 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:54.241 | INFO | Uploaded 36GiB to GPU in 2.12186s (16.9663GiB/s) | 20260505 10:23:54.241 | INFO | Downloaded 34GiB to GPU in 2.14853s (15.8248GiB/s) | 20260505 10:23:59.386 | INFO | Analyzing chunk frames = [2465280,2465792) | 20260505 10:23:59.386 | INFO | MemInfo: { | 20260505 10:23:59.386 | INFO | "gpu" : " 17 GB", | 20260505 10:23:59.386 | INFO | "huge" : "", | 20260505 10:23:59.386 | INFO | "huge_shared" : " 27 GB", | 20260505 10:23:59.386 | INFO | "malloc" : " 14 GB", | 20260505 10:23:59.386 | INFO | "pinned" : " 29 GB" | 20260505 10:23:59.386 | INFO | } | 20260505 10:23:59.386 | INFO | Perf report for the last 5.14529 seconds: | 20260505 10:23:59.386 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:23:59.386 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:59.386 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:59.386 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:59.386 | INFO | ANALYSIS : 64.3%, 3.0, 100.5ms, 0 | 20260505 10:23:59.386 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:59.386 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:59.386 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 10:23:59.386 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:59.386 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:23:59.386 | INFO | Uploaded 36GiB to GPU in 2.24581s (16.0298GiB/s) | 20260505 10:23:59.386 | INFO | Downloaded 34GiB to GPU in 2.09247s (16.2488GiB/s) | 20260505 10:23:59.386 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 10:23:59.386 | INFO | Total mem usage: 15.3 GiB | 20260505 10:23:59.386 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 10:23:59.386 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.61%) | 20260505 10:23:59.386 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 10:23:59.386 | INFO | High Water Breakdown: | 20260505 10:23:59.386 | INFO | 61% : 8.77 GiB ( 93.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 10:23:59.386 | INFO | 38.8% : 5.58 GiB ( 84% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 10:23:59.386 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 10:23:59.386 | INFO | ------------------------------------------ | 20260505 10:23:59.386 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:23:59.386 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 10:23:59.386 | INFO | Total mem usage: 26.5 GiB | 20260505 10:23:59.386 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 10:23:59.386 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 10:23:59.386 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 10:23:59.386 | INFO | High Water Breakdown: | 20260505 10:23:59.386 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:23:59.386 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:23:59.386 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:23:59.386 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:23:59.386 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:23:59.386 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:23:59.386 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 10:23:59.386 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 10:23:59.387 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:23:59.387 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 10:23:59.387 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 10:23:59.387 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 10:23:59.387 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 10:23:59.387 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 10:23:59.387 | INFO | ------------------------------------------ | 20260505 10:23:59.387 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:23:59.387 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 10:23:59.387 | INFO | Total mem usage: 25.6 GiB | 20260505 10:23:59.387 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 10:23:59.387 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 10:23:59.387 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 10:23:59.387 | INFO | High Water Breakdown: | 20260505 10:23:59.387 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 10:23:59.387 | INFO | ------------------------------------------ | 20260505 10:23:59.387 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:23:59.387 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 10:23:59.387 | INFO | Total mem usage: 15.7 GiB | 20260505 10:23:59.387 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 10:23:59.387 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 10:23:59.387 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 10:23:59.387 | INFO | High Water Breakdown: | 20260505 10:23:59.387 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 10:23:59.387 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 10:23:59.387 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 10:23:59.387 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 10:23:59.387 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 10:23:59.387 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 10:23:59.387 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 10:23:59.387 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 10:23:59.387 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 10:23:59.387 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:23:59.387 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 10:23:59.387 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:23:59.387 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 10:23:59.387 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 10:23:59.387 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 10:23:59.387 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 10:23:59.387 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:23:59.387 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 10:23:59.387 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:23:59.387 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:23:59.387 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:23:59.387 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 10:23:59.387 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 10:23:59.387 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:23:59.387 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 10:23:59.387 | INFO | ------------------------------------------ | 20260505 10:23:59.387 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:23:59.387 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 10:23:59.387 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:23:59.387 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:23:59.387 | INFO | PerfReport: Full Execution -- 3.23e+01 / 7.75e+01, 1.39e+01 / 1.68e+00 | 20260505 10:23:59.387 | INFO | PerfReport: 0.29%, AggregateStats -- 9.31e-02 / 1.02e-01, 1.10e-02 / 3.75e-03 | 20260505 10:23:59.387 | INFO | PerfReport: 11.70%, Baseline -- 3.78e+00 / 4.47e+00, 6.08e-01 / 2.36e-01 | 20260505 10:23:59.387 | INFO | PerfReport: 3.32%, Histogram -- 1.07e+00 / 1.50e+00, 2.07e-01 / 8.92e-02 | 20260505 10:23:59.387 | INFO | PerfReport: 15.69%, DME -- 4.06e+01 / 4.49e+01, 4.44e+00 / 2.13e+00 | 20260505 10:23:59.387 | INFO | PerfReport: 56.50%, FrameLabeling -- 1.83e+01 / 2.17e+01, 1.75e+00 / 1.84e+00 | 20260505 10:23:59.387 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.39e+00 / 2.88e+00, 4.17e-01 / 1.75e-01 | 20260505 10:23:59.387 | INFO | PerfReport: 3.78%, Metrics -- 1.22e+00 / 1.51e+00, 3.52e-01 / 2.11e-01 | 20260505 10:23:59.387 | INFO | PerfReport: 1.33%, PulseSerializing -- 4.31e-01 / 4.75e-01, 8.01e-02 / 3.05e-02 | 20260505 10:23:59.387 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 10:23:59.387 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:23:59.387 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:23:59.387 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.42e+01 / 0.00e+00 | 20260505 10:23:59.387 | INFO | PerfReport: 22.58%, Upload -- 2.29e+01 / 0.00e+00, 8.17e+00 / 0.00e+00 | 20260505 10:23:59.387 | INFO | PerfReport: 21.96%, Download -- 2.22e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260505 10:23:59.387 | INFO | PerfReport: 31.92%, Compute -- 3.23e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 10:23:59.387 | INFO | PerfReport: 1.90%, LockUpload -- 1.93e+00 / 0.00e+00, 7.31e+00 / 0.00e+00 | 20260505 10:23:59.387 | INFO | PerfReport: 1.18%, LockDownload -- 1.20e+00 / 0.00e+00, 2.79e+00 / 0.00e+00 | 20260505 10:23:59.387 | INFO | PerfReport: 20.44%, LockCompute -- 2.07e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260505 10:23:59.387 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.64e-02 / 0.00e+00 | 20260505 10:23:59.524 | INFO | Starting a round of Baz IO | 20260505 10:24:02.782 | INFO | Populating RTMetrics report | 20260505 10:24:02.782 | INFO | RTMetrics report populated | 20260505 10:24:02.782 | INFO | Opening tmp RTMetrics json file | 20260505 10:24:02.798 | INFO | Serializing RTMetrics to json | 20260505 10:24:02.802 | INFO | Serialization done, swapping tmp file now... | 20260505 10:24:02.802 | INFO | RTMetrics done | 20260505 10:24:02.955 | INFO | Finished a round of Baz IO: Wrote 2.415 GiB in 3.431 seconds | 20260505 10:24:02.955 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.03% / 0.01% / 74.96% | 20260505 10:24:02.955 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.51921% / 7.94729e-06% / 8.34465e-05% | 20260505 10:24:02.955 | INFO | Baz IO overall: | 20260505 10:24:02.955 | INFO | BytesWritten : 1443764941113 | 20260505 10:24:02.956 | INFO | NumEvents : 687383618698 | 20260505 10:24:04.523 | INFO | Analyzing chunk frames = [2465792,2466304) | 20260505 10:24:04.523 | INFO | MemInfo: { | 20260505 10:24:04.523 | INFO | "gpu" : " 17 GB", | 20260505 10:24:04.523 | INFO | "huge" : "", | 20260505 10:24:04.523 | INFO | "huge_shared" : " 27 GB", | 20260505 10:24:04.523 | INFO | "malloc" : " 14 GB", | 20260505 10:24:04.523 | INFO | "pinned" : " 29 GB" | 20260505 10:24:04.523 | INFO | } | 20260505 10:24:04.523 | INFO | Perf report for the last 5.13708 seconds: | 20260505 10:24:04.523 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:24:04.523 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:04.523 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:04.523 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:04.523 | INFO | ANALYSIS : 65.0%, 2.9, 99.5ms, 0 | 20260505 10:24:04.523 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:04.523 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260505 10:24:04.523 | INFO | PRE_HQ : 61.6%, 1.9, 59.3ms, 0 | 20260505 10:24:04.523 | INFO | BAZWRITER : 66.1%, 4.2, 142.1ms, 0 | 20260505 10:24:04.523 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 10:24:04.523 | INFO | Uploaded 36GiB to GPU in 2.20117s (16.3549GiB/s) | 20260505 10:24:04.523 | INFO | Downloaded 38GiB to GPU in 2.30351s (16.4966GiB/s) | 20260505 10:24:09.654 | INFO | Analyzing chunk frames = [2466304,2466816) | 20260505 10:24:09.654 | INFO | MemInfo: { | 20260505 10:24:09.654 | INFO | "gpu" : " 17 GB", | 20260505 10:24:09.654 | INFO | "huge" : "", | 20260505 10:24:09.654 | INFO | "huge_shared" : " 27 GB", | 20260505 10:24:09.654 | INFO | "malloc" : " 14 GB", | 20260505 10:24:09.654 | INFO | "pinned" : " 29 GB" | 20260505 10:24:09.654 | INFO | } | 20260505 10:24:09.654 | INFO | Perf report for the last 5.13115 seconds: | 20260505 10:24:09.654 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:24:09.654 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:09.654 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:09.654 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:09.654 | INFO | ANALYSIS : 64.3%, 3.0, 100.3ms, 0 | 20260505 10:24:09.654 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:09.654 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:09.654 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:24:09.654 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:09.654 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:09.654 | INFO | Uploaded 36GiB to GPU in 2.17985s (16.5149GiB/s) | 20260505 10:24:09.655 | INFO | Downloaded 34GiB to GPU in 2.09918s (16.1968GiB/s) | 20260505 10:24:14.795 | INFO | Analyzing chunk frames = [2466816,2467328) | 20260505 10:24:14.795 | INFO | MemInfo: { | 20260505 10:24:14.795 | INFO | "gpu" : " 17 GB", | 20260505 10:24:14.795 | INFO | "huge" : "", | 20260505 10:24:14.795 | INFO | "huge_shared" : " 27 GB", | 20260505 10:24:14.795 | INFO | "malloc" : " 14 GB", | 20260505 10:24:14.795 | INFO | "pinned" : " 29 GB" | 20260505 10:24:14.795 | INFO | } | 20260505 10:24:14.795 | INFO | Perf report for the last 5.14067 seconds: | 20260505 10:24:14.795 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:24:14.795 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:14.795 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:14.795 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:14.795 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 10:24:14.795 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:14.795 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:14.795 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:24:14.795 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:14.795 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:14.795 | INFO | Uploaded 36GiB to GPU in 2.17363s (16.5621GiB/s) | 20260505 10:24:14.795 | INFO | Downloaded 34GiB to GPU in 2.07179s (16.4109GiB/s) | 20260505 10:24:19.943 | INFO | Analyzing chunk frames = [2467328,2467840) | 20260505 10:24:19.943 | INFO | MemInfo: { | 20260505 10:24:19.943 | INFO | "gpu" : " 17 GB", | 20260505 10:24:19.943 | INFO | "huge" : "", | 20260505 10:24:19.943 | INFO | "huge_shared" : " 27 GB", | 20260505 10:24:19.943 | INFO | "malloc" : " 14 GB", | 20260505 10:24:19.943 | INFO | "pinned" : " 29 GB" | 20260505 10:24:19.943 | INFO | } | 20260505 10:24:19.943 | INFO | Perf report for the last 5.1483 seconds: | 20260505 10:24:19.943 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:24:19.943 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:19.943 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:19.943 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:19.943 | INFO | ANALYSIS : 63.0%, 3.0, 98.1ms, 0 | 20260505 10:24:19.943 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:19.943 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:19.943 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 10:24:19.943 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:19.943 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:19.944 | INFO | Uploaded 36GiB to GPU in 2.18199s (16.4987GiB/s) | 20260505 10:24:19.944 | INFO | Downloaded 34GiB to GPU in 2.08114s (16.3372GiB/s) | 20260505 10:24:25.082 | INFO | Analyzing chunk frames = [2467840,2468352) | 20260505 10:24:25.082 | INFO | MemInfo: { | 20260505 10:24:25.082 | INFO | "gpu" : " 17 GB", | 20260505 10:24:25.082 | INFO | "huge" : "", | 20260505 10:24:25.082 | INFO | "huge_shared" : " 27 GB", | 20260505 10:24:25.082 | INFO | "malloc" : " 14 GB", | 20260505 10:24:25.082 | INFO | "pinned" : " 29 GB" | 20260505 10:24:25.082 | INFO | } | 20260505 10:24:25.082 | INFO | Perf report for the last 5.13877 seconds: | 20260505 10:24:25.082 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:24:25.082 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:25.082 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:25.082 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:25.082 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 10:24:25.082 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:25.082 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:25.082 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 10:24:25.082 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:25.082 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:25.082 | INFO | Uploaded 36GiB to GPU in 2.15332s (16.7184GiB/s) | 20260505 10:24:25.082 | INFO | Downloaded 34GiB to GPU in 2.12346s (16.0116GiB/s) | 20260505 10:24:30.222 | INFO | Analyzing chunk frames = [2468352,2468864) | 20260505 10:24:30.222 | INFO | MemInfo: { | 20260505 10:24:30.222 | INFO | "gpu" : " 17 GB", | 20260505 10:24:30.222 | INFO | "huge" : "", | 20260505 10:24:30.222 | INFO | "huge_shared" : " 27 GB", | 20260505 10:24:30.222 | INFO | "malloc" : " 14 GB", | 20260505 10:24:30.222 | INFO | "pinned" : " 29 GB" | 20260505 10:24:30.222 | INFO | } | 20260505 10:24:30.222 | INFO | Perf report for the last 5.13968 seconds: | 20260505 10:24:30.222 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:24:30.222 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:30.222 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:30.222 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:30.222 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 10:24:30.222 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:30.222 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:30.222 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:24:30.222 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:30.222 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:30.222 | INFO | Uploaded 36GiB to GPU in 2.1631s (16.6428GiB/s) | 20260505 10:24:30.222 | INFO | Downloaded 34GiB to GPU in 2.11902s (16.0451GiB/s) | 20260505 10:24:35.353 | INFO | Analyzing chunk frames = [2468864,2469376) | 20260505 10:24:35.353 | INFO | MemInfo: { | 20260505 10:24:35.353 | INFO | "gpu" : " 17 GB", | 20260505 10:24:35.353 | INFO | "huge" : "", | 20260505 10:24:35.353 | INFO | "huge_shared" : " 27 GB", | 20260505 10:24:35.353 | INFO | "malloc" : " 14 GB", | 20260505 10:24:35.353 | INFO | "pinned" : " 29 GB" | 20260505 10:24:35.353 | INFO | } | 20260505 10:24:35.353 | INFO | Perf report for the last 5.13147 seconds: | 20260505 10:24:35.353 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:24:35.353 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:35.353 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:35.353 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:35.353 | INFO | ANALYSIS : 63.9%, 3.0, 99.6ms, 0 | 20260505 10:24:35.353 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:35.353 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:35.353 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 10:24:35.353 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:35.353 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:35.353 | INFO | Uploaded 36GiB to GPU in 2.18305s (16.4907GiB/s) | 20260505 10:24:35.353 | INFO | Downloaded 34GiB to GPU in 2.14435s (15.8556GiB/s) | 20260505 10:24:40.492 | INFO | Analyzing chunk frames = [2469376,2469888) | 20260505 10:24:40.492 | INFO | MemInfo: { | 20260505 10:24:40.492 | INFO | "gpu" : " 17 GB", | 20260505 10:24:40.492 | INFO | "huge" : "", | 20260505 10:24:40.492 | INFO | "huge_shared" : " 27 GB", | 20260505 10:24:40.492 | INFO | "malloc" : " 14 GB", | 20260505 10:24:40.492 | INFO | "pinned" : " 29 GB" | 20260505 10:24:40.492 | INFO | } | 20260505 10:24:40.492 | INFO | Perf report for the last 5.13898 seconds: | 20260505 10:24:40.492 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:24:40.492 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:40.492 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:40.492 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:40.492 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 10:24:40.492 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:40.492 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:40.492 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 10:24:40.492 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:40.492 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:40.493 | INFO | Uploaded 36GiB to GPU in 2.20809s (16.3037GiB/s) | 20260505 10:24:40.493 | INFO | Downloaded 34GiB to GPU in 2.0599s (16.5056GiB/s) | 20260505 10:24:40.626 | INFO | Starting a round of Baz IO | 20260505 10:24:43.899 | INFO | Populating RTMetrics report | 20260505 10:24:43.900 | INFO | RTMetrics report populated | 20260505 10:24:43.900 | INFO | Opening tmp RTMetrics json file | 20260505 10:24:43.905 | INFO | Serializing RTMetrics to json | 20260505 10:24:43.906 | INFO | Serialization done, swapping tmp file now... | 20260505 10:24:43.906 | INFO | RTMetrics done | 20260505 10:24:44.024 | INFO | Finished a round of Baz IO: Wrote 2.415 GiB in 3.398 seconds | 20260505 10:24:44.024 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.02% / 0.01% / 74.97% | 20260505 10:24:44.024 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.51467% / 1.58946e-05% / 0.000115236% | 20260505 10:24:44.024 | INFO | Baz IO overall: | 20260505 10:24:44.024 | INFO | BytesWritten : 1446358503827 | 20260505 10:24:44.024 | INFO | NumEvents : 688620406826 | 20260505 10:24:45.635 | INFO | Analyzing chunk frames = [2469888,2470400) | 20260505 10:24:45.635 | INFO | MemInfo: { | 20260505 10:24:45.635 | INFO | "gpu" : " 17 GB", | 20260505 10:24:45.635 | INFO | "huge" : "", | 20260505 10:24:45.635 | INFO | "huge_shared" : " 27 GB", | 20260505 10:24:45.635 | INFO | "malloc" : " 14 GB", | 20260505 10:24:45.635 | INFO | "pinned" : " 29 GB" | 20260505 10:24:45.635 | INFO | } | 20260505 10:24:45.635 | INFO | Perf report for the last 5.14284 seconds: | 20260505 10:24:45.635 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:24:45.635 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:45.635 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:45.635 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:45.635 | INFO | ANALYSIS : 65.1%, 2.9, 99.9ms, 0 | 20260505 10:24:45.635 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:45.635 | INFO | RT_METRICS : 45.8%, 1.0, 24.1ms, 0 | 20260505 10:24:45.635 | INFO | PRE_HQ : 61.8%, 1.9, 59.6ms, 0 | 20260505 10:24:45.635 | INFO | BAZWRITER : 65.3%, 4.1, 138.7ms, 0 | 20260505 10:24:45.635 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 10:24:45.635 | INFO | Uploaded 36GiB to GPU in 2.19668s (16.3884GiB/s) | 20260505 10:24:45.635 | INFO | Downloaded 38GiB to GPU in 2.30722s (16.47GiB/s) | 20260505 10:24:50.782 | INFO | Analyzing chunk frames = [2470400,2470912) | 20260505 10:24:50.782 | INFO | MemInfo: { | 20260505 10:24:50.782 | INFO | "gpu" : " 17 GB", | 20260505 10:24:50.782 | INFO | "huge" : "", | 20260505 10:24:50.782 | INFO | "huge_shared" : " 27 GB", | 20260505 10:24:50.782 | INFO | "malloc" : " 14 GB", | 20260505 10:24:50.782 | INFO | "pinned" : " 29 GB" | 20260505 10:24:50.782 | INFO | } | 20260505 10:24:50.782 | INFO | Perf report for the last 5.14677 seconds: | 20260505 10:24:50.782 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:24:50.782 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:50.782 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:50.782 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:50.782 | INFO | ANALYSIS : 64.4%, 3.0, 100.7ms, 0 | 20260505 10:24:50.782 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:50.782 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:50.782 | INFO | PRE_HQ : 29.1%, 1.0, 15.2ms, 0 | 20260505 10:24:50.782 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:50.782 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:50.782 | INFO | Uploaded 36GiB to GPU in 2.24094s (16.0647GiB/s) | 20260505 10:24:50.782 | INFO | Downloaded 34GiB to GPU in 2.07927s (16.3519GiB/s) | 20260505 10:24:55.913 | INFO | Analyzing chunk frames = [2470912,2471424) | 20260505 10:24:55.914 | INFO | MemInfo: { | 20260505 10:24:55.914 | INFO | "gpu" : " 17 GB", | 20260505 10:24:55.914 | INFO | "huge" : "", | 20260505 10:24:55.914 | INFO | "huge_shared" : " 27 GB", | 20260505 10:24:55.914 | INFO | "malloc" : " 14 GB", | 20260505 10:24:55.914 | INFO | "pinned" : " 29 GB" | 20260505 10:24:55.914 | INFO | } | 20260505 10:24:55.914 | INFO | Perf report for the last 5.13189 seconds: | 20260505 10:24:55.914 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:24:55.914 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:55.914 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:55.914 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:55.914 | INFO | ANALYSIS : 63.9%, 3.0, 99.3ms, 0 | 20260505 10:24:55.914 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:55.914 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:55.914 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 10:24:55.914 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:55.914 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:24:55.914 | INFO | Uploaded 36GiB to GPU in 2.22817s (16.1568GiB/s) | 20260505 10:24:55.914 | INFO | Downloaded 34GiB to GPU in 2.12253s (16.0186GiB/s) | 20260505 10:25:01.050 | INFO | Analyzing chunk frames = [2471424,2471936) | 20260505 10:25:01.050 | INFO | MemInfo: { | 20260505 10:25:01.050 | INFO | "gpu" : " 17 GB", | 20260505 10:25:01.050 | INFO | "huge" : "", | 20260505 10:25:01.050 | INFO | "huge_shared" : " 27 GB", | 20260505 10:25:01.050 | INFO | "malloc" : " 14 GB", | 20260505 10:25:01.050 | INFO | "pinned" : " 29 GB" | 20260505 10:25:01.050 | INFO | } | 20260505 10:25:01.050 | INFO | Perf report for the last 5.13608 seconds: | 20260505 10:25:01.050 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:25:01.050 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:01.050 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:01.050 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:01.050 | INFO | ANALYSIS : 63.1%, 3.0, 98.0ms, 0 | 20260505 10:25:01.050 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:01.050 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:01.050 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:25:01.050 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:01.050 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:01.050 | INFO | Uploaded 36GiB to GPU in 2.17854s (16.5248GiB/s) | 20260505 10:25:01.050 | INFO | Downloaded 34GiB to GPU in 2.07795s (16.3623GiB/s) | 20260505 10:25:06.200 | INFO | Analyzing chunk frames = [2471936,2472448) | 20260505 10:25:06.200 | INFO | MemInfo: { | 20260505 10:25:06.200 | INFO | "gpu" : " 17 GB", | 20260505 10:25:06.200 | INFO | "huge" : "", | 20260505 10:25:06.200 | INFO | "huge_shared" : " 27 GB", | 20260505 10:25:06.200 | INFO | "malloc" : " 14 GB", | 20260505 10:25:06.200 | INFO | "pinned" : " 29 GB" | 20260505 10:25:06.200 | INFO | } | 20260505 10:25:06.200 | INFO | Perf report for the last 5.15015 seconds: | 20260505 10:25:06.200 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:25:06.200 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:06.200 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:06.200 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:06.200 | INFO | ANALYSIS : 63.5%, 3.0, 99.4ms, 0 | 20260505 10:25:06.200 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:06.200 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:06.200 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 10:25:06.200 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:06.200 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:06.200 | INFO | Uploaded 36GiB to GPU in 2.24059s (16.0672GiB/s) | 20260505 10:25:06.200 | INFO | Downloaded 34GiB to GPU in 2.10205s (16.1747GiB/s) | 20260505 10:25:11.333 | INFO | Analyzing chunk frames = [2472448,2472960) | 20260505 10:25:11.333 | INFO | MemInfo: { | 20260505 10:25:11.333 | INFO | "gpu" : " 17 GB", | 20260505 10:25:11.333 | INFO | "huge" : "", | 20260505 10:25:11.333 | INFO | "huge_shared" : " 27 GB", | 20260505 10:25:11.333 | INFO | "malloc" : " 14 GB", | 20260505 10:25:11.333 | INFO | "pinned" : " 29 GB" | 20260505 10:25:11.333 | INFO | } | 20260505 10:25:11.333 | INFO | Perf report for the last 5.13314 seconds: | 20260505 10:25:11.333 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:25:11.333 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:11.333 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:11.333 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:11.333 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 10:25:11.333 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:11.333 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:11.333 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 10:25:11.333 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:11.333 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:11.333 | INFO | Uploaded 36GiB to GPU in 2.20242s (16.3457GiB/s) | 20260505 10:25:11.333 | INFO | Downloaded 34GiB to GPU in 2.10197s (16.1753GiB/s) | 20260505 10:25:16.474 | INFO | Analyzing chunk frames = [2472960,2473472) | 20260505 10:25:16.474 | INFO | MemInfo: { | 20260505 10:25:16.474 | INFO | "gpu" : " 17 GB", | 20260505 10:25:16.474 | INFO | "huge" : "", | 20260505 10:25:16.474 | INFO | "huge_shared" : " 27 GB", | 20260505 10:25:16.474 | INFO | "malloc" : " 14 GB", | 20260505 10:25:16.474 | INFO | "pinned" : " 29 GB" | 20260505 10:25:16.474 | INFO | } | 20260505 10:25:16.474 | INFO | Perf report for the last 5.141 seconds: | 20260505 10:25:16.474 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:25:16.474 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:16.474 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:16.474 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:16.474 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 10:25:16.474 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:16.474 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:16.474 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 10:25:16.474 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:16.474 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:16.475 | INFO | Uploaded 36GiB to GPU in 2.17639s (16.5412GiB/s) | 20260505 10:25:16.475 | INFO | Downloaded 34GiB to GPU in 2.1182s (16.0514GiB/s) | 20260505 10:25:21.611 | INFO | Analyzing chunk frames = [2473472,2473984) | 20260505 10:25:21.611 | INFO | MemInfo: { | 20260505 10:25:21.611 | INFO | "gpu" : " 17 GB", | 20260505 10:25:21.611 | INFO | "huge" : "", | 20260505 10:25:21.611 | INFO | "huge_shared" : " 27 GB", | 20260505 10:25:21.611 | INFO | "malloc" : " 14 GB", | 20260505 10:25:21.611 | INFO | "pinned" : " 29 GB" | 20260505 10:25:21.611 | INFO | } | 20260505 10:25:21.611 | INFO | Perf report for the last 5.13672 seconds: | 20260505 10:25:21.611 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:25:21.611 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:21.611 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:21.611 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:21.611 | INFO | ANALYSIS : 64.6%, 3.0, 100.9ms, 0 | 20260505 10:25:21.611 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:21.611 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:21.611 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 10:25:21.611 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:21.611 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:21.611 | INFO | Uploaded 36GiB to GPU in 2.24891s (16.0077GiB/s) | 20260505 10:25:21.611 | INFO | Downloaded 34GiB to GPU in 2.0864s (16.296GiB/s) | 20260505 10:25:21.611 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 10:25:21.611 | INFO | Total mem usage: 15.3 GiB | 20260505 10:25:21.611 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 10:25:21.611 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.39%) | 20260505 10:25:21.611 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 10:25:21.611 | INFO | High Water Breakdown: | 20260505 10:25:21.611 | INFO | 60.9% : 8.78 GiB ( 93.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 10:25:21.611 | INFO | 38.9% : 5.61 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 10:25:21.611 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 10:25:21.611 | INFO | ------------------------------------------ | 20260505 10:25:21.611 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:25:21.611 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 10:25:21.611 | INFO | Total mem usage: 26.5 GiB | 20260505 10:25:21.611 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 10:25:21.611 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 10:25:21.611 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 10:25:21.611 | INFO | High Water Breakdown: | 20260505 10:25:21.611 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:25:21.611 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:25:21.611 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:25:21.611 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:25:21.611 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:25:21.611 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:25:21.611 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 10:25:21.611 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 10:25:21.611 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:25:21.612 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 10:25:21.612 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 10:25:21.612 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 10:25:21.612 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 10:25:21.612 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 10:25:21.612 | INFO | ------------------------------------------ | 20260505 10:25:21.612 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:25:21.612 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 10:25:21.612 | INFO | Total mem usage: 25.6 GiB | 20260505 10:25:21.612 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 10:25:21.612 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 10:25:21.612 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 10:25:21.612 | INFO | High Water Breakdown: | 20260505 10:25:21.612 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 10:25:21.612 | INFO | ------------------------------------------ | 20260505 10:25:21.612 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:25:21.612 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 10:25:21.612 | INFO | Total mem usage: 15.7 GiB | 20260505 10:25:21.612 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 10:25:21.612 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 10:25:21.612 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 10:25:21.612 | INFO | High Water Breakdown: | 20260505 10:25:21.612 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 10:25:21.612 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 10:25:21.612 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 10:25:21.612 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 10:25:21.612 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 10:25:21.612 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 10:25:21.612 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 10:25:21.612 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 10:25:21.612 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 10:25:21.612 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:25:21.612 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 10:25:21.612 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:25:21.612 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 10:25:21.612 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 10:25:21.612 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 10:25:21.612 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 10:25:21.612 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:25:21.612 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 10:25:21.612 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:25:21.612 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:25:21.612 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:25:21.612 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 10:25:21.612 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 10:25:21.612 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:25:21.612 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 10:25:21.612 | INFO | ------------------------------------------ | 20260505 10:25:21.612 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:25:21.612 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 10:25:21.612 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:25:21.612 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:25:21.612 | INFO | PerfReport: Full Execution -- 3.24e+01 / 7.74e+01, 1.39e+01 / 1.87e+00 | 20260505 10:25:21.612 | INFO | PerfReport: 0.29%, AggregateStats -- 9.33e-02 / 1.03e-01, 1.09e-02 / 5.42e-03 | 20260505 10:25:21.612 | INFO | PerfReport: 11.69%, Baseline -- 3.78e+00 / 4.52e+00, 6.11e-01 / 1.72e-01 | 20260505 10:25:21.612 | INFO | PerfReport: 3.32%, Histogram -- 1.07e+00 / 1.51e+00, 2.10e-01 / 7.11e-02 | 20260505 10:25:21.612 | INFO | PerfReport: 15.80%, DME -- 4.09e+01 / 4.56e+01, 4.61e+00 / 1.95e+00 | 20260505 10:25:21.612 | INFO | PerfReport: 56.47%, FrameLabeling -- 1.83e+01 / 2.07e+01, 1.78e+00 / 2.03e+00 | 20260505 10:25:21.612 | INFO | PerfReport: 7.36%, PulseAccumulating -- 2.38e+00 / 2.95e+00, 4.16e-01 / 2.54e-01 | 20260505 10:25:21.612 | INFO | PerfReport: 3.75%, Metrics -- 1.21e+00 / 1.51e+00, 3.49e-01 / 2.09e-01 | 20260505 10:25:21.612 | INFO | PerfReport: 1.33%, PulseSerializing -- 4.30e-01 / 4.77e-01, 8.16e-02 / 4.59e-02 | 20260505 10:25:21.612 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 10:25:21.612 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:25:21.612 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:25:21.612 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.45e+01 / 0.00e+00 | 20260505 10:25:21.612 | INFO | PerfReport: 22.57%, Upload -- 2.29e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260505 10:25:21.612 | INFO | PerfReport: 21.82%, Download -- 2.21e+01 / 0.00e+00, 8.16e+00 / 0.00e+00 | 20260505 10:25:21.612 | INFO | PerfReport: 31.91%, Compute -- 3.24e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 10:25:21.612 | INFO | PerfReport: 2.04%, LockUpload -- 2.06e+00 / 0.00e+00, 7.64e+00 / 0.00e+00 | 20260505 10:25:21.612 | INFO | PerfReport: 1.07%, LockDownload -- 1.09e+00 / 0.00e+00, 2.57e+00 / 0.00e+00 | 20260505 10:25:21.612 | INFO | PerfReport: 20.57%, LockCompute -- 2.09e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260505 10:25:21.612 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.90e-02 / 0.00e+00 | 20260505 10:25:21.752 | INFO | Starting a round of Baz IO | 20260505 10:25:25.020 | INFO | Populating RTMetrics report | 20260505 10:25:25.020 | INFO | RTMetrics report populated | 20260505 10:25:25.020 | INFO | Opening tmp RTMetrics json file | 20260505 10:25:25.022 | INFO | Serializing RTMetrics to json | 20260505 10:25:25.037 | INFO | Serialization done, swapping tmp file now... | 20260505 10:25:25.048 | INFO | RTMetrics done | 20260505 10:25:25.177 | INFO | Finished a round of Baz IO: Wrote 2.416 GiB in 3.424 seconds | 20260505 10:25:25.177 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.01% / 0.01% / 74.98% | 20260505 10:25:25.177 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.51004% / 2.38419e-05% / 7.54992e-05% | 20260505 10:25:25.177 | INFO | Baz IO overall: | 20260505 10:25:25.177 | INFO | BytesWritten : 1448953175321 | 20260505 10:25:25.177 | INFO | NumEvents : 689858021801 | 20260505 10:25:26.746 | INFO | Analyzing chunk frames = [2473984,2474496) | 20260505 10:25:26.746 | INFO | MemInfo: { | 20260505 10:25:26.746 | INFO | "gpu" : " 17 GB", | 20260505 10:25:26.746 | INFO | "huge" : "", | 20260505 10:25:26.746 | INFO | "huge_shared" : " 27 GB", | 20260505 10:25:26.746 | INFO | "malloc" : " 14 GB", | 20260505 10:25:26.746 | INFO | "pinned" : " 29 GB" | 20260505 10:25:26.746 | INFO | } | 20260505 10:25:26.746 | INFO | Perf report for the last 5.13491 seconds: | 20260505 10:25:26.746 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:25:26.746 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:26.746 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:26.746 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:26.746 | INFO | ANALYSIS : 65.3%, 2.9, 99.9ms, 0 | 20260505 10:25:26.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:26.746 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260505 10:25:26.746 | INFO | PRE_HQ : 61.6%, 1.9, 59.3ms, 0 | 20260505 10:25:26.746 | INFO | BAZWRITER : 66.0%, 4.1, 140.6ms, 0 | 20260505 10:25:26.746 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260505 10:25:26.746 | INFO | Uploaded 36GiB to GPU in 2.19909s (16.3704GiB/s) | 20260505 10:25:26.746 | INFO | Downloaded 38GiB to GPU in 2.31841s (16.3906GiB/s) | 20260505 10:25:31.897 | INFO | Analyzing chunk frames = [2474496,2475008) | 20260505 10:25:31.897 | INFO | MemInfo: { | 20260505 10:25:31.897 | INFO | "gpu" : " 17 GB", | 20260505 10:25:31.897 | INFO | "huge" : "", | 20260505 10:25:31.897 | INFO | "huge_shared" : " 27 GB", | 20260505 10:25:31.897 | INFO | "malloc" : " 14 GB", | 20260505 10:25:31.897 | INFO | "pinned" : " 29 GB" | 20260505 10:25:31.897 | INFO | } | 20260505 10:25:31.897 | INFO | Perf report for the last 5.15121 seconds: | 20260505 10:25:31.897 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:25:31.897 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:31.897 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:31.897 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:31.897 | INFO | ANALYSIS : 64.2%, 3.0, 100.5ms, 0 | 20260505 10:25:31.897 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:31.897 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:31.897 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:25:31.897 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:31.897 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:31.897 | INFO | Uploaded 36GiB to GPU in 2.20883s (16.2982GiB/s) | 20260505 10:25:31.897 | INFO | Downloaded 34GiB to GPU in 2.08671s (16.2936GiB/s) | 20260505 10:25:37.036 | INFO | Analyzing chunk frames = [2475008,2475520) | 20260505 10:25:37.036 | INFO | MemInfo: { | 20260505 10:25:37.036 | INFO | "gpu" : " 17 GB", | 20260505 10:25:37.036 | INFO | "huge" : "", | 20260505 10:25:37.036 | INFO | "huge_shared" : " 27 GB", | 20260505 10:25:37.036 | INFO | "malloc" : " 14 GB", | 20260505 10:25:37.036 | INFO | "pinned" : " 29 GB" | 20260505 10:25:37.036 | INFO | } | 20260505 10:25:37.036 | INFO | Perf report for the last 5.13929 seconds: | 20260505 10:25:37.036 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:25:37.036 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:37.036 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:37.036 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:37.036 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260505 10:25:37.036 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:37.036 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:37.036 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 10:25:37.036 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:37.036 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:37.037 | INFO | Uploaded 36GiB to GPU in 2.20898s (16.2971GiB/s) | 20260505 10:25:37.037 | INFO | Downloaded 34GiB to GPU in 2.1119s (16.0992GiB/s) | 20260505 10:25:42.168 | INFO | Analyzing chunk frames = [2475520,2476032) | 20260505 10:25:42.168 | INFO | MemInfo: { | 20260505 10:25:42.168 | INFO | "gpu" : " 17 GB", | 20260505 10:25:42.168 | INFO | "huge" : "", | 20260505 10:25:42.168 | INFO | "huge_shared" : " 27 GB", | 20260505 10:25:42.168 | INFO | "malloc" : " 14 GB", | 20260505 10:25:42.168 | INFO | "pinned" : " 29 GB" | 20260505 10:25:42.168 | INFO | } | 20260505 10:25:42.168 | INFO | Perf report for the last 5.13205 seconds: | 20260505 10:25:42.168 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:25:42.168 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:42.168 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:42.168 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:42.168 | INFO | ANALYSIS : 64.0%, 3.0, 99.2ms, 0 | 20260505 10:25:42.168 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:42.168 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:42.168 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:25:42.168 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:42.168 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:42.169 | INFO | Uploaded 36GiB to GPU in 2.21985s (16.2173GiB/s) | 20260505 10:25:42.169 | INFO | Downloaded 34GiB to GPU in 2.12867s (15.9724GiB/s) | 20260505 10:25:47.300 | INFO | Analyzing chunk frames = [2476032,2476544) | 20260505 10:25:47.300 | INFO | MemInfo: { | 20260505 10:25:47.300 | INFO | "gpu" : " 17 GB", | 20260505 10:25:47.300 | INFO | "huge" : "", | 20260505 10:25:47.300 | INFO | "huge_shared" : " 27 GB", | 20260505 10:25:47.300 | INFO | "malloc" : " 14 GB", | 20260505 10:25:47.300 | INFO | "pinned" : " 29 GB" | 20260505 10:25:47.300 | INFO | } | 20260505 10:25:47.300 | INFO | Perf report for the last 5.13194 seconds: | 20260505 10:25:47.300 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:25:47.300 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:47.300 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:47.300 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:47.300 | INFO | ANALYSIS : 63.8%, 3.0, 99.5ms, 0 | 20260505 10:25:47.300 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:47.300 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:47.300 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:25:47.300 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:47.300 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:47.301 | INFO | Uploaded 36GiB to GPU in 2.23734s (16.0905GiB/s) | 20260505 10:25:47.301 | INFO | Downloaded 34GiB to GPU in 2.11451s (16.0794GiB/s) | 20260505 10:25:52.446 | INFO | Analyzing chunk frames = [2476544,2477056) | 20260505 10:25:52.446 | INFO | MemInfo: { | 20260505 10:25:52.446 | INFO | "gpu" : " 17 GB", | 20260505 10:25:52.446 | INFO | "huge" : "", | 20260505 10:25:52.446 | INFO | "huge_shared" : " 27 GB", | 20260505 10:25:52.446 | INFO | "malloc" : " 14 GB", | 20260505 10:25:52.446 | INFO | "pinned" : " 29 GB" | 20260505 10:25:52.446 | INFO | } | 20260505 10:25:52.446 | INFO | Perf report for the last 5.14536 seconds: | 20260505 10:25:52.446 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:25:52.446 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:52.446 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:52.446 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:52.446 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 10:25:52.446 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:52.446 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:52.446 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 10:25:52.446 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:52.446 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:52.446 | INFO | Uploaded 36GiB to GPU in 2.23754s (16.0891GiB/s) | 20260505 10:25:52.446 | INFO | Downloaded 34GiB to GPU in 2.10513s (16.151GiB/s) | 20260505 10:25:57.591 | INFO | Analyzing chunk frames = [2477056,2477568) | 20260505 10:25:57.591 | INFO | MemInfo: { | 20260505 10:25:57.591 | INFO | "gpu" : " 17 GB", | 20260505 10:25:57.591 | INFO | "huge" : "", | 20260505 10:25:57.591 | INFO | "huge_shared" : " 27 GB", | 20260505 10:25:57.591 | INFO | "malloc" : " 14 GB", | 20260505 10:25:57.591 | INFO | "pinned" : " 29 GB" | 20260505 10:25:57.591 | INFO | } | 20260505 10:25:57.591 | INFO | Perf report for the last 5.145 seconds: | 20260505 10:25:57.591 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:25:57.591 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:57.591 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:57.591 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:57.591 | INFO | ANALYSIS : 63.6%, 3.0, 99.5ms, 0 | 20260505 10:25:57.591 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:57.591 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:57.591 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 10:25:57.591 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:57.591 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:25:57.591 | INFO | Uploaded 36GiB to GPU in 2.18887s (16.4468GiB/s) | 20260505 10:25:57.591 | INFO | Downloaded 34GiB to GPU in 2.14474s (15.8527GiB/s) | 20260505 10:26:02.725 | INFO | Analyzing chunk frames = [2477568,2478080) | 20260505 10:26:02.725 | INFO | MemInfo: { | 20260505 10:26:02.725 | INFO | "gpu" : " 17 GB", | 20260505 10:26:02.725 | INFO | "huge" : "", | 20260505 10:26:02.725 | INFO | "huge_shared" : " 27 GB", | 20260505 10:26:02.725 | INFO | "malloc" : " 14 GB", | 20260505 10:26:02.725 | INFO | "pinned" : " 29 GB" | 20260505 10:26:02.725 | INFO | } | 20260505 10:26:02.725 | INFO | Perf report for the last 5.13403 seconds: | 20260505 10:26:02.725 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:26:02.725 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:02.725 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:02.725 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:02.725 | INFO | ANALYSIS : 64.2%, 3.0, 100.1ms, 0 | 20260505 10:26:02.725 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:02.725 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:02.725 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260505 10:26:02.725 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:02.725 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:02.725 | INFO | Uploaded 36GiB to GPU in 2.22096s (16.2092GiB/s) | 20260505 10:26:02.725 | INFO | Downloaded 34GiB to GPU in 2.09482s (16.2305GiB/s) | 20260505 10:26:02.854 | INFO | Starting a round of Baz IO | 20260505 10:26:06.137 | INFO | Populating RTMetrics report | 20260505 10:26:06.138 | INFO | RTMetrics report populated | 20260505 10:26:06.138 | INFO | Opening tmp RTMetrics json file | 20260505 10:26:06.158 | INFO | Serializing RTMetrics to json | 20260505 10:26:06.159 | INFO | Serialization done, swapping tmp file now... | 20260505 10:26:06.159 | INFO | RTMetrics done | 20260505 10:26:06.306 | INFO | Finished a round of Baz IO: Wrote 2.413 GiB in 3.452 seconds | 20260505 10:26:06.306 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 25.00% / 0.01% / 74.99% | 20260505 10:26:06.306 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.50204% / 1.98682e-05% / 7.15256e-05% | 20260505 10:26:06.306 | INFO | Baz IO overall: | 20260505 10:26:06.306 | INFO | BytesWritten : 1451544471746 | 20260505 10:26:06.306 | INFO | NumEvents : 691093147003 | 20260505 10:26:07.863 | INFO | Analyzing chunk frames = [2478080,2478592) | 20260505 10:26:07.863 | INFO | MemInfo: { | 20260505 10:26:07.863 | INFO | "gpu" : " 17 GB", | 20260505 10:26:07.863 | INFO | "huge" : "", | 20260505 10:26:07.863 | INFO | "huge_shared" : " 27 GB", | 20260505 10:26:07.863 | INFO | "malloc" : " 14 GB", | 20260505 10:26:07.863 | INFO | "pinned" : " 29 GB" | 20260505 10:26:07.863 | INFO | } | 20260505 10:26:07.863 | INFO | Perf report for the last 5.13857 seconds: | 20260505 10:26:07.863 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:26:07.863 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:07.863 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:07.863 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:07.863 | INFO | ANALYSIS : 65.2%, 2.9, 99.7ms, 0 | 20260505 10:26:07.863 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:07.863 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260505 10:26:07.863 | INFO | PRE_HQ : 62.6%, 1.8, 59.9ms, 0 | 20260505 10:26:07.863 | INFO | BAZWRITER : 66.5%, 4.2, 143.3ms, 0 | 20260505 10:26:07.863 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 10:26:07.864 | INFO | Uploaded 36GiB to GPU in 2.21433s (16.2577GiB/s) | 20260505 10:26:07.864 | INFO | Downloaded 38GiB to GPU in 2.32168s (16.3674GiB/s) | 20260505 10:26:13.007 | INFO | Analyzing chunk frames = [2478592,2479104) | 20260505 10:26:13.007 | INFO | MemInfo: { | 20260505 10:26:13.007 | INFO | "gpu" : " 17 GB", | 20260505 10:26:13.007 | INFO | "huge" : "", | 20260505 10:26:13.007 | INFO | "huge_shared" : " 27 GB", | 20260505 10:26:13.007 | INFO | "malloc" : " 14 GB", | 20260505 10:26:13.007 | INFO | "pinned" : " 29 GB" | 20260505 10:26:13.007 | INFO | } | 20260505 10:26:13.007 | INFO | Perf report for the last 5.1433 seconds: | 20260505 10:26:13.007 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:26:13.007 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:13.007 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:13.007 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:13.007 | INFO | ANALYSIS : 64.1%, 3.0, 100.2ms, 0 | 20260505 10:26:13.007 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:13.007 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:13.007 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:26:13.007 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:13.007 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:13.007 | INFO | Uploaded 36GiB to GPU in 2.17903s (16.5211GiB/s) | 20260505 10:26:13.007 | INFO | Downloaded 34GiB to GPU in 2.09917s (16.1969GiB/s) | 20260505 10:26:18.155 | INFO | Analyzing chunk frames = [2479104,2479616) | 20260505 10:26:18.155 | INFO | MemInfo: { | 20260505 10:26:18.155 | INFO | "gpu" : " 17 GB", | 20260505 10:26:18.155 | INFO | "huge" : "", | 20260505 10:26:18.155 | INFO | "huge_shared" : " 27 GB", | 20260505 10:26:18.155 | INFO | "malloc" : " 14 GB", | 20260505 10:26:18.155 | INFO | "pinned" : " 29 GB" | 20260505 10:26:18.155 | INFO | } | 20260505 10:26:18.155 | INFO | Perf report for the last 5.14818 seconds: | 20260505 10:26:18.155 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:26:18.155 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:18.155 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:18.155 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:18.155 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 10:26:18.155 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:18.155 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:18.155 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:26:18.155 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:18.155 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:18.155 | INFO | Uploaded 36GiB to GPU in 2.22724s (16.1635GiB/s) | 20260505 10:26:18.155 | INFO | Downloaded 34GiB to GPU in 2.12583s (15.9937GiB/s) | 20260505 10:26:23.287 | INFO | Analyzing chunk frames = [2479616,2480128) | 20260505 10:26:23.287 | INFO | MemInfo: { | 20260505 10:26:23.287 | INFO | "gpu" : " 17 GB", | 20260505 10:26:23.287 | INFO | "huge" : "", | 20260505 10:26:23.287 | INFO | "huge_shared" : " 27 GB", | 20260505 10:26:23.287 | INFO | "malloc" : " 14 GB", | 20260505 10:26:23.287 | INFO | "pinned" : " 29 GB" | 20260505 10:26:23.287 | INFO | } | 20260505 10:26:23.287 | INFO | Perf report for the last 5.13233 seconds: | 20260505 10:26:23.287 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:26:23.287 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:23.287 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:23.287 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:23.287 | INFO | ANALYSIS : 63.6%, 3.0, 98.6ms, 0 | 20260505 10:26:23.287 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:23.287 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:23.287 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 10:26:23.287 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:23.287 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:23.287 | INFO | Uploaded 36GiB to GPU in 2.21019s (16.2882GiB/s) | 20260505 10:26:23.287 | INFO | Downloaded 34GiB to GPU in 2.12495s (16.0003GiB/s) | 20260505 10:26:28.419 | INFO | Analyzing chunk frames = [2480128,2480640) | 20260505 10:26:28.419 | INFO | MemInfo: { | 20260505 10:26:28.419 | INFO | "gpu" : " 17 GB", | 20260505 10:26:28.419 | INFO | "huge" : "", | 20260505 10:26:28.419 | INFO | "huge_shared" : " 27 GB", | 20260505 10:26:28.419 | INFO | "malloc" : " 14 GB", | 20260505 10:26:28.419 | INFO | "pinned" : " 29 GB" | 20260505 10:26:28.419 | INFO | } | 20260505 10:26:28.419 | INFO | Perf report for the last 5.13214 seconds: | 20260505 10:26:28.419 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:26:28.419 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:28.419 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:28.419 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:28.419 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 10:26:28.419 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:28.419 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:28.419 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 10:26:28.419 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:28.419 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:28.420 | INFO | Uploaded 36GiB to GPU in 2.2118s (16.2763GiB/s) | 20260505 10:26:28.420 | INFO | Downloaded 34GiB to GPU in 2.12242s (16.0195GiB/s) | 20260505 10:26:33.560 | INFO | Analyzing chunk frames = [2480640,2481152) | 20260505 10:26:33.560 | INFO | MemInfo: { | 20260505 10:26:33.560 | INFO | "gpu" : " 17 GB", | 20260505 10:26:33.560 | INFO | "huge" : "", | 20260505 10:26:33.560 | INFO | "huge_shared" : " 27 GB", | 20260505 10:26:33.560 | INFO | "malloc" : " 14 GB", | 20260505 10:26:33.560 | INFO | "pinned" : " 29 GB" | 20260505 10:26:33.560 | INFO | } | 20260505 10:26:33.560 | INFO | Perf report for the last 5.14094 seconds: | 20260505 10:26:33.560 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:26:33.560 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:33.560 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:33.560 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:33.560 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 10:26:33.560 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:33.560 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:33.560 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 10:26:33.560 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:33.560 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:33.561 | INFO | Uploaded 36GiB to GPU in 2.16792s (16.6058GiB/s) | 20260505 10:26:33.561 | INFO | Downloaded 34GiB to GPU in 2.06054s (16.5005GiB/s) | 20260505 10:26:38.698 | INFO | Analyzing chunk frames = [2481152,2481664) | 20260505 10:26:38.698 | INFO | MemInfo: { | 20260505 10:26:38.698 | INFO | "gpu" : " 17 GB", | 20260505 10:26:38.698 | INFO | "huge" : "", | 20260505 10:26:38.698 | INFO | "huge_shared" : " 27 GB", | 20260505 10:26:38.698 | INFO | "malloc" : " 14 GB", | 20260505 10:26:38.698 | INFO | "pinned" : " 29 GB" | 20260505 10:26:38.698 | INFO | } | 20260505 10:26:38.698 | INFO | Perf report for the last 5.13794 seconds: | 20260505 10:26:38.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:26:38.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:38.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:38.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:38.698 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 10:26:38.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:38.698 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:38.698 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 10:26:38.698 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:38.698 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:38.699 | INFO | Uploaded 36GiB to GPU in 2.18399s (16.4836GiB/s) | 20260505 10:26:38.699 | INFO | Downloaded 34GiB to GPU in 2.12823s (15.9757GiB/s) | 20260505 10:26:43.843 | INFO | Analyzing chunk frames = [2481664,2482176) | 20260505 10:26:43.843 | INFO | MemInfo: { | 20260505 10:26:43.843 | INFO | "gpu" : " 17 GB", | 20260505 10:26:43.843 | INFO | "huge" : "", | 20260505 10:26:43.843 | INFO | "huge_shared" : " 27 GB", | 20260505 10:26:43.843 | INFO | "malloc" : " 14 GB", | 20260505 10:26:43.843 | INFO | "pinned" : " 29 GB" | 20260505 10:26:43.843 | INFO | } | 20260505 10:26:43.843 | INFO | Perf report for the last 5.14456 seconds: | 20260505 10:26:43.844 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:26:43.844 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:43.844 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:43.844 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:43.844 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 10:26:43.844 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:43.844 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:43.844 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 10:26:43.844 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:43.844 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:43.844 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 10:26:43.844 | INFO | Total mem usage: 15.3 GiB | 20260505 10:26:43.844 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 10:26:43.844 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.63%) | 20260505 10:26:43.844 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 10:26:43.844 | INFO | High Water Breakdown: | 20260505 10:26:43.844 | INFO | 61% : 8.77 GiB ( 93.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 10:26:43.844 | INFO | 38.8% : 5.58 GiB ( 84% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 10:26:43.844 | INFO | Uploaded 36GiB to GPU in 2.22051s (16.2125GiB/s) | 20260505 10:26:43.844 | INFO | Downloaded 34GiB to GPU in 2.0913s (16.2578GiB/s) | 20260505 10:26:43.844 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 10:26:43.845 | INFO | ------------------------------------------ | 20260505 10:26:43.845 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:26:43.845 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 10:26:43.845 | INFO | Total mem usage: 26.5 GiB | 20260505 10:26:43.845 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 10:26:43.845 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 10:26:43.845 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 10:26:43.845 | INFO | High Water Breakdown: | 20260505 10:26:43.845 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:26:43.845 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:26:43.845 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:26:43.845 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:26:43.845 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:26:43.845 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:26:43.845 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 10:26:43.845 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 10:26:43.845 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:26:43.845 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 10:26:43.845 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 10:26:43.845 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 10:26:43.845 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 10:26:43.845 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 10:26:43.845 | INFO | ------------------------------------------ | 20260505 10:26:43.845 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:26:43.845 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 10:26:43.845 | INFO | Total mem usage: 25.6 GiB | 20260505 10:26:43.845 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 10:26:43.845 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 10:26:43.845 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 10:26:43.845 | INFO | High Water Breakdown: | 20260505 10:26:43.845 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 10:26:43.845 | INFO | ------------------------------------------ | 20260505 10:26:43.845 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:26:43.845 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 10:26:43.845 | INFO | Total mem usage: 15.7 GiB | 20260505 10:26:43.845 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 10:26:43.845 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 10:26:43.845 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 10:26:43.845 | INFO | High Water Breakdown: | 20260505 10:26:43.845 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 10:26:43.845 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 10:26:43.845 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 10:26:43.845 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 10:26:43.845 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 10:26:43.845 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 10:26:43.845 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 10:26:43.845 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 10:26:43.845 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 10:26:43.845 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:26:43.845 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 10:26:43.845 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:26:43.845 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 10:26:43.845 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 10:26:43.845 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 10:26:43.845 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 10:26:43.845 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:26:43.845 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 10:26:43.845 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:26:43.845 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:26:43.845 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:26:43.845 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 10:26:43.845 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 10:26:43.845 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:26:43.845 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 10:26:43.845 | INFO | ------------------------------------------ | 20260505 10:26:43.845 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:26:43.845 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 10:26:43.845 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:26:43.845 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:26:43.845 | INFO | PerfReport: Full Execution -- 3.24e+01 / 7.81e+01, 1.37e+01 / 1.73e+00 | 20260505 10:26:43.845 | INFO | PerfReport: 0.29%, AggregateStats -- 9.33e-02 / 1.05e-01, 1.17e-02 / 1.75e-02 | 20260505 10:26:43.845 | INFO | PerfReport: 11.69%, Baseline -- 3.78e+00 / 4.48e+00, 6.15e-01 / 2.09e-01 | 20260505 10:26:43.845 | INFO | PerfReport: 3.32%, Histogram -- 1.07e+00 / 1.48e+00, 2.06e-01 / 1.09e-01 | 20260505 10:26:43.845 | INFO | PerfReport: 15.63%, DME -- 4.05e+01 / 4.45e+01, 4.30e+00 / 2.39e+00 | 20260505 10:26:43.845 | INFO | PerfReport: 56.68%, FrameLabeling -- 1.83e+01 / 2.27e+01, 1.83e+00 / 2.01e+00 | 20260505 10:26:43.845 | INFO | PerfReport: 7.32%, PulseAccumulating -- 2.37e+00 / 2.84e+00, 4.00e-01 / 2.18e-01 | 20260505 10:26:43.845 | INFO | PerfReport: 3.75%, Metrics -- 1.21e+00 / 1.50e+00, 3.50e-01 / 1.78e-01 | 20260505 10:26:43.845 | INFO | PerfReport: 1.33%, PulseSerializing -- 4.30e-01 / 4.71e-01, 8.04e-02 / 3.47e-02 | 20260505 10:26:43.845 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 10:26:43.845 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:26:43.845 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:26:43.845 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.44e+01 / 0.00e+00 | 20260505 10:26:43.845 | INFO | PerfReport: 22.65%, Upload -- 2.30e+01 / 0.00e+00, 8.36e+00 / 0.00e+00 | 20260505 10:26:43.845 | INFO | PerfReport: 21.91%, Download -- 2.23e+01 / 0.00e+00, 8.34e+00 / 0.00e+00 | 20260505 10:26:43.845 | INFO | PerfReport: 31.86%, Compute -- 3.24e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 10:26:43.845 | INFO | PerfReport: 2.01%, LockUpload -- 2.04e+00 / 0.00e+00, 7.48e+00 / 0.00e+00 | 20260505 10:26:43.845 | INFO | PerfReport: 1.12%, LockDownload -- 1.14e+00 / 0.00e+00, 2.71e+00 / 0.00e+00 | 20260505 10:26:43.845 | INFO | PerfReport: 20.44%, LockCompute -- 2.08e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260505 10:26:43.845 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.57e-02 / 0.00e+00 | 20260505 10:26:43.982 | INFO | Starting a round of Baz IO | 20260505 10:26:47.250 | INFO | Populating RTMetrics report | 20260505 10:26:47.250 | INFO | RTMetrics report populated | 20260505 10:26:47.250 | INFO | Opening tmp RTMetrics json file | 20260505 10:26:47.270 | INFO | Serializing RTMetrics to json | 20260505 10:26:47.288 | INFO | Serialization done, swapping tmp file now... | 20260505 10:26:47.301 | INFO | RTMetrics done | 20260505 10:26:47.389 | INFO | Finished a round of Baz IO: Wrote 2.414 GiB in 3.406 seconds | 20260505 10:26:47.389 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.99% / 0.01% / 75.00% | 20260505 10:26:47.389 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.49817% / 1.19209e-05% / 9.13938e-05% | 20260505 10:26:47.389 | INFO | Baz IO overall: | 20260505 10:26:47.389 | INFO | BytesWritten : 1454136099739 | 20260505 10:26:47.389 | INFO | NumEvents : 692328505228 | 20260505 10:26:48.990 | INFO | Analyzing chunk frames = [2482176,2482688) | 20260505 10:26:48.990 | INFO | MemInfo: { | 20260505 10:26:48.990 | INFO | "gpu" : " 17 GB", | 20260505 10:26:48.990 | INFO | "huge" : "", | 20260505 10:26:48.990 | INFO | "huge_shared" : " 27 GB", | 20260505 10:26:48.990 | INFO | "malloc" : " 14 GB", | 20260505 10:26:48.990 | INFO | "pinned" : " 29 GB" | 20260505 10:26:48.990 | INFO | } | 20260505 10:26:48.990 | INFO | Perf report for the last 5.1467 seconds: | 20260505 10:26:48.990 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:26:48.990 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:48.990 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:48.990 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:48.990 | INFO | ANALYSIS : 65.0%, 2.9, 99.9ms, 0 | 20260505 10:26:48.990 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:48.990 | INFO | RT_METRICS : 46.5%, 1.0, 24.4ms, 0 | 20260505 10:26:48.990 | INFO | PRE_HQ : 61.6%, 1.8, 59.2ms, 0 | 20260505 10:26:48.990 | INFO | BAZWRITER : 65.5%, 4.2, 141.3ms, 0 | 20260505 10:26:48.990 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 10:26:48.990 | INFO | Uploaded 36GiB to GPU in 2.23916s (16.0774GiB/s) | 20260505 10:26:48.990 | INFO | Downloaded 38GiB to GPU in 2.29989s (16.5225GiB/s) | 20260505 10:26:54.118 | INFO | Analyzing chunk frames = [2482688,2483200) | 20260505 10:26:54.118 | INFO | MemInfo: { | 20260505 10:26:54.118 | INFO | "gpu" : " 17 GB", | 20260505 10:26:54.118 | INFO | "huge" : "", | 20260505 10:26:54.118 | INFO | "huge_shared" : " 27 GB", | 20260505 10:26:54.118 | INFO | "malloc" : " 14 GB", | 20260505 10:26:54.118 | INFO | "pinned" : " 29 GB" | 20260505 10:26:54.118 | INFO | } | 20260505 10:26:54.118 | INFO | Perf report for the last 5.12856 seconds: | 20260505 10:26:54.118 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:26:54.118 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:54.118 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:54.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:54.118 | INFO | ANALYSIS : 64.3%, 3.0, 100.2ms, 0 | 20260505 10:26:54.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:54.118 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:54.118 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 10:26:54.118 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:54.118 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:54.118 | INFO | Uploaded 36GiB to GPU in 2.20895s (16.2973GiB/s) | 20260505 10:26:54.118 | INFO | Downloaded 34GiB to GPU in 2.08722s (16.2896GiB/s) | 20260505 10:26:59.257 | INFO | Analyzing chunk frames = [2483200,2483712) | 20260505 10:26:59.257 | INFO | MemInfo: { | 20260505 10:26:59.257 | INFO | "gpu" : " 17 GB", | 20260505 10:26:59.257 | INFO | "huge" : "", | 20260505 10:26:59.257 | INFO | "huge_shared" : " 27 GB", | 20260505 10:26:59.257 | INFO | "malloc" : " 14 GB", | 20260505 10:26:59.257 | INFO | "pinned" : " 29 GB" | 20260505 10:26:59.258 | INFO | } | 20260505 10:26:59.258 | INFO | Perf report for the last 5.1394 seconds: | 20260505 10:26:59.258 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:26:59.258 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:59.258 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:59.258 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:59.258 | INFO | ANALYSIS : 63.2%, 3.0, 98.3ms, 0 | 20260505 10:26:59.258 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:59.258 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:59.258 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:26:59.258 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:59.258 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:26:59.258 | INFO | Uploaded 36GiB to GPU in 2.14941s (16.7488GiB/s) | 20260505 10:26:59.258 | INFO | Downloaded 34GiB to GPU in 2.06745s (16.4453GiB/s) | 20260505 10:27:04.398 | INFO | Analyzing chunk frames = [2483712,2484224) | 20260505 10:27:04.398 | INFO | MemInfo: { | 20260505 10:27:04.398 | INFO | "gpu" : " 17 GB", | 20260505 10:27:04.398 | INFO | "huge" : "", | 20260505 10:27:04.398 | INFO | "huge_shared" : " 27 GB", | 20260505 10:27:04.398 | INFO | "malloc" : " 14 GB", | 20260505 10:27:04.398 | INFO | "pinned" : " 29 GB" | 20260505 10:27:04.398 | INFO | } | 20260505 10:27:04.398 | INFO | Perf report for the last 5.14075 seconds: | 20260505 10:27:04.398 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:27:04.398 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:04.398 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:04.398 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:04.398 | INFO | ANALYSIS : 63.0%, 3.0, 97.9ms, 0 | 20260505 10:27:04.398 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:04.398 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:04.398 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:27:04.398 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:04.398 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:04.399 | INFO | Uploaded 36GiB to GPU in 2.15434s (16.7105GiB/s) | 20260505 10:27:04.399 | INFO | Downloaded 34GiB to GPU in 2.07577s (16.3795GiB/s) | 20260505 10:27:09.541 | INFO | Analyzing chunk frames = [2484224,2484736) | 20260505 10:27:09.541 | INFO | MemInfo: { | 20260505 10:27:09.541 | INFO | "gpu" : " 17 GB", | 20260505 10:27:09.541 | INFO | "huge" : "", | 20260505 10:27:09.541 | INFO | "huge_shared" : " 27 GB", | 20260505 10:27:09.541 | INFO | "malloc" : " 14 GB", | 20260505 10:27:09.541 | INFO | "pinned" : " 29 GB" | 20260505 10:27:09.541 | INFO | } | 20260505 10:27:09.541 | INFO | Perf report for the last 5.1431 seconds: | 20260505 10:27:09.541 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:27:09.541 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:09.541 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:09.541 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:09.541 | INFO | ANALYSIS : 63.3%, 3.0, 99.0ms, 0 | 20260505 10:27:09.542 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:09.542 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:09.542 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:27:09.542 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:09.542 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:09.542 | INFO | Uploaded 36GiB to GPU in 2.16068s (16.6614GiB/s) | 20260505 10:27:09.542 | INFO | Downloaded 34GiB to GPU in 2.12342s (16.0119GiB/s) | 20260505 10:27:14.669 | INFO | Analyzing chunk frames = [2484736,2485248) | 20260505 10:27:14.669 | INFO | MemInfo: { | 20260505 10:27:14.669 | INFO | "gpu" : " 17 GB", | 20260505 10:27:14.669 | INFO | "huge" : "", | 20260505 10:27:14.669 | INFO | "huge_shared" : " 27 GB", | 20260505 10:27:14.669 | INFO | "malloc" : " 14 GB", | 20260505 10:27:14.669 | INFO | "pinned" : " 29 GB" | 20260505 10:27:14.669 | INFO | } | 20260505 10:27:14.669 | INFO | Perf report for the last 5.12771 seconds: | 20260505 10:27:14.670 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:27:14.670 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:14.670 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:14.670 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:14.670 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 10:27:14.670 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:14.670 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:14.670 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 10:27:14.670 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:14.670 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:14.670 | INFO | Uploaded 36GiB to GPU in 2.2093s (16.2948GiB/s) | 20260505 10:27:14.670 | INFO | Downloaded 34GiB to GPU in 2.09953s (16.1941GiB/s) | 20260505 10:27:19.813 | INFO | Analyzing chunk frames = [2485248,2485760) | 20260505 10:27:19.813 | INFO | MemInfo: { | 20260505 10:27:19.813 | INFO | "gpu" : " 17 GB", | 20260505 10:27:19.813 | INFO | "huge" : "", | 20260505 10:27:19.813 | INFO | "huge_shared" : " 27 GB", | 20260505 10:27:19.813 | INFO | "malloc" : " 14 GB", | 20260505 10:27:19.813 | INFO | "pinned" : " 29 GB" | 20260505 10:27:19.813 | INFO | } | 20260505 10:27:19.813 | INFO | Perf report for the last 5.14434 seconds: | 20260505 10:27:19.814 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:27:19.814 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:19.814 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:19.814 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:19.814 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 10:27:19.814 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:19.814 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:19.814 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 10:27:19.814 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:19.814 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:19.814 | INFO | Uploaded 36GiB to GPU in 2.21657s (16.2413GiB/s) | 20260505 10:27:19.814 | INFO | Downloaded 34GiB to GPU in 2.15112s (15.8057GiB/s) | 20260505 10:27:24.959 | INFO | Analyzing chunk frames = [2485760,2486272) | 20260505 10:27:24.959 | INFO | MemInfo: { | 20260505 10:27:24.959 | INFO | "gpu" : " 17 GB", | 20260505 10:27:24.959 | INFO | "huge" : "", | 20260505 10:27:24.959 | INFO | "huge_shared" : " 27 GB", | 20260505 10:27:24.959 | INFO | "malloc" : " 14 GB", | 20260505 10:27:24.959 | INFO | "pinned" : " 29 GB" | 20260505 10:27:24.959 | INFO | } | 20260505 10:27:24.959 | INFO | Perf report for the last 5.14558 seconds: | 20260505 10:27:24.959 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:27:24.959 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:24.959 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:24.959 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:24.959 | INFO | ANALYSIS : 64.4%, 3.0, 100.7ms, 0 | 20260505 10:27:24.959 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:24.959 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:24.959 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 10:27:24.959 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:24.959 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:24.959 | INFO | Uploaded 36GiB to GPU in 2.2462s (16.0271GiB/s) | 20260505 10:27:24.959 | INFO | Downloaded 34GiB to GPU in 2.09036s (16.2651GiB/s) | 20260505 10:27:25.098 | INFO | Starting a round of Baz IO | 20260505 10:27:28.358 | INFO | Populating RTMetrics report | 20260505 10:27:28.358 | INFO | RTMetrics report populated | 20260505 10:27:28.358 | INFO | Opening tmp RTMetrics json file | 20260505 10:27:28.367 | INFO | Serializing RTMetrics to json | 20260505 10:27:28.369 | INFO | Serialization done, swapping tmp file now... | 20260505 10:27:28.369 | INFO | RTMetrics done | 20260505 10:27:28.520 | INFO | Finished a round of Baz IO: Wrote 2.414 GiB in 3.421 seconds | 20260505 10:27:28.520 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.98% / 0.01% / 75.01% | 20260505 10:27:28.520 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.4949% / 7.94729e-06% / 7.54992e-05% | 20260505 10:27:28.520 | INFO | Baz IO overall: | 20260505 10:27:28.520 | INFO | BytesWritten : 1456727996982 | 20260505 10:27:28.520 | INFO | NumEvents : 693564258286 | 20260505 10:27:30.096 | INFO | Analyzing chunk frames = [2486272,2486784) | 20260505 10:27:30.096 | INFO | MemInfo: { | 20260505 10:27:30.096 | INFO | "gpu" : " 17 GB", | 20260505 10:27:30.096 | INFO | "huge" : "", | 20260505 10:27:30.096 | INFO | "huge_shared" : " 27 GB", | 20260505 10:27:30.096 | INFO | "malloc" : " 14 GB", | 20260505 10:27:30.096 | INFO | "pinned" : " 29 GB" | 20260505 10:27:30.096 | INFO | } | 20260505 10:27:30.096 | INFO | Perf report for the last 5.13714 seconds: | 20260505 10:27:30.096 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:27:30.096 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:30.096 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:30.096 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:30.096 | INFO | ANALYSIS : 65.0%, 2.9, 99.8ms, 0 | 20260505 10:27:30.096 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:30.096 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260505 10:27:30.096 | INFO | PRE_HQ : 61.6%, 1.9, 59.4ms, 0 | 20260505 10:27:30.096 | INFO | BAZWRITER : 65.9%, 4.2, 140.6ms, 0 | 20260505 10:27:30.096 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 10:27:30.096 | INFO | Uploaded 36GiB to GPU in 2.21047s (16.2861GiB/s) | 20260505 10:27:30.097 | INFO | Downloaded 38GiB to GPU in 2.30418s (16.4918GiB/s) | 20260505 10:27:35.236 | INFO | Analyzing chunk frames = [2486784,2487296) | 20260505 10:27:35.236 | INFO | MemInfo: { | 20260505 10:27:35.236 | INFO | "gpu" : " 17 GB", | 20260505 10:27:35.236 | INFO | "huge" : "", | 20260505 10:27:35.236 | INFO | "huge_shared" : " 27 GB", | 20260505 10:27:35.236 | INFO | "malloc" : " 14 GB", | 20260505 10:27:35.236 | INFO | "pinned" : " 29 GB" | 20260505 10:27:35.236 | INFO | } | 20260505 10:27:35.236 | INFO | Perf report for the last 5.14007 seconds: | 20260505 10:27:35.236 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:27:35.236 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:35.236 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:35.236 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:35.236 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 10:27:35.236 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:35.236 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:35.236 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 10:27:35.236 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:35.236 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:35.237 | INFO | Uploaded 36GiB to GPU in 2.17676s (16.5384GiB/s) | 20260505 10:27:35.237 | INFO | Downloaded 34GiB to GPU in 2.096s (16.2214GiB/s) | 20260505 10:27:40.365 | INFO | Analyzing chunk frames = [2487296,2487808) | 20260505 10:27:40.365 | INFO | MemInfo: { | 20260505 10:27:40.365 | INFO | "gpu" : " 17 GB", | 20260505 10:27:40.365 | INFO | "huge" : "", | 20260505 10:27:40.365 | INFO | "huge_shared" : " 27 GB", | 20260505 10:27:40.365 | INFO | "malloc" : " 14 GB", | 20260505 10:27:40.365 | INFO | "pinned" : " 29 GB" | 20260505 10:27:40.365 | INFO | } | 20260505 10:27:40.365 | INFO | Perf report for the last 5.12875 seconds: | 20260505 10:27:40.365 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:27:40.365 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:40.365 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:40.365 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:40.365 | INFO | ANALYSIS : 63.0%, 3.0, 97.9ms, 0 | 20260505 10:27:40.365 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:40.365 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:40.365 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 10:27:40.365 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:40.365 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:40.365 | INFO | Uploaded 36GiB to GPU in 2.14595s (16.7758GiB/s) | 20260505 10:27:40.365 | INFO | Downloaded 34GiB to GPU in 2.05449s (16.5491GiB/s) | 20260505 10:27:45.510 | INFO | Analyzing chunk frames = [2487808,2488320) | 20260505 10:27:45.510 | INFO | MemInfo: { | 20260505 10:27:45.510 | INFO | "gpu" : " 17 GB", | 20260505 10:27:45.510 | INFO | "huge" : "", | 20260505 10:27:45.510 | INFO | "huge_shared" : " 27 GB", | 20260505 10:27:45.510 | INFO | "malloc" : " 14 GB", | 20260505 10:27:45.510 | INFO | "pinned" : " 29 GB" | 20260505 10:27:45.510 | INFO | } | 20260505 10:27:45.510 | INFO | Perf report for the last 5.14464 seconds: | 20260505 10:27:45.510 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:27:45.510 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:45.510 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:45.510 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:45.510 | INFO | ANALYSIS : 63.1%, 3.0, 98.2ms, 0 | 20260505 10:27:45.510 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:45.510 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:45.510 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:27:45.510 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:45.510 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:45.510 | INFO | Uploaded 36GiB to GPU in 2.16248s (16.6476GiB/s) | 20260505 10:27:45.510 | INFO | Downloaded 34GiB to GPU in 2.06857s (16.4365GiB/s) | 20260505 10:27:50.651 | INFO | Analyzing chunk frames = [2488320,2488832) | 20260505 10:27:50.651 | INFO | MemInfo: { | 20260505 10:27:50.651 | INFO | "gpu" : " 17 GB", | 20260505 10:27:50.651 | INFO | "huge" : "", | 20260505 10:27:50.651 | INFO | "huge_shared" : " 27 GB", | 20260505 10:27:50.651 | INFO | "malloc" : " 14 GB", | 20260505 10:27:50.651 | INFO | "pinned" : " 29 GB" | 20260505 10:27:50.651 | INFO | } | 20260505 10:27:50.652 | INFO | Perf report for the last 5.14173 seconds: | 20260505 10:27:50.652 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:27:50.652 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:50.652 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:50.652 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:50.652 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 10:27:50.652 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:50.652 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:50.652 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:27:50.652 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:50.652 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:50.652 | INFO | Uploaded 36GiB to GPU in 2.15355s (16.7166GiB/s) | 20260505 10:27:50.652 | INFO | Downloaded 34GiB to GPU in 2.12708s (15.9843GiB/s) | 20260505 10:27:55.801 | INFO | Analyzing chunk frames = [2488832,2489344) | 20260505 10:27:55.801 | INFO | MemInfo: { | 20260505 10:27:55.801 | INFO | "gpu" : " 17 GB", | 20260505 10:27:55.801 | INFO | "huge" : "", | 20260505 10:27:55.801 | INFO | "huge_shared" : " 27 GB", | 20260505 10:27:55.801 | INFO | "malloc" : " 14 GB", | 20260505 10:27:55.801 | INFO | "pinned" : " 29 GB" | 20260505 10:27:55.801 | INFO | } | 20260505 10:27:55.801 | INFO | Perf report for the last 5.14993 seconds: | 20260505 10:27:55.801 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:27:55.801 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:55.801 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:55.802 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:55.802 | INFO | ANALYSIS : 63.5%, 3.0, 99.4ms, 0 | 20260505 10:27:55.802 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:55.802 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:55.802 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:27:55.802 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:55.802 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:27:55.802 | INFO | Uploaded 36GiB to GPU in 2.16928s (16.5954GiB/s) | 20260505 10:27:55.802 | INFO | Downloaded 34GiB to GPU in 2.12245s (16.0192GiB/s) | 20260505 10:28:00.936 | INFO | Analyzing chunk frames = [2489344,2489856) | 20260505 10:28:00.936 | INFO | MemInfo: { | 20260505 10:28:00.936 | INFO | "gpu" : " 17 GB", | 20260505 10:28:00.936 | INFO | "huge" : "", | 20260505 10:28:00.936 | INFO | "huge_shared" : " 27 GB", | 20260505 10:28:00.936 | INFO | "malloc" : " 14 GB", | 20260505 10:28:00.936 | INFO | "pinned" : " 29 GB" | 20260505 10:28:00.936 | INFO | } | 20260505 10:28:00.936 | INFO | Perf report for the last 5.1343 seconds: | 20260505 10:28:00.936 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:28:00.936 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:00.936 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:00.936 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:00.936 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 10:28:00.936 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:00.936 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:00.936 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 10:28:00.936 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:00.936 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:00.936 | INFO | Uploaded 36GiB to GPU in 2.17718s (16.5352GiB/s) | 20260505 10:28:00.936 | INFO | Downloaded 34GiB to GPU in 2.13993s (15.8884GiB/s) | 20260505 10:28:06.063 | INFO | Analyzing chunk frames = [2489856,2490368) | 20260505 10:28:06.063 | INFO | MemInfo: { | 20260505 10:28:06.063 | INFO | "gpu" : " 17 GB", | 20260505 10:28:06.063 | INFO | "huge" : "", | 20260505 10:28:06.063 | INFO | "huge_shared" : " 27 GB", | 20260505 10:28:06.063 | INFO | "malloc" : " 14 GB", | 20260505 10:28:06.063 | INFO | "pinned" : " 29 GB" | 20260505 10:28:06.063 | INFO | } | 20260505 10:28:06.063 | INFO | Perf report for the last 5.12699 seconds: | 20260505 10:28:06.063 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:28:06.063 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:06.063 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:06.063 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:06.063 | INFO | ANALYSIS : 64.7%, 3.0, 100.8ms, 0 | 20260505 10:28:06.063 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:06.063 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:06.063 | INFO | PRE_HQ : 29.8%, 1.0, 15.4ms, 0 | 20260505 10:28:06.063 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:06.063 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:06.063 | INFO | Uploaded 36GiB to GPU in 2.24742s (16.0184GiB/s) | 20260505 10:28:06.063 | INFO | Downloaded 34GiB to GPU in 2.09024s (16.266GiB/s) | 20260505 10:28:06.063 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 10:28:06.063 | INFO | Total mem usage: 15.3 GiB | 20260505 10:28:06.063 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 10:28:06.063 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.65%) | 20260505 10:28:06.063 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 10:28:06.063 | INFO | High Water Breakdown: | 20260505 10:28:06.063 | INFO | 61% : 8.77 GiB ( 93.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 10:28:06.063 | INFO | 38.8% : 5.58 GiB ( 84% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 10:28:06.063 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 10:28:06.063 | INFO | ------------------------------------------ | 20260505 10:28:06.063 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:28:06.063 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 10:28:06.063 | INFO | Total mem usage: 26.5 GiB | 20260505 10:28:06.063 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 10:28:06.063 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 10:28:06.063 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 10:28:06.063 | INFO | High Water Breakdown: | 20260505 10:28:06.063 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:28:06.063 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:28:06.063 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:28:06.063 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:28:06.063 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:28:06.064 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:28:06.064 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 10:28:06.064 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 10:28:06.064 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:28:06.064 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 10:28:06.064 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 10:28:06.064 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 10:28:06.064 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 10:28:06.064 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 10:28:06.064 | INFO | ------------------------------------------ | 20260505 10:28:06.064 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:28:06.064 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 10:28:06.064 | INFO | Total mem usage: 25.6 GiB | 20260505 10:28:06.064 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 10:28:06.064 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 10:28:06.064 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 10:28:06.064 | INFO | High Water Breakdown: | 20260505 10:28:06.064 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 10:28:06.064 | INFO | ------------------------------------------ | 20260505 10:28:06.064 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:28:06.064 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 10:28:06.064 | INFO | Total mem usage: 15.7 GiB | 20260505 10:28:06.064 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 10:28:06.064 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 10:28:06.064 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 10:28:06.064 | INFO | High Water Breakdown: | 20260505 10:28:06.064 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 10:28:06.064 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 10:28:06.064 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 10:28:06.064 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 10:28:06.064 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 10:28:06.064 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 10:28:06.064 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 10:28:06.064 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 10:28:06.064 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 10:28:06.064 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:28:06.064 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 10:28:06.064 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:28:06.064 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 10:28:06.064 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 10:28:06.064 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 10:28:06.064 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 10:28:06.064 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:28:06.064 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 10:28:06.064 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:28:06.064 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:28:06.064 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:28:06.064 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 10:28:06.064 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 10:28:06.064 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:28:06.064 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 10:28:06.064 | INFO | ------------------------------------------ | 20260505 10:28:06.064 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:28:06.064 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 10:28:06.064 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:28:06.064 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:28:06.064 | INFO | PerfReport: Full Execution -- 3.24e+01 / 7.75e+01, 1.36e+01 / 2.26e+00 | 20260505 10:28:06.064 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 1.00e-01, 1.01e-02 / 4.20e-03 | 20260505 10:28:06.064 | INFO | PerfReport: 11.70%, Baseline -- 3.79e+00 / 4.52e+00, 6.16e-01 / 1.96e-01 | 20260505 10:28:06.064 | INFO | PerfReport: 3.31%, Histogram -- 1.07e+00 / 1.50e+00, 2.08e-01 / 9.88e-02 | 20260505 10:28:06.064 | INFO | PerfReport: 15.88%, DME -- 4.11e+01 / 4.61e+01, 4.56e+00 / 2.19e+00 | 20260505 10:28:06.064 | INFO | PerfReport: 56.38%, FrameLabeling -- 1.82e+01 / 2.04e+01, 1.73e+00 / 2.15e+00 | 20260505 10:28:06.064 | INFO | PerfReport: 7.37%, PulseAccumulating -- 2.39e+00 / 2.94e+00, 4.23e-01 / 2.59e-01 | 20260505 10:28:06.064 | INFO | PerfReport: 3.74%, Metrics -- 1.21e+00 / 1.52e+00, 3.47e-01 / 2.11e-01 | 20260505 10:28:06.064 | INFO | PerfReport: 1.33%, PulseSerializing -- 4.29e-01 / 4.78e-01, 7.92e-02 / 3.93e-02 | 20260505 10:28:06.064 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 10:28:06.064 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:28:06.064 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:28:06.064 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.46e+01 / 0.00e+00 | 20260505 10:28:06.064 | INFO | PerfReport: 22.49%, Upload -- 2.28e+01 / 0.00e+00, 8.16e+00 / 0.00e+00 | 20260505 10:28:06.064 | INFO | PerfReport: 21.83%, Download -- 2.21e+01 / 0.00e+00, 8.13e+00 / 0.00e+00 | 20260505 10:28:06.064 | INFO | PerfReport: 31.91%, Compute -- 3.24e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 10:28:06.064 | INFO | PerfReport: 1.92%, LockUpload -- 1.95e+00 / 0.00e+00, 7.53e+00 / 0.00e+00 | 20260505 10:28:06.064 | INFO | PerfReport: 1.18%, LockDownload -- 1.19e+00 / 0.00e+00, 2.69e+00 / 0.00e+00 | 20260505 10:28:06.064 | INFO | PerfReport: 20.66%, LockCompute -- 2.10e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260505 10:28:06.064 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.77e-02 / 0.00e+00 | 20260505 10:28:06.195 | INFO | Starting a round of Baz IO | 20260505 10:28:09.453 | INFO | Populating RTMetrics report | 20260505 10:28:09.453 | INFO | RTMetrics report populated | 20260505 10:28:09.453 | INFO | Opening tmp RTMetrics json file | 20260505 10:28:09.507 | INFO | Serializing RTMetrics to json | 20260505 10:28:09.529 | INFO | Serialization done, swapping tmp file now... | 20260505 10:28:09.538 | INFO | RTMetrics done | 20260505 10:28:09.658 | INFO | Finished a round of Baz IO: Wrote 2.416 GiB in 3.463 seconds | 20260505 10:28:09.658 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.97% / 0.01% / 75.02% | 20260505 10:28:09.658 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.49294% / 3.97364e-06% / 4.37101e-05% | 20260505 10:28:09.658 | INFO | Baz IO overall: | 20260505 10:28:09.658 | INFO | BytesWritten : 1459321835818 | 20260505 10:28:09.658 | INFO | NumEvents : 694801438549 | 20260505 10:28:11.210 | INFO | Analyzing chunk frames = [2490368,2490880) | 20260505 10:28:11.210 | INFO | MemInfo: { | 20260505 10:28:11.210 | INFO | "gpu" : " 17 GB", | 20260505 10:28:11.210 | INFO | "huge" : "", | 20260505 10:28:11.211 | INFO | "huge_shared" : " 27 GB", | 20260505 10:28:11.211 | INFO | "malloc" : " 14 GB", | 20260505 10:28:11.211 | INFO | "pinned" : " 29 GB" | 20260505 10:28:11.211 | INFO | } | 20260505 10:28:11.211 | INFO | Perf report for the last 5.14779 seconds: | 20260505 10:28:11.211 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:28:11.211 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:11.211 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:11.211 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:11.211 | INFO | ANALYSIS : 64.7%, 2.9, 99.1ms, 0 | 20260505 10:28:11.211 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:11.211 | INFO | RT_METRICS : 47.3%, 1.0, 24.8ms, 0 | 20260505 10:28:11.211 | INFO | PRE_HQ : 62.6%, 1.8, 59.6ms, 0 | 20260505 10:28:11.211 | INFO | BAZWRITER : 66.6%, 4.2, 142.7ms, 0 | 20260505 10:28:11.211 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260505 10:28:11.211 | INFO | Uploaded 36GiB to GPU in 2.28559s (15.7509GiB/s) | 20260505 10:28:11.211 | INFO | Downloaded 38GiB to GPU in 2.38481s (15.9342GiB/s) | 20260505 10:28:16.348 | INFO | Analyzing chunk frames = [2490880,2491392) | 20260505 10:28:16.348 | INFO | MemInfo: { | 20260505 10:28:16.348 | INFO | "gpu" : " 17 GB", | 20260505 10:28:16.348 | INFO | "huge" : "", | 20260505 10:28:16.348 | INFO | "huge_shared" : " 27 GB", | 20260505 10:28:16.348 | INFO | "malloc" : " 14 GB", | 20260505 10:28:16.348 | INFO | "pinned" : " 29 GB" | 20260505 10:28:16.348 | INFO | } | 20260505 10:28:16.348 | INFO | Perf report for the last 5.13784 seconds: | 20260505 10:28:16.348 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:28:16.348 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:16.348 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:16.348 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:16.348 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 10:28:16.348 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:16.349 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:16.349 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:28:16.349 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:16.349 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:16.349 | INFO | Uploaded 36GiB to GPU in 2.21357s (16.2633GiB/s) | 20260505 10:28:16.349 | INFO | Downloaded 34GiB to GPU in 2.09236s (16.2496GiB/s) | 20260505 10:28:21.489 | INFO | Analyzing chunk frames = [2491392,2491904) | 20260505 10:28:21.489 | INFO | MemInfo: { | 20260505 10:28:21.489 | INFO | "gpu" : " 17 GB", | 20260505 10:28:21.489 | INFO | "huge" : "", | 20260505 10:28:21.489 | INFO | "huge_shared" : " 27 GB", | 20260505 10:28:21.489 | INFO | "malloc" : " 14 GB", | 20260505 10:28:21.489 | INFO | "pinned" : " 29 GB" | 20260505 10:28:21.489 | INFO | } | 20260505 10:28:21.489 | INFO | Perf report for the last 5.14048 seconds: | 20260505 10:28:21.489 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:28:21.489 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:21.489 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:21.489 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:21.489 | INFO | ANALYSIS : 63.0%, 3.0, 98.1ms, 0 | 20260505 10:28:21.489 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:21.489 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:21.489 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:28:21.489 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:21.489 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:21.489 | INFO | Uploaded 36GiB to GPU in 2.15305s (16.7204GiB/s) | 20260505 10:28:21.489 | INFO | Downloaded 34GiB to GPU in 2.06555s (16.4605GiB/s) | 20260505 10:28:26.623 | INFO | Analyzing chunk frames = [2491904,2492416) | 20260505 10:28:26.623 | INFO | MemInfo: { | 20260505 10:28:26.623 | INFO | "gpu" : " 17 GB", | 20260505 10:28:26.623 | INFO | "huge" : "", | 20260505 10:28:26.623 | INFO | "huge_shared" : " 27 GB", | 20260505 10:28:26.623 | INFO | "malloc" : " 14 GB", | 20260505 10:28:26.623 | INFO | "pinned" : " 29 GB" | 20260505 10:28:26.624 | INFO | } | 20260505 10:28:26.624 | INFO | Perf report for the last 5.13471 seconds: | 20260505 10:28:26.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:28:26.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:26.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:26.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:26.624 | INFO | ANALYSIS : 63.2%, 3.0, 98.0ms, 0 | 20260505 10:28:26.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:26.624 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:26.624 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:28:26.624 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:26.624 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:26.624 | INFO | Uploaded 36GiB to GPU in 2.15986s (16.6677GiB/s) | 20260505 10:28:26.624 | INFO | Downloaded 34GiB to GPU in 2.07132s (16.4147GiB/s) | 20260505 10:28:31.771 | INFO | Analyzing chunk frames = [2492416,2492928) | 20260505 10:28:31.771 | INFO | MemInfo: { | 20260505 10:28:31.771 | INFO | "gpu" : " 17 GB", | 20260505 10:28:31.771 | INFO | "huge" : "", | 20260505 10:28:31.771 | INFO | "huge_shared" : " 27 GB", | 20260505 10:28:31.771 | INFO | "malloc" : " 14 GB", | 20260505 10:28:31.771 | INFO | "pinned" : " 29 GB" | 20260505 10:28:31.771 | INFO | } | 20260505 10:28:31.771 | INFO | Perf report for the last 5.14736 seconds: | 20260505 10:28:31.771 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:28:31.771 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:31.771 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:31.771 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:31.771 | INFO | ANALYSIS : 63.5%, 3.0, 99.4ms, 0 | 20260505 10:28:31.771 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:31.771 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:31.771 | INFO | PRE_HQ : 29.7%, 1.0, 15.5ms, 0 | 20260505 10:28:31.771 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:31.771 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:31.771 | INFO | Uploaded 36GiB to GPU in 2.23795s (16.0862GiB/s) | 20260505 10:28:31.771 | INFO | Downloaded 34GiB to GPU in 2.10938s (16.1185GiB/s) | 20260505 10:28:36.916 | INFO | Analyzing chunk frames = [2492928,2493440) | 20260505 10:28:36.916 | INFO | MemInfo: { | 20260505 10:28:36.916 | INFO | "gpu" : " 17 GB", | 20260505 10:28:36.916 | INFO | "huge" : "", | 20260505 10:28:36.916 | INFO | "huge_shared" : " 27 GB", | 20260505 10:28:36.916 | INFO | "malloc" : " 14 GB", | 20260505 10:28:36.916 | INFO | "pinned" : " 29 GB" | 20260505 10:28:36.916 | INFO | } | 20260505 10:28:36.916 | INFO | Perf report for the last 5.14538 seconds: | 20260505 10:28:36.916 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:28:36.916 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:36.916 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:36.916 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:36.916 | INFO | ANALYSIS : 63.4%, 3.0, 99.2ms, 0 | 20260505 10:28:36.916 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:36.916 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:36.916 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:28:36.917 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:36.917 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:36.917 | INFO | Uploaded 36GiB to GPU in 2.13954s (16.8261GiB/s) | 20260505 10:28:36.917 | INFO | Downloaded 34GiB to GPU in 2.11423s (16.0815GiB/s) | 20260505 10:28:42.049 | INFO | Analyzing chunk frames = [2493440,2493952) | 20260505 10:28:42.049 | INFO | MemInfo: { | 20260505 10:28:42.049 | INFO | "gpu" : " 17 GB", | 20260505 10:28:42.049 | INFO | "huge" : "", | 20260505 10:28:42.049 | INFO | "huge_shared" : " 27 GB", | 20260505 10:28:42.049 | INFO | "malloc" : " 14 GB", | 20260505 10:28:42.049 | INFO | "pinned" : " 29 GB" | 20260505 10:28:42.049 | INFO | } | 20260505 10:28:42.049 | INFO | Perf report for the last 5.13261 seconds: | 20260505 10:28:42.049 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:28:42.049 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:42.049 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:42.049 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:42.049 | INFO | ANALYSIS : 63.9%, 3.0, 99.6ms, 0 | 20260505 10:28:42.049 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:42.049 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:42.049 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 10:28:42.049 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:42.049 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:42.049 | INFO | Uploaded 36GiB to GPU in 2.18685s (16.462GiB/s) | 20260505 10:28:42.049 | INFO | Downloaded 34GiB to GPU in 2.12168s (16.025GiB/s) | 20260505 10:28:47.178 | INFO | Analyzing chunk frames = [2493952,2494464) | 20260505 10:28:47.178 | INFO | MemInfo: { | 20260505 10:28:47.178 | INFO | "gpu" : " 17 GB", | 20260505 10:28:47.178 | INFO | "huge" : "", | 20260505 10:28:47.178 | INFO | "huge_shared" : " 27 GB", | 20260505 10:28:47.178 | INFO | "malloc" : " 14 GB", | 20260505 10:28:47.178 | INFO | "pinned" : " 29 GB" | 20260505 10:28:47.178 | INFO | } | 20260505 10:28:47.178 | INFO | Perf report for the last 5.12894 seconds: | 20260505 10:28:47.178 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:28:47.178 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:47.178 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:47.178 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:47.178 | INFO | ANALYSIS : 64.7%, 3.0, 100.8ms, 0 | 20260505 10:28:47.178 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:47.178 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:47.178 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 10:28:47.178 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:47.178 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:47.178 | INFO | Uploaded 36GiB to GPU in 2.24704s (16.0211GiB/s) | 20260505 10:28:47.178 | INFO | Downloaded 34GiB to GPU in 2.08876s (16.2776GiB/s) | 20260505 10:28:47.312 | INFO | Starting a round of Baz IO | 20260505 10:28:50.578 | INFO | Populating RTMetrics report | 20260505 10:28:50.578 | INFO | RTMetrics report populated | 20260505 10:28:50.578 | INFO | Opening tmp RTMetrics json file | 20260505 10:28:50.582 | INFO | Serializing RTMetrics to json | 20260505 10:28:50.585 | INFO | Serialization done, swapping tmp file now... | 20260505 10:28:50.585 | INFO | RTMetrics done | 20260505 10:28:50.780 | INFO | Finished a round of Baz IO: Wrote 2.414 GiB in 3.467 seconds | 20260505 10:28:50.780 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.96% / 0.01% / 75.03% | 20260505 10:28:50.780 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.48629% / 2.38419e-05% / 6.75519e-05% | 20260505 10:28:50.780 | INFO | Baz IO overall: | 20260505 10:28:50.780 | INFO | BytesWritten : 1461914001176 | 20260505 10:28:50.780 | INFO | NumEvents : 696037262417 | 20260505 10:28:52.320 | INFO | Analyzing chunk frames = [2494464,2494976) | 20260505 10:28:52.320 | INFO | MemInfo: { | 20260505 10:28:52.320 | INFO | "gpu" : " 17 GB", | 20260505 10:28:52.320 | INFO | "huge" : "", | 20260505 10:28:52.320 | INFO | "huge_shared" : " 27 GB", | 20260505 10:28:52.320 | INFO | "malloc" : " 14 GB", | 20260505 10:28:52.320 | INFO | "pinned" : " 29 GB" | 20260505 10:28:52.320 | INFO | } | 20260505 10:28:52.320 | INFO | Perf report for the last 5.14251 seconds: | 20260505 10:28:52.320 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:28:52.320 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:52.320 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:52.320 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:52.320 | INFO | ANALYSIS : 65.0%, 2.9, 99.1ms, 0 | 20260505 10:28:52.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:52.321 | INFO | RT_METRICS : 46.4%, 1.0, 24.4ms, 0 | 20260505 10:28:52.321 | INFO | PRE_HQ : 63.3%, 1.9, 61.0ms, 0 | 20260505 10:28:52.321 | INFO | BAZWRITER : 66.7%, 4.2, 144.3ms, 0 | 20260505 10:28:52.321 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 10:28:52.321 | INFO | Uploaded 36GiB to GPU in 2.24775s (16.0161GiB/s) | 20260505 10:28:52.321 | INFO | Downloaded 38GiB to GPU in 2.38151s (15.9563GiB/s) | 20260505 10:28:57.465 | INFO | Analyzing chunk frames = [2494976,2495488) | 20260505 10:28:57.465 | INFO | MemInfo: { | 20260505 10:28:57.465 | INFO | "gpu" : " 17 GB", | 20260505 10:28:57.465 | INFO | "huge" : "", | 20260505 10:28:57.465 | INFO | "huge_shared" : " 27 GB", | 20260505 10:28:57.465 | INFO | "malloc" : " 14 GB", | 20260505 10:28:57.465 | INFO | "pinned" : " 29 GB" | 20260505 10:28:57.465 | INFO | } | 20260505 10:28:57.465 | INFO | Perf report for the last 5.14455 seconds: | 20260505 10:28:57.465 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:28:57.465 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:57.465 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:57.465 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:57.465 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 10:28:57.465 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:57.465 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:57.465 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 10:28:57.465 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:57.465 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:28:57.465 | INFO | Uploaded 36GiB to GPU in 2.17844s (16.5256GiB/s) | 20260505 10:28:57.465 | INFO | Downloaded 34GiB to GPU in 2.09615s (16.2202GiB/s) | 20260505 10:29:02.607 | INFO | Analyzing chunk frames = [2495488,2496000) | 20260505 10:29:02.607 | INFO | MemInfo: { | 20260505 10:29:02.607 | INFO | "gpu" : " 17 GB", | 20260505 10:29:02.607 | INFO | "huge" : "", | 20260505 10:29:02.607 | INFO | "huge_shared" : " 27 GB", | 20260505 10:29:02.607 | INFO | "malloc" : " 14 GB", | 20260505 10:29:02.607 | INFO | "pinned" : " 29 GB" | 20260505 10:29:02.607 | INFO | } | 20260505 10:29:02.607 | INFO | Perf report for the last 5.14231 seconds: | 20260505 10:29:02.607 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:29:02.607 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:02.607 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:02.607 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:02.607 | INFO | ANALYSIS : 63.6%, 3.0, 99.1ms, 0 | 20260505 10:29:02.607 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:02.607 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:02.608 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 10:29:02.608 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:02.608 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:02.608 | INFO | Uploaded 36GiB to GPU in 2.23522s (16.1058GiB/s) | 20260505 10:29:02.608 | INFO | Downloaded 34GiB to GPU in 2.12404s (16.0073GiB/s) | 20260505 10:29:07.744 | INFO | Analyzing chunk frames = [2496000,2496512) | 20260505 10:29:07.744 | INFO | MemInfo: { | 20260505 10:29:07.744 | INFO | "gpu" : " 17 GB", | 20260505 10:29:07.744 | INFO | "huge" : "", | 20260505 10:29:07.744 | INFO | "huge_shared" : " 27 GB", | 20260505 10:29:07.744 | INFO | "malloc" : " 14 GB", | 20260505 10:29:07.744 | INFO | "pinned" : " 29 GB" | 20260505 10:29:07.744 | INFO | } | 20260505 10:29:07.744 | INFO | Perf report for the last 5.1368 seconds: | 20260505 10:29:07.744 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:29:07.744 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:07.744 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:07.744 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:07.744 | INFO | ANALYSIS : 62.9%, 3.0, 97.7ms, 0 | 20260505 10:29:07.744 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:07.744 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:07.744 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:29:07.744 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:07.744 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:07.744 | INFO | Uploaded 36GiB to GPU in 2.15321s (16.7192GiB/s) | 20260505 10:29:07.745 | INFO | Downloaded 34GiB to GPU in 2.07109s (16.4165GiB/s) | 20260505 10:29:12.879 | INFO | Analyzing chunk frames = [2496512,2497024) | 20260505 10:29:12.879 | INFO | MemInfo: { | 20260505 10:29:12.879 | INFO | "gpu" : " 17 GB", | 20260505 10:29:12.879 | INFO | "huge" : "", | 20260505 10:29:12.879 | INFO | "huge_shared" : " 27 GB", | 20260505 10:29:12.879 | INFO | "malloc" : " 14 GB", | 20260505 10:29:12.879 | INFO | "pinned" : " 29 GB" | 20260505 10:29:12.879 | INFO | } | 20260505 10:29:12.879 | INFO | Perf report for the last 5.13511 seconds: | 20260505 10:29:12.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:29:12.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:12.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:12.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:12.879 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 10:29:12.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:12.879 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:12.879 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 10:29:12.879 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:12.879 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:12.880 | INFO | Uploaded 36GiB to GPU in 2.23859s (16.0815GiB/s) | 20260505 10:29:12.880 | INFO | Downloaded 34GiB to GPU in 2.10995s (16.1141GiB/s) | 20260505 10:29:18.017 | INFO | Analyzing chunk frames = [2497024,2497536) | 20260505 10:29:18.017 | INFO | MemInfo: { | 20260505 10:29:18.017 | INFO | "gpu" : " 17 GB", | 20260505 10:29:18.017 | INFO | "huge" : "", | 20260505 10:29:18.017 | INFO | "huge_shared" : " 27 GB", | 20260505 10:29:18.017 | INFO | "malloc" : " 14 GB", | 20260505 10:29:18.017 | INFO | "pinned" : " 29 GB" | 20260505 10:29:18.017 | INFO | } | 20260505 10:29:18.018 | INFO | Perf report for the last 5.13819 seconds: | 20260505 10:29:18.018 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:29:18.018 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:18.018 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:18.018 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:18.018 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 10:29:18.018 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:18.018 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:18.018 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:29:18.018 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:18.018 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:18.018 | INFO | Uploaded 36GiB to GPU in 2.16336s (16.6408GiB/s) | 20260505 10:29:18.018 | INFO | Downloaded 34GiB to GPU in 2.0621s (16.488GiB/s) | 20260505 10:29:23.164 | INFO | Analyzing chunk frames = [2497536,2498048) | 20260505 10:29:23.164 | INFO | MemInfo: { | 20260505 10:29:23.164 | INFO | "gpu" : " 17 GB", | 20260505 10:29:23.164 | INFO | "huge" : "", | 20260505 10:29:23.164 | INFO | "huge_shared" : " 27 GB", | 20260505 10:29:23.164 | INFO | "malloc" : " 14 GB", | 20260505 10:29:23.164 | INFO | "pinned" : " 29 GB" | 20260505 10:29:23.164 | INFO | } | 20260505 10:29:23.164 | INFO | Perf report for the last 5.14634 seconds: | 20260505 10:29:23.164 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:29:23.164 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:23.164 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:23.164 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:23.164 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 10:29:23.164 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:23.164 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:23.164 | INFO | PRE_HQ : 30.3%, 1.0, 15.8ms, 0 | 20260505 10:29:23.164 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:23.164 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:23.164 | INFO | Uploaded 36GiB to GPU in 2.19879s (16.3726GiB/s) | 20260505 10:29:23.164 | INFO | Downloaded 34GiB to GPU in 2.11882s (16.0467GiB/s) | 20260505 10:29:28.303 | INFO | Analyzing chunk frames = [2498048,2498560) | 20260505 10:29:28.303 | INFO | MemInfo: { | 20260505 10:29:28.303 | INFO | "gpu" : " 17 GB", | 20260505 10:29:28.303 | INFO | "huge" : "", | 20260505 10:29:28.303 | INFO | "huge_shared" : " 27 GB", | 20260505 10:29:28.303 | INFO | "malloc" : " 14 GB", | 20260505 10:29:28.303 | INFO | "pinned" : " 29 GB" | 20260505 10:29:28.303 | INFO | } | 20260505 10:29:28.303 | INFO | Perf report for the last 5.13945 seconds: | 20260505 10:29:28.303 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:29:28.303 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:28.303 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:28.303 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:28.303 | INFO | ANALYSIS : 64.1%, 3.0, 100.2ms, 0 | 20260505 10:29:28.303 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:28.303 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:28.303 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 10:29:28.304 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:28.304 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:28.304 | INFO | Uploaded 36GiB to GPU in 2.23069s (16.1385GiB/s) | 20260505 10:29:28.304 | INFO | Downloaded 34GiB to GPU in 2.09549s (16.2253GiB/s) | 20260505 10:29:28.304 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 10:29:28.304 | INFO | Total mem usage: 15.3 GiB | 20260505 10:29:28.304 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 4.32%) | 20260505 10:29:28.304 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -5.57%) | 20260505 10:29:28.304 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 10:29:28.304 | INFO | High Water Breakdown: | 20260505 10:29:28.304 | INFO | 61.1% : 8.87 GiB ( 94.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 10:29:28.304 | INFO | 38.8% : 5.63 GiB ( 84.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 10:29:28.304 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 10:29:28.304 | INFO | ------------------------------------------ | 20260505 10:29:28.304 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:29:28.304 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 10:29:28.304 | INFO | Total mem usage: 26.5 GiB | 20260505 10:29:28.304 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 10:29:28.304 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 10:29:28.304 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 10:29:28.304 | INFO | High Water Breakdown: | 20260505 10:29:28.304 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:29:28.304 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:29:28.304 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:29:28.304 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:29:28.304 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:29:28.304 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:29:28.304 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 10:29:28.304 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 10:29:28.304 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:29:28.304 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 10:29:28.304 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 10:29:28.304 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 10:29:28.304 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 10:29:28.304 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 10:29:28.304 | INFO | ------------------------------------------ | 20260505 10:29:28.304 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:29:28.304 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 10:29:28.304 | INFO | Total mem usage: 25.6 GiB | 20260505 10:29:28.304 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 10:29:28.304 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 10:29:28.304 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 10:29:28.304 | INFO | High Water Breakdown: | 20260505 10:29:28.304 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 10:29:28.304 | INFO | ------------------------------------------ | 20260505 10:29:28.304 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:29:28.304 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 10:29:28.304 | INFO | Total mem usage: 15.7 GiB | 20260505 10:29:28.304 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 10:29:28.304 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 10:29:28.304 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 10:29:28.304 | INFO | High Water Breakdown: | 20260505 10:29:28.304 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 10:29:28.304 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 10:29:28.304 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 10:29:28.304 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 10:29:28.304 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 10:29:28.304 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 10:29:28.304 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 10:29:28.304 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 10:29:28.304 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 10:29:28.304 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:29:28.304 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 10:29:28.304 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:29:28.304 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 10:29:28.304 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 10:29:28.305 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 10:29:28.305 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 10:29:28.305 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:29:28.305 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 10:29:28.305 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:29:28.305 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:29:28.305 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:29:28.305 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 10:29:28.305 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 10:29:28.305 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:29:28.305 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 10:29:28.305 | INFO | ------------------------------------------ | 20260505 10:29:28.305 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:29:28.305 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 10:29:28.305 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:29:28.305 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:29:28.305 | INFO | PerfReport: Full Execution -- 3.23e+01 / 7.70e+01, 1.39e+01 / 2.19e+00 | 20260505 10:29:28.305 | INFO | PerfReport: 0.29%, AggregateStats -- 9.35e-02 / 1.04e-01, 1.09e-02 / 4.51e-03 | 20260505 10:29:28.305 | INFO | PerfReport: 11.69%, Baseline -- 3.78e+00 / 4.42e+00, 6.13e-01 / 2.59e-01 | 20260505 10:29:28.305 | INFO | PerfReport: 3.32%, Histogram -- 1.07e+00 / 1.48e+00, 2.07e-01 / 8.70e-02 | 20260505 10:29:28.305 | INFO | PerfReport: 15.67%, DME -- 4.05e+01 / 4.48e+01, 4.39e+00 / 1.80e+00 | 20260505 10:29:28.305 | INFO | PerfReport: 56.57%, FrameLabeling -- 1.83e+01 / 2.14e+01, 1.76e+00 / 2.02e+00 | 20260505 10:29:28.305 | INFO | PerfReport: 7.37%, PulseAccumulating -- 2.38e+00 / 2.90e+00, 4.16e-01 / 2.19e-01 | 20260505 10:29:28.305 | INFO | PerfReport: 3.76%, Metrics -- 1.22e+00 / 1.50e+00, 3.53e-01 / 2.05e-01 | 20260505 10:29:28.305 | INFO | PerfReport: 1.33%, PulseSerializing -- 4.30e-01 / 4.70e-01, 7.85e-02 / 4.15e-02 | 20260505 10:29:28.305 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 10:29:28.305 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:29:28.305 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:29:28.305 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.42e+01 / 0.00e+00 | 20260505 10:29:28.305 | INFO | PerfReport: 22.66%, Upload -- 2.30e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260505 10:29:28.305 | INFO | PerfReport: 21.91%, Download -- 2.22e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260505 10:29:28.305 | INFO | PerfReport: 31.92%, Compute -- 3.23e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 10:29:28.305 | INFO | PerfReport: 1.91%, LockUpload -- 1.93e+00 / 0.00e+00, 7.57e+00 / 0.00e+00 | 20260505 10:29:28.305 | INFO | PerfReport: 1.18%, LockDownload -- 1.19e+00 / 0.00e+00, 2.77e+00 / 0.00e+00 | 20260505 10:29:28.305 | INFO | PerfReport: 20.41%, LockCompute -- 2.07e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260505 10:29:28.305 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.41e-02 / 0.00e+00 | 20260505 10:29:28.435 | INFO | Starting a round of Baz IO | 20260505 10:29:31.697 | INFO | Populating RTMetrics report | 20260505 10:29:31.698 | INFO | RTMetrics report populated | 20260505 10:29:31.698 | INFO | Opening tmp RTMetrics json file | 20260505 10:29:31.739 | INFO | Serializing RTMetrics to json | 20260505 10:29:31.763 | INFO | Serialization done, swapping tmp file now... | 20260505 10:29:31.764 | INFO | RTMetrics done | 20260505 10:29:31.885 | INFO | Finished a round of Baz IO: Wrote 2.416 GiB in 3.45 seconds | 20260505 10:29:31.885 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.95% / 0.01% / 75.04% | 20260505 10:29:31.885 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.48886% / 1.58946e-05% / 8.34465e-05% | 20260505 10:29:31.885 | INFO | Baz IO overall: | 20260505 10:29:31.885 | INFO | BytesWritten : 1464508356083 | 20260505 10:29:31.885 | INFO | NumEvents : 697275108464 | 20260505 10:29:33.437 | INFO | Analyzing chunk frames = [2498560,2499072) | 20260505 10:29:33.437 | INFO | MemInfo: { | 20260505 10:29:33.437 | INFO | "gpu" : " 17 GB", | 20260505 10:29:33.437 | INFO | "huge" : "", | 20260505 10:29:33.437 | INFO | "huge_shared" : " 27 GB", | 20260505 10:29:33.437 | INFO | "malloc" : " 14 GB", | 20260505 10:29:33.437 | INFO | "pinned" : " 29 GB" | 20260505 10:29:33.437 | INFO | } | 20260505 10:29:33.437 | INFO | Perf report for the last 5.13363 seconds: | 20260505 10:29:33.437 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:29:33.437 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:33.437 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:33.437 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:33.437 | INFO | ANALYSIS : 65.0%, 2.9, 99.6ms, 0 | 20260505 10:29:33.437 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:33.437 | INFO | RT_METRICS : 48.7%, 1.0, 25.5ms, 0 | 20260505 10:29:33.437 | INFO | PRE_HQ : 63.1%, 1.9, 62.2ms, 0 | 20260505 10:29:33.437 | INFO | BAZWRITER : 66.5%, 4.3, 145.3ms, 0 | 20260505 10:29:33.437 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260505 10:29:33.437 | INFO | Uploaded 36GiB to GPU in 2.26376s (15.9027GiB/s) | 20260505 10:29:33.437 | INFO | Downloaded 38GiB to GPU in 2.37244s (16.0172GiB/s) | 20260505 10:29:38.575 | INFO | Analyzing chunk frames = [2499072,2499584) | 20260505 10:29:38.575 | INFO | MemInfo: { | 20260505 10:29:38.575 | INFO | "gpu" : " 17 GB", | 20260505 10:29:38.575 | INFO | "huge" : "", | 20260505 10:29:38.575 | INFO | "huge_shared" : " 27 GB", | 20260505 10:29:38.575 | INFO | "malloc" : " 14 GB", | 20260505 10:29:38.575 | INFO | "pinned" : " 29 GB" | 20260505 10:29:38.575 | INFO | } | 20260505 10:29:38.575 | INFO | Perf report for the last 5.13824 seconds: | 20260505 10:29:38.575 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:29:38.575 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:38.575 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:38.575 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:38.575 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 10:29:38.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:38.575 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:38.575 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:29:38.575 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:38.575 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:38.576 | INFO | Uploaded 36GiB to GPU in 2.1776s (16.5319GiB/s) | 20260505 10:29:38.576 | INFO | Downloaded 34GiB to GPU in 2.09086s (16.2612GiB/s) | 20260505 10:29:43.722 | INFO | Analyzing chunk frames = [2499584,2500096) | 20260505 10:29:43.722 | INFO | MemInfo: { | 20260505 10:29:43.722 | INFO | "gpu" : " 17 GB", | 20260505 10:29:43.722 | INFO | "huge" : "", | 20260505 10:29:43.722 | INFO | "huge_shared" : " 27 GB", | 20260505 10:29:43.722 | INFO | "malloc" : " 14 GB", | 20260505 10:29:43.722 | INFO | "pinned" : " 29 GB" | 20260505 10:29:43.722 | INFO | } | 20260505 10:29:43.722 | INFO | Perf report for the last 5.14655 seconds: | 20260505 10:29:43.722 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:29:43.722 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:43.722 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:43.722 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:43.722 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260505 10:29:43.722 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:43.722 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:43.722 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:29:43.722 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:43.722 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:43.722 | INFO | Uploaded 36GiB to GPU in 2.14711s (16.7668GiB/s) | 20260505 10:29:43.722 | INFO | Downloaded 34GiB to GPU in 2.05975s (16.5069GiB/s) | 20260505 10:29:48.858 | INFO | Analyzing chunk frames = [2500096,2500608) | 20260505 10:29:48.858 | INFO | MemInfo: { | 20260505 10:29:48.858 | INFO | "gpu" : " 17 GB", | 20260505 10:29:48.858 | INFO | "huge" : "", | 20260505 10:29:48.858 | INFO | "huge_shared" : " 27 GB", | 20260505 10:29:48.858 | INFO | "malloc" : " 14 GB", | 20260505 10:29:48.858 | INFO | "pinned" : " 29 GB" | 20260505 10:29:48.858 | INFO | } | 20260505 10:29:48.859 | INFO | Perf report for the last 5.13684 seconds: | 20260505 10:29:48.859 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:29:48.859 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:48.859 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:48.859 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:48.859 | INFO | ANALYSIS : 63.7%, 3.0, 99.0ms, 0 | 20260505 10:29:48.859 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:48.859 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:48.859 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 10:29:48.859 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:48.859 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:48.859 | INFO | Uploaded 36GiB to GPU in 2.22365s (16.1896GiB/s) | 20260505 10:29:48.859 | INFO | Downloaded 34GiB to GPU in 2.12915s (15.9688GiB/s) | 20260505 10:29:53.995 | INFO | Analyzing chunk frames = [2500608,2501120) | 20260505 10:29:53.995 | INFO | MemInfo: { | 20260505 10:29:53.995 | INFO | "gpu" : " 17 GB", | 20260505 10:29:53.995 | INFO | "huge" : "", | 20260505 10:29:53.995 | INFO | "huge_shared" : " 27 GB", | 20260505 10:29:53.995 | INFO | "malloc" : " 14 GB", | 20260505 10:29:53.995 | INFO | "pinned" : " 29 GB" | 20260505 10:29:53.995 | INFO | } | 20260505 10:29:53.995 | INFO | Perf report for the last 5.13642 seconds: | 20260505 10:29:53.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:29:53.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:53.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:53.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:53.995 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 10:29:53.995 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:53.995 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:53.995 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 10:29:53.995 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:53.995 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:53.995 | INFO | Uploaded 36GiB to GPU in 2.2154s (16.2499GiB/s) | 20260505 10:29:53.996 | INFO | Downloaded 34GiB to GPU in 2.12326s (16.0131GiB/s) | 20260505 10:29:59.130 | INFO | Analyzing chunk frames = [2501120,2501632) | 20260505 10:29:59.131 | INFO | MemInfo: { | 20260505 10:29:59.131 | INFO | "gpu" : " 17 GB", | 20260505 10:29:59.131 | INFO | "huge" : "", | 20260505 10:29:59.131 | INFO | "huge_shared" : " 27 GB", | 20260505 10:29:59.131 | INFO | "malloc" : " 14 GB", | 20260505 10:29:59.131 | INFO | "pinned" : " 29 GB" | 20260505 10:29:59.131 | INFO | } | 20260505 10:29:59.131 | INFO | Perf report for the last 5.13551 seconds: | 20260505 10:29:59.131 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:29:59.131 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:59.131 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:59.131 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:59.131 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 10:29:59.131 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:59.131 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:59.131 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 10:29:59.131 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:59.131 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:29:59.131 | INFO | Uploaded 36GiB to GPU in 2.16377s (16.6376GiB/s) | 20260505 10:29:59.131 | INFO | Downloaded 34GiB to GPU in 2.0598s (16.5065GiB/s) | 20260505 10:30:04.278 | INFO | Analyzing chunk frames = [2501632,2502144) | 20260505 10:30:04.278 | INFO | MemInfo: { | 20260505 10:30:04.278 | INFO | "gpu" : " 17 GB", | 20260505 10:30:04.278 | INFO | "huge" : "", | 20260505 10:30:04.278 | INFO | "huge_shared" : " 27 GB", | 20260505 10:30:04.278 | INFO | "malloc" : " 14 GB", | 20260505 10:30:04.278 | INFO | "pinned" : " 29 GB" | 20260505 10:30:04.278 | INFO | } | 20260505 10:30:04.278 | INFO | Perf report for the last 5.14737 seconds: | 20260505 10:30:04.278 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:30:04.278 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:04.278 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:04.278 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:04.278 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 10:30:04.278 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:04.278 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:04.278 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 10:30:04.278 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:04.278 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:04.278 | INFO | Uploaded 36GiB to GPU in 2.21968s (16.2186GiB/s) | 20260505 10:30:04.279 | INFO | Downloaded 34GiB to GPU in 2.14642s (15.8403GiB/s) | 20260505 10:30:09.417 | INFO | Analyzing chunk frames = [2502144,2502656) | 20260505 10:30:09.417 | INFO | MemInfo: { | 20260505 10:30:09.417 | INFO | "gpu" : " 17 GB", | 20260505 10:30:09.417 | INFO | "huge" : "", | 20260505 10:30:09.417 | INFO | "huge_shared" : " 27 GB", | 20260505 10:30:09.417 | INFO | "malloc" : " 14 GB", | 20260505 10:30:09.417 | INFO | "pinned" : " 29 GB" | 20260505 10:30:09.417 | INFO | } | 20260505 10:30:09.417 | INFO | Perf report for the last 5.13901 seconds: | 20260505 10:30:09.417 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:30:09.417 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:09.417 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:09.417 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:09.417 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 10:30:09.417 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:09.417 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:09.417 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 10:30:09.417 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:09.417 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:09.417 | INFO | Uploaded 36GiB to GPU in 2.21954s (16.2196GiB/s) | 20260505 10:30:09.417 | INFO | Downloaded 34GiB to GPU in 2.08563s (16.302GiB/s) | 20260505 10:30:09.555 | INFO | Starting a round of Baz IO | 20260505 10:30:12.818 | INFO | Populating RTMetrics report | 20260505 10:30:12.818 | INFO | RTMetrics report populated | 20260505 10:30:12.818 | INFO | Opening tmp RTMetrics json file | 20260505 10:30:12.839 | INFO | Serializing RTMetrics to json | 20260505 10:30:12.840 | INFO | Serialization done, swapping tmp file now... | 20260505 10:30:12.841 | INFO | RTMetrics done | 20260505 10:30:13.270 | INFO | Finished a round of Baz IO: Wrote 2.411 GiB in 3.714 seconds | 20260505 10:30:13.270 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.94% / 0.01% / 75.05% | 20260505 10:30:13.270 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.47645% / 1.19209e-05% / 7.94729e-05% | 20260505 10:30:13.270 | INFO | Baz IO overall: | 20260505 10:30:13.270 | INFO | BytesWritten : 1467096913355 | 20260505 10:30:13.274 | INFO | NumEvents : 698509194169 | 20260505 10:30:14.566 | INFO | Analyzing chunk frames = [2502656,2503168) | 20260505 10:30:14.566 | INFO | MemInfo: { | 20260505 10:30:14.566 | INFO | "gpu" : " 17 GB", | 20260505 10:30:14.566 | INFO | "huge" : "", | 20260505 10:30:14.566 | INFO | "huge_shared" : " 27 GB", | 20260505 10:30:14.566 | INFO | "malloc" : " 14 GB", | 20260505 10:30:14.566 | INFO | "pinned" : " 29 GB" | 20260505 10:30:14.567 | INFO | } | 20260505 10:30:14.567 | INFO | Perf report for the last 5.14954 seconds: | 20260505 10:30:14.567 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:30:14.567 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:14.567 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:14.567 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:14.567 | INFO | ANALYSIS : 64.9%, 2.9, 99.4ms, 0 | 20260505 10:30:14.567 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:14.567 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260505 10:30:14.567 | INFO | PRE_HQ : 61.4%, 1.8, 58.7ms, 0 | 20260505 10:30:14.567 | INFO | BAZWRITER : 71.5%, 4.3, 158.4ms, 0 | 20260505 10:30:14.567 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 10:30:14.567 | INFO | Uploaded 36GiB to GPU in 2.24454s (16.039GiB/s) | 20260505 10:30:14.567 | INFO | Downloaded 38GiB to GPU in 2.30495s (16.4862GiB/s) | 20260505 10:30:19.692 | INFO | Analyzing chunk frames = [2503168,2503680) | 20260505 10:30:19.692 | INFO | MemInfo: { | 20260505 10:30:19.692 | INFO | "gpu" : " 17 GB", | 20260505 10:30:19.692 | INFO | "huge" : "", | 20260505 10:30:19.692 | INFO | "huge_shared" : " 27 GB", | 20260505 10:30:19.692 | INFO | "malloc" : " 14 GB", | 20260505 10:30:19.692 | INFO | "pinned" : " 29 GB" | 20260505 10:30:19.692 | INFO | } | 20260505 10:30:19.692 | INFO | Perf report for the last 5.12518 seconds: | 20260505 10:30:19.692 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:30:19.692 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:19.692 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:19.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:19.692 | INFO | ANALYSIS : 64.4%, 3.0, 100.3ms, 0 | 20260505 10:30:19.692 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:19.692 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:19.692 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 10:30:19.692 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:19.692 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:19.692 | INFO | Uploaded 36GiB to GPU in 2.23149s (16.1327GiB/s) | 20260505 10:30:19.692 | INFO | Downloaded 34GiB to GPU in 2.07664s (16.3726GiB/s) | 20260505 10:30:24.826 | INFO | Analyzing chunk frames = [2503680,2504192) | 20260505 10:30:24.826 | INFO | MemInfo: { | 20260505 10:30:24.826 | INFO | "gpu" : " 17 GB", | 20260505 10:30:24.826 | INFO | "huge" : "", | 20260505 10:30:24.826 | INFO | "huge_shared" : " 27 GB", | 20260505 10:30:24.826 | INFO | "malloc" : " 14 GB", | 20260505 10:30:24.826 | INFO | "pinned" : " 29 GB" | 20260505 10:30:24.826 | INFO | } | 20260505 10:30:24.826 | INFO | Perf report for the last 5.13387 seconds: | 20260505 10:30:24.826 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:30:24.826 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:24.826 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:24.826 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:24.826 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 10:30:24.826 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:24.826 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:24.826 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 10:30:24.826 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:24.826 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:24.826 | INFO | Uploaded 36GiB to GPU in 2.15022s (16.7424GiB/s) | 20260505 10:30:24.826 | INFO | Downloaded 34GiB to GPU in 2.07035s (16.4224GiB/s) | 20260505 10:30:29.979 | INFO | Analyzing chunk frames = [2504192,2504704) | 20260505 10:30:29.979 | INFO | MemInfo: { | 20260505 10:30:29.979 | INFO | "gpu" : " 17 GB", | 20260505 10:30:29.979 | INFO | "huge" : "", | 20260505 10:30:29.979 | INFO | "huge_shared" : " 27 GB", | 20260505 10:30:29.979 | INFO | "malloc" : " 14 GB", | 20260505 10:30:29.979 | INFO | "pinned" : " 29 GB" | 20260505 10:30:29.979 | INFO | } | 20260505 10:30:29.979 | INFO | Perf report for the last 5.15368 seconds: | 20260505 10:30:29.979 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:30:29.979 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:29.979 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:29.979 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:29.979 | INFO | ANALYSIS : 62.9%, 3.0, 97.9ms, 0 | 20260505 10:30:29.979 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:29.979 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:29.979 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 10:30:29.979 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:29.979 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:29.980 | INFO | Uploaded 36GiB to GPU in 2.16334s (16.6409GiB/s) | 20260505 10:30:29.980 | INFO | Downloaded 34GiB to GPU in 2.079s (16.3541GiB/s) | 20260505 10:30:35.111 | INFO | Analyzing chunk frames = [2504704,2505216) | 20260505 10:30:35.111 | INFO | MemInfo: { | 20260505 10:30:35.111 | INFO | "gpu" : " 17 GB", | 20260505 10:30:35.111 | INFO | "huge" : "", | 20260505 10:30:35.111 | INFO | "huge_shared" : " 27 GB", | 20260505 10:30:35.111 | INFO | "malloc" : " 14 GB", | 20260505 10:30:35.111 | INFO | "pinned" : " 29 GB" | 20260505 10:30:35.111 | INFO | } | 20260505 10:30:35.112 | INFO | Perf report for the last 5.13222 seconds: | 20260505 10:30:35.112 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:30:35.112 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:35.112 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:35.112 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:35.112 | INFO | ANALYSIS : 63.4%, 3.0, 98.8ms, 0 | 20260505 10:30:35.112 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:35.112 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:35.112 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 10:30:35.112 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:35.112 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:35.112 | INFO | Uploaded 36GiB to GPU in 2.15747s (16.6862GiB/s) | 20260505 10:30:35.112 | INFO | Downloaded 34GiB to GPU in 2.12663s (15.9878GiB/s) | 20260505 10:30:40.248 | INFO | Analyzing chunk frames = [2505216,2505728) | 20260505 10:30:40.248 | INFO | MemInfo: { | 20260505 10:30:40.248 | INFO | "gpu" : " 17 GB", | 20260505 10:30:40.248 | INFO | "huge" : "", | 20260505 10:30:40.248 | INFO | "huge_shared" : " 27 GB", | 20260505 10:30:40.248 | INFO | "malloc" : " 14 GB", | 20260505 10:30:40.248 | INFO | "pinned" : " 29 GB" | 20260505 10:30:40.248 | INFO | } | 20260505 10:30:40.248 | INFO | Perf report for the last 5.13651 seconds: | 20260505 10:30:40.248 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:30:40.248 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:40.248 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:40.248 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:40.248 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 10:30:40.248 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:40.248 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:40.248 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:30:40.248 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:40.248 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:40.248 | INFO | Uploaded 36GiB to GPU in 2.23585s (16.1013GiB/s) | 20260505 10:30:40.249 | INFO | Downloaded 34GiB to GPU in 2.10271s (16.1696GiB/s) | 20260505 10:30:45.386 | INFO | Analyzing chunk frames = [2505728,2506240) | 20260505 10:30:45.386 | INFO | MemInfo: { | 20260505 10:30:45.386 | INFO | "gpu" : " 17 GB", | 20260505 10:30:45.386 | INFO | "huge" : "", | 20260505 10:30:45.386 | INFO | "huge_shared" : " 27 GB", | 20260505 10:30:45.386 | INFO | "malloc" : " 14 GB", | 20260505 10:30:45.386 | INFO | "pinned" : " 29 GB" | 20260505 10:30:45.386 | INFO | } | 20260505 10:30:45.386 | INFO | Perf report for the last 5.1381 seconds: | 20260505 10:30:45.386 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:30:45.386 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:45.386 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:45.386 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:45.386 | INFO | ANALYSIS : 63.8%, 3.0, 99.5ms, 0 | 20260505 10:30:45.386 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:45.386 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:45.386 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 10:30:45.386 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:45.386 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:45.386 | INFO | Uploaded 36GiB to GPU in 2.20904s (16.2967GiB/s) | 20260505 10:30:45.387 | INFO | Downloaded 34GiB to GPU in 2.14964s (15.8166GiB/s) | 20260505 10:30:50.534 | INFO | Analyzing chunk frames = [2506240,2506752) | 20260505 10:30:50.534 | INFO | MemInfo: { | 20260505 10:30:50.534 | INFO | "gpu" : " 17 GB", | 20260505 10:30:50.534 | INFO | "huge" : "", | 20260505 10:30:50.534 | INFO | "huge_shared" : " 27 GB", | 20260505 10:30:50.534 | INFO | "malloc" : " 14 GB", | 20260505 10:30:50.534 | INFO | "pinned" : " 29 GB" | 20260505 10:30:50.534 | INFO | } | 20260505 10:30:50.534 | INFO | Perf report for the last 5.14768 seconds: | 20260505 10:30:50.534 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:30:50.534 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:50.534 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:50.534 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:50.534 | INFO | ANALYSIS : 63.7%, 3.0, 99.6ms, 0 | 20260505 10:30:50.534 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:50.534 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:50.534 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 10:30:50.534 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:50.534 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:50.534 | INFO | Uploaded 36GiB to GPU in 2.18135s (16.5035GiB/s) | 20260505 10:30:50.534 | INFO | Downloaded 34GiB to GPU in 2.0577s (16.5233GiB/s) | 20260505 10:30:50.534 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 10:30:50.534 | INFO | Total mem usage: 15.3 GiB | 20260505 10:30:50.534 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 10:30:50.534 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -0.0657%) | 20260505 10:30:50.534 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 10:30:50.534 | INFO | High Water Breakdown: | 20260505 10:30:50.534 | INFO | 61.9% : 9.49 GiB ( 100% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 10:30:50.534 | INFO | 37.9% : 5.81 GiB ( 87.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 10:30:50.534 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 10:30:50.534 | INFO | ------------------------------------------ | 20260505 10:30:50.534 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:30:50.534 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 10:30:50.534 | INFO | Total mem usage: 26.5 GiB | 20260505 10:30:50.534 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 10:30:50.534 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 10:30:50.534 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 10:30:50.535 | INFO | High Water Breakdown: | 20260505 10:30:50.535 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:30:50.535 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:30:50.535 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:30:50.535 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:30:50.535 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:30:50.535 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:30:50.535 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 10:30:50.535 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 10:30:50.535 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:30:50.535 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 10:30:50.535 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 10:30:50.535 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 10:30:50.535 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 10:30:50.535 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 10:30:50.535 | INFO | ------------------------------------------ | 20260505 10:30:50.535 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:30:50.535 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 10:30:50.535 | INFO | Total mem usage: 25.6 GiB | 20260505 10:30:50.535 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 10:30:50.535 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 10:30:50.535 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 10:30:50.535 | INFO | High Water Breakdown: | 20260505 10:30:50.535 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 10:30:50.535 | INFO | ------------------------------------------ | 20260505 10:30:50.535 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:30:50.535 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 10:30:50.535 | INFO | Total mem usage: 15.7 GiB | 20260505 10:30:50.535 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 10:30:50.535 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 10:30:50.535 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 10:30:50.535 | INFO | High Water Breakdown: | 20260505 10:30:50.535 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 10:30:50.535 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 10:30:50.535 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 10:30:50.535 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 10:30:50.535 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 10:30:50.535 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 10:30:50.535 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 10:30:50.535 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 10:30:50.535 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 10:30:50.535 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:30:50.535 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 10:30:50.535 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:30:50.535 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 10:30:50.535 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 10:30:50.535 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 10:30:50.535 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 10:30:50.535 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:30:50.535 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 10:30:50.535 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:30:50.535 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:30:50.535 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:30:50.535 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 10:30:50.535 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 10:30:50.535 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:30:50.535 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 10:30:50.535 | INFO | ------------------------------------------ | 20260505 10:30:50.535 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:30:50.535 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 10:30:50.535 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:30:50.535 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:30:50.535 | INFO | PerfReport: Full Execution -- 3.23e+01 / 7.68e+01, 1.38e+01 / 1.70e+00 | 20260505 10:30:50.535 | INFO | PerfReport: 0.29%, AggregateStats -- 9.33e-02 / 1.02e-01, 1.11e-02 / 4.59e-03 | 20260505 10:30:50.535 | INFO | PerfReport: 11.68%, Baseline -- 3.77e+00 / 4.43e+00, 6.05e-01 / 2.56e-01 | 20260505 10:30:50.535 | INFO | PerfReport: 3.31%, Histogram -- 1.07e+00 / 1.49e+00, 2.05e-01 / 9.69e-02 | 20260505 10:30:50.535 | INFO | PerfReport: 15.71%, DME -- 4.06e+01 / 4.50e+01, 4.40e+00 / 2.19e+00 | 20260505 10:30:50.535 | INFO | PerfReport: 56.51%, FrameLabeling -- 1.83e+01 / 2.08e+01, 1.77e+00 / 1.97e+00 | 20260505 10:30:50.535 | INFO | PerfReport: 7.42%, PulseAccumulating -- 2.40e+00 / 2.92e+00, 4.21e-01 / 2.26e-01 | 20260505 10:30:50.535 | INFO | PerfReport: 3.76%, Metrics -- 1.21e+00 / 1.54e+00, 3.50e-01 / 2.02e-01 | 20260505 10:30:50.535 | INFO | PerfReport: 1.33%, PulseSerializing -- 4.29e-01 / 4.81e-01, 7.87e-02 / 2.83e-02 | 20260505 10:30:50.535 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 10:30:50.535 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:30:50.535 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:30:50.535 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.40e+01 / 0.00e+00 | 20260505 10:30:50.535 | INFO | PerfReport: 22.63%, Upload -- 2.29e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260505 10:30:50.535 | INFO | PerfReport: 21.88%, Download -- 2.22e+01 / 0.00e+00, 8.18e+00 / 0.00e+00 | 20260505 10:30:50.535 | INFO | PerfReport: 31.90%, Compute -- 3.23e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 10:30:50.535 | INFO | PerfReport: 1.93%, LockUpload -- 1.96e+00 / 0.00e+00, 7.45e+00 / 0.00e+00 | 20260505 10:30:50.536 | INFO | PerfReport: 1.13%, LockDownload -- 1.14e+00 / 0.00e+00, 2.66e+00 / 0.00e+00 | 20260505 10:30:50.536 | INFO | PerfReport: 20.52%, LockCompute -- 2.08e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260505 10:30:50.536 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.72e-02 / 0.00e+00 | 20260505 10:30:50.671 | INFO | Starting a round of Baz IO | 20260505 10:30:53.926 | INFO | Populating RTMetrics report | 20260505 10:30:53.926 | INFO | RTMetrics report populated | 20260505 10:30:53.926 | INFO | Opening tmp RTMetrics json file | 20260505 10:30:53.933 | INFO | Serializing RTMetrics to json | 20260505 10:30:53.937 | INFO | Serialization done, swapping tmp file now... | 20260505 10:30:53.939 | INFO | RTMetrics done | 20260505 10:30:54.114 | INFO | Finished a round of Baz IO: Wrote 2.411 GiB in 3.443 seconds | 20260505 10:30:54.114 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.93% / 0.01% / 75.06% | 20260505 10:30:54.114 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.47016% / 1.58946e-05% / 9.13938e-05% | 20260505 10:30:54.114 | INFO | Baz IO overall: | 20260505 10:30:54.114 | INFO | BytesWritten : 1469685207372 | 20260505 10:30:54.114 | INFO | NumEvents : 699743157453 | 20260505 10:30:55.674 | INFO | Analyzing chunk frames = [2506752,2507264) | 20260505 10:30:55.674 | INFO | MemInfo: { | 20260505 10:30:55.674 | INFO | "gpu" : " 17 GB", | 20260505 10:30:55.674 | INFO | "huge" : "", | 20260505 10:30:55.674 | INFO | "huge_shared" : " 27 GB", | 20260505 10:30:55.674 | INFO | "malloc" : " 14 GB", | 20260505 10:30:55.674 | INFO | "pinned" : " 29 GB" | 20260505 10:30:55.674 | INFO | } | 20260505 10:30:55.674 | INFO | Perf report for the last 5.14061 seconds: | 20260505 10:30:55.675 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:30:55.675 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:55.675 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:55.675 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:55.675 | INFO | ANALYSIS : 64.8%, 2.9, 98.9ms, 0 | 20260505 10:30:55.675 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:30:55.675 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 10:30:55.675 | INFO | PRE_HQ : 61.4%, 1.9, 59.6ms, 0 | 20260505 10:30:55.675 | INFO | BAZWRITER : 66.3%, 4.3, 144.9ms, 0 | 20260505 10:30:55.675 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 10:30:55.675 | INFO | Uploaded 36GiB to GPU in 2.20514s (16.3255GiB/s) | 20260505 10:30:55.675 | INFO | Downloaded 38GiB to GPU in 2.30116s (16.5134GiB/s) | 20260505 10:31:00.809 | INFO | Analyzing chunk frames = [2507264,2507776) | 20260505 10:31:00.809 | INFO | MemInfo: { | 20260505 10:31:00.809 | INFO | "gpu" : " 17 GB", | 20260505 10:31:00.809 | INFO | "huge" : "", | 20260505 10:31:00.809 | INFO | "huge_shared" : " 27 GB", | 20260505 10:31:00.809 | INFO | "malloc" : " 14 GB", | 20260505 10:31:00.809 | INFO | "pinned" : " 29 GB" | 20260505 10:31:00.809 | INFO | } | 20260505 10:31:00.809 | INFO | Perf report for the last 5.1349 seconds: | 20260505 10:31:00.809 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:31:00.809 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:00.809 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:00.809 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:00.809 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 10:31:00.809 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:00.809 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:00.809 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260505 10:31:00.809 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:00.809 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:00.810 | INFO | Uploaded 36GiB to GPU in 2.23397s (16.1148GiB/s) | 20260505 10:31:00.810 | INFO | Downloaded 34GiB to GPU in 2.08398s (16.3149GiB/s) | 20260505 10:31:05.943 | INFO | Analyzing chunk frames = [2507776,2508288) | 20260505 10:31:05.943 | INFO | MemInfo: { | 20260505 10:31:05.943 | INFO | "gpu" : " 17 GB", | 20260505 10:31:05.943 | INFO | "huge" : "", | 20260505 10:31:05.943 | INFO | "huge_shared" : " 27 GB", | 20260505 10:31:05.943 | INFO | "malloc" : " 14 GB", | 20260505 10:31:05.943 | INFO | "pinned" : " 29 GB" | 20260505 10:31:05.943 | INFO | } | 20260505 10:31:05.944 | INFO | Perf report for the last 5.13412 seconds: | 20260505 10:31:05.944 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:31:05.944 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:05.944 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:05.944 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:05.944 | INFO | ANALYSIS : 63.1%, 3.0, 98.1ms, 0 | 20260505 10:31:05.944 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:05.944 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:05.944 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 10:31:05.944 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:05.944 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:05.944 | INFO | Uploaded 36GiB to GPU in 2.15839s (16.6791GiB/s) | 20260505 10:31:05.944 | INFO | Downloaded 34GiB to GPU in 2.06492s (16.4655GiB/s) | 20260505 10:31:11.079 | INFO | Analyzing chunk frames = [2508288,2508800) | 20260505 10:31:11.079 | INFO | MemInfo: { | 20260505 10:31:11.079 | INFO | "gpu" : " 17 GB", | 20260505 10:31:11.079 | INFO | "huge" : "", | 20260505 10:31:11.079 | INFO | "huge_shared" : " 27 GB", | 20260505 10:31:11.079 | INFO | "malloc" : " 14 GB", | 20260505 10:31:11.079 | INFO | "pinned" : " 29 GB" | 20260505 10:31:11.079 | INFO | } | 20260505 10:31:11.080 | INFO | Perf report for the last 5.13603 seconds: | 20260505 10:31:11.080 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:31:11.080 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:11.080 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:11.080 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:11.080 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 10:31:11.080 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:11.080 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:11.080 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:31:11.080 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:11.080 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:11.080 | INFO | Uploaded 36GiB to GPU in 2.16607s (16.62GiB/s) | 20260505 10:31:11.080 | INFO | Downloaded 34GiB to GPU in 2.06673s (16.4511GiB/s) | 20260505 10:31:16.232 | INFO | Analyzing chunk frames = [2508800,2509312) | 20260505 10:31:16.232 | INFO | MemInfo: { | 20260505 10:31:16.232 | INFO | "gpu" : " 17 GB", | 20260505 10:31:16.232 | INFO | "huge" : "", | 20260505 10:31:16.233 | INFO | "huge_shared" : " 27 GB", | 20260505 10:31:16.233 | INFO | "malloc" : " 14 GB", | 20260505 10:31:16.233 | INFO | "pinned" : " 29 GB" | 20260505 10:31:16.233 | INFO | } | 20260505 10:31:16.233 | INFO | Perf report for the last 5.15302 seconds: | 20260505 10:31:16.233 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:31:16.233 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:16.233 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:16.233 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:16.233 | INFO | ANALYSIS : 63.5%, 3.0, 99.4ms, 0 | 20260505 10:31:16.233 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:16.233 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:16.233 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 10:31:16.233 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:16.233 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:16.233 | INFO | Uploaded 36GiB to GPU in 2.23696s (16.0933GiB/s) | 20260505 10:31:16.233 | INFO | Downloaded 34GiB to GPU in 2.10578s (16.1461GiB/s) | 20260505 10:31:21.377 | INFO | Analyzing chunk frames = [2509312,2509824) | 20260505 10:31:21.377 | INFO | MemInfo: { | 20260505 10:31:21.377 | INFO | "gpu" : " 17 GB", | 20260505 10:31:21.377 | INFO | "huge" : "", | 20260505 10:31:21.377 | INFO | "huge_shared" : " 27 GB", | 20260505 10:31:21.377 | INFO | "malloc" : " 14 GB", | 20260505 10:31:21.377 | INFO | "pinned" : " 29 GB" | 20260505 10:31:21.377 | INFO | } | 20260505 10:31:21.377 | INFO | Perf report for the last 5.14448 seconds: | 20260505 10:31:21.377 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:31:21.377 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:21.377 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:21.377 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:21.377 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 10:31:21.377 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:21.377 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:21.377 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:31:21.377 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:21.377 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:21.377 | INFO | Uploaded 36GiB to GPU in 2.18961s (16.4413GiB/s) | 20260505 10:31:21.377 | INFO | Downloaded 34GiB to GPU in 2.10115s (16.1816GiB/s) | 20260505 10:31:26.514 | INFO | Analyzing chunk frames = [2509824,2510336) | 20260505 10:31:26.514 | INFO | MemInfo: { | 20260505 10:31:26.514 | INFO | "gpu" : " 17 GB", | 20260505 10:31:26.514 | INFO | "huge" : "", | 20260505 10:31:26.514 | INFO | "huge_shared" : " 27 GB", | 20260505 10:31:26.514 | INFO | "malloc" : " 14 GB", | 20260505 10:31:26.514 | INFO | "pinned" : " 29 GB" | 20260505 10:31:26.514 | INFO | } | 20260505 10:31:26.514 | INFO | Perf report for the last 5.13698 seconds: | 20260505 10:31:26.514 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:31:26.514 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:26.514 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:26.514 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:26.514 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 10:31:26.514 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:26.514 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:26.514 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 10:31:26.514 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:26.514 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:26.514 | INFO | Uploaded 36GiB to GPU in 2.17531s (16.5494GiB/s) | 20260505 10:31:26.514 | INFO | Downloaded 34GiB to GPU in 2.13894s (15.8957GiB/s) | 20260505 10:31:31.640 | INFO | Analyzing chunk frames = [2510336,2510848) | 20260505 10:31:31.640 | INFO | MemInfo: { | 20260505 10:31:31.640 | INFO | "gpu" : " 17 GB", | 20260505 10:31:31.640 | INFO | "huge" : "", | 20260505 10:31:31.640 | INFO | "huge_shared" : " 27 GB", | 20260505 10:31:31.640 | INFO | "malloc" : " 14 GB", | 20260505 10:31:31.640 | INFO | "pinned" : " 29 GB" | 20260505 10:31:31.640 | INFO | } | 20260505 10:31:31.640 | INFO | Perf report for the last 5.12598 seconds: | 20260505 10:31:31.640 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:31:31.640 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:31.640 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:31.640 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:31.640 | INFO | ANALYSIS : 64.3%, 3.0, 100.1ms, 0 | 20260505 10:31:31.640 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:31.640 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:31.640 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 10:31:31.640 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:31.640 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:31.640 | INFO | Uploaded 36GiB to GPU in 2.21728s (16.2361GiB/s) | 20260505 10:31:31.640 | INFO | Downloaded 34GiB to GPU in 2.09035s (16.2652GiB/s) | 20260505 10:31:31.770 | INFO | Starting a round of Baz IO | 20260505 10:31:35.027 | INFO | Populating RTMetrics report | 20260505 10:31:35.027 | INFO | RTMetrics report populated | 20260505 10:31:35.027 | INFO | Opening tmp RTMetrics json file | 20260505 10:31:35.039 | INFO | Serializing RTMetrics to json | 20260505 10:31:35.040 | INFO | Serialization done, swapping tmp file now... | 20260505 10:31:35.040 | INFO | RTMetrics done | 20260505 10:31:35.208 | INFO | Finished a round of Baz IO: Wrote 2.411 GiB in 3.437 seconds | 20260505 10:31:35.208 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.92% / 0.01% / 75.07% | 20260505 10:31:35.208 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.4674% / 3.97364e-06% / 9.93411e-05% | 20260505 10:31:35.208 | INFO | Baz IO overall: | 20260505 10:31:35.208 | INFO | BytesWritten : 1472273509242 | 20260505 10:31:35.208 | INFO | NumEvents : 700977520006 | 20260505 10:31:36.781 | INFO | Analyzing chunk frames = [2510848,2511360) | 20260505 10:31:36.781 | INFO | MemInfo: { | 20260505 10:31:36.781 | INFO | "gpu" : " 17 GB", | 20260505 10:31:36.781 | INFO | "huge" : "", | 20260505 10:31:36.781 | INFO | "huge_shared" : " 27 GB", | 20260505 10:31:36.781 | INFO | "malloc" : " 14 GB", | 20260505 10:31:36.781 | INFO | "pinned" : " 29 GB" | 20260505 10:31:36.781 | INFO | } | 20260505 10:31:36.781 | INFO | Perf report for the last 5.14093 seconds: | 20260505 10:31:36.781 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:31:36.781 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:36.781 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:36.781 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:36.781 | INFO | ANALYSIS : 64.7%, 2.9, 98.9ms, 0 | 20260505 10:31:36.781 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:36.781 | INFO | RT_METRICS : 46.2%, 1.0, 24.3ms, 0 | 20260505 10:31:36.781 | INFO | PRE_HQ : 62.7%, 1.8, 59.4ms, 0 | 20260505 10:31:36.781 | INFO | BAZWRITER : 66.2%, 4.2, 143.3ms, 0 | 20260505 10:31:36.781 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 10:31:36.782 | INFO | Uploaded 36GiB to GPU in 2.253s (15.9787GiB/s) | 20260505 10:31:36.782 | INFO | Downloaded 38GiB to GPU in 2.37558s (15.9961GiB/s) | 20260505 10:31:41.926 | INFO | Analyzing chunk frames = [2511360,2511872) | 20260505 10:31:41.926 | INFO | MemInfo: { | 20260505 10:31:41.926 | INFO | "gpu" : " 17 GB", | 20260505 10:31:41.926 | INFO | "huge" : "", | 20260505 10:31:41.926 | INFO | "huge_shared" : " 27 GB", | 20260505 10:31:41.926 | INFO | "malloc" : " 14 GB", | 20260505 10:31:41.926 | INFO | "pinned" : " 29 GB" | 20260505 10:31:41.926 | INFO | } | 20260505 10:31:41.926 | INFO | Perf report for the last 5.14536 seconds: | 20260505 10:31:41.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:31:41.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:41.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:41.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:41.926 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 10:31:41.927 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:41.927 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:41.927 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 10:31:41.927 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:41.927 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:41.927 | INFO | Uploaded 36GiB to GPU in 2.19177s (16.4251GiB/s) | 20260505 10:31:41.927 | INFO | Downloaded 34GiB to GPU in 2.09725s (16.2117GiB/s) | 20260505 10:31:47.064 | INFO | Analyzing chunk frames = [2511872,2512384) | 20260505 10:31:47.064 | INFO | MemInfo: { | 20260505 10:31:47.064 | INFO | "gpu" : " 17 GB", | 20260505 10:31:47.064 | INFO | "huge" : "", | 20260505 10:31:47.064 | INFO | "huge_shared" : " 27 GB", | 20260505 10:31:47.064 | INFO | "malloc" : " 14 GB", | 20260505 10:31:47.064 | INFO | "pinned" : " 29 GB" | 20260505 10:31:47.064 | INFO | } | 20260505 10:31:47.064 | INFO | Perf report for the last 5.13787 seconds: | 20260505 10:31:47.064 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:31:47.064 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:47.064 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:47.064 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:47.064 | INFO | ANALYSIS : 62.9%, 3.0, 97.8ms, 0 | 20260505 10:31:47.064 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:47.064 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:47.064 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 10:31:47.064 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:47.064 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:47.065 | INFO | Uploaded 36GiB to GPU in 2.15574s (16.6996GiB/s) | 20260505 10:31:47.065 | INFO | Downloaded 34GiB to GPU in 2.06362s (16.4759GiB/s) | 20260505 10:31:52.199 | INFO | Analyzing chunk frames = [2512384,2512896) | 20260505 10:31:52.199 | INFO | MemInfo: { | 20260505 10:31:52.199 | INFO | "gpu" : " 17 GB", | 20260505 10:31:52.199 | INFO | "huge" : "", | 20260505 10:31:52.199 | INFO | "huge_shared" : " 27 GB", | 20260505 10:31:52.199 | INFO | "malloc" : " 14 GB", | 20260505 10:31:52.199 | INFO | "pinned" : " 29 GB" | 20260505 10:31:52.199 | INFO | } | 20260505 10:31:52.199 | INFO | Perf report for the last 5.13466 seconds: | 20260505 10:31:52.199 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:31:52.199 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:52.199 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:52.199 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:52.199 | INFO | ANALYSIS : 63.1%, 3.0, 97.9ms, 0 | 20260505 10:31:52.199 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:52.199 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:52.199 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 10:31:52.199 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:52.199 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:52.199 | INFO | Uploaded 36GiB to GPU in 2.17026s (16.5879GiB/s) | 20260505 10:31:52.199 | INFO | Downloaded 34GiB to GPU in 2.07555s (16.3812GiB/s) | 20260505 10:31:57.335 | INFO | Analyzing chunk frames = [2512896,2513408) | 20260505 10:31:57.335 | INFO | MemInfo: { | 20260505 10:31:57.335 | INFO | "gpu" : " 17 GB", | 20260505 10:31:57.335 | INFO | "huge" : "", | 20260505 10:31:57.335 | INFO | "huge_shared" : " 27 GB", | 20260505 10:31:57.335 | INFO | "malloc" : " 14 GB", | 20260505 10:31:57.335 | INFO | "pinned" : " 29 GB" | 20260505 10:31:57.335 | INFO | } | 20260505 10:31:57.335 | INFO | Perf report for the last 5.1365 seconds: | 20260505 10:31:57.335 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:31:57.335 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:57.335 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:57.335 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:57.335 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 10:31:57.336 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:57.336 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:57.336 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260505 10:31:57.336 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:57.336 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:31:57.336 | INFO | Uploaded 36GiB to GPU in 2.23335s (16.1193GiB/s) | 20260505 10:31:57.336 | INFO | Downloaded 34GiB to GPU in 2.10983s (16.115GiB/s) | 20260505 10:32:02.488 | INFO | Analyzing chunk frames = [2513408,2513920) | 20260505 10:32:02.488 | INFO | MemInfo: { | 20260505 10:32:02.488 | INFO | "gpu" : " 17 GB", | 20260505 10:32:02.488 | INFO | "huge" : "", | 20260505 10:32:02.488 | INFO | "huge_shared" : " 27 GB", | 20260505 10:32:02.488 | INFO | "malloc" : " 14 GB", | 20260505 10:32:02.488 | INFO | "pinned" : " 29 GB" | 20260505 10:32:02.488 | INFO | } | 20260505 10:32:02.488 | INFO | Perf report for the last 5.15244 seconds: | 20260505 10:32:02.488 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:32:02.488 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:02.488 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:02.488 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:02.488 | INFO | ANALYSIS : 63.8%, 3.0, 99.8ms, 0 | 20260505 10:32:02.488 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:02.488 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:02.488 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:32:02.488 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:02.488 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:02.488 | INFO | Uploaded 36GiB to GPU in 2.23486s (16.1084GiB/s) | 20260505 10:32:02.488 | INFO | Downloaded 34GiB to GPU in 2.10509s (16.1513GiB/s) | 20260505 10:32:07.622 | INFO | Analyzing chunk frames = [2513920,2514432) | 20260505 10:32:07.622 | INFO | MemInfo: { | 20260505 10:32:07.622 | INFO | "gpu" : " 17 GB", | 20260505 10:32:07.622 | INFO | "huge" : "", | 20260505 10:32:07.622 | INFO | "huge_shared" : " 27 GB", | 20260505 10:32:07.622 | INFO | "malloc" : " 14 GB", | 20260505 10:32:07.622 | INFO | "pinned" : " 29 GB" | 20260505 10:32:07.622 | INFO | } | 20260505 10:32:07.622 | INFO | Perf report for the last 5.13428 seconds: | 20260505 10:32:07.622 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:32:07.622 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:07.622 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:07.622 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:07.622 | INFO | ANALYSIS : 63.9%, 3.0, 99.6ms, 0 | 20260505 10:32:07.622 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:07.622 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:07.622 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 10:32:07.622 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:07.622 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:07.623 | INFO | Uploaded 36GiB to GPU in 2.18494s (16.4764GiB/s) | 20260505 10:32:07.623 | INFO | Downloaded 34GiB to GPU in 2.13129s (15.9528GiB/s) | 20260505 10:32:12.756 | INFO | Analyzing chunk frames = [2514432,2514944) | 20260505 10:32:12.756 | INFO | MemInfo: { | 20260505 10:32:12.756 | INFO | "gpu" : " 17 GB", | 20260505 10:32:12.756 | INFO | "huge" : "", | 20260505 10:32:12.756 | INFO | "huge_shared" : " 27 GB", | 20260505 10:32:12.756 | INFO | "malloc" : " 14 GB", | 20260505 10:32:12.756 | INFO | "pinned" : " 29 GB" | 20260505 10:32:12.756 | INFO | } | 20260505 10:32:12.756 | INFO | Perf report for the last 5.13377 seconds: | 20260505 10:32:12.756 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:32:12.756 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:12.756 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:12.756 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:12.756 | INFO | ANALYSIS : 64.5%, 3.0, 100.7ms, 0 | 20260505 10:32:12.756 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:12.756 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:12.756 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 10:32:12.756 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:12.756 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:12.756 | INFO | Uploaded 36GiB to GPU in 2.24442s (16.0398GiB/s) | 20260505 10:32:12.756 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 10:32:12.756 | INFO | Total mem usage: 15.3 GiB | 20260505 10:32:12.756 | INFO | Downloaded 34GiB to GPU in 2.09503s (16.2289GiB/s) | 20260505 10:32:12.756 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 10:32:12.756 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -5.92%) | 20260505 10:32:12.756 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 10:32:12.756 | INFO | High Water Breakdown: | 20260505 10:32:12.756 | INFO | 61.1% : 8.84 GiB ( 93.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 10:32:12.756 | INFO | 38.7% : 5.61 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 10:32:12.756 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 10:32:12.756 | INFO | ------------------------------------------ | 20260505 10:32:12.756 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:32:12.756 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 10:32:12.756 | INFO | Total mem usage: 26.5 GiB | 20260505 10:32:12.756 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 10:32:12.756 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 10:32:12.756 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 10:32:12.756 | INFO | High Water Breakdown: | 20260505 10:32:12.756 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:32:12.756 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:32:12.756 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:32:12.756 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:32:12.756 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:32:12.756 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:32:12.757 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 10:32:12.757 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 10:32:12.757 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:32:12.757 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 10:32:12.757 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 10:32:12.757 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 10:32:12.757 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 10:32:12.757 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 10:32:12.757 | INFO | ------------------------------------------ | 20260505 10:32:12.757 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:32:12.757 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 10:32:12.757 | INFO | Total mem usage: 25.6 GiB | 20260505 10:32:12.757 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 10:32:12.757 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 10:32:12.757 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 10:32:12.757 | INFO | High Water Breakdown: | 20260505 10:32:12.757 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 10:32:12.757 | INFO | ------------------------------------------ | 20260505 10:32:12.757 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:32:12.757 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 10:32:12.757 | INFO | Total mem usage: 15.7 GiB | 20260505 10:32:12.757 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 10:32:12.757 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 10:32:12.757 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 10:32:12.757 | INFO | High Water Breakdown: | 20260505 10:32:12.757 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 10:32:12.757 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 10:32:12.757 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 10:32:12.757 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 10:32:12.757 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 10:32:12.757 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 10:32:12.757 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 10:32:12.757 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 10:32:12.757 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 10:32:12.757 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:32:12.757 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 10:32:12.757 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:32:12.757 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 10:32:12.757 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 10:32:12.757 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 10:32:12.757 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 10:32:12.757 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:32:12.757 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 10:32:12.757 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:32:12.757 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:32:12.757 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:32:12.757 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 10:32:12.757 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 10:32:12.757 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:32:12.757 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 10:32:12.757 | INFO | ------------------------------------------ | 20260505 10:32:12.757 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:32:12.757 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 10:32:12.757 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:32:12.757 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:32:12.757 | INFO | PerfReport: Full Execution -- 3.23e+01 / 7.67e+01, 1.39e+01 / 1.87e+00 | 20260505 10:32:12.757 | INFO | PerfReport: 0.29%, AggregateStats -- 9.31e-02 / 1.02e-01, 1.05e-02 / 3.41e-03 | 20260505 10:32:12.757 | INFO | PerfReport: 11.69%, Baseline -- 3.78e+00 / 4.47e+00, 6.03e-01 / 2.85e-01 | 20260505 10:32:12.757 | INFO | PerfReport: 3.32%, Histogram -- 1.07e+00 / 1.49e+00, 2.06e-01 / 1.07e-01 | 20260505 10:32:12.757 | INFO | PerfReport: 15.60%, DME -- 4.03e+01 / 4.42e+01, 4.27e+00 / 1.78e+00 | 20260505 10:32:12.757 | INFO | PerfReport: 56.67%, FrameLabeling -- 1.83e+01 / 2.16e+01, 1.82e+00 / 1.84e+00 | 20260505 10:32:12.757 | INFO | PerfReport: 7.35%, PulseAccumulating -- 2.38e+00 / 2.91e+00, 4.03e-01 / 1.96e-01 | 20260505 10:32:12.757 | INFO | PerfReport: 3.75%, Metrics -- 1.21e+00 / 1.48e+00, 3.46e-01 / 1.89e-01 | 20260505 10:32:12.757 | INFO | PerfReport: 1.33%, PulseSerializing -- 4.29e-01 / 4.66e-01, 7.79e-02 / 3.88e-02 | 20260505 10:32:12.757 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 10:32:12.757 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:32:12.757 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:32:12.757 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260505 10:32:12.757 | INFO | PerfReport: 22.65%, Upload -- 2.30e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260505 10:32:12.757 | INFO | PerfReport: 21.85%, Download -- 2.21e+01 / 0.00e+00, 8.12e+00 / 0.00e+00 | 20260505 10:32:12.757 | INFO | PerfReport: 31.91%, Compute -- 3.23e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 10:32:12.757 | INFO | PerfReport: 1.97%, LockUpload -- 2.00e+00 / 0.00e+00, 7.57e+00 / 0.00e+00 | 20260505 10:32:12.757 | INFO | PerfReport: 1.12%, LockDownload -- 1.13e+00 / 0.00e+00, 2.70e+00 / 0.00e+00 | 20260505 10:32:12.757 | INFO | PerfReport: 20.48%, LockCompute -- 2.08e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260505 10:32:12.757 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.32e-02 / 0.00e+00 | 20260505 10:32:12.887 | INFO | Starting a round of Baz IO | 20260505 10:32:16.149 | INFO | Populating RTMetrics report | 20260505 10:32:16.149 | INFO | RTMetrics report populated | 20260505 10:32:16.149 | INFO | Opening tmp RTMetrics json file | 20260505 10:32:16.186 | INFO | Serializing RTMetrics to json | 20260505 10:32:16.211 | INFO | Serialization done, swapping tmp file now... | 20260505 10:32:16.216 | INFO | RTMetrics done | 20260505 10:32:16.338 | INFO | Finished a round of Baz IO: Wrote 2.408 GiB in 3.451 seconds | 20260505 10:32:16.338 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.91% / 0.01% / 75.08% | 20260505 10:32:16.338 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.45962% / 1.58946e-05% / 9.13938e-05% | 20260505 10:32:16.338 | INFO | Baz IO overall: | 20260505 10:32:16.338 | INFO | BytesWritten : 1474859303155 | 20260505 10:32:16.339 | INFO | NumEvents : 702209896488 | 20260505 10:32:17.894 | INFO | Analyzing chunk frames = [2514944,2515456) | 20260505 10:32:17.894 | INFO | MemInfo: { | 20260505 10:32:17.894 | INFO | "gpu" : " 17 GB", | 20260505 10:32:17.894 | INFO | "huge" : "", | 20260505 10:32:17.894 | INFO | "huge_shared" : " 27 GB", | 20260505 10:32:17.894 | INFO | "malloc" : " 14 GB", | 20260505 10:32:17.894 | INFO | "pinned" : " 29 GB" | 20260505 10:32:17.894 | INFO | } | 20260505 10:32:17.894 | INFO | Perf report for the last 5.13783 seconds: | 20260505 10:32:17.894 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:32:17.894 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:17.894 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:17.894 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:17.894 | INFO | ANALYSIS : 64.9%, 2.9, 99.5ms, 0 | 20260505 10:32:17.894 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:17.894 | INFO | RT_METRICS : 47.0%, 1.0, 24.6ms, 0 | 20260505 10:32:17.894 | INFO | PRE_HQ : 62.8%, 1.8, 59.6ms, 0 | 20260505 10:32:17.894 | INFO | BAZWRITER : 66.5%, 4.2, 142.9ms, 0 | 20260505 10:32:17.894 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260505 10:32:17.894 | INFO | Uploaded 36GiB to GPU in 2.26395s (15.9014GiB/s) | 20260505 10:32:17.894 | INFO | Downloaded 38GiB to GPU in 2.37093s (16.0275GiB/s) | 20260505 10:32:23.033 | INFO | Analyzing chunk frames = [2515456,2515968) | 20260505 10:32:23.033 | INFO | MemInfo: { | 20260505 10:32:23.033 | INFO | "gpu" : " 17 GB", | 20260505 10:32:23.033 | INFO | "huge" : "", | 20260505 10:32:23.033 | INFO | "huge_shared" : " 27 GB", | 20260505 10:32:23.033 | INFO | "malloc" : " 14 GB", | 20260505 10:32:23.033 | INFO | "pinned" : " 29 GB" | 20260505 10:32:23.033 | INFO | } | 20260505 10:32:23.033 | INFO | Perf report for the last 5.13935 seconds: | 20260505 10:32:23.033 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:32:23.033 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:23.033 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:23.033 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:23.033 | INFO | ANALYSIS : 64.5%, 3.0, 100.7ms, 0 | 20260505 10:32:23.033 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:23.033 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:23.033 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 10:32:23.033 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:23.033 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:23.034 | INFO | Uploaded 36GiB to GPU in 2.23248s (16.1255GiB/s) | 20260505 10:32:23.034 | INFO | Downloaded 34GiB to GPU in 2.10984s (16.1149GiB/s) | 20260505 10:32:28.179 | INFO | Analyzing chunk frames = [2515968,2516480) | 20260505 10:32:28.179 | INFO | MemInfo: { | 20260505 10:32:28.179 | INFO | "gpu" : " 17 GB", | 20260505 10:32:28.179 | INFO | "huge" : "", | 20260505 10:32:28.179 | INFO | "huge_shared" : " 27 GB", | 20260505 10:32:28.179 | INFO | "malloc" : " 14 GB", | 20260505 10:32:28.179 | INFO | "pinned" : " 29 GB" | 20260505 10:32:28.179 | INFO | } | 20260505 10:32:28.179 | INFO | Perf report for the last 5.14564 seconds: | 20260505 10:32:28.179 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:32:28.179 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:28.179 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:28.179 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:28.179 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260505 10:32:28.179 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:28.179 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:28.179 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:32:28.179 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:28.179 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:28.179 | INFO | Uploaded 36GiB to GPU in 2.14182s (16.8082GiB/s) | 20260505 10:32:28.179 | INFO | Downloaded 34GiB to GPU in 2.05665s (16.5318GiB/s) | 20260505 10:32:33.322 | INFO | Analyzing chunk frames = [2516480,2516992) | 20260505 10:32:33.322 | INFO | MemInfo: { | 20260505 10:32:33.322 | INFO | "gpu" : " 17 GB", | 20260505 10:32:33.322 | INFO | "huge" : "", | 20260505 10:32:33.322 | INFO | "huge_shared" : " 27 GB", | 20260505 10:32:33.322 | INFO | "malloc" : " 14 GB", | 20260505 10:32:33.322 | INFO | "pinned" : " 29 GB" | 20260505 10:32:33.322 | INFO | } | 20260505 10:32:33.322 | INFO | Perf report for the last 5.14336 seconds: | 20260505 10:32:33.322 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:32:33.322 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:33.322 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:33.322 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:33.322 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260505 10:32:33.322 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:33.322 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:33.322 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:32:33.322 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:33.322 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:33.322 | INFO | Uploaded 36GiB to GPU in 2.14329s (16.7966GiB/s) | 20260505 10:32:33.323 | INFO | Downloaded 34GiB to GPU in 2.07318s (16.3999GiB/s) | 20260505 10:32:38.454 | INFO | Analyzing chunk frames = [2516992,2517504) | 20260505 10:32:38.454 | INFO | MemInfo: { | 20260505 10:32:38.454 | INFO | "gpu" : " 17 GB", | 20260505 10:32:38.454 | INFO | "huge" : "", | 20260505 10:32:38.454 | INFO | "huge_shared" : " 27 GB", | 20260505 10:32:38.454 | INFO | "malloc" : " 14 GB", | 20260505 10:32:38.454 | INFO | "pinned" : " 29 GB" | 20260505 10:32:38.454 | INFO | } | 20260505 10:32:38.454 | INFO | Perf report for the last 5.13162 seconds: | 20260505 10:32:38.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:32:38.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:38.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:38.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:38.454 | INFO | ANALYSIS : 63.5%, 3.0, 99.0ms, 0 | 20260505 10:32:38.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:38.454 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:38.454 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:32:38.454 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:38.454 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:38.454 | INFO | Uploaded 36GiB to GPU in 2.16586s (16.6216GiB/s) | 20260505 10:32:38.454 | INFO | Downloaded 34GiB to GPU in 2.12983s (15.9637GiB/s) | 20260505 10:32:43.592 | INFO | Analyzing chunk frames = [2517504,2518016) | 20260505 10:32:43.592 | INFO | MemInfo: { | 20260505 10:32:43.592 | INFO | "gpu" : " 17 GB", | 20260505 10:32:43.592 | INFO | "huge" : "", | 20260505 10:32:43.592 | INFO | "huge_shared" : " 27 GB", | 20260505 10:32:43.592 | INFO | "malloc" : " 14 GB", | 20260505 10:32:43.592 | INFO | "pinned" : " 29 GB" | 20260505 10:32:43.592 | INFO | } | 20260505 10:32:43.592 | INFO | Perf report for the last 5.1378 seconds: | 20260505 10:32:43.592 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:32:43.592 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:43.592 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:43.592 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:43.592 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 10:32:43.592 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:43.592 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:43.592 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 10:32:43.592 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:43.592 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:43.592 | INFO | Uploaded 36GiB to GPU in 2.16248s (16.6476GiB/s) | 20260505 10:32:43.592 | INFO | Downloaded 34GiB to GPU in 2.05683s (16.5303GiB/s) | 20260505 10:32:48.740 | INFO | Analyzing chunk frames = [2518016,2518528) | 20260505 10:32:48.740 | INFO | MemInfo: { | 20260505 10:32:48.740 | INFO | "gpu" : " 17 GB", | 20260505 10:32:48.740 | INFO | "huge" : "", | 20260505 10:32:48.740 | INFO | "huge_shared" : " 27 GB", | 20260505 10:32:48.740 | INFO | "malloc" : " 14 GB", | 20260505 10:32:48.740 | INFO | "pinned" : " 29 GB" | 20260505 10:32:48.740 | INFO | } | 20260505 10:32:48.741 | INFO | Perf report for the last 5.14886 seconds: | 20260505 10:32:48.741 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:32:48.741 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:48.741 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:48.741 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:48.741 | INFO | ANALYSIS : 63.5%, 3.0, 99.3ms, 0 | 20260505 10:32:48.741 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:48.741 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:48.741 | INFO | PRE_HQ : 30.3%, 1.0, 15.8ms, 0 | 20260505 10:32:48.741 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:48.741 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:48.741 | INFO | Uploaded 36GiB to GPU in 2.18112s (16.5053GiB/s) | 20260505 10:32:48.741 | INFO | Downloaded 34GiB to GPU in 2.15227s (15.7973GiB/s) | 20260505 10:32:53.879 | INFO | Analyzing chunk frames = [2518528,2519040) | 20260505 10:32:53.879 | INFO | MemInfo: { | 20260505 10:32:53.879 | INFO | "gpu" : " 17 GB", | 20260505 10:32:53.879 | INFO | "huge" : "", | 20260505 10:32:53.879 | INFO | "huge_shared" : " 27 GB", | 20260505 10:32:53.879 | INFO | "malloc" : " 14 GB", | 20260505 10:32:53.879 | INFO | "pinned" : " 29 GB" | 20260505 10:32:53.879 | INFO | } | 20260505 10:32:53.879 | INFO | Perf report for the last 5.1382 seconds: | 20260505 10:32:53.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:32:53.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:53.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:53.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:53.879 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 10:32:53.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:53.879 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:53.879 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 10:32:53.879 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:53.879 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:53.879 | INFO | Uploaded 36GiB to GPU in 2.21893s (16.224GiB/s) | 20260505 10:32:53.879 | INFO | Downloaded 34GiB to GPU in 2.08072s (16.3405GiB/s) | 20260505 10:32:54.013 | INFO | Starting a round of Baz IO | 20260505 10:32:57.272 | INFO | Populating RTMetrics report | 20260505 10:32:57.272 | INFO | RTMetrics report populated | 20260505 10:32:57.272 | INFO | Opening tmp RTMetrics json file | 20260505 10:32:57.273 | INFO | Serializing RTMetrics to json | 20260505 10:32:57.274 | INFO | Serialization done, swapping tmp file now... | 20260505 10:32:57.275 | INFO | RTMetrics done | 20260505 10:32:57.445 | INFO | Finished a round of Baz IO: Wrote 2.409 GiB in 3.432 seconds | 20260505 10:32:57.445 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.90% / 0.01% / 75.09% | 20260505 10:32:57.445 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.45849% / 2.38419e-05% / 9.13938e-05% | 20260505 10:32:57.445 | INFO | Baz IO overall: | 20260505 10:32:57.445 | INFO | BytesWritten : 1477445880251 | 20260505 10:32:57.445 | INFO | NumEvents : 703443011207 | 20260505 10:32:59.017 | INFO | Analyzing chunk frames = [2519040,2519552) | 20260505 10:32:59.017 | INFO | MemInfo: { | 20260505 10:32:59.017 | INFO | "gpu" : " 17 GB", | 20260505 10:32:59.017 | INFO | "huge" : "", | 20260505 10:32:59.017 | INFO | "huge_shared" : " 27 GB", | 20260505 10:32:59.017 | INFO | "malloc" : " 14 GB", | 20260505 10:32:59.017 | INFO | "pinned" : " 29 GB" | 20260505 10:32:59.017 | INFO | } | 20260505 10:32:59.018 | INFO | Perf report for the last 5.13875 seconds: | 20260505 10:32:59.018 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:32:59.018 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:59.018 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:59.018 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:59.018 | INFO | ANALYSIS : 64.9%, 2.9, 99.5ms, 0 | 20260505 10:32:59.018 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:32:59.018 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260505 10:32:59.018 | INFO | PRE_HQ : 62.5%, 1.8, 59.1ms, 0 | 20260505 10:32:59.018 | INFO | BAZWRITER : 66.1%, 4.2, 142.2ms, 0 | 20260505 10:32:59.018 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 10:32:59.018 | INFO | Uploaded 36GiB to GPU in 2.29522s (15.6848GiB/s) | 20260505 10:32:59.018 | INFO | Downloaded 38GiB to GPU in 2.37875s (15.9748GiB/s) | 20260505 10:33:04.151 | INFO | Analyzing chunk frames = [2519552,2520064) | 20260505 10:33:04.151 | INFO | MemInfo: { | 20260505 10:33:04.151 | INFO | "gpu" : " 17 GB", | 20260505 10:33:04.151 | INFO | "huge" : "", | 20260505 10:33:04.151 | INFO | "huge_shared" : " 27 GB", | 20260505 10:33:04.151 | INFO | "malloc" : " 14 GB", | 20260505 10:33:04.151 | INFO | "pinned" : " 29 GB" | 20260505 10:33:04.151 | INFO | } | 20260505 10:33:04.151 | INFO | Perf report for the last 5.13358 seconds: | 20260505 10:33:04.151 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:33:04.151 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:04.151 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:04.151 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:04.151 | INFO | ANALYSIS : 64.3%, 3.0, 100.3ms, 0 | 20260505 10:33:04.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:04.151 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:04.151 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:33:04.151 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:04.151 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:04.151 | INFO | Uploaded 36GiB to GPU in 2.17346s (16.5635GiB/s) | 20260505 10:33:04.151 | INFO | Downloaded 34GiB to GPU in 2.09081s (16.2617GiB/s) | 20260505 10:33:09.296 | INFO | Analyzing chunk frames = [2520064,2520576) | 20260505 10:33:09.296 | INFO | MemInfo: { | 20260505 10:33:09.296 | INFO | "gpu" : " 17 GB", | 20260505 10:33:09.296 | INFO | "huge" : "", | 20260505 10:33:09.296 | INFO | "huge_shared" : " 27 GB", | 20260505 10:33:09.296 | INFO | "malloc" : " 14 GB", | 20260505 10:33:09.296 | INFO | "pinned" : " 29 GB" | 20260505 10:33:09.296 | INFO | } | 20260505 10:33:09.297 | INFO | Perf report for the last 5.14541 seconds: | 20260505 10:33:09.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:33:09.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:09.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:09.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:09.297 | INFO | ANALYSIS : 62.9%, 3.0, 98.0ms, 0 | 20260505 10:33:09.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:09.297 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:09.297 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:33:09.297 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:09.297 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:09.297 | INFO | Uploaded 36GiB to GPU in 2.15245s (16.7251GiB/s) | 20260505 10:33:09.297 | INFO | Downloaded 34GiB to GPU in 2.0626s (16.4841GiB/s) | 20260505 10:33:14.434 | INFO | Analyzing chunk frames = [2520576,2521088) | 20260505 10:33:14.434 | INFO | MemInfo: { | 20260505 10:33:14.434 | INFO | "gpu" : " 17 GB", | 20260505 10:33:14.435 | INFO | "huge" : "", | 20260505 10:33:14.435 | INFO | "huge_shared" : " 27 GB", | 20260505 10:33:14.435 | INFO | "malloc" : " 14 GB", | 20260505 10:33:14.435 | INFO | "pinned" : " 29 GB" | 20260505 10:33:14.435 | INFO | } | 20260505 10:33:14.435 | INFO | Perf report for the last 5.1382 seconds: | 20260505 10:33:14.435 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:33:14.435 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:14.435 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:14.435 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:14.435 | INFO | ANALYSIS : 63.7%, 3.0, 98.9ms, 0 | 20260505 10:33:14.435 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:14.435 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:14.435 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:33:14.435 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:14.435 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:14.435 | INFO | Uploaded 36GiB to GPU in 2.22075s (16.2107GiB/s) | 20260505 10:33:14.435 | INFO | Downloaded 34GiB to GPU in 2.13094s (15.9554GiB/s) | 20260505 10:33:19.573 | INFO | Analyzing chunk frames = [2521088,2521600) | 20260505 10:33:19.573 | INFO | MemInfo: { | 20260505 10:33:19.573 | INFO | "gpu" : " 17 GB", | 20260505 10:33:19.573 | INFO | "huge" : "", | 20260505 10:33:19.573 | INFO | "huge_shared" : " 27 GB", | 20260505 10:33:19.573 | INFO | "malloc" : " 14 GB", | 20260505 10:33:19.573 | INFO | "pinned" : " 29 GB" | 20260505 10:33:19.573 | INFO | } | 20260505 10:33:19.573 | INFO | Perf report for the last 5.13803 seconds: | 20260505 10:33:19.573 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:33:19.573 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:19.573 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:19.573 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:19.573 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 10:33:19.573 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:19.573 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:19.573 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 10:33:19.573 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:19.573 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:19.573 | INFO | Uploaded 36GiB to GPU in 2.16739s (16.6099GiB/s) | 20260505 10:33:19.573 | INFO | Downloaded 34GiB to GPU in 2.13005s (15.9621GiB/s) | 20260505 10:33:24.705 | INFO | Analyzing chunk frames = [2521600,2522112) | 20260505 10:33:24.705 | INFO | MemInfo: { | 20260505 10:33:24.705 | INFO | "gpu" : " 17 GB", | 20260505 10:33:24.705 | INFO | "huge" : "", | 20260505 10:33:24.705 | INFO | "huge_shared" : " 27 GB", | 20260505 10:33:24.705 | INFO | "malloc" : " 14 GB", | 20260505 10:33:24.705 | INFO | "pinned" : " 29 GB" | 20260505 10:33:24.705 | INFO | } | 20260505 10:33:24.705 | INFO | Perf report for the last 5.13208 seconds: | 20260505 10:33:24.705 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:33:24.705 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:24.705 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:24.705 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:24.705 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 10:33:24.705 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:24.705 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:24.705 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 10:33:24.705 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:24.705 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:24.705 | INFO | Uploaded 36GiB to GPU in 2.19025s (16.4365GiB/s) | 20260505 10:33:24.705 | INFO | Downloaded 34GiB to GPU in 2.09598s (16.2215GiB/s) | 20260505 10:33:29.844 | INFO | Analyzing chunk frames = [2522112,2522624) | 20260505 10:33:29.844 | INFO | MemInfo: { | 20260505 10:33:29.844 | INFO | "gpu" : " 17 GB", | 20260505 10:33:29.844 | INFO | "huge" : "", | 20260505 10:33:29.844 | INFO | "huge_shared" : " 27 GB", | 20260505 10:33:29.844 | INFO | "malloc" : " 14 GB", | 20260505 10:33:29.844 | INFO | "pinned" : " 29 GB" | 20260505 10:33:29.844 | INFO | } | 20260505 10:33:29.845 | INFO | Perf report for the last 5.13963 seconds: | 20260505 10:33:29.845 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:33:29.845 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:29.845 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:29.845 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:29.845 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 10:33:29.845 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:29.845 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:29.845 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 10:33:29.845 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:29.845 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:29.845 | INFO | Uploaded 36GiB to GPU in 2.1562s (16.696GiB/s) | 20260505 10:33:29.845 | INFO | Downloaded 34GiB to GPU in 2.15712s (15.7618GiB/s) | 20260505 10:33:34.991 | INFO | Analyzing chunk frames = [2522624,2523136) | 20260505 10:33:34.991 | INFO | MemInfo: { | 20260505 10:33:34.991 | INFO | "gpu" : " 17 GB", | 20260505 10:33:34.991 | INFO | "huge" : "", | 20260505 10:33:34.991 | INFO | "huge_shared" : " 27 GB", | 20260505 10:33:34.991 | INFO | "malloc" : " 14 GB", | 20260505 10:33:34.991 | INFO | "pinned" : " 29 GB" | 20260505 10:33:34.992 | INFO | } | 20260505 10:33:34.992 | INFO | Perf report for the last 5.14701 seconds: | 20260505 10:33:34.992 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:33:34.992 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:34.992 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:34.992 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:34.992 | INFO | ANALYSIS : 64.1%, 3.0, 100.2ms, 0 | 20260505 10:33:34.992 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:34.992 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:34.992 | INFO | PRE_HQ : 29.7%, 1.0, 15.5ms, 0 | 20260505 10:33:34.992 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:34.992 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:34.992 | INFO | Uploaded 36GiB to GPU in 2.21927s (16.2215GiB/s) | 20260505 10:33:34.992 | INFO | Downloaded 34GiB to GPU in 2.09431s (16.2345GiB/s) | 20260505 10:33:34.992 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 10:33:34.992 | INFO | Total mem usage: 15.3 GiB | 20260505 10:33:34.992 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 10:33:34.992 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.41%) | 20260505 10:33:34.992 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 10:33:34.992 | INFO | High Water Breakdown: | 20260505 10:33:34.992 | INFO | 60.9% : 8.78 GiB ( 92.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 10:33:34.992 | INFO | 38.9% : 5.6 GiB ( 84.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 10:33:34.992 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 10:33:34.992 | INFO | ------------------------------------------ | 20260505 10:33:34.992 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:33:34.992 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 10:33:34.992 | INFO | Total mem usage: 26.5 GiB | 20260505 10:33:34.992 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 10:33:34.992 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 10:33:34.992 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 10:33:34.992 | INFO | High Water Breakdown: | 20260505 10:33:34.992 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:33:34.992 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:33:34.992 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:33:34.992 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:33:34.992 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:33:34.992 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:33:34.992 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 10:33:34.992 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 10:33:34.992 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:33:34.992 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 10:33:34.992 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 10:33:34.992 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 10:33:34.992 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 10:33:34.992 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 10:33:34.992 | INFO | ------------------------------------------ | 20260505 10:33:34.992 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:33:34.992 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 10:33:34.992 | INFO | Total mem usage: 25.6 GiB | 20260505 10:33:34.992 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 10:33:34.992 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 10:33:34.992 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 10:33:34.992 | INFO | High Water Breakdown: | 20260505 10:33:34.993 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 10:33:34.993 | INFO | ------------------------------------------ | 20260505 10:33:34.993 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:33:34.993 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 10:33:34.993 | INFO | Total mem usage: 15.7 GiB | 20260505 10:33:34.993 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 10:33:34.993 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 10:33:34.993 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 10:33:34.993 | INFO | High Water Breakdown: | 20260505 10:33:34.993 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 10:33:34.993 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 10:33:34.993 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 10:33:34.993 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 10:33:34.993 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 10:33:34.993 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 10:33:34.993 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 10:33:34.993 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 10:33:34.993 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 10:33:34.993 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:33:34.993 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 10:33:34.993 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:33:34.993 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 10:33:34.993 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 10:33:34.993 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 10:33:34.993 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 10:33:34.993 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:33:34.993 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 10:33:34.993 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:33:34.993 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:33:34.993 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:33:34.993 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 10:33:34.993 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 10:33:34.993 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:33:34.993 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 10:33:34.993 | INFO | ------------------------------------------ | 20260505 10:33:34.993 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:33:34.993 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 10:33:34.993 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:33:34.993 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:33:34.993 | INFO | PerfReport: Full Execution -- 3.23e+01 / 7.70e+01, 1.38e+01 / 2.50e+00 | 20260505 10:33:34.993 | INFO | PerfReport: 0.29%, AggregateStats -- 9.33e-02 / 1.05e-01, 1.18e-02 / 1.45e-02 | 20260505 10:33:34.993 | INFO | PerfReport: 11.67%, Baseline -- 3.77e+00 / 4.38e+00, 6.05e-01 / 3.12e-01 | 20260505 10:33:34.993 | INFO | PerfReport: 3.31%, Histogram -- 1.07e+00 / 1.45e+00, 2.04e-01 / 1.29e-01 | 20260505 10:33:34.993 | INFO | PerfReport: 15.79%, DME -- 4.08e+01 / 4.52e+01, 4.53e+00 / 2.92e+00 | 20260505 10:33:34.993 | INFO | PerfReport: 56.43%, FrameLabeling -- 1.82e+01 / 2.09e+01, 1.72e+00 / 1.99e+00 | 20260505 10:33:34.993 | INFO | PerfReport: 7.44%, PulseAccumulating -- 2.40e+00 / 2.97e+00, 4.32e-01 / 2.26e-01 | 20260505 10:33:34.993 | INFO | PerfReport: 3.75%, Metrics -- 1.21e+00 / 1.54e+00, 3.47e-01 / 2.01e-01 | 20260505 10:33:34.993 | INFO | PerfReport: 1.32%, PulseSerializing -- 4.27e-01 / 4.78e-01, 7.82e-02 / 2.98e-02 | 20260505 10:33:34.993 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 10:33:34.993 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:33:34.993 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:33:34.993 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260505 10:33:34.993 | INFO | PerfReport: 22.56%, Upload -- 2.28e+01 / 0.00e+00, 8.15e+00 / 0.00e+00 | 20260505 10:33:34.993 | INFO | PerfReport: 21.97%, Download -- 2.22e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260505 10:33:34.993 | INFO | PerfReport: 31.89%, Compute -- 3.23e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 10:33:34.993 | INFO | PerfReport: 1.87%, LockUpload -- 1.89e+00 / 0.00e+00, 7.36e+00 / 0.00e+00 | 20260505 10:33:34.993 | INFO | PerfReport: 1.27%, LockDownload -- 1.29e+00 / 0.00e+00, 2.95e+00 / 0.00e+00 | 20260505 10:33:34.993 | INFO | PerfReport: 20.42%, LockCompute -- 2.07e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260505 10:33:34.993 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 2.36e-02 / 0.00e+00 | 20260505 10:33:35.127 | INFO | Starting a round of Baz IO | 20260505 10:33:38.385 | INFO | Populating RTMetrics report | 20260505 10:33:38.385 | INFO | RTMetrics report populated | 20260505 10:33:38.385 | INFO | Opening tmp RTMetrics json file | 20260505 10:33:38.389 | INFO | Serializing RTMetrics to json | 20260505 10:33:38.392 | INFO | Serialization done, swapping tmp file now... | 20260505 10:33:38.392 | INFO | RTMetrics done | 20260505 10:33:38.519 | INFO | Finished a round of Baz IO: Wrote 2.408 GiB in 3.391 seconds | 20260505 10:33:38.519 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.89% / 0.01% / 75.10% | 20260505 10:33:38.519 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.44961% / 1.98682e-05% / 0.000135104% | 20260505 10:33:38.519 | INFO | Baz IO overall: | 20260505 10:33:38.519 | INFO | BytesWritten : 1480031153930 | 20260505 10:33:38.519 | INFO | NumEvents : 704674801631 | 20260505 10:33:40.140 | INFO | Analyzing chunk frames = [2523136,2523648) | 20260505 10:33:40.140 | INFO | MemInfo: { | 20260505 10:33:40.140 | INFO | "gpu" : " 17 GB", | 20260505 10:33:40.140 | INFO | "huge" : "", | 20260505 10:33:40.140 | INFO | "huge_shared" : " 27 GB", | 20260505 10:33:40.140 | INFO | "malloc" : " 14 GB", | 20260505 10:33:40.140 | INFO | "pinned" : " 29 GB" | 20260505 10:33:40.140 | INFO | } | 20260505 10:33:40.140 | INFO | Perf report for the last 5.14872 seconds: | 20260505 10:33:40.140 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:33:40.140 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:40.140 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:40.140 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:40.140 | INFO | ANALYSIS : 64.8%, 2.9, 99.4ms, 0 | 20260505 10:33:40.140 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:40.140 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260505 10:33:40.140 | INFO | PRE_HQ : 61.2%, 1.9, 59.3ms, 0 | 20260505 10:33:40.141 | INFO | BAZWRITER : 65.0%, 4.2, 140.0ms, 0 | 20260505 10:33:40.141 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 10:33:40.141 | INFO | Uploaded 36GiB to GPU in 2.1924s (16.4204GiB/s) | 20260505 10:33:40.141 | INFO | Downloaded 38GiB to GPU in 2.29944s (16.5258GiB/s) | 20260505 10:33:45.266 | INFO | Analyzing chunk frames = [2523648,2524160) | 20260505 10:33:45.266 | INFO | MemInfo: { | 20260505 10:33:45.266 | INFO | "gpu" : " 17 GB", | 20260505 10:33:45.266 | INFO | "huge" : "", | 20260505 10:33:45.266 | INFO | "huge_shared" : " 27 GB", | 20260505 10:33:45.266 | INFO | "malloc" : " 14 GB", | 20260505 10:33:45.266 | INFO | "pinned" : " 29 GB" | 20260505 10:33:45.266 | INFO | } | 20260505 10:33:45.266 | INFO | Perf report for the last 5.12603 seconds: | 20260505 10:33:45.266 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:33:45.266 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:45.266 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:45.266 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:45.266 | INFO | ANALYSIS : 64.4%, 3.0, 100.4ms, 0 | 20260505 10:33:45.266 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:45.266 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:45.266 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 10:33:45.266 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:45.266 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:45.267 | INFO | Uploaded 36GiB to GPU in 2.23383s (16.1158GiB/s) | 20260505 10:33:45.267 | INFO | Downloaded 34GiB to GPU in 2.08831s (16.2811GiB/s) | 20260505 10:33:50.400 | INFO | Analyzing chunk frames = [2524160,2524672) | 20260505 10:33:50.400 | INFO | MemInfo: { | 20260505 10:33:50.400 | INFO | "gpu" : " 17 GB", | 20260505 10:33:50.400 | INFO | "huge" : "", | 20260505 10:33:50.400 | INFO | "huge_shared" : " 27 GB", | 20260505 10:33:50.400 | INFO | "malloc" : " 14 GB", | 20260505 10:33:50.400 | INFO | "pinned" : " 29 GB" | 20260505 10:33:50.400 | INFO | } | 20260505 10:33:50.400 | INFO | Perf report for the last 5.13341 seconds: | 20260505 10:33:50.400 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:33:50.400 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:50.400 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:50.400 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:50.400 | INFO | ANALYSIS : 63.2%, 3.0, 98.3ms, 0 | 20260505 10:33:50.400 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:50.400 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:50.400 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 10:33:50.400 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:50.400 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:50.400 | INFO | Uploaded 36GiB to GPU in 2.15606s (16.6971GiB/s) | 20260505 10:33:50.400 | INFO | Downloaded 34GiB to GPU in 2.07222s (16.4075GiB/s) | 20260505 10:33:55.549 | INFO | Analyzing chunk frames = [2524672,2525184) | 20260505 10:33:55.549 | INFO | MemInfo: { | 20260505 10:33:55.549 | INFO | "gpu" : " 17 GB", | 20260505 10:33:55.549 | INFO | "huge" : "", | 20260505 10:33:55.549 | INFO | "huge_shared" : " 27 GB", | 20260505 10:33:55.549 | INFO | "malloc" : " 14 GB", | 20260505 10:33:55.549 | INFO | "pinned" : " 29 GB" | 20260505 10:33:55.549 | INFO | } | 20260505 10:33:55.549 | INFO | Perf report for the last 5.14926 seconds: | 20260505 10:33:55.549 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:33:55.549 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:55.549 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:55.549 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:55.549 | INFO | ANALYSIS : 62.9%, 3.0, 97.9ms, 0 | 20260505 10:33:55.549 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:55.549 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:55.549 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:33:55.549 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:55.549 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:33:55.549 | INFO | Uploaded 36GiB to GPU in 2.16782s (16.6066GiB/s) | 20260505 10:33:55.549 | INFO | Downloaded 34GiB to GPU in 2.07761s (16.365GiB/s) | 20260505 10:34:00.685 | INFO | Analyzing chunk frames = [2525184,2525696) | 20260505 10:34:00.685 | INFO | MemInfo: { | 20260505 10:34:00.685 | INFO | "gpu" : " 17 GB", | 20260505 10:34:00.686 | INFO | "huge" : "", | 20260505 10:34:00.686 | INFO | "huge_shared" : " 27 GB", | 20260505 10:34:00.686 | INFO | "malloc" : " 14 GB", | 20260505 10:34:00.686 | INFO | "pinned" : " 29 GB" | 20260505 10:34:00.686 | INFO | } | 20260505 10:34:00.686 | INFO | Perf report for the last 5.13663 seconds: | 20260505 10:34:00.686 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:34:00.686 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:00.686 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:00.686 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:00.686 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 10:34:00.686 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:00.686 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:00.686 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:34:00.686 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:00.686 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:00.686 | INFO | Uploaded 36GiB to GPU in 2.15779s (16.6837GiB/s) | 20260505 10:34:00.686 | INFO | Downloaded 34GiB to GPU in 2.1244s (16.0045GiB/s) | 20260505 10:34:05.828 | INFO | Analyzing chunk frames = [2525696,2526208) | 20260505 10:34:05.828 | INFO | MemInfo: { | 20260505 10:34:05.828 | INFO | "gpu" : " 17 GB", | 20260505 10:34:05.828 | INFO | "huge" : "", | 20260505 10:34:05.828 | INFO | "huge_shared" : " 27 GB", | 20260505 10:34:05.828 | INFO | "malloc" : " 14 GB", | 20260505 10:34:05.828 | INFO | "pinned" : " 29 GB" | 20260505 10:34:05.828 | INFO | } | 20260505 10:34:05.828 | INFO | Perf report for the last 5.1421 seconds: | 20260505 10:34:05.828 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:34:05.828 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:05.828 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:05.828 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:05.828 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 10:34:05.828 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:05.828 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:05.828 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 10:34:05.828 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:05.828 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:05.828 | INFO | Uploaded 36GiB to GPU in 2.21011s (16.2888GiB/s) | 20260505 10:34:05.828 | INFO | Downloaded 34GiB to GPU in 2.10196s (16.1754GiB/s) | 20260505 10:34:10.965 | INFO | Analyzing chunk frames = [2526208,2526720) | 20260505 10:34:10.965 | INFO | MemInfo: { | 20260505 10:34:10.965 | INFO | "gpu" : " 17 GB", | 20260505 10:34:10.965 | INFO | "huge" : "", | 20260505 10:34:10.965 | INFO | "huge_shared" : " 27 GB", | 20260505 10:34:10.965 | INFO | "malloc" : " 14 GB", | 20260505 10:34:10.965 | INFO | "pinned" : " 29 GB" | 20260505 10:34:10.965 | INFO | } | 20260505 10:34:10.965 | INFO | Perf report for the last 5.13742 seconds: | 20260505 10:34:10.965 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:34:10.965 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:10.965 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:10.965 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:10.965 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 10:34:10.965 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:10.965 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:10.965 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 10:34:10.965 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:10.965 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:10.965 | INFO | Uploaded 36GiB to GPU in 2.17772s (16.5311GiB/s) | 20260505 10:34:10.966 | INFO | Downloaded 34GiB to GPU in 2.14681s (15.8374GiB/s) | 20260505 10:34:16.104 | INFO | Analyzing chunk frames = [2526720,2527232) | 20260505 10:34:16.104 | INFO | MemInfo: { | 20260505 10:34:16.104 | INFO | "gpu" : " 17 GB", | 20260505 10:34:16.104 | INFO | "huge" : "", | 20260505 10:34:16.104 | INFO | "huge_shared" : " 27 GB", | 20260505 10:34:16.104 | INFO | "malloc" : " 14 GB", | 20260505 10:34:16.104 | INFO | "pinned" : " 29 GB" | 20260505 10:34:16.104 | INFO | } | 20260505 10:34:16.104 | INFO | Perf report for the last 5.13863 seconds: | 20260505 10:34:16.104 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:34:16.104 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:16.104 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:16.104 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:16.104 | INFO | ANALYSIS : 64.6%, 3.0, 100.8ms, 0 | 20260505 10:34:16.104 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:16.104 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:16.104 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 10:34:16.104 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:16.104 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:16.104 | INFO | Uploaded 36GiB to GPU in 2.24543s (16.0326GiB/s) | 20260505 10:34:16.104 | INFO | Downloaded 34GiB to GPU in 2.10102s (16.1826GiB/s) | 20260505 10:34:16.246 | INFO | Starting a round of Baz IO | 20260505 10:34:19.507 | INFO | Populating RTMetrics report | 20260505 10:34:19.507 | INFO | RTMetrics report populated | 20260505 10:34:19.507 | INFO | Opening tmp RTMetrics json file | 20260505 10:34:19.577 | INFO | Serializing RTMetrics to json | 20260505 10:34:19.579 | INFO | Serialization done, swapping tmp file now... | 20260505 10:34:19.580 | INFO | RTMetrics done | 20260505 10:34:19.645 | INFO | Finished a round of Baz IO: Wrote 2.409 GiB in 3.398 seconds | 20260505 10:34:19.645 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.87% / 0.01% / 75.11% | 20260505 10:34:19.645 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.44815% / 1.19209e-05% / 8.74201e-05% | 20260505 10:34:19.645 | INFO | Baz IO overall: | 20260505 10:34:19.645 | INFO | BytesWritten : 1482618163434 | 20260505 10:34:19.645 | INFO | NumEvents : 705908143642 | 20260505 10:34:21.254 | INFO | Analyzing chunk frames = [2527232,2527744) | 20260505 10:34:21.254 | INFO | MemInfo: { | 20260505 10:34:21.254 | INFO | "gpu" : " 17 GB", | 20260505 10:34:21.254 | INFO | "huge" : "", | 20260505 10:34:21.254 | INFO | "huge_shared" : " 27 GB", | 20260505 10:34:21.254 | INFO | "malloc" : " 14 GB", | 20260505 10:34:21.254 | INFO | "pinned" : " 29 GB" | 20260505 10:34:21.254 | INFO | } | 20260505 10:34:21.255 | INFO | Perf report for the last 5.15067 seconds: | 20260505 10:34:21.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:34:21.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:21.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:21.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:21.255 | INFO | ANALYSIS : 64.9%, 2.9, 99.6ms, 0 | 20260505 10:34:21.255 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:21.255 | INFO | RT_METRICS : 46.9%, 1.0, 24.6ms, 0 | 20260505 10:34:21.255 | INFO | PRE_HQ : 61.5%, 1.8, 59.1ms, 0 | 20260505 10:34:21.255 | INFO | BAZWRITER : 65.3%, 4.2, 140.0ms, 0 | 20260505 10:34:21.255 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260505 10:34:21.255 | INFO | Uploaded 36GiB to GPU in 2.19378s (16.41GiB/s) | 20260505 10:34:21.255 | INFO | Downloaded 38GiB to GPU in 2.31091s (16.4438GiB/s) | 20260505 10:34:26.384 | INFO | Analyzing chunk frames = [2527744,2528256) | 20260505 10:34:26.385 | INFO | MemInfo: { | 20260505 10:34:26.385 | INFO | "gpu" : " 17 GB", | 20260505 10:34:26.385 | INFO | "huge" : "", | 20260505 10:34:26.385 | INFO | "huge_shared" : " 27 GB", | 20260505 10:34:26.385 | INFO | "malloc" : " 14 GB", | 20260505 10:34:26.385 | INFO | "pinned" : " 29 GB" | 20260505 10:34:26.385 | INFO | } | 20260505 10:34:26.385 | INFO | Perf report for the last 5.13013 seconds: | 20260505 10:34:26.385 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:34:26.385 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:26.385 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:26.385 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:26.385 | INFO | ANALYSIS : 64.4%, 3.0, 100.4ms, 0 | 20260505 10:34:26.385 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:26.385 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:26.385 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:34:26.385 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:26.385 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:26.385 | INFO | Uploaded 36GiB to GPU in 2.2345s (16.111GiB/s) | 20260505 10:34:26.385 | INFO | Downloaded 34GiB to GPU in 2.09981s (16.1919GiB/s) | 20260505 10:34:31.522 | INFO | Analyzing chunk frames = [2528256,2528768) | 20260505 10:34:31.522 | INFO | MemInfo: { | 20260505 10:34:31.522 | INFO | "gpu" : " 17 GB", | 20260505 10:34:31.522 | INFO | "huge" : "", | 20260505 10:34:31.522 | INFO | "huge_shared" : " 27 GB", | 20260505 10:34:31.522 | INFO | "malloc" : " 14 GB", | 20260505 10:34:31.522 | INFO | "pinned" : " 29 GB" | 20260505 10:34:31.522 | INFO | } | 20260505 10:34:31.522 | INFO | Perf report for the last 5.13754 seconds: | 20260505 10:34:31.522 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:34:31.522 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:31.522 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:31.522 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:31.522 | INFO | ANALYSIS : 63.0%, 3.0, 98.0ms, 0 | 20260505 10:34:31.522 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:31.522 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:31.522 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:34:31.522 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:31.522 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:31.523 | INFO | Uploaded 36GiB to GPU in 2.15075s (16.7384GiB/s) | 20260505 10:34:31.523 | INFO | Downloaded 34GiB to GPU in 2.06889s (16.434GiB/s) | 20260505 10:34:36.657 | INFO | Analyzing chunk frames = [2528768,2529280) | 20260505 10:34:36.657 | INFO | MemInfo: { | 20260505 10:34:36.657 | INFO | "gpu" : " 17 GB", | 20260505 10:34:36.657 | INFO | "huge" : "", | 20260505 10:34:36.657 | INFO | "huge_shared" : " 27 GB", | 20260505 10:34:36.657 | INFO | "malloc" : " 14 GB", | 20260505 10:34:36.657 | INFO | "pinned" : " 29 GB" | 20260505 10:34:36.657 | INFO | } | 20260505 10:34:36.657 | INFO | Perf report for the last 5.13463 seconds: | 20260505 10:34:36.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:34:36.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:36.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:36.657 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:36.657 | INFO | ANALYSIS : 63.0%, 3.0, 97.9ms, 0 | 20260505 10:34:36.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:36.657 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:36.657 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 10:34:36.657 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:36.657 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:36.657 | INFO | Uploaded 36GiB to GPU in 2.16083s (16.6603GiB/s) | 20260505 10:34:36.657 | INFO | Downloaded 34GiB to GPU in 2.07331s (16.3989GiB/s) | 20260505 10:34:41.802 | INFO | Analyzing chunk frames = [2529280,2529792) | 20260505 10:34:41.802 | INFO | MemInfo: { | 20260505 10:34:41.802 | INFO | "gpu" : " 17 GB", | 20260505 10:34:41.802 | INFO | "huge" : "", | 20260505 10:34:41.803 | INFO | "huge_shared" : " 27 GB", | 20260505 10:34:41.803 | INFO | "malloc" : " 14 GB", | 20260505 10:34:41.803 | INFO | "pinned" : " 29 GB" | 20260505 10:34:41.803 | INFO | } | 20260505 10:34:41.803 | INFO | Perf report for the last 5.14576 seconds: | 20260505 10:34:41.803 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:34:41.803 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:41.803 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:41.803 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:41.803 | INFO | ANALYSIS : 63.6%, 3.0, 99.5ms, 0 | 20260505 10:34:41.803 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:41.803 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:41.803 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 10:34:41.803 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:41.803 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:41.803 | INFO | Uploaded 36GiB to GPU in 2.23646s (16.0969GiB/s) | 20260505 10:34:41.803 | INFO | Downloaded 34GiB to GPU in 2.11173s (16.1006GiB/s) | 20260505 10:34:46.940 | INFO | Analyzing chunk frames = [2529792,2530304) | 20260505 10:34:46.940 | INFO | MemInfo: { | 20260505 10:34:46.940 | INFO | "gpu" : " 17 GB", | 20260505 10:34:46.940 | INFO | "huge" : "", | 20260505 10:34:46.940 | INFO | "huge_shared" : " 27 GB", | 20260505 10:34:46.940 | INFO | "malloc" : " 14 GB", | 20260505 10:34:46.940 | INFO | "pinned" : " 29 GB" | 20260505 10:34:46.940 | INFO | } | 20260505 10:34:46.940 | INFO | Perf report for the last 5.13773 seconds: | 20260505 10:34:46.940 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:34:46.940 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:46.940 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:46.940 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:46.940 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 10:34:46.940 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:46.940 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:46.940 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 10:34:46.940 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:46.940 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:46.941 | INFO | Uploaded 36GiB to GPU in 2.16108s (16.6583GiB/s) | 20260505 10:34:46.941 | INFO | Downloaded 34GiB to GPU in 2.11602s (16.0679GiB/s) | 20260505 10:34:52.084 | INFO | Analyzing chunk frames = [2530304,2530816) | 20260505 10:34:52.084 | INFO | MemInfo: { | 20260505 10:34:52.084 | INFO | "gpu" : " 17 GB", | 20260505 10:34:52.084 | INFO | "huge" : "", | 20260505 10:34:52.084 | INFO | "huge_shared" : " 27 GB", | 20260505 10:34:52.084 | INFO | "malloc" : " 14 GB", | 20260505 10:34:52.084 | INFO | "pinned" : " 29 GB" | 20260505 10:34:52.084 | INFO | } | 20260505 10:34:52.084 | INFO | Perf report for the last 5.14368 seconds: | 20260505 10:34:52.084 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:34:52.084 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:52.084 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:52.084 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:52.084 | INFO | ANALYSIS : 63.7%, 3.0, 99.6ms, 0 | 20260505 10:34:52.084 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:52.084 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:52.084 | INFO | PRE_HQ : 30.5%, 1.0, 15.9ms, 0 | 20260505 10:34:52.084 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:52.084 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:52.084 | INFO | Uploaded 36GiB to GPU in 2.17275s (16.5689GiB/s) | 20260505 10:34:52.084 | INFO | Downloaded 34GiB to GPU in 2.14338s (15.8628GiB/s) | 20260505 10:34:57.214 | INFO | Analyzing chunk frames = [2530816,2531328) | 20260505 10:34:57.214 | INFO | MemInfo: { | 20260505 10:34:57.214 | INFO | "gpu" : " 17 GB", | 20260505 10:34:57.214 | INFO | "huge" : "", | 20260505 10:34:57.214 | INFO | "huge_shared" : " 27 GB", | 20260505 10:34:57.215 | INFO | "malloc" : " 14 GB", | 20260505 10:34:57.215 | INFO | "pinned" : " 29 GB" | 20260505 10:34:57.215 | INFO | } | 20260505 10:34:57.215 | INFO | Perf report for the last 5.13048 seconds: | 20260505 10:34:57.215 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:34:57.215 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:57.215 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:57.215 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:57.215 | INFO | ANALYSIS : 64.5%, 3.0, 100.6ms, 0 | 20260505 10:34:57.215 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:57.215 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:57.215 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 10:34:57.215 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:57.215 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:34:57.215 | INFO | Uploaded 36GiB to GPU in 2.24538s (16.0329GiB/s) | 20260505 10:34:57.215 | INFO | Downloaded 34GiB to GPU in 2.0868s (16.2929GiB/s) | 20260505 10:34:57.215 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 10:34:57.215 | INFO | Total mem usage: 15.3 GiB | 20260505 10:34:57.215 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 10:34:57.215 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.76%) | 20260505 10:34:57.215 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 10:34:57.215 | INFO | High Water Breakdown: | 20260505 10:34:57.215 | INFO | 61% : 8.76 GiB ( 92.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 10:34:57.215 | INFO | 38.9% : 5.58 GiB ( 84% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 10:34:57.215 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 10:34:57.215 | INFO | ------------------------------------------ | 20260505 10:34:57.215 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:34:57.215 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 10:34:57.215 | INFO | Total mem usage: 26.5 GiB | 20260505 10:34:57.215 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 10:34:57.215 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 10:34:57.215 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 10:34:57.215 | INFO | High Water Breakdown: | 20260505 10:34:57.215 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:34:57.215 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:34:57.215 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:34:57.215 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:34:57.215 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:34:57.215 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:34:57.215 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 10:34:57.215 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 10:34:57.215 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:34:57.215 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 10:34:57.215 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 10:34:57.215 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 10:34:57.215 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 10:34:57.215 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 10:34:57.215 | INFO | ------------------------------------------ | 20260505 10:34:57.215 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:34:57.215 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 10:34:57.215 | INFO | Total mem usage: 25.6 GiB | 20260505 10:34:57.215 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 10:34:57.215 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 10:34:57.215 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 10:34:57.215 | INFO | High Water Breakdown: | 20260505 10:34:57.215 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 10:34:57.215 | INFO | ------------------------------------------ | 20260505 10:34:57.215 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:34:57.215 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 10:34:57.215 | INFO | Total mem usage: 15.7 GiB | 20260505 10:34:57.215 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 10:34:57.215 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 10:34:57.215 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 10:34:57.215 | INFO | High Water Breakdown: | 20260505 10:34:57.215 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 10:34:57.215 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 10:34:57.215 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 10:34:57.215 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 10:34:57.215 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 10:34:57.215 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 10:34:57.216 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 10:34:57.216 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 10:34:57.216 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 10:34:57.216 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:34:57.216 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 10:34:57.216 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:34:57.216 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 10:34:57.216 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 10:34:57.216 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 10:34:57.216 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 10:34:57.216 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:34:57.216 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 10:34:57.216 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:34:57.216 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:34:57.216 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:34:57.216 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 10:34:57.216 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 10:34:57.216 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:34:57.216 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 10:34:57.216 | INFO | ------------------------------------------ | 20260505 10:34:57.216 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:34:57.216 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 10:34:57.216 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:34:57.216 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:34:57.216 | INFO | PerfReport: Full Execution -- 3.23e+01 / 7.72e+01, 1.39e+01 / 1.70e+00 | 20260505 10:34:57.216 | INFO | PerfReport: 0.29%, AggregateStats -- 9.33e-02 / 1.04e-01, 1.11e-02 / 8.66e-03 | 20260505 10:34:57.216 | INFO | PerfReport: 11.67%, Baseline -- 3.77e+00 / 4.49e+00, 6.00e-01 / 1.80e-01 | 20260505 10:34:57.216 | INFO | PerfReport: 3.31%, Histogram -- 1.07e+00 / 1.51e+00, 2.09e-01 / 7.55e-02 | 20260505 10:34:57.216 | INFO | PerfReport: 15.76%, DME -- 4.08e+01 / 4.53e+01, 4.29e+00 / 1.67e+00 | 20260505 10:34:57.216 | INFO | PerfReport: 56.50%, FrameLabeling -- 1.83e+01 / 2.09e+01, 1.76e+00 / 2.06e+00 | 20260505 10:34:57.216 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.38e+00 / 2.93e+00, 4.09e-01 / 1.92e-01 | 20260505 10:34:57.216 | INFO | PerfReport: 3.75%, Metrics -- 1.21e+00 / 1.52e+00, 3.50e-01 / 2.03e-01 | 20260505 10:34:57.216 | INFO | PerfReport: 1.32%, PulseSerializing -- 4.28e-01 / 4.74e-01, 7.84e-02 / 3.56e-02 | 20260505 10:34:57.216 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 10:34:57.216 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:34:57.216 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:34:57.216 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.46e+01 / 0.00e+00 | 20260505 10:34:57.216 | INFO | PerfReport: 22.54%, Upload -- 2.28e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260505 10:34:57.216 | INFO | PerfReport: 21.85%, Download -- 2.22e+01 / 0.00e+00, 8.17e+00 / 0.00e+00 | 20260505 10:34:57.216 | INFO | PerfReport: 31.89%, Compute -- 3.23e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 10:34:57.216 | INFO | PerfReport: 2.01%, LockUpload -- 2.04e+00 / 0.00e+00, 7.62e+00 / 0.00e+00 | 20260505 10:34:57.216 | INFO | PerfReport: 1.14%, LockDownload -- 1.16e+00 / 0.00e+00, 2.72e+00 / 0.00e+00 | 20260505 10:34:57.216 | INFO | PerfReport: 20.56%, LockCompute -- 2.08e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260505 10:34:57.216 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.76e-02 / 0.00e+00 | 20260505 10:34:57.348 | INFO | Starting a round of Baz IO | 20260505 10:35:00.607 | INFO | Populating RTMetrics report | 20260505 10:35:00.607 | INFO | RTMetrics report populated | 20260505 10:35:00.607 | INFO | Opening tmp RTMetrics json file | 20260505 10:35:00.622 | INFO | Serializing RTMetrics to json | 20260505 10:35:00.626 | INFO | Serialization done, swapping tmp file now... | 20260505 10:35:00.627 | INFO | RTMetrics done | 20260505 10:35:00.848 | INFO | Finished a round of Baz IO: Wrote 2.407 GiB in 3.499 seconds | 20260505 10:35:00.848 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.86% / 0.01% / 75.13% | 20260505 10:35:00.848 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.44158% / 3.17891e-05% / 9.53674e-05% | 20260505 10:35:00.848 | INFO | Baz IO overall: | 20260505 10:35:00.848 | INFO | BytesWritten : 1485202637543 | 20260505 10:35:00.848 | INFO | NumEvents : 707139369511 | 20260505 10:35:02.352 | INFO | Analyzing chunk frames = [2531328,2531840) | 20260505 10:35:02.352 | INFO | MemInfo: { | 20260505 10:35:02.352 | INFO | "gpu" : " 17 GB", | 20260505 10:35:02.352 | INFO | "huge" : "", | 20260505 10:35:02.352 | INFO | "huge_shared" : " 27 GB", | 20260505 10:35:02.352 | INFO | "malloc" : " 14 GB", | 20260505 10:35:02.352 | INFO | "pinned" : " 29 GB" | 20260505 10:35:02.352 | INFO | } | 20260505 10:35:02.352 | INFO | Perf report for the last 5.13766 seconds: | 20260505 10:35:02.352 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:35:02.352 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:02.352 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:02.352 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:02.352 | INFO | ANALYSIS : 64.9%, 2.9, 98.8ms, 0 | 20260505 10:35:02.352 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:02.352 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260505 10:35:02.352 | INFO | PRE_HQ : 63.0%, 1.8, 59.6ms, 0 | 20260505 10:35:02.352 | INFO | BAZWRITER : 67.4%, 4.3, 148.6ms, 0 | 20260505 10:35:02.352 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 10:35:02.353 | INFO | Uploaded 36GiB to GPU in 2.27049s (15.8556GiB/s) | 20260505 10:35:02.353 | INFO | Downloaded 38GiB to GPU in 2.39492s (15.8669GiB/s) | 20260505 10:35:07.507 | INFO | Analyzing chunk frames = [2531840,2532352) | 20260505 10:35:07.507 | INFO | MemInfo: { | 20260505 10:35:07.507 | INFO | "gpu" : " 17 GB", | 20260505 10:35:07.507 | INFO | "huge" : "", | 20260505 10:35:07.507 | INFO | "huge_shared" : " 27 GB", | 20260505 10:35:07.507 | INFO | "malloc" : " 14 GB", | 20260505 10:35:07.507 | INFO | "pinned" : " 29 GB" | 20260505 10:35:07.507 | INFO | } | 20260505 10:35:07.507 | INFO | Perf report for the last 5.15467 seconds: | 20260505 10:35:07.507 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:35:07.507 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:07.507 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:07.507 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:07.507 | INFO | ANALYSIS : 64.1%, 3.0, 100.3ms, 0 | 20260505 10:35:07.507 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:07.507 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:07.507 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 10:35:07.507 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:07.507 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:07.507 | INFO | Uploaded 36GiB to GPU in 2.17508s (16.5511GiB/s) | 20260505 10:35:07.507 | INFO | Downloaded 34GiB to GPU in 2.09105s (16.2598GiB/s) | 20260505 10:35:12.643 | INFO | Analyzing chunk frames = [2532352,2532864) | 20260505 10:35:12.643 | INFO | MemInfo: { | 20260505 10:35:12.643 | INFO | "gpu" : " 17 GB", | 20260505 10:35:12.643 | INFO | "huge" : "", | 20260505 10:35:12.643 | INFO | "huge_shared" : " 27 GB", | 20260505 10:35:12.643 | INFO | "malloc" : " 14 GB", | 20260505 10:35:12.643 | INFO | "pinned" : " 29 GB" | 20260505 10:35:12.643 | INFO | } | 20260505 10:35:12.643 | INFO | Perf report for the last 5.13604 seconds: | 20260505 10:35:12.643 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:35:12.643 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:12.643 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:12.643 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:12.643 | INFO | ANALYSIS : 63.4%, 3.0, 98.6ms, 0 | 20260505 10:35:12.643 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:12.643 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:12.643 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:35:12.643 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:12.643 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:12.643 | INFO | Uploaded 36GiB to GPU in 2.21412s (16.2593GiB/s) | 20260505 10:35:12.643 | INFO | Downloaded 34GiB to GPU in 2.11967s (16.0403GiB/s) | 20260505 10:35:17.780 | INFO | Analyzing chunk frames = [2532864,2533376) | 20260505 10:35:17.780 | INFO | MemInfo: { | 20260505 10:35:17.780 | INFO | "gpu" : " 17 GB", | 20260505 10:35:17.780 | INFO | "huge" : "", | 20260505 10:35:17.780 | INFO | "huge_shared" : " 27 GB", | 20260505 10:35:17.780 | INFO | "malloc" : " 14 GB", | 20260505 10:35:17.780 | INFO | "pinned" : " 29 GB" | 20260505 10:35:17.780 | INFO | } | 20260505 10:35:17.780 | INFO | Perf report for the last 5.13668 seconds: | 20260505 10:35:17.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:35:17.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:17.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:17.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:17.780 | INFO | ANALYSIS : 63.0%, 3.0, 97.9ms, 0 | 20260505 10:35:17.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:17.780 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:17.780 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:35:17.780 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:17.780 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:17.780 | INFO | Uploaded 36GiB to GPU in 2.15456s (16.7088GiB/s) | 20260505 10:35:17.780 | INFO | Downloaded 34GiB to GPU in 2.07727s (16.3676GiB/s) | 20260505 10:35:22.911 | INFO | Analyzing chunk frames = [2533376,2533888) | 20260505 10:35:22.911 | INFO | MemInfo: { | 20260505 10:35:22.911 | INFO | "gpu" : " 17 GB", | 20260505 10:35:22.911 | INFO | "huge" : "", | 20260505 10:35:22.911 | INFO | "huge_shared" : " 27 GB", | 20260505 10:35:22.911 | INFO | "malloc" : " 14 GB", | 20260505 10:35:22.911 | INFO | "pinned" : " 29 GB" | 20260505 10:35:22.911 | INFO | } | 20260505 10:35:22.911 | INFO | Perf report for the last 5.13133 seconds: | 20260505 10:35:22.911 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:35:22.911 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:22.911 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:22.911 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:22.911 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 10:35:22.911 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:22.911 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:22.911 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:35:22.911 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:22.911 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:22.911 | INFO | Uploaded 36GiB to GPU in 2.20634s (16.3166GiB/s) | 20260505 10:35:22.911 | INFO | Downloaded 34GiB to GPU in 2.10308s (16.1668GiB/s) | 20260505 10:35:28.059 | INFO | Analyzing chunk frames = [2533888,2534400) | 20260505 10:35:28.059 | INFO | MemInfo: { | 20260505 10:35:28.059 | INFO | "gpu" : " 17 GB", | 20260505 10:35:28.059 | INFO | "huge" : "", | 20260505 10:35:28.059 | INFO | "huge_shared" : " 27 GB", | 20260505 10:35:28.059 | INFO | "malloc" : " 14 GB", | 20260505 10:35:28.059 | INFO | "pinned" : " 29 GB" | 20260505 10:35:28.060 | INFO | } | 20260505 10:35:28.060 | INFO | Perf report for the last 5.14864 seconds: | 20260505 10:35:28.060 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:35:28.060 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:28.060 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:28.060 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:28.060 | INFO | ANALYSIS : 63.8%, 3.0, 99.8ms, 0 | 20260505 10:35:28.060 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:28.060 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:28.060 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 10:35:28.060 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:28.060 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:28.060 | INFO | Uploaded 36GiB to GPU in 2.23452s (16.1109GiB/s) | 20260505 10:35:28.060 | INFO | Downloaded 34GiB to GPU in 2.10014s (16.1894GiB/s) | 20260505 10:35:33.197 | INFO | Analyzing chunk frames = [2534400,2534912) | 20260505 10:35:33.197 | INFO | MemInfo: { | 20260505 10:35:33.197 | INFO | "gpu" : " 17 GB", | 20260505 10:35:33.197 | INFO | "huge" : "", | 20260505 10:35:33.197 | INFO | "huge_shared" : " 27 GB", | 20260505 10:35:33.197 | INFO | "malloc" : " 14 GB", | 20260505 10:35:33.197 | INFO | "pinned" : " 29 GB" | 20260505 10:35:33.197 | INFO | } | 20260505 10:35:33.197 | INFO | Perf report for the last 5.13776 seconds: | 20260505 10:35:33.197 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:35:33.197 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:33.197 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:33.197 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:33.198 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 10:35:33.198 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:33.198 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:33.198 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 10:35:33.198 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:33.198 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:33.198 | INFO | Uploaded 36GiB to GPU in 2.19163s (16.4262GiB/s) | 20260505 10:35:33.198 | INFO | Downloaded 34GiB to GPU in 2.13703s (15.9099GiB/s) | 20260505 10:35:38.331 | INFO | Analyzing chunk frames = [2534912,2535424) | 20260505 10:35:38.331 | INFO | MemInfo: { | 20260505 10:35:38.331 | INFO | "gpu" : " 17 GB", | 20260505 10:35:38.331 | INFO | "huge" : "", | 20260505 10:35:38.331 | INFO | "huge_shared" : " 27 GB", | 20260505 10:35:38.331 | INFO | "malloc" : " 14 GB", | 20260505 10:35:38.331 | INFO | "pinned" : " 29 GB" | 20260505 10:35:38.331 | INFO | } | 20260505 10:35:38.332 | INFO | Perf report for the last 5.13424 seconds: | 20260505 10:35:38.332 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:35:38.332 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:38.332 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:38.332 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:38.332 | INFO | ANALYSIS : 64.6%, 3.0, 100.7ms, 0 | 20260505 10:35:38.332 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:38.332 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:38.332 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 10:35:38.332 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:38.332 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:38.332 | INFO | Uploaded 36GiB to GPU in 2.24506s (16.0352GiB/s) | 20260505 10:35:38.332 | INFO | Downloaded 34GiB to GPU in 2.09204s (16.252GiB/s) | 20260505 10:35:38.460 | INFO | Starting a round of Baz IO | 20260505 10:35:41.721 | INFO | Populating RTMetrics report | 20260505 10:35:41.721 | INFO | RTMetrics report populated | 20260505 10:35:41.721 | INFO | Opening tmp RTMetrics json file | 20260505 10:35:41.763 | INFO | Serializing RTMetrics to json | 20260505 10:35:41.789 | INFO | Serialization done, swapping tmp file now... | 20260505 10:35:41.789 | INFO | RTMetrics done | 20260505 10:35:41.857 | INFO | Finished a round of Baz IO: Wrote 2.408 GiB in 3.397 seconds | 20260505 10:35:41.857 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.85% / 0.01% / 75.14% | 20260505 10:35:41.857 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.43859% / 1.19209e-05% / 7.54992e-05% | 20260505 10:35:41.857 | INFO | Baz IO overall: | 20260505 10:35:41.858 | INFO | BytesWritten : 1487788246701 | 20260505 10:35:41.858 | INFO | NumEvents : 708371532412 | 20260505 10:35:43.470 | INFO | Analyzing chunk frames = [2535424,2535936) | 20260505 10:35:43.470 | INFO | MemInfo: { | 20260505 10:35:43.470 | INFO | "gpu" : " 17 GB", | 20260505 10:35:43.470 | INFO | "huge" : "", | 20260505 10:35:43.470 | INFO | "huge_shared" : " 27 GB", | 20260505 10:35:43.470 | INFO | "malloc" : " 14 GB", | 20260505 10:35:43.470 | INFO | "pinned" : " 29 GB" | 20260505 10:35:43.470 | INFO | } | 20260505 10:35:43.470 | INFO | Perf report for the last 5.13864 seconds: | 20260505 10:35:43.470 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:35:43.470 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:43.470 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:43.470 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:43.470 | INFO | ANALYSIS : 64.8%, 2.9, 99.2ms, 0 | 20260505 10:35:43.470 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:43.470 | INFO | RT_METRICS : 46.9%, 1.0, 24.6ms, 0 | 20260505 10:35:43.470 | INFO | PRE_HQ : 62.7%, 1.8, 59.1ms, 0 | 20260505 10:35:43.470 | INFO | BAZWRITER : 65.4%, 4.2, 141.2ms, 0 | 20260505 10:35:43.470 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260505 10:35:43.471 | INFO | Uploaded 36GiB to GPU in 2.2452s (16.0342GiB/s) | 20260505 10:35:43.471 | INFO | Downloaded 38GiB to GPU in 2.35064s (16.1658GiB/s) | 20260505 10:35:48.613 | INFO | Analyzing chunk frames = [2535936,2536448) | 20260505 10:35:48.613 | INFO | MemInfo: { | 20260505 10:35:48.613 | INFO | "gpu" : " 17 GB", | 20260505 10:35:48.613 | INFO | "huge" : "", | 20260505 10:35:48.613 | INFO | "huge_shared" : " 27 GB", | 20260505 10:35:48.613 | INFO | "malloc" : " 14 GB", | 20260505 10:35:48.613 | INFO | "pinned" : " 29 GB" | 20260505 10:35:48.613 | INFO | } | 20260505 10:35:48.613 | INFO | Perf report for the last 5.14273 seconds: | 20260505 10:35:48.613 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:35:48.613 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:48.613 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:48.613 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:48.613 | INFO | ANALYSIS : 64.5%, 3.0, 100.8ms, 0 | 20260505 10:35:48.613 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:48.613 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:48.613 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:35:48.613 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:48.613 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:48.613 | INFO | Uploaded 36GiB to GPU in 2.23325s (16.12GiB/s) | 20260505 10:35:48.613 | INFO | Downloaded 34GiB to GPU in 2.1102s (16.1122GiB/s) | 20260505 10:35:53.756 | INFO | Analyzing chunk frames = [2536448,2536960) | 20260505 10:35:53.756 | INFO | MemInfo: { | 20260505 10:35:53.756 | INFO | "gpu" : " 17 GB", | 20260505 10:35:53.756 | INFO | "huge" : "", | 20260505 10:35:53.756 | INFO | "huge_shared" : " 27 GB", | 20260505 10:35:53.756 | INFO | "malloc" : " 14 GB", | 20260505 10:35:53.756 | INFO | "pinned" : " 29 GB" | 20260505 10:35:53.756 | INFO | } | 20260505 10:35:53.757 | INFO | Perf report for the last 5.14351 seconds: | 20260505 10:35:53.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:35:53.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:53.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:53.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:53.757 | INFO | ANALYSIS : 63.8%, 3.0, 99.3ms, 0 | 20260505 10:35:53.757 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:53.757 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:53.757 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:35:53.757 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:53.757 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:53.757 | INFO | Uploaded 36GiB to GPU in 2.22999s (16.1436GiB/s) | 20260505 10:35:53.757 | INFO | Downloaded 34GiB to GPU in 2.12355s (16.0109GiB/s) | 20260505 10:35:58.896 | INFO | Analyzing chunk frames = [2536960,2537472) | 20260505 10:35:58.896 | INFO | MemInfo: { | 20260505 10:35:58.896 | INFO | "gpu" : " 17 GB", | 20260505 10:35:58.896 | INFO | "huge" : "", | 20260505 10:35:58.896 | INFO | "huge_shared" : " 27 GB", | 20260505 10:35:58.896 | INFO | "malloc" : " 14 GB", | 20260505 10:35:58.896 | INFO | "pinned" : " 29 GB" | 20260505 10:35:58.896 | INFO | } | 20260505 10:35:58.896 | INFO | Perf report for the last 5.13947 seconds: | 20260505 10:35:58.896 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:35:58.896 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:58.896 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:58.896 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:58.896 | INFO | ANALYSIS : 63.7%, 3.0, 99.0ms, 0 | 20260505 10:35:58.896 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:58.896 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:58.896 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:35:58.896 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:58.896 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:35:58.896 | INFO | Uploaded 36GiB to GPU in 2.22004s (16.2159GiB/s) | 20260505 10:35:58.896 | INFO | Downloaded 34GiB to GPU in 2.1279s (15.9782GiB/s) | 20260505 10:36:04.024 | INFO | Analyzing chunk frames = [2537472,2537984) | 20260505 10:36:04.024 | INFO | MemInfo: { | 20260505 10:36:04.024 | INFO | "gpu" : " 17 GB", | 20260505 10:36:04.024 | INFO | "huge" : "", | 20260505 10:36:04.024 | INFO | "huge_shared" : " 27 GB", | 20260505 10:36:04.024 | INFO | "malloc" : " 14 GB", | 20260505 10:36:04.024 | INFO | "pinned" : " 29 GB" | 20260505 10:36:04.024 | INFO | } | 20260505 10:36:04.024 | INFO | Perf report for the last 5.12836 seconds: | 20260505 10:36:04.025 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:36:04.025 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:04.025 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:04.025 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:04.025 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 10:36:04.025 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:04.025 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:04.025 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 10:36:04.025 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:04.025 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:04.025 | INFO | Uploaded 36GiB to GPU in 2.2053s (16.3243GiB/s) | 20260505 10:36:04.025 | INFO | Downloaded 34GiB to GPU in 2.12569s (15.9948GiB/s) | 20260505 10:36:09.161 | INFO | Analyzing chunk frames = [2537984,2538496) | 20260505 10:36:09.161 | INFO | MemInfo: { | 20260505 10:36:09.161 | INFO | "gpu" : " 17 GB", | 20260505 10:36:09.161 | INFO | "huge" : "", | 20260505 10:36:09.161 | INFO | "huge_shared" : " 27 GB", | 20260505 10:36:09.161 | INFO | "malloc" : " 14 GB", | 20260505 10:36:09.161 | INFO | "pinned" : " 29 GB" | 20260505 10:36:09.161 | INFO | } | 20260505 10:36:09.162 | INFO | Perf report for the last 5.13724 seconds: | 20260505 10:36:09.162 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:36:09.162 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:09.162 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:09.162 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:09.162 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 10:36:09.162 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:09.162 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:09.162 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:36:09.162 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:09.162 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:09.162 | INFO | Uploaded 36GiB to GPU in 2.16253s (16.6471GiB/s) | 20260505 10:36:09.162 | INFO | Downloaded 34GiB to GPU in 2.05849s (16.517GiB/s) | 20260505 10:36:14.309 | INFO | Analyzing chunk frames = [2538496,2539008) | 20260505 10:36:14.309 | INFO | MemInfo: { | 20260505 10:36:14.309 | INFO | "gpu" : " 17 GB", | 20260505 10:36:14.309 | INFO | "huge" : "", | 20260505 10:36:14.309 | INFO | "huge_shared" : " 27 GB", | 20260505 10:36:14.309 | INFO | "malloc" : " 14 GB", | 20260505 10:36:14.309 | INFO | "pinned" : " 29 GB" | 20260505 10:36:14.309 | INFO | } | 20260505 10:36:14.309 | INFO | Perf report for the last 5.14693 seconds: | 20260505 10:36:14.309 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:36:14.309 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:14.309 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:14.309 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:14.309 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 10:36:14.309 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:14.309 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:14.309 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260505 10:36:14.309 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:14.309 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:14.309 | INFO | Uploaded 36GiB to GPU in 2.21523s (16.2512GiB/s) | 20260505 10:36:14.309 | INFO | Downloaded 34GiB to GPU in 2.1518s (15.8007GiB/s) | 20260505 10:36:19.455 | INFO | Analyzing chunk frames = [2539008,2539520) | 20260505 10:36:19.455 | INFO | MemInfo: { | 20260505 10:36:19.455 | INFO | "gpu" : " 17 GB", | 20260505 10:36:19.455 | INFO | "huge" : "", | 20260505 10:36:19.455 | INFO | "huge_shared" : " 27 GB", | 20260505 10:36:19.455 | INFO | "malloc" : " 14 GB", | 20260505 10:36:19.455 | INFO | "pinned" : " 29 GB" | 20260505 10:36:19.455 | INFO | } | 20260505 10:36:19.456 | INFO | Perf report for the last 5.14688 seconds: | 20260505 10:36:19.456 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:36:19.456 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:19.456 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:19.456 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:19.456 | INFO | ANALYSIS : 64.0%, 3.0, 100.1ms, 0 | 20260505 10:36:19.456 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:19.456 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:19.456 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 10:36:19.456 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:19.456 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:19.456 | INFO | Uploaded 36GiB to GPU in 2.21846s (16.2275GiB/s) | 20260505 10:36:19.456 | INFO | Downloaded 34GiB to GPU in 2.095s (16.2291GiB/s) | 20260505 10:36:19.456 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 10:36:19.456 | INFO | Total mem usage: 15.3 GiB | 20260505 10:36:19.456 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 10:36:19.456 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -4.78%) | 20260505 10:36:19.456 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 10:36:19.456 | INFO | High Water Breakdown: | 20260505 10:36:19.456 | INFO | 61.2% : 8.95 GiB ( 94.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 10:36:19.456 | INFO | 38.7% : 5.65 GiB ( 85.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 10:36:19.456 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 10:36:19.456 | INFO | ------------------------------------------ | 20260505 10:36:19.456 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:36:19.456 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 10:36:19.456 | INFO | Total mem usage: 26.5 GiB | 20260505 10:36:19.456 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 10:36:19.456 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260505 10:36:19.456 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260505 10:36:19.456 | INFO | High Water Breakdown: | 20260505 10:36:19.456 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:36:19.456 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:36:19.456 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:36:19.456 | INFO | 2% : 525 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:36:19.456 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:36:19.456 | INFO | 0.695% : 182 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:36:19.456 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 10:36:19.456 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 10:36:19.456 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:36:19.456 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 10:36:19.456 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 10:36:19.456 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 10:36:19.456 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 10:36:19.456 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 10:36:19.456 | INFO | ------------------------------------------ | 20260505 10:36:19.456 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:36:19.456 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 10:36:19.456 | INFO | Total mem usage: 25.6 GiB | 20260505 10:36:19.456 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 10:36:19.456 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 10:36:19.457 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 10:36:19.457 | INFO | High Water Breakdown: | 20260505 10:36:19.457 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 10:36:19.457 | INFO | ------------------------------------------ | 20260505 10:36:19.457 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:36:19.457 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 10:36:19.457 | INFO | Total mem usage: 15.7 GiB | 20260505 10:36:19.457 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 10:36:19.457 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 10:36:19.457 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 10:36:19.457 | INFO | High Water Breakdown: | 20260505 10:36:19.457 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 10:36:19.457 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 10:36:19.457 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 10:36:19.457 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 10:36:19.457 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 10:36:19.457 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 10:36:19.457 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 10:36:19.457 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 10:36:19.457 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 10:36:19.457 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:36:19.457 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 10:36:19.457 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:36:19.457 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 10:36:19.457 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 10:36:19.457 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 10:36:19.457 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 10:36:19.457 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:36:19.457 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 10:36:19.457 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:36:19.457 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:36:19.457 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:36:19.457 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 10:36:19.457 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 10:36:19.457 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:36:19.457 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 10:36:19.457 | INFO | ------------------------------------------ | 20260505 10:36:19.457 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:36:19.457 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 10:36:19.457 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:36:19.457 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:36:19.457 | INFO | PerfReport: Full Execution -- 3.23e+01 / 7.73e+01, 1.38e+01 / 2.08e+00 | 20260505 10:36:19.457 | INFO | PerfReport: 0.29%, AggregateStats -- 9.39e-02 / 1.06e-01, 2.49e-02 / 1.31e-02 | 20260505 10:36:19.457 | INFO | PerfReport: 11.67%, Baseline -- 3.77e+00 / 4.41e+00, 6.09e-01 / 2.65e-01 | 20260505 10:36:19.457 | INFO | PerfReport: 3.31%, Histogram -- 1.07e+00 / 1.48e+00, 2.07e-01 / 1.10e-01 | 20260505 10:36:19.457 | INFO | PerfReport: 15.61%, DME -- 4.03e+01 / 4.41e+01, 4.14e+00 / 2.13e+00 | 20260505 10:36:19.457 | INFO | PerfReport: 56.62%, FrameLabeling -- 1.83e+01 / 2.24e+01, 1.79e+00 / 1.98e+00 | 20260505 10:36:19.457 | INFO | PerfReport: 7.39%, PulseAccumulating -- 2.39e+00 / 2.87e+00, 4.20e-01 / 2.17e-01 | 20260505 10:36:19.457 | INFO | PerfReport: 3.76%, Metrics -- 1.22e+00 / 1.51e+00, 3.51e-01 / 1.74e-01 | 20260505 10:36:19.457 | INFO | PerfReport: 1.33%, PulseSerializing -- 4.28e-01 / 4.71e-01, 7.92e-02 / 2.66e-02 | 20260505 10:36:19.457 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 10:36:19.457 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:36:19.457 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:36:19.457 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.42e+01 / 0.00e+00 | 20260505 10:36:19.457 | INFO | PerfReport: 22.74%, Upload -- 2.31e+01 / 0.00e+00, 8.39e+00 / 0.00e+00 | 20260505 10:36:19.457 | INFO | PerfReport: 21.99%, Download -- 2.23e+01 / 0.00e+00, 8.35e+00 / 0.00e+00 | 20260505 10:36:19.457 | INFO | PerfReport: 31.84%, Compute -- 3.23e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 10:36:19.457 | INFO | PerfReport: 1.98%, LockUpload -- 2.01e+00 / 0.00e+00, 7.49e+00 / 0.00e+00 | 20260505 10:36:19.457 | INFO | PerfReport: 1.17%, LockDownload -- 1.19e+00 / 0.00e+00, 2.79e+00 / 0.00e+00 | 20260505 10:36:19.457 | INFO | PerfReport: 20.26%, LockCompute -- 2.05e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260505 10:36:19.457 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.48e-02 / 0.00e+00 | 20260505 10:36:19.595 | INFO | Starting a round of Baz IO | 20260505 10:36:22.852 | INFO | Populating RTMetrics report | 20260505 10:36:22.852 | INFO | RTMetrics report populated | 20260505 10:36:22.852 | INFO | Opening tmp RTMetrics json file | 20260505 10:36:22.855 | INFO | Serializing RTMetrics to json | 20260505 10:36:22.856 | INFO | Serialization done, swapping tmp file now... | 20260505 10:36:22.857 | INFO | RTMetrics done | 20260505 10:36:23.021 | INFO | Finished a round of Baz IO: Wrote 2.407 GiB in 3.426 seconds | 20260505 10:36:23.021 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.84% / 0.01% / 75.15% | 20260505 10:36:23.021 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.43329% / 2.78155e-05% / 0.000127157% | 20260505 10:36:23.021 | INFO | Baz IO overall: | 20260505 10:36:23.021 | INFO | BytesWritten : 1490372983737 | 20260505 10:36:23.021 | INFO | NumEvents : 709603130469 | 20260505 10:36:24.597 | INFO | Analyzing chunk frames = [2539520,2540032) | 20260505 10:36:24.597 | INFO | MemInfo: { | 20260505 10:36:24.597 | INFO | "gpu" : " 17 GB", | 20260505 10:36:24.597 | INFO | "huge" : "", | 20260505 10:36:24.597 | INFO | "huge_shared" : " 27 GB", | 20260505 10:36:24.597 | INFO | "malloc" : " 14 GB", | 20260505 10:36:24.597 | INFO | "pinned" : " 29 GB" | 20260505 10:36:24.597 | INFO | } | 20260505 10:36:24.597 | INFO | Perf report for the last 5.14155 seconds: | 20260505 10:36:24.597 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:36:24.597 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:24.597 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:24.597 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:24.597 | INFO | ANALYSIS : 64.9%, 2.9, 99.5ms, 0 | 20260505 10:36:24.597 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:24.597 | INFO | RT_METRICS : 45.8%, 1.0, 24.1ms, 0 | 20260505 10:36:24.597 | INFO | PRE_HQ : 61.7%, 1.9, 59.7ms, 0 | 20260505 10:36:24.597 | INFO | BAZWRITER : 66.0%, 4.2, 142.3ms, 0 | 20260505 10:36:24.597 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 10:36:24.597 | INFO | Uploaded 36GiB to GPU in 2.21257s (16.2707GiB/s) | 20260505 10:36:24.598 | INFO | Downloaded 38GiB to GPU in 2.31463s (16.4173GiB/s) | 20260505 10:36:29.726 | INFO | Analyzing chunk frames = [2540032,2540544) | 20260505 10:36:29.726 | INFO | MemInfo: { | 20260505 10:36:29.726 | INFO | "gpu" : " 17 GB", | 20260505 10:36:29.726 | INFO | "huge" : "", | 20260505 10:36:29.726 | INFO | "huge_shared" : " 27 GB", | 20260505 10:36:29.726 | INFO | "malloc" : " 14 GB", | 20260505 10:36:29.726 | INFO | "pinned" : " 29 GB" | 20260505 10:36:29.726 | INFO | } | 20260505 10:36:29.726 | INFO | Perf report for the last 5.12882 seconds: | 20260505 10:36:29.726 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:36:29.726 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:29.726 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:29.726 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:29.726 | INFO | ANALYSIS : 64.3%, 3.0, 100.2ms, 0 | 20260505 10:36:29.726 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:29.726 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:29.726 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:36:29.726 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:29.726 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:29.726 | INFO | Uploaded 36GiB to GPU in 2.22938s (16.148GiB/s) | 20260505 10:36:29.726 | INFO | Downloaded 34GiB to GPU in 2.08224s (16.3286GiB/s) | 20260505 10:36:34.862 | INFO | Analyzing chunk frames = [2540544,2541056) | 20260505 10:36:34.862 | INFO | MemInfo: { | 20260505 10:36:34.862 | INFO | "gpu" : " 17 GB", | 20260505 10:36:34.862 | INFO | "huge" : "", | 20260505 10:36:34.862 | INFO | "huge_shared" : " 27 GB", | 20260505 10:36:34.862 | INFO | "malloc" : " 14 GB", | 20260505 10:36:34.862 | INFO | "pinned" : " 29 GB" | 20260505 10:36:34.862 | INFO | } | 20260505 10:36:34.862 | INFO | Perf report for the last 5.13629 seconds: | 20260505 10:36:34.862 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:36:34.862 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:34.862 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:34.862 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:34.862 | INFO | ANALYSIS : 63.0%, 3.0, 98.0ms, 0 | 20260505 10:36:34.862 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:34.862 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:34.862 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 10:36:34.862 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:34.862 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:34.863 | INFO | Uploaded 36GiB to GPU in 2.15856s (16.6778GiB/s) | 20260505 10:36:34.863 | INFO | Downloaded 34GiB to GPU in 2.06467s (16.4675GiB/s) | 20260505 10:36:40.005 | INFO | Analyzing chunk frames = [2541056,2541568) | 20260505 10:36:40.005 | INFO | MemInfo: { | 20260505 10:36:40.005 | INFO | "gpu" : " 17 GB", | 20260505 10:36:40.005 | INFO | "huge" : "", | 20260505 10:36:40.005 | INFO | "huge_shared" : " 27 GB", | 20260505 10:36:40.005 | INFO | "malloc" : " 14 GB", | 20260505 10:36:40.005 | INFO | "pinned" : " 29 GB" | 20260505 10:36:40.005 | INFO | } | 20260505 10:36:40.006 | INFO | Perf report for the last 5.14328 seconds: | 20260505 10:36:40.006 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:36:40.006 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:40.006 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:40.006 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:40.006 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260505 10:36:40.006 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:40.006 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:40.006 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:36:40.006 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:40.006 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:40.006 | INFO | Uploaded 36GiB to GPU in 2.14685s (16.7687GiB/s) | 20260505 10:36:40.006 | INFO | Downloaded 34GiB to GPU in 2.07132s (16.4146GiB/s) | 20260505 10:36:45.152 | INFO | Analyzing chunk frames = [2541568,2542080) | 20260505 10:36:45.152 | INFO | MemInfo: { | 20260505 10:36:45.152 | INFO | "gpu" : " 17 GB", | 20260505 10:36:45.152 | INFO | "huge" : "", | 20260505 10:36:45.152 | INFO | "huge_shared" : " 27 GB", | 20260505 10:36:45.152 | INFO | "malloc" : " 14 GB", | 20260505 10:36:45.152 | INFO | "pinned" : " 29 GB" | 20260505 10:36:45.152 | INFO | } | 20260505 10:36:45.152 | INFO | Perf report for the last 5.14678 seconds: | 20260505 10:36:45.152 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:36:45.152 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:45.152 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:45.152 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:45.152 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 10:36:45.152 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:45.152 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:45.152 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:36:45.152 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:45.152 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:45.153 | INFO | Uploaded 36GiB to GPU in 2.16922s (16.5959GiB/s) | 20260505 10:36:45.153 | INFO | Downloaded 34GiB to GPU in 2.12939s (15.967GiB/s) | 20260505 10:36:50.285 | INFO | Analyzing chunk frames = [2542080,2542592) | 20260505 10:36:50.285 | INFO | MemInfo: { | 20260505 10:36:50.285 | INFO | "gpu" : " 17 GB", | 20260505 10:36:50.285 | INFO | "huge" : "", | 20260505 10:36:50.285 | INFO | "huge_shared" : " 27 GB", | 20260505 10:36:50.285 | INFO | "malloc" : " 14 GB", | 20260505 10:36:50.285 | INFO | "pinned" : " 29 GB" | 20260505 10:36:50.285 | INFO | } | 20260505 10:36:50.285 | INFO | Perf report for the last 5.13242 seconds: | 20260505 10:36:50.285 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:36:50.285 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:50.285 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:50.285 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:50.285 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 10:36:50.285 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:50.285 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:50.285 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:36:50.285 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:50.285 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:50.285 | INFO | Uploaded 36GiB to GPU in 2.21851s (16.2271GiB/s) | 20260505 10:36:50.285 | INFO | Downloaded 34GiB to GPU in 2.10187s (16.1761GiB/s) | 20260505 10:36:55.419 | INFO | Analyzing chunk frames = [2542592,2543104) | 20260505 10:36:55.419 | INFO | MemInfo: { | 20260505 10:36:55.419 | INFO | "gpu" : " 17 GB", | 20260505 10:36:55.419 | INFO | "huge" : "", | 20260505 10:36:55.419 | INFO | "huge_shared" : " 27 GB", | 20260505 10:36:55.419 | INFO | "malloc" : " 14 GB", | 20260505 10:36:55.419 | INFO | "pinned" : " 29 GB" | 20260505 10:36:55.419 | INFO | } | 20260505 10:36:55.419 | INFO | Perf report for the last 5.13447 seconds: | 20260505 10:36:55.419 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:36:55.419 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:55.419 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:55.419 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:55.419 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 10:36:55.419 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:55.419 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:55.419 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 10:36:55.419 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:55.419 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:36:55.420 | INFO | Uploaded 36GiB to GPU in 2.20749s (16.3081GiB/s) | 20260505 10:36:55.420 | INFO | Downloaded 34GiB to GPU in 2.14973s (15.816GiB/s) | 20260505 10:37:00.569 | INFO | Analyzing chunk frames = [2543104,2543616) | 20260505 10:37:00.569 | INFO | MemInfo: { | 20260505 10:37:00.569 | INFO | "gpu" : " 17 GB", | 20260505 10:37:00.569 | INFO | "huge" : "", | 20260505 10:37:00.569 | INFO | "huge_shared" : " 27 GB", | 20260505 10:37:00.569 | INFO | "malloc" : " 14 GB", | 20260505 10:37:00.569 | INFO | "pinned" : " 29 GB" | 20260505 10:37:00.569 | INFO | } | 20260505 10:37:00.569 | INFO | Perf report for the last 5.15011 seconds: | 20260505 10:37:00.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:37:00.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:00.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:00.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:00.569 | INFO | ANALYSIS : 64.4%, 3.0, 100.7ms, 0 | 20260505 10:37:00.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:00.569 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:00.570 | INFO | PRE_HQ : 29.5%, 1.0, 15.4ms, 0 | 20260505 10:37:00.570 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:00.570 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:00.570 | INFO | Uploaded 36GiB to GPU in 2.24776s (16.0159GiB/s) | 20260505 10:37:00.570 | INFO | Downloaded 34GiB to GPU in 2.09841s (16.2027GiB/s) | 20260505 10:37:00.706 | INFO | Starting a round of Baz IO | 20260505 10:37:03.965 | INFO | Populating RTMetrics report | 20260505 10:37:03.965 | INFO | RTMetrics report populated | 20260505 10:37:03.965 | INFO | Opening tmp RTMetrics json file | 20260505 10:37:04.004 | INFO | Serializing RTMetrics to json | 20260505 10:37:04.038 | INFO | Serialization done, swapping tmp file now... | 20260505 10:37:04.051 | INFO | RTMetrics done | 20260505 10:37:04.089 | INFO | Finished a round of Baz IO: Wrote 2.408 GiB in 3.383 seconds | 20260505 10:37:04.089 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.83% / 0.01% / 75.16% | 20260505 10:37:04.089 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.43019% / 1.58946e-05% / 9.93411e-05% | 20260505 10:37:04.089 | INFO | Baz IO overall: | 20260505 10:37:04.089 | INFO | BytesWritten : 1492958515791 | 20260505 10:37:04.089 | INFO | NumEvents : 710835122322 | 20260505 10:37:05.708 | INFO | Analyzing chunk frames = [2543616,2544128) | 20260505 10:37:05.708 | INFO | MemInfo: { | 20260505 10:37:05.708 | INFO | "gpu" : " 17 GB", | 20260505 10:37:05.708 | INFO | "huge" : "", | 20260505 10:37:05.708 | INFO | "huge_shared" : " 27 GB", | 20260505 10:37:05.708 | INFO | "malloc" : " 14 GB", | 20260505 10:37:05.708 | INFO | "pinned" : " 29 GB" | 20260505 10:37:05.708 | INFO | } | 20260505 10:37:05.708 | INFO | Perf report for the last 5.139 seconds: | 20260505 10:37:05.708 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:37:05.708 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:05.708 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:05.708 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:05.708 | INFO | ANALYSIS : 64.9%, 2.9, 99.8ms, 0 | 20260505 10:37:05.708 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:05.708 | INFO | RT_METRICS : 47.1%, 1.0, 24.7ms, 0 | 20260505 10:37:05.709 | INFO | PRE_HQ : 61.4%, 1.9, 59.3ms, 0 | 20260505 10:37:05.709 | INFO | BAZWRITER : 65.0%, 4.2, 139.8ms, 0 | 20260505 10:37:05.709 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260505 10:37:05.709 | INFO | Uploaded 36GiB to GPU in 2.21501s (16.2528GiB/s) | 20260505 10:37:05.709 | INFO | Downloaded 38GiB to GPU in 2.29898s (16.5291GiB/s) | 20260505 10:37:10.841 | INFO | Analyzing chunk frames = [2544128,2544640) | 20260505 10:37:10.841 | INFO | MemInfo: { | 20260505 10:37:10.841 | INFO | "gpu" : " 17 GB", | 20260505 10:37:10.841 | INFO | "huge" : "", | 20260505 10:37:10.841 | INFO | "huge_shared" : " 27 GB", | 20260505 10:37:10.841 | INFO | "malloc" : " 14 GB", | 20260505 10:37:10.841 | INFO | "pinned" : " 29 GB" | 20260505 10:37:10.841 | INFO | } | 20260505 10:37:10.841 | INFO | Perf report for the last 5.133 seconds: | 20260505 10:37:10.841 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:37:10.841 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:10.841 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:10.841 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:10.841 | INFO | ANALYSIS : 64.5%, 3.0, 100.5ms, 0 | 20260505 10:37:10.841 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:10.841 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:10.842 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:37:10.842 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:10.842 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:10.842 | INFO | Uploaded 36GiB to GPU in 2.23861s (16.0814GiB/s) | 20260505 10:37:10.842 | INFO | Downloaded 34GiB to GPU in 2.08089s (16.3392GiB/s) | 20260505 10:37:15.974 | INFO | Analyzing chunk frames = [2544640,2545152) | 20260505 10:37:15.974 | INFO | MemInfo: { | 20260505 10:37:15.974 | INFO | "gpu" : " 17 GB", | 20260505 10:37:15.974 | INFO | "huge" : "", | 20260505 10:37:15.974 | INFO | "huge_shared" : " 27 GB", | 20260505 10:37:15.974 | INFO | "malloc" : " 14 GB", | 20260505 10:37:15.974 | INFO | "pinned" : " 29 GB" | 20260505 10:37:15.974 | INFO | } | 20260505 10:37:15.974 | INFO | Perf report for the last 5.13228 seconds: | 20260505 10:37:15.974 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:37:15.974 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:15.974 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:15.974 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:15.974 | INFO | ANALYSIS : 63.1%, 3.0, 98.1ms, 0 | 20260505 10:37:15.974 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:15.974 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:15.974 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 10:37:15.974 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:15.974 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:15.974 | INFO | Uploaded 36GiB to GPU in 2.15602s (16.6975GiB/s) | 20260505 10:37:15.974 | INFO | Downloaded 34GiB to GPU in 2.06298s (16.481GiB/s) | 20260505 10:37:21.116 | INFO | Analyzing chunk frames = [2545152,2545664) | 20260505 10:37:21.116 | INFO | MemInfo: { | 20260505 10:37:21.116 | INFO | "gpu" : " 17 GB", | 20260505 10:37:21.116 | INFO | "huge" : "", | 20260505 10:37:21.116 | INFO | "huge_shared" : " 27 GB", | 20260505 10:37:21.117 | INFO | "malloc" : " 14 GB", | 20260505 10:37:21.117 | INFO | "pinned" : " 29 GB" | 20260505 10:37:21.117 | INFO | } | 20260505 10:37:21.117 | INFO | Perf report for the last 5.143 seconds: | 20260505 10:37:21.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:37:21.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:21.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:21.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:21.117 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 10:37:21.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:21.117 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:21.117 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 10:37:21.117 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:21.117 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:21.117 | INFO | Uploaded 36GiB to GPU in 2.16853s (16.6011GiB/s) | 20260505 10:37:21.117 | INFO | Downloaded 34GiB to GPU in 2.07459s (16.3888GiB/s) | 20260505 10:37:26.259 | INFO | Analyzing chunk frames = [2545664,2546176) | 20260505 10:37:26.259 | INFO | MemInfo: { | 20260505 10:37:26.259 | INFO | "gpu" : " 17 GB", | 20260505 10:37:26.259 | INFO | "huge" : "", | 20260505 10:37:26.259 | INFO | "huge_shared" : " 27 GB", | 20260505 10:37:26.259 | INFO | "malloc" : " 14 GB", | 20260505 10:37:26.259 | INFO | "pinned" : " 29 GB" | 20260505 10:37:26.259 | INFO | } | 20260505 10:37:26.259 | INFO | Perf report for the last 5.14233 seconds: | 20260505 10:37:26.259 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:37:26.259 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:26.259 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:26.259 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:26.259 | INFO | ANALYSIS : 63.3%, 3.0, 99.0ms, 0 | 20260505 10:37:26.259 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:26.259 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:26.259 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:37:26.259 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:26.259 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:26.259 | INFO | Uploaded 36GiB to GPU in 2.16167s (16.6538GiB/s) | 20260505 10:37:26.259 | INFO | Downloaded 34GiB to GPU in 2.1282s (15.976GiB/s) | 20260505 10:37:31.411 | INFO | Analyzing chunk frames = [2546176,2546688) | 20260505 10:37:31.411 | INFO | MemInfo: { | 20260505 10:37:31.411 | INFO | "gpu" : " 17 GB", | 20260505 10:37:31.411 | INFO | "huge" : "", | 20260505 10:37:31.411 | INFO | "huge_shared" : " 27 GB", | 20260505 10:37:31.411 | INFO | "malloc" : " 14 GB", | 20260505 10:37:31.411 | INFO | "pinned" : " 29 GB" | 20260505 10:37:31.411 | INFO | } | 20260505 10:37:31.411 | INFO | Perf report for the last 5.15214 seconds: | 20260505 10:37:31.411 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:37:31.411 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:31.411 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:31.411 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:31.411 | INFO | ANALYSIS : 63.4%, 3.0, 99.3ms, 0 | 20260505 10:37:31.411 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:31.411 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:31.411 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 10:37:31.411 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:31.411 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:31.411 | INFO | Uploaded 36GiB to GPU in 2.21061s (16.2851GiB/s) | 20260505 10:37:31.411 | INFO | Downloaded 34GiB to GPU in 2.10172s (16.1772GiB/s) | 20260505 10:37:36.545 | INFO | Analyzing chunk frames = [2546688,2547200) | 20260505 10:37:36.545 | INFO | MemInfo: { | 20260505 10:37:36.545 | INFO | "gpu" : " 17 GB", | 20260505 10:37:36.545 | INFO | "huge" : "", | 20260505 10:37:36.545 | INFO | "huge_shared" : " 27 GB", | 20260505 10:37:36.545 | INFO | "malloc" : " 14 GB", | 20260505 10:37:36.545 | INFO | "pinned" : " 29 GB" | 20260505 10:37:36.545 | INFO | } | 20260505 10:37:36.545 | INFO | Perf report for the last 5.134 seconds: | 20260505 10:37:36.545 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:37:36.545 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:36.545 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:36.545 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:36.545 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 10:37:36.545 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:36.545 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:36.545 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 10:37:36.545 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:36.545 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:36.546 | INFO | Uploaded 36GiB to GPU in 2.17088s (16.5831GiB/s) | 20260505 10:37:36.546 | INFO | Downloaded 34GiB to GPU in 2.15532s (15.7749GiB/s) | 20260505 10:37:41.678 | INFO | Analyzing chunk frames = [2547200,2547712) | 20260505 10:37:41.678 | INFO | MemInfo: { | 20260505 10:37:41.678 | INFO | "gpu" : " 17 GB", | 20260505 10:37:41.678 | INFO | "huge" : "", | 20260505 10:37:41.678 | INFO | "huge_shared" : " 27 GB", | 20260505 10:37:41.678 | INFO | "malloc" : " 14 GB", | 20260505 10:37:41.679 | INFO | "pinned" : " 29 GB" | 20260505 10:37:41.679 | INFO | } | 20260505 10:37:41.679 | INFO | Perf report for the last 5.13334 seconds: | 20260505 10:37:41.679 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:37:41.679 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:41.679 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:41.679 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:41.679 | INFO | ANALYSIS : 64.5%, 3.0, 100.6ms, 0 | 20260505 10:37:41.679 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:41.679 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:41.679 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 10:37:41.679 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:41.679 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:41.679 | INFO | Uploaded 36GiB to GPU in 2.24853s (16.0105GiB/s) | 20260505 10:37:41.679 | INFO | Downloaded 34GiB to GPU in 2.0892s (16.2742GiB/s) | 20260505 10:37:41.679 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 10:37:41.679 | INFO | Total mem usage: 15.3 GiB | 20260505 10:37:41.679 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 10:37:41.679 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.52%) | 20260505 10:37:41.679 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 10:37:41.679 | INFO | High Water Breakdown: | 20260505 10:37:41.679 | INFO | 61% : 8.78 GiB ( 92.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 10:37:41.679 | INFO | 38.9% : 5.59 GiB ( 84.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 10:37:41.679 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 10:37:41.679 | INFO | ------------------------------------------ | 20260505 10:37:41.679 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:37:41.679 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 10:37:41.679 | INFO | Total mem usage: 26.5 GiB | 20260505 10:37:41.679 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 10:37:41.679 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 10:37:41.679 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 10:37:41.679 | INFO | High Water Breakdown: | 20260505 10:37:41.679 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:37:41.679 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:37:41.679 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:37:41.679 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:37:41.679 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:37:41.679 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:37:41.679 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 10:37:41.679 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 10:37:41.679 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:37:41.679 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 10:37:41.679 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 10:37:41.679 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 10:37:41.679 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 10:37:41.679 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 10:37:41.679 | INFO | ------------------------------------------ | 20260505 10:37:41.679 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:37:41.679 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 10:37:41.679 | INFO | Total mem usage: 25.6 GiB | 20260505 10:37:41.679 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 10:37:41.679 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 10:37:41.680 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 10:37:41.680 | INFO | High Water Breakdown: | 20260505 10:37:41.680 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 10:37:41.680 | INFO | ------------------------------------------ | 20260505 10:37:41.680 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:37:41.680 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 10:37:41.680 | INFO | Total mem usage: 15.7 GiB | 20260505 10:37:41.680 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 10:37:41.680 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 10:37:41.680 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 10:37:41.680 | INFO | High Water Breakdown: | 20260505 10:37:41.680 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 10:37:41.680 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 10:37:41.680 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 10:37:41.680 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 10:37:41.680 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 10:37:41.680 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 10:37:41.680 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 10:37:41.680 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 10:37:41.680 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 10:37:41.680 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:37:41.680 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 10:37:41.680 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:37:41.680 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 10:37:41.680 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 10:37:41.680 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 10:37:41.680 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 10:37:41.680 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:37:41.680 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 10:37:41.680 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:37:41.680 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:37:41.680 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:37:41.680 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 10:37:41.680 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 10:37:41.680 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:37:41.680 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 10:37:41.680 | INFO | ------------------------------------------ | 20260505 10:37:41.680 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:37:41.680 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 10:37:41.680 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:37:41.680 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:37:41.680 | INFO | PerfReport: Full Execution -- 3.23e+01 / 7.72e+01, 1.38e+01 / 1.83e+00 | 20260505 10:37:41.680 | INFO | PerfReport: 0.29%, AggregateStats -- 9.30e-02 / 1.01e-01, 1.02e-02 / 3.11e-03 | 20260505 10:37:41.680 | INFO | PerfReport: 11.68%, Baseline -- 3.78e+00 / 4.51e+00, 6.02e-01 / 1.85e-01 | 20260505 10:37:41.680 | INFO | PerfReport: 3.31%, Histogram -- 1.07e+00 / 1.51e+00, 2.06e-01 / 8.17e-02 | 20260505 10:37:41.680 | INFO | PerfReport: 15.78%, DME -- 4.08e+01 / 4.54e+01, 4.43e+00 / 1.65e+00 | 20260505 10:37:41.680 | INFO | PerfReport: 56.48%, FrameLabeling -- 1.83e+01 / 2.08e+01, 1.75e+00 / 1.90e+00 | 20260505 10:37:41.680 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.39e+00 / 2.89e+00, 4.15e-01 / 2.33e-01 | 20260505 10:37:41.680 | INFO | PerfReport: 3.74%, Metrics -- 1.21e+00 / 1.49e+00, 3.46e-01 / 1.60e-01 | 20260505 10:37:41.680 | INFO | PerfReport: 1.32%, PulseSerializing -- 4.28e-01 / 4.79e-01, 7.88e-02 / 4.29e-02 | 20260505 10:37:41.680 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 10:37:41.680 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:37:41.680 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:37:41.680 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.45e+01 / 0.00e+00 | 20260505 10:37:41.680 | INFO | PerfReport: 22.58%, Upload -- 2.29e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260505 10:37:41.680 | INFO | PerfReport: 21.84%, Download -- 2.21e+01 / 0.00e+00, 8.08e+00 / 0.00e+00 | 20260505 10:37:41.680 | INFO | PerfReport: 31.89%, Compute -- 3.23e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 10:37:41.680 | INFO | PerfReport: 2.01%, LockUpload -- 2.03e+00 / 0.00e+00, 7.62e+00 / 0.00e+00 | 20260505 10:37:41.680 | INFO | PerfReport: 1.15%, LockDownload -- 1.16e+00 / 0.00e+00, 2.69e+00 / 0.00e+00 | 20260505 10:37:41.680 | INFO | PerfReport: 20.52%, LockCompute -- 2.08e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260505 10:37:41.680 | INFO | PerfReport: 0.01%, Misc -- 1.24e-02 / 0.00e+00, 1.29e-02 / 0.00e+00 | 20260505 10:37:41.820 | INFO | Starting a round of Baz IO | 20260505 10:37:45.081 | INFO | Populating RTMetrics report | 20260505 10:37:45.081 | INFO | RTMetrics report populated | 20260505 10:37:45.081 | INFO | Opening tmp RTMetrics json file | 20260505 10:37:45.090 | INFO | Serializing RTMetrics to json | 20260505 10:37:45.092 | INFO | Serialization done, swapping tmp file now... | 20260505 10:37:45.095 | INFO | RTMetrics done | 20260505 10:37:45.216 | INFO | Finished a round of Baz IO: Wrote 2.408 GiB in 3.396 seconds | 20260505 10:37:45.216 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.82% / 0.01% / 75.17% | 20260505 10:37:45.216 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.42927% / 1.19209e-05% / 0.000115236% | 20260505 10:37:45.216 | INFO | Baz IO overall: | 20260505 10:37:45.216 | INFO | BytesWritten : 1495544027834 | 20260505 10:37:45.216 | INFO | NumEvents : 712067228548 | 20260505 10:37:46.816 | INFO | Analyzing chunk frames = [2547712,2548224) | 20260505 10:37:46.816 | INFO | MemInfo: { | 20260505 10:37:46.816 | INFO | "gpu" : " 17 GB", | 20260505 10:37:46.816 | INFO | "huge" : "", | 20260505 10:37:46.816 | INFO | "huge_shared" : " 27 GB", | 20260505 10:37:46.816 | INFO | "malloc" : " 14 GB", | 20260505 10:37:46.816 | INFO | "pinned" : " 29 GB" | 20260505 10:37:46.816 | INFO | } | 20260505 10:37:46.816 | INFO | Perf report for the last 5.13741 seconds: | 20260505 10:37:46.816 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:37:46.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:46.816 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:46.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:46.816 | INFO | ANALYSIS : 65.1%, 2.9, 99.8ms, 0 | 20260505 10:37:46.816 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:46.816 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260505 10:37:46.816 | INFO | PRE_HQ : 61.6%, 1.9, 59.3ms, 0 | 20260505 10:37:46.816 | INFO | BAZWRITER : 65.4%, 4.2, 140.2ms, 0 | 20260505 10:37:46.816 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 10:37:46.816 | INFO | Uploaded 36GiB to GPU in 2.21493s (16.2533GiB/s) | 20260505 10:37:46.816 | INFO | Downloaded 38GiB to GPU in 2.30936s (16.4548GiB/s) | 20260505 10:37:51.956 | INFO | Analyzing chunk frames = [2548224,2548736) | 20260505 10:37:51.956 | INFO | MemInfo: { | 20260505 10:37:51.956 | INFO | "gpu" : " 17 GB", | 20260505 10:37:51.956 | INFO | "huge" : "", | 20260505 10:37:51.956 | INFO | "huge_shared" : " 27 GB", | 20260505 10:37:51.956 | INFO | "malloc" : " 14 GB", | 20260505 10:37:51.956 | INFO | "pinned" : " 29 GB" | 20260505 10:37:51.956 | INFO | } | 20260505 10:37:51.957 | INFO | Perf report for the last 5.14053 seconds: | 20260505 10:37:51.957 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:37:51.957 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:51.957 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:51.957 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:51.957 | INFO | ANALYSIS : 64.3%, 3.0, 100.5ms, 0 | 20260505 10:37:51.957 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:51.957 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:51.957 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:37:51.957 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:51.957 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:51.957 | INFO | Uploaded 36GiB to GPU in 2.23873s (16.0805GiB/s) | 20260505 10:37:51.957 | INFO | Downloaded 34GiB to GPU in 2.0822s (16.3289GiB/s) | 20260505 10:37:57.099 | INFO | Analyzing chunk frames = [2548736,2549248) | 20260505 10:37:57.099 | INFO | MemInfo: { | 20260505 10:37:57.099 | INFO | "gpu" : " 17 GB", | 20260505 10:37:57.099 | INFO | "huge" : "", | 20260505 10:37:57.099 | INFO | "huge_shared" : " 27 GB", | 20260505 10:37:57.099 | INFO | "malloc" : " 14 GB", | 20260505 10:37:57.099 | INFO | "pinned" : " 29 GB" | 20260505 10:37:57.099 | INFO | } | 20260505 10:37:57.099 | INFO | Perf report for the last 5.14267 seconds: | 20260505 10:37:57.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:37:57.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:57.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:57.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:57.099 | INFO | ANALYSIS : 62.9%, 3.0, 97.9ms, 0 | 20260505 10:37:57.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:57.099 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:57.099 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 10:37:57.099 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:57.099 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:37:57.099 | INFO | Uploaded 36GiB to GPU in 2.15399s (16.7132GiB/s) | 20260505 10:37:57.099 | INFO | Downloaded 34GiB to GPU in 2.05577s (16.5388GiB/s) | 20260505 10:38:02.231 | INFO | Analyzing chunk frames = [2549248,2549760) | 20260505 10:38:02.231 | INFO | MemInfo: { | 20260505 10:38:02.231 | INFO | "gpu" : " 17 GB", | 20260505 10:38:02.231 | INFO | "huge" : "", | 20260505 10:38:02.231 | INFO | "huge_shared" : " 27 GB", | 20260505 10:38:02.231 | INFO | "malloc" : " 14 GB", | 20260505 10:38:02.231 | INFO | "pinned" : " 29 GB" | 20260505 10:38:02.231 | INFO | } | 20260505 10:38:02.231 | INFO | Perf report for the last 5.13202 seconds: | 20260505 10:38:02.231 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:38:02.231 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:02.231 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:02.231 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:02.231 | INFO | ANALYSIS : 63.1%, 3.0, 97.9ms, 0 | 20260505 10:38:02.231 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:02.231 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:02.231 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 10:38:02.231 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:02.231 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:02.231 | INFO | Uploaded 36GiB to GPU in 2.16713s (16.6118GiB/s) | 20260505 10:38:02.232 | INFO | Downloaded 34GiB to GPU in 2.0779s (16.3627GiB/s) | 20260505 10:38:07.377 | INFO | Analyzing chunk frames = [2549760,2550272) | 20260505 10:38:07.377 | INFO | MemInfo: { | 20260505 10:38:07.377 | INFO | "gpu" : " 17 GB", | 20260505 10:38:07.377 | INFO | "huge" : "", | 20260505 10:38:07.377 | INFO | "huge_shared" : " 27 GB", | 20260505 10:38:07.377 | INFO | "malloc" : " 14 GB", | 20260505 10:38:07.377 | INFO | "pinned" : " 29 GB" | 20260505 10:38:07.377 | INFO | } | 20260505 10:38:07.377 | INFO | Perf report for the last 5.14594 seconds: | 20260505 10:38:07.377 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:38:07.377 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:07.377 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:07.377 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:07.377 | INFO | ANALYSIS : 63.6%, 3.0, 99.5ms, 0 | 20260505 10:38:07.377 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:07.377 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:07.377 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:38:07.377 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:07.377 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:07.377 | INFO | Uploaded 36GiB to GPU in 2.23597s (16.1004GiB/s) | 20260505 10:38:07.378 | INFO | Downloaded 34GiB to GPU in 2.11242s (16.0953GiB/s) | 20260505 10:38:12.527 | INFO | Analyzing chunk frames = [2550272,2550784) | 20260505 10:38:12.527 | INFO | MemInfo: { | 20260505 10:38:12.527 | INFO | "gpu" : " 17 GB", | 20260505 10:38:12.527 | INFO | "huge" : "", | 20260505 10:38:12.527 | INFO | "huge_shared" : " 27 GB", | 20260505 10:38:12.527 | INFO | "malloc" : " 14 GB", | 20260505 10:38:12.527 | INFO | "pinned" : " 29 GB" | 20260505 10:38:12.527 | INFO | } | 20260505 10:38:12.527 | INFO | Perf report for the last 5.14988 seconds: | 20260505 10:38:12.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:38:12.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:12.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:12.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:12.527 | INFO | ANALYSIS : 63.4%, 3.0, 99.3ms, 0 | 20260505 10:38:12.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:12.527 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:12.527 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 10:38:12.527 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:12.527 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:12.527 | INFO | Uploaded 36GiB to GPU in 2.16539s (16.6252GiB/s) | 20260505 10:38:12.527 | INFO | Downloaded 34GiB to GPU in 2.1237s (16.0098GiB/s) | 20260505 10:38:17.657 | INFO | Analyzing chunk frames = [2550784,2551296) | 20260505 10:38:17.657 | INFO | MemInfo: { | 20260505 10:38:17.657 | INFO | "gpu" : " 17 GB", | 20260505 10:38:17.657 | INFO | "huge" : "", | 20260505 10:38:17.657 | INFO | "huge_shared" : " 27 GB", | 20260505 10:38:17.657 | INFO | "malloc" : " 14 GB", | 20260505 10:38:17.657 | INFO | "pinned" : " 29 GB" | 20260505 10:38:17.657 | INFO | } | 20260505 10:38:17.657 | INFO | Perf report for the last 5.12983 seconds: | 20260505 10:38:17.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:38:17.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:17.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:17.657 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:17.657 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 10:38:17.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:17.657 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:17.657 | INFO | PRE_HQ : 30.6%, 1.0, 15.8ms, 0 | 20260505 10:38:17.657 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:17.657 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:17.657 | INFO | Uploaded 36GiB to GPU in 2.17521s (16.5501GiB/s) | 20260505 10:38:17.657 | INFO | Downloaded 34GiB to GPU in 2.15376s (15.7864GiB/s) | 20260505 10:38:22.784 | INFO | Analyzing chunk frames = [2551296,2551808) | 20260505 10:38:22.784 | INFO | MemInfo: { | 20260505 10:38:22.784 | INFO | "gpu" : " 17 GB", | 20260505 10:38:22.784 | INFO | "huge" : "", | 20260505 10:38:22.784 | INFO | "huge_shared" : " 27 GB", | 20260505 10:38:22.784 | INFO | "malloc" : " 14 GB", | 20260505 10:38:22.784 | INFO | "pinned" : " 29 GB" | 20260505 10:38:22.784 | INFO | } | 20260505 10:38:22.785 | INFO | Perf report for the last 5.1276 seconds: | 20260505 10:38:22.785 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:38:22.785 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:22.785 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:22.785 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:22.785 | INFO | ANALYSIS : 64.7%, 3.0, 100.8ms, 0 | 20260505 10:38:22.785 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:22.785 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:22.785 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 10:38:22.785 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:22.785 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:22.785 | INFO | Uploaded 36GiB to GPU in 2.24797s (16.0144GiB/s) | 20260505 10:38:22.785 | INFO | Downloaded 34GiB to GPU in 2.08778s (16.2853GiB/s) | 20260505 10:38:22.914 | INFO | Starting a round of Baz IO | 20260505 10:38:26.172 | INFO | Populating RTMetrics report | 20260505 10:38:26.172 | INFO | RTMetrics report populated | 20260505 10:38:26.172 | INFO | Opening tmp RTMetrics json file | 20260505 10:38:26.201 | INFO | Serializing RTMetrics to json | 20260505 10:38:26.254 | INFO | Serialization done, swapping tmp file now... | 20260505 10:38:26.267 | INFO | RTMetrics done | 20260505 10:38:26.302 | INFO | Finished a round of Baz IO: Wrote 2.408 GiB in 3.388 seconds | 20260505 10:38:26.302 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.81% / 0.01% / 75.18% | 20260505 10:38:26.302 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.42492% / 2.38419e-05% / 0.000103315% | 20260505 10:38:26.302 | INFO | Baz IO overall: | 20260505 10:38:26.302 | INFO | BytesWritten : 1498129451211 | 20260505 10:38:26.302 | INFO | NumEvents : 713299459407 | 20260505 10:38:27.926 | INFO | Analyzing chunk frames = [2551808,2552320) | 20260505 10:38:27.926 | INFO | MemInfo: { | 20260505 10:38:27.926 | INFO | "gpu" : " 17 GB", | 20260505 10:38:27.926 | INFO | "huge" : "", | 20260505 10:38:27.926 | INFO | "huge_shared" : " 27 GB", | 20260505 10:38:27.926 | INFO | "malloc" : " 14 GB", | 20260505 10:38:27.926 | INFO | "pinned" : " 29 GB" | 20260505 10:38:27.926 | INFO | } | 20260505 10:38:27.926 | INFO | Perf report for the last 5.14129 seconds: | 20260505 10:38:27.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:38:27.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:27.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:27.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:27.926 | INFO | ANALYSIS : 64.8%, 2.9, 98.6ms, 0 | 20260505 10:38:27.926 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:27.926 | INFO | RT_METRICS : 47.6%, 1.0, 25.0ms, 0 | 20260505 10:38:27.926 | INFO | PRE_HQ : 62.8%, 1.8, 59.5ms, 0 | 20260505 10:38:27.926 | INFO | BAZWRITER : 65.3%, 4.2, 141.8ms, 0 | 20260505 10:38:27.926 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260505 10:38:27.926 | INFO | Uploaded 36GiB to GPU in 2.24569s (16.0307GiB/s) | 20260505 10:38:27.926 | INFO | Downloaded 38GiB to GPU in 2.36835s (16.0449GiB/s) | 20260505 10:38:33.073 | INFO | Analyzing chunk frames = [2552320,2552832) | 20260505 10:38:33.073 | INFO | MemInfo: { | 20260505 10:38:33.073 | INFO | "gpu" : " 17 GB", | 20260505 10:38:33.073 | INFO | "huge" : "", | 20260505 10:38:33.073 | INFO | "huge_shared" : " 27 GB", | 20260505 10:38:33.073 | INFO | "malloc" : " 14 GB", | 20260505 10:38:33.073 | INFO | "pinned" : " 29 GB" | 20260505 10:38:33.073 | INFO | } | 20260505 10:38:33.073 | INFO | Perf report for the last 5.14731 seconds: | 20260505 10:38:33.073 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:38:33.073 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:33.073 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:33.073 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:33.073 | INFO | ANALYSIS : 64.1%, 3.0, 100.3ms, 0 | 20260505 10:38:33.073 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:33.073 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:33.073 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:38:33.073 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:33.073 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:33.073 | INFO | Uploaded 36GiB to GPU in 2.17643s (16.5408GiB/s) | 20260505 10:38:33.073 | INFO | Downloaded 34GiB to GPU in 2.09008s (16.2673GiB/s) | 20260505 10:38:38.216 | INFO | Analyzing chunk frames = [2552832,2553344) | 20260505 10:38:38.216 | INFO | MemInfo: { | 20260505 10:38:38.216 | INFO | "gpu" : " 17 GB", | 20260505 10:38:38.216 | INFO | "huge" : "", | 20260505 10:38:38.216 | INFO | "huge_shared" : " 27 GB", | 20260505 10:38:38.216 | INFO | "malloc" : " 14 GB", | 20260505 10:38:38.216 | INFO | "pinned" : " 29 GB" | 20260505 10:38:38.216 | INFO | } | 20260505 10:38:38.216 | INFO | Perf report for the last 5.14322 seconds: | 20260505 10:38:38.216 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:38:38.216 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:38.216 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:38.216 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:38.216 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 10:38:38.217 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:38.217 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:38.217 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:38:38.217 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:38.217 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:38.217 | INFO | Uploaded 36GiB to GPU in 2.21571s (16.2476GiB/s) | 20260505 10:38:38.217 | INFO | Downloaded 34GiB to GPU in 2.12002s (16.0376GiB/s) | 20260505 10:38:43.358 | INFO | Analyzing chunk frames = [2553344,2553856) | 20260505 10:38:43.358 | INFO | MemInfo: { | 20260505 10:38:43.358 | INFO | "gpu" : " 17 GB", | 20260505 10:38:43.358 | INFO | "huge" : "", | 20260505 10:38:43.358 | INFO | "huge_shared" : " 27 GB", | 20260505 10:38:43.358 | INFO | "malloc" : " 14 GB", | 20260505 10:38:43.358 | INFO | "pinned" : " 29 GB" | 20260505 10:38:43.358 | INFO | } | 20260505 10:38:43.358 | INFO | Perf report for the last 5.14204 seconds: | 20260505 10:38:43.358 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:38:43.358 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:43.358 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:43.359 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:43.359 | INFO | ANALYSIS : 62.9%, 3.0, 97.8ms, 0 | 20260505 10:38:43.359 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:43.359 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:43.359 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 10:38:43.359 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:43.359 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:43.359 | INFO | Uploaded 36GiB to GPU in 2.15723s (16.6881GiB/s) | 20260505 10:38:43.359 | INFO | Downloaded 34GiB to GPU in 2.07406s (16.393GiB/s) | 20260505 10:38:48.488 | INFO | Analyzing chunk frames = [2553856,2554368) | 20260505 10:38:48.488 | INFO | MemInfo: { | 20260505 10:38:48.488 | INFO | "gpu" : " 17 GB", | 20260505 10:38:48.488 | INFO | "huge" : "", | 20260505 10:38:48.488 | INFO | "huge_shared" : " 27 GB", | 20260505 10:38:48.488 | INFO | "malloc" : " 14 GB", | 20260505 10:38:48.488 | INFO | "pinned" : " 29 GB" | 20260505 10:38:48.488 | INFO | } | 20260505 10:38:48.488 | INFO | Perf report for the last 5.12994 seconds: | 20260505 10:38:48.488 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:38:48.488 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:48.488 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:48.488 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:48.488 | INFO | ANALYSIS : 63.5%, 3.0, 99.0ms, 0 | 20260505 10:38:48.488 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:48.488 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:48.489 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:38:48.489 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:48.489 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:48.489 | INFO | Uploaded 36GiB to GPU in 2.16025s (16.6647GiB/s) | 20260505 10:38:48.489 | INFO | Downloaded 34GiB to GPU in 2.12906s (15.9695GiB/s) | 20260505 10:38:53.627 | INFO | Analyzing chunk frames = [2554368,2554880) | 20260505 10:38:53.627 | INFO | MemInfo: { | 20260505 10:38:53.627 | INFO | "gpu" : " 17 GB", | 20260505 10:38:53.627 | INFO | "huge" : "", | 20260505 10:38:53.627 | INFO | "huge_shared" : " 27 GB", | 20260505 10:38:53.627 | INFO | "malloc" : " 14 GB", | 20260505 10:38:53.627 | INFO | "pinned" : " 29 GB" | 20260505 10:38:53.627 | INFO | } | 20260505 10:38:53.627 | INFO | Perf report for the last 5.13898 seconds: | 20260505 10:38:53.627 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:38:53.627 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:53.627 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:53.627 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:53.627 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 10:38:53.627 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:53.627 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:53.628 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 10:38:53.628 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:53.628 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:53.628 | INFO | Uploaded 36GiB to GPU in 2.22173s (16.2036GiB/s) | 20260505 10:38:53.628 | INFO | Downloaded 34GiB to GPU in 2.0992s (16.1966GiB/s) | 20260505 10:38:58.776 | INFO | Analyzing chunk frames = [2554880,2555392) | 20260505 10:38:58.776 | INFO | MemInfo: { | 20260505 10:38:58.776 | INFO | "gpu" : " 17 GB", | 20260505 10:38:58.776 | INFO | "huge" : "", | 20260505 10:38:58.776 | INFO | "huge_shared" : " 27 GB", | 20260505 10:38:58.776 | INFO | "malloc" : " 14 GB", | 20260505 10:38:58.776 | INFO | "pinned" : " 29 GB" | 20260505 10:38:58.776 | INFO | } | 20260505 10:38:58.776 | INFO | Perf report for the last 5.14899 seconds: | 20260505 10:38:58.776 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:38:58.776 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:58.776 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:58.776 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:58.776 | INFO | ANALYSIS : 63.5%, 3.0, 99.3ms, 0 | 20260505 10:38:58.776 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:58.777 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:58.777 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 10:38:58.777 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:58.777 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:38:58.777 | INFO | Uploaded 36GiB to GPU in 2.19874s (16.373GiB/s) | 20260505 10:38:58.777 | INFO | Downloaded 34GiB to GPU in 2.15951s (15.7443GiB/s) | 20260505 10:39:03.910 | INFO | Analyzing chunk frames = [2555392,2555904) | 20260505 10:39:03.910 | INFO | MemInfo: { | 20260505 10:39:03.910 | INFO | "gpu" : " 17 GB", | 20260505 10:39:03.910 | INFO | "huge" : "", | 20260505 10:39:03.910 | INFO | "huge_shared" : " 27 GB", | 20260505 10:39:03.910 | INFO | "malloc" : " 14 GB", | 20260505 10:39:03.910 | INFO | "pinned" : " 29 GB" | 20260505 10:39:03.910 | INFO | } | 20260505 10:39:03.910 | INFO | Perf report for the last 5.13371 seconds: | 20260505 10:39:03.910 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:39:03.910 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:03.910 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:03.910 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:03.910 | INFO | ANALYSIS : 64.1%, 3.0, 100.0ms, 0 | 20260505 10:39:03.910 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:03.910 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:03.910 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 10:39:03.910 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:03.910 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:03.910 | INFO | Uploaded 36GiB to GPU in 2.22023s (16.2145GiB/s) | 20260505 10:39:03.911 | INFO | Downloaded 34GiB to GPU in 2.07852s (16.3578GiB/s) | 20260505 10:39:03.911 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 10:39:03.911 | INFO | Total mem usage: 15.3 GiB | 20260505 10:39:03.911 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 10:39:03.911 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -6.03%) | 20260505 10:39:03.911 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 10:39:03.911 | INFO | High Water Breakdown: | 20260505 10:39:03.911 | INFO | 61.1% : 8.83 GiB ( 93.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 10:39:03.911 | INFO | 38.8% : 5.6 GiB ( 84.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 10:39:03.911 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 10:39:03.911 | INFO | ------------------------------------------ | 20260505 10:39:03.911 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:39:03.911 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 10:39:03.911 | INFO | Total mem usage: 26.5 GiB | 20260505 10:39:03.911 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 10:39:03.911 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 10:39:03.911 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 10:39:03.911 | INFO | High Water Breakdown: | 20260505 10:39:03.911 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:39:03.911 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:39:03.911 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:39:03.911 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:39:03.911 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:39:03.911 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:39:03.911 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 10:39:03.911 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 10:39:03.911 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:39:03.911 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 10:39:03.911 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 10:39:03.911 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 10:39:03.911 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 10:39:03.911 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 10:39:03.911 | INFO | ------------------------------------------ | 20260505 10:39:03.911 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:39:03.911 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 10:39:03.911 | INFO | Total mem usage: 25.6 GiB | 20260505 10:39:03.911 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 10:39:03.911 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 10:39:03.911 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 10:39:03.911 | INFO | High Water Breakdown: | 20260505 10:39:03.911 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 10:39:03.911 | INFO | ------------------------------------------ | 20260505 10:39:03.911 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:39:03.911 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 10:39:03.911 | INFO | Total mem usage: 15.7 GiB | 20260505 10:39:03.911 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 10:39:03.911 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 10:39:03.911 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 10:39:03.911 | INFO | High Water Breakdown: | 20260505 10:39:03.911 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 10:39:03.911 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 10:39:03.911 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 10:39:03.911 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 10:39:03.911 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 10:39:03.911 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 10:39:03.912 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 10:39:03.912 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 10:39:03.912 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 10:39:03.912 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:39:03.912 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 10:39:03.912 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:39:03.912 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 10:39:03.912 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 10:39:03.912 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 10:39:03.912 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 10:39:03.912 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:39:03.912 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 10:39:03.912 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:39:03.912 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:39:03.912 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:39:03.912 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 10:39:03.912 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 10:39:03.912 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:39:03.912 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 10:39:03.912 | INFO | ------------------------------------------ | 20260505 10:39:03.912 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:39:03.912 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 10:39:03.912 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:39:03.912 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:39:03.912 | INFO | PerfReport: Full Execution -- 3.23e+01 / 7.72e+01, 1.39e+01 / 2.17e+00 | 20260505 10:39:03.912 | INFO | PerfReport: 0.29%, AggregateStats -- 9.33e-02 / 1.01e-01, 1.07e-02 / 3.49e-03 | 20260505 10:39:03.912 | INFO | PerfReport: 11.71%, Baseline -- 3.78e+00 / 4.44e+00, 6.10e-01 / 2.00e-01 | 20260505 10:39:03.912 | INFO | PerfReport: 3.32%, Histogram -- 1.07e+00 / 1.50e+00, 2.08e-01 / 8.67e-02 | 20260505 10:39:03.912 | INFO | PerfReport: 15.75%, DME -- 4.07e+01 / 4.50e+01, 4.39e+00 / 1.85e+00 | 20260505 10:39:03.912 | INFO | PerfReport: 56.44%, FrameLabeling -- 1.82e+01 / 2.13e+01, 1.71e+00 / 1.91e+00 | 20260505 10:39:03.912 | INFO | PerfReport: 7.41%, PulseAccumulating -- 2.39e+00 / 2.90e+00, 4.31e-01 / 2.37e-01 | 20260505 10:39:03.912 | INFO | PerfReport: 3.75%, Metrics -- 1.21e+00 / 1.50e+00, 3.48e-01 / 1.91e-01 | 20260505 10:39:03.912 | INFO | PerfReport: 1.32%, PulseSerializing -- 4.28e-01 / 4.70e-01, 7.93e-02 / 4.06e-02 | 20260505 10:39:03.912 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 10:39:03.912 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:39:03.912 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:39:03.912 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.45e+01 / 0.00e+00 | 20260505 10:39:03.912 | INFO | PerfReport: 22.62%, Upload -- 2.29e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260505 10:39:03.912 | INFO | PerfReport: 21.93%, Download -- 2.22e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260505 10:39:03.912 | INFO | PerfReport: 31.88%, Compute -- 3.23e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 10:39:03.912 | INFO | PerfReport: 1.98%, LockUpload -- 2.00e+00 / 0.00e+00, 7.57e+00 / 0.00e+00 | 20260505 10:39:03.912 | INFO | PerfReport: 1.21%, LockDownload -- 1.23e+00 / 0.00e+00, 2.84e+00 / 0.00e+00 | 20260505 10:39:03.912 | INFO | PerfReport: 20.38%, LockCompute -- 2.06e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260505 10:39:03.912 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.57e-02 / 0.00e+00 | 20260505 10:39:04.044 | INFO | Starting a round of Baz IO | 20260505 10:39:07.317 | INFO | Populating RTMetrics report | 20260505 10:39:07.318 | INFO | RTMetrics report populated | 20260505 10:39:07.318 | INFO | Opening tmp RTMetrics json file | 20260505 10:39:07.329 | INFO | Serializing RTMetrics to json | 20260505 10:39:07.330 | INFO | Serialization done, swapping tmp file now... | 20260505 10:39:07.331 | INFO | RTMetrics done | 20260505 10:39:07.458 | INFO | Finished a round of Baz IO: Wrote 2.408 GiB in 3.414 seconds | 20260505 10:39:07.458 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.80% / 0.01% / 75.19% | 20260505 10:39:07.458 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.41973% / 1.19209e-05% / 8.34465e-05% | 20260505 10:39:07.458 | INFO | Baz IO overall: | 20260505 10:39:07.458 | INFO | BytesWritten : 1500714567513 | 20260505 10:39:07.458 | INFO | NumEvents : 714531703381 | 20260505 10:39:09.053 | INFO | Analyzing chunk frames = [2555904,2556416) | 20260505 10:39:09.053 | INFO | MemInfo: { | 20260505 10:39:09.053 | INFO | "gpu" : " 17 GB", | 20260505 10:39:09.053 | INFO | "huge" : "", | 20260505 10:39:09.053 | INFO | "huge_shared" : " 27 GB", | 20260505 10:39:09.053 | INFO | "malloc" : " 14 GB", | 20260505 10:39:09.053 | INFO | "pinned" : " 29 GB" | 20260505 10:39:09.053 | INFO | } | 20260505 10:39:09.053 | INFO | Perf report for the last 5.14265 seconds: | 20260505 10:39:09.053 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:39:09.053 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:09.053 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:09.053 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:09.053 | INFO | ANALYSIS : 65.1%, 2.9, 100.1ms, 0 | 20260505 10:39:09.053 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:09.053 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260505 10:39:09.053 | INFO | PRE_HQ : 62.0%, 1.8, 59.5ms, 0 | 20260505 10:39:09.053 | INFO | BAZWRITER : 65.7%, 4.1, 139.6ms, 0 | 20260505 10:39:09.053 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 10:39:09.053 | INFO | Uploaded 36GiB to GPU in 2.21937s (16.2208GiB/s) | 20260505 10:39:09.053 | INFO | Downloaded 38GiB to GPU in 2.3021s (16.5067GiB/s) | 20260505 10:39:14.184 | INFO | Analyzing chunk frames = [2556416,2556928) | 20260505 10:39:14.184 | INFO | MemInfo: { | 20260505 10:39:14.184 | INFO | "gpu" : " 17 GB", | 20260505 10:39:14.184 | INFO | "huge" : "", | 20260505 10:39:14.184 | INFO | "huge_shared" : " 27 GB", | 20260505 10:39:14.184 | INFO | "malloc" : " 14 GB", | 20260505 10:39:14.184 | INFO | "pinned" : " 29 GB" | 20260505 10:39:14.184 | INFO | } | 20260505 10:39:14.184 | INFO | Perf report for the last 5.13107 seconds: | 20260505 10:39:14.184 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:39:14.184 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:14.184 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:14.184 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:14.184 | INFO | ANALYSIS : 64.3%, 3.0, 100.2ms, 0 | 20260505 10:39:14.184 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:14.184 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:14.184 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 10:39:14.184 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:14.184 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:14.184 | INFO | Uploaded 36GiB to GPU in 2.17537s (16.5489GiB/s) | 20260505 10:39:14.184 | INFO | Downloaded 34GiB to GPU in 2.08825s (16.2816GiB/s) | 20260505 10:39:19.330 | INFO | Analyzing chunk frames = [2556928,2557440) | 20260505 10:39:19.330 | INFO | MemInfo: { | 20260505 10:39:19.330 | INFO | "gpu" : " 17 GB", | 20260505 10:39:19.330 | INFO | "huge" : "", | 20260505 10:39:19.330 | INFO | "huge_shared" : " 27 GB", | 20260505 10:39:19.330 | INFO | "malloc" : " 14 GB", | 20260505 10:39:19.330 | INFO | "pinned" : " 29 GB" | 20260505 10:39:19.330 | INFO | } | 20260505 10:39:19.330 | INFO | Perf report for the last 5.14601 seconds: | 20260505 10:39:19.330 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:39:19.330 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:19.330 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:19.330 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:19.330 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260505 10:39:19.330 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:19.330 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:19.330 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:39:19.330 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:19.330 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:19.330 | INFO | Uploaded 36GiB to GPU in 2.15506s (16.7049GiB/s) | 20260505 10:39:19.330 | INFO | Downloaded 34GiB to GPU in 2.06297s (16.4811GiB/s) | 20260505 10:39:24.470 | INFO | Analyzing chunk frames = [2557440,2557952) | 20260505 10:39:24.470 | INFO | MemInfo: { | 20260505 10:39:24.470 | INFO | "gpu" : " 17 GB", | 20260505 10:39:24.470 | INFO | "huge" : "", | 20260505 10:39:24.470 | INFO | "huge_shared" : " 27 GB", | 20260505 10:39:24.470 | INFO | "malloc" : " 14 GB", | 20260505 10:39:24.470 | INFO | "pinned" : " 29 GB" | 20260505 10:39:24.470 | INFO | } | 20260505 10:39:24.471 | INFO | Perf report for the last 5.14063 seconds: | 20260505 10:39:24.471 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:39:24.471 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:24.471 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:24.471 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:24.471 | INFO | ANALYSIS : 63.6%, 3.0, 98.9ms, 0 | 20260505 10:39:24.471 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:24.471 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:24.471 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:39:24.471 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:24.471 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:24.471 | INFO | Uploaded 36GiB to GPU in 2.21968s (16.2186GiB/s) | 20260505 10:39:24.471 | INFO | Downloaded 34GiB to GPU in 2.1307s (15.9572GiB/s) | 20260505 10:39:29.601 | INFO | Analyzing chunk frames = [2557952,2558464) | 20260505 10:39:29.601 | INFO | MemInfo: { | 20260505 10:39:29.601 | INFO | "gpu" : " 17 GB", | 20260505 10:39:29.601 | INFO | "huge" : "", | 20260505 10:39:29.601 | INFO | "huge_shared" : " 27 GB", | 20260505 10:39:29.601 | INFO | "malloc" : " 14 GB", | 20260505 10:39:29.601 | INFO | "pinned" : " 29 GB" | 20260505 10:39:29.601 | INFO | } | 20260505 10:39:29.601 | INFO | Perf report for the last 5.13094 seconds: | 20260505 10:39:29.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:39:29.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:29.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:29.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:29.602 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 10:39:29.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:29.602 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:29.602 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:39:29.602 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:29.602 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:29.602 | INFO | Uploaded 36GiB to GPU in 2.15779s (16.6837GiB/s) | 20260505 10:39:29.602 | INFO | Downloaded 34GiB to GPU in 2.12842s (15.9743GiB/s) | 20260505 10:39:34.738 | INFO | Analyzing chunk frames = [2558464,2558976) | 20260505 10:39:34.738 | INFO | MemInfo: { | 20260505 10:39:34.738 | INFO | "gpu" : " 17 GB", | 20260505 10:39:34.738 | INFO | "huge" : "", | 20260505 10:39:34.738 | INFO | "huge_shared" : " 27 GB", | 20260505 10:39:34.738 | INFO | "malloc" : " 14 GB", | 20260505 10:39:34.738 | INFO | "pinned" : " 29 GB" | 20260505 10:39:34.738 | INFO | } | 20260505 10:39:34.739 | INFO | Perf report for the last 5.13708 seconds: | 20260505 10:39:34.739 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:39:34.739 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:34.739 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:34.739 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:34.739 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 10:39:34.739 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:34.739 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:34.739 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 10:39:34.739 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:34.739 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:34.739 | INFO | Uploaded 36GiB to GPU in 2.16705s (16.6125GiB/s) | 20260505 10:39:34.739 | INFO | Downloaded 34GiB to GPU in 2.05986s (16.506GiB/s) | 20260505 10:39:39.881 | INFO | Analyzing chunk frames = [2558976,2559488) | 20260505 10:39:39.881 | INFO | MemInfo: { | 20260505 10:39:39.881 | INFO | "gpu" : " 17 GB", | 20260505 10:39:39.881 | INFO | "huge" : "", | 20260505 10:39:39.881 | INFO | "huge_shared" : " 27 GB", | 20260505 10:39:39.881 | INFO | "malloc" : " 14 GB", | 20260505 10:39:39.881 | INFO | "pinned" : " 29 GB" | 20260505 10:39:39.881 | INFO | } | 20260505 10:39:39.881 | INFO | Perf report for the last 5.14252 seconds: | 20260505 10:39:39.881 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:39:39.881 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:39.881 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:39.881 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:39.881 | INFO | ANALYSIS : 63.2%, 3.0, 98.8ms, 0 | 20260505 10:39:39.881 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:39.881 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:39.881 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 10:39:39.881 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:39.881 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:39.881 | INFO | Uploaded 36GiB to GPU in 2.16774s (16.6072GiB/s) | 20260505 10:39:39.881 | INFO | Downloaded 34GiB to GPU in 2.15385s (15.7857GiB/s) | 20260505 10:39:45.025 | INFO | Analyzing chunk frames = [2559488,2560000) | 20260505 10:39:45.025 | INFO | MemInfo: { | 20260505 10:39:45.025 | INFO | "gpu" : " 17 GB", | 20260505 10:39:45.025 | INFO | "huge" : "", | 20260505 10:39:45.025 | INFO | "huge_shared" : " 27 GB", | 20260505 10:39:45.025 | INFO | "malloc" : " 14 GB", | 20260505 10:39:45.025 | INFO | "pinned" : " 29 GB" | 20260505 10:39:45.025 | INFO | } | 20260505 10:39:45.025 | INFO | Perf report for the last 5.14407 seconds: | 20260505 10:39:45.025 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:39:45.025 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:45.025 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:45.025 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:45.025 | INFO | ANALYSIS : 64.4%, 3.0, 100.6ms, 0 | 20260505 10:39:45.025 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:45.025 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:45.025 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 10:39:45.025 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:45.025 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:45.025 | INFO | Uploaded 36GiB to GPU in 2.24866s (16.0096GiB/s) | 20260505 10:39:45.026 | INFO | Downloaded 34GiB to GPU in 2.08721s (16.2897GiB/s) | 20260505 10:39:45.162 | INFO | Starting a round of Baz IO | 20260505 10:39:48.429 | INFO | Populating RTMetrics report | 20260505 10:39:48.429 | INFO | RTMetrics report populated | 20260505 10:39:48.429 | INFO | Opening tmp RTMetrics json file | 20260505 10:39:48.453 | INFO | Serializing RTMetrics to json | 20260505 10:39:48.454 | INFO | Serialization done, swapping tmp file now... | 20260505 10:39:48.455 | INFO | RTMetrics done | 20260505 10:39:48.619 | INFO | Finished a round of Baz IO: Wrote 2.406 GiB in 3.456 seconds | 20260505 10:39:48.619 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.79% / 0.01% / 75.20% | 20260505 10:39:48.619 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.41257% / 1.19209e-05% / 6.35783e-05% | 20260505 10:39:48.619 | INFO | Baz IO overall: | 20260505 10:39:48.619 | INFO | BytesWritten : 1503298339273 | 20260505 10:39:48.619 | INFO | NumEvents : 715762730358 | 20260505 10:39:50.170 | INFO | Analyzing chunk frames = [2560000,2560512) | 20260505 10:39:50.170 | INFO | MemInfo: { | 20260505 10:39:50.170 | INFO | "gpu" : " 17 GB", | 20260505 10:39:50.170 | INFO | "huge" : "", | 20260505 10:39:50.170 | INFO | "huge_shared" : " 27 GB", | 20260505 10:39:50.170 | INFO | "malloc" : " 14 GB", | 20260505 10:39:50.170 | INFO | "pinned" : " 29 GB" | 20260505 10:39:50.170 | INFO | } | 20260505 10:39:50.170 | INFO | Perf report for the last 5.14472 seconds: | 20260505 10:39:50.170 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:39:50.170 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:50.170 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:50.170 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:50.170 | INFO | ANALYSIS : 65.0%, 2.9, 99.9ms, 0 | 20260505 10:39:50.170 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:50.170 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260505 10:39:50.170 | INFO | PRE_HQ : 61.1%, 1.9, 59.4ms, 0 | 20260505 10:39:50.170 | INFO | BAZWRITER : 66.5%, 4.1, 141.7ms, 0 | 20260505 10:39:50.170 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 10:39:50.170 | INFO | Uploaded 36GiB to GPU in 2.20921s (16.2955GiB/s) | 20260505 10:39:50.170 | INFO | Downloaded 38GiB to GPU in 2.30262s (16.503GiB/s) | 20260505 10:39:55.304 | INFO | Analyzing chunk frames = [2560512,2561024) | 20260505 10:39:55.304 | INFO | MemInfo: { | 20260505 10:39:55.304 | INFO | "gpu" : " 17 GB", | 20260505 10:39:55.304 | INFO | "huge" : "", | 20260505 10:39:55.304 | INFO | "huge_shared" : " 27 GB", | 20260505 10:39:55.304 | INFO | "malloc" : " 14 GB", | 20260505 10:39:55.304 | INFO | "pinned" : " 29 GB" | 20260505 10:39:55.304 | INFO | } | 20260505 10:39:55.304 | INFO | Perf report for the last 5.13393 seconds: | 20260505 10:39:55.304 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:39:55.304 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:55.304 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:55.304 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:55.304 | INFO | ANALYSIS : 64.4%, 3.0, 100.4ms, 0 | 20260505 10:39:55.304 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:55.304 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:55.304 | INFO | PRE_HQ : 29.3%, 1.0, 15.3ms, 0 | 20260505 10:39:55.304 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:55.304 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:39:55.304 | INFO | Uploaded 36GiB to GPU in 2.23664s (16.0955GiB/s) | 20260505 10:39:55.304 | INFO | Downloaded 34GiB to GPU in 2.08336s (16.3198GiB/s) | 20260505 10:40:00.438 | INFO | Analyzing chunk frames = [2561024,2561536) | 20260505 10:40:00.438 | INFO | MemInfo: { | 20260505 10:40:00.438 | INFO | "gpu" : " 17 GB", | 20260505 10:40:00.438 | INFO | "huge" : "", | 20260505 10:40:00.438 | INFO | "huge_shared" : " 27 GB", | 20260505 10:40:00.438 | INFO | "malloc" : " 14 GB", | 20260505 10:40:00.438 | INFO | "pinned" : " 29 GB" | 20260505 10:40:00.438 | INFO | } | 20260505 10:40:00.438 | INFO | Perf report for the last 5.13393 seconds: | 20260505 10:40:00.438 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:40:00.438 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:00.438 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:00.438 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:00.438 | INFO | ANALYSIS : 63.1%, 3.0, 98.1ms, 0 | 20260505 10:40:00.438 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:00.438 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:00.438 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 10:40:00.438 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:00.438 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:00.438 | INFO | Uploaded 36GiB to GPU in 2.14722s (16.7659GiB/s) | 20260505 10:40:00.438 | INFO | Downloaded 34GiB to GPU in 2.0715s (16.4133GiB/s) | 20260505 10:40:05.580 | INFO | Analyzing chunk frames = [2561536,2562048) | 20260505 10:40:05.580 | INFO | MemInfo: { | 20260505 10:40:05.580 | INFO | "gpu" : " 17 GB", | 20260505 10:40:05.580 | INFO | "huge" : "", | 20260505 10:40:05.580 | INFO | "huge_shared" : " 27 GB", | 20260505 10:40:05.580 | INFO | "malloc" : " 14 GB", | 20260505 10:40:05.580 | INFO | "pinned" : " 29 GB" | 20260505 10:40:05.580 | INFO | } | 20260505 10:40:05.580 | INFO | Perf report for the last 5.14234 seconds: | 20260505 10:40:05.580 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:40:05.580 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:05.580 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:05.580 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:05.580 | INFO | ANALYSIS : 63.0%, 3.0, 97.9ms, 0 | 20260505 10:40:05.580 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:05.580 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:05.580 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 10:40:05.580 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:05.580 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:05.580 | INFO | Uploaded 36GiB to GPU in 2.17603s (16.5439GiB/s) | 20260505 10:40:05.580 | INFO | Downloaded 34GiB to GPU in 2.07265s (16.4041GiB/s) | 20260505 10:40:10.727 | INFO | Analyzing chunk frames = [2562048,2562560) | 20260505 10:40:10.727 | INFO | MemInfo: { | 20260505 10:40:10.727 | INFO | "gpu" : " 17 GB", | 20260505 10:40:10.727 | INFO | "huge" : "", | 20260505 10:40:10.727 | INFO | "huge_shared" : " 27 GB", | 20260505 10:40:10.727 | INFO | "malloc" : " 14 GB", | 20260505 10:40:10.727 | INFO | "pinned" : " 29 GB" | 20260505 10:40:10.727 | INFO | } | 20260505 10:40:10.727 | INFO | Perf report for the last 5.1473 seconds: | 20260505 10:40:10.728 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:40:10.728 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:10.728 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:10.728 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:10.728 | INFO | ANALYSIS : 63.2%, 3.0, 98.8ms, 0 | 20260505 10:40:10.728 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:10.728 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:10.728 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 10:40:10.728 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:10.728 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:10.728 | INFO | Uploaded 36GiB to GPU in 2.12504s (16.9409GiB/s) | 20260505 10:40:10.728 | INFO | Downloaded 34GiB to GPU in 2.12536s (15.9973GiB/s) | 20260505 10:40:15.861 | INFO | Analyzing chunk frames = [2562560,2563072) | 20260505 10:40:15.861 | INFO | MemInfo: { | 20260505 10:40:15.861 | INFO | "gpu" : " 17 GB", | 20260505 10:40:15.861 | INFO | "huge" : "", | 20260505 10:40:15.861 | INFO | "huge_shared" : " 27 GB", | 20260505 10:40:15.861 | INFO | "malloc" : " 14 GB", | 20260505 10:40:15.861 | INFO | "pinned" : " 29 GB" | 20260505 10:40:15.861 | INFO | } | 20260505 10:40:15.861 | INFO | Perf report for the last 5.13351 seconds: | 20260505 10:40:15.861 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:40:15.861 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:15.861 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:15.861 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:15.861 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 10:40:15.861 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:15.861 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:15.861 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:40:15.861 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:15.861 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:15.861 | INFO | Uploaded 36GiB to GPU in 2.2223s (16.1994GiB/s) | 20260505 10:40:15.861 | INFO | Downloaded 34GiB to GPU in 2.10409s (16.159GiB/s) | 20260505 10:40:20.993 | INFO | Analyzing chunk frames = [2563072,2563584) | 20260505 10:40:20.993 | INFO | MemInfo: { | 20260505 10:40:20.993 | INFO | "gpu" : " 17 GB", | 20260505 10:40:20.993 | INFO | "huge" : "", | 20260505 10:40:20.993 | INFO | "huge_shared" : " 27 GB", | 20260505 10:40:20.993 | INFO | "malloc" : " 14 GB", | 20260505 10:40:20.993 | INFO | "pinned" : " 29 GB" | 20260505 10:40:20.993 | INFO | } | 20260505 10:40:20.993 | INFO | Perf report for the last 5.13226 seconds: | 20260505 10:40:20.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:40:20.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:20.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:20.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:20.993 | INFO | ANALYSIS : 63.8%, 3.0, 99.5ms, 0 | 20260505 10:40:20.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:20.993 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:20.993 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 10:40:20.993 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:20.993 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:20.994 | INFO | Uploaded 36GiB to GPU in 2.16459s (16.6313GiB/s) | 20260505 10:40:20.994 | INFO | Downloaded 34GiB to GPU in 2.14746s (15.8327GiB/s) | 20260505 10:40:26.140 | INFO | Analyzing chunk frames = [2563584,2564096) | 20260505 10:40:26.140 | INFO | MemInfo: { | 20260505 10:40:26.140 | INFO | "gpu" : " 17 GB", | 20260505 10:40:26.140 | INFO | "huge" : "", | 20260505 10:40:26.140 | INFO | "huge_shared" : " 27 GB", | 20260505 10:40:26.140 | INFO | "malloc" : " 14 GB", | 20260505 10:40:26.140 | INFO | "pinned" : " 29 GB" | 20260505 10:40:26.140 | INFO | } | 20260505 10:40:26.140 | INFO | Perf report for the last 5.14672 seconds: | 20260505 10:40:26.140 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:40:26.140 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:26.140 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:26.140 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:26.140 | INFO | ANALYSIS : 64.4%, 3.0, 100.6ms, 0 | 20260505 10:40:26.140 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:26.140 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:26.140 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 10:40:26.140 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:26.140 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:26.140 | INFO | Uploaded 36GiB to GPU in 2.24755s (16.0174GiB/s) | 20260505 10:40:26.140 | INFO | Downloaded 34GiB to GPU in 2.08987s (16.269GiB/s) | 20260505 10:40:26.140 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 10:40:26.140 | INFO | Total mem usage: 15.3 GiB | 20260505 10:40:26.140 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 10:40:26.140 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.39%) | 20260505 10:40:26.140 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 10:40:26.141 | INFO | High Water Breakdown: | 20260505 10:40:26.141 | INFO | 61% : 8.78 GiB ( 92.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 10:40:26.141 | INFO | 38.9% : 5.6 GiB ( 84.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 10:40:26.141 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 10:40:26.141 | INFO | ------------------------------------------ | 20260505 10:40:26.141 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:40:26.141 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 10:40:26.141 | INFO | Total mem usage: 26.5 GiB | 20260505 10:40:26.141 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 10:40:26.141 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 10:40:26.141 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 10:40:26.141 | INFO | High Water Breakdown: | 20260505 10:40:26.141 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:40:26.141 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:40:26.141 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:40:26.141 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:40:26.141 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:40:26.141 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:40:26.141 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 10:40:26.141 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 10:40:26.141 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:40:26.141 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 10:40:26.141 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 10:40:26.141 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 10:40:26.141 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 10:40:26.141 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 10:40:26.141 | INFO | ------------------------------------------ | 20260505 10:40:26.141 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:40:26.141 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 10:40:26.141 | INFO | Total mem usage: 25.6 GiB | 20260505 10:40:26.141 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 10:40:26.141 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 10:40:26.141 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 10:40:26.141 | INFO | High Water Breakdown: | 20260505 10:40:26.141 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 10:40:26.141 | INFO | ------------------------------------------ | 20260505 10:40:26.141 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:40:26.141 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 10:40:26.141 | INFO | Total mem usage: 15.7 GiB | 20260505 10:40:26.141 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 10:40:26.141 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 10:40:26.141 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 10:40:26.141 | INFO | High Water Breakdown: | 20260505 10:40:26.141 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 10:40:26.141 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 10:40:26.141 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 10:40:26.141 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 10:40:26.141 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 10:40:26.141 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 10:40:26.141 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 10:40:26.141 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 10:40:26.141 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 10:40:26.141 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:40:26.141 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 10:40:26.141 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:40:26.141 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 10:40:26.141 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 10:40:26.141 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 10:40:26.141 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 10:40:26.141 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:40:26.141 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 10:40:26.141 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:40:26.141 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:40:26.141 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:40:26.141 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 10:40:26.141 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 10:40:26.141 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:40:26.141 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 10:40:26.141 | INFO | ------------------------------------------ | 20260505 10:40:26.141 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:40:26.141 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 10:40:26.141 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:40:26.141 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:40:26.142 | INFO | PerfReport: Full Execution -- 3.23e+01 / 7.73e+01, 1.37e+01 / 2.11e+00 | 20260505 10:40:26.142 | INFO | PerfReport: 0.29%, AggregateStats -- 9.31e-02 / 1.02e-01, 1.04e-02 / 6.81e-03 | 20260505 10:40:26.142 | INFO | PerfReport: 11.69%, Baseline -- 3.78e+00 / 4.50e+00, 6.13e-01 / 2.08e-01 | 20260505 10:40:26.142 | INFO | PerfReport: 3.31%, Histogram -- 1.07e+00 / 1.52e+00, 2.08e-01 / 7.01e-02 | 20260505 10:40:26.142 | INFO | PerfReport: 15.77%, DME -- 4.08e+01 / 4.55e+01, 4.36e+00 / 2.16e+00 | 20260505 10:40:26.142 | INFO | PerfReport: 56.52%, FrameLabeling -- 1.83e+01 / 2.08e+01, 1.77e+00 / 2.21e+00 | 20260505 10:40:26.142 | INFO | PerfReport: 7.35%, PulseAccumulating -- 2.38e+00 / 2.91e+00, 4.13e-01 / 2.02e-01 | 20260505 10:40:26.142 | INFO | PerfReport: 3.74%, Metrics -- 1.21e+00 / 1.52e+00, 3.50e-01 / 1.87e-01 | 20260505 10:40:26.142 | INFO | PerfReport: 1.32%, PulseSerializing -- 4.27e-01 / 4.78e-01, 7.92e-02 / 3.65e-02 | 20260505 10:40:26.142 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 10:40:26.142 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:40:26.142 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:40:26.142 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.45e+01 / 0.00e+00 | 20260505 10:40:26.142 | INFO | PerfReport: 22.51%, Upload -- 2.28e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260505 10:40:26.142 | INFO | PerfReport: 21.85%, Download -- 2.21e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260505 10:40:26.142 | INFO | PerfReport: 31.91%, Compute -- 3.23e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 10:40:26.142 | INFO | PerfReport: 1.97%, LockUpload -- 2.00e+00 / 0.00e+00, 7.50e+00 / 0.00e+00 | 20260505 10:40:26.142 | INFO | PerfReport: 1.15%, LockDownload -- 1.17e+00 / 0.00e+00, 2.73e+00 / 0.00e+00 | 20260505 10:40:26.142 | INFO | PerfReport: 20.59%, LockCompute -- 2.09e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260505 10:40:26.142 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.28e-02 / 0.00e+00 | 20260505 10:40:26.279 | INFO | Starting a round of Baz IO | 20260505 10:40:29.538 | INFO | Populating RTMetrics report | 20260505 10:40:29.538 | INFO | RTMetrics report populated | 20260505 10:40:29.538 | INFO | Opening tmp RTMetrics json file | 20260505 10:40:29.551 | INFO | Serializing RTMetrics to json | 20260505 10:40:29.580 | INFO | Serialization done, swapping tmp file now... | 20260505 10:40:29.598 | INFO | RTMetrics done | 20260505 10:40:29.681 | INFO | Finished a round of Baz IO: Wrote 2.407 GiB in 3.402 seconds | 20260505 10:40:29.681 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.78% / 0.01% / 75.21% | 20260505 10:40:29.681 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.41187% / 1.19209e-05% / 8.34465e-05% | 20260505 10:40:29.681 | INFO | Baz IO overall: | 20260505 10:40:29.681 | INFO | BytesWritten : 1505882628022 | 20260505 10:40:29.681 | INFO | NumEvents : 716994605916 | 20260505 10:40:31.284 | INFO | Analyzing chunk frames = [2564096,2564608) | 20260505 10:40:31.284 | INFO | MemInfo: { | 20260505 10:40:31.284 | INFO | "gpu" : " 17 GB", | 20260505 10:40:31.284 | INFO | "huge" : "", | 20260505 10:40:31.284 | INFO | "huge_shared" : " 27 GB", | 20260505 10:40:31.284 | INFO | "malloc" : " 14 GB", | 20260505 10:40:31.284 | INFO | "pinned" : " 29 GB" | 20260505 10:40:31.284 | INFO | } | 20260505 10:40:31.284 | INFO | Perf report for the last 5.14386 seconds: | 20260505 10:40:31.284 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:40:31.284 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:31.284 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:31.284 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:31.284 | INFO | ANALYSIS : 64.9%, 2.9, 99.5ms, 0 | 20260505 10:40:31.284 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:31.284 | INFO | RT_METRICS : 46.8%, 1.0, 24.6ms, 0 | 20260505 10:40:31.284 | INFO | PRE_HQ : 61.5%, 1.9, 59.8ms, 0 | 20260505 10:40:31.284 | INFO | BAZWRITER : 65.3%, 4.2, 140.0ms, 0 | 20260505 10:40:31.284 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 10:40:31.284 | INFO | Uploaded 36GiB to GPU in 2.21912s (16.2226GiB/s) | 20260505 10:40:31.284 | INFO | Downloaded 38GiB to GPU in 2.29664s (16.5459GiB/s) | 20260505 10:40:36.420 | INFO | Analyzing chunk frames = [2564608,2565120) | 20260505 10:40:36.420 | INFO | MemInfo: { | 20260505 10:40:36.420 | INFO | "gpu" : " 17 GB", | 20260505 10:40:36.420 | INFO | "huge" : "", | 20260505 10:40:36.420 | INFO | "huge_shared" : " 27 GB", | 20260505 10:40:36.420 | INFO | "malloc" : " 14 GB", | 20260505 10:40:36.420 | INFO | "pinned" : " 29 GB" | 20260505 10:40:36.420 | INFO | } | 20260505 10:40:36.420 | INFO | Perf report for the last 5.13638 seconds: | 20260505 10:40:36.420 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:40:36.420 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:36.420 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:36.420 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:36.420 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 10:40:36.420 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:36.420 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:36.420 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:40:36.421 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:36.421 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:36.421 | INFO | Uploaded 36GiB to GPU in 2.23412s (16.1137GiB/s) | 20260505 10:40:36.421 | INFO | Downloaded 34GiB to GPU in 2.07957s (16.3495GiB/s) | 20260505 10:40:41.547 | INFO | Analyzing chunk frames = [2565120,2565632) | 20260505 10:40:41.547 | INFO | MemInfo: { | 20260505 10:40:41.547 | INFO | "gpu" : " 17 GB", | 20260505 10:40:41.547 | INFO | "huge" : "", | 20260505 10:40:41.547 | INFO | "huge_shared" : " 27 GB", | 20260505 10:40:41.547 | INFO | "malloc" : " 14 GB", | 20260505 10:40:41.547 | INFO | "pinned" : " 29 GB" | 20260505 10:40:41.547 | INFO | } | 20260505 10:40:41.547 | INFO | Perf report for the last 5.12684 seconds: | 20260505 10:40:41.547 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:40:41.547 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:41.547 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:41.547 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:41.547 | INFO | ANALYSIS : 63.2%, 3.0, 98.1ms, 0 | 20260505 10:40:41.547 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:41.547 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:41.547 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:40:41.547 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:41.547 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:41.548 | INFO | Uploaded 36GiB to GPU in 2.15842s (16.6789GiB/s) | 20260505 10:40:41.548 | INFO | Downloaded 34GiB to GPU in 2.06689s (16.4498GiB/s) | 20260505 10:40:46.688 | INFO | Analyzing chunk frames = [2565632,2566144) | 20260505 10:40:46.688 | INFO | MemInfo: { | 20260505 10:40:46.688 | INFO | "gpu" : " 17 GB", | 20260505 10:40:46.688 | INFO | "huge" : "", | 20260505 10:40:46.688 | INFO | "huge_shared" : " 27 GB", | 20260505 10:40:46.688 | INFO | "malloc" : " 14 GB", | 20260505 10:40:46.688 | INFO | "pinned" : " 29 GB" | 20260505 10:40:46.689 | INFO | } | 20260505 10:40:46.689 | INFO | Perf report for the last 5.14149 seconds: | 20260505 10:40:46.689 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:40:46.689 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:46.689 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:46.689 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:46.689 | INFO | ANALYSIS : 63.2%, 3.0, 98.3ms, 0 | 20260505 10:40:46.689 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:46.689 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:46.689 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:40:46.689 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:46.689 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:46.689 | INFO | Uploaded 36GiB to GPU in 2.17238s (16.5717GiB/s) | 20260505 10:40:46.689 | INFO | Downloaded 34GiB to GPU in 2.07487s (16.3866GiB/s) | 20260505 10:40:51.835 | INFO | Analyzing chunk frames = [2566144,2566656) | 20260505 10:40:51.835 | INFO | MemInfo: { | 20260505 10:40:51.835 | INFO | "gpu" : " 17 GB", | 20260505 10:40:51.835 | INFO | "huge" : "", | 20260505 10:40:51.835 | INFO | "huge_shared" : " 27 GB", | 20260505 10:40:51.835 | INFO | "malloc" : " 14 GB", | 20260505 10:40:51.835 | INFO | "pinned" : " 29 GB" | 20260505 10:40:51.835 | INFO | } | 20260505 10:40:51.835 | INFO | Perf report for the last 5.14618 seconds: | 20260505 10:40:51.835 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:40:51.835 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:51.835 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:51.835 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:51.835 | INFO | ANALYSIS : 63.6%, 3.0, 99.5ms, 0 | 20260505 10:40:51.835 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:51.835 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:51.835 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 10:40:51.835 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:51.835 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:51.835 | INFO | Uploaded 36GiB to GPU in 2.2361s (16.0995GiB/s) | 20260505 10:40:51.835 | INFO | Downloaded 34GiB to GPU in 2.11014s (16.1127GiB/s) | 20260505 10:40:56.976 | INFO | Analyzing chunk frames = [2566656,2567168) | 20260505 10:40:56.976 | INFO | MemInfo: { | 20260505 10:40:56.976 | INFO | "gpu" : " 17 GB", | 20260505 10:40:56.976 | INFO | "huge" : "", | 20260505 10:40:56.976 | INFO | "huge_shared" : " 27 GB", | 20260505 10:40:56.976 | INFO | "malloc" : " 14 GB", | 20260505 10:40:56.976 | INFO | "pinned" : " 29 GB" | 20260505 10:40:56.976 | INFO | } | 20260505 10:40:56.977 | INFO | Perf report for the last 5.14171 seconds: | 20260505 10:40:56.977 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:40:56.977 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:56.977 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:56.977 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:56.977 | INFO | ANALYSIS : 63.5%, 3.0, 99.3ms, 0 | 20260505 10:40:56.977 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:56.977 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:56.977 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:40:56.977 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:56.977 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:40:56.977 | INFO | Uploaded 36GiB to GPU in 2.16943s (16.5942GiB/s) | 20260505 10:40:56.977 | INFO | Downloaded 34GiB to GPU in 2.12139s (16.0272GiB/s) | 20260505 10:41:02.116 | INFO | Analyzing chunk frames = [2567168,2567680) | 20260505 10:41:02.116 | INFO | MemInfo: { | 20260505 10:41:02.116 | INFO | "gpu" : " 17 GB", | 20260505 10:41:02.116 | INFO | "huge" : "", | 20260505 10:41:02.116 | INFO | "huge_shared" : " 27 GB", | 20260505 10:41:02.117 | INFO | "malloc" : " 14 GB", | 20260505 10:41:02.117 | INFO | "pinned" : " 29 GB" | 20260505 10:41:02.117 | INFO | } | 20260505 10:41:02.117 | INFO | Perf report for the last 5.14008 seconds: | 20260505 10:41:02.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:41:02.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:02.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:02.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:02.117 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 10:41:02.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:02.117 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:02.117 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 10:41:02.117 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:02.117 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:02.117 | INFO | Uploaded 36GiB to GPU in 2.19329s (16.4137GiB/s) | 20260505 10:41:02.117 | INFO | Downloaded 34GiB to GPU in 2.16444s (15.7085GiB/s) | 20260505 10:41:07.249 | INFO | Analyzing chunk frames = [2567680,2568192) | 20260505 10:41:07.249 | INFO | MemInfo: { | 20260505 10:41:07.249 | INFO | "gpu" : " 17 GB", | 20260505 10:41:07.249 | INFO | "huge" : "", | 20260505 10:41:07.249 | INFO | "huge_shared" : " 27 GB", | 20260505 10:41:07.249 | INFO | "malloc" : " 14 GB", | 20260505 10:41:07.249 | INFO | "pinned" : " 29 GB" | 20260505 10:41:07.249 | INFO | } | 20260505 10:41:07.249 | INFO | Perf report for the last 5.13214 seconds: | 20260505 10:41:07.249 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:41:07.249 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:07.249 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:07.249 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:07.249 | INFO | ANALYSIS : 64.6%, 3.0, 100.7ms, 0 | 20260505 10:41:07.249 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:07.249 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:07.249 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260505 10:41:07.249 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:07.249 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:07.249 | INFO | Uploaded 36GiB to GPU in 2.24608s (16.028GiB/s) | 20260505 10:41:07.249 | INFO | Downloaded 34GiB to GPU in 2.09448s (16.2331GiB/s) | 20260505 10:41:07.379 | INFO | Starting a round of Baz IO | 20260505 10:41:10.635 | INFO | Populating RTMetrics report | 20260505 10:41:10.635 | INFO | RTMetrics report populated | 20260505 10:41:10.635 | INFO | Opening tmp RTMetrics json file | 20260505 10:41:10.691 | INFO | Serializing RTMetrics to json | 20260505 10:41:10.697 | INFO | Serialization done, swapping tmp file now... | 20260505 10:41:10.702 | INFO | RTMetrics done | 20260505 10:41:10.775 | INFO | Finished a round of Baz IO: Wrote 2.407 GiB in 3.396 seconds | 20260505 10:41:10.775 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.77% / 0.01% / 75.22% | 20260505 10:41:10.776 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.40804% / 2.38419e-05% / 6.75519e-05% | 20260505 10:41:10.776 | INFO | Baz IO overall: | 20260505 10:41:10.776 | INFO | BytesWritten : 1508467452781 | 20260505 10:41:10.776 | INFO | NumEvents : 718226954183 | 20260505 10:41:12.393 | INFO | Analyzing chunk frames = [2568192,2568704) | 20260505 10:41:12.393 | INFO | MemInfo: { | 20260505 10:41:12.393 | INFO | "gpu" : " 17 GB", | 20260505 10:41:12.393 | INFO | "huge" : "", | 20260505 10:41:12.393 | INFO | "huge_shared" : " 27 GB", | 20260505 10:41:12.393 | INFO | "malloc" : " 14 GB", | 20260505 10:41:12.393 | INFO | "pinned" : " 29 GB" | 20260505 10:41:12.393 | INFO | } | 20260505 10:41:12.393 | INFO | Perf report for the last 5.14417 seconds: | 20260505 10:41:12.393 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:41:12.393 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:12.393 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:12.393 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:12.393 | INFO | ANALYSIS : 64.7%, 2.9, 99.0ms, 0 | 20260505 10:41:12.393 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:12.393 | INFO | RT_METRICS : 46.9%, 1.0, 24.6ms, 0 | 20260505 10:41:12.393 | INFO | PRE_HQ : 62.7%, 1.8, 59.2ms, 0 | 20260505 10:41:12.393 | INFO | BAZWRITER : 65.3%, 4.2, 141.1ms, 0 | 20260505 10:41:12.393 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260505 10:41:12.393 | INFO | Uploaded 36GiB to GPU in 2.26689s (15.8808GiB/s) | 20260505 10:41:12.393 | INFO | Downloaded 38GiB to GPU in 2.37433s (16.0045GiB/s) | 20260505 10:41:17.533 | INFO | Analyzing chunk frames = [2568704,2569216) | 20260505 10:41:17.533 | INFO | MemInfo: { | 20260505 10:41:17.533 | INFO | "gpu" : " 17 GB", | 20260505 10:41:17.533 | INFO | "huge" : "", | 20260505 10:41:17.533 | INFO | "huge_shared" : " 27 GB", | 20260505 10:41:17.533 | INFO | "malloc" : " 14 GB", | 20260505 10:41:17.533 | INFO | "pinned" : " 29 GB" | 20260505 10:41:17.533 | INFO | } | 20260505 10:41:17.533 | INFO | Perf report for the last 5.1405 seconds: | 20260505 10:41:17.533 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:41:17.533 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:17.533 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:17.533 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:17.533 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 10:41:17.534 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:17.534 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:17.534 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:41:17.534 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:17.534 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:17.534 | INFO | Uploaded 36GiB to GPU in 2.23916s (16.0774GiB/s) | 20260505 10:41:17.534 | INFO | Downloaded 34GiB to GPU in 2.08414s (16.3137GiB/s) | 20260505 10:41:22.678 | INFO | Analyzing chunk frames = [2569216,2569728) | 20260505 10:41:22.678 | INFO | MemInfo: { | 20260505 10:41:22.678 | INFO | "gpu" : " 17 GB", | 20260505 10:41:22.678 | INFO | "huge" : "", | 20260505 10:41:22.678 | INFO | "huge_shared" : " 27 GB", | 20260505 10:41:22.678 | INFO | "malloc" : " 14 GB", | 20260505 10:41:22.678 | INFO | "pinned" : " 29 GB" | 20260505 10:41:22.678 | INFO | } | 20260505 10:41:22.678 | INFO | Perf report for the last 5.14493 seconds: | 20260505 10:41:22.678 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:41:22.678 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:22.678 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:22.679 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:22.679 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260505 10:41:22.679 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:22.679 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:22.679 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:41:22.679 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:22.679 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:22.679 | INFO | Uploaded 36GiB to GPU in 2.16509s (16.6275GiB/s) | 20260505 10:41:22.679 | INFO | Downloaded 34GiB to GPU in 2.06254s (16.4845GiB/s) | 20260505 10:41:27.811 | INFO | Analyzing chunk frames = [2569728,2570240) | 20260505 10:41:27.811 | INFO | MemInfo: { | 20260505 10:41:27.811 | INFO | "gpu" : " 17 GB", | 20260505 10:41:27.811 | INFO | "huge" : "", | 20260505 10:41:27.811 | INFO | "huge_shared" : " 27 GB", | 20260505 10:41:27.811 | INFO | "malloc" : " 14 GB", | 20260505 10:41:27.811 | INFO | "pinned" : " 29 GB" | 20260505 10:41:27.811 | INFO | } | 20260505 10:41:27.811 | INFO | Perf report for the last 5.13299 seconds: | 20260505 10:41:27.811 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:41:27.811 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:27.811 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:27.811 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:27.811 | INFO | ANALYSIS : 63.0%, 3.0, 97.7ms, 0 | 20260505 10:41:27.811 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:27.811 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:27.811 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:41:27.812 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:27.812 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:27.812 | INFO | Uploaded 36GiB to GPU in 2.1515s (16.7325GiB/s) | 20260505 10:41:27.812 | INFO | Downloaded 34GiB to GPU in 2.07409s (16.3928GiB/s) | 20260505 10:41:32.947 | INFO | Analyzing chunk frames = [2570240,2570752) | 20260505 10:41:32.947 | INFO | MemInfo: { | 20260505 10:41:32.947 | INFO | "gpu" : " 17 GB", | 20260505 10:41:32.947 | INFO | "huge" : "", | 20260505 10:41:32.947 | INFO | "huge_shared" : " 27 GB", | 20260505 10:41:32.947 | INFO | "malloc" : " 14 GB", | 20260505 10:41:32.947 | INFO | "pinned" : " 29 GB" | 20260505 10:41:32.947 | INFO | } | 20260505 10:41:32.947 | INFO | Perf report for the last 5.1357 seconds: | 20260505 10:41:32.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:41:32.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:32.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:32.947 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:32.947 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 10:41:32.947 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:32.947 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:32.947 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:41:32.947 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:32.947 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:32.947 | INFO | Uploaded 36GiB to GPU in 2.2352s (16.106GiB/s) | 20260505 10:41:32.948 | INFO | Downloaded 34GiB to GPU in 2.10809s (16.1284GiB/s) | 20260505 10:41:38.095 | INFO | Analyzing chunk frames = [2570752,2571264) | 20260505 10:41:38.095 | INFO | MemInfo: { | 20260505 10:41:38.095 | INFO | "gpu" : " 17 GB", | 20260505 10:41:38.095 | INFO | "huge" : "", | 20260505 10:41:38.095 | INFO | "huge_shared" : " 27 GB", | 20260505 10:41:38.095 | INFO | "malloc" : " 14 GB", | 20260505 10:41:38.095 | INFO | "pinned" : " 29 GB" | 20260505 10:41:38.095 | INFO | } | 20260505 10:41:38.095 | INFO | Perf report for the last 5.14822 seconds: | 20260505 10:41:38.095 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:41:38.095 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:38.095 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:38.095 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:38.095 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 10:41:38.095 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:38.095 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:38.095 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:41:38.095 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:38.095 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:38.096 | INFO | Uploaded 36GiB to GPU in 2.22283s (16.1956GiB/s) | 20260505 10:41:38.096 | INFO | Downloaded 34GiB to GPU in 2.10157s (16.1784GiB/s) | 20260505 10:41:43.233 | INFO | Analyzing chunk frames = [2571264,2571776) | 20260505 10:41:43.233 | INFO | MemInfo: { | 20260505 10:41:43.233 | INFO | "gpu" : " 17 GB", | 20260505 10:41:43.233 | INFO | "huge" : "", | 20260505 10:41:43.233 | INFO | "huge_shared" : " 27 GB", | 20260505 10:41:43.233 | INFO | "malloc" : " 14 GB", | 20260505 10:41:43.233 | INFO | "pinned" : " 29 GB" | 20260505 10:41:43.233 | INFO | } | 20260505 10:41:43.233 | INFO | Perf report for the last 5.13741 seconds: | 20260505 10:41:43.233 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:41:43.233 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:43.233 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:43.233 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:43.233 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 10:41:43.233 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:43.233 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:43.233 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 10:41:43.233 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:43.233 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:43.233 | INFO | Uploaded 36GiB to GPU in 2.16936s (16.5948GiB/s) | 20260505 10:41:43.233 | INFO | Downloaded 34GiB to GPU in 2.13825s (15.9009GiB/s) | 20260505 10:41:48.368 | INFO | Analyzing chunk frames = [2571776,2572288) | 20260505 10:41:48.368 | INFO | MemInfo: { | 20260505 10:41:48.368 | INFO | "gpu" : " 17 GB", | 20260505 10:41:48.368 | INFO | "huge" : "", | 20260505 10:41:48.368 | INFO | "huge_shared" : " 27 GB", | 20260505 10:41:48.368 | INFO | "malloc" : " 14 GB", | 20260505 10:41:48.368 | INFO | "pinned" : " 29 GB" | 20260505 10:41:48.368 | INFO | } | 20260505 10:41:48.368 | INFO | Perf report for the last 5.13551 seconds: | 20260505 10:41:48.368 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:41:48.368 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:48.368 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:48.368 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:48.368 | INFO | ANALYSIS : 64.1%, 3.0, 100.0ms, 0 | 20260505 10:41:48.368 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:48.368 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:48.368 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 10:41:48.368 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:48.368 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:48.369 | INFO | Uploaded 36GiB to GPU in 2.22266s (16.1968GiB/s) | 20260505 10:41:48.369 | INFO | Downloaded 34GiB to GPU in 2.09718s (16.2123GiB/s) | 20260505 10:41:48.369 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 10:41:48.369 | INFO | Total mem usage: 15.3 GiB | 20260505 10:41:48.369 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 10:41:48.369 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -6%) | 20260505 10:41:48.369 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 10:41:48.369 | INFO | High Water Breakdown: | 20260505 10:41:48.369 | INFO | 61.1% : 8.84 GiB ( 93.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 10:41:48.369 | INFO | 38.7% : 5.6 GiB ( 84.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 10:41:48.369 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 10:41:48.369 | INFO | ------------------------------------------ | 20260505 10:41:48.369 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:41:48.369 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 10:41:48.369 | INFO | Total mem usage: 26.5 GiB | 20260505 10:41:48.369 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 10:41:48.369 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 10:41:48.369 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 10:41:48.369 | INFO | High Water Breakdown: | 20260505 10:41:48.369 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:41:48.369 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:41:48.369 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:41:48.369 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:41:48.369 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:41:48.369 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:41:48.369 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 10:41:48.369 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 10:41:48.369 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:41:48.369 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 10:41:48.369 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 10:41:48.369 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 10:41:48.369 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 10:41:48.369 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 10:41:48.369 | INFO | ------------------------------------------ | 20260505 10:41:48.369 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:41:48.369 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 10:41:48.369 | INFO | Total mem usage: 25.6 GiB | 20260505 10:41:48.369 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 10:41:48.369 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 10:41:48.369 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 10:41:48.369 | INFO | High Water Breakdown: | 20260505 10:41:48.369 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 10:41:48.369 | INFO | ------------------------------------------ | 20260505 10:41:48.369 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:41:48.369 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 10:41:48.369 | INFO | Total mem usage: 15.7 GiB | 20260505 10:41:48.369 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 10:41:48.369 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 10:41:48.369 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 10:41:48.369 | INFO | High Water Breakdown: | 20260505 10:41:48.369 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 10:41:48.369 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 10:41:48.369 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 10:41:48.369 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 10:41:48.369 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 10:41:48.369 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 10:41:48.369 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 10:41:48.369 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 10:41:48.369 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 10:41:48.369 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:41:48.369 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 10:41:48.369 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:41:48.369 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 10:41:48.369 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 10:41:48.369 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 10:41:48.369 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 10:41:48.369 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:41:48.369 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 10:41:48.369 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:41:48.369 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:41:48.369 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:41:48.369 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 10:41:48.369 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 10:41:48.369 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:41:48.369 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 10:41:48.369 | INFO | ------------------------------------------ | 20260505 10:41:48.369 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:41:48.369 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 10:41:48.369 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:41:48.369 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:41:48.369 | INFO | PerfReport: Full Execution -- 3.23e+01 / 7.68e+01, 1.39e+01 / 1.70e+00 | 20260505 10:41:48.369 | INFO | PerfReport: 0.29%, AggregateStats -- 9.31e-02 / 1.02e-01, 1.06e-02 / 3.93e-03 | 20260505 10:41:48.370 | INFO | PerfReport: 11.67%, Baseline -- 3.77e+00 / 4.46e+00, 6.01e-01 / 2.55e-01 | 20260505 10:41:48.370 | INFO | PerfReport: 3.31%, Histogram -- 1.07e+00 / 1.50e+00, 2.08e-01 / 8.92e-02 | 20260505 10:41:48.370 | INFO | PerfReport: 15.67%, DME -- 4.05e+01 / 4.45e+01, 4.21e+00 / 1.76e+00 | 20260505 10:41:48.370 | INFO | PerfReport: 56.60%, FrameLabeling -- 1.83e+01 / 2.14e+01, 1.78e+00 / 1.99e+00 | 20260505 10:41:48.370 | INFO | PerfReport: 7.37%, PulseAccumulating -- 2.38e+00 / 2.88e+00, 4.12e-01 / 2.21e-01 | 20260505 10:41:48.370 | INFO | PerfReport: 3.76%, Metrics -- 1.21e+00 / 1.48e+00, 3.48e-01 / 1.90e-01 | 20260505 10:41:48.370 | INFO | PerfReport: 1.32%, PulseSerializing -- 4.27e-01 / 4.71e-01, 7.87e-02 / 4.66e-02 | 20260505 10:41:48.370 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 10:41:48.370 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:41:48.370 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:41:48.370 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.45e+01 / 0.00e+00 | 20260505 10:41:48.370 | INFO | PerfReport: 22.67%, Upload -- 2.30e+01 / 0.00e+00, 8.35e+00 / 0.00e+00 | 20260505 10:41:48.370 | INFO | PerfReport: 21.87%, Download -- 2.22e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260505 10:41:48.370 | INFO | PerfReport: 31.85%, Compute -- 3.23e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 10:41:48.370 | INFO | PerfReport: 2.07%, LockUpload -- 2.09e+00 / 0.00e+00, 7.67e+00 / 0.00e+00 | 20260505 10:41:48.370 | INFO | PerfReport: 1.11%, LockDownload -- 1.13e+00 / 0.00e+00, 2.70e+00 / 0.00e+00 | 20260505 10:41:48.370 | INFO | PerfReport: 20.42%, LockCompute -- 2.07e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260505 10:41:48.370 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.28e-02 / 0.00e+00 | 20260505 10:41:48.511 | INFO | Starting a round of Baz IO | 20260505 10:41:51.770 | INFO | Populating RTMetrics report | 20260505 10:41:51.771 | INFO | RTMetrics report populated | 20260505 10:41:51.771 | INFO | Opening tmp RTMetrics json file | 20260505 10:41:51.792 | INFO | Serializing RTMetrics to json | 20260505 10:41:51.839 | INFO | Serialization done, swapping tmp file now... | 20260505 10:41:51.841 | INFO | RTMetrics done | 20260505 10:41:51.904 | INFO | Finished a round of Baz IO: Wrote 2.407 GiB in 3.393 seconds | 20260505 10:41:51.904 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.76% / 0.01% / 75.23% | 20260505 10:41:51.904 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.40486% / 0% / 5.5631e-05% | 20260505 10:41:51.904 | INFO | Baz IO overall: | 20260505 10:41:51.904 | INFO | BytesWritten : 1511051804080 | 20260505 10:41:51.904 | INFO | NumEvents : 719459199691 | 20260505 10:41:53.506 | INFO | Analyzing chunk frames = [2572288,2572800) | 20260505 10:41:53.506 | INFO | MemInfo: { | 20260505 10:41:53.506 | INFO | "gpu" : " 17 GB", | 20260505 10:41:53.506 | INFO | "huge" : "", | 20260505 10:41:53.506 | INFO | "huge_shared" : " 27 GB", | 20260505 10:41:53.506 | INFO | "malloc" : " 14 GB", | 20260505 10:41:53.506 | INFO | "pinned" : " 29 GB" | 20260505 10:41:53.506 | INFO | } | 20260505 10:41:53.506 | INFO | Perf report for the last 5.13766 seconds: | 20260505 10:41:53.506 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:41:53.506 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:53.506 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:53.506 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:53.506 | INFO | ANALYSIS : 65.1%, 2.9, 99.4ms, 0 | 20260505 10:41:53.506 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:53.506 | INFO | RT_METRICS : 47.3%, 1.0, 24.8ms, 0 | 20260505 10:41:53.506 | INFO | PRE_HQ : 62.5%, 1.8, 59.5ms, 0 | 20260505 10:41:53.506 | INFO | BAZWRITER : 65.7%, 4.2, 141.7ms, 0 | 20260505 10:41:53.506 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260505 10:41:53.506 | INFO | Uploaded 36GiB to GPU in 2.23789s (16.0866GiB/s) | 20260505 10:41:53.506 | INFO | Downloaded 38GiB to GPU in 2.34194s (16.2259GiB/s) | 20260505 10:41:58.647 | INFO | Analyzing chunk frames = [2572800,2573312) | 20260505 10:41:58.647 | INFO | MemInfo: { | 20260505 10:41:58.647 | INFO | "gpu" : " 17 GB", | 20260505 10:41:58.647 | INFO | "huge" : "", | 20260505 10:41:58.647 | INFO | "huge_shared" : " 27 GB", | 20260505 10:41:58.647 | INFO | "malloc" : " 14 GB", | 20260505 10:41:58.647 | INFO | "pinned" : " 29 GB" | 20260505 10:41:58.647 | INFO | } | 20260505 10:41:58.647 | INFO | Perf report for the last 5.14131 seconds: | 20260505 10:41:58.647 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:41:58.647 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:58.647 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:58.647 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:58.647 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 10:41:58.647 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:58.647 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:58.647 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:41:58.647 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:58.647 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:41:58.648 | INFO | Uploaded 36GiB to GPU in 2.18448s (16.4799GiB/s) | 20260505 10:41:58.648 | INFO | Downloaded 34GiB to GPU in 2.10235s (16.1724GiB/s) | 20260505 10:42:03.790 | INFO | Analyzing chunk frames = [2573312,2573824) | 20260505 10:42:03.790 | INFO | MemInfo: { | 20260505 10:42:03.790 | INFO | "gpu" : " 17 GB", | 20260505 10:42:03.790 | INFO | "huge" : "", | 20260505 10:42:03.790 | INFO | "huge_shared" : " 27 GB", | 20260505 10:42:03.790 | INFO | "malloc" : " 14 GB", | 20260505 10:42:03.790 | INFO | "pinned" : " 29 GB" | 20260505 10:42:03.790 | INFO | } | 20260505 10:42:03.790 | INFO | Perf report for the last 5.14286 seconds: | 20260505 10:42:03.790 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:42:03.790 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:03.790 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:03.790 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:03.790 | INFO | ANALYSIS : 63.7%, 3.0, 99.1ms, 0 | 20260505 10:42:03.790 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:03.790 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:03.790 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:42:03.790 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:03.790 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:03.790 | INFO | Uploaded 36GiB to GPU in 2.22997s (16.1437GiB/s) | 20260505 10:42:03.790 | INFO | Downloaded 34GiB to GPU in 2.12761s (15.9804GiB/s) | 20260505 10:42:08.932 | INFO | Analyzing chunk frames = [2573824,2574336) | 20260505 10:42:08.932 | INFO | MemInfo: { | 20260505 10:42:08.932 | INFO | "gpu" : " 17 GB", | 20260505 10:42:08.932 | INFO | "huge" : "", | 20260505 10:42:08.932 | INFO | "huge_shared" : " 27 GB", | 20260505 10:42:08.932 | INFO | "malloc" : " 14 GB", | 20260505 10:42:08.932 | INFO | "pinned" : " 29 GB" | 20260505 10:42:08.932 | INFO | } | 20260505 10:42:08.933 | INFO | Perf report for the last 5.14239 seconds: | 20260505 10:42:08.933 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:42:08.933 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:08.933 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:08.933 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:08.933 | INFO | ANALYSIS : 63.7%, 3.0, 99.0ms, 0 | 20260505 10:42:08.933 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:08.933 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:08.933 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:42:08.933 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:08.933 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:08.933 | INFO | Uploaded 36GiB to GPU in 2.22053s (16.2123GiB/s) | 20260505 10:42:08.933 | INFO | Downloaded 34GiB to GPU in 2.12173s (16.0247GiB/s) | 20260505 10:42:14.062 | INFO | Analyzing chunk frames = [2574336,2574848) | 20260505 10:42:14.062 | INFO | MemInfo: { | 20260505 10:42:14.062 | INFO | "gpu" : " 17 GB", | 20260505 10:42:14.062 | INFO | "huge" : "", | 20260505 10:42:14.062 | INFO | "huge_shared" : " 27 GB", | 20260505 10:42:14.062 | INFO | "malloc" : " 14 GB", | 20260505 10:42:14.062 | INFO | "pinned" : " 29 GB" | 20260505 10:42:14.062 | INFO | } | 20260505 10:42:14.062 | INFO | Perf report for the last 5.12963 seconds: | 20260505 10:42:14.062 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:42:14.062 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:14.062 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:14.062 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:14.062 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 10:42:14.062 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:14.062 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:14.062 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:42:14.062 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:14.062 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:14.062 | INFO | Uploaded 36GiB to GPU in 2.15544s (16.7019GiB/s) | 20260505 10:42:14.063 | INFO | Downloaded 34GiB to GPU in 2.12739s (15.982GiB/s) | 20260505 10:42:19.198 | INFO | Analyzing chunk frames = [2574848,2575360) | 20260505 10:42:19.198 | INFO | MemInfo: { | 20260505 10:42:19.198 | INFO | "gpu" : " 17 GB", | 20260505 10:42:19.198 | INFO | "huge" : "", | 20260505 10:42:19.198 | INFO | "huge_shared" : " 27 GB", | 20260505 10:42:19.198 | INFO | "malloc" : " 14 GB", | 20260505 10:42:19.198 | INFO | "pinned" : " 29 GB" | 20260505 10:42:19.198 | INFO | } | 20260505 10:42:19.198 | INFO | Perf report for the last 5.13544 seconds: | 20260505 10:42:19.198 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:42:19.198 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:19.198 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:19.198 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:19.198 | INFO | ANALYSIS : 64.0%, 3.0, 99.8ms, 0 | 20260505 10:42:19.198 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:19.198 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:19.198 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:42:19.198 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:19.198 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:19.198 | INFO | Uploaded 36GiB to GPU in 2.2331s (16.1211GiB/s) | 20260505 10:42:19.198 | INFO | Downloaded 34GiB to GPU in 2.10584s (16.1456GiB/s) | 20260505 10:42:24.344 | INFO | Analyzing chunk frames = [2575360,2575872) | 20260505 10:42:24.344 | INFO | MemInfo: { | 20260505 10:42:24.344 | INFO | "gpu" : " 17 GB", | 20260505 10:42:24.344 | INFO | "huge" : "", | 20260505 10:42:24.344 | INFO | "huge_shared" : " 27 GB", | 20260505 10:42:24.344 | INFO | "malloc" : " 14 GB", | 20260505 10:42:24.345 | INFO | "pinned" : " 29 GB" | 20260505 10:42:24.345 | INFO | } | 20260505 10:42:24.345 | INFO | Perf report for the last 5.14688 seconds: | 20260505 10:42:24.345 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:42:24.345 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:24.345 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:24.345 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:24.345 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 10:42:24.345 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:24.345 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:24.345 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 10:42:24.345 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:24.345 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:24.345 | INFO | Uploaded 36GiB to GPU in 2.19155s (16.4267GiB/s) | 20260505 10:42:24.345 | INFO | Downloaded 34GiB to GPU in 2.15479s (15.7788GiB/s) | 20260505 10:42:29.484 | INFO | Analyzing chunk frames = [2575872,2576384) | 20260505 10:42:29.484 | INFO | MemInfo: { | 20260505 10:42:29.484 | INFO | "gpu" : " 17 GB", | 20260505 10:42:29.484 | INFO | "huge" : "", | 20260505 10:42:29.484 | INFO | "huge_shared" : " 27 GB", | 20260505 10:42:29.484 | INFO | "malloc" : " 14 GB", | 20260505 10:42:29.484 | INFO | "pinned" : " 29 GB" | 20260505 10:42:29.484 | INFO | } | 20260505 10:42:29.484 | INFO | Perf report for the last 5.13946 seconds: | 20260505 10:42:29.484 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:42:29.484 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:29.484 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:29.484 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:29.484 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 10:42:29.484 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:29.484 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:29.484 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:42:29.484 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:29.484 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:29.484 | INFO | Uploaded 36GiB to GPU in 2.23404s (16.1143GiB/s) | 20260505 10:42:29.484 | INFO | Downloaded 34GiB to GPU in 2.09101s (16.2601GiB/s) | 20260505 10:42:29.623 | INFO | Starting a round of Baz IO | 20260505 10:42:32.885 | INFO | Populating RTMetrics report | 20260505 10:42:32.886 | INFO | RTMetrics report populated | 20260505 10:42:32.886 | INFO | Opening tmp RTMetrics json file | 20260505 10:42:32.901 | INFO | Serializing RTMetrics to json | 20260505 10:42:32.905 | INFO | Serialization done, swapping tmp file now... | 20260505 10:42:32.908 | INFO | RTMetrics done | 20260505 10:42:33.039 | INFO | Finished a round of Baz IO: Wrote 2.406 GiB in 3.415 seconds | 20260505 10:42:33.039 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.75% / 0.01% / 75.24% | 20260505 10:42:33.039 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.39553% / 2.38419e-05% / 7.94729e-05% | 20260505 10:42:33.039 | INFO | Baz IO overall: | 20260505 10:42:33.039 | INFO | BytesWritten : 1513635360078 | 20260505 10:42:33.039 | INFO | NumEvents : 720690584187 | 20260505 10:42:34.624 | INFO | Analyzing chunk frames = [2576384,2576896) | 20260505 10:42:34.624 | INFO | MemInfo: { | 20260505 10:42:34.624 | INFO | "gpu" : " 17 GB", | 20260505 10:42:34.624 | INFO | "huge" : "", | 20260505 10:42:34.624 | INFO | "huge_shared" : " 27 GB", | 20260505 10:42:34.624 | INFO | "malloc" : " 14 GB", | 20260505 10:42:34.624 | INFO | "pinned" : " 29 GB" | 20260505 10:42:34.624 | INFO | } | 20260505 10:42:34.624 | INFO | Perf report for the last 5.14014 seconds: | 20260505 10:42:34.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:42:34.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:34.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:34.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:34.624 | INFO | ANALYSIS : 65.0%, 2.9, 99.4ms, 0 | 20260505 10:42:34.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:34.624 | INFO | RT_METRICS : 46.2%, 1.0, 24.2ms, 0 | 20260505 10:42:34.624 | INFO | PRE_HQ : 61.7%, 1.8, 59.2ms, 0 | 20260505 10:42:34.624 | INFO | BAZWRITER : 65.8%, 4.2, 140.8ms, 0 | 20260505 10:42:34.624 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 10:42:34.624 | INFO | Uploaded 36GiB to GPU in 2.20429s (16.3318GiB/s) | 20260505 10:42:34.625 | INFO | Downloaded 38GiB to GPU in 2.31262s (16.4316GiB/s) | 20260505 10:42:39.765 | INFO | Analyzing chunk frames = [2576896,2577408) | 20260505 10:42:39.765 | INFO | MemInfo: { | 20260505 10:42:39.765 | INFO | "gpu" : " 17 GB", | 20260505 10:42:39.765 | INFO | "huge" : "", | 20260505 10:42:39.765 | INFO | "huge_shared" : " 27 GB", | 20260505 10:42:39.765 | INFO | "malloc" : " 14 GB", | 20260505 10:42:39.765 | INFO | "pinned" : " 29 GB" | 20260505 10:42:39.765 | INFO | } | 20260505 10:42:39.765 | INFO | Perf report for the last 5.14097 seconds: | 20260505 10:42:39.765 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:42:39.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:39.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:39.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:39.765 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 10:42:39.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:39.765 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:39.765 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:42:39.765 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:39.765 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:39.765 | INFO | Uploaded 36GiB to GPU in 2.17217s (16.5733GiB/s) | 20260505 10:42:39.765 | INFO | Downloaded 34GiB to GPU in 2.09325s (16.2427GiB/s) | 20260505 10:42:44.905 | INFO | Analyzing chunk frames = [2577408,2577920) | 20260505 10:42:44.905 | INFO | MemInfo: { | 20260505 10:42:44.905 | INFO | "gpu" : " 17 GB", | 20260505 10:42:44.905 | INFO | "huge" : "", | 20260505 10:42:44.905 | INFO | "huge_shared" : " 27 GB", | 20260505 10:42:44.905 | INFO | "malloc" : " 14 GB", | 20260505 10:42:44.905 | INFO | "pinned" : " 29 GB" | 20260505 10:42:44.905 | INFO | } | 20260505 10:42:44.905 | INFO | Perf report for the last 5.13976 seconds: | 20260505 10:42:44.905 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:42:44.905 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:44.905 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:44.905 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:44.905 | INFO | ANALYSIS : 63.9%, 3.0, 99.4ms, 0 | 20260505 10:42:44.905 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:44.905 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:44.905 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:42:44.905 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:44.905 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:44.905 | INFO | Uploaded 36GiB to GPU in 2.22922s (16.1492GiB/s) | 20260505 10:42:44.905 | INFO | Downloaded 34GiB to GPU in 2.12318s (16.0137GiB/s) | 20260505 10:42:50.042 | INFO | Analyzing chunk frames = [2577920,2578432) | 20260505 10:42:50.042 | INFO | MemInfo: { | 20260505 10:42:50.042 | INFO | "gpu" : " 17 GB", | 20260505 10:42:50.042 | INFO | "huge" : "", | 20260505 10:42:50.042 | INFO | "huge_shared" : " 27 GB", | 20260505 10:42:50.042 | INFO | "malloc" : " 14 GB", | 20260505 10:42:50.042 | INFO | "pinned" : " 29 GB" | 20260505 10:42:50.042 | INFO | } | 20260505 10:42:50.042 | INFO | Perf report for the last 5.1371 seconds: | 20260505 10:42:50.042 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:42:50.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:50.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:50.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:50.042 | INFO | ANALYSIS : 63.8%, 3.0, 99.0ms, 0 | 20260505 10:42:50.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:50.042 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:50.042 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 10:42:50.042 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:50.042 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:50.042 | INFO | Uploaded 36GiB to GPU in 2.22166s (16.2041GiB/s) | 20260505 10:42:50.042 | INFO | Downloaded 34GiB to GPU in 2.1279s (15.9782GiB/s) | 20260505 10:42:55.180 | INFO | Analyzing chunk frames = [2578432,2578944) | 20260505 10:42:55.180 | INFO | MemInfo: { | 20260505 10:42:55.180 | INFO | "gpu" : " 17 GB", | 20260505 10:42:55.180 | INFO | "huge" : "", | 20260505 10:42:55.180 | INFO | "huge_shared" : " 27 GB", | 20260505 10:42:55.180 | INFO | "malloc" : " 14 GB", | 20260505 10:42:55.180 | INFO | "pinned" : " 29 GB" | 20260505 10:42:55.180 | INFO | } | 20260505 10:42:55.180 | INFO | Perf report for the last 5.13804 seconds: | 20260505 10:42:55.180 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:42:55.180 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:55.180 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:55.180 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:55.180 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 10:42:55.180 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:55.180 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:55.180 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:42:55.180 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:55.180 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:42:55.180 | INFO | Uploaded 36GiB to GPU in 2.15577s (16.6994GiB/s) | 20260505 10:42:55.180 | INFO | Downloaded 34GiB to GPU in 2.12582s (15.9938GiB/s) | 20260505 10:43:00.316 | INFO | Analyzing chunk frames = [2578944,2579456) | 20260505 10:43:00.316 | INFO | MemInfo: { | 20260505 10:43:00.316 | INFO | "gpu" : " 17 GB", | 20260505 10:43:00.316 | INFO | "huge" : "", | 20260505 10:43:00.316 | INFO | "huge_shared" : " 27 GB", | 20260505 10:43:00.316 | INFO | "malloc" : " 14 GB", | 20260505 10:43:00.316 | INFO | "pinned" : " 29 GB" | 20260505 10:43:00.316 | INFO | } | 20260505 10:43:00.316 | INFO | Perf report for the last 5.1356 seconds: | 20260505 10:43:00.316 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:43:00.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:00.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:00.316 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:00.316 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 10:43:00.316 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:00.316 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:00.316 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:43:00.316 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:00.316 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:00.316 | INFO | Uploaded 36GiB to GPU in 2.23663s (16.0956GiB/s) | 20260505 10:43:00.316 | INFO | Downloaded 34GiB to GPU in 2.10392s (16.1603GiB/s) | 20260505 10:43:05.456 | INFO | Analyzing chunk frames = [2579456,2579968) | 20260505 10:43:05.456 | INFO | MemInfo: { | 20260505 10:43:05.456 | INFO | "gpu" : " 17 GB", | 20260505 10:43:05.456 | INFO | "huge" : "", | 20260505 10:43:05.456 | INFO | "huge_shared" : " 27 GB", | 20260505 10:43:05.456 | INFO | "malloc" : " 14 GB", | 20260505 10:43:05.456 | INFO | "pinned" : " 29 GB" | 20260505 10:43:05.456 | INFO | } | 20260505 10:43:05.457 | INFO | Perf report for the last 5.14074 seconds: | 20260505 10:43:05.457 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:43:05.457 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:05.457 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:05.457 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:05.457 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 10:43:05.457 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:05.457 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:05.457 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 10:43:05.457 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:05.457 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:05.457 | INFO | Uploaded 36GiB to GPU in 2.21009s (16.2889GiB/s) | 20260505 10:43:05.457 | INFO | Downloaded 34GiB to GPU in 2.15216s (15.7981GiB/s) | 20260505 10:43:10.603 | INFO | Analyzing chunk frames = [2579968,2580480) | 20260505 10:43:10.603 | INFO | MemInfo: { | 20260505 10:43:10.603 | INFO | "gpu" : " 17 GB", | 20260505 10:43:10.603 | INFO | "huge" : "", | 20260505 10:43:10.603 | INFO | "huge_shared" : " 27 GB", | 20260505 10:43:10.603 | INFO | "malloc" : " 14 GB", | 20260505 10:43:10.603 | INFO | "pinned" : " 29 GB" | 20260505 10:43:10.603 | INFO | } | 20260505 10:43:10.603 | INFO | Perf report for the last 5.14647 seconds: | 20260505 10:43:10.603 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:43:10.603 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:10.603 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:10.603 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:10.603 | INFO | ANALYSIS : 64.1%, 3.0, 100.3ms, 0 | 20260505 10:43:10.603 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:10.603 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:10.603 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 10:43:10.603 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:10.603 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:10.603 | INFO | Uploaded 36GiB to GPU in 2.22128s (16.2069GiB/s) | 20260505 10:43:10.603 | INFO | Downloaded 34GiB to GPU in 2.09473s (16.2312GiB/s) | 20260505 10:43:10.603 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 10:43:10.603 | INFO | Total mem usage: 15.3 GiB | 20260505 10:43:10.603 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 10:43:10.603 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -6.04%) | 20260505 10:43:10.604 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 10:43:10.604 | INFO | High Water Breakdown: | 20260505 10:43:10.604 | INFO | 61.1% : 8.83 GiB ( 93.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 10:43:10.604 | INFO | 38.8% : 5.6 GiB ( 84.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 10:43:10.604 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 10:43:10.604 | INFO | ------------------------------------------ | 20260505 10:43:10.604 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:43:10.604 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 10:43:10.604 | INFO | Total mem usage: 26.5 GiB | 20260505 10:43:10.604 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 10:43:10.604 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 10:43:10.604 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 10:43:10.604 | INFO | High Water Breakdown: | 20260505 10:43:10.604 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:43:10.604 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:43:10.604 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:43:10.604 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:43:10.604 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:43:10.604 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:43:10.604 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 10:43:10.604 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 10:43:10.604 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:43:10.604 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 10:43:10.604 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 10:43:10.604 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 10:43:10.604 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 10:43:10.604 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 10:43:10.604 | INFO | ------------------------------------------ | 20260505 10:43:10.604 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:43:10.604 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 10:43:10.604 | INFO | Total mem usage: 25.6 GiB | 20260505 10:43:10.604 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 10:43:10.604 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 10:43:10.604 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 10:43:10.604 | INFO | High Water Breakdown: | 20260505 10:43:10.604 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 10:43:10.604 | INFO | ------------------------------------------ | 20260505 10:43:10.604 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:43:10.604 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 10:43:10.604 | INFO | Total mem usage: 15.7 GiB | 20260505 10:43:10.604 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 10:43:10.604 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 10:43:10.604 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 10:43:10.604 | INFO | High Water Breakdown: | 20260505 10:43:10.604 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 10:43:10.604 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 10:43:10.604 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 10:43:10.604 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 10:43:10.604 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 10:43:10.604 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 10:43:10.604 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 10:43:10.604 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 10:43:10.604 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 10:43:10.604 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:43:10.604 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 10:43:10.604 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:43:10.604 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 10:43:10.604 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 10:43:10.604 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 10:43:10.604 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 10:43:10.604 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:43:10.604 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 10:43:10.604 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:43:10.604 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:43:10.604 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:43:10.604 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 10:43:10.604 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 10:43:10.604 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:43:10.604 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 10:43:10.604 | INFO | ------------------------------------------ | 20260505 10:43:10.604 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:43:10.604 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 10:43:10.604 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:43:10.604 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:43:10.604 | INFO | PerfReport: Full Execution -- 3.24e+01 / 7.82e+01, 1.40e+01 / 2.14e+00 | 20260505 10:43:10.604 | INFO | PerfReport: 0.29%, AggregateStats -- 9.32e-02 / 1.02e-01, 1.05e-02 / 4.79e-03 | 20260505 10:43:10.604 | INFO | PerfReport: 11.66%, Baseline -- 3.78e+00 / 4.45e+00, 6.11e-01 / 2.49e-01 | 20260505 10:43:10.604 | INFO | PerfReport: 3.32%, Histogram -- 1.08e+00 / 1.50e+00, 2.11e-01 / 1.00e-01 | 20260505 10:43:10.604 | INFO | PerfReport: 15.77%, DME -- 4.09e+01 / 4.53e+01, 4.51e+00 / 2.22e+00 | 20260505 10:43:10.604 | INFO | PerfReport: 56.63%, FrameLabeling -- 1.83e+01 / 2.20e+01, 1.81e+00 / 1.85e+00 | 20260505 10:43:10.604 | INFO | PerfReport: 7.29%, PulseAccumulating -- 2.36e+00 / 2.89e+00, 4.09e-01 / 2.41e-01 | 20260505 10:43:10.604 | INFO | PerfReport: 3.72%, Metrics -- 1.20e+00 / 1.47e+00, 3.51e-01 / 1.78e-01 | 20260505 10:43:10.604 | INFO | PerfReport: 1.32%, PulseSerializing -- 4.27e-01 / 4.66e-01, 8.08e-02 / 3.35e-02 | 20260505 10:43:10.604 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 10:43:10.604 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:43:10.604 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:43:10.604 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260505 10:43:10.604 | INFO | PerfReport: 22.65%, Upload -- 2.30e+01 / 0.00e+00, 8.35e+00 / 0.00e+00 | 20260505 10:43:10.604 | INFO | PerfReport: 21.99%, Download -- 2.23e+01 / 0.00e+00, 8.31e+00 / 0.00e+00 | 20260505 10:43:10.604 | INFO | PerfReport: 31.88%, Compute -- 3.24e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260505 10:43:10.604 | INFO | PerfReport: 2.02%, LockUpload -- 2.05e+00 / 0.00e+00, 7.50e+00 / 0.00e+00 | 20260505 10:43:10.604 | INFO | PerfReport: 1.13%, LockDownload -- 1.15e+00 / 0.00e+00, 2.73e+00 / 0.00e+00 | 20260505 10:43:10.604 | INFO | PerfReport: 20.32%, LockCompute -- 2.06e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260505 10:43:10.604 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.83e-02 / 0.00e+00 | 20260505 10:43:10.735 | INFO | Starting a round of Baz IO | 20260505 10:43:13.992 | INFO | Populating RTMetrics report | 20260505 10:43:13.992 | INFO | RTMetrics report populated | 20260505 10:43:13.992 | INFO | Opening tmp RTMetrics json file | 20260505 10:43:13.993 | INFO | Serializing RTMetrics to json | 20260505 10:43:13.994 | INFO | Serialization done, swapping tmp file now... | 20260505 10:43:13.994 | INFO | RTMetrics done | 20260505 10:43:14.247 | INFO | Finished a round of Baz IO: Wrote 2.403 GiB in 3.512 seconds | 20260505 10:43:14.248 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.74% / 0.01% / 75.25% | 20260505 10:43:14.248 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.37955% / 1.98682e-05% / 7.54992e-05% | 20260505 10:43:14.248 | INFO | Baz IO overall: | 20260505 10:43:14.248 | INFO | BytesWritten : 1516215955687 | 20260505 10:43:14.248 | INFO | NumEvents : 721920067352 | 20260505 10:43:15.743 | INFO | Analyzing chunk frames = [2580480,2580992) | 20260505 10:43:15.743 | INFO | MemInfo: { | 20260505 10:43:15.743 | INFO | "gpu" : " 17 GB", | 20260505 10:43:15.743 | INFO | "huge" : "", | 20260505 10:43:15.743 | INFO | "huge_shared" : " 27 GB", | 20260505 10:43:15.743 | INFO | "malloc" : " 14 GB", | 20260505 10:43:15.743 | INFO | "pinned" : " 29 GB" | 20260505 10:43:15.743 | INFO | } | 20260505 10:43:15.743 | INFO | Perf report for the last 5.13991 seconds: | 20260505 10:43:15.743 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:43:15.743 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:15.743 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:15.743 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:15.743 | INFO | ANALYSIS : 64.8%, 2.9, 99.3ms, 0 | 20260505 10:43:15.743 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:15.743 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260505 10:43:15.743 | INFO | PRE_HQ : 61.4%, 1.9, 59.1ms, 0 | 20260505 10:43:15.743 | INFO | BAZWRITER : 67.6%, 4.2, 144.8ms, 0 | 20260505 10:43:15.743 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 10:43:15.743 | INFO | Uploaded 36GiB to GPU in 2.19441s (16.4053GiB/s) | 20260505 10:43:15.743 | INFO | Downloaded 38GiB to GPU in 2.30257s (16.5033GiB/s) | 20260505 10:43:20.881 | INFO | Analyzing chunk frames = [2580992,2581504) | 20260505 10:43:20.881 | INFO | MemInfo: { | 20260505 10:43:20.881 | INFO | "gpu" : " 17 GB", | 20260505 10:43:20.881 | INFO | "huge" : "", | 20260505 10:43:20.881 | INFO | "huge_shared" : " 27 GB", | 20260505 10:43:20.882 | INFO | "malloc" : " 14 GB", | 20260505 10:43:20.882 | INFO | "pinned" : " 29 GB" | 20260505 10:43:20.882 | INFO | } | 20260505 10:43:20.882 | INFO | Perf report for the last 5.13864 seconds: | 20260505 10:43:20.882 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:43:20.882 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:20.882 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:20.882 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:20.882 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 10:43:20.882 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:20.882 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:20.882 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 10:43:20.882 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:20.882 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:20.882 | INFO | Uploaded 36GiB to GPU in 2.23446s (16.1113GiB/s) | 20260505 10:43:20.882 | INFO | Downloaded 34GiB to GPU in 2.08476s (16.3089GiB/s) | 20260505 10:43:26.010 | INFO | Analyzing chunk frames = [2581504,2582016) | 20260505 10:43:26.010 | INFO | MemInfo: { | 20260505 10:43:26.010 | INFO | "gpu" : " 17 GB", | 20260505 10:43:26.010 | INFO | "huge" : "", | 20260505 10:43:26.010 | INFO | "huge_shared" : " 27 GB", | 20260505 10:43:26.010 | INFO | "malloc" : " 14 GB", | 20260505 10:43:26.010 | INFO | "pinned" : " 29 GB" | 20260505 10:43:26.011 | INFO | } | 20260505 10:43:26.011 | INFO | Perf report for the last 5.12898 seconds: | 20260505 10:43:26.011 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:43:26.011 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:26.011 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:26.011 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:26.011 | INFO | ANALYSIS : 63.2%, 3.0, 98.1ms, 0 | 20260505 10:43:26.011 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:26.011 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:26.011 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:43:26.011 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:26.011 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:26.011 | INFO | Uploaded 36GiB to GPU in 2.15299s (16.7209GiB/s) | 20260505 10:43:26.011 | INFO | Downloaded 34GiB to GPU in 2.06156s (16.4924GiB/s) | 20260505 10:43:31.153 | INFO | Analyzing chunk frames = [2582016,2582528) | 20260505 10:43:31.153 | INFO | MemInfo: { | 20260505 10:43:31.153 | INFO | "gpu" : " 17 GB", | 20260505 10:43:31.153 | INFO | "huge" : "", | 20260505 10:43:31.153 | INFO | "huge_shared" : " 27 GB", | 20260505 10:43:31.154 | INFO | "malloc" : " 14 GB", | 20260505 10:43:31.154 | INFO | "pinned" : " 29 GB" | 20260505 10:43:31.154 | INFO | } | 20260505 10:43:31.154 | INFO | Perf report for the last 5.14303 seconds: | 20260505 10:43:31.154 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:43:31.154 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:31.154 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:31.154 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:31.154 | INFO | ANALYSIS : 62.9%, 3.0, 97.8ms, 0 | 20260505 10:43:31.154 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:31.154 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:31.154 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 10:43:31.154 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:31.154 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:31.154 | INFO | Uploaded 36GiB to GPU in 2.15132s (16.7339GiB/s) | 20260505 10:43:31.154 | INFO | Downloaded 34GiB to GPU in 2.07455s (16.3891GiB/s) | 20260505 10:43:36.295 | INFO | Analyzing chunk frames = [2582528,2583040) | 20260505 10:43:36.295 | INFO | MemInfo: { | 20260505 10:43:36.295 | INFO | "gpu" : " 17 GB", | 20260505 10:43:36.295 | INFO | "huge" : "", | 20260505 10:43:36.295 | INFO | "huge_shared" : " 27 GB", | 20260505 10:43:36.295 | INFO | "malloc" : " 14 GB", | 20260505 10:43:36.295 | INFO | "pinned" : " 29 GB" | 20260505 10:43:36.295 | INFO | } | 20260505 10:43:36.295 | INFO | Perf report for the last 5.14152 seconds: | 20260505 10:43:36.295 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:43:36.295 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:36.295 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:36.295 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:36.295 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 10:43:36.295 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:36.295 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:36.295 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 10:43:36.295 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:36.295 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:36.295 | INFO | Uploaded 36GiB to GPU in 2.13742s (16.8427GiB/s) | 20260505 10:43:36.295 | INFO | Downloaded 34GiB to GPU in 2.12115s (16.029GiB/s) | 20260505 10:43:41.441 | INFO | Analyzing chunk frames = [2583040,2583552) | 20260505 10:43:41.441 | INFO | MemInfo: { | 20260505 10:43:41.441 | INFO | "gpu" : " 17 GB", | 20260505 10:43:41.441 | INFO | "huge" : "", | 20260505 10:43:41.441 | INFO | "huge_shared" : " 27 GB", | 20260505 10:43:41.441 | INFO | "malloc" : " 14 GB", | 20260505 10:43:41.441 | INFO | "pinned" : " 29 GB" | 20260505 10:43:41.441 | INFO | } | 20260505 10:43:41.441 | INFO | Perf report for the last 5.14574 seconds: | 20260505 10:43:41.441 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:43:41.441 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:41.441 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:41.441 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:41.441 | INFO | ANALYSIS : 63.3%, 3.0, 99.0ms, 0 | 20260505 10:43:41.441 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:41.441 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:41.441 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 10:43:41.441 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:41.441 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:41.441 | INFO | Uploaded 36GiB to GPU in 2.1905s (16.4346GiB/s) | 20260505 10:43:41.441 | INFO | Downloaded 34GiB to GPU in 2.10675s (16.1386GiB/s) | 20260505 10:43:46.570 | INFO | Analyzing chunk frames = [2583552,2584064) | 20260505 10:43:46.570 | INFO | MemInfo: { | 20260505 10:43:46.570 | INFO | "gpu" : " 17 GB", | 20260505 10:43:46.570 | INFO | "huge" : "", | 20260505 10:43:46.570 | INFO | "huge_shared" : " 27 GB", | 20260505 10:43:46.570 | INFO | "malloc" : " 14 GB", | 20260505 10:43:46.570 | INFO | "pinned" : " 29 GB" | 20260505 10:43:46.570 | INFO | } | 20260505 10:43:46.570 | INFO | Perf report for the last 5.12918 seconds: | 20260505 10:43:46.570 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:43:46.570 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:46.570 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:46.570 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:46.570 | INFO | ANALYSIS : 64.0%, 3.0, 99.7ms, 0 | 20260505 10:43:46.570 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:46.570 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:46.570 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 10:43:46.570 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:46.570 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:46.570 | INFO | Uploaded 36GiB to GPU in 2.19861s (16.374GiB/s) | 20260505 10:43:46.570 | INFO | Downloaded 34GiB to GPU in 2.15087s (15.8076GiB/s) | 20260505 10:43:51.709 | INFO | Analyzing chunk frames = [2584064,2584576) | 20260505 10:43:51.709 | INFO | MemInfo: { | 20260505 10:43:51.709 | INFO | "gpu" : " 17 GB", | 20260505 10:43:51.709 | INFO | "huge" : "", | 20260505 10:43:51.709 | INFO | "huge_shared" : " 27 GB", | 20260505 10:43:51.709 | INFO | "malloc" : " 14 GB", | 20260505 10:43:51.709 | INFO | "pinned" : " 29 GB" | 20260505 10:43:51.709 | INFO | } | 20260505 10:43:51.709 | INFO | Perf report for the last 5.13936 seconds: | 20260505 10:43:51.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:43:51.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:51.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:51.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:51.709 | INFO | ANALYSIS : 64.5%, 3.0, 100.7ms, 0 | 20260505 10:43:51.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:51.709 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:51.709 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 10:43:51.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:51.710 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:51.710 | INFO | Uploaded 36GiB to GPU in 2.24521s (16.0342GiB/s) | 20260505 10:43:51.710 | INFO | Downloaded 34GiB to GPU in 2.09976s (16.1923GiB/s) | 20260505 10:43:51.850 | INFO | Starting a round of Baz IO | 20260505 10:43:55.112 | INFO | Populating RTMetrics report | 20260505 10:43:55.112 | INFO | RTMetrics report populated | 20260505 10:43:55.112 | INFO | Opening tmp RTMetrics json file | 20260505 10:43:55.129 | INFO | Serializing RTMetrics to json | 20260505 10:43:55.131 | INFO | Serialization done, swapping tmp file now... | 20260505 10:43:55.132 | INFO | RTMetrics done | 20260505 10:43:55.255 | INFO | Finished a round of Baz IO: Wrote 2.401 GiB in 3.405 seconds | 20260505 10:43:55.255 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.73% / 0.01% / 75.26% | 20260505 10:43:55.255 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.3745% / 3.97364e-06% / 9.53674e-05% | 20260505 10:43:55.255 | INFO | Baz IO overall: | 20260505 10:43:55.256 | INFO | BytesWritten : 1518793583435 | 20260505 10:43:55.256 | INFO | NumEvents : 723147662955 | 20260505 10:43:56.859 | INFO | Analyzing chunk frames = [2584576,2585088) | 20260505 10:43:56.859 | INFO | MemInfo: { | 20260505 10:43:56.859 | INFO | "gpu" : " 17 GB", | 20260505 10:43:56.859 | INFO | "huge" : "", | 20260505 10:43:56.859 | INFO | "huge_shared" : " 27 GB", | 20260505 10:43:56.859 | INFO | "malloc" : " 14 GB", | 20260505 10:43:56.859 | INFO | "pinned" : " 29 GB" | 20260505 10:43:56.859 | INFO | } | 20260505 10:43:56.859 | INFO | Perf report for the last 5.14968 seconds: | 20260505 10:43:56.859 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:43:56.859 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:56.859 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:56.859 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:56.859 | INFO | ANALYSIS : 65.0%, 2.9, 100.1ms, 0 | 20260505 10:43:56.859 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:43:56.859 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260505 10:43:56.859 | INFO | PRE_HQ : 61.5%, 1.8, 59.0ms, 0 | 20260505 10:43:56.859 | INFO | BAZWRITER : 65.4%, 4.1, 138.9ms, 0 | 20260505 10:43:56.859 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 10:43:56.859 | INFO | Uploaded 36GiB to GPU in 2.19358s (16.4115GiB/s) | 20260505 10:43:56.859 | INFO | Downloaded 38GiB to GPU in 2.31235s (16.4335GiB/s) | 20260505 10:44:01.996 | INFO | Analyzing chunk frames = [2585088,2585600) | 20260505 10:44:01.996 | INFO | MemInfo: { | 20260505 10:44:01.996 | INFO | "gpu" : " 17 GB", | 20260505 10:44:01.996 | INFO | "huge" : "", | 20260505 10:44:01.996 | INFO | "huge_shared" : " 27 GB", | 20260505 10:44:01.996 | INFO | "malloc" : " 14 GB", | 20260505 10:44:01.996 | INFO | "pinned" : " 29 GB" | 20260505 10:44:01.996 | INFO | } | 20260505 10:44:01.997 | INFO | Perf report for the last 5.13737 seconds: | 20260505 10:44:01.997 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:44:01.997 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:01.997 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:01.997 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:01.997 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 10:44:01.997 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:01.997 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:01.997 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:44:01.997 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:01.997 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:01.997 | INFO | Uploaded 36GiB to GPU in 2.23631s (16.0979GiB/s) | 20260505 10:44:01.997 | INFO | Downloaded 34GiB to GPU in 2.08866s (16.2784GiB/s) | 20260505 10:44:07.125 | INFO | Analyzing chunk frames = [2585600,2586112) | 20260505 10:44:07.125 | INFO | MemInfo: { | 20260505 10:44:07.125 | INFO | "gpu" : " 17 GB", | 20260505 10:44:07.125 | INFO | "huge" : "", | 20260505 10:44:07.125 | INFO | "huge_shared" : " 27 GB", | 20260505 10:44:07.125 | INFO | "malloc" : " 14 GB", | 20260505 10:44:07.126 | INFO | "pinned" : " 29 GB" | 20260505 10:44:07.126 | INFO | } | 20260505 10:44:07.126 | INFO | Perf report for the last 5.12904 seconds: | 20260505 10:44:07.126 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:44:07.126 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:07.126 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:07.126 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:07.126 | INFO | ANALYSIS : 64.0%, 3.0, 99.4ms, 0 | 20260505 10:44:07.126 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:07.126 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:07.126 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 10:44:07.126 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:07.126 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:07.126 | INFO | Uploaded 36GiB to GPU in 2.229s (16.1507GiB/s) | 20260505 10:44:07.126 | INFO | Downloaded 34GiB to GPU in 2.12208s (16.022GiB/s) | 20260505 10:44:12.266 | INFO | Analyzing chunk frames = [2586112,2586624) | 20260505 10:44:12.266 | INFO | MemInfo: { | 20260505 10:44:12.266 | INFO | "gpu" : " 17 GB", | 20260505 10:44:12.266 | INFO | "huge" : "", | 20260505 10:44:12.266 | INFO | "huge_shared" : " 27 GB", | 20260505 10:44:12.266 | INFO | "malloc" : " 14 GB", | 20260505 10:44:12.266 | INFO | "pinned" : " 29 GB" | 20260505 10:44:12.266 | INFO | } | 20260505 10:44:12.266 | INFO | Perf report for the last 5.1409 seconds: | 20260505 10:44:12.266 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:44:12.266 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:12.266 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:12.267 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:12.267 | INFO | ANALYSIS : 63.0%, 3.0, 97.9ms, 0 | 20260505 10:44:12.267 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:12.267 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:12.267 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 10:44:12.267 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:12.267 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:12.267 | INFO | Uploaded 36GiB to GPU in 2.17452s (16.5554GiB/s) | 20260505 10:44:12.267 | INFO | Downloaded 34GiB to GPU in 2.0757s (16.38GiB/s) | 20260505 10:44:17.412 | INFO | Analyzing chunk frames = [2586624,2587136) | 20260505 10:44:17.412 | INFO | MemInfo: { | 20260505 10:44:17.412 | INFO | "gpu" : " 17 GB", | 20260505 10:44:17.412 | INFO | "huge" : "", | 20260505 10:44:17.412 | INFO | "huge_shared" : " 27 GB", | 20260505 10:44:17.412 | INFO | "malloc" : " 14 GB", | 20260505 10:44:17.412 | INFO | "pinned" : " 29 GB" | 20260505 10:44:17.412 | INFO | } | 20260505 10:44:17.412 | INFO | Perf report for the last 5.14566 seconds: | 20260505 10:44:17.412 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:44:17.412 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:17.412 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:17.412 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:17.412 | INFO | ANALYSIS : 63.5%, 3.0, 99.3ms, 0 | 20260505 10:44:17.412 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:17.412 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:17.412 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:44:17.412 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:17.412 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:17.412 | INFO | Uploaded 36GiB to GPU in 2.23851s (16.0821GiB/s) | 20260505 10:44:17.413 | INFO | Downloaded 34GiB to GPU in 2.11152s (16.1021GiB/s) | 20260505 10:44:22.551 | INFO | Analyzing chunk frames = [2587136,2587648) | 20260505 10:44:22.551 | INFO | MemInfo: { | 20260505 10:44:22.551 | INFO | "gpu" : " 17 GB", | 20260505 10:44:22.551 | INFO | "huge" : "", | 20260505 10:44:22.551 | INFO | "huge_shared" : " 27 GB", | 20260505 10:44:22.551 | INFO | "malloc" : " 14 GB", | 20260505 10:44:22.551 | INFO | "pinned" : " 29 GB" | 20260505 10:44:22.551 | INFO | } | 20260505 10:44:22.551 | INFO | Perf report for the last 5.13864 seconds: | 20260505 10:44:22.551 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:44:22.551 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:22.551 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:22.551 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:22.551 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 10:44:22.551 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:22.551 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:22.551 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:44:22.551 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:22.551 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:22.551 | INFO | Uploaded 36GiB to GPU in 2.19644s (16.3902GiB/s) | 20260505 10:44:22.551 | INFO | Downloaded 34GiB to GPU in 2.10254s (16.1709GiB/s) | 20260505 10:44:27.694 | INFO | Analyzing chunk frames = [2587648,2588160) | 20260505 10:44:27.694 | INFO | MemInfo: { | 20260505 10:44:27.694 | INFO | "gpu" : " 17 GB", | 20260505 10:44:27.694 | INFO | "huge" : "", | 20260505 10:44:27.694 | INFO | "huge_shared" : " 27 GB", | 20260505 10:44:27.694 | INFO | "malloc" : " 14 GB", | 20260505 10:44:27.694 | INFO | "pinned" : " 29 GB" | 20260505 10:44:27.694 | INFO | } | 20260505 10:44:27.694 | INFO | Perf report for the last 5.14332 seconds: | 20260505 10:44:27.694 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:44:27.694 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:27.694 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:27.694 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:27.694 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 10:44:27.694 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:27.694 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:27.694 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 10:44:27.694 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:27.694 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:27.694 | INFO | Uploaded 36GiB to GPU in 2.16406s (16.6354GiB/s) | 20260505 10:44:27.694 | INFO | Downloaded 34GiB to GPU in 2.15013s (15.813GiB/s) | 20260505 10:44:32.826 | INFO | Analyzing chunk frames = [2588160,2588672) | 20260505 10:44:32.826 | INFO | MemInfo: { | 20260505 10:44:32.826 | INFO | "gpu" : " 17 GB", | 20260505 10:44:32.826 | INFO | "huge" : "", | 20260505 10:44:32.826 | INFO | "huge_shared" : " 27 GB", | 20260505 10:44:32.826 | INFO | "malloc" : " 14 GB", | 20260505 10:44:32.826 | INFO | "pinned" : " 29 GB" | 20260505 10:44:32.826 | INFO | } | 20260505 10:44:32.826 | INFO | Perf report for the last 5.13207 seconds: | 20260505 10:44:32.826 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:44:32.826 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:32.826 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:32.826 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:32.826 | INFO | ANALYSIS : 64.1%, 3.0, 100.0ms, 0 | 20260505 10:44:32.826 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:32.826 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:32.826 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 10:44:32.826 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:32.826 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:32.827 | INFO | Uploaded 36GiB to GPU in 2.22461s (16.1826GiB/s) | 20260505 10:44:32.827 | INFO | Downloaded 34GiB to GPU in 2.09582s (16.2228GiB/s) | 20260505 10:44:32.827 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 10:44:32.827 | INFO | Total mem usage: 15.3 GiB | 20260505 10:44:32.827 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 10:44:32.827 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -4.79%) | 20260505 10:44:32.827 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 10:44:32.827 | INFO | High Water Breakdown: | 20260505 10:44:32.827 | INFO | 61.2% : 8.95 GiB ( 94.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 10:44:32.827 | INFO | 38.7% : 5.65 GiB ( 85.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 10:44:32.827 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 10:44:32.827 | INFO | ------------------------------------------ | 20260505 10:44:32.827 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:44:32.827 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 10:44:32.827 | INFO | Total mem usage: 26.5 GiB | 20260505 10:44:32.827 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 10:44:32.827 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 10:44:32.827 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 10:44:32.827 | INFO | High Water Breakdown: | 20260505 10:44:32.827 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:44:32.827 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:44:32.827 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:44:32.827 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:44:32.827 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:44:32.827 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:44:32.827 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 10:44:32.827 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 10:44:32.827 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:44:32.827 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 10:44:32.827 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 10:44:32.827 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 10:44:32.827 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 10:44:32.827 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 10:44:32.827 | INFO | ------------------------------------------ | 20260505 10:44:32.827 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:44:32.827 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 10:44:32.827 | INFO | Total mem usage: 25.6 GiB | 20260505 10:44:32.827 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 10:44:32.827 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 10:44:32.827 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 10:44:32.827 | INFO | High Water Breakdown: | 20260505 10:44:32.827 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 10:44:32.827 | INFO | ------------------------------------------ | 20260505 10:44:32.827 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:44:32.827 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 10:44:32.827 | INFO | Total mem usage: 15.7 GiB | 20260505 10:44:32.827 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 10:44:32.827 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 10:44:32.827 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 10:44:32.827 | INFO | High Water Breakdown: | 20260505 10:44:32.827 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 10:44:32.827 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 10:44:32.827 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 10:44:32.828 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 10:44:32.828 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 10:44:32.828 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 10:44:32.828 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 10:44:32.828 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 10:44:32.828 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 10:44:32.828 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:44:32.828 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 10:44:32.828 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:44:32.828 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 10:44:32.828 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 10:44:32.828 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 10:44:32.828 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 10:44:32.828 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:44:32.828 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 10:44:32.828 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:44:32.828 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:44:32.828 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:44:32.828 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 10:44:32.828 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 10:44:32.828 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:44:32.828 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 10:44:32.828 | INFO | ------------------------------------------ | 20260505 10:44:32.828 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:44:32.828 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 10:44:32.828 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:44:32.828 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:44:32.828 | INFO | PerfReport: Full Execution -- 3.23e+01 / 7.72e+01, 1.39e+01 / 1.76e+00 | 20260505 10:44:32.828 | INFO | PerfReport: 0.29%, AggregateStats -- 9.32e-02 / 1.02e-01, 1.07e-02 / 4.49e-03 | 20260505 10:44:32.828 | INFO | PerfReport: 11.70%, Baseline -- 3.78e+00 / 4.52e+00, 6.08e-01 / 1.76e-01 | 20260505 10:44:32.828 | INFO | PerfReport: 3.30%, Histogram -- 1.07e+00 / 1.49e+00, 2.07e-01 / 9.17e-02 | 20260505 10:44:32.828 | INFO | PerfReport: 15.69%, DME -- 4.05e+01 / 4.46e+01, 4.21e+00 / 1.53e+00 | 20260505 10:44:32.828 | INFO | PerfReport: 56.62%, FrameLabeling -- 1.83e+01 / 2.17e+01, 1.81e+00 / 2.08e+00 | 20260505 10:44:32.828 | INFO | PerfReport: 7.35%, PulseAccumulating -- 2.37e+00 / 2.87e+00, 4.12e-01 / 2.17e-01 | 20260505 10:44:32.828 | INFO | PerfReport: 3.73%, Metrics -- 1.20e+00 / 1.51e+00, 3.46e-01 / 2.18e-01 | 20260505 10:44:32.828 | INFO | PerfReport: 1.32%, PulseSerializing -- 4.25e-01 / 4.72e-01, 7.82e-02 / 4.23e-02 | 20260505 10:44:32.828 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 10:44:32.828 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:44:32.828 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:44:32.828 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.46e+01 / 0.00e+00 | 20260505 10:44:32.828 | INFO | PerfReport: 22.58%, Upload -- 2.29e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260505 10:44:32.828 | INFO | PerfReport: 21.87%, Download -- 2.22e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260505 10:44:32.828 | INFO | PerfReport: 31.84%, Compute -- 3.23e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 10:44:32.828 | INFO | PerfReport: 2.03%, LockUpload -- 2.06e+00 / 0.00e+00, 7.60e+00 / 0.00e+00 | 20260505 10:44:32.828 | INFO | PerfReport: 1.16%, LockDownload -- 1.18e+00 / 0.00e+00, 2.76e+00 / 0.00e+00 | 20260505 10:44:32.828 | INFO | PerfReport: 20.50%, LockCompute -- 2.08e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260505 10:44:32.828 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 2.21e-02 / 0.00e+00 | 20260505 10:44:32.955 | INFO | Starting a round of Baz IO | 20260505 10:44:36.213 | INFO | Populating RTMetrics report | 20260505 10:44:36.213 | INFO | RTMetrics report populated | 20260505 10:44:36.213 | INFO | Opening tmp RTMetrics json file | 20260505 10:44:36.217 | INFO | Serializing RTMetrics to json | 20260505 10:44:36.224 | INFO | Serialization done, swapping tmp file now... | 20260505 10:44:36.241 | INFO | RTMetrics done | 20260505 10:44:36.352 | INFO | Finished a round of Baz IO: Wrote 2.4 GiB in 3.396 seconds | 20260505 10:44:36.352 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.72% / 0.01% / 75.27% | 20260505 10:44:36.352 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.36918% / 1.58946e-05% / 9.13938e-05% | 20260505 10:44:36.352 | INFO | Baz IO overall: | 20260505 10:44:36.352 | INFO | BytesWritten : 1521370603826 | 20260505 10:44:36.352 | INFO | NumEvents : 724374857587 | 20260505 10:44:37.961 | INFO | Analyzing chunk frames = [2588672,2589184) | 20260505 10:44:37.961 | INFO | MemInfo: { | 20260505 10:44:37.961 | INFO | "gpu" : " 17 GB", | 20260505 10:44:37.961 | INFO | "huge" : "", | 20260505 10:44:37.961 | INFO | "huge_shared" : " 27 GB", | 20260505 10:44:37.961 | INFO | "malloc" : " 14 GB", | 20260505 10:44:37.961 | INFO | "pinned" : " 29 GB" | 20260505 10:44:37.961 | INFO | } | 20260505 10:44:37.961 | INFO | Perf report for the last 5.13445 seconds: | 20260505 10:44:37.961 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:44:37.961 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:37.961 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:37.961 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:37.961 | INFO | ANALYSIS : 64.8%, 2.9, 99.4ms, 0 | 20260505 10:44:37.961 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:37.961 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 10:44:37.961 | INFO | PRE_HQ : 62.6%, 1.8, 59.2ms, 0 | 20260505 10:44:37.961 | INFO | BAZWRITER : 65.4%, 4.2, 140.4ms, 0 | 20260505 10:44:37.961 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 10:44:37.961 | INFO | Uploaded 36GiB to GPU in 2.25714s (15.9494GiB/s) | 20260505 10:44:37.961 | INFO | Downloaded 38GiB to GPU in 2.37307s (16.013GiB/s) | 20260505 10:44:43.111 | INFO | Analyzing chunk frames = [2589184,2589696) | 20260505 10:44:43.111 | INFO | MemInfo: { | 20260505 10:44:43.111 | INFO | "gpu" : " 17 GB", | 20260505 10:44:43.111 | INFO | "huge" : "", | 20260505 10:44:43.111 | INFO | "huge_shared" : " 27 GB", | 20260505 10:44:43.111 | INFO | "malloc" : " 14 GB", | 20260505 10:44:43.111 | INFO | "pinned" : " 29 GB" | 20260505 10:44:43.111 | INFO | } | 20260505 10:44:43.111 | INFO | Perf report for the last 5.15016 seconds: | 20260505 10:44:43.111 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:44:43.111 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:43.111 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:43.111 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:43.111 | INFO | ANALYSIS : 64.0%, 3.0, 100.2ms, 0 | 20260505 10:44:43.111 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:43.111 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:43.111 | INFO | PRE_HQ : 28.9%, 1.0, 15.1ms, 0 | 20260505 10:44:43.111 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:43.111 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:43.111 | INFO | Uploaded 36GiB to GPU in 2.17943s (16.5181GiB/s) | 20260505 10:44:43.111 | INFO | Downloaded 34GiB to GPU in 2.08605s (16.2987GiB/s) | 20260505 10:44:48.252 | INFO | Analyzing chunk frames = [2589696,2590208) | 20260505 10:44:48.252 | INFO | MemInfo: { | 20260505 10:44:48.252 | INFO | "gpu" : " 17 GB", | 20260505 10:44:48.252 | INFO | "huge" : "", | 20260505 10:44:48.252 | INFO | "huge_shared" : " 27 GB", | 20260505 10:44:48.252 | INFO | "malloc" : " 14 GB", | 20260505 10:44:48.252 | INFO | "pinned" : " 29 GB" | 20260505 10:44:48.252 | INFO | } | 20260505 10:44:48.252 | INFO | Perf report for the last 5.14147 seconds: | 20260505 10:44:48.252 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:44:48.252 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:48.252 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:48.252 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:48.252 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 10:44:48.252 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:48.252 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:48.252 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:44:48.253 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:48.253 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:48.253 | INFO | Uploaded 36GiB to GPU in 2.21411s (16.2594GiB/s) | 20260505 10:44:48.253 | INFO | Downloaded 34GiB to GPU in 2.1198s (16.0393GiB/s) | 20260505 10:44:53.388 | INFO | Analyzing chunk frames = [2590208,2590720) | 20260505 10:44:53.388 | INFO | MemInfo: { | 20260505 10:44:53.388 | INFO | "gpu" : " 17 GB", | 20260505 10:44:53.388 | INFO | "huge" : "", | 20260505 10:44:53.388 | INFO | "huge_shared" : " 27 GB", | 20260505 10:44:53.388 | INFO | "malloc" : " 14 GB", | 20260505 10:44:53.388 | INFO | "pinned" : " 29 GB" | 20260505 10:44:53.388 | INFO | } | 20260505 10:44:53.388 | INFO | Perf report for the last 5.13608 seconds: | 20260505 10:44:53.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:44:53.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:53.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:53.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:53.389 | INFO | ANALYSIS : 63.7%, 3.0, 99.0ms, 0 | 20260505 10:44:53.389 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:53.389 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:53.389 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:44:53.389 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:53.389 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:53.389 | INFO | Uploaded 36GiB to GPU in 2.22368s (16.1894GiB/s) | 20260505 10:44:53.389 | INFO | Downloaded 34GiB to GPU in 2.12277s (16.0168GiB/s) | 20260505 10:44:58.521 | INFO | Analyzing chunk frames = [2590720,2591232) | 20260505 10:44:58.521 | INFO | MemInfo: { | 20260505 10:44:58.521 | INFO | "gpu" : " 17 GB", | 20260505 10:44:58.521 | INFO | "huge" : "", | 20260505 10:44:58.521 | INFO | "huge_shared" : " 27 GB", | 20260505 10:44:58.521 | INFO | "malloc" : " 14 GB", | 20260505 10:44:58.521 | INFO | "pinned" : " 29 GB" | 20260505 10:44:58.521 | INFO | } | 20260505 10:44:58.521 | INFO | Perf report for the last 5.13261 seconds: | 20260505 10:44:58.521 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:44:58.521 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:58.521 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:58.521 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:58.521 | INFO | ANALYSIS : 63.6%, 3.0, 99.1ms, 0 | 20260505 10:44:58.521 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:58.521 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:58.521 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 10:44:58.521 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:58.521 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:44:58.521 | INFO | Uploaded 36GiB to GPU in 2.23138s (16.1335GiB/s) | 20260505 10:44:58.521 | INFO | Downloaded 34GiB to GPU in 2.10663s (16.1396GiB/s) | 20260505 10:45:03.665 | INFO | Analyzing chunk frames = [2591232,2591744) | 20260505 10:45:03.665 | INFO | MemInfo: { | 20260505 10:45:03.665 | INFO | "gpu" : " 17 GB", | 20260505 10:45:03.665 | INFO | "huge" : "", | 20260505 10:45:03.665 | INFO | "huge_shared" : " 27 GB", | 20260505 10:45:03.665 | INFO | "malloc" : " 14 GB", | 20260505 10:45:03.666 | INFO | "pinned" : " 29 GB" | 20260505 10:45:03.666 | INFO | } | 20260505 10:45:03.666 | INFO | Perf report for the last 5.14461 seconds: | 20260505 10:45:03.666 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:45:03.666 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:03.666 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:03.666 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:03.666 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 10:45:03.666 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:03.666 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:03.666 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:45:03.666 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:03.666 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:03.666 | INFO | Uploaded 36GiB to GPU in 2.23667s (16.0953GiB/s) | 20260505 10:45:03.666 | INFO | Downloaded 34GiB to GPU in 2.1107s (16.1084GiB/s) | 20260505 10:45:08.809 | INFO | Analyzing chunk frames = [2591744,2592256) | 20260505 10:45:08.809 | INFO | MemInfo: { | 20260505 10:45:08.809 | INFO | "gpu" : " 17 GB", | 20260505 10:45:08.809 | INFO | "huge" : "", | 20260505 10:45:08.809 | INFO | "huge_shared" : " 27 GB", | 20260505 10:45:08.809 | INFO | "malloc" : " 14 GB", | 20260505 10:45:08.809 | INFO | "pinned" : " 29 GB" | 20260505 10:45:08.809 | INFO | } | 20260505 10:45:08.809 | INFO | Perf report for the last 5.14344 seconds: | 20260505 10:45:08.809 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:45:08.809 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:08.809 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:08.809 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:08.809 | INFO | ANALYSIS : 63.7%, 3.0, 99.6ms, 0 | 20260505 10:45:08.809 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:08.809 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:08.809 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 10:45:08.809 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:08.809 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:08.809 | INFO | Uploaded 36GiB to GPU in 2.19815s (16.3774GiB/s) | 20260505 10:45:08.809 | INFO | Downloaded 34GiB to GPU in 2.14662s (15.8389GiB/s) | 20260505 10:45:13.945 | INFO | Analyzing chunk frames = [2592256,2592768) | 20260505 10:45:13.945 | INFO | MemInfo: { | 20260505 10:45:13.945 | INFO | "gpu" : " 17 GB", | 20260505 10:45:13.945 | INFO | "huge" : "", | 20260505 10:45:13.945 | INFO | "huge_shared" : " 27 GB", | 20260505 10:45:13.945 | INFO | "malloc" : " 14 GB", | 20260505 10:45:13.945 | INFO | "pinned" : " 29 GB" | 20260505 10:45:13.945 | INFO | } | 20260505 10:45:13.945 | INFO | Perf report for the last 5.13561 seconds: | 20260505 10:45:13.945 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:45:13.945 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:13.945 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:13.945 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:13.945 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 10:45:13.945 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:13.945 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:13.945 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 10:45:13.945 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:13.945 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:13.945 | INFO | Uploaded 36GiB to GPU in 2.22467s (16.1822GiB/s) | 20260505 10:45:13.945 | INFO | Downloaded 34GiB to GPU in 2.08047s (16.3425GiB/s) | 20260505 10:45:14.085 | INFO | Starting a round of Baz IO | 20260505 10:45:17.344 | INFO | Populating RTMetrics report | 20260505 10:45:17.345 | INFO | RTMetrics report populated | 20260505 10:45:17.345 | INFO | Opening tmp RTMetrics json file | 20260505 10:45:17.357 | INFO | Serializing RTMetrics to json | 20260505 10:45:17.357 | INFO | Serialization done, swapping tmp file now... | 20260505 10:45:17.358 | INFO | RTMetrics done | 20260505 10:45:17.478 | INFO | Finished a round of Baz IO: Wrote 2.4 GiB in 3.392 seconds | 20260505 10:45:17.478 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.71% / 0.01% / 75.28% | 20260505 10:45:17.478 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.36757% / 7.94729e-06% / 8.74201e-05% | 20260505 10:45:17.478 | INFO | Baz IO overall: | 20260505 10:45:17.478 | INFO | BytesWritten : 1523947713814 | 20260505 10:45:17.478 | INFO | NumEvents : 725602179651 | 20260505 10:45:19.075 | INFO | Analyzing chunk frames = [2592768,2593280) | 20260505 10:45:19.075 | INFO | MemInfo: { | 20260505 10:45:19.075 | INFO | "gpu" : " 17 GB", | 20260505 10:45:19.075 | INFO | "huge" : "", | 20260505 10:45:19.075 | INFO | "huge_shared" : " 27 GB", | 20260505 10:45:19.075 | INFO | "malloc" : " 14 GB", | 20260505 10:45:19.075 | INFO | "pinned" : " 29 GB" | 20260505 10:45:19.075 | INFO | } | 20260505 10:45:19.075 | INFO | Perf report for the last 5.13054 seconds: | 20260505 10:45:19.075 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:45:19.075 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:19.075 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:19.075 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:19.075 | INFO | ANALYSIS : 65.1%, 2.9, 99.8ms, 0 | 20260505 10:45:19.075 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:19.075 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260505 10:45:19.075 | INFO | PRE_HQ : 61.7%, 1.8, 59.0ms, 0 | 20260505 10:45:19.075 | INFO | BAZWRITER : 65.3%, 4.2, 139.2ms, 0 | 20260505 10:45:19.075 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 10:45:19.076 | INFO | Uploaded 36GiB to GPU in 2.19165s (16.426GiB/s) | 20260505 10:45:19.076 | INFO | Downloaded 38GiB to GPU in 2.30536s (16.4833GiB/s) | 20260505 10:45:24.217 | INFO | Analyzing chunk frames = [2593280,2593792) | 20260505 10:45:24.217 | INFO | MemInfo: { | 20260505 10:45:24.217 | INFO | "gpu" : " 17 GB", | 20260505 10:45:24.217 | INFO | "huge" : "", | 20260505 10:45:24.217 | INFO | "huge_shared" : " 27 GB", | 20260505 10:45:24.218 | INFO | "malloc" : " 14 GB", | 20260505 10:45:24.218 | INFO | "pinned" : " 29 GB" | 20260505 10:45:24.218 | INFO | } | 20260505 10:45:24.218 | INFO | Perf report for the last 5.14234 seconds: | 20260505 10:45:24.218 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:45:24.218 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:24.218 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:24.218 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:24.218 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 10:45:24.218 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:24.218 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:24.218 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 10:45:24.218 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:24.218 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:24.218 | INFO | Uploaded 36GiB to GPU in 2.23153s (16.1324GiB/s) | 20260505 10:45:24.218 | INFO | Downloaded 34GiB to GPU in 2.12074s (16.0321GiB/s) | 20260505 10:45:29.362 | INFO | Analyzing chunk frames = [2593792,2594304) | 20260505 10:45:29.362 | INFO | MemInfo: { | 20260505 10:45:29.362 | INFO | "gpu" : " 17 GB", | 20260505 10:45:29.362 | INFO | "huge" : "", | 20260505 10:45:29.362 | INFO | "huge_shared" : " 27 GB", | 20260505 10:45:29.362 | INFO | "malloc" : " 14 GB", | 20260505 10:45:29.363 | INFO | "pinned" : " 29 GB" | 20260505 10:45:29.363 | INFO | } | 20260505 10:45:29.363 | INFO | Perf report for the last 5.14504 seconds: | 20260505 10:45:29.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:45:29.363 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:29.363 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:29.363 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:29.363 | INFO | ANALYSIS : 63.8%, 3.0, 99.4ms, 0 | 20260505 10:45:29.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:29.363 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:29.363 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 10:45:29.363 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:29.363 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:29.363 | INFO | Uploaded 36GiB to GPU in 2.22988s (16.1444GiB/s) | 20260505 10:45:29.363 | INFO | Downloaded 34GiB to GPU in 2.12253s (16.0186GiB/s) | 20260505 10:45:34.515 | INFO | Analyzing chunk frames = [2594304,2594816) | 20260505 10:45:34.515 | INFO | MemInfo: { | 20260505 10:45:34.515 | INFO | "gpu" : " 17 GB", | 20260505 10:45:34.515 | INFO | "huge" : "", | 20260505 10:45:34.515 | INFO | "huge_shared" : " 27 GB", | 20260505 10:45:34.515 | INFO | "malloc" : " 14 GB", | 20260505 10:45:34.515 | INFO | "pinned" : " 29 GB" | 20260505 10:45:34.515 | INFO | } | 20260505 10:45:34.515 | INFO | Perf report for the last 5.15253 seconds: | 20260505 10:45:34.515 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:45:34.515 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:34.515 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:34.515 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:34.515 | INFO | ANALYSIS : 63.5%, 3.0, 99.0ms, 0 | 20260505 10:45:34.515 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:34.515 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:34.515 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:45:34.515 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:34.515 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:34.516 | INFO | Uploaded 36GiB to GPU in 2.22194s (16.202GiB/s) | 20260505 10:45:34.516 | INFO | Downloaded 34GiB to GPU in 2.12593s (15.993GiB/s) | 20260505 10:45:39.638 | INFO | Analyzing chunk frames = [2594816,2595328) | 20260505 10:45:39.638 | INFO | MemInfo: { | 20260505 10:45:39.638 | INFO | "gpu" : " 17 GB", | 20260505 10:45:39.638 | INFO | "huge" : "", | 20260505 10:45:39.638 | INFO | "huge_shared" : " 27 GB", | 20260505 10:45:39.638 | INFO | "malloc" : " 14 GB", | 20260505 10:45:39.638 | INFO | "pinned" : " 29 GB" | 20260505 10:45:39.638 | INFO | } | 20260505 10:45:39.638 | INFO | Perf report for the last 5.12248 seconds: | 20260505 10:45:39.638 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:45:39.638 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:39.638 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:39.638 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:39.638 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260505 10:45:39.638 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:39.638 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:39.638 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 10:45:39.638 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:39.638 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:39.638 | INFO | Uploaded 36GiB to GPU in 2.182s (16.4986GiB/s) | 20260505 10:45:39.638 | INFO | Downloaded 34GiB to GPU in 2.12587s (15.9934GiB/s) | 20260505 10:45:44.773 | INFO | Analyzing chunk frames = [2595328,2595840) | 20260505 10:45:44.773 | INFO | MemInfo: { | 20260505 10:45:44.773 | INFO | "gpu" : " 17 GB", | 20260505 10:45:44.773 | INFO | "huge" : "", | 20260505 10:45:44.773 | INFO | "huge_shared" : " 27 GB", | 20260505 10:45:44.773 | INFO | "malloc" : " 14 GB", | 20260505 10:45:44.773 | INFO | "pinned" : " 29 GB" | 20260505 10:45:44.773 | INFO | } | 20260505 10:45:44.773 | INFO | Perf report for the last 5.13567 seconds: | 20260505 10:45:44.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:45:44.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:44.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:44.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:44.774 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 10:45:44.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:44.774 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:44.774 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:45:44.774 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:44.774 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:44.774 | INFO | Uploaded 36GiB to GPU in 2.23493s (16.1079GiB/s) | 20260505 10:45:44.774 | INFO | Downloaded 34GiB to GPU in 2.10053s (16.1864GiB/s) | 20260505 10:45:49.921 | INFO | Analyzing chunk frames = [2595840,2596352) | 20260505 10:45:49.921 | INFO | MemInfo: { | 20260505 10:45:49.921 | INFO | "gpu" : " 17 GB", | 20260505 10:45:49.921 | INFO | "huge" : "", | 20260505 10:45:49.921 | INFO | "huge_shared" : " 27 GB", | 20260505 10:45:49.921 | INFO | "malloc" : " 14 GB", | 20260505 10:45:49.921 | INFO | "pinned" : " 29 GB" | 20260505 10:45:49.921 | INFO | } | 20260505 10:45:49.921 | INFO | Perf report for the last 5.1475 seconds: | 20260505 10:45:49.921 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:45:49.921 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:49.921 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:49.921 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:49.921 | INFO | ANALYSIS : 63.2%, 3.0, 98.9ms, 0 | 20260505 10:45:49.921 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:49.921 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:49.921 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 10:45:49.921 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:49.921 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:49.921 | INFO | Uploaded 36GiB to GPU in 2.16697s (16.6131GiB/s) | 20260505 10:45:49.921 | INFO | Downloaded 34GiB to GPU in 2.15102s (15.8064GiB/s) | 20260505 10:45:55.054 | INFO | Analyzing chunk frames = [2596352,2596864) | 20260505 10:45:55.054 | INFO | MemInfo: { | 20260505 10:45:55.054 | INFO | "gpu" : " 17 GB", | 20260505 10:45:55.054 | INFO | "huge" : "", | 20260505 10:45:55.054 | INFO | "huge_shared" : " 27 GB", | 20260505 10:45:55.054 | INFO | "malloc" : " 14 GB", | 20260505 10:45:55.054 | INFO | "pinned" : " 29 GB" | 20260505 10:45:55.054 | INFO | } | 20260505 10:45:55.054 | INFO | Perf report for the last 5.13345 seconds: | 20260505 10:45:55.054 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:45:55.054 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:55.054 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:55.054 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:55.054 | INFO | ANALYSIS : 64.1%, 3.0, 100.0ms, 0 | 20260505 10:45:55.054 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:55.054 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:55.055 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:45:55.055 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:55.055 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:45:55.055 | INFO | Uploaded 36GiB to GPU in 2.21585s (16.2466GiB/s) | 20260505 10:45:55.055 | INFO | Downloaded 34GiB to GPU in 2.0955s (16.2252GiB/s) | 20260505 10:45:55.055 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 10:45:55.055 | INFO | Total mem usage: 15.3 GiB | 20260505 10:45:55.055 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 10:45:55.055 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -6.03%) | 20260505 10:45:55.055 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 10:45:55.055 | INFO | High Water Breakdown: | 20260505 10:45:55.055 | INFO | 61.1% : 8.83 GiB ( 93.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 10:45:55.055 | INFO | 38.8% : 5.6 GiB ( 84.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 10:45:55.055 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 10:45:55.055 | INFO | ------------------------------------------ | 20260505 10:45:55.055 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:45:55.055 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 10:45:55.055 | INFO | Total mem usage: 26.5 GiB | 20260505 10:45:55.055 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 10:45:55.055 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 10:45:55.055 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 10:45:55.055 | INFO | High Water Breakdown: | 20260505 10:45:55.055 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:45:55.055 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:45:55.055 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:45:55.055 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:45:55.055 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:45:55.055 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:45:55.055 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 10:45:55.055 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 10:45:55.055 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:45:55.055 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 10:45:55.055 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 10:45:55.055 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 10:45:55.055 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 10:45:55.055 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 10:45:55.055 | INFO | ------------------------------------------ | 20260505 10:45:55.055 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:45:55.055 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 10:45:55.055 | INFO | Total mem usage: 25.6 GiB | 20260505 10:45:55.055 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 10:45:55.055 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 10:45:55.055 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 10:45:55.055 | INFO | High Water Breakdown: | 20260505 10:45:55.055 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 10:45:55.055 | INFO | ------------------------------------------ | 20260505 10:45:55.055 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:45:55.055 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 10:45:55.055 | INFO | Total mem usage: 15.7 GiB | 20260505 10:45:55.055 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 10:45:55.055 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 10:45:55.055 | INFO | Unused Cache: 880 MiB (Waste: 5.48%) | 20260505 10:45:55.055 | INFO | High Water Breakdown: | 20260505 10:45:55.055 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 10:45:55.055 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 10:45:55.055 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 10:45:55.055 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 10:45:55.055 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 10:45:55.055 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 10:45:55.055 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 10:45:55.055 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 10:45:55.055 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 10:45:55.055 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:45:55.055 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 10:45:55.055 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:45:55.055 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 10:45:55.055 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 10:45:55.055 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 10:45:55.055 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 10:45:55.055 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:45:55.055 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 10:45:55.055 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:45:55.055 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:45:55.055 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:45:55.055 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 10:45:55.055 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 10:45:55.055 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:45:55.056 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 10:45:55.056 | INFO | ------------------------------------------ | 20260505 10:45:55.056 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:45:55.056 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 10:45:55.056 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:45:55.056 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:45:55.056 | INFO | PerfReport: Full Execution -- 3.23e+01 / 7.80e+01, 1.40e+01 / 1.75e+00 | 20260505 10:45:55.056 | INFO | PerfReport: 0.29%, AggregateStats -- 9.33e-02 / 1.02e-01, 1.09e-02 / 5.26e-03 | 20260505 10:45:55.056 | INFO | PerfReport: 11.70%, Baseline -- 3.78e+00 / 4.48e+00, 6.15e-01 / 2.33e-01 | 20260505 10:45:55.056 | INFO | PerfReport: 3.32%, Histogram -- 1.07e+00 / 1.48e+00, 2.09e-01 / 1.07e-01 | 20260505 10:45:55.056 | INFO | PerfReport: 15.60%, DME -- 4.04e+01 / 4.42e+01, 4.20e+00 / 1.91e+00 | 20260505 10:45:55.056 | INFO | PerfReport: 56.70%, FrameLabeling -- 1.83e+01 / 2.29e+01, 1.81e+00 / 1.72e+00 | 20260505 10:45:55.056 | INFO | PerfReport: 7.32%, PulseAccumulating -- 2.36e+00 / 2.85e+00, 4.03e-01 / 2.19e-01 | 20260505 10:45:55.056 | INFO | PerfReport: 3.74%, Metrics -- 1.21e+00 / 1.49e+00, 3.52e-01 / 2.24e-01 | 20260505 10:45:55.056 | INFO | PerfReport: 1.32%, PulseSerializing -- 4.27e-01 / 4.67e-01, 7.97e-02 / 3.23e-02 | 20260505 10:45:55.056 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 10:45:55.056 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:45:55.056 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:45:55.056 | INFO | PerfReport: Full Execution -- 1.02e+02 / -nan, 2.46e+01 / 0.00e+00 | 20260505 10:45:55.056 | INFO | PerfReport: 22.73%, Upload -- 2.31e+01 / 0.00e+00, 8.38e+00 / 0.00e+00 | 20260505 10:45:55.056 | INFO | PerfReport: 21.99%, Download -- 2.23e+01 / 0.00e+00, 8.30e+00 / 0.00e+00 | 20260505 10:45:55.056 | INFO | PerfReport: 31.84%, Compute -- 3.23e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260505 10:45:55.056 | INFO | PerfReport: 2.02%, LockUpload -- 2.05e+00 / 0.00e+00, 7.51e+00 / 0.00e+00 | 20260505 10:45:55.056 | INFO | PerfReport: 1.12%, LockDownload -- 1.13e+00 / 0.00e+00, 2.68e+00 / 0.00e+00 | 20260505 10:45:55.056 | INFO | PerfReport: 20.29%, LockCompute -- 2.06e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260505 10:45:55.056 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.71e-02 / 0.00e+00 | 20260505 10:45:55.193 | INFO | Starting a round of Baz IO | 20260505 10:45:58.455 | INFO | Populating RTMetrics report | 20260505 10:45:58.455 | INFO | RTMetrics report populated | 20260505 10:45:58.455 | INFO | Opening tmp RTMetrics json file | 20260505 10:45:58.465 | INFO | Serializing RTMetrics to json | 20260505 10:45:58.490 | INFO | Serialization done, swapping tmp file now... | 20260505 10:45:58.492 | INFO | RTMetrics done | 20260505 10:45:58.576 | INFO | Finished a round of Baz IO: Wrote 2.4 GiB in 3.382 seconds | 20260505 10:45:58.576 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.70% / 0.01% / 75.29% | 20260505 10:45:58.576 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.36434% / 7.94729e-06% / 8.74201e-05% | 20260505 10:45:58.576 | INFO | Baz IO overall: | 20260505 10:45:58.576 | INFO | BytesWritten : 1526525210664 | 20260505 10:45:58.576 | INFO | NumEvents : 726829871976 | 20260505 10:46:00.198 | INFO | Analyzing chunk frames = [2596864,2597376) | 20260505 10:46:00.198 | INFO | MemInfo: { | 20260505 10:46:00.198 | INFO | "gpu" : " 17 GB", | 20260505 10:46:00.198 | INFO | "huge" : "", | 20260505 10:46:00.198 | INFO | "huge_shared" : " 27 GB", | 20260505 10:46:00.198 | INFO | "malloc" : " 14 GB", | 20260505 10:46:00.198 | INFO | "pinned" : " 29 GB" | 20260505 10:46:00.198 | INFO | } | 20260505 10:46:00.198 | INFO | Perf report for the last 5.14358 seconds: | 20260505 10:46:00.198 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:46:00.198 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:00.198 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:00.198 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:00.198 | INFO | ANALYSIS : 65.0%, 2.9, 99.8ms, 0 | 20260505 10:46:00.198 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:00.198 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260505 10:46:00.198 | INFO | PRE_HQ : 61.5%, 1.8, 58.9ms, 0 | 20260505 10:46:00.198 | INFO | BAZWRITER : 65.1%, 4.2, 139.2ms, 0 | 20260505 10:46:00.198 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260505 10:46:00.198 | INFO | Uploaded 36GiB to GPU in 2.19986s (16.3647GiB/s) | 20260505 10:46:00.198 | INFO | Downloaded 38GiB to GPU in 2.31153s (16.4393GiB/s) | 20260505 10:46:05.336 | INFO | Analyzing chunk frames = [2597376,2597888) | 20260505 10:46:05.336 | INFO | MemInfo: { | 20260505 10:46:05.336 | INFO | "gpu" : " 17 GB", | 20260505 10:46:05.336 | INFO | "huge" : "", | 20260505 10:46:05.336 | INFO | "huge_shared" : " 27 GB", | 20260505 10:46:05.336 | INFO | "malloc" : " 14 GB", | 20260505 10:46:05.336 | INFO | "pinned" : " 29 GB" | 20260505 10:46:05.336 | INFO | } | 20260505 10:46:05.336 | INFO | Perf report for the last 5.13776 seconds: | 20260505 10:46:05.336 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:46:05.336 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:05.336 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:05.336 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:05.336 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 10:46:05.336 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:05.336 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:05.336 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 10:46:05.336 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:05.336 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:05.336 | INFO | Uploaded 36GiB to GPU in 2.21675s (16.24GiB/s) | 20260505 10:46:05.336 | INFO | Downloaded 34GiB to GPU in 2.08462s (16.31GiB/s) | 20260505 10:46:10.476 | INFO | Analyzing chunk frames = [2597888,2598400) | 20260505 10:46:10.476 | INFO | MemInfo: { | 20260505 10:46:10.476 | INFO | "gpu" : " 17 GB", | 20260505 10:46:10.476 | INFO | "huge" : "", | 20260505 10:46:10.476 | INFO | "huge_shared" : " 27 GB", | 20260505 10:46:10.476 | INFO | "malloc" : " 14 GB", | 20260505 10:46:10.476 | INFO | "pinned" : " 29 GB" | 20260505 10:46:10.476 | INFO | } | 20260505 10:46:10.476 | INFO | Perf report for the last 5.14048 seconds: | 20260505 10:46:10.476 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:46:10.476 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:10.476 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:10.476 | INFO | SAVE_TRACE : 0.1%, 1.0, 0.0ms, 0 | 20260505 10:46:10.476 | INFO | ANALYSIS : 62.9%, 3.0, 97.9ms, 0 | 20260505 10:46:10.476 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:10.476 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:10.476 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 10:46:10.476 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:10.476 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:10.477 | INFO | Uploaded 36GiB to GPU in 2.1458s (16.777GiB/s) | 20260505 10:46:10.477 | INFO | Downloaded 34GiB to GPU in 2.05478s (16.5468GiB/s) | 20260505 10:46:15.624 | INFO | Analyzing chunk frames = [2598400,2598912) | 20260505 10:46:15.624 | INFO | MemInfo: { | 20260505 10:46:15.624 | INFO | "gpu" : " 17 GB", | 20260505 10:46:15.624 | INFO | "huge" : "", | 20260505 10:46:15.624 | INFO | "huge_shared" : " 27 GB", | 20260505 10:46:15.625 | INFO | "malloc" : " 14 GB", | 20260505 10:46:15.625 | INFO | "pinned" : " 29 GB" | 20260505 10:46:15.625 | INFO | } | 20260505 10:46:15.625 | INFO | Perf report for the last 5.14832 seconds: | 20260505 10:46:15.625 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:46:15.625 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:15.625 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:15.625 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:15.625 | INFO | ANALYSIS : 62.9%, 3.0, 97.9ms, 0 | 20260505 10:46:15.625 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:15.625 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:15.625 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 10:46:15.625 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:15.625 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:15.625 | INFO | Uploaded 36GiB to GPU in 2.15968s (16.6691GiB/s) | 20260505 10:46:15.625 | INFO | Downloaded 34GiB to GPU in 2.0742s (16.3919GiB/s) | 20260505 10:46:20.760 | INFO | Analyzing chunk frames = [2598912,2599424) | 20260505 10:46:20.760 | INFO | MemInfo: { | 20260505 10:46:20.760 | INFO | "gpu" : " 17 GB", | 20260505 10:46:20.760 | INFO | "huge" : "", | 20260505 10:46:20.760 | INFO | "huge_shared" : " 27 GB", | 20260505 10:46:20.760 | INFO | "malloc" : " 14 GB", | 20260505 10:46:20.760 | INFO | "pinned" : " 29 GB" | 20260505 10:46:20.760 | INFO | } | 20260505 10:46:20.760 | INFO | Perf report for the last 5.13526 seconds: | 20260505 10:46:20.760 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:46:20.760 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:20.760 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:20.760 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:20.760 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 10:46:20.760 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:20.760 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:20.760 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 10:46:20.760 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:20.760 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:20.760 | INFO | Uploaded 36GiB to GPU in 2.13705s (16.8457GiB/s) | 20260505 10:46:20.760 | INFO | Downloaded 34GiB to GPU in 2.10813s (16.128GiB/s) | 20260505 10:46:25.891 | INFO | Analyzing chunk frames = [2599424,2599936) | 20260505 10:46:25.891 | INFO | MemInfo: { | 20260505 10:46:25.891 | INFO | "gpu" : " 17 GB", | 20260505 10:46:25.891 | INFO | "huge" : "", | 20260505 10:46:25.891 | INFO | "huge_shared" : " 27 GB", | 20260505 10:46:25.892 | INFO | "malloc" : " 14 GB", | 20260505 10:46:25.892 | INFO | "pinned" : " 29 GB" | 20260505 10:46:25.892 | INFO | } | 20260505 10:46:25.892 | INFO | Perf report for the last 5.1318 seconds: | 20260505 10:46:25.892 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:46:25.892 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:25.892 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:25.892 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:25.892 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 10:46:25.892 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:25.892 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:25.892 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 10:46:25.892 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:25.892 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:25.892 | INFO | Uploaded 36GiB to GPU in 2.22443s (16.1839GiB/s) | 20260505 10:46:25.892 | INFO | Downloaded 34GiB to GPU in 2.09947s (16.1945GiB/s) | 20260505 10:46:31.031 | INFO | Analyzing chunk frames = [2599936,2600448) | 20260505 10:46:31.031 | INFO | MemInfo: { | 20260505 10:46:31.031 | INFO | "gpu" : " 17 GB", | 20260505 10:46:31.031 | INFO | "huge" : "", | 20260505 10:46:31.031 | INFO | "huge_shared" : " 27 GB", | 20260505 10:46:31.031 | INFO | "malloc" : " 14 GB", | 20260505 10:46:31.031 | INFO | "pinned" : " 29 GB" | 20260505 10:46:31.031 | INFO | } | 20260505 10:46:31.031 | INFO | Perf report for the last 5.13975 seconds: | 20260505 10:46:31.031 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:46:31.031 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:31.031 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:31.031 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:31.032 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 10:46:31.032 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:31.032 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:31.032 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 10:46:31.032 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:31.032 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:31.032 | INFO | Uploaded 36GiB to GPU in 2.18716s (16.4597GiB/s) | 20260505 10:46:31.032 | INFO | Downloaded 34GiB to GPU in 2.12933s (15.9674GiB/s) | 20260505 10:46:36.173 | INFO | Analyzing chunk frames = [2600448,2600960) | 20260505 10:46:36.173 | INFO | MemInfo: { | 20260505 10:46:36.173 | INFO | "gpu" : " 17 GB", | 20260505 10:46:36.173 | INFO | "huge" : "", | 20260505 10:46:36.173 | INFO | "huge_shared" : " 27 GB", | 20260505 10:46:36.173 | INFO | "malloc" : " 14 GB", | 20260505 10:46:36.173 | INFO | "pinned" : " 29 GB" | 20260505 10:46:36.173 | INFO | } | 20260505 10:46:36.173 | INFO | Perf report for the last 5.14186 seconds: | 20260505 10:46:36.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:46:36.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:36.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:36.174 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:36.174 | INFO | ANALYSIS : 64.4%, 3.0, 100.6ms, 0 | 20260505 10:46:36.174 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:36.174 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:36.174 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 10:46:36.174 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:36.174 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:36.174 | INFO | Uploaded 36GiB to GPU in 2.24501s (16.0356GiB/s) | 20260505 10:46:36.174 | INFO | Downloaded 34GiB to GPU in 2.09634s (16.2188GiB/s) | 20260505 10:46:36.308 | INFO | Starting a round of Baz IO | 20260505 10:46:39.570 | INFO | Populating RTMetrics report | 20260505 10:46:39.571 | INFO | RTMetrics report populated | 20260505 10:46:39.571 | INFO | Opening tmp RTMetrics json file | 20260505 10:46:39.575 | INFO | Serializing RTMetrics to json | 20260505 10:46:39.578 | INFO | Serialization done, swapping tmp file now... | 20260505 10:46:39.580 | INFO | RTMetrics done | 20260505 10:46:39.715 | INFO | Finished a round of Baz IO: Wrote 2.401 GiB in 3.407 seconds | 20260505 10:46:39.715 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.69% / 0.01% / 75.30% | 20260505 10:46:39.716 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.36356% / 1.19209e-05% / 9.53674e-05% | 20260505 10:46:39.716 | INFO | Baz IO overall: | 20260505 10:46:39.716 | INFO | BytesWritten : 1529103249542 | 20260505 10:46:39.716 | INFO | NumEvents : 728058148773 | 20260505 10:46:41.315 | INFO | Analyzing chunk frames = [2600960,2601472) | 20260505 10:46:41.315 | INFO | MemInfo: { | 20260505 10:46:41.315 | INFO | "gpu" : " 17 GB", | 20260505 10:46:41.315 | INFO | "huge" : "", | 20260505 10:46:41.315 | INFO | "huge_shared" : " 27 GB", | 20260505 10:46:41.315 | INFO | "malloc" : " 14 GB", | 20260505 10:46:41.315 | INFO | "pinned" : " 29 GB" | 20260505 10:46:41.315 | INFO | } | 20260505 10:46:41.315 | INFO | Perf report for the last 5.14151 seconds: | 20260505 10:46:41.315 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:46:41.315 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:41.315 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:41.315 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:41.315 | INFO | ANALYSIS : 64.9%, 2.9, 99.9ms, 0 | 20260505 10:46:41.315 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:41.315 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260505 10:46:41.315 | INFO | PRE_HQ : 61.4%, 1.9, 59.5ms, 0 | 20260505 10:46:41.315 | INFO | BAZWRITER : 65.5%, 4.1, 138.6ms, 0 | 20260505 10:46:41.315 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 10:46:41.315 | INFO | Uploaded 36GiB to GPU in 2.23012s (16.1426GiB/s) | 20260505 10:46:41.315 | INFO | Downloaded 38GiB to GPU in 2.28148s (16.6558GiB/s) | 20260505 10:46:46.457 | INFO | Analyzing chunk frames = [2601472,2601984) | 20260505 10:46:46.457 | INFO | MemInfo: { | 20260505 10:46:46.457 | INFO | "gpu" : " 17 GB", | 20260505 10:46:46.457 | INFO | "huge" : "", | 20260505 10:46:46.457 | INFO | "huge_shared" : " 27 GB", | 20260505 10:46:46.457 | INFO | "malloc" : " 14 GB", | 20260505 10:46:46.457 | INFO | "pinned" : " 29 GB" | 20260505 10:46:46.457 | INFO | } | 20260505 10:46:46.457 | INFO | Perf report for the last 5.14253 seconds: | 20260505 10:46:46.457 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:46:46.457 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:46.457 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:46.457 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:46.457 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 10:46:46.458 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:46.458 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:46.458 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 10:46:46.458 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:46.458 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:46.458 | INFO | Uploaded 36GiB to GPU in 2.22767s (16.1604GiB/s) | 20260505 10:46:46.458 | INFO | Downloaded 34GiB to GPU in 2.0878s (16.2851GiB/s) | 20260505 10:46:51.587 | INFO | Analyzing chunk frames = [2601984,2602496) | 20260505 10:46:51.587 | INFO | MemInfo: { | 20260505 10:46:51.587 | INFO | "gpu" : " 17 GB", | 20260505 10:46:51.587 | INFO | "huge" : "", | 20260505 10:46:51.587 | INFO | "huge_shared" : " 27 GB", | 20260505 10:46:51.587 | INFO | "malloc" : " 14 GB", | 20260505 10:46:51.587 | INFO | "pinned" : " 29 GB" | 20260505 10:46:51.587 | INFO | } | 20260505 10:46:51.587 | INFO | Perf report for the last 5.12939 seconds: | 20260505 10:46:51.587 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:46:51.587 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:51.587 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:51.587 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:51.587 | INFO | ANALYSIS : 63.0%, 3.0, 97.8ms, 0 | 20260505 10:46:51.587 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:51.587 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:51.587 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:46:51.587 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:51.587 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:51.587 | INFO | Uploaded 36GiB to GPU in 2.1605s (16.6628GiB/s) | 20260505 10:46:51.587 | INFO | Downloaded 34GiB to GPU in 2.06313s (16.4798GiB/s) | 20260505 10:46:56.725 | INFO | Analyzing chunk frames = [2602496,2603008) | 20260505 10:46:56.725 | INFO | MemInfo: { | 20260505 10:46:56.725 | INFO | "gpu" : " 17 GB", | 20260505 10:46:56.725 | INFO | "huge" : "", | 20260505 10:46:56.725 | INFO | "huge_shared" : " 27 GB", | 20260505 10:46:56.725 | INFO | "malloc" : " 14 GB", | 20260505 10:46:56.725 | INFO | "pinned" : " 29 GB" | 20260505 10:46:56.725 | INFO | } | 20260505 10:46:56.725 | INFO | Perf report for the last 5.13846 seconds: | 20260505 10:46:56.725 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:46:56.725 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:56.725 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:56.725 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:56.725 | INFO | ANALYSIS : 63.0%, 3.0, 97.9ms, 0 | 20260505 10:46:56.725 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:56.725 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:56.725 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 10:46:56.725 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:56.725 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:46:56.726 | INFO | Uploaded 36GiB to GPU in 2.15311s (16.72GiB/s) | 20260505 10:46:56.726 | INFO | Downloaded 34GiB to GPU in 2.07602s (16.3775GiB/s) | 20260505 10:47:01.871 | INFO | Analyzing chunk frames = [2603008,2603520) | 20260505 10:47:01.871 | INFO | MemInfo: { | 20260505 10:47:01.871 | INFO | "gpu" : " 17 GB", | 20260505 10:47:01.871 | INFO | "huge" : "", | 20260505 10:47:01.871 | INFO | "huge_shared" : " 27 GB", | 20260505 10:47:01.871 | INFO | "malloc" : " 14 GB", | 20260505 10:47:01.871 | INFO | "pinned" : " 29 GB" | 20260505 10:47:01.871 | INFO | } | 20260505 10:47:01.871 | INFO | Perf report for the last 5.14553 seconds: | 20260505 10:47:01.871 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:47:01.871 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:01.871 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:01.871 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:01.871 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 10:47:01.871 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:01.871 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:01.871 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 10:47:01.871 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:01.871 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:01.871 | INFO | Uploaded 36GiB to GPU in 2.23472s (16.1094GiB/s) | 20260505 10:47:01.871 | INFO | Downloaded 34GiB to GPU in 2.11074s (16.1081GiB/s) | 20260505 10:47:07.015 | INFO | Analyzing chunk frames = [2603520,2604032) | 20260505 10:47:07.015 | INFO | MemInfo: { | 20260505 10:47:07.015 | INFO | "gpu" : " 17 GB", | 20260505 10:47:07.015 | INFO | "huge" : "", | 20260505 10:47:07.015 | INFO | "huge_shared" : " 27 GB", | 20260505 10:47:07.015 | INFO | "malloc" : " 14 GB", | 20260505 10:47:07.015 | INFO | "pinned" : " 29 GB" | 20260505 10:47:07.015 | INFO | } | 20260505 10:47:07.015 | INFO | Perf report for the last 5.14436 seconds: | 20260505 10:47:07.015 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:47:07.015 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:07.015 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:07.015 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:07.015 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 10:47:07.015 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:07.015 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:07.015 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 10:47:07.015 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:07.015 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:07.015 | INFO | Uploaded 36GiB to GPU in 2.19898s (16.3712GiB/s) | 20260505 10:47:07.016 | INFO | Downloaded 34GiB to GPU in 2.10833s (16.1265GiB/s) | 20260505 10:47:12.146 | INFO | Analyzing chunk frames = [2604032,2604544) | 20260505 10:47:12.146 | INFO | MemInfo: { | 20260505 10:47:12.146 | INFO | "gpu" : " 17 GB", | 20260505 10:47:12.146 | INFO | "huge" : "", | 20260505 10:47:12.146 | INFO | "huge_shared" : " 27 GB", | 20260505 10:47:12.146 | INFO | "malloc" : " 14 GB", | 20260505 10:47:12.146 | INFO | "pinned" : " 29 GB" | 20260505 10:47:12.146 | INFO | } | 20260505 10:47:12.146 | INFO | Perf report for the last 5.13119 seconds: | 20260505 10:47:12.146 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:47:12.146 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:12.146 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:12.146 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:12.146 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 10:47:12.147 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:12.147 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:12.147 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 10:47:12.147 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:12.147 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:12.147 | INFO | Uploaded 36GiB to GPU in 2.17185s (16.5757GiB/s) | 20260505 10:47:12.147 | INFO | Downloaded 34GiB to GPU in 2.15809s (15.7547GiB/s) | 20260505 10:47:17.282 | INFO | Analyzing chunk frames = [2604544,2605056) | 20260505 10:47:17.282 | INFO | MemInfo: { | 20260505 10:47:17.282 | INFO | "gpu" : " 17 GB", | 20260505 10:47:17.282 | INFO | "huge" : "", | 20260505 10:47:17.282 | INFO | "huge_shared" : " 27 GB", | 20260505 10:47:17.282 | INFO | "malloc" : " 14 GB", | 20260505 10:47:17.282 | INFO | "pinned" : " 29 GB" | 20260505 10:47:17.282 | INFO | } | 20260505 10:47:17.282 | INFO | Perf report for the last 5.13566 seconds: | 20260505 10:47:17.282 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:47:17.282 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:17.282 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:17.282 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:17.282 | INFO | ANALYSIS : 64.5%, 3.0, 100.7ms, 0 | 20260505 10:47:17.282 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:17.282 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:17.282 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 10:47:17.282 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:17.282 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:17.282 | INFO | Uploaded 36GiB to GPU in 2.24758s (16.0173GiB/s) | 20260505 10:47:17.282 | INFO | Downloaded 34GiB to GPU in 2.09226s (16.2503GiB/s) | 20260505 10:47:17.283 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 10:47:17.283 | INFO | Total mem usage: 15.3 GiB | 20260505 10:47:17.283 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 10:47:17.283 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -6.81%) | 20260505 10:47:17.283 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 10:47:17.283 | INFO | High Water Breakdown: | 20260505 10:47:17.283 | INFO | 61% : 8.75 GiB ( 92.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 10:47:17.283 | INFO | 38.9% : 5.58 GiB ( 83.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 10:47:17.283 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 10:47:17.283 | INFO | ------------------------------------------ | 20260505 10:47:17.283 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:47:17.283 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 10:47:17.283 | INFO | Total mem usage: 26.5 GiB | 20260505 10:47:17.283 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 10:47:17.283 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 10:47:17.283 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 10:47:17.283 | INFO | High Water Breakdown: | 20260505 10:47:17.283 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:47:17.283 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:47:17.283 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:47:17.283 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:47:17.283 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:47:17.283 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:47:17.283 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 10:47:17.283 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 10:47:17.283 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:47:17.283 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 10:47:17.283 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 10:47:17.283 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 10:47:17.283 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 10:47:17.283 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 10:47:17.283 | INFO | ------------------------------------------ | 20260505 10:47:17.283 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:47:17.283 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 10:47:17.283 | INFO | Total mem usage: 25.6 GiB | 20260505 10:47:17.283 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 10:47:17.283 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 10:47:17.283 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 10:47:17.283 | INFO | High Water Breakdown: | 20260505 10:47:17.283 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 10:47:17.283 | INFO | ------------------------------------------ | 20260505 10:47:17.283 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:47:17.283 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 10:47:17.283 | INFO | Total mem usage: 15.7 GiB | 20260505 10:47:17.283 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 10:47:17.283 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 10:47:17.283 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 10:47:17.283 | INFO | High Water Breakdown: | 20260505 10:47:17.283 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 10:47:17.283 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 10:47:17.283 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 10:47:17.283 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 10:47:17.283 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 10:47:17.283 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 10:47:17.283 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 10:47:17.283 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 10:47:17.283 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 10:47:17.283 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:47:17.283 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 10:47:17.283 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:47:17.283 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 10:47:17.283 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 10:47:17.283 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 10:47:17.283 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 10:47:17.283 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:47:17.283 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 10:47:17.283 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:47:17.283 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:47:17.283 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:47:17.283 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 10:47:17.283 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 10:47:17.283 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:47:17.283 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 10:47:17.283 | INFO | ------------------------------------------ | 20260505 10:47:17.283 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:47:17.283 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 10:47:17.283 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:47:17.283 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:47:17.283 | INFO | PerfReport: Full Execution -- 3.23e+01 / 7.71e+01, 1.38e+01 / 1.48e+00 | 20260505 10:47:17.283 | INFO | PerfReport: 0.29%, AggregateStats -- 9.30e-02 / 1.02e-01, 1.05e-02 / 4.92e-03 | 20260505 10:47:17.283 | INFO | PerfReport: 11.70%, Baseline -- 3.78e+00 / 4.51e+00, 6.05e-01 / 2.33e-01 | 20260505 10:47:17.283 | INFO | PerfReport: 3.32%, Histogram -- 1.07e+00 / 1.49e+00, 2.07e-01 / 9.87e-02 | 20260505 10:47:17.283 | INFO | PerfReport: 15.70%, DME -- 4.06e+01 / 4.50e+01, 4.30e+00 / 1.34e+00 | 20260505 10:47:17.283 | INFO | PerfReport: 56.55%, FrameLabeling -- 1.83e+01 / 2.12e+01, 1.78e+00 / 1.95e+00 | 20260505 10:47:17.283 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.38e+00 / 2.91e+00, 4.16e-01 / 2.08e-01 | 20260505 10:47:17.283 | INFO | PerfReport: 3.74%, Metrics -- 1.21e+00 / 1.51e+00, 3.44e-01 / 2.01e-01 | 20260505 10:47:17.283 | INFO | PerfReport: 1.32%, PulseSerializing -- 4.25e-01 / 4.73e-01, 7.78e-02 / 3.51e-02 | 20260505 10:47:17.283 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 10:47:17.283 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:47:17.283 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:47:17.283 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.42e+01 / 0.00e+00 | 20260505 10:47:17.283 | INFO | PerfReport: 22.59%, Upload -- 2.29e+01 / 0.00e+00, 8.18e+00 / 0.00e+00 | 20260505 10:47:17.283 | INFO | PerfReport: 21.81%, Download -- 2.21e+01 / 0.00e+00, 8.15e+00 / 0.00e+00 | 20260505 10:47:17.283 | INFO | PerfReport: 31.88%, Compute -- 3.23e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 10:47:17.284 | INFO | PerfReport: 1.96%, LockUpload -- 1.98e+00 / 0.00e+00, 7.55e+00 / 0.00e+00 | 20260505 10:47:17.284 | INFO | PerfReport: 1.17%, LockDownload -- 1.19e+00 / 0.00e+00, 2.71e+00 / 0.00e+00 | 20260505 10:47:17.284 | INFO | PerfReport: 20.58%, LockCompute -- 2.08e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260505 10:47:17.284 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.84e-02 / 0.00e+00 | 20260505 10:47:17.416 | INFO | Starting a round of Baz IO | 20260505 10:47:20.686 | INFO | Populating RTMetrics report | 20260505 10:47:20.686 | INFO | RTMetrics report populated | 20260505 10:47:20.686 | INFO | Opening tmp RTMetrics json file | 20260505 10:47:20.689 | INFO | Serializing RTMetrics to json | 20260505 10:47:20.698 | INFO | Serialization done, swapping tmp file now... | 20260505 10:47:20.714 | INFO | RTMetrics done | 20260505 10:47:20.815 | INFO | Finished a round of Baz IO: Wrote 2.403 GiB in 3.399 seconds | 20260505 10:47:20.815 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.69% / 0.01% / 75.31% | 20260505 10:47:20.815 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.36095% / 3.97364e-06% / 7.94729e-05% | 20260505 10:47:20.815 | INFO | Baz IO overall: | 20260505 10:47:20.815 | INFO | BytesWritten : 1531683021319 | 20260505 10:47:20.815 | INFO | NumEvents : 729287701271 | 20260505 10:47:22.431 | INFO | Analyzing chunk frames = [2605056,2605568) | 20260505 10:47:22.431 | INFO | MemInfo: { | 20260505 10:47:22.431 | INFO | "gpu" : " 17 GB", | 20260505 10:47:22.431 | INFO | "huge" : "", | 20260505 10:47:22.431 | INFO | "huge_shared" : " 27 GB", | 20260505 10:47:22.431 | INFO | "malloc" : " 14 GB", | 20260505 10:47:22.431 | INFO | "pinned" : " 29 GB" | 20260505 10:47:22.431 | INFO | } | 20260505 10:47:22.431 | INFO | Perf report for the last 5.14868 seconds: | 20260505 10:47:22.431 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:47:22.431 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:22.431 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:22.431 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:22.431 | INFO | ANALYSIS : 65.0%, 2.9, 100.3ms, 0 | 20260505 10:47:22.431 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:22.431 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260505 10:47:22.431 | INFO | PRE_HQ : 61.6%, 1.8, 59.1ms, 0 | 20260505 10:47:22.431 | INFO | BAZWRITER : 65.3%, 4.1, 137.9ms, 0 | 20260505 10:47:22.431 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 10:47:22.432 | INFO | Uploaded 36GiB to GPU in 2.18737s (16.4581GiB/s) | 20260505 10:47:22.432 | INFO | Downloaded 38GiB to GPU in 2.31025s (16.4485GiB/s) | 20260505 10:47:27.567 | INFO | Analyzing chunk frames = [2605568,2606080) | 20260505 10:47:27.567 | INFO | MemInfo: { | 20260505 10:47:27.567 | INFO | "gpu" : " 17 GB", | 20260505 10:47:27.567 | INFO | "huge" : "", | 20260505 10:47:27.567 | INFO | "huge_shared" : " 27 GB", | 20260505 10:47:27.567 | INFO | "malloc" : " 14 GB", | 20260505 10:47:27.567 | INFO | "pinned" : " 29 GB" | 20260505 10:47:27.567 | INFO | } | 20260505 10:47:27.567 | INFO | Perf report for the last 5.13672 seconds: | 20260505 10:47:27.568 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:47:27.568 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:27.568 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:27.568 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:27.568 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 10:47:27.568 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:27.568 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:27.568 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 10:47:27.568 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:27.568 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:27.568 | INFO | Uploaded 36GiB to GPU in 2.23367s (16.117GiB/s) | 20260505 10:47:27.568 | INFO | Downloaded 34GiB to GPU in 2.09289s (16.2455GiB/s) | 20260505 10:47:32.704 | INFO | Analyzing chunk frames = [2606080,2606592) | 20260505 10:47:32.704 | INFO | MemInfo: { | 20260505 10:47:32.704 | INFO | "gpu" : " 17 GB", | 20260505 10:47:32.704 | INFO | "huge" : "", | 20260505 10:47:32.704 | INFO | "huge_shared" : " 27 GB", | 20260505 10:47:32.704 | INFO | "malloc" : " 14 GB", | 20260505 10:47:32.704 | INFO | "pinned" : " 29 GB" | 20260505 10:47:32.705 | INFO | } | 20260505 10:47:32.705 | INFO | Perf report for the last 5.1372 seconds: | 20260505 10:47:32.705 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:47:32.705 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:32.705 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:32.705 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:32.705 | INFO | ANALYSIS : 62.9%, 3.0, 97.9ms, 0 | 20260505 10:47:32.705 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:32.705 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:32.705 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 10:47:32.705 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:32.705 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:32.705 | INFO | Uploaded 36GiB to GPU in 2.16048s (16.6629GiB/s) | 20260505 10:47:32.705 | INFO | Downloaded 34GiB to GPU in 2.06203s (16.4886GiB/s) | 20260505 10:47:37.839 | INFO | Analyzing chunk frames = [2606592,2607104) | 20260505 10:47:37.839 | INFO | MemInfo: { | 20260505 10:47:37.839 | INFO | "gpu" : " 17 GB", | 20260505 10:47:37.839 | INFO | "huge" : "", | 20260505 10:47:37.839 | INFO | "huge_shared" : " 27 GB", | 20260505 10:47:37.839 | INFO | "malloc" : " 14 GB", | 20260505 10:47:37.839 | INFO | "pinned" : " 29 GB" | 20260505 10:47:37.839 | INFO | } | 20260505 10:47:37.839 | INFO | Perf report for the last 5.13445 seconds: | 20260505 10:47:37.839 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:47:37.839 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:37.839 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:37.839 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:37.839 | INFO | ANALYSIS : 63.1%, 3.0, 98.0ms, 0 | 20260505 10:47:37.839 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:37.839 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:37.839 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:47:37.839 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:37.839 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:37.839 | INFO | Uploaded 36GiB to GPU in 2.18346s (16.4876GiB/s) | 20260505 10:47:37.839 | INFO | Downloaded 34GiB to GPU in 2.08039s (16.3431GiB/s) | 20260505 10:47:42.985 | INFO | Analyzing chunk frames = [2607104,2607616) | 20260505 10:47:42.985 | INFO | MemInfo: { | 20260505 10:47:42.985 | INFO | "gpu" : " 17 GB", | 20260505 10:47:42.985 | INFO | "huge" : "", | 20260505 10:47:42.985 | INFO | "huge_shared" : " 27 GB", | 20260505 10:47:42.986 | INFO | "malloc" : " 14 GB", | 20260505 10:47:42.986 | INFO | "pinned" : " 29 GB" | 20260505 10:47:42.986 | INFO | } | 20260505 10:47:42.986 | INFO | Perf report for the last 5.1465 seconds: | 20260505 10:47:42.986 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:47:42.986 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:42.986 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:42.986 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:42.986 | INFO | ANALYSIS : 63.6%, 3.0, 99.5ms, 0 | 20260505 10:47:42.986 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:42.986 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:42.986 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:47:42.986 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:42.986 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:42.986 | INFO | Uploaded 36GiB to GPU in 2.23716s (16.0918GiB/s) | 20260505 10:47:42.986 | INFO | Downloaded 34GiB to GPU in 2.10126s (16.1807GiB/s) | 20260505 10:47:48.130 | INFO | Analyzing chunk frames = [2607616,2608128) | 20260505 10:47:48.130 | INFO | MemInfo: { | 20260505 10:47:48.130 | INFO | "gpu" : " 17 GB", | 20260505 10:47:48.130 | INFO | "huge" : "", | 20260505 10:47:48.130 | INFO | "huge_shared" : " 27 GB", | 20260505 10:47:48.130 | INFO | "malloc" : " 14 GB", | 20260505 10:47:48.130 | INFO | "pinned" : " 29 GB" | 20260505 10:47:48.130 | INFO | } | 20260505 10:47:48.131 | INFO | Perf report for the last 5.14487 seconds: | 20260505 10:47:48.131 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:47:48.131 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:48.131 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:48.131 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:48.131 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 10:47:48.131 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:48.131 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:48.131 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 10:47:48.131 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:48.131 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:48.131 | INFO | Uploaded 36GiB to GPU in 2.15933s (16.6718GiB/s) | 20260505 10:47:48.131 | INFO | Downloaded 34GiB to GPU in 2.12157s (16.0259GiB/s) | 20260505 10:47:53.268 | INFO | Analyzing chunk frames = [2608128,2608640) | 20260505 10:47:53.268 | INFO | MemInfo: { | 20260505 10:47:53.268 | INFO | "gpu" : " 17 GB", | 20260505 10:47:53.268 | INFO | "huge" : "", | 20260505 10:47:53.268 | INFO | "huge_shared" : " 27 GB", | 20260505 10:47:53.268 | INFO | "malloc" : " 14 GB", | 20260505 10:47:53.268 | INFO | "pinned" : " 29 GB" | 20260505 10:47:53.268 | INFO | } | 20260505 10:47:53.268 | INFO | Perf report for the last 5.13776 seconds: | 20260505 10:47:53.268 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:47:53.268 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:53.268 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:53.268 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:53.268 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 10:47:53.268 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:53.268 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:53.268 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 10:47:53.268 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:53.268 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:53.269 | INFO | Uploaded 36GiB to GPU in 2.15997s (16.6669GiB/s) | 20260505 10:47:53.269 | INFO | Downloaded 34GiB to GPU in 2.14056s (15.8837GiB/s) | 20260505 10:47:58.401 | INFO | Analyzing chunk frames = [2608640,2609152) | 20260505 10:47:58.401 | INFO | MemInfo: { | 20260505 10:47:58.401 | INFO | "gpu" : " 17 GB", | 20260505 10:47:58.401 | INFO | "huge" : "", | 20260505 10:47:58.401 | INFO | "huge_shared" : " 27 GB", | 20260505 10:47:58.401 | INFO | "malloc" : " 14 GB", | 20260505 10:47:58.401 | INFO | "pinned" : " 29 GB" | 20260505 10:47:58.401 | INFO | } | 20260505 10:47:58.401 | INFO | Perf report for the last 5.13281 seconds: | 20260505 10:47:58.401 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:47:58.401 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:58.401 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:58.401 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:58.401 | INFO | ANALYSIS : 64.1%, 3.0, 99.9ms, 0 | 20260505 10:47:58.401 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:58.401 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:58.401 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 10:47:58.401 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:58.401 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:47:58.401 | INFO | Uploaded 36GiB to GPU in 2.22721s (16.1637GiB/s) | 20260505 10:47:58.401 | INFO | Downloaded 34GiB to GPU in 2.09657s (16.217GiB/s) | 20260505 10:47:58.533 | INFO | Starting a round of Baz IO | 20260505 10:48:01.809 | INFO | Populating RTMetrics report | 20260505 10:48:01.810 | INFO | RTMetrics report populated | 20260505 10:48:01.810 | INFO | Opening tmp RTMetrics json file | 20260505 10:48:01.816 | INFO | Serializing RTMetrics to json | 20260505 10:48:01.818 | INFO | Serialization done, swapping tmp file now... | 20260505 10:48:01.819 | INFO | RTMetrics done | 20260505 10:48:01.979 | INFO | Finished a round of Baz IO: Wrote 2.401 GiB in 3.445 seconds | 20260505 10:48:01.979 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.68% / 0.01% / 75.31% | 20260505 10:48:01.979 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.35606% / 3.57628e-05% / 8.34465e-05% | 20260505 10:48:01.979 | INFO | Baz IO overall: | 20260505 10:48:01.979 | INFO | BytesWritten : 1534261424842 | 20260505 10:48:01.979 | INFO | NumEvents : 730515899649 | 20260505 10:48:03.535 | INFO | Analyzing chunk frames = [2609152,2609664) | 20260505 10:48:03.535 | INFO | MemInfo: { | 20260505 10:48:03.535 | INFO | "gpu" : " 17 GB", | 20260505 10:48:03.535 | INFO | "huge" : "", | 20260505 10:48:03.535 | INFO | "huge_shared" : " 27 GB", | 20260505 10:48:03.535 | INFO | "malloc" : " 14 GB", | 20260505 10:48:03.535 | INFO | "pinned" : " 29 GB" | 20260505 10:48:03.535 | INFO | } | 20260505 10:48:03.535 | INFO | Perf report for the last 5.13381 seconds: | 20260505 10:48:03.535 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:48:03.535 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:03.535 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:03.535 | INFO | SAVE_TRACE : 0.1%, 1.0, 0.1ms, 0 | 20260505 10:48:03.535 | INFO | ANALYSIS : 65.3%, 2.9, 100.1ms, 0 | 20260505 10:48:03.535 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:03.535 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260505 10:48:03.535 | INFO | PRE_HQ : 61.7%, 1.9, 59.2ms, 0 | 20260505 10:48:03.535 | INFO | BAZWRITER : 66.4%, 4.2, 141.9ms, 0 | 20260505 10:48:03.535 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 10:48:03.535 | INFO | Uploaded 36GiB to GPU in 2.19665s (16.3886GiB/s) | 20260505 10:48:03.535 | INFO | Downloaded 38GiB to GPU in 2.30977s (16.4519GiB/s) | 20260505 10:48:08.679 | INFO | Analyzing chunk frames = [2609664,2610176) | 20260505 10:48:08.679 | INFO | MemInfo: { | 20260505 10:48:08.679 | INFO | "gpu" : " 17 GB", | 20260505 10:48:08.679 | INFO | "huge" : "", | 20260505 10:48:08.679 | INFO | "huge_shared" : " 27 GB", | 20260505 10:48:08.679 | INFO | "malloc" : " 14 GB", | 20260505 10:48:08.679 | INFO | "pinned" : " 29 GB" | 20260505 10:48:08.679 | INFO | } | 20260505 10:48:08.679 | INFO | Perf report for the last 5.14404 seconds: | 20260505 10:48:08.679 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:48:08.679 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:08.679 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:08.679 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:08.679 | INFO | ANALYSIS : 64.1%, 3.0, 100.2ms, 0 | 20260505 10:48:08.679 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:08.679 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:08.679 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 10:48:08.679 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:08.679 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:08.679 | INFO | Uploaded 36GiB to GPU in 2.17838s (16.5261GiB/s) | 20260505 10:48:08.680 | INFO | Downloaded 34GiB to GPU in 2.10163s (16.1779GiB/s) | 20260505 10:48:13.820 | INFO | Analyzing chunk frames = [2610176,2610688) | 20260505 10:48:13.820 | INFO | MemInfo: { | 20260505 10:48:13.820 | INFO | "gpu" : " 17 GB", | 20260505 10:48:13.820 | INFO | "huge" : "", | 20260505 10:48:13.820 | INFO | "huge_shared" : " 27 GB", | 20260505 10:48:13.820 | INFO | "malloc" : " 14 GB", | 20260505 10:48:13.820 | INFO | "pinned" : " 29 GB" | 20260505 10:48:13.820 | INFO | } | 20260505 10:48:13.820 | INFO | Perf report for the last 5.14077 seconds: | 20260505 10:48:13.820 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:48:13.820 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:13.820 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:13.820 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:13.820 | INFO | ANALYSIS : 63.8%, 3.0, 99.3ms, 0 | 20260505 10:48:13.820 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:13.820 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:13.820 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:48:13.820 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:13.820 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:13.820 | INFO | Uploaded 36GiB to GPU in 2.22972s (16.1455GiB/s) | 20260505 10:48:13.820 | INFO | Downloaded 34GiB to GPU in 2.12446s (16.004GiB/s) | 20260505 10:48:18.965 | INFO | Analyzing chunk frames = [2610688,2611200) | 20260505 10:48:18.965 | INFO | MemInfo: { | 20260505 10:48:18.965 | INFO | "gpu" : " 17 GB", | 20260505 10:48:18.965 | INFO | "huge" : "", | 20260505 10:48:18.965 | INFO | "huge_shared" : " 27 GB", | 20260505 10:48:18.965 | INFO | "malloc" : " 14 GB", | 20260505 10:48:18.965 | INFO | "pinned" : " 29 GB" | 20260505 10:48:18.965 | INFO | } | 20260505 10:48:18.965 | INFO | Perf report for the last 5.14515 seconds: | 20260505 10:48:18.965 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:48:18.965 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:18.965 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:18.965 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:18.965 | INFO | ANALYSIS : 62.9%, 3.0, 97.9ms, 0 | 20260505 10:48:18.965 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:18.965 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:18.965 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 10:48:18.965 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:18.965 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:18.965 | INFO | Uploaded 36GiB to GPU in 2.16414s (16.6348GiB/s) | 20260505 10:48:18.965 | INFO | Downloaded 34GiB to GPU in 2.07999s (16.3462GiB/s) | 20260505 10:48:24.095 | INFO | Analyzing chunk frames = [2611200,2611712) | 20260505 10:48:24.095 | INFO | MemInfo: { | 20260505 10:48:24.095 | INFO | "gpu" : " 17 GB", | 20260505 10:48:24.095 | INFO | "huge" : "", | 20260505 10:48:24.095 | INFO | "huge_shared" : " 27 GB", | 20260505 10:48:24.095 | INFO | "malloc" : " 14 GB", | 20260505 10:48:24.095 | INFO | "pinned" : " 29 GB" | 20260505 10:48:24.095 | INFO | } | 20260505 10:48:24.095 | INFO | Perf report for the last 5.13034 seconds: | 20260505 10:48:24.095 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:48:24.095 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:24.095 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:24.095 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:24.095 | INFO | ANALYSIS : 63.5%, 3.0, 99.0ms, 0 | 20260505 10:48:24.095 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:24.095 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:24.095 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 10:48:24.095 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:24.095 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:24.096 | INFO | Uploaded 36GiB to GPU in 2.18386s (16.4846GiB/s) | 20260505 10:48:24.096 | INFO | Downloaded 34GiB to GPU in 2.12696s (15.9852GiB/s) | 20260505 10:48:29.235 | INFO | Analyzing chunk frames = [2611712,2612224) | 20260505 10:48:29.235 | INFO | MemInfo: { | 20260505 10:48:29.235 | INFO | "gpu" : " 17 GB", | 20260505 10:48:29.235 | INFO | "huge" : "", | 20260505 10:48:29.235 | INFO | "huge_shared" : " 27 GB", | 20260505 10:48:29.235 | INFO | "malloc" : " 14 GB", | 20260505 10:48:29.235 | INFO | "pinned" : " 29 GB" | 20260505 10:48:29.235 | INFO | } | 20260505 10:48:29.235 | INFO | Perf report for the last 5.13977 seconds: | 20260505 10:48:29.235 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:48:29.235 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:29.235 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:29.235 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:29.235 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 10:48:29.235 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:29.235 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:29.235 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:48:29.235 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:29.235 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:29.235 | INFO | Uploaded 36GiB to GPU in 2.23438s (16.1118GiB/s) | 20260505 10:48:29.235 | INFO | Downloaded 34GiB to GPU in 2.10636s (16.1416GiB/s) | 20260505 10:48:34.381 | INFO | Analyzing chunk frames = [2612224,2612736) | 20260505 10:48:34.381 | INFO | MemInfo: { | 20260505 10:48:34.381 | INFO | "gpu" : " 17 GB", | 20260505 10:48:34.381 | INFO | "huge" : "", | 20260505 10:48:34.381 | INFO | "huge_shared" : " 27 GB", | 20260505 10:48:34.381 | INFO | "malloc" : " 14 GB", | 20260505 10:48:34.381 | INFO | "pinned" : " 29 GB" | 20260505 10:48:34.381 | INFO | } | 20260505 10:48:34.381 | INFO | Perf report for the last 5.14637 seconds: | 20260505 10:48:34.381 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:48:34.381 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:34.381 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:34.382 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:34.382 | INFO | ANALYSIS : 63.3%, 3.0, 99.0ms, 0 | 20260505 10:48:34.382 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:34.382 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:34.382 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 10:48:34.382 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:34.382 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:34.382 | INFO | Uploaded 36GiB to GPU in 2.15545s (16.7019GiB/s) | 20260505 10:48:34.382 | INFO | Downloaded 34GiB to GPU in 2.15622s (15.7684GiB/s) | 20260505 10:48:39.519 | INFO | Analyzing chunk frames = [2612736,2613248) | 20260505 10:48:39.519 | INFO | MemInfo: { | 20260505 10:48:39.519 | INFO | "gpu" : " 17 GB", | 20260505 10:48:39.519 | INFO | "huge" : "", | 20260505 10:48:39.519 | INFO | "huge_shared" : " 27 GB", | 20260505 10:48:39.519 | INFO | "malloc" : " 14 GB", | 20260505 10:48:39.519 | INFO | "pinned" : " 29 GB" | 20260505 10:48:39.519 | INFO | } | 20260505 10:48:39.519 | INFO | Perf report for the last 5.13774 seconds: | 20260505 10:48:39.519 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:48:39.519 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:39.519 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:39.519 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:39.519 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 10:48:39.519 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:39.519 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:39.519 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:48:39.519 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:39.519 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:39.519 | INFO | Uploaded 36GiB to GPU in 2.21692s (16.2388GiB/s) | 20260505 10:48:39.520 | INFO | Downloaded 34GiB to GPU in 2.08545s (16.3034GiB/s) | 20260505 10:48:39.520 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 10:48:39.520 | INFO | Total mem usage: 15.3 GiB | 20260505 10:48:39.520 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 10:48:39.520 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.42%) | 20260505 10:48:39.520 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 10:48:39.520 | INFO | High Water Breakdown: | 20260505 10:48:39.520 | INFO | 61% : 8.78 GiB ( 92.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 10:48:39.520 | INFO | 38.9% : 5.6 GiB ( 84.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 10:48:39.520 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 10:48:39.520 | INFO | ------------------------------------------ | 20260505 10:48:39.520 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:48:39.520 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 10:48:39.520 | INFO | Total mem usage: 26.5 GiB | 20260505 10:48:39.520 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 10:48:39.520 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 10:48:39.520 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 10:48:39.520 | INFO | High Water Breakdown: | 20260505 10:48:39.520 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:48:39.520 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:48:39.520 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:48:39.520 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:48:39.520 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:48:39.520 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:48:39.520 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 10:48:39.520 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 10:48:39.520 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:48:39.520 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 10:48:39.520 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 10:48:39.520 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 10:48:39.520 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 10:48:39.520 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 10:48:39.520 | INFO | ------------------------------------------ | 20260505 10:48:39.520 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:48:39.520 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 10:48:39.520 | INFO | Total mem usage: 25.6 GiB | 20260505 10:48:39.520 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 10:48:39.520 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 10:48:39.520 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 10:48:39.520 | INFO | High Water Breakdown: | 20260505 10:48:39.520 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 10:48:39.520 | INFO | ------------------------------------------ | 20260505 10:48:39.520 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:48:39.520 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 10:48:39.520 | INFO | Total mem usage: 15.7 GiB | 20260505 10:48:39.520 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 10:48:39.520 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 10:48:39.520 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 10:48:39.520 | INFO | High Water Breakdown: | 20260505 10:48:39.520 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 10:48:39.520 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 10:48:39.520 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 10:48:39.520 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 10:48:39.520 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 10:48:39.520 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 10:48:39.520 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 10:48:39.520 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 10:48:39.520 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 10:48:39.520 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:48:39.520 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 10:48:39.520 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:48:39.520 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 10:48:39.520 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 10:48:39.520 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 10:48:39.520 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 10:48:39.520 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:48:39.520 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 10:48:39.520 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:48:39.520 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:48:39.520 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:48:39.520 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 10:48:39.520 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 10:48:39.520 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:48:39.520 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 10:48:39.520 | INFO | ------------------------------------------ | 20260505 10:48:39.520 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:48:39.521 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 10:48:39.521 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:48:39.521 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:48:39.521 | INFO | PerfReport: Full Execution -- 3.23e+01 / 7.74e+01, 1.38e+01 / 2.25e+00 | 20260505 10:48:39.521 | INFO | PerfReport: 0.29%, AggregateStats -- 9.31e-02 / 1.05e-01, 1.10e-02 / 1.52e-02 | 20260505 10:48:39.521 | INFO | PerfReport: 11.67%, Baseline -- 3.77e+00 / 4.47e+00, 5.97e-01 / 2.33e-01 | 20260505 10:48:39.521 | INFO | PerfReport: 3.31%, Histogram -- 1.07e+00 / 1.49e+00, 2.05e-01 / 9.96e-02 | 20260505 10:48:39.521 | INFO | PerfReport: 15.77%, DME -- 4.07e+01 / 4.52e+01, 4.55e+00 / 2.66e+00 | 20260505 10:48:39.521 | INFO | PerfReport: 56.55%, FrameLabeling -- 1.83e+01 / 2.13e+01, 1.74e+00 / 2.15e+00 | 20260505 10:48:39.521 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.38e+00 / 2.92e+00, 4.17e-01 / 2.42e-01 | 20260505 10:48:39.521 | INFO | PerfReport: 3.71%, Metrics -- 1.20e+00 / 1.49e+00, 3.39e-01 / 1.89e-01 | 20260505 10:48:39.521 | INFO | PerfReport: 1.32%, PulseSerializing -- 4.25e-01 / 4.69e-01, 7.90e-02 / 4.79e-02 | 20260505 10:48:39.521 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 10:48:39.521 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:48:39.521 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:48:39.521 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.44e+01 / 0.00e+00 | 20260505 10:48:39.521 | INFO | PerfReport: 22.54%, Upload -- 2.29e+01 / 0.00e+00, 8.13e+00 / 0.00e+00 | 20260505 10:48:39.521 | INFO | PerfReport: 21.89%, Download -- 2.22e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260505 10:48:39.521 | INFO | PerfReport: 31.85%, Compute -- 3.23e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 10:48:39.521 | INFO | PerfReport: 2.00%, LockUpload -- 2.02e+00 / 0.00e+00, 7.57e+00 / 0.00e+00 | 20260505 10:48:39.521 | INFO | PerfReport: 1.18%, LockDownload -- 1.20e+00 / 0.00e+00, 2.82e+00 / 0.00e+00 | 20260505 10:48:39.521 | INFO | PerfReport: 20.52%, LockCompute -- 2.08e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260505 10:48:39.521 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 2.08e-02 / 0.00e+00 | 20260505 10:48:39.658 | INFO | Starting a round of Baz IO | 20260505 10:48:42.915 | INFO | Populating RTMetrics report | 20260505 10:48:42.915 | INFO | RTMetrics report populated | 20260505 10:48:42.915 | INFO | Opening tmp RTMetrics json file | 20260505 10:48:42.937 | INFO | Serializing RTMetrics to json | 20260505 10:48:42.982 | INFO | Serialization done, swapping tmp file now... | 20260505 10:48:42.984 | INFO | RTMetrics done | 20260505 10:48:43.072 | INFO | Finished a round of Baz IO: Wrote 2.401 GiB in 3.413 seconds | 20260505 10:48:43.072 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.67% / 0.01% / 75.32% | 20260505 10:48:43.072 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.35444% / 3.97364e-06% / 8.74201e-05% | 20260505 10:48:43.072 | INFO | Baz IO overall: | 20260505 10:48:43.072 | INFO | BytesWritten : 1536839629841 | 20260505 10:48:43.072 | INFO | NumEvents : 731744176552 | 20260505 10:48:44.657 | INFO | Analyzing chunk frames = [2613248,2613760) | 20260505 10:48:44.657 | INFO | MemInfo: { | 20260505 10:48:44.657 | INFO | "gpu" : " 17 GB", | 20260505 10:48:44.657 | INFO | "huge" : "", | 20260505 10:48:44.657 | INFO | "huge_shared" : " 27 GB", | 20260505 10:48:44.657 | INFO | "malloc" : " 14 GB", | 20260505 10:48:44.657 | INFO | "pinned" : " 29 GB" | 20260505 10:48:44.657 | INFO | } | 20260505 10:48:44.657 | INFO | Perf report for the last 5.13749 seconds: | 20260505 10:48:44.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:48:44.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:44.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:44.657 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:44.657 | INFO | ANALYSIS : 65.0%, 2.9, 99.7ms, 0 | 20260505 10:48:44.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:44.657 | INFO | RT_METRICS : 46.8%, 1.0, 24.5ms, 0 | 20260505 10:48:44.657 | INFO | PRE_HQ : 61.4%, 1.8, 58.8ms, 0 | 20260505 10:48:44.657 | INFO | BAZWRITER : 65.8%, 4.1, 139.6ms, 0 | 20260505 10:48:44.657 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260505 10:48:44.657 | INFO | Uploaded 36GiB to GPU in 2.20134s (16.3537GiB/s) | 20260505 10:48:44.657 | INFO | Downloaded 38GiB to GPU in 2.3024s (16.5045GiB/s) | 20260505 10:48:49.789 | INFO | Analyzing chunk frames = [2613760,2614272) | 20260505 10:48:49.789 | INFO | MemInfo: { | 20260505 10:48:49.789 | INFO | "gpu" : " 17 GB", | 20260505 10:48:49.789 | INFO | "huge" : "", | 20260505 10:48:49.789 | INFO | "huge_shared" : " 27 GB", | 20260505 10:48:49.789 | INFO | "malloc" : " 14 GB", | 20260505 10:48:49.789 | INFO | "pinned" : " 29 GB" | 20260505 10:48:49.789 | INFO | } | 20260505 10:48:49.789 | INFO | Perf report for the last 5.13275 seconds: | 20260505 10:48:49.789 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:48:49.790 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:49.790 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:49.790 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:49.790 | INFO | ANALYSIS : 64.2%, 3.0, 100.1ms, 0 | 20260505 10:48:49.790 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:49.790 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:49.790 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:48:49.790 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:49.790 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:49.790 | INFO | Uploaded 36GiB to GPU in 2.21785s (16.2319GiB/s) | 20260505 10:48:49.790 | INFO | Downloaded 34GiB to GPU in 2.12233s (16.0202GiB/s) | 20260505 10:48:54.944 | INFO | Analyzing chunk frames = [2614272,2614784) | 20260505 10:48:54.945 | INFO | MemInfo: { | 20260505 10:48:54.945 | INFO | "gpu" : " 17 GB", | 20260505 10:48:54.945 | INFO | "huge" : "", | 20260505 10:48:54.945 | INFO | "huge_shared" : " 27 GB", | 20260505 10:48:54.945 | INFO | "malloc" : " 14 GB", | 20260505 10:48:54.945 | INFO | "pinned" : " 29 GB" | 20260505 10:48:54.945 | INFO | } | 20260505 10:48:54.945 | INFO | Perf report for the last 5.15519 seconds: | 20260505 10:48:54.945 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:48:54.945 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:54.945 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:54.945 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:54.945 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 10:48:54.945 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:54.945 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:54.945 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 10:48:54.945 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:54.945 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:48:54.945 | INFO | Uploaded 36GiB to GPU in 2.15371s (16.7153GiB/s) | 20260505 10:48:54.945 | INFO | Downloaded 34GiB to GPU in 2.06108s (16.4962GiB/s) | 20260505 10:49:00.083 | INFO | Analyzing chunk frames = [2614784,2615296) | 20260505 10:49:00.083 | INFO | MemInfo: { | 20260505 10:49:00.083 | INFO | "gpu" : " 17 GB", | 20260505 10:49:00.083 | INFO | "huge" : "", | 20260505 10:49:00.083 | INFO | "huge_shared" : " 27 GB", | 20260505 10:49:00.083 | INFO | "malloc" : " 14 GB", | 20260505 10:49:00.083 | INFO | "pinned" : " 29 GB" | 20260505 10:49:00.083 | INFO | } | 20260505 10:49:00.083 | INFO | Perf report for the last 5.13803 seconds: | 20260505 10:49:00.083 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:49:00.083 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:00.083 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:00.083 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:00.083 | INFO | ANALYSIS : 63.4%, 3.0, 98.5ms, 0 | 20260505 10:49:00.083 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:00.083 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:00.083 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:49:00.083 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:00.083 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:00.083 | INFO | Uploaded 36GiB to GPU in 2.20678s (16.3133GiB/s) | 20260505 10:49:00.083 | INFO | Downloaded 34GiB to GPU in 2.12647s (15.9889GiB/s) | 20260505 10:49:05.216 | INFO | Analyzing chunk frames = [2615296,2615808) | 20260505 10:49:05.216 | INFO | MemInfo: { | 20260505 10:49:05.216 | INFO | "gpu" : " 17 GB", | 20260505 10:49:05.216 | INFO | "huge" : "", | 20260505 10:49:05.216 | INFO | "huge_shared" : " 27 GB", | 20260505 10:49:05.216 | INFO | "malloc" : " 14 GB", | 20260505 10:49:05.216 | INFO | "pinned" : " 29 GB" | 20260505 10:49:05.216 | INFO | } | 20260505 10:49:05.216 | INFO | Perf report for the last 5.13303 seconds: | 20260505 10:49:05.216 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:49:05.216 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:05.216 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:05.216 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:05.216 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 10:49:05.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:05.216 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:05.216 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 10:49:05.216 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:05.216 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:05.216 | INFO | Uploaded 36GiB to GPU in 2.15022s (16.7425GiB/s) | 20260505 10:49:05.216 | INFO | Downloaded 34GiB to GPU in 2.12443s (16.0043GiB/s) | 20260505 10:49:10.348 | INFO | Analyzing chunk frames = [2615808,2616320) | 20260505 10:49:10.348 | INFO | MemInfo: { | 20260505 10:49:10.348 | INFO | "gpu" : " 17 GB", | 20260505 10:49:10.348 | INFO | "huge" : "", | 20260505 10:49:10.348 | INFO | "huge_shared" : " 27 GB", | 20260505 10:49:10.348 | INFO | "malloc" : " 14 GB", | 20260505 10:49:10.348 | INFO | "pinned" : " 29 GB" | 20260505 10:49:10.348 | INFO | } | 20260505 10:49:10.348 | INFO | Perf report for the last 5.13259 seconds: | 20260505 10:49:10.348 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:49:10.348 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:10.348 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:10.348 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:10.348 | INFO | ANALYSIS : 64.0%, 3.0, 99.8ms, 0 | 20260505 10:49:10.348 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:10.348 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:10.349 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 10:49:10.349 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:10.349 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:10.349 | INFO | Uploaded 36GiB to GPU in 2.23188s (16.1299GiB/s) | 20260505 10:49:10.349 | INFO | Downloaded 34GiB to GPU in 2.10093s (16.1833GiB/s) | 20260505 10:49:15.490 | INFO | Analyzing chunk frames = [2616320,2616832) | 20260505 10:49:15.490 | INFO | MemInfo: { | 20260505 10:49:15.490 | INFO | "gpu" : " 17 GB", | 20260505 10:49:15.490 | INFO | "huge" : "", | 20260505 10:49:15.490 | INFO | "huge_shared" : " 27 GB", | 20260505 10:49:15.490 | INFO | "malloc" : " 14 GB", | 20260505 10:49:15.490 | INFO | "pinned" : " 29 GB" | 20260505 10:49:15.490 | INFO | } | 20260505 10:49:15.490 | INFO | Perf report for the last 5.14173 seconds: | 20260505 10:49:15.490 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:49:15.490 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:15.490 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:15.490 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:15.490 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 10:49:15.490 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:15.490 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:15.490 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 10:49:15.490 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:15.490 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:15.491 | INFO | Uploaded 36GiB to GPU in 2.21227s (16.2729GiB/s) | 20260505 10:49:15.491 | INFO | Downloaded 34GiB to GPU in 2.14302s (15.8654GiB/s) | 20260505 10:49:20.635 | INFO | Analyzing chunk frames = [2616832,2617344) | 20260505 10:49:20.635 | INFO | MemInfo: { | 20260505 10:49:20.635 | INFO | "gpu" : " 17 GB", | 20260505 10:49:20.635 | INFO | "huge" : "", | 20260505 10:49:20.635 | INFO | "huge_shared" : " 27 GB", | 20260505 10:49:20.635 | INFO | "malloc" : " 14 GB", | 20260505 10:49:20.635 | INFO | "pinned" : " 29 GB" | 20260505 10:49:20.635 | INFO | } | 20260505 10:49:20.635 | INFO | Perf report for the last 5.14501 seconds: | 20260505 10:49:20.635 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:49:20.635 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:20.635 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:20.635 | INFO | SAVE_TRACE : 0.1%, 1.0, 0.0ms, 0 | 20260505 10:49:20.635 | INFO | ANALYSIS : 64.0%, 3.0, 100.1ms, 0 | 20260505 10:49:20.635 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:20.635 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:20.635 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:49:20.635 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:20.635 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:20.635 | INFO | Uploaded 36GiB to GPU in 2.21933s (16.2211GiB/s) | 20260505 10:49:20.635 | INFO | Downloaded 34GiB to GPU in 2.09234s (16.2498GiB/s) | 20260505 10:49:20.770 | INFO | Starting a round of Baz IO | 20260505 10:49:24.031 | INFO | Populating RTMetrics report | 20260505 10:49:24.032 | INFO | RTMetrics report populated | 20260505 10:49:24.032 | INFO | Opening tmp RTMetrics json file | 20260505 10:49:24.075 | INFO | Serializing RTMetrics to json | 20260505 10:49:24.077 | INFO | Serialization done, swapping tmp file now... | 20260505 10:49:24.077 | INFO | RTMetrics done | 20260505 10:49:24.244 | INFO | Finished a round of Baz IO: Wrote 2.4 GiB in 3.473 seconds | 20260505 10:49:24.244 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.66% / 0.01% / 75.33% | 20260505 10:49:24.244 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.34809% / 1.58946e-05% / 9.93411e-05% | 20260505 10:49:24.244 | INFO | Baz IO overall: | 20260505 10:49:24.244 | INFO | BytesWritten : 1539416594305 | 20260505 10:49:24.244 | INFO | NumEvents : 732971321231 | 20260505 10:49:25.778 | INFO | Analyzing chunk frames = [2617344,2617856) | 20260505 10:49:25.778 | INFO | MemInfo: { | 20260505 10:49:25.778 | INFO | "gpu" : " 17 GB", | 20260505 10:49:25.778 | INFO | "huge" : "", | 20260505 10:49:25.778 | INFO | "huge_shared" : " 27 GB", | 20260505 10:49:25.778 | INFO | "malloc" : " 14 GB", | 20260505 10:49:25.778 | INFO | "pinned" : " 29 GB" | 20260505 10:49:25.778 | INFO | } | 20260505 10:49:25.778 | INFO | Perf report for the last 5.1427 seconds: | 20260505 10:49:25.778 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:49:25.778 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:25.778 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:25.778 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:25.778 | INFO | ANALYSIS : 64.9%, 2.9, 99.5ms, 0 | 20260505 10:49:25.778 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:25.778 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260505 10:49:25.778 | INFO | PRE_HQ : 61.3%, 1.9, 59.1ms, 0 | 20260505 10:49:25.778 | INFO | BAZWRITER : 66.8%, 4.2, 143.3ms, 0 | 20260505 10:49:25.778 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260505 10:49:25.778 | INFO | Uploaded 36GiB to GPU in 2.20686s (16.3128GiB/s) | 20260505 10:49:25.778 | INFO | Downloaded 38GiB to GPU in 2.30518s (16.4846GiB/s) | 20260505 10:49:30.914 | INFO | Analyzing chunk frames = [2617856,2618368) | 20260505 10:49:30.914 | INFO | MemInfo: { | 20260505 10:49:30.914 | INFO | "gpu" : " 17 GB", | 20260505 10:49:30.914 | INFO | "huge" : "", | 20260505 10:49:30.914 | INFO | "huge_shared" : " 27 GB", | 20260505 10:49:30.914 | INFO | "malloc" : " 14 GB", | 20260505 10:49:30.914 | INFO | "pinned" : " 29 GB" | 20260505 10:49:30.914 | INFO | } | 20260505 10:49:30.914 | INFO | Perf report for the last 5.13644 seconds: | 20260505 10:49:30.915 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:49:30.915 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:30.915 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:30.915 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:30.915 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 10:49:30.915 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:30.915 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:30.915 | INFO | PRE_HQ : 29.3%, 1.0, 15.3ms, 0 | 20260505 10:49:30.915 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:30.915 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:30.915 | INFO | Uploaded 36GiB to GPU in 2.23735s (16.0905GiB/s) | 20260505 10:49:30.915 | INFO | Downloaded 34GiB to GPU in 2.07738s (16.3668GiB/s) | 20260505 10:49:36.047 | INFO | Analyzing chunk frames = [2618368,2618880) | 20260505 10:49:36.047 | INFO | MemInfo: { | 20260505 10:49:36.047 | INFO | "gpu" : " 17 GB", | 20260505 10:49:36.047 | INFO | "huge" : "", | 20260505 10:49:36.047 | INFO | "huge_shared" : " 27 GB", | 20260505 10:49:36.047 | INFO | "malloc" : " 14 GB", | 20260505 10:49:36.047 | INFO | "pinned" : " 29 GB" | 20260505 10:49:36.047 | INFO | } | 20260505 10:49:36.047 | INFO | Perf report for the last 5.13316 seconds: | 20260505 10:49:36.048 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:49:36.048 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:36.048 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:36.048 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:36.048 | INFO | ANALYSIS : 63.0%, 3.0, 98.0ms, 0 | 20260505 10:49:36.048 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:36.048 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:36.048 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:49:36.048 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:36.048 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:36.048 | INFO | Uploaded 36GiB to GPU in 2.1634s (16.6404GiB/s) | 20260505 10:49:36.048 | INFO | Downloaded 34GiB to GPU in 2.06451s (16.4688GiB/s) | 20260505 10:49:41.193 | INFO | Analyzing chunk frames = [2618880,2619392) | 20260505 10:49:41.193 | INFO | MemInfo: { | 20260505 10:49:41.193 | INFO | "gpu" : " 17 GB", | 20260505 10:49:41.193 | INFO | "huge" : "", | 20260505 10:49:41.193 | INFO | "huge_shared" : " 27 GB", | 20260505 10:49:41.193 | INFO | "malloc" : " 14 GB", | 20260505 10:49:41.193 | INFO | "pinned" : " 29 GB" | 20260505 10:49:41.193 | INFO | } | 20260505 10:49:41.193 | INFO | Perf report for the last 5.14559 seconds: | 20260505 10:49:41.193 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:49:41.193 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:41.193 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:41.193 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:41.193 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260505 10:49:41.193 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:41.193 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:41.193 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 10:49:41.193 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:41.193 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:41.193 | INFO | Uploaded 36GiB to GPU in 2.15909s (16.6737GiB/s) | 20260505 10:49:41.193 | INFO | Downloaded 34GiB to GPU in 2.0721s (16.4084GiB/s) | 20260505 10:49:46.328 | INFO | Analyzing chunk frames = [2619392,2619904) | 20260505 10:49:46.329 | INFO | MemInfo: { | 20260505 10:49:46.329 | INFO | "gpu" : " 17 GB", | 20260505 10:49:46.329 | INFO | "huge" : "", | 20260505 10:49:46.329 | INFO | "huge_shared" : " 27 GB", | 20260505 10:49:46.329 | INFO | "malloc" : " 14 GB", | 20260505 10:49:46.329 | INFO | "pinned" : " 29 GB" | 20260505 10:49:46.329 | INFO | } | 20260505 10:49:46.329 | INFO | Perf report for the last 5.13573 seconds: | 20260505 10:49:46.329 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:49:46.329 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:46.329 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:46.329 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:46.329 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 10:49:46.329 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:46.329 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:46.329 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:49:46.329 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:46.329 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:46.329 | INFO | Uploaded 36GiB to GPU in 2.19654s (16.3894GiB/s) | 20260505 10:49:46.329 | INFO | Downloaded 34GiB to GPU in 2.12986s (15.9635GiB/s) | 20260505 10:49:51.467 | INFO | Analyzing chunk frames = [2619904,2620416) | 20260505 10:49:51.467 | INFO | MemInfo: { | 20260505 10:49:51.467 | INFO | "gpu" : " 17 GB", | 20260505 10:49:51.467 | INFO | "huge" : "", | 20260505 10:49:51.467 | INFO | "huge_shared" : " 27 GB", | 20260505 10:49:51.467 | INFO | "malloc" : " 14 GB", | 20260505 10:49:51.467 | INFO | "pinned" : " 29 GB" | 20260505 10:49:51.467 | INFO | } | 20260505 10:49:51.467 | INFO | Perf report for the last 5.13825 seconds: | 20260505 10:49:51.467 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:49:51.467 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:51.467 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:51.467 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:51.467 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 10:49:51.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:51.467 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:51.467 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:49:51.467 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:51.467 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:51.467 | INFO | Uploaded 36GiB to GPU in 2.2119s (16.2756GiB/s) | 20260505 10:49:51.467 | INFO | Downloaded 34GiB to GPU in 2.10285s (16.1685GiB/s) | 20260505 10:49:56.607 | INFO | Analyzing chunk frames = [2620416,2620928) | 20260505 10:49:56.607 | INFO | MemInfo: { | 20260505 10:49:56.607 | INFO | "gpu" : " 17 GB", | 20260505 10:49:56.607 | INFO | "huge" : "", | 20260505 10:49:56.607 | INFO | "huge_shared" : " 27 GB", | 20260505 10:49:56.607 | INFO | "malloc" : " 14 GB", | 20260505 10:49:56.607 | INFO | "pinned" : " 29 GB" | 20260505 10:49:56.607 | INFO | } | 20260505 10:49:56.607 | INFO | Perf report for the last 5.14009 seconds: | 20260505 10:49:56.607 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:49:56.607 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:56.607 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:56.607 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:56.607 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 10:49:56.607 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:56.607 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:56.607 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 10:49:56.607 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:56.607 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:49:56.607 | INFO | Uploaded 36GiB to GPU in 2.18865s (16.4485GiB/s) | 20260505 10:49:56.608 | INFO | Downloaded 34GiB to GPU in 2.12888s (15.9709GiB/s) | 20260505 10:50:01.746 | INFO | Analyzing chunk frames = [2620928,2621440) | 20260505 10:50:01.746 | INFO | MemInfo: { | 20260505 10:50:01.746 | INFO | "gpu" : " 17 GB", | 20260505 10:50:01.747 | INFO | "huge" : "", | 20260505 10:50:01.747 | INFO | "huge_shared" : " 27 GB", | 20260505 10:50:01.747 | INFO | "malloc" : " 14 GB", | 20260505 10:50:01.747 | INFO | "pinned" : " 29 GB" | 20260505 10:50:01.747 | INFO | } | 20260505 10:50:01.747 | INFO | Perf report for the last 5.13944 seconds: | 20260505 10:50:01.747 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:50:01.747 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:01.747 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:01.747 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:01.747 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 10:50:01.747 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:01.747 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:01.747 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 10:50:01.747 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:01.747 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:01.747 | INFO | Uploaded 36GiB to GPU in 2.24567s (16.0309GiB/s) | 20260505 10:50:01.747 | INFO | Downloaded 34GiB to GPU in 2.09586s (16.2224GiB/s) | 20260505 10:50:01.747 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 10:50:01.747 | INFO | Total mem usage: 15.3 GiB | 20260505 10:50:01.747 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 10:50:01.747 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -5.54%) | 20260505 10:50:01.747 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 10:50:01.747 | INFO | High Water Breakdown: | 20260505 10:50:01.747 | INFO | 61.1% : 8.88 GiB ( 93.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 10:50:01.747 | INFO | 38.7% : 5.63 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 10:50:01.747 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 10:50:01.747 | INFO | ------------------------------------------ | 20260505 10:50:01.747 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:50:01.747 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 10:50:01.747 | INFO | Total mem usage: 26.5 GiB | 20260505 10:50:01.747 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 10:50:01.747 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 10:50:01.747 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 10:50:01.747 | INFO | High Water Breakdown: | 20260505 10:50:01.747 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:50:01.747 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:50:01.747 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:50:01.747 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:50:01.747 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:50:01.747 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:50:01.747 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 10:50:01.747 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 10:50:01.748 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:50:01.748 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 10:50:01.748 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 10:50:01.748 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 10:50:01.748 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 10:50:01.748 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 10:50:01.748 | INFO | ------------------------------------------ | 20260505 10:50:01.748 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:50:01.748 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 10:50:01.748 | INFO | Total mem usage: 25.6 GiB | 20260505 10:50:01.748 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 10:50:01.748 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 10:50:01.748 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 10:50:01.748 | INFO | High Water Breakdown: | 20260505 10:50:01.748 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 10:50:01.748 | INFO | ------------------------------------------ | 20260505 10:50:01.748 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:50:01.748 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 10:50:01.748 | INFO | Total mem usage: 15.7 GiB | 20260505 10:50:01.748 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 10:50:01.748 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 10:50:01.748 | INFO | Unused Cache: 880 MiB (Waste: 5.48%) | 20260505 10:50:01.748 | INFO | High Water Breakdown: | 20260505 10:50:01.748 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 10:50:01.748 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 10:50:01.748 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 10:50:01.748 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 10:50:01.748 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 10:50:01.748 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 10:50:01.748 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 10:50:01.748 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 10:50:01.748 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 10:50:01.748 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:50:01.748 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 10:50:01.748 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:50:01.748 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 10:50:01.748 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 10:50:01.748 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 10:50:01.748 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 10:50:01.748 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:50:01.748 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 10:50:01.748 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:50:01.748 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:50:01.748 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:50:01.748 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 10:50:01.748 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 10:50:01.748 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:50:01.748 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 10:50:01.748 | INFO | ------------------------------------------ | 20260505 10:50:01.748 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:50:01.748 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 10:50:01.748 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:50:01.748 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:50:01.748 | INFO | PerfReport: Full Execution -- 3.23e+01 / 7.71e+01, 1.39e+01 / 1.71e+00 | 20260505 10:50:01.748 | INFO | PerfReport: 0.29%, AggregateStats -- 9.32e-02 / 1.02e-01, 1.09e-02 / 5.19e-03 | 20260505 10:50:01.748 | INFO | PerfReport: 11.72%, Baseline -- 3.78e+00 / 4.50e+00, 6.15e-01 / 1.91e-01 | 20260505 10:50:01.748 | INFO | PerfReport: 3.31%, Histogram -- 1.07e+00 / 1.51e+00, 2.09e-01 / 8.41e-02 | 20260505 10:50:01.748 | INFO | PerfReport: 15.69%, DME -- 4.05e+01 / 4.48e+01, 4.21e+00 / 2.25e+00 | 20260505 10:50:01.748 | INFO | PerfReport: 56.54%, FrameLabeling -- 1.83e+01 / 2.14e+01, 1.75e+00 / 2.32e+00 | 20260505 10:50:01.748 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.38e+00 / 2.86e+00, 4.11e-01 / 2.33e-01 | 20260505 10:50:01.748 | INFO | PerfReport: 3.74%, Metrics -- 1.21e+00 / 1.51e+00, 3.47e-01 / 1.96e-01 | 20260505 10:50:01.748 | INFO | PerfReport: 1.32%, PulseSerializing -- 4.26e-01 / 4.74e-01, 7.87e-02 / 2.51e-02 | 20260505 10:50:01.748 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 10:50:01.748 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:50:01.748 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:50:01.748 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.46e+01 / 0.00e+00 | 20260505 10:50:01.748 | INFO | PerfReport: 22.63%, Upload -- 2.29e+01 / 0.00e+00, 8.31e+00 / 0.00e+00 | 20260505 10:50:01.748 | INFO | PerfReport: 21.89%, Download -- 2.22e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260505 10:50:01.748 | INFO | PerfReport: 31.87%, Compute -- 3.23e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 10:50:01.748 | INFO | PerfReport: 2.01%, LockUpload -- 2.03e+00 / 0.00e+00, 7.48e+00 / 0.00e+00 | 20260505 10:50:01.748 | INFO | PerfReport: 1.12%, LockDownload -- 1.13e+00 / 0.00e+00, 2.64e+00 / 0.00e+00 | 20260505 10:50:01.748 | INFO | PerfReport: 20.47%, LockCompute -- 2.07e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260505 10:50:01.748 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 2.86e-02 / 0.00e+00 | 20260505 10:50:01.884 | INFO | Starting a round of Baz IO | 20260505 10:50:05.142 | INFO | Populating RTMetrics report | 20260505 10:50:05.143 | INFO | RTMetrics report populated | 20260505 10:50:05.143 | INFO | Opening tmp RTMetrics json file | 20260505 10:50:05.162 | INFO | Serializing RTMetrics to json | 20260505 10:50:05.189 | INFO | Serialization done, swapping tmp file now... | 20260505 10:50:05.190 | INFO | RTMetrics done | 20260505 10:50:05.293 | INFO | Finished a round of Baz IO: Wrote 2.4 GiB in 3.409 seconds | 20260505 10:50:05.293 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.65% / 0.01% / 75.34% | 20260505 10:50:05.293 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.34625% / 2.78155e-05% / 6.75519e-05% | 20260505 10:50:05.294 | INFO | Baz IO overall: | 20260505 10:50:05.294 | INFO | BytesWritten : 1541993113589 | 20260505 10:50:05.294 | INFO | NumEvents : 734198408486 | 20260505 10:50:06.894 | INFO | Analyzing chunk frames = [2621440,2621952) | 20260505 10:50:06.894 | INFO | MemInfo: { | 20260505 10:50:06.894 | INFO | "gpu" : " 17 GB", | 20260505 10:50:06.894 | INFO | "huge" : "", | 20260505 10:50:06.894 | INFO | "huge_shared" : " 27 GB", | 20260505 10:50:06.894 | INFO | "malloc" : " 14 GB", | 20260505 10:50:06.894 | INFO | "pinned" : " 29 GB" | 20260505 10:50:06.894 | INFO | } | 20260505 10:50:06.895 | INFO | Perf report for the last 5.14787 seconds: | 20260505 10:50:06.895 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:50:06.895 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:06.895 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:06.895 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:06.895 | INFO | ANALYSIS : 64.8%, 2.9, 99.6ms, 0 | 20260505 10:50:06.895 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:06.895 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260505 10:50:06.895 | INFO | PRE_HQ : 61.3%, 1.9, 59.2ms, 0 | 20260505 10:50:06.895 | INFO | BAZWRITER : 65.6%, 4.1, 139.3ms, 0 | 20260505 10:50:06.895 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260505 10:50:06.895 | INFO | Uploaded 36GiB to GPU in 2.20775s (16.3062GiB/s) | 20260505 10:50:06.895 | INFO | Downloaded 38GiB to GPU in 2.30782s (16.4657GiB/s) | 20260505 10:50:12.036 | INFO | Analyzing chunk frames = [2621952,2622464) | 20260505 10:50:12.036 | INFO | MemInfo: { | 20260505 10:50:12.036 | INFO | "gpu" : " 17 GB", | 20260505 10:50:12.036 | INFO | "huge" : "", | 20260505 10:50:12.036 | INFO | "huge_shared" : " 27 GB", | 20260505 10:50:12.036 | INFO | "malloc" : " 14 GB", | 20260505 10:50:12.036 | INFO | "pinned" : " 29 GB" | 20260505 10:50:12.036 | INFO | } | 20260505 10:50:12.036 | INFO | Perf report for the last 5.1418 seconds: | 20260505 10:50:12.037 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:50:12.037 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:12.037 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:12.037 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:12.037 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 10:50:12.037 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:12.037 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:12.037 | INFO | PRE_HQ : 29.3%, 1.0, 15.4ms, 0 | 20260505 10:50:12.037 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:12.037 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:12.037 | INFO | Uploaded 36GiB to GPU in 2.14201s (16.8067GiB/s) | 20260505 10:50:12.037 | INFO | Downloaded 34GiB to GPU in 2.04379s (16.6357GiB/s) | 20260505 10:50:17.159 | INFO | Analyzing chunk frames = [2622464,2622976) | 20260505 10:50:17.160 | INFO | MemInfo: { | 20260505 10:50:17.160 | INFO | "gpu" : " 17 GB", | 20260505 10:50:17.160 | INFO | "huge" : "", | 20260505 10:50:17.160 | INFO | "huge_shared" : " 27 GB", | 20260505 10:50:17.160 | INFO | "malloc" : " 14 GB", | 20260505 10:50:17.160 | INFO | "pinned" : " 29 GB" | 20260505 10:50:17.160 | INFO | } | 20260505 10:50:17.160 | INFO | Perf report for the last 5.12322 seconds: | 20260505 10:50:17.160 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:50:17.160 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:17.160 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:17.160 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:17.160 | INFO | ANALYSIS : 63.7%, 3.0, 98.8ms, 0 | 20260505 10:50:17.160 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:17.160 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:17.160 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:50:17.160 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:17.160 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:17.160 | INFO | Uploaded 36GiB to GPU in 2.23395s (16.1149GiB/s) | 20260505 10:50:17.160 | INFO | Downloaded 34GiB to GPU in 2.12658s (15.9881GiB/s) | 20260505 10:50:22.296 | INFO | Analyzing chunk frames = [2622976,2623488) | 20260505 10:50:22.296 | INFO | MemInfo: { | 20260505 10:50:22.296 | INFO | "gpu" : " 17 GB", | 20260505 10:50:22.296 | INFO | "huge" : "", | 20260505 10:50:22.296 | INFO | "huge_shared" : " 27 GB", | 20260505 10:50:22.296 | INFO | "malloc" : " 14 GB", | 20260505 10:50:22.296 | INFO | "pinned" : " 29 GB" | 20260505 10:50:22.296 | INFO | } | 20260505 10:50:22.296 | INFO | Perf report for the last 5.13681 seconds: | 20260505 10:50:22.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:50:22.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:22.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:22.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:22.297 | INFO | ANALYSIS : 63.1%, 3.0, 97.9ms, 0 | 20260505 10:50:22.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:22.297 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:22.297 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 10:50:22.297 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:22.297 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:22.297 | INFO | Uploaded 36GiB to GPU in 2.15331s (16.7184GiB/s) | 20260505 10:50:22.297 | INFO | Downloaded 34GiB to GPU in 2.07066s (16.4199GiB/s) | 20260505 10:50:27.446 | INFO | Analyzing chunk frames = [2623488,2624000) | 20260505 10:50:27.446 | INFO | MemInfo: { | 20260505 10:50:27.446 | INFO | "gpu" : " 17 GB", | 20260505 10:50:27.446 | INFO | "huge" : "", | 20260505 10:50:27.446 | INFO | "huge_shared" : " 27 GB", | 20260505 10:50:27.446 | INFO | "malloc" : " 14 GB", | 20260505 10:50:27.446 | INFO | "pinned" : " 29 GB" | 20260505 10:50:27.447 | INFO | } | 20260505 10:50:27.447 | INFO | Perf report for the last 5.15024 seconds: | 20260505 10:50:27.447 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:50:27.447 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:27.447 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:27.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:27.447 | INFO | ANALYSIS : 63.4%, 3.0, 99.2ms, 0 | 20260505 10:50:27.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:27.447 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:27.447 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 10:50:27.447 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:27.447 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:27.447 | INFO | Uploaded 36GiB to GPU in 2.23074s (16.1381GiB/s) | 20260505 10:50:27.447 | INFO | Downloaded 34GiB to GPU in 2.11083s (16.1074GiB/s) | 20260505 10:50:32.592 | INFO | Analyzing chunk frames = [2624000,2624512) | 20260505 10:50:32.592 | INFO | MemInfo: { | 20260505 10:50:32.592 | INFO | "gpu" : " 17 GB", | 20260505 10:50:32.592 | INFO | "huge" : "", | 20260505 10:50:32.592 | INFO | "huge_shared" : " 27 GB", | 20260505 10:50:32.592 | INFO | "malloc" : " 14 GB", | 20260505 10:50:32.592 | INFO | "pinned" : " 29 GB" | 20260505 10:50:32.592 | INFO | } | 20260505 10:50:32.592 | INFO | Perf report for the last 5.14512 seconds: | 20260505 10:50:32.592 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:50:32.592 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:32.592 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:32.592 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:32.592 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 10:50:32.592 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:32.592 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:32.592 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 10:50:32.592 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:32.592 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:32.592 | INFO | Uploaded 36GiB to GPU in 2.19566s (16.396GiB/s) | 20260505 10:50:32.592 | INFO | Downloaded 34GiB to GPU in 2.11682s (16.0618GiB/s) | 20260505 10:50:37.731 | INFO | Analyzing chunk frames = [2624512,2625024) | 20260505 10:50:37.731 | INFO | MemInfo: { | 20260505 10:50:37.731 | INFO | "gpu" : " 17 GB", | 20260505 10:50:37.731 | INFO | "huge" : "", | 20260505 10:50:37.731 | INFO | "huge_shared" : " 27 GB", | 20260505 10:50:37.731 | INFO | "malloc" : " 14 GB", | 20260505 10:50:37.731 | INFO | "pinned" : " 29 GB" | 20260505 10:50:37.731 | INFO | } | 20260505 10:50:37.731 | INFO | Perf report for the last 5.13959 seconds: | 20260505 10:50:37.732 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:50:37.732 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:37.732 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:37.732 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:37.732 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 10:50:37.732 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:37.732 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:37.732 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 10:50:37.732 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:37.732 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:37.732 | INFO | Uploaded 36GiB to GPU in 2.17037s (16.587GiB/s) | 20260505 10:50:37.732 | INFO | Downloaded 34GiB to GPU in 2.14043s (15.8846GiB/s) | 20260505 10:50:42.859 | INFO | Analyzing chunk frames = [2625024,2625536) | 20260505 10:50:42.859 | INFO | MemInfo: { | 20260505 10:50:42.859 | INFO | "gpu" : " 17 GB", | 20260505 10:50:42.859 | INFO | "huge" : "", | 20260505 10:50:42.859 | INFO | "huge_shared" : " 27 GB", | 20260505 10:50:42.859 | INFO | "malloc" : " 14 GB", | 20260505 10:50:42.859 | INFO | "pinned" : " 29 GB" | 20260505 10:50:42.859 | INFO | } | 20260505 10:50:42.859 | INFO | Perf report for the last 5.12743 seconds: | 20260505 10:50:42.859 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:50:42.859 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:42.859 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:42.859 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:42.859 | INFO | ANALYSIS : 64.1%, 3.0, 99.9ms, 0 | 20260505 10:50:42.859 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:42.859 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:42.859 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:50:42.859 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:42.859 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:42.859 | INFO | Uploaded 36GiB to GPU in 2.21772s (16.2329GiB/s) | 20260505 10:50:42.859 | INFO | Downloaded 34GiB to GPU in 2.10036s (16.1877GiB/s) | 20260505 10:50:42.992 | INFO | Starting a round of Baz IO | 20260505 10:50:46.262 | INFO | Populating RTMetrics report | 20260505 10:50:46.262 | INFO | RTMetrics report populated | 20260505 10:50:46.262 | INFO | Opening tmp RTMetrics json file | 20260505 10:50:46.298 | INFO | Serializing RTMetrics to json | 20260505 10:50:46.300 | INFO | Serialization done, swapping tmp file now... | 20260505 10:50:46.300 | INFO | RTMetrics done | 20260505 10:50:46.401 | INFO | Finished a round of Baz IO: Wrote 2.4 GiB in 3.408 seconds | 20260505 10:50:46.401 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.64% / 0.01% / 75.35% | 20260505 10:50:46.401 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.34465% / 1.98682e-05% / 9.53674e-05% | 20260505 10:50:46.401 | INFO | Baz IO overall: | 20260505 10:50:46.401 | INFO | BytesWritten : 1544570579190 | 20260505 10:50:46.401 | INFO | NumEvents : 735426054046 | 20260505 10:50:48.000 | INFO | Analyzing chunk frames = [2625536,2626048) | 20260505 10:50:48.000 | INFO | MemInfo: { | 20260505 10:50:48.000 | INFO | "gpu" : " 17 GB", | 20260505 10:50:48.000 | INFO | "huge" : "", | 20260505 10:50:48.000 | INFO | "huge_shared" : " 27 GB", | 20260505 10:50:48.000 | INFO | "malloc" : " 14 GB", | 20260505 10:50:48.000 | INFO | "pinned" : " 29 GB" | 20260505 10:50:48.000 | INFO | } | 20260505 10:50:48.000 | INFO | Perf report for the last 5.14117 seconds: | 20260505 10:50:48.000 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:50:48.000 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:48.000 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:48.000 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:48.000 | INFO | ANALYSIS : 65.1%, 2.9, 100.3ms, 0 | 20260505 10:50:48.000 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:48.000 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260505 10:50:48.000 | INFO | PRE_HQ : 61.7%, 1.9, 59.5ms, 0 | 20260505 10:50:48.000 | INFO | BAZWRITER : 65.6%, 4.1, 138.7ms, 0 | 20260505 10:50:48.000 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 10:50:48.000 | INFO | Uploaded 36GiB to GPU in 2.20135s (16.3536GiB/s) | 20260505 10:50:48.000 | INFO | Downloaded 38GiB to GPU in 2.32279s (16.3597GiB/s) | 20260505 10:50:53.141 | INFO | Analyzing chunk frames = [2626048,2626560) | 20260505 10:50:53.141 | INFO | MemInfo: { | 20260505 10:50:53.141 | INFO | "gpu" : " 17 GB", | 20260505 10:50:53.141 | INFO | "huge" : "", | 20260505 10:50:53.141 | INFO | "huge_shared" : " 27 GB", | 20260505 10:50:53.141 | INFO | "malloc" : " 14 GB", | 20260505 10:50:53.141 | INFO | "pinned" : " 29 GB" | 20260505 10:50:53.142 | INFO | } | 20260505 10:50:53.142 | INFO | Perf report for the last 5.14159 seconds: | 20260505 10:50:53.142 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:50:53.142 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:53.142 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:53.142 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:53.142 | INFO | ANALYSIS : 64.3%, 3.0, 100.5ms, 0 | 20260505 10:50:53.142 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:53.142 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:53.142 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 10:50:53.142 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:53.142 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:53.142 | INFO | Uploaded 36GiB to GPU in 2.23641s (16.0972GiB/s) | 20260505 10:50:53.142 | INFO | Downloaded 34GiB to GPU in 2.08756s (16.287GiB/s) | 20260505 10:50:58.280 | INFO | Analyzing chunk frames = [2626560,2627072) | 20260505 10:50:58.280 | INFO | MemInfo: { | 20260505 10:50:58.280 | INFO | "gpu" : " 17 GB", | 20260505 10:50:58.280 | INFO | "huge" : "", | 20260505 10:50:58.280 | INFO | "huge_shared" : " 27 GB", | 20260505 10:50:58.280 | INFO | "malloc" : " 14 GB", | 20260505 10:50:58.280 | INFO | "pinned" : " 29 GB" | 20260505 10:50:58.280 | INFO | } | 20260505 10:50:58.280 | INFO | Perf report for the last 5.13824 seconds: | 20260505 10:50:58.280 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:50:58.280 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:58.280 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:58.280 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:58.280 | INFO | ANALYSIS : 63.8%, 3.0, 99.2ms, 0 | 20260505 10:50:58.280 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:58.280 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:58.280 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 10:50:58.280 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:58.280 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:50:58.280 | INFO | Uploaded 36GiB to GPU in 2.2253s (16.1776GiB/s) | 20260505 10:50:58.280 | INFO | Downloaded 34GiB to GPU in 2.13179s (15.949GiB/s) | 20260505 10:51:03.415 | INFO | Analyzing chunk frames = [2627072,2627584) | 20260505 10:51:03.415 | INFO | MemInfo: { | 20260505 10:51:03.415 | INFO | "gpu" : " 17 GB", | 20260505 10:51:03.415 | INFO | "huge" : "", | 20260505 10:51:03.415 | INFO | "huge_shared" : " 27 GB", | 20260505 10:51:03.415 | INFO | "malloc" : " 14 GB", | 20260505 10:51:03.415 | INFO | "pinned" : " 29 GB" | 20260505 10:51:03.415 | INFO | } | 20260505 10:51:03.415 | INFO | Perf report for the last 5.13537 seconds: | 20260505 10:51:03.415 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:51:03.415 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:03.415 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:03.415 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:03.415 | INFO | ANALYSIS : 63.0%, 3.0, 97.7ms, 0 | 20260505 10:51:03.415 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:03.415 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:03.415 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:51:03.415 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:03.415 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:03.416 | INFO | Uploaded 36GiB to GPU in 2.16678s (16.6145GiB/s) | 20260505 10:51:03.416 | INFO | Downloaded 34GiB to GPU in 2.07995s (16.3466GiB/s) | 20260505 10:51:08.556 | INFO | Analyzing chunk frames = [2627584,2628096) | 20260505 10:51:08.557 | INFO | MemInfo: { | 20260505 10:51:08.557 | INFO | "gpu" : " 17 GB", | 20260505 10:51:08.557 | INFO | "huge" : "", | 20260505 10:51:08.557 | INFO | "huge_shared" : " 27 GB", | 20260505 10:51:08.557 | INFO | "malloc" : " 14 GB", | 20260505 10:51:08.557 | INFO | "pinned" : " 29 GB" | 20260505 10:51:08.557 | INFO | } | 20260505 10:51:08.557 | INFO | Perf report for the last 5.14131 seconds: | 20260505 10:51:08.557 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:51:08.557 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:08.557 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:08.557 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:08.557 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 10:51:08.557 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:08.557 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:08.557 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:51:08.557 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:08.557 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:08.557 | INFO | Uploaded 36GiB to GPU in 2.22958s (16.1466GiB/s) | 20260505 10:51:08.557 | INFO | Downloaded 34GiB to GPU in 2.11213s (16.0975GiB/s) | 20260505 10:51:13.706 | INFO | Analyzing chunk frames = [2628096,2628608) | 20260505 10:51:13.706 | INFO | MemInfo: { | 20260505 10:51:13.706 | INFO | "gpu" : " 17 GB", | 20260505 10:51:13.706 | INFO | "huge" : "", | 20260505 10:51:13.706 | INFO | "huge_shared" : " 27 GB", | 20260505 10:51:13.706 | INFO | "malloc" : " 14 GB", | 20260505 10:51:13.706 | INFO | "pinned" : " 29 GB" | 20260505 10:51:13.706 | INFO | } | 20260505 10:51:13.706 | INFO | Perf report for the last 5.14919 seconds: | 20260505 10:51:13.706 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:51:13.706 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:13.706 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:13.706 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:13.706 | INFO | ANALYSIS : 63.2%, 3.0, 98.9ms, 0 | 20260505 10:51:13.706 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:13.706 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:13.706 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 10:51:13.706 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:13.706 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:13.706 | INFO | Uploaded 36GiB to GPU in 2.19317s (16.4146GiB/s) | 20260505 10:51:13.706 | INFO | Downloaded 34GiB to GPU in 2.10838s (16.1261GiB/s) | 20260505 10:51:18.842 | INFO | Analyzing chunk frames = [2628608,2629120) | 20260505 10:51:18.842 | INFO | MemInfo: { | 20260505 10:51:18.842 | INFO | "gpu" : " 17 GB", | 20260505 10:51:18.842 | INFO | "huge" : "", | 20260505 10:51:18.842 | INFO | "huge_shared" : " 27 GB", | 20260505 10:51:18.842 | INFO | "malloc" : " 14 GB", | 20260505 10:51:18.842 | INFO | "pinned" : " 29 GB" | 20260505 10:51:18.842 | INFO | } | 20260505 10:51:18.842 | INFO | Perf report for the last 5.13654 seconds: | 20260505 10:51:18.842 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:51:18.842 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:18.842 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:18.842 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:18.842 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 10:51:18.842 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:18.842 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:18.842 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 10:51:18.842 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:18.842 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:18.843 | INFO | Uploaded 36GiB to GPU in 2.17473s (16.5538GiB/s) | 20260505 10:51:18.843 | INFO | Downloaded 34GiB to GPU in 2.14869s (15.8236GiB/s) | 20260505 10:51:23.977 | INFO | Analyzing chunk frames = [2629120,2629632) | 20260505 10:51:23.977 | INFO | MemInfo: { | 20260505 10:51:23.977 | INFO | "gpu" : " 17 GB", | 20260505 10:51:23.977 | INFO | "huge" : "", | 20260505 10:51:23.977 | INFO | "huge_shared" : " 27 GB", | 20260505 10:51:23.977 | INFO | "malloc" : " 14 GB", | 20260505 10:51:23.977 | INFO | "pinned" : " 29 GB" | 20260505 10:51:23.977 | INFO | } | 20260505 10:51:23.977 | INFO | Perf report for the last 5.13484 seconds: | 20260505 10:51:23.977 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:51:23.977 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:23.977 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:23.977 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:23.977 | INFO | ANALYSIS : 64.1%, 3.0, 100.0ms, 0 | 20260505 10:51:23.977 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:23.977 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:23.977 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 10:51:23.977 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:23.977 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:23.977 | INFO | Uploaded 36GiB to GPU in 2.21857s (16.2267GiB/s) | 20260505 10:51:23.977 | INFO | Downloaded 34GiB to GPU in 2.09004s (16.2676GiB/s) | 20260505 10:51:23.978 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 10:51:23.978 | INFO | Total mem usage: 15.3 GiB | 20260505 10:51:23.978 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 10:51:23.978 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -6.81%) | 20260505 10:51:23.978 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 10:51:23.978 | INFO | High Water Breakdown: | 20260505 10:51:23.978 | INFO | 61% : 8.75 GiB ( 92.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 10:51:23.978 | INFO | 38.9% : 5.58 GiB ( 83.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 10:51:23.978 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 10:51:23.978 | INFO | ------------------------------------------ | 20260505 10:51:23.978 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:51:23.978 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 10:51:23.978 | INFO | Total mem usage: 26.5 GiB | 20260505 10:51:23.978 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 10:51:23.978 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 10:51:23.978 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 10:51:23.978 | INFO | High Water Breakdown: | 20260505 10:51:23.978 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:51:23.978 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:51:23.978 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:51:23.978 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:51:23.978 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:51:23.978 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:51:23.978 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 10:51:23.978 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 10:51:23.978 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:51:23.978 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 10:51:23.978 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 10:51:23.978 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 10:51:23.978 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 10:51:23.978 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 10:51:23.978 | INFO | ------------------------------------------ | 20260505 10:51:23.978 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:51:23.978 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 10:51:23.978 | INFO | Total mem usage: 25.6 GiB | 20260505 10:51:23.978 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 10:51:23.978 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 10:51:23.978 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 10:51:23.978 | INFO | High Water Breakdown: | 20260505 10:51:23.978 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 10:51:23.978 | INFO | ------------------------------------------ | 20260505 10:51:23.978 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:51:23.978 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 10:51:23.978 | INFO | Total mem usage: 15.7 GiB | 20260505 10:51:23.978 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 10:51:23.978 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 10:51:23.978 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 10:51:23.978 | INFO | High Water Breakdown: | 20260505 10:51:23.978 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 10:51:23.978 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 10:51:23.978 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 10:51:23.978 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 10:51:23.978 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 10:51:23.978 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 10:51:23.978 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 10:51:23.978 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 10:51:23.978 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 10:51:23.978 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:51:23.978 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 10:51:23.978 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:51:23.978 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 10:51:23.978 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 10:51:23.978 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 10:51:23.978 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 10:51:23.978 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:51:23.978 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 10:51:23.978 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:51:23.978 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:51:23.978 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:51:23.978 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 10:51:23.978 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 10:51:23.978 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:51:23.978 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 10:51:23.978 | INFO | ------------------------------------------ | 20260505 10:51:23.979 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:51:23.979 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 10:51:23.979 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:51:23.979 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:51:23.979 | INFO | PerfReport: Full Execution -- 3.23e+01 / 7.71e+01, 1.38e+01 / 1.89e+00 | 20260505 10:51:23.979 | INFO | PerfReport: 0.29%, AggregateStats -- 9.32e-02 / 1.02e-01, 1.09e-02 / 4.94e-03 | 20260505 10:51:23.979 | INFO | PerfReport: 11.71%, Baseline -- 3.78e+00 / 4.49e+00, 6.06e-01 / 2.03e-01 | 20260505 10:51:23.979 | INFO | PerfReport: 3.31%, Histogram -- 1.07e+00 / 1.48e+00, 2.06e-01 / 1.06e-01 | 20260505 10:51:23.979 | INFO | PerfReport: 15.64%, DME -- 4.04e+01 / 4.43e+01, 4.35e+00 / 2.18e+00 | 20260505 10:51:23.979 | INFO | PerfReport: 56.64%, FrameLabeling -- 1.83e+01 / 2.20e+01, 1.79e+00 / 2.29e+00 | 20260505 10:51:23.979 | INFO | PerfReport: 7.36%, PulseAccumulating -- 2.37e+00 / 2.84e+00, 4.08e-01 / 2.27e-01 | 20260505 10:51:23.979 | INFO | PerfReport: 3.74%, Metrics -- 1.21e+00 / 1.46e+00, 3.48e-01 / 1.63e-01 | 20260505 10:51:23.979 | INFO | PerfReport: 1.31%, PulseSerializing -- 4.24e-01 / 4.65e-01, 7.85e-02 / 3.48e-02 | 20260505 10:51:23.979 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 10:51:23.979 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:51:23.979 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:51:23.979 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.45e+01 / 0.00e+00 | 20260505 10:51:23.979 | INFO | PerfReport: 22.61%, Upload -- 2.29e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260505 10:51:23.979 | INFO | PerfReport: 21.91%, Download -- 2.22e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260505 10:51:23.979 | INFO | PerfReport: 31.84%, Compute -- 3.23e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 10:51:23.979 | INFO | PerfReport: 2.03%, LockUpload -- 2.05e+00 / 0.00e+00, 7.65e+00 / 0.00e+00 | 20260505 10:51:23.979 | INFO | PerfReport: 1.18%, LockDownload -- 1.20e+00 / 0.00e+00, 2.78e+00 / 0.00e+00 | 20260505 10:51:23.979 | INFO | PerfReport: 20.41%, LockCompute -- 2.07e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260505 10:51:23.979 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.83e-02 / 0.00e+00 | 20260505 10:51:24.111 | INFO | Starting a round of Baz IO | 20260505 10:51:27.379 | INFO | Populating RTMetrics report | 20260505 10:51:27.379 | INFO | RTMetrics report populated | 20260505 10:51:27.379 | INFO | Opening tmp RTMetrics json file | 20260505 10:51:27.391 | INFO | Serializing RTMetrics to json | 20260505 10:51:27.416 | INFO | Serialization done, swapping tmp file now... | 20260505 10:51:27.436 | INFO | RTMetrics done | 20260505 10:51:27.501 | INFO | Finished a round of Baz IO: Wrote 2.399 GiB in 3.39 seconds | 20260505 10:51:27.501 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.63% / 0.01% / 75.36% | 20260505 10:51:27.501 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.33819% / 1.19209e-05% / 9.53674e-05% | 20260505 10:51:27.501 | INFO | Baz IO overall: | 20260505 10:51:27.501 | INFO | BytesWritten : 1547145985153 | 20260505 10:51:27.501 | INFO | NumEvents : 736652566223 | 20260505 10:51:29.111 | INFO | Analyzing chunk frames = [2629632,2630144) | 20260505 10:51:29.111 | INFO | MemInfo: { | 20260505 10:51:29.111 | INFO | "gpu" : " 17 GB", | 20260505 10:51:29.111 | INFO | "huge" : "", | 20260505 10:51:29.111 | INFO | "huge_shared" : " 27 GB", | 20260505 10:51:29.111 | INFO | "malloc" : " 14 GB", | 20260505 10:51:29.111 | INFO | "pinned" : " 29 GB" | 20260505 10:51:29.111 | INFO | } | 20260505 10:51:29.111 | INFO | Perf report for the last 5.13418 seconds: | 20260505 10:51:29.112 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:51:29.112 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:29.112 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:29.112 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:29.112 | INFO | ANALYSIS : 65.1%, 2.9, 99.9ms, 0 | 20260505 10:51:29.112 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:29.112 | INFO | RT_METRICS : 46.7%, 1.0, 24.5ms, 0 | 20260505 10:51:29.112 | INFO | PRE_HQ : 61.8%, 1.8, 59.1ms, 0 | 20260505 10:51:29.112 | INFO | BAZWRITER : 65.3%, 4.1, 139.1ms, 0 | 20260505 10:51:29.112 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260505 10:51:29.112 | INFO | Uploaded 36GiB to GPU in 2.18486s (16.477GiB/s) | 20260505 10:51:29.112 | INFO | Downloaded 38GiB to GPU in 2.3093s (16.4552GiB/s) | 20260505 10:51:34.253 | INFO | Analyzing chunk frames = [2630144,2630656) | 20260505 10:51:34.253 | INFO | MemInfo: { | 20260505 10:51:34.253 | INFO | "gpu" : " 17 GB", | 20260505 10:51:34.253 | INFO | "huge" : "", | 20260505 10:51:34.253 | INFO | "huge_shared" : " 27 GB", | 20260505 10:51:34.253 | INFO | "malloc" : " 14 GB", | 20260505 10:51:34.253 | INFO | "pinned" : " 29 GB" | 20260505 10:51:34.253 | INFO | } | 20260505 10:51:34.253 | INFO | Perf report for the last 5.14197 seconds: | 20260505 10:51:34.253 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:51:34.253 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:34.253 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:34.253 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:34.253 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 10:51:34.253 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:34.253 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:34.253 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:51:34.253 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:34.253 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:34.254 | INFO | Uploaded 36GiB to GPU in 2.23183s (16.1303GiB/s) | 20260505 10:51:34.254 | INFO | Downloaded 34GiB to GPU in 2.10849s (16.1253GiB/s) | 20260505 10:51:39.397 | INFO | Analyzing chunk frames = [2630656,2631168) | 20260505 10:51:39.397 | INFO | MemInfo: { | 20260505 10:51:39.397 | INFO | "gpu" : " 17 GB", | 20260505 10:51:39.397 | INFO | "huge" : "", | 20260505 10:51:39.397 | INFO | "huge_shared" : " 27 GB", | 20260505 10:51:39.397 | INFO | "malloc" : " 14 GB", | 20260505 10:51:39.397 | INFO | "pinned" : " 29 GB" | 20260505 10:51:39.397 | INFO | } | 20260505 10:51:39.397 | INFO | Perf report for the last 5.14409 seconds: | 20260505 10:51:39.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:51:39.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:39.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:39.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:39.397 | INFO | ANALYSIS : 63.6%, 3.0, 99.1ms, 0 | 20260505 10:51:39.398 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:39.398 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:39.398 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:51:39.398 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:39.398 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:39.398 | INFO | Uploaded 36GiB to GPU in 2.22516s (16.1786GiB/s) | 20260505 10:51:39.398 | INFO | Downloaded 34GiB to GPU in 2.12573s (15.9945GiB/s) | 20260505 10:51:44.540 | INFO | Analyzing chunk frames = [2631168,2631680) | 20260505 10:51:44.540 | INFO | MemInfo: { | 20260505 10:51:44.540 | INFO | "gpu" : " 17 GB", | 20260505 10:51:44.540 | INFO | "huge" : "", | 20260505 10:51:44.540 | INFO | "huge_shared" : " 27 GB", | 20260505 10:51:44.540 | INFO | "malloc" : " 14 GB", | 20260505 10:51:44.540 | INFO | "pinned" : " 29 GB" | 20260505 10:51:44.540 | INFO | } | 20260505 10:51:44.540 | INFO | Perf report for the last 5.14254 seconds: | 20260505 10:51:44.540 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:51:44.540 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:44.540 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:44.540 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:44.540 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 10:51:44.540 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:44.540 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:44.540 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:51:44.540 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:44.540 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:44.540 | INFO | Uploaded 36GiB to GPU in 2.2112s (16.2808GiB/s) | 20260505 10:51:44.540 | INFO | Downloaded 34GiB to GPU in 2.12634s (15.9899GiB/s) | 20260505 10:51:49.672 | INFO | Analyzing chunk frames = [2631680,2632192) | 20260505 10:51:49.672 | INFO | MemInfo: { | 20260505 10:51:49.672 | INFO | "gpu" : " 17 GB", | 20260505 10:51:49.672 | INFO | "huge" : "", | 20260505 10:51:49.672 | INFO | "huge_shared" : " 27 GB", | 20260505 10:51:49.672 | INFO | "malloc" : " 14 GB", | 20260505 10:51:49.672 | INFO | "pinned" : " 29 GB" | 20260505 10:51:49.672 | INFO | } | 20260505 10:51:49.672 | INFO | Perf report for the last 5.13212 seconds: | 20260505 10:51:49.672 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:51:49.672 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:49.672 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:49.672 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:49.672 | INFO | ANALYSIS : 63.4%, 3.0, 98.8ms, 0 | 20260505 10:51:49.672 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:49.672 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:49.672 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 10:51:49.672 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:49.672 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:49.672 | INFO | Uploaded 36GiB to GPU in 2.15917s (16.673GiB/s) | 20260505 10:51:49.673 | INFO | Downloaded 34GiB to GPU in 2.12775s (15.9793GiB/s) | 20260505 10:51:54.808 | INFO | Analyzing chunk frames = [2632192,2632704) | 20260505 10:51:54.808 | INFO | MemInfo: { | 20260505 10:51:54.808 | INFO | "gpu" : " 17 GB", | 20260505 10:51:54.808 | INFO | "huge" : "", | 20260505 10:51:54.808 | INFO | "huge_shared" : " 27 GB", | 20260505 10:51:54.808 | INFO | "malloc" : " 14 GB", | 20260505 10:51:54.808 | INFO | "pinned" : " 29 GB" | 20260505 10:51:54.808 | INFO | } | 20260505 10:51:54.808 | INFO | Perf report for the last 5.13629 seconds: | 20260505 10:51:54.808 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:51:54.808 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:54.808 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:54.808 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:54.808 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 10:51:54.808 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:54.809 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:54.809 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 10:51:54.809 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:54.809 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:54.809 | INFO | Uploaded 36GiB to GPU in 2.20891s (16.2976GiB/s) | 20260505 10:51:54.809 | INFO | Downloaded 34GiB to GPU in 2.10085s (16.1839GiB/s) | 20260505 10:51:59.947 | INFO | Analyzing chunk frames = [2632704,2633216) | 20260505 10:51:59.947 | INFO | MemInfo: { | 20260505 10:51:59.947 | INFO | "gpu" : " 17 GB", | 20260505 10:51:59.947 | INFO | "huge" : "", | 20260505 10:51:59.947 | INFO | "huge_shared" : " 27 GB", | 20260505 10:51:59.947 | INFO | "malloc" : " 14 GB", | 20260505 10:51:59.947 | INFO | "pinned" : " 29 GB" | 20260505 10:51:59.947 | INFO | } | 20260505 10:51:59.947 | INFO | Perf report for the last 5.1384 seconds: | 20260505 10:51:59.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:51:59.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:59.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:59.947 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:59.947 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 10:51:59.947 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:59.947 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:59.947 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 10:51:59.947 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:59.947 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:51:59.947 | INFO | Uploaded 36GiB to GPU in 2.18362s (16.4864GiB/s) | 20260505 10:51:59.947 | INFO | Downloaded 34GiB to GPU in 2.15216s (15.7981GiB/s) | 20260505 10:52:05.093 | INFO | Analyzing chunk frames = [2633216,2633728) | 20260505 10:52:05.093 | INFO | MemInfo: { | 20260505 10:52:05.093 | INFO | "gpu" : " 17 GB", | 20260505 10:52:05.093 | INFO | "huge" : "", | 20260505 10:52:05.093 | INFO | "huge_shared" : " 27 GB", | 20260505 10:52:05.093 | INFO | "malloc" : " 14 GB", | 20260505 10:52:05.093 | INFO | "pinned" : " 29 GB" | 20260505 10:52:05.093 | INFO | } | 20260505 10:52:05.093 | INFO | Perf report for the last 5.14645 seconds: | 20260505 10:52:05.093 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:52:05.093 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:05.093 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:05.093 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:05.093 | INFO | ANALYSIS : 64.3%, 3.0, 100.5ms, 0 | 20260505 10:52:05.093 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:05.093 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:05.093 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 10:52:05.093 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:05.093 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:05.094 | INFO | Uploaded 36GiB to GPU in 2.24575s (16.0303GiB/s) | 20260505 10:52:05.094 | INFO | Downloaded 34GiB to GPU in 2.10041s (16.1873GiB/s) | 20260505 10:52:05.227 | INFO | Starting a round of Baz IO | 20260505 10:52:08.493 | INFO | Populating RTMetrics report | 20260505 10:52:08.493 | INFO | RTMetrics report populated | 20260505 10:52:08.493 | INFO | Opening tmp RTMetrics json file | 20260505 10:52:08.499 | INFO | Serializing RTMetrics to json | 20260505 10:52:08.522 | INFO | Serialization done, swapping tmp file now... | 20260505 10:52:08.524 | INFO | RTMetrics done | 20260505 10:52:08.629 | INFO | Finished a round of Baz IO: Wrote 2.397 GiB in 3.402 seconds | 20260505 10:52:08.629 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.62% / 0.01% / 75.37% | 20260505 10:52:08.629 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.33024% / 1.19209e-05% / 0.000107288% | 20260505 10:52:08.630 | INFO | Baz IO overall: | 20260505 10:52:08.630 | INFO | BytesWritten : 1549719260157 | 20260505 10:52:08.630 | INFO | NumEvents : 737877631073 | 20260505 10:52:10.231 | INFO | Analyzing chunk frames = [2633728,2634240) | 20260505 10:52:10.231 | INFO | MemInfo: { | 20260505 10:52:10.231 | INFO | "gpu" : " 17 GB", | 20260505 10:52:10.231 | INFO | "huge" : "", | 20260505 10:52:10.231 | INFO | "huge_shared" : " 27 GB", | 20260505 10:52:10.231 | INFO | "malloc" : " 14 GB", | 20260505 10:52:10.231 | INFO | "pinned" : " 29 GB" | 20260505 10:52:10.231 | INFO | } | 20260505 10:52:10.231 | INFO | Perf report for the last 5.13782 seconds: | 20260505 10:52:10.231 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:52:10.231 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:10.231 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:10.231 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:10.231 | INFO | ANALYSIS : 65.0%, 2.9, 100.0ms, 0 | 20260505 10:52:10.231 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:10.231 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260505 10:52:10.231 | INFO | PRE_HQ : 62.4%, 1.8, 59.1ms, 0 | 20260505 10:52:10.231 | INFO | BAZWRITER : 65.5%, 4.1, 137.4ms, 0 | 20260505 10:52:10.231 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 10:52:10.231 | INFO | Uploaded 36GiB to GPU in 2.25094s (15.9933GiB/s) | 20260505 10:52:10.232 | INFO | Downloaded 38GiB to GPU in 2.35363s (16.1453GiB/s) | 20260505 10:52:15.367 | INFO | Analyzing chunk frames = [2634240,2634752) | 20260505 10:52:15.367 | INFO | MemInfo: { | 20260505 10:52:15.367 | INFO | "gpu" : " 17 GB", | 20260505 10:52:15.367 | INFO | "huge" : "", | 20260505 10:52:15.367 | INFO | "huge_shared" : " 27 GB", | 20260505 10:52:15.367 | INFO | "malloc" : " 14 GB", | 20260505 10:52:15.367 | INFO | "pinned" : " 29 GB" | 20260505 10:52:15.368 | INFO | } | 20260505 10:52:15.368 | INFO | Perf report for the last 5.13648 seconds: | 20260505 10:52:15.368 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:52:15.368 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:15.368 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:15.368 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:15.368 | INFO | ANALYSIS : 64.2%, 3.0, 100.1ms, 0 | 20260505 10:52:15.368 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:15.368 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:15.368 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 10:52:15.368 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:15.368 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:15.368 | INFO | Uploaded 36GiB to GPU in 2.17633s (16.5416GiB/s) | 20260505 10:52:15.368 | INFO | Downloaded 34GiB to GPU in 2.09532s (16.2267GiB/s) | 20260505 10:52:20.508 | INFO | Analyzing chunk frames = [2634752,2635264) | 20260505 10:52:20.508 | INFO | MemInfo: { | 20260505 10:52:20.508 | INFO | "gpu" : " 17 GB", | 20260505 10:52:20.508 | INFO | "huge" : "", | 20260505 10:52:20.508 | INFO | "huge_shared" : " 27 GB", | 20260505 10:52:20.508 | INFO | "malloc" : " 14 GB", | 20260505 10:52:20.508 | INFO | "pinned" : " 29 GB" | 20260505 10:52:20.508 | INFO | } | 20260505 10:52:20.508 | INFO | Perf report for the last 5.14063 seconds: | 20260505 10:52:20.508 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:52:20.508 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:20.508 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:20.508 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:20.508 | INFO | ANALYSIS : 62.9%, 3.0, 97.8ms, 0 | 20260505 10:52:20.508 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:20.508 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:20.509 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 10:52:20.509 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:20.509 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:20.509 | INFO | Uploaded 36GiB to GPU in 2.15709s (16.6891GiB/s) | 20260505 10:52:20.509 | INFO | Downloaded 34GiB to GPU in 2.0644s (16.4697GiB/s) | 20260505 10:52:25.653 | INFO | Analyzing chunk frames = [2635264,2635776) | 20260505 10:52:25.653 | INFO | MemInfo: { | 20260505 10:52:25.653 | INFO | "gpu" : " 17 GB", | 20260505 10:52:25.653 | INFO | "huge" : "", | 20260505 10:52:25.653 | INFO | "huge_shared" : " 27 GB", | 20260505 10:52:25.653 | INFO | "malloc" : " 14 GB", | 20260505 10:52:25.653 | INFO | "pinned" : " 29 GB" | 20260505 10:52:25.653 | INFO | } | 20260505 10:52:25.654 | INFO | Perf report for the last 5.14523 seconds: | 20260505 10:52:25.654 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:52:25.654 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:25.654 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:25.654 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:25.654 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260505 10:52:25.654 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:25.654 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:25.654 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 10:52:25.654 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:25.654 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:25.654 | INFO | Uploaded 36GiB to GPU in 2.15461s (16.7084GiB/s) | 20260505 10:52:25.654 | INFO | Downloaded 34GiB to GPU in 2.07487s (16.3866GiB/s) | 20260505 10:52:30.787 | INFO | Analyzing chunk frames = [2635776,2636288) | 20260505 10:52:30.787 | INFO | MemInfo: { | 20260505 10:52:30.787 | INFO | "gpu" : " 17 GB", | 20260505 10:52:30.787 | INFO | "huge" : "", | 20260505 10:52:30.787 | INFO | "huge_shared" : " 27 GB", | 20260505 10:52:30.787 | INFO | "malloc" : " 14 GB", | 20260505 10:52:30.787 | INFO | "pinned" : " 29 GB" | 20260505 10:52:30.787 | INFO | } | 20260505 10:52:30.787 | INFO | Perf report for the last 5.13384 seconds: | 20260505 10:52:30.788 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:52:30.788 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:30.788 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:30.788 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:30.788 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 10:52:30.788 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:30.788 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:30.788 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 10:52:30.788 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:30.788 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:30.788 | INFO | Uploaded 36GiB to GPU in 2.18353s (16.487GiB/s) | 20260505 10:52:30.788 | INFO | Downloaded 34GiB to GPU in 2.12628s (15.9904GiB/s) | 20260505 10:52:35.925 | INFO | Analyzing chunk frames = [2636288,2636800) | 20260505 10:52:35.925 | INFO | MemInfo: { | 20260505 10:52:35.925 | INFO | "gpu" : " 17 GB", | 20260505 10:52:35.925 | INFO | "huge" : "", | 20260505 10:52:35.925 | INFO | "huge_shared" : " 27 GB", | 20260505 10:52:35.925 | INFO | "malloc" : " 14 GB", | 20260505 10:52:35.925 | INFO | "pinned" : " 29 GB" | 20260505 10:52:35.925 | INFO | } | 20260505 10:52:35.925 | INFO | Perf report for the last 5.13803 seconds: | 20260505 10:52:35.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:52:35.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:35.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:35.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:35.926 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 10:52:35.926 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:35.926 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:35.926 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 10:52:35.926 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:35.926 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:35.926 | INFO | Uploaded 36GiB to GPU in 2.23588s (16.1011GiB/s) | 20260505 10:52:35.926 | INFO | Downloaded 34GiB to GPU in 2.10379s (16.1613GiB/s) | 20260505 10:52:41.063 | INFO | Analyzing chunk frames = [2636800,2637312) | 20260505 10:52:41.063 | INFO | MemInfo: { | 20260505 10:52:41.063 | INFO | "gpu" : " 17 GB", | 20260505 10:52:41.063 | INFO | "huge" : "", | 20260505 10:52:41.063 | INFO | "huge_shared" : " 27 GB", | 20260505 10:52:41.063 | INFO | "malloc" : " 14 GB", | 20260505 10:52:41.063 | INFO | "pinned" : " 29 GB" | 20260505 10:52:41.063 | INFO | } | 20260505 10:52:41.063 | INFO | Perf report for the last 5.13792 seconds: | 20260505 10:52:41.064 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:52:41.064 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:41.064 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:41.064 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:41.064 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 10:52:41.064 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:41.064 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:41.064 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 10:52:41.064 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:41.064 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:41.064 | INFO | Uploaded 36GiB to GPU in 2.20654s (16.3151GiB/s) | 20260505 10:52:41.064 | INFO | Downloaded 34GiB to GPU in 2.12863s (15.9727GiB/s) | 20260505 10:52:46.211 | INFO | Analyzing chunk frames = [2637312,2637824) | 20260505 10:52:46.211 | INFO | MemInfo: { | 20260505 10:52:46.211 | INFO | "gpu" : " 17 GB", | 20260505 10:52:46.211 | INFO | "huge" : "", | 20260505 10:52:46.211 | INFO | "huge_shared" : " 27 GB", | 20260505 10:52:46.211 | INFO | "malloc" : " 14 GB", | 20260505 10:52:46.211 | INFO | "pinned" : " 29 GB" | 20260505 10:52:46.211 | INFO | } | 20260505 10:52:46.212 | INFO | Perf report for the last 5.14816 seconds: | 20260505 10:52:46.212 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:52:46.212 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:46.212 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:46.212 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:46.212 | INFO | ANALYSIS : 64.4%, 3.0, 100.7ms, 0 | 20260505 10:52:46.212 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:46.212 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:46.212 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 10:52:46.212 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:46.212 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:46.212 | INFO | Uploaded 36GiB to GPU in 2.25097s (15.9931GiB/s) | 20260505 10:52:46.212 | INFO | Downloaded 34GiB to GPU in 2.09077s (16.262GiB/s) | 20260505 10:52:46.212 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 10:52:46.212 | INFO | Total mem usage: 15.3 GiB | 20260505 10:52:46.212 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 10:52:46.212 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -6.82%) | 20260505 10:52:46.212 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 10:52:46.212 | INFO | High Water Breakdown: | 20260505 10:52:46.212 | INFO | 61% : 8.75 GiB ( 92.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 10:52:46.212 | INFO | 38.9% : 5.57 GiB ( 83.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 10:52:46.212 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 10:52:46.212 | INFO | ------------------------------------------ | 20260505 10:52:46.212 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:52:46.212 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 10:52:46.212 | INFO | Total mem usage: 26.5 GiB | 20260505 10:52:46.212 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 10:52:46.212 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 10:52:46.212 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 10:52:46.212 | INFO | High Water Breakdown: | 20260505 10:52:46.212 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:52:46.212 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:52:46.212 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:52:46.212 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:52:46.212 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:52:46.212 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:52:46.212 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 10:52:46.212 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 10:52:46.212 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:52:46.212 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 10:52:46.212 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 10:52:46.212 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 10:52:46.212 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 10:52:46.213 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 10:52:46.213 | INFO | ------------------------------------------ | 20260505 10:52:46.213 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:52:46.213 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 10:52:46.213 | INFO | Total mem usage: 25.6 GiB | 20260505 10:52:46.213 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 10:52:46.213 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 10:52:46.213 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 10:52:46.213 | INFO | High Water Breakdown: | 20260505 10:52:46.213 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 10:52:46.213 | INFO | ------------------------------------------ | 20260505 10:52:46.213 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:52:46.213 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 10:52:46.213 | INFO | Total mem usage: 15.7 GiB | 20260505 10:52:46.213 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 10:52:46.213 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 10:52:46.213 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 10:52:46.213 | INFO | High Water Breakdown: | 20260505 10:52:46.213 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 10:52:46.213 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 10:52:46.213 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 10:52:46.213 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 10:52:46.213 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 10:52:46.213 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 10:52:46.213 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 10:52:46.213 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 10:52:46.213 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 10:52:46.213 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:52:46.213 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 10:52:46.213 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:52:46.213 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 10:52:46.213 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 10:52:46.213 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 10:52:46.213 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 10:52:46.213 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:52:46.213 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 10:52:46.213 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:52:46.213 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:52:46.213 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:52:46.213 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 10:52:46.213 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 10:52:46.213 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:52:46.213 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 10:52:46.213 | INFO | ------------------------------------------ | 20260505 10:52:46.213 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:52:46.213 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 10:52:46.213 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:52:46.213 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:52:46.213 | INFO | PerfReport: Full Execution -- 3.23e+01 / 7.80e+01, 1.39e+01 / 1.73e+00 | 20260505 10:52:46.213 | INFO | PerfReport: 0.29%, AggregateStats -- 9.30e-02 / 1.02e-01, 1.05e-02 / 3.28e-03 | 20260505 10:52:46.213 | INFO | PerfReport: 11.67%, Baseline -- 3.77e+00 / 4.46e+00, 6.02e-01 / 1.99e-01 | 20260505 10:52:46.213 | INFO | PerfReport: 3.32%, Histogram -- 1.07e+00 / 1.49e+00, 2.08e-01 / 8.69e-02 | 20260505 10:52:46.213 | INFO | PerfReport: 15.77%, DME -- 4.08e+01 / 4.54e+01, 4.39e+00 / 2.15e+00 | 20260505 10:52:46.213 | INFO | PerfReport: 56.58%, FrameLabeling -- 1.83e+01 / 2.17e+01, 1.76e+00 / 1.46e+00 | 20260505 10:52:46.213 | INFO | PerfReport: 7.33%, PulseAccumulating -- 2.37e+00 / 2.87e+00, 4.09e-01 / 2.22e-01 | 20260505 10:52:46.213 | INFO | PerfReport: 3.73%, Metrics -- 1.20e+00 / 1.50e+00, 3.47e-01 / 1.81e-01 | 20260505 10:52:46.213 | INFO | PerfReport: 1.32%, PulseSerializing -- 4.25e-01 / 4.72e-01, 7.88e-02 / 3.29e-02 | 20260505 10:52:46.213 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 10:52:46.213 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:52:46.213 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:52:46.213 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.46e+01 / 0.00e+00 | 20260505 10:52:46.213 | INFO | PerfReport: 22.63%, Upload -- 2.30e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260505 10:52:46.213 | INFO | PerfReport: 21.94%, Download -- 2.23e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260505 10:52:46.213 | INFO | PerfReport: 31.84%, Compute -- 3.23e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 10:52:46.213 | INFO | PerfReport: 1.96%, LockUpload -- 1.99e+00 / 0.00e+00, 7.45e+00 / 0.00e+00 | 20260505 10:52:46.213 | INFO | PerfReport: 1.20%, LockDownload -- 1.22e+00 / 0.00e+00, 2.82e+00 / 0.00e+00 | 20260505 10:52:46.213 | INFO | PerfReport: 20.42%, LockCompute -- 2.07e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260505 10:52:46.213 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.86e-02 / 0.00e+00 | 20260505 10:52:46.343 | INFO | Starting a round of Baz IO | 20260505 10:52:49.612 | INFO | Populating RTMetrics report | 20260505 10:52:49.612 | INFO | RTMetrics report populated | 20260505 10:52:49.612 | INFO | Opening tmp RTMetrics json file | 20260505 10:52:49.631 | INFO | Serializing RTMetrics to json | 20260505 10:52:49.645 | INFO | Serialization done, swapping tmp file now... | 20260505 10:52:49.646 | INFO | RTMetrics done | 20260505 10:52:49.745 | INFO | Finished a round of Baz IO: Wrote 2.397 GiB in 3.402 seconds | 20260505 10:52:49.745 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.61% / 0.01% / 75.38% | 20260505 10:52:49.745 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.32924% / 2.38419e-05% / 7.54992e-05% | 20260505 10:52:49.745 | INFO | Baz IO overall: | 20260505 10:52:49.745 | INFO | BytesWritten : 1552293504097 | 20260505 10:52:49.745 | INFO | NumEvents : 739103846017 | 20260505 10:52:51.353 | INFO | Analyzing chunk frames = [2637824,2638336) | 20260505 10:52:51.353 | INFO | MemInfo: { | 20260505 10:52:51.353 | INFO | "gpu" : " 17 GB", | 20260505 10:52:51.353 | INFO | "huge" : "", | 20260505 10:52:51.353 | INFO | "huge_shared" : " 27 GB", | 20260505 10:52:51.353 | INFO | "malloc" : " 14 GB", | 20260505 10:52:51.353 | INFO | "pinned" : " 29 GB" | 20260505 10:52:51.353 | INFO | } | 20260505 10:52:51.353 | INFO | Perf report for the last 5.14189 seconds: | 20260505 10:52:51.354 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:52:51.354 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:51.354 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:51.354 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:51.354 | INFO | ANALYSIS : 64.9%, 2.9, 99.9ms, 0 | 20260505 10:52:51.354 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:51.354 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260505 10:52:51.354 | INFO | PRE_HQ : 61.1%, 1.9, 59.5ms, 0 | 20260505 10:52:51.354 | INFO | BAZWRITER : 65.4%, 4.1, 139.4ms, 0 | 20260505 10:52:51.354 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 10:52:51.354 | INFO | Uploaded 36GiB to GPU in 2.23128s (16.1342GiB/s) | 20260505 10:52:51.354 | INFO | Downloaded 38GiB to GPU in 2.28824s (16.6066GiB/s) | 20260505 10:52:56.488 | INFO | Analyzing chunk frames = [2638336,2638848) | 20260505 10:52:56.488 | INFO | MemInfo: { | 20260505 10:52:56.488 | INFO | "gpu" : " 17 GB", | 20260505 10:52:56.488 | INFO | "huge" : "", | 20260505 10:52:56.488 | INFO | "huge_shared" : " 27 GB", | 20260505 10:52:56.488 | INFO | "malloc" : " 14 GB", | 20260505 10:52:56.488 | INFO | "pinned" : " 29 GB" | 20260505 10:52:56.488 | INFO | } | 20260505 10:52:56.488 | INFO | Perf report for the last 5.13433 seconds: | 20260505 10:52:56.488 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:52:56.488 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:56.488 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:56.488 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:56.488 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 10:52:56.488 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:56.488 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:56.488 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:52:56.488 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:56.488 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:52:56.488 | INFO | Uploaded 36GiB to GPU in 2.23569s (16.1024GiB/s) | 20260505 10:52:56.488 | INFO | Downloaded 34GiB to GPU in 2.08274s (16.3247GiB/s) | 20260505 10:53:01.620 | INFO | Analyzing chunk frames = [2638848,2639360) | 20260505 10:53:01.620 | INFO | MemInfo: { | 20260505 10:53:01.620 | INFO | "gpu" : " 17 GB", | 20260505 10:53:01.620 | INFO | "huge" : "", | 20260505 10:53:01.620 | INFO | "huge_shared" : " 27 GB", | 20260505 10:53:01.620 | INFO | "malloc" : " 14 GB", | 20260505 10:53:01.621 | INFO | "pinned" : " 29 GB" | 20260505 10:53:01.621 | INFO | } | 20260505 10:53:01.621 | INFO | Perf report for the last 5.1328 seconds: | 20260505 10:53:01.621 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:53:01.621 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:01.621 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:01.621 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:01.621 | INFO | ANALYSIS : 63.0%, 3.0, 97.9ms, 0 | 20260505 10:53:01.621 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:01.621 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:01.621 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:53:01.621 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:01.621 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:01.621 | INFO | Uploaded 36GiB to GPU in 2.17221s (16.573GiB/s) | 20260505 10:53:01.621 | INFO | Downloaded 34GiB to GPU in 2.06317s (16.4795GiB/s) | 20260505 10:53:06.763 | INFO | Analyzing chunk frames = [2639360,2639872) | 20260505 10:53:06.763 | INFO | MemInfo: { | 20260505 10:53:06.763 | INFO | "gpu" : " 17 GB", | 20260505 10:53:06.763 | INFO | "huge" : "", | 20260505 10:53:06.763 | INFO | "huge_shared" : " 27 GB", | 20260505 10:53:06.764 | INFO | "malloc" : " 14 GB", | 20260505 10:53:06.764 | INFO | "pinned" : " 29 GB" | 20260505 10:53:06.764 | INFO | } | 20260505 10:53:06.764 | INFO | Perf report for the last 5.14299 seconds: | 20260505 10:53:06.764 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:53:06.764 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:06.764 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:06.764 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:06.764 | INFO | ANALYSIS : 63.1%, 3.0, 98.0ms, 0 | 20260505 10:53:06.764 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:06.764 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:06.764 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:53:06.764 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:06.764 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:06.764 | INFO | Uploaded 36GiB to GPU in 2.17962s (16.5166GiB/s) | 20260505 10:53:06.764 | INFO | Downloaded 34GiB to GPU in 2.075s (16.3855GiB/s) | 20260505 10:53:11.905 | INFO | Analyzing chunk frames = [2639872,2640384) | 20260505 10:53:11.905 | INFO | MemInfo: { | 20260505 10:53:11.905 | INFO | "gpu" : " 17 GB", | 20260505 10:53:11.905 | INFO | "huge" : "", | 20260505 10:53:11.905 | INFO | "huge_shared" : " 27 GB", | 20260505 10:53:11.905 | INFO | "malloc" : " 14 GB", | 20260505 10:53:11.905 | INFO | "pinned" : " 29 GB" | 20260505 10:53:11.905 | INFO | } | 20260505 10:53:11.905 | INFO | Perf report for the last 5.14143 seconds: | 20260505 10:53:11.905 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:53:11.905 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:11.905 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:11.905 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:11.905 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 10:53:11.905 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:11.905 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:11.905 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:53:11.905 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:11.905 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:11.905 | INFO | Uploaded 36GiB to GPU in 2.12505s (16.9408GiB/s) | 20260505 10:53:11.905 | INFO | Downloaded 34GiB to GPU in 2.12526s (15.998GiB/s) | 20260505 10:53:17.050 | INFO | Analyzing chunk frames = [2640384,2640896) | 20260505 10:53:17.050 | INFO | MemInfo: { | 20260505 10:53:17.050 | INFO | "gpu" : " 17 GB", | 20260505 10:53:17.050 | INFO | "huge" : "", | 20260505 10:53:17.050 | INFO | "huge_shared" : " 27 GB", | 20260505 10:53:17.051 | INFO | "malloc" : " 14 GB", | 20260505 10:53:17.051 | INFO | "pinned" : " 29 GB" | 20260505 10:53:17.051 | INFO | } | 20260505 10:53:17.051 | INFO | Perf report for the last 5.14558 seconds: | 20260505 10:53:17.051 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:53:17.051 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:17.051 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:17.051 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:17.051 | INFO | ANALYSIS : 63.4%, 3.0, 99.2ms, 0 | 20260505 10:53:17.051 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:17.051 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:17.051 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:53:17.051 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:17.051 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:17.051 | INFO | Uploaded 36GiB to GPU in 2.20559s (16.3222GiB/s) | 20260505 10:53:17.051 | INFO | Downloaded 34GiB to GPU in 2.10255s (16.1708GiB/s) | 20260505 10:53:22.177 | INFO | Analyzing chunk frames = [2640896,2641408) | 20260505 10:53:22.177 | INFO | MemInfo: { | 20260505 10:53:22.177 | INFO | "gpu" : " 17 GB", | 20260505 10:53:22.177 | INFO | "huge" : "", | 20260505 10:53:22.177 | INFO | "huge_shared" : " 27 GB", | 20260505 10:53:22.177 | INFO | "malloc" : " 14 GB", | 20260505 10:53:22.177 | INFO | "pinned" : " 29 GB" | 20260505 10:53:22.177 | INFO | } | 20260505 10:53:22.177 | INFO | Perf report for the last 5.12664 seconds: | 20260505 10:53:22.177 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:53:22.177 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:22.177 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:22.177 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:22.177 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 10:53:22.177 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:22.177 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:22.177 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260505 10:53:22.177 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:22.177 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:22.178 | INFO | Uploaded 36GiB to GPU in 2.17926s (16.5194GiB/s) | 20260505 10:53:22.178 | INFO | Downloaded 34GiB to GPU in 2.15537s (15.7745GiB/s) | 20260505 10:53:27.318 | INFO | Analyzing chunk frames = [2641408,2641920) | 20260505 10:53:27.318 | INFO | MemInfo: { | 20260505 10:53:27.318 | INFO | "gpu" : " 17 GB", | 20260505 10:53:27.318 | INFO | "huge" : "", | 20260505 10:53:27.318 | INFO | "huge_shared" : " 27 GB", | 20260505 10:53:27.318 | INFO | "malloc" : " 14 GB", | 20260505 10:53:27.318 | INFO | "pinned" : " 29 GB" | 20260505 10:53:27.318 | INFO | } | 20260505 10:53:27.318 | INFO | Perf report for the last 5.14056 seconds: | 20260505 10:53:27.318 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:53:27.318 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:27.318 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:27.318 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:27.318 | INFO | ANALYSIS : 64.4%, 3.0, 100.7ms, 0 | 20260505 10:53:27.318 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:27.318 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:27.318 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 10:53:27.318 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:27.318 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:27.318 | INFO | Uploaded 36GiB to GPU in 2.24815s (16.0131GiB/s) | 20260505 10:53:27.318 | INFO | Downloaded 34GiB to GPU in 2.08867s (16.2783GiB/s) | 20260505 10:53:27.457 | INFO | Starting a round of Baz IO | 20260505 10:53:30.719 | INFO | Populating RTMetrics report | 20260505 10:53:30.719 | INFO | RTMetrics report populated | 20260505 10:53:30.719 | INFO | Opening tmp RTMetrics json file | 20260505 10:53:30.720 | INFO | Serializing RTMetrics to json | 20260505 10:53:30.721 | INFO | Serialization done, swapping tmp file now... | 20260505 10:53:30.721 | INFO | RTMetrics done | 20260505 10:53:30.849 | INFO | Finished a round of Baz IO: Wrote 2.396 GiB in 3.391 seconds | 20260505 10:53:30.849 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.60% / 0.01% / 75.39% | 20260505 10:53:30.849 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.32399% / 0% / 0.000111262% | 20260505 10:53:30.849 | INFO | Baz IO overall: | 20260505 10:53:30.849 | INFO | BytesWritten : 1554866538288 | 20260505 10:53:30.849 | INFO | NumEvents : 740329131302 | 20260505 10:53:32.466 | INFO | Analyzing chunk frames = [2641920,2642432) | 20260505 10:53:32.466 | INFO | MemInfo: { | 20260505 10:53:32.466 | INFO | "gpu" : " 17 GB", | 20260505 10:53:32.466 | INFO | "huge" : "", | 20260505 10:53:32.466 | INFO | "huge_shared" : " 27 GB", | 20260505 10:53:32.466 | INFO | "malloc" : " 14 GB", | 20260505 10:53:32.466 | INFO | "pinned" : " 29 GB" | 20260505 10:53:32.466 | INFO | } | 20260505 10:53:32.466 | INFO | Perf report for the last 5.14862 seconds: | 20260505 10:53:32.467 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:53:32.467 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:32.467 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:32.467 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:32.467 | INFO | ANALYSIS : 64.9%, 2.9, 99.9ms, 0 | 20260505 10:53:32.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:32.467 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260505 10:53:32.467 | INFO | PRE_HQ : 61.6%, 1.8, 59.2ms, 0 | 20260505 10:53:32.467 | INFO | BAZWRITER : 65.2%, 4.1, 138.5ms, 0 | 20260505 10:53:32.467 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 10:53:32.467 | INFO | Uploaded 36GiB to GPU in 2.22525s (16.178GiB/s) | 20260505 10:53:32.467 | INFO | Downloaded 38GiB to GPU in 2.29195s (16.5798GiB/s) | 20260505 10:53:37.604 | INFO | Analyzing chunk frames = [2642432,2642944) | 20260505 10:53:37.604 | INFO | MemInfo: { | 20260505 10:53:37.604 | INFO | "gpu" : " 17 GB", | 20260505 10:53:37.604 | INFO | "huge" : "", | 20260505 10:53:37.604 | INFO | "huge_shared" : " 27 GB", | 20260505 10:53:37.604 | INFO | "malloc" : " 14 GB", | 20260505 10:53:37.604 | INFO | "pinned" : " 29 GB" | 20260505 10:53:37.605 | INFO | } | 20260505 10:53:37.605 | INFO | Perf report for the last 5.13808 seconds: | 20260505 10:53:37.605 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:53:37.605 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:37.605 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:37.605 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:37.605 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 10:53:37.605 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:37.605 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:37.605 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 10:53:37.605 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:37.605 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:37.605 | INFO | Uploaded 36GiB to GPU in 2.23454s (16.1107GiB/s) | 20260505 10:53:37.605 | INFO | Downloaded 34GiB to GPU in 2.09409s (16.2362GiB/s) | 20260505 10:53:42.736 | INFO | Analyzing chunk frames = [2642944,2643456) | 20260505 10:53:42.736 | INFO | MemInfo: { | 20260505 10:53:42.736 | INFO | "gpu" : " 17 GB", | 20260505 10:53:42.736 | INFO | "huge" : "", | 20260505 10:53:42.736 | INFO | "huge_shared" : " 27 GB", | 20260505 10:53:42.736 | INFO | "malloc" : " 14 GB", | 20260505 10:53:42.736 | INFO | "pinned" : " 29 GB" | 20260505 10:53:42.736 | INFO | } | 20260505 10:53:42.736 | INFO | Perf report for the last 5.13154 seconds: | 20260505 10:53:42.736 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:53:42.736 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:42.736 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:42.736 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:42.736 | INFO | ANALYSIS : 63.7%, 3.0, 99.0ms, 0 | 20260505 10:53:42.736 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:42.736 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:42.736 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 10:53:42.736 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:42.736 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:42.736 | INFO | Uploaded 36GiB to GPU in 2.23224s (16.1273GiB/s) | 20260505 10:53:42.736 | INFO | Downloaded 34GiB to GPU in 2.12387s (16.0085GiB/s) | 20260505 10:53:47.876 | INFO | Analyzing chunk frames = [2643456,2643968) | 20260505 10:53:47.876 | INFO | MemInfo: { | 20260505 10:53:47.876 | INFO | "gpu" : " 17 GB", | 20260505 10:53:47.876 | INFO | "huge" : "", | 20260505 10:53:47.876 | INFO | "huge_shared" : " 27 GB", | 20260505 10:53:47.876 | INFO | "malloc" : " 14 GB", | 20260505 10:53:47.876 | INFO | "pinned" : " 29 GB" | 20260505 10:53:47.876 | INFO | } | 20260505 10:53:47.876 | INFO | Perf report for the last 5.14008 seconds: | 20260505 10:53:47.876 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:53:47.876 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:47.876 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:47.876 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:47.876 | INFO | ANALYSIS : 63.1%, 3.0, 98.0ms, 0 | 20260505 10:53:47.876 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:47.876 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:47.876 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:53:47.876 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:47.876 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:47.877 | INFO | Uploaded 36GiB to GPU in 2.17393s (16.5599GiB/s) | 20260505 10:53:47.877 | INFO | Downloaded 34GiB to GPU in 2.07407s (16.3929GiB/s) | 20260505 10:53:53.021 | INFO | Analyzing chunk frames = [2643968,2644480) | 20260505 10:53:53.021 | INFO | MemInfo: { | 20260505 10:53:53.021 | INFO | "gpu" : " 17 GB", | 20260505 10:53:53.021 | INFO | "huge" : "", | 20260505 10:53:53.021 | INFO | "huge_shared" : " 27 GB", | 20260505 10:53:53.021 | INFO | "malloc" : " 14 GB", | 20260505 10:53:53.021 | INFO | "pinned" : " 29 GB" | 20260505 10:53:53.021 | INFO | } | 20260505 10:53:53.021 | INFO | Perf report for the last 5.1446 seconds: | 20260505 10:53:53.021 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:53:53.021 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:53.021 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:53.021 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:53.021 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 10:53:53.021 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:53.021 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:53.021 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:53:53.021 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:53.021 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:53.021 | INFO | Uploaded 36GiB to GPU in 2.23495s (16.1077GiB/s) | 20260505 10:53:53.021 | INFO | Downloaded 34GiB to GPU in 2.1114s (16.103GiB/s) | 20260505 10:53:58.161 | INFO | Analyzing chunk frames = [2644480,2644992) | 20260505 10:53:58.161 | INFO | MemInfo: { | 20260505 10:53:58.161 | INFO | "gpu" : " 17 GB", | 20260505 10:53:58.161 | INFO | "huge" : "", | 20260505 10:53:58.161 | INFO | "huge_shared" : " 27 GB", | 20260505 10:53:58.161 | INFO | "malloc" : " 14 GB", | 20260505 10:53:58.161 | INFO | "pinned" : " 29 GB" | 20260505 10:53:58.161 | INFO | } | 20260505 10:53:58.161 | INFO | Perf report for the last 5.14048 seconds: | 20260505 10:53:58.161 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:53:58.161 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:58.161 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:58.161 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:58.161 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 10:53:58.161 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:58.161 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:58.161 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 10:53:58.161 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:58.161 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:53:58.162 | INFO | Uploaded 36GiB to GPU in 2.19053s (16.4343GiB/s) | 20260505 10:53:58.162 | INFO | Downloaded 34GiB to GPU in 2.11923s (16.0436GiB/s) | 20260505 10:54:03.307 | INFO | Analyzing chunk frames = [2644992,2645504) | 20260505 10:54:03.307 | INFO | MemInfo: { | 20260505 10:54:03.307 | INFO | "gpu" : " 17 GB", | 20260505 10:54:03.307 | INFO | "huge" : "", | 20260505 10:54:03.307 | INFO | "huge_shared" : " 27 GB", | 20260505 10:54:03.307 | INFO | "malloc" : " 14 GB", | 20260505 10:54:03.307 | INFO | "pinned" : " 29 GB" | 20260505 10:54:03.307 | INFO | } | 20260505 10:54:03.307 | INFO | Perf report for the last 5.14543 seconds: | 20260505 10:54:03.307 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:54:03.307 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:03.307 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:03.307 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:03.307 | INFO | ANALYSIS : 63.8%, 3.0, 99.8ms, 0 | 20260505 10:54:03.307 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:03.307 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:03.307 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 10:54:03.307 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:03.307 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:03.307 | INFO | Uploaded 36GiB to GPU in 2.18686s (16.4619GiB/s) | 20260505 10:54:03.307 | INFO | Downloaded 34GiB to GPU in 2.13476s (15.9269GiB/s) | 20260505 10:54:08.434 | INFO | Analyzing chunk frames = [2645504,2646016) | 20260505 10:54:08.434 | INFO | MemInfo: { | 20260505 10:54:08.434 | INFO | "gpu" : " 17 GB", | 20260505 10:54:08.434 | INFO | "huge" : "", | 20260505 10:54:08.434 | INFO | "huge_shared" : " 27 GB", | 20260505 10:54:08.434 | INFO | "malloc" : " 14 GB", | 20260505 10:54:08.434 | INFO | "pinned" : " 29 GB" | 20260505 10:54:08.434 | INFO | } | 20260505 10:54:08.435 | INFO | Perf report for the last 5.12771 seconds: | 20260505 10:54:08.435 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:54:08.435 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:08.435 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:08.435 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:08.435 | INFO | ANALYSIS : 64.2%, 3.0, 100.0ms, 0 | 20260505 10:54:08.435 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:08.435 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:08.435 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 10:54:08.435 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:08.435 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:08.435 | INFO | Uploaded 36GiB to GPU in 2.22731s (16.163GiB/s) | 20260505 10:54:08.435 | INFO | Downloaded 34GiB to GPU in 2.0916s (16.2555GiB/s) | 20260505 10:54:08.435 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 10:54:08.435 | INFO | Total mem usage: 15.3 GiB | 20260505 10:54:08.435 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 10:54:08.435 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -6.82%) | 20260505 10:54:08.435 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 10:54:08.435 | INFO | High Water Breakdown: | 20260505 10:54:08.435 | INFO | 61% : 8.75 GiB ( 92.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 10:54:08.435 | INFO | 38.9% : 5.57 GiB ( 83.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 10:54:08.435 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 10:54:08.435 | INFO | ------------------------------------------ | 20260505 10:54:08.435 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:54:08.435 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 10:54:08.435 | INFO | Total mem usage: 26.5 GiB | 20260505 10:54:08.435 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 10:54:08.435 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 10:54:08.435 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 10:54:08.435 | INFO | High Water Breakdown: | 20260505 10:54:08.435 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:54:08.435 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:54:08.435 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:54:08.435 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:54:08.435 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:54:08.435 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:54:08.435 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 10:54:08.435 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 10:54:08.435 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:54:08.435 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 10:54:08.435 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 10:54:08.435 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 10:54:08.435 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 10:54:08.435 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 10:54:08.435 | INFO | ------------------------------------------ | 20260505 10:54:08.435 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:54:08.435 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 10:54:08.435 | INFO | Total mem usage: 25.6 GiB | 20260505 10:54:08.435 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 10:54:08.435 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 10:54:08.435 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 10:54:08.435 | INFO | High Water Breakdown: | 20260505 10:54:08.435 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 10:54:08.435 | INFO | ------------------------------------------ | 20260505 10:54:08.436 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:54:08.436 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 10:54:08.436 | INFO | Total mem usage: 15.7 GiB | 20260505 10:54:08.436 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 10:54:08.436 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 10:54:08.436 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 10:54:08.436 | INFO | High Water Breakdown: | 20260505 10:54:08.436 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 10:54:08.436 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 10:54:08.436 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 10:54:08.436 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 10:54:08.436 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 10:54:08.436 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 10:54:08.436 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 10:54:08.436 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 10:54:08.436 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 10:54:08.436 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:54:08.436 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 10:54:08.436 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:54:08.436 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 10:54:08.436 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 10:54:08.436 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 10:54:08.436 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 10:54:08.436 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:54:08.436 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 10:54:08.436 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:54:08.436 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:54:08.436 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:54:08.436 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 10:54:08.436 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 10:54:08.436 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:54:08.436 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 10:54:08.436 | INFO | ------------------------------------------ | 20260505 10:54:08.436 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:54:08.436 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 10:54:08.436 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:54:08.436 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:54:08.436 | INFO | PerfReport: Full Execution -- 3.22e+01 / 7.71e+01, 1.37e+01 / 1.85e+00 | 20260505 10:54:08.436 | INFO | PerfReport: 0.29%, AggregateStats -- 9.34e-02 / 1.02e-01, 1.11e-02 / 3.24e-03 | 20260505 10:54:08.436 | INFO | PerfReport: 11.70%, Baseline -- 3.77e+00 / 4.54e+00, 6.03e-01 / 1.32e-01 | 20260505 10:54:08.436 | INFO | PerfReport: 3.32%, Histogram -- 1.07e+00 / 1.52e+00, 2.08e-01 / 6.19e-02 | 20260505 10:54:08.436 | INFO | PerfReport: 15.77%, DME -- 4.07e+01 / 4.50e+01, 4.42e+00 / 1.45e+00 | 20260505 10:54:08.436 | INFO | PerfReport: 56.51%, FrameLabeling -- 1.82e+01 / 2.10e+01, 1.73e+00 / 1.54e+00 | 20260505 10:54:08.436 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.38e+00 / 2.92e+00, 4.19e-01 / 1.73e-01 | 20260505 10:54:08.436 | INFO | PerfReport: 3.71%, Metrics -- 1.20e+00 / 1.49e+00, 3.43e-01 / 1.51e-01 | 20260505 10:54:08.436 | INFO | PerfReport: 1.32%, PulseSerializing -- 4.25e-01 / 4.74e-01, 7.94e-02 / 3.43e-02 | 20260505 10:54:08.436 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 10:54:08.436 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:54:08.436 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:54:08.436 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.46e+01 / 0.00e+00 | 20260505 10:54:08.436 | INFO | PerfReport: 22.66%, Upload -- 2.30e+01 / 0.00e+00, 8.31e+00 / 0.00e+00 | 20260505 10:54:08.436 | INFO | PerfReport: 21.85%, Download -- 2.22e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260505 10:54:08.436 | INFO | PerfReport: 31.81%, Compute -- 3.23e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 10:54:08.436 | INFO | PerfReport: 2.10%, LockUpload -- 2.13e+00 / 0.00e+00, 7.67e+00 / 0.00e+00 | 20260505 10:54:08.436 | INFO | PerfReport: 1.09%, LockDownload -- 1.11e+00 / 0.00e+00, 2.52e+00 / 0.00e+00 | 20260505 10:54:08.436 | INFO | PerfReport: 20.47%, LockCompute -- 2.08e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260505 10:54:08.436 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.98e-02 / 0.00e+00 | 20260505 10:54:08.566 | INFO | Starting a round of Baz IO | 20260505 10:54:11.823 | INFO | Populating RTMetrics report | 20260505 10:54:11.823 | INFO | RTMetrics report populated | 20260505 10:54:11.823 | INFO | Opening tmp RTMetrics json file | 20260505 10:54:11.839 | INFO | Serializing RTMetrics to json | 20260505 10:54:11.886 | INFO | Serialization done, swapping tmp file now... | 20260505 10:54:11.888 | INFO | RTMetrics done | 20260505 10:54:11.961 | INFO | Finished a round of Baz IO: Wrote 2.395 GiB in 3.394 seconds | 20260505 10:54:11.961 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.59% / 0.01% / 75.40% | 20260505 10:54:11.961 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.31812% / 2.38419e-05% / 8.74201e-05% | 20260505 10:54:11.961 | INFO | Baz IO overall: | 20260505 10:54:11.961 | INFO | BytesWritten : 1557437875036 | 20260505 10:54:11.961 | INFO | NumEvents : 741553239006 | 20260505 10:54:13.572 | INFO | Analyzing chunk frames = [2646016,2646528) | 20260505 10:54:13.572 | INFO | MemInfo: { | 20260505 10:54:13.572 | INFO | "gpu" : " 17 GB", | 20260505 10:54:13.572 | INFO | "huge" : "", | 20260505 10:54:13.572 | INFO | "huge_shared" : " 27 GB", | 20260505 10:54:13.572 | INFO | "malloc" : " 14 GB", | 20260505 10:54:13.572 | INFO | "pinned" : " 29 GB" | 20260505 10:54:13.572 | INFO | } | 20260505 10:54:13.572 | INFO | Perf report for the last 5.13785 seconds: | 20260505 10:54:13.573 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:54:13.573 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:13.573 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:13.573 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:13.573 | INFO | ANALYSIS : 64.8%, 2.9, 99.6ms, 0 | 20260505 10:54:13.573 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:13.573 | INFO | RT_METRICS : 46.7%, 1.0, 24.5ms, 0 | 20260505 10:54:13.573 | INFO | PRE_HQ : 62.8%, 1.8, 59.0ms, 0 | 20260505 10:54:13.573 | INFO | BAZWRITER : 65.4%, 4.2, 139.7ms, 0 | 20260505 10:54:13.573 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260505 10:54:13.573 | INFO | Uploaded 36GiB to GPU in 2.25542s (15.9616GiB/s) | 20260505 10:54:13.573 | INFO | Downloaded 38GiB to GPU in 2.3837s (15.9416GiB/s) | 20260505 10:54:18.716 | INFO | Analyzing chunk frames = [2646528,2647040) | 20260505 10:54:18.716 | INFO | MemInfo: { | 20260505 10:54:18.716 | INFO | "gpu" : " 17 GB", | 20260505 10:54:18.716 | INFO | "huge" : "", | 20260505 10:54:18.716 | INFO | "huge_shared" : " 27 GB", | 20260505 10:54:18.716 | INFO | "malloc" : " 14 GB", | 20260505 10:54:18.716 | INFO | "pinned" : " 29 GB" | 20260505 10:54:18.716 | INFO | } | 20260505 10:54:18.716 | INFO | Perf report for the last 5.14369 seconds: | 20260505 10:54:18.716 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:54:18.716 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:18.716 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:18.716 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:18.716 | INFO | ANALYSIS : 64.3%, 3.0, 100.5ms, 0 | 20260505 10:54:18.716 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:18.716 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:18.716 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 10:54:18.716 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:18.716 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:18.716 | INFO | Uploaded 36GiB to GPU in 2.23879s (16.0801GiB/s) | 20260505 10:54:18.716 | INFO | Downloaded 34GiB to GPU in 2.08453s (16.3106GiB/s) | 20260505 10:54:23.864 | INFO | Analyzing chunk frames = [2647040,2647552) | 20260505 10:54:23.864 | INFO | MemInfo: { | 20260505 10:54:23.864 | INFO | "gpu" : " 17 GB", | 20260505 10:54:23.864 | INFO | "huge" : "", | 20260505 10:54:23.864 | INFO | "huge_shared" : " 27 GB", | 20260505 10:54:23.864 | INFO | "malloc" : " 14 GB", | 20260505 10:54:23.864 | INFO | "pinned" : " 29 GB" | 20260505 10:54:23.864 | INFO | } | 20260505 10:54:23.864 | INFO | Perf report for the last 5.14782 seconds: | 20260505 10:54:23.864 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:54:23.864 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:23.864 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:23.864 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:23.864 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 10:54:23.864 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:23.864 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:23.864 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:54:23.864 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:23.864 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:23.864 | INFO | Uploaded 36GiB to GPU in 2.2325s (16.1254GiB/s) | 20260505 10:54:23.864 | INFO | Downloaded 34GiB to GPU in 2.11559s (16.0712GiB/s) | 20260505 10:54:28.998 | INFO | Analyzing chunk frames = [2647552,2648064) | 20260505 10:54:28.998 | INFO | MemInfo: { | 20260505 10:54:28.998 | INFO | "gpu" : " 17 GB", | 20260505 10:54:28.998 | INFO | "huge" : "", | 20260505 10:54:28.998 | INFO | "huge_shared" : " 27 GB", | 20260505 10:54:28.998 | INFO | "malloc" : " 14 GB", | 20260505 10:54:28.998 | INFO | "pinned" : " 29 GB" | 20260505 10:54:28.998 | INFO | } | 20260505 10:54:28.998 | INFO | Perf report for the last 5.13417 seconds: | 20260505 10:54:28.998 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:54:28.998 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:28.998 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:28.998 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:28.998 | INFO | ANALYSIS : 63.6%, 3.0, 98.8ms, 0 | 20260505 10:54:28.998 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:28.998 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:28.998 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 10:54:28.998 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:28.998 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:28.998 | INFO | Uploaded 36GiB to GPU in 2.22172s (16.2037GiB/s) | 20260505 10:54:28.998 | INFO | Downloaded 34GiB to GPU in 2.13007s (15.9619GiB/s) | 20260505 10:54:34.130 | INFO | Analyzing chunk frames = [2648064,2648576) | 20260505 10:54:34.130 | INFO | MemInfo: { | 20260505 10:54:34.130 | INFO | "gpu" : " 17 GB", | 20260505 10:54:34.130 | INFO | "huge" : "", | 20260505 10:54:34.130 | INFO | "huge_shared" : " 27 GB", | 20260505 10:54:34.130 | INFO | "malloc" : " 14 GB", | 20260505 10:54:34.130 | INFO | "pinned" : " 29 GB" | 20260505 10:54:34.130 | INFO | } | 20260505 10:54:34.130 | INFO | Perf report for the last 5.13193 seconds: | 20260505 10:54:34.130 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:54:34.130 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:34.130 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:34.130 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:34.130 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 10:54:34.130 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:34.130 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:34.130 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 10:54:34.130 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:34.130 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:34.130 | INFO | Uploaded 36GiB to GPU in 2.21564s (16.2481GiB/s) | 20260505 10:54:34.130 | INFO | Downloaded 34GiB to GPU in 2.1213s (16.0279GiB/s) | 20260505 10:54:39.274 | INFO | Analyzing chunk frames = [2648576,2649088) | 20260505 10:54:39.274 | INFO | MemInfo: { | 20260505 10:54:39.274 | INFO | "gpu" : " 17 GB", | 20260505 10:54:39.274 | INFO | "huge" : "", | 20260505 10:54:39.274 | INFO | "huge_shared" : " 27 GB", | 20260505 10:54:39.274 | INFO | "malloc" : " 14 GB", | 20260505 10:54:39.274 | INFO | "pinned" : " 29 GB" | 20260505 10:54:39.274 | INFO | } | 20260505 10:54:39.274 | INFO | Perf report for the last 5.14394 seconds: | 20260505 10:54:39.274 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:54:39.274 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:39.274 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:39.274 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:39.274 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 10:54:39.274 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:39.274 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:39.274 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 10:54:39.274 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:39.274 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:39.274 | INFO | Uploaded 36GiB to GPU in 2.23505s (16.107GiB/s) | 20260505 10:54:39.274 | INFO | Downloaded 34GiB to GPU in 2.10166s (16.1777GiB/s) | 20260505 10:54:44.414 | INFO | Analyzing chunk frames = [2649088,2649600) | 20260505 10:54:44.414 | INFO | MemInfo: { | 20260505 10:54:44.414 | INFO | "gpu" : " 17 GB", | 20260505 10:54:44.414 | INFO | "huge" : "", | 20260505 10:54:44.414 | INFO | "huge_shared" : " 27 GB", | 20260505 10:54:44.414 | INFO | "malloc" : " 14 GB", | 20260505 10:54:44.414 | INFO | "pinned" : " 29 GB" | 20260505 10:54:44.414 | INFO | } | 20260505 10:54:44.415 | INFO | Perf report for the last 5.14055 seconds: | 20260505 10:54:44.415 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:54:44.415 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:44.415 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:44.415 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:44.415 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 10:54:44.415 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:44.415 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:44.415 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 10:54:44.415 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:44.415 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:44.415 | INFO | Uploaded 36GiB to GPU in 2.18483s (16.4772GiB/s) | 20260505 10:54:44.415 | INFO | Downloaded 34GiB to GPU in 2.14904s (15.821GiB/s) | 20260505 10:54:49.552 | INFO | Analyzing chunk frames = [2649600,2650112) | 20260505 10:54:49.552 | INFO | MemInfo: { | 20260505 10:54:49.552 | INFO | "gpu" : " 17 GB", | 20260505 10:54:49.552 | INFO | "huge" : "", | 20260505 10:54:49.552 | INFO | "huge_shared" : " 27 GB", | 20260505 10:54:49.552 | INFO | "malloc" : " 14 GB", | 20260505 10:54:49.552 | INFO | "pinned" : " 29 GB" | 20260505 10:54:49.552 | INFO | } | 20260505 10:54:49.552 | INFO | Perf report for the last 5.13781 seconds: | 20260505 10:54:49.552 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:54:49.552 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:49.552 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:49.552 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:49.552 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 10:54:49.552 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:49.552 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:49.552 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:54:49.552 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:49.552 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:49.553 | INFO | Uploaded 36GiB to GPU in 2.21941s (16.2206GiB/s) | 20260505 10:54:49.553 | INFO | Downloaded 34GiB to GPU in 2.0833s (16.3203GiB/s) | 20260505 10:54:49.693 | INFO | Starting a round of Baz IO | 20260505 10:54:52.953 | INFO | Populating RTMetrics report | 20260505 10:54:52.953 | INFO | RTMetrics report populated | 20260505 10:54:52.953 | INFO | Opening tmp RTMetrics json file | 20260505 10:54:52.983 | INFO | Serializing RTMetrics to json | 20260505 10:54:53.005 | INFO | Serialization done, swapping tmp file now... | 20260505 10:54:53.008 | INFO | RTMetrics done | 20260505 10:54:53.076 | INFO | Finished a round of Baz IO: Wrote 2.394 GiB in 3.382 seconds | 20260505 10:54:53.076 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.58% / 0.01% / 75.41% | 20260505 10:54:53.076 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.31354% / 0% / 0.000115236% | 20260505 10:54:53.076 | INFO | Baz IO overall: | 20260505 10:54:53.076 | INFO | BytesWritten : 1560008884141 | 20260505 10:54:53.076 | INFO | NumEvents : 742777465086 | 20260505 10:54:54.690 | INFO | Analyzing chunk frames = [2650112,2650624) | 20260505 10:54:54.690 | INFO | MemInfo: { | 20260505 10:54:54.690 | INFO | "gpu" : " 17 GB", | 20260505 10:54:54.690 | INFO | "huge" : "", | 20260505 10:54:54.690 | INFO | "huge_shared" : " 27 GB", | 20260505 10:54:54.690 | INFO | "malloc" : " 14 GB", | 20260505 10:54:54.690 | INFO | "pinned" : " 29 GB" | 20260505 10:54:54.690 | INFO | } | 20260505 10:54:54.691 | INFO | Perf report for the last 5.13816 seconds: | 20260505 10:54:54.691 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:54:54.691 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:54.691 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:54.691 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:54.691 | INFO | ANALYSIS : 65.1%, 2.9, 100.2ms, 0 | 20260505 10:54:54.691 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:54.691 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260505 10:54:54.691 | INFO | PRE_HQ : 61.5%, 1.8, 58.9ms, 0 | 20260505 10:54:54.691 | INFO | BAZWRITER : 65.1%, 4.1, 137.8ms, 0 | 20260505 10:54:54.691 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 10:54:54.691 | INFO | Uploaded 36GiB to GPU in 2.18237s (16.4958GiB/s) | 20260505 10:54:54.691 | INFO | Downloaded 38GiB to GPU in 2.31117s (16.4419GiB/s) | 20260505 10:54:59.825 | INFO | Analyzing chunk frames = [2650624,2651136) | 20260505 10:54:59.825 | INFO | MemInfo: { | 20260505 10:54:59.825 | INFO | "gpu" : " 17 GB", | 20260505 10:54:59.825 | INFO | "huge" : "", | 20260505 10:54:59.825 | INFO | "huge_shared" : " 27 GB", | 20260505 10:54:59.825 | INFO | "malloc" : " 14 GB", | 20260505 10:54:59.826 | INFO | "pinned" : " 29 GB" | 20260505 10:54:59.826 | INFO | } | 20260505 10:54:59.826 | INFO | Perf report for the last 5.13504 seconds: | 20260505 10:54:59.826 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:54:59.826 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:59.826 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:59.826 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:59.826 | INFO | ANALYSIS : 64.3%, 3.0, 100.3ms, 0 | 20260505 10:54:59.826 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:59.826 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:59.826 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:54:59.826 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:59.826 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:54:59.826 | INFO | Uploaded 36GiB to GPU in 2.19589s (16.3943GiB/s) | 20260505 10:54:59.826 | INFO | Downloaded 34GiB to GPU in 2.10878s (16.1231GiB/s) | 20260505 10:55:04.975 | INFO | Analyzing chunk frames = [2651136,2651648) | 20260505 10:55:04.975 | INFO | MemInfo: { | 20260505 10:55:04.975 | INFO | "gpu" : " 17 GB", | 20260505 10:55:04.975 | INFO | "huge" : "", | 20260505 10:55:04.975 | INFO | "huge_shared" : " 27 GB", | 20260505 10:55:04.975 | INFO | "malloc" : " 14 GB", | 20260505 10:55:04.975 | INFO | "pinned" : " 29 GB" | 20260505 10:55:04.975 | INFO | } | 20260505 10:55:04.975 | INFO | Perf report for the last 5.14973 seconds: | 20260505 10:55:04.975 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:55:04.975 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:04.975 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:04.975 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:04.975 | INFO | ANALYSIS : 62.9%, 3.0, 98.0ms, 0 | 20260505 10:55:04.975 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:04.975 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:04.975 | INFO | PRE_HQ : 28.9%, 1.0, 15.1ms, 0 | 20260505 10:55:04.975 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:04.975 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:04.976 | INFO | Uploaded 36GiB to GPU in 2.15327s (16.7187GiB/s) | 20260505 10:55:04.976 | INFO | Downloaded 34GiB to GPU in 2.06634s (16.4542GiB/s) | 20260505 10:55:10.120 | INFO | Analyzing chunk frames = [2651648,2652160) | 20260505 10:55:10.120 | INFO | MemInfo: { | 20260505 10:55:10.120 | INFO | "gpu" : " 17 GB", | 20260505 10:55:10.120 | INFO | "huge" : "", | 20260505 10:55:10.120 | INFO | "huge_shared" : " 27 GB", | 20260505 10:55:10.120 | INFO | "malloc" : " 14 GB", | 20260505 10:55:10.120 | INFO | "pinned" : " 29 GB" | 20260505 10:55:10.120 | INFO | } | 20260505 10:55:10.120 | INFO | Perf report for the last 5.14513 seconds: | 20260505 10:55:10.120 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:55:10.120 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:10.120 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:10.120 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:10.121 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 10:55:10.121 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:10.121 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:10.121 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 10:55:10.121 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:10.121 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:10.121 | INFO | Uploaded 36GiB to GPU in 2.21156s (16.2781GiB/s) | 20260505 10:55:10.121 | INFO | Downloaded 34GiB to GPU in 2.13232s (15.9451GiB/s) | 20260505 10:55:15.251 | INFO | Analyzing chunk frames = [2652160,2652672) | 20260505 10:55:15.251 | INFO | MemInfo: { | 20260505 10:55:15.251 | INFO | "gpu" : " 17 GB", | 20260505 10:55:15.251 | INFO | "huge" : "", | 20260505 10:55:15.251 | INFO | "huge_shared" : " 27 GB", | 20260505 10:55:15.252 | INFO | "malloc" : " 14 GB", | 20260505 10:55:15.252 | INFO | "pinned" : " 29 GB" | 20260505 10:55:15.252 | INFO | } | 20260505 10:55:15.252 | INFO | Perf report for the last 5.13112 seconds: | 20260505 10:55:15.252 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:55:15.252 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:15.252 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:15.252 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:15.252 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 10:55:15.252 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:15.252 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:15.252 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 10:55:15.252 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:15.252 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:15.252 | INFO | Uploaded 36GiB to GPU in 2.17742s (16.5333GiB/s) | 20260505 10:55:15.252 | INFO | Downloaded 34GiB to GPU in 2.12624s (15.9906GiB/s) | 20260505 10:55:20.380 | INFO | Analyzing chunk frames = [2652672,2653184) | 20260505 10:55:20.380 | INFO | MemInfo: { | 20260505 10:55:20.380 | INFO | "gpu" : " 17 GB", | 20260505 10:55:20.380 | INFO | "huge" : "", | 20260505 10:55:20.380 | INFO | "huge_shared" : " 27 GB", | 20260505 10:55:20.380 | INFO | "malloc" : " 14 GB", | 20260505 10:55:20.380 | INFO | "pinned" : " 29 GB" | 20260505 10:55:20.380 | INFO | } | 20260505 10:55:20.380 | INFO | Perf report for the last 5.12886 seconds: | 20260505 10:55:20.380 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:55:20.380 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:20.381 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:20.381 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:20.381 | INFO | ANALYSIS : 63.7%, 3.0, 99.2ms, 0 | 20260505 10:55:20.381 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:20.381 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:20.381 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 10:55:20.381 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:20.381 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:20.381 | INFO | Uploaded 36GiB to GPU in 2.17547s (16.5482GiB/s) | 20260505 10:55:20.381 | INFO | Downloaded 34GiB to GPU in 2.11861s (16.0482GiB/s) | 20260505 10:55:25.525 | INFO | Analyzing chunk frames = [2653184,2653696) | 20260505 10:55:25.525 | INFO | MemInfo: { | 20260505 10:55:25.525 | INFO | "gpu" : " 17 GB", | 20260505 10:55:25.525 | INFO | "huge" : "", | 20260505 10:55:25.525 | INFO | "huge_shared" : " 27 GB", | 20260505 10:55:25.525 | INFO | "malloc" : " 14 GB", | 20260505 10:55:25.525 | INFO | "pinned" : " 29 GB" | 20260505 10:55:25.525 | INFO | } | 20260505 10:55:25.526 | INFO | Perf report for the last 5.14508 seconds: | 20260505 10:55:25.526 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:55:25.526 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:25.526 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:25.526 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:25.526 | INFO | ANALYSIS : 63.6%, 3.0, 99.5ms, 0 | 20260505 10:55:25.526 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:25.526 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:25.526 | INFO | PRE_HQ : 29.9%, 1.0, 15.6ms, 0 | 20260505 10:55:25.526 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:25.526 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:25.526 | INFO | Uploaded 36GiB to GPU in 2.15258s (16.7241GiB/s) | 20260505 10:55:25.526 | INFO | Downloaded 34GiB to GPU in 2.13861s (15.8982GiB/s) | 20260505 10:55:30.671 | INFO | Analyzing chunk frames = [2653696,2654208) | 20260505 10:55:30.671 | INFO | MemInfo: { | 20260505 10:55:30.671 | INFO | "gpu" : " 17 GB", | 20260505 10:55:30.671 | INFO | "huge" : "", | 20260505 10:55:30.671 | INFO | "huge_shared" : " 27 GB", | 20260505 10:55:30.671 | INFO | "malloc" : " 14 GB", | 20260505 10:55:30.671 | INFO | "pinned" : " 29 GB" | 20260505 10:55:30.671 | INFO | } | 20260505 10:55:30.671 | INFO | Perf report for the last 5.14519 seconds: | 20260505 10:55:30.671 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:55:30.671 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:30.671 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:30.671 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:30.671 | INFO | ANALYSIS : 63.9%, 3.0, 99.9ms, 0 | 20260505 10:55:30.671 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:30.671 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:30.671 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:55:30.671 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:30.671 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:30.671 | INFO | Uploaded 36GiB to GPU in 2.21819s (16.2295GiB/s) | 20260505 10:55:30.671 | INFO | Downloaded 34GiB to GPU in 2.09483s (16.2305GiB/s) | 20260505 10:55:30.671 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 10:55:30.671 | INFO | Total mem usage: 15.3 GiB | 20260505 10:55:30.671 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 10:55:30.671 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -6.04%) | 20260505 10:55:30.671 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 10:55:30.671 | INFO | High Water Breakdown: | 20260505 10:55:30.671 | INFO | 61.1% : 8.83 GiB ( 93.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 10:55:30.671 | INFO | 38.7% : 5.6 GiB ( 84.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 10:55:30.671 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 10:55:30.671 | INFO | ------------------------------------------ | 20260505 10:55:30.671 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:55:30.671 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 10:55:30.671 | INFO | Total mem usage: 26.5 GiB | 20260505 10:55:30.671 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 10:55:30.671 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 10:55:30.671 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 10:55:30.671 | INFO | High Water Breakdown: | 20260505 10:55:30.672 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:55:30.672 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:55:30.672 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:55:30.672 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:55:30.672 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:55:30.672 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:55:30.672 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 10:55:30.672 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 10:55:30.672 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:55:30.672 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 10:55:30.672 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 10:55:30.672 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 10:55:30.672 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 10:55:30.672 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 10:55:30.672 | INFO | ------------------------------------------ | 20260505 10:55:30.672 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:55:30.672 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 10:55:30.672 | INFO | Total mem usage: 25.6 GiB | 20260505 10:55:30.672 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 10:55:30.672 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 10:55:30.672 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 10:55:30.672 | INFO | High Water Breakdown: | 20260505 10:55:30.672 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 10:55:30.672 | INFO | ------------------------------------------ | 20260505 10:55:30.672 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:55:30.672 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 10:55:30.672 | INFO | Total mem usage: 15.7 GiB | 20260505 10:55:30.672 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 10:55:30.672 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 10:55:30.672 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 10:55:30.672 | INFO | High Water Breakdown: | 20260505 10:55:30.672 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 10:55:30.672 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 10:55:30.672 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 10:55:30.672 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 10:55:30.672 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 10:55:30.672 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 10:55:30.672 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 10:55:30.672 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 10:55:30.672 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 10:55:30.672 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:55:30.672 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 10:55:30.672 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:55:30.672 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 10:55:30.672 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 10:55:30.672 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 10:55:30.672 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 10:55:30.672 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:55:30.672 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 10:55:30.672 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:55:30.672 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:55:30.672 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:55:30.672 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 10:55:30.672 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 10:55:30.672 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:55:30.672 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 10:55:30.672 | INFO | ------------------------------------------ | 20260505 10:55:30.672 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:55:30.673 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 10:55:30.673 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:55:30.673 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:55:30.673 | INFO | PerfReport: Full Execution -- 3.23e+01 / 7.75e+01, 1.38e+01 / 2.05e+00 | 20260505 10:55:30.673 | INFO | PerfReport: 0.29%, AggregateStats -- 9.31e-02 / 1.01e-01, 1.05e-02 / 4.51e-03 | 20260505 10:55:30.673 | INFO | PerfReport: 11.70%, Baseline -- 3.77e+00 / 4.44e+00, 6.03e-01 / 2.37e-01 | 20260505 10:55:30.673 | INFO | PerfReport: 3.32%, Histogram -- 1.07e+00 / 1.48e+00, 2.06e-01 / 1.01e-01 | 20260505 10:55:30.673 | INFO | PerfReport: 15.76%, DME -- 4.07e+01 / 4.47e+01, 4.39e+00 / 2.53e+00 | 20260505 10:55:30.673 | INFO | PerfReport: 56.52%, FrameLabeling -- 1.82e+01 / 2.19e+01, 1.70e+00 / 2.09e+00 | 20260505 10:55:30.673 | INFO | PerfReport: 7.37%, PulseAccumulating -- 2.38e+00 / 2.86e+00, 4.16e-01 / 2.29e-01 | 20260505 10:55:30.673 | INFO | PerfReport: 3.73%, Metrics -- 1.20e+00 / 1.48e+00, 3.44e-01 / 2.02e-01 | 20260505 10:55:30.673 | INFO | PerfReport: 1.31%, PulseSerializing -- 4.24e-01 / 4.62e-01, 8.00e-02 / 4.11e-02 | 20260505 10:55:30.673 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 10:55:30.673 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:55:30.673 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:55:30.673 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.46e+01 / 0.00e+00 | 20260505 10:55:30.673 | INFO | PerfReport: 22.64%, Upload -- 2.30e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260505 10:55:30.673 | INFO | PerfReport: 21.99%, Download -- 2.23e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260505 10:55:30.673 | INFO | PerfReport: 31.80%, Compute -- 3.23e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 10:55:30.673 | INFO | PerfReport: 2.01%, LockUpload -- 2.04e+00 / 0.00e+00, 7.50e+00 / 0.00e+00 | 20260505 10:55:30.673 | INFO | PerfReport: 1.23%, LockDownload -- 1.25e+00 / 0.00e+00, 2.84e+00 / 0.00e+00 | 20260505 10:55:30.673 | INFO | PerfReport: 20.32%, LockCompute -- 2.06e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260505 10:55:30.673 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.67e-02 / 0.00e+00 | 20260505 10:55:30.806 | INFO | Starting a round of Baz IO | 20260505 10:55:34.064 | INFO | Populating RTMetrics report | 20260505 10:55:34.064 | INFO | RTMetrics report populated | 20260505 10:55:34.064 | INFO | Opening tmp RTMetrics json file | 20260505 10:55:34.065 | INFO | Serializing RTMetrics to json | 20260505 10:55:34.067 | INFO | Serialization done, swapping tmp file now... | 20260505 10:55:34.067 | INFO | RTMetrics done | 20260505 10:55:34.193 | INFO | Finished a round of Baz IO: Wrote 2.397 GiB in 3.387 seconds | 20260505 10:55:34.193 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.57% / 0.01% / 75.42% | 20260505 10:55:34.193 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.31449% / 0% / 7.54992e-05% | 20260505 10:55:34.193 | INFO | Baz IO overall: | 20260505 10:55:34.193 | INFO | BytesWritten : 1562582394295 | 20260505 10:55:34.194 | INFO | NumEvents : 744003309169 | 20260505 10:55:35.811 | INFO | Analyzing chunk frames = [2654208,2654720) | 20260505 10:55:35.811 | INFO | MemInfo: { | 20260505 10:55:35.811 | INFO | "gpu" : " 17 GB", | 20260505 10:55:35.811 | INFO | "huge" : "", | 20260505 10:55:35.811 | INFO | "huge_shared" : " 27 GB", | 20260505 10:55:35.811 | INFO | "malloc" : " 14 GB", | 20260505 10:55:35.811 | INFO | "pinned" : " 29 GB" | 20260505 10:55:35.811 | INFO | } | 20260505 10:55:35.811 | INFO | Perf report for the last 5.14004 seconds: | 20260505 10:55:35.811 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:55:35.811 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:35.811 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:35.811 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:35.811 | INFO | ANALYSIS : 64.9%, 2.9, 99.7ms, 0 | 20260505 10:55:35.811 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:35.811 | INFO | RT_METRICS : 45.5%, 1.0, 23.8ms, 0 | 20260505 10:55:35.811 | INFO | PRE_HQ : 61.4%, 1.9, 59.1ms, 0 | 20260505 10:55:35.811 | INFO | BAZWRITER : 65.1%, 4.1, 138.0ms, 0 | 20260505 10:55:35.811 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 10:55:35.811 | INFO | Uploaded 36GiB to GPU in 2.21357s (16.2633GiB/s) | 20260505 10:55:35.811 | INFO | Downloaded 38GiB to GPU in 2.29509s (16.5571GiB/s) | 20260505 10:55:40.948 | INFO | Analyzing chunk frames = [2654720,2655232) | 20260505 10:55:40.948 | INFO | MemInfo: { | 20260505 10:55:40.948 | INFO | "gpu" : " 17 GB", | 20260505 10:55:40.948 | INFO | "huge" : "", | 20260505 10:55:40.948 | INFO | "huge_shared" : " 27 GB", | 20260505 10:55:40.948 | INFO | "malloc" : " 14 GB", | 20260505 10:55:40.948 | INFO | "pinned" : " 29 GB" | 20260505 10:55:40.948 | INFO | } | 20260505 10:55:40.948 | INFO | Perf report for the last 5.13756 seconds: | 20260505 10:55:40.948 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:55:40.948 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:40.948 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:40.948 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:40.948 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 10:55:40.948 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:40.948 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:40.948 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 10:55:40.949 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:40.949 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:40.949 | INFO | Uploaded 36GiB to GPU in 2.22929s (16.1487GiB/s) | 20260505 10:55:40.949 | INFO | Downloaded 34GiB to GPU in 2.08782s (16.2849GiB/s) | 20260505 10:55:46.084 | INFO | Analyzing chunk frames = [2655232,2655744) | 20260505 10:55:46.084 | INFO | MemInfo: { | 20260505 10:55:46.084 | INFO | "gpu" : " 17 GB", | 20260505 10:55:46.084 | INFO | "huge" : "", | 20260505 10:55:46.084 | INFO | "huge_shared" : " 27 GB", | 20260505 10:55:46.084 | INFO | "malloc" : " 14 GB", | 20260505 10:55:46.084 | INFO | "pinned" : " 29 GB" | 20260505 10:55:46.084 | INFO | } | 20260505 10:55:46.084 | INFO | Perf report for the last 5.1354 seconds: | 20260505 10:55:46.084 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:55:46.084 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:46.084 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:46.084 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:46.084 | INFO | ANALYSIS : 63.8%, 3.0, 99.2ms, 0 | 20260505 10:55:46.084 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:46.084 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:46.084 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 10:55:46.084 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:46.084 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:46.084 | INFO | Uploaded 36GiB to GPU in 2.22483s (16.181GiB/s) | 20260505 10:55:46.084 | INFO | Downloaded 34GiB to GPU in 2.12743s (15.9817GiB/s) | 20260505 10:55:51.229 | INFO | Analyzing chunk frames = [2655744,2656256) | 20260505 10:55:51.229 | INFO | MemInfo: { | 20260505 10:55:51.229 | INFO | "gpu" : " 17 GB", | 20260505 10:55:51.230 | INFO | "huge" : "", | 20260505 10:55:51.230 | INFO | "huge_shared" : " 27 GB", | 20260505 10:55:51.230 | INFO | "malloc" : " 14 GB", | 20260505 10:55:51.230 | INFO | "pinned" : " 29 GB" | 20260505 10:55:51.230 | INFO | } | 20260505 10:55:51.230 | INFO | Perf report for the last 5.14587 seconds: | 20260505 10:55:51.230 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:55:51.230 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:51.230 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:51.230 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:51.230 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260505 10:55:51.230 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:51.230 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:51.230 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 10:55:51.230 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:51.230 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:51.230 | INFO | Uploaded 36GiB to GPU in 2.1955s (16.3972GiB/s) | 20260505 10:55:51.230 | INFO | Downloaded 34GiB to GPU in 2.13149s (15.9513GiB/s) | 20260505 10:55:56.363 | INFO | Analyzing chunk frames = [2656256,2656768) | 20260505 10:55:56.363 | INFO | MemInfo: { | 20260505 10:55:56.363 | INFO | "gpu" : " 17 GB", | 20260505 10:55:56.363 | INFO | "huge" : "", | 20260505 10:55:56.363 | INFO | "huge_shared" : " 27 GB", | 20260505 10:55:56.363 | INFO | "malloc" : " 14 GB", | 20260505 10:55:56.363 | INFO | "pinned" : " 29 GB" | 20260505 10:55:56.363 | INFO | } | 20260505 10:55:56.364 | INFO | Perf report for the last 5.13382 seconds: | 20260505 10:55:56.364 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:55:56.364 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:56.364 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:56.364 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:56.364 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 10:55:56.364 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:56.364 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:56.364 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 10:55:56.364 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:56.364 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:55:56.364 | INFO | Uploaded 36GiB to GPU in 2.17809s (16.5282GiB/s) | 20260505 10:55:56.364 | INFO | Downloaded 34GiB to GPU in 2.12773s (15.9795GiB/s) | 20260505 10:56:01.499 | INFO | Analyzing chunk frames = [2656768,2657280) | 20260505 10:56:01.499 | INFO | MemInfo: { | 20260505 10:56:01.499 | INFO | "gpu" : " 17 GB", | 20260505 10:56:01.499 | INFO | "huge" : "", | 20260505 10:56:01.499 | INFO | "huge_shared" : " 27 GB", | 20260505 10:56:01.499 | INFO | "malloc" : " 14 GB", | 20260505 10:56:01.499 | INFO | "pinned" : " 29 GB" | 20260505 10:56:01.499 | INFO | } | 20260505 10:56:01.499 | INFO | Perf report for the last 5.13569 seconds: | 20260505 10:56:01.499 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:56:01.499 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:01.499 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:01.499 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:01.499 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 10:56:01.499 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:01.499 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:01.499 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 10:56:01.499 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:01.499 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:01.499 | INFO | Uploaded 36GiB to GPU in 2.21797s (16.231GiB/s) | 20260505 10:56:01.499 | INFO | Downloaded 34GiB to GPU in 2.10262s (16.1703GiB/s) | 20260505 10:56:06.639 | INFO | Analyzing chunk frames = [2657280,2657792) | 20260505 10:56:06.639 | INFO | MemInfo: { | 20260505 10:56:06.639 | INFO | "gpu" : " 17 GB", | 20260505 10:56:06.639 | INFO | "huge" : "", | 20260505 10:56:06.639 | INFO | "huge_shared" : " 27 GB", | 20260505 10:56:06.639 | INFO | "malloc" : " 14 GB", | 20260505 10:56:06.639 | INFO | "pinned" : " 29 GB" | 20260505 10:56:06.639 | INFO | } | 20260505 10:56:06.639 | INFO | Perf report for the last 5.13998 seconds: | 20260505 10:56:06.639 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:56:06.639 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:06.639 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:06.639 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:06.639 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 10:56:06.639 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:06.639 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:06.639 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 10:56:06.639 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:06.639 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:06.640 | INFO | Uploaded 36GiB to GPU in 2.15969s (16.669GiB/s) | 20260505 10:56:06.640 | INFO | Downloaded 34GiB to GPU in 2.12649s (15.9888GiB/s) | 20260505 10:56:11.784 | INFO | Analyzing chunk frames = [2657792,2658304) | 20260505 10:56:11.784 | INFO | MemInfo: { | 20260505 10:56:11.784 | INFO | "gpu" : " 17 GB", | 20260505 10:56:11.784 | INFO | "huge" : "", | 20260505 10:56:11.784 | INFO | "huge_shared" : " 27 GB", | 20260505 10:56:11.785 | INFO | "malloc" : " 14 GB", | 20260505 10:56:11.785 | INFO | "pinned" : " 29 GB" | 20260505 10:56:11.785 | INFO | } | 20260505 10:56:11.785 | INFO | Perf report for the last 5.14546 seconds: | 20260505 10:56:11.785 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:56:11.785 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:11.785 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:11.785 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:11.785 | INFO | ANALYSIS : 64.1%, 3.0, 100.2ms, 0 | 20260505 10:56:11.785 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:11.785 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:11.785 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:56:11.785 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:11.785 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:11.785 | INFO | Uploaded 36GiB to GPU in 2.24217s (16.0559GiB/s) | 20260505 10:56:11.785 | INFO | Downloaded 34GiB to GPU in 2.09709s (16.2129GiB/s) | 20260505 10:56:11.921 | INFO | Starting a round of Baz IO | 20260505 10:56:15.175 | INFO | Populating RTMetrics report | 20260505 10:56:15.175 | INFO | RTMetrics report populated | 20260505 10:56:15.175 | INFO | Opening tmp RTMetrics json file | 20260505 10:56:15.231 | INFO | Serializing RTMetrics to json | 20260505 10:56:15.258 | INFO | Serialization done, swapping tmp file now... | 20260505 10:56:15.271 | INFO | RTMetrics done | 20260505 10:56:15.294 | INFO | Finished a round of Baz IO: Wrote 2.397 GiB in 3.372 seconds | 20260505 10:56:15.294 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.56% / 0.01% / 75.43% | 20260505 10:56:15.294 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.30938% / 1.58946e-05% / 7.54992e-05% | 20260505 10:56:15.294 | INFO | Baz IO overall: | 20260505 10:56:15.294 | INFO | BytesWritten : 1565155629419 | 20260505 10:56:15.294 | INFO | NumEvents : 745228892347 | 20260505 10:56:16.925 | INFO | Analyzing chunk frames = [2658304,2658816) | 20260505 10:56:16.925 | INFO | MemInfo: { | 20260505 10:56:16.925 | INFO | "gpu" : " 17 GB", | 20260505 10:56:16.925 | INFO | "huge" : "", | 20260505 10:56:16.925 | INFO | "huge_shared" : " 27 GB", | 20260505 10:56:16.925 | INFO | "malloc" : " 14 GB", | 20260505 10:56:16.925 | INFO | "pinned" : " 29 GB" | 20260505 10:56:16.925 | INFO | } | 20260505 10:56:16.925 | INFO | Perf report for the last 5.14008 seconds: | 20260505 10:56:16.925 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:56:16.925 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:16.925 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:16.925 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:16.925 | INFO | ANALYSIS : 64.8%, 2.9, 99.6ms, 0 | 20260505 10:56:16.925 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:16.925 | INFO | RT_METRICS : 47.5%, 1.0, 24.9ms, 0 | 20260505 10:56:16.925 | INFO | PRE_HQ : 61.5%, 1.9, 59.2ms, 0 | 20260505 10:56:16.925 | INFO | BAZWRITER : 65.0%, 4.1, 137.6ms, 0 | 20260505 10:56:16.925 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 10:56:16.925 | INFO | Uploaded 36GiB to GPU in 2.19333s (16.4134GiB/s) | 20260505 10:56:16.925 | INFO | Downloaded 38GiB to GPU in 2.29494s (16.5581GiB/s) | 20260505 10:56:22.066 | INFO | Analyzing chunk frames = [2658816,2659328) | 20260505 10:56:22.066 | INFO | MemInfo: { | 20260505 10:56:22.066 | INFO | "gpu" : " 17 GB", | 20260505 10:56:22.066 | INFO | "huge" : "", | 20260505 10:56:22.066 | INFO | "huge_shared" : " 27 GB", | 20260505 10:56:22.066 | INFO | "malloc" : " 14 GB", | 20260505 10:56:22.066 | INFO | "pinned" : " 29 GB" | 20260505 10:56:22.066 | INFO | } | 20260505 10:56:22.066 | INFO | Perf report for the last 5.14136 seconds: | 20260505 10:56:22.066 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:56:22.066 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:22.066 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:22.066 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:22.066 | INFO | ANALYSIS : 64.2%, 3.0, 100.4ms, 0 | 20260505 10:56:22.066 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:22.066 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:22.066 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 10:56:22.066 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:22.066 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:22.066 | INFO | Uploaded 36GiB to GPU in 2.23321s (16.1203GiB/s) | 20260505 10:56:22.066 | INFO | Downloaded 34GiB to GPU in 2.07782s (16.3633GiB/s) | 20260505 10:56:27.201 | INFO | Analyzing chunk frames = [2659328,2659840) | 20260505 10:56:27.201 | INFO | MemInfo: { | 20260505 10:56:27.201 | INFO | "gpu" : " 17 GB", | 20260505 10:56:27.202 | INFO | "huge" : "", | 20260505 10:56:27.202 | INFO | "huge_shared" : " 27 GB", | 20260505 10:56:27.202 | INFO | "malloc" : " 14 GB", | 20260505 10:56:27.202 | INFO | "pinned" : " 29 GB" | 20260505 10:56:27.202 | INFO | } | 20260505 10:56:27.202 | INFO | Perf report for the last 5.13541 seconds: | 20260505 10:56:27.202 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:56:27.202 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:27.202 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:27.202 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:27.202 | INFO | ANALYSIS : 62.9%, 3.0, 97.8ms, 0 | 20260505 10:56:27.202 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:27.202 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:27.202 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:56:27.202 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:27.202 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:27.202 | INFO | Uploaded 36GiB to GPU in 2.16776s (16.607GiB/s) | 20260505 10:56:27.202 | INFO | Downloaded 34GiB to GPU in 2.06191s (16.4896GiB/s) | 20260505 10:56:32.338 | INFO | Analyzing chunk frames = [2659840,2660352) | 20260505 10:56:32.338 | INFO | MemInfo: { | 20260505 10:56:32.338 | INFO | "gpu" : " 17 GB", | 20260505 10:56:32.338 | INFO | "huge" : "", | 20260505 10:56:32.338 | INFO | "huge_shared" : " 27 GB", | 20260505 10:56:32.338 | INFO | "malloc" : " 14 GB", | 20260505 10:56:32.338 | INFO | "pinned" : " 29 GB" | 20260505 10:56:32.338 | INFO | } | 20260505 10:56:32.338 | INFO | Perf report for the last 5.13617 seconds: | 20260505 10:56:32.338 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:56:32.338 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:32.338 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:32.338 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:32.338 | INFO | ANALYSIS : 63.0%, 3.0, 97.9ms, 0 | 20260505 10:56:32.338 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:32.338 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:32.338 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:56:32.338 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:32.338 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:32.338 | INFO | Uploaded 36GiB to GPU in 2.15602s (16.6974GiB/s) | 20260505 10:56:32.338 | INFO | Downloaded 34GiB to GPU in 2.07836s (16.3591GiB/s) | 20260505 10:56:37.480 | INFO | Analyzing chunk frames = [2660352,2660864) | 20260505 10:56:37.480 | INFO | MemInfo: { | 20260505 10:56:37.480 | INFO | "gpu" : " 17 GB", | 20260505 10:56:37.480 | INFO | "huge" : "", | 20260505 10:56:37.480 | INFO | "huge_shared" : " 27 GB", | 20260505 10:56:37.480 | INFO | "malloc" : " 14 GB", | 20260505 10:56:37.480 | INFO | "pinned" : " 29 GB" | 20260505 10:56:37.480 | INFO | } | 20260505 10:56:37.480 | INFO | Perf report for the last 5.1422 seconds: | 20260505 10:56:37.480 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:56:37.480 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:37.480 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:37.480 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:37.480 | INFO | ANALYSIS : 63.2%, 3.0, 98.8ms, 0 | 20260505 10:56:37.480 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:37.480 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:37.480 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 10:56:37.480 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:37.480 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:37.480 | INFO | Uploaded 36GiB to GPU in 2.15307s (16.7203GiB/s) | 20260505 10:56:37.480 | INFO | Downloaded 34GiB to GPU in 2.12999s (15.9625GiB/s) | 20260505 10:56:42.630 | INFO | Analyzing chunk frames = [2660864,2661376) | 20260505 10:56:42.630 | INFO | MemInfo: { | 20260505 10:56:42.630 | INFO | "gpu" : " 17 GB", | 20260505 10:56:42.630 | INFO | "huge" : "", | 20260505 10:56:42.630 | INFO | "huge_shared" : " 27 GB", | 20260505 10:56:42.630 | INFO | "malloc" : " 14 GB", | 20260505 10:56:42.630 | INFO | "pinned" : " 29 GB" | 20260505 10:56:42.630 | INFO | } | 20260505 10:56:42.630 | INFO | Perf report for the last 5.15032 seconds: | 20260505 10:56:42.630 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:56:42.630 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:42.630 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:42.630 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:42.630 | INFO | ANALYSIS : 63.3%, 3.0, 99.1ms, 0 | 20260505 10:56:42.630 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:42.631 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:42.631 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 10:56:42.631 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:42.631 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:42.631 | INFO | Uploaded 36GiB to GPU in 2.1546s (16.7084GiB/s) | 20260505 10:56:42.631 | INFO | Downloaded 34GiB to GPU in 2.11734s (16.0579GiB/s) | 20260505 10:56:47.758 | INFO | Analyzing chunk frames = [2661376,2661888) | 20260505 10:56:47.758 | INFO | MemInfo: { | 20260505 10:56:47.758 | INFO | "gpu" : " 17 GB", | 20260505 10:56:47.758 | INFO | "huge" : "", | 20260505 10:56:47.758 | INFO | "huge_shared" : " 27 GB", | 20260505 10:56:47.758 | INFO | "malloc" : " 14 GB", | 20260505 10:56:47.758 | INFO | "pinned" : " 29 GB" | 20260505 10:56:47.758 | INFO | } | 20260505 10:56:47.758 | INFO | Perf report for the last 5.12793 seconds: | 20260505 10:56:47.758 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:56:47.758 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:47.758 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:47.758 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:47.758 | INFO | ANALYSIS : 63.6%, 3.0, 99.0ms, 0 | 20260505 10:56:47.758 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:47.758 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:47.758 | INFO | PRE_HQ : 30.8%, 1.0, 15.9ms, 0 | 20260505 10:56:47.758 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:47.758 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:47.759 | INFO | Uploaded 36GiB to GPU in 2.17296s (16.5673GiB/s) | 20260505 10:56:47.759 | INFO | Downloaded 34GiB to GPU in 2.13935s (15.8927GiB/s) | 20260505 10:56:52.891 | INFO | Analyzing chunk frames = [2661888,2662400) | 20260505 10:56:52.891 | INFO | MemInfo: { | 20260505 10:56:52.891 | INFO | "gpu" : " 17 GB", | 20260505 10:56:52.891 | INFO | "huge" : "", | 20260505 10:56:52.891 | INFO | "huge_shared" : " 27 GB", | 20260505 10:56:52.891 | INFO | "malloc" : " 14 GB", | 20260505 10:56:52.891 | INFO | "pinned" : " 29 GB" | 20260505 10:56:52.891 | INFO | } | 20260505 10:56:52.892 | INFO | Perf report for the last 5.1333 seconds: | 20260505 10:56:52.892 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:56:52.892 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:52.892 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:52.892 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:52.892 | INFO | ANALYSIS : 64.6%, 3.0, 100.7ms, 0 | 20260505 10:56:52.892 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:52.892 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:52.892 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 10:56:52.892 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:52.892 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:52.892 | INFO | Uploaded 36GiB to GPU in 2.24752s (16.0176GiB/s) | 20260505 10:56:52.892 | INFO | Downloaded 34GiB to GPU in 2.09204s (16.2521GiB/s) | 20260505 10:56:52.892 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 10:56:52.892 | INFO | Total mem usage: 15.3 GiB | 20260505 10:56:52.892 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 10:56:52.892 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -6.81%) | 20260505 10:56:52.892 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 10:56:52.892 | INFO | High Water Breakdown: | 20260505 10:56:52.892 | INFO | 61% : 8.76 GiB ( 92.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 10:56:52.892 | INFO | 38.8% : 5.57 GiB ( 83.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 10:56:52.892 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 10:56:52.892 | INFO | ------------------------------------------ | 20260505 10:56:52.892 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:56:52.892 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 10:56:52.892 | INFO | Total mem usage: 26.5 GiB | 20260505 10:56:52.892 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 10:56:52.892 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 10:56:52.892 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 10:56:52.892 | INFO | High Water Breakdown: | 20260505 10:56:52.892 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:56:52.892 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:56:52.892 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:56:52.892 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:56:52.892 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:56:52.892 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:56:52.892 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 10:56:52.892 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 10:56:52.892 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:56:52.892 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 10:56:52.892 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 10:56:52.892 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 10:56:52.892 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 10:56:52.892 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 10:56:52.892 | INFO | ------------------------------------------ | 20260505 10:56:52.892 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:56:52.892 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 10:56:52.892 | INFO | Total mem usage: 25.6 GiB | 20260505 10:56:52.892 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 10:56:52.892 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 10:56:52.892 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 10:56:52.892 | INFO | High Water Breakdown: | 20260505 10:56:52.892 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 10:56:52.892 | INFO | ------------------------------------------ | 20260505 10:56:52.893 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:56:52.893 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 10:56:52.893 | INFO | Total mem usage: 15.7 GiB | 20260505 10:56:52.893 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 10:56:52.893 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 10:56:52.893 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 10:56:52.893 | INFO | High Water Breakdown: | 20260505 10:56:52.893 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 10:56:52.893 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 10:56:52.893 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 10:56:52.893 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 10:56:52.893 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 10:56:52.893 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 10:56:52.893 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 10:56:52.893 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 10:56:52.893 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 10:56:52.893 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:56:52.893 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 10:56:52.893 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:56:52.893 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 10:56:52.893 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 10:56:52.893 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 10:56:52.893 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 10:56:52.893 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:56:52.893 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 10:56:52.893 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:56:52.893 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:56:52.893 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:56:52.893 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 10:56:52.893 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 10:56:52.893 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:56:52.893 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 10:56:52.893 | INFO | ------------------------------------------ | 20260505 10:56:52.893 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:56:52.893 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 10:56:52.893 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:56:52.893 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:56:52.893 | INFO | PerfReport: Full Execution -- 3.22e+01 / 7.76e+01, 1.39e+01 / 1.42e+00 | 20260505 10:56:52.893 | INFO | PerfReport: 0.29%, AggregateStats -- 9.30e-02 / 1.01e-01, 1.06e-02 / 3.37e-03 | 20260505 10:56:52.893 | INFO | PerfReport: 11.72%, Baseline -- 3.78e+00 / 4.51e+00, 6.05e-01 / 1.75e-01 | 20260505 10:56:52.893 | INFO | PerfReport: 3.31%, Histogram -- 1.07e+00 / 1.52e+00, 2.08e-01 / 6.91e-02 | 20260505 10:56:52.893 | INFO | PerfReport: 15.84%, DME -- 4.08e+01 / 4.55e+01, 4.45e+00 / 1.90e+00 | 20260505 10:56:52.893 | INFO | PerfReport: 56.40%, FrameLabeling -- 1.82e+01 / 2.11e+01, 1.68e+00 / 1.90e+00 | 20260505 10:56:52.893 | INFO | PerfReport: 7.40%, PulseAccumulating -- 2.39e+00 / 2.88e+00, 4.26e-01 / 2.05e-01 | 20260505 10:56:52.893 | INFO | PerfReport: 3.72%, Metrics -- 1.20e+00 / 1.52e+00, 3.41e-01 / 1.98e-01 | 20260505 10:56:52.893 | INFO | PerfReport: 1.31%, PulseSerializing -- 4.24e-01 / 4.73e-01, 7.84e-02 / 3.64e-02 | 20260505 10:56:52.893 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 10:56:52.893 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:56:52.893 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:56:52.893 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260505 10:56:52.893 | INFO | PerfReport: 22.58%, Upload -- 2.29e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260505 10:56:52.893 | INFO | PerfReport: 21.90%, Download -- 2.22e+01 / 0.00e+00, 8.15e+00 / 0.00e+00 | 20260505 10:56:52.893 | INFO | PerfReport: 31.81%, Compute -- 3.22e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 10:56:52.893 | INFO | PerfReport: 2.08%, LockUpload -- 2.11e+00 / 0.00e+00, 7.61e+00 / 0.00e+00 | 20260505 10:56:52.893 | INFO | PerfReport: 1.22%, LockDownload -- 1.23e+00 / 0.00e+00, 2.75e+00 / 0.00e+00 | 20260505 10:56:52.893 | INFO | PerfReport: 20.41%, LockCompute -- 2.07e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260505 10:56:52.893 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 2.02e-02 / 0.00e+00 | 20260505 10:56:53.031 | INFO | Starting a round of Baz IO | 20260505 10:56:56.287 | INFO | Populating RTMetrics report | 20260505 10:56:56.288 | INFO | RTMetrics report populated | 20260505 10:56:56.288 | INFO | Opening tmp RTMetrics json file | 20260505 10:56:56.290 | INFO | Serializing RTMetrics to json | 20260505 10:56:56.292 | INFO | Serialization done, swapping tmp file now... | 20260505 10:56:56.293 | INFO | RTMetrics done | 20260505 10:56:56.423 | INFO | Finished a round of Baz IO: Wrote 2.395 GiB in 3.391 seconds | 20260505 10:56:56.423 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.55% / 0.01% / 75.44% | 20260505 10:56:56.423 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.30283% / 2.78155e-05% / 8.34465e-05% | 20260505 10:56:56.423 | INFO | Baz IO overall: | 20260505 10:56:56.423 | INFO | BytesWritten : 1567726946306 | 20260505 10:56:56.423 | INFO | NumEvents : 746453202663 | 20260505 10:56:58.037 | INFO | Analyzing chunk frames = [2662400,2662912) | 20260505 10:56:58.037 | INFO | MemInfo: { | 20260505 10:56:58.037 | INFO | "gpu" : " 17 GB", | 20260505 10:56:58.037 | INFO | "huge" : "", | 20260505 10:56:58.037 | INFO | "huge_shared" : " 27 GB", | 20260505 10:56:58.037 | INFO | "malloc" : " 14 GB", | 20260505 10:56:58.037 | INFO | "pinned" : " 29 GB" | 20260505 10:56:58.037 | INFO | } | 20260505 10:56:58.037 | INFO | Perf report for the last 5.14566 seconds: | 20260505 10:56:58.037 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:56:58.037 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:58.037 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:58.037 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:58.037 | INFO | ANALYSIS : 64.9%, 2.9, 100.0ms, 0 | 20260505 10:56:58.037 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:56:58.037 | INFO | RT_METRICS : 45.5%, 1.0, 23.9ms, 0 | 20260505 10:56:58.038 | INFO | PRE_HQ : 61.6%, 1.9, 59.4ms, 0 | 20260505 10:56:58.038 | INFO | BAZWRITER : 65.2%, 4.1, 137.3ms, 0 | 20260505 10:56:58.038 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 10:56:58.038 | INFO | Uploaded 36GiB to GPU in 2.18697s (16.4611GiB/s) | 20260505 10:56:58.038 | INFO | Downloaded 38GiB to GPU in 2.30289s (16.501GiB/s) | 20260505 10:57:03.181 | INFO | Analyzing chunk frames = [2662912,2663424) | 20260505 10:57:03.181 | INFO | MemInfo: { | 20260505 10:57:03.181 | INFO | "gpu" : " 17 GB", | 20260505 10:57:03.181 | INFO | "huge" : "", | 20260505 10:57:03.181 | INFO | "huge_shared" : " 27 GB", | 20260505 10:57:03.181 | INFO | "malloc" : " 14 GB", | 20260505 10:57:03.181 | INFO | "pinned" : " 29 GB" | 20260505 10:57:03.181 | INFO | } | 20260505 10:57:03.181 | INFO | Perf report for the last 5.14412 seconds: | 20260505 10:57:03.181 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:57:03.181 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:03.181 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:03.181 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:03.181 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 10:57:03.181 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:03.181 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:03.181 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:57:03.182 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:03.182 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:03.182 | INFO | Uploaded 36GiB to GPU in 2.23776s (16.0875GiB/s) | 20260505 10:57:03.182 | INFO | Downloaded 34GiB to GPU in 2.07158s (16.4126GiB/s) | 20260505 10:57:08.316 | INFO | Analyzing chunk frames = [2663424,2663936) | 20260505 10:57:08.317 | INFO | MemInfo: { | 20260505 10:57:08.317 | INFO | "gpu" : " 17 GB", | 20260505 10:57:08.317 | INFO | "huge" : "", | 20260505 10:57:08.317 | INFO | "huge_shared" : " 27 GB", | 20260505 10:57:08.317 | INFO | "malloc" : " 14 GB", | 20260505 10:57:08.317 | INFO | "pinned" : " 29 GB" | 20260505 10:57:08.317 | INFO | } | 20260505 10:57:08.317 | INFO | Perf report for the last 5.1354 seconds: | 20260505 10:57:08.317 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:57:08.317 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:08.317 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:08.317 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:08.317 | INFO | ANALYSIS : 63.8%, 3.0, 99.2ms, 0 | 20260505 10:57:08.317 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:08.317 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:08.317 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 10:57:08.317 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:08.317 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:08.317 | INFO | Uploaded 36GiB to GPU in 2.22785s (16.1591GiB/s) | 20260505 10:57:08.317 | INFO | Downloaded 34GiB to GPU in 2.12277s (16.0168GiB/s) | 20260505 10:57:13.447 | INFO | Analyzing chunk frames = [2663936,2664448) | 20260505 10:57:13.447 | INFO | MemInfo: { | 20260505 10:57:13.447 | INFO | "gpu" : " 17 GB", | 20260505 10:57:13.447 | INFO | "huge" : "", | 20260505 10:57:13.447 | INFO | "huge_shared" : " 27 GB", | 20260505 10:57:13.447 | INFO | "malloc" : " 14 GB", | 20260505 10:57:13.447 | INFO | "pinned" : " 29 GB" | 20260505 10:57:13.447 | INFO | } | 20260505 10:57:13.447 | INFO | Perf report for the last 5.13036 seconds: | 20260505 10:57:13.447 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:57:13.447 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:13.447 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:13.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:13.447 | INFO | ANALYSIS : 63.1%, 3.0, 97.9ms, 0 | 20260505 10:57:13.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:13.447 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:13.447 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 10:57:13.447 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:13.447 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:13.448 | INFO | Uploaded 36GiB to GPU in 2.15357s (16.7164GiB/s) | 20260505 10:57:13.448 | INFO | Downloaded 34GiB to GPU in 2.07451s (16.3894GiB/s) | 20260505 10:57:18.590 | INFO | Analyzing chunk frames = [2664448,2664960) | 20260505 10:57:18.590 | INFO | MemInfo: { | 20260505 10:57:18.590 | INFO | "gpu" : " 17 GB", | 20260505 10:57:18.590 | INFO | "huge" : "", | 20260505 10:57:18.590 | INFO | "huge_shared" : " 27 GB", | 20260505 10:57:18.590 | INFO | "malloc" : " 14 GB", | 20260505 10:57:18.590 | INFO | "pinned" : " 29 GB" | 20260505 10:57:18.590 | INFO | } | 20260505 10:57:18.590 | INFO | Perf report for the last 5.1433 seconds: | 20260505 10:57:18.590 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:57:18.590 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:18.590 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:18.591 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:18.591 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 10:57:18.591 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:18.591 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:18.591 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 10:57:18.591 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:18.591 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:18.591 | INFO | Uploaded 36GiB to GPU in 2.24062s (16.067GiB/s) | 20260505 10:57:18.591 | INFO | Downloaded 34GiB to GPU in 2.10295s (16.1678GiB/s) | 20260505 10:57:23.739 | INFO | Analyzing chunk frames = [2664960,2665472) | 20260505 10:57:23.739 | INFO | MemInfo: { | 20260505 10:57:23.739 | INFO | "gpu" : " 17 GB", | 20260505 10:57:23.739 | INFO | "huge" : "", | 20260505 10:57:23.739 | INFO | "huge_shared" : " 27 GB", | 20260505 10:57:23.739 | INFO | "malloc" : " 14 GB", | 20260505 10:57:23.739 | INFO | "pinned" : " 29 GB" | 20260505 10:57:23.739 | INFO | } | 20260505 10:57:23.739 | INFO | Perf report for the last 5.14853 seconds: | 20260505 10:57:23.739 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:57:23.739 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:23.739 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:23.739 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:23.739 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 10:57:23.739 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:23.739 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:23.739 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 10:57:23.739 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:23.739 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:23.739 | INFO | Uploaded 36GiB to GPU in 2.18432s (16.4811GiB/s) | 20260505 10:57:23.739 | INFO | Downloaded 34GiB to GPU in 2.11678s (16.0621GiB/s) | 20260505 10:57:28.880 | INFO | Analyzing chunk frames = [2665472,2665984) | 20260505 10:57:28.880 | INFO | MemInfo: { | 20260505 10:57:28.880 | INFO | "gpu" : " 17 GB", | 20260505 10:57:28.880 | INFO | "huge" : "", | 20260505 10:57:28.880 | INFO | "huge_shared" : " 27 GB", | 20260505 10:57:28.880 | INFO | "malloc" : " 14 GB", | 20260505 10:57:28.880 | INFO | "pinned" : " 29 GB" | 20260505 10:57:28.880 | INFO | } | 20260505 10:57:28.880 | INFO | Perf report for the last 5.14072 seconds: | 20260505 10:57:28.880 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:57:28.880 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:28.880 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:28.880 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:28.880 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 10:57:28.880 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:28.880 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:28.880 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 10:57:28.880 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:28.880 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:28.880 | INFO | Uploaded 36GiB to GPU in 2.1742s (16.5578GiB/s) | 20260505 10:57:28.880 | INFO | Downloaded 34GiB to GPU in 2.14604s (15.8431GiB/s) | 20260505 10:57:34.010 | INFO | Analyzing chunk frames = [2665984,2666496) | 20260505 10:57:34.010 | INFO | MemInfo: { | 20260505 10:57:34.010 | INFO | "gpu" : " 17 GB", | 20260505 10:57:34.010 | INFO | "huge" : "", | 20260505 10:57:34.010 | INFO | "huge_shared" : " 27 GB", | 20260505 10:57:34.010 | INFO | "malloc" : " 14 GB", | 20260505 10:57:34.010 | INFO | "pinned" : " 29 GB" | 20260505 10:57:34.010 | INFO | } | 20260505 10:57:34.010 | INFO | Perf report for the last 5.13058 seconds: | 20260505 10:57:34.010 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:57:34.010 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:34.010 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:34.010 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:34.010 | INFO | ANALYSIS : 64.0%, 3.0, 99.8ms, 0 | 20260505 10:57:34.010 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:34.010 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:34.010 | INFO | PRE_HQ : 29.8%, 1.0, 15.4ms, 0 | 20260505 10:57:34.010 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:34.010 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:34.011 | INFO | Uploaded 36GiB to GPU in 2.21945s (16.2202GiB/s) | 20260505 10:57:34.011 | INFO | Downloaded 34GiB to GPU in 2.08377s (16.3166GiB/s) | 20260505 10:57:34.141 | INFO | Starting a round of Baz IO | 20260505 10:57:37.418 | INFO | Populating RTMetrics report | 20260505 10:57:37.418 | INFO | RTMetrics report populated | 20260505 10:57:37.418 | INFO | Opening tmp RTMetrics json file | 20260505 10:57:37.421 | INFO | Serializing RTMetrics to json | 20260505 10:57:37.422 | INFO | Serialization done, swapping tmp file now... | 20260505 10:57:37.422 | INFO | RTMetrics done | 20260505 10:57:37.554 | INFO | Finished a round of Baz IO: Wrote 2.4 GiB in 3.413 seconds | 20260505 10:57:37.554 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.54% / 0.01% / 75.45% | 20260505 10:57:37.554 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.30852% / 3.97364e-06% / 9.13938e-05% | 20260505 10:57:37.554 | INFO | Baz IO overall: | 20260505 10:57:37.554 | INFO | BytesWritten : 1570304284442 | 20260505 10:57:37.554 | INFO | NumEvents : 747681908688 | 20260505 10:57:39.144 | INFO | Analyzing chunk frames = [2666496,2667008) | 20260505 10:57:39.144 | INFO | MemInfo: { | 20260505 10:57:39.144 | INFO | "gpu" : " 17 GB", | 20260505 10:57:39.144 | INFO | "huge" : "", | 20260505 10:57:39.145 | INFO | "huge_shared" : " 27 GB", | 20260505 10:57:39.145 | INFO | "malloc" : " 14 GB", | 20260505 10:57:39.145 | INFO | "pinned" : " 29 GB" | 20260505 10:57:39.145 | INFO | } | 20260505 10:57:39.145 | INFO | Perf report for the last 5.13435 seconds: | 20260505 10:57:39.145 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:57:39.145 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:39.145 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:39.145 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:39.145 | INFO | ANALYSIS : 65.2%, 2.9, 100.5ms, 0 | 20260505 10:57:39.145 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:39.145 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260505 10:57:39.145 | INFO | PRE_HQ : 62.1%, 1.8, 59.3ms, 0 | 20260505 10:57:39.145 | INFO | BAZWRITER : 65.8%, 4.1, 138.1ms, 0 | 20260505 10:57:39.145 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 10:57:39.145 | INFO | Uploaded 36GiB to GPU in 2.23342s (16.1188GiB/s) | 20260505 10:57:39.145 | INFO | Downloaded 38GiB to GPU in 2.32096s (16.3726GiB/s) | 20260505 10:57:44.295 | INFO | Analyzing chunk frames = [2667008,2667520) | 20260505 10:57:44.295 | INFO | MemInfo: { | 20260505 10:57:44.295 | INFO | "gpu" : " 17 GB", | 20260505 10:57:44.295 | INFO | "huge" : "", | 20260505 10:57:44.295 | INFO | "huge_shared" : " 27 GB", | 20260505 10:57:44.295 | INFO | "malloc" : " 14 GB", | 20260505 10:57:44.295 | INFO | "pinned" : " 29 GB" | 20260505 10:57:44.295 | INFO | } | 20260505 10:57:44.295 | INFO | Perf report for the last 5.15025 seconds: | 20260505 10:57:44.295 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:57:44.295 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:44.295 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:44.295 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:44.295 | INFO | ANALYSIS : 63.9%, 3.0, 100.0ms, 0 | 20260505 10:57:44.295 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:44.295 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:44.295 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 10:57:44.295 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:44.295 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:44.295 | INFO | Uploaded 36GiB to GPU in 2.17524s (16.5499GiB/s) | 20260505 10:57:44.295 | INFO | Downloaded 34GiB to GPU in 2.09109s (16.2594GiB/s) | 20260505 10:57:49.430 | INFO | Analyzing chunk frames = [2667520,2668032) | 20260505 10:57:49.430 | INFO | MemInfo: { | 20260505 10:57:49.430 | INFO | "gpu" : " 17 GB", | 20260505 10:57:49.430 | INFO | "huge" : "", | 20260505 10:57:49.430 | INFO | "huge_shared" : " 27 GB", | 20260505 10:57:49.430 | INFO | "malloc" : " 14 GB", | 20260505 10:57:49.430 | INFO | "pinned" : " 29 GB" | 20260505 10:57:49.430 | INFO | } | 20260505 10:57:49.430 | INFO | Perf report for the last 5.13506 seconds: | 20260505 10:57:49.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:57:49.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:49.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:49.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:49.430 | INFO | ANALYSIS : 63.4%, 3.0, 98.6ms, 0 | 20260505 10:57:49.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:49.430 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:49.430 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 10:57:49.430 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:49.430 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:49.430 | INFO | Uploaded 36GiB to GPU in 2.20728s (16.3096GiB/s) | 20260505 10:57:49.430 | INFO | Downloaded 34GiB to GPU in 2.11993s (16.0382GiB/s) | 20260505 10:57:54.574 | INFO | Analyzing chunk frames = [2668032,2668544) | 20260505 10:57:54.574 | INFO | MemInfo: { | 20260505 10:57:54.574 | INFO | "gpu" : " 17 GB", | 20260505 10:57:54.574 | INFO | "huge" : "", | 20260505 10:57:54.574 | INFO | "huge_shared" : " 27 GB", | 20260505 10:57:54.574 | INFO | "malloc" : " 14 GB", | 20260505 10:57:54.574 | INFO | "pinned" : " 29 GB" | 20260505 10:57:54.574 | INFO | } | 20260505 10:57:54.574 | INFO | Perf report for the last 5.14446 seconds: | 20260505 10:57:54.574 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:57:54.574 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:54.574 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:54.575 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:54.575 | INFO | ANALYSIS : 63.7%, 3.0, 99.0ms, 0 | 20260505 10:57:54.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:54.575 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:54.575 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 10:57:54.575 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:54.575 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:54.575 | INFO | Uploaded 36GiB to GPU in 2.22693s (16.1657GiB/s) | 20260505 10:57:54.575 | INFO | Downloaded 34GiB to GPU in 2.12102s (16.03GiB/s) | 20260505 10:57:59.709 | INFO | Analyzing chunk frames = [2668544,2669056) | 20260505 10:57:59.709 | INFO | MemInfo: { | 20260505 10:57:59.709 | INFO | "gpu" : " 17 GB", | 20260505 10:57:59.709 | INFO | "huge" : "", | 20260505 10:57:59.709 | INFO | "huge_shared" : " 27 GB", | 20260505 10:57:59.709 | INFO | "malloc" : " 14 GB", | 20260505 10:57:59.709 | INFO | "pinned" : " 29 GB" | 20260505 10:57:59.709 | INFO | } | 20260505 10:57:59.709 | INFO | Perf report for the last 5.13498 seconds: | 20260505 10:57:59.710 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:57:59.710 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:59.710 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:59.710 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:59.710 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 10:57:59.710 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:59.710 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:59.710 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 10:57:59.710 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:59.710 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:57:59.710 | INFO | Uploaded 36GiB to GPU in 2.16041s (16.6635GiB/s) | 20260505 10:57:59.710 | INFO | Downloaded 34GiB to GPU in 2.12573s (15.9945GiB/s) | 20260505 10:58:04.846 | INFO | Analyzing chunk frames = [2669056,2669568) | 20260505 10:58:04.846 | INFO | MemInfo: { | 20260505 10:58:04.846 | INFO | "gpu" : " 17 GB", | 20260505 10:58:04.846 | INFO | "huge" : "", | 20260505 10:58:04.846 | INFO | "huge_shared" : " 27 GB", | 20260505 10:58:04.846 | INFO | "malloc" : " 14 GB", | 20260505 10:58:04.846 | INFO | "pinned" : " 29 GB" | 20260505 10:58:04.846 | INFO | } | 20260505 10:58:04.846 | INFO | Perf report for the last 5.13615 seconds: | 20260505 10:58:04.846 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:58:04.846 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:04.846 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:04.846 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:04.846 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 10:58:04.846 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:04.846 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:04.846 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 10:58:04.846 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:04.846 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:04.846 | INFO | Uploaded 36GiB to GPU in 2.22195s (16.202GiB/s) | 20260505 10:58:04.846 | INFO | Downloaded 34GiB to GPU in 2.1022s (16.1735GiB/s) | 20260505 10:58:09.995 | INFO | Analyzing chunk frames = [2669568,2670080) | 20260505 10:58:09.995 | INFO | MemInfo: { | 20260505 10:58:09.995 | INFO | "gpu" : " 17 GB", | 20260505 10:58:09.995 | INFO | "huge" : "", | 20260505 10:58:09.995 | INFO | "huge_shared" : " 27 GB", | 20260505 10:58:09.995 | INFO | "malloc" : " 14 GB", | 20260505 10:58:09.995 | INFO | "pinned" : " 29 GB" | 20260505 10:58:09.995 | INFO | } | 20260505 10:58:09.995 | INFO | Perf report for the last 5.14934 seconds: | 20260505 10:58:09.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:58:09.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:09.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:09.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:09.995 | INFO | ANALYSIS : 63.5%, 3.0, 99.4ms, 0 | 20260505 10:58:09.995 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:09.995 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:09.995 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 10:58:09.995 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:09.995 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:09.995 | INFO | Uploaded 36GiB to GPU in 2.19058s (16.434GiB/s) | 20260505 10:58:09.995 | INFO | Downloaded 34GiB to GPU in 2.15942s (15.745GiB/s) | 20260505 10:58:15.129 | INFO | Analyzing chunk frames = [2670080,2670592) | 20260505 10:58:15.129 | INFO | MemInfo: { | 20260505 10:58:15.129 | INFO | "gpu" : " 17 GB", | 20260505 10:58:15.129 | INFO | "huge" : "", | 20260505 10:58:15.129 | INFO | "huge_shared" : " 27 GB", | 20260505 10:58:15.129 | INFO | "malloc" : " 14 GB", | 20260505 10:58:15.129 | INFO | "pinned" : " 29 GB" | 20260505 10:58:15.129 | INFO | } | 20260505 10:58:15.129 | INFO | Perf report for the last 5.13424 seconds: | 20260505 10:58:15.129 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:58:15.129 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:15.129 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:15.129 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:15.129 | INFO | ANALYSIS : 63.9%, 3.0, 99.6ms, 0 | 20260505 10:58:15.129 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:15.129 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:15.129 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:58:15.129 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:15.129 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:15.129 | INFO | Uploaded 36GiB to GPU in 2.13977s (16.8242GiB/s) | 20260505 10:58:15.130 | INFO | Downloaded 34GiB to GPU in 2.10996s (16.1141GiB/s) | 20260505 10:58:15.130 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 10:58:15.130 | INFO | Total mem usage: 15.3 GiB | 20260505 10:58:15.130 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 10:58:15.130 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.8%) | 20260505 10:58:15.130 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 10:58:15.130 | INFO | High Water Breakdown: | 20260505 10:58:15.130 | INFO | 61% : 8.76 GiB ( 92.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 10:58:15.130 | INFO | 38.8% : 5.57 GiB ( 83.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 10:58:15.130 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 10:58:15.130 | INFO | ------------------------------------------ | 20260505 10:58:15.130 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:58:15.130 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 10:58:15.130 | INFO | Total mem usage: 26.5 GiB | 20260505 10:58:15.130 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 10:58:15.130 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 10:58:15.130 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 10:58:15.130 | INFO | High Water Breakdown: | 20260505 10:58:15.130 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:58:15.130 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:58:15.130 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:58:15.130 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:58:15.130 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:58:15.130 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:58:15.130 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 10:58:15.130 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 10:58:15.130 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:58:15.130 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 10:58:15.130 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 10:58:15.130 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 10:58:15.130 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 10:58:15.130 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 10:58:15.130 | INFO | ------------------------------------------ | 20260505 10:58:15.130 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:58:15.130 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 10:58:15.130 | INFO | Total mem usage: 25.6 GiB | 20260505 10:58:15.130 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 10:58:15.130 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 10:58:15.130 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 10:58:15.130 | INFO | High Water Breakdown: | 20260505 10:58:15.130 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 10:58:15.130 | INFO | ------------------------------------------ | 20260505 10:58:15.130 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:58:15.130 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 10:58:15.130 | INFO | Total mem usage: 15.7 GiB | 20260505 10:58:15.130 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 10:58:15.130 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 10:58:15.130 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 10:58:15.130 | INFO | High Water Breakdown: | 20260505 10:58:15.130 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 10:58:15.131 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 10:58:15.131 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 10:58:15.131 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 10:58:15.131 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 10:58:15.131 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 10:58:15.131 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 10:58:15.131 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 10:58:15.131 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 10:58:15.131 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:58:15.131 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 10:58:15.131 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:58:15.131 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 10:58:15.131 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 10:58:15.131 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 10:58:15.131 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 10:58:15.131 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:58:15.131 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 10:58:15.131 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:58:15.131 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:58:15.131 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:58:15.131 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 10:58:15.131 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 10:58:15.131 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:58:15.131 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 10:58:15.131 | INFO | ------------------------------------------ | 20260505 10:58:15.131 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:58:15.131 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 10:58:15.131 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:58:15.131 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:58:15.131 | INFO | PerfReport: Full Execution -- 3.22e+01 / 7.78e+01, 1.36e+01 / 2.19e+00 | 20260505 10:58:15.131 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 1.01e-01, 1.03e-02 / 4.87e-03 | 20260505 10:58:15.131 | INFO | PerfReport: 11.68%, Baseline -- 3.77e+00 / 4.45e+00, 6.00e-01 / 2.63e-01 | 20260505 10:58:15.131 | INFO | PerfReport: 3.30%, Histogram -- 1.07e+00 / 1.48e+00, 2.05e-01 / 1.18e-01 | 20260505 10:58:15.131 | INFO | PerfReport: 15.85%, DME -- 4.09e+01 / 4.55e+01, 4.63e+00 / 2.48e+00 | 20260505 10:58:15.131 | INFO | PerfReport: 56.44%, FrameLabeling -- 1.82e+01 / 2.14e+01, 1.65e+00 / 2.17e+00 | 20260505 10:58:15.131 | INFO | PerfReport: 7.40%, PulseAccumulating -- 2.38e+00 / 2.91e+00, 4.27e-01 / 2.47e-01 | 20260505 10:58:15.131 | INFO | PerfReport: 3.72%, Metrics -- 1.20e+00 / 1.51e+00, 3.47e-01 / 2.09e-01 | 20260505 10:58:15.131 | INFO | PerfReport: 1.31%, PulseSerializing -- 4.24e-01 / 4.70e-01, 7.96e-02 / 3.97e-02 | 20260505 10:58:15.131 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 10:58:15.131 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:58:15.131 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:58:15.131 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.47e+01 / 0.00e+00 | 20260505 10:58:15.131 | INFO | PerfReport: 22.57%, Upload -- 2.29e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260505 10:58:15.131 | INFO | PerfReport: 21.92%, Download -- 2.22e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260505 10:58:15.131 | INFO | PerfReport: 31.77%, Compute -- 3.22e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 10:58:15.131 | INFO | PerfReport: 2.09%, LockUpload -- 2.12e+00 / 0.00e+00, 7.67e+00 / 0.00e+00 | 20260505 10:58:15.131 | INFO | PerfReport: 1.21%, LockDownload -- 1.23e+00 / 0.00e+00, 2.79e+00 / 0.00e+00 | 20260505 10:58:15.131 | INFO | PerfReport: 20.42%, LockCompute -- 2.07e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260505 10:58:15.131 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.79e-02 / 0.00e+00 | 20260505 10:58:15.269 | INFO | Starting a round of Baz IO | 20260505 10:58:18.532 | INFO | Populating RTMetrics report | 20260505 10:58:18.533 | INFO | RTMetrics report populated | 20260505 10:58:18.533 | INFO | Opening tmp RTMetrics json file | 20260505 10:58:18.546 | INFO | Serializing RTMetrics to json | 20260505 10:58:18.591 | INFO | Serialization done, swapping tmp file now... | 20260505 10:58:18.611 | INFO | RTMetrics done | 20260505 10:58:18.658 | INFO | Finished a round of Baz IO: Wrote 2.398 GiB in 3.389 seconds | 20260505 10:58:18.658 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.53% / 0.01% / 75.46% | 20260505 10:58:18.658 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.30492% / 1.19209e-05% / 6.75519e-05% | 20260505 10:58:18.658 | INFO | Baz IO overall: | 20260505 10:58:18.658 | INFO | BytesWritten : 1572879195664 | 20260505 10:58:18.658 | INFO | NumEvents : 748909118875 | 20260505 10:58:20.264 | INFO | Analyzing chunk frames = [2670592,2671104) | 20260505 10:58:20.264 | INFO | MemInfo: { | 20260505 10:58:20.264 | INFO | "gpu" : " 17 GB", | 20260505 10:58:20.264 | INFO | "huge" : "", | 20260505 10:58:20.264 | INFO | "huge_shared" : " 27 GB", | 20260505 10:58:20.264 | INFO | "malloc" : " 14 GB", | 20260505 10:58:20.264 | INFO | "pinned" : " 29 GB" | 20260505 10:58:20.264 | INFO | } | 20260505 10:58:20.264 | INFO | Perf report for the last 5.13471 seconds: | 20260505 10:58:20.264 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:58:20.264 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:20.264 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:20.264 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:20.264 | INFO | ANALYSIS : 65.2%, 2.9, 100.4ms, 0 | 20260505 10:58:20.264 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:20.264 | INFO | RT_METRICS : 47.0%, 1.0, 24.6ms, 0 | 20260505 10:58:20.264 | INFO | PRE_HQ : 61.9%, 1.8, 59.4ms, 0 | 20260505 10:58:20.264 | INFO | BAZWRITER : 65.4%, 4.1, 137.8ms, 0 | 20260505 10:58:20.264 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 10:58:20.264 | INFO | Uploaded 36GiB to GPU in 2.2193s (16.2213GiB/s) | 20260505 10:58:20.264 | INFO | Downloaded 38GiB to GPU in 2.29729s (16.5412GiB/s) | 20260505 10:58:25.396 | INFO | Analyzing chunk frames = [2671104,2671616) | 20260505 10:58:25.396 | INFO | MemInfo: { | 20260505 10:58:25.396 | INFO | "gpu" : " 17 GB", | 20260505 10:58:25.396 | INFO | "huge" : "", | 20260505 10:58:25.396 | INFO | "huge_shared" : " 27 GB", | 20260505 10:58:25.396 | INFO | "malloc" : " 14 GB", | 20260505 10:58:25.396 | INFO | "pinned" : " 29 GB" | 20260505 10:58:25.396 | INFO | } | 20260505 10:58:25.396 | INFO | Perf report for the last 5.13252 seconds: | 20260505 10:58:25.396 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:58:25.396 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:25.396 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:25.396 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:25.397 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 10:58:25.397 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:25.397 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:25.397 | INFO | PRE_HQ : 29.3%, 1.0, 15.5ms, 0 | 20260505 10:58:25.397 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:25.397 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:25.397 | INFO | Uploaded 36GiB to GPU in 2.19679s (16.3875GiB/s) | 20260505 10:58:25.397 | INFO | Downloaded 34GiB to GPU in 2.1046s (16.1551GiB/s) | 20260505 10:58:30.545 | INFO | Analyzing chunk frames = [2671616,2672128) | 20260505 10:58:30.545 | INFO | MemInfo: { | 20260505 10:58:30.545 | INFO | "gpu" : " 17 GB", | 20260505 10:58:30.545 | INFO | "huge" : "", | 20260505 10:58:30.545 | INFO | "huge_shared" : " 27 GB", | 20260505 10:58:30.545 | INFO | "malloc" : " 14 GB", | 20260505 10:58:30.545 | INFO | "pinned" : " 29 GB" | 20260505 10:58:30.545 | INFO | } | 20260505 10:58:30.545 | INFO | Perf report for the last 5.14856 seconds: | 20260505 10:58:30.545 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:58:30.545 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:30.545 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:30.545 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:30.545 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 10:58:30.545 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:30.545 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:30.545 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:58:30.545 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:30.545 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:30.545 | INFO | Uploaded 36GiB to GPU in 2.16489s (16.629GiB/s) | 20260505 10:58:30.545 | INFO | Downloaded 34GiB to GPU in 2.06896s (16.4334GiB/s) | 20260505 10:58:35.689 | INFO | Analyzing chunk frames = [2672128,2672640) | 20260505 10:58:35.689 | INFO | MemInfo: { | 20260505 10:58:35.689 | INFO | "gpu" : " 17 GB", | 20260505 10:58:35.689 | INFO | "huge" : "", | 20260505 10:58:35.689 | INFO | "huge_shared" : " 27 GB", | 20260505 10:58:35.689 | INFO | "malloc" : " 14 GB", | 20260505 10:58:35.689 | INFO | "pinned" : " 29 GB" | 20260505 10:58:35.689 | INFO | } | 20260505 10:58:35.689 | INFO | Perf report for the last 5.14414 seconds: | 20260505 10:58:35.689 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:58:35.689 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:35.689 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:35.689 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:35.689 | INFO | ANALYSIS : 63.4%, 3.0, 98.6ms, 0 | 20260505 10:58:35.689 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:35.689 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:35.689 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 10:58:35.689 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:35.689 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:35.690 | INFO | Uploaded 36GiB to GPU in 2.21508s (16.2523GiB/s) | 20260505 10:58:35.690 | INFO | Downloaded 34GiB to GPU in 2.12884s (15.9711GiB/s) | 20260505 10:58:40.831 | INFO | Analyzing chunk frames = [2672640,2673152) | 20260505 10:58:40.831 | INFO | MemInfo: { | 20260505 10:58:40.831 | INFO | "gpu" : " 17 GB", | 20260505 10:58:40.831 | INFO | "huge" : "", | 20260505 10:58:40.831 | INFO | "huge_shared" : " 27 GB", | 20260505 10:58:40.831 | INFO | "malloc" : " 14 GB", | 20260505 10:58:40.831 | INFO | "pinned" : " 29 GB" | 20260505 10:58:40.831 | INFO | } | 20260505 10:58:40.831 | INFO | Perf report for the last 5.14183 seconds: | 20260505 10:58:40.831 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:58:40.831 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:40.831 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:40.831 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:40.831 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 10:58:40.831 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:40.831 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:40.831 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 10:58:40.831 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:40.831 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:40.831 | INFO | Uploaded 36GiB to GPU in 2.11888s (16.9901GiB/s) | 20260505 10:58:40.831 | INFO | Downloaded 34GiB to GPU in 2.12021s (16.0362GiB/s) | 20260505 10:58:45.957 | INFO | Analyzing chunk frames = [2673152,2673664) | 20260505 10:58:45.957 | INFO | MemInfo: { | 20260505 10:58:45.957 | INFO | "gpu" : " 17 GB", | 20260505 10:58:45.957 | INFO | "huge" : "", | 20260505 10:58:45.957 | INFO | "huge_shared" : " 27 GB", | 20260505 10:58:45.957 | INFO | "malloc" : " 14 GB", | 20260505 10:58:45.957 | INFO | "pinned" : " 29 GB" | 20260505 10:58:45.957 | INFO | } | 20260505 10:58:45.957 | INFO | Perf report for the last 5.12604 seconds: | 20260505 10:58:45.957 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:58:45.957 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:45.957 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:45.957 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:45.957 | INFO | ANALYSIS : 63.7%, 3.0, 99.2ms, 0 | 20260505 10:58:45.957 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:45.957 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:45.957 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 10:58:45.957 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:45.957 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:45.957 | INFO | Uploaded 36GiB to GPU in 2.16985s (16.591GiB/s) | 20260505 10:58:45.957 | INFO | Downloaded 34GiB to GPU in 2.11998s (16.0379GiB/s) | 20260505 10:58:51.097 | INFO | Analyzing chunk frames = [2673664,2674176) | 20260505 10:58:51.097 | INFO | MemInfo: { | 20260505 10:58:51.097 | INFO | "gpu" : " 17 GB", | 20260505 10:58:51.097 | INFO | "huge" : "", | 20260505 10:58:51.097 | INFO | "huge_shared" : " 27 GB", | 20260505 10:58:51.097 | INFO | "malloc" : " 14 GB", | 20260505 10:58:51.097 | INFO | "pinned" : " 29 GB" | 20260505 10:58:51.097 | INFO | } | 20260505 10:58:51.097 | INFO | Perf report for the last 5.13966 seconds: | 20260505 10:58:51.097 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:58:51.097 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:51.097 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:51.097 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:51.097 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 10:58:51.097 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:51.097 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:51.097 | INFO | PRE_HQ : 30.3%, 1.0, 15.8ms, 0 | 20260505 10:58:51.097 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:51.097 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:51.097 | INFO | Uploaded 36GiB to GPU in 2.16646s (16.617GiB/s) | 20260505 10:58:51.097 | INFO | Downloaded 34GiB to GPU in 2.15915s (15.7469GiB/s) | 20260505 10:58:56.249 | INFO | Analyzing chunk frames = [2674176,2674688) | 20260505 10:58:56.249 | INFO | MemInfo: { | 20260505 10:58:56.249 | INFO | "gpu" : " 17 GB", | 20260505 10:58:56.249 | INFO | "huge" : "", | 20260505 10:58:56.249 | INFO | "huge_shared" : " 27 GB", | 20260505 10:58:56.249 | INFO | "malloc" : " 14 GB", | 20260505 10:58:56.249 | INFO | "pinned" : " 29 GB" | 20260505 10:58:56.249 | INFO | } | 20260505 10:58:56.249 | INFO | Perf report for the last 5.15199 seconds: | 20260505 10:58:56.249 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:58:56.249 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:56.249 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:56.249 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:56.249 | INFO | ANALYSIS : 63.9%, 3.0, 100.0ms, 0 | 20260505 10:58:56.249 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:56.249 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:56.249 | INFO | PRE_HQ : 29.5%, 1.0, 15.4ms, 0 | 20260505 10:58:56.249 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:56.249 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:58:56.249 | INFO | Uploaded 36GiB to GPU in 2.23083s (16.1375GiB/s) | 20260505 10:58:56.249 | INFO | Downloaded 34GiB to GPU in 2.09048s (16.2642GiB/s) | 20260505 10:58:56.381 | INFO | Starting a round of Baz IO | 20260505 10:58:59.641 | INFO | Populating RTMetrics report | 20260505 10:58:59.641 | INFO | RTMetrics report populated | 20260505 10:58:59.641 | INFO | Opening tmp RTMetrics json file | 20260505 10:58:59.700 | INFO | Serializing RTMetrics to json | 20260505 10:58:59.751 | INFO | Serialization done, swapping tmp file now... | 20260505 10:58:59.758 | INFO | RTMetrics done | 20260505 10:58:59.775 | INFO | Finished a round of Baz IO: Wrote 2.398 GiB in 3.393 seconds | 20260505 10:58:59.775 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.53% / 0.01% / 75.47% | 20260505 10:58:59.775 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.30016% / 7.94729e-06% / 6.35783e-05% | 20260505 10:58:59.775 | INFO | Baz IO overall: | 20260505 10:58:59.775 | INFO | BytesWritten : 1575454058867 | 20260505 10:58:59.775 | INFO | NumEvents : 750136168295 | 20260505 10:59:01.385 | INFO | Analyzing chunk frames = [2674688,2675200) | 20260505 10:59:01.385 | INFO | MemInfo: { | 20260505 10:59:01.385 | INFO | "gpu" : " 17 GB", | 20260505 10:59:01.385 | INFO | "huge" : "", | 20260505 10:59:01.385 | INFO | "huge_shared" : " 27 GB", | 20260505 10:59:01.385 | INFO | "malloc" : " 14 GB", | 20260505 10:59:01.385 | INFO | "pinned" : " 29 GB" | 20260505 10:59:01.385 | INFO | } | 20260505 10:59:01.385 | INFO | Perf report for the last 5.13663 seconds: | 20260505 10:59:01.385 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:59:01.385 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:01.385 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:01.385 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:01.385 | INFO | ANALYSIS : 64.9%, 2.9, 99.9ms, 0 | 20260505 10:59:01.385 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:01.385 | INFO | RT_METRICS : 47.8%, 1.0, 25.1ms, 0 | 20260505 10:59:01.385 | INFO | PRE_HQ : 61.4%, 1.9, 59.4ms, 0 | 20260505 10:59:01.385 | INFO | BAZWRITER : 65.4%, 4.1, 138.2ms, 0 | 20260505 10:59:01.385 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260505 10:59:01.386 | INFO | Uploaded 36GiB to GPU in 2.17963s (16.5166GiB/s) | 20260505 10:59:01.386 | INFO | Downloaded 38GiB to GPU in 2.27384s (16.7119GiB/s) | 20260505 10:59:06.522 | INFO | Analyzing chunk frames = [2675200,2675712) | 20260505 10:59:06.522 | INFO | MemInfo: { | 20260505 10:59:06.522 | INFO | "gpu" : " 17 GB", | 20260505 10:59:06.522 | INFO | "huge" : "", | 20260505 10:59:06.522 | INFO | "huge_shared" : " 27 GB", | 20260505 10:59:06.522 | INFO | "malloc" : " 14 GB", | 20260505 10:59:06.522 | INFO | "pinned" : " 29 GB" | 20260505 10:59:06.522 | INFO | } | 20260505 10:59:06.522 | INFO | Perf report for the last 5.13673 seconds: | 20260505 10:59:06.522 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:59:06.522 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:06.522 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:06.522 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:06.522 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 10:59:06.522 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:06.522 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:06.522 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 10:59:06.522 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:06.522 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:06.522 | INFO | Uploaded 36GiB to GPU in 2.23126s (16.1344GiB/s) | 20260505 10:59:06.522 | INFO | Downloaded 34GiB to GPU in 2.08877s (16.2775GiB/s) | 20260505 10:59:11.655 | INFO | Analyzing chunk frames = [2675712,2676224) | 20260505 10:59:11.655 | INFO | MemInfo: { | 20260505 10:59:11.655 | INFO | "gpu" : " 17 GB", | 20260505 10:59:11.655 | INFO | "huge" : "", | 20260505 10:59:11.655 | INFO | "huge_shared" : " 27 GB", | 20260505 10:59:11.655 | INFO | "malloc" : " 14 GB", | 20260505 10:59:11.655 | INFO | "pinned" : " 29 GB" | 20260505 10:59:11.655 | INFO | } | 20260505 10:59:11.655 | INFO | Perf report for the last 5.13245 seconds: | 20260505 10:59:11.655 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:59:11.655 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:11.655 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:11.655 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:11.655 | INFO | ANALYSIS : 63.0%, 3.0, 97.9ms, 0 | 20260505 10:59:11.655 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:11.655 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:11.655 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 10:59:11.655 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:11.655 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:11.655 | INFO | Uploaded 36GiB to GPU in 2.16882s (16.5989GiB/s) | 20260505 10:59:11.655 | INFO | Downloaded 34GiB to GPU in 2.06764s (16.4438GiB/s) | 20260505 10:59:16.802 | INFO | Analyzing chunk frames = [2676224,2676736) | 20260505 10:59:16.802 | INFO | MemInfo: { | 20260505 10:59:16.802 | INFO | "gpu" : " 17 GB", | 20260505 10:59:16.802 | INFO | "huge" : "", | 20260505 10:59:16.802 | INFO | "huge_shared" : " 27 GB", | 20260505 10:59:16.802 | INFO | "malloc" : " 14 GB", | 20260505 10:59:16.802 | INFO | "pinned" : " 29 GB" | 20260505 10:59:16.802 | INFO | } | 20260505 10:59:16.802 | INFO | Perf report for the last 5.1476 seconds: | 20260505 10:59:16.802 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:59:16.802 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:16.802 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:16.802 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:16.802 | INFO | ANALYSIS : 62.9%, 3.0, 97.8ms, 0 | 20260505 10:59:16.802 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:16.802 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:16.802 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 10:59:16.802 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:16.802 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:16.802 | INFO | Uploaded 36GiB to GPU in 2.14908s (16.7513GiB/s) | 20260505 10:59:16.802 | INFO | Downloaded 34GiB to GPU in 2.07545s (16.382GiB/s) | 20260505 10:59:21.939 | INFO | Analyzing chunk frames = [2676736,2677248) | 20260505 10:59:21.939 | INFO | MemInfo: { | 20260505 10:59:21.939 | INFO | "gpu" : " 17 GB", | 20260505 10:59:21.939 | INFO | "huge" : "", | 20260505 10:59:21.939 | INFO | "huge_shared" : " 27 GB", | 20260505 10:59:21.939 | INFO | "malloc" : " 14 GB", | 20260505 10:59:21.939 | INFO | "pinned" : " 29 GB" | 20260505 10:59:21.939 | INFO | } | 20260505 10:59:21.939 | INFO | Perf report for the last 5.13649 seconds: | 20260505 10:59:21.939 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:59:21.939 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:21.939 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:21.939 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:21.939 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 10:59:21.939 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:21.939 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:21.939 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 10:59:21.939 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:21.939 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:21.939 | INFO | Uploaded 36GiB to GPU in 2.16912s (16.5966GiB/s) | 20260505 10:59:21.939 | INFO | Downloaded 34GiB to GPU in 2.12672s (15.987GiB/s) | 20260505 10:59:27.074 | INFO | Analyzing chunk frames = [2677248,2677760) | 20260505 10:59:27.074 | INFO | MemInfo: { | 20260505 10:59:27.074 | INFO | "gpu" : " 17 GB", | 20260505 10:59:27.074 | INFO | "huge" : "", | 20260505 10:59:27.074 | INFO | "huge_shared" : " 27 GB", | 20260505 10:59:27.074 | INFO | "malloc" : " 14 GB", | 20260505 10:59:27.074 | INFO | "pinned" : " 29 GB" | 20260505 10:59:27.074 | INFO | } | 20260505 10:59:27.074 | INFO | Perf report for the last 5.13499 seconds: | 20260505 10:59:27.074 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:59:27.074 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:27.074 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:27.074 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:27.074 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 10:59:27.074 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:27.074 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:27.074 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 10:59:27.074 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:27.074 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:27.074 | INFO | Uploaded 36GiB to GPU in 2.16135s (16.6563GiB/s) | 20260505 10:59:27.074 | INFO | Downloaded 34GiB to GPU in 2.12381s (16.009GiB/s) | 20260505 10:59:32.215 | INFO | Analyzing chunk frames = [2677760,2678272) | 20260505 10:59:32.215 | INFO | MemInfo: { | 20260505 10:59:32.215 | INFO | "gpu" : " 17 GB", | 20260505 10:59:32.215 | INFO | "huge" : "", | 20260505 10:59:32.215 | INFO | "huge_shared" : " 27 GB", | 20260505 10:59:32.215 | INFO | "malloc" : " 14 GB", | 20260505 10:59:32.215 | INFO | "pinned" : " 29 GB" | 20260505 10:59:32.215 | INFO | } | 20260505 10:59:32.215 | INFO | Perf report for the last 5.14176 seconds: | 20260505 10:59:32.215 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:59:32.215 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:32.215 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:32.216 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:32.216 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 10:59:32.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:32.216 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:32.216 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 10:59:32.216 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:32.216 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:32.216 | INFO | Uploaded 36GiB to GPU in 2.17748s (16.5328GiB/s) | 20260505 10:59:32.216 | INFO | Downloaded 34GiB to GPU in 2.11983s (16.039GiB/s) | 20260505 10:59:37.352 | INFO | Analyzing chunk frames = [2678272,2678784) | 20260505 10:59:37.352 | INFO | MemInfo: { | 20260505 10:59:37.352 | INFO | "gpu" : " 17 GB", | 20260505 10:59:37.352 | INFO | "huge" : "", | 20260505 10:59:37.353 | INFO | "huge_shared" : " 27 GB", | 20260505 10:59:37.353 | INFO | "malloc" : " 14 GB", | 20260505 10:59:37.353 | INFO | "pinned" : " 29 GB" | 20260505 10:59:37.353 | INFO | } | 20260505 10:59:37.353 | INFO | Perf report for the last 5.13714 seconds: | 20260505 10:59:37.353 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:59:37.353 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:37.353 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:37.353 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:37.353 | INFO | ANALYSIS : 64.5%, 3.0, 100.6ms, 0 | 20260505 10:59:37.353 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:37.353 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:37.353 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:59:37.353 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:37.353 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:37.353 | INFO | Uploaded 36GiB to GPU in 2.24426s (16.0409GiB/s) | 20260505 10:59:37.353 | INFO | Downloaded 34GiB to GPU in 2.09925s (16.1963GiB/s) | 20260505 10:59:37.353 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 10:59:37.353 | INFO | Total mem usage: 15.3 GiB | 20260505 10:59:37.353 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 10:59:37.353 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.79%) | 20260505 10:59:37.353 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 10:59:37.353 | INFO | High Water Breakdown: | 20260505 10:59:37.353 | INFO | 61% : 8.76 GiB ( 92.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 10:59:37.353 | INFO | 38.8% : 5.57 GiB ( 83.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 10:59:37.353 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 10:59:37.353 | INFO | ------------------------------------------ | 20260505 10:59:37.353 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:59:37.353 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 10:59:37.353 | INFO | Total mem usage: 26.5 GiB | 20260505 10:59:37.353 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 10:59:37.353 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 10:59:37.353 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 10:59:37.353 | INFO | High Water Breakdown: | 20260505 10:59:37.353 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:59:37.353 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:59:37.353 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:59:37.353 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:59:37.353 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:59:37.353 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:59:37.353 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 10:59:37.353 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 10:59:37.353 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:59:37.353 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 10:59:37.353 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 10:59:37.353 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 10:59:37.353 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 10:59:37.353 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 10:59:37.353 | INFO | ------------------------------------------ | 20260505 10:59:37.353 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:59:37.353 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 10:59:37.353 | INFO | Total mem usage: 25.6 GiB | 20260505 10:59:37.353 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 10:59:37.353 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 10:59:37.353 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 10:59:37.353 | INFO | High Water Breakdown: | 20260505 10:59:37.353 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 10:59:37.353 | INFO | ------------------------------------------ | 20260505 10:59:37.353 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:59:37.353 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 10:59:37.353 | INFO | Total mem usage: 15.7 GiB | 20260505 10:59:37.353 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 10:59:37.353 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 10:59:37.353 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 10:59:37.353 | INFO | High Water Breakdown: | 20260505 10:59:37.353 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 10:59:37.353 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 10:59:37.353 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 10:59:37.353 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 10:59:37.353 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 10:59:37.353 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 10:59:37.353 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 10:59:37.353 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 10:59:37.353 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 10:59:37.353 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 10:59:37.353 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 10:59:37.353 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 10:59:37.353 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 10:59:37.354 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 10:59:37.354 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 10:59:37.354 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 10:59:37.354 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 10:59:37.354 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 10:59:37.354 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 10:59:37.354 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 10:59:37.354 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 10:59:37.354 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 10:59:37.354 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 10:59:37.354 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 10:59:37.354 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 10:59:37.354 | INFO | ------------------------------------------ | 20260505 10:59:37.354 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 10:59:37.354 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 10:59:37.354 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:59:37.354 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:59:37.354 | INFO | PerfReport: Full Execution -- 3.23e+01 / 7.73e+01, 1.37e+01 / 2.29e+00 | 20260505 10:59:37.354 | INFO | PerfReport: 0.29%, AggregateStats -- 9.30e-02 / 1.01e-01, 1.07e-02 / 3.63e-03 | 20260505 10:59:37.354 | INFO | PerfReport: 11.71%, Baseline -- 3.78e+00 / 4.50e+00, 6.05e-01 / 1.85e-01 | 20260505 10:59:37.354 | INFO | PerfReport: 3.31%, Histogram -- 1.07e+00 / 1.51e+00, 2.05e-01 / 6.57e-02 | 20260505 10:59:37.354 | INFO | PerfReport: 15.91%, DME -- 4.11e+01 / 4.61e+01, 4.58e+00 / 2.32e+00 | 20260505 10:59:37.354 | INFO | PerfReport: 56.34%, FrameLabeling -- 1.82e+01 / 2.02e+01, 1.64e+00 / 2.06e+00 | 20260505 10:59:37.354 | INFO | PerfReport: 7.40%, PulseAccumulating -- 2.39e+00 / 2.88e+00, 4.28e-01 / 2.65e-01 | 20260505 10:59:37.354 | INFO | PerfReport: 3.72%, Metrics -- 1.20e+00 / 1.50e+00, 3.40e-01 / 2.14e-01 | 20260505 10:59:37.354 | INFO | PerfReport: 1.31%, PulseSerializing -- 4.23e-01 / 4.73e-01, 7.88e-02 / 5.09e-02 | 20260505 10:59:37.354 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 10:59:37.354 | INFO | PerfReport: Gathered 1536 samples. | 20260505 10:59:37.354 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 10:59:37.354 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260505 10:59:37.354 | INFO | PerfReport: 22.47%, Upload -- 2.28e+01 / 0.00e+00, 8.09e+00 / 0.00e+00 | 20260505 10:59:37.354 | INFO | PerfReport: 21.89%, Download -- 2.22e+01 / 0.00e+00, 8.12e+00 / 0.00e+00 | 20260505 10:59:37.354 | INFO | PerfReport: 31.85%, Compute -- 3.23e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 10:59:37.354 | INFO | PerfReport: 1.98%, LockUpload -- 2.00e+00 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260505 10:59:37.354 | INFO | PerfReport: 1.23%, LockDownload -- 1.25e+00 / 0.00e+00, 2.77e+00 / 0.00e+00 | 20260505 10:59:37.354 | INFO | PerfReport: 20.57%, LockCompute -- 2.08e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260505 10:59:37.354 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 2.01e-02 / 0.00e+00 | 20260505 10:59:37.489 | INFO | Starting a round of Baz IO | 20260505 10:59:40.750 | INFO | Populating RTMetrics report | 20260505 10:59:40.750 | INFO | RTMetrics report populated | 20260505 10:59:40.750 | INFO | Opening tmp RTMetrics json file | 20260505 10:59:40.774 | INFO | Serializing RTMetrics to json | 20260505 10:59:40.778 | INFO | Serialization done, swapping tmp file now... | 20260505 10:59:40.778 | INFO | RTMetrics done | 20260505 10:59:40.879 | INFO | Finished a round of Baz IO: Wrote 2.395 GiB in 3.389 seconds | 20260505 10:59:40.879 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.52% / 0.01% / 75.47% | 20260505 10:59:40.879 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.28953% / 3.97364e-06% / 7.54992e-05% | 20260505 10:59:40.879 | INFO | Baz IO overall: | 20260505 10:59:40.879 | INFO | BytesWritten : 1578026194386 | 20260505 10:59:40.879 | INFO | NumEvents : 751361325717 | 20260505 10:59:42.501 | INFO | Analyzing chunk frames = [2678784,2679296) | 20260505 10:59:42.501 | INFO | MemInfo: { | 20260505 10:59:42.501 | INFO | "gpu" : " 17 GB", | 20260505 10:59:42.501 | INFO | "huge" : "", | 20260505 10:59:42.501 | INFO | "huge_shared" : " 27 GB", | 20260505 10:59:42.501 | INFO | "malloc" : " 14 GB", | 20260505 10:59:42.501 | INFO | "pinned" : " 29 GB" | 20260505 10:59:42.501 | INFO | } | 20260505 10:59:42.501 | INFO | Perf report for the last 5.14842 seconds: | 20260505 10:59:42.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:59:42.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:42.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:42.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:42.501 | INFO | ANALYSIS : 64.9%, 2.9, 100.5ms, 0 | 20260505 10:59:42.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:42.501 | INFO | RT_METRICS : 46.0%, 1.0, 24.2ms, 0 | 20260505 10:59:42.501 | INFO | PRE_HQ : 61.2%, 1.9, 59.0ms, 0 | 20260505 10:59:42.501 | INFO | BAZWRITER : 65.2%, 4.1, 136.8ms, 0 | 20260505 10:59:42.501 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 10:59:42.501 | INFO | Uploaded 36GiB to GPU in 2.22547s (16.1764GiB/s) | 20260505 10:59:42.501 | INFO | Downloaded 38GiB to GPU in 2.28853s (16.6046GiB/s) | 20260505 10:59:47.639 | INFO | Analyzing chunk frames = [2679296,2679808) | 20260505 10:59:47.639 | INFO | MemInfo: { | 20260505 10:59:47.639 | INFO | "gpu" : " 17 GB", | 20260505 10:59:47.639 | INFO | "huge" : "", | 20260505 10:59:47.639 | INFO | "huge_shared" : " 27 GB", | 20260505 10:59:47.639 | INFO | "malloc" : " 14 GB", | 20260505 10:59:47.639 | INFO | "pinned" : " 29 GB" | 20260505 10:59:47.639 | INFO | } | 20260505 10:59:47.639 | INFO | Perf report for the last 5.1382 seconds: | 20260505 10:59:47.639 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:59:47.639 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:47.639 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:47.639 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:47.639 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260505 10:59:47.639 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:47.639 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:47.639 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 10:59:47.639 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:47.639 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:47.639 | INFO | Uploaded 36GiB to GPU in 2.23787s (16.0867GiB/s) | 20260505 10:59:47.640 | INFO | Downloaded 34GiB to GPU in 2.09401s (16.2368GiB/s) | 20260505 10:59:52.774 | INFO | Analyzing chunk frames = [2679808,2680320) | 20260505 10:59:52.774 | INFO | MemInfo: { | 20260505 10:59:52.774 | INFO | "gpu" : " 17 GB", | 20260505 10:59:52.774 | INFO | "huge" : "", | 20260505 10:59:52.774 | INFO | "huge_shared" : " 27 GB", | 20260505 10:59:52.774 | INFO | "malloc" : " 14 GB", | 20260505 10:59:52.774 | INFO | "pinned" : " 29 GB" | 20260505 10:59:52.774 | INFO | } | 20260505 10:59:52.774 | INFO | Perf report for the last 5.13518 seconds: | 20260505 10:59:52.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:59:52.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:52.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:52.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:52.774 | INFO | ANALYSIS : 63.7%, 3.0, 99.1ms, 0 | 20260505 10:59:52.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:52.774 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:52.774 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 10:59:52.774 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:52.774 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:52.775 | INFO | Uploaded 36GiB to GPU in 2.23432s (16.1123GiB/s) | 20260505 10:59:52.775 | INFO | Downloaded 34GiB to GPU in 2.12493s (16.0005GiB/s) | 20260505 10:59:57.906 | INFO | Analyzing chunk frames = [2680320,2680832) | 20260505 10:59:57.906 | INFO | MemInfo: { | 20260505 10:59:57.906 | INFO | "gpu" : " 17 GB", | 20260505 10:59:57.907 | INFO | "huge" : "", | 20260505 10:59:57.907 | INFO | "huge_shared" : " 27 GB", | 20260505 10:59:57.907 | INFO | "malloc" : " 14 GB", | 20260505 10:59:57.907 | INFO | "pinned" : " 29 GB" | 20260505 10:59:57.907 | INFO | } | 20260505 10:59:57.907 | INFO | Perf report for the last 5.13211 seconds: | 20260505 10:59:57.907 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 10:59:57.907 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:57.907 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:57.907 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:57.907 | INFO | ANALYSIS : 63.0%, 3.0, 97.8ms, 0 | 20260505 10:59:57.907 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:57.907 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:57.907 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 10:59:57.907 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:57.907 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 10:59:57.907 | INFO | Uploaded 36GiB to GPU in 2.15442s (16.7098GiB/s) | 20260505 10:59:57.907 | INFO | Downloaded 34GiB to GPU in 2.07465s (16.3883GiB/s) | 20260505 11:00:03.053 | INFO | Analyzing chunk frames = [2680832,2681344) | 20260505 11:00:03.053 | INFO | MemInfo: { | 20260505 11:00:03.053 | INFO | "gpu" : " 17 GB", | 20260505 11:00:03.053 | INFO | "huge" : "", | 20260505 11:00:03.053 | INFO | "huge_shared" : " 27 GB", | 20260505 11:00:03.053 | INFO | "malloc" : " 14 GB", | 20260505 11:00:03.053 | INFO | "pinned" : " 29 GB" | 20260505 11:00:03.053 | INFO | } | 20260505 11:00:03.053 | INFO | Perf report for the last 5.14659 seconds: | 20260505 11:00:03.053 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:00:03.053 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:03.053 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:03.053 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:03.053 | INFO | ANALYSIS : 63.2%, 3.0, 98.8ms, 0 | 20260505 11:00:03.053 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:03.053 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:03.053 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 11:00:03.053 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:03.053 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:03.053 | INFO | Uploaded 36GiB to GPU in 2.20213s (16.3478GiB/s) | 20260505 11:00:03.053 | INFO | Downloaded 34GiB to GPU in 2.10248s (16.1714GiB/s) | 20260505 11:00:08.202 | INFO | Analyzing chunk frames = [2681344,2681856) | 20260505 11:00:08.202 | INFO | MemInfo: { | 20260505 11:00:08.202 | INFO | "gpu" : " 17 GB", | 20260505 11:00:08.202 | INFO | "huge" : "", | 20260505 11:00:08.202 | INFO | "huge_shared" : " 27 GB", | 20260505 11:00:08.202 | INFO | "malloc" : " 14 GB", | 20260505 11:00:08.202 | INFO | "pinned" : " 29 GB" | 20260505 11:00:08.202 | INFO | } | 20260505 11:00:08.202 | INFO | Perf report for the last 5.14847 seconds: | 20260505 11:00:08.202 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:00:08.202 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:08.202 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:08.202 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:08.202 | INFO | ANALYSIS : 63.5%, 3.0, 99.3ms, 0 | 20260505 11:00:08.202 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:08.202 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:08.202 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 11:00:08.202 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:08.202 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:08.202 | INFO | Uploaded 36GiB to GPU in 2.19533s (16.3984GiB/s) | 20260505 11:00:08.202 | INFO | Downloaded 34GiB to GPU in 2.11676s (16.0623GiB/s) | 20260505 11:00:13.337 | INFO | Analyzing chunk frames = [2681856,2682368) | 20260505 11:00:13.337 | INFO | MemInfo: { | 20260505 11:00:13.337 | INFO | "gpu" : " 17 GB", | 20260505 11:00:13.337 | INFO | "huge" : "", | 20260505 11:00:13.337 | INFO | "huge_shared" : " 27 GB", | 20260505 11:00:13.337 | INFO | "malloc" : " 14 GB", | 20260505 11:00:13.337 | INFO | "pinned" : " 29 GB" | 20260505 11:00:13.337 | INFO | } | 20260505 11:00:13.338 | INFO | Perf report for the last 5.1358 seconds: | 20260505 11:00:13.338 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:00:13.338 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:13.338 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:13.338 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:13.338 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 11:00:13.338 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:13.338 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:13.338 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 11:00:13.338 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:13.338 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:13.338 | INFO | Uploaded 36GiB to GPU in 2.10806s (17.0773GiB/s) | 20260505 11:00:13.338 | INFO | Downloaded 34GiB to GPU in 2.14615s (15.8424GiB/s) | 20260505 11:00:18.475 | INFO | Analyzing chunk frames = [2682368,2682880) | 20260505 11:00:18.475 | INFO | MemInfo: { | 20260505 11:00:18.475 | INFO | "gpu" : " 17 GB", | 20260505 11:00:18.475 | INFO | "huge" : "", | 20260505 11:00:18.475 | INFO | "huge_shared" : " 27 GB", | 20260505 11:00:18.475 | INFO | "malloc" : " 14 GB", | 20260505 11:00:18.475 | INFO | "pinned" : " 29 GB" | 20260505 11:00:18.475 | INFO | } | 20260505 11:00:18.475 | INFO | Perf report for the last 5.13743 seconds: | 20260505 11:00:18.475 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:00:18.475 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:18.475 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:18.475 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:18.475 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 11:00:18.475 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:18.475 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:18.475 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 11:00:18.475 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:18.475 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:18.475 | INFO | Uploaded 36GiB to GPU in 2.23219s (16.1277GiB/s) | 20260505 11:00:18.475 | INFO | Downloaded 34GiB to GPU in 2.09138s (16.2572GiB/s) | 20260505 11:00:18.612 | INFO | Starting a round of Baz IO | 20260505 11:00:21.864 | INFO | Populating RTMetrics report | 20260505 11:00:21.864 | INFO | RTMetrics report populated | 20260505 11:00:21.864 | INFO | Opening tmp RTMetrics json file | 20260505 11:00:21.867 | INFO | Serializing RTMetrics to json | 20260505 11:00:21.868 | INFO | Serialization done, swapping tmp file now... | 20260505 11:00:21.868 | INFO | RTMetrics done | 20260505 11:00:21.997 | INFO | Finished a round of Baz IO: Wrote 2.396 GiB in 3.385 seconds | 20260505 11:00:21.997 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.51% / 0.01% / 75.48% | 20260505 11:00:21.997 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.28766% / 3.97364e-06% / 5.5631e-05% | 20260505 11:00:21.997 | INFO | Baz IO overall: | 20260505 11:00:21.997 | INFO | BytesWritten : 1580598369396 | 20260505 11:00:21.997 | INFO | NumEvents : 752586268605 | 20260505 11:00:23.610 | INFO | Analyzing chunk frames = [2682880,2683392) | 20260505 11:00:23.610 | INFO | MemInfo: { | 20260505 11:00:23.610 | INFO | "gpu" : " 17 GB", | 20260505 11:00:23.610 | INFO | "huge" : "", | 20260505 11:00:23.610 | INFO | "huge_shared" : " 27 GB", | 20260505 11:00:23.610 | INFO | "malloc" : " 14 GB", | 20260505 11:00:23.610 | INFO | "pinned" : " 29 GB" | 20260505 11:00:23.610 | INFO | } | 20260505 11:00:23.610 | INFO | Perf report for the last 5.13523 seconds: | 20260505 11:00:23.610 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:00:23.610 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:23.610 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:23.610 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:23.610 | INFO | ANALYSIS : 64.9%, 2.9, 99.6ms, 0 | 20260505 11:00:23.610 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:23.610 | INFO | RT_METRICS : 45.5%, 1.0, 23.9ms, 0 | 20260505 11:00:23.610 | INFO | PRE_HQ : 61.2%, 1.9, 59.0ms, 0 | 20260505 11:00:23.610 | INFO | BAZWRITER : 65.2%, 4.1, 138.1ms, 0 | 20260505 11:00:23.610 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 11:00:23.610 | INFO | Uploaded 36GiB to GPU in 2.19334s (16.4133GiB/s) | 20260505 11:00:23.611 | INFO | Downloaded 38GiB to GPU in 2.2936s (16.5678GiB/s) | 20260505 11:00:28.756 | INFO | Analyzing chunk frames = [2683392,2683904) | 20260505 11:00:28.756 | INFO | MemInfo: { | 20260505 11:00:28.756 | INFO | "gpu" : " 17 GB", | 20260505 11:00:28.756 | INFO | "huge" : "", | 20260505 11:00:28.756 | INFO | "huge_shared" : " 27 GB", | 20260505 11:00:28.756 | INFO | "malloc" : " 14 GB", | 20260505 11:00:28.756 | INFO | "pinned" : " 29 GB" | 20260505 11:00:28.756 | INFO | } | 20260505 11:00:28.756 | INFO | Perf report for the last 5.14578 seconds: | 20260505 11:00:28.756 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:00:28.756 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:28.756 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:28.756 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:28.756 | INFO | ANALYSIS : 64.2%, 3.0, 100.4ms, 0 | 20260505 11:00:28.756 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:28.756 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:28.756 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:00:28.756 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:28.756 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:28.756 | INFO | Uploaded 36GiB to GPU in 2.23525s (16.1056GiB/s) | 20260505 11:00:28.756 | INFO | Downloaded 34GiB to GPU in 2.07664s (16.3726GiB/s) | 20260505 11:00:33.894 | INFO | Analyzing chunk frames = [2683904,2684416) | 20260505 11:00:33.894 | INFO | MemInfo: { | 20260505 11:00:33.894 | INFO | "gpu" : " 17 GB", | 20260505 11:00:33.894 | INFO | "huge" : "", | 20260505 11:00:33.894 | INFO | "huge_shared" : " 27 GB", | 20260505 11:00:33.894 | INFO | "malloc" : " 14 GB", | 20260505 11:00:33.894 | INFO | "pinned" : " 29 GB" | 20260505 11:00:33.894 | INFO | } | 20260505 11:00:33.894 | INFO | Perf report for the last 5.13813 seconds: | 20260505 11:00:33.894 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:00:33.894 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:33.894 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:33.894 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:33.894 | INFO | ANALYSIS : 63.7%, 3.0, 99.1ms, 0 | 20260505 11:00:33.894 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:33.894 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:33.894 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 11:00:33.894 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:33.894 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:33.894 | INFO | Uploaded 36GiB to GPU in 2.2364s (16.0973GiB/s) | 20260505 11:00:33.894 | INFO | Downloaded 34GiB to GPU in 2.12364s (16.0102GiB/s) | 20260505 11:00:39.024 | INFO | Analyzing chunk frames = [2684416,2684928) | 20260505 11:00:39.024 | INFO | MemInfo: { | 20260505 11:00:39.024 | INFO | "gpu" : " 17 GB", | 20260505 11:00:39.024 | INFO | "huge" : "", | 20260505 11:00:39.024 | INFO | "huge_shared" : " 27 GB", | 20260505 11:00:39.024 | INFO | "malloc" : " 14 GB", | 20260505 11:00:39.024 | INFO | "pinned" : " 29 GB" | 20260505 11:00:39.024 | INFO | } | 20260505 11:00:39.025 | INFO | Perf report for the last 5.13038 seconds: | 20260505 11:00:39.025 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:00:39.025 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:39.025 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:39.025 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:39.025 | INFO | ANALYSIS : 63.9%, 3.0, 99.1ms, 0 | 20260505 11:00:39.025 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:39.025 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:39.025 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 11:00:39.025 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:39.025 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:39.025 | INFO | Uploaded 36GiB to GPU in 2.2259s (16.1733GiB/s) | 20260505 11:00:39.025 | INFO | Downloaded 34GiB to GPU in 2.1198s (16.0392GiB/s) | 20260505 11:00:44.159 | INFO | Analyzing chunk frames = [2684928,2685440) | 20260505 11:00:44.159 | INFO | MemInfo: { | 20260505 11:00:44.159 | INFO | "gpu" : " 17 GB", | 20260505 11:00:44.159 | INFO | "huge" : "", | 20260505 11:00:44.159 | INFO | "huge_shared" : " 27 GB", | 20260505 11:00:44.159 | INFO | "malloc" : " 14 GB", | 20260505 11:00:44.159 | INFO | "pinned" : " 29 GB" | 20260505 11:00:44.159 | INFO | } | 20260505 11:00:44.159 | INFO | Perf report for the last 5.13482 seconds: | 20260505 11:00:44.160 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:00:44.160 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:44.160 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:44.160 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:44.160 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 11:00:44.160 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:44.160 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:44.160 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 11:00:44.160 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:44.160 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:44.160 | INFO | Uploaded 36GiB to GPU in 2.24046s (16.0681GiB/s) | 20260505 11:00:44.160 | INFO | Downloaded 34GiB to GPU in 2.11073s (16.1082GiB/s) | 20260505 11:00:49.312 | INFO | Analyzing chunk frames = [2685440,2685952) | 20260505 11:00:49.312 | INFO | MemInfo: { | 20260505 11:00:49.312 | INFO | "gpu" : " 17 GB", | 20260505 11:00:49.312 | INFO | "huge" : "", | 20260505 11:00:49.312 | INFO | "huge_shared" : " 27 GB", | 20260505 11:00:49.312 | INFO | "malloc" : " 14 GB", | 20260505 11:00:49.312 | INFO | "pinned" : " 29 GB" | 20260505 11:00:49.312 | INFO | } | 20260505 11:00:49.312 | INFO | Perf report for the last 5.15278 seconds: | 20260505 11:00:49.312 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:00:49.312 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:49.312 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:49.312 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:49.312 | INFO | ANALYSIS : 63.6%, 3.0, 99.5ms, 0 | 20260505 11:00:49.312 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:49.312 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:49.312 | INFO | PRE_HQ : 29.1%, 1.0, 15.2ms, 0 | 20260505 11:00:49.312 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:49.312 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:49.312 | INFO | Uploaded 36GiB to GPU in 2.23369s (16.1169GiB/s) | 20260505 11:00:49.312 | INFO | Downloaded 34GiB to GPU in 2.09155s (16.2559GiB/s) | 20260505 11:00:54.451 | INFO | Analyzing chunk frames = [2685952,2686464) | 20260505 11:00:54.451 | INFO | MemInfo: { | 20260505 11:00:54.451 | INFO | "gpu" : " 17 GB", | 20260505 11:00:54.451 | INFO | "huge" : "", | 20260505 11:00:54.451 | INFO | "huge_shared" : " 27 GB", | 20260505 11:00:54.451 | INFO | "malloc" : " 14 GB", | 20260505 11:00:54.451 | INFO | "pinned" : " 29 GB" | 20260505 11:00:54.451 | INFO | } | 20260505 11:00:54.451 | INFO | Perf report for the last 5.13869 seconds: | 20260505 11:00:54.451 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:00:54.451 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:54.451 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:54.451 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:54.451 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 11:00:54.451 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:54.451 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:54.451 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 11:00:54.451 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:54.451 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:54.451 | INFO | Uploaded 36GiB to GPU in 2.1741s (16.5586GiB/s) | 20260505 11:00:54.451 | INFO | Downloaded 34GiB to GPU in 2.12641s (15.9894GiB/s) | 20260505 11:00:59.588 | INFO | Analyzing chunk frames = [2686464,2686976) | 20260505 11:00:59.588 | INFO | MemInfo: { | 20260505 11:00:59.588 | INFO | "gpu" : " 17 GB", | 20260505 11:00:59.588 | INFO | "huge" : "", | 20260505 11:00:59.588 | INFO | "huge_shared" : " 27 GB", | 20260505 11:00:59.588 | INFO | "malloc" : " 14 GB", | 20260505 11:00:59.588 | INFO | "pinned" : " 29 GB" | 20260505 11:00:59.588 | INFO | } | 20260505 11:00:59.588 | INFO | Perf report for the last 5.13739 seconds: | 20260505 11:00:59.588 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:00:59.588 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:59.588 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:59.588 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:59.588 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 11:00:59.588 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:59.588 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:59.589 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 11:00:59.589 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:59.589 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:00:59.589 | INFO | Uploaded 36GiB to GPU in 2.22029s (16.2141GiB/s) | 20260505 11:00:59.589 | INFO | Downloaded 34GiB to GPU in 2.07727s (16.3676GiB/s) | 20260505 11:00:59.589 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 11:00:59.589 | INFO | Total mem usage: 15.3 GiB | 20260505 11:00:59.589 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 11:00:59.589 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -6.82%) | 20260505 11:00:59.589 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 11:00:59.589 | INFO | High Water Breakdown: | 20260505 11:00:59.589 | INFO | 61% : 8.76 GiB ( 92.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 11:00:59.589 | INFO | 38.8% : 5.57 GiB ( 83.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 11:00:59.589 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 11:00:59.589 | INFO | ------------------------------------------ | 20260505 11:00:59.589 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:00:59.589 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 11:00:59.589 | INFO | Total mem usage: 26.5 GiB | 20260505 11:00:59.589 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 11:00:59.589 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 11:00:59.589 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 11:00:59.589 | INFO | High Water Breakdown: | 20260505 11:00:59.589 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:00:59.589 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:00:59.589 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:00:59.589 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:00:59.589 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:00:59.589 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:00:59.589 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 11:00:59.589 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 11:00:59.589 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:00:59.589 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 11:00:59.589 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 11:00:59.589 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 11:00:59.589 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 11:00:59.589 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 11:00:59.589 | INFO | ------------------------------------------ | 20260505 11:00:59.589 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:00:59.589 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 11:00:59.589 | INFO | Total mem usage: 25.6 GiB | 20260505 11:00:59.589 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 11:00:59.589 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 11:00:59.589 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 11:00:59.589 | INFO | High Water Breakdown: | 20260505 11:00:59.589 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 11:00:59.589 | INFO | ------------------------------------------ | 20260505 11:00:59.589 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:00:59.589 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 11:00:59.589 | INFO | Total mem usage: 15.7 GiB | 20260505 11:00:59.589 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 11:00:59.589 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 11:00:59.589 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 11:00:59.589 | INFO | High Water Breakdown: | 20260505 11:00:59.589 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 11:00:59.589 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 11:00:59.589 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 11:00:59.589 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 11:00:59.589 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 11:00:59.589 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 11:00:59.589 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 11:00:59.589 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 11:00:59.589 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 11:00:59.589 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:00:59.589 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 11:00:59.589 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:00:59.589 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 11:00:59.589 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 11:00:59.589 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 11:00:59.589 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 11:00:59.589 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:00:59.589 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 11:00:59.589 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:00:59.589 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:00:59.590 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:00:59.590 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 11:00:59.590 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 11:00:59.590 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:00:59.590 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 11:00:59.590 | INFO | ------------------------------------------ | 20260505 11:00:59.590 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:00:59.590 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 11:00:59.590 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:00:59.590 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:00:59.590 | INFO | PerfReport: Full Execution -- 3.22e+01 / 7.74e+01, 1.37e+01 / 1.45e+00 | 20260505 11:00:59.590 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 1.02e-01, 1.06e-02 / 5.88e-03 | 20260505 11:00:59.590 | INFO | PerfReport: 11.68%, Baseline -- 3.77e+00 / 4.52e+00, 6.02e-01 / 1.75e-01 | 20260505 11:00:59.590 | INFO | PerfReport: 3.31%, Histogram -- 1.07e+00 / 1.50e+00, 2.08e-01 / 9.08e-02 | 20260505 11:00:59.590 | INFO | PerfReport: 15.64%, DME -- 4.03e+01 / 4.43e+01, 4.19e+00 / 1.45e+00 | 20260505 11:00:59.590 | INFO | PerfReport: 56.67%, FrameLabeling -- 1.83e+01 / 2.23e+01, 1.77e+00 / 1.74e+00 | 20260505 11:00:59.590 | INFO | PerfReport: 7.35%, PulseAccumulating -- 2.37e+00 / 2.83e+00, 4.07e-01 / 2.03e-01 | 20260505 11:00:59.590 | INFO | PerfReport: 3.73%, Metrics -- 1.20e+00 / 1.46e+00, 3.47e-01 / 1.83e-01 | 20260505 11:00:59.590 | INFO | PerfReport: 1.31%, PulseSerializing -- 4.24e-01 / 4.69e-01, 8.03e-02 / 2.85e-02 | 20260505 11:00:59.590 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 11:00:59.590 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:00:59.590 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:00:59.590 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.46e+01 / 0.00e+00 | 20260505 11:00:59.590 | INFO | PerfReport: 22.68%, Upload -- 2.30e+01 / 0.00e+00, 8.38e+00 / 0.00e+00 | 20260505 11:00:59.590 | INFO | PerfReport: 21.86%, Download -- 2.22e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260505 11:00:59.590 | INFO | PerfReport: 31.78%, Compute -- 3.22e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 11:00:59.590 | INFO | PerfReport: 2.18%, LockUpload -- 2.21e+00 / 0.00e+00, 7.68e+00 / 0.00e+00 | 20260505 11:00:59.590 | INFO | PerfReport: 1.06%, LockDownload -- 1.08e+00 / 0.00e+00, 2.55e+00 / 0.00e+00 | 20260505 11:00:59.590 | INFO | PerfReport: 20.43%, LockCompute -- 2.07e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260505 11:00:59.590 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.76e-02 / 0.00e+00 | 20260505 11:00:59.729 | INFO | Starting a round of Baz IO | 20260505 11:01:02.987 | INFO | Populating RTMetrics report | 20260505 11:01:02.988 | INFO | RTMetrics report populated | 20260505 11:01:02.988 | INFO | Opening tmp RTMetrics json file | 20260505 11:01:03.000 | INFO | Serializing RTMetrics to json | 20260505 11:01:03.025 | INFO | Serialization done, swapping tmp file now... | 20260505 11:01:03.026 | INFO | RTMetrics done | 20260505 11:01:03.120 | INFO | Finished a round of Baz IO: Wrote 2.394 GiB in 3.391 seconds | 20260505 11:01:03.120 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.50% / 0.01% / 75.49% | 20260505 11:01:03.120 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.28346% / 1.58946e-05% / 6.35783e-05% | 20260505 11:01:03.120 | INFO | Baz IO overall: | 20260505 11:01:03.120 | INFO | BytesWritten : 1583169153210 | 20260505 11:01:03.120 | INFO | NumEvents : 753810442387 | 20260505 11:01:04.717 | INFO | Analyzing chunk frames = [2686976,2687488) | 20260505 11:01:04.717 | INFO | MemInfo: { | 20260505 11:01:04.717 | INFO | "gpu" : " 17 GB", | 20260505 11:01:04.717 | INFO | "huge" : "", | 20260505 11:01:04.717 | INFO | "huge_shared" : " 27 GB", | 20260505 11:01:04.717 | INFO | "malloc" : " 14 GB", | 20260505 11:01:04.717 | INFO | "pinned" : " 29 GB" | 20260505 11:01:04.717 | INFO | } | 20260505 11:01:04.717 | INFO | Perf report for the last 5.12919 seconds: | 20260505 11:01:04.718 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:01:04.718 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:04.718 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:04.718 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:04.718 | INFO | ANALYSIS : 65.2%, 2.9, 100.1ms, 0 | 20260505 11:01:04.718 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:04.718 | INFO | RT_METRICS : 46.2%, 1.0, 24.2ms, 0 | 20260505 11:01:04.718 | INFO | PRE_HQ : 61.8%, 1.8, 59.2ms, 0 | 20260505 11:01:04.718 | INFO | BAZWRITER : 65.3%, 4.1, 138.8ms, 0 | 20260505 11:01:04.718 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 11:01:04.718 | INFO | Uploaded 36GiB to GPU in 2.21363s (16.2629GiB/s) | 20260505 11:01:04.718 | INFO | Downloaded 38GiB to GPU in 2.29706s (16.5429GiB/s) | 20260505 11:01:09.861 | INFO | Analyzing chunk frames = [2687488,2688000) | 20260505 11:01:09.861 | INFO | MemInfo: { | 20260505 11:01:09.861 | INFO | "gpu" : " 17 GB", | 20260505 11:01:09.861 | INFO | "huge" : "", | 20260505 11:01:09.861 | INFO | "huge_shared" : " 27 GB", | 20260505 11:01:09.861 | INFO | "malloc" : " 14 GB", | 20260505 11:01:09.861 | INFO | "pinned" : " 29 GB" | 20260505 11:01:09.861 | INFO | } | 20260505 11:01:09.862 | INFO | Perf report for the last 5.14423 seconds: | 20260505 11:01:09.862 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:01:09.862 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:09.862 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:09.862 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:09.862 | INFO | ANALYSIS : 64.1%, 3.0, 100.2ms, 0 | 20260505 11:01:09.862 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:09.862 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:09.862 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:01:09.862 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:09.862 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:09.862 | INFO | Uploaded 36GiB to GPU in 2.18049s (16.51GiB/s) | 20260505 11:01:09.862 | INFO | Downloaded 34GiB to GPU in 2.09945s (16.1947GiB/s) | 20260505 11:01:15.003 | INFO | Analyzing chunk frames = [2688000,2688512) | 20260505 11:01:15.003 | INFO | MemInfo: { | 20260505 11:01:15.003 | INFO | "gpu" : " 17 GB", | 20260505 11:01:15.003 | INFO | "huge" : "", | 20260505 11:01:15.003 | INFO | "huge_shared" : " 27 GB", | 20260505 11:01:15.003 | INFO | "malloc" : " 14 GB", | 20260505 11:01:15.003 | INFO | "pinned" : " 29 GB" | 20260505 11:01:15.003 | INFO | } | 20260505 11:01:15.003 | INFO | Perf report for the last 5.14098 seconds: | 20260505 11:01:15.003 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:01:15.003 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:15.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:15.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:15.003 | INFO | ANALYSIS : 63.7%, 3.0, 99.2ms, 0 | 20260505 11:01:15.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:15.003 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:15.003 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 11:01:15.003 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:15.003 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:15.003 | INFO | Uploaded 36GiB to GPU in 2.22601s (16.1724GiB/s) | 20260505 11:01:15.003 | INFO | Downloaded 34GiB to GPU in 2.12445s (16.0042GiB/s) | 20260505 11:01:20.150 | INFO | Analyzing chunk frames = [2688512,2689024) | 20260505 11:01:20.150 | INFO | MemInfo: { | 20260505 11:01:20.150 | INFO | "gpu" : " 17 GB", | 20260505 11:01:20.150 | INFO | "huge" : "", | 20260505 11:01:20.150 | INFO | "huge_shared" : " 27 GB", | 20260505 11:01:20.150 | INFO | "malloc" : " 14 GB", | 20260505 11:01:20.150 | INFO | "pinned" : " 29 GB" | 20260505 11:01:20.150 | INFO | } | 20260505 11:01:20.150 | INFO | Perf report for the last 5.14773 seconds: | 20260505 11:01:20.150 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:01:20.150 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:20.150 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:20.150 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:20.150 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 11:01:20.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:20.151 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:20.151 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 11:01:20.151 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:20.151 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:20.151 | INFO | Uploaded 36GiB to GPU in 2.22645s (16.1692GiB/s) | 20260505 11:01:20.151 | INFO | Downloaded 34GiB to GPU in 2.12764s (15.9801GiB/s) | 20260505 11:01:25.284 | INFO | Analyzing chunk frames = [2689024,2689536) | 20260505 11:01:25.284 | INFO | MemInfo: { | 20260505 11:01:25.284 | INFO | "gpu" : " 17 GB", | 20260505 11:01:25.284 | INFO | "huge" : "", | 20260505 11:01:25.284 | INFO | "huge_shared" : " 27 GB", | 20260505 11:01:25.284 | INFO | "malloc" : " 14 GB", | 20260505 11:01:25.284 | INFO | "pinned" : " 29 GB" | 20260505 11:01:25.284 | INFO | } | 20260505 11:01:25.284 | INFO | Perf report for the last 5.13372 seconds: | 20260505 11:01:25.284 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:01:25.284 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:25.284 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:25.284 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:25.284 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 11:01:25.284 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:25.284 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:25.284 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 11:01:25.284 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:25.284 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:25.284 | INFO | Uploaded 36GiB to GPU in 2.13207s (16.885GiB/s) | 20260505 11:01:25.284 | INFO | Downloaded 34GiB to GPU in 2.12245s (16.0192GiB/s) | 20260505 11:01:30.422 | INFO | Analyzing chunk frames = [2689536,2690048) | 20260505 11:01:30.422 | INFO | MemInfo: { | 20260505 11:01:30.422 | INFO | "gpu" : " 17 GB", | 20260505 11:01:30.422 | INFO | "huge" : "", | 20260505 11:01:30.422 | INFO | "huge_shared" : " 27 GB", | 20260505 11:01:30.422 | INFO | "malloc" : " 14 GB", | 20260505 11:01:30.422 | INFO | "pinned" : " 29 GB" | 20260505 11:01:30.422 | INFO | } | 20260505 11:01:30.422 | INFO | Perf report for the last 5.13752 seconds: | 20260505 11:01:30.422 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:01:30.422 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:30.422 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:30.422 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:30.422 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 11:01:30.422 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:30.422 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:30.422 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 11:01:30.422 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:30.422 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:30.422 | INFO | Uploaded 36GiB to GPU in 2.22943s (16.1476GiB/s) | 20260505 11:01:30.422 | INFO | Downloaded 34GiB to GPU in 2.10187s (16.1761GiB/s) | 20260505 11:01:35.572 | INFO | Analyzing chunk frames = [2690048,2690560) | 20260505 11:01:35.572 | INFO | MemInfo: { | 20260505 11:01:35.572 | INFO | "gpu" : " 17 GB", | 20260505 11:01:35.572 | INFO | "huge" : "", | 20260505 11:01:35.572 | INFO | "huge_shared" : " 27 GB", | 20260505 11:01:35.572 | INFO | "malloc" : " 14 GB", | 20260505 11:01:35.572 | INFO | "pinned" : " 29 GB" | 20260505 11:01:35.572 | INFO | } | 20260505 11:01:35.572 | INFO | Perf report for the last 5.15015 seconds: | 20260505 11:01:35.572 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:01:35.572 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:35.572 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:35.572 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:35.572 | INFO | ANALYSIS : 63.4%, 3.0, 99.3ms, 0 | 20260505 11:01:35.572 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:35.572 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:35.572 | INFO | PRE_HQ : 29.9%, 1.0, 15.6ms, 0 | 20260505 11:01:35.572 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:35.572 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:35.572 | INFO | Uploaded 36GiB to GPU in 2.16359s (16.639GiB/s) | 20260505 11:01:35.572 | INFO | Downloaded 34GiB to GPU in 2.1361s (15.9169GiB/s) | 20260505 11:01:40.699 | INFO | Analyzing chunk frames = [2690560,2691072) | 20260505 11:01:40.699 | INFO | MemInfo: { | 20260505 11:01:40.699 | INFO | "gpu" : " 17 GB", | 20260505 11:01:40.699 | INFO | "huge" : "", | 20260505 11:01:40.699 | INFO | "huge_shared" : " 27 GB", | 20260505 11:01:40.699 | INFO | "malloc" : " 14 GB", | 20260505 11:01:40.699 | INFO | "pinned" : " 29 GB" | 20260505 11:01:40.699 | INFO | } | 20260505 11:01:40.699 | INFO | Perf report for the last 5.12726 seconds: | 20260505 11:01:40.699 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:01:40.699 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:40.699 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:40.699 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:40.699 | INFO | ANALYSIS : 64.0%, 3.0, 99.8ms, 0 | 20260505 11:01:40.699 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:40.699 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:40.699 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 11:01:40.699 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:40.699 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:40.700 | INFO | Uploaded 36GiB to GPU in 2.17148s (16.5785GiB/s) | 20260505 11:01:40.700 | INFO | Downloaded 34GiB to GPU in 2.10585s (16.1455GiB/s) | 20260505 11:01:40.834 | INFO | Starting a round of Baz IO | 20260505 11:01:44.107 | INFO | Populating RTMetrics report | 20260505 11:01:44.107 | INFO | RTMetrics report populated | 20260505 11:01:44.107 | INFO | Opening tmp RTMetrics json file | 20260505 11:01:44.123 | INFO | Serializing RTMetrics to json | 20260505 11:01:44.126 | INFO | Serialization done, swapping tmp file now... | 20260505 11:01:44.128 | INFO | RTMetrics done | 20260505 11:01:44.259 | INFO | Finished a round of Baz IO: Wrote 2.395 GiB in 3.424 seconds | 20260505 11:01:44.259 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.49% / 0.01% / 75.50% | 20260505 11:01:44.259 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.28101% / 1.58946e-05% / 7.94729e-05% | 20260505 11:01:44.259 | INFO | Baz IO overall: | 20260505 11:01:44.259 | INFO | BytesWritten : 1585740721132 | 20260505 11:01:44.259 | INFO | NumEvents : 755035092397 | 20260505 11:01:45.841 | INFO | Analyzing chunk frames = [2691072,2691584) | 20260505 11:01:45.841 | INFO | MemInfo: { | 20260505 11:01:45.841 | INFO | "gpu" : " 17 GB", | 20260505 11:01:45.841 | INFO | "huge" : "", | 20260505 11:01:45.841 | INFO | "huge_shared" : " 27 GB", | 20260505 11:01:45.841 | INFO | "malloc" : " 14 GB", | 20260505 11:01:45.841 | INFO | "pinned" : " 29 GB" | 20260505 11:01:45.841 | INFO | } | 20260505 11:01:45.841 | INFO | Perf report for the last 5.14173 seconds: | 20260505 11:01:45.841 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:01:45.841 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:45.841 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:45.841 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:45.841 | INFO | ANALYSIS : 65.1%, 2.9, 100.1ms, 0 | 20260505 11:01:45.841 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:45.841 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260505 11:01:45.841 | INFO | PRE_HQ : 61.8%, 1.9, 59.6ms, 0 | 20260505 11:01:45.841 | INFO | BAZWRITER : 65.9%, 4.1, 138.9ms, 0 | 20260505 11:01:45.841 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260505 11:01:45.841 | INFO | Uploaded 36GiB to GPU in 2.21571s (16.2476GiB/s) | 20260505 11:01:45.841 | INFO | Downloaded 38GiB to GPU in 2.29464s (16.5603GiB/s) | 20260505 11:01:50.978 | INFO | Analyzing chunk frames = [2691584,2692096) | 20260505 11:01:50.978 | INFO | MemInfo: { | 20260505 11:01:50.978 | INFO | "gpu" : " 17 GB", | 20260505 11:01:50.978 | INFO | "huge" : "", | 20260505 11:01:50.978 | INFO | "huge_shared" : " 27 GB", | 20260505 11:01:50.978 | INFO | "malloc" : " 14 GB", | 20260505 11:01:50.978 | INFO | "pinned" : " 29 GB" | 20260505 11:01:50.978 | INFO | } | 20260505 11:01:50.978 | INFO | Perf report for the last 5.13747 seconds: | 20260505 11:01:50.978 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:01:50.978 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:50.978 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:50.978 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:50.978 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 11:01:50.978 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:50.979 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:50.979 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:01:50.979 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:50.979 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:50.979 | INFO | Uploaded 36GiB to GPU in 2.23442s (16.1116GiB/s) | 20260505 11:01:50.979 | INFO | Downloaded 34GiB to GPU in 2.08408s (16.3142GiB/s) | 20260505 11:01:56.117 | INFO | Analyzing chunk frames = [2692096,2692608) | 20260505 11:01:56.117 | INFO | MemInfo: { | 20260505 11:01:56.117 | INFO | "gpu" : " 17 GB", | 20260505 11:01:56.117 | INFO | "huge" : "", | 20260505 11:01:56.117 | INFO | "huge_shared" : " 27 GB", | 20260505 11:01:56.117 | INFO | "malloc" : " 14 GB", | 20260505 11:01:56.117 | INFO | "pinned" : " 29 GB" | 20260505 11:01:56.117 | INFO | } | 20260505 11:01:56.117 | INFO | Perf report for the last 5.1389 seconds: | 20260505 11:01:56.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:01:56.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:56.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:56.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:56.117 | INFO | ANALYSIS : 63.0%, 3.0, 98.0ms, 0 | 20260505 11:01:56.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:56.117 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:56.117 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 11:01:56.117 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:56.117 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:01:56.117 | INFO | Uploaded 36GiB to GPU in 2.15324s (16.719GiB/s) | 20260505 11:01:56.117 | INFO | Downloaded 34GiB to GPU in 2.06259s (16.4841GiB/s) | 20260505 11:02:01.260 | INFO | Analyzing chunk frames = [2692608,2693120) | 20260505 11:02:01.260 | INFO | MemInfo: { | 20260505 11:02:01.260 | INFO | "gpu" : " 17 GB", | 20260505 11:02:01.260 | INFO | "huge" : "", | 20260505 11:02:01.260 | INFO | "huge_shared" : " 27 GB", | 20260505 11:02:01.260 | INFO | "malloc" : " 14 GB", | 20260505 11:02:01.260 | INFO | "pinned" : " 29 GB" | 20260505 11:02:01.260 | INFO | } | 20260505 11:02:01.260 | INFO | Perf report for the last 5.14319 seconds: | 20260505 11:02:01.260 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:02:01.260 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:01.260 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:01.260 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:01.260 | INFO | ANALYSIS : 63.6%, 3.0, 98.9ms, 0 | 20260505 11:02:01.261 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:01.261 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:01.261 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 11:02:01.261 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:01.261 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:01.261 | INFO | Uploaded 36GiB to GPU in 2.22414s (16.186GiB/s) | 20260505 11:02:01.261 | INFO | Downloaded 34GiB to GPU in 2.129s (15.97GiB/s) | 20260505 11:02:06.404 | INFO | Analyzing chunk frames = [2693120,2693632) | 20260505 11:02:06.405 | INFO | MemInfo: { | 20260505 11:02:06.405 | INFO | "gpu" : " 17 GB", | 20260505 11:02:06.405 | INFO | "huge" : "", | 20260505 11:02:06.405 | INFO | "huge_shared" : " 27 GB", | 20260505 11:02:06.405 | INFO | "malloc" : " 14 GB", | 20260505 11:02:06.405 | INFO | "pinned" : " 29 GB" | 20260505 11:02:06.405 | INFO | } | 20260505 11:02:06.405 | INFO | Perf report for the last 5.14408 seconds: | 20260505 11:02:06.405 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:02:06.405 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:06.405 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:06.405 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:06.405 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 11:02:06.405 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:06.405 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:06.405 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 11:02:06.405 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:06.405 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:06.405 | INFO | Uploaded 36GiB to GPU in 2.18084s (16.5074GiB/s) | 20260505 11:02:06.405 | INFO | Downloaded 34GiB to GPU in 2.12538s (15.9972GiB/s) | 20260505 11:02:11.532 | INFO | Analyzing chunk frames = [2693632,2694144) | 20260505 11:02:11.532 | INFO | MemInfo: { | 20260505 11:02:11.532 | INFO | "gpu" : " 17 GB", | 20260505 11:02:11.532 | INFO | "huge" : "", | 20260505 11:02:11.532 | INFO | "huge_shared" : " 27 GB", | 20260505 11:02:11.532 | INFO | "malloc" : " 14 GB", | 20260505 11:02:11.532 | INFO | "pinned" : " 29 GB" | 20260505 11:02:11.532 | INFO | } | 20260505 11:02:11.532 | INFO | Perf report for the last 5.12731 seconds: | 20260505 11:02:11.532 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:02:11.532 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:11.532 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:11.532 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:11.532 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 11:02:11.532 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:11.532 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:11.532 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 11:02:11.532 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:11.532 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:11.532 | INFO | Uploaded 36GiB to GPU in 2.17997s (16.514GiB/s) | 20260505 11:02:11.532 | INFO | Downloaded 34GiB to GPU in 2.11729s (16.0582GiB/s) | 20260505 11:02:16.667 | INFO | Analyzing chunk frames = [2694144,2694656) | 20260505 11:02:16.667 | INFO | MemInfo: { | 20260505 11:02:16.667 | INFO | "gpu" : " 17 GB", | 20260505 11:02:16.667 | INFO | "huge" : "", | 20260505 11:02:16.667 | INFO | "huge_shared" : " 27 GB", | 20260505 11:02:16.667 | INFO | "malloc" : " 14 GB", | 20260505 11:02:16.667 | INFO | "pinned" : " 29 GB" | 20260505 11:02:16.667 | INFO | } | 20260505 11:02:16.667 | INFO | Perf report for the last 5.13536 seconds: | 20260505 11:02:16.667 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:02:16.667 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:16.667 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:16.667 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:16.667 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 11:02:16.667 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:16.667 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:16.667 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 11:02:16.667 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:16.667 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:16.668 | INFO | Uploaded 36GiB to GPU in 2.17888s (16.5222GiB/s) | 20260505 11:02:16.668 | INFO | Downloaded 34GiB to GPU in 2.12135s (16.0275GiB/s) | 20260505 11:02:21.815 | INFO | Analyzing chunk frames = [2694656,2695168) | 20260505 11:02:21.815 | INFO | MemInfo: { | 20260505 11:02:21.815 | INFO | "gpu" : " 17 GB", | 20260505 11:02:21.816 | INFO | "huge" : "", | 20260505 11:02:21.816 | INFO | "huge_shared" : " 27 GB", | 20260505 11:02:21.816 | INFO | "malloc" : " 14 GB", | 20260505 11:02:21.816 | INFO | "pinned" : " 29 GB" | 20260505 11:02:21.816 | INFO | } | 20260505 11:02:21.816 | INFO | Perf report for the last 5.14828 seconds: | 20260505 11:02:21.816 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:02:21.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:21.816 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:21.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:21.816 | INFO | ANALYSIS : 64.3%, 3.0, 100.5ms, 0 | 20260505 11:02:21.816 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:21.816 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:21.816 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 11:02:21.816 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:21.816 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:21.816 | INFO | Uploaded 36GiB to GPU in 2.24857s (16.0102GiB/s) | 20260505 11:02:21.816 | INFO | Downloaded 34GiB to GPU in 2.0917s (16.2547GiB/s) | 20260505 11:02:21.816 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 11:02:21.816 | INFO | Total mem usage: 15.3 GiB | 20260505 11:02:21.816 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 11:02:21.816 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -6.83%) | 20260505 11:02:21.816 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 11:02:21.816 | INFO | High Water Breakdown: | 20260505 11:02:21.816 | INFO | 61% : 8.75 GiB ( 92.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 11:02:21.816 | INFO | 38.8% : 5.57 GiB ( 83.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 11:02:21.816 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 11:02:21.816 | INFO | ------------------------------------------ | 20260505 11:02:21.816 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:02:21.816 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 11:02:21.816 | INFO | Total mem usage: 26.5 GiB | 20260505 11:02:21.816 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 11:02:21.816 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 11:02:21.816 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 11:02:21.816 | INFO | High Water Breakdown: | 20260505 11:02:21.816 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:02:21.816 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:02:21.816 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:02:21.816 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:02:21.816 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:02:21.816 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:02:21.816 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 11:02:21.816 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 11:02:21.816 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:02:21.816 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 11:02:21.816 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 11:02:21.816 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 11:02:21.816 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 11:02:21.816 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 11:02:21.816 | INFO | ------------------------------------------ | 20260505 11:02:21.816 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:02:21.816 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 11:02:21.816 | INFO | Total mem usage: 25.6 GiB | 20260505 11:02:21.816 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 11:02:21.816 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 11:02:21.816 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 11:02:21.816 | INFO | High Water Breakdown: | 20260505 11:02:21.816 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 11:02:21.816 | INFO | ------------------------------------------ | 20260505 11:02:21.816 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:02:21.816 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 11:02:21.816 | INFO | Total mem usage: 15.7 GiB | 20260505 11:02:21.817 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 11:02:21.817 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 11:02:21.817 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 11:02:21.817 | INFO | High Water Breakdown: | 20260505 11:02:21.817 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 11:02:21.817 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 11:02:21.817 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 11:02:21.817 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 11:02:21.817 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 11:02:21.817 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 11:02:21.817 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 11:02:21.817 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 11:02:21.817 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 11:02:21.817 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:02:21.817 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 11:02:21.817 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:02:21.817 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 11:02:21.817 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 11:02:21.817 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 11:02:21.817 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 11:02:21.817 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:02:21.817 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 11:02:21.817 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:02:21.817 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:02:21.817 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:02:21.817 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 11:02:21.817 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 11:02:21.817 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:02:21.817 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 11:02:21.817 | INFO | ------------------------------------------ | 20260505 11:02:21.817 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:02:21.817 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 11:02:21.817 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:02:21.817 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:02:21.817 | INFO | PerfReport: Full Execution -- 3.23e+01 / 7.78e+01, 1.35e+01 / 1.99e+00 | 20260505 11:02:21.817 | INFO | PerfReport: 0.29%, AggregateStats -- 9.31e-02 / 1.00e-01, 1.02e-02 / 3.77e-03 | 20260505 11:02:21.817 | INFO | PerfReport: 11.70%, Baseline -- 3.78e+00 / 4.49e+00, 6.03e-01 / 2.11e-01 | 20260505 11:02:21.817 | INFO | PerfReport: 3.32%, Histogram -- 1.07e+00 / 1.52e+00, 2.06e-01 / 6.23e-02 | 20260505 11:02:21.817 | INFO | PerfReport: 15.88%, DME -- 4.10e+01 / 4.60e+01, 4.60e+00 / 2.36e+00 | 20260505 11:02:21.817 | INFO | PerfReport: 56.44%, FrameLabeling -- 1.82e+01 / 2.08e+01, 1.67e+00 / 2.19e+00 | 20260505 11:02:21.817 | INFO | PerfReport: 7.34%, PulseAccumulating -- 2.37e+00 / 2.95e+00, 4.22e-01 / 2.07e-01 | 20260505 11:02:21.817 | INFO | PerfReport: 3.70%, Metrics -- 1.19e+00 / 1.51e+00, 3.42e-01 / 1.71e-01 | 20260505 11:02:21.817 | INFO | PerfReport: 1.31%, PulseSerializing -- 4.23e-01 / 4.74e-01, 7.93e-02 / 3.41e-02 | 20260505 11:02:21.817 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 11:02:21.817 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:02:21.817 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:02:21.817 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260505 11:02:21.817 | INFO | PerfReport: 22.57%, Upload -- 2.29e+01 / 0.00e+00, 8.17e+00 / 0.00e+00 | 20260505 11:02:21.817 | INFO | PerfReport: 21.92%, Download -- 2.22e+01 / 0.00e+00, 8.17e+00 / 0.00e+00 | 20260505 11:02:21.817 | INFO | PerfReport: 31.81%, Compute -- 3.23e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 11:02:21.817 | INFO | PerfReport: 2.10%, LockUpload -- 2.13e+00 / 0.00e+00, 7.66e+00 / 0.00e+00 | 20260505 11:02:21.817 | INFO | PerfReport: 1.19%, LockDownload -- 1.21e+00 / 0.00e+00, 2.70e+00 / 0.00e+00 | 20260505 11:02:21.817 | INFO | PerfReport: 20.40%, LockCompute -- 2.07e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260505 11:02:21.817 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 3.49e-02 / 0.00e+00 | 20260505 11:02:21.950 | INFO | Starting a round of Baz IO | 20260505 11:02:25.209 | INFO | Populating RTMetrics report | 20260505 11:02:25.210 | INFO | RTMetrics report populated | 20260505 11:02:25.210 | INFO | Opening tmp RTMetrics json file | 20260505 11:02:25.237 | INFO | Serializing RTMetrics to json | 20260505 11:02:25.238 | INFO | Serialization done, swapping tmp file now... | 20260505 11:02:25.239 | INFO | RTMetrics done | 20260505 11:02:25.339 | INFO | Finished a round of Baz IO: Wrote 2.393 GiB in 3.389 seconds | 20260505 11:02:25.339 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.48% / 0.01% / 75.51% | 20260505 11:02:25.339 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.27556% / 2.38419e-05% / 9.13938e-05% | 20260505 11:02:25.339 | INFO | Baz IO overall: | 20260505 11:02:25.339 | INFO | BytesWritten : 1588310597999 | 20260505 11:02:25.339 | INFO | NumEvents : 756258540082 | 20260505 11:02:26.961 | INFO | Analyzing chunk frames = [2695168,2695680) | 20260505 11:02:26.961 | INFO | MemInfo: { | 20260505 11:02:26.961 | INFO | "gpu" : " 17 GB", | 20260505 11:02:26.961 | INFO | "huge" : "", | 20260505 11:02:26.961 | INFO | "huge_shared" : " 27 GB", | 20260505 11:02:26.961 | INFO | "malloc" : " 14 GB", | 20260505 11:02:26.961 | INFO | "pinned" : " 29 GB" | 20260505 11:02:26.961 | INFO | } | 20260505 11:02:26.961 | INFO | Perf report for the last 5.14513 seconds: | 20260505 11:02:26.961 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:02:26.961 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:26.961 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:26.961 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:26.961 | INFO | ANALYSIS : 64.8%, 2.9, 99.7ms, 0 | 20260505 11:02:26.961 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:26.961 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260505 11:02:26.961 | INFO | PRE_HQ : 61.3%, 1.9, 59.0ms, 0 | 20260505 11:02:26.961 | INFO | BAZWRITER : 65.2%, 4.1, 137.7ms, 0 | 20260505 11:02:26.961 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 11:02:26.961 | INFO | Uploaded 36GiB to GPU in 2.20892s (16.2976GiB/s) | 20260505 11:02:26.961 | INFO | Downloaded 38GiB to GPU in 2.29578s (16.5521GiB/s) | 20260505 11:02:32.100 | INFO | Analyzing chunk frames = [2695680,2696192) | 20260505 11:02:32.100 | INFO | MemInfo: { | 20260505 11:02:32.100 | INFO | "gpu" : " 17 GB", | 20260505 11:02:32.100 | INFO | "huge" : "", | 20260505 11:02:32.100 | INFO | "huge_shared" : " 27 GB", | 20260505 11:02:32.100 | INFO | "malloc" : " 14 GB", | 20260505 11:02:32.100 | INFO | "pinned" : " 29 GB" | 20260505 11:02:32.100 | INFO | } | 20260505 11:02:32.101 | INFO | Perf report for the last 5.13984 seconds: | 20260505 11:02:32.101 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:02:32.101 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:32.101 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:32.101 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:32.101 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 11:02:32.101 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:32.101 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:32.101 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 11:02:32.101 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:32.101 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:32.101 | INFO | Uploaded 36GiB to GPU in 2.23351s (16.1181GiB/s) | 20260505 11:02:32.101 | INFO | Downloaded 34GiB to GPU in 2.08616s (16.2979GiB/s) | 20260505 11:02:37.231 | INFO | Analyzing chunk frames = [2696192,2696704) | 20260505 11:02:37.231 | INFO | MemInfo: { | 20260505 11:02:37.231 | INFO | "gpu" : " 17 GB", | 20260505 11:02:37.231 | INFO | "huge" : "", | 20260505 11:02:37.231 | INFO | "huge_shared" : " 27 GB", | 20260505 11:02:37.231 | INFO | "malloc" : " 14 GB", | 20260505 11:02:37.231 | INFO | "pinned" : " 29 GB" | 20260505 11:02:37.231 | INFO | } | 20260505 11:02:37.231 | INFO | Perf report for the last 5.1302 seconds: | 20260505 11:02:37.231 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:02:37.231 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:37.231 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:37.231 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:37.231 | INFO | ANALYSIS : 63.0%, 3.0, 97.9ms, 0 | 20260505 11:02:37.231 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:37.231 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:37.231 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 11:02:37.231 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:37.231 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:37.231 | INFO | Uploaded 36GiB to GPU in 2.16412s (16.635GiB/s) | 20260505 11:02:37.231 | INFO | Downloaded 34GiB to GPU in 2.06519s (16.4634GiB/s) | 20260505 11:02:42.375 | INFO | Analyzing chunk frames = [2696704,2697216) | 20260505 11:02:42.375 | INFO | MemInfo: { | 20260505 11:02:42.375 | INFO | "gpu" : " 17 GB", | 20260505 11:02:42.375 | INFO | "huge" : "", | 20260505 11:02:42.375 | INFO | "huge_shared" : " 27 GB", | 20260505 11:02:42.375 | INFO | "malloc" : " 14 GB", | 20260505 11:02:42.375 | INFO | "pinned" : " 29 GB" | 20260505 11:02:42.375 | INFO | } | 20260505 11:02:42.375 | INFO | Perf report for the last 5.1444 seconds: | 20260505 11:02:42.375 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:02:42.375 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:42.375 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:42.375 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:42.375 | INFO | ANALYSIS : 62.9%, 3.0, 97.8ms, 0 | 20260505 11:02:42.375 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:42.375 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:42.375 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:02:42.375 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:42.375 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:42.376 | INFO | Uploaded 36GiB to GPU in 2.15959s (16.6698GiB/s) | 20260505 11:02:42.376 | INFO | Downloaded 34GiB to GPU in 2.07506s (16.3851GiB/s) | 20260505 11:02:47.517 | INFO | Analyzing chunk frames = [2697216,2697728) | 20260505 11:02:47.517 | INFO | MemInfo: { | 20260505 11:02:47.517 | INFO | "gpu" : " 17 GB", | 20260505 11:02:47.517 | INFO | "huge" : "", | 20260505 11:02:47.517 | INFO | "huge_shared" : " 27 GB", | 20260505 11:02:47.517 | INFO | "malloc" : " 14 GB", | 20260505 11:02:47.517 | INFO | "pinned" : " 29 GB" | 20260505 11:02:47.517 | INFO | } | 20260505 11:02:47.517 | INFO | Perf report for the last 5.14177 seconds: | 20260505 11:02:47.517 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:02:47.517 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:47.517 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:47.517 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:47.517 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 11:02:47.517 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:47.517 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:47.517 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 11:02:47.517 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:47.517 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:47.517 | INFO | Uploaded 36GiB to GPU in 2.14898s (16.7522GiB/s) | 20260505 11:02:47.517 | INFO | Downloaded 34GiB to GPU in 2.12517s (15.9987GiB/s) | 20260505 11:02:52.654 | INFO | Analyzing chunk frames = [2697728,2698240) | 20260505 11:02:52.654 | INFO | MemInfo: { | 20260505 11:02:52.654 | INFO | "gpu" : " 17 GB", | 20260505 11:02:52.654 | INFO | "huge" : "", | 20260505 11:02:52.654 | INFO | "huge_shared" : " 27 GB", | 20260505 11:02:52.654 | INFO | "malloc" : " 14 GB", | 20260505 11:02:52.654 | INFO | "pinned" : " 29 GB" | 20260505 11:02:52.654 | INFO | } | 20260505 11:02:52.654 | INFO | Perf report for the last 5.13678 seconds: | 20260505 11:02:52.654 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:02:52.654 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:52.654 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:52.654 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:52.654 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 11:02:52.654 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:52.654 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:52.654 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 11:02:52.654 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:52.654 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:52.654 | INFO | Uploaded 36GiB to GPU in 2.16297s (16.6438GiB/s) | 20260505 11:02:52.654 | INFO | Downloaded 34GiB to GPU in 2.10742s (16.1334GiB/s) | 20260505 11:02:57.790 | INFO | Analyzing chunk frames = [2698240,2698752) | 20260505 11:02:57.790 | INFO | MemInfo: { | 20260505 11:02:57.790 | INFO | "gpu" : " 17 GB", | 20260505 11:02:57.790 | INFO | "huge" : "", | 20260505 11:02:57.790 | INFO | "huge_shared" : " 27 GB", | 20260505 11:02:57.790 | INFO | "malloc" : " 14 GB", | 20260505 11:02:57.790 | INFO | "pinned" : " 29 GB" | 20260505 11:02:57.790 | INFO | } | 20260505 11:02:57.790 | INFO | Perf report for the last 5.13675 seconds: | 20260505 11:02:57.791 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:02:57.791 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:57.791 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:57.791 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:57.791 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 11:02:57.791 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:57.791 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:57.791 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 11:02:57.791 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:57.791 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:02:57.791 | INFO | Uploaded 36GiB to GPU in 2.19429s (16.4062GiB/s) | 20260505 11:02:57.791 | INFO | Downloaded 34GiB to GPU in 2.1373s (15.9079GiB/s) | 20260505 11:03:02.935 | INFO | Analyzing chunk frames = [2698752,2699264) | 20260505 11:03:02.935 | INFO | MemInfo: { | 20260505 11:03:02.935 | INFO | "gpu" : " 17 GB", | 20260505 11:03:02.935 | INFO | "huge" : "", | 20260505 11:03:02.935 | INFO | "huge_shared" : " 27 GB", | 20260505 11:03:02.935 | INFO | "malloc" : " 14 GB", | 20260505 11:03:02.935 | INFO | "pinned" : " 29 GB" | 20260505 11:03:02.935 | INFO | } | 20260505 11:03:02.935 | INFO | Perf report for the last 5.14452 seconds: | 20260505 11:03:02.935 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:03:02.935 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:02.935 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:02.935 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:02.935 | INFO | ANALYSIS : 64.5%, 3.0, 100.8ms, 0 | 20260505 11:03:02.935 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:02.935 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:02.935 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 11:03:02.935 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:02.935 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:02.935 | INFO | Uploaded 36GiB to GPU in 2.24572s (16.0305GiB/s) | 20260505 11:03:02.935 | INFO | Downloaded 34GiB to GPU in 2.09455s (16.2326GiB/s) | 20260505 11:03:03.070 | INFO | Starting a round of Baz IO | 20260505 11:03:06.328 | INFO | Populating RTMetrics report | 20260505 11:03:06.328 | INFO | RTMetrics report populated | 20260505 11:03:06.328 | INFO | Opening tmp RTMetrics json file | 20260505 11:03:06.356 | INFO | Serializing RTMetrics to json | 20260505 11:03:06.358 | INFO | Serialization done, swapping tmp file now... | 20260505 11:03:06.358 | INFO | RTMetrics done | 20260505 11:03:06.450 | INFO | Finished a round of Baz IO: Wrote 2.394 GiB in 3.38 seconds | 20260505 11:03:06.450 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.47% / 0.01% / 75.52% | 20260505 11:03:06.450 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.2752% / 1.98682e-05% / 7.15256e-05% | 20260505 11:03:06.450 | INFO | Baz IO overall: | 20260505 11:03:06.450 | INFO | BytesWritten : 1590880776976 | 20260505 11:03:06.450 | INFO | NumEvents : 757482533939 | 20260505 11:03:08.077 | INFO | Analyzing chunk frames = [2699264,2699776) | 20260505 11:03:08.077 | INFO | MemInfo: { | 20260505 11:03:08.077 | INFO | "gpu" : " 17 GB", | 20260505 11:03:08.077 | INFO | "huge" : "", | 20260505 11:03:08.077 | INFO | "huge_shared" : " 27 GB", | 20260505 11:03:08.077 | INFO | "malloc" : " 14 GB", | 20260505 11:03:08.077 | INFO | "pinned" : " 29 GB" | 20260505 11:03:08.077 | INFO | } | 20260505 11:03:08.077 | INFO | Perf report for the last 5.14205 seconds: | 20260505 11:03:08.077 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:03:08.077 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:08.077 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:08.077 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:08.077 | INFO | ANALYSIS : 64.9%, 2.9, 100.0ms, 0 | 20260505 11:03:08.077 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:08.077 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260505 11:03:08.077 | INFO | PRE_HQ : 61.3%, 1.9, 59.1ms, 0 | 20260505 11:03:08.077 | INFO | BAZWRITER : 65.0%, 4.1, 137.2ms, 0 | 20260505 11:03:08.077 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 11:03:08.077 | INFO | Uploaded 36GiB to GPU in 2.21489s (16.2536GiB/s) | 20260505 11:03:08.077 | INFO | Downloaded 38GiB to GPU in 2.30018s (16.5205GiB/s) | 20260505 11:03:13.214 | INFO | Analyzing chunk frames = [2699776,2700288) | 20260505 11:03:13.214 | INFO | MemInfo: { | 20260505 11:03:13.214 | INFO | "gpu" : " 17 GB", | 20260505 11:03:13.214 | INFO | "huge" : "", | 20260505 11:03:13.214 | INFO | "huge_shared" : " 27 GB", | 20260505 11:03:13.214 | INFO | "malloc" : " 14 GB", | 20260505 11:03:13.214 | INFO | "pinned" : " 29 GB" | 20260505 11:03:13.214 | INFO | } | 20260505 11:03:13.214 | INFO | Perf report for the last 5.13706 seconds: | 20260505 11:03:13.214 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:03:13.214 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:13.214 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:13.214 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:13.214 | INFO | ANALYSIS : 64.0%, 3.0, 99.8ms, 0 | 20260505 11:03:13.214 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:13.214 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:13.214 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 11:03:13.214 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:13.214 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:13.214 | INFO | Uploaded 36GiB to GPU in 2.17605s (16.5437GiB/s) | 20260505 11:03:13.215 | INFO | Downloaded 34GiB to GPU in 2.04754s (16.6053GiB/s) | 20260505 11:03:18.347 | INFO | Analyzing chunk frames = [2700288,2700800) | 20260505 11:03:18.347 | INFO | MemInfo: { | 20260505 11:03:18.347 | INFO | "gpu" : " 17 GB", | 20260505 11:03:18.347 | INFO | "huge" : "", | 20260505 11:03:18.347 | INFO | "huge_shared" : " 27 GB", | 20260505 11:03:18.347 | INFO | "malloc" : " 14 GB", | 20260505 11:03:18.347 | INFO | "pinned" : " 29 GB" | 20260505 11:03:18.347 | INFO | } | 20260505 11:03:18.347 | INFO | Perf report for the last 5.13266 seconds: | 20260505 11:03:18.347 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:03:18.347 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:18.347 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:18.347 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:18.347 | INFO | ANALYSIS : 63.7%, 3.0, 99.0ms, 0 | 20260505 11:03:18.347 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:18.347 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:18.347 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 11:03:18.347 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:18.347 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:18.347 | INFO | Uploaded 36GiB to GPU in 2.22206s (16.2012GiB/s) | 20260505 11:03:18.347 | INFO | Downloaded 34GiB to GPU in 2.13089s (15.9558GiB/s) | 20260505 11:03:23.480 | INFO | Analyzing chunk frames = [2700800,2701312) | 20260505 11:03:23.480 | INFO | MemInfo: { | 20260505 11:03:23.480 | INFO | "gpu" : " 17 GB", | 20260505 11:03:23.480 | INFO | "huge" : "", | 20260505 11:03:23.480 | INFO | "huge_shared" : " 27 GB", | 20260505 11:03:23.480 | INFO | "malloc" : " 14 GB", | 20260505 11:03:23.480 | INFO | "pinned" : " 29 GB" | 20260505 11:03:23.480 | INFO | } | 20260505 11:03:23.480 | INFO | Perf report for the last 5.13327 seconds: | 20260505 11:03:23.480 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:03:23.480 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:23.480 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:23.480 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:23.480 | INFO | ANALYSIS : 63.1%, 3.0, 97.9ms, 0 | 20260505 11:03:23.480 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:23.480 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:23.480 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 11:03:23.480 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:23.480 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:23.480 | INFO | Uploaded 36GiB to GPU in 2.165s (16.6282GiB/s) | 20260505 11:03:23.480 | INFO | Downloaded 34GiB to GPU in 2.07566s (16.3803GiB/s) | 20260505 11:03:28.624 | INFO | Analyzing chunk frames = [2701312,2701824) | 20260505 11:03:28.624 | INFO | MemInfo: { | 20260505 11:03:28.624 | INFO | "gpu" : " 17 GB", | 20260505 11:03:28.624 | INFO | "huge" : "", | 20260505 11:03:28.624 | INFO | "huge_shared" : " 27 GB", | 20260505 11:03:28.624 | INFO | "malloc" : " 14 GB", | 20260505 11:03:28.624 | INFO | "pinned" : " 29 GB" | 20260505 11:03:28.624 | INFO | } | 20260505 11:03:28.624 | INFO | Perf report for the last 5.14347 seconds: | 20260505 11:03:28.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:03:28.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:28.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:28.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:28.624 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 11:03:28.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:28.624 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:28.624 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 11:03:28.624 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:28.624 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:28.624 | INFO | Uploaded 36GiB to GPU in 2.23626s (16.0983GiB/s) | 20260505 11:03:28.624 | INFO | Downloaded 34GiB to GPU in 2.11259s (16.094GiB/s) | 20260505 11:03:33.769 | INFO | Analyzing chunk frames = [2701824,2702336) | 20260505 11:03:33.769 | INFO | MemInfo: { | 20260505 11:03:33.769 | INFO | "gpu" : " 17 GB", | 20260505 11:03:33.769 | INFO | "huge" : "", | 20260505 11:03:33.769 | INFO | "huge_shared" : " 27 GB", | 20260505 11:03:33.769 | INFO | "malloc" : " 14 GB", | 20260505 11:03:33.769 | INFO | "pinned" : " 29 GB" | 20260505 11:03:33.769 | INFO | } | 20260505 11:03:33.769 | INFO | Perf report for the last 5.14546 seconds: | 20260505 11:03:33.769 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:03:33.769 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:33.769 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:33.769 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:33.769 | INFO | ANALYSIS : 63.5%, 3.0, 99.3ms, 0 | 20260505 11:03:33.769 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:33.769 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:33.769 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 11:03:33.769 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:33.769 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:33.769 | INFO | Uploaded 36GiB to GPU in 2.16347s (16.6399GiB/s) | 20260505 11:03:33.770 | INFO | Downloaded 34GiB to GPU in 2.1172s (16.059GiB/s) | 20260505 11:03:38.918 | INFO | Analyzing chunk frames = [2702336,2702848) | 20260505 11:03:38.918 | INFO | MemInfo: { | 20260505 11:03:38.918 | INFO | "gpu" : " 17 GB", | 20260505 11:03:38.918 | INFO | "huge" : "", | 20260505 11:03:38.918 | INFO | "huge_shared" : " 27 GB", | 20260505 11:03:38.918 | INFO | "malloc" : " 14 GB", | 20260505 11:03:38.918 | INFO | "pinned" : " 29 GB" | 20260505 11:03:38.918 | INFO | } | 20260505 11:03:38.919 | INFO | Perf report for the last 5.14942 seconds: | 20260505 11:03:38.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:03:38.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:38.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:38.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:38.919 | INFO | ANALYSIS : 63.3%, 3.0, 99.0ms, 0 | 20260505 11:03:38.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:38.919 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:38.919 | INFO | PRE_HQ : 29.9%, 1.0, 15.6ms, 0 | 20260505 11:03:38.919 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:38.919 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:38.919 | INFO | Uploaded 36GiB to GPU in 2.18143s (16.503GiB/s) | 20260505 11:03:38.919 | INFO | Downloaded 34GiB to GPU in 2.13971s (15.89GiB/s) | 20260505 11:03:44.046 | INFO | Analyzing chunk frames = [2702848,2703360) | 20260505 11:03:44.046 | INFO | MemInfo: { | 20260505 11:03:44.046 | INFO | "gpu" : " 17 GB", | 20260505 11:03:44.046 | INFO | "huge" : "", | 20260505 11:03:44.046 | INFO | "huge_shared" : " 27 GB", | 20260505 11:03:44.046 | INFO | "malloc" : " 14 GB", | 20260505 11:03:44.046 | INFO | "pinned" : " 29 GB" | 20260505 11:03:44.046 | INFO | } | 20260505 11:03:44.046 | INFO | Perf report for the last 5.12793 seconds: | 20260505 11:03:44.046 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:03:44.046 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:44.046 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:44.047 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:44.047 | INFO | ANALYSIS : 64.0%, 3.0, 99.7ms, 0 | 20260505 11:03:44.047 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:44.047 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:44.047 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 11:03:44.047 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:44.047 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:44.047 | INFO | Uploaded 36GiB to GPU in 2.12421s (16.9475GiB/s) | 20260505 11:03:44.047 | INFO | Downloaded 34GiB to GPU in 2.11105s (16.1057GiB/s) | 20260505 11:03:44.047 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 11:03:44.047 | INFO | Total mem usage: 15.3 GiB | 20260505 11:03:44.047 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 11:03:44.047 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -6.83%) | 20260505 11:03:44.047 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 11:03:44.047 | INFO | High Water Breakdown: | 20260505 11:03:44.047 | INFO | 61% : 8.75 GiB ( 92.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 11:03:44.047 | INFO | 38.8% : 5.57 GiB ( 83.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 11:03:44.047 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 11:03:44.047 | INFO | ------------------------------------------ | 20260505 11:03:44.047 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:03:44.047 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 11:03:44.047 | INFO | Total mem usage: 26.5 GiB | 20260505 11:03:44.047 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 11:03:44.047 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 11:03:44.047 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 11:03:44.047 | INFO | High Water Breakdown: | 20260505 11:03:44.047 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:03:44.047 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:03:44.047 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:03:44.047 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:03:44.047 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:03:44.047 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:03:44.047 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 11:03:44.047 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 11:03:44.047 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:03:44.047 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 11:03:44.047 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 11:03:44.047 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 11:03:44.047 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 11:03:44.047 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 11:03:44.047 | INFO | ------------------------------------------ | 20260505 11:03:44.047 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:03:44.047 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 11:03:44.047 | INFO | Total mem usage: 25.6 GiB | 20260505 11:03:44.047 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 11:03:44.047 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 11:03:44.047 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 11:03:44.047 | INFO | High Water Breakdown: | 20260505 11:03:44.047 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 11:03:44.047 | INFO | ------------------------------------------ | 20260505 11:03:44.047 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:03:44.047 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 11:03:44.047 | INFO | Total mem usage: 15.7 GiB | 20260505 11:03:44.047 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 11:03:44.047 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 11:03:44.047 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 11:03:44.047 | INFO | High Water Breakdown: | 20260505 11:03:44.047 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 11:03:44.047 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 11:03:44.047 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 11:03:44.047 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 11:03:44.047 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 11:03:44.047 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 11:03:44.047 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 11:03:44.047 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 11:03:44.047 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 11:03:44.047 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:03:44.047 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 11:03:44.047 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:03:44.047 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 11:03:44.047 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 11:03:44.047 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 11:03:44.047 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 11:03:44.047 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:03:44.047 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 11:03:44.047 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:03:44.047 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:03:44.047 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:03:44.048 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 11:03:44.048 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 11:03:44.048 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:03:44.048 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 11:03:44.048 | INFO | ------------------------------------------ | 20260505 11:03:44.048 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:03:44.048 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 11:03:44.048 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:03:44.048 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:03:44.048 | INFO | PerfReport: Full Execution -- 3.22e+01 / 7.73e+01, 1.37e+01 / 1.89e+00 | 20260505 11:03:44.048 | INFO | PerfReport: 0.29%, AggregateStats -- 9.32e-02 / 9.99e-02, 1.10e-02 / 3.33e-03 | 20260505 11:03:44.048 | INFO | PerfReport: 11.72%, Baseline -- 3.78e+00 / 4.51e+00, 6.09e-01 / 1.70e-01 | 20260505 11:03:44.048 | INFO | PerfReport: 3.31%, Histogram -- 1.07e+00 / 1.50e+00, 2.06e-01 / 8.07e-02 | 20260505 11:03:44.048 | INFO | PerfReport: 15.86%, DME -- 4.09e+01 / 4.57e+01, 4.41e+00 / 1.95e+00 | 20260505 11:03:44.048 | INFO | PerfReport: 56.42%, FrameLabeling -- 1.82e+01 / 2.06e+01, 1.69e+00 / 2.18e+00 | 20260505 11:03:44.048 | INFO | PerfReport: 7.39%, PulseAccumulating -- 2.38e+00 / 2.96e+00, 4.31e-01 / 2.22e-01 | 20260505 11:03:44.048 | INFO | PerfReport: 3.70%, Metrics -- 1.19e+00 / 1.51e+00, 3.40e-01 / 1.84e-01 | 20260505 11:03:44.048 | INFO | PerfReport: 1.31%, PulseSerializing -- 4.22e-01 / 4.69e-01, 7.91e-02 / 4.53e-02 | 20260505 11:03:44.048 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 11:03:44.049 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:03:44.049 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:03:44.049 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.46e+01 / 0.00e+00 | 20260505 11:03:44.049 | INFO | PerfReport: 22.50%, Upload -- 2.28e+01 / 0.00e+00, 8.18e+00 / 0.00e+00 | 20260505 11:03:44.049 | INFO | PerfReport: 21.87%, Download -- 2.22e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260505 11:03:44.049 | INFO | PerfReport: 31.83%, Compute -- 3.22e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 11:03:44.049 | INFO | PerfReport: 2.02%, LockUpload -- 2.04e+00 / 0.00e+00, 7.63e+00 / 0.00e+00 | 20260505 11:03:44.049 | INFO | PerfReport: 1.22%, LockDownload -- 1.23e+00 / 0.00e+00, 2.74e+00 / 0.00e+00 | 20260505 11:03:44.049 | INFO | PerfReport: 20.55%, LockCompute -- 2.08e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260505 11:03:44.049 | INFO | PerfReport: 0.01%, Misc -- 1.49e-02 / 0.00e+00, 2.99e-02 / 0.00e+00 | 20260505 11:03:44.181 | INFO | Starting a round of Baz IO | 20260505 11:03:47.452 | INFO | Populating RTMetrics report | 20260505 11:03:47.452 | INFO | RTMetrics report populated | 20260505 11:03:47.452 | INFO | Opening tmp RTMetrics json file | 20260505 11:03:47.471 | INFO | Serializing RTMetrics to json | 20260505 11:03:47.510 | INFO | Serialization done, swapping tmp file now... | 20260505 11:03:47.529 | INFO | RTMetrics done | 20260505 11:03:47.578 | INFO | Finished a round of Baz IO: Wrote 2.391 GiB in 3.397 seconds | 20260505 11:03:47.578 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.46% / 0.01% / 75.53% | 20260505 11:03:47.578 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.26588% / 1.98682e-05% / 8.74201e-05% | 20260505 11:03:47.578 | INFO | Baz IO overall: | 20260505 11:03:47.578 | INFO | BytesWritten : 1593447599030 | 20260505 11:03:47.578 | INFO | NumEvents : 758704046204 | 20260505 11:03:49.181 | INFO | Analyzing chunk frames = [2703360,2703872) | 20260505 11:03:49.181 | INFO | MemInfo: { | 20260505 11:03:49.181 | INFO | "gpu" : " 17 GB", | 20260505 11:03:49.181 | INFO | "huge" : "", | 20260505 11:03:49.181 | INFO | "huge_shared" : " 27 GB", | 20260505 11:03:49.181 | INFO | "malloc" : " 14 GB", | 20260505 11:03:49.181 | INFO | "pinned" : " 29 GB" | 20260505 11:03:49.181 | INFO | } | 20260505 11:03:49.182 | INFO | Perf report for the last 5.13504 seconds: | 20260505 11:03:49.182 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:03:49.182 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:49.182 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:49.182 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:49.182 | INFO | ANALYSIS : 65.2%, 2.9, 100.0ms, 0 | 20260505 11:03:49.182 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:49.182 | INFO | RT_METRICS : 47.3%, 1.0, 24.8ms, 0 | 20260505 11:03:49.182 | INFO | PRE_HQ : 62.0%, 1.8, 59.0ms, 0 | 20260505 11:03:49.182 | INFO | BAZWRITER : 65.5%, 4.1, 138.5ms, 0 | 20260505 11:03:49.182 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260505 11:03:49.182 | INFO | Uploaded 36GiB to GPU in 2.22334s (16.1919GiB/s) | 20260505 11:03:49.182 | INFO | Downloaded 38GiB to GPU in 2.32479s (16.3455GiB/s) | 20260505 11:03:54.325 | INFO | Analyzing chunk frames = [2703872,2704384) | 20260505 11:03:54.326 | INFO | MemInfo: { | 20260505 11:03:54.326 | INFO | "gpu" : " 17 GB", | 20260505 11:03:54.326 | INFO | "huge" : "", | 20260505 11:03:54.326 | INFO | "huge_shared" : " 27 GB", | 20260505 11:03:54.326 | INFO | "malloc" : " 14 GB", | 20260505 11:03:54.326 | INFO | "pinned" : " 29 GB" | 20260505 11:03:54.326 | INFO | } | 20260505 11:03:54.326 | INFO | Perf report for the last 5.14419 seconds: | 20260505 11:03:54.326 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:03:54.326 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:54.326 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:54.326 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:54.326 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 11:03:54.326 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:54.326 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:54.326 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 11:03:54.326 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:54.326 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:54.326 | INFO | Uploaded 36GiB to GPU in 2.23812s (16.085GiB/s) | 20260505 11:03:54.326 | INFO | Downloaded 34GiB to GPU in 2.08027s (16.344GiB/s) | 20260505 11:03:59.469 | INFO | Analyzing chunk frames = [2704384,2704896) | 20260505 11:03:59.469 | INFO | MemInfo: { | 20260505 11:03:59.469 | INFO | "gpu" : " 17 GB", | 20260505 11:03:59.469 | INFO | "huge" : "", | 20260505 11:03:59.469 | INFO | "huge_shared" : " 27 GB", | 20260505 11:03:59.469 | INFO | "malloc" : " 14 GB", | 20260505 11:03:59.469 | INFO | "pinned" : " 29 GB" | 20260505 11:03:59.469 | INFO | } | 20260505 11:03:59.469 | INFO | Perf report for the last 5.14338 seconds: | 20260505 11:03:59.469 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:03:59.469 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:59.469 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:59.469 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:59.469 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 11:03:59.469 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:59.469 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:59.469 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 11:03:59.469 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:59.469 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:03:59.469 | INFO | Uploaded 36GiB to GPU in 2.20938s (16.2942GiB/s) | 20260505 11:03:59.469 | INFO | Downloaded 34GiB to GPU in 2.11375s (16.0851GiB/s) | 20260505 11:04:04.603 | INFO | Analyzing chunk frames = [2704896,2705408) | 20260505 11:04:04.603 | INFO | MemInfo: { | 20260505 11:04:04.603 | INFO | "gpu" : " 17 GB", | 20260505 11:04:04.603 | INFO | "huge" : "", | 20260505 11:04:04.603 | INFO | "huge_shared" : " 27 GB", | 20260505 11:04:04.603 | INFO | "malloc" : " 14 GB", | 20260505 11:04:04.603 | INFO | "pinned" : " 29 GB" | 20260505 11:04:04.603 | INFO | } | 20260505 11:04:04.603 | INFO | Perf report for the last 5.13409 seconds: | 20260505 11:04:04.603 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:04:04.603 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:04.603 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:04.603 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:04.603 | INFO | ANALYSIS : 63.6%, 3.0, 98.7ms, 0 | 20260505 11:04:04.603 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:04.603 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:04.603 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 11:04:04.603 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:04.603 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:04.604 | INFO | Uploaded 36GiB to GPU in 2.21627s (16.2435GiB/s) | 20260505 11:04:04.604 | INFO | Downloaded 34GiB to GPU in 2.13097s (15.9552GiB/s) | 20260505 11:04:09.738 | INFO | Analyzing chunk frames = [2705408,2705920) | 20260505 11:04:09.738 | INFO | MemInfo: { | 20260505 11:04:09.738 | INFO | "gpu" : " 17 GB", | 20260505 11:04:09.738 | INFO | "huge" : "", | 20260505 11:04:09.738 | INFO | "huge_shared" : " 27 GB", | 20260505 11:04:09.739 | INFO | "malloc" : " 14 GB", | 20260505 11:04:09.739 | INFO | "pinned" : " 29 GB" | 20260505 11:04:09.739 | INFO | } | 20260505 11:04:09.739 | INFO | Perf report for the last 5.13544 seconds: | 20260505 11:04:09.739 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:04:09.739 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:09.739 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:09.739 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:09.739 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 11:04:09.739 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:09.739 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:09.739 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 11:04:09.739 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:09.739 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:09.739 | INFO | Uploaded 36GiB to GPU in 2.13071s (16.8958GiB/s) | 20260505 11:04:09.739 | INFO | Downloaded 34GiB to GPU in 2.12668s (15.9873GiB/s) | 20260505 11:04:14.887 | INFO | Analyzing chunk frames = [2705920,2706432) | 20260505 11:04:14.887 | INFO | MemInfo: { | 20260505 11:04:14.887 | INFO | "gpu" : " 17 GB", | 20260505 11:04:14.887 | INFO | "huge" : "", | 20260505 11:04:14.887 | INFO | "huge_shared" : " 27 GB", | 20260505 11:04:14.887 | INFO | "malloc" : " 14 GB", | 20260505 11:04:14.887 | INFO | "pinned" : " 29 GB" | 20260505 11:04:14.887 | INFO | } | 20260505 11:04:14.887 | INFO | Perf report for the last 5.1487 seconds: | 20260505 11:04:14.887 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:04:14.887 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:14.887 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:14.887 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:14.887 | INFO | ANALYSIS : 63.5%, 3.0, 99.3ms, 0 | 20260505 11:04:14.887 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:14.887 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:14.887 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 11:04:14.887 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:14.888 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:14.888 | INFO | Uploaded 36GiB to GPU in 2.16611s (16.6196GiB/s) | 20260505 11:04:14.888 | INFO | Downloaded 34GiB to GPU in 2.12395s (16.0079GiB/s) | 20260505 11:04:20.025 | INFO | Analyzing chunk frames = [2706432,2706944) | 20260505 11:04:20.025 | INFO | MemInfo: { | 20260505 11:04:20.025 | INFO | "gpu" : " 17 GB", | 20260505 11:04:20.025 | INFO | "huge" : "", | 20260505 11:04:20.025 | INFO | "huge_shared" : " 27 GB", | 20260505 11:04:20.025 | INFO | "malloc" : " 14 GB", | 20260505 11:04:20.025 | INFO | "pinned" : " 29 GB" | 20260505 11:04:20.025 | INFO | } | 20260505 11:04:20.025 | INFO | Perf report for the last 5.13758 seconds: | 20260505 11:04:20.025 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:04:20.025 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:20.025 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:20.025 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:20.025 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 11:04:20.025 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:20.025 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:20.025 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 11:04:20.025 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:20.025 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:20.025 | INFO | Uploaded 36GiB to GPU in 2.15024s (16.7423GiB/s) | 20260505 11:04:20.025 | INFO | Downloaded 34GiB to GPU in 2.15135s (15.804GiB/s) | 20260505 11:04:25.163 | INFO | Analyzing chunk frames = [2706944,2707456) | 20260505 11:04:25.163 | INFO | MemInfo: { | 20260505 11:04:25.163 | INFO | "gpu" : " 17 GB", | 20260505 11:04:25.163 | INFO | "huge" : "", | 20260505 11:04:25.163 | INFO | "huge_shared" : " 27 GB", | 20260505 11:04:25.163 | INFO | "malloc" : " 14 GB", | 20260505 11:04:25.163 | INFO | "pinned" : " 29 GB" | 20260505 11:04:25.163 | INFO | } | 20260505 11:04:25.163 | INFO | Perf report for the last 5.13825 seconds: | 20260505 11:04:25.163 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:04:25.163 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:25.163 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:25.163 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:25.163 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 11:04:25.163 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:25.163 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:25.163 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 11:04:25.163 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:25.163 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:25.164 | INFO | Uploaded 36GiB to GPU in 2.21248s (16.2714GiB/s) | 20260505 11:04:25.164 | INFO | Downloaded 34GiB to GPU in 2.08423s (16.3129GiB/s) | 20260505 11:04:25.302 | INFO | Starting a round of Baz IO | 20260505 11:04:28.557 | INFO | Populating RTMetrics report | 20260505 11:04:28.558 | INFO | RTMetrics report populated | 20260505 11:04:28.558 | INFO | Opening tmp RTMetrics json file | 20260505 11:04:28.592 | INFO | Serializing RTMetrics to json | 20260505 11:04:28.594 | INFO | Serialization done, swapping tmp file now... | 20260505 11:04:28.594 | INFO | RTMetrics done | 20260505 11:04:28.708 | INFO | Finished a round of Baz IO: Wrote 2.392 GiB in 3.406 seconds | 20260505 11:04:28.708 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.45% / 0.01% / 75.54% | 20260505 11:04:28.708 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.26511% / 1.19209e-05% / 8.34465e-05% | 20260505 11:04:28.708 | INFO | Baz IO overall: | 20260505 11:04:28.708 | INFO | BytesWritten : 1596015597508 | 20260505 11:04:28.708 | INFO | NumEvents : 759926703137 | 20260505 11:04:30.294 | INFO | Analyzing chunk frames = [2707456,2707968) | 20260505 11:04:30.294 | INFO | MemInfo: { | 20260505 11:04:30.294 | INFO | "gpu" : " 17 GB", | 20260505 11:04:30.294 | INFO | "huge" : "", | 20260505 11:04:30.294 | INFO | "huge_shared" : " 27 GB", | 20260505 11:04:30.294 | INFO | "malloc" : " 14 GB", | 20260505 11:04:30.294 | INFO | "pinned" : " 29 GB" | 20260505 11:04:30.294 | INFO | } | 20260505 11:04:30.294 | INFO | Perf report for the last 5.13062 seconds: | 20260505 11:04:30.294 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:04:30.294 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:30.294 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:30.294 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:30.294 | INFO | ANALYSIS : 65.0%, 2.9, 99.6ms, 0 | 20260505 11:04:30.294 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:30.294 | INFO | RT_METRICS : 46.3%, 1.0, 24.2ms, 0 | 20260505 11:04:30.294 | INFO | PRE_HQ : 61.6%, 1.9, 59.2ms, 0 | 20260505 11:04:30.294 | INFO | BAZWRITER : 65.7%, 4.1, 139.4ms, 0 | 20260505 11:04:30.294 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 11:04:30.294 | INFO | Uploaded 36GiB to GPU in 2.19365s (16.411GiB/s) | 20260505 11:04:30.294 | INFO | Downloaded 38GiB to GPU in 2.30147s (16.5112GiB/s) | 20260505 11:04:35.435 | INFO | Analyzing chunk frames = [2707968,2708480) | 20260505 11:04:35.435 | INFO | MemInfo: { | 20260505 11:04:35.435 | INFO | "gpu" : " 17 GB", | 20260505 11:04:35.435 | INFO | "huge" : "", | 20260505 11:04:35.435 | INFO | "huge_shared" : " 27 GB", | 20260505 11:04:35.435 | INFO | "malloc" : " 14 GB", | 20260505 11:04:35.435 | INFO | "pinned" : " 29 GB" | 20260505 11:04:35.435 | INFO | } | 20260505 11:04:35.435 | INFO | Perf report for the last 5.14142 seconds: | 20260505 11:04:35.435 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:04:35.435 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:35.435 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:35.435 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:35.435 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 11:04:35.435 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:35.435 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:35.435 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 11:04:35.435 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:35.435 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:35.436 | INFO | Uploaded 36GiB to GPU in 2.22864s (16.1534GiB/s) | 20260505 11:04:35.436 | INFO | Downloaded 34GiB to GPU in 2.1259s (15.9932GiB/s) | 20260505 11:04:40.578 | INFO | Analyzing chunk frames = [2708480,2708992) | 20260505 11:04:40.578 | INFO | MemInfo: { | 20260505 11:04:40.578 | INFO | "gpu" : " 17 GB", | 20260505 11:04:40.578 | INFO | "huge" : "", | 20260505 11:04:40.578 | INFO | "huge_shared" : " 27 GB", | 20260505 11:04:40.579 | INFO | "malloc" : " 14 GB", | 20260505 11:04:40.579 | INFO | "pinned" : " 29 GB" | 20260505 11:04:40.579 | INFO | } | 20260505 11:04:40.579 | INFO | Perf report for the last 5.14341 seconds: | 20260505 11:04:40.579 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:04:40.579 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:40.579 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:40.579 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:40.579 | INFO | ANALYSIS : 62.9%, 3.0, 98.0ms, 0 | 20260505 11:04:40.579 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:40.579 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:40.579 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 11:04:40.579 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:40.579 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:40.579 | INFO | Uploaded 36GiB to GPU in 2.15496s (16.7056GiB/s) | 20260505 11:04:40.579 | INFO | Downloaded 34GiB to GPU in 2.06704s (16.4486GiB/s) | 20260505 11:04:45.725 | INFO | Analyzing chunk frames = [2708992,2709504) | 20260505 11:04:45.725 | INFO | MemInfo: { | 20260505 11:04:45.725 | INFO | "gpu" : " 17 GB", | 20260505 11:04:45.725 | INFO | "huge" : "", | 20260505 11:04:45.725 | INFO | "huge_shared" : " 27 GB", | 20260505 11:04:45.725 | INFO | "malloc" : " 14 GB", | 20260505 11:04:45.725 | INFO | "pinned" : " 29 GB" | 20260505 11:04:45.725 | INFO | } | 20260505 11:04:45.725 | INFO | Perf report for the last 5.14622 seconds: | 20260505 11:04:45.725 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:04:45.725 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:45.725 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:45.725 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:45.725 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 11:04:45.725 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:45.725 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:45.725 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 11:04:45.725 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:45.725 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:45.725 | INFO | Uploaded 36GiB to GPU in 2.21074s (16.2841GiB/s) | 20260505 11:04:45.725 | INFO | Downloaded 34GiB to GPU in 2.12533s (15.9975GiB/s) | 20260505 11:04:50.859 | INFO | Analyzing chunk frames = [2709504,2710016) | 20260505 11:04:50.859 | INFO | MemInfo: { | 20260505 11:04:50.859 | INFO | "gpu" : " 17 GB", | 20260505 11:04:50.859 | INFO | "huge" : "", | 20260505 11:04:50.859 | INFO | "huge_shared" : " 27 GB", | 20260505 11:04:50.859 | INFO | "malloc" : " 14 GB", | 20260505 11:04:50.859 | INFO | "pinned" : " 29 GB" | 20260505 11:04:50.859 | INFO | } | 20260505 11:04:50.859 | INFO | Perf report for the last 5.1344 seconds: | 20260505 11:04:50.859 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:04:50.859 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:50.859 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:50.859 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:50.859 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 11:04:50.859 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:50.859 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:50.859 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 11:04:50.859 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:50.860 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:50.860 | INFO | Uploaded 36GiB to GPU in 2.14249s (16.8029GiB/s) | 20260505 11:04:50.860 | INFO | Downloaded 34GiB to GPU in 2.12326s (16.0131GiB/s) | 20260505 11:04:55.991 | INFO | Analyzing chunk frames = [2710016,2710528) | 20260505 11:04:55.991 | INFO | MemInfo: { | 20260505 11:04:55.991 | INFO | "gpu" : " 17 GB", | 20260505 11:04:55.991 | INFO | "huge" : "", | 20260505 11:04:55.991 | INFO | "huge_shared" : " 27 GB", | 20260505 11:04:55.991 | INFO | "malloc" : " 14 GB", | 20260505 11:04:55.991 | INFO | "pinned" : " 29 GB" | 20260505 11:04:55.991 | INFO | } | 20260505 11:04:55.991 | INFO | Perf report for the last 5.1315 seconds: | 20260505 11:04:55.991 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:04:55.991 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:55.991 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:55.991 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:55.991 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 11:04:55.991 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:55.991 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:55.991 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:04:55.991 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:55.991 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:04:55.991 | INFO | Uploaded 36GiB to GPU in 2.16609s (16.6198GiB/s) | 20260505 11:04:55.991 | INFO | Downloaded 34GiB to GPU in 2.12533s (15.9975GiB/s) | 20260505 11:05:01.137 | INFO | Analyzing chunk frames = [2710528,2711040) | 20260505 11:05:01.137 | INFO | MemInfo: { | 20260505 11:05:01.137 | INFO | "gpu" : " 17 GB", | 20260505 11:05:01.137 | INFO | "huge" : "", | 20260505 11:05:01.137 | INFO | "huge_shared" : " 27 GB", | 20260505 11:05:01.137 | INFO | "malloc" : " 14 GB", | 20260505 11:05:01.137 | INFO | "pinned" : " 29 GB" | 20260505 11:05:01.137 | INFO | } | 20260505 11:05:01.138 | INFO | Perf report for the last 5.14662 seconds: | 20260505 11:05:01.138 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:05:01.138 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:01.138 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:01.138 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:01.138 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 11:05:01.138 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:01.138 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:01.138 | INFO | PRE_HQ : 30.1%, 1.0, 15.7ms, 0 | 20260505 11:05:01.138 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:01.138 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:01.138 | INFO | Uploaded 36GiB to GPU in 2.16985s (16.591GiB/s) | 20260505 11:05:01.138 | INFO | Downloaded 34GiB to GPU in 2.14022s (15.8862GiB/s) | 20260505 11:05:06.276 | INFO | Analyzing chunk frames = [2711040,2711552) | 20260505 11:05:06.276 | INFO | MemInfo: { | 20260505 11:05:06.276 | INFO | "gpu" : " 17 GB", | 20260505 11:05:06.276 | INFO | "huge" : "", | 20260505 11:05:06.276 | INFO | "huge_shared" : " 27 GB", | 20260505 11:05:06.276 | INFO | "malloc" : " 14 GB", | 20260505 11:05:06.276 | INFO | "pinned" : " 29 GB" | 20260505 11:05:06.276 | INFO | } | 20260505 11:05:06.277 | INFO | Perf report for the last 5.13901 seconds: | 20260505 11:05:06.277 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:05:06.277 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:06.277 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:06.277 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:06.277 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 11:05:06.277 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:06.277 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:06.277 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 11:05:06.277 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:06.277 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:06.277 | INFO | Uploaded 36GiB to GPU in 2.21069s (16.2845GiB/s) | 20260505 11:05:06.277 | INFO | Downloaded 34GiB to GPU in 2.09333s (16.242GiB/s) | 20260505 11:05:06.277 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 11:05:06.277 | INFO | Total mem usage: 15.3 GiB | 20260505 11:05:06.277 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 11:05:06.277 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -6.83%) | 20260505 11:05:06.277 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 11:05:06.277 | INFO | High Water Breakdown: | 20260505 11:05:06.277 | INFO | 61% : 8.75 GiB ( 92.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 11:05:06.277 | INFO | 38.8% : 5.57 GiB ( 83.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 11:05:06.277 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 11:05:06.277 | INFO | ------------------------------------------ | 20260505 11:05:06.277 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:05:06.277 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 11:05:06.277 | INFO | Total mem usage: 26.5 GiB | 20260505 11:05:06.277 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 11:05:06.277 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 11:05:06.277 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 11:05:06.277 | INFO | High Water Breakdown: | 20260505 11:05:06.277 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:05:06.277 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:05:06.277 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:05:06.277 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:05:06.277 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:05:06.277 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:05:06.277 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 11:05:06.277 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 11:05:06.277 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:05:06.277 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 11:05:06.277 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 11:05:06.277 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 11:05:06.277 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 11:05:06.277 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 11:05:06.277 | INFO | ------------------------------------------ | 20260505 11:05:06.277 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:05:06.277 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 11:05:06.277 | INFO | Total mem usage: 25.6 GiB | 20260505 11:05:06.277 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 11:05:06.277 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 11:05:06.277 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 11:05:06.278 | INFO | High Water Breakdown: | 20260505 11:05:06.278 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 11:05:06.278 | INFO | ------------------------------------------ | 20260505 11:05:06.278 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:05:06.278 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 11:05:06.278 | INFO | Total mem usage: 15.7 GiB | 20260505 11:05:06.278 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 11:05:06.278 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 11:05:06.278 | INFO | Unused Cache: 880 MiB (Waste: 5.48%) | 20260505 11:05:06.278 | INFO | High Water Breakdown: | 20260505 11:05:06.278 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 11:05:06.278 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 11:05:06.278 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 11:05:06.278 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 11:05:06.278 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 11:05:06.278 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 11:05:06.278 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 11:05:06.278 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 11:05:06.278 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 11:05:06.278 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:05:06.278 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 11:05:06.278 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:05:06.278 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 11:05:06.278 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 11:05:06.278 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 11:05:06.278 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 11:05:06.278 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:05:06.278 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 11:05:06.278 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:05:06.278 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:05:06.278 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:05:06.278 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 11:05:06.278 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 11:05:06.278 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:05:06.278 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 11:05:06.278 | INFO | ------------------------------------------ | 20260505 11:05:06.278 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:05:06.278 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 11:05:06.278 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:05:06.278 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:05:06.278 | INFO | PerfReport: Full Execution -- 3.22e+01 / 7.72e+01, 1.39e+01 / 2.23e+00 | 20260505 11:05:06.278 | INFO | PerfReport: 0.29%, AggregateStats -- 9.32e-02 / 1.01e-01, 1.07e-02 / 5.14e-03 | 20260505 11:05:06.278 | INFO | PerfReport: 11.72%, Baseline -- 3.78e+00 / 4.44e+00, 6.11e-01 / 2.78e-01 | 20260505 11:05:06.278 | INFO | PerfReport: 3.31%, Histogram -- 1.07e+00 / 1.46e+00, 2.06e-01 / 1.16e-01 | 20260505 11:05:06.278 | INFO | PerfReport: 15.84%, DME -- 4.08e+01 / 4.51e+01, 4.43e+00 / 2.65e+00 | 20260505 11:05:06.278 | INFO | PerfReport: 56.43%, FrameLabeling -- 1.82e+01 / 2.13e+01, 1.64e+00 / 2.10e+00 | 20260505 11:05:06.278 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.38e+00 / 2.84e+00, 4.18e-01 / 2.70e-01 | 20260505 11:05:06.278 | INFO | PerfReport: 3.71%, Metrics -- 1.20e+00 / 1.50e+00, 3.40e-01 / 1.95e-01 | 20260505 11:05:06.278 | INFO | PerfReport: 1.31%, PulseSerializing -- 4.23e-01 / 4.65e-01, 7.92e-02 / 5.05e-02 | 20260505 11:05:06.278 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 11:05:06.278 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:05:06.278 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:05:06.278 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.48e+01 / 0.00e+00 | 20260505 11:05:06.278 | INFO | PerfReport: 22.51%, Upload -- 2.28e+01 / 0.00e+00, 8.11e+00 / 0.00e+00 | 20260505 11:05:06.278 | INFO | PerfReport: 22.00%, Download -- 2.23e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260505 11:05:06.278 | INFO | PerfReport: 31.82%, Compute -- 3.22e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 11:05:06.279 | INFO | PerfReport: 2.00%, LockUpload -- 2.02e+00 / 0.00e+00, 7.57e+00 / 0.00e+00 | 20260505 11:05:06.279 | INFO | PerfReport: 1.27%, LockDownload -- 1.29e+00 / 0.00e+00, 2.95e+00 / 0.00e+00 | 20260505 11:05:06.279 | INFO | PerfReport: 20.39%, LockCompute -- 2.07e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260505 11:05:06.279 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.80e-02 / 0.00e+00 | 20260505 11:05:06.416 | INFO | Starting a round of Baz IO | 20260505 11:05:09.675 | INFO | Populating RTMetrics report | 20260505 11:05:09.676 | INFO | RTMetrics report populated | 20260505 11:05:09.676 | INFO | Opening tmp RTMetrics json file | 20260505 11:05:09.690 | INFO | Serializing RTMetrics to json | 20260505 11:05:09.724 | INFO | Serialization done, swapping tmp file now... | 20260505 11:05:09.748 | INFO | RTMetrics done | 20260505 11:05:09.827 | INFO | Finished a round of Baz IO: Wrote 2.391 GiB in 3.41 seconds | 20260505 11:05:09.827 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.45% / 0.01% / 75.55% | 20260505 11:05:09.827 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.26223% / 1.58946e-05% / 8.74201e-05% | 20260505 11:05:09.827 | INFO | Baz IO overall: | 20260505 11:05:09.827 | INFO | BytesWritten : 1598582756288 | 20260505 11:05:09.827 | INFO | NumEvents : 761148846564 | 20260505 11:05:11.423 | INFO | Analyzing chunk frames = [2711552,2712064) | 20260505 11:05:11.423 | INFO | MemInfo: { | 20260505 11:05:11.423 | INFO | "gpu" : " 17 GB", | 20260505 11:05:11.423 | INFO | "huge" : "", | 20260505 11:05:11.423 | INFO | "huge_shared" : " 27 GB", | 20260505 11:05:11.423 | INFO | "malloc" : " 14 GB", | 20260505 11:05:11.423 | INFO | "pinned" : " 29 GB" | 20260505 11:05:11.423 | INFO | } | 20260505 11:05:11.423 | INFO | Perf report for the last 5.14689 seconds: | 20260505 11:05:11.423 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:05:11.424 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:11.424 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:11.424 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:11.424 | INFO | ANALYSIS : 64.9%, 2.9, 99.7ms, 0 | 20260505 11:05:11.424 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:11.424 | INFO | RT_METRICS : 46.8%, 1.0, 24.6ms, 0 | 20260505 11:05:11.424 | INFO | PRE_HQ : 61.5%, 1.8, 58.7ms, 0 | 20260505 11:05:11.424 | INFO | BAZWRITER : 65.6%, 4.2, 141.2ms, 0 | 20260505 11:05:11.424 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260505 11:05:11.424 | INFO | Uploaded 36GiB to GPU in 2.20943s (16.2938GiB/s) | 20260505 11:05:11.424 | INFO | Downloaded 38GiB to GPU in 2.30723s (16.47GiB/s) | 20260505 11:05:16.557 | INFO | Analyzing chunk frames = [2712064,2712576) | 20260505 11:05:16.557 | INFO | MemInfo: { | 20260505 11:05:16.557 | INFO | "gpu" : " 17 GB", | 20260505 11:05:16.557 | INFO | "huge" : "", | 20260505 11:05:16.557 | INFO | "huge_shared" : " 27 GB", | 20260505 11:05:16.557 | INFO | "malloc" : " 14 GB", | 20260505 11:05:16.557 | INFO | "pinned" : " 29 GB" | 20260505 11:05:16.557 | INFO | } | 20260505 11:05:16.557 | INFO | Perf report for the last 5.13399 seconds: | 20260505 11:05:16.557 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:05:16.557 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:16.557 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:16.557 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:16.557 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 11:05:16.558 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:16.558 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:16.558 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 11:05:16.558 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:16.558 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:16.558 | INFO | Uploaded 36GiB to GPU in 2.23899s (16.0787GiB/s) | 20260505 11:05:16.558 | INFO | Downloaded 34GiB to GPU in 2.08481s (16.3084GiB/s) | 20260505 11:05:21.690 | INFO | Analyzing chunk frames = [2712576,2713088) | 20260505 11:05:21.690 | INFO | MemInfo: { | 20260505 11:05:21.690 | INFO | "gpu" : " 17 GB", | 20260505 11:05:21.690 | INFO | "huge" : "", | 20260505 11:05:21.690 | INFO | "huge_shared" : " 27 GB", | 20260505 11:05:21.690 | INFO | "malloc" : " 14 GB", | 20260505 11:05:21.690 | INFO | "pinned" : " 29 GB" | 20260505 11:05:21.690 | INFO | } | 20260505 11:05:21.690 | INFO | Perf report for the last 5.13245 seconds: | 20260505 11:05:21.690 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:05:21.690 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:21.690 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:21.690 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:21.690 | INFO | ANALYSIS : 62.9%, 3.0, 97.8ms, 0 | 20260505 11:05:21.690 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:21.690 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:21.690 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 11:05:21.690 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:21.690 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:21.690 | INFO | Uploaded 36GiB to GPU in 2.15707s (16.6893GiB/s) | 20260505 11:05:21.690 | INFO | Downloaded 34GiB to GPU in 2.06095s (16.4972GiB/s) | 20260505 11:05:26.839 | INFO | Analyzing chunk frames = [2713088,2713600) | 20260505 11:05:26.839 | INFO | MemInfo: { | 20260505 11:05:26.839 | INFO | "gpu" : " 17 GB", | 20260505 11:05:26.839 | INFO | "huge" : "", | 20260505 11:05:26.839 | INFO | "huge_shared" : " 27 GB", | 20260505 11:05:26.839 | INFO | "malloc" : " 14 GB", | 20260505 11:05:26.839 | INFO | "pinned" : " 29 GB" | 20260505 11:05:26.839 | INFO | } | 20260505 11:05:26.839 | INFO | Perf report for the last 5.1494 seconds: | 20260505 11:05:26.839 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:05:26.839 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:26.839 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:26.839 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:26.839 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 11:05:26.839 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:26.839 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:26.839 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 11:05:26.839 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:26.839 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:26.840 | INFO | Uploaded 36GiB to GPU in 2.14856s (16.7554GiB/s) | 20260505 11:05:26.840 | INFO | Downloaded 34GiB to GPU in 2.07242s (16.406GiB/s) | 20260505 11:05:31.975 | INFO | Analyzing chunk frames = [2713600,2714112) | 20260505 11:05:31.975 | INFO | MemInfo: { | 20260505 11:05:31.975 | INFO | "gpu" : " 17 GB", | 20260505 11:05:31.975 | INFO | "huge" : "", | 20260505 11:05:31.975 | INFO | "huge_shared" : " 27 GB", | 20260505 11:05:31.975 | INFO | "malloc" : " 14 GB", | 20260505 11:05:31.975 | INFO | "pinned" : " 29 GB" | 20260505 11:05:31.975 | INFO | } | 20260505 11:05:31.975 | INFO | Perf report for the last 5.13595 seconds: | 20260505 11:05:31.975 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:05:31.975 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:31.975 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:31.975 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:31.975 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 11:05:31.975 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:31.975 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:31.975 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 11:05:31.975 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:31.975 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:31.976 | INFO | Uploaded 36GiB to GPU in 2.17614s (16.543GiB/s) | 20260505 11:05:31.976 | INFO | Downloaded 34GiB to GPU in 2.12614s (15.9915GiB/s) | 20260505 11:05:37.110 | INFO | Analyzing chunk frames = [2714112,2714624) | 20260505 11:05:37.110 | INFO | MemInfo: { | 20260505 11:05:37.110 | INFO | "gpu" : " 17 GB", | 20260505 11:05:37.110 | INFO | "huge" : "", | 20260505 11:05:37.110 | INFO | "huge_shared" : " 27 GB", | 20260505 11:05:37.110 | INFO | "malloc" : " 14 GB", | 20260505 11:05:37.110 | INFO | "pinned" : " 29 GB" | 20260505 11:05:37.110 | INFO | } | 20260505 11:05:37.110 | INFO | Perf report for the last 5.13459 seconds: | 20260505 11:05:37.110 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:05:37.110 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:37.110 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:37.110 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:37.110 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 11:05:37.110 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:37.110 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:37.110 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 11:05:37.110 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:37.110 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:37.110 | INFO | Uploaded 36GiB to GPU in 2.17029s (16.5876GiB/s) | 20260505 11:05:37.110 | INFO | Downloaded 34GiB to GPU in 2.12161s (16.0255GiB/s) | 20260505 11:05:42.251 | INFO | Analyzing chunk frames = [2714624,2715136) | 20260505 11:05:42.251 | INFO | MemInfo: { | 20260505 11:05:42.251 | INFO | "gpu" : " 17 GB", | 20260505 11:05:42.251 | INFO | "huge" : "", | 20260505 11:05:42.251 | INFO | "huge_shared" : " 27 GB", | 20260505 11:05:42.251 | INFO | "malloc" : " 14 GB", | 20260505 11:05:42.251 | INFO | "pinned" : " 29 GB" | 20260505 11:05:42.251 | INFO | } | 20260505 11:05:42.251 | INFO | Perf report for the last 5.14111 seconds: | 20260505 11:05:42.251 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:05:42.251 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:42.251 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:42.251 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:42.251 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 11:05:42.251 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:42.251 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:42.251 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 11:05:42.251 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:42.251 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:42.251 | INFO | Uploaded 36GiB to GPU in 2.19512s (16.4GiB/s) | 20260505 11:05:42.251 | INFO | Downloaded 34GiB to GPU in 2.15563s (15.7727GiB/s) | 20260505 11:05:47.395 | INFO | Analyzing chunk frames = [2715136,2715648) | 20260505 11:05:47.395 | INFO | MemInfo: { | 20260505 11:05:47.395 | INFO | "gpu" : " 17 GB", | 20260505 11:05:47.395 | INFO | "huge" : "", | 20260505 11:05:47.395 | INFO | "huge_shared" : " 27 GB", | 20260505 11:05:47.395 | INFO | "malloc" : " 14 GB", | 20260505 11:05:47.395 | INFO | "pinned" : " 29 GB" | 20260505 11:05:47.395 | INFO | } | 20260505 11:05:47.395 | INFO | Perf report for the last 5.14431 seconds: | 20260505 11:05:47.395 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:05:47.395 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:47.395 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:47.395 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:47.395 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 11:05:47.395 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:47.395 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:47.395 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 11:05:47.395 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:47.395 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:47.395 | INFO | Uploaded 36GiB to GPU in 2.23652s (16.0964GiB/s) | 20260505 11:05:47.396 | INFO | Downloaded 34GiB to GPU in 2.09332s (16.2422GiB/s) | 20260505 11:05:47.524 | INFO | Starting a round of Baz IO | 20260505 11:05:50.775 | INFO | Populating RTMetrics report | 20260505 11:05:50.775 | INFO | RTMetrics report populated | 20260505 11:05:50.775 | INFO | Opening tmp RTMetrics json file | 20260505 11:05:50.778 | INFO | Serializing RTMetrics to json | 20260505 11:05:50.781 | INFO | Serialization done, swapping tmp file now... | 20260505 11:05:50.782 | INFO | RTMetrics done | 20260505 11:05:50.944 | INFO | Finished a round of Baz IO: Wrote 2.393 GiB in 3.42 seconds | 20260505 11:05:50.944 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.44% / 0.01% / 75.55% | 20260505 11:05:50.944 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.26206% / 7.94729e-06% / 7.94729e-05% | 20260505 11:05:50.944 | INFO | Baz IO overall: | 20260505 11:05:50.944 | INFO | BytesWritten : 1601152752108 | 20260505 11:05:50.944 | INFO | NumEvents : 762373117276 | 20260505 11:05:52.536 | INFO | Analyzing chunk frames = [2715648,2716160) | 20260505 11:05:52.536 | INFO | MemInfo: { | 20260505 11:05:52.536 | INFO | "gpu" : " 17 GB", | 20260505 11:05:52.536 | INFO | "huge" : "", | 20260505 11:05:52.536 | INFO | "huge_shared" : " 27 GB", | 20260505 11:05:52.536 | INFO | "malloc" : " 14 GB", | 20260505 11:05:52.536 | INFO | "pinned" : " 29 GB" | 20260505 11:05:52.536 | INFO | } | 20260505 11:05:52.536 | INFO | Perf report for the last 5.14065 seconds: | 20260505 11:05:52.536 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:05:52.536 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:52.536 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:52.536 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:52.536 | INFO | ANALYSIS : 64.6%, 2.9, 99.3ms, 0 | 20260505 11:05:52.536 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:52.536 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260505 11:05:52.536 | INFO | PRE_HQ : 61.3%, 1.9, 59.1ms, 0 | 20260505 11:05:52.536 | INFO | BAZWRITER : 65.8%, 4.2, 141.7ms, 0 | 20260505 11:05:52.536 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 11:05:52.536 | INFO | Uploaded 36GiB to GPU in 2.1787s (16.5236GiB/s) | 20260505 11:05:52.536 | INFO | Downloaded 38GiB to GPU in 2.28545s (16.6269GiB/s) | 20260505 11:05:57.675 | INFO | Analyzing chunk frames = [2716160,2716672) | 20260505 11:05:57.675 | INFO | MemInfo: { | 20260505 11:05:57.675 | INFO | "gpu" : " 17 GB", | 20260505 11:05:57.675 | INFO | "huge" : "", | 20260505 11:05:57.675 | INFO | "huge_shared" : " 27 GB", | 20260505 11:05:57.675 | INFO | "malloc" : " 14 GB", | 20260505 11:05:57.675 | INFO | "pinned" : " 29 GB" | 20260505 11:05:57.675 | INFO | } | 20260505 11:05:57.675 | INFO | Perf report for the last 5.13902 seconds: | 20260505 11:05:57.675 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:05:57.675 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:57.675 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:57.675 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:57.675 | INFO | ANALYSIS : 64.3%, 3.0, 100.3ms, 0 | 20260505 11:05:57.675 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:57.675 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:57.675 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 11:05:57.675 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:57.675 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:05:57.675 | INFO | Uploaded 36GiB to GPU in 2.23417s (16.1133GiB/s) | 20260505 11:05:57.675 | INFO | Downloaded 34GiB to GPU in 2.09726s (16.2117GiB/s) | 20260505 11:06:02.807 | INFO | Analyzing chunk frames = [2716672,2717184) | 20260505 11:06:02.807 | INFO | MemInfo: { | 20260505 11:06:02.807 | INFO | "gpu" : " 17 GB", | 20260505 11:06:02.807 | INFO | "huge" : "", | 20260505 11:06:02.807 | INFO | "huge_shared" : " 27 GB", | 20260505 11:06:02.807 | INFO | "malloc" : " 14 GB", | 20260505 11:06:02.807 | INFO | "pinned" : " 29 GB" | 20260505 11:06:02.807 | INFO | } | 20260505 11:06:02.807 | INFO | Perf report for the last 5.13236 seconds: | 20260505 11:06:02.807 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:06:02.807 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:02.807 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:02.807 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:02.807 | INFO | ANALYSIS : 64.0%, 3.0, 99.5ms, 0 | 20260505 11:06:02.807 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:02.808 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:02.808 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 11:06:02.808 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:02.808 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:02.808 | INFO | Uploaded 36GiB to GPU in 2.23249s (16.1255GiB/s) | 20260505 11:06:02.808 | INFO | Downloaded 34GiB to GPU in 2.12635s (15.9898GiB/s) | 20260505 11:06:07.940 | INFO | Analyzing chunk frames = [2717184,2717696) | 20260505 11:06:07.940 | INFO | MemInfo: { | 20260505 11:06:07.940 | INFO | "gpu" : " 17 GB", | 20260505 11:06:07.940 | INFO | "huge" : "", | 20260505 11:06:07.940 | INFO | "huge_shared" : " 27 GB", | 20260505 11:06:07.940 | INFO | "malloc" : " 14 GB", | 20260505 11:06:07.940 | INFO | "pinned" : " 29 GB" | 20260505 11:06:07.940 | INFO | } | 20260505 11:06:07.940 | INFO | Perf report for the last 5.13291 seconds: | 20260505 11:06:07.940 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:06:07.940 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:07.940 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:07.940 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:07.940 | INFO | ANALYSIS : 63.1%, 3.0, 97.9ms, 0 | 20260505 11:06:07.940 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:07.940 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:07.940 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 11:06:07.940 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:07.940 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:07.941 | INFO | Uploaded 36GiB to GPU in 2.1757s (16.5464GiB/s) | 20260505 11:06:07.941 | INFO | Downloaded 34GiB to GPU in 2.08065s (16.3411GiB/s) | 20260505 11:06:13.089 | INFO | Analyzing chunk frames = [2717696,2718208) | 20260505 11:06:13.089 | INFO | MemInfo: { | 20260505 11:06:13.089 | INFO | "gpu" : " 17 GB", | 20260505 11:06:13.089 | INFO | "huge" : "", | 20260505 11:06:13.089 | INFO | "huge_shared" : " 27 GB", | 20260505 11:06:13.089 | INFO | "malloc" : " 14 GB", | 20260505 11:06:13.089 | INFO | "pinned" : " 29 GB" | 20260505 11:06:13.089 | INFO | } | 20260505 11:06:13.090 | INFO | Perf report for the last 5.14921 seconds: | 20260505 11:06:13.090 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:06:13.090 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:13.090 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:13.090 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:13.090 | INFO | ANALYSIS : 63.2%, 3.0, 98.9ms, 0 | 20260505 11:06:13.090 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:13.090 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:13.090 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 11:06:13.090 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:13.090 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:13.090 | INFO | Uploaded 36GiB to GPU in 2.16628s (16.6184GiB/s) | 20260505 11:06:13.090 | INFO | Downloaded 34GiB to GPU in 2.12857s (15.9732GiB/s) | 20260505 11:06:18.236 | INFO | Analyzing chunk frames = [2718208,2718720) | 20260505 11:06:18.237 | INFO | MemInfo: { | 20260505 11:06:18.237 | INFO | "gpu" : " 17 GB", | 20260505 11:06:18.237 | INFO | "huge" : "", | 20260505 11:06:18.237 | INFO | "huge_shared" : " 27 GB", | 20260505 11:06:18.237 | INFO | "malloc" : " 14 GB", | 20260505 11:06:18.237 | INFO | "pinned" : " 29 GB" | 20260505 11:06:18.237 | INFO | } | 20260505 11:06:18.237 | INFO | Perf report for the last 5.14726 seconds: | 20260505 11:06:18.237 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:06:18.237 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:18.237 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:18.237 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:18.237 | INFO | ANALYSIS : 63.3%, 3.0, 99.0ms, 0 | 20260505 11:06:18.237 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:18.237 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:18.237 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 11:06:18.237 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:18.237 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:18.237 | INFO | Uploaded 36GiB to GPU in 2.11579s (17.0149GiB/s) | 20260505 11:06:18.237 | INFO | Downloaded 34GiB to GPU in 2.10869s (16.1237GiB/s) | 20260505 11:06:23.374 | INFO | Analyzing chunk frames = [2718720,2719232) | 20260505 11:06:23.374 | INFO | MemInfo: { | 20260505 11:06:23.374 | INFO | "gpu" : " 17 GB", | 20260505 11:06:23.374 | INFO | "huge" : "", | 20260505 11:06:23.374 | INFO | "huge_shared" : " 27 GB", | 20260505 11:06:23.374 | INFO | "malloc" : " 14 GB", | 20260505 11:06:23.374 | INFO | "pinned" : " 29 GB" | 20260505 11:06:23.374 | INFO | } | 20260505 11:06:23.374 | INFO | Perf report for the last 5.13718 seconds: | 20260505 11:06:23.374 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:06:23.374 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:23.374 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:23.374 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:23.374 | INFO | ANALYSIS : 63.7%, 3.0, 98.4ms, 0 | 20260505 11:06:23.374 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:23.374 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:23.374 | INFO | PRE_HQ : 30.0%, 1.0, 15.4ms, 0 | 20260505 11:06:23.374 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:23.374 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:23.374 | INFO | Uploaded 36GiB to GPU in 2.18676s (16.4627GiB/s) | 20260505 11:06:23.374 | INFO | Downloaded 34GiB to GPU in 2.12839s (15.9745GiB/s) | 20260505 11:06:28.504 | INFO | Analyzing chunk frames = [2719232,2719744) | 20260505 11:06:28.504 | INFO | MemInfo: { | 20260505 11:06:28.504 | INFO | "gpu" : " 17 GB", | 20260505 11:06:28.504 | INFO | "huge" : "", | 20260505 11:06:28.504 | INFO | "huge_shared" : " 27 GB", | 20260505 11:06:28.504 | INFO | "malloc" : " 14 GB", | 20260505 11:06:28.504 | INFO | "pinned" : " 29 GB" | 20260505 11:06:28.504 | INFO | } | 20260505 11:06:28.504 | INFO | Perf report for the last 5.13025 seconds: | 20260505 11:06:28.504 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:06:28.504 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:28.504 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:28.504 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:28.504 | INFO | ANALYSIS : 64.2%, 3.0, 100.1ms, 0 | 20260505 11:06:28.504 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:28.504 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:28.504 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 11:06:28.504 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:28.504 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:28.505 | INFO | Uploaded 36GiB to GPU in 2.22148s (16.2054GiB/s) | 20260505 11:06:28.505 | INFO | Downloaded 34GiB to GPU in 2.09521s (16.2275GiB/s) | 20260505 11:06:28.505 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 11:06:28.505 | INFO | Total mem usage: 15.3 GiB | 20260505 11:06:28.505 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 11:06:28.505 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -6.82%) | 20260505 11:06:28.505 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 11:06:28.505 | INFO | High Water Breakdown: | 20260505 11:06:28.505 | INFO | 61% : 8.76 GiB ( 92.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 11:06:28.505 | INFO | 38.8% : 5.57 GiB ( 83.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 11:06:28.505 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 11:06:28.505 | INFO | ------------------------------------------ | 20260505 11:06:28.505 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:06:28.505 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 11:06:28.505 | INFO | Total mem usage: 26.5 GiB | 20260505 11:06:28.505 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 11:06:28.505 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 11:06:28.505 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 11:06:28.505 | INFO | High Water Breakdown: | 20260505 11:06:28.505 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:06:28.505 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:06:28.505 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:06:28.505 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:06:28.505 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:06:28.505 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:06:28.505 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 11:06:28.505 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 11:06:28.505 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:06:28.505 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 11:06:28.505 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 11:06:28.505 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 11:06:28.505 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 11:06:28.505 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 11:06:28.505 | INFO | ------------------------------------------ | 20260505 11:06:28.505 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:06:28.505 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 11:06:28.505 | INFO | Total mem usage: 25.6 GiB | 20260505 11:06:28.505 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 11:06:28.505 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 11:06:28.505 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 11:06:28.505 | INFO | High Water Breakdown: | 20260505 11:06:28.505 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 11:06:28.505 | INFO | ------------------------------------------ | 20260505 11:06:28.505 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:06:28.505 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 11:06:28.505 | INFO | Total mem usage: 15.7 GiB | 20260505 11:06:28.505 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 11:06:28.505 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 11:06:28.505 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 11:06:28.505 | INFO | High Water Breakdown: | 20260505 11:06:28.505 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 11:06:28.505 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 11:06:28.505 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 11:06:28.505 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 11:06:28.505 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 11:06:28.505 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 11:06:28.505 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 11:06:28.505 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 11:06:28.505 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 11:06:28.505 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:06:28.505 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 11:06:28.505 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:06:28.505 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 11:06:28.505 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 11:06:28.505 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 11:06:28.505 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 11:06:28.505 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:06:28.505 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 11:06:28.505 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:06:28.505 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:06:28.505 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:06:28.505 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 11:06:28.505 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 11:06:28.505 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:06:28.505 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 11:06:28.505 | INFO | ------------------------------------------ | 20260505 11:06:28.505 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:06:28.505 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 11:06:28.505 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:06:28.505 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:06:28.505 | INFO | PerfReport: Full Execution -- 3.22e+01 / 7.72e+01, 1.38e+01 / 2.01e+00 | 20260505 11:06:28.505 | INFO | PerfReport: 0.29%, AggregateStats -- 9.31e-02 / 1.01e-01, 1.06e-02 / 4.53e-03 | 20260505 11:06:28.505 | INFO | PerfReport: 11.70%, Baseline -- 3.77e+00 / 4.49e+00, 5.99e-01 / 1.93e-01 | 20260505 11:06:28.505 | INFO | PerfReport: 3.30%, Histogram -- 1.06e+00 / 1.50e+00, 2.05e-01 / 7.18e-02 | 20260505 11:06:28.505 | INFO | PerfReport: 15.89%, DME -- 4.09e+01 / 4.57e+01, 4.57e+00 / 2.47e+00 | 20260505 11:06:28.505 | INFO | PerfReport: 56.36%, FrameLabeling -- 1.81e+01 / 2.05e+01, 1.64e+00 / 1.81e+00 | 20260505 11:06:28.506 | INFO | PerfReport: 7.43%, PulseAccumulating -- 2.39e+00 / 2.92e+00, 4.33e-01 / 2.27e-01 | 20260505 11:06:28.506 | INFO | PerfReport: 3.72%, Metrics -- 1.20e+00 / 1.51e+00, 3.50e-01 / 2.02e-01 | 20260505 11:06:28.506 | INFO | PerfReport: 1.31%, PulseSerializing -- 4.22e-01 / 4.70e-01, 7.87e-02 / 4.69e-02 | 20260505 11:06:28.506 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 11:06:28.506 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:06:28.506 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:06:28.506 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260505 11:06:28.506 | INFO | PerfReport: 22.53%, Upload -- 2.28e+01 / 0.00e+00, 8.18e+00 / 0.00e+00 | 20260505 11:06:28.506 | INFO | PerfReport: 21.91%, Download -- 2.22e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260505 11:06:28.506 | INFO | PerfReport: 31.79%, Compute -- 3.22e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 11:06:28.506 | INFO | PerfReport: 2.10%, LockUpload -- 2.13e+00 / 0.00e+00, 7.74e+00 / 0.00e+00 | 20260505 11:06:28.506 | INFO | PerfReport: 1.22%, LockDownload -- 1.23e+00 / 0.00e+00, 2.79e+00 / 0.00e+00 | 20260505 11:06:28.506 | INFO | PerfReport: 20.44%, LockCompute -- 2.07e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260505 11:06:28.506 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.67e-02 / 0.00e+00 | 20260505 11:06:28.645 | INFO | Starting a round of Baz IO | 20260505 11:06:31.904 | INFO | Populating RTMetrics report | 20260505 11:06:31.904 | INFO | RTMetrics report populated | 20260505 11:06:31.904 | INFO | Opening tmp RTMetrics json file | 20260505 11:06:31.908 | INFO | Serializing RTMetrics to json | 20260505 11:06:31.911 | INFO | Serialization done, swapping tmp file now... | 20260505 11:06:31.913 | INFO | RTMetrics done | 20260505 11:06:32.035 | INFO | Finished a round of Baz IO: Wrote 2.393 GiB in 3.39 seconds | 20260505 11:06:32.035 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.43% / 0.01% / 75.56% | 20260505 11:06:32.035 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.25682% / 1.98682e-05% / 7.15256e-05% | 20260505 11:06:32.035 | INFO | Baz IO overall: | 20260505 11:06:32.035 | INFO | BytesWritten : 1603722224272 | 20260505 11:06:32.035 | INFO | NumEvents : 763596928520 | 20260505 11:06:33.648 | INFO | Analyzing chunk frames = [2719744,2720256) | 20260505 11:06:33.648 | INFO | MemInfo: { | 20260505 11:06:33.648 | INFO | "gpu" : " 17 GB", | 20260505 11:06:33.648 | INFO | "huge" : "", | 20260505 11:06:33.648 | INFO | "huge_shared" : " 27 GB", | 20260505 11:06:33.648 | INFO | "malloc" : " 14 GB", | 20260505 11:06:33.648 | INFO | "pinned" : " 29 GB" | 20260505 11:06:33.648 | INFO | } | 20260505 11:06:33.648 | INFO | Perf report for the last 5.14381 seconds: | 20260505 11:06:33.648 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:06:33.648 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:33.648 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:33.648 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:33.648 | INFO | ANALYSIS : 65.0%, 2.9, 100.0ms, 0 | 20260505 11:06:33.648 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:33.648 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260505 11:06:33.648 | INFO | PRE_HQ : 61.4%, 1.9, 59.2ms, 0 | 20260505 11:06:33.648 | INFO | BAZWRITER : 65.3%, 4.1, 138.0ms, 0 | 20260505 11:06:33.648 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 11:06:33.648 | INFO | Uploaded 36GiB to GPU in 2.20215s (16.3477GiB/s) | 20260505 11:06:33.648 | INFO | Downloaded 38GiB to GPU in 2.31051s (16.4466GiB/s) | 20260505 11:06:38.785 | INFO | Analyzing chunk frames = [2720256,2720768) | 20260505 11:06:38.785 | INFO | MemInfo: { | 20260505 11:06:38.786 | INFO | "gpu" : " 17 GB", | 20260505 11:06:38.786 | INFO | "huge" : "", | 20260505 11:06:38.786 | INFO | "huge_shared" : " 27 GB", | 20260505 11:06:38.786 | INFO | "malloc" : " 14 GB", | 20260505 11:06:38.786 | INFO | "pinned" : " 29 GB" | 20260505 11:06:38.786 | INFO | } | 20260505 11:06:38.786 | INFO | Perf report for the last 5.13766 seconds: | 20260505 11:06:38.786 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:06:38.786 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:38.786 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:38.786 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:38.786 | INFO | ANALYSIS : 64.3%, 3.0, 100.3ms, 0 | 20260505 11:06:38.786 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:38.786 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:38.786 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:06:38.786 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:38.786 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:38.786 | INFO | Uploaded 36GiB to GPU in 2.23407s (16.1141GiB/s) | 20260505 11:06:38.786 | INFO | Downloaded 34GiB to GPU in 2.07877s (16.3558GiB/s) | 20260505 11:06:43.926 | INFO | Analyzing chunk frames = [2720768,2721280) | 20260505 11:06:43.926 | INFO | MemInfo: { | 20260505 11:06:43.926 | INFO | "gpu" : " 17 GB", | 20260505 11:06:43.926 | INFO | "huge" : "", | 20260505 11:06:43.926 | INFO | "huge_shared" : " 27 GB", | 20260505 11:06:43.926 | INFO | "malloc" : " 14 GB", | 20260505 11:06:43.926 | INFO | "pinned" : " 29 GB" | 20260505 11:06:43.926 | INFO | } | 20260505 11:06:43.926 | INFO | Perf report for the last 5.14038 seconds: | 20260505 11:06:43.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:06:43.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:43.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:43.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:43.926 | INFO | ANALYSIS : 63.7%, 3.0, 99.2ms, 0 | 20260505 11:06:43.926 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:43.926 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:43.926 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 11:06:43.926 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:43.926 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:43.926 | INFO | Uploaded 36GiB to GPU in 2.22897s (16.151GiB/s) | 20260505 11:06:43.926 | INFO | Downloaded 34GiB to GPU in 2.12515s (15.9988GiB/s) | 20260505 11:06:49.058 | INFO | Analyzing chunk frames = [2721280,2721792) | 20260505 11:06:49.058 | INFO | MemInfo: { | 20260505 11:06:49.058 | INFO | "gpu" : " 17 GB", | 20260505 11:06:49.058 | INFO | "huge" : "", | 20260505 11:06:49.058 | INFO | "huge_shared" : " 27 GB", | 20260505 11:06:49.058 | INFO | "malloc" : " 14 GB", | 20260505 11:06:49.058 | INFO | "pinned" : " 29 GB" | 20260505 11:06:49.058 | INFO | } | 20260505 11:06:49.058 | INFO | Perf report for the last 5.13213 seconds: | 20260505 11:06:49.058 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:06:49.058 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:49.058 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:49.058 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:49.058 | INFO | ANALYSIS : 63.8%, 3.0, 99.1ms, 0 | 20260505 11:06:49.058 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:49.058 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:49.058 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 11:06:49.058 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:49.058 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:49.059 | INFO | Uploaded 36GiB to GPU in 2.22337s (16.1917GiB/s) | 20260505 11:06:49.059 | INFO | Downloaded 34GiB to GPU in 2.122s (16.0226GiB/s) | 20260505 11:06:54.199 | INFO | Analyzing chunk frames = [2721792,2722304) | 20260505 11:06:54.199 | INFO | MemInfo: { | 20260505 11:06:54.199 | INFO | "gpu" : " 17 GB", | 20260505 11:06:54.200 | INFO | "huge" : "", | 20260505 11:06:54.200 | INFO | "huge_shared" : " 27 GB", | 20260505 11:06:54.200 | INFO | "malloc" : " 14 GB", | 20260505 11:06:54.200 | INFO | "pinned" : " 29 GB" | 20260505 11:06:54.200 | INFO | } | 20260505 11:06:54.200 | INFO | Perf report for the last 5.14136 seconds: | 20260505 11:06:54.200 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:06:54.200 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:54.200 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:54.200 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:54.200 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 11:06:54.200 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:54.200 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:54.200 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:06:54.200 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:54.200 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:54.200 | INFO | Uploaded 36GiB to GPU in 2.23029s (16.1414GiB/s) | 20260505 11:06:54.200 | INFO | Downloaded 34GiB to GPU in 2.11248s (16.0948GiB/s) | 20260505 11:06:59.349 | INFO | Analyzing chunk frames = [2722304,2722816) | 20260505 11:06:59.349 | INFO | MemInfo: { | 20260505 11:06:59.349 | INFO | "gpu" : " 17 GB", | 20260505 11:06:59.349 | INFO | "huge" : "", | 20260505 11:06:59.349 | INFO | "huge_shared" : " 27 GB", | 20260505 11:06:59.349 | INFO | "malloc" : " 14 GB", | 20260505 11:06:59.349 | INFO | "pinned" : " 29 GB" | 20260505 11:06:59.349 | INFO | } | 20260505 11:06:59.349 | INFO | Perf report for the last 5.14978 seconds: | 20260505 11:06:59.349 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:06:59.349 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:59.349 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:59.349 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:59.350 | INFO | ANALYSIS : 63.5%, 3.0, 99.3ms, 0 | 20260505 11:06:59.350 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:59.350 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:59.350 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 11:06:59.350 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:59.350 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:06:59.350 | INFO | Uploaded 36GiB to GPU in 2.19701s (16.3859GiB/s) | 20260505 11:06:59.350 | INFO | Downloaded 34GiB to GPU in 2.11737s (16.0576GiB/s) | 20260505 11:07:04.490 | INFO | Analyzing chunk frames = [2722816,2723328) | 20260505 11:07:04.490 | INFO | MemInfo: { | 20260505 11:07:04.490 | INFO | "gpu" : " 17 GB", | 20260505 11:07:04.490 | INFO | "huge" : "", | 20260505 11:07:04.490 | INFO | "huge_shared" : " 27 GB", | 20260505 11:07:04.490 | INFO | "malloc" : " 14 GB", | 20260505 11:07:04.490 | INFO | "pinned" : " 29 GB" | 20260505 11:07:04.490 | INFO | } | 20260505 11:07:04.490 | INFO | Perf report for the last 5.14089 seconds: | 20260505 11:07:04.490 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:07:04.490 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:04.490 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:04.490 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:04.490 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 11:07:04.490 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:04.491 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:04.491 | INFO | PRE_HQ : 30.1%, 1.0, 15.7ms, 0 | 20260505 11:07:04.491 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:04.491 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:04.491 | INFO | Uploaded 36GiB to GPU in 2.18201s (16.4986GiB/s) | 20260505 11:07:04.491 | INFO | Downloaded 34GiB to GPU in 2.14805s (15.8283GiB/s) | 20260505 11:07:09.621 | INFO | Analyzing chunk frames = [2723328,2723840) | 20260505 11:07:09.621 | INFO | MemInfo: { | 20260505 11:07:09.621 | INFO | "gpu" : " 17 GB", | 20260505 11:07:09.621 | INFO | "huge" : "", | 20260505 11:07:09.621 | INFO | "huge_shared" : " 27 GB", | 20260505 11:07:09.621 | INFO | "malloc" : " 14 GB", | 20260505 11:07:09.621 | INFO | "pinned" : " 29 GB" | 20260505 11:07:09.621 | INFO | } | 20260505 11:07:09.621 | INFO | Perf report for the last 5.13076 seconds: | 20260505 11:07:09.621 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:07:09.621 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:09.621 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:09.621 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:09.621 | INFO | ANALYSIS : 64.0%, 3.0, 99.8ms, 0 | 20260505 11:07:09.621 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:09.621 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:09.621 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 11:07:09.621 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:09.621 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:09.621 | INFO | Uploaded 36GiB to GPU in 2.21564s (16.2481GiB/s) | 20260505 11:07:09.621 | INFO | Downloaded 34GiB to GPU in 2.08898s (16.2759GiB/s) | 20260505 11:07:09.754 | INFO | Starting a round of Baz IO | 20260505 11:07:13.009 | INFO | Populating RTMetrics report | 20260505 11:07:13.010 | INFO | RTMetrics report populated | 20260505 11:07:13.010 | INFO | Opening tmp RTMetrics json file | 20260505 11:07:13.010 | INFO | Serializing RTMetrics to json | 20260505 11:07:13.011 | INFO | Serialization done, swapping tmp file now... | 20260505 11:07:13.012 | INFO | RTMetrics done | 20260505 11:07:13.133 | INFO | Finished a round of Baz IO: Wrote 2.392 GiB in 3.379 seconds | 20260505 11:07:13.133 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.42% / 0.01% / 75.57% | 20260505 11:07:13.133 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.25144% / 1.58946e-05% / 8.74201e-05% | 20260505 11:07:13.133 | INFO | Baz IO overall: | 20260505 11:07:13.133 | INFO | BytesWritten : 1606290527179 | 20260505 11:07:13.133 | INFO | NumEvents : 764819754347 | 20260505 11:07:14.752 | INFO | Analyzing chunk frames = [2723840,2724352) | 20260505 11:07:14.752 | INFO | MemInfo: { | 20260505 11:07:14.752 | INFO | "gpu" : " 17 GB", | 20260505 11:07:14.752 | INFO | "huge" : "", | 20260505 11:07:14.752 | INFO | "huge_shared" : " 27 GB", | 20260505 11:07:14.752 | INFO | "malloc" : " 14 GB", | 20260505 11:07:14.752 | INFO | "pinned" : " 29 GB" | 20260505 11:07:14.752 | INFO | } | 20260505 11:07:14.753 | INFO | Perf report for the last 5.13133 seconds: | 20260505 11:07:14.753 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:07:14.753 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:14.753 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:14.753 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:14.753 | INFO | ANALYSIS : 64.9%, 2.9, 99.7ms, 0 | 20260505 11:07:14.753 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:14.753 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260505 11:07:14.753 | INFO | PRE_HQ : 62.4%, 1.8, 58.7ms, 0 | 20260505 11:07:14.753 | INFO | BAZWRITER : 65.1%, 4.2, 138.7ms, 0 | 20260505 11:07:14.753 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 11:07:14.753 | INFO | Uploaded 36GiB to GPU in 2.30521s (15.6168GiB/s) | 20260505 11:07:14.753 | INFO | Downloaded 38GiB to GPU in 2.39229s (15.8844GiB/s) | 20260505 11:07:19.899 | INFO | Analyzing chunk frames = [2724352,2724864) | 20260505 11:07:19.899 | INFO | MemInfo: { | 20260505 11:07:19.899 | INFO | "gpu" : " 17 GB", | 20260505 11:07:19.899 | INFO | "huge" : "", | 20260505 11:07:19.899 | INFO | "huge_shared" : " 27 GB", | 20260505 11:07:19.900 | INFO | "malloc" : " 14 GB", | 20260505 11:07:19.900 | INFO | "pinned" : " 29 GB" | 20260505 11:07:19.900 | INFO | } | 20260505 11:07:19.900 | INFO | Perf report for the last 5.14716 seconds: | 20260505 11:07:19.900 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:07:19.900 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:19.900 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:19.900 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:19.900 | INFO | ANALYSIS : 64.2%, 3.0, 100.4ms, 0 | 20260505 11:07:19.900 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:19.900 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:19.900 | INFO | PRE_HQ : 29.2%, 1.0, 15.3ms, 0 | 20260505 11:07:19.900 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:19.900 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:19.900 | INFO | Uploaded 36GiB to GPU in 2.21801s (16.2308GiB/s) | 20260505 11:07:19.900 | INFO | Downloaded 34GiB to GPU in 2.10403s (16.1595GiB/s) | 20260505 11:07:25.042 | INFO | Analyzing chunk frames = [2724864,2725376) | 20260505 11:07:25.042 | INFO | MemInfo: { | 20260505 11:07:25.042 | INFO | "gpu" : " 17 GB", | 20260505 11:07:25.042 | INFO | "huge" : "", | 20260505 11:07:25.042 | INFO | "huge_shared" : " 27 GB", | 20260505 11:07:25.042 | INFO | "malloc" : " 14 GB", | 20260505 11:07:25.042 | INFO | "pinned" : " 29 GB" | 20260505 11:07:25.042 | INFO | } | 20260505 11:07:25.042 | INFO | Perf report for the last 5.14246 seconds: | 20260505 11:07:25.042 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:07:25.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:25.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:25.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:25.042 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 11:07:25.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:25.042 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:25.042 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 11:07:25.042 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:25.042 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:25.042 | INFO | Uploaded 36GiB to GPU in 2.23s (16.1435GiB/s) | 20260505 11:07:25.042 | INFO | Downloaded 34GiB to GPU in 2.12602s (15.9923GiB/s) | 20260505 11:07:30.184 | INFO | Analyzing chunk frames = [2725376,2725888) | 20260505 11:07:30.184 | INFO | MemInfo: { | 20260505 11:07:30.184 | INFO | "gpu" : " 17 GB", | 20260505 11:07:30.184 | INFO | "huge" : "", | 20260505 11:07:30.184 | INFO | "huge_shared" : " 27 GB", | 20260505 11:07:30.185 | INFO | "malloc" : " 14 GB", | 20260505 11:07:30.185 | INFO | "pinned" : " 29 GB" | 20260505 11:07:30.185 | INFO | } | 20260505 11:07:30.185 | INFO | Perf report for the last 5.14249 seconds: | 20260505 11:07:30.185 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:07:30.185 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:30.185 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:30.185 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:30.185 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 11:07:30.185 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:30.185 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:30.185 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 11:07:30.185 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:30.185 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:30.185 | INFO | Uploaded 36GiB to GPU in 2.21142s (16.2791GiB/s) | 20260505 11:07:30.185 | INFO | Downloaded 34GiB to GPU in 2.13055s (15.9583GiB/s) | 20260505 11:07:35.318 | INFO | Analyzing chunk frames = [2725888,2726400) | 20260505 11:07:35.318 | INFO | MemInfo: { | 20260505 11:07:35.318 | INFO | "gpu" : " 17 GB", | 20260505 11:07:35.318 | INFO | "huge" : "", | 20260505 11:07:35.318 | INFO | "huge_shared" : " 27 GB", | 20260505 11:07:35.318 | INFO | "malloc" : " 14 GB", | 20260505 11:07:35.318 | INFO | "pinned" : " 29 GB" | 20260505 11:07:35.318 | INFO | } | 20260505 11:07:35.318 | INFO | Perf report for the last 5.13333 seconds: | 20260505 11:07:35.318 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:07:35.318 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:35.318 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:35.318 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:35.318 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 11:07:35.318 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:35.318 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:35.318 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:07:35.318 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:35.318 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:35.318 | INFO | Uploaded 36GiB to GPU in 2.14856s (16.7554GiB/s) | 20260505 11:07:35.318 | INFO | Downloaded 34GiB to GPU in 2.12286s (16.0161GiB/s) | 20260505 11:07:40.458 | INFO | Analyzing chunk frames = [2726400,2726912) | 20260505 11:07:40.458 | INFO | MemInfo: { | 20260505 11:07:40.458 | INFO | "gpu" : " 17 GB", | 20260505 11:07:40.458 | INFO | "huge" : "", | 20260505 11:07:40.458 | INFO | "huge_shared" : " 27 GB", | 20260505 11:07:40.458 | INFO | "malloc" : " 14 GB", | 20260505 11:07:40.458 | INFO | "pinned" : " 29 GB" | 20260505 11:07:40.458 | INFO | } | 20260505 11:07:40.458 | INFO | Perf report for the last 5.14041 seconds: | 20260505 11:07:40.458 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:07:40.458 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:40.458 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:40.458 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:40.458 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 11:07:40.458 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:40.458 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:40.459 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 11:07:40.459 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:40.459 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:40.459 | INFO | Uploaded 36GiB to GPU in 2.20977s (16.2913GiB/s) | 20260505 11:07:40.459 | INFO | Downloaded 34GiB to GPU in 2.10729s (16.1345GiB/s) | 20260505 11:07:45.599 | INFO | Analyzing chunk frames = [2726912,2727424) | 20260505 11:07:45.599 | INFO | MemInfo: { | 20260505 11:07:45.599 | INFO | "gpu" : " 17 GB", | 20260505 11:07:45.599 | INFO | "huge" : "", | 20260505 11:07:45.599 | INFO | "huge_shared" : " 27 GB", | 20260505 11:07:45.599 | INFO | "malloc" : " 14 GB", | 20260505 11:07:45.599 | INFO | "pinned" : " 29 GB" | 20260505 11:07:45.599 | INFO | } | 20260505 11:07:45.599 | INFO | Perf report for the last 5.14065 seconds: | 20260505 11:07:45.599 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:07:45.599 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:45.599 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:45.599 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:45.599 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 11:07:45.599 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:45.599 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:45.599 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 11:07:45.599 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:45.599 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:45.599 | INFO | Uploaded 36GiB to GPU in 2.19034s (16.4358GiB/s) | 20260505 11:07:45.599 | INFO | Downloaded 34GiB to GPU in 2.1399s (15.8886GiB/s) | 20260505 11:07:50.740 | INFO | Analyzing chunk frames = [2727424,2727936) | 20260505 11:07:50.740 | INFO | MemInfo: { | 20260505 11:07:50.740 | INFO | "gpu" : " 17 GB", | 20260505 11:07:50.740 | INFO | "huge" : "", | 20260505 11:07:50.740 | INFO | "huge_shared" : " 27 GB", | 20260505 11:07:50.740 | INFO | "malloc" : " 14 GB", | 20260505 11:07:50.740 | INFO | "pinned" : " 29 GB" | 20260505 11:07:50.740 | INFO | } | 20260505 11:07:50.740 | INFO | Perf report for the last 5.14076 seconds: | 20260505 11:07:50.740 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:07:50.740 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:50.740 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:50.740 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:50.740 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 11:07:50.740 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:50.740 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:50.740 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 11:07:50.740 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:50.740 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:50.740 | INFO | Uploaded 36GiB to GPU in 2.2132s (16.266GiB/s) | 20260505 11:07:50.740 | INFO | Downloaded 34GiB to GPU in 2.09109s (16.2595GiB/s) | 20260505 11:07:50.740 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 11:07:50.740 | INFO | Total mem usage: 15.3 GiB | 20260505 11:07:50.740 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 11:07:50.740 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.71%) | 20260505 11:07:50.740 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 11:07:50.740 | INFO | High Water Breakdown: | 20260505 11:07:50.740 | INFO | 61.1% : 8.77 GiB ( 92.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 11:07:50.740 | INFO | 38.8% : 5.57 GiB ( 83.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 11:07:50.740 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 11:07:50.740 | INFO | ------------------------------------------ | 20260505 11:07:50.741 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:07:50.741 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 11:07:50.741 | INFO | Total mem usage: 26.5 GiB | 20260505 11:07:50.741 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 11:07:50.741 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 11:07:50.741 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 11:07:50.741 | INFO | High Water Breakdown: | 20260505 11:07:50.741 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:07:50.741 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:07:50.741 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:07:50.741 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:07:50.741 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:07:50.741 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:07:50.741 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 11:07:50.741 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 11:07:50.741 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:07:50.741 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 11:07:50.741 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 11:07:50.741 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 11:07:50.741 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 11:07:50.741 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 11:07:50.741 | INFO | ------------------------------------------ | 20260505 11:07:50.741 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:07:50.741 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 11:07:50.741 | INFO | Total mem usage: 25.6 GiB | 20260505 11:07:50.741 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 11:07:50.741 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 11:07:50.741 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 11:07:50.741 | INFO | High Water Breakdown: | 20260505 11:07:50.741 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 11:07:50.741 | INFO | ------------------------------------------ | 20260505 11:07:50.741 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:07:50.741 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 11:07:50.741 | INFO | Total mem usage: 15.7 GiB | 20260505 11:07:50.741 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 11:07:50.741 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 11:07:50.741 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 11:07:50.741 | INFO | High Water Breakdown: | 20260505 11:07:50.741 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 11:07:50.741 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 11:07:50.741 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 11:07:50.741 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 11:07:50.741 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 11:07:50.741 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 11:07:50.741 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 11:07:50.741 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 11:07:50.741 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 11:07:50.741 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:07:50.741 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 11:07:50.741 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:07:50.741 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 11:07:50.741 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 11:07:50.741 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 11:07:50.741 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 11:07:50.741 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:07:50.741 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 11:07:50.741 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:07:50.741 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:07:50.741 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:07:50.741 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 11:07:50.741 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 11:07:50.741 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:07:50.741 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 11:07:50.741 | INFO | ------------------------------------------ | 20260505 11:07:50.741 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:07:50.741 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 11:07:50.741 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:07:50.741 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:07:50.741 | INFO | PerfReport: Full Execution -- 3.22e+01 / 7.76e+01, 1.38e+01 / 1.84e+00 | 20260505 11:07:50.741 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 1.02e-01, 1.05e-02 / 4.81e-03 | 20260505 11:07:50.741 | INFO | PerfReport: 11.69%, Baseline -- 3.77e+00 / 4.45e+00, 6.09e-01 / 2.78e-01 | 20260505 11:07:50.741 | INFO | PerfReport: 3.30%, Histogram -- 1.07e+00 / 1.48e+00, 2.05e-01 / 1.06e-01 | 20260505 11:07:50.741 | INFO | PerfReport: 15.70%, DME -- 4.05e+01 / 4.46e+01, 4.35e+00 / 2.13e+00 | 20260505 11:07:50.741 | INFO | PerfReport: 56.65%, FrameLabeling -- 1.83e+01 / 2.22e+01, 1.75e+00 / 2.12e+00 | 20260505 11:07:50.741 | INFO | PerfReport: 7.34%, PulseAccumulating -- 2.36e+00 / 2.82e+00, 4.12e-01 / 2.43e-01 | 20260505 11:07:50.741 | INFO | PerfReport: 3.72%, Metrics -- 1.20e+00 / 1.47e+00, 3.49e-01 / 2.30e-01 | 20260505 11:07:50.741 | INFO | PerfReport: 1.31%, PulseSerializing -- 4.22e-01 / 4.64e-01, 7.86e-02 / 3.92e-02 | 20260505 11:07:50.741 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 11:07:50.741 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:07:50.741 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:07:50.741 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.48e+01 / 0.00e+00 | 20260505 11:07:50.741 | INFO | PerfReport: 22.74%, Upload -- 2.31e+01 / 0.00e+00, 8.40e+00 / 0.00e+00 | 20260505 11:07:50.741 | INFO | PerfReport: 22.02%, Download -- 2.23e+01 / 0.00e+00, 8.38e+00 / 0.00e+00 | 20260505 11:07:50.741 | INFO | PerfReport: 31.77%, Compute -- 3.22e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 11:07:50.741 | INFO | PerfReport: 2.05%, LockUpload -- 2.08e+00 / 0.00e+00, 7.63e+00 / 0.00e+00 | 20260505 11:07:50.741 | INFO | PerfReport: 1.18%, LockDownload -- 1.20e+00 / 0.00e+00, 2.72e+00 / 0.00e+00 | 20260505 11:07:50.741 | INFO | PerfReport: 20.22%, LockCompute -- 2.05e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260505 11:07:50.741 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.36e-02 / 0.00e+00 | 20260505 11:07:50.878 | INFO | Starting a round of Baz IO | 20260505 11:07:54.140 | INFO | Populating RTMetrics report | 20260505 11:07:54.141 | INFO | RTMetrics report populated | 20260505 11:07:54.141 | INFO | Opening tmp RTMetrics json file | 20260505 11:07:54.142 | INFO | Serializing RTMetrics to json | 20260505 11:07:54.143 | INFO | Serialization done, swapping tmp file now... | 20260505 11:07:54.144 | INFO | RTMetrics done | 20260505 11:07:54.272 | INFO | Finished a round of Baz IO: Wrote 2.39 GiB in 3.393 seconds | 20260505 11:07:54.272 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.41% / 0.01% / 75.58% | 20260505 11:07:54.272 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.24444% / 1.98682e-05% / 5.16574e-05% | 20260505 11:07:54.272 | INFO | Baz IO overall: | 20260505 11:07:54.272 | INFO | BytesWritten : 1608856801072 | 20260505 11:07:54.272 | INFO | NumEvents : 766041043629 | 20260505 11:07:55.875 | INFO | Analyzing chunk frames = [2727936,2728448) | 20260505 11:07:55.875 | INFO | MemInfo: { | 20260505 11:07:55.875 | INFO | "gpu" : " 17 GB", | 20260505 11:07:55.875 | INFO | "huge" : "", | 20260505 11:07:55.875 | INFO | "huge_shared" : " 27 GB", | 20260505 11:07:55.875 | INFO | "malloc" : " 14 GB", | 20260505 11:07:55.875 | INFO | "pinned" : " 29 GB" | 20260505 11:07:55.875 | INFO | } | 20260505 11:07:55.876 | INFO | Perf report for the last 5.13568 seconds: | 20260505 11:07:55.876 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:07:55.876 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:55.876 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:55.876 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:55.876 | INFO | ANALYSIS : 65.1%, 2.9, 100.3ms, 0 | 20260505 11:07:55.876 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:07:55.876 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260505 11:07:55.876 | INFO | PRE_HQ : 61.5%, 1.8, 58.9ms, 0 | 20260505 11:07:55.876 | INFO | BAZWRITER : 65.4%, 4.1, 137.5ms, 0 | 20260505 11:07:55.876 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 11:07:55.876 | INFO | Uploaded 36GiB to GPU in 2.22973s (16.1455GiB/s) | 20260505 11:07:55.876 | INFO | Downloaded 38GiB to GPU in 2.2941s (16.5643GiB/s) | 20260505 11:08:01.010 | INFO | Analyzing chunk frames = [2728448,2728960) | 20260505 11:08:01.010 | INFO | MemInfo: { | 20260505 11:08:01.010 | INFO | "gpu" : " 17 GB", | 20260505 11:08:01.010 | INFO | "huge" : "", | 20260505 11:08:01.010 | INFO | "huge_shared" : " 27 GB", | 20260505 11:08:01.010 | INFO | "malloc" : " 14 GB", | 20260505 11:08:01.011 | INFO | "pinned" : " 29 GB" | 20260505 11:08:01.011 | INFO | } | 20260505 11:08:01.011 | INFO | Perf report for the last 5.13504 seconds: | 20260505 11:08:01.011 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:08:01.011 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:01.011 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:01.011 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:01.011 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 11:08:01.011 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:01.011 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:01.011 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 11:08:01.011 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:01.011 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:01.011 | INFO | Uploaded 36GiB to GPU in 2.22938s (16.148GiB/s) | 20260505 11:08:01.011 | INFO | Downloaded 34GiB to GPU in 2.08809s (16.2828GiB/s) | 20260505 11:08:06.160 | INFO | Analyzing chunk frames = [2728960,2729472) | 20260505 11:08:06.160 | INFO | MemInfo: { | 20260505 11:08:06.160 | INFO | "gpu" : " 17 GB", | 20260505 11:08:06.160 | INFO | "huge" : "", | 20260505 11:08:06.160 | INFO | "huge_shared" : " 27 GB", | 20260505 11:08:06.160 | INFO | "malloc" : " 14 GB", | 20260505 11:08:06.160 | INFO | "pinned" : " 29 GB" | 20260505 11:08:06.160 | INFO | } | 20260505 11:08:06.160 | INFO | Perf report for the last 5.14952 seconds: | 20260505 11:08:06.160 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:08:06.160 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:06.160 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:06.160 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:06.160 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 11:08:06.160 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:06.160 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:06.160 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 11:08:06.160 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:06.160 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:06.160 | INFO | Uploaded 36GiB to GPU in 2.16262s (16.6465GiB/s) | 20260505 11:08:06.160 | INFO | Downloaded 34GiB to GPU in 2.0636s (16.4761GiB/s) | 20260505 11:08:11.297 | INFO | Analyzing chunk frames = [2729472,2729984) | 20260505 11:08:11.297 | INFO | MemInfo: { | 20260505 11:08:11.297 | INFO | "gpu" : " 17 GB", | 20260505 11:08:11.297 | INFO | "huge" : "", | 20260505 11:08:11.297 | INFO | "huge_shared" : " 27 GB", | 20260505 11:08:11.297 | INFO | "malloc" : " 14 GB", | 20260505 11:08:11.297 | INFO | "pinned" : " 29 GB" | 20260505 11:08:11.297 | INFO | } | 20260505 11:08:11.297 | INFO | Perf report for the last 5.13715 seconds: | 20260505 11:08:11.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:08:11.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:11.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:11.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:11.297 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 11:08:11.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:11.297 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:11.297 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 11:08:11.297 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:11.297 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:11.297 | INFO | Uploaded 36GiB to GPU in 2.20352s (16.3375GiB/s) | 20260505 11:08:11.298 | INFO | Downloaded 34GiB to GPU in 2.12948s (15.9663GiB/s) | 20260505 11:08:16.435 | INFO | Analyzing chunk frames = [2729984,2730496) | 20260505 11:08:16.435 | INFO | MemInfo: { | 20260505 11:08:16.435 | INFO | "gpu" : " 17 GB", | 20260505 11:08:16.435 | INFO | "huge" : "", | 20260505 11:08:16.435 | INFO | "huge_shared" : " 27 GB", | 20260505 11:08:16.435 | INFO | "malloc" : " 14 GB", | 20260505 11:08:16.435 | INFO | "pinned" : " 29 GB" | 20260505 11:08:16.436 | INFO | } | 20260505 11:08:16.436 | INFO | Perf report for the last 5.13829 seconds: | 20260505 11:08:16.436 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:08:16.436 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:16.436 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:16.436 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:16.436 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 11:08:16.436 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:16.436 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:16.436 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 11:08:16.436 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:16.436 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:16.436 | INFO | Uploaded 36GiB to GPU in 2.1832s (16.4895GiB/s) | 20260505 11:08:16.436 | INFO | Downloaded 34GiB to GPU in 2.12799s (15.9775GiB/s) | 20260505 11:08:21.569 | INFO | Analyzing chunk frames = [2730496,2731008) | 20260505 11:08:21.569 | INFO | MemInfo: { | 20260505 11:08:21.569 | INFO | "gpu" : " 17 GB", | 20260505 11:08:21.569 | INFO | "huge" : "", | 20260505 11:08:21.569 | INFO | "huge_shared" : " 27 GB", | 20260505 11:08:21.569 | INFO | "malloc" : " 14 GB", | 20260505 11:08:21.569 | INFO | "pinned" : " 29 GB" | 20260505 11:08:21.569 | INFO | } | 20260505 11:08:21.570 | INFO | Perf report for the last 5.13395 seconds: | 20260505 11:08:21.570 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:08:21.570 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:21.570 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:21.570 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:21.570 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 11:08:21.570 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:21.570 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:21.570 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 11:08:21.570 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:21.570 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:21.570 | INFO | Uploaded 36GiB to GPU in 2.16381s (16.6374GiB/s) | 20260505 11:08:21.570 | INFO | Downloaded 34GiB to GPU in 2.12015s (16.0366GiB/s) | 20260505 11:08:26.707 | INFO | Analyzing chunk frames = [2731008,2731520) | 20260505 11:08:26.707 | INFO | MemInfo: { | 20260505 11:08:26.707 | INFO | "gpu" : " 17 GB", | 20260505 11:08:26.707 | INFO | "huge" : "", | 20260505 11:08:26.707 | INFO | "huge_shared" : " 27 GB", | 20260505 11:08:26.707 | INFO | "malloc" : " 14 GB", | 20260505 11:08:26.707 | INFO | "pinned" : " 29 GB" | 20260505 11:08:26.707 | INFO | } | 20260505 11:08:26.707 | INFO | Perf report for the last 5.13781 seconds: | 20260505 11:08:26.707 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:08:26.707 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:26.707 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:26.707 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:26.707 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 11:08:26.707 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:26.707 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:26.707 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 11:08:26.707 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:26.708 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:26.708 | INFO | Uploaded 36GiB to GPU in 2.18127s (16.5041GiB/s) | 20260505 11:08:26.708 | INFO | Downloaded 34GiB to GPU in 2.13365s (15.9351GiB/s) | 20260505 11:08:31.855 | INFO | Analyzing chunk frames = [2731520,2732032) | 20260505 11:08:31.855 | INFO | MemInfo: { | 20260505 11:08:31.855 | INFO | "gpu" : " 17 GB", | 20260505 11:08:31.855 | INFO | "huge" : "", | 20260505 11:08:31.855 | INFO | "huge_shared" : " 27 GB", | 20260505 11:08:31.855 | INFO | "malloc" : " 14 GB", | 20260505 11:08:31.855 | INFO | "pinned" : " 29 GB" | 20260505 11:08:31.855 | INFO | } | 20260505 11:08:31.855 | INFO | Perf report for the last 5.14765 seconds: | 20260505 11:08:31.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:08:31.855 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:31.855 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:31.855 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:31.855 | INFO | ANALYSIS : 64.1%, 3.0, 100.2ms, 0 | 20260505 11:08:31.855 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:31.855 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:31.855 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 11:08:31.855 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:31.855 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:31.855 | INFO | Uploaded 36GiB to GPU in 2.24367s (16.0451GiB/s) | 20260505 11:08:31.855 | INFO | Downloaded 34GiB to GPU in 2.10594s (16.1448GiB/s) | 20260505 11:08:31.983 | INFO | Starting a round of Baz IO | 20260505 11:08:35.235 | INFO | Populating RTMetrics report | 20260505 11:08:35.235 | INFO | RTMetrics report populated | 20260505 11:08:35.235 | INFO | Opening tmp RTMetrics json file | 20260505 11:08:35.236 | INFO | Serializing RTMetrics to json | 20260505 11:08:35.237 | INFO | Serialization done, swapping tmp file now... | 20260505 11:08:35.237 | INFO | RTMetrics done | 20260505 11:08:35.378 | INFO | Finished a round of Baz IO: Wrote 2.386 GiB in 3.394 seconds | 20260505 11:08:35.378 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.40% / 0.01% / 75.59% | 20260505 11:08:35.378 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.23579% / 1.19209e-05% / 3.97364e-05% | 20260505 11:08:35.378 | INFO | Baz IO overall: | 20260505 11:08:35.378 | INFO | BytesWritten : 1611419155445 | 20260505 11:08:35.378 | INFO | NumEvents : 767259446899 | 20260505 11:08:36.998 | INFO | Analyzing chunk frames = [2732032,2732544) | 20260505 11:08:36.998 | INFO | MemInfo: { | 20260505 11:08:36.998 | INFO | "gpu" : " 17 GB", | 20260505 11:08:36.998 | INFO | "huge" : "", | 20260505 11:08:36.998 | INFO | "huge_shared" : " 27 GB", | 20260505 11:08:36.998 | INFO | "malloc" : " 14 GB", | 20260505 11:08:36.998 | INFO | "pinned" : " 29 GB" | 20260505 11:08:36.998 | INFO | } | 20260505 11:08:36.998 | INFO | Perf report for the last 5.14282 seconds: | 20260505 11:08:36.998 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:08:36.998 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:36.998 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:36.998 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:36.998 | INFO | ANALYSIS : 64.6%, 2.9, 99.6ms, 0 | 20260505 11:08:36.998 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:36.998 | INFO | RT_METRICS : 45.3%, 1.0, 23.8ms, 0 | 20260505 11:08:36.998 | INFO | PRE_HQ : 61.1%, 1.9, 58.7ms, 0 | 20260505 11:08:36.998 | INFO | BAZWRITER : 65.3%, 4.1, 138.7ms, 0 | 20260505 11:08:36.998 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 11:08:36.998 | INFO | Uploaded 36GiB to GPU in 2.18029s (16.5116GiB/s) | 20260505 11:08:36.998 | INFO | Downloaded 38GiB to GPU in 2.28207s (16.6516GiB/s) | 20260505 11:08:42.130 | INFO | Analyzing chunk frames = [2732544,2733056) | 20260505 11:08:42.130 | INFO | MemInfo: { | 20260505 11:08:42.130 | INFO | "gpu" : " 17 GB", | 20260505 11:08:42.130 | INFO | "huge" : "", | 20260505 11:08:42.130 | INFO | "huge_shared" : " 27 GB", | 20260505 11:08:42.130 | INFO | "malloc" : " 14 GB", | 20260505 11:08:42.130 | INFO | "pinned" : " 29 GB" | 20260505 11:08:42.130 | INFO | } | 20260505 11:08:42.130 | INFO | Perf report for the last 5.13194 seconds: | 20260505 11:08:42.130 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:08:42.130 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:42.130 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:42.130 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:42.130 | INFO | ANALYSIS : 64.2%, 3.0, 100.0ms, 0 | 20260505 11:08:42.130 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:42.130 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:42.130 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:08:42.130 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:42.130 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:42.130 | INFO | Uploaded 36GiB to GPU in 2.23193s (16.1296GiB/s) | 20260505 11:08:42.130 | INFO | Downloaded 34GiB to GPU in 2.09888s (16.1991GiB/s) | 20260505 11:08:47.261 | INFO | Analyzing chunk frames = [2733056,2733568) | 20260505 11:08:47.261 | INFO | MemInfo: { | 20260505 11:08:47.261 | INFO | "gpu" : " 17 GB", | 20260505 11:08:47.261 | INFO | "huge" : "", | 20260505 11:08:47.261 | INFO | "huge_shared" : " 27 GB", | 20260505 11:08:47.261 | INFO | "malloc" : " 14 GB", | 20260505 11:08:47.261 | INFO | "pinned" : " 29 GB" | 20260505 11:08:47.261 | INFO | } | 20260505 11:08:47.261 | INFO | Perf report for the last 5.13159 seconds: | 20260505 11:08:47.261 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:08:47.261 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:47.261 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:47.261 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:47.261 | INFO | ANALYSIS : 63.1%, 3.0, 98.0ms, 0 | 20260505 11:08:47.261 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:47.262 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:47.262 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 11:08:47.262 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:47.262 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:47.262 | INFO | Uploaded 36GiB to GPU in 2.1636s (16.639GiB/s) | 20260505 11:08:47.262 | INFO | Downloaded 34GiB to GPU in 2.06701s (16.4489GiB/s) | 20260505 11:08:52.413 | INFO | Analyzing chunk frames = [2733568,2734080) | 20260505 11:08:52.413 | INFO | MemInfo: { | 20260505 11:08:52.413 | INFO | "gpu" : " 17 GB", | 20260505 11:08:52.413 | INFO | "huge" : "", | 20260505 11:08:52.413 | INFO | "huge_shared" : " 27 GB", | 20260505 11:08:52.413 | INFO | "malloc" : " 14 GB", | 20260505 11:08:52.413 | INFO | "pinned" : " 29 GB" | 20260505 11:08:52.413 | INFO | } | 20260505 11:08:52.413 | INFO | Perf report for the last 5.15192 seconds: | 20260505 11:08:52.413 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:08:52.413 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:52.413 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:52.413 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:52.413 | INFO | ANALYSIS : 62.9%, 3.0, 97.9ms, 0 | 20260505 11:08:52.413 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:52.413 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:52.413 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 11:08:52.413 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:52.413 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:52.414 | INFO | Uploaded 36GiB to GPU in 2.17672s (16.5387GiB/s) | 20260505 11:08:52.414 | INFO | Downloaded 34GiB to GPU in 2.08031s (16.3437GiB/s) | 20260505 11:08:57.548 | INFO | Analyzing chunk frames = [2734080,2734592) | 20260505 11:08:57.548 | INFO | MemInfo: { | 20260505 11:08:57.548 | INFO | "gpu" : " 17 GB", | 20260505 11:08:57.548 | INFO | "huge" : "", | 20260505 11:08:57.548 | INFO | "huge_shared" : " 27 GB", | 20260505 11:08:57.548 | INFO | "malloc" : " 14 GB", | 20260505 11:08:57.548 | INFO | "pinned" : " 29 GB" | 20260505 11:08:57.548 | INFO | } | 20260505 11:08:57.548 | INFO | Perf report for the last 5.13488 seconds: | 20260505 11:08:57.548 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:08:57.548 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:57.548 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:57.548 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:57.548 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 11:08:57.548 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:57.548 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:57.548 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 11:08:57.548 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:57.548 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:08:57.548 | INFO | Uploaded 36GiB to GPU in 2.18508s (16.4754GiB/s) | 20260505 11:08:57.549 | INFO | Downloaded 34GiB to GPU in 2.12486s (16.0011GiB/s) | 20260505 11:09:02.692 | INFO | Analyzing chunk frames = [2734592,2735104) | 20260505 11:09:02.692 | INFO | MemInfo: { | 20260505 11:09:02.692 | INFO | "gpu" : " 17 GB", | 20260505 11:09:02.692 | INFO | "huge" : "", | 20260505 11:09:02.692 | INFO | "huge_shared" : " 27 GB", | 20260505 11:09:02.692 | INFO | "malloc" : " 14 GB", | 20260505 11:09:02.692 | INFO | "pinned" : " 29 GB" | 20260505 11:09:02.692 | INFO | } | 20260505 11:09:02.692 | INFO | Perf report for the last 5.14368 seconds: | 20260505 11:09:02.692 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:09:02.692 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:02.692 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:02.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:02.692 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 11:09:02.692 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:02.692 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:02.692 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 11:09:02.692 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:02.692 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:02.692 | INFO | Uploaded 36GiB to GPU in 2.17583s (16.5454GiB/s) | 20260505 11:09:02.692 | INFO | Downloaded 34GiB to GPU in 2.12076s (16.032GiB/s) | 20260505 11:09:07.825 | INFO | Analyzing chunk frames = [2735104,2735616) | 20260505 11:09:07.825 | INFO | MemInfo: { | 20260505 11:09:07.825 | INFO | "gpu" : " 17 GB", | 20260505 11:09:07.825 | INFO | "huge" : "", | 20260505 11:09:07.825 | INFO | "huge_shared" : " 27 GB", | 20260505 11:09:07.825 | INFO | "malloc" : " 14 GB", | 20260505 11:09:07.825 | INFO | "pinned" : " 29 GB" | 20260505 11:09:07.825 | INFO | } | 20260505 11:09:07.825 | INFO | Perf report for the last 5.1333 seconds: | 20260505 11:09:07.825 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:09:07.825 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:07.825 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:07.825 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:07.825 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 11:09:07.825 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:07.825 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:07.825 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 11:09:07.825 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:07.825 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:07.826 | INFO | Uploaded 36GiB to GPU in 2.16908s (16.5969GiB/s) | 20260505 11:09:07.826 | INFO | Downloaded 34GiB to GPU in 2.13057s (15.9581GiB/s) | 20260505 11:09:12.965 | INFO | Analyzing chunk frames = [2735616,2736128) | 20260505 11:09:12.965 | INFO | MemInfo: { | 20260505 11:09:12.965 | INFO | "gpu" : " 17 GB", | 20260505 11:09:12.965 | INFO | "huge" : "", | 20260505 11:09:12.965 | INFO | "huge_shared" : " 27 GB", | 20260505 11:09:12.965 | INFO | "malloc" : " 14 GB", | 20260505 11:09:12.966 | INFO | "pinned" : " 29 GB" | 20260505 11:09:12.966 | INFO | } | 20260505 11:09:12.966 | INFO | Perf report for the last 5.14035 seconds: | 20260505 11:09:12.966 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:09:12.966 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:12.966 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:12.966 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:12.966 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 11:09:12.966 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:12.966 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:12.966 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 11:09:12.966 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:12.966 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:12.966 | INFO | Uploaded 36GiB to GPU in 2.23146s (16.1329GiB/s) | 20260505 11:09:12.966 | INFO | Downloaded 34GiB to GPU in 2.09826s (16.2039GiB/s) | 20260505 11:09:12.966 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 11:09:12.966 | INFO | Total mem usage: 15.3 GiB | 20260505 11:09:12.966 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 11:09:12.966 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -6.83%) | 20260505 11:09:12.966 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 11:09:12.966 | INFO | High Water Breakdown: | 20260505 11:09:12.966 | INFO | 61% : 8.75 GiB ( 92.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 11:09:12.966 | INFO | 38.8% : 5.57 GiB ( 83.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 11:09:12.966 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 11:09:12.966 | INFO | ------------------------------------------ | 20260505 11:09:12.966 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:09:12.966 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 11:09:12.967 | INFO | Total mem usage: 26.5 GiB | 20260505 11:09:12.967 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 11:09:12.967 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 11:09:12.967 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 11:09:12.967 | INFO | High Water Breakdown: | 20260505 11:09:12.967 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:09:12.967 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:09:12.967 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:09:12.967 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:09:12.967 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:09:12.967 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:09:12.967 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 11:09:12.967 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 11:09:12.967 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:09:12.967 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 11:09:12.967 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 11:09:12.967 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 11:09:12.967 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 11:09:12.967 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 11:09:12.967 | INFO | ------------------------------------------ | 20260505 11:09:12.967 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:09:12.967 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 11:09:12.967 | INFO | Total mem usage: 25.6 GiB | 20260505 11:09:12.967 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 11:09:12.967 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 11:09:12.967 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 11:09:12.967 | INFO | High Water Breakdown: | 20260505 11:09:12.967 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 11:09:12.967 | INFO | ------------------------------------------ | 20260505 11:09:12.967 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:09:12.967 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 11:09:12.967 | INFO | Total mem usage: 15.7 GiB | 20260505 11:09:12.967 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 11:09:12.967 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 11:09:12.967 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 11:09:12.967 | INFO | High Water Breakdown: | 20260505 11:09:12.967 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 11:09:12.967 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 11:09:12.967 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 11:09:12.967 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 11:09:12.967 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 11:09:12.967 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 11:09:12.967 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 11:09:12.967 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 11:09:12.967 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 11:09:12.967 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:09:12.967 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 11:09:12.967 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:09:12.967 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 11:09:12.967 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 11:09:12.967 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 11:09:12.967 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 11:09:12.967 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:09:12.967 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 11:09:12.967 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:09:12.967 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:09:12.967 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:09:12.967 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 11:09:12.967 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 11:09:12.967 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:09:12.967 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 11:09:12.967 | INFO | ------------------------------------------ | 20260505 11:09:12.967 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:09:12.967 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 11:09:12.967 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:09:12.967 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:09:12.967 | INFO | PerfReport: Full Execution -- 3.22e+01 / 7.69e+01, 1.36e+01 / 1.80e+00 | 20260505 11:09:12.967 | INFO | PerfReport: 0.29%, AggregateStats -- 9.32e-02 / 1.01e-01, 1.04e-02 / 3.25e-03 | 20260505 11:09:12.967 | INFO | PerfReport: 11.72%, Baseline -- 3.77e+00 / 4.50e+00, 5.99e-01 / 1.50e-01 | 20260505 11:09:12.967 | INFO | PerfReport: 3.31%, Histogram -- 1.07e+00 / 1.51e+00, 2.04e-01 / 6.19e-02 | 20260505 11:09:12.967 | INFO | PerfReport: 15.89%, DME -- 4.09e+01 / 4.56e+01, 4.60e+00 / 2.18e+00 | 20260505 11:09:12.967 | INFO | PerfReport: 56.33%, FrameLabeling -- 1.81e+01 / 2.03e+01, 1.61e+00 / 1.99e+00 | 20260505 11:09:12.967 | INFO | PerfReport: 7.43%, PulseAccumulating -- 2.39e+00 / 2.89e+00, 4.28e-01 / 2.65e-01 | 20260505 11:09:12.967 | INFO | PerfReport: 3.73%, Metrics -- 1.20e+00 / 1.49e+00, 3.43e-01 / 2.23e-01 | 20260505 11:09:12.967 | INFO | PerfReport: 1.31%, PulseSerializing -- 4.21e-01 / 4.70e-01, 7.76e-02 / 5.35e-02 | 20260505 11:09:12.967 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 11:09:12.967 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:09:12.967 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:09:12.967 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.46e+01 / 0.00e+00 | 20260505 11:09:12.967 | INFO | PerfReport: 22.59%, Upload -- 2.29e+01 / 0.00e+00, 8.16e+00 / 0.00e+00 | 20260505 11:09:12.967 | INFO | PerfReport: 21.92%, Download -- 2.22e+01 / 0.00e+00, 8.16e+00 / 0.00e+00 | 20260505 11:09:12.967 | INFO | PerfReport: 31.82%, Compute -- 3.22e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 11:09:12.967 | INFO | PerfReport: 2.01%, LockUpload -- 2.03e+00 / 0.00e+00, 7.56e+00 / 0.00e+00 | 20260505 11:09:12.967 | INFO | PerfReport: 1.21%, LockDownload -- 1.22e+00 / 0.00e+00, 2.68e+00 / 0.00e+00 | 20260505 11:09:12.967 | INFO | PerfReport: 20.45%, LockCompute -- 2.07e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260505 11:09:12.967 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 2.12e-02 / 0.00e+00 | 20260505 11:09:13.100 | INFO | Starting a round of Baz IO | 20260505 11:09:16.346 | INFO | Populating RTMetrics report | 20260505 11:09:16.346 | INFO | RTMetrics report populated | 20260505 11:09:16.346 | INFO | Opening tmp RTMetrics json file | 20260505 11:09:16.366 | INFO | Serializing RTMetrics to json | 20260505 11:09:16.417 | INFO | Serialization done, swapping tmp file now... | 20260505 11:09:16.444 | INFO | RTMetrics done | 20260505 11:09:16.524 | INFO | Finished a round of Baz IO: Wrote 2.39 GiB in 3.423 seconds | 20260505 11:09:16.524 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.39% / 0.01% / 75.60% | 20260505 11:09:16.524 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.23953% / 1.19209e-05% / 6.35783e-05% | 20260505 11:09:16.524 | INFO | Baz IO overall: | 20260505 11:09:16.524 | INFO | BytesWritten : 1613985319688 | 20260505 11:09:16.524 | INFO | NumEvents : 768481063803 | 20260505 11:09:18.112 | INFO | Analyzing chunk frames = [2736128,2736640) | 20260505 11:09:18.112 | INFO | MemInfo: { | 20260505 11:09:18.112 | INFO | "gpu" : " 17 GB", | 20260505 11:09:18.112 | INFO | "huge" : "", | 20260505 11:09:18.112 | INFO | "huge_shared" : " 27 GB", | 20260505 11:09:18.112 | INFO | "malloc" : " 14 GB", | 20260505 11:09:18.112 | INFO | "pinned" : " 29 GB" | 20260505 11:09:18.112 | INFO | } | 20260505 11:09:18.112 | INFO | Perf report for the last 5.14685 seconds: | 20260505 11:09:18.113 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:09:18.113 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:18.113 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:18.113 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:18.113 | INFO | ANALYSIS : 64.6%, 2.9, 98.2ms, 0 | 20260505 11:09:18.113 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:18.113 | INFO | RT_METRICS : 47.2%, 1.0, 24.8ms, 0 | 20260505 11:09:18.113 | INFO | PRE_HQ : 61.2%, 1.8, 58.6ms, 0 | 20260505 11:09:18.113 | INFO | BAZWRITER : 65.8%, 4.4, 148.5ms, 0 | 20260505 11:09:18.113 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 11:09:18.113 | INFO | Uploaded 36GiB to GPU in 2.19865s (16.3737GiB/s) | 20260505 11:09:18.113 | INFO | Downloaded 38GiB to GPU in 2.30966s (16.4526GiB/s) | 20260505 11:09:23.247 | INFO | Analyzing chunk frames = [2736640,2737152) | 20260505 11:09:23.247 | INFO | MemInfo: { | 20260505 11:09:23.247 | INFO | "gpu" : " 17 GB", | 20260505 11:09:23.247 | INFO | "huge" : "", | 20260505 11:09:23.247 | INFO | "huge_shared" : " 27 GB", | 20260505 11:09:23.247 | INFO | "malloc" : " 14 GB", | 20260505 11:09:23.247 | INFO | "pinned" : " 29 GB" | 20260505 11:09:23.247 | INFO | } | 20260505 11:09:23.247 | INFO | Perf report for the last 5.13495 seconds: | 20260505 11:09:23.247 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:09:23.247 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:23.247 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:23.247 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:23.248 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 11:09:23.248 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:23.248 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:23.248 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 11:09:23.248 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:23.248 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:23.248 | INFO | Uploaded 36GiB to GPU in 2.17492s (16.5523GiB/s) | 20260505 11:09:23.248 | INFO | Downloaded 34GiB to GPU in 2.04702s (16.6095GiB/s) | 20260505 11:09:28.382 | INFO | Analyzing chunk frames = [2737152,2737664) | 20260505 11:09:28.382 | INFO | MemInfo: { | 20260505 11:09:28.382 | INFO | "gpu" : " 17 GB", | 20260505 11:09:28.382 | INFO | "huge" : "", | 20260505 11:09:28.382 | INFO | "huge_shared" : " 27 GB", | 20260505 11:09:28.383 | INFO | "malloc" : " 14 GB", | 20260505 11:09:28.383 | INFO | "pinned" : " 29 GB" | 20260505 11:09:28.383 | INFO | } | 20260505 11:09:28.383 | INFO | Perf report for the last 5.13522 seconds: | 20260505 11:09:28.383 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:09:28.383 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:28.383 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:28.383 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:28.383 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260505 11:09:28.383 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:28.383 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:28.383 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 11:09:28.383 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:28.383 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:28.383 | INFO | Uploaded 36GiB to GPU in 2.21878s (16.2251GiB/s) | 20260505 11:09:28.383 | INFO | Downloaded 34GiB to GPU in 2.12991s (15.9631GiB/s) | 20260505 11:09:33.525 | INFO | Analyzing chunk frames = [2737664,2738176) | 20260505 11:09:33.525 | INFO | MemInfo: { | 20260505 11:09:33.525 | INFO | "gpu" : " 17 GB", | 20260505 11:09:33.525 | INFO | "huge" : "", | 20260505 11:09:33.525 | INFO | "huge_shared" : " 27 GB", | 20260505 11:09:33.525 | INFO | "malloc" : " 14 GB", | 20260505 11:09:33.525 | INFO | "pinned" : " 29 GB" | 20260505 11:09:33.525 | INFO | } | 20260505 11:09:33.525 | INFO | Perf report for the last 5.1421 seconds: | 20260505 11:09:33.525 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:09:33.525 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:33.525 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:33.525 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:33.525 | INFO | ANALYSIS : 62.9%, 3.0, 97.8ms, 0 | 20260505 11:09:33.525 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:33.525 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:33.525 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 11:09:33.525 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:33.525 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:33.525 | INFO | Uploaded 36GiB to GPU in 2.16846s (16.6016GiB/s) | 20260505 11:09:33.525 | INFO | Downloaded 34GiB to GPU in 2.08s (16.3461GiB/s) | 20260505 11:09:38.664 | INFO | Analyzing chunk frames = [2738176,2738688) | 20260505 11:09:38.664 | INFO | MemInfo: { | 20260505 11:09:38.664 | INFO | "gpu" : " 17 GB", | 20260505 11:09:38.664 | INFO | "huge" : "", | 20260505 11:09:38.664 | INFO | "huge_shared" : " 27 GB", | 20260505 11:09:38.664 | INFO | "malloc" : " 14 GB", | 20260505 11:09:38.664 | INFO | "pinned" : " 29 GB" | 20260505 11:09:38.664 | INFO | } | 20260505 11:09:38.664 | INFO | Perf report for the last 5.13918 seconds: | 20260505 11:09:38.664 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:09:38.664 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:38.664 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:38.664 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:38.664 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 11:09:38.664 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:38.664 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:38.664 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 11:09:38.664 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:38.664 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:38.664 | INFO | Uploaded 36GiB to GPU in 2.16096s (16.6593GiB/s) | 20260505 11:09:38.664 | INFO | Downloaded 34GiB to GPU in 2.123s (16.0151GiB/s) | 20260505 11:09:43.804 | INFO | Analyzing chunk frames = [2738688,2739200) | 20260505 11:09:43.804 | INFO | MemInfo: { | 20260505 11:09:43.804 | INFO | "gpu" : " 17 GB", | 20260505 11:09:43.804 | INFO | "huge" : "", | 20260505 11:09:43.804 | INFO | "huge_shared" : " 27 GB", | 20260505 11:09:43.804 | INFO | "malloc" : " 14 GB", | 20260505 11:09:43.805 | INFO | "pinned" : " 29 GB" | 20260505 11:09:43.805 | INFO | } | 20260505 11:09:43.805 | INFO | Perf report for the last 5.14075 seconds: | 20260505 11:09:43.805 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:09:43.805 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:43.805 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:43.805 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:43.805 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 11:09:43.805 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:43.805 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:43.805 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 11:09:43.805 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:43.805 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:43.805 | INFO | Uploaded 36GiB to GPU in 2.12694s (16.9258GiB/s) | 20260505 11:09:43.805 | INFO | Downloaded 34GiB to GPU in 2.10386s (16.1608GiB/s) | 20260505 11:09:48.944 | INFO | Analyzing chunk frames = [2739200,2739712) | 20260505 11:09:48.944 | INFO | MemInfo: { | 20260505 11:09:48.944 | INFO | "gpu" : " 17 GB", | 20260505 11:09:48.944 | INFO | "huge" : "", | 20260505 11:09:48.944 | INFO | "huge_shared" : " 27 GB", | 20260505 11:09:48.944 | INFO | "malloc" : " 14 GB", | 20260505 11:09:48.944 | INFO | "pinned" : " 29 GB" | 20260505 11:09:48.944 | INFO | } | 20260505 11:09:48.944 | INFO | Perf report for the last 5.1392 seconds: | 20260505 11:09:48.944 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:09:48.944 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:48.944 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:48.944 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:48.944 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 11:09:48.944 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:48.944 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:48.944 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 11:09:48.944 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:48.944 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:48.944 | INFO | Uploaded 36GiB to GPU in 2.15697s (16.6901GiB/s) | 20260505 11:09:48.944 | INFO | Downloaded 34GiB to GPU in 2.14224s (15.8713GiB/s) | 20260505 11:09:54.078 | INFO | Analyzing chunk frames = [2739712,2740224) | 20260505 11:09:54.078 | INFO | MemInfo: { | 20260505 11:09:54.078 | INFO | "gpu" : " 17 GB", | 20260505 11:09:54.078 | INFO | "huge" : "", | 20260505 11:09:54.078 | INFO | "huge_shared" : " 27 GB", | 20260505 11:09:54.079 | INFO | "malloc" : " 14 GB", | 20260505 11:09:54.079 | INFO | "pinned" : " 29 GB" | 20260505 11:09:54.079 | INFO | } | 20260505 11:09:54.079 | INFO | Perf report for the last 5.13475 seconds: | 20260505 11:09:54.079 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:09:54.079 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:54.079 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:54.079 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:54.079 | INFO | ANALYSIS : 64.0%, 3.0, 99.8ms, 0 | 20260505 11:09:54.079 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:54.079 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:54.079 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 11:09:54.079 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:54.079 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:54.079 | INFO | Uploaded 36GiB to GPU in 2.22807s (16.1575GiB/s) | 20260505 11:09:54.079 | INFO | Downloaded 34GiB to GPU in 2.09291s (16.2453GiB/s) | 20260505 11:09:54.211 | INFO | Starting a round of Baz IO | 20260505 11:09:57.467 | INFO | Populating RTMetrics report | 20260505 11:09:57.467 | INFO | RTMetrics report populated | 20260505 11:09:57.467 | INFO | Opening tmp RTMetrics json file | 20260505 11:09:57.507 | INFO | Serializing RTMetrics to json | 20260505 11:09:57.509 | INFO | Serialization done, swapping tmp file now... | 20260505 11:09:57.509 | INFO | RTMetrics done | 20260505 11:09:57.596 | INFO | Finished a round of Baz IO: Wrote 2.389 GiB in 3.385 seconds | 20260505 11:09:57.596 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.38% / 0.01% / 75.61% | 20260505 11:09:57.596 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.23299% / 0% / 9.13938e-05% | 20260505 11:09:57.596 | INFO | Baz IO overall: | 20260505 11:09:57.596 | INFO | BytesWritten : 1616550202392 | 20260505 11:09:57.596 | INFO | NumEvents : 769701471025 | 20260505 11:09:59.213 | INFO | Analyzing chunk frames = [2740224,2740736) | 20260505 11:09:59.213 | INFO | MemInfo: { | 20260505 11:09:59.213 | INFO | "gpu" : " 17 GB", | 20260505 11:09:59.213 | INFO | "huge" : "", | 20260505 11:09:59.213 | INFO | "huge_shared" : " 27 GB", | 20260505 11:09:59.213 | INFO | "malloc" : " 14 GB", | 20260505 11:09:59.213 | INFO | "pinned" : " 29 GB" | 20260505 11:09:59.213 | INFO | } | 20260505 11:09:59.213 | INFO | Perf report for the last 5.13439 seconds: | 20260505 11:09:59.213 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:09:59.213 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:59.213 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:59.213 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:59.213 | INFO | ANALYSIS : 64.9%, 2.9, 99.8ms, 0 | 20260505 11:09:59.213 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:09:59.213 | INFO | RT_METRICS : 46.2%, 1.0, 24.2ms, 0 | 20260505 11:09:59.213 | INFO | PRE_HQ : 62.4%, 1.8, 59.0ms, 0 | 20260505 11:09:59.213 | INFO | BAZWRITER : 65.2%, 4.2, 139.4ms, 0 | 20260505 11:09:59.213 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 11:09:59.214 | INFO | Uploaded 36GiB to GPU in 2.30117s (15.6442GiB/s) | 20260505 11:09:59.214 | INFO | Downloaded 38GiB to GPU in 2.38403s (15.9394GiB/s) | 20260505 11:10:04.365 | INFO | Analyzing chunk frames = [2740736,2741248) | 20260505 11:10:04.365 | INFO | MemInfo: { | 20260505 11:10:04.365 | INFO | "gpu" : " 17 GB", | 20260505 11:10:04.365 | INFO | "huge" : "", | 20260505 11:10:04.365 | INFO | "huge_shared" : " 27 GB", | 20260505 11:10:04.365 | INFO | "malloc" : " 14 GB", | 20260505 11:10:04.365 | INFO | "pinned" : " 29 GB" | 20260505 11:10:04.365 | INFO | } | 20260505 11:10:04.365 | INFO | Perf report for the last 5.15167 seconds: | 20260505 11:10:04.365 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:10:04.365 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:04.365 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:04.365 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:04.365 | INFO | ANALYSIS : 64.0%, 3.0, 100.2ms, 0 | 20260505 11:10:04.365 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:04.365 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:04.365 | INFO | PRE_HQ : 28.7%, 1.0, 15.0ms, 0 | 20260505 11:10:04.365 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:04.365 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:04.365 | INFO | Uploaded 36GiB to GPU in 2.23111s (16.1354GiB/s) | 20260505 11:10:04.365 | INFO | Downloaded 34GiB to GPU in 2.08513s (16.3059GiB/s) | 20260505 11:10:09.506 | INFO | Analyzing chunk frames = [2741248,2741760) | 20260505 11:10:09.506 | INFO | MemInfo: { | 20260505 11:10:09.506 | INFO | "gpu" : " 17 GB", | 20260505 11:10:09.507 | INFO | "huge" : "", | 20260505 11:10:09.507 | INFO | "huge_shared" : " 27 GB", | 20260505 11:10:09.507 | INFO | "malloc" : " 14 GB", | 20260505 11:10:09.507 | INFO | "pinned" : " 29 GB" | 20260505 11:10:09.507 | INFO | } | 20260505 11:10:09.507 | INFO | Perf report for the last 5.14198 seconds: | 20260505 11:10:09.507 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:10:09.507 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:09.507 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:09.507 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:09.507 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 11:10:09.507 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:09.507 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:09.507 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:10:09.507 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:09.507 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:09.507 | INFO | Uploaded 36GiB to GPU in 2.21363s (16.2629GiB/s) | 20260505 11:10:09.507 | INFO | Downloaded 34GiB to GPU in 2.11722s (16.0588GiB/s) | 20260505 11:10:14.643 | INFO | Analyzing chunk frames = [2741760,2742272) | 20260505 11:10:14.643 | INFO | MemInfo: { | 20260505 11:10:14.643 | INFO | "gpu" : " 17 GB", | 20260505 11:10:14.643 | INFO | "huge" : "", | 20260505 11:10:14.643 | INFO | "huge_shared" : " 27 GB", | 20260505 11:10:14.643 | INFO | "malloc" : " 14 GB", | 20260505 11:10:14.643 | INFO | "pinned" : " 29 GB" | 20260505 11:10:14.643 | INFO | } | 20260505 11:10:14.643 | INFO | Perf report for the last 5.13618 seconds: | 20260505 11:10:14.643 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:10:14.643 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:14.643 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:14.643 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:14.643 | INFO | ANALYSIS : 63.6%, 3.0, 98.7ms, 0 | 20260505 11:10:14.643 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:14.643 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:14.643 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 11:10:14.643 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:14.643 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:14.643 | INFO | Uploaded 36GiB to GPU in 2.21683s (16.2394GiB/s) | 20260505 11:10:14.643 | INFO | Downloaded 34GiB to GPU in 2.13015s (15.9613GiB/s) | 20260505 11:10:19.774 | INFO | Analyzing chunk frames = [2742272,2742784) | 20260505 11:10:19.774 | INFO | MemInfo: { | 20260505 11:10:19.774 | INFO | "gpu" : " 17 GB", | 20260505 11:10:19.774 | INFO | "huge" : "", | 20260505 11:10:19.774 | INFO | "huge_shared" : " 27 GB", | 20260505 11:10:19.774 | INFO | "malloc" : " 14 GB", | 20260505 11:10:19.774 | INFO | "pinned" : " 29 GB" | 20260505 11:10:19.774 | INFO | } | 20260505 11:10:19.774 | INFO | Perf report for the last 5.1313 seconds: | 20260505 11:10:19.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:10:19.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:19.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:19.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:19.774 | INFO | ANALYSIS : 63.4%, 3.0, 97.9ms, 0 | 20260505 11:10:19.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:19.774 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:19.774 | INFO | PRE_HQ : 29.2%, 1.0, 15.0ms, 0 | 20260505 11:10:19.774 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:19.774 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:19.775 | INFO | Uploaded 36GiB to GPU in 2.16622s (16.6188GiB/s) | 20260505 11:10:19.775 | INFO | Downloaded 34GiB to GPU in 2.12439s (16.0046GiB/s) | 20260505 11:10:24.919 | INFO | Analyzing chunk frames = [2742784,2743296) | 20260505 11:10:24.919 | INFO | MemInfo: { | 20260505 11:10:24.919 | INFO | "gpu" : " 17 GB", | 20260505 11:10:24.920 | INFO | "huge" : "", | 20260505 11:10:24.920 | INFO | "huge_shared" : " 27 GB", | 20260505 11:10:24.920 | INFO | "malloc" : " 14 GB", | 20260505 11:10:24.920 | INFO | "pinned" : " 29 GB" | 20260505 11:10:24.920 | INFO | } | 20260505 11:10:24.920 | INFO | Perf report for the last 5.14541 seconds: | 20260505 11:10:24.920 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:10:24.920 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:24.920 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:24.920 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:24.920 | INFO | ANALYSIS : 63.5%, 3.0, 99.3ms, 0 | 20260505 11:10:24.920 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:24.920 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:24.920 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 11:10:24.920 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:24.920 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:24.920 | INFO | Uploaded 36GiB to GPU in 2.21257s (16.2706GiB/s) | 20260505 11:10:24.920 | INFO | Downloaded 34GiB to GPU in 2.10901s (16.1213GiB/s) | 20260505 11:10:30.061 | INFO | Analyzing chunk frames = [2743296,2743808) | 20260505 11:10:30.061 | INFO | MemInfo: { | 20260505 11:10:30.061 | INFO | "gpu" : " 17 GB", | 20260505 11:10:30.061 | INFO | "huge" : "", | 20260505 11:10:30.061 | INFO | "huge_shared" : " 27 GB", | 20260505 11:10:30.061 | INFO | "malloc" : " 14 GB", | 20260505 11:10:30.061 | INFO | "pinned" : " 29 GB" | 20260505 11:10:30.061 | INFO | } | 20260505 11:10:30.061 | INFO | Perf report for the last 5.14178 seconds: | 20260505 11:10:30.062 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:10:30.062 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:30.062 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:30.062 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:30.062 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 11:10:30.062 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:30.062 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:30.062 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 11:10:30.062 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:30.062 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:30.062 | INFO | Uploaded 36GiB to GPU in 2.17678s (16.5382GiB/s) | 20260505 11:10:30.062 | INFO | Downloaded 34GiB to GPU in 2.12054s (16.0336GiB/s) | 20260505 11:10:35.193 | INFO | Analyzing chunk frames = [2743808,2744320) | 20260505 11:10:35.193 | INFO | MemInfo: { | 20260505 11:10:35.193 | INFO | "gpu" : " 17 GB", | 20260505 11:10:35.193 | INFO | "huge" : "", | 20260505 11:10:35.193 | INFO | "huge_shared" : " 27 GB", | 20260505 11:10:35.193 | INFO | "malloc" : " 14 GB", | 20260505 11:10:35.193 | INFO | "pinned" : " 29 GB" | 20260505 11:10:35.193 | INFO | } | 20260505 11:10:35.193 | INFO | Perf report for the last 5.13179 seconds: | 20260505 11:10:35.193 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:10:35.193 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:35.193 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:35.193 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:35.193 | INFO | ANALYSIS : 64.0%, 3.0, 99.7ms, 0 | 20260505 11:10:35.193 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:35.193 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:35.193 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 11:10:35.193 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:35.193 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:35.194 | INFO | Uploaded 36GiB to GPU in 2.21194s (16.2753GiB/s) | 20260505 11:10:35.194 | INFO | Downloaded 34GiB to GPU in 2.08895s (16.2761GiB/s) | 20260505 11:10:35.194 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 11:10:35.194 | INFO | Total mem usage: 15.3 GiB | 20260505 11:10:35.194 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 11:10:35.194 | INFO | Recent High Water Marks Sum: 14.7 GiB (Savings: -4.36%) | 20260505 11:10:35.194 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 11:10:35.194 | INFO | High Water Breakdown: | 20260505 11:10:35.194 | INFO | 61.4% : 9.02 GiB ( 95.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 11:10:35.194 | INFO | 38.5% : 5.65 GiB ( 85% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 11:10:35.194 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 11:10:35.194 | INFO | ------------------------------------------ | 20260505 11:10:35.194 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:10:35.194 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 11:10:35.194 | INFO | Total mem usage: 26.5 GiB | 20260505 11:10:35.194 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 11:10:35.194 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 11:10:35.194 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 11:10:35.194 | INFO | High Water Breakdown: | 20260505 11:10:35.194 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:10:35.194 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:10:35.194 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:10:35.194 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:10:35.194 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:10:35.194 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:10:35.194 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 11:10:35.194 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 11:10:35.194 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:10:35.194 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 11:10:35.194 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 11:10:35.194 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 11:10:35.194 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 11:10:35.194 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 11:10:35.194 | INFO | ------------------------------------------ | 20260505 11:10:35.194 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:10:35.194 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 11:10:35.194 | INFO | Total mem usage: 25.6 GiB | 20260505 11:10:35.194 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 11:10:35.194 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 11:10:35.194 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 11:10:35.194 | INFO | High Water Breakdown: | 20260505 11:10:35.194 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 11:10:35.194 | INFO | ------------------------------------------ | 20260505 11:10:35.194 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:10:35.194 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 11:10:35.194 | INFO | Total mem usage: 15.7 GiB | 20260505 11:10:35.194 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 11:10:35.194 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 11:10:35.194 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 11:10:35.194 | INFO | High Water Breakdown: | 20260505 11:10:35.194 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 11:10:35.194 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 11:10:35.194 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 11:10:35.194 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 11:10:35.194 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 11:10:35.194 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 11:10:35.194 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 11:10:35.194 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 11:10:35.194 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 11:10:35.194 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:10:35.194 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 11:10:35.195 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:10:35.195 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 11:10:35.195 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 11:10:35.195 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 11:10:35.195 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 11:10:35.195 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:10:35.195 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 11:10:35.195 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:10:35.195 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:10:35.195 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:10:35.195 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 11:10:35.195 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 11:10:35.195 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:10:35.195 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 11:10:35.195 | INFO | ------------------------------------------ | 20260505 11:10:35.195 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:10:35.195 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 11:10:35.195 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:10:35.195 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:10:35.195 | INFO | PerfReport: Full Execution -- 3.22e+01 / 7.71e+01, 1.37e+01 / 1.74e+00 | 20260505 11:10:35.195 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 1.01e-01, 1.10e-02 / 4.39e-03 | 20260505 11:10:35.195 | INFO | PerfReport: 11.72%, Baseline -- 3.77e+00 / 4.43e+00, 6.10e-01 / 2.42e-01 | 20260505 11:10:35.195 | INFO | PerfReport: 3.31%, Histogram -- 1.07e+00 / 1.48e+00, 2.04e-01 / 8.93e-02 | 20260505 11:10:35.195 | INFO | PerfReport: 15.74%, DME -- 4.05e+01 / 4.47e+01, 4.28e+00 / 2.34e+00 | 20260505 11:10:35.195 | INFO | PerfReport: 56.53%, FrameLabeling -- 1.82e+01 / 2.15e+01, 1.67e+00 / 2.16e+00 | 20260505 11:10:35.195 | INFO | PerfReport: 7.40%, PulseAccumulating -- 2.38e+00 / 2.89e+00, 4.28e-01 / 2.14e-01 | 20260505 11:10:35.195 | INFO | PerfReport: 3.71%, Metrics -- 1.19e+00 / 1.51e+00, 3.44e-01 / 2.07e-01 | 20260505 11:10:35.195 | INFO | PerfReport: 1.30%, PulseSerializing -- 4.19e-01 / 4.69e-01, 7.69e-02 / 3.18e-02 | 20260505 11:10:35.195 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 11:10:35.195 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:10:35.195 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:10:35.195 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.47e+01 / 0.00e+00 | 20260505 11:10:35.195 | INFO | PerfReport: 22.64%, Upload -- 2.29e+01 / 0.00e+00, 8.15e+00 / 0.00e+00 | 20260505 11:10:35.195 | INFO | PerfReport: 22.01%, Download -- 2.23e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260505 11:10:35.195 | INFO | PerfReport: 31.84%, Compute -- 3.22e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 11:10:35.195 | INFO | PerfReport: 1.95%, LockUpload -- 1.98e+00 / 0.00e+00, 7.64e+00 / 0.00e+00 | 20260505 11:10:35.195 | INFO | PerfReport: 1.30%, LockDownload -- 1.32e+00 / 0.00e+00, 2.89e+00 / 0.00e+00 | 20260505 11:10:35.195 | INFO | PerfReport: 20.24%, LockCompute -- 2.05e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260505 11:10:35.195 | INFO | PerfReport: 0.01%, Misc -- 1.46e-02 / 0.00e+00, 2.56e-02 / 0.00e+00 | 20260505 11:10:35.328 | INFO | Starting a round of Baz IO | 20260505 11:10:38.595 | INFO | Populating RTMetrics report | 20260505 11:10:38.595 | INFO | RTMetrics report populated | 20260505 11:10:38.595 | INFO | Opening tmp RTMetrics json file | 20260505 11:10:38.596 | INFO | Serializing RTMetrics to json | 20260505 11:10:38.599 | INFO | Serialization done, swapping tmp file now... | 20260505 11:10:38.601 | INFO | RTMetrics done | 20260505 11:10:38.733 | INFO | Finished a round of Baz IO: Wrote 2.388 GiB in 3.405 seconds | 20260505 11:10:38.733 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.38% / 0.01% / 75.62% | 20260505 11:10:38.733 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.22858% / 1.19209e-05% / 8.74201e-05% | 20260505 11:10:38.733 | INFO | Baz IO overall: | 20260505 11:10:38.733 | INFO | BytesWritten : 1619114247261 | 20260505 11:10:38.733 | INFO | NumEvents : 770921025057 | 20260505 11:10:40.332 | INFO | Analyzing chunk frames = [2744320,2744832) | 20260505 11:10:40.332 | INFO | MemInfo: { | 20260505 11:10:40.332 | INFO | "gpu" : " 17 GB", | 20260505 11:10:40.332 | INFO | "huge" : "", | 20260505 11:10:40.332 | INFO | "huge_shared" : " 27 GB", | 20260505 11:10:40.332 | INFO | "malloc" : " 14 GB", | 20260505 11:10:40.332 | INFO | "pinned" : " 29 GB" | 20260505 11:10:40.332 | INFO | } | 20260505 11:10:40.332 | INFO | Perf report for the last 5.13875 seconds: | 20260505 11:10:40.332 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:10:40.332 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:40.332 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:40.332 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:40.332 | INFO | ANALYSIS : 65.1%, 2.9, 100.0ms, 0 | 20260505 11:10:40.332 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:40.332 | INFO | RT_METRICS : 45.5%, 1.0, 23.9ms, 0 | 20260505 11:10:40.332 | INFO | PRE_HQ : 61.4%, 1.8, 58.7ms, 0 | 20260505 11:10:40.332 | INFO | BAZWRITER : 65.5%, 4.1, 139.3ms, 0 | 20260505 11:10:40.332 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 11:10:40.332 | INFO | Uploaded 36GiB to GPU in 2.18234s (16.4961GiB/s) | 20260505 11:10:40.333 | INFO | Downloaded 38GiB to GPU in 2.31138s (16.4404GiB/s) | 20260505 11:10:45.476 | INFO | Analyzing chunk frames = [2744832,2745344) | 20260505 11:10:45.476 | INFO | MemInfo: { | 20260505 11:10:45.476 | INFO | "gpu" : " 17 GB", | 20260505 11:10:45.476 | INFO | "huge" : "", | 20260505 11:10:45.476 | INFO | "huge_shared" : " 27 GB", | 20260505 11:10:45.476 | INFO | "malloc" : " 14 GB", | 20260505 11:10:45.476 | INFO | "pinned" : " 29 GB" | 20260505 11:10:45.476 | INFO | } | 20260505 11:10:45.476 | INFO | Perf report for the last 5.14435 seconds: | 20260505 11:10:45.476 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:10:45.476 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:45.476 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:45.476 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:45.476 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 11:10:45.476 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:45.476 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:45.476 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:10:45.476 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:45.476 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:45.477 | INFO | Uploaded 36GiB to GPU in 2.19663s (16.3887GiB/s) | 20260505 11:10:45.477 | INFO | Downloaded 34GiB to GPU in 2.09363s (16.2398GiB/s) | 20260505 11:10:50.613 | INFO | Analyzing chunk frames = [2745344,2745856) | 20260505 11:10:50.613 | INFO | MemInfo: { | 20260505 11:10:50.613 | INFO | "gpu" : " 17 GB", | 20260505 11:10:50.613 | INFO | "huge" : "", | 20260505 11:10:50.613 | INFO | "huge_shared" : " 27 GB", | 20260505 11:10:50.613 | INFO | "malloc" : " 14 GB", | 20260505 11:10:50.613 | INFO | "pinned" : " 29 GB" | 20260505 11:10:50.613 | INFO | } | 20260505 11:10:50.613 | INFO | Perf report for the last 5.13672 seconds: | 20260505 11:10:50.613 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:10:50.613 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:50.613 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:50.613 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:50.613 | INFO | ANALYSIS : 63.4%, 3.0, 98.6ms, 0 | 20260505 11:10:50.613 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:50.613 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:50.613 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 11:10:50.613 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:50.613 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:50.613 | INFO | Uploaded 36GiB to GPU in 2.22859s (16.1537GiB/s) | 20260505 11:10:50.613 | INFO | Downloaded 34GiB to GPU in 2.1267s (15.9872GiB/s) | 20260505 11:10:55.757 | INFO | Analyzing chunk frames = [2745856,2746368) | 20260505 11:10:55.757 | INFO | MemInfo: { | 20260505 11:10:55.757 | INFO | "gpu" : " 17 GB", | 20260505 11:10:55.757 | INFO | "huge" : "", | 20260505 11:10:55.757 | INFO | "huge_shared" : " 27 GB", | 20260505 11:10:55.757 | INFO | "malloc" : " 14 GB", | 20260505 11:10:55.757 | INFO | "pinned" : " 29 GB" | 20260505 11:10:55.757 | INFO | } | 20260505 11:10:55.758 | INFO | Perf report for the last 5.14438 seconds: | 20260505 11:10:55.758 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:10:55.758 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:55.758 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:55.758 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:55.758 | INFO | ANALYSIS : 63.2%, 3.0, 98.4ms, 0 | 20260505 11:10:55.758 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:55.758 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:55.758 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 11:10:55.758 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:55.758 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:10:55.758 | INFO | Uploaded 36GiB to GPU in 2.20806s (16.3039GiB/s) | 20260505 11:10:55.758 | INFO | Downloaded 34GiB to GPU in 2.13134s (15.9524GiB/s) | 20260505 11:11:00.889 | INFO | Analyzing chunk frames = [2746368,2746880) | 20260505 11:11:00.889 | INFO | MemInfo: { | 20260505 11:11:00.889 | INFO | "gpu" : " 17 GB", | 20260505 11:11:00.889 | INFO | "huge" : "", | 20260505 11:11:00.889 | INFO | "huge_shared" : " 27 GB", | 20260505 11:11:00.889 | INFO | "malloc" : " 14 GB", | 20260505 11:11:00.889 | INFO | "pinned" : " 29 GB" | 20260505 11:11:00.889 | INFO | } | 20260505 11:11:00.889 | INFO | Perf report for the last 5.13186 seconds: | 20260505 11:11:00.889 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:11:00.889 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:00.889 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:00.889 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:00.889 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 11:11:00.890 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:00.890 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:00.890 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 11:11:00.890 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:00.890 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:00.890 | INFO | Uploaded 36GiB to GPU in 2.15788s (16.6831GiB/s) | 20260505 11:11:00.890 | INFO | Downloaded 34GiB to GPU in 2.12828s (15.9754GiB/s) | 20260505 11:11:06.026 | INFO | Analyzing chunk frames = [2746880,2747392) | 20260505 11:11:06.026 | INFO | MemInfo: { | 20260505 11:11:06.026 | INFO | "gpu" : " 17 GB", | 20260505 11:11:06.026 | INFO | "huge" : "", | 20260505 11:11:06.026 | INFO | "huge_shared" : " 27 GB", | 20260505 11:11:06.026 | INFO | "malloc" : " 14 GB", | 20260505 11:11:06.026 | INFO | "pinned" : " 29 GB" | 20260505 11:11:06.026 | INFO | } | 20260505 11:11:06.026 | INFO | Perf report for the last 5.13686 seconds: | 20260505 11:11:06.026 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:11:06.026 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:06.026 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:06.026 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:06.026 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 11:11:06.026 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:06.026 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:06.026 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 11:11:06.026 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:06.026 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:06.026 | INFO | Uploaded 36GiB to GPU in 2.22996s (16.1438GiB/s) | 20260505 11:11:06.027 | INFO | Downloaded 34GiB to GPU in 2.10289s (16.1682GiB/s) | 20260505 11:11:11.174 | INFO | Analyzing chunk frames = [2747392,2747904) | 20260505 11:11:11.174 | INFO | MemInfo: { | 20260505 11:11:11.174 | INFO | "gpu" : " 17 GB", | 20260505 11:11:11.174 | INFO | "huge" : "", | 20260505 11:11:11.174 | INFO | "huge_shared" : " 27 GB", | 20260505 11:11:11.174 | INFO | "malloc" : " 14 GB", | 20260505 11:11:11.174 | INFO | "pinned" : " 29 GB" | 20260505 11:11:11.175 | INFO | } | 20260505 11:11:11.175 | INFO | Perf report for the last 5.14831 seconds: | 20260505 11:11:11.175 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:11:11.175 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:11.175 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:11.175 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:11.175 | INFO | ANALYSIS : 63.4%, 3.0, 99.2ms, 0 | 20260505 11:11:11.175 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:11.175 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:11.175 | INFO | PRE_HQ : 29.7%, 1.0, 15.5ms, 0 | 20260505 11:11:11.175 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:11.175 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:11.175 | INFO | Uploaded 36GiB to GPU in 2.19745s (16.3826GiB/s) | 20260505 11:11:11.175 | INFO | Downloaded 34GiB to GPU in 2.15887s (15.749GiB/s) | 20260505 11:11:16.313 | INFO | Analyzing chunk frames = [2747904,2748416) | 20260505 11:11:16.313 | INFO | MemInfo: { | 20260505 11:11:16.313 | INFO | "gpu" : " 17 GB", | 20260505 11:11:16.313 | INFO | "huge" : "", | 20260505 11:11:16.313 | INFO | "huge_shared" : " 27 GB", | 20260505 11:11:16.313 | INFO | "malloc" : " 14 GB", | 20260505 11:11:16.313 | INFO | "pinned" : " 29 GB" | 20260505 11:11:16.313 | INFO | } | 20260505 11:11:16.313 | INFO | Perf report for the last 5.13886 seconds: | 20260505 11:11:16.313 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:11:16.313 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:16.314 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:16.314 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:16.314 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 11:11:16.314 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:16.314 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:16.314 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 11:11:16.314 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:16.314 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:16.314 | INFO | Uploaded 36GiB to GPU in 2.20117s (16.3549GiB/s) | 20260505 11:11:16.314 | INFO | Downloaded 34GiB to GPU in 2.115s (16.0757GiB/s) | 20260505 11:11:16.453 | INFO | Starting a round of Baz IO | 20260505 11:11:19.716 | INFO | Populating RTMetrics report | 20260505 11:11:19.716 | INFO | RTMetrics report populated | 20260505 11:11:19.716 | INFO | Opening tmp RTMetrics json file | 20260505 11:11:19.752 | INFO | Serializing RTMetrics to json | 20260505 11:11:19.776 | INFO | Serialization done, swapping tmp file now... | 20260505 11:11:19.781 | INFO | RTMetrics done | 20260505 11:11:19.843 | INFO | Finished a round of Baz IO: Wrote 2.391 GiB in 3.39 seconds | 20260505 11:11:19.843 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.37% / 0.01% / 75.62% | 20260505 11:11:19.843 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.23039% / 1.19209e-05% / 7.15256e-05% | 20260505 11:11:19.843 | INFO | Baz IO overall: | 20260505 11:11:19.843 | INFO | BytesWritten : 1621681500269 | 20260505 11:11:19.843 | INFO | NumEvents : 772143395430 | 20260505 11:11:21.454 | INFO | Analyzing chunk frames = [2748416,2748928) | 20260505 11:11:21.454 | INFO | MemInfo: { | 20260505 11:11:21.454 | INFO | "gpu" : " 17 GB", | 20260505 11:11:21.454 | INFO | "huge" : "", | 20260505 11:11:21.454 | INFO | "huge_shared" : " 27 GB", | 20260505 11:11:21.454 | INFO | "malloc" : " 14 GB", | 20260505 11:11:21.454 | INFO | "pinned" : " 29 GB" | 20260505 11:11:21.454 | INFO | } | 20260505 11:11:21.454 | INFO | Perf report for the last 5.14031 seconds: | 20260505 11:11:21.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:11:21.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:21.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:21.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:21.454 | INFO | ANALYSIS : 65.1%, 2.9, 100.4ms, 0 | 20260505 11:11:21.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:21.454 | INFO | RT_METRICS : 47.1%, 1.0, 24.7ms, 0 | 20260505 11:11:21.454 | INFO | PRE_HQ : 61.5%, 1.8, 58.9ms, 0 | 20260505 11:11:21.454 | INFO | BAZWRITER : 65.3%, 4.1, 138.2ms, 0 | 20260505 11:11:21.454 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260505 11:11:21.454 | INFO | Uploaded 36GiB to GPU in 2.22484s (16.1809GiB/s) | 20260505 11:11:21.454 | INFO | Downloaded 38GiB to GPU in 2.2988s (16.5304GiB/s) | 20260505 11:11:26.588 | INFO | Analyzing chunk frames = [2748928,2749440) | 20260505 11:11:26.588 | INFO | MemInfo: { | 20260505 11:11:26.588 | INFO | "gpu" : " 17 GB", | 20260505 11:11:26.588 | INFO | "huge" : "", | 20260505 11:11:26.588 | INFO | "huge_shared" : " 27 GB", | 20260505 11:11:26.588 | INFO | "malloc" : " 14 GB", | 20260505 11:11:26.588 | INFO | "pinned" : " 29 GB" | 20260505 11:11:26.588 | INFO | } | 20260505 11:11:26.588 | INFO | Perf report for the last 5.13437 seconds: | 20260505 11:11:26.588 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:11:26.588 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:26.588 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:26.588 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:26.588 | INFO | ANALYSIS : 64.3%, 3.0, 100.2ms, 0 | 20260505 11:11:26.588 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:26.588 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:26.588 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 11:11:26.588 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:26.588 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:26.588 | INFO | Uploaded 36GiB to GPU in 2.23043s (16.1404GiB/s) | 20260505 11:11:26.588 | INFO | Downloaded 34GiB to GPU in 2.08951s (16.2718GiB/s) | 20260505 11:11:31.727 | INFO | Analyzing chunk frames = [2749440,2749952) | 20260505 11:11:31.727 | INFO | MemInfo: { | 20260505 11:11:31.727 | INFO | "gpu" : " 17 GB", | 20260505 11:11:31.727 | INFO | "huge" : "", | 20260505 11:11:31.727 | INFO | "huge_shared" : " 27 GB", | 20260505 11:11:31.727 | INFO | "malloc" : " 14 GB", | 20260505 11:11:31.727 | INFO | "pinned" : " 29 GB" | 20260505 11:11:31.727 | INFO | } | 20260505 11:11:31.727 | INFO | Perf report for the last 5.13908 seconds: | 20260505 11:11:31.727 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:11:31.727 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:31.727 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:31.727 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:31.727 | INFO | ANALYSIS : 62.9%, 3.0, 97.5ms, 0 | 20260505 11:11:31.727 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:31.727 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:31.727 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:11:31.727 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:31.727 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:31.728 | INFO | Uploaded 36GiB to GPU in 2.14655s (16.7711GiB/s) | 20260505 11:11:31.728 | INFO | Downloaded 34GiB to GPU in 2.05943s (16.5094GiB/s) | 20260505 11:11:36.875 | INFO | Analyzing chunk frames = [2749952,2750464) | 20260505 11:11:36.875 | INFO | MemInfo: { | 20260505 11:11:36.875 | INFO | "gpu" : " 17 GB", | 20260505 11:11:36.875 | INFO | "huge" : "", | 20260505 11:11:36.875 | INFO | "huge_shared" : " 27 GB", | 20260505 11:11:36.875 | INFO | "malloc" : " 14 GB", | 20260505 11:11:36.875 | INFO | "pinned" : " 29 GB" | 20260505 11:11:36.875 | INFO | } | 20260505 11:11:36.875 | INFO | Perf report for the last 5.14789 seconds: | 20260505 11:11:36.875 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:11:36.875 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:36.875 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:36.875 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:36.875 | INFO | ANALYSIS : 63.4%, 3.0, 98.6ms, 0 | 20260505 11:11:36.875 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:36.875 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:36.875 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 11:11:36.875 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:36.875 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:36.875 | INFO | Uploaded 36GiB to GPU in 2.21439s (16.2573GiB/s) | 20260505 11:11:36.875 | INFO | Downloaded 34GiB to GPU in 2.13418s (15.9312GiB/s) | 20260505 11:11:42.011 | INFO | Analyzing chunk frames = [2750464,2750976) | 20260505 11:11:42.011 | INFO | MemInfo: { | 20260505 11:11:42.011 | INFO | "gpu" : " 17 GB", | 20260505 11:11:42.011 | INFO | "huge" : "", | 20260505 11:11:42.011 | INFO | "huge_shared" : " 27 GB", | 20260505 11:11:42.011 | INFO | "malloc" : " 14 GB", | 20260505 11:11:42.011 | INFO | "pinned" : " 29 GB" | 20260505 11:11:42.011 | INFO | } | 20260505 11:11:42.012 | INFO | Perf report for the last 5.1363 seconds: | 20260505 11:11:42.012 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:11:42.012 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:42.012 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:42.012 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:42.012 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 11:11:42.012 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:42.012 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:42.012 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:11:42.012 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:42.012 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:42.012 | INFO | Uploaded 36GiB to GPU in 2.15052s (16.7401GiB/s) | 20260505 11:11:42.012 | INFO | Downloaded 34GiB to GPU in 2.12483s (16.0013GiB/s) | 20260505 11:11:47.142 | INFO | Analyzing chunk frames = [2750976,2751488) | 20260505 11:11:47.142 | INFO | MemInfo: { | 20260505 11:11:47.142 | INFO | "gpu" : " 17 GB", | 20260505 11:11:47.142 | INFO | "huge" : "", | 20260505 11:11:47.142 | INFO | "huge_shared" : " 27 GB", | 20260505 11:11:47.142 | INFO | "malloc" : " 14 GB", | 20260505 11:11:47.142 | INFO | "pinned" : " 29 GB" | 20260505 11:11:47.142 | INFO | } | 20260505 11:11:47.142 | INFO | Perf report for the last 5.13092 seconds: | 20260505 11:11:47.142 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:11:47.142 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:47.142 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:47.142 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:47.142 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 11:11:47.142 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:47.143 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:47.143 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 11:11:47.143 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:47.143 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:47.143 | INFO | Uploaded 36GiB to GPU in 2.19933s (16.3686GiB/s) | 20260505 11:11:47.143 | INFO | Downloaded 34GiB to GPU in 2.1229s (16.0158GiB/s) | 20260505 11:11:52.278 | INFO | Analyzing chunk frames = [2751488,2752000) | 20260505 11:11:52.278 | INFO | MemInfo: { | 20260505 11:11:52.278 | INFO | "gpu" : " 17 GB", | 20260505 11:11:52.278 | INFO | "huge" : "", | 20260505 11:11:52.278 | INFO | "huge_shared" : " 27 GB", | 20260505 11:11:52.278 | INFO | "malloc" : " 14 GB", | 20260505 11:11:52.278 | INFO | "pinned" : " 29 GB" | 20260505 11:11:52.278 | INFO | } | 20260505 11:11:52.278 | INFO | Perf report for the last 5.13559 seconds: | 20260505 11:11:52.278 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:11:52.278 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:52.278 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:52.278 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:52.278 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 11:11:52.278 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:52.278 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:52.278 | INFO | PRE_HQ : 30.0%, 1.0, 15.5ms, 0 | 20260505 11:11:52.278 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:52.278 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:52.278 | INFO | Uploaded 36GiB to GPU in 2.19223s (16.4217GiB/s) | 20260505 11:11:52.278 | INFO | Downloaded 34GiB to GPU in 2.12454s (16.0035GiB/s) | 20260505 11:11:57.428 | INFO | Analyzing chunk frames = [2752000,2752512) | 20260505 11:11:57.428 | INFO | MemInfo: { | 20260505 11:11:57.428 | INFO | "gpu" : " 17 GB", | 20260505 11:11:57.428 | INFO | "huge" : "", | 20260505 11:11:57.428 | INFO | "huge_shared" : " 27 GB", | 20260505 11:11:57.428 | INFO | "malloc" : " 14 GB", | 20260505 11:11:57.428 | INFO | "pinned" : " 29 GB" | 20260505 11:11:57.428 | INFO | } | 20260505 11:11:57.428 | INFO | Perf report for the last 5.15037 seconds: | 20260505 11:11:57.428 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:11:57.428 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:57.428 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:57.428 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:57.429 | INFO | ANALYSIS : 64.2%, 3.0, 100.4ms, 0 | 20260505 11:11:57.429 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:57.429 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:57.429 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 11:11:57.429 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:57.429 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:11:57.429 | INFO | Uploaded 36GiB to GPU in 2.24326s (16.0481GiB/s) | 20260505 11:11:57.429 | INFO | Downloaded 34GiB to GPU in 2.10697s (16.1369GiB/s) | 20260505 11:11:57.429 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 11:11:57.429 | INFO | Total mem usage: 15.3 GiB | 20260505 11:11:57.429 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 11:11:57.429 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -6.81%) | 20260505 11:11:57.429 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 11:11:57.429 | INFO | High Water Breakdown: | 20260505 11:11:57.429 | INFO | 61% : 8.76 GiB ( 92.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 11:11:57.429 | INFO | 38.8% : 5.57 GiB ( 83.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 11:11:57.429 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 11:11:57.429 | INFO | ------------------------------------------ | 20260505 11:11:57.429 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:11:57.429 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 11:11:57.429 | INFO | Total mem usage: 26.5 GiB | 20260505 11:11:57.429 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 11:11:57.429 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 11:11:57.429 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 11:11:57.429 | INFO | High Water Breakdown: | 20260505 11:11:57.429 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:11:57.429 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:11:57.429 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:11:57.429 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:11:57.429 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:11:57.429 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:11:57.429 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 11:11:57.429 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 11:11:57.429 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:11:57.429 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 11:11:57.429 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 11:11:57.429 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 11:11:57.429 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 11:11:57.429 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 11:11:57.429 | INFO | ------------------------------------------ | 20260505 11:11:57.429 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:11:57.429 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 11:11:57.429 | INFO | Total mem usage: 25.6 GiB | 20260505 11:11:57.430 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 11:11:57.430 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 11:11:57.430 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 11:11:57.430 | INFO | High Water Breakdown: | 20260505 11:11:57.430 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 11:11:57.430 | INFO | ------------------------------------------ | 20260505 11:11:57.430 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:11:57.430 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 11:11:57.430 | INFO | Total mem usage: 15.7 GiB | 20260505 11:11:57.430 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 11:11:57.430 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 11:11:57.430 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 11:11:57.430 | INFO | High Water Breakdown: | 20260505 11:11:57.430 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 11:11:57.430 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 11:11:57.430 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 11:11:57.430 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 11:11:57.430 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 11:11:57.430 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 11:11:57.430 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 11:11:57.430 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 11:11:57.430 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 11:11:57.430 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:11:57.430 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 11:11:57.430 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:11:57.430 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 11:11:57.430 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 11:11:57.430 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 11:11:57.430 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 11:11:57.430 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:11:57.430 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 11:11:57.430 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:11:57.430 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:11:57.430 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:11:57.430 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 11:11:57.430 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 11:11:57.430 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:11:57.430 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 11:11:57.430 | INFO | ------------------------------------------ | 20260505 11:11:57.430 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:11:57.430 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 11:11:57.430 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:11:57.430 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:11:57.430 | INFO | PerfReport: Full Execution -- 3.23e+01 / 7.79e+01, 1.40e+01 / 1.93e+00 | 20260505 11:11:57.430 | INFO | PerfReport: 0.29%, AggregateStats -- 9.31e-02 / 1.02e-01, 1.07e-02 / 4.62e-03 | 20260505 11:11:57.430 | INFO | PerfReport: 11.71%, Baseline -- 3.78e+00 / 4.50e+00, 6.12e-01 / 1.82e-01 | 20260505 11:11:57.430 | INFO | PerfReport: 3.30%, Histogram -- 1.06e+00 / 1.50e+00, 2.07e-01 / 8.38e-02 | 20260505 11:11:57.430 | INFO | PerfReport: 15.91%, DME -- 4.11e+01 / 4.61e+01, 4.62e+00 / 2.64e+00 | 20260505 11:11:57.430 | INFO | PerfReport: 56.40%, FrameLabeling -- 1.82e+01 / 2.09e+01, 1.68e+00 / 2.24e+00 | 20260505 11:11:57.430 | INFO | PerfReport: 7.37%, PulseAccumulating -- 2.38e+00 / 2.88e+00, 4.29e-01 / 2.15e-01 | 20260505 11:11:57.430 | INFO | PerfReport: 3.71%, Metrics -- 1.20e+00 / 1.47e+00, 3.44e-01 / 2.01e-01 | 20260505 11:11:57.430 | INFO | PerfReport: 1.31%, PulseSerializing -- 4.21e-01 / 4.66e-01, 7.87e-02 / 4.25e-02 | 20260505 11:11:57.430 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 11:11:57.430 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:11:57.430 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:11:57.430 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260505 11:11:57.430 | INFO | PerfReport: 22.61%, Upload -- 2.29e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260505 11:11:57.430 | INFO | PerfReport: 21.99%, Download -- 2.23e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260505 11:11:57.430 | INFO | PerfReport: 31.83%, Compute -- 3.23e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260505 11:11:57.430 | INFO | PerfReport: 2.01%, LockUpload -- 2.04e+00 / 0.00e+00, 7.52e+00 / 0.00e+00 | 20260505 11:11:57.430 | INFO | PerfReport: 1.23%, LockDownload -- 1.25e+00 / 0.00e+00, 2.80e+00 / 0.00e+00 | 20260505 11:11:57.430 | INFO | PerfReport: 20.32%, LockCompute -- 2.06e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260505 11:11:57.430 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 2.01e-02 / 0.00e+00 | 20260505 11:11:57.559 | INFO | Starting a round of Baz IO | 20260505 11:12:00.814 | INFO | Populating RTMetrics report | 20260505 11:12:00.814 | INFO | RTMetrics report populated | 20260505 11:12:00.814 | INFO | Opening tmp RTMetrics json file | 20260505 11:12:00.815 | INFO | Serializing RTMetrics to json | 20260505 11:12:00.816 | INFO | Serialization done, swapping tmp file now... | 20260505 11:12:00.816 | INFO | RTMetrics done | 20260505 11:12:00.928 | INFO | Finished a round of Baz IO: Wrote 2.391 GiB in 3.369 seconds | 20260505 11:12:00.928 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.36% / 0.01% / 75.63% | 20260505 11:12:00.928 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.22683% / 7.94729e-06% / 7.15256e-05% | 20260505 11:12:00.928 | INFO | Baz IO overall: | 20260505 11:12:00.928 | INFO | BytesWritten : 1624248285296 | 20260505 11:12:00.928 | INFO | NumEvents : 773365309949 | 20260505 11:12:02.570 | INFO | Analyzing chunk frames = [2752512,2753024) | 20260505 11:12:02.570 | INFO | MemInfo: { | 20260505 11:12:02.570 | INFO | "gpu" : " 17 GB", | 20260505 11:12:02.570 | INFO | "huge" : "", | 20260505 11:12:02.570 | INFO | "huge_shared" : " 27 GB", | 20260505 11:12:02.570 | INFO | "malloc" : " 14 GB", | 20260505 11:12:02.570 | INFO | "pinned" : " 29 GB" | 20260505 11:12:02.570 | INFO | } | 20260505 11:12:02.570 | INFO | Perf report for the last 5.14165 seconds: | 20260505 11:12:02.570 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:12:02.570 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:02.570 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:02.570 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:02.570 | INFO | ANALYSIS : 64.7%, 2.9, 99.6ms, 0 | 20260505 11:12:02.570 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:02.570 | INFO | RT_METRICS : 45.5%, 1.0, 23.9ms, 0 | 20260505 11:12:02.570 | INFO | PRE_HQ : 61.4%, 1.8, 59.0ms, 0 | 20260505 11:12:02.570 | INFO | BAZWRITER : 64.8%, 4.2, 138.6ms, 0 | 20260505 11:12:02.570 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 11:12:02.570 | INFO | Uploaded 36GiB to GPU in 2.18028s (16.5116GiB/s) | 20260505 11:12:02.571 | INFO | Downloaded 38GiB to GPU in 2.28018s (16.6653GiB/s) | 20260505 11:12:07.708 | INFO | Analyzing chunk frames = [2753024,2753536) | 20260505 11:12:07.708 | INFO | MemInfo: { | 20260505 11:12:07.708 | INFO | "gpu" : " 17 GB", | 20260505 11:12:07.708 | INFO | "huge" : "", | 20260505 11:12:07.708 | INFO | "huge_shared" : " 27 GB", | 20260505 11:12:07.708 | INFO | "malloc" : " 14 GB", | 20260505 11:12:07.708 | INFO | "pinned" : " 29 GB" | 20260505 11:12:07.708 | INFO | } | 20260505 11:12:07.708 | INFO | Perf report for the last 5.13808 seconds: | 20260505 11:12:07.708 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:12:07.708 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:07.708 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:07.708 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:07.708 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 11:12:07.708 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:07.708 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:07.708 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:12:07.708 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:07.708 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:07.708 | INFO | Uploaded 36GiB to GPU in 2.17851s (16.525GiB/s) | 20260505 11:12:07.708 | INFO | Downloaded 34GiB to GPU in 2.05012s (16.5844GiB/s) | 20260505 11:12:12.840 | INFO | Analyzing chunk frames = [2753536,2754048) | 20260505 11:12:12.840 | INFO | MemInfo: { | 20260505 11:12:12.840 | INFO | "gpu" : " 17 GB", | 20260505 11:12:12.840 | INFO | "huge" : "", | 20260505 11:12:12.840 | INFO | "huge_shared" : " 27 GB", | 20260505 11:12:12.840 | INFO | "malloc" : " 14 GB", | 20260505 11:12:12.840 | INFO | "pinned" : " 29 GB" | 20260505 11:12:12.840 | INFO | } | 20260505 11:12:12.840 | INFO | Perf report for the last 5.13142 seconds: | 20260505 11:12:12.840 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:12:12.840 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:12.840 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:12.840 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:12.840 | INFO | ANALYSIS : 63.8%, 3.0, 99.1ms, 0 | 20260505 11:12:12.840 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:12.840 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:12.840 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 11:12:12.840 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:12.840 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:12.840 | INFO | Uploaded 36GiB to GPU in 2.22522s (16.1782GiB/s) | 20260505 11:12:12.840 | INFO | Downloaded 34GiB to GPU in 2.12972s (15.9645GiB/s) | 20260505 11:12:17.979 | INFO | Analyzing chunk frames = [2754048,2754560) | 20260505 11:12:17.979 | INFO | MemInfo: { | 20260505 11:12:17.979 | INFO | "gpu" : " 17 GB", | 20260505 11:12:17.979 | INFO | "huge" : "", | 20260505 11:12:17.979 | INFO | "huge_shared" : " 27 GB", | 20260505 11:12:17.979 | INFO | "malloc" : " 14 GB", | 20260505 11:12:17.979 | INFO | "pinned" : " 29 GB" | 20260505 11:12:17.979 | INFO | } | 20260505 11:12:17.979 | INFO | Perf report for the last 5.13915 seconds: | 20260505 11:12:17.979 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:12:17.979 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:17.979 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:17.979 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:17.979 | INFO | ANALYSIS : 63.0%, 3.0, 97.8ms, 0 | 20260505 11:12:17.979 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:17.979 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:17.979 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 11:12:17.979 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:17.979 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:17.979 | INFO | Uploaded 36GiB to GPU in 2.17065s (16.5849GiB/s) | 20260505 11:12:17.979 | INFO | Downloaded 34GiB to GPU in 2.07912s (16.3531GiB/s) | 20260505 11:12:23.124 | INFO | Analyzing chunk frames = [2754560,2755072) | 20260505 11:12:23.124 | INFO | MemInfo: { | 20260505 11:12:23.124 | INFO | "gpu" : " 17 GB", | 20260505 11:12:23.124 | INFO | "huge" : "", | 20260505 11:12:23.124 | INFO | "huge_shared" : " 27 GB", | 20260505 11:12:23.124 | INFO | "malloc" : " 14 GB", | 20260505 11:12:23.124 | INFO | "pinned" : " 29 GB" | 20260505 11:12:23.125 | INFO | } | 20260505 11:12:23.125 | INFO | Perf report for the last 5.14593 seconds: | 20260505 11:12:23.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:12:23.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:23.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:23.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:23.125 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 11:12:23.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:23.125 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:23.125 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 11:12:23.125 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:23.125 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:23.125 | INFO | Uploaded 36GiB to GPU in 2.16266s (16.6461GiB/s) | 20260505 11:12:23.125 | INFO | Downloaded 34GiB to GPU in 2.12349s (16.0114GiB/s) | 20260505 11:12:28.263 | INFO | Analyzing chunk frames = [2755072,2755584) | 20260505 11:12:28.263 | INFO | MemInfo: { | 20260505 11:12:28.263 | INFO | "gpu" : " 17 GB", | 20260505 11:12:28.263 | INFO | "huge" : "", | 20260505 11:12:28.263 | INFO | "huge_shared" : " 27 GB", | 20260505 11:12:28.263 | INFO | "malloc" : " 14 GB", | 20260505 11:12:28.263 | INFO | "pinned" : " 29 GB" | 20260505 11:12:28.263 | INFO | } | 20260505 11:12:28.264 | INFO | Perf report for the last 5.13892 seconds: | 20260505 11:12:28.264 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:12:28.264 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:28.264 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:28.264 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:28.264 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 11:12:28.264 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:28.264 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:28.264 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 11:12:28.264 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:28.264 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:28.264 | INFO | Uploaded 36GiB to GPU in 2.19611s (16.3926GiB/s) | 20260505 11:12:28.264 | INFO | Downloaded 34GiB to GPU in 2.11785s (16.054GiB/s) | 20260505 11:12:33.400 | INFO | Analyzing chunk frames = [2755584,2756096) | 20260505 11:12:33.400 | INFO | MemInfo: { | 20260505 11:12:33.400 | INFO | "gpu" : " 17 GB", | 20260505 11:12:33.400 | INFO | "huge" : "", | 20260505 11:12:33.400 | INFO | "huge_shared" : " 27 GB", | 20260505 11:12:33.400 | INFO | "malloc" : " 14 GB", | 20260505 11:12:33.400 | INFO | "pinned" : " 29 GB" | 20260505 11:12:33.401 | INFO | } | 20260505 11:12:33.401 | INFO | Perf report for the last 5.13692 seconds: | 20260505 11:12:33.401 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:12:33.401 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:33.401 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:33.401 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:33.401 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 11:12:33.401 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:33.401 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:33.401 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 11:12:33.401 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:33.401 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:33.401 | INFO | Uploaded 36GiB to GPU in 2.19512s (16.4GiB/s) | 20260505 11:12:33.401 | INFO | Downloaded 34GiB to GPU in 2.13662s (15.913GiB/s) | 20260505 11:12:38.537 | INFO | Analyzing chunk frames = [2756096,2756608) | 20260505 11:12:38.537 | INFO | MemInfo: { | 20260505 11:12:38.537 | INFO | "gpu" : " 17 GB", | 20260505 11:12:38.537 | INFO | "huge" : "", | 20260505 11:12:38.537 | INFO | "huge_shared" : " 27 GB", | 20260505 11:12:38.537 | INFO | "malloc" : " 14 GB", | 20260505 11:12:38.537 | INFO | "pinned" : " 29 GB" | 20260505 11:12:38.537 | INFO | } | 20260505 11:12:38.537 | INFO | Perf report for the last 5.13687 seconds: | 20260505 11:12:38.537 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:12:38.537 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:38.537 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:38.538 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:38.538 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 11:12:38.538 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:38.538 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:38.538 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 11:12:38.538 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:38.538 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:38.538 | INFO | Uploaded 36GiB to GPU in 2.21962s (16.219GiB/s) | 20260505 11:12:38.538 | INFO | Downloaded 34GiB to GPU in 2.09001s (16.2678GiB/s) | 20260505 11:12:38.676 | INFO | Starting a round of Baz IO | 20260505 11:12:41.940 | INFO | Populating RTMetrics report | 20260505 11:12:41.941 | INFO | RTMetrics report populated | 20260505 11:12:41.941 | INFO | Opening tmp RTMetrics json file | 20260505 11:12:41.952 | INFO | Serializing RTMetrics to json | 20260505 11:12:41.953 | INFO | Serialization done, swapping tmp file now... | 20260505 11:12:41.954 | INFO | RTMetrics done | 20260505 11:12:42.076 | INFO | Finished a round of Baz IO: Wrote 2.389 GiB in 3.399 seconds | 20260505 11:12:42.076 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.35% / 0.01% / 75.64% | 20260505 11:12:42.076 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.22206% / 1.19209e-05% / 7.94729e-05% | 20260505 11:12:42.076 | INFO | Baz IO overall: | 20260505 11:12:42.076 | INFO | BytesWritten : 1626813125716 | 20260505 11:12:42.076 | INFO | NumEvents : 774585924744 | 20260505 11:12:43.690 | INFO | Analyzing chunk frames = [2756608,2757120) | 20260505 11:12:43.690 | INFO | MemInfo: { | 20260505 11:12:43.690 | INFO | "gpu" : " 17 GB", | 20260505 11:12:43.690 | INFO | "huge" : "", | 20260505 11:12:43.690 | INFO | "huge_shared" : " 27 GB", | 20260505 11:12:43.690 | INFO | "malloc" : " 14 GB", | 20260505 11:12:43.690 | INFO | "pinned" : " 29 GB" | 20260505 11:12:43.690 | INFO | } | 20260505 11:12:43.690 | INFO | Perf report for the last 5.15239 seconds: | 20260505 11:12:43.690 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:12:43.690 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:43.690 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:43.690 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:43.690 | INFO | ANALYSIS : 64.9%, 2.9, 100.0ms, 0 | 20260505 11:12:43.690 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:43.690 | INFO | RT_METRICS : 45.5%, 1.0, 23.9ms, 0 | 20260505 11:12:43.690 | INFO | PRE_HQ : 61.5%, 1.8, 59.0ms, 0 | 20260505 11:12:43.690 | INFO | BAZWRITER : 65.3%, 4.2, 140.6ms, 0 | 20260505 11:12:43.690 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 11:12:43.690 | INFO | Uploaded 36GiB to GPU in 2.23664s (16.0956GiB/s) | 20260505 11:12:43.690 | INFO | Downloaded 38GiB to GPU in 2.30229s (16.5053GiB/s) | 20260505 11:12:48.820 | INFO | Analyzing chunk frames = [2757120,2757632) | 20260505 11:12:48.820 | INFO | MemInfo: { | 20260505 11:12:48.820 | INFO | "gpu" : " 17 GB", | 20260505 11:12:48.820 | INFO | "huge" : "", | 20260505 11:12:48.820 | INFO | "huge_shared" : " 27 GB", | 20260505 11:12:48.820 | INFO | "malloc" : " 14 GB", | 20260505 11:12:48.820 | INFO | "pinned" : " 29 GB" | 20260505 11:12:48.820 | INFO | } | 20260505 11:12:48.820 | INFO | Perf report for the last 5.13041 seconds: | 20260505 11:12:48.820 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:12:48.820 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:48.820 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:48.820 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:48.820 | INFO | ANALYSIS : 64.0%, 3.0, 99.7ms, 0 | 20260505 11:12:48.820 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:48.820 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:48.820 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 11:12:48.820 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:48.820 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:48.821 | INFO | Uploaded 36GiB to GPU in 2.18289s (16.4919GiB/s) | 20260505 11:12:48.821 | INFO | Downloaded 34GiB to GPU in 2.04973s (16.5876GiB/s) | 20260505 11:12:53.956 | INFO | Analyzing chunk frames = [2757632,2758144) | 20260505 11:12:53.956 | INFO | MemInfo: { | 20260505 11:12:53.956 | INFO | "gpu" : " 17 GB", | 20260505 11:12:53.956 | INFO | "huge" : "", | 20260505 11:12:53.956 | INFO | "huge_shared" : " 27 GB", | 20260505 11:12:53.956 | INFO | "malloc" : " 14 GB", | 20260505 11:12:53.956 | INFO | "pinned" : " 29 GB" | 20260505 11:12:53.956 | INFO | } | 20260505 11:12:53.957 | INFO | Perf report for the last 5.13624 seconds: | 20260505 11:12:53.957 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:12:53.957 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:53.957 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:53.957 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:53.957 | INFO | ANALYSIS : 63.7%, 3.0, 99.0ms, 0 | 20260505 11:12:53.957 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:53.957 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:53.957 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 11:12:53.957 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:53.957 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:53.957 | INFO | Uploaded 36GiB to GPU in 2.22583s (16.1737GiB/s) | 20260505 11:12:53.957 | INFO | Downloaded 34GiB to GPU in 2.12947s (15.9664GiB/s) | 20260505 11:12:59.100 | INFO | Analyzing chunk frames = [2758144,2758656) | 20260505 11:12:59.100 | INFO | MemInfo: { | 20260505 11:12:59.100 | INFO | "gpu" : " 17 GB", | 20260505 11:12:59.100 | INFO | "huge" : "", | 20260505 11:12:59.100 | INFO | "huge_shared" : " 27 GB", | 20260505 11:12:59.100 | INFO | "malloc" : " 14 GB", | 20260505 11:12:59.100 | INFO | "pinned" : " 29 GB" | 20260505 11:12:59.100 | INFO | } | 20260505 11:12:59.100 | INFO | Perf report for the last 5.14343 seconds: | 20260505 11:12:59.100 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:12:59.100 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:59.100 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:59.100 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:59.100 | INFO | ANALYSIS : 62.8%, 3.0, 97.6ms, 0 | 20260505 11:12:59.100 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:59.100 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:59.100 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 11:12:59.100 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:59.100 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:12:59.100 | INFO | Uploaded 36GiB to GPU in 2.14891s (16.7527GiB/s) | 20260505 11:12:59.100 | INFO | Downloaded 34GiB to GPU in 2.07274s (16.4034GiB/s) | 20260505 11:13:04.235 | INFO | Analyzing chunk frames = [2758656,2759168) | 20260505 11:13:04.235 | INFO | MemInfo: { | 20260505 11:13:04.235 | INFO | "gpu" : " 17 GB", | 20260505 11:13:04.235 | INFO | "huge" : "", | 20260505 11:13:04.235 | INFO | "huge_shared" : " 27 GB", | 20260505 11:13:04.235 | INFO | "malloc" : " 14 GB", | 20260505 11:13:04.235 | INFO | "pinned" : " 29 GB" | 20260505 11:13:04.235 | INFO | } | 20260505 11:13:04.235 | INFO | Perf report for the last 5.13516 seconds: | 20260505 11:13:04.235 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:13:04.235 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:04.235 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:04.235 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:04.235 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 11:13:04.235 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:04.235 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:04.235 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 11:13:04.235 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:04.235 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:04.235 | INFO | Uploaded 36GiB to GPU in 2.16697s (16.6131GiB/s) | 20260505 11:13:04.236 | INFO | Downloaded 34GiB to GPU in 2.12195s (16.023GiB/s) | 20260505 11:13:09.381 | INFO | Analyzing chunk frames = [2759168,2759680) | 20260505 11:13:09.381 | INFO | MemInfo: { | 20260505 11:13:09.381 | INFO | "gpu" : " 17 GB", | 20260505 11:13:09.381 | INFO | "huge" : "", | 20260505 11:13:09.381 | INFO | "huge_shared" : " 27 GB", | 20260505 11:13:09.381 | INFO | "malloc" : " 14 GB", | 20260505 11:13:09.381 | INFO | "pinned" : " 29 GB" | 20260505 11:13:09.381 | INFO | } | 20260505 11:13:09.381 | INFO | Perf report for the last 5.14556 seconds: | 20260505 11:13:09.381 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:13:09.381 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:09.381 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:09.381 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:09.381 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 11:13:09.381 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:09.381 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:09.381 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 11:13:09.381 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:09.381 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:09.381 | INFO | Uploaded 36GiB to GPU in 2.19481s (16.4024GiB/s) | 20260505 11:13:09.381 | INFO | Downloaded 34GiB to GPU in 2.11661s (16.0634GiB/s) | 20260505 11:13:14.528 | INFO | Analyzing chunk frames = [2759680,2760192) | 20260505 11:13:14.528 | INFO | MemInfo: { | 20260505 11:13:14.528 | INFO | "gpu" : " 17 GB", | 20260505 11:13:14.528 | INFO | "huge" : "", | 20260505 11:13:14.528 | INFO | "huge_shared" : " 27 GB", | 20260505 11:13:14.528 | INFO | "malloc" : " 14 GB", | 20260505 11:13:14.528 | INFO | "pinned" : " 29 GB" | 20260505 11:13:14.528 | INFO | } | 20260505 11:13:14.528 | INFO | Perf report for the last 5.14735 seconds: | 20260505 11:13:14.528 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:13:14.528 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:14.528 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:14.528 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:14.528 | INFO | ANALYSIS : 63.3%, 3.0, 99.0ms, 0 | 20260505 11:13:14.528 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:14.528 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:14.528 | INFO | PRE_HQ : 29.9%, 1.0, 15.6ms, 0 | 20260505 11:13:14.528 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:14.528 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:14.528 | INFO | Uploaded 36GiB to GPU in 2.16662s (16.6158GiB/s) | 20260505 11:13:14.528 | INFO | Downloaded 34GiB to GPU in 2.1395s (15.8916GiB/s) | 20260505 11:13:19.652 | INFO | Analyzing chunk frames = [2760192,2760704) | 20260505 11:13:19.652 | INFO | MemInfo: { | 20260505 11:13:19.652 | INFO | "gpu" : " 17 GB", | 20260505 11:13:19.652 | INFO | "huge" : "", | 20260505 11:13:19.652 | INFO | "huge_shared" : " 27 GB", | 20260505 11:13:19.652 | INFO | "malloc" : " 14 GB", | 20260505 11:13:19.652 | INFO | "pinned" : " 29 GB" | 20260505 11:13:19.652 | INFO | } | 20260505 11:13:19.652 | INFO | Perf report for the last 5.12394 seconds: | 20260505 11:13:19.652 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:13:19.652 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:19.652 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:19.652 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:19.652 | INFO | ANALYSIS : 64.0%, 3.0, 99.7ms, 0 | 20260505 11:13:19.652 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:19.652 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:19.652 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 11:13:19.652 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:19.652 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:19.652 | INFO | Uploaded 36GiB to GPU in 2.22461s (16.1826GiB/s) | 20260505 11:13:19.652 | INFO | Downloaded 34GiB to GPU in 2.10841s (16.1259GiB/s) | 20260505 11:13:19.653 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 11:13:19.653 | INFO | Total mem usage: 15.3 GiB | 20260505 11:13:19.653 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 11:13:19.653 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -6.9%) | 20260505 11:13:19.653 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 11:13:19.653 | INFO | High Water Breakdown: | 20260505 11:13:19.653 | INFO | 61% : 8.75 GiB ( 92.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 11:13:19.653 | INFO | 38.9% : 5.57 GiB ( 83.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 11:13:19.653 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 11:13:19.653 | INFO | ------------------------------------------ | 20260505 11:13:19.653 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:13:19.653 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 11:13:19.653 | INFO | Total mem usage: 26.5 GiB | 20260505 11:13:19.653 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 11:13:19.653 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 11:13:19.653 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 11:13:19.653 | INFO | High Water Breakdown: | 20260505 11:13:19.653 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:13:19.653 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:13:19.653 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:13:19.653 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:13:19.653 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:13:19.653 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:13:19.653 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 11:13:19.653 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 11:13:19.653 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:13:19.653 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 11:13:19.653 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 11:13:19.653 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 11:13:19.653 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 11:13:19.653 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 11:13:19.653 | INFO | ------------------------------------------ | 20260505 11:13:19.653 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:13:19.653 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 11:13:19.653 | INFO | Total mem usage: 25.6 GiB | 20260505 11:13:19.653 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 11:13:19.653 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 11:13:19.653 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 11:13:19.653 | INFO | High Water Breakdown: | 20260505 11:13:19.653 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 11:13:19.653 | INFO | ------------------------------------------ | 20260505 11:13:19.653 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:13:19.653 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 11:13:19.653 | INFO | Total mem usage: 15.7 GiB | 20260505 11:13:19.653 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 11:13:19.653 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 11:13:19.653 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 11:13:19.653 | INFO | High Water Breakdown: | 20260505 11:13:19.653 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 11:13:19.653 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 11:13:19.653 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 11:13:19.653 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 11:13:19.653 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 11:13:19.653 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 11:13:19.653 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 11:13:19.653 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 11:13:19.653 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 11:13:19.653 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:13:19.653 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 11:13:19.653 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:13:19.653 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 11:13:19.653 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 11:13:19.653 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 11:13:19.653 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 11:13:19.653 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:13:19.653 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 11:13:19.653 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:13:19.653 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:13:19.653 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:13:19.653 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 11:13:19.653 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 11:13:19.653 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:13:19.653 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 11:13:19.653 | INFO | ------------------------------------------ | 20260505 11:13:19.653 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:13:19.653 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 11:13:19.653 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:13:19.653 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:13:19.653 | INFO | PerfReport: Full Execution -- 3.22e+01 / 7.74e+01, 1.38e+01 / 1.80e+00 | 20260505 11:13:19.653 | INFO | PerfReport: 0.29%, AggregateStats -- 9.32e-02 / 1.02e-01, 1.05e-02 / 4.17e-03 | 20260505 11:13:19.654 | INFO | PerfReport: 11.71%, Baseline -- 3.77e+00 / 4.51e+00, 6.08e-01 / 1.67e-01 | 20260505 11:13:19.654 | INFO | PerfReport: 3.31%, Histogram -- 1.06e+00 / 1.50e+00, 2.07e-01 / 8.92e-02 | 20260505 11:13:19.654 | INFO | PerfReport: 15.83%, DME -- 4.08e+01 / 4.55e+01, 4.45e+00 / 1.88e+00 | 20260505 11:13:19.654 | INFO | PerfReport: 56.47%, FrameLabeling -- 1.82e+01 / 2.08e+01, 1.69e+00 / 1.92e+00 | 20260505 11:13:19.654 | INFO | PerfReport: 7.41%, PulseAccumulating -- 2.39e+00 / 2.95e+00, 4.34e-01 / 1.90e-01 | 20260505 11:13:19.654 | INFO | PerfReport: 3.68%, Metrics -- 1.19e+00 / 1.52e+00, 3.37e-01 / 1.84e-01 | 20260505 11:13:19.654 | INFO | PerfReport: 1.30%, PulseSerializing -- 4.20e-01 / 4.68e-01, 7.85e-02 / 3.75e-02 | 20260505 11:13:19.654 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 11:13:19.654 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:13:19.654 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:13:19.654 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260505 11:13:19.654 | INFO | PerfReport: 22.56%, Upload -- 2.28e+01 / 0.00e+00, 8.15e+00 / 0.00e+00 | 20260505 11:13:19.654 | INFO | PerfReport: 21.90%, Download -- 2.22e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260505 11:13:19.654 | INFO | PerfReport: 31.83%, Compute -- 3.22e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 11:13:19.654 | INFO | PerfReport: 1.99%, LockUpload -- 2.01e+00 / 0.00e+00, 7.60e+00 / 0.00e+00 | 20260505 11:13:19.654 | INFO | PerfReport: 1.23%, LockDownload -- 1.25e+00 / 0.00e+00, 2.76e+00 / 0.00e+00 | 20260505 11:13:19.654 | INFO | PerfReport: 20.47%, LockCompute -- 2.07e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260505 11:13:19.654 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.97e-02 / 0.00e+00 | 20260505 11:13:19.784 | INFO | Starting a round of Baz IO | 20260505 11:13:23.037 | INFO | Populating RTMetrics report | 20260505 11:13:23.038 | INFO | RTMetrics report populated | 20260505 11:13:23.038 | INFO | Opening tmp RTMetrics json file | 20260505 11:13:23.056 | INFO | Serializing RTMetrics to json | 20260505 11:13:23.088 | INFO | Serialization done, swapping tmp file now... | 20260505 11:13:23.107 | INFO | RTMetrics done | 20260505 11:13:23.177 | INFO | Finished a round of Baz IO: Wrote 2.39 GiB in 3.393 seconds | 20260505 11:13:23.177 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.34% / 0.01% / 75.65% | 20260505 11:13:23.177 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.22035% / 7.94729e-06% / 5.16574e-05% | 20260505 11:13:23.177 | INFO | Baz IO overall: | 20260505 11:13:23.177 | INFO | BytesWritten : 1629379145320 | 20260505 11:13:23.177 | INFO | NumEvents : 775807838013 | 20260505 11:13:24.788 | INFO | Analyzing chunk frames = [2760704,2761216) | 20260505 11:13:24.788 | INFO | MemInfo: { | 20260505 11:13:24.788 | INFO | "gpu" : " 17 GB", | 20260505 11:13:24.788 | INFO | "huge" : "", | 20260505 11:13:24.788 | INFO | "huge_shared" : " 27 GB", | 20260505 11:13:24.788 | INFO | "malloc" : " 14 GB", | 20260505 11:13:24.788 | INFO | "pinned" : " 29 GB" | 20260505 11:13:24.788 | INFO | } | 20260505 11:13:24.788 | INFO | Perf report for the last 5.1363 seconds: | 20260505 11:13:24.788 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:13:24.788 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:24.788 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:24.788 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:24.789 | INFO | ANALYSIS : 64.8%, 2.9, 99.1ms, 0 | 20260505 11:13:24.789 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:24.789 | INFO | RT_METRICS : 46.7%, 1.0, 24.5ms, 0 | 20260505 11:13:24.789 | INFO | PRE_HQ : 62.4%, 1.8, 58.6ms, 0 | 20260505 11:13:24.789 | INFO | BAZWRITER : 65.4%, 4.2, 141.1ms, 0 | 20260505 11:13:24.789 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 11:13:24.789 | INFO | Uploaded 36GiB to GPU in 2.29123s (15.7121GiB/s) | 20260505 11:13:24.789 | INFO | Downloaded 38GiB to GPU in 2.38607s (15.9258GiB/s) | 20260505 11:13:29.935 | INFO | Analyzing chunk frames = [2761216,2761728) | 20260505 11:13:29.935 | INFO | MemInfo: { | 20260505 11:13:29.935 | INFO | "gpu" : " 17 GB", | 20260505 11:13:29.935 | INFO | "huge" : "", | 20260505 11:13:29.935 | INFO | "huge_shared" : " 27 GB", | 20260505 11:13:29.935 | INFO | "malloc" : " 14 GB", | 20260505 11:13:29.935 | INFO | "pinned" : " 29 GB" | 20260505 11:13:29.935 | INFO | } | 20260505 11:13:29.935 | INFO | Perf report for the last 5.14683 seconds: | 20260505 11:13:29.935 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:13:29.935 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:29.935 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:29.935 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:29.935 | INFO | ANALYSIS : 64.1%, 3.0, 100.2ms, 0 | 20260505 11:13:29.935 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:29.935 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:29.935 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 11:13:29.935 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:29.935 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:29.935 | INFO | Uploaded 36GiB to GPU in 2.2167s (16.2404GiB/s) | 20260505 11:13:29.935 | INFO | Downloaded 34GiB to GPU in 2.08511s (16.3061GiB/s) | 20260505 11:13:35.079 | INFO | Analyzing chunk frames = [2761728,2762240) | 20260505 11:13:35.079 | INFO | MemInfo: { | 20260505 11:13:35.079 | INFO | "gpu" : " 17 GB", | 20260505 11:13:35.079 | INFO | "huge" : "", | 20260505 11:13:35.079 | INFO | "huge_shared" : " 27 GB", | 20260505 11:13:35.079 | INFO | "malloc" : " 14 GB", | 20260505 11:13:35.079 | INFO | "pinned" : " 29 GB" | 20260505 11:13:35.079 | INFO | } | 20260505 11:13:35.079 | INFO | Perf report for the last 5.14391 seconds: | 20260505 11:13:35.079 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:13:35.079 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:35.079 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:35.079 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:35.079 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 11:13:35.079 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:35.079 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:35.079 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 11:13:35.079 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:35.079 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:35.079 | INFO | Uploaded 36GiB to GPU in 2.20996s (16.2899GiB/s) | 20260505 11:13:35.079 | INFO | Downloaded 34GiB to GPU in 2.1138s (16.0847GiB/s) | 20260505 11:13:40.215 | INFO | Analyzing chunk frames = [2762240,2762752) | 20260505 11:13:40.215 | INFO | MemInfo: { | 20260505 11:13:40.215 | INFO | "gpu" : " 17 GB", | 20260505 11:13:40.215 | INFO | "huge" : "", | 20260505 11:13:40.215 | INFO | "huge_shared" : " 27 GB", | 20260505 11:13:40.215 | INFO | "malloc" : " 14 GB", | 20260505 11:13:40.215 | INFO | "pinned" : " 29 GB" | 20260505 11:13:40.215 | INFO | } | 20260505 11:13:40.215 | INFO | Perf report for the last 5.13596 seconds: | 20260505 11:13:40.215 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:13:40.215 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:40.215 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:40.215 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:40.215 | INFO | ANALYSIS : 63.4%, 3.0, 98.4ms, 0 | 20260505 11:13:40.215 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:40.215 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:40.215 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 11:13:40.215 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:40.215 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:40.215 | INFO | Uploaded 36GiB to GPU in 2.21185s (16.276GiB/s) | 20260505 11:13:40.215 | INFO | Downloaded 34GiB to GPU in 2.12935s (15.9673GiB/s) | 20260505 11:13:45.349 | INFO | Analyzing chunk frames = [2762752,2763264) | 20260505 11:13:45.349 | INFO | MemInfo: { | 20260505 11:13:45.349 | INFO | "gpu" : " 17 GB", | 20260505 11:13:45.349 | INFO | "huge" : "", | 20260505 11:13:45.349 | INFO | "huge_shared" : " 27 GB", | 20260505 11:13:45.349 | INFO | "malloc" : " 14 GB", | 20260505 11:13:45.349 | INFO | "pinned" : " 29 GB" | 20260505 11:13:45.349 | INFO | } | 20260505 11:13:45.349 | INFO | Perf report for the last 5.13387 seconds: | 20260505 11:13:45.349 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:13:45.349 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:45.349 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:45.349 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:45.349 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 11:13:45.349 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:45.349 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:45.349 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 11:13:45.349 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:45.349 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:45.349 | INFO | Uploaded 36GiB to GPU in 2.19143s (16.4276GiB/s) | 20260505 11:13:45.349 | INFO | Downloaded 34GiB to GPU in 2.1259s (15.9932GiB/s) | 20260505 11:13:50.493 | INFO | Analyzing chunk frames = [2763264,2763776) | 20260505 11:13:50.493 | INFO | MemInfo: { | 20260505 11:13:50.493 | INFO | "gpu" : " 17 GB", | 20260505 11:13:50.493 | INFO | "huge" : "", | 20260505 11:13:50.493 | INFO | "huge_shared" : " 27 GB", | 20260505 11:13:50.493 | INFO | "malloc" : " 14 GB", | 20260505 11:13:50.493 | INFO | "pinned" : " 29 GB" | 20260505 11:13:50.493 | INFO | } | 20260505 11:13:50.493 | INFO | Perf report for the last 5.14394 seconds: | 20260505 11:13:50.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:13:50.493 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:50.493 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:50.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:50.493 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 11:13:50.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:50.493 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:50.493 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 11:13:50.493 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:50.493 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:50.493 | INFO | Uploaded 36GiB to GPU in 2.17042s (16.5866GiB/s) | 20260505 11:13:50.493 | INFO | Downloaded 34GiB to GPU in 2.116s (16.0681GiB/s) | 20260505 11:13:55.638 | INFO | Analyzing chunk frames = [2763776,2764288) | 20260505 11:13:55.638 | INFO | MemInfo: { | 20260505 11:13:55.638 | INFO | "gpu" : " 17 GB", | 20260505 11:13:55.638 | INFO | "huge" : "", | 20260505 11:13:55.638 | INFO | "huge_shared" : " 27 GB", | 20260505 11:13:55.638 | INFO | "malloc" : " 14 GB", | 20260505 11:13:55.638 | INFO | "pinned" : " 29 GB" | 20260505 11:13:55.638 | INFO | } | 20260505 11:13:55.638 | INFO | Perf report for the last 5.1451 seconds: | 20260505 11:13:55.638 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:13:55.638 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:55.638 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:55.638 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:55.638 | INFO | ANALYSIS : 63.5%, 3.0, 99.3ms, 0 | 20260505 11:13:55.638 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:55.638 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:55.638 | INFO | PRE_HQ : 29.9%, 1.0, 15.6ms, 0 | 20260505 11:13:55.638 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:55.638 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:13:55.638 | INFO | Uploaded 36GiB to GPU in 2.17875s (16.5232GiB/s) | 20260505 11:13:55.638 | INFO | Downloaded 34GiB to GPU in 2.12424s (16.0057GiB/s) | 20260505 11:14:00.771 | INFO | Analyzing chunk frames = [2764288,2764800) | 20260505 11:14:00.771 | INFO | MemInfo: { | 20260505 11:14:00.771 | INFO | "gpu" : " 17 GB", | 20260505 11:14:00.771 | INFO | "huge" : "", | 20260505 11:14:00.771 | INFO | "huge_shared" : " 27 GB", | 20260505 11:14:00.771 | INFO | "malloc" : " 14 GB", | 20260505 11:14:00.771 | INFO | "pinned" : " 29 GB" | 20260505 11:14:00.771 | INFO | } | 20260505 11:14:00.772 | INFO | Perf report for the last 5.13363 seconds: | 20260505 11:14:00.772 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:14:00.772 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:00.772 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:00.772 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:00.772 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 11:14:00.772 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:00.772 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:00.772 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 11:14:00.772 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:00.772 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:00.772 | INFO | Uploaded 36GiB to GPU in 2.21549s (16.2492GiB/s) | 20260505 11:14:00.772 | INFO | Downloaded 34GiB to GPU in 2.09679s (16.2153GiB/s) | 20260505 11:14:00.910 | INFO | Starting a round of Baz IO | 20260505 11:14:04.168 | INFO | Populating RTMetrics report | 20260505 11:14:04.169 | INFO | RTMetrics report populated | 20260505 11:14:04.169 | INFO | Opening tmp RTMetrics json file | 20260505 11:14:04.170 | INFO | Serializing RTMetrics to json | 20260505 11:14:04.171 | INFO | Serialization done, swapping tmp file now... | 20260505 11:14:04.172 | INFO | RTMetrics done | 20260505 11:14:04.306 | INFO | Finished a round of Baz IO: Wrote 2.386 GiB in 3.396 seconds | 20260505 11:14:04.306 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.33% / 0.01% / 75.66% | 20260505 11:14:04.306 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.21143% / 7.94729e-06% / 6.75519e-05% | 20260505 11:14:04.306 | INFO | Baz IO overall: | 20260505 11:14:04.306 | INFO | BytesWritten : 1631940589513 | 20260505 11:14:04.306 | INFO | NumEvents : 777026627244 | 20260505 11:14:05.902 | INFO | Analyzing chunk frames = [2764800,2765312) | 20260505 11:14:05.902 | INFO | MemInfo: { | 20260505 11:14:05.902 | INFO | "gpu" : " 17 GB", | 20260505 11:14:05.902 | INFO | "huge" : "", | 20260505 11:14:05.902 | INFO | "huge_shared" : " 27 GB", | 20260505 11:14:05.902 | INFO | "malloc" : " 14 GB", | 20260505 11:14:05.902 | INFO | "pinned" : " 29 GB" | 20260505 11:14:05.902 | INFO | } | 20260505 11:14:05.902 | INFO | Perf report for the last 5.13048 seconds: | 20260505 11:14:05.902 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:14:05.902 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:05.902 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:05.902 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:05.902 | INFO | ANALYSIS : 65.1%, 2.9, 99.9ms, 0 | 20260505 11:14:05.902 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:05.902 | INFO | RT_METRICS : 45.5%, 1.0, 23.8ms, 0 | 20260505 11:14:05.902 | INFO | PRE_HQ : 61.5%, 1.8, 58.8ms, 0 | 20260505 11:14:05.902 | INFO | BAZWRITER : 65.5%, 4.1, 139.1ms, 0 | 20260505 11:14:05.902 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 11:14:05.903 | INFO | Uploaded 36GiB to GPU in 2.19704s (16.3857GiB/s) | 20260505 11:14:05.903 | INFO | Downloaded 38GiB to GPU in 2.29994s (16.5222GiB/s) | 20260505 11:14:11.047 | INFO | Analyzing chunk frames = [2765312,2765824) | 20260505 11:14:11.047 | INFO | MemInfo: { | 20260505 11:14:11.047 | INFO | "gpu" : " 17 GB", | 20260505 11:14:11.047 | INFO | "huge" : "", | 20260505 11:14:11.047 | INFO | "huge_shared" : " 27 GB", | 20260505 11:14:11.047 | INFO | "malloc" : " 14 GB", | 20260505 11:14:11.047 | INFO | "pinned" : " 29 GB" | 20260505 11:14:11.047 | INFO | } | 20260505 11:14:11.047 | INFO | Perf report for the last 5.14519 seconds: | 20260505 11:14:11.047 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:14:11.047 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:11.047 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:11.047 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:11.047 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 11:14:11.047 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:11.048 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:11.048 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 11:14:11.048 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:11.048 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:11.048 | INFO | Uploaded 36GiB to GPU in 2.17683s (16.5378GiB/s) | 20260505 11:14:11.048 | INFO | Downloaded 34GiB to GPU in 2.09005s (16.2676GiB/s) | 20260505 11:14:16.195 | INFO | Analyzing chunk frames = [2765824,2766336) | 20260505 11:14:16.195 | INFO | MemInfo: { | 20260505 11:14:16.195 | INFO | "gpu" : " 17 GB", | 20260505 11:14:16.195 | INFO | "huge" : "", | 20260505 11:14:16.195 | INFO | "huge_shared" : " 27 GB", | 20260505 11:14:16.195 | INFO | "malloc" : " 14 GB", | 20260505 11:14:16.195 | INFO | "pinned" : " 29 GB" | 20260505 11:14:16.195 | INFO | } | 20260505 11:14:16.195 | INFO | Perf report for the last 5.14793 seconds: | 20260505 11:14:16.195 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:14:16.195 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:16.195 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:16.195 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:16.195 | INFO | ANALYSIS : 63.5%, 3.0, 99.0ms, 0 | 20260505 11:14:16.195 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:16.195 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:16.195 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 11:14:16.195 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:16.195 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:16.196 | INFO | Uploaded 36GiB to GPU in 2.22572s (16.1745GiB/s) | 20260505 11:14:16.196 | INFO | Downloaded 34GiB to GPU in 2.13405s (15.9322GiB/s) | 20260505 11:14:21.330 | INFO | Analyzing chunk frames = [2766336,2766848) | 20260505 11:14:21.330 | INFO | MemInfo: { | 20260505 11:14:21.330 | INFO | "gpu" : " 17 GB", | 20260505 11:14:21.330 | INFO | "huge" : "", | 20260505 11:14:21.330 | INFO | "huge_shared" : " 27 GB", | 20260505 11:14:21.330 | INFO | "malloc" : " 14 GB", | 20260505 11:14:21.330 | INFO | "pinned" : " 29 GB" | 20260505 11:14:21.330 | INFO | } | 20260505 11:14:21.330 | INFO | Perf report for the last 5.13476 seconds: | 20260505 11:14:21.330 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:14:21.330 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:21.330 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:21.330 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:21.330 | INFO | ANALYSIS : 63.2%, 3.0, 98.1ms, 0 | 20260505 11:14:21.330 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:21.330 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:21.330 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 11:14:21.330 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:21.330 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:21.330 | INFO | Uploaded 36GiB to GPU in 2.1995s (16.3673GiB/s) | 20260505 11:14:21.330 | INFO | Downloaded 34GiB to GPU in 2.12925s (15.968GiB/s) | 20260505 11:14:26.470 | INFO | Analyzing chunk frames = [2766848,2767360) | 20260505 11:14:26.470 | INFO | MemInfo: { | 20260505 11:14:26.470 | INFO | "gpu" : " 17 GB", | 20260505 11:14:26.470 | INFO | "huge" : "", | 20260505 11:14:26.470 | INFO | "huge_shared" : " 27 GB", | 20260505 11:14:26.470 | INFO | "malloc" : " 14 GB", | 20260505 11:14:26.471 | INFO | "pinned" : " 29 GB" | 20260505 11:14:26.471 | INFO | } | 20260505 11:14:26.471 | INFO | Perf report for the last 5.14047 seconds: | 20260505 11:14:26.471 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:14:26.471 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:26.471 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:26.471 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:26.471 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 11:14:26.471 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:26.471 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:26.471 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 11:14:26.471 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:26.471 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:26.471 | INFO | Uploaded 36GiB to GPU in 2.17181s (16.5761GiB/s) | 20260505 11:14:26.471 | INFO | Downloaded 34GiB to GPU in 2.12663s (15.9878GiB/s) | 20260505 11:14:31.601 | INFO | Analyzing chunk frames = [2767360,2767872) | 20260505 11:14:31.601 | INFO | MemInfo: { | 20260505 11:14:31.601 | INFO | "gpu" : " 17 GB", | 20260505 11:14:31.601 | INFO | "huge" : "", | 20260505 11:14:31.601 | INFO | "huge_shared" : " 27 GB", | 20260505 11:14:31.601 | INFO | "malloc" : " 14 GB", | 20260505 11:14:31.601 | INFO | "pinned" : " 29 GB" | 20260505 11:14:31.601 | INFO | } | 20260505 11:14:31.601 | INFO | Perf report for the last 5.13076 seconds: | 20260505 11:14:31.601 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:14:31.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:31.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:31.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:31.602 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 11:14:31.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:31.602 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:31.602 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 11:14:31.602 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:31.602 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:31.602 | INFO | Uploaded 36GiB to GPU in 2.16729s (16.6106GiB/s) | 20260505 11:14:31.602 | INFO | Downloaded 34GiB to GPU in 2.12298s (16.0152GiB/s) | 20260505 11:14:36.744 | INFO | Analyzing chunk frames = [2767872,2768384) | 20260505 11:14:36.744 | INFO | MemInfo: { | 20260505 11:14:36.744 | INFO | "gpu" : " 17 GB", | 20260505 11:14:36.744 | INFO | "huge" : "", | 20260505 11:14:36.744 | INFO | "huge_shared" : " 27 GB", | 20260505 11:14:36.744 | INFO | "malloc" : " 14 GB", | 20260505 11:14:36.744 | INFO | "pinned" : " 29 GB" | 20260505 11:14:36.744 | INFO | } | 20260505 11:14:36.744 | INFO | Perf report for the last 5.14306 seconds: | 20260505 11:14:36.744 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:14:36.744 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:36.744 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:36.744 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:36.745 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 11:14:36.745 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:36.745 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:36.745 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 11:14:36.745 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:36.745 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:36.745 | INFO | Uploaded 36GiB to GPU in 2.16299s (16.6436GiB/s) | 20260505 11:14:36.745 | INFO | Downloaded 34GiB to GPU in 2.15559s (15.773GiB/s) | 20260505 11:14:41.881 | INFO | Analyzing chunk frames = [2768384,2768896) | 20260505 11:14:41.881 | INFO | MemInfo: { | 20260505 11:14:41.881 | INFO | "gpu" : " 17 GB", | 20260505 11:14:41.881 | INFO | "huge" : "", | 20260505 11:14:41.881 | INFO | "huge_shared" : " 27 GB", | 20260505 11:14:41.881 | INFO | "malloc" : " 14 GB", | 20260505 11:14:41.881 | INFO | "pinned" : " 29 GB" | 20260505 11:14:41.881 | INFO | } | 20260505 11:14:41.881 | INFO | Perf report for the last 5.13679 seconds: | 20260505 11:14:41.881 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:14:41.881 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:41.881 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:41.881 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:41.881 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 11:14:41.881 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:41.881 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:41.881 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 11:14:41.881 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:41.881 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:41.881 | INFO | Uploaded 36GiB to GPU in 2.21507s (16.2523GiB/s) | 20260505 11:14:41.882 | INFO | Downloaded 34GiB to GPU in 2.09448s (16.2331GiB/s) | 20260505 11:14:41.882 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 11:14:41.882 | INFO | Total mem usage: 15.3 GiB | 20260505 11:14:41.882 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 11:14:41.882 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.07%) | 20260505 11:14:41.882 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 11:14:41.882 | INFO | High Water Breakdown: | 20260505 11:14:41.882 | INFO | 61.1% : 8.83 GiB ( 93.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 11:14:41.882 | INFO | 38.7% : 5.6 GiB ( 84.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 11:14:41.882 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 11:14:41.882 | INFO | ------------------------------------------ | 20260505 11:14:41.882 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:14:41.882 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 11:14:41.882 | INFO | Total mem usage: 26.5 GiB | 20260505 11:14:41.882 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 11:14:41.882 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 11:14:41.882 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 11:14:41.882 | INFO | High Water Breakdown: | 20260505 11:14:41.882 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:14:41.882 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:14:41.882 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:14:41.882 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:14:41.882 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:14:41.882 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:14:41.882 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 11:14:41.882 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 11:14:41.882 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:14:41.882 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 11:14:41.882 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 11:14:41.882 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 11:14:41.882 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 11:14:41.882 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 11:14:41.882 | INFO | ------------------------------------------ | 20260505 11:14:41.882 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:14:41.882 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 11:14:41.882 | INFO | Total mem usage: 25.6 GiB | 20260505 11:14:41.882 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 11:14:41.882 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 11:14:41.882 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 11:14:41.882 | INFO | High Water Breakdown: | 20260505 11:14:41.882 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 11:14:41.882 | INFO | ------------------------------------------ | 20260505 11:14:41.882 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:14:41.882 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 11:14:41.882 | INFO | Total mem usage: 15.7 GiB | 20260505 11:14:41.882 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 11:14:41.882 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 11:14:41.882 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 11:14:41.882 | INFO | High Water Breakdown: | 20260505 11:14:41.882 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 11:14:41.882 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 11:14:41.882 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 11:14:41.882 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 11:14:41.882 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 11:14:41.882 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 11:14:41.882 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 11:14:41.882 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 11:14:41.882 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 11:14:41.882 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:14:41.883 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 11:14:41.883 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:14:41.883 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 11:14:41.883 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 11:14:41.883 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 11:14:41.883 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 11:14:41.883 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:14:41.883 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 11:14:41.883 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:14:41.883 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:14:41.883 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:14:41.883 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 11:14:41.883 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 11:14:41.883 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:14:41.883 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 11:14:41.883 | INFO | ------------------------------------------ | 20260505 11:14:41.883 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:14:41.883 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 11:14:41.883 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:14:41.883 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:14:41.883 | INFO | PerfReport: Full Execution -- 3.22e+01 / 7.74e+01, 1.40e+01 / 1.73e+00 | 20260505 11:14:41.883 | INFO | PerfReport: 0.29%, AggregateStats -- 9.33e-02 / 1.02e-01, 1.11e-02 / 7.15e-03 | 20260505 11:14:41.883 | INFO | PerfReport: 11.72%, Baseline -- 3.77e+00 / 4.42e+00, 6.11e-01 / 2.81e-01 | 20260505 11:14:41.883 | INFO | PerfReport: 3.31%, Histogram -- 1.07e+00 / 1.47e+00, 2.05e-01 / 1.13e-01 | 20260505 11:14:41.883 | INFO | PerfReport: 15.87%, DME -- 4.09e+01 / 4.53e+01, 4.63e+00 / 2.46e+00 | 20260505 11:14:41.883 | INFO | PerfReport: 56.39%, FrameLabeling -- 1.81e+01 / 2.14e+01, 1.58e+00 / 2.00e+00 | 20260505 11:14:41.883 | INFO | PerfReport: 7.41%, PulseAccumulating -- 2.38e+00 / 2.84e+00, 4.31e-01 / 2.47e-01 | 20260505 11:14:41.883 | INFO | PerfReport: 3.70%, Metrics -- 1.19e+00 / 1.49e+00, 3.40e-01 / 2.06e-01 | 20260505 11:14:41.883 | INFO | PerfReport: 1.31%, PulseSerializing -- 4.21e-01 / 4.64e-01, 7.79e-02 / 4.72e-02 | 20260505 11:14:41.883 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 11:14:41.883 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:14:41.883 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:14:41.883 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260505 11:14:41.883 | INFO | PerfReport: 22.64%, Upload -- 2.29e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260505 11:14:41.883 | INFO | PerfReport: 22.08%, Download -- 2.24e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260505 11:14:41.883 | INFO | PerfReport: 31.80%, Compute -- 3.22e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260505 11:14:41.883 | INFO | PerfReport: 2.02%, LockUpload -- 2.04e+00 / 0.00e+00, 7.62e+00 / 0.00e+00 | 20260505 11:14:41.883 | INFO | PerfReport: 1.32%, LockDownload -- 1.34e+00 / 0.00e+00, 2.93e+00 / 0.00e+00 | 20260505 11:14:41.883 | INFO | PerfReport: 20.13%, LockCompute -- 2.04e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260505 11:14:41.883 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 3.36e-02 / 0.00e+00 | 20260505 11:14:42.020 | INFO | Starting a round of Baz IO | 20260505 11:14:45.277 | INFO | Populating RTMetrics report | 20260505 11:14:45.277 | INFO | RTMetrics report populated | 20260505 11:14:45.277 | INFO | Opening tmp RTMetrics json file | 20260505 11:14:45.310 | INFO | Serializing RTMetrics to json | 20260505 11:14:45.361 | INFO | Serialization done, swapping tmp file now... | 20260505 11:14:45.364 | INFO | RTMetrics done | 20260505 11:14:45.450 | INFO | Finished a round of Baz IO: Wrote 2.388 GiB in 3.43 seconds | 20260505 11:14:45.450 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.32% / 0.01% / 75.67% | 20260505 11:14:45.450 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.21147% / 1.58946e-05% / 7.15256e-05% | 20260505 11:14:45.450 | INFO | Baz IO overall: | 20260505 11:14:45.450 | INFO | BytesWritten : 1634504336507 | 20260505 11:14:45.451 | INFO | NumEvents : 778247530004 | 20260505 11:14:47.031 | INFO | Analyzing chunk frames = [2768896,2769408) | 20260505 11:14:47.031 | INFO | MemInfo: { | 20260505 11:14:47.031 | INFO | "gpu" : " 17 GB", | 20260505 11:14:47.031 | INFO | "huge" : "", | 20260505 11:14:47.031 | INFO | "huge_shared" : " 27 GB", | 20260505 11:14:47.031 | INFO | "malloc" : " 14 GB", | 20260505 11:14:47.031 | INFO | "pinned" : " 29 GB" | 20260505 11:14:47.031 | INFO | } | 20260505 11:14:47.031 | INFO | Perf report for the last 5.15002 seconds: | 20260505 11:14:47.031 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:14:47.032 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:47.032 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:47.032 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:47.032 | INFO | ANALYSIS : 64.8%, 2.9, 99.9ms, 0 | 20260505 11:14:47.032 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:47.032 | INFO | RT_METRICS : 47.1%, 1.0, 24.8ms, 0 | 20260505 11:14:47.032 | INFO | PRE_HQ : 61.4%, 1.8, 59.1ms, 0 | 20260505 11:14:47.032 | INFO | BAZWRITER : 66.0%, 4.1, 139.1ms, 0 | 20260505 11:14:47.032 | INFO | BAZ_SAMPLING : 2.2%, 1.0, 1.1ms, 0 | 20260505 11:14:47.032 | INFO | Uploaded 36GiB to GPU in 2.20136s (16.3535GiB/s) | 20260505 11:14:47.032 | INFO | Downloaded 38GiB to GPU in 2.29133s (16.5843GiB/s) | 20260505 11:14:52.161 | INFO | Analyzing chunk frames = [2769408,2769920) | 20260505 11:14:52.161 | INFO | MemInfo: { | 20260505 11:14:52.161 | INFO | "gpu" : " 17 GB", | 20260505 11:14:52.161 | INFO | "huge" : "", | 20260505 11:14:52.161 | INFO | "huge_shared" : " 27 GB", | 20260505 11:14:52.161 | INFO | "malloc" : " 14 GB", | 20260505 11:14:52.161 | INFO | "pinned" : " 29 GB" | 20260505 11:14:52.161 | INFO | } | 20260505 11:14:52.161 | INFO | Perf report for the last 5.12935 seconds: | 20260505 11:14:52.161 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:14:52.161 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:52.161 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:52.161 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:52.161 | INFO | ANALYSIS : 64.3%, 3.0, 100.2ms, 0 | 20260505 11:14:52.161 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:52.161 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:52.161 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 11:14:52.161 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:52.161 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:52.161 | INFO | Uploaded 36GiB to GPU in 2.23184s (16.1302GiB/s) | 20260505 11:14:52.161 | INFO | Downloaded 34GiB to GPU in 2.10365s (16.1624GiB/s) | 20260505 11:14:57.299 | INFO | Analyzing chunk frames = [2769920,2770432) | 20260505 11:14:57.299 | INFO | MemInfo: { | 20260505 11:14:57.299 | INFO | "gpu" : " 17 GB", | 20260505 11:14:57.299 | INFO | "huge" : "", | 20260505 11:14:57.299 | INFO | "huge_shared" : " 27 GB", | 20260505 11:14:57.299 | INFO | "malloc" : " 14 GB", | 20260505 11:14:57.299 | INFO | "pinned" : " 29 GB" | 20260505 11:14:57.299 | INFO | } | 20260505 11:14:57.299 | INFO | Perf report for the last 5.13839 seconds: | 20260505 11:14:57.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:14:57.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:57.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:57.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:57.299 | INFO | ANALYSIS : 63.0%, 3.0, 98.1ms, 0 | 20260505 11:14:57.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:57.299 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:57.299 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:14:57.299 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:57.299 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:14:57.299 | INFO | Uploaded 36GiB to GPU in 2.16313s (16.6425GiB/s) | 20260505 11:14:57.299 | INFO | Downloaded 34GiB to GPU in 2.06816s (16.4398GiB/s) | 20260505 11:15:02.448 | INFO | Analyzing chunk frames = [2770432,2770944) | 20260505 11:15:02.448 | INFO | MemInfo: { | 20260505 11:15:02.448 | INFO | "gpu" : " 17 GB", | 20260505 11:15:02.448 | INFO | "huge" : "", | 20260505 11:15:02.448 | INFO | "huge_shared" : " 27 GB", | 20260505 11:15:02.448 | INFO | "malloc" : " 14 GB", | 20260505 11:15:02.448 | INFO | "pinned" : " 29 GB" | 20260505 11:15:02.448 | INFO | } | 20260505 11:15:02.448 | INFO | Perf report for the last 5.14899 seconds: | 20260505 11:15:02.448 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:15:02.448 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:02.448 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:02.448 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:02.448 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 11:15:02.448 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:02.448 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:02.448 | INFO | PRE_HQ : 29.3%, 1.0, 15.3ms, 0 | 20260505 11:15:02.448 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:02.448 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:02.448 | INFO | Uploaded 36GiB to GPU in 2.21918s (16.2222GiB/s) | 20260505 11:15:02.448 | INFO | Downloaded 34GiB to GPU in 2.13354s (15.936GiB/s) | 20260505 11:15:07.585 | INFO | Analyzing chunk frames = [2770944,2771456) | 20260505 11:15:07.585 | INFO | MemInfo: { | 20260505 11:15:07.585 | INFO | "gpu" : " 17 GB", | 20260505 11:15:07.585 | INFO | "huge" : "", | 20260505 11:15:07.585 | INFO | "huge_shared" : " 27 GB", | 20260505 11:15:07.585 | INFO | "malloc" : " 14 GB", | 20260505 11:15:07.585 | INFO | "pinned" : " 29 GB" | 20260505 11:15:07.585 | INFO | } | 20260505 11:15:07.585 | INFO | Perf report for the last 5.13722 seconds: | 20260505 11:15:07.585 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:15:07.585 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:07.585 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:07.585 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:07.585 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 11:15:07.585 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:07.585 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:07.585 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 11:15:07.585 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:07.585 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:07.586 | INFO | Uploaded 36GiB to GPU in 2.17612s (16.5432GiB/s) | 20260505 11:15:07.586 | INFO | Downloaded 34GiB to GPU in 2.12554s (15.9959GiB/s) | 20260505 11:15:12.717 | INFO | Analyzing chunk frames = [2771456,2771968) | 20260505 11:15:12.717 | INFO | MemInfo: { | 20260505 11:15:12.717 | INFO | "gpu" : " 17 GB", | 20260505 11:15:12.717 | INFO | "huge" : "", | 20260505 11:15:12.717 | INFO | "huge_shared" : " 27 GB", | 20260505 11:15:12.717 | INFO | "malloc" : " 14 GB", | 20260505 11:15:12.718 | INFO | "pinned" : " 29 GB" | 20260505 11:15:12.718 | INFO | } | 20260505 11:15:12.718 | INFO | Perf report for the last 5.1323 seconds: | 20260505 11:15:12.718 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:15:12.718 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:12.718 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:12.718 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:12.718 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 11:15:12.718 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:12.718 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:12.718 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:15:12.718 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:12.718 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:12.718 | INFO | Uploaded 36GiB to GPU in 2.16695s (16.6132GiB/s) | 20260505 11:15:12.718 | INFO | Downloaded 34GiB to GPU in 2.12211s (16.0218GiB/s) | 20260505 11:15:17.855 | INFO | Analyzing chunk frames = [2771968,2772480) | 20260505 11:15:17.855 | INFO | MemInfo: { | 20260505 11:15:17.855 | INFO | "gpu" : " 17 GB", | 20260505 11:15:17.855 | INFO | "huge" : "", | 20260505 11:15:17.855 | INFO | "huge_shared" : " 27 GB", | 20260505 11:15:17.855 | INFO | "malloc" : " 14 GB", | 20260505 11:15:17.855 | INFO | "pinned" : " 29 GB" | 20260505 11:15:17.855 | INFO | } | 20260505 11:15:17.855 | INFO | Perf report for the last 5.13722 seconds: | 20260505 11:15:17.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:15:17.855 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:17.855 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:17.855 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:17.855 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 11:15:17.855 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:17.855 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:17.855 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260505 11:15:17.855 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:17.855 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:17.855 | INFO | Uploaded 36GiB to GPU in 2.18429s (16.4813GiB/s) | 20260505 11:15:17.855 | INFO | Downloaded 34GiB to GPU in 2.13473s (15.9271GiB/s) | 20260505 11:15:23.005 | INFO | Analyzing chunk frames = [2772480,2772992) | 20260505 11:15:23.005 | INFO | MemInfo: { | 20260505 11:15:23.005 | INFO | "gpu" : " 17 GB", | 20260505 11:15:23.005 | INFO | "huge" : "", | 20260505 11:15:23.005 | INFO | "huge_shared" : " 27 GB", | 20260505 11:15:23.005 | INFO | "malloc" : " 14 GB", | 20260505 11:15:23.005 | INFO | "pinned" : " 29 GB" | 20260505 11:15:23.005 | INFO | } | 20260505 11:15:23.005 | INFO | Perf report for the last 5.15014 seconds: | 20260505 11:15:23.005 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:15:23.005 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:23.005 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:23.005 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:23.005 | INFO | ANALYSIS : 64.1%, 3.0, 100.4ms, 0 | 20260505 11:15:23.005 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:23.005 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:23.005 | INFO | PRE_HQ : 29.3%, 1.0, 15.3ms, 0 | 20260505 11:15:23.005 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:23.005 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:23.005 | INFO | Uploaded 36GiB to GPU in 2.24265s (16.0524GiB/s) | 20260505 11:15:23.005 | INFO | Downloaded 34GiB to GPU in 2.1041s (16.1589GiB/s) | 20260505 11:15:23.135 | INFO | Starting a round of Baz IO | 20260505 11:15:26.386 | INFO | Populating RTMetrics report | 20260505 11:15:26.386 | INFO | RTMetrics report populated | 20260505 11:15:26.386 | INFO | Opening tmp RTMetrics json file | 20260505 11:15:26.391 | INFO | Serializing RTMetrics to json | 20260505 11:15:26.394 | INFO | Serialization done, swapping tmp file now... | 20260505 11:15:26.395 | INFO | RTMetrics done | 20260505 11:15:26.514 | INFO | Finished a round of Baz IO: Wrote 2.385 GiB in 3.379 seconds | 20260505 11:15:26.515 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.32% / 0.01% / 75.68% | 20260505 11:15:26.515 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.20436% / 3.97364e-06% / 5.96046e-05% | 20260505 11:15:26.515 | INFO | Baz IO overall: | 20260505 11:15:26.515 | INFO | BytesWritten : 1637065303147 | 20260505 11:15:26.515 | INFO | NumEvents : 779466564428 | 20260505 11:15:28.144 | INFO | Analyzing chunk frames = [2772992,2773504) | 20260505 11:15:28.144 | INFO | MemInfo: { | 20260505 11:15:28.144 | INFO | "gpu" : " 17 GB", | 20260505 11:15:28.144 | INFO | "huge" : "", | 20260505 11:15:28.144 | INFO | "huge_shared" : " 27 GB", | 20260505 11:15:28.144 | INFO | "malloc" : " 14 GB", | 20260505 11:15:28.144 | INFO | "pinned" : " 29 GB" | 20260505 11:15:28.144 | INFO | } | 20260505 11:15:28.144 | INFO | Perf report for the last 5.1392 seconds: | 20260505 11:15:28.144 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:15:28.144 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:28.144 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:28.144 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:28.144 | INFO | ANALYSIS : 64.7%, 2.9, 99.9ms, 0 | 20260505 11:15:28.144 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:28.144 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260505 11:15:28.144 | INFO | PRE_HQ : 61.1%, 1.9, 58.9ms, 0 | 20260505 11:15:28.144 | INFO | BAZWRITER : 65.1%, 4.1, 137.2ms, 0 | 20260505 11:15:28.144 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 11:15:28.144 | INFO | Uploaded 36GiB to GPU in 2.18723s (16.4592GiB/s) | 20260505 11:15:28.145 | INFO | Downloaded 38GiB to GPU in 2.27834s (16.6788GiB/s) | 20260505 11:15:33.285 | INFO | Analyzing chunk frames = [2773504,2774016) | 20260505 11:15:33.285 | INFO | MemInfo: { | 20260505 11:15:33.285 | INFO | "gpu" : " 17 GB", | 20260505 11:15:33.285 | INFO | "huge" : "", | 20260505 11:15:33.285 | INFO | "huge_shared" : " 27 GB", | 20260505 11:15:33.285 | INFO | "malloc" : " 14 GB", | 20260505 11:15:33.285 | INFO | "pinned" : " 29 GB" | 20260505 11:15:33.285 | INFO | } | 20260505 11:15:33.285 | INFO | Perf report for the last 5.14059 seconds: | 20260505 11:15:33.285 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:15:33.285 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:33.285 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:33.285 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:33.285 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 11:15:33.285 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:33.285 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:33.285 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 11:15:33.285 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:33.285 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:33.285 | INFO | Uploaded 36GiB to GPU in 2.17282s (16.5683GiB/s) | 20260505 11:15:33.285 | INFO | Downloaded 34GiB to GPU in 2.04361s (16.6372GiB/s) | 20260505 11:15:38.412 | INFO | Analyzing chunk frames = [2774016,2774528) | 20260505 11:15:38.412 | INFO | MemInfo: { | 20260505 11:15:38.412 | INFO | "gpu" : " 17 GB", | 20260505 11:15:38.412 | INFO | "huge" : "", | 20260505 11:15:38.412 | INFO | "huge_shared" : " 27 GB", | 20260505 11:15:38.412 | INFO | "malloc" : " 14 GB", | 20260505 11:15:38.412 | INFO | "pinned" : " 29 GB" | 20260505 11:15:38.412 | INFO | } | 20260505 11:15:38.412 | INFO | Perf report for the last 5.12758 seconds: | 20260505 11:15:38.412 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:15:38.412 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:38.412 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:38.412 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:38.412 | INFO | ANALYSIS : 63.6%, 3.0, 98.7ms, 0 | 20260505 11:15:38.412 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:38.412 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:38.413 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 11:15:38.413 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:38.413 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:38.413 | INFO | Uploaded 36GiB to GPU in 2.23187s (16.13GiB/s) | 20260505 11:15:38.413 | INFO | Downloaded 34GiB to GPU in 2.12597s (15.9927GiB/s) | 20260505 11:15:43.551 | INFO | Analyzing chunk frames = [2774528,2775040) | 20260505 11:15:43.551 | INFO | MemInfo: { | 20260505 11:15:43.551 | INFO | "gpu" : " 17 GB", | 20260505 11:15:43.551 | INFO | "huge" : "", | 20260505 11:15:43.551 | INFO | "huge_shared" : " 27 GB", | 20260505 11:15:43.551 | INFO | "malloc" : " 14 GB", | 20260505 11:15:43.551 | INFO | "pinned" : " 29 GB" | 20260505 11:15:43.551 | INFO | } | 20260505 11:15:43.551 | INFO | Perf report for the last 5.13888 seconds: | 20260505 11:15:43.551 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:15:43.551 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:43.551 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:43.551 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:43.551 | INFO | ANALYSIS : 63.0%, 3.0, 97.8ms, 0 | 20260505 11:15:43.551 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:43.551 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:43.551 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 11:15:43.551 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:43.551 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:43.552 | INFO | Uploaded 36GiB to GPU in 2.17174s (16.5766GiB/s) | 20260505 11:15:43.552 | INFO | Downloaded 34GiB to GPU in 2.07833s (16.3593GiB/s) | 20260505 11:15:48.697 | INFO | Analyzing chunk frames = [2775040,2775552) | 20260505 11:15:48.698 | INFO | MemInfo: { | 20260505 11:15:48.698 | INFO | "gpu" : " 17 GB", | 20260505 11:15:48.698 | INFO | "huge" : "", | 20260505 11:15:48.698 | INFO | "huge_shared" : " 27 GB", | 20260505 11:15:48.698 | INFO | "malloc" : " 14 GB", | 20260505 11:15:48.698 | INFO | "pinned" : " 29 GB" | 20260505 11:15:48.698 | INFO | } | 20260505 11:15:48.698 | INFO | Perf report for the last 5.14648 seconds: | 20260505 11:15:48.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:15:48.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:48.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:48.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:48.698 | INFO | ANALYSIS : 63.2%, 3.0, 98.9ms, 0 | 20260505 11:15:48.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:48.698 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:48.698 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 11:15:48.698 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:48.698 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:48.698 | INFO | Uploaded 36GiB to GPU in 2.22223s (16.1999GiB/s) | 20260505 11:15:48.698 | INFO | Downloaded 34GiB to GPU in 2.12235s (16.02GiB/s) | 20260505 11:15:53.838 | INFO | Analyzing chunk frames = [2775552,2776064) | 20260505 11:15:53.838 | INFO | MemInfo: { | 20260505 11:15:53.838 | INFO | "gpu" : " 17 GB", | 20260505 11:15:53.838 | INFO | "huge" : "", | 20260505 11:15:53.838 | INFO | "huge_shared" : " 27 GB", | 20260505 11:15:53.838 | INFO | "malloc" : " 14 GB", | 20260505 11:15:53.838 | INFO | "pinned" : " 29 GB" | 20260505 11:15:53.838 | INFO | } | 20260505 11:15:53.838 | INFO | Perf report for the last 5.14047 seconds: | 20260505 11:15:53.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:15:53.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:53.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:53.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:53.838 | INFO | ANALYSIS : 63.2%, 3.0, 98.8ms, 0 | 20260505 11:15:53.838 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:53.838 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:53.838 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:15:53.838 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:53.838 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:53.839 | INFO | Uploaded 36GiB to GPU in 2.15788s (16.6831GiB/s) | 20260505 11:15:53.839 | INFO | Downloaded 34GiB to GPU in 2.09307s (16.2441GiB/s) | 20260505 11:15:58.982 | INFO | Analyzing chunk frames = [2776064,2776576) | 20260505 11:15:58.982 | INFO | MemInfo: { | 20260505 11:15:58.982 | INFO | "gpu" : " 17 GB", | 20260505 11:15:58.982 | INFO | "huge" : "", | 20260505 11:15:58.982 | INFO | "huge_shared" : " 27 GB", | 20260505 11:15:58.982 | INFO | "malloc" : " 14 GB", | 20260505 11:15:58.982 | INFO | "pinned" : " 29 GB" | 20260505 11:15:58.982 | INFO | } | 20260505 11:15:58.982 | INFO | Perf report for the last 5.14372 seconds: | 20260505 11:15:58.982 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:15:58.982 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:58.982 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:58.982 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:58.982 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 11:15:58.982 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:58.982 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:58.982 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 11:15:58.982 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:58.982 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:15:58.982 | INFO | Uploaded 36GiB to GPU in 2.16682s (16.6142GiB/s) | 20260505 11:15:58.982 | INFO | Downloaded 34GiB to GPU in 2.12788s (15.9783GiB/s) | 20260505 11:16:04.113 | INFO | Analyzing chunk frames = [2776576,2777088) | 20260505 11:16:04.113 | INFO | MemInfo: { | 20260505 11:16:04.113 | INFO | "gpu" : " 17 GB", | 20260505 11:16:04.113 | INFO | "huge" : "", | 20260505 11:16:04.113 | INFO | "huge_shared" : " 27 GB", | 20260505 11:16:04.113 | INFO | "malloc" : " 14 GB", | 20260505 11:16:04.113 | INFO | "pinned" : " 29 GB" | 20260505 11:16:04.113 | INFO | } | 20260505 11:16:04.113 | INFO | Perf report for the last 5.13132 seconds: | 20260505 11:16:04.113 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:16:04.113 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:04.113 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:04.113 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:04.113 | INFO | ANALYSIS : 64.0%, 3.0, 99.7ms, 0 | 20260505 11:16:04.113 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:04.113 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:04.113 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 11:16:04.113 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:04.113 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:04.114 | INFO | Uploaded 36GiB to GPU in 2.22838s (16.1552GiB/s) | 20260505 11:16:04.114 | INFO | Downloaded 34GiB to GPU in 2.09619s (16.2199GiB/s) | 20260505 11:16:04.114 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 11:16:04.114 | INFO | Total mem usage: 15.3 GiB | 20260505 11:16:04.114 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 11:16:04.114 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -6.96%) | 20260505 11:16:04.114 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 11:16:04.114 | INFO | High Water Breakdown: | 20260505 11:16:04.114 | INFO | 61% : 8.74 GiB ( 92.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 11:16:04.114 | INFO | 38.9% : 5.57 GiB ( 83.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 11:16:04.114 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 11:16:04.114 | INFO | ------------------------------------------ | 20260505 11:16:04.114 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:16:04.114 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 11:16:04.114 | INFO | Total mem usage: 26.5 GiB | 20260505 11:16:04.114 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 11:16:04.114 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 11:16:04.114 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 11:16:04.114 | INFO | High Water Breakdown: | 20260505 11:16:04.114 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:16:04.114 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:16:04.114 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:16:04.114 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:16:04.114 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:16:04.114 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:16:04.114 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 11:16:04.114 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 11:16:04.114 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:16:04.114 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 11:16:04.114 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 11:16:04.114 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 11:16:04.114 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 11:16:04.114 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 11:16:04.114 | INFO | ------------------------------------------ | 20260505 11:16:04.114 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:16:04.114 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 11:16:04.114 | INFO | Total mem usage: 25.6 GiB | 20260505 11:16:04.114 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 11:16:04.114 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 11:16:04.114 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 11:16:04.114 | INFO | High Water Breakdown: | 20260505 11:16:04.114 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 11:16:04.114 | INFO | ------------------------------------------ | 20260505 11:16:04.114 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:16:04.114 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 11:16:04.114 | INFO | Total mem usage: 15.7 GiB | 20260505 11:16:04.114 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 11:16:04.114 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 11:16:04.114 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 11:16:04.114 | INFO | High Water Breakdown: | 20260505 11:16:04.114 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 11:16:04.114 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 11:16:04.114 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 11:16:04.114 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 11:16:04.114 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 11:16:04.114 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 11:16:04.114 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 11:16:04.115 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 11:16:04.115 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 11:16:04.115 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:16:04.115 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 11:16:04.115 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:16:04.115 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 11:16:04.115 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 11:16:04.115 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 11:16:04.115 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 11:16:04.115 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:16:04.115 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 11:16:04.115 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:16:04.115 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:16:04.115 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:16:04.115 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 11:16:04.115 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 11:16:04.115 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:16:04.115 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 11:16:04.115 | INFO | ------------------------------------------ | 20260505 11:16:04.115 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:16:04.115 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 11:16:04.115 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:16:04.115 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:16:04.115 | INFO | PerfReport: Full Execution -- 3.22e+01 / 7.71e+01, 1.35e+01 / 1.47e+00 | 20260505 11:16:04.115 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 1.02e-01, 1.03e-02 / 3.31e-03 | 20260505 11:16:04.115 | INFO | PerfReport: 11.72%, Baseline -- 3.77e+00 / 4.51e+00, 6.07e-01 / 1.65e-01 | 20260505 11:16:04.115 | INFO | PerfReport: 3.31%, Histogram -- 1.06e+00 / 1.51e+00, 2.07e-01 / 8.99e-02 | 20260505 11:16:04.115 | INFO | PerfReport: 15.79%, DME -- 4.07e+01 / 4.50e+01, 4.23e+00 / 1.62e+00 | 20260505 11:16:04.115 | INFO | PerfReport: 56.49%, FrameLabeling -- 1.82e+01 / 2.11e+01, 1.68e+00 / 1.91e+00 | 20260505 11:16:04.115 | INFO | PerfReport: 7.40%, PulseAccumulating -- 2.38e+00 / 2.88e+00, 4.24e-01 / 2.19e-01 | 20260505 11:16:04.115 | INFO | PerfReport: 3.70%, Metrics -- 1.19e+00 / 1.47e+00, 3.39e-01 / 1.66e-01 | 20260505 11:16:04.115 | INFO | PerfReport: 1.30%, PulseSerializing -- 4.20e-01 / 4.64e-01, 7.73e-02 / 4.34e-02 | 20260505 11:16:04.115 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 11:16:04.115 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:16:04.115 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:16:04.115 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.47e+01 / 0.00e+00 | 20260505 11:16:04.115 | INFO | PerfReport: 22.59%, Upload -- 2.29e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260505 11:16:04.115 | INFO | PerfReport: 21.90%, Download -- 2.22e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260505 11:16:04.115 | INFO | PerfReport: 31.80%, Compute -- 3.22e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 11:16:04.115 | INFO | PerfReport: 2.01%, LockUpload -- 2.04e+00 / 0.00e+00, 7.61e+00 / 0.00e+00 | 20260505 11:16:04.115 | INFO | PerfReport: 1.22%, LockDownload -- 1.23e+00 / 0.00e+00, 2.77e+00 / 0.00e+00 | 20260505 11:16:04.115 | INFO | PerfReport: 20.48%, LockCompute -- 2.07e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260505 11:16:04.115 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.87e-02 / 0.00e+00 | 20260505 11:16:04.252 | INFO | Starting a round of Baz IO | 20260505 11:16:07.523 | INFO | Populating RTMetrics report | 20260505 11:16:07.524 | INFO | RTMetrics report populated | 20260505 11:16:07.524 | INFO | Opening tmp RTMetrics json file | 20260505 11:16:07.541 | INFO | Serializing RTMetrics to json | 20260505 11:16:07.543 | INFO | Serialization done, swapping tmp file now... | 20260505 11:16:07.543 | INFO | RTMetrics done | 20260505 11:16:07.658 | INFO | Finished a round of Baz IO: Wrote 2.384 GiB in 3.405 seconds | 20260505 11:16:07.658 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.31% / 0.01% / 75.68% | 20260505 11:16:07.658 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.20107% / 3.97364e-06% / 7.94729e-05% | 20260505 11:16:07.658 | INFO | Baz IO overall: | 20260505 11:16:07.658 | INFO | BytesWritten : 1639624644708 | 20260505 11:16:07.658 | INFO | NumEvents : 780684298023 | 20260505 11:16:09.256 | INFO | Analyzing chunk frames = [2777088,2777600) | 20260505 11:16:09.256 | INFO | MemInfo: { | 20260505 11:16:09.256 | INFO | "gpu" : " 17 GB", | 20260505 11:16:09.256 | INFO | "huge" : "", | 20260505 11:16:09.256 | INFO | "huge_shared" : " 27 GB", | 20260505 11:16:09.256 | INFO | "malloc" : " 14 GB", | 20260505 11:16:09.256 | INFO | "pinned" : " 29 GB" | 20260505 11:16:09.256 | INFO | } | 20260505 11:16:09.256 | INFO | Perf report for the last 5.14255 seconds: | 20260505 11:16:09.256 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:16:09.256 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:09.256 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:09.256 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:09.256 | INFO | ANALYSIS : 65.2%, 2.9, 100.4ms, 0 | 20260505 11:16:09.256 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:09.256 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260505 11:16:09.256 | INFO | PRE_HQ : 61.7%, 1.8, 59.1ms, 0 | 20260505 11:16:09.256 | INFO | BAZWRITER : 65.6%, 4.2, 140.2ms, 0 | 20260505 11:16:09.256 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 11:16:09.256 | INFO | Uploaded 36GiB to GPU in 2.23257s (16.1249GiB/s) | 20260505 11:16:09.256 | INFO | Downloaded 38GiB to GPU in 2.29964s (16.5243GiB/s) | 20260505 11:16:14.392 | INFO | Analyzing chunk frames = [2777600,2778112) | 20260505 11:16:14.392 | INFO | MemInfo: { | 20260505 11:16:14.392 | INFO | "gpu" : " 17 GB", | 20260505 11:16:14.392 | INFO | "huge" : "", | 20260505 11:16:14.392 | INFO | "huge_shared" : " 27 GB", | 20260505 11:16:14.392 | INFO | "malloc" : " 14 GB", | 20260505 11:16:14.392 | INFO | "pinned" : " 29 GB" | 20260505 11:16:14.392 | INFO | } | 20260505 11:16:14.392 | INFO | Perf report for the last 5.13631 seconds: | 20260505 11:16:14.392 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:16:14.392 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:14.392 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:14.392 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:14.392 | INFO | ANALYSIS : 64.2%, 3.0, 100.1ms, 0 | 20260505 11:16:14.392 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:14.392 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:14.392 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:16:14.392 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:14.392 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:14.393 | INFO | Uploaded 36GiB to GPU in 2.22843s (16.1549GiB/s) | 20260505 11:16:14.393 | INFO | Downloaded 34GiB to GPU in 2.10074s (16.1848GiB/s) | 20260505 11:16:19.540 | INFO | Analyzing chunk frames = [2778112,2778624) | 20260505 11:16:19.540 | INFO | MemInfo: { | 20260505 11:16:19.540 | INFO | "gpu" : " 17 GB", | 20260505 11:16:19.540 | INFO | "huge" : "", | 20260505 11:16:19.540 | INFO | "huge_shared" : " 27 GB", | 20260505 11:16:19.540 | INFO | "malloc" : " 14 GB", | 20260505 11:16:19.540 | INFO | "pinned" : " 29 GB" | 20260505 11:16:19.540 | INFO | } | 20260505 11:16:19.540 | INFO | Perf report for the last 5.14809 seconds: | 20260505 11:16:19.540 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:16:19.541 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:19.541 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:19.541 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:19.541 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 11:16:19.541 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:19.541 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:19.541 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 11:16:19.541 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:19.541 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:19.541 | INFO | Uploaded 36GiB to GPU in 2.22082s (16.2102GiB/s) | 20260505 11:16:19.541 | INFO | Downloaded 34GiB to GPU in 2.13008s (15.9618GiB/s) | 20260505 11:16:24.668 | INFO | Analyzing chunk frames = [2778624,2779136) | 20260505 11:16:24.669 | INFO | MemInfo: { | 20260505 11:16:24.669 | INFO | "gpu" : " 17 GB", | 20260505 11:16:24.669 | INFO | "huge" : "", | 20260505 11:16:24.669 | INFO | "huge_shared" : " 27 GB", | 20260505 11:16:24.669 | INFO | "malloc" : " 14 GB", | 20260505 11:16:24.669 | INFO | "pinned" : " 29 GB" | 20260505 11:16:24.669 | INFO | } | 20260505 11:16:24.669 | INFO | Perf report for the last 5.12841 seconds: | 20260505 11:16:24.669 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:16:24.669 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:24.669 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:24.669 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:24.669 | INFO | ANALYSIS : 63.0%, 3.0, 97.7ms, 0 | 20260505 11:16:24.669 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:24.669 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:24.669 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 11:16:24.669 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:24.669 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:24.669 | INFO | Uploaded 36GiB to GPU in 2.15498s (16.7055GiB/s) | 20260505 11:16:24.669 | INFO | Downloaded 34GiB to GPU in 2.0749s (16.3864GiB/s) | 20260505 11:16:29.807 | INFO | Analyzing chunk frames = [2779136,2779648) | 20260505 11:16:29.807 | INFO | MemInfo: { | 20260505 11:16:29.807 | INFO | "gpu" : " 17 GB", | 20260505 11:16:29.807 | INFO | "huge" : "", | 20260505 11:16:29.807 | INFO | "huge_shared" : " 27 GB", | 20260505 11:16:29.807 | INFO | "malloc" : " 14 GB", | 20260505 11:16:29.807 | INFO | "pinned" : " 29 GB" | 20260505 11:16:29.807 | INFO | } | 20260505 11:16:29.807 | INFO | Perf report for the last 5.13795 seconds: | 20260505 11:16:29.807 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:16:29.807 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:29.807 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:29.807 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:29.807 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 11:16:29.807 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:29.807 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:29.807 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 11:16:29.807 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:29.807 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:29.807 | INFO | Uploaded 36GiB to GPU in 2.22924s (16.149GiB/s) | 20260505 11:16:29.807 | INFO | Downloaded 34GiB to GPU in 2.1087s (16.1237GiB/s) | 20260505 11:16:34.960 | INFO | Analyzing chunk frames = [2779648,2780160) | 20260505 11:16:34.960 | INFO | MemInfo: { | 20260505 11:16:34.960 | INFO | "gpu" : " 17 GB", | 20260505 11:16:34.960 | INFO | "huge" : "", | 20260505 11:16:34.960 | INFO | "huge_shared" : " 27 GB", | 20260505 11:16:34.960 | INFO | "malloc" : " 14 GB", | 20260505 11:16:34.960 | INFO | "pinned" : " 29 GB" | 20260505 11:16:34.960 | INFO | } | 20260505 11:16:34.960 | INFO | Perf report for the last 5.15363 seconds: | 20260505 11:16:34.960 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:16:34.960 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:34.960 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:34.960 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:34.960 | INFO | ANALYSIS : 63.4%, 3.0, 99.3ms, 0 | 20260505 11:16:34.960 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:34.960 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:34.960 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 11:16:34.960 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:34.960 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:34.961 | INFO | Uploaded 36GiB to GPU in 2.21737s (16.2355GiB/s) | 20260505 11:16:34.961 | INFO | Downloaded 34GiB to GPU in 2.10577s (16.1461GiB/s) | 20260505 11:16:40.098 | INFO | Analyzing chunk frames = [2780160,2780672) | 20260505 11:16:40.098 | INFO | MemInfo: { | 20260505 11:16:40.098 | INFO | "gpu" : " 17 GB", | 20260505 11:16:40.098 | INFO | "huge" : "", | 20260505 11:16:40.098 | INFO | "huge_shared" : " 27 GB", | 20260505 11:16:40.098 | INFO | "malloc" : " 14 GB", | 20260505 11:16:40.099 | INFO | "pinned" : " 29 GB" | 20260505 11:16:40.099 | INFO | } | 20260505 11:16:40.099 | INFO | Perf report for the last 5.13825 seconds: | 20260505 11:16:40.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:16:40.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:40.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:40.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:40.099 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 11:16:40.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:40.099 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:40.099 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 11:16:40.099 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:40.099 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:40.099 | INFO | Uploaded 36GiB to GPU in 2.1551s (16.7046GiB/s) | 20260505 11:16:40.099 | INFO | Downloaded 34GiB to GPU in 2.13003s (15.9622GiB/s) | 20260505 11:16:45.228 | INFO | Analyzing chunk frames = [2780672,2781184) | 20260505 11:16:45.228 | INFO | MemInfo: { | 20260505 11:16:45.228 | INFO | "gpu" : " 17 GB", | 20260505 11:16:45.228 | INFO | "huge" : "", | 20260505 11:16:45.228 | INFO | "huge_shared" : " 27 GB", | 20260505 11:16:45.229 | INFO | "malloc" : " 14 GB", | 20260505 11:16:45.229 | INFO | "pinned" : " 29 GB" | 20260505 11:16:45.229 | INFO | } | 20260505 11:16:45.229 | INFO | Perf report for the last 5.13008 seconds: | 20260505 11:16:45.229 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:16:45.229 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:45.229 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:45.229 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:45.229 | INFO | ANALYSIS : 64.0%, 3.0, 99.7ms, 0 | 20260505 11:16:45.229 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:45.229 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:45.229 | INFO | PRE_HQ : 29.8%, 1.0, 15.4ms, 0 | 20260505 11:16:45.229 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:45.229 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:45.229 | INFO | Uploaded 36GiB to GPU in 2.22424s (16.1853GiB/s) | 20260505 11:16:45.229 | INFO | Downloaded 34GiB to GPU in 2.10608s (16.1437GiB/s) | 20260505 11:16:45.368 | INFO | Starting a round of Baz IO | 20260505 11:16:48.625 | INFO | Populating RTMetrics report | 20260505 11:16:48.625 | INFO | RTMetrics report populated | 20260505 11:16:48.625 | INFO | Opening tmp RTMetrics json file | 20260505 11:16:48.627 | INFO | Serializing RTMetrics to json | 20260505 11:16:48.628 | INFO | Serialization done, swapping tmp file now... | 20260505 11:16:48.629 | INFO | RTMetrics done | 20260505 11:16:48.766 | INFO | Finished a round of Baz IO: Wrote 2.386 GiB in 3.398 seconds | 20260505 11:16:48.766 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.30% / 0.01% / 75.69% | 20260505 11:16:48.766 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.19993% / 2.38419e-05% / 3.97364e-05% | 20260505 11:16:48.766 | INFO | Baz IO overall: | 20260505 11:16:48.766 | INFO | BytesWritten : 1642186953302 | 20260505 11:16:48.766 | INFO | NumEvents : 781904374847 | 20260505 11:16:50.360 | INFO | Analyzing chunk frames = [2781184,2781696) | 20260505 11:16:50.360 | INFO | MemInfo: { | 20260505 11:16:50.360 | INFO | "gpu" : " 17 GB", | 20260505 11:16:50.360 | INFO | "huge" : "", | 20260505 11:16:50.360 | INFO | "huge_shared" : " 27 GB", | 20260505 11:16:50.360 | INFO | "malloc" : " 14 GB", | 20260505 11:16:50.361 | INFO | "pinned" : " 29 GB" | 20260505 11:16:50.361 | INFO | } | 20260505 11:16:50.361 | INFO | Perf report for the last 5.13194 seconds: | 20260505 11:16:50.361 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:16:50.361 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:50.361 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:50.361 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:50.361 | INFO | ANALYSIS : 65.1%, 2.9, 99.7ms, 0 | 20260505 11:16:50.361 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:50.361 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260505 11:16:50.361 | INFO | PRE_HQ : 61.6%, 1.9, 59.1ms, 0 | 20260505 11:16:50.361 | INFO | BAZWRITER : 65.5%, 4.2, 140.1ms, 0 | 20260505 11:16:50.361 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 11:16:50.361 | INFO | Uploaded 36GiB to GPU in 2.18204s (16.4983GiB/s) | 20260505 11:16:50.361 | INFO | Downloaded 38GiB to GPU in 2.30152s (16.5108GiB/s) | 20260505 11:16:55.507 | INFO | Analyzing chunk frames = [2781696,2782208) | 20260505 11:16:55.507 | INFO | MemInfo: { | 20260505 11:16:55.507 | INFO | "gpu" : " 17 GB", | 20260505 11:16:55.507 | INFO | "huge" : "", | 20260505 11:16:55.507 | INFO | "huge_shared" : " 27 GB", | 20260505 11:16:55.507 | INFO | "malloc" : " 14 GB", | 20260505 11:16:55.507 | INFO | "pinned" : " 29 GB" | 20260505 11:16:55.507 | INFO | } | 20260505 11:16:55.508 | INFO | Perf report for the last 5.14688 seconds: | 20260505 11:16:55.508 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:16:55.508 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:55.508 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:55.508 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:55.508 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 11:16:55.508 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:55.508 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:55.508 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 11:16:55.508 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:55.508 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:16:55.508 | INFO | Uploaded 36GiB to GPU in 2.17815s (16.5278GiB/s) | 20260505 11:16:55.508 | INFO | Downloaded 34GiB to GPU in 2.08603s (16.2989GiB/s) | 20260505 11:17:00.651 | INFO | Analyzing chunk frames = [2782208,2782720) | 20260505 11:17:00.651 | INFO | MemInfo: { | 20260505 11:17:00.651 | INFO | "gpu" : " 17 GB", | 20260505 11:17:00.651 | INFO | "huge" : "", | 20260505 11:17:00.651 | INFO | "huge_shared" : " 27 GB", | 20260505 11:17:00.651 | INFO | "malloc" : " 14 GB", | 20260505 11:17:00.651 | INFO | "pinned" : " 29 GB" | 20260505 11:17:00.651 | INFO | } | 20260505 11:17:00.651 | INFO | Perf report for the last 5.14339 seconds: | 20260505 11:17:00.651 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:17:00.651 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:00.651 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:00.651 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:00.651 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 11:17:00.651 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:00.651 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:00.651 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 11:17:00.651 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:00.651 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:00.651 | INFO | Uploaded 36GiB to GPU in 2.21182s (16.2762GiB/s) | 20260505 11:17:00.651 | INFO | Downloaded 34GiB to GPU in 2.11027s (16.1117GiB/s) | 20260505 11:17:05.792 | INFO | Analyzing chunk frames = [2782720,2783232) | 20260505 11:17:05.792 | INFO | MemInfo: { | 20260505 11:17:05.792 | INFO | "gpu" : " 17 GB", | 20260505 11:17:05.792 | INFO | "huge" : "", | 20260505 11:17:05.792 | INFO | "huge_shared" : " 27 GB", | 20260505 11:17:05.792 | INFO | "malloc" : " 14 GB", | 20260505 11:17:05.792 | INFO | "pinned" : " 29 GB" | 20260505 11:17:05.792 | INFO | } | 20260505 11:17:05.792 | INFO | Perf report for the last 5.14145 seconds: | 20260505 11:17:05.792 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:17:05.792 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:05.792 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:05.792 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:05.792 | INFO | ANALYSIS : 63.4%, 3.0, 98.5ms, 0 | 20260505 11:17:05.792 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:05.792 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:05.793 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:17:05.793 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:05.793 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:05.793 | INFO | Uploaded 36GiB to GPU in 2.21055s (16.2856GiB/s) | 20260505 11:17:05.793 | INFO | Downloaded 34GiB to GPU in 2.13207s (15.947GiB/s) | 20260505 11:17:10.927 | INFO | Analyzing chunk frames = [2783232,2783744) | 20260505 11:17:10.927 | INFO | MemInfo: { | 20260505 11:17:10.927 | INFO | "gpu" : " 17 GB", | 20260505 11:17:10.927 | INFO | "huge" : "", | 20260505 11:17:10.927 | INFO | "huge_shared" : " 27 GB", | 20260505 11:17:10.927 | INFO | "malloc" : " 14 GB", | 20260505 11:17:10.927 | INFO | "pinned" : " 29 GB" | 20260505 11:17:10.927 | INFO | } | 20260505 11:17:10.927 | INFO | Perf report for the last 5.1349 seconds: | 20260505 11:17:10.927 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:17:10.927 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:10.927 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:10.927 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:10.927 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 11:17:10.927 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:10.927 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:10.927 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 11:17:10.927 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:10.927 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:10.928 | INFO | Uploaded 36GiB to GPU in 2.15491s (16.706GiB/s) | 20260505 11:17:10.928 | INFO | Downloaded 34GiB to GPU in 2.12053s (16.0337GiB/s) | 20260505 11:17:16.066 | INFO | Analyzing chunk frames = [2783744,2784256) | 20260505 11:17:16.066 | INFO | MemInfo: { | 20260505 11:17:16.066 | INFO | "gpu" : " 17 GB", | 20260505 11:17:16.066 | INFO | "huge" : "", | 20260505 11:17:16.066 | INFO | "huge_shared" : " 27 GB", | 20260505 11:17:16.066 | INFO | "malloc" : " 14 GB", | 20260505 11:17:16.066 | INFO | "pinned" : " 29 GB" | 20260505 11:17:16.066 | INFO | } | 20260505 11:17:16.066 | INFO | Perf report for the last 5.13908 seconds: | 20260505 11:17:16.067 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:17:16.067 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:16.067 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:16.067 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:16.067 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 11:17:16.067 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:16.067 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:16.067 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:17:16.067 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:16.067 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:16.067 | INFO | Uploaded 36GiB to GPU in 2.20987s (16.2905GiB/s) | 20260505 11:17:16.067 | INFO | Downloaded 34GiB to GPU in 2.10439s (16.1567GiB/s) | 20260505 11:17:21.211 | INFO | Analyzing chunk frames = [2784256,2784768) | 20260505 11:17:21.211 | INFO | MemInfo: { | 20260505 11:17:21.211 | INFO | "gpu" : " 17 GB", | 20260505 11:17:21.211 | INFO | "huge" : "", | 20260505 11:17:21.211 | INFO | "huge_shared" : " 27 GB", | 20260505 11:17:21.211 | INFO | "malloc" : " 14 GB", | 20260505 11:17:21.211 | INFO | "pinned" : " 29 GB" | 20260505 11:17:21.211 | INFO | } | 20260505 11:17:21.211 | INFO | Perf report for the last 5.14459 seconds: | 20260505 11:17:21.211 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:17:21.211 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:21.211 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:21.211 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:21.211 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 11:17:21.211 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:21.211 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:21.211 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 11:17:21.211 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:21.211 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:21.211 | INFO | Uploaded 36GiB to GPU in 2.16833s (16.6026GiB/s) | 20260505 11:17:21.211 | INFO | Downloaded 34GiB to GPU in 2.12306s (16.0146GiB/s) | 20260505 11:17:26.348 | INFO | Analyzing chunk frames = [2784768,2785280) | 20260505 11:17:26.348 | INFO | MemInfo: { | 20260505 11:17:26.348 | INFO | "gpu" : " 17 GB", | 20260505 11:17:26.348 | INFO | "huge" : "", | 20260505 11:17:26.348 | INFO | "huge_shared" : " 27 GB", | 20260505 11:17:26.348 | INFO | "malloc" : " 14 GB", | 20260505 11:17:26.348 | INFO | "pinned" : " 29 GB" | 20260505 11:17:26.348 | INFO | } | 20260505 11:17:26.348 | INFO | Perf report for the last 5.13728 seconds: | 20260505 11:17:26.348 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:17:26.348 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:26.348 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:26.348 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:26.348 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 11:17:26.348 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:26.348 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:26.348 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 11:17:26.348 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:26.348 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:26.349 | INFO | Uploaded 36GiB to GPU in 2.21779s (16.2324GiB/s) | 20260505 11:17:26.349 | INFO | Downloaded 34GiB to GPU in 2.10524s (16.1502GiB/s) | 20260505 11:17:26.349 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 11:17:26.349 | INFO | Total mem usage: 15.3 GiB | 20260505 11:17:26.349 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 11:17:26.349 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -6.9%) | 20260505 11:17:26.349 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 11:17:26.349 | INFO | High Water Breakdown: | 20260505 11:17:26.349 | INFO | 61% : 8.75 GiB ( 92.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 11:17:26.349 | INFO | 38.9% : 5.57 GiB ( 83.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 11:17:26.349 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 11:17:26.349 | INFO | ------------------------------------------ | 20260505 11:17:26.349 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:17:26.349 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 11:17:26.349 | INFO | Total mem usage: 26.5 GiB | 20260505 11:17:26.349 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 11:17:26.349 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 11:17:26.349 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 11:17:26.349 | INFO | High Water Breakdown: | 20260505 11:17:26.349 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:17:26.349 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:17:26.349 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:17:26.349 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:17:26.349 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:17:26.349 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:17:26.349 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 11:17:26.349 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 11:17:26.349 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:17:26.349 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 11:17:26.349 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 11:17:26.349 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 11:17:26.349 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 11:17:26.349 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 11:17:26.349 | INFO | ------------------------------------------ | 20260505 11:17:26.349 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:17:26.349 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 11:17:26.349 | INFO | Total mem usage: 25.6 GiB | 20260505 11:17:26.349 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 11:17:26.349 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 11:17:26.349 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 11:17:26.349 | INFO | High Water Breakdown: | 20260505 11:17:26.349 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 11:17:26.349 | INFO | ------------------------------------------ | 20260505 11:17:26.349 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:17:26.349 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 11:17:26.349 | INFO | Total mem usage: 15.7 GiB | 20260505 11:17:26.349 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 11:17:26.349 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 11:17:26.349 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 11:17:26.349 | INFO | High Water Breakdown: | 20260505 11:17:26.349 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 11:17:26.349 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 11:17:26.349 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 11:17:26.349 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 11:17:26.349 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 11:17:26.349 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 11:17:26.349 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 11:17:26.350 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 11:17:26.350 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 11:17:26.350 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:17:26.350 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 11:17:26.350 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:17:26.350 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 11:17:26.350 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 11:17:26.350 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 11:17:26.350 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 11:17:26.350 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:17:26.350 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 11:17:26.350 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:17:26.350 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:17:26.350 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:17:26.350 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 11:17:26.350 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 11:17:26.350 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:17:26.350 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 11:17:26.350 | INFO | ------------------------------------------ | 20260505 11:17:26.350 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:17:26.350 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 11:17:26.350 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:17:26.350 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:17:26.350 | INFO | PerfReport: Full Execution -- 3.22e+01 / 7.77e+01, 1.39e+01 / 1.63e+00 | 20260505 11:17:26.350 | INFO | PerfReport: 0.29%, AggregateStats -- 9.30e-02 / 1.02e-01, 1.02e-02 / 3.99e-03 | 20260505 11:17:26.350 | INFO | PerfReport: 11.71%, Baseline -- 3.77e+00 / 4.54e+00, 6.03e-01 / 1.51e-01 | 20260505 11:17:26.350 | INFO | PerfReport: 3.32%, Histogram -- 1.07e+00 / 1.50e+00, 2.06e-01 / 8.81e-02 | 20260505 11:17:26.350 | INFO | PerfReport: 15.68%, DME -- 4.04e+01 / 4.47e+01, 4.41e+00 / 2.00e+00 | 20260505 11:17:26.350 | INFO | PerfReport: 56.67%, FrameLabeling -- 1.83e+01 / 2.21e+01, 1.73e+00 / 2.26e+00 | 20260505 11:17:26.350 | INFO | PerfReport: 7.34%, PulseAccumulating -- 2.36e+00 / 2.82e+00, 4.07e-01 / 2.27e-01 | 20260505 11:17:26.350 | INFO | PerfReport: 3.70%, Metrics -- 1.19e+00 / 1.48e+00, 3.41e-01 / 1.88e-01 | 20260505 11:17:26.350 | INFO | PerfReport: 1.30%, PulseSerializing -- 4.20e-01 / 4.70e-01, 7.70e-02 / 3.41e-02 | 20260505 11:17:26.350 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 11:17:26.350 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:17:26.350 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:17:26.350 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.44e+01 / 0.00e+00 | 20260505 11:17:26.350 | INFO | PerfReport: 22.63%, Upload -- 2.29e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260505 11:17:26.350 | INFO | PerfReport: 21.95%, Download -- 2.22e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260505 11:17:26.350 | INFO | PerfReport: 31.82%, Compute -- 3.22e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 11:17:26.350 | INFO | PerfReport: 1.93%, LockUpload -- 1.96e+00 / 0.00e+00, 7.54e+00 / 0.00e+00 | 20260505 11:17:26.350 | INFO | PerfReport: 1.23%, LockDownload -- 1.24e+00 / 0.00e+00, 2.77e+00 / 0.00e+00 | 20260505 11:17:26.350 | INFO | PerfReport: 20.44%, LockCompute -- 2.07e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260505 11:17:26.350 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.65e-02 / 0.00e+00 | 20260505 11:17:26.481 | INFO | Starting a round of Baz IO | 20260505 11:17:29.732 | INFO | Populating RTMetrics report | 20260505 11:17:29.732 | INFO | RTMetrics report populated | 20260505 11:17:29.732 | INFO | Opening tmp RTMetrics json file | 20260505 11:17:29.742 | INFO | Serializing RTMetrics to json | 20260505 11:17:29.780 | INFO | Serialization done, swapping tmp file now... | 20260505 11:17:29.783 | INFO | RTMetrics done | 20260505 11:17:29.871 | INFO | Finished a round of Baz IO: Wrote 2.384 GiB in 3.39 seconds | 20260505 11:17:29.871 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.29% / 0.01% / 75.70% | 20260505 11:17:29.871 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.19248% / 1.19209e-05% / 8.34465e-05% | 20260505 11:17:29.871 | INFO | Baz IO overall: | 20260505 11:17:29.871 | INFO | BytesWritten : 1644746988469 | 20260505 11:17:29.871 | INFO | NumEvents : 783122498950 | 20260505 11:17:31.485 | INFO | Analyzing chunk frames = [2785280,2785792) | 20260505 11:17:31.486 | INFO | MemInfo: { | 20260505 11:17:31.486 | INFO | "gpu" : " 17 GB", | 20260505 11:17:31.486 | INFO | "huge" : "", | 20260505 11:17:31.486 | INFO | "huge_shared" : " 27 GB", | 20260505 11:17:31.486 | INFO | "malloc" : " 14 GB", | 20260505 11:17:31.486 | INFO | "pinned" : " 29 GB" | 20260505 11:17:31.486 | INFO | } | 20260505 11:17:31.486 | INFO | Perf report for the last 5.13727 seconds: | 20260505 11:17:31.486 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:17:31.486 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:31.486 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:31.486 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:31.486 | INFO | ANALYSIS : 64.7%, 2.9, 99.3ms, 0 | 20260505 11:17:31.486 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:31.486 | INFO | RT_METRICS : 46.5%, 1.0, 24.4ms, 0 | 20260505 11:17:31.486 | INFO | PRE_HQ : 61.3%, 1.9, 59.2ms, 0 | 20260505 11:17:31.486 | INFO | BAZWRITER : 65.3%, 4.1, 137.8ms, 0 | 20260505 11:17:31.486 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 11:17:31.486 | INFO | Uploaded 36GiB to GPU in 2.18046s (16.5103GiB/s) | 20260505 11:17:31.486 | INFO | Downloaded 38GiB to GPU in 2.29086s (16.5877GiB/s) | 20260505 11:17:36.620 | INFO | Analyzing chunk frames = [2785792,2786304) | 20260505 11:17:36.620 | INFO | MemInfo: { | 20260505 11:17:36.620 | INFO | "gpu" : " 17 GB", | 20260505 11:17:36.620 | INFO | "huge" : "", | 20260505 11:17:36.620 | INFO | "huge_shared" : " 27 GB", | 20260505 11:17:36.620 | INFO | "malloc" : " 14 GB", | 20260505 11:17:36.620 | INFO | "pinned" : " 29 GB" | 20260505 11:17:36.620 | INFO | } | 20260505 11:17:36.620 | INFO | Perf report for the last 5.13483 seconds: | 20260505 11:17:36.620 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:17:36.620 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:36.620 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:36.620 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:36.621 | INFO | ANALYSIS : 64.1%, 3.0, 100.0ms, 0 | 20260505 11:17:36.621 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:36.621 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:36.621 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:17:36.621 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:36.621 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:36.621 | INFO | Uploaded 36GiB to GPU in 2.22727s (16.1633GiB/s) | 20260505 11:17:36.621 | INFO | Downloaded 34GiB to GPU in 2.08696s (16.2917GiB/s) | 20260505 11:17:41.766 | INFO | Analyzing chunk frames = [2786304,2786816) | 20260505 11:17:41.766 | INFO | MemInfo: { | 20260505 11:17:41.766 | INFO | "gpu" : " 17 GB", | 20260505 11:17:41.766 | INFO | "huge" : "", | 20260505 11:17:41.766 | INFO | "huge_shared" : " 27 GB", | 20260505 11:17:41.766 | INFO | "malloc" : " 14 GB", | 20260505 11:17:41.766 | INFO | "pinned" : " 29 GB" | 20260505 11:17:41.766 | INFO | } | 20260505 11:17:41.766 | INFO | Perf report for the last 5.14548 seconds: | 20260505 11:17:41.766 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:17:41.766 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:41.766 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:41.766 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:41.766 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 11:17:41.766 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:41.766 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:41.766 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:17:41.766 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:41.766 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:41.766 | INFO | Uploaded 36GiB to GPU in 2.21626s (16.2436GiB/s) | 20260505 11:17:41.766 | INFO | Downloaded 34GiB to GPU in 2.13278s (15.9417GiB/s) | 20260505 11:17:46.907 | INFO | Analyzing chunk frames = [2786816,2787328) | 20260505 11:17:46.907 | INFO | MemInfo: { | 20260505 11:17:46.907 | INFO | "gpu" : " 17 GB", | 20260505 11:17:46.907 | INFO | "huge" : "", | 20260505 11:17:46.907 | INFO | "huge_shared" : " 27 GB", | 20260505 11:17:46.907 | INFO | "malloc" : " 14 GB", | 20260505 11:17:46.908 | INFO | "pinned" : " 29 GB" | 20260505 11:17:46.908 | INFO | } | 20260505 11:17:46.908 | INFO | Perf report for the last 5.14163 seconds: | 20260505 11:17:46.908 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:17:46.908 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:46.908 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:46.908 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:46.908 | INFO | ANALYSIS : 63.3%, 3.0, 98.3ms, 0 | 20260505 11:17:46.908 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:46.908 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:46.908 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 11:17:46.908 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:46.908 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:46.908 | INFO | Uploaded 36GiB to GPU in 2.20647s (16.3156GiB/s) | 20260505 11:17:46.908 | INFO | Downloaded 34GiB to GPU in 2.13069s (15.9573GiB/s) | 20260505 11:17:52.046 | INFO | Analyzing chunk frames = [2787328,2787840) | 20260505 11:17:52.046 | INFO | MemInfo: { | 20260505 11:17:52.046 | INFO | "gpu" : " 17 GB", | 20260505 11:17:52.046 | INFO | "huge" : "", | 20260505 11:17:52.046 | INFO | "huge_shared" : " 27 GB", | 20260505 11:17:52.046 | INFO | "malloc" : " 14 GB", | 20260505 11:17:52.046 | INFO | "pinned" : " 29 GB" | 20260505 11:17:52.046 | INFO | } | 20260505 11:17:52.046 | INFO | Perf report for the last 5.13881 seconds: | 20260505 11:17:52.046 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:17:52.046 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:52.046 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:52.046 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:52.046 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 11:17:52.046 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:52.047 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:52.047 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:17:52.047 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:52.047 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:52.047 | INFO | Uploaded 36GiB to GPU in 2.152s (16.7287GiB/s) | 20260505 11:17:52.047 | INFO | Downloaded 34GiB to GPU in 2.12524s (15.9982GiB/s) | 20260505 11:17:57.174 | INFO | Analyzing chunk frames = [2787840,2788352) | 20260505 11:17:57.174 | INFO | MemInfo: { | 20260505 11:17:57.174 | INFO | "gpu" : " 17 GB", | 20260505 11:17:57.174 | INFO | "huge" : "", | 20260505 11:17:57.174 | INFO | "huge_shared" : " 27 GB", | 20260505 11:17:57.174 | INFO | "malloc" : " 14 GB", | 20260505 11:17:57.174 | INFO | "pinned" : " 29 GB" | 20260505 11:17:57.174 | INFO | } | 20260505 11:17:57.174 | INFO | Perf report for the last 5.12772 seconds: | 20260505 11:17:57.174 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:17:57.174 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:57.174 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:57.174 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:57.174 | INFO | ANALYSIS : 63.7%, 3.0, 99.2ms, 0 | 20260505 11:17:57.174 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:57.174 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:57.174 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 11:17:57.174 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:57.174 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:17:57.174 | INFO | Uploaded 36GiB to GPU in 2.16557s (16.6238GiB/s) | 20260505 11:17:57.174 | INFO | Downloaded 34GiB to GPU in 2.11666s (16.0631GiB/s) | 20260505 11:18:02.313 | INFO | Analyzing chunk frames = [2788352,2788864) | 20260505 11:18:02.313 | INFO | MemInfo: { | 20260505 11:18:02.313 | INFO | "gpu" : " 17 GB", | 20260505 11:18:02.313 | INFO | "huge" : "", | 20260505 11:18:02.313 | INFO | "huge_shared" : " 27 GB", | 20260505 11:18:02.313 | INFO | "malloc" : " 14 GB", | 20260505 11:18:02.314 | INFO | "pinned" : " 29 GB" | 20260505 11:18:02.314 | INFO | } | 20260505 11:18:02.314 | INFO | Perf report for the last 5.13946 seconds: | 20260505 11:18:02.314 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:18:02.314 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:02.314 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:02.314 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:02.314 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 11:18:02.314 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:02.314 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:02.314 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 11:18:02.314 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:02.314 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:02.314 | INFO | Uploaded 36GiB to GPU in 2.17438s (16.5565GiB/s) | 20260505 11:18:02.314 | INFO | Downloaded 34GiB to GPU in 2.1521s (15.7986GiB/s) | 20260505 11:18:07.464 | INFO | Analyzing chunk frames = [2788864,2789376) | 20260505 11:18:07.464 | INFO | MemInfo: { | 20260505 11:18:07.464 | INFO | "gpu" : " 17 GB", | 20260505 11:18:07.464 | INFO | "huge" : "", | 20260505 11:18:07.464 | INFO | "huge_shared" : " 27 GB", | 20260505 11:18:07.464 | INFO | "malloc" : " 14 GB", | 20260505 11:18:07.464 | INFO | "pinned" : " 29 GB" | 20260505 11:18:07.464 | INFO | } | 20260505 11:18:07.464 | INFO | Perf report for the last 5.15047 seconds: | 20260505 11:18:07.464 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:18:07.464 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:07.464 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:07.464 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:07.464 | INFO | ANALYSIS : 63.7%, 3.0, 99.7ms, 0 | 20260505 11:18:07.464 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:07.464 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:07.464 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 11:18:07.464 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:07.464 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:07.464 | INFO | Uploaded 36GiB to GPU in 2.21295s (16.2679GiB/s) | 20260505 11:18:07.464 | INFO | Downloaded 34GiB to GPU in 2.08249s (16.3266GiB/s) | 20260505 11:18:07.597 | INFO | Starting a round of Baz IO | 20260505 11:18:10.848 | INFO | Populating RTMetrics report | 20260505 11:18:10.848 | INFO | RTMetrics report populated | 20260505 11:18:10.848 | INFO | Opening tmp RTMetrics json file | 20260505 11:18:10.873 | INFO | Serializing RTMetrics to json | 20260505 11:18:10.897 | INFO | Serialization done, swapping tmp file now... | 20260505 11:18:10.898 | INFO | RTMetrics done | 20260505 11:18:10.971 | INFO | Finished a round of Baz IO: Wrote 2.384 GiB in 3.374 seconds | 20260505 11:18:10.971 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.28% / 0.01% / 75.71% | 20260505 11:18:10.971 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.19057% / 1.19209e-05% / 5.16574e-05% | 20260505 11:18:10.971 | INFO | Baz IO overall: | 20260505 11:18:10.971 | INFO | BytesWritten : 1647306613467 | 20260505 11:18:10.971 | INFO | NumEvents : 784340752317 | 20260505 11:18:12.605 | INFO | Analyzing chunk frames = [2789376,2789888) | 20260505 11:18:12.605 | INFO | MemInfo: { | 20260505 11:18:12.605 | INFO | "gpu" : " 17 GB", | 20260505 11:18:12.605 | INFO | "huge" : "", | 20260505 11:18:12.605 | INFO | "huge_shared" : " 27 GB", | 20260505 11:18:12.605 | INFO | "malloc" : " 14 GB", | 20260505 11:18:12.605 | INFO | "pinned" : " 29 GB" | 20260505 11:18:12.605 | INFO | } | 20260505 11:18:12.605 | INFO | Perf report for the last 5.1413 seconds: | 20260505 11:18:12.605 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:18:12.605 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:12.605 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:12.605 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:12.605 | INFO | ANALYSIS : 64.7%, 2.9, 99.7ms, 0 | 20260505 11:18:12.605 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:12.605 | INFO | RT_METRICS : 46.4%, 1.0, 24.4ms, 0 | 20260505 11:18:12.605 | INFO | PRE_HQ : 61.2%, 1.9, 58.9ms, 0 | 20260505 11:18:12.606 | INFO | BAZWRITER : 64.9%, 4.1, 137.2ms, 0 | 20260505 11:18:12.606 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 11:18:12.606 | INFO | Uploaded 36GiB to GPU in 2.20566s (16.3216GiB/s) | 20260505 11:18:12.606 | INFO | Downloaded 38GiB to GPU in 2.28133s (16.657GiB/s) | 20260505 11:18:17.739 | INFO | Analyzing chunk frames = [2789888,2790400) | 20260505 11:18:17.739 | INFO | MemInfo: { | 20260505 11:18:17.739 | INFO | "gpu" : " 17 GB", | 20260505 11:18:17.739 | INFO | "huge" : "", | 20260505 11:18:17.739 | INFO | "huge_shared" : " 27 GB", | 20260505 11:18:17.739 | INFO | "malloc" : " 14 GB", | 20260505 11:18:17.739 | INFO | "pinned" : " 29 GB" | 20260505 11:18:17.739 | INFO | } | 20260505 11:18:17.739 | INFO | Perf report for the last 5.1336 seconds: | 20260505 11:18:17.739 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:18:17.739 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:17.739 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:17.739 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:17.739 | INFO | ANALYSIS : 64.3%, 3.0, 100.3ms, 0 | 20260505 11:18:17.739 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:17.739 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:17.739 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 11:18:17.739 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:17.739 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:17.739 | INFO | Uploaded 36GiB to GPU in 2.23012s (16.1427GiB/s) | 20260505 11:18:17.739 | INFO | Downloaded 34GiB to GPU in 2.09354s (16.2405GiB/s) | 20260505 11:18:22.870 | INFO | Analyzing chunk frames = [2790400,2790912) | 20260505 11:18:22.870 | INFO | MemInfo: { | 20260505 11:18:22.870 | INFO | "gpu" : " 17 GB", | 20260505 11:18:22.870 | INFO | "huge" : "", | 20260505 11:18:22.870 | INFO | "huge_shared" : " 27 GB", | 20260505 11:18:22.870 | INFO | "malloc" : " 14 GB", | 20260505 11:18:22.870 | INFO | "pinned" : " 29 GB" | 20260505 11:18:22.870 | INFO | } | 20260505 11:18:22.870 | INFO | Perf report for the last 5.13086 seconds: | 20260505 11:18:22.870 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:18:22.870 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:22.870 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:22.870 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:22.870 | INFO | ANALYSIS : 63.0%, 3.0, 97.9ms, 0 | 20260505 11:18:22.870 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:22.870 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:22.870 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 11:18:22.870 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:22.870 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:22.870 | INFO | Uploaded 36GiB to GPU in 2.15566s (16.7002GiB/s) | 20260505 11:18:22.870 | INFO | Downloaded 34GiB to GPU in 2.06156s (16.4923GiB/s) | 20260505 11:18:28.017 | INFO | Analyzing chunk frames = [2790912,2791424) | 20260505 11:18:28.017 | INFO | MemInfo: { | 20260505 11:18:28.017 | INFO | "gpu" : " 17 GB", | 20260505 11:18:28.017 | INFO | "huge" : "", | 20260505 11:18:28.017 | INFO | "huge_shared" : " 27 GB", | 20260505 11:18:28.017 | INFO | "malloc" : " 14 GB", | 20260505 11:18:28.017 | INFO | "pinned" : " 29 GB" | 20260505 11:18:28.017 | INFO | } | 20260505 11:18:28.017 | INFO | Perf report for the last 5.14702 seconds: | 20260505 11:18:28.017 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:18:28.017 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:28.017 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:28.017 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:28.017 | INFO | ANALYSIS : 62.9%, 3.0, 97.8ms, 0 | 20260505 11:18:28.017 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:28.017 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:28.017 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 11:18:28.017 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:28.017 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:28.017 | INFO | Uploaded 36GiB to GPU in 2.16898s (16.5976GiB/s) | 20260505 11:18:28.017 | INFO | Downloaded 34GiB to GPU in 2.08029s (16.3438GiB/s) | 20260505 11:18:33.152 | INFO | Analyzing chunk frames = [2791424,2791936) | 20260505 11:18:33.152 | INFO | MemInfo: { | 20260505 11:18:33.152 | INFO | "gpu" : " 17 GB", | 20260505 11:18:33.152 | INFO | "huge" : "", | 20260505 11:18:33.152 | INFO | "huge_shared" : " 27 GB", | 20260505 11:18:33.152 | INFO | "malloc" : " 14 GB", | 20260505 11:18:33.152 | INFO | "pinned" : " 29 GB" | 20260505 11:18:33.152 | INFO | } | 20260505 11:18:33.152 | INFO | Perf report for the last 5.13545 seconds: | 20260505 11:18:33.152 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:18:33.152 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:33.152 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:33.152 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:33.152 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 11:18:33.152 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:33.152 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:33.152 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 11:18:33.152 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:33.153 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:33.153 | INFO | Uploaded 36GiB to GPU in 2.18346s (16.4876GiB/s) | 20260505 11:18:33.153 | INFO | Downloaded 34GiB to GPU in 2.1288s (15.9714GiB/s) | 20260505 11:18:38.300 | INFO | Analyzing chunk frames = [2791936,2792448) | 20260505 11:18:38.300 | INFO | MemInfo: { | 20260505 11:18:38.300 | INFO | "gpu" : " 17 GB", | 20260505 11:18:38.300 | INFO | "huge" : "", | 20260505 11:18:38.300 | INFO | "huge_shared" : " 27 GB", | 20260505 11:18:38.300 | INFO | "malloc" : " 14 GB", | 20260505 11:18:38.301 | INFO | "pinned" : " 29 GB" | 20260505 11:18:38.301 | INFO | } | 20260505 11:18:38.301 | INFO | Perf report for the last 5.14831 seconds: | 20260505 11:18:38.301 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:18:38.301 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:38.301 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:38.301 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:38.301 | INFO | ANALYSIS : 63.5%, 3.0, 99.3ms, 0 | 20260505 11:18:38.301 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:38.301 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:38.301 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 11:18:38.301 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:38.301 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:38.301 | INFO | Uploaded 36GiB to GPU in 2.16291s (16.6443GiB/s) | 20260505 11:18:38.301 | INFO | Downloaded 34GiB to GPU in 2.12223s (16.0209GiB/s) | 20260505 11:18:43.434 | INFO | Analyzing chunk frames = [2792448,2792960) | 20260505 11:18:43.434 | INFO | MemInfo: { | 20260505 11:18:43.434 | INFO | "gpu" : " 17 GB", | 20260505 11:18:43.434 | INFO | "huge" : "", | 20260505 11:18:43.434 | INFO | "huge_shared" : " 27 GB", | 20260505 11:18:43.434 | INFO | "malloc" : " 14 GB", | 20260505 11:18:43.434 | INFO | "pinned" : " 29 GB" | 20260505 11:18:43.434 | INFO | } | 20260505 11:18:43.434 | INFO | Perf report for the last 5.13369 seconds: | 20260505 11:18:43.434 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:18:43.434 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:43.434 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:43.434 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:43.434 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 11:18:43.434 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:43.435 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:43.435 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 11:18:43.435 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:43.435 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:43.435 | INFO | Uploaded 36GiB to GPU in 2.1846s (16.479GiB/s) | 20260505 11:18:43.435 | INFO | Downloaded 34GiB to GPU in 2.14325s (15.8638GiB/s) | 20260505 11:18:48.572 | INFO | Analyzing chunk frames = [2792960,2793472) | 20260505 11:18:48.572 | INFO | MemInfo: { | 20260505 11:18:48.572 | INFO | "gpu" : " 17 GB", | 20260505 11:18:48.572 | INFO | "huge" : "", | 20260505 11:18:48.572 | INFO | "huge_shared" : " 27 GB", | 20260505 11:18:48.572 | INFO | "malloc" : " 14 GB", | 20260505 11:18:48.572 | INFO | "pinned" : " 29 GB" | 20260505 11:18:48.572 | INFO | } | 20260505 11:18:48.572 | INFO | Perf report for the last 5.13777 seconds: | 20260505 11:18:48.572 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:18:48.572 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:48.572 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:48.572 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:48.572 | INFO | ANALYSIS : 64.1%, 3.0, 100.0ms, 0 | 20260505 11:18:48.572 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:48.572 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:48.572 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 11:18:48.572 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:48.572 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:48.572 | INFO | Uploaded 36GiB to GPU in 2.23143s (16.1332GiB/s) | 20260505 11:18:48.572 | INFO | Downloaded 34GiB to GPU in 2.10138s (16.1798GiB/s) | 20260505 11:18:48.573 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 11:18:48.573 | INFO | Total mem usage: 15.3 GiB | 20260505 11:18:48.573 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 11:18:48.573 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -6.83%) | 20260505 11:18:48.573 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 11:18:48.573 | INFO | High Water Breakdown: | 20260505 11:18:48.573 | INFO | 61% : 8.76 GiB ( 92.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 11:18:48.573 | INFO | 38.8% : 5.57 GiB ( 83.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 11:18:48.573 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 11:18:48.573 | INFO | ------------------------------------------ | 20260505 11:18:48.573 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:18:48.573 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 11:18:48.573 | INFO | Total mem usage: 26.5 GiB | 20260505 11:18:48.573 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 11:18:48.573 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 11:18:48.573 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 11:18:48.573 | INFO | High Water Breakdown: | 20260505 11:18:48.573 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:18:48.573 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:18:48.573 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:18:48.573 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:18:48.573 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:18:48.573 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:18:48.573 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 11:18:48.573 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 11:18:48.573 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:18:48.573 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 11:18:48.573 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 11:18:48.573 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 11:18:48.573 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 11:18:48.573 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 11:18:48.573 | INFO | ------------------------------------------ | 20260505 11:18:48.573 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:18:48.573 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 11:18:48.573 | INFO | Total mem usage: 25.6 GiB | 20260505 11:18:48.573 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 11:18:48.573 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 11:18:48.573 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 11:18:48.573 | INFO | High Water Breakdown: | 20260505 11:18:48.573 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 11:18:48.573 | INFO | ------------------------------------------ | 20260505 11:18:48.573 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:18:48.573 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 11:18:48.573 | INFO | Total mem usage: 15.7 GiB | 20260505 11:18:48.573 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 11:18:48.573 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 11:18:48.573 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 11:18:48.573 | INFO | High Water Breakdown: | 20260505 11:18:48.573 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 11:18:48.573 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 11:18:48.573 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 11:18:48.573 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 11:18:48.574 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 11:18:48.574 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 11:18:48.574 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 11:18:48.574 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 11:18:48.574 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 11:18:48.574 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:18:48.574 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 11:18:48.574 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:18:48.574 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 11:18:48.574 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 11:18:48.574 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 11:18:48.574 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 11:18:48.574 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:18:48.574 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 11:18:48.574 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:18:48.574 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:18:48.574 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:18:48.574 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 11:18:48.574 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 11:18:48.574 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:18:48.574 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 11:18:48.574 | INFO | ------------------------------------------ | 20260505 11:18:48.574 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:18:48.574 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 11:18:48.574 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:18:48.574 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:18:48.574 | INFO | PerfReport: Full Execution -- 3.22e+01 / 7.72e+01, 1.38e+01 / 1.86e+00 | 20260505 11:18:48.574 | INFO | PerfReport: 0.29%, AggregateStats -- 9.27e-02 / 1.01e-01, 1.00e-02 / 4.49e-03 | 20260505 11:18:48.574 | INFO | PerfReport: 11.73%, Baseline -- 3.77e+00 / 4.51e+00, 5.91e-01 / 1.88e-01 | 20260505 11:18:48.574 | INFO | PerfReport: 3.30%, Histogram -- 1.06e+00 / 1.50e+00, 2.02e-01 / 9.91e-02 | 20260505 11:18:48.574 | INFO | PerfReport: 15.88%, DME -- 4.09e+01 / 4.55e+01, 4.45e+00 / 2.12e+00 | 20260505 11:18:48.574 | INFO | PerfReport: 56.37%, FrameLabeling -- 1.81e+01 / 2.07e+01, 1.58e+00 / 1.97e+00 | 20260505 11:18:48.574 | INFO | PerfReport: 7.43%, PulseAccumulating -- 2.39e+00 / 2.86e+00, 4.31e-01 / 2.67e-01 | 20260505 11:18:48.574 | INFO | PerfReport: 3.70%, Metrics -- 1.19e+00 / 1.48e+00, 3.35e-01 / 2.17e-01 | 20260505 11:18:48.574 | INFO | PerfReport: 1.30%, PulseSerializing -- 4.19e-01 / 4.65e-01, 7.70e-02 / 5.07e-02 | 20260505 11:18:48.574 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 11:18:48.574 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:18:48.574 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:18:48.574 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260505 11:18:48.574 | INFO | PerfReport: 22.55%, Upload -- 2.28e+01 / 0.00e+00, 8.00e+00 / 0.00e+00 | 20260505 11:18:48.574 | INFO | PerfReport: 21.96%, Download -- 2.22e+01 / 0.00e+00, 8.07e+00 / 0.00e+00 | 20260505 11:18:48.574 | INFO | PerfReport: 31.80%, Compute -- 3.22e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 11:18:48.574 | INFO | PerfReport: 2.03%, LockUpload -- 2.06e+00 / 0.00e+00, 7.68e+00 / 0.00e+00 | 20260505 11:18:48.574 | INFO | PerfReport: 1.27%, LockDownload -- 1.28e+00 / 0.00e+00, 2.79e+00 / 0.00e+00 | 20260505 11:18:48.574 | INFO | PerfReport: 20.37%, LockCompute -- 2.06e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260505 11:18:48.574 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.89e-02 / 0.00e+00 | 20260505 11:18:48.725 | INFO | Starting a round of Baz IO | 20260505 11:18:51.966 | INFO | Populating RTMetrics report | 20260505 11:18:51.966 | INFO | RTMetrics report populated | 20260505 11:18:51.966 | INFO | Opening tmp RTMetrics json file | 20260505 11:18:51.968 | INFO | Serializing RTMetrics to json | 20260505 11:18:51.969 | INFO | Serialization done, swapping tmp file now... | 20260505 11:18:51.969 | INFO | RTMetrics done | 20260505 11:18:52.099 | INFO | Finished a round of Baz IO: Wrote 2.383 GiB in 3.374 seconds | 20260505 11:18:52.099 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.27% / 0.01% / 75.72% | 20260505 11:18:52.099 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.18626% / 7.94729e-06% / 9.93411e-05% | 20260505 11:18:52.099 | INFO | Baz IO overall: | 20260505 11:18:52.099 | INFO | BytesWritten : 1649865133103 | 20260505 11:18:52.099 | INFO | NumEvents : 785558293606 | 20260505 11:18:53.716 | INFO | Analyzing chunk frames = [2793472,2793984) | 20260505 11:18:53.716 | INFO | MemInfo: { | 20260505 11:18:53.716 | INFO | "gpu" : " 17 GB", | 20260505 11:18:53.716 | INFO | "huge" : "", | 20260505 11:18:53.716 | INFO | "huge_shared" : " 27 GB", | 20260505 11:18:53.716 | INFO | "malloc" : " 14 GB", | 20260505 11:18:53.716 | INFO | "pinned" : " 29 GB" | 20260505 11:18:53.716 | INFO | } | 20260505 11:18:53.717 | INFO | Perf report for the last 5.14432 seconds: | 20260505 11:18:53.717 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:18:53.717 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:53.717 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:53.717 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:53.717 | INFO | ANALYSIS : 64.9%, 2.9, 100.3ms, 0 | 20260505 11:18:53.717 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:53.717 | INFO | RT_METRICS : 45.5%, 1.0, 23.9ms, 0 | 20260505 11:18:53.717 | INFO | PRE_HQ : 61.4%, 1.9, 59.1ms, 0 | 20260505 11:18:53.717 | INFO | BAZWRITER : 65.0%, 4.1, 137.2ms, 0 | 20260505 11:18:53.717 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 11:18:53.717 | INFO | Uploaded 36GiB to GPU in 2.23111s (16.1355GiB/s) | 20260505 11:18:53.717 | INFO | Downloaded 38GiB to GPU in 2.28846s (16.605GiB/s) | 20260505 11:18:58.859 | INFO | Analyzing chunk frames = [2793984,2794496) | 20260505 11:18:58.859 | INFO | MemInfo: { | 20260505 11:18:58.859 | INFO | "gpu" : " 17 GB", | 20260505 11:18:58.859 | INFO | "huge" : "", | 20260505 11:18:58.859 | INFO | "huge_shared" : " 27 GB", | 20260505 11:18:58.859 | INFO | "malloc" : " 14 GB", | 20260505 11:18:58.859 | INFO | "pinned" : " 29 GB" | 20260505 11:18:58.859 | INFO | } | 20260505 11:18:58.859 | INFO | Perf report for the last 5.14254 seconds: | 20260505 11:18:58.859 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:18:58.859 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:58.859 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:58.859 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:58.859 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 11:18:58.859 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:58.859 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:58.859 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:18:58.859 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:58.859 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:18:58.859 | INFO | Uploaded 36GiB to GPU in 2.22667s (16.1676GiB/s) | 20260505 11:18:58.859 | INFO | Downloaded 34GiB to GPU in 2.10402s (16.1596GiB/s) | 20260505 11:19:03.993 | INFO | Analyzing chunk frames = [2794496,2795008) | 20260505 11:19:03.993 | INFO | MemInfo: { | 20260505 11:19:03.993 | INFO | "gpu" : " 17 GB", | 20260505 11:19:03.993 | INFO | "huge" : "", | 20260505 11:19:03.993 | INFO | "huge_shared" : " 27 GB", | 20260505 11:19:03.993 | INFO | "malloc" : " 14 GB", | 20260505 11:19:03.993 | INFO | "pinned" : " 29 GB" | 20260505 11:19:03.993 | INFO | } | 20260505 11:19:03.993 | INFO | Perf report for the last 5.13403 seconds: | 20260505 11:19:03.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:19:03.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:03.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:03.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:03.993 | INFO | ANALYSIS : 63.6%, 3.0, 98.8ms, 0 | 20260505 11:19:03.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:03.993 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:03.993 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 11:19:03.993 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:03.993 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:03.993 | INFO | Uploaded 36GiB to GPU in 2.23408s (16.114GiB/s) | 20260505 11:19:03.993 | INFO | Downloaded 34GiB to GPU in 2.13046s (15.959GiB/s) | 20260505 11:19:09.123 | INFO | Analyzing chunk frames = [2795008,2795520) | 20260505 11:19:09.123 | INFO | MemInfo: { | 20260505 11:19:09.123 | INFO | "gpu" : " 17 GB", | 20260505 11:19:09.123 | INFO | "huge" : "", | 20260505 11:19:09.123 | INFO | "huge_shared" : " 27 GB", | 20260505 11:19:09.123 | INFO | "malloc" : " 14 GB", | 20260505 11:19:09.123 | INFO | "pinned" : " 29 GB" | 20260505 11:19:09.123 | INFO | } | 20260505 11:19:09.123 | INFO | Perf report for the last 5.13019 seconds: | 20260505 11:19:09.123 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:19:09.123 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:09.123 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:09.123 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:09.123 | INFO | ANALYSIS : 63.0%, 3.0, 97.7ms, 0 | 20260505 11:19:09.123 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:09.123 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:09.123 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 11:19:09.123 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:09.123 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:09.124 | INFO | Uploaded 36GiB to GPU in 2.15553s (16.7012GiB/s) | 20260505 11:19:09.124 | INFO | Downloaded 34GiB to GPU in 2.07921s (16.3524GiB/s) | 20260505 11:19:14.278 | INFO | Analyzing chunk frames = [2795520,2796032) | 20260505 11:19:14.278 | INFO | MemInfo: { | 20260505 11:19:14.278 | INFO | "gpu" : " 17 GB", | 20260505 11:19:14.278 | INFO | "huge" : "", | 20260505 11:19:14.278 | INFO | "huge_shared" : " 27 GB", | 20260505 11:19:14.278 | INFO | "malloc" : " 14 GB", | 20260505 11:19:14.278 | INFO | "pinned" : " 29 GB" | 20260505 11:19:14.278 | INFO | } | 20260505 11:19:14.278 | INFO | Perf report for the last 5.15504 seconds: | 20260505 11:19:14.278 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:19:14.278 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:14.278 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:14.278 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:14.278 | INFO | ANALYSIS : 63.1%, 3.0, 98.8ms, 0 | 20260505 11:19:14.278 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:14.278 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:14.278 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 11:19:14.279 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:14.279 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:14.279 | INFO | Uploaded 36GiB to GPU in 2.21724s (16.2364GiB/s) | 20260505 11:19:14.279 | INFO | Downloaded 34GiB to GPU in 2.12251s (16.0188GiB/s) | 20260505 11:19:19.417 | INFO | Analyzing chunk frames = [2796032,2796544) | 20260505 11:19:19.417 | INFO | MemInfo: { | 20260505 11:19:19.417 | INFO | "gpu" : " 17 GB", | 20260505 11:19:19.417 | INFO | "huge" : "", | 20260505 11:19:19.417 | INFO | "huge_shared" : " 27 GB", | 20260505 11:19:19.417 | INFO | "malloc" : " 14 GB", | 20260505 11:19:19.418 | INFO | "pinned" : " 29 GB" | 20260505 11:19:19.418 | INFO | } | 20260505 11:19:19.418 | INFO | Perf report for the last 5.13923 seconds: | 20260505 11:19:19.418 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:19:19.418 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:19.418 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:19.418 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:19.418 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 11:19:19.418 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:19.418 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:19.418 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 11:19:19.418 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:19.418 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:19.418 | INFO | Uploaded 36GiB to GPU in 2.14751s (16.7636GiB/s) | 20260505 11:19:19.418 | INFO | Downloaded 34GiB to GPU in 2.10928s (16.1192GiB/s) | 20260505 11:19:24.554 | INFO | Analyzing chunk frames = [2796544,2797056) | 20260505 11:19:24.554 | INFO | MemInfo: { | 20260505 11:19:24.554 | INFO | "gpu" : " 17 GB", | 20260505 11:19:24.554 | INFO | "huge" : "", | 20260505 11:19:24.554 | INFO | "huge_shared" : " 27 GB", | 20260505 11:19:24.554 | INFO | "malloc" : " 14 GB", | 20260505 11:19:24.554 | INFO | "pinned" : " 29 GB" | 20260505 11:19:24.554 | INFO | } | 20260505 11:19:24.554 | INFO | Perf report for the last 5.13632 seconds: | 20260505 11:19:24.554 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:19:24.554 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:24.554 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:24.554 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:24.554 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 11:19:24.554 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:24.554 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:24.554 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 11:19:24.554 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:24.554 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:24.554 | INFO | Uploaded 36GiB to GPU in 2.16975s (16.5918GiB/s) | 20260505 11:19:24.554 | INFO | Downloaded 34GiB to GPU in 2.14808s (15.8281GiB/s) | 20260505 11:19:29.683 | INFO | Analyzing chunk frames = [2797056,2797568) | 20260505 11:19:29.683 | INFO | MemInfo: { | 20260505 11:19:29.683 | INFO | "gpu" : " 17 GB", | 20260505 11:19:29.683 | INFO | "huge" : "", | 20260505 11:19:29.683 | INFO | "huge_shared" : " 27 GB", | 20260505 11:19:29.683 | INFO | "malloc" : " 14 GB", | 20260505 11:19:29.683 | INFO | "pinned" : " 29 GB" | 20260505 11:19:29.683 | INFO | } | 20260505 11:19:29.683 | INFO | Perf report for the last 5.12906 seconds: | 20260505 11:19:29.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:19:29.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:29.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:29.683 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:29.683 | INFO | ANALYSIS : 64.0%, 3.0, 99.8ms, 0 | 20260505 11:19:29.683 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:29.683 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:29.683 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 11:19:29.683 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:29.683 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:29.683 | INFO | Uploaded 36GiB to GPU in 2.21232s (16.2725GiB/s) | 20260505 11:19:29.683 | INFO | Downloaded 34GiB to GPU in 2.09231s (16.25GiB/s) | 20260505 11:19:29.814 | INFO | Starting a round of Baz IO | 20260505 11:19:33.071 | INFO | Populating RTMetrics report | 20260505 11:19:33.071 | INFO | RTMetrics report populated | 20260505 11:19:33.071 | INFO | Opening tmp RTMetrics json file | 20260505 11:19:33.075 | INFO | Serializing RTMetrics to json | 20260505 11:19:33.079 | INFO | Serialization done, swapping tmp file now... | 20260505 11:19:33.080 | INFO | RTMetrics done | 20260505 11:19:33.215 | INFO | Finished a round of Baz IO: Wrote 2.385 GiB in 3.4 seconds | 20260505 11:19:33.215 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.27% / 0.01% / 75.73% | 20260505 11:19:33.215 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.1862% / 1.19209e-05% / 7.15256e-05% | 20260505 11:19:33.215 | INFO | Baz IO overall: | 20260505 11:19:33.215 | INFO | BytesWritten : 1652425775344 | 20260505 11:19:33.215 | INFO | NumEvents : 786777228586 | 20260505 11:19:34.826 | INFO | Analyzing chunk frames = [2797568,2798080) | 20260505 11:19:34.826 | INFO | MemInfo: { | 20260505 11:19:34.826 | INFO | "gpu" : " 17 GB", | 20260505 11:19:34.826 | INFO | "huge" : "", | 20260505 11:19:34.826 | INFO | "huge_shared" : " 27 GB", | 20260505 11:19:34.826 | INFO | "malloc" : " 14 GB", | 20260505 11:19:34.826 | INFO | "pinned" : " 29 GB" | 20260505 11:19:34.826 | INFO | } | 20260505 11:19:34.826 | INFO | Perf report for the last 5.14341 seconds: | 20260505 11:19:34.826 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:19:34.826 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:34.826 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:34.826 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:34.827 | INFO | ANALYSIS : 64.7%, 2.9, 99.2ms, 0 | 20260505 11:19:34.827 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:34.827 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260505 11:19:34.827 | INFO | PRE_HQ : 62.5%, 1.8, 58.9ms, 0 | 20260505 11:19:34.827 | INFO | BAZWRITER : 65.4%, 4.2, 141.9ms, 0 | 20260505 11:19:34.827 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 11:19:34.827 | INFO | Uploaded 36GiB to GPU in 2.30627s (15.6096GiB/s) | 20260505 11:19:34.827 | INFO | Downloaded 38GiB to GPU in 2.38849s (15.9096GiB/s) | 20260505 11:19:39.974 | INFO | Analyzing chunk frames = [2798080,2798592) | 20260505 11:19:39.974 | INFO | MemInfo: { | 20260505 11:19:39.974 | INFO | "gpu" : " 17 GB", | 20260505 11:19:39.974 | INFO | "huge" : "", | 20260505 11:19:39.974 | INFO | "huge_shared" : " 27 GB", | 20260505 11:19:39.974 | INFO | "malloc" : " 14 GB", | 20260505 11:19:39.974 | INFO | "pinned" : " 29 GB" | 20260505 11:19:39.974 | INFO | } | 20260505 11:19:39.974 | INFO | Perf report for the last 5.1476 seconds: | 20260505 11:19:39.974 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:19:39.974 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:39.974 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:39.974 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:39.974 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 11:19:39.974 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:39.974 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:39.974 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:19:39.974 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:39.974 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:39.974 | INFO | Uploaded 36GiB to GPU in 2.22976s (16.1452GiB/s) | 20260505 11:19:39.974 | INFO | Downloaded 34GiB to GPU in 2.08313s (16.3216GiB/s) | 20260505 11:19:45.113 | INFO | Analyzing chunk frames = [2798592,2799104) | 20260505 11:19:45.113 | INFO | MemInfo: { | 20260505 11:19:45.113 | INFO | "gpu" : " 17 GB", | 20260505 11:19:45.113 | INFO | "huge" : "", | 20260505 11:19:45.113 | INFO | "huge_shared" : " 27 GB", | 20260505 11:19:45.113 | INFO | "malloc" : " 14 GB", | 20260505 11:19:45.113 | INFO | "pinned" : " 29 GB" | 20260505 11:19:45.113 | INFO | } | 20260505 11:19:45.113 | INFO | Perf report for the last 5.13887 seconds: | 20260505 11:19:45.113 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:19:45.113 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:45.113 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:45.113 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:45.113 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 11:19:45.113 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:45.113 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:45.113 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 11:19:45.113 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:45.113 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:45.113 | INFO | Uploaded 36GiB to GPU in 2.20941s (16.294GiB/s) | 20260505 11:19:45.113 | INFO | Downloaded 34GiB to GPU in 2.11551s (16.0718GiB/s) | 20260505 11:19:50.242 | INFO | Analyzing chunk frames = [2799104,2799616) | 20260505 11:19:50.242 | INFO | MemInfo: { | 20260505 11:19:50.242 | INFO | "gpu" : " 17 GB", | 20260505 11:19:50.242 | INFO | "huge" : "", | 20260505 11:19:50.242 | INFO | "huge_shared" : " 27 GB", | 20260505 11:19:50.242 | INFO | "malloc" : " 14 GB", | 20260505 11:19:50.242 | INFO | "pinned" : " 29 GB" | 20260505 11:19:50.242 | INFO | } | 20260505 11:19:50.242 | INFO | Perf report for the last 5.12924 seconds: | 20260505 11:19:50.242 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:19:50.242 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:50.242 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:50.242 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:50.242 | INFO | ANALYSIS : 63.4%, 3.0, 98.4ms, 0 | 20260505 11:19:50.242 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:50.242 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:50.242 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 11:19:50.242 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:50.242 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:50.242 | INFO | Uploaded 36GiB to GPU in 2.20967s (16.292GiB/s) | 20260505 11:19:50.243 | INFO | Downloaded 34GiB to GPU in 2.13032s (15.9601GiB/s) | 20260505 11:19:55.384 | INFO | Analyzing chunk frames = [2799616,2800128) | 20260505 11:19:55.384 | INFO | MemInfo: { | 20260505 11:19:55.384 | INFO | "gpu" : " 17 GB", | 20260505 11:19:55.384 | INFO | "huge" : "", | 20260505 11:19:55.384 | INFO | "huge_shared" : " 27 GB", | 20260505 11:19:55.384 | INFO | "malloc" : " 14 GB", | 20260505 11:19:55.384 | INFO | "pinned" : " 29 GB" | 20260505 11:19:55.384 | INFO | } | 20260505 11:19:55.384 | INFO | Perf report for the last 5.14183 seconds: | 20260505 11:19:55.384 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:19:55.384 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:55.384 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:55.384 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:55.384 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 11:19:55.384 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:55.384 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:55.384 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 11:19:55.384 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:55.384 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:19:55.384 | INFO | Uploaded 36GiB to GPU in 2.22698s (16.1654GiB/s) | 20260505 11:19:55.384 | INFO | Downloaded 34GiB to GPU in 2.1106s (16.1092GiB/s) | 20260505 11:20:00.525 | INFO | Analyzing chunk frames = [2800128,2800640) | 20260505 11:20:00.525 | INFO | MemInfo: { | 20260505 11:20:00.525 | INFO | "gpu" : " 17 GB", | 20260505 11:20:00.525 | INFO | "huge" : "", | 20260505 11:20:00.525 | INFO | "huge_shared" : " 27 GB", | 20260505 11:20:00.525 | INFO | "malloc" : " 14 GB", | 20260505 11:20:00.525 | INFO | "pinned" : " 29 GB" | 20260505 11:20:00.525 | INFO | } | 20260505 11:20:00.525 | INFO | Perf report for the last 5.14148 seconds: | 20260505 11:20:00.526 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:20:00.526 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:00.526 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:00.526 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:00.526 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 11:20:00.526 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:00.526 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:00.526 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:20:00.526 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:00.526 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:00.526 | INFO | Uploaded 36GiB to GPU in 2.21758s (16.2339GiB/s) | 20260505 11:20:00.526 | INFO | Downloaded 34GiB to GPU in 2.10734s (16.1341GiB/s) | 20260505 11:20:05.663 | INFO | Analyzing chunk frames = [2800640,2801152) | 20260505 11:20:05.663 | INFO | MemInfo: { | 20260505 11:20:05.663 | INFO | "gpu" : " 17 GB", | 20260505 11:20:05.663 | INFO | "huge" : "", | 20260505 11:20:05.663 | INFO | "huge_shared" : " 27 GB", | 20260505 11:20:05.663 | INFO | "malloc" : " 14 GB", | 20260505 11:20:05.663 | INFO | "pinned" : " 29 GB" | 20260505 11:20:05.663 | INFO | } | 20260505 11:20:05.663 | INFO | Perf report for the last 5.13782 seconds: | 20260505 11:20:05.663 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:20:05.663 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:05.663 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:05.663 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:05.663 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 11:20:05.663 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:05.664 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:05.664 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 11:20:05.664 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:05.664 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:05.664 | INFO | Uploaded 36GiB to GPU in 2.18908s (16.4453GiB/s) | 20260505 11:20:05.664 | INFO | Downloaded 34GiB to GPU in 2.15899s (15.7481GiB/s) | 20260505 11:20:10.804 | INFO | Analyzing chunk frames = [2801152,2801664) | 20260505 11:20:10.804 | INFO | MemInfo: { | 20260505 11:20:10.804 | INFO | "gpu" : " 17 GB", | 20260505 11:20:10.804 | INFO | "huge" : "", | 20260505 11:20:10.804 | INFO | "huge_shared" : " 27 GB", | 20260505 11:20:10.804 | INFO | "malloc" : " 14 GB", | 20260505 11:20:10.804 | INFO | "pinned" : " 29 GB" | 20260505 11:20:10.804 | INFO | } | 20260505 11:20:10.804 | INFO | Perf report for the last 5.14061 seconds: | 20260505 11:20:10.804 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:20:10.804 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:10.804 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:10.804 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:10.804 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 11:20:10.804 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:10.804 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:10.804 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 11:20:10.804 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:10.804 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:10.804 | INFO | Uploaded 36GiB to GPU in 2.2181s (16.2301GiB/s) | 20260505 11:20:10.804 | INFO | Downloaded 34GiB to GPU in 2.08703s (16.2911GiB/s) | 20260505 11:20:10.804 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 11:20:10.804 | INFO | Total mem usage: 15.3 GiB | 20260505 11:20:10.804 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 11:20:10.804 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -6.05%) | 20260505 11:20:10.804 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 11:20:10.804 | INFO | High Water Breakdown: | 20260505 11:20:10.804 | INFO | 61.1% : 8.84 GiB ( 93.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 11:20:10.804 | INFO | 38.7% : 5.6 GiB ( 84.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 11:20:10.805 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 11:20:10.805 | INFO | ------------------------------------------ | 20260505 11:20:10.805 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:20:10.805 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 11:20:10.805 | INFO | Total mem usage: 26.5 GiB | 20260505 11:20:10.805 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 11:20:10.805 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 11:20:10.805 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 11:20:10.805 | INFO | High Water Breakdown: | 20260505 11:20:10.805 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:20:10.805 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:20:10.805 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:20:10.805 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:20:10.805 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:20:10.805 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:20:10.805 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 11:20:10.805 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 11:20:10.805 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:20:10.805 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 11:20:10.805 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 11:20:10.805 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 11:20:10.805 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 11:20:10.805 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 11:20:10.805 | INFO | ------------------------------------------ | 20260505 11:20:10.805 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:20:10.805 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 11:20:10.805 | INFO | Total mem usage: 25.6 GiB | 20260505 11:20:10.805 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 11:20:10.805 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 11:20:10.805 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 11:20:10.805 | INFO | High Water Breakdown: | 20260505 11:20:10.805 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 11:20:10.805 | INFO | ------------------------------------------ | 20260505 11:20:10.805 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:20:10.805 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 11:20:10.805 | INFO | Total mem usage: 15.7 GiB | 20260505 11:20:10.805 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 11:20:10.805 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 11:20:10.805 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 11:20:10.805 | INFO | High Water Breakdown: | 20260505 11:20:10.805 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 11:20:10.805 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 11:20:10.805 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 11:20:10.805 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 11:20:10.805 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 11:20:10.805 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 11:20:10.805 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 11:20:10.805 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 11:20:10.805 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 11:20:10.805 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:20:10.805 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 11:20:10.805 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:20:10.805 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 11:20:10.805 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 11:20:10.805 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 11:20:10.805 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 11:20:10.805 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:20:10.805 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 11:20:10.805 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:20:10.805 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:20:10.805 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:20:10.805 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 11:20:10.805 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 11:20:10.805 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:20:10.805 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 11:20:10.805 | INFO | ------------------------------------------ | 20260505 11:20:10.805 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:20:10.805 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 11:20:10.806 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:20:10.806 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:20:10.806 | INFO | PerfReport: Full Execution -- 3.21e+01 / 7.68e+01, 1.38e+01 / 1.92e+00 | 20260505 11:20:10.806 | INFO | PerfReport: 0.29%, AggregateStats -- 9.31e-02 / 1.02e-01, 1.09e-02 / 4.58e-03 | 20260505 11:20:10.806 | INFO | PerfReport: 11.70%, Baseline -- 3.76e+00 / 4.42e+00, 5.87e-01 / 3.08e-01 | 20260505 11:20:10.806 | INFO | PerfReport: 3.30%, Histogram -- 1.06e+00 / 1.48e+00, 2.01e-01 / 1.02e-01 | 20260505 11:20:10.806 | INFO | PerfReport: 15.68%, DME -- 4.03e+01 / 4.38e+01, 3.85e+00 / 1.45e+00 | 20260505 11:20:10.806 | INFO | PerfReport: 56.55%, FrameLabeling -- 1.82e+01 / 2.23e+01, 1.63e+00 / 1.65e+00 | 20260505 11:20:10.806 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.39e+00 / 2.79e+00, 4.17e-01 / 2.09e-01 | 20260505 11:20:10.806 | INFO | PerfReport: 3.72%, Metrics -- 1.19e+00 / 1.48e+00, 3.39e-01 / 1.84e-01 | 20260505 11:20:10.806 | INFO | PerfReport: 1.31%, PulseSerializing -- 4.20e-01 / 4.72e-01, 7.82e-02 / 3.33e-02 | 20260505 11:20:10.806 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 11:20:10.806 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:20:10.806 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:20:10.806 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.42e+01 / 0.00e+00 | 20260505 11:20:10.806 | INFO | PerfReport: 22.77%, Upload -- 2.30e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260505 11:20:10.806 | INFO | PerfReport: 22.03%, Download -- 2.23e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260505 11:20:10.806 | INFO | PerfReport: 31.72%, Compute -- 3.21e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 11:20:10.806 | INFO | PerfReport: 2.01%, LockUpload -- 2.04e+00 / 0.00e+00, 7.65e+00 / 0.00e+00 | 20260505 11:20:10.806 | INFO | PerfReport: 1.26%, LockDownload -- 1.27e+00 / 0.00e+00, 2.76e+00 / 0.00e+00 | 20260505 11:20:10.806 | INFO | PerfReport: 20.19%, LockCompute -- 2.04e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260505 11:20:10.806 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.61e-02 / 0.00e+00 | 20260505 11:20:10.941 | INFO | Starting a round of Baz IO | 20260505 11:20:14.209 | INFO | Populating RTMetrics report | 20260505 11:20:14.209 | INFO | RTMetrics report populated | 20260505 11:20:14.209 | INFO | Opening tmp RTMetrics json file | 20260505 11:20:14.226 | INFO | Serializing RTMetrics to json | 20260505 11:20:14.228 | INFO | Serialization done, swapping tmp file now... | 20260505 11:20:14.228 | INFO | RTMetrics done | 20260505 11:20:14.358 | INFO | Finished a round of Baz IO: Wrote 2.384 GiB in 3.416 seconds | 20260505 11:20:14.358 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.26% / 0.01% / 75.73% | 20260505 11:20:14.358 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.18403% / 1.58946e-05% / 6.75519e-05% | 20260505 11:20:14.358 | INFO | Baz IO overall: | 20260505 11:20:14.358 | INFO | BytesWritten : 1654986084402 | 20260505 11:20:14.358 | INFO | NumEvents : 787995924401 | 20260505 11:20:15.947 | INFO | Analyzing chunk frames = [2801664,2802176) | 20260505 11:20:15.947 | INFO | MemInfo: { | 20260505 11:20:15.947 | INFO | "gpu" : " 17 GB", | 20260505 11:20:15.947 | INFO | "huge" : "", | 20260505 11:20:15.947 | INFO | "huge_shared" : " 27 GB", | 20260505 11:20:15.947 | INFO | "malloc" : " 14 GB", | 20260505 11:20:15.947 | INFO | "pinned" : " 29 GB" | 20260505 11:20:15.947 | INFO | } | 20260505 11:20:15.947 | INFO | Perf report for the last 5.14274 seconds: | 20260505 11:20:15.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:20:15.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:15.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:15.947 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:15.947 | INFO | ANALYSIS : 65.0%, 2.9, 99.3ms, 0 | 20260505 11:20:15.947 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:15.947 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260505 11:20:15.947 | INFO | PRE_HQ : 61.2%, 1.9, 59.3ms, 0 | 20260505 11:20:15.947 | INFO | BAZWRITER : 65.8%, 4.2, 141.3ms, 0 | 20260505 11:20:15.947 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 11:20:15.947 | INFO | Uploaded 36GiB to GPU in 2.18424s (16.4817GiB/s) | 20260505 11:20:15.947 | INFO | Downloaded 38GiB to GPU in 2.30123s (16.5129GiB/s) | 20260505 11:20:21.080 | INFO | Analyzing chunk frames = [2802176,2802688) | 20260505 11:20:21.080 | INFO | MemInfo: { | 20260505 11:20:21.080 | INFO | "gpu" : " 17 GB", | 20260505 11:20:21.080 | INFO | "huge" : "", | 20260505 11:20:21.080 | INFO | "huge_shared" : " 27 GB", | 20260505 11:20:21.080 | INFO | "malloc" : " 14 GB", | 20260505 11:20:21.080 | INFO | "pinned" : " 29 GB" | 20260505 11:20:21.080 | INFO | } | 20260505 11:20:21.080 | INFO | Perf report for the last 5.1333 seconds: | 20260505 11:20:21.080 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:20:21.080 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:21.080 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:21.080 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:21.080 | INFO | ANALYSIS : 64.3%, 3.0, 100.3ms, 0 | 20260505 11:20:21.080 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:21.080 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:21.080 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 11:20:21.080 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:21.080 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:21.080 | INFO | Uploaded 36GiB to GPU in 2.20848s (16.3008GiB/s) | 20260505 11:20:21.080 | INFO | Downloaded 34GiB to GPU in 2.08891s (16.2765GiB/s) | 20260505 11:20:26.228 | INFO | Analyzing chunk frames = [2802688,2803200) | 20260505 11:20:26.228 | INFO | MemInfo: { | 20260505 11:20:26.228 | INFO | "gpu" : " 17 GB", | 20260505 11:20:26.228 | INFO | "huge" : "", | 20260505 11:20:26.228 | INFO | "huge_shared" : " 27 GB", | 20260505 11:20:26.228 | INFO | "malloc" : " 14 GB", | 20260505 11:20:26.228 | INFO | "pinned" : " 29 GB" | 20260505 11:20:26.228 | INFO | } | 20260505 11:20:26.228 | INFO | Perf report for the last 5.14837 seconds: | 20260505 11:20:26.229 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:20:26.229 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:26.229 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:26.229 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:26.229 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 11:20:26.229 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:26.229 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:26.229 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 11:20:26.229 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:26.229 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:26.229 | INFO | Uploaded 36GiB to GPU in 2.22145s (16.2056GiB/s) | 20260505 11:20:26.229 | INFO | Downloaded 34GiB to GPU in 2.1311s (15.9542GiB/s) | 20260505 11:20:31.371 | INFO | Analyzing chunk frames = [2803200,2803712) | 20260505 11:20:31.371 | INFO | MemInfo: { | 20260505 11:20:31.371 | INFO | "gpu" : " 17 GB", | 20260505 11:20:31.371 | INFO | "huge" : "", | 20260505 11:20:31.371 | INFO | "huge_shared" : " 27 GB", | 20260505 11:20:31.371 | INFO | "malloc" : " 14 GB", | 20260505 11:20:31.371 | INFO | "pinned" : " 29 GB" | 20260505 11:20:31.371 | INFO | } | 20260505 11:20:31.371 | INFO | Perf report for the last 5.14257 seconds: | 20260505 11:20:31.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:20:31.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:31.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:31.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:31.371 | INFO | ANALYSIS : 63.2%, 3.0, 98.3ms, 0 | 20260505 11:20:31.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:31.371 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:31.371 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 11:20:31.371 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:31.371 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:31.371 | INFO | Uploaded 36GiB to GPU in 2.20433s (16.3315GiB/s) | 20260505 11:20:31.371 | INFO | Downloaded 34GiB to GPU in 2.13222s (15.9458GiB/s) | 20260505 11:20:36.495 | INFO | Analyzing chunk frames = [2803712,2804224) | 20260505 11:20:36.495 | INFO | MemInfo: { | 20260505 11:20:36.495 | INFO | "gpu" : " 17 GB", | 20260505 11:20:36.495 | INFO | "huge" : "", | 20260505 11:20:36.495 | INFO | "huge_shared" : " 27 GB", | 20260505 11:20:36.495 | INFO | "malloc" : " 14 GB", | 20260505 11:20:36.495 | INFO | "pinned" : " 29 GB" | 20260505 11:20:36.495 | INFO | } | 20260505 11:20:36.496 | INFO | Perf report for the last 5.12469 seconds: | 20260505 11:20:36.496 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:20:36.496 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:36.496 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:36.496 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:36.496 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 11:20:36.496 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:36.496 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:36.496 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 11:20:36.496 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:36.496 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:36.496 | INFO | Uploaded 36GiB to GPU in 2.15683s (16.6912GiB/s) | 20260505 11:20:36.496 | INFO | Downloaded 34GiB to GPU in 2.12287s (16.016GiB/s) | 20260505 11:20:41.634 | INFO | Analyzing chunk frames = [2804224,2804736) | 20260505 11:20:41.634 | INFO | MemInfo: { | 20260505 11:20:41.634 | INFO | "gpu" : " 17 GB", | 20260505 11:20:41.634 | INFO | "huge" : "", | 20260505 11:20:41.634 | INFO | "huge_shared" : " 27 GB", | 20260505 11:20:41.634 | INFO | "malloc" : " 14 GB", | 20260505 11:20:41.634 | INFO | "pinned" : " 29 GB" | 20260505 11:20:41.634 | INFO | } | 20260505 11:20:41.634 | INFO | Perf report for the last 5.13859 seconds: | 20260505 11:20:41.634 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:20:41.634 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:41.634 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:41.634 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:41.634 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 11:20:41.634 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:41.634 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:41.634 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:20:41.634 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:41.634 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:41.635 | INFO | Uploaded 36GiB to GPU in 2.22893s (16.1513GiB/s) | 20260505 11:20:41.635 | INFO | Downloaded 34GiB to GPU in 2.10888s (16.1223GiB/s) | 20260505 11:20:46.781 | INFO | Analyzing chunk frames = [2804736,2805248) | 20260505 11:20:46.781 | INFO | MemInfo: { | 20260505 11:20:46.781 | INFO | "gpu" : " 17 GB", | 20260505 11:20:46.781 | INFO | "huge" : "", | 20260505 11:20:46.781 | INFO | "huge_shared" : " 27 GB", | 20260505 11:20:46.781 | INFO | "malloc" : " 14 GB", | 20260505 11:20:46.781 | INFO | "pinned" : " 29 GB" | 20260505 11:20:46.781 | INFO | } | 20260505 11:20:46.781 | INFO | Perf report for the last 5.14674 seconds: | 20260505 11:20:46.781 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:20:46.781 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:46.781 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:46.781 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:46.781 | INFO | ANALYSIS : 63.2%, 3.0, 98.9ms, 0 | 20260505 11:20:46.781 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:46.781 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:46.781 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 11:20:46.781 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:46.781 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:46.781 | INFO | Uploaded 36GiB to GPU in 2.16331s (16.6412GiB/s) | 20260505 11:20:46.781 | INFO | Downloaded 34GiB to GPU in 2.16137s (15.7307GiB/s) | 20260505 11:20:51.924 | INFO | Analyzing chunk frames = [2805248,2805760) | 20260505 11:20:51.924 | INFO | MemInfo: { | 20260505 11:20:51.924 | INFO | "gpu" : " 17 GB", | 20260505 11:20:51.924 | INFO | "huge" : "", | 20260505 11:20:51.924 | INFO | "huge_shared" : " 27 GB", | 20260505 11:20:51.924 | INFO | "malloc" : " 14 GB", | 20260505 11:20:51.924 | INFO | "pinned" : " 29 GB" | 20260505 11:20:51.924 | INFO | } | 20260505 11:20:51.924 | INFO | Perf report for the last 5.14332 seconds: | 20260505 11:20:51.924 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:20:51.924 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:51.924 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:51.924 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:51.924 | INFO | ANALYSIS : 63.9%, 3.0, 99.9ms, 0 | 20260505 11:20:51.925 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:51.925 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:51.925 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 11:20:51.925 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:51.925 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:51.925 | INFO | Uploaded 36GiB to GPU in 2.19875s (16.3729GiB/s) | 20260505 11:20:51.925 | INFO | Downloaded 34GiB to GPU in 2.11403s (16.083GiB/s) | 20260505 11:20:52.056 | INFO | Starting a round of Baz IO | 20260505 11:20:55.306 | INFO | Populating RTMetrics report | 20260505 11:20:55.307 | INFO | RTMetrics report populated | 20260505 11:20:55.307 | INFO | Opening tmp RTMetrics json file | 20260505 11:20:55.317 | INFO | Serializing RTMetrics to json | 20260505 11:20:55.319 | INFO | Serialization done, swapping tmp file now... | 20260505 11:20:55.322 | INFO | RTMetrics done | 20260505 11:20:55.449 | INFO | Finished a round of Baz IO: Wrote 2.383 GiB in 3.393 seconds | 20260505 11:20:55.449 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.25% / 0.01% / 75.74% | 20260505 11:20:55.449 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.17919% / 3.97364e-06% / 5.5631e-05% | 20260505 11:20:55.450 | INFO | Baz IO overall: | 20260505 11:20:55.450 | INFO | BytesWritten : 1657545059640 | 20260505 11:20:55.450 | INFO | NumEvents : 789213845008 | 20260505 11:20:57.065 | INFO | Analyzing chunk frames = [2805760,2806272) | 20260505 11:20:57.065 | INFO | MemInfo: { | 20260505 11:20:57.065 | INFO | "gpu" : " 17 GB", | 20260505 11:20:57.065 | INFO | "huge" : "", | 20260505 11:20:57.065 | INFO | "huge_shared" : " 27 GB", | 20260505 11:20:57.065 | INFO | "malloc" : " 14 GB", | 20260505 11:20:57.065 | INFO | "pinned" : " 29 GB" | 20260505 11:20:57.065 | INFO | } | 20260505 11:20:57.065 | INFO | Perf report for the last 5.14105 seconds: | 20260505 11:20:57.065 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:20:57.065 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:57.066 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:57.066 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:57.066 | INFO | ANALYSIS : 64.7%, 2.9, 99.5ms, 0 | 20260505 11:20:57.066 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:20:57.066 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260505 11:20:57.066 | INFO | PRE_HQ : 61.0%, 1.8, 58.5ms, 0 | 20260505 11:20:57.066 | INFO | BAZWRITER : 65.2%, 4.1, 138.7ms, 0 | 20260505 11:20:57.066 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 11:20:57.066 | INFO | Uploaded 36GiB to GPU in 2.18227s (16.4966GiB/s) | 20260505 11:20:57.066 | INFO | Downloaded 38GiB to GPU in 2.291s (16.5866GiB/s) | 20260505 11:21:02.196 | INFO | Analyzing chunk frames = [2806272,2806784) | 20260505 11:21:02.196 | INFO | MemInfo: { | 20260505 11:21:02.196 | INFO | "gpu" : " 17 GB", | 20260505 11:21:02.196 | INFO | "huge" : "", | 20260505 11:21:02.196 | INFO | "huge_shared" : " 27 GB", | 20260505 11:21:02.196 | INFO | "malloc" : " 14 GB", | 20260505 11:21:02.196 | INFO | "pinned" : " 29 GB" | 20260505 11:21:02.196 | INFO | } | 20260505 11:21:02.196 | INFO | Perf report for the last 5.13099 seconds: | 20260505 11:21:02.196 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:21:02.196 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:02.196 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:02.196 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:02.196 | INFO | ANALYSIS : 64.3%, 3.0, 100.3ms, 0 | 20260505 11:21:02.197 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:02.197 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:02.197 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 11:21:02.197 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:02.197 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:02.197 | INFO | Uploaded 36GiB to GPU in 2.23055s (16.1395GiB/s) | 20260505 11:21:02.197 | INFO | Downloaded 34GiB to GPU in 2.09568s (16.2238GiB/s) | 20260505 11:21:07.331 | INFO | Analyzing chunk frames = [2806784,2807296) | 20260505 11:21:07.331 | INFO | MemInfo: { | 20260505 11:21:07.331 | INFO | "gpu" : " 17 GB", | 20260505 11:21:07.331 | INFO | "huge" : "", | 20260505 11:21:07.331 | INFO | "huge_shared" : " 27 GB", | 20260505 11:21:07.332 | INFO | "malloc" : " 14 GB", | 20260505 11:21:07.332 | INFO | "pinned" : " 29 GB" | 20260505 11:21:07.332 | INFO | } | 20260505 11:21:07.332 | INFO | Perf report for the last 5.13526 seconds: | 20260505 11:21:07.332 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:21:07.332 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:07.332 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:07.332 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:07.332 | INFO | ANALYSIS : 63.0%, 3.0, 97.7ms, 0 | 20260505 11:21:07.332 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:07.332 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:07.332 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 11:21:07.332 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:07.332 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:07.332 | INFO | Uploaded 36GiB to GPU in 2.16001s (16.6666GiB/s) | 20260505 11:21:07.332 | INFO | Downloaded 34GiB to GPU in 2.06668s (16.4515GiB/s) | 20260505 11:21:12.478 | INFO | Analyzing chunk frames = [2807296,2807808) | 20260505 11:21:12.478 | INFO | MemInfo: { | 20260505 11:21:12.478 | INFO | "gpu" : " 17 GB", | 20260505 11:21:12.478 | INFO | "huge" : "", | 20260505 11:21:12.478 | INFO | "huge_shared" : " 27 GB", | 20260505 11:21:12.478 | INFO | "malloc" : " 14 GB", | 20260505 11:21:12.478 | INFO | "pinned" : " 29 GB" | 20260505 11:21:12.478 | INFO | } | 20260505 11:21:12.478 | INFO | Perf report for the last 5.14618 seconds: | 20260505 11:21:12.478 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:21:12.478 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:12.478 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:12.478 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:12.478 | INFO | ANALYSIS : 62.9%, 3.0, 97.8ms, 0 | 20260505 11:21:12.478 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:12.478 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:12.478 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 11:21:12.478 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:12.478 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:12.478 | INFO | Uploaded 36GiB to GPU in 2.15688s (16.6908GiB/s) | 20260505 11:21:12.478 | INFO | Downloaded 34GiB to GPU in 2.07757s (16.3653GiB/s) | 20260505 11:21:17.618 | INFO | Analyzing chunk frames = [2807808,2808320) | 20260505 11:21:17.618 | INFO | MemInfo: { | 20260505 11:21:17.618 | INFO | "gpu" : " 17 GB", | 20260505 11:21:17.618 | INFO | "huge" : "", | 20260505 11:21:17.618 | INFO | "huge_shared" : " 27 GB", | 20260505 11:21:17.618 | INFO | "malloc" : " 14 GB", | 20260505 11:21:17.619 | INFO | "pinned" : " 29 GB" | 20260505 11:21:17.619 | INFO | } | 20260505 11:21:17.619 | INFO | Perf report for the last 5.14076 seconds: | 20260505 11:21:17.619 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:21:17.619 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:17.619 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:17.619 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:17.619 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 11:21:17.619 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:17.619 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:17.619 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 11:21:17.619 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:17.619 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:17.619 | INFO | Uploaded 36GiB to GPU in 2.18085s (16.5074GiB/s) | 20260505 11:21:17.619 | INFO | Downloaded 34GiB to GPU in 2.12766s (15.98GiB/s) | 20260505 11:21:22.752 | INFO | Analyzing chunk frames = [2808320,2808832) | 20260505 11:21:22.752 | INFO | MemInfo: { | 20260505 11:21:22.752 | INFO | "gpu" : " 17 GB", | 20260505 11:21:22.752 | INFO | "huge" : "", | 20260505 11:21:22.752 | INFO | "huge_shared" : " 27 GB", | 20260505 11:21:22.752 | INFO | "malloc" : " 14 GB", | 20260505 11:21:22.752 | INFO | "pinned" : " 29 GB" | 20260505 11:21:22.752 | INFO | } | 20260505 11:21:22.752 | INFO | Perf report for the last 5.13332 seconds: | 20260505 11:21:22.752 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:21:22.752 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:22.752 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:22.752 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:22.752 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 11:21:22.752 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:22.752 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:22.752 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 11:21:22.752 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:22.752 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:22.753 | INFO | Uploaded 36GiB to GPU in 2.19224s (16.4215GiB/s) | 20260505 11:21:22.753 | INFO | Downloaded 34GiB to GPU in 2.11588s (16.069GiB/s) | 20260505 11:21:27.888 | INFO | Analyzing chunk frames = [2808832,2809344) | 20260505 11:21:27.888 | INFO | MemInfo: { | 20260505 11:21:27.888 | INFO | "gpu" : " 17 GB", | 20260505 11:21:27.888 | INFO | "huge" : "", | 20260505 11:21:27.889 | INFO | "huge_shared" : " 27 GB", | 20260505 11:21:27.889 | INFO | "malloc" : " 14 GB", | 20260505 11:21:27.889 | INFO | "pinned" : " 29 GB" | 20260505 11:21:27.889 | INFO | } | 20260505 11:21:27.889 | INFO | Perf report for the last 5.13667 seconds: | 20260505 11:21:27.889 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:21:27.889 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:27.889 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:27.889 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:27.889 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 11:21:27.889 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:27.889 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:27.889 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 11:21:27.889 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:27.889 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:27.889 | INFO | Uploaded 36GiB to GPU in 2.15076s (16.7382GiB/s) | 20260505 11:21:27.889 | INFO | Downloaded 34GiB to GPU in 2.15029s (15.8118GiB/s) | 20260505 11:21:33.039 | INFO | Analyzing chunk frames = [2809344,2809856) | 20260505 11:21:33.039 | INFO | MemInfo: { | 20260505 11:21:33.039 | INFO | "gpu" : " 17 GB", | 20260505 11:21:33.039 | INFO | "huge" : "", | 20260505 11:21:33.039 | INFO | "huge_shared" : " 27 GB", | 20260505 11:21:33.039 | INFO | "malloc" : " 14 GB", | 20260505 11:21:33.039 | INFO | "pinned" : " 29 GB" | 20260505 11:21:33.039 | INFO | } | 20260505 11:21:33.039 | INFO | Perf report for the last 5.15042 seconds: | 20260505 11:21:33.039 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:21:33.039 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:33.039 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:33.039 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:33.039 | INFO | ANALYSIS : 63.7%, 3.0, 99.7ms, 0 | 20260505 11:21:33.039 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:33.039 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:33.039 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 11:21:33.039 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:33.039 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:33.039 | INFO | Uploaded 36GiB to GPU in 2.22291s (16.195GiB/s) | 20260505 11:21:33.039 | INFO | Downloaded 34GiB to GPU in 2.09726s (16.2116GiB/s) | 20260505 11:21:33.040 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 11:21:33.040 | INFO | Total mem usage: 15.3 GiB | 20260505 11:21:33.040 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 11:21:33.040 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -6.91%) | 20260505 11:21:33.040 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 11:21:33.040 | INFO | High Water Breakdown: | 20260505 11:21:33.040 | INFO | 61% : 8.75 GiB ( 92.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 11:21:33.040 | INFO | 38.9% : 5.57 GiB ( 83.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 11:21:33.040 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 11:21:33.040 | INFO | ------------------------------------------ | 20260505 11:21:33.040 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:21:33.040 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 11:21:33.040 | INFO | Total mem usage: 26.5 GiB | 20260505 11:21:33.040 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 11:21:33.040 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 11:21:33.040 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 11:21:33.040 | INFO | High Water Breakdown: | 20260505 11:21:33.040 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:21:33.040 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:21:33.040 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:21:33.040 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:21:33.040 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:21:33.040 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:21:33.040 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 11:21:33.040 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 11:21:33.040 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:21:33.040 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 11:21:33.040 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 11:21:33.040 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 11:21:33.040 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 11:21:33.040 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 11:21:33.040 | INFO | ------------------------------------------ | 20260505 11:21:33.040 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:21:33.040 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 11:21:33.040 | INFO | Total mem usage: 25.6 GiB | 20260505 11:21:33.040 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 11:21:33.040 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 11:21:33.040 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 11:21:33.040 | INFO | High Water Breakdown: | 20260505 11:21:33.040 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 11:21:33.040 | INFO | ------------------------------------------ | 20260505 11:21:33.040 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:21:33.040 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 11:21:33.040 | INFO | Total mem usage: 15.7 GiB | 20260505 11:21:33.040 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 11:21:33.040 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 11:21:33.040 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 11:21:33.040 | INFO | High Water Breakdown: | 20260505 11:21:33.040 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 11:21:33.040 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 11:21:33.040 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 11:21:33.040 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 11:21:33.040 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 11:21:33.040 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 11:21:33.040 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 11:21:33.040 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 11:21:33.040 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 11:21:33.040 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:21:33.040 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 11:21:33.040 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:21:33.041 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 11:21:33.041 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 11:21:33.041 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 11:21:33.041 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 11:21:33.041 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:21:33.041 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 11:21:33.041 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:21:33.041 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:21:33.041 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:21:33.041 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 11:21:33.041 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 11:21:33.041 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:21:33.041 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 11:21:33.041 | INFO | ------------------------------------------ | 20260505 11:21:33.041 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:21:33.041 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 11:21:33.041 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:21:33.041 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:21:33.041 | INFO | PerfReport: Full Execution -- 3.22e+01 / 7.73e+01, 1.39e+01 / 1.58e+00 | 20260505 11:21:33.041 | INFO | PerfReport: 0.29%, AggregateStats -- 9.31e-02 / 1.02e-01, 1.04e-02 / 3.70e-03 | 20260505 11:21:33.041 | INFO | PerfReport: 11.73%, Baseline -- 3.78e+00 / 4.52e+00, 6.02e-01 / 1.53e-01 | 20260505 11:21:33.041 | INFO | PerfReport: 3.30%, Histogram -- 1.06e+00 / 1.50e+00, 2.02e-01 / 7.81e-02 | 20260505 11:21:33.041 | INFO | PerfReport: 15.89%, DME -- 4.09e+01 / 4.56e+01, 4.55e+00 / 2.24e+00 | 20260505 11:21:33.041 | INFO | PerfReport: 56.35%, FrameLabeling -- 1.81e+01 / 2.06e+01, 1.58e+00 / 1.71e+00 | 20260505 11:21:33.041 | INFO | PerfReport: 7.43%, PulseAccumulating -- 2.39e+00 / 2.90e+00, 4.33e-01 / 2.18e-01 | 20260505 11:21:33.041 | INFO | PerfReport: 3.71%, Metrics -- 1.19e+00 / 1.50e+00, 3.41e-01 / 2.17e-01 | 20260505 11:21:33.041 | INFO | PerfReport: 1.30%, PulseSerializing -- 4.20e-01 / 4.64e-01, 7.78e-02 / 4.52e-02 | 20260505 11:21:33.041 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 11:21:33.041 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:21:33.041 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:21:33.041 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.47e+01 / 0.00e+00 | 20260505 11:21:33.041 | INFO | PerfReport: 22.56%, Upload -- 2.28e+01 / 0.00e+00, 8.06e+00 / 0.00e+00 | 20260505 11:21:33.041 | INFO | PerfReport: 22.00%, Download -- 2.23e+01 / 0.00e+00, 8.16e+00 / 0.00e+00 | 20260505 11:21:33.041 | INFO | PerfReport: 31.83%, Compute -- 3.22e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 11:21:33.041 | INFO | PerfReport: 2.00%, LockUpload -- 2.03e+00 / 0.00e+00, 7.51e+00 / 0.00e+00 | 20260505 11:21:33.041 | INFO | PerfReport: 1.27%, LockDownload -- 1.29e+00 / 0.00e+00, 2.85e+00 / 0.00e+00 | 20260505 11:21:33.041 | INFO | PerfReport: 20.32%, LockCompute -- 2.06e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260505 11:21:33.041 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.74e-02 / 0.00e+00 | 20260505 11:21:33.171 | INFO | Starting a round of Baz IO | 20260505 11:21:36.420 | INFO | Populating RTMetrics report | 20260505 11:21:36.420 | INFO | RTMetrics report populated | 20260505 11:21:36.420 | INFO | Opening tmp RTMetrics json file | 20260505 11:21:36.426 | INFO | Serializing RTMetrics to json | 20260505 11:21:36.429 | INFO | Serialization done, swapping tmp file now... | 20260505 11:21:36.429 | INFO | RTMetrics done | 20260505 11:21:36.563 | INFO | Finished a round of Baz IO: Wrote 2.382 GiB in 3.392 seconds | 20260505 11:21:36.563 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.24% / 0.01% / 75.75% | 20260505 11:21:36.563 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.17313% / 1.58946e-05% / 8.74201e-05% | 20260505 11:21:36.563 | INFO | Baz IO overall: | 20260505 11:21:36.563 | INFO | BytesWritten : 1660102833877 | 20260505 11:21:36.563 | INFO | NumEvents : 790430948392 | 20260505 11:21:38.178 | INFO | Analyzing chunk frames = [2809856,2810368) | 20260505 11:21:38.178 | INFO | MemInfo: { | 20260505 11:21:38.178 | INFO | "gpu" : " 17 GB", | 20260505 11:21:38.178 | INFO | "huge" : "", | 20260505 11:21:38.178 | INFO | "huge_shared" : " 27 GB", | 20260505 11:21:38.178 | INFO | "malloc" : " 14 GB", | 20260505 11:21:38.178 | INFO | "pinned" : " 29 GB" | 20260505 11:21:38.178 | INFO | } | 20260505 11:21:38.178 | INFO | Perf report for the last 5.13847 seconds: | 20260505 11:21:38.178 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:21:38.178 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:38.178 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:38.178 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:38.178 | INFO | ANALYSIS : 64.7%, 2.9, 99.3ms, 0 | 20260505 11:21:38.178 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:38.178 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260505 11:21:38.178 | INFO | PRE_HQ : 61.2%, 1.9, 59.1ms, 0 | 20260505 11:21:38.178 | INFO | BAZWRITER : 65.3%, 4.1, 138.0ms, 0 | 20260505 11:21:38.178 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260505 11:21:38.178 | INFO | Uploaded 36GiB to GPU in 2.19674s (16.3879GiB/s) | 20260505 11:21:38.178 | INFO | Downloaded 38GiB to GPU in 2.27865s (16.6765GiB/s) | 20260505 11:21:43.315 | INFO | Analyzing chunk frames = [2810368,2810880) | 20260505 11:21:43.315 | INFO | MemInfo: { | 20260505 11:21:43.315 | INFO | "gpu" : " 17 GB", | 20260505 11:21:43.315 | INFO | "huge" : "", | 20260505 11:21:43.315 | INFO | "huge_shared" : " 27 GB", | 20260505 11:21:43.315 | INFO | "malloc" : " 14 GB", | 20260505 11:21:43.315 | INFO | "pinned" : " 29 GB" | 20260505 11:21:43.315 | INFO | } | 20260505 11:21:43.316 | INFO | Perf report for the last 5.13788 seconds: | 20260505 11:21:43.316 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:21:43.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:43.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:43.316 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:43.316 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 11:21:43.316 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:43.316 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:43.316 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:21:43.316 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:43.316 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:43.316 | INFO | Uploaded 36GiB to GPU in 2.22967s (16.1459GiB/s) | 20260505 11:21:43.316 | INFO | Downloaded 34GiB to GPU in 2.09135s (16.2574GiB/s) | 20260505 11:21:48.444 | INFO | Analyzing chunk frames = [2810880,2811392) | 20260505 11:21:48.444 | INFO | MemInfo: { | 20260505 11:21:48.444 | INFO | "gpu" : " 17 GB", | 20260505 11:21:48.444 | INFO | "huge" : "", | 20260505 11:21:48.445 | INFO | "huge_shared" : " 27 GB", | 20260505 11:21:48.445 | INFO | "malloc" : " 14 GB", | 20260505 11:21:48.445 | INFO | "pinned" : " 29 GB" | 20260505 11:21:48.445 | INFO | } | 20260505 11:21:48.445 | INFO | Perf report for the last 5.12911 seconds: | 20260505 11:21:48.445 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:21:48.445 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:48.445 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:48.445 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:48.445 | INFO | ANALYSIS : 63.8%, 3.0, 99.1ms, 0 | 20260505 11:21:48.445 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:48.445 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:48.445 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 11:21:48.445 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:48.445 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:48.445 | INFO | Uploaded 36GiB to GPU in 2.22094s (16.2093GiB/s) | 20260505 11:21:48.445 | INFO | Downloaded 34GiB to GPU in 2.13293s (15.9405GiB/s) | 20260505 11:21:53.590 | INFO | Analyzing chunk frames = [2811392,2811904) | 20260505 11:21:53.590 | INFO | MemInfo: { | 20260505 11:21:53.590 | INFO | "gpu" : " 17 GB", | 20260505 11:21:53.590 | INFO | "huge" : "", | 20260505 11:21:53.590 | INFO | "huge_shared" : " 27 GB", | 20260505 11:21:53.590 | INFO | "malloc" : " 14 GB", | 20260505 11:21:53.590 | INFO | "pinned" : " 29 GB" | 20260505 11:21:53.590 | INFO | } | 20260505 11:21:53.590 | INFO | Perf report for the last 5.14572 seconds: | 20260505 11:21:53.590 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:21:53.590 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:53.590 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:53.590 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:53.590 | INFO | ANALYSIS : 62.9%, 3.0, 97.9ms, 0 | 20260505 11:21:53.590 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:53.590 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:53.591 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 11:21:53.591 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:53.591 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:53.591 | INFO | Uploaded 36GiB to GPU in 2.17147s (16.5786GiB/s) | 20260505 11:21:53.591 | INFO | Downloaded 34GiB to GPU in 2.08023s (16.3443GiB/s) | 20260505 11:21:58.732 | INFO | Analyzing chunk frames = [2811904,2812416) | 20260505 11:21:58.732 | INFO | MemInfo: { | 20260505 11:21:58.732 | INFO | "gpu" : " 17 GB", | 20260505 11:21:58.732 | INFO | "huge" : "", | 20260505 11:21:58.732 | INFO | "huge_shared" : " 27 GB", | 20260505 11:21:58.732 | INFO | "malloc" : " 14 GB", | 20260505 11:21:58.732 | INFO | "pinned" : " 29 GB" | 20260505 11:21:58.732 | INFO | } | 20260505 11:21:58.732 | INFO | Perf report for the last 5.14153 seconds: | 20260505 11:21:58.732 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:21:58.732 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:58.732 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:58.732 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:58.732 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 11:21:58.732 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:58.732 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:58.732 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 11:21:58.732 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:58.732 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:21:58.732 | INFO | Uploaded 36GiB to GPU in 2.17255s (16.5704GiB/s) | 20260505 11:21:58.732 | INFO | Downloaded 34GiB to GPU in 2.12897s (15.9701GiB/s) | 20260505 11:22:03.877 | INFO | Analyzing chunk frames = [2812416,2812928) | 20260505 11:22:03.877 | INFO | MemInfo: { | 20260505 11:22:03.877 | INFO | "gpu" : " 17 GB", | 20260505 11:22:03.877 | INFO | "huge" : "", | 20260505 11:22:03.877 | INFO | "huge_shared" : " 27 GB", | 20260505 11:22:03.877 | INFO | "malloc" : " 14 GB", | 20260505 11:22:03.877 | INFO | "pinned" : " 29 GB" | 20260505 11:22:03.877 | INFO | } | 20260505 11:22:03.877 | INFO | Perf report for the last 5.14531 seconds: | 20260505 11:22:03.877 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:22:03.877 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:03.877 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:03.877 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:03.877 | INFO | ANALYSIS : 63.1%, 3.0, 98.7ms, 0 | 20260505 11:22:03.877 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:03.877 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:03.877 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 11:22:03.877 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:03.877 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:03.878 | INFO | Uploaded 36GiB to GPU in 2.14833s (16.7572GiB/s) | 20260505 11:22:03.878 | INFO | Downloaded 34GiB to GPU in 2.10847s (16.1254GiB/s) | 20260505 11:22:09.011 | INFO | Analyzing chunk frames = [2812928,2813440) | 20260505 11:22:09.011 | INFO | MemInfo: { | 20260505 11:22:09.011 | INFO | "gpu" : " 17 GB", | 20260505 11:22:09.011 | INFO | "huge" : "", | 20260505 11:22:09.011 | INFO | "huge_shared" : " 27 GB", | 20260505 11:22:09.011 | INFO | "malloc" : " 14 GB", | 20260505 11:22:09.011 | INFO | "pinned" : " 29 GB" | 20260505 11:22:09.011 | INFO | } | 20260505 11:22:09.011 | INFO | Perf report for the last 5.13418 seconds: | 20260505 11:22:09.011 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:22:09.011 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:09.011 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:09.011 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:09.011 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 11:22:09.011 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:09.012 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:09.012 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 11:22:09.012 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:09.012 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:09.012 | INFO | Uploaded 36GiB to GPU in 2.18048s (16.5101GiB/s) | 20260505 11:22:09.012 | INFO | Downloaded 34GiB to GPU in 2.1523s (15.7971GiB/s) | 20260505 11:22:14.141 | INFO | Analyzing chunk frames = [2813440,2813952) | 20260505 11:22:14.141 | INFO | MemInfo: { | 20260505 11:22:14.141 | INFO | "gpu" : " 17 GB", | 20260505 11:22:14.142 | INFO | "huge" : "", | 20260505 11:22:14.142 | INFO | "huge_shared" : " 27 GB", | 20260505 11:22:14.142 | INFO | "malloc" : " 14 GB", | 20260505 11:22:14.142 | INFO | "pinned" : " 29 GB" | 20260505 11:22:14.142 | INFO | } | 20260505 11:22:14.142 | INFO | Perf report for the last 5.13035 seconds: | 20260505 11:22:14.142 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:22:14.142 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:14.142 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:14.142 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:14.142 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 11:22:14.142 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:14.142 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:14.142 | INFO | PRE_HQ : 29.8%, 1.0, 15.4ms, 0 | 20260505 11:22:14.142 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:14.142 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:14.142 | INFO | Uploaded 36GiB to GPU in 2.217s (16.2382GiB/s) | 20260505 11:22:14.142 | INFO | Downloaded 34GiB to GPU in 2.08547s (16.3033GiB/s) | 20260505 11:22:14.276 | INFO | Starting a round of Baz IO | 20260505 11:22:17.540 | INFO | Populating RTMetrics report | 20260505 11:22:17.541 | INFO | RTMetrics report populated | 20260505 11:22:17.541 | INFO | Opening tmp RTMetrics json file | 20260505 11:22:17.542 | INFO | Serializing RTMetrics to json | 20260505 11:22:17.544 | INFO | Serialization done, swapping tmp file now... | 20260505 11:22:17.545 | INFO | RTMetrics done | 20260505 11:22:17.674 | INFO | Finished a round of Baz IO: Wrote 2.384 GiB in 3.398 seconds | 20260505 11:22:17.674 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.23% / 0.01% / 75.76% | 20260505 11:22:17.674 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.1721% / 3.97364e-06% / 7.54992e-05% | 20260505 11:22:17.674 | INFO | Baz IO overall: | 20260505 11:22:17.674 | INFO | BytesWritten : 1662662312059 | 20260505 11:22:17.674 | INFO | NumEvents : 791649270465 | 20260505 11:22:19.287 | INFO | Analyzing chunk frames = [2813952,2814464) | 20260505 11:22:19.287 | INFO | MemInfo: { | 20260505 11:22:19.287 | INFO | "gpu" : " 17 GB", | 20260505 11:22:19.287 | INFO | "huge" : "", | 20260505 11:22:19.287 | INFO | "huge_shared" : " 27 GB", | 20260505 11:22:19.287 | INFO | "malloc" : " 14 GB", | 20260505 11:22:19.287 | INFO | "pinned" : " 29 GB" | 20260505 11:22:19.288 | INFO | } | 20260505 11:22:19.288 | INFO | Perf report for the last 5.14586 seconds: | 20260505 11:22:19.288 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:22:19.288 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:19.288 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:19.288 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:19.288 | INFO | ANALYSIS : 64.9%, 2.9, 100.1ms, 0 | 20260505 11:22:19.288 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:19.288 | INFO | RT_METRICS : 45.5%, 1.0, 23.9ms, 0 | 20260505 11:22:19.288 | INFO | PRE_HQ : 62.4%, 1.8, 59.1ms, 0 | 20260505 11:22:19.288 | INFO | BAZWRITER : 65.3%, 4.1, 137.6ms, 0 | 20260505 11:22:19.288 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 11:22:19.288 | INFO | Uploaded 36GiB to GPU in 2.30245s (15.6355GiB/s) | 20260505 11:22:19.288 | INFO | Downloaded 38GiB to GPU in 2.37514s (15.999GiB/s) | 20260505 11:22:24.427 | INFO | Analyzing chunk frames = [2814464,2814976) | 20260505 11:22:24.427 | INFO | MemInfo: { | 20260505 11:22:24.427 | INFO | "gpu" : " 17 GB", | 20260505 11:22:24.427 | INFO | "huge" : "", | 20260505 11:22:24.427 | INFO | "huge_shared" : " 27 GB", | 20260505 11:22:24.427 | INFO | "malloc" : " 14 GB", | 20260505 11:22:24.427 | INFO | "pinned" : " 29 GB" | 20260505 11:22:24.427 | INFO | } | 20260505 11:22:24.427 | INFO | Perf report for the last 5.13978 seconds: | 20260505 11:22:24.427 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:22:24.427 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:24.427 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:24.427 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:24.427 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 11:22:24.428 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:24.428 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:24.428 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 11:22:24.428 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:24.428 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:24.428 | INFO | Uploaded 36GiB to GPU in 2.22939s (16.1479GiB/s) | 20260505 11:22:24.428 | INFO | Downloaded 34GiB to GPU in 2.09755s (16.2094GiB/s) | 20260505 11:22:29.568 | INFO | Analyzing chunk frames = [2814976,2815488) | 20260505 11:22:29.568 | INFO | MemInfo: { | 20260505 11:22:29.568 | INFO | "gpu" : " 17 GB", | 20260505 11:22:29.568 | INFO | "huge" : "", | 20260505 11:22:29.568 | INFO | "huge_shared" : " 27 GB", | 20260505 11:22:29.568 | INFO | "malloc" : " 14 GB", | 20260505 11:22:29.569 | INFO | "pinned" : " 29 GB" | 20260505 11:22:29.569 | INFO | } | 20260505 11:22:29.569 | INFO | Perf report for the last 5.14117 seconds: | 20260505 11:22:29.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:22:29.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:29.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:29.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:29.569 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 11:22:29.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:29.569 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:29.569 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 11:22:29.569 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:29.569 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:29.569 | INFO | Uploaded 36GiB to GPU in 2.21686s (16.2392GiB/s) | 20260505 11:22:29.569 | INFO | Downloaded 34GiB to GPU in 2.12851s (15.9736GiB/s) | 20260505 11:22:34.706 | INFO | Analyzing chunk frames = [2815488,2816000) | 20260505 11:22:34.706 | INFO | MemInfo: { | 20260505 11:22:34.706 | INFO | "gpu" : " 17 GB", | 20260505 11:22:34.706 | INFO | "huge" : "", | 20260505 11:22:34.706 | INFO | "huge_shared" : " 27 GB", | 20260505 11:22:34.706 | INFO | "malloc" : " 14 GB", | 20260505 11:22:34.706 | INFO | "pinned" : " 29 GB" | 20260505 11:22:34.706 | INFO | } | 20260505 11:22:34.706 | INFO | Perf report for the last 5.13766 seconds: | 20260505 11:22:34.706 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:22:34.706 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:34.706 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:34.706 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:34.706 | INFO | ANALYSIS : 63.6%, 3.0, 98.8ms, 0 | 20260505 11:22:34.706 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:34.706 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:34.706 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 11:22:34.706 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:34.706 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:34.707 | INFO | Uploaded 36GiB to GPU in 2.21925s (16.2217GiB/s) | 20260505 11:22:34.707 | INFO | Downloaded 34GiB to GPU in 2.13226s (15.9455GiB/s) | 20260505 11:22:39.841 | INFO | Analyzing chunk frames = [2816000,2816512) | 20260505 11:22:39.841 | INFO | MemInfo: { | 20260505 11:22:39.841 | INFO | "gpu" : " 17 GB", | 20260505 11:22:39.841 | INFO | "huge" : "", | 20260505 11:22:39.841 | INFO | "huge_shared" : " 27 GB", | 20260505 11:22:39.841 | INFO | "malloc" : " 14 GB", | 20260505 11:22:39.841 | INFO | "pinned" : " 29 GB" | 20260505 11:22:39.841 | INFO | } | 20260505 11:22:39.841 | INFO | Perf report for the last 5.13489 seconds: | 20260505 11:22:39.841 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:22:39.841 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:39.841 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:39.841 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:39.841 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 11:22:39.841 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:39.841 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:39.841 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 11:22:39.841 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:39.841 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:39.841 | INFO | Uploaded 36GiB to GPU in 2.17278s (16.5687GiB/s) | 20260505 11:22:39.842 | INFO | Downloaded 34GiB to GPU in 2.12612s (15.9916GiB/s) | 20260505 11:22:44.985 | INFO | Analyzing chunk frames = [2816512,2817024) | 20260505 11:22:44.985 | INFO | MemInfo: { | 20260505 11:22:44.985 | INFO | "gpu" : " 17 GB", | 20260505 11:22:44.985 | INFO | "huge" : "", | 20260505 11:22:44.985 | INFO | "huge_shared" : " 27 GB", | 20260505 11:22:44.985 | INFO | "malloc" : " 14 GB", | 20260505 11:22:44.985 | INFO | "pinned" : " 29 GB" | 20260505 11:22:44.986 | INFO | } | 20260505 11:22:44.986 | INFO | Perf report for the last 5.1445 seconds: | 20260505 11:22:44.986 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:22:44.986 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:44.986 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:44.986 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:44.986 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 11:22:44.986 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:44.986 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:44.986 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 11:22:44.986 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:44.986 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:44.986 | INFO | Uploaded 36GiB to GPU in 2.16758s (16.6084GiB/s) | 20260505 11:22:44.986 | INFO | Downloaded 34GiB to GPU in 2.1117s (16.1008GiB/s) | 20260505 11:22:50.127 | INFO | Analyzing chunk frames = [2817024,2817536) | 20260505 11:22:50.127 | INFO | MemInfo: { | 20260505 11:22:50.127 | INFO | "gpu" : " 17 GB", | 20260505 11:22:50.127 | INFO | "huge" : "", | 20260505 11:22:50.127 | INFO | "huge_shared" : " 27 GB", | 20260505 11:22:50.127 | INFO | "malloc" : " 14 GB", | 20260505 11:22:50.127 | INFO | "pinned" : " 29 GB" | 20260505 11:22:50.127 | INFO | } | 20260505 11:22:50.127 | INFO | Perf report for the last 5.14124 seconds: | 20260505 11:22:50.127 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:22:50.127 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:50.127 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:50.127 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:50.127 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 11:22:50.127 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:50.127 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:50.127 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 11:22:50.127 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:50.127 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:50.127 | INFO | Uploaded 36GiB to GPU in 2.20165s (16.3514GiB/s) | 20260505 11:22:50.127 | INFO | Downloaded 34GiB to GPU in 2.14937s (15.8186GiB/s) | 20260505 11:22:55.259 | INFO | Analyzing chunk frames = [2817536,2818048) | 20260505 11:22:55.259 | INFO | MemInfo: { | 20260505 11:22:55.259 | INFO | "gpu" : " 17 GB", | 20260505 11:22:55.259 | INFO | "huge" : "", | 20260505 11:22:55.259 | INFO | "huge_shared" : " 27 GB", | 20260505 11:22:55.259 | INFO | "malloc" : " 14 GB", | 20260505 11:22:55.259 | INFO | "pinned" : " 29 GB" | 20260505 11:22:55.259 | INFO | } | 20260505 11:22:55.259 | INFO | Perf report for the last 5.1321 seconds: | 20260505 11:22:55.259 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:22:55.259 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:55.259 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:55.259 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:55.259 | INFO | ANALYSIS : 64.0%, 3.0, 99.8ms, 0 | 20260505 11:22:55.259 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:55.259 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:55.259 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 11:22:55.259 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:55.259 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:22:55.259 | INFO | Uploaded 36GiB to GPU in 2.21848s (16.2273GiB/s) | 20260505 11:22:55.259 | INFO | Downloaded 34GiB to GPU in 2.09945s (16.1947GiB/s) | 20260505 11:22:55.260 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 11:22:55.260 | INFO | Total mem usage: 15.3 GiB | 20260505 11:22:55.260 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 11:22:55.260 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -6.81%) | 20260505 11:22:55.260 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 11:22:55.260 | INFO | High Water Breakdown: | 20260505 11:22:55.260 | INFO | 61% : 8.76 GiB ( 92.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 11:22:55.260 | INFO | 38.8% : 5.57 GiB ( 83.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 11:22:55.260 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 11:22:55.260 | INFO | ------------------------------------------ | 20260505 11:22:55.260 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:22:55.260 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 11:22:55.260 | INFO | Total mem usage: 26.5 GiB | 20260505 11:22:55.260 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 11:22:55.260 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 11:22:55.260 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 11:22:55.260 | INFO | High Water Breakdown: | 20260505 11:22:55.260 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:22:55.260 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:22:55.260 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:22:55.260 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:22:55.260 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:22:55.260 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:22:55.260 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 11:22:55.260 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 11:22:55.260 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:22:55.260 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 11:22:55.260 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 11:22:55.260 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 11:22:55.260 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 11:22:55.260 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 11:22:55.260 | INFO | ------------------------------------------ | 20260505 11:22:55.260 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:22:55.260 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 11:22:55.260 | INFO | Total mem usage: 25.6 GiB | 20260505 11:22:55.260 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 11:22:55.260 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 11:22:55.260 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 11:22:55.260 | INFO | High Water Breakdown: | 20260505 11:22:55.260 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 11:22:55.260 | INFO | ------------------------------------------ | 20260505 11:22:55.260 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:22:55.260 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 11:22:55.260 | INFO | Total mem usage: 15.7 GiB | 20260505 11:22:55.260 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 11:22:55.260 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 11:22:55.260 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 11:22:55.260 | INFO | High Water Breakdown: | 20260505 11:22:55.260 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 11:22:55.260 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 11:22:55.260 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 11:22:55.261 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 11:22:55.261 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 11:22:55.261 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 11:22:55.261 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 11:22:55.261 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 11:22:55.261 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 11:22:55.261 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:22:55.261 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 11:22:55.261 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:22:55.261 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 11:22:55.261 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 11:22:55.261 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 11:22:55.261 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 11:22:55.261 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:22:55.261 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 11:22:55.261 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:22:55.261 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:22:55.261 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:22:55.261 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 11:22:55.261 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 11:22:55.261 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:22:55.261 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 11:22:55.261 | INFO | ------------------------------------------ | 20260505 11:22:55.261 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:22:55.261 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 11:22:55.261 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:22:55.261 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:22:55.261 | INFO | PerfReport: Full Execution -- 3.21e+01 / 7.72e+01, 1.38e+01 / 1.75e+00 | 20260505 11:22:55.261 | INFO | PerfReport: 0.29%, AggregateStats -- 9.26e-02 / 1.01e-01, 1.03e-02 / 4.12e-03 | 20260505 11:22:55.261 | INFO | PerfReport: 11.70%, Baseline -- 3.76e+00 / 4.44e+00, 5.87e-01 / 2.42e-01 | 20260505 11:22:55.261 | INFO | PerfReport: 3.28%, Histogram -- 1.06e+00 / 1.49e+00, 2.00e-01 / 8.27e-02 | 20260505 11:22:55.261 | INFO | PerfReport: 15.87%, DME -- 4.08e+01 / 4.52e+01, 4.39e+00 / 2.27e+00 | 20260505 11:22:55.261 | INFO | PerfReport: 56.41%, FrameLabeling -- 1.81e+01 / 2.11e+01, 1.60e+00 / 2.29e+00 | 20260505 11:22:55.261 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.39e+00 / 2.90e+00, 4.35e-01 / 2.25e-01 | 20260505 11:22:55.261 | INFO | PerfReport: 3.70%, Metrics -- 1.19e+00 / 1.51e+00, 3.40e-01 / 1.94e-01 | 20260505 11:22:55.261 | INFO | PerfReport: 1.30%, PulseSerializing -- 4.19e-01 / 4.66e-01, 7.60e-02 / 3.45e-02 | 20260505 11:22:55.261 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 11:22:55.261 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:22:55.261 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:22:55.261 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.48e+01 / 0.00e+00 | 20260505 11:22:55.261 | INFO | PerfReport: 22.66%, Upload -- 2.30e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260505 11:22:55.261 | INFO | PerfReport: 22.03%, Download -- 2.23e+01 / 0.00e+00, 8.11e+00 / 0.00e+00 | 20260505 11:22:55.261 | INFO | PerfReport: 31.74%, Compute -- 3.22e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 11:22:55.261 | INFO | PerfReport: 2.07%, LockUpload -- 2.10e+00 / 0.00e+00, 7.68e+00 / 0.00e+00 | 20260505 11:22:55.261 | INFO | PerfReport: 1.26%, LockDownload -- 1.27e+00 / 0.00e+00, 2.78e+00 / 0.00e+00 | 20260505 11:22:55.261 | INFO | PerfReport: 20.23%, LockCompute -- 2.05e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260505 11:22:55.261 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.66e-02 / 0.00e+00 | 20260505 11:22:55.391 | INFO | Starting a round of Baz IO | 20260505 11:22:58.650 | INFO | Populating RTMetrics report | 20260505 11:22:58.650 | INFO | RTMetrics report populated | 20260505 11:22:58.650 | INFO | Opening tmp RTMetrics json file | 20260505 11:22:58.674 | INFO | Serializing RTMetrics to json | 20260505 11:22:58.705 | INFO | Serialization done, swapping tmp file now... | 20260505 11:22:58.727 | INFO | RTMetrics done | 20260505 11:22:58.799 | INFO | Finished a round of Baz IO: Wrote 2.385 GiB in 3.408 seconds | 20260505 11:22:58.799 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.23% / 0.01% / 75.77% | 20260505 11:22:58.799 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.1732% / 1.19209e-05% / 5.5631e-05% | 20260505 11:22:58.799 | INFO | Baz IO overall: | 20260505 11:22:58.799 | INFO | BytesWritten : 1665223138887 | 20260505 11:22:58.799 | INFO | NumEvents : 792868867889 | 20260505 11:23:00.400 | INFO | Analyzing chunk frames = [2818048,2818560) | 20260505 11:23:00.400 | INFO | MemInfo: { | 20260505 11:23:00.400 | INFO | "gpu" : " 17 GB", | 20260505 11:23:00.400 | INFO | "huge" : "", | 20260505 11:23:00.400 | INFO | "huge_shared" : " 27 GB", | 20260505 11:23:00.400 | INFO | "malloc" : " 14 GB", | 20260505 11:23:00.400 | INFO | "pinned" : " 29 GB" | 20260505 11:23:00.400 | INFO | } | 20260505 11:23:00.400 | INFO | Perf report for the last 5.14076 seconds: | 20260505 11:23:00.400 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:23:00.400 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:00.400 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:00.400 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:00.400 | INFO | ANALYSIS : 64.8%, 2.9, 99.7ms, 0 | 20260505 11:23:00.400 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:00.400 | INFO | RT_METRICS : 46.9%, 1.0, 24.6ms, 0 | 20260505 11:23:00.400 | INFO | PRE_HQ : 62.5%, 1.8, 58.9ms, 0 | 20260505 11:23:00.400 | INFO | BAZWRITER : 65.6%, 4.1, 138.5ms, 0 | 20260505 11:23:00.400 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 11:23:00.400 | INFO | Uploaded 36GiB to GPU in 2.29671s (15.6746GiB/s) | 20260505 11:23:00.400 | INFO | Downloaded 38GiB to GPU in 2.37905s (15.9728GiB/s) | 20260505 11:23:05.550 | INFO | Analyzing chunk frames = [2818560,2819072) | 20260505 11:23:05.550 | INFO | MemInfo: { | 20260505 11:23:05.550 | INFO | "gpu" : " 17 GB", | 20260505 11:23:05.550 | INFO | "huge" : "", | 20260505 11:23:05.550 | INFO | "huge_shared" : " 27 GB", | 20260505 11:23:05.550 | INFO | "malloc" : " 14 GB", | 20260505 11:23:05.550 | INFO | "pinned" : " 29 GB" | 20260505 11:23:05.550 | INFO | } | 20260505 11:23:05.550 | INFO | Perf report for the last 5.15012 seconds: | 20260505 11:23:05.550 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:23:05.550 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:05.550 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:05.550 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:05.550 | INFO | ANALYSIS : 64.0%, 3.0, 100.1ms, 0 | 20260505 11:23:05.550 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:05.550 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:05.550 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 11:23:05.550 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:05.550 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:05.550 | INFO | Uploaded 36GiB to GPU in 2.2307s (16.1385GiB/s) | 20260505 11:23:05.550 | INFO | Downloaded 34GiB to GPU in 2.09097s (16.2604GiB/s) | 20260505 11:23:10.689 | INFO | Analyzing chunk frames = [2819072,2819584) | 20260505 11:23:10.689 | INFO | MemInfo: { | 20260505 11:23:10.689 | INFO | "gpu" : " 17 GB", | 20260505 11:23:10.689 | INFO | "huge" : "", | 20260505 11:23:10.689 | INFO | "huge_shared" : " 27 GB", | 20260505 11:23:10.689 | INFO | "malloc" : " 14 GB", | 20260505 11:23:10.689 | INFO | "pinned" : " 29 GB" | 20260505 11:23:10.689 | INFO | } | 20260505 11:23:10.689 | INFO | Perf report for the last 5.13931 seconds: | 20260505 11:23:10.689 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:23:10.689 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:10.689 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:10.690 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:10.690 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 11:23:10.690 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:10.690 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:10.690 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:23:10.690 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:10.690 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:10.690 | INFO | Uploaded 36GiB to GPU in 2.19576s (16.3952GiB/s) | 20260505 11:23:10.690 | INFO | Downloaded 34GiB to GPU in 2.11987s (16.0387GiB/s) | 20260505 11:23:15.821 | INFO | Analyzing chunk frames = [2819584,2820096) | 20260505 11:23:15.821 | INFO | MemInfo: { | 20260505 11:23:15.821 | INFO | "gpu" : " 17 GB", | 20260505 11:23:15.821 | INFO | "huge" : "", | 20260505 11:23:15.821 | INFO | "huge_shared" : " 27 GB", | 20260505 11:23:15.821 | INFO | "malloc" : " 14 GB", | 20260505 11:23:15.821 | INFO | "pinned" : " 29 GB" | 20260505 11:23:15.821 | INFO | } | 20260505 11:23:15.821 | INFO | Perf report for the last 5.13187 seconds: | 20260505 11:23:15.821 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:23:15.821 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:15.821 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:15.821 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:15.821 | INFO | ANALYSIS : 63.3%, 3.0, 98.1ms, 0 | 20260505 11:23:15.821 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:15.821 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:15.821 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 11:23:15.821 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:15.821 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:15.821 | INFO | Uploaded 36GiB to GPU in 2.20529s (16.3244GiB/s) | 20260505 11:23:15.821 | INFO | Downloaded 34GiB to GPU in 2.13267s (15.9424GiB/s) | 20260505 11:23:20.957 | INFO | Analyzing chunk frames = [2820096,2820608) | 20260505 11:23:20.957 | INFO | MemInfo: { | 20260505 11:23:20.957 | INFO | "gpu" : " 17 GB", | 20260505 11:23:20.957 | INFO | "huge" : "", | 20260505 11:23:20.957 | INFO | "huge_shared" : " 27 GB", | 20260505 11:23:20.957 | INFO | "malloc" : " 14 GB", | 20260505 11:23:20.957 | INFO | "pinned" : " 29 GB" | 20260505 11:23:20.957 | INFO | } | 20260505 11:23:20.957 | INFO | Perf report for the last 5.13597 seconds: | 20260505 11:23:20.957 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:23:20.957 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:20.957 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:20.957 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:20.957 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 11:23:20.957 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:20.957 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:20.957 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 11:23:20.957 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:20.957 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:20.957 | INFO | Uploaded 36GiB to GPU in 2.13845s (16.8346GiB/s) | 20260505 11:23:20.958 | INFO | Downloaded 34GiB to GPU in 2.11186s (16.0996GiB/s) | 20260505 11:23:26.098 | INFO | Analyzing chunk frames = [2820608,2821120) | 20260505 11:23:26.098 | INFO | MemInfo: { | 20260505 11:23:26.098 | INFO | "gpu" : " 17 GB", | 20260505 11:23:26.098 | INFO | "huge" : "", | 20260505 11:23:26.098 | INFO | "huge_shared" : " 27 GB", | 20260505 11:23:26.098 | INFO | "malloc" : " 14 GB", | 20260505 11:23:26.098 | INFO | "pinned" : " 29 GB" | 20260505 11:23:26.098 | INFO | } | 20260505 11:23:26.098 | INFO | Perf report for the last 5.14078 seconds: | 20260505 11:23:26.098 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:23:26.098 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:26.098 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:26.098 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:26.098 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 11:23:26.098 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:26.098 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:26.098 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:23:26.098 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:26.098 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:26.098 | INFO | Uploaded 36GiB to GPU in 2.21574s (16.2474GiB/s) | 20260505 11:23:26.098 | INFO | Downloaded 34GiB to GPU in 2.10493s (16.1526GiB/s) | 20260505 11:23:31.239 | INFO | Analyzing chunk frames = [2821120,2821632) | 20260505 11:23:31.239 | INFO | MemInfo: { | 20260505 11:23:31.239 | INFO | "gpu" : " 17 GB", | 20260505 11:23:31.239 | INFO | "huge" : "", | 20260505 11:23:31.239 | INFO | "huge_shared" : " 27 GB", | 20260505 11:23:31.239 | INFO | "malloc" : " 14 GB", | 20260505 11:23:31.239 | INFO | "pinned" : " 29 GB" | 20260505 11:23:31.239 | INFO | } | 20260505 11:23:31.240 | INFO | Perf report for the last 5.14162 seconds: | 20260505 11:23:31.240 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:23:31.240 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:31.240 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:31.240 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:31.240 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 11:23:31.240 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:31.240 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:31.240 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 11:23:31.240 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:31.240 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:31.240 | INFO | Uploaded 36GiB to GPU in 2.16862s (16.6004GiB/s) | 20260505 11:23:31.240 | INFO | Downloaded 34GiB to GPU in 2.13315s (15.9389GiB/s) | 20260505 11:23:36.384 | INFO | Analyzing chunk frames = [2821632,2822144) | 20260505 11:23:36.384 | INFO | MemInfo: { | 20260505 11:23:36.384 | INFO | "gpu" : " 17 GB", | 20260505 11:23:36.384 | INFO | "huge" : "", | 20260505 11:23:36.384 | INFO | "huge_shared" : " 27 GB", | 20260505 11:23:36.384 | INFO | "malloc" : " 14 GB", | 20260505 11:23:36.384 | INFO | "pinned" : " 29 GB" | 20260505 11:23:36.384 | INFO | } | 20260505 11:23:36.384 | INFO | Perf report for the last 5.14461 seconds: | 20260505 11:23:36.384 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:23:36.384 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:36.384 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:36.384 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:36.384 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 11:23:36.384 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:36.384 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:36.384 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 11:23:36.384 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:36.384 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:36.385 | INFO | Uploaded 36GiB to GPU in 2.21006s (16.2891GiB/s) | 20260505 11:23:36.385 | INFO | Downloaded 34GiB to GPU in 2.094s (16.2369GiB/s) | 20260505 11:23:36.518 | INFO | Starting a round of Baz IO | 20260505 11:23:39.777 | INFO | Populating RTMetrics report | 20260505 11:23:39.778 | INFO | RTMetrics report populated | 20260505 11:23:39.778 | INFO | Opening tmp RTMetrics json file | 20260505 11:23:39.794 | INFO | Serializing RTMetrics to json | 20260505 11:23:39.803 | INFO | Serialization done, swapping tmp file now... | 20260505 11:23:39.822 | INFO | RTMetrics done | 20260505 11:23:39.912 | INFO | Finished a round of Baz IO: Wrote 2.381 GiB in 3.393 seconds | 20260505 11:23:39.912 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.22% / 0.01% / 75.77% | 20260505 11:23:39.912 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.16635% / 1.58946e-05% / 7.54992e-05% | 20260505 11:23:39.912 | INFO | Baz IO overall: | 20260505 11:23:39.912 | INFO | BytesWritten : 1667779544983 | 20260505 11:23:39.912 | INFO | NumEvents : 794085425803 | 20260505 11:23:41.522 | INFO | Analyzing chunk frames = [2822144,2822656) | 20260505 11:23:41.522 | INFO | MemInfo: { | 20260505 11:23:41.522 | INFO | "gpu" : " 17 GB", | 20260505 11:23:41.522 | INFO | "huge" : "", | 20260505 11:23:41.522 | INFO | "huge_shared" : " 27 GB", | 20260505 11:23:41.522 | INFO | "malloc" : " 14 GB", | 20260505 11:23:41.522 | INFO | "pinned" : " 29 GB" | 20260505 11:23:41.523 | INFO | } | 20260505 11:23:41.523 | INFO | Perf report for the last 5.13834 seconds: | 20260505 11:23:41.523 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:23:41.523 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:41.523 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:41.523 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:41.523 | INFO | ANALYSIS : 64.9%, 2.9, 99.7ms, 0 | 20260505 11:23:41.523 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:41.523 | INFO | RT_METRICS : 46.2%, 1.0, 24.2ms, 0 | 20260505 11:23:41.523 | INFO | PRE_HQ : 61.2%, 1.8, 58.4ms, 0 | 20260505 11:23:41.523 | INFO | BAZWRITER : 65.3%, 4.2, 139.4ms, 0 | 20260505 11:23:41.523 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 11:23:41.523 | INFO | Uploaded 36GiB to GPU in 2.2253s (16.1776GiB/s) | 20260505 11:23:41.523 | INFO | Downloaded 38GiB to GPU in 2.29881s (16.5303GiB/s) | 20260505 11:23:46.653 | INFO | Analyzing chunk frames = [2822656,2823168) | 20260505 11:23:46.653 | INFO | MemInfo: { | 20260505 11:23:46.653 | INFO | "gpu" : " 17 GB", | 20260505 11:23:46.653 | INFO | "huge" : "", | 20260505 11:23:46.653 | INFO | "huge_shared" : " 27 GB", | 20260505 11:23:46.653 | INFO | "malloc" : " 14 GB", | 20260505 11:23:46.653 | INFO | "pinned" : " 29 GB" | 20260505 11:23:46.653 | INFO | } | 20260505 11:23:46.653 | INFO | Perf report for the last 5.13062 seconds: | 20260505 11:23:46.653 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:23:46.653 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:46.653 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:46.653 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:46.653 | INFO | ANALYSIS : 64.1%, 3.0, 99.9ms, 0 | 20260505 11:23:46.653 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:46.653 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:46.653 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:23:46.653 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:46.653 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:46.654 | INFO | Uploaded 36GiB to GPU in 2.21819s (16.2294GiB/s) | 20260505 11:23:46.654 | INFO | Downloaded 34GiB to GPU in 2.08345s (16.3191GiB/s) | 20260505 11:23:51.796 | INFO | Analyzing chunk frames = [2823168,2823680) | 20260505 11:23:51.796 | INFO | MemInfo: { | 20260505 11:23:51.796 | INFO | "gpu" : " 17 GB", | 20260505 11:23:51.796 | INFO | "huge" : "", | 20260505 11:23:51.796 | INFO | "huge_shared" : " 27 GB", | 20260505 11:23:51.796 | INFO | "malloc" : " 14 GB", | 20260505 11:23:51.796 | INFO | "pinned" : " 29 GB" | 20260505 11:23:51.796 | INFO | } | 20260505 11:23:51.796 | INFO | Perf report for the last 5.1429 seconds: | 20260505 11:23:51.796 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:23:51.796 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:51.796 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:51.796 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:51.796 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 11:23:51.796 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:51.796 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:51.796 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:23:51.796 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:51.796 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:51.796 | INFO | Uploaded 36GiB to GPU in 2.22351s (16.1906GiB/s) | 20260505 11:23:51.796 | INFO | Downloaded 34GiB to GPU in 2.129s (15.9699GiB/s) | 20260505 11:23:56.937 | INFO | Analyzing chunk frames = [2823680,2824192) | 20260505 11:23:56.937 | INFO | MemInfo: { | 20260505 11:23:56.937 | INFO | "gpu" : " 17 GB", | 20260505 11:23:56.937 | INFO | "huge" : "", | 20260505 11:23:56.937 | INFO | "huge_shared" : " 27 GB", | 20260505 11:23:56.937 | INFO | "malloc" : " 14 GB", | 20260505 11:23:56.937 | INFO | "pinned" : " 29 GB" | 20260505 11:23:56.937 | INFO | } | 20260505 11:23:56.937 | INFO | Perf report for the last 5.14074 seconds: | 20260505 11:23:56.937 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:23:56.937 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:56.937 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:56.937 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:56.937 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 11:23:56.937 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:56.937 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:56.937 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 11:23:56.937 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:56.937 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:23:56.937 | INFO | Uploaded 36GiB to GPU in 2.20076s (16.358GiB/s) | 20260505 11:23:56.937 | INFO | Downloaded 34GiB to GPU in 2.13063s (15.9577GiB/s) | 20260505 11:24:02.073 | INFO | Analyzing chunk frames = [2824192,2824704) | 20260505 11:24:02.073 | INFO | MemInfo: { | 20260505 11:24:02.073 | INFO | "gpu" : " 17 GB", | 20260505 11:24:02.073 | INFO | "huge" : "", | 20260505 11:24:02.073 | INFO | "huge_shared" : " 27 GB", | 20260505 11:24:02.073 | INFO | "malloc" : " 14 GB", | 20260505 11:24:02.073 | INFO | "pinned" : " 29 GB" | 20260505 11:24:02.073 | INFO | } | 20260505 11:24:02.074 | INFO | Perf report for the last 5.13662 seconds: | 20260505 11:24:02.074 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:24:02.074 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:02.074 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:02.074 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:02.074 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 11:24:02.074 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:02.074 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:02.074 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 11:24:02.074 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:02.074 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:02.074 | INFO | Uploaded 36GiB to GPU in 2.15312s (16.7199GiB/s) | 20260505 11:24:02.074 | INFO | Downloaded 34GiB to GPU in 2.12546s (15.9965GiB/s) | 20260505 11:24:07.208 | INFO | Analyzing chunk frames = [2824704,2825216) | 20260505 11:24:07.208 | INFO | MemInfo: { | 20260505 11:24:07.208 | INFO | "gpu" : " 17 GB", | 20260505 11:24:07.208 | INFO | "huge" : "", | 20260505 11:24:07.208 | INFO | "huge_shared" : " 27 GB", | 20260505 11:24:07.208 | INFO | "malloc" : " 14 GB", | 20260505 11:24:07.208 | INFO | "pinned" : " 29 GB" | 20260505 11:24:07.208 | INFO | } | 20260505 11:24:07.208 | INFO | Perf report for the last 5.13466 seconds: | 20260505 11:24:07.208 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:24:07.208 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:07.208 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:07.208 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:07.208 | INFO | ANALYSIS : 63.6%, 3.0, 99.1ms, 0 | 20260505 11:24:07.208 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:07.208 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:07.208 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 11:24:07.208 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:07.208 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:07.208 | INFO | Uploaded 36GiB to GPU in 2.20641s (16.3161GiB/s) | 20260505 11:24:07.209 | INFO | Downloaded 34GiB to GPU in 2.10491s (16.1527GiB/s) | 20260505 11:24:12.352 | INFO | Analyzing chunk frames = [2825216,2825728) | 20260505 11:24:12.352 | INFO | MemInfo: { | 20260505 11:24:12.352 | INFO | "gpu" : " 17 GB", | 20260505 11:24:12.352 | INFO | "huge" : "", | 20260505 11:24:12.352 | INFO | "huge_shared" : " 27 GB", | 20260505 11:24:12.352 | INFO | "malloc" : " 14 GB", | 20260505 11:24:12.352 | INFO | "pinned" : " 29 GB" | 20260505 11:24:12.352 | INFO | } | 20260505 11:24:12.352 | INFO | Perf report for the last 5.1442 seconds: | 20260505 11:24:12.352 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:24:12.352 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:12.352 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:12.352 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:12.352 | INFO | ANALYSIS : 63.1%, 3.0, 98.7ms, 0 | 20260505 11:24:12.352 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:12.352 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:12.353 | INFO | PRE_HQ : 30.3%, 1.0, 15.8ms, 0 | 20260505 11:24:12.353 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:12.353 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:12.353 | INFO | Uploaded 36GiB to GPU in 2.13628s (16.8517GiB/s) | 20260505 11:24:12.353 | INFO | Downloaded 34GiB to GPU in 2.14902s (15.8212GiB/s) | 20260505 11:24:17.503 | INFO | Analyzing chunk frames = [2825728,2826240) | 20260505 11:24:17.503 | INFO | MemInfo: { | 20260505 11:24:17.503 | INFO | "gpu" : " 17 GB", | 20260505 11:24:17.503 | INFO | "huge" : "", | 20260505 11:24:17.503 | INFO | "huge_shared" : " 27 GB", | 20260505 11:24:17.503 | INFO | "malloc" : " 14 GB", | 20260505 11:24:17.503 | INFO | "pinned" : " 29 GB" | 20260505 11:24:17.503 | INFO | } | 20260505 11:24:17.503 | INFO | Perf report for the last 5.15083 seconds: | 20260505 11:24:17.503 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:24:17.503 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:17.503 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:17.503 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:17.503 | INFO | ANALYSIS : 63.8%, 3.0, 99.9ms, 0 | 20260505 11:24:17.503 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:17.503 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:17.503 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 11:24:17.503 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:17.503 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:17.504 | INFO | Uploaded 36GiB to GPU in 2.21681s (16.2396GiB/s) | 20260505 11:24:17.504 | INFO | Downloaded 34GiB to GPU in 2.08806s (16.283GiB/s) | 20260505 11:24:17.504 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 11:24:17.504 | INFO | Total mem usage: 15.3 GiB | 20260505 11:24:17.504 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 11:24:17.504 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -6.82%) | 20260505 11:24:17.504 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 11:24:17.504 | INFO | High Water Breakdown: | 20260505 11:24:17.504 | INFO | 61% : 8.76 GiB ( 92.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 11:24:17.504 | INFO | 38.8% : 5.57 GiB ( 83.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 11:24:17.504 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 11:24:17.504 | INFO | ------------------------------------------ | 20260505 11:24:17.504 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:24:17.504 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 11:24:17.504 | INFO | Total mem usage: 26.5 GiB | 20260505 11:24:17.504 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 11:24:17.504 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 11:24:17.504 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 11:24:17.504 | INFO | High Water Breakdown: | 20260505 11:24:17.504 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:24:17.504 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:24:17.504 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:24:17.504 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:24:17.504 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:24:17.504 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:24:17.504 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 11:24:17.504 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 11:24:17.504 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:24:17.504 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 11:24:17.504 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 11:24:17.504 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 11:24:17.504 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 11:24:17.504 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 11:24:17.504 | INFO | ------------------------------------------ | 20260505 11:24:17.504 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:24:17.504 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 11:24:17.504 | INFO | Total mem usage: 25.6 GiB | 20260505 11:24:17.504 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 11:24:17.504 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 11:24:17.504 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 11:24:17.504 | INFO | High Water Breakdown: | 20260505 11:24:17.504 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 11:24:17.504 | INFO | ------------------------------------------ | 20260505 11:24:17.504 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:24:17.504 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 11:24:17.504 | INFO | Total mem usage: 15.7 GiB | 20260505 11:24:17.504 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 11:24:17.504 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 11:24:17.504 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 11:24:17.504 | INFO | High Water Breakdown: | 20260505 11:24:17.504 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 11:24:17.504 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 11:24:17.504 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 11:24:17.504 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 11:24:17.504 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 11:24:17.504 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 11:24:17.504 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 11:24:17.504 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 11:24:17.505 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 11:24:17.505 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:24:17.505 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 11:24:17.505 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:24:17.505 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 11:24:17.505 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 11:24:17.505 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 11:24:17.505 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 11:24:17.505 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:24:17.505 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 11:24:17.505 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:24:17.505 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:24:17.505 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:24:17.505 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 11:24:17.505 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 11:24:17.505 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:24:17.505 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 11:24:17.505 | INFO | ------------------------------------------ | 20260505 11:24:17.505 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:24:17.505 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 11:24:17.505 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:24:17.505 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:24:17.505 | INFO | PerfReport: Full Execution -- 3.22e+01 / 7.73e+01, 1.38e+01 / 1.73e+00 | 20260505 11:24:17.505 | INFO | PerfReport: 0.29%, AggregateStats -- 9.31e-02 / 1.01e-01, 1.07e-02 / 4.23e-03 | 20260505 11:24:17.505 | INFO | PerfReport: 11.73%, Baseline -- 3.77e+00 / 4.47e+00, 6.07e-01 / 2.63e-01 | 20260505 11:24:17.505 | INFO | PerfReport: 3.32%, Histogram -- 1.07e+00 / 1.49e+00, 2.05e-01 / 1.06e-01 | 20260505 11:24:17.505 | INFO | PerfReport: 15.74%, DME -- 4.05e+01 / 4.48e+01, 4.35e+00 / 1.98e+00 | 20260505 11:24:17.505 | INFO | PerfReport: 56.50%, FrameLabeling -- 1.82e+01 / 2.17e+01, 1.62e+00 / 2.03e+00 | 20260505 11:24:17.505 | INFO | PerfReport: 7.40%, PulseAccumulating -- 2.38e+00 / 2.81e+00, 4.23e-01 / 2.10e-01 | 20260505 11:24:17.505 | INFO | PerfReport: 3.72%, Metrics -- 1.20e+00 / 1.50e+00, 3.48e-01 / 2.09e-01 | 20260505 11:24:17.505 | INFO | PerfReport: 1.30%, PulseSerializing -- 4.20e-01 / 4.68e-01, 7.67e-02 / 2.73e-02 | 20260505 11:24:17.505 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 11:24:17.505 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:24:17.505 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:24:17.505 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.47e+01 / 0.00e+00 | 20260505 11:24:17.505 | INFO | PerfReport: 22.67%, Upload -- 2.29e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260505 11:24:17.505 | INFO | PerfReport: 22.05%, Download -- 2.23e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260505 11:24:17.505 | INFO | PerfReport: 31.79%, Compute -- 3.22e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 11:24:17.505 | INFO | PerfReport: 2.02%, LockUpload -- 2.04e+00 / 0.00e+00, 7.70e+00 / 0.00e+00 | 20260505 11:24:17.505 | INFO | PerfReport: 1.28%, LockDownload -- 1.29e+00 / 0.00e+00, 2.80e+00 / 0.00e+00 | 20260505 11:24:17.505 | INFO | PerfReport: 20.18%, LockCompute -- 2.04e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260505 11:24:17.505 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.51e-02 / 0.00e+00 | 20260505 11:24:17.639 | INFO | Starting a round of Baz IO | 20260505 11:24:20.878 | INFO | Populating RTMetrics report | 20260505 11:24:20.879 | INFO | RTMetrics report populated | 20260505 11:24:20.879 | INFO | Opening tmp RTMetrics json file | 20260505 11:24:20.896 | INFO | Serializing RTMetrics to json | 20260505 11:24:20.935 | INFO | Serialization done, swapping tmp file now... | 20260505 11:24:20.943 | INFO | RTMetrics done | 20260505 11:24:21.011 | INFO | Finished a round of Baz IO: Wrote 2.382 GiB in 3.371 seconds | 20260505 11:24:21.011 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.21% / 0.01% / 75.78% | 20260505 11:24:21.011 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.16361% / 0% / 8.74201e-05% | 20260505 11:24:21.011 | INFO | Baz IO overall: | 20260505 11:24:21.011 | INFO | BytesWritten : 1670337404618 | 20260505 11:24:21.011 | INFO | NumEvents : 795302893838 | 20260505 11:24:22.639 | INFO | Analyzing chunk frames = [2826240,2826752) | 20260505 11:24:22.639 | INFO | MemInfo: { | 20260505 11:24:22.639 | INFO | "gpu" : " 17 GB", | 20260505 11:24:22.639 | INFO | "huge" : "", | 20260505 11:24:22.639 | INFO | "huge_shared" : " 27 GB", | 20260505 11:24:22.639 | INFO | "malloc" : " 14 GB", | 20260505 11:24:22.639 | INFO | "pinned" : " 29 GB" | 20260505 11:24:22.639 | INFO | } | 20260505 11:24:22.639 | INFO | Perf report for the last 5.13582 seconds: | 20260505 11:24:22.639 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:24:22.639 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:22.639 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:22.639 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:22.639 | INFO | ANALYSIS : 64.6%, 2.9, 98.9ms, 0 | 20260505 11:24:22.639 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:22.639 | INFO | RT_METRICS : 46.7%, 1.0, 24.5ms, 0 | 20260505 11:24:22.639 | INFO | PRE_HQ : 61.1%, 1.9, 59.0ms, 0 | 20260505 11:24:22.639 | INFO | BAZWRITER : 65.0%, 4.2, 138.8ms, 0 | 20260505 11:24:22.639 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260505 11:24:22.639 | INFO | Uploaded 36GiB to GPU in 2.19166s (16.4259GiB/s) | 20260505 11:24:22.639 | INFO | Downloaded 38GiB to GPU in 2.29129s (16.5845GiB/s) | 20260505 11:24:27.765 | INFO | Analyzing chunk frames = [2826752,2827264) | 20260505 11:24:27.765 | INFO | MemInfo: { | 20260505 11:24:27.765 | INFO | "gpu" : " 17 GB", | 20260505 11:24:27.765 | INFO | "huge" : "", | 20260505 11:24:27.765 | INFO | "huge_shared" : " 27 GB", | 20260505 11:24:27.765 | INFO | "malloc" : " 14 GB", | 20260505 11:24:27.765 | INFO | "pinned" : " 29 GB" | 20260505 11:24:27.765 | INFO | } | 20260505 11:24:27.765 | INFO | Perf report for the last 5.12577 seconds: | 20260505 11:24:27.765 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:24:27.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:27.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:27.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:27.765 | INFO | ANALYSIS : 64.3%, 3.0, 100.2ms, 0 | 20260505 11:24:27.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:27.765 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:27.765 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 11:24:27.765 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:27.765 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:27.765 | INFO | Uploaded 36GiB to GPU in 2.22786s (16.159GiB/s) | 20260505 11:24:27.765 | INFO | Downloaded 34GiB to GPU in 2.08988s (16.2689GiB/s) | 20260505 11:24:32.904 | INFO | Analyzing chunk frames = [2827264,2827776) | 20260505 11:24:32.904 | INFO | MemInfo: { | 20260505 11:24:32.904 | INFO | "gpu" : " 17 GB", | 20260505 11:24:32.904 | INFO | "huge" : "", | 20260505 11:24:32.904 | INFO | "huge_shared" : " 27 GB", | 20260505 11:24:32.904 | INFO | "malloc" : " 14 GB", | 20260505 11:24:32.904 | INFO | "pinned" : " 29 GB" | 20260505 11:24:32.904 | INFO | } | 20260505 11:24:32.904 | INFO | Perf report for the last 5.13959 seconds: | 20260505 11:24:32.905 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:24:32.905 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:32.905 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:32.905 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:32.905 | INFO | ANALYSIS : 62.9%, 3.0, 97.8ms, 0 | 20260505 11:24:32.905 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:32.905 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:32.905 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 11:24:32.905 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:32.905 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:32.905 | INFO | Uploaded 36GiB to GPU in 2.16079s (16.6606GiB/s) | 20260505 11:24:32.905 | INFO | Downloaded 34GiB to GPU in 2.06785s (16.4422GiB/s) | 20260505 11:24:38.052 | INFO | Analyzing chunk frames = [2827776,2828288) | 20260505 11:24:38.052 | INFO | MemInfo: { | 20260505 11:24:38.052 | INFO | "gpu" : " 17 GB", | 20260505 11:24:38.052 | INFO | "huge" : "", | 20260505 11:24:38.052 | INFO | "huge_shared" : " 27 GB", | 20260505 11:24:38.052 | INFO | "malloc" : " 14 GB", | 20260505 11:24:38.052 | INFO | "pinned" : " 29 GB" | 20260505 11:24:38.052 | INFO | } | 20260505 11:24:38.052 | INFO | Perf report for the last 5.14758 seconds: | 20260505 11:24:38.052 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:24:38.052 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:38.052 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:38.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:38.052 | INFO | ANALYSIS : 62.9%, 3.0, 97.8ms, 0 | 20260505 11:24:38.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:38.052 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:38.052 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 11:24:38.052 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:38.052 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:38.052 | INFO | Uploaded 36GiB to GPU in 2.16436s (16.6331GiB/s) | 20260505 11:24:38.052 | INFO | Downloaded 34GiB to GPU in 2.07948s (16.3503GiB/s) | 20260505 11:24:43.194 | INFO | Analyzing chunk frames = [2828288,2828800) | 20260505 11:24:43.194 | INFO | MemInfo: { | 20260505 11:24:43.194 | INFO | "gpu" : " 17 GB", | 20260505 11:24:43.194 | INFO | "huge" : "", | 20260505 11:24:43.194 | INFO | "huge_shared" : " 27 GB", | 20260505 11:24:43.194 | INFO | "malloc" : " 14 GB", | 20260505 11:24:43.194 | INFO | "pinned" : " 29 GB" | 20260505 11:24:43.194 | INFO | } | 20260505 11:24:43.194 | INFO | Perf report for the last 5.14227 seconds: | 20260505 11:24:43.194 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:24:43.194 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:43.194 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:43.194 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:43.194 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 11:24:43.194 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:43.194 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:43.194 | INFO | PRE_HQ : 29.1%, 1.0, 15.2ms, 0 | 20260505 11:24:43.194 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:43.194 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:43.195 | INFO | Uploaded 36GiB to GPU in 2.18716s (16.4597GiB/s) | 20260505 11:24:43.195 | INFO | Downloaded 34GiB to GPU in 2.12467s (16.0025GiB/s) | 20260505 11:24:48.335 | INFO | Analyzing chunk frames = [2828800,2829312) | 20260505 11:24:48.335 | INFO | MemInfo: { | 20260505 11:24:48.335 | INFO | "gpu" : " 17 GB", | 20260505 11:24:48.335 | INFO | "huge" : "", | 20260505 11:24:48.335 | INFO | "huge_shared" : " 27 GB", | 20260505 11:24:48.335 | INFO | "malloc" : " 14 GB", | 20260505 11:24:48.335 | INFO | "pinned" : " 29 GB" | 20260505 11:24:48.335 | INFO | } | 20260505 11:24:48.335 | INFO | Perf report for the last 5.14063 seconds: | 20260505 11:24:48.335 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:24:48.335 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:48.335 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:48.335 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:48.335 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 11:24:48.335 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:48.335 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:48.335 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 11:24:48.335 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:48.335 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:48.335 | INFO | Uploaded 36GiB to GPU in 2.13927s (16.8282GiB/s) | 20260505 11:24:48.335 | INFO | Downloaded 34GiB to GPU in 2.1077s (16.1314GiB/s) | 20260505 11:24:53.468 | INFO | Analyzing chunk frames = [2829312,2829824) | 20260505 11:24:53.468 | INFO | MemInfo: { | 20260505 11:24:53.468 | INFO | "gpu" : " 17 GB", | 20260505 11:24:53.468 | INFO | "huge" : "", | 20260505 11:24:53.468 | INFO | "huge_shared" : " 27 GB", | 20260505 11:24:53.468 | INFO | "malloc" : " 14 GB", | 20260505 11:24:53.468 | INFO | "pinned" : " 29 GB" | 20260505 11:24:53.468 | INFO | } | 20260505 11:24:53.469 | INFO | Perf report for the last 5.13354 seconds: | 20260505 11:24:53.469 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:24:53.469 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:53.469 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:53.469 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:53.469 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 11:24:53.469 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:53.469 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:53.469 | INFO | PRE_HQ : 30.0%, 1.0, 15.5ms, 0 | 20260505 11:24:53.469 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:53.469 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:53.469 | INFO | Uploaded 36GiB to GPU in 2.16925s (16.5956GiB/s) | 20260505 11:24:53.469 | INFO | Downloaded 34GiB to GPU in 2.12011s (16.0369GiB/s) | 20260505 11:24:58.610 | INFO | Analyzing chunk frames = [2829824,2830336) | 20260505 11:24:58.610 | INFO | MemInfo: { | 20260505 11:24:58.610 | INFO | "gpu" : " 17 GB", | 20260505 11:24:58.610 | INFO | "huge" : "", | 20260505 11:24:58.610 | INFO | "huge_shared" : " 27 GB", | 20260505 11:24:58.610 | INFO | "malloc" : " 14 GB", | 20260505 11:24:58.610 | INFO | "pinned" : " 29 GB" | 20260505 11:24:58.610 | INFO | } | 20260505 11:24:58.610 | INFO | Perf report for the last 5.14126 seconds: | 20260505 11:24:58.610 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:24:58.610 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:58.610 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:58.610 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:58.610 | INFO | ANALYSIS : 63.9%, 3.0, 99.9ms, 0 | 20260505 11:24:58.610 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:58.610 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:58.610 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 11:24:58.610 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:58.610 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:24:58.610 | INFO | Uploaded 36GiB to GPU in 2.21684s (16.2393GiB/s) | 20260505 11:24:58.610 | INFO | Downloaded 34GiB to GPU in 2.09712s (16.2127GiB/s) | 20260505 11:24:58.746 | INFO | Starting a round of Baz IO | 20260505 11:25:01.997 | INFO | Populating RTMetrics report | 20260505 11:25:01.997 | INFO | RTMetrics report populated | 20260505 11:25:01.997 | INFO | Opening tmp RTMetrics json file | 20260505 11:25:02.036 | INFO | Serializing RTMetrics to json | 20260505 11:25:02.040 | INFO | Serialization done, swapping tmp file now... | 20260505 11:25:02.042 | INFO | RTMetrics done | 20260505 11:25:02.129 | INFO | Finished a round of Baz IO: Wrote 2.379 GiB in 3.382 seconds | 20260505 11:25:02.129 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.20% / 0.01% / 75.79% | 20260505 11:25:02.129 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.15705% / 1.58946e-05% / 4.76837e-05% | 20260505 11:25:02.129 | INFO | Baz IO overall: | 20260505 11:25:02.129 | INFO | BytesWritten : 1672892332701 | 20260505 11:25:02.129 | INFO | NumEvents : 796518349595 | 20260505 11:25:03.750 | INFO | Analyzing chunk frames = [2830336,2830848) | 20260505 11:25:03.750 | INFO | MemInfo: { | 20260505 11:25:03.750 | INFO | "gpu" : " 17 GB", | 20260505 11:25:03.750 | INFO | "huge" : "", | 20260505 11:25:03.750 | INFO | "huge_shared" : " 27 GB", | 20260505 11:25:03.750 | INFO | "malloc" : " 14 GB", | 20260505 11:25:03.750 | INFO | "pinned" : " 29 GB" | 20260505 11:25:03.750 | INFO | } | 20260505 11:25:03.750 | INFO | Perf report for the last 5.14007 seconds: | 20260505 11:25:03.750 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:25:03.750 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:03.750 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:03.750 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:03.750 | INFO | ANALYSIS : 64.8%, 2.9, 99.6ms, 0 | 20260505 11:25:03.750 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:03.750 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260505 11:25:03.750 | INFO | PRE_HQ : 61.3%, 1.8, 58.8ms, 0 | 20260505 11:25:03.750 | INFO | BAZWRITER : 65.1%, 4.2, 139.1ms, 0 | 20260505 11:25:03.750 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 11:25:03.750 | INFO | Uploaded 36GiB to GPU in 2.20215s (16.3477GiB/s) | 20260505 11:25:03.750 | INFO | Downloaded 38GiB to GPU in 2.29943s (16.5258GiB/s) | 20260505 11:25:08.891 | INFO | Analyzing chunk frames = [2830848,2831360) | 20260505 11:25:08.891 | INFO | MemInfo: { | 20260505 11:25:08.891 | INFO | "gpu" : " 17 GB", | 20260505 11:25:08.891 | INFO | "huge" : "", | 20260505 11:25:08.891 | INFO | "huge_shared" : " 27 GB", | 20260505 11:25:08.891 | INFO | "malloc" : " 14 GB", | 20260505 11:25:08.891 | INFO | "pinned" : " 29 GB" | 20260505 11:25:08.891 | INFO | } | 20260505 11:25:08.892 | INFO | Perf report for the last 5.14168 seconds: | 20260505 11:25:08.892 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:25:08.892 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:08.892 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:08.892 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:08.892 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 11:25:08.892 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:08.892 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:08.892 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 11:25:08.892 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:08.892 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:08.892 | INFO | Uploaded 36GiB to GPU in 2.23001s (16.1434GiB/s) | 20260505 11:25:08.892 | INFO | Downloaded 34GiB to GPU in 2.09929s (16.196GiB/s) | 20260505 11:25:14.019 | INFO | Analyzing chunk frames = [2831360,2831872) | 20260505 11:25:14.019 | INFO | MemInfo: { | 20260505 11:25:14.019 | INFO | "gpu" : " 17 GB", | 20260505 11:25:14.019 | INFO | "huge" : "", | 20260505 11:25:14.019 | INFO | "huge_shared" : " 27 GB", | 20260505 11:25:14.019 | INFO | "malloc" : " 14 GB", | 20260505 11:25:14.019 | INFO | "pinned" : " 29 GB" | 20260505 11:25:14.019 | INFO | } | 20260505 11:25:14.019 | INFO | Perf report for the last 5.12789 seconds: | 20260505 11:25:14.019 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:25:14.019 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:14.019 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:14.019 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:14.020 | INFO | ANALYSIS : 63.7%, 3.0, 98.8ms, 0 | 20260505 11:25:14.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:14.020 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:14.020 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:25:14.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:14.020 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:14.020 | INFO | Uploaded 36GiB to GPU in 2.22519s (16.1784GiB/s) | 20260505 11:25:14.020 | INFO | Downloaded 34GiB to GPU in 2.1282s (15.976GiB/s) | 20260505 11:25:19.160 | INFO | Analyzing chunk frames = [2831872,2832384) | 20260505 11:25:19.160 | INFO | MemInfo: { | 20260505 11:25:19.160 | INFO | "gpu" : " 17 GB", | 20260505 11:25:19.160 | INFO | "huge" : "", | 20260505 11:25:19.160 | INFO | "huge_shared" : " 27 GB", | 20260505 11:25:19.160 | INFO | "malloc" : " 14 GB", | 20260505 11:25:19.160 | INFO | "pinned" : " 29 GB" | 20260505 11:25:19.160 | INFO | } | 20260505 11:25:19.160 | INFO | Perf report for the last 5.14101 seconds: | 20260505 11:25:19.160 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:25:19.160 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:19.160 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:19.161 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:19.161 | INFO | ANALYSIS : 62.9%, 3.0, 97.8ms, 0 | 20260505 11:25:19.161 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:19.161 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:19.161 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 11:25:19.161 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:19.161 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:19.161 | INFO | Uploaded 36GiB to GPU in 2.16176s (16.6531GiB/s) | 20260505 11:25:19.161 | INFO | Downloaded 34GiB to GPU in 2.0732s (16.3997GiB/s) | 20260505 11:25:24.311 | INFO | Analyzing chunk frames = [2832384,2832896) | 20260505 11:25:24.311 | INFO | MemInfo: { | 20260505 11:25:24.311 | INFO | "gpu" : " 17 GB", | 20260505 11:25:24.311 | INFO | "huge" : "", | 20260505 11:25:24.311 | INFO | "huge_shared" : " 27 GB", | 20260505 11:25:24.311 | INFO | "malloc" : " 14 GB", | 20260505 11:25:24.311 | INFO | "pinned" : " 29 GB" | 20260505 11:25:24.312 | INFO | } | 20260505 11:25:24.312 | INFO | Perf report for the last 5.15111 seconds: | 20260505 11:25:24.312 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:25:24.312 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:24.312 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:24.312 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:24.312 | INFO | ANALYSIS : 63.1%, 3.0, 98.8ms, 0 | 20260505 11:25:24.312 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:24.312 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:24.312 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 11:25:24.312 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:24.312 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:24.312 | INFO | Uploaded 36GiB to GPU in 2.15916s (16.6731GiB/s) | 20260505 11:25:24.312 | INFO | Downloaded 34GiB to GPU in 2.12117s (16.0289GiB/s) | 20260505 11:25:29.447 | INFO | Analyzing chunk frames = [2832896,2833408) | 20260505 11:25:29.447 | INFO | MemInfo: { | 20260505 11:25:29.447 | INFO | "gpu" : " 17 GB", | 20260505 11:25:29.447 | INFO | "huge" : "", | 20260505 11:25:29.447 | INFO | "huge_shared" : " 27 GB", | 20260505 11:25:29.447 | INFO | "malloc" : " 14 GB", | 20260505 11:25:29.447 | INFO | "pinned" : " 29 GB" | 20260505 11:25:29.447 | INFO | } | 20260505 11:25:29.447 | INFO | Perf report for the last 5.13553 seconds: | 20260505 11:25:29.447 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:25:29.447 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:29.447 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:29.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:29.447 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 11:25:29.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:29.447 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:29.447 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 11:25:29.447 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:29.447 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:29.447 | INFO | Uploaded 36GiB to GPU in 2.14965s (16.7469GiB/s) | 20260505 11:25:29.447 | INFO | Downloaded 34GiB to GPU in 2.1064s (16.1413GiB/s) | 20260505 11:25:34.585 | INFO | Analyzing chunk frames = [2833408,2833920) | 20260505 11:25:34.585 | INFO | MemInfo: { | 20260505 11:25:34.585 | INFO | "gpu" : " 17 GB", | 20260505 11:25:34.585 | INFO | "huge" : "", | 20260505 11:25:34.585 | INFO | "huge_shared" : " 27 GB", | 20260505 11:25:34.585 | INFO | "malloc" : " 14 GB", | 20260505 11:25:34.585 | INFO | "pinned" : " 29 GB" | 20260505 11:25:34.585 | INFO | } | 20260505 11:25:34.585 | INFO | Perf report for the last 5.13768 seconds: | 20260505 11:25:34.585 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:25:34.585 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:34.585 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:34.585 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:34.585 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 11:25:34.585 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:34.585 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:34.585 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 11:25:34.585 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:34.585 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:34.585 | INFO | Uploaded 36GiB to GPU in 2.17465s (16.5544GiB/s) | 20260505 11:25:34.585 | INFO | Downloaded 34GiB to GPU in 2.12067s (16.0327GiB/s) | 20260505 11:25:39.718 | INFO | Analyzing chunk frames = [2833920,2834432) | 20260505 11:25:39.718 | INFO | MemInfo: { | 20260505 11:25:39.718 | INFO | "gpu" : " 17 GB", | 20260505 11:25:39.718 | INFO | "huge" : "", | 20260505 11:25:39.718 | INFO | "huge_shared" : " 27 GB", | 20260505 11:25:39.718 | INFO | "malloc" : " 14 GB", | 20260505 11:25:39.718 | INFO | "pinned" : " 29 GB" | 20260505 11:25:39.718 | INFO | } | 20260505 11:25:39.718 | INFO | Perf report for the last 5.13316 seconds: | 20260505 11:25:39.718 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:25:39.718 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:39.718 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:39.718 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:39.718 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 11:25:39.718 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:39.718 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:39.718 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 11:25:39.718 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:39.718 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:39.718 | INFO | Uploaded 36GiB to GPU in 2.23069s (16.1385GiB/s) | 20260505 11:25:39.718 | INFO | Downloaded 34GiB to GPU in 2.09418s (16.2355GiB/s) | 20260505 11:25:39.719 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 11:25:39.719 | INFO | Total mem usage: 15.3 GiB | 20260505 11:25:39.719 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 11:25:39.719 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -6.81%) | 20260505 11:25:39.719 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 11:25:39.719 | INFO | High Water Breakdown: | 20260505 11:25:39.719 | INFO | 61% : 8.76 GiB ( 92.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 11:25:39.719 | INFO | 38.8% : 5.57 GiB ( 83.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 11:25:39.719 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 11:25:39.719 | INFO | ------------------------------------------ | 20260505 11:25:39.719 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:25:39.719 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 11:25:39.719 | INFO | Total mem usage: 26.5 GiB | 20260505 11:25:39.719 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 11:25:39.719 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 11:25:39.719 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 11:25:39.719 | INFO | High Water Breakdown: | 20260505 11:25:39.719 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:25:39.719 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:25:39.719 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:25:39.719 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:25:39.719 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:25:39.719 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:25:39.719 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 11:25:39.719 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 11:25:39.719 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:25:39.719 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 11:25:39.719 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 11:25:39.719 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 11:25:39.719 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 11:25:39.719 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 11:25:39.719 | INFO | ------------------------------------------ | 20260505 11:25:39.719 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:25:39.719 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 11:25:39.719 | INFO | Total mem usage: 25.6 GiB | 20260505 11:25:39.719 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 11:25:39.719 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 11:25:39.719 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 11:25:39.719 | INFO | High Water Breakdown: | 20260505 11:25:39.719 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 11:25:39.719 | INFO | ------------------------------------------ | 20260505 11:25:39.719 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:25:39.719 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 11:25:39.719 | INFO | Total mem usage: 15.7 GiB | 20260505 11:25:39.719 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 11:25:39.719 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 11:25:39.719 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 11:25:39.719 | INFO | High Water Breakdown: | 20260505 11:25:39.719 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 11:25:39.719 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 11:25:39.719 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 11:25:39.719 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 11:25:39.719 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 11:25:39.719 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 11:25:39.719 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 11:25:39.719 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 11:25:39.719 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 11:25:39.719 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:25:39.719 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 11:25:39.719 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:25:39.719 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 11:25:39.719 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 11:25:39.719 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 11:25:39.719 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 11:25:39.719 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:25:39.720 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 11:25:39.720 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:25:39.720 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:25:39.720 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:25:39.720 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 11:25:39.720 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 11:25:39.720 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:25:39.720 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 11:25:39.720 | INFO | ------------------------------------------ | 20260505 11:25:39.720 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:25:39.720 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 11:25:39.720 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:25:39.720 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:25:39.720 | INFO | PerfReport: Full Execution -- 3.21e+01 / 7.69e+01, 1.39e+01 / 1.52e+00 | 20260505 11:25:39.720 | INFO | PerfReport: 0.29%, AggregateStats -- 9.30e-02 / 1.00e-01, 1.05e-02 / 3.30e-03 | 20260505 11:25:39.720 | INFO | PerfReport: 11.72%, Baseline -- 3.76e+00 / 4.53e+00, 5.98e-01 / 1.13e-01 | 20260505 11:25:39.720 | INFO | PerfReport: 3.31%, Histogram -- 1.06e+00 / 1.49e+00, 2.05e-01 / 9.26e-02 | 20260505 11:25:39.720 | INFO | PerfReport: 15.88%, DME -- 4.08e+01 / 4.55e+01, 4.31e+00 / 1.78e+00 | 20260505 11:25:39.720 | INFO | PerfReport: 56.31%, FrameLabeling -- 1.81e+01 / 2.04e+01, 1.56e+00 / 1.71e+00 | 20260505 11:25:39.720 | INFO | PerfReport: 7.48%, PulseAccumulating -- 2.40e+00 / 2.93e+00, 4.34e-01 / 1.85e-01 | 20260505 11:25:39.720 | INFO | PerfReport: 3.70%, Metrics -- 1.19e+00 / 1.51e+00, 3.31e-01 / 1.96e-01 | 20260505 11:25:39.720 | INFO | PerfReport: 1.30%, PulseSerializing -- 4.18e-01 / 4.70e-01, 7.56e-02 / 3.30e-02 | 20260505 11:25:39.720 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 11:25:39.720 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:25:39.720 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:25:39.720 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.47e+01 / 0.00e+00 | 20260505 11:25:39.720 | INFO | PerfReport: 22.54%, Upload -- 2.28e+01 / 0.00e+00, 8.09e+00 / 0.00e+00 | 20260505 11:25:39.720 | INFO | PerfReport: 21.91%, Download -- 2.22e+01 / 0.00e+00, 8.14e+00 / 0.00e+00 | 20260505 11:25:39.720 | INFO | PerfReport: 31.78%, Compute -- 3.21e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 11:25:39.720 | INFO | PerfReport: 2.04%, LockUpload -- 2.06e+00 / 0.00e+00, 7.74e+00 / 0.00e+00 | 20260505 11:25:39.720 | INFO | PerfReport: 1.27%, LockDownload -- 1.28e+00 / 0.00e+00, 2.79e+00 / 0.00e+00 | 20260505 11:25:39.720 | INFO | PerfReport: 20.45%, LockCompute -- 2.07e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260505 11:25:39.720 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 2.55e-02 / 0.00e+00 | 20260505 11:25:39.857 | INFO | Starting a round of Baz IO | 20260505 11:25:43.122 | INFO | Populating RTMetrics report | 20260505 11:25:43.123 | INFO | RTMetrics report populated | 20260505 11:25:43.123 | INFO | Opening tmp RTMetrics json file | 20260505 11:25:43.139 | INFO | Serializing RTMetrics to json | 20260505 11:25:43.188 | INFO | Serialization done, swapping tmp file now... | 20260505 11:25:43.191 | INFO | RTMetrics done | 20260505 11:25:43.253 | INFO | Finished a round of Baz IO: Wrote 2.38 GiB in 3.396 seconds | 20260505 11:25:43.253 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.19% / 0.01% / 75.80% | 20260505 11:25:43.253 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.15736% / 0% / 7.94729e-05% | 20260505 11:25:43.253 | INFO | Baz IO overall: | 20260505 11:25:43.253 | INFO | BytesWritten : 1675447864424 | 20260505 11:25:43.253 | INFO | NumEvents : 797734696142 | 20260505 11:25:44.861 | INFO | Analyzing chunk frames = [2834432,2834944) | 20260505 11:25:44.861 | INFO | MemInfo: { | 20260505 11:25:44.861 | INFO | "gpu" : " 17 GB", | 20260505 11:25:44.861 | INFO | "huge" : "", | 20260505 11:25:44.861 | INFO | "huge_shared" : " 27 GB", | 20260505 11:25:44.861 | INFO | "malloc" : " 14 GB", | 20260505 11:25:44.861 | INFO | "pinned" : " 29 GB" | 20260505 11:25:44.861 | INFO | } | 20260505 11:25:44.861 | INFO | Perf report for the last 5.14274 seconds: | 20260505 11:25:44.861 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:25:44.861 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:44.861 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:44.861 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:44.861 | INFO | ANALYSIS : 65.1%, 2.9, 100.4ms, 0 | 20260505 11:25:44.861 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:44.861 | INFO | RT_METRICS : 46.6%, 1.0, 24.5ms, 0 | 20260505 11:25:44.861 | INFO | PRE_HQ : 61.5%, 1.8, 58.6ms, 0 | 20260505 11:25:44.861 | INFO | BAZWRITER : 65.4%, 4.1, 137.9ms, 0 | 20260505 11:25:44.861 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 11:25:44.861 | INFO | Uploaded 36GiB to GPU in 2.22003s (16.216GiB/s) | 20260505 11:25:44.861 | INFO | Downloaded 38GiB to GPU in 2.29956s (16.5249GiB/s) | 20260505 11:25:50.001 | INFO | Analyzing chunk frames = [2834944,2835456) | 20260505 11:25:50.001 | INFO | MemInfo: { | 20260505 11:25:50.001 | INFO | "gpu" : " 17 GB", | 20260505 11:25:50.001 | INFO | "huge" : "", | 20260505 11:25:50.001 | INFO | "huge_shared" : " 27 GB", | 20260505 11:25:50.001 | INFO | "malloc" : " 14 GB", | 20260505 11:25:50.001 | INFO | "pinned" : " 29 GB" | 20260505 11:25:50.002 | INFO | } | 20260505 11:25:50.002 | INFO | Perf report for the last 5.14097 seconds: | 20260505 11:25:50.002 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:25:50.002 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:50.002 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:50.002 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:50.002 | INFO | ANALYSIS : 64.1%, 3.0, 100.2ms, 0 | 20260505 11:25:50.002 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:50.002 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:50.002 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 11:25:50.002 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:50.002 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:50.002 | INFO | Uploaded 36GiB to GPU in 2.2321s (16.1283GiB/s) | 20260505 11:25:50.002 | INFO | Downloaded 34GiB to GPU in 2.10218s (16.1737GiB/s) | 20260505 11:25:55.150 | INFO | Analyzing chunk frames = [2835456,2835968) | 20260505 11:25:55.150 | INFO | MemInfo: { | 20260505 11:25:55.150 | INFO | "gpu" : " 17 GB", | 20260505 11:25:55.150 | INFO | "huge" : "", | 20260505 11:25:55.150 | INFO | "huge_shared" : " 27 GB", | 20260505 11:25:55.150 | INFO | "malloc" : " 14 GB", | 20260505 11:25:55.150 | INFO | "pinned" : " 29 GB" | 20260505 11:25:55.150 | INFO | } | 20260505 11:25:55.150 | INFO | Perf report for the last 5.14826 seconds: | 20260505 11:25:55.150 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:25:55.150 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:55.150 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:55.150 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:55.150 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 11:25:55.150 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:55.150 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:55.150 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 11:25:55.150 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:55.150 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:25:55.150 | INFO | Uploaded 36GiB to GPU in 2.2052s (16.325GiB/s) | 20260505 11:25:55.150 | INFO | Downloaded 34GiB to GPU in 2.11345s (16.0875GiB/s) | 20260505 11:26:00.282 | INFO | Analyzing chunk frames = [2835968,2836480) | 20260505 11:26:00.282 | INFO | MemInfo: { | 20260505 11:26:00.282 | INFO | "gpu" : " 17 GB", | 20260505 11:26:00.282 | INFO | "huge" : "", | 20260505 11:26:00.282 | INFO | "huge_shared" : " 27 GB", | 20260505 11:26:00.282 | INFO | "malloc" : " 14 GB", | 20260505 11:26:00.282 | INFO | "pinned" : " 29 GB" | 20260505 11:26:00.282 | INFO | } | 20260505 11:26:00.282 | INFO | Perf report for the last 5.13249 seconds: | 20260505 11:26:00.283 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:26:00.283 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:00.283 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:00.283 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:00.283 | INFO | ANALYSIS : 63.5%, 3.0, 98.5ms, 0 | 20260505 11:26:00.283 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:00.283 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:00.283 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 11:26:00.283 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:00.283 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:00.283 | INFO | Uploaded 36GiB to GPU in 2.21044s (16.2863GiB/s) | 20260505 11:26:00.283 | INFO | Downloaded 34GiB to GPU in 2.13222s (15.9459GiB/s) | 20260505 11:26:05.414 | INFO | Analyzing chunk frames = [2836480,2836992) | 20260505 11:26:05.414 | INFO | MemInfo: { | 20260505 11:26:05.414 | INFO | "gpu" : " 17 GB", | 20260505 11:26:05.414 | INFO | "huge" : "", | 20260505 11:26:05.414 | INFO | "huge_shared" : " 27 GB", | 20260505 11:26:05.414 | INFO | "malloc" : " 14 GB", | 20260505 11:26:05.414 | INFO | "pinned" : " 29 GB" | 20260505 11:26:05.414 | INFO | } | 20260505 11:26:05.414 | INFO | Perf report for the last 5.13159 seconds: | 20260505 11:26:05.414 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:26:05.414 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:05.414 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:05.414 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:05.414 | INFO | ANALYSIS : 63.4%, 3.0, 98.8ms, 0 | 20260505 11:26:05.414 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:05.414 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:05.414 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 11:26:05.414 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:05.414 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:05.415 | INFO | Uploaded 36GiB to GPU in 2.15984s (16.6679GiB/s) | 20260505 11:26:05.415 | INFO | Downloaded 34GiB to GPU in 2.12559s (15.9955GiB/s) | 20260505 11:26:10.562 | INFO | Analyzing chunk frames = [2836992,2837504) | 20260505 11:26:10.562 | INFO | MemInfo: { | 20260505 11:26:10.562 | INFO | "gpu" : " 17 GB", | 20260505 11:26:10.562 | INFO | "huge" : "", | 20260505 11:26:10.562 | INFO | "huge_shared" : " 27 GB", | 20260505 11:26:10.562 | INFO | "malloc" : " 14 GB", | 20260505 11:26:10.562 | INFO | "pinned" : " 29 GB" | 20260505 11:26:10.562 | INFO | } | 20260505 11:26:10.562 | INFO | Perf report for the last 5.14832 seconds: | 20260505 11:26:10.562 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:26:10.562 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:10.562 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:10.562 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:10.562 | INFO | ANALYSIS : 63.4%, 3.0, 99.2ms, 0 | 20260505 11:26:10.563 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:10.563 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:10.563 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 11:26:10.563 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:10.563 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:10.563 | INFO | Uploaded 36GiB to GPU in 2.19914s (16.3701GiB/s) | 20260505 11:26:10.563 | INFO | Downloaded 34GiB to GPU in 2.11999s (16.0378GiB/s) | 20260505 11:26:15.698 | INFO | Analyzing chunk frames = [2837504,2838016) | 20260505 11:26:15.698 | INFO | MemInfo: { | 20260505 11:26:15.698 | INFO | "gpu" : " 17 GB", | 20260505 11:26:15.698 | INFO | "huge" : "", | 20260505 11:26:15.698 | INFO | "huge_shared" : " 27 GB", | 20260505 11:26:15.698 | INFO | "malloc" : " 14 GB", | 20260505 11:26:15.698 | INFO | "pinned" : " 29 GB" | 20260505 11:26:15.698 | INFO | } | 20260505 11:26:15.698 | INFO | Perf report for the last 5.13564 seconds: | 20260505 11:26:15.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:26:15.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:15.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:15.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:15.698 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 11:26:15.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:15.698 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:15.698 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 11:26:15.698 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:15.698 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:15.698 | INFO | Uploaded 36GiB to GPU in 2.15377s (16.7149GiB/s) | 20260505 11:26:15.698 | INFO | Downloaded 34GiB to GPU in 2.12576s (15.9943GiB/s) | 20260505 11:26:20.838 | INFO | Analyzing chunk frames = [2838016,2838528) | 20260505 11:26:20.838 | INFO | MemInfo: { | 20260505 11:26:20.838 | INFO | "gpu" : " 17 GB", | 20260505 11:26:20.838 | INFO | "huge" : "", | 20260505 11:26:20.838 | INFO | "huge_shared" : " 27 GB", | 20260505 11:26:20.838 | INFO | "malloc" : " 14 GB", | 20260505 11:26:20.838 | INFO | "pinned" : " 29 GB" | 20260505 11:26:20.838 | INFO | } | 20260505 11:26:20.838 | INFO | Perf report for the last 5.14 seconds: | 20260505 11:26:20.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:26:20.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:20.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:20.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:20.838 | INFO | ANALYSIS : 63.9%, 3.0, 99.9ms, 0 | 20260505 11:26:20.838 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:20.838 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:20.838 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 11:26:20.838 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:20.838 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:20.838 | INFO | Uploaded 36GiB to GPU in 2.2224s (16.1987GiB/s) | 20260505 11:26:20.838 | INFO | Downloaded 34GiB to GPU in 2.09579s (16.223GiB/s) | 20260505 11:26:20.977 | INFO | Starting a round of Baz IO | 20260505 11:26:24.238 | INFO | Populating RTMetrics report | 20260505 11:26:24.239 | INFO | RTMetrics report populated | 20260505 11:26:24.239 | INFO | Opening tmp RTMetrics json file | 20260505 11:26:24.240 | INFO | Serializing RTMetrics to json | 20260505 11:26:24.249 | INFO | Serialization done, swapping tmp file now... | 20260505 11:26:24.266 | INFO | RTMetrics done | 20260505 11:26:24.383 | INFO | Finished a round of Baz IO: Wrote 2.38 GiB in 3.405 seconds | 20260505 11:26:24.383 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.19% / 0.01% / 75.81% | 20260505 11:26:24.383 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.15306% / 1.19209e-05% / 7.54992e-05% | 20260505 11:26:24.383 | INFO | Baz IO overall: | 20260505 11:26:24.383 | INFO | BytesWritten : 1678003557758 | 20260505 11:26:24.383 | INFO | NumEvents : 798950945213 | 20260505 11:26:25.972 | INFO | Analyzing chunk frames = [2838528,2839040) | 20260505 11:26:25.972 | INFO | MemInfo: { | 20260505 11:26:25.972 | INFO | "gpu" : " 17 GB", | 20260505 11:26:25.972 | INFO | "huge" : "", | 20260505 11:26:25.972 | INFO | "huge_shared" : " 27 GB", | 20260505 11:26:25.972 | INFO | "malloc" : " 14 GB", | 20260505 11:26:25.972 | INFO | "pinned" : " 29 GB" | 20260505 11:26:25.972 | INFO | } | 20260505 11:26:25.972 | INFO | Perf report for the last 5.13429 seconds: | 20260505 11:26:25.973 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:26:25.973 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:25.973 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:25.973 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:25.973 | INFO | ANALYSIS : 65.1%, 2.9, 100.2ms, 0 | 20260505 11:26:25.973 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:25.973 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260505 11:26:25.973 | INFO | PRE_HQ : 61.5%, 1.8, 58.5ms, 0 | 20260505 11:26:25.973 | INFO | BAZWRITER : 65.7%, 4.1, 138.7ms, 0 | 20260505 11:26:25.973 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 11:26:25.973 | INFO | Uploaded 36GiB to GPU in 2.19551s (16.3971GiB/s) | 20260505 11:26:25.973 | INFO | Downloaded 38GiB to GPU in 2.30681s (16.4729GiB/s) | 20260505 11:26:31.118 | INFO | Analyzing chunk frames = [2839040,2839552) | 20260505 11:26:31.118 | INFO | MemInfo: { | 20260505 11:26:31.118 | INFO | "gpu" : " 17 GB", | 20260505 11:26:31.118 | INFO | "huge" : "", | 20260505 11:26:31.118 | INFO | "huge_shared" : " 27 GB", | 20260505 11:26:31.118 | INFO | "malloc" : " 14 GB", | 20260505 11:26:31.118 | INFO | "pinned" : " 29 GB" | 20260505 11:26:31.118 | INFO | } | 20260505 11:26:31.119 | INFO | Perf report for the last 5.1461 seconds: | 20260505 11:26:31.119 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:26:31.119 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:31.119 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:31.119 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:31.119 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 11:26:31.119 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:31.119 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:31.119 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:26:31.119 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:31.119 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:31.119 | INFO | Uploaded 36GiB to GPU in 2.23021s (16.142GiB/s) | 20260505 11:26:31.119 | INFO | Downloaded 34GiB to GPU in 2.08709s (16.2906GiB/s) | 20260505 11:26:36.261 | INFO | Analyzing chunk frames = [2839552,2840064) | 20260505 11:26:36.261 | INFO | MemInfo: { | 20260505 11:26:36.261 | INFO | "gpu" : " 17 GB", | 20260505 11:26:36.261 | INFO | "huge" : "", | 20260505 11:26:36.261 | INFO | "huge_shared" : " 27 GB", | 20260505 11:26:36.261 | INFO | "malloc" : " 14 GB", | 20260505 11:26:36.261 | INFO | "pinned" : " 29 GB" | 20260505 11:26:36.261 | INFO | } | 20260505 11:26:36.261 | INFO | Perf report for the last 5.14262 seconds: | 20260505 11:26:36.261 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:26:36.261 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:36.261 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:36.261 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:36.261 | INFO | ANALYSIS : 63.2%, 3.0, 98.4ms, 0 | 20260505 11:26:36.261 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:36.261 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:36.261 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:26:36.261 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:36.261 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:36.262 | INFO | Uploaded 36GiB to GPU in 2.17852s (16.525GiB/s) | 20260505 11:26:36.262 | INFO | Downloaded 34GiB to GPU in 2.11183s (16.0997GiB/s) | 20260505 11:26:41.397 | INFO | Analyzing chunk frames = [2840064,2840576) | 20260505 11:26:41.397 | INFO | MemInfo: { | 20260505 11:26:41.397 | INFO | "gpu" : " 17 GB", | 20260505 11:26:41.397 | INFO | "huge" : "", | 20260505 11:26:41.397 | INFO | "huge_shared" : " 27 GB", | 20260505 11:26:41.397 | INFO | "malloc" : " 14 GB", | 20260505 11:26:41.397 | INFO | "pinned" : " 29 GB" | 20260505 11:26:41.397 | INFO | } | 20260505 11:26:41.397 | INFO | Perf report for the last 5.13619 seconds: | 20260505 11:26:41.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:26:41.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:41.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:41.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:41.397 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 11:26:41.397 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:41.397 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:41.397 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:26:41.397 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:41.397 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:41.398 | INFO | Uploaded 36GiB to GPU in 2.20462s (16.3294GiB/s) | 20260505 11:26:41.398 | INFO | Downloaded 34GiB to GPU in 2.12964s (15.9651GiB/s) | 20260505 11:26:46.528 | INFO | Analyzing chunk frames = [2840576,2841088) | 20260505 11:26:46.528 | INFO | MemInfo: { | 20260505 11:26:46.528 | INFO | "gpu" : " 17 GB", | 20260505 11:26:46.528 | INFO | "huge" : "", | 20260505 11:26:46.528 | INFO | "huge_shared" : " 27 GB", | 20260505 11:26:46.528 | INFO | "malloc" : " 14 GB", | 20260505 11:26:46.528 | INFO | "pinned" : " 29 GB" | 20260505 11:26:46.528 | INFO | } | 20260505 11:26:46.528 | INFO | Perf report for the last 5.13084 seconds: | 20260505 11:26:46.528 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:26:46.528 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:46.528 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:46.528 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:46.528 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 11:26:46.528 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:46.528 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:46.528 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:26:46.528 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:46.528 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:46.528 | INFO | Uploaded 36GiB to GPU in 2.14874s (16.754GiB/s) | 20260505 11:26:46.529 | INFO | Downloaded 34GiB to GPU in 2.12501s (15.9999GiB/s) | 20260505 11:26:51.669 | INFO | Analyzing chunk frames = [2841088,2841600) | 20260505 11:26:51.669 | INFO | MemInfo: { | 20260505 11:26:51.669 | INFO | "gpu" : " 17 GB", | 20260505 11:26:51.669 | INFO | "huge" : "", | 20260505 11:26:51.669 | INFO | "huge_shared" : " 27 GB", | 20260505 11:26:51.669 | INFO | "malloc" : " 14 GB", | 20260505 11:26:51.669 | INFO | "pinned" : " 29 GB" | 20260505 11:26:51.669 | INFO | } | 20260505 11:26:51.669 | INFO | Perf report for the last 5.14124 seconds: | 20260505 11:26:51.670 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:26:51.670 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:51.670 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:51.670 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:51.670 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 11:26:51.670 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:51.670 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:51.670 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:26:51.670 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:51.670 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:51.670 | INFO | Uploaded 36GiB to GPU in 2.22379s (16.1886GiB/s) | 20260505 11:26:51.670 | INFO | Downloaded 34GiB to GPU in 2.10555s (16.1478GiB/s) | 20260505 11:26:56.816 | INFO | Analyzing chunk frames = [2841600,2842112) | 20260505 11:26:56.816 | INFO | MemInfo: { | 20260505 11:26:56.816 | INFO | "gpu" : " 17 GB", | 20260505 11:26:56.816 | INFO | "huge" : "", | 20260505 11:26:56.816 | INFO | "huge_shared" : " 27 GB", | 20260505 11:26:56.816 | INFO | "malloc" : " 14 GB", | 20260505 11:26:56.816 | INFO | "pinned" : " 29 GB" | 20260505 11:26:56.816 | INFO | } | 20260505 11:26:56.816 | INFO | Perf report for the last 5.14681 seconds: | 20260505 11:26:56.816 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:26:56.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:56.816 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:56.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:56.816 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 11:26:56.817 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:56.817 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:56.817 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 11:26:56.817 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:56.817 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:26:56.817 | INFO | Uploaded 36GiB to GPU in 2.17185s (16.5758GiB/s) | 20260505 11:26:56.817 | INFO | Downloaded 34GiB to GPU in 2.15968s (15.7431GiB/s) | 20260505 11:27:01.957 | INFO | Analyzing chunk frames = [2842112,2842624) | 20260505 11:27:01.957 | INFO | MemInfo: { | 20260505 11:27:01.957 | INFO | "gpu" : " 17 GB", | 20260505 11:27:01.957 | INFO | "huge" : "", | 20260505 11:27:01.957 | INFO | "huge_shared" : " 27 GB", | 20260505 11:27:01.957 | INFO | "malloc" : " 14 GB", | 20260505 11:27:01.957 | INFO | "pinned" : " 29 GB" | 20260505 11:27:01.957 | INFO | } | 20260505 11:27:01.957 | INFO | Perf report for the last 5.14098 seconds: | 20260505 11:27:01.957 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:27:01.958 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:01.958 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:01.958 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:01.958 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 11:27:01.958 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:01.958 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:01.958 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 11:27:01.958 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:01.958 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:01.958 | INFO | Uploaded 36GiB to GPU in 2.17705s (16.5362GiB/s) | 20260505 11:27:01.958 | INFO | Downloaded 34GiB to GPU in 2.11534s (16.0731GiB/s) | 20260505 11:27:01.958 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 11:27:01.958 | INFO | Total mem usage: 15.3 GiB | 20260505 11:27:01.958 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 11:27:01.958 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -6.81%) | 20260505 11:27:01.958 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 11:27:01.958 | INFO | High Water Breakdown: | 20260505 11:27:01.958 | INFO | 61% : 8.76 GiB ( 92.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 11:27:01.958 | INFO | 38.8% : 5.57 GiB ( 83.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 11:27:01.958 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 11:27:01.958 | INFO | ------------------------------------------ | 20260505 11:27:01.958 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:27:01.958 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 11:27:01.958 | INFO | Total mem usage: 26.5 GiB | 20260505 11:27:01.958 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 11:27:01.958 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 11:27:01.958 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 11:27:01.958 | INFO | High Water Breakdown: | 20260505 11:27:01.958 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:27:01.958 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:27:01.958 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:27:01.958 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:27:01.958 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:27:01.958 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:27:01.958 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 11:27:01.958 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 11:27:01.958 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:27:01.958 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 11:27:01.958 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 11:27:01.958 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 11:27:01.958 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 11:27:01.958 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 11:27:01.958 | INFO | ------------------------------------------ | 20260505 11:27:01.958 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:27:01.958 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 11:27:01.958 | INFO | Total mem usage: 25.6 GiB | 20260505 11:27:01.958 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 11:27:01.958 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 11:27:01.958 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 11:27:01.958 | INFO | High Water Breakdown: | 20260505 11:27:01.958 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 11:27:01.958 | INFO | ------------------------------------------ | 20260505 11:27:01.958 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:27:01.958 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 11:27:01.958 | INFO | Total mem usage: 15.7 GiB | 20260505 11:27:01.958 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 11:27:01.958 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 11:27:01.958 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 11:27:01.958 | INFO | High Water Breakdown: | 20260505 11:27:01.958 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 11:27:01.958 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 11:27:01.958 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 11:27:01.958 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 11:27:01.959 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 11:27:01.959 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 11:27:01.959 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 11:27:01.959 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 11:27:01.959 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 11:27:01.959 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:27:01.959 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 11:27:01.959 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:27:01.959 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 11:27:01.959 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 11:27:01.959 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 11:27:01.959 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 11:27:01.959 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:27:01.959 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 11:27:01.959 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:27:01.959 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:27:01.959 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:27:01.959 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 11:27:01.959 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 11:27:01.959 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:27:01.959 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 11:27:01.959 | INFO | ------------------------------------------ | 20260505 11:27:01.959 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:27:01.959 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 11:27:01.959 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:27:01.959 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:27:01.959 | INFO | PerfReport: Full Execution -- 3.22e+01 / 7.76e+01, 1.40e+01 / 2.01e+00 | 20260505 11:27:01.959 | INFO | PerfReport: 0.29%, AggregateStats -- 9.30e-02 / 1.01e-01, 1.08e-02 / 4.41e-03 | 20260505 11:27:01.959 | INFO | PerfReport: 11.72%, Baseline -- 3.77e+00 / 4.48e+00, 6.07e-01 / 2.21e-01 | 20260505 11:27:01.959 | INFO | PerfReport: 3.31%, Histogram -- 1.06e+00 / 1.49e+00, 2.06e-01 / 1.04e-01 | 20260505 11:27:01.959 | INFO | PerfReport: 15.90%, DME -- 4.09e+01 / 4.55e+01, 4.39e+00 / 2.09e+00 | 20260505 11:27:01.959 | INFO | PerfReport: 56.34%, FrameLabeling -- 1.81e+01 / 2.12e+01, 1.57e+00 / 1.78e+00 | 20260505 11:27:01.959 | INFO | PerfReport: 7.44%, PulseAccumulating -- 2.39e+00 / 2.89e+00, 4.39e-01 / 2.15e-01 | 20260505 11:27:01.959 | INFO | PerfReport: 3.70%, Metrics -- 1.19e+00 / 1.47e+00, 3.38e-01 / 1.62e-01 | 20260505 11:27:01.959 | INFO | PerfReport: 1.30%, PulseSerializing -- 4.19e-01 / 4.73e-01, 7.69e-02 / 3.48e-02 | 20260505 11:27:01.959 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 11:27:01.959 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:27:01.959 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:27:01.959 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260505 11:27:01.959 | INFO | PerfReport: 22.58%, Upload -- 2.29e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260505 11:27:01.959 | INFO | PerfReport: 22.02%, Download -- 2.23e+01 / 0.00e+00, 8.30e+00 / 0.00e+00 | 20260505 11:27:01.959 | INFO | PerfReport: 31.74%, Compute -- 3.22e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260505 11:27:01.959 | INFO | PerfReport: 2.08%, LockUpload -- 2.11e+00 / 0.00e+00, 7.68e+00 / 0.00e+00 | 20260505 11:27:01.959 | INFO | PerfReport: 1.27%, LockDownload -- 1.29e+00 / 0.00e+00, 2.85e+00 / 0.00e+00 | 20260505 11:27:01.959 | INFO | PerfReport: 20.29%, LockCompute -- 2.06e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260505 11:27:01.959 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.35e-02 / 0.00e+00 | 20260505 11:27:02.086 | INFO | Starting a round of Baz IO | 20260505 11:27:05.336 | INFO | Populating RTMetrics report | 20260505 11:27:05.337 | INFO | RTMetrics report populated | 20260505 11:27:05.337 | INFO | Opening tmp RTMetrics json file | 20260505 11:27:05.338 | INFO | Serializing RTMetrics to json | 20260505 11:27:05.346 | INFO | Serialization done, swapping tmp file now... | 20260505 11:27:05.364 | INFO | RTMetrics done | 20260505 11:27:05.451 | INFO | Finished a round of Baz IO: Wrote 2.381 GiB in 3.365 seconds | 20260505 11:27:05.451 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.18% / 0.01% / 75.81% | 20260505 11:27:05.451 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.15173% / 3.97364e-06% / 5.96046e-05% | 20260505 11:27:05.451 | INFO | Baz IO overall: | 20260505 11:27:05.451 | INFO | BytesWritten : 1680560365367 | 20260505 11:27:05.451 | INFO | NumEvents : 800167893391 | 20260505 11:27:07.096 | INFO | Analyzing chunk frames = [2842624,2843136) | 20260505 11:27:07.096 | INFO | MemInfo: { | 20260505 11:27:07.096 | INFO | "gpu" : " 17 GB", | 20260505 11:27:07.096 | INFO | "huge" : "", | 20260505 11:27:07.096 | INFO | "huge_shared" : " 27 GB", | 20260505 11:27:07.096 | INFO | "malloc" : " 14 GB", | 20260505 11:27:07.096 | INFO | "pinned" : " 29 GB" | 20260505 11:27:07.096 | INFO | } | 20260505 11:27:07.096 | INFO | Perf report for the last 5.13888 seconds: | 20260505 11:27:07.097 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:27:07.097 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:07.097 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:07.097 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:07.097 | INFO | ANALYSIS : 64.6%, 2.9, 99.3ms, 0 | 20260505 11:27:07.097 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:07.097 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260505 11:27:07.097 | INFO | PRE_HQ : 61.1%, 1.8, 58.6ms, 0 | 20260505 11:27:07.097 | INFO | BAZWRITER : 64.8%, 4.2, 138.6ms, 0 | 20260505 11:27:07.097 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260505 11:27:07.097 | INFO | Uploaded 36GiB to GPU in 2.17326s (16.565GiB/s) | 20260505 11:27:07.097 | INFO | Downloaded 38GiB to GPU in 2.27932s (16.6716GiB/s) | 20260505 11:27:12.230 | INFO | Analyzing chunk frames = [2843136,2843648) | 20260505 11:27:12.230 | INFO | MemInfo: { | 20260505 11:27:12.230 | INFO | "gpu" : " 17 GB", | 20260505 11:27:12.230 | INFO | "huge" : "", | 20260505 11:27:12.230 | INFO | "huge_shared" : " 27 GB", | 20260505 11:27:12.230 | INFO | "malloc" : " 14 GB", | 20260505 11:27:12.230 | INFO | "pinned" : " 29 GB" | 20260505 11:27:12.230 | INFO | } | 20260505 11:27:12.230 | INFO | Perf report for the last 5.1339 seconds: | 20260505 11:27:12.230 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:27:12.230 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:12.230 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:12.230 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:12.230 | INFO | ANALYSIS : 64.3%, 3.0, 100.2ms, 0 | 20260505 11:27:12.230 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:12.230 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:12.230 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 11:27:12.230 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:12.230 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:12.230 | INFO | Uploaded 36GiB to GPU in 2.23237s (16.1264GiB/s) | 20260505 11:27:12.231 | INFO | Downloaded 34GiB to GPU in 2.08332s (16.3201GiB/s) | 20260505 11:27:17.373 | INFO | Analyzing chunk frames = [2843648,2844160) | 20260505 11:27:17.373 | INFO | MemInfo: { | 20260505 11:27:17.373 | INFO | "gpu" : " 17 GB", | 20260505 11:27:17.373 | INFO | "huge" : "", | 20260505 11:27:17.373 | INFO | "huge_shared" : " 27 GB", | 20260505 11:27:17.373 | INFO | "malloc" : " 14 GB", | 20260505 11:27:17.373 | INFO | "pinned" : " 29 GB" | 20260505 11:27:17.373 | INFO | } | 20260505 11:27:17.373 | INFO | Perf report for the last 5.1427 seconds: | 20260505 11:27:17.373 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:27:17.373 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:17.373 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:17.373 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:17.373 | INFO | ANALYSIS : 62.8%, 3.0, 97.8ms, 0 | 20260505 11:27:17.373 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:17.373 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:17.373 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:27:17.373 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:17.373 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:17.373 | INFO | Uploaded 36GiB to GPU in 2.16897s (16.5978GiB/s) | 20260505 11:27:17.373 | INFO | Downloaded 34GiB to GPU in 2.06712s (16.448GiB/s) | 20260505 11:27:22.510 | INFO | Analyzing chunk frames = [2844160,2844672) | 20260505 11:27:22.510 | INFO | MemInfo: { | 20260505 11:27:22.510 | INFO | "gpu" : " 17 GB", | 20260505 11:27:22.510 | INFO | "huge" : "", | 20260505 11:27:22.510 | INFO | "huge_shared" : " 27 GB", | 20260505 11:27:22.510 | INFO | "malloc" : " 14 GB", | 20260505 11:27:22.510 | INFO | "pinned" : " 29 GB" | 20260505 11:27:22.510 | INFO | } | 20260505 11:27:22.510 | INFO | Perf report for the last 5.13695 seconds: | 20260505 11:27:22.510 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:27:22.510 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:22.510 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:22.510 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:22.510 | INFO | ANALYSIS : 63.5%, 3.0, 98.6ms, 0 | 20260505 11:27:22.510 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:22.510 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:22.510 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 11:27:22.510 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:22.510 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:22.510 | INFO | Uploaded 36GiB to GPU in 2.21389s (16.261GiB/s) | 20260505 11:27:22.510 | INFO | Downloaded 34GiB to GPU in 2.13442s (15.9294GiB/s) | 20260505 11:27:27.654 | INFO | Analyzing chunk frames = [2844672,2845184) | 20260505 11:27:27.654 | INFO | MemInfo: { | 20260505 11:27:27.654 | INFO | "gpu" : " 17 GB", | 20260505 11:27:27.654 | INFO | "huge" : "", | 20260505 11:27:27.654 | INFO | "huge_shared" : " 27 GB", | 20260505 11:27:27.654 | INFO | "malloc" : " 14 GB", | 20260505 11:27:27.654 | INFO | "pinned" : " 29 GB" | 20260505 11:27:27.654 | INFO | } | 20260505 11:27:27.654 | INFO | Perf report for the last 5.14448 seconds: | 20260505 11:27:27.654 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:27:27.654 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:27.654 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:27.654 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:27.654 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 11:27:27.654 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:27.654 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:27.654 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 11:27:27.654 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:27.654 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:27.654 | INFO | Uploaded 36GiB to GPU in 2.15646s (16.694GiB/s) | 20260505 11:27:27.655 | INFO | Downloaded 34GiB to GPU in 2.12279s (16.0167GiB/s) | 20260505 11:27:32.786 | INFO | Analyzing chunk frames = [2845184,2845696) | 20260505 11:27:32.786 | INFO | MemInfo: { | 20260505 11:27:32.786 | INFO | "gpu" : " 17 GB", | 20260505 11:27:32.786 | INFO | "huge" : "", | 20260505 11:27:32.786 | INFO | "huge_shared" : " 27 GB", | 20260505 11:27:32.786 | INFO | "malloc" : " 14 GB", | 20260505 11:27:32.786 | INFO | "pinned" : " 29 GB" | 20260505 11:27:32.786 | INFO | } | 20260505 11:27:32.786 | INFO | Perf report for the last 5.13199 seconds: | 20260505 11:27:32.786 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:27:32.786 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:32.786 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:32.786 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:32.786 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 11:27:32.786 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:32.786 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:32.786 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 11:27:32.786 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:32.786 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:32.786 | INFO | Uploaded 36GiB to GPU in 2.17364s (16.5621GiB/s) | 20260505 11:27:32.786 | INFO | Downloaded 34GiB to GPU in 2.12287s (16.016GiB/s) | 20260505 11:27:37.924 | INFO | Analyzing chunk frames = [2845696,2846208) | 20260505 11:27:37.924 | INFO | MemInfo: { | 20260505 11:27:37.924 | INFO | "gpu" : " 17 GB", | 20260505 11:27:37.924 | INFO | "huge" : "", | 20260505 11:27:37.924 | INFO | "huge_shared" : " 27 GB", | 20260505 11:27:37.924 | INFO | "malloc" : " 14 GB", | 20260505 11:27:37.924 | INFO | "pinned" : " 29 GB" | 20260505 11:27:37.924 | INFO | } | 20260505 11:27:37.924 | INFO | Perf report for the last 5.138 seconds: | 20260505 11:27:37.924 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:27:37.924 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:37.924 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:37.924 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:37.924 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 11:27:37.924 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:37.924 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:37.924 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 11:27:37.924 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:37.924 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:37.925 | INFO | Uploaded 36GiB to GPU in 2.1807s (16.5085GiB/s) | 20260505 11:27:37.925 | INFO | Downloaded 34GiB to GPU in 2.15951s (15.7443GiB/s) | 20260505 11:27:43.070 | INFO | Analyzing chunk frames = [2846208,2846720) | 20260505 11:27:43.070 | INFO | MemInfo: { | 20260505 11:27:43.070 | INFO | "gpu" : " 17 GB", | 20260505 11:27:43.070 | INFO | "huge" : "", | 20260505 11:27:43.070 | INFO | "huge_shared" : " 27 GB", | 20260505 11:27:43.070 | INFO | "malloc" : " 14 GB", | 20260505 11:27:43.070 | INFO | "pinned" : " 29 GB" | 20260505 11:27:43.070 | INFO | } | 20260505 11:27:43.070 | INFO | Perf report for the last 5.14611 seconds: | 20260505 11:27:43.070 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:27:43.070 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:43.070 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:43.071 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:43.071 | INFO | ANALYSIS : 63.9%, 3.0, 99.9ms, 0 | 20260505 11:27:43.071 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:43.071 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:43.071 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 11:27:43.071 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:43.071 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:43.071 | INFO | Uploaded 36GiB to GPU in 2.21996s (16.2165GiB/s) | 20260505 11:27:43.071 | INFO | Downloaded 34GiB to GPU in 2.07897s (16.3543GiB/s) | 20260505 11:27:43.200 | INFO | Starting a round of Baz IO | 20260505 11:27:46.453 | INFO | Populating RTMetrics report | 20260505 11:27:46.453 | INFO | RTMetrics report populated | 20260505 11:27:46.453 | INFO | Opening tmp RTMetrics json file | 20260505 11:27:46.462 | INFO | Serializing RTMetrics to json | 20260505 11:27:46.463 | INFO | Serialization done, swapping tmp file now... | 20260505 11:27:46.464 | INFO | RTMetrics done | 20260505 11:27:46.595 | INFO | Finished a round of Baz IO: Wrote 2.382 GiB in 3.395 seconds | 20260505 11:27:46.595 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.17% / 0.01% / 75.82% | 20260505 11:27:46.595 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.14898% / 1.98682e-05% / 9.13938e-05% | 20260505 11:27:46.595 | INFO | Baz IO overall: | 20260505 11:27:46.595 | INFO | BytesWritten : 1683118105371 | 20260505 11:27:46.595 | INFO | NumEvents : 801385749431 | 20260505 11:27:48.215 | INFO | Analyzing chunk frames = [2846720,2847232) | 20260505 11:27:48.215 | INFO | MemInfo: { | 20260505 11:27:48.215 | INFO | "gpu" : " 17 GB", | 20260505 11:27:48.215 | INFO | "huge" : "", | 20260505 11:27:48.215 | INFO | "huge_shared" : " 27 GB", | 20260505 11:27:48.215 | INFO | "malloc" : " 14 GB", | 20260505 11:27:48.215 | INFO | "pinned" : " 29 GB" | 20260505 11:27:48.215 | INFO | } | 20260505 11:27:48.215 | INFO | Perf report for the last 5.14477 seconds: | 20260505 11:27:48.215 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:27:48.215 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:48.215 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:48.215 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:48.215 | INFO | ANALYSIS : 64.6%, 2.9, 99.8ms, 0 | 20260505 11:27:48.215 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:48.215 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260505 11:27:48.215 | INFO | PRE_HQ : 61.1%, 1.9, 59.0ms, 0 | 20260505 11:27:48.215 | INFO | BAZWRITER : 65.3%, 4.1, 138.6ms, 0 | 20260505 11:27:48.215 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260505 11:27:48.215 | INFO | Uploaded 36GiB to GPU in 2.19126s (16.4289GiB/s) | 20260505 11:27:48.215 | INFO | Downloaded 38GiB to GPU in 2.29063s (16.5893GiB/s) | 20260505 11:27:53.344 | INFO | Analyzing chunk frames = [2847232,2847744) | 20260505 11:27:53.344 | INFO | MemInfo: { | 20260505 11:27:53.344 | INFO | "gpu" : " 17 GB", | 20260505 11:27:53.344 | INFO | "huge" : "", | 20260505 11:27:53.344 | INFO | "huge_shared" : " 27 GB", | 20260505 11:27:53.344 | INFO | "malloc" : " 14 GB", | 20260505 11:27:53.344 | INFO | "pinned" : " 29 GB" | 20260505 11:27:53.344 | INFO | } | 20260505 11:27:53.344 | INFO | Perf report for the last 5.12887 seconds: | 20260505 11:27:53.344 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:27:53.344 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:53.344 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:53.344 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:53.344 | INFO | ANALYSIS : 64.3%, 3.0, 100.1ms, 0 | 20260505 11:27:53.344 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:53.344 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:53.344 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 11:27:53.344 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:53.344 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:53.344 | INFO | Uploaded 36GiB to GPU in 2.22867s (16.1531GiB/s) | 20260505 11:27:53.344 | INFO | Downloaded 34GiB to GPU in 2.10245s (16.1716GiB/s) | 20260505 11:27:58.478 | INFO | Analyzing chunk frames = [2847744,2848256) | 20260505 11:27:58.478 | INFO | MemInfo: { | 20260505 11:27:58.478 | INFO | "gpu" : " 17 GB", | 20260505 11:27:58.478 | INFO | "huge" : "", | 20260505 11:27:58.478 | INFO | "huge_shared" : " 27 GB", | 20260505 11:27:58.478 | INFO | "malloc" : " 14 GB", | 20260505 11:27:58.478 | INFO | "pinned" : " 29 GB" | 20260505 11:27:58.478 | INFO | } | 20260505 11:27:58.478 | INFO | Perf report for the last 5.13395 seconds: | 20260505 11:27:58.478 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:27:58.478 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:58.478 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:58.478 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:58.478 | INFO | ANALYSIS : 63.1%, 3.0, 98.0ms, 0 | 20260505 11:27:58.478 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:58.478 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:58.478 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 11:27:58.478 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:58.478 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:27:58.478 | INFO | Uploaded 36GiB to GPU in 2.15743s (16.6865GiB/s) | 20260505 11:27:58.478 | INFO | Downloaded 34GiB to GPU in 2.0633s (16.4785GiB/s) | 20260505 11:28:03.625 | INFO | Analyzing chunk frames = [2848256,2848768) | 20260505 11:28:03.625 | INFO | MemInfo: { | 20260505 11:28:03.625 | INFO | "gpu" : " 17 GB", | 20260505 11:28:03.625 | INFO | "huge" : "", | 20260505 11:28:03.625 | INFO | "huge_shared" : " 27 GB", | 20260505 11:28:03.625 | INFO | "malloc" : " 14 GB", | 20260505 11:28:03.625 | INFO | "pinned" : " 29 GB" | 20260505 11:28:03.626 | INFO | } | 20260505 11:28:03.626 | INFO | Perf report for the last 5.14753 seconds: | 20260505 11:28:03.626 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:28:03.626 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:03.626 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:03.626 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:03.626 | INFO | ANALYSIS : 62.9%, 3.0, 97.9ms, 0 | 20260505 11:28:03.626 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:03.626 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:03.626 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 11:28:03.626 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:03.626 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:03.626 | INFO | Uploaded 36GiB to GPU in 2.1688s (16.599GiB/s) | 20260505 11:28:03.626 | INFO | Downloaded 34GiB to GPU in 2.07557s (16.381GiB/s) | 20260505 11:28:08.768 | INFO | Analyzing chunk frames = [2848768,2849280) | 20260505 11:28:08.768 | INFO | MemInfo: { | 20260505 11:28:08.768 | INFO | "gpu" : " 17 GB", | 20260505 11:28:08.768 | INFO | "huge" : "", | 20260505 11:28:08.768 | INFO | "huge_shared" : " 27 GB", | 20260505 11:28:08.768 | INFO | "malloc" : " 14 GB", | 20260505 11:28:08.768 | INFO | "pinned" : " 29 GB" | 20260505 11:28:08.768 | INFO | } | 20260505 11:28:08.768 | INFO | Perf report for the last 5.14237 seconds: | 20260505 11:28:08.768 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:28:08.768 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:08.768 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:08.768 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:08.768 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 11:28:08.768 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:08.768 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:08.768 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 11:28:08.768 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:08.768 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:08.768 | INFO | Uploaded 36GiB to GPU in 2.15636s (16.6948GiB/s) | 20260505 11:28:08.768 | INFO | Downloaded 34GiB to GPU in 2.12235s (16.02GiB/s) | 20260505 11:28:13.911 | INFO | Analyzing chunk frames = [2849280,2849792) | 20260505 11:28:13.911 | INFO | MemInfo: { | 20260505 11:28:13.911 | INFO | "gpu" : " 17 GB", | 20260505 11:28:13.911 | INFO | "huge" : "", | 20260505 11:28:13.911 | INFO | "huge_shared" : " 27 GB", | 20260505 11:28:13.911 | INFO | "malloc" : " 14 GB", | 20260505 11:28:13.911 | INFO | "pinned" : " 29 GB" | 20260505 11:28:13.911 | INFO | } | 20260505 11:28:13.911 | INFO | Perf report for the last 5.14273 seconds: | 20260505 11:28:13.911 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:28:13.911 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:13.911 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:13.911 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:13.911 | INFO | ANALYSIS : 63.2%, 3.0, 98.8ms, 0 | 20260505 11:28:13.911 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:13.911 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:13.911 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 11:28:13.911 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:13.911 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:13.911 | INFO | Uploaded 36GiB to GPU in 2.14289s (16.7998GiB/s) | 20260505 11:28:13.911 | INFO | Downloaded 34GiB to GPU in 2.10546s (16.1485GiB/s) | 20260505 11:28:19.039 | INFO | Analyzing chunk frames = [2849792,2850304) | 20260505 11:28:19.039 | INFO | MemInfo: { | 20260505 11:28:19.039 | INFO | "gpu" : " 17 GB", | 20260505 11:28:19.039 | INFO | "huge" : "", | 20260505 11:28:19.039 | INFO | "huge_shared" : " 27 GB", | 20260505 11:28:19.039 | INFO | "malloc" : " 14 GB", | 20260505 11:28:19.039 | INFO | "pinned" : " 29 GB" | 20260505 11:28:19.039 | INFO | } | 20260505 11:28:19.039 | INFO | Perf report for the last 5.12823 seconds: | 20260505 11:28:19.039 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:28:19.039 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:19.039 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:19.039 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:19.039 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 11:28:19.039 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:19.039 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:19.039 | INFO | PRE_HQ : 30.2%, 1.0, 15.6ms, 0 | 20260505 11:28:19.039 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:19.039 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:19.039 | INFO | Uploaded 36GiB to GPU in 2.16427s (16.6338GiB/s) | 20260505 11:28:19.039 | INFO | Downloaded 34GiB to GPU in 2.14339s (15.8627GiB/s) | 20260505 11:28:24.179 | INFO | Analyzing chunk frames = [2850304,2850816) | 20260505 11:28:24.179 | INFO | MemInfo: { | 20260505 11:28:24.179 | INFO | "gpu" : " 17 GB", | 20260505 11:28:24.179 | INFO | "huge" : "", | 20260505 11:28:24.179 | INFO | "huge_shared" : " 27 GB", | 20260505 11:28:24.179 | INFO | "malloc" : " 14 GB", | 20260505 11:28:24.179 | INFO | "pinned" : " 29 GB" | 20260505 11:28:24.179 | INFO | } | 20260505 11:28:24.179 | INFO | Perf report for the last 5.13993 seconds: | 20260505 11:28:24.179 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:28:24.179 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:24.179 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:24.179 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:24.179 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 11:28:24.179 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:24.179 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:24.179 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 11:28:24.179 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:24.179 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:24.179 | INFO | Uploaded 36GiB to GPU in 2.24344s (16.0468GiB/s) | 20260505 11:28:24.179 | INFO | Downloaded 34GiB to GPU in 2.10173s (16.1771GiB/s) | 20260505 11:28:24.179 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 11:28:24.179 | INFO | Total mem usage: 15.3 GiB | 20260505 11:28:24.179 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 11:28:24.179 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.34%) | 20260505 11:28:24.179 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 11:28:24.179 | INFO | High Water Breakdown: | 20260505 11:28:24.179 | INFO | 61.2% : 8.82 GiB ( 92.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 11:28:24.179 | INFO | 38.7% : 5.57 GiB ( 83.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 11:28:24.179 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 11:28:24.179 | INFO | ------------------------------------------ | 20260505 11:28:24.179 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:28:24.179 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 11:28:24.179 | INFO | Total mem usage: 26.5 GiB | 20260505 11:28:24.180 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 11:28:24.180 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 11:28:24.180 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 11:28:24.180 | INFO | High Water Breakdown: | 20260505 11:28:24.180 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:28:24.180 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:28:24.180 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:28:24.180 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:28:24.180 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:28:24.180 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:28:24.180 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 11:28:24.180 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 11:28:24.180 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:28:24.180 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 11:28:24.180 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 11:28:24.180 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 11:28:24.180 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 11:28:24.180 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 11:28:24.180 | INFO | ------------------------------------------ | 20260505 11:28:24.180 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:28:24.180 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 11:28:24.180 | INFO | Total mem usage: 25.6 GiB | 20260505 11:28:24.180 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 11:28:24.180 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 11:28:24.180 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 11:28:24.180 | INFO | High Water Breakdown: | 20260505 11:28:24.180 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 11:28:24.180 | INFO | ------------------------------------------ | 20260505 11:28:24.180 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:28:24.180 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 11:28:24.180 | INFO | Total mem usage: 15.7 GiB | 20260505 11:28:24.180 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 11:28:24.180 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 11:28:24.180 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 11:28:24.180 | INFO | High Water Breakdown: | 20260505 11:28:24.180 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 11:28:24.180 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 11:28:24.180 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 11:28:24.180 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 11:28:24.180 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 11:28:24.180 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 11:28:24.180 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 11:28:24.180 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 11:28:24.180 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 11:28:24.180 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:28:24.180 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 11:28:24.180 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:28:24.180 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 11:28:24.180 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 11:28:24.180 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 11:28:24.180 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 11:28:24.180 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:28:24.180 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 11:28:24.180 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:28:24.180 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:28:24.180 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:28:24.180 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 11:28:24.180 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 11:28:24.180 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:28:24.180 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 11:28:24.180 | INFO | ------------------------------------------ | 20260505 11:28:24.180 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:28:24.180 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 11:28:24.180 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:28:24.180 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:28:24.180 | INFO | PerfReport: Full Execution -- 3.22e+01 / 7.68e+01, 1.33e+01 / 1.83e+00 | 20260505 11:28:24.180 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 1.01e-01, 1.03e-02 / 4.45e-03 | 20260505 11:28:24.180 | INFO | PerfReport: 11.74%, Baseline -- 3.78e+00 / 4.51e+00, 6.00e-01 / 1.66e-01 | 20260505 11:28:24.180 | INFO | PerfReport: 3.28%, Histogram -- 1.06e+00 / 1.50e+00, 2.00e-01 / 8.20e-02 | 20260505 11:28:24.180 | INFO | PerfReport: 15.86%, DME -- 4.08e+01 / 4.55e+01, 4.42e+00 / 1.95e+00 | 20260505 11:28:24.180 | INFO | PerfReport: 56.35%, FrameLabeling -- 1.81e+01 / 2.04e+01, 1.59e+00 / 1.97e+00 | 20260505 11:28:24.180 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.40e+00 / 2.90e+00, 4.33e-01 / 2.30e-01 | 20260505 11:28:24.180 | INFO | PerfReport: 3.72%, Metrics -- 1.20e+00 / 1.49e+00, 3.39e-01 / 1.94e-01 | 20260505 11:28:24.180 | INFO | PerfReport: 1.30%, PulseSerializing -- 4.19e-01 / 4.65e-01, 7.63e-02 / 4.15e-02 | 20260505 11:28:24.180 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 11:28:24.180 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:28:24.180 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:28:24.180 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.45e+01 / 0.00e+00 | 20260505 11:28:24.180 | INFO | PerfReport: 22.52%, Upload -- 2.28e+01 / 0.00e+00, 8.13e+00 / 0.00e+00 | 20260505 11:28:24.180 | INFO | PerfReport: 21.93%, Download -- 2.22e+01 / 0.00e+00, 8.10e+00 / 0.00e+00 | 20260505 11:28:24.180 | INFO | PerfReport: 31.80%, Compute -- 3.22e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260505 11:28:24.180 | INFO | PerfReport: 2.07%, LockUpload -- 2.09e+00 / 0.00e+00, 7.63e+00 / 0.00e+00 | 20260505 11:28:24.180 | INFO | PerfReport: 1.23%, LockDownload -- 1.24e+00 / 0.00e+00, 2.81e+00 / 0.00e+00 | 20260505 11:28:24.180 | INFO | PerfReport: 20.44%, LockCompute -- 2.07e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260505 11:28:24.180 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.93e-02 / 0.00e+00 | 20260505 11:28:24.316 | INFO | Starting a round of Baz IO | 20260505 11:28:27.581 | INFO | Populating RTMetrics report | 20260505 11:28:27.581 | INFO | RTMetrics report populated | 20260505 11:28:27.581 | INFO | Opening tmp RTMetrics json file | 20260505 11:28:27.587 | INFO | Serializing RTMetrics to json | 20260505 11:28:27.591 | INFO | Serialization done, swapping tmp file now... | 20260505 11:28:27.593 | INFO | RTMetrics done | 20260505 11:28:27.713 | INFO | Finished a round of Baz IO: Wrote 2.382 GiB in 3.397 seconds | 20260505 11:28:27.713 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.16% / 0.01% / 75.83% | 20260505 11:28:27.713 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.1451% / 1.19209e-05% / 6.35783e-05% | 20260505 11:28:27.713 | INFO | Baz IO overall: | 20260505 11:28:27.713 | INFO | BytesWritten : 1685675964881 | 20260505 11:28:27.713 | INFO | NumEvents : 802603896109 | 20260505 11:28:29.330 | INFO | Analyzing chunk frames = [2850816,2851328) | 20260505 11:28:29.330 | INFO | MemInfo: { | 20260505 11:28:29.330 | INFO | "gpu" : " 17 GB", | 20260505 11:28:29.330 | INFO | "huge" : "", | 20260505 11:28:29.330 | INFO | "huge_shared" : " 27 GB", | 20260505 11:28:29.330 | INFO | "malloc" : " 14 GB", | 20260505 11:28:29.330 | INFO | "pinned" : " 29 GB" | 20260505 11:28:29.330 | INFO | } | 20260505 11:28:29.330 | INFO | Perf report for the last 5.15109 seconds: | 20260505 11:28:29.330 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:28:29.330 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:29.330 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:29.330 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:29.330 | INFO | ANALYSIS : 64.9%, 2.9, 100.2ms, 0 | 20260505 11:28:29.330 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:29.330 | INFO | RT_METRICS : 45.5%, 1.0, 23.9ms, 0 | 20260505 11:28:29.330 | INFO | PRE_HQ : 61.5%, 1.8, 59.0ms, 0 | 20260505 11:28:29.330 | INFO | BAZWRITER : 65.3%, 4.1, 137.6ms, 0 | 20260505 11:28:29.330 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 11:28:29.330 | INFO | Uploaded 36GiB to GPU in 2.23841s (16.0829GiB/s) | 20260505 11:28:29.330 | INFO | Downloaded 38GiB to GPU in 2.29793s (16.5366GiB/s) | 20260505 11:28:34.465 | INFO | Analyzing chunk frames = [2851328,2851840) | 20260505 11:28:34.465 | INFO | MemInfo: { | 20260505 11:28:34.465 | INFO | "gpu" : " 17 GB", | 20260505 11:28:34.465 | INFO | "huge" : "", | 20260505 11:28:34.465 | INFO | "huge_shared" : " 27 GB", | 20260505 11:28:34.465 | INFO | "malloc" : " 14 GB", | 20260505 11:28:34.465 | INFO | "pinned" : " 29 GB" | 20260505 11:28:34.465 | INFO | } | 20260505 11:28:34.465 | INFO | Perf report for the last 5.13544 seconds: | 20260505 11:28:34.465 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:28:34.465 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:34.465 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:34.465 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:34.465 | INFO | ANALYSIS : 64.0%, 3.0, 99.8ms, 0 | 20260505 11:28:34.466 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:34.466 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:34.466 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:28:34.466 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:34.466 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:34.466 | INFO | Uploaded 36GiB to GPU in 2.17821s (16.5273GiB/s) | 20260505 11:28:34.466 | INFO | Downloaded 34GiB to GPU in 2.04981s (16.5869GiB/s) | 20260505 11:28:39.599 | INFO | Analyzing chunk frames = [2851840,2852352) | 20260505 11:28:39.599 | INFO | MemInfo: { | 20260505 11:28:39.599 | INFO | "gpu" : " 17 GB", | 20260505 11:28:39.599 | INFO | "huge" : "", | 20260505 11:28:39.599 | INFO | "huge_shared" : " 27 GB", | 20260505 11:28:39.599 | INFO | "malloc" : " 14 GB", | 20260505 11:28:39.599 | INFO | "pinned" : " 29 GB" | 20260505 11:28:39.599 | INFO | } | 20260505 11:28:39.600 | INFO | Perf report for the last 5.13413 seconds: | 20260505 11:28:39.600 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:28:39.600 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:39.600 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:39.600 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:39.600 | INFO | ANALYSIS : 63.6%, 3.0, 98.9ms, 0 | 20260505 11:28:39.600 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:39.600 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:39.600 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 11:28:39.600 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:39.600 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:39.600 | INFO | Uploaded 36GiB to GPU in 2.22299s (16.1944GiB/s) | 20260505 11:28:39.600 | INFO | Downloaded 34GiB to GPU in 2.13071s (15.9571GiB/s) | 20260505 11:28:44.735 | INFO | Analyzing chunk frames = [2852352,2852864) | 20260505 11:28:44.735 | INFO | MemInfo: { | 20260505 11:28:44.735 | INFO | "gpu" : " 17 GB", | 20260505 11:28:44.735 | INFO | "huge" : "", | 20260505 11:28:44.735 | INFO | "huge_shared" : " 27 GB", | 20260505 11:28:44.735 | INFO | "malloc" : " 14 GB", | 20260505 11:28:44.735 | INFO | "pinned" : " 29 GB" | 20260505 11:28:44.735 | INFO | } | 20260505 11:28:44.735 | INFO | Perf report for the last 5.13554 seconds: | 20260505 11:28:44.735 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:28:44.735 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:44.735 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:44.735 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:44.735 | INFO | ANALYSIS : 62.9%, 3.0, 97.6ms, 0 | 20260505 11:28:44.735 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:44.735 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:44.735 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 11:28:44.735 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:44.735 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:44.735 | INFO | Uploaded 36GiB to GPU in 2.15225s (16.7267GiB/s) | 20260505 11:28:44.735 | INFO | Downloaded 34GiB to GPU in 2.07295s (16.4018GiB/s) | 20260505 11:28:49.881 | INFO | Analyzing chunk frames = [2852864,2853376) | 20260505 11:28:49.881 | INFO | MemInfo: { | 20260505 11:28:49.881 | INFO | "gpu" : " 17 GB", | 20260505 11:28:49.881 | INFO | "huge" : "", | 20260505 11:28:49.881 | INFO | "huge_shared" : " 27 GB", | 20260505 11:28:49.881 | INFO | "malloc" : " 14 GB", | 20260505 11:28:49.881 | INFO | "pinned" : " 29 GB" | 20260505 11:28:49.881 | INFO | } | 20260505 11:28:49.881 | INFO | Perf report for the last 5.14636 seconds: | 20260505 11:28:49.881 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:28:49.881 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:49.881 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:49.881 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:49.882 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 11:28:49.882 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:49.882 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:49.882 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 11:28:49.882 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:49.882 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:49.882 | INFO | Uploaded 36GiB to GPU in 2.16158s (16.6545GiB/s) | 20260505 11:28:49.882 | INFO | Downloaded 34GiB to GPU in 2.12417s (16.0063GiB/s) | 20260505 11:28:55.020 | INFO | Analyzing chunk frames = [2853376,2853888) | 20260505 11:28:55.020 | INFO | MemInfo: { | 20260505 11:28:55.020 | INFO | "gpu" : " 17 GB", | 20260505 11:28:55.020 | INFO | "huge" : "", | 20260505 11:28:55.020 | INFO | "huge_shared" : " 27 GB", | 20260505 11:28:55.020 | INFO | "malloc" : " 14 GB", | 20260505 11:28:55.020 | INFO | "pinned" : " 29 GB" | 20260505 11:28:55.020 | INFO | } | 20260505 11:28:55.020 | INFO | Perf report for the last 5.13856 seconds: | 20260505 11:28:55.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:28:55.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:55.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:55.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:55.020 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 11:28:55.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:55.020 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:55.020 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 11:28:55.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:55.020 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:28:55.020 | INFO | Uploaded 36GiB to GPU in 2.15895s (16.6748GiB/s) | 20260505 11:28:55.020 | INFO | Downloaded 34GiB to GPU in 2.10713s (16.1357GiB/s) | 20260505 11:29:00.161 | INFO | Analyzing chunk frames = [2853888,2854400) | 20260505 11:29:00.161 | INFO | MemInfo: { | 20260505 11:29:00.161 | INFO | "gpu" : " 17 GB", | 20260505 11:29:00.161 | INFO | "huge" : "", | 20260505 11:29:00.161 | INFO | "huge_shared" : " 27 GB", | 20260505 11:29:00.161 | INFO | "malloc" : " 14 GB", | 20260505 11:29:00.161 | INFO | "pinned" : " 29 GB" | 20260505 11:29:00.161 | INFO | } | 20260505 11:29:00.161 | INFO | Perf report for the last 5.14107 seconds: | 20260505 11:29:00.161 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:29:00.161 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:00.161 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:00.161 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:00.161 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 11:29:00.161 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:00.161 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:00.161 | INFO | PRE_HQ : 30.3%, 1.0, 15.8ms, 0 | 20260505 11:29:00.161 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:00.161 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:00.161 | INFO | Uploaded 36GiB to GPU in 2.15385s (16.7142GiB/s) | 20260505 11:29:00.161 | INFO | Downloaded 34GiB to GPU in 2.13165s (15.9501GiB/s) | 20260505 11:29:05.294 | INFO | Analyzing chunk frames = [2854400,2854912) | 20260505 11:29:05.294 | INFO | MemInfo: { | 20260505 11:29:05.294 | INFO | "gpu" : " 17 GB", | 20260505 11:29:05.294 | INFO | "huge" : "", | 20260505 11:29:05.294 | INFO | "huge_shared" : " 27 GB", | 20260505 11:29:05.294 | INFO | "malloc" : " 14 GB", | 20260505 11:29:05.294 | INFO | "pinned" : " 29 GB" | 20260505 11:29:05.294 | INFO | } | 20260505 11:29:05.295 | INFO | Perf report for the last 5.13342 seconds: | 20260505 11:29:05.295 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:29:05.295 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:05.295 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:05.295 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:05.295 | INFO | ANALYSIS : 63.9%, 3.0, 99.6ms, 0 | 20260505 11:29:05.295 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:05.295 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:05.295 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 11:29:05.295 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:05.295 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:05.295 | INFO | Uploaded 36GiB to GPU in 2.21416s (16.259GiB/s) | 20260505 11:29:05.295 | INFO | Downloaded 34GiB to GPU in 2.08716s (16.2901GiB/s) | 20260505 11:29:05.426 | INFO | Starting a round of Baz IO | 20260505 11:29:08.685 | INFO | Populating RTMetrics report | 20260505 11:29:08.686 | INFO | RTMetrics report populated | 20260505 11:29:08.686 | INFO | Opening tmp RTMetrics json file | 20260505 11:29:08.689 | INFO | Serializing RTMetrics to json | 20260505 11:29:08.691 | INFO | Serialization done, swapping tmp file now... | 20260505 11:29:08.691 | INFO | RTMetrics done | 20260505 11:29:08.828 | INFO | Finished a round of Baz IO: Wrote 2.379 GiB in 3.401 seconds | 20260505 11:29:08.828 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.15% / 0.01% / 75.84% | 20260505 11:29:08.828 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.13896% / 1.58946e-05% / 6.35783e-05% | 20260505 11:29:08.828 | INFO | Baz IO overall: | 20260505 11:29:08.828 | INFO | BytesWritten : 1688230361787 | 20260505 11:29:08.828 | INFO | NumEvents : 803819323079 | 20260505 11:29:10.431 | INFO | Analyzing chunk frames = [2854912,2855424) | 20260505 11:29:10.431 | INFO | MemInfo: { | 20260505 11:29:10.431 | INFO | "gpu" : " 17 GB", | 20260505 11:29:10.431 | INFO | "huge" : "", | 20260505 11:29:10.431 | INFO | "huge_shared" : " 27 GB", | 20260505 11:29:10.431 | INFO | "malloc" : " 14 GB", | 20260505 11:29:10.431 | INFO | "pinned" : " 29 GB" | 20260505 11:29:10.431 | INFO | } | 20260505 11:29:10.431 | INFO | Perf report for the last 5.13681 seconds: | 20260505 11:29:10.431 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:29:10.431 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:10.431 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:10.431 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:10.431 | INFO | ANALYSIS : 64.9%, 2.9, 99.8ms, 0 | 20260505 11:29:10.431 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:10.431 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260505 11:29:10.431 | INFO | PRE_HQ : 62.8%, 1.8, 59.1ms, 0 | 20260505 11:29:10.431 | INFO | BAZWRITER : 65.5%, 4.1, 137.9ms, 0 | 20260505 11:29:10.431 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 11:29:10.432 | INFO | Uploaded 36GiB to GPU in 2.29347s (15.6968GiB/s) | 20260505 11:29:10.432 | INFO | Downloaded 38GiB to GPU in 2.35765s (16.1178GiB/s) | 20260505 11:29:15.576 | INFO | Analyzing chunk frames = [2855424,2855936) | 20260505 11:29:15.576 | INFO | MemInfo: { | 20260505 11:29:15.576 | INFO | "gpu" : " 17 GB", | 20260505 11:29:15.576 | INFO | "huge" : "", | 20260505 11:29:15.576 | INFO | "huge_shared" : " 27 GB", | 20260505 11:29:15.576 | INFO | "malloc" : " 14 GB", | 20260505 11:29:15.576 | INFO | "pinned" : " 29 GB" | 20260505 11:29:15.576 | INFO | } | 20260505 11:29:15.576 | INFO | Perf report for the last 5.14484 seconds: | 20260505 11:29:15.576 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:29:15.576 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:15.576 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:15.576 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:15.576 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 11:29:15.576 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:15.576 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:15.576 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:29:15.576 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:15.576 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:15.576 | INFO | Uploaded 36GiB to GPU in 2.17886s (16.5224GiB/s) | 20260505 11:29:15.576 | INFO | Downloaded 34GiB to GPU in 2.08301s (16.3226GiB/s) | 20260505 11:29:20.728 | INFO | Analyzing chunk frames = [2855936,2856448) | 20260505 11:29:20.728 | INFO | MemInfo: { | 20260505 11:29:20.728 | INFO | "gpu" : " 17 GB", | 20260505 11:29:20.728 | INFO | "huge" : "", | 20260505 11:29:20.728 | INFO | "huge_shared" : " 27 GB", | 20260505 11:29:20.728 | INFO | "malloc" : " 14 GB", | 20260505 11:29:20.728 | INFO | "pinned" : " 29 GB" | 20260505 11:29:20.728 | INFO | } | 20260505 11:29:20.728 | INFO | Perf report for the last 5.1515 seconds: | 20260505 11:29:20.728 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:29:20.728 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:20.728 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:20.728 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:20.728 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 11:29:20.728 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:20.728 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:20.728 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:29:20.728 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:20.728 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:20.728 | INFO | Uploaded 36GiB to GPU in 2.21642s (16.2424GiB/s) | 20260505 11:29:20.728 | INFO | Downloaded 34GiB to GPU in 2.12513s (15.999GiB/s) | 20260505 11:29:25.861 | INFO | Analyzing chunk frames = [2856448,2856960) | 20260505 11:29:25.861 | INFO | MemInfo: { | 20260505 11:29:25.861 | INFO | "gpu" : " 17 GB", | 20260505 11:29:25.861 | INFO | "huge" : "", | 20260505 11:29:25.861 | INFO | "huge_shared" : " 27 GB", | 20260505 11:29:25.861 | INFO | "malloc" : " 14 GB", | 20260505 11:29:25.861 | INFO | "pinned" : " 29 GB" | 20260505 11:29:25.861 | INFO | } | 20260505 11:29:25.861 | INFO | Perf report for the last 5.13337 seconds: | 20260505 11:29:25.861 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:29:25.861 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:25.861 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:25.861 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:25.861 | INFO | ANALYSIS : 63.2%, 3.0, 98.0ms, 0 | 20260505 11:29:25.861 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:25.861 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:25.861 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:29:25.861 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:25.861 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:25.861 | INFO | Uploaded 36GiB to GPU in 2.19999s (16.3637GiB/s) | 20260505 11:29:25.861 | INFO | Downloaded 34GiB to GPU in 2.13121s (15.9534GiB/s) | 20260505 11:29:30.988 | INFO | Analyzing chunk frames = [2856960,2857472) | 20260505 11:29:30.988 | INFO | MemInfo: { | 20260505 11:29:30.988 | INFO | "gpu" : " 17 GB", | 20260505 11:29:30.988 | INFO | "huge" : "", | 20260505 11:29:30.988 | INFO | "huge_shared" : " 27 GB", | 20260505 11:29:30.988 | INFO | "malloc" : " 14 GB", | 20260505 11:29:30.988 | INFO | "pinned" : " 29 GB" | 20260505 11:29:30.988 | INFO | } | 20260505 11:29:30.988 | INFO | Perf report for the last 5.12694 seconds: | 20260505 11:29:30.988 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:29:30.988 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:30.988 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:30.988 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:30.988 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 11:29:30.988 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:30.988 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:30.988 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 11:29:30.988 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:30.988 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:30.988 | INFO | Uploaded 36GiB to GPU in 2.15486s (16.7064GiB/s) | 20260505 11:29:30.988 | INFO | Downloaded 34GiB to GPU in 2.12414s (16.0065GiB/s) | 20260505 11:29:36.132 | INFO | Analyzing chunk frames = [2857472,2857984) | 20260505 11:29:36.132 | INFO | MemInfo: { | 20260505 11:29:36.132 | INFO | "gpu" : " 17 GB", | 20260505 11:29:36.132 | INFO | "huge" : "", | 20260505 11:29:36.132 | INFO | "huge_shared" : " 27 GB", | 20260505 11:29:36.132 | INFO | "malloc" : " 14 GB", | 20260505 11:29:36.132 | INFO | "pinned" : " 29 GB" | 20260505 11:29:36.132 | INFO | } | 20260505 11:29:36.132 | INFO | Perf report for the last 5.14375 seconds: | 20260505 11:29:36.132 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:29:36.132 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:36.132 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:36.132 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:36.132 | INFO | ANALYSIS : 63.4%, 3.0, 99.2ms, 0 | 20260505 11:29:36.132 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:36.132 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:36.132 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 11:29:36.132 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:36.132 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:36.132 | INFO | Uploaded 36GiB to GPU in 2.21735s (16.2356GiB/s) | 20260505 11:29:36.132 | INFO | Downloaded 34GiB to GPU in 2.10596s (16.1447GiB/s) | 20260505 11:29:41.275 | INFO | Analyzing chunk frames = [2857984,2858496) | 20260505 11:29:41.275 | INFO | MemInfo: { | 20260505 11:29:41.275 | INFO | "gpu" : " 17 GB", | 20260505 11:29:41.275 | INFO | "huge" : "", | 20260505 11:29:41.275 | INFO | "huge_shared" : " 27 GB", | 20260505 11:29:41.275 | INFO | "malloc" : " 14 GB", | 20260505 11:29:41.275 | INFO | "pinned" : " 29 GB" | 20260505 11:29:41.275 | INFO | } | 20260505 11:29:41.275 | INFO | Perf report for the last 5.1432 seconds: | 20260505 11:29:41.275 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:29:41.275 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:41.275 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:41.275 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:41.275 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 11:29:41.275 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:41.275 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:41.275 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 11:29:41.275 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:41.275 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:41.275 | INFO | Uploaded 36GiB to GPU in 2.16811s (16.6044GiB/s) | 20260505 11:29:41.275 | INFO | Downloaded 34GiB to GPU in 2.13435s (15.9299GiB/s) | 20260505 11:29:46.416 | INFO | Analyzing chunk frames = [2858496,2859008) | 20260505 11:29:46.416 | INFO | MemInfo: { | 20260505 11:29:46.416 | INFO | "gpu" : " 17 GB", | 20260505 11:29:46.416 | INFO | "huge" : "", | 20260505 11:29:46.416 | INFO | "huge_shared" : " 27 GB", | 20260505 11:29:46.416 | INFO | "malloc" : " 14 GB", | 20260505 11:29:46.416 | INFO | "pinned" : " 29 GB" | 20260505 11:29:46.416 | INFO | } | 20260505 11:29:46.416 | INFO | Perf report for the last 5.14103 seconds: | 20260505 11:29:46.416 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:29:46.416 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:46.416 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:46.416 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:46.416 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 11:29:46.416 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:46.416 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:46.416 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 11:29:46.416 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:46.416 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:46.416 | INFO | Uploaded 36GiB to GPU in 2.2151s (16.2521GiB/s) | 20260505 11:29:46.416 | INFO | Downloaded 34GiB to GPU in 2.0902s (16.2664GiB/s) | 20260505 11:29:46.416 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 11:29:46.417 | INFO | Total mem usage: 15.3 GiB | 20260505 11:29:46.417 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 11:29:46.417 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -6.05%) | 20260505 11:29:46.417 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 11:29:46.417 | INFO | High Water Breakdown: | 20260505 11:29:46.417 | INFO | 61.1% : 8.84 GiB ( 93.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 11:29:46.417 | INFO | 38.7% : 5.6 GiB ( 84.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 11:29:46.417 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 11:29:46.417 | INFO | ------------------------------------------ | 20260505 11:29:46.417 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:29:46.417 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 11:29:46.417 | INFO | Total mem usage: 26.5 GiB | 20260505 11:29:46.417 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 11:29:46.417 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 11:29:46.417 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 11:29:46.417 | INFO | High Water Breakdown: | 20260505 11:29:46.417 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:29:46.417 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:29:46.417 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:29:46.417 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:29:46.417 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:29:46.417 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:29:46.417 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 11:29:46.417 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 11:29:46.417 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:29:46.417 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 11:29:46.417 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 11:29:46.417 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 11:29:46.417 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 11:29:46.417 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 11:29:46.417 | INFO | ------------------------------------------ | 20260505 11:29:46.417 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:29:46.417 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 11:29:46.417 | INFO | Total mem usage: 25.6 GiB | 20260505 11:29:46.417 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 11:29:46.417 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 11:29:46.417 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 11:29:46.417 | INFO | High Water Breakdown: | 20260505 11:29:46.417 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 11:29:46.417 | INFO | ------------------------------------------ | 20260505 11:29:46.417 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:29:46.417 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 11:29:46.417 | INFO | Total mem usage: 15.7 GiB | 20260505 11:29:46.417 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 11:29:46.417 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 11:29:46.417 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 11:29:46.417 | INFO | High Water Breakdown: | 20260505 11:29:46.417 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 11:29:46.417 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 11:29:46.417 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 11:29:46.417 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 11:29:46.417 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 11:29:46.417 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 11:29:46.417 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 11:29:46.417 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 11:29:46.417 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 11:29:46.417 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:29:46.417 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 11:29:46.417 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:29:46.417 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 11:29:46.417 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 11:29:46.418 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 11:29:46.418 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 11:29:46.418 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:29:46.418 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 11:29:46.418 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:29:46.418 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:29:46.418 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:29:46.418 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 11:29:46.418 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 11:29:46.418 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:29:46.418 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 11:29:46.418 | INFO | ------------------------------------------ | 20260505 11:29:46.418 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:29:46.418 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 11:29:46.418 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:29:46.418 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:29:46.418 | INFO | PerfReport: Full Execution -- 3.22e+01 / 7.73e+01, 1.38e+01 / 1.87e+00 | 20260505 11:29:46.418 | INFO | PerfReport: 0.29%, AggregateStats -- 9.32e-02 / 1.01e-01, 1.05e-02 / 3.72e-03 | 20260505 11:29:46.418 | INFO | PerfReport: 11.74%, Baseline -- 3.78e+00 / 4.43e+00, 6.15e-01 / 3.14e-01 | 20260505 11:29:46.418 | INFO | PerfReport: 3.32%, Histogram -- 1.07e+00 / 1.49e+00, 2.08e-01 / 1.07e-01 | 20260505 11:29:46.418 | INFO | PerfReport: 15.76%, DME -- 4.06e+01 / 4.47e+01, 4.33e+00 / 2.38e+00 | 20260505 11:29:46.418 | INFO | PerfReport: 56.49%, FrameLabeling -- 1.82e+01 / 2.17e+01, 1.65e+00 / 2.27e+00 | 20260505 11:29:46.418 | INFO | PerfReport: 7.41%, PulseAccumulating -- 2.38e+00 / 2.85e+00, 4.30e-01 / 2.21e-01 | 20260505 11:29:46.418 | INFO | PerfReport: 3.70%, Metrics -- 1.19e+00 / 1.47e+00, 3.41e-01 / 1.85e-01 | 20260505 11:29:46.418 | INFO | PerfReport: 1.30%, PulseSerializing -- 4.19e-01 / 4.65e-01, 7.71e-02 / 4.06e-02 | 20260505 11:29:46.418 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 11:29:46.418 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:29:46.418 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:29:46.418 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.45e+01 / 0.00e+00 | 20260505 11:29:46.418 | INFO | PerfReport: 22.60%, Upload -- 2.29e+01 / 0.00e+00, 8.17e+00 / 0.00e+00 | 20260505 11:29:46.418 | INFO | PerfReport: 21.98%, Download -- 2.22e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260505 11:29:46.418 | INFO | PerfReport: 31.82%, Compute -- 3.22e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 11:29:46.418 | INFO | PerfReport: 1.95%, LockUpload -- 1.97e+00 / 0.00e+00, 7.59e+00 / 0.00e+00 | 20260505 11:29:46.418 | INFO | PerfReport: 1.32%, LockDownload -- 1.33e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260505 11:29:46.418 | INFO | PerfReport: 20.32%, LockCompute -- 2.06e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260505 11:29:46.418 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.94e-02 / 0.00e+00 | 20260505 11:29:46.552 | INFO | Starting a round of Baz IO | 20260505 11:29:49.817 | INFO | Populating RTMetrics report | 20260505 11:29:49.817 | INFO | RTMetrics report populated | 20260505 11:29:49.817 | INFO | Opening tmp RTMetrics json file | 20260505 11:29:49.827 | INFO | Serializing RTMetrics to json | 20260505 11:29:49.868 | INFO | Serialization done, swapping tmp file now... | 20260505 11:29:49.869 | INFO | RTMetrics done | 20260505 11:29:49.973 | INFO | Finished a round of Baz IO: Wrote 2.378 GiB in 3.421 seconds | 20260505 11:29:49.973 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.15% / 0.01% / 75.85% | 20260505 11:29:49.973 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.13308% / 1.58946e-05% / 5.5631e-05% | 20260505 11:29:49.973 | INFO | Baz IO overall: | 20260505 11:29:49.973 | INFO | BytesWritten : 1690783333624 | 20260505 11:29:49.973 | INFO | NumEvents : 805033432952 | 20260505 11:29:51.547 | INFO | Analyzing chunk frames = [2859008,2859520) | 20260505 11:29:51.548 | INFO | MemInfo: { | 20260505 11:29:51.548 | INFO | "gpu" : " 17 GB", | 20260505 11:29:51.548 | INFO | "huge" : "", | 20260505 11:29:51.548 | INFO | "huge_shared" : " 27 GB", | 20260505 11:29:51.548 | INFO | "malloc" : " 14 GB", | 20260505 11:29:51.548 | INFO | "pinned" : " 29 GB" | 20260505 11:29:51.548 | INFO | } | 20260505 11:29:51.548 | INFO | Perf report for the last 5.13147 seconds: | 20260505 11:29:51.548 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:29:51.548 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:51.548 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:51.548 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:51.548 | INFO | ANALYSIS : 65.2%, 2.9, 99.8ms, 0 | 20260505 11:29:51.548 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:51.548 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260505 11:29:51.548 | INFO | PRE_HQ : 61.7%, 1.8, 59.1ms, 0 | 20260505 11:29:51.548 | INFO | BAZWRITER : 66.0%, 4.1, 138.7ms, 0 | 20260505 11:29:51.548 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260505 11:29:51.548 | INFO | Uploaded 36GiB to GPU in 2.22673s (16.1672GiB/s) | 20260505 11:29:51.548 | INFO | Downloaded 38GiB to GPU in 2.3023s (16.5052GiB/s) | 20260505 11:29:56.688 | INFO | Analyzing chunk frames = [2859520,2860032) | 20260505 11:29:56.688 | INFO | MemInfo: { | 20260505 11:29:56.688 | INFO | "gpu" : " 17 GB", | 20260505 11:29:56.688 | INFO | "huge" : "", | 20260505 11:29:56.688 | INFO | "huge_shared" : " 27 GB", | 20260505 11:29:56.688 | INFO | "malloc" : " 14 GB", | 20260505 11:29:56.689 | INFO | "pinned" : " 29 GB" | 20260505 11:29:56.689 | INFO | } | 20260505 11:29:56.689 | INFO | Perf report for the last 5.14098 seconds: | 20260505 11:29:56.689 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:29:56.689 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:56.689 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:56.689 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:56.689 | INFO | ANALYSIS : 64.1%, 3.0, 100.2ms, 0 | 20260505 11:29:56.689 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:56.689 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:56.689 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:29:56.689 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:56.689 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:29:56.689 | INFO | Uploaded 36GiB to GPU in 2.20981s (16.291GiB/s) | 20260505 11:29:56.689 | INFO | Downloaded 34GiB to GPU in 2.08878s (16.2774GiB/s) | 20260505 11:30:01.834 | INFO | Analyzing chunk frames = [2860032,2860544) | 20260505 11:30:01.834 | INFO | MemInfo: { | 20260505 11:30:01.834 | INFO | "gpu" : " 17 GB", | 20260505 11:30:01.834 | INFO | "huge" : "", | 20260505 11:30:01.834 | INFO | "huge_shared" : " 27 GB", | 20260505 11:30:01.834 | INFO | "malloc" : " 14 GB", | 20260505 11:30:01.834 | INFO | "pinned" : " 29 GB" | 20260505 11:30:01.834 | INFO | } | 20260505 11:30:01.834 | INFO | Perf report for the last 5.14588 seconds: | 20260505 11:30:01.835 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:30:01.835 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:01.835 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:01.835 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:01.835 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 11:30:01.835 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:01.835 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:01.835 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 11:30:01.835 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:01.835 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:01.835 | INFO | Uploaded 36GiB to GPU in 2.22461s (16.1826GiB/s) | 20260505 11:30:01.835 | INFO | Downloaded 34GiB to GPU in 2.12808s (15.9768GiB/s) | 20260505 11:30:06.974 | INFO | Analyzing chunk frames = [2860544,2861056) | 20260505 11:30:06.974 | INFO | MemInfo: { | 20260505 11:30:06.974 | INFO | "gpu" : " 17 GB", | 20260505 11:30:06.974 | INFO | "huge" : "", | 20260505 11:30:06.974 | INFO | "huge_shared" : " 27 GB", | 20260505 11:30:06.974 | INFO | "malloc" : " 14 GB", | 20260505 11:30:06.974 | INFO | "pinned" : " 29 GB" | 20260505 11:30:06.974 | INFO | } | 20260505 11:30:06.974 | INFO | Perf report for the last 5.13995 seconds: | 20260505 11:30:06.975 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:30:06.975 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:06.975 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:06.975 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:06.975 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 11:30:06.975 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:06.975 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:06.975 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:30:06.975 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:06.975 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:06.975 | INFO | Uploaded 36GiB to GPU in 2.20352s (16.3375GiB/s) | 20260505 11:30:06.975 | INFO | Downloaded 34GiB to GPU in 2.1333s (15.9378GiB/s) | 20260505 11:30:12.106 | INFO | Analyzing chunk frames = [2861056,2861568) | 20260505 11:30:12.106 | INFO | MemInfo: { | 20260505 11:30:12.106 | INFO | "gpu" : " 17 GB", | 20260505 11:30:12.106 | INFO | "huge" : "", | 20260505 11:30:12.106 | INFO | "huge_shared" : " 27 GB", | 20260505 11:30:12.106 | INFO | "malloc" : " 14 GB", | 20260505 11:30:12.106 | INFO | "pinned" : " 29 GB" | 20260505 11:30:12.106 | INFO | } | 20260505 11:30:12.106 | INFO | Perf report for the last 5.13198 seconds: | 20260505 11:30:12.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:30:12.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:12.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:12.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:12.107 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 11:30:12.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:12.107 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:12.107 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 11:30:12.107 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:12.107 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:12.107 | INFO | Uploaded 36GiB to GPU in 2.18633s (16.466GiB/s) | 20260505 11:30:12.107 | INFO | Downloaded 34GiB to GPU in 2.12937s (15.9671GiB/s) | 20260505 11:30:17.242 | INFO | Analyzing chunk frames = [2861568,2862080) | 20260505 11:30:17.242 | INFO | MemInfo: { | 20260505 11:30:17.242 | INFO | "gpu" : " 17 GB", | 20260505 11:30:17.242 | INFO | "huge" : "", | 20260505 11:30:17.242 | INFO | "huge_shared" : " 27 GB", | 20260505 11:30:17.242 | INFO | "malloc" : " 14 GB", | 20260505 11:30:17.242 | INFO | "pinned" : " 29 GB" | 20260505 11:30:17.242 | INFO | } | 20260505 11:30:17.242 | INFO | Perf report for the last 5.13535 seconds: | 20260505 11:30:17.242 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:30:17.242 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:17.242 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:17.242 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:17.242 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 11:30:17.242 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:17.242 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:17.242 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 11:30:17.242 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:17.242 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:17.242 | INFO | Uploaded 36GiB to GPU in 2.1987s (16.3733GiB/s) | 20260505 11:30:17.242 | INFO | Downloaded 34GiB to GPU in 2.10707s (16.1361GiB/s) | 20260505 11:30:22.387 | INFO | Analyzing chunk frames = [2862080,2862592) | 20260505 11:30:22.387 | INFO | MemInfo: { | 20260505 11:30:22.387 | INFO | "gpu" : " 17 GB", | 20260505 11:30:22.387 | INFO | "huge" : "", | 20260505 11:30:22.387 | INFO | "huge_shared" : " 27 GB", | 20260505 11:30:22.387 | INFO | "malloc" : " 14 GB", | 20260505 11:30:22.387 | INFO | "pinned" : " 29 GB" | 20260505 11:30:22.387 | INFO | } | 20260505 11:30:22.387 | INFO | Perf report for the last 5.14532 seconds: | 20260505 11:30:22.387 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:30:22.387 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:22.387 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:22.387 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:22.387 | INFO | ANALYSIS : 63.5%, 3.0, 99.3ms, 0 | 20260505 11:30:22.387 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:22.387 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:22.387 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 11:30:22.387 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:22.387 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:22.387 | INFO | Uploaded 36GiB to GPU in 2.19038s (16.4355GiB/s) | 20260505 11:30:22.387 | INFO | Downloaded 34GiB to GPU in 2.13375s (15.9344GiB/s) | 20260505 11:30:27.527 | INFO | Analyzing chunk frames = [2862592,2863104) | 20260505 11:30:27.527 | INFO | MemInfo: { | 20260505 11:30:27.527 | INFO | "gpu" : " 17 GB", | 20260505 11:30:27.527 | INFO | "huge" : "", | 20260505 11:30:27.527 | INFO | "huge_shared" : " 27 GB", | 20260505 11:30:27.527 | INFO | "malloc" : " 14 GB", | 20260505 11:30:27.527 | INFO | "pinned" : " 29 GB" | 20260505 11:30:27.527 | INFO | } | 20260505 11:30:27.527 | INFO | Perf report for the last 5.13981 seconds: | 20260505 11:30:27.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:30:27.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:27.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:27.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:27.527 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 11:30:27.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:27.527 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:27.527 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 11:30:27.527 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:27.527 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:27.527 | INFO | Uploaded 36GiB to GPU in 2.21189s (16.2757GiB/s) | 20260505 11:30:27.527 | INFO | Downloaded 34GiB to GPU in 2.08396s (16.3151GiB/s) | 20260505 11:30:27.660 | INFO | Starting a round of Baz IO | 20260505 11:30:30.921 | INFO | Populating RTMetrics report | 20260505 11:30:30.922 | INFO | RTMetrics report populated | 20260505 11:30:30.922 | INFO | Opening tmp RTMetrics json file | 20260505 11:30:30.961 | INFO | Serializing RTMetrics to json | 20260505 11:30:30.985 | INFO | Serialization done, swapping tmp file now... | 20260505 11:30:30.987 | INFO | RTMetrics done | 20260505 11:30:31.088 | INFO | Finished a round of Baz IO: Wrote 2.379 GiB in 3.428 seconds | 20260505 11:30:31.088 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.14% / 0.01% / 75.85% | 20260505 11:30:31.088 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.133% / 3.97364e-06% / 6.35783e-05% | 20260505 11:30:31.088 | INFO | Baz IO overall: | 20260505 11:30:31.088 | INFO | BytesWritten : 1693337515788 | 20260505 11:30:31.088 | INFO | NumEvents : 806248628602 | 20260505 11:30:32.671 | INFO | Analyzing chunk frames = [2863104,2863616) | 20260505 11:30:32.671 | INFO | MemInfo: { | 20260505 11:30:32.671 | INFO | "gpu" : " 17 GB", | 20260505 11:30:32.671 | INFO | "huge" : "", | 20260505 11:30:32.671 | INFO | "huge_shared" : " 27 GB", | 20260505 11:30:32.671 | INFO | "malloc" : " 14 GB", | 20260505 11:30:32.671 | INFO | "pinned" : " 29 GB" | 20260505 11:30:32.671 | INFO | } | 20260505 11:30:32.671 | INFO | Perf report for the last 5.14443 seconds: | 20260505 11:30:32.671 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:30:32.671 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:32.671 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:32.671 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:32.671 | INFO | ANALYSIS : 64.9%, 2.9, 99.6ms, 0 | 20260505 11:30:32.671 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:32.671 | INFO | RT_METRICS : 46.9%, 1.0, 24.6ms, 0 | 20260505 11:30:32.671 | INFO | PRE_HQ : 61.3%, 1.9, 59.1ms, 0 | 20260505 11:30:32.672 | INFO | BAZWRITER : 65.9%, 4.2, 140.9ms, 0 | 20260505 11:30:32.672 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 11:30:32.672 | INFO | Uploaded 36GiB to GPU in 2.23859s (16.0816GiB/s) | 20260505 11:30:32.672 | INFO | Downloaded 38GiB to GPU in 2.28984s (16.595GiB/s) | 20260505 11:30:37.809 | INFO | Analyzing chunk frames = [2863616,2864128) | 20260505 11:30:37.809 | INFO | MemInfo: { | 20260505 11:30:37.809 | INFO | "gpu" : " 17 GB", | 20260505 11:30:37.809 | INFO | "huge" : "", | 20260505 11:30:37.809 | INFO | "huge_shared" : " 27 GB", | 20260505 11:30:37.809 | INFO | "malloc" : " 14 GB", | 20260505 11:30:37.809 | INFO | "pinned" : " 29 GB" | 20260505 11:30:37.809 | INFO | } | 20260505 11:30:37.810 | INFO | Perf report for the last 5.13825 seconds: | 20260505 11:30:37.810 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:30:37.810 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:37.810 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:37.810 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:37.810 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 11:30:37.810 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:37.810 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:37.810 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 11:30:37.810 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:37.810 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:37.810 | INFO | Uploaded 36GiB to GPU in 2.22803s (16.1578GiB/s) | 20260505 11:30:37.810 | INFO | Downloaded 34GiB to GPU in 2.10085s (16.1839GiB/s) | 20260505 11:30:42.943 | INFO | Analyzing chunk frames = [2864128,2864640) | 20260505 11:30:42.943 | INFO | MemInfo: { | 20260505 11:30:42.943 | INFO | "gpu" : " 17 GB", | 20260505 11:30:42.943 | INFO | "huge" : "", | 20260505 11:30:42.943 | INFO | "huge_shared" : " 27 GB", | 20260505 11:30:42.943 | INFO | "malloc" : " 14 GB", | 20260505 11:30:42.943 | INFO | "pinned" : " 29 GB" | 20260505 11:30:42.943 | INFO | } | 20260505 11:30:42.943 | INFO | Perf report for the last 5.13373 seconds: | 20260505 11:30:42.943 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:30:42.943 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:42.943 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:42.943 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:42.943 | INFO | ANALYSIS : 63.6%, 3.0, 98.9ms, 0 | 20260505 11:30:42.943 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:42.943 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:42.943 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 11:30:42.943 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:42.943 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:42.944 | INFO | Uploaded 36GiB to GPU in 2.2218s (16.2031GiB/s) | 20260505 11:30:42.944 | INFO | Downloaded 34GiB to GPU in 2.13087s (15.9559GiB/s) | 20260505 11:30:48.088 | INFO | Analyzing chunk frames = [2864640,2865152) | 20260505 11:30:48.088 | INFO | MemInfo: { | 20260505 11:30:48.088 | INFO | "gpu" : " 17 GB", | 20260505 11:30:48.088 | INFO | "huge" : "", | 20260505 11:30:48.088 | INFO | "huge_shared" : " 27 GB", | 20260505 11:30:48.088 | INFO | "malloc" : " 14 GB", | 20260505 11:30:48.088 | INFO | "pinned" : " 29 GB" | 20260505 11:30:48.088 | INFO | } | 20260505 11:30:48.088 | INFO | Perf report for the last 5.14473 seconds: | 20260505 11:30:48.088 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:30:48.088 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:48.088 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:48.088 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:48.088 | INFO | ANALYSIS : 63.4%, 3.0, 98.5ms, 0 | 20260505 11:30:48.088 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:48.088 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:48.088 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 11:30:48.088 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:48.088 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:48.088 | INFO | Uploaded 36GiB to GPU in 2.21164s (16.2775GiB/s) | 20260505 11:30:48.088 | INFO | Downloaded 34GiB to GPU in 2.134s (15.9325GiB/s) | 20260505 11:30:53.235 | INFO | Analyzing chunk frames = [2865152,2865664) | 20260505 11:30:53.235 | INFO | MemInfo: { | 20260505 11:30:53.235 | INFO | "gpu" : " 17 GB", | 20260505 11:30:53.235 | INFO | "huge" : "", | 20260505 11:30:53.235 | INFO | "huge_shared" : " 27 GB", | 20260505 11:30:53.235 | INFO | "malloc" : " 14 GB", | 20260505 11:30:53.235 | INFO | "pinned" : " 29 GB" | 20260505 11:30:53.235 | INFO | } | 20260505 11:30:53.235 | INFO | Perf report for the last 5.14734 seconds: | 20260505 11:30:53.236 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:30:53.236 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:53.236 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:53.236 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:53.236 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 11:30:53.236 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:53.236 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:53.236 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 11:30:53.236 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:53.236 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:53.236 | INFO | Uploaded 36GiB to GPU in 2.15012s (16.7432GiB/s) | 20260505 11:30:53.236 | INFO | Downloaded 34GiB to GPU in 2.12368s (16.0099GiB/s) | 20260505 11:30:58.364 | INFO | Analyzing chunk frames = [2865664,2866176) | 20260505 11:30:58.364 | INFO | MemInfo: { | 20260505 11:30:58.364 | INFO | "gpu" : " 17 GB", | 20260505 11:30:58.364 | INFO | "huge" : "", | 20260505 11:30:58.364 | INFO | "huge_shared" : " 27 GB", | 20260505 11:30:58.364 | INFO | "malloc" : " 14 GB", | 20260505 11:30:58.364 | INFO | "pinned" : " 29 GB" | 20260505 11:30:58.364 | INFO | } | 20260505 11:30:58.364 | INFO | Perf report for the last 5.12833 seconds: | 20260505 11:30:58.364 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:30:58.364 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:58.364 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:58.364 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:58.364 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 11:30:58.364 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:58.364 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:58.364 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 11:30:58.364 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:58.364 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:30:58.364 | INFO | Uploaded 36GiB to GPU in 2.15351s (16.7169GiB/s) | 20260505 11:30:58.364 | INFO | Downloaded 34GiB to GPU in 2.11045s (16.1103GiB/s) | 20260505 11:31:03.495 | INFO | Analyzing chunk frames = [2866176,2866688) | 20260505 11:31:03.495 | INFO | MemInfo: { | 20260505 11:31:03.495 | INFO | "gpu" : " 17 GB", | 20260505 11:31:03.495 | INFO | "huge" : "", | 20260505 11:31:03.495 | INFO | "huge_shared" : " 27 GB", | 20260505 11:31:03.495 | INFO | "malloc" : " 14 GB", | 20260505 11:31:03.495 | INFO | "pinned" : " 29 GB" | 20260505 11:31:03.495 | INFO | } | 20260505 11:31:03.495 | INFO | Perf report for the last 5.13105 seconds: | 20260505 11:31:03.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:31:03.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:03.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:03.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:03.495 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 11:31:03.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:03.495 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:03.495 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 11:31:03.495 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:03.495 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:03.495 | INFO | Uploaded 36GiB to GPU in 2.1657s (16.6228GiB/s) | 20260505 11:31:03.495 | INFO | Downloaded 34GiB to GPU in 2.15476s (15.779GiB/s) | 20260505 11:31:08.642 | INFO | Analyzing chunk frames = [2866688,2867200) | 20260505 11:31:08.642 | INFO | MemInfo: { | 20260505 11:31:08.642 | INFO | "gpu" : " 17 GB", | 20260505 11:31:08.642 | INFO | "huge" : "", | 20260505 11:31:08.642 | INFO | "huge_shared" : " 27 GB", | 20260505 11:31:08.642 | INFO | "malloc" : " 14 GB", | 20260505 11:31:08.642 | INFO | "pinned" : " 29 GB" | 20260505 11:31:08.642 | INFO | } | 20260505 11:31:08.642 | INFO | Perf report for the last 5.14734 seconds: | 20260505 11:31:08.642 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:31:08.642 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:08.642 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:08.642 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:08.642 | INFO | ANALYSIS : 64.0%, 3.0, 100.1ms, 0 | 20260505 11:31:08.642 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:08.642 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:08.642 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 11:31:08.642 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:08.642 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:08.642 | INFO | Uploaded 36GiB to GPU in 2.24094s (16.0647GiB/s) | 20260505 11:31:08.642 | INFO | Downloaded 34GiB to GPU in 2.10458s (16.1552GiB/s) | 20260505 11:31:08.643 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 11:31:08.643 | INFO | Total mem usage: 15.3 GiB | 20260505 11:31:08.643 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 11:31:08.643 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.49%) | 20260505 11:31:08.643 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 11:31:08.643 | INFO | High Water Breakdown: | 20260505 11:31:08.643 | INFO | 61% : 8.78 GiB ( 92.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 11:31:08.643 | INFO | 38.9% : 5.6 GiB ( 84.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 11:31:08.643 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 11:31:08.643 | INFO | ------------------------------------------ | 20260505 11:31:08.643 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:31:08.643 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 11:31:08.643 | INFO | Total mem usage: 26.5 GiB | 20260505 11:31:08.643 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 11:31:08.643 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 11:31:08.643 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 11:31:08.643 | INFO | High Water Breakdown: | 20260505 11:31:08.643 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:31:08.643 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:31:08.643 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:31:08.643 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:31:08.643 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:31:08.643 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:31:08.643 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 11:31:08.643 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 11:31:08.643 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:31:08.643 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 11:31:08.643 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 11:31:08.643 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 11:31:08.643 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 11:31:08.643 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 11:31:08.643 | INFO | ------------------------------------------ | 20260505 11:31:08.643 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:31:08.643 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 11:31:08.643 | INFO | Total mem usage: 25.6 GiB | 20260505 11:31:08.643 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 11:31:08.643 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 11:31:08.643 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 11:31:08.643 | INFO | High Water Breakdown: | 20260505 11:31:08.643 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 11:31:08.643 | INFO | ------------------------------------------ | 20260505 11:31:08.643 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:31:08.643 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 11:31:08.643 | INFO | Total mem usage: 15.7 GiB | 20260505 11:31:08.643 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 11:31:08.643 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 11:31:08.643 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 11:31:08.643 | INFO | High Water Breakdown: | 20260505 11:31:08.643 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 11:31:08.643 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 11:31:08.643 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 11:31:08.643 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 11:31:08.643 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 11:31:08.643 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 11:31:08.643 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 11:31:08.643 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 11:31:08.643 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 11:31:08.643 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:31:08.643 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 11:31:08.643 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:31:08.643 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 11:31:08.643 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 11:31:08.643 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 11:31:08.643 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 11:31:08.643 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:31:08.643 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 11:31:08.643 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:31:08.643 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:31:08.643 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:31:08.643 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 11:31:08.643 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 11:31:08.643 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:31:08.643 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 11:31:08.643 | INFO | ------------------------------------------ | 20260505 11:31:08.643 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:31:08.643 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 11:31:08.643 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:31:08.644 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:31:08.644 | INFO | PerfReport: Full Execution -- 3.22e+01 / 7.76e+01, 1.40e+01 / 1.60e+00 | 20260505 11:31:08.644 | INFO | PerfReport: 0.29%, AggregateStats -- 9.30e-02 / 1.01e-01, 1.11e-02 / 4.23e-03 | 20260505 11:31:08.644 | INFO | PerfReport: 11.73%, Baseline -- 3.78e+00 / 4.51e+00, 6.07e-01 / 1.83e-01 | 20260505 11:31:08.644 | INFO | PerfReport: 3.31%, Histogram -- 1.07e+00 / 1.51e+00, 2.08e-01 / 9.30e-02 | 20260505 11:31:08.644 | INFO | PerfReport: 15.79%, DME -- 4.07e+01 / 4.49e+01, 4.27e+00 / 1.91e+00 | 20260505 11:31:08.644 | INFO | PerfReport: 56.51%, FrameLabeling -- 1.82e+01 / 2.17e+01, 1.66e+00 / 1.95e+00 | 20260505 11:31:08.644 | INFO | PerfReport: 7.35%, PulseAccumulating -- 2.37e+00 / 2.84e+00, 4.12e-01 / 2.20e-01 | 20260505 11:31:08.644 | INFO | PerfReport: 3.72%, Metrics -- 1.20e+00 / 1.49e+00, 3.43e-01 / 1.78e-01 | 20260505 11:31:08.644 | INFO | PerfReport: 1.30%, PulseSerializing -- 4.19e-01 / 4.70e-01, 7.75e-02 / 3.72e-02 | 20260505 11:31:08.644 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 11:31:08.644 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:31:08.644 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:31:08.644 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260505 11:31:08.644 | INFO | PerfReport: 22.67%, Upload -- 2.30e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260505 11:31:08.644 | INFO | PerfReport: 22.02%, Download -- 2.23e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260505 11:31:08.644 | INFO | PerfReport: 31.79%, Compute -- 3.22e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260505 11:31:08.644 | INFO | PerfReport: 2.05%, LockUpload -- 2.07e+00 / 0.00e+00, 7.62e+00 / 0.00e+00 | 20260505 11:31:08.644 | INFO | PerfReport: 1.20%, LockDownload -- 1.21e+00 / 0.00e+00, 2.63e+00 / 0.00e+00 | 20260505 11:31:08.644 | INFO | PerfReport: 20.26%, LockCompute -- 2.05e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260505 11:31:08.644 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.52e-02 / 0.00e+00 | 20260505 11:31:08.773 | INFO | Starting a round of Baz IO | 20260505 11:31:12.028 | INFO | Populating RTMetrics report | 20260505 11:31:12.029 | INFO | RTMetrics report populated | 20260505 11:31:12.029 | INFO | Opening tmp RTMetrics json file | 20260505 11:31:12.031 | INFO | Serializing RTMetrics to json | 20260505 11:31:12.033 | INFO | Serialization done, swapping tmp file now... | 20260505 11:31:12.034 | INFO | RTMetrics done | 20260505 11:31:12.179 | INFO | Finished a round of Baz IO: Wrote 2.376 GiB in 3.405 seconds | 20260505 11:31:12.179 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.13% / 0.01% / 75.86% | 20260505 11:31:12.179 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.12608% / 1.58946e-05% / 8.74201e-05% | 20260505 11:31:12.179 | INFO | Baz IO overall: | 20260505 11:31:12.179 | INFO | BytesWritten : 1695889173607 | 20260505 11:31:12.179 | INFO | NumEvents : 807462088092 | 20260505 11:31:13.786 | INFO | Analyzing chunk frames = [2867200,2867712) | 20260505 11:31:13.786 | INFO | MemInfo: { | 20260505 11:31:13.786 | INFO | "gpu" : " 17 GB", | 20260505 11:31:13.786 | INFO | "huge" : "", | 20260505 11:31:13.786 | INFO | "huge_shared" : " 27 GB", | 20260505 11:31:13.786 | INFO | "malloc" : " 14 GB", | 20260505 11:31:13.786 | INFO | "pinned" : " 29 GB" | 20260505 11:31:13.786 | INFO | } | 20260505 11:31:13.786 | INFO | Perf report for the last 5.14406 seconds: | 20260505 11:31:13.786 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:31:13.786 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:13.786 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:13.786 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:13.786 | INFO | ANALYSIS : 64.7%, 2.9, 99.8ms, 0 | 20260505 11:31:13.786 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:13.786 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260505 11:31:13.786 | INFO | PRE_HQ : 61.1%, 1.9, 58.8ms, 0 | 20260505 11:31:13.786 | INFO | BAZWRITER : 65.5%, 4.1, 139.1ms, 0 | 20260505 11:31:13.786 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 11:31:13.787 | INFO | Uploaded 36GiB to GPU in 2.1898s (16.4399GiB/s) | 20260505 11:31:13.787 | INFO | Downloaded 38GiB to GPU in 2.29274s (16.574GiB/s) | 20260505 11:31:18.923 | INFO | Analyzing chunk frames = [2867712,2868224) | 20260505 11:31:18.923 | INFO | MemInfo: { | 20260505 11:31:18.923 | INFO | "gpu" : " 17 GB", | 20260505 11:31:18.923 | INFO | "huge" : "", | 20260505 11:31:18.923 | INFO | "huge_shared" : " 27 GB", | 20260505 11:31:18.923 | INFO | "malloc" : " 14 GB", | 20260505 11:31:18.923 | INFO | "pinned" : " 29 GB" | 20260505 11:31:18.923 | INFO | } | 20260505 11:31:18.924 | INFO | Perf report for the last 5.13736 seconds: | 20260505 11:31:18.924 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:31:18.924 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:18.924 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:18.924 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:18.924 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 11:31:18.924 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:18.924 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:18.924 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:31:18.924 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:18.924 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:18.924 | INFO | Uploaded 36GiB to GPU in 2.18493s (16.4765GiB/s) | 20260505 11:31:18.924 | INFO | Downloaded 34GiB to GPU in 2.05088s (16.5783GiB/s) | 20260505 11:31:24.055 | INFO | Analyzing chunk frames = [2868224,2868736) | 20260505 11:31:24.055 | INFO | MemInfo: { | 20260505 11:31:24.055 | INFO | "gpu" : " 17 GB", | 20260505 11:31:24.055 | INFO | "huge" : "", | 20260505 11:31:24.055 | INFO | "huge_shared" : " 27 GB", | 20260505 11:31:24.055 | INFO | "malloc" : " 14 GB", | 20260505 11:31:24.055 | INFO | "pinned" : " 29 GB" | 20260505 11:31:24.055 | INFO | } | 20260505 11:31:24.055 | INFO | Perf report for the last 5.13137 seconds: | 20260505 11:31:24.055 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:31:24.055 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:24.055 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:24.055 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:24.055 | INFO | ANALYSIS : 63.4%, 3.0, 98.4ms, 0 | 20260505 11:31:24.055 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:24.055 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:24.055 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 11:31:24.055 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:24.055 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:24.055 | INFO | Uploaded 36GiB to GPU in 2.21044s (16.2864GiB/s) | 20260505 11:31:24.055 | INFO | Downloaded 34GiB to GPU in 2.11927s (16.0433GiB/s) | 20260505 11:31:29.195 | INFO | Analyzing chunk frames = [2868736,2869248) | 20260505 11:31:29.195 | INFO | MemInfo: { | 20260505 11:31:29.195 | INFO | "gpu" : " 17 GB", | 20260505 11:31:29.195 | INFO | "huge" : "", | 20260505 11:31:29.195 | INFO | "huge_shared" : " 27 GB", | 20260505 11:31:29.195 | INFO | "malloc" : " 14 GB", | 20260505 11:31:29.195 | INFO | "pinned" : " 29 GB" | 20260505 11:31:29.195 | INFO | } | 20260505 11:31:29.195 | INFO | Perf report for the last 5.14042 seconds: | 20260505 11:31:29.195 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:31:29.195 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:29.195 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:29.195 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:29.196 | INFO | ANALYSIS : 63.0%, 3.0, 97.9ms, 0 | 20260505 11:31:29.196 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:29.196 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:29.196 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 11:31:29.196 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:29.196 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:29.196 | INFO | Uploaded 36GiB to GPU in 2.1676s (16.6082GiB/s) | 20260505 11:31:29.196 | INFO | Downloaded 34GiB to GPU in 2.07832s (16.3594GiB/s) | 20260505 11:31:34.341 | INFO | Analyzing chunk frames = [2869248,2869760) | 20260505 11:31:34.341 | INFO | MemInfo: { | 20260505 11:31:34.341 | INFO | "gpu" : " 17 GB", | 20260505 11:31:34.341 | INFO | "huge" : "", | 20260505 11:31:34.341 | INFO | "huge_shared" : " 27 GB", | 20260505 11:31:34.341 | INFO | "malloc" : " 14 GB", | 20260505 11:31:34.341 | INFO | "pinned" : " 29 GB" | 20260505 11:31:34.341 | INFO | } | 20260505 11:31:34.341 | INFO | Perf report for the last 5.14574 seconds: | 20260505 11:31:34.341 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:31:34.341 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:34.341 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:34.341 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:34.341 | INFO | ANALYSIS : 62.9%, 3.0, 98.4ms, 0 | 20260505 11:31:34.341 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:34.341 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:34.341 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 11:31:34.341 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:34.341 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:34.341 | INFO | Uploaded 36GiB to GPU in 2.16803s (16.6049GiB/s) | 20260505 11:31:34.341 | INFO | Downloaded 34GiB to GPU in 2.1086s (16.1245GiB/s) | 20260505 11:31:39.486 | INFO | Analyzing chunk frames = [2869760,2870272) | 20260505 11:31:39.486 | INFO | MemInfo: { | 20260505 11:31:39.486 | INFO | "gpu" : " 17 GB", | 20260505 11:31:39.486 | INFO | "huge" : "", | 20260505 11:31:39.487 | INFO | "huge_shared" : " 27 GB", | 20260505 11:31:39.487 | INFO | "malloc" : " 14 GB", | 20260505 11:31:39.487 | INFO | "pinned" : " 29 GB" | 20260505 11:31:39.487 | INFO | } | 20260505 11:31:39.487 | INFO | Perf report for the last 5.14539 seconds: | 20260505 11:31:39.487 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:31:39.487 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:39.487 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:39.487 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:39.487 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 11:31:39.487 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:39.487 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:39.487 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 11:31:39.487 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:39.487 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:39.487 | INFO | Uploaded 36GiB to GPU in 2.15388s (16.714GiB/s) | 20260505 11:31:39.487 | INFO | Downloaded 34GiB to GPU in 2.11519s (16.0742GiB/s) | 20260505 11:31:44.619 | INFO | Analyzing chunk frames = [2870272,2870784) | 20260505 11:31:44.619 | INFO | MemInfo: { | 20260505 11:31:44.619 | INFO | "gpu" : " 17 GB", | 20260505 11:31:44.619 | INFO | "huge" : "", | 20260505 11:31:44.619 | INFO | "huge_shared" : " 27 GB", | 20260505 11:31:44.619 | INFO | "malloc" : " 14 GB", | 20260505 11:31:44.619 | INFO | "pinned" : " 29 GB" | 20260505 11:31:44.619 | INFO | } | 20260505 11:31:44.619 | INFO | Perf report for the last 5.13229 seconds: | 20260505 11:31:44.619 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:31:44.619 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:44.619 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:44.619 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:44.619 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 11:31:44.619 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:44.619 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:44.619 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 11:31:44.619 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:44.619 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:44.619 | INFO | Uploaded 36GiB to GPU in 2.17133s (16.5797GiB/s) | 20260505 11:31:44.620 | INFO | Downloaded 34GiB to GPU in 2.14977s (15.8156GiB/s) | 20260505 11:31:49.753 | INFO | Analyzing chunk frames = [2870784,2871296) | 20260505 11:31:49.753 | INFO | MemInfo: { | 20260505 11:31:49.753 | INFO | "gpu" : " 17 GB", | 20260505 11:31:49.753 | INFO | "huge" : "", | 20260505 11:31:49.753 | INFO | "huge_shared" : " 27 GB", | 20260505 11:31:49.753 | INFO | "malloc" : " 14 GB", | 20260505 11:31:49.753 | INFO | "pinned" : " 29 GB" | 20260505 11:31:49.753 | INFO | } | 20260505 11:31:49.753 | INFO | Perf report for the last 5.13405 seconds: | 20260505 11:31:49.753 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:31:49.753 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:49.753 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:49.753 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:49.753 | INFO | ANALYSIS : 64.0%, 3.0, 99.8ms, 0 | 20260505 11:31:49.753 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:49.753 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:49.753 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 11:31:49.753 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:49.753 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:49.753 | INFO | Uploaded 36GiB to GPU in 2.22739s (16.1624GiB/s) | 20260505 11:31:49.753 | INFO | Downloaded 34GiB to GPU in 2.09347s (16.241GiB/s) | 20260505 11:31:49.890 | INFO | Starting a round of Baz IO | 20260505 11:31:53.143 | INFO | Populating RTMetrics report | 20260505 11:31:53.143 | INFO | RTMetrics report populated | 20260505 11:31:53.143 | INFO | Opening tmp RTMetrics json file | 20260505 11:31:53.174 | INFO | Serializing RTMetrics to json | 20260505 11:31:53.176 | INFO | Serialization done, swapping tmp file now... | 20260505 11:31:53.176 | INFO | RTMetrics done | 20260505 11:31:53.268 | INFO | Finished a round of Baz IO: Wrote 2.377 GiB in 3.378 seconds | 20260505 11:31:53.268 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.12% / 0.01% / 75.87% | 20260505 11:31:53.268 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.12381% / 7.94729e-06% / 9.53674e-05% | 20260505 11:31:53.268 | INFO | Baz IO overall: | 20260505 11:31:53.268 | INFO | BytesWritten : 1698440964098 | 20260505 11:31:53.268 | INFO | NumEvents : 808675533159 | 20260505 11:31:54.902 | INFO | Analyzing chunk frames = [2871296,2871808) | 20260505 11:31:54.902 | INFO | MemInfo: { | 20260505 11:31:54.902 | INFO | "gpu" : " 17 GB", | 20260505 11:31:54.902 | INFO | "huge" : "", | 20260505 11:31:54.902 | INFO | "huge_shared" : " 27 GB", | 20260505 11:31:54.902 | INFO | "malloc" : " 14 GB", | 20260505 11:31:54.902 | INFO | "pinned" : " 29 GB" | 20260505 11:31:54.902 | INFO | } | 20260505 11:31:54.902 | INFO | Perf report for the last 5.14928 seconds: | 20260505 11:31:54.902 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:31:54.902 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:54.902 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:54.902 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:54.902 | INFO | ANALYSIS : 64.7%, 2.9, 100.0ms, 0 | 20260505 11:31:54.902 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:31:54.902 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 11:31:54.902 | INFO | PRE_HQ : 61.1%, 1.8, 58.8ms, 0 | 20260505 11:31:54.902 | INFO | BAZWRITER : 65.0%, 4.1, 135.7ms, 0 | 20260505 11:31:54.902 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 11:31:54.903 | INFO | Uploaded 36GiB to GPU in 2.19853s (16.3746GiB/s) | 20260505 11:31:54.903 | INFO | Downloaded 38GiB to GPU in 2.30243s (16.5043GiB/s) | 20260505 11:32:00.034 | INFO | Analyzing chunk frames = [2871808,2872320) | 20260505 11:32:00.034 | INFO | MemInfo: { | 20260505 11:32:00.034 | INFO | "gpu" : " 17 GB", | 20260505 11:32:00.034 | INFO | "huge" : "", | 20260505 11:32:00.034 | INFO | "huge_shared" : " 27 GB", | 20260505 11:32:00.034 | INFO | "malloc" : " 14 GB", | 20260505 11:32:00.034 | INFO | "pinned" : " 29 GB" | 20260505 11:32:00.034 | INFO | } | 20260505 11:32:00.034 | INFO | Perf report for the last 5.13208 seconds: | 20260505 11:32:00.034 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:32:00.034 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:00.034 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:00.034 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:00.034 | INFO | ANALYSIS : 63.8%, 3.0, 99.4ms, 0 | 20260505 11:32:00.034 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:00.034 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:00.034 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 11:32:00.034 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:00.034 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:00.035 | INFO | Uploaded 36GiB to GPU in 2.1526s (16.724GiB/s) | 20260505 11:32:00.035 | INFO | Downloaded 34GiB to GPU in 2.03527s (16.7054GiB/s) | 20260505 11:32:05.177 | INFO | Analyzing chunk frames = [2872320,2872832) | 20260505 11:32:05.177 | INFO | MemInfo: { | 20260505 11:32:05.177 | INFO | "gpu" : " 17 GB", | 20260505 11:32:05.177 | INFO | "huge" : "", | 20260505 11:32:05.177 | INFO | "huge_shared" : " 27 GB", | 20260505 11:32:05.177 | INFO | "malloc" : " 14 GB", | 20260505 11:32:05.177 | INFO | "pinned" : " 29 GB" | 20260505 11:32:05.177 | INFO | } | 20260505 11:32:05.177 | INFO | Perf report for the last 5.14259 seconds: | 20260505 11:32:05.177 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:32:05.177 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:05.177 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:05.177 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:05.177 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260505 11:32:05.177 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:05.177 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:05.177 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:32:05.177 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:05.177 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:05.177 | INFO | Uploaded 36GiB to GPU in 2.22618s (16.1712GiB/s) | 20260505 11:32:05.177 | INFO | Downloaded 34GiB to GPU in 2.12995s (15.9628GiB/s) | 20260505 11:32:10.309 | INFO | Analyzing chunk frames = [2872832,2873344) | 20260505 11:32:10.309 | INFO | MemInfo: { | 20260505 11:32:10.309 | INFO | "gpu" : " 17 GB", | 20260505 11:32:10.309 | INFO | "huge" : "", | 20260505 11:32:10.309 | INFO | "huge_shared" : " 27 GB", | 20260505 11:32:10.309 | INFO | "malloc" : " 14 GB", | 20260505 11:32:10.309 | INFO | "pinned" : " 29 GB" | 20260505 11:32:10.309 | INFO | } | 20260505 11:32:10.309 | INFO | Perf report for the last 5.13236 seconds: | 20260505 11:32:10.309 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:32:10.309 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:10.309 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:10.309 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:10.309 | INFO | ANALYSIS : 63.4%, 3.0, 98.3ms, 0 | 20260505 11:32:10.309 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:10.309 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:10.309 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 11:32:10.309 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:10.309 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:10.310 | INFO | Uploaded 36GiB to GPU in 2.20946s (16.2936GiB/s) | 20260505 11:32:10.310 | INFO | Downloaded 34GiB to GPU in 2.13043s (15.9593GiB/s) | 20260505 11:32:15.454 | INFO | Analyzing chunk frames = [2873344,2873856) | 20260505 11:32:15.454 | INFO | MemInfo: { | 20260505 11:32:15.454 | INFO | "gpu" : " 17 GB", | 20260505 11:32:15.454 | INFO | "huge" : "", | 20260505 11:32:15.454 | INFO | "huge_shared" : " 27 GB", | 20260505 11:32:15.454 | INFO | "malloc" : " 14 GB", | 20260505 11:32:15.454 | INFO | "pinned" : " 29 GB" | 20260505 11:32:15.454 | INFO | } | 20260505 11:32:15.454 | INFO | Perf report for the last 5.14443 seconds: | 20260505 11:32:15.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:32:15.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:15.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:15.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:15.454 | INFO | ANALYSIS : 63.2%, 3.0, 98.8ms, 0 | 20260505 11:32:15.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:15.454 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:15.454 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:32:15.454 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:15.454 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:15.454 | INFO | Uploaded 36GiB to GPU in 2.22792s (16.1586GiB/s) | 20260505 11:32:15.454 | INFO | Downloaded 34GiB to GPU in 2.11067s (16.1087GiB/s) | 20260505 11:32:20.592 | INFO | Analyzing chunk frames = [2873856,2874368) | 20260505 11:32:20.592 | INFO | MemInfo: { | 20260505 11:32:20.592 | INFO | "gpu" : " 17 GB", | 20260505 11:32:20.592 | INFO | "huge" : "", | 20260505 11:32:20.592 | INFO | "huge_shared" : " 27 GB", | 20260505 11:32:20.592 | INFO | "malloc" : " 14 GB", | 20260505 11:32:20.592 | INFO | "pinned" : " 29 GB" | 20260505 11:32:20.592 | INFO | } | 20260505 11:32:20.592 | INFO | Perf report for the last 5.13864 seconds: | 20260505 11:32:20.593 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:32:20.593 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:20.593 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:20.593 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:20.593 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 11:32:20.593 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:20.593 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:20.593 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 11:32:20.593 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:20.593 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:20.593 | INFO | Uploaded 36GiB to GPU in 2.14996s (16.7445GiB/s) | 20260505 11:32:20.593 | INFO | Downloaded 34GiB to GPU in 2.10859s (16.1245GiB/s) | 20260505 11:32:25.736 | INFO | Analyzing chunk frames = [2874368,2874880) | 20260505 11:32:25.736 | INFO | MemInfo: { | 20260505 11:32:25.736 | INFO | "gpu" : " 17 GB", | 20260505 11:32:25.736 | INFO | "huge" : "", | 20260505 11:32:25.736 | INFO | "huge_shared" : " 27 GB", | 20260505 11:32:25.736 | INFO | "malloc" : " 14 GB", | 20260505 11:32:25.736 | INFO | "pinned" : " 29 GB" | 20260505 11:32:25.736 | INFO | } | 20260505 11:32:25.736 | INFO | Perf report for the last 5.1439 seconds: | 20260505 11:32:25.736 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:32:25.736 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:25.736 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:25.736 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:25.736 | INFO | ANALYSIS : 63.5%, 3.0, 99.3ms, 0 | 20260505 11:32:25.736 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:25.736 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:25.736 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 11:32:25.736 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:25.736 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:25.737 | INFO | Uploaded 36GiB to GPU in 2.17647s (16.5405GiB/s) | 20260505 11:32:25.737 | INFO | Downloaded 34GiB to GPU in 2.14868s (15.8236GiB/s) | 20260505 11:32:30.868 | INFO | Analyzing chunk frames = [2874880,2875392) | 20260505 11:32:30.868 | INFO | MemInfo: { | 20260505 11:32:30.868 | INFO | "gpu" : " 17 GB", | 20260505 11:32:30.868 | INFO | "huge" : "", | 20260505 11:32:30.868 | INFO | "huge_shared" : " 27 GB", | 20260505 11:32:30.868 | INFO | "malloc" : " 14 GB", | 20260505 11:32:30.868 | INFO | "pinned" : " 29 GB" | 20260505 11:32:30.868 | INFO | } | 20260505 11:32:30.868 | INFO | Perf report for the last 5.13198 seconds: | 20260505 11:32:30.868 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:32:30.868 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:30.868 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:30.868 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:30.868 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 11:32:30.868 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:30.868 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:30.868 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 11:32:30.868 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:30.868 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:30.869 | INFO | Uploaded 36GiB to GPU in 2.21202s (16.2747GiB/s) | 20260505 11:32:30.869 | INFO | Downloaded 34GiB to GPU in 2.09801s (16.2059GiB/s) | 20260505 11:32:30.869 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 11:32:30.869 | INFO | Total mem usage: 15.3 GiB | 20260505 11:32:30.869 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 11:32:30.869 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.79%) | 20260505 11:32:30.869 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 11:32:30.869 | INFO | High Water Breakdown: | 20260505 11:32:30.869 | INFO | 61.1% : 8.76 GiB ( 92.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 11:32:30.869 | INFO | 38.8% : 5.57 GiB ( 83.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 11:32:30.869 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 11:32:30.869 | INFO | ------------------------------------------ | 20260505 11:32:30.869 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:32:30.869 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 11:32:30.869 | INFO | Total mem usage: 26.5 GiB | 20260505 11:32:30.869 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 11:32:30.869 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 11:32:30.869 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 11:32:30.869 | INFO | High Water Breakdown: | 20260505 11:32:30.869 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:32:30.869 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:32:30.869 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:32:30.869 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:32:30.869 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:32:30.869 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:32:30.869 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 11:32:30.869 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 11:32:30.869 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:32:30.869 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 11:32:30.869 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 11:32:30.869 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 11:32:30.869 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 11:32:30.869 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 11:32:30.869 | INFO | ------------------------------------------ | 20260505 11:32:30.869 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:32:30.869 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 11:32:30.869 | INFO | Total mem usage: 25.6 GiB | 20260505 11:32:30.869 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 11:32:30.869 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 11:32:30.869 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 11:32:30.869 | INFO | High Water Breakdown: | 20260505 11:32:30.869 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 11:32:30.869 | INFO | ------------------------------------------ | 20260505 11:32:30.869 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:32:30.869 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 11:32:30.869 | INFO | Total mem usage: 15.7 GiB | 20260505 11:32:30.869 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 11:32:30.869 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 11:32:30.870 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 11:32:30.870 | INFO | High Water Breakdown: | 20260505 11:32:30.870 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 11:32:30.870 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 11:32:30.870 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 11:32:30.870 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 11:32:30.870 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 11:32:30.870 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 11:32:30.870 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 11:32:30.870 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 11:32:30.870 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 11:32:30.870 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:32:30.870 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 11:32:30.870 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:32:30.870 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 11:32:30.870 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 11:32:30.870 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 11:32:30.870 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 11:32:30.870 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:32:30.870 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 11:32:30.870 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:32:30.870 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:32:30.870 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:32:30.870 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 11:32:30.870 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 11:32:30.870 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:32:30.870 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 11:32:30.870 | INFO | ------------------------------------------ | 20260505 11:32:30.870 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:32:30.870 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 11:32:30.870 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:32:30.870 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:32:30.870 | INFO | PerfReport: Full Execution -- 3.22e+01 / 7.71e+01, 1.39e+01 / 1.66e+00 | 20260505 11:32:30.870 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 1.01e-01, 1.03e-02 / 3.46e-03 | 20260505 11:32:30.870 | INFO | PerfReport: 11.77%, Baseline -- 3.79e+00 / 4.50e+00, 6.14e-01 / 2.13e-01 | 20260505 11:32:30.870 | INFO | PerfReport: 3.31%, Histogram -- 1.06e+00 / 1.49e+00, 2.04e-01 / 1.03e-01 | 20260505 11:32:30.870 | INFO | PerfReport: 15.72%, DME -- 4.05e+01 / 4.46e+01, 4.36e+00 / 1.87e+00 | 20260505 11:32:30.870 | INFO | PerfReport: 56.54%, FrameLabeling -- 1.82e+01 / 2.15e+01, 1.68e+00 / 1.92e+00 | 20260505 11:32:30.870 | INFO | PerfReport: 7.39%, PulseAccumulating -- 2.38e+00 / 2.85e+00, 4.20e-01 / 2.18e-01 | 20260505 11:32:30.870 | INFO | PerfReport: 3.69%, Metrics -- 1.19e+00 / 1.50e+00, 3.36e-01 / 1.79e-01 | 20260505 11:32:30.870 | INFO | PerfReport: 1.30%, PulseSerializing -- 4.18e-01 / 4.73e-01, 7.71e-02 / 3.61e-02 | 20260505 11:32:30.870 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 11:32:30.870 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:32:30.870 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:32:30.870 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.47e+01 / 0.00e+00 | 20260505 11:32:30.870 | INFO | PerfReport: 22.55%, Upload -- 2.28e+01 / 0.00e+00, 8.08e+00 / 0.00e+00 | 20260505 11:32:30.870 | INFO | PerfReport: 21.94%, Download -- 2.22e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260505 11:32:30.870 | INFO | PerfReport: 31.82%, Compute -- 3.22e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 11:32:30.870 | INFO | PerfReport: 1.96%, LockUpload -- 1.98e+00 / 0.00e+00, 7.61e+00 / 0.00e+00 | 20260505 11:32:30.870 | INFO | PerfReport: 1.30%, LockDownload -- 1.31e+00 / 0.00e+00, 2.84e+00 / 0.00e+00 | 20260505 11:32:30.870 | INFO | PerfReport: 20.42%, LockCompute -- 2.07e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260505 11:32:30.870 | INFO | PerfReport: 0.02%, Misc -- 1.62e-02 / 0.00e+00, 4.06e-02 / 0.00e+00 | 20260505 11:32:31.008 | INFO | Starting a round of Baz IO | 20260505 11:32:34.263 | INFO | Populating RTMetrics report | 20260505 11:32:34.263 | INFO | RTMetrics report populated | 20260505 11:32:34.263 | INFO | Opening tmp RTMetrics json file | 20260505 11:32:34.267 | INFO | Serializing RTMetrics to json | 20260505 11:32:34.271 | INFO | Serialization done, swapping tmp file now... | 20260505 11:32:34.271 | INFO | RTMetrics done | 20260505 11:32:34.398 | INFO | Finished a round of Baz IO: Wrote 2.376 GiB in 3.39 seconds | 20260505 11:32:34.398 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.12% / 0.01% / 75.88% | 20260505 11:32:34.398 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.11816% / 3.97364e-06% / 9.93411e-05% | 20260505 11:32:34.398 | INFO | Baz IO overall: | 20260505 11:32:34.398 | INFO | BytesWritten : 1700992595708 | 20260505 11:32:34.398 | INFO | NumEvents : 809888767559 | 20260505 11:32:36.006 | INFO | Analyzing chunk frames = [2875392,2875904) | 20260505 11:32:36.006 | INFO | MemInfo: { | 20260505 11:32:36.006 | INFO | "gpu" : " 17 GB", | 20260505 11:32:36.006 | INFO | "huge" : "", | 20260505 11:32:36.006 | INFO | "huge_shared" : " 27 GB", | 20260505 11:32:36.006 | INFO | "malloc" : " 14 GB", | 20260505 11:32:36.006 | INFO | "pinned" : " 29 GB" | 20260505 11:32:36.006 | INFO | } | 20260505 11:32:36.006 | INFO | Perf report for the last 5.13732 seconds: | 20260505 11:32:36.006 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:32:36.006 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:36.006 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:36.006 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:36.006 | INFO | ANALYSIS : 65.0%, 2.9, 100.3ms, 0 | 20260505 11:32:36.006 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:36.006 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260505 11:32:36.006 | INFO | PRE_HQ : 61.4%, 1.9, 59.0ms, 0 | 20260505 11:32:36.006 | INFO | BAZWRITER : 65.4%, 4.1, 137.6ms, 0 | 20260505 11:32:36.006 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 11:32:36.006 | INFO | Uploaded 36GiB to GPU in 2.19869s (16.3734GiB/s) | 20260505 11:32:36.006 | INFO | Downloaded 38GiB to GPU in 2.30069s (16.5168GiB/s) | 20260505 11:32:41.152 | INFO | Analyzing chunk frames = [2875904,2876416) | 20260505 11:32:41.152 | INFO | MemInfo: { | 20260505 11:32:41.152 | INFO | "gpu" : " 17 GB", | 20260505 11:32:41.152 | INFO | "huge" : "", | 20260505 11:32:41.152 | INFO | "huge_shared" : " 27 GB", | 20260505 11:32:41.152 | INFO | "malloc" : " 14 GB", | 20260505 11:32:41.152 | INFO | "pinned" : " 29 GB" | 20260505 11:32:41.152 | INFO | } | 20260505 11:32:41.153 | INFO | Perf report for the last 5.14685 seconds: | 20260505 11:32:41.153 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:32:41.153 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:41.153 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:41.153 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:41.153 | INFO | ANALYSIS : 64.1%, 3.0, 100.2ms, 0 | 20260505 11:32:41.153 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:41.153 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:41.153 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:32:41.153 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:41.153 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:41.153 | INFO | Uploaded 36GiB to GPU in 2.23375s (16.1164GiB/s) | 20260505 11:32:41.153 | INFO | Downloaded 34GiB to GPU in 2.09123s (16.2584GiB/s) | 20260505 11:32:46.292 | INFO | Analyzing chunk frames = [2876416,2876928) | 20260505 11:32:46.292 | INFO | MemInfo: { | 20260505 11:32:46.292 | INFO | "gpu" : " 17 GB", | 20260505 11:32:46.292 | INFO | "huge" : "", | 20260505 11:32:46.292 | INFO | "huge_shared" : " 27 GB", | 20260505 11:32:46.292 | INFO | "malloc" : " 14 GB", | 20260505 11:32:46.292 | INFO | "pinned" : " 29 GB" | 20260505 11:32:46.292 | INFO | } | 20260505 11:32:46.293 | INFO | Perf report for the last 5.14002 seconds: | 20260505 11:32:46.293 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:32:46.293 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:46.293 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:46.293 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:46.293 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 11:32:46.293 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:46.293 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:46.293 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 11:32:46.293 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:46.293 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:46.293 | INFO | Uploaded 36GiB to GPU in 2.17233s (16.5721GiB/s) | 20260505 11:32:46.293 | INFO | Downloaded 34GiB to GPU in 2.11664s (16.0632GiB/s) | 20260505 11:32:51.439 | INFO | Analyzing chunk frames = [2876928,2877440) | 20260505 11:32:51.439 | INFO | MemInfo: { | 20260505 11:32:51.439 | INFO | "gpu" : " 17 GB", | 20260505 11:32:51.439 | INFO | "huge" : "", | 20260505 11:32:51.439 | INFO | "huge_shared" : " 27 GB", | 20260505 11:32:51.439 | INFO | "malloc" : " 14 GB", | 20260505 11:32:51.439 | INFO | "pinned" : " 29 GB" | 20260505 11:32:51.439 | INFO | } | 20260505 11:32:51.439 | INFO | Perf report for the last 5.14634 seconds: | 20260505 11:32:51.439 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:32:51.439 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:51.439 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:51.439 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:51.439 | INFO | ANALYSIS : 63.1%, 3.0, 98.2ms, 0 | 20260505 11:32:51.439 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:51.439 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:51.439 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 11:32:51.439 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:51.439 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:51.439 | INFO | Uploaded 36GiB to GPU in 2.20789s (16.3052GiB/s) | 20260505 11:32:51.439 | INFO | Downloaded 34GiB to GPU in 2.13345s (15.9366GiB/s) | 20260505 11:32:56.567 | INFO | Analyzing chunk frames = [2877440,2877952) | 20260505 11:32:56.567 | INFO | MemInfo: { | 20260505 11:32:56.567 | INFO | "gpu" : " 17 GB", | 20260505 11:32:56.567 | INFO | "huge" : "", | 20260505 11:32:56.567 | INFO | "huge_shared" : " 27 GB", | 20260505 11:32:56.567 | INFO | "malloc" : " 14 GB", | 20260505 11:32:56.567 | INFO | "pinned" : " 29 GB" | 20260505 11:32:56.567 | INFO | } | 20260505 11:32:56.567 | INFO | Perf report for the last 5.12828 seconds: | 20260505 11:32:56.567 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:32:56.567 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:56.567 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:56.567 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:56.567 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 11:32:56.567 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:56.567 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:56.567 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:32:56.567 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:56.567 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:32:56.567 | INFO | Uploaded 36GiB to GPU in 2.15377s (16.7149GiB/s) | 20260505 11:32:56.567 | INFO | Downloaded 34GiB to GPU in 2.12218s (16.0213GiB/s) | 20260505 11:33:01.705 | INFO | Analyzing chunk frames = [2877952,2878464) | 20260505 11:33:01.705 | INFO | MemInfo: { | 20260505 11:33:01.705 | INFO | "gpu" : " 17 GB", | 20260505 11:33:01.705 | INFO | "huge" : "", | 20260505 11:33:01.705 | INFO | "huge_shared" : " 27 GB", | 20260505 11:33:01.705 | INFO | "malloc" : " 14 GB", | 20260505 11:33:01.705 | INFO | "pinned" : " 29 GB" | 20260505 11:33:01.705 | INFO | } | 20260505 11:33:01.705 | INFO | Perf report for the last 5.13755 seconds: | 20260505 11:33:01.705 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:33:01.705 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:01.705 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:01.705 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:01.705 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 11:33:01.705 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:01.705 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:01.705 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:33:01.705 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:01.705 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:01.706 | INFO | Uploaded 36GiB to GPU in 2.19005s (16.438GiB/s) | 20260505 11:33:01.706 | INFO | Downloaded 34GiB to GPU in 2.10661s (16.1397GiB/s) | 20260505 11:33:06.852 | INFO | Analyzing chunk frames = [2878464,2878976) | 20260505 11:33:06.852 | INFO | MemInfo: { | 20260505 11:33:06.852 | INFO | "gpu" : " 17 GB", | 20260505 11:33:06.852 | INFO | "huge" : "", | 20260505 11:33:06.852 | INFO | "huge_shared" : " 27 GB", | 20260505 11:33:06.852 | INFO | "malloc" : " 14 GB", | 20260505 11:33:06.852 | INFO | "pinned" : " 29 GB" | 20260505 11:33:06.852 | INFO | } | 20260505 11:33:06.852 | INFO | Perf report for the last 5.14743 seconds: | 20260505 11:33:06.852 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:33:06.852 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:06.852 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:06.852 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:06.852 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 11:33:06.852 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:06.852 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:06.852 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 11:33:06.852 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:06.852 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:06.852 | INFO | Uploaded 36GiB to GPU in 2.17433s (16.5569GiB/s) | 20260505 11:33:06.852 | INFO | Downloaded 34GiB to GPU in 2.13338s (15.9371GiB/s) | 20260505 11:33:11.984 | INFO | Analyzing chunk frames = [2878976,2879488) | 20260505 11:33:11.984 | INFO | MemInfo: { | 20260505 11:33:11.984 | INFO | "gpu" : " 17 GB", | 20260505 11:33:11.984 | INFO | "huge" : "", | 20260505 11:33:11.984 | INFO | "huge_shared" : " 27 GB", | 20260505 11:33:11.984 | INFO | "malloc" : " 14 GB", | 20260505 11:33:11.984 | INFO | "pinned" : " 29 GB" | 20260505 11:33:11.984 | INFO | } | 20260505 11:33:11.984 | INFO | Perf report for the last 5.13179 seconds: | 20260505 11:33:11.984 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:33:11.984 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:11.984 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:11.984 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:11.984 | INFO | ANALYSIS : 63.9%, 3.0, 99.6ms, 0 | 20260505 11:33:11.984 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:11.984 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:11.984 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 11:33:11.984 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:11.984 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:11.984 | INFO | Uploaded 36GiB to GPU in 2.18054s (16.5097GiB/s) | 20260505 11:33:11.984 | INFO | Downloaded 34GiB to GPU in 2.11182s (16.0999GiB/s) | 20260505 11:33:12.126 | INFO | Starting a round of Baz IO | 20260505 11:33:15.390 | INFO | Populating RTMetrics report | 20260505 11:33:15.390 | INFO | RTMetrics report populated | 20260505 11:33:15.390 | INFO | Opening tmp RTMetrics json file | 20260505 11:33:15.416 | INFO | Serializing RTMetrics to json | 20260505 11:33:15.418 | INFO | Serialization done, swapping tmp file now... | 20260505 11:33:15.418 | INFO | RTMetrics done | 20260505 11:33:15.521 | INFO | Finished a round of Baz IO: Wrote 2.377 GiB in 3.395 seconds | 20260505 11:33:15.521 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.11% / 0.01% / 75.88% | 20260505 11:33:15.522 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.11814% / 3.97364e-06% / 7.15256e-05% | 20260505 11:33:15.522 | INFO | Baz IO overall: | 20260505 11:33:15.522 | INFO | BytesWritten : 1703544421517 | 20260505 11:33:15.522 | INFO | NumEvents : 811102124001 | 20260505 11:33:17.119 | INFO | Analyzing chunk frames = [2879488,2880000) | 20260505 11:33:17.119 | INFO | MemInfo: { | 20260505 11:33:17.119 | INFO | "gpu" : " 17 GB", | 20260505 11:33:17.119 | INFO | "huge" : "", | 20260505 11:33:17.119 | INFO | "huge_shared" : " 27 GB", | 20260505 11:33:17.119 | INFO | "malloc" : " 14 GB", | 20260505 11:33:17.119 | INFO | "pinned" : " 29 GB" | 20260505 11:33:17.119 | INFO | } | 20260505 11:33:17.120 | INFO | Perf report for the last 5.13556 seconds: | 20260505 11:33:17.120 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:33:17.120 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:17.120 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:17.120 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:17.120 | INFO | ANALYSIS : 65.2%, 2.9, 100.2ms, 0 | 20260505 11:33:17.120 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:17.120 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 11:33:17.120 | INFO | PRE_HQ : 61.5%, 1.8, 59.0ms, 0 | 20260505 11:33:17.120 | INFO | BAZWRITER : 65.5%, 4.1, 139.2ms, 0 | 20260505 11:33:17.120 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 11:33:17.120 | INFO | Uploaded 36GiB to GPU in 2.19266s (16.4184GiB/s) | 20260505 11:33:17.120 | INFO | Downloaded 38GiB to GPU in 2.31256s (16.432GiB/s) | 20260505 11:33:22.260 | INFO | Analyzing chunk frames = [2880000,2880512) | 20260505 11:33:22.260 | INFO | MemInfo: { | 20260505 11:33:22.260 | INFO | "gpu" : " 17 GB", | 20260505 11:33:22.260 | INFO | "huge" : "", | 20260505 11:33:22.260 | INFO | "huge_shared" : " 27 GB", | 20260505 11:33:22.260 | INFO | "malloc" : " 14 GB", | 20260505 11:33:22.260 | INFO | "pinned" : " 29 GB" | 20260505 11:33:22.260 | INFO | } | 20260505 11:33:22.260 | INFO | Perf report for the last 5.1407 seconds: | 20260505 11:33:22.260 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:33:22.260 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:22.260 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:22.260 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:22.260 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 11:33:22.260 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:22.260 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:22.260 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 11:33:22.260 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:22.260 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:22.261 | INFO | Uploaded 36GiB to GPU in 2.23405s (16.1142GiB/s) | 20260505 11:33:22.261 | INFO | Downloaded 34GiB to GPU in 2.08674s (16.2934GiB/s) | 20260505 11:33:27.409 | INFO | Analyzing chunk frames = [2880512,2881024) | 20260505 11:33:27.409 | INFO | MemInfo: { | 20260505 11:33:27.409 | INFO | "gpu" : " 17 GB", | 20260505 11:33:27.409 | INFO | "huge" : "", | 20260505 11:33:27.409 | INFO | "huge_shared" : " 27 GB", | 20260505 11:33:27.409 | INFO | "malloc" : " 14 GB", | 20260505 11:33:27.409 | INFO | "pinned" : " 29 GB" | 20260505 11:33:27.409 | INFO | } | 20260505 11:33:27.409 | INFO | Perf report for the last 5.14879 seconds: | 20260505 11:33:27.409 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:33:27.409 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:27.409 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:27.409 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:27.409 | INFO | ANALYSIS : 63.5%, 3.0, 99.0ms, 0 | 20260505 11:33:27.409 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:27.409 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:27.409 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 11:33:27.409 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:27.409 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:27.409 | INFO | Uploaded 36GiB to GPU in 2.22204s (16.2013GiB/s) | 20260505 11:33:27.409 | INFO | Downloaded 34GiB to GPU in 2.12697s (15.9852GiB/s) | 20260505 11:33:32.548 | INFO | Analyzing chunk frames = [2881024,2881536) | 20260505 11:33:32.548 | INFO | MemInfo: { | 20260505 11:33:32.548 | INFO | "gpu" : " 17 GB", | 20260505 11:33:32.548 | INFO | "huge" : "", | 20260505 11:33:32.548 | INFO | "huge_shared" : " 27 GB", | 20260505 11:33:32.548 | INFO | "malloc" : " 14 GB", | 20260505 11:33:32.548 | INFO | "pinned" : " 29 GB" | 20260505 11:33:32.548 | INFO | } | 20260505 11:33:32.548 | INFO | Perf report for the last 5.13895 seconds: | 20260505 11:33:32.548 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:33:32.548 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:32.548 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:32.548 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:32.548 | INFO | ANALYSIS : 63.3%, 3.0, 98.3ms, 0 | 20260505 11:33:32.548 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:32.548 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:32.548 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 11:33:32.548 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:32.548 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:32.548 | INFO | Uploaded 36GiB to GPU in 2.20875s (16.2988GiB/s) | 20260505 11:33:32.548 | INFO | Downloaded 34GiB to GPU in 2.13183s (15.9488GiB/s) | 20260505 11:33:37.683 | INFO | Analyzing chunk frames = [2881536,2882048) | 20260505 11:33:37.683 | INFO | MemInfo: { | 20260505 11:33:37.683 | INFO | "gpu" : " 17 GB", | 20260505 11:33:37.683 | INFO | "huge" : "", | 20260505 11:33:37.683 | INFO | "huge_shared" : " 27 GB", | 20260505 11:33:37.683 | INFO | "malloc" : " 14 GB", | 20260505 11:33:37.683 | INFO | "pinned" : " 29 GB" | 20260505 11:33:37.683 | INFO | } | 20260505 11:33:37.683 | INFO | Perf report for the last 5.13511 seconds: | 20260505 11:33:37.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:33:37.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:37.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:37.683 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:37.683 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 11:33:37.683 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:37.683 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:37.683 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 11:33:37.683 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:37.683 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:37.683 | INFO | Uploaded 36GiB to GPU in 2.18969s (16.4407GiB/s) | 20260505 11:33:37.683 | INFO | Downloaded 34GiB to GPU in 2.12496s (16.0003GiB/s) | 20260505 11:33:42.816 | INFO | Analyzing chunk frames = [2882048,2882560) | 20260505 11:33:42.816 | INFO | MemInfo: { | 20260505 11:33:42.816 | INFO | "gpu" : " 17 GB", | 20260505 11:33:42.816 | INFO | "huge" : "", | 20260505 11:33:42.816 | INFO | "huge_shared" : " 27 GB", | 20260505 11:33:42.816 | INFO | "malloc" : " 14 GB", | 20260505 11:33:42.816 | INFO | "pinned" : " 29 GB" | 20260505 11:33:42.816 | INFO | } | 20260505 11:33:42.816 | INFO | Perf report for the last 5.13266 seconds: | 20260505 11:33:42.816 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:33:42.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:42.816 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:42.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:42.816 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 11:33:42.816 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:42.816 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:42.816 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 11:33:42.816 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:42.816 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:42.816 | INFO | Uploaded 36GiB to GPU in 2.19758s (16.3817GiB/s) | 20260505 11:33:42.816 | INFO | Downloaded 34GiB to GPU in 2.12413s (16.0066GiB/s) | 20260505 11:33:47.955 | INFO | Analyzing chunk frames = [2882560,2883072) | 20260505 11:33:47.955 | INFO | MemInfo: { | 20260505 11:33:47.955 | INFO | "gpu" : " 17 GB", | 20260505 11:33:47.955 | INFO | "huge" : "", | 20260505 11:33:47.955 | INFO | "huge_shared" : " 27 GB", | 20260505 11:33:47.955 | INFO | "malloc" : " 14 GB", | 20260505 11:33:47.955 | INFO | "pinned" : " 29 GB" | 20260505 11:33:47.955 | INFO | } | 20260505 11:33:47.955 | INFO | Perf report for the last 5.13937 seconds: | 20260505 11:33:47.955 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:33:47.955 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:47.955 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:47.955 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:47.955 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 11:33:47.955 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:47.955 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:47.955 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 11:33:47.955 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:47.955 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:47.955 | INFO | Uploaded 36GiB to GPU in 2.17246s (16.5711GiB/s) | 20260505 11:33:47.956 | INFO | Downloaded 34GiB to GPU in 2.14475s (15.8526GiB/s) | 20260505 11:33:53.103 | INFO | Analyzing chunk frames = [2883072,2883584) | 20260505 11:33:53.103 | INFO | MemInfo: { | 20260505 11:33:53.103 | INFO | "gpu" : " 17 GB", | 20260505 11:33:53.103 | INFO | "huge" : "", | 20260505 11:33:53.103 | INFO | "huge_shared" : " 27 GB", | 20260505 11:33:53.103 | INFO | "malloc" : " 14 GB", | 20260505 11:33:53.103 | INFO | "pinned" : " 29 GB" | 20260505 11:33:53.103 | INFO | } | 20260505 11:33:53.103 | INFO | Perf report for the last 5.14757 seconds: | 20260505 11:33:53.103 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:33:53.103 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:53.103 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:53.103 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:53.103 | INFO | ANALYSIS : 63.9%, 3.0, 99.9ms, 0 | 20260505 11:33:53.103 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:53.103 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:53.103 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 11:33:53.103 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:53.103 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:53.103 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 11:33:53.103 | INFO | Total mem usage: 15.3 GiB | 20260505 11:33:53.103 | INFO | Uploaded 36GiB to GPU in 2.22404s (16.1868GiB/s) | 20260505 11:33:53.103 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 11:33:53.103 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.8%) | 20260505 11:33:53.103 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 11:33:53.103 | INFO | Downloaded 34GiB to GPU in 2.09354s (16.2404GiB/s) | 20260505 11:33:53.103 | INFO | High Water Breakdown: | 20260505 11:33:53.103 | INFO | 61.1% : 8.76 GiB ( 92.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 11:33:53.103 | INFO | 38.8% : 5.57 GiB ( 83.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 11:33:53.103 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 11:33:53.103 | INFO | ------------------------------------------ | 20260505 11:33:53.103 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:33:53.103 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 11:33:53.103 | INFO | Total mem usage: 26.5 GiB | 20260505 11:33:53.104 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 11:33:53.104 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 11:33:53.104 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 11:33:53.104 | INFO | High Water Breakdown: | 20260505 11:33:53.104 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:33:53.104 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:33:53.104 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:33:53.104 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:33:53.104 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:33:53.104 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:33:53.104 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 11:33:53.104 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 11:33:53.104 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:33:53.104 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 11:33:53.104 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 11:33:53.104 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 11:33:53.104 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 11:33:53.104 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 11:33:53.104 | INFO | ------------------------------------------ | 20260505 11:33:53.104 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:33:53.104 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 11:33:53.104 | INFO | Total mem usage: 25.6 GiB | 20260505 11:33:53.104 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 11:33:53.104 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 11:33:53.104 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 11:33:53.104 | INFO | High Water Breakdown: | 20260505 11:33:53.104 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 11:33:53.104 | INFO | ------------------------------------------ | 20260505 11:33:53.104 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:33:53.104 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 11:33:53.104 | INFO | Total mem usage: 15.7 GiB | 20260505 11:33:53.104 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 11:33:53.104 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 11:33:53.104 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 11:33:53.104 | INFO | High Water Breakdown: | 20260505 11:33:53.104 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 11:33:53.104 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 11:33:53.104 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 11:33:53.104 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 11:33:53.104 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 11:33:53.104 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 11:33:53.104 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 11:33:53.104 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 11:33:53.104 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 11:33:53.104 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:33:53.104 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 11:33:53.104 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:33:53.104 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 11:33:53.104 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 11:33:53.104 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 11:33:53.104 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 11:33:53.104 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:33:53.104 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 11:33:53.104 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:33:53.104 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:33:53.104 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:33:53.104 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 11:33:53.104 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 11:33:53.104 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:33:53.104 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 11:33:53.104 | INFO | ------------------------------------------ | 20260505 11:33:53.104 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:33:53.104 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 11:33:53.104 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:33:53.105 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:33:53.105 | INFO | PerfReport: Full Execution -- 3.22e+01 / 7.80e+01, 1.39e+01 / 1.78e+00 | 20260505 11:33:53.105 | INFO | PerfReport: 0.29%, AggregateStats -- 9.31e-02 / 1.01e-01, 1.07e-02 / 4.62e-03 | 20260505 11:33:53.105 | INFO | PerfReport: 11.74%, Baseline -- 3.78e+00 / 4.51e+00, 6.06e-01 / 1.40e-01 | 20260505 11:33:53.105 | INFO | PerfReport: 3.31%, Histogram -- 1.06e+00 / 1.51e+00, 2.06e-01 / 7.85e-02 | 20260505 11:33:53.105 | INFO | PerfReport: 15.94%, DME -- 4.10e+01 / 4.61e+01, 4.84e+00 / 2.59e+00 | 20260505 11:33:53.105 | INFO | PerfReport: 56.32%, FrameLabeling -- 1.81e+01 / 2.10e+01, 1.55e+00 / 2.20e+00 | 20260505 11:33:53.105 | INFO | PerfReport: 7.41%, PulseAccumulating -- 2.38e+00 / 2.87e+00, 4.39e-01 / 2.12e-01 | 20260505 11:33:53.105 | INFO | PerfReport: 3.70%, Metrics -- 1.19e+00 / 1.46e+00, 3.43e-01 / 2.15e-01 | 20260505 11:33:53.105 | INFO | PerfReport: 1.30%, PulseSerializing -- 4.18e-01 / 4.56e-01, 7.70e-02 / 4.49e-02 | 20260505 11:33:53.105 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 11:33:53.105 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:33:53.105 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:33:53.105 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260505 11:33:53.105 | INFO | PerfReport: 22.59%, Upload -- 2.29e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260505 11:33:53.105 | INFO | PerfReport: 22.02%, Download -- 2.23e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260505 11:33:53.105 | INFO | PerfReport: 31.75%, Compute -- 3.22e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 11:33:53.105 | INFO | PerfReport: 2.10%, LockUpload -- 2.13e+00 / 0.00e+00, 7.70e+00 / 0.00e+00 | 20260505 11:33:53.105 | INFO | PerfReport: 1.30%, LockDownload -- 1.31e+00 / 0.00e+00, 2.85e+00 / 0.00e+00 | 20260505 11:33:53.105 | INFO | PerfReport: 20.24%, LockCompute -- 2.05e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260505 11:33:53.105 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.49e-02 / 0.00e+00 | 20260505 11:33:53.236 | INFO | Starting a round of Baz IO | 20260505 11:33:56.485 | INFO | Populating RTMetrics report | 20260505 11:33:56.485 | INFO | RTMetrics report populated | 20260505 11:33:56.485 | INFO | Opening tmp RTMetrics json file | 20260505 11:33:56.493 | INFO | Serializing RTMetrics to json | 20260505 11:33:56.542 | INFO | Serialization done, swapping tmp file now... | 20260505 11:33:56.544 | INFO | RTMetrics done | 20260505 11:33:56.621 | INFO | Finished a round of Baz IO: Wrote 2.376 GiB in 3.384 seconds | 20260505 11:33:56.621 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.10% / 0.01% / 75.89% | 20260505 11:33:56.621 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.11639% / 1.98682e-05% / 8.34465e-05% | 20260505 11:33:56.621 | INFO | Baz IO overall: | 20260505 11:33:56.621 | INFO | BytesWritten : 1706096153873 | 20260505 11:33:56.621 | INFO | NumEvents : 812315707642 | 20260505 11:33:58.247 | INFO | Analyzing chunk frames = [2883584,2884096) | 20260505 11:33:58.247 | INFO | MemInfo: { | 20260505 11:33:58.247 | INFO | "gpu" : " 17 GB", | 20260505 11:33:58.247 | INFO | "huge" : "", | 20260505 11:33:58.247 | INFO | "huge_shared" : " 27 GB", | 20260505 11:33:58.247 | INFO | "malloc" : " 14 GB", | 20260505 11:33:58.247 | INFO | "pinned" : " 29 GB" | 20260505 11:33:58.247 | INFO | } | 20260505 11:33:58.247 | INFO | Perf report for the last 5.14452 seconds: | 20260505 11:33:58.247 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:33:58.247 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:58.247 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:58.247 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:58.248 | INFO | ANALYSIS : 64.6%, 2.9, 99.6ms, 0 | 20260505 11:33:58.248 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:33:58.248 | INFO | RT_METRICS : 46.4%, 1.0, 24.4ms, 0 | 20260505 11:33:58.248 | INFO | PRE_HQ : 61.1%, 1.9, 59.1ms, 0 | 20260505 11:33:58.248 | INFO | BAZWRITER : 64.9%, 4.1, 137.9ms, 0 | 20260505 11:33:58.248 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 11:33:58.248 | INFO | Uploaded 36GiB to GPU in 2.19341s (16.4128GiB/s) | 20260505 11:33:58.248 | INFO | Downloaded 38GiB to GPU in 2.28788s (16.6093GiB/s) | 20260505 11:34:03.382 | INFO | Analyzing chunk frames = [2884096,2884608) | 20260505 11:34:03.382 | INFO | MemInfo: { | 20260505 11:34:03.382 | INFO | "gpu" : " 17 GB", | 20260505 11:34:03.382 | INFO | "huge" : "", | 20260505 11:34:03.382 | INFO | "huge_shared" : " 27 GB", | 20260505 11:34:03.382 | INFO | "malloc" : " 14 GB", | 20260505 11:34:03.382 | INFO | "pinned" : " 29 GB" | 20260505 11:34:03.382 | INFO | } | 20260505 11:34:03.382 | INFO | Perf report for the last 5.13437 seconds: | 20260505 11:34:03.382 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:34:03.382 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:03.382 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:03.382 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:03.382 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 11:34:03.382 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:03.382 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:03.382 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:34:03.382 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:03.382 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:03.382 | INFO | Uploaded 36GiB to GPU in 2.18749s (16.4572GiB/s) | 20260505 11:34:03.382 | INFO | Downloaded 34GiB to GPU in 2.05122s (16.5755GiB/s) | 20260505 11:34:08.513 | INFO | Analyzing chunk frames = [2884608,2885120) | 20260505 11:34:08.513 | INFO | MemInfo: { | 20260505 11:34:08.513 | INFO | "gpu" : " 17 GB", | 20260505 11:34:08.513 | INFO | "huge" : "", | 20260505 11:34:08.513 | INFO | "huge_shared" : " 27 GB", | 20260505 11:34:08.513 | INFO | "malloc" : " 14 GB", | 20260505 11:34:08.514 | INFO | "pinned" : " 29 GB" | 20260505 11:34:08.514 | INFO | } | 20260505 11:34:08.514 | INFO | Perf report for the last 5.13182 seconds: | 20260505 11:34:08.514 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:34:08.514 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:08.514 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:08.514 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:08.514 | INFO | ANALYSIS : 63.6%, 3.0, 98.9ms, 0 | 20260505 11:34:08.514 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:08.514 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:08.514 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 11:34:08.514 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:08.514 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:08.514 | INFO | Uploaded 36GiB to GPU in 2.2229s (16.195GiB/s) | 20260505 11:34:08.514 | INFO | Downloaded 34GiB to GPU in 2.1308s (15.9564GiB/s) | 20260505 11:34:13.660 | INFO | Analyzing chunk frames = [2885120,2885632) | 20260505 11:34:13.660 | INFO | MemInfo: { | 20260505 11:34:13.660 | INFO | "gpu" : " 17 GB", | 20260505 11:34:13.660 | INFO | "huge" : "", | 20260505 11:34:13.660 | INFO | "huge_shared" : " 27 GB", | 20260505 11:34:13.660 | INFO | "malloc" : " 14 GB", | 20260505 11:34:13.660 | INFO | "pinned" : " 29 GB" | 20260505 11:34:13.660 | INFO | } | 20260505 11:34:13.660 | INFO | Perf report for the last 5.14612 seconds: | 20260505 11:34:13.660 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:34:13.660 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:13.660 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:13.660 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:13.660 | INFO | ANALYSIS : 62.9%, 3.0, 97.8ms, 0 | 20260505 11:34:13.660 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:13.660 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:13.660 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 11:34:13.660 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:13.660 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:13.660 | INFO | Uploaded 36GiB to GPU in 2.1542s (16.7115GiB/s) | 20260505 11:34:13.660 | INFO | Downloaded 34GiB to GPU in 2.07738s (16.3668GiB/s) | 20260505 11:34:18.804 | INFO | Analyzing chunk frames = [2885632,2886144) | 20260505 11:34:18.804 | INFO | MemInfo: { | 20260505 11:34:18.804 | INFO | "gpu" : " 17 GB", | 20260505 11:34:18.804 | INFO | "huge" : "", | 20260505 11:34:18.804 | INFO | "huge_shared" : " 27 GB", | 20260505 11:34:18.804 | INFO | "malloc" : " 14 GB", | 20260505 11:34:18.804 | INFO | "pinned" : " 29 GB" | 20260505 11:34:18.804 | INFO | } | 20260505 11:34:18.804 | INFO | Perf report for the last 5.14449 seconds: | 20260505 11:34:18.804 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:34:18.804 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:18.804 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:18.804 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:18.804 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 11:34:18.804 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:18.804 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:18.804 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 11:34:18.804 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:18.804 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:18.804 | INFO | Uploaded 36GiB to GPU in 2.18036s (16.5111GiB/s) | 20260505 11:34:18.804 | INFO | Downloaded 34GiB to GPU in 2.12552s (15.9961GiB/s) | 20260505 11:34:23.935 | INFO | Analyzing chunk frames = [2886144,2886656) | 20260505 11:34:23.935 | INFO | MemInfo: { | 20260505 11:34:23.935 | INFO | "gpu" : " 17 GB", | 20260505 11:34:23.935 | INFO | "huge" : "", | 20260505 11:34:23.935 | INFO | "huge_shared" : " 27 GB", | 20260505 11:34:23.935 | INFO | "malloc" : " 14 GB", | 20260505 11:34:23.935 | INFO | "pinned" : " 29 GB" | 20260505 11:34:23.935 | INFO | } | 20260505 11:34:23.935 | INFO | Perf report for the last 5.13086 seconds: | 20260505 11:34:23.935 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:34:23.935 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:23.935 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:23.935 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:23.935 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 11:34:23.935 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:23.935 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:23.935 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:34:23.935 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:23.935 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:23.935 | INFO | Uploaded 36GiB to GPU in 2.1152s (17.0197GiB/s) | 20260505 11:34:23.935 | INFO | Downloaded 34GiB to GPU in 2.10146s (16.1792GiB/s) | 20260505 11:34:29.079 | INFO | Analyzing chunk frames = [2886656,2887168) | 20260505 11:34:29.079 | INFO | MemInfo: { | 20260505 11:34:29.079 | INFO | "gpu" : " 17 GB", | 20260505 11:34:29.079 | INFO | "huge" : "", | 20260505 11:34:29.079 | INFO | "huge_shared" : " 27 GB", | 20260505 11:34:29.079 | INFO | "malloc" : " 14 GB", | 20260505 11:34:29.079 | INFO | "pinned" : " 29 GB" | 20260505 11:34:29.079 | INFO | } | 20260505 11:34:29.079 | INFO | Perf report for the last 5.1437 seconds: | 20260505 11:34:29.079 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:34:29.079 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:29.079 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:29.079 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:29.079 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 11:34:29.079 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:29.079 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:29.079 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 11:34:29.079 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:29.079 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:29.079 | INFO | Uploaded 36GiB to GPU in 2.17866s (16.5239GiB/s) | 20260505 11:34:29.079 | INFO | Downloaded 34GiB to GPU in 2.15016s (15.8128GiB/s) | 20260505 11:34:34.213 | INFO | Analyzing chunk frames = [2887168,2887680) | 20260505 11:34:34.213 | INFO | MemInfo: { | 20260505 11:34:34.213 | INFO | "gpu" : " 17 GB", | 20260505 11:34:34.213 | INFO | "huge" : "", | 20260505 11:34:34.213 | INFO | "huge_shared" : " 27 GB", | 20260505 11:34:34.213 | INFO | "malloc" : " 14 GB", | 20260505 11:34:34.213 | INFO | "pinned" : " 29 GB" | 20260505 11:34:34.213 | INFO | } | 20260505 11:34:34.213 | INFO | Perf report for the last 5.13443 seconds: | 20260505 11:34:34.213 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:34:34.213 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:34.213 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:34.213 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:34.213 | INFO | ANALYSIS : 63.9%, 3.0, 99.6ms, 0 | 20260505 11:34:34.213 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:34.213 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:34.213 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 11:34:34.213 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:34.213 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:34.213 | INFO | Uploaded 36GiB to GPU in 2.21924s (16.2218GiB/s) | 20260505 11:34:34.213 | INFO | Downloaded 34GiB to GPU in 2.07646s (16.3741GiB/s) | 20260505 11:34:34.347 | INFO | Starting a round of Baz IO | 20260505 11:34:37.601 | INFO | Populating RTMetrics report | 20260505 11:34:37.601 | INFO | RTMetrics report populated | 20260505 11:34:37.601 | INFO | Opening tmp RTMetrics json file | 20260505 11:34:37.628 | INFO | Serializing RTMetrics to json | 20260505 11:34:37.633 | INFO | Serialization done, swapping tmp file now... | 20260505 11:34:37.635 | INFO | RTMetrics done | 20260505 11:34:37.728 | INFO | Finished a round of Baz IO: Wrote 2.376 GiB in 3.38 seconds | 20260505 11:34:37.728 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.09% / 0.01% / 75.90% | 20260505 11:34:37.728 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.1127% / 1.98682e-05% / 7.54992e-05% | 20260505 11:34:37.728 | INFO | Baz IO overall: | 20260505 11:34:37.728 | INFO | BytesWritten : 1708647834524 | 20260505 11:34:37.728 | INFO | NumEvents : 813529641662 | 20260505 11:34:39.358 | INFO | Analyzing chunk frames = [2887680,2888192) | 20260505 11:34:39.358 | INFO | MemInfo: { | 20260505 11:34:39.358 | INFO | "gpu" : " 17 GB", | 20260505 11:34:39.358 | INFO | "huge" : "", | 20260505 11:34:39.358 | INFO | "huge_shared" : " 27 GB", | 20260505 11:34:39.358 | INFO | "malloc" : " 14 GB", | 20260505 11:34:39.358 | INFO | "pinned" : " 29 GB" | 20260505 11:34:39.358 | INFO | } | 20260505 11:34:39.358 | INFO | Perf report for the last 5.14477 seconds: | 20260505 11:34:39.358 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:34:39.358 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:39.358 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:39.358 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:39.358 | INFO | ANALYSIS : 64.7%, 2.9, 99.8ms, 0 | 20260505 11:34:39.358 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:39.358 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260505 11:34:39.358 | INFO | PRE_HQ : 61.1%, 1.9, 58.9ms, 0 | 20260505 11:34:39.358 | INFO | BAZWRITER : 65.0%, 4.1, 137.2ms, 0 | 20260505 11:34:39.358 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 11:34:39.358 | INFO | Uploaded 36GiB to GPU in 2.19795s (16.3789GiB/s) | 20260505 11:34:39.358 | INFO | Downloaded 38GiB to GPU in 2.29429s (16.5629GiB/s) | 20260505 11:34:44.498 | INFO | Analyzing chunk frames = [2888192,2888704) | 20260505 11:34:44.498 | INFO | MemInfo: { | 20260505 11:34:44.498 | INFO | "gpu" : " 17 GB", | 20260505 11:34:44.498 | INFO | "huge" : "", | 20260505 11:34:44.498 | INFO | "huge_shared" : " 27 GB", | 20260505 11:34:44.498 | INFO | "malloc" : " 14 GB", | 20260505 11:34:44.498 | INFO | "pinned" : " 29 GB" | 20260505 11:34:44.498 | INFO | } | 20260505 11:34:44.498 | INFO | Perf report for the last 5.1398 seconds: | 20260505 11:34:44.498 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:34:44.498 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:44.498 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:44.498 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:44.498 | INFO | ANALYSIS : 64.1%, 3.0, 100.2ms, 0 | 20260505 11:34:44.498 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:44.498 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:44.498 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:34:44.498 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:44.498 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:44.498 | INFO | Uploaded 36GiB to GPU in 2.23178s (16.1306GiB/s) | 20260505 11:34:44.498 | INFO | Downloaded 34GiB to GPU in 2.0998s (16.192GiB/s) | 20260505 11:34:49.627 | INFO | Analyzing chunk frames = [2888704,2889216) | 20260505 11:34:49.627 | INFO | MemInfo: { | 20260505 11:34:49.627 | INFO | "gpu" : " 17 GB", | 20260505 11:34:49.627 | INFO | "huge" : "", | 20260505 11:34:49.627 | INFO | "huge_shared" : " 27 GB", | 20260505 11:34:49.627 | INFO | "malloc" : " 14 GB", | 20260505 11:34:49.627 | INFO | "pinned" : " 29 GB" | 20260505 11:34:49.627 | INFO | } | 20260505 11:34:49.627 | INFO | Perf report for the last 5.12921 seconds: | 20260505 11:34:49.627 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:34:49.627 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:49.627 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:49.627 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:49.627 | INFO | ANALYSIS : 63.6%, 3.0, 98.8ms, 0 | 20260505 11:34:49.627 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:49.627 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:49.627 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 11:34:49.627 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:49.627 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:49.627 | INFO | Uploaded 36GiB to GPU in 2.23103s (16.136GiB/s) | 20260505 11:34:49.627 | INFO | Downloaded 34GiB to GPU in 2.12969s (15.9648GiB/s) | 20260505 11:34:54.765 | INFO | Analyzing chunk frames = [2889216,2889728) | 20260505 11:34:54.765 | INFO | MemInfo: { | 20260505 11:34:54.765 | INFO | "gpu" : " 17 GB", | 20260505 11:34:54.765 | INFO | "huge" : "", | 20260505 11:34:54.765 | INFO | "huge_shared" : " 27 GB", | 20260505 11:34:54.765 | INFO | "malloc" : " 14 GB", | 20260505 11:34:54.765 | INFO | "pinned" : " 29 GB" | 20260505 11:34:54.765 | INFO | } | 20260505 11:34:54.765 | INFO | Perf report for the last 5.13833 seconds: | 20260505 11:34:54.765 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:34:54.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:54.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:54.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:54.765 | INFO | ANALYSIS : 63.0%, 3.0, 97.9ms, 0 | 20260505 11:34:54.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:54.765 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:54.766 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:34:54.766 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:54.766 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:54.766 | INFO | Uploaded 36GiB to GPU in 2.17152s (16.5782GiB/s) | 20260505 11:34:54.766 | INFO | Downloaded 34GiB to GPU in 2.07688s (16.3707GiB/s) | 20260505 11:34:59.912 | INFO | Analyzing chunk frames = [2889728,2890240) | 20260505 11:34:59.912 | INFO | MemInfo: { | 20260505 11:34:59.913 | INFO | "gpu" : " 17 GB", | 20260505 11:34:59.913 | INFO | "huge" : "", | 20260505 11:34:59.913 | INFO | "huge_shared" : " 27 GB", | 20260505 11:34:59.913 | INFO | "malloc" : " 14 GB", | 20260505 11:34:59.913 | INFO | "pinned" : " 29 GB" | 20260505 11:34:59.913 | INFO | } | 20260505 11:34:59.913 | INFO | Perf report for the last 5.14728 seconds: | 20260505 11:34:59.913 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:34:59.913 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:59.913 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:59.913 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:59.913 | INFO | ANALYSIS : 63.2%, 3.0, 98.8ms, 0 | 20260505 11:34:59.913 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:59.913 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:59.913 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 11:34:59.913 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:59.913 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:34:59.913 | INFO | Uploaded 36GiB to GPU in 2.1648s (16.6297GiB/s) | 20260505 11:34:59.913 | INFO | Downloaded 34GiB to GPU in 2.12446s (16.004GiB/s) | 20260505 11:35:05.056 | INFO | Analyzing chunk frames = [2890240,2890752) | 20260505 11:35:05.056 | INFO | MemInfo: { | 20260505 11:35:05.056 | INFO | "gpu" : " 17 GB", | 20260505 11:35:05.056 | INFO | "huge" : "", | 20260505 11:35:05.056 | INFO | "huge_shared" : " 27 GB", | 20260505 11:35:05.056 | INFO | "malloc" : " 14 GB", | 20260505 11:35:05.056 | INFO | "pinned" : " 29 GB" | 20260505 11:35:05.056 | INFO | } | 20260505 11:35:05.056 | INFO | Perf report for the last 5.14358 seconds: | 20260505 11:35:05.056 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:35:05.056 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:05.056 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:05.056 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:05.056 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 11:35:05.056 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:05.056 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:05.056 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:35:05.056 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:05.056 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:05.057 | INFO | Uploaded 36GiB to GPU in 2.16579s (16.6221GiB/s) | 20260505 11:35:05.057 | INFO | Downloaded 34GiB to GPU in 2.10851s (16.1251GiB/s) | 20260505 11:35:10.189 | INFO | Analyzing chunk frames = [2890752,2891264) | 20260505 11:35:10.189 | INFO | MemInfo: { | 20260505 11:35:10.189 | INFO | "gpu" : " 17 GB", | 20260505 11:35:10.189 | INFO | "huge" : "", | 20260505 11:35:10.190 | INFO | "huge_shared" : " 27 GB", | 20260505 11:35:10.190 | INFO | "malloc" : " 14 GB", | 20260505 11:35:10.190 | INFO | "pinned" : " 29 GB" | 20260505 11:35:10.190 | INFO | } | 20260505 11:35:10.190 | INFO | Perf report for the last 5.13335 seconds: | 20260505 11:35:10.190 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:35:10.190 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:10.190 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:10.190 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:10.190 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 11:35:10.190 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:10.190 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:10.190 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 11:35:10.190 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:10.190 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:10.190 | INFO | Uploaded 36GiB to GPU in 2.1278s (16.9189GiB/s) | 20260505 11:35:10.190 | INFO | Downloaded 34GiB to GPU in 2.1477s (15.8309GiB/s) | 20260505 11:35:15.325 | INFO | Analyzing chunk frames = [2891264,2891776) | 20260505 11:35:15.325 | INFO | MemInfo: { | 20260505 11:35:15.325 | INFO | "gpu" : " 17 GB", | 20260505 11:35:15.325 | INFO | "huge" : "", | 20260505 11:35:15.325 | INFO | "huge_shared" : " 27 GB", | 20260505 11:35:15.325 | INFO | "malloc" : " 14 GB", | 20260505 11:35:15.325 | INFO | "pinned" : " 29 GB" | 20260505 11:35:15.325 | INFO | } | 20260505 11:35:15.325 | INFO | Perf report for the last 5.13546 seconds: | 20260505 11:35:15.325 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:35:15.325 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:15.325 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:15.325 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:15.325 | INFO | ANALYSIS : 64.0%, 3.0, 99.8ms, 0 | 20260505 11:35:15.325 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:15.325 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:15.325 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 11:35:15.325 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:15.325 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:15.325 | INFO | Uploaded 36GiB to GPU in 2.21712s (16.2373GiB/s) | 20260505 11:35:15.325 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 11:35:15.325 | INFO | Downloaded 34GiB to GPU in 2.10104s (16.1825GiB/s) | 20260505 11:35:15.325 | INFO | Total mem usage: 15.3 GiB | 20260505 11:35:15.325 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 11:35:15.326 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.78%) | 20260505 11:35:15.326 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 11:35:15.326 | INFO | High Water Breakdown: | 20260505 11:35:15.326 | INFO | 61.1% : 8.76 GiB ( 92.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 11:35:15.326 | INFO | 38.8% : 5.57 GiB ( 83.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 11:35:15.326 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 11:35:15.326 | INFO | ------------------------------------------ | 20260505 11:35:15.326 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:35:15.326 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 11:35:15.326 | INFO | Total mem usage: 26.5 GiB | 20260505 11:35:15.326 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 11:35:15.326 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 11:35:15.326 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 11:35:15.326 | INFO | High Water Breakdown: | 20260505 11:35:15.326 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:35:15.326 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:35:15.326 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:35:15.326 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:35:15.326 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:35:15.326 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:35:15.326 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 11:35:15.326 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 11:35:15.326 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:35:15.326 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 11:35:15.326 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 11:35:15.326 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 11:35:15.326 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 11:35:15.326 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 11:35:15.326 | INFO | ------------------------------------------ | 20260505 11:35:15.326 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:35:15.326 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 11:35:15.326 | INFO | Total mem usage: 25.6 GiB | 20260505 11:35:15.326 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 11:35:15.326 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 11:35:15.326 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 11:35:15.326 | INFO | High Water Breakdown: | 20260505 11:35:15.326 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 11:35:15.326 | INFO | ------------------------------------------ | 20260505 11:35:15.326 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:35:15.326 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 11:35:15.326 | INFO | Total mem usage: 15.7 GiB | 20260505 11:35:15.326 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 11:35:15.326 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 11:35:15.326 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 11:35:15.326 | INFO | High Water Breakdown: | 20260505 11:35:15.326 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 11:35:15.326 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 11:35:15.326 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 11:35:15.326 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 11:35:15.326 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 11:35:15.326 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 11:35:15.326 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 11:35:15.326 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 11:35:15.326 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 11:35:15.326 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:35:15.326 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 11:35:15.326 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:35:15.326 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 11:35:15.326 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 11:35:15.326 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 11:35:15.326 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 11:35:15.326 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:35:15.326 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 11:35:15.326 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:35:15.326 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:35:15.326 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:35:15.326 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 11:35:15.326 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 11:35:15.326 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:35:15.326 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 11:35:15.326 | INFO | ------------------------------------------ | 20260505 11:35:15.326 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:35:15.326 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 11:35:15.326 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:35:15.326 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:35:15.326 | INFO | PerfReport: Full Execution -- 3.21e+01 / 7.68e+01, 1.39e+01 / 1.75e+00 | 20260505 11:35:15.326 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 1.01e-01, 1.06e-02 / 5.63e-03 | 20260505 11:35:15.326 | INFO | PerfReport: 11.74%, Baseline -- 3.77e+00 / 4.47e+00, 6.05e-01 / 2.55e-01 | 20260505 11:35:15.326 | INFO | PerfReport: 3.32%, Histogram -- 1.07e+00 / 1.52e+00, 2.09e-01 / 7.03e-02 | 20260505 11:35:15.326 | INFO | PerfReport: 15.82%, DME -- 4.07e+01 / 4.49e+01, 4.25e+00 / 1.89e+00 | 20260505 11:35:15.326 | INFO | PerfReport: 56.43%, FrameLabeling -- 1.81e+01 / 2.10e+01, 1.62e+00 / 1.85e+00 | 20260505 11:35:15.326 | INFO | PerfReport: 7.42%, PulseAccumulating -- 2.38e+00 / 2.89e+00, 4.29e-01 / 1.85e-01 | 20260505 11:35:15.326 | INFO | PerfReport: 3.69%, Metrics -- 1.19e+00 / 1.49e+00, 3.36e-01 / 1.80e-01 | 20260505 11:35:15.326 | INFO | PerfReport: 1.30%, PulseSerializing -- 4.17e-01 / 4.65e-01, 7.65e-02 / 4.23e-02 | 20260505 11:35:15.326 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 11:35:15.326 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:35:15.326 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:35:15.326 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.48e+01 / 0.00e+00 | 20260505 11:35:15.326 | INFO | PerfReport: 22.51%, Upload -- 2.28e+01 / 0.00e+00, 8.11e+00 / 0.00e+00 | 20260505 11:35:15.326 | INFO | PerfReport: 21.95%, Download -- 2.22e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260505 11:35:15.326 | INFO | PerfReport: 31.79%, Compute -- 3.21e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 11:35:15.326 | INFO | PerfReport: 2.04%, LockUpload -- 2.06e+00 / 0.00e+00, 7.74e+00 / 0.00e+00 | 20260505 11:35:15.326 | INFO | PerfReport: 1.30%, LockDownload -- 1.32e+00 / 0.00e+00, 2.92e+00 / 0.00e+00 | 20260505 11:35:15.326 | INFO | PerfReport: 20.39%, LockCompute -- 2.06e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260505 11:35:15.326 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.20e-02 / 0.00e+00 | 20260505 11:35:15.459 | INFO | Starting a round of Baz IO | 20260505 11:35:18.715 | INFO | Populating RTMetrics report | 20260505 11:35:18.716 | INFO | RTMetrics report populated | 20260505 11:35:18.716 | INFO | Opening tmp RTMetrics json file | 20260505 11:35:18.725 | INFO | Serializing RTMetrics to json | 20260505 11:35:18.774 | INFO | Serialization done, swapping tmp file now... | 20260505 11:35:18.796 | INFO | RTMetrics done | 20260505 11:35:18.832 | INFO | Finished a round of Baz IO: Wrote 2.372 GiB in 3.373 seconds | 20260505 11:35:18.832 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.08% / 0.01% / 75.91% | 20260505 11:35:18.832 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.1052% / 1.58946e-05% / 9.13938e-05% | 20260505 11:35:18.832 | INFO | Baz IO overall: | 20260505 11:35:18.832 | INFO | BytesWritten : 1711195216548 | 20260505 11:35:18.832 | INFO | NumEvents : 814740559213 | 20260505 11:35:20.467 | INFO | Analyzing chunk frames = [2891776,2892288) | 20260505 11:35:20.467 | INFO | MemInfo: { | 20260505 11:35:20.467 | INFO | "gpu" : " 17 GB", | 20260505 11:35:20.467 | INFO | "huge" : "", | 20260505 11:35:20.467 | INFO | "huge_shared" : " 27 GB", | 20260505 11:35:20.467 | INFO | "malloc" : " 14 GB", | 20260505 11:35:20.467 | INFO | "pinned" : " 29 GB" | 20260505 11:35:20.467 | INFO | } | 20260505 11:35:20.467 | INFO | Perf report for the last 5.14224 seconds: | 20260505 11:35:20.467 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:35:20.467 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:20.467 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:20.467 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:20.467 | INFO | ANALYSIS : 64.8%, 2.9, 99.8ms, 0 | 20260505 11:35:20.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:20.467 | INFO | RT_METRICS : 46.9%, 1.0, 24.6ms, 0 | 20260505 11:35:20.467 | INFO | PRE_HQ : 62.3%, 1.8, 58.7ms, 0 | 20260505 11:35:20.467 | INFO | BAZWRITER : 64.9%, 4.1, 137.7ms, 0 | 20260505 11:35:20.467 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 11:35:20.468 | INFO | Uploaded 36GiB to GPU in 2.30171s (15.6406GiB/s) | 20260505 11:35:20.468 | INFO | Downloaded 38GiB to GPU in 2.39254s (15.8827GiB/s) | 20260505 11:35:25.609 | INFO | Analyzing chunk frames = [2892288,2892800) | 20260505 11:35:25.609 | INFO | MemInfo: { | 20260505 11:35:25.609 | INFO | "gpu" : " 17 GB", | 20260505 11:35:25.609 | INFO | "huge" : "", | 20260505 11:35:25.609 | INFO | "huge_shared" : " 27 GB", | 20260505 11:35:25.609 | INFO | "malloc" : " 14 GB", | 20260505 11:35:25.609 | INFO | "pinned" : " 29 GB" | 20260505 11:35:25.609 | INFO | } | 20260505 11:35:25.609 | INFO | Perf report for the last 5.14137 seconds: | 20260505 11:35:25.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:35:25.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:25.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:25.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:25.609 | INFO | ANALYSIS : 64.1%, 3.0, 100.0ms, 0 | 20260505 11:35:25.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:25.609 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:25.609 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 11:35:25.609 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:25.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:25.609 | INFO | Uploaded 36GiB to GPU in 2.22813s (16.1571GiB/s) | 20260505 11:35:25.609 | INFO | Downloaded 34GiB to GPU in 2.0793s (16.3516GiB/s) | 20260505 11:35:30.758 | INFO | Analyzing chunk frames = [2892800,2893312) | 20260505 11:35:30.758 | INFO | MemInfo: { | 20260505 11:35:30.758 | INFO | "gpu" : " 17 GB", | 20260505 11:35:30.758 | INFO | "huge" : "", | 20260505 11:35:30.759 | INFO | "huge_shared" : " 27 GB", | 20260505 11:35:30.759 | INFO | "malloc" : " 14 GB", | 20260505 11:35:30.759 | INFO | "pinned" : " 29 GB" | 20260505 11:35:30.759 | INFO | } | 20260505 11:35:30.759 | INFO | Perf report for the last 5.14989 seconds: | 20260505 11:35:30.759 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:35:30.759 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:30.759 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:30.759 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:30.759 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 11:35:30.759 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:30.759 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:30.759 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:35:30.759 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:30.759 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:30.759 | INFO | Uploaded 36GiB to GPU in 2.21243s (16.2717GiB/s) | 20260505 11:35:30.759 | INFO | Downloaded 34GiB to GPU in 2.12525s (15.9981GiB/s) | 20260505 11:35:35.889 | INFO | Analyzing chunk frames = [2893312,2893824) | 20260505 11:35:35.889 | INFO | MemInfo: { | 20260505 11:35:35.889 | INFO | "gpu" : " 17 GB", | 20260505 11:35:35.889 | INFO | "huge" : "", | 20260505 11:35:35.889 | INFO | "huge_shared" : " 27 GB", | 20260505 11:35:35.889 | INFO | "malloc" : " 14 GB", | 20260505 11:35:35.889 | INFO | "pinned" : " 29 GB" | 20260505 11:35:35.889 | INFO | } | 20260505 11:35:35.889 | INFO | Perf report for the last 5.1301 seconds: | 20260505 11:35:35.889 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:35:35.889 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:35.889 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:35.889 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:35.889 | INFO | ANALYSIS : 63.3%, 3.0, 98.1ms, 0 | 20260505 11:35:35.889 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:35.889 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:35.889 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 11:35:35.889 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:35.889 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:35.889 | INFO | Uploaded 36GiB to GPU in 2.20377s (16.3356GiB/s) | 20260505 11:35:35.889 | INFO | Downloaded 34GiB to GPU in 2.12862s (15.9728GiB/s) | 20260505 11:35:41.025 | INFO | Analyzing chunk frames = [2893824,2894336) | 20260505 11:35:41.026 | INFO | MemInfo: { | 20260505 11:35:41.026 | INFO | "gpu" : " 17 GB", | 20260505 11:35:41.026 | INFO | "huge" : "", | 20260505 11:35:41.026 | INFO | "huge_shared" : " 27 GB", | 20260505 11:35:41.026 | INFO | "malloc" : " 14 GB", | 20260505 11:35:41.026 | INFO | "pinned" : " 29 GB" | 20260505 11:35:41.026 | INFO | } | 20260505 11:35:41.026 | INFO | Perf report for the last 5.13689 seconds: | 20260505 11:35:41.026 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:35:41.026 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:41.026 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:41.026 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:41.026 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 11:35:41.026 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:41.026 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:41.026 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 11:35:41.026 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:41.026 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:41.026 | INFO | Uploaded 36GiB to GPU in 2.1607s (16.6613GiB/s) | 20260505 11:35:41.026 | INFO | Downloaded 34GiB to GPU in 2.12461s (16.003GiB/s) | 20260505 11:35:46.167 | INFO | Analyzing chunk frames = [2894336,2894848) | 20260505 11:35:46.167 | INFO | MemInfo: { | 20260505 11:35:46.167 | INFO | "gpu" : " 17 GB", | 20260505 11:35:46.167 | INFO | "huge" : "", | 20260505 11:35:46.167 | INFO | "huge_shared" : " 27 GB", | 20260505 11:35:46.167 | INFO | "malloc" : " 14 GB", | 20260505 11:35:46.167 | INFO | "pinned" : " 29 GB" | 20260505 11:35:46.167 | INFO | } | 20260505 11:35:46.167 | INFO | Perf report for the last 5.14135 seconds: | 20260505 11:35:46.167 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:35:46.167 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:46.167 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:46.167 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:46.167 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 11:35:46.167 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:46.167 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:46.167 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 11:35:46.167 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:46.167 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:46.167 | INFO | Uploaded 36GiB to GPU in 2.16068s (16.6614GiB/s) | 20260505 11:35:46.167 | INFO | Downloaded 34GiB to GPU in 2.11129s (16.1039GiB/s) | 20260505 11:35:51.304 | INFO | Analyzing chunk frames = [2894848,2895360) | 20260505 11:35:51.304 | INFO | MemInfo: { | 20260505 11:35:51.304 | INFO | "gpu" : " 17 GB", | 20260505 11:35:51.304 | INFO | "huge" : "", | 20260505 11:35:51.304 | INFO | "huge_shared" : " 27 GB", | 20260505 11:35:51.304 | INFO | "malloc" : " 14 GB", | 20260505 11:35:51.304 | INFO | "pinned" : " 29 GB" | 20260505 11:35:51.304 | INFO | } | 20260505 11:35:51.304 | INFO | Perf report for the last 5.13723 seconds: | 20260505 11:35:51.304 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:35:51.304 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:51.304 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:51.304 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:51.304 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 11:35:51.304 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:51.304 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:51.304 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 11:35:51.304 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:51.304 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:51.305 | INFO | Uploaded 36GiB to GPU in 2.18425s (16.4816GiB/s) | 20260505 11:35:51.305 | INFO | Downloaded 34GiB to GPU in 2.13326s (15.9381GiB/s) | 20260505 11:35:56.443 | INFO | Analyzing chunk frames = [2895360,2895872) | 20260505 11:35:56.443 | INFO | MemInfo: { | 20260505 11:35:56.443 | INFO | "gpu" : " 17 GB", | 20260505 11:35:56.443 | INFO | "huge" : "", | 20260505 11:35:56.443 | INFO | "huge_shared" : " 27 GB", | 20260505 11:35:56.443 | INFO | "malloc" : " 14 GB", | 20260505 11:35:56.443 | INFO | "pinned" : " 29 GB" | 20260505 11:35:56.443 | INFO | } | 20260505 11:35:56.443 | INFO | Perf report for the last 5.13901 seconds: | 20260505 11:35:56.443 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:35:56.443 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:56.443 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:56.443 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:56.443 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 11:35:56.443 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:56.443 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:56.443 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 11:35:56.443 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:56.443 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:35:56.444 | INFO | Uploaded 36GiB to GPU in 2.21932s (16.2212GiB/s) | 20260505 11:35:56.444 | INFO | Downloaded 34GiB to GPU in 2.09283s (16.2459GiB/s) | 20260505 11:35:56.581 | INFO | Starting a round of Baz IO | 20260505 11:35:59.836 | INFO | Populating RTMetrics report | 20260505 11:35:59.836 | INFO | RTMetrics report populated | 20260505 11:35:59.836 | INFO | Opening tmp RTMetrics json file | 20260505 11:35:59.869 | INFO | Serializing RTMetrics to json | 20260505 11:35:59.881 | INFO | Serialization done, swapping tmp file now... | 20260505 11:35:59.899 | INFO | RTMetrics done | 20260505 11:35:59.957 | INFO | Finished a round of Baz IO: Wrote 2.373 GiB in 3.375 seconds | 20260505 11:35:59.957 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.08% / 0.01% / 75.92% | 20260505 11:35:59.957 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.10297% / 2.38419e-05% / 8.74201e-05% | 20260505 11:35:59.957 | INFO | Baz IO overall: | 20260505 11:35:59.957 | INFO | BytesWritten : 1713743384440 | 20260505 11:35:59.957 | INFO | NumEvents : 815952056491 | 20260505 11:36:01.576 | INFO | Analyzing chunk frames = [2895872,2896384) | 20260505 11:36:01.576 | INFO | MemInfo: { | 20260505 11:36:01.576 | INFO | "gpu" : " 17 GB", | 20260505 11:36:01.576 | INFO | "huge" : "", | 20260505 11:36:01.576 | INFO | "huge_shared" : " 27 GB", | 20260505 11:36:01.576 | INFO | "malloc" : " 14 GB", | 20260505 11:36:01.576 | INFO | "pinned" : " 29 GB" | 20260505 11:36:01.576 | INFO | } | 20260505 11:36:01.576 | INFO | Perf report for the last 5.13273 seconds: | 20260505 11:36:01.576 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:36:01.576 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:01.576 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:01.576 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:01.576 | INFO | ANALYSIS : 65.0%, 2.9, 99.9ms, 0 | 20260505 11:36:01.576 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:01.576 | INFO | RT_METRICS : 46.6%, 1.0, 24.4ms, 0 | 20260505 11:36:01.576 | INFO | PRE_HQ : 61.3%, 1.8, 58.8ms, 0 | 20260505 11:36:01.576 | INFO | BAZWRITER : 65.1%, 4.1, 136.9ms, 0 | 20260505 11:36:01.576 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 11:36:01.576 | INFO | Uploaded 36GiB to GPU in 2.19701s (16.3859GiB/s) | 20260505 11:36:01.576 | INFO | Downloaded 38GiB to GPU in 2.30289s (16.501GiB/s) | 20260505 11:36:06.722 | INFO | Analyzing chunk frames = [2896384,2896896) | 20260505 11:36:06.722 | INFO | MemInfo: { | 20260505 11:36:06.722 | INFO | "gpu" : " 17 GB", | 20260505 11:36:06.723 | INFO | "huge" : "", | 20260505 11:36:06.723 | INFO | "huge_shared" : " 27 GB", | 20260505 11:36:06.723 | INFO | "malloc" : " 14 GB", | 20260505 11:36:06.723 | INFO | "pinned" : " 29 GB" | 20260505 11:36:06.723 | INFO | } | 20260505 11:36:06.723 | INFO | Perf report for the last 5.14662 seconds: | 20260505 11:36:06.723 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:36:06.723 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:06.723 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:06.723 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:06.723 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 11:36:06.723 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:06.723 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:06.723 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:36:06.723 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:06.723 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:06.723 | INFO | Uploaded 36GiB to GPU in 2.20654s (16.3151GiB/s) | 20260505 11:36:06.723 | INFO | Downloaded 34GiB to GPU in 2.08168s (16.3329GiB/s) | 20260505 11:36:11.867 | INFO | Analyzing chunk frames = [2896896,2897408) | 20260505 11:36:11.867 | INFO | MemInfo: { | 20260505 11:36:11.867 | INFO | "gpu" : " 17 GB", | 20260505 11:36:11.867 | INFO | "huge" : "", | 20260505 11:36:11.868 | INFO | "huge_shared" : " 27 GB", | 20260505 11:36:11.868 | INFO | "malloc" : " 14 GB", | 20260505 11:36:11.868 | INFO | "pinned" : " 29 GB" | 20260505 11:36:11.868 | INFO | } | 20260505 11:36:11.868 | INFO | Perf report for the last 5.14497 seconds: | 20260505 11:36:11.868 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:36:11.868 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:11.868 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:11.868 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:11.868 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 11:36:11.868 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:11.868 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:11.868 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 11:36:11.868 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:11.868 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:11.868 | INFO | Uploaded 36GiB to GPU in 2.21474s (16.2547GiB/s) | 20260505 11:36:11.868 | INFO | Downloaded 34GiB to GPU in 2.11373s (16.0853GiB/s) | 20260505 11:36:17.009 | INFO | Analyzing chunk frames = [2897408,2897920) | 20260505 11:36:17.009 | INFO | MemInfo: { | 20260505 11:36:17.009 | INFO | "gpu" : " 17 GB", | 20260505 11:36:17.009 | INFO | "huge" : "", | 20260505 11:36:17.009 | INFO | "huge_shared" : " 27 GB", | 20260505 11:36:17.009 | INFO | "malloc" : " 14 GB", | 20260505 11:36:17.009 | INFO | "pinned" : " 29 GB" | 20260505 11:36:17.009 | INFO | } | 20260505 11:36:17.009 | INFO | Perf report for the last 5.14122 seconds: | 20260505 11:36:17.009 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:36:17.009 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:17.009 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:17.009 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:17.009 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 11:36:17.009 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:17.009 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:17.009 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 11:36:17.009 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:17.009 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:17.009 | INFO | Uploaded 36GiB to GPU in 2.20523s (16.3249GiB/s) | 20260505 11:36:17.009 | INFO | Downloaded 34GiB to GPU in 2.13065s (15.9576GiB/s) | 20260505 11:36:22.136 | INFO | Analyzing chunk frames = [2897920,2898432) | 20260505 11:36:22.136 | INFO | MemInfo: { | 20260505 11:36:22.136 | INFO | "gpu" : " 17 GB", | 20260505 11:36:22.136 | INFO | "huge" : "", | 20260505 11:36:22.136 | INFO | "huge_shared" : " 27 GB", | 20260505 11:36:22.136 | INFO | "malloc" : " 14 GB", | 20260505 11:36:22.136 | INFO | "pinned" : " 29 GB" | 20260505 11:36:22.136 | INFO | } | 20260505 11:36:22.136 | INFO | Perf report for the last 5.12756 seconds: | 20260505 11:36:22.136 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:36:22.136 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:22.136 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:22.136 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:22.136 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 11:36:22.136 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:22.137 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:22.137 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 11:36:22.137 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:22.137 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:22.137 | INFO | Uploaded 36GiB to GPU in 2.15676s (16.6917GiB/s) | 20260505 11:36:22.137 | INFO | Downloaded 34GiB to GPU in 2.1251s (15.9992GiB/s) | 20260505 11:36:27.276 | INFO | Analyzing chunk frames = [2898432,2898944) | 20260505 11:36:27.276 | INFO | MemInfo: { | 20260505 11:36:27.276 | INFO | "gpu" : " 17 GB", | 20260505 11:36:27.276 | INFO | "huge" : "", | 20260505 11:36:27.276 | INFO | "huge_shared" : " 27 GB", | 20260505 11:36:27.276 | INFO | "malloc" : " 14 GB", | 20260505 11:36:27.276 | INFO | "pinned" : " 29 GB" | 20260505 11:36:27.276 | INFO | } | 20260505 11:36:27.276 | INFO | Perf report for the last 5.13927 seconds: | 20260505 11:36:27.276 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:36:27.276 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:27.276 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:27.276 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:27.276 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 11:36:27.276 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:27.276 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:27.276 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 11:36:27.276 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:27.276 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:27.276 | INFO | Uploaded 36GiB to GPU in 2.20897s (16.2972GiB/s) | 20260505 11:36:27.276 | INFO | Downloaded 34GiB to GPU in 2.10405s (16.1593GiB/s) | 20260505 11:36:32.419 | INFO | Analyzing chunk frames = [2898944,2899456) | 20260505 11:36:32.419 | INFO | MemInfo: { | 20260505 11:36:32.419 | INFO | "gpu" : " 17 GB", | 20260505 11:36:32.419 | INFO | "huge" : "", | 20260505 11:36:32.419 | INFO | "huge_shared" : " 27 GB", | 20260505 11:36:32.419 | INFO | "malloc" : " 14 GB", | 20260505 11:36:32.419 | INFO | "pinned" : " 29 GB" | 20260505 11:36:32.419 | INFO | } | 20260505 11:36:32.419 | INFO | Perf report for the last 5.14356 seconds: | 20260505 11:36:32.419 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:36:32.419 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:32.419 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:32.419 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:32.419 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 11:36:32.419 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:32.419 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:32.419 | INFO | PRE_HQ : 29.9%, 1.0, 15.6ms, 0 | 20260505 11:36:32.419 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:32.419 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:32.419 | INFO | Uploaded 36GiB to GPU in 2.19317s (16.4146GiB/s) | 20260505 11:36:32.420 | INFO | Downloaded 34GiB to GPU in 2.14016s (15.8866GiB/s) | 20260505 11:36:37.560 | INFO | Analyzing chunk frames = [2899456,2899968) | 20260505 11:36:37.560 | INFO | MemInfo: { | 20260505 11:36:37.560 | INFO | "gpu" : " 17 GB", | 20260505 11:36:37.560 | INFO | "huge" : "", | 20260505 11:36:37.560 | INFO | "huge_shared" : " 27 GB", | 20260505 11:36:37.560 | INFO | "malloc" : " 14 GB", | 20260505 11:36:37.560 | INFO | "pinned" : " 29 GB" | 20260505 11:36:37.560 | INFO | } | 20260505 11:36:37.560 | INFO | Perf report for the last 5.14078 seconds: | 20260505 11:36:37.560 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:36:37.560 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:37.560 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:37.560 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:37.560 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 11:36:37.560 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:37.560 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:37.560 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:36:37.560 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:37.560 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:37.560 | INFO | Uploaded 36GiB to GPU in 2.21953s (16.2197GiB/s) | 20260505 11:36:37.560 | INFO | Downloaded 34GiB to GPU in 2.10377s (16.1614GiB/s) | 20260505 11:36:37.560 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 11:36:37.560 | INFO | Total mem usage: 15.3 GiB | 20260505 11:36:37.560 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 11:36:37.560 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.77%) | 20260505 11:36:37.560 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 11:36:37.560 | INFO | High Water Breakdown: | 20260505 11:36:37.560 | INFO | 61.1% : 8.77 GiB ( 92.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 11:36:37.560 | INFO | 38.8% : 5.57 GiB ( 83.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 11:36:37.560 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 11:36:37.560 | INFO | ------------------------------------------ | 20260505 11:36:37.560 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:36:37.560 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 11:36:37.560 | INFO | Total mem usage: 26.5 GiB | 20260505 11:36:37.560 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 11:36:37.560 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 11:36:37.560 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 11:36:37.561 | INFO | High Water Breakdown: | 20260505 11:36:37.561 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:36:37.561 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:36:37.561 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:36:37.561 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:36:37.561 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:36:37.561 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:36:37.561 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 11:36:37.561 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 11:36:37.561 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:36:37.561 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 11:36:37.561 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 11:36:37.561 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 11:36:37.561 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 11:36:37.561 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 11:36:37.561 | INFO | ------------------------------------------ | 20260505 11:36:37.561 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:36:37.561 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 11:36:37.561 | INFO | Total mem usage: 25.6 GiB | 20260505 11:36:37.561 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 11:36:37.561 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 11:36:37.561 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 11:36:37.561 | INFO | High Water Breakdown: | 20260505 11:36:37.561 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 11:36:37.561 | INFO | ------------------------------------------ | 20260505 11:36:37.561 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:36:37.561 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 11:36:37.561 | INFO | Total mem usage: 15.7 GiB | 20260505 11:36:37.561 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 11:36:37.561 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 11:36:37.561 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 11:36:37.561 | INFO | High Water Breakdown: | 20260505 11:36:37.561 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 11:36:37.561 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 11:36:37.561 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 11:36:37.561 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 11:36:37.561 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 11:36:37.561 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 11:36:37.561 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 11:36:37.561 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 11:36:37.561 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 11:36:37.561 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:36:37.561 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 11:36:37.561 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:36:37.561 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 11:36:37.561 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 11:36:37.561 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 11:36:37.561 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 11:36:37.561 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:36:37.561 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 11:36:37.561 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:36:37.561 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:36:37.561 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:36:37.561 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 11:36:37.561 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 11:36:37.561 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:36:37.561 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 11:36:37.561 | INFO | ------------------------------------------ | 20260505 11:36:37.561 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:36:37.561 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 11:36:37.561 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:36:37.561 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:36:37.561 | INFO | PerfReport: Full Execution -- 3.21e+01 / 7.73e+01, 1.38e+01 / 1.39e+00 | 20260505 11:36:37.561 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 1.02e-01, 1.02e-02 / 4.39e-03 | 20260505 11:36:37.561 | INFO | PerfReport: 11.74%, Baseline -- 3.77e+00 / 4.45e+00, 6.07e-01 / 2.32e-01 | 20260505 11:36:37.561 | INFO | PerfReport: 3.32%, Histogram -- 1.07e+00 / 1.50e+00, 2.06e-01 / 9.04e-02 | 20260505 11:36:37.561 | INFO | PerfReport: 15.78%, DME -- 4.05e+01 / 4.47e+01, 4.30e+00 / 2.07e+00 | 20260505 11:36:37.561 | INFO | PerfReport: 56.44%, FrameLabeling -- 1.81e+01 / 2.17e+01, 1.58e+00 / 1.84e+00 | 20260505 11:36:37.561 | INFO | PerfReport: 7.44%, PulseAccumulating -- 2.39e+00 / 2.82e+00, 4.26e-01 / 1.95e-01 | 20260505 11:36:37.561 | INFO | PerfReport: 3.70%, Metrics -- 1.19e+00 / 1.51e+00, 3.38e-01 / 1.98e-01 | 20260505 11:36:37.561 | INFO | PerfReport: 1.30%, PulseSerializing -- 4.17e-01 / 4.66e-01, 7.59e-02 / 2.64e-02 | 20260505 11:36:37.561 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 11:36:37.561 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:36:37.561 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:36:37.561 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.47e+01 / 0.00e+00 | 20260505 11:36:37.561 | INFO | PerfReport: 22.69%, Upload -- 2.30e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260505 11:36:37.561 | INFO | PerfReport: 22.06%, Download -- 2.23e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260505 11:36:37.561 | INFO | PerfReport: 31.73%, Compute -- 3.21e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 11:36:37.561 | INFO | PerfReport: 2.03%, LockUpload -- 2.06e+00 / 0.00e+00, 7.61e+00 / 0.00e+00 | 20260505 11:36:37.561 | INFO | PerfReport: 1.33%, LockDownload -- 1.35e+00 / 0.00e+00, 2.93e+00 / 0.00e+00 | 20260505 11:36:37.561 | INFO | PerfReport: 20.14%, LockCompute -- 2.04e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260505 11:36:37.561 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.04e-02 / 0.00e+00 | 20260505 11:36:37.696 | INFO | Starting a round of Baz IO | 20260505 11:36:40.948 | INFO | Populating RTMetrics report | 20260505 11:36:40.949 | INFO | RTMetrics report populated | 20260505 11:36:40.949 | INFO | Opening tmp RTMetrics json file | 20260505 11:36:40.949 | INFO | Serializing RTMetrics to json | 20260505 11:36:40.950 | INFO | Serialization done, swapping tmp file now... | 20260505 11:36:40.951 | INFO | RTMetrics done | 20260505 11:36:41.081 | INFO | Finished a round of Baz IO: Wrote 2.372 GiB in 3.385 seconds | 20260505 11:36:41.081 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.07% / 0.01% / 75.92% | 20260505 11:36:41.081 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.09758% / 1.58946e-05% / 9.13938e-05% | 20260505 11:36:41.081 | INFO | Baz IO overall: | 20260505 11:36:41.081 | INFO | BytesWritten : 1716289895305 | 20260505 11:36:41.081 | INFO | NumEvents : 817162703508 | 20260505 11:36:42.705 | INFO | Analyzing chunk frames = [2899968,2900480) | 20260505 11:36:42.705 | INFO | MemInfo: { | 20260505 11:36:42.705 | INFO | "gpu" : " 17 GB", | 20260505 11:36:42.705 | INFO | "huge" : "", | 20260505 11:36:42.705 | INFO | "huge_shared" : " 27 GB", | 20260505 11:36:42.705 | INFO | "malloc" : " 14 GB", | 20260505 11:36:42.705 | INFO | "pinned" : " 29 GB" | 20260505 11:36:42.705 | INFO | } | 20260505 11:36:42.705 | INFO | Perf report for the last 5.14474 seconds: | 20260505 11:36:42.705 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:36:42.705 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:42.705 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:42.705 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:42.705 | INFO | ANALYSIS : 64.7%, 2.9, 99.9ms, 0 | 20260505 11:36:42.705 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:42.705 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260505 11:36:42.705 | INFO | PRE_HQ : 61.0%, 1.9, 58.9ms, 0 | 20260505 11:36:42.705 | INFO | BAZWRITER : 65.1%, 4.1, 137.3ms, 0 | 20260505 11:36:42.705 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 11:36:42.705 | INFO | Uploaded 36GiB to GPU in 2.19291s (16.4165GiB/s) | 20260505 11:36:42.705 | INFO | Downloaded 38GiB to GPU in 2.29044s (16.5907GiB/s) | 20260505 11:36:47.839 | INFO | Analyzing chunk frames = [2900480,2900992) | 20260505 11:36:47.839 | INFO | MemInfo: { | 20260505 11:36:47.839 | INFO | "gpu" : " 17 GB", | 20260505 11:36:47.839 | INFO | "huge" : "", | 20260505 11:36:47.839 | INFO | "huge_shared" : " 27 GB", | 20260505 11:36:47.839 | INFO | "malloc" : " 14 GB", | 20260505 11:36:47.839 | INFO | "pinned" : " 29 GB" | 20260505 11:36:47.839 | INFO | } | 20260505 11:36:47.839 | INFO | Perf report for the last 5.13438 seconds: | 20260505 11:36:47.839 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:36:47.839 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:47.839 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:47.839 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:47.839 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 11:36:47.839 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:47.839 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:47.839 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 11:36:47.839 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:47.839 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:47.839 | INFO | Uploaded 36GiB to GPU in 2.18369s (16.4858GiB/s) | 20260505 11:36:47.839 | INFO | Downloaded 34GiB to GPU in 2.0564s (16.5337GiB/s) | 20260505 11:36:52.981 | INFO | Analyzing chunk frames = [2900992,2901504) | 20260505 11:36:52.981 | INFO | MemInfo: { | 20260505 11:36:52.981 | INFO | "gpu" : " 17 GB", | 20260505 11:36:52.981 | INFO | "huge" : "", | 20260505 11:36:52.981 | INFO | "huge_shared" : " 27 GB", | 20260505 11:36:52.981 | INFO | "malloc" : " 14 GB", | 20260505 11:36:52.981 | INFO | "pinned" : " 29 GB" | 20260505 11:36:52.981 | INFO | } | 20260505 11:36:52.981 | INFO | Perf report for the last 5.14219 seconds: | 20260505 11:36:52.982 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:36:52.982 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:52.982 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:52.982 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:52.982 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 11:36:52.982 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:52.982 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:52.982 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:36:52.982 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:52.982 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:52.982 | INFO | Uploaded 36GiB to GPU in 2.22143s (16.2058GiB/s) | 20260505 11:36:52.982 | INFO | Downloaded 34GiB to GPU in 2.12998s (15.9626GiB/s) | 20260505 11:36:58.120 | INFO | Analyzing chunk frames = [2901504,2902016) | 20260505 11:36:58.120 | INFO | MemInfo: { | 20260505 11:36:58.120 | INFO | "gpu" : " 17 GB", | 20260505 11:36:58.120 | INFO | "huge" : "", | 20260505 11:36:58.120 | INFO | "huge_shared" : " 27 GB", | 20260505 11:36:58.120 | INFO | "malloc" : " 14 GB", | 20260505 11:36:58.120 | INFO | "pinned" : " 29 GB" | 20260505 11:36:58.120 | INFO | } | 20260505 11:36:58.120 | INFO | Perf report for the last 5.1388 seconds: | 20260505 11:36:58.120 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:36:58.120 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:58.120 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:58.120 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:58.120 | INFO | ANALYSIS : 63.2%, 3.0, 98.1ms, 0 | 20260505 11:36:58.120 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:58.120 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:58.120 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 11:36:58.120 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:58.120 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:36:58.120 | INFO | Uploaded 36GiB to GPU in 2.20586s (16.3201GiB/s) | 20260505 11:36:58.120 | INFO | Downloaded 34GiB to GPU in 2.13133s (15.9525GiB/s) | 20260505 11:37:03.255 | INFO | Analyzing chunk frames = [2902016,2902528) | 20260505 11:37:03.255 | INFO | MemInfo: { | 20260505 11:37:03.255 | INFO | "gpu" : " 17 GB", | 20260505 11:37:03.255 | INFO | "huge" : "", | 20260505 11:37:03.255 | INFO | "huge_shared" : " 27 GB", | 20260505 11:37:03.255 | INFO | "malloc" : " 14 GB", | 20260505 11:37:03.255 | INFO | "pinned" : " 29 GB" | 20260505 11:37:03.255 | INFO | } | 20260505 11:37:03.255 | INFO | Perf report for the last 5.13484 seconds: | 20260505 11:37:03.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:37:03.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:03.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:03.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:03.255 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 11:37:03.255 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:03.255 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:03.255 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 11:37:03.255 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:03.255 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:03.255 | INFO | Uploaded 36GiB to GPU in 2.18168s (16.5011GiB/s) | 20260505 11:37:03.255 | INFO | Downloaded 34GiB to GPU in 2.12956s (15.9657GiB/s) | 20260505 11:37:08.395 | INFO | Analyzing chunk frames = [2902528,2903040) | 20260505 11:37:08.395 | INFO | MemInfo: { | 20260505 11:37:08.395 | INFO | "gpu" : " 17 GB", | 20260505 11:37:08.395 | INFO | "huge" : "", | 20260505 11:37:08.395 | INFO | "huge_shared" : " 27 GB", | 20260505 11:37:08.395 | INFO | "malloc" : " 14 GB", | 20260505 11:37:08.395 | INFO | "pinned" : " 29 GB" | 20260505 11:37:08.395 | INFO | } | 20260505 11:37:08.395 | INFO | Perf report for the last 5.14013 seconds: | 20260505 11:37:08.395 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:37:08.395 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:08.395 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:08.395 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:08.395 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 11:37:08.395 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:08.395 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:08.395 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:37:08.395 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:08.395 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:08.395 | INFO | Uploaded 36GiB to GPU in 2.19993s (16.3642GiB/s) | 20260505 11:37:08.395 | INFO | Downloaded 34GiB to GPU in 2.11995s (16.0381GiB/s) | 20260505 11:37:13.527 | INFO | Analyzing chunk frames = [2903040,2903552) | 20260505 11:37:13.527 | INFO | MemInfo: { | 20260505 11:37:13.527 | INFO | "gpu" : " 17 GB", | 20260505 11:37:13.527 | INFO | "huge" : "", | 20260505 11:37:13.527 | INFO | "huge_shared" : " 27 GB", | 20260505 11:37:13.527 | INFO | "malloc" : " 14 GB", | 20260505 11:37:13.527 | INFO | "pinned" : " 29 GB" | 20260505 11:37:13.527 | INFO | } | 20260505 11:37:13.527 | INFO | Perf report for the last 5.13197 seconds: | 20260505 11:37:13.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:37:13.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:13.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:13.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:13.527 | INFO | ANALYSIS : 63.5%, 3.0, 99.0ms, 0 | 20260505 11:37:13.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:13.527 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:13.527 | INFO | PRE_HQ : 30.2%, 1.0, 15.6ms, 0 | 20260505 11:37:13.527 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:13.527 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:13.527 | INFO | Uploaded 36GiB to GPU in 2.18599s (16.4685GiB/s) | 20260505 11:37:13.527 | INFO | Downloaded 34GiB to GPU in 2.15256s (15.7951GiB/s) | 20260505 11:37:18.680 | INFO | Analyzing chunk frames = [2903552,2904064) | 20260505 11:37:18.680 | INFO | MemInfo: { | 20260505 11:37:18.680 | INFO | "gpu" : " 17 GB", | 20260505 11:37:18.680 | INFO | "huge" : "", | 20260505 11:37:18.680 | INFO | "huge_shared" : " 27 GB", | 20260505 11:37:18.680 | INFO | "malloc" : " 14 GB", | 20260505 11:37:18.680 | INFO | "pinned" : " 29 GB" | 20260505 11:37:18.680 | INFO | } | 20260505 11:37:18.680 | INFO | Perf report for the last 5.15318 seconds: | 20260505 11:37:18.680 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:37:18.680 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:18.680 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:18.680 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:18.680 | INFO | ANALYSIS : 63.7%, 3.0, 99.8ms, 0 | 20260505 11:37:18.680 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:18.680 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:18.680 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 11:37:18.680 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:18.681 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:18.681 | INFO | Uploaded 36GiB to GPU in 2.22246s (16.1982GiB/s) | 20260505 11:37:18.681 | INFO | Downloaded 34GiB to GPU in 2.09902s (16.198GiB/s) | 20260505 11:37:18.814 | INFO | Starting a round of Baz IO | 20260505 11:37:22.050 | INFO | Populating RTMetrics report | 20260505 11:37:22.050 | INFO | RTMetrics report populated | 20260505 11:37:22.050 | INFO | Opening tmp RTMetrics json file | 20260505 11:37:22.054 | INFO | Serializing RTMetrics to json | 20260505 11:37:22.055 | INFO | Serialization done, swapping tmp file now... | 20260505 11:37:22.055 | INFO | RTMetrics done | 20260505 11:37:22.172 | INFO | Finished a round of Baz IO: Wrote 2.373 GiB in 3.358 seconds | 20260505 11:37:22.172 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.06% / 0.01% / 75.93% | 20260505 11:37:22.172 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.09871% / 1.19209e-05% / 4.37101e-05% | 20260505 11:37:22.172 | INFO | Baz IO overall: | 20260505 11:37:22.172 | INFO | BytesWritten : 1718837540821 | 20260505 11:37:22.172 | INFO | NumEvents : 818374671372 | 20260505 11:37:23.821 | INFO | Analyzing chunk frames = [2904064,2904576) | 20260505 11:37:23.822 | INFO | MemInfo: { | 20260505 11:37:23.822 | INFO | "gpu" : " 17 GB", | 20260505 11:37:23.822 | INFO | "huge" : "", | 20260505 11:37:23.822 | INFO | "huge_shared" : " 27 GB", | 20260505 11:37:23.822 | INFO | "malloc" : " 14 GB", | 20260505 11:37:23.822 | INFO | "pinned" : " 29 GB" | 20260505 11:37:23.822 | INFO | } | 20260505 11:37:23.822 | INFO | Perf report for the last 5.14122 seconds: | 20260505 11:37:23.822 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:37:23.822 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:23.822 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:23.822 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:23.822 | INFO | ANALYSIS : 64.4%, 2.9, 99.1ms, 0 | 20260505 11:37:23.822 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:23.822 | INFO | RT_METRICS : 45.5%, 1.0, 23.9ms, 0 | 20260505 11:37:23.822 | INFO | PRE_HQ : 61.5%, 1.8, 58.7ms, 0 | 20260505 11:37:23.822 | INFO | BAZWRITER : 64.6%, 4.1, 137.6ms, 0 | 20260505 11:37:23.822 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260505 11:37:23.822 | INFO | Uploaded 36GiB to GPU in 2.18757s (16.4566GiB/s) | 20260505 11:37:23.822 | INFO | Downloaded 38GiB to GPU in 2.27593s (16.6964GiB/s) | 20260505 11:37:28.955 | INFO | Analyzing chunk frames = [2904576,2905088) | 20260505 11:37:28.955 | INFO | MemInfo: { | 20260505 11:37:28.955 | INFO | "gpu" : " 17 GB", | 20260505 11:37:28.955 | INFO | "huge" : "", | 20260505 11:37:28.955 | INFO | "huge_shared" : " 27 GB", | 20260505 11:37:28.955 | INFO | "malloc" : " 14 GB", | 20260505 11:37:28.955 | INFO | "pinned" : " 29 GB" | 20260505 11:37:28.955 | INFO | } | 20260505 11:37:28.955 | INFO | Perf report for the last 5.13347 seconds: | 20260505 11:37:28.955 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:37:28.955 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:28.955 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:28.955 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:28.955 | INFO | ANALYSIS : 63.8%, 3.0, 99.5ms, 0 | 20260505 11:37:28.955 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:28.955 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:28.955 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 11:37:28.955 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:28.955 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:28.955 | INFO | Uploaded 36GiB to GPU in 2.17338s (16.564GiB/s) | 20260505 11:37:28.955 | INFO | Downloaded 34GiB to GPU in 2.03488s (16.7086GiB/s) | 20260505 11:37:34.086 | INFO | Analyzing chunk frames = [2905088,2905600) | 20260505 11:37:34.086 | INFO | MemInfo: { | 20260505 11:37:34.086 | INFO | "gpu" : " 17 GB", | 20260505 11:37:34.086 | INFO | "huge" : "", | 20260505 11:37:34.086 | INFO | "huge_shared" : " 27 GB", | 20260505 11:37:34.086 | INFO | "malloc" : " 14 GB", | 20260505 11:37:34.086 | INFO | "pinned" : " 29 GB" | 20260505 11:37:34.086 | INFO | } | 20260505 11:37:34.086 | INFO | Perf report for the last 5.13124 seconds: | 20260505 11:37:34.086 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:37:34.086 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:34.086 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:34.086 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:34.086 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260505 11:37:34.086 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:34.086 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:34.086 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:37:34.086 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:34.086 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:34.087 | INFO | Uploaded 36GiB to GPU in 2.23307s (16.1213GiB/s) | 20260505 11:37:34.087 | INFO | Downloaded 34GiB to GPU in 2.12688s (15.9859GiB/s) | 20260505 11:37:39.227 | INFO | Analyzing chunk frames = [2905600,2906112) | 20260505 11:37:39.227 | INFO | MemInfo: { | 20260505 11:37:39.227 | INFO | "gpu" : " 17 GB", | 20260505 11:37:39.227 | INFO | "huge" : "", | 20260505 11:37:39.227 | INFO | "huge_shared" : " 27 GB", | 20260505 11:37:39.227 | INFO | "malloc" : " 14 GB", | 20260505 11:37:39.227 | INFO | "pinned" : " 29 GB" | 20260505 11:37:39.227 | INFO | } | 20260505 11:37:39.227 | INFO | Perf report for the last 5.14097 seconds: | 20260505 11:37:39.227 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:37:39.227 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:39.227 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:39.227 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:39.227 | INFO | ANALYSIS : 62.9%, 3.0, 97.8ms, 0 | 20260505 11:37:39.227 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:39.227 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:39.227 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 11:37:39.227 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:39.227 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:39.228 | INFO | Uploaded 36GiB to GPU in 2.15447s (16.7095GiB/s) | 20260505 11:37:39.228 | INFO | Downloaded 34GiB to GPU in 2.0756s (16.3808GiB/s) | 20260505 11:37:44.371 | INFO | Analyzing chunk frames = [2906112,2906624) | 20260505 11:37:44.371 | INFO | MemInfo: { | 20260505 11:37:44.371 | INFO | "gpu" : " 17 GB", | 20260505 11:37:44.371 | INFO | "huge" : "", | 20260505 11:37:44.371 | INFO | "huge_shared" : " 27 GB", | 20260505 11:37:44.371 | INFO | "malloc" : " 14 GB", | 20260505 11:37:44.371 | INFO | "pinned" : " 29 GB" | 20260505 11:37:44.371 | INFO | } | 20260505 11:37:44.371 | INFO | Perf report for the last 5.14359 seconds: | 20260505 11:37:44.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:37:44.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:44.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:44.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:44.371 | INFO | ANALYSIS : 63.2%, 3.0, 98.8ms, 0 | 20260505 11:37:44.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:44.371 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:44.371 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 11:37:44.371 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:44.371 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:44.371 | INFO | Uploaded 36GiB to GPU in 2.16674s (16.6148GiB/s) | 20260505 11:37:44.371 | INFO | Downloaded 34GiB to GPU in 2.12598s (15.9927GiB/s) | 20260505 11:37:49.514 | INFO | Analyzing chunk frames = [2906624,2907136) | 20260505 11:37:49.514 | INFO | MemInfo: { | 20260505 11:37:49.514 | INFO | "gpu" : " 17 GB", | 20260505 11:37:49.514 | INFO | "huge" : "", | 20260505 11:37:49.514 | INFO | "huge_shared" : " 27 GB", | 20260505 11:37:49.514 | INFO | "malloc" : " 14 GB", | 20260505 11:37:49.514 | INFO | "pinned" : " 29 GB" | 20260505 11:37:49.514 | INFO | } | 20260505 11:37:49.514 | INFO | Perf report for the last 5.14345 seconds: | 20260505 11:37:49.514 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:37:49.514 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:49.514 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:49.514 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:49.514 | INFO | ANALYSIS : 63.2%, 3.0, 98.8ms, 0 | 20260505 11:37:49.514 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:49.514 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:49.514 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 11:37:49.514 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:49.514 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:49.515 | INFO | Uploaded 36GiB to GPU in 2.14604s (16.7751GiB/s) | 20260505 11:37:49.515 | INFO | Downloaded 34GiB to GPU in 2.10455s (16.1555GiB/s) | 20260505 11:37:54.653 | INFO | Analyzing chunk frames = [2907136,2907648) | 20260505 11:37:54.653 | INFO | MemInfo: { | 20260505 11:37:54.653 | INFO | "gpu" : " 17 GB", | 20260505 11:37:54.653 | INFO | "huge" : "", | 20260505 11:37:54.653 | INFO | "huge_shared" : " 27 GB", | 20260505 11:37:54.653 | INFO | "malloc" : " 14 GB", | 20260505 11:37:54.653 | INFO | "pinned" : " 29 GB" | 20260505 11:37:54.653 | INFO | } | 20260505 11:37:54.653 | INFO | Perf report for the last 5.13845 seconds: | 20260505 11:37:54.653 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:37:54.653 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:54.653 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:54.653 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:54.653 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 11:37:54.653 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:54.653 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:54.653 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260505 11:37:54.653 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:54.653 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:54.653 | INFO | Uploaded 36GiB to GPU in 2.16433s (16.6333GiB/s) | 20260505 11:37:54.653 | INFO | Downloaded 34GiB to GPU in 2.15627s (15.768GiB/s) | 20260505 11:37:59.788 | INFO | Analyzing chunk frames = [2907648,2908160) | 20260505 11:37:59.788 | INFO | MemInfo: { | 20260505 11:37:59.788 | INFO | "gpu" : " 17 GB", | 20260505 11:37:59.788 | INFO | "huge" : "", | 20260505 11:37:59.788 | INFO | "huge_shared" : " 27 GB", | 20260505 11:37:59.788 | INFO | "malloc" : " 14 GB", | 20260505 11:37:59.788 | INFO | "pinned" : " 29 GB" | 20260505 11:37:59.788 | INFO | } | 20260505 11:37:59.788 | INFO | Perf report for the last 5.13491 seconds: | 20260505 11:37:59.788 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:37:59.788 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:59.788 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:59.788 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:59.788 | INFO | ANALYSIS : 64.3%, 3.0, 100.3ms, 0 | 20260505 11:37:59.788 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:59.788 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:59.788 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 11:37:59.788 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:59.788 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:37:59.788 | INFO | Uploaded 36GiB to GPU in 2.24467s (16.038GiB/s) | 20260505 11:37:59.788 | INFO | Downloaded 34GiB to GPU in 2.09657s (16.2169GiB/s) | 20260505 11:37:59.788 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 11:37:59.788 | INFO | Total mem usage: 15.3 GiB | 20260505 11:37:59.788 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 11:37:59.788 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.77%) | 20260505 11:37:59.788 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 11:37:59.788 | INFO | High Water Breakdown: | 20260505 11:37:59.788 | INFO | 61.1% : 8.77 GiB ( 92.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 11:37:59.788 | INFO | 38.8% : 5.57 GiB ( 83.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 11:37:59.788 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 11:37:59.788 | INFO | ------------------------------------------ | 20260505 11:37:59.788 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:37:59.788 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 11:37:59.788 | INFO | Total mem usage: 26.5 GiB | 20260505 11:37:59.788 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 11:37:59.788 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 11:37:59.788 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 11:37:59.788 | INFO | High Water Breakdown: | 20260505 11:37:59.788 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:37:59.788 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:37:59.788 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:37:59.788 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:37:59.788 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:37:59.788 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:37:59.788 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 11:37:59.788 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 11:37:59.788 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:37:59.788 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 11:37:59.788 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 11:37:59.788 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 11:37:59.788 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 11:37:59.788 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 11:37:59.788 | INFO | ------------------------------------------ | 20260505 11:37:59.788 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:37:59.788 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 11:37:59.788 | INFO | Total mem usage: 25.6 GiB | 20260505 11:37:59.788 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 11:37:59.789 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 11:37:59.789 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 11:37:59.789 | INFO | High Water Breakdown: | 20260505 11:37:59.789 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 11:37:59.789 | INFO | ------------------------------------------ | 20260505 11:37:59.789 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:37:59.789 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 11:37:59.789 | INFO | Total mem usage: 15.7 GiB | 20260505 11:37:59.789 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 11:37:59.789 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 11:37:59.789 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 11:37:59.789 | INFO | High Water Breakdown: | 20260505 11:37:59.789 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 11:37:59.789 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 11:37:59.789 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 11:37:59.789 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 11:37:59.789 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 11:37:59.789 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 11:37:59.789 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 11:37:59.789 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 11:37:59.789 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 11:37:59.789 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:37:59.789 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 11:37:59.789 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:37:59.789 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 11:37:59.789 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 11:37:59.789 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 11:37:59.789 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 11:37:59.789 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:37:59.789 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 11:37:59.789 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:37:59.789 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:37:59.789 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:37:59.789 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 11:37:59.789 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 11:37:59.789 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:37:59.789 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 11:37:59.789 | INFO | ------------------------------------------ | 20260505 11:37:59.789 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:37:59.789 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 11:37:59.789 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:37:59.789 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:37:59.789 | INFO | PerfReport: Full Execution -- 3.22e+01 / 7.69e+01, 1.40e+01 / 1.65e+00 | 20260505 11:37:59.789 | INFO | PerfReport: 0.29%, AggregateStats -- 9.32e-02 / 1.03e-01, 1.09e-02 / 6.83e-03 | 20260505 11:37:59.789 | INFO | PerfReport: 11.74%, Baseline -- 3.78e+00 / 4.47e+00, 6.05e-01 / 2.04e-01 | 20260505 11:37:59.789 | INFO | PerfReport: 3.31%, Histogram -- 1.07e+00 / 1.50e+00, 2.07e-01 / 8.90e-02 | 20260505 11:37:59.789 | INFO | PerfReport: 15.77%, DME -- 4.06e+01 / 4.48e+01, 4.33e+00 / 2.13e+00 | 20260505 11:37:59.789 | INFO | PerfReport: 56.54%, FrameLabeling -- 1.82e+01 / 2.13e+01, 1.69e+00 / 2.05e+00 | 20260505 11:37:59.789 | INFO | PerfReport: 7.37%, PulseAccumulating -- 2.37e+00 / 2.85e+00, 4.24e-01 / 2.43e-01 | 20260505 11:37:59.789 | INFO | PerfReport: 3.69%, Metrics -- 1.19e+00 / 1.48e+00, 3.44e-01 / 2.18e-01 | 20260505 11:37:59.789 | INFO | PerfReport: 1.29%, PulseSerializing -- 4.17e-01 / 4.63e-01, 7.72e-02 / 4.56e-02 | 20260505 11:37:59.789 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 11:37:59.789 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:37:59.789 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:37:59.789 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260505 11:37:59.789 | INFO | PerfReport: 22.57%, Upload -- 2.28e+01 / 0.00e+00, 8.15e+00 / 0.00e+00 | 20260505 11:37:59.789 | INFO | PerfReport: 21.96%, Download -- 2.22e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260505 11:37:59.789 | INFO | PerfReport: 31.83%, Compute -- 3.22e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260505 11:37:59.789 | INFO | PerfReport: 1.99%, LockUpload -- 2.02e+00 / 0.00e+00, 7.58e+00 / 0.00e+00 | 20260505 11:37:59.789 | INFO | PerfReport: 1.24%, LockDownload -- 1.26e+00 / 0.00e+00, 2.76e+00 / 0.00e+00 | 20260505 11:37:59.789 | INFO | PerfReport: 20.39%, LockCompute -- 2.06e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260505 11:37:59.789 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.66e-02 / 0.00e+00 | 20260505 11:37:59.922 | INFO | Starting a round of Baz IO | 20260505 11:38:03.177 | INFO | Populating RTMetrics report | 20260505 11:38:03.177 | INFO | RTMetrics report populated | 20260505 11:38:03.177 | INFO | Opening tmp RTMetrics json file | 20260505 11:38:03.187 | INFO | Serializing RTMetrics to json | 20260505 11:38:03.192 | INFO | Serialization done, swapping tmp file now... | 20260505 11:38:03.192 | INFO | RTMetrics done | 20260505 11:38:03.307 | INFO | Finished a round of Baz IO: Wrote 2.371 GiB in 3.384 seconds | 20260505 11:38:03.307 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.05% / 0.01% / 75.94% | 20260505 11:38:03.307 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.09387% / 1.19209e-05% / 8.74201e-05% | 20260505 11:38:03.307 | INFO | Baz IO overall: | 20260505 11:38:03.307 | INFO | BytesWritten : 1721383582528 | 20260505 11:38:03.307 | INFO | NumEvents : 819585316715 | 20260505 11:38:04.934 | INFO | Analyzing chunk frames = [2908160,2908672) | 20260505 11:38:04.934 | INFO | MemInfo: { | 20260505 11:38:04.934 | INFO | "gpu" : " 17 GB", | 20260505 11:38:04.934 | INFO | "huge" : "", | 20260505 11:38:04.934 | INFO | "huge_shared" : " 27 GB", | 20260505 11:38:04.934 | INFO | "malloc" : " 14 GB", | 20260505 11:38:04.934 | INFO | "pinned" : " 29 GB" | 20260505 11:38:04.934 | INFO | } | 20260505 11:38:04.934 | INFO | Perf report for the last 5.14603 seconds: | 20260505 11:38:04.934 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:38:04.934 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:04.934 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:04.934 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:04.934 | INFO | ANALYSIS : 64.7%, 2.9, 99.9ms, 0 | 20260505 11:38:04.934 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:04.934 | INFO | RT_METRICS : 45.5%, 1.0, 23.9ms, 0 | 20260505 11:38:04.934 | INFO | PRE_HQ : 61.4%, 1.9, 59.0ms, 0 | 20260505 11:38:04.934 | INFO | BAZWRITER : 65.1%, 4.1, 137.9ms, 0 | 20260505 11:38:04.934 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 11:38:04.934 | INFO | Uploaded 36GiB to GPU in 2.20082s (16.3576GiB/s) | 20260505 11:38:04.934 | INFO | Downloaded 38GiB to GPU in 2.30036s (16.5192GiB/s) | 20260505 11:38:10.073 | INFO | Analyzing chunk frames = [2908672,2909184) | 20260505 11:38:10.073 | INFO | MemInfo: { | 20260505 11:38:10.073 | INFO | "gpu" : " 17 GB", | 20260505 11:38:10.073 | INFO | "huge" : "", | 20260505 11:38:10.073 | INFO | "huge_shared" : " 27 GB", | 20260505 11:38:10.073 | INFO | "malloc" : " 14 GB", | 20260505 11:38:10.073 | INFO | "pinned" : " 29 GB" | 20260505 11:38:10.073 | INFO | } | 20260505 11:38:10.073 | INFO | Perf report for the last 5.13903 seconds: | 20260505 11:38:10.073 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:38:10.073 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:10.073 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:10.073 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:10.073 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 11:38:10.073 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:10.073 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:10.073 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 11:38:10.073 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:10.073 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:10.073 | INFO | Uploaded 36GiB to GPU in 2.18541s (16.4729GiB/s) | 20260505 11:38:10.073 | INFO | Downloaded 34GiB to GPU in 2.04286s (16.6433GiB/s) | 20260505 11:38:15.203 | INFO | Analyzing chunk frames = [2909184,2909696) | 20260505 11:38:15.203 | INFO | MemInfo: { | 20260505 11:38:15.203 | INFO | "gpu" : " 17 GB", | 20260505 11:38:15.203 | INFO | "huge" : "", | 20260505 11:38:15.203 | INFO | "huge_shared" : " 27 GB", | 20260505 11:38:15.203 | INFO | "malloc" : " 14 GB", | 20260505 11:38:15.203 | INFO | "pinned" : " 29 GB" | 20260505 11:38:15.203 | INFO | } | 20260505 11:38:15.203 | INFO | Perf report for the last 5.12978 seconds: | 20260505 11:38:15.203 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:38:15.203 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:15.203 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:15.203 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:15.203 | INFO | ANALYSIS : 63.5%, 3.0, 98.6ms, 0 | 20260505 11:38:15.203 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:15.203 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:15.203 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 11:38:15.203 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:15.203 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:15.203 | INFO | Uploaded 36GiB to GPU in 2.21229s (16.2727GiB/s) | 20260505 11:38:15.203 | INFO | Downloaded 34GiB to GPU in 2.11357s (16.0865GiB/s) | 20260505 11:38:20.342 | INFO | Analyzing chunk frames = [2909696,2910208) | 20260505 11:38:20.342 | INFO | MemInfo: { | 20260505 11:38:20.342 | INFO | "gpu" : " 17 GB", | 20260505 11:38:20.342 | INFO | "huge" : "", | 20260505 11:38:20.342 | INFO | "huge_shared" : " 27 GB", | 20260505 11:38:20.342 | INFO | "malloc" : " 14 GB", | 20260505 11:38:20.342 | INFO | "pinned" : " 29 GB" | 20260505 11:38:20.342 | INFO | } | 20260505 11:38:20.342 | INFO | Perf report for the last 5.13977 seconds: | 20260505 11:38:20.342 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:38:20.342 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:20.343 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:20.343 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:20.343 | INFO | ANALYSIS : 62.9%, 3.0, 97.7ms, 0 | 20260505 11:38:20.343 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:20.343 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:20.343 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 11:38:20.343 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:20.343 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:20.343 | INFO | Uploaded 36GiB to GPU in 2.15111s (16.7355GiB/s) | 20260505 11:38:20.343 | INFO | Downloaded 34GiB to GPU in 2.07573s (16.3798GiB/s) | 20260505 11:38:25.487 | INFO | Analyzing chunk frames = [2910208,2910720) | 20260505 11:38:25.487 | INFO | MemInfo: { | 20260505 11:38:25.487 | INFO | "gpu" : " 17 GB", | 20260505 11:38:25.487 | INFO | "huge" : "", | 20260505 11:38:25.487 | INFO | "huge_shared" : " 27 GB", | 20260505 11:38:25.487 | INFO | "malloc" : " 14 GB", | 20260505 11:38:25.487 | INFO | "pinned" : " 29 GB" | 20260505 11:38:25.487 | INFO | } | 20260505 11:38:25.487 | INFO | Perf report for the last 5.14449 seconds: | 20260505 11:38:25.487 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:38:25.487 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:25.487 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:25.487 | INFO | SAVE_TRACE : 0.1%, 1.0, 0.1ms, 0 | 20260505 11:38:25.487 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 11:38:25.487 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:25.487 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:25.487 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:38:25.487 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:25.487 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:25.487 | INFO | Uploaded 36GiB to GPU in 2.15339s (16.7178GiB/s) | 20260505 11:38:25.487 | INFO | Downloaded 34GiB to GPU in 2.1217s (16.0249GiB/s) | 20260505 11:38:30.627 | INFO | Analyzing chunk frames = [2910720,2911232) | 20260505 11:38:30.627 | INFO | MemInfo: { | 20260505 11:38:30.627 | INFO | "gpu" : " 17 GB", | 20260505 11:38:30.627 | INFO | "huge" : "", | 20260505 11:38:30.627 | INFO | "huge_shared" : " 27 GB", | 20260505 11:38:30.627 | INFO | "malloc" : " 14 GB", | 20260505 11:38:30.627 | INFO | "pinned" : " 29 GB" | 20260505 11:38:30.627 | INFO | } | 20260505 11:38:30.627 | INFO | Perf report for the last 5.13984 seconds: | 20260505 11:38:30.627 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:38:30.627 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:30.627 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:30.627 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:30.627 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 11:38:30.627 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:30.627 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:30.627 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 11:38:30.627 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:30.627 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:30.627 | INFO | Uploaded 36GiB to GPU in 2.15499s (16.7054GiB/s) | 20260505 11:38:30.627 | INFO | Downloaded 34GiB to GPU in 2.1082s (16.1275GiB/s) | 20260505 11:38:35.765 | INFO | Analyzing chunk frames = [2911232,2911744) | 20260505 11:38:35.765 | INFO | MemInfo: { | 20260505 11:38:35.765 | INFO | "gpu" : " 17 GB", | 20260505 11:38:35.765 | INFO | "huge" : "", | 20260505 11:38:35.765 | INFO | "huge_shared" : " 27 GB", | 20260505 11:38:35.765 | INFO | "malloc" : " 14 GB", | 20260505 11:38:35.766 | INFO | "pinned" : " 29 GB" | 20260505 11:38:35.766 | INFO | } | 20260505 11:38:35.766 | INFO | Perf report for the last 5.13875 seconds: | 20260505 11:38:35.766 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:38:35.766 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:35.766 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:35.766 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:35.766 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 11:38:35.766 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:35.766 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:35.766 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 11:38:35.766 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:35.766 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:35.766 | INFO | Uploaded 36GiB to GPU in 2.17069s (16.5846GiB/s) | 20260505 11:38:35.766 | INFO | Downloaded 34GiB to GPU in 2.14516s (15.8497GiB/s) | 20260505 11:38:40.901 | INFO | Analyzing chunk frames = [2911744,2912256) | 20260505 11:38:40.901 | INFO | MemInfo: { | 20260505 11:38:40.901 | INFO | "gpu" : " 17 GB", | 20260505 11:38:40.901 | INFO | "huge" : "", | 20260505 11:38:40.901 | INFO | "huge_shared" : " 27 GB", | 20260505 11:38:40.901 | INFO | "malloc" : " 14 GB", | 20260505 11:38:40.901 | INFO | "pinned" : " 29 GB" | 20260505 11:38:40.901 | INFO | } | 20260505 11:38:40.901 | INFO | Perf report for the last 5.1356 seconds: | 20260505 11:38:40.901 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:38:40.901 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:40.901 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:40.901 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:40.901 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 11:38:40.901 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:40.901 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:40.901 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 11:38:40.901 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:40.901 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:40.901 | INFO | Uploaded 36GiB to GPU in 2.21426s (16.2582GiB/s) | 20260505 11:38:40.901 | INFO | Downloaded 34GiB to GPU in 2.09074s (16.2622GiB/s) | 20260505 11:38:41.040 | INFO | Starting a round of Baz IO | 20260505 11:38:44.295 | INFO | Populating RTMetrics report | 20260505 11:38:44.295 | INFO | RTMetrics report populated | 20260505 11:38:44.295 | INFO | Opening tmp RTMetrics json file | 20260505 11:38:44.298 | INFO | Serializing RTMetrics to json | 20260505 11:38:44.300 | INFO | Serialization done, swapping tmp file now... | 20260505 11:38:44.302 | INFO | RTMetrics done | 20260505 11:38:44.435 | INFO | Finished a round of Baz IO: Wrote 2.372 GiB in 3.395 seconds | 20260505 11:38:44.435 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.05% / 0.01% / 75.95% | 20260505 11:38:44.435 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.09186% / 1.98682e-05% / 8.34465e-05% | 20260505 11:38:44.435 | INFO | Baz IO overall: | 20260505 11:38:44.435 | INFO | BytesWritten : 1723930976641 | 20260505 11:38:44.435 | INFO | NumEvents : 820797185338 | 20260505 11:38:46.040 | INFO | Analyzing chunk frames = [2912256,2912768) | 20260505 11:38:46.040 | INFO | MemInfo: { | 20260505 11:38:46.040 | INFO | "gpu" : " 17 GB", | 20260505 11:38:46.040 | INFO | "huge" : "", | 20260505 11:38:46.040 | INFO | "huge_shared" : " 27 GB", | 20260505 11:38:46.040 | INFO | "malloc" : " 14 GB", | 20260505 11:38:46.040 | INFO | "pinned" : " 29 GB" | 20260505 11:38:46.040 | INFO | } | 20260505 11:38:46.040 | INFO | Perf report for the last 5.13907 seconds: | 20260505 11:38:46.040 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:38:46.040 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:46.040 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:46.040 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:46.040 | INFO | ANALYSIS : 64.9%, 2.9, 100.0ms, 0 | 20260505 11:38:46.040 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:46.040 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260505 11:38:46.040 | INFO | PRE_HQ : 61.4%, 1.8, 58.7ms, 0 | 20260505 11:38:46.040 | INFO | BAZWRITER : 65.3%, 4.1, 137.7ms, 0 | 20260505 11:38:46.040 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 11:38:46.041 | INFO | Uploaded 36GiB to GPU in 2.20476s (16.3283GiB/s) | 20260505 11:38:46.041 | INFO | Downloaded 38GiB to GPU in 2.30397s (16.4933GiB/s) | 20260505 11:38:51.179 | INFO | Analyzing chunk frames = [2912768,2913280) | 20260505 11:38:51.179 | INFO | MemInfo: { | 20260505 11:38:51.179 | INFO | "gpu" : " 17 GB", | 20260505 11:38:51.179 | INFO | "huge" : "", | 20260505 11:38:51.179 | INFO | "huge_shared" : " 27 GB", | 20260505 11:38:51.179 | INFO | "malloc" : " 14 GB", | 20260505 11:38:51.179 | INFO | "pinned" : " 29 GB" | 20260505 11:38:51.179 | INFO | } | 20260505 11:38:51.179 | INFO | Perf report for the last 5.13888 seconds: | 20260505 11:38:51.179 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:38:51.179 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:51.179 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:51.179 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:51.179 | INFO | ANALYSIS : 64.3%, 3.0, 100.3ms, 0 | 20260505 11:38:51.179 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:51.179 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:51.179 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 11:38:51.179 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:51.179 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:51.179 | INFO | Uploaded 36GiB to GPU in 2.2365s (16.0966GiB/s) | 20260505 11:38:51.179 | INFO | Downloaded 34GiB to GPU in 2.08522s (16.3052GiB/s) | 20260505 11:38:56.326 | INFO | Analyzing chunk frames = [2913280,2913792) | 20260505 11:38:56.326 | INFO | MemInfo: { | 20260505 11:38:56.326 | INFO | "gpu" : " 17 GB", | 20260505 11:38:56.326 | INFO | "huge" : "", | 20260505 11:38:56.326 | INFO | "huge_shared" : " 27 GB", | 20260505 11:38:56.326 | INFO | "malloc" : " 14 GB", | 20260505 11:38:56.326 | INFO | "pinned" : " 29 GB" | 20260505 11:38:56.326 | INFO | } | 20260505 11:38:56.326 | INFO | Perf report for the last 5.1472 seconds: | 20260505 11:38:56.326 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:38:56.326 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:56.327 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:56.327 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:56.327 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 11:38:56.327 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:56.327 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:56.327 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:38:56.327 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:56.327 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:38:56.327 | INFO | Uploaded 36GiB to GPU in 2.23011s (16.1427GiB/s) | 20260505 11:38:56.327 | INFO | Downloaded 34GiB to GPU in 2.12037s (16.0349GiB/s) | 20260505 11:39:01.464 | INFO | Analyzing chunk frames = [2913792,2914304) | 20260505 11:39:01.464 | INFO | MemInfo: { | 20260505 11:39:01.464 | INFO | "gpu" : " 17 GB", | 20260505 11:39:01.464 | INFO | "huge" : "", | 20260505 11:39:01.464 | INFO | "huge_shared" : " 27 GB", | 20260505 11:39:01.464 | INFO | "malloc" : " 14 GB", | 20260505 11:39:01.464 | INFO | "pinned" : " 29 GB" | 20260505 11:39:01.465 | INFO | } | 20260505 11:39:01.465 | INFO | Perf report for the last 5.13817 seconds: | 20260505 11:39:01.465 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:39:01.465 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:01.465 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:01.465 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:01.465 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 11:39:01.465 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:01.465 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:01.465 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:39:01.465 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:01.465 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:01.465 | INFO | Uploaded 36GiB to GPU in 2.20668s (16.3141GiB/s) | 20260505 11:39:01.465 | INFO | Downloaded 34GiB to GPU in 2.12779s (15.9791GiB/s) | 20260505 11:39:06.597 | INFO | Analyzing chunk frames = [2914304,2914816) | 20260505 11:39:06.597 | INFO | MemInfo: { | 20260505 11:39:06.597 | INFO | "gpu" : " 17 GB", | 20260505 11:39:06.597 | INFO | "huge" : "", | 20260505 11:39:06.597 | INFO | "huge_shared" : " 27 GB", | 20260505 11:39:06.597 | INFO | "malloc" : " 14 GB", | 20260505 11:39:06.597 | INFO | "pinned" : " 29 GB" | 20260505 11:39:06.597 | INFO | } | 20260505 11:39:06.597 | INFO | Perf report for the last 5.13272 seconds: | 20260505 11:39:06.597 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:39:06.597 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:06.597 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:06.597 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:06.597 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 11:39:06.597 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:06.597 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:06.597 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 11:39:06.597 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:06.597 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:06.598 | INFO | Uploaded 36GiB to GPU in 2.18874s (16.4478GiB/s) | 20260505 11:39:06.598 | INFO | Downloaded 34GiB to GPU in 2.12466s (16.0026GiB/s) | 20260505 11:39:11.741 | INFO | Analyzing chunk frames = [2914816,2915328) | 20260505 11:39:11.741 | INFO | MemInfo: { | 20260505 11:39:11.741 | INFO | "gpu" : " 17 GB", | 20260505 11:39:11.741 | INFO | "huge" : "", | 20260505 11:39:11.741 | INFO | "huge_shared" : " 27 GB", | 20260505 11:39:11.741 | INFO | "malloc" : " 14 GB", | 20260505 11:39:11.741 | INFO | "pinned" : " 29 GB" | 20260505 11:39:11.741 | INFO | } | 20260505 11:39:11.741 | INFO | Perf report for the last 5.14343 seconds: | 20260505 11:39:11.741 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:39:11.741 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:11.741 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:11.741 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:11.741 | INFO | ANALYSIS : 63.5%, 3.0, 99.3ms, 0 | 20260505 11:39:11.741 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:11.741 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:11.741 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 11:39:11.741 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:11.741 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:11.741 | INFO | Uploaded 36GiB to GPU in 2.16967s (16.5924GiB/s) | 20260505 11:39:11.741 | INFO | Downloaded 34GiB to GPU in 2.12067s (16.0327GiB/s) | 20260505 11:39:16.878 | INFO | Analyzing chunk frames = [2915328,2915840) | 20260505 11:39:16.878 | INFO | MemInfo: { | 20260505 11:39:16.878 | INFO | "gpu" : " 17 GB", | 20260505 11:39:16.878 | INFO | "huge" : "", | 20260505 11:39:16.878 | INFO | "huge_shared" : " 27 GB", | 20260505 11:39:16.878 | INFO | "malloc" : " 14 GB", | 20260505 11:39:16.878 | INFO | "pinned" : " 29 GB" | 20260505 11:39:16.878 | INFO | } | 20260505 11:39:16.878 | INFO | Perf report for the last 5.13751 seconds: | 20260505 11:39:16.878 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:39:16.878 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:16.878 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:16.878 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:16.878 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 11:39:16.878 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:16.878 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:16.878 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 11:39:16.878 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:16.878 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:16.878 | INFO | Uploaded 36GiB to GPU in 2.16009s (16.6659GiB/s) | 20260505 11:39:16.878 | INFO | Downloaded 34GiB to GPU in 2.12515s (15.9989GiB/s) | 20260505 11:39:22.025 | INFO | Analyzing chunk frames = [2915840,2916352) | 20260505 11:39:22.025 | INFO | MemInfo: { | 20260505 11:39:22.025 | INFO | "gpu" : " 17 GB", | 20260505 11:39:22.025 | INFO | "huge" : "", | 20260505 11:39:22.025 | INFO | "huge_shared" : " 27 GB", | 20260505 11:39:22.026 | INFO | "malloc" : " 14 GB", | 20260505 11:39:22.026 | INFO | "pinned" : " 29 GB" | 20260505 11:39:22.026 | INFO | } | 20260505 11:39:22.026 | INFO | Perf report for the last 5.14738 seconds: | 20260505 11:39:22.026 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:39:22.026 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:22.026 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:22.026 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:22.026 | INFO | ANALYSIS : 63.7%, 3.0, 99.6ms, 0 | 20260505 11:39:22.026 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:22.026 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:22.026 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 11:39:22.026 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:22.026 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:22.026 | INFO | Uploaded 36GiB to GPU in 2.21782s (16.2322GiB/s) | 20260505 11:39:22.026 | INFO | Downloaded 34GiB to GPU in 2.09186s (16.2535GiB/s) | 20260505 11:39:22.026 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 11:39:22.026 | INFO | Total mem usage: 15.3 GiB | 20260505 11:39:22.026 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 11:39:22.026 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.76%) | 20260505 11:39:22.026 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 11:39:22.026 | INFO | High Water Breakdown: | 20260505 11:39:22.026 | INFO | 61.1% : 8.77 GiB ( 92.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 11:39:22.026 | INFO | 38.8% : 5.57 GiB ( 83.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 11:39:22.026 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 11:39:22.026 | INFO | ------------------------------------------ | 20260505 11:39:22.026 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:39:22.026 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 11:39:22.026 | INFO | Total mem usage: 26.5 GiB | 20260505 11:39:22.026 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 11:39:22.026 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 11:39:22.026 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 11:39:22.026 | INFO | High Water Breakdown: | 20260505 11:39:22.026 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:39:22.026 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:39:22.026 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:39:22.026 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:39:22.026 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:39:22.026 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:39:22.026 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 11:39:22.026 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 11:39:22.026 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:39:22.026 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 11:39:22.026 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 11:39:22.026 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 11:39:22.026 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 11:39:22.026 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 11:39:22.026 | INFO | ------------------------------------------ | 20260505 11:39:22.026 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:39:22.026 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 11:39:22.026 | INFO | Total mem usage: 25.6 GiB | 20260505 11:39:22.026 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 11:39:22.026 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 11:39:22.026 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 11:39:22.026 | INFO | High Water Breakdown: | 20260505 11:39:22.026 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 11:39:22.026 | INFO | ------------------------------------------ | 20260505 11:39:22.026 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:39:22.026 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 11:39:22.026 | INFO | Total mem usage: 15.7 GiB | 20260505 11:39:22.026 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 11:39:22.026 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 11:39:22.026 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 11:39:22.026 | INFO | High Water Breakdown: | 20260505 11:39:22.026 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 11:39:22.026 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 11:39:22.026 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 11:39:22.026 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 11:39:22.026 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 11:39:22.026 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 11:39:22.026 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 11:39:22.026 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 11:39:22.027 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 11:39:22.027 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:39:22.027 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 11:39:22.027 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:39:22.027 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 11:39:22.027 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 11:39:22.027 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 11:39:22.027 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 11:39:22.027 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:39:22.027 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 11:39:22.027 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:39:22.027 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:39:22.027 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:39:22.027 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 11:39:22.027 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 11:39:22.027 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:39:22.027 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 11:39:22.027 | INFO | ------------------------------------------ | 20260505 11:39:22.027 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:39:22.027 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 11:39:22.027 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:39:22.027 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:39:22.027 | INFO | PerfReport: Full Execution -- 3.21e+01 / 7.71e+01, 1.35e+01 / 1.81e+00 | 20260505 11:39:22.027 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 1.01e-01, 1.03e-02 / 3.78e-03 | 20260505 11:39:22.027 | INFO | PerfReport: 11.74%, Baseline -- 3.77e+00 / 4.48e+00, 6.03e-01 / 2.23e-01 | 20260505 11:39:22.027 | INFO | PerfReport: 3.31%, Histogram -- 1.06e+00 / 1.50e+00, 2.03e-01 / 8.75e-02 | 20260505 11:39:22.027 | INFO | PerfReport: 15.84%, DME -- 4.07e+01 / 4.52e+01, 4.47e+00 / 2.02e+00 | 20260505 11:39:22.027 | INFO | PerfReport: 56.39%, FrameLabeling -- 1.81e+01 / 2.10e+01, 1.55e+00 / 2.15e+00 | 20260505 11:39:22.027 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.39e+00 / 2.86e+00, 4.37e-01 / 2.45e-01 | 20260505 11:39:22.027 | INFO | PerfReport: 3.68%, Metrics -- 1.18e+00 / 1.52e+00, 3.34e-01 / 1.89e-01 | 20260505 11:39:22.027 | INFO | PerfReport: 1.30%, PulseSerializing -- 4.16e-01 / 4.71e-01, 7.71e-02 / 3.11e-02 | 20260505 11:39:22.027 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 11:39:22.027 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:39:22.027 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:39:22.027 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260505 11:39:22.027 | INFO | PerfReport: 22.57%, Upload -- 2.28e+01 / 0.00e+00, 8.14e+00 / 0.00e+00 | 20260505 11:39:22.027 | INFO | PerfReport: 21.95%, Download -- 2.22e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260505 11:39:22.027 | INFO | PerfReport: 31.73%, Compute -- 3.21e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 11:39:22.027 | INFO | PerfReport: 2.11%, LockUpload -- 2.14e+00 / 0.00e+00, 7.76e+00 / 0.00e+00 | 20260505 11:39:22.027 | INFO | PerfReport: 1.31%, LockDownload -- 1.33e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260505 11:39:22.027 | INFO | PerfReport: 20.30%, LockCompute -- 2.05e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260505 11:39:22.027 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.91e-02 / 0.00e+00 | 20260505 11:39:22.162 | INFO | Starting a round of Baz IO | 20260505 11:39:25.409 | INFO | Populating RTMetrics report | 20260505 11:39:25.410 | INFO | RTMetrics report populated | 20260505 11:39:25.410 | INFO | Opening tmp RTMetrics json file | 20260505 11:39:25.410 | INFO | Serializing RTMetrics to json | 20260505 11:39:25.412 | INFO | Serialization done, swapping tmp file now... | 20260505 11:39:25.412 | INFO | RTMetrics done | 20260505 11:39:25.529 | INFO | Finished a round of Baz IO: Wrote 2.372 GiB in 3.367 seconds | 20260505 11:39:25.529 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.04% / 0.01% / 75.95% | 20260505 11:39:25.529 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.0881% / 1.58946e-05% / 6.75519e-05% | 20260505 11:39:25.530 | INFO | Baz IO overall: | 20260505 11:39:25.530 | INFO | BytesWritten : 1726477810499 | 20260505 11:39:25.530 | INFO | NumEvents : 822008367984 | 20260505 11:39:27.153 | INFO | Analyzing chunk frames = [2916352,2916864) | 20260505 11:39:27.153 | INFO | MemInfo: { | 20260505 11:39:27.153 | INFO | "gpu" : " 17 GB", | 20260505 11:39:27.153 | INFO | "huge" : "", | 20260505 11:39:27.153 | INFO | "huge_shared" : " 27 GB", | 20260505 11:39:27.153 | INFO | "malloc" : " 14 GB", | 20260505 11:39:27.153 | INFO | "pinned" : " 29 GB" | 20260505 11:39:27.153 | INFO | } | 20260505 11:39:27.154 | INFO | Perf report for the last 5.12786 seconds: | 20260505 11:39:27.154 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:39:27.154 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:27.154 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:27.154 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:27.154 | INFO | ANALYSIS : 64.9%, 2.9, 99.9ms, 0 | 20260505 11:39:27.154 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:27.154 | INFO | RT_METRICS : 45.5%, 1.0, 23.8ms, 0 | 20260505 11:39:27.154 | INFO | PRE_HQ : 61.2%, 1.9, 58.8ms, 0 | 20260505 11:39:27.154 | INFO | BAZWRITER : 65.0%, 4.1, 135.8ms, 0 | 20260505 11:39:27.154 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 11:39:27.154 | INFO | Uploaded 36GiB to GPU in 2.18962s (16.4412GiB/s) | 20260505 11:39:27.154 | INFO | Downloaded 38GiB to GPU in 2.28876s (16.6029GiB/s) | 20260505 11:39:32.294 | INFO | Analyzing chunk frames = [2916864,2917376) | 20260505 11:39:32.294 | INFO | MemInfo: { | 20260505 11:39:32.294 | INFO | "gpu" : " 17 GB", | 20260505 11:39:32.294 | INFO | "huge" : "", | 20260505 11:39:32.294 | INFO | "huge_shared" : " 27 GB", | 20260505 11:39:32.294 | INFO | "malloc" : " 14 GB", | 20260505 11:39:32.294 | INFO | "pinned" : " 29 GB" | 20260505 11:39:32.294 | INFO | } | 20260505 11:39:32.295 | INFO | Perf report for the last 5.14102 seconds: | 20260505 11:39:32.295 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:39:32.295 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:32.295 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:32.295 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:32.295 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 11:39:32.295 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:32.295 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:32.295 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 11:39:32.295 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:32.295 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:32.295 | INFO | Uploaded 36GiB to GPU in 2.20813s (16.3034GiB/s) | 20260505 11:39:32.295 | INFO | Downloaded 34GiB to GPU in 2.08466s (16.3096GiB/s) | 20260505 11:39:37.447 | INFO | Analyzing chunk frames = [2917376,2917888) | 20260505 11:39:37.447 | INFO | MemInfo: { | 20260505 11:39:37.447 | INFO | "gpu" : " 17 GB", | 20260505 11:39:37.447 | INFO | "huge" : "", | 20260505 11:39:37.447 | INFO | "huge_shared" : " 27 GB", | 20260505 11:39:37.447 | INFO | "malloc" : " 14 GB", | 20260505 11:39:37.447 | INFO | "pinned" : " 29 GB" | 20260505 11:39:37.447 | INFO | } | 20260505 11:39:37.447 | INFO | Perf report for the last 5.15283 seconds: | 20260505 11:39:37.447 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:39:37.447 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:37.447 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:37.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:37.447 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 11:39:37.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:37.447 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:37.447 | INFO | PRE_HQ : 28.7%, 1.0, 15.0ms, 0 | 20260505 11:39:37.447 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:37.447 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:37.448 | INFO | Uploaded 36GiB to GPU in 2.23298s (16.122GiB/s) | 20260505 11:39:37.448 | INFO | Downloaded 34GiB to GPU in 2.13182s (15.9488GiB/s) | 20260505 11:39:42.584 | INFO | Analyzing chunk frames = [2917888,2918400) | 20260505 11:39:42.584 | INFO | MemInfo: { | 20260505 11:39:42.584 | INFO | "gpu" : " 17 GB", | 20260505 11:39:42.584 | INFO | "huge" : "", | 20260505 11:39:42.584 | INFO | "huge_shared" : " 27 GB", | 20260505 11:39:42.584 | INFO | "malloc" : " 14 GB", | 20260505 11:39:42.584 | INFO | "pinned" : " 29 GB" | 20260505 11:39:42.584 | INFO | } | 20260505 11:39:42.584 | INFO | Perf report for the last 5.13647 seconds: | 20260505 11:39:42.584 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:39:42.584 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:42.584 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:42.584 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:42.584 | INFO | ANALYSIS : 63.0%, 3.0, 97.9ms, 0 | 20260505 11:39:42.584 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:42.584 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:42.584 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 11:39:42.584 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:42.584 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:42.584 | INFO | Uploaded 36GiB to GPU in 2.19484s (16.4021GiB/s) | 20260505 11:39:42.584 | INFO | Downloaded 34GiB to GPU in 2.12946s (15.9665GiB/s) | 20260505 11:39:47.713 | INFO | Analyzing chunk frames = [2918400,2918912) | 20260505 11:39:47.713 | INFO | MemInfo: { | 20260505 11:39:47.713 | INFO | "gpu" : " 17 GB", | 20260505 11:39:47.713 | INFO | "huge" : "", | 20260505 11:39:47.713 | INFO | "huge_shared" : " 27 GB", | 20260505 11:39:47.713 | INFO | "malloc" : " 14 GB", | 20260505 11:39:47.713 | INFO | "pinned" : " 29 GB" | 20260505 11:39:47.713 | INFO | } | 20260505 11:39:47.713 | INFO | Perf report for the last 5.1292 seconds: | 20260505 11:39:47.713 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:39:47.713 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:47.713 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:47.713 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:47.713 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 11:39:47.713 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:47.713 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:47.713 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 11:39:47.713 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:47.713 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:47.714 | INFO | Uploaded 36GiB to GPU in 2.17879s (16.523GiB/s) | 20260505 11:39:47.714 | INFO | Downloaded 34GiB to GPU in 2.12947s (15.9664GiB/s) | 20260505 11:39:52.846 | INFO | Analyzing chunk frames = [2918912,2919424) | 20260505 11:39:52.846 | INFO | MemInfo: { | 20260505 11:39:52.846 | INFO | "gpu" : " 17 GB", | 20260505 11:39:52.846 | INFO | "huge" : "", | 20260505 11:39:52.846 | INFO | "huge_shared" : " 27 GB", | 20260505 11:39:52.846 | INFO | "malloc" : " 14 GB", | 20260505 11:39:52.846 | INFO | "pinned" : " 29 GB" | 20260505 11:39:52.846 | INFO | } | 20260505 11:39:52.846 | INFO | Perf report for the last 5.13262 seconds: | 20260505 11:39:52.846 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:39:52.846 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:52.846 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:52.846 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:52.846 | INFO | ANALYSIS : 63.5%, 3.0, 99.0ms, 0 | 20260505 11:39:52.846 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:52.846 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:52.846 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:39:52.846 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:52.846 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:52.846 | INFO | Uploaded 36GiB to GPU in 2.20799s (16.3044GiB/s) | 20260505 11:39:52.846 | INFO | Downloaded 34GiB to GPU in 2.1102s (16.1122GiB/s) | 20260505 11:39:57.995 | INFO | Analyzing chunk frames = [2919424,2919936) | 20260505 11:39:57.995 | INFO | MemInfo: { | 20260505 11:39:57.995 | INFO | "gpu" : " 17 GB", | 20260505 11:39:57.995 | INFO | "huge" : "", | 20260505 11:39:57.995 | INFO | "huge_shared" : " 27 GB", | 20260505 11:39:57.995 | INFO | "malloc" : " 14 GB", | 20260505 11:39:57.995 | INFO | "pinned" : " 29 GB" | 20260505 11:39:57.995 | INFO | } | 20260505 11:39:57.995 | INFO | Perf report for the last 5.14917 seconds: | 20260505 11:39:57.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:39:57.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:57.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:57.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:57.995 | INFO | ANALYSIS : 63.5%, 3.0, 99.4ms, 0 | 20260505 11:39:57.995 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:57.995 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:57.995 | INFO | PRE_HQ : 29.9%, 1.0, 15.6ms, 0 | 20260505 11:39:57.995 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:57.995 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:39:57.995 | INFO | Uploaded 36GiB to GPU in 2.18829s (16.4512GiB/s) | 20260505 11:39:57.995 | INFO | Downloaded 34GiB to GPU in 2.15037s (15.8113GiB/s) | 20260505 11:40:03.129 | INFO | Analyzing chunk frames = [2919936,2920448) | 20260505 11:40:03.129 | INFO | MemInfo: { | 20260505 11:40:03.129 | INFO | "gpu" : " 17 GB", | 20260505 11:40:03.130 | INFO | "huge" : "", | 20260505 11:40:03.130 | INFO | "huge_shared" : " 27 GB", | 20260505 11:40:03.130 | INFO | "malloc" : " 14 GB", | 20260505 11:40:03.130 | INFO | "pinned" : " 29 GB" | 20260505 11:40:03.130 | INFO | } | 20260505 11:40:03.130 | INFO | Perf report for the last 5.13469 seconds: | 20260505 11:40:03.130 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:40:03.130 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:03.130 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:03.130 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:03.130 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 11:40:03.130 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:03.130 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:03.130 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 11:40:03.130 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:03.130 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:03.130 | INFO | Uploaded 36GiB to GPU in 2.21062s (16.285GiB/s) | 20260505 11:40:03.130 | INFO | Downloaded 34GiB to GPU in 2.0949s (16.2299GiB/s) | 20260505 11:40:03.264 | INFO | Starting a round of Baz IO | 20260505 11:40:06.522 | INFO | Populating RTMetrics report | 20260505 11:40:06.522 | INFO | RTMetrics report populated | 20260505 11:40:06.522 | INFO | Opening tmp RTMetrics json file | 20260505 11:40:06.531 | INFO | Serializing RTMetrics to json | 20260505 11:40:06.533 | INFO | Serialization done, swapping tmp file now... | 20260505 11:40:06.535 | INFO | RTMetrics done | 20260505 11:40:06.659 | INFO | Finished a round of Baz IO: Wrote 2.371 GiB in 3.394 seconds | 20260505 11:40:06.659 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.03% / 0.01% / 75.96% | 20260505 11:40:06.659 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.08438% / 1.58946e-05% / 7.54992e-05% | 20260505 11:40:06.659 | INFO | Baz IO overall: | 20260505 11:40:06.659 | INFO | BytesWritten : 1729023839911 | 20260505 11:40:06.659 | INFO | NumEvents : 823219063310 | 20260505 11:40:08.274 | INFO | Analyzing chunk frames = [2920448,2920960) | 20260505 11:40:08.274 | INFO | MemInfo: { | 20260505 11:40:08.274 | INFO | "gpu" : " 17 GB", | 20260505 11:40:08.274 | INFO | "huge" : "", | 20260505 11:40:08.274 | INFO | "huge_shared" : " 27 GB", | 20260505 11:40:08.274 | INFO | "malloc" : " 14 GB", | 20260505 11:40:08.274 | INFO | "pinned" : " 29 GB" | 20260505 11:40:08.274 | INFO | } | 20260505 11:40:08.274 | INFO | Perf report for the last 5.14446 seconds: | 20260505 11:40:08.274 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:40:08.274 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:08.274 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:08.274 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:08.274 | INFO | ANALYSIS : 64.8%, 2.9, 99.8ms, 0 | 20260505 11:40:08.274 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:08.274 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260505 11:40:08.274 | INFO | PRE_HQ : 61.3%, 1.8, 58.9ms, 0 | 20260505 11:40:08.274 | INFO | BAZWRITER : 65.3%, 4.1, 138.4ms, 0 | 20260505 11:40:08.274 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 11:40:08.274 | INFO | Uploaded 36GiB to GPU in 2.23113s (16.1353GiB/s) | 20260505 11:40:08.274 | INFO | Downloaded 38GiB to GPU in 2.29237s (16.5767GiB/s) | 20260505 11:40:13.409 | INFO | Analyzing chunk frames = [2920960,2921472) | 20260505 11:40:13.409 | INFO | MemInfo: { | 20260505 11:40:13.409 | INFO | "gpu" : " 17 GB", | 20260505 11:40:13.409 | INFO | "huge" : "", | 20260505 11:40:13.409 | INFO | "huge_shared" : " 27 GB", | 20260505 11:40:13.409 | INFO | "malloc" : " 14 GB", | 20260505 11:40:13.409 | INFO | "pinned" : " 29 GB" | 20260505 11:40:13.409 | INFO | } | 20260505 11:40:13.409 | INFO | Perf report for the last 5.13532 seconds: | 20260505 11:40:13.409 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:40:13.409 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:13.409 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:13.409 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:13.409 | INFO | ANALYSIS : 64.1%, 3.0, 100.0ms, 0 | 20260505 11:40:13.409 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:13.409 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:13.409 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 11:40:13.409 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:13.409 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:13.410 | INFO | Uploaded 36GiB to GPU in 2.22989s (16.1443GiB/s) | 20260505 11:40:13.410 | INFO | Downloaded 34GiB to GPU in 2.1042s (16.1582GiB/s) | 20260505 11:40:18.548 | INFO | Analyzing chunk frames = [2921472,2921984) | 20260505 11:40:18.548 | INFO | MemInfo: { | 20260505 11:40:18.548 | INFO | "gpu" : " 17 GB", | 20260505 11:40:18.548 | INFO | "huge" : "", | 20260505 11:40:18.548 | INFO | "huge_shared" : " 27 GB", | 20260505 11:40:18.548 | INFO | "malloc" : " 14 GB", | 20260505 11:40:18.548 | INFO | "pinned" : " 29 GB" | 20260505 11:40:18.548 | INFO | } | 20260505 11:40:18.548 | INFO | Perf report for the last 5.13844 seconds: | 20260505 11:40:18.548 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:40:18.548 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:18.548 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:18.548 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:18.548 | INFO | ANALYSIS : 63.6%, 3.0, 99.0ms, 0 | 20260505 11:40:18.548 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:18.548 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:18.548 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:40:18.548 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:18.548 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:18.548 | INFO | Uploaded 36GiB to GPU in 2.21894s (16.224GiB/s) | 20260505 11:40:18.548 | INFO | Downloaded 34GiB to GPU in 2.13271s (15.9421GiB/s) | 20260505 11:40:23.692 | INFO | Analyzing chunk frames = [2921984,2922496) | 20260505 11:40:23.692 | INFO | MemInfo: { | 20260505 11:40:23.692 | INFO | "gpu" : " 17 GB", | 20260505 11:40:23.692 | INFO | "huge" : "", | 20260505 11:40:23.692 | INFO | "huge_shared" : " 27 GB", | 20260505 11:40:23.692 | INFO | "malloc" : " 14 GB", | 20260505 11:40:23.692 | INFO | "pinned" : " 29 GB" | 20260505 11:40:23.692 | INFO | } | 20260505 11:40:23.692 | INFO | Perf report for the last 5.14423 seconds: | 20260505 11:40:23.692 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:40:23.692 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:23.692 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:23.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:23.692 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 11:40:23.692 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:23.692 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:23.692 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:40:23.692 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:23.692 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:23.693 | INFO | Uploaded 36GiB to GPU in 2.2156s (16.2484GiB/s) | 20260505 11:40:23.693 | INFO | Downloaded 34GiB to GPU in 2.13348s (15.9364GiB/s) | 20260505 11:40:28.829 | INFO | Analyzing chunk frames = [2922496,2923008) | 20260505 11:40:28.829 | INFO | MemInfo: { | 20260505 11:40:28.829 | INFO | "gpu" : " 17 GB", | 20260505 11:40:28.829 | INFO | "huge" : "", | 20260505 11:40:28.829 | INFO | "huge_shared" : " 27 GB", | 20260505 11:40:28.829 | INFO | "malloc" : " 14 GB", | 20260505 11:40:28.829 | INFO | "pinned" : " 29 GB" | 20260505 11:40:28.829 | INFO | } | 20260505 11:40:28.829 | INFO | Perf report for the last 5.13728 seconds: | 20260505 11:40:28.829 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:40:28.829 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:28.829 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:28.829 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:28.829 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 11:40:28.829 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:28.829 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:28.829 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 11:40:28.829 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:28.829 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:28.830 | INFO | Uploaded 36GiB to GPU in 2.16074s (16.6609GiB/s) | 20260505 11:40:28.830 | INFO | Downloaded 34GiB to GPU in 2.12748s (15.9814GiB/s) | 20260505 11:40:33.970 | INFO | Analyzing chunk frames = [2923008,2923520) | 20260505 11:40:33.970 | INFO | MemInfo: { | 20260505 11:40:33.970 | INFO | "gpu" : " 17 GB", | 20260505 11:40:33.970 | INFO | "huge" : "", | 20260505 11:40:33.970 | INFO | "huge_shared" : " 27 GB", | 20260505 11:40:33.970 | INFO | "malloc" : " 14 GB", | 20260505 11:40:33.970 | INFO | "pinned" : " 29 GB" | 20260505 11:40:33.970 | INFO | } | 20260505 11:40:33.970 | INFO | Perf report for the last 5.14093 seconds: | 20260505 11:40:33.970 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:40:33.970 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:33.970 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:33.970 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:33.970 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 11:40:33.970 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:33.970 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:33.970 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 11:40:33.970 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:33.970 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:33.971 | INFO | Uploaded 36GiB to GPU in 2.17514s (16.5506GiB/s) | 20260505 11:40:33.971 | INFO | Downloaded 34GiB to GPU in 2.11693s (16.061GiB/s) | 20260505 11:40:39.104 | INFO | Analyzing chunk frames = [2923520,2924032) | 20260505 11:40:39.104 | INFO | MemInfo: { | 20260505 11:40:39.104 | INFO | "gpu" : " 17 GB", | 20260505 11:40:39.104 | INFO | "huge" : "", | 20260505 11:40:39.105 | INFO | "huge_shared" : " 27 GB", | 20260505 11:40:39.105 | INFO | "malloc" : " 14 GB", | 20260505 11:40:39.105 | INFO | "pinned" : " 29 GB" | 20260505 11:40:39.105 | INFO | } | 20260505 11:40:39.105 | INFO | Perf report for the last 5.13426 seconds: | 20260505 11:40:39.105 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:40:39.105 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:39.105 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:39.105 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:39.105 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 11:40:39.105 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:39.105 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:39.105 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 11:40:39.105 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:39.105 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:39.105 | INFO | Uploaded 36GiB to GPU in 2.14448s (16.7873GiB/s) | 20260505 11:40:39.105 | INFO | Downloaded 34GiB to GPU in 2.16271s (15.721GiB/s) | 20260505 11:40:44.250 | INFO | Analyzing chunk frames = [2924032,2924544) | 20260505 11:40:44.250 | INFO | MemInfo: { | 20260505 11:40:44.250 | INFO | "gpu" : " 17 GB", | 20260505 11:40:44.250 | INFO | "huge" : "", | 20260505 11:40:44.250 | INFO | "huge_shared" : " 27 GB", | 20260505 11:40:44.250 | INFO | "malloc" : " 14 GB", | 20260505 11:40:44.250 | INFO | "pinned" : " 29 GB" | 20260505 11:40:44.250 | INFO | } | 20260505 11:40:44.250 | INFO | Perf report for the last 5.14554 seconds: | 20260505 11:40:44.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:40:44.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:44.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:44.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:44.250 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 11:40:44.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:44.250 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:44.250 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 11:40:44.250 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:44.250 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:44.250 | INFO | Uploaded 36GiB to GPU in 2.21527s (16.2508GiB/s) | 20260505 11:40:44.250 | INFO | Downloaded 34GiB to GPU in 2.09139s (16.2571GiB/s) | 20260505 11:40:44.250 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 11:40:44.250 | INFO | Total mem usage: 15.3 GiB | 20260505 11:40:44.250 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 11:40:44.250 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -7.33%) | 20260505 11:40:44.251 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 11:40:44.251 | INFO | High Water Breakdown: | 20260505 11:40:44.251 | INFO | 60.9% : 8.7 GiB ( 91.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 11:40:44.251 | INFO | 38.9% : 5.56 GiB ( 83.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 11:40:44.251 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 11:40:44.251 | INFO | ------------------------------------------ | 20260505 11:40:44.251 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:40:44.251 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 11:40:44.251 | INFO | Total mem usage: 26.5 GiB | 20260505 11:40:44.251 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 11:40:44.251 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 11:40:44.251 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 11:40:44.251 | INFO | High Water Breakdown: | 20260505 11:40:44.251 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:40:44.251 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:40:44.251 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:40:44.251 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:40:44.251 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:40:44.251 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:40:44.251 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 11:40:44.251 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 11:40:44.251 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:40:44.251 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 11:40:44.251 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 11:40:44.251 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 11:40:44.251 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 11:40:44.251 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 11:40:44.251 | INFO | ------------------------------------------ | 20260505 11:40:44.251 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:40:44.251 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 11:40:44.251 | INFO | Total mem usage: 25.6 GiB | 20260505 11:40:44.251 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 11:40:44.251 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 11:40:44.251 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 11:40:44.251 | INFO | High Water Breakdown: | 20260505 11:40:44.251 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 11:40:44.251 | INFO | ------------------------------------------ | 20260505 11:40:44.251 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:40:44.251 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 11:40:44.251 | INFO | Total mem usage: 15.7 GiB | 20260505 11:40:44.251 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 11:40:44.251 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 11:40:44.251 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 11:40:44.251 | INFO | High Water Breakdown: | 20260505 11:40:44.251 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 11:40:44.251 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 11:40:44.251 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 11:40:44.251 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 11:40:44.251 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 11:40:44.251 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 11:40:44.251 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 11:40:44.251 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 11:40:44.251 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 11:40:44.251 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:40:44.251 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 11:40:44.251 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:40:44.251 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 11:40:44.251 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 11:40:44.251 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 11:40:44.251 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 11:40:44.251 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:40:44.251 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 11:40:44.251 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:40:44.251 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:40:44.251 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:40:44.251 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 11:40:44.251 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 11:40:44.251 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:40:44.251 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 11:40:44.251 | INFO | ------------------------------------------ | 20260505 11:40:44.251 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:40:44.251 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 11:40:44.251 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:40:44.251 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:40:44.251 | INFO | PerfReport: Full Execution -- 3.21e+01 / 7.75e+01, 1.38e+01 / 1.39e+00 | 20260505 11:40:44.251 | INFO | PerfReport: 0.29%, AggregateStats -- 9.32e-02 / 1.01e-01, 1.07e-02 / 3.53e-03 | 20260505 11:40:44.251 | INFO | PerfReport: 11.72%, Baseline -- 3.77e+00 / 4.50e+00, 6.02e-01 / 1.90e-01 | 20260505 11:40:44.251 | INFO | PerfReport: 3.32%, Histogram -- 1.07e+00 / 1.51e+00, 2.07e-01 / 8.14e-02 | 20260505 11:40:44.251 | INFO | PerfReport: 15.80%, DME -- 4.06e+01 / 4.49e+01, 4.29e+00 / 1.93e+00 | 20260505 11:40:44.251 | INFO | PerfReport: 56.51%, FrameLabeling -- 1.82e+01 / 2.17e+01, 1.65e+00 / 2.43e+00 | 20260505 11:40:44.251 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.37e+00 / 2.84e+00, 4.21e-01 / 2.17e-01 | 20260505 11:40:44.251 | INFO | PerfReport: 3.68%, Metrics -- 1.18e+00 / 1.47e+00, 3.40e-01 / 1.92e-01 | 20260505 11:40:44.251 | INFO | PerfReport: 1.29%, PulseSerializing -- 4.16e-01 / 4.61e-01, 7.62e-02 / 3.98e-02 | 20260505 11:40:44.251 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 11:40:44.251 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:40:44.251 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:40:44.251 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.47e+01 / 0.00e+00 | 20260505 11:40:44.251 | INFO | PerfReport: 22.64%, Upload -- 2.29e+01 / 0.00e+00, 8.29e+00 / 0.00e+00 | 20260505 11:40:44.251 | INFO | PerfReport: 22.05%, Download -- 2.23e+01 / 0.00e+00, 8.30e+00 / 0.00e+00 | 20260505 11:40:44.251 | INFO | PerfReport: 31.76%, Compute -- 3.21e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 11:40:44.251 | INFO | PerfReport: 2.01%, LockUpload -- 2.03e+00 / 0.00e+00, 7.60e+00 / 0.00e+00 | 20260505 11:40:44.251 | INFO | PerfReport: 1.26%, LockDownload -- 1.28e+00 / 0.00e+00, 2.74e+00 / 0.00e+00 | 20260505 11:40:44.251 | INFO | PerfReport: 20.26%, LockCompute -- 2.05e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260505 11:40:44.251 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.87e-02 / 0.00e+00 | 20260505 11:40:44.384 | INFO | Starting a round of Baz IO | 20260505 11:40:47.626 | INFO | Populating RTMetrics report | 20260505 11:40:47.626 | INFO | RTMetrics report populated | 20260505 11:40:47.626 | INFO | Opening tmp RTMetrics json file | 20260505 11:40:47.629 | INFO | Serializing RTMetrics to json | 20260505 11:40:47.631 | INFO | Serialization done, swapping tmp file now... | 20260505 11:40:47.632 | INFO | RTMetrics done | 20260505 11:40:47.762 | INFO | Finished a round of Baz IO: Wrote 2.368 GiB in 3.377 seconds | 20260505 11:40:47.762 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.02% / 0.01% / 75.97% | 20260505 11:40:47.762 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.07865% / 3.97364e-06% / 5.16574e-05% | 20260505 11:40:47.762 | INFO | Baz IO overall: | 20260505 11:40:47.762 | INFO | BytesWritten : 1731566848314 | 20260505 11:40:47.762 | INFO | NumEvents : 824427648531 | 20260505 11:40:49.392 | INFO | Analyzing chunk frames = [2924544,2925056) | 20260505 11:40:49.392 | INFO | MemInfo: { | 20260505 11:40:49.392 | INFO | "gpu" : " 17 GB", | 20260505 11:40:49.392 | INFO | "huge" : "", | 20260505 11:40:49.392 | INFO | "huge_shared" : " 27 GB", | 20260505 11:40:49.392 | INFO | "malloc" : " 14 GB", | 20260505 11:40:49.392 | INFO | "pinned" : " 29 GB" | 20260505 11:40:49.392 | INFO | } | 20260505 11:40:49.393 | INFO | Perf report for the last 5.14235 seconds: | 20260505 11:40:49.393 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:40:49.393 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:49.393 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:49.393 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:49.393 | INFO | ANALYSIS : 64.5%, 2.9, 99.1ms, 0 | 20260505 11:40:49.393 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:49.393 | INFO | RT_METRICS : 45.5%, 1.0, 23.9ms, 0 | 20260505 11:40:49.393 | INFO | PRE_HQ : 60.9%, 1.9, 58.6ms, 0 | 20260505 11:40:49.393 | INFO | BAZWRITER : 65.0%, 4.2, 139.0ms, 0 | 20260505 11:40:49.393 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 11:40:49.393 | INFO | Uploaded 36GiB to GPU in 2.19223s (16.4216GiB/s) | 20260505 11:40:49.393 | INFO | Downloaded 38GiB to GPU in 2.28857s (16.6042GiB/s) | 20260505 11:40:54.525 | INFO | Analyzing chunk frames = [2925056,2925568) | 20260505 11:40:54.525 | INFO | MemInfo: { | 20260505 11:40:54.525 | INFO | "gpu" : " 17 GB", | 20260505 11:40:54.525 | INFO | "huge" : "", | 20260505 11:40:54.525 | INFO | "huge_shared" : " 27 GB", | 20260505 11:40:54.525 | INFO | "malloc" : " 14 GB", | 20260505 11:40:54.525 | INFO | "pinned" : " 29 GB" | 20260505 11:40:54.525 | INFO | } | 20260505 11:40:54.525 | INFO | Perf report for the last 5.1327 seconds: | 20260505 11:40:54.525 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:40:54.525 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:54.525 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:54.525 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:54.525 | INFO | ANALYSIS : 64.2%, 3.0, 100.0ms, 0 | 20260505 11:40:54.525 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:54.525 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:54.525 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:40:54.525 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:54.525 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:54.525 | INFO | Uploaded 36GiB to GPU in 2.22895s (16.1511GiB/s) | 20260505 11:40:54.525 | INFO | Downloaded 34GiB to GPU in 2.10342s (16.1641GiB/s) | 20260505 11:40:59.663 | INFO | Analyzing chunk frames = [2925568,2926080) | 20260505 11:40:59.663 | INFO | MemInfo: { | 20260505 11:40:59.663 | INFO | "gpu" : " 17 GB", | 20260505 11:40:59.663 | INFO | "huge" : "", | 20260505 11:40:59.663 | INFO | "huge_shared" : " 27 GB", | 20260505 11:40:59.663 | INFO | "malloc" : " 14 GB", | 20260505 11:40:59.663 | INFO | "pinned" : " 29 GB" | 20260505 11:40:59.663 | INFO | } | 20260505 11:40:59.663 | INFO | Perf report for the last 5.13803 seconds: | 20260505 11:40:59.663 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:40:59.663 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:59.663 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:59.663 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:59.663 | INFO | ANALYSIS : 63.6%, 3.0, 98.9ms, 0 | 20260505 11:40:59.663 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:59.663 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:59.663 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 11:40:59.663 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:59.663 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:40:59.663 | INFO | Uploaded 36GiB to GPU in 2.22072s (16.2109GiB/s) | 20260505 11:40:59.663 | INFO | Downloaded 34GiB to GPU in 2.13197s (15.9477GiB/s) | 20260505 11:41:04.800 | INFO | Analyzing chunk frames = [2926080,2926592) | 20260505 11:41:04.800 | INFO | MemInfo: { | 20260505 11:41:04.800 | INFO | "gpu" : " 17 GB", | 20260505 11:41:04.800 | INFO | "huge" : "", | 20260505 11:41:04.800 | INFO | "huge_shared" : " 27 GB", | 20260505 11:41:04.800 | INFO | "malloc" : " 14 GB", | 20260505 11:41:04.800 | INFO | "pinned" : " 29 GB" | 20260505 11:41:04.800 | INFO | } | 20260505 11:41:04.800 | INFO | Perf report for the last 5.13664 seconds: | 20260505 11:41:04.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:41:04.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:04.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:04.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:04.800 | INFO | ANALYSIS : 63.0%, 3.0, 97.8ms, 0 | 20260505 11:41:04.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:04.800 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:04.800 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 11:41:04.800 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:04.800 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:04.800 | INFO | Uploaded 36GiB to GPU in 2.15937s (16.6715GiB/s) | 20260505 11:41:04.800 | INFO | Downloaded 34GiB to GPU in 2.08051s (16.3421GiB/s) | 20260505 11:41:09.948 | INFO | Analyzing chunk frames = [2926592,2927104) | 20260505 11:41:09.948 | INFO | MemInfo: { | 20260505 11:41:09.948 | INFO | "gpu" : " 17 GB", | 20260505 11:41:09.948 | INFO | "huge" : "", | 20260505 11:41:09.948 | INFO | "huge_shared" : " 27 GB", | 20260505 11:41:09.948 | INFO | "malloc" : " 14 GB", | 20260505 11:41:09.948 | INFO | "pinned" : " 29 GB" | 20260505 11:41:09.948 | INFO | } | 20260505 11:41:09.948 | INFO | Perf report for the last 5.14843 seconds: | 20260505 11:41:09.948 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:41:09.948 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:09.948 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:09.948 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:09.948 | INFO | ANALYSIS : 63.1%, 3.0, 98.7ms, 0 | 20260505 11:41:09.948 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:09.948 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:09.948 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 11:41:09.948 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:09.948 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:09.949 | INFO | Uploaded 36GiB to GPU in 2.16202s (16.6511GiB/s) | 20260505 11:41:09.949 | INFO | Downloaded 34GiB to GPU in 2.12547s (15.9964GiB/s) | 20260505 11:41:15.087 | INFO | Analyzing chunk frames = [2927104,2927616) | 20260505 11:41:15.087 | INFO | MemInfo: { | 20260505 11:41:15.087 | INFO | "gpu" : " 17 GB", | 20260505 11:41:15.087 | INFO | "huge" : "", | 20260505 11:41:15.087 | INFO | "huge_shared" : " 27 GB", | 20260505 11:41:15.087 | INFO | "malloc" : " 14 GB", | 20260505 11:41:15.087 | INFO | "pinned" : " 29 GB" | 20260505 11:41:15.087 | INFO | } | 20260505 11:41:15.087 | INFO | Perf report for the last 5.13855 seconds: | 20260505 11:41:15.087 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:41:15.087 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:15.087 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:15.087 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:15.087 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 11:41:15.087 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:15.087 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:15.087 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 11:41:15.087 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:15.087 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:15.087 | INFO | Uploaded 36GiB to GPU in 2.15861s (16.6774GiB/s) | 20260505 11:41:15.087 | INFO | Downloaded 34GiB to GPU in 2.11016s (16.1125GiB/s) | 20260505 11:41:20.228 | INFO | Analyzing chunk frames = [2927616,2928128) | 20260505 11:41:20.228 | INFO | MemInfo: { | 20260505 11:41:20.228 | INFO | "gpu" : " 17 GB", | 20260505 11:41:20.228 | INFO | "huge" : "", | 20260505 11:41:20.228 | INFO | "huge_shared" : " 27 GB", | 20260505 11:41:20.228 | INFO | "malloc" : " 14 GB", | 20260505 11:41:20.228 | INFO | "pinned" : " 29 GB" | 20260505 11:41:20.228 | INFO | } | 20260505 11:41:20.228 | INFO | Perf report for the last 5.14094 seconds: | 20260505 11:41:20.228 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:41:20.228 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:20.228 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:20.228 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:20.228 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 11:41:20.228 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:20.228 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:20.228 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 11:41:20.228 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:20.228 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:20.228 | INFO | Uploaded 36GiB to GPU in 2.18088s (16.5071GiB/s) | 20260505 11:41:20.228 | INFO | Downloaded 34GiB to GPU in 2.13834s (15.9002GiB/s) | 20260505 11:41:25.360 | INFO | Analyzing chunk frames = [2928128,2928640) | 20260505 11:41:25.360 | INFO | MemInfo: { | 20260505 11:41:25.360 | INFO | "gpu" : " 17 GB", | 20260505 11:41:25.360 | INFO | "huge" : "", | 20260505 11:41:25.360 | INFO | "huge_shared" : " 27 GB", | 20260505 11:41:25.360 | INFO | "malloc" : " 14 GB", | 20260505 11:41:25.360 | INFO | "pinned" : " 29 GB" | 20260505 11:41:25.360 | INFO | } | 20260505 11:41:25.360 | INFO | Perf report for the last 5.13238 seconds: | 20260505 11:41:25.360 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:41:25.360 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:25.360 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:25.360 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:25.360 | INFO | ANALYSIS : 64.0%, 3.0, 99.8ms, 0 | 20260505 11:41:25.360 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:25.360 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:25.360 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 11:41:25.360 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:25.360 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:25.360 | INFO | Uploaded 36GiB to GPU in 2.20144s (16.3529GiB/s) | 20260505 11:41:25.360 | INFO | Downloaded 34GiB to GPU in 2.11681s (16.0619GiB/s) | 20260505 11:41:25.497 | INFO | Starting a round of Baz IO | 20260505 11:41:28.742 | INFO | Populating RTMetrics report | 20260505 11:41:28.742 | INFO | RTMetrics report populated | 20260505 11:41:28.742 | INFO | Opening tmp RTMetrics json file | 20260505 11:41:28.743 | INFO | Serializing RTMetrics to json | 20260505 11:41:28.744 | INFO | Serialization done, swapping tmp file now... | 20260505 11:41:28.747 | INFO | RTMetrics done | 20260505 11:41:28.897 | INFO | Finished a round of Baz IO: Wrote 2.372 GiB in 3.399 seconds | 20260505 11:41:28.897 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.02% / 0.01% / 75.98% | 20260505 11:41:28.897 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.07815% / 7.94729e-06% / 6.75519e-05% | 20260505 11:41:28.897 | INFO | Baz IO overall: | 20260505 11:41:28.897 | INFO | BytesWritten : 1734113358547 | 20260505 11:41:28.897 | INFO | NumEvents : 825639012008 | 20260505 11:41:30.501 | INFO | Analyzing chunk frames = [2928640,2929152) | 20260505 11:41:30.501 | INFO | MemInfo: { | 20260505 11:41:30.501 | INFO | "gpu" : " 17 GB", | 20260505 11:41:30.501 | INFO | "huge" : "", | 20260505 11:41:30.501 | INFO | "huge_shared" : " 27 GB", | 20260505 11:41:30.501 | INFO | "malloc" : " 14 GB", | 20260505 11:41:30.501 | INFO | "pinned" : " 29 GB" | 20260505 11:41:30.501 | INFO | } | 20260505 11:41:30.501 | INFO | Perf report for the last 5.14086 seconds: | 20260505 11:41:30.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:41:30.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:30.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:30.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:30.501 | INFO | ANALYSIS : 64.7%, 2.9, 99.7ms, 0 | 20260505 11:41:30.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:30.501 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260505 11:41:30.501 | INFO | PRE_HQ : 61.0%, 1.9, 59.0ms, 0 | 20260505 11:41:30.501 | INFO | BAZWRITER : 65.5%, 4.1, 138.5ms, 0 | 20260505 11:41:30.501 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 11:41:30.501 | INFO | Uploaded 36GiB to GPU in 2.17611s (16.5433GiB/s) | 20260505 11:41:30.501 | INFO | Downloaded 38GiB to GPU in 2.28979s (16.5954GiB/s) | 20260505 11:41:35.644 | INFO | Analyzing chunk frames = [2929152,2929664) | 20260505 11:41:35.644 | INFO | MemInfo: { | 20260505 11:41:35.644 | INFO | "gpu" : " 17 GB", | 20260505 11:41:35.644 | INFO | "huge" : "", | 20260505 11:41:35.644 | INFO | "huge_shared" : " 27 GB", | 20260505 11:41:35.644 | INFO | "malloc" : " 14 GB", | 20260505 11:41:35.644 | INFO | "pinned" : " 29 GB" | 20260505 11:41:35.644 | INFO | } | 20260505 11:41:35.644 | INFO | Perf report for the last 5.14278 seconds: | 20260505 11:41:35.644 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:41:35.644 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:35.644 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:35.644 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:35.644 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 11:41:35.644 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:35.644 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:35.644 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 11:41:35.644 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:35.644 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:35.644 | INFO | Uploaded 36GiB to GPU in 2.22853s (16.1541GiB/s) | 20260505 11:41:35.644 | INFO | Downloaded 34GiB to GPU in 2.10233s (16.1725GiB/s) | 20260505 11:41:40.780 | INFO | Analyzing chunk frames = [2929664,2930176) | 20260505 11:41:40.780 | INFO | MemInfo: { | 20260505 11:41:40.780 | INFO | "gpu" : " 17 GB", | 20260505 11:41:40.780 | INFO | "huge" : "", | 20260505 11:41:40.780 | INFO | "huge_shared" : " 27 GB", | 20260505 11:41:40.780 | INFO | "malloc" : " 14 GB", | 20260505 11:41:40.780 | INFO | "pinned" : " 29 GB" | 20260505 11:41:40.780 | INFO | } | 20260505 11:41:40.780 | INFO | Perf report for the last 5.13575 seconds: | 20260505 11:41:40.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:41:40.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:40.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:40.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:40.780 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 11:41:40.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:40.780 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:40.780 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 11:41:40.780 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:40.780 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:40.780 | INFO | Uploaded 36GiB to GPU in 2.2192s (16.2221GiB/s) | 20260505 11:41:40.780 | INFO | Downloaded 34GiB to GPU in 2.12447s (16.004GiB/s) | 20260505 11:41:45.917 | INFO | Analyzing chunk frames = [2930176,2930688) | 20260505 11:41:45.917 | INFO | MemInfo: { | 20260505 11:41:45.917 | INFO | "gpu" : " 17 GB", | 20260505 11:41:45.917 | INFO | "huge" : "", | 20260505 11:41:45.917 | INFO | "huge_shared" : " 27 GB", | 20260505 11:41:45.917 | INFO | "malloc" : " 14 GB", | 20260505 11:41:45.917 | INFO | "pinned" : " 29 GB" | 20260505 11:41:45.917 | INFO | } | 20260505 11:41:45.917 | INFO | Perf report for the last 5.13756 seconds: | 20260505 11:41:45.917 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:41:45.917 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:45.917 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:45.917 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:45.917 | INFO | ANALYSIS : 62.9%, 3.0, 97.7ms, 0 | 20260505 11:41:45.917 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:45.917 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:45.917 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 11:41:45.917 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:45.917 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:45.917 | INFO | Uploaded 36GiB to GPU in 2.15477s (16.7071GiB/s) | 20260505 11:41:45.918 | INFO | Downloaded 34GiB to GPU in 2.07844s (16.3585GiB/s) | 20260505 11:41:51.057 | INFO | Analyzing chunk frames = [2930688,2931200) | 20260505 11:41:51.057 | INFO | MemInfo: { | 20260505 11:41:51.057 | INFO | "gpu" : " 17 GB", | 20260505 11:41:51.057 | INFO | "huge" : "", | 20260505 11:41:51.057 | INFO | "huge_shared" : " 27 GB", | 20260505 11:41:51.057 | INFO | "malloc" : " 14 GB", | 20260505 11:41:51.057 | INFO | "pinned" : " 29 GB" | 20260505 11:41:51.057 | INFO | } | 20260505 11:41:51.057 | INFO | Perf report for the last 5.13953 seconds: | 20260505 11:41:51.057 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:41:51.057 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:51.057 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:51.057 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:51.057 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 11:41:51.057 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:51.057 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:51.057 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 11:41:51.057 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:51.057 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:51.057 | INFO | Uploaded 36GiB to GPU in 2.11924s (16.9872GiB/s) | 20260505 11:41:51.057 | INFO | Downloaded 34GiB to GPU in 2.12306s (16.0146GiB/s) | 20260505 11:41:56.203 | INFO | Analyzing chunk frames = [2931200,2931712) | 20260505 11:41:56.203 | INFO | MemInfo: { | 20260505 11:41:56.203 | INFO | "gpu" : " 17 GB", | 20260505 11:41:56.203 | INFO | "huge" : "", | 20260505 11:41:56.203 | INFO | "huge_shared" : " 27 GB", | 20260505 11:41:56.203 | INFO | "malloc" : " 14 GB", | 20260505 11:41:56.203 | INFO | "pinned" : " 29 GB" | 20260505 11:41:56.203 | INFO | } | 20260505 11:41:56.203 | INFO | Perf report for the last 5.14662 seconds: | 20260505 11:41:56.203 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:41:56.203 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:56.203 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:56.203 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:56.203 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 11:41:56.203 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:56.203 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:56.204 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 11:41:56.204 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:56.204 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:41:56.204 | INFO | Uploaded 36GiB to GPU in 2.17808s (16.5283GiB/s) | 20260505 11:41:56.204 | INFO | Downloaded 34GiB to GPU in 2.11383s (16.0846GiB/s) | 20260505 11:42:01.343 | INFO | Analyzing chunk frames = [2931712,2932224) | 20260505 11:42:01.343 | INFO | MemInfo: { | 20260505 11:42:01.343 | INFO | "gpu" : " 17 GB", | 20260505 11:42:01.343 | INFO | "huge" : "", | 20260505 11:42:01.343 | INFO | "huge_shared" : " 27 GB", | 20260505 11:42:01.343 | INFO | "malloc" : " 14 GB", | 20260505 11:42:01.343 | INFO | "pinned" : " 29 GB" | 20260505 11:42:01.343 | INFO | } | 20260505 11:42:01.343 | INFO | Perf report for the last 5.13971 seconds: | 20260505 11:42:01.343 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:42:01.343 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:01.343 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:01.343 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:01.343 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 11:42:01.343 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:01.343 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:01.343 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260505 11:42:01.343 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:01.343 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:01.343 | INFO | Uploaded 36GiB to GPU in 2.16599s (16.6206GiB/s) | 20260505 11:42:01.343 | INFO | Downloaded 34GiB to GPU in 2.14676s (15.8378GiB/s) | 20260505 11:42:06.469 | INFO | Analyzing chunk frames = [2932224,2932736) | 20260505 11:42:06.469 | INFO | MemInfo: { | 20260505 11:42:06.469 | INFO | "gpu" : " 17 GB", | 20260505 11:42:06.469 | INFO | "huge" : "", | 20260505 11:42:06.469 | INFO | "huge_shared" : " 27 GB", | 20260505 11:42:06.469 | INFO | "malloc" : " 14 GB", | 20260505 11:42:06.469 | INFO | "pinned" : " 29 GB" | 20260505 11:42:06.469 | INFO | } | 20260505 11:42:06.469 | INFO | Perf report for the last 5.12557 seconds: | 20260505 11:42:06.469 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:42:06.469 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:06.469 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:06.469 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:06.469 | INFO | ANALYSIS : 64.0%, 3.0, 99.6ms, 0 | 20260505 11:42:06.469 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:06.469 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:06.469 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 11:42:06.469 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:06.469 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:06.469 | INFO | Uploaded 36GiB to GPU in 2.21567s (16.2479GiB/s) | 20260505 11:42:06.469 | INFO | Downloaded 34GiB to GPU in 2.10547s (16.1484GiB/s) | 20260505 11:42:06.469 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 11:42:06.469 | INFO | Total mem usage: 15.3 GiB | 20260505 11:42:06.469 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 11:42:06.469 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -6.84%) | 20260505 11:42:06.469 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 11:42:06.469 | INFO | High Water Breakdown: | 20260505 11:42:06.469 | INFO | 61% : 8.76 GiB ( 92.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 11:42:06.469 | INFO | 38.8% : 5.57 GiB ( 83.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 11:42:06.469 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 11:42:06.469 | INFO | ------------------------------------------ | 20260505 11:42:06.469 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:42:06.469 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 11:42:06.469 | INFO | Total mem usage: 26.5 GiB | 20260505 11:42:06.469 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 11:42:06.469 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 11:42:06.469 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 11:42:06.469 | INFO | High Water Breakdown: | 20260505 11:42:06.469 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:42:06.469 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:42:06.469 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:42:06.469 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:42:06.469 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:42:06.469 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:42:06.469 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 11:42:06.469 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 11:42:06.469 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:42:06.469 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 11:42:06.469 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 11:42:06.469 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 11:42:06.469 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 11:42:06.469 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 11:42:06.469 | INFO | ------------------------------------------ | 20260505 11:42:06.469 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:42:06.469 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 11:42:06.469 | INFO | Total mem usage: 25.6 GiB | 20260505 11:42:06.469 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 11:42:06.469 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 11:42:06.470 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 11:42:06.470 | INFO | High Water Breakdown: | 20260505 11:42:06.470 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 11:42:06.470 | INFO | ------------------------------------------ | 20260505 11:42:06.470 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:42:06.470 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 11:42:06.470 | INFO | Total mem usage: 15.7 GiB | 20260505 11:42:06.470 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 11:42:06.470 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 11:42:06.470 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 11:42:06.470 | INFO | High Water Breakdown: | 20260505 11:42:06.470 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 11:42:06.470 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 11:42:06.470 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 11:42:06.470 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 11:42:06.470 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 11:42:06.470 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 11:42:06.470 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 11:42:06.470 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 11:42:06.470 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 11:42:06.470 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:42:06.470 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 11:42:06.470 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:42:06.470 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 11:42:06.470 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 11:42:06.470 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 11:42:06.470 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 11:42:06.470 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:42:06.470 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 11:42:06.470 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:42:06.470 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:42:06.470 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:42:06.470 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 11:42:06.470 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 11:42:06.470 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:42:06.470 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 11:42:06.470 | INFO | ------------------------------------------ | 20260505 11:42:06.470 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:42:06.470 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 11:42:06.470 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:42:06.470 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:42:06.470 | INFO | PerfReport: Full Execution -- 3.21e+01 / 7.70e+01, 1.38e+01 / 1.94e+00 | 20260505 11:42:06.470 | INFO | PerfReport: 0.29%, AggregateStats -- 9.30e-02 / 9.99e-02, 1.08e-02 / 4.06e-03 | 20260505 11:42:06.470 | INFO | PerfReport: 11.71%, Baseline -- 3.76e+00 / 4.48e+00, 5.96e-01 / 2.34e-01 | 20260505 11:42:06.470 | INFO | PerfReport: 3.30%, Histogram -- 1.06e+00 / 1.48e+00, 2.03e-01 / 8.58e-02 | 20260505 11:42:06.470 | INFO | PerfReport: 15.95%, DME -- 4.09e+01 / 4.57e+01, 4.56e+00 / 1.93e+00 | 20260505 11:42:06.470 | INFO | PerfReport: 56.32%, FrameLabeling -- 1.81e+01 / 2.03e+01, 1.52e+00 / 1.57e+00 | 20260505 11:42:06.470 | INFO | PerfReport: 7.46%, PulseAccumulating -- 2.39e+00 / 2.93e+00, 4.38e-01 / 2.05e-01 | 20260505 11:42:06.470 | INFO | PerfReport: 3.67%, Metrics -- 1.18e+00 / 1.52e+00, 3.32e-01 / 1.49e-01 | 20260505 11:42:06.470 | INFO | PerfReport: 1.30%, PulseSerializing -- 4.15e-01 / 4.68e-01, 7.67e-02 / 2.57e-02 | 20260505 11:42:06.470 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 11:42:06.470 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:42:06.470 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:42:06.470 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.48e+01 / 0.00e+00 | 20260505 11:42:06.470 | INFO | PerfReport: 22.52%, Upload -- 2.28e+01 / 0.00e+00, 8.11e+00 / 0.00e+00 | 20260505 11:42:06.470 | INFO | PerfReport: 22.01%, Download -- 2.23e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260505 11:42:06.470 | INFO | PerfReport: 31.72%, Compute -- 3.21e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 11:42:06.470 | INFO | PerfReport: 2.08%, LockUpload -- 2.11e+00 / 0.00e+00, 7.80e+00 / 0.00e+00 | 20260505 11:42:06.470 | INFO | PerfReport: 1.34%, LockDownload -- 1.35e+00 / 0.00e+00, 2.96e+00 / 0.00e+00 | 20260505 11:42:06.470 | INFO | PerfReport: 20.31%, LockCompute -- 2.05e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260505 11:42:06.470 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.83e-02 / 0.00e+00 | 20260505 11:42:06.609 | INFO | Starting a round of Baz IO | 20260505 11:42:09.872 | INFO | Populating RTMetrics report | 20260505 11:42:09.872 | INFO | RTMetrics report populated | 20260505 11:42:09.872 | INFO | Opening tmp RTMetrics json file | 20260505 11:42:09.875 | INFO | Serializing RTMetrics to json | 20260505 11:42:09.876 | INFO | Serialization done, swapping tmp file now... | 20260505 11:42:09.876 | INFO | RTMetrics done | 20260505 11:42:10.004 | INFO | Finished a round of Baz IO: Wrote 2.369 GiB in 3.394 seconds | 20260505 11:42:10.004 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.01% / 0.01% / 75.98% | 20260505 11:42:10.004 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.07186% / 1.19209e-05% / 7.54992e-05% | 20260505 11:42:10.004 | INFO | Baz IO overall: | 20260505 11:42:10.004 | INFO | BytesWritten : 1736656546773 | 20260505 11:42:10.004 | INFO | NumEvents : 826847902754 | 20260505 11:42:11.611 | INFO | Analyzing chunk frames = [2932736,2933248) | 20260505 11:42:11.611 | INFO | MemInfo: { | 20260505 11:42:11.611 | INFO | "gpu" : " 17 GB", | 20260505 11:42:11.611 | INFO | "huge" : "", | 20260505 11:42:11.611 | INFO | "huge_shared" : " 27 GB", | 20260505 11:42:11.611 | INFO | "malloc" : " 14 GB", | 20260505 11:42:11.611 | INFO | "pinned" : " 29 GB" | 20260505 11:42:11.611 | INFO | } | 20260505 11:42:11.611 | INFO | Perf report for the last 5.14241 seconds: | 20260505 11:42:11.611 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:42:11.611 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:11.611 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:11.611 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:11.611 | INFO | ANALYSIS : 65.1%, 2.9, 100.1ms, 0 | 20260505 11:42:11.611 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:11.611 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260505 11:42:11.611 | INFO | PRE_HQ : 61.5%, 1.8, 58.2ms, 0 | 20260505 11:42:11.611 | INFO | BAZWRITER : 65.3%, 4.1, 138.5ms, 0 | 20260505 11:42:11.611 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 11:42:11.611 | INFO | Uploaded 36GiB to GPU in 2.21221s (16.2733GiB/s) | 20260505 11:42:11.611 | INFO | Downloaded 38GiB to GPU in 2.3098s (16.4516GiB/s) | 20260505 11:42:16.759 | INFO | Analyzing chunk frames = [2933248,2933760) | 20260505 11:42:16.759 | INFO | MemInfo: { | 20260505 11:42:16.759 | INFO | "gpu" : " 17 GB", | 20260505 11:42:16.759 | INFO | "huge" : "", | 20260505 11:42:16.759 | INFO | "huge_shared" : " 27 GB", | 20260505 11:42:16.759 | INFO | "malloc" : " 14 GB", | 20260505 11:42:16.759 | INFO | "pinned" : " 29 GB" | 20260505 11:42:16.759 | INFO | } | 20260505 11:42:16.759 | INFO | Perf report for the last 5.14827 seconds: | 20260505 11:42:16.759 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:42:16.759 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:16.759 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:16.759 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:16.759 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 11:42:16.760 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:16.760 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:16.760 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 11:42:16.760 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:16.760 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:16.760 | INFO | Uploaded 36GiB to GPU in 2.23647s (16.0968GiB/s) | 20260505 11:42:16.760 | INFO | Downloaded 34GiB to GPU in 2.08782s (16.285GiB/s) | 20260505 11:42:21.894 | INFO | Analyzing chunk frames = [2933760,2934272) | 20260505 11:42:21.894 | INFO | MemInfo: { | 20260505 11:42:21.894 | INFO | "gpu" : " 17 GB", | 20260505 11:42:21.894 | INFO | "huge" : "", | 20260505 11:42:21.894 | INFO | "huge_shared" : " 27 GB", | 20260505 11:42:21.895 | INFO | "malloc" : " 14 GB", | 20260505 11:42:21.895 | INFO | "pinned" : " 29 GB" | 20260505 11:42:21.895 | INFO | } | 20260505 11:42:21.895 | INFO | Perf report for the last 5.13508 seconds: | 20260505 11:42:21.895 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:42:21.895 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:21.895 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:21.895 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:21.895 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 11:42:21.895 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:21.895 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:21.895 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:42:21.895 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:21.895 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:21.895 | INFO | Uploaded 36GiB to GPU in 2.21061s (16.2851GiB/s) | 20260505 11:42:21.895 | INFO | Downloaded 34GiB to GPU in 2.12261s (16.018GiB/s) | 20260505 11:42:27.038 | INFO | Analyzing chunk frames = [2934272,2934784) | 20260505 11:42:27.038 | INFO | MemInfo: { | 20260505 11:42:27.038 | INFO | "gpu" : " 17 GB", | 20260505 11:42:27.038 | INFO | "huge" : "", | 20260505 11:42:27.038 | INFO | "huge_shared" : " 27 GB", | 20260505 11:42:27.038 | INFO | "malloc" : " 14 GB", | 20260505 11:42:27.038 | INFO | "pinned" : " 29 GB" | 20260505 11:42:27.038 | INFO | } | 20260505 11:42:27.038 | INFO | Perf report for the last 5.14363 seconds: | 20260505 11:42:27.038 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:42:27.038 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:27.038 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:27.039 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:27.039 | INFO | ANALYSIS : 63.4%, 3.0, 98.5ms, 0 | 20260505 11:42:27.039 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:27.039 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:27.039 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 11:42:27.039 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:27.039 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:27.039 | INFO | Uploaded 36GiB to GPU in 2.21213s (16.2739GiB/s) | 20260505 11:42:27.039 | INFO | Downloaded 34GiB to GPU in 2.13083s (15.9562GiB/s) | 20260505 11:42:32.173 | INFO | Analyzing chunk frames = [2934784,2935296) | 20260505 11:42:32.173 | INFO | MemInfo: { | 20260505 11:42:32.173 | INFO | "gpu" : " 17 GB", | 20260505 11:42:32.173 | INFO | "huge" : "", | 20260505 11:42:32.173 | INFO | "huge_shared" : " 27 GB", | 20260505 11:42:32.173 | INFO | "malloc" : " 14 GB", | 20260505 11:42:32.173 | INFO | "pinned" : " 29 GB" | 20260505 11:42:32.173 | INFO | } | 20260505 11:42:32.174 | INFO | Perf report for the last 5.13531 seconds: | 20260505 11:42:32.174 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:42:32.174 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:32.174 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:32.174 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:32.174 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 11:42:32.174 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:32.174 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:32.174 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 11:42:32.174 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:32.174 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:32.174 | INFO | Uploaded 36GiB to GPU in 2.17736s (16.5338GiB/s) | 20260505 11:42:32.174 | INFO | Downloaded 34GiB to GPU in 2.12866s (15.9725GiB/s) | 20260505 11:42:37.312 | INFO | Analyzing chunk frames = [2935296,2935808) | 20260505 11:42:37.312 | INFO | MemInfo: { | 20260505 11:42:37.312 | INFO | "gpu" : " 17 GB", | 20260505 11:42:37.312 | INFO | "huge" : "", | 20260505 11:42:37.312 | INFO | "huge_shared" : " 27 GB", | 20260505 11:42:37.312 | INFO | "malloc" : " 14 GB", | 20260505 11:42:37.312 | INFO | "pinned" : " 29 GB" | 20260505 11:42:37.312 | INFO | } | 20260505 11:42:37.312 | INFO | Perf report for the last 5.13876 seconds: | 20260505 11:42:37.312 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:42:37.312 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:37.312 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:37.312 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:37.312 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 11:42:37.312 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:37.312 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:37.312 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 11:42:37.312 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:37.312 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:37.313 | INFO | Uploaded 36GiB to GPU in 2.18776s (16.4552GiB/s) | 20260505 11:42:37.313 | INFO | Downloaded 34GiB to GPU in 2.1183s (16.0506GiB/s) | 20260505 11:42:42.454 | INFO | Analyzing chunk frames = [2935808,2936320) | 20260505 11:42:42.454 | INFO | MemInfo: { | 20260505 11:42:42.454 | INFO | "gpu" : " 17 GB", | 20260505 11:42:42.454 | INFO | "huge" : "", | 20260505 11:42:42.454 | INFO | "huge_shared" : " 27 GB", | 20260505 11:42:42.454 | INFO | "malloc" : " 14 GB", | 20260505 11:42:42.454 | INFO | "pinned" : " 29 GB" | 20260505 11:42:42.454 | INFO | } | 20260505 11:42:42.454 | INFO | Perf report for the last 5.14162 seconds: | 20260505 11:42:42.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:42:42.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:42.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:42.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:42.454 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 11:42:42.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:42.454 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:42.454 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 11:42:42.454 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:42.454 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:42.454 | INFO | Uploaded 36GiB to GPU in 2.18769s (16.4557GiB/s) | 20260505 11:42:42.454 | INFO | Downloaded 34GiB to GPU in 2.14491s (15.8515GiB/s) | 20260505 11:42:47.598 | INFO | Analyzing chunk frames = [2936320,2936832) | 20260505 11:42:47.598 | INFO | MemInfo: { | 20260505 11:42:47.598 | INFO | "gpu" : " 17 GB", | 20260505 11:42:47.598 | INFO | "huge" : "", | 20260505 11:42:47.598 | INFO | "huge_shared" : " 27 GB", | 20260505 11:42:47.598 | INFO | "malloc" : " 14 GB", | 20260505 11:42:47.598 | INFO | "pinned" : " 29 GB" | 20260505 11:42:47.598 | INFO | } | 20260505 11:42:47.598 | INFO | Perf report for the last 5.14432 seconds: | 20260505 11:42:47.598 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:42:47.598 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:47.598 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:47.598 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:47.598 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 11:42:47.598 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:47.598 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:47.598 | INFO | PRE_HQ : 29.5%, 1.0, 15.4ms, 0 | 20260505 11:42:47.598 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:47.598 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:47.598 | INFO | Uploaded 36GiB to GPU in 2.17814s (16.5279GiB/s) | 20260505 11:42:47.599 | INFO | Downloaded 34GiB to GPU in 2.10905s (16.121GiB/s) | 20260505 11:42:47.735 | INFO | Starting a round of Baz IO | 20260505 11:42:50.988 | INFO | Populating RTMetrics report | 20260505 11:42:50.989 | INFO | RTMetrics report populated | 20260505 11:42:50.989 | INFO | Opening tmp RTMetrics json file | 20260505 11:42:51.001 | INFO | Serializing RTMetrics to json | 20260505 11:42:51.004 | INFO | Serialization done, swapping tmp file now... | 20260505 11:42:51.011 | INFO | RTMetrics done | 20260505 11:42:51.118 | INFO | Finished a round of Baz IO: Wrote 2.367 GiB in 3.382 seconds | 20260505 11:42:51.118 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.00% / 0.01% / 75.99% | 20260505 11:42:51.118 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.05319% / 2.38419e-05% / 6.75519e-05% | 20260505 11:42:51.118 | INFO | Baz IO overall: | 20260505 11:42:51.118 | INFO | BytesWritten : 1739197645996 | 20260505 11:42:51.118 | INFO | NumEvents : 828054067533 | 20260505 11:42:52.728 | INFO | Analyzing chunk frames = [2936832,2937344) | 20260505 11:42:52.729 | INFO | MemInfo: { | 20260505 11:42:52.729 | INFO | "gpu" : " 17 GB", | 20260505 11:42:52.729 | INFO | "huge" : "", | 20260505 11:42:52.729 | INFO | "huge_shared" : " 27 GB", | 20260505 11:42:52.729 | INFO | "malloc" : " 14 GB", | 20260505 11:42:52.729 | INFO | "pinned" : " 29 GB" | 20260505 11:42:52.729 | INFO | } | 20260505 11:42:52.729 | INFO | Perf report for the last 5.13034 seconds: | 20260505 11:42:52.729 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:42:52.729 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:52.729 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:52.729 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:52.729 | INFO | ANALYSIS : 65.0%, 2.9, 99.9ms, 0 | 20260505 11:42:52.729 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:52.729 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260505 11:42:52.729 | INFO | PRE_HQ : 61.3%, 1.9, 58.9ms, 0 | 20260505 11:42:52.729 | INFO | BAZWRITER : 65.3%, 4.1, 137.3ms, 0 | 20260505 11:42:52.729 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 11:42:52.729 | INFO | Uploaded 36GiB to GPU in 2.20252s (16.3449GiB/s) | 20260505 11:42:52.729 | INFO | Downloaded 38GiB to GPU in 2.29725s (16.5415GiB/s) | 20260505 11:42:57.864 | INFO | Analyzing chunk frames = [2937344,2937856) | 20260505 11:42:57.864 | INFO | MemInfo: { | 20260505 11:42:57.864 | INFO | "gpu" : " 17 GB", | 20260505 11:42:57.864 | INFO | "huge" : "", | 20260505 11:42:57.864 | INFO | "huge_shared" : " 27 GB", | 20260505 11:42:57.864 | INFO | "malloc" : " 14 GB", | 20260505 11:42:57.864 | INFO | "pinned" : " 29 GB" | 20260505 11:42:57.864 | INFO | } | 20260505 11:42:57.864 | INFO | Perf report for the last 5.13549 seconds: | 20260505 11:42:57.864 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:42:57.864 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:57.864 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:57.864 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:57.864 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 11:42:57.864 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:57.864 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:57.864 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 11:42:57.864 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:57.864 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:42:57.864 | INFO | Uploaded 36GiB to GPU in 2.20928s (16.2949GiB/s) | 20260505 11:42:57.864 | INFO | Downloaded 34GiB to GPU in 2.08896s (16.2761GiB/s) | 20260505 11:43:03.015 | INFO | Analyzing chunk frames = [2937856,2938368) | 20260505 11:43:03.015 | INFO | MemInfo: { | 20260505 11:43:03.015 | INFO | "gpu" : " 17 GB", | 20260505 11:43:03.015 | INFO | "huge" : "", | 20260505 11:43:03.015 | INFO | "huge_shared" : " 27 GB", | 20260505 11:43:03.015 | INFO | "malloc" : " 14 GB", | 20260505 11:43:03.015 | INFO | "pinned" : " 29 GB" | 20260505 11:43:03.015 | INFO | } | 20260505 11:43:03.015 | INFO | Perf report for the last 5.15057 seconds: | 20260505 11:43:03.015 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:43:03.015 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:03.015 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:03.015 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:03.015 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 11:43:03.015 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:03.015 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:03.015 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 11:43:03.015 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:03.015 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:03.015 | INFO | Uploaded 36GiB to GPU in 2.14882s (16.7534GiB/s) | 20260505 11:43:03.015 | INFO | Downloaded 34GiB to GPU in 2.06324s (16.479GiB/s) | 20260505 11:43:08.154 | INFO | Analyzing chunk frames = [2938368,2938880) | 20260505 11:43:08.154 | INFO | MemInfo: { | 20260505 11:43:08.154 | INFO | "gpu" : " 17 GB", | 20260505 11:43:08.154 | INFO | "huge" : "", | 20260505 11:43:08.154 | INFO | "huge_shared" : " 27 GB", | 20260505 11:43:08.154 | INFO | "malloc" : " 14 GB", | 20260505 11:43:08.154 | INFO | "pinned" : " 29 GB" | 20260505 11:43:08.154 | INFO | } | 20260505 11:43:08.154 | INFO | Perf report for the last 5.13946 seconds: | 20260505 11:43:08.154 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:43:08.154 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:08.154 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:08.154 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:08.154 | INFO | ANALYSIS : 63.1%, 3.0, 98.0ms, 0 | 20260505 11:43:08.154 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:08.154 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:08.154 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 11:43:08.154 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:08.154 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:08.154 | INFO | Uploaded 36GiB to GPU in 2.19727s (16.384GiB/s) | 20260505 11:43:08.154 | INFO | Downloaded 34GiB to GPU in 2.13109s (15.9543GiB/s) | 20260505 11:43:13.293 | INFO | Analyzing chunk frames = [2938880,2939392) | 20260505 11:43:13.293 | INFO | MemInfo: { | 20260505 11:43:13.293 | INFO | "gpu" : " 17 GB", | 20260505 11:43:13.293 | INFO | "huge" : "", | 20260505 11:43:13.293 | INFO | "huge_shared" : " 27 GB", | 20260505 11:43:13.293 | INFO | "malloc" : " 14 GB", | 20260505 11:43:13.293 | INFO | "pinned" : " 29 GB" | 20260505 11:43:13.293 | INFO | } | 20260505 11:43:13.293 | INFO | Perf report for the last 5.1386 seconds: | 20260505 11:43:13.293 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:43:13.293 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:13.293 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:13.293 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:13.293 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 11:43:13.293 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:13.293 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:13.293 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 11:43:13.293 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:13.293 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:13.293 | INFO | Uploaded 36GiB to GPU in 2.17644s (16.5408GiB/s) | 20260505 11:43:13.293 | INFO | Downloaded 34GiB to GPU in 2.12633s (15.99GiB/s) | 20260505 11:43:18.419 | INFO | Analyzing chunk frames = [2939392,2939904) | 20260505 11:43:18.419 | INFO | MemInfo: { | 20260505 11:43:18.419 | INFO | "gpu" : " 17 GB", | 20260505 11:43:18.419 | INFO | "huge" : "", | 20260505 11:43:18.419 | INFO | "huge_shared" : " 27 GB", | 20260505 11:43:18.419 | INFO | "malloc" : " 14 GB", | 20260505 11:43:18.419 | INFO | "pinned" : " 29 GB" | 20260505 11:43:18.419 | INFO | } | 20260505 11:43:18.419 | INFO | Perf report for the last 5.12629 seconds: | 20260505 11:43:18.419 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:43:18.419 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:18.419 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:18.419 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:18.419 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 11:43:18.419 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:18.419 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:18.419 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 11:43:18.419 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:18.419 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:18.419 | INFO | Uploaded 36GiB to GPU in 2.17042s (16.5867GiB/s) | 20260505 11:43:18.419 | INFO | Downloaded 34GiB to GPU in 2.11544s (16.0723GiB/s) | 20260505 11:43:23.562 | INFO | Analyzing chunk frames = [2939904,2940416) | 20260505 11:43:23.562 | INFO | MemInfo: { | 20260505 11:43:23.562 | INFO | "gpu" : " 17 GB", | 20260505 11:43:23.562 | INFO | "huge" : "", | 20260505 11:43:23.562 | INFO | "huge_shared" : " 27 GB", | 20260505 11:43:23.562 | INFO | "malloc" : " 14 GB", | 20260505 11:43:23.562 | INFO | "pinned" : " 29 GB" | 20260505 11:43:23.562 | INFO | } | 20260505 11:43:23.562 | INFO | Perf report for the last 5.14281 seconds: | 20260505 11:43:23.562 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:43:23.562 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:23.562 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:23.562 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:23.562 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 11:43:23.562 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:23.562 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:23.562 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 11:43:23.562 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:23.562 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:23.562 | INFO | Uploaded 36GiB to GPU in 2.15677s (16.6916GiB/s) | 20260505 11:43:23.562 | INFO | Downloaded 34GiB to GPU in 2.14179s (15.8746GiB/s) | 20260505 11:43:28.710 | INFO | Analyzing chunk frames = [2940416,2940928) | 20260505 11:43:28.710 | INFO | MemInfo: { | 20260505 11:43:28.710 | INFO | "gpu" : " 17 GB", | 20260505 11:43:28.710 | INFO | "huge" : "", | 20260505 11:43:28.710 | INFO | "huge_shared" : " 27 GB", | 20260505 11:43:28.710 | INFO | "malloc" : " 14 GB", | 20260505 11:43:28.710 | INFO | "pinned" : " 29 GB" | 20260505 11:43:28.710 | INFO | } | 20260505 11:43:28.710 | INFO | Perf report for the last 5.14778 seconds: | 20260505 11:43:28.710 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:43:28.710 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:28.710 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:28.710 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:28.710 | INFO | ANALYSIS : 63.8%, 3.0, 99.9ms, 0 | 20260505 11:43:28.710 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:28.710 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:28.710 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 11:43:28.710 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:28.710 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:28.710 | INFO | Uploaded 36GiB to GPU in 2.22801s (16.1579GiB/s) | 20260505 11:43:28.710 | INFO | Downloaded 34GiB to GPU in 2.0988s (16.1997GiB/s) | 20260505 11:43:28.710 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 11:43:28.710 | INFO | Total mem usage: 15.3 GiB | 20260505 11:43:28.710 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 11:43:28.710 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.75%) | 20260505 11:43:28.710 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 11:43:28.710 | INFO | High Water Breakdown: | 20260505 11:43:28.710 | INFO | 61.1% : 8.77 GiB ( 92.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 11:43:28.710 | INFO | 38.8% : 5.57 GiB ( 83.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 11:43:28.710 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 11:43:28.710 | INFO | ------------------------------------------ | 20260505 11:43:28.710 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:43:28.710 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 11:43:28.710 | INFO | Total mem usage: 26.5 GiB | 20260505 11:43:28.710 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 11:43:28.710 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 11:43:28.710 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 11:43:28.710 | INFO | High Water Breakdown: | 20260505 11:43:28.710 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:43:28.710 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:43:28.710 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:43:28.710 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:43:28.710 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:43:28.710 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:43:28.710 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 11:43:28.710 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 11:43:28.710 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:43:28.710 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 11:43:28.710 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 11:43:28.710 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 11:43:28.710 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 11:43:28.710 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 11:43:28.710 | INFO | ------------------------------------------ | 20260505 11:43:28.710 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:43:28.710 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 11:43:28.710 | INFO | Total mem usage: 25.6 GiB | 20260505 11:43:28.710 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 11:43:28.710 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 11:43:28.710 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 11:43:28.710 | INFO | High Water Breakdown: | 20260505 11:43:28.710 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 11:43:28.710 | INFO | ------------------------------------------ | 20260505 11:43:28.710 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:43:28.710 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 11:43:28.710 | INFO | Total mem usage: 15.7 GiB | 20260505 11:43:28.710 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 11:43:28.710 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 11:43:28.710 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 11:43:28.710 | INFO | High Water Breakdown: | 20260505 11:43:28.710 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 11:43:28.710 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 11:43:28.711 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 11:43:28.711 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 11:43:28.711 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 11:43:28.711 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 11:43:28.711 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 11:43:28.711 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 11:43:28.711 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 11:43:28.711 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:43:28.711 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 11:43:28.711 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:43:28.711 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 11:43:28.711 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 11:43:28.711 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 11:43:28.711 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 11:43:28.711 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:43:28.711 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 11:43:28.711 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:43:28.711 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:43:28.711 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:43:28.711 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 11:43:28.711 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 11:43:28.711 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:43:28.711 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 11:43:28.711 | INFO | ------------------------------------------ | 20260505 11:43:28.711 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:43:28.711 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 11:43:28.711 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:43:28.711 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:43:28.711 | INFO | PerfReport: Full Execution -- 3.21e+01 / 7.75e+01, 1.37e+01 / 1.66e+00 | 20260505 11:43:28.711 | INFO | PerfReport: 0.29%, AggregateStats -- 9.30e-02 / 1.01e-01, 1.08e-02 / 4.73e-03 | 20260505 11:43:28.711 | INFO | PerfReport: 11.75%, Baseline -- 3.77e+00 / 4.49e+00, 6.08e-01 / 2.09e-01 | 20260505 11:43:28.711 | INFO | PerfReport: 3.31%, Histogram -- 1.06e+00 / 1.52e+00, 2.05e-01 / 6.02e-02 | 20260505 11:43:28.711 | INFO | PerfReport: 15.97%, DME -- 4.10e+01 / 4.60e+01, 4.64e+00 / 2.50e+00 | 20260505 11:43:28.711 | INFO | PerfReport: 56.25%, FrameLabeling -- 1.81e+01 / 2.05e+01, 1.51e+00 / 2.32e+00 | 20260505 11:43:28.711 | INFO | PerfReport: 7.47%, PulseAccumulating -- 2.40e+00 / 2.92e+00, 4.44e-01 / 1.99e-01 | 20260505 11:43:28.711 | INFO | PerfReport: 3.67%, Metrics -- 1.18e+00 / 1.51e+00, 3.38e-01 / 2.03e-01 | 20260505 11:43:28.711 | INFO | PerfReport: 1.29%, PulseSerializing -- 4.14e-01 / 4.68e-01, 7.61e-02 / 2.61e-02 | 20260505 11:43:28.711 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 11:43:28.711 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:43:28.711 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:43:28.711 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260505 11:43:28.711 | INFO | PerfReport: 22.58%, Upload -- 2.28e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260505 11:43:28.711 | INFO | PerfReport: 22.02%, Download -- 2.23e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260505 11:43:28.711 | INFO | PerfReport: 31.74%, Compute -- 3.21e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 11:43:28.711 | INFO | PerfReport: 2.06%, LockUpload -- 2.08e+00 / 0.00e+00, 7.66e+00 / 0.00e+00 | 20260505 11:43:28.711 | INFO | PerfReport: 1.33%, LockDownload -- 1.35e+00 / 0.00e+00, 2.85e+00 / 0.00e+00 | 20260505 11:43:28.711 | INFO | PerfReport: 20.25%, LockCompute -- 2.05e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260505 11:43:28.711 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.11e-02 / 0.00e+00 | 20260505 11:43:28.840 | INFO | Starting a round of Baz IO | 20260505 11:43:32.090 | INFO | Populating RTMetrics report | 20260505 11:43:32.091 | INFO | RTMetrics report populated | 20260505 11:43:32.091 | INFO | Opening tmp RTMetrics json file | 20260505 11:43:32.108 | INFO | Serializing RTMetrics to json | 20260505 11:43:32.187 | INFO | Serialization done, swapping tmp file now... | 20260505 11:43:32.216 | INFO | RTMetrics done | 20260505 11:43:32.320 | INFO | Finished a round of Baz IO: Wrote 2.366 GiB in 3.48 seconds | 20260505 11:43:32.320 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 24.00% / 0.01% / 76.00% | 20260505 11:43:32.320 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.04702% / 7.94729e-06% / 9.13938e-05% | 20260505 11:43:32.320 | INFO | Baz IO overall: | 20260505 11:43:32.320 | INFO | BytesWritten : 1741738482519 | 20260505 11:43:32.321 | INFO | NumEvents : 829260200131 | 20260505 11:43:33.851 | INFO | Analyzing chunk frames = [2940928,2941440) | 20260505 11:43:33.851 | INFO | MemInfo: { | 20260505 11:43:33.851 | INFO | "gpu" : " 17 GB", | 20260505 11:43:33.851 | INFO | "huge" : "", | 20260505 11:43:33.851 | INFO | "huge_shared" : " 27 GB", | 20260505 11:43:33.851 | INFO | "malloc" : " 14 GB", | 20260505 11:43:33.851 | INFO | "pinned" : " 29 GB" | 20260505 11:43:33.851 | INFO | } | 20260505 11:43:33.851 | INFO | Perf report for the last 5.14104 seconds: | 20260505 11:43:33.851 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:43:33.851 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:33.851 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:33.851 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:33.851 | INFO | ANALYSIS : 64.6%, 2.9, 99.4ms, 0 | 20260505 11:43:33.851 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:33.851 | INFO | RT_METRICS : 48.3%, 1.0, 25.3ms, 0 | 20260505 11:43:33.851 | INFO | PRE_HQ : 61.1%, 1.9, 58.9ms, 0 | 20260505 11:43:33.851 | INFO | BAZWRITER : 66.9%, 4.1, 141.6ms, 0 | 20260505 11:43:33.851 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260505 11:43:33.851 | INFO | Uploaded 36GiB to GPU in 2.18418s (16.4822GiB/s) | 20260505 11:43:33.851 | INFO | Downloaded 38GiB to GPU in 2.29119s (16.5853GiB/s) | 20260505 11:43:38.987 | INFO | Analyzing chunk frames = [2941440,2941952) | 20260505 11:43:38.987 | INFO | MemInfo: { | 20260505 11:43:38.987 | INFO | "gpu" : " 17 GB", | 20260505 11:43:38.987 | INFO | "huge" : "", | 20260505 11:43:38.987 | INFO | "huge_shared" : " 27 GB", | 20260505 11:43:38.987 | INFO | "malloc" : " 14 GB", | 20260505 11:43:38.987 | INFO | "pinned" : " 29 GB" | 20260505 11:43:38.987 | INFO | } | 20260505 11:43:38.987 | INFO | Perf report for the last 5.13604 seconds: | 20260505 11:43:38.987 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:43:38.987 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:38.987 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:38.987 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:38.987 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 11:43:38.987 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:38.987 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:38.987 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 11:43:38.987 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:38.987 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:38.987 | INFO | Uploaded 36GiB to GPU in 2.23134s (16.1338GiB/s) | 20260505 11:43:38.987 | INFO | Downloaded 34GiB to GPU in 2.10451s (16.1558GiB/s) | 20260505 11:43:44.123 | INFO | Analyzing chunk frames = [2941952,2942464) | 20260505 11:43:44.123 | INFO | MemInfo: { | 20260505 11:43:44.123 | INFO | "gpu" : " 17 GB", | 20260505 11:43:44.123 | INFO | "huge" : "", | 20260505 11:43:44.123 | INFO | "huge_shared" : " 27 GB", | 20260505 11:43:44.123 | INFO | "malloc" : " 14 GB", | 20260505 11:43:44.123 | INFO | "pinned" : " 29 GB" | 20260505 11:43:44.123 | INFO | } | 20260505 11:43:44.123 | INFO | Perf report for the last 5.13639 seconds: | 20260505 11:43:44.123 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:43:44.123 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:44.123 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:44.123 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:44.123 | INFO | ANALYSIS : 63.4%, 3.0, 98.6ms, 0 | 20260505 11:43:44.123 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:44.123 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:44.123 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 11:43:44.123 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:44.123 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:44.124 | INFO | Uploaded 36GiB to GPU in 2.20892s (16.2976GiB/s) | 20260505 11:43:44.124 | INFO | Downloaded 34GiB to GPU in 2.12383s (16.0088GiB/s) | 20260505 11:43:49.270 | INFO | Analyzing chunk frames = [2942464,2942976) | 20260505 11:43:49.270 | INFO | MemInfo: { | 20260505 11:43:49.270 | INFO | "gpu" : " 17 GB", | 20260505 11:43:49.270 | INFO | "huge" : "", | 20260505 11:43:49.270 | INFO | "huge_shared" : " 27 GB", | 20260505 11:43:49.270 | INFO | "malloc" : " 14 GB", | 20260505 11:43:49.270 | INFO | "pinned" : " 29 GB" | 20260505 11:43:49.270 | INFO | } | 20260505 11:43:49.270 | INFO | Perf report for the last 5.14725 seconds: | 20260505 11:43:49.270 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:43:49.270 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:49.270 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:49.270 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:49.270 | INFO | ANALYSIS : 63.2%, 3.0, 98.4ms, 0 | 20260505 11:43:49.270 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:49.271 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:49.271 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 11:43:49.271 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:49.271 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:49.271 | INFO | Uploaded 36GiB to GPU in 2.21458s (16.2559GiB/s) | 20260505 11:43:49.271 | INFO | Downloaded 34GiB to GPU in 2.13049s (15.9587GiB/s) | 20260505 11:43:54.400 | INFO | Analyzing chunk frames = [2942976,2943488) | 20260505 11:43:54.400 | INFO | MemInfo: { | 20260505 11:43:54.400 | INFO | "gpu" : " 17 GB", | 20260505 11:43:54.400 | INFO | "huge" : "", | 20260505 11:43:54.400 | INFO | "huge_shared" : " 27 GB", | 20260505 11:43:54.401 | INFO | "malloc" : " 14 GB", | 20260505 11:43:54.401 | INFO | "pinned" : " 29 GB" | 20260505 11:43:54.401 | INFO | } | 20260505 11:43:54.401 | INFO | Perf report for the last 5.13043 seconds: | 20260505 11:43:54.401 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:43:54.401 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:54.401 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:54.401 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:54.401 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 11:43:54.401 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:54.401 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:54.401 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 11:43:54.401 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:54.401 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:54.401 | INFO | Uploaded 36GiB to GPU in 2.15513s (16.7044GiB/s) | 20260505 11:43:54.401 | INFO | Downloaded 34GiB to GPU in 2.1215s (16.0264GiB/s) | 20260505 11:43:59.547 | INFO | Analyzing chunk frames = [2943488,2944000) | 20260505 11:43:59.547 | INFO | MemInfo: { | 20260505 11:43:59.547 | INFO | "gpu" : " 17 GB", | 20260505 11:43:59.547 | INFO | "huge" : "", | 20260505 11:43:59.547 | INFO | "huge_shared" : " 27 GB", | 20260505 11:43:59.547 | INFO | "malloc" : " 14 GB", | 20260505 11:43:59.547 | INFO | "pinned" : " 29 GB" | 20260505 11:43:59.547 | INFO | } | 20260505 11:43:59.547 | INFO | Perf report for the last 5.14625 seconds: | 20260505 11:43:59.547 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:43:59.547 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:59.547 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:59.547 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:59.547 | INFO | ANALYSIS : 63.4%, 3.0, 99.2ms, 0 | 20260505 11:43:59.547 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:59.547 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:59.547 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 11:43:59.547 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:59.547 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:43:59.547 | INFO | Uploaded 36GiB to GPU in 2.20165s (16.3514GiB/s) | 20260505 11:43:59.547 | INFO | Downloaded 34GiB to GPU in 2.12245s (16.0192GiB/s) | 20260505 11:44:04.682 | INFO | Analyzing chunk frames = [2944000,2944512) | 20260505 11:44:04.682 | INFO | MemInfo: { | 20260505 11:44:04.682 | INFO | "gpu" : " 17 GB", | 20260505 11:44:04.682 | INFO | "huge" : "", | 20260505 11:44:04.682 | INFO | "huge_shared" : " 27 GB", | 20260505 11:44:04.682 | INFO | "malloc" : " 14 GB", | 20260505 11:44:04.682 | INFO | "pinned" : " 29 GB" | 20260505 11:44:04.682 | INFO | } | 20260505 11:44:04.682 | INFO | Perf report for the last 5.1351 seconds: | 20260505 11:44:04.682 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:44:04.682 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:04.682 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:04.682 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:04.682 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 11:44:04.682 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:04.682 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:04.682 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 11:44:04.682 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:04.682 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:04.682 | INFO | Uploaded 36GiB to GPU in 2.17092s (16.5828GiB/s) | 20260505 11:44:04.682 | INFO | Downloaded 34GiB to GPU in 2.13623s (15.9159GiB/s) | 20260505 11:44:09.815 | INFO | Analyzing chunk frames = [2944512,2945024) | 20260505 11:44:09.815 | INFO | MemInfo: { | 20260505 11:44:09.815 | INFO | "gpu" : " 17 GB", | 20260505 11:44:09.815 | INFO | "huge" : "", | 20260505 11:44:09.815 | INFO | "huge_shared" : " 27 GB", | 20260505 11:44:09.815 | INFO | "malloc" : " 14 GB", | 20260505 11:44:09.815 | INFO | "pinned" : " 29 GB" | 20260505 11:44:09.815 | INFO | } | 20260505 11:44:09.815 | INFO | Perf report for the last 5.13315 seconds: | 20260505 11:44:09.815 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:44:09.815 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:09.815 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:09.815 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:09.815 | INFO | ANALYSIS : 64.0%, 3.0, 99.8ms, 0 | 20260505 11:44:09.815 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:09.815 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:09.815 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 11:44:09.816 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:09.816 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:09.816 | INFO | Uploaded 36GiB to GPU in 2.22905s (16.1504GiB/s) | 20260505 11:44:09.816 | INFO | Downloaded 34GiB to GPU in 2.09955s (16.194GiB/s) | 20260505 11:44:09.953 | INFO | Starting a round of Baz IO | 20260505 11:44:13.206 | INFO | Populating RTMetrics report | 20260505 11:44:13.206 | INFO | RTMetrics report populated | 20260505 11:44:13.206 | INFO | Opening tmp RTMetrics json file | 20260505 11:44:13.233 | INFO | Serializing RTMetrics to json | 20260505 11:44:13.235 | INFO | Serialization done, swapping tmp file now... | 20260505 11:44:13.239 | INFO | RTMetrics done | 20260505 11:44:13.320 | INFO | Finished a round of Baz IO: Wrote 2.362 GiB in 3.366 seconds | 20260505 11:44:13.320 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.99% / 0.01% / 76.00% | 20260505 11:44:13.320 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.04388% / 3.97364e-06% / 7.54992e-05% | 20260505 11:44:13.320 | INFO | Baz IO overall: | 20260505 11:44:13.320 | INFO | BytesWritten : 1744274536394 | 20260505 11:44:13.320 | INFO | NumEvents : 830464125906 | 20260505 11:44:14.966 | INFO | Analyzing chunk frames = [2945024,2945536) | 20260505 11:44:14.966 | INFO | MemInfo: { | 20260505 11:44:14.966 | INFO | "gpu" : " 17 GB", | 20260505 11:44:14.966 | INFO | "huge" : "", | 20260505 11:44:14.966 | INFO | "huge_shared" : " 27 GB", | 20260505 11:44:14.966 | INFO | "malloc" : " 14 GB", | 20260505 11:44:14.966 | INFO | "pinned" : " 29 GB" | 20260505 11:44:14.966 | INFO | } | 20260505 11:44:14.966 | INFO | Perf report for the last 5.15071 seconds: | 20260505 11:44:14.966 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:44:14.966 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:14.966 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:14.966 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:14.966 | INFO | ANALYSIS : 64.7%, 2.9, 99.9ms, 0 | 20260505 11:44:14.966 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:14.966 | INFO | RT_METRICS : 45.8%, 1.0, 24.1ms, 0 | 20260505 11:44:14.966 | INFO | PRE_HQ : 61.5%, 1.8, 58.9ms, 0 | 20260505 11:44:14.966 | INFO | BAZWRITER : 64.7%, 4.1, 136.7ms, 0 | 20260505 11:44:14.966 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 11:44:14.966 | INFO | Uploaded 36GiB to GPU in 2.18178s (16.5003GiB/s) | 20260505 11:44:14.966 | INFO | Downloaded 38GiB to GPU in 2.29537s (16.555GiB/s) | 20260505 11:44:20.110 | INFO | Analyzing chunk frames = [2945536,2946048) | 20260505 11:44:20.110 | INFO | MemInfo: { | 20260505 11:44:20.110 | INFO | "gpu" : " 17 GB", | 20260505 11:44:20.110 | INFO | "huge" : "", | 20260505 11:44:20.110 | INFO | "huge_shared" : " 27 GB", | 20260505 11:44:20.110 | INFO | "malloc" : " 14 GB", | 20260505 11:44:20.110 | INFO | "pinned" : " 29 GB" | 20260505 11:44:20.110 | INFO | } | 20260505 11:44:20.110 | INFO | Perf report for the last 5.14401 seconds: | 20260505 11:44:20.110 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:44:20.110 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:20.110 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:20.110 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:20.110 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 11:44:20.110 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:20.110 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:20.110 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:44:20.110 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:20.110 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:20.110 | INFO | Uploaded 36GiB to GPU in 2.16742s (16.6096GiB/s) | 20260505 11:44:20.110 | INFO | Downloaded 34GiB to GPU in 2.04062s (16.6616GiB/s) | 20260505 11:44:25.236 | INFO | Analyzing chunk frames = [2946048,2946560) | 20260505 11:44:25.236 | INFO | MemInfo: { | 20260505 11:44:25.236 | INFO | "gpu" : " 17 GB", | 20260505 11:44:25.236 | INFO | "huge" : "", | 20260505 11:44:25.236 | INFO | "huge_shared" : " 27 GB", | 20260505 11:44:25.236 | INFO | "malloc" : " 14 GB", | 20260505 11:44:25.236 | INFO | "pinned" : " 29 GB" | 20260505 11:44:25.236 | INFO | } | 20260505 11:44:25.236 | INFO | Perf report for the last 5.12563 seconds: | 20260505 11:44:25.236 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:44:25.236 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:25.236 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:25.236 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:25.236 | INFO | ANALYSIS : 63.5%, 3.0, 98.6ms, 0 | 20260505 11:44:25.236 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:25.236 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:25.236 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 11:44:25.236 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:25.236 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:25.236 | INFO | Uploaded 36GiB to GPU in 2.18703s (16.4607GiB/s) | 20260505 11:44:25.236 | INFO | Downloaded 34GiB to GPU in 2.13556s (15.9209GiB/s) | 20260505 11:44:30.371 | INFO | Analyzing chunk frames = [2946560,2947072) | 20260505 11:44:30.371 | INFO | MemInfo: { | 20260505 11:44:30.371 | INFO | "gpu" : " 17 GB", | 20260505 11:44:30.371 | INFO | "huge" : "", | 20260505 11:44:30.371 | INFO | "huge_shared" : " 27 GB", | 20260505 11:44:30.371 | INFO | "malloc" : " 14 GB", | 20260505 11:44:30.371 | INFO | "pinned" : " 29 GB" | 20260505 11:44:30.371 | INFO | } | 20260505 11:44:30.372 | INFO | Perf report for the last 5.13573 seconds: | 20260505 11:44:30.372 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:44:30.372 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:30.372 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:30.372 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:30.372 | INFO | ANALYSIS : 63.6%, 3.0, 98.8ms, 0 | 20260505 11:44:30.372 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:30.372 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:30.372 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 11:44:30.372 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:30.372 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:30.372 | INFO | Uploaded 36GiB to GPU in 2.21858s (16.2266GiB/s) | 20260505 11:44:30.372 | INFO | Downloaded 34GiB to GPU in 2.1343s (15.9303GiB/s) | 20260505 11:44:35.519 | INFO | Analyzing chunk frames = [2947072,2947584) | 20260505 11:44:35.519 | INFO | MemInfo: { | 20260505 11:44:35.519 | INFO | "gpu" : " 17 GB", | 20260505 11:44:35.519 | INFO | "huge" : "", | 20260505 11:44:35.520 | INFO | "huge_shared" : " 27 GB", | 20260505 11:44:35.520 | INFO | "malloc" : " 14 GB", | 20260505 11:44:35.520 | INFO | "pinned" : " 29 GB" | 20260505 11:44:35.520 | INFO | } | 20260505 11:44:35.520 | INFO | Perf report for the last 5.14809 seconds: | 20260505 11:44:35.520 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:44:35.520 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:35.520 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:35.520 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:35.520 | INFO | ANALYSIS : 63.2%, 3.0, 98.8ms, 0 | 20260505 11:44:35.520 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:35.520 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:35.520 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 11:44:35.520 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:35.520 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:35.520 | INFO | Uploaded 36GiB to GPU in 2.15815s (16.6809GiB/s) | 20260505 11:44:35.520 | INFO | Downloaded 34GiB to GPU in 2.12839s (15.9745GiB/s) | 20260505 11:44:40.667 | INFO | Analyzing chunk frames = [2947584,2948096) | 20260505 11:44:40.667 | INFO | MemInfo: { | 20260505 11:44:40.667 | INFO | "gpu" : " 17 GB", | 20260505 11:44:40.667 | INFO | "huge" : "", | 20260505 11:44:40.667 | INFO | "huge_shared" : " 27 GB", | 20260505 11:44:40.667 | INFO | "malloc" : " 14 GB", | 20260505 11:44:40.667 | INFO | "pinned" : " 29 GB" | 20260505 11:44:40.667 | INFO | } | 20260505 11:44:40.667 | INFO | Perf report for the last 5.14714 seconds: | 20260505 11:44:40.667 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:44:40.667 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:40.667 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:40.667 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:40.667 | INFO | ANALYSIS : 63.2%, 3.0, 98.8ms, 0 | 20260505 11:44:40.667 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:40.667 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:40.667 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 11:44:40.667 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:40.667 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:40.667 | INFO | Uploaded 36GiB to GPU in 2.15048s (16.7404GiB/s) | 20260505 11:44:40.667 | INFO | Downloaded 34GiB to GPU in 2.10815s (16.1279GiB/s) | 20260505 11:44:45.800 | INFO | Analyzing chunk frames = [2948096,2948608) | 20260505 11:44:45.800 | INFO | MemInfo: { | 20260505 11:44:45.800 | INFO | "gpu" : " 17 GB", | 20260505 11:44:45.800 | INFO | "huge" : "", | 20260505 11:44:45.800 | INFO | "huge_shared" : " 27 GB", | 20260505 11:44:45.800 | INFO | "malloc" : " 14 GB", | 20260505 11:44:45.800 | INFO | "pinned" : " 29 GB" | 20260505 11:44:45.800 | INFO | } | 20260505 11:44:45.800 | INFO | Perf report for the last 5.13353 seconds: | 20260505 11:44:45.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:44:45.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:45.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:45.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:45.800 | INFO | ANALYSIS : 63.4%, 3.0, 98.8ms, 0 | 20260505 11:44:45.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:45.800 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:45.800 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 11:44:45.800 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:45.800 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:45.801 | INFO | Uploaded 36GiB to GPU in 2.16599s (16.6205GiB/s) | 20260505 11:44:45.801 | INFO | Downloaded 34GiB to GPU in 2.16037s (15.7381GiB/s) | 20260505 11:44:50.936 | INFO | Analyzing chunk frames = [2948608,2949120) | 20260505 11:44:50.936 | INFO | MemInfo: { | 20260505 11:44:50.936 | INFO | "gpu" : " 17 GB", | 20260505 11:44:50.936 | INFO | "huge" : "", | 20260505 11:44:50.936 | INFO | "huge_shared" : " 27 GB", | 20260505 11:44:50.936 | INFO | "malloc" : " 14 GB", | 20260505 11:44:50.936 | INFO | "pinned" : " 29 GB" | 20260505 11:44:50.936 | INFO | } | 20260505 11:44:50.936 | INFO | Perf report for the last 5.13533 seconds: | 20260505 11:44:50.936 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:44:50.936 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:50.936 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:50.936 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:50.936 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 11:44:50.936 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:50.936 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:50.936 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 11:44:50.936 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:50.936 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:50.936 | INFO | Uploaded 36GiB to GPU in 2.17826s (16.527GiB/s) | 20260505 11:44:50.936 | INFO | Downloaded 34GiB to GPU in 2.097s (16.2136GiB/s) | 20260505 11:44:50.936 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 11:44:50.936 | INFO | Total mem usage: 15.3 GiB | 20260505 11:44:50.936 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 11:44:50.936 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.43%) | 20260505 11:44:50.936 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 11:44:50.936 | INFO | High Water Breakdown: | 20260505 11:44:50.936 | INFO | 61% : 8.79 GiB ( 92.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 11:44:50.936 | INFO | 38.8% : 5.59 GiB ( 84.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 11:44:50.936 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 11:44:50.936 | INFO | ------------------------------------------ | 20260505 11:44:50.936 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:44:50.936 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 11:44:50.936 | INFO | Total mem usage: 26.5 GiB | 20260505 11:44:50.936 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 11:44:50.936 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 11:44:50.936 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 11:44:50.936 | INFO | High Water Breakdown: | 20260505 11:44:50.936 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:44:50.936 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:44:50.936 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:44:50.936 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:44:50.936 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:44:50.936 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:44:50.936 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 11:44:50.936 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 11:44:50.936 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:44:50.936 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 11:44:50.936 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 11:44:50.936 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 11:44:50.936 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 11:44:50.936 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 11:44:50.936 | INFO | ------------------------------------------ | 20260505 11:44:50.936 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:44:50.936 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 11:44:50.936 | INFO | Total mem usage: 25.6 GiB | 20260505 11:44:50.936 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 11:44:50.936 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 11:44:50.936 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 11:44:50.936 | INFO | High Water Breakdown: | 20260505 11:44:50.936 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 11:44:50.936 | INFO | ------------------------------------------ | 20260505 11:44:50.936 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:44:50.936 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 11:44:50.936 | INFO | Total mem usage: 15.7 GiB | 20260505 11:44:50.936 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 11:44:50.936 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 11:44:50.936 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 11:44:50.936 | INFO | High Water Breakdown: | 20260505 11:44:50.936 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 11:44:50.936 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 11:44:50.936 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 11:44:50.936 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 11:44:50.936 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 11:44:50.937 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 11:44:50.937 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 11:44:50.937 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 11:44:50.937 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 11:44:50.937 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:44:50.937 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 11:44:50.937 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:44:50.937 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 11:44:50.937 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 11:44:50.937 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 11:44:50.937 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 11:44:50.937 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:44:50.937 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 11:44:50.937 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:44:50.937 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:44:50.937 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:44:50.937 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 11:44:50.937 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 11:44:50.937 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:44:50.937 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 11:44:50.937 | INFO | ------------------------------------------ | 20260505 11:44:50.937 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:44:50.937 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 11:44:50.937 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:44:50.937 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:44:50.937 | INFO | PerfReport: Full Execution -- 3.21e+01 / 7.73e+01, 1.38e+01 / 1.68e+00 | 20260505 11:44:50.937 | INFO | PerfReport: 0.29%, AggregateStats -- 9.27e-02 / 9.96e-02, 1.01e-02 / 5.94e-03 | 20260505 11:44:50.937 | INFO | PerfReport: 11.75%, Baseline -- 3.77e+00 / 4.48e+00, 6.02e-01 / 2.52e-01 | 20260505 11:44:50.937 | INFO | PerfReport: 3.30%, Histogram -- 1.06e+00 / 1.49e+00, 2.02e-01 / 1.02e-01 | 20260505 11:44:50.937 | INFO | PerfReport: 15.90%, DME -- 4.08e+01 / 4.53e+01, 4.30e+00 / 2.26e+00 | 20260505 11:44:50.937 | INFO | PerfReport: 56.39%, FrameLabeling -- 1.81e+01 / 2.11e+01, 1.56e+00 / 2.35e+00 | 20260505 11:44:50.937 | INFO | PerfReport: 7.43%, PulseAccumulating -- 2.39e+00 / 2.85e+00, 4.42e-01 / 2.75e-01 | 20260505 11:44:50.937 | INFO | PerfReport: 3.66%, Metrics -- 1.17e+00 / 1.47e+00, 3.33e-01 / 1.91e-01 | 20260505 11:44:50.937 | INFO | PerfReport: 1.29%, PulseSerializing -- 4.14e-01 / 4.64e-01, 7.58e-02 / 4.66e-02 | 20260505 11:44:50.937 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 11:44:50.937 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:44:50.937 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:44:50.937 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260505 11:44:50.937 | INFO | PerfReport: 22.53%, Upload -- 2.28e+01 / 0.00e+00, 8.07e+00 / 0.00e+00 | 20260505 11:44:50.937 | INFO | PerfReport: 22.03%, Download -- 2.23e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260505 11:44:50.937 | INFO | PerfReport: 31.74%, Compute -- 3.21e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 11:44:50.937 | INFO | PerfReport: 2.10%, LockUpload -- 2.13e+00 / 0.00e+00, 7.79e+00 / 0.00e+00 | 20260505 11:44:50.937 | INFO | PerfReport: 1.37%, LockDownload -- 1.39e+00 / 0.00e+00, 3.00e+00 / 0.00e+00 | 20260505 11:44:50.937 | INFO | PerfReport: 20.20%, LockCompute -- 2.04e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260505 11:44:50.937 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.51e-02 / 0.00e+00 | 20260505 11:44:51.072 | INFO | Starting a round of Baz IO | 20260505 11:44:54.331 | INFO | Populating RTMetrics report | 20260505 11:44:54.332 | INFO | RTMetrics report populated | 20260505 11:44:54.332 | INFO | Opening tmp RTMetrics json file | 20260505 11:44:54.352 | INFO | Serializing RTMetrics to json | 20260505 11:44:54.361 | INFO | Serialization done, swapping tmp file now... | 20260505 11:44:54.377 | INFO | RTMetrics done | 20260505 11:44:54.449 | INFO | Finished a round of Baz IO: Wrote 2.362 GiB in 3.376 seconds | 20260505 11:44:54.449 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.98% / 0.01% / 76.01% | 20260505 11:44:54.449 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.04483% / 2.38419e-05% / 6.35783e-05% | 20260505 11:44:54.449 | INFO | Baz IO overall: | 20260505 11:44:54.449 | INFO | BytesWritten : 1746810820390 | 20260505 11:44:54.449 | INFO | NumEvents : 831668862621 | 20260505 11:44:56.072 | INFO | Analyzing chunk frames = [2949120,2949632) | 20260505 11:44:56.072 | INFO | MemInfo: { | 20260505 11:44:56.072 | INFO | "gpu" : " 17 GB", | 20260505 11:44:56.072 | INFO | "huge" : "", | 20260505 11:44:56.072 | INFO | "huge_shared" : " 27 GB", | 20260505 11:44:56.072 | INFO | "malloc" : " 14 GB", | 20260505 11:44:56.072 | INFO | "pinned" : " 29 GB" | 20260505 11:44:56.072 | INFO | } | 20260505 11:44:56.072 | INFO | Perf report for the last 5.13672 seconds: | 20260505 11:44:56.072 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:44:56.072 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:56.072 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:56.073 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:56.073 | INFO | ANALYSIS : 65.0%, 2.9, 100.3ms, 0 | 20260505 11:44:56.073 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:44:56.073 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260505 11:44:56.073 | INFO | PRE_HQ : 61.4%, 1.8, 58.7ms, 0 | 20260505 11:44:56.073 | INFO | BAZWRITER : 65.1%, 4.0, 135.1ms, 0 | 20260505 11:44:56.073 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 11:44:56.073 | INFO | Uploaded 36GiB to GPU in 2.23172s (16.1311GiB/s) | 20260505 11:44:56.073 | INFO | Downloaded 38GiB to GPU in 2.28673s (16.6176GiB/s) | 20260505 11:45:01.214 | INFO | Analyzing chunk frames = [2949632,2950144) | 20260505 11:45:01.215 | INFO | MemInfo: { | 20260505 11:45:01.215 | INFO | "gpu" : " 17 GB", | 20260505 11:45:01.215 | INFO | "huge" : "", | 20260505 11:45:01.215 | INFO | "huge_shared" : " 27 GB", | 20260505 11:45:01.215 | INFO | "malloc" : " 14 GB", | 20260505 11:45:01.215 | INFO | "pinned" : " 29 GB" | 20260505 11:45:01.215 | INFO | } | 20260505 11:45:01.215 | INFO | Perf report for the last 5.14218 seconds: | 20260505 11:45:01.215 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:45:01.215 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:01.215 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:01.215 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:01.215 | INFO | ANALYSIS : 64.1%, 3.0, 100.2ms, 0 | 20260505 11:45:01.215 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:01.215 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:01.215 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 11:45:01.215 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:01.215 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:01.215 | INFO | Uploaded 36GiB to GPU in 2.2304s (16.1406GiB/s) | 20260505 11:45:01.215 | INFO | Downloaded 34GiB to GPU in 2.09007s (16.2674GiB/s) | 20260505 11:45:06.355 | INFO | Analyzing chunk frames = [2950144,2950656) | 20260505 11:45:06.355 | INFO | MemInfo: { | 20260505 11:45:06.355 | INFO | "gpu" : " 17 GB", | 20260505 11:45:06.355 | INFO | "huge" : "", | 20260505 11:45:06.355 | INFO | "huge_shared" : " 27 GB", | 20260505 11:45:06.355 | INFO | "malloc" : " 14 GB", | 20260505 11:45:06.355 | INFO | "pinned" : " 29 GB" | 20260505 11:45:06.355 | INFO | } | 20260505 11:45:06.355 | INFO | Perf report for the last 5.14065 seconds: | 20260505 11:45:06.355 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:45:06.355 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:06.355 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:06.355 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:06.355 | INFO | ANALYSIS : 63.2%, 3.0, 98.4ms, 0 | 20260505 11:45:06.355 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:06.355 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:06.355 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 11:45:06.355 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:06.355 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:06.355 | INFO | Uploaded 36GiB to GPU in 2.21752s (16.2344GiB/s) | 20260505 11:45:06.355 | INFO | Downloaded 34GiB to GPU in 2.11593s (16.0686GiB/s) | 20260505 11:45:11.493 | INFO | Analyzing chunk frames = [2950656,2951168) | 20260505 11:45:11.493 | INFO | MemInfo: { | 20260505 11:45:11.493 | INFO | "gpu" : " 17 GB", | 20260505 11:45:11.493 | INFO | "huge" : "", | 20260505 11:45:11.493 | INFO | "huge_shared" : " 27 GB", | 20260505 11:45:11.493 | INFO | "malloc" : " 14 GB", | 20260505 11:45:11.493 | INFO | "pinned" : " 29 GB" | 20260505 11:45:11.493 | INFO | } | 20260505 11:45:11.493 | INFO | Perf report for the last 5.13815 seconds: | 20260505 11:45:11.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:45:11.493 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:11.493 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:11.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:11.493 | INFO | ANALYSIS : 63.4%, 3.0, 98.4ms, 0 | 20260505 11:45:11.494 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:11.494 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:11.494 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 11:45:11.494 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:11.494 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:11.494 | INFO | Uploaded 36GiB to GPU in 2.20953s (16.293GiB/s) | 20260505 11:45:11.494 | INFO | Downloaded 34GiB to GPU in 2.13234s (15.9449GiB/s) | 20260505 11:45:16.631 | INFO | Analyzing chunk frames = [2951168,2951680) | 20260505 11:45:16.631 | INFO | MemInfo: { | 20260505 11:45:16.631 | INFO | "gpu" : " 17 GB", | 20260505 11:45:16.631 | INFO | "huge" : "", | 20260505 11:45:16.631 | INFO | "huge_shared" : " 27 GB", | 20260505 11:45:16.631 | INFO | "malloc" : " 14 GB", | 20260505 11:45:16.631 | INFO | "pinned" : " 29 GB" | 20260505 11:45:16.631 | INFO | } | 20260505 11:45:16.631 | INFO | Perf report for the last 5.13763 seconds: | 20260505 11:45:16.631 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:45:16.631 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:16.631 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:16.631 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:16.631 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 11:45:16.631 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:16.631 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:16.631 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 11:45:16.631 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:16.631 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:16.631 | INFO | Uploaded 36GiB to GPU in 2.16459s (16.6313GiB/s) | 20260505 11:45:16.631 | INFO | Downloaded 34GiB to GPU in 2.11371s (16.0855GiB/s) | 20260505 11:45:21.777 | INFO | Analyzing chunk frames = [2951680,2952192) | 20260505 11:45:21.777 | INFO | MemInfo: { | 20260505 11:45:21.777 | INFO | "gpu" : " 17 GB", | 20260505 11:45:21.777 | INFO | "huge" : "", | 20260505 11:45:21.777 | INFO | "huge_shared" : " 27 GB", | 20260505 11:45:21.777 | INFO | "malloc" : " 14 GB", | 20260505 11:45:21.777 | INFO | "pinned" : " 29 GB" | 20260505 11:45:21.777 | INFO | } | 20260505 11:45:21.777 | INFO | Perf report for the last 5.14597 seconds: | 20260505 11:45:21.777 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:45:21.777 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:21.777 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:21.777 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:21.777 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 11:45:21.777 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:21.777 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:21.777 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 11:45:21.777 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:21.777 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:21.777 | INFO | Uploaded 36GiB to GPU in 2.18377s (16.4852GiB/s) | 20260505 11:45:21.777 | INFO | Downloaded 34GiB to GPU in 2.10509s (16.1513GiB/s) | 20260505 11:45:26.919 | INFO | Analyzing chunk frames = [2952192,2952704) | 20260505 11:45:26.919 | INFO | MemInfo: { | 20260505 11:45:26.919 | INFO | "gpu" : " 17 GB", | 20260505 11:45:26.919 | INFO | "huge" : "", | 20260505 11:45:26.919 | INFO | "huge_shared" : " 27 GB", | 20260505 11:45:26.919 | INFO | "malloc" : " 14 GB", | 20260505 11:45:26.919 | INFO | "pinned" : " 29 GB" | 20260505 11:45:26.919 | INFO | } | 20260505 11:45:26.919 | INFO | Perf report for the last 5.14215 seconds: | 20260505 11:45:26.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:45:26.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:26.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:26.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:26.919 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 11:45:26.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:26.919 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:26.919 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 11:45:26.919 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:26.919 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:26.920 | INFO | Uploaded 36GiB to GPU in 2.15718s (16.6884GiB/s) | 20260505 11:45:26.920 | INFO | Downloaded 34GiB to GPU in 2.13617s (15.9163GiB/s) | 20260505 11:45:32.057 | INFO | Analyzing chunk frames = [2952704,2953216) | 20260505 11:45:32.057 | INFO | MemInfo: { | 20260505 11:45:32.057 | INFO | "gpu" : " 17 GB", | 20260505 11:45:32.057 | INFO | "huge" : "", | 20260505 11:45:32.058 | INFO | "huge_shared" : " 27 GB", | 20260505 11:45:32.058 | INFO | "malloc" : " 14 GB", | 20260505 11:45:32.058 | INFO | "pinned" : " 29 GB" | 20260505 11:45:32.058 | INFO | } | 20260505 11:45:32.058 | INFO | Perf report for the last 5.13834 seconds: | 20260505 11:45:32.058 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:45:32.058 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:32.058 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:32.058 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:32.058 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 11:45:32.058 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:32.058 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:32.058 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 11:45:32.058 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:32.058 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:32.058 | INFO | Uploaded 36GiB to GPU in 2.21011s (16.2888GiB/s) | 20260505 11:45:32.058 | INFO | Downloaded 34GiB to GPU in 2.09264s (16.2474GiB/s) | 20260505 11:45:32.189 | INFO | Starting a round of Baz IO | 20260505 11:45:35.433 | INFO | Populating RTMetrics report | 20260505 11:45:35.433 | INFO | RTMetrics report populated | 20260505 11:45:35.433 | INFO | Opening tmp RTMetrics json file | 20260505 11:45:35.436 | INFO | Serializing RTMetrics to json | 20260505 11:45:35.438 | INFO | Serialization done, swapping tmp file now... | 20260505 11:45:35.440 | INFO | RTMetrics done | 20260505 11:45:35.562 | INFO | Finished a round of Baz IO: Wrote 2.361 GiB in 3.372 seconds | 20260505 11:45:35.562 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.97% / 0.01% / 76.02% | 20260505 11:45:35.562 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.04215% / 3.97364e-06% / 7.15256e-05% | 20260505 11:45:35.562 | INFO | Baz IO overall: | 20260505 11:45:35.562 | INFO | BytesWritten : 1749345697694 | 20260505 11:45:35.562 | INFO | NumEvents : 832872578397 | 20260505 11:45:37.185 | INFO | Analyzing chunk frames = [2953216,2953728) | 20260505 11:45:37.185 | INFO | MemInfo: { | 20260505 11:45:37.185 | INFO | "gpu" : " 17 GB", | 20260505 11:45:37.185 | INFO | "huge" : "", | 20260505 11:45:37.185 | INFO | "huge_shared" : " 27 GB", | 20260505 11:45:37.185 | INFO | "malloc" : " 14 GB", | 20260505 11:45:37.185 | INFO | "pinned" : " 29 GB" | 20260505 11:45:37.185 | INFO | } | 20260505 11:45:37.185 | INFO | Perf report for the last 5.12751 seconds: | 20260505 11:45:37.185 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:45:37.185 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:37.185 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:37.185 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:37.185 | INFO | ANALYSIS : 64.7%, 2.9, 99.6ms, 0 | 20260505 11:45:37.185 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:37.185 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260505 11:45:37.185 | INFO | PRE_HQ : 61.2%, 1.9, 58.8ms, 0 | 20260505 11:45:37.185 | INFO | BAZWRITER : 64.9%, 4.1, 136.9ms, 0 | 20260505 11:45:37.185 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 11:45:37.185 | INFO | Uploaded 36GiB to GPU in 2.17556s (16.5475GiB/s) | 20260505 11:45:37.185 | INFO | Downloaded 38GiB to GPU in 2.27816s (16.6801GiB/s) | 20260505 11:45:42.328 | INFO | Analyzing chunk frames = [2953728,2954240) | 20260505 11:45:42.328 | INFO | MemInfo: { | 20260505 11:45:42.328 | INFO | "gpu" : " 17 GB", | 20260505 11:45:42.328 | INFO | "huge" : "", | 20260505 11:45:42.328 | INFO | "huge_shared" : " 27 GB", | 20260505 11:45:42.328 | INFO | "malloc" : " 14 GB", | 20260505 11:45:42.328 | INFO | "pinned" : " 29 GB" | 20260505 11:45:42.328 | INFO | } | 20260505 11:45:42.328 | INFO | Perf report for the last 5.14277 seconds: | 20260505 11:45:42.328 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:45:42.328 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:42.328 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:42.328 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:42.328 | INFO | ANALYSIS : 64.0%, 3.0, 100.1ms, 0 | 20260505 11:45:42.328 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:42.328 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:42.328 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 11:45:42.328 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:42.328 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:42.328 | INFO | Uploaded 36GiB to GPU in 2.23064s (16.1388GiB/s) | 20260505 11:45:42.328 | INFO | Downloaded 34GiB to GPU in 2.08407s (16.3142GiB/s) | 20260505 11:45:47.471 | INFO | Analyzing chunk frames = [2954240,2954752) | 20260505 11:45:47.471 | INFO | MemInfo: { | 20260505 11:45:47.471 | INFO | "gpu" : " 17 GB", | 20260505 11:45:47.471 | INFO | "huge" : "", | 20260505 11:45:47.471 | INFO | "huge_shared" : " 27 GB", | 20260505 11:45:47.471 | INFO | "malloc" : " 14 GB", | 20260505 11:45:47.471 | INFO | "pinned" : " 29 GB" | 20260505 11:45:47.471 | INFO | } | 20260505 11:45:47.471 | INFO | Perf report for the last 5.14347 seconds: | 20260505 11:45:47.471 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:45:47.471 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:47.471 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:47.471 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:47.471 | INFO | ANALYSIS : 63.4%, 3.0, 98.8ms, 0 | 20260505 11:45:47.471 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:47.471 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:47.471 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 11:45:47.471 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:47.471 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:47.472 | INFO | Uploaded 36GiB to GPU in 2.23332s (16.1195GiB/s) | 20260505 11:45:47.472 | INFO | Downloaded 34GiB to GPU in 2.12886s (15.971GiB/s) | 20260505 11:45:52.619 | INFO | Analyzing chunk frames = [2954752,2955264) | 20260505 11:45:52.620 | INFO | MemInfo: { | 20260505 11:45:52.620 | INFO | "gpu" : " 17 GB", | 20260505 11:45:52.620 | INFO | "huge" : "", | 20260505 11:45:52.620 | INFO | "huge_shared" : " 27 GB", | 20260505 11:45:52.620 | INFO | "malloc" : " 14 GB", | 20260505 11:45:52.620 | INFO | "pinned" : " 29 GB" | 20260505 11:45:52.620 | INFO | } | 20260505 11:45:52.620 | INFO | Perf report for the last 5.14826 seconds: | 20260505 11:45:52.620 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:45:52.620 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:52.620 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:52.620 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:52.620 | INFO | ANALYSIS : 63.0%, 3.0, 98.1ms, 0 | 20260505 11:45:52.620 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:52.620 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:52.620 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:45:52.620 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:52.620 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:52.620 | INFO | Uploaded 36GiB to GPU in 2.20207s (16.3482GiB/s) | 20260505 11:45:52.620 | INFO | Downloaded 34GiB to GPU in 2.12898s (15.9701GiB/s) | 20260505 11:45:57.751 | INFO | Analyzing chunk frames = [2955264,2955776) | 20260505 11:45:57.751 | INFO | MemInfo: { | 20260505 11:45:57.751 | INFO | "gpu" : " 17 GB", | 20260505 11:45:57.751 | INFO | "huge" : "", | 20260505 11:45:57.751 | INFO | "huge_shared" : " 27 GB", | 20260505 11:45:57.751 | INFO | "malloc" : " 14 GB", | 20260505 11:45:57.751 | INFO | "pinned" : " 29 GB" | 20260505 11:45:57.751 | INFO | } | 20260505 11:45:57.751 | INFO | Perf report for the last 5.13135 seconds: | 20260505 11:45:57.751 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:45:57.751 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:57.751 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:57.751 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:57.751 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 11:45:57.751 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:57.751 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:57.751 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:45:57.751 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:57.751 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:45:57.751 | INFO | Uploaded 36GiB to GPU in 2.15077s (16.7382GiB/s) | 20260505 11:45:57.751 | INFO | Downloaded 34GiB to GPU in 2.12241s (16.0195GiB/s) | 20260505 11:46:02.884 | INFO | Analyzing chunk frames = [2955776,2956288) | 20260505 11:46:02.884 | INFO | MemInfo: { | 20260505 11:46:02.884 | INFO | "gpu" : " 17 GB", | 20260505 11:46:02.884 | INFO | "huge" : "", | 20260505 11:46:02.884 | INFO | "huge_shared" : " 27 GB", | 20260505 11:46:02.884 | INFO | "malloc" : " 14 GB", | 20260505 11:46:02.884 | INFO | "pinned" : " 29 GB" | 20260505 11:46:02.884 | INFO | } | 20260505 11:46:02.884 | INFO | Perf report for the last 5.13305 seconds: | 20260505 11:46:02.884 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:46:02.884 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:02.884 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:02.884 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:02.884 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 11:46:02.884 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:02.884 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:02.884 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 11:46:02.884 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:02.884 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:02.884 | INFO | Uploaded 36GiB to GPU in 2.15512s (16.7044GiB/s) | 20260505 11:46:02.884 | INFO | Downloaded 34GiB to GPU in 2.0943s (16.2346GiB/s) | 20260505 11:46:08.027 | INFO | Analyzing chunk frames = [2956288,2956800) | 20260505 11:46:08.027 | INFO | MemInfo: { | 20260505 11:46:08.027 | INFO | "gpu" : " 17 GB", | 20260505 11:46:08.027 | INFO | "huge" : "", | 20260505 11:46:08.027 | INFO | "huge_shared" : " 27 GB", | 20260505 11:46:08.027 | INFO | "malloc" : " 14 GB", | 20260505 11:46:08.027 | INFO | "pinned" : " 29 GB" | 20260505 11:46:08.027 | INFO | } | 20260505 11:46:08.027 | INFO | Perf report for the last 5.1431 seconds: | 20260505 11:46:08.027 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:46:08.027 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:08.027 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:08.027 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:08.027 | INFO | ANALYSIS : 63.3%, 3.0, 99.0ms, 0 | 20260505 11:46:08.027 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:08.027 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:08.027 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260505 11:46:08.027 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:08.027 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:08.027 | INFO | Uploaded 36GiB to GPU in 2.1693s (16.5952GiB/s) | 20260505 11:46:08.028 | INFO | Downloaded 34GiB to GPU in 2.15995s (15.7411GiB/s) | 20260505 11:46:13.165 | INFO | Analyzing chunk frames = [2956800,2957312) | 20260505 11:46:13.165 | INFO | MemInfo: { | 20260505 11:46:13.165 | INFO | "gpu" : " 17 GB", | 20260505 11:46:13.165 | INFO | "huge" : "", | 20260505 11:46:13.165 | INFO | "huge_shared" : " 27 GB", | 20260505 11:46:13.165 | INFO | "malloc" : " 14 GB", | 20260505 11:46:13.165 | INFO | "pinned" : " 29 GB" | 20260505 11:46:13.165 | INFO | } | 20260505 11:46:13.165 | INFO | Perf report for the last 5.13787 seconds: | 20260505 11:46:13.165 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:46:13.165 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:13.165 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:13.165 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:13.165 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 11:46:13.165 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:13.165 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:13.165 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 11:46:13.165 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:13.165 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:13.165 | INFO | Uploaded 36GiB to GPU in 2.15084s (16.7376GiB/s) | 20260505 11:46:13.165 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 11:46:13.165 | INFO | Total mem usage: 15.3 GiB | 20260505 11:46:13.165 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 11:46:13.165 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.79%) | 20260505 11:46:13.165 | INFO | Downloaded 34GiB to GPU in 2.10814s (16.1279GiB/s) | 20260505 11:46:13.165 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 11:46:13.165 | INFO | High Water Breakdown: | 20260505 11:46:13.165 | INFO | 61.1% : 8.76 GiB ( 92.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 11:46:13.165 | INFO | 38.8% : 5.57 GiB ( 83.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 11:46:13.165 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 11:46:13.165 | INFO | ------------------------------------------ | 20260505 11:46:13.165 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:46:13.165 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 11:46:13.165 | INFO | Total mem usage: 26.5 GiB | 20260505 11:46:13.165 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 11:46:13.165 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 11:46:13.165 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 11:46:13.165 | INFO | High Water Breakdown: | 20260505 11:46:13.166 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:46:13.166 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:46:13.166 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:46:13.166 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:46:13.166 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:46:13.166 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:46:13.166 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 11:46:13.166 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 11:46:13.166 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:46:13.166 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 11:46:13.166 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 11:46:13.166 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 11:46:13.166 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 11:46:13.166 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 11:46:13.166 | INFO | ------------------------------------------ | 20260505 11:46:13.166 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:46:13.166 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 11:46:13.166 | INFO | Total mem usage: 25.6 GiB | 20260505 11:46:13.166 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 11:46:13.166 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 11:46:13.166 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 11:46:13.166 | INFO | High Water Breakdown: | 20260505 11:46:13.166 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 11:46:13.166 | INFO | ------------------------------------------ | 20260505 11:46:13.166 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:46:13.166 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 11:46:13.166 | INFO | Total mem usage: 15.7 GiB | 20260505 11:46:13.166 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 11:46:13.166 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 11:46:13.166 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 11:46:13.166 | INFO | High Water Breakdown: | 20260505 11:46:13.166 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 11:46:13.166 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 11:46:13.166 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 11:46:13.166 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 11:46:13.166 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 11:46:13.166 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 11:46:13.166 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 11:46:13.166 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 11:46:13.166 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 11:46:13.166 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:46:13.166 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 11:46:13.166 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:46:13.166 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 11:46:13.166 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 11:46:13.166 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 11:46:13.166 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 11:46:13.166 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:46:13.166 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 11:46:13.166 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:46:13.166 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:46:13.166 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:46:13.166 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 11:46:13.166 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 11:46:13.166 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:46:13.166 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 11:46:13.166 | INFO | ------------------------------------------ | 20260505 11:46:13.166 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:46:13.166 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 11:46:13.166 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:46:13.166 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:46:13.166 | INFO | PerfReport: Full Execution -- 3.21e+01 / 7.74e+01, 1.41e+01 / 1.43e+00 | 20260505 11:46:13.166 | INFO | PerfReport: 0.29%, AggregateStats -- 9.32e-02 / 1.02e-01, 1.07e-02 / 6.54e-03 | 20260505 11:46:13.166 | INFO | PerfReport: 11.75%, Baseline -- 3.77e+00 / 4.50e+00, 6.02e-01 / 2.13e-01 | 20260505 11:46:13.166 | INFO | PerfReport: 3.31%, Histogram -- 1.06e+00 / 1.51e+00, 2.06e-01 / 8.83e-02 | 20260505 11:46:13.166 | INFO | PerfReport: 15.90%, DME -- 4.08e+01 / 4.55e+01, 4.59e+00 / 1.93e+00 | 20260505 11:46:13.166 | INFO | PerfReport: 56.33%, FrameLabeling -- 1.81e+01 / 2.09e+01, 1.54e+00 / 1.58e+00 | 20260505 11:46:13.166 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.39e+00 / 2.83e+00, 4.37e-01 / 2.22e-01 | 20260505 11:46:13.166 | INFO | PerfReport: 3.68%, Metrics -- 1.18e+00 / 1.50e+00, 3.33e-01 / 1.68e-01 | 20260505 11:46:13.166 | INFO | PerfReport: 1.29%, PulseSerializing -- 4.15e-01 / 4.73e-01, 7.50e-02 / 3.65e-02 | 20260505 11:46:13.166 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 11:46:13.166 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:46:13.166 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:46:13.166 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260505 11:46:13.166 | INFO | PerfReport: 22.57%, Upload -- 2.28e+01 / 0.00e+00, 8.15e+00 / 0.00e+00 | 20260505 11:46:13.166 | INFO | PerfReport: 22.00%, Download -- 2.23e+01 / 0.00e+00, 8.17e+00 / 0.00e+00 | 20260505 11:46:13.166 | INFO | PerfReport: 31.71%, Compute -- 3.21e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260505 11:46:13.166 | INFO | PerfReport: 2.12%, LockUpload -- 2.15e+00 / 0.00e+00, 7.82e+00 / 0.00e+00 | 20260505 11:46:13.166 | INFO | PerfReport: 1.35%, LockDownload -- 1.37e+00 / 0.00e+00, 2.90e+00 / 0.00e+00 | 20260505 11:46:13.166 | INFO | PerfReport: 20.23%, LockCompute -- 2.05e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260505 11:46:13.166 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.84e-02 / 0.00e+00 | 20260505 11:46:13.298 | INFO | Starting a round of Baz IO | 20260505 11:46:16.551 | INFO | Populating RTMetrics report | 20260505 11:46:16.551 | INFO | RTMetrics report populated | 20260505 11:46:16.551 | INFO | Opening tmp RTMetrics json file | 20260505 11:46:16.552 | INFO | Serializing RTMetrics to json | 20260505 11:46:16.564 | INFO | Serialization done, swapping tmp file now... | 20260505 11:46:16.579 | INFO | RTMetrics done | 20260505 11:46:16.677 | INFO | Finished a round of Baz IO: Wrote 2.363 GiB in 3.379 seconds | 20260505 11:46:16.678 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.97% / 0.01% / 76.03% | 20260505 11:46:16.678 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.04204% / 1.98682e-05% / 7.54992e-05% | 20260505 11:46:16.678 | INFO | Baz IO overall: | 20260505 11:46:16.678 | INFO | BytesWritten : 1751882683662 | 20260505 11:46:16.678 | INFO | NumEvents : 834078085633 | 20260505 11:46:18.304 | INFO | Analyzing chunk frames = [2957312,2957824) | 20260505 11:46:18.304 | INFO | MemInfo: { | 20260505 11:46:18.304 | INFO | "gpu" : " 17 GB", | 20260505 11:46:18.304 | INFO | "huge" : "", | 20260505 11:46:18.304 | INFO | "huge_shared" : " 27 GB", | 20260505 11:46:18.304 | INFO | "malloc" : " 14 GB", | 20260505 11:46:18.304 | INFO | "pinned" : " 29 GB" | 20260505 11:46:18.304 | INFO | } | 20260505 11:46:18.304 | INFO | Perf report for the last 5.13891 seconds: | 20260505 11:46:18.304 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:46:18.304 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:18.304 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:18.304 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:18.304 | INFO | ANALYSIS : 64.8%, 2.9, 99.8ms, 0 | 20260505 11:46:18.304 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:18.304 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 11:46:18.304 | INFO | PRE_HQ : 61.1%, 1.9, 58.8ms, 0 | 20260505 11:46:18.304 | INFO | BAZWRITER : 65.1%, 4.1, 136.3ms, 0 | 20260505 11:46:18.304 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260505 11:46:18.304 | INFO | Uploaded 36GiB to GPU in 2.19541s (16.3979GiB/s) | 20260505 11:46:18.304 | INFO | Downloaded 38GiB to GPU in 2.30252s (16.5037GiB/s) | 20260505 11:46:23.440 | INFO | Analyzing chunk frames = [2957824,2958336) | 20260505 11:46:23.440 | INFO | MemInfo: { | 20260505 11:46:23.440 | INFO | "gpu" : " 17 GB", | 20260505 11:46:23.440 | INFO | "huge" : "", | 20260505 11:46:23.440 | INFO | "huge_shared" : " 27 GB", | 20260505 11:46:23.440 | INFO | "malloc" : " 14 GB", | 20260505 11:46:23.440 | INFO | "pinned" : " 29 GB" | 20260505 11:46:23.440 | INFO | } | 20260505 11:46:23.440 | INFO | Perf report for the last 5.13585 seconds: | 20260505 11:46:23.440 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:46:23.440 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:23.440 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:23.440 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:23.440 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 11:46:23.440 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:23.440 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:23.440 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 11:46:23.440 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:23.440 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:23.440 | INFO | Uploaded 36GiB to GPU in 2.19152s (16.4269GiB/s) | 20260505 11:46:23.440 | INFO | Downloaded 34GiB to GPU in 2.06796s (16.4413GiB/s) | 20260505 11:46:28.580 | INFO | Analyzing chunk frames = [2958336,2958848) | 20260505 11:46:28.580 | INFO | MemInfo: { | 20260505 11:46:28.580 | INFO | "gpu" : " 17 GB", | 20260505 11:46:28.580 | INFO | "huge" : "", | 20260505 11:46:28.580 | INFO | "huge_shared" : " 27 GB", | 20260505 11:46:28.580 | INFO | "malloc" : " 14 GB", | 20260505 11:46:28.580 | INFO | "pinned" : " 29 GB" | 20260505 11:46:28.580 | INFO | } | 20260505 11:46:28.580 | INFO | Perf report for the last 5.14037 seconds: | 20260505 11:46:28.580 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:46:28.580 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:28.580 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:28.580 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:28.580 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 11:46:28.580 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:28.580 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:28.580 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 11:46:28.580 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:28.580 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:28.580 | INFO | Uploaded 36GiB to GPU in 2.22211s (16.2009GiB/s) | 20260505 11:46:28.580 | INFO | Downloaded 34GiB to GPU in 2.13336s (15.9373GiB/s) | 20260505 11:46:33.726 | INFO | Analyzing chunk frames = [2958848,2959360) | 20260505 11:46:33.726 | INFO | MemInfo: { | 20260505 11:46:33.726 | INFO | "gpu" : " 17 GB", | 20260505 11:46:33.726 | INFO | "huge" : "", | 20260505 11:46:33.726 | INFO | "huge_shared" : " 27 GB", | 20260505 11:46:33.726 | INFO | "malloc" : " 14 GB", | 20260505 11:46:33.726 | INFO | "pinned" : " 29 GB" | 20260505 11:46:33.726 | INFO | } | 20260505 11:46:33.726 | INFO | Perf report for the last 5.14602 seconds: | 20260505 11:46:33.726 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:46:33.726 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:33.726 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:33.726 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:33.726 | INFO | ANALYSIS : 63.1%, 3.0, 98.2ms, 0 | 20260505 11:46:33.726 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:33.726 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:33.726 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 11:46:33.726 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:33.726 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:33.726 | INFO | Uploaded 36GiB to GPU in 2.20849s (16.3007GiB/s) | 20260505 11:46:33.726 | INFO | Downloaded 34GiB to GPU in 2.13379s (15.9341GiB/s) | 20260505 11:46:38.868 | INFO | Analyzing chunk frames = [2959360,2959872) | 20260505 11:46:38.868 | INFO | MemInfo: { | 20260505 11:46:38.868 | INFO | "gpu" : " 17 GB", | 20260505 11:46:38.868 | INFO | "huge" : "", | 20260505 11:46:38.868 | INFO | "huge_shared" : " 27 GB", | 20260505 11:46:38.868 | INFO | "malloc" : " 14 GB", | 20260505 11:46:38.868 | INFO | "pinned" : " 29 GB" | 20260505 11:46:38.868 | INFO | } | 20260505 11:46:38.868 | INFO | Perf report for the last 5.14178 seconds: | 20260505 11:46:38.868 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:46:38.868 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:38.868 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:38.868 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:38.868 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 11:46:38.868 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:38.868 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:38.868 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 11:46:38.868 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:38.868 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:38.868 | INFO | Uploaded 36GiB to GPU in 2.17567s (16.5466GiB/s) | 20260505 11:46:38.868 | INFO | Downloaded 34GiB to GPU in 2.126s (15.9925GiB/s) | 20260505 11:46:44.002 | INFO | Analyzing chunk frames = [2959872,2960384) | 20260505 11:46:44.002 | INFO | MemInfo: { | 20260505 11:46:44.002 | INFO | "gpu" : " 17 GB", | 20260505 11:46:44.002 | INFO | "huge" : "", | 20260505 11:46:44.002 | INFO | "huge_shared" : " 27 GB", | 20260505 11:46:44.002 | INFO | "malloc" : " 14 GB", | 20260505 11:46:44.002 | INFO | "pinned" : " 29 GB" | 20260505 11:46:44.002 | INFO | } | 20260505 11:46:44.002 | INFO | Perf report for the last 5.13371 seconds: | 20260505 11:46:44.002 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:46:44.002 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:44.002 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:44.002 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:44.002 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 11:46:44.002 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:44.002 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:44.002 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 11:46:44.002 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:44.002 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:44.002 | INFO | Uploaded 36GiB to GPU in 2.162s (16.6512GiB/s) | 20260505 11:46:44.002 | INFO | Downloaded 34GiB to GPU in 2.111s (16.1061GiB/s) | 20260505 11:46:49.136 | INFO | Analyzing chunk frames = [2960384,2960896) | 20260505 11:46:49.136 | INFO | MemInfo: { | 20260505 11:46:49.136 | INFO | "gpu" : " 17 GB", | 20260505 11:46:49.136 | INFO | "huge" : "", | 20260505 11:46:49.136 | INFO | "huge_shared" : " 27 GB", | 20260505 11:46:49.136 | INFO | "malloc" : " 14 GB", | 20260505 11:46:49.136 | INFO | "pinned" : " 29 GB" | 20260505 11:46:49.136 | INFO | } | 20260505 11:46:49.136 | INFO | Perf report for the last 5.1341 seconds: | 20260505 11:46:49.136 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:46:49.136 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:49.136 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:49.136 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:49.136 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 11:46:49.136 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:49.136 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:49.136 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 11:46:49.136 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:49.136 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:49.136 | INFO | Uploaded 36GiB to GPU in 2.17591s (16.5448GiB/s) | 20260505 11:46:49.136 | INFO | Downloaded 34GiB to GPU in 2.16184s (15.7274GiB/s) | 20260505 11:46:54.286 | INFO | Analyzing chunk frames = [2960896,2961408) | 20260505 11:46:54.286 | INFO | MemInfo: { | 20260505 11:46:54.286 | INFO | "gpu" : " 17 GB", | 20260505 11:46:54.286 | INFO | "huge" : "", | 20260505 11:46:54.286 | INFO | "huge_shared" : " 27 GB", | 20260505 11:46:54.286 | INFO | "malloc" : " 14 GB", | 20260505 11:46:54.286 | INFO | "pinned" : " 29 GB" | 20260505 11:46:54.286 | INFO | } | 20260505 11:46:54.286 | INFO | Perf report for the last 5.15004 seconds: | 20260505 11:46:54.286 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:46:54.286 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:54.286 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:54.286 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:54.286 | INFO | ANALYSIS : 63.7%, 3.0, 99.7ms, 0 | 20260505 11:46:54.286 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:54.286 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:54.286 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 11:46:54.286 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:54.286 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:54.286 | INFO | Uploaded 36GiB to GPU in 2.2252s (16.1783GiB/s) | 20260505 11:46:54.286 | INFO | Downloaded 34GiB to GPU in 2.09114s (16.2591GiB/s) | 20260505 11:46:54.417 | INFO | Starting a round of Baz IO | 20260505 11:46:57.663 | INFO | Populating RTMetrics report | 20260505 11:46:57.664 | INFO | RTMetrics report populated | 20260505 11:46:57.664 | INFO | Opening tmp RTMetrics json file | 20260505 11:46:57.665 | INFO | Serializing RTMetrics to json | 20260505 11:46:57.667 | INFO | Serialization done, swapping tmp file now... | 20260505 11:46:57.667 | INFO | RTMetrics done | 20260505 11:46:57.786 | INFO | Finished a round of Baz IO: Wrote 2.36 GiB in 3.368 seconds | 20260505 11:46:57.786 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.96% / 0.01% / 76.03% | 20260505 11:46:57.786 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.0385% / 1.58946e-05% / 8.34465e-05% | 20260505 11:46:57.786 | INFO | Baz IO overall: | 20260505 11:46:57.786 | INFO | BytesWritten : 1754416880542 | 20260505 11:46:57.786 | INFO | NumEvents : 835281701911 | 20260505 11:46:59.422 | INFO | Analyzing chunk frames = [2961408,2961920) | 20260505 11:46:59.422 | INFO | MemInfo: { | 20260505 11:46:59.422 | INFO | "gpu" : " 17 GB", | 20260505 11:46:59.422 | INFO | "huge" : "", | 20260505 11:46:59.422 | INFO | "huge_shared" : " 27 GB", | 20260505 11:46:59.422 | INFO | "malloc" : " 14 GB", | 20260505 11:46:59.422 | INFO | "pinned" : " 29 GB" | 20260505 11:46:59.422 | INFO | } | 20260505 11:46:59.422 | INFO | Perf report for the last 5.13581 seconds: | 20260505 11:46:59.422 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:46:59.422 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:59.422 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:59.422 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:59.422 | INFO | ANALYSIS : 64.6%, 2.9, 99.8ms, 0 | 20260505 11:46:59.422 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:46:59.422 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260505 11:46:59.422 | INFO | PRE_HQ : 61.0%, 1.8, 58.4ms, 0 | 20260505 11:46:59.422 | INFO | BAZWRITER : 64.9%, 4.1, 136.7ms, 0 | 20260505 11:46:59.422 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 11:46:59.422 | INFO | Uploaded 36GiB to GPU in 2.16871s (16.5998GiB/s) | 20260505 11:46:59.422 | INFO | Downloaded 38GiB to GPU in 2.27488s (16.7042GiB/s) | 20260505 11:47:04.560 | INFO | Analyzing chunk frames = [2961920,2962432) | 20260505 11:47:04.560 | INFO | MemInfo: { | 20260505 11:47:04.560 | INFO | "gpu" : " 17 GB", | 20260505 11:47:04.560 | INFO | "huge" : "", | 20260505 11:47:04.560 | INFO | "huge_shared" : " 27 GB", | 20260505 11:47:04.560 | INFO | "malloc" : " 14 GB", | 20260505 11:47:04.560 | INFO | "pinned" : " 29 GB" | 20260505 11:47:04.560 | INFO | } | 20260505 11:47:04.560 | INFO | Perf report for the last 5.13863 seconds: | 20260505 11:47:04.560 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:47:04.560 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:04.560 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:04.560 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:04.560 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 11:47:04.560 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:04.560 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:04.560 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 11:47:04.560 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:04.560 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:04.561 | INFO | Uploaded 36GiB to GPU in 2.17729s (16.5343GiB/s) | 20260505 11:47:04.561 | INFO | Downloaded 34GiB to GPU in 2.0419s (16.6511GiB/s) | 20260505 11:47:09.697 | INFO | Analyzing chunk frames = [2962432,2962944) | 20260505 11:47:09.697 | INFO | MemInfo: { | 20260505 11:47:09.697 | INFO | "gpu" : " 17 GB", | 20260505 11:47:09.697 | INFO | "huge" : "", | 20260505 11:47:09.697 | INFO | "huge_shared" : " 27 GB", | 20260505 11:47:09.697 | INFO | "malloc" : " 14 GB", | 20260505 11:47:09.697 | INFO | "pinned" : " 29 GB" | 20260505 11:47:09.698 | INFO | } | 20260505 11:47:09.698 | INFO | Perf report for the last 5.13716 seconds: | 20260505 11:47:09.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:47:09.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:09.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:09.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:09.698 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260505 11:47:09.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:09.698 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:09.698 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 11:47:09.698 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:09.698 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:09.698 | INFO | Uploaded 36GiB to GPU in 2.23345s (16.1186GiB/s) | 20260505 11:47:09.698 | INFO | Downloaded 34GiB to GPU in 2.12591s (15.9932GiB/s) | 20260505 11:47:14.840 | INFO | Analyzing chunk frames = [2962944,2963456) | 20260505 11:47:14.840 | INFO | MemInfo: { | 20260505 11:47:14.840 | INFO | "gpu" : " 17 GB", | 20260505 11:47:14.840 | INFO | "huge" : "", | 20260505 11:47:14.840 | INFO | "huge_shared" : " 27 GB", | 20260505 11:47:14.840 | INFO | "malloc" : " 14 GB", | 20260505 11:47:14.840 | INFO | "pinned" : " 29 GB" | 20260505 11:47:14.840 | INFO | } | 20260505 11:47:14.840 | INFO | Perf report for the last 5.14207 seconds: | 20260505 11:47:14.840 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:47:14.840 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:14.840 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:14.840 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:14.840 | INFO | ANALYSIS : 63.4%, 3.0, 98.6ms, 0 | 20260505 11:47:14.840 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:14.840 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:14.840 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 11:47:14.840 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:14.840 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:14.840 | INFO | Uploaded 36GiB to GPU in 2.22043s (16.2131GiB/s) | 20260505 11:47:14.840 | INFO | Downloaded 34GiB to GPU in 2.13533s (15.9226GiB/s) | 20260505 11:47:19.977 | INFO | Analyzing chunk frames = [2963456,2963968) | 20260505 11:47:19.977 | INFO | MemInfo: { | 20260505 11:47:19.977 | INFO | "gpu" : " 17 GB", | 20260505 11:47:19.977 | INFO | "huge" : "", | 20260505 11:47:19.977 | INFO | "huge_shared" : " 27 GB", | 20260505 11:47:19.977 | INFO | "malloc" : " 14 GB", | 20260505 11:47:19.977 | INFO | "pinned" : " 29 GB" | 20260505 11:47:19.977 | INFO | } | 20260505 11:47:19.977 | INFO | Perf report for the last 5.1373 seconds: | 20260505 11:47:19.977 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:47:19.977 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:19.977 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:19.977 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:19.977 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 11:47:19.977 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:19.977 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:19.977 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 11:47:19.977 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:19.977 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:19.977 | INFO | Uploaded 36GiB to GPU in 2.19335s (16.4133GiB/s) | 20260505 11:47:19.977 | INFO | Downloaded 34GiB to GPU in 2.12702s (15.9848GiB/s) | 20260505 11:47:25.113 | INFO | Analyzing chunk frames = [2963968,2964480) | 20260505 11:47:25.113 | INFO | MemInfo: { | 20260505 11:47:25.113 | INFO | "gpu" : " 17 GB", | 20260505 11:47:25.113 | INFO | "huge" : "", | 20260505 11:47:25.113 | INFO | "huge_shared" : " 27 GB", | 20260505 11:47:25.114 | INFO | "malloc" : " 14 GB", | 20260505 11:47:25.114 | INFO | "pinned" : " 29 GB" | 20260505 11:47:25.114 | INFO | } | 20260505 11:47:25.114 | INFO | Perf report for the last 5.13668 seconds: | 20260505 11:47:25.114 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:47:25.114 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:25.114 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:25.114 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:25.114 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 11:47:25.114 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:25.114 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:25.114 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 11:47:25.114 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:25.114 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:25.114 | INFO | Uploaded 36GiB to GPU in 2.15006s (16.7437GiB/s) | 20260505 11:47:25.114 | INFO | Downloaded 34GiB to GPU in 2.10867s (16.1239GiB/s) | 20260505 11:47:30.253 | INFO | Analyzing chunk frames = [2964480,2964992) | 20260505 11:47:30.253 | INFO | MemInfo: { | 20260505 11:47:30.253 | INFO | "gpu" : " 17 GB", | 20260505 11:47:30.253 | INFO | "huge" : "", | 20260505 11:47:30.253 | INFO | "huge_shared" : " 27 GB", | 20260505 11:47:30.253 | INFO | "malloc" : " 14 GB", | 20260505 11:47:30.253 | INFO | "pinned" : " 29 GB" | 20260505 11:47:30.253 | INFO | } | 20260505 11:47:30.253 | INFO | Perf report for the last 5.13935 seconds: | 20260505 11:47:30.253 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:47:30.253 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:30.253 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:30.253 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:30.253 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 11:47:30.253 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:30.253 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:30.253 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 11:47:30.253 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:30.253 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:30.253 | INFO | Uploaded 36GiB to GPU in 2.17402s (16.5592GiB/s) | 20260505 11:47:30.253 | INFO | Downloaded 34GiB to GPU in 2.16014s (15.7397GiB/s) | 20260505 11:47:35.390 | INFO | Analyzing chunk frames = [2964992,2965504) | 20260505 11:47:35.390 | INFO | MemInfo: { | 20260505 11:47:35.390 | INFO | "gpu" : " 17 GB", | 20260505 11:47:35.390 | INFO | "huge" : "", | 20260505 11:47:35.390 | INFO | "huge_shared" : " 27 GB", | 20260505 11:47:35.390 | INFO | "malloc" : " 14 GB", | 20260505 11:47:35.390 | INFO | "pinned" : " 29 GB" | 20260505 11:47:35.390 | INFO | } | 20260505 11:47:35.390 | INFO | Perf report for the last 5.13709 seconds: | 20260505 11:47:35.390 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:47:35.390 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:35.390 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:35.390 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:35.390 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 11:47:35.390 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:35.390 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:35.390 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 11:47:35.390 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:35.390 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:35.390 | INFO | Uploaded 36GiB to GPU in 2.24509s (16.035GiB/s) | 20260505 11:47:35.390 | INFO | Downloaded 34GiB to GPU in 2.10301s (16.1673GiB/s) | 20260505 11:47:35.390 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 11:47:35.390 | INFO | Total mem usage: 15.3 GiB | 20260505 11:47:35.390 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 11:47:35.390 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -7.37%) | 20260505 11:47:35.390 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 11:47:35.390 | INFO | High Water Breakdown: | 20260505 11:47:35.390 | INFO | 60.9% : 8.7 GiB ( 91.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 11:47:35.390 | INFO | 38.9% : 5.56 GiB ( 83.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 11:47:35.390 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 11:47:35.390 | INFO | ------------------------------------------ | 20260505 11:47:35.391 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:47:35.391 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 11:47:35.391 | INFO | Total mem usage: 26.5 GiB | 20260505 11:47:35.391 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 11:47:35.391 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 11:47:35.391 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 11:47:35.391 | INFO | High Water Breakdown: | 20260505 11:47:35.391 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:47:35.391 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:47:35.391 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:47:35.391 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:47:35.391 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:47:35.391 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:47:35.391 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 11:47:35.391 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 11:47:35.391 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:47:35.391 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 11:47:35.391 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 11:47:35.391 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 11:47:35.391 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 11:47:35.391 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 11:47:35.391 | INFO | ------------------------------------------ | 20260505 11:47:35.391 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:47:35.391 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 11:47:35.391 | INFO | Total mem usage: 25.6 GiB | 20260505 11:47:35.391 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 11:47:35.391 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 11:47:35.391 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 11:47:35.391 | INFO | High Water Breakdown: | 20260505 11:47:35.391 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 11:47:35.391 | INFO | ------------------------------------------ | 20260505 11:47:35.391 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:47:35.391 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 11:47:35.391 | INFO | Total mem usage: 15.7 GiB | 20260505 11:47:35.391 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 11:47:35.391 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 11:47:35.391 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 11:47:35.391 | INFO | High Water Breakdown: | 20260505 11:47:35.391 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 11:47:35.391 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 11:47:35.391 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 11:47:35.391 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 11:47:35.391 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 11:47:35.391 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 11:47:35.391 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 11:47:35.391 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 11:47:35.391 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 11:47:35.391 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:47:35.391 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 11:47:35.391 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:47:35.391 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 11:47:35.391 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 11:47:35.391 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 11:47:35.391 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 11:47:35.391 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:47:35.391 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 11:47:35.391 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:47:35.391 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:47:35.391 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:47:35.391 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 11:47:35.392 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 11:47:35.392 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:47:35.392 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 11:47:35.392 | INFO | ------------------------------------------ | 20260505 11:47:35.392 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:47:35.392 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 11:47:35.392 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:47:35.392 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:47:35.392 | INFO | PerfReport: Full Execution -- 3.21e+01 / 7.71e+01, 1.38e+01 / 1.47e+00 | 20260505 11:47:35.392 | INFO | PerfReport: 0.29%, AggregateStats -- 9.30e-02 / 1.02e-01, 1.06e-02 / 4.41e-03 | 20260505 11:47:35.392 | INFO | PerfReport: 11.73%, Baseline -- 3.77e+00 / 4.50e+00, 6.06e-01 / 2.10e-01 | 20260505 11:47:35.392 | INFO | PerfReport: 3.31%, Histogram -- 1.06e+00 / 1.51e+00, 2.08e-01 / 8.83e-02 | 20260505 11:47:35.392 | INFO | PerfReport: 15.79%, DME -- 4.06e+01 / 4.48e+01, 4.22e+00 / 1.81e+00 | 20260505 11:47:35.392 | INFO | PerfReport: 56.53%, FrameLabeling -- 1.82e+01 / 2.14e+01, 1.65e+00 / 2.35e+00 | 20260505 11:47:35.392 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.37e+00 / 2.86e+00, 4.28e-01 / 2.07e-01 | 20260505 11:47:35.392 | INFO | PerfReport: 3.68%, Metrics -- 1.18e+00 / 1.47e+00, 3.38e-01 / 2.21e-01 | 20260505 11:47:35.392 | INFO | PerfReport: 1.29%, PulseSerializing -- 4.15e-01 / 4.63e-01, 7.62e-02 / 4.13e-02 | 20260505 11:47:35.392 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 11:47:35.392 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:47:35.392 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:47:35.392 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260505 11:47:35.392 | INFO | PerfReport: 22.60%, Upload -- 2.29e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260505 11:47:35.392 | INFO | PerfReport: 22.01%, Download -- 2.23e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260505 11:47:35.392 | INFO | PerfReport: 31.77%, Compute -- 3.21e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 11:47:35.392 | INFO | PerfReport: 2.08%, LockUpload -- 2.11e+00 / 0.00e+00, 7.64e+00 / 0.00e+00 | 20260505 11:47:35.392 | INFO | PerfReport: 1.25%, LockDownload -- 1.27e+00 / 0.00e+00, 2.81e+00 / 0.00e+00 | 20260505 11:47:35.392 | INFO | PerfReport: 20.27%, LockCompute -- 2.05e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260505 11:47:35.392 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.03e-02 / 0.00e+00 | 20260505 11:47:35.527 | INFO | Starting a round of Baz IO | 20260505 11:47:38.785 | INFO | Populating RTMetrics report | 20260505 11:47:38.785 | INFO | RTMetrics report populated | 20260505 11:47:38.785 | INFO | Opening tmp RTMetrics json file | 20260505 11:47:38.791 | INFO | Serializing RTMetrics to json | 20260505 11:47:38.795 | INFO | Serialization done, swapping tmp file now... | 20260505 11:47:38.795 | INFO | RTMetrics done | 20260505 11:47:38.922 | INFO | Finished a round of Baz IO: Wrote 2.362 GiB in 3.395 seconds | 20260505 11:47:38.922 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.95% / 0.01% / 76.04% | 20260505 11:47:38.922 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.04074% / 1.98682e-05% / 7.94729e-05% | 20260505 11:47:38.922 | INFO | Baz IO overall: | 20260505 11:47:38.922 | INFO | BytesWritten : 1756953182283 | 20260505 11:47:38.922 | INFO | NumEvents : 836487034463 | 20260505 11:47:40.535 | INFO | Analyzing chunk frames = [2965504,2966016) | 20260505 11:47:40.535 | INFO | MemInfo: { | 20260505 11:47:40.535 | INFO | "gpu" : " 17 GB", | 20260505 11:47:40.535 | INFO | "huge" : "", | 20260505 11:47:40.535 | INFO | "huge_shared" : " 27 GB", | 20260505 11:47:40.535 | INFO | "malloc" : " 14 GB", | 20260505 11:47:40.535 | INFO | "pinned" : " 29 GB" | 20260505 11:47:40.535 | INFO | } | 20260505 11:47:40.535 | INFO | Perf report for the last 5.1448 seconds: | 20260505 11:47:40.535 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:47:40.535 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:40.535 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:40.535 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:40.535 | INFO | ANALYSIS : 64.9%, 2.9, 100.2ms, 0 | 20260505 11:47:40.535 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:40.535 | INFO | RT_METRICS : 45.5%, 1.0, 23.9ms, 0 | 20260505 11:47:40.535 | INFO | PRE_HQ : 61.2%, 1.8, 58.4ms, 0 | 20260505 11:47:40.535 | INFO | BAZWRITER : 65.3%, 4.1, 137.7ms, 0 | 20260505 11:47:40.535 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 11:47:40.535 | INFO | Uploaded 36GiB to GPU in 2.21722s (16.2366GiB/s) | 20260505 11:47:40.535 | INFO | Downloaded 38GiB to GPU in 2.30358s (16.496GiB/s) | 20260505 11:47:45.676 | INFO | Analyzing chunk frames = [2966016,2966528) | 20260505 11:47:45.676 | INFO | MemInfo: { | 20260505 11:47:45.676 | INFO | "gpu" : " 17 GB", | 20260505 11:47:45.676 | INFO | "huge" : "", | 20260505 11:47:45.676 | INFO | "huge_shared" : " 27 GB", | 20260505 11:47:45.676 | INFO | "malloc" : " 14 GB", | 20260505 11:47:45.676 | INFO | "pinned" : " 29 GB" | 20260505 11:47:45.676 | INFO | } | 20260505 11:47:45.676 | INFO | Perf report for the last 5.14142 seconds: | 20260505 11:47:45.676 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:47:45.676 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:45.676 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:45.676 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:45.676 | INFO | ANALYSIS : 63.7%, 3.0, 99.6ms, 0 | 20260505 11:47:45.676 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:45.676 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:45.676 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 11:47:45.676 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:45.676 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:45.677 | INFO | Uploaded 36GiB to GPU in 2.17478s (16.5534GiB/s) | 20260505 11:47:45.677 | INFO | Downloaded 34GiB to GPU in 2.0422s (16.6487GiB/s) | 20260505 11:47:50.815 | INFO | Analyzing chunk frames = [2966528,2967040) | 20260505 11:47:50.815 | INFO | MemInfo: { | 20260505 11:47:50.815 | INFO | "gpu" : " 17 GB", | 20260505 11:47:50.815 | INFO | "huge" : "", | 20260505 11:47:50.815 | INFO | "huge_shared" : " 27 GB", | 20260505 11:47:50.815 | INFO | "malloc" : " 14 GB", | 20260505 11:47:50.815 | INFO | "pinned" : " 29 GB" | 20260505 11:47:50.815 | INFO | } | 20260505 11:47:50.815 | INFO | Perf report for the last 5.13841 seconds: | 20260505 11:47:50.815 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:47:50.815 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:50.815 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:50.815 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:50.815 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 11:47:50.815 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:50.815 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:50.815 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 11:47:50.815 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:50.815 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:50.815 | INFO | Uploaded 36GiB to GPU in 2.1624s (16.6482GiB/s) | 20260505 11:47:50.815 | INFO | Downloaded 34GiB to GPU in 2.11788s (16.0538GiB/s) | 20260505 11:47:55.946 | INFO | Analyzing chunk frames = [2967040,2967552) | 20260505 11:47:55.946 | INFO | MemInfo: { | 20260505 11:47:55.946 | INFO | "gpu" : " 17 GB", | 20260505 11:47:55.946 | INFO | "huge" : "", | 20260505 11:47:55.946 | INFO | "huge_shared" : " 27 GB", | 20260505 11:47:55.946 | INFO | "malloc" : " 14 GB", | 20260505 11:47:55.946 | INFO | "pinned" : " 29 GB" | 20260505 11:47:55.946 | INFO | } | 20260505 11:47:55.946 | INFO | Perf report for the last 5.13098 seconds: | 20260505 11:47:55.946 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:47:55.946 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:55.946 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:55.946 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:55.946 | INFO | ANALYSIS : 63.6%, 3.0, 98.6ms, 0 | 20260505 11:47:55.946 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:55.946 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:55.946 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 11:47:55.946 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:55.946 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:47:55.946 | INFO | Uploaded 36GiB to GPU in 2.21679s (16.2397GiB/s) | 20260505 11:47:55.946 | INFO | Downloaded 34GiB to GPU in 2.13151s (15.9512GiB/s) | 20260505 11:48:01.089 | INFO | Analyzing chunk frames = [2967552,2968064) | 20260505 11:48:01.089 | INFO | MemInfo: { | 20260505 11:48:01.089 | INFO | "gpu" : " 17 GB", | 20260505 11:48:01.089 | INFO | "huge" : "", | 20260505 11:48:01.089 | INFO | "huge_shared" : " 27 GB", | 20260505 11:48:01.089 | INFO | "malloc" : " 14 GB", | 20260505 11:48:01.089 | INFO | "pinned" : " 29 GB" | 20260505 11:48:01.089 | INFO | } | 20260505 11:48:01.089 | INFO | Perf report for the last 5.14364 seconds: | 20260505 11:48:01.089 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:48:01.089 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:01.089 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:01.089 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:01.089 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 11:48:01.089 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:01.090 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:01.090 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 11:48:01.090 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:01.090 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:01.090 | INFO | Uploaded 36GiB to GPU in 2.16906s (16.5971GiB/s) | 20260505 11:48:01.090 | INFO | Downloaded 34GiB to GPU in 2.12072s (16.0323GiB/s) | 20260505 11:48:06.237 | INFO | Analyzing chunk frames = [2968064,2968576) | 20260505 11:48:06.237 | INFO | MemInfo: { | 20260505 11:48:06.237 | INFO | "gpu" : " 17 GB", | 20260505 11:48:06.237 | INFO | "huge" : "", | 20260505 11:48:06.237 | INFO | "huge_shared" : " 27 GB", | 20260505 11:48:06.237 | INFO | "malloc" : " 14 GB", | 20260505 11:48:06.237 | INFO | "pinned" : " 29 GB" | 20260505 11:48:06.237 | INFO | } | 20260505 11:48:06.237 | INFO | Perf report for the last 5.14794 seconds: | 20260505 11:48:06.237 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:48:06.237 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:06.237 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:06.237 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:06.237 | INFO | ANALYSIS : 63.2%, 3.0, 98.9ms, 0 | 20260505 11:48:06.237 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:06.237 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:06.237 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 11:48:06.237 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:06.237 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:06.238 | INFO | Uploaded 36GiB to GPU in 2.1721s (16.5738GiB/s) | 20260505 11:48:06.238 | INFO | Downloaded 34GiB to GPU in 2.11546s (16.0721GiB/s) | 20260505 11:48:11.378 | INFO | Analyzing chunk frames = [2968576,2969088) | 20260505 11:48:11.378 | INFO | MemInfo: { | 20260505 11:48:11.378 | INFO | "gpu" : " 17 GB", | 20260505 11:48:11.378 | INFO | "huge" : "", | 20260505 11:48:11.378 | INFO | "huge_shared" : " 27 GB", | 20260505 11:48:11.378 | INFO | "malloc" : " 14 GB", | 20260505 11:48:11.378 | INFO | "pinned" : " 29 GB" | 20260505 11:48:11.378 | INFO | } | 20260505 11:48:11.378 | INFO | Perf report for the last 5.14036 seconds: | 20260505 11:48:11.378 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:48:11.378 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:11.378 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:11.378 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:11.378 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 11:48:11.378 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:11.378 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:11.378 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 11:48:11.378 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:11.378 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:11.378 | INFO | Uploaded 36GiB to GPU in 2.1511s (16.7356GiB/s) | 20260505 11:48:11.378 | INFO | Downloaded 34GiB to GPU in 2.17321s (15.645GiB/s) | 20260505 11:48:16.506 | INFO | Analyzing chunk frames = [2969088,2969600) | 20260505 11:48:16.506 | INFO | MemInfo: { | 20260505 11:48:16.506 | INFO | "gpu" : " 17 GB", | 20260505 11:48:16.506 | INFO | "huge" : "", | 20260505 11:48:16.507 | INFO | "huge_shared" : " 27 GB", | 20260505 11:48:16.507 | INFO | "malloc" : " 14 GB", | 20260505 11:48:16.507 | INFO | "pinned" : " 29 GB" | 20260505 11:48:16.507 | INFO | } | 20260505 11:48:16.507 | INFO | Perf report for the last 5.12885 seconds: | 20260505 11:48:16.507 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:48:16.507 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:16.507 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:16.507 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:16.507 | INFO | ANALYSIS : 63.9%, 3.0, 99.6ms, 0 | 20260505 11:48:16.507 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:16.507 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:16.507 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 11:48:16.507 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:16.507 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:16.507 | INFO | Uploaded 36GiB to GPU in 2.13496s (16.8622GiB/s) | 20260505 11:48:16.507 | INFO | Downloaded 34GiB to GPU in 2.10752s (16.1327GiB/s) | 20260505 11:48:16.645 | INFO | Starting a round of Baz IO | 20260505 11:48:19.899 | INFO | Populating RTMetrics report | 20260505 11:48:19.899 | INFO | RTMetrics report populated | 20260505 11:48:19.899 | INFO | Opening tmp RTMetrics json file | 20260505 11:48:19.906 | INFO | Serializing RTMetrics to json | 20260505 11:48:19.909 | INFO | Serialization done, swapping tmp file now... | 20260505 11:48:19.909 | INFO | RTMetrics done | 20260505 11:48:20.045 | INFO | Finished a round of Baz IO: Wrote 2.361 GiB in 3.4 seconds | 20260505 11:48:20.046 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.95% / 0.01% / 76.05% | 20260505 11:48:20.046 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.036% / 1.98682e-05% / 5.16574e-05% | 20260505 11:48:20.046 | INFO | Baz IO overall: | 20260505 11:48:20.046 | INFO | BytesWritten : 1759488802082 | 20260505 11:48:20.046 | INFO | NumEvents : 837691819724 | 20260505 11:48:21.643 | INFO | Analyzing chunk frames = [2969600,2970112) | 20260505 11:48:21.643 | INFO | MemInfo: { | 20260505 11:48:21.643 | INFO | "gpu" : " 17 GB", | 20260505 11:48:21.644 | INFO | "huge" : "", | 20260505 11:48:21.644 | INFO | "huge_shared" : " 27 GB", | 20260505 11:48:21.644 | INFO | "malloc" : " 14 GB", | 20260505 11:48:21.644 | INFO | "pinned" : " 29 GB" | 20260505 11:48:21.644 | INFO | } | 20260505 11:48:21.644 | INFO | Perf report for the last 5.13697 seconds: | 20260505 11:48:21.644 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:48:21.644 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:21.644 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:21.644 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:21.644 | INFO | ANALYSIS : 64.9%, 2.9, 99.9ms, 0 | 20260505 11:48:21.644 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:21.644 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 11:48:21.644 | INFO | PRE_HQ : 61.6%, 1.8, 59.1ms, 0 | 20260505 11:48:21.644 | INFO | BAZWRITER : 65.6%, 4.1, 137.0ms, 0 | 20260505 11:48:21.644 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 11:48:21.644 | INFO | Uploaded 36GiB to GPU in 2.20325s (16.3395GiB/s) | 20260505 11:48:21.644 | INFO | Downloaded 38GiB to GPU in 2.30511s (16.4851GiB/s) | 20260505 11:48:26.792 | INFO | Analyzing chunk frames = [2970112,2970624) | 20260505 11:48:26.792 | INFO | MemInfo: { | 20260505 11:48:26.792 | INFO | "gpu" : " 17 GB", | 20260505 11:48:26.792 | INFO | "huge" : "", | 20260505 11:48:26.792 | INFO | "huge_shared" : " 27 GB", | 20260505 11:48:26.792 | INFO | "malloc" : " 14 GB", | 20260505 11:48:26.792 | INFO | "pinned" : " 29 GB" | 20260505 11:48:26.792 | INFO | } | 20260505 11:48:26.792 | INFO | Perf report for the last 5.14826 seconds: | 20260505 11:48:26.792 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:48:26.792 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:26.792 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:26.792 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:26.792 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 11:48:26.792 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:26.792 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:26.792 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 11:48:26.792 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:26.792 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:26.792 | INFO | Uploaded 36GiB to GPU in 2.23211s (16.1282GiB/s) | 20260505 11:48:26.792 | INFO | Downloaded 34GiB to GPU in 2.08479s (16.3086GiB/s) | 20260505 11:48:31.927 | INFO | Analyzing chunk frames = [2970624,2971136) | 20260505 11:48:31.927 | INFO | MemInfo: { | 20260505 11:48:31.927 | INFO | "gpu" : " 17 GB", | 20260505 11:48:31.927 | INFO | "huge" : "", | 20260505 11:48:31.927 | INFO | "huge_shared" : " 27 GB", | 20260505 11:48:31.927 | INFO | "malloc" : " 14 GB", | 20260505 11:48:31.927 | INFO | "pinned" : " 29 GB" | 20260505 11:48:31.927 | INFO | } | 20260505 11:48:31.927 | INFO | Perf report for the last 5.13554 seconds: | 20260505 11:48:31.927 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:48:31.927 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:31.927 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:31.927 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:31.927 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 11:48:31.927 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:31.927 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:31.927 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:48:31.928 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:31.928 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:31.928 | INFO | Uploaded 36GiB to GPU in 2.16876s (16.5993GiB/s) | 20260505 11:48:31.928 | INFO | Downloaded 34GiB to GPU in 2.12675s (15.9869GiB/s) | 20260505 11:48:37.064 | INFO | Analyzing chunk frames = [2971136,2971648) | 20260505 11:48:37.064 | INFO | MemInfo: { | 20260505 11:48:37.064 | INFO | "gpu" : " 17 GB", | 20260505 11:48:37.065 | INFO | "huge" : "", | 20260505 11:48:37.065 | INFO | "huge_shared" : " 27 GB", | 20260505 11:48:37.065 | INFO | "malloc" : " 14 GB", | 20260505 11:48:37.065 | INFO | "pinned" : " 29 GB" | 20260505 11:48:37.065 | INFO | } | 20260505 11:48:37.065 | INFO | Perf report for the last 5.13723 seconds: | 20260505 11:48:37.065 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:48:37.065 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:37.065 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:37.065 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:37.065 | INFO | ANALYSIS : 63.4%, 3.0, 98.5ms, 0 | 20260505 11:48:37.065 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:37.065 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:37.065 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:48:37.065 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:37.065 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:37.065 | INFO | Uploaded 36GiB to GPU in 2.21632s (16.2431GiB/s) | 20260505 11:48:37.065 | INFO | Downloaded 34GiB to GPU in 2.13145s (15.9516GiB/s) | 20260505 11:48:42.205 | INFO | Analyzing chunk frames = [2971648,2972160) | 20260505 11:48:42.205 | INFO | MemInfo: { | 20260505 11:48:42.205 | INFO | "gpu" : " 17 GB", | 20260505 11:48:42.205 | INFO | "huge" : "", | 20260505 11:48:42.206 | INFO | "huge_shared" : " 27 GB", | 20260505 11:48:42.206 | INFO | "malloc" : " 14 GB", | 20260505 11:48:42.206 | INFO | "pinned" : " 29 GB" | 20260505 11:48:42.206 | INFO | } | 20260505 11:48:42.206 | INFO | Perf report for the last 5.14088 seconds: | 20260505 11:48:42.206 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:48:42.206 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:42.206 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:42.206 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:42.206 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 11:48:42.206 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:42.206 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:42.206 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 11:48:42.206 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:42.206 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:42.206 | INFO | Uploaded 36GiB to GPU in 2.16224s (16.6494GiB/s) | 20260505 11:48:42.206 | INFO | Downloaded 34GiB to GPU in 2.12648s (15.9889GiB/s) | 20260505 11:48:47.346 | INFO | Analyzing chunk frames = [2972160,2972672) | 20260505 11:48:47.346 | INFO | MemInfo: { | 20260505 11:48:47.346 | INFO | "gpu" : " 17 GB", | 20260505 11:48:47.346 | INFO | "huge" : "", | 20260505 11:48:47.346 | INFO | "huge_shared" : " 27 GB", | 20260505 11:48:47.346 | INFO | "malloc" : " 14 GB", | 20260505 11:48:47.346 | INFO | "pinned" : " 29 GB" | 20260505 11:48:47.346 | INFO | } | 20260505 11:48:47.346 | INFO | Perf report for the last 5.1407 seconds: | 20260505 11:48:47.346 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:48:47.346 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:47.346 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:47.346 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:47.346 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 11:48:47.346 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:47.346 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:47.346 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 11:48:47.346 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:47.346 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:47.346 | INFO | Uploaded 36GiB to GPU in 2.19211s (16.4225GiB/s) | 20260505 11:48:47.347 | INFO | Downloaded 34GiB to GPU in 2.11577s (16.0698GiB/s) | 20260505 11:48:52.484 | INFO | Analyzing chunk frames = [2972672,2973184) | 20260505 11:48:52.484 | INFO | MemInfo: { | 20260505 11:48:52.484 | INFO | "gpu" : " 17 GB", | 20260505 11:48:52.484 | INFO | "huge" : "", | 20260505 11:48:52.484 | INFO | "huge_shared" : " 27 GB", | 20260505 11:48:52.484 | INFO | "malloc" : " 14 GB", | 20260505 11:48:52.484 | INFO | "pinned" : " 29 GB" | 20260505 11:48:52.484 | INFO | } | 20260505 11:48:52.484 | INFO | Perf report for the last 5.13812 seconds: | 20260505 11:48:52.484 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:48:52.484 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:52.484 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:52.484 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:52.484 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 11:48:52.484 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:52.484 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:52.484 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 11:48:52.484 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:52.484 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:52.485 | INFO | Uploaded 36GiB to GPU in 2.1968s (16.3875GiB/s) | 20260505 11:48:52.485 | INFO | Downloaded 34GiB to GPU in 2.13375s (15.9344GiB/s) | 20260505 11:48:57.626 | INFO | Analyzing chunk frames = [2973184,2973696) | 20260505 11:48:57.626 | INFO | MemInfo: { | 20260505 11:48:57.626 | INFO | "gpu" : " 17 GB", | 20260505 11:48:57.626 | INFO | "huge" : "", | 20260505 11:48:57.626 | INFO | "huge_shared" : " 27 GB", | 20260505 11:48:57.626 | INFO | "malloc" : " 14 GB", | 20260505 11:48:57.626 | INFO | "pinned" : " 29 GB" | 20260505 11:48:57.626 | INFO | } | 20260505 11:48:57.626 | INFO | Perf report for the last 5.14168 seconds: | 20260505 11:48:57.626 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:48:57.626 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:57.626 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:57.626 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:57.626 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 11:48:57.626 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:57.626 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:57.626 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 11:48:57.626 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:57.626 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:48:57.626 | INFO | Uploaded 36GiB to GPU in 2.19621s (16.3919GiB/s) | 20260505 11:48:57.626 | INFO | Downloaded 34GiB to GPU in 2.11406s (16.0828GiB/s) | 20260505 11:48:57.626 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 11:48:57.626 | INFO | Total mem usage: 15.3 GiB | 20260505 11:48:57.626 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 11:48:57.626 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.77%) | 20260505 11:48:57.626 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 11:48:57.626 | INFO | High Water Breakdown: | 20260505 11:48:57.626 | INFO | 61.1% : 8.77 GiB ( 92.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 11:48:57.626 | INFO | 38.8% : 5.57 GiB ( 83.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 11:48:57.626 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 11:48:57.626 | INFO | ------------------------------------------ | 20260505 11:48:57.626 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:48:57.627 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 11:48:57.627 | INFO | Total mem usage: 26.5 GiB | 20260505 11:48:57.627 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 11:48:57.627 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 11:48:57.627 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 11:48:57.627 | INFO | High Water Breakdown: | 20260505 11:48:57.627 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:48:57.627 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:48:57.627 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:48:57.627 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:48:57.627 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:48:57.627 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:48:57.627 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 11:48:57.627 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 11:48:57.627 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:48:57.627 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 11:48:57.627 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 11:48:57.627 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 11:48:57.627 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 11:48:57.627 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 11:48:57.627 | INFO | ------------------------------------------ | 20260505 11:48:57.627 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:48:57.627 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 11:48:57.627 | INFO | Total mem usage: 25.6 GiB | 20260505 11:48:57.627 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 11:48:57.627 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 11:48:57.627 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 11:48:57.627 | INFO | High Water Breakdown: | 20260505 11:48:57.627 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 11:48:57.627 | INFO | ------------------------------------------ | 20260505 11:48:57.627 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:48:57.627 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 11:48:57.627 | INFO | Total mem usage: 15.7 GiB | 20260505 11:48:57.627 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 11:48:57.627 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 11:48:57.627 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 11:48:57.627 | INFO | High Water Breakdown: | 20260505 11:48:57.627 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 11:48:57.627 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 11:48:57.627 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 11:48:57.627 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 11:48:57.627 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 11:48:57.627 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 11:48:57.627 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 11:48:57.627 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 11:48:57.627 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 11:48:57.627 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:48:57.627 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 11:48:57.627 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:48:57.627 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 11:48:57.627 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 11:48:57.627 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 11:48:57.627 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 11:48:57.627 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:48:57.627 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 11:48:57.627 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:48:57.627 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:48:57.627 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:48:57.627 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 11:48:57.627 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 11:48:57.627 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:48:57.627 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 11:48:57.627 | INFO | ------------------------------------------ | 20260505 11:48:57.627 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:48:57.627 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 11:48:57.627 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:48:57.627 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:48:57.627 | INFO | PerfReport: Full Execution -- 3.21e+01 / 7.78e+01, 1.41e+01 / 1.94e+00 | 20260505 11:48:57.627 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 9.98e-02, 1.02e-02 / 3.73e-03 | 20260505 11:48:57.627 | INFO | PerfReport: 11.75%, Baseline -- 3.77e+00 / 4.48e+00, 6.05e-01 / 1.69e-01 | 20260505 11:48:57.627 | INFO | PerfReport: 3.30%, Histogram -- 1.06e+00 / 1.50e+00, 2.04e-01 / 7.79e-02 | 20260505 11:48:57.627 | INFO | PerfReport: 16.10%, DME -- 4.13e+01 / 4.70e+01, 4.89e+00 / 2.30e+00 | 20260505 11:48:57.627 | INFO | PerfReport: 56.16%, FrameLabeling -- 1.80e+01 / 1.98e+01, 1.43e+00 / 1.90e+00 | 20260505 11:48:57.627 | INFO | PerfReport: 7.44%, PulseAccumulating -- 2.39e+00 / 2.95e+00, 4.51e-01 / 1.92e-01 | 20260505 11:48:57.627 | INFO | PerfReport: 3.66%, Metrics -- 1.18e+00 / 1.53e+00, 3.36e-01 / 1.86e-01 | 20260505 11:48:57.627 | INFO | PerfReport: 1.29%, PulseSerializing -- 4.14e-01 / 4.66e-01, 7.56e-02 / 3.38e-02 | 20260505 11:48:57.627 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 11:48:57.627 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:48:57.627 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:48:57.627 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.56e+01 / 0.00e+00 | 20260505 11:48:57.627 | INFO | PerfReport: 22.49%, Upload -- 2.28e+01 / 0.00e+00, 8.05e+00 / 0.00e+00 | 20260505 11:48:57.627 | INFO | PerfReport: 22.03%, Download -- 2.23e+01 / 0.00e+00, 8.14e+00 / 0.00e+00 | 20260505 11:48:57.627 | INFO | PerfReport: 31.71%, Compute -- 3.21e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260505 11:48:57.627 | INFO | PerfReport: 2.17%, LockUpload -- 2.20e+00 / 0.00e+00, 7.95e+00 / 0.00e+00 | 20260505 11:48:57.627 | INFO | PerfReport: 1.43%, LockDownload -- 1.44e+00 / 0.00e+00, 3.01e+00 / 0.00e+00 | 20260505 11:48:57.627 | INFO | PerfReport: 20.16%, LockCompute -- 2.04e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 11:48:57.627 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 2.30e-02 / 0.00e+00 | 20260505 11:48:57.761 | INFO | Starting a round of Baz IO | 20260505 11:49:01.006 | INFO | Populating RTMetrics report | 20260505 11:49:01.006 | INFO | RTMetrics report populated | 20260505 11:49:01.006 | INFO | Opening tmp RTMetrics json file | 20260505 11:49:01.012 | INFO | Serializing RTMetrics to json | 20260505 11:49:01.030 | INFO | Serialization done, swapping tmp file now... | 20260505 11:49:01.031 | INFO | RTMetrics done | 20260505 11:49:01.125 | INFO | Finished a round of Baz IO: Wrote 2.361 GiB in 3.363 seconds | 20260505 11:49:01.125 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.94% / 0.01% / 76.05% | 20260505 11:49:01.125 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.03358% / 3.97364e-06% / 6.75519e-05% | 20260505 11:49:01.125 | INFO | Baz IO overall: | 20260505 11:49:01.125 | INFO | BytesWritten : 1762023575397 | 20260505 11:49:01.125 | INFO | NumEvents : 838896075441 | 20260505 11:49:02.758 | INFO | Analyzing chunk frames = [2973696,2974208) | 20260505 11:49:02.758 | INFO | MemInfo: { | 20260505 11:49:02.758 | INFO | "gpu" : " 17 GB", | 20260505 11:49:02.758 | INFO | "huge" : "", | 20260505 11:49:02.758 | INFO | "huge_shared" : " 27 GB", | 20260505 11:49:02.758 | INFO | "malloc" : " 14 GB", | 20260505 11:49:02.758 | INFO | "pinned" : " 29 GB" | 20260505 11:49:02.758 | INFO | } | 20260505 11:49:02.758 | INFO | Perf report for the last 5.132 seconds: | 20260505 11:49:02.758 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:49:02.758 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:02.758 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:02.758 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:02.758 | INFO | ANALYSIS : 64.7%, 2.9, 99.6ms, 0 | 20260505 11:49:02.758 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:02.758 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260505 11:49:02.758 | INFO | PRE_HQ : 61.3%, 1.8, 58.7ms, 0 | 20260505 11:49:02.758 | INFO | BAZWRITER : 64.8%, 4.1, 137.0ms, 0 | 20260505 11:49:02.758 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260505 11:49:02.759 | INFO | Uploaded 36GiB to GPU in 2.198s (16.3785GiB/s) | 20260505 11:49:02.759 | INFO | Downloaded 38GiB to GPU in 2.28984s (16.5951GiB/s) | 20260505 11:49:07.897 | INFO | Analyzing chunk frames = [2974208,2974720) | 20260505 11:49:07.897 | INFO | MemInfo: { | 20260505 11:49:07.897 | INFO | "gpu" : " 17 GB", | 20260505 11:49:07.897 | INFO | "huge" : "", | 20260505 11:49:07.897 | INFO | "huge_shared" : " 27 GB", | 20260505 11:49:07.898 | INFO | "malloc" : " 14 GB", | 20260505 11:49:07.898 | INFO | "pinned" : " 29 GB" | 20260505 11:49:07.898 | INFO | } | 20260505 11:49:07.898 | INFO | Perf report for the last 5.13945 seconds: | 20260505 11:49:07.898 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:49:07.898 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:07.898 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:07.898 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:07.898 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 11:49:07.898 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:07.898 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:07.898 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 11:49:07.898 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:07.898 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:07.898 | INFO | Uploaded 36GiB to GPU in 2.20773s (16.3063GiB/s) | 20260505 11:49:07.898 | INFO | Downloaded 34GiB to GPU in 2.08251s (16.3264GiB/s) | 20260505 11:49:13.046 | INFO | Analyzing chunk frames = [2974720,2975232) | 20260505 11:49:13.046 | INFO | MemInfo: { | 20260505 11:49:13.046 | INFO | "gpu" : " 17 GB", | 20260505 11:49:13.046 | INFO | "huge" : "", | 20260505 11:49:13.046 | INFO | "huge_shared" : " 27 GB", | 20260505 11:49:13.046 | INFO | "malloc" : " 14 GB", | 20260505 11:49:13.046 | INFO | "pinned" : " 29 GB" | 20260505 11:49:13.046 | INFO | } | 20260505 11:49:13.046 | INFO | Perf report for the last 5.14849 seconds: | 20260505 11:49:13.046 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:49:13.046 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:13.046 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:13.046 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:13.046 | INFO | ANALYSIS : 63.5%, 3.0, 99.0ms, 0 | 20260505 11:49:13.046 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:13.046 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:13.046 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 11:49:13.046 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:13.046 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:13.046 | INFO | Uploaded 36GiB to GPU in 2.22243s (16.1985GiB/s) | 20260505 11:49:13.046 | INFO | Downloaded 34GiB to GPU in 2.13216s (15.9463GiB/s) | 20260505 11:49:18.191 | INFO | Analyzing chunk frames = [2975232,2975744) | 20260505 11:49:18.191 | INFO | MemInfo: { | 20260505 11:49:18.191 | INFO | "gpu" : " 17 GB", | 20260505 11:49:18.191 | INFO | "huge" : "", | 20260505 11:49:18.191 | INFO | "huge_shared" : " 27 GB", | 20260505 11:49:18.191 | INFO | "malloc" : " 14 GB", | 20260505 11:49:18.191 | INFO | "pinned" : " 29 GB" | 20260505 11:49:18.191 | INFO | } | 20260505 11:49:18.191 | INFO | Perf report for the last 5.14512 seconds: | 20260505 11:49:18.191 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:49:18.191 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:18.191 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:18.191 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:18.191 | INFO | ANALYSIS : 63.1%, 3.0, 98.1ms, 0 | 20260505 11:49:18.191 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:18.191 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:18.191 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 11:49:18.191 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:18.191 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:18.192 | INFO | Uploaded 36GiB to GPU in 2.20119s (16.3548GiB/s) | 20260505 11:49:18.192 | INFO | Downloaded 34GiB to GPU in 2.13313s (15.939GiB/s) | 20260505 11:49:23.324 | INFO | Analyzing chunk frames = [2975744,2976256) | 20260505 11:49:23.324 | INFO | MemInfo: { | 20260505 11:49:23.324 | INFO | "gpu" : " 17 GB", | 20260505 11:49:23.324 | INFO | "huge" : "", | 20260505 11:49:23.324 | INFO | "huge_shared" : " 27 GB", | 20260505 11:49:23.324 | INFO | "malloc" : " 14 GB", | 20260505 11:49:23.324 | INFO | "pinned" : " 29 GB" | 20260505 11:49:23.324 | INFO | } | 20260505 11:49:23.325 | INFO | Perf report for the last 5.13328 seconds: | 20260505 11:49:23.325 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:49:23.325 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:23.325 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:23.325 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:23.325 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260505 11:49:23.325 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:23.325 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:23.325 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:49:23.325 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:23.325 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:23.325 | INFO | Uploaded 36GiB to GPU in 2.14025s (16.8205GiB/s) | 20260505 11:49:23.325 | INFO | Downloaded 34GiB to GPU in 2.12171s (16.0248GiB/s) | 20260505 11:49:28.454 | INFO | Analyzing chunk frames = [2976256,2976768) | 20260505 11:49:28.454 | INFO | MemInfo: { | 20260505 11:49:28.454 | INFO | "gpu" : " 17 GB", | 20260505 11:49:28.454 | INFO | "huge" : "", | 20260505 11:49:28.454 | INFO | "huge_shared" : " 27 GB", | 20260505 11:49:28.454 | INFO | "malloc" : " 14 GB", | 20260505 11:49:28.454 | INFO | "pinned" : " 29 GB" | 20260505 11:49:28.454 | INFO | } | 20260505 11:49:28.454 | INFO | Perf report for the last 5.12941 seconds: | 20260505 11:49:28.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:49:28.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:28.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:28.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:28.454 | INFO | ANALYSIS : 63.5%, 3.0, 99.0ms, 0 | 20260505 11:49:28.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:28.454 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:28.454 | INFO | PRE_HQ : 28.7%, 1.0, 14.8ms, 0 | 20260505 11:49:28.454 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:28.454 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:28.454 | INFO | Uploaded 36GiB to GPU in 2.17683s (16.5378GiB/s) | 20260505 11:49:28.454 | INFO | Downloaded 34GiB to GPU in 2.11722s (16.0588GiB/s) | 20260505 11:49:33.600 | INFO | Analyzing chunk frames = [2976768,2977280) | 20260505 11:49:33.600 | INFO | MemInfo: { | 20260505 11:49:33.600 | INFO | "gpu" : " 17 GB", | 20260505 11:49:33.600 | INFO | "huge" : "", | 20260505 11:49:33.600 | INFO | "huge_shared" : " 27 GB", | 20260505 11:49:33.600 | INFO | "malloc" : " 14 GB", | 20260505 11:49:33.600 | INFO | "pinned" : " 29 GB" | 20260505 11:49:33.600 | INFO | } | 20260505 11:49:33.600 | INFO | Perf report for the last 5.1462 seconds: | 20260505 11:49:33.600 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:49:33.600 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:33.600 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:33.600 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:33.600 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 11:49:33.600 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:33.600 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:33.600 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 11:49:33.600 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:33.600 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:33.600 | INFO | Uploaded 36GiB to GPU in 2.19209s (16.4227GiB/s) | 20260505 11:49:33.600 | INFO | Downloaded 34GiB to GPU in 2.16256s (15.7221GiB/s) | 20260505 11:49:38.739 | INFO | Analyzing chunk frames = [2977280,2977792) | 20260505 11:49:38.739 | INFO | MemInfo: { | 20260505 11:49:38.739 | INFO | "gpu" : " 17 GB", | 20260505 11:49:38.739 | INFO | "huge" : "", | 20260505 11:49:38.739 | INFO | "huge_shared" : " 27 GB", | 20260505 11:49:38.739 | INFO | "malloc" : " 14 GB", | 20260505 11:49:38.739 | INFO | "pinned" : " 29 GB" | 20260505 11:49:38.739 | INFO | } | 20260505 11:49:38.739 | INFO | Perf report for the last 5.1388 seconds: | 20260505 11:49:38.739 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:49:38.739 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:38.739 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:38.739 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:38.739 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 11:49:38.739 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:38.739 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:38.739 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 11:49:38.739 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:38.739 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:38.739 | INFO | Uploaded 36GiB to GPU in 2.21375s (16.262GiB/s) | 20260505 11:49:38.739 | INFO | Downloaded 34GiB to GPU in 2.07983s (16.3475GiB/s) | 20260505 11:49:38.869 | INFO | Starting a round of Baz IO | 20260505 11:49:42.120 | INFO | Populating RTMetrics report | 20260505 11:49:42.120 | INFO | RTMetrics report populated | 20260505 11:49:42.120 | INFO | Opening tmp RTMetrics json file | 20260505 11:49:42.123 | INFO | Serializing RTMetrics to json | 20260505 11:49:42.125 | INFO | Serialization done, swapping tmp file now... | 20260505 11:49:42.125 | INFO | RTMetrics done | 20260505 11:49:42.247 | INFO | Finished a round of Baz IO: Wrote 2.359 GiB in 3.377 seconds | 20260505 11:49:42.247 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.93% / 0.01% / 76.06% | 20260505 11:49:42.247 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.02794% / 3.97364e-06% / 8.34465e-05% | 20260505 11:49:42.247 | INFO | Baz IO overall: | 20260505 11:49:42.247 | INFO | BytesWritten : 1764556213277 | 20260505 11:49:42.247 | INFO | NumEvents : 840098886540 | 20260505 11:49:43.881 | INFO | Analyzing chunk frames = [2977792,2978304) | 20260505 11:49:43.881 | INFO | MemInfo: { | 20260505 11:49:43.881 | INFO | "gpu" : " 17 GB", | 20260505 11:49:43.881 | INFO | "huge" : "", | 20260505 11:49:43.881 | INFO | "huge_shared" : " 27 GB", | 20260505 11:49:43.881 | INFO | "malloc" : " 14 GB", | 20260505 11:49:43.881 | INFO | "pinned" : " 29 GB" | 20260505 11:49:43.881 | INFO | } | 20260505 11:49:43.881 | INFO | Perf report for the last 5.14181 seconds: | 20260505 11:49:43.881 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:49:43.881 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:43.881 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:43.881 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:43.881 | INFO | ANALYSIS : 64.6%, 2.9, 99.6ms, 0 | 20260505 11:49:43.881 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:43.881 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260505 11:49:43.881 | INFO | PRE_HQ : 61.0%, 1.8, 58.5ms, 0 | 20260505 11:49:43.881 | INFO | BAZWRITER : 65.0%, 4.1, 136.2ms, 0 | 20260505 11:49:43.881 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 11:49:43.881 | INFO | Uploaded 36GiB to GPU in 2.17458s (16.5549GiB/s) | 20260505 11:49:43.881 | INFO | Downloaded 38GiB to GPU in 2.27819s (16.6799GiB/s) | 20260505 11:49:49.019 | INFO | Analyzing chunk frames = [2978304,2978816) | 20260505 11:49:49.019 | INFO | MemInfo: { | 20260505 11:49:49.019 | INFO | "gpu" : " 17 GB", | 20260505 11:49:49.019 | INFO | "huge" : "", | 20260505 11:49:49.019 | INFO | "huge_shared" : " 27 GB", | 20260505 11:49:49.019 | INFO | "malloc" : " 14 GB", | 20260505 11:49:49.019 | INFO | "pinned" : " 29 GB" | 20260505 11:49:49.019 | INFO | } | 20260505 11:49:49.019 | INFO | Perf report for the last 5.13812 seconds: | 20260505 11:49:49.019 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:49:49.019 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:49.019 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:49.019 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:49.019 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 11:49:49.019 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:49.019 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:49.019 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 11:49:49.019 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:49.019 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:49.019 | INFO | Uploaded 36GiB to GPU in 2.23149s (16.1327GiB/s) | 20260505 11:49:49.019 | INFO | Downloaded 34GiB to GPU in 2.09997s (16.1907GiB/s) | 20260505 11:49:54.156 | INFO | Analyzing chunk frames = [2978816,2979328) | 20260505 11:49:54.156 | INFO | MemInfo: { | 20260505 11:49:54.156 | INFO | "gpu" : " 17 GB", | 20260505 11:49:54.156 | INFO | "huge" : "", | 20260505 11:49:54.156 | INFO | "huge_shared" : " 27 GB", | 20260505 11:49:54.156 | INFO | "malloc" : " 14 GB", | 20260505 11:49:54.156 | INFO | "pinned" : " 29 GB" | 20260505 11:49:54.156 | INFO | } | 20260505 11:49:54.156 | INFO | Perf report for the last 5.13727 seconds: | 20260505 11:49:54.156 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:49:54.156 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:54.156 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:54.156 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:54.156 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 11:49:54.156 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:54.156 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:54.156 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:49:54.156 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:54.156 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:54.156 | INFO | Uploaded 36GiB to GPU in 2.20625s (16.3172GiB/s) | 20260505 11:49:54.156 | INFO | Downloaded 34GiB to GPU in 2.11402s (16.0831GiB/s) | 20260505 11:49:59.304 | INFO | Analyzing chunk frames = [2979328,2979840) | 20260505 11:49:59.304 | INFO | MemInfo: { | 20260505 11:49:59.304 | INFO | "gpu" : " 17 GB", | 20260505 11:49:59.304 | INFO | "huge" : "", | 20260505 11:49:59.304 | INFO | "huge_shared" : " 27 GB", | 20260505 11:49:59.304 | INFO | "malloc" : " 14 GB", | 20260505 11:49:59.304 | INFO | "pinned" : " 29 GB" | 20260505 11:49:59.304 | INFO | } | 20260505 11:49:59.304 | INFO | Perf report for the last 5.14769 seconds: | 20260505 11:49:59.304 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:49:59.304 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:59.304 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:59.304 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:59.304 | INFO | ANALYSIS : 63.2%, 3.0, 98.4ms, 0 | 20260505 11:49:59.304 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:59.304 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:59.304 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 11:49:59.304 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:59.304 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:49:59.304 | INFO | Uploaded 36GiB to GPU in 2.21487s (16.2538GiB/s) | 20260505 11:49:59.304 | INFO | Downloaded 34GiB to GPU in 2.13565s (15.9202GiB/s) | 20260505 11:50:04.434 | INFO | Analyzing chunk frames = [2979840,2980352) | 20260505 11:50:04.434 | INFO | MemInfo: { | 20260505 11:50:04.434 | INFO | "gpu" : " 17 GB", | 20260505 11:50:04.434 | INFO | "huge" : "", | 20260505 11:50:04.435 | INFO | "huge_shared" : " 27 GB", | 20260505 11:50:04.435 | INFO | "malloc" : " 14 GB", | 20260505 11:50:04.435 | INFO | "pinned" : " 29 GB" | 20260505 11:50:04.435 | INFO | } | 20260505 11:50:04.435 | INFO | Perf report for the last 5.13078 seconds: | 20260505 11:50:04.435 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:50:04.435 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:04.435 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:04.435 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:04.435 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 11:50:04.435 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:04.435 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:04.435 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:50:04.435 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:04.435 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:04.435 | INFO | Uploaded 36GiB to GPU in 2.14397s (16.7913GiB/s) | 20260505 11:50:04.435 | INFO | Downloaded 34GiB to GPU in 2.12045s (16.0343GiB/s) | 20260505 11:50:09.572 | INFO | Analyzing chunk frames = [2980352,2980864) | 20260505 11:50:09.572 | INFO | MemInfo: { | 20260505 11:50:09.572 | INFO | "gpu" : " 17 GB", | 20260505 11:50:09.572 | INFO | "huge" : "", | 20260505 11:50:09.572 | INFO | "huge_shared" : " 27 GB", | 20260505 11:50:09.572 | INFO | "malloc" : " 14 GB", | 20260505 11:50:09.572 | INFO | "pinned" : " 29 GB" | 20260505 11:50:09.572 | INFO | } | 20260505 11:50:09.572 | INFO | Perf report for the last 5.1372 seconds: | 20260505 11:50:09.572 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:50:09.572 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:09.572 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:09.572 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:09.572 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 11:50:09.572 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:09.572 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:09.572 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:50:09.572 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:09.572 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:09.572 | INFO | Uploaded 36GiB to GPU in 2.18763s (16.4561GiB/s) | 20260505 11:50:09.572 | INFO | Downloaded 34GiB to GPU in 2.11815s (16.0518GiB/s) | 20260505 11:50:14.709 | INFO | Analyzing chunk frames = [2980864,2981376) | 20260505 11:50:14.709 | INFO | MemInfo: { | 20260505 11:50:14.709 | INFO | "gpu" : " 17 GB", | 20260505 11:50:14.709 | INFO | "huge" : "", | 20260505 11:50:14.709 | INFO | "huge_shared" : " 27 GB", | 20260505 11:50:14.709 | INFO | "malloc" : " 14 GB", | 20260505 11:50:14.709 | INFO | "pinned" : " 29 GB" | 20260505 11:50:14.709 | INFO | } | 20260505 11:50:14.709 | INFO | Perf report for the last 5.1371 seconds: | 20260505 11:50:14.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:50:14.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:14.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:14.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:14.709 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 11:50:14.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:14.709 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:14.709 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 11:50:14.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:14.709 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:14.709 | INFO | Uploaded 36GiB to GPU in 2.20154s (16.3522GiB/s) | 20260505 11:50:14.709 | INFO | Downloaded 34GiB to GPU in 2.15155s (15.8025GiB/s) | 20260505 11:50:19.853 | INFO | Analyzing chunk frames = [2981376,2981888) | 20260505 11:50:19.853 | INFO | MemInfo: { | 20260505 11:50:19.853 | INFO | "gpu" : " 17 GB", | 20260505 11:50:19.853 | INFO | "huge" : "", | 20260505 11:50:19.853 | INFO | "huge_shared" : " 27 GB", | 20260505 11:50:19.853 | INFO | "malloc" : " 14 GB", | 20260505 11:50:19.853 | INFO | "pinned" : " 29 GB" | 20260505 11:50:19.853 | INFO | } | 20260505 11:50:19.853 | INFO | Perf report for the last 5.14442 seconds: | 20260505 11:50:19.853 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:50:19.853 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:19.853 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:19.854 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:19.854 | INFO | ANALYSIS : 63.7%, 3.0, 99.6ms, 0 | 20260505 11:50:19.854 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:19.854 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:19.854 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 11:50:19.854 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:19.854 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:19.854 | INFO | Uploaded 36GiB to GPU in 2.21587s (16.2465GiB/s) | 20260505 11:50:19.854 | INFO | Downloaded 34GiB to GPU in 2.09572s (16.2235GiB/s) | 20260505 11:50:19.854 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 11:50:19.854 | INFO | Total mem usage: 15.3 GiB | 20260505 11:50:19.854 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 11:50:19.854 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.76%) | 20260505 11:50:19.854 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 11:50:19.854 | INFO | High Water Breakdown: | 20260505 11:50:19.854 | INFO | 61.1% : 8.77 GiB ( 92.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 11:50:19.854 | INFO | 38.8% : 5.57 GiB ( 83.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 11:50:19.854 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 11:50:19.854 | INFO | ------------------------------------------ | 20260505 11:50:19.854 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:50:19.854 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 11:50:19.854 | INFO | Total mem usage: 26.5 GiB | 20260505 11:50:19.854 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 11:50:19.854 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 11:50:19.854 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 11:50:19.854 | INFO | High Water Breakdown: | 20260505 11:50:19.854 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:50:19.854 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:50:19.854 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:50:19.854 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:50:19.854 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:50:19.854 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:50:19.854 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 11:50:19.854 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 11:50:19.854 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:50:19.854 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 11:50:19.854 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 11:50:19.854 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 11:50:19.854 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 11:50:19.854 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 11:50:19.854 | INFO | ------------------------------------------ | 20260505 11:50:19.854 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:50:19.854 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 11:50:19.854 | INFO | Total mem usage: 25.6 GiB | 20260505 11:50:19.854 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 11:50:19.855 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 11:50:19.855 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 11:50:19.855 | INFO | High Water Breakdown: | 20260505 11:50:19.855 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 11:50:19.855 | INFO | ------------------------------------------ | 20260505 11:50:19.855 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:50:19.855 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 11:50:19.855 | INFO | Total mem usage: 15.7 GiB | 20260505 11:50:19.855 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 11:50:19.855 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 11:50:19.855 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 11:50:19.855 | INFO | High Water Breakdown: | 20260505 11:50:19.855 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 11:50:19.855 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 11:50:19.855 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 11:50:19.855 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 11:50:19.855 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 11:50:19.855 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 11:50:19.855 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 11:50:19.855 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 11:50:19.855 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 11:50:19.855 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:50:19.855 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 11:50:19.855 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:50:19.855 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 11:50:19.855 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 11:50:19.855 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 11:50:19.855 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 11:50:19.855 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:50:19.855 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 11:50:19.855 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:50:19.855 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:50:19.855 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:50:19.855 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 11:50:19.855 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 11:50:19.855 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:50:19.855 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 11:50:19.855 | INFO | ------------------------------------------ | 20260505 11:50:19.855 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:50:19.855 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 11:50:19.855 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:50:19.855 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:50:19.855 | INFO | PerfReport: Full Execution -- 3.21e+01 / 7.72e+01, 1.41e+01 / 1.34e+00 | 20260505 11:50:19.855 | INFO | PerfReport: 0.29%, AggregateStats -- 9.30e-02 / 1.02e-01, 1.09e-02 / 6.44e-03 | 20260505 11:50:19.855 | INFO | PerfReport: 11.76%, Baseline -- 3.77e+00 / 4.49e+00, 5.99e-01 / 1.99e-01 | 20260505 11:50:19.855 | INFO | PerfReport: 3.30%, Histogram -- 1.06e+00 / 1.51e+00, 2.04e-01 / 9.39e-02 | 20260505 11:50:19.855 | INFO | PerfReport: 15.84%, DME -- 4.07e+01 / 4.49e+01, 4.22e+00 / 1.91e+00 | 20260505 11:50:19.855 | INFO | PerfReport: 56.43%, FrameLabeling -- 1.81e+01 / 2.13e+01, 1.56e+00 / 1.84e+00 | 20260505 11:50:19.855 | INFO | PerfReport: 7.42%, PulseAccumulating -- 2.38e+00 / 2.85e+00, 4.29e-01 / 2.43e-01 | 20260505 11:50:19.855 | INFO | PerfReport: 3.67%, Metrics -- 1.18e+00 / 1.49e+00, 3.32e-01 / 1.82e-01 | 20260505 11:50:19.855 | INFO | PerfReport: 1.29%, PulseSerializing -- 4.14e-01 / 4.61e-01, 7.68e-02 / 3.08e-02 | 20260505 11:50:19.855 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 11:50:19.855 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:50:19.855 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:50:19.855 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260505 11:50:19.855 | INFO | PerfReport: 22.62%, Upload -- 2.29e+01 / 0.00e+00, 8.12e+00 / 0.00e+00 | 20260505 11:50:19.855 | INFO | PerfReport: 22.04%, Download -- 2.23e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260505 11:50:19.855 | INFO | PerfReport: 31.74%, Compute -- 3.21e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260505 11:50:19.855 | INFO | PerfReport: 2.06%, LockUpload -- 2.08e+00 / 0.00e+00, 7.60e+00 / 0.00e+00 | 20260505 11:50:19.855 | INFO | PerfReport: 1.29%, LockDownload -- 1.30e+00 / 0.00e+00, 2.80e+00 / 0.00e+00 | 20260505 11:50:19.855 | INFO | PerfReport: 20.24%, LockCompute -- 2.05e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260505 11:50:19.856 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.69e-02 / 0.00e+00 | 20260505 11:50:19.987 | INFO | Starting a round of Baz IO | 20260505 11:50:23.232 | INFO | Populating RTMetrics report | 20260505 11:50:23.233 | INFO | RTMetrics report populated | 20260505 11:50:23.233 | INFO | Opening tmp RTMetrics json file | 20260505 11:50:23.262 | INFO | Serializing RTMetrics to json | 20260505 11:50:23.310 | INFO | Serialization done, swapping tmp file now... | 20260505 11:50:23.312 | INFO | RTMetrics done | 20260505 11:50:23.391 | INFO | Finished a round of Baz IO: Wrote 2.361 GiB in 3.404 seconds | 20260505 11:50:23.391 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.92% / 0.01% / 76.07% | 20260505 11:50:23.391 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.0298% / 1.98682e-05% / 6.75519e-05% | 20260505 11:50:23.391 | INFO | Baz IO overall: | 20260505 11:50:23.391 | INFO | BytesWritten : 1767091154200 | 20260505 11:50:23.391 | INFO | NumEvents : 841303323049 | 20260505 11:50:24.995 | INFO | Analyzing chunk frames = [2981888,2982400) | 20260505 11:50:24.995 | INFO | MemInfo: { | 20260505 11:50:24.995 | INFO | "gpu" : " 17 GB", | 20260505 11:50:24.995 | INFO | "huge" : "", | 20260505 11:50:24.995 | INFO | "huge_shared" : " 27 GB", | 20260505 11:50:24.995 | INFO | "malloc" : " 14 GB", | 20260505 11:50:24.995 | INFO | "pinned" : " 29 GB" | 20260505 11:50:24.995 | INFO | } | 20260505 11:50:24.995 | INFO | Perf report for the last 5.14142 seconds: | 20260505 11:50:24.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:50:24.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:24.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:24.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:24.995 | INFO | ANALYSIS : 64.6%, 2.9, 99.6ms, 0 | 20260505 11:50:24.995 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:24.995 | INFO | RT_METRICS : 46.8%, 1.0, 24.6ms, 0 | 20260505 11:50:24.995 | INFO | PRE_HQ : 61.0%, 1.8, 58.4ms, 0 | 20260505 11:50:24.995 | INFO | BAZWRITER : 65.5%, 4.1, 138.4ms, 0 | 20260505 11:50:24.995 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260505 11:50:24.995 | INFO | Uploaded 36GiB to GPU in 2.19613s (16.3924GiB/s) | 20260505 11:50:24.995 | INFO | Downloaded 38GiB to GPU in 2.29642s (16.5475GiB/s) | 20260505 11:50:30.138 | INFO | Analyzing chunk frames = [2982400,2982912) | 20260505 11:50:30.138 | INFO | MemInfo: { | 20260505 11:50:30.138 | INFO | "gpu" : " 17 GB", | 20260505 11:50:30.138 | INFO | "huge" : "", | 20260505 11:50:30.138 | INFO | "huge_shared" : " 27 GB", | 20260505 11:50:30.138 | INFO | "malloc" : " 14 GB", | 20260505 11:50:30.138 | INFO | "pinned" : " 29 GB" | 20260505 11:50:30.138 | INFO | } | 20260505 11:50:30.138 | INFO | Perf report for the last 5.14329 seconds: | 20260505 11:50:30.138 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:50:30.138 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:30.138 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:30.138 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:30.138 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 11:50:30.138 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:30.138 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:30.138 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 11:50:30.138 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:30.138 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:30.138 | INFO | Uploaded 36GiB to GPU in 2.18796s (16.4536GiB/s) | 20260505 11:50:30.139 | INFO | Downloaded 34GiB to GPU in 2.05086s (16.5784GiB/s) | 20260505 11:50:35.270 | INFO | Analyzing chunk frames = [2982912,2983424) | 20260505 11:50:35.270 | INFO | MemInfo: { | 20260505 11:50:35.270 | INFO | "gpu" : " 17 GB", | 20260505 11:50:35.270 | INFO | "huge" : "", | 20260505 11:50:35.270 | INFO | "huge_shared" : " 27 GB", | 20260505 11:50:35.270 | INFO | "malloc" : " 14 GB", | 20260505 11:50:35.270 | INFO | "pinned" : " 29 GB" | 20260505 11:50:35.270 | INFO | } | 20260505 11:50:35.270 | INFO | Perf report for the last 5.13184 seconds: | 20260505 11:50:35.270 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:50:35.270 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:35.270 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:35.270 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:35.270 | INFO | ANALYSIS : 63.4%, 3.0, 98.5ms, 0 | 20260505 11:50:35.270 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:35.270 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:35.270 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 11:50:35.270 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:35.270 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:35.270 | INFO | Uploaded 36GiB to GPU in 2.21544s (16.2496GiB/s) | 20260505 11:50:35.270 | INFO | Downloaded 34GiB to GPU in 2.11335s (16.0882GiB/s) | 20260505 11:50:40.407 | INFO | Analyzing chunk frames = [2983424,2983936) | 20260505 11:50:40.407 | INFO | MemInfo: { | 20260505 11:50:40.407 | INFO | "gpu" : " 17 GB", | 20260505 11:50:40.407 | INFO | "huge" : "", | 20260505 11:50:40.407 | INFO | "huge_shared" : " 27 GB", | 20260505 11:50:40.407 | INFO | "malloc" : " 14 GB", | 20260505 11:50:40.407 | INFO | "pinned" : " 29 GB" | 20260505 11:50:40.407 | INFO | } | 20260505 11:50:40.407 | INFO | Perf report for the last 5.13722 seconds: | 20260505 11:50:40.407 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:50:40.407 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:40.407 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:40.407 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:40.407 | INFO | ANALYSIS : 62.9%, 3.0, 97.7ms, 0 | 20260505 11:50:40.407 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:40.407 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:40.407 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 11:50:40.407 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:40.407 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:40.407 | INFO | Uploaded 36GiB to GPU in 2.15971s (16.6689GiB/s) | 20260505 11:50:40.408 | INFO | Downloaded 34GiB to GPU in 2.07796s (16.3622GiB/s) | 20260505 11:50:45.552 | INFO | Analyzing chunk frames = [2983936,2984448) | 20260505 11:50:45.552 | INFO | MemInfo: { | 20260505 11:50:45.552 | INFO | "gpu" : " 17 GB", | 20260505 11:50:45.552 | INFO | "huge" : "", | 20260505 11:50:45.552 | INFO | "huge_shared" : " 27 GB", | 20260505 11:50:45.552 | INFO | "malloc" : " 14 GB", | 20260505 11:50:45.552 | INFO | "pinned" : " 29 GB" | 20260505 11:50:45.552 | INFO | } | 20260505 11:50:45.552 | INFO | Perf report for the last 5.14479 seconds: | 20260505 11:50:45.552 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:50:45.552 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:45.552 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:45.552 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:45.552 | INFO | ANALYSIS : 63.1%, 3.0, 98.7ms, 0 | 20260505 11:50:45.552 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:45.552 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:45.552 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:50:45.552 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:45.552 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:45.552 | INFO | Uploaded 36GiB to GPU in 2.15586s (16.6987GiB/s) | 20260505 11:50:45.552 | INFO | Downloaded 34GiB to GPU in 2.12143s (16.0269GiB/s) | 20260505 11:50:50.694 | INFO | Analyzing chunk frames = [2984448,2984960) | 20260505 11:50:50.694 | INFO | MemInfo: { | 20260505 11:50:50.694 | INFO | "gpu" : " 17 GB", | 20260505 11:50:50.694 | INFO | "huge" : "", | 20260505 11:50:50.694 | INFO | "huge_shared" : " 27 GB", | 20260505 11:50:50.694 | INFO | "malloc" : " 14 GB", | 20260505 11:50:50.694 | INFO | "pinned" : " 29 GB" | 20260505 11:50:50.694 | INFO | } | 20260505 11:50:50.694 | INFO | Perf report for the last 5.14233 seconds: | 20260505 11:50:50.694 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:50:50.694 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:50.694 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:50.694 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:50.694 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 11:50:50.694 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:50.694 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:50.695 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 11:50:50.695 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:50.695 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:50.695 | INFO | Uploaded 36GiB to GPU in 2.1694s (16.5944GiB/s) | 20260505 11:50:50.695 | INFO | Downloaded 34GiB to GPU in 2.10683s (16.138GiB/s) | 20260505 11:50:55.830 | INFO | Analyzing chunk frames = [2984960,2985472) | 20260505 11:50:55.830 | INFO | MemInfo: { | 20260505 11:50:55.830 | INFO | "gpu" : " 17 GB", | 20260505 11:50:55.830 | INFO | "huge" : "", | 20260505 11:50:55.830 | INFO | "huge_shared" : " 27 GB", | 20260505 11:50:55.830 | INFO | "malloc" : " 14 GB", | 20260505 11:50:55.830 | INFO | "pinned" : " 29 GB" | 20260505 11:50:55.830 | INFO | } | 20260505 11:50:55.830 | INFO | Perf report for the last 5.13567 seconds: | 20260505 11:50:55.830 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:50:55.830 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:55.830 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:55.830 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:55.830 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 11:50:55.830 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:55.830 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:55.830 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 11:50:55.830 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:55.830 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:50:55.830 | INFO | Uploaded 36GiB to GPU in 2.16385s (16.637GiB/s) | 20260505 11:50:55.830 | INFO | Downloaded 34GiB to GPU in 2.13028s (15.9603GiB/s) | 20260505 11:51:00.966 | INFO | Analyzing chunk frames = [2985472,2985984) | 20260505 11:51:00.966 | INFO | MemInfo: { | 20260505 11:51:00.966 | INFO | "gpu" : " 17 GB", | 20260505 11:51:00.966 | INFO | "huge" : "", | 20260505 11:51:00.966 | INFO | "huge_shared" : " 27 GB", | 20260505 11:51:00.966 | INFO | "malloc" : " 14 GB", | 20260505 11:51:00.966 | INFO | "pinned" : " 29 GB" | 20260505 11:51:00.966 | INFO | } | 20260505 11:51:00.966 | INFO | Perf report for the last 5.13583 seconds: | 20260505 11:51:00.966 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:51:00.966 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:00.966 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:00.966 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:00.966 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 11:51:00.966 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:00.966 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:00.966 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 11:51:00.966 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:00.966 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:00.966 | INFO | Uploaded 36GiB to GPU in 2.22178s (16.2032GiB/s) | 20260505 11:51:00.966 | INFO | Downloaded 34GiB to GPU in 2.08045s (16.3426GiB/s) | 20260505 11:51:01.103 | INFO | Starting a round of Baz IO | 20260505 11:51:04.358 | INFO | Populating RTMetrics report | 20260505 11:51:04.359 | INFO | RTMetrics report populated | 20260505 11:51:04.359 | INFO | Opening tmp RTMetrics json file | 20260505 11:51:04.359 | INFO | Serializing RTMetrics to json | 20260505 11:51:04.360 | INFO | Serialization done, swapping tmp file now... | 20260505 11:51:04.361 | INFO | RTMetrics done | 20260505 11:51:04.492 | INFO | Finished a round of Baz IO: Wrote 2.359 GiB in 3.389 seconds | 20260505 11:51:04.492 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.92% / 0.01% / 76.08% | 20260505 11:51:04.492 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.02593% / 7.94729e-06% / 7.94729e-05% | 20260505 11:51:04.492 | INFO | Baz IO overall: | 20260505 11:51:04.492 | INFO | BytesWritten : 1769624623767 | 20260505 11:51:04.492 | INFO | NumEvents : 842506516568 | 20260505 11:51:06.108 | INFO | Analyzing chunk frames = [2985984,2986496) | 20260505 11:51:06.108 | INFO | MemInfo: { | 20260505 11:51:06.108 | INFO | "gpu" : " 17 GB", | 20260505 11:51:06.108 | INFO | "huge" : "", | 20260505 11:51:06.108 | INFO | "huge_shared" : " 27 GB", | 20260505 11:51:06.108 | INFO | "malloc" : " 14 GB", | 20260505 11:51:06.108 | INFO | "pinned" : " 29 GB" | 20260505 11:51:06.108 | INFO | } | 20260505 11:51:06.108 | INFO | Perf report for the last 5.14236 seconds: | 20260505 11:51:06.108 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:51:06.108 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:06.108 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:06.108 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:06.108 | INFO | ANALYSIS : 64.9%, 2.9, 99.7ms, 0 | 20260505 11:51:06.108 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:06.108 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260505 11:51:06.108 | INFO | PRE_HQ : 61.5%, 1.8, 58.8ms, 0 | 20260505 11:51:06.108 | INFO | BAZWRITER : 65.3%, 4.1, 138.4ms, 0 | 20260505 11:51:06.108 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 11:51:06.109 | INFO | Uploaded 36GiB to GPU in 2.19668s (16.3884GiB/s) | 20260505 11:51:06.109 | INFO | Downloaded 38GiB to GPU in 2.30757s (16.4675GiB/s) | 20260505 11:51:11.246 | INFO | Analyzing chunk frames = [2986496,2987008) | 20260505 11:51:11.246 | INFO | MemInfo: { | 20260505 11:51:11.246 | INFO | "gpu" : " 17 GB", | 20260505 11:51:11.246 | INFO | "huge" : "", | 20260505 11:51:11.246 | INFO | "huge_shared" : " 27 GB", | 20260505 11:51:11.246 | INFO | "malloc" : " 14 GB", | 20260505 11:51:11.246 | INFO | "pinned" : " 29 GB" | 20260505 11:51:11.246 | INFO | } | 20260505 11:51:11.246 | INFO | Perf report for the last 5.13794 seconds: | 20260505 11:51:11.246 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:51:11.246 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:11.246 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:11.246 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:11.246 | INFO | ANALYSIS : 63.8%, 3.0, 99.5ms, 0 | 20260505 11:51:11.246 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:11.246 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:11.246 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 11:51:11.246 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:11.247 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:11.247 | INFO | Uploaded 36GiB to GPU in 2.1836s (16.4866GiB/s) | 20260505 11:51:11.247 | INFO | Downloaded 34GiB to GPU in 2.04964s (16.5883GiB/s) | 20260505 11:51:16.387 | INFO | Analyzing chunk frames = [2987008,2987520) | 20260505 11:51:16.387 | INFO | MemInfo: { | 20260505 11:51:16.387 | INFO | "gpu" : " 17 GB", | 20260505 11:51:16.387 | INFO | "huge" : "", | 20260505 11:51:16.387 | INFO | "huge_shared" : " 27 GB", | 20260505 11:51:16.387 | INFO | "malloc" : " 14 GB", | 20260505 11:51:16.387 | INFO | "pinned" : " 29 GB" | 20260505 11:51:16.387 | INFO | } | 20260505 11:51:16.387 | INFO | Perf report for the last 5.14107 seconds: | 20260505 11:51:16.387 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:51:16.387 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:16.387 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:16.387 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:16.387 | INFO | ANALYSIS : 63.2%, 3.0, 98.4ms, 0 | 20260505 11:51:16.387 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:16.387 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:16.387 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 11:51:16.387 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:16.387 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:16.388 | INFO | Uploaded 36GiB to GPU in 2.20565s (16.3217GiB/s) | 20260505 11:51:16.388 | INFO | Downloaded 34GiB to GPU in 2.11778s (16.0545GiB/s) | 20260505 11:51:21.523 | INFO | Analyzing chunk frames = [2987520,2988032) | 20260505 11:51:21.523 | INFO | MemInfo: { | 20260505 11:51:21.523 | INFO | "gpu" : " 17 GB", | 20260505 11:51:21.523 | INFO | "huge" : "", | 20260505 11:51:21.523 | INFO | "huge_shared" : " 27 GB", | 20260505 11:51:21.523 | INFO | "malloc" : " 14 GB", | 20260505 11:51:21.523 | INFO | "pinned" : " 29 GB" | 20260505 11:51:21.523 | INFO | } | 20260505 11:51:21.523 | INFO | Perf report for the last 5.13556 seconds: | 20260505 11:51:21.523 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:51:21.523 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:21.523 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:21.523 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:21.523 | INFO | ANALYSIS : 63.5%, 3.0, 98.6ms, 0 | 20260505 11:51:21.523 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:21.523 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:21.523 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 11:51:21.523 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:21.523 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:21.523 | INFO | Uploaded 36GiB to GPU in 2.2167s (16.2404GiB/s) | 20260505 11:51:21.523 | INFO | Downloaded 34GiB to GPU in 2.13291s (15.9406GiB/s) | 20260505 11:51:26.662 | INFO | Analyzing chunk frames = [2988032,2988544) | 20260505 11:51:26.662 | INFO | MemInfo: { | 20260505 11:51:26.662 | INFO | "gpu" : " 17 GB", | 20260505 11:51:26.662 | INFO | "huge" : "", | 20260505 11:51:26.662 | INFO | "huge_shared" : " 27 GB", | 20260505 11:51:26.662 | INFO | "malloc" : " 14 GB", | 20260505 11:51:26.662 | INFO | "pinned" : " 29 GB" | 20260505 11:51:26.662 | INFO | } | 20260505 11:51:26.662 | INFO | Perf report for the last 5.13942 seconds: | 20260505 11:51:26.662 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:51:26.662 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:26.662 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:26.662 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:26.662 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 11:51:26.662 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:26.662 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:26.662 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 11:51:26.662 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:26.662 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:26.663 | INFO | Uploaded 36GiB to GPU in 2.15868s (16.6769GiB/s) | 20260505 11:51:26.663 | INFO | Downloaded 34GiB to GPU in 2.12456s (16.0033GiB/s) | 20260505 11:51:31.807 | INFO | Analyzing chunk frames = [2988544,2989056) | 20260505 11:51:31.807 | INFO | MemInfo: { | 20260505 11:51:31.807 | INFO | "gpu" : " 17 GB", | 20260505 11:51:31.807 | INFO | "huge" : "", | 20260505 11:51:31.807 | INFO | "huge_shared" : " 27 GB", | 20260505 11:51:31.807 | INFO | "malloc" : " 14 GB", | 20260505 11:51:31.807 | INFO | "pinned" : " 29 GB" | 20260505 11:51:31.807 | INFO | } | 20260505 11:51:31.807 | INFO | Perf report for the last 5.14444 seconds: | 20260505 11:51:31.807 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:51:31.807 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:31.807 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:31.807 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:31.807 | INFO | ANALYSIS : 63.3%, 3.0, 99.0ms, 0 | 20260505 11:51:31.807 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:31.807 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:31.807 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 11:51:31.807 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:31.807 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:31.807 | INFO | Uploaded 36GiB to GPU in 2.18151s (16.5024GiB/s) | 20260505 11:51:31.807 | INFO | Downloaded 34GiB to GPU in 2.11475s (16.0775GiB/s) | 20260505 11:51:36.949 | INFO | Analyzing chunk frames = [2989056,2989568) | 20260505 11:51:36.949 | INFO | MemInfo: { | 20260505 11:51:36.949 | INFO | "gpu" : " 17 GB", | 20260505 11:51:36.949 | INFO | "huge" : "", | 20260505 11:51:36.949 | INFO | "huge_shared" : " 27 GB", | 20260505 11:51:36.949 | INFO | "malloc" : " 14 GB", | 20260505 11:51:36.949 | INFO | "pinned" : " 29 GB" | 20260505 11:51:36.949 | INFO | } | 20260505 11:51:36.950 | INFO | Perf report for the last 5.1427 seconds: | 20260505 11:51:36.950 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:51:36.950 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:36.950 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:36.950 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:36.950 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 11:51:36.950 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:36.950 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:36.950 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 11:51:36.950 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:36.950 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:36.950 | INFO | Uploaded 36GiB to GPU in 2.16001s (16.6666GiB/s) | 20260505 11:51:36.950 | INFO | Downloaded 34GiB to GPU in 2.13078s (15.9566GiB/s) | 20260505 11:51:42.080 | INFO | Analyzing chunk frames = [2989568,2990080) | 20260505 11:51:42.080 | INFO | MemInfo: { | 20260505 11:51:42.080 | INFO | "gpu" : " 17 GB", | 20260505 11:51:42.080 | INFO | "huge" : "", | 20260505 11:51:42.080 | INFO | "huge_shared" : " 27 GB", | 20260505 11:51:42.080 | INFO | "malloc" : " 14 GB", | 20260505 11:51:42.080 | INFO | "pinned" : " 29 GB" | 20260505 11:51:42.080 | INFO | } | 20260505 11:51:42.080 | INFO | Perf report for the last 5.1306 seconds: | 20260505 11:51:42.080 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:51:42.080 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:42.080 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:42.080 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:42.080 | INFO | ANALYSIS : 64.0%, 3.0, 99.8ms, 0 | 20260505 11:51:42.080 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:42.080 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:42.080 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 11:51:42.080 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:42.080 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:42.080 | INFO | Uploaded 36GiB to GPU in 2.19693s (16.3865GiB/s) | 20260505 11:51:42.080 | INFO | Downloaded 34GiB to GPU in 2.11751s (16.0566GiB/s) | 20260505 11:51:42.081 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 11:51:42.081 | INFO | Total mem usage: 15.3 GiB | 20260505 11:51:42.081 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 11:51:42.081 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.77%) | 20260505 11:51:42.081 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 11:51:42.081 | INFO | High Water Breakdown: | 20260505 11:51:42.081 | INFO | 61.1% : 8.77 GiB ( 92.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 11:51:42.081 | INFO | 38.8% : 5.57 GiB ( 83.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 11:51:42.081 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 11:51:42.081 | INFO | ------------------------------------------ | 20260505 11:51:42.081 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:51:42.081 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 11:51:42.081 | INFO | Total mem usage: 26.5 GiB | 20260505 11:51:42.081 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 11:51:42.081 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 11:51:42.081 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 11:51:42.081 | INFO | High Water Breakdown: | 20260505 11:51:42.081 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:51:42.081 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:51:42.081 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:51:42.081 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:51:42.081 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:51:42.081 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:51:42.081 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 11:51:42.081 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 11:51:42.081 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:51:42.081 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 11:51:42.081 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 11:51:42.081 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 11:51:42.081 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 11:51:42.081 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 11:51:42.081 | INFO | ------------------------------------------ | 20260505 11:51:42.081 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:51:42.081 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 11:51:42.081 | INFO | Total mem usage: 25.6 GiB | 20260505 11:51:42.081 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 11:51:42.081 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 11:51:42.081 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 11:51:42.081 | INFO | High Water Breakdown: | 20260505 11:51:42.081 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 11:51:42.081 | INFO | ------------------------------------------ | 20260505 11:51:42.081 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:51:42.081 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 11:51:42.081 | INFO | Total mem usage: 15.7 GiB | 20260505 11:51:42.081 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 11:51:42.081 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 11:51:42.081 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 11:51:42.081 | INFO | High Water Breakdown: | 20260505 11:51:42.081 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 11:51:42.081 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 11:51:42.081 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 11:51:42.081 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 11:51:42.081 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 11:51:42.081 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 11:51:42.081 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 11:51:42.081 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 11:51:42.081 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 11:51:42.081 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:51:42.081 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 11:51:42.081 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:51:42.081 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 11:51:42.082 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 11:51:42.082 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 11:51:42.082 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 11:51:42.082 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:51:42.082 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 11:51:42.082 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:51:42.082 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:51:42.082 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:51:42.082 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 11:51:42.082 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 11:51:42.082 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:51:42.082 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 11:51:42.082 | INFO | ------------------------------------------ | 20260505 11:51:42.082 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:51:42.082 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 11:51:42.082 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:51:42.082 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:51:42.082 | INFO | PerfReport: Full Execution -- 3.21e+01 / 7.71e+01, 1.38e+01 / 1.73e+00 | 20260505 11:51:42.082 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 1.02e-01, 1.04e-02 / 4.88e-03 | 20260505 11:51:42.082 | INFO | PerfReport: 11.74%, Baseline -- 3.77e+00 / 4.45e+00, 6.03e-01 / 2.53e-01 | 20260505 11:51:42.082 | INFO | PerfReport: 3.29%, Histogram -- 1.06e+00 / 1.49e+00, 2.03e-01 / 9.86e-02 | 20260505 11:51:42.082 | INFO | PerfReport: 15.87%, DME -- 4.07e+01 / 4.55e+01, 4.31e+00 / 2.22e+00 | 20260505 11:51:42.082 | INFO | PerfReport: 56.46%, FrameLabeling -- 1.81e+01 / 2.07e+01, 1.60e+00 / 2.02e+00 | 20260505 11:51:42.082 | INFO | PerfReport: 7.41%, PulseAccumulating -- 2.38e+00 / 2.92e+00, 4.37e-01 / 2.07e-01 | 20260505 11:51:42.082 | INFO | PerfReport: 3.64%, Metrics -- 1.17e+00 / 1.51e+00, 3.28e-01 / 1.55e-01 | 20260505 11:51:42.082 | INFO | PerfReport: 1.29%, PulseSerializing -- 4.13e-01 / 4.67e-01, 7.62e-02 / 2.73e-02 | 20260505 11:51:42.082 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 11:51:42.082 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:51:42.082 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:51:42.082 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260505 11:51:42.082 | INFO | PerfReport: 22.53%, Upload -- 2.28e+01 / 0.00e+00, 8.10e+00 / 0.00e+00 | 20260505 11:51:42.082 | INFO | PerfReport: 21.96%, Download -- 2.22e+01 / 0.00e+00, 8.17e+00 / 0.00e+00 | 20260505 11:51:42.082 | INFO | PerfReport: 31.77%, Compute -- 3.21e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 11:51:42.082 | INFO | PerfReport: 2.07%, LockUpload -- 2.10e+00 / 0.00e+00, 7.74e+00 / 0.00e+00 | 20260505 11:51:42.082 | INFO | PerfReport: 1.34%, LockDownload -- 1.36e+00 / 0.00e+00, 2.94e+00 / 0.00e+00 | 20260505 11:51:42.082 | INFO | PerfReport: 20.31%, LockCompute -- 2.05e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260505 11:51:42.082 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.66e-02 / 0.00e+00 | 20260505 11:51:42.217 | INFO | Starting a round of Baz IO | 20260505 11:51:45.475 | INFO | Populating RTMetrics report | 20260505 11:51:45.475 | INFO | RTMetrics report populated | 20260505 11:51:45.475 | INFO | Opening tmp RTMetrics json file | 20260505 11:51:45.481 | INFO | Serializing RTMetrics to json | 20260505 11:51:45.518 | INFO | Serialization done, swapping tmp file now... | 20260505 11:51:45.530 | INFO | RTMetrics done | 20260505 11:51:45.589 | INFO | Finished a round of Baz IO: Wrote 2.359 GiB in 3.371 seconds | 20260505 11:51:45.589 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.91% / 0.01% / 76.08% | 20260505 11:51:45.589 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.023% / 7.94729e-06% / 6.35783e-05% | 20260505 11:51:45.589 | INFO | Baz IO overall: | 20260505 11:51:45.589 | INFO | BytesWritten : 1772157351082 | 20260505 11:51:45.589 | INFO | NumEvents : 843709145497 | 20260505 11:51:47.216 | INFO | Analyzing chunk frames = [2990080,2990592) | 20260505 11:51:47.216 | INFO | MemInfo: { | 20260505 11:51:47.216 | INFO | "gpu" : " 17 GB", | 20260505 11:51:47.216 | INFO | "huge" : "", | 20260505 11:51:47.216 | INFO | "huge_shared" : " 27 GB", | 20260505 11:51:47.216 | INFO | "malloc" : " 14 GB", | 20260505 11:51:47.216 | INFO | "pinned" : " 29 GB" | 20260505 11:51:47.216 | INFO | } | 20260505 11:51:47.216 | INFO | Perf report for the last 5.13601 seconds: | 20260505 11:51:47.216 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:51:47.216 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:47.216 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:47.216 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:47.216 | INFO | ANALYSIS : 65.0%, 2.9, 100.1ms, 0 | 20260505 11:51:47.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:47.216 | INFO | RT_METRICS : 46.6%, 1.0, 24.4ms, 0 | 20260505 11:51:47.216 | INFO | PRE_HQ : 61.4%, 1.8, 58.6ms, 0 | 20260505 11:51:47.216 | INFO | BAZWRITER : 65.0%, 4.1, 136.8ms, 0 | 20260505 11:51:47.216 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 11:51:47.216 | INFO | Uploaded 36GiB to GPU in 2.20888s (16.2979GiB/s) | 20260505 11:51:47.217 | INFO | Downloaded 38GiB to GPU in 2.30158s (16.5104GiB/s) | 20260505 11:51:52.362 | INFO | Analyzing chunk frames = [2990592,2991104) | 20260505 11:51:52.362 | INFO | MemInfo: { | 20260505 11:51:52.362 | INFO | "gpu" : " 17 GB", | 20260505 11:51:52.362 | INFO | "huge" : "", | 20260505 11:51:52.362 | INFO | "huge_shared" : " 27 GB", | 20260505 11:51:52.362 | INFO | "malloc" : " 14 GB", | 20260505 11:51:52.362 | INFO | "pinned" : " 29 GB" | 20260505 11:51:52.362 | INFO | } | 20260505 11:51:52.362 | INFO | Perf report for the last 5.14625 seconds: | 20260505 11:51:52.362 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:51:52.362 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:52.362 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:52.362 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:52.362 | INFO | ANALYSIS : 64.0%, 3.0, 100.1ms, 0 | 20260505 11:51:52.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:52.362 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:52.362 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 11:51:52.363 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:52.363 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:52.363 | INFO | Uploaded 36GiB to GPU in 2.23006s (16.143GiB/s) | 20260505 11:51:52.363 | INFO | Downloaded 34GiB to GPU in 2.08802s (16.2834GiB/s) | 20260505 11:51:57.505 | INFO | Analyzing chunk frames = [2991104,2991616) | 20260505 11:51:57.505 | INFO | MemInfo: { | 20260505 11:51:57.505 | INFO | "gpu" : " 17 GB", | 20260505 11:51:57.506 | INFO | "huge" : "", | 20260505 11:51:57.506 | INFO | "huge_shared" : " 27 GB", | 20260505 11:51:57.506 | INFO | "malloc" : " 14 GB", | 20260505 11:51:57.506 | INFO | "pinned" : " 29 GB" | 20260505 11:51:57.506 | INFO | } | 20260505 11:51:57.506 | INFO | Perf report for the last 5.14317 seconds: | 20260505 11:51:57.506 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:51:57.506 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:57.506 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:57.506 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:57.506 | INFO | ANALYSIS : 63.1%, 3.0, 98.3ms, 0 | 20260505 11:51:57.506 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:57.506 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:57.506 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 11:51:57.506 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:57.506 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:51:57.506 | INFO | Uploaded 36GiB to GPU in 2.17872s (16.5235GiB/s) | 20260505 11:51:57.506 | INFO | Downloaded 34GiB to GPU in 2.11746s (16.0569GiB/s) | 20260505 11:52:02.650 | INFO | Analyzing chunk frames = [2991616,2992128) | 20260505 11:52:02.650 | INFO | MemInfo: { | 20260505 11:52:02.650 | INFO | "gpu" : " 17 GB", | 20260505 11:52:02.650 | INFO | "huge" : "", | 20260505 11:52:02.650 | INFO | "huge_shared" : " 27 GB", | 20260505 11:52:02.650 | INFO | "malloc" : " 14 GB", | 20260505 11:52:02.650 | INFO | "pinned" : " 29 GB" | 20260505 11:52:02.650 | INFO | } | 20260505 11:52:02.650 | INFO | Perf report for the last 5.14483 seconds: | 20260505 11:52:02.650 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:52:02.650 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:02.650 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:02.650 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:02.650 | INFO | ANALYSIS : 63.1%, 3.0, 98.1ms, 0 | 20260505 11:52:02.650 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:02.650 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:02.650 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 11:52:02.650 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:02.650 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:02.651 | INFO | Uploaded 36GiB to GPU in 2.19906s (16.3706GiB/s) | 20260505 11:52:02.651 | INFO | Downloaded 34GiB to GPU in 2.13211s (15.9466GiB/s) | 20260505 11:52:07.780 | INFO | Analyzing chunk frames = [2992128,2992640) | 20260505 11:52:07.780 | INFO | MemInfo: { | 20260505 11:52:07.780 | INFO | "gpu" : " 17 GB", | 20260505 11:52:07.780 | INFO | "huge" : "", | 20260505 11:52:07.780 | INFO | "huge_shared" : " 27 GB", | 20260505 11:52:07.780 | INFO | "malloc" : " 14 GB", | 20260505 11:52:07.780 | INFO | "pinned" : " 29 GB" | 20260505 11:52:07.780 | INFO | } | 20260505 11:52:07.780 | INFO | Perf report for the last 5.12952 seconds: | 20260505 11:52:07.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:52:07.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:07.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:07.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:07.780 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 11:52:07.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:07.780 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:07.780 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 11:52:07.780 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:07.780 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:07.780 | INFO | Uploaded 36GiB to GPU in 2.17074s (16.5842GiB/s) | 20260505 11:52:07.780 | INFO | Downloaded 34GiB to GPU in 2.12394s (16.008GiB/s) | 20260505 11:52:12.921 | INFO | Analyzing chunk frames = [2992640,2993152) | 20260505 11:52:12.921 | INFO | MemInfo: { | 20260505 11:52:12.921 | INFO | "gpu" : " 17 GB", | 20260505 11:52:12.921 | INFO | "huge" : "", | 20260505 11:52:12.921 | INFO | "huge_shared" : " 27 GB", | 20260505 11:52:12.921 | INFO | "malloc" : " 14 GB", | 20260505 11:52:12.921 | INFO | "pinned" : " 29 GB" | 20260505 11:52:12.921 | INFO | } | 20260505 11:52:12.921 | INFO | Perf report for the last 5.14102 seconds: | 20260505 11:52:12.921 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:52:12.921 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:12.921 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:12.921 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:12.921 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 11:52:12.921 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:12.921 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:12.921 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 11:52:12.921 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:12.921 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:12.921 | INFO | Uploaded 36GiB to GPU in 2.16459s (16.6313GiB/s) | 20260505 11:52:12.921 | INFO | Downloaded 34GiB to GPU in 2.12503s (15.9998GiB/s) | 20260505 11:52:18.060 | INFO | Analyzing chunk frames = [2993152,2993664) | 20260505 11:52:18.060 | INFO | MemInfo: { | 20260505 11:52:18.060 | INFO | "gpu" : " 17 GB", | 20260505 11:52:18.060 | INFO | "huge" : "", | 20260505 11:52:18.060 | INFO | "huge_shared" : " 27 GB", | 20260505 11:52:18.060 | INFO | "malloc" : " 14 GB", | 20260505 11:52:18.060 | INFO | "pinned" : " 29 GB" | 20260505 11:52:18.060 | INFO | } | 20260505 11:52:18.060 | INFO | Perf report for the last 5.13945 seconds: | 20260505 11:52:18.060 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:52:18.060 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:18.061 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:18.061 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:18.061 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 11:52:18.061 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:18.061 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:18.061 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 11:52:18.061 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:18.061 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:18.061 | INFO | Uploaded 36GiB to GPU in 2.15832s (16.6797GiB/s) | 20260505 11:52:18.061 | INFO | Downloaded 34GiB to GPU in 2.14909s (15.8206GiB/s) | 20260505 11:52:23.205 | INFO | Analyzing chunk frames = [2993664,2994176) | 20260505 11:52:23.205 | INFO | MemInfo: { | 20260505 11:52:23.205 | INFO | "gpu" : " 17 GB", | 20260505 11:52:23.205 | INFO | "huge" : "", | 20260505 11:52:23.205 | INFO | "huge_shared" : " 27 GB", | 20260505 11:52:23.205 | INFO | "malloc" : " 14 GB", | 20260505 11:52:23.205 | INFO | "pinned" : " 29 GB" | 20260505 11:52:23.205 | INFO | } | 20260505 11:52:23.205 | INFO | Perf report for the last 5.14471 seconds: | 20260505 11:52:23.205 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:52:23.205 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:23.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:23.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:23.205 | INFO | ANALYSIS : 63.7%, 3.0, 99.6ms, 0 | 20260505 11:52:23.205 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:23.205 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:23.205 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 11:52:23.205 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:23.205 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:23.205 | INFO | Uploaded 36GiB to GPU in 2.21388s (16.261GiB/s) | 20260505 11:52:23.205 | INFO | Downloaded 34GiB to GPU in 2.10386s (16.1607GiB/s) | 20260505 11:52:23.339 | INFO | Starting a round of Baz IO | 20260505 11:52:26.573 | INFO | Populating RTMetrics report | 20260505 11:52:26.573 | INFO | RTMetrics report populated | 20260505 11:52:26.573 | INFO | Opening tmp RTMetrics json file | 20260505 11:52:26.679 | INFO | Serializing RTMetrics to json | 20260505 11:52:26.689 | INFO | Serialization done, swapping tmp file now... | 20260505 11:52:26.689 | INFO | RTMetrics done | 20260505 11:52:26.728 | INFO | Finished a round of Baz IO: Wrote 2.361 GiB in 3.389 seconds | 20260505 11:52:26.728 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.90% / 0.01% / 76.09% | 20260505 11:52:26.728 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.02313% / 7.94729e-06% / 6.75519e-05% | 20260505 11:52:26.728 | INFO | Baz IO overall: | 20260505 11:52:26.728 | INFO | BytesWritten : 1774692282695 | 20260505 11:52:26.728 | INFO | NumEvents : 844913722169 | 20260505 11:52:28.335 | INFO | Analyzing chunk frames = [2994176,2994688) | 20260505 11:52:28.335 | INFO | MemInfo: { | 20260505 11:52:28.335 | INFO | "gpu" : " 17 GB", | 20260505 11:52:28.335 | INFO | "huge" : "", | 20260505 11:52:28.335 | INFO | "huge_shared" : " 27 GB", | 20260505 11:52:28.335 | INFO | "malloc" : " 14 GB", | 20260505 11:52:28.335 | INFO | "pinned" : " 29 GB" | 20260505 11:52:28.335 | INFO | } | 20260505 11:52:28.335 | INFO | Perf report for the last 5.13028 seconds: | 20260505 11:52:28.336 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:52:28.336 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:28.336 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:28.336 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:28.336 | INFO | ANALYSIS : 64.5%, 2.9, 99.4ms, 0 | 20260505 11:52:28.336 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:28.336 | INFO | RT_METRICS : 47.6%, 1.0, 24.9ms, 0 | 20260505 11:52:28.336 | INFO | PRE_HQ : 61.3%, 1.8, 58.5ms, 0 | 20260505 11:52:28.336 | INFO | BAZWRITER : 65.2%, 4.1, 137.8ms, 0 | 20260505 11:52:28.336 | INFO | BAZ_SAMPLING : 2.1%, 1.0, 1.1ms, 0 | 20260505 11:52:28.336 | INFO | Uploaded 36GiB to GPU in 2.19024s (16.4366GiB/s) | 20260505 11:52:28.336 | INFO | Downloaded 38GiB to GPU in 2.28072s (16.6614GiB/s) | 20260505 11:52:33.471 | INFO | Analyzing chunk frames = [2994688,2995200) | 20260505 11:52:33.471 | INFO | MemInfo: { | 20260505 11:52:33.471 | INFO | "gpu" : " 17 GB", | 20260505 11:52:33.471 | INFO | "huge" : "", | 20260505 11:52:33.471 | INFO | "huge_shared" : " 27 GB", | 20260505 11:52:33.471 | INFO | "malloc" : " 14 GB", | 20260505 11:52:33.471 | INFO | "pinned" : " 29 GB" | 20260505 11:52:33.471 | INFO | } | 20260505 11:52:33.471 | INFO | Perf report for the last 5.13595 seconds: | 20260505 11:52:33.471 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:52:33.471 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:33.471 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:33.471 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:33.471 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 11:52:33.471 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:33.472 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:33.472 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 11:52:33.472 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:33.472 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:33.472 | INFO | Uploaded 36GiB to GPU in 2.23015s (16.1424GiB/s) | 20260505 11:52:33.472 | INFO | Downloaded 34GiB to GPU in 2.08707s (16.2908GiB/s) | 20260505 11:52:38.617 | INFO | Analyzing chunk frames = [2995200,2995712) | 20260505 11:52:38.617 | INFO | MemInfo: { | 20260505 11:52:38.617 | INFO | "gpu" : " 17 GB", | 20260505 11:52:38.617 | INFO | "huge" : "", | 20260505 11:52:38.617 | INFO | "huge_shared" : " 27 GB", | 20260505 11:52:38.617 | INFO | "malloc" : " 14 GB", | 20260505 11:52:38.617 | INFO | "pinned" : " 29 GB" | 20260505 11:52:38.617 | INFO | } | 20260505 11:52:38.617 | INFO | Perf report for the last 5.14563 seconds: | 20260505 11:52:38.617 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:52:38.617 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:38.617 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:38.617 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:38.617 | INFO | ANALYSIS : 63.5%, 3.0, 99.0ms, 0 | 20260505 11:52:38.617 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:38.617 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:38.617 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 11:52:38.617 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:38.617 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:38.617 | INFO | Uploaded 36GiB to GPU in 2.2257s (16.1747GiB/s) | 20260505 11:52:38.617 | INFO | Downloaded 34GiB to GPU in 2.13117s (15.9537GiB/s) | 20260505 11:52:43.755 | INFO | Analyzing chunk frames = [2995712,2996224) | 20260505 11:52:43.755 | INFO | MemInfo: { | 20260505 11:52:43.755 | INFO | "gpu" : " 17 GB", | 20260505 11:52:43.755 | INFO | "huge" : "", | 20260505 11:52:43.756 | INFO | "huge_shared" : " 27 GB", | 20260505 11:52:43.756 | INFO | "malloc" : " 14 GB", | 20260505 11:52:43.756 | INFO | "pinned" : " 29 GB" | 20260505 11:52:43.756 | INFO | } | 20260505 11:52:43.756 | INFO | Perf report for the last 5.13852 seconds: | 20260505 11:52:43.756 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:52:43.756 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:43.756 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:43.756 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:43.756 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 11:52:43.756 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:43.756 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:43.756 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 11:52:43.756 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:43.756 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:43.756 | INFO | Uploaded 36GiB to GPU in 2.208s (16.3044GiB/s) | 20260505 11:52:43.756 | INFO | Downloaded 34GiB to GPU in 2.12984s (15.9637GiB/s) | 20260505 11:52:48.895 | INFO | Analyzing chunk frames = [2996224,2996736) | 20260505 11:52:48.895 | INFO | MemInfo: { | 20260505 11:52:48.895 | INFO | "gpu" : " 17 GB", | 20260505 11:52:48.895 | INFO | "huge" : "", | 20260505 11:52:48.895 | INFO | "huge_shared" : " 27 GB", | 20260505 11:52:48.895 | INFO | "malloc" : " 14 GB", | 20260505 11:52:48.895 | INFO | "pinned" : " 29 GB" | 20260505 11:52:48.895 | INFO | } | 20260505 11:52:48.895 | INFO | Perf report for the last 5.13952 seconds: | 20260505 11:52:48.895 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:52:48.895 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:48.895 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:48.895 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:48.895 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 11:52:48.895 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:48.895 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:48.895 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 11:52:48.895 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:48.895 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:48.896 | INFO | Uploaded 36GiB to GPU in 2.15196s (16.7289GiB/s) | 20260505 11:52:48.896 | INFO | Downloaded 34GiB to GPU in 2.12546s (15.9966GiB/s) | 20260505 11:52:54.028 | INFO | Analyzing chunk frames = [2996736,2997248) | 20260505 11:52:54.028 | INFO | MemInfo: { | 20260505 11:52:54.028 | INFO | "gpu" : " 17 GB", | 20260505 11:52:54.028 | INFO | "huge" : "", | 20260505 11:52:54.028 | INFO | "huge_shared" : " 27 GB", | 20260505 11:52:54.028 | INFO | "malloc" : " 14 GB", | 20260505 11:52:54.028 | INFO | "pinned" : " 29 GB" | 20260505 11:52:54.028 | INFO | } | 20260505 11:52:54.028 | INFO | Perf report for the last 5.13311 seconds: | 20260505 11:52:54.028 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:52:54.028 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:54.028 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:54.028 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:54.028 | INFO | ANALYSIS : 63.5%, 3.0, 99.0ms, 0 | 20260505 11:52:54.028 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:54.028 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:54.028 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 11:52:54.028 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:54.028 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:54.029 | INFO | Uploaded 36GiB to GPU in 2.18245s (16.4953GiB/s) | 20260505 11:52:54.029 | INFO | Downloaded 34GiB to GPU in 2.11541s (16.0725GiB/s) | 20260505 11:52:59.170 | INFO | Analyzing chunk frames = [2997248,2997760) | 20260505 11:52:59.170 | INFO | MemInfo: { | 20260505 11:52:59.170 | INFO | "gpu" : " 17 GB", | 20260505 11:52:59.170 | INFO | "huge" : "", | 20260505 11:52:59.170 | INFO | "huge_shared" : " 27 GB", | 20260505 11:52:59.170 | INFO | "malloc" : " 14 GB", | 20260505 11:52:59.170 | INFO | "pinned" : " 29 GB" | 20260505 11:52:59.170 | INFO | } | 20260505 11:52:59.170 | INFO | Perf report for the last 5.14144 seconds: | 20260505 11:52:59.170 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:52:59.170 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:59.170 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:59.170 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:59.170 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 11:52:59.170 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:59.170 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:59.170 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 11:52:59.170 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:59.170 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:52:59.170 | INFO | Uploaded 36GiB to GPU in 2.15972s (16.6689GiB/s) | 20260505 11:52:59.170 | INFO | Downloaded 34GiB to GPU in 2.14518s (15.8495GiB/s) | 20260505 11:53:04.318 | INFO | Analyzing chunk frames = [2997760,2998272) | 20260505 11:53:04.318 | INFO | MemInfo: { | 20260505 11:53:04.318 | INFO | "gpu" : " 17 GB", | 20260505 11:53:04.318 | INFO | "huge" : "", | 20260505 11:53:04.318 | INFO | "huge_shared" : " 27 GB", | 20260505 11:53:04.318 | INFO | "malloc" : " 14 GB", | 20260505 11:53:04.318 | INFO | "pinned" : " 29 GB" | 20260505 11:53:04.318 | INFO | } | 20260505 11:53:04.318 | INFO | Perf report for the last 5.14828 seconds: | 20260505 11:53:04.318 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:53:04.318 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:04.318 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:04.318 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:04.318 | INFO | ANALYSIS : 63.7%, 3.0, 99.7ms, 0 | 20260505 11:53:04.318 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:04.318 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:04.318 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 11:53:04.318 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:04.318 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:04.318 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 11:53:04.318 | INFO | Total mem usage: 15.3 GiB | 20260505 11:53:04.318 | INFO | Uploaded 36GiB to GPU in 2.22264s (16.197GiB/s) | 20260505 11:53:04.318 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 11:53:04.318 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.76%) | 20260505 11:53:04.318 | INFO | Downloaded 34GiB to GPU in 2.08462s (16.3099GiB/s) | 20260505 11:53:04.318 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 11:53:04.318 | INFO | High Water Breakdown: | 20260505 11:53:04.318 | INFO | 61.1% : 8.77 GiB ( 92.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 11:53:04.318 | INFO | 38.8% : 5.57 GiB ( 83.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 11:53:04.318 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 11:53:04.318 | INFO | ------------------------------------------ | 20260505 11:53:04.318 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:53:04.318 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 11:53:04.318 | INFO | Total mem usage: 26.5 GiB | 20260505 11:53:04.318 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 11:53:04.318 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 11:53:04.318 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 11:53:04.319 | INFO | High Water Breakdown: | 20260505 11:53:04.319 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:53:04.319 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:53:04.319 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:53:04.319 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:53:04.319 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:53:04.319 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:53:04.319 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 11:53:04.319 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 11:53:04.319 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:53:04.319 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 11:53:04.319 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 11:53:04.319 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 11:53:04.319 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 11:53:04.319 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 11:53:04.319 | INFO | ------------------------------------------ | 20260505 11:53:04.319 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:53:04.319 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 11:53:04.319 | INFO | Total mem usage: 25.6 GiB | 20260505 11:53:04.319 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 11:53:04.319 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 11:53:04.319 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 11:53:04.319 | INFO | High Water Breakdown: | 20260505 11:53:04.319 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 11:53:04.319 | INFO | ------------------------------------------ | 20260505 11:53:04.319 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:53:04.319 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 11:53:04.319 | INFO | Total mem usage: 15.7 GiB | 20260505 11:53:04.319 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 11:53:04.319 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 11:53:04.319 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 11:53:04.319 | INFO | High Water Breakdown: | 20260505 11:53:04.319 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 11:53:04.319 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 11:53:04.319 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 11:53:04.319 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 11:53:04.319 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 11:53:04.319 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 11:53:04.319 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 11:53:04.319 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 11:53:04.319 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 11:53:04.319 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:53:04.319 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 11:53:04.319 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:53:04.319 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 11:53:04.319 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 11:53:04.319 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 11:53:04.319 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 11:53:04.319 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:53:04.319 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 11:53:04.319 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:53:04.319 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:53:04.319 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:53:04.319 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 11:53:04.319 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 11:53:04.319 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:53:04.319 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 11:53:04.319 | INFO | ------------------------------------------ | 20260505 11:53:04.319 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:53:04.319 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 11:53:04.319 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:53:04.319 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:53:04.319 | INFO | PerfReport: Full Execution -- 3.21e+01 / 7.73e+01, 1.38e+01 / 1.62e+00 | 20260505 11:53:04.319 | INFO | PerfReport: 0.29%, AggregateStats -- 9.26e-02 / 1.00e-01, 1.05e-02 / 4.17e-03 | 20260505 11:53:04.319 | INFO | PerfReport: 11.76%, Baseline -- 3.77e+00 / 4.51e+00, 6.07e-01 / 2.10e-01 | 20260505 11:53:04.319 | INFO | PerfReport: 3.31%, Histogram -- 1.06e+00 / 1.49e+00, 2.04e-01 / 1.07e-01 | 20260505 11:53:04.319 | INFO | PerfReport: 15.86%, DME -- 4.07e+01 / 4.51e+01, 4.34e+00 / 1.37e+00 | 20260505 11:53:04.319 | INFO | PerfReport: 56.43%, FrameLabeling -- 1.81e+01 / 2.13e+01, 1.56e+00 / 1.59e+00 | 20260505 11:53:04.319 | INFO | PerfReport: 7.41%, PulseAccumulating -- 2.38e+00 / 2.84e+00, 4.29e-01 / 2.06e-01 | 20260505 11:53:04.319 | INFO | PerfReport: 3.66%, Metrics -- 1.18e+00 / 1.45e+00, 3.34e-01 / 2.06e-01 | 20260505 11:53:04.319 | INFO | PerfReport: 1.29%, PulseSerializing -- 4.13e-01 / 4.58e-01, 7.54e-02 / 3.91e-02 | 20260505 11:53:04.319 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 11:53:04.319 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:53:04.319 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:53:04.319 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260505 11:53:04.319 | INFO | PerfReport: 22.59%, Upload -- 2.28e+01 / 0.00e+00, 8.15e+00 / 0.00e+00 | 20260505 11:53:04.319 | INFO | PerfReport: 22.04%, Download -- 2.23e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260505 11:53:04.319 | INFO | PerfReport: 31.73%, Compute -- 3.21e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 11:53:04.319 | INFO | PerfReport: 2.06%, LockUpload -- 2.08e+00 / 0.00e+00, 7.59e+00 / 0.00e+00 | 20260505 11:53:04.319 | INFO | PerfReport: 1.30%, LockDownload -- 1.32e+00 / 0.00e+00, 2.84e+00 / 0.00e+00 | 20260505 11:53:04.319 | INFO | PerfReport: 20.27%, LockCompute -- 2.05e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260505 11:53:04.319 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.83e-02 / 0.00e+00 | 20260505 11:53:04.449 | INFO | Starting a round of Baz IO | 20260505 11:53:07.699 | INFO | Populating RTMetrics report | 20260505 11:53:07.699 | INFO | RTMetrics report populated | 20260505 11:53:07.699 | INFO | Opening tmp RTMetrics json file | 20260505 11:53:07.700 | INFO | Serializing RTMetrics to json | 20260505 11:53:07.701 | INFO | Serialization done, swapping tmp file now... | 20260505 11:53:07.701 | INFO | RTMetrics done | 20260505 11:53:07.817 | INFO | Finished a round of Baz IO: Wrote 2.359 GiB in 3.368 seconds | 20260505 11:53:07.817 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.90% / 0.01% / 76.10% | 20260505 11:53:07.817 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.01801% / 7.94729e-06% / 4.76837e-05% | 20260505 11:53:07.817 | INFO | Baz IO overall: | 20260505 11:53:07.817 | INFO | BytesWritten : 1777224960847 | 20260505 11:53:07.817 | INFO | NumEvents : 846116698942 | 20260505 11:53:09.458 | INFO | Analyzing chunk frames = [2998272,2998784) | 20260505 11:53:09.458 | INFO | MemInfo: { | 20260505 11:53:09.458 | INFO | "gpu" : " 17 GB", | 20260505 11:53:09.458 | INFO | "huge" : "", | 20260505 11:53:09.458 | INFO | "huge_shared" : " 27 GB", | 20260505 11:53:09.458 | INFO | "malloc" : " 14 GB", | 20260505 11:53:09.458 | INFO | "pinned" : " 29 GB" | 20260505 11:53:09.458 | INFO | } | 20260505 11:53:09.458 | INFO | Perf report for the last 5.14006 seconds: | 20260505 11:53:09.458 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:53:09.458 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:09.458 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:09.458 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:09.458 | INFO | ANALYSIS : 64.7%, 3.0, 100.1ms, 0 | 20260505 11:53:09.458 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:09.458 | INFO | RT_METRICS : 45.3%, 1.0, 23.8ms, 0 | 20260505 11:53:09.458 | INFO | PRE_HQ : 61.3%, 1.8, 58.7ms, 0 | 20260505 11:53:09.458 | INFO | BAZWRITER : 64.8%, 4.1, 136.3ms, 0 | 20260505 11:53:09.458 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 11:53:09.458 | INFO | Uploaded 36GiB to GPU in 2.18946s (16.4424GiB/s) | 20260505 11:53:09.458 | INFO | Downloaded 38GiB to GPU in 2.27693s (16.6891GiB/s) | 20260505 11:53:14.594 | INFO | Analyzing chunk frames = [2998784,2999296) | 20260505 11:53:14.594 | INFO | MemInfo: { | 20260505 11:53:14.594 | INFO | "gpu" : " 17 GB", | 20260505 11:53:14.594 | INFO | "huge" : "", | 20260505 11:53:14.594 | INFO | "huge_shared" : " 27 GB", | 20260505 11:53:14.594 | INFO | "malloc" : " 14 GB", | 20260505 11:53:14.594 | INFO | "pinned" : " 29 GB" | 20260505 11:53:14.594 | INFO | } | 20260505 11:53:14.594 | INFO | Perf report for the last 5.13568 seconds: | 20260505 11:53:14.594 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:53:14.594 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:14.594 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:14.594 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:14.594 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 11:53:14.594 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:14.594 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:14.594 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:53:14.594 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:14.594 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:14.594 | INFO | Uploaded 36GiB to GPU in 2.18212s (16.4977GiB/s) | 20260505 11:53:14.594 | INFO | Downloaded 34GiB to GPU in 2.05418s (16.5516GiB/s) | 20260505 11:53:19.723 | INFO | Analyzing chunk frames = [2999296,2999808) | 20260505 11:53:19.723 | INFO | MemInfo: { | 20260505 11:53:19.723 | INFO | "gpu" : " 17 GB", | 20260505 11:53:19.724 | INFO | "huge" : "", | 20260505 11:53:19.724 | INFO | "huge_shared" : " 27 GB", | 20260505 11:53:19.724 | INFO | "malloc" : " 14 GB", | 20260505 11:53:19.724 | INFO | "pinned" : " 29 GB" | 20260505 11:53:19.724 | INFO | } | 20260505 11:53:19.724 | INFO | Perf report for the last 5.12987 seconds: | 20260505 11:53:19.724 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:53:19.724 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:19.724 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:19.724 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:19.724 | INFO | ANALYSIS : 63.5%, 3.0, 98.6ms, 0 | 20260505 11:53:19.724 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:19.724 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:19.724 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 11:53:19.724 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:19.724 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:19.724 | INFO | Uploaded 36GiB to GPU in 2.22478s (16.1814GiB/s) | 20260505 11:53:19.724 | INFO | Downloaded 34GiB to GPU in 2.12949s (15.9663GiB/s) | 20260505 11:53:24.869 | INFO | Analyzing chunk frames = [2999808,3000320) | 20260505 11:53:24.869 | INFO | MemInfo: { | 20260505 11:53:24.869 | INFO | "gpu" : " 17 GB", | 20260505 11:53:24.869 | INFO | "huge" : "", | 20260505 11:53:24.869 | INFO | "huge_shared" : " 27 GB", | 20260505 11:53:24.869 | INFO | "malloc" : " 14 GB", | 20260505 11:53:24.869 | INFO | "pinned" : " 29 GB" | 20260505 11:53:24.869 | INFO | } | 20260505 11:53:24.869 | INFO | Perf report for the last 5.14522 seconds: | 20260505 11:53:24.869 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:53:24.869 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:24.869 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:24.869 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:24.869 | INFO | ANALYSIS : 63.4%, 3.0, 98.6ms, 0 | 20260505 11:53:24.869 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:24.869 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:24.869 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 11:53:24.869 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:24.869 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:24.869 | INFO | Uploaded 36GiB to GPU in 2.20816s (16.3031GiB/s) | 20260505 11:53:24.869 | INFO | Downloaded 34GiB to GPU in 2.1225s (16.0189GiB/s) | 20260505 11:53:30.008 | INFO | Analyzing chunk frames = [3000320,3000832) | 20260505 11:53:30.008 | INFO | MemInfo: { | 20260505 11:53:30.008 | INFO | "gpu" : " 17 GB", | 20260505 11:53:30.008 | INFO | "huge" : "", | 20260505 11:53:30.008 | INFO | "huge_shared" : " 27 GB", | 20260505 11:53:30.008 | INFO | "malloc" : " 14 GB", | 20260505 11:53:30.008 | INFO | "pinned" : " 29 GB" | 20260505 11:53:30.008 | INFO | } | 20260505 11:53:30.008 | INFO | Perf report for the last 5.13901 seconds: | 20260505 11:53:30.008 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:53:30.008 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:30.008 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:30.008 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:30.008 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 11:53:30.008 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:30.008 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:30.008 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 11:53:30.008 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:30.008 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:30.008 | INFO | Uploaded 36GiB to GPU in 2.14694s (16.7681GiB/s) | 20260505 11:53:30.008 | INFO | Downloaded 34GiB to GPU in 2.12207s (16.0221GiB/s) | 20260505 11:53:35.156 | INFO | Analyzing chunk frames = [3000832,3001344) | 20260505 11:53:35.156 | INFO | MemInfo: { | 20260505 11:53:35.156 | INFO | "gpu" : " 17 GB", | 20260505 11:53:35.156 | INFO | "huge" : "", | 20260505 11:53:35.156 | INFO | "huge_shared" : " 27 GB", | 20260505 11:53:35.156 | INFO | "malloc" : " 14 GB", | 20260505 11:53:35.156 | INFO | "pinned" : " 29 GB" | 20260505 11:53:35.156 | INFO | } | 20260505 11:53:35.156 | INFO | Perf report for the last 5.1482 seconds: | 20260505 11:53:35.156 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:53:35.156 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:35.156 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:35.156 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:35.156 | INFO | ANALYSIS : 63.1%, 3.0, 98.7ms, 0 | 20260505 11:53:35.156 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:35.156 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:35.156 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 11:53:35.156 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:35.156 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:35.156 | INFO | Uploaded 36GiB to GPU in 2.12718s (16.9238GiB/s) | 20260505 11:53:35.156 | INFO | Downloaded 34GiB to GPU in 2.10867s (16.1239GiB/s) | 20260505 11:53:40.292 | INFO | Analyzing chunk frames = [3001344,3001856) | 20260505 11:53:40.292 | INFO | MemInfo: { | 20260505 11:53:40.292 | INFO | "gpu" : " 17 GB", | 20260505 11:53:40.292 | INFO | "huge" : "", | 20260505 11:53:40.292 | INFO | "huge_shared" : " 27 GB", | 20260505 11:53:40.292 | INFO | "malloc" : " 14 GB", | 20260505 11:53:40.292 | INFO | "pinned" : " 29 GB" | 20260505 11:53:40.292 | INFO | } | 20260505 11:53:40.292 | INFO | Perf report for the last 5.13595 seconds: | 20260505 11:53:40.292 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:53:40.292 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:40.292 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:40.292 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:40.292 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 11:53:40.292 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:40.292 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:40.292 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 11:53:40.292 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:40.292 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:40.292 | INFO | Uploaded 36GiB to GPU in 2.16158s (16.6545GiB/s) | 20260505 11:53:40.292 | INFO | Downloaded 34GiB to GPU in 2.1486s (15.8242GiB/s) | 20260505 11:53:45.426 | INFO | Analyzing chunk frames = [3001856,3002368) | 20260505 11:53:45.426 | INFO | MemInfo: { | 20260505 11:53:45.426 | INFO | "gpu" : " 17 GB", | 20260505 11:53:45.426 | INFO | "huge" : "", | 20260505 11:53:45.426 | INFO | "huge_shared" : " 27 GB", | 20260505 11:53:45.426 | INFO | "malloc" : " 14 GB", | 20260505 11:53:45.426 | INFO | "pinned" : " 29 GB" | 20260505 11:53:45.426 | INFO | } | 20260505 11:53:45.426 | INFO | Perf report for the last 5.13401 seconds: | 20260505 11:53:45.426 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:53:45.426 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:45.426 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:45.426 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:45.426 | INFO | ANALYSIS : 63.8%, 3.0, 99.5ms, 0 | 20260505 11:53:45.426 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:45.426 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:45.426 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 11:53:45.426 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:45.426 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:45.426 | INFO | Uploaded 36GiB to GPU in 2.21083s (16.2835GiB/s) | 20260505 11:53:45.426 | INFO | Downloaded 34GiB to GPU in 2.09709s (16.2129GiB/s) | 20260505 11:53:45.560 | INFO | Starting a round of Baz IO | 20260505 11:53:48.810 | INFO | Populating RTMetrics report | 20260505 11:53:48.810 | INFO | RTMetrics report populated | 20260505 11:53:48.810 | INFO | Opening tmp RTMetrics json file | 20260505 11:53:48.845 | INFO | Serializing RTMetrics to json | 20260505 11:53:48.853 | INFO | Serialization done, swapping tmp file now... | 20260505 11:53:48.870 | INFO | RTMetrics done | 20260505 11:53:48.935 | INFO | Finished a round of Baz IO: Wrote 2.357 GiB in 3.375 seconds | 20260505 11:53:48.935 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.89% / 0.01% / 76.10% | 20260505 11:53:48.935 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.01433% / 1.19209e-05% / 4.76837e-05% | 20260505 11:53:48.935 | INFO | Baz IO overall: | 20260505 11:53:48.935 | INFO | BytesWritten : 1779755810306 | 20260505 11:53:48.935 | INFO | NumEvents : 847318055592 | 20260505 11:53:50.570 | INFO | Analyzing chunk frames = [3002368,3002880) | 20260505 11:53:50.570 | INFO | MemInfo: { | 20260505 11:53:50.570 | INFO | "gpu" : " 17 GB", | 20260505 11:53:50.570 | INFO | "huge" : "", | 20260505 11:53:50.570 | INFO | "huge_shared" : " 27 GB", | 20260505 11:53:50.570 | INFO | "malloc" : " 14 GB", | 20260505 11:53:50.570 | INFO | "pinned" : " 29 GB" | 20260505 11:53:50.570 | INFO | } | 20260505 11:53:50.570 | INFO | Perf report for the last 5.14377 seconds: | 20260505 11:53:50.570 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:53:50.570 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:50.570 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:50.570 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:50.570 | INFO | ANALYSIS : 64.7%, 2.9, 99.9ms, 0 | 20260505 11:53:50.570 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:50.570 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260505 11:53:50.570 | INFO | PRE_HQ : 61.1%, 1.8, 58.4ms, 0 | 20260505 11:53:50.570 | INFO | BAZWRITER : 64.9%, 4.1, 136.7ms, 0 | 20260505 11:53:50.570 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 11:53:50.570 | INFO | Uploaded 36GiB to GPU in 2.19367s (16.4109GiB/s) | 20260505 11:53:50.570 | INFO | Downloaded 38GiB to GPU in 2.29395s (16.5653GiB/s) | 20260505 11:53:55.703 | INFO | Analyzing chunk frames = [3002880,3003392) | 20260505 11:53:55.703 | INFO | MemInfo: { | 20260505 11:53:55.703 | INFO | "gpu" : " 17 GB", | 20260505 11:53:55.703 | INFO | "huge" : "", | 20260505 11:53:55.703 | INFO | "huge_shared" : " 27 GB", | 20260505 11:53:55.703 | INFO | "malloc" : " 14 GB", | 20260505 11:53:55.703 | INFO | "pinned" : " 29 GB" | 20260505 11:53:55.703 | INFO | } | 20260505 11:53:55.703 | INFO | Perf report for the last 5.1333 seconds: | 20260505 11:53:55.703 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:53:55.703 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:55.703 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:55.703 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:55.703 | INFO | ANALYSIS : 63.8%, 3.0, 99.5ms, 0 | 20260505 11:53:55.703 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:55.703 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:55.703 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 11:53:55.703 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:55.703 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:53:55.704 | INFO | Uploaded 36GiB to GPU in 2.19728s (16.3839GiB/s) | 20260505 11:53:55.704 | INFO | Downloaded 34GiB to GPU in 2.04271s (16.6446GiB/s) | 20260505 11:54:00.846 | INFO | Analyzing chunk frames = [3003392,3003904) | 20260505 11:54:00.847 | INFO | MemInfo: { | 20260505 11:54:00.847 | INFO | "gpu" : " 17 GB", | 20260505 11:54:00.847 | INFO | "huge" : "", | 20260505 11:54:00.847 | INFO | "huge_shared" : " 27 GB", | 20260505 11:54:00.847 | INFO | "malloc" : " 14 GB", | 20260505 11:54:00.847 | INFO | "pinned" : " 29 GB" | 20260505 11:54:00.847 | INFO | } | 20260505 11:54:00.847 | INFO | Perf report for the last 5.1434 seconds: | 20260505 11:54:00.847 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:54:00.847 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:00.847 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:00.847 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:00.847 | INFO | ANALYSIS : 63.4%, 3.0, 98.8ms, 0 | 20260505 11:54:00.847 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:00.847 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:00.847 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 11:54:00.847 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:00.847 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:00.847 | INFO | Uploaded 36GiB to GPU in 2.22883s (16.152GiB/s) | 20260505 11:54:00.847 | INFO | Downloaded 34GiB to GPU in 2.13029s (15.9603GiB/s) | 20260505 11:54:05.976 | INFO | Analyzing chunk frames = [3003904,3004416) | 20260505 11:54:05.976 | INFO | MemInfo: { | 20260505 11:54:05.976 | INFO | "gpu" : " 17 GB", | 20260505 11:54:05.976 | INFO | "huge" : "", | 20260505 11:54:05.976 | INFO | "huge_shared" : " 27 GB", | 20260505 11:54:05.976 | INFO | "malloc" : " 14 GB", | 20260505 11:54:05.976 | INFO | "pinned" : " 29 GB" | 20260505 11:54:05.976 | INFO | } | 20260505 11:54:05.976 | INFO | Perf report for the last 5.12962 seconds: | 20260505 11:54:05.976 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:54:05.976 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:05.976 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:05.976 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:05.976 | INFO | ANALYSIS : 63.6%, 3.0, 98.6ms, 0 | 20260505 11:54:05.976 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:05.976 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:05.976 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:54:05.976 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:05.976 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:05.976 | INFO | Uploaded 36GiB to GPU in 2.21792s (16.2314GiB/s) | 20260505 11:54:05.977 | INFO | Downloaded 34GiB to GPU in 2.13367s (15.935GiB/s) | 20260505 11:54:11.127 | INFO | Analyzing chunk frames = [3004416,3004928) | 20260505 11:54:11.127 | INFO | MemInfo: { | 20260505 11:54:11.127 | INFO | "gpu" : " 17 GB", | 20260505 11:54:11.127 | INFO | "huge" : "", | 20260505 11:54:11.127 | INFO | "huge_shared" : " 27 GB", | 20260505 11:54:11.127 | INFO | "malloc" : " 14 GB", | 20260505 11:54:11.127 | INFO | "pinned" : " 29 GB" | 20260505 11:54:11.127 | INFO | } | 20260505 11:54:11.128 | INFO | Perf report for the last 5.15124 seconds: | 20260505 11:54:11.128 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:54:11.128 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:11.128 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:11.128 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:11.128 | INFO | ANALYSIS : 63.1%, 3.0, 98.8ms, 0 | 20260505 11:54:11.128 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:11.128 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:11.128 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 11:54:11.128 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:11.128 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:11.128 | INFO | Uploaded 36GiB to GPU in 2.16252s (16.6473GiB/s) | 20260505 11:54:11.128 | INFO | Downloaded 34GiB to GPU in 2.12463s (16.0028GiB/s) | 20260505 11:54:16.267 | INFO | Analyzing chunk frames = [3004928,3005440) | 20260505 11:54:16.267 | INFO | MemInfo: { | 20260505 11:54:16.267 | INFO | "gpu" : " 17 GB", | 20260505 11:54:16.267 | INFO | "huge" : "", | 20260505 11:54:16.267 | INFO | "huge_shared" : " 27 GB", | 20260505 11:54:16.267 | INFO | "malloc" : " 14 GB", | 20260505 11:54:16.267 | INFO | "pinned" : " 29 GB" | 20260505 11:54:16.267 | INFO | } | 20260505 11:54:16.267 | INFO | Perf report for the last 5.13943 seconds: | 20260505 11:54:16.267 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:54:16.267 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:16.267 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:16.267 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:16.267 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 11:54:16.267 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:16.267 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:16.267 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 11:54:16.267 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:16.267 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:16.267 | INFO | Uploaded 36GiB to GPU in 2.16532s (16.6257GiB/s) | 20260505 11:54:16.267 | INFO | Downloaded 34GiB to GPU in 2.09519s (16.2276GiB/s) | 20260505 11:54:21.406 | INFO | Analyzing chunk frames = [3005440,3005952) | 20260505 11:54:21.406 | INFO | MemInfo: { | 20260505 11:54:21.406 | INFO | "gpu" : " 17 GB", | 20260505 11:54:21.406 | INFO | "huge" : "", | 20260505 11:54:21.406 | INFO | "huge_shared" : " 27 GB", | 20260505 11:54:21.406 | INFO | "malloc" : " 14 GB", | 20260505 11:54:21.406 | INFO | "pinned" : " 29 GB" | 20260505 11:54:21.406 | INFO | } | 20260505 11:54:21.406 | INFO | Perf report for the last 5.13916 seconds: | 20260505 11:54:21.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:54:21.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:21.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:21.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:21.406 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 11:54:21.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:21.406 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:21.406 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 11:54:21.406 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:21.406 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:21.406 | INFO | Uploaded 36GiB to GPU in 2.15153s (16.7323GiB/s) | 20260505 11:54:21.406 | INFO | Downloaded 34GiB to GPU in 2.13169s (15.9498GiB/s) | 20260505 11:54:26.536 | INFO | Analyzing chunk frames = [3005952,3006464) | 20260505 11:54:26.536 | INFO | MemInfo: { | 20260505 11:54:26.536 | INFO | "gpu" : " 17 GB", | 20260505 11:54:26.536 | INFO | "huge" : "", | 20260505 11:54:26.536 | INFO | "huge_shared" : " 27 GB", | 20260505 11:54:26.536 | INFO | "malloc" : " 14 GB", | 20260505 11:54:26.536 | INFO | "pinned" : " 29 GB" | 20260505 11:54:26.536 | INFO | } | 20260505 11:54:26.536 | INFO | Perf report for the last 5.12972 seconds: | 20260505 11:54:26.536 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:54:26.536 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:26.536 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:26.536 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:26.536 | INFO | ANALYSIS : 63.9%, 3.0, 99.6ms, 0 | 20260505 11:54:26.536 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:26.536 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:26.536 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 11:54:26.536 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:26.536 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:26.536 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 11:54:26.536 | INFO | Total mem usage: 15.3 GiB | 20260505 11:54:26.536 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 11:54:26.536 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -6.83%) | 20260505 11:54:26.536 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 11:54:26.536 | INFO | High Water Breakdown: | 20260505 11:54:26.536 | INFO | 61.1% : 8.77 GiB ( 92.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 11:54:26.536 | INFO | 38.7% : 5.56 GiB ( 83.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 11:54:26.536 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 11:54:26.536 | INFO | ------------------------------------------ | 20260505 11:54:26.536 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:54:26.536 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 11:54:26.536 | INFO | Total mem usage: 26.5 GiB | 20260505 11:54:26.536 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 11:54:26.536 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 11:54:26.536 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 11:54:26.536 | INFO | High Water Breakdown: | 20260505 11:54:26.536 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:54:26.536 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:54:26.536 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:54:26.536 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:54:26.536 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:54:26.536 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:54:26.536 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 11:54:26.536 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 11:54:26.536 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:54:26.536 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 11:54:26.536 | INFO | Uploaded 36GiB to GPU in 2.22095s (16.2093GiB/s) | 20260505 11:54:26.536 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 11:54:26.536 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 11:54:26.536 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 11:54:26.536 | INFO | Downloaded 34GiB to GPU in 2.11078s (16.1078GiB/s) | 20260505 11:54:26.536 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 11:54:26.536 | INFO | ------------------------------------------ | 20260505 11:54:26.536 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:54:26.537 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 11:54:26.537 | INFO | Total mem usage: 25.6 GiB | 20260505 11:54:26.537 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 11:54:26.537 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 11:54:26.537 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 11:54:26.537 | INFO | High Water Breakdown: | 20260505 11:54:26.537 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 11:54:26.537 | INFO | ------------------------------------------ | 20260505 11:54:26.537 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:54:26.537 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 11:54:26.537 | INFO | Total mem usage: 15.7 GiB | 20260505 11:54:26.537 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 11:54:26.537 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 11:54:26.537 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 11:54:26.537 | INFO | High Water Breakdown: | 20260505 11:54:26.537 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 11:54:26.537 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 11:54:26.537 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 11:54:26.537 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 11:54:26.537 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 11:54:26.537 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 11:54:26.537 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 11:54:26.537 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 11:54:26.537 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 11:54:26.537 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:54:26.537 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 11:54:26.537 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:54:26.537 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 11:54:26.537 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 11:54:26.537 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 11:54:26.537 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 11:54:26.537 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:54:26.537 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 11:54:26.537 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:54:26.537 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:54:26.537 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:54:26.537 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 11:54:26.537 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 11:54:26.537 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:54:26.537 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 11:54:26.537 | INFO | ------------------------------------------ | 20260505 11:54:26.537 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:54:26.537 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 11:54:26.537 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:54:26.537 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:54:26.537 | INFO | PerfReport: Full Execution -- 3.21e+01 / 7.72e+01, 1.38e+01 / 1.53e+00 | 20260505 11:54:26.537 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 1.02e-01, 1.01e-02 / 6.73e-03 | 20260505 11:54:26.537 | INFO | PerfReport: 11.74%, Baseline -- 3.77e+00 / 4.52e+00, 6.09e-01 / 2.00e-01 | 20260505 11:54:26.537 | INFO | PerfReport: 3.30%, Histogram -- 1.06e+00 / 1.51e+00, 2.05e-01 / 8.96e-02 | 20260505 11:54:26.537 | INFO | PerfReport: 15.75%, DME -- 4.05e+01 / 4.49e+01, 4.22e+00 / 1.91e+00 | 20260505 11:54:26.537 | INFO | PerfReport: 56.62%, FrameLabeling -- 1.82e+01 / 2.14e+01, 1.68e+00 / 1.97e+00 | 20260505 11:54:26.537 | INFO | PerfReport: 7.35%, PulseAccumulating -- 2.36e+00 / 2.83e+00, 4.22e-01 / 2.22e-01 | 20260505 11:54:26.537 | INFO | PerfReport: 3.65%, Metrics -- 1.17e+00 / 1.49e+00, 3.36e-01 / 1.87e-01 | 20260505 11:54:26.537 | INFO | PerfReport: 1.29%, PulseSerializing -- 4.13e-01 / 4.67e-01, 7.68e-02 / 3.02e-02 | 20260505 11:54:26.537 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 11:54:26.537 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:54:26.537 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:54:26.537 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260505 11:54:26.537 | INFO | PerfReport: 22.53%, Upload -- 2.28e+01 / 0.00e+00, 8.14e+00 / 0.00e+00 | 20260505 11:54:26.537 | INFO | PerfReport: 21.97%, Download -- 2.22e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260505 11:54:26.537 | INFO | PerfReport: 31.80%, Compute -- 3.21e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 11:54:26.537 | INFO | PerfReport: 2.02%, LockUpload -- 2.04e+00 / 0.00e+00, 7.63e+00 / 0.00e+00 | 20260505 11:54:26.537 | INFO | PerfReport: 1.30%, LockDownload -- 1.31e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260505 11:54:26.537 | INFO | PerfReport: 20.37%, LockCompute -- 2.06e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260505 11:54:26.537 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.84e-02 / 0.00e+00 | 20260505 11:54:26.677 | INFO | Starting a round of Baz IO | 20260505 11:54:29.929 | INFO | Populating RTMetrics report | 20260505 11:54:29.930 | INFO | RTMetrics report populated | 20260505 11:54:29.930 | INFO | Opening tmp RTMetrics json file | 20260505 11:54:29.938 | INFO | Serializing RTMetrics to json | 20260505 11:54:29.986 | INFO | Serialization done, swapping tmp file now... | 20260505 11:54:30.009 | INFO | RTMetrics done | 20260505 11:54:30.048 | INFO | Finished a round of Baz IO: Wrote 2.36 GiB in 3.371 seconds | 20260505 11:54:30.048 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.88% / 0.01% / 76.11% | 20260505 11:54:30.048 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.0145% / 2.38419e-05% / 7.15256e-05% | 20260505 11:54:30.048 | INFO | Baz IO overall: | 20260505 11:54:30.048 | INFO | BytesWritten : 1782289596013 | 20260505 11:54:30.048 | INFO | NumEvents : 848521969574 | 20260505 11:54:31.676 | INFO | Analyzing chunk frames = [3006464,3006976) | 20260505 11:54:31.676 | INFO | MemInfo: { | 20260505 11:54:31.676 | INFO | "gpu" : " 17 GB", | 20260505 11:54:31.676 | INFO | "huge" : "", | 20260505 11:54:31.676 | INFO | "huge_shared" : " 27 GB", | 20260505 11:54:31.676 | INFO | "malloc" : " 14 GB", | 20260505 11:54:31.676 | INFO | "pinned" : " 29 GB" | 20260505 11:54:31.676 | INFO | } | 20260505 11:54:31.676 | INFO | Perf report for the last 5.14024 seconds: | 20260505 11:54:31.676 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:54:31.676 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:31.676 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:31.676 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:31.676 | INFO | ANALYSIS : 64.9%, 2.9, 100.0ms, 0 | 20260505 11:54:31.676 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:31.676 | INFO | RT_METRICS : 46.9%, 1.0, 24.6ms, 0 | 20260505 11:54:31.676 | INFO | PRE_HQ : 61.3%, 1.8, 58.3ms, 0 | 20260505 11:54:31.676 | INFO | BAZWRITER : 64.9%, 4.1, 136.6ms, 0 | 20260505 11:54:31.676 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 11:54:31.676 | INFO | Uploaded 36GiB to GPU in 2.19325s (16.414GiB/s) | 20260505 11:54:31.676 | INFO | Downloaded 38GiB to GPU in 2.29515s (16.5566GiB/s) | 20260505 11:54:36.821 | INFO | Analyzing chunk frames = [3006976,3007488) | 20260505 11:54:36.821 | INFO | MemInfo: { | 20260505 11:54:36.821 | INFO | "gpu" : " 17 GB", | 20260505 11:54:36.821 | INFO | "huge" : "", | 20260505 11:54:36.821 | INFO | "huge_shared" : " 27 GB", | 20260505 11:54:36.821 | INFO | "malloc" : " 14 GB", | 20260505 11:54:36.822 | INFO | "pinned" : " 29 GB" | 20260505 11:54:36.822 | INFO | } | 20260505 11:54:36.822 | INFO | Perf report for the last 5.14547 seconds: | 20260505 11:54:36.822 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:54:36.822 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:36.822 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:36.822 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:36.822 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 11:54:36.822 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:36.822 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:36.822 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 11:54:36.822 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:36.822 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:36.822 | INFO | Uploaded 36GiB to GPU in 2.23001s (16.1435GiB/s) | 20260505 11:54:36.822 | INFO | Downloaded 34GiB to GPU in 2.08219s (16.3289GiB/s) | 20260505 11:54:41.963 | INFO | Analyzing chunk frames = [3007488,3008000) | 20260505 11:54:41.963 | INFO | MemInfo: { | 20260505 11:54:41.963 | INFO | "gpu" : " 17 GB", | 20260505 11:54:41.963 | INFO | "huge" : "", | 20260505 11:54:41.963 | INFO | "huge_shared" : " 27 GB", | 20260505 11:54:41.963 | INFO | "malloc" : " 14 GB", | 20260505 11:54:41.963 | INFO | "pinned" : " 29 GB" | 20260505 11:54:41.963 | INFO | } | 20260505 11:54:41.963 | INFO | Perf report for the last 5.14175 seconds: | 20260505 11:54:41.963 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:54:41.963 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:41.963 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:41.963 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:41.963 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 11:54:41.963 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:41.963 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:41.963 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 11:54:41.963 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:41.963 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:41.964 | INFO | Uploaded 36GiB to GPU in 2.21574s (16.2474GiB/s) | 20260505 11:54:41.964 | INFO | Downloaded 34GiB to GPU in 2.12364s (16.0102GiB/s) | 20260505 11:54:47.099 | INFO | Analyzing chunk frames = [3008000,3008512) | 20260505 11:54:47.099 | INFO | MemInfo: { | 20260505 11:54:47.099 | INFO | "gpu" : " 17 GB", | 20260505 11:54:47.100 | INFO | "huge" : "", | 20260505 11:54:47.100 | INFO | "huge_shared" : " 27 GB", | 20260505 11:54:47.100 | INFO | "malloc" : " 14 GB", | 20260505 11:54:47.100 | INFO | "pinned" : " 29 GB" | 20260505 11:54:47.100 | INFO | } | 20260505 11:54:47.100 | INFO | Perf report for the last 5.13632 seconds: | 20260505 11:54:47.100 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:54:47.100 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:47.100 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:47.100 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:47.100 | INFO | ANALYSIS : 63.2%, 3.0, 98.1ms, 0 | 20260505 11:54:47.100 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:47.100 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:47.100 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:54:47.100 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:47.100 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:47.100 | INFO | Uploaded 36GiB to GPU in 2.20756s (16.3076GiB/s) | 20260505 11:54:47.100 | INFO | Downloaded 34GiB to GPU in 2.13076s (15.9567GiB/s) | 20260505 11:54:52.239 | INFO | Analyzing chunk frames = [3008512,3009024) | 20260505 11:54:52.239 | INFO | MemInfo: { | 20260505 11:54:52.239 | INFO | "gpu" : " 17 GB", | 20260505 11:54:52.239 | INFO | "huge" : "", | 20260505 11:54:52.239 | INFO | "huge_shared" : " 27 GB", | 20260505 11:54:52.239 | INFO | "malloc" : " 14 GB", | 20260505 11:54:52.239 | INFO | "pinned" : " 29 GB" | 20260505 11:54:52.239 | INFO | } | 20260505 11:54:52.239 | INFO | Perf report for the last 5.13972 seconds: | 20260505 11:54:52.239 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:54:52.239 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:52.239 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:52.239 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:52.239 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 11:54:52.239 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:52.239 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:52.240 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 11:54:52.240 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:52.240 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:52.240 | INFO | Uploaded 36GiB to GPU in 2.18668s (16.4633GiB/s) | 20260505 11:54:52.240 | INFO | Downloaded 34GiB to GPU in 2.12933s (15.9675GiB/s) | 20260505 11:54:57.381 | INFO | Analyzing chunk frames = [3009024,3009536) | 20260505 11:54:57.381 | INFO | MemInfo: { | 20260505 11:54:57.381 | INFO | "gpu" : " 17 GB", | 20260505 11:54:57.381 | INFO | "huge" : "", | 20260505 11:54:57.381 | INFO | "huge_shared" : " 27 GB", | 20260505 11:54:57.381 | INFO | "malloc" : " 14 GB", | 20260505 11:54:57.381 | INFO | "pinned" : " 29 GB" | 20260505 11:54:57.381 | INFO | } | 20260505 11:54:57.381 | INFO | Perf report for the last 5.14196 seconds: | 20260505 11:54:57.381 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:54:57.381 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:57.381 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:57.381 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:57.381 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 11:54:57.381 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:57.381 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:57.381 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 11:54:57.381 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:57.382 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:54:57.382 | INFO | Uploaded 36GiB to GPU in 2.17567s (16.5466GiB/s) | 20260505 11:54:57.382 | INFO | Downloaded 34GiB to GPU in 2.11471s (16.0779GiB/s) | 20260505 11:55:02.515 | INFO | Analyzing chunk frames = [3009536,3010048) | 20260505 11:55:02.515 | INFO | MemInfo: { | 20260505 11:55:02.515 | INFO | "gpu" : " 17 GB", | 20260505 11:55:02.515 | INFO | "huge" : "", | 20260505 11:55:02.515 | INFO | "huge_shared" : " 27 GB", | 20260505 11:55:02.515 | INFO | "malloc" : " 14 GB", | 20260505 11:55:02.515 | INFO | "pinned" : " 29 GB" | 20260505 11:55:02.515 | INFO | } | 20260505 11:55:02.515 | INFO | Perf report for the last 5.1332 seconds: | 20260505 11:55:02.515 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:55:02.515 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:02.515 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:02.515 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:02.515 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 11:55:02.515 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:02.515 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:02.515 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 11:55:02.515 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:02.515 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:02.515 | INFO | Uploaded 36GiB to GPU in 2.13898s (16.8304GiB/s) | 20260505 11:55:02.515 | INFO | Downloaded 34GiB to GPU in 2.12967s (15.9649GiB/s) | 20260505 11:55:07.655 | INFO | Analyzing chunk frames = [3010048,3010560) | 20260505 11:55:07.655 | INFO | MemInfo: { | 20260505 11:55:07.655 | INFO | "gpu" : " 17 GB", | 20260505 11:55:07.655 | INFO | "huge" : "", | 20260505 11:55:07.655 | INFO | "huge_shared" : " 27 GB", | 20260505 11:55:07.655 | INFO | "malloc" : " 14 GB", | 20260505 11:55:07.655 | INFO | "pinned" : " 29 GB" | 20260505 11:55:07.655 | INFO | } | 20260505 11:55:07.655 | INFO | Perf report for the last 5.14024 seconds: | 20260505 11:55:07.655 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:55:07.655 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:07.655 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:07.655 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:07.655 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 11:55:07.655 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:07.655 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:07.655 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 11:55:07.655 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:07.655 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:07.655 | INFO | Uploaded 36GiB to GPU in 2.21185s (16.276GiB/s) | 20260505 11:55:07.655 | INFO | Downloaded 34GiB to GPU in 2.08413s (16.3138GiB/s) | 20260505 11:55:07.792 | INFO | Starting a round of Baz IO | 20260505 11:55:11.051 | INFO | Populating RTMetrics report | 20260505 11:55:11.052 | INFO | RTMetrics report populated | 20260505 11:55:11.052 | INFO | Opening tmp RTMetrics json file | 20260505 11:55:11.053 | INFO | Serializing RTMetrics to json | 20260505 11:55:11.056 | INFO | Serialization done, swapping tmp file now... | 20260505 11:55:11.057 | INFO | RTMetrics done | 20260505 11:55:11.179 | INFO | Finished a round of Baz IO: Wrote 2.358 GiB in 3.387 seconds | 20260505 11:55:11.179 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.88% / 0.01% / 76.12% | 20260505 11:55:11.179 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.00694% / 7.94729e-06% / 6.75519e-05% | 20260505 11:55:11.179 | INFO | Baz IO overall: | 20260505 11:55:11.179 | INFO | BytesWritten : 1784821181101 | 20260505 11:55:11.179 | INFO | NumEvents : 849724089954 | 20260505 11:55:12.795 | INFO | Analyzing chunk frames = [3010560,3011072) | 20260505 11:55:12.795 | INFO | MemInfo: { | 20260505 11:55:12.795 | INFO | "gpu" : " 17 GB", | 20260505 11:55:12.795 | INFO | "huge" : "", | 20260505 11:55:12.795 | INFO | "huge_shared" : " 27 GB", | 20260505 11:55:12.795 | INFO | "malloc" : " 14 GB", | 20260505 11:55:12.795 | INFO | "pinned" : " 29 GB" | 20260505 11:55:12.795 | INFO | } | 20260505 11:55:12.795 | INFO | Perf report for the last 5.13996 seconds: | 20260505 11:55:12.795 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:55:12.795 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:12.795 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:12.795 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:12.795 | INFO | ANALYSIS : 65.0%, 2.9, 100.2ms, 0 | 20260505 11:55:12.795 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:12.795 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260505 11:55:12.795 | INFO | PRE_HQ : 61.0%, 1.8, 58.4ms, 0 | 20260505 11:55:12.795 | INFO | BAZWRITER : 65.2%, 4.1, 137.3ms, 0 | 20260505 11:55:12.795 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 11:55:12.795 | INFO | Uploaded 36GiB to GPU in 2.19682s (16.3873GiB/s) | 20260505 11:55:12.795 | INFO | Downloaded 38GiB to GPU in 2.30236s (16.5048GiB/s) | 20260505 11:55:17.930 | INFO | Analyzing chunk frames = [3011072,3011584) | 20260505 11:55:17.930 | INFO | MemInfo: { | 20260505 11:55:17.930 | INFO | "gpu" : " 17 GB", | 20260505 11:55:17.930 | INFO | "huge" : "", | 20260505 11:55:17.930 | INFO | "huge_shared" : " 27 GB", | 20260505 11:55:17.930 | INFO | "malloc" : " 14 GB", | 20260505 11:55:17.930 | INFO | "pinned" : " 29 GB" | 20260505 11:55:17.930 | INFO | } | 20260505 11:55:17.930 | INFO | Perf report for the last 5.13551 seconds: | 20260505 11:55:17.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:55:17.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:17.930 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:17.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:17.930 | INFO | ANALYSIS : 64.2%, 3.0, 100.1ms, 0 | 20260505 11:55:17.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:17.931 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:17.931 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 11:55:17.931 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:17.931 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:17.931 | INFO | Uploaded 36GiB to GPU in 2.23352s (16.1181GiB/s) | 20260505 11:55:17.931 | INFO | Downloaded 34GiB to GPU in 2.08992s (16.2686GiB/s) | 20260505 11:55:23.079 | INFO | Analyzing chunk frames = [3011584,3012096) | 20260505 11:55:23.079 | INFO | MemInfo: { | 20260505 11:55:23.079 | INFO | "gpu" : " 17 GB", | 20260505 11:55:23.079 | INFO | "huge" : "", | 20260505 11:55:23.079 | INFO | "huge_shared" : " 27 GB", | 20260505 11:55:23.079 | INFO | "malloc" : " 14 GB", | 20260505 11:55:23.079 | INFO | "pinned" : " 29 GB" | 20260505 11:55:23.079 | INFO | } | 20260505 11:55:23.079 | INFO | Perf report for the last 5.14862 seconds: | 20260505 11:55:23.079 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:55:23.079 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:23.079 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:23.079 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:23.079 | INFO | ANALYSIS : 63.1%, 3.0, 98.3ms, 0 | 20260505 11:55:23.079 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:23.079 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:23.079 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 11:55:23.079 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:23.079 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:23.079 | INFO | Uploaded 36GiB to GPU in 2.20983s (16.2909GiB/s) | 20260505 11:55:23.079 | INFO | Downloaded 34GiB to GPU in 2.11814s (16.0518GiB/s) | 20260505 11:55:28.224 | INFO | Analyzing chunk frames = [3012096,3012608) | 20260505 11:55:28.224 | INFO | MemInfo: { | 20260505 11:55:28.224 | INFO | "gpu" : " 17 GB", | 20260505 11:55:28.224 | INFO | "huge" : "", | 20260505 11:55:28.224 | INFO | "huge_shared" : " 27 GB", | 20260505 11:55:28.224 | INFO | "malloc" : " 14 GB", | 20260505 11:55:28.224 | INFO | "pinned" : " 29 GB" | 20260505 11:55:28.224 | INFO | } | 20260505 11:55:28.224 | INFO | Perf report for the last 5.14538 seconds: | 20260505 11:55:28.224 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:55:28.224 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:28.224 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:28.224 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:28.224 | INFO | ANALYSIS : 63.0%, 3.0, 98.0ms, 0 | 20260505 11:55:28.224 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:28.224 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:28.224 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 11:55:28.225 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:28.225 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:28.225 | INFO | Uploaded 36GiB to GPU in 2.19977s (16.3653GiB/s) | 20260505 11:55:28.225 | INFO | Downloaded 34GiB to GPU in 2.13343s (15.9368GiB/s) | 20260505 11:55:33.352 | INFO | Analyzing chunk frames = [3012608,3013120) | 20260505 11:55:33.352 | INFO | MemInfo: { | 20260505 11:55:33.352 | INFO | "gpu" : " 17 GB", | 20260505 11:55:33.352 | INFO | "huge" : "", | 20260505 11:55:33.352 | INFO | "huge_shared" : " 27 GB", | 20260505 11:55:33.352 | INFO | "malloc" : " 14 GB", | 20260505 11:55:33.352 | INFO | "pinned" : " 29 GB" | 20260505 11:55:33.352 | INFO | } | 20260505 11:55:33.352 | INFO | Perf report for the last 5.12737 seconds: | 20260505 11:55:33.352 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:55:33.352 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:33.352 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:33.352 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:33.352 | INFO | ANALYSIS : 63.0%, 3.0, 98.1ms, 0 | 20260505 11:55:33.352 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:33.352 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:33.352 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:55:33.352 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:33.352 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:33.352 | INFO | Uploaded 36GiB to GPU in 2.15056s (16.7398GiB/s) | 20260505 11:55:33.352 | INFO | Downloaded 34GiB to GPU in 2.12402s (16.0074GiB/s) | 20260505 11:55:38.486 | INFO | Analyzing chunk frames = [3013120,3013632) | 20260505 11:55:38.486 | INFO | MemInfo: { | 20260505 11:55:38.486 | INFO | "gpu" : " 17 GB", | 20260505 11:55:38.486 | INFO | "huge" : "", | 20260505 11:55:38.486 | INFO | "huge_shared" : " 27 GB", | 20260505 11:55:38.486 | INFO | "malloc" : " 14 GB", | 20260505 11:55:38.486 | INFO | "pinned" : " 29 GB" | 20260505 11:55:38.486 | INFO | } | 20260505 11:55:38.487 | INFO | Perf report for the last 5.13473 seconds: | 20260505 11:55:38.487 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:55:38.487 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:38.487 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:38.487 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:38.487 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 11:55:38.487 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:38.487 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:38.487 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:55:38.487 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:38.487 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:38.487 | INFO | Uploaded 36GiB to GPU in 2.16606s (16.62GiB/s) | 20260505 11:55:38.487 | INFO | Downloaded 34GiB to GPU in 2.11888s (16.0462GiB/s) | 20260505 11:55:43.632 | INFO | Analyzing chunk frames = [3013632,3014144) | 20260505 11:55:43.632 | INFO | MemInfo: { | 20260505 11:55:43.632 | INFO | "gpu" : " 17 GB", | 20260505 11:55:43.632 | INFO | "huge" : "", | 20260505 11:55:43.632 | INFO | "huge_shared" : " 27 GB", | 20260505 11:55:43.632 | INFO | "malloc" : " 14 GB", | 20260505 11:55:43.632 | INFO | "pinned" : " 29 GB" | 20260505 11:55:43.632 | INFO | } | 20260505 11:55:43.632 | INFO | Perf report for the last 5.14514 seconds: | 20260505 11:55:43.632 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:55:43.632 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:43.632 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:43.632 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:43.632 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 11:55:43.632 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:43.632 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:43.632 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 11:55:43.632 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:43.632 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:43.632 | INFO | Uploaded 36GiB to GPU in 2.1945s (16.4046GiB/s) | 20260505 11:55:43.632 | INFO | Downloaded 34GiB to GPU in 2.14212s (15.8721GiB/s) | 20260505 11:55:48.774 | INFO | Analyzing chunk frames = [3014144,3014656) | 20260505 11:55:48.774 | INFO | MemInfo: { | 20260505 11:55:48.774 | INFO | "gpu" : " 17 GB", | 20260505 11:55:48.774 | INFO | "huge" : "", | 20260505 11:55:48.774 | INFO | "huge_shared" : " 27 GB", | 20260505 11:55:48.774 | INFO | "malloc" : " 14 GB", | 20260505 11:55:48.774 | INFO | "pinned" : " 29 GB" | 20260505 11:55:48.774 | INFO | } | 20260505 11:55:48.775 | INFO | Perf report for the last 5.14285 seconds: | 20260505 11:55:48.775 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:55:48.775 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:48.775 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:48.775 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:48.775 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 11:55:48.775 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:48.775 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:48.775 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 11:55:48.775 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:48.775 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:48.775 | INFO | Uploaded 36GiB to GPU in 2.17983s (16.5151GiB/s) | 20260505 11:55:48.775 | INFO | Downloaded 34GiB to GPU in 2.11277s (16.0926GiB/s) | 20260505 11:55:48.775 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 11:55:48.775 | INFO | Total mem usage: 15.3 GiB | 20260505 11:55:48.775 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 11:55:48.775 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.76%) | 20260505 11:55:48.775 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 11:55:48.775 | INFO | High Water Breakdown: | 20260505 11:55:48.775 | INFO | 61.1% : 8.77 GiB ( 92.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 11:55:48.775 | INFO | 38.8% : 5.57 GiB ( 83.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 11:55:48.775 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 11:55:48.775 | INFO | ------------------------------------------ | 20260505 11:55:48.775 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:55:48.775 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 11:55:48.775 | INFO | Total mem usage: 26.5 GiB | 20260505 11:55:48.775 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 11:55:48.775 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 11:55:48.775 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 11:55:48.775 | INFO | High Water Breakdown: | 20260505 11:55:48.775 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:55:48.775 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:55:48.775 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:55:48.775 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:55:48.775 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:55:48.775 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:55:48.775 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 11:55:48.775 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 11:55:48.775 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:55:48.775 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 11:55:48.775 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 11:55:48.775 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 11:55:48.775 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 11:55:48.775 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 11:55:48.775 | INFO | ------------------------------------------ | 20260505 11:55:48.775 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:55:48.775 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 11:55:48.775 | INFO | Total mem usage: 25.6 GiB | 20260505 11:55:48.775 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 11:55:48.775 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 11:55:48.775 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 11:55:48.775 | INFO | High Water Breakdown: | 20260505 11:55:48.775 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 11:55:48.775 | INFO | ------------------------------------------ | 20260505 11:55:48.775 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:55:48.775 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 11:55:48.775 | INFO | Total mem usage: 15.7 GiB | 20260505 11:55:48.775 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 11:55:48.775 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 11:55:48.775 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 11:55:48.775 | INFO | High Water Breakdown: | 20260505 11:55:48.775 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 11:55:48.775 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 11:55:48.775 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 11:55:48.776 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 11:55:48.776 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 11:55:48.776 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 11:55:48.776 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 11:55:48.776 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 11:55:48.776 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 11:55:48.776 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:55:48.776 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 11:55:48.776 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:55:48.776 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 11:55:48.776 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 11:55:48.776 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 11:55:48.776 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 11:55:48.776 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:55:48.776 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 11:55:48.776 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:55:48.776 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:55:48.776 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:55:48.776 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 11:55:48.776 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 11:55:48.776 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:55:48.776 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 11:55:48.776 | INFO | ------------------------------------------ | 20260505 11:55:48.776 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:55:48.776 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 11:55:48.776 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:55:48.776 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:55:48.776 | INFO | PerfReport: Full Execution -- 3.20e+01 / 7.73e+01, 1.38e+01 / 1.69e+00 | 20260505 11:55:48.776 | INFO | PerfReport: 0.29%, AggregateStats -- 9.30e-02 / 1.00e-01, 1.08e-02 / 4.04e-03 | 20260505 11:55:48.776 | INFO | PerfReport: 11.76%, Baseline -- 3.77e+00 / 4.48e+00, 5.99e-01 / 2.33e-01 | 20260505 11:55:48.776 | INFO | PerfReport: 3.30%, Histogram -- 1.06e+00 / 1.49e+00, 2.03e-01 / 1.03e-01 | 20260505 11:55:48.776 | INFO | PerfReport: 15.92%, DME -- 4.08e+01 / 4.55e+01, 4.54e+00 / 2.59e+00 | 20260505 11:55:48.776 | INFO | PerfReport: 56.31%, FrameLabeling -- 1.80e+01 / 2.09e+01, 1.47e+00 / 2.17e+00 | 20260505 11:55:48.776 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.39e+00 / 2.87e+00, 4.41e-01 / 2.40e-01 | 20260505 11:55:48.776 | INFO | PerfReport: 3.67%, Metrics -- 1.18e+00 / 1.48e+00, 3.32e-01 / 2.04e-01 | 20260505 11:55:48.776 | INFO | PerfReport: 1.29%, PulseSerializing -- 4.12e-01 / 4.60e-01, 7.49e-02 / 3.88e-02 | 20260505 11:55:48.776 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 11:55:48.776 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:55:48.776 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:55:48.776 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260505 11:55:48.776 | INFO | PerfReport: 22.59%, Upload -- 2.28e+01 / 0.00e+00, 8.14e+00 / 0.00e+00 | 20260505 11:55:48.776 | INFO | PerfReport: 22.04%, Download -- 2.23e+01 / 0.00e+00, 8.13e+00 / 0.00e+00 | 20260505 11:55:48.776 | INFO | PerfReport: 31.69%, Compute -- 3.20e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 11:55:48.776 | INFO | PerfReport: 2.12%, LockUpload -- 2.15e+00 / 0.00e+00, 7.68e+00 / 0.00e+00 | 20260505 11:55:48.776 | INFO | PerfReport: 1.34%, LockDownload -- 1.36e+00 / 0.00e+00, 2.90e+00 / 0.00e+00 | 20260505 11:55:48.776 | INFO | PerfReport: 20.20%, LockCompute -- 2.04e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260505 11:55:48.776 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.36e-02 / 0.00e+00 | 20260505 11:55:48.907 | INFO | Starting a round of Baz IO | 20260505 11:55:52.151 | INFO | Populating RTMetrics report | 20260505 11:55:52.151 | INFO | RTMetrics report populated | 20260505 11:55:52.151 | INFO | Opening tmp RTMetrics json file | 20260505 11:55:52.170 | INFO | Serializing RTMetrics to json | 20260505 11:55:52.174 | INFO | Serialization done, swapping tmp file now... | 20260505 11:55:52.177 | INFO | RTMetrics done | 20260505 11:55:52.271 | INFO | Finished a round of Baz IO: Wrote 2.359 GiB in 3.363 seconds | 20260505 11:55:52.271 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.87% / 0.01% / 76.12% | 20260505 11:55:52.271 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.00742% / 1.19209e-05% / 5.5631e-05% | 20260505 11:55:52.271 | INFO | Baz IO overall: | 20260505 11:55:52.271 | INFO | BytesWritten : 1787354176556 | 20260505 11:55:52.271 | INFO | NumEvents : 850927358882 | 20260505 11:55:53.912 | INFO | Analyzing chunk frames = [3014656,3015168) | 20260505 11:55:53.912 | INFO | MemInfo: { | 20260505 11:55:53.912 | INFO | "gpu" : " 17 GB", | 20260505 11:55:53.912 | INFO | "huge" : "", | 20260505 11:55:53.912 | INFO | "huge_shared" : " 27 GB", | 20260505 11:55:53.912 | INFO | "malloc" : " 14 GB", | 20260505 11:55:53.912 | INFO | "pinned" : " 29 GB" | 20260505 11:55:53.912 | INFO | } | 20260505 11:55:53.912 | INFO | Perf report for the last 5.13748 seconds: | 20260505 11:55:53.912 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:55:53.912 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:53.912 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:53.912 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:53.912 | INFO | ANALYSIS : 64.6%, 2.9, 99.5ms, 0 | 20260505 11:55:53.912 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:53.912 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260505 11:55:53.912 | INFO | PRE_HQ : 60.9%, 1.9, 58.6ms, 0 | 20260505 11:55:53.912 | INFO | BAZWRITER : 64.7%, 4.1, 136.5ms, 0 | 20260505 11:55:53.912 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260505 11:55:53.912 | INFO | Uploaded 36GiB to GPU in 2.18534s (16.4734GiB/s) | 20260505 11:55:53.912 | INFO | Downloaded 38GiB to GPU in 2.2826s (16.6477GiB/s) | 20260505 11:55:59.048 | INFO | Analyzing chunk frames = [3015168,3015680) | 20260505 11:55:59.048 | INFO | MemInfo: { | 20260505 11:55:59.048 | INFO | "gpu" : " 17 GB", | 20260505 11:55:59.048 | INFO | "huge" : "", | 20260505 11:55:59.048 | INFO | "huge_shared" : " 27 GB", | 20260505 11:55:59.048 | INFO | "malloc" : " 14 GB", | 20260505 11:55:59.048 | INFO | "pinned" : " 29 GB" | 20260505 11:55:59.048 | INFO | } | 20260505 11:55:59.048 | INFO | Perf report for the last 5.13645 seconds: | 20260505 11:55:59.048 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:55:59.048 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:59.048 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:59.048 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:59.049 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 11:55:59.049 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:59.049 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:59.049 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 11:55:59.049 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:59.049 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:55:59.049 | INFO | Uploaded 36GiB to GPU in 2.23161s (16.1319GiB/s) | 20260505 11:55:59.049 | INFO | Downloaded 34GiB to GPU in 2.10061s (16.1858GiB/s) | 20260505 11:56:04.189 | INFO | Analyzing chunk frames = [3015680,3016192) | 20260505 11:56:04.189 | INFO | MemInfo: { | 20260505 11:56:04.189 | INFO | "gpu" : " 17 GB", | 20260505 11:56:04.189 | INFO | "huge" : "", | 20260505 11:56:04.189 | INFO | "huge_shared" : " 27 GB", | 20260505 11:56:04.189 | INFO | "malloc" : " 14 GB", | 20260505 11:56:04.189 | INFO | "pinned" : " 29 GB" | 20260505 11:56:04.189 | INFO | } | 20260505 11:56:04.189 | INFO | Perf report for the last 5.14056 seconds: | 20260505 11:56:04.189 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:56:04.189 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:04.189 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:04.189 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:04.189 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 11:56:04.189 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:04.189 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:04.189 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:56:04.189 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:04.189 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:04.189 | INFO | Uploaded 36GiB to GPU in 2.23287s (16.1227GiB/s) | 20260505 11:56:04.189 | INFO | Downloaded 34GiB to GPU in 2.12798s (15.9776GiB/s) | 20260505 11:56:09.333 | INFO | Analyzing chunk frames = [3016192,3016704) | 20260505 11:56:09.333 | INFO | MemInfo: { | 20260505 11:56:09.333 | INFO | "gpu" : " 17 GB", | 20260505 11:56:09.333 | INFO | "huge" : "", | 20260505 11:56:09.333 | INFO | "huge_shared" : " 27 GB", | 20260505 11:56:09.333 | INFO | "malloc" : " 14 GB", | 20260505 11:56:09.333 | INFO | "pinned" : " 29 GB" | 20260505 11:56:09.333 | INFO | } | 20260505 11:56:09.333 | INFO | Perf report for the last 5.1439 seconds: | 20260505 11:56:09.333 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:56:09.333 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:09.333 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:09.333 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:09.333 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 11:56:09.333 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:09.333 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:09.333 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 11:56:09.333 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:09.333 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:09.333 | INFO | Uploaded 36GiB to GPU in 2.2095s (16.2932GiB/s) | 20260505 11:56:09.333 | INFO | Downloaded 34GiB to GPU in 2.13174s (15.9494GiB/s) | 20260505 11:56:14.471 | INFO | Analyzing chunk frames = [3016704,3017216) | 20260505 11:56:14.471 | INFO | MemInfo: { | 20260505 11:56:14.471 | INFO | "gpu" : " 17 GB", | 20260505 11:56:14.471 | INFO | "huge" : "", | 20260505 11:56:14.471 | INFO | "huge_shared" : " 27 GB", | 20260505 11:56:14.471 | INFO | "malloc" : " 14 GB", | 20260505 11:56:14.471 | INFO | "pinned" : " 29 GB" | 20260505 11:56:14.471 | INFO | } | 20260505 11:56:14.471 | INFO | Perf report for the last 5.13848 seconds: | 20260505 11:56:14.471 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:56:14.471 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:14.472 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:14.472 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:14.472 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 11:56:14.472 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:14.472 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:14.472 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 11:56:14.472 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:14.472 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:14.472 | INFO | Uploaded 36GiB to GPU in 2.16534s (16.6255GiB/s) | 20260505 11:56:14.472 | INFO | Downloaded 34GiB to GPU in 2.12408s (16.0069GiB/s) | 20260505 11:56:19.603 | INFO | Analyzing chunk frames = [3017216,3017728) | 20260505 11:56:19.603 | INFO | MemInfo: { | 20260505 11:56:19.603 | INFO | "gpu" : " 17 GB", | 20260505 11:56:19.603 | INFO | "huge" : "", | 20260505 11:56:19.603 | INFO | "huge_shared" : " 27 GB", | 20260505 11:56:19.603 | INFO | "malloc" : " 14 GB", | 20260505 11:56:19.603 | INFO | "pinned" : " 29 GB" | 20260505 11:56:19.603 | INFO | } | 20260505 11:56:19.603 | INFO | Perf report for the last 5.13133 seconds: | 20260505 11:56:19.603 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:56:19.603 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:19.603 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:19.603 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:19.603 | INFO | ANALYSIS : 63.5%, 3.0, 99.0ms, 0 | 20260505 11:56:19.603 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:19.603 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:19.603 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 11:56:19.603 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:19.603 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:19.603 | INFO | Uploaded 36GiB to GPU in 2.1691s (16.5968GiB/s) | 20260505 11:56:19.603 | INFO | Downloaded 34GiB to GPU in 2.11424s (16.0814GiB/s) | 20260505 11:56:24.744 | INFO | Analyzing chunk frames = [3017728,3018240) | 20260505 11:56:24.744 | INFO | MemInfo: { | 20260505 11:56:24.744 | INFO | "gpu" : " 17 GB", | 20260505 11:56:24.744 | INFO | "huge" : "", | 20260505 11:56:24.744 | INFO | "huge_shared" : " 27 GB", | 20260505 11:56:24.744 | INFO | "malloc" : " 14 GB", | 20260505 11:56:24.744 | INFO | "pinned" : " 29 GB" | 20260505 11:56:24.744 | INFO | } | 20260505 11:56:24.744 | INFO | Perf report for the last 5.14113 seconds: | 20260505 11:56:24.744 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:56:24.744 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:24.744 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:24.744 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:24.744 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 11:56:24.744 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:24.744 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:24.744 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 11:56:24.744 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:24.744 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:24.744 | INFO | Uploaded 36GiB to GPU in 2.17085s (16.5834GiB/s) | 20260505 11:56:24.744 | INFO | Downloaded 34GiB to GPU in 2.16143s (15.7303GiB/s) | 20260505 11:56:29.885 | INFO | Analyzing chunk frames = [3018240,3018752) | 20260505 11:56:29.885 | INFO | MemInfo: { | 20260505 11:56:29.885 | INFO | "gpu" : " 17 GB", | 20260505 11:56:29.885 | INFO | "huge" : "", | 20260505 11:56:29.885 | INFO | "huge_shared" : " 27 GB", | 20260505 11:56:29.885 | INFO | "malloc" : " 14 GB", | 20260505 11:56:29.885 | INFO | "pinned" : " 29 GB" | 20260505 11:56:29.885 | INFO | } | 20260505 11:56:29.885 | INFO | Perf report for the last 5.14123 seconds: | 20260505 11:56:29.885 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:56:29.885 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:29.885 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:29.885 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:29.885 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 11:56:29.885 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:29.885 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:29.885 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 11:56:29.885 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:29.885 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:29.885 | INFO | Uploaded 36GiB to GPU in 2.21272s (16.2696GiB/s) | 20260505 11:56:29.885 | INFO | Downloaded 34GiB to GPU in 2.08601s (16.299GiB/s) | 20260505 11:56:30.018 | INFO | Starting a round of Baz IO | 20260505 11:56:33.267 | INFO | Populating RTMetrics report | 20260505 11:56:33.267 | INFO | RTMetrics report populated | 20260505 11:56:33.267 | INFO | Opening tmp RTMetrics json file | 20260505 11:56:33.268 | INFO | Serializing RTMetrics to json | 20260505 11:56:33.270 | INFO | Serialization done, swapping tmp file now... | 20260505 11:56:33.270 | INFO | RTMetrics done | 20260505 11:56:33.385 | INFO | Finished a round of Baz IO: Wrote 2.357 GiB in 3.367 seconds | 20260505 11:56:33.386 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.86% / 0.01% / 76.13% | 20260505 11:56:33.386 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.00302% / 2.38419e-05% / 7.15256e-05% | 20260505 11:56:33.386 | INFO | Baz IO overall: | 20260505 11:56:33.386 | INFO | BytesWritten : 1789884451966 | 20260505 11:56:33.386 | INFO | NumEvents : 852128385691 | 20260505 11:56:35.032 | INFO | Analyzing chunk frames = [3018752,3019264) | 20260505 11:56:35.032 | INFO | MemInfo: { | 20260505 11:56:35.032 | INFO | "gpu" : " 17 GB", | 20260505 11:56:35.032 | INFO | "huge" : "", | 20260505 11:56:35.032 | INFO | "huge_shared" : " 27 GB", | 20260505 11:56:35.032 | INFO | "malloc" : " 14 GB", | 20260505 11:56:35.032 | INFO | "pinned" : " 29 GB" | 20260505 11:56:35.032 | INFO | } | 20260505 11:56:35.032 | INFO | Perf report for the last 5.14718 seconds: | 20260505 11:56:35.032 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:56:35.032 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:35.032 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:35.032 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:35.032 | INFO | ANALYSIS : 64.6%, 3.0, 100.1ms, 0 | 20260505 11:56:35.032 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:35.032 | INFO | RT_METRICS : 45.2%, 1.0, 23.8ms, 0 | 20260505 11:56:35.032 | INFO | PRE_HQ : 61.0%, 1.9, 58.8ms, 0 | 20260505 11:56:35.032 | INFO | BAZWRITER : 64.7%, 4.1, 135.1ms, 0 | 20260505 11:56:35.032 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 11:56:35.033 | INFO | Uploaded 36GiB to GPU in 2.19475s (16.4027GiB/s) | 20260505 11:56:35.033 | INFO | Downloaded 38GiB to GPU in 2.2707s (16.7349GiB/s) | 20260505 11:56:40.169 | INFO | Analyzing chunk frames = [3019264,3019776) | 20260505 11:56:40.169 | INFO | MemInfo: { | 20260505 11:56:40.169 | INFO | "gpu" : " 17 GB", | 20260505 11:56:40.170 | INFO | "huge" : "", | 20260505 11:56:40.170 | INFO | "huge_shared" : " 27 GB", | 20260505 11:56:40.170 | INFO | "malloc" : " 14 GB", | 20260505 11:56:40.170 | INFO | "pinned" : " 29 GB" | 20260505 11:56:40.170 | INFO | } | 20260505 11:56:40.170 | INFO | Perf report for the last 5.1373 seconds: | 20260505 11:56:40.170 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:56:40.170 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:40.170 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:40.170 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:40.170 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 11:56:40.170 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:40.170 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:40.170 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 11:56:40.170 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:40.170 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:40.170 | INFO | Uploaded 36GiB to GPU in 2.19474s (16.4029GiB/s) | 20260505 11:56:40.170 | INFO | Downloaded 34GiB to GPU in 2.04383s (16.6354GiB/s) | 20260505 11:56:45.300 | INFO | Analyzing chunk frames = [3019776,3020288) | 20260505 11:56:45.300 | INFO | MemInfo: { | 20260505 11:56:45.300 | INFO | "gpu" : " 17 GB", | 20260505 11:56:45.300 | INFO | "huge" : "", | 20260505 11:56:45.300 | INFO | "huge_shared" : " 27 GB", | 20260505 11:56:45.300 | INFO | "malloc" : " 14 GB", | 20260505 11:56:45.300 | INFO | "pinned" : " 29 GB" | 20260505 11:56:45.300 | INFO | } | 20260505 11:56:45.300 | INFO | Perf report for the last 5.13052 seconds: | 20260505 11:56:45.300 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:56:45.300 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:45.300 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:45.300 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:45.300 | INFO | ANALYSIS : 63.4%, 3.0, 98.5ms, 0 | 20260505 11:56:45.300 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:45.300 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:45.300 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:56:45.300 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:45.300 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:45.300 | INFO | Uploaded 36GiB to GPU in 2.20337s (16.3386GiB/s) | 20260505 11:56:45.300 | INFO | Downloaded 34GiB to GPU in 2.12137s (16.0274GiB/s) | 20260505 11:56:50.442 | INFO | Analyzing chunk frames = [3020288,3020800) | 20260505 11:56:50.442 | INFO | MemInfo: { | 20260505 11:56:50.442 | INFO | "gpu" : " 17 GB", | 20260505 11:56:50.442 | INFO | "huge" : "", | 20260505 11:56:50.442 | INFO | "huge_shared" : " 27 GB", | 20260505 11:56:50.442 | INFO | "malloc" : " 14 GB", | 20260505 11:56:50.442 | INFO | "pinned" : " 29 GB" | 20260505 11:56:50.442 | INFO | } | 20260505 11:56:50.442 | INFO | Perf report for the last 5.14179 seconds: | 20260505 11:56:50.442 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:56:50.442 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:50.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:50.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:50.442 | INFO | ANALYSIS : 62.7%, 3.0, 97.5ms, 0 | 20260505 11:56:50.442 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:50.442 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:50.442 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:56:50.442 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:50.442 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:50.442 | INFO | Uploaded 36GiB to GPU in 2.14649s (16.7716GiB/s) | 20260505 11:56:50.442 | INFO | Downloaded 34GiB to GPU in 2.07666s (16.3724GiB/s) | 20260505 11:56:55.584 | INFO | Analyzing chunk frames = [3020800,3021312) | 20260505 11:56:55.584 | INFO | MemInfo: { | 20260505 11:56:55.584 | INFO | "gpu" : " 17 GB", | 20260505 11:56:55.584 | INFO | "huge" : "", | 20260505 11:56:55.584 | INFO | "huge_shared" : " 27 GB", | 20260505 11:56:55.584 | INFO | "malloc" : " 14 GB", | 20260505 11:56:55.585 | INFO | "pinned" : " 29 GB" | 20260505 11:56:55.585 | INFO | } | 20260505 11:56:55.585 | INFO | Perf report for the last 5.14259 seconds: | 20260505 11:56:55.585 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:56:55.585 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:55.585 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:55.585 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:55.585 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 11:56:55.585 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:55.585 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:55.585 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:56:55.585 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:55.585 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:56:55.585 | INFO | Uploaded 36GiB to GPU in 2.1542s (16.7116GiB/s) | 20260505 11:56:55.585 | INFO | Downloaded 34GiB to GPU in 2.12548s (15.9964GiB/s) | 20260505 11:57:00.727 | INFO | Analyzing chunk frames = [3021312,3021824) | 20260505 11:57:00.727 | INFO | MemInfo: { | 20260505 11:57:00.727 | INFO | "gpu" : " 17 GB", | 20260505 11:57:00.727 | INFO | "huge" : "", | 20260505 11:57:00.727 | INFO | "huge_shared" : " 27 GB", | 20260505 11:57:00.727 | INFO | "malloc" : " 14 GB", | 20260505 11:57:00.727 | INFO | "pinned" : " 29 GB" | 20260505 11:57:00.727 | INFO | } | 20260505 11:57:00.727 | INFO | Perf report for the last 5.14242 seconds: | 20260505 11:57:00.727 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:57:00.727 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:00.727 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:00.727 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:00.727 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 11:57:00.727 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:00.727 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:00.727 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:57:00.727 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:00.727 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:00.727 | INFO | Uploaded 36GiB to GPU in 2.14054s (16.8182GiB/s) | 20260505 11:57:00.727 | INFO | Downloaded 34GiB to GPU in 2.10867s (16.1239GiB/s) | 20260505 11:57:05.864 | INFO | Analyzing chunk frames = [3021824,3022336) | 20260505 11:57:05.864 | INFO | MemInfo: { | 20260505 11:57:05.864 | INFO | "gpu" : " 17 GB", | 20260505 11:57:05.864 | INFO | "huge" : "", | 20260505 11:57:05.864 | INFO | "huge_shared" : " 27 GB", | 20260505 11:57:05.864 | INFO | "malloc" : " 14 GB", | 20260505 11:57:05.864 | INFO | "pinned" : " 29 GB" | 20260505 11:57:05.864 | INFO | } | 20260505 11:57:05.864 | INFO | Perf report for the last 5.13739 seconds: | 20260505 11:57:05.864 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:57:05.864 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:05.864 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:05.864 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:05.864 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 11:57:05.864 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:05.864 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:05.864 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 11:57:05.865 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:05.865 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:05.865 | INFO | Uploaded 36GiB to GPU in 2.13869s (16.8328GiB/s) | 20260505 11:57:05.865 | INFO | Downloaded 34GiB to GPU in 2.15271s (15.794GiB/s) | 20260505 11:57:10.996 | INFO | Analyzing chunk frames = [3022336,3022848) | 20260505 11:57:10.996 | INFO | MemInfo: { | 20260505 11:57:10.996 | INFO | "gpu" : " 17 GB", | 20260505 11:57:10.996 | INFO | "huge" : "", | 20260505 11:57:10.996 | INFO | "huge_shared" : " 27 GB", | 20260505 11:57:10.996 | INFO | "malloc" : " 14 GB", | 20260505 11:57:10.996 | INFO | "pinned" : " 29 GB" | 20260505 11:57:10.996 | INFO | } | 20260505 11:57:10.996 | INFO | Perf report for the last 5.13138 seconds: | 20260505 11:57:10.996 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:57:10.996 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:10.996 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:10.996 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:10.996 | INFO | ANALYSIS : 64.0%, 3.0, 99.7ms, 0 | 20260505 11:57:10.996 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:10.996 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:10.996 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 11:57:10.996 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:10.996 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:10.996 | INFO | Uploaded 36GiB to GPU in 2.22055s (16.2122GiB/s) | 20260505 11:57:10.996 | INFO | Downloaded 34GiB to GPU in 2.08675s (16.2933GiB/s) | 20260505 11:57:10.996 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 11:57:10.996 | INFO | Total mem usage: 15.3 GiB | 20260505 11:57:10.996 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.11%) | 20260505 11:57:10.996 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.76%) | 20260505 11:57:10.996 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 11:57:10.996 | INFO | High Water Breakdown: | 20260505 11:57:10.996 | INFO | 61.1% : 8.77 GiB ( 92.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 11:57:10.996 | INFO | 38.8% : 5.56 GiB ( 83.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 11:57:10.996 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 11:57:10.996 | INFO | ------------------------------------------ | 20260505 11:57:10.996 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:57:10.996 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 11:57:10.996 | INFO | Total mem usage: 26.5 GiB | 20260505 11:57:10.996 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 11:57:10.996 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 11:57:10.996 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 11:57:10.996 | INFO | High Water Breakdown: | 20260505 11:57:10.996 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:57:10.996 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:57:10.996 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:57:10.996 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:57:10.996 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:57:10.996 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:57:10.996 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 11:57:10.996 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 11:57:10.996 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:57:10.996 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 11:57:10.996 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 11:57:10.996 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 11:57:10.996 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 11:57:10.996 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 11:57:10.996 | INFO | ------------------------------------------ | 20260505 11:57:10.996 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:57:10.996 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 11:57:10.996 | INFO | Total mem usage: 25.6 GiB | 20260505 11:57:10.996 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 11:57:10.996 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 11:57:10.997 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 11:57:10.997 | INFO | High Water Breakdown: | 20260505 11:57:10.997 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 11:57:10.997 | INFO | ------------------------------------------ | 20260505 11:57:10.997 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:57:10.997 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 11:57:10.997 | INFO | Total mem usage: 15.7 GiB | 20260505 11:57:10.997 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 11:57:10.997 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 11:57:10.997 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 11:57:10.997 | INFO | High Water Breakdown: | 20260505 11:57:10.997 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 11:57:10.997 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 11:57:10.997 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 11:57:10.997 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 11:57:10.997 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 11:57:10.997 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 11:57:10.997 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 11:57:10.997 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 11:57:10.997 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 11:57:10.997 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:57:10.997 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 11:57:10.997 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:57:10.997 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 11:57:10.997 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 11:57:10.997 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 11:57:10.997 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 11:57:10.997 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:57:10.997 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 11:57:10.997 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:57:10.997 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:57:10.997 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:57:10.997 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 11:57:10.997 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 11:57:10.997 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:57:10.997 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 11:57:10.997 | INFO | ------------------------------------------ | 20260505 11:57:10.997 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:57:10.997 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 11:57:10.997 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:57:10.997 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:57:10.997 | INFO | PerfReport: Full Execution -- 3.21e+01 / 7.67e+01, 1.37e+01 / 1.71e+00 | 20260505 11:57:10.997 | INFO | PerfReport: 0.29%, AggregateStats -- 9.30e-02 / 1.00e-01, 1.06e-02 / 5.51e-03 | 20260505 11:57:10.997 | INFO | PerfReport: 11.74%, Baseline -- 3.76e+00 / 4.49e+00, 6.02e-01 / 2.15e-01 | 20260505 11:57:10.997 | INFO | PerfReport: 3.31%, Histogram -- 1.06e+00 / 1.49e+00, 2.04e-01 / 1.03e-01 | 20260505 11:57:10.997 | INFO | PerfReport: 15.82%, DME -- 4.06e+01 / 4.49e+01, 4.27e+00 / 2.33e+00 | 20260505 11:57:10.997 | INFO | PerfReport: 56.49%, FrameLabeling -- 1.81e+01 / 2.10e+01, 1.59e+00 / 2.23e+00 | 20260505 11:57:10.997 | INFO | PerfReport: 7.41%, PulseAccumulating -- 2.37e+00 / 2.84e+00, 4.31e-01 / 2.54e-01 | 20260505 11:57:10.997 | INFO | PerfReport: 3.66%, Metrics -- 1.18e+00 / 1.48e+00, 3.33e-01 / 1.98e-01 | 20260505 11:57:10.997 | INFO | PerfReport: 1.29%, PulseSerializing -- 4.12e-01 / 4.63e-01, 7.46e-02 / 3.53e-02 | 20260505 11:57:10.997 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 11:57:10.997 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:57:10.997 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:57:10.997 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.46e+01 / 0.00e+00 | 20260505 11:57:10.997 | INFO | PerfReport: 22.54%, Upload -- 2.28e+01 / 0.00e+00, 8.10e+00 / 0.00e+00 | 20260505 11:57:10.997 | INFO | PerfReport: 21.99%, Download -- 2.22e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260505 11:57:10.997 | INFO | PerfReport: 31.75%, Compute -- 3.21e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 11:57:10.997 | INFO | PerfReport: 2.10%, LockUpload -- 2.12e+00 / 0.00e+00, 7.68e+00 / 0.00e+00 | 20260505 11:57:10.997 | INFO | PerfReport: 1.27%, LockDownload -- 1.29e+00 / 0.00e+00, 2.84e+00 / 0.00e+00 | 20260505 11:57:10.997 | INFO | PerfReport: 20.34%, LockCompute -- 2.06e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260505 11:57:10.997 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.85e-02 / 0.00e+00 | 20260505 11:57:11.133 | INFO | Starting a round of Baz IO | 20260505 11:57:14.385 | INFO | Populating RTMetrics report | 20260505 11:57:14.385 | INFO | RTMetrics report populated | 20260505 11:57:14.385 | INFO | Opening tmp RTMetrics json file | 20260505 11:57:14.398 | INFO | Serializing RTMetrics to json | 20260505 11:57:14.423 | INFO | Serialization done, swapping tmp file now... | 20260505 11:57:14.424 | INFO | RTMetrics done | 20260505 11:57:14.789 | INFO | Finished a round of Baz IO: Wrote 2.355 GiB in 3.655 seconds | 20260505 11:57:14.789 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.86% / 0.01% / 76.14% | 20260505 11:57:14.789 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.998926% / 3.97364e-06% / 7.15256e-05% | 20260505 11:57:14.789 | INFO | Baz IO overall: | 20260505 11:57:14.789 | INFO | BytesWritten : 1792413363066 | 20260505 11:57:14.789 | INFO | NumEvents : 853328922491 | 20260505 11:57:16.145 | INFO | Analyzing chunk frames = [3022848,3023360) | 20260505 11:57:16.145 | INFO | MemInfo: { | 20260505 11:57:16.145 | INFO | "gpu" : " 17 GB", | 20260505 11:57:16.145 | INFO | "huge" : "", | 20260505 11:57:16.145 | INFO | "huge_shared" : " 27 GB", | 20260505 11:57:16.145 | INFO | "malloc" : " 14 GB", | 20260505 11:57:16.145 | INFO | "pinned" : " 29 GB" | 20260505 11:57:16.145 | INFO | } | 20260505 11:57:16.145 | INFO | Perf report for the last 5.14959 seconds: | 20260505 11:57:16.145 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:57:16.145 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:16.145 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:16.145 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:16.145 | INFO | ANALYSIS : 64.7%, 2.9, 98.7ms, 0 | 20260505 11:57:16.145 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:16.145 | INFO | RT_METRICS : 46.2%, 1.0, 24.3ms, 0 | 20260505 11:57:16.145 | INFO | PRE_HQ : 61.2%, 1.8, 58.0ms, 0 | 20260505 11:57:16.146 | INFO | BAZWRITER : 70.4%, 4.3, 155.4ms, 0 | 20260505 11:57:16.146 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 11:57:16.146 | INFO | Uploaded 36GiB to GPU in 2.20631s (16.3169GiB/s) | 20260505 11:57:16.146 | INFO | Downloaded 38GiB to GPU in 2.31989s (16.3801GiB/s) | 20260505 11:57:21.282 | INFO | Analyzing chunk frames = [3023360,3023872) | 20260505 11:57:21.282 | INFO | MemInfo: { | 20260505 11:57:21.282 | INFO | "gpu" : " 17 GB", | 20260505 11:57:21.282 | INFO | "huge" : "", | 20260505 11:57:21.282 | INFO | "huge_shared" : " 27 GB", | 20260505 11:57:21.282 | INFO | "malloc" : " 14 GB", | 20260505 11:57:21.282 | INFO | "pinned" : " 29 GB" | 20260505 11:57:21.282 | INFO | } | 20260505 11:57:21.282 | INFO | Perf report for the last 5.13631 seconds: | 20260505 11:57:21.282 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:57:21.282 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:21.282 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:21.282 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:21.282 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 11:57:21.282 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:21.282 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:21.282 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 11:57:21.282 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:21.282 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:21.282 | INFO | Uploaded 36GiB to GPU in 2.17895s (16.5218GiB/s) | 20260505 11:57:21.282 | INFO | Downloaded 34GiB to GPU in 2.03299s (16.7241GiB/s) | 20260505 11:57:26.420 | INFO | Analyzing chunk frames = [3023872,3024384) | 20260505 11:57:26.420 | INFO | MemInfo: { | 20260505 11:57:26.420 | INFO | "gpu" : " 17 GB", | 20260505 11:57:26.420 | INFO | "huge" : "", | 20260505 11:57:26.420 | INFO | "huge_shared" : " 27 GB", | 20260505 11:57:26.420 | INFO | "malloc" : " 14 GB", | 20260505 11:57:26.420 | INFO | "pinned" : " 29 GB" | 20260505 11:57:26.420 | INFO | } | 20260505 11:57:26.420 | INFO | Perf report for the last 5.13856 seconds: | 20260505 11:57:26.420 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:57:26.420 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:26.420 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:26.420 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:26.420 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 11:57:26.420 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:26.420 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:26.420 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 11:57:26.420 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:26.420 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:26.421 | INFO | Uploaded 36GiB to GPU in 2.22839s (16.1552GiB/s) | 20260505 11:57:26.421 | INFO | Downloaded 34GiB to GPU in 2.12899s (15.97GiB/s) | 20260505 11:57:31.551 | INFO | Analyzing chunk frames = [3024384,3024896) | 20260505 11:57:31.551 | INFO | MemInfo: { | 20260505 11:57:31.551 | INFO | "gpu" : " 17 GB", | 20260505 11:57:31.551 | INFO | "huge" : "", | 20260505 11:57:31.552 | INFO | "huge_shared" : " 27 GB", | 20260505 11:57:31.552 | INFO | "malloc" : " 14 GB", | 20260505 11:57:31.552 | INFO | "pinned" : " 29 GB" | 20260505 11:57:31.552 | INFO | } | 20260505 11:57:31.552 | INFO | Perf report for the last 5.13122 seconds: | 20260505 11:57:31.552 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:57:31.552 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:31.552 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:31.552 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:31.552 | INFO | ANALYSIS : 63.6%, 3.0, 98.7ms, 0 | 20260505 11:57:31.552 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:31.552 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:31.552 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 11:57:31.552 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:31.552 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:31.552 | INFO | Uploaded 36GiB to GPU in 2.21848s (16.2273GiB/s) | 20260505 11:57:31.552 | INFO | Downloaded 34GiB to GPU in 2.13478s (15.9267GiB/s) | 20260505 11:57:36.692 | INFO | Analyzing chunk frames = [3024896,3025408) | 20260505 11:57:36.692 | INFO | MemInfo: { | 20260505 11:57:36.692 | INFO | "gpu" : " 17 GB", | 20260505 11:57:36.692 | INFO | "huge" : "", | 20260505 11:57:36.693 | INFO | "huge_shared" : " 27 GB", | 20260505 11:57:36.693 | INFO | "malloc" : " 14 GB", | 20260505 11:57:36.693 | INFO | "pinned" : " 29 GB" | 20260505 11:57:36.693 | INFO | } | 20260505 11:57:36.693 | INFO | Perf report for the last 5.14098 seconds: | 20260505 11:57:36.693 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:57:36.693 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:36.693 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:36.693 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:36.693 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 11:57:36.693 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:36.693 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:36.693 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 11:57:36.693 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:36.693 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:36.693 | INFO | Uploaded 36GiB to GPU in 2.18413s (16.4826GiB/s) | 20260505 11:57:36.693 | INFO | Downloaded 34GiB to GPU in 2.12323s (16.0133GiB/s) | 20260505 11:57:41.838 | INFO | Analyzing chunk frames = [3025408,3025920) | 20260505 11:57:41.838 | INFO | MemInfo: { | 20260505 11:57:41.838 | INFO | "gpu" : " 17 GB", | 20260505 11:57:41.838 | INFO | "huge" : "", | 20260505 11:57:41.838 | INFO | "huge_shared" : " 27 GB", | 20260505 11:57:41.838 | INFO | "malloc" : " 14 GB", | 20260505 11:57:41.838 | INFO | "pinned" : " 29 GB" | 20260505 11:57:41.838 | INFO | } | 20260505 11:57:41.838 | INFO | Perf report for the last 5.1459 seconds: | 20260505 11:57:41.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:57:41.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:41.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:41.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:41.839 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 11:57:41.839 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:41.839 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:41.839 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 11:57:41.839 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:41.839 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:41.839 | INFO | Uploaded 36GiB to GPU in 2.18724s (16.4591GiB/s) | 20260505 11:57:41.839 | INFO | Downloaded 34GiB to GPU in 2.12143s (16.0269GiB/s) | 20260505 11:57:46.984 | INFO | Analyzing chunk frames = [3025920,3026432) | 20260505 11:57:46.984 | INFO | MemInfo: { | 20260505 11:57:46.984 | INFO | "gpu" : " 17 GB", | 20260505 11:57:46.984 | INFO | "huge" : "", | 20260505 11:57:46.984 | INFO | "huge_shared" : " 27 GB", | 20260505 11:57:46.984 | INFO | "malloc" : " 14 GB", | 20260505 11:57:46.984 | INFO | "pinned" : " 29 GB" | 20260505 11:57:46.984 | INFO | } | 20260505 11:57:46.984 | INFO | Perf report for the last 5.14598 seconds: | 20260505 11:57:46.984 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:57:46.984 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:46.985 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:46.985 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:46.985 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 11:57:46.985 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:46.985 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:46.985 | INFO | PRE_HQ : 29.9%, 1.0, 15.6ms, 0 | 20260505 11:57:46.985 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:46.985 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:46.985 | INFO | Uploaded 36GiB to GPU in 2.13993s (16.8229GiB/s) | 20260505 11:57:46.985 | INFO | Downloaded 34GiB to GPU in 2.13256s (15.9433GiB/s) | 20260505 11:57:52.112 | INFO | Analyzing chunk frames = [3026432,3026944) | 20260505 11:57:52.112 | INFO | MemInfo: { | 20260505 11:57:52.112 | INFO | "gpu" : " 17 GB", | 20260505 11:57:52.112 | INFO | "huge" : "", | 20260505 11:57:52.112 | INFO | "huge_shared" : " 27 GB", | 20260505 11:57:52.112 | INFO | "malloc" : " 14 GB", | 20260505 11:57:52.112 | INFO | "pinned" : " 29 GB" | 20260505 11:57:52.112 | INFO | } | 20260505 11:57:52.112 | INFO | Perf report for the last 5.12798 seconds: | 20260505 11:57:52.112 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:57:52.112 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:52.112 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:52.112 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:52.112 | INFO | ANALYSIS : 63.8%, 3.0, 99.3ms, 0 | 20260505 11:57:52.113 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:52.113 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:52.113 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 11:57:52.113 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:52.113 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:52.113 | INFO | Uploaded 36GiB to GPU in 2.13081s (16.895GiB/s) | 20260505 11:57:52.113 | INFO | Downloaded 34GiB to GPU in 2.10327s (16.1653GiB/s) | 20260505 11:57:52.250 | INFO | Starting a round of Baz IO | 20260505 11:57:55.505 | INFO | Populating RTMetrics report | 20260505 11:57:55.505 | INFO | RTMetrics report populated | 20260505 11:57:55.505 | INFO | Opening tmp RTMetrics json file | 20260505 11:57:55.528 | INFO | Serializing RTMetrics to json | 20260505 11:57:55.544 | INFO | Serialization done, swapping tmp file now... | 20260505 11:57:55.560 | INFO | RTMetrics done | 20260505 11:57:55.613 | INFO | Finished a round of Baz IO: Wrote 2.356 GiB in 3.363 seconds | 20260505 11:57:55.613 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.85% / 0.01% / 76.14% | 20260505 11:57:55.613 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.997178% / 3.97364e-06% / 7.54992e-05% | 20260505 11:57:55.613 | INFO | Baz IO overall: | 20260505 11:57:55.613 | INFO | BytesWritten : 1794942636765 | 20260505 11:57:55.613 | INFO | NumEvents : 854529990888 | 20260505 11:57:57.250 | INFO | Analyzing chunk frames = [3026944,3027456) | 20260505 11:57:57.250 | INFO | MemInfo: { | 20260505 11:57:57.250 | INFO | "gpu" : " 17 GB", | 20260505 11:57:57.250 | INFO | "huge" : "", | 20260505 11:57:57.250 | INFO | "huge_shared" : " 27 GB", | 20260505 11:57:57.250 | INFO | "malloc" : " 14 GB", | 20260505 11:57:57.250 | INFO | "pinned" : " 29 GB" | 20260505 11:57:57.250 | INFO | } | 20260505 11:57:57.250 | INFO | Perf report for the last 5.13763 seconds: | 20260505 11:57:57.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:57:57.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:57.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:57.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:57.250 | INFO | ANALYSIS : 64.9%, 3.0, 100.6ms, 0 | 20260505 11:57:57.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:57:57.250 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260505 11:57:57.250 | INFO | PRE_HQ : 61.3%, 1.8, 58.6ms, 0 | 20260505 11:57:57.250 | INFO | BAZWRITER : 64.8%, 4.0, 134.2ms, 0 | 20260505 11:57:57.250 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 11:57:57.250 | INFO | Uploaded 36GiB to GPU in 2.18264s (16.4938GiB/s) | 20260505 11:57:57.251 | INFO | Downloaded 38GiB to GPU in 2.29307s (16.5717GiB/s) | 20260505 11:58:02.397 | INFO | Analyzing chunk frames = [3027456,3027968) | 20260505 11:58:02.397 | INFO | MemInfo: { | 20260505 11:58:02.397 | INFO | "gpu" : " 17 GB", | 20260505 11:58:02.397 | INFO | "huge" : "", | 20260505 11:58:02.397 | INFO | "huge_shared" : " 27 GB", | 20260505 11:58:02.397 | INFO | "malloc" : " 14 GB", | 20260505 11:58:02.397 | INFO | "pinned" : " 29 GB" | 20260505 11:58:02.397 | INFO | } | 20260505 11:58:02.397 | INFO | Perf report for the last 5.14693 seconds: | 20260505 11:58:02.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:58:02.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:02.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:02.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:02.397 | INFO | ANALYSIS : 64.0%, 3.0, 100.1ms, 0 | 20260505 11:58:02.397 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:02.397 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:02.397 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 11:58:02.397 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:02.397 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:02.397 | INFO | Uploaded 36GiB to GPU in 2.23301s (16.1218GiB/s) | 20260505 11:58:02.397 | INFO | Downloaded 34GiB to GPU in 2.09079s (16.2618GiB/s) | 20260505 11:58:07.538 | INFO | Analyzing chunk frames = [3027968,3028480) | 20260505 11:58:07.538 | INFO | MemInfo: { | 20260505 11:58:07.538 | INFO | "gpu" : " 17 GB", | 20260505 11:58:07.539 | INFO | "huge" : "", | 20260505 11:58:07.539 | INFO | "huge_shared" : " 27 GB", | 20260505 11:58:07.539 | INFO | "malloc" : " 14 GB", | 20260505 11:58:07.539 | INFO | "pinned" : " 29 GB" | 20260505 11:58:07.539 | INFO | } | 20260505 11:58:07.539 | INFO | Perf report for the last 5.14155 seconds: | 20260505 11:58:07.539 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:58:07.539 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:07.539 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:07.539 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:07.539 | INFO | ANALYSIS : 63.2%, 3.0, 98.4ms, 0 | 20260505 11:58:07.539 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:07.539 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:07.539 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 11:58:07.539 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:07.539 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:07.539 | INFO | Uploaded 36GiB to GPU in 2.20893s (16.2975GiB/s) | 20260505 11:58:07.539 | INFO | Downloaded 34GiB to GPU in 2.12035s (16.0351GiB/s) | 20260505 11:58:12.672 | INFO | Analyzing chunk frames = [3028480,3028992) | 20260505 11:58:12.672 | INFO | MemInfo: { | 20260505 11:58:12.672 | INFO | "gpu" : " 17 GB", | 20260505 11:58:12.672 | INFO | "huge" : "", | 20260505 11:58:12.672 | INFO | "huge_shared" : " 27 GB", | 20260505 11:58:12.672 | INFO | "malloc" : " 14 GB", | 20260505 11:58:12.672 | INFO | "pinned" : " 29 GB" | 20260505 11:58:12.672 | INFO | } | 20260505 11:58:12.672 | INFO | Perf report for the last 5.13324 seconds: | 20260505 11:58:12.672 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:58:12.672 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:12.672 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:12.672 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:12.672 | INFO | ANALYSIS : 63.2%, 3.0, 98.1ms, 0 | 20260505 11:58:12.672 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:12.672 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:12.672 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:58:12.672 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:12.672 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:12.672 | INFO | Uploaded 36GiB to GPU in 2.20539s (16.3237GiB/s) | 20260505 11:58:12.672 | INFO | Downloaded 34GiB to GPU in 2.13017s (15.9611GiB/s) | 20260505 11:58:17.815 | INFO | Analyzing chunk frames = [3028992,3029504) | 20260505 11:58:17.815 | INFO | MemInfo: { | 20260505 11:58:17.815 | INFO | "gpu" : " 17 GB", | 20260505 11:58:17.815 | INFO | "huge" : "", | 20260505 11:58:17.815 | INFO | "huge_shared" : " 27 GB", | 20260505 11:58:17.815 | INFO | "malloc" : " 14 GB", | 20260505 11:58:17.815 | INFO | "pinned" : " 29 GB" | 20260505 11:58:17.815 | INFO | } | 20260505 11:58:17.815 | INFO | Perf report for the last 5.14337 seconds: | 20260505 11:58:17.815 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:58:17.815 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:17.815 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:17.815 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:17.815 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 11:58:17.815 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:17.815 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:17.815 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 11:58:17.815 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:17.815 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:17.815 | INFO | Uploaded 36GiB to GPU in 2.17267s (16.5695GiB/s) | 20260505 11:58:17.816 | INFO | Downloaded 34GiB to GPU in 2.12957s (15.9657GiB/s) | 20260505 11:58:22.951 | INFO | Analyzing chunk frames = [3029504,3030016) | 20260505 11:58:22.951 | INFO | MemInfo: { | 20260505 11:58:22.951 | INFO | "gpu" : " 17 GB", | 20260505 11:58:22.951 | INFO | "huge" : "", | 20260505 11:58:22.951 | INFO | "huge_shared" : " 27 GB", | 20260505 11:58:22.951 | INFO | "malloc" : " 14 GB", | 20260505 11:58:22.951 | INFO | "pinned" : " 29 GB" | 20260505 11:58:22.951 | INFO | } | 20260505 11:58:22.951 | INFO | Perf report for the last 5.13599 seconds: | 20260505 11:58:22.951 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:58:22.951 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:22.951 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:22.951 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:22.951 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 11:58:22.951 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:22.951 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:22.951 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 11:58:22.951 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:22.951 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:22.952 | INFO | Uploaded 36GiB to GPU in 2.18047s (16.5102GiB/s) | 20260505 11:58:22.952 | INFO | Downloaded 34GiB to GPU in 2.11685s (16.0616GiB/s) | 20260505 11:58:28.093 | INFO | Analyzing chunk frames = [3030016,3030528) | 20260505 11:58:28.093 | INFO | MemInfo: { | 20260505 11:58:28.093 | INFO | "gpu" : " 17 GB", | 20260505 11:58:28.093 | INFO | "huge" : "", | 20260505 11:58:28.093 | INFO | "huge_shared" : " 27 GB", | 20260505 11:58:28.093 | INFO | "malloc" : " 14 GB", | 20260505 11:58:28.093 | INFO | "pinned" : " 29 GB" | 20260505 11:58:28.093 | INFO | } | 20260505 11:58:28.093 | INFO | Perf report for the last 5.1422 seconds: | 20260505 11:58:28.093 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:58:28.093 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:28.094 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:28.094 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:28.094 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 11:58:28.094 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:28.094 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:28.094 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 11:58:28.094 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:28.094 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:28.094 | INFO | Uploaded 36GiB to GPU in 2.16229s (16.649GiB/s) | 20260505 11:58:28.094 | INFO | Downloaded 34GiB to GPU in 2.15016s (15.8128GiB/s) | 20260505 11:58:33.234 | INFO | Analyzing chunk frames = [3030528,3031040) | 20260505 11:58:33.234 | INFO | MemInfo: { | 20260505 11:58:33.234 | INFO | "gpu" : " 17 GB", | 20260505 11:58:33.234 | INFO | "huge" : "", | 20260505 11:58:33.234 | INFO | "huge_shared" : " 27 GB", | 20260505 11:58:33.234 | INFO | "malloc" : " 14 GB", | 20260505 11:58:33.234 | INFO | "pinned" : " 29 GB" | 20260505 11:58:33.234 | INFO | } | 20260505 11:58:33.234 | INFO | Perf report for the last 5.14059 seconds: | 20260505 11:58:33.234 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:58:33.234 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:33.234 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:33.234 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:33.234 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 11:58:33.234 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:33.234 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:33.234 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 11:58:33.234 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:33.234 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:33.234 | INFO | Uploaded 36GiB to GPU in 2.17364s (16.5621GiB/s) | 20260505 11:58:33.234 | INFO | Downloaded 34GiB to GPU in 2.10882s (16.1228GiB/s) | 20260505 11:58:33.234 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 11:58:33.234 | INFO | Total mem usage: 15.3 GiB | 20260505 11:58:33.234 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 11:58:33.234 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -0.132%) | 20260505 11:58:33.234 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 11:58:33.234 | INFO | High Water Breakdown: | 20260505 11:58:33.234 | INFO | 62.2% : 9.52 GiB ( 100% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 11:58:33.234 | INFO | 37.7% : 5.77 GiB ( 86.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 11:58:33.235 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 11:58:33.235 | INFO | ------------------------------------------ | 20260505 11:58:33.235 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:58:33.235 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 11:58:33.235 | INFO | Total mem usage: 26.5 GiB | 20260505 11:58:33.235 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 11:58:33.235 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 11:58:33.235 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 11:58:33.235 | INFO | High Water Breakdown: | 20260505 11:58:33.235 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:58:33.235 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:58:33.235 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:58:33.235 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:58:33.235 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:58:33.235 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:58:33.235 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 11:58:33.235 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 11:58:33.235 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:58:33.235 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 11:58:33.235 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 11:58:33.235 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 11:58:33.235 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 11:58:33.235 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 11:58:33.235 | INFO | ------------------------------------------ | 20260505 11:58:33.235 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:58:33.235 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 11:58:33.235 | INFO | Total mem usage: 25.6 GiB | 20260505 11:58:33.235 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 11:58:33.235 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 11:58:33.235 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 11:58:33.235 | INFO | High Water Breakdown: | 20260505 11:58:33.235 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 11:58:33.235 | INFO | ------------------------------------------ | 20260505 11:58:33.235 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:58:33.235 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 11:58:33.235 | INFO | Total mem usage: 15.7 GiB | 20260505 11:58:33.235 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 11:58:33.235 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 11:58:33.235 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 11:58:33.235 | INFO | High Water Breakdown: | 20260505 11:58:33.235 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 11:58:33.235 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 11:58:33.235 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 11:58:33.235 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 11:58:33.235 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 11:58:33.235 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 11:58:33.235 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 11:58:33.235 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 11:58:33.235 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 11:58:33.235 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:58:33.235 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 11:58:33.235 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:58:33.235 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 11:58:33.235 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 11:58:33.235 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 11:58:33.235 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 11:58:33.235 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:58:33.235 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 11:58:33.235 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:58:33.235 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:58:33.235 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:58:33.235 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 11:58:33.235 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 11:58:33.235 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:58:33.235 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 11:58:33.235 | INFO | ------------------------------------------ | 20260505 11:58:33.235 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:58:33.235 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 11:58:33.235 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:58:33.235 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:58:33.235 | INFO | PerfReport: Full Execution -- 3.21e+01 / 7.74e+01, 1.38e+01 / 1.80e+00 | 20260505 11:58:33.235 | INFO | PerfReport: 0.29%, AggregateStats -- 9.27e-02 / 9.99e-02, 1.09e-02 / 5.08e-03 | 20260505 11:58:33.235 | INFO | PerfReport: 11.76%, Baseline -- 3.77e+00 / 4.49e+00, 5.97e-01 / 1.53e-01 | 20260505 11:58:33.235 | INFO | PerfReport: 3.31%, Histogram -- 1.06e+00 / 1.49e+00, 2.03e-01 / 9.75e-02 | 20260505 11:58:33.235 | INFO | PerfReport: 15.93%, DME -- 4.09e+01 / 4.56e+01, 4.36e+00 / 2.36e+00 | 20260505 11:58:33.235 | INFO | PerfReport: 56.31%, FrameLabeling -- 1.81e+01 / 2.09e+01, 1.50e+00 / 2.09e+00 | 20260505 11:58:33.235 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.39e+00 / 2.87e+00, 4.40e-01 / 2.32e-01 | 20260505 11:58:33.235 | INFO | PerfReport: 3.66%, Metrics -- 1.17e+00 / 1.49e+00, 3.29e-01 / 1.88e-01 | 20260505 11:58:33.235 | INFO | PerfReport: 1.29%, PulseSerializing -- 4.12e-01 / 4.60e-01, 7.47e-02 / 3.90e-02 | 20260505 11:58:33.235 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 11:58:33.235 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:58:33.235 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:58:33.235 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260505 11:58:33.235 | INFO | PerfReport: 22.55%, Upload -- 2.28e+01 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260505 11:58:33.235 | INFO | PerfReport: 22.06%, Download -- 2.23e+01 / 0.00e+00, 8.12e+00 / 0.00e+00 | 20260505 11:58:33.235 | INFO | PerfReport: 31.74%, Compute -- 3.21e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 11:58:33.235 | INFO | PerfReport: 2.08%, LockUpload -- 2.10e+00 / 0.00e+00, 7.73e+00 / 0.00e+00 | 20260505 11:58:33.235 | INFO | PerfReport: 1.40%, LockDownload -- 1.41e+00 / 0.00e+00, 3.00e+00 / 0.00e+00 | 20260505 11:58:33.235 | INFO | PerfReport: 20.16%, LockCompute -- 2.04e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260505 11:58:33.235 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.15e-02 / 0.00e+00 | 20260505 11:58:33.366 | INFO | Starting a round of Baz IO | 20260505 11:58:36.619 | INFO | Populating RTMetrics report | 20260505 11:58:36.619 | INFO | RTMetrics report populated | 20260505 11:58:36.619 | INFO | Opening tmp RTMetrics json file | 20260505 11:58:36.624 | INFO | Serializing RTMetrics to json | 20260505 11:58:36.647 | INFO | Serialization done, swapping tmp file now... | 20260505 11:58:36.672 | INFO | RTMetrics done | 20260505 11:58:36.736 | INFO | Finished a round of Baz IO: Wrote 2.355 GiB in 3.37 seconds | 20260505 11:58:36.736 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.84% / 0.01% / 76.15% | 20260505 11:58:36.736 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.996308% / 3.97364e-06% / 4.37101e-05% | 20260505 11:58:36.736 | INFO | Baz IO overall: | 20260505 11:58:36.736 | INFO | BytesWritten : 1797471502362 | 20260505 11:58:36.736 | INFO | NumEvents : 855730937788 | 20260505 11:58:38.363 | INFO | Analyzing chunk frames = [3031040,3031552) | 20260505 11:58:38.363 | INFO | MemInfo: { | 20260505 11:58:38.363 | INFO | "gpu" : " 17 GB", | 20260505 11:58:38.363 | INFO | "huge" : "", | 20260505 11:58:38.363 | INFO | "huge_shared" : " 27 GB", | 20260505 11:58:38.363 | INFO | "malloc" : " 14 GB", | 20260505 11:58:38.363 | INFO | "pinned" : " 29 GB" | 20260505 11:58:38.363 | INFO | } | 20260505 11:58:38.363 | INFO | Perf report for the last 5.12914 seconds: | 20260505 11:58:38.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:58:38.363 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:38.363 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:38.363 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:38.363 | INFO | ANALYSIS : 64.9%, 2.9, 100.0ms, 0 | 20260505 11:58:38.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:38.363 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260505 11:58:38.363 | INFO | PRE_HQ : 60.9%, 1.9, 58.5ms, 0 | 20260505 11:58:38.363 | INFO | BAZWRITER : 65.0%, 4.1, 136.4ms, 0 | 20260505 11:58:38.363 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 11:58:38.363 | INFO | Uploaded 36GiB to GPU in 2.18873s (16.4479GiB/s) | 20260505 11:58:38.364 | INFO | Downloaded 38GiB to GPU in 2.29876s (16.5306GiB/s) | 20260505 11:58:43.501 | INFO | Analyzing chunk frames = [3031552,3032064) | 20260505 11:58:43.501 | INFO | MemInfo: { | 20260505 11:58:43.501 | INFO | "gpu" : " 17 GB", | 20260505 11:58:43.501 | INFO | "huge" : "", | 20260505 11:58:43.501 | INFO | "huge_shared" : " 27 GB", | 20260505 11:58:43.501 | INFO | "malloc" : " 14 GB", | 20260505 11:58:43.501 | INFO | "pinned" : " 29 GB" | 20260505 11:58:43.501 | INFO | } | 20260505 11:58:43.501 | INFO | Perf report for the last 5.13827 seconds: | 20260505 11:58:43.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:58:43.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:43.502 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:43.502 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:43.502 | INFO | ANALYSIS : 64.1%, 3.0, 100.0ms, 0 | 20260505 11:58:43.502 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:43.502 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:43.502 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 11:58:43.502 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:43.502 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:43.502 | INFO | Uploaded 36GiB to GPU in 2.20714s (16.3107GiB/s) | 20260505 11:58:43.502 | INFO | Downloaded 34GiB to GPU in 2.08309s (16.3219GiB/s) | 20260505 11:58:48.650 | INFO | Analyzing chunk frames = [3032064,3032576) | 20260505 11:58:48.650 | INFO | MemInfo: { | 20260505 11:58:48.650 | INFO | "gpu" : " 17 GB", | 20260505 11:58:48.650 | INFO | "huge" : "", | 20260505 11:58:48.650 | INFO | "huge_shared" : " 27 GB", | 20260505 11:58:48.650 | INFO | "malloc" : " 14 GB", | 20260505 11:58:48.650 | INFO | "pinned" : " 29 GB" | 20260505 11:58:48.650 | INFO | } | 20260505 11:58:48.650 | INFO | Perf report for the last 5.14854 seconds: | 20260505 11:58:48.650 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:58:48.650 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:48.650 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:48.650 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:48.650 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260505 11:58:48.650 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:48.650 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:48.650 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 11:58:48.650 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:48.650 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:48.650 | INFO | Uploaded 36GiB to GPU in 2.15715s (16.6887GiB/s) | 20260505 11:58:48.650 | INFO | Downloaded 34GiB to GPU in 2.06718s (16.4476GiB/s) | 20260505 11:58:53.792 | INFO | Analyzing chunk frames = [3032576,3033088) | 20260505 11:58:53.792 | INFO | MemInfo: { | 20260505 11:58:53.792 | INFO | "gpu" : " 17 GB", | 20260505 11:58:53.792 | INFO | "huge" : "", | 20260505 11:58:53.793 | INFO | "huge_shared" : " 27 GB", | 20260505 11:58:53.793 | INFO | "malloc" : " 14 GB", | 20260505 11:58:53.793 | INFO | "pinned" : " 29 GB" | 20260505 11:58:53.793 | INFO | } | 20260505 11:58:53.793 | INFO | Perf report for the last 5.14253 seconds: | 20260505 11:58:53.793 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:58:53.793 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:53.793 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:53.793 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:53.793 | INFO | ANALYSIS : 63.1%, 3.0, 98.0ms, 0 | 20260505 11:58:53.793 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:53.793 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:53.793 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 11:58:53.793 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:53.793 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:53.793 | INFO | Uploaded 36GiB to GPU in 2.20165s (16.3513GiB/s) | 20260505 11:58:53.793 | INFO | Downloaded 34GiB to GPU in 2.13246s (15.944GiB/s) | 20260505 11:58:58.930 | INFO | Analyzing chunk frames = [3033088,3033600) | 20260505 11:58:58.930 | INFO | MemInfo: { | 20260505 11:58:58.930 | INFO | "gpu" : " 17 GB", | 20260505 11:58:58.930 | INFO | "huge" : "", | 20260505 11:58:58.930 | INFO | "huge_shared" : " 27 GB", | 20260505 11:58:58.930 | INFO | "malloc" : " 14 GB", | 20260505 11:58:58.930 | INFO | "pinned" : " 29 GB" | 20260505 11:58:58.930 | INFO | } | 20260505 11:58:58.930 | INFO | Perf report for the last 5.13746 seconds: | 20260505 11:58:58.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:58:58.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:58.930 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:58.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:58.930 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 11:58:58.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:58.930 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:58.930 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 11:58:58.930 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:58.930 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:58:58.930 | INFO | Uploaded 36GiB to GPU in 2.14188s (16.8077GiB/s) | 20260505 11:58:58.930 | INFO | Downloaded 34GiB to GPU in 2.12358s (16.0107GiB/s) | 20260505 11:59:04.063 | INFO | Analyzing chunk frames = [3033600,3034112) | 20260505 11:59:04.063 | INFO | MemInfo: { | 20260505 11:59:04.063 | INFO | "gpu" : " 17 GB", | 20260505 11:59:04.063 | INFO | "huge" : "", | 20260505 11:59:04.063 | INFO | "huge_shared" : " 27 GB", | 20260505 11:59:04.063 | INFO | "malloc" : " 14 GB", | 20260505 11:59:04.063 | INFO | "pinned" : " 29 GB" | 20260505 11:59:04.063 | INFO | } | 20260505 11:59:04.064 | INFO | Perf report for the last 5.13345 seconds: | 20260505 11:59:04.064 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:59:04.064 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:04.064 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:04.064 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:04.064 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 11:59:04.064 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:04.064 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:04.064 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 11:59:04.064 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:04.064 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:04.064 | INFO | Uploaded 36GiB to GPU in 2.14262s (16.8019GiB/s) | 20260505 11:59:04.064 | INFO | Downloaded 34GiB to GPU in 2.10923s (16.1196GiB/s) | 20260505 11:59:09.204 | INFO | Analyzing chunk frames = [3034112,3034624) | 20260505 11:59:09.204 | INFO | MemInfo: { | 20260505 11:59:09.204 | INFO | "gpu" : " 17 GB", | 20260505 11:59:09.204 | INFO | "huge" : "", | 20260505 11:59:09.204 | INFO | "huge_shared" : " 27 GB", | 20260505 11:59:09.204 | INFO | "malloc" : " 14 GB", | 20260505 11:59:09.204 | INFO | "pinned" : " 29 GB" | 20260505 11:59:09.204 | INFO | } | 20260505 11:59:09.205 | INFO | Perf report for the last 5.14098 seconds: | 20260505 11:59:09.205 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:59:09.205 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:09.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:09.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:09.205 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 11:59:09.205 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:09.205 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:09.205 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 11:59:09.205 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:09.205 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:09.205 | INFO | Uploaded 36GiB to GPU in 2.19975s (16.3655GiB/s) | 20260505 11:59:09.205 | INFO | Downloaded 34GiB to GPU in 2.14156s (15.8763GiB/s) | 20260505 11:59:14.343 | INFO | Analyzing chunk frames = [3034624,3035136) | 20260505 11:59:14.343 | INFO | MemInfo: { | 20260505 11:59:14.343 | INFO | "gpu" : " 17 GB", | 20260505 11:59:14.343 | INFO | "huge" : "", | 20260505 11:59:14.343 | INFO | "huge_shared" : " 27 GB", | 20260505 11:59:14.343 | INFO | "malloc" : " 14 GB", | 20260505 11:59:14.343 | INFO | "pinned" : " 29 GB" | 20260505 11:59:14.343 | INFO | } | 20260505 11:59:14.343 | INFO | Perf report for the last 5.13868 seconds: | 20260505 11:59:14.343 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:59:14.343 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:14.343 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:14.343 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:14.343 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 11:59:14.343 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:14.343 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:14.343 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 11:59:14.343 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:14.343 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:14.343 | INFO | Uploaded 36GiB to GPU in 2.21412s (16.2593GiB/s) | 20260505 11:59:14.343 | INFO | Downloaded 34GiB to GPU in 2.10562s (16.1472GiB/s) | 20260505 11:59:14.475 | INFO | Starting a round of Baz IO | 20260505 11:59:17.716 | INFO | Populating RTMetrics report | 20260505 11:59:17.716 | INFO | RTMetrics report populated | 20260505 11:59:17.716 | INFO | Opening tmp RTMetrics json file | 20260505 11:59:17.717 | INFO | Serializing RTMetrics to json | 20260505 11:59:17.718 | INFO | Serialization done, swapping tmp file now... | 20260505 11:59:17.719 | INFO | RTMetrics done | 20260505 11:59:17.840 | INFO | Finished a round of Baz IO: Wrote 2.354 GiB in 3.365 seconds | 20260505 11:59:17.841 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.84% / 0.01% / 76.16% | 20260505 11:59:17.841 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.992843% / 1.58946e-05% / 6.35783e-05% | 20260505 11:59:17.841 | INFO | Baz IO overall: | 20260505 11:59:17.841 | INFO | BytesWritten : 1799999549926 | 20260505 11:59:17.841 | INFO | NumEvents : 856931542069 | 20260505 11:59:19.496 | INFO | Analyzing chunk frames = [3035136,3035648) | 20260505 11:59:19.496 | INFO | MemInfo: { | 20260505 11:59:19.496 | INFO | "gpu" : " 17 GB", | 20260505 11:59:19.496 | INFO | "huge" : "", | 20260505 11:59:19.496 | INFO | "huge_shared" : " 27 GB", | 20260505 11:59:19.496 | INFO | "malloc" : " 14 GB", | 20260505 11:59:19.496 | INFO | "pinned" : " 29 GB" | 20260505 11:59:19.496 | INFO | } | 20260505 11:59:19.496 | INFO | Perf report for the last 5.15249 seconds: | 20260505 11:59:19.496 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:59:19.496 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:19.496 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:19.496 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:19.496 | INFO | ANALYSIS : 64.3%, 2.9, 99.6ms, 0 | 20260505 11:59:19.496 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:19.496 | INFO | RT_METRICS : 45.2%, 1.0, 23.8ms, 0 | 20260505 11:59:19.496 | INFO | PRE_HQ : 60.7%, 1.8, 58.3ms, 0 | 20260505 11:59:19.496 | INFO | BAZWRITER : 64.6%, 4.1, 136.7ms, 0 | 20260505 11:59:19.496 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 11:59:19.496 | INFO | Uploaded 36GiB to GPU in 2.1769s (16.5373GiB/s) | 20260505 11:59:19.496 | INFO | Downloaded 38GiB to GPU in 2.29052s (16.5901GiB/s) | 20260505 11:59:24.622 | INFO | Analyzing chunk frames = [3035648,3036160) | 20260505 11:59:24.622 | INFO | MemInfo: { | 20260505 11:59:24.622 | INFO | "gpu" : " 17 GB", | 20260505 11:59:24.622 | INFO | "huge" : "", | 20260505 11:59:24.622 | INFO | "huge_shared" : " 27 GB", | 20260505 11:59:24.622 | INFO | "malloc" : " 14 GB", | 20260505 11:59:24.622 | INFO | "pinned" : " 29 GB" | 20260505 11:59:24.622 | INFO | } | 20260505 11:59:24.622 | INFO | Perf report for the last 5.12602 seconds: | 20260505 11:59:24.622 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:59:24.622 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:24.622 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:24.622 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:24.622 | INFO | ANALYSIS : 63.8%, 3.0, 99.3ms, 0 | 20260505 11:59:24.622 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:24.622 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:24.622 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 11:59:24.622 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:24.622 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:24.622 | INFO | Uploaded 36GiB to GPU in 2.15807s (16.6816GiB/s) | 20260505 11:59:24.622 | INFO | Downloaded 34GiB to GPU in 2.02237s (16.8119GiB/s) | 20260505 11:59:29.761 | INFO | Analyzing chunk frames = [3036160,3036672) | 20260505 11:59:29.761 | INFO | MemInfo: { | 20260505 11:59:29.761 | INFO | "gpu" : " 17 GB", | 20260505 11:59:29.761 | INFO | "huge" : "", | 20260505 11:59:29.761 | INFO | "huge_shared" : " 27 GB", | 20260505 11:59:29.761 | INFO | "malloc" : " 14 GB", | 20260505 11:59:29.761 | INFO | "pinned" : " 29 GB" | 20260505 11:59:29.761 | INFO | } | 20260505 11:59:29.761 | INFO | Perf report for the last 5.13952 seconds: | 20260505 11:59:29.761 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:59:29.761 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:29.761 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:29.761 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:29.761 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260505 11:59:29.761 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:29.761 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:29.761 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 11:59:29.761 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:29.761 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:29.762 | INFO | Uploaded 36GiB to GPU in 2.22389s (16.1879GiB/s) | 20260505 11:59:29.762 | INFO | Downloaded 34GiB to GPU in 2.12939s (15.967GiB/s) | 20260505 11:59:34.902 | INFO | Analyzing chunk frames = [3036672,3037184) | 20260505 11:59:34.902 | INFO | MemInfo: { | 20260505 11:59:34.902 | INFO | "gpu" : " 17 GB", | 20260505 11:59:34.902 | INFO | "huge" : "", | 20260505 11:59:34.902 | INFO | "huge_shared" : " 27 GB", | 20260505 11:59:34.902 | INFO | "malloc" : " 14 GB", | 20260505 11:59:34.902 | INFO | "pinned" : " 29 GB" | 20260505 11:59:34.902 | INFO | } | 20260505 11:59:34.902 | INFO | Perf report for the last 5.14096 seconds: | 20260505 11:59:34.902 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:59:34.902 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:34.902 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:34.902 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:34.902 | INFO | ANALYSIS : 63.3%, 3.0, 98.3ms, 0 | 20260505 11:59:34.902 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:34.902 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:34.902 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 11:59:34.902 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:34.902 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:34.902 | INFO | Uploaded 36GiB to GPU in 2.2125s (16.2712GiB/s) | 20260505 11:59:34.902 | INFO | Downloaded 34GiB to GPU in 2.12855s (15.9733GiB/s) | 20260505 11:59:40.042 | INFO | Analyzing chunk frames = [3037184,3037696) | 20260505 11:59:40.042 | INFO | MemInfo: { | 20260505 11:59:40.042 | INFO | "gpu" : " 17 GB", | 20260505 11:59:40.042 | INFO | "huge" : "", | 20260505 11:59:40.042 | INFO | "huge_shared" : " 27 GB", | 20260505 11:59:40.042 | INFO | "malloc" : " 14 GB", | 20260505 11:59:40.042 | INFO | "pinned" : " 29 GB" | 20260505 11:59:40.042 | INFO | } | 20260505 11:59:40.042 | INFO | Perf report for the last 5.13982 seconds: | 20260505 11:59:40.042 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:59:40.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:40.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:40.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:40.042 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 11:59:40.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:40.042 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:40.042 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 11:59:40.042 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:40.042 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:40.042 | INFO | Uploaded 36GiB to GPU in 2.15528s (16.7031GiB/s) | 20260505 11:59:40.042 | INFO | Downloaded 34GiB to GPU in 2.12594s (15.9929GiB/s) | 20260505 11:59:45.179 | INFO | Analyzing chunk frames = [3037696,3038208) | 20260505 11:59:45.179 | INFO | MemInfo: { | 20260505 11:59:45.179 | INFO | "gpu" : " 17 GB", | 20260505 11:59:45.179 | INFO | "huge" : "", | 20260505 11:59:45.179 | INFO | "huge_shared" : " 27 GB", | 20260505 11:59:45.179 | INFO | "malloc" : " 14 GB", | 20260505 11:59:45.179 | INFO | "pinned" : " 29 GB" | 20260505 11:59:45.179 | INFO | } | 20260505 11:59:45.179 | INFO | Perf report for the last 5.13718 seconds: | 20260505 11:59:45.179 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:59:45.179 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:45.179 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:45.179 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:45.179 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 11:59:45.179 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:45.179 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:45.179 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 11:59:45.179 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:45.179 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:45.179 | INFO | Uploaded 36GiB to GPU in 2.1766s (16.5396GiB/s) | 20260505 11:59:45.179 | INFO | Downloaded 34GiB to GPU in 2.11603s (16.0678GiB/s) | 20260505 11:59:50.314 | INFO | Analyzing chunk frames = [3038208,3038720) | 20260505 11:59:50.314 | INFO | MemInfo: { | 20260505 11:59:50.314 | INFO | "gpu" : " 17 GB", | 20260505 11:59:50.314 | INFO | "huge" : "", | 20260505 11:59:50.314 | INFO | "huge_shared" : " 27 GB", | 20260505 11:59:50.314 | INFO | "malloc" : " 14 GB", | 20260505 11:59:50.314 | INFO | "pinned" : " 29 GB" | 20260505 11:59:50.314 | INFO | } | 20260505 11:59:50.314 | INFO | Perf report for the last 5.13463 seconds: | 20260505 11:59:50.314 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:59:50.314 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:50.314 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:50.314 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:50.314 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 11:59:50.314 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:50.314 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:50.314 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 11:59:50.314 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:50.314 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:50.314 | INFO | Uploaded 36GiB to GPU in 2.18548s (16.4724GiB/s) | 20260505 11:59:50.314 | INFO | Downloaded 34GiB to GPU in 2.13475s (15.9269GiB/s) | 20260505 11:59:55.460 | INFO | Analyzing chunk frames = [3038720,3039232) | 20260505 11:59:55.460 | INFO | MemInfo: { | 20260505 11:59:55.460 | INFO | "gpu" : " 17 GB", | 20260505 11:59:55.460 | INFO | "huge" : "", | 20260505 11:59:55.460 | INFO | "huge_shared" : " 27 GB", | 20260505 11:59:55.460 | INFO | "malloc" : " 14 GB", | 20260505 11:59:55.460 | INFO | "pinned" : " 29 GB" | 20260505 11:59:55.460 | INFO | } | 20260505 11:59:55.460 | INFO | Perf report for the last 5.1458 seconds: | 20260505 11:59:55.460 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 11:59:55.460 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:55.460 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:55.460 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:55.460 | INFO | ANALYSIS : 63.9%, 3.0, 99.9ms, 0 | 20260505 11:59:55.460 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:55.460 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:55.460 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 11:59:55.460 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:55.460 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 11:59:55.460 | INFO | Uploaded 36GiB to GPU in 2.22072s (16.211GiB/s) | 20260505 11:59:55.460 | INFO | Downloaded 34GiB to GPU in 2.09704s (16.2134GiB/s) | 20260505 11:59:55.460 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 11:59:55.460 | INFO | Total mem usage: 15.3 GiB | 20260505 11:59:55.460 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 11:59:55.460 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -7.51%) | 20260505 11:59:55.460 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 11:59:55.460 | INFO | High Water Breakdown: | 20260505 11:59:55.460 | INFO | 61% : 8.7 GiB ( 91.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 11:59:55.460 | INFO | 38.8% : 5.54 GiB ( 83.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 11:59:55.460 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 11:59:55.460 | INFO | ------------------------------------------ | 20260505 11:59:55.460 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:59:55.460 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 11:59:55.460 | INFO | Total mem usage: 26.5 GiB | 20260505 11:59:55.460 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 11:59:55.460 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 11:59:55.460 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 11:59:55.460 | INFO | High Water Breakdown: | 20260505 11:59:55.460 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:59:55.460 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:59:55.460 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:59:55.460 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:59:55.460 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:59:55.460 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:59:55.460 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 11:59:55.460 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 11:59:55.460 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:59:55.460 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 11:59:55.460 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 11:59:55.461 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 11:59:55.461 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 11:59:55.461 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 11:59:55.461 | INFO | ------------------------------------------ | 20260505 11:59:55.461 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:59:55.461 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 11:59:55.461 | INFO | Total mem usage: 25.6 GiB | 20260505 11:59:55.461 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 11:59:55.461 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 11:59:55.461 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 11:59:55.461 | INFO | High Water Breakdown: | 20260505 11:59:55.461 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 11:59:55.461 | INFO | ------------------------------------------ | 20260505 11:59:55.461 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:59:55.461 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 11:59:55.461 | INFO | Total mem usage: 15.7 GiB | 20260505 11:59:55.461 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 11:59:55.461 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 11:59:55.461 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 11:59:55.461 | INFO | High Water Breakdown: | 20260505 11:59:55.461 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 11:59:55.461 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 11:59:55.461 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 11:59:55.461 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 11:59:55.461 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 11:59:55.461 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 11:59:55.461 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 11:59:55.461 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 11:59:55.461 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 11:59:55.461 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 11:59:55.461 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 11:59:55.461 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 11:59:55.461 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 11:59:55.461 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 11:59:55.461 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 11:59:55.461 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 11:59:55.461 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 11:59:55.461 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 11:59:55.461 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 11:59:55.461 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 11:59:55.461 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 11:59:55.461 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 11:59:55.461 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 11:59:55.461 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 11:59:55.461 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 11:59:55.461 | INFO | ------------------------------------------ | 20260505 11:59:55.461 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 11:59:55.461 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 11:59:55.461 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:59:55.461 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:59:55.461 | INFO | PerfReport: Full Execution -- 3.21e+01 / 7.69e+01, 1.37e+01 / 1.24e+00 | 20260505 11:59:55.461 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 1.01e-01, 1.07e-02 / 3.64e-03 | 20260505 11:59:55.461 | INFO | PerfReport: 11.75%, Baseline -- 3.77e+00 / 4.51e+00, 6.05e-01 / 1.82e-01 | 20260505 11:59:55.461 | INFO | PerfReport: 3.30%, Histogram -- 1.06e+00 / 1.50e+00, 2.05e-01 / 8.61e-02 | 20260505 11:59:55.461 | INFO | PerfReport: 15.81%, DME -- 4.06e+01 / 4.51e+01, 4.32e+00 / 1.77e+00 | 20260505 11:59:55.461 | INFO | PerfReport: 56.48%, FrameLabeling -- 1.81e+01 / 2.09e+01, 1.59e+00 / 2.00e+00 | 20260505 11:59:55.461 | INFO | PerfReport: 7.41%, PulseAccumulating -- 2.38e+00 / 2.86e+00, 4.29e-01 / 1.70e-01 | 20260505 11:59:55.461 | INFO | PerfReport: 3.67%, Metrics -- 1.18e+00 / 1.47e+00, 3.36e-01 / 1.47e-01 | 20260505 11:59:55.461 | INFO | PerfReport: 1.29%, PulseSerializing -- 4.13e-01 / 4.63e-01, 7.49e-02 / 2.93e-02 | 20260505 11:59:55.461 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 11:59:55.461 | INFO | PerfReport: Gathered 1536 samples. | 20260505 11:59:55.461 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 11:59:55.461 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.48e+01 / 0.00e+00 | 20260505 11:59:55.461 | INFO | PerfReport: 22.53%, Upload -- 2.28e+01 / 0.00e+00, 8.11e+00 / 0.00e+00 | 20260505 11:59:55.461 | INFO | PerfReport: 21.98%, Download -- 2.22e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260505 11:59:55.461 | INFO | PerfReport: 31.77%, Compute -- 3.21e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 11:59:55.461 | INFO | PerfReport: 1.98%, LockUpload -- 2.00e+00 / 0.00e+00, 7.58e+00 / 0.00e+00 | 20260505 11:59:55.461 | INFO | PerfReport: 1.36%, LockDownload -- 1.37e+00 / 0.00e+00, 2.94e+00 / 0.00e+00 | 20260505 11:59:55.461 | INFO | PerfReport: 20.37%, LockCompute -- 2.06e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260505 11:59:55.461 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 2.00e-02 / 0.00e+00 | 20260505 11:59:55.591 | INFO | Starting a round of Baz IO | 20260505 11:59:58.836 | INFO | Populating RTMetrics report | 20260505 11:59:58.836 | INFO | RTMetrics report populated | 20260505 11:59:58.836 | INFO | Opening tmp RTMetrics json file | 20260505 11:59:58.837 | INFO | Serializing RTMetrics to json | 20260505 11:59:58.838 | INFO | Serialization done, swapping tmp file now... | 20260505 11:59:58.838 | INFO | RTMetrics done | 20260505 11:59:58.961 | INFO | Finished a round of Baz IO: Wrote 2.355 GiB in 3.369 seconds | 20260505 11:59:58.961 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.83% / 0.01% / 76.16% | 20260505 11:59:58.961 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.991162% / 1.19209e-05% / 9.13938e-05% | 20260505 11:59:58.961 | INFO | Baz IO overall: | 20260505 11:59:58.961 | INFO | BytesWritten : 1802528226743 | 20260505 11:59:58.961 | INFO | NumEvents : 858132808247 | 20260505 12:00:00.600 | INFO | Analyzing chunk frames = [3039232,3039744) | 20260505 12:00:00.600 | INFO | MemInfo: { | 20260505 12:00:00.600 | INFO | "gpu" : " 17 GB", | 20260505 12:00:00.600 | INFO | "huge" : "", | 20260505 12:00:00.600 | INFO | "huge_shared" : " 27 GB", | 20260505 12:00:00.600 | INFO | "malloc" : " 14 GB", | 20260505 12:00:00.600 | INFO | "pinned" : " 29 GB" | 20260505 12:00:00.600 | INFO | } | 20260505 12:00:00.600 | INFO | Perf report for the last 5.14066 seconds: | 20260505 12:00:00.601 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:00:00.601 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:00.601 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:00.601 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:00.601 | INFO | ANALYSIS : 64.6%, 2.9, 99.8ms, 0 | 20260505 12:00:00.601 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:00.601 | INFO | RT_METRICS : 45.3%, 1.0, 23.7ms, 0 | 20260505 12:00:00.601 | INFO | PRE_HQ : 61.0%, 1.9, 58.7ms, 0 | 20260505 12:00:00.601 | INFO | BAZWRITER : 64.9%, 4.1, 135.8ms, 0 | 20260505 12:00:00.601 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 12:00:00.601 | INFO | Uploaded 36GiB to GPU in 2.1782s (16.5274GiB/s) | 20260505 12:00:00.601 | INFO | Downloaded 38GiB to GPU in 2.28474s (16.6321GiB/s) | 20260505 12:00:05.745 | INFO | Analyzing chunk frames = [3039744,3040256) | 20260505 12:00:05.745 | INFO | MemInfo: { | 20260505 12:00:05.745 | INFO | "gpu" : " 17 GB", | 20260505 12:00:05.745 | INFO | "huge" : "", | 20260505 12:00:05.745 | INFO | "huge_shared" : " 27 GB", | 20260505 12:00:05.745 | INFO | "malloc" : " 14 GB", | 20260505 12:00:05.745 | INFO | "pinned" : " 29 GB" | 20260505 12:00:05.745 | INFO | } | 20260505 12:00:05.745 | INFO | Perf report for the last 5.14509 seconds: | 20260505 12:00:05.746 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:00:05.746 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:05.746 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:05.746 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:05.746 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 12:00:05.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:05.746 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:05.746 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 12:00:05.746 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:05.746 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:05.746 | INFO | Uploaded 36GiB to GPU in 2.174s (16.5593GiB/s) | 20260505 12:00:05.746 | INFO | Downloaded 34GiB to GPU in 2.04379s (16.6358GiB/s) | 20260505 12:00:10.875 | INFO | Analyzing chunk frames = [3040256,3040768) | 20260505 12:00:10.876 | INFO | MemInfo: { | 20260505 12:00:10.876 | INFO | "gpu" : " 17 GB", | 20260505 12:00:10.876 | INFO | "huge" : "", | 20260505 12:00:10.876 | INFO | "huge_shared" : " 27 GB", | 20260505 12:00:10.876 | INFO | "malloc" : " 14 GB", | 20260505 12:00:10.876 | INFO | "pinned" : " 29 GB" | 20260505 12:00:10.876 | INFO | } | 20260505 12:00:10.876 | INFO | Perf report for the last 5.13011 seconds: | 20260505 12:00:10.876 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:00:10.876 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:10.876 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:10.876 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:10.876 | INFO | ANALYSIS : 63.4%, 3.0, 98.5ms, 0 | 20260505 12:00:10.876 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:10.876 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:10.876 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 12:00:10.876 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:10.876 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:10.876 | INFO | Uploaded 36GiB to GPU in 2.15935s (16.6717GiB/s) | 20260505 12:00:10.876 | INFO | Downloaded 34GiB to GPU in 2.10858s (16.1246GiB/s) | 20260505 12:00:16.009 | INFO | Analyzing chunk frames = [3040768,3041280) | 20260505 12:00:16.009 | INFO | MemInfo: { | 20260505 12:00:16.009 | INFO | "gpu" : " 17 GB", | 20260505 12:00:16.009 | INFO | "huge" : "", | 20260505 12:00:16.009 | INFO | "huge_shared" : " 27 GB", | 20260505 12:00:16.009 | INFO | "malloc" : " 14 GB", | 20260505 12:00:16.009 | INFO | "pinned" : " 29 GB" | 20260505 12:00:16.009 | INFO | } | 20260505 12:00:16.009 | INFO | Perf report for the last 5.13318 seconds: | 20260505 12:00:16.009 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:00:16.009 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:16.009 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:16.009 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:16.009 | INFO | ANALYSIS : 63.6%, 3.0, 98.6ms, 0 | 20260505 12:00:16.009 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:16.009 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:16.009 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 12:00:16.009 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:16.009 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:16.009 | INFO | Uploaded 36GiB to GPU in 2.21929s (16.2214GiB/s) | 20260505 12:00:16.009 | INFO | Downloaded 34GiB to GPU in 2.13666s (15.9127GiB/s) | 20260505 12:00:21.154 | INFO | Analyzing chunk frames = [3041280,3041792) | 20260505 12:00:21.154 | INFO | MemInfo: { | 20260505 12:00:21.154 | INFO | "gpu" : " 17 GB", | 20260505 12:00:21.154 | INFO | "huge" : "", | 20260505 12:00:21.154 | INFO | "huge_shared" : " 27 GB", | 20260505 12:00:21.154 | INFO | "malloc" : " 14 GB", | 20260505 12:00:21.154 | INFO | "pinned" : " 29 GB" | 20260505 12:00:21.154 | INFO | } | 20260505 12:00:21.155 | INFO | Perf report for the last 5.14568 seconds: | 20260505 12:00:21.155 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:00:21.155 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:21.155 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:21.155 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:21.155 | INFO | ANALYSIS : 63.1%, 3.0, 98.7ms, 0 | 20260505 12:00:21.155 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:21.155 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:21.155 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:00:21.155 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:21.155 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:21.155 | INFO | Uploaded 36GiB to GPU in 2.15661s (16.6928GiB/s) | 20260505 12:00:21.155 | INFO | Downloaded 34GiB to GPU in 2.12597s (15.9927GiB/s) | 20260505 12:00:26.295 | INFO | Analyzing chunk frames = [3041792,3042304) | 20260505 12:00:26.295 | INFO | MemInfo: { | 20260505 12:00:26.295 | INFO | "gpu" : " 17 GB", | 20260505 12:00:26.295 | INFO | "huge" : "", | 20260505 12:00:26.296 | INFO | "huge_shared" : " 27 GB", | 20260505 12:00:26.296 | INFO | "malloc" : " 14 GB", | 20260505 12:00:26.296 | INFO | "pinned" : " 29 GB" | 20260505 12:00:26.296 | INFO | } | 20260505 12:00:26.296 | INFO | Perf report for the last 5.14108 seconds: | 20260505 12:00:26.296 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:00:26.296 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:26.296 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:26.296 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:26.296 | INFO | ANALYSIS : 63.2%, 3.0, 98.8ms, 0 | 20260505 12:00:26.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:26.296 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:26.296 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 12:00:26.296 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:26.296 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:26.296 | INFO | Uploaded 36GiB to GPU in 2.16678s (16.6145GiB/s) | 20260505 12:00:26.296 | INFO | Downloaded 34GiB to GPU in 2.11191s (16.0992GiB/s) | 20260505 12:00:31.441 | INFO | Analyzing chunk frames = [3042304,3042816) | 20260505 12:00:31.441 | INFO | MemInfo: { | 20260505 12:00:31.441 | INFO | "gpu" : " 17 GB", | 20260505 12:00:31.441 | INFO | "huge" : "", | 20260505 12:00:31.442 | INFO | "huge_shared" : " 27 GB", | 20260505 12:00:31.442 | INFO | "malloc" : " 14 GB", | 20260505 12:00:31.442 | INFO | "pinned" : " 29 GB" | 20260505 12:00:31.442 | INFO | } | 20260505 12:00:31.442 | INFO | Perf report for the last 5.14597 seconds: | 20260505 12:00:31.442 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:00:31.442 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:31.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:31.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:31.442 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 12:00:31.442 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:31.442 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:31.442 | INFO | PRE_HQ : 29.9%, 1.0, 15.6ms, 0 | 20260505 12:00:31.442 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:31.442 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:31.442 | INFO | Uploaded 36GiB to GPU in 2.13426s (16.8676GiB/s) | 20260505 12:00:31.442 | INFO | Downloaded 34GiB to GPU in 2.13341s (15.9369GiB/s) | 20260505 12:00:36.575 | INFO | Analyzing chunk frames = [3042816,3043328) | 20260505 12:00:36.575 | INFO | MemInfo: { | 20260505 12:00:36.575 | INFO | "gpu" : " 17 GB", | 20260505 12:00:36.575 | INFO | "huge" : "", | 20260505 12:00:36.575 | INFO | "huge_shared" : " 27 GB", | 20260505 12:00:36.575 | INFO | "malloc" : " 14 GB", | 20260505 12:00:36.575 | INFO | "pinned" : " 29 GB" | 20260505 12:00:36.575 | INFO | } | 20260505 12:00:36.575 | INFO | Perf report for the last 5.13351 seconds: | 20260505 12:00:36.575 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:00:36.575 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:36.575 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:36.575 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:36.575 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 12:00:36.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:36.575 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:36.575 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 12:00:36.575 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:36.575 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:36.575 | INFO | Uploaded 36GiB to GPU in 2.19432s (16.406GiB/s) | 20260505 12:00:36.575 | INFO | Downloaded 34GiB to GPU in 2.09618s (16.22GiB/s) | 20260505 12:00:36.707 | INFO | Starting a round of Baz IO | 20260505 12:00:39.959 | INFO | Populating RTMetrics report | 20260505 12:00:39.960 | INFO | RTMetrics report populated | 20260505 12:00:39.960 | INFO | Opening tmp RTMetrics json file | 20260505 12:00:39.960 | INFO | Serializing RTMetrics to json | 20260505 12:00:39.961 | INFO | Serialization done, swapping tmp file now... | 20260505 12:00:39.962 | INFO | RTMetrics done | 20260505 12:00:40.085 | INFO | Finished a round of Baz IO: Wrote 2.353 GiB in 3.377 seconds | 20260505 12:00:40.085 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.82% / 0.01% / 76.17% | 20260505 12:00:40.085 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.987756% / 0% / 9.13938e-05% | 20260505 12:00:40.085 | INFO | Baz IO overall: | 20260505 12:00:40.085 | INFO | BytesWritten : 1805055002108 | 20260505 12:00:40.085 | INFO | NumEvents : 859332723150 | 20260505 12:00:41.719 | INFO | Analyzing chunk frames = [3043328,3043840) | 20260505 12:00:41.719 | INFO | MemInfo: { | 20260505 12:00:41.719 | INFO | "gpu" : " 17 GB", | 20260505 12:00:41.719 | INFO | "huge" : "", | 20260505 12:00:41.719 | INFO | "huge_shared" : " 27 GB", | 20260505 12:00:41.719 | INFO | "malloc" : " 14 GB", | 20260505 12:00:41.720 | INFO | "pinned" : " 29 GB" | 20260505 12:00:41.720 | INFO | } | 20260505 12:00:41.720 | INFO | Perf report for the last 5.14445 seconds: | 20260505 12:00:41.720 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:00:41.720 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:41.720 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:41.720 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:41.720 | INFO | ANALYSIS : 64.7%, 3.0, 100.3ms, 0 | 20260505 12:00:41.720 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:41.720 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260505 12:00:41.720 | INFO | PRE_HQ : 60.9%, 1.8, 58.3ms, 0 | 20260505 12:00:41.720 | INFO | BAZWRITER : 65.0%, 4.0, 134.5ms, 0 | 20260505 12:00:41.720 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 12:00:41.720 | INFO | Uploaded 36GiB to GPU in 2.18896s (16.4461GiB/s) | 20260505 12:00:41.720 | INFO | Downloaded 38GiB to GPU in 2.2879s (16.6091GiB/s) | 20260505 12:00:46.853 | INFO | Analyzing chunk frames = [3043840,3044352) | 20260505 12:00:46.854 | INFO | MemInfo: { | 20260505 12:00:46.854 | INFO | "gpu" : " 17 GB", | 20260505 12:00:46.854 | INFO | "huge" : "", | 20260505 12:00:46.854 | INFO | "huge_shared" : " 27 GB", | 20260505 12:00:46.854 | INFO | "malloc" : " 14 GB", | 20260505 12:00:46.854 | INFO | "pinned" : " 29 GB" | 20260505 12:00:46.854 | INFO | } | 20260505 12:00:46.854 | INFO | Perf report for the last 5.13398 seconds: | 20260505 12:00:46.854 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:00:46.854 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:46.854 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:46.854 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:46.854 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 12:00:46.854 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:46.854 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:46.854 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:00:46.854 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:46.854 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:46.854 | INFO | Uploaded 36GiB to GPU in 2.16863s (16.6003GiB/s) | 20260505 12:00:46.854 | INFO | Downloaded 34GiB to GPU in 2.0294s (16.7538GiB/s) | 20260505 12:00:51.993 | INFO | Analyzing chunk frames = [3044352,3044864) | 20260505 12:00:51.993 | INFO | MemInfo: { | 20260505 12:00:51.993 | INFO | "gpu" : " 17 GB", | 20260505 12:00:51.993 | INFO | "huge" : "", | 20260505 12:00:51.993 | INFO | "huge_shared" : " 27 GB", | 20260505 12:00:51.993 | INFO | "malloc" : " 14 GB", | 20260505 12:00:51.993 | INFO | "pinned" : " 29 GB" | 20260505 12:00:51.993 | INFO | } | 20260505 12:00:51.993 | INFO | Perf report for the last 5.13955 seconds: | 20260505 12:00:51.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:00:51.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:51.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:51.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:51.993 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 12:00:51.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:51.993 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:51.993 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 12:00:51.993 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:51.993 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:51.993 | INFO | Uploaded 36GiB to GPU in 2.19555s (16.3968GiB/s) | 20260505 12:00:51.993 | INFO | Downloaded 34GiB to GPU in 2.12268s (16.0175GiB/s) | 20260505 12:00:57.126 | INFO | Analyzing chunk frames = [3044864,3045376) | 20260505 12:00:57.126 | INFO | MemInfo: { | 20260505 12:00:57.126 | INFO | "gpu" : " 17 GB", | 20260505 12:00:57.126 | INFO | "huge" : "", | 20260505 12:00:57.126 | INFO | "huge_shared" : " 27 GB", | 20260505 12:00:57.126 | INFO | "malloc" : " 14 GB", | 20260505 12:00:57.126 | INFO | "pinned" : " 29 GB" | 20260505 12:00:57.126 | INFO | } | 20260505 12:00:57.126 | INFO | Perf report for the last 5.13325 seconds: | 20260505 12:00:57.126 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:00:57.126 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:57.126 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:57.127 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:57.127 | INFO | ANALYSIS : 63.3%, 3.0, 98.1ms, 0 | 20260505 12:00:57.127 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:57.127 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:57.127 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 12:00:57.127 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:57.127 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:00:57.127 | INFO | Uploaded 36GiB to GPU in 2.20846s (16.301GiB/s) | 20260505 12:00:57.127 | INFO | Downloaded 34GiB to GPU in 2.13112s (15.9541GiB/s) | 20260505 12:01:02.263 | INFO | Analyzing chunk frames = [3045376,3045888) | 20260505 12:01:02.263 | INFO | MemInfo: { | 20260505 12:01:02.263 | INFO | "gpu" : " 17 GB", | 20260505 12:01:02.263 | INFO | "huge" : "", | 20260505 12:01:02.263 | INFO | "huge_shared" : " 27 GB", | 20260505 12:01:02.263 | INFO | "malloc" : " 14 GB", | 20260505 12:01:02.263 | INFO | "pinned" : " 29 GB" | 20260505 12:01:02.263 | INFO | } | 20260505 12:01:02.263 | INFO | Perf report for the last 5.13644 seconds: | 20260505 12:01:02.263 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:01:02.263 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:02.263 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:02.263 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:02.263 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 12:01:02.263 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:02.263 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:02.263 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 12:01:02.263 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:02.263 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:02.263 | INFO | Uploaded 36GiB to GPU in 2.14613s (16.7744GiB/s) | 20260505 12:01:02.263 | INFO | Downloaded 34GiB to GPU in 2.12708s (15.9843GiB/s) | 20260505 12:01:07.414 | INFO | Analyzing chunk frames = [3045888,3046400) | 20260505 12:01:07.414 | INFO | MemInfo: { | 20260505 12:01:07.414 | INFO | "gpu" : " 17 GB", | 20260505 12:01:07.414 | INFO | "huge" : "", | 20260505 12:01:07.414 | INFO | "huge_shared" : " 27 GB", | 20260505 12:01:07.414 | INFO | "malloc" : " 14 GB", | 20260505 12:01:07.414 | INFO | "pinned" : " 29 GB" | 20260505 12:01:07.414 | INFO | } | 20260505 12:01:07.414 | INFO | Perf report for the last 5.15134 seconds: | 20260505 12:01:07.414 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:01:07.414 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:07.414 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:07.414 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:07.414 | INFO | ANALYSIS : 63.3%, 3.0, 99.1ms, 0 | 20260505 12:01:07.414 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:07.414 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:07.414 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 12:01:07.414 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:07.414 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:07.414 | INFO | Uploaded 36GiB to GPU in 2.19146s (16.4274GiB/s) | 20260505 12:01:07.414 | INFO | Downloaded 34GiB to GPU in 2.12383s (16.0088GiB/s) | 20260505 12:01:12.560 | INFO | Analyzing chunk frames = [3046400,3046912) | 20260505 12:01:12.560 | INFO | MemInfo: { | 20260505 12:01:12.560 | INFO | "gpu" : " 17 GB", | 20260505 12:01:12.560 | INFO | "huge" : "", | 20260505 12:01:12.560 | INFO | "huge_shared" : " 27 GB", | 20260505 12:01:12.560 | INFO | "malloc" : " 14 GB", | 20260505 12:01:12.560 | INFO | "pinned" : " 29 GB" | 20260505 12:01:12.560 | INFO | } | 20260505 12:01:12.560 | INFO | Perf report for the last 5.14626 seconds: | 20260505 12:01:12.560 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:01:12.560 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:12.560 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:12.560 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:12.560 | INFO | ANALYSIS : 64.3%, 3.0, 100.5ms, 0 | 20260505 12:01:12.561 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:12.561 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:12.561 | INFO | PRE_HQ : 30.4%, 1.0, 15.9ms, 0 | 20260505 12:01:12.561 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:12.561 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:12.561 | INFO | Uploaded 36GiB to GPU in 2.20145s (16.3529GiB/s) | 20260505 12:01:12.561 | INFO | Downloaded 34GiB to GPU in 2.20103s (15.4473GiB/s) | 20260505 12:01:17.688 | INFO | Analyzing chunk frames = [3046912,3047424) | 20260505 12:01:17.688 | INFO | MemInfo: { | 20260505 12:01:17.688 | INFO | "gpu" : " 17 GB", | 20260505 12:01:17.688 | INFO | "huge" : "", | 20260505 12:01:17.688 | INFO | "huge_shared" : " 27 GB", | 20260505 12:01:17.688 | INFO | "malloc" : " 14 GB", | 20260505 12:01:17.688 | INFO | "pinned" : " 29 GB" | 20260505 12:01:17.688 | INFO | } | 20260505 12:01:17.688 | INFO | Perf report for the last 5.12755 seconds: | 20260505 12:01:17.688 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:01:17.688 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:17.688 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:17.688 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:17.688 | INFO | ANALYSIS : 63.9%, 3.0, 99.5ms, 0 | 20260505 12:01:17.688 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:17.688 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:17.688 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 12:01:17.688 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:17.688 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:17.688 | INFO | Uploaded 36GiB to GPU in 2.21511s (16.252GiB/s) | 20260505 12:01:17.688 | INFO | Downloaded 34GiB to GPU in 2.10479s (16.1537GiB/s) | 20260505 12:01:17.689 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 12:01:17.689 | INFO | Total mem usage: 15.3 GiB | 20260505 12:01:17.689 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 12:01:17.689 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -6.81%) | 20260505 12:01:17.689 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 12:01:17.689 | INFO | High Water Breakdown: | 20260505 12:01:17.689 | INFO | 61.1% : 8.77 GiB ( 92.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 12:01:17.689 | INFO | 38.8% : 5.56 GiB ( 83.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 12:01:17.689 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 12:01:17.689 | INFO | ------------------------------------------ | 20260505 12:01:17.689 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:01:17.689 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 12:01:17.689 | INFO | Total mem usage: 26.5 GiB | 20260505 12:01:17.689 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 12:01:17.689 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 12:01:17.689 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 12:01:17.689 | INFO | High Water Breakdown: | 20260505 12:01:17.689 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:01:17.689 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:01:17.689 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:01:17.689 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:01:17.689 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:01:17.689 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:01:17.689 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 12:01:17.689 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 12:01:17.689 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:01:17.689 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 12:01:17.689 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 12:01:17.689 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 12:01:17.689 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 12:01:17.689 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 12:01:17.689 | INFO | ------------------------------------------ | 20260505 12:01:17.689 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:01:17.689 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 12:01:17.689 | INFO | Total mem usage: 25.6 GiB | 20260505 12:01:17.689 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 12:01:17.689 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 12:01:17.689 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 12:01:17.689 | INFO | High Water Breakdown: | 20260505 12:01:17.689 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 12:01:17.689 | INFO | ------------------------------------------ | 20260505 12:01:17.689 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:01:17.689 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 12:01:17.689 | INFO | Total mem usage: 15.7 GiB | 20260505 12:01:17.689 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 12:01:17.689 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 12:01:17.689 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 12:01:17.689 | INFO | High Water Breakdown: | 20260505 12:01:17.689 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 12:01:17.689 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 12:01:17.689 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 12:01:17.689 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 12:01:17.689 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 12:01:17.689 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 12:01:17.689 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 12:01:17.689 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 12:01:17.689 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 12:01:17.689 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:01:17.689 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 12:01:17.689 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:01:17.689 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 12:01:17.689 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 12:01:17.689 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 12:01:17.689 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 12:01:17.689 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:01:17.689 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 12:01:17.689 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:01:17.689 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:01:17.689 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:01:17.689 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 12:01:17.689 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 12:01:17.689 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:01:17.689 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 12:01:17.689 | INFO | ------------------------------------------ | 20260505 12:01:17.689 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:01:17.689 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 12:01:17.689 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:01:17.689 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:01:17.689 | INFO | PerfReport: Full Execution -- 3.21e+01 / 7.79e+01, 1.37e+01 / 1.98e+00 | 20260505 12:01:17.689 | INFO | PerfReport: 0.29%, AggregateStats -- 9.30e-02 / 1.01e-01, 1.10e-02 / 6.19e-03 | 20260505 12:01:17.689 | INFO | PerfReport: 11.73%, Baseline -- 3.76e+00 / 4.48e+00, 6.00e-01 / 1.32e-01 | 20260505 12:01:17.689 | INFO | PerfReport: 3.29%, Histogram -- 1.06e+00 / 1.50e+00, 2.03e-01 / 9.00e-02 | 20260505 12:01:17.689 | INFO | PerfReport: 16.10%, DME -- 4.13e+01 / 4.70e+01, 5.14e+00 / 2.45e+00 | 20260505 12:01:17.689 | INFO | PerfReport: 56.25%, FrameLabeling -- 1.80e+01 / 1.99e+01, 1.46e+00 / 1.80e+00 | 20260505 12:01:17.689 | INFO | PerfReport: 7.43%, PulseAccumulating -- 2.38e+00 / 2.97e+00, 4.50e-01 / 2.07e-01 | 20260505 12:01:17.689 | INFO | PerfReport: 3.62%, Metrics -- 1.16e+00 / 1.48e+00, 3.27e-01 / 2.01e-01 | 20260505 12:01:17.689 | INFO | PerfReport: 1.28%, PulseSerializing -- 4.11e-01 / 4.56e-01, 7.45e-02 / 4.16e-02 | 20260505 12:01:17.689 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 12:01:17.689 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:01:17.689 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:01:17.689 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.56e+01 / 0.00e+00 | 20260505 12:01:17.689 | INFO | PerfReport: 22.45%, Upload -- 2.27e+01 / 0.00e+00, 8.00e+00 / 0.00e+00 | 20260505 12:01:17.689 | INFO | PerfReport: 21.99%, Download -- 2.22e+01 / 0.00e+00, 8.11e+00 / 0.00e+00 | 20260505 12:01:17.689 | INFO | PerfReport: 31.71%, Compute -- 3.21e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 12:01:17.689 | INFO | PerfReport: 2.15%, LockUpload -- 2.17e+00 / 0.00e+00, 7.87e+00 / 0.00e+00 | 20260505 12:01:17.689 | INFO | PerfReport: 1.42%, LockDownload -- 1.44e+00 / 0.00e+00, 3.00e+00 / 0.00e+00 | 20260505 12:01:17.689 | INFO | PerfReport: 20.26%, LockCompute -- 2.05e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260505 12:01:17.689 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.17e-02 / 0.00e+00 | 20260505 12:01:17.827 | INFO | Starting a round of Baz IO | 20260505 12:01:21.078 | INFO | Populating RTMetrics report | 20260505 12:01:21.079 | INFO | RTMetrics report populated | 20260505 12:01:21.079 | INFO | Opening tmp RTMetrics json file | 20260505 12:01:21.126 | INFO | Serializing RTMetrics to json | 20260505 12:01:21.153 | INFO | Serialization done, swapping tmp file now... | 20260505 12:01:21.155 | INFO | RTMetrics done | 20260505 12:01:21.186 | INFO | Finished a round of Baz IO: Wrote 2.352 GiB in 3.358 seconds | 20260505 12:01:21.186 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.82% / 0.01% / 76.18% | 20260505 12:01:21.186 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.983516% / 1.19209e-05% / 8.34465e-05% | 20260505 12:01:21.186 | INFO | Baz IO overall: | 20260505 12:01:21.186 | INFO | BytesWritten : 1807580337838 | 20260505 12:01:21.186 | INFO | NumEvents : 860531622079 | 20260505 12:01:22.822 | INFO | Analyzing chunk frames = [3047424,3047936) | 20260505 12:01:22.822 | INFO | MemInfo: { | 20260505 12:01:22.822 | INFO | "gpu" : " 17 GB", | 20260505 12:01:22.822 | INFO | "huge" : "", | 20260505 12:01:22.823 | INFO | "huge_shared" : " 27 GB", | 20260505 12:01:22.823 | INFO | "malloc" : " 14 GB", | 20260505 12:01:22.823 | INFO | "pinned" : " 29 GB" | 20260505 12:01:22.823 | INFO | } | 20260505 12:01:22.823 | INFO | Perf report for the last 5.13454 seconds: | 20260505 12:01:22.823 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:01:22.823 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:22.823 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:22.823 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:22.823 | INFO | ANALYSIS : 64.9%, 2.9, 100.3ms, 0 | 20260505 12:01:22.823 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:22.823 | INFO | RT_METRICS : 46.8%, 1.0, 24.5ms, 0 | 20260505 12:01:22.823 | INFO | PRE_HQ : 61.3%, 1.8, 58.6ms, 0 | 20260505 12:01:22.823 | INFO | BAZWRITER : 64.8%, 4.0, 133.5ms, 0 | 20260505 12:01:22.823 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260505 12:01:22.823 | INFO | Uploaded 36GiB to GPU in 2.1961s (16.3927GiB/s) | 20260505 12:01:22.823 | INFO | Downloaded 38GiB to GPU in 2.29735s (16.5408GiB/s) | 20260505 12:01:27.967 | INFO | Analyzing chunk frames = [3047936,3048448) | 20260505 12:01:27.967 | INFO | MemInfo: { | 20260505 12:01:27.967 | INFO | "gpu" : " 17 GB", | 20260505 12:01:27.967 | INFO | "huge" : "", | 20260505 12:01:27.967 | INFO | "huge_shared" : " 27 GB", | 20260505 12:01:27.967 | INFO | "malloc" : " 14 GB", | 20260505 12:01:27.967 | INFO | "pinned" : " 29 GB" | 20260505 12:01:27.967 | INFO | } | 20260505 12:01:27.967 | INFO | Perf report for the last 5.14458 seconds: | 20260505 12:01:27.967 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:01:27.967 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:27.967 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:27.967 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:27.967 | INFO | ANALYSIS : 63.9%, 3.0, 99.9ms, 0 | 20260505 12:01:27.967 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:27.967 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:27.967 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 12:01:27.967 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:27.967 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:27.968 | INFO | Uploaded 36GiB to GPU in 2.22876s (16.1525GiB/s) | 20260505 12:01:27.968 | INFO | Downloaded 34GiB to GPU in 2.08495s (16.3073GiB/s) | 20260505 12:01:33.105 | INFO | Analyzing chunk frames = [3048448,3048960) | 20260505 12:01:33.105 | INFO | MemInfo: { | 20260505 12:01:33.105 | INFO | "gpu" : " 17 GB", | 20260505 12:01:33.105 | INFO | "huge" : "", | 20260505 12:01:33.105 | INFO | "huge_shared" : " 27 GB", | 20260505 12:01:33.105 | INFO | "malloc" : " 14 GB", | 20260505 12:01:33.105 | INFO | "pinned" : " 29 GB" | 20260505 12:01:33.105 | INFO | } | 20260505 12:01:33.105 | INFO | Perf report for the last 5.13778 seconds: | 20260505 12:01:33.105 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:01:33.105 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:33.105 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:33.105 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:33.105 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 12:01:33.105 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:33.105 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:33.105 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 12:01:33.105 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:33.105 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:33.105 | INFO | Uploaded 36GiB to GPU in 2.20953s (16.2931GiB/s) | 20260505 12:01:33.105 | INFO | Downloaded 34GiB to GPU in 2.12307s (16.0146GiB/s) | 20260505 12:01:38.247 | INFO | Analyzing chunk frames = [3048960,3049472) | 20260505 12:01:38.247 | INFO | MemInfo: { | 20260505 12:01:38.247 | INFO | "gpu" : " 17 GB", | 20260505 12:01:38.247 | INFO | "huge" : "", | 20260505 12:01:38.247 | INFO | "huge_shared" : " 27 GB", | 20260505 12:01:38.247 | INFO | "malloc" : " 14 GB", | 20260505 12:01:38.247 | INFO | "pinned" : " 29 GB" | 20260505 12:01:38.247 | INFO | } | 20260505 12:01:38.247 | INFO | Perf report for the last 5.14233 seconds: | 20260505 12:01:38.247 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:01:38.247 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:38.247 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:38.247 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:38.247 | INFO | ANALYSIS : 63.2%, 3.0, 98.3ms, 0 | 20260505 12:01:38.247 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:38.247 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:38.247 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 12:01:38.247 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:38.247 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:38.248 | INFO | Uploaded 36GiB to GPU in 2.20651s (16.3153GiB/s) | 20260505 12:01:38.248 | INFO | Downloaded 34GiB to GPU in 2.13192s (15.9481GiB/s) | 20260505 12:01:43.388 | INFO | Analyzing chunk frames = [3049472,3049984) | 20260505 12:01:43.388 | INFO | MemInfo: { | 20260505 12:01:43.388 | INFO | "gpu" : " 17 GB", | 20260505 12:01:43.388 | INFO | "huge" : "", | 20260505 12:01:43.388 | INFO | "huge_shared" : " 27 GB", | 20260505 12:01:43.388 | INFO | "malloc" : " 14 GB", | 20260505 12:01:43.388 | INFO | "pinned" : " 29 GB" | 20260505 12:01:43.388 | INFO | } | 20260505 12:01:43.388 | INFO | Perf report for the last 5.14065 seconds: | 20260505 12:01:43.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:01:43.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:43.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:43.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:43.388 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 12:01:43.388 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:43.388 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:43.388 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 12:01:43.388 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:43.388 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:43.388 | INFO | Uploaded 36GiB to GPU in 2.17622s (16.5425GiB/s) | 20260505 12:01:43.388 | INFO | Downloaded 34GiB to GPU in 2.12684s (15.9861GiB/s) | 20260505 12:01:48.522 | INFO | Analyzing chunk frames = [3049984,3050496) | 20260505 12:01:48.522 | INFO | MemInfo: { | 20260505 12:01:48.522 | INFO | "gpu" : " 17 GB", | 20260505 12:01:48.522 | INFO | "huge" : "", | 20260505 12:01:48.522 | INFO | "huge_shared" : " 27 GB", | 20260505 12:01:48.522 | INFO | "malloc" : " 14 GB", | 20260505 12:01:48.522 | INFO | "pinned" : " 29 GB" | 20260505 12:01:48.522 | INFO | } | 20260505 12:01:48.522 | INFO | Perf report for the last 5.13451 seconds: | 20260505 12:01:48.522 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:01:48.522 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:48.522 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:48.522 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:48.522 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 12:01:48.522 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:48.523 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:48.523 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 12:01:48.523 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:48.523 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:48.523 | INFO | Uploaded 36GiB to GPU in 2.16212s (16.6503GiB/s) | 20260505 12:01:48.523 | INFO | Downloaded 34GiB to GPU in 2.1215s (16.0264GiB/s) | 20260505 12:01:53.672 | INFO | Analyzing chunk frames = [3050496,3051008) | 20260505 12:01:53.672 | INFO | MemInfo: { | 20260505 12:01:53.672 | INFO | "gpu" : " 17 GB", | 20260505 12:01:53.672 | INFO | "huge" : "", | 20260505 12:01:53.672 | INFO | "huge_shared" : " 27 GB", | 20260505 12:01:53.672 | INFO | "malloc" : " 14 GB", | 20260505 12:01:53.672 | INFO | "pinned" : " 29 GB" | 20260505 12:01:53.672 | INFO | } | 20260505 12:01:53.672 | INFO | Perf report for the last 5.14932 seconds: | 20260505 12:01:53.672 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:01:53.672 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:53.672 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:53.672 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:53.672 | INFO | ANALYSIS : 63.3%, 3.0, 99.1ms, 0 | 20260505 12:01:53.672 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:53.672 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:53.672 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 12:01:53.672 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:53.672 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:53.672 | INFO | Uploaded 36GiB to GPU in 2.1787s (16.5236GiB/s) | 20260505 12:01:53.672 | INFO | Downloaded 34GiB to GPU in 2.13461s (15.928GiB/s) | 20260505 12:01:58.812 | INFO | Analyzing chunk frames = [3051008,3051520) | 20260505 12:01:58.812 | INFO | MemInfo: { | 20260505 12:01:58.812 | INFO | "gpu" : " 17 GB", | 20260505 12:01:58.812 | INFO | "huge" : "", | 20260505 12:01:58.812 | INFO | "huge_shared" : " 27 GB", | 20260505 12:01:58.812 | INFO | "malloc" : " 14 GB", | 20260505 12:01:58.812 | INFO | "pinned" : " 29 GB" | 20260505 12:01:58.812 | INFO | } | 20260505 12:01:58.812 | INFO | Perf report for the last 5.13995 seconds: | 20260505 12:01:58.812 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:01:58.812 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:58.812 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:58.812 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:58.812 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 12:01:58.812 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:58.812 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:58.812 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 12:01:58.812 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:58.812 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:01:58.812 | INFO | Uploaded 36GiB to GPU in 2.12634s (16.9305GiB/s) | 20260505 12:01:58.812 | INFO | Downloaded 34GiB to GPU in 2.10117s (16.1815GiB/s) | 20260505 12:01:58.954 | INFO | Starting a round of Baz IO | 20260505 12:02:02.179 | INFO | Populating RTMetrics report | 20260505 12:02:02.179 | INFO | RTMetrics report populated | 20260505 12:02:02.179 | INFO | Opening tmp RTMetrics json file | 20260505 12:02:02.181 | INFO | Serializing RTMetrics to json | 20260505 12:02:02.204 | INFO | Serialization done, swapping tmp file now... | 20260505 12:02:02.225 | INFO | RTMetrics done | 20260505 12:02:02.289 | INFO | Finished a round of Baz IO: Wrote 2.352 GiB in 3.335 seconds | 20260505 12:02:02.289 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.81% / 0.01% / 76.18% | 20260505 12:02:02.289 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.980417% / 3.97364e-06% / 7.15256e-05% | 20260505 12:02:02.289 | INFO | Baz IO overall: | 20260505 12:02:02.289 | INFO | BytesWritten : 1810105959306 | 20260505 12:02:02.289 | INFO | NumEvents : 861730745990 | 20260505 12:02:03.936 | INFO | Analyzing chunk frames = [3051520,3052032) | 20260505 12:02:03.936 | INFO | MemInfo: { | 20260505 12:02:03.936 | INFO | "gpu" : " 17 GB", | 20260505 12:02:03.936 | INFO | "huge" : "", | 20260505 12:02:03.936 | INFO | "huge_shared" : " 27 GB", | 20260505 12:02:03.936 | INFO | "malloc" : " 14 GB", | 20260505 12:02:03.936 | INFO | "pinned" : " 29 GB" | 20260505 12:02:03.936 | INFO | } | 20260505 12:02:03.936 | INFO | Perf report for the last 5.12461 seconds: | 20260505 12:02:03.936 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:02:03.936 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:03.936 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:03.936 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:03.936 | INFO | ANALYSIS : 64.6%, 3.0, 99.7ms, 0 | 20260505 12:02:03.936 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:03.936 | INFO | RT_METRICS : 46.3%, 1.0, 24.2ms, 0 | 20260505 12:02:03.937 | INFO | PRE_HQ : 61.4%, 1.8, 58.3ms, 0 | 20260505 12:02:03.937 | INFO | BAZWRITER : 64.4%, 4.1, 135.4ms, 0 | 20260505 12:02:03.937 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260505 12:02:03.937 | INFO | Uploaded 36GiB to GPU in 2.18349s (16.4873GiB/s) | 20260505 12:02:03.937 | INFO | Downloaded 38GiB to GPU in 2.27548s (16.6998GiB/s) | 20260505 12:02:09.080 | INFO | Analyzing chunk frames = [3052032,3052544) | 20260505 12:02:09.080 | INFO | MemInfo: { | 20260505 12:02:09.080 | INFO | "gpu" : " 17 GB", | 20260505 12:02:09.080 | INFO | "huge" : "", | 20260505 12:02:09.080 | INFO | "huge_shared" : " 27 GB", | 20260505 12:02:09.080 | INFO | "malloc" : " 14 GB", | 20260505 12:02:09.080 | INFO | "pinned" : " 29 GB" | 20260505 12:02:09.080 | INFO | } | 20260505 12:02:09.080 | INFO | Perf report for the last 5.14361 seconds: | 20260505 12:02:09.080 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:02:09.080 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:09.080 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:09.080 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:09.080 | INFO | ANALYSIS : 64.1%, 3.0, 100.2ms, 0 | 20260505 12:02:09.080 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:09.080 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:09.080 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 12:02:09.080 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:09.080 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:09.080 | INFO | Uploaded 36GiB to GPU in 2.23629s (16.0981GiB/s) | 20260505 12:02:09.080 | INFO | Downloaded 34GiB to GPU in 2.08761s (16.2866GiB/s) | 20260505 12:02:14.221 | INFO | Analyzing chunk frames = [3052544,3053056) | 20260505 12:02:14.221 | INFO | MemInfo: { | 20260505 12:02:14.221 | INFO | "gpu" : " 17 GB", | 20260505 12:02:14.221 | INFO | "huge" : "", | 20260505 12:02:14.221 | INFO | "huge_shared" : " 27 GB", | 20260505 12:02:14.221 | INFO | "malloc" : " 14 GB", | 20260505 12:02:14.221 | INFO | "pinned" : " 29 GB" | 20260505 12:02:14.222 | INFO | } | 20260505 12:02:14.222 | INFO | Perf report for the last 5.14154 seconds: | 20260505 12:02:14.222 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:02:14.222 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:14.222 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:14.222 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:14.222 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 12:02:14.222 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:14.222 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:14.222 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 12:02:14.222 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:14.222 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:14.222 | INFO | Uploaded 36GiB to GPU in 2.21203s (16.2746GiB/s) | 20260505 12:02:14.222 | INFO | Downloaded 34GiB to GPU in 2.11821s (16.0513GiB/s) | 20260505 12:02:19.362 | INFO | Analyzing chunk frames = [3053056,3053568) | 20260505 12:02:19.362 | INFO | MemInfo: { | 20260505 12:02:19.362 | INFO | "gpu" : " 17 GB", | 20260505 12:02:19.362 | INFO | "huge" : "", | 20260505 12:02:19.362 | INFO | "huge_shared" : " 27 GB", | 20260505 12:02:19.362 | INFO | "malloc" : " 14 GB", | 20260505 12:02:19.362 | INFO | "pinned" : " 29 GB" | 20260505 12:02:19.362 | INFO | } | 20260505 12:02:19.362 | INFO | Perf report for the last 5.14059 seconds: | 20260505 12:02:19.362 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:02:19.362 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:19.362 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:19.362 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:19.362 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 12:02:19.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:19.362 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:19.362 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:02:19.362 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:19.362 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:19.362 | INFO | Uploaded 36GiB to GPU in 2.20702s (16.3116GiB/s) | 20260505 12:02:19.362 | INFO | Downloaded 34GiB to GPU in 2.13055s (15.9583GiB/s) | 20260505 12:02:24.503 | INFO | Analyzing chunk frames = [3053568,3054080) | 20260505 12:02:24.503 | INFO | MemInfo: { | 20260505 12:02:24.503 | INFO | "gpu" : " 17 GB", | 20260505 12:02:24.503 | INFO | "huge" : "", | 20260505 12:02:24.503 | INFO | "huge_shared" : " 27 GB", | 20260505 12:02:24.503 | INFO | "malloc" : " 14 GB", | 20260505 12:02:24.503 | INFO | "pinned" : " 29 GB" | 20260505 12:02:24.503 | INFO | } | 20260505 12:02:24.503 | INFO | Perf report for the last 5.14057 seconds: | 20260505 12:02:24.503 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:02:24.503 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:24.503 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:24.503 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:24.503 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 12:02:24.503 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:24.503 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:24.503 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:02:24.503 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:24.503 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:24.503 | INFO | Uploaded 36GiB to GPU in 2.14123s (16.8128GiB/s) | 20260505 12:02:24.503 | INFO | Downloaded 34GiB to GPU in 2.12502s (15.9999GiB/s) | 20260505 12:02:29.641 | INFO | Analyzing chunk frames = [3054080,3054592) | 20260505 12:02:29.641 | INFO | MemInfo: { | 20260505 12:02:29.641 | INFO | "gpu" : " 17 GB", | 20260505 12:02:29.642 | INFO | "huge" : "", | 20260505 12:02:29.642 | INFO | "huge_shared" : " 27 GB", | 20260505 12:02:29.642 | INFO | "malloc" : " 14 GB", | 20260505 12:02:29.642 | INFO | "pinned" : " 29 GB" | 20260505 12:02:29.642 | INFO | } | 20260505 12:02:29.642 | INFO | Perf report for the last 5.13889 seconds: | 20260505 12:02:29.642 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:02:29.642 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:29.642 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:29.642 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:29.642 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 12:02:29.642 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:29.642 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:29.642 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 12:02:29.642 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:29.642 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:29.642 | INFO | Uploaded 36GiB to GPU in 2.15282s (16.7223GiB/s) | 20260505 12:02:29.642 | INFO | Downloaded 34GiB to GPU in 2.10587s (16.1454GiB/s) | 20260505 12:02:34.777 | INFO | Analyzing chunk frames = [3054592,3055104) | 20260505 12:02:34.777 | INFO | MemInfo: { | 20260505 12:02:34.777 | INFO | "gpu" : " 17 GB", | 20260505 12:02:34.777 | INFO | "huge" : "", | 20260505 12:02:34.777 | INFO | "huge_shared" : " 27 GB", | 20260505 12:02:34.777 | INFO | "malloc" : " 14 GB", | 20260505 12:02:34.777 | INFO | "pinned" : " 29 GB" | 20260505 12:02:34.778 | INFO | } | 20260505 12:02:34.778 | INFO | Perf report for the last 5.13589 seconds: | 20260505 12:02:34.778 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:02:34.778 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:34.778 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:34.778 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:34.778 | INFO | ANALYSIS : 63.5%, 3.0, 99.0ms, 0 | 20260505 12:02:34.778 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:34.778 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:34.778 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 12:02:34.778 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:34.778 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:34.778 | INFO | Uploaded 36GiB to GPU in 2.16509s (16.6274GiB/s) | 20260505 12:02:34.778 | INFO | Downloaded 34GiB to GPU in 2.13944s (15.892GiB/s) | 20260505 12:02:39.917 | INFO | Analyzing chunk frames = [3055104,3055616) | 20260505 12:02:39.917 | INFO | MemInfo: { | 20260505 12:02:39.917 | INFO | "gpu" : " 17 GB", | 20260505 12:02:39.917 | INFO | "huge" : "", | 20260505 12:02:39.917 | INFO | "huge_shared" : " 27 GB", | 20260505 12:02:39.917 | INFO | "malloc" : " 14 GB", | 20260505 12:02:39.917 | INFO | "pinned" : " 29 GB" | 20260505 12:02:39.917 | INFO | } | 20260505 12:02:39.917 | INFO | Perf report for the last 5.13934 seconds: | 20260505 12:02:39.917 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:02:39.917 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:39.917 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:39.917 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:39.917 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 12:02:39.917 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:39.917 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:39.917 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 12:02:39.917 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:39.917 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:39.917 | INFO | Uploaded 36GiB to GPU in 2.21319s (16.2661GiB/s) | 20260505 12:02:39.917 | INFO | Downloaded 34GiB to GPU in 2.0949s (16.2299GiB/s) | 20260505 12:02:39.917 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 12:02:39.917 | INFO | Total mem usage: 15.3 GiB | 20260505 12:02:39.917 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 12:02:39.917 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -6.81%) | 20260505 12:02:39.917 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 12:02:39.917 | INFO | High Water Breakdown: | 20260505 12:02:39.917 | INFO | 61.1% : 8.77 GiB ( 92.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 12:02:39.917 | INFO | 38.8% : 5.56 GiB ( 83.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 12:02:39.917 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 12:02:39.917 | INFO | ------------------------------------------ | 20260505 12:02:39.917 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:02:39.917 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 12:02:39.917 | INFO | Total mem usage: 26.5 GiB | 20260505 12:02:39.918 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 12:02:39.918 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 12:02:39.918 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 12:02:39.918 | INFO | High Water Breakdown: | 20260505 12:02:39.918 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:02:39.918 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:02:39.918 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:02:39.918 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:02:39.918 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:02:39.918 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:02:39.918 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 12:02:39.918 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 12:02:39.918 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:02:39.918 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 12:02:39.918 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 12:02:39.918 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 12:02:39.918 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 12:02:39.918 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 12:02:39.918 | INFO | ------------------------------------------ | 20260505 12:02:39.918 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:02:39.918 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 12:02:39.918 | INFO | Total mem usage: 25.6 GiB | 20260505 12:02:39.918 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 12:02:39.918 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 12:02:39.918 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 12:02:39.918 | INFO | High Water Breakdown: | 20260505 12:02:39.918 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 12:02:39.918 | INFO | ------------------------------------------ | 20260505 12:02:39.918 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:02:39.918 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 12:02:39.918 | INFO | Total mem usage: 15.7 GiB | 20260505 12:02:39.918 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 12:02:39.918 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 12:02:39.918 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 12:02:39.918 | INFO | High Water Breakdown: | 20260505 12:02:39.918 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 12:02:39.918 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 12:02:39.918 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 12:02:39.918 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 12:02:39.918 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 12:02:39.918 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 12:02:39.918 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 12:02:39.918 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 12:02:39.918 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 12:02:39.918 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:02:39.918 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 12:02:39.918 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:02:39.918 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 12:02:39.918 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 12:02:39.918 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 12:02:39.918 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 12:02:39.918 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:02:39.918 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 12:02:39.918 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:02:39.918 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:02:39.918 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:02:39.918 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 12:02:39.918 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 12:02:39.918 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:02:39.918 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 12:02:39.918 | INFO | ------------------------------------------ | 20260505 12:02:39.918 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:02:39.918 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 12:02:39.918 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:02:39.918 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:02:39.918 | INFO | PerfReport: Full Execution -- 3.20e+01 / 7.69e+01, 1.38e+01 / 1.52e+00 | 20260505 12:02:39.918 | INFO | PerfReport: 0.29%, AggregateStats -- 9.27e-02 / 1.01e-01, 1.06e-02 / 7.10e-03 | 20260505 12:02:39.918 | INFO | PerfReport: 11.77%, Baseline -- 3.77e+00 / 4.46e+00, 6.04e-01 / 2.54e-01 | 20260505 12:02:39.918 | INFO | PerfReport: 3.31%, Histogram -- 1.06e+00 / 1.49e+00, 2.03e-01 / 9.91e-02 | 20260505 12:02:39.918 | INFO | PerfReport: 15.91%, DME -- 4.08e+01 / 4.52e+01, 4.51e+00 / 2.44e+00 | 20260505 12:02:39.918 | INFO | PerfReport: 56.35%, FrameLabeling -- 1.81e+01 / 2.09e+01, 1.51e+00 / 2.21e+00 | 20260505 12:02:39.918 | INFO | PerfReport: 7.43%, PulseAccumulating -- 2.38e+00 / 2.84e+00, 4.35e-01 / 2.27e-01 | 20260505 12:02:39.918 | INFO | PerfReport: 3.66%, Metrics -- 1.17e+00 / 1.47e+00, 3.32e-01 / 1.91e-01 | 20260505 12:02:39.918 | INFO | PerfReport: 1.29%, PulseSerializing -- 4.12e-01 / 4.57e-01, 7.50e-02 / 4.53e-02 | 20260505 12:02:39.918 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 12:02:39.918 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:02:39.918 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:02:39.918 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260505 12:02:39.918 | INFO | PerfReport: 22.53%, Upload -- 2.28e+01 / 0.00e+00, 8.14e+00 / 0.00e+00 | 20260505 12:02:39.918 | INFO | PerfReport: 22.02%, Download -- 2.23e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260505 12:02:39.918 | INFO | PerfReport: 31.69%, Compute -- 3.20e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 12:02:39.918 | INFO | PerfReport: 2.14%, LockUpload -- 2.16e+00 / 0.00e+00, 7.80e+00 / 0.00e+00 | 20260505 12:02:39.918 | INFO | PerfReport: 1.36%, LockDownload -- 1.37e+00 / 0.00e+00, 2.97e+00 / 0.00e+00 | 20260505 12:02:39.918 | INFO | PerfReport: 20.25%, LockCompute -- 2.05e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260505 12:02:39.918 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.98e-02 / 0.00e+00 | 20260505 12:02:40.048 | INFO | Starting a round of Baz IO | 20260505 12:02:43.292 | INFO | Populating RTMetrics report | 20260505 12:02:43.293 | INFO | RTMetrics report populated | 20260505 12:02:43.293 | INFO | Opening tmp RTMetrics json file | 20260505 12:02:43.317 | INFO | Serializing RTMetrics to json | 20260505 12:02:43.318 | INFO | Serialization done, swapping tmp file now... | 20260505 12:02:43.318 | INFO | RTMetrics done | 20260505 12:02:43.405 | INFO | Finished a round of Baz IO: Wrote 2.352 GiB in 3.356 seconds | 20260505 12:02:43.405 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.80% / 0.01% / 76.19% | 20260505 12:02:43.405 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.976245% / 0% / 7.15256e-05% | 20260505 12:02:43.405 | INFO | Baz IO overall: | 20260505 12:02:43.405 | INFO | BytesWritten : 1812631519141 | 20260505 12:02:43.405 | INFO | NumEvents : 862929664102 | 20260505 12:02:45.056 | INFO | Analyzing chunk frames = [3055616,3056128) | 20260505 12:02:45.056 | INFO | MemInfo: { | 20260505 12:02:45.056 | INFO | "gpu" : " 17 GB", | 20260505 12:02:45.056 | INFO | "huge" : "", | 20260505 12:02:45.056 | INFO | "huge_shared" : " 27 GB", | 20260505 12:02:45.056 | INFO | "malloc" : " 14 GB", | 20260505 12:02:45.056 | INFO | "pinned" : " 29 GB" | 20260505 12:02:45.056 | INFO | } | 20260505 12:02:45.056 | INFO | Perf report for the last 5.13939 seconds: | 20260505 12:02:45.056 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:02:45.056 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:45.056 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:45.056 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:45.056 | INFO | ANALYSIS : 64.6%, 2.9, 99.5ms, 0 | 20260505 12:02:45.056 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:45.056 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260505 12:02:45.056 | INFO | PRE_HQ : 60.9%, 1.8, 58.2ms, 0 | 20260505 12:02:45.056 | INFO | BAZWRITER : 64.6%, 4.1, 135.5ms, 0 | 20260505 12:02:45.056 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260505 12:02:45.056 | INFO | Uploaded 36GiB to GPU in 2.19722s (16.3843GiB/s) | 20260505 12:02:45.057 | INFO | Downloaded 38GiB to GPU in 2.28368s (16.6398GiB/s) | 20260505 12:02:50.202 | INFO | Analyzing chunk frames = [3056128,3056640) | 20260505 12:02:50.202 | INFO | MemInfo: { | 20260505 12:02:50.202 | INFO | "gpu" : " 17 GB", | 20260505 12:02:50.202 | INFO | "huge" : "", | 20260505 12:02:50.202 | INFO | "huge_shared" : " 27 GB", | 20260505 12:02:50.202 | INFO | "malloc" : " 14 GB", | 20260505 12:02:50.202 | INFO | "pinned" : " 29 GB" | 20260505 12:02:50.202 | INFO | } | 20260505 12:02:50.202 | INFO | Perf report for the last 5.14577 seconds: | 20260505 12:02:50.202 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:02:50.202 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:50.202 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:50.202 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:50.202 | INFO | ANALYSIS : 63.7%, 3.0, 99.6ms, 0 | 20260505 12:02:50.202 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:50.202 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:50.202 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 12:02:50.202 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:50.202 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:50.202 | INFO | Uploaded 36GiB to GPU in 2.18719s (16.4595GiB/s) | 20260505 12:02:50.202 | INFO | Downloaded 34GiB to GPU in 2.05267s (16.5638GiB/s) | 20260505 12:02:55.333 | INFO | Analyzing chunk frames = [3056640,3057152) | 20260505 12:02:55.333 | INFO | MemInfo: { | 20260505 12:02:55.333 | INFO | "gpu" : " 17 GB", | 20260505 12:02:55.333 | INFO | "huge" : "", | 20260505 12:02:55.333 | INFO | "huge_shared" : " 27 GB", | 20260505 12:02:55.333 | INFO | "malloc" : " 14 GB", | 20260505 12:02:55.333 | INFO | "pinned" : " 29 GB" | 20260505 12:02:55.333 | INFO | } | 20260505 12:02:55.333 | INFO | Perf report for the last 5.1311 seconds: | 20260505 12:02:55.333 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:02:55.333 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:55.333 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:55.333 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:55.333 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 12:02:55.333 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:55.333 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:55.333 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:02:55.333 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:55.333 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:02:55.333 | INFO | Uploaded 36GiB to GPU in 2.21377s (16.2618GiB/s) | 20260505 12:02:55.333 | INFO | Downloaded 34GiB to GPU in 2.11706s (16.06GiB/s) | 20260505 12:03:00.477 | INFO | Analyzing chunk frames = [3057152,3057664) | 20260505 12:03:00.477 | INFO | MemInfo: { | 20260505 12:03:00.477 | INFO | "gpu" : " 17 GB", | 20260505 12:03:00.477 | INFO | "huge" : "", | 20260505 12:03:00.477 | INFO | "huge_shared" : " 27 GB", | 20260505 12:03:00.477 | INFO | "malloc" : " 14 GB", | 20260505 12:03:00.477 | INFO | "pinned" : " 29 GB" | 20260505 12:03:00.477 | INFO | } | 20260505 12:03:00.477 | INFO | Perf report for the last 5.14397 seconds: | 20260505 12:03:00.477 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:03:00.477 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:00.477 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:00.477 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:00.477 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260505 12:03:00.477 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:00.477 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:00.477 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:03:00.477 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:00.477 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:00.477 | INFO | Uploaded 36GiB to GPU in 2.21959s (16.2192GiB/s) | 20260505 12:03:00.478 | INFO | Downloaded 34GiB to GPU in 2.13357s (15.9357GiB/s) | 20260505 12:03:05.615 | INFO | Analyzing chunk frames = [3057664,3058176) | 20260505 12:03:05.615 | INFO | MemInfo: { | 20260505 12:03:05.615 | INFO | "gpu" : " 17 GB", | 20260505 12:03:05.615 | INFO | "huge" : "", | 20260505 12:03:05.615 | INFO | "huge_shared" : " 27 GB", | 20260505 12:03:05.615 | INFO | "malloc" : " 14 GB", | 20260505 12:03:05.615 | INFO | "pinned" : " 29 GB" | 20260505 12:03:05.615 | INFO | } | 20260505 12:03:05.615 | INFO | Perf report for the last 5.13789 seconds: | 20260505 12:03:05.615 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:03:05.615 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:05.615 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:05.615 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:05.615 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 12:03:05.615 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:05.615 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:05.615 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 12:03:05.615 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:05.615 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:05.615 | INFO | Uploaded 36GiB to GPU in 2.16908s (16.5969GiB/s) | 20260505 12:03:05.615 | INFO | Downloaded 34GiB to GPU in 2.12403s (16.0073GiB/s) | 20260505 12:03:10.759 | INFO | Analyzing chunk frames = [3058176,3058688) | 20260505 12:03:10.759 | INFO | MemInfo: { | 20260505 12:03:10.759 | INFO | "gpu" : " 17 GB", | 20260505 12:03:10.759 | INFO | "huge" : "", | 20260505 12:03:10.759 | INFO | "huge_shared" : " 27 GB", | 20260505 12:03:10.759 | INFO | "malloc" : " 14 GB", | 20260505 12:03:10.759 | INFO | "pinned" : " 29 GB" | 20260505 12:03:10.759 | INFO | } | 20260505 12:03:10.759 | INFO | Perf report for the last 5.14388 seconds: | 20260505 12:03:10.759 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:03:10.759 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:10.759 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:10.759 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:10.759 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 12:03:10.759 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:10.759 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:10.759 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:03:10.759 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:10.759 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:10.759 | INFO | Uploaded 36GiB to GPU in 2.15201s (16.7285GiB/s) | 20260505 12:03:10.759 | INFO | Downloaded 34GiB to GPU in 2.10684s (16.138GiB/s) | 20260505 12:03:15.889 | INFO | Analyzing chunk frames = [3058688,3059200) | 20260505 12:03:15.889 | INFO | MemInfo: { | 20260505 12:03:15.889 | INFO | "gpu" : " 17 GB", | 20260505 12:03:15.889 | INFO | "huge" : "", | 20260505 12:03:15.889 | INFO | "huge_shared" : " 27 GB", | 20260505 12:03:15.889 | INFO | "malloc" : " 14 GB", | 20260505 12:03:15.889 | INFO | "pinned" : " 29 GB" | 20260505 12:03:15.889 | INFO | } | 20260505 12:03:15.889 | INFO | Perf report for the last 5.13015 seconds: | 20260505 12:03:15.889 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:03:15.889 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:15.889 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:15.889 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:15.889 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 12:03:15.889 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:15.889 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:15.889 | INFO | PRE_HQ : 29.8%, 1.0, 15.4ms, 0 | 20260505 12:03:15.889 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:15.889 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:15.889 | INFO | Uploaded 36GiB to GPU in 2.17169s (16.577GiB/s) | 20260505 12:03:15.889 | INFO | Downloaded 34GiB to GPU in 2.15317s (15.7907GiB/s) | 20260505 12:03:21.031 | INFO | Analyzing chunk frames = [3059200,3059712) | 20260505 12:03:21.031 | INFO | MemInfo: { | 20260505 12:03:21.031 | INFO | "gpu" : " 17 GB", | 20260505 12:03:21.031 | INFO | "huge" : "", | 20260505 12:03:21.031 | INFO | "huge_shared" : " 27 GB", | 20260505 12:03:21.031 | INFO | "malloc" : " 14 GB", | 20260505 12:03:21.031 | INFO | "pinned" : " 29 GB" | 20260505 12:03:21.031 | INFO | } | 20260505 12:03:21.031 | INFO | Perf report for the last 5.14179 seconds: | 20260505 12:03:21.031 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:03:21.031 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:21.031 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:21.031 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:21.031 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 12:03:21.031 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:21.031 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:21.031 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 12:03:21.031 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:21.031 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:21.031 | INFO | Uploaded 36GiB to GPU in 2.22733s (16.1629GiB/s) | 20260505 12:03:21.031 | INFO | Downloaded 34GiB to GPU in 2.09017s (16.2666GiB/s) | 20260505 12:03:21.163 | INFO | Starting a round of Baz IO | 20260505 12:03:24.409 | INFO | Populating RTMetrics report | 20260505 12:03:24.409 | INFO | RTMetrics report populated | 20260505 12:03:24.409 | INFO | Opening tmp RTMetrics json file | 20260505 12:03:24.413 | INFO | Serializing RTMetrics to json | 20260505 12:03:24.415 | INFO | Serialization done, swapping tmp file now... | 20260505 12:03:24.416 | INFO | RTMetrics done | 20260505 12:03:24.528 | INFO | Finished a round of Baz IO: Wrote 2.352 GiB in 3.364 seconds | 20260505 12:03:24.528 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.80% / 0.01% / 76.20% | 20260505 12:03:24.528 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.975823% / 1.58946e-05% / 7.15256e-05% | 20260505 12:03:24.528 | INFO | Baz IO overall: | 20260505 12:03:24.528 | INFO | BytesWritten : 1815156488949 | 20260505 12:03:24.528 | INFO | NumEvents : 864128148690 | 20260505 12:03:26.180 | INFO | Analyzing chunk frames = [3059712,3060224) | 20260505 12:03:26.180 | INFO | MemInfo: { | 20260505 12:03:26.180 | INFO | "gpu" : " 17 GB", | 20260505 12:03:26.180 | INFO | "huge" : "", | 20260505 12:03:26.180 | INFO | "huge_shared" : " 27 GB", | 20260505 12:03:26.180 | INFO | "malloc" : " 14 GB", | 20260505 12:03:26.180 | INFO | "pinned" : " 29 GB" | 20260505 12:03:26.180 | INFO | } | 20260505 12:03:26.180 | INFO | Perf report for the last 5.14928 seconds: | 20260505 12:03:26.180 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:03:26.180 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:26.180 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:26.180 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:26.180 | INFO | ANALYSIS : 64.5%, 2.9, 99.8ms, 0 | 20260505 12:03:26.180 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:26.180 | INFO | RT_METRICS : 45.3%, 1.0, 23.8ms, 0 | 20260505 12:03:26.180 | INFO | PRE_HQ : 60.9%, 1.8, 58.5ms, 0 | 20260505 12:03:26.180 | INFO | BAZWRITER : 64.6%, 4.0, 134.8ms, 0 | 20260505 12:03:26.180 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 12:03:26.181 | INFO | Uploaded 36GiB to GPU in 2.18442s (16.4803GiB/s) | 20260505 12:03:26.181 | INFO | Downloaded 38GiB to GPU in 2.28961s (16.5967GiB/s) | 20260505 12:03:31.315 | INFO | Analyzing chunk frames = [3060224,3060736) | 20260505 12:03:31.315 | INFO | MemInfo: { | 20260505 12:03:31.315 | INFO | "gpu" : " 17 GB", | 20260505 12:03:31.315 | INFO | "huge" : "", | 20260505 12:03:31.315 | INFO | "huge_shared" : " 27 GB", | 20260505 12:03:31.315 | INFO | "malloc" : " 14 GB", | 20260505 12:03:31.315 | INFO | "pinned" : " 29 GB" | 20260505 12:03:31.315 | INFO | } | 20260505 12:03:31.315 | INFO | Perf report for the last 5.13467 seconds: | 20260505 12:03:31.315 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:03:31.315 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:31.315 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:31.315 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:31.315 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 12:03:31.315 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:31.315 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:31.315 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:03:31.315 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:31.315 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:31.315 | INFO | Uploaded 36GiB to GPU in 2.17621s (16.5425GiB/s) | 20260505 12:03:31.315 | INFO | Downloaded 34GiB to GPU in 2.03257s (16.7276GiB/s) | 20260505 12:03:36.449 | INFO | Analyzing chunk frames = [3060736,3061248) | 20260505 12:03:36.449 | INFO | MemInfo: { | 20260505 12:03:36.449 | INFO | "gpu" : " 17 GB", | 20260505 12:03:36.449 | INFO | "huge" : "", | 20260505 12:03:36.449 | INFO | "huge_shared" : " 27 GB", | 20260505 12:03:36.449 | INFO | "malloc" : " 14 GB", | 20260505 12:03:36.449 | INFO | "pinned" : " 29 GB" | 20260505 12:03:36.449 | INFO | } | 20260505 12:03:36.449 | INFO | Perf report for the last 5.13371 seconds: | 20260505 12:03:36.449 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:03:36.449 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:36.449 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:36.449 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:36.449 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 12:03:36.449 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:36.449 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:36.449 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 12:03:36.449 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:36.449 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:36.449 | INFO | Uploaded 36GiB to GPU in 2.19742s (16.3829GiB/s) | 20260505 12:03:36.449 | INFO | Downloaded 34GiB to GPU in 2.12136s (16.0274GiB/s) | 20260505 12:03:41.581 | INFO | Analyzing chunk frames = [3061248,3061760) | 20260505 12:03:41.581 | INFO | MemInfo: { | 20260505 12:03:41.581 | INFO | "gpu" : " 17 GB", | 20260505 12:03:41.581 | INFO | "huge" : "", | 20260505 12:03:41.581 | INFO | "huge_shared" : " 27 GB", | 20260505 12:03:41.581 | INFO | "malloc" : " 14 GB", | 20260505 12:03:41.582 | INFO | "pinned" : " 29 GB" | 20260505 12:03:41.582 | INFO | } | 20260505 12:03:41.582 | INFO | Perf report for the last 5.13301 seconds: | 20260505 12:03:41.582 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:03:41.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:41.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:41.582 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:41.582 | INFO | ANALYSIS : 63.3%, 3.0, 98.2ms, 0 | 20260505 12:03:41.582 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:41.582 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:41.582 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 12:03:41.582 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:41.582 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:41.582 | INFO | Uploaded 36GiB to GPU in 2.20954s (16.293GiB/s) | 20260505 12:03:41.582 | INFO | Downloaded 34GiB to GPU in 2.13225s (15.9456GiB/s) | 20260505 12:03:46.729 | INFO | Analyzing chunk frames = [3061760,3062272) | 20260505 12:03:46.729 | INFO | MemInfo: { | 20260505 12:03:46.729 | INFO | "gpu" : " 17 GB", | 20260505 12:03:46.729 | INFO | "huge" : "", | 20260505 12:03:46.729 | INFO | "huge_shared" : " 27 GB", | 20260505 12:03:46.729 | INFO | "malloc" : " 14 GB", | 20260505 12:03:46.729 | INFO | "pinned" : " 29 GB" | 20260505 12:03:46.729 | INFO | } | 20260505 12:03:46.730 | INFO | Perf report for the last 5.14787 seconds: | 20260505 12:03:46.730 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:03:46.730 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:46.730 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:46.730 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:46.730 | INFO | ANALYSIS : 63.1%, 3.0, 98.8ms, 0 | 20260505 12:03:46.730 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:46.730 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:46.730 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 12:03:46.730 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:46.730 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:46.730 | INFO | Uploaded 36GiB to GPU in 2.13873s (16.8324GiB/s) | 20260505 12:03:46.730 | INFO | Downloaded 34GiB to GPU in 2.12125s (16.0283GiB/s) | 20260505 12:03:51.866 | INFO | Analyzing chunk frames = [3062272,3062784) | 20260505 12:03:51.866 | INFO | MemInfo: { | 20260505 12:03:51.866 | INFO | "gpu" : " 17 GB", | 20260505 12:03:51.866 | INFO | "huge" : "", | 20260505 12:03:51.866 | INFO | "huge_shared" : " 27 GB", | 20260505 12:03:51.866 | INFO | "malloc" : " 14 GB", | 20260505 12:03:51.866 | INFO | "pinned" : " 29 GB" | 20260505 12:03:51.866 | INFO | } | 20260505 12:03:51.866 | INFO | Perf report for the last 5.13614 seconds: | 20260505 12:03:51.866 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:03:51.866 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:51.866 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:51.866 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:51.866 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 12:03:51.866 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:51.866 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:51.866 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 12:03:51.866 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:51.866 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:51.866 | INFO | Uploaded 36GiB to GPU in 2.1507s (16.7387GiB/s) | 20260505 12:03:51.866 | INFO | Downloaded 34GiB to GPU in 2.10847s (16.1254GiB/s) | 20260505 12:03:57.013 | INFO | Analyzing chunk frames = [3062784,3063296) | 20260505 12:03:57.013 | INFO | MemInfo: { | 20260505 12:03:57.013 | INFO | "gpu" : " 17 GB", | 20260505 12:03:57.013 | INFO | "huge" : "", | 20260505 12:03:57.013 | INFO | "huge_shared" : " 27 GB", | 20260505 12:03:57.013 | INFO | "malloc" : " 14 GB", | 20260505 12:03:57.013 | INFO | "pinned" : " 29 GB" | 20260505 12:03:57.013 | INFO | } | 20260505 12:03:57.013 | INFO | Perf report for the last 5.14725 seconds: | 20260505 12:03:57.013 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:03:57.013 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:57.013 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:57.013 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:57.013 | INFO | ANALYSIS : 63.1%, 3.0, 98.7ms, 0 | 20260505 12:03:57.013 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:57.013 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:57.013 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 12:03:57.013 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:57.013 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:03:57.013 | INFO | Uploaded 36GiB to GPU in 2.16329s (16.6413GiB/s) | 20260505 12:03:57.013 | INFO | Downloaded 34GiB to GPU in 2.1563s (15.7678GiB/s) | 20260505 12:04:02.147 | INFO | Analyzing chunk frames = [3063296,3063808) | 20260505 12:04:02.147 | INFO | MemInfo: { | 20260505 12:04:02.147 | INFO | "gpu" : " 17 GB", | 20260505 12:04:02.147 | INFO | "huge" : "", | 20260505 12:04:02.148 | INFO | "huge_shared" : " 27 GB", | 20260505 12:04:02.148 | INFO | "malloc" : " 14 GB", | 20260505 12:04:02.148 | INFO | "pinned" : " 29 GB" | 20260505 12:04:02.148 | INFO | } | 20260505 12:04:02.148 | INFO | Perf report for the last 5.13461 seconds: | 20260505 12:04:02.148 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:04:02.148 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:02.148 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:02.148 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:02.148 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 12:04:02.148 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:02.148 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:02.148 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 12:04:02.148 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:02.148 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:02.148 | INFO | Uploaded 36GiB to GPU in 2.13292s (16.8783GiB/s) | 20260505 12:04:02.148 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 12:04:02.148 | INFO | Total mem usage: 15.3 GiB | 20260505 12:04:02.148 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 12:04:02.148 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -6.82%) | 20260505 12:04:02.148 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 12:04:02.148 | INFO | Downloaded 34GiB to GPU in 2.10211s (16.1742GiB/s) | 20260505 12:04:02.148 | INFO | High Water Breakdown: | 20260505 12:04:02.148 | INFO | 61.1% : 8.76 GiB ( 92.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 12:04:02.148 | INFO | 38.8% : 5.56 GiB ( 83.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 12:04:02.148 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 12:04:02.148 | INFO | ------------------------------------------ | 20260505 12:04:02.148 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:04:02.148 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 12:04:02.148 | INFO | Total mem usage: 26.5 GiB | 20260505 12:04:02.148 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 12:04:02.148 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 12:04:02.148 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 12:04:02.148 | INFO | High Water Breakdown: | 20260505 12:04:02.148 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:04:02.148 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:04:02.148 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:04:02.148 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:04:02.148 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:04:02.148 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:04:02.148 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 12:04:02.148 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 12:04:02.148 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:04:02.148 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 12:04:02.148 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 12:04:02.148 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 12:04:02.148 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 12:04:02.148 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 12:04:02.148 | INFO | ------------------------------------------ | 20260505 12:04:02.148 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:04:02.148 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 12:04:02.148 | INFO | Total mem usage: 25.6 GiB | 20260505 12:04:02.148 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 12:04:02.148 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 12:04:02.148 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 12:04:02.148 | INFO | High Water Breakdown: | 20260505 12:04:02.148 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 12:04:02.148 | INFO | ------------------------------------------ | 20260505 12:04:02.148 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:04:02.148 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 12:04:02.148 | INFO | Total mem usage: 15.7 GiB | 20260505 12:04:02.148 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 12:04:02.148 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 12:04:02.148 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 12:04:02.148 | INFO | High Water Breakdown: | 20260505 12:04:02.148 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 12:04:02.148 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 12:04:02.148 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 12:04:02.148 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 12:04:02.148 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 12:04:02.148 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 12:04:02.148 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 12:04:02.148 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 12:04:02.148 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 12:04:02.148 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:04:02.148 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 12:04:02.148 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:04:02.148 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 12:04:02.148 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 12:04:02.148 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 12:04:02.148 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 12:04:02.148 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:04:02.148 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 12:04:02.148 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:04:02.148 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:04:02.149 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:04:02.149 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 12:04:02.149 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 12:04:02.149 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:04:02.149 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 12:04:02.149 | INFO | ------------------------------------------ | 20260505 12:04:02.149 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:04:02.149 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 12:04:02.149 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:04:02.149 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:04:02.149 | INFO | PerfReport: Full Execution -- 3.21e+01 / 7.71e+01, 1.38e+01 / 1.82e+00 | 20260505 12:04:02.149 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 1.01e-01, 1.03e-02 / 4.58e-03 | 20260505 12:04:02.149 | INFO | PerfReport: 11.74%, Baseline -- 3.77e+00 / 4.44e+00, 6.01e-01 / 2.70e-01 | 20260505 12:04:02.149 | INFO | PerfReport: 3.30%, Histogram -- 1.06e+00 / 1.49e+00, 2.01e-01 / 9.81e-02 | 20260505 12:04:02.149 | INFO | PerfReport: 15.88%, DME -- 4.07e+01 / 4.56e+01, 4.65e+00 / 2.33e+00 | 20260505 12:04:02.149 | INFO | PerfReport: 56.44%, FrameLabeling -- 1.81e+01 / 2.06e+01, 1.58e+00 / 1.93e+00 | 20260505 12:04:02.149 | INFO | PerfReport: 7.43%, PulseAccumulating -- 2.38e+00 / 2.91e+00, 4.39e-01 / 2.19e-01 | 20260505 12:04:02.149 | INFO | PerfReport: 3.63%, Metrics -- 1.16e+00 / 1.52e+00, 3.29e-01 / 1.69e-01 | 20260505 12:04:02.149 | INFO | PerfReport: 1.28%, PulseSerializing -- 4.11e-01 / 4.70e-01, 7.48e-02 / 3.02e-02 | 20260505 12:04:02.149 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 12:04:02.149 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:04:02.149 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:04:02.149 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260505 12:04:02.149 | INFO | PerfReport: 22.50%, Upload -- 2.27e+01 / 0.00e+00, 8.01e+00 / 0.00e+00 | 20260505 12:04:02.149 | INFO | PerfReport: 22.00%, Download -- 2.22e+01 / 0.00e+00, 8.11e+00 / 0.00e+00 | 20260505 12:04:02.149 | INFO | PerfReport: 31.75%, Compute -- 3.21e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 12:04:02.149 | INFO | PerfReport: 2.08%, LockUpload -- 2.10e+00 / 0.00e+00, 7.71e+00 / 0.00e+00 | 20260505 12:04:02.149 | INFO | PerfReport: 1.40%, LockDownload -- 1.42e+00 / 0.00e+00, 3.02e+00 / 0.00e+00 | 20260505 12:04:02.149 | INFO | PerfReport: 20.25%, LockCompute -- 2.05e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260505 12:04:02.149 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.70e-02 / 0.00e+00 | 20260505 12:04:02.282 | INFO | Starting a round of Baz IO | 20260505 12:04:05.527 | INFO | Populating RTMetrics report | 20260505 12:04:05.528 | INFO | RTMetrics report populated | 20260505 12:04:05.528 | INFO | Opening tmp RTMetrics json file | 20260505 12:04:05.533 | INFO | Serializing RTMetrics to json | 20260505 12:04:05.534 | INFO | Serialization done, swapping tmp file now... | 20260505 12:04:05.535 | INFO | RTMetrics done | 20260505 12:04:05.655 | INFO | Finished a round of Baz IO: Wrote 2.35 GiB in 3.372 seconds | 20260505 12:04:05.655 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.79% / 0.01% / 76.20% | 20260505 12:04:05.655 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.970948% / 3.97364e-06% / 7.15256e-05% | 20260505 12:04:05.655 | INFO | Baz IO overall: | 20260505 12:04:05.655 | INFO | BytesWritten : 1817679864134 | 20260505 12:04:05.655 | INFO | NumEvents : 865325539454 | 20260505 12:04:07.287 | INFO | Analyzing chunk frames = [3063808,3064320) | 20260505 12:04:07.287 | INFO | MemInfo: { | 20260505 12:04:07.287 | INFO | "gpu" : " 17 GB", | 20260505 12:04:07.287 | INFO | "huge" : "", | 20260505 12:04:07.287 | INFO | "huge_shared" : " 27 GB", | 20260505 12:04:07.287 | INFO | "malloc" : " 14 GB", | 20260505 12:04:07.287 | INFO | "pinned" : " 29 GB" | 20260505 12:04:07.287 | INFO | } | 20260505 12:04:07.287 | INFO | Perf report for the last 5.13921 seconds: | 20260505 12:04:07.287 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:04:07.287 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:07.287 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:07.287 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:07.287 | INFO | ANALYSIS : 64.7%, 2.9, 99.7ms, 0 | 20260505 12:04:07.287 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:07.287 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260505 12:04:07.287 | INFO | PRE_HQ : 61.4%, 1.8, 58.6ms, 0 | 20260505 12:04:07.287 | INFO | BAZWRITER : 64.9%, 4.1, 136.5ms, 0 | 20260505 12:04:07.287 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 12:04:07.287 | INFO | Uploaded 36GiB to GPU in 2.18926s (16.4439GiB/s) | 20260505 12:04:07.287 | INFO | Downloaded 38GiB to GPU in 2.292s (16.5794GiB/s) | 20260505 12:04:12.430 | INFO | Analyzing chunk frames = [3064320,3064832) | 20260505 12:04:12.430 | INFO | MemInfo: { | 20260505 12:04:12.430 | INFO | "gpu" : " 17 GB", | 20260505 12:04:12.430 | INFO | "huge" : "", | 20260505 12:04:12.430 | INFO | "huge_shared" : " 27 GB", | 20260505 12:04:12.430 | INFO | "malloc" : " 14 GB", | 20260505 12:04:12.430 | INFO | "pinned" : " 29 GB" | 20260505 12:04:12.430 | INFO | } | 20260505 12:04:12.430 | INFO | Perf report for the last 5.14288 seconds: | 20260505 12:04:12.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:04:12.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:12.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:12.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:12.430 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 12:04:12.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:12.430 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:12.430 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:04:12.430 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:12.430 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:12.430 | INFO | Uploaded 36GiB to GPU in 2.23069s (16.1385GiB/s) | 20260505 12:04:12.430 | INFO | Downloaded 34GiB to GPU in 2.10065s (16.1855GiB/s) | 20260505 12:04:17.564 | INFO | Analyzing chunk frames = [3064832,3065344) | 20260505 12:04:17.564 | INFO | MemInfo: { | 20260505 12:04:17.564 | INFO | "gpu" : " 17 GB", | 20260505 12:04:17.564 | INFO | "huge" : "", | 20260505 12:04:17.564 | INFO | "huge_shared" : " 27 GB", | 20260505 12:04:17.564 | INFO | "malloc" : " 14 GB", | 20260505 12:04:17.564 | INFO | "pinned" : " 29 GB" | 20260505 12:04:17.564 | INFO | } | 20260505 12:04:17.564 | INFO | Perf report for the last 5.13431 seconds: | 20260505 12:04:17.564 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:04:17.564 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:17.564 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:17.564 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:17.564 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 12:04:17.564 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:17.564 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:17.564 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 12:04:17.564 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:17.564 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:17.564 | INFO | Uploaded 36GiB to GPU in 2.16671s (16.615GiB/s) | 20260505 12:04:17.564 | INFO | Downloaded 34GiB to GPU in 2.11652s (16.0641GiB/s) | 20260505 12:04:22.705 | INFO | Analyzing chunk frames = [3065344,3065856) | 20260505 12:04:22.705 | INFO | MemInfo: { | 20260505 12:04:22.705 | INFO | "gpu" : " 17 GB", | 20260505 12:04:22.705 | INFO | "huge" : "", | 20260505 12:04:22.705 | INFO | "huge_shared" : " 27 GB", | 20260505 12:04:22.705 | INFO | "malloc" : " 14 GB", | 20260505 12:04:22.705 | INFO | "pinned" : " 29 GB" | 20260505 12:04:22.705 | INFO | } | 20260505 12:04:22.705 | INFO | Perf report for the last 5.14122 seconds: | 20260505 12:04:22.705 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:04:22.705 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:22.705 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:22.705 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:22.705 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 12:04:22.705 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:22.705 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:22.705 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 12:04:22.705 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:22.705 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:22.705 | INFO | Uploaded 36GiB to GPU in 2.21251s (16.2711GiB/s) | 20260505 12:04:22.705 | INFO | Downloaded 34GiB to GPU in 2.12987s (15.9634GiB/s) | 20260505 12:04:27.838 | INFO | Analyzing chunk frames = [3065856,3066368) | 20260505 12:04:27.838 | INFO | MemInfo: { | 20260505 12:04:27.838 | INFO | "gpu" : " 17 GB", | 20260505 12:04:27.838 | INFO | "huge" : "", | 20260505 12:04:27.838 | INFO | "huge_shared" : " 27 GB", | 20260505 12:04:27.838 | INFO | "malloc" : " 14 GB", | 20260505 12:04:27.838 | INFO | "pinned" : " 29 GB" | 20260505 12:04:27.838 | INFO | } | 20260505 12:04:27.838 | INFO | Perf report for the last 5.13251 seconds: | 20260505 12:04:27.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:04:27.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:27.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:27.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:27.838 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 12:04:27.838 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:27.838 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:27.838 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 12:04:27.838 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:27.838 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:27.838 | INFO | Uploaded 36GiB to GPU in 2.1575s (16.686GiB/s) | 20260505 12:04:27.838 | INFO | Downloaded 34GiB to GPU in 2.12446s (16.004GiB/s) | 20260505 12:04:32.986 | INFO | Analyzing chunk frames = [3066368,3066880) | 20260505 12:04:32.986 | INFO | MemInfo: { | 20260505 12:04:32.986 | INFO | "gpu" : " 17 GB", | 20260505 12:04:32.986 | INFO | "huge" : "", | 20260505 12:04:32.986 | INFO | "huge_shared" : " 27 GB", | 20260505 12:04:32.986 | INFO | "malloc" : " 14 GB", | 20260505 12:04:32.986 | INFO | "pinned" : " 29 GB" | 20260505 12:04:32.986 | INFO | } | 20260505 12:04:32.986 | INFO | Perf report for the last 5.14848 seconds: | 20260505 12:04:32.986 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:04:32.986 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:32.986 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:32.986 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:32.986 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 12:04:32.986 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:32.986 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:32.986 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 12:04:32.986 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:32.986 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:32.986 | INFO | Uploaded 36GiB to GPU in 2.17234s (16.572GiB/s) | 20260505 12:04:32.987 | INFO | Downloaded 34GiB to GPU in 2.12342s (16.0119GiB/s) | 20260505 12:04:38.129 | INFO | Analyzing chunk frames = [3066880,3067392) | 20260505 12:04:38.129 | INFO | MemInfo: { | 20260505 12:04:38.130 | INFO | "gpu" : " 17 GB", | 20260505 12:04:38.130 | INFO | "huge" : "", | 20260505 12:04:38.130 | INFO | "huge_shared" : " 27 GB", | 20260505 12:04:38.130 | INFO | "malloc" : " 14 GB", | 20260505 12:04:38.130 | INFO | "pinned" : " 29 GB" | 20260505 12:04:38.130 | INFO | } | 20260505 12:04:38.130 | INFO | Perf report for the last 5.14344 seconds: | 20260505 12:04:38.130 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:04:38.130 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:38.130 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:38.130 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:38.130 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 12:04:38.130 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:38.130 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:38.130 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 12:04:38.130 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:38.130 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:38.130 | INFO | Uploaded 36GiB to GPU in 2.15144s (16.733GiB/s) | 20260505 12:04:38.130 | INFO | Downloaded 34GiB to GPU in 2.13327s (15.938GiB/s) | 20260505 12:04:43.264 | INFO | Analyzing chunk frames = [3067392,3067904) | 20260505 12:04:43.264 | INFO | MemInfo: { | 20260505 12:04:43.264 | INFO | "gpu" : " 17 GB", | 20260505 12:04:43.264 | INFO | "huge" : "", | 20260505 12:04:43.264 | INFO | "huge_shared" : " 27 GB", | 20260505 12:04:43.264 | INFO | "malloc" : " 14 GB", | 20260505 12:04:43.264 | INFO | "pinned" : " 29 GB" | 20260505 12:04:43.264 | INFO | } | 20260505 12:04:43.264 | INFO | Perf report for the last 5.13392 seconds: | 20260505 12:04:43.264 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:04:43.264 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:43.264 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:43.264 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:43.264 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 12:04:43.264 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:43.264 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:43.264 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 12:04:43.264 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:43.264 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:43.264 | INFO | Uploaded 36GiB to GPU in 2.17042s (16.5867GiB/s) | 20260505 12:04:43.264 | INFO | Downloaded 34GiB to GPU in 2.10519s (16.1506GiB/s) | 20260505 12:04:43.401 | INFO | Starting a round of Baz IO | 20260505 12:04:46.660 | INFO | Populating RTMetrics report | 20260505 12:04:46.660 | INFO | RTMetrics report populated | 20260505 12:04:46.660 | INFO | Opening tmp RTMetrics json file | 20260505 12:04:46.677 | INFO | Serializing RTMetrics to json | 20260505 12:04:46.704 | INFO | Serialization done, swapping tmp file now... | 20260505 12:04:46.736 | INFO | RTMetrics done | 20260505 12:04:46.769 | INFO | Finished a round of Baz IO: Wrote 2.352 GiB in 3.368 seconds | 20260505 12:04:46.769 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.78% / 0.01% / 76.21% | 20260505 12:04:46.769 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.970439% / 1.19209e-05% / 9.13938e-05% | 20260505 12:04:46.769 | INFO | Baz IO overall: | 20260505 12:04:46.769 | INFO | BytesWritten : 1820205076232 | 20260505 12:04:46.769 | INFO | NumEvents : 866524573412 | 20260505 12:04:48.395 | INFO | Analyzing chunk frames = [3067904,3068416) | 20260505 12:04:48.395 | INFO | MemInfo: { | 20260505 12:04:48.395 | INFO | "gpu" : " 17 GB", | 20260505 12:04:48.395 | INFO | "huge" : "", | 20260505 12:04:48.395 | INFO | "huge_shared" : " 27 GB", | 20260505 12:04:48.395 | INFO | "malloc" : " 14 GB", | 20260505 12:04:48.395 | INFO | "pinned" : " 29 GB" | 20260505 12:04:48.395 | INFO | } | 20260505 12:04:48.395 | INFO | Perf report for the last 5.13136 seconds: | 20260505 12:04:48.395 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:04:48.395 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:48.395 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:48.395 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:48.395 | INFO | ANALYSIS : 65.1%, 3.0, 100.6ms, 0 | 20260505 12:04:48.395 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:48.395 | INFO | RT_METRICS : 46.9%, 1.0, 24.6ms, 0 | 20260505 12:04:48.395 | INFO | PRE_HQ : 61.3%, 1.8, 58.7ms, 0 | 20260505 12:04:48.395 | INFO | BAZWRITER : 65.0%, 4.1, 135.6ms, 0 | 20260505 12:04:48.395 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 12:04:48.395 | INFO | Uploaded 36GiB to GPU in 2.23665s (16.0955GiB/s) | 20260505 12:04:48.395 | INFO | Downloaded 38GiB to GPU in 2.29379s (16.5664GiB/s) | 20260505 12:04:53.535 | INFO | Analyzing chunk frames = [3068416,3068928) | 20260505 12:04:53.535 | INFO | MemInfo: { | 20260505 12:04:53.535 | INFO | "gpu" : " 17 GB", | 20260505 12:04:53.535 | INFO | "huge" : "", | 20260505 12:04:53.535 | INFO | "huge_shared" : " 27 GB", | 20260505 12:04:53.535 | INFO | "malloc" : " 14 GB", | 20260505 12:04:53.535 | INFO | "pinned" : " 29 GB" | 20260505 12:04:53.535 | INFO | } | 20260505 12:04:53.535 | INFO | Perf report for the last 5.14027 seconds: | 20260505 12:04:53.535 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:04:53.535 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:53.535 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:53.535 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:53.535 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 12:04:53.535 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:53.535 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:53.535 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 12:04:53.535 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:53.535 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:53.535 | INFO | Uploaded 36GiB to GPU in 2.23131s (16.134GiB/s) | 20260505 12:04:53.536 | INFO | Downloaded 34GiB to GPU in 2.0835s (16.3187GiB/s) | 20260505 12:04:58.682 | INFO | Analyzing chunk frames = [3068928,3069440) | 20260505 12:04:58.682 | INFO | MemInfo: { | 20260505 12:04:58.682 | INFO | "gpu" : " 17 GB", | 20260505 12:04:58.682 | INFO | "huge" : "", | 20260505 12:04:58.682 | INFO | "huge_shared" : " 27 GB", | 20260505 12:04:58.682 | INFO | "malloc" : " 14 GB", | 20260505 12:04:58.682 | INFO | "pinned" : " 29 GB" | 20260505 12:04:58.682 | INFO | } | 20260505 12:04:58.682 | INFO | Perf report for the last 5.14657 seconds: | 20260505 12:04:58.682 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:04:58.682 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:58.682 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:58.682 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:58.682 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 12:04:58.682 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:58.682 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:58.682 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:04:58.682 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:58.682 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:04:58.682 | INFO | Uploaded 36GiB to GPU in 2.22788s (16.1588GiB/s) | 20260505 12:04:58.682 | INFO | Downloaded 34GiB to GPU in 2.12915s (15.9688GiB/s) | 20260505 12:05:03.824 | INFO | Analyzing chunk frames = [3069440,3069952) | 20260505 12:05:03.824 | INFO | MemInfo: { | 20260505 12:05:03.824 | INFO | "gpu" : " 17 GB", | 20260505 12:05:03.824 | INFO | "huge" : "", | 20260505 12:05:03.824 | INFO | "huge_shared" : " 27 GB", | 20260505 12:05:03.824 | INFO | "malloc" : " 14 GB", | 20260505 12:05:03.824 | INFO | "pinned" : " 29 GB" | 20260505 12:05:03.824 | INFO | } | 20260505 12:05:03.824 | INFO | Perf report for the last 5.14231 seconds: | 20260505 12:05:03.824 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:05:03.824 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:03.824 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:03.824 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:03.824 | INFO | ANALYSIS : 63.1%, 3.0, 98.1ms, 0 | 20260505 12:05:03.824 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:03.824 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:03.824 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 12:05:03.824 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:03.824 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:03.824 | INFO | Uploaded 36GiB to GPU in 2.20633s (16.3167GiB/s) | 20260505 12:05:03.825 | INFO | Downloaded 34GiB to GPU in 2.13002s (15.9623GiB/s) | 20260505 12:05:08.960 | INFO | Analyzing chunk frames = [3069952,3070464) | 20260505 12:05:08.960 | INFO | MemInfo: { | 20260505 12:05:08.960 | INFO | "gpu" : " 17 GB", | 20260505 12:05:08.960 | INFO | "huge" : "", | 20260505 12:05:08.960 | INFO | "huge_shared" : " 27 GB", | 20260505 12:05:08.960 | INFO | "malloc" : " 14 GB", | 20260505 12:05:08.960 | INFO | "pinned" : " 29 GB" | 20260505 12:05:08.960 | INFO | } | 20260505 12:05:08.960 | INFO | Perf report for the last 5.13569 seconds: | 20260505 12:05:08.960 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:05:08.960 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:08.960 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:08.960 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:08.960 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 12:05:08.960 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:08.960 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:08.960 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:05:08.960 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:08.960 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:08.960 | INFO | Uploaded 36GiB to GPU in 2.17166s (16.5772GiB/s) | 20260505 12:05:08.960 | INFO | Downloaded 34GiB to GPU in 2.12388s (16.0085GiB/s) | 20260505 12:05:14.094 | INFO | Analyzing chunk frames = [3070464,3070976) | 20260505 12:05:14.094 | INFO | MemInfo: { | 20260505 12:05:14.094 | INFO | "gpu" : " 17 GB", | 20260505 12:05:14.095 | INFO | "huge" : "", | 20260505 12:05:14.095 | INFO | "huge_shared" : " 27 GB", | 20260505 12:05:14.095 | INFO | "malloc" : " 14 GB", | 20260505 12:05:14.095 | INFO | "pinned" : " 29 GB" | 20260505 12:05:14.095 | INFO | } | 20260505 12:05:14.095 | INFO | Perf report for the last 5.13467 seconds: | 20260505 12:05:14.095 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:05:14.095 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:14.095 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:14.095 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:14.095 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 12:05:14.095 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:14.095 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:14.095 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 12:05:14.095 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:14.095 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:14.095 | INFO | Uploaded 36GiB to GPU in 2.17537s (16.5489GiB/s) | 20260505 12:05:14.095 | INFO | Downloaded 34GiB to GPU in 2.1133s (16.0886GiB/s) | 20260505 12:05:19.240 | INFO | Analyzing chunk frames = [3070976,3071488) | 20260505 12:05:19.240 | INFO | MemInfo: { | 20260505 12:05:19.240 | INFO | "gpu" : " 17 GB", | 20260505 12:05:19.240 | INFO | "huge" : "", | 20260505 12:05:19.240 | INFO | "huge_shared" : " 27 GB", | 20260505 12:05:19.240 | INFO | "malloc" : " 14 GB", | 20260505 12:05:19.240 | INFO | "pinned" : " 29 GB" | 20260505 12:05:19.240 | INFO | } | 20260505 12:05:19.240 | INFO | Perf report for the last 5.14501 seconds: | 20260505 12:05:19.240 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:05:19.240 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:19.240 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:19.240 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:19.240 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 12:05:19.240 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:19.240 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:19.240 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 12:05:19.240 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:19.240 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:19.240 | INFO | Uploaded 36GiB to GPU in 2.1927s (16.4181GiB/s) | 20260505 12:05:19.240 | INFO | Downloaded 34GiB to GPU in 2.14441s (15.8551GiB/s) | 20260505 12:05:24.374 | INFO | Analyzing chunk frames = [3071488,3072000) | 20260505 12:05:24.374 | INFO | MemInfo: { | 20260505 12:05:24.374 | INFO | "gpu" : " 17 GB", | 20260505 12:05:24.374 | INFO | "huge" : "", | 20260505 12:05:24.374 | INFO | "huge_shared" : " 27 GB", | 20260505 12:05:24.374 | INFO | "malloc" : " 14 GB", | 20260505 12:05:24.374 | INFO | "pinned" : " 29 GB" | 20260505 12:05:24.374 | INFO | } | 20260505 12:05:24.374 | INFO | Perf report for the last 5.13435 seconds: | 20260505 12:05:24.374 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:05:24.374 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:24.374 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:24.374 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:24.374 | INFO | ANALYSIS : 63.9%, 3.0, 99.6ms, 0 | 20260505 12:05:24.374 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:24.374 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:24.374 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 12:05:24.374 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:24.374 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:24.374 | INFO | Uploaded 36GiB to GPU in 2.19353s (16.4119GiB/s) | 20260505 12:05:24.374 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 12:05:24.374 | INFO | Downloaded 34GiB to GPU in 2.11059s (16.1092GiB/s) | 20260505 12:05:24.374 | INFO | Total mem usage: 15.3 GiB | 20260505 12:05:24.374 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 12:05:24.374 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -6.84%) | 20260505 12:05:24.374 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 12:05:24.374 | INFO | High Water Breakdown: | 20260505 12:05:24.374 | INFO | 61.1% : 8.76 GiB ( 92.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 12:05:24.374 | INFO | 38.8% : 5.56 GiB ( 83.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 12:05:24.374 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 12:05:24.374 | INFO | ------------------------------------------ | 20260505 12:05:24.374 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:05:24.374 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 12:05:24.374 | INFO | Total mem usage: 26.5 GiB | 20260505 12:05:24.374 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 12:05:24.374 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 12:05:24.374 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 12:05:24.374 | INFO | High Water Breakdown: | 20260505 12:05:24.374 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:05:24.374 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:05:24.374 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:05:24.374 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:05:24.374 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:05:24.374 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:05:24.374 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 12:05:24.375 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 12:05:24.375 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:05:24.375 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 12:05:24.375 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 12:05:24.375 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 12:05:24.375 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 12:05:24.375 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 12:05:24.375 | INFO | ------------------------------------------ | 20260505 12:05:24.375 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:05:24.375 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 12:05:24.375 | INFO | Total mem usage: 25.6 GiB | 20260505 12:05:24.375 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 12:05:24.375 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 12:05:24.375 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 12:05:24.375 | INFO | High Water Breakdown: | 20260505 12:05:24.375 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 12:05:24.375 | INFO | ------------------------------------------ | 20260505 12:05:24.375 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:05:24.375 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 12:05:24.375 | INFO | Total mem usage: 15.7 GiB | 20260505 12:05:24.375 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 12:05:24.375 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 12:05:24.375 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 12:05:24.375 | INFO | High Water Breakdown: | 20260505 12:05:24.375 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 12:05:24.375 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 12:05:24.375 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 12:05:24.375 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 12:05:24.375 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 12:05:24.375 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 12:05:24.375 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 12:05:24.375 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 12:05:24.375 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 12:05:24.375 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:05:24.375 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 12:05:24.375 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:05:24.375 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 12:05:24.375 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 12:05:24.375 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 12:05:24.375 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 12:05:24.375 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:05:24.375 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 12:05:24.375 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:05:24.375 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:05:24.375 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:05:24.375 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 12:05:24.375 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 12:05:24.375 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:05:24.375 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 12:05:24.375 | INFO | ------------------------------------------ | 20260505 12:05:24.375 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:05:24.375 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 12:05:24.375 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:05:24.375 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:05:24.375 | INFO | PerfReport: Full Execution -- 3.20e+01 / 7.76e+01, 1.37e+01 / 1.59e+00 | 20260505 12:05:24.375 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 9.91e-02, 1.03e-02 / 4.27e-03 | 20260505 12:05:24.375 | INFO | PerfReport: 11.74%, Baseline -- 3.76e+00 / 4.51e+00, 6.01e-01 / 1.62e-01 | 20260505 12:05:24.375 | INFO | PerfReport: 3.30%, Histogram -- 1.06e+00 / 1.51e+00, 2.03e-01 / 7.41e-02 | 20260505 12:05:24.375 | INFO | PerfReport: 16.06%, DME -- 4.12e+01 / 4.65e+01, 4.96e+00 / 2.12e+00 | 20260505 12:05:24.375 | INFO | PerfReport: 56.24%, FrameLabeling -- 1.80e+01 / 2.00e+01, 1.44e+00 / 1.97e+00 | 20260505 12:05:24.375 | INFO | PerfReport: 7.44%, PulseAccumulating -- 2.38e+00 / 2.92e+00, 4.48e-01 / 1.78e-01 | 20260505 12:05:24.375 | INFO | PerfReport: 3.64%, Metrics -- 1.17e+00 / 1.48e+00, 3.31e-01 / 1.74e-01 | 20260505 12:05:24.375 | INFO | PerfReport: 1.28%, PulseSerializing -- 4.12e-01 / 4.56e-01, 7.53e-02 / 3.71e-02 | 20260505 12:05:24.375 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 12:05:24.375 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:05:24.375 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:05:24.375 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260505 12:05:24.375 | INFO | PerfReport: 22.57%, Upload -- 2.28e+01 / 0.00e+00, 8.15e+00 / 0.00e+00 | 20260505 12:05:24.375 | INFO | PerfReport: 22.04%, Download -- 2.23e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260505 12:05:24.375 | INFO | PerfReport: 31.68%, Compute -- 3.21e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 12:05:24.375 | INFO | PerfReport: 2.16%, LockUpload -- 2.19e+00 / 0.00e+00, 7.78e+00 / 0.00e+00 | 20260505 12:05:24.375 | INFO | PerfReport: 1.34%, LockDownload -- 1.36e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260505 12:05:24.375 | INFO | PerfReport: 20.19%, LockCompute -- 2.04e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260505 12:05:24.375 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.80e-02 / 0.00e+00 | 20260505 12:05:24.507 | INFO | Starting a round of Baz IO | 20260505 12:05:27.751 | INFO | Populating RTMetrics report | 20260505 12:05:27.751 | INFO | RTMetrics report populated | 20260505 12:05:27.751 | INFO | Opening tmp RTMetrics json file | 20260505 12:05:27.753 | INFO | Serializing RTMetrics to json | 20260505 12:05:27.755 | INFO | Serialization done, swapping tmp file now... | 20260505 12:05:27.756 | INFO | RTMetrics done | 20260505 12:05:27.874 | INFO | Finished a round of Baz IO: Wrote 2.351 GiB in 3.366 seconds | 20260505 12:05:27.874 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.78% / 0.01% / 76.22% | 20260505 12:05:27.874 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.969032% / 3.97364e-06% / 6.35783e-05% | 20260505 12:05:27.874 | INFO | Baz IO overall: | 20260505 12:05:27.874 | INFO | BytesWritten : 1822729745548 | 20260505 12:05:27.874 | INFO | NumEvents : 867723146546 | 20260505 12:05:29.517 | INFO | Analyzing chunk frames = [3072000,3072512) | 20260505 12:05:29.517 | INFO | MemInfo: { | 20260505 12:05:29.517 | INFO | "gpu" : " 17 GB", | 20260505 12:05:29.517 | INFO | "huge" : "", | 20260505 12:05:29.517 | INFO | "huge_shared" : " 27 GB", | 20260505 12:05:29.517 | INFO | "malloc" : " 14 GB", | 20260505 12:05:29.517 | INFO | "pinned" : " 29 GB" | 20260505 12:05:29.517 | INFO | } | 20260505 12:05:29.517 | INFO | Perf report for the last 5.14329 seconds: | 20260505 12:05:29.517 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:05:29.517 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:29.517 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:29.517 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:29.517 | INFO | ANALYSIS : 64.5%, 2.9, 99.6ms, 0 | 20260505 12:05:29.517 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:29.517 | INFO | RT_METRICS : 45.3%, 1.0, 23.8ms, 0 | 20260505 12:05:29.517 | INFO | PRE_HQ : 61.2%, 1.8, 58.8ms, 0 | 20260505 12:05:29.517 | INFO | BAZWRITER : 64.8%, 4.1, 135.5ms, 0 | 20260505 12:05:29.517 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 12:05:29.517 | INFO | Uploaded 36GiB to GPU in 2.19236s (16.4207GiB/s) | 20260505 12:05:29.517 | INFO | Downloaded 38GiB to GPU in 2.2951s (16.557GiB/s) | 20260505 12:05:34.656 | INFO | Analyzing chunk frames = [3072512,3073024) | 20260505 12:05:34.656 | INFO | MemInfo: { | 20260505 12:05:34.656 | INFO | "gpu" : " 17 GB", | 20260505 12:05:34.656 | INFO | "huge" : "", | 20260505 12:05:34.656 | INFO | "huge_shared" : " 27 GB", | 20260505 12:05:34.656 | INFO | "malloc" : " 14 GB", | 20260505 12:05:34.656 | INFO | "pinned" : " 29 GB" | 20260505 12:05:34.656 | INFO | } | 20260505 12:05:34.656 | INFO | Perf report for the last 5.13903 seconds: | 20260505 12:05:34.656 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:05:34.656 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:34.656 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:34.656 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:34.656 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 12:05:34.656 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:34.656 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:34.656 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 12:05:34.656 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:34.656 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:34.657 | INFO | Uploaded 36GiB to GPU in 2.23394s (16.115GiB/s) | 20260505 12:05:34.657 | INFO | Downloaded 34GiB to GPU in 2.07695s (16.3701GiB/s) | 20260505 12:05:39.793 | INFO | Analyzing chunk frames = [3073024,3073536) | 20260505 12:05:39.793 | INFO | MemInfo: { | 20260505 12:05:39.793 | INFO | "gpu" : " 17 GB", | 20260505 12:05:39.793 | INFO | "huge" : "", | 20260505 12:05:39.793 | INFO | "huge_shared" : " 27 GB", | 20260505 12:05:39.793 | INFO | "malloc" : " 14 GB", | 20260505 12:05:39.793 | INFO | "pinned" : " 29 GB" | 20260505 12:05:39.793 | INFO | } | 20260505 12:05:39.793 | INFO | Perf report for the last 5.1369 seconds: | 20260505 12:05:39.793 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:05:39.793 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:39.793 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:39.793 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:39.793 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 12:05:39.793 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:39.793 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:39.793 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 12:05:39.793 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:39.793 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:39.793 | INFO | Uploaded 36GiB to GPU in 2.21355s (16.2634GiB/s) | 20260505 12:05:39.793 | INFO | Downloaded 34GiB to GPU in 2.12359s (16.0106GiB/s) | 20260505 12:05:44.939 | INFO | Analyzing chunk frames = [3073536,3074048) | 20260505 12:05:44.939 | INFO | MemInfo: { | 20260505 12:05:44.939 | INFO | "gpu" : " 17 GB", | 20260505 12:05:44.940 | INFO | "huge" : "", | 20260505 12:05:44.940 | INFO | "huge_shared" : " 27 GB", | 20260505 12:05:44.940 | INFO | "malloc" : " 14 GB", | 20260505 12:05:44.940 | INFO | "pinned" : " 29 GB" | 20260505 12:05:44.940 | INFO | } | 20260505 12:05:44.940 | INFO | Perf report for the last 5.14644 seconds: | 20260505 12:05:44.940 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:05:44.940 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:44.940 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:44.940 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:44.940 | INFO | ANALYSIS : 63.2%, 3.0, 98.4ms, 0 | 20260505 12:05:44.940 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:44.940 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:44.940 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 12:05:44.940 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:44.940 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:44.940 | INFO | Uploaded 36GiB to GPU in 2.21163s (16.2776GiB/s) | 20260505 12:05:44.940 | INFO | Downloaded 34GiB to GPU in 2.13216s (15.9463GiB/s) | 20260505 12:05:50.081 | INFO | Analyzing chunk frames = [3074048,3074560) | 20260505 12:05:50.081 | INFO | MemInfo: { | 20260505 12:05:50.081 | INFO | "gpu" : " 17 GB", | 20260505 12:05:50.081 | INFO | "huge" : "", | 20260505 12:05:50.081 | INFO | "huge_shared" : " 27 GB", | 20260505 12:05:50.081 | INFO | "malloc" : " 14 GB", | 20260505 12:05:50.081 | INFO | "pinned" : " 29 GB" | 20260505 12:05:50.081 | INFO | } | 20260505 12:05:50.081 | INFO | Perf report for the last 5.14109 seconds: | 20260505 12:05:50.081 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:05:50.081 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:50.081 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:50.081 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:50.081 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 12:05:50.081 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:50.081 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:50.081 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:05:50.081 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:50.081 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:50.081 | INFO | Uploaded 36GiB to GPU in 2.14548s (16.7795GiB/s) | 20260505 12:05:50.081 | INFO | Downloaded 34GiB to GPU in 2.12114s (16.0291GiB/s) | 20260505 12:05:55.206 | INFO | Analyzing chunk frames = [3074560,3075072) | 20260505 12:05:55.206 | INFO | MemInfo: { | 20260505 12:05:55.206 | INFO | "gpu" : " 17 GB", | 20260505 12:05:55.207 | INFO | "huge" : "", | 20260505 12:05:55.207 | INFO | "huge_shared" : " 27 GB", | 20260505 12:05:55.207 | INFO | "malloc" : " 14 GB", | 20260505 12:05:55.207 | INFO | "pinned" : " 29 GB" | 20260505 12:05:55.207 | INFO | } | 20260505 12:05:55.207 | INFO | Perf report for the last 5.12579 seconds: | 20260505 12:05:55.207 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:05:55.207 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:55.207 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:55.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:55.207 | INFO | ANALYSIS : 63.4%, 3.0, 98.8ms, 0 | 20260505 12:05:55.207 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:55.207 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:55.207 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 12:05:55.207 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:55.207 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:05:55.207 | INFO | Uploaded 36GiB to GPU in 2.1537s (16.7154GiB/s) | 20260505 12:05:55.207 | INFO | Downloaded 34GiB to GPU in 2.10624s (16.1425GiB/s) | 20260505 12:06:00.347 | INFO | Analyzing chunk frames = [3075072,3075584) | 20260505 12:06:00.347 | INFO | MemInfo: { | 20260505 12:06:00.347 | INFO | "gpu" : " 17 GB", | 20260505 12:06:00.347 | INFO | "huge" : "", | 20260505 12:06:00.347 | INFO | "huge_shared" : " 27 GB", | 20260505 12:06:00.347 | INFO | "malloc" : " 14 GB", | 20260505 12:06:00.347 | INFO | "pinned" : " 29 GB" | 20260505 12:06:00.347 | INFO | } | 20260505 12:06:00.347 | INFO | Perf report for the last 5.14085 seconds: | 20260505 12:06:00.347 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:06:00.347 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:00.347 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:00.348 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:00.348 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 12:06:00.348 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:00.348 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:00.348 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 12:06:00.348 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:00.348 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:00.348 | INFO | Uploaded 36GiB to GPU in 2.19304s (16.4155GiB/s) | 20260505 12:06:00.348 | INFO | Downloaded 34GiB to GPU in 2.15172s (15.8013GiB/s) | 20260505 12:06:05.498 | INFO | Analyzing chunk frames = [3075584,3076096) | 20260505 12:06:05.498 | INFO | MemInfo: { | 20260505 12:06:05.498 | INFO | "gpu" : " 17 GB", | 20260505 12:06:05.498 | INFO | "huge" : "", | 20260505 12:06:05.498 | INFO | "huge_shared" : " 27 GB", | 20260505 12:06:05.498 | INFO | "malloc" : " 14 GB", | 20260505 12:06:05.498 | INFO | "pinned" : " 29 GB" | 20260505 12:06:05.498 | INFO | } | 20260505 12:06:05.498 | INFO | Perf report for the last 5.15027 seconds: | 20260505 12:06:05.498 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:06:05.498 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:05.498 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:05.498 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:05.498 | INFO | ANALYSIS : 63.8%, 3.0, 99.8ms, 0 | 20260505 12:06:05.498 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:05.498 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:05.498 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 12:06:05.498 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:05.498 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:05.498 | INFO | Uploaded 36GiB to GPU in 2.2305s (16.1398GiB/s) | 20260505 12:06:05.498 | INFO | Downloaded 34GiB to GPU in 2.09259s (16.2478GiB/s) | 20260505 12:06:05.639 | INFO | Starting a round of Baz IO | 20260505 12:06:08.868 | INFO | Populating RTMetrics report | 20260505 12:06:08.868 | INFO | RTMetrics report populated | 20260505 12:06:08.868 | INFO | Opening tmp RTMetrics json file | 20260505 12:06:08.873 | INFO | Serializing RTMetrics to json | 20260505 12:06:08.893 | INFO | Serialization done, swapping tmp file now... | 20260505 12:06:08.894 | INFO | RTMetrics done | 20260505 12:06:08.987 | INFO | Finished a round of Baz IO: Wrote 2.353 GiB in 3.348 seconds | 20260505 12:06:08.987 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.77% / 0.01% / 76.22% | 20260505 12:06:08.987 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.96887% / 3.97364e-06% / 9.13938e-05% | 20260505 12:06:08.987 | INFO | Baz IO overall: | 20260505 12:06:08.987 | INFO | BytesWritten : 1825256638862 | 20260505 12:06:08.987 | INFO | NumEvents : 868923379272 | 20260505 12:06:10.636 | INFO | Analyzing chunk frames = [3076096,3076608) | 20260505 12:06:10.636 | INFO | MemInfo: { | 20260505 12:06:10.636 | INFO | "gpu" : " 17 GB", | 20260505 12:06:10.636 | INFO | "huge" : "", | 20260505 12:06:10.636 | INFO | "huge_shared" : " 27 GB", | 20260505 12:06:10.636 | INFO | "malloc" : " 14 GB", | 20260505 12:06:10.636 | INFO | "pinned" : " 29 GB" | 20260505 12:06:10.636 | INFO | } | 20260505 12:06:10.636 | INFO | Perf report for the last 5.13802 seconds: | 20260505 12:06:10.636 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:06:10.636 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:10.636 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:10.636 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:10.636 | INFO | ANALYSIS : 64.5%, 2.9, 99.7ms, 0 | 20260505 12:06:10.636 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:10.636 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260505 12:06:10.636 | INFO | PRE_HQ : 61.2%, 1.8, 58.2ms, 0 | 20260505 12:06:10.636 | INFO | BAZWRITER : 64.5%, 4.1, 135.0ms, 0 | 20260505 12:06:10.636 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 12:06:10.636 | INFO | Uploaded 36GiB to GPU in 2.19723s (16.3842GiB/s) | 20260505 12:06:10.636 | INFO | Downloaded 38GiB to GPU in 2.27835s (16.6787GiB/s) | 20260505 12:06:15.773 | INFO | Analyzing chunk frames = [3076608,3077120) | 20260505 12:06:15.773 | INFO | MemInfo: { | 20260505 12:06:15.773 | INFO | "gpu" : " 17 GB", | 20260505 12:06:15.773 | INFO | "huge" : "", | 20260505 12:06:15.773 | INFO | "huge_shared" : " 27 GB", | 20260505 12:06:15.773 | INFO | "malloc" : " 14 GB", | 20260505 12:06:15.773 | INFO | "pinned" : " 29 GB" | 20260505 12:06:15.773 | INFO | } | 20260505 12:06:15.773 | INFO | Perf report for the last 5.1374 seconds: | 20260505 12:06:15.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:06:15.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:15.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:15.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:15.773 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 12:06:15.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:15.773 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:15.773 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:06:15.773 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:15.773 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:15.773 | INFO | Uploaded 36GiB to GPU in 2.15869s (16.6768GiB/s) | 20260505 12:06:15.774 | INFO | Downloaded 34GiB to GPU in 2.03564s (16.7024GiB/s) | 20260505 12:06:20.909 | INFO | Analyzing chunk frames = [3077120,3077632) | 20260505 12:06:20.909 | INFO | MemInfo: { | 20260505 12:06:20.909 | INFO | "gpu" : " 17 GB", | 20260505 12:06:20.909 | INFO | "huge" : "", | 20260505 12:06:20.909 | INFO | "huge_shared" : " 27 GB", | 20260505 12:06:20.909 | INFO | "malloc" : " 14 GB", | 20260505 12:06:20.909 | INFO | "pinned" : " 29 GB" | 20260505 12:06:20.909 | INFO | } | 20260505 12:06:20.909 | INFO | Perf report for the last 5.13579 seconds: | 20260505 12:06:20.909 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:06:20.909 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:20.909 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:20.909 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:20.909 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 12:06:20.909 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:20.909 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:20.909 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:06:20.909 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:20.909 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:20.909 | INFO | Uploaded 36GiB to GPU in 2.22461s (16.1826GiB/s) | 20260505 12:06:20.909 | INFO | Downloaded 34GiB to GPU in 2.12159s (16.0257GiB/s) | 20260505 12:06:26.048 | INFO | Analyzing chunk frames = [3077632,3078144) | 20260505 12:06:26.048 | INFO | MemInfo: { | 20260505 12:06:26.048 | INFO | "gpu" : " 17 GB", | 20260505 12:06:26.048 | INFO | "huge" : "", | 20260505 12:06:26.048 | INFO | "huge_shared" : " 27 GB", | 20260505 12:06:26.048 | INFO | "malloc" : " 14 GB", | 20260505 12:06:26.048 | INFO | "pinned" : " 29 GB" | 20260505 12:06:26.048 | INFO | } | 20260505 12:06:26.048 | INFO | Perf report for the last 5.13905 seconds: | 20260505 12:06:26.048 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:06:26.048 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:26.048 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:26.048 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:26.048 | INFO | ANALYSIS : 63.5%, 3.0, 98.6ms, 0 | 20260505 12:06:26.048 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:26.048 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:26.048 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:06:26.048 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:26.048 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:26.048 | INFO | Uploaded 36GiB to GPU in 2.21864s (16.2262GiB/s) | 20260505 12:06:26.048 | INFO | Downloaded 34GiB to GPU in 2.13433s (15.93GiB/s) | 20260505 12:06:31.186 | INFO | Analyzing chunk frames = [3078144,3078656) | 20260505 12:06:31.186 | INFO | MemInfo: { | 20260505 12:06:31.186 | INFO | "gpu" : " 17 GB", | 20260505 12:06:31.186 | INFO | "huge" : "", | 20260505 12:06:31.186 | INFO | "huge_shared" : " 27 GB", | 20260505 12:06:31.186 | INFO | "malloc" : " 14 GB", | 20260505 12:06:31.186 | INFO | "pinned" : " 29 GB" | 20260505 12:06:31.186 | INFO | } | 20260505 12:06:31.186 | INFO | Perf report for the last 5.13806 seconds: | 20260505 12:06:31.186 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:06:31.186 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:31.186 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:31.186 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:31.186 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 12:06:31.186 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:31.186 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:31.186 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:06:31.186 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:31.186 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:31.186 | INFO | Uploaded 36GiB to GPU in 2.16643s (16.6172GiB/s) | 20260505 12:06:31.186 | INFO | Downloaded 34GiB to GPU in 2.12697s (15.9852GiB/s) | 20260505 12:06:36.333 | INFO | Analyzing chunk frames = [3078656,3079168) | 20260505 12:06:36.333 | INFO | MemInfo: { | 20260505 12:06:36.333 | INFO | "gpu" : " 17 GB", | 20260505 12:06:36.333 | INFO | "huge" : "", | 20260505 12:06:36.333 | INFO | "huge_shared" : " 27 GB", | 20260505 12:06:36.333 | INFO | "malloc" : " 14 GB", | 20260505 12:06:36.333 | INFO | "pinned" : " 29 GB" | 20260505 12:06:36.333 | INFO | } | 20260505 12:06:36.333 | INFO | Perf report for the last 5.14717 seconds: | 20260505 12:06:36.333 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:06:36.334 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:36.334 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:36.334 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:36.334 | INFO | ANALYSIS : 63.1%, 3.0, 98.7ms, 0 | 20260505 12:06:36.334 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:36.334 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:36.334 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:06:36.334 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:36.334 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:36.334 | INFO | Uploaded 36GiB to GPU in 2.15571s (16.6998GiB/s) | 20260505 12:06:36.334 | INFO | Downloaded 34GiB to GPU in 2.10866s (16.124GiB/s) | 20260505 12:06:41.470 | INFO | Analyzing chunk frames = [3079168,3079680) | 20260505 12:06:41.470 | INFO | MemInfo: { | 20260505 12:06:41.470 | INFO | "gpu" : " 17 GB", | 20260505 12:06:41.470 | INFO | "huge" : "", | 20260505 12:06:41.470 | INFO | "huge_shared" : " 27 GB", | 20260505 12:06:41.470 | INFO | "malloc" : " 14 GB", | 20260505 12:06:41.470 | INFO | "pinned" : " 29 GB" | 20260505 12:06:41.470 | INFO | } | 20260505 12:06:41.470 | INFO | Perf report for the last 5.13682 seconds: | 20260505 12:06:41.470 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:06:41.470 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:41.470 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:41.470 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:41.470 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 12:06:41.470 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:41.470 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:41.470 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 12:06:41.470 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:41.470 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:41.470 | INFO | Uploaded 36GiB to GPU in 2.17852s (16.525GiB/s) | 20260505 12:06:41.470 | INFO | Downloaded 34GiB to GPU in 2.14386s (15.8593GiB/s) | 20260505 12:06:46.603 | INFO | Analyzing chunk frames = [3079680,3080192) | 20260505 12:06:46.603 | INFO | MemInfo: { | 20260505 12:06:46.603 | INFO | "gpu" : " 17 GB", | 20260505 12:06:46.603 | INFO | "huge" : "", | 20260505 12:06:46.603 | INFO | "huge_shared" : " 27 GB", | 20260505 12:06:46.603 | INFO | "malloc" : " 14 GB", | 20260505 12:06:46.603 | INFO | "pinned" : " 29 GB" | 20260505 12:06:46.603 | INFO | } | 20260505 12:06:46.603 | INFO | Perf report for the last 5.13308 seconds: | 20260505 12:06:46.603 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:06:46.603 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:46.603 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:46.603 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:46.603 | INFO | ANALYSIS : 63.8%, 3.0, 99.5ms, 0 | 20260505 12:06:46.603 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:46.603 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:46.603 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 12:06:46.603 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:46.603 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:46.603 | INFO | Uploaded 36GiB to GPU in 2.21752s (16.2344GiB/s) | 20260505 12:06:46.604 | INFO | Downloaded 34GiB to GPU in 2.10447s (16.1561GiB/s) | 20260505 12:06:46.604 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 12:06:46.604 | INFO | Total mem usage: 15.3 GiB | 20260505 12:06:46.604 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 12:06:46.604 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -7.19%) | 20260505 12:06:46.604 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 12:06:46.604 | INFO | High Water Breakdown: | 20260505 12:06:46.604 | INFO | 61% : 8.73 GiB ( 91.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 12:06:46.604 | INFO | 38.8% : 5.55 GiB ( 83.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 12:06:46.604 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 12:06:46.604 | INFO | ------------------------------------------ | 20260505 12:06:46.604 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:06:46.604 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 12:06:46.604 | INFO | Total mem usage: 26.5 GiB | 20260505 12:06:46.604 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 12:06:46.604 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 12:06:46.604 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 12:06:46.604 | INFO | High Water Breakdown: | 20260505 12:06:46.604 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:06:46.604 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:06:46.604 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:06:46.604 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:06:46.604 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:06:46.604 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:06:46.604 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 12:06:46.604 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 12:06:46.604 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:06:46.604 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 12:06:46.604 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 12:06:46.604 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 12:06:46.604 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 12:06:46.604 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 12:06:46.604 | INFO | ------------------------------------------ | 20260505 12:06:46.604 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:06:46.604 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 12:06:46.604 | INFO | Total mem usage: 25.6 GiB | 20260505 12:06:46.604 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 12:06:46.604 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 12:06:46.604 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 12:06:46.604 | INFO | High Water Breakdown: | 20260505 12:06:46.604 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 12:06:46.604 | INFO | ------------------------------------------ | 20260505 12:06:46.604 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:06:46.604 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 12:06:46.604 | INFO | Total mem usage: 15.7 GiB | 20260505 12:06:46.604 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 12:06:46.604 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 12:06:46.604 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 12:06:46.604 | INFO | High Water Breakdown: | 20260505 12:06:46.604 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 12:06:46.604 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 12:06:46.604 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 12:06:46.604 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 12:06:46.604 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 12:06:46.604 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 12:06:46.604 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 12:06:46.604 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 12:06:46.604 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 12:06:46.604 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:06:46.604 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 12:06:46.604 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:06:46.604 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 12:06:46.604 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 12:06:46.604 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 12:06:46.604 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 12:06:46.604 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:06:46.604 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 12:06:46.604 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:06:46.604 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:06:46.604 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:06:46.604 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 12:06:46.604 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 12:06:46.604 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:06:46.604 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 12:06:46.604 | INFO | ------------------------------------------ | 20260505 12:06:46.604 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:06:46.604 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 12:06:46.604 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:06:46.604 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:06:46.604 | INFO | PerfReport: Full Execution -- 3.21e+01 / 7.68e+01, 1.38e+01 / 1.29e+00 | 20260505 12:06:46.604 | INFO | PerfReport: 0.29%, AggregateStats -- 9.31e-02 / 1.01e-01, 1.06e-02 / 3.65e-03 | 20260505 12:06:46.605 | INFO | PerfReport: 11.75%, Baseline -- 3.77e+00 / 4.49e+00, 6.02e-01 / 1.80e-01 | 20260505 12:06:46.605 | INFO | PerfReport: 3.31%, Histogram -- 1.06e+00 / 1.50e+00, 2.04e-01 / 9.81e-02 | 20260505 12:06:46.605 | INFO | PerfReport: 15.78%, DME -- 4.05e+01 / 4.47e+01, 4.18e+00 / 2.07e+00 | 20260505 12:06:46.605 | INFO | PerfReport: 56.51%, FrameLabeling -- 1.81e+01 / 2.12e+01, 1.60e+00 / 2.41e+00 | 20260505 12:06:46.605 | INFO | PerfReport: 7.42%, PulseAccumulating -- 2.38e+00 / 2.84e+00, 4.32e-01 / 2.49e-01 | 20260505 12:06:46.605 | INFO | PerfReport: 3.66%, Metrics -- 1.17e+00 / 1.48e+00, 3.33e-01 / 1.93e-01 | 20260505 12:06:46.605 | INFO | PerfReport: 1.29%, PulseSerializing -- 4.12e-01 / 4.66e-01, 7.50e-02 / 3.35e-02 | 20260505 12:06:46.605 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 12:06:46.605 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:06:46.605 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:06:46.605 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260505 12:06:46.605 | INFO | PerfReport: 22.61%, Upload -- 2.28e+01 / 0.00e+00, 8.13e+00 / 0.00e+00 | 20260505 12:06:46.605 | INFO | PerfReport: 22.01%, Download -- 2.22e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260505 12:06:46.605 | INFO | PerfReport: 31.74%, Compute -- 3.21e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 12:06:46.605 | INFO | PerfReport: 2.11%, LockUpload -- 2.13e+00 / 0.00e+00, 7.72e+00 / 0.00e+00 | 20260505 12:06:46.605 | INFO | PerfReport: 1.26%, LockDownload -- 1.28e+00 / 0.00e+00, 2.79e+00 / 0.00e+00 | 20260505 12:06:46.605 | INFO | PerfReport: 20.25%, LockCompute -- 2.05e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260505 12:06:46.605 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.80e-02 / 0.00e+00 | 20260505 12:06:46.742 | INFO | Starting a round of Baz IO | 20260505 12:06:49.993 | INFO | Populating RTMetrics report | 20260505 12:06:49.994 | INFO | RTMetrics report populated | 20260505 12:06:49.994 | INFO | Opening tmp RTMetrics json file | 20260505 12:06:50.017 | INFO | Serializing RTMetrics to json | 20260505 12:06:50.022 | INFO | Serialization done, swapping tmp file now... | 20260505 12:06:50.024 | INFO | RTMetrics done | 20260505 12:06:50.118 | INFO | Finished a round of Baz IO: Wrote 2.349 GiB in 3.376 seconds | 20260505 12:06:50.118 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.76% / 0.01% / 76.23% | 20260505 12:06:50.118 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.962317% / 3.97364e-06% / 7.94729e-05% | 20260505 12:06:50.118 | INFO | Baz IO overall: | 20260505 12:06:50.118 | INFO | BytesWritten : 1827779078906 | 20260505 12:06:50.118 | INFO | NumEvents : 870120327197 | 20260505 12:06:51.753 | INFO | Analyzing chunk frames = [3080192,3080704) | 20260505 12:06:51.753 | INFO | MemInfo: { | 20260505 12:06:51.753 | INFO | "gpu" : " 17 GB", | 20260505 12:06:51.753 | INFO | "huge" : "", | 20260505 12:06:51.753 | INFO | "huge_shared" : " 27 GB", | 20260505 12:06:51.753 | INFO | "malloc" : " 14 GB", | 20260505 12:06:51.753 | INFO | "pinned" : " 29 GB" | 20260505 12:06:51.753 | INFO | } | 20260505 12:06:51.753 | INFO | Perf report for the last 5.14944 seconds: | 20260505 12:06:51.753 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:06:51.753 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:51.753 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:51.753 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:51.753 | INFO | ANALYSIS : 64.7%, 2.9, 100.2ms, 0 | 20260505 12:06:51.753 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:51.753 | INFO | RT_METRICS : 46.0%, 1.0, 24.2ms, 0 | 20260505 12:06:51.753 | INFO | PRE_HQ : 61.2%, 1.8, 58.5ms, 0 | 20260505 12:06:51.753 | INFO | BAZWRITER : 64.9%, 4.1, 135.4ms, 0 | 20260505 12:06:51.753 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 12:06:51.753 | INFO | Uploaded 36GiB to GPU in 2.18278s (16.4927GiB/s) | 20260505 12:06:51.753 | INFO | Downloaded 38GiB to GPU in 2.29977s (16.5234GiB/s) | 20260505 12:06:56.890 | INFO | Analyzing chunk frames = [3080704,3081216) | 20260505 12:06:56.890 | INFO | MemInfo: { | 20260505 12:06:56.890 | INFO | "gpu" : " 17 GB", | 20260505 12:06:56.890 | INFO | "huge" : "", | 20260505 12:06:56.890 | INFO | "huge_shared" : " 27 GB", | 20260505 12:06:56.890 | INFO | "malloc" : " 14 GB", | 20260505 12:06:56.890 | INFO | "pinned" : " 29 GB" | 20260505 12:06:56.890 | INFO | } | 20260505 12:06:56.890 | INFO | Perf report for the last 5.13717 seconds: | 20260505 12:06:56.890 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:06:56.890 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:56.890 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:56.890 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:56.890 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 12:06:56.890 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:56.890 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:56.890 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 12:06:56.890 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:56.890 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:06:56.890 | INFO | Uploaded 36GiB to GPU in 2.1613s (16.6566GiB/s) | 20260505 12:06:56.890 | INFO | Downloaded 34GiB to GPU in 2.03346s (16.7203GiB/s) | 20260505 12:07:02.022 | INFO | Analyzing chunk frames = [3081216,3081728) | 20260505 12:07:02.022 | INFO | MemInfo: { | 20260505 12:07:02.022 | INFO | "gpu" : " 17 GB", | 20260505 12:07:02.022 | INFO | "huge" : "", | 20260505 12:07:02.022 | INFO | "huge_shared" : " 27 GB", | 20260505 12:07:02.022 | INFO | "malloc" : " 14 GB", | 20260505 12:07:02.022 | INFO | "pinned" : " 29 GB" | 20260505 12:07:02.022 | INFO | } | 20260505 12:07:02.023 | INFO | Perf report for the last 5.13268 seconds: | 20260505 12:07:02.023 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:07:02.023 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:02.023 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:02.023 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:02.023 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 12:07:02.023 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:02.023 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:02.023 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 12:07:02.023 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:02.023 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:02.023 | INFO | Uploaded 36GiB to GPU in 2.16033s (16.6641GiB/s) | 20260505 12:07:02.023 | INFO | Downloaded 34GiB to GPU in 2.1135s (16.0871GiB/s) | 20260505 12:07:07.157 | INFO | Analyzing chunk frames = [3081728,3082240) | 20260505 12:07:07.157 | INFO | MemInfo: { | 20260505 12:07:07.157 | INFO | "gpu" : " 17 GB", | 20260505 12:07:07.157 | INFO | "huge" : "", | 20260505 12:07:07.157 | INFO | "huge_shared" : " 27 GB", | 20260505 12:07:07.157 | INFO | "malloc" : " 14 GB", | 20260505 12:07:07.157 | INFO | "pinned" : " 29 GB" | 20260505 12:07:07.157 | INFO | } | 20260505 12:07:07.157 | INFO | Perf report for the last 5.1348 seconds: | 20260505 12:07:07.158 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:07:07.158 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:07.158 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:07.158 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:07.158 | INFO | ANALYSIS : 63.3%, 3.0, 98.3ms, 0 | 20260505 12:07:07.158 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:07.158 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:07.158 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 12:07:07.158 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:07.158 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:07.158 | INFO | Uploaded 36GiB to GPU in 2.21106s (16.2818GiB/s) | 20260505 12:07:07.158 | INFO | Downloaded 34GiB to GPU in 2.12934s (15.9674GiB/s) | 20260505 12:07:12.300 | INFO | Analyzing chunk frames = [3082240,3082752) | 20260505 12:07:12.300 | INFO | MemInfo: { | 20260505 12:07:12.300 | INFO | "gpu" : " 17 GB", | 20260505 12:07:12.300 | INFO | "huge" : "", | 20260505 12:07:12.300 | INFO | "huge_shared" : " 27 GB", | 20260505 12:07:12.300 | INFO | "malloc" : " 14 GB", | 20260505 12:07:12.300 | INFO | "pinned" : " 29 GB" | 20260505 12:07:12.300 | INFO | } | 20260505 12:07:12.300 | INFO | Perf report for the last 5.14245 seconds: | 20260505 12:07:12.300 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:07:12.300 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:12.300 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:12.300 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:12.300 | INFO | ANALYSIS : 63.2%, 3.0, 98.8ms, 0 | 20260505 12:07:12.300 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:12.300 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:12.300 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 12:07:12.300 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:12.300 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:12.300 | INFO | Uploaded 36GiB to GPU in 2.19858s (16.3742GiB/s) | 20260505 12:07:12.300 | INFO | Downloaded 34GiB to GPU in 2.12985s (15.9635GiB/s) | 20260505 12:07:17.449 | INFO | Analyzing chunk frames = [3082752,3083264) | 20260505 12:07:17.449 | INFO | MemInfo: { | 20260505 12:07:17.449 | INFO | "gpu" : " 17 GB", | 20260505 12:07:17.449 | INFO | "huge" : "", | 20260505 12:07:17.449 | INFO | "huge_shared" : " 27 GB", | 20260505 12:07:17.449 | INFO | "malloc" : " 14 GB", | 20260505 12:07:17.449 | INFO | "pinned" : " 29 GB" | 20260505 12:07:17.449 | INFO | } | 20260505 12:07:17.449 | INFO | Perf report for the last 5.14918 seconds: | 20260505 12:07:17.449 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:07:17.449 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:17.449 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:17.449 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:17.449 | INFO | ANALYSIS : 63.3%, 3.0, 99.0ms, 0 | 20260505 12:07:17.449 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:17.449 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:17.449 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 12:07:17.449 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:17.449 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:17.449 | INFO | Uploaded 36GiB to GPU in 2.18587s (16.4694GiB/s) | 20260505 12:07:17.449 | INFO | Downloaded 34GiB to GPU in 2.1186s (16.0483GiB/s) | 20260505 12:07:22.589 | INFO | Analyzing chunk frames = [3083264,3083776) | 20260505 12:07:22.589 | INFO | MemInfo: { | 20260505 12:07:22.589 | INFO | "gpu" : " 17 GB", | 20260505 12:07:22.589 | INFO | "huge" : "", | 20260505 12:07:22.589 | INFO | "huge_shared" : " 27 GB", | 20260505 12:07:22.589 | INFO | "malloc" : " 14 GB", | 20260505 12:07:22.589 | INFO | "pinned" : " 29 GB" | 20260505 12:07:22.589 | INFO | } | 20260505 12:07:22.589 | INFO | Perf report for the last 5.14019 seconds: | 20260505 12:07:22.589 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:07:22.589 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:22.589 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:22.589 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:22.589 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 12:07:22.589 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:22.589 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:22.589 | INFO | PRE_HQ : 30.1%, 1.0, 15.7ms, 0 | 20260505 12:07:22.589 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:22.589 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:22.589 | INFO | Uploaded 36GiB to GPU in 2.15929s (16.6722GiB/s) | 20260505 12:07:22.590 | INFO | Downloaded 34GiB to GPU in 2.14398s (15.8583GiB/s) | 20260505 12:07:27.721 | INFO | Analyzing chunk frames = [3083776,3084288) | 20260505 12:07:27.721 | INFO | MemInfo: { | 20260505 12:07:27.721 | INFO | "gpu" : " 17 GB", | 20260505 12:07:27.721 | INFO | "huge" : "", | 20260505 12:07:27.721 | INFO | "huge_shared" : " 27 GB", | 20260505 12:07:27.721 | INFO | "malloc" : " 14 GB", | 20260505 12:07:27.721 | INFO | "pinned" : " 29 GB" | 20260505 12:07:27.721 | INFO | } | 20260505 12:07:27.721 | INFO | Perf report for the last 5.13202 seconds: | 20260505 12:07:27.721 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:07:27.721 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:27.721 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:27.721 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:27.721 | INFO | ANALYSIS : 63.8%, 3.0, 99.5ms, 0 | 20260505 12:07:27.721 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:27.721 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:27.721 | INFO | PRE_HQ : 29.8%, 1.0, 15.4ms, 0 | 20260505 12:07:27.721 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:27.721 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:27.721 | INFO | Uploaded 36GiB to GPU in 2.16392s (16.6365GiB/s) | 20260505 12:07:27.722 | INFO | Downloaded 34GiB to GPU in 2.10488s (16.1529GiB/s) | 20260505 12:07:27.852 | INFO | Starting a round of Baz IO | 20260505 12:07:31.100 | INFO | Populating RTMetrics report | 20260505 12:07:31.100 | INFO | RTMetrics report populated | 20260505 12:07:31.100 | INFO | Opening tmp RTMetrics json file | 20260505 12:07:31.109 | INFO | Serializing RTMetrics to json | 20260505 12:07:31.131 | INFO | Serialization done, swapping tmp file now... | 20260505 12:07:31.135 | INFO | RTMetrics done | 20260505 12:07:31.236 | INFO | Finished a round of Baz IO: Wrote 2.35 GiB in 3.384 seconds | 20260505 12:07:31.237 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.76% / 0.01% / 76.24% | 20260505 12:07:31.237 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.958991% / 3.97364e-06% / 7.54992e-05% | 20260505 12:07:31.237 | INFO | Baz IO overall: | 20260505 12:07:31.237 | INFO | BytesWritten : 1830301866890 | 20260505 12:07:31.237 | INFO | NumEvents : 871317702597 | 20260505 12:07:32.863 | INFO | Analyzing chunk frames = [3084288,3084800) | 20260505 12:07:32.863 | INFO | MemInfo: { | 20260505 12:07:32.863 | INFO | "gpu" : " 17 GB", | 20260505 12:07:32.863 | INFO | "huge" : "", | 20260505 12:07:32.863 | INFO | "huge_shared" : " 27 GB", | 20260505 12:07:32.863 | INFO | "malloc" : " 14 GB", | 20260505 12:07:32.863 | INFO | "pinned" : " 29 GB" | 20260505 12:07:32.863 | INFO | } | 20260505 12:07:32.863 | INFO | Perf report for the last 5.14196 seconds: | 20260505 12:07:32.863 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:07:32.863 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:32.863 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:32.863 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:32.863 | INFO | ANALYSIS : 64.6%, 2.9, 99.1ms, 0 | 20260505 12:07:32.863 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:32.863 | INFO | RT_METRICS : 46.6%, 1.0, 24.4ms, 0 | 20260505 12:07:32.863 | INFO | PRE_HQ : 61.1%, 1.9, 58.8ms, 0 | 20260505 12:07:32.863 | INFO | BAZWRITER : 65.2%, 4.1, 139.0ms, 0 | 20260505 12:07:32.863 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 12:07:32.863 | INFO | Uploaded 36GiB to GPU in 2.18698s (16.461GiB/s) | 20260505 12:07:32.863 | INFO | Downloaded 38GiB to GPU in 2.28193s (16.6525GiB/s) | 20260505 12:07:38.001 | INFO | Analyzing chunk frames = [3084800,3085312) | 20260505 12:07:38.001 | INFO | MemInfo: { | 20260505 12:07:38.001 | INFO | "gpu" : " 17 GB", | 20260505 12:07:38.001 | INFO | "huge" : "", | 20260505 12:07:38.001 | INFO | "huge_shared" : " 27 GB", | 20260505 12:07:38.001 | INFO | "malloc" : " 14 GB", | 20260505 12:07:38.001 | INFO | "pinned" : " 29 GB" | 20260505 12:07:38.002 | INFO | } | 20260505 12:07:38.002 | INFO | Perf report for the last 5.13831 seconds: | 20260505 12:07:38.002 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:07:38.002 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:38.002 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:38.002 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:38.002 | INFO | ANALYSIS : 64.1%, 3.0, 100.0ms, 0 | 20260505 12:07:38.002 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:38.002 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:38.002 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 12:07:38.002 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:38.002 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:38.002 | INFO | Uploaded 36GiB to GPU in 2.23118s (16.1349GiB/s) | 20260505 12:07:38.002 | INFO | Downloaded 34GiB to GPU in 2.09736s (16.2109GiB/s) | 20260505 12:07:43.139 | INFO | Analyzing chunk frames = [3085312,3085824) | 20260505 12:07:43.139 | INFO | MemInfo: { | 20260505 12:07:43.139 | INFO | "gpu" : " 17 GB", | 20260505 12:07:43.139 | INFO | "huge" : "", | 20260505 12:07:43.139 | INFO | "huge_shared" : " 27 GB", | 20260505 12:07:43.139 | INFO | "malloc" : " 14 GB", | 20260505 12:07:43.139 | INFO | "pinned" : " 29 GB" | 20260505 12:07:43.139 | INFO | } | 20260505 12:07:43.139 | INFO | Perf report for the last 5.13759 seconds: | 20260505 12:07:43.139 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:07:43.139 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:43.139 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:43.139 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:43.139 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 12:07:43.139 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:43.139 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:43.139 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:07:43.139 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:43.139 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:43.139 | INFO | Uploaded 36GiB to GPU in 2.22399s (16.1871GiB/s) | 20260505 12:07:43.139 | INFO | Downloaded 34GiB to GPU in 2.11919s (16.0439GiB/s) | 20260505 12:07:48.282 | INFO | Analyzing chunk frames = [3085824,3086336) | 20260505 12:07:48.282 | INFO | MemInfo: { | 20260505 12:07:48.282 | INFO | "gpu" : " 17 GB", | 20260505 12:07:48.282 | INFO | "huge" : "", | 20260505 12:07:48.282 | INFO | "huge_shared" : " 27 GB", | 20260505 12:07:48.282 | INFO | "malloc" : " 14 GB", | 20260505 12:07:48.282 | INFO | "pinned" : " 29 GB" | 20260505 12:07:48.282 | INFO | } | 20260505 12:07:48.282 | INFO | Perf report for the last 5.14274 seconds: | 20260505 12:07:48.282 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:07:48.282 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:48.282 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:48.282 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:48.282 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 12:07:48.282 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:48.282 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:48.282 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 12:07:48.282 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:48.282 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:48.282 | INFO | Uploaded 36GiB to GPU in 2.21076s (16.284GiB/s) | 20260505 12:07:48.282 | INFO | Downloaded 34GiB to GPU in 2.13452s (15.9286GiB/s) | 20260505 12:07:53.414 | INFO | Analyzing chunk frames = [3086336,3086848) | 20260505 12:07:53.414 | INFO | MemInfo: { | 20260505 12:07:53.414 | INFO | "gpu" : " 17 GB", | 20260505 12:07:53.414 | INFO | "huge" : "", | 20260505 12:07:53.414 | INFO | "huge_shared" : " 27 GB", | 20260505 12:07:53.414 | INFO | "malloc" : " 14 GB", | 20260505 12:07:53.414 | INFO | "pinned" : " 29 GB" | 20260505 12:07:53.414 | INFO | } | 20260505 12:07:53.415 | INFO | Perf report for the last 5.13261 seconds: | 20260505 12:07:53.415 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:07:53.415 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:53.415 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:53.415 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:53.415 | INFO | ANALYSIS : 63.1%, 3.0, 98.3ms, 0 | 20260505 12:07:53.415 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:53.415 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:53.415 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:07:53.415 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:53.415 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:53.415 | INFO | Uploaded 36GiB to GPU in 2.17473s (16.5538GiB/s) | 20260505 12:07:53.415 | INFO | Downloaded 34GiB to GPU in 2.12508s (15.9994GiB/s) | 20260505 12:07:58.558 | INFO | Analyzing chunk frames = [3086848,3087360) | 20260505 12:07:58.558 | INFO | MemInfo: { | 20260505 12:07:58.558 | INFO | "gpu" : " 17 GB", | 20260505 12:07:58.558 | INFO | "huge" : "", | 20260505 12:07:58.558 | INFO | "huge_shared" : " 27 GB", | 20260505 12:07:58.558 | INFO | "malloc" : " 14 GB", | 20260505 12:07:58.558 | INFO | "pinned" : " 29 GB" | 20260505 12:07:58.558 | INFO | } | 20260505 12:07:58.558 | INFO | Perf report for the last 5.14387 seconds: | 20260505 12:07:58.558 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:07:58.558 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:58.558 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:58.558 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:58.558 | INFO | ANALYSIS : 63.2%, 3.0, 98.8ms, 0 | 20260505 12:07:58.558 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:58.558 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:58.558 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:07:58.558 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:58.559 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:07:58.559 | INFO | Uploaded 36GiB to GPU in 2.17485s (16.5529GiB/s) | 20260505 12:07:58.559 | INFO | Downloaded 34GiB to GPU in 2.11034s (16.1111GiB/s) | 20260505 12:08:03.700 | INFO | Analyzing chunk frames = [3087360,3087872) | 20260505 12:08:03.700 | INFO | MemInfo: { | 20260505 12:08:03.700 | INFO | "gpu" : " 17 GB", | 20260505 12:08:03.700 | INFO | "huge" : "", | 20260505 12:08:03.700 | INFO | "huge_shared" : " 27 GB", | 20260505 12:08:03.700 | INFO | "malloc" : " 14 GB", | 20260505 12:08:03.700 | INFO | "pinned" : " 29 GB" | 20260505 12:08:03.700 | INFO | } | 20260505 12:08:03.700 | INFO | Perf report for the last 5.14149 seconds: | 20260505 12:08:03.700 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:08:03.700 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:03.700 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:03.700 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:03.700 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 12:08:03.700 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:03.700 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:03.700 | INFO | PRE_HQ : 29.9%, 1.0, 15.6ms, 0 | 20260505 12:08:03.700 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:03.700 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:03.700 | INFO | Uploaded 36GiB to GPU in 2.17252s (16.5706GiB/s) | 20260505 12:08:03.700 | INFO | Downloaded 34GiB to GPU in 2.12749s (15.9813GiB/s) | 20260505 12:08:08.840 | INFO | Analyzing chunk frames = [3087872,3088384) | 20260505 12:08:08.840 | INFO | MemInfo: { | 20260505 12:08:08.840 | INFO | "gpu" : " 17 GB", | 20260505 12:08:08.840 | INFO | "huge" : "", | 20260505 12:08:08.841 | INFO | "huge_shared" : " 27 GB", | 20260505 12:08:08.841 | INFO | "malloc" : " 14 GB", | 20260505 12:08:08.841 | INFO | "pinned" : " 29 GB" | 20260505 12:08:08.841 | INFO | } | 20260505 12:08:08.841 | INFO | Perf report for the last 5.14067 seconds: | 20260505 12:08:08.841 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:08:08.841 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:08.841 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:08.841 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:08.841 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 12:08:08.841 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:08.841 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:08.841 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 12:08:08.841 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:08.841 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:08.841 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 12:08:08.841 | INFO | Total mem usage: 15.3 GiB | 20260505 12:08:08.841 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 12:08:08.841 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -6.03%) | 20260505 12:08:08.841 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 12:08:08.841 | INFO | High Water Breakdown: | 20260505 12:08:08.841 | INFO | 61.2% : 8.85 GiB ( 93% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 12:08:08.841 | INFO | 38.7% : 5.59 GiB ( 84.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 12:08:08.841 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 12:08:08.841 | INFO | ------------------------------------------ | 20260505 12:08:08.841 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:08:08.841 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 12:08:08.841 | INFO | Total mem usage: 26.5 GiB | 20260505 12:08:08.841 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 12:08:08.841 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 12:08:08.841 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 12:08:08.841 | INFO | High Water Breakdown: | 20260505 12:08:08.841 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:08:08.841 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:08:08.841 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:08:08.841 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:08:08.841 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:08:08.841 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:08:08.841 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 12:08:08.841 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 12:08:08.841 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:08:08.841 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 12:08:08.841 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 12:08:08.841 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 12:08:08.841 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 12:08:08.841 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 12:08:08.841 | INFO | ------------------------------------------ | 20260505 12:08:08.841 | INFO | Uploaded 36GiB to GPU in 2.10453s (17.1059GiB/s) | 20260505 12:08:08.841 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:08:08.841 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 12:08:08.841 | INFO | Total mem usage: 25.6 GiB | 20260505 12:08:08.841 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 12:08:08.841 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 12:08:08.841 | INFO | Downloaded 34GiB to GPU in 2.09886s (16.1993GiB/s) | 20260505 12:08:08.841 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 12:08:08.841 | INFO | High Water Breakdown: | 20260505 12:08:08.841 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 12:08:08.841 | INFO | ------------------------------------------ | 20260505 12:08:08.841 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:08:08.841 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 12:08:08.841 | INFO | Total mem usage: 15.7 GiB | 20260505 12:08:08.841 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 12:08:08.841 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 12:08:08.841 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 12:08:08.841 | INFO | High Water Breakdown: | 20260505 12:08:08.841 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 12:08:08.841 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 12:08:08.841 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 12:08:08.841 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 12:08:08.841 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 12:08:08.841 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 12:08:08.842 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 12:08:08.842 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 12:08:08.842 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 12:08:08.842 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:08:08.842 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 12:08:08.842 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:08:08.842 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 12:08:08.842 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 12:08:08.842 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 12:08:08.842 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 12:08:08.842 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:08:08.842 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 12:08:08.842 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:08:08.842 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:08:08.842 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:08:08.842 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 12:08:08.842 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 12:08:08.842 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:08:08.842 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 12:08:08.842 | INFO | ------------------------------------------ | 20260505 12:08:08.842 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:08:08.842 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 12:08:08.842 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:08:08.842 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:08:08.842 | INFO | PerfReport: Full Execution -- 3.20e+01 / 7.73e+01, 1.40e+01 / 1.64e+00 | 20260505 12:08:08.842 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 1.01e-01, 1.04e-02 / 5.86e-03 | 20260505 12:08:08.842 | INFO | PerfReport: 11.75%, Baseline -- 3.77e+00 / 4.49e+00, 6.01e-01 / 1.98e-01 | 20260505 12:08:08.842 | INFO | PerfReport: 3.29%, Histogram -- 1.06e+00 / 1.51e+00, 2.04e-01 / 8.26e-02 | 20260505 12:08:08.842 | INFO | PerfReport: 16.02%, DME -- 4.11e+01 / 4.62e+01, 4.63e+00 / 2.07e+00 | 20260505 12:08:08.842 | INFO | PerfReport: 56.26%, FrameLabeling -- 1.80e+01 / 2.01e+01, 1.44e+00 / 2.00e+00 | 20260505 12:08:08.842 | INFO | PerfReport: 7.46%, PulseAccumulating -- 2.39e+00 / 2.90e+00, 4.49e-01 / 2.50e-01 | 20260505 12:08:08.842 | INFO | PerfReport: 3.65%, Metrics -- 1.17e+00 / 1.50e+00, 3.30e-01 / 1.90e-01 | 20260505 12:08:08.842 | INFO | PerfReport: 1.28%, PulseSerializing -- 4.11e-01 / 4.64e-01, 7.51e-02 / 4.03e-02 | 20260505 12:08:08.842 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 12:08:08.842 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:08:08.842 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:08:08.842 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260505 12:08:08.842 | INFO | PerfReport: 22.49%, Upload -- 2.27e+01 / 0.00e+00, 8.00e+00 / 0.00e+00 | 20260505 12:08:08.842 | INFO | PerfReport: 22.02%, Download -- 2.22e+01 / 0.00e+00, 8.14e+00 / 0.00e+00 | 20260505 12:08:08.842 | INFO | PerfReport: 31.71%, Compute -- 3.20e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260505 12:08:08.842 | INFO | PerfReport: 2.13%, LockUpload -- 2.16e+00 / 0.00e+00, 7.83e+00 / 0.00e+00 | 20260505 12:08:08.842 | INFO | PerfReport: 1.43%, LockDownload -- 1.44e+00 / 0.00e+00, 3.07e+00 / 0.00e+00 | 20260505 12:08:08.842 | INFO | PerfReport: 20.21%, LockCompute -- 2.04e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260505 12:08:08.842 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.96e-02 / 0.00e+00 | 20260505 12:08:08.973 | INFO | Starting a round of Baz IO | 20260505 12:08:12.224 | INFO | Populating RTMetrics report | 20260505 12:08:12.224 | INFO | RTMetrics report populated | 20260505 12:08:12.224 | INFO | Opening tmp RTMetrics json file | 20260505 12:08:12.257 | INFO | Serializing RTMetrics to json | 20260505 12:08:12.280 | INFO | Serialization done, swapping tmp file now... | 20260505 12:08:12.311 | INFO | RTMetrics done | 20260505 12:08:12.343 | INFO | Finished a round of Baz IO: Wrote 2.351 GiB in 3.369 seconds | 20260505 12:08:12.343 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.75% / 0.01% / 76.24% | 20260505 12:08:12.343 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.959253% / 1.19209e-05% / 7.54992e-05% | 20260505 12:08:12.343 | INFO | Baz IO overall: | 20260505 12:08:12.343 | INFO | BytesWritten : 1832826237619 | 20260505 12:08:12.343 | INFO | NumEvents : 872516265307 | 20260505 12:08:13.972 | INFO | Analyzing chunk frames = [3088384,3088896) | 20260505 12:08:13.972 | INFO | MemInfo: { | 20260505 12:08:13.972 | INFO | "gpu" : " 17 GB", | 20260505 12:08:13.972 | INFO | "huge" : "", | 20260505 12:08:13.972 | INFO | "huge_shared" : " 27 GB", | 20260505 12:08:13.972 | INFO | "malloc" : " 14 GB", | 20260505 12:08:13.972 | INFO | "pinned" : " 29 GB" | 20260505 12:08:13.972 | INFO | } | 20260505 12:08:13.972 | INFO | Perf report for the last 5.13111 seconds: | 20260505 12:08:13.972 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:08:13.972 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:13.972 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:13.972 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:13.972 | INFO | ANALYSIS : 64.8%, 3.0, 100.2ms, 0 | 20260505 12:08:13.972 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:13.972 | INFO | RT_METRICS : 47.3%, 1.0, 24.8ms, 0 | 20260505 12:08:13.972 | INFO | PRE_HQ : 61.2%, 1.8, 58.4ms, 0 | 20260505 12:08:13.972 | INFO | BAZWRITER : 65.0%, 4.1, 135.3ms, 0 | 20260505 12:08:13.972 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 12:08:13.972 | INFO | Uploaded 36GiB to GPU in 2.2026s (16.3443GiB/s) | 20260505 12:08:13.972 | INFO | Downloaded 38GiB to GPU in 2.2977s (16.5382GiB/s) | 20260505 12:08:19.109 | INFO | Analyzing chunk frames = [3088896,3089408) | 20260505 12:08:19.109 | INFO | MemInfo: { | 20260505 12:08:19.109 | INFO | "gpu" : " 17 GB", | 20260505 12:08:19.109 | INFO | "huge" : "", | 20260505 12:08:19.109 | INFO | "huge_shared" : " 27 GB", | 20260505 12:08:19.109 | INFO | "malloc" : " 14 GB", | 20260505 12:08:19.109 | INFO | "pinned" : " 29 GB" | 20260505 12:08:19.109 | INFO | } | 20260505 12:08:19.109 | INFO | Perf report for the last 5.13769 seconds: | 20260505 12:08:19.109 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:08:19.109 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:19.109 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:19.109 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:19.109 | INFO | ANALYSIS : 64.1%, 3.0, 100.0ms, 0 | 20260505 12:08:19.109 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:19.109 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:19.109 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:08:19.110 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:19.110 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:19.110 | INFO | Uploaded 36GiB to GPU in 2.23327s (16.1198GiB/s) | 20260505 12:08:19.110 | INFO | Downloaded 34GiB to GPU in 2.08639s (16.2961GiB/s) | 20260505 12:08:24.253 | INFO | Analyzing chunk frames = [3089408,3089920) | 20260505 12:08:24.253 | INFO | MemInfo: { | 20260505 12:08:24.253 | INFO | "gpu" : " 17 GB", | 20260505 12:08:24.253 | INFO | "huge" : "", | 20260505 12:08:24.253 | INFO | "huge_shared" : " 27 GB", | 20260505 12:08:24.253 | INFO | "malloc" : " 14 GB", | 20260505 12:08:24.253 | INFO | "pinned" : " 29 GB" | 20260505 12:08:24.253 | INFO | } | 20260505 12:08:24.253 | INFO | Perf report for the last 5.14377 seconds: | 20260505 12:08:24.253 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:08:24.253 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:24.253 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:24.253 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:24.253 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 12:08:24.253 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:24.253 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:24.253 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 12:08:24.253 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:24.253 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:24.253 | INFO | Uploaded 36GiB to GPU in 2.23435s (16.1121GiB/s) | 20260505 12:08:24.253 | INFO | Downloaded 34GiB to GPU in 2.12923s (15.9682GiB/s) | 20260505 12:08:29.399 | INFO | Analyzing chunk frames = [3089920,3090432) | 20260505 12:08:29.399 | INFO | MemInfo: { | 20260505 12:08:29.399 | INFO | "gpu" : " 17 GB", | 20260505 12:08:29.399 | INFO | "huge" : "", | 20260505 12:08:29.399 | INFO | "huge_shared" : " 27 GB", | 20260505 12:08:29.399 | INFO | "malloc" : " 14 GB", | 20260505 12:08:29.399 | INFO | "pinned" : " 29 GB" | 20260505 12:08:29.399 | INFO | } | 20260505 12:08:29.400 | INFO | Perf report for the last 5.14631 seconds: | 20260505 12:08:29.400 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:08:29.400 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:29.400 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:29.400 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:29.400 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260505 12:08:29.400 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:29.400 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:29.400 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 12:08:29.400 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:29.400 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:29.400 | INFO | Uploaded 36GiB to GPU in 2.1752s (16.5502GiB/s) | 20260505 12:08:29.400 | INFO | Downloaded 34GiB to GPU in 2.10279s (16.169GiB/s) | 20260505 12:08:34.534 | INFO | Analyzing chunk frames = [3090432,3090944) | 20260505 12:08:34.534 | INFO | MemInfo: { | 20260505 12:08:34.534 | INFO | "gpu" : " 17 GB", | 20260505 12:08:34.534 | INFO | "huge" : "", | 20260505 12:08:34.534 | INFO | "huge_shared" : " 27 GB", | 20260505 12:08:34.534 | INFO | "malloc" : " 14 GB", | 20260505 12:08:34.534 | INFO | "pinned" : " 29 GB" | 20260505 12:08:34.534 | INFO | } | 20260505 12:08:34.534 | INFO | Perf report for the last 5.13479 seconds: | 20260505 12:08:34.534 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:08:34.534 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:34.534 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:34.534 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:34.534 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 12:08:34.534 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:34.534 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:34.534 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:08:34.534 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:34.534 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:34.535 | INFO | Uploaded 36GiB to GPU in 2.16245s (16.6478GiB/s) | 20260505 12:08:34.535 | INFO | Downloaded 34GiB to GPU in 2.12511s (15.9991GiB/s) | 20260505 12:08:39.672 | INFO | Analyzing chunk frames = [3090944,3091456) | 20260505 12:08:39.672 | INFO | MemInfo: { | 20260505 12:08:39.672 | INFO | "gpu" : " 17 GB", | 20260505 12:08:39.672 | INFO | "huge" : "", | 20260505 12:08:39.672 | INFO | "huge_shared" : " 27 GB", | 20260505 12:08:39.672 | INFO | "malloc" : " 14 GB", | 20260505 12:08:39.672 | INFO | "pinned" : " 29 GB" | 20260505 12:08:39.672 | INFO | } | 20260505 12:08:39.672 | INFO | Perf report for the last 5.13786 seconds: | 20260505 12:08:39.672 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:08:39.672 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:39.672 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:39.672 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:39.672 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 12:08:39.672 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:39.672 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:39.672 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 12:08:39.672 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:39.672 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:39.672 | INFO | Uploaded 36GiB to GPU in 2.16983s (16.5912GiB/s) | 20260505 12:08:39.673 | INFO | Downloaded 34GiB to GPU in 2.1139s (16.084GiB/s) | 20260505 12:08:44.812 | INFO | Analyzing chunk frames = [3091456,3091968) | 20260505 12:08:44.812 | INFO | MemInfo: { | 20260505 12:08:44.812 | INFO | "gpu" : " 17 GB", | 20260505 12:08:44.812 | INFO | "huge" : "", | 20260505 12:08:44.812 | INFO | "huge_shared" : " 27 GB", | 20260505 12:08:44.812 | INFO | "malloc" : " 14 GB", | 20260505 12:08:44.812 | INFO | "pinned" : " 29 GB" | 20260505 12:08:44.812 | INFO | } | 20260505 12:08:44.812 | INFO | Perf report for the last 5.13986 seconds: | 20260505 12:08:44.812 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:08:44.812 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:44.812 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:44.812 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:44.812 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 12:08:44.812 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:44.812 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:44.812 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 12:08:44.812 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:44.812 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:44.812 | INFO | Uploaded 36GiB to GPU in 2.17982s (16.5151GiB/s) | 20260505 12:08:44.812 | INFO | Downloaded 34GiB to GPU in 2.15576s (15.7717GiB/s) | 20260505 12:08:49.948 | INFO | Analyzing chunk frames = [3091968,3092480) | 20260505 12:08:49.948 | INFO | MemInfo: { | 20260505 12:08:49.948 | INFO | "gpu" : " 17 GB", | 20260505 12:08:49.948 | INFO | "huge" : "", | 20260505 12:08:49.948 | INFO | "huge_shared" : " 27 GB", | 20260505 12:08:49.948 | INFO | "malloc" : " 14 GB", | 20260505 12:08:49.948 | INFO | "pinned" : " 29 GB" | 20260505 12:08:49.948 | INFO | } | 20260505 12:08:49.948 | INFO | Perf report for the last 5.13643 seconds: | 20260505 12:08:49.948 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:08:49.948 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:49.948 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:49.948 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:49.948 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 12:08:49.948 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:49.949 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:49.949 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 12:08:49.949 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:49.949 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:49.949 | INFO | Uploaded 36GiB to GPU in 2.21947s (16.2201GiB/s) | 20260505 12:08:49.949 | INFO | Downloaded 34GiB to GPU in 2.08327s (16.3205GiB/s) | 20260505 12:08:50.078 | INFO | Starting a round of Baz IO | 20260505 12:08:53.321 | INFO | Populating RTMetrics report | 20260505 12:08:53.321 | INFO | RTMetrics report populated | 20260505 12:08:53.321 | INFO | Opening tmp RTMetrics json file | 20260505 12:08:53.322 | INFO | Serializing RTMetrics to json | 20260505 12:08:53.323 | INFO | Serialization done, swapping tmp file now... | 20260505 12:08:53.324 | INFO | RTMetrics done | 20260505 12:08:53.449 | INFO | Finished a round of Baz IO: Wrote 2.348 GiB in 3.37 seconds | 20260505 12:08:53.449 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.74% / 0.01% / 76.25% | 20260505 12:08:53.449 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.954032% / 1.58946e-05% / 9.93411e-05% | 20260505 12:08:53.449 | INFO | Baz IO overall: | 20260505 12:08:53.449 | INFO | BytesWritten : 1835347139524 | 20260505 12:08:53.449 | INFO | NumEvents : 873712997489 | 20260505 12:08:55.093 | INFO | Analyzing chunk frames = [3092480,3092992) | 20260505 12:08:55.093 | INFO | MemInfo: { | 20260505 12:08:55.093 | INFO | "gpu" : " 17 GB", | 20260505 12:08:55.093 | INFO | "huge" : "", | 20260505 12:08:55.094 | INFO | "huge_shared" : " 27 GB", | 20260505 12:08:55.094 | INFO | "malloc" : " 14 GB", | 20260505 12:08:55.094 | INFO | "pinned" : " 29 GB" | 20260505 12:08:55.094 | INFO | } | 20260505 12:08:55.094 | INFO | Perf report for the last 5.14509 seconds: | 20260505 12:08:55.094 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:08:55.094 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:55.094 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:55.094 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:55.094 | INFO | ANALYSIS : 64.4%, 2.9, 99.6ms, 0 | 20260505 12:08:55.094 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:08:55.094 | INFO | RT_METRICS : 45.3%, 1.0, 23.8ms, 0 | 20260505 12:08:55.094 | INFO | PRE_HQ : 61.0%, 1.8, 58.6ms, 0 | 20260505 12:08:55.094 | INFO | BAZWRITER : 64.8%, 4.1, 136.1ms, 0 | 20260505 12:08:55.094 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 12:08:55.094 | INFO | Uploaded 36GiB to GPU in 2.19445s (16.405GiB/s) | 20260505 12:08:55.094 | INFO | Downloaded 38GiB to GPU in 2.28523s (16.6285GiB/s) | 20260505 12:09:00.234 | INFO | Analyzing chunk frames = [3092992,3093504) | 20260505 12:09:00.234 | INFO | MemInfo: { | 20260505 12:09:00.234 | INFO | "gpu" : " 17 GB", | 20260505 12:09:00.234 | INFO | "huge" : "", | 20260505 12:09:00.234 | INFO | "huge_shared" : " 27 GB", | 20260505 12:09:00.234 | INFO | "malloc" : " 14 GB", | 20260505 12:09:00.234 | INFO | "pinned" : " 29 GB" | 20260505 12:09:00.235 | INFO | } | 20260505 12:09:00.235 | INFO | Perf report for the last 5.14097 seconds: | 20260505 12:09:00.235 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:09:00.235 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:00.235 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:00.235 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:00.235 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 12:09:00.235 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:00.235 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:00.235 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 12:09:00.235 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:00.235 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:00.235 | INFO | Uploaded 36GiB to GPU in 2.22975s (16.1453GiB/s) | 20260505 12:09:00.235 | INFO | Downloaded 34GiB to GPU in 2.10686s (16.1378GiB/s) | 20260505 12:09:05.364 | INFO | Analyzing chunk frames = [3093504,3094016) | 20260505 12:09:05.364 | INFO | MemInfo: { | 20260505 12:09:05.364 | INFO | "gpu" : " 17 GB", | 20260505 12:09:05.364 | INFO | "huge" : "", | 20260505 12:09:05.364 | INFO | "huge_shared" : " 27 GB", | 20260505 12:09:05.364 | INFO | "malloc" : " 14 GB", | 20260505 12:09:05.364 | INFO | "pinned" : " 29 GB" | 20260505 12:09:05.364 | INFO | } | 20260505 12:09:05.364 | INFO | Perf report for the last 5.12937 seconds: | 20260505 12:09:05.364 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:09:05.364 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:05.364 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:05.364 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:05.364 | INFO | ANALYSIS : 63.3%, 3.0, 98.3ms, 0 | 20260505 12:09:05.364 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:05.364 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:05.364 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 12:09:05.364 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:05.364 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:05.364 | INFO | Uploaded 36GiB to GPU in 2.20845s (16.301GiB/s) | 20260505 12:09:05.364 | INFO | Downloaded 34GiB to GPU in 2.10936s (16.1187GiB/s) | 20260505 12:09:10.510 | INFO | Analyzing chunk frames = [3094016,3094528) | 20260505 12:09:10.510 | INFO | MemInfo: { | 20260505 12:09:10.510 | INFO | "gpu" : " 17 GB", | 20260505 12:09:10.510 | INFO | "huge" : "", | 20260505 12:09:10.510 | INFO | "huge_shared" : " 27 GB", | 20260505 12:09:10.510 | INFO | "malloc" : " 14 GB", | 20260505 12:09:10.510 | INFO | "pinned" : " 29 GB" | 20260505 12:09:10.510 | INFO | } | 20260505 12:09:10.510 | INFO | Perf report for the last 5.14602 seconds: | 20260505 12:09:10.510 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:09:10.510 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:10.510 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:10.510 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:10.510 | INFO | ANALYSIS : 63.1%, 3.0, 98.2ms, 0 | 20260505 12:09:10.510 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:10.510 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:10.510 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 12:09:10.510 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:10.510 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:10.510 | INFO | Uploaded 36GiB to GPU in 2.20818s (16.303GiB/s) | 20260505 12:09:10.510 | INFO | Downloaded 34GiB to GPU in 2.13202s (15.9474GiB/s) | 20260505 12:09:15.649 | INFO | Analyzing chunk frames = [3094528,3095040) | 20260505 12:09:15.649 | INFO | MemInfo: { | 20260505 12:09:15.649 | INFO | "gpu" : " 17 GB", | 20260505 12:09:15.649 | INFO | "huge" : "", | 20260505 12:09:15.649 | INFO | "huge_shared" : " 27 GB", | 20260505 12:09:15.649 | INFO | "malloc" : " 14 GB", | 20260505 12:09:15.649 | INFO | "pinned" : " 29 GB" | 20260505 12:09:15.649 | INFO | } | 20260505 12:09:15.649 | INFO | Perf report for the last 5.13916 seconds: | 20260505 12:09:15.649 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:09:15.649 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:15.649 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:15.649 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:15.649 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 12:09:15.649 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:15.649 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:15.649 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 12:09:15.649 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:15.649 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:15.649 | INFO | Uploaded 36GiB to GPU in 2.15504s (16.705GiB/s) | 20260505 12:09:15.649 | INFO | Downloaded 34GiB to GPU in 2.12526s (15.9981GiB/s) | 20260505 12:09:20.787 | INFO | Analyzing chunk frames = [3095040,3095552) | 20260505 12:09:20.787 | INFO | MemInfo: { | 20260505 12:09:20.787 | INFO | "gpu" : " 17 GB", | 20260505 12:09:20.787 | INFO | "huge" : "", | 20260505 12:09:20.787 | INFO | "huge_shared" : " 27 GB", | 20260505 12:09:20.787 | INFO | "malloc" : " 14 GB", | 20260505 12:09:20.787 | INFO | "pinned" : " 29 GB" | 20260505 12:09:20.787 | INFO | } | 20260505 12:09:20.787 | INFO | Perf report for the last 5.13768 seconds: | 20260505 12:09:20.787 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:09:20.787 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:20.787 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:20.787 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:20.787 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 12:09:20.787 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:20.787 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:20.787 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:09:20.787 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:20.787 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:20.787 | INFO | Uploaded 36GiB to GPU in 2.15438s (16.7101GiB/s) | 20260505 12:09:20.787 | INFO | Downloaded 34GiB to GPU in 2.10761s (16.132GiB/s) | 20260505 12:09:25.922 | INFO | Analyzing chunk frames = [3095552,3096064) | 20260505 12:09:25.922 | INFO | MemInfo: { | 20260505 12:09:25.922 | INFO | "gpu" : " 17 GB", | 20260505 12:09:25.922 | INFO | "huge" : "", | 20260505 12:09:25.922 | INFO | "huge_shared" : " 27 GB", | 20260505 12:09:25.922 | INFO | "malloc" : " 14 GB", | 20260505 12:09:25.922 | INFO | "pinned" : " 29 GB" | 20260505 12:09:25.922 | INFO | } | 20260505 12:09:25.923 | INFO | Perf report for the last 5.13564 seconds: | 20260505 12:09:25.923 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:09:25.923 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:25.923 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:25.923 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:25.923 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 12:09:25.923 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:25.923 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:25.923 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 12:09:25.923 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:25.923 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:25.923 | INFO | Uploaded 36GiB to GPU in 2.1801s (16.513GiB/s) | 20260505 12:09:25.923 | INFO | Downloaded 34GiB to GPU in 2.1328s (15.9415GiB/s) | 20260505 12:09:31.062 | INFO | Analyzing chunk frames = [3096064,3096576) | 20260505 12:09:31.062 | INFO | MemInfo: { | 20260505 12:09:31.062 | INFO | "gpu" : " 17 GB", | 20260505 12:09:31.062 | INFO | "huge" : "", | 20260505 12:09:31.062 | INFO | "huge_shared" : " 27 GB", | 20260505 12:09:31.062 | INFO | "malloc" : " 14 GB", | 20260505 12:09:31.062 | INFO | "pinned" : " 29 GB" | 20260505 12:09:31.062 | INFO | } | 20260505 12:09:31.063 | INFO | Perf report for the last 5.14003 seconds: | 20260505 12:09:31.063 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:09:31.063 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:31.063 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:31.063 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:31.063 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 12:09:31.063 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:31.063 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:31.063 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 12:09:31.063 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:31.063 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:31.063 | INFO | Uploaded 36GiB to GPU in 2.21778s (16.2325GiB/s) | 20260505 12:09:31.063 | INFO | Downloaded 34GiB to GPU in 2.08537s (16.3041GiB/s) | 20260505 12:09:31.063 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 12:09:31.063 | INFO | Total mem usage: 15.3 GiB | 20260505 12:09:31.063 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 12:09:31.063 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -6.84%) | 20260505 12:09:31.063 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 12:09:31.063 | INFO | High Water Breakdown: | 20260505 12:09:31.063 | INFO | 61.1% : 8.76 GiB ( 92.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 12:09:31.063 | INFO | 38.8% : 5.56 GiB ( 83.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 12:09:31.063 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 12:09:31.063 | INFO | ------------------------------------------ | 20260505 12:09:31.063 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:09:31.063 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 12:09:31.063 | INFO | Total mem usage: 26.5 GiB | 20260505 12:09:31.063 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 12:09:31.063 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 12:09:31.063 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 12:09:31.063 | INFO | High Water Breakdown: | 20260505 12:09:31.063 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:09:31.063 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:09:31.063 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:09:31.063 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:09:31.063 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:09:31.063 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:09:31.063 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 12:09:31.063 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 12:09:31.063 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:09:31.063 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 12:09:31.063 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 12:09:31.063 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 12:09:31.063 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 12:09:31.063 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 12:09:31.063 | INFO | ------------------------------------------ | 20260505 12:09:31.063 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:09:31.063 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 12:09:31.063 | INFO | Total mem usage: 25.6 GiB | 20260505 12:09:31.063 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 12:09:31.063 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 12:09:31.063 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 12:09:31.063 | INFO | High Water Breakdown: | 20260505 12:09:31.063 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 12:09:31.063 | INFO | ------------------------------------------ | 20260505 12:09:31.063 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:09:31.063 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 12:09:31.063 | INFO | Total mem usage: 15.7 GiB | 20260505 12:09:31.063 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 12:09:31.063 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 12:09:31.063 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 12:09:31.063 | INFO | High Water Breakdown: | 20260505 12:09:31.063 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 12:09:31.063 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 12:09:31.063 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 12:09:31.063 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 12:09:31.063 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 12:09:31.063 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 12:09:31.063 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 12:09:31.063 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 12:09:31.063 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 12:09:31.063 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:09:31.063 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 12:09:31.063 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:09:31.063 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 12:09:31.063 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 12:09:31.063 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 12:09:31.063 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 12:09:31.063 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:09:31.063 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 12:09:31.063 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:09:31.063 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:09:31.063 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:09:31.063 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 12:09:31.064 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 12:09:31.064 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:09:31.064 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 12:09:31.064 | INFO | ------------------------------------------ | 20260505 12:09:31.064 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:09:31.064 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 12:09:31.064 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:09:31.064 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:09:31.064 | INFO | PerfReport: Full Execution -- 3.20e+01 / 7.68e+01, 1.37e+01 / 1.50e+00 | 20260505 12:09:31.064 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 1.00e-01, 1.02e-02 / 4.44e-03 | 20260505 12:09:31.064 | INFO | PerfReport: 11.75%, Baseline -- 3.76e+00 / 4.47e+00, 5.91e-01 / 2.29e-01 | 20260505 12:09:31.064 | INFO | PerfReport: 3.30%, Histogram -- 1.06e+00 / 1.48e+00, 2.00e-01 / 1.11e-01 | 20260505 12:09:31.064 | INFO | PerfReport: 15.82%, DME -- 4.05e+01 / 4.47e+01, 4.08e+00 / 1.97e+00 | 20260505 12:09:31.064 | INFO | PerfReport: 56.47%, FrameLabeling -- 1.81e+01 / 2.12e+01, 1.57e+00 / 2.08e+00 | 20260505 12:09:31.064 | INFO | PerfReport: 7.42%, PulseAccumulating -- 2.38e+00 / 2.84e+00, 4.25e-01 / 2.02e-01 | 20260505 12:09:31.064 | INFO | PerfReport: 3.66%, Metrics -- 1.17e+00 / 1.48e+00, 3.30e-01 / 2.06e-01 | 20260505 12:09:31.064 | INFO | PerfReport: 1.28%, PulseSerializing -- 4.11e-01 / 4.54e-01, 7.34e-02 / 3.90e-02 | 20260505 12:09:31.064 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 12:09:31.064 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:09:31.064 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:09:31.064 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.48e+01 / 0.00e+00 | 20260505 12:09:31.064 | INFO | PerfReport: 22.62%, Upload -- 2.29e+01 / 0.00e+00, 8.17e+00 / 0.00e+00 | 20260505 12:09:31.064 | INFO | PerfReport: 22.01%, Download -- 2.23e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260505 12:09:31.064 | INFO | PerfReport: 31.68%, Compute -- 3.20e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 12:09:31.064 | INFO | PerfReport: 2.13%, LockUpload -- 2.15e+00 / 0.00e+00, 7.71e+00 / 0.00e+00 | 20260505 12:09:31.064 | INFO | PerfReport: 1.30%, LockDownload -- 1.32e+00 / 0.00e+00, 2.82e+00 / 0.00e+00 | 20260505 12:09:31.064 | INFO | PerfReport: 20.24%, LockCompute -- 2.05e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260505 12:09:31.064 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.78e-02 / 0.00e+00 | 20260505 12:09:31.194 | INFO | Starting a round of Baz IO | 20260505 12:09:34.444 | INFO | Populating RTMetrics report | 20260505 12:09:34.445 | INFO | RTMetrics report populated | 20260505 12:09:34.445 | INFO | Opening tmp RTMetrics json file | 20260505 12:09:34.471 | INFO | Serializing RTMetrics to json | 20260505 12:09:34.473 | INFO | Serialization done, swapping tmp file now... | 20260505 12:09:34.474 | INFO | RTMetrics done | 20260505 12:09:34.582 | INFO | Finished a round of Baz IO: Wrote 2.346 GiB in 3.387 seconds | 20260505 12:09:34.582 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.74% / 0.01% / 76.26% | 20260505 12:09:34.582 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.948521% / 1.58946e-05% / 6.35783e-05% | 20260505 12:09:34.582 | INFO | Baz IO overall: | 20260505 12:09:34.582 | INFO | BytesWritten : 1837866147299 | 20260505 12:09:34.582 | INFO | NumEvents : 874908575949 | 20260505 12:09:36.207 | INFO | Analyzing chunk frames = [3096576,3097088) | 20260505 12:09:36.207 | INFO | MemInfo: { | 20260505 12:09:36.207 | INFO | "gpu" : " 17 GB", | 20260505 12:09:36.207 | INFO | "huge" : "", | 20260505 12:09:36.207 | INFO | "huge_shared" : " 27 GB", | 20260505 12:09:36.207 | INFO | "malloc" : " 14 GB", | 20260505 12:09:36.207 | INFO | "pinned" : " 29 GB" | 20260505 12:09:36.207 | INFO | } | 20260505 12:09:36.207 | INFO | Perf report for the last 5.14408 seconds: | 20260505 12:09:36.207 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:09:36.207 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:36.207 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:36.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:36.207 | INFO | ANALYSIS : 64.6%, 2.9, 100.0ms, 0 | 20260505 12:09:36.207 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:36.207 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260505 12:09:36.207 | INFO | PRE_HQ : 61.0%, 1.9, 58.6ms, 0 | 20260505 12:09:36.207 | INFO | BAZWRITER : 65.0%, 4.1, 136.2ms, 0 | 20260505 12:09:36.207 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 12:09:36.207 | INFO | Uploaded 36GiB to GPU in 2.20085s (16.3573GiB/s) | 20260505 12:09:36.207 | INFO | Downloaded 38GiB to GPU in 2.29093s (16.5871GiB/s) | 20260505 12:09:41.351 | INFO | Analyzing chunk frames = [3097088,3097600) | 20260505 12:09:41.351 | INFO | MemInfo: { | 20260505 12:09:41.351 | INFO | "gpu" : " 17 GB", | 20260505 12:09:41.351 | INFO | "huge" : "", | 20260505 12:09:41.351 | INFO | "huge_shared" : " 27 GB", | 20260505 12:09:41.351 | INFO | "malloc" : " 14 GB", | 20260505 12:09:41.351 | INFO | "pinned" : " 29 GB" | 20260505 12:09:41.351 | INFO | } | 20260505 12:09:41.351 | INFO | Perf report for the last 5.14407 seconds: | 20260505 12:09:41.351 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:09:41.351 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:41.351 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:41.351 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:41.351 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 12:09:41.351 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:41.351 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:41.351 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 12:09:41.351 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:41.351 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:41.351 | INFO | Uploaded 36GiB to GPU in 2.17105s (16.5818GiB/s) | 20260505 12:09:41.351 | INFO | Downloaded 34GiB to GPU in 2.03034s (16.7459GiB/s) | 20260505 12:09:46.477 | INFO | Analyzing chunk frames = [3097600,3098112) | 20260505 12:09:46.477 | INFO | MemInfo: { | 20260505 12:09:46.477 | INFO | "gpu" : " 17 GB", | 20260505 12:09:46.477 | INFO | "huge" : "", | 20260505 12:09:46.477 | INFO | "huge_shared" : " 27 GB", | 20260505 12:09:46.477 | INFO | "malloc" : " 14 GB", | 20260505 12:09:46.477 | INFO | "pinned" : " 29 GB" | 20260505 12:09:46.477 | INFO | } | 20260505 12:09:46.477 | INFO | Perf report for the last 5.12652 seconds: | 20260505 12:09:46.477 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:09:46.477 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:46.477 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:46.477 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:46.477 | INFO | ANALYSIS : 63.5%, 3.0, 98.6ms, 0 | 20260505 12:09:46.477 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:46.477 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:46.477 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 12:09:46.477 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:46.477 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:46.478 | INFO | Uploaded 36GiB to GPU in 2.20972s (16.2917GiB/s) | 20260505 12:09:46.478 | INFO | Downloaded 34GiB to GPU in 2.11449s (16.0795GiB/s) | 20260505 12:09:51.619 | INFO | Analyzing chunk frames = [3098112,3098624) | 20260505 12:09:51.619 | INFO | MemInfo: { | 20260505 12:09:51.619 | INFO | "gpu" : " 17 GB", | 20260505 12:09:51.619 | INFO | "huge" : "", | 20260505 12:09:51.619 | INFO | "huge_shared" : " 27 GB", | 20260505 12:09:51.619 | INFO | "malloc" : " 14 GB", | 20260505 12:09:51.619 | INFO | "pinned" : " 29 GB" | 20260505 12:09:51.619 | INFO | } | 20260505 12:09:51.619 | INFO | Perf report for the last 5.14136 seconds: | 20260505 12:09:51.619 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:09:51.619 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:51.619 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:51.619 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:51.619 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260505 12:09:51.619 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:51.619 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:51.619 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 12:09:51.619 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:51.619 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:51.619 | INFO | Uploaded 36GiB to GPU in 2.22151s (16.2052GiB/s) | 20260505 12:09:51.619 | INFO | Downloaded 34GiB to GPU in 2.13343s (15.9368GiB/s) | 20260505 12:09:56.763 | INFO | Analyzing chunk frames = [3098624,3099136) | 20260505 12:09:56.763 | INFO | MemInfo: { | 20260505 12:09:56.763 | INFO | "gpu" : " 17 GB", | 20260505 12:09:56.763 | INFO | "huge" : "", | 20260505 12:09:56.763 | INFO | "huge_shared" : " 27 GB", | 20260505 12:09:56.763 | INFO | "malloc" : " 14 GB", | 20260505 12:09:56.763 | INFO | "pinned" : " 29 GB" | 20260505 12:09:56.763 | INFO | } | 20260505 12:09:56.763 | INFO | Perf report for the last 5.14412 seconds: | 20260505 12:09:56.763 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:09:56.763 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:56.763 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:56.763 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:56.763 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 12:09:56.763 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:56.763 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:56.763 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:09:56.763 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:56.763 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:09:56.763 | INFO | Uploaded 36GiB to GPU in 2.16163s (16.6541GiB/s) | 20260505 12:09:56.763 | INFO | Downloaded 34GiB to GPU in 2.12516s (15.9988GiB/s) | 20260505 12:10:01.901 | INFO | Analyzing chunk frames = [3099136,3099648) | 20260505 12:10:01.901 | INFO | MemInfo: { | 20260505 12:10:01.901 | INFO | "gpu" : " 17 GB", | 20260505 12:10:01.902 | INFO | "huge" : "", | 20260505 12:10:01.902 | INFO | "huge_shared" : " 27 GB", | 20260505 12:10:01.902 | INFO | "malloc" : " 14 GB", | 20260505 12:10:01.902 | INFO | "pinned" : " 29 GB" | 20260505 12:10:01.902 | INFO | } | 20260505 12:10:01.902 | INFO | Perf report for the last 5.13887 seconds: | 20260505 12:10:01.902 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:10:01.902 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:01.902 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:01.902 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:01.902 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 12:10:01.902 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:01.902 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:01.902 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:10:01.902 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:01.902 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:01.902 | INFO | Uploaded 36GiB to GPU in 2.16199s (16.6513GiB/s) | 20260505 12:10:01.902 | INFO | Downloaded 34GiB to GPU in 2.10964s (16.1165GiB/s) | 20260505 12:10:07.042 | INFO | Analyzing chunk frames = [3099648,3100160) | 20260505 12:10:07.042 | INFO | MemInfo: { | 20260505 12:10:07.042 | INFO | "gpu" : " 17 GB", | 20260505 12:10:07.042 | INFO | "huge" : "", | 20260505 12:10:07.042 | INFO | "huge_shared" : " 27 GB", | 20260505 12:10:07.042 | INFO | "malloc" : " 14 GB", | 20260505 12:10:07.042 | INFO | "pinned" : " 29 GB" | 20260505 12:10:07.042 | INFO | } | 20260505 12:10:07.042 | INFO | Perf report for the last 5.14013 seconds: | 20260505 12:10:07.042 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:10:07.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:07.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:07.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:07.042 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 12:10:07.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:07.042 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:07.042 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 12:10:07.042 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:07.042 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:07.042 | INFO | Uploaded 36GiB to GPU in 2.15775s (16.6841GiB/s) | 20260505 12:10:07.042 | INFO | Downloaded 34GiB to GPU in 2.14246s (15.8696GiB/s) | 20260505 12:10:12.179 | INFO | Analyzing chunk frames = [3100160,3100672) | 20260505 12:10:12.179 | INFO | MemInfo: { | 20260505 12:10:12.179 | INFO | "gpu" : " 17 GB", | 20260505 12:10:12.179 | INFO | "huge" : "", | 20260505 12:10:12.179 | INFO | "huge_shared" : " 27 GB", | 20260505 12:10:12.179 | INFO | "malloc" : " 14 GB", | 20260505 12:10:12.179 | INFO | "pinned" : " 29 GB" | 20260505 12:10:12.179 | INFO | } | 20260505 12:10:12.179 | INFO | Perf report for the last 5.13696 seconds: | 20260505 12:10:12.179 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:10:12.179 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:12.179 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:12.179 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:12.179 | INFO | ANALYSIS : 63.8%, 3.0, 99.5ms, 0 | 20260505 12:10:12.179 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:12.179 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:12.179 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 12:10:12.179 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:12.179 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:12.179 | INFO | Uploaded 36GiB to GPU in 2.21556s (16.2487GiB/s) | 20260505 12:10:12.179 | INFO | Downloaded 34GiB to GPU in 2.09442s (16.2336GiB/s) | 20260505 12:10:12.313 | INFO | Starting a round of Baz IO | 20260505 12:10:15.556 | INFO | Populating RTMetrics report | 20260505 12:10:15.556 | INFO | RTMetrics report populated | 20260505 12:10:15.556 | INFO | Opening tmp RTMetrics json file | 20260505 12:10:15.557 | INFO | Serializing RTMetrics to json | 20260505 12:10:15.568 | INFO | Serialization done, swapping tmp file now... | 20260505 12:10:15.583 | INFO | RTMetrics done | 20260505 12:10:15.681 | INFO | Finished a round of Baz IO: Wrote 2.348 GiB in 3.367 seconds | 20260505 12:10:15.681 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.73% / 0.01% / 76.26% | 20260505 12:10:15.681 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.948783% / 3.97364e-06% / 8.34465e-05% | 20260505 12:10:15.681 | INFO | Baz IO overall: | 20260505 12:10:15.681 | INFO | BytesWritten : 1840386801846 | 20260505 12:10:15.681 | INFO | NumEvents : 876105323474 | 20260505 12:10:17.320 | INFO | Analyzing chunk frames = [3100672,3101184) | 20260505 12:10:17.320 | INFO | MemInfo: { | 20260505 12:10:17.320 | INFO | "gpu" : " 17 GB", | 20260505 12:10:17.320 | INFO | "huge" : "", | 20260505 12:10:17.320 | INFO | "huge_shared" : " 27 GB", | 20260505 12:10:17.320 | INFO | "malloc" : " 14 GB", | 20260505 12:10:17.320 | INFO | "pinned" : " 29 GB" | 20260505 12:10:17.320 | INFO | } | 20260505 12:10:17.320 | INFO | Perf report for the last 5.14103 seconds: | 20260505 12:10:17.320 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:10:17.320 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:17.320 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:17.320 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:17.320 | INFO | ANALYSIS : 64.6%, 2.9, 99.4ms, 0 | 20260505 12:10:17.320 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:17.320 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 12:10:17.320 | INFO | PRE_HQ : 60.9%, 1.9, 58.7ms, 0 | 20260505 12:10:17.320 | INFO | BAZWRITER : 64.8%, 4.1, 136.1ms, 0 | 20260505 12:10:17.320 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 12:10:17.320 | INFO | Uploaded 36GiB to GPU in 2.18655s (16.4643GiB/s) | 20260505 12:10:17.320 | INFO | Downloaded 38GiB to GPU in 2.29562s (16.5532GiB/s) | 20260505 12:10:22.458 | INFO | Analyzing chunk frames = [3101184,3101696) | 20260505 12:10:22.458 | INFO | MemInfo: { | 20260505 12:10:22.458 | INFO | "gpu" : " 17 GB", | 20260505 12:10:22.458 | INFO | "huge" : "", | 20260505 12:10:22.458 | INFO | "huge_shared" : " 27 GB", | 20260505 12:10:22.458 | INFO | "malloc" : " 14 GB", | 20260505 12:10:22.458 | INFO | "pinned" : " 29 GB" | 20260505 12:10:22.458 | INFO | } | 20260505 12:10:22.458 | INFO | Perf report for the last 5.13861 seconds: | 20260505 12:10:22.458 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:10:22.458 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:22.458 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:22.458 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:22.458 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 12:10:22.458 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:22.458 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:22.458 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 12:10:22.458 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:22.458 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:22.459 | INFO | Uploaded 36GiB to GPU in 2.17739s (16.5335GiB/s) | 20260505 12:10:22.459 | INFO | Downloaded 34GiB to GPU in 2.05537s (16.542GiB/s) | 20260505 12:10:27.599 | INFO | Analyzing chunk frames = [3101696,3102208) | 20260505 12:10:27.599 | INFO | MemInfo: { | 20260505 12:10:27.599 | INFO | "gpu" : " 17 GB", | 20260505 12:10:27.600 | INFO | "huge" : "", | 20260505 12:10:27.600 | INFO | "huge_shared" : " 27 GB", | 20260505 12:10:27.600 | INFO | "malloc" : " 14 GB", | 20260505 12:10:27.600 | INFO | "pinned" : " 29 GB" | 20260505 12:10:27.600 | INFO | } | 20260505 12:10:27.600 | INFO | Perf report for the last 5.14117 seconds: | 20260505 12:10:27.600 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:10:27.600 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:27.600 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:27.600 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:27.600 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 12:10:27.600 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:27.600 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:27.600 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 12:10:27.600 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:27.600 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:27.600 | INFO | Uploaded 36GiB to GPU in 2.2158s (16.247GiB/s) | 20260505 12:10:27.600 | INFO | Downloaded 34GiB to GPU in 2.12406s (16.0071GiB/s) | 20260505 12:10:32.733 | INFO | Analyzing chunk frames = [3102208,3102720) | 20260505 12:10:32.733 | INFO | MemInfo: { | 20260505 12:10:32.733 | INFO | "gpu" : " 17 GB", | 20260505 12:10:32.733 | INFO | "huge" : "", | 20260505 12:10:32.733 | INFO | "huge_shared" : " 27 GB", | 20260505 12:10:32.733 | INFO | "malloc" : " 14 GB", | 20260505 12:10:32.733 | INFO | "pinned" : " 29 GB" | 20260505 12:10:32.733 | INFO | } | 20260505 12:10:32.733 | INFO | Perf report for the last 5.13305 seconds: | 20260505 12:10:32.733 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:10:32.733 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:32.733 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:32.733 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:32.733 | INFO | ANALYSIS : 63.4%, 3.0, 98.4ms, 0 | 20260505 12:10:32.733 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:32.733 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:32.733 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 12:10:32.733 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:32.733 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:32.733 | INFO | Uploaded 36GiB to GPU in 2.21383s (16.2614GiB/s) | 20260505 12:10:32.733 | INFO | Downloaded 34GiB to GPU in 2.13401s (15.9325GiB/s) | 20260505 12:10:37.873 | INFO | Analyzing chunk frames = [3102720,3103232) | 20260505 12:10:37.873 | INFO | MemInfo: { | 20260505 12:10:37.873 | INFO | "gpu" : " 17 GB", | 20260505 12:10:37.873 | INFO | "huge" : "", | 20260505 12:10:37.873 | INFO | "huge_shared" : " 27 GB", | 20260505 12:10:37.873 | INFO | "malloc" : " 14 GB", | 20260505 12:10:37.873 | INFO | "pinned" : " 29 GB" | 20260505 12:10:37.873 | INFO | } | 20260505 12:10:37.873 | INFO | Perf report for the last 5.14059 seconds: | 20260505 12:10:37.873 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:10:37.873 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:37.873 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:37.873 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:37.873 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 12:10:37.873 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:37.873 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:37.873 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 12:10:37.873 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:37.873 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:37.874 | INFO | Uploaded 36GiB to GPU in 2.15003s (16.744GiB/s) | 20260505 12:10:37.874 | INFO | Downloaded 34GiB to GPU in 2.12528s (15.9979GiB/s) | 20260505 12:10:43.024 | INFO | Analyzing chunk frames = [3103232,3103744) | 20260505 12:10:43.024 | INFO | MemInfo: { | 20260505 12:10:43.024 | INFO | "gpu" : " 17 GB", | 20260505 12:10:43.024 | INFO | "huge" : "", | 20260505 12:10:43.024 | INFO | "huge_shared" : " 27 GB", | 20260505 12:10:43.024 | INFO | "malloc" : " 14 GB", | 20260505 12:10:43.024 | INFO | "pinned" : " 29 GB" | 20260505 12:10:43.024 | INFO | } | 20260505 12:10:43.024 | INFO | Perf report for the last 5.15077 seconds: | 20260505 12:10:43.024 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:10:43.024 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:43.024 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:43.024 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:43.024 | INFO | ANALYSIS : 63.2%, 3.0, 98.9ms, 0 | 20260505 12:10:43.024 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:43.024 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:43.024 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 12:10:43.024 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:43.024 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:43.024 | INFO | Uploaded 36GiB to GPU in 2.17818s (16.5276GiB/s) | 20260505 12:10:43.024 | INFO | Downloaded 34GiB to GPU in 2.11904s (16.045GiB/s) | 20260505 12:10:48.161 | INFO | Analyzing chunk frames = [3103744,3104256) | 20260505 12:10:48.161 | INFO | MemInfo: { | 20260505 12:10:48.161 | INFO | "gpu" : " 17 GB", | 20260505 12:10:48.161 | INFO | "huge" : "", | 20260505 12:10:48.161 | INFO | "huge_shared" : " 27 GB", | 20260505 12:10:48.161 | INFO | "malloc" : " 14 GB", | 20260505 12:10:48.161 | INFO | "pinned" : " 29 GB" | 20260505 12:10:48.161 | INFO | } | 20260505 12:10:48.161 | INFO | Perf report for the last 5.13668 seconds: | 20260505 12:10:48.161 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:10:48.161 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:48.161 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:48.161 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:48.161 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 12:10:48.161 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:48.161 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:48.161 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 12:10:48.161 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:48.161 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:48.161 | INFO | Uploaded 36GiB to GPU in 2.15104s (16.7361GiB/s) | 20260505 12:10:48.161 | INFO | Downloaded 34GiB to GPU in 2.14969s (15.8163GiB/s) | 20260505 12:10:53.296 | INFO | Analyzing chunk frames = [3104256,3104768) | 20260505 12:10:53.296 | INFO | MemInfo: { | 20260505 12:10:53.296 | INFO | "gpu" : " 17 GB", | 20260505 12:10:53.296 | INFO | "huge" : "", | 20260505 12:10:53.296 | INFO | "huge_shared" : " 27 GB", | 20260505 12:10:53.296 | INFO | "malloc" : " 14 GB", | 20260505 12:10:53.296 | INFO | "pinned" : " 29 GB" | 20260505 12:10:53.296 | INFO | } | 20260505 12:10:53.296 | INFO | Perf report for the last 5.13494 seconds: | 20260505 12:10:53.296 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:10:53.296 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:53.296 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:53.296 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:53.296 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 12:10:53.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:53.296 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:53.296 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 12:10:53.296 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:53.296 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:53.296 | INFO | Uploaded 36GiB to GPU in 2.15355s (16.7166GiB/s) | 20260505 12:10:53.296 | INFO | Downloaded 34GiB to GPU in 2.10487s (16.153GiB/s) | 20260505 12:10:53.296 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 12:10:53.296 | INFO | Total mem usage: 15.3 GiB | 20260505 12:10:53.296 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 12:10:53.296 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -6.82%) | 20260505 12:10:53.296 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 12:10:53.296 | INFO | High Water Breakdown: | 20260505 12:10:53.296 | INFO | 61.1% : 8.77 GiB ( 92.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 12:10:53.296 | INFO | 38.8% : 5.56 GiB ( 83.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 12:10:53.296 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 12:10:53.296 | INFO | ------------------------------------------ | 20260505 12:10:53.296 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:10:53.296 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 12:10:53.296 | INFO | Total mem usage: 26.5 GiB | 20260505 12:10:53.296 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 12:10:53.296 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 12:10:53.296 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 12:10:53.296 | INFO | High Water Breakdown: | 20260505 12:10:53.296 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:10:53.296 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:10:53.296 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:10:53.296 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:10:53.296 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:10:53.296 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:10:53.296 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 12:10:53.296 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 12:10:53.296 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:10:53.296 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 12:10:53.296 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 12:10:53.296 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 12:10:53.296 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 12:10:53.296 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 12:10:53.296 | INFO | ------------------------------------------ | 20260505 12:10:53.296 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:10:53.296 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 12:10:53.296 | INFO | Total mem usage: 25.6 GiB | 20260505 12:10:53.296 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 12:10:53.296 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 12:10:53.296 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 12:10:53.296 | INFO | High Water Breakdown: | 20260505 12:10:53.296 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 12:10:53.296 | INFO | ------------------------------------------ | 20260505 12:10:53.296 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:10:53.296 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 12:10:53.296 | INFO | Total mem usage: 15.7 GiB | 20260505 12:10:53.296 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 12:10:53.296 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 12:10:53.296 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 12:10:53.297 | INFO | High Water Breakdown: | 20260505 12:10:53.297 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 12:10:53.297 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 12:10:53.297 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 12:10:53.297 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 12:10:53.297 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 12:10:53.297 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 12:10:53.297 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 12:10:53.297 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 12:10:53.297 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 12:10:53.297 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:10:53.297 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 12:10:53.297 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:10:53.297 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 12:10:53.297 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 12:10:53.297 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 12:10:53.297 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 12:10:53.297 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:10:53.297 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 12:10:53.297 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:10:53.297 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:10:53.297 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:10:53.297 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 12:10:53.297 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 12:10:53.297 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:10:53.297 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 12:10:53.297 | INFO | ------------------------------------------ | 20260505 12:10:53.297 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:10:53.297 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 12:10:53.297 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:10:53.297 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:10:53.297 | INFO | PerfReport: Full Execution -- 3.20e+01 / 7.74e+01, 1.38e+01 / 1.84e+00 | 20260505 12:10:53.297 | INFO | PerfReport: 0.29%, AggregateStats -- 9.33e-02 / 1.01e-01, 1.18e-02 / 3.59e-03 | 20260505 12:10:53.297 | INFO | PerfReport: 11.75%, Baseline -- 3.77e+00 / 4.51e+00, 5.95e-01 / 1.92e-01 | 20260505 12:10:53.297 | INFO | PerfReport: 3.29%, Histogram -- 1.06e+00 / 1.49e+00, 2.00e-01 / 8.60e-02 | 20260505 12:10:53.297 | INFO | PerfReport: 15.97%, DME -- 4.09e+01 / 4.61e+01, 4.79e+00 / 2.28e+00 | 20260505 12:10:53.297 | INFO | PerfReport: 56.34%, FrameLabeling -- 1.81e+01 / 2.02e+01, 1.52e+00 / 1.98e+00 | 20260505 12:10:53.297 | INFO | PerfReport: 7.43%, PulseAccumulating -- 2.38e+00 / 2.94e+00, 4.45e-01 / 1.90e-01 | 20260505 12:10:53.297 | INFO | PerfReport: 3.64%, Metrics -- 1.17e+00 / 1.51e+00, 3.29e-01 / 1.55e-01 | 20260505 12:10:53.297 | INFO | PerfReport: 1.28%, PulseSerializing -- 4.11e-01 / 4.63e-01, 7.38e-02 / 3.02e-02 | 20260505 12:10:53.297 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 12:10:53.297 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:10:53.297 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:10:53.297 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260505 12:10:53.297 | INFO | PerfReport: 22.50%, Upload -- 2.27e+01 / 0.00e+00, 8.01e+00 / 0.00e+00 | 20260505 12:10:53.297 | INFO | PerfReport: 22.00%, Download -- 2.22e+01 / 0.00e+00, 8.11e+00 / 0.00e+00 | 20260505 12:10:53.297 | INFO | PerfReport: 31.72%, Compute -- 3.21e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 12:10:53.297 | INFO | PerfReport: 2.13%, LockUpload -- 2.16e+00 / 0.00e+00, 7.84e+00 / 0.00e+00 | 20260505 12:10:53.297 | INFO | PerfReport: 1.42%, LockDownload -- 1.43e+00 / 0.00e+00, 3.02e+00 / 0.00e+00 | 20260505 12:10:53.297 | INFO | PerfReport: 20.20%, LockCompute -- 2.04e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260505 12:10:53.297 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.76e-02 / 0.00e+00 | 20260505 12:10:53.430 | INFO | Starting a round of Baz IO | 20260505 12:10:56.683 | INFO | Populating RTMetrics report | 20260505 12:10:56.683 | INFO | RTMetrics report populated | 20260505 12:10:56.683 | INFO | Opening tmp RTMetrics json file | 20260505 12:10:56.687 | INFO | Serializing RTMetrics to json | 20260505 12:10:56.690 | INFO | Serialization done, swapping tmp file now... | 20260505 12:10:56.692 | INFO | RTMetrics done | 20260505 12:10:56.806 | INFO | Finished a round of Baz IO: Wrote 2.347 GiB in 3.376 seconds | 20260505 12:10:56.806 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.72% / 0.01% / 76.27% | 20260505 12:10:56.806 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.945989% / 1.19209e-05% / 5.5631e-05% | 20260505 12:10:56.806 | INFO | Baz IO overall: | 20260505 12:10:56.806 | INFO | BytesWritten : 1842906955335 | 20260505 12:10:56.806 | INFO | NumEvents : 877301607492 | 20260505 12:10:58.431 | INFO | Analyzing chunk frames = [3104768,3105280) | 20260505 12:10:58.431 | INFO | MemInfo: { | 20260505 12:10:58.431 | INFO | "gpu" : " 17 GB", | 20260505 12:10:58.431 | INFO | "huge" : "", | 20260505 12:10:58.431 | INFO | "huge_shared" : " 27 GB", | 20260505 12:10:58.431 | INFO | "malloc" : " 14 GB", | 20260505 12:10:58.431 | INFO | "pinned" : " 29 GB" | 20260505 12:10:58.431 | INFO | } | 20260505 12:10:58.431 | INFO | Perf report for the last 5.13569 seconds: | 20260505 12:10:58.431 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:10:58.431 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:58.431 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:58.431 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:58.431 | INFO | ANALYSIS : 64.8%, 2.9, 100.2ms, 0 | 20260505 12:10:58.431 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:10:58.432 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260505 12:10:58.432 | INFO | PRE_HQ : 61.1%, 1.8, 58.4ms, 0 | 20260505 12:10:58.432 | INFO | BAZWRITER : 65.0%, 4.1, 136.4ms, 0 | 20260505 12:10:58.432 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 12:10:58.432 | INFO | Uploaded 36GiB to GPU in 2.20692s (16.3123GiB/s) | 20260505 12:10:58.432 | INFO | Downloaded 38GiB to GPU in 2.29717s (16.5421GiB/s) | 20260505 12:11:03.575 | INFO | Analyzing chunk frames = [3105280,3105792) | 20260505 12:11:03.575 | INFO | MemInfo: { | 20260505 12:11:03.575 | INFO | "gpu" : " 17 GB", | 20260505 12:11:03.575 | INFO | "huge" : "", | 20260505 12:11:03.575 | INFO | "huge_shared" : " 27 GB", | 20260505 12:11:03.575 | INFO | "malloc" : " 14 GB", | 20260505 12:11:03.575 | INFO | "pinned" : " 29 GB" | 20260505 12:11:03.575 | INFO | } | 20260505 12:11:03.575 | INFO | Perf report for the last 5.14341 seconds: | 20260505 12:11:03.575 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:11:03.575 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:03.575 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:03.575 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:03.575 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 12:11:03.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:03.575 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:03.575 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 12:11:03.575 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:03.575 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:03.575 | INFO | Uploaded 36GiB to GPU in 2.22924s (16.149GiB/s) | 20260505 12:11:03.575 | INFO | Downloaded 34GiB to GPU in 2.09245s (16.2489GiB/s) | 20260505 12:11:08.709 | INFO | Analyzing chunk frames = [3105792,3106304) | 20260505 12:11:08.709 | INFO | MemInfo: { | 20260505 12:11:08.709 | INFO | "gpu" : " 17 GB", | 20260505 12:11:08.709 | INFO | "huge" : "", | 20260505 12:11:08.709 | INFO | "huge_shared" : " 27 GB", | 20260505 12:11:08.709 | INFO | "malloc" : " 14 GB", | 20260505 12:11:08.709 | INFO | "pinned" : " 29 GB" | 20260505 12:11:08.709 | INFO | } | 20260505 12:11:08.710 | INFO | Perf report for the last 5.13469 seconds: | 20260505 12:11:08.710 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:11:08.710 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:08.710 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:08.710 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:08.710 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 12:11:08.710 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:08.710 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:08.710 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 12:11:08.710 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:08.710 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:08.710 | INFO | Uploaded 36GiB to GPU in 2.20698s (16.3119GiB/s) | 20260505 12:11:08.710 | INFO | Downloaded 34GiB to GPU in 2.11115s (16.105GiB/s) | 20260505 12:11:13.854 | INFO | Analyzing chunk frames = [3106304,3106816) | 20260505 12:11:13.854 | INFO | MemInfo: { | 20260505 12:11:13.854 | INFO | "gpu" : " 17 GB", | 20260505 12:11:13.854 | INFO | "huge" : "", | 20260505 12:11:13.854 | INFO | "huge_shared" : " 27 GB", | 20260505 12:11:13.854 | INFO | "malloc" : " 14 GB", | 20260505 12:11:13.854 | INFO | "pinned" : " 29 GB" | 20260505 12:11:13.854 | INFO | } | 20260505 12:11:13.854 | INFO | Perf report for the last 5.14411 seconds: | 20260505 12:11:13.854 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:11:13.854 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:13.854 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:13.854 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:13.854 | INFO | ANALYSIS : 63.0%, 3.0, 98.0ms, 0 | 20260505 12:11:13.854 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:13.854 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:13.854 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 12:11:13.854 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:13.854 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:13.854 | INFO | Uploaded 36GiB to GPU in 2.19905s (16.3707GiB/s) | 20260505 12:11:13.854 | INFO | Downloaded 34GiB to GPU in 2.13252s (15.9436GiB/s) | 20260505 12:11:18.992 | INFO | Analyzing chunk frames = [3106816,3107328) | 20260505 12:11:18.992 | INFO | MemInfo: { | 20260505 12:11:18.992 | INFO | "gpu" : " 17 GB", | 20260505 12:11:18.992 | INFO | "huge" : "", | 20260505 12:11:18.992 | INFO | "huge_shared" : " 27 GB", | 20260505 12:11:18.992 | INFO | "malloc" : " 14 GB", | 20260505 12:11:18.992 | INFO | "pinned" : " 29 GB" | 20260505 12:11:18.992 | INFO | } | 20260505 12:11:18.992 | INFO | Perf report for the last 5.13869 seconds: | 20260505 12:11:18.992 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:11:18.992 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:18.992 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:18.992 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:18.992 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 12:11:18.992 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:18.992 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:18.992 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:11:18.992 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:18.992 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:18.993 | INFO | Uploaded 36GiB to GPU in 2.17814s (16.5279GiB/s) | 20260505 12:11:18.993 | INFO | Downloaded 34GiB to GPU in 2.12752s (15.9811GiB/s) | 20260505 12:11:24.127 | INFO | Analyzing chunk frames = [3107328,3107840) | 20260505 12:11:24.127 | INFO | MemInfo: { | 20260505 12:11:24.127 | INFO | "gpu" : " 17 GB", | 20260505 12:11:24.127 | INFO | "huge" : "", | 20260505 12:11:24.127 | INFO | "huge_shared" : " 27 GB", | 20260505 12:11:24.127 | INFO | "malloc" : " 14 GB", | 20260505 12:11:24.127 | INFO | "pinned" : " 29 GB" | 20260505 12:11:24.127 | INFO | } | 20260505 12:11:24.127 | INFO | Perf report for the last 5.13474 seconds: | 20260505 12:11:24.127 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:11:24.127 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:24.127 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:24.127 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:24.127 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 12:11:24.127 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:24.127 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:24.127 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 12:11:24.127 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:24.127 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:24.127 | INFO | Uploaded 36GiB to GPU in 2.15398s (16.7133GiB/s) | 20260505 12:11:24.127 | INFO | Downloaded 34GiB to GPU in 2.11087s (16.1071GiB/s) | 20260505 12:11:29.271 | INFO | Analyzing chunk frames = [3107840,3108352) | 20260505 12:11:29.271 | INFO | MemInfo: { | 20260505 12:11:29.271 | INFO | "gpu" : " 17 GB", | 20260505 12:11:29.271 | INFO | "huge" : "", | 20260505 12:11:29.271 | INFO | "huge_shared" : " 27 GB", | 20260505 12:11:29.271 | INFO | "malloc" : " 14 GB", | 20260505 12:11:29.271 | INFO | "pinned" : " 29 GB" | 20260505 12:11:29.271 | INFO | } | 20260505 12:11:29.271 | INFO | Perf report for the last 5.14423 seconds: | 20260505 12:11:29.271 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:11:29.271 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:29.271 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:29.271 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:29.271 | INFO | ANALYSIS : 63.1%, 3.0, 98.7ms, 0 | 20260505 12:11:29.271 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:29.271 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:29.271 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 12:11:29.271 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:29.271 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:29.272 | INFO | Uploaded 36GiB to GPU in 2.14616s (16.7741GiB/s) | 20260505 12:11:29.272 | INFO | Downloaded 34GiB to GPU in 2.13733s (15.9077GiB/s) | 20260505 12:11:34.409 | INFO | Analyzing chunk frames = [3108352,3108864) | 20260505 12:11:34.409 | INFO | MemInfo: { | 20260505 12:11:34.409 | INFO | "gpu" : " 17 GB", | 20260505 12:11:34.409 | INFO | "huge" : "", | 20260505 12:11:34.409 | INFO | "huge_shared" : " 27 GB", | 20260505 12:11:34.409 | INFO | "malloc" : " 14 GB", | 20260505 12:11:34.409 | INFO | "pinned" : " 29 GB" | 20260505 12:11:34.409 | INFO | } | 20260505 12:11:34.410 | INFO | Perf report for the last 5.13816 seconds: | 20260505 12:11:34.410 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:11:34.410 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:34.410 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:34.410 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:34.410 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 12:11:34.410 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:34.410 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:34.410 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 12:11:34.410 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:34.410 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:34.410 | INFO | Uploaded 36GiB to GPU in 2.17386s (16.5604GiB/s) | 20260505 12:11:34.410 | INFO | Downloaded 34GiB to GPU in 2.10542s (16.1488GiB/s) | 20260505 12:11:34.543 | INFO | Starting a round of Baz IO | 20260505 12:11:37.784 | INFO | Populating RTMetrics report | 20260505 12:11:37.784 | INFO | RTMetrics report populated | 20260505 12:11:37.784 | INFO | Opening tmp RTMetrics json file | 20260505 12:11:37.791 | INFO | Serializing RTMetrics to json | 20260505 12:11:37.834 | INFO | Serialization done, swapping tmp file now... | 20260505 12:11:37.837 | INFO | RTMetrics done | 20260505 12:11:37.904 | INFO | Finished a round of Baz IO: Wrote 2.349 GiB in 3.36 seconds | 20260505 12:11:37.904 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.72% / 0.01% / 76.28% | 20260505 12:11:37.904 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.946208% / 1.58946e-05% / 8.34465e-05% | 20260505 12:11:37.904 | INFO | Baz IO overall: | 20260505 12:11:37.904 | INFO | BytesWritten : 1845428902205 | 20260505 12:11:37.904 | INFO | NumEvents : 878499427069 | 20260505 12:11:39.543 | INFO | Analyzing chunk frames = [3108864,3109376) | 20260505 12:11:39.543 | INFO | MemInfo: { | 20260505 12:11:39.543 | INFO | "gpu" : " 17 GB", | 20260505 12:11:39.543 | INFO | "huge" : "", | 20260505 12:11:39.543 | INFO | "huge_shared" : " 27 GB", | 20260505 12:11:39.543 | INFO | "malloc" : " 14 GB", | 20260505 12:11:39.543 | INFO | "pinned" : " 29 GB" | 20260505 12:11:39.543 | INFO | } | 20260505 12:11:39.544 | INFO | Perf report for the last 5.134 seconds: | 20260505 12:11:39.544 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:11:39.544 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:39.544 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:39.544 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:39.544 | INFO | ANALYSIS : 64.6%, 2.9, 99.5ms, 0 | 20260505 12:11:39.544 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:39.544 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260505 12:11:39.544 | INFO | PRE_HQ : 60.9%, 1.8, 58.1ms, 0 | 20260505 12:11:39.544 | INFO | BAZWRITER : 64.8%, 4.1, 135.8ms, 0 | 20260505 12:11:39.544 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260505 12:11:39.544 | INFO | Uploaded 36GiB to GPU in 2.17817s (16.5277GiB/s) | 20260505 12:11:39.544 | INFO | Downloaded 38GiB to GPU in 2.28998s (16.594GiB/s) | 20260505 12:11:44.680 | INFO | Analyzing chunk frames = [3109376,3109888) | 20260505 12:11:44.680 | INFO | MemInfo: { | 20260505 12:11:44.680 | INFO | "gpu" : " 17 GB", | 20260505 12:11:44.680 | INFO | "huge" : "", | 20260505 12:11:44.680 | INFO | "huge_shared" : " 27 GB", | 20260505 12:11:44.680 | INFO | "malloc" : " 14 GB", | 20260505 12:11:44.680 | INFO | "pinned" : " 29 GB" | 20260505 12:11:44.680 | INFO | } | 20260505 12:11:44.680 | INFO | Perf report for the last 5.13676 seconds: | 20260505 12:11:44.680 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:11:44.680 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:44.680 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:44.680 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:44.680 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 12:11:44.680 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:44.680 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:44.680 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:11:44.680 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:44.680 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:44.681 | INFO | Uploaded 36GiB to GPU in 2.21585s (16.2466GiB/s) | 20260505 12:11:44.681 | INFO | Downloaded 34GiB to GPU in 2.08131s (16.3359GiB/s) | 20260505 12:11:49.826 | INFO | Analyzing chunk frames = [3109888,3110400) | 20260505 12:11:49.826 | INFO | MemInfo: { | 20260505 12:11:49.826 | INFO | "gpu" : " 17 GB", | 20260505 12:11:49.826 | INFO | "huge" : "", | 20260505 12:11:49.826 | INFO | "huge_shared" : " 27 GB", | 20260505 12:11:49.826 | INFO | "malloc" : " 14 GB", | 20260505 12:11:49.826 | INFO | "pinned" : " 29 GB" | 20260505 12:11:49.826 | INFO | } | 20260505 12:11:49.826 | INFO | Perf report for the last 5.14544 seconds: | 20260505 12:11:49.826 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:11:49.826 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:49.826 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:49.826 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:49.826 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 12:11:49.826 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:49.826 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:49.826 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:11:49.826 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:49.826 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:49.826 | INFO | Uploaded 36GiB to GPU in 2.22662s (16.168GiB/s) | 20260505 12:11:49.826 | INFO | Downloaded 34GiB to GPU in 2.12954s (15.9659GiB/s) | 20260505 12:11:54.976 | INFO | Analyzing chunk frames = [3110400,3110912) | 20260505 12:11:54.976 | INFO | MemInfo: { | 20260505 12:11:54.976 | INFO | "gpu" : " 17 GB", | 20260505 12:11:54.976 | INFO | "huge" : "", | 20260505 12:11:54.976 | INFO | "huge_shared" : " 27 GB", | 20260505 12:11:54.976 | INFO | "malloc" : " 14 GB", | 20260505 12:11:54.976 | INFO | "pinned" : " 29 GB" | 20260505 12:11:54.976 | INFO | } | 20260505 12:11:54.976 | INFO | Perf report for the last 5.15051 seconds: | 20260505 12:11:54.976 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:11:54.976 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:54.976 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:54.976 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:54.976 | INFO | ANALYSIS : 63.0%, 3.0, 98.0ms, 0 | 20260505 12:11:54.976 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:54.976 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:54.976 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 12:11:54.976 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:54.976 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:11:54.976 | INFO | Uploaded 36GiB to GPU in 2.2077s (16.3066GiB/s) | 20260505 12:11:54.977 | INFO | Downloaded 34GiB to GPU in 2.13276s (15.9418GiB/s) | 20260505 12:12:00.111 | INFO | Analyzing chunk frames = [3110912,3111424) | 20260505 12:12:00.111 | INFO | MemInfo: { | 20260505 12:12:00.111 | INFO | "gpu" : " 17 GB", | 20260505 12:12:00.111 | INFO | "huge" : "", | 20260505 12:12:00.111 | INFO | "huge_shared" : " 27 GB", | 20260505 12:12:00.111 | INFO | "malloc" : " 14 GB", | 20260505 12:12:00.111 | INFO | "pinned" : " 29 GB" | 20260505 12:12:00.111 | INFO | } | 20260505 12:12:00.111 | INFO | Perf report for the last 5.13467 seconds: | 20260505 12:12:00.111 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:12:00.111 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:00.111 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:00.111 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:00.111 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260505 12:12:00.111 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:00.111 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:00.111 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 12:12:00.111 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:00.111 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:00.111 | INFO | Uploaded 36GiB to GPU in 2.14053s (16.8182GiB/s) | 20260505 12:12:00.111 | INFO | Downloaded 34GiB to GPU in 2.12195s (16.023GiB/s) | 20260505 12:12:05.243 | INFO | Analyzing chunk frames = [3111424,3111936) | 20260505 12:12:05.243 | INFO | MemInfo: { | 20260505 12:12:05.243 | INFO | "gpu" : " 17 GB", | 20260505 12:12:05.243 | INFO | "huge" : "", | 20260505 12:12:05.243 | INFO | "huge_shared" : " 27 GB", | 20260505 12:12:05.243 | INFO | "malloc" : " 14 GB", | 20260505 12:12:05.243 | INFO | "pinned" : " 29 GB" | 20260505 12:12:05.243 | INFO | } | 20260505 12:12:05.243 | INFO | Perf report for the last 5.13251 seconds: | 20260505 12:12:05.243 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:12:05.243 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:05.243 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:05.244 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:05.244 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 12:12:05.244 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:05.244 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:05.244 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 12:12:05.244 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:05.244 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:05.244 | INFO | Uploaded 36GiB to GPU in 2.1461s (16.7747GiB/s) | 20260505 12:12:05.244 | INFO | Downloaded 34GiB to GPU in 2.10662s (16.1396GiB/s) | 20260505 12:12:10.381 | INFO | Analyzing chunk frames = [3111936,3112448) | 20260505 12:12:10.381 | INFO | MemInfo: { | 20260505 12:12:10.381 | INFO | "gpu" : " 17 GB", | 20260505 12:12:10.381 | INFO | "huge" : "", | 20260505 12:12:10.381 | INFO | "huge_shared" : " 27 GB", | 20260505 12:12:10.381 | INFO | "malloc" : " 14 GB", | 20260505 12:12:10.381 | INFO | "pinned" : " 29 GB" | 20260505 12:12:10.381 | INFO | } | 20260505 12:12:10.381 | INFO | Perf report for the last 5.13741 seconds: | 20260505 12:12:10.381 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:12:10.381 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:10.381 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:10.381 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:10.381 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 12:12:10.381 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:10.381 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:10.381 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 12:12:10.381 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:10.381 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:10.381 | INFO | Uploaded 36GiB to GPU in 2.18423s (16.4818GiB/s) | 20260505 12:12:10.381 | INFO | Downloaded 34GiB to GPU in 2.12908s (15.9694GiB/s) | 20260505 12:12:15.525 | INFO | Analyzing chunk frames = [3112448,3112960) | 20260505 12:12:15.525 | INFO | MemInfo: { | 20260505 12:12:15.525 | INFO | "gpu" : " 17 GB", | 20260505 12:12:15.525 | INFO | "huge" : "", | 20260505 12:12:15.525 | INFO | "huge_shared" : " 27 GB", | 20260505 12:12:15.525 | INFO | "malloc" : " 14 GB", | 20260505 12:12:15.525 | INFO | "pinned" : " 29 GB" | 20260505 12:12:15.525 | INFO | } | 20260505 12:12:15.525 | INFO | Perf report for the last 5.1444 seconds: | 20260505 12:12:15.525 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:12:15.525 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:15.525 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:15.525 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:15.525 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 12:12:15.525 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:15.525 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:15.525 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 12:12:15.525 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:15.525 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:15.525 | INFO | Uploaded 36GiB to GPU in 2.22129s (16.2068GiB/s) | 20260505 12:12:15.526 | INFO | Downloaded 34GiB to GPU in 2.08678s (16.293GiB/s) | 20260505 12:12:15.526 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 12:12:15.526 | INFO | Total mem usage: 15.3 GiB | 20260505 12:12:15.526 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 12:12:15.526 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -7.41%) | 20260505 12:12:15.526 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 12:12:15.526 | INFO | High Water Breakdown: | 20260505 12:12:15.526 | INFO | 60.9% : 8.7 GiB ( 91.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 12:12:15.526 | INFO | 38.9% : 5.55 GiB ( 83.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 12:12:15.526 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 12:12:15.526 | INFO | ------------------------------------------ | 20260505 12:12:15.526 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:12:15.526 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 12:12:15.526 | INFO | Total mem usage: 26.5 GiB | 20260505 12:12:15.526 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 12:12:15.526 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 12:12:15.526 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 12:12:15.526 | INFO | High Water Breakdown: | 20260505 12:12:15.526 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:12:15.526 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:12:15.526 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:12:15.526 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:12:15.526 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:12:15.526 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:12:15.526 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 12:12:15.526 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 12:12:15.526 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:12:15.526 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 12:12:15.526 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 12:12:15.526 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 12:12:15.526 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 12:12:15.526 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 12:12:15.526 | INFO | ------------------------------------------ | 20260505 12:12:15.526 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:12:15.526 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 12:12:15.526 | INFO | Total mem usage: 25.6 GiB | 20260505 12:12:15.526 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 12:12:15.526 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 12:12:15.526 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 12:12:15.526 | INFO | High Water Breakdown: | 20260505 12:12:15.526 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 12:12:15.526 | INFO | ------------------------------------------ | 20260505 12:12:15.526 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:12:15.526 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 12:12:15.526 | INFO | Total mem usage: 15.7 GiB | 20260505 12:12:15.526 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 12:12:15.526 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 12:12:15.526 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 12:12:15.526 | INFO | High Water Breakdown: | 20260505 12:12:15.526 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 12:12:15.526 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 12:12:15.526 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 12:12:15.526 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 12:12:15.526 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 12:12:15.526 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 12:12:15.526 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 12:12:15.526 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 12:12:15.526 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 12:12:15.526 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:12:15.526 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 12:12:15.526 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:12:15.526 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 12:12:15.526 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 12:12:15.526 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 12:12:15.526 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 12:12:15.526 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:12:15.526 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 12:12:15.526 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:12:15.526 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:12:15.526 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:12:15.526 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 12:12:15.526 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 12:12:15.526 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:12:15.526 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 12:12:15.526 | INFO | ------------------------------------------ | 20260505 12:12:15.526 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:12:15.526 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 12:12:15.526 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:12:15.526 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:12:15.526 | INFO | PerfReport: Full Execution -- 3.20e+01 / 7.71e+01, 1.38e+01 / 1.25e+00 | 20260505 12:12:15.526 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 9.95e-02, 1.02e-02 / 4.18e-03 | 20260505 12:12:15.526 | INFO | PerfReport: 11.78%, Baseline -- 3.77e+00 / 4.49e+00, 6.01e-01 / 2.14e-01 | 20260505 12:12:15.526 | INFO | PerfReport: 3.31%, Histogram -- 1.06e+00 / 1.50e+00, 2.04e-01 / 8.60e-02 | 20260505 12:12:15.526 | INFO | PerfReport: 15.91%, DME -- 4.07e+01 / 4.54e+01, 4.49e+00 / 1.98e+00 | 20260505 12:12:15.526 | INFO | PerfReport: 56.33%, FrameLabeling -- 1.80e+01 / 2.09e+01, 1.47e+00 / 2.11e+00 | 20260505 12:12:15.526 | INFO | PerfReport: 7.43%, PulseAccumulating -- 2.38e+00 / 2.85e+00, 4.31e-01 / 2.27e-01 | 20260505 12:12:15.526 | INFO | PerfReport: 3.66%, Metrics -- 1.17e+00 / 1.50e+00, 3.32e-01 / 2.10e-01 | 20260505 12:12:15.526 | INFO | PerfReport: 1.29%, PulseSerializing -- 4.11e-01 / 4.60e-01, 7.43e-02 / 3.48e-02 | 20260505 12:12:15.526 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 12:12:15.526 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:12:15.526 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:12:15.527 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260505 12:12:15.527 | INFO | PerfReport: 22.56%, Upload -- 2.28e+01 / 0.00e+00, 8.10e+00 / 0.00e+00 | 20260505 12:12:15.527 | INFO | PerfReport: 22.03%, Download -- 2.23e+01 / 0.00e+00, 8.18e+00 / 0.00e+00 | 20260505 12:12:15.527 | INFO | PerfReport: 31.68%, Compute -- 3.20e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 12:12:15.527 | INFO | PerfReport: 2.12%, LockUpload -- 2.14e+00 / 0.00e+00, 7.73e+00 / 0.00e+00 | 20260505 12:12:15.527 | INFO | PerfReport: 1.39%, LockDownload -- 1.41e+00 / 0.00e+00, 2.94e+00 / 0.00e+00 | 20260505 12:12:15.527 | INFO | PerfReport: 20.20%, LockCompute -- 2.04e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260505 12:12:15.527 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.17e-02 / 0.00e+00 | 20260505 12:12:15.657 | INFO | Starting a round of Baz IO | 20260505 12:12:18.902 | INFO | Populating RTMetrics report | 20260505 12:12:18.902 | INFO | RTMetrics report populated | 20260505 12:12:18.902 | INFO | Opening tmp RTMetrics json file | 20260505 12:12:18.921 | INFO | Serializing RTMetrics to json | 20260505 12:12:18.923 | INFO | Serialization done, swapping tmp file now... | 20260505 12:12:18.929 | INFO | RTMetrics done | 20260505 12:12:19.009 | INFO | Finished a round of Baz IO: Wrote 2.348 GiB in 3.351 seconds | 20260505 12:12:19.009 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.71% / 0.01% / 76.28% | 20260505 12:12:19.009 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.943025% / 7.94729e-06% / 9.13938e-05% | 20260505 12:12:19.009 | INFO | Baz IO overall: | 20260505 12:12:19.009 | INFO | BytesWritten : 1847950214813 | 20260505 12:12:19.009 | INFO | NumEvents : 879696805483 | 20260505 12:12:20.664 | INFO | Analyzing chunk frames = [3112960,3113472) | 20260505 12:12:20.664 | INFO | MemInfo: { | 20260505 12:12:20.664 | INFO | "gpu" : " 17 GB", | 20260505 12:12:20.664 | INFO | "huge" : "", | 20260505 12:12:20.664 | INFO | "huge_shared" : " 27 GB", | 20260505 12:12:20.664 | INFO | "malloc" : " 14 GB", | 20260505 12:12:20.664 | INFO | "pinned" : " 29 GB" | 20260505 12:12:20.664 | INFO | } | 20260505 12:12:20.664 | INFO | Perf report for the last 5.13869 seconds: | 20260505 12:12:20.664 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:12:20.664 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:20.664 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:20.664 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:20.664 | INFO | ANALYSIS : 64.6%, 3.0, 99.9ms, 0 | 20260505 12:12:20.664 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:20.664 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260505 12:12:20.664 | INFO | PRE_HQ : 60.9%, 1.8, 58.1ms, 0 | 20260505 12:12:20.664 | INFO | BAZWRITER : 64.5%, 4.1, 134.4ms, 0 | 20260505 12:12:20.664 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260505 12:12:20.664 | INFO | Uploaded 36GiB to GPU in 2.19638s (16.3906GiB/s) | 20260505 12:12:20.664 | INFO | Downloaded 38GiB to GPU in 2.27636s (16.6933GiB/s) | 20260505 12:12:25.804 | INFO | Analyzing chunk frames = [3113472,3113984) | 20260505 12:12:25.804 | INFO | MemInfo: { | 20260505 12:12:25.804 | INFO | "gpu" : " 17 GB", | 20260505 12:12:25.804 | INFO | "huge" : "", | 20260505 12:12:25.804 | INFO | "huge_shared" : " 27 GB", | 20260505 12:12:25.804 | INFO | "malloc" : " 14 GB", | 20260505 12:12:25.804 | INFO | "pinned" : " 29 GB" | 20260505 12:12:25.804 | INFO | } | 20260505 12:12:25.804 | INFO | Perf report for the last 5.14012 seconds: | 20260505 12:12:25.804 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:12:25.804 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:25.804 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:25.804 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:25.804 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 12:12:25.804 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:25.804 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:25.804 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:12:25.804 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:25.804 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:25.804 | INFO | Uploaded 36GiB to GPU in 2.23169s (16.1313GiB/s) | 20260505 12:12:25.804 | INFO | Downloaded 34GiB to GPU in 2.09873s (16.2003GiB/s) | 20260505 12:12:30.942 | INFO | Analyzing chunk frames = [3113984,3114496) | 20260505 12:12:30.942 | INFO | MemInfo: { | 20260505 12:12:30.942 | INFO | "gpu" : " 17 GB", | 20260505 12:12:30.942 | INFO | "huge" : "", | 20260505 12:12:30.942 | INFO | "huge_shared" : " 27 GB", | 20260505 12:12:30.942 | INFO | "malloc" : " 14 GB", | 20260505 12:12:30.942 | INFO | "pinned" : " 29 GB" | 20260505 12:12:30.942 | INFO | } | 20260505 12:12:30.942 | INFO | Perf report for the last 5.13813 seconds: | 20260505 12:12:30.942 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:12:30.942 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:30.942 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:30.942 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:30.942 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 12:12:30.942 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:30.942 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:30.942 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 12:12:30.942 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:30.942 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:30.943 | INFO | Uploaded 36GiB to GPU in 2.22134s (16.2065GiB/s) | 20260505 12:12:30.943 | INFO | Downloaded 34GiB to GPU in 2.11997s (16.038GiB/s) | 20260505 12:12:36.083 | INFO | Analyzing chunk frames = [3114496,3115008) | 20260505 12:12:36.083 | INFO | MemInfo: { | 20260505 12:12:36.083 | INFO | "gpu" : " 17 GB", | 20260505 12:12:36.083 | INFO | "huge" : "", | 20260505 12:12:36.083 | INFO | "huge_shared" : " 27 GB", | 20260505 12:12:36.083 | INFO | "malloc" : " 14 GB", | 20260505 12:12:36.083 | INFO | "pinned" : " 29 GB" | 20260505 12:12:36.083 | INFO | } | 20260505 12:12:36.083 | INFO | Perf report for the last 5.14051 seconds: | 20260505 12:12:36.083 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:12:36.083 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:36.083 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:36.083 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:36.083 | INFO | ANALYSIS : 63.3%, 3.0, 98.3ms, 0 | 20260505 12:12:36.083 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:36.083 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:36.083 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 12:12:36.083 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:36.083 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:36.083 | INFO | Uploaded 36GiB to GPU in 2.21311s (16.2667GiB/s) | 20260505 12:12:36.083 | INFO | Downloaded 34GiB to GPU in 2.13343s (15.9368GiB/s) | 20260505 12:12:41.222 | INFO | Analyzing chunk frames = [3115008,3115520) | 20260505 12:12:41.222 | INFO | MemInfo: { | 20260505 12:12:41.222 | INFO | "gpu" : " 17 GB", | 20260505 12:12:41.222 | INFO | "huge" : "", | 20260505 12:12:41.222 | INFO | "huge_shared" : " 27 GB", | 20260505 12:12:41.222 | INFO | "malloc" : " 14 GB", | 20260505 12:12:41.222 | INFO | "pinned" : " 29 GB" | 20260505 12:12:41.222 | INFO | } | 20260505 12:12:41.222 | INFO | Perf report for the last 5.13893 seconds: | 20260505 12:12:41.222 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:12:41.222 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:41.222 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:41.222 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:41.222 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 12:12:41.222 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:41.222 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:41.222 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:12:41.222 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:41.222 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:41.222 | INFO | Uploaded 36GiB to GPU in 2.16148s (16.6553GiB/s) | 20260505 12:12:41.222 | INFO | Downloaded 34GiB to GPU in 2.12482s (16.0014GiB/s) | 20260505 12:12:46.363 | INFO | Analyzing chunk frames = [3115520,3116032) | 20260505 12:12:46.363 | INFO | MemInfo: { | 20260505 12:12:46.363 | INFO | "gpu" : " 17 GB", | 20260505 12:12:46.363 | INFO | "huge" : "", | 20260505 12:12:46.363 | INFO | "huge_shared" : " 27 GB", | 20260505 12:12:46.363 | INFO | "malloc" : " 14 GB", | 20260505 12:12:46.363 | INFO | "pinned" : " 29 GB" | 20260505 12:12:46.363 | INFO | } | 20260505 12:12:46.363 | INFO | Perf report for the last 5.14148 seconds: | 20260505 12:12:46.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:12:46.363 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:46.363 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:46.363 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:46.363 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 12:12:46.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:46.363 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:46.363 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:12:46.363 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:46.363 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:46.363 | INFO | Uploaded 36GiB to GPU in 2.15251s (16.7247GiB/s) | 20260505 12:12:46.363 | INFO | Downloaded 34GiB to GPU in 2.10832s (16.1266GiB/s) | 20260505 12:12:51.494 | INFO | Analyzing chunk frames = [3116032,3116544) | 20260505 12:12:51.494 | INFO | MemInfo: { | 20260505 12:12:51.494 | INFO | "gpu" : " 17 GB", | 20260505 12:12:51.494 | INFO | "huge" : "", | 20260505 12:12:51.494 | INFO | "huge_shared" : " 27 GB", | 20260505 12:12:51.494 | INFO | "malloc" : " 14 GB", | 20260505 12:12:51.494 | INFO | "pinned" : " 29 GB" | 20260505 12:12:51.494 | INFO | } | 20260505 12:12:51.494 | INFO | Perf report for the last 5.13077 seconds: | 20260505 12:12:51.494 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:12:51.494 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:51.494 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:51.494 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:51.494 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 12:12:51.494 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:51.494 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:51.494 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 12:12:51.494 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:51.494 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:51.494 | INFO | Uploaded 36GiB to GPU in 2.17521s (16.5501GiB/s) | 20260505 12:12:51.494 | INFO | Downloaded 34GiB to GPU in 2.14295s (15.866GiB/s) | 20260505 12:12:56.635 | INFO | Analyzing chunk frames = [3116544,3117056) | 20260505 12:12:56.635 | INFO | MemInfo: { | 20260505 12:12:56.635 | INFO | "gpu" : " 17 GB", | 20260505 12:12:56.635 | INFO | "huge" : "", | 20260505 12:12:56.635 | INFO | "huge_shared" : " 27 GB", | 20260505 12:12:56.635 | INFO | "malloc" : " 14 GB", | 20260505 12:12:56.635 | INFO | "pinned" : " 29 GB" | 20260505 12:12:56.635 | INFO | } | 20260505 12:12:56.636 | INFO | Perf report for the last 5.14149 seconds: | 20260505 12:12:56.636 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:12:56.636 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:56.636 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:56.636 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:56.636 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 12:12:56.636 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:56.636 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:56.636 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 12:12:56.636 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:56.636 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:12:56.636 | INFO | Uploaded 36GiB to GPU in 2.22311s (16.1935GiB/s) | 20260505 12:12:56.636 | INFO | Downloaded 34GiB to GPU in 2.09332s (16.2422GiB/s) | 20260505 12:12:56.772 | INFO | Starting a round of Baz IO | 20260505 12:13:00.022 | INFO | Populating RTMetrics report | 20260505 12:13:00.023 | INFO | RTMetrics report populated | 20260505 12:13:00.023 | INFO | Opening tmp RTMetrics json file | 20260505 12:13:00.023 | INFO | Serializing RTMetrics to json | 20260505 12:13:00.024 | INFO | Serialization done, swapping tmp file now... | 20260505 12:13:00.025 | INFO | RTMetrics done | 20260505 12:13:00.144 | INFO | Finished a round of Baz IO: Wrote 2.349 GiB in 3.371 seconds | 20260505 12:13:00.144 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.70% / 0.01% / 76.29% | 20260505 12:13:00.144 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.942147% / 1.58946e-05% / 9.13938e-05% | 20260505 12:13:00.144 | INFO | Baz IO overall: | 20260505 12:13:00.144 | INFO | BytesWritten : 1850472774062 | 20260505 12:13:00.144 | INFO | NumEvents : 880895020979 | 20260505 12:13:01.774 | INFO | Analyzing chunk frames = [3117056,3117568) | 20260505 12:13:01.774 | INFO | MemInfo: { | 20260505 12:13:01.774 | INFO | "gpu" : " 17 GB", | 20260505 12:13:01.774 | INFO | "huge" : "", | 20260505 12:13:01.774 | INFO | "huge_shared" : " 27 GB", | 20260505 12:13:01.774 | INFO | "malloc" : " 14 GB", | 20260505 12:13:01.774 | INFO | "pinned" : " 29 GB" | 20260505 12:13:01.774 | INFO | } | 20260505 12:13:01.774 | INFO | Perf report for the last 5.1388 seconds: | 20260505 12:13:01.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:13:01.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:01.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:01.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:01.774 | INFO | ANALYSIS : 64.8%, 2.9, 99.9ms, 0 | 20260505 12:13:01.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:01.774 | INFO | RT_METRICS : 45.3%, 1.0, 23.7ms, 0 | 20260505 12:13:01.774 | INFO | PRE_HQ : 61.1%, 1.8, 58.3ms, 0 | 20260505 12:13:01.774 | INFO | BAZWRITER : 65.0%, 4.1, 136.1ms, 0 | 20260505 12:13:01.774 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 12:13:01.775 | INFO | Uploaded 36GiB to GPU in 2.19378s (16.41GiB/s) | 20260505 12:13:01.775 | INFO | Downloaded 38GiB to GPU in 2.29982s (16.5231GiB/s) | 20260505 12:13:06.927 | INFO | Analyzing chunk frames = [3117568,3118080) | 20260505 12:13:06.927 | INFO | MemInfo: { | 20260505 12:13:06.927 | INFO | "gpu" : " 17 GB", | 20260505 12:13:06.927 | INFO | "huge" : "", | 20260505 12:13:06.927 | INFO | "huge_shared" : " 27 GB", | 20260505 12:13:06.927 | INFO | "malloc" : " 14 GB", | 20260505 12:13:06.927 | INFO | "pinned" : " 29 GB" | 20260505 12:13:06.927 | INFO | } | 20260505 12:13:06.927 | INFO | Perf report for the last 5.15299 seconds: | 20260505 12:13:06.927 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:13:06.927 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:06.927 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:06.927 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:06.927 | INFO | ANALYSIS : 63.9%, 3.0, 100.1ms, 0 | 20260505 12:13:06.927 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:06.928 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:06.928 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:13:06.928 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:06.928 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:06.928 | INFO | Uploaded 36GiB to GPU in 2.23074s (16.1381GiB/s) | 20260505 12:13:06.928 | INFO | Downloaded 34GiB to GPU in 2.09567s (16.2239GiB/s) | 20260505 12:13:12.057 | INFO | Analyzing chunk frames = [3118080,3118592) | 20260505 12:13:12.057 | INFO | MemInfo: { | 20260505 12:13:12.057 | INFO | "gpu" : " 17 GB", | 20260505 12:13:12.057 | INFO | "huge" : "", | 20260505 12:13:12.057 | INFO | "huge_shared" : " 27 GB", | 20260505 12:13:12.057 | INFO | "malloc" : " 14 GB", | 20260505 12:13:12.057 | INFO | "pinned" : " 29 GB" | 20260505 12:13:12.057 | INFO | } | 20260505 12:13:12.057 | INFO | Perf report for the last 5.13011 seconds: | 20260505 12:13:12.058 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:13:12.058 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:12.058 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:12.058 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:12.058 | INFO | ANALYSIS : 63.4%, 3.0, 98.5ms, 0 | 20260505 12:13:12.058 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:12.058 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:12.058 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 12:13:12.058 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:12.058 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:12.058 | INFO | Uploaded 36GiB to GPU in 2.21799s (16.2309GiB/s) | 20260505 12:13:12.058 | INFO | Downloaded 34GiB to GPU in 2.09773s (16.208GiB/s) | 20260505 12:13:17.194 | INFO | Analyzing chunk frames = [3118592,3119104) | 20260505 12:13:17.194 | INFO | MemInfo: { | 20260505 12:13:17.194 | INFO | "gpu" : " 17 GB", | 20260505 12:13:17.194 | INFO | "huge" : "", | 20260505 12:13:17.194 | INFO | "huge_shared" : " 27 GB", | 20260505 12:13:17.194 | INFO | "malloc" : " 14 GB", | 20260505 12:13:17.194 | INFO | "pinned" : " 29 GB" | 20260505 12:13:17.194 | INFO | } | 20260505 12:13:17.194 | INFO | Perf report for the last 5.13652 seconds: | 20260505 12:13:17.194 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:13:17.194 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:17.194 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:17.194 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:17.194 | INFO | ANALYSIS : 63.3%, 3.0, 98.2ms, 0 | 20260505 12:13:17.194 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:17.194 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:17.194 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 12:13:17.194 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:17.194 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:17.194 | INFO | Uploaded 36GiB to GPU in 2.20815s (16.3032GiB/s) | 20260505 12:13:17.194 | INFO | Downloaded 34GiB to GPU in 2.12949s (15.9663GiB/s) | 20260505 12:13:22.338 | INFO | Analyzing chunk frames = [3119104,3119616) | 20260505 12:13:22.338 | INFO | MemInfo: { | 20260505 12:13:22.338 | INFO | "gpu" : " 17 GB", | 20260505 12:13:22.338 | INFO | "huge" : "", | 20260505 12:13:22.338 | INFO | "huge_shared" : " 27 GB", | 20260505 12:13:22.338 | INFO | "malloc" : " 14 GB", | 20260505 12:13:22.338 | INFO | "pinned" : " 29 GB" | 20260505 12:13:22.338 | INFO | } | 20260505 12:13:22.338 | INFO | Perf report for the last 5.14439 seconds: | 20260505 12:13:22.338 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:13:22.338 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:22.338 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:22.338 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:22.338 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 12:13:22.338 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:22.338 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:22.338 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 12:13:22.338 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:22.338 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:22.339 | INFO | Uploaded 36GiB to GPU in 2.18562s (16.4713GiB/s) | 20260505 12:13:22.339 | INFO | Downloaded 34GiB to GPU in 2.12728s (15.9829GiB/s) | 20260505 12:13:27.472 | INFO | Analyzing chunk frames = [3119616,3120128) | 20260505 12:13:27.472 | INFO | MemInfo: { | 20260505 12:13:27.472 | INFO | "gpu" : " 17 GB", | 20260505 12:13:27.472 | INFO | "huge" : "", | 20260505 12:13:27.472 | INFO | "huge_shared" : " 27 GB", | 20260505 12:13:27.472 | INFO | "malloc" : " 14 GB", | 20260505 12:13:27.472 | INFO | "pinned" : " 29 GB" | 20260505 12:13:27.472 | INFO | } | 20260505 12:13:27.472 | INFO | Perf report for the last 5.13349 seconds: | 20260505 12:13:27.472 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:13:27.472 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:27.472 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:27.472 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:27.472 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 12:13:27.472 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:27.472 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:27.472 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 12:13:27.472 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:27.472 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:27.472 | INFO | Uploaded 36GiB to GPU in 2.15589s (16.6984GiB/s) | 20260505 12:13:27.472 | INFO | Downloaded 34GiB to GPU in 2.10854s (16.1249GiB/s) | 20260505 12:13:32.623 | INFO | Analyzing chunk frames = [3120128,3120640) | 20260505 12:13:32.623 | INFO | MemInfo: { | 20260505 12:13:32.623 | INFO | "gpu" : " 17 GB", | 20260505 12:13:32.623 | INFO | "huge" : "", | 20260505 12:13:32.623 | INFO | "huge_shared" : " 27 GB", | 20260505 12:13:32.623 | INFO | "malloc" : " 14 GB", | 20260505 12:13:32.623 | INFO | "pinned" : " 29 GB" | 20260505 12:13:32.623 | INFO | } | 20260505 12:13:32.623 | INFO | Perf report for the last 5.15128 seconds: | 20260505 12:13:32.623 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:13:32.623 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:32.623 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:32.623 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:32.623 | INFO | ANALYSIS : 63.2%, 3.0, 98.8ms, 0 | 20260505 12:13:32.623 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:32.623 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:32.623 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 12:13:32.623 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:32.623 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:32.623 | INFO | Uploaded 36GiB to GPU in 2.17859s (16.5245GiB/s) | 20260505 12:13:32.623 | INFO | Downloaded 34GiB to GPU in 2.14379s (15.8598GiB/s) | 20260505 12:13:37.758 | INFO | Analyzing chunk frames = [3120640,3121152) | 20260505 12:13:37.758 | INFO | MemInfo: { | 20260505 12:13:37.758 | INFO | "gpu" : " 17 GB", | 20260505 12:13:37.758 | INFO | "huge" : "", | 20260505 12:13:37.758 | INFO | "huge_shared" : " 27 GB", | 20260505 12:13:37.758 | INFO | "malloc" : " 14 GB", | 20260505 12:13:37.758 | INFO | "pinned" : " 29 GB" | 20260505 12:13:37.758 | INFO | } | 20260505 12:13:37.758 | INFO | Perf report for the last 5.13462 seconds: | 20260505 12:13:37.758 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:13:37.758 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:37.758 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:37.758 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:37.758 | INFO | ANALYSIS : 64.0%, 3.0, 99.8ms, 0 | 20260505 12:13:37.758 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:37.758 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:37.758 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 12:13:37.758 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:37.758 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:37.758 | INFO | Uploaded 36GiB to GPU in 2.20344s (16.3381GiB/s) | 20260505 12:13:37.758 | INFO | Downloaded 34GiB to GPU in 2.11549s (16.0719GiB/s) | 20260505 12:13:37.758 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 12:13:37.758 | INFO | Total mem usage: 15.3 GiB | 20260505 12:13:37.758 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 12:13:37.758 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -6.82%) | 20260505 12:13:37.758 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 12:13:37.758 | INFO | High Water Breakdown: | 20260505 12:13:37.758 | INFO | 61.1% : 8.77 GiB ( 92.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 12:13:37.758 | INFO | 38.8% : 5.56 GiB ( 83.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 12:13:37.758 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 12:13:37.758 | INFO | ------------------------------------------ | 20260505 12:13:37.758 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:13:37.758 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 12:13:37.758 | INFO | Total mem usage: 26.5 GiB | 20260505 12:13:37.758 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 12:13:37.758 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 12:13:37.758 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 12:13:37.758 | INFO | High Water Breakdown: | 20260505 12:13:37.758 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:13:37.758 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:13:37.758 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:13:37.758 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:13:37.758 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:13:37.758 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:13:37.758 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 12:13:37.758 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 12:13:37.758 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:13:37.758 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 12:13:37.758 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 12:13:37.758 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 12:13:37.758 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 12:13:37.758 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 12:13:37.758 | INFO | ------------------------------------------ | 20260505 12:13:37.758 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:13:37.758 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 12:13:37.758 | INFO | Total mem usage: 25.6 GiB | 20260505 12:13:37.758 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 12:13:37.758 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 12:13:37.758 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 12:13:37.758 | INFO | High Water Breakdown: | 20260505 12:13:37.758 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 12:13:37.758 | INFO | ------------------------------------------ | 20260505 12:13:37.758 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:13:37.758 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 12:13:37.758 | INFO | Total mem usage: 15.7 GiB | 20260505 12:13:37.758 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 12:13:37.758 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 12:13:37.758 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 12:13:37.759 | INFO | High Water Breakdown: | 20260505 12:13:37.759 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 12:13:37.759 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 12:13:37.759 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 12:13:37.759 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 12:13:37.759 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 12:13:37.759 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 12:13:37.759 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 12:13:37.759 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 12:13:37.759 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 12:13:37.759 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:13:37.759 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 12:13:37.759 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:13:37.759 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 12:13:37.759 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 12:13:37.759 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 12:13:37.759 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 12:13:37.759 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:13:37.759 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 12:13:37.759 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:13:37.759 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:13:37.759 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:13:37.759 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 12:13:37.759 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 12:13:37.759 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:13:37.759 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 12:13:37.759 | INFO | ------------------------------------------ | 20260505 12:13:37.759 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:13:37.759 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 12:13:37.759 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:13:37.759 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:13:37.759 | INFO | PerfReport: Full Execution -- 3.20e+01 / 7.70e+01, 1.38e+01 / 1.53e+00 | 20260505 12:13:37.759 | INFO | PerfReport: 0.29%, AggregateStats -- 9.25e-02 / 1.01e-01, 1.03e-02 / 4.38e-03 | 20260505 12:13:37.759 | INFO | PerfReport: 11.74%, Baseline -- 3.75e+00 / 4.48e+00, 5.89e-01 / 2.11e-01 | 20260505 12:13:37.759 | INFO | PerfReport: 3.29%, Histogram -- 1.05e+00 / 1.50e+00, 2.00e-01 / 8.81e-02 | 20260505 12:13:37.759 | INFO | PerfReport: 15.97%, DME -- 4.08e+01 / 4.56e+01, 4.43e+00 / 2.08e+00 | 20260505 12:13:37.759 | INFO | PerfReport: 56.26%, FrameLabeling -- 1.80e+01 / 2.06e+01, 1.40e+00 / 1.93e+00 | 20260505 12:13:37.759 | INFO | PerfReport: 7.50%, PulseAccumulating -- 2.40e+00 / 2.87e+00, 4.43e-01 / 2.21e-01 | 20260505 12:13:37.759 | INFO | PerfReport: 3.67%, Metrics -- 1.17e+00 / 1.48e+00, 3.29e-01 / 1.49e-01 | 20260505 12:13:37.759 | INFO | PerfReport: 1.29%, PulseSerializing -- 4.11e-01 / 4.62e-01, 7.36e-02 / 2.91e-02 | 20260505 12:13:37.759 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 12:13:37.759 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:13:37.759 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:13:37.759 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260505 12:13:37.759 | INFO | PerfReport: 22.63%, Upload -- 2.29e+01 / 0.00e+00, 8.15e+00 / 0.00e+00 | 20260505 12:13:37.759 | INFO | PerfReport: 22.03%, Download -- 2.23e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260505 12:13:37.759 | INFO | PerfReport: 31.60%, Compute -- 3.20e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 12:13:37.759 | INFO | PerfReport: 2.29%, LockUpload -- 2.32e+00 / 0.00e+00, 8.04e+00 / 0.00e+00 | 20260505 12:13:37.759 | INFO | PerfReport: 1.34%, LockDownload -- 1.36e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260505 12:13:37.759 | INFO | PerfReport: 20.09%, LockCompute -- 2.03e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260505 12:13:37.759 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.89e-02 / 0.00e+00 | 20260505 12:13:37.891 | INFO | Starting a round of Baz IO | 20260505 12:13:41.142 | INFO | Populating RTMetrics report | 20260505 12:13:41.142 | INFO | RTMetrics report populated | 20260505 12:13:41.142 | INFO | Opening tmp RTMetrics json file | 20260505 12:13:41.148 | INFO | Serializing RTMetrics to json | 20260505 12:13:41.149 | INFO | Serialization done, swapping tmp file now... | 20260505 12:13:41.150 | INFO | RTMetrics done | 20260505 12:13:41.267 | INFO | Finished a round of Baz IO: Wrote 2.346 GiB in 3.375 seconds | 20260505 12:13:41.267 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.70% / 0.01% / 76.30% | 20260505 12:13:41.267 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.938062% / 3.97364e-06% / 9.93411e-05% | 20260505 12:13:41.267 | INFO | Baz IO overall: | 20260505 12:13:41.267 | INFO | BytesWritten : 1852992296996 | 20260505 12:13:41.267 | INFO | NumEvents : 882090982853 | 20260505 12:13:42.885 | INFO | Analyzing chunk frames = [3121152,3121664) | 20260505 12:13:42.885 | INFO | MemInfo: { | 20260505 12:13:42.885 | INFO | "gpu" : " 17 GB", | 20260505 12:13:42.885 | INFO | "huge" : "", | 20260505 12:13:42.885 | INFO | "huge_shared" : " 27 GB", | 20260505 12:13:42.885 | INFO | "malloc" : " 14 GB", | 20260505 12:13:42.885 | INFO | "pinned" : " 29 GB" | 20260505 12:13:42.885 | INFO | } | 20260505 12:13:42.885 | INFO | Perf report for the last 5.12756 seconds: | 20260505 12:13:42.885 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:13:42.885 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:42.885 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:42.885 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:42.885 | INFO | ANALYSIS : 64.9%, 2.9, 99.8ms, 0 | 20260505 12:13:42.885 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:42.885 | INFO | RT_METRICS : 45.5%, 1.0, 23.8ms, 0 | 20260505 12:13:42.885 | INFO | PRE_HQ : 61.1%, 1.8, 58.2ms, 0 | 20260505 12:13:42.885 | INFO | BAZWRITER : 65.2%, 4.1, 136.9ms, 0 | 20260505 12:13:42.885 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 12:13:42.886 | INFO | Uploaded 36GiB to GPU in 2.20433s (16.3315GiB/s) | 20260505 12:13:42.886 | INFO | Downloaded 38GiB to GPU in 2.3044s (16.4902GiB/s) | 20260505 12:13:48.035 | INFO | Analyzing chunk frames = [3121664,3122176) | 20260505 12:13:48.035 | INFO | MemInfo: { | 20260505 12:13:48.035 | INFO | "gpu" : " 17 GB", | 20260505 12:13:48.035 | INFO | "huge" : "", | 20260505 12:13:48.035 | INFO | "huge_shared" : " 27 GB", | 20260505 12:13:48.035 | INFO | "malloc" : " 14 GB", | 20260505 12:13:48.035 | INFO | "pinned" : " 29 GB" | 20260505 12:13:48.035 | INFO | } | 20260505 12:13:48.035 | INFO | Perf report for the last 5.1501 seconds: | 20260505 12:13:48.035 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:13:48.036 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:48.036 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:48.036 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:48.036 | INFO | ANALYSIS : 64.0%, 3.0, 100.2ms, 0 | 20260505 12:13:48.036 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:48.036 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:48.036 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 12:13:48.036 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:48.036 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:48.036 | INFO | Uploaded 36GiB to GPU in 2.23382s (16.1159GiB/s) | 20260505 12:13:48.036 | INFO | Downloaded 34GiB to GPU in 2.09101s (16.2601GiB/s) | 20260505 12:13:53.179 | INFO | Analyzing chunk frames = [3122176,3122688) | 20260505 12:13:53.179 | INFO | MemInfo: { | 20260505 12:13:53.179 | INFO | "gpu" : " 17 GB", | 20260505 12:13:53.179 | INFO | "huge" : "", | 20260505 12:13:53.179 | INFO | "huge_shared" : " 27 GB", | 20260505 12:13:53.179 | INFO | "malloc" : " 14 GB", | 20260505 12:13:53.179 | INFO | "pinned" : " 29 GB" | 20260505 12:13:53.179 | INFO | } | 20260505 12:13:53.179 | INFO | Perf report for the last 5.14393 seconds: | 20260505 12:13:53.179 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:13:53.179 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:53.179 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:53.179 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:53.179 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 12:13:53.179 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:53.179 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:53.179 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 12:13:53.180 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:53.180 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:53.180 | INFO | Uploaded 36GiB to GPU in 2.21839s (16.228GiB/s) | 20260505 12:13:53.180 | INFO | Downloaded 34GiB to GPU in 2.10908s (16.1208GiB/s) | 20260505 12:13:58.310 | INFO | Analyzing chunk frames = [3122688,3123200) | 20260505 12:13:58.310 | INFO | MemInfo: { | 20260505 12:13:58.310 | INFO | "gpu" : " 17 GB", | 20260505 12:13:58.310 | INFO | "huge" : "", | 20260505 12:13:58.310 | INFO | "huge_shared" : " 27 GB", | 20260505 12:13:58.310 | INFO | "malloc" : " 14 GB", | 20260505 12:13:58.310 | INFO | "pinned" : " 29 GB" | 20260505 12:13:58.310 | INFO | } | 20260505 12:13:58.310 | INFO | Perf report for the last 5.1308 seconds: | 20260505 12:13:58.310 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:13:58.310 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:58.310 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:58.310 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:58.310 | INFO | ANALYSIS : 63.2%, 3.0, 98.0ms, 0 | 20260505 12:13:58.310 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:58.310 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:58.310 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:13:58.310 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:58.310 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:13:58.310 | INFO | Uploaded 36GiB to GPU in 2.20022s (16.362GiB/s) | 20260505 12:13:58.311 | INFO | Downloaded 34GiB to GPU in 2.12981s (15.9639GiB/s) | 20260505 12:14:03.441 | INFO | Analyzing chunk frames = [3123200,3123712) | 20260505 12:14:03.441 | INFO | MemInfo: { | 20260505 12:14:03.442 | INFO | "gpu" : " 17 GB", | 20260505 12:14:03.442 | INFO | "huge" : "", | 20260505 12:14:03.442 | INFO | "huge_shared" : " 27 GB", | 20260505 12:14:03.442 | INFO | "malloc" : " 14 GB", | 20260505 12:14:03.442 | INFO | "pinned" : " 29 GB" | 20260505 12:14:03.442 | INFO | } | 20260505 12:14:03.442 | INFO | Perf report for the last 5.1314 seconds: | 20260505 12:14:03.442 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:14:03.442 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:03.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:03.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:03.442 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 12:14:03.442 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:03.442 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:03.442 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:14:03.442 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:03.442 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:03.442 | INFO | Uploaded 36GiB to GPU in 2.15157s (16.732GiB/s) | 20260505 12:14:03.442 | INFO | Downloaded 34GiB to GPU in 2.12596s (15.9928GiB/s) | 20260505 12:14:08.592 | INFO | Analyzing chunk frames = [3123712,3124224) | 20260505 12:14:08.592 | INFO | MemInfo: { | 20260505 12:14:08.592 | INFO | "gpu" : " 17 GB", | 20260505 12:14:08.592 | INFO | "huge" : "", | 20260505 12:14:08.593 | INFO | "huge_shared" : " 27 GB", | 20260505 12:14:08.593 | INFO | "malloc" : " 14 GB", | 20260505 12:14:08.593 | INFO | "pinned" : " 29 GB" | 20260505 12:14:08.593 | INFO | } | 20260505 12:14:08.593 | INFO | Perf report for the last 5.15094 seconds: | 20260505 12:14:08.593 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:14:08.593 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:08.593 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:08.593 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:08.593 | INFO | ANALYSIS : 63.4%, 3.0, 99.2ms, 0 | 20260505 12:14:08.593 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:08.593 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:08.593 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:14:08.593 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:08.593 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:08.593 | INFO | Uploaded 36GiB to GPU in 2.20413s (16.333GiB/s) | 20260505 12:14:08.593 | INFO | Downloaded 34GiB to GPU in 2.12167s (16.0251GiB/s) | 20260505 12:14:13.731 | INFO | Analyzing chunk frames = [3124224,3124736) | 20260505 12:14:13.731 | INFO | MemInfo: { | 20260505 12:14:13.731 | INFO | "gpu" : " 17 GB", | 20260505 12:14:13.731 | INFO | "huge" : "", | 20260505 12:14:13.731 | INFO | "huge_shared" : " 27 GB", | 20260505 12:14:13.731 | INFO | "malloc" : " 14 GB", | 20260505 12:14:13.731 | INFO | "pinned" : " 29 GB" | 20260505 12:14:13.731 | INFO | } | 20260505 12:14:13.731 | INFO | Perf report for the last 5.13836 seconds: | 20260505 12:14:13.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:14:13.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:13.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:13.731 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:13.731 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 12:14:13.731 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:13.731 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:13.731 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 12:14:13.731 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:13.731 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:13.731 | INFO | Uploaded 36GiB to GPU in 2.15955s (16.6702GiB/s) | 20260505 12:14:13.731 | INFO | Downloaded 34GiB to GPU in 2.13499s (15.9251GiB/s) | 20260505 12:14:18.870 | INFO | Analyzing chunk frames = [3124736,3125248) | 20260505 12:14:18.870 | INFO | MemInfo: { | 20260505 12:14:18.870 | INFO | "gpu" : " 17 GB", | 20260505 12:14:18.870 | INFO | "huge" : "", | 20260505 12:14:18.870 | INFO | "huge_shared" : " 27 GB", | 20260505 12:14:18.870 | INFO | "malloc" : " 14 GB", | 20260505 12:14:18.870 | INFO | "pinned" : " 29 GB" | 20260505 12:14:18.870 | INFO | } | 20260505 12:14:18.870 | INFO | Perf report for the last 5.13903 seconds: | 20260505 12:14:18.870 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:14:18.870 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:18.870 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:18.870 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:18.870 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 12:14:18.870 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:18.870 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:18.870 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 12:14:18.870 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:18.870 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:18.870 | INFO | Uploaded 36GiB to GPU in 2.17565s (16.5468GiB/s) | 20260505 12:14:18.870 | INFO | Downloaded 34GiB to GPU in 2.1143s (16.081GiB/s) | 20260505 12:14:19.000 | INFO | Starting a round of Baz IO | 20260505 12:14:22.247 | INFO | Populating RTMetrics report | 20260505 12:14:22.247 | INFO | RTMetrics report populated | 20260505 12:14:22.247 | INFO | Opening tmp RTMetrics json file | 20260505 12:14:22.251 | INFO | Serializing RTMetrics to json | 20260505 12:14:22.253 | INFO | Serialization done, swapping tmp file now... | 20260505 12:14:22.256 | INFO | RTMetrics done | 20260505 12:14:22.366 | INFO | Finished a round of Baz IO: Wrote 2.347 GiB in 3.366 seconds | 20260505 12:14:22.366 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.69% / 0.01% / 76.30% | 20260505 12:14:22.366 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.935638% / 1.98682e-05% / 6.75519e-05% | 20260505 12:14:22.366 | INFO | Baz IO overall: | 20260505 12:14:22.366 | INFO | BytesWritten : 1855512784145 | 20260505 12:14:22.366 | INFO | NumEvents : 883287832192 | 20260505 12:14:24.011 | INFO | Analyzing chunk frames = [3125248,3125760) | 20260505 12:14:24.011 | INFO | MemInfo: { | 20260505 12:14:24.011 | INFO | "gpu" : " 17 GB", | 20260505 12:14:24.011 | INFO | "huge" : "", | 20260505 12:14:24.011 | INFO | "huge_shared" : " 27 GB", | 20260505 12:14:24.011 | INFO | "malloc" : " 14 GB", | 20260505 12:14:24.011 | INFO | "pinned" : " 29 GB" | 20260505 12:14:24.011 | INFO | } | 20260505 12:14:24.011 | INFO | Perf report for the last 5.14089 seconds: | 20260505 12:14:24.011 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:14:24.011 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:24.011 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:24.011 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:24.011 | INFO | ANALYSIS : 64.6%, 3.0, 100.1ms, 0 | 20260505 12:14:24.011 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:24.011 | INFO | RT_METRICS : 45.3%, 1.0, 23.8ms, 0 | 20260505 12:14:24.011 | INFO | PRE_HQ : 60.9%, 1.8, 58.4ms, 0 | 20260505 12:14:24.011 | INFO | BAZWRITER : 64.8%, 4.1, 135.5ms, 0 | 20260505 12:14:24.011 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 12:14:24.011 | INFO | Uploaded 36GiB to GPU in 2.17155s (16.578GiB/s) | 20260505 12:14:24.012 | INFO | Downloaded 38GiB to GPU in 2.27893s (16.6745GiB/s) | 20260505 12:14:29.146 | INFO | Analyzing chunk frames = [3125760,3126272) | 20260505 12:14:29.146 | INFO | MemInfo: { | 20260505 12:14:29.146 | INFO | "gpu" : " 17 GB", | 20260505 12:14:29.146 | INFO | "huge" : "", | 20260505 12:14:29.146 | INFO | "huge_shared" : " 27 GB", | 20260505 12:14:29.146 | INFO | "malloc" : " 14 GB", | 20260505 12:14:29.146 | INFO | "pinned" : " 29 GB" | 20260505 12:14:29.146 | INFO | } | 20260505 12:14:29.146 | INFO | Perf report for the last 5.13496 seconds: | 20260505 12:14:29.146 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:14:29.146 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:29.146 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:29.146 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:29.146 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 12:14:29.146 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:29.146 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:29.146 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:14:29.146 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:29.146 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:29.146 | INFO | Uploaded 36GiB to GPU in 2.23121s (16.1348GiB/s) | 20260505 12:14:29.146 | INFO | Downloaded 34GiB to GPU in 2.10126s (16.1808GiB/s) | 20260505 12:14:34.291 | INFO | Analyzing chunk frames = [3126272,3126784) | 20260505 12:14:34.291 | INFO | MemInfo: { | 20260505 12:14:34.291 | INFO | "gpu" : " 17 GB", | 20260505 12:14:34.291 | INFO | "huge" : "", | 20260505 12:14:34.291 | INFO | "huge_shared" : " 27 GB", | 20260505 12:14:34.291 | INFO | "malloc" : " 14 GB", | 20260505 12:14:34.291 | INFO | "pinned" : " 29 GB" | 20260505 12:14:34.291 | INFO | } | 20260505 12:14:34.291 | INFO | Perf report for the last 5.14553 seconds: | 20260505 12:14:34.291 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:14:34.291 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:34.291 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:34.291 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:34.291 | INFO | ANALYSIS : 63.2%, 3.0, 98.4ms, 0 | 20260505 12:14:34.291 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:34.292 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:34.292 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 12:14:34.292 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:34.292 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:34.292 | INFO | Uploaded 36GiB to GPU in 2.21392s (16.2607GiB/s) | 20260505 12:14:34.292 | INFO | Downloaded 34GiB to GPU in 2.12261s (16.018GiB/s) | 20260505 12:14:39.429 | INFO | Analyzing chunk frames = [3126784,3127296) | 20260505 12:14:39.429 | INFO | MemInfo: { | 20260505 12:14:39.429 | INFO | "gpu" : " 17 GB", | 20260505 12:14:39.429 | INFO | "huge" : "", | 20260505 12:14:39.429 | INFO | "huge_shared" : " 27 GB", | 20260505 12:14:39.429 | INFO | "malloc" : " 14 GB", | 20260505 12:14:39.429 | INFO | "pinned" : " 29 GB" | 20260505 12:14:39.429 | INFO | } | 20260505 12:14:39.429 | INFO | Perf report for the last 5.13769 seconds: | 20260505 12:14:39.429 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:14:39.429 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:39.429 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:39.429 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:39.429 | INFO | ANALYSIS : 63.0%, 3.0, 97.9ms, 0 | 20260505 12:14:39.429 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:39.429 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:39.429 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 12:14:39.429 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:39.429 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:39.429 | INFO | Uploaded 36GiB to GPU in 2.20155s (16.3521GiB/s) | 20260505 12:14:39.429 | INFO | Downloaded 34GiB to GPU in 2.13076s (15.9568GiB/s) | 20260505 12:14:44.574 | INFO | Analyzing chunk frames = [3127296,3127808) | 20260505 12:14:44.574 | INFO | MemInfo: { | 20260505 12:14:44.574 | INFO | "gpu" : " 17 GB", | 20260505 12:14:44.574 | INFO | "huge" : "", | 20260505 12:14:44.574 | INFO | "huge_shared" : " 27 GB", | 20260505 12:14:44.574 | INFO | "malloc" : " 14 GB", | 20260505 12:14:44.574 | INFO | "pinned" : " 29 GB" | 20260505 12:14:44.574 | INFO | } | 20260505 12:14:44.574 | INFO | Perf report for the last 5.14498 seconds: | 20260505 12:14:44.574 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:14:44.574 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:44.574 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:44.574 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:44.574 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 12:14:44.574 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:44.574 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:44.574 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 12:14:44.574 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:44.574 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:44.574 | INFO | Uploaded 36GiB to GPU in 2.15388s (16.714GiB/s) | 20260505 12:14:44.574 | INFO | Downloaded 34GiB to GPU in 2.124s (16.0075GiB/s) | 20260505 12:14:49.709 | INFO | Analyzing chunk frames = [3127808,3128320) | 20260505 12:14:49.709 | INFO | MemInfo: { | 20260505 12:14:49.709 | INFO | "gpu" : " 17 GB", | 20260505 12:14:49.709 | INFO | "huge" : "", | 20260505 12:14:49.709 | INFO | "huge_shared" : " 27 GB", | 20260505 12:14:49.709 | INFO | "malloc" : " 14 GB", | 20260505 12:14:49.709 | INFO | "pinned" : " 29 GB" | 20260505 12:14:49.709 | INFO | } | 20260505 12:14:49.709 | INFO | Perf report for the last 5.13461 seconds: | 20260505 12:14:49.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:14:49.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:49.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:49.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:49.709 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 12:14:49.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:49.709 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:49.709 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 12:14:49.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:49.709 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:49.709 | INFO | Uploaded 36GiB to GPU in 2.165s (16.6282GiB/s) | 20260505 12:14:49.709 | INFO | Downloaded 34GiB to GPU in 2.10742s (16.1335GiB/s) | 20260505 12:14:54.842 | INFO | Analyzing chunk frames = [3128320,3128832) | 20260505 12:14:54.842 | INFO | MemInfo: { | 20260505 12:14:54.842 | INFO | "gpu" : " 17 GB", | 20260505 12:14:54.842 | INFO | "huge" : "", | 20260505 12:14:54.842 | INFO | "huge_shared" : " 27 GB", | 20260505 12:14:54.842 | INFO | "malloc" : " 14 GB", | 20260505 12:14:54.842 | INFO | "pinned" : " 29 GB" | 20260505 12:14:54.842 | INFO | } | 20260505 12:14:54.842 | INFO | Perf report for the last 5.13296 seconds: | 20260505 12:14:54.842 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:14:54.842 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:54.842 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:54.842 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:54.842 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 12:14:54.842 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:54.842 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:54.842 | INFO | PRE_HQ : 29.8%, 1.0, 15.4ms, 0 | 20260505 12:14:54.842 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:54.842 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:54.842 | INFO | Uploaded 36GiB to GPU in 2.16905s (16.5971GiB/s) | 20260505 12:14:54.842 | INFO | Downloaded 34GiB to GPU in 2.13455s (15.9284GiB/s) | 20260505 12:14:59.983 | INFO | Analyzing chunk frames = [3128832,3129344) | 20260505 12:14:59.983 | INFO | MemInfo: { | 20260505 12:14:59.983 | INFO | "gpu" : " 17 GB", | 20260505 12:14:59.983 | INFO | "huge" : "", | 20260505 12:14:59.983 | INFO | "huge_shared" : " 27 GB", | 20260505 12:14:59.983 | INFO | "malloc" : " 14 GB", | 20260505 12:14:59.983 | INFO | "pinned" : " 29 GB" | 20260505 12:14:59.983 | INFO | } | 20260505 12:14:59.983 | INFO | Perf report for the last 5.14169 seconds: | 20260505 12:14:59.983 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:14:59.983 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:59.983 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:59.983 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:59.983 | INFO | ANALYSIS : 63.7%, 3.0, 99.6ms, 0 | 20260505 12:14:59.983 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:59.983 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:59.984 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 12:14:59.984 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:59.984 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:14:59.984 | INFO | Uploaded 36GiB to GPU in 2.21658s (16.2413GiB/s) | 20260505 12:14:59.984 | INFO | Downloaded 34GiB to GPU in 2.09312s (16.2437GiB/s) | 20260505 12:14:59.984 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 12:14:59.984 | INFO | Total mem usage: 15.3 GiB | 20260505 12:14:59.984 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 12:14:59.984 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -6.82%) | 20260505 12:14:59.984 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 12:14:59.984 | INFO | High Water Breakdown: | 20260505 12:14:59.984 | INFO | 61.1% : 8.77 GiB ( 92.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 12:14:59.984 | INFO | 38.8% : 5.56 GiB ( 83.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 12:14:59.984 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 12:14:59.984 | INFO | ------------------------------------------ | 20260505 12:14:59.984 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:14:59.984 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 12:14:59.984 | INFO | Total mem usage: 26.5 GiB | 20260505 12:14:59.984 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 12:14:59.984 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 12:14:59.984 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 12:14:59.984 | INFO | High Water Breakdown: | 20260505 12:14:59.984 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:14:59.984 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:14:59.984 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:14:59.984 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:14:59.984 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:14:59.984 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:14:59.984 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 12:14:59.984 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 12:14:59.984 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:14:59.984 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 12:14:59.984 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 12:14:59.984 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 12:14:59.984 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 12:14:59.984 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 12:14:59.984 | INFO | ------------------------------------------ | 20260505 12:14:59.984 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:14:59.984 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 12:14:59.984 | INFO | Total mem usage: 25.6 GiB | 20260505 12:14:59.984 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 12:14:59.984 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 12:14:59.984 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 12:14:59.984 | INFO | High Water Breakdown: | 20260505 12:14:59.984 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 12:14:59.984 | INFO | ------------------------------------------ | 20260505 12:14:59.984 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:14:59.984 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 12:14:59.984 | INFO | Total mem usage: 15.7 GiB | 20260505 12:14:59.984 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 12:14:59.984 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 12:14:59.984 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 12:14:59.984 | INFO | High Water Breakdown: | 20260505 12:14:59.984 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 12:14:59.984 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 12:14:59.984 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 12:14:59.984 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 12:14:59.984 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 12:14:59.984 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 12:14:59.984 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 12:14:59.984 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 12:14:59.984 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 12:14:59.984 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:14:59.984 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 12:14:59.984 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:14:59.984 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 12:14:59.984 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 12:14:59.986 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 12:14:59.986 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 12:14:59.986 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:14:59.986 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 12:14:59.986 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:14:59.986 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:14:59.986 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:14:59.986 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 12:14:59.986 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 12:14:59.986 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:14:59.986 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 12:14:59.986 | INFO | ------------------------------------------ | 20260505 12:14:59.986 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:14:59.986 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 12:14:59.986 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:14:59.986 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:14:59.986 | INFO | PerfReport: Full Execution -- 3.20e+01 / 7.72e+01, 1.38e+01 / 1.48e+00 | 20260505 12:14:59.986 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 1.01e-01, 1.06e-02 / 4.40e-03 | 20260505 12:14:59.986 | INFO | PerfReport: 11.75%, Baseline -- 3.76e+00 / 4.48e+00, 5.95e-01 / 1.78e-01 | 20260505 12:14:59.986 | INFO | PerfReport: 3.29%, Histogram -- 1.05e+00 / 1.49e+00, 2.01e-01 / 8.90e-02 | 20260505 12:14:59.986 | INFO | PerfReport: 15.97%, DME -- 4.09e+01 / 4.58e+01, 4.63e+00 / 2.31e+00 | 20260505 12:14:59.986 | INFO | PerfReport: 56.25%, FrameLabeling -- 1.80e+01 / 2.06e+01, 1.42e+00 / 2.00e+00 | 20260505 12:14:59.986 | INFO | PerfReport: 7.50%, PulseAccumulating -- 2.40e+00 / 2.89e+00, 4.48e-01 / 1.81e-01 | 20260505 12:14:59.986 | INFO | PerfReport: 3.66%, Metrics -- 1.17e+00 / 1.48e+00, 3.29e-01 / 1.74e-01 | 20260505 12:14:59.986 | INFO | PerfReport: 1.28%, PulseSerializing -- 4.10e-01 / 4.59e-01, 7.43e-02 / 4.11e-02 | 20260505 12:14:59.986 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 12:14:59.986 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:14:59.986 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:14:59.986 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260505 12:14:59.986 | INFO | PerfReport: 22.58%, Upload -- 2.28e+01 / 0.00e+00, 8.15e+00 / 0.00e+00 | 20260505 12:14:59.986 | INFO | PerfReport: 22.04%, Download -- 2.23e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260505 12:14:59.986 | INFO | PerfReport: 31.64%, Compute -- 3.20e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 12:14:59.986 | INFO | PerfReport: 2.22%, LockUpload -- 2.25e+00 / 0.00e+00, 7.86e+00 / 0.00e+00 | 20260505 12:14:59.986 | INFO | PerfReport: 1.37%, LockDownload -- 1.39e+00 / 0.00e+00, 2.96e+00 / 0.00e+00 | 20260505 12:14:59.986 | INFO | PerfReport: 20.13%, LockCompute -- 2.04e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260505 12:14:59.986 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.24e-02 / 0.00e+00 | 20260505 12:15:00.117 | INFO | Starting a round of Baz IO | 20260505 12:15:03.359 | INFO | Populating RTMetrics report | 20260505 12:15:03.360 | INFO | RTMetrics report populated | 20260505 12:15:03.360 | INFO | Opening tmp RTMetrics json file | 20260505 12:15:03.360 | INFO | Serializing RTMetrics to json | 20260505 12:15:03.361 | INFO | Serialization done, swapping tmp file now... | 20260505 12:15:03.362 | INFO | RTMetrics done | 20260505 12:15:03.487 | INFO | Finished a round of Baz IO: Wrote 2.347 GiB in 3.369 seconds | 20260505 12:15:03.487 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.69% / 0.01% / 76.31% | 20260505 12:15:03.487 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.932757% / 1.98682e-05% / 5.5631e-05% | 20260505 12:15:03.487 | INFO | Baz IO overall: | 20260505 12:15:03.487 | INFO | BytesWritten : 1858033005954 | 20260505 12:15:03.487 | INFO | NumEvents : 884484443545 | 20260505 12:15:05.126 | INFO | Analyzing chunk frames = [3129344,3129856) | 20260505 12:15:05.126 | INFO | MemInfo: { | 20260505 12:15:05.126 | INFO | "gpu" : " 17 GB", | 20260505 12:15:05.126 | INFO | "huge" : "", | 20260505 12:15:05.126 | INFO | "huge_shared" : " 27 GB", | 20260505 12:15:05.126 | INFO | "malloc" : " 14 GB", | 20260505 12:15:05.126 | INFO | "pinned" : " 29 GB" | 20260505 12:15:05.126 | INFO | } | 20260505 12:15:05.126 | INFO | Perf report for the last 5.14253 seconds: | 20260505 12:15:05.126 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:15:05.126 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:05.126 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:05.126 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:05.126 | INFO | ANALYSIS : 64.5%, 2.9, 99.9ms, 0 | 20260505 12:15:05.126 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:05.126 | INFO | RT_METRICS : 45.3%, 1.0, 23.8ms, 0 | 20260505 12:15:05.126 | INFO | PRE_HQ : 60.7%, 1.9, 58.6ms, 0 | 20260505 12:15:05.126 | INFO | BAZWRITER : 64.9%, 4.1, 135.9ms, 0 | 20260505 12:15:05.126 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 12:15:05.126 | INFO | Uploaded 36GiB to GPU in 2.17014s (16.5888GiB/s) | 20260505 12:15:05.126 | INFO | Downloaded 38GiB to GPU in 2.28444s (16.6342GiB/s) | 20260505 12:15:10.255 | INFO | Analyzing chunk frames = [3129856,3130368) | 20260505 12:15:10.255 | INFO | MemInfo: { | 20260505 12:15:10.255 | INFO | "gpu" : " 17 GB", | 20260505 12:15:10.255 | INFO | "huge" : "", | 20260505 12:15:10.255 | INFO | "huge_shared" : " 27 GB", | 20260505 12:15:10.255 | INFO | "malloc" : " 14 GB", | 20260505 12:15:10.255 | INFO | "pinned" : " 29 GB" | 20260505 12:15:10.255 | INFO | } | 20260505 12:15:10.255 | INFO | Perf report for the last 5.12938 seconds: | 20260505 12:15:10.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:15:10.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:10.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:10.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:10.255 | INFO | ANALYSIS : 63.8%, 3.0, 99.4ms, 0 | 20260505 12:15:10.255 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:10.255 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:10.255 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:15:10.255 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:10.255 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:10.256 | INFO | Uploaded 36GiB to GPU in 2.17905s (16.521GiB/s) | 20260505 12:15:10.256 | INFO | Downloaded 34GiB to GPU in 2.04062s (16.6616GiB/s) | 20260505 12:15:15.397 | INFO | Analyzing chunk frames = [3130368,3130880) | 20260505 12:15:15.397 | INFO | MemInfo: { | 20260505 12:15:15.397 | INFO | "gpu" : " 17 GB", | 20260505 12:15:15.397 | INFO | "huge" : "", | 20260505 12:15:15.397 | INFO | "huge_shared" : " 27 GB", | 20260505 12:15:15.397 | INFO | "malloc" : " 14 GB", | 20260505 12:15:15.397 | INFO | "pinned" : " 29 GB" | 20260505 12:15:15.397 | INFO | } | 20260505 12:15:15.397 | INFO | Perf report for the last 5.14185 seconds: | 20260505 12:15:15.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:15:15.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:15.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:15.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:15.397 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260505 12:15:15.397 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:15.397 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:15.397 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 12:15:15.397 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:15.397 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:15.397 | INFO | Uploaded 36GiB to GPU in 2.22246s (16.1983GiB/s) | 20260505 12:15:15.397 | INFO | Downloaded 34GiB to GPU in 2.13194s (15.9479GiB/s) | 20260505 12:15:20.545 | INFO | Analyzing chunk frames = [3130880,3131392) | 20260505 12:15:20.545 | INFO | MemInfo: { | 20260505 12:15:20.545 | INFO | "gpu" : " 17 GB", | 20260505 12:15:20.546 | INFO | "huge" : "", | 20260505 12:15:20.546 | INFO | "huge_shared" : " 27 GB", | 20260505 12:15:20.546 | INFO | "malloc" : " 14 GB", | 20260505 12:15:20.546 | INFO | "pinned" : " 29 GB" | 20260505 12:15:20.546 | INFO | } | 20260505 12:15:20.546 | INFO | Perf report for the last 5.14835 seconds: | 20260505 12:15:20.546 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:15:20.546 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:20.546 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:20.546 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:20.546 | INFO | ANALYSIS : 63.2%, 3.0, 98.4ms, 0 | 20260505 12:15:20.546 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:20.546 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:20.546 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 12:15:20.546 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:20.546 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:20.546 | INFO | Uploaded 36GiB to GPU in 2.21083s (16.2835GiB/s) | 20260505 12:15:20.546 | INFO | Downloaded 34GiB to GPU in 2.13387s (15.9335GiB/s) | 20260505 12:15:25.691 | INFO | Analyzing chunk frames = [3131392,3131904) | 20260505 12:15:25.691 | INFO | MemInfo: { | 20260505 12:15:25.691 | INFO | "gpu" : " 17 GB", | 20260505 12:15:25.691 | INFO | "huge" : "", | 20260505 12:15:25.691 | INFO | "huge_shared" : " 27 GB", | 20260505 12:15:25.691 | INFO | "malloc" : " 14 GB", | 20260505 12:15:25.691 | INFO | "pinned" : " 29 GB" | 20260505 12:15:25.691 | INFO | } | 20260505 12:15:25.691 | INFO | Perf report for the last 5.14506 seconds: | 20260505 12:15:25.691 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:15:25.691 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:25.691 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:25.691 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:25.691 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 12:15:25.691 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:25.691 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:25.691 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 12:15:25.691 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:25.691 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:25.691 | INFO | Uploaded 36GiB to GPU in 2.1391s (16.8295GiB/s) | 20260505 12:15:25.691 | INFO | Downloaded 34GiB to GPU in 2.12204s (16.0223GiB/s) | 20260505 12:15:30.818 | INFO | Analyzing chunk frames = [3131904,3132416) | 20260505 12:15:30.818 | INFO | MemInfo: { | 20260505 12:15:30.818 | INFO | "gpu" : " 17 GB", | 20260505 12:15:30.818 | INFO | "huge" : "", | 20260505 12:15:30.818 | INFO | "huge_shared" : " 27 GB", | 20260505 12:15:30.818 | INFO | "malloc" : " 14 GB", | 20260505 12:15:30.818 | INFO | "pinned" : " 29 GB" | 20260505 12:15:30.818 | INFO | } | 20260505 12:15:30.818 | INFO | Perf report for the last 5.12745 seconds: | 20260505 12:15:30.818 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:15:30.818 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:30.818 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:30.818 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:30.818 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 12:15:30.818 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:30.818 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:30.818 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 12:15:30.818 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:30.818 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:30.818 | INFO | Uploaded 36GiB to GPU in 2.16293s (16.6441GiB/s) | 20260505 12:15:30.818 | INFO | Downloaded 34GiB to GPU in 2.10879s (16.123GiB/s) | 20260505 12:15:35.957 | INFO | Analyzing chunk frames = [3132416,3132928) | 20260505 12:15:35.957 | INFO | MemInfo: { | 20260505 12:15:35.957 | INFO | "gpu" : " 17 GB", | 20260505 12:15:35.957 | INFO | "huge" : "", | 20260505 12:15:35.957 | INFO | "huge_shared" : " 27 GB", | 20260505 12:15:35.957 | INFO | "malloc" : " 14 GB", | 20260505 12:15:35.957 | INFO | "pinned" : " 29 GB" | 20260505 12:15:35.957 | INFO | } | 20260505 12:15:35.957 | INFO | Perf report for the last 5.13861 seconds: | 20260505 12:15:35.957 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:15:35.957 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:35.957 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:35.957 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:35.957 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 12:15:35.957 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:35.957 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:35.957 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 12:15:35.957 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:35.957 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:35.957 | INFO | Uploaded 36GiB to GPU in 2.16864s (16.6003GiB/s) | 20260505 12:15:35.957 | INFO | Downloaded 34GiB to GPU in 2.16137s (15.7308GiB/s) | 20260505 12:15:41.100 | INFO | Analyzing chunk frames = [3132928,3133440) | 20260505 12:15:41.100 | INFO | MemInfo: { | 20260505 12:15:41.100 | INFO | "gpu" : " 17 GB", | 20260505 12:15:41.100 | INFO | "huge" : "", | 20260505 12:15:41.100 | INFO | "huge_shared" : " 27 GB", | 20260505 12:15:41.100 | INFO | "malloc" : " 14 GB", | 20260505 12:15:41.100 | INFO | "pinned" : " 29 GB" | 20260505 12:15:41.100 | INFO | } | 20260505 12:15:41.101 | INFO | Perf report for the last 5.14376 seconds: | 20260505 12:15:41.101 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:15:41.101 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:41.101 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:41.101 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:41.101 | INFO | ANALYSIS : 63.6%, 3.0, 99.5ms, 0 | 20260505 12:15:41.101 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:41.101 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:41.101 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 12:15:41.101 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:41.101 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:41.101 | INFO | Uploaded 36GiB to GPU in 2.21505s (16.2525GiB/s) | 20260505 12:15:41.101 | INFO | Downloaded 34GiB to GPU in 2.1074s (16.1336GiB/s) | 20260505 12:15:41.234 | INFO | Starting a round of Baz IO | 20260505 12:15:44.476 | INFO | Populating RTMetrics report | 20260505 12:15:44.477 | INFO | RTMetrics report populated | 20260505 12:15:44.477 | INFO | Opening tmp RTMetrics json file | 20260505 12:15:44.509 | INFO | Serializing RTMetrics to json | 20260505 12:15:44.558 | INFO | Serialization done, swapping tmp file now... | 20260505 12:15:44.560 | INFO | RTMetrics done | 20260505 12:15:44.645 | INFO | Finished a round of Baz IO: Wrote 2.345 GiB in 3.411 seconds | 20260505 12:15:44.645 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.68% / 0.01% / 76.31% | 20260505 12:15:44.646 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.92866% / 1.19209e-05% / 5.16574e-05% | 20260505 12:15:44.646 | INFO | Baz IO overall: | 20260505 12:15:44.646 | INFO | BytesWritten : 1860550532066 | 20260505 12:15:44.646 | INFO | NumEvents : 885678916111 | 20260505 12:15:46.235 | INFO | Analyzing chunk frames = [3133440,3133952) | 20260505 12:15:46.235 | INFO | MemInfo: { | 20260505 12:15:46.235 | INFO | "gpu" : " 17 GB", | 20260505 12:15:46.235 | INFO | "huge" : "", | 20260505 12:15:46.235 | INFO | "huge_shared" : " 27 GB", | 20260505 12:15:46.235 | INFO | "malloc" : " 14 GB", | 20260505 12:15:46.235 | INFO | "pinned" : " 29 GB" | 20260505 12:15:46.235 | INFO | } | 20260505 12:15:46.235 | INFO | Perf report for the last 5.13425 seconds: | 20260505 12:15:46.235 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:15:46.235 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:46.235 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:46.235 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:46.235 | INFO | ANALYSIS : 64.6%, 2.9, 99.6ms, 0 | 20260505 12:15:46.235 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:46.235 | INFO | RT_METRICS : 47.2%, 1.0, 24.8ms, 0 | 20260505 12:15:46.235 | INFO | PRE_HQ : 61.2%, 1.9, 58.8ms, 0 | 20260505 12:15:46.235 | INFO | BAZWRITER : 65.7%, 4.1, 138.1ms, 0 | 20260505 12:15:46.235 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260505 12:15:46.235 | INFO | Uploaded 36GiB to GPU in 2.18411s (16.4826GiB/s) | 20260505 12:15:46.235 | INFO | Downloaded 38GiB to GPU in 2.29049s (16.5903GiB/s) | 20260505 12:15:51.377 | INFO | Analyzing chunk frames = [3133952,3134464) | 20260505 12:15:51.377 | INFO | MemInfo: { | 20260505 12:15:51.377 | INFO | "gpu" : " 17 GB", | 20260505 12:15:51.377 | INFO | "huge" : "", | 20260505 12:15:51.377 | INFO | "huge_shared" : " 27 GB", | 20260505 12:15:51.377 | INFO | "malloc" : " 14 GB", | 20260505 12:15:51.377 | INFO | "pinned" : " 29 GB" | 20260505 12:15:51.377 | INFO | } | 20260505 12:15:51.377 | INFO | Perf report for the last 5.14191 seconds: | 20260505 12:15:51.377 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:15:51.377 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:51.377 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:51.377 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:51.377 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 12:15:51.377 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:51.377 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:51.377 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 12:15:51.377 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:51.377 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:51.377 | INFO | Uploaded 36GiB to GPU in 2.22969s (16.1458GiB/s) | 20260505 12:15:51.377 | INFO | Downloaded 34GiB to GPU in 2.102s (16.175GiB/s) | 20260505 12:15:56.519 | INFO | Analyzing chunk frames = [3134464,3134976) | 20260505 12:15:56.519 | INFO | MemInfo: { | 20260505 12:15:56.519 | INFO | "gpu" : " 17 GB", | 20260505 12:15:56.519 | INFO | "huge" : "", | 20260505 12:15:56.519 | INFO | "huge_shared" : " 27 GB", | 20260505 12:15:56.519 | INFO | "malloc" : " 14 GB", | 20260505 12:15:56.519 | INFO | "pinned" : " 29 GB" | 20260505 12:15:56.519 | INFO | } | 20260505 12:15:56.519 | INFO | Perf report for the last 5.14274 seconds: | 20260505 12:15:56.519 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:15:56.520 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:56.520 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:56.520 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:56.520 | INFO | ANALYSIS : 63.1%, 3.0, 98.3ms, 0 | 20260505 12:15:56.520 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:56.520 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:56.520 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 12:15:56.520 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:56.520 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:15:56.520 | INFO | Uploaded 36GiB to GPU in 2.20401s (16.3338GiB/s) | 20260505 12:15:56.520 | INFO | Downloaded 34GiB to GPU in 2.11969s (16.0401GiB/s) | 20260505 12:16:01.658 | INFO | Analyzing chunk frames = [3134976,3135488) | 20260505 12:16:01.658 | INFO | MemInfo: { | 20260505 12:16:01.658 | INFO | "gpu" : " 17 GB", | 20260505 12:16:01.658 | INFO | "huge" : "", | 20260505 12:16:01.658 | INFO | "huge_shared" : " 27 GB", | 20260505 12:16:01.658 | INFO | "malloc" : " 14 GB", | 20260505 12:16:01.658 | INFO | "pinned" : " 29 GB" | 20260505 12:16:01.658 | INFO | } | 20260505 12:16:01.658 | INFO | Perf report for the last 5.13839 seconds: | 20260505 12:16:01.658 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:16:01.658 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:01.658 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:01.658 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:01.658 | INFO | ANALYSIS : 63.3%, 3.0, 98.3ms, 0 | 20260505 12:16:01.658 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:01.658 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:01.658 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 12:16:01.658 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:01.658 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:01.658 | INFO | Uploaded 36GiB to GPU in 2.20851s (16.3006GiB/s) | 20260505 12:16:01.658 | INFO | Downloaded 34GiB to GPU in 2.13449s (15.9289GiB/s) | 20260505 12:16:06.793 | INFO | Analyzing chunk frames = [3135488,3136000) | 20260505 12:16:06.793 | INFO | MemInfo: { | 20260505 12:16:06.793 | INFO | "gpu" : " 17 GB", | 20260505 12:16:06.793 | INFO | "huge" : "", | 20260505 12:16:06.793 | INFO | "huge_shared" : " 27 GB", | 20260505 12:16:06.793 | INFO | "malloc" : " 14 GB", | 20260505 12:16:06.793 | INFO | "pinned" : " 29 GB" | 20260505 12:16:06.793 | INFO | } | 20260505 12:16:06.793 | INFO | Perf report for the last 5.13515 seconds: | 20260505 12:16:06.793 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:16:06.793 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:06.793 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:06.793 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:06.793 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260505 12:16:06.793 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:06.793 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:06.793 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 12:16:06.793 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:06.793 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:06.793 | INFO | Uploaded 36GiB to GPU in 2.14388s (16.792GiB/s) | 20260505 12:16:06.793 | INFO | Downloaded 34GiB to GPU in 2.12518s (15.9986GiB/s) | 20260505 12:16:11.936 | INFO | Analyzing chunk frames = [3136000,3136512) | 20260505 12:16:11.936 | INFO | MemInfo: { | 20260505 12:16:11.936 | INFO | "gpu" : " 17 GB", | 20260505 12:16:11.936 | INFO | "huge" : "", | 20260505 12:16:11.936 | INFO | "huge_shared" : " 27 GB", | 20260505 12:16:11.936 | INFO | "malloc" : " 14 GB", | 20260505 12:16:11.936 | INFO | "pinned" : " 29 GB" | 20260505 12:16:11.936 | INFO | } | 20260505 12:16:11.937 | INFO | Perf report for the last 5.14347 seconds: | 20260505 12:16:11.937 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:16:11.937 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:11.937 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:11.937 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:11.937 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 12:16:11.937 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:11.937 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:11.937 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 12:16:11.937 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:11.937 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:11.937 | INFO | Uploaded 36GiB to GPU in 2.14588s (16.7763GiB/s) | 20260505 12:16:11.937 | INFO | Downloaded 34GiB to GPU in 2.1081s (16.1282GiB/s) | 20260505 12:16:17.071 | INFO | Analyzing chunk frames = [3136512,3137024) | 20260505 12:16:17.071 | INFO | MemInfo: { | 20260505 12:16:17.071 | INFO | "gpu" : " 17 GB", | 20260505 12:16:17.071 | INFO | "huge" : "", | 20260505 12:16:17.071 | INFO | "huge_shared" : " 27 GB", | 20260505 12:16:17.071 | INFO | "malloc" : " 14 GB", | 20260505 12:16:17.071 | INFO | "pinned" : " 29 GB" | 20260505 12:16:17.071 | INFO | } | 20260505 12:16:17.071 | INFO | Perf report for the last 5.13434 seconds: | 20260505 12:16:17.071 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:16:17.071 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:17.071 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:17.071 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:17.071 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 12:16:17.071 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:17.071 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:17.071 | INFO | PRE_HQ : 29.8%, 1.0, 15.4ms, 0 | 20260505 12:16:17.071 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:17.071 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:17.071 | INFO | Uploaded 36GiB to GPU in 2.16537s (16.6253GiB/s) | 20260505 12:16:17.071 | INFO | Downloaded 34GiB to GPU in 2.15243s (15.7961GiB/s) | 20260505 12:16:22.204 | INFO | Analyzing chunk frames = [3137024,3137536) | 20260505 12:16:22.204 | INFO | MemInfo: { | 20260505 12:16:22.204 | INFO | "gpu" : " 17 GB", | 20260505 12:16:22.204 | INFO | "huge" : "", | 20260505 12:16:22.204 | INFO | "huge_shared" : " 27 GB", | 20260505 12:16:22.204 | INFO | "malloc" : " 14 GB", | 20260505 12:16:22.204 | INFO | "pinned" : " 29 GB" | 20260505 12:16:22.204 | INFO | } | 20260505 12:16:22.204 | INFO | Perf report for the last 5.13354 seconds: | 20260505 12:16:22.205 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:16:22.205 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:22.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:22.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:22.205 | INFO | ANALYSIS : 63.9%, 3.0, 99.6ms, 0 | 20260505 12:16:22.205 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:22.205 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:22.205 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 12:16:22.205 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:22.205 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:22.205 | INFO | Uploaded 36GiB to GPU in 2.21609s (16.2448GiB/s) | 20260505 12:16:22.205 | INFO | Downloaded 34GiB to GPU in 2.09043s (16.2646GiB/s) | 20260505 12:16:22.205 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 12:16:22.205 | INFO | Total mem usage: 15.3 GiB | 20260505 12:16:22.205 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 12:16:22.205 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -6.82%) | 20260505 12:16:22.205 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 12:16:22.205 | INFO | High Water Breakdown: | 20260505 12:16:22.205 | INFO | 61.1% : 8.77 GiB ( 92.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 12:16:22.205 | INFO | 38.8% : 5.56 GiB ( 83.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 12:16:22.205 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 12:16:22.205 | INFO | ------------------------------------------ | 20260505 12:16:22.205 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:16:22.205 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 12:16:22.205 | INFO | Total mem usage: 26.5 GiB | 20260505 12:16:22.205 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 12:16:22.205 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 12:16:22.205 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 12:16:22.205 | INFO | High Water Breakdown: | 20260505 12:16:22.205 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:16:22.205 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:16:22.205 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:16:22.205 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:16:22.205 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:16:22.205 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:16:22.205 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 12:16:22.205 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 12:16:22.205 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:16:22.205 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 12:16:22.205 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 12:16:22.206 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 12:16:22.206 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 12:16:22.206 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 12:16:22.206 | INFO | ------------------------------------------ | 20260505 12:16:22.206 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:16:22.206 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 12:16:22.206 | INFO | Total mem usage: 25.6 GiB | 20260505 12:16:22.206 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 12:16:22.206 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 12:16:22.206 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 12:16:22.206 | INFO | High Water Breakdown: | 20260505 12:16:22.206 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 12:16:22.206 | INFO | ------------------------------------------ | 20260505 12:16:22.206 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:16:22.206 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 12:16:22.206 | INFO | Total mem usage: 15.7 GiB | 20260505 12:16:22.206 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 12:16:22.206 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 12:16:22.206 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 12:16:22.206 | INFO | High Water Breakdown: | 20260505 12:16:22.206 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 12:16:22.206 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 12:16:22.206 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 12:16:22.206 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 12:16:22.206 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 12:16:22.206 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 12:16:22.206 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 12:16:22.206 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 12:16:22.206 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 12:16:22.206 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:16:22.206 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 12:16:22.206 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:16:22.206 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 12:16:22.206 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 12:16:22.206 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 12:16:22.206 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 12:16:22.206 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:16:22.206 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 12:16:22.206 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:16:22.206 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:16:22.206 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:16:22.206 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 12:16:22.206 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 12:16:22.206 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:16:22.206 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 12:16:22.206 | INFO | ------------------------------------------ | 20260505 12:16:22.206 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:16:22.206 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 12:16:22.206 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:16:22.206 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:16:22.206 | INFO | PerfReport: Full Execution -- 3.20e+01 / 7.68e+01, 1.34e+01 / 1.32e+00 | 20260505 12:16:22.206 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 1.01e-01, 1.03e-02 / 3.77e-03 | 20260505 12:16:22.206 | INFO | PerfReport: 11.75%, Baseline -- 3.76e+00 / 4.48e+00, 6.03e-01 / 2.35e-01 | 20260505 12:16:22.206 | INFO | PerfReport: 3.30%, Histogram -- 1.06e+00 / 1.49e+00, 2.04e-01 / 9.70e-02 | 20260505 12:16:22.206 | INFO | PerfReport: 15.76%, DME -- 4.04e+01 / 4.46e+01, 4.28e+00 / 1.99e+00 | 20260505 12:16:22.206 | INFO | PerfReport: 56.58%, FrameLabeling -- 1.81e+01 / 2.14e+01, 1.61e+00 / 2.23e+00 | 20260505 12:16:22.206 | INFO | PerfReport: 7.39%, PulseAccumulating -- 2.37e+00 / 2.83e+00, 4.26e-01 / 2.39e-01 | 20260505 12:16:22.206 | INFO | PerfReport: 3.65%, Metrics -- 1.17e+00 / 1.48e+00, 3.31e-01 / 1.55e-01 | 20260505 12:16:22.206 | INFO | PerfReport: 1.28%, PulseSerializing -- 4.10e-01 / 4.57e-01, 7.46e-02 / 3.31e-02 | 20260505 12:16:22.206 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 12:16:22.206 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:16:22.206 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:16:22.206 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260505 12:16:22.206 | INFO | PerfReport: 22.54%, Upload -- 2.28e+01 / 0.00e+00, 8.15e+00 / 0.00e+00 | 20260505 12:16:22.206 | INFO | PerfReport: 22.05%, Download -- 2.23e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260505 12:16:22.206 | INFO | PerfReport: 31.73%, Compute -- 3.20e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 12:16:22.206 | INFO | PerfReport: 2.06%, LockUpload -- 2.08e+00 / 0.00e+00, 7.61e+00 / 0.00e+00 | 20260505 12:16:22.206 | INFO | PerfReport: 1.35%, LockDownload -- 1.36e+00 / 0.00e+00, 2.97e+00 / 0.00e+00 | 20260505 12:16:22.206 | INFO | PerfReport: 20.26%, LockCompute -- 2.05e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260505 12:16:22.206 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.67e-02 / 0.00e+00 | 20260505 12:16:22.342 | INFO | Starting a round of Baz IO | 20260505 12:16:25.595 | INFO | Populating RTMetrics report | 20260505 12:16:25.595 | INFO | RTMetrics report populated | 20260505 12:16:25.595 | INFO | Opening tmp RTMetrics json file | 20260505 12:16:25.596 | INFO | Serializing RTMetrics to json | 20260505 12:16:25.597 | INFO | Serialization done, swapping tmp file now... | 20260505 12:16:25.598 | INFO | RTMetrics done | 20260505 12:16:25.718 | INFO | Finished a round of Baz IO: Wrote 2.345 GiB in 3.376 seconds | 20260505 12:16:25.718 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.67% / 0.01% / 76.32% | 20260505 12:16:25.718 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.926423% / 7.94729e-06% / 6.75519e-05% | 20260505 12:16:25.718 | INFO | Baz IO overall: | 20260505 12:16:25.719 | INFO | BytesWritten : 1863068542056 | 20260505 12:16:25.719 | INFO | NumEvents : 886873629636 | 20260505 12:16:27.349 | INFO | Analyzing chunk frames = [3137536,3138048) | 20260505 12:16:27.349 | INFO | MemInfo: { | 20260505 12:16:27.349 | INFO | "gpu" : " 17 GB", | 20260505 12:16:27.349 | INFO | "huge" : "", | 20260505 12:16:27.349 | INFO | "huge_shared" : " 27 GB", | 20260505 12:16:27.349 | INFO | "malloc" : " 14 GB", | 20260505 12:16:27.349 | INFO | "pinned" : " 29 GB" | 20260505 12:16:27.349 | INFO | } | 20260505 12:16:27.349 | INFO | Perf report for the last 5.1443 seconds: | 20260505 12:16:27.349 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:16:27.349 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:27.349 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:27.349 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:27.349 | INFO | ANALYSIS : 64.8%, 3.0, 100.5ms, 0 | 20260505 12:16:27.349 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:27.349 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260505 12:16:27.349 | INFO | PRE_HQ : 61.2%, 1.8, 58.4ms, 0 | 20260505 12:16:27.349 | INFO | BAZWRITER : 65.0%, 4.1, 135.9ms, 0 | 20260505 12:16:27.349 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 12:16:27.349 | INFO | Uploaded 36GiB to GPU in 2.2002s (16.3622GiB/s) | 20260505 12:16:27.349 | INFO | Downloaded 38GiB to GPU in 2.30066s (16.517GiB/s) | 20260505 12:16:32.497 | INFO | Analyzing chunk frames = [3138048,3138560) | 20260505 12:16:32.497 | INFO | MemInfo: { | 20260505 12:16:32.497 | INFO | "gpu" : " 17 GB", | 20260505 12:16:32.497 | INFO | "huge" : "", | 20260505 12:16:32.497 | INFO | "huge_shared" : " 27 GB", | 20260505 12:16:32.497 | INFO | "malloc" : " 14 GB", | 20260505 12:16:32.497 | INFO | "pinned" : " 29 GB" | 20260505 12:16:32.497 | INFO | } | 20260505 12:16:32.497 | INFO | Perf report for the last 5.14784 seconds: | 20260505 12:16:32.497 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:16:32.497 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:32.497 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:32.497 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:32.497 | INFO | ANALYSIS : 63.7%, 3.0, 99.6ms, 0 | 20260505 12:16:32.497 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:32.497 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:32.497 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 12:16:32.497 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:32.497 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:32.497 | INFO | Uploaded 36GiB to GPU in 2.18075s (16.5081GiB/s) | 20260505 12:16:32.497 | INFO | Downloaded 34GiB to GPU in 2.05291s (16.5618GiB/s) | 20260505 12:16:37.637 | INFO | Analyzing chunk frames = [3138560,3139072) | 20260505 12:16:37.637 | INFO | MemInfo: { | 20260505 12:16:37.637 | INFO | "gpu" : " 17 GB", | 20260505 12:16:37.637 | INFO | "huge" : "", | 20260505 12:16:37.637 | INFO | "huge_shared" : " 27 GB", | 20260505 12:16:37.638 | INFO | "malloc" : " 14 GB", | 20260505 12:16:37.638 | INFO | "pinned" : " 29 GB" | 20260505 12:16:37.638 | INFO | } | 20260505 12:16:37.638 | INFO | Perf report for the last 5.14104 seconds: | 20260505 12:16:37.638 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:16:37.638 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:37.638 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:37.638 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:37.638 | INFO | ANALYSIS : 63.1%, 3.0, 98.3ms, 0 | 20260505 12:16:37.638 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:37.638 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:37.638 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:16:37.638 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:37.638 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:37.638 | INFO | Uploaded 36GiB to GPU in 2.16334s (16.6409GiB/s) | 20260505 12:16:37.638 | INFO | Downloaded 34GiB to GPU in 2.11798s (16.0531GiB/s) | 20260505 12:16:42.766 | INFO | Analyzing chunk frames = [3139072,3139584) | 20260505 12:16:42.766 | INFO | MemInfo: { | 20260505 12:16:42.766 | INFO | "gpu" : " 17 GB", | 20260505 12:16:42.766 | INFO | "huge" : "", | 20260505 12:16:42.766 | INFO | "huge_shared" : " 27 GB", | 20260505 12:16:42.766 | INFO | "malloc" : " 14 GB", | 20260505 12:16:42.766 | INFO | "pinned" : " 29 GB" | 20260505 12:16:42.766 | INFO | } | 20260505 12:16:42.766 | INFO | Perf report for the last 5.12808 seconds: | 20260505 12:16:42.766 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:16:42.766 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:42.766 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:42.766 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:42.766 | INFO | ANALYSIS : 63.3%, 3.0, 98.2ms, 0 | 20260505 12:16:42.766 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:42.766 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:42.766 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:16:42.766 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:42.766 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:42.766 | INFO | Uploaded 36GiB to GPU in 2.20643s (16.316GiB/s) | 20260505 12:16:42.766 | INFO | Downloaded 34GiB to GPU in 2.13439s (15.9296GiB/s) | 20260505 12:16:47.907 | INFO | Analyzing chunk frames = [3139584,3140096) | 20260505 12:16:47.907 | INFO | MemInfo: { | 20260505 12:16:47.907 | INFO | "gpu" : " 17 GB", | 20260505 12:16:47.907 | INFO | "huge" : "", | 20260505 12:16:47.907 | INFO | "huge_shared" : " 27 GB", | 20260505 12:16:47.907 | INFO | "malloc" : " 14 GB", | 20260505 12:16:47.907 | INFO | "pinned" : " 29 GB" | 20260505 12:16:47.907 | INFO | } | 20260505 12:16:47.907 | INFO | Perf report for the last 5.14109 seconds: | 20260505 12:16:47.907 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:16:47.907 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:47.907 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:47.907 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:47.907 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 12:16:47.907 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:47.907 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:47.907 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:16:47.907 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:47.907 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:47.907 | INFO | Uploaded 36GiB to GPU in 2.15492s (16.7059GiB/s) | 20260505 12:16:47.907 | INFO | Downloaded 34GiB to GPU in 2.12867s (15.9724GiB/s) | 20260505 12:16:53.051 | INFO | Analyzing chunk frames = [3140096,3140608) | 20260505 12:16:53.051 | INFO | MemInfo: { | 20260505 12:16:53.051 | INFO | "gpu" : " 17 GB", | 20260505 12:16:53.051 | INFO | "huge" : "", | 20260505 12:16:53.051 | INFO | "huge_shared" : " 27 GB", | 20260505 12:16:53.051 | INFO | "malloc" : " 14 GB", | 20260505 12:16:53.051 | INFO | "pinned" : " 29 GB" | 20260505 12:16:53.051 | INFO | } | 20260505 12:16:53.051 | INFO | Perf report for the last 5.14458 seconds: | 20260505 12:16:53.051 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:16:53.051 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:53.051 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:53.051 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:53.051 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 12:16:53.051 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:53.051 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:53.051 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:16:53.052 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:53.052 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:53.052 | INFO | Uploaded 36GiB to GPU in 2.14567s (16.7779GiB/s) | 20260505 12:16:53.052 | INFO | Downloaded 34GiB to GPU in 2.10994s (16.1142GiB/s) | 20260505 12:16:58.190 | INFO | Analyzing chunk frames = [3140608,3141120) | 20260505 12:16:58.190 | INFO | MemInfo: { | 20260505 12:16:58.190 | INFO | "gpu" : " 17 GB", | 20260505 12:16:58.190 | INFO | "huge" : "", | 20260505 12:16:58.190 | INFO | "huge_shared" : " 27 GB", | 20260505 12:16:58.190 | INFO | "malloc" : " 14 GB", | 20260505 12:16:58.190 | INFO | "pinned" : " 29 GB" | 20260505 12:16:58.190 | INFO | } | 20260505 12:16:58.190 | INFO | Perf report for the last 5.13826 seconds: | 20260505 12:16:58.190 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:16:58.190 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:58.190 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:58.190 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:58.190 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 12:16:58.190 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:58.190 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:58.190 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 12:16:58.190 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:58.190 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:16:58.190 | INFO | Uploaded 36GiB to GPU in 2.18176s (16.5005GiB/s) | 20260505 12:16:58.190 | INFO | Downloaded 34GiB to GPU in 2.16585s (15.6982GiB/s) | 20260505 12:17:03.323 | INFO | Analyzing chunk frames = [3141120,3141632) | 20260505 12:17:03.323 | INFO | MemInfo: { | 20260505 12:17:03.323 | INFO | "gpu" : " 17 GB", | 20260505 12:17:03.323 | INFO | "huge" : "", | 20260505 12:17:03.323 | INFO | "huge_shared" : " 27 GB", | 20260505 12:17:03.323 | INFO | "malloc" : " 14 GB", | 20260505 12:17:03.323 | INFO | "pinned" : " 29 GB" | 20260505 12:17:03.323 | INFO | } | 20260505 12:17:03.323 | INFO | Perf report for the last 5.1334 seconds: | 20260505 12:17:03.323 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:17:03.323 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:03.323 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:03.323 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:03.323 | INFO | ANALYSIS : 64.0%, 3.0, 99.7ms, 0 | 20260505 12:17:03.323 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:03.323 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:03.323 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 12:17:03.323 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:03.323 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:03.323 | INFO | Uploaded 36GiB to GPU in 2.19751s (16.3822GiB/s) | 20260505 12:17:03.323 | INFO | Downloaded 34GiB to GPU in 2.11776s (16.0547GiB/s) | 20260505 12:17:03.456 | INFO | Starting a round of Baz IO | 20260505 12:17:06.706 | INFO | Populating RTMetrics report | 20260505 12:17:06.706 | INFO | RTMetrics report populated | 20260505 12:17:06.706 | INFO | Opening tmp RTMetrics json file | 20260505 12:17:06.707 | INFO | Serializing RTMetrics to json | 20260505 12:17:06.708 | INFO | Serialization done, swapping tmp file now... | 20260505 12:17:06.708 | INFO | RTMetrics done | 20260505 12:17:06.837 | INFO | Finished a round of Baz IO: Wrote 2.344 GiB in 3.38 seconds | 20260505 12:17:06.837 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.67% / 0.01% / 76.33% | 20260505 12:17:06.837 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.922445% / 0% / 7.94729e-05% | 20260505 12:17:06.837 | INFO | Baz IO overall: | 20260505 12:17:06.837 | INFO | BytesWritten : 1865585389209 | 20260505 12:17:06.837 | INFO | NumEvents : 888067532648 | 20260505 12:17:08.460 | INFO | Analyzing chunk frames = [3141632,3142144) | 20260505 12:17:08.460 | INFO | MemInfo: { | 20260505 12:17:08.460 | INFO | "gpu" : " 17 GB", | 20260505 12:17:08.460 | INFO | "huge" : "", | 20260505 12:17:08.460 | INFO | "huge_shared" : " 27 GB", | 20260505 12:17:08.460 | INFO | "malloc" : " 14 GB", | 20260505 12:17:08.460 | INFO | "pinned" : " 29 GB" | 20260505 12:17:08.460 | INFO | } | 20260505 12:17:08.460 | INFO | Perf report for the last 5.13693 seconds: | 20260505 12:17:08.460 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:17:08.460 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:08.460 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:08.460 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:08.460 | INFO | ANALYSIS : 64.7%, 2.9, 100.1ms, 0 | 20260505 12:17:08.460 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:08.460 | INFO | RT_METRICS : 45.3%, 1.0, 23.8ms, 0 | 20260505 12:17:08.460 | INFO | PRE_HQ : 61.0%, 1.8, 58.1ms, 0 | 20260505 12:17:08.460 | INFO | BAZWRITER : 65.1%, 4.0, 135.3ms, 0 | 20260505 12:17:08.460 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 12:17:08.460 | INFO | Uploaded 36GiB to GPU in 2.19066s (16.4334GiB/s) | 20260505 12:17:08.460 | INFO | Downloaded 38GiB to GPU in 2.29213s (16.5785GiB/s) | 20260505 12:17:13.612 | INFO | Analyzing chunk frames = [3142144,3142656) | 20260505 12:17:13.612 | INFO | MemInfo: { | 20260505 12:17:13.612 | INFO | "gpu" : " 17 GB", | 20260505 12:17:13.612 | INFO | "huge" : "", | 20260505 12:17:13.612 | INFO | "huge_shared" : " 27 GB", | 20260505 12:17:13.612 | INFO | "malloc" : " 14 GB", | 20260505 12:17:13.612 | INFO | "pinned" : " 29 GB" | 20260505 12:17:13.612 | INFO | } | 20260505 12:17:13.612 | INFO | Perf report for the last 5.15163 seconds: | 20260505 12:17:13.612 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:17:13.612 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:13.612 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:13.612 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:13.612 | INFO | ANALYSIS : 63.8%, 3.0, 99.9ms, 0 | 20260505 12:17:13.612 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:13.612 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:13.612 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:17:13.612 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:13.612 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:13.612 | INFO | Uploaded 36GiB to GPU in 2.2259s (16.1733GiB/s) | 20260505 12:17:13.612 | INFO | Downloaded 34GiB to GPU in 2.07664s (16.3726GiB/s) | 20260505 12:17:18.745 | INFO | Analyzing chunk frames = [3142656,3143168) | 20260505 12:17:18.745 | INFO | MemInfo: { | 20260505 12:17:18.745 | INFO | "gpu" : " 17 GB", | 20260505 12:17:18.745 | INFO | "huge" : "", | 20260505 12:17:18.745 | INFO | "huge_shared" : " 27 GB", | 20260505 12:17:18.745 | INFO | "malloc" : " 14 GB", | 20260505 12:17:18.745 | INFO | "pinned" : " 29 GB" | 20260505 12:17:18.745 | INFO | } | 20260505 12:17:18.745 | INFO | Perf report for the last 5.13374 seconds: | 20260505 12:17:18.745 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:17:18.746 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:18.746 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:18.746 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:18.746 | INFO | ANALYSIS : 63.4%, 3.0, 98.5ms, 0 | 20260505 12:17:18.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:18.746 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:18.746 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 12:17:18.746 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:18.746 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:18.746 | INFO | Uploaded 36GiB to GPU in 2.19988s (16.3645GiB/s) | 20260505 12:17:18.746 | INFO | Downloaded 34GiB to GPU in 2.11496s (16.076GiB/s) | 20260505 12:17:23.884 | INFO | Analyzing chunk frames = [3143168,3143680) | 20260505 12:17:23.884 | INFO | MemInfo: { | 20260505 12:17:23.884 | INFO | "gpu" : " 17 GB", | 20260505 12:17:23.884 | INFO | "huge" : "", | 20260505 12:17:23.884 | INFO | "huge_shared" : " 27 GB", | 20260505 12:17:23.884 | INFO | "malloc" : " 14 GB", | 20260505 12:17:23.884 | INFO | "pinned" : " 29 GB" | 20260505 12:17:23.885 | INFO | } | 20260505 12:17:23.885 | INFO | Perf report for the last 5.13908 seconds: | 20260505 12:17:23.885 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:17:23.885 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:23.885 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:23.885 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:23.885 | INFO | ANALYSIS : 63.3%, 3.0, 98.3ms, 0 | 20260505 12:17:23.885 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:23.885 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:23.885 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 12:17:23.885 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:23.885 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:23.885 | INFO | Uploaded 36GiB to GPU in 2.20624s (16.3174GiB/s) | 20260505 12:17:23.885 | INFO | Downloaded 34GiB to GPU in 2.13574s (15.9195GiB/s) | 20260505 12:17:29.019 | INFO | Analyzing chunk frames = [3143680,3144192) | 20260505 12:17:29.019 | INFO | MemInfo: { | 20260505 12:17:29.019 | INFO | "gpu" : " 17 GB", | 20260505 12:17:29.019 | INFO | "huge" : "", | 20260505 12:17:29.019 | INFO | "huge_shared" : " 27 GB", | 20260505 12:17:29.019 | INFO | "malloc" : " 14 GB", | 20260505 12:17:29.019 | INFO | "pinned" : " 29 GB" | 20260505 12:17:29.019 | INFO | } | 20260505 12:17:29.019 | INFO | Perf report for the last 5.13444 seconds: | 20260505 12:17:29.019 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:17:29.019 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:29.019 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:29.019 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:29.019 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 12:17:29.019 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:29.019 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:29.019 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 12:17:29.019 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:29.019 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:29.019 | INFO | Uploaded 36GiB to GPU in 2.15656s (16.6933GiB/s) | 20260505 12:17:29.019 | INFO | Downloaded 34GiB to GPU in 2.12736s (15.9822GiB/s) | 20260505 12:17:34.158 | INFO | Analyzing chunk frames = [3144192,3144704) | 20260505 12:17:34.158 | INFO | MemInfo: { | 20260505 12:17:34.158 | INFO | "gpu" : " 17 GB", | 20260505 12:17:34.158 | INFO | "huge" : "", | 20260505 12:17:34.158 | INFO | "huge_shared" : " 27 GB", | 20260505 12:17:34.158 | INFO | "malloc" : " 14 GB", | 20260505 12:17:34.158 | INFO | "pinned" : " 29 GB" | 20260505 12:17:34.158 | INFO | } | 20260505 12:17:34.158 | INFO | Perf report for the last 5.13925 seconds: | 20260505 12:17:34.158 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:17:34.158 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:34.158 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:34.158 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:34.158 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 12:17:34.158 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:34.158 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:34.158 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:17:34.158 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:34.158 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:34.159 | INFO | Uploaded 36GiB to GPU in 2.20904s (16.2967GiB/s) | 20260505 12:17:34.159 | INFO | Downloaded 34GiB to GPU in 2.10735s (16.134GiB/s) | 20260505 12:17:39.302 | INFO | Analyzing chunk frames = [3144704,3145216) | 20260505 12:17:39.302 | INFO | MemInfo: { | 20260505 12:17:39.302 | INFO | "gpu" : " 17 GB", | 20260505 12:17:39.302 | INFO | "huge" : "", | 20260505 12:17:39.302 | INFO | "huge_shared" : " 27 GB", | 20260505 12:17:39.302 | INFO | "malloc" : " 14 GB", | 20260505 12:17:39.303 | INFO | "pinned" : " 29 GB" | 20260505 12:17:39.303 | INFO | } | 20260505 12:17:39.303 | INFO | Perf report for the last 5.14426 seconds: | 20260505 12:17:39.303 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:17:39.303 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:39.303 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:39.303 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:39.303 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 12:17:39.303 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:39.303 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:39.303 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 12:17:39.303 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:39.303 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:39.303 | INFO | Uploaded 36GiB to GPU in 2.17523s (16.55GiB/s) | 20260505 12:17:39.303 | INFO | Downloaded 34GiB to GPU in 2.1195s (16.0415GiB/s) | 20260505 12:17:44.448 | INFO | Analyzing chunk frames = [3145216,3145728) | 20260505 12:17:44.448 | INFO | MemInfo: { | 20260505 12:17:44.448 | INFO | "gpu" : " 17 GB", | 20260505 12:17:44.448 | INFO | "huge" : "", | 20260505 12:17:44.448 | INFO | "huge_shared" : " 27 GB", | 20260505 12:17:44.448 | INFO | "malloc" : " 14 GB", | 20260505 12:17:44.448 | INFO | "pinned" : " 29 GB" | 20260505 12:17:44.448 | INFO | } | 20260505 12:17:44.448 | INFO | Perf report for the last 5.14574 seconds: | 20260505 12:17:44.448 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:17:44.448 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:44.448 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:44.448 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:44.448 | INFO | ANALYSIS : 63.5%, 3.0, 99.3ms, 0 | 20260505 12:17:44.448 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:44.448 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:44.448 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 12:17:44.448 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:44.448 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:44.449 | INFO | Uploaded 36GiB to GPU in 2.15102s (16.7362GiB/s) | 20260505 12:17:44.449 | INFO | Downloaded 34GiB to GPU in 2.10307s (16.1668GiB/s) | 20260505 12:17:44.449 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 12:17:44.449 | INFO | Total mem usage: 15.3 GiB | 20260505 12:17:44.449 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 12:17:44.449 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -7.53%) | 20260505 12:17:44.449 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 12:17:44.449 | INFO | High Water Breakdown: | 20260505 12:17:44.449 | INFO | 61% : 8.7 GiB ( 91.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 12:17:44.449 | INFO | 38.8% : 5.54 GiB ( 83.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 12:17:44.449 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 12:17:44.449 | INFO | ------------------------------------------ | 20260505 12:17:44.449 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:17:44.449 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 12:17:44.449 | INFO | Total mem usage: 26.5 GiB | 20260505 12:17:44.449 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 12:17:44.449 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 12:17:44.449 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 12:17:44.449 | INFO | High Water Breakdown: | 20260505 12:17:44.449 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:17:44.449 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:17:44.449 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:17:44.449 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:17:44.449 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:17:44.449 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:17:44.449 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 12:17:44.449 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 12:17:44.449 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:17:44.449 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 12:17:44.449 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 12:17:44.449 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 12:17:44.449 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 12:17:44.449 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 12:17:44.449 | INFO | ------------------------------------------ | 20260505 12:17:44.449 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:17:44.449 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 12:17:44.449 | INFO | Total mem usage: 25.6 GiB | 20260505 12:17:44.449 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 12:17:44.449 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 12:17:44.449 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 12:17:44.449 | INFO | High Water Breakdown: | 20260505 12:17:44.449 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 12:17:44.449 | INFO | ------------------------------------------ | 20260505 12:17:44.449 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:17:44.449 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 12:17:44.449 | INFO | Total mem usage: 15.7 GiB | 20260505 12:17:44.449 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 12:17:44.449 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 12:17:44.449 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 12:17:44.449 | INFO | High Water Breakdown: | 20260505 12:17:44.449 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 12:17:44.449 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 12:17:44.450 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 12:17:44.450 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 12:17:44.450 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 12:17:44.450 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 12:17:44.450 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 12:17:44.450 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 12:17:44.450 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 12:17:44.450 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:17:44.450 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 12:17:44.450 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:17:44.450 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 12:17:44.450 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 12:17:44.450 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 12:17:44.450 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 12:17:44.450 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:17:44.450 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 12:17:44.450 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:17:44.450 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:17:44.450 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:17:44.450 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 12:17:44.450 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 12:17:44.450 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:17:44.450 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 12:17:44.450 | INFO | ------------------------------------------ | 20260505 12:17:44.450 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:17:44.450 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 12:17:44.450 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:17:44.450 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:17:44.450 | INFO | PerfReport: Full Execution -- 3.20e+01 / 7.75e+01, 1.39e+01 / 1.78e+00 | 20260505 12:17:44.450 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 1.01e-01, 1.02e-02 / 4.32e-03 | 20260505 12:17:44.450 | INFO | PerfReport: 11.77%, Baseline -- 3.77e+00 / 4.49e+00, 6.08e-01 / 2.00e-01 | 20260505 12:17:44.450 | INFO | PerfReport: 3.31%, Histogram -- 1.06e+00 / 1.50e+00, 2.05e-01 / 8.14e-02 | 20260505 12:17:44.450 | INFO | PerfReport: 16.01%, DME -- 4.11e+01 / 4.64e+01, 4.89e+00 / 2.51e+00 | 20260505 12:17:44.450 | INFO | PerfReport: 56.32%, FrameLabeling -- 1.80e+01 / 2.02e+01, 1.50e+00 / 1.98e+00 | 20260505 12:17:44.450 | INFO | PerfReport: 7.41%, PulseAccumulating -- 2.37e+00 / 2.92e+00, 4.40e-01 / 2.07e-01 | 20260505 12:17:44.450 | INFO | PerfReport: 3.61%, Metrics -- 1.16e+00 / 1.47e+00, 3.23e-01 / 1.63e-01 | 20260505 12:17:44.450 | INFO | PerfReport: 1.28%, PulseSerializing -- 4.09e-01 / 4.59e-01, 7.35e-02 / 3.07e-02 | 20260505 12:17:44.450 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 12:17:44.450 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:17:44.450 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:17:44.450 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260505 12:17:44.450 | INFO | PerfReport: 22.48%, Upload -- 2.28e+01 / 0.00e+00, 8.10e+00 / 0.00e+00 | 20260505 12:17:44.450 | INFO | PerfReport: 22.01%, Download -- 2.23e+01 / 0.00e+00, 8.18e+00 / 0.00e+00 | 20260505 12:17:44.450 | INFO | PerfReport: 31.67%, Compute -- 3.21e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 12:17:44.450 | INFO | PerfReport: 2.12%, LockUpload -- 2.15e+00 / 0.00e+00, 7.87e+00 / 0.00e+00 | 20260505 12:17:44.450 | INFO | PerfReport: 1.43%, LockDownload -- 1.45e+00 / 0.00e+00, 3.01e+00 / 0.00e+00 | 20260505 12:17:44.450 | INFO | PerfReport: 20.27%, LockCompute -- 2.05e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260505 12:17:44.450 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.54e-02 / 0.00e+00 | 20260505 12:17:44.583 | INFO | Starting a round of Baz IO | 20260505 12:17:47.814 | INFO | Populating RTMetrics report | 20260505 12:17:47.814 | INFO | RTMetrics report populated | 20260505 12:17:47.814 | INFO | Opening tmp RTMetrics json file | 20260505 12:17:47.817 | INFO | Serializing RTMetrics to json | 20260505 12:17:47.841 | INFO | Serialization done, swapping tmp file now... | 20260505 12:17:47.864 | INFO | RTMetrics done | 20260505 12:17:47.933 | INFO | Finished a round of Baz IO: Wrote 2.345 GiB in 3.35 seconds | 20260505 12:17:47.933 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.66% / 0.01% / 76.33% | 20260505 12:17:47.933 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.923185% / 1.19209e-05% / 9.53674e-05% | 20260505 12:17:47.933 | INFO | Baz IO overall: | 20260505 12:17:47.933 | INFO | BytesWritten : 1868103763643 | 20260505 12:17:47.933 | INFO | NumEvents : 889262455895 | 20260505 12:17:49.581 | INFO | Analyzing chunk frames = [3145728,3146240) | 20260505 12:17:49.581 | INFO | MemInfo: { | 20260505 12:17:49.581 | INFO | "gpu" : " 17 GB", | 20260505 12:17:49.581 | INFO | "huge" : "", | 20260505 12:17:49.581 | INFO | "huge_shared" : " 27 GB", | 20260505 12:17:49.581 | INFO | "malloc" : " 14 GB", | 20260505 12:17:49.581 | INFO | "pinned" : " 29 GB" | 20260505 12:17:49.581 | INFO | } | 20260505 12:17:49.581 | INFO | Perf report for the last 5.13314 seconds: | 20260505 12:17:49.582 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:17:49.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:49.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:49.582 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:49.582 | INFO | ANALYSIS : 64.5%, 2.9, 99.5ms, 0 | 20260505 12:17:49.582 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:49.582 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260505 12:17:49.582 | INFO | PRE_HQ : 61.2%, 1.8, 58.2ms, 0 | 20260505 12:17:49.582 | INFO | BAZWRITER : 64.6%, 4.1, 135.9ms, 0 | 20260505 12:17:49.582 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260505 12:17:49.582 | INFO | Uploaded 36GiB to GPU in 2.18618s (16.4671GiB/s) | 20260505 12:17:49.582 | INFO | Downloaded 38GiB to GPU in 2.27917s (16.6727GiB/s) | 20260505 12:17:54.714 | INFO | Analyzing chunk frames = [3146240,3146752) | 20260505 12:17:54.714 | INFO | MemInfo: { | 20260505 12:17:54.714 | INFO | "gpu" : " 17 GB", | 20260505 12:17:54.714 | INFO | "huge" : "", | 20260505 12:17:54.714 | INFO | "huge_shared" : " 27 GB", | 20260505 12:17:54.714 | INFO | "malloc" : " 14 GB", | 20260505 12:17:54.714 | INFO | "pinned" : " 29 GB" | 20260505 12:17:54.714 | INFO | } | 20260505 12:17:54.714 | INFO | Perf report for the last 5.13271 seconds: | 20260505 12:17:54.714 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:17:54.714 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:54.714 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:54.714 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:54.714 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 12:17:54.714 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:54.714 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:54.714 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:17:54.714 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:54.714 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:54.714 | INFO | Uploaded 36GiB to GPU in 2.1826s (16.4941GiB/s) | 20260505 12:17:54.715 | INFO | Downloaded 34GiB to GPU in 2.07247s (16.4055GiB/s) | 20260505 12:17:59.865 | INFO | Analyzing chunk frames = [3146752,3147264) | 20260505 12:17:59.865 | INFO | MemInfo: { | 20260505 12:17:59.865 | INFO | "gpu" : " 17 GB", | 20260505 12:17:59.865 | INFO | "huge" : "", | 20260505 12:17:59.865 | INFO | "huge_shared" : " 27 GB", | 20260505 12:17:59.865 | INFO | "malloc" : " 14 GB", | 20260505 12:17:59.865 | INFO | "pinned" : " 29 GB" | 20260505 12:17:59.865 | INFO | } | 20260505 12:17:59.865 | INFO | Perf report for the last 5.15056 seconds: | 20260505 12:17:59.865 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:17:59.865 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:59.865 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:59.865 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:59.865 | INFO | ANALYSIS : 63.5%, 3.0, 99.0ms, 0 | 20260505 12:17:59.865 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:59.865 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:59.865 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:17:59.865 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:59.865 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:17:59.865 | INFO | Uploaded 36GiB to GPU in 2.22284s (16.1955GiB/s) | 20260505 12:17:59.865 | INFO | Downloaded 34GiB to GPU in 2.1317s (15.9497GiB/s) | 20260505 12:18:05.001 | INFO | Analyzing chunk frames = [3147264,3147776) | 20260505 12:18:05.001 | INFO | MemInfo: { | 20260505 12:18:05.001 | INFO | "gpu" : " 17 GB", | 20260505 12:18:05.001 | INFO | "huge" : "", | 20260505 12:18:05.001 | INFO | "huge_shared" : " 27 GB", | 20260505 12:18:05.001 | INFO | "malloc" : " 14 GB", | 20260505 12:18:05.001 | INFO | "pinned" : " 29 GB" | 20260505 12:18:05.001 | INFO | } | 20260505 12:18:05.002 | INFO | Perf report for the last 5.13678 seconds: | 20260505 12:18:05.002 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:18:05.002 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:05.002 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:05.002 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:05.002 | INFO | ANALYSIS : 63.1%, 3.0, 98.0ms, 0 | 20260505 12:18:05.002 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:05.002 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:05.002 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 12:18:05.002 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:05.002 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:05.002 | INFO | Uploaded 36GiB to GPU in 2.20444s (16.3307GiB/s) | 20260505 12:18:05.002 | INFO | Downloaded 34GiB to GPU in 2.13212s (15.9466GiB/s) | 20260505 12:18:10.141 | INFO | Analyzing chunk frames = [3147776,3148288) | 20260505 12:18:10.141 | INFO | MemInfo: { | 20260505 12:18:10.141 | INFO | "gpu" : " 17 GB", | 20260505 12:18:10.141 | INFO | "huge" : "", | 20260505 12:18:10.141 | INFO | "huge_shared" : " 27 GB", | 20260505 12:18:10.141 | INFO | "malloc" : " 14 GB", | 20260505 12:18:10.141 | INFO | "pinned" : " 29 GB" | 20260505 12:18:10.141 | INFO | } | 20260505 12:18:10.141 | INFO | Perf report for the last 5.13952 seconds: | 20260505 12:18:10.141 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:18:10.141 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:10.141 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:10.141 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:10.141 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 12:18:10.141 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:10.141 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:10.141 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 12:18:10.141 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:10.141 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:10.141 | INFO | Uploaded 36GiB to GPU in 2.15623s (16.6958GiB/s) | 20260505 12:18:10.141 | INFO | Downloaded 34GiB to GPU in 2.12493s (16.0006GiB/s) | 20260505 12:18:15.280 | INFO | Analyzing chunk frames = [3148288,3148800) | 20260505 12:18:15.280 | INFO | MemInfo: { | 20260505 12:18:15.280 | INFO | "gpu" : " 17 GB", | 20260505 12:18:15.281 | INFO | "huge" : "", | 20260505 12:18:15.281 | INFO | "huge_shared" : " 27 GB", | 20260505 12:18:15.281 | INFO | "malloc" : " 14 GB", | 20260505 12:18:15.281 | INFO | "pinned" : " 29 GB" | 20260505 12:18:15.281 | INFO | } | 20260505 12:18:15.281 | INFO | Perf report for the last 5.13947 seconds: | 20260505 12:18:15.281 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:18:15.281 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:15.281 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:15.281 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:15.281 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 12:18:15.281 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:15.281 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:15.281 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 12:18:15.281 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:15.281 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:15.281 | INFO | Uploaded 36GiB to GPU in 2.15584s (16.6988GiB/s) | 20260505 12:18:15.281 | INFO | Downloaded 34GiB to GPU in 2.10931s (16.119GiB/s) | 20260505 12:18:20.413 | INFO | Analyzing chunk frames = [3148800,3149312) | 20260505 12:18:20.413 | INFO | MemInfo: { | 20260505 12:18:20.413 | INFO | "gpu" : " 17 GB", | 20260505 12:18:20.413 | INFO | "huge" : "", | 20260505 12:18:20.413 | INFO | "huge_shared" : " 27 GB", | 20260505 12:18:20.413 | INFO | "malloc" : " 14 GB", | 20260505 12:18:20.413 | INFO | "pinned" : " 29 GB" | 20260505 12:18:20.413 | INFO | } | 20260505 12:18:20.413 | INFO | Perf report for the last 5.13278 seconds: | 20260505 12:18:20.413 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:18:20.413 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:20.413 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:20.413 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:20.414 | INFO | ANALYSIS : 63.4%, 3.0, 98.8ms, 0 | 20260505 12:18:20.414 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:20.414 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:20.414 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 12:18:20.414 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:20.414 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:20.414 | INFO | Uploaded 36GiB to GPU in 2.17106s (16.5817GiB/s) | 20260505 12:18:20.414 | INFO | Downloaded 34GiB to GPU in 2.13995s (15.8883GiB/s) | 20260505 12:18:25.556 | INFO | Analyzing chunk frames = [3149312,3149824) | 20260505 12:18:25.556 | INFO | MemInfo: { | 20260505 12:18:25.556 | INFO | "gpu" : " 17 GB", | 20260505 12:18:25.556 | INFO | "huge" : "", | 20260505 12:18:25.556 | INFO | "huge_shared" : " 27 GB", | 20260505 12:18:25.556 | INFO | "malloc" : " 14 GB", | 20260505 12:18:25.556 | INFO | "pinned" : " 29 GB" | 20260505 12:18:25.556 | INFO | } | 20260505 12:18:25.556 | INFO | Perf report for the last 5.14305 seconds: | 20260505 12:18:25.557 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:18:25.557 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:25.557 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:25.557 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:25.557 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 12:18:25.557 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:25.557 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:25.557 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:18:25.557 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:25.557 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:25.557 | INFO | Uploaded 36GiB to GPU in 2.199s (16.3711GiB/s) | 20260505 12:18:25.557 | INFO | Downloaded 34GiB to GPU in 2.11627s (16.066GiB/s) | 20260505 12:18:25.690 | INFO | Starting a round of Baz IO | 20260505 12:18:28.935 | INFO | Populating RTMetrics report | 20260505 12:18:28.936 | INFO | RTMetrics report populated | 20260505 12:18:28.936 | INFO | Opening tmp RTMetrics json file | 20260505 12:18:28.936 | INFO | Serializing RTMetrics to json | 20260505 12:18:28.938 | INFO | Serialization done, swapping tmp file now... | 20260505 12:18:28.938 | INFO | RTMetrics done | 20260505 12:18:29.055 | INFO | Finished a round of Baz IO: Wrote 2.347 GiB in 3.364 seconds | 20260505 12:18:29.055 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.65% / 0.01% / 76.34% | 20260505 12:18:29.055 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.92417% / 0% / 7.15256e-05% | 20260505 12:18:29.055 | INFO | Baz IO overall: | 20260505 12:18:29.055 | INFO | BytesWritten : 1870623585584 | 20260505 12:18:29.055 | INFO | NumEvents : 890458787423 | 20260505 12:18:30.702 | INFO | Analyzing chunk frames = [3149824,3150336) | 20260505 12:18:30.702 | INFO | MemInfo: { | 20260505 12:18:30.702 | INFO | "gpu" : " 17 GB", | 20260505 12:18:30.702 | INFO | "huge" : "", | 20260505 12:18:30.702 | INFO | "huge_shared" : " 27 GB", | 20260505 12:18:30.702 | INFO | "malloc" : " 14 GB", | 20260505 12:18:30.702 | INFO | "pinned" : " 29 GB" | 20260505 12:18:30.702 | INFO | } | 20260505 12:18:30.702 | INFO | Perf report for the last 5.14566 seconds: | 20260505 12:18:30.702 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:18:30.702 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:30.702 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:30.702 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:30.702 | INFO | ANALYSIS : 64.5%, 2.9, 99.8ms, 0 | 20260505 12:18:30.702 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:30.702 | INFO | RT_METRICS : 45.2%, 1.0, 23.8ms, 0 | 20260505 12:18:30.702 | INFO | PRE_HQ : 60.7%, 1.8, 58.2ms, 0 | 20260505 12:18:30.702 | INFO | BAZWRITER : 64.7%, 4.1, 135.5ms, 0 | 20260505 12:18:30.702 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 12:18:30.702 | INFO | Uploaded 36GiB to GPU in 2.17684s (16.5377GiB/s) | 20260505 12:18:30.702 | INFO | Downloaded 38GiB to GPU in 2.28384s (16.6386GiB/s) | 20260505 12:18:35.828 | INFO | Analyzing chunk frames = [3150336,3150848) | 20260505 12:18:35.828 | INFO | MemInfo: { | 20260505 12:18:35.828 | INFO | "gpu" : " 17 GB", | 20260505 12:18:35.828 | INFO | "huge" : "", | 20260505 12:18:35.828 | INFO | "huge_shared" : " 27 GB", | 20260505 12:18:35.828 | INFO | "malloc" : " 14 GB", | 20260505 12:18:35.828 | INFO | "pinned" : " 29 GB" | 20260505 12:18:35.828 | INFO | } | 20260505 12:18:35.828 | INFO | Perf report for the last 5.12628 seconds: | 20260505 12:18:35.828 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:18:35.828 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:35.828 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:35.828 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:35.828 | INFO | ANALYSIS : 63.9%, 3.0, 99.5ms, 0 | 20260505 12:18:35.828 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:35.828 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:35.828 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:18:35.828 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:35.829 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:35.829 | INFO | Uploaded 36GiB to GPU in 2.17803s (16.5287GiB/s) | 20260505 12:18:35.829 | INFO | Downloaded 34GiB to GPU in 2.04792s (16.6022GiB/s) | 20260505 12:18:40.968 | INFO | Analyzing chunk frames = [3150848,3151360) | 20260505 12:18:40.968 | INFO | MemInfo: { | 20260505 12:18:40.968 | INFO | "gpu" : " 17 GB", | 20260505 12:18:40.968 | INFO | "huge" : "", | 20260505 12:18:40.968 | INFO | "huge_shared" : " 27 GB", | 20260505 12:18:40.968 | INFO | "malloc" : " 14 GB", | 20260505 12:18:40.968 | INFO | "pinned" : " 29 GB" | 20260505 12:18:40.968 | INFO | } | 20260505 12:18:40.968 | INFO | Perf report for the last 5.13987 seconds: | 20260505 12:18:40.968 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:18:40.968 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:40.968 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:40.968 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:40.968 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260505 12:18:40.968 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:40.968 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:40.968 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:18:40.968 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:40.968 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:40.968 | INFO | Uploaded 36GiB to GPU in 2.15399s (16.7131GiB/s) | 20260505 12:18:40.969 | INFO | Downloaded 34GiB to GPU in 2.06329s (16.4786GiB/s) | 20260505 12:18:46.112 | INFO | Analyzing chunk frames = [3151360,3151872) | 20260505 12:18:46.112 | INFO | MemInfo: { | 20260505 12:18:46.112 | INFO | "gpu" : " 17 GB", | 20260505 12:18:46.112 | INFO | "huge" : "", | 20260505 12:18:46.112 | INFO | "huge_shared" : " 27 GB", | 20260505 12:18:46.112 | INFO | "malloc" : " 14 GB", | 20260505 12:18:46.112 | INFO | "pinned" : " 29 GB" | 20260505 12:18:46.112 | INFO | } | 20260505 12:18:46.112 | INFO | Perf report for the last 5.14355 seconds: | 20260505 12:18:46.112 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:18:46.112 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:46.112 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:46.112 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:46.112 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260505 12:18:46.112 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:46.112 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:46.112 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 12:18:46.112 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:46.112 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:46.112 | INFO | Uploaded 36GiB to GPU in 2.21895s (16.2239GiB/s) | 20260505 12:18:46.112 | INFO | Downloaded 34GiB to GPU in 2.13359s (15.9356GiB/s) | 20260505 12:18:51.259 | INFO | Analyzing chunk frames = [3151872,3152384) | 20260505 12:18:51.259 | INFO | MemInfo: { | 20260505 12:18:51.259 | INFO | "gpu" : " 17 GB", | 20260505 12:18:51.259 | INFO | "huge" : "", | 20260505 12:18:51.259 | INFO | "huge_shared" : " 27 GB", | 20260505 12:18:51.259 | INFO | "malloc" : " 14 GB", | 20260505 12:18:51.259 | INFO | "pinned" : " 29 GB" | 20260505 12:18:51.259 | INFO | } | 20260505 12:18:51.259 | INFO | Perf report for the last 5.14717 seconds: | 20260505 12:18:51.259 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:18:51.259 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:51.259 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:51.259 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:51.259 | INFO | ANALYSIS : 62.8%, 3.0, 98.3ms, 0 | 20260505 12:18:51.259 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:51.259 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:51.259 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:18:51.259 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:51.259 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:51.259 | INFO | Uploaded 36GiB to GPU in 2.15937s (16.6715GiB/s) | 20260505 12:18:51.259 | INFO | Downloaded 34GiB to GPU in 2.12285s (16.0162GiB/s) | 20260505 12:18:56.393 | INFO | Analyzing chunk frames = [3152384,3152896) | 20260505 12:18:56.393 | INFO | MemInfo: { | 20260505 12:18:56.393 | INFO | "gpu" : " 17 GB", | 20260505 12:18:56.393 | INFO | "huge" : "", | 20260505 12:18:56.393 | INFO | "huge_shared" : " 27 GB", | 20260505 12:18:56.393 | INFO | "malloc" : " 14 GB", | 20260505 12:18:56.394 | INFO | "pinned" : " 29 GB" | 20260505 12:18:56.394 | INFO | } | 20260505 12:18:56.394 | INFO | Perf report for the last 5.13468 seconds: | 20260505 12:18:56.394 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:18:56.394 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:56.394 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:56.394 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:56.394 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 12:18:56.394 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:56.394 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:56.394 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 12:18:56.394 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:56.394 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:18:56.394 | INFO | Uploaded 36GiB to GPU in 2.14637s (16.7725GiB/s) | 20260505 12:18:56.394 | INFO | Downloaded 34GiB to GPU in 2.10569s (16.1468GiB/s) | 20260505 12:19:01.527 | INFO | Analyzing chunk frames = [3152896,3153408) | 20260505 12:19:01.527 | INFO | MemInfo: { | 20260505 12:19:01.527 | INFO | "gpu" : " 17 GB", | 20260505 12:19:01.527 | INFO | "huge" : "", | 20260505 12:19:01.527 | INFO | "huge_shared" : " 27 GB", | 20260505 12:19:01.527 | INFO | "malloc" : " 14 GB", | 20260505 12:19:01.527 | INFO | "pinned" : " 29 GB" | 20260505 12:19:01.527 | INFO | } | 20260505 12:19:01.527 | INFO | Perf report for the last 5.13378 seconds: | 20260505 12:19:01.528 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:19:01.528 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:01.528 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:01.528 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:01.528 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 12:19:01.528 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:01.528 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:01.528 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260505 12:19:01.528 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:01.528 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:01.528 | INFO | Uploaded 36GiB to GPU in 2.16126s (16.657GiB/s) | 20260505 12:19:01.528 | INFO | Downloaded 34GiB to GPU in 2.14571s (15.8455GiB/s) | 20260505 12:19:06.668 | INFO | Analyzing chunk frames = [3153408,3153920) | 20260505 12:19:06.668 | INFO | MemInfo: { | 20260505 12:19:06.668 | INFO | "gpu" : " 17 GB", | 20260505 12:19:06.668 | INFO | "huge" : "", | 20260505 12:19:06.668 | INFO | "huge_shared" : " 27 GB", | 20260505 12:19:06.668 | INFO | "malloc" : " 14 GB", | 20260505 12:19:06.668 | INFO | "pinned" : " 29 GB" | 20260505 12:19:06.668 | INFO | } | 20260505 12:19:06.668 | INFO | Perf report for the last 5.14088 seconds: | 20260505 12:19:06.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:19:06.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:06.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:06.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:06.668 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 12:19:06.668 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:06.668 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:06.668 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 12:19:06.668 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:06.669 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:06.669 | INFO | Uploaded 36GiB to GPU in 2.22618s (16.1712GiB/s) | 20260505 12:19:06.669 | INFO | Downloaded 34GiB to GPU in 2.09933s (16.1956GiB/s) | 20260505 12:19:06.669 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 12:19:06.669 | INFO | Total mem usage: 15.3 GiB | 20260505 12:19:06.669 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 12:19:06.669 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -6.88%) | 20260505 12:19:06.669 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 12:19:06.669 | INFO | High Water Breakdown: | 20260505 12:19:06.669 | INFO | 61.1% : 8.77 GiB ( 92.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 12:19:06.669 | INFO | 38.7% : 5.55 GiB ( 83.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 12:19:06.669 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 12:19:06.669 | INFO | ------------------------------------------ | 20260505 12:19:06.669 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:19:06.669 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 12:19:06.669 | INFO | Total mem usage: 26.5 GiB | 20260505 12:19:06.669 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 12:19:06.669 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 12:19:06.669 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 12:19:06.669 | INFO | High Water Breakdown: | 20260505 12:19:06.669 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:19:06.669 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:19:06.669 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:19:06.669 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:19:06.669 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:19:06.669 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:19:06.669 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 12:19:06.669 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 12:19:06.669 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:19:06.669 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 12:19:06.669 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 12:19:06.669 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 12:19:06.669 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 12:19:06.669 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 12:19:06.669 | INFO | ------------------------------------------ | 20260505 12:19:06.669 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:19:06.669 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 12:19:06.669 | INFO | Total mem usage: 25.6 GiB | 20260505 12:19:06.669 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 12:19:06.669 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 12:19:06.669 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 12:19:06.669 | INFO | High Water Breakdown: | 20260505 12:19:06.669 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 12:19:06.669 | INFO | ------------------------------------------ | 20260505 12:19:06.669 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:19:06.669 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 12:19:06.669 | INFO | Total mem usage: 15.7 GiB | 20260505 12:19:06.669 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 12:19:06.669 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 12:19:06.669 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 12:19:06.669 | INFO | High Water Breakdown: | 20260505 12:19:06.669 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 12:19:06.670 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 12:19:06.670 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 12:19:06.670 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 12:19:06.670 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 12:19:06.670 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 12:19:06.670 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 12:19:06.670 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 12:19:06.670 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 12:19:06.670 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:19:06.670 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 12:19:06.670 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:19:06.670 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 12:19:06.670 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 12:19:06.670 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 12:19:06.670 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 12:19:06.670 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:19:06.670 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 12:19:06.670 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:19:06.670 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:19:06.670 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:19:06.670 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 12:19:06.670 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 12:19:06.670 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:19:06.670 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 12:19:06.670 | INFO | ------------------------------------------ | 20260505 12:19:06.670 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:19:06.670 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 12:19:06.670 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:19:06.670 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:19:06.670 | INFO | PerfReport: Full Execution -- 3.21e+01 / 7.71e+01, 1.37e+01 / 1.83e+00 | 20260505 12:19:06.670 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 1.01e-01, 1.07e-02 / 4.20e-03 | 20260505 12:19:06.670 | INFO | PerfReport: 11.74%, Baseline -- 3.77e+00 / 4.48e+00, 5.96e-01 / 2.13e-01 | 20260505 12:19:06.670 | INFO | PerfReport: 3.31%, Histogram -- 1.06e+00 / 1.49e+00, 2.04e-01 / 9.63e-02 | 20260505 12:19:06.670 | INFO | PerfReport: 15.82%, DME -- 4.06e+01 / 4.53e+01, 4.47e+00 / 2.17e+00 | 20260505 12:19:06.670 | INFO | PerfReport: 56.57%, FrameLabeling -- 1.82e+01 / 2.09e+01, 1.66e+00 / 1.78e+00 | 20260505 12:19:06.670 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.37e+00 / 2.90e+00, 4.28e-01 / 2.31e-01 | 20260505 12:19:06.670 | INFO | PerfReport: 3.63%, Metrics -- 1.16e+00 / 1.48e+00, 3.28e-01 / 1.79e-01 | 20260505 12:19:06.670 | INFO | PerfReport: 1.28%, PulseSerializing -- 4.10e-01 / 4.57e-01, 7.48e-02 / 3.06e-02 | 20260505 12:19:06.670 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 12:19:06.670 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:19:06.670 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:19:06.670 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260505 12:19:06.670 | INFO | PerfReport: 22.50%, Upload -- 2.27e+01 / 0.00e+00, 8.05e+00 / 0.00e+00 | 20260505 12:19:06.670 | INFO | PerfReport: 21.99%, Download -- 2.22e+01 / 0.00e+00, 8.08e+00 / 0.00e+00 | 20260505 12:19:06.670 | INFO | PerfReport: 31.81%, Compute -- 3.21e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 12:19:06.670 | INFO | PerfReport: 1.98%, LockUpload -- 2.00e+00 / 0.00e+00, 7.51e+00 / 0.00e+00 | 20260505 12:19:06.670 | INFO | PerfReport: 1.34%, LockDownload -- 1.35e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260505 12:19:06.670 | INFO | PerfReport: 20.37%, LockCompute -- 2.06e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260505 12:19:06.670 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 2.18e-02 / 0.00e+00 | 20260505 12:19:06.797 | INFO | Starting a round of Baz IO | 20260505 12:19:10.045 | INFO | Populating RTMetrics report | 20260505 12:19:10.045 | INFO | RTMetrics report populated | 20260505 12:19:10.045 | INFO | Opening tmp RTMetrics json file | 20260505 12:19:10.074 | INFO | Serializing RTMetrics to json | 20260505 12:19:10.112 | INFO | Serialization done, swapping tmp file now... | 20260505 12:19:10.118 | INFO | RTMetrics done | 20260505 12:19:10.152 | INFO | Finished a round of Baz IO: Wrote 2.345 GiB in 3.354 seconds | 20260505 12:19:10.152 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.65% / 0.01% / 76.35% | 20260505 12:19:10.152 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.920777% / 1.19209e-05% / 7.94729e-05% | 20260505 12:19:10.152 | INFO | Baz IO overall: | 20260505 12:19:10.152 | INFO | BytesWritten : 1873141388991 | 20260505 12:19:10.152 | INFO | NumEvents : 891654077031 | 20260505 12:19:11.812 | INFO | Analyzing chunk frames = [3153920,3154432) | 20260505 12:19:11.812 | INFO | MemInfo: { | 20260505 12:19:11.812 | INFO | "gpu" : " 17 GB", | 20260505 12:19:11.812 | INFO | "huge" : "", | 20260505 12:19:11.812 | INFO | "huge_shared" : " 27 GB", | 20260505 12:19:11.812 | INFO | "malloc" : " 14 GB", | 20260505 12:19:11.812 | INFO | "pinned" : " 29 GB" | 20260505 12:19:11.812 | INFO | } | 20260505 12:19:11.812 | INFO | Perf report for the last 5.14331 seconds: | 20260505 12:19:11.812 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:19:11.812 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:11.812 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:11.812 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:11.812 | INFO | ANALYSIS : 64.5%, 2.9, 99.8ms, 0 | 20260505 12:19:11.812 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:11.812 | INFO | RT_METRICS : 46.6%, 1.0, 24.4ms, 0 | 20260505 12:19:11.812 | INFO | PRE_HQ : 60.9%, 1.8, 58.2ms, 0 | 20260505 12:19:11.812 | INFO | BAZWRITER : 64.5%, 4.1, 135.1ms, 0 | 20260505 12:19:11.812 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 12:19:11.812 | INFO | Uploaded 36GiB to GPU in 2.18213s (16.4977GiB/s) | 20260505 12:19:11.812 | INFO | Downloaded 38GiB to GPU in 2.27823s (16.6796GiB/s) | 20260505 12:19:16.956 | INFO | Analyzing chunk frames = [3154432,3154944) | 20260505 12:19:16.956 | INFO | MemInfo: { | 20260505 12:19:16.956 | INFO | "gpu" : " 17 GB", | 20260505 12:19:16.957 | INFO | "huge" : "", | 20260505 12:19:16.957 | INFO | "huge_shared" : " 27 GB", | 20260505 12:19:16.957 | INFO | "malloc" : " 14 GB", | 20260505 12:19:16.957 | INFO | "pinned" : " 29 GB" | 20260505 12:19:16.957 | INFO | } | 20260505 12:19:16.957 | INFO | Perf report for the last 5.1449 seconds: | 20260505 12:19:16.957 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:19:16.957 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:16.957 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:16.957 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:16.957 | INFO | ANALYSIS : 63.7%, 3.0, 99.6ms, 0 | 20260505 12:19:16.957 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:16.957 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:16.957 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:19:16.957 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:16.957 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:16.957 | INFO | Uploaded 36GiB to GPU in 2.20063s (16.3589GiB/s) | 20260505 12:19:16.957 | INFO | Downloaded 34GiB to GPU in 2.04437s (16.6311GiB/s) | 20260505 12:19:22.090 | INFO | Analyzing chunk frames = [3154944,3155456) | 20260505 12:19:22.090 | INFO | MemInfo: { | 20260505 12:19:22.090 | INFO | "gpu" : " 17 GB", | 20260505 12:19:22.090 | INFO | "huge" : "", | 20260505 12:19:22.090 | INFO | "huge_shared" : " 27 GB", | 20260505 12:19:22.090 | INFO | "malloc" : " 14 GB", | 20260505 12:19:22.090 | INFO | "pinned" : " 29 GB" | 20260505 12:19:22.090 | INFO | } | 20260505 12:19:22.090 | INFO | Perf report for the last 5.13369 seconds: | 20260505 12:19:22.090 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:19:22.090 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:22.090 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:22.090 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:22.090 | INFO | ANALYSIS : 63.4%, 3.0, 98.6ms, 0 | 20260505 12:19:22.090 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:22.090 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:22.090 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:19:22.090 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:22.090 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:22.091 | INFO | Uploaded 36GiB to GPU in 2.20075s (16.358GiB/s) | 20260505 12:19:22.091 | INFO | Downloaded 34GiB to GPU in 2.12034s (16.0352GiB/s) | 20260505 12:19:27.231 | INFO | Analyzing chunk frames = [3155456,3155968) | 20260505 12:19:27.231 | INFO | MemInfo: { | 20260505 12:19:27.231 | INFO | "gpu" : " 17 GB", | 20260505 12:19:27.231 | INFO | "huge" : "", | 20260505 12:19:27.231 | INFO | "huge_shared" : " 27 GB", | 20260505 12:19:27.231 | INFO | "malloc" : " 14 GB", | 20260505 12:19:27.231 | INFO | "pinned" : " 29 GB" | 20260505 12:19:27.231 | INFO | } | 20260505 12:19:27.231 | INFO | Perf report for the last 5.14047 seconds: | 20260505 12:19:27.231 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:19:27.231 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:27.231 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:27.231 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:27.231 | INFO | ANALYSIS : 63.1%, 3.0, 98.0ms, 0 | 20260505 12:19:27.231 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:27.231 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:27.231 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 12:19:27.231 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:27.231 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:27.231 | INFO | Uploaded 36GiB to GPU in 2.20369s (16.3363GiB/s) | 20260505 12:19:27.231 | INFO | Downloaded 34GiB to GPU in 2.13071s (15.9571GiB/s) | 20260505 12:19:32.374 | INFO | Analyzing chunk frames = [3155968,3156480) | 20260505 12:19:32.374 | INFO | MemInfo: { | 20260505 12:19:32.374 | INFO | "gpu" : " 17 GB", | 20260505 12:19:32.374 | INFO | "huge" : "", | 20260505 12:19:32.374 | INFO | "huge_shared" : " 27 GB", | 20260505 12:19:32.374 | INFO | "malloc" : " 14 GB", | 20260505 12:19:32.374 | INFO | "pinned" : " 29 GB" | 20260505 12:19:32.375 | INFO | } | 20260505 12:19:32.375 | INFO | Perf report for the last 5.1438 seconds: | 20260505 12:19:32.375 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:19:32.375 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:32.375 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:32.375 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:32.375 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 12:19:32.375 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:32.375 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:32.375 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:19:32.375 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:32.375 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:32.375 | INFO | Uploaded 36GiB to GPU in 2.1864s (16.4655GiB/s) | 20260505 12:19:32.375 | INFO | Downloaded 34GiB to GPU in 2.12862s (15.9728GiB/s) | 20260505 12:19:37.508 | INFO | Analyzing chunk frames = [3156480,3156992) | 20260505 12:19:37.508 | INFO | MemInfo: { | 20260505 12:19:37.508 | INFO | "gpu" : " 17 GB", | 20260505 12:19:37.508 | INFO | "huge" : "", | 20260505 12:19:37.508 | INFO | "huge_shared" : " 27 GB", | 20260505 12:19:37.508 | INFO | "malloc" : " 14 GB", | 20260505 12:19:37.508 | INFO | "pinned" : " 29 GB" | 20260505 12:19:37.508 | INFO | } | 20260505 12:19:37.509 | INFO | Perf report for the last 5.13388 seconds: | 20260505 12:19:37.509 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:19:37.509 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:37.509 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:37.509 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:37.509 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 12:19:37.509 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:37.509 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:37.509 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 12:19:37.509 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:37.509 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:37.509 | INFO | Uploaded 36GiB to GPU in 2.13289s (16.8785GiB/s) | 20260505 12:19:37.509 | INFO | Downloaded 34GiB to GPU in 2.10254s (16.1709GiB/s) | 20260505 12:19:42.650 | INFO | Analyzing chunk frames = [3156992,3157504) | 20260505 12:19:42.651 | INFO | MemInfo: { | 20260505 12:19:42.651 | INFO | "gpu" : " 17 GB", | 20260505 12:19:42.651 | INFO | "huge" : "", | 20260505 12:19:42.651 | INFO | "huge_shared" : " 27 GB", | 20260505 12:19:42.651 | INFO | "malloc" : " 14 GB", | 20260505 12:19:42.651 | INFO | "pinned" : " 29 GB" | 20260505 12:19:42.651 | INFO | } | 20260505 12:19:42.651 | INFO | Perf report for the last 5.14225 seconds: | 20260505 12:19:42.651 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:19:42.651 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:42.651 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:42.651 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:42.651 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 12:19:42.651 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:42.651 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:42.651 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 12:19:42.651 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:42.651 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:42.651 | INFO | Uploaded 36GiB to GPU in 2.14304s (16.7985GiB/s) | 20260505 12:19:42.651 | INFO | Downloaded 34GiB to GPU in 2.12725s (15.9831GiB/s) | 20260505 12:19:47.781 | INFO | Analyzing chunk frames = [3157504,3158016) | 20260505 12:19:47.781 | INFO | MemInfo: { | 20260505 12:19:47.781 | INFO | "gpu" : " 17 GB", | 20260505 12:19:47.781 | INFO | "huge" : "", | 20260505 12:19:47.781 | INFO | "huge_shared" : " 27 GB", | 20260505 12:19:47.781 | INFO | "malloc" : " 14 GB", | 20260505 12:19:47.781 | INFO | "pinned" : " 29 GB" | 20260505 12:19:47.781 | INFO | } | 20260505 12:19:47.781 | INFO | Perf report for the last 5.13017 seconds: | 20260505 12:19:47.781 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:19:47.781 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:47.781 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:47.781 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:47.781 | INFO | ANALYSIS : 63.9%, 3.0, 99.6ms, 0 | 20260505 12:19:47.781 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:47.781 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:47.781 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 12:19:47.781 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:47.781 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:47.781 | INFO | Uploaded 36GiB to GPU in 2.21301s (16.2674GiB/s) | 20260505 12:19:47.782 | INFO | Downloaded 34GiB to GPU in 2.10689s (16.1376GiB/s) | 20260505 12:19:47.916 | INFO | Starting a round of Baz IO | 20260505 12:19:51.161 | INFO | Populating RTMetrics report | 20260505 12:19:51.162 | INFO | RTMetrics report populated | 20260505 12:19:51.162 | INFO | Opening tmp RTMetrics json file | 20260505 12:19:51.181 | INFO | Serializing RTMetrics to json | 20260505 12:19:51.192 | INFO | Serialization done, swapping tmp file now... | 20260505 12:19:51.208 | INFO | RTMetrics done | 20260505 12:19:51.270 | INFO | Finished a round of Baz IO: Wrote 2.342 GiB in 3.353 seconds | 20260505 12:19:51.270 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.64% / 0.01% / 76.35% | 20260505 12:19:51.270 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.912686% / 1.19209e-05% / 9.13938e-05% | 20260505 12:19:51.270 | INFO | Baz IO overall: | 20260505 12:19:51.270 | INFO | BytesWritten : 1875656501795 | 20260505 12:19:51.270 | INFO | NumEvents : 892847833233 | 20260505 12:19:52.919 | INFO | Analyzing chunk frames = [3158016,3158528) | 20260505 12:19:52.919 | INFO | MemInfo: { | 20260505 12:19:52.919 | INFO | "gpu" : " 17 GB", | 20260505 12:19:52.919 | INFO | "huge" : "", | 20260505 12:19:52.919 | INFO | "huge_shared" : " 27 GB", | 20260505 12:19:52.919 | INFO | "malloc" : " 14 GB", | 20260505 12:19:52.919 | INFO | "pinned" : " 29 GB" | 20260505 12:19:52.919 | INFO | } | 20260505 12:19:52.919 | INFO | Perf report for the last 5.1384 seconds: | 20260505 12:19:52.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:19:52.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:52.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:52.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:52.919 | INFO | ANALYSIS : 64.7%, 2.9, 99.9ms, 0 | 20260505 12:19:52.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:52.919 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 12:19:52.919 | INFO | PRE_HQ : 60.9%, 1.8, 58.0ms, 0 | 20260505 12:19:52.919 | INFO | BAZWRITER : 64.6%, 4.1, 134.8ms, 0 | 20260505 12:19:52.920 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 12:19:52.920 | INFO | Uploaded 36GiB to GPU in 2.18319s (16.4897GiB/s) | 20260505 12:19:52.920 | INFO | Downloaded 38GiB to GPU in 2.28645s (16.6197GiB/s) | 20260505 12:19:58.063 | INFO | Analyzing chunk frames = [3158528,3159040) | 20260505 12:19:58.063 | INFO | MemInfo: { | 20260505 12:19:58.063 | INFO | "gpu" : " 17 GB", | 20260505 12:19:58.063 | INFO | "huge" : "", | 20260505 12:19:58.063 | INFO | "huge_shared" : " 27 GB", | 20260505 12:19:58.063 | INFO | "malloc" : " 14 GB", | 20260505 12:19:58.063 | INFO | "pinned" : " 29 GB" | 20260505 12:19:58.063 | INFO | } | 20260505 12:19:58.063 | INFO | Perf report for the last 5.14397 seconds: | 20260505 12:19:58.063 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:19:58.063 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:58.063 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:58.063 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:58.063 | INFO | ANALYSIS : 64.0%, 3.0, 100.1ms, 0 | 20260505 12:19:58.063 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:58.063 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:58.063 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:19:58.063 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:58.063 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:19:58.064 | INFO | Uploaded 36GiB to GPU in 2.23286s (16.1228GiB/s) | 20260505 12:19:58.064 | INFO | Downloaded 34GiB to GPU in 2.09395s (16.2373GiB/s) | 20260505 12:20:03.211 | INFO | Analyzing chunk frames = [3159040,3159552) | 20260505 12:20:03.211 | INFO | MemInfo: { | 20260505 12:20:03.211 | INFO | "gpu" : " 17 GB", | 20260505 12:20:03.211 | INFO | "huge" : "", | 20260505 12:20:03.211 | INFO | "huge_shared" : " 27 GB", | 20260505 12:20:03.212 | INFO | "malloc" : " 14 GB", | 20260505 12:20:03.212 | INFO | "pinned" : " 29 GB" | 20260505 12:20:03.212 | INFO | } | 20260505 12:20:03.212 | INFO | Perf report for the last 5.14833 seconds: | 20260505 12:20:03.212 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:20:03.212 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:03.212 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:03.212 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:03.212 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 12:20:03.212 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:03.212 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:03.212 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:20:03.212 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:03.212 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:03.212 | INFO | Uploaded 36GiB to GPU in 2.20346s (16.3379GiB/s) | 20260505 12:20:03.212 | INFO | Downloaded 34GiB to GPU in 2.1312s (15.9534GiB/s) | 20260505 12:20:08.339 | INFO | Analyzing chunk frames = [3159552,3160064) | 20260505 12:20:08.339 | INFO | MemInfo: { | 20260505 12:20:08.339 | INFO | "gpu" : " 17 GB", | 20260505 12:20:08.339 | INFO | "huge" : "", | 20260505 12:20:08.339 | INFO | "huge_shared" : " 27 GB", | 20260505 12:20:08.339 | INFO | "malloc" : " 14 GB", | 20260505 12:20:08.339 | INFO | "pinned" : " 29 GB" | 20260505 12:20:08.339 | INFO | } | 20260505 12:20:08.339 | INFO | Perf report for the last 5.12719 seconds: | 20260505 12:20:08.339 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:20:08.339 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:08.339 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:08.339 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:08.339 | INFO | ANALYSIS : 63.2%, 3.0, 98.0ms, 0 | 20260505 12:20:08.339 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:08.339 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:08.339 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 12:20:08.339 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:08.339 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:08.339 | INFO | Uploaded 36GiB to GPU in 2.2004s (16.3607GiB/s) | 20260505 12:20:08.339 | INFO | Downloaded 34GiB to GPU in 2.13535s (15.9225GiB/s) | 20260505 12:20:13.476 | INFO | Analyzing chunk frames = [3160064,3160576) | 20260505 12:20:13.476 | INFO | MemInfo: { | 20260505 12:20:13.476 | INFO | "gpu" : " 17 GB", | 20260505 12:20:13.476 | INFO | "huge" : "", | 20260505 12:20:13.476 | INFO | "huge_shared" : " 27 GB", | 20260505 12:20:13.476 | INFO | "malloc" : " 14 GB", | 20260505 12:20:13.476 | INFO | "pinned" : " 29 GB" | 20260505 12:20:13.476 | INFO | } | 20260505 12:20:13.476 | INFO | Perf report for the last 5.13694 seconds: | 20260505 12:20:13.476 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:20:13.476 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:13.476 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:13.476 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:13.476 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 12:20:13.476 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:13.476 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:13.476 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 12:20:13.476 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:13.476 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:13.476 | INFO | Uploaded 36GiB to GPU in 2.19485s (16.402GiB/s) | 20260505 12:20:13.476 | INFO | Downloaded 34GiB to GPU in 2.12684s (15.9862GiB/s) | 20260505 12:20:18.623 | INFO | Analyzing chunk frames = [3160576,3161088) | 20260505 12:20:18.623 | INFO | MemInfo: { | 20260505 12:20:18.623 | INFO | "gpu" : " 17 GB", | 20260505 12:20:18.623 | INFO | "huge" : "", | 20260505 12:20:18.623 | INFO | "huge_shared" : " 27 GB", | 20260505 12:20:18.623 | INFO | "malloc" : " 14 GB", | 20260505 12:20:18.624 | INFO | "pinned" : " 29 GB" | 20260505 12:20:18.624 | INFO | } | 20260505 12:20:18.624 | INFO | Perf report for the last 5.14773 seconds: | 20260505 12:20:18.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:20:18.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:18.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:18.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:18.624 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 12:20:18.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:18.624 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:18.624 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 12:20:18.624 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:18.624 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:18.624 | INFO | Uploaded 36GiB to GPU in 2.16849s (16.6014GiB/s) | 20260505 12:20:18.624 | INFO | Downloaded 34GiB to GPU in 2.12039s (16.0348GiB/s) | 20260505 12:20:23.764 | INFO | Analyzing chunk frames = [3161088,3161600) | 20260505 12:20:23.764 | INFO | MemInfo: { | 20260505 12:20:23.764 | INFO | "gpu" : " 17 GB", | 20260505 12:20:23.764 | INFO | "huge" : "", | 20260505 12:20:23.764 | INFO | "huge_shared" : " 27 GB", | 20260505 12:20:23.764 | INFO | "malloc" : " 14 GB", | 20260505 12:20:23.764 | INFO | "pinned" : " 29 GB" | 20260505 12:20:23.764 | INFO | } | 20260505 12:20:23.765 | INFO | Perf report for the last 5.14096 seconds: | 20260505 12:20:23.765 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:20:23.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:23.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:23.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:23.765 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 12:20:23.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:23.765 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:23.765 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 12:20:23.765 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:23.765 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:23.765 | INFO | Uploaded 36GiB to GPU in 2.16441s (16.6327GiB/s) | 20260505 12:20:23.765 | INFO | Downloaded 34GiB to GPU in 2.15552s (15.7735GiB/s) | 20260505 12:20:28.903 | INFO | Analyzing chunk frames = [3161600,3162112) | 20260505 12:20:28.903 | INFO | MemInfo: { | 20260505 12:20:28.903 | INFO | "gpu" : " 17 GB", | 20260505 12:20:28.903 | INFO | "huge" : "", | 20260505 12:20:28.903 | INFO | "huge_shared" : " 27 GB", | 20260505 12:20:28.903 | INFO | "malloc" : " 14 GB", | 20260505 12:20:28.903 | INFO | "pinned" : " 29 GB" | 20260505 12:20:28.903 | INFO | } | 20260505 12:20:28.903 | INFO | Perf report for the last 5.13835 seconds: | 20260505 12:20:28.903 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:20:28.903 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:28.903 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:28.903 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:28.903 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 12:20:28.903 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:28.903 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:28.903 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:20:28.903 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:28.903 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:28.903 | INFO | Uploaded 36GiB to GPU in 2.18949s (16.4422GiB/s) | 20260505 12:20:28.903 | INFO | Downloaded 34GiB to GPU in 2.11252s (16.0945GiB/s) | 20260505 12:20:28.903 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 12:20:28.903 | INFO | Total mem usage: 15.3 GiB | 20260505 12:20:28.903 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 12:20:28.903 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -6.82%) | 20260505 12:20:28.903 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 12:20:28.903 | INFO | High Water Breakdown: | 20260505 12:20:28.903 | INFO | 61.1% : 8.77 GiB ( 92.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 12:20:28.903 | INFO | 38.8% : 5.56 GiB ( 83.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 12:20:28.903 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 12:20:28.903 | INFO | ------------------------------------------ | 20260505 12:20:28.903 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:20:28.904 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 12:20:28.904 | INFO | Total mem usage: 26.5 GiB | 20260505 12:20:28.904 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 12:20:28.904 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 12:20:28.904 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 12:20:28.904 | INFO | High Water Breakdown: | 20260505 12:20:28.904 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:20:28.904 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:20:28.904 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:20:28.904 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:20:28.904 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:20:28.904 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:20:28.904 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 12:20:28.904 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 12:20:28.904 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:20:28.904 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 12:20:28.904 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 12:20:28.904 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 12:20:28.904 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 12:20:28.904 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 12:20:28.904 | INFO | ------------------------------------------ | 20260505 12:20:28.904 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:20:28.904 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 12:20:28.904 | INFO | Total mem usage: 25.6 GiB | 20260505 12:20:28.904 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 12:20:28.904 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 12:20:28.904 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 12:20:28.904 | INFO | High Water Breakdown: | 20260505 12:20:28.904 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 12:20:28.904 | INFO | ------------------------------------------ | 20260505 12:20:28.904 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:20:28.904 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 12:20:28.904 | INFO | Total mem usage: 15.7 GiB | 20260505 12:20:28.904 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 12:20:28.904 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 12:20:28.904 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 12:20:28.904 | INFO | High Water Breakdown: | 20260505 12:20:28.904 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 12:20:28.904 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 12:20:28.904 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 12:20:28.904 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 12:20:28.904 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 12:20:28.904 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 12:20:28.904 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 12:20:28.904 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 12:20:28.904 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 12:20:28.904 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:20:28.904 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 12:20:28.904 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:20:28.904 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 12:20:28.904 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 12:20:28.904 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 12:20:28.904 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 12:20:28.904 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:20:28.904 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 12:20:28.904 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:20:28.904 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:20:28.904 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:20:28.904 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 12:20:28.904 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 12:20:28.904 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:20:28.904 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 12:20:28.904 | INFO | ------------------------------------------ | 20260505 12:20:28.904 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:20:28.904 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 12:20:28.904 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:20:28.904 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:20:28.904 | INFO | PerfReport: Full Execution -- 3.20e+01 / 7.74e+01, 1.39e+01 / 1.68e+00 | 20260505 12:20:28.904 | INFO | PerfReport: 0.29%, AggregateStats -- 9.26e-02 / 9.98e-02, 1.05e-02 / 3.12e-03 | 20260505 12:20:28.904 | INFO | PerfReport: 11.75%, Baseline -- 3.76e+00 / 4.49e+00, 6.00e-01 / 1.86e-01 | 20260505 12:20:28.904 | INFO | PerfReport: 3.30%, Histogram -- 1.06e+00 / 1.50e+00, 2.05e-01 / 9.57e-02 | 20260505 12:20:28.904 | INFO | PerfReport: 15.98%, DME -- 4.09e+01 / 4.61e+01, 4.78e+00 / 2.34e+00 | 20260505 12:20:28.904 | INFO | PerfReport: 56.32%, FrameLabeling -- 1.80e+01 / 2.03e+01, 1.47e+00 / 1.85e+00 | 20260505 12:20:28.904 | INFO | PerfReport: 7.43%, PulseAccumulating -- 2.38e+00 / 2.90e+00, 4.45e-01 / 2.39e-01 | 20260505 12:20:28.904 | INFO | PerfReport: 3.64%, Metrics -- 1.17e+00 / 1.50e+00, 3.31e-01 / 1.79e-01 | 20260505 12:20:28.904 | INFO | PerfReport: 1.28%, PulseSerializing -- 4.10e-01 / 4.65e-01, 7.49e-02 / 4.46e-02 | 20260505 12:20:28.904 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 12:20:28.904 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:20:28.904 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:20:28.904 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260505 12:20:28.904 | INFO | PerfReport: 22.55%, Upload -- 2.28e+01 / 0.00e+00, 8.13e+00 / 0.00e+00 | 20260505 12:20:28.905 | INFO | PerfReport: 22.03%, Download -- 2.23e+01 / 0.00e+00, 8.16e+00 / 0.00e+00 | 20260505 12:20:28.905 | INFO | PerfReport: 31.68%, Compute -- 3.20e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 12:20:28.905 | INFO | PerfReport: 2.19%, LockUpload -- 2.22e+00 / 0.00e+00, 7.80e+00 / 0.00e+00 | 20260505 12:20:28.905 | INFO | PerfReport: 1.34%, LockDownload -- 1.36e+00 / 0.00e+00, 2.94e+00 / 0.00e+00 | 20260505 12:20:28.905 | INFO | PerfReport: 20.19%, LockCompute -- 2.04e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260505 12:20:28.905 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 2.58e-02 / 0.00e+00 | 20260505 12:20:29.036 | INFO | Starting a round of Baz IO | 20260505 12:20:32.282 | INFO | Populating RTMetrics report | 20260505 12:20:32.283 | INFO | RTMetrics report populated | 20260505 12:20:32.283 | INFO | Opening tmp RTMetrics json file | 20260505 12:20:32.287 | INFO | Serializing RTMetrics to json | 20260505 12:20:32.288 | INFO | Serialization done, swapping tmp file now... | 20260505 12:20:32.289 | INFO | RTMetrics done | 20260505 12:20:32.405 | INFO | Finished a round of Baz IO: Wrote 2.341 GiB in 3.369 seconds | 20260505 12:20:32.405 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.64% / 0.01% / 76.36% | 20260505 12:20:32.405 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.91213% / 1.19209e-05% / 5.96046e-05% | 20260505 12:20:32.405 | INFO | Baz IO overall: | 20260505 12:20:32.405 | INFO | BytesWritten : 1878170296473 | 20260505 12:20:32.406 | INFO | NumEvents : 894040861784 | 20260505 12:20:34.038 | INFO | Analyzing chunk frames = [3162112,3162624) | 20260505 12:20:34.038 | INFO | MemInfo: { | 20260505 12:20:34.038 | INFO | "gpu" : " 17 GB", | 20260505 12:20:34.038 | INFO | "huge" : "", | 20260505 12:20:34.038 | INFO | "huge_shared" : " 27 GB", | 20260505 12:20:34.038 | INFO | "malloc" : " 14 GB", | 20260505 12:20:34.038 | INFO | "pinned" : " 29 GB" | 20260505 12:20:34.038 | INFO | } | 20260505 12:20:34.038 | INFO | Perf report for the last 5.1355 seconds: | 20260505 12:20:34.039 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:20:34.039 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:34.039 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:34.039 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:34.039 | INFO | ANALYSIS : 64.7%, 2.9, 99.8ms, 0 | 20260505 12:20:34.039 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:34.039 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260505 12:20:34.039 | INFO | PRE_HQ : 61.3%, 1.8, 58.4ms, 0 | 20260505 12:20:34.039 | INFO | BAZWRITER : 64.9%, 4.0, 135.0ms, 0 | 20260505 12:20:34.039 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 12:20:34.039 | INFO | Uploaded 36GiB to GPU in 2.18581s (16.4699GiB/s) | 20260505 12:20:34.039 | INFO | Downloaded 38GiB to GPU in 2.29081s (16.588GiB/s) | 20260505 12:20:39.184 | INFO | Analyzing chunk frames = [3162624,3163136) | 20260505 12:20:39.184 | INFO | MemInfo: { | 20260505 12:20:39.184 | INFO | "gpu" : " 17 GB", | 20260505 12:20:39.184 | INFO | "huge" : "", | 20260505 12:20:39.184 | INFO | "huge_shared" : " 27 GB", | 20260505 12:20:39.184 | INFO | "malloc" : " 14 GB", | 20260505 12:20:39.184 | INFO | "pinned" : " 29 GB" | 20260505 12:20:39.184 | INFO | } | 20260505 12:20:39.184 | INFO | Perf report for the last 5.1458 seconds: | 20260505 12:20:39.184 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:20:39.184 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:39.184 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:39.184 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:39.184 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 12:20:39.184 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:39.184 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:39.185 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 12:20:39.185 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:39.185 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:39.185 | INFO | Uploaded 36GiB to GPU in 2.23215s (16.1279GiB/s) | 20260505 12:20:39.185 | INFO | Downloaded 34GiB to GPU in 2.10272s (16.1695GiB/s) | 20260505 12:20:44.322 | INFO | Analyzing chunk frames = [3163136,3163648) | 20260505 12:20:44.322 | INFO | MemInfo: { | 20260505 12:20:44.322 | INFO | "gpu" : " 17 GB", | 20260505 12:20:44.322 | INFO | "huge" : "", | 20260505 12:20:44.322 | INFO | "huge_shared" : " 27 GB", | 20260505 12:20:44.322 | INFO | "malloc" : " 14 GB", | 20260505 12:20:44.322 | INFO | "pinned" : " 29 GB" | 20260505 12:20:44.322 | INFO | } | 20260505 12:20:44.322 | INFO | Perf report for the last 5.13782 seconds: | 20260505 12:20:44.322 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:20:44.322 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:44.322 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:44.322 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:44.322 | INFO | ANALYSIS : 63.4%, 3.0, 98.6ms, 0 | 20260505 12:20:44.322 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:44.322 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:44.322 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 12:20:44.322 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:44.322 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:44.322 | INFO | Uploaded 36GiB to GPU in 2.20281s (16.3428GiB/s) | 20260505 12:20:44.322 | INFO | Downloaded 34GiB to GPU in 2.10821s (16.1274GiB/s) | 20260505 12:20:49.458 | INFO | Analyzing chunk frames = [3163648,3164160) | 20260505 12:20:49.458 | INFO | MemInfo: { | 20260505 12:20:49.458 | INFO | "gpu" : " 17 GB", | 20260505 12:20:49.458 | INFO | "huge" : "", | 20260505 12:20:49.458 | INFO | "huge_shared" : " 27 GB", | 20260505 12:20:49.458 | INFO | "malloc" : " 14 GB", | 20260505 12:20:49.458 | INFO | "pinned" : " 29 GB" | 20260505 12:20:49.458 | INFO | } | 20260505 12:20:49.458 | INFO | Perf report for the last 5.13592 seconds: | 20260505 12:20:49.458 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:20:49.458 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:49.458 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:49.458 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:49.458 | INFO | ANALYSIS : 63.1%, 3.0, 98.0ms, 0 | 20260505 12:20:49.458 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:49.458 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:49.458 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:20:49.458 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:49.458 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:49.458 | INFO | Uploaded 36GiB to GPU in 2.20009s (16.363GiB/s) | 20260505 12:20:49.458 | INFO | Downloaded 34GiB to GPU in 2.13067s (15.9574GiB/s) | 20260505 12:20:54.599 | INFO | Analyzing chunk frames = [3164160,3164672) | 20260505 12:20:54.599 | INFO | MemInfo: { | 20260505 12:20:54.599 | INFO | "gpu" : " 17 GB", | 20260505 12:20:54.599 | INFO | "huge" : "", | 20260505 12:20:54.599 | INFO | "huge_shared" : " 27 GB", | 20260505 12:20:54.599 | INFO | "malloc" : " 14 GB", | 20260505 12:20:54.599 | INFO | "pinned" : " 29 GB" | 20260505 12:20:54.599 | INFO | } | 20260505 12:20:54.599 | INFO | Perf report for the last 5.14137 seconds: | 20260505 12:20:54.599 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:20:54.599 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:54.599 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:54.599 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:54.600 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 12:20:54.600 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:54.600 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:54.600 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 12:20:54.600 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:54.600 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:54.600 | INFO | Uploaded 36GiB to GPU in 2.17335s (16.5643GiB/s) | 20260505 12:20:54.600 | INFO | Downloaded 34GiB to GPU in 2.12581s (15.9939GiB/s) | 20260505 12:20:59.733 | INFO | Analyzing chunk frames = [3164672,3165184) | 20260505 12:20:59.733 | INFO | MemInfo: { | 20260505 12:20:59.733 | INFO | "gpu" : " 17 GB", | 20260505 12:20:59.733 | INFO | "huge" : "", | 20260505 12:20:59.733 | INFO | "huge_shared" : " 27 GB", | 20260505 12:20:59.733 | INFO | "malloc" : " 14 GB", | 20260505 12:20:59.733 | INFO | "pinned" : " 29 GB" | 20260505 12:20:59.733 | INFO | } | 20260505 12:20:59.734 | INFO | Perf report for the last 5.1341 seconds: | 20260505 12:20:59.734 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:20:59.734 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:59.734 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:59.734 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:59.734 | INFO | ANALYSIS : 63.4%, 3.0, 98.8ms, 0 | 20260505 12:20:59.734 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:59.734 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:59.734 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:20:59.734 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:59.734 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:20:59.734 | INFO | Uploaded 36GiB to GPU in 2.16544s (16.6248GiB/s) | 20260505 12:20:59.734 | INFO | Downloaded 34GiB to GPU in 2.10891s (16.122GiB/s) | 20260505 12:21:04.877 | INFO | Analyzing chunk frames = [3165184,3165696) | 20260505 12:21:04.877 | INFO | MemInfo: { | 20260505 12:21:04.877 | INFO | "gpu" : " 17 GB", | 20260505 12:21:04.877 | INFO | "huge" : "", | 20260505 12:21:04.877 | INFO | "huge_shared" : " 27 GB", | 20260505 12:21:04.877 | INFO | "malloc" : " 14 GB", | 20260505 12:21:04.877 | INFO | "pinned" : " 29 GB" | 20260505 12:21:04.877 | INFO | } | 20260505 12:21:04.877 | INFO | Perf report for the last 5.14364 seconds: | 20260505 12:21:04.877 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:21:04.877 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:04.877 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:04.877 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:04.877 | INFO | ANALYSIS : 63.3%, 3.0, 99.0ms, 0 | 20260505 12:21:04.877 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:04.877 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:04.877 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 12:21:04.877 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:04.877 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:04.878 | INFO | Uploaded 36GiB to GPU in 2.19346s (16.4124GiB/s) | 20260505 12:21:04.878 | INFO | Downloaded 34GiB to GPU in 2.15677s (15.7643GiB/s) | 20260505 12:21:10.018 | INFO | Analyzing chunk frames = [3165696,3166208) | 20260505 12:21:10.018 | INFO | MemInfo: { | 20260505 12:21:10.018 | INFO | "gpu" : " 17 GB", | 20260505 12:21:10.018 | INFO | "huge" : "", | 20260505 12:21:10.018 | INFO | "huge_shared" : " 27 GB", | 20260505 12:21:10.018 | INFO | "malloc" : " 14 GB", | 20260505 12:21:10.018 | INFO | "pinned" : " 29 GB" | 20260505 12:21:10.018 | INFO | } | 20260505 12:21:10.018 | INFO | Perf report for the last 5.14126 seconds: | 20260505 12:21:10.019 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:21:10.019 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:10.019 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:10.019 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:10.019 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 12:21:10.019 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:10.019 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:10.019 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:21:10.019 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:10.019 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:10.019 | INFO | Uploaded 36GiB to GPU in 2.16836s (16.6024GiB/s) | 20260505 12:21:10.019 | INFO | Downloaded 34GiB to GPU in 2.09957s (16.1938GiB/s) | 20260505 12:21:10.153 | INFO | Starting a round of Baz IO | 20260505 12:21:13.392 | INFO | Populating RTMetrics report | 20260505 12:21:13.392 | INFO | RTMetrics report populated | 20260505 12:21:13.392 | INFO | Opening tmp RTMetrics json file | 20260505 12:21:13.393 | INFO | Serializing RTMetrics to json | 20260505 12:21:13.395 | INFO | Serialization done, swapping tmp file now... | 20260505 12:21:13.395 | INFO | RTMetrics done | 20260505 12:21:13.511 | INFO | Finished a round of Baz IO: Wrote 2.342 GiB in 3.358 seconds | 20260505 12:21:13.511 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.63% / 0.01% / 76.36% | 20260505 12:21:13.511 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.911935% / 1.58946e-05% / 7.54992e-05% | 20260505 12:21:13.511 | INFO | Baz IO overall: | 20260505 12:21:13.511 | INFO | BytesWritten : 1880685316579 | 20260505 12:21:13.511 | INFO | NumEvents : 895234927573 | 20260505 12:21:15.160 | INFO | Analyzing chunk frames = [3166208,3166720) | 20260505 12:21:15.160 | INFO | MemInfo: { | 20260505 12:21:15.160 | INFO | "gpu" : " 17 GB", | 20260505 12:21:15.160 | INFO | "huge" : "", | 20260505 12:21:15.160 | INFO | "huge_shared" : " 27 GB", | 20260505 12:21:15.160 | INFO | "malloc" : " 14 GB", | 20260505 12:21:15.160 | INFO | "pinned" : " 29 GB" | 20260505 12:21:15.160 | INFO | } | 20260505 12:21:15.160 | INFO | Perf report for the last 5.14155 seconds: | 20260505 12:21:15.160 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:21:15.160 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:15.160 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:15.160 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:15.160 | INFO | ANALYSIS : 64.5%, 2.9, 99.8ms, 0 | 20260505 12:21:15.160 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:15.160 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260505 12:21:15.160 | INFO | PRE_HQ : 60.7%, 1.8, 57.9ms, 0 | 20260505 12:21:15.160 | INFO | BAZWRITER : 64.6%, 4.1, 135.1ms, 0 | 20260505 12:21:15.160 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 12:21:15.160 | INFO | Uploaded 36GiB to GPU in 2.18685s (16.462GiB/s) | 20260505 12:21:15.161 | INFO | Downloaded 38GiB to GPU in 2.28678s (16.6172GiB/s) | 20260505 12:21:20.290 | INFO | Analyzing chunk frames = [3166720,3167232) | 20260505 12:21:20.290 | INFO | MemInfo: { | 20260505 12:21:20.290 | INFO | "gpu" : " 17 GB", | 20260505 12:21:20.290 | INFO | "huge" : "", | 20260505 12:21:20.290 | INFO | "huge_shared" : " 27 GB", | 20260505 12:21:20.290 | INFO | "malloc" : " 14 GB", | 20260505 12:21:20.290 | INFO | "pinned" : " 29 GB" | 20260505 12:21:20.290 | INFO | } | 20260505 12:21:20.290 | INFO | Perf report for the last 5.12965 seconds: | 20260505 12:21:20.290 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:21:20.290 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:20.290 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:20.290 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:20.290 | INFO | ANALYSIS : 63.8%, 3.0, 99.5ms, 0 | 20260505 12:21:20.290 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:20.290 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:20.290 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 12:21:20.290 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:20.290 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:20.290 | INFO | Uploaded 36GiB to GPU in 2.17993s (16.5143GiB/s) | 20260505 12:21:20.290 | INFO | Downloaded 34GiB to GPU in 2.04278s (16.644GiB/s) | 20260505 12:21:25.431 | INFO | Analyzing chunk frames = [3167232,3167744) | 20260505 12:21:25.431 | INFO | MemInfo: { | 20260505 12:21:25.431 | INFO | "gpu" : " 17 GB", | 20260505 12:21:25.431 | INFO | "huge" : "", | 20260505 12:21:25.431 | INFO | "huge_shared" : " 27 GB", | 20260505 12:21:25.431 | INFO | "malloc" : " 14 GB", | 20260505 12:21:25.431 | INFO | "pinned" : " 29 GB" | 20260505 12:21:25.431 | INFO | } | 20260505 12:21:25.431 | INFO | Perf report for the last 5.14155 seconds: | 20260505 12:21:25.431 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:21:25.431 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:25.431 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:25.431 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:25.431 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260505 12:21:25.431 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:25.431 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:25.431 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 12:21:25.432 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:25.432 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:25.432 | INFO | Uploaded 36GiB to GPU in 2.22009s (16.2155GiB/s) | 20260505 12:21:25.432 | INFO | Downloaded 34GiB to GPU in 2.13164s (15.9501GiB/s) | 20260505 12:21:30.572 | INFO | Analyzing chunk frames = [3167744,3168256) | 20260505 12:21:30.572 | INFO | MemInfo: { | 20260505 12:21:30.572 | INFO | "gpu" : " 17 GB", | 20260505 12:21:30.572 | INFO | "huge" : "", | 20260505 12:21:30.572 | INFO | "huge_shared" : " 27 GB", | 20260505 12:21:30.572 | INFO | "malloc" : " 14 GB", | 20260505 12:21:30.572 | INFO | "pinned" : " 29 GB" | 20260505 12:21:30.572 | INFO | } | 20260505 12:21:30.572 | INFO | Perf report for the last 5.14047 seconds: | 20260505 12:21:30.572 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:21:30.572 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:30.572 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:30.572 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:30.572 | INFO | ANALYSIS : 63.2%, 3.0, 98.1ms, 0 | 20260505 12:21:30.572 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:30.572 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:30.572 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 12:21:30.572 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:30.572 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:30.572 | INFO | Uploaded 36GiB to GPU in 2.21012s (16.2887GiB/s) | 20260505 12:21:30.572 | INFO | Downloaded 34GiB to GPU in 2.13043s (15.9592GiB/s) | 20260505 12:21:35.720 | INFO | Analyzing chunk frames = [3168256,3168768) | 20260505 12:21:35.720 | INFO | MemInfo: { | 20260505 12:21:35.720 | INFO | "gpu" : " 17 GB", | 20260505 12:21:35.720 | INFO | "huge" : "", | 20260505 12:21:35.720 | INFO | "huge_shared" : " 27 GB", | 20260505 12:21:35.720 | INFO | "malloc" : " 14 GB", | 20260505 12:21:35.720 | INFO | "pinned" : " 29 GB" | 20260505 12:21:35.720 | INFO | } | 20260505 12:21:35.721 | INFO | Perf report for the last 5.14896 seconds: | 20260505 12:21:35.721 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:21:35.721 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:35.721 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:35.721 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:35.721 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 12:21:35.721 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:35.721 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:35.721 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:21:35.721 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:35.721 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:35.721 | INFO | Uploaded 36GiB to GPU in 2.15987s (16.6676GiB/s) | 20260505 12:21:35.721 | INFO | Downloaded 34GiB to GPU in 2.12671s (15.9872GiB/s) | 20260505 12:21:40.853 | INFO | Analyzing chunk frames = [3168768,3169280) | 20260505 12:21:40.853 | INFO | MemInfo: { | 20260505 12:21:40.853 | INFO | "gpu" : " 17 GB", | 20260505 12:21:40.853 | INFO | "huge" : "", | 20260505 12:21:40.853 | INFO | "huge_shared" : " 27 GB", | 20260505 12:21:40.853 | INFO | "malloc" : " 14 GB", | 20260505 12:21:40.853 | INFO | "pinned" : " 29 GB" | 20260505 12:21:40.853 | INFO | } | 20260505 12:21:40.853 | INFO | Perf report for the last 5.13229 seconds: | 20260505 12:21:40.853 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:21:40.853 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:40.853 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:40.853 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:40.853 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 12:21:40.853 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:40.853 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:40.853 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:21:40.853 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:40.853 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:40.853 | INFO | Uploaded 36GiB to GPU in 2.13403s (16.8695GiB/s) | 20260505 12:21:40.853 | INFO | Downloaded 34GiB to GPU in 2.10329s (16.1652GiB/s) | 20260505 12:21:45.986 | INFO | Analyzing chunk frames = [3169280,3169792) | 20260505 12:21:45.986 | INFO | MemInfo: { | 20260505 12:21:45.986 | INFO | "gpu" : " 17 GB", | 20260505 12:21:45.986 | INFO | "huge" : "", | 20260505 12:21:45.986 | INFO | "huge_shared" : " 27 GB", | 20260505 12:21:45.986 | INFO | "malloc" : " 14 GB", | 20260505 12:21:45.986 | INFO | "pinned" : " 29 GB" | 20260505 12:21:45.986 | INFO | } | 20260505 12:21:45.986 | INFO | Perf report for the last 5.13333 seconds: | 20260505 12:21:45.986 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:21:45.986 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:45.986 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:45.986 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:45.986 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 12:21:45.986 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:45.986 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:45.986 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 12:21:45.986 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:45.986 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:45.987 | INFO | Uploaded 36GiB to GPU in 2.16367s (16.6384GiB/s) | 20260505 12:21:45.987 | INFO | Downloaded 34GiB to GPU in 2.15529s (15.7751GiB/s) | 20260505 12:21:51.135 | INFO | Analyzing chunk frames = [3169792,3170304) | 20260505 12:21:51.135 | INFO | MemInfo: { | 20260505 12:21:51.135 | INFO | "gpu" : " 17 GB", | 20260505 12:21:51.135 | INFO | "huge" : "", | 20260505 12:21:51.135 | INFO | "huge_shared" : " 27 GB", | 20260505 12:21:51.135 | INFO | "malloc" : " 14 GB", | 20260505 12:21:51.135 | INFO | "pinned" : " 29 GB" | 20260505 12:21:51.135 | INFO | } | 20260505 12:21:51.135 | INFO | Perf report for the last 5.14912 seconds: | 20260505 12:21:51.135 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:21:51.135 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:51.135 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:51.135 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:51.136 | INFO | ANALYSIS : 63.7%, 3.0, 99.7ms, 0 | 20260505 12:21:51.136 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:51.136 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:51.136 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 12:21:51.136 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:51.136 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:51.136 | INFO | Uploaded 36GiB to GPU in 2.21692s (16.2387GiB/s) | 20260505 12:21:51.136 | INFO | Downloaded 34GiB to GPU in 2.09283s (16.2459GiB/s) | 20260505 12:21:51.136 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 12:21:51.136 | INFO | Total mem usage: 15.3 GiB | 20260505 12:21:51.136 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 12:21:51.136 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -7.4%) | 20260505 12:21:51.136 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 12:21:51.136 | INFO | High Water Breakdown: | 20260505 12:21:51.136 | INFO | 61% : 8.7 GiB ( 91.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 12:21:51.136 | INFO | 38.9% : 5.55 GiB ( 83.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 12:21:51.136 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 12:21:51.136 | INFO | ------------------------------------------ | 20260505 12:21:51.136 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:21:51.136 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 12:21:51.136 | INFO | Total mem usage: 26.5 GiB | 20260505 12:21:51.136 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 12:21:51.136 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 12:21:51.136 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 12:21:51.136 | INFO | High Water Breakdown: | 20260505 12:21:51.136 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:21:51.136 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:21:51.136 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:21:51.136 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:21:51.136 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:21:51.136 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:21:51.136 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 12:21:51.136 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 12:21:51.136 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:21:51.136 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 12:21:51.137 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 12:21:51.137 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 12:21:51.137 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 12:21:51.137 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 12:21:51.137 | INFO | ------------------------------------------ | 20260505 12:21:51.137 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:21:51.137 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 12:21:51.137 | INFO | Total mem usage: 25.6 GiB | 20260505 12:21:51.137 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 12:21:51.137 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 12:21:51.137 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 12:21:51.137 | INFO | High Water Breakdown: | 20260505 12:21:51.137 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 12:21:51.137 | INFO | ------------------------------------------ | 20260505 12:21:51.137 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:21:51.137 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 12:21:51.137 | INFO | Total mem usage: 15.7 GiB | 20260505 12:21:51.137 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 12:21:51.137 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 12:21:51.137 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 12:21:51.137 | INFO | High Water Breakdown: | 20260505 12:21:51.137 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 12:21:51.137 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 12:21:51.137 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 12:21:51.137 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 12:21:51.137 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 12:21:51.137 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 12:21:51.137 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 12:21:51.137 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 12:21:51.137 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 12:21:51.137 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:21:51.137 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 12:21:51.137 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:21:51.137 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 12:21:51.137 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 12:21:51.137 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 12:21:51.137 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 12:21:51.137 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:21:51.137 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 12:21:51.137 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:21:51.137 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:21:51.137 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:21:51.137 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 12:21:51.137 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 12:21:51.137 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:21:51.137 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 12:21:51.137 | INFO | ------------------------------------------ | 20260505 12:21:51.137 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:21:51.137 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 12:21:51.137 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:21:51.137 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:21:51.137 | INFO | PerfReport: Full Execution -- 3.20e+01 / 7.70e+01, 1.38e+01 / 1.37e+00 | 20260505 12:21:51.137 | INFO | PerfReport: 0.29%, AggregateStats -- 9.26e-02 / 1.00e-01, 1.06e-02 / 3.93e-03 | 20260505 12:21:51.137 | INFO | PerfReport: 11.76%, Baseline -- 3.76e+00 / 4.50e+00, 6.00e-01 / 1.94e-01 | 20260505 12:21:51.137 | INFO | PerfReport: 3.30%, Histogram -- 1.06e+00 / 1.49e+00, 2.03e-01 / 9.49e-02 | 20260505 12:21:51.137 | INFO | PerfReport: 15.89%, DME -- 4.07e+01 / 4.54e+01, 4.40e+00 / 2.03e+00 | 20260505 12:21:51.137 | INFO | PerfReport: 56.38%, FrameLabeling -- 1.81e+01 / 2.07e+01, 1.50e+00 / 2.09e+00 | 20260505 12:21:51.137 | INFO | PerfReport: 7.46%, PulseAccumulating -- 2.39e+00 / 2.87e+00, 4.55e-01 / 2.52e-01 | 20260505 12:21:51.137 | INFO | PerfReport: 3.64%, Metrics -- 1.17e+00 / 1.48e+00, 3.28e-01 / 1.81e-01 | 20260505 12:21:51.137 | INFO | PerfReport: 1.28%, PulseSerializing -- 4.09e-01 / 4.65e-01, 7.47e-02 / 3.24e-02 | 20260505 12:21:51.137 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 12:21:51.137 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:21:51.137 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:21:51.137 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260505 12:21:51.137 | INFO | PerfReport: 22.55%, Upload -- 2.28e+01 / 0.00e+00, 8.11e+00 / 0.00e+00 | 20260505 12:21:51.137 | INFO | PerfReport: 22.03%, Download -- 2.23e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260505 12:21:51.137 | INFO | PerfReport: 31.69%, Compute -- 3.20e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 12:21:51.137 | INFO | PerfReport: 2.14%, LockUpload -- 2.17e+00 / 0.00e+00, 7.76e+00 / 0.00e+00 | 20260505 12:21:51.137 | INFO | PerfReport: 1.35%, LockDownload -- 1.36e+00 / 0.00e+00, 2.93e+00 / 0.00e+00 | 20260505 12:21:51.137 | INFO | PerfReport: 20.22%, LockCompute -- 2.04e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260505 12:21:51.137 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.93e-02 / 0.00e+00 | 20260505 12:21:51.277 | INFO | Starting a round of Baz IO | 20260505 12:21:54.508 | INFO | Populating RTMetrics report | 20260505 12:21:54.509 | INFO | RTMetrics report populated | 20260505 12:21:54.509 | INFO | Opening tmp RTMetrics json file | 20260505 12:21:54.534 | INFO | Serializing RTMetrics to json | 20260505 12:21:54.537 | INFO | Serialization done, swapping tmp file now... | 20260505 12:21:54.537 | INFO | RTMetrics done | 20260505 12:21:54.640 | INFO | Finished a round of Baz IO: Wrote 2.342 GiB in 3.362 seconds | 20260505 12:21:54.640 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.62% / 0.01% / 76.37% | 20260505 12:21:54.640 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.908554% / 1.98682e-05% / 7.54992e-05% | 20260505 12:21:54.640 | INFO | Baz IO overall: | 20260505 12:21:54.640 | INFO | BytesWritten : 1883200313003 | 20260505 12:21:54.640 | INFO | NumEvents : 896429217678 | 20260505 12:21:56.271 | INFO | Analyzing chunk frames = [3170304,3170816) | 20260505 12:21:56.271 | INFO | MemInfo: { | 20260505 12:21:56.271 | INFO | "gpu" : " 17 GB", | 20260505 12:21:56.271 | INFO | "huge" : "", | 20260505 12:21:56.271 | INFO | "huge_shared" : " 27 GB", | 20260505 12:21:56.271 | INFO | "malloc" : " 14 GB", | 20260505 12:21:56.271 | INFO | "pinned" : " 29 GB" | 20260505 12:21:56.271 | INFO | } | 20260505 12:21:56.271 | INFO | Perf report for the last 5.13553 seconds: | 20260505 12:21:56.271 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:21:56.271 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:56.271 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:56.271 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:56.271 | INFO | ANALYSIS : 64.6%, 2.9, 99.6ms, 0 | 20260505 12:21:56.271 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:21:56.271 | INFO | RT_METRICS : 45.9%, 1.0, 24.0ms, 0 | 20260505 12:21:56.271 | INFO | PRE_HQ : 61.1%, 1.8, 58.0ms, 0 | 20260505 12:21:56.271 | INFO | BAZWRITER : 64.8%, 4.1, 136.2ms, 0 | 20260505 12:21:56.271 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 12:21:56.271 | INFO | Uploaded 36GiB to GPU in 2.1626s (16.6466GiB/s) | 20260505 12:21:56.271 | INFO | Downloaded 38GiB to GPU in 2.27737s (16.6859GiB/s) | 20260505 12:22:01.407 | INFO | Analyzing chunk frames = [3170816,3171328) | 20260505 12:22:01.407 | INFO | MemInfo: { | 20260505 12:22:01.407 | INFO | "gpu" : " 17 GB", | 20260505 12:22:01.407 | INFO | "huge" : "", | 20260505 12:22:01.407 | INFO | "huge_shared" : " 27 GB", | 20260505 12:22:01.407 | INFO | "malloc" : " 14 GB", | 20260505 12:22:01.408 | INFO | "pinned" : " 29 GB" | 20260505 12:22:01.408 | INFO | } | 20260505 12:22:01.408 | INFO | Perf report for the last 5.13666 seconds: | 20260505 12:22:01.408 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:22:01.408 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:01.408 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:01.408 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:01.408 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 12:22:01.408 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:01.408 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:01.408 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 12:22:01.408 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:01.408 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:01.408 | INFO | Uploaded 36GiB to GPU in 2.15714s (16.6888GiB/s) | 20260505 12:22:01.408 | INFO | Downloaded 34GiB to GPU in 2.03605s (16.699GiB/s) | 20260505 12:22:06.547 | INFO | Analyzing chunk frames = [3171328,3171840) | 20260505 12:22:06.547 | INFO | MemInfo: { | 20260505 12:22:06.547 | INFO | "gpu" : " 17 GB", | 20260505 12:22:06.547 | INFO | "huge" : "", | 20260505 12:22:06.547 | INFO | "huge_shared" : " 27 GB", | 20260505 12:22:06.547 | INFO | "malloc" : " 14 GB", | 20260505 12:22:06.547 | INFO | "pinned" : " 29 GB" | 20260505 12:22:06.547 | INFO | } | 20260505 12:22:06.547 | INFO | Perf report for the last 5.13909 seconds: | 20260505 12:22:06.547 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:22:06.547 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:06.547 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:06.547 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:06.547 | INFO | ANALYSIS : 63.2%, 3.0, 98.3ms, 0 | 20260505 12:22:06.547 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:06.547 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:06.547 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 12:22:06.547 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:06.547 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:06.547 | INFO | Uploaded 36GiB to GPU in 2.19762s (16.3814GiB/s) | 20260505 12:22:06.547 | INFO | Downloaded 34GiB to GPU in 2.12086s (16.0312GiB/s) | 20260505 12:22:11.685 | INFO | Analyzing chunk frames = [3171840,3172352) | 20260505 12:22:11.685 | INFO | MemInfo: { | 20260505 12:22:11.685 | INFO | "gpu" : " 17 GB", | 20260505 12:22:11.685 | INFO | "huge" : "", | 20260505 12:22:11.685 | INFO | "huge_shared" : " 27 GB", | 20260505 12:22:11.685 | INFO | "malloc" : " 14 GB", | 20260505 12:22:11.685 | INFO | "pinned" : " 29 GB" | 20260505 12:22:11.685 | INFO | } | 20260505 12:22:11.685 | INFO | Perf report for the last 5.1386 seconds: | 20260505 12:22:11.685 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:22:11.685 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:11.685 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:11.685 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:11.685 | INFO | ANALYSIS : 63.2%, 3.0, 98.1ms, 0 | 20260505 12:22:11.685 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:11.686 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:11.686 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 12:22:11.686 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:11.686 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:11.686 | INFO | Uploaded 36GiB to GPU in 2.20782s (16.3057GiB/s) | 20260505 12:22:11.686 | INFO | Downloaded 34GiB to GPU in 2.13145s (15.9516GiB/s) | 20260505 12:22:16.827 | INFO | Analyzing chunk frames = [3172352,3172864) | 20260505 12:22:16.827 | INFO | MemInfo: { | 20260505 12:22:16.827 | INFO | "gpu" : " 17 GB", | 20260505 12:22:16.827 | INFO | "huge" : "", | 20260505 12:22:16.827 | INFO | "huge_shared" : " 27 GB", | 20260505 12:22:16.827 | INFO | "malloc" : " 14 GB", | 20260505 12:22:16.827 | INFO | "pinned" : " 29 GB" | 20260505 12:22:16.827 | INFO | } | 20260505 12:22:16.827 | INFO | Perf report for the last 5.14184 seconds: | 20260505 12:22:16.827 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:22:16.827 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:16.827 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:16.827 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:16.827 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 12:22:16.827 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:16.827 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:16.827 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 12:22:16.827 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:16.827 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:16.828 | INFO | Uploaded 36GiB to GPU in 2.15892s (16.675GiB/s) | 20260505 12:22:16.828 | INFO | Downloaded 34GiB to GPU in 2.12722s (15.9833GiB/s) | 20260505 12:22:21.970 | INFO | Analyzing chunk frames = [3172864,3173376) | 20260505 12:22:21.970 | INFO | MemInfo: { | 20260505 12:22:21.970 | INFO | "gpu" : " 17 GB", | 20260505 12:22:21.970 | INFO | "huge" : "", | 20260505 12:22:21.970 | INFO | "huge_shared" : " 27 GB", | 20260505 12:22:21.970 | INFO | "malloc" : " 14 GB", | 20260505 12:22:21.970 | INFO | "pinned" : " 29 GB" | 20260505 12:22:21.970 | INFO | } | 20260505 12:22:21.970 | INFO | Perf report for the last 5.143 seconds: | 20260505 12:22:21.970 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:22:21.970 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:21.970 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:21.970 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:21.970 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 12:22:21.970 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:21.970 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:21.970 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 12:22:21.971 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:21.971 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:21.971 | INFO | Uploaded 36GiB to GPU in 2.16075s (16.6609GiB/s) | 20260505 12:22:21.971 | INFO | Downloaded 34GiB to GPU in 2.10642s (16.1411GiB/s) | 20260505 12:22:27.103 | INFO | Analyzing chunk frames = [3173376,3173888) | 20260505 12:22:27.103 | INFO | MemInfo: { | 20260505 12:22:27.103 | INFO | "gpu" : " 17 GB", | 20260505 12:22:27.103 | INFO | "huge" : "", | 20260505 12:22:27.103 | INFO | "huge_shared" : " 27 GB", | 20260505 12:22:27.103 | INFO | "malloc" : " 14 GB", | 20260505 12:22:27.103 | INFO | "pinned" : " 29 GB" | 20260505 12:22:27.103 | INFO | } | 20260505 12:22:27.104 | INFO | Perf report for the last 5.13322 seconds: | 20260505 12:22:27.104 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:22:27.104 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:27.104 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:27.104 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:27.104 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 12:22:27.104 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:27.104 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:27.104 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 12:22:27.104 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:27.104 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:27.104 | INFO | Uploaded 36GiB to GPU in 2.16194s (16.6517GiB/s) | 20260505 12:22:27.104 | INFO | Downloaded 34GiB to GPU in 2.1379s (15.9034GiB/s) | 20260505 12:22:32.237 | INFO | Analyzing chunk frames = [3173888,3174400) | 20260505 12:22:32.237 | INFO | MemInfo: { | 20260505 12:22:32.237 | INFO | "gpu" : " 17 GB", | 20260505 12:22:32.237 | INFO | "huge" : "", | 20260505 12:22:32.237 | INFO | "huge_shared" : " 27 GB", | 20260505 12:22:32.237 | INFO | "malloc" : " 14 GB", | 20260505 12:22:32.237 | INFO | "pinned" : " 29 GB" | 20260505 12:22:32.237 | INFO | } | 20260505 12:22:32.237 | INFO | Perf report for the last 5.13344 seconds: | 20260505 12:22:32.237 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:22:32.237 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:32.237 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:32.237 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:32.237 | INFO | ANALYSIS : 63.8%, 3.0, 99.5ms, 0 | 20260505 12:22:32.237 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:32.237 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:32.237 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 12:22:32.237 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:32.237 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:32.237 | INFO | Uploaded 36GiB to GPU in 2.21255s (16.2708GiB/s) | 20260505 12:22:32.237 | INFO | Downloaded 34GiB to GPU in 2.09607s (16.2208GiB/s) | 20260505 12:22:32.390 | INFO | Starting a round of Baz IO | 20260505 12:22:35.623 | INFO | Populating RTMetrics report | 20260505 12:22:35.623 | INFO | RTMetrics report populated | 20260505 12:22:35.623 | INFO | Opening tmp RTMetrics json file | 20260505 12:22:35.641 | INFO | Serializing RTMetrics to json | 20260505 12:22:35.643 | INFO | Serialization done, swapping tmp file now... | 20260505 12:22:35.647 | INFO | RTMetrics done | 20260505 12:22:35.740 | INFO | Finished a round of Baz IO: Wrote 2.339 GiB in 3.349 seconds | 20260505 12:22:35.740 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.62% / 0.01% / 76.38% | 20260505 12:22:35.740 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.90301% / 2.38419e-05% / 9.93411e-05% | 20260505 12:22:35.740 | INFO | Baz IO overall: | 20260505 12:22:35.740 | INFO | BytesWritten : 1885712328832 | 20260505 12:22:35.740 | INFO | NumEvents : 897621221913 | 20260505 12:22:37.387 | INFO | Analyzing chunk frames = [3174400,3174912) | 20260505 12:22:37.387 | INFO | MemInfo: { | 20260505 12:22:37.387 | INFO | "gpu" : " 17 GB", | 20260505 12:22:37.387 | INFO | "huge" : "", | 20260505 12:22:37.387 | INFO | "huge_shared" : " 27 GB", | 20260505 12:22:37.387 | INFO | "malloc" : " 14 GB", | 20260505 12:22:37.387 | INFO | "pinned" : " 29 GB" | 20260505 12:22:37.387 | INFO | } | 20260505 12:22:37.387 | INFO | Perf report for the last 5.14977 seconds: | 20260505 12:22:37.387 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:22:37.387 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:37.387 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:37.387 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:37.387 | INFO | ANALYSIS : 64.6%, 2.9, 100.0ms, 0 | 20260505 12:22:37.387 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:37.387 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260505 12:22:37.387 | INFO | PRE_HQ : 61.1%, 1.8, 58.3ms, 0 | 20260505 12:22:37.387 | INFO | BAZWRITER : 65.0%, 4.0, 135.3ms, 0 | 20260505 12:22:37.387 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 12:22:37.387 | INFO | Uploaded 36GiB to GPU in 2.20043s (16.3604GiB/s) | 20260505 12:22:37.387 | INFO | Downloaded 38GiB to GPU in 2.2991s (16.5282GiB/s) | 20260505 12:22:42.532 | INFO | Analyzing chunk frames = [3174912,3175424) | 20260505 12:22:42.532 | INFO | MemInfo: { | 20260505 12:22:42.532 | INFO | "gpu" : " 17 GB", | 20260505 12:22:42.532 | INFO | "huge" : "", | 20260505 12:22:42.532 | INFO | "huge_shared" : " 27 GB", | 20260505 12:22:42.532 | INFO | "malloc" : " 14 GB", | 20260505 12:22:42.532 | INFO | "pinned" : " 29 GB" | 20260505 12:22:42.532 | INFO | } | 20260505 12:22:42.533 | INFO | Perf report for the last 5.14577 seconds: | 20260505 12:22:42.533 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:22:42.533 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:42.533 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:42.533 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:42.533 | INFO | ANALYSIS : 63.4%, 3.0, 99.2ms, 0 | 20260505 12:22:42.533 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:42.533 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:42.533 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:22:42.533 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:42.533 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:42.533 | INFO | Uploaded 36GiB to GPU in 2.17302s (16.5668GiB/s) | 20260505 12:22:42.533 | INFO | Downloaded 34GiB to GPU in 2.02559s (16.7853GiB/s) | 20260505 12:22:47.666 | INFO | Analyzing chunk frames = [3175424,3175936) | 20260505 12:22:47.666 | INFO | MemInfo: { | 20260505 12:22:47.666 | INFO | "gpu" : " 17 GB", | 20260505 12:22:47.666 | INFO | "huge" : "", | 20260505 12:22:47.666 | INFO | "huge_shared" : " 27 GB", | 20260505 12:22:47.666 | INFO | "malloc" : " 14 GB", | 20260505 12:22:47.666 | INFO | "pinned" : " 29 GB" | 20260505 12:22:47.666 | INFO | } | 20260505 12:22:47.666 | INFO | Perf report for the last 5.13324 seconds: | 20260505 12:22:47.666 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:22:47.666 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:47.666 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:47.666 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:47.666 | INFO | ANALYSIS : 63.7%, 3.0, 99.0ms, 0 | 20260505 12:22:47.666 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:47.666 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:47.666 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:22:47.666 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:47.666 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:47.666 | INFO | Uploaded 36GiB to GPU in 2.23921s (16.0771GiB/s) | 20260505 12:22:47.666 | INFO | Downloaded 34GiB to GPU in 2.11114s (16.105GiB/s) | 20260505 12:22:52.801 | INFO | Analyzing chunk frames = [3175936,3176448) | 20260505 12:22:52.801 | INFO | MemInfo: { | 20260505 12:22:52.801 | INFO | "gpu" : " 17 GB", | 20260505 12:22:52.801 | INFO | "huge" : "", | 20260505 12:22:52.801 | INFO | "huge_shared" : " 27 GB", | 20260505 12:22:52.801 | INFO | "malloc" : " 14 GB", | 20260505 12:22:52.801 | INFO | "pinned" : " 29 GB" | 20260505 12:22:52.801 | INFO | } | 20260505 12:22:52.801 | INFO | Perf report for the last 5.13552 seconds: | 20260505 12:22:52.801 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:22:52.801 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:52.801 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:52.801 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:52.801 | INFO | ANALYSIS : 63.4%, 3.0, 98.4ms, 0 | 20260505 12:22:52.801 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:52.801 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:52.801 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:22:52.801 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:52.801 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:52.801 | INFO | Uploaded 36GiB to GPU in 2.21289s (16.2683GiB/s) | 20260505 12:22:52.802 | INFO | Downloaded 34GiB to GPU in 2.13202s (15.9473GiB/s) | 20260505 12:22:57.942 | INFO | Analyzing chunk frames = [3176448,3176960) | 20260505 12:22:57.942 | INFO | MemInfo: { | 20260505 12:22:57.942 | INFO | "gpu" : " 17 GB", | 20260505 12:22:57.942 | INFO | "huge" : "", | 20260505 12:22:57.942 | INFO | "huge_shared" : " 27 GB", | 20260505 12:22:57.942 | INFO | "malloc" : " 14 GB", | 20260505 12:22:57.942 | INFO | "pinned" : " 29 GB" | 20260505 12:22:57.942 | INFO | } | 20260505 12:22:57.943 | INFO | Perf report for the last 5.14118 seconds: | 20260505 12:22:57.943 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:22:57.943 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:57.943 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:57.943 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:57.943 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 12:22:57.943 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:57.943 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:57.943 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 12:22:57.943 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:57.943 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:22:57.943 | INFO | Uploaded 36GiB to GPU in 2.18736s (16.4582GiB/s) | 20260505 12:22:57.943 | INFO | Downloaded 34GiB to GPU in 2.12776s (15.9792GiB/s) | 20260505 12:23:03.081 | INFO | Analyzing chunk frames = [3176960,3177472) | 20260505 12:23:03.081 | INFO | MemInfo: { | 20260505 12:23:03.081 | INFO | "gpu" : " 17 GB", | 20260505 12:23:03.081 | INFO | "huge" : "", | 20260505 12:23:03.081 | INFO | "huge_shared" : " 27 GB", | 20260505 12:23:03.081 | INFO | "malloc" : " 14 GB", | 20260505 12:23:03.081 | INFO | "pinned" : " 29 GB" | 20260505 12:23:03.081 | INFO | } | 20260505 12:23:03.081 | INFO | Perf report for the last 5.13838 seconds: | 20260505 12:23:03.081 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:23:03.081 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:03.081 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:03.081 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:03.081 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 12:23:03.081 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:03.081 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:03.081 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 12:23:03.081 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:03.081 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:03.081 | INFO | Uploaded 36GiB to GPU in 2.16708s (16.6122GiB/s) | 20260505 12:23:03.081 | INFO | Downloaded 34GiB to GPU in 2.11055s (16.1095GiB/s) | 20260505 12:23:08.230 | INFO | Analyzing chunk frames = [3177472,3177984) | 20260505 12:23:08.230 | INFO | MemInfo: { | 20260505 12:23:08.230 | INFO | "gpu" : " 17 GB", | 20260505 12:23:08.230 | INFO | "huge" : "", | 20260505 12:23:08.230 | INFO | "huge_shared" : " 27 GB", | 20260505 12:23:08.230 | INFO | "malloc" : " 14 GB", | 20260505 12:23:08.230 | INFO | "pinned" : " 29 GB" | 20260505 12:23:08.230 | INFO | } | 20260505 12:23:08.230 | INFO | Perf report for the last 5.1492 seconds: | 20260505 12:23:08.230 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:23:08.230 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:08.230 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:08.230 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:08.230 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 12:23:08.230 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:08.230 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:08.230 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 12:23:08.230 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:08.230 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:08.230 | INFO | Uploaded 36GiB to GPU in 2.15174s (16.7306GiB/s) | 20260505 12:23:08.230 | INFO | Downloaded 34GiB to GPU in 2.16919s (15.6741GiB/s) | 20260505 12:23:13.356 | INFO | Analyzing chunk frames = [3177984,3178496) | 20260505 12:23:13.356 | INFO | MemInfo: { | 20260505 12:23:13.356 | INFO | "gpu" : " 17 GB", | 20260505 12:23:13.356 | INFO | "huge" : "", | 20260505 12:23:13.356 | INFO | "huge_shared" : " 27 GB", | 20260505 12:23:13.356 | INFO | "malloc" : " 14 GB", | 20260505 12:23:13.356 | INFO | "pinned" : " 29 GB" | 20260505 12:23:13.356 | INFO | } | 20260505 12:23:13.356 | INFO | Perf report for the last 5.12569 seconds: | 20260505 12:23:13.356 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:23:13.356 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:13.356 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:13.356 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:13.356 | INFO | ANALYSIS : 63.8%, 3.0, 99.3ms, 0 | 20260505 12:23:13.356 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:13.356 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:13.356 | INFO | PRE_HQ : 29.1%, 1.0, 15.0ms, 0 | 20260505 12:23:13.356 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:13.356 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:13.356 | INFO | Uploaded 36GiB to GPU in 2.15424s (16.7112GiB/s) | 20260505 12:23:13.356 | INFO | Downloaded 34GiB to GPU in 2.10177s (16.1768GiB/s) | 20260505 12:23:13.356 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 12:23:13.356 | INFO | Total mem usage: 15.3 GiB | 20260505 12:23:13.356 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 12:23:13.356 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -7.43%) | 20260505 12:23:13.356 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 12:23:13.356 | INFO | High Water Breakdown: | 20260505 12:23:13.356 | INFO | 60.9% : 8.7 GiB ( 91.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 12:23:13.356 | INFO | 38.9% : 5.55 GiB ( 83.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 12:23:13.356 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 12:23:13.356 | INFO | ------------------------------------------ | 20260505 12:23:13.356 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:23:13.356 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 12:23:13.356 | INFO | Total mem usage: 26.5 GiB | 20260505 12:23:13.356 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 12:23:13.356 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 12:23:13.356 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 12:23:13.356 | INFO | High Water Breakdown: | 20260505 12:23:13.356 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:23:13.356 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:23:13.356 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:23:13.356 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:23:13.356 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:23:13.356 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:23:13.356 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 12:23:13.356 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 12:23:13.356 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:23:13.356 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 12:23:13.356 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 12:23:13.356 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 12:23:13.356 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 12:23:13.356 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 12:23:13.356 | INFO | ------------------------------------------ | 20260505 12:23:13.356 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:23:13.356 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 12:23:13.356 | INFO | Total mem usage: 25.6 GiB | 20260505 12:23:13.356 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 12:23:13.356 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 12:23:13.356 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 12:23:13.356 | INFO | High Water Breakdown: | 20260505 12:23:13.356 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 12:23:13.356 | INFO | ------------------------------------------ | 20260505 12:23:13.356 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:23:13.356 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 12:23:13.356 | INFO | Total mem usage: 15.7 GiB | 20260505 12:23:13.356 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 12:23:13.356 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 12:23:13.357 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 12:23:13.357 | INFO | High Water Breakdown: | 20260505 12:23:13.357 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 12:23:13.357 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 12:23:13.357 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 12:23:13.357 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 12:23:13.357 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 12:23:13.357 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 12:23:13.357 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 12:23:13.357 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 12:23:13.357 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 12:23:13.357 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:23:13.357 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 12:23:13.357 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:23:13.357 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 12:23:13.357 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 12:23:13.357 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 12:23:13.357 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 12:23:13.357 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:23:13.357 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 12:23:13.357 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:23:13.357 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:23:13.357 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:23:13.357 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 12:23:13.357 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 12:23:13.357 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:23:13.357 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 12:23:13.357 | INFO | ------------------------------------------ | 20260505 12:23:13.357 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:23:13.357 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 12:23:13.357 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:23:13.357 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:23:13.357 | INFO | PerfReport: Full Execution -- 3.20e+01 / 7.70e+01, 1.36e+01 / 1.62e+00 | 20260505 12:23:13.357 | INFO | PerfReport: 0.29%, AggregateStats -- 9.30e-02 / 1.01e-01, 1.05e-02 / 4.46e-03 | 20260505 12:23:13.357 | INFO | PerfReport: 11.75%, Baseline -- 3.76e+00 / 4.47e+00, 5.96e-01 / 2.24e-01 | 20260505 12:23:13.357 | INFO | PerfReport: 3.30%, Histogram -- 1.06e+00 / 1.48e+00, 2.00e-01 / 1.14e-01 | 20260505 12:23:13.357 | INFO | PerfReport: 15.90%, DME -- 4.07e+01 / 4.55e+01, 4.45e+00 / 1.85e+00 | 20260505 12:23:13.357 | INFO | PerfReport: 56.41%, FrameLabeling -- 1.81e+01 / 2.05e+01, 1.50e+00 / 1.72e+00 | 20260505 12:23:13.357 | INFO | PerfReport: 7.43%, PulseAccumulating -- 2.38e+00 / 2.91e+00, 4.42e-01 / 2.14e-01 | 20260505 12:23:13.357 | INFO | PerfReport: 3.63%, Metrics -- 1.16e+00 / 1.50e+00, 3.28e-01 / 1.83e-01 | 20260505 12:23:13.357 | INFO | PerfReport: 1.28%, PulseSerializing -- 4.08e-01 / 4.59e-01, 7.41e-02 / 2.83e-02 | 20260505 12:23:13.357 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 12:23:13.357 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:23:13.357 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:23:13.357 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260505 12:23:13.357 | INFO | PerfReport: 22.50%, Upload -- 2.27e+01 / 0.00e+00, 8.03e+00 / 0.00e+00 | 20260505 12:23:13.357 | INFO | PerfReport: 21.99%, Download -- 2.22e+01 / 0.00e+00, 8.14e+00 / 0.00e+00 | 20260505 12:23:13.357 | INFO | PerfReport: 31.70%, Compute -- 3.20e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 12:23:13.357 | INFO | PerfReport: 2.17%, LockUpload -- 2.19e+00 / 0.00e+00, 7.86e+00 / 0.00e+00 | 20260505 12:23:13.357 | INFO | PerfReport: 1.37%, LockDownload -- 1.39e+00 / 0.00e+00, 2.97e+00 / 0.00e+00 | 20260505 12:23:13.357 | INFO | PerfReport: 20.26%, LockCompute -- 2.05e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260505 12:23:13.357 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.50e-02 / 0.00e+00 | 20260505 12:23:13.494 | INFO | Starting a round of Baz IO | 20260505 12:23:16.744 | INFO | Populating RTMetrics report | 20260505 12:23:16.745 | INFO | RTMetrics report populated | 20260505 12:23:16.745 | INFO | Opening tmp RTMetrics json file | 20260505 12:23:16.801 | INFO | Serializing RTMetrics to json | 20260505 12:23:16.827 | INFO | Serialization done, swapping tmp file now... | 20260505 12:23:16.838 | INFO | RTMetrics done | 20260505 12:23:16.855 | INFO | Finished a round of Baz IO: Wrote 2.338 GiB in 3.36 seconds | 20260505 12:23:16.855 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.61% / 0.01% / 76.38% | 20260505 12:23:16.855 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.897845% / 7.94729e-06% / 6.75519e-05% | 20260505 12:23:16.855 | INFO | Baz IO overall: | 20260505 12:23:16.855 | INFO | BytesWritten : 1888222241217 | 20260505 12:23:16.855 | INFO | NumEvents : 898811726924 | 20260505 12:23:18.493 | INFO | Analyzing chunk frames = [3178496,3179008) | 20260505 12:23:18.493 | INFO | MemInfo: { | 20260505 12:23:18.493 | INFO | "gpu" : " 17 GB", | 20260505 12:23:18.493 | INFO | "huge" : "", | 20260505 12:23:18.493 | INFO | "huge_shared" : " 27 GB", | 20260505 12:23:18.493 | INFO | "malloc" : " 14 GB", | 20260505 12:23:18.493 | INFO | "pinned" : " 29 GB" | 20260505 12:23:18.493 | INFO | } | 20260505 12:23:18.493 | INFO | Perf report for the last 5.13741 seconds: | 20260505 12:23:18.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:23:18.493 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:18.493 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:18.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:18.493 | INFO | ANALYSIS : 64.9%, 2.9, 100.3ms, 0 | 20260505 12:23:18.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:18.493 | INFO | RT_METRICS : 47.1%, 1.0, 24.7ms, 0 | 20260505 12:23:18.493 | INFO | PRE_HQ : 61.2%, 1.8, 58.4ms, 0 | 20260505 12:23:18.493 | INFO | BAZWRITER : 64.8%, 4.0, 134.6ms, 0 | 20260505 12:23:18.493 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 12:23:18.493 | INFO | Uploaded 36GiB to GPU in 2.2042s (16.3325GiB/s) | 20260505 12:23:18.493 | INFO | Downloaded 38GiB to GPU in 2.2925s (16.5758GiB/s) | 20260505 12:23:23.641 | INFO | Analyzing chunk frames = [3179008,3179520) | 20260505 12:23:23.641 | INFO | MemInfo: { | 20260505 12:23:23.641 | INFO | "gpu" : " 17 GB", | 20260505 12:23:23.641 | INFO | "huge" : "", | 20260505 12:23:23.641 | INFO | "huge_shared" : " 27 GB", | 20260505 12:23:23.641 | INFO | "malloc" : " 14 GB", | 20260505 12:23:23.641 | INFO | "pinned" : " 29 GB" | 20260505 12:23:23.641 | INFO | } | 20260505 12:23:23.642 | INFO | Perf report for the last 5.14847 seconds: | 20260505 12:23:23.642 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:23:23.642 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:23.642 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:23.642 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:23.642 | INFO | ANALYSIS : 64.2%, 3.0, 100.4ms, 0 | 20260505 12:23:23.642 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:23.642 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:23.642 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:23:23.642 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:23.642 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:23.642 | INFO | Uploaded 36GiB to GPU in 2.23778s (16.0874GiB/s) | 20260505 12:23:23.642 | INFO | Downloaded 34GiB to GPU in 2.09156s (16.2558GiB/s) | 20260505 12:23:28.786 | INFO | Analyzing chunk frames = [3179520,3180032) | 20260505 12:23:28.786 | INFO | MemInfo: { | 20260505 12:23:28.786 | INFO | "gpu" : " 17 GB", | 20260505 12:23:28.786 | INFO | "huge" : "", | 20260505 12:23:28.786 | INFO | "huge_shared" : " 27 GB", | 20260505 12:23:28.787 | INFO | "malloc" : " 14 GB", | 20260505 12:23:28.787 | INFO | "pinned" : " 29 GB" | 20260505 12:23:28.787 | INFO | } | 20260505 12:23:28.787 | INFO | Perf report for the last 5.1449 seconds: | 20260505 12:23:28.787 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:23:28.787 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:28.787 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:28.787 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:28.787 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 12:23:28.787 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:28.787 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:28.787 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 12:23:28.787 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:28.787 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:28.787 | INFO | Uploaded 36GiB to GPU in 2.23778s (16.0874GiB/s) | 20260505 12:23:28.787 | INFO | Downloaded 34GiB to GPU in 2.08965s (16.2706GiB/s) | 20260505 12:23:33.918 | INFO | Analyzing chunk frames = [3180032,3180544) | 20260505 12:23:33.918 | INFO | MemInfo: { | 20260505 12:23:33.918 | INFO | "gpu" : " 17 GB", | 20260505 12:23:33.918 | INFO | "huge" : "", | 20260505 12:23:33.918 | INFO | "huge_shared" : " 27 GB", | 20260505 12:23:33.918 | INFO | "malloc" : " 14 GB", | 20260505 12:23:33.918 | INFO | "pinned" : " 29 GB" | 20260505 12:23:33.918 | INFO | } | 20260505 12:23:33.918 | INFO | Perf report for the last 5.13179 seconds: | 20260505 12:23:33.918 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:23:33.918 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:33.918 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:33.918 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:33.918 | INFO | ANALYSIS : 63.3%, 3.0, 98.1ms, 0 | 20260505 12:23:33.918 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:33.918 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:33.918 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 12:23:33.918 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:33.919 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:33.919 | INFO | Uploaded 36GiB to GPU in 2.20282s (16.3427GiB/s) | 20260505 12:23:33.919 | INFO | Downloaded 34GiB to GPU in 2.13072s (15.957GiB/s) | 20260505 12:23:39.047 | INFO | Analyzing chunk frames = [3180544,3181056) | 20260505 12:23:39.047 | INFO | MemInfo: { | 20260505 12:23:39.047 | INFO | "gpu" : " 17 GB", | 20260505 12:23:39.047 | INFO | "huge" : "", | 20260505 12:23:39.047 | INFO | "huge_shared" : " 27 GB", | 20260505 12:23:39.047 | INFO | "malloc" : " 14 GB", | 20260505 12:23:39.047 | INFO | "pinned" : " 29 GB" | 20260505 12:23:39.047 | INFO | } | 20260505 12:23:39.047 | INFO | Perf report for the last 5.12883 seconds: | 20260505 12:23:39.047 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:23:39.047 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:39.047 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:39.047 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:39.047 | INFO | ANALYSIS : 63.2%, 3.0, 98.4ms, 0 | 20260505 12:23:39.047 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:39.047 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:39.047 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 12:23:39.047 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:39.047 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:39.047 | INFO | Uploaded 36GiB to GPU in 2.17448s (16.5557GiB/s) | 20260505 12:23:39.048 | INFO | Downloaded 34GiB to GPU in 2.12625s (15.9906GiB/s) | 20260505 12:23:44.193 | INFO | Analyzing chunk frames = [3181056,3181568) | 20260505 12:23:44.193 | INFO | MemInfo: { | 20260505 12:23:44.193 | INFO | "gpu" : " 17 GB", | 20260505 12:23:44.193 | INFO | "huge" : "", | 20260505 12:23:44.193 | INFO | "huge_shared" : " 27 GB", | 20260505 12:23:44.193 | INFO | "malloc" : " 14 GB", | 20260505 12:23:44.193 | INFO | "pinned" : " 29 GB" | 20260505 12:23:44.193 | INFO | } | 20260505 12:23:44.193 | INFO | Perf report for the last 5.14621 seconds: | 20260505 12:23:44.193 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:23:44.194 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:44.194 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:44.194 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:44.194 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 12:23:44.194 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:44.194 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:44.194 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 12:23:44.194 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:44.194 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:44.194 | INFO | Uploaded 36GiB to GPU in 2.17155s (16.578GiB/s) | 20260505 12:23:44.194 | INFO | Downloaded 34GiB to GPU in 2.12332s (16.0127GiB/s) | 20260505 12:23:49.334 | INFO | Analyzing chunk frames = [3181568,3182080) | 20260505 12:23:49.334 | INFO | MemInfo: { | 20260505 12:23:49.334 | INFO | "gpu" : " 17 GB", | 20260505 12:23:49.334 | INFO | "huge" : "", | 20260505 12:23:49.334 | INFO | "huge_shared" : " 27 GB", | 20260505 12:23:49.334 | INFO | "malloc" : " 14 GB", | 20260505 12:23:49.334 | INFO | "pinned" : " 29 GB" | 20260505 12:23:49.334 | INFO | } | 20260505 12:23:49.334 | INFO | Perf report for the last 5.1409 seconds: | 20260505 12:23:49.334 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:23:49.334 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:49.334 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:49.334 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:49.334 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 12:23:49.334 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:49.334 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:49.334 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 12:23:49.334 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:49.334 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:49.335 | INFO | Uploaded 36GiB to GPU in 2.16995s (16.5902GiB/s) | 20260505 12:23:49.335 | INFO | Downloaded 34GiB to GPU in 2.13425s (15.9306GiB/s) | 20260505 12:23:54.478 | INFO | Analyzing chunk frames = [3182080,3182592) | 20260505 12:23:54.478 | INFO | MemInfo: { | 20260505 12:23:54.478 | INFO | "gpu" : " 17 GB", | 20260505 12:23:54.478 | INFO | "huge" : "", | 20260505 12:23:54.479 | INFO | "huge_shared" : " 27 GB", | 20260505 12:23:54.479 | INFO | "malloc" : " 14 GB", | 20260505 12:23:54.479 | INFO | "pinned" : " 29 GB" | 20260505 12:23:54.479 | INFO | } | 20260505 12:23:54.479 | INFO | Perf report for the last 5.14419 seconds: | 20260505 12:23:54.479 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:23:54.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:54.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:54.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:54.479 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 12:23:54.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:54.479 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:54.479 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 12:23:54.479 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:54.479 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:54.479 | INFO | Uploaded 36GiB to GPU in 2.16269s (16.6459GiB/s) | 20260505 12:23:54.479 | INFO | Downloaded 34GiB to GPU in 2.10722s (16.135GiB/s) | 20260505 12:23:54.619 | INFO | Starting a round of Baz IO | 20260505 12:23:57.842 | INFO | Populating RTMetrics report | 20260505 12:23:57.842 | INFO | RTMetrics report populated | 20260505 12:23:57.842 | INFO | Opening tmp RTMetrics json file | 20260505 12:23:57.859 | INFO | Serializing RTMetrics to json | 20260505 12:23:57.860 | INFO | Serialization done, swapping tmp file now... | 20260505 12:23:57.861 | INFO | RTMetrics done | 20260505 12:23:57.950 | INFO | Finished a round of Baz IO: Wrote 2.338 GiB in 3.331 seconds | 20260505 12:23:57.950 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.61% / 0.01% / 76.39% | 20260505 12:23:57.950 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.896962% / 7.94729e-06% / 8.34465e-05% | 20260505 12:23:57.950 | INFO | Baz IO overall: | 20260505 12:23:57.950 | INFO | BytesWritten : 1890732127181 | 20260505 12:23:57.950 | INFO | NumEvents : 900002241468 | 20260505 12:23:59.614 | INFO | Analyzing chunk frames = [3182592,3183104) | 20260505 12:23:59.614 | INFO | MemInfo: { | 20260505 12:23:59.614 | INFO | "gpu" : " 17 GB", | 20260505 12:23:59.614 | INFO | "huge" : "", | 20260505 12:23:59.614 | INFO | "huge_shared" : " 27 GB", | 20260505 12:23:59.614 | INFO | "malloc" : " 14 GB", | 20260505 12:23:59.614 | INFO | "pinned" : " 29 GB" | 20260505 12:23:59.614 | INFO | } | 20260505 12:23:59.614 | INFO | Perf report for the last 5.1355 seconds: | 20260505 12:23:59.614 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:23:59.614 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:59.614 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:59.614 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:59.614 | INFO | ANALYSIS : 64.4%, 3.0, 99.5ms, 0 | 20260505 12:23:59.614 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:23:59.614 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260505 12:23:59.614 | INFO | PRE_HQ : 61.1%, 1.8, 58.2ms, 0 | 20260505 12:23:59.614 | INFO | BAZWRITER : 64.2%, 4.1, 134.2ms, 0 | 20260505 12:23:59.614 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 12:23:59.614 | INFO | Uploaded 36GiB to GPU in 2.16417s (16.6346GiB/s) | 20260505 12:23:59.614 | INFO | Downloaded 38GiB to GPU in 2.27509s (16.7026GiB/s) | 20260505 12:24:04.749 | INFO | Analyzing chunk frames = [3183104,3183616) | 20260505 12:24:04.749 | INFO | MemInfo: { | 20260505 12:24:04.749 | INFO | "gpu" : " 17 GB", | 20260505 12:24:04.749 | INFO | "huge" : "", | 20260505 12:24:04.749 | INFO | "huge_shared" : " 27 GB", | 20260505 12:24:04.749 | INFO | "malloc" : " 14 GB", | 20260505 12:24:04.749 | INFO | "pinned" : " 29 GB" | 20260505 12:24:04.749 | INFO | } | 20260505 12:24:04.749 | INFO | Perf report for the last 5.13522 seconds: | 20260505 12:24:04.749 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:24:04.749 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:04.749 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:04.749 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:04.749 | INFO | ANALYSIS : 64.3%, 3.0, 100.4ms, 0 | 20260505 12:24:04.749 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:04.749 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:04.749 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 12:24:04.749 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:04.749 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:04.750 | INFO | Uploaded 36GiB to GPU in 2.23424s (16.1128GiB/s) | 20260505 12:24:04.750 | INFO | Downloaded 34GiB to GPU in 2.09366s (16.2395GiB/s) | 20260505 12:24:09.891 | INFO | Analyzing chunk frames = [3183616,3184128) | 20260505 12:24:09.891 | INFO | MemInfo: { | 20260505 12:24:09.891 | INFO | "gpu" : " 17 GB", | 20260505 12:24:09.891 | INFO | "huge" : "", | 20260505 12:24:09.891 | INFO | "huge_shared" : " 27 GB", | 20260505 12:24:09.891 | INFO | "malloc" : " 14 GB", | 20260505 12:24:09.891 | INFO | "pinned" : " 29 GB" | 20260505 12:24:09.891 | INFO | } | 20260505 12:24:09.891 | INFO | Perf report for the last 5.14199 seconds: | 20260505 12:24:09.891 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:24:09.891 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:09.891 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:09.891 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:09.891 | INFO | ANALYSIS : 63.1%, 3.0, 98.3ms, 0 | 20260505 12:24:09.891 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:09.891 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:09.891 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:24:09.891 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:09.891 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:09.892 | INFO | Uploaded 36GiB to GPU in 2.20725s (16.3099GiB/s) | 20260505 12:24:09.892 | INFO | Downloaded 34GiB to GPU in 2.12629s (15.9903GiB/s) | 20260505 12:24:15.028 | INFO | Analyzing chunk frames = [3184128,3184640) | 20260505 12:24:15.028 | INFO | MemInfo: { | 20260505 12:24:15.028 | INFO | "gpu" : " 17 GB", | 20260505 12:24:15.028 | INFO | "huge" : "", | 20260505 12:24:15.028 | INFO | "huge_shared" : " 27 GB", | 20260505 12:24:15.028 | INFO | "malloc" : " 14 GB", | 20260505 12:24:15.028 | INFO | "pinned" : " 29 GB" | 20260505 12:24:15.028 | INFO | } | 20260505 12:24:15.028 | INFO | Perf report for the last 5.13645 seconds: | 20260505 12:24:15.028 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:24:15.028 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:15.028 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:15.028 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:15.028 | INFO | ANALYSIS : 63.4%, 3.0, 98.4ms, 0 | 20260505 12:24:15.028 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:15.028 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:15.028 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 12:24:15.028 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:15.028 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:15.028 | INFO | Uploaded 36GiB to GPU in 2.20818s (16.303GiB/s) | 20260505 12:24:15.028 | INFO | Downloaded 34GiB to GPU in 2.13663s (15.9129GiB/s) | 20260505 12:24:20.172 | INFO | Analyzing chunk frames = [3184640,3185152) | 20260505 12:24:20.172 | INFO | MemInfo: { | 20260505 12:24:20.172 | INFO | "gpu" : " 17 GB", | 20260505 12:24:20.173 | INFO | "huge" : "", | 20260505 12:24:20.173 | INFO | "huge_shared" : " 27 GB", | 20260505 12:24:20.173 | INFO | "malloc" : " 14 GB", | 20260505 12:24:20.173 | INFO | "pinned" : " 29 GB" | 20260505 12:24:20.173 | INFO | } | 20260505 12:24:20.173 | INFO | Perf report for the last 5.14485 seconds: | 20260505 12:24:20.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:24:20.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:20.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:20.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:20.173 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 12:24:20.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:20.173 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:20.173 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:24:20.173 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:20.173 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:20.173 | INFO | Uploaded 36GiB to GPU in 2.17976s (16.5156GiB/s) | 20260505 12:24:20.173 | INFO | Downloaded 34GiB to GPU in 2.12946s (15.9665GiB/s) | 20260505 12:24:25.308 | INFO | Analyzing chunk frames = [3185152,3185664) | 20260505 12:24:25.308 | INFO | MemInfo: { | 20260505 12:24:25.308 | INFO | "gpu" : " 17 GB", | 20260505 12:24:25.308 | INFO | "huge" : "", | 20260505 12:24:25.308 | INFO | "huge_shared" : " 27 GB", | 20260505 12:24:25.308 | INFO | "malloc" : " 14 GB", | 20260505 12:24:25.308 | INFO | "pinned" : " 29 GB" | 20260505 12:24:25.308 | INFO | } | 20260505 12:24:25.308 | INFO | Perf report for the last 5.13551 seconds: | 20260505 12:24:25.308 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:24:25.308 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:25.308 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:25.308 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:25.308 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 12:24:25.308 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:25.308 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:25.308 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 12:24:25.308 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:25.308 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:25.308 | INFO | Uploaded 36GiB to GPU in 2.16705s (16.6125GiB/s) | 20260505 12:24:25.308 | INFO | Downloaded 34GiB to GPU in 2.11477s (16.0774GiB/s) | 20260505 12:24:30.453 | INFO | Analyzing chunk frames = [3185664,3186176) | 20260505 12:24:30.453 | INFO | MemInfo: { | 20260505 12:24:30.453 | INFO | "gpu" : " 17 GB", | 20260505 12:24:30.453 | INFO | "huge" : "", | 20260505 12:24:30.453 | INFO | "huge_shared" : " 27 GB", | 20260505 12:24:30.453 | INFO | "malloc" : " 14 GB", | 20260505 12:24:30.453 | INFO | "pinned" : " 29 GB" | 20260505 12:24:30.453 | INFO | } | 20260505 12:24:30.453 | INFO | Perf report for the last 5.14498 seconds: | 20260505 12:24:30.453 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:24:30.453 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:30.453 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:30.453 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:30.453 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 12:24:30.453 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:30.453 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:30.453 | INFO | PRE_HQ : 29.5%, 1.0, 15.4ms, 0 | 20260505 12:24:30.453 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:30.453 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:30.453 | INFO | Uploaded 36GiB to GPU in 2.16705s (16.6125GiB/s) | 20260505 12:24:30.453 | INFO | Downloaded 34GiB to GPU in 2.13204s (15.9472GiB/s) | 20260505 12:24:35.591 | INFO | Analyzing chunk frames = [3186176,3186688) | 20260505 12:24:35.591 | INFO | MemInfo: { | 20260505 12:24:35.591 | INFO | "gpu" : " 17 GB", | 20260505 12:24:35.591 | INFO | "huge" : "", | 20260505 12:24:35.591 | INFO | "huge_shared" : " 27 GB", | 20260505 12:24:35.591 | INFO | "malloc" : " 14 GB", | 20260505 12:24:35.591 | INFO | "pinned" : " 29 GB" | 20260505 12:24:35.591 | INFO | } | 20260505 12:24:35.591 | INFO | Perf report for the last 5.1378 seconds: | 20260505 12:24:35.591 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:24:35.591 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:35.591 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:35.591 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:35.591 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 12:24:35.591 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:35.591 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:35.591 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 12:24:35.591 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:35.591 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:35.591 | INFO | Uploaded 36GiB to GPU in 2.2067s (16.314GiB/s) | 20260505 12:24:35.591 | INFO | Downloaded 34GiB to GPU in 2.10578s (16.146GiB/s) | 20260505 12:24:35.591 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 12:24:35.591 | INFO | Total mem usage: 15.3 GiB | 20260505 12:24:35.591 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 12:24:35.591 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -7.54%) | 20260505 12:24:35.591 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 12:24:35.591 | INFO | High Water Breakdown: | 20260505 12:24:35.591 | INFO | 61% : 8.7 GiB ( 91.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 12:24:35.591 | INFO | 38.8% : 5.54 GiB ( 83.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 12:24:35.592 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 12:24:35.592 | INFO | ------------------------------------------ | 20260505 12:24:35.592 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:24:35.592 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 12:24:35.592 | INFO | Total mem usage: 26.5 GiB | 20260505 12:24:35.592 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 12:24:35.592 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 12:24:35.592 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 12:24:35.592 | INFO | High Water Breakdown: | 20260505 12:24:35.592 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:24:35.592 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:24:35.592 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:24:35.592 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:24:35.592 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:24:35.592 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:24:35.592 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 12:24:35.592 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 12:24:35.592 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:24:35.592 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 12:24:35.592 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 12:24:35.592 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 12:24:35.592 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 12:24:35.592 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 12:24:35.592 | INFO | ------------------------------------------ | 20260505 12:24:35.592 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:24:35.592 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 12:24:35.592 | INFO | Total mem usage: 25.6 GiB | 20260505 12:24:35.592 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 12:24:35.592 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 12:24:35.592 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 12:24:35.592 | INFO | High Water Breakdown: | 20260505 12:24:35.592 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 12:24:35.592 | INFO | ------------------------------------------ | 20260505 12:24:35.592 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:24:35.592 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 12:24:35.592 | INFO | Total mem usage: 15.7 GiB | 20260505 12:24:35.592 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 12:24:35.592 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 12:24:35.592 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 12:24:35.592 | INFO | High Water Breakdown: | 20260505 12:24:35.592 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 12:24:35.592 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 12:24:35.592 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 12:24:35.592 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 12:24:35.592 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 12:24:35.592 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 12:24:35.592 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 12:24:35.592 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 12:24:35.592 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 12:24:35.592 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:24:35.592 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 12:24:35.592 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:24:35.592 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 12:24:35.592 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 12:24:35.592 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 12:24:35.592 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 12:24:35.592 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:24:35.592 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 12:24:35.592 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:24:35.592 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:24:35.592 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:24:35.592 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 12:24:35.592 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 12:24:35.592 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:24:35.592 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 12:24:35.592 | INFO | ------------------------------------------ | 20260505 12:24:35.592 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:24:35.592 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 12:24:35.592 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:24:35.592 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:24:35.592 | INFO | PerfReport: Full Execution -- 3.20e+01 / 7.72e+01, 1.38e+01 / 1.68e+00 | 20260505 12:24:35.592 | INFO | PerfReport: 0.29%, AggregateStats -- 9.25e-02 / 9.95e-02, 1.05e-02 / 4.99e-03 | 20260505 12:24:35.592 | INFO | PerfReport: 11.72%, Baseline -- 3.74e+00 / 4.50e+00, 5.85e-01 / 1.95e-01 | 20260505 12:24:35.592 | INFO | PerfReport: 3.29%, Histogram -- 1.05e+00 / 1.49e+00, 2.01e-01 / 9.50e-02 | 20260505 12:24:35.592 | INFO | PerfReport: 16.04%, DME -- 4.10e+01 / 4.61e+01, 4.56e+00 / 2.03e+00 | 20260505 12:24:35.592 | INFO | PerfReport: 56.29%, FrameLabeling -- 1.80e+01 / 2.02e+01, 1.44e+00 / 1.76e+00 | 20260505 12:24:35.592 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.38e+00 / 2.91e+00, 4.41e-01 / 2.06e-01 | 20260505 12:24:35.592 | INFO | PerfReport: 3.64%, Metrics -- 1.16e+00 / 1.48e+00, 3.24e-01 / 1.74e-01 | 20260505 12:24:35.592 | INFO | PerfReport: 1.28%, PulseSerializing -- 4.09e-01 / 4.60e-01, 7.30e-02 / 3.82e-02 | 20260505 12:24:35.592 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 12:24:35.592 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:24:35.592 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:24:35.592 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260505 12:24:35.592 | INFO | PerfReport: 22.58%, Upload -- 2.28e+01 / 0.00e+00, 8.14e+00 / 0.00e+00 | 20260505 12:24:35.592 | INFO | PerfReport: 22.01%, Download -- 2.23e+01 / 0.00e+00, 8.14e+00 / 0.00e+00 | 20260505 12:24:35.592 | INFO | PerfReport: 31.59%, Compute -- 3.20e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 12:24:35.592 | INFO | PerfReport: 2.34%, LockUpload -- 2.37e+00 / 0.00e+00, 7.97e+00 / 0.00e+00 | 20260505 12:24:35.592 | INFO | PerfReport: 1.29%, LockDownload -- 1.31e+00 / 0.00e+00, 2.87e+00 / 0.00e+00 | 20260505 12:24:35.592 | INFO | PerfReport: 20.18%, LockCompute -- 2.04e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260505 12:24:35.592 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.52e-02 / 0.00e+00 | 20260505 12:24:35.726 | INFO | Starting a round of Baz IO | 20260505 12:24:38.973 | INFO | Populating RTMetrics report | 20260505 12:24:38.973 | INFO | RTMetrics report populated | 20260505 12:24:38.974 | INFO | Opening tmp RTMetrics json file | 20260505 12:24:38.980 | INFO | Serializing RTMetrics to json | 20260505 12:24:38.982 | INFO | Serialization done, swapping tmp file now... | 20260505 12:24:38.982 | INFO | RTMetrics done | 20260505 12:24:39.098 | INFO | Finished a round of Baz IO: Wrote 2.339 GiB in 3.371 seconds | 20260505 12:24:39.098 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.60% / 0.01% / 76.39% | 20260505 12:24:39.098 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.896633% / 3.97364e-06% / 8.74201e-05% | 20260505 12:24:39.098 | INFO | Baz IO overall: | 20260505 12:24:39.098 | INFO | BytesWritten : 1893243747540 | 20260505 12:24:39.098 | INFO | NumEvents : 901193860114 | 20260505 12:24:40.731 | INFO | Analyzing chunk frames = [3186688,3187200) | 20260505 12:24:40.731 | INFO | MemInfo: { | 20260505 12:24:40.731 | INFO | "gpu" : " 17 GB", | 20260505 12:24:40.731 | INFO | "huge" : "", | 20260505 12:24:40.731 | INFO | "huge_shared" : " 27 GB", | 20260505 12:24:40.731 | INFO | "malloc" : " 14 GB", | 20260505 12:24:40.731 | INFO | "pinned" : " 29 GB" | 20260505 12:24:40.731 | INFO | } | 20260505 12:24:40.731 | INFO | Perf report for the last 5.13989 seconds: | 20260505 12:24:40.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:24:40.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:40.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:40.731 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:40.731 | INFO | ANALYSIS : 64.7%, 2.9, 100.0ms, 0 | 20260505 12:24:40.731 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:40.731 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260505 12:24:40.731 | INFO | PRE_HQ : 61.2%, 1.8, 58.5ms, 0 | 20260505 12:24:40.731 | INFO | BAZWRITER : 64.9%, 4.1, 135.8ms, 0 | 20260505 12:24:40.731 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260505 12:24:40.731 | INFO | Uploaded 36GiB to GPU in 2.21334s (16.265GiB/s) | 20260505 12:24:40.731 | INFO | Downloaded 38GiB to GPU in 2.28938s (16.5984GiB/s) | 20260505 12:24:45.866 | INFO | Analyzing chunk frames = [3187200,3187712) | 20260505 12:24:45.866 | INFO | MemInfo: { | 20260505 12:24:45.866 | INFO | "gpu" : " 17 GB", | 20260505 12:24:45.866 | INFO | "huge" : "", | 20260505 12:24:45.866 | INFO | "huge_shared" : " 27 GB", | 20260505 12:24:45.866 | INFO | "malloc" : " 14 GB", | 20260505 12:24:45.866 | INFO | "pinned" : " 29 GB" | 20260505 12:24:45.866 | INFO | } | 20260505 12:24:45.866 | INFO | Perf report for the last 5.13554 seconds: | 20260505 12:24:45.866 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:24:45.866 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:45.866 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:45.866 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:45.866 | INFO | ANALYSIS : 63.8%, 3.0, 99.5ms, 0 | 20260505 12:24:45.866 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:45.866 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:45.867 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 12:24:45.867 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:45.867 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:45.867 | INFO | Uploaded 36GiB to GPU in 2.17854s (16.5248GiB/s) | 20260505 12:24:45.867 | INFO | Downloaded 34GiB to GPU in 2.0501s (16.5846GiB/s) | 20260505 12:24:51.000 | INFO | Analyzing chunk frames = [3187712,3188224) | 20260505 12:24:51.000 | INFO | MemInfo: { | 20260505 12:24:51.000 | INFO | "gpu" : " 17 GB", | 20260505 12:24:51.000 | INFO | "huge" : "", | 20260505 12:24:51.000 | INFO | "huge_shared" : " 27 GB", | 20260505 12:24:51.000 | INFO | "malloc" : " 14 GB", | 20260505 12:24:51.000 | INFO | "pinned" : " 29 GB" | 20260505 12:24:51.000 | INFO | } | 20260505 12:24:51.000 | INFO | Perf report for the last 5.13347 seconds: | 20260505 12:24:51.000 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:24:51.000 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:51.000 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:51.000 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:51.000 | INFO | ANALYSIS : 63.7%, 3.0, 99.0ms, 0 | 20260505 12:24:51.000 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:51.000 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:51.000 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 12:24:51.000 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:51.000 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:51.000 | INFO | Uploaded 36GiB to GPU in 2.22752s (16.1614GiB/s) | 20260505 12:24:51.000 | INFO | Downloaded 34GiB to GPU in 2.12636s (15.9898GiB/s) | 20260505 12:24:56.148 | INFO | Analyzing chunk frames = [3188224,3188736) | 20260505 12:24:56.148 | INFO | MemInfo: { | 20260505 12:24:56.148 | INFO | "gpu" : " 17 GB", | 20260505 12:24:56.148 | INFO | "huge" : "", | 20260505 12:24:56.148 | INFO | "huge_shared" : " 27 GB", | 20260505 12:24:56.148 | INFO | "malloc" : " 14 GB", | 20260505 12:24:56.148 | INFO | "pinned" : " 29 GB" | 20260505 12:24:56.148 | INFO | } | 20260505 12:24:56.148 | INFO | Perf report for the last 5.14788 seconds: | 20260505 12:24:56.148 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:24:56.148 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:56.148 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:56.148 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:56.148 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 12:24:56.148 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:56.148 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:56.148 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:24:56.148 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:56.148 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:24:56.148 | INFO | Uploaded 36GiB to GPU in 2.22032s (16.2139GiB/s) | 20260505 12:24:56.148 | INFO | Downloaded 34GiB to GPU in 2.13593s (15.9181GiB/s) | 20260505 12:25:01.290 | INFO | Analyzing chunk frames = [3188736,3189248) | 20260505 12:25:01.290 | INFO | MemInfo: { | 20260505 12:25:01.290 | INFO | "gpu" : " 17 GB", | 20260505 12:25:01.290 | INFO | "huge" : "", | 20260505 12:25:01.290 | INFO | "huge_shared" : " 27 GB", | 20260505 12:25:01.290 | INFO | "malloc" : " 14 GB", | 20260505 12:25:01.290 | INFO | "pinned" : " 29 GB" | 20260505 12:25:01.290 | INFO | } | 20260505 12:25:01.290 | INFO | Perf report for the last 5.14194 seconds: | 20260505 12:25:01.290 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:25:01.290 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:01.290 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:01.290 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:01.290 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 12:25:01.290 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:01.290 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:01.290 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 12:25:01.290 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:01.290 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:01.290 | INFO | Uploaded 36GiB to GPU in 2.155s (16.7053GiB/s) | 20260505 12:25:01.290 | INFO | Downloaded 34GiB to GPU in 2.12553s (15.996GiB/s) | 20260505 12:25:06.426 | INFO | Analyzing chunk frames = [3189248,3189760) | 20260505 12:25:06.426 | INFO | MemInfo: { | 20260505 12:25:06.426 | INFO | "gpu" : " 17 GB", | 20260505 12:25:06.426 | INFO | "huge" : "", | 20260505 12:25:06.426 | INFO | "huge_shared" : " 27 GB", | 20260505 12:25:06.426 | INFO | "malloc" : " 14 GB", | 20260505 12:25:06.426 | INFO | "pinned" : " 29 GB" | 20260505 12:25:06.426 | INFO | } | 20260505 12:25:06.427 | INFO | Perf report for the last 5.13672 seconds: | 20260505 12:25:06.427 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:25:06.427 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:06.427 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:06.427 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:06.427 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 12:25:06.427 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:06.427 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:06.427 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 12:25:06.427 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:06.427 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:06.427 | INFO | Uploaded 36GiB to GPU in 2.14787s (16.7608GiB/s) | 20260505 12:25:06.427 | INFO | Downloaded 34GiB to GPU in 2.10689s (16.1375GiB/s) | 20260505 12:25:11.563 | INFO | Analyzing chunk frames = [3189760,3190272) | 20260505 12:25:11.563 | INFO | MemInfo: { | 20260505 12:25:11.563 | INFO | "gpu" : " 17 GB", | 20260505 12:25:11.563 | INFO | "huge" : "", | 20260505 12:25:11.563 | INFO | "huge_shared" : " 27 GB", | 20260505 12:25:11.563 | INFO | "malloc" : " 14 GB", | 20260505 12:25:11.563 | INFO | "pinned" : " 29 GB" | 20260505 12:25:11.563 | INFO | } | 20260505 12:25:11.563 | INFO | Perf report for the last 5.13698 seconds: | 20260505 12:25:11.563 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:25:11.563 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:11.564 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:11.564 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:11.564 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 12:25:11.564 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:11.564 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:11.564 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 12:25:11.564 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:11.564 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:11.564 | INFO | Uploaded 36GiB to GPU in 2.1857s (16.4707GiB/s) | 20260505 12:25:11.564 | INFO | Downloaded 34GiB to GPU in 2.15063s (15.8093GiB/s) | 20260505 12:25:16.704 | INFO | Analyzing chunk frames = [3190272,3190784) | 20260505 12:25:16.704 | INFO | MemInfo: { | 20260505 12:25:16.704 | INFO | "gpu" : " 17 GB", | 20260505 12:25:16.704 | INFO | "huge" : "", | 20260505 12:25:16.704 | INFO | "huge_shared" : " 27 GB", | 20260505 12:25:16.704 | INFO | "malloc" : " 14 GB", | 20260505 12:25:16.704 | INFO | "pinned" : " 29 GB" | 20260505 12:25:16.704 | INFO | } | 20260505 12:25:16.704 | INFO | Perf report for the last 5.14087 seconds: | 20260505 12:25:16.704 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:25:16.704 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:16.704 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:16.704 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:16.704 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 12:25:16.704 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:16.704 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:16.704 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 12:25:16.704 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:16.704 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:16.705 | INFO | Uploaded 36GiB to GPU in 2.17278s (16.5687GiB/s) | 20260505 12:25:16.705 | INFO | Downloaded 34GiB to GPU in 2.11361s (16.0862GiB/s) | 20260505 12:25:16.838 | INFO | Starting a round of Baz IO | 20260505 12:25:20.086 | INFO | Populating RTMetrics report | 20260505 12:25:20.086 | INFO | RTMetrics report populated | 20260505 12:25:20.086 | INFO | Opening tmp RTMetrics json file | 20260505 12:25:20.107 | INFO | Serializing RTMetrics to json | 20260505 12:25:20.130 | INFO | Serialization done, swapping tmp file now... | 20260505 12:25:20.152 | INFO | RTMetrics done | 20260505 12:25:20.191 | INFO | Finished a round of Baz IO: Wrote 2.339 GiB in 3.353 seconds | 20260505 12:25:20.191 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.59% / 0.01% / 76.40% | 20260505 12:25:20.191 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.893851% / 3.97364e-06% / 9.13938e-05% | 20260505 12:25:20.191 | INFO | Baz IO overall: | 20260505 12:25:20.191 | INFO | BytesWritten : 1895755467522 | 20260505 12:25:20.191 | INFO | NumEvents : 902385461293 | 20260505 12:25:21.840 | INFO | Analyzing chunk frames = [3190784,3191296) | 20260505 12:25:21.840 | INFO | MemInfo: { | 20260505 12:25:21.840 | INFO | "gpu" : " 17 GB", | 20260505 12:25:21.840 | INFO | "huge" : "", | 20260505 12:25:21.840 | INFO | "huge_shared" : " 27 GB", | 20260505 12:25:21.840 | INFO | "malloc" : " 14 GB", | 20260505 12:25:21.840 | INFO | "pinned" : " 29 GB" | 20260505 12:25:21.840 | INFO | } | 20260505 12:25:21.840 | INFO | Perf report for the last 5.13599 seconds: | 20260505 12:25:21.840 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:25:21.840 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:21.840 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:21.840 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:21.840 | INFO | ANALYSIS : 64.7%, 2.9, 99.9ms, 0 | 20260505 12:25:21.840 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:21.840 | INFO | RT_METRICS : 46.5%, 1.0, 24.4ms, 0 | 20260505 12:25:21.840 | INFO | PRE_HQ : 60.9%, 1.8, 58.1ms, 0 | 20260505 12:25:21.840 | INFO | BAZWRITER : 64.6%, 4.1, 135.6ms, 0 | 20260505 12:25:21.840 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260505 12:25:21.841 | INFO | Uploaded 36GiB to GPU in 2.18735s (16.4583GiB/s) | 20260505 12:25:21.841 | INFO | Downloaded 38GiB to GPU in 2.28221s (16.6505GiB/s) | 20260505 12:25:26.977 | INFO | Analyzing chunk frames = [3191296,3191808) | 20260505 12:25:26.977 | INFO | MemInfo: { | 20260505 12:25:26.977 | INFO | "gpu" : " 17 GB", | 20260505 12:25:26.977 | INFO | "huge" : "", | 20260505 12:25:26.977 | INFO | "huge_shared" : " 27 GB", | 20260505 12:25:26.978 | INFO | "malloc" : " 14 GB", | 20260505 12:25:26.978 | INFO | "pinned" : " 29 GB" | 20260505 12:25:26.978 | INFO | } | 20260505 12:25:26.978 | INFO | Perf report for the last 5.13718 seconds: | 20260505 12:25:26.978 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:25:26.978 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:26.978 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:26.978 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:26.978 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 12:25:26.978 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:26.978 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:26.978 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 12:25:26.978 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:26.978 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:26.978 | INFO | Uploaded 36GiB to GPU in 2.20537s (16.3238GiB/s) | 20260505 12:25:26.978 | INFO | Downloaded 34GiB to GPU in 2.04739s (16.6065GiB/s) | 20260505 12:25:32.121 | INFO | Analyzing chunk frames = [3191808,3192320) | 20260505 12:25:32.121 | INFO | MemInfo: { | 20260505 12:25:32.121 | INFO | "gpu" : " 17 GB", | 20260505 12:25:32.121 | INFO | "huge" : "", | 20260505 12:25:32.121 | INFO | "huge_shared" : " 27 GB", | 20260505 12:25:32.121 | INFO | "malloc" : " 14 GB", | 20260505 12:25:32.121 | INFO | "pinned" : " 29 GB" | 20260505 12:25:32.121 | INFO | } | 20260505 12:25:32.121 | INFO | Perf report for the last 5.14313 seconds: | 20260505 12:25:32.121 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:25:32.121 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:32.121 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:32.121 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:32.121 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260505 12:25:32.121 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:32.121 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:32.121 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 12:25:32.121 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:32.121 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:32.121 | INFO | Uploaded 36GiB to GPU in 2.23947s (16.0753GiB/s) | 20260505 12:25:32.121 | INFO | Downloaded 34GiB to GPU in 2.12432s (16.0051GiB/s) | 20260505 12:25:37.256 | INFO | Analyzing chunk frames = [3192320,3192832) | 20260505 12:25:37.256 | INFO | MemInfo: { | 20260505 12:25:37.256 | INFO | "gpu" : " 17 GB", | 20260505 12:25:37.256 | INFO | "huge" : "", | 20260505 12:25:37.256 | INFO | "huge_shared" : " 27 GB", | 20260505 12:25:37.256 | INFO | "malloc" : " 14 GB", | 20260505 12:25:37.256 | INFO | "pinned" : " 29 GB" | 20260505 12:25:37.256 | INFO | } | 20260505 12:25:37.256 | INFO | Perf report for the last 5.13505 seconds: | 20260505 12:25:37.256 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:25:37.256 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:37.256 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:37.256 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:37.256 | INFO | ANALYSIS : 63.7%, 3.0, 98.9ms, 0 | 20260505 12:25:37.256 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:37.256 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:37.256 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:25:37.256 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:37.256 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:37.256 | INFO | Uploaded 36GiB to GPU in 2.22779s (16.1595GiB/s) | 20260505 12:25:37.256 | INFO | Downloaded 34GiB to GPU in 2.12218s (16.0213GiB/s) | 20260505 12:25:42.406 | INFO | Analyzing chunk frames = [3192832,3193344) | 20260505 12:25:42.406 | INFO | MemInfo: { | 20260505 12:25:42.406 | INFO | "gpu" : " 17 GB", | 20260505 12:25:42.406 | INFO | "huge" : "", | 20260505 12:25:42.406 | INFO | "huge_shared" : " 27 GB", | 20260505 12:25:42.406 | INFO | "malloc" : " 14 GB", | 20260505 12:25:42.406 | INFO | "pinned" : " 29 GB" | 20260505 12:25:42.406 | INFO | } | 20260505 12:25:42.406 | INFO | Perf report for the last 5.15005 seconds: | 20260505 12:25:42.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:25:42.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:42.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:42.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:42.406 | INFO | ANALYSIS : 62.9%, 3.0, 98.4ms, 0 | 20260505 12:25:42.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:42.406 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:42.406 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 12:25:42.406 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:42.406 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:42.406 | INFO | Uploaded 36GiB to GPU in 2.15092s (16.737GiB/s) | 20260505 12:25:42.406 | INFO | Downloaded 34GiB to GPU in 2.12223s (16.0209GiB/s) | 20260505 12:25:47.551 | INFO | Analyzing chunk frames = [3193344,3193856) | 20260505 12:25:47.551 | INFO | MemInfo: { | 20260505 12:25:47.551 | INFO | "gpu" : " 17 GB", | 20260505 12:25:47.551 | INFO | "huge" : "", | 20260505 12:25:47.551 | INFO | "huge_shared" : " 27 GB", | 20260505 12:25:47.551 | INFO | "malloc" : " 14 GB", | 20260505 12:25:47.551 | INFO | "pinned" : " 29 GB" | 20260505 12:25:47.551 | INFO | } | 20260505 12:25:47.551 | INFO | Perf report for the last 5.14535 seconds: | 20260505 12:25:47.551 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:25:47.551 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:47.551 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:47.551 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:47.551 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 12:25:47.551 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:47.551 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:47.551 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:25:47.551 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:47.551 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:47.551 | INFO | Uploaded 36GiB to GPU in 2.20141s (16.3532GiB/s) | 20260505 12:25:47.552 | INFO | Downloaded 34GiB to GPU in 2.11217s (16.0972GiB/s) | 20260505 12:25:52.674 | INFO | Analyzing chunk frames = [3193856,3194368) | 20260505 12:25:52.674 | INFO | MemInfo: { | 20260505 12:25:52.674 | INFO | "gpu" : " 17 GB", | 20260505 12:25:52.674 | INFO | "huge" : "", | 20260505 12:25:52.674 | INFO | "huge_shared" : " 27 GB", | 20260505 12:25:52.674 | INFO | "malloc" : " 14 GB", | 20260505 12:25:52.674 | INFO | "pinned" : " 29 GB" | 20260505 12:25:52.674 | INFO | } | 20260505 12:25:52.674 | INFO | Perf report for the last 5.12305 seconds: | 20260505 12:25:52.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:25:52.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:52.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:52.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:52.674 | INFO | ANALYSIS : 64.0%, 3.0, 99.6ms, 0 | 20260505 12:25:52.674 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:52.674 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:52.674 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260505 12:25:52.674 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:52.674 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:52.674 | INFO | Uploaded 36GiB to GPU in 2.16345s (16.6401GiB/s) | 20260505 12:25:52.675 | INFO | Downloaded 34GiB to GPU in 2.17854s (15.6068GiB/s) | 20260505 12:25:57.810 | INFO | Analyzing chunk frames = [3194368,3194880) | 20260505 12:25:57.810 | INFO | MemInfo: { | 20260505 12:25:57.810 | INFO | "gpu" : " 17 GB", | 20260505 12:25:57.810 | INFO | "huge" : "", | 20260505 12:25:57.810 | INFO | "huge_shared" : " 27 GB", | 20260505 12:25:57.810 | INFO | "malloc" : " 14 GB", | 20260505 12:25:57.810 | INFO | "pinned" : " 29 GB" | 20260505 12:25:57.810 | INFO | } | 20260505 12:25:57.810 | INFO | Perf report for the last 5.13618 seconds: | 20260505 12:25:57.810 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:25:57.810 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:57.810 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:57.810 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:57.810 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 12:25:57.810 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:57.811 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:57.811 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:25:57.811 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:57.811 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:25:57.811 | INFO | Uploaded 36GiB to GPU in 2.22419s (16.1857GiB/s) | 20260505 12:25:57.811 | INFO | Downloaded 34GiB to GPU in 2.08025s (16.3442GiB/s) | 20260505 12:25:57.811 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 12:25:57.811 | INFO | Total mem usage: 15.3 GiB | 20260505 12:25:57.811 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 12:25:57.811 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -6.04%) | 20260505 12:25:57.811 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 12:25:57.811 | INFO | High Water Breakdown: | 20260505 12:25:57.811 | INFO | 61.2% : 8.85 GiB ( 93% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 12:25:57.811 | INFO | 38.6% : 5.59 GiB ( 84.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 12:25:57.811 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 12:25:57.811 | INFO | ------------------------------------------ | 20260505 12:25:57.811 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:25:57.811 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 12:25:57.811 | INFO | Total mem usage: 26.5 GiB | 20260505 12:25:57.811 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 12:25:57.811 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 12:25:57.811 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 12:25:57.811 | INFO | High Water Breakdown: | 20260505 12:25:57.811 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:25:57.811 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:25:57.811 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:25:57.811 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:25:57.811 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:25:57.811 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:25:57.811 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 12:25:57.811 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 12:25:57.811 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:25:57.811 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 12:25:57.811 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 12:25:57.811 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 12:25:57.811 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 12:25:57.811 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 12:25:57.811 | INFO | ------------------------------------------ | 20260505 12:25:57.811 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:25:57.811 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 12:25:57.811 | INFO | Total mem usage: 25.6 GiB | 20260505 12:25:57.811 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 12:25:57.811 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 12:25:57.811 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 12:25:57.811 | INFO | High Water Breakdown: | 20260505 12:25:57.811 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 12:25:57.811 | INFO | ------------------------------------------ | 20260505 12:25:57.811 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:25:57.811 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 12:25:57.811 | INFO | Total mem usage: 15.7 GiB | 20260505 12:25:57.811 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 12:25:57.811 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 12:25:57.811 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 12:25:57.811 | INFO | High Water Breakdown: | 20260505 12:25:57.811 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 12:25:57.811 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 12:25:57.811 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 12:25:57.811 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 12:25:57.811 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 12:25:57.811 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 12:25:57.811 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 12:25:57.811 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 12:25:57.811 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 12:25:57.811 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:25:57.811 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 12:25:57.811 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:25:57.811 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 12:25:57.811 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 12:25:57.811 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 12:25:57.811 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 12:25:57.811 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:25:57.811 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 12:25:57.811 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:25:57.811 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:25:57.811 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:25:57.811 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 12:25:57.811 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 12:25:57.811 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:25:57.811 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 12:25:57.811 | INFO | ------------------------------------------ | 20260505 12:25:57.811 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:25:57.811 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 12:25:57.811 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:25:57.811 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:25:57.811 | INFO | PerfReport: Full Execution -- 3.21e+01 / 7.73e+01, 1.37e+01 / 1.72e+00 | 20260505 12:25:57.811 | INFO | PerfReport: 0.29%, AggregateStats -- 9.30e-02 / 1.01e-01, 1.02e-02 / 4.58e-03 | 20260505 12:25:57.811 | INFO | PerfReport: 11.72%, Baseline -- 3.76e+00 / 4.49e+00, 5.93e-01 / 2.25e-01 | 20260505 12:25:57.811 | INFO | PerfReport: 3.31%, Histogram -- 1.06e+00 / 1.51e+00, 2.05e-01 / 8.09e-02 | 20260505 12:25:57.812 | INFO | PerfReport: 15.85%, DME -- 4.07e+01 / 4.53e+01, 4.40e+00 / 1.96e+00 | 20260505 12:25:57.812 | INFO | PerfReport: 56.63%, FrameLabeling -- 1.82e+01 / 2.11e+01, 1.64e+00 / 2.12e+00 | 20260505 12:25:57.812 | INFO | PerfReport: 7.33%, PulseAccumulating -- 2.35e+00 / 2.90e+00, 4.27e-01 / 2.12e-01 | 20260505 12:25:57.812 | INFO | PerfReport: 3.60%, Metrics -- 1.16e+00 / 1.48e+00, 3.27e-01 / 1.83e-01 | 20260505 12:25:57.812 | INFO | PerfReport: 1.27%, PulseSerializing -- 4.08e-01 / 4.54e-01, 7.52e-02 / 3.77e-02 | 20260505 12:25:57.812 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 12:25:57.812 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:25:57.812 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:25:57.812 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260505 12:25:57.812 | INFO | PerfReport: 22.58%, Upload -- 2.29e+01 / 0.00e+00, 8.18e+00 / 0.00e+00 | 20260505 12:25:57.812 | INFO | PerfReport: 21.98%, Download -- 2.22e+01 / 0.00e+00, 8.12e+00 / 0.00e+00 | 20260505 12:25:57.812 | INFO | PerfReport: 31.70%, Compute -- 3.21e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 12:25:57.812 | INFO | PerfReport: 2.16%, LockUpload -- 2.18e+00 / 0.00e+00, 7.73e+00 / 0.00e+00 | 20260505 12:25:57.812 | INFO | PerfReport: 1.28%, LockDownload -- 1.29e+00 / 0.00e+00, 2.81e+00 / 0.00e+00 | 20260505 12:25:57.812 | INFO | PerfReport: 20.28%, LockCompute -- 2.05e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260505 12:25:57.812 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.59e-02 / 0.00e+00 | 20260505 12:25:57.946 | INFO | Starting a round of Baz IO | 20260505 12:26:01.202 | INFO | Populating RTMetrics report | 20260505 12:26:01.203 | INFO | RTMetrics report populated | 20260505 12:26:01.203 | INFO | Opening tmp RTMetrics json file | 20260505 12:26:01.256 | INFO | Serializing RTMetrics to json | 20260505 12:26:01.259 | INFO | Serialization done, swapping tmp file now... | 20260505 12:26:01.259 | INFO | RTMetrics done | 20260505 12:26:01.326 | INFO | Finished a round of Baz IO: Wrote 2.338 GiB in 3.379 seconds | 20260505 12:26:01.326 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.59% / 0.01% / 76.41% | 20260505 12:26:01.326 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.889107% / 3.97364e-06% / 6.75519e-05% | 20260505 12:26:01.326 | INFO | Baz IO overall: | 20260505 12:26:01.326 | INFO | BytesWritten : 1898265472009 | 20260505 12:26:01.326 | INFO | NumEvents : 903575997103 | 20260505 12:26:02.959 | INFO | Analyzing chunk frames = [3194880,3195392) | 20260505 12:26:02.959 | INFO | MemInfo: { | 20260505 12:26:02.959 | INFO | "gpu" : " 17 GB", | 20260505 12:26:02.959 | INFO | "huge" : "", | 20260505 12:26:02.959 | INFO | "huge_shared" : " 27 GB", | 20260505 12:26:02.959 | INFO | "malloc" : " 14 GB", | 20260505 12:26:02.959 | INFO | "pinned" : " 29 GB" | 20260505 12:26:02.959 | INFO | } | 20260505 12:26:02.959 | INFO | Perf report for the last 5.14861 seconds: | 20260505 12:26:02.959 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:26:02.959 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:02.959 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:02.959 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:02.959 | INFO | ANALYSIS : 64.8%, 3.0, 100.4ms, 0 | 20260505 12:26:02.959 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:02.959 | INFO | RT_METRICS : 46.4%, 1.0, 24.4ms, 0 | 20260505 12:26:02.959 | INFO | PRE_HQ : 61.0%, 1.8, 58.4ms, 0 | 20260505 12:26:02.959 | INFO | BAZWRITER : 65.0%, 4.1, 135.7ms, 0 | 20260505 12:26:02.959 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.8ms, 0 | 20260505 12:26:02.959 | INFO | Uploaded 36GiB to GPU in 2.22464s (16.1824GiB/s) | 20260505 12:26:02.959 | INFO | Downloaded 38GiB to GPU in 2.29083s (16.5879GiB/s) | 20260505 12:26:08.096 | INFO | Analyzing chunk frames = [3195392,3195904) | 20260505 12:26:08.096 | INFO | MemInfo: { | 20260505 12:26:08.096 | INFO | "gpu" : " 17 GB", | 20260505 12:26:08.096 | INFO | "huge" : "", | 20260505 12:26:08.096 | INFO | "huge_shared" : " 27 GB", | 20260505 12:26:08.096 | INFO | "malloc" : " 14 GB", | 20260505 12:26:08.096 | INFO | "pinned" : " 29 GB" | 20260505 12:26:08.096 | INFO | } | 20260505 12:26:08.096 | INFO | Perf report for the last 5.13697 seconds: | 20260505 12:26:08.096 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:26:08.096 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:08.096 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:08.096 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:08.096 | INFO | ANALYSIS : 64.3%, 3.0, 100.3ms, 0 | 20260505 12:26:08.096 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:08.096 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:08.096 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 12:26:08.096 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:08.096 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:08.096 | INFO | Uploaded 36GiB to GPU in 2.23811s (16.085GiB/s) | 20260505 12:26:08.096 | INFO | Downloaded 34GiB to GPU in 2.09474s (16.2311GiB/s) | 20260505 12:26:13.239 | INFO | Analyzing chunk frames = [3195904,3196416) | 20260505 12:26:13.239 | INFO | MemInfo: { | 20260505 12:26:13.239 | INFO | "gpu" : " 17 GB", | 20260505 12:26:13.239 | INFO | "huge" : "", | 20260505 12:26:13.240 | INFO | "huge_shared" : " 27 GB", | 20260505 12:26:13.240 | INFO | "malloc" : " 14 GB", | 20260505 12:26:13.240 | INFO | "pinned" : " 29 GB" | 20260505 12:26:13.240 | INFO | } | 20260505 12:26:13.240 | INFO | Perf report for the last 5.14357 seconds: | 20260505 12:26:13.240 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:26:13.240 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:13.240 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:13.240 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:13.240 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 12:26:13.240 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:13.240 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:13.240 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 12:26:13.240 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:13.240 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:13.240 | INFO | Uploaded 36GiB to GPU in 2.22337s (16.1916GiB/s) | 20260505 12:26:13.240 | INFO | Downloaded 34GiB to GPU in 2.12398s (16.0077GiB/s) | 20260505 12:26:18.374 | INFO | Analyzing chunk frames = [3196416,3196928) | 20260505 12:26:18.374 | INFO | MemInfo: { | 20260505 12:26:18.374 | INFO | "gpu" : " 17 GB", | 20260505 12:26:18.374 | INFO | "huge" : "", | 20260505 12:26:18.374 | INFO | "huge_shared" : " 27 GB", | 20260505 12:26:18.374 | INFO | "malloc" : " 14 GB", | 20260505 12:26:18.374 | INFO | "pinned" : " 29 GB" | 20260505 12:26:18.374 | INFO | } | 20260505 12:26:18.374 | INFO | Perf report for the last 5.13465 seconds: | 20260505 12:26:18.374 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:26:18.374 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:18.374 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:18.374 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:18.374 | INFO | ANALYSIS : 63.3%, 3.0, 98.2ms, 0 | 20260505 12:26:18.374 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:18.374 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:18.374 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:26:18.374 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:18.374 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:18.375 | INFO | Uploaded 36GiB to GPU in 2.20865s (16.2995GiB/s) | 20260505 12:26:18.375 | INFO | Downloaded 34GiB to GPU in 2.13056s (15.9582GiB/s) | 20260505 12:26:23.514 | INFO | Analyzing chunk frames = [3196928,3197440) | 20260505 12:26:23.514 | INFO | MemInfo: { | 20260505 12:26:23.514 | INFO | "gpu" : " 17 GB", | 20260505 12:26:23.514 | INFO | "huge" : "", | 20260505 12:26:23.514 | INFO | "huge_shared" : " 27 GB", | 20260505 12:26:23.514 | INFO | "malloc" : " 14 GB", | 20260505 12:26:23.514 | INFO | "pinned" : " 29 GB" | 20260505 12:26:23.514 | INFO | } | 20260505 12:26:23.514 | INFO | Perf report for the last 5.14006 seconds: | 20260505 12:26:23.514 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:26:23.514 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:23.514 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:23.514 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:23.514 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 12:26:23.514 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:23.514 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:23.514 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:26:23.514 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:23.514 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:23.515 | INFO | Uploaded 36GiB to GPU in 2.17732s (16.5341GiB/s) | 20260505 12:26:23.515 | INFO | Downloaded 34GiB to GPU in 2.12761s (15.9804GiB/s) | 20260505 12:26:28.657 | INFO | Analyzing chunk frames = [3197440,3197952) | 20260505 12:26:28.657 | INFO | MemInfo: { | 20260505 12:26:28.657 | INFO | "gpu" : " 17 GB", | 20260505 12:26:28.657 | INFO | "huge" : "", | 20260505 12:26:28.657 | INFO | "huge_shared" : " 27 GB", | 20260505 12:26:28.657 | INFO | "malloc" : " 14 GB", | 20260505 12:26:28.657 | INFO | "pinned" : " 29 GB" | 20260505 12:26:28.657 | INFO | } | 20260505 12:26:28.657 | INFO | Perf report for the last 5.14299 seconds: | 20260505 12:26:28.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:26:28.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:28.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:28.657 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:28.657 | INFO | ANALYSIS : 63.2%, 3.0, 98.8ms, 0 | 20260505 12:26:28.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:28.657 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:28.657 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 12:26:28.658 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:28.658 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:28.658 | INFO | Uploaded 36GiB to GPU in 2.15418s (16.7117GiB/s) | 20260505 12:26:28.658 | INFO | Downloaded 34GiB to GPU in 2.10843s (16.1258GiB/s) | 20260505 12:26:33.793 | INFO | Analyzing chunk frames = [3197952,3198464) | 20260505 12:26:33.793 | INFO | MemInfo: { | 20260505 12:26:33.793 | INFO | "gpu" : " 17 GB", | 20260505 12:26:33.793 | INFO | "huge" : "", | 20260505 12:26:33.793 | INFO | "huge_shared" : " 27 GB", | 20260505 12:26:33.793 | INFO | "malloc" : " 14 GB", | 20260505 12:26:33.793 | INFO | "pinned" : " 29 GB" | 20260505 12:26:33.793 | INFO | } | 20260505 12:26:33.793 | INFO | Perf report for the last 5.1358 seconds: | 20260505 12:26:33.793 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:26:33.793 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:33.793 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:33.793 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:33.793 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 12:26:33.793 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:33.793 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:33.793 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 12:26:33.793 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:33.793 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:33.793 | INFO | Uploaded 36GiB to GPU in 2.14641s (16.7722GiB/s) | 20260505 12:26:33.794 | INFO | Downloaded 34GiB to GPU in 2.1727s (15.6488GiB/s) | 20260505 12:26:38.927 | INFO | Analyzing chunk frames = [3198464,3198976) | 20260505 12:26:38.927 | INFO | MemInfo: { | 20260505 12:26:38.928 | INFO | "gpu" : " 17 GB", | 20260505 12:26:38.928 | INFO | "huge" : "", | 20260505 12:26:38.928 | INFO | "huge_shared" : " 27 GB", | 20260505 12:26:38.928 | INFO | "malloc" : " 14 GB", | 20260505 12:26:38.928 | INFO | "pinned" : " 29 GB" | 20260505 12:26:38.928 | INFO | } | 20260505 12:26:38.928 | INFO | Perf report for the last 5.13451 seconds: | 20260505 12:26:38.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:26:38.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:38.928 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:38.928 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:38.928 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 12:26:38.928 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:38.928 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:38.928 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 12:26:38.928 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:38.928 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:38.928 | INFO | Uploaded 36GiB to GPU in 2.22079s (16.2104GiB/s) | 20260505 12:26:38.928 | INFO | Downloaded 34GiB to GPU in 2.10825s (16.1271GiB/s) | 20260505 12:26:39.067 | INFO | Starting a round of Baz IO | 20260505 12:26:42.319 | INFO | Populating RTMetrics report | 20260505 12:26:42.319 | INFO | RTMetrics report populated | 20260505 12:26:42.319 | INFO | Opening tmp RTMetrics json file | 20260505 12:26:42.343 | INFO | Serializing RTMetrics to json | 20260505 12:26:42.382 | INFO | Serialization done, swapping tmp file now... | 20260505 12:26:42.389 | INFO | RTMetrics done | 20260505 12:26:42.428 | INFO | Finished a round of Baz IO: Wrote 2.338 GiB in 3.361 seconds | 20260505 12:26:42.428 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.58% / 0.01% / 76.41% | 20260505 12:26:42.428 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.887799% / 1.58946e-05% / 5.96046e-05% | 20260505 12:26:42.428 | INFO | Baz IO overall: | 20260505 12:26:42.429 | INFO | BytesWritten : 1900775799394 | 20260505 12:26:42.429 | INFO | NumEvents : 904766819992 | 20260505 12:26:44.068 | INFO | Analyzing chunk frames = [3198976,3199488) | 20260505 12:26:44.068 | INFO | MemInfo: { | 20260505 12:26:44.068 | INFO | "gpu" : " 17 GB", | 20260505 12:26:44.068 | INFO | "huge" : "", | 20260505 12:26:44.068 | INFO | "huge_shared" : " 27 GB", | 20260505 12:26:44.068 | INFO | "malloc" : " 14 GB", | 20260505 12:26:44.068 | INFO | "pinned" : " 29 GB" | 20260505 12:26:44.068 | INFO | } | 20260505 12:26:44.068 | INFO | Perf report for the last 5.14025 seconds: | 20260505 12:26:44.068 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:26:44.068 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:44.068 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:44.068 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:44.068 | INFO | ANALYSIS : 64.8%, 3.0, 100.5ms, 0 | 20260505 12:26:44.068 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:44.068 | INFO | RT_METRICS : 46.7%, 1.0, 24.5ms, 0 | 20260505 12:26:44.068 | INFO | PRE_HQ : 61.2%, 1.8, 58.1ms, 0 | 20260505 12:26:44.068 | INFO | BAZWRITER : 64.7%, 4.0, 134.5ms, 0 | 20260505 12:26:44.068 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260505 12:26:44.068 | INFO | Uploaded 36GiB to GPU in 2.20663s (16.3144GiB/s) | 20260505 12:26:44.068 | INFO | Downloaded 38GiB to GPU in 2.29124s (16.5849GiB/s) | 20260505 12:26:49.212 | INFO | Analyzing chunk frames = [3199488,3200000) | 20260505 12:26:49.212 | INFO | MemInfo: { | 20260505 12:26:49.212 | INFO | "gpu" : " 17 GB", | 20260505 12:26:49.212 | INFO | "huge" : "", | 20260505 12:26:49.212 | INFO | "huge_shared" : " 27 GB", | 20260505 12:26:49.212 | INFO | "malloc" : " 14 GB", | 20260505 12:26:49.212 | INFO | "pinned" : " 29 GB" | 20260505 12:26:49.212 | INFO | } | 20260505 12:26:49.212 | INFO | Perf report for the last 5.14378 seconds: | 20260505 12:26:49.212 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:26:49.212 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:49.212 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:49.212 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:49.212 | INFO | ANALYSIS : 64.2%, 3.0, 100.4ms, 0 | 20260505 12:26:49.212 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:49.212 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:49.212 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 12:26:49.212 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:49.212 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:49.212 | INFO | Uploaded 36GiB to GPU in 2.23705s (16.0927GiB/s) | 20260505 12:26:49.212 | INFO | Downloaded 34GiB to GPU in 2.07929s (16.3517GiB/s) | 20260505 12:26:54.354 | INFO | Analyzing chunk frames = [3200000,3200512) | 20260505 12:26:54.354 | INFO | MemInfo: { | 20260505 12:26:54.354 | INFO | "gpu" : " 17 GB", | 20260505 12:26:54.354 | INFO | "huge" : "", | 20260505 12:26:54.354 | INFO | "huge_shared" : " 27 GB", | 20260505 12:26:54.354 | INFO | "malloc" : " 14 GB", | 20260505 12:26:54.354 | INFO | "pinned" : " 29 GB" | 20260505 12:26:54.354 | INFO | } | 20260505 12:26:54.354 | INFO | Perf report for the last 5.14194 seconds: | 20260505 12:26:54.354 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:26:54.354 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:54.354 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:54.354 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:54.354 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 12:26:54.354 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:54.354 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:54.354 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 12:26:54.354 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:54.354 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:54.354 | INFO | Uploaded 36GiB to GPU in 2.20057s (16.3594GiB/s) | 20260505 12:26:54.354 | INFO | Downloaded 34GiB to GPU in 2.13254s (15.9434GiB/s) | 20260505 12:26:59.496 | INFO | Analyzing chunk frames = [3200512,3201024) | 20260505 12:26:59.496 | INFO | MemInfo: { | 20260505 12:26:59.496 | INFO | "gpu" : " 17 GB", | 20260505 12:26:59.496 | INFO | "huge" : "", | 20260505 12:26:59.496 | INFO | "huge_shared" : " 27 GB", | 20260505 12:26:59.496 | INFO | "malloc" : " 14 GB", | 20260505 12:26:59.496 | INFO | "pinned" : " 29 GB" | 20260505 12:26:59.496 | INFO | } | 20260505 12:26:59.496 | INFO | Perf report for the last 5.14244 seconds: | 20260505 12:26:59.496 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:26:59.496 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:59.496 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:59.496 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:59.496 | INFO | ANALYSIS : 63.1%, 3.0, 98.2ms, 0 | 20260505 12:26:59.496 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:59.496 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:59.496 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:26:59.496 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:59.496 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:26:59.496 | INFO | Uploaded 36GiB to GPU in 2.20162s (16.3516GiB/s) | 20260505 12:26:59.497 | INFO | Downloaded 34GiB to GPU in 2.13279s (15.9416GiB/s) | 20260505 12:27:04.625 | INFO | Analyzing chunk frames = [3201024,3201536) | 20260505 12:27:04.625 | INFO | MemInfo: { | 20260505 12:27:04.625 | INFO | "gpu" : " 17 GB", | 20260505 12:27:04.625 | INFO | "huge" : "", | 20260505 12:27:04.625 | INFO | "huge_shared" : " 27 GB", | 20260505 12:27:04.625 | INFO | "malloc" : " 14 GB", | 20260505 12:27:04.625 | INFO | "pinned" : " 29 GB" | 20260505 12:27:04.625 | INFO | } | 20260505 12:27:04.625 | INFO | Perf report for the last 5.12865 seconds: | 20260505 12:27:04.625 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:27:04.625 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:04.625 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:04.625 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:04.625 | INFO | ANALYSIS : 63.2%, 3.0, 98.4ms, 0 | 20260505 12:27:04.625 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:04.625 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:04.625 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 12:27:04.625 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:04.625 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:04.625 | INFO | Uploaded 36GiB to GPU in 2.16975s (16.5917GiB/s) | 20260505 12:27:04.625 | INFO | Downloaded 34GiB to GPU in 2.12492s (16.0006GiB/s) | 20260505 12:27:09.766 | INFO | Analyzing chunk frames = [3201536,3202048) | 20260505 12:27:09.766 | INFO | MemInfo: { | 20260505 12:27:09.766 | INFO | "gpu" : " 17 GB", | 20260505 12:27:09.766 | INFO | "huge" : "", | 20260505 12:27:09.766 | INFO | "huge_shared" : " 27 GB", | 20260505 12:27:09.766 | INFO | "malloc" : " 14 GB", | 20260505 12:27:09.766 | INFO | "pinned" : " 29 GB" | 20260505 12:27:09.766 | INFO | } | 20260505 12:27:09.766 | INFO | Perf report for the last 5.14093 seconds: | 20260505 12:27:09.766 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:27:09.766 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:09.766 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:09.766 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:09.766 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 12:27:09.766 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:09.766 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:09.766 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 12:27:09.766 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:09.766 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:09.766 | INFO | Uploaded 36GiB to GPU in 2.20244s (16.3455GiB/s) | 20260505 12:27:09.766 | INFO | Downloaded 34GiB to GPU in 2.12156s (16.0259GiB/s) | 20260505 12:27:14.912 | INFO | Analyzing chunk frames = [3202048,3202560) | 20260505 12:27:14.912 | INFO | MemInfo: { | 20260505 12:27:14.912 | INFO | "gpu" : " 17 GB", | 20260505 12:27:14.912 | INFO | "huge" : "", | 20260505 12:27:14.912 | INFO | "huge_shared" : " 27 GB", | 20260505 12:27:14.912 | INFO | "malloc" : " 14 GB", | 20260505 12:27:14.912 | INFO | "pinned" : " 29 GB" | 20260505 12:27:14.912 | INFO | } | 20260505 12:27:14.912 | INFO | Perf report for the last 5.14611 seconds: | 20260505 12:27:14.912 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:27:14.912 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:14.912 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:14.912 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:14.912 | INFO | ANALYSIS : 63.4%, 3.0, 99.2ms, 0 | 20260505 12:27:14.912 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:14.912 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:14.912 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 12:27:14.912 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:14.912 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:14.912 | INFO | Uploaded 36GiB to GPU in 2.18474s (16.478GiB/s) | 20260505 12:27:14.912 | INFO | Downloaded 34GiB to GPU in 2.14401s (15.8582GiB/s) | 20260505 12:27:20.050 | INFO | Analyzing chunk frames = [3202560,3203072) | 20260505 12:27:20.050 | INFO | MemInfo: { | 20260505 12:27:20.050 | INFO | "gpu" : " 17 GB", | 20260505 12:27:20.050 | INFO | "huge" : "", | 20260505 12:27:20.050 | INFO | "huge_shared" : " 27 GB", | 20260505 12:27:20.050 | INFO | "malloc" : " 14 GB", | 20260505 12:27:20.050 | INFO | "pinned" : " 29 GB" | 20260505 12:27:20.050 | INFO | } | 20260505 12:27:20.050 | INFO | Perf report for the last 5.13811 seconds: | 20260505 12:27:20.050 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:27:20.050 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:20.050 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:20.050 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:20.050 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 12:27:20.050 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:20.050 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:20.050 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 12:27:20.050 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:20.050 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:20.050 | INFO | Uploaded 36GiB to GPU in 2.19431s (16.406GiB/s) | 20260505 12:27:20.050 | INFO | Downloaded 34GiB to GPU in 2.11431s (16.0809GiB/s) | 20260505 12:27:20.051 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 12:27:20.051 | INFO | Total mem usage: 15.3 GiB | 20260505 12:27:20.051 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 12:27:20.051 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -7.45%) | 20260505 12:27:20.051 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 12:27:20.051 | INFO | High Water Breakdown: | 20260505 12:27:20.051 | INFO | 60.9% : 8.69 GiB ( 91.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 12:27:20.051 | INFO | 38.9% : 5.55 GiB ( 83.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 12:27:20.051 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 12:27:20.051 | INFO | ------------------------------------------ | 20260505 12:27:20.051 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:27:20.051 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 12:27:20.051 | INFO | Total mem usage: 26.5 GiB | 20260505 12:27:20.051 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 12:27:20.051 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 12:27:20.051 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 12:27:20.051 | INFO | High Water Breakdown: | 20260505 12:27:20.051 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:27:20.051 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:27:20.051 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:27:20.051 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:27:20.051 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:27:20.051 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:27:20.051 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 12:27:20.051 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 12:27:20.051 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:27:20.051 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 12:27:20.051 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 12:27:20.051 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 12:27:20.051 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 12:27:20.051 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 12:27:20.051 | INFO | ------------------------------------------ | 20260505 12:27:20.051 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:27:20.051 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 12:27:20.051 | INFO | Total mem usage: 25.6 GiB | 20260505 12:27:20.051 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 12:27:20.051 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 12:27:20.051 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 12:27:20.051 | INFO | High Water Breakdown: | 20260505 12:27:20.051 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 12:27:20.051 | INFO | ------------------------------------------ | 20260505 12:27:20.051 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:27:20.051 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 12:27:20.051 | INFO | Total mem usage: 15.7 GiB | 20260505 12:27:20.051 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 12:27:20.051 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 12:27:20.051 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 12:27:20.051 | INFO | High Water Breakdown: | 20260505 12:27:20.051 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 12:27:20.051 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 12:27:20.051 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 12:27:20.051 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 12:27:20.051 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 12:27:20.051 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 12:27:20.051 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 12:27:20.051 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 12:27:20.051 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 12:27:20.051 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:27:20.051 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 12:27:20.051 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:27:20.051 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 12:27:20.051 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 12:27:20.051 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 12:27:20.051 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 12:27:20.051 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:27:20.051 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 12:27:20.051 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:27:20.051 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:27:20.051 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:27:20.051 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 12:27:20.051 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 12:27:20.051 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:27:20.051 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 12:27:20.051 | INFO | ------------------------------------------ | 20260505 12:27:20.051 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:27:20.052 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 12:27:20.052 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:27:20.052 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:27:20.052 | INFO | PerfReport: Full Execution -- 3.20e+01 / 7.73e+01, 1.40e+01 / 1.80e+00 | 20260505 12:27:20.052 | INFO | PerfReport: 0.29%, AggregateStats -- 9.27e-02 / 1.00e-01, 1.03e-02 / 4.74e-03 | 20260505 12:27:20.052 | INFO | PerfReport: 11.72%, Baseline -- 3.75e+00 / 4.46e+00, 5.89e-01 / 2.37e-01 | 20260505 12:27:20.052 | INFO | PerfReport: 3.29%, Histogram -- 1.05e+00 / 1.50e+00, 2.02e-01 / 7.65e-02 | 20260505 12:27:20.052 | INFO | PerfReport: 16.01%, DME -- 4.10e+01 / 4.59e+01, 4.72e+00 / 2.01e+00 | 20260505 12:27:20.052 | INFO | PerfReport: 56.33%, FrameLabeling -- 1.80e+01 / 2.05e+01, 1.45e+00 / 1.70e+00 | 20260505 12:27:20.052 | INFO | PerfReport: 7.44%, PulseAccumulating -- 2.38e+00 / 2.90e+00, 4.43e-01 / 1.76e-01 | 20260505 12:27:20.052 | INFO | PerfReport: 3.64%, Metrics -- 1.16e+00 / 1.46e+00, 3.29e-01 / 1.90e-01 | 20260505 12:27:20.052 | INFO | PerfReport: 1.28%, PulseSerializing -- 4.09e-01 / 4.59e-01, 7.48e-02 / 3.69e-02 | 20260505 12:27:20.052 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 12:27:20.052 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:27:20.052 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:27:20.052 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260505 12:27:20.052 | INFO | PerfReport: 22.63%, Upload -- 2.29e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260505 12:27:20.052 | INFO | PerfReport: 22.05%, Download -- 2.23e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260505 12:27:20.052 | INFO | PerfReport: 31.59%, Compute -- 3.20e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260505 12:27:20.052 | INFO | PerfReport: 2.25%, LockUpload -- 2.28e+00 / 0.00e+00, 7.82e+00 / 0.00e+00 | 20260505 12:27:20.052 | INFO | PerfReport: 1.33%, LockDownload -- 1.34e+00 / 0.00e+00, 2.89e+00 / 0.00e+00 | 20260505 12:27:20.052 | INFO | PerfReport: 20.14%, LockCompute -- 2.04e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260505 12:27:20.052 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.76e-02 / 0.00e+00 | 20260505 12:27:20.179 | INFO | Starting a round of Baz IO | 20260505 12:27:23.426 | INFO | Populating RTMetrics report | 20260505 12:27:23.427 | INFO | RTMetrics report populated | 20260505 12:27:23.427 | INFO | Opening tmp RTMetrics json file | 20260505 12:27:23.427 | INFO | Serializing RTMetrics to json | 20260505 12:27:23.428 | INFO | Serialization done, swapping tmp file now... | 20260505 12:27:23.428 | INFO | RTMetrics done | 20260505 12:27:23.546 | INFO | Finished a round of Baz IO: Wrote 2.338 GiB in 3.367 seconds | 20260505 12:27:23.546 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.58% / 0.01% / 76.42% | 20260505 12:27:23.546 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.886432% / 1.19209e-05% / 7.15256e-05% | 20260505 12:27:23.546 | INFO | Baz IO overall: | 20260505 12:27:23.546 | INFO | BytesWritten : 1903286484802 | 20260505 12:27:23.546 | INFO | NumEvents : 905958170192 | 20260505 12:27:25.185 | INFO | Analyzing chunk frames = [3203072,3203584) | 20260505 12:27:25.185 | INFO | MemInfo: { | 20260505 12:27:25.185 | INFO | "gpu" : " 17 GB", | 20260505 12:27:25.185 | INFO | "huge" : "", | 20260505 12:27:25.185 | INFO | "huge_shared" : " 27 GB", | 20260505 12:27:25.185 | INFO | "malloc" : " 14 GB", | 20260505 12:27:25.185 | INFO | "pinned" : " 29 GB" | 20260505 12:27:25.185 | INFO | } | 20260505 12:27:25.185 | INFO | Perf report for the last 5.13511 seconds: | 20260505 12:27:25.185 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:27:25.185 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:25.185 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:25.185 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:25.185 | INFO | ANALYSIS : 64.6%, 3.0, 100.0ms, 0 | 20260505 12:27:25.185 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:25.185 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260505 12:27:25.185 | INFO | PRE_HQ : 60.9%, 1.8, 58.1ms, 0 | 20260505 12:27:25.185 | INFO | BAZWRITER : 64.9%, 4.1, 135.2ms, 0 | 20260505 12:27:25.185 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 12:27:25.185 | INFO | Uploaded 36GiB to GPU in 2.1933s (16.4136GiB/s) | 20260505 12:27:25.186 | INFO | Downloaded 38GiB to GPU in 2.27472s (16.7054GiB/s) | 20260505 12:27:30.321 | INFO | Analyzing chunk frames = [3203584,3204096) | 20260505 12:27:30.321 | INFO | MemInfo: { | 20260505 12:27:30.321 | INFO | "gpu" : " 17 GB", | 20260505 12:27:30.321 | INFO | "huge" : "", | 20260505 12:27:30.321 | INFO | "huge_shared" : " 27 GB", | 20260505 12:27:30.322 | INFO | "malloc" : " 14 GB", | 20260505 12:27:30.322 | INFO | "pinned" : " 29 GB" | 20260505 12:27:30.322 | INFO | } | 20260505 12:27:30.322 | INFO | Perf report for the last 5.13645 seconds: | 20260505 12:27:30.322 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:27:30.322 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:30.322 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:30.322 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:30.322 | INFO | ANALYSIS : 64.1%, 3.0, 99.2ms, 0 | 20260505 12:27:30.322 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:30.322 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:30.322 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:27:30.322 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:30.322 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:30.322 | INFO | Uploaded 36GiB to GPU in 2.22909s (16.1501GiB/s) | 20260505 12:27:30.322 | INFO | Downloaded 34GiB to GPU in 2.0746s (16.3887GiB/s) | 20260505 12:27:35.468 | INFO | Analyzing chunk frames = [3204096,3204608) | 20260505 12:27:35.469 | INFO | MemInfo: { | 20260505 12:27:35.469 | INFO | "gpu" : " 17 GB", | 20260505 12:27:35.469 | INFO | "huge" : "", | 20260505 12:27:35.469 | INFO | "huge_shared" : " 27 GB", | 20260505 12:27:35.469 | INFO | "malloc" : " 14 GB", | 20260505 12:27:35.469 | INFO | "pinned" : " 29 GB" | 20260505 12:27:35.469 | INFO | } | 20260505 12:27:35.469 | INFO | Perf report for the last 5.14714 seconds: | 20260505 12:27:35.469 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:27:35.469 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:35.469 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:35.469 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:35.469 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 12:27:35.469 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:35.469 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:35.469 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 12:27:35.469 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:35.469 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:35.469 | INFO | Uploaded 36GiB to GPU in 2.211s (16.2822GiB/s) | 20260505 12:27:35.469 | INFO | Downloaded 34GiB to GPU in 2.12175s (16.0245GiB/s) | 20260505 12:27:40.604 | INFO | Analyzing chunk frames = [3204608,3205120) | 20260505 12:27:40.604 | INFO | MemInfo: { | 20260505 12:27:40.604 | INFO | "gpu" : " 17 GB", | 20260505 12:27:40.604 | INFO | "huge" : "", | 20260505 12:27:40.604 | INFO | "huge_shared" : " 27 GB", | 20260505 12:27:40.604 | INFO | "malloc" : " 14 GB", | 20260505 12:27:40.604 | INFO | "pinned" : " 29 GB" | 20260505 12:27:40.604 | INFO | } | 20260505 12:27:40.604 | INFO | Perf report for the last 5.13491 seconds: | 20260505 12:27:40.604 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:27:40.604 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:40.604 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:40.604 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:40.604 | INFO | ANALYSIS : 63.1%, 3.0, 97.9ms, 0 | 20260505 12:27:40.604 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:40.604 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:40.604 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:27:40.604 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:40.604 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:40.604 | INFO | Uploaded 36GiB to GPU in 2.1956s (16.3964GiB/s) | 20260505 12:27:40.604 | INFO | Downloaded 34GiB to GPU in 2.13351s (15.9362GiB/s) | 20260505 12:27:45.744 | INFO | Analyzing chunk frames = [3205120,3205632) | 20260505 12:27:45.744 | INFO | MemInfo: { | 20260505 12:27:45.744 | INFO | "gpu" : " 17 GB", | 20260505 12:27:45.744 | INFO | "huge" : "", | 20260505 12:27:45.744 | INFO | "huge_shared" : " 27 GB", | 20260505 12:27:45.744 | INFO | "malloc" : " 14 GB", | 20260505 12:27:45.744 | INFO | "pinned" : " 29 GB" | 20260505 12:27:45.744 | INFO | } | 20260505 12:27:45.744 | INFO | Perf report for the last 5.14055 seconds: | 20260505 12:27:45.744 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:27:45.744 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:45.744 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:45.744 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:45.744 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 12:27:45.744 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:45.744 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:45.744 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:27:45.744 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:45.744 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:45.745 | INFO | Uploaded 36GiB to GPU in 2.17458s (16.5549GiB/s) | 20260505 12:27:45.745 | INFO | Downloaded 34GiB to GPU in 2.12961s (15.9654GiB/s) | 20260505 12:27:50.881 | INFO | Analyzing chunk frames = [3205632,3206144) | 20260505 12:27:50.882 | INFO | MemInfo: { | 20260505 12:27:50.882 | INFO | "gpu" : " 17 GB", | 20260505 12:27:50.882 | INFO | "huge" : "", | 20260505 12:27:50.882 | INFO | "huge_shared" : " 27 GB", | 20260505 12:27:50.882 | INFO | "malloc" : " 14 GB", | 20260505 12:27:50.882 | INFO | "pinned" : " 29 GB" | 20260505 12:27:50.882 | INFO | } | 20260505 12:27:50.882 | INFO | Perf report for the last 5.13741 seconds: | 20260505 12:27:50.882 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:27:50.882 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:50.882 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:50.882 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:50.882 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 12:27:50.882 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:50.882 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:50.882 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 12:27:50.882 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:50.882 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:50.882 | INFO | Uploaded 36GiB to GPU in 2.15338s (16.7179GiB/s) | 20260505 12:27:50.882 | INFO | Downloaded 34GiB to GPU in 2.11298s (16.091GiB/s) | 20260505 12:27:56.020 | INFO | Analyzing chunk frames = [3206144,3206656) | 20260505 12:27:56.020 | INFO | MemInfo: { | 20260505 12:27:56.020 | INFO | "gpu" : " 17 GB", | 20260505 12:27:56.020 | INFO | "huge" : "", | 20260505 12:27:56.020 | INFO | "huge_shared" : " 27 GB", | 20260505 12:27:56.020 | INFO | "malloc" : " 14 GB", | 20260505 12:27:56.020 | INFO | "pinned" : " 29 GB" | 20260505 12:27:56.020 | INFO | } | 20260505 12:27:56.020 | INFO | Perf report for the last 5.13849 seconds: | 20260505 12:27:56.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:27:56.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:56.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:56.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:56.020 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 12:27:56.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:56.020 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:56.020 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 12:27:56.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:56.020 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:27:56.020 | INFO | Uploaded 36GiB to GPU in 2.17041s (16.5867GiB/s) | 20260505 12:27:56.021 | INFO | Downloaded 34GiB to GPU in 2.16434s (15.7092GiB/s) | 20260505 12:28:01.170 | INFO | Analyzing chunk frames = [3206656,3207168) | 20260505 12:28:01.170 | INFO | MemInfo: { | 20260505 12:28:01.170 | INFO | "gpu" : " 17 GB", | 20260505 12:28:01.170 | INFO | "huge" : "", | 20260505 12:28:01.170 | INFO | "huge_shared" : " 27 GB", | 20260505 12:28:01.170 | INFO | "malloc" : " 14 GB", | 20260505 12:28:01.170 | INFO | "pinned" : " 29 GB" | 20260505 12:28:01.170 | INFO | } | 20260505 12:28:01.170 | INFO | Perf report for the last 5.15026 seconds: | 20260505 12:28:01.170 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:28:01.170 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:01.170 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:01.171 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:01.171 | INFO | ANALYSIS : 63.6%, 3.0, 99.6ms, 0 | 20260505 12:28:01.171 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:01.171 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:01.171 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:28:01.171 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:01.171 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:01.171 | INFO | Uploaded 36GiB to GPU in 2.1724s (16.5715GiB/s) | 20260505 12:28:01.171 | INFO | Downloaded 34GiB to GPU in 2.11658s (16.0636GiB/s) | 20260505 12:28:01.307 | INFO | Starting a round of Baz IO | 20260505 12:28:04.531 | INFO | Populating RTMetrics report | 20260505 12:28:04.532 | INFO | RTMetrics report populated | 20260505 12:28:04.532 | INFO | Opening tmp RTMetrics json file | 20260505 12:28:04.580 | INFO | Serializing RTMetrics to json | 20260505 12:28:04.606 | INFO | Serialization done, swapping tmp file now... | 20260505 12:28:04.612 | INFO | RTMetrics done | 20260505 12:28:04.641 | INFO | Finished a round of Baz IO: Wrote 2.335 GiB in 3.333 seconds | 20260505 12:28:04.641 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.57% / 0.01% / 76.42% | 20260505 12:28:04.641 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.879502% / 1.58946e-05% / 7.94729e-05% | 20260505 12:28:04.641 | INFO | Baz IO overall: | 20260505 12:28:04.641 | INFO | BytesWritten : 1905793525177 | 20260505 12:28:04.641 | INFO | NumEvents : 907146851808 | 20260505 12:28:06.308 | INFO | Analyzing chunk frames = [3207168,3207680) | 20260505 12:28:06.308 | INFO | MemInfo: { | 20260505 12:28:06.308 | INFO | "gpu" : " 17 GB", | 20260505 12:28:06.308 | INFO | "huge" : "", | 20260505 12:28:06.308 | INFO | "huge_shared" : " 27 GB", | 20260505 12:28:06.308 | INFO | "malloc" : " 14 GB", | 20260505 12:28:06.308 | INFO | "pinned" : " 29 GB" | 20260505 12:28:06.308 | INFO | } | 20260505 12:28:06.308 | INFO | Perf report for the last 5.13802 seconds: | 20260505 12:28:06.308 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:28:06.308 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:06.309 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:06.309 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:06.309 | INFO | ANALYSIS : 64.3%, 3.0, 99.5ms, 0 | 20260505 12:28:06.309 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:06.309 | INFO | RT_METRICS : 46.7%, 1.0, 24.5ms, 0 | 20260505 12:28:06.309 | INFO | PRE_HQ : 61.1%, 1.8, 58.1ms, 0 | 20260505 12:28:06.309 | INFO | BAZWRITER : 64.2%, 4.1, 134.3ms, 0 | 20260505 12:28:06.309 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260505 12:28:06.309 | INFO | Uploaded 36GiB to GPU in 2.18816s (16.4522GiB/s) | 20260505 12:28:06.309 | INFO | Downloaded 38GiB to GPU in 2.26739s (16.7594GiB/s) | 20260505 12:28:11.441 | INFO | Analyzing chunk frames = [3207680,3208192) | 20260505 12:28:11.441 | INFO | MemInfo: { | 20260505 12:28:11.441 | INFO | "gpu" : " 17 GB", | 20260505 12:28:11.441 | INFO | "huge" : "", | 20260505 12:28:11.441 | INFO | "huge_shared" : " 27 GB", | 20260505 12:28:11.441 | INFO | "malloc" : " 14 GB", | 20260505 12:28:11.441 | INFO | "pinned" : " 29 GB" | 20260505 12:28:11.441 | INFO | } | 20260505 12:28:11.441 | INFO | Perf report for the last 5.13225 seconds: | 20260505 12:28:11.441 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:28:11.441 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:11.441 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:11.441 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:11.441 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 12:28:11.441 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:11.441 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:11.441 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 12:28:11.441 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:11.441 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:11.442 | INFO | Uploaded 36GiB to GPU in 2.17639s (16.5412GiB/s) | 20260505 12:28:11.442 | INFO | Downloaded 34GiB to GPU in 2.03183s (16.7337GiB/s) | 20260505 12:28:16.572 | INFO | Analyzing chunk frames = [3208192,3208704) | 20260505 12:28:16.572 | INFO | MemInfo: { | 20260505 12:28:16.572 | INFO | "gpu" : " 17 GB", | 20260505 12:28:16.572 | INFO | "huge" : "", | 20260505 12:28:16.572 | INFO | "huge_shared" : " 27 GB", | 20260505 12:28:16.572 | INFO | "malloc" : " 14 GB", | 20260505 12:28:16.572 | INFO | "pinned" : " 29 GB" | 20260505 12:28:16.572 | INFO | } | 20260505 12:28:16.572 | INFO | Perf report for the last 5.13102 seconds: | 20260505 12:28:16.572 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:28:16.572 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:16.572 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:16.572 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:16.572 | INFO | ANALYSIS : 63.4%, 3.0, 98.5ms, 0 | 20260505 12:28:16.572 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:16.572 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:16.572 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:28:16.572 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:16.572 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:16.572 | INFO | Uploaded 36GiB to GPU in 2.21041s (16.2866GiB/s) | 20260505 12:28:16.572 | INFO | Downloaded 34GiB to GPU in 2.1129s (16.0917GiB/s) | 20260505 12:28:21.719 | INFO | Analyzing chunk frames = [3208704,3209216) | 20260505 12:28:21.719 | INFO | MemInfo: { | 20260505 12:28:21.719 | INFO | "gpu" : " 17 GB", | 20260505 12:28:21.719 | INFO | "huge" : "", | 20260505 12:28:21.719 | INFO | "huge_shared" : " 27 GB", | 20260505 12:28:21.719 | INFO | "malloc" : " 14 GB", | 20260505 12:28:21.719 | INFO | "pinned" : " 29 GB" | 20260505 12:28:21.719 | INFO | } | 20260505 12:28:21.719 | INFO | Perf report for the last 5.14761 seconds: | 20260505 12:28:21.719 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:28:21.719 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:21.719 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:21.719 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:21.719 | INFO | ANALYSIS : 63.2%, 3.0, 98.4ms, 0 | 20260505 12:28:21.720 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:21.720 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:21.720 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 12:28:21.720 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:21.720 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:21.720 | INFO | Uploaded 36GiB to GPU in 2.21133s (16.2798GiB/s) | 20260505 12:28:21.720 | INFO | Downloaded 34GiB to GPU in 2.13254s (15.9435GiB/s) | 20260505 12:28:26.864 | INFO | Analyzing chunk frames = [3209216,3209728) | 20260505 12:28:26.864 | INFO | MemInfo: { | 20260505 12:28:26.864 | INFO | "gpu" : " 17 GB", | 20260505 12:28:26.864 | INFO | "huge" : "", | 20260505 12:28:26.864 | INFO | "huge_shared" : " 27 GB", | 20260505 12:28:26.864 | INFO | "malloc" : " 14 GB", | 20260505 12:28:26.864 | INFO | "pinned" : " 29 GB" | 20260505 12:28:26.864 | INFO | } | 20260505 12:28:26.864 | INFO | Perf report for the last 5.14476 seconds: | 20260505 12:28:26.864 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:28:26.864 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:26.864 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:26.864 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:26.864 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 12:28:26.864 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:26.864 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:26.864 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 12:28:26.864 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:26.864 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:26.865 | INFO | Uploaded 36GiB to GPU in 2.16975s (16.5918GiB/s) | 20260505 12:28:26.865 | INFO | Downloaded 34GiB to GPU in 2.12863s (15.9727GiB/s) | 20260505 12:28:31.998 | INFO | Analyzing chunk frames = [3209728,3210240) | 20260505 12:28:31.998 | INFO | MemInfo: { | 20260505 12:28:31.998 | INFO | "gpu" : " 17 GB", | 20260505 12:28:31.998 | INFO | "huge" : "", | 20260505 12:28:31.998 | INFO | "huge_shared" : " 27 GB", | 20260505 12:28:31.998 | INFO | "malloc" : " 14 GB", | 20260505 12:28:31.998 | INFO | "pinned" : " 29 GB" | 20260505 12:28:31.998 | INFO | } | 20260505 12:28:31.998 | INFO | Perf report for the last 5.13395 seconds: | 20260505 12:28:31.998 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:28:31.998 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:31.998 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:31.998 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:31.998 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 12:28:31.998 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:31.998 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:31.998 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:28:31.998 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:31.998 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:31.998 | INFO | Uploaded 36GiB to GPU in 2.16356s (16.6393GiB/s) | 20260505 12:28:31.998 | INFO | Downloaded 34GiB to GPU in 2.11345s (16.0874GiB/s) | 20260505 12:28:37.133 | INFO | Analyzing chunk frames = [3210240,3210752) | 20260505 12:28:37.133 | INFO | MemInfo: { | 20260505 12:28:37.133 | INFO | "gpu" : " 17 GB", | 20260505 12:28:37.133 | INFO | "huge" : "", | 20260505 12:28:37.134 | INFO | "huge_shared" : " 27 GB", | 20260505 12:28:37.134 | INFO | "malloc" : " 14 GB", | 20260505 12:28:37.134 | INFO | "pinned" : " 29 GB" | 20260505 12:28:37.134 | INFO | } | 20260505 12:28:37.134 | INFO | Perf report for the last 5.13548 seconds: | 20260505 12:28:37.134 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:28:37.134 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:37.134 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:37.134 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:37.134 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 12:28:37.134 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:37.134 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:37.134 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 12:28:37.134 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:37.134 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:37.134 | INFO | Uploaded 36GiB to GPU in 2.17091s (16.5829GiB/s) | 20260505 12:28:37.134 | INFO | Downloaded 34GiB to GPU in 2.14537s (15.8481GiB/s) | 20260505 12:28:42.276 | INFO | Analyzing chunk frames = [3210752,3211264) | 20260505 12:28:42.276 | INFO | MemInfo: { | 20260505 12:28:42.276 | INFO | "gpu" : " 17 GB", | 20260505 12:28:42.276 | INFO | "huge" : "", | 20260505 12:28:42.276 | INFO | "huge_shared" : " 27 GB", | 20260505 12:28:42.276 | INFO | "malloc" : " 14 GB", | 20260505 12:28:42.276 | INFO | "pinned" : " 29 GB" | 20260505 12:28:42.276 | INFO | } | 20260505 12:28:42.276 | INFO | Perf report for the last 5.14233 seconds: | 20260505 12:28:42.276 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:28:42.276 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:42.276 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:42.276 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:42.276 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 12:28:42.276 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:42.276 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:42.276 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 12:28:42.276 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:42.276 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:42.276 | INFO | Uploaded 36GiB to GPU in 2.21598s (16.2456GiB/s) | 20260505 12:28:42.276 | INFO | Downloaded 34GiB to GPU in 2.08772s (16.2857GiB/s) | 20260505 12:28:42.276 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 12:28:42.276 | INFO | Total mem usage: 15.3 GiB | 20260505 12:28:42.276 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 12:28:42.276 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -6.07%) | 20260505 12:28:42.276 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 12:28:42.276 | INFO | High Water Breakdown: | 20260505 12:28:42.276 | INFO | 61.2% : 8.84 GiB ( 92.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 12:28:42.276 | INFO | 38.7% : 5.59 GiB ( 84.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 12:28:42.276 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 12:28:42.276 | INFO | ------------------------------------------ | 20260505 12:28:42.276 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:28:42.276 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 12:28:42.276 | INFO | Total mem usage: 26.5 GiB | 20260505 12:28:42.276 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 12:28:42.276 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 12:28:42.276 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 12:28:42.276 | INFO | High Water Breakdown: | 20260505 12:28:42.276 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:28:42.276 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:28:42.277 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:28:42.277 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:28:42.277 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:28:42.277 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:28:42.277 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 12:28:42.277 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 12:28:42.277 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:28:42.277 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 12:28:42.277 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 12:28:42.277 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 12:28:42.277 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 12:28:42.277 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 12:28:42.277 | INFO | ------------------------------------------ | 20260505 12:28:42.277 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:28:42.277 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 12:28:42.277 | INFO | Total mem usage: 25.6 GiB | 20260505 12:28:42.277 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 12:28:42.277 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 12:28:42.277 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 12:28:42.277 | INFO | High Water Breakdown: | 20260505 12:28:42.277 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 12:28:42.277 | INFO | ------------------------------------------ | 20260505 12:28:42.277 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:28:42.277 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 12:28:42.277 | INFO | Total mem usage: 15.7 GiB | 20260505 12:28:42.277 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 12:28:42.277 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 12:28:42.277 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 12:28:42.277 | INFO | High Water Breakdown: | 20260505 12:28:42.277 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 12:28:42.277 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 12:28:42.277 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 12:28:42.277 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 12:28:42.277 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 12:28:42.277 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 12:28:42.277 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 12:28:42.277 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 12:28:42.277 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 12:28:42.277 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:28:42.277 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 12:28:42.277 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:28:42.277 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 12:28:42.277 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 12:28:42.277 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 12:28:42.277 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 12:28:42.277 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:28:42.277 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 12:28:42.277 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:28:42.277 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:28:42.277 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:28:42.277 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 12:28:42.277 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 12:28:42.277 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:28:42.277 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 12:28:42.277 | INFO | ------------------------------------------ | 20260505 12:28:42.277 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:28:42.277 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 12:28:42.277 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:28:42.277 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:28:42.277 | INFO | PerfReport: Full Execution -- 3.20e+01 / 7.68e+01, 1.37e+01 / 1.60e+00 | 20260505 12:28:42.277 | INFO | PerfReport: 0.29%, AggregateStats -- 9.30e-02 / 1.01e-01, 1.04e-02 / 5.81e-03 | 20260505 12:28:42.277 | INFO | PerfReport: 11.75%, Baseline -- 3.76e+00 / 4.47e+00, 5.87e-01 / 2.29e-01 | 20260505 12:28:42.277 | INFO | PerfReport: 3.31%, Histogram -- 1.06e+00 / 1.48e+00, 2.00e-01 / 1.09e-01 | 20260505 12:28:42.277 | INFO | PerfReport: 15.88%, DME -- 4.07e+01 / 4.50e+01, 4.38e+00 / 2.27e+00 | 20260505 12:28:42.277 | INFO | PerfReport: 56.38%, FrameLabeling -- 1.80e+01 / 2.09e+01, 1.49e+00 / 2.36e+00 | 20260505 12:28:42.277 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.38e+00 / 2.85e+00, 4.36e-01 / 2.45e-01 | 20260505 12:28:42.277 | INFO | PerfReport: 3.65%, Metrics -- 1.17e+00 / 1.46e+00, 3.31e-01 / 1.85e-01 | 20260505 12:28:42.277 | INFO | PerfReport: 1.28%, PulseSerializing -- 4.09e-01 / 4.58e-01, 7.55e-02 / 3.57e-02 | 20260505 12:28:42.277 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 12:28:42.277 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:28:42.277 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:28:42.277 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260505 12:28:42.277 | INFO | PerfReport: 22.57%, Upload -- 2.28e+01 / 0.00e+00, 8.03e+00 / 0.00e+00 | 20260505 12:28:42.277 | INFO | PerfReport: 22.01%, Download -- 2.22e+01 / 0.00e+00, 8.09e+00 / 0.00e+00 | 20260505 12:28:42.277 | INFO | PerfReport: 31.70%, Compute -- 3.20e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 12:28:42.277 | INFO | PerfReport: 2.13%, LockUpload -- 2.15e+00 / 0.00e+00, 7.71e+00 / 0.00e+00 | 20260505 12:28:42.277 | INFO | PerfReport: 1.36%, LockDownload -- 1.37e+00 / 0.00e+00, 2.90e+00 / 0.00e+00 | 20260505 12:28:42.277 | INFO | PerfReport: 20.22%, LockCompute -- 2.04e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260505 12:28:42.277 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.77e-02 / 0.00e+00 | 20260505 12:28:42.406 | INFO | Starting a round of Baz IO | 20260505 12:28:45.660 | INFO | Populating RTMetrics report | 20260505 12:28:45.660 | INFO | RTMetrics report populated | 20260505 12:28:45.660 | INFO | Opening tmp RTMetrics json file | 20260505 12:28:45.661 | INFO | Serializing RTMetrics to json | 20260505 12:28:45.669 | INFO | Serialization done, swapping tmp file now... | 20260505 12:28:45.687 | INFO | RTMetrics done | 20260505 12:28:45.782 | INFO | Finished a round of Baz IO: Wrote 2.336 GiB in 3.376 seconds | 20260505 12:28:45.782 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.56% / 0.01% / 76.43% | 20260505 12:28:45.782 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.880822% / 1.58946e-05% / 8.74201e-05% | 20260505 12:28:45.782 | INFO | Baz IO overall: | 20260505 12:28:45.782 | INFO | BytesWritten : 1908301981249 | 20260505 12:28:45.782 | INFO | NumEvents : 908336416079 | 20260505 12:28:47.418 | INFO | Analyzing chunk frames = [3211264,3211776) | 20260505 12:28:47.418 | INFO | MemInfo: { | 20260505 12:28:47.418 | INFO | "gpu" : " 17 GB", | 20260505 12:28:47.418 | INFO | "huge" : "", | 20260505 12:28:47.418 | INFO | "huge_shared" : " 27 GB", | 20260505 12:28:47.418 | INFO | "malloc" : " 14 GB", | 20260505 12:28:47.418 | INFO | "pinned" : " 29 GB" | 20260505 12:28:47.418 | INFO | } | 20260505 12:28:47.418 | INFO | Perf report for the last 5.14192 seconds: | 20260505 12:28:47.418 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:28:47.418 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:47.418 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:47.418 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:47.418 | INFO | ANALYSIS : 64.7%, 2.9, 99.7ms, 0 | 20260505 12:28:47.418 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:47.418 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260505 12:28:47.418 | INFO | PRE_HQ : 61.1%, 1.8, 58.4ms, 0 | 20260505 12:28:47.418 | INFO | BAZWRITER : 64.8%, 4.1, 136.2ms, 0 | 20260505 12:28:47.418 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260505 12:28:47.418 | INFO | Uploaded 36GiB to GPU in 2.20215s (16.3476GiB/s) | 20260505 12:28:47.418 | INFO | Downloaded 38GiB to GPU in 2.2837s (16.6397GiB/s) | 20260505 12:28:52.558 | INFO | Analyzing chunk frames = [3211776,3212288) | 20260505 12:28:52.558 | INFO | MemInfo: { | 20260505 12:28:52.558 | INFO | "gpu" : " 17 GB", | 20260505 12:28:52.558 | INFO | "huge" : "", | 20260505 12:28:52.558 | INFO | "huge_shared" : " 27 GB", | 20260505 12:28:52.558 | INFO | "malloc" : " 14 GB", | 20260505 12:28:52.558 | INFO | "pinned" : " 29 GB" | 20260505 12:28:52.558 | INFO | } | 20260505 12:28:52.558 | INFO | Perf report for the last 5.1405 seconds: | 20260505 12:28:52.558 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:28:52.558 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:52.558 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:52.558 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:52.558 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 12:28:52.559 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:52.559 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:52.559 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 12:28:52.559 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:52.559 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:52.559 | INFO | Uploaded 36GiB to GPU in 2.17634s (16.5415GiB/s) | 20260505 12:28:52.559 | INFO | Downloaded 34GiB to GPU in 2.02926s (16.7549GiB/s) | 20260505 12:28:57.697 | INFO | Analyzing chunk frames = [3212288,3212800) | 20260505 12:28:57.697 | INFO | MemInfo: { | 20260505 12:28:57.697 | INFO | "gpu" : " 17 GB", | 20260505 12:28:57.697 | INFO | "huge" : "", | 20260505 12:28:57.697 | INFO | "huge_shared" : " 27 GB", | 20260505 12:28:57.697 | INFO | "malloc" : " 14 GB", | 20260505 12:28:57.697 | INFO | "pinned" : " 29 GB" | 20260505 12:28:57.698 | INFO | } | 20260505 12:28:57.698 | INFO | Perf report for the last 5.13913 seconds: | 20260505 12:28:57.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:28:57.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:57.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:57.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:57.698 | INFO | ANALYSIS : 63.2%, 3.0, 98.3ms, 0 | 20260505 12:28:57.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:57.698 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:57.698 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 12:28:57.698 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:57.698 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:28:57.698 | INFO | Uploaded 36GiB to GPU in 2.1501s (16.7434GiB/s) | 20260505 12:28:57.698 | INFO | Downloaded 34GiB to GPU in 2.13153s (15.951GiB/s) | 20260505 12:29:02.827 | INFO | Analyzing chunk frames = [3212800,3213312) | 20260505 12:29:02.827 | INFO | MemInfo: { | 20260505 12:29:02.827 | INFO | "gpu" : " 17 GB", | 20260505 12:29:02.827 | INFO | "huge" : "", | 20260505 12:29:02.827 | INFO | "huge_shared" : " 27 GB", | 20260505 12:29:02.827 | INFO | "malloc" : " 14 GB", | 20260505 12:29:02.827 | INFO | "pinned" : " 29 GB" | 20260505 12:29:02.827 | INFO | } | 20260505 12:29:02.827 | INFO | Perf report for the last 5.12913 seconds: | 20260505 12:29:02.827 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:29:02.827 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:02.827 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:02.827 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:02.827 | INFO | ANALYSIS : 63.4%, 3.0, 98.3ms, 0 | 20260505 12:29:02.827 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:02.827 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:02.827 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 12:29:02.827 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:02.827 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:02.827 | INFO | Uploaded 36GiB to GPU in 2.21414s (16.2591GiB/s) | 20260505 12:29:02.827 | INFO | Downloaded 34GiB to GPU in 2.12978s (15.9641GiB/s) | 20260505 12:29:07.976 | INFO | Analyzing chunk frames = [3213312,3213824) | 20260505 12:29:07.976 | INFO | MemInfo: { | 20260505 12:29:07.976 | INFO | "gpu" : " 17 GB", | 20260505 12:29:07.976 | INFO | "huge" : "", | 20260505 12:29:07.976 | INFO | "huge_shared" : " 27 GB", | 20260505 12:29:07.976 | INFO | "malloc" : " 14 GB", | 20260505 12:29:07.976 | INFO | "pinned" : " 29 GB" | 20260505 12:29:07.976 | INFO | } | 20260505 12:29:07.976 | INFO | Perf report for the last 5.14942 seconds: | 20260505 12:29:07.976 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:29:07.976 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:07.976 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:07.976 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:07.976 | INFO | ANALYSIS : 63.1%, 3.0, 98.7ms, 0 | 20260505 12:29:07.976 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:07.976 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:07.976 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 12:29:07.976 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:07.976 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:07.976 | INFO | Uploaded 36GiB to GPU in 2.15594s (16.6981GiB/s) | 20260505 12:29:07.976 | INFO | Downloaded 34GiB to GPU in 2.12843s (15.9742GiB/s) | 20260505 12:29:13.120 | INFO | Analyzing chunk frames = [3213824,3214336) | 20260505 12:29:13.120 | INFO | MemInfo: { | 20260505 12:29:13.120 | INFO | "gpu" : " 17 GB", | 20260505 12:29:13.120 | INFO | "huge" : "", | 20260505 12:29:13.120 | INFO | "huge_shared" : " 27 GB", | 20260505 12:29:13.120 | INFO | "malloc" : " 14 GB", | 20260505 12:29:13.120 | INFO | "pinned" : " 29 GB" | 20260505 12:29:13.120 | INFO | } | 20260505 12:29:13.120 | INFO | Perf report for the last 5.14423 seconds: | 20260505 12:29:13.120 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:29:13.120 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:13.120 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:13.120 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:13.120 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 12:29:13.120 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:13.120 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:13.121 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:29:13.121 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:13.121 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:13.121 | INFO | Uploaded 36GiB to GPU in 2.13578s (16.8557GiB/s) | 20260505 12:29:13.121 | INFO | Downloaded 34GiB to GPU in 2.1016s (16.1782GiB/s) | 20260505 12:29:18.259 | INFO | Analyzing chunk frames = [3214336,3214848) | 20260505 12:29:18.259 | INFO | MemInfo: { | 20260505 12:29:18.259 | INFO | "gpu" : " 17 GB", | 20260505 12:29:18.259 | INFO | "huge" : "", | 20260505 12:29:18.259 | INFO | "huge_shared" : " 27 GB", | 20260505 12:29:18.259 | INFO | "malloc" : " 14 GB", | 20260505 12:29:18.259 | INFO | "pinned" : " 29 GB" | 20260505 12:29:18.259 | INFO | } | 20260505 12:29:18.259 | INFO | Perf report for the last 5.13868 seconds: | 20260505 12:29:18.259 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:29:18.259 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:18.260 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:18.260 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:18.260 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 12:29:18.260 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:18.260 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:18.260 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 12:29:18.260 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:18.260 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:18.260 | INFO | Uploaded 36GiB to GPU in 2.14988s (16.7452GiB/s) | 20260505 12:29:18.260 | INFO | Downloaded 34GiB to GPU in 2.16675s (15.6917GiB/s) | 20260505 12:29:23.384 | INFO | Analyzing chunk frames = [3214848,3215360) | 20260505 12:29:23.384 | INFO | MemInfo: { | 20260505 12:29:23.384 | INFO | "gpu" : " 17 GB", | 20260505 12:29:23.384 | INFO | "huge" : "", | 20260505 12:29:23.384 | INFO | "huge_shared" : " 27 GB", | 20260505 12:29:23.384 | INFO | "malloc" : " 14 GB", | 20260505 12:29:23.384 | INFO | "pinned" : " 29 GB" | 20260505 12:29:23.384 | INFO | } | 20260505 12:29:23.384 | INFO | Perf report for the last 5.12533 seconds: | 20260505 12:29:23.384 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:29:23.384 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:23.384 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:23.384 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:23.384 | INFO | ANALYSIS : 63.7%, 3.0, 99.2ms, 0 | 20260505 12:29:23.384 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:23.384 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:23.384 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260505 12:29:23.384 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:23.384 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:23.385 | INFO | Uploaded 36GiB to GPU in 2.12064s (16.976GiB/s) | 20260505 12:29:23.385 | INFO | Downloaded 34GiB to GPU in 2.09985s (16.1916GiB/s) | 20260505 12:29:23.519 | INFO | Starting a round of Baz IO | 20260505 12:29:26.776 | INFO | Populating RTMetrics report | 20260505 12:29:26.776 | INFO | RTMetrics report populated | 20260505 12:29:26.776 | INFO | Opening tmp RTMetrics json file | 20260505 12:29:26.780 | INFO | Serializing RTMetrics to json | 20260505 12:29:26.782 | INFO | Serialization done, swapping tmp file now... | 20260505 12:29:26.782 | INFO | RTMetrics done | 20260505 12:29:26.913 | INFO | Finished a round of Baz IO: Wrote 2.334 GiB in 3.393 seconds | 20260505 12:29:26.913 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.56% / 0.01% / 76.44% | 20260505 12:29:26.913 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.876232% / 3.97364e-06% / 6.75519e-05% | 20260505 12:29:26.913 | INFO | Baz IO overall: | 20260505 12:29:26.913 | INFO | BytesWritten : 1910808557780 | 20260505 12:29:26.913 | INFO | NumEvents : 909524709414 | 20260505 12:29:28.526 | INFO | Analyzing chunk frames = [3215360,3215872) | 20260505 12:29:28.526 | INFO | MemInfo: { | 20260505 12:29:28.526 | INFO | "gpu" : " 17 GB", | 20260505 12:29:28.526 | INFO | "huge" : "", | 20260505 12:29:28.527 | INFO | "huge_shared" : " 27 GB", | 20260505 12:29:28.527 | INFO | "malloc" : " 14 GB", | 20260505 12:29:28.527 | INFO | "pinned" : " 29 GB" | 20260505 12:29:28.527 | INFO | } | 20260505 12:29:28.527 | INFO | Perf report for the last 5.14226 seconds: | 20260505 12:29:28.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:29:28.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:28.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:28.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:28.527 | INFO | ANALYSIS : 64.8%, 2.9, 100.2ms, 0 | 20260505 12:29:28.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:28.527 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260505 12:29:28.527 | INFO | PRE_HQ : 61.1%, 1.9, 58.7ms, 0 | 20260505 12:29:28.527 | INFO | BAZWRITER : 65.3%, 4.1, 137.1ms, 0 | 20260505 12:29:28.527 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 12:29:28.527 | INFO | Uploaded 36GiB to GPU in 2.20432s (16.3316GiB/s) | 20260505 12:29:28.527 | INFO | Downloaded 38GiB to GPU in 2.29179s (16.5809GiB/s) | 20260505 12:29:33.669 | INFO | Analyzing chunk frames = [3215872,3216384) | 20260505 12:29:33.669 | INFO | MemInfo: { | 20260505 12:29:33.669 | INFO | "gpu" : " 17 GB", | 20260505 12:29:33.669 | INFO | "huge" : "", | 20260505 12:29:33.669 | INFO | "huge_shared" : " 27 GB", | 20260505 12:29:33.669 | INFO | "malloc" : " 14 GB", | 20260505 12:29:33.669 | INFO | "pinned" : " 29 GB" | 20260505 12:29:33.669 | INFO | } | 20260505 12:29:33.669 | INFO | Perf report for the last 5.14226 seconds: | 20260505 12:29:33.669 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:29:33.669 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:33.669 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:33.669 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:33.669 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 12:29:33.669 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:33.669 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:33.669 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 12:29:33.669 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:33.669 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:33.669 | INFO | Uploaded 36GiB to GPU in 2.23513s (16.1064GiB/s) | 20260505 12:29:33.669 | INFO | Downloaded 34GiB to GPU in 2.10136s (16.18GiB/s) | 20260505 12:29:38.814 | INFO | Analyzing chunk frames = [3216384,3216896) | 20260505 12:29:38.814 | INFO | MemInfo: { | 20260505 12:29:38.814 | INFO | "gpu" : " 17 GB", | 20260505 12:29:38.814 | INFO | "huge" : "", | 20260505 12:29:38.814 | INFO | "huge_shared" : " 27 GB", | 20260505 12:29:38.814 | INFO | "malloc" : " 14 GB", | 20260505 12:29:38.814 | INFO | "pinned" : " 29 GB" | 20260505 12:29:38.814 | INFO | } | 20260505 12:29:38.814 | INFO | Perf report for the last 5.14497 seconds: | 20260505 12:29:38.814 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:29:38.814 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:38.814 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:38.814 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:38.814 | INFO | ANALYSIS : 63.2%, 3.0, 98.4ms, 0 | 20260505 12:29:38.814 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:38.814 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:38.814 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 12:29:38.814 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:38.814 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:38.814 | INFO | Uploaded 36GiB to GPU in 2.16651s (16.6166GiB/s) | 20260505 12:29:38.814 | INFO | Downloaded 34GiB to GPU in 2.12881s (15.9713GiB/s) | 20260505 12:29:43.946 | INFO | Analyzing chunk frames = [3216896,3217408) | 20260505 12:29:43.946 | INFO | MemInfo: { | 20260505 12:29:43.946 | INFO | "gpu" : " 17 GB", | 20260505 12:29:43.946 | INFO | "huge" : "", | 20260505 12:29:43.946 | INFO | "huge_shared" : " 27 GB", | 20260505 12:29:43.946 | INFO | "malloc" : " 14 GB", | 20260505 12:29:43.946 | INFO | "pinned" : " 29 GB" | 20260505 12:29:43.946 | INFO | } | 20260505 12:29:43.946 | INFO | Perf report for the last 5.13201 seconds: | 20260505 12:29:43.946 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:29:43.946 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:43.946 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:43.946 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:43.946 | INFO | ANALYSIS : 63.4%, 3.0, 98.3ms, 0 | 20260505 12:29:43.946 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:43.946 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:43.946 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 12:29:43.946 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:43.946 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:43.946 | INFO | Uploaded 36GiB to GPU in 2.20605s (16.3188GiB/s) | 20260505 12:29:43.946 | INFO | Downloaded 34GiB to GPU in 2.13616s (15.9164GiB/s) | 20260505 12:29:49.085 | INFO | Analyzing chunk frames = [3217408,3217920) | 20260505 12:29:49.085 | INFO | MemInfo: { | 20260505 12:29:49.085 | INFO | "gpu" : " 17 GB", | 20260505 12:29:49.085 | INFO | "huge" : "", | 20260505 12:29:49.085 | INFO | "huge_shared" : " 27 GB", | 20260505 12:29:49.085 | INFO | "malloc" : " 14 GB", | 20260505 12:29:49.086 | INFO | "pinned" : " 29 GB" | 20260505 12:29:49.086 | INFO | } | 20260505 12:29:49.086 | INFO | Perf report for the last 5.13966 seconds: | 20260505 12:29:49.086 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:29:49.086 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:49.086 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:49.086 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:49.086 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 12:29:49.086 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:49.086 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:49.086 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 12:29:49.086 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:49.086 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:49.086 | INFO | Uploaded 36GiB to GPU in 2.14899s (16.7521GiB/s) | 20260505 12:29:49.086 | INFO | Downloaded 34GiB to GPU in 2.12562s (15.9954GiB/s) | 20260505 12:29:54.232 | INFO | Analyzing chunk frames = [3217920,3218432) | 20260505 12:29:54.233 | INFO | MemInfo: { | 20260505 12:29:54.233 | INFO | "gpu" : " 17 GB", | 20260505 12:29:54.233 | INFO | "huge" : "", | 20260505 12:29:54.233 | INFO | "huge_shared" : " 27 GB", | 20260505 12:29:54.233 | INFO | "malloc" : " 14 GB", | 20260505 12:29:54.233 | INFO | "pinned" : " 29 GB" | 20260505 12:29:54.233 | INFO | } | 20260505 12:29:54.233 | INFO | Perf report for the last 5.14713 seconds: | 20260505 12:29:54.233 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:29:54.233 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:54.233 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:54.233 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:54.233 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 12:29:54.233 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:54.233 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:54.233 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 12:29:54.233 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:54.233 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:54.233 | INFO | Uploaded 36GiB to GPU in 2.19477s (16.4026GiB/s) | 20260505 12:29:54.233 | INFO | Downloaded 34GiB to GPU in 2.1197s (16.04GiB/s) | 20260505 12:29:59.363 | INFO | Analyzing chunk frames = [3218432,3218944) | 20260505 12:29:59.363 | INFO | MemInfo: { | 20260505 12:29:59.363 | INFO | "gpu" : " 17 GB", | 20260505 12:29:59.363 | INFO | "huge" : "", | 20260505 12:29:59.363 | INFO | "huge_shared" : " 27 GB", | 20260505 12:29:59.363 | INFO | "malloc" : " 14 GB", | 20260505 12:29:59.363 | INFO | "pinned" : " 29 GB" | 20260505 12:29:59.363 | INFO | } | 20260505 12:29:59.363 | INFO | Perf report for the last 5.13035 seconds: | 20260505 12:29:59.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:29:59.363 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:59.363 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:59.363 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:59.363 | INFO | ANALYSIS : 63.5%, 3.0, 99.0ms, 0 | 20260505 12:29:59.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:59.363 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:59.363 | INFO | PRE_HQ : 29.8%, 1.0, 15.4ms, 0 | 20260505 12:29:59.363 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:59.363 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:29:59.364 | INFO | Uploaded 36GiB to GPU in 2.15196s (16.7289GiB/s) | 20260505 12:29:59.364 | INFO | Downloaded 34GiB to GPU in 2.17571s (15.6271GiB/s) | 20260505 12:30:04.505 | INFO | Analyzing chunk frames = [3218944,3219456) | 20260505 12:30:04.505 | INFO | MemInfo: { | 20260505 12:30:04.505 | INFO | "gpu" : " 17 GB", | 20260505 12:30:04.505 | INFO | "huge" : "", | 20260505 12:30:04.505 | INFO | "huge_shared" : " 27 GB", | 20260505 12:30:04.505 | INFO | "malloc" : " 14 GB", | 20260505 12:30:04.505 | INFO | "pinned" : " 29 GB" | 20260505 12:30:04.505 | INFO | } | 20260505 12:30:04.505 | INFO | Perf report for the last 5.14214 seconds: | 20260505 12:30:04.505 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:30:04.505 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:04.505 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:04.505 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:04.505 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 12:30:04.505 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:04.505 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:04.505 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 12:30:04.505 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:04.505 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:04.506 | INFO | Uploaded 36GiB to GPU in 2.14655s (16.7711GiB/s) | 20260505 12:30:04.506 | INFO | Downloaded 34GiB to GPU in 2.11437s (16.0804GiB/s) | 20260505 12:30:04.506 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 12:30:04.506 | INFO | Total mem usage: 15.3 GiB | 20260505 12:30:04.506 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 12:30:04.506 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -6.87%) | 20260505 12:30:04.506 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 12:30:04.506 | INFO | High Water Breakdown: | 20260505 12:30:04.506 | INFO | 61.1% : 8.76 GiB ( 92.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 12:30:04.506 | INFO | 38.8% : 5.56 GiB ( 83.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 12:30:04.506 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 12:30:04.506 | INFO | ------------------------------------------ | 20260505 12:30:04.506 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:30:04.506 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 12:30:04.506 | INFO | Total mem usage: 26.5 GiB | 20260505 12:30:04.506 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 12:30:04.506 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 12:30:04.506 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 12:30:04.506 | INFO | High Water Breakdown: | 20260505 12:30:04.506 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:30:04.506 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:30:04.506 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:30:04.506 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:30:04.506 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:30:04.506 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:30:04.506 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 12:30:04.506 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 12:30:04.506 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:30:04.506 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 12:30:04.506 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 12:30:04.506 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 12:30:04.506 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 12:30:04.506 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 12:30:04.506 | INFO | ------------------------------------------ | 20260505 12:30:04.506 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:30:04.506 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 12:30:04.506 | INFO | Total mem usage: 25.6 GiB | 20260505 12:30:04.506 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 12:30:04.506 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 12:30:04.506 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 12:30:04.506 | INFO | High Water Breakdown: | 20260505 12:30:04.506 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 12:30:04.506 | INFO | ------------------------------------------ | 20260505 12:30:04.506 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:30:04.506 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 12:30:04.506 | INFO | Total mem usage: 15.7 GiB | 20260505 12:30:04.506 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 12:30:04.506 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 12:30:04.506 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 12:30:04.506 | INFO | High Water Breakdown: | 20260505 12:30:04.506 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 12:30:04.507 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 12:30:04.507 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 12:30:04.507 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 12:30:04.507 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 12:30:04.507 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 12:30:04.507 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 12:30:04.507 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 12:30:04.507 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 12:30:04.507 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:30:04.507 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 12:30:04.507 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:30:04.507 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 12:30:04.507 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 12:30:04.507 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 12:30:04.507 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 12:30:04.507 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:30:04.507 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 12:30:04.507 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:30:04.507 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:30:04.507 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:30:04.507 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 12:30:04.507 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 12:30:04.507 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:30:04.507 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 12:30:04.507 | INFO | ------------------------------------------ | 20260505 12:30:04.507 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:30:04.507 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 12:30:04.507 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:30:04.507 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:30:04.507 | INFO | PerfReport: Full Execution -- 3.20e+01 / 7.76e+01, 1.39e+01 / 1.99e+00 | 20260505 12:30:04.507 | INFO | PerfReport: 0.29%, AggregateStats -- 9.30e-02 / 1.00e-01, 1.03e-02 / 6.03e-03 | 20260505 12:30:04.507 | INFO | PerfReport: 11.75%, Baseline -- 3.76e+00 / 4.49e+00, 5.92e-01 / 1.71e-01 | 20260505 12:30:04.507 | INFO | PerfReport: 3.30%, Histogram -- 1.06e+00 / 1.51e+00, 2.03e-01 / 9.17e-02 | 20260505 12:30:04.507 | INFO | PerfReport: 16.11%, DME -- 4.12e+01 / 4.68e+01, 4.97e+00 / 2.05e+00 | 20260505 12:30:04.507 | INFO | PerfReport: 56.23%, FrameLabeling -- 1.80e+01 / 1.97e+01, 1.38e+00 / 1.84e+00 | 20260505 12:30:04.507 | INFO | PerfReport: 7.43%, PulseAccumulating -- 2.38e+00 / 2.98e+00, 4.53e-01 / 1.60e-01 | 20260505 12:30:04.507 | INFO | PerfReport: 3.61%, Metrics -- 1.16e+00 / 1.49e+00, 3.25e-01 / 1.94e-01 | 20260505 12:30:04.507 | INFO | PerfReport: 1.28%, PulseSerializing -- 4.09e-01 / 4.59e-01, 7.50e-02 / 3.52e-02 | 20260505 12:30:04.507 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 12:30:04.507 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:30:04.507 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:30:04.507 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.56e+01 / 0.00e+00 | 20260505 12:30:04.507 | INFO | PerfReport: 22.39%, Upload -- 2.26e+01 / 0.00e+00, 7.97e+00 / 0.00e+00 | 20260505 12:30:04.507 | INFO | PerfReport: 22.07%, Download -- 2.23e+01 / 0.00e+00, 8.17e+00 / 0.00e+00 | 20260505 12:30:04.507 | INFO | PerfReport: 31.66%, Compute -- 3.20e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 12:30:04.507 | INFO | PerfReport: 2.16%, LockUpload -- 2.18e+00 / 0.00e+00, 7.87e+00 / 0.00e+00 | 20260505 12:30:04.507 | INFO | PerfReport: 1.48%, LockDownload -- 1.49e+00 / 0.00e+00, 3.13e+00 / 0.00e+00 | 20260505 12:30:04.507 | INFO | PerfReport: 20.23%, LockCompute -- 2.04e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260505 12:30:04.507 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.16e-02 / 0.00e+00 | 20260505 12:30:04.635 | INFO | Starting a round of Baz IO | 20260505 12:30:07.885 | INFO | Populating RTMetrics report | 20260505 12:30:07.885 | INFO | RTMetrics report populated | 20260505 12:30:07.885 | INFO | Opening tmp RTMetrics json file | 20260505 12:30:07.904 | INFO | Serializing RTMetrics to json | 20260505 12:30:07.907 | INFO | Serialization done, swapping tmp file now... | 20260505 12:30:07.908 | INFO | RTMetrics done | 20260505 12:30:07.999 | INFO | Finished a round of Baz IO: Wrote 2.334 GiB in 3.364 seconds | 20260505 12:30:07.999 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.55% / 0.01% / 76.44% | 20260505 12:30:07.999 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.875171% / 1.19209e-05% / 8.34465e-05% | 20260505 12:30:08.000 | INFO | Baz IO overall: | 20260505 12:30:08.000 | INFO | BytesWritten : 1913315072990 | 20260505 12:30:08.000 | INFO | NumEvents : 910713091564 | 20260505 12:30:09.640 | INFO | Analyzing chunk frames = [3219456,3219968) | 20260505 12:30:09.640 | INFO | MemInfo: { | 20260505 12:30:09.640 | INFO | "gpu" : " 17 GB", | 20260505 12:30:09.640 | INFO | "huge" : "", | 20260505 12:30:09.640 | INFO | "huge_shared" : " 27 GB", | 20260505 12:30:09.640 | INFO | "malloc" : " 14 GB", | 20260505 12:30:09.640 | INFO | "pinned" : " 29 GB" | 20260505 12:30:09.640 | INFO | } | 20260505 12:30:09.640 | INFO | Perf report for the last 5.13517 seconds: | 20260505 12:30:09.641 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:30:09.641 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:09.641 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:09.641 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:09.641 | INFO | ANALYSIS : 64.7%, 3.0, 100.0ms, 0 | 20260505 12:30:09.641 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:09.641 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260505 12:30:09.641 | INFO | PRE_HQ : 60.9%, 1.8, 58.3ms, 0 | 20260505 12:30:09.641 | INFO | BAZWRITER : 64.8%, 4.0, 134.4ms, 0 | 20260505 12:30:09.641 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 12:30:09.641 | INFO | Uploaded 36GiB to GPU in 2.20007s (16.3631GiB/s) | 20260505 12:30:09.641 | INFO | Downloaded 38GiB to GPU in 2.28688s (16.6165GiB/s) | 20260505 12:30:14.779 | INFO | Analyzing chunk frames = [3219968,3220480) | 20260505 12:30:14.779 | INFO | MemInfo: { | 20260505 12:30:14.779 | INFO | "gpu" : " 17 GB", | 20260505 12:30:14.779 | INFO | "huge" : "", | 20260505 12:30:14.779 | INFO | "huge_shared" : " 27 GB", | 20260505 12:30:14.779 | INFO | "malloc" : " 14 GB", | 20260505 12:30:14.779 | INFO | "pinned" : " 29 GB" | 20260505 12:30:14.779 | INFO | } | 20260505 12:30:14.779 | INFO | Perf report for the last 5.13856 seconds: | 20260505 12:30:14.779 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:30:14.779 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:14.779 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:14.779 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:14.779 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 12:30:14.779 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:14.779 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:14.779 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 12:30:14.779 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:14.779 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:14.779 | INFO | Uploaded 36GiB to GPU in 2.23277s (16.1235GiB/s) | 20260505 12:30:14.779 | INFO | Downloaded 34GiB to GPU in 2.08643s (16.2958GiB/s) | 20260505 12:30:19.929 | INFO | Analyzing chunk frames = [3220480,3220992) | 20260505 12:30:19.929 | INFO | MemInfo: { | 20260505 12:30:19.929 | INFO | "gpu" : " 17 GB", | 20260505 12:30:19.929 | INFO | "huge" : "", | 20260505 12:30:19.929 | INFO | "huge_shared" : " 27 GB", | 20260505 12:30:19.929 | INFO | "malloc" : " 14 GB", | 20260505 12:30:19.929 | INFO | "pinned" : " 29 GB" | 20260505 12:30:19.929 | INFO | } | 20260505 12:30:19.929 | INFO | Perf report for the last 5.15031 seconds: | 20260505 12:30:19.929 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:30:19.929 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:19.929 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:19.929 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:19.929 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 12:30:19.929 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:19.929 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:19.929 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 12:30:19.929 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:19.930 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:19.930 | INFO | Uploaded 36GiB to GPU in 2.20438s (16.3311GiB/s) | 20260505 12:30:19.930 | INFO | Downloaded 34GiB to GPU in 2.1161s (16.0673GiB/s) | 20260505 12:30:25.073 | INFO | Analyzing chunk frames = [3220992,3221504) | 20260505 12:30:25.073 | INFO | MemInfo: { | 20260505 12:30:25.073 | INFO | "gpu" : " 17 GB", | 20260505 12:30:25.073 | INFO | "huge" : "", | 20260505 12:30:25.073 | INFO | "huge_shared" : " 27 GB", | 20260505 12:30:25.073 | INFO | "malloc" : " 14 GB", | 20260505 12:30:25.073 | INFO | "pinned" : " 29 GB" | 20260505 12:30:25.073 | INFO | } | 20260505 12:30:25.073 | INFO | Perf report for the last 5.14365 seconds: | 20260505 12:30:25.073 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:30:25.073 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:25.073 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:25.073 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:25.073 | INFO | ANALYSIS : 63.0%, 3.0, 98.0ms, 0 | 20260505 12:30:25.073 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:25.073 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:25.073 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 12:30:25.073 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:25.073 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:25.073 | INFO | Uploaded 36GiB to GPU in 2.19942s (16.3679GiB/s) | 20260505 12:30:25.073 | INFO | Downloaded 34GiB to GPU in 2.12833s (15.975GiB/s) | 20260505 12:30:30.208 | INFO | Analyzing chunk frames = [3221504,3222016) | 20260505 12:30:30.208 | INFO | MemInfo: { | 20260505 12:30:30.208 | INFO | "gpu" : " 17 GB", | 20260505 12:30:30.208 | INFO | "huge" : "", | 20260505 12:30:30.208 | INFO | "huge_shared" : " 27 GB", | 20260505 12:30:30.208 | INFO | "malloc" : " 14 GB", | 20260505 12:30:30.208 | INFO | "pinned" : " 29 GB" | 20260505 12:30:30.208 | INFO | } | 20260505 12:30:30.208 | INFO | Perf report for the last 5.13475 seconds: | 20260505 12:30:30.208 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:30:30.208 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:30.208 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:30.208 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:30.208 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 12:30:30.208 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:30.208 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:30.208 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 12:30:30.208 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:30.208 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:30.208 | INFO | Uploaded 36GiB to GPU in 2.15773s (16.6842GiB/s) | 20260505 12:30:30.208 | INFO | Downloaded 34GiB to GPU in 2.12257s (16.0183GiB/s) | 20260505 12:30:35.336 | INFO | Analyzing chunk frames = [3222016,3222528) | 20260505 12:30:35.336 | INFO | MemInfo: { | 20260505 12:30:35.336 | INFO | "gpu" : " 17 GB", | 20260505 12:30:35.336 | INFO | "huge" : "", | 20260505 12:30:35.336 | INFO | "huge_shared" : " 27 GB", | 20260505 12:30:35.336 | INFO | "malloc" : " 14 GB", | 20260505 12:30:35.336 | INFO | "pinned" : " 29 GB" | 20260505 12:30:35.336 | INFO | } | 20260505 12:30:35.336 | INFO | Perf report for the last 5.12855 seconds: | 20260505 12:30:35.337 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:30:35.337 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:35.337 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:35.337 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:35.337 | INFO | ANALYSIS : 63.4%, 3.0, 98.8ms, 0 | 20260505 12:30:35.337 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:35.337 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:35.337 | INFO | PRE_HQ : 28.5%, 1.0, 14.7ms, 0 | 20260505 12:30:35.337 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:35.337 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:35.337 | INFO | Uploaded 36GiB to GPU in 2.17349s (16.5632GiB/s) | 20260505 12:30:35.337 | INFO | Downloaded 34GiB to GPU in 2.11153s (16.1021GiB/s) | 20260505 12:30:40.483 | INFO | Analyzing chunk frames = [3222528,3223040) | 20260505 12:30:40.483 | INFO | MemInfo: { | 20260505 12:30:40.483 | INFO | "gpu" : " 17 GB", | 20260505 12:30:40.483 | INFO | "huge" : "", | 20260505 12:30:40.483 | INFO | "huge_shared" : " 27 GB", | 20260505 12:30:40.483 | INFO | "malloc" : " 14 GB", | 20260505 12:30:40.483 | INFO | "pinned" : " 29 GB" | 20260505 12:30:40.483 | INFO | } | 20260505 12:30:40.483 | INFO | Perf report for the last 5.14652 seconds: | 20260505 12:30:40.483 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:30:40.483 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:40.483 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:40.483 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:40.483 | INFO | ANALYSIS : 63.3%, 3.0, 99.0ms, 0 | 20260505 12:30:40.483 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:40.483 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:40.483 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 12:30:40.483 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:40.483 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:40.483 | INFO | Uploaded 36GiB to GPU in 2.16504s (16.6279GiB/s) | 20260505 12:30:40.483 | INFO | Downloaded 34GiB to GPU in 2.12861s (15.9729GiB/s) | 20260505 12:30:45.621 | INFO | Analyzing chunk frames = [3223040,3223552) | 20260505 12:30:45.621 | INFO | MemInfo: { | 20260505 12:30:45.621 | INFO | "gpu" : " 17 GB", | 20260505 12:30:45.621 | INFO | "huge" : "", | 20260505 12:30:45.621 | INFO | "huge_shared" : " 27 GB", | 20260505 12:30:45.621 | INFO | "malloc" : " 14 GB", | 20260505 12:30:45.621 | INFO | "pinned" : " 29 GB" | 20260505 12:30:45.621 | INFO | } | 20260505 12:30:45.621 | INFO | Perf report for the last 5.13861 seconds: | 20260505 12:30:45.621 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:30:45.621 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:45.621 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:45.621 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:45.621 | INFO | ANALYSIS : 63.8%, 3.0, 99.5ms, 0 | 20260505 12:30:45.622 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:45.622 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:45.622 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 12:30:45.622 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:45.622 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:45.622 | INFO | Uploaded 36GiB to GPU in 2.14268s (16.8014GiB/s) | 20260505 12:30:45.622 | INFO | Downloaded 34GiB to GPU in 2.10805s (16.1286GiB/s) | 20260505 12:30:45.755 | INFO | Starting a round of Baz IO | 20260505 12:30:48.990 | INFO | Populating RTMetrics report | 20260505 12:30:48.991 | INFO | RTMetrics report populated | 20260505 12:30:48.991 | INFO | Opening tmp RTMetrics json file | 20260505 12:30:48.993 | INFO | Serializing RTMetrics to json | 20260505 12:30:48.997 | INFO | Serialization done, swapping tmp file now... | 20260505 12:30:49.005 | INFO | RTMetrics done | 20260505 12:30:49.100 | INFO | Finished a round of Baz IO: Wrote 2.335 GiB in 3.344 seconds | 20260505 12:30:49.100 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.55% / 0.01% / 76.45% | 20260505 12:30:49.100 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.87351% / 7.94729e-06% / 7.15256e-05% | 20260505 12:30:49.100 | INFO | Baz IO overall: | 20260505 12:30:49.100 | INFO | BytesWritten : 1915822666559 | 20260505 12:30:49.100 | INFO | NumEvents : 911902719065 | 20260505 12:30:50.763 | INFO | Analyzing chunk frames = [3223552,3224064) | 20260505 12:30:50.763 | INFO | MemInfo: { | 20260505 12:30:50.763 | INFO | "gpu" : " 17 GB", | 20260505 12:30:50.764 | INFO | "huge" : "", | 20260505 12:30:50.764 | INFO | "huge_shared" : " 27 GB", | 20260505 12:30:50.764 | INFO | "malloc" : " 14 GB", | 20260505 12:30:50.764 | INFO | "pinned" : " 29 GB" | 20260505 12:30:50.764 | INFO | } | 20260505 12:30:50.764 | INFO | Perf report for the last 5.14217 seconds: | 20260505 12:30:50.764 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:30:50.764 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:50.764 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:50.764 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:50.764 | INFO | ANALYSIS : 64.4%, 2.9, 99.4ms, 0 | 20260505 12:30:50.764 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:50.764 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260505 12:30:50.764 | INFO | PRE_HQ : 61.2%, 1.8, 58.2ms, 0 | 20260505 12:30:50.764 | INFO | BAZWRITER : 64.3%, 4.1, 135.9ms, 0 | 20260505 12:30:50.764 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 12:30:50.764 | INFO | Uploaded 36GiB to GPU in 2.19121s (16.4293GiB/s) | 20260505 12:30:50.764 | INFO | Downloaded 38GiB to GPU in 2.27849s (16.6777GiB/s) | 20260505 12:30:55.900 | INFO | Analyzing chunk frames = [3224064,3224576) | 20260505 12:30:55.900 | INFO | MemInfo: { | 20260505 12:30:55.900 | INFO | "gpu" : " 17 GB", | 20260505 12:30:55.900 | INFO | "huge" : "", | 20260505 12:30:55.900 | INFO | "huge_shared" : " 27 GB", | 20260505 12:30:55.900 | INFO | "malloc" : " 14 GB", | 20260505 12:30:55.900 | INFO | "pinned" : " 29 GB" | 20260505 12:30:55.900 | INFO | } | 20260505 12:30:55.900 | INFO | Perf report for the last 5.13606 seconds: | 20260505 12:30:55.900 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:30:55.900 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:55.900 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:55.900 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:55.900 | INFO | ANALYSIS : 63.8%, 3.0, 99.5ms, 0 | 20260505 12:30:55.900 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:55.900 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:55.900 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 12:30:55.900 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:55.900 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:30:55.900 | INFO | Uploaded 36GiB to GPU in 2.19804s (16.3782GiB/s) | 20260505 12:30:55.900 | INFO | Downloaded 34GiB to GPU in 2.04431s (16.6315GiB/s) | 20260505 12:31:01.037 | INFO | Analyzing chunk frames = [3224576,3225088) | 20260505 12:31:01.037 | INFO | MemInfo: { | 20260505 12:31:01.037 | INFO | "gpu" : " 17 GB", | 20260505 12:31:01.037 | INFO | "huge" : "", | 20260505 12:31:01.037 | INFO | "huge_shared" : " 27 GB", | 20260505 12:31:01.037 | INFO | "malloc" : " 14 GB", | 20260505 12:31:01.037 | INFO | "pinned" : " 29 GB" | 20260505 12:31:01.037 | INFO | } | 20260505 12:31:01.037 | INFO | Perf report for the last 5.13768 seconds: | 20260505 12:31:01.037 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:31:01.037 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:01.037 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:01.037 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:01.038 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 12:31:01.038 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:01.038 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:01.038 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 12:31:01.038 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:01.038 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:01.038 | INFO | Uploaded 36GiB to GPU in 2.2079s (16.3051GiB/s) | 20260505 12:31:01.038 | INFO | Downloaded 34GiB to GPU in 2.11531s (16.0733GiB/s) | 20260505 12:31:06.177 | INFO | Analyzing chunk frames = [3225088,3225600) | 20260505 12:31:06.177 | INFO | MemInfo: { | 20260505 12:31:06.177 | INFO | "gpu" : " 17 GB", | 20260505 12:31:06.177 | INFO | "huge" : "", | 20260505 12:31:06.177 | INFO | "huge_shared" : " 27 GB", | 20260505 12:31:06.177 | INFO | "malloc" : " 14 GB", | 20260505 12:31:06.177 | INFO | "pinned" : " 29 GB" | 20260505 12:31:06.177 | INFO | } | 20260505 12:31:06.177 | INFO | Perf report for the last 5.13944 seconds: | 20260505 12:31:06.177 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:31:06.177 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:06.177 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:06.177 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:06.177 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 12:31:06.177 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:06.177 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:06.177 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:31:06.177 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:06.177 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:06.177 | INFO | Uploaded 36GiB to GPU in 2.21299s (16.2676GiB/s) | 20260505 12:31:06.177 | INFO | Downloaded 34GiB to GPU in 2.13247s (15.944GiB/s) | 20260505 12:31:11.320 | INFO | Analyzing chunk frames = [3225600,3226112) | 20260505 12:31:11.320 | INFO | MemInfo: { | 20260505 12:31:11.320 | INFO | "gpu" : " 17 GB", | 20260505 12:31:11.320 | INFO | "huge" : "", | 20260505 12:31:11.320 | INFO | "huge_shared" : " 27 GB", | 20260505 12:31:11.320 | INFO | "malloc" : " 14 GB", | 20260505 12:31:11.320 | INFO | "pinned" : " 29 GB" | 20260505 12:31:11.320 | INFO | } | 20260505 12:31:11.320 | INFO | Perf report for the last 5.14273 seconds: | 20260505 12:31:11.320 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:31:11.320 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:11.320 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:11.320 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:11.320 | INFO | ANALYSIS : 62.9%, 3.0, 98.4ms, 0 | 20260505 12:31:11.320 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:11.320 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:11.320 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 12:31:11.320 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:11.320 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:11.320 | INFO | Uploaded 36GiB to GPU in 2.17956s (16.5171GiB/s) | 20260505 12:31:11.320 | INFO | Downloaded 34GiB to GPU in 2.12741s (15.9818GiB/s) | 20260505 12:31:16.450 | INFO | Analyzing chunk frames = [3226112,3226624) | 20260505 12:31:16.450 | INFO | MemInfo: { | 20260505 12:31:16.450 | INFO | "gpu" : " 17 GB", | 20260505 12:31:16.450 | INFO | "huge" : "", | 20260505 12:31:16.450 | INFO | "huge_shared" : " 27 GB", | 20260505 12:31:16.450 | INFO | "malloc" : " 14 GB", | 20260505 12:31:16.450 | INFO | "pinned" : " 29 GB" | 20260505 12:31:16.450 | INFO | } | 20260505 12:31:16.450 | INFO | Perf report for the last 5.13071 seconds: | 20260505 12:31:16.450 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:31:16.450 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:16.450 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:16.450 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:16.450 | INFO | ANALYSIS : 63.4%, 3.0, 98.8ms, 0 | 20260505 12:31:16.450 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:16.450 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:16.450 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 12:31:16.450 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:16.451 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:16.451 | INFO | Uploaded 36GiB to GPU in 2.16417s (16.6346GiB/s) | 20260505 12:31:16.451 | INFO | Downloaded 34GiB to GPU in 2.11193s (16.099GiB/s) | 20260505 12:31:21.589 | INFO | Analyzing chunk frames = [3226624,3227136) | 20260505 12:31:21.589 | INFO | MemInfo: { | 20260505 12:31:21.589 | INFO | "gpu" : " 17 GB", | 20260505 12:31:21.589 | INFO | "huge" : "", | 20260505 12:31:21.589 | INFO | "huge_shared" : " 27 GB", | 20260505 12:31:21.589 | INFO | "malloc" : " 14 GB", | 20260505 12:31:21.589 | INFO | "pinned" : " 29 GB" | 20260505 12:31:21.589 | INFO | } | 20260505 12:31:21.589 | INFO | Perf report for the last 5.13865 seconds: | 20260505 12:31:21.589 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:31:21.589 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:21.589 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:21.589 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:21.589 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 12:31:21.589 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:21.589 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:21.589 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 12:31:21.589 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:21.589 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:21.589 | INFO | Uploaded 36GiB to GPU in 2.18013s (16.5128GiB/s) | 20260505 12:31:21.589 | INFO | Downloaded 34GiB to GPU in 2.13556s (15.9209GiB/s) | 20260505 12:31:26.735 | INFO | Analyzing chunk frames = [3227136,3227648) | 20260505 12:31:26.735 | INFO | MemInfo: { | 20260505 12:31:26.735 | INFO | "gpu" : " 17 GB", | 20260505 12:31:26.735 | INFO | "huge" : "", | 20260505 12:31:26.735 | INFO | "huge_shared" : " 27 GB", | 20260505 12:31:26.735 | INFO | "malloc" : " 14 GB", | 20260505 12:31:26.735 | INFO | "pinned" : " 29 GB" | 20260505 12:31:26.735 | INFO | } | 20260505 12:31:26.736 | INFO | Perf report for the last 5.14669 seconds: | 20260505 12:31:26.736 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:31:26.736 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:26.736 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:26.736 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:26.736 | INFO | ANALYSIS : 63.7%, 3.0, 99.6ms, 0 | 20260505 12:31:26.736 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:26.736 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:26.736 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 12:31:26.736 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:26.736 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:26.736 | INFO | Uploaded 36GiB to GPU in 2.21308s (16.2669GiB/s) | 20260505 12:31:26.736 | INFO | Downloaded 34GiB to GPU in 2.09141s (16.2569GiB/s) | 20260505 12:31:26.736 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 12:31:26.736 | INFO | Total mem usage: 15.3 GiB | 20260505 12:31:26.736 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 12:31:26.736 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -6.9%) | 20260505 12:31:26.736 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 12:31:26.736 | INFO | High Water Breakdown: | 20260505 12:31:26.736 | INFO | 61.1% : 8.76 GiB ( 92% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 12:31:26.736 | INFO | 38.8% : 5.56 GiB ( 83.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 12:31:26.736 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 12:31:26.736 | INFO | ------------------------------------------ | 20260505 12:31:26.736 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:31:26.736 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 12:31:26.736 | INFO | Total mem usage: 26.5 GiB | 20260505 12:31:26.736 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 12:31:26.736 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 12:31:26.736 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 12:31:26.736 | INFO | High Water Breakdown: | 20260505 12:31:26.736 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:31:26.736 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:31:26.736 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:31:26.736 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:31:26.736 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:31:26.736 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:31:26.736 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 12:31:26.736 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 12:31:26.736 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:31:26.736 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 12:31:26.736 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 12:31:26.736 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 12:31:26.736 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 12:31:26.736 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 12:31:26.737 | INFO | ------------------------------------------ | 20260505 12:31:26.737 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:31:26.737 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 12:31:26.737 | INFO | Total mem usage: 25.6 GiB | 20260505 12:31:26.737 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 12:31:26.737 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 12:31:26.737 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 12:31:26.737 | INFO | High Water Breakdown: | 20260505 12:31:26.737 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 12:31:26.737 | INFO | ------------------------------------------ | 20260505 12:31:26.737 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:31:26.737 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 12:31:26.737 | INFO | Total mem usage: 15.7 GiB | 20260505 12:31:26.737 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 12:31:26.737 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 12:31:26.737 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 12:31:26.737 | INFO | High Water Breakdown: | 20260505 12:31:26.737 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 12:31:26.737 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 12:31:26.737 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 12:31:26.737 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 12:31:26.737 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 12:31:26.737 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 12:31:26.737 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 12:31:26.737 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 12:31:26.737 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 12:31:26.737 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:31:26.737 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 12:31:26.737 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:31:26.737 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 12:31:26.737 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 12:31:26.737 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 12:31:26.737 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 12:31:26.737 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:31:26.737 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 12:31:26.737 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:31:26.737 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:31:26.737 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:31:26.737 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 12:31:26.737 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 12:31:26.737 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:31:26.737 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 12:31:26.737 | INFO | ------------------------------------------ | 20260505 12:31:26.737 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:31:26.737 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 12:31:26.737 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:31:26.737 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:31:26.737 | INFO | PerfReport: Full Execution -- 3.20e+01 / 7.69e+01, 1.38e+01 / 1.50e+00 | 20260505 12:31:26.737 | INFO | PerfReport: 0.29%, AggregateStats -- 9.30e-02 / 9.96e-02, 1.11e-02 / 4.75e-03 | 20260505 12:31:26.737 | INFO | PerfReport: 11.77%, Baseline -- 3.77e+00 / 4.50e+00, 6.06e-01 / 1.52e-01 | 20260505 12:31:26.737 | INFO | PerfReport: 3.31%, Histogram -- 1.06e+00 / 1.48e+00, 2.03e-01 / 1.13e-01 | 20260505 12:31:26.737 | INFO | PerfReport: 15.84%, DME -- 4.06e+01 / 4.51e+01, 4.49e+00 / 2.02e+00 | 20260505 12:31:26.737 | INFO | PerfReport: 56.48%, FrameLabeling -- 1.81e+01 / 2.09e+01, 1.56e+00 / 1.86e+00 | 20260505 12:31:26.737 | INFO | PerfReport: 7.41%, PulseAccumulating -- 2.37e+00 / 2.86e+00, 4.35e-01 / 2.14e-01 | 20260505 12:31:26.737 | INFO | PerfReport: 3.63%, Metrics -- 1.16e+00 / 1.48e+00, 3.27e-01 / 1.79e-01 | 20260505 12:31:26.737 | INFO | PerfReport: 1.27%, PulseSerializing -- 4.08e-01 / 4.52e-01, 7.36e-02 / 3.31e-02 | 20260505 12:31:26.737 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 12:31:26.737 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:31:26.737 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:31:26.737 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260505 12:31:26.737 | INFO | PerfReport: 22.57%, Upload -- 2.28e+01 / 0.00e+00, 8.11e+00 / 0.00e+00 | 20260505 12:31:26.737 | INFO | PerfReport: 21.99%, Download -- 2.22e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260505 12:31:26.737 | INFO | PerfReport: 31.71%, Compute -- 3.20e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 12:31:26.737 | INFO | PerfReport: 2.17%, LockUpload -- 2.19e+00 / 0.00e+00, 7.72e+00 / 0.00e+00 | 20260505 12:31:26.737 | INFO | PerfReport: 1.30%, LockDownload -- 1.32e+00 / 0.00e+00, 2.86e+00 / 0.00e+00 | 20260505 12:31:26.737 | INFO | PerfReport: 20.25%, LockCompute -- 2.05e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260505 12:31:26.737 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.81e-02 / 0.00e+00 | 20260505 12:31:26.870 | INFO | Starting a round of Baz IO | 20260505 12:31:30.107 | INFO | Populating RTMetrics report | 20260505 12:31:30.108 | INFO | RTMetrics report populated | 20260505 12:31:30.108 | INFO | Opening tmp RTMetrics json file | 20260505 12:31:30.132 | INFO | Serializing RTMetrics to json | 20260505 12:31:30.175 | INFO | Serialization done, swapping tmp file now... | 20260505 12:31:30.182 | INFO | RTMetrics done | 20260505 12:31:30.216 | INFO | Finished a round of Baz IO: Wrote 2.334 GiB in 3.345 seconds | 20260505 12:31:30.216 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.54% / 0.01% / 76.45% | 20260505 12:31:30.216 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.871468% / 3.97364e-06% / 7.15256e-05% | 20260505 12:31:30.216 | INFO | Baz IO overall: | 20260505 12:31:30.216 | INFO | BytesWritten : 1918329178431 | 20260505 12:31:30.216 | INFO | NumEvents : 913091491493 | 20260505 12:31:31.882 | INFO | Analyzing chunk frames = [3227648,3228160) | 20260505 12:31:31.882 | INFO | MemInfo: { | 20260505 12:31:31.882 | INFO | "gpu" : " 17 GB", | 20260505 12:31:31.882 | INFO | "huge" : "", | 20260505 12:31:31.882 | INFO | "huge_shared" : " 27 GB", | 20260505 12:31:31.882 | INFO | "malloc" : " 14 GB", | 20260505 12:31:31.882 | INFO | "pinned" : " 29 GB" | 20260505 12:31:31.882 | INFO | } | 20260505 12:31:31.882 | INFO | Perf report for the last 5.14604 seconds: | 20260505 12:31:31.882 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:31:31.882 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:31.882 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:31.882 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:31.882 | INFO | ANALYSIS : 64.4%, 2.9, 99.5ms, 0 | 20260505 12:31:31.882 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:31.882 | INFO | RT_METRICS : 46.7%, 1.0, 24.5ms, 0 | 20260505 12:31:31.882 | INFO | PRE_HQ : 61.2%, 1.8, 58.5ms, 0 | 20260505 12:31:31.882 | INFO | BAZWRITER : 64.3%, 4.1, 135.4ms, 0 | 20260505 12:31:31.882 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 12:31:31.882 | INFO | Uploaded 36GiB to GPU in 2.19889s (16.3719GiB/s) | 20260505 12:31:31.882 | INFO | Downloaded 38GiB to GPU in 2.28502s (16.6301GiB/s) | 20260505 12:31:37.017 | INFO | Analyzing chunk frames = [3228160,3228672) | 20260505 12:31:37.017 | INFO | MemInfo: { | 20260505 12:31:37.017 | INFO | "gpu" : " 17 GB", | 20260505 12:31:37.017 | INFO | "huge" : "", | 20260505 12:31:37.017 | INFO | "huge_shared" : " 27 GB", | 20260505 12:31:37.017 | INFO | "malloc" : " 14 GB", | 20260505 12:31:37.017 | INFO | "pinned" : " 29 GB" | 20260505 12:31:37.017 | INFO | } | 20260505 12:31:37.017 | INFO | Perf report for the last 5.13556 seconds: | 20260505 12:31:37.017 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:31:37.017 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:37.017 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:37.017 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:37.017 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 12:31:37.017 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:37.017 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:37.017 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 12:31:37.017 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:37.017 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:37.018 | INFO | Uploaded 36GiB to GPU in 2.16343s (16.6402GiB/s) | 20260505 12:31:37.018 | INFO | Downloaded 34GiB to GPU in 2.03256s (16.7277GiB/s) | 20260505 12:31:42.149 | INFO | Analyzing chunk frames = [3228672,3229184) | 20260505 12:31:42.149 | INFO | MemInfo: { | 20260505 12:31:42.149 | INFO | "gpu" : " 17 GB", | 20260505 12:31:42.149 | INFO | "huge" : "", | 20260505 12:31:42.149 | INFO | "huge_shared" : " 27 GB", | 20260505 12:31:42.149 | INFO | "malloc" : " 14 GB", | 20260505 12:31:42.149 | INFO | "pinned" : " 29 GB" | 20260505 12:31:42.149 | INFO | } | 20260505 12:31:42.149 | INFO | Perf report for the last 5.13188 seconds: | 20260505 12:31:42.149 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:31:42.149 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:42.149 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:42.149 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:42.149 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 12:31:42.149 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:42.149 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:42.149 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:31:42.149 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:42.149 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:42.149 | INFO | Uploaded 36GiB to GPU in 2.21344s (16.2643GiB/s) | 20260505 12:31:42.150 | INFO | Downloaded 34GiB to GPU in 2.12035s (16.0351GiB/s) | 20260505 12:31:47.292 | INFO | Analyzing chunk frames = [3229184,3229696) | 20260505 12:31:47.292 | INFO | MemInfo: { | 20260505 12:31:47.292 | INFO | "gpu" : " 17 GB", | 20260505 12:31:47.292 | INFO | "huge" : "", | 20260505 12:31:47.292 | INFO | "huge_shared" : " 27 GB", | 20260505 12:31:47.292 | INFO | "malloc" : " 14 GB", | 20260505 12:31:47.292 | INFO | "pinned" : " 29 GB" | 20260505 12:31:47.292 | INFO | } | 20260505 12:31:47.292 | INFO | Perf report for the last 5.14312 seconds: | 20260505 12:31:47.292 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:31:47.292 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:47.292 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:47.292 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:47.292 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260505 12:31:47.292 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:47.292 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:47.292 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 12:31:47.293 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:47.293 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:47.293 | INFO | Uploaded 36GiB to GPU in 2.22476s (16.1815GiB/s) | 20260505 12:31:47.293 | INFO | Downloaded 34GiB to GPU in 2.13512s (15.9242GiB/s) | 20260505 12:31:52.433 | INFO | Analyzing chunk frames = [3229696,3230208) | 20260505 12:31:52.433 | INFO | MemInfo: { | 20260505 12:31:52.433 | INFO | "gpu" : " 17 GB", | 20260505 12:31:52.433 | INFO | "huge" : "", | 20260505 12:31:52.433 | INFO | "huge_shared" : " 27 GB", | 20260505 12:31:52.433 | INFO | "malloc" : " 14 GB", | 20260505 12:31:52.433 | INFO | "pinned" : " 29 GB" | 20260505 12:31:52.433 | INFO | } | 20260505 12:31:52.433 | INFO | Perf report for the last 5.14075 seconds: | 20260505 12:31:52.433 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:31:52.433 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:52.433 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:52.433 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:52.433 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 12:31:52.433 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:52.433 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:52.433 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:31:52.433 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:52.433 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:52.433 | INFO | Uploaded 36GiB to GPU in 2.18179s (16.5002GiB/s) | 20260505 12:31:52.433 | INFO | Downloaded 34GiB to GPU in 2.12872s (15.9721GiB/s) | 20260505 12:31:57.573 | INFO | Analyzing chunk frames = [3230208,3230720) | 20260505 12:31:57.573 | INFO | MemInfo: { | 20260505 12:31:57.573 | INFO | "gpu" : " 17 GB", | 20260505 12:31:57.573 | INFO | "huge" : "", | 20260505 12:31:57.573 | INFO | "huge_shared" : " 27 GB", | 20260505 12:31:57.573 | INFO | "malloc" : " 14 GB", | 20260505 12:31:57.573 | INFO | "pinned" : " 29 GB" | 20260505 12:31:57.573 | INFO | } | 20260505 12:31:57.573 | INFO | Perf report for the last 5.13965 seconds: | 20260505 12:31:57.573 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:31:57.573 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:57.573 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:57.573 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:57.573 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 12:31:57.573 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:57.573 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:57.573 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 12:31:57.573 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:57.573 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:31:57.573 | INFO | Uploaded 36GiB to GPU in 2.11688s (17.0062GiB/s) | 20260505 12:31:57.573 | INFO | Downloaded 34GiB to GPU in 2.0996s (16.1936GiB/s) | 20260505 12:32:02.718 | INFO | Analyzing chunk frames = [3230720,3231232) | 20260505 12:32:02.718 | INFO | MemInfo: { | 20260505 12:32:02.718 | INFO | "gpu" : " 17 GB", | 20260505 12:32:02.718 | INFO | "huge" : "", | 20260505 12:32:02.718 | INFO | "huge_shared" : " 27 GB", | 20260505 12:32:02.718 | INFO | "malloc" : " 14 GB", | 20260505 12:32:02.718 | INFO | "pinned" : " 29 GB" | 20260505 12:32:02.718 | INFO | } | 20260505 12:32:02.718 | INFO | Perf report for the last 5.14543 seconds: | 20260505 12:32:02.718 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:32:02.718 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:02.718 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:02.718 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:02.718 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 12:32:02.718 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:02.718 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:02.718 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 12:32:02.718 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:02.718 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:02.719 | INFO | Uploaded 36GiB to GPU in 2.16779s (16.6068GiB/s) | 20260505 12:32:02.719 | INFO | Downloaded 34GiB to GPU in 2.14595s (15.8438GiB/s) | 20260505 12:32:07.852 | INFO | Analyzing chunk frames = [3231232,3231744) | 20260505 12:32:07.852 | INFO | MemInfo: { | 20260505 12:32:07.852 | INFO | "gpu" : " 17 GB", | 20260505 12:32:07.852 | INFO | "huge" : "", | 20260505 12:32:07.852 | INFO | "huge_shared" : " 27 GB", | 20260505 12:32:07.852 | INFO | "malloc" : " 14 GB", | 20260505 12:32:07.852 | INFO | "pinned" : " 29 GB" | 20260505 12:32:07.852 | INFO | } | 20260505 12:32:07.852 | INFO | Perf report for the last 5.13359 seconds: | 20260505 12:32:07.852 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:32:07.852 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:07.852 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:07.852 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:07.852 | INFO | ANALYSIS : 63.8%, 3.0, 99.5ms, 0 | 20260505 12:32:07.852 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:07.852 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:07.852 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 12:32:07.852 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:07.852 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:07.852 | INFO | Uploaded 36GiB to GPU in 2.16706s (16.6123GiB/s) | 20260505 12:32:07.852 | INFO | Downloaded 34GiB to GPU in 2.10869s (16.1238GiB/s) | 20260505 12:32:07.982 | INFO | Starting a round of Baz IO | 20260505 12:32:11.234 | INFO | Populating RTMetrics report | 20260505 12:32:11.235 | INFO | RTMetrics report populated | 20260505 12:32:11.235 | INFO | Opening tmp RTMetrics json file | 20260505 12:32:11.239 | INFO | Serializing RTMetrics to json | 20260505 12:32:11.240 | INFO | Serialization done, swapping tmp file now... | 20260505 12:32:11.240 | INFO | RTMetrics done | 20260505 12:32:11.352 | INFO | Finished a round of Baz IO: Wrote 2.333 GiB in 3.37 seconds | 20260505 12:32:11.352 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.53% / 0.01% / 76.46% | 20260505 12:32:11.352 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.868408% / 7.94729e-06% / 7.15256e-05% | 20260505 12:32:11.352 | INFO | Baz IO overall: | 20260505 12:32:11.352 | INFO | BytesWritten : 1920833995619 | 20260505 12:32:11.352 | INFO | NumEvents : 914279189285 | 20260505 12:32:12.996 | INFO | Analyzing chunk frames = [3231744,3232256) | 20260505 12:32:12.996 | INFO | MemInfo: { | 20260505 12:32:12.996 | INFO | "gpu" : " 17 GB", | 20260505 12:32:12.996 | INFO | "huge" : "", | 20260505 12:32:12.996 | INFO | "huge_shared" : " 27 GB", | 20260505 12:32:12.996 | INFO | "malloc" : " 14 GB", | 20260505 12:32:12.996 | INFO | "pinned" : " 29 GB" | 20260505 12:32:12.996 | INFO | } | 20260505 12:32:12.996 | INFO | Perf report for the last 5.14406 seconds: | 20260505 12:32:12.996 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:32:12.996 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:12.996 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:12.996 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:12.996 | INFO | ANALYSIS : 64.7%, 2.9, 100.1ms, 0 | 20260505 12:32:12.996 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:12.996 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260505 12:32:12.996 | INFO | PRE_HQ : 61.1%, 1.8, 58.5ms, 0 | 20260505 12:32:12.996 | INFO | BAZWRITER : 64.7%, 4.1, 136.2ms, 0 | 20260505 12:32:12.996 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260505 12:32:12.996 | INFO | Uploaded 36GiB to GPU in 2.19105s (16.4305GiB/s) | 20260505 12:32:12.996 | INFO | Downloaded 38GiB to GPU in 2.2803s (16.6645GiB/s) | 20260505 12:32:18.130 | INFO | Analyzing chunk frames = [3232256,3232768) | 20260505 12:32:18.130 | INFO | MemInfo: { | 20260505 12:32:18.130 | INFO | "gpu" : " 17 GB", | 20260505 12:32:18.130 | INFO | "huge" : "", | 20260505 12:32:18.130 | INFO | "huge_shared" : " 27 GB", | 20260505 12:32:18.130 | INFO | "malloc" : " 14 GB", | 20260505 12:32:18.130 | INFO | "pinned" : " 29 GB" | 20260505 12:32:18.130 | INFO | } | 20260505 12:32:18.130 | INFO | Perf report for the last 5.1344 seconds: | 20260505 12:32:18.130 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:32:18.130 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:18.130 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:18.130 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:18.130 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 12:32:18.130 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:18.130 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:18.130 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:32:18.130 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:18.130 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:18.131 | INFO | Uploaded 36GiB to GPU in 2.18924s (16.4441GiB/s) | 20260505 12:32:18.131 | INFO | Downloaded 34GiB to GPU in 2.03346s (16.7203GiB/s) | 20260505 12:32:23.263 | INFO | Analyzing chunk frames = [3232768,3233280) | 20260505 12:32:23.263 | INFO | MemInfo: { | 20260505 12:32:23.263 | INFO | "gpu" : " 17 GB", | 20260505 12:32:23.263 | INFO | "huge" : "", | 20260505 12:32:23.263 | INFO | "huge_shared" : " 27 GB", | 20260505 12:32:23.263 | INFO | "malloc" : " 14 GB", | 20260505 12:32:23.263 | INFO | "pinned" : " 29 GB" | 20260505 12:32:23.263 | INFO | } | 20260505 12:32:23.263 | INFO | Perf report for the last 5.13296 seconds: | 20260505 12:32:23.263 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:32:23.263 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:23.263 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:23.263 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:23.263 | INFO | ANALYSIS : 63.4%, 3.0, 98.5ms, 0 | 20260505 12:32:23.263 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:23.263 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:23.263 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:32:23.263 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:23.263 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:23.264 | INFO | Uploaded 36GiB to GPU in 2.21937s (16.2208GiB/s) | 20260505 12:32:23.264 | INFO | Downloaded 34GiB to GPU in 2.12266s (16.0177GiB/s) | 20260505 12:32:28.403 | INFO | Analyzing chunk frames = [3233280,3233792) | 20260505 12:32:28.403 | INFO | MemInfo: { | 20260505 12:32:28.403 | INFO | "gpu" : " 17 GB", | 20260505 12:32:28.403 | INFO | "huge" : "", | 20260505 12:32:28.403 | INFO | "huge_shared" : " 27 GB", | 20260505 12:32:28.403 | INFO | "malloc" : " 14 GB", | 20260505 12:32:28.403 | INFO | "pinned" : " 29 GB" | 20260505 12:32:28.403 | INFO | } | 20260505 12:32:28.403 | INFO | Perf report for the last 5.13964 seconds: | 20260505 12:32:28.403 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:32:28.403 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:28.403 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:28.403 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:28.403 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260505 12:32:28.403 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:28.403 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:28.403 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:32:28.403 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:28.403 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:28.403 | INFO | Uploaded 36GiB to GPU in 2.22337s (16.1917GiB/s) | 20260505 12:32:28.403 | INFO | Downloaded 34GiB to GPU in 2.13319s (15.9386GiB/s) | 20260505 12:32:33.543 | INFO | Analyzing chunk frames = [3233792,3234304) | 20260505 12:32:33.543 | INFO | MemInfo: { | 20260505 12:32:33.543 | INFO | "gpu" : " 17 GB", | 20260505 12:32:33.543 | INFO | "huge" : "", | 20260505 12:32:33.543 | INFO | "huge_shared" : " 27 GB", | 20260505 12:32:33.543 | INFO | "malloc" : " 14 GB", | 20260505 12:32:33.543 | INFO | "pinned" : " 29 GB" | 20260505 12:32:33.543 | INFO | } | 20260505 12:32:33.544 | INFO | Perf report for the last 5.14069 seconds: | 20260505 12:32:33.544 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:32:33.544 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:33.544 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:33.544 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:33.544 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 12:32:33.544 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:33.544 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:33.544 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 12:32:33.544 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:33.544 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:33.544 | INFO | Uploaded 36GiB to GPU in 2.18176s (16.5005GiB/s) | 20260505 12:32:33.544 | INFO | Downloaded 34GiB to GPU in 2.13243s (15.9443GiB/s) | 20260505 12:32:38.686 | INFO | Analyzing chunk frames = [3234304,3234816) | 20260505 12:32:38.686 | INFO | MemInfo: { | 20260505 12:32:38.686 | INFO | "gpu" : " 17 GB", | 20260505 12:32:38.687 | INFO | "huge" : "", | 20260505 12:32:38.687 | INFO | "huge_shared" : " 27 GB", | 20260505 12:32:38.687 | INFO | "malloc" : " 14 GB", | 20260505 12:32:38.687 | INFO | "pinned" : " 29 GB" | 20260505 12:32:38.687 | INFO | } | 20260505 12:32:38.687 | INFO | Perf report for the last 5.14301 seconds: | 20260505 12:32:38.687 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:32:38.687 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:38.687 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:38.687 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:38.687 | INFO | ANALYSIS : 63.2%, 3.0, 98.8ms, 0 | 20260505 12:32:38.687 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:38.687 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:38.687 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 12:32:38.687 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:38.687 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:38.687 | INFO | Uploaded 36GiB to GPU in 2.17367s (16.5618GiB/s) | 20260505 12:32:38.687 | INFO | Downloaded 34GiB to GPU in 2.11174s (16.1004GiB/s) | 20260505 12:32:43.830 | INFO | Analyzing chunk frames = [3234816,3235328) | 20260505 12:32:43.830 | INFO | MemInfo: { | 20260505 12:32:43.830 | INFO | "gpu" : " 17 GB", | 20260505 12:32:43.830 | INFO | "huge" : "", | 20260505 12:32:43.830 | INFO | "huge_shared" : " 27 GB", | 20260505 12:32:43.830 | INFO | "malloc" : " 14 GB", | 20260505 12:32:43.830 | INFO | "pinned" : " 29 GB" | 20260505 12:32:43.830 | INFO | } | 20260505 12:32:43.830 | INFO | Perf report for the last 5.14368 seconds: | 20260505 12:32:43.830 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:32:43.830 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:43.830 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:43.830 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:43.830 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 12:32:43.830 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:43.831 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:43.831 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 12:32:43.831 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:43.831 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:43.831 | INFO | Uploaded 36GiB to GPU in 2.16393s (16.6364GiB/s) | 20260505 12:32:43.831 | INFO | Downloaded 34GiB to GPU in 2.16586s (15.6981GiB/s) | 20260505 12:32:48.964 | INFO | Analyzing chunk frames = [3235328,3235840) | 20260505 12:32:48.964 | INFO | MemInfo: { | 20260505 12:32:48.964 | INFO | "gpu" : " 17 GB", | 20260505 12:32:48.964 | INFO | "huge" : "", | 20260505 12:32:48.964 | INFO | "huge_shared" : " 27 GB", | 20260505 12:32:48.964 | INFO | "malloc" : " 14 GB", | 20260505 12:32:48.964 | INFO | "pinned" : " 29 GB" | 20260505 12:32:48.964 | INFO | } | 20260505 12:32:48.964 | INFO | Perf report for the last 5.13344 seconds: | 20260505 12:32:48.964 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:32:48.964 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:48.964 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:48.964 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:48.964 | INFO | ANALYSIS : 63.8%, 3.0, 99.5ms, 0 | 20260505 12:32:48.964 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:48.964 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:48.964 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 12:32:48.964 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:48.964 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:48.964 | INFO | Uploaded 36GiB to GPU in 2.18604s (16.4682GiB/s) | 20260505 12:32:48.964 | INFO | Downloaded 34GiB to GPU in 2.11047s (16.1102GiB/s) | 20260505 12:32:48.964 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 12:32:48.964 | INFO | Total mem usage: 15.3 GiB | 20260505 12:32:48.964 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 12:32:48.964 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -6.92%) | 20260505 12:32:48.964 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 12:32:48.964 | INFO | High Water Breakdown: | 20260505 12:32:48.964 | INFO | 61.1% : 8.76 GiB ( 92% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 12:32:48.964 | INFO | 38.8% : 5.56 GiB ( 83.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 12:32:48.964 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 12:32:48.964 | INFO | ------------------------------------------ | 20260505 12:32:48.964 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:32:48.964 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 12:32:48.964 | INFO | Total mem usage: 26.5 GiB | 20260505 12:32:48.964 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 12:32:48.964 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 12:32:48.964 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 12:32:48.964 | INFO | High Water Breakdown: | 20260505 12:32:48.964 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:32:48.964 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:32:48.964 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:32:48.964 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:32:48.964 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:32:48.964 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:32:48.965 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 12:32:48.965 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 12:32:48.965 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:32:48.965 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 12:32:48.965 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 12:32:48.965 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 12:32:48.965 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 12:32:48.965 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 12:32:48.965 | INFO | ------------------------------------------ | 20260505 12:32:48.965 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:32:48.965 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 12:32:48.965 | INFO | Total mem usage: 25.6 GiB | 20260505 12:32:48.965 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 12:32:48.965 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 12:32:48.965 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 12:32:48.965 | INFO | High Water Breakdown: | 20260505 12:32:48.965 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 12:32:48.965 | INFO | ------------------------------------------ | 20260505 12:32:48.965 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:32:48.965 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 12:32:48.965 | INFO | Total mem usage: 15.7 GiB | 20260505 12:32:48.965 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 12:32:48.965 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 12:32:48.965 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 12:32:48.965 | INFO | High Water Breakdown: | 20260505 12:32:48.965 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 12:32:48.965 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 12:32:48.965 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 12:32:48.965 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 12:32:48.965 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 12:32:48.965 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 12:32:48.965 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 12:32:48.965 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 12:32:48.965 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 12:32:48.965 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:32:48.965 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 12:32:48.965 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:32:48.965 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 12:32:48.965 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 12:32:48.965 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 12:32:48.965 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 12:32:48.965 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:32:48.965 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 12:32:48.965 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:32:48.965 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:32:48.965 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:32:48.965 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 12:32:48.965 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 12:32:48.965 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:32:48.965 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 12:32:48.965 | INFO | ------------------------------------------ | 20260505 12:32:48.965 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:32:48.965 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 12:32:48.965 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:32:48.965 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:32:48.965 | INFO | PerfReport: Full Execution -- 3.20e+01 / 7.70e+01, 1.39e+01 / 2.06e+00 | 20260505 12:32:48.965 | INFO | PerfReport: 0.29%, AggregateStats -- 9.31e-02 / 1.00e-01, 1.05e-02 / 5.11e-03 | 20260505 12:32:48.965 | INFO | PerfReport: 11.75%, Baseline -- 3.76e+00 / 4.46e+00, 5.97e-01 / 2.19e-01 | 20260505 12:32:48.965 | INFO | PerfReport: 3.31%, Histogram -- 1.06e+00 / 1.49e+00, 2.04e-01 / 9.75e-02 | 20260505 12:32:48.965 | INFO | PerfReport: 15.92%, DME -- 4.08e+01 / 4.56e+01, 4.52e+00 / 2.37e+00 | 20260505 12:32:48.965 | INFO | PerfReport: 56.45%, FrameLabeling -- 1.81e+01 / 2.05e+01, 1.53e+00 / 2.01e+00 | 20260505 12:32:48.965 | INFO | PerfReport: 7.40%, PulseAccumulating -- 2.37e+00 / 2.93e+00, 4.47e-01 / 1.84e-01 | 20260505 12:32:48.965 | INFO | PerfReport: 3.60%, Metrics -- 1.15e+00 / 1.47e+00, 3.21e-01 / 1.48e-01 | 20260505 12:32:48.965 | INFO | PerfReport: 1.28%, PulseSerializing -- 4.09e-01 / 4.56e-01, 7.49e-02 / 2.71e-02 | 20260505 12:32:48.965 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 12:32:48.965 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:32:48.965 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:32:48.965 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260505 12:32:48.965 | INFO | PerfReport: 22.53%, Upload -- 2.28e+01 / 0.00e+00, 8.07e+00 / 0.00e+00 | 20260505 12:32:48.965 | INFO | PerfReport: 22.01%, Download -- 2.22e+01 / 0.00e+00, 8.16e+00 / 0.00e+00 | 20260505 12:32:48.965 | INFO | PerfReport: 31.71%, Compute -- 3.20e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 12:32:48.965 | INFO | PerfReport: 2.15%, LockUpload -- 2.17e+00 / 0.00e+00, 7.83e+00 / 0.00e+00 | 20260505 12:32:48.965 | INFO | PerfReport: 1.34%, LockDownload -- 1.35e+00 / 0.00e+00, 2.92e+00 / 0.00e+00 | 20260505 12:32:48.965 | INFO | PerfReport: 20.26%, LockCompute -- 2.05e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260505 12:32:48.965 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.83e-02 / 0.00e+00 | 20260505 12:32:49.097 | INFO | Starting a round of Baz IO | 20260505 12:32:52.350 | INFO | Populating RTMetrics report | 20260505 12:32:52.351 | INFO | RTMetrics report populated | 20260505 12:32:52.351 | INFO | Opening tmp RTMetrics json file | 20260505 12:32:52.352 | INFO | Serializing RTMetrics to json | 20260505 12:32:52.353 | INFO | Serialization done, swapping tmp file now... | 20260505 12:32:52.353 | INFO | RTMetrics done | 20260505 12:32:52.476 | INFO | Finished a round of Baz IO: Wrote 2.334 GiB in 3.378 seconds | 20260505 12:32:52.476 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.53% / 0.01% / 76.47% | 20260505 12:32:52.476 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.867093% / 1.98682e-05% / 8.34465e-05% | 20260505 12:32:52.476 | INFO | Baz IO overall: | 20260505 12:32:52.476 | INFO | BytesWritten : 1923339650718 | 20260505 12:32:52.476 | INFO | NumEvents : 915467543978 | 20260505 12:32:54.096 | INFO | Analyzing chunk frames = [3235840,3236352) | 20260505 12:32:54.096 | INFO | MemInfo: { | 20260505 12:32:54.096 | INFO | "gpu" : " 17 GB", | 20260505 12:32:54.096 | INFO | "huge" : "", | 20260505 12:32:54.096 | INFO | "huge_shared" : " 27 GB", | 20260505 12:32:54.096 | INFO | "malloc" : " 14 GB", | 20260505 12:32:54.096 | INFO | "pinned" : " 29 GB" | 20260505 12:32:54.096 | INFO | } | 20260505 12:32:54.096 | INFO | Perf report for the last 5.13177 seconds: | 20260505 12:32:54.096 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:32:54.096 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:54.096 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:54.096 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:54.096 | INFO | ANALYSIS : 64.9%, 2.9, 100.1ms, 0 | 20260505 12:32:54.096 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:54.096 | INFO | RT_METRICS : 45.5%, 1.0, 23.8ms, 0 | 20260505 12:32:54.096 | INFO | PRE_HQ : 61.2%, 1.8, 58.3ms, 0 | 20260505 12:32:54.096 | INFO | BAZWRITER : 65.2%, 4.1, 136.9ms, 0 | 20260505 12:32:54.096 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 12:32:54.096 | INFO | Uploaded 36GiB to GPU in 2.20203s (16.3486GiB/s) | 20260505 12:32:54.096 | INFO | Downloaded 38GiB to GPU in 2.29256s (16.5754GiB/s) | 20260505 12:32:59.244 | INFO | Analyzing chunk frames = [3236352,3236864) | 20260505 12:32:59.244 | INFO | MemInfo: { | 20260505 12:32:59.244 | INFO | "gpu" : " 17 GB", | 20260505 12:32:59.244 | INFO | "huge" : "", | 20260505 12:32:59.244 | INFO | "huge_shared" : " 27 GB", | 20260505 12:32:59.244 | INFO | "malloc" : " 14 GB", | 20260505 12:32:59.244 | INFO | "pinned" : " 29 GB" | 20260505 12:32:59.244 | INFO | } | 20260505 12:32:59.244 | INFO | Perf report for the last 5.14862 seconds: | 20260505 12:32:59.244 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:32:59.244 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:59.244 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:59.244 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:59.244 | INFO | ANALYSIS : 63.9%, 3.0, 100.0ms, 0 | 20260505 12:32:59.244 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:59.244 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:59.244 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:32:59.244 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:59.244 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:32:59.244 | INFO | Uploaded 36GiB to GPU in 2.23134s (16.1338GiB/s) | 20260505 12:32:59.245 | INFO | Downloaded 34GiB to GPU in 2.08073s (16.3404GiB/s) | 20260505 12:33:04.392 | INFO | Analyzing chunk frames = [3236864,3237376) | 20260505 12:33:04.392 | INFO | MemInfo: { | 20260505 12:33:04.392 | INFO | "gpu" : " 17 GB", | 20260505 12:33:04.392 | INFO | "huge" : "", | 20260505 12:33:04.392 | INFO | "huge_shared" : " 27 GB", | 20260505 12:33:04.392 | INFO | "malloc" : " 14 GB", | 20260505 12:33:04.392 | INFO | "pinned" : " 29 GB" | 20260505 12:33:04.392 | INFO | } | 20260505 12:33:04.392 | INFO | Perf report for the last 5.1477 seconds: | 20260505 12:33:04.392 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:33:04.392 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:04.392 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:04.392 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:04.392 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 12:33:04.392 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:04.392 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:04.392 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 12:33:04.392 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:04.392 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:04.392 | INFO | Uploaded 36GiB to GPU in 2.20909s (16.2963GiB/s) | 20260505 12:33:04.392 | INFO | Downloaded 34GiB to GPU in 2.11949s (16.0416GiB/s) | 20260505 12:33:09.524 | INFO | Analyzing chunk frames = [3237376,3237888) | 20260505 12:33:09.524 | INFO | MemInfo: { | 20260505 12:33:09.524 | INFO | "gpu" : " 17 GB", | 20260505 12:33:09.524 | INFO | "huge" : "", | 20260505 12:33:09.524 | INFO | "huge_shared" : " 27 GB", | 20260505 12:33:09.524 | INFO | "malloc" : " 14 GB", | 20260505 12:33:09.524 | INFO | "pinned" : " 29 GB" | 20260505 12:33:09.524 | INFO | } | 20260505 12:33:09.524 | INFO | Perf report for the last 5.13241 seconds: | 20260505 12:33:09.524 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:33:09.524 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:09.524 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:09.524 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:09.524 | INFO | ANALYSIS : 63.2%, 3.0, 98.0ms, 0 | 20260505 12:33:09.524 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:09.524 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:09.524 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:33:09.524 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:09.524 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:09.525 | INFO | Uploaded 36GiB to GPU in 2.1963s (16.3912GiB/s) | 20260505 12:33:09.525 | INFO | Downloaded 34GiB to GPU in 2.13416s (15.9313GiB/s) | 20260505 12:33:14.656 | INFO | Analyzing chunk frames = [3237888,3238400) | 20260505 12:33:14.656 | INFO | MemInfo: { | 20260505 12:33:14.656 | INFO | "gpu" : " 17 GB", | 20260505 12:33:14.656 | INFO | "huge" : "", | 20260505 12:33:14.656 | INFO | "huge_shared" : " 27 GB", | 20260505 12:33:14.656 | INFO | "malloc" : " 14 GB", | 20260505 12:33:14.656 | INFO | "pinned" : " 29 GB" | 20260505 12:33:14.656 | INFO | } | 20260505 12:33:14.657 | INFO | Perf report for the last 5.13216 seconds: | 20260505 12:33:14.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:33:14.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:14.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:14.657 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:14.657 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 12:33:14.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:14.657 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:14.657 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 12:33:14.657 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:14.657 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:14.657 | INFO | Uploaded 36GiB to GPU in 2.16044s (16.6633GiB/s) | 20260505 12:33:14.657 | INFO | Downloaded 34GiB to GPU in 2.12334s (16.0125GiB/s) | 20260505 12:33:19.806 | INFO | Analyzing chunk frames = [3238400,3238912) | 20260505 12:33:19.806 | INFO | MemInfo: { | 20260505 12:33:19.806 | INFO | "gpu" : " 17 GB", | 20260505 12:33:19.806 | INFO | "huge" : "", | 20260505 12:33:19.806 | INFO | "huge_shared" : " 27 GB", | 20260505 12:33:19.806 | INFO | "malloc" : " 14 GB", | 20260505 12:33:19.806 | INFO | "pinned" : " 29 GB" | 20260505 12:33:19.806 | INFO | } | 20260505 12:33:19.806 | INFO | Perf report for the last 5.14916 seconds: | 20260505 12:33:19.806 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:33:19.806 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:19.806 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:19.806 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:19.806 | INFO | ANALYSIS : 63.2%, 3.0, 98.9ms, 0 | 20260505 12:33:19.806 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:19.806 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:19.806 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 12:33:19.806 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:19.806 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:19.806 | INFO | Uploaded 36GiB to GPU in 2.1542s (16.7116GiB/s) | 20260505 12:33:19.806 | INFO | Downloaded 34GiB to GPU in 2.1166s (16.0635GiB/s) | 20260505 12:33:24.943 | INFO | Analyzing chunk frames = [3238912,3239424) | 20260505 12:33:24.943 | INFO | MemInfo: { | 20260505 12:33:24.943 | INFO | "gpu" : " 17 GB", | 20260505 12:33:24.943 | INFO | "huge" : "", | 20260505 12:33:24.943 | INFO | "huge_shared" : " 27 GB", | 20260505 12:33:24.943 | INFO | "malloc" : " 14 GB", | 20260505 12:33:24.943 | INFO | "pinned" : " 29 GB" | 20260505 12:33:24.943 | INFO | } | 20260505 12:33:24.943 | INFO | Perf report for the last 5.13708 seconds: | 20260505 12:33:24.943 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:33:24.943 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:24.943 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:24.943 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:24.943 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 12:33:24.943 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:24.943 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:24.943 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260505 12:33:24.943 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:24.943 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:24.943 | INFO | Uploaded 36GiB to GPU in 2.1665s (16.6167GiB/s) | 20260505 12:33:24.943 | INFO | Downloaded 34GiB to GPU in 2.15567s (15.7723GiB/s) | 20260505 12:33:30.078 | INFO | Analyzing chunk frames = [3239424,3239936) | 20260505 12:33:30.078 | INFO | MemInfo: { | 20260505 12:33:30.078 | INFO | "gpu" : " 17 GB", | 20260505 12:33:30.078 | INFO | "huge" : "", | 20260505 12:33:30.078 | INFO | "huge_shared" : " 27 GB", | 20260505 12:33:30.078 | INFO | "malloc" : " 14 GB", | 20260505 12:33:30.078 | INFO | "pinned" : " 29 GB" | 20260505 12:33:30.078 | INFO | } | 20260505 12:33:30.078 | INFO | Perf report for the last 5.13562 seconds: | 20260505 12:33:30.078 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:33:30.078 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:30.078 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:30.079 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:30.079 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 12:33:30.079 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:30.079 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:30.079 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 12:33:30.079 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:30.079 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:30.079 | INFO | Uploaded 36GiB to GPU in 2.14837s (16.7569GiB/s) | 20260505 12:33:30.079 | INFO | Downloaded 34GiB to GPU in 2.10277s (16.1691GiB/s) | 20260505 12:33:30.210 | INFO | Starting a round of Baz IO | 20260505 12:33:33.457 | INFO | Populating RTMetrics report | 20260505 12:33:33.457 | INFO | RTMetrics report populated | 20260505 12:33:33.457 | INFO | Opening tmp RTMetrics json file | 20260505 12:33:33.480 | INFO | Serializing RTMetrics to json | 20260505 12:33:33.482 | INFO | Serialization done, swapping tmp file now... | 20260505 12:33:33.485 | INFO | RTMetrics done | 20260505 12:33:33.586 | INFO | Finished a round of Baz IO: Wrote 2.335 GiB in 3.376 seconds | 20260505 12:33:33.586 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.52% / 0.01% / 76.47% | 20260505 12:33:33.586 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.866091% / 1.19209e-05% / 5.96046e-05% | 20260505 12:33:33.586 | INFO | Baz IO overall: | 20260505 12:33:33.587 | INFO | BytesWritten : 1925846936236 | 20260505 12:33:33.587 | INFO | NumEvents : 916657257026 | 20260505 12:33:35.213 | INFO | Analyzing chunk frames = [3239936,3240448) | 20260505 12:33:35.213 | INFO | MemInfo: { | 20260505 12:33:35.213 | INFO | "gpu" : " 17 GB", | 20260505 12:33:35.213 | INFO | "huge" : "", | 20260505 12:33:35.213 | INFO | "huge_shared" : " 27 GB", | 20260505 12:33:35.213 | INFO | "malloc" : " 14 GB", | 20260505 12:33:35.213 | INFO | "pinned" : " 29 GB" | 20260505 12:33:35.213 | INFO | } | 20260505 12:33:35.214 | INFO | Perf report for the last 5.13508 seconds: | 20260505 12:33:35.214 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:33:35.214 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:35.214 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:35.214 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:35.214 | INFO | ANALYSIS : 64.7%, 3.0, 100.1ms, 0 | 20260505 12:33:35.214 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:35.214 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260505 12:33:35.214 | INFO | PRE_HQ : 61.0%, 1.8, 58.4ms, 0 | 20260505 12:33:35.214 | INFO | BAZWRITER : 64.9%, 4.0, 134.8ms, 0 | 20260505 12:33:35.214 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 12:33:35.214 | INFO | Uploaded 36GiB to GPU in 2.18778s (16.455GiB/s) | 20260505 12:33:35.214 | INFO | Downloaded 38GiB to GPU in 2.28885s (16.6022GiB/s) | 20260505 12:33:40.355 | INFO | Analyzing chunk frames = [3240448,3240960) | 20260505 12:33:40.355 | INFO | MemInfo: { | 20260505 12:33:40.355 | INFO | "gpu" : " 17 GB", | 20260505 12:33:40.355 | INFO | "huge" : "", | 20260505 12:33:40.355 | INFO | "huge_shared" : " 27 GB", | 20260505 12:33:40.355 | INFO | "malloc" : " 14 GB", | 20260505 12:33:40.355 | INFO | "pinned" : " 29 GB" | 20260505 12:33:40.355 | INFO | } | 20260505 12:33:40.355 | INFO | Perf report for the last 5.14181 seconds: | 20260505 12:33:40.355 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:33:40.355 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:40.355 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:40.355 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:40.355 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 12:33:40.355 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:40.355 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:40.355 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:33:40.355 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:40.356 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:40.356 | INFO | Uploaded 36GiB to GPU in 2.23344s (16.1186GiB/s) | 20260505 12:33:40.356 | INFO | Downloaded 34GiB to GPU in 2.08582s (16.3005GiB/s) | 20260505 12:33:45.500 | INFO | Analyzing chunk frames = [3240960,3241472) | 20260505 12:33:45.500 | INFO | MemInfo: { | 20260505 12:33:45.500 | INFO | "gpu" : " 17 GB", | 20260505 12:33:45.500 | INFO | "huge" : "", | 20260505 12:33:45.500 | INFO | "huge_shared" : " 27 GB", | 20260505 12:33:45.500 | INFO | "malloc" : " 14 GB", | 20260505 12:33:45.500 | INFO | "pinned" : " 29 GB" | 20260505 12:33:45.500 | INFO | } | 20260505 12:33:45.500 | INFO | Perf report for the last 5.14444 seconds: | 20260505 12:33:45.500 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:33:45.500 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:45.500 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:45.500 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:45.500 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 12:33:45.500 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:45.500 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:45.500 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 12:33:45.500 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:45.500 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:45.500 | INFO | Uploaded 36GiB to GPU in 2.22102s (16.2088GiB/s) | 20260505 12:33:45.500 | INFO | Downloaded 34GiB to GPU in 2.12205s (16.0223GiB/s) | 20260505 12:33:50.642 | INFO | Analyzing chunk frames = [3241472,3241984) | 20260505 12:33:50.642 | INFO | MemInfo: { | 20260505 12:33:50.642 | INFO | "gpu" : " 17 GB", | 20260505 12:33:50.642 | INFO | "huge" : "", | 20260505 12:33:50.642 | INFO | "huge_shared" : " 27 GB", | 20260505 12:33:50.642 | INFO | "malloc" : " 14 GB", | 20260505 12:33:50.642 | INFO | "pinned" : " 29 GB" | 20260505 12:33:50.642 | INFO | } | 20260505 12:33:50.642 | INFO | Perf report for the last 5.14215 seconds: | 20260505 12:33:50.642 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:33:50.642 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:50.642 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:50.642 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:50.642 | INFO | ANALYSIS : 63.1%, 3.0, 98.1ms, 0 | 20260505 12:33:50.642 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:50.642 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:50.642 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 12:33:50.642 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:50.642 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:50.642 | INFO | Uploaded 36GiB to GPU in 2.20219s (16.3474GiB/s) | 20260505 12:33:50.642 | INFO | Downloaded 34GiB to GPU in 2.13432s (15.9301GiB/s) | 20260505 12:33:55.778 | INFO | Analyzing chunk frames = [3241984,3242496) | 20260505 12:33:55.778 | INFO | MemInfo: { | 20260505 12:33:55.778 | INFO | "gpu" : " 17 GB", | 20260505 12:33:55.778 | INFO | "huge" : "", | 20260505 12:33:55.778 | INFO | "huge_shared" : " 27 GB", | 20260505 12:33:55.778 | INFO | "malloc" : " 14 GB", | 20260505 12:33:55.778 | INFO | "pinned" : " 29 GB" | 20260505 12:33:55.778 | INFO | } | 20260505 12:33:55.778 | INFO | Perf report for the last 5.13574 seconds: | 20260505 12:33:55.778 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:33:55.778 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:55.778 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:55.778 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:55.778 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 12:33:55.778 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:55.778 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:55.778 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:33:55.778 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:55.778 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:33:55.778 | INFO | Uploaded 36GiB to GPU in 2.14411s (16.7902GiB/s) | 20260505 12:33:55.778 | INFO | Downloaded 34GiB to GPU in 2.12395s (16.0079GiB/s) | 20260505 12:34:00.908 | INFO | Analyzing chunk frames = [3242496,3243008) | 20260505 12:34:00.908 | INFO | MemInfo: { | 20260505 12:34:00.908 | INFO | "gpu" : " 17 GB", | 20260505 12:34:00.908 | INFO | "huge" : "", | 20260505 12:34:00.908 | INFO | "huge_shared" : " 27 GB", | 20260505 12:34:00.908 | INFO | "malloc" : " 14 GB", | 20260505 12:34:00.908 | INFO | "pinned" : " 29 GB" | 20260505 12:34:00.908 | INFO | } | 20260505 12:34:00.908 | INFO | Perf report for the last 5.12994 seconds: | 20260505 12:34:00.908 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:34:00.908 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:00.908 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:00.908 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:00.908 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 12:34:00.908 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:00.908 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:00.908 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:34:00.908 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:00.908 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:00.908 | INFO | Uploaded 36GiB to GPU in 2.13197s (16.8858GiB/s) | 20260505 12:34:00.908 | INFO | Downloaded 34GiB to GPU in 2.1066s (16.1398GiB/s) | 20260505 12:34:06.052 | INFO | Analyzing chunk frames = [3243008,3243520) | 20260505 12:34:06.052 | INFO | MemInfo: { | 20260505 12:34:06.052 | INFO | "gpu" : " 17 GB", | 20260505 12:34:06.052 | INFO | "huge" : "", | 20260505 12:34:06.052 | INFO | "huge_shared" : " 27 GB", | 20260505 12:34:06.052 | INFO | "malloc" : " 14 GB", | 20260505 12:34:06.052 | INFO | "pinned" : " 29 GB" | 20260505 12:34:06.052 | INFO | } | 20260505 12:34:06.052 | INFO | Perf report for the last 5.14475 seconds: | 20260505 12:34:06.052 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:34:06.052 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:06.052 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:06.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:06.052 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 12:34:06.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:06.052 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:06.052 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 12:34:06.053 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:06.053 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:06.053 | INFO | Uploaded 36GiB to GPU in 2.14829s (16.7575GiB/s) | 20260505 12:34:06.053 | INFO | Downloaded 34GiB to GPU in 2.14425s (15.8564GiB/s) | 20260505 12:34:11.194 | INFO | Analyzing chunk frames = [3243520,3244032) | 20260505 12:34:11.194 | INFO | MemInfo: { | 20260505 12:34:11.194 | INFO | "gpu" : " 17 GB", | 20260505 12:34:11.194 | INFO | "huge" : "", | 20260505 12:34:11.194 | INFO | "huge_shared" : " 27 GB", | 20260505 12:34:11.194 | INFO | "malloc" : " 14 GB", | 20260505 12:34:11.194 | INFO | "pinned" : " 29 GB" | 20260505 12:34:11.195 | INFO | } | 20260505 12:34:11.195 | INFO | Perf report for the last 5.1422 seconds: | 20260505 12:34:11.195 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:34:11.195 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:11.195 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:11.195 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:11.195 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 12:34:11.195 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:11.195 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:11.195 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 12:34:11.195 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:11.195 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:11.195 | INFO | Uploaded 36GiB to GPU in 2.19242s (16.4202GiB/s) | 20260505 12:34:11.195 | INFO | Downloaded 34GiB to GPU in 2.11795s (16.0532GiB/s) | 20260505 12:34:11.195 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 12:34:11.195 | INFO | Total mem usage: 15.3 GiB | 20260505 12:34:11.195 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 12:34:11.195 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -6.98%) | 20260505 12:34:11.195 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 12:34:11.195 | INFO | High Water Breakdown: | 20260505 12:34:11.195 | INFO | 61.1% : 8.76 GiB ( 92% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 12:34:11.195 | INFO | 38.7% : 5.55 GiB ( 83.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 12:34:11.195 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 12:34:11.195 | INFO | ------------------------------------------ | 20260505 12:34:11.195 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:34:11.195 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 12:34:11.195 | INFO | Total mem usage: 26.5 GiB | 20260505 12:34:11.195 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 12:34:11.195 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 12:34:11.195 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 12:34:11.195 | INFO | High Water Breakdown: | 20260505 12:34:11.195 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:34:11.195 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:34:11.195 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:34:11.195 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:34:11.195 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:34:11.195 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:34:11.195 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 12:34:11.195 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 12:34:11.195 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:34:11.195 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 12:34:11.195 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 12:34:11.195 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 12:34:11.195 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 12:34:11.195 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 12:34:11.195 | INFO | ------------------------------------------ | 20260505 12:34:11.195 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:34:11.195 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 12:34:11.195 | INFO | Total mem usage: 25.6 GiB | 20260505 12:34:11.195 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 12:34:11.195 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 12:34:11.195 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 12:34:11.195 | INFO | High Water Breakdown: | 20260505 12:34:11.195 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 12:34:11.195 | INFO | ------------------------------------------ | 20260505 12:34:11.196 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:34:11.196 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 12:34:11.196 | INFO | Total mem usage: 15.7 GiB | 20260505 12:34:11.196 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 12:34:11.196 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 12:34:11.196 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 12:34:11.196 | INFO | High Water Breakdown: | 20260505 12:34:11.196 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 12:34:11.196 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 12:34:11.196 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 12:34:11.196 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 12:34:11.196 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 12:34:11.196 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 12:34:11.196 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 12:34:11.196 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 12:34:11.196 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 12:34:11.196 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:34:11.196 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 12:34:11.196 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:34:11.196 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 12:34:11.196 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 12:34:11.196 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 12:34:11.196 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 12:34:11.196 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:34:11.196 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 12:34:11.196 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:34:11.196 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:34:11.196 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:34:11.196 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 12:34:11.196 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 12:34:11.196 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:34:11.196 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 12:34:11.196 | INFO | ------------------------------------------ | 20260505 12:34:11.196 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:34:11.196 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 12:34:11.196 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:34:11.196 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:34:11.196 | INFO | PerfReport: Full Execution -- 3.20e+01 / 7.73e+01, 1.37e+01 / 1.44e+00 | 20260505 12:34:11.196 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 9.89e-02, 1.05e-02 / 3.42e-03 | 20260505 12:34:11.196 | INFO | PerfReport: 11.77%, Baseline -- 3.76e+00 / 4.50e+00, 5.92e-01 / 1.73e-01 | 20260505 12:34:11.196 | INFO | PerfReport: 3.30%, Histogram -- 1.05e+00 / 1.50e+00, 1.99e-01 / 8.17e-02 | 20260505 12:34:11.196 | INFO | PerfReport: 16.04%, DME -- 4.10e+01 / 4.63e+01, 4.64e+00 / 2.01e+00 | 20260505 12:34:11.196 | INFO | PerfReport: 56.21%, FrameLabeling -- 1.80e+01 / 2.00e+01, 1.37e+00 / 2.13e+00 | 20260505 12:34:11.196 | INFO | PerfReport: 7.47%, PulseAccumulating -- 2.39e+00 / 2.91e+00, 4.55e-01 / 2.07e-01 | 20260505 12:34:11.196 | INFO | PerfReport: 3.64%, Metrics -- 1.16e+00 / 1.49e+00, 3.26e-01 / 1.59e-01 | 20260505 12:34:11.196 | INFO | PerfReport: 1.28%, PulseSerializing -- 4.10e-01 / 4.56e-01, 1.08e-01 / 3.14e-02 | 20260505 12:34:11.196 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 12:34:11.196 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:34:11.196 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:34:11.196 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260505 12:34:11.196 | INFO | PerfReport: 22.50%, Upload -- 2.27e+01 / 0.00e+00, 8.07e+00 / 0.00e+00 | 20260505 12:34:11.196 | INFO | PerfReport: 22.06%, Download -- 2.23e+01 / 0.00e+00, 8.18e+00 / 0.00e+00 | 20260505 12:34:11.196 | INFO | PerfReport: 31.62%, Compute -- 3.20e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 12:34:11.196 | INFO | PerfReport: 2.20%, LockUpload -- 2.22e+00 / 0.00e+00, 7.79e+00 / 0.00e+00 | 20260505 12:34:11.196 | INFO | PerfReport: 1.41%, LockDownload -- 1.43e+00 / 0.00e+00, 3.01e+00 / 0.00e+00 | 20260505 12:34:11.196 | INFO | PerfReport: 20.19%, LockCompute -- 2.04e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260505 12:34:11.196 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.11e-02 / 0.00e+00 | 20260505 12:34:11.329 | INFO | Starting a round of Baz IO | 20260505 12:34:14.577 | INFO | Populating RTMetrics report | 20260505 12:34:14.577 | INFO | RTMetrics report populated | 20260505 12:34:14.577 | INFO | Opening tmp RTMetrics json file | 20260505 12:34:14.588 | INFO | Serializing RTMetrics to json | 20260505 12:34:14.627 | INFO | Serialization done, swapping tmp file now... | 20260505 12:34:14.634 | INFO | RTMetrics done | 20260505 12:34:14.703 | INFO | Finished a round of Baz IO: Wrote 2.334 GiB in 3.375 seconds | 20260505 12:34:14.704 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.52% / 0.01% / 76.48% | 20260505 12:34:14.704 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.863524% / 1.19209e-05% / 7.15256e-05% | 20260505 12:34:14.704 | INFO | Baz IO overall: | 20260505 12:34:14.704 | INFO | BytesWritten : 1928353155954 | 20260505 12:34:14.704 | INFO | NumEvents : 917846385447 | 20260505 12:34:16.342 | INFO | Analyzing chunk frames = [3244032,3244544) | 20260505 12:34:16.342 | INFO | MemInfo: { | 20260505 12:34:16.342 | INFO | "gpu" : " 17 GB", | 20260505 12:34:16.342 | INFO | "huge" : "", | 20260505 12:34:16.342 | INFO | "huge_shared" : " 27 GB", | 20260505 12:34:16.342 | INFO | "malloc" : " 14 GB", | 20260505 12:34:16.342 | INFO | "pinned" : " 29 GB" | 20260505 12:34:16.342 | INFO | } | 20260505 12:34:16.343 | INFO | Perf report for the last 5.14788 seconds: | 20260505 12:34:16.343 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:34:16.343 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:16.343 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:16.343 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:16.343 | INFO | ANALYSIS : 64.6%, 2.9, 99.5ms, 0 | 20260505 12:34:16.343 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:16.343 | INFO | RT_METRICS : 46.4%, 1.0, 24.4ms, 0 | 20260505 12:34:16.343 | INFO | PRE_HQ : 60.9%, 1.9, 58.8ms, 0 | 20260505 12:34:16.343 | INFO | BAZWRITER : 64.9%, 4.1, 137.2ms, 0 | 20260505 12:34:16.343 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 12:34:16.343 | INFO | Uploaded 36GiB to GPU in 2.20567s (16.3215GiB/s) | 20260505 12:34:16.343 | INFO | Downloaded 38GiB to GPU in 2.2905s (16.5903GiB/s) | 20260505 12:34:21.471 | INFO | Analyzing chunk frames = [3244544,3245056) | 20260505 12:34:21.471 | INFO | MemInfo: { | 20260505 12:34:21.471 | INFO | "gpu" : " 17 GB", | 20260505 12:34:21.471 | INFO | "huge" : "", | 20260505 12:34:21.471 | INFO | "huge_shared" : " 27 GB", | 20260505 12:34:21.471 | INFO | "malloc" : " 14 GB", | 20260505 12:34:21.472 | INFO | "pinned" : " 29 GB" | 20260505 12:34:21.472 | INFO | } | 20260505 12:34:21.472 | INFO | Perf report for the last 5.12906 seconds: | 20260505 12:34:21.472 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:34:21.472 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:21.472 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:21.472 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:21.472 | INFO | ANALYSIS : 63.7%, 3.0, 99.2ms, 0 | 20260505 12:34:21.472 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:21.472 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:21.472 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 12:34:21.472 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:21.472 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:21.472 | INFO | Uploaded 36GiB to GPU in 2.17003s (16.5896GiB/s) | 20260505 12:34:21.472 | INFO | Downloaded 34GiB to GPU in 2.02849s (16.7612GiB/s) | 20260505 12:34:26.610 | INFO | Analyzing chunk frames = [3245056,3245568) | 20260505 12:34:26.610 | INFO | MemInfo: { | 20260505 12:34:26.610 | INFO | "gpu" : " 17 GB", | 20260505 12:34:26.610 | INFO | "huge" : "", | 20260505 12:34:26.610 | INFO | "huge_shared" : " 27 GB", | 20260505 12:34:26.610 | INFO | "malloc" : " 14 GB", | 20260505 12:34:26.610 | INFO | "pinned" : " 29 GB" | 20260505 12:34:26.610 | INFO | } | 20260505 12:34:26.610 | INFO | Perf report for the last 5.1382 seconds: | 20260505 12:34:26.610 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:34:26.610 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:26.610 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:26.610 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:26.610 | INFO | ANALYSIS : 63.2%, 3.0, 98.3ms, 0 | 20260505 12:34:26.610 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:26.610 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:26.610 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:34:26.610 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:26.610 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:26.610 | INFO | Uploaded 36GiB to GPU in 2.20933s (16.2946GiB/s) | 20260505 12:34:26.610 | INFO | Downloaded 34GiB to GPU in 2.1251s (15.9992GiB/s) | 20260505 12:34:31.758 | INFO | Analyzing chunk frames = [3245568,3246080) | 20260505 12:34:31.758 | INFO | MemInfo: { | 20260505 12:34:31.758 | INFO | "gpu" : " 17 GB", | 20260505 12:34:31.758 | INFO | "huge" : "", | 20260505 12:34:31.758 | INFO | "huge_shared" : " 27 GB", | 20260505 12:34:31.758 | INFO | "malloc" : " 14 GB", | 20260505 12:34:31.758 | INFO | "pinned" : " 29 GB" | 20260505 12:34:31.758 | INFO | } | 20260505 12:34:31.758 | INFO | Perf report for the last 5.14792 seconds: | 20260505 12:34:31.758 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:34:31.758 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:31.758 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:31.758 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:31.758 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 12:34:31.758 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:31.758 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:31.758 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 12:34:31.758 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:31.758 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:31.758 | INFO | Uploaded 36GiB to GPU in 2.21564s (16.2481GiB/s) | 20260505 12:34:31.758 | INFO | Downloaded 34GiB to GPU in 2.13381s (15.934GiB/s) | 20260505 12:34:36.891 | INFO | Analyzing chunk frames = [3246080,3246592) | 20260505 12:34:36.891 | INFO | MemInfo: { | 20260505 12:34:36.891 | INFO | "gpu" : " 17 GB", | 20260505 12:34:36.891 | INFO | "huge" : "", | 20260505 12:34:36.891 | INFO | "huge_shared" : " 27 GB", | 20260505 12:34:36.891 | INFO | "malloc" : " 14 GB", | 20260505 12:34:36.891 | INFO | "pinned" : " 29 GB" | 20260505 12:34:36.891 | INFO | } | 20260505 12:34:36.891 | INFO | Perf report for the last 5.13353 seconds: | 20260505 12:34:36.891 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:34:36.891 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:36.891 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:36.891 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:36.891 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 12:34:36.891 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:36.891 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:36.891 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 12:34:36.891 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:36.891 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:36.892 | INFO | Uploaded 36GiB to GPU in 2.13639s (16.8509GiB/s) | 20260505 12:34:36.892 | INFO | Downloaded 34GiB to GPU in 2.122s (16.0226GiB/s) | 20260505 12:34:42.029 | INFO | Analyzing chunk frames = [3246592,3247104) | 20260505 12:34:42.029 | INFO | MemInfo: { | 20260505 12:34:42.029 | INFO | "gpu" : " 17 GB", | 20260505 12:34:42.029 | INFO | "huge" : "", | 20260505 12:34:42.029 | INFO | "huge_shared" : " 27 GB", | 20260505 12:34:42.029 | INFO | "malloc" : " 14 GB", | 20260505 12:34:42.029 | INFO | "pinned" : " 29 GB" | 20260505 12:34:42.029 | INFO | } | 20260505 12:34:42.029 | INFO | Perf report for the last 5.13785 seconds: | 20260505 12:34:42.029 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:34:42.029 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:42.029 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:42.029 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:42.029 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 12:34:42.029 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:42.029 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:42.029 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:34:42.029 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:42.029 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:42.030 | INFO | Uploaded 36GiB to GPU in 2.16354s (16.6394GiB/s) | 20260505 12:34:42.030 | INFO | Downloaded 34GiB to GPU in 2.11264s (16.0936GiB/s) | 20260505 12:34:47.170 | INFO | Analyzing chunk frames = [3247104,3247616) | 20260505 12:34:47.170 | INFO | MemInfo: { | 20260505 12:34:47.170 | INFO | "gpu" : " 17 GB", | 20260505 12:34:47.170 | INFO | "huge" : "", | 20260505 12:34:47.170 | INFO | "huge_shared" : " 27 GB", | 20260505 12:34:47.170 | INFO | "malloc" : " 14 GB", | 20260505 12:34:47.170 | INFO | "pinned" : " 29 GB" | 20260505 12:34:47.171 | INFO | } | 20260505 12:34:47.171 | INFO | Perf report for the last 5.14152 seconds: | 20260505 12:34:47.171 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:34:47.171 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:47.171 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:47.171 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:47.171 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 12:34:47.171 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:47.171 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:47.171 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 12:34:47.171 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:47.171 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:47.171 | INFO | Uploaded 36GiB to GPU in 2.13436s (16.8669GiB/s) | 20260505 12:34:47.171 | INFO | Downloaded 34GiB to GPU in 2.14551s (15.8471GiB/s) | 20260505 12:34:52.311 | INFO | Analyzing chunk frames = [3247616,3248128) | 20260505 12:34:52.311 | INFO | MemInfo: { | 20260505 12:34:52.311 | INFO | "gpu" : " 17 GB", | 20260505 12:34:52.311 | INFO | "huge" : "", | 20260505 12:34:52.311 | INFO | "huge_shared" : " 27 GB", | 20260505 12:34:52.311 | INFO | "malloc" : " 14 GB", | 20260505 12:34:52.311 | INFO | "pinned" : " 29 GB" | 20260505 12:34:52.311 | INFO | } | 20260505 12:34:52.311 | INFO | Perf report for the last 5.1401 seconds: | 20260505 12:34:52.311 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:34:52.311 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:52.311 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:52.311 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:52.311 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 12:34:52.311 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:52.311 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:52.311 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:34:52.311 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:52.311 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:52.311 | INFO | Uploaded 36GiB to GPU in 2.21294s (16.268GiB/s) | 20260505 12:34:52.311 | INFO | Downloaded 34GiB to GPU in 2.10515s (16.1509GiB/s) | 20260505 12:34:52.445 | INFO | Starting a round of Baz IO | 20260505 12:34:55.684 | INFO | Populating RTMetrics report | 20260505 12:34:55.684 | INFO | RTMetrics report populated | 20260505 12:34:55.685 | INFO | Opening tmp RTMetrics json file | 20260505 12:34:55.728 | INFO | Serializing RTMetrics to json | 20260505 12:34:55.757 | INFO | Serialization done, swapping tmp file now... | 20260505 12:34:55.762 | INFO | RTMetrics done | 20260505 12:34:55.793 | INFO | Finished a round of Baz IO: Wrote 2.332 GiB in 3.348 seconds | 20260505 12:34:55.793 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.51% / 0.01% / 76.48% | 20260505 12:34:55.793 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.860818% / 7.94729e-06% / 7.54992e-05% | 20260505 12:34:55.793 | INFO | Baz IO overall: | 20260505 12:34:55.793 | INFO | BytesWritten : 1930857417834 | 20260505 12:34:55.793 | INFO | NumEvents : 919033999079 | 20260505 12:34:57.448 | INFO | Analyzing chunk frames = [3248128,3248640) | 20260505 12:34:57.448 | INFO | MemInfo: { | 20260505 12:34:57.448 | INFO | "gpu" : " 17 GB", | 20260505 12:34:57.448 | INFO | "huge" : "", | 20260505 12:34:57.448 | INFO | "huge_shared" : " 27 GB", | 20260505 12:34:57.448 | INFO | "malloc" : " 14 GB", | 20260505 12:34:57.448 | INFO | "pinned" : " 29 GB" | 20260505 12:34:57.448 | INFO | } | 20260505 12:34:57.448 | INFO | Perf report for the last 5.13691 seconds: | 20260505 12:34:57.448 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:34:57.448 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:57.448 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:57.448 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:57.448 | INFO | ANALYSIS : 64.5%, 2.9, 99.5ms, 0 | 20260505 12:34:57.448 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:34:57.448 | INFO | RT_METRICS : 46.6%, 1.0, 24.4ms, 0 | 20260505 12:34:57.448 | INFO | PRE_HQ : 61.1%, 1.8, 58.1ms, 0 | 20260505 12:34:57.448 | INFO | BAZWRITER : 64.5%, 4.1, 135.2ms, 0 | 20260505 12:34:57.448 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 12:34:57.448 | INFO | Uploaded 36GiB to GPU in 2.19997s (16.3639GiB/s) | 20260505 12:34:57.448 | INFO | Downloaded 38GiB to GPU in 2.27922s (16.6724GiB/s) | 20260505 12:35:02.589 | INFO | Analyzing chunk frames = [3248640,3249152) | 20260505 12:35:02.589 | INFO | MemInfo: { | 20260505 12:35:02.589 | INFO | "gpu" : " 17 GB", | 20260505 12:35:02.589 | INFO | "huge" : "", | 20260505 12:35:02.589 | INFO | "huge_shared" : " 27 GB", | 20260505 12:35:02.589 | INFO | "malloc" : " 14 GB", | 20260505 12:35:02.589 | INFO | "pinned" : " 29 GB" | 20260505 12:35:02.589 | INFO | } | 20260505 12:35:02.589 | INFO | Perf report for the last 5.14174 seconds: | 20260505 12:35:02.589 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:35:02.589 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:02.589 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:02.589 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:02.589 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 12:35:02.590 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:02.590 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:02.590 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:35:02.590 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:02.590 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:02.590 | INFO | Uploaded 36GiB to GPU in 2.17041s (16.5867GiB/s) | 20260505 12:35:02.590 | INFO | Downloaded 34GiB to GPU in 2.0281s (16.7644GiB/s) | 20260505 12:35:07.721 | INFO | Analyzing chunk frames = [3249152,3249664) | 20260505 12:35:07.721 | INFO | MemInfo: { | 20260505 12:35:07.721 | INFO | "gpu" : " 17 GB", | 20260505 12:35:07.721 | INFO | "huge" : "", | 20260505 12:35:07.721 | INFO | "huge_shared" : " 27 GB", | 20260505 12:35:07.721 | INFO | "malloc" : " 14 GB", | 20260505 12:35:07.721 | INFO | "pinned" : " 29 GB" | 20260505 12:35:07.721 | INFO | } | 20260505 12:35:07.721 | INFO | Perf report for the last 5.13191 seconds: | 20260505 12:35:07.721 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:35:07.721 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:07.721 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:07.721 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:07.721 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 12:35:07.722 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:07.722 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:07.722 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:35:07.722 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:07.722 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:07.722 | INFO | Uploaded 36GiB to GPU in 2.21492s (16.2534GiB/s) | 20260505 12:35:07.722 | INFO | Downloaded 34GiB to GPU in 2.12027s (16.0357GiB/s) | 20260505 12:35:12.858 | INFO | Analyzing chunk frames = [3249664,3250176) | 20260505 12:35:12.858 | INFO | MemInfo: { | 20260505 12:35:12.858 | INFO | "gpu" : " 17 GB", | 20260505 12:35:12.858 | INFO | "huge" : "", | 20260505 12:35:12.858 | INFO | "huge_shared" : " 27 GB", | 20260505 12:35:12.858 | INFO | "malloc" : " 14 GB", | 20260505 12:35:12.858 | INFO | "pinned" : " 29 GB" | 20260505 12:35:12.858 | INFO | } | 20260505 12:35:12.858 | INFO | Perf report for the last 5.13702 seconds: | 20260505 12:35:12.858 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:35:12.858 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:12.858 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:12.858 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:12.858 | INFO | ANALYSIS : 63.4%, 3.0, 98.4ms, 0 | 20260505 12:35:12.859 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:12.859 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:12.859 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 12:35:12.859 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:12.859 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:12.859 | INFO | Uploaded 36GiB to GPU in 2.21499s (16.2529GiB/s) | 20260505 12:35:12.859 | INFO | Downloaded 34GiB to GPU in 2.13157s (15.9507GiB/s) | 20260505 12:35:18.008 | INFO | Analyzing chunk frames = [3250176,3250688) | 20260505 12:35:18.008 | INFO | MemInfo: { | 20260505 12:35:18.008 | INFO | "gpu" : " 17 GB", | 20260505 12:35:18.008 | INFO | "huge" : "", | 20260505 12:35:18.008 | INFO | "huge_shared" : " 27 GB", | 20260505 12:35:18.008 | INFO | "malloc" : " 14 GB", | 20260505 12:35:18.008 | INFO | "pinned" : " 29 GB" | 20260505 12:35:18.008 | INFO | } | 20260505 12:35:18.008 | INFO | Perf report for the last 5.14999 seconds: | 20260505 12:35:18.008 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:35:18.008 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:18.008 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:18.008 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:18.009 | INFO | ANALYSIS : 63.1%, 3.0, 98.7ms, 0 | 20260505 12:35:18.009 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:18.009 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:18.009 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 12:35:18.009 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:18.009 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:18.009 | INFO | Uploaded 36GiB to GPU in 2.1873s (16.4586GiB/s) | 20260505 12:35:18.009 | INFO | Downloaded 34GiB to GPU in 2.13165s (15.9501GiB/s) | 20260505 12:35:23.153 | INFO | Analyzing chunk frames = [3250688,3251200) | 20260505 12:35:23.153 | INFO | MemInfo: { | 20260505 12:35:23.153 | INFO | "gpu" : " 17 GB", | 20260505 12:35:23.153 | INFO | "huge" : "", | 20260505 12:35:23.153 | INFO | "huge_shared" : " 27 GB", | 20260505 12:35:23.153 | INFO | "malloc" : " 14 GB", | 20260505 12:35:23.153 | INFO | "pinned" : " 29 GB" | 20260505 12:35:23.153 | INFO | } | 20260505 12:35:23.153 | INFO | Perf report for the last 5.14465 seconds: | 20260505 12:35:23.153 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:35:23.153 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:23.153 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:23.153 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:23.153 | INFO | ANALYSIS : 63.1%, 3.0, 98.7ms, 0 | 20260505 12:35:23.153 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:23.153 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:23.153 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 12:35:23.153 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:23.153 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:23.153 | INFO | Uploaded 36GiB to GPU in 2.16435s (16.6332GiB/s) | 20260505 12:35:23.153 | INFO | Downloaded 34GiB to GPU in 2.10804s (16.1287GiB/s) | 20260505 12:35:28.290 | INFO | Analyzing chunk frames = [3251200,3251712) | 20260505 12:35:28.290 | INFO | MemInfo: { | 20260505 12:35:28.290 | INFO | "gpu" : " 17 GB", | 20260505 12:35:28.290 | INFO | "huge" : "", | 20260505 12:35:28.290 | INFO | "huge_shared" : " 27 GB", | 20260505 12:35:28.290 | INFO | "malloc" : " 14 GB", | 20260505 12:35:28.290 | INFO | "pinned" : " 29 GB" | 20260505 12:35:28.290 | INFO | } | 20260505 12:35:28.290 | INFO | Perf report for the last 5.13678 seconds: | 20260505 12:35:28.290 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:35:28.290 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:28.290 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:28.290 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:28.290 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 12:35:28.290 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:28.290 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:28.290 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 12:35:28.290 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:28.290 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:28.290 | INFO | Uploaded 36GiB to GPU in 2.13469s (16.8643GiB/s) | 20260505 12:35:28.290 | INFO | Downloaded 34GiB to GPU in 2.14921s (15.8198GiB/s) | 20260505 12:35:33.418 | INFO | Analyzing chunk frames = [3251712,3252224) | 20260505 12:35:33.418 | INFO | MemInfo: { | 20260505 12:35:33.418 | INFO | "gpu" : " 17 GB", | 20260505 12:35:33.418 | INFO | "huge" : "", | 20260505 12:35:33.418 | INFO | "huge_shared" : " 27 GB", | 20260505 12:35:33.418 | INFO | "malloc" : " 14 GB", | 20260505 12:35:33.418 | INFO | "pinned" : " 29 GB" | 20260505 12:35:33.418 | INFO | } | 20260505 12:35:33.418 | INFO | Perf report for the last 5.12792 seconds: | 20260505 12:35:33.418 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:35:33.418 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:33.418 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:33.418 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:33.418 | INFO | ANALYSIS : 63.9%, 3.0, 99.6ms, 0 | 20260505 12:35:33.418 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:33.418 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:33.418 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 12:35:33.418 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:33.418 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:33.418 | INFO | Uploaded 36GiB to GPU in 2.1792s (16.5198GiB/s) | 20260505 12:35:33.418 | INFO | Downloaded 34GiB to GPU in 2.11869s (16.0477GiB/s) | 20260505 12:35:33.418 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 12:35:33.418 | INFO | Total mem usage: 15.3 GiB | 20260505 12:35:33.418 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 12:35:33.418 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -6.88%) | 20260505 12:35:33.418 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 12:35:33.418 | INFO | High Water Breakdown: | 20260505 12:35:33.418 | INFO | 61.1% : 8.76 GiB ( 92.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 12:35:33.418 | INFO | 38.8% : 5.56 GiB ( 83.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 12:35:33.418 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 12:35:33.418 | INFO | ------------------------------------------ | 20260505 12:35:33.418 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:35:33.418 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 12:35:33.418 | INFO | Total mem usage: 26.5 GiB | 20260505 12:35:33.418 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 12:35:33.418 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 12:35:33.419 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 12:35:33.419 | INFO | High Water Breakdown: | 20260505 12:35:33.419 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:35:33.419 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:35:33.419 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:35:33.419 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:35:33.419 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:35:33.419 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:35:33.419 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 12:35:33.419 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 12:35:33.419 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:35:33.419 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 12:35:33.419 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 12:35:33.419 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 12:35:33.419 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 12:35:33.419 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 12:35:33.419 | INFO | ------------------------------------------ | 20260505 12:35:33.419 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:35:33.419 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 12:35:33.419 | INFO | Total mem usage: 25.6 GiB | 20260505 12:35:33.419 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 12:35:33.419 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 12:35:33.419 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 12:35:33.419 | INFO | High Water Breakdown: | 20260505 12:35:33.419 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 12:35:33.419 | INFO | ------------------------------------------ | 20260505 12:35:33.419 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:35:33.419 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 12:35:33.419 | INFO | Total mem usage: 15.7 GiB | 20260505 12:35:33.419 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 12:35:33.419 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 12:35:33.419 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 12:35:33.419 | INFO | High Water Breakdown: | 20260505 12:35:33.419 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 12:35:33.420 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 12:35:33.420 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 12:35:33.420 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 12:35:33.420 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 12:35:33.420 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 12:35:33.420 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 12:35:33.420 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 12:35:33.420 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 12:35:33.420 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:35:33.420 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 12:35:33.420 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:35:33.420 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 12:35:33.420 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 12:35:33.420 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 12:35:33.420 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 12:35:33.420 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:35:33.420 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 12:35:33.420 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:35:33.420 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:35:33.420 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:35:33.420 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 12:35:33.420 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 12:35:33.420 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:35:33.420 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 12:35:33.420 | INFO | ------------------------------------------ | 20260505 12:35:33.420 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:35:33.420 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 12:35:33.420 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:35:33.420 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:35:33.420 | INFO | PerfReport: Full Execution -- 3.20e+01 / 7.69e+01, 1.37e+01 / 1.47e+00 | 20260505 12:35:33.420 | INFO | PerfReport: 0.29%, AggregateStats -- 9.32e-02 / 1.02e-01, 1.09e-02 / 5.63e-03 | 20260505 12:35:33.420 | INFO | PerfReport: 11.76%, Baseline -- 3.76e+00 / 4.51e+00, 5.94e-01 / 2.06e-01 | 20260505 12:35:33.420 | INFO | PerfReport: 3.32%, Histogram -- 1.06e+00 / 1.50e+00, 2.02e-01 / 9.60e-02 | 20260505 12:35:33.420 | INFO | PerfReport: 15.83%, DME -- 4.05e+01 / 4.51e+01, 4.36e+00 / 1.82e+00 | 20260505 12:35:33.420 | INFO | PerfReport: 56.46%, FrameLabeling -- 1.81e+01 / 2.09e+01, 1.50e+00 / 1.80e+00 | 20260505 12:35:33.420 | INFO | PerfReport: 7.42%, PulseAccumulating -- 2.37e+00 / 2.87e+00, 4.36e-01 / 1.76e-01 | 20260505 12:35:33.420 | INFO | PerfReport: 3.64%, Metrics -- 1.16e+00 / 1.47e+00, 3.29e-01 / 1.67e-01 | 20260505 12:35:33.420 | INFO | PerfReport: 1.27%, PulseSerializing -- 4.07e-01 / 4.53e-01, 7.31e-02 / 2.32e-02 | 20260505 12:35:33.420 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 12:35:33.420 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:35:33.420 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:35:33.420 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260505 12:35:33.420 | INFO | PerfReport: 22.53%, Upload -- 2.27e+01 / 0.00e+00, 7.93e+00 / 0.00e+00 | 20260505 12:35:33.420 | INFO | PerfReport: 22.02%, Download -- 2.22e+01 / 0.00e+00, 8.09e+00 / 0.00e+00 | 20260505 12:35:33.420 | INFO | PerfReport: 31.70%, Compute -- 3.20e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 12:35:33.420 | INFO | PerfReport: 2.14%, LockUpload -- 2.16e+00 / 0.00e+00, 7.80e+00 / 0.00e+00 | 20260505 12:35:33.420 | INFO | PerfReport: 1.37%, LockDownload -- 1.38e+00 / 0.00e+00, 2.99e+00 / 0.00e+00 | 20260505 12:35:33.420 | INFO | PerfReport: 20.22%, LockCompute -- 2.04e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260505 12:35:33.420 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.89e-02 / 0.00e+00 | 20260505 12:35:33.553 | INFO | Starting a round of Baz IO | 20260505 12:35:36.804 | INFO | Populating RTMetrics report | 20260505 12:35:36.805 | INFO | RTMetrics report populated | 20260505 12:35:36.805 | INFO | Opening tmp RTMetrics json file | 20260505 12:35:36.821 | INFO | Serializing RTMetrics to json | 20260505 12:35:36.827 | INFO | Serialization done, swapping tmp file now... | 20260505 12:35:36.827 | INFO | RTMetrics done | 20260505 12:35:36.926 | INFO | Finished a round of Baz IO: Wrote 2.332 GiB in 3.372 seconds | 20260505 12:35:36.926 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.51% / 0.01% / 76.49% | 20260505 12:35:36.926 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.85694% / 7.94729e-06% / 7.94729e-05% | 20260505 12:35:36.926 | INFO | Baz IO overall: | 20260505 12:35:36.926 | INFO | BytesWritten : 1933361103087 | 20260505 12:35:36.926 | INFO | NumEvents : 920221269710 | 20260505 12:35:38.563 | INFO | Analyzing chunk frames = [3252224,3252736) | 20260505 12:35:38.563 | INFO | MemInfo: { | 20260505 12:35:38.563 | INFO | "gpu" : " 17 GB", | 20260505 12:35:38.563 | INFO | "huge" : "", | 20260505 12:35:38.563 | INFO | "huge_shared" : " 27 GB", | 20260505 12:35:38.563 | INFO | "malloc" : " 14 GB", | 20260505 12:35:38.563 | INFO | "pinned" : " 29 GB" | 20260505 12:35:38.563 | INFO | } | 20260505 12:35:38.563 | INFO | Perf report for the last 5.14493 seconds: | 20260505 12:35:38.563 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:35:38.563 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:38.563 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:38.563 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:38.563 | INFO | ANALYSIS : 64.7%, 3.0, 100.3ms, 0 | 20260505 12:35:38.563 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:38.563 | INFO | RT_METRICS : 45.5%, 1.0, 23.9ms, 0 | 20260505 12:35:38.563 | INFO | PRE_HQ : 61.0%, 1.8, 58.0ms, 0 | 20260505 12:35:38.563 | INFO | BAZWRITER : 64.9%, 4.0, 134.9ms, 0 | 20260505 12:35:38.563 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 12:35:38.563 | INFO | Uploaded 36GiB to GPU in 2.19423s (16.4066GiB/s) | 20260505 12:35:38.563 | INFO | Downloaded 38GiB to GPU in 2.30126s (16.5127GiB/s) | 20260505 12:35:43.706 | INFO | Analyzing chunk frames = [3252736,3253248) | 20260505 12:35:43.706 | INFO | MemInfo: { | 20260505 12:35:43.706 | INFO | "gpu" : " 17 GB", | 20260505 12:35:43.706 | INFO | "huge" : "", | 20260505 12:35:43.706 | INFO | "huge_shared" : " 27 GB", | 20260505 12:35:43.706 | INFO | "malloc" : " 14 GB", | 20260505 12:35:43.706 | INFO | "pinned" : " 29 GB" | 20260505 12:35:43.706 | INFO | } | 20260505 12:35:43.706 | INFO | Perf report for the last 5.14335 seconds: | 20260505 12:35:43.706 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:35:43.706 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:43.706 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:43.706 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:43.706 | INFO | ANALYSIS : 63.7%, 3.0, 99.6ms, 0 | 20260505 12:35:43.706 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:43.706 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:43.706 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 12:35:43.706 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:43.706 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:43.706 | INFO | Uploaded 36GiB to GPU in 2.17865s (16.524GiB/s) | 20260505 12:35:43.707 | INFO | Downloaded 34GiB to GPU in 2.05176s (16.5711GiB/s) | 20260505 12:35:48.845 | INFO | Analyzing chunk frames = [3253248,3253760) | 20260505 12:35:48.845 | INFO | MemInfo: { | 20260505 12:35:48.845 | INFO | "gpu" : " 17 GB", | 20260505 12:35:48.845 | INFO | "huge" : "", | 20260505 12:35:48.845 | INFO | "huge_shared" : " 27 GB", | 20260505 12:35:48.845 | INFO | "malloc" : " 14 GB", | 20260505 12:35:48.845 | INFO | "pinned" : " 29 GB" | 20260505 12:35:48.845 | INFO | } | 20260505 12:35:48.845 | INFO | Perf report for the last 5.13862 seconds: | 20260505 12:35:48.845 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:35:48.845 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:48.845 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:48.845 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:48.845 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 12:35:48.845 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:48.845 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:48.845 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 12:35:48.845 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:48.845 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:48.845 | INFO | Uploaded 36GiB to GPU in 2.19993s (16.3642GiB/s) | 20260505 12:35:48.845 | INFO | Downloaded 34GiB to GPU in 2.11756s (16.0562GiB/s) | 20260505 12:35:53.981 | INFO | Analyzing chunk frames = [3253760,3254272) | 20260505 12:35:53.981 | INFO | MemInfo: { | 20260505 12:35:53.981 | INFO | "gpu" : " 17 GB", | 20260505 12:35:53.981 | INFO | "huge" : "", | 20260505 12:35:53.981 | INFO | "huge_shared" : " 27 GB", | 20260505 12:35:53.981 | INFO | "malloc" : " 14 GB", | 20260505 12:35:53.981 | INFO | "pinned" : " 29 GB" | 20260505 12:35:53.982 | INFO | } | 20260505 12:35:53.982 | INFO | Perf report for the last 5.13688 seconds: | 20260505 12:35:53.982 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:35:53.982 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:53.982 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:53.982 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:53.982 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 12:35:53.982 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:53.982 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:53.982 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 12:35:53.982 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:53.982 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:53.982 | INFO | Uploaded 36GiB to GPU in 2.20675s (16.3136GiB/s) | 20260505 12:35:53.982 | INFO | Downloaded 34GiB to GPU in 2.13244s (15.9442GiB/s) | 20260505 12:35:59.118 | INFO | Analyzing chunk frames = [3254272,3254784) | 20260505 12:35:59.118 | INFO | MemInfo: { | 20260505 12:35:59.118 | INFO | "gpu" : " 17 GB", | 20260505 12:35:59.118 | INFO | "huge" : "", | 20260505 12:35:59.118 | INFO | "huge_shared" : " 27 GB", | 20260505 12:35:59.118 | INFO | "malloc" : " 14 GB", | 20260505 12:35:59.118 | INFO | "pinned" : " 29 GB" | 20260505 12:35:59.118 | INFO | } | 20260505 12:35:59.118 | INFO | Perf report for the last 5.13671 seconds: | 20260505 12:35:59.118 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:35:59.118 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:59.118 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:59.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:59.118 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 12:35:59.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:59.118 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:59.118 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 12:35:59.118 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:59.118 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:35:59.119 | INFO | Uploaded 36GiB to GPU in 2.17873s (16.5234GiB/s) | 20260505 12:35:59.119 | INFO | Downloaded 34GiB to GPU in 2.12635s (15.9899GiB/s) | 20260505 12:36:04.266 | INFO | Analyzing chunk frames = [3254784,3255296) | 20260505 12:36:04.266 | INFO | MemInfo: { | 20260505 12:36:04.266 | INFO | "gpu" : " 17 GB", | 20260505 12:36:04.266 | INFO | "huge" : "", | 20260505 12:36:04.266 | INFO | "huge_shared" : " 27 GB", | 20260505 12:36:04.266 | INFO | "malloc" : " 14 GB", | 20260505 12:36:04.266 | INFO | "pinned" : " 29 GB" | 20260505 12:36:04.267 | INFO | } | 20260505 12:36:04.267 | INFO | Perf report for the last 5.14837 seconds: | 20260505 12:36:04.267 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:36:04.267 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:04.267 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:04.267 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:04.267 | INFO | ANALYSIS : 63.1%, 3.0, 98.7ms, 0 | 20260505 12:36:04.267 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:04.267 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:04.267 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 12:36:04.267 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:04.267 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:04.267 | INFO | Uploaded 36GiB to GPU in 2.16734s (16.6102GiB/s) | 20260505 12:36:04.267 | INFO | Downloaded 34GiB to GPU in 2.11399s (16.0833GiB/s) | 20260505 12:36:09.397 | INFO | Analyzing chunk frames = [3255296,3255808) | 20260505 12:36:09.397 | INFO | MemInfo: { | 20260505 12:36:09.397 | INFO | "gpu" : " 17 GB", | 20260505 12:36:09.397 | INFO | "huge" : "", | 20260505 12:36:09.397 | INFO | "huge_shared" : " 27 GB", | 20260505 12:36:09.397 | INFO | "malloc" : " 14 GB", | 20260505 12:36:09.397 | INFO | "pinned" : " 29 GB" | 20260505 12:36:09.397 | INFO | } | 20260505 12:36:09.397 | INFO | Perf report for the last 5.13026 seconds: | 20260505 12:36:09.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:36:09.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:09.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:09.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:09.397 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 12:36:09.397 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:09.397 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:09.397 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 12:36:09.397 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:09.397 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:09.397 | INFO | Uploaded 36GiB to GPU in 2.15699s (16.6899GiB/s) | 20260505 12:36:09.397 | INFO | Downloaded 34GiB to GPU in 2.14955s (15.8172GiB/s) | 20260505 12:36:14.533 | INFO | Analyzing chunk frames = [3255808,3256320) | 20260505 12:36:14.533 | INFO | MemInfo: { | 20260505 12:36:14.533 | INFO | "gpu" : " 17 GB", | 20260505 12:36:14.533 | INFO | "huge" : "", | 20260505 12:36:14.533 | INFO | "huge_shared" : " 27 GB", | 20260505 12:36:14.533 | INFO | "malloc" : " 14 GB", | 20260505 12:36:14.533 | INFO | "pinned" : " 29 GB" | 20260505 12:36:14.533 | INFO | } | 20260505 12:36:14.533 | INFO | Perf report for the last 5.13561 seconds: | 20260505 12:36:14.533 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:36:14.533 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:14.533 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:14.533 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:14.533 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 12:36:14.533 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:14.533 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:14.533 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 12:36:14.533 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:14.533 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:14.533 | INFO | Uploaded 36GiB to GPU in 2.22161s (16.2044GiB/s) | 20260505 12:36:14.533 | INFO | Downloaded 34GiB to GPU in 2.11064s (16.1089GiB/s) | 20260505 12:36:14.670 | INFO | Starting a round of Baz IO | 20260505 12:36:17.918 | INFO | Populating RTMetrics report | 20260505 12:36:17.918 | INFO | RTMetrics report populated | 20260505 12:36:17.918 | INFO | Opening tmp RTMetrics json file | 20260505 12:36:17.920 | INFO | Serializing RTMetrics to json | 20260505 12:36:17.921 | INFO | Serialization done, swapping tmp file now... | 20260505 12:36:17.923 | INFO | RTMetrics done | 20260505 12:36:18.040 | INFO | Finished a round of Baz IO: Wrote 2.333 GiB in 3.37 seconds | 20260505 12:36:18.040 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.50% / 0.01% / 76.49% | 20260505 12:36:18.040 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.857127% / 3.97364e-06% / 6.35783e-05% | 20260505 12:36:18.040 | INFO | Baz IO overall: | 20260505 12:36:18.040 | INFO | BytesWritten : 1935866312089 | 20260505 12:36:18.040 | INFO | NumEvents : 921409689357 | 20260505 12:36:19.675 | INFO | Analyzing chunk frames = [3256320,3256832) | 20260505 12:36:19.675 | INFO | MemInfo: { | 20260505 12:36:19.675 | INFO | "gpu" : " 17 GB", | 20260505 12:36:19.675 | INFO | "huge" : "", | 20260505 12:36:19.675 | INFO | "huge_shared" : " 27 GB", | 20260505 12:36:19.675 | INFO | "malloc" : " 14 GB", | 20260505 12:36:19.675 | INFO | "pinned" : " 29 GB" | 20260505 12:36:19.675 | INFO | } | 20260505 12:36:19.675 | INFO | Perf report for the last 5.14266 seconds: | 20260505 12:36:19.675 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:36:19.675 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:19.675 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:19.675 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:19.675 | INFO | ANALYSIS : 64.7%, 2.9, 100.1ms, 0 | 20260505 12:36:19.675 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:19.675 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260505 12:36:19.675 | INFO | PRE_HQ : 61.0%, 1.8, 58.1ms, 0 | 20260505 12:36:19.675 | INFO | BAZWRITER : 64.9%, 4.0, 134.8ms, 0 | 20260505 12:36:19.675 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 12:36:19.676 | INFO | Uploaded 36GiB to GPU in 2.19218s (16.422GiB/s) | 20260505 12:36:19.676 | INFO | Downloaded 38GiB to GPU in 2.30374s (16.4949GiB/s) | 20260505 12:36:24.818 | INFO | Analyzing chunk frames = [3256832,3257344) | 20260505 12:36:24.818 | INFO | MemInfo: { | 20260505 12:36:24.818 | INFO | "gpu" : " 17 GB", | 20260505 12:36:24.818 | INFO | "huge" : "", | 20260505 12:36:24.818 | INFO | "huge_shared" : " 27 GB", | 20260505 12:36:24.819 | INFO | "malloc" : " 14 GB", | 20260505 12:36:24.819 | INFO | "pinned" : " 29 GB" | 20260505 12:36:24.819 | INFO | } | 20260505 12:36:24.819 | INFO | Perf report for the last 5.14324 seconds: | 20260505 12:36:24.819 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:36:24.819 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:24.819 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:24.819 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:24.819 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 12:36:24.819 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:24.819 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:24.819 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:36:24.819 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:24.819 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:24.819 | INFO | Uploaded 36GiB to GPU in 2.23791s (16.0864GiB/s) | 20260505 12:36:24.819 | INFO | Downloaded 34GiB to GPU in 2.10092s (16.1834GiB/s) | 20260505 12:36:29.960 | INFO | Analyzing chunk frames = [3257344,3257856) | 20260505 12:36:29.960 | INFO | MemInfo: { | 20260505 12:36:29.960 | INFO | "gpu" : " 17 GB", | 20260505 12:36:29.960 | INFO | "huge" : "", | 20260505 12:36:29.960 | INFO | "huge_shared" : " 27 GB", | 20260505 12:36:29.960 | INFO | "malloc" : " 14 GB", | 20260505 12:36:29.960 | INFO | "pinned" : " 29 GB" | 20260505 12:36:29.960 | INFO | } | 20260505 12:36:29.960 | INFO | Perf report for the last 5.14162 seconds: | 20260505 12:36:29.960 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:36:29.960 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:29.960 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:29.960 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:29.960 | INFO | ANALYSIS : 63.6%, 3.0, 98.9ms, 0 | 20260505 12:36:29.960 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:29.960 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:29.960 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 12:36:29.960 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:29.960 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:29.960 | INFO | Uploaded 36GiB to GPU in 2.23933s (16.0762GiB/s) | 20260505 12:36:29.960 | INFO | Downloaded 34GiB to GPU in 2.08932s (16.2732GiB/s) | 20260505 12:36:35.101 | INFO | Analyzing chunk frames = [3257856,3258368) | 20260505 12:36:35.101 | INFO | MemInfo: { | 20260505 12:36:35.101 | INFO | "gpu" : " 17 GB", | 20260505 12:36:35.101 | INFO | "huge" : "", | 20260505 12:36:35.101 | INFO | "huge_shared" : " 27 GB", | 20260505 12:36:35.101 | INFO | "malloc" : " 14 GB", | 20260505 12:36:35.101 | INFO | "pinned" : " 29 GB" | 20260505 12:36:35.101 | INFO | } | 20260505 12:36:35.101 | INFO | Perf report for the last 5.14116 seconds: | 20260505 12:36:35.101 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:36:35.101 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:35.101 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:35.101 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:35.101 | INFO | ANALYSIS : 63.0%, 3.0, 98.0ms, 0 | 20260505 12:36:35.101 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:35.101 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:35.101 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:36:35.101 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:35.101 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:35.102 | INFO | Uploaded 36GiB to GPU in 2.20255s (16.3447GiB/s) | 20260505 12:36:35.102 | INFO | Downloaded 34GiB to GPU in 2.12565s (15.9951GiB/s) | 20260505 12:36:40.230 | INFO | Analyzing chunk frames = [3258368,3258880) | 20260505 12:36:40.230 | INFO | MemInfo: { | 20260505 12:36:40.230 | INFO | "gpu" : " 17 GB", | 20260505 12:36:40.230 | INFO | "huge" : "", | 20260505 12:36:40.230 | INFO | "huge_shared" : " 27 GB", | 20260505 12:36:40.230 | INFO | "malloc" : " 14 GB", | 20260505 12:36:40.230 | INFO | "pinned" : " 29 GB" | 20260505 12:36:40.230 | INFO | } | 20260505 12:36:40.230 | INFO | Perf report for the last 5.12891 seconds: | 20260505 12:36:40.230 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:36:40.230 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:40.230 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:40.230 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:40.230 | INFO | ANALYSIS : 63.1%, 3.0, 98.3ms, 0 | 20260505 12:36:40.230 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:40.230 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:40.230 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:36:40.230 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:40.230 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:40.231 | INFO | Uploaded 36GiB to GPU in 2.15521s (16.7037GiB/s) | 20260505 12:36:40.231 | INFO | Downloaded 34GiB to GPU in 2.1212s (16.0287GiB/s) | 20260505 12:36:45.370 | INFO | Analyzing chunk frames = [3258880,3259392) | 20260505 12:36:45.370 | INFO | MemInfo: { | 20260505 12:36:45.370 | INFO | "gpu" : " 17 GB", | 20260505 12:36:45.370 | INFO | "huge" : "", | 20260505 12:36:45.370 | INFO | "huge_shared" : " 27 GB", | 20260505 12:36:45.370 | INFO | "malloc" : " 14 GB", | 20260505 12:36:45.370 | INFO | "pinned" : " 29 GB" | 20260505 12:36:45.370 | INFO | } | 20260505 12:36:45.370 | INFO | Perf report for the last 5.13937 seconds: | 20260505 12:36:45.370 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:36:45.370 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:45.370 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:45.370 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:45.370 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 12:36:45.370 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:45.370 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:45.370 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 12:36:45.370 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:45.370 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:45.370 | INFO | Uploaded 36GiB to GPU in 2.15743s (16.6865GiB/s) | 20260505 12:36:45.370 | INFO | Downloaded 34GiB to GPU in 2.11945s (16.0419GiB/s) | 20260505 12:36:50.517 | INFO | Analyzing chunk frames = [3259392,3259904) | 20260505 12:36:50.517 | INFO | MemInfo: { | 20260505 12:36:50.517 | INFO | "gpu" : " 17 GB", | 20260505 12:36:50.517 | INFO | "huge" : "", | 20260505 12:36:50.517 | INFO | "huge_shared" : " 27 GB", | 20260505 12:36:50.517 | INFO | "malloc" : " 14 GB", | 20260505 12:36:50.517 | INFO | "pinned" : " 29 GB" | 20260505 12:36:50.518 | INFO | } | 20260505 12:36:50.518 | INFO | Perf report for the last 5.14805 seconds: | 20260505 12:36:50.518 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:36:50.518 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:50.518 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:50.518 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:50.518 | INFO | ANALYSIS : 63.4%, 3.0, 99.2ms, 0 | 20260505 12:36:50.518 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:50.518 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:50.518 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 12:36:50.518 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:50.518 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:50.518 | INFO | Uploaded 36GiB to GPU in 2.19029s (16.4362GiB/s) | 20260505 12:36:50.518 | INFO | Downloaded 34GiB to GPU in 2.14186s (15.874GiB/s) | 20260505 12:36:55.658 | INFO | Analyzing chunk frames = [3259904,3260416) | 20260505 12:36:55.658 | INFO | MemInfo: { | 20260505 12:36:55.658 | INFO | "gpu" : " 17 GB", | 20260505 12:36:55.658 | INFO | "huge" : "", | 20260505 12:36:55.658 | INFO | "huge_shared" : " 27 GB", | 20260505 12:36:55.658 | INFO | "malloc" : " 14 GB", | 20260505 12:36:55.658 | INFO | "pinned" : " 29 GB" | 20260505 12:36:55.658 | INFO | } | 20260505 12:36:55.658 | INFO | Perf report for the last 5.14026 seconds: | 20260505 12:36:55.658 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:36:55.658 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:55.658 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:55.658 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:55.658 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 12:36:55.658 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:55.658 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:55.658 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 12:36:55.658 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:55.658 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:36:55.658 | INFO | Uploaded 36GiB to GPU in 2.18974s (16.4403GiB/s) | 20260505 12:36:55.658 | INFO | Downloaded 34GiB to GPU in 2.10765s (16.1317GiB/s) | 20260505 12:36:55.659 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 12:36:55.659 | INFO | Total mem usage: 15.3 GiB | 20260505 12:36:55.659 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 12:36:55.659 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -7.01%) | 20260505 12:36:55.659 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 12:36:55.659 | INFO | High Water Breakdown: | 20260505 12:36:55.659 | INFO | 61.1% : 8.75 GiB ( 92% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 12:36:55.659 | INFO | 38.7% : 5.55 GiB ( 83.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 12:36:55.659 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 12:36:55.659 | INFO | ------------------------------------------ | 20260505 12:36:55.659 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:36:55.659 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 12:36:55.659 | INFO | Total mem usage: 26.5 GiB | 20260505 12:36:55.659 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 12:36:55.659 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 12:36:55.659 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 12:36:55.659 | INFO | High Water Breakdown: | 20260505 12:36:55.659 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:36:55.659 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:36:55.659 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:36:55.659 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:36:55.659 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:36:55.659 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:36:55.659 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 12:36:55.659 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 12:36:55.659 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:36:55.659 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 12:36:55.659 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 12:36:55.659 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 12:36:55.659 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 12:36:55.659 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 12:36:55.659 | INFO | ------------------------------------------ | 20260505 12:36:55.659 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:36:55.659 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 12:36:55.659 | INFO | Total mem usage: 25.6 GiB | 20260505 12:36:55.659 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 12:36:55.659 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 12:36:55.659 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 12:36:55.659 | INFO | High Water Breakdown: | 20260505 12:36:55.659 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 12:36:55.659 | INFO | ------------------------------------------ | 20260505 12:36:55.659 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:36:55.659 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 12:36:55.659 | INFO | Total mem usage: 15.7 GiB | 20260505 12:36:55.659 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 12:36:55.659 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 12:36:55.659 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 12:36:55.659 | INFO | High Water Breakdown: | 20260505 12:36:55.659 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 12:36:55.659 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 12:36:55.659 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 12:36:55.659 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 12:36:55.659 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 12:36:55.659 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 12:36:55.659 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 12:36:55.659 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 12:36:55.659 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 12:36:55.659 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:36:55.659 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 12:36:55.659 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:36:55.659 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 12:36:55.659 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 12:36:55.659 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 12:36:55.659 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 12:36:55.659 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:36:55.659 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 12:36:55.659 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:36:55.659 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:36:55.660 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:36:55.660 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 12:36:55.660 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 12:36:55.660 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:36:55.660 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 12:36:55.660 | INFO | ------------------------------------------ | 20260505 12:36:55.660 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:36:55.660 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 12:36:55.660 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:36:55.660 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:36:55.660 | INFO | PerfReport: Full Execution -- 3.20e+01 / 7.73e+01, 1.36e+01 / 1.47e+00 | 20260505 12:36:55.660 | INFO | PerfReport: 0.29%, AggregateStats -- 9.26e-02 / 9.93e-02, 1.00e-02 / 5.25e-03 | 20260505 12:36:55.660 | INFO | PerfReport: 11.75%, Baseline -- 3.76e+00 / 4.51e+00, 5.91e-01 / 1.80e-01 | 20260505 12:36:55.660 | INFO | PerfReport: 3.30%, Histogram -- 1.05e+00 / 1.51e+00, 2.01e-01 / 7.79e-02 | 20260505 12:36:55.660 | INFO | PerfReport: 15.99%, DME -- 4.09e+01 / 4.61e+01, 4.66e+00 / 1.79e+00 | 20260505 12:36:55.660 | INFO | PerfReport: 56.32%, FrameLabeling -- 1.80e+01 / 2.03e+01, 1.44e+00 / 2.03e+00 | 20260505 12:36:55.660 | INFO | PerfReport: 7.44%, PulseAccumulating -- 2.38e+00 / 2.92e+00, 4.44e-01 / 2.07e-01 | 20260505 12:36:55.660 | INFO | PerfReport: 3.64%, Metrics -- 1.16e+00 / 1.47e+00, 3.27e-01 / 1.81e-01 | 20260505 12:36:55.660 | INFO | PerfReport: 1.28%, PulseSerializing -- 4.08e-01 / 4.55e-01, 7.40e-02 / 3.78e-02 | 20260505 12:36:55.660 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 12:36:55.660 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:36:55.660 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:36:55.660 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260505 12:36:55.660 | INFO | PerfReport: 22.57%, Upload -- 2.28e+01 / 0.00e+00, 8.09e+00 / 0.00e+00 | 20260505 12:36:55.660 | INFO | PerfReport: 22.02%, Download -- 2.23e+01 / 0.00e+00, 8.18e+00 / 0.00e+00 | 20260505 12:36:55.660 | INFO | PerfReport: 31.62%, Compute -- 3.20e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 12:36:55.660 | INFO | PerfReport: 2.32%, LockUpload -- 2.34e+00 / 0.00e+00, 8.00e+00 / 0.00e+00 | 20260505 12:36:55.660 | INFO | PerfReport: 1.32%, LockDownload -- 1.33e+00 / 0.00e+00, 2.90e+00 / 0.00e+00 | 20260505 12:36:55.660 | INFO | PerfReport: 20.14%, LockCompute -- 2.04e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260505 12:36:55.660 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.99e-02 / 0.00e+00 | 20260505 12:36:55.796 | INFO | Starting a round of Baz IO | 20260505 12:36:59.024 | INFO | Populating RTMetrics report | 20260505 12:36:59.024 | INFO | RTMetrics report populated | 20260505 12:36:59.024 | INFO | Opening tmp RTMetrics json file | 20260505 12:36:59.025 | INFO | Serializing RTMetrics to json | 20260505 12:36:59.028 | INFO | Serialization done, swapping tmp file now... | 20260505 12:36:59.030 | INFO | RTMetrics done | 20260505 12:36:59.151 | INFO | Finished a round of Baz IO: Wrote 2.333 GiB in 3.354 seconds | 20260505 12:36:59.151 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.49% / 0.01% / 76.50% | 20260505 12:36:59.151 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.853344% / 3.97364e-06% / 7.94729e-05% | 20260505 12:36:59.151 | INFO | Baz IO overall: | 20260505 12:36:59.151 | INFO | BytesWritten : 1938371591532 | 20260505 12:36:59.151 | INFO | NumEvents : 922597975350 | 20260505 12:37:00.795 | INFO | Analyzing chunk frames = [3260416,3260928) | 20260505 12:37:00.795 | INFO | MemInfo: { | 20260505 12:37:00.795 | INFO | "gpu" : " 17 GB", | 20260505 12:37:00.795 | INFO | "huge" : "", | 20260505 12:37:00.795 | INFO | "huge_shared" : " 27 GB", | 20260505 12:37:00.795 | INFO | "malloc" : " 14 GB", | 20260505 12:37:00.795 | INFO | "pinned" : " 29 GB" | 20260505 12:37:00.795 | INFO | } | 20260505 12:37:00.795 | INFO | Perf report for the last 5.13699 seconds: | 20260505 12:37:00.795 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:37:00.795 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:00.795 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:00.795 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:00.795 | INFO | ANALYSIS : 64.4%, 2.9, 99.5ms, 0 | 20260505 12:37:00.795 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:00.795 | INFO | RT_METRICS : 45.5%, 1.0, 23.8ms, 0 | 20260505 12:37:00.795 | INFO | PRE_HQ : 61.2%, 1.8, 58.3ms, 0 | 20260505 12:37:00.795 | INFO | BAZWRITER : 64.7%, 4.1, 135.3ms, 0 | 20260505 12:37:00.795 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 12:37:00.795 | INFO | Uploaded 36GiB to GPU in 2.18184s (16.4998GiB/s) | 20260505 12:37:00.795 | INFO | Downloaded 38GiB to GPU in 2.27952s (16.6702GiB/s) | 20260505 12:37:05.928 | INFO | Analyzing chunk frames = [3260928,3261440) | 20260505 12:37:05.928 | INFO | MemInfo: { | 20260505 12:37:05.928 | INFO | "gpu" : " 17 GB", | 20260505 12:37:05.928 | INFO | "huge" : "", | 20260505 12:37:05.928 | INFO | "huge_shared" : " 27 GB", | 20260505 12:37:05.928 | INFO | "malloc" : " 14 GB", | 20260505 12:37:05.928 | INFO | "pinned" : " 29 GB" | 20260505 12:37:05.928 | INFO | } | 20260505 12:37:05.928 | INFO | Perf report for the last 5.13284 seconds: | 20260505 12:37:05.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:37:05.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:05.928 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:05.928 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:05.928 | INFO | ANALYSIS : 63.8%, 3.0, 99.4ms, 0 | 20260505 12:37:05.928 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:05.928 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:05.928 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 12:37:05.928 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:05.928 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:05.928 | INFO | Uploaded 36GiB to GPU in 2.17679s (16.5381GiB/s) | 20260505 12:37:05.928 | INFO | Downloaded 34GiB to GPU in 2.04534s (16.6232GiB/s) | 20260505 12:37:11.075 | INFO | Analyzing chunk frames = [3261440,3261952) | 20260505 12:37:11.075 | INFO | MemInfo: { | 20260505 12:37:11.075 | INFO | "gpu" : " 17 GB", | 20260505 12:37:11.075 | INFO | "huge" : "", | 20260505 12:37:11.075 | INFO | "huge_shared" : " 27 GB", | 20260505 12:37:11.075 | INFO | "malloc" : " 14 GB", | 20260505 12:37:11.075 | INFO | "pinned" : " 29 GB" | 20260505 12:37:11.075 | INFO | } | 20260505 12:37:11.075 | INFO | Perf report for the last 5.14739 seconds: | 20260505 12:37:11.075 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:37:11.075 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:11.075 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:11.075 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:11.075 | INFO | ANALYSIS : 63.1%, 3.0, 98.3ms, 0 | 20260505 12:37:11.075 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:11.075 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:11.075 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:37:11.076 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:11.076 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:11.076 | INFO | Uploaded 36GiB to GPU in 2.20584s (16.3203GiB/s) | 20260505 12:37:11.076 | INFO | Downloaded 34GiB to GPU in 2.11514s (16.0746GiB/s) | 20260505 12:37:16.209 | INFO | Analyzing chunk frames = [3261952,3262464) | 20260505 12:37:16.209 | INFO | MemInfo: { | 20260505 12:37:16.209 | INFO | "gpu" : " 17 GB", | 20260505 12:37:16.209 | INFO | "huge" : "", | 20260505 12:37:16.209 | INFO | "huge_shared" : " 27 GB", | 20260505 12:37:16.209 | INFO | "malloc" : " 14 GB", | 20260505 12:37:16.209 | INFO | "pinned" : " 29 GB" | 20260505 12:37:16.209 | INFO | } | 20260505 12:37:16.209 | INFO | Perf report for the last 5.13388 seconds: | 20260505 12:37:16.209 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:37:16.209 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:16.209 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:16.209 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:16.209 | INFO | ANALYSIS : 63.3%, 3.0, 98.2ms, 0 | 20260505 12:37:16.209 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:16.209 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:16.209 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:37:16.209 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:16.209 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:16.209 | INFO | Uploaded 36GiB to GPU in 2.20782s (16.3057GiB/s) | 20260505 12:37:16.210 | INFO | Downloaded 34GiB to GPU in 2.13334s (15.9375GiB/s) | 20260505 12:37:21.351 | INFO | Analyzing chunk frames = [3262464,3262976) | 20260505 12:37:21.351 | INFO | MemInfo: { | 20260505 12:37:21.351 | INFO | "gpu" : " 17 GB", | 20260505 12:37:21.351 | INFO | "huge" : "", | 20260505 12:37:21.351 | INFO | "huge_shared" : " 27 GB", | 20260505 12:37:21.352 | INFO | "malloc" : " 14 GB", | 20260505 12:37:21.352 | INFO | "pinned" : " 29 GB" | 20260505 12:37:21.352 | INFO | } | 20260505 12:37:21.352 | INFO | Perf report for the last 5.14247 seconds: | 20260505 12:37:21.352 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:37:21.352 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:21.352 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:21.352 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:21.352 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 12:37:21.352 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:21.352 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:21.352 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:37:21.352 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:21.352 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:21.352 | INFO | Uploaded 36GiB to GPU in 2.17962s (16.5167GiB/s) | 20260505 12:37:21.352 | INFO | Downloaded 34GiB to GPU in 2.1291s (15.9692GiB/s) | 20260505 12:37:26.490 | INFO | Analyzing chunk frames = [3262976,3263488) | 20260505 12:37:26.490 | INFO | MemInfo: { | 20260505 12:37:26.490 | INFO | "gpu" : " 17 GB", | 20260505 12:37:26.490 | INFO | "huge" : "", | 20260505 12:37:26.490 | INFO | "huge_shared" : " 27 GB", | 20260505 12:37:26.490 | INFO | "malloc" : " 14 GB", | 20260505 12:37:26.490 | INFO | "pinned" : " 29 GB" | 20260505 12:37:26.490 | INFO | } | 20260505 12:37:26.490 | INFO | Perf report for the last 5.13857 seconds: | 20260505 12:37:26.490 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:37:26.490 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:26.490 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:26.490 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:26.490 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 12:37:26.490 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:26.490 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:26.490 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 12:37:26.490 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:26.490 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:26.491 | INFO | Uploaded 36GiB to GPU in 2.15617s (16.6962GiB/s) | 20260505 12:37:26.491 | INFO | Downloaded 34GiB to GPU in 2.10779s (16.1306GiB/s) | 20260505 12:37:31.626 | INFO | Analyzing chunk frames = [3263488,3264000) | 20260505 12:37:31.626 | INFO | MemInfo: { | 20260505 12:37:31.626 | INFO | "gpu" : " 17 GB", | 20260505 12:37:31.626 | INFO | "huge" : "", | 20260505 12:37:31.626 | INFO | "huge_shared" : " 27 GB", | 20260505 12:37:31.627 | INFO | "malloc" : " 14 GB", | 20260505 12:37:31.627 | INFO | "pinned" : " 29 GB" | 20260505 12:37:31.627 | INFO | } | 20260505 12:37:31.627 | INFO | Perf report for the last 5.13652 seconds: | 20260505 12:37:31.627 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:37:31.627 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:31.627 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:31.627 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:31.627 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 12:37:31.627 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:31.627 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:31.627 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 12:37:31.627 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:31.627 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:31.627 | INFO | Uploaded 36GiB to GPU in 2.17146s (16.5787GiB/s) | 20260505 12:37:31.627 | INFO | Downloaded 34GiB to GPU in 2.13823s (15.901GiB/s) | 20260505 12:37:36.771 | INFO | Analyzing chunk frames = [3264000,3264512) | 20260505 12:37:36.771 | INFO | MemInfo: { | 20260505 12:37:36.771 | INFO | "gpu" : " 17 GB", | 20260505 12:37:36.771 | INFO | "huge" : "", | 20260505 12:37:36.771 | INFO | "huge_shared" : " 27 GB", | 20260505 12:37:36.771 | INFO | "malloc" : " 14 GB", | 20260505 12:37:36.771 | INFO | "pinned" : " 29 GB" | 20260505 12:37:36.771 | INFO | } | 20260505 12:37:36.772 | INFO | Perf report for the last 5.14475 seconds: | 20260505 12:37:36.772 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:37:36.772 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:36.772 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:36.772 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:36.772 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 12:37:36.772 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:36.772 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:36.772 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 12:37:36.772 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:36.772 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:36.772 | INFO | Uploaded 36GiB to GPU in 2.14067s (16.8172GiB/s) | 20260505 12:37:36.772 | INFO | Downloaded 34GiB to GPU in 2.10691s (16.1374GiB/s) | 20260505 12:37:36.908 | INFO | Starting a round of Baz IO | 20260505 12:37:40.131 | INFO | Populating RTMetrics report | 20260505 12:37:40.131 | INFO | RTMetrics report populated | 20260505 12:37:40.131 | INFO | Opening tmp RTMetrics json file | 20260505 12:37:40.150 | INFO | Serializing RTMetrics to json | 20260505 12:37:40.155 | INFO | Serialization done, swapping tmp file now... | 20260505 12:37:40.162 | INFO | RTMetrics done | 20260505 12:37:40.237 | INFO | Finished a round of Baz IO: Wrote 2.33 GiB in 3.328 seconds | 20260505 12:37:40.237 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.49% / 0.01% / 76.51% | 20260505 12:37:40.237 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.849732% / 3.97364e-06% / 7.94729e-05% | 20260505 12:37:40.237 | INFO | Baz IO overall: | 20260505 12:37:40.237 | INFO | BytesWritten : 1940873081397 | 20260505 12:37:40.237 | INFO | NumEvents : 923783232599 | 20260505 12:37:41.915 | INFO | Analyzing chunk frames = [3264512,3265024) | 20260505 12:37:41.915 | INFO | MemInfo: { | 20260505 12:37:41.916 | INFO | "gpu" : " 17 GB", | 20260505 12:37:41.916 | INFO | "huge" : "", | 20260505 12:37:41.916 | INFO | "huge_shared" : " 27 GB", | 20260505 12:37:41.916 | INFO | "malloc" : " 14 GB", | 20260505 12:37:41.916 | INFO | "pinned" : " 29 GB" | 20260505 12:37:41.916 | INFO | } | 20260505 12:37:41.916 | INFO | Perf report for the last 5.14419 seconds: | 20260505 12:37:41.916 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:37:41.916 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:41.916 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:41.916 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:41.916 | INFO | ANALYSIS : 64.2%, 2.9, 99.3ms, 0 | 20260505 12:37:41.916 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:41.916 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260505 12:37:41.916 | INFO | PRE_HQ : 61.1%, 1.8, 58.2ms, 0 | 20260505 12:37:41.916 | INFO | BAZWRITER : 64.1%, 4.1, 134.3ms, 0 | 20260505 12:37:41.916 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 12:37:41.916 | INFO | Uploaded 36GiB to GPU in 2.20274s (16.3433GiB/s) | 20260505 12:37:41.916 | INFO | Downloaded 38GiB to GPU in 2.27535s (16.7008GiB/s) | 20260505 12:37:47.047 | INFO | Analyzing chunk frames = [3265024,3265536) | 20260505 12:37:47.047 | INFO | MemInfo: { | 20260505 12:37:47.047 | INFO | "gpu" : " 17 GB", | 20260505 12:37:47.047 | INFO | "huge" : "", | 20260505 12:37:47.047 | INFO | "huge_shared" : " 27 GB", | 20260505 12:37:47.047 | INFO | "malloc" : " 14 GB", | 20260505 12:37:47.047 | INFO | "pinned" : " 29 GB" | 20260505 12:37:47.047 | INFO | } | 20260505 12:37:47.047 | INFO | Perf report for the last 5.13121 seconds: | 20260505 12:37:47.047 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:37:47.047 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:47.047 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:47.047 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:47.047 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 12:37:47.047 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:47.047 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:47.047 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:37:47.047 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:47.047 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:47.047 | INFO | Uploaded 36GiB to GPU in 2.19783s (16.3798GiB/s) | 20260505 12:37:47.047 | INFO | Downloaded 34GiB to GPU in 2.0398s (16.6683GiB/s) | 20260505 12:37:52.176 | INFO | Analyzing chunk frames = [3265536,3266048) | 20260505 12:37:52.176 | INFO | MemInfo: { | 20260505 12:37:52.176 | INFO | "gpu" : " 17 GB", | 20260505 12:37:52.176 | INFO | "huge" : "", | 20260505 12:37:52.176 | INFO | "huge_shared" : " 27 GB", | 20260505 12:37:52.176 | INFO | "malloc" : " 14 GB", | 20260505 12:37:52.176 | INFO | "pinned" : " 29 GB" | 20260505 12:37:52.176 | INFO | } | 20260505 12:37:52.176 | INFO | Perf report for the last 5.12926 seconds: | 20260505 12:37:52.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:37:52.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:52.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:52.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:52.176 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 12:37:52.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:52.176 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:52.177 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 12:37:52.177 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:52.177 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:52.177 | INFO | Uploaded 36GiB to GPU in 2.21475s (16.2547GiB/s) | 20260505 12:37:52.177 | INFO | Downloaded 34GiB to GPU in 2.12423s (16.0058GiB/s) | 20260505 12:37:57.328 | INFO | Analyzing chunk frames = [3266048,3266560) | 20260505 12:37:57.328 | INFO | MemInfo: { | 20260505 12:37:57.328 | INFO | "gpu" : " 17 GB", | 20260505 12:37:57.328 | INFO | "huge" : "", | 20260505 12:37:57.328 | INFO | "huge_shared" : " 27 GB", | 20260505 12:37:57.328 | INFO | "malloc" : " 14 GB", | 20260505 12:37:57.329 | INFO | "pinned" : " 29 GB" | 20260505 12:37:57.329 | INFO | } | 20260505 12:37:57.329 | INFO | Perf report for the last 5.15236 seconds: | 20260505 12:37:57.329 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:37:57.329 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:57.329 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:57.329 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:57.329 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 12:37:57.329 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:57.329 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:57.329 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 12:37:57.329 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:57.329 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:37:57.329 | INFO | Uploaded 36GiB to GPU in 2.22514s (16.1788GiB/s) | 20260505 12:37:57.329 | INFO | Downloaded 34GiB to GPU in 2.12469s (16.0023GiB/s) | 20260505 12:38:02.464 | INFO | Analyzing chunk frames = [3266560,3267072) | 20260505 12:38:02.464 | INFO | MemInfo: { | 20260505 12:38:02.464 | INFO | "gpu" : " 17 GB", | 20260505 12:38:02.464 | INFO | "huge" : "", | 20260505 12:38:02.464 | INFO | "huge_shared" : " 27 GB", | 20260505 12:38:02.464 | INFO | "malloc" : " 14 GB", | 20260505 12:38:02.464 | INFO | "pinned" : " 29 GB" | 20260505 12:38:02.464 | INFO | } | 20260505 12:38:02.464 | INFO | Perf report for the last 5.13562 seconds: | 20260505 12:38:02.464 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:38:02.464 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:02.464 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:02.464 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:02.464 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 12:38:02.464 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:02.464 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:02.464 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 12:38:02.464 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:02.464 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:02.464 | INFO | Uploaded 36GiB to GPU in 2.16871s (16.5997GiB/s) | 20260505 12:38:02.464 | INFO | Downloaded 34GiB to GPU in 2.12806s (15.977GiB/s) | 20260505 12:38:07.607 | INFO | Analyzing chunk frames = [3267072,3267584) | 20260505 12:38:07.607 | INFO | MemInfo: { | 20260505 12:38:07.607 | INFO | "gpu" : " 17 GB", | 20260505 12:38:07.607 | INFO | "huge" : "", | 20260505 12:38:07.607 | INFO | "huge_shared" : " 27 GB", | 20260505 12:38:07.607 | INFO | "malloc" : " 14 GB", | 20260505 12:38:07.608 | INFO | "pinned" : " 29 GB" | 20260505 12:38:07.608 | INFO | } | 20260505 12:38:07.608 | INFO | Perf report for the last 5.14332 seconds: | 20260505 12:38:07.608 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:38:07.608 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:07.608 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:07.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:07.608 | INFO | ANALYSIS : 63.2%, 3.0, 98.8ms, 0 | 20260505 12:38:07.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:07.608 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:07.608 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:38:07.608 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:07.608 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:07.608 | INFO | Uploaded 36GiB to GPU in 2.16422s (16.6342GiB/s) | 20260505 12:38:07.608 | INFO | Downloaded 34GiB to GPU in 2.10969s (16.1161GiB/s) | 20260505 12:38:12.743 | INFO | Analyzing chunk frames = [3267584,3268096) | 20260505 12:38:12.743 | INFO | MemInfo: { | 20260505 12:38:12.743 | INFO | "gpu" : " 17 GB", | 20260505 12:38:12.743 | INFO | "huge" : "", | 20260505 12:38:12.743 | INFO | "huge_shared" : " 27 GB", | 20260505 12:38:12.743 | INFO | "malloc" : " 14 GB", | 20260505 12:38:12.743 | INFO | "pinned" : " 29 GB" | 20260505 12:38:12.743 | INFO | } | 20260505 12:38:12.743 | INFO | Perf report for the last 5.13571 seconds: | 20260505 12:38:12.743 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:38:12.743 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:12.743 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:12.743 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:12.743 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 12:38:12.743 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:12.743 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:12.743 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 12:38:12.743 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:12.743 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:12.744 | INFO | Uploaded 36GiB to GPU in 2.12982s (16.9028GiB/s) | 20260505 12:38:12.744 | INFO | Downloaded 34GiB to GPU in 2.12508s (15.9994GiB/s) | 20260505 12:38:17.883 | INFO | Analyzing chunk frames = [3268096,3268608) | 20260505 12:38:17.883 | INFO | MemInfo: { | 20260505 12:38:17.883 | INFO | "gpu" : " 17 GB", | 20260505 12:38:17.883 | INFO | "huge" : "", | 20260505 12:38:17.883 | INFO | "huge_shared" : " 27 GB", | 20260505 12:38:17.883 | INFO | "malloc" : " 14 GB", | 20260505 12:38:17.883 | INFO | "pinned" : " 29 GB" | 20260505 12:38:17.883 | INFO | } | 20260505 12:38:17.883 | INFO | Perf report for the last 5.13954 seconds: | 20260505 12:38:17.883 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:38:17.883 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:17.883 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:17.883 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:17.883 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 12:38:17.883 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:17.883 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:17.883 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 12:38:17.883 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:17.883 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:17.883 | INFO | Uploaded 36GiB to GPU in 2.21391s (16.2608GiB/s) | 20260505 12:38:17.883 | INFO | Downloaded 34GiB to GPU in 2.12062s (16.0331GiB/s) | 20260505 12:38:17.883 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 12:38:17.883 | INFO | Total mem usage: 15.3 GiB | 20260505 12:38:17.883 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 12:38:17.883 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -6.93%) | 20260505 12:38:17.883 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 12:38:17.883 | INFO | High Water Breakdown: | 20260505 12:38:17.883 | INFO | 61.1% : 8.76 GiB ( 92% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 12:38:17.883 | INFO | 38.8% : 5.56 GiB ( 83.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 12:38:17.883 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 12:38:17.883 | INFO | ------------------------------------------ | 20260505 12:38:17.883 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:38:17.884 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 12:38:17.884 | INFO | Total mem usage: 26.5 GiB | 20260505 12:38:17.884 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 12:38:17.884 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 12:38:17.884 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 12:38:17.884 | INFO | High Water Breakdown: | 20260505 12:38:17.884 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:38:17.884 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:38:17.884 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:38:17.884 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:38:17.884 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:38:17.884 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:38:17.884 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 12:38:17.884 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 12:38:17.884 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:38:17.884 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 12:38:17.884 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 12:38:17.884 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 12:38:17.884 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 12:38:17.884 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 12:38:17.884 | INFO | ------------------------------------------ | 20260505 12:38:17.884 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:38:17.884 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 12:38:17.884 | INFO | Total mem usage: 25.6 GiB | 20260505 12:38:17.884 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 12:38:17.884 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 12:38:17.884 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 12:38:17.884 | INFO | High Water Breakdown: | 20260505 12:38:17.884 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 12:38:17.884 | INFO | ------------------------------------------ | 20260505 12:38:17.884 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:38:17.884 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 12:38:17.884 | INFO | Total mem usage: 15.7 GiB | 20260505 12:38:17.884 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 12:38:17.884 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 12:38:17.884 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 12:38:17.884 | INFO | High Water Breakdown: | 20260505 12:38:17.884 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 12:38:17.884 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 12:38:17.884 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 12:38:17.884 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 12:38:17.884 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 12:38:17.884 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 12:38:17.884 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 12:38:17.884 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 12:38:17.884 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 12:38:17.884 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:38:17.884 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 12:38:17.884 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:38:17.884 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 12:38:17.884 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 12:38:17.884 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 12:38:17.884 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 12:38:17.884 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:38:17.884 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 12:38:17.884 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:38:17.884 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:38:17.884 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:38:17.884 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 12:38:17.884 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 12:38:17.884 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:38:17.884 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 12:38:17.884 | INFO | ------------------------------------------ | 20260505 12:38:17.884 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:38:17.884 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 12:38:17.884 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:38:17.884 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:38:17.884 | INFO | PerfReport: Full Execution -- 3.20e+01 / 7.69e+01, 1.36e+01 / 1.65e+00 | 20260505 12:38:17.884 | INFO | PerfReport: 0.29%, AggregateStats -- 9.31e-02 / 9.90e-02, 1.37e-02 / 4.34e-03 | 20260505 12:38:17.884 | INFO | PerfReport: 11.76%, Baseline -- 3.76e+00 / 4.46e+00, 6.00e-01 / 2.34e-01 | 20260505 12:38:17.884 | INFO | PerfReport: 3.31%, Histogram -- 1.06e+00 / 1.49e+00, 2.04e-01 / 9.96e-02 | 20260505 12:38:17.884 | INFO | PerfReport: 15.90%, DME -- 4.07e+01 / 4.55e+01, 4.61e+00 / 2.25e+00 | 20260505 12:38:17.884 | INFO | PerfReport: 56.46%, FrameLabeling -- 1.81e+01 / 2.05e+01, 1.54e+00 / 2.15e+00 | 20260505 12:38:17.884 | INFO | PerfReport: 7.40%, PulseAccumulating -- 2.37e+00 / 2.91e+00, 4.43e-01 / 2.10e-01 | 20260505 12:38:17.884 | INFO | PerfReport: 3.60%, Metrics -- 1.15e+00 / 1.48e+00, 3.22e-01 / 1.63e-01 | 20260505 12:38:17.884 | INFO | PerfReport: 1.27%, PulseSerializing -- 4.06e-01 / 4.52e-01, 7.33e-02 / 3.08e-02 | 20260505 12:38:17.885 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 12:38:17.885 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:38:17.885 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:38:17.885 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260505 12:38:17.885 | INFO | PerfReport: 22.54%, Upload -- 2.27e+01 / 0.00e+00, 8.05e+00 / 0.00e+00 | 20260505 12:38:17.885 | INFO | PerfReport: 22.00%, Download -- 2.22e+01 / 0.00e+00, 8.12e+00 / 0.00e+00 | 20260505 12:38:17.885 | INFO | PerfReport: 31.71%, Compute -- 3.20e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 12:38:17.885 | INFO | PerfReport: 2.12%, LockUpload -- 2.14e+00 / 0.00e+00, 7.66e+00 / 0.00e+00 | 20260505 12:38:17.885 | INFO | PerfReport: 1.36%, LockDownload -- 1.37e+00 / 0.00e+00, 2.97e+00 / 0.00e+00 | 20260505 12:38:17.885 | INFO | PerfReport: 20.26%, LockCompute -- 2.04e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260505 12:38:17.885 | INFO | PerfReport: 0.01%, Misc -- 1.25e-02 / 0.00e+00, 1.28e-02 / 0.00e+00 | 20260505 12:38:18.016 | INFO | Starting a round of Baz IO | 20260505 12:38:21.260 | INFO | Populating RTMetrics report | 20260505 12:38:21.260 | INFO | RTMetrics report populated | 20260505 12:38:21.260 | INFO | Opening tmp RTMetrics json file | 20260505 12:38:21.261 | INFO | Serializing RTMetrics to json | 20260505 12:38:21.262 | INFO | Serialization done, swapping tmp file now... | 20260505 12:38:21.262 | INFO | RTMetrics done | 20260505 12:38:21.378 | INFO | Finished a round of Baz IO: Wrote 2.331 GiB in 3.361 seconds | 20260505 12:38:21.378 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.48% / 0.01% / 76.51% | 20260505 12:38:21.378 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.849605% / 3.97364e-06% / 9.13938e-05% | 20260505 12:38:21.378 | INFO | Baz IO overall: | 20260505 12:38:21.378 | INFO | BytesWritten : 1943376101129 | 20260505 12:38:21.378 | INFO | NumEvents : 924969928117 | 20260505 12:38:23.027 | INFO | Analyzing chunk frames = [3268608,3269120) | 20260505 12:38:23.027 | INFO | MemInfo: { | 20260505 12:38:23.027 | INFO | "gpu" : " 17 GB", | 20260505 12:38:23.027 | INFO | "huge" : "", | 20260505 12:38:23.027 | INFO | "huge_shared" : " 27 GB", | 20260505 12:38:23.027 | INFO | "malloc" : " 14 GB", | 20260505 12:38:23.027 | INFO | "pinned" : " 29 GB" | 20260505 12:38:23.027 | INFO | } | 20260505 12:38:23.027 | INFO | Perf report for the last 5.14424 seconds: | 20260505 12:38:23.027 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:38:23.027 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:23.027 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:23.027 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:23.027 | INFO | ANALYSIS : 64.5%, 2.9, 99.9ms, 0 | 20260505 12:38:23.027 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:23.027 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260505 12:38:23.027 | INFO | PRE_HQ : 61.1%, 1.9, 59.2ms, 0 | 20260505 12:38:23.027 | INFO | BAZWRITER : 64.7%, 4.1, 136.9ms, 0 | 20260505 12:38:23.027 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 12:38:23.027 | INFO | Uploaded 36GiB to GPU in 2.18545s (16.4726GiB/s) | 20260505 12:38:23.027 | INFO | Downloaded 38GiB to GPU in 2.28069s (16.6616GiB/s) | 20260505 12:38:28.164 | INFO | Analyzing chunk frames = [3269120,3269632) | 20260505 12:38:28.164 | INFO | MemInfo: { | 20260505 12:38:28.164 | INFO | "gpu" : " 17 GB", | 20260505 12:38:28.164 | INFO | "huge" : "", | 20260505 12:38:28.164 | INFO | "huge_shared" : " 27 GB", | 20260505 12:38:28.164 | INFO | "malloc" : " 14 GB", | 20260505 12:38:28.164 | INFO | "pinned" : " 29 GB" | 20260505 12:38:28.164 | INFO | } | 20260505 12:38:28.164 | INFO | Perf report for the last 5.13687 seconds: | 20260505 12:38:28.164 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:38:28.164 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:28.164 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:28.164 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:28.164 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 12:38:28.164 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:28.164 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:28.164 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:38:28.164 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:28.164 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:28.164 | INFO | Uploaded 36GiB to GPU in 2.19426s (16.4064GiB/s) | 20260505 12:38:28.165 | INFO | Downloaded 34GiB to GPU in 2.03058s (16.744GiB/s) | 20260505 12:38:33.307 | INFO | Analyzing chunk frames = [3269632,3270144) | 20260505 12:38:33.307 | INFO | MemInfo: { | 20260505 12:38:33.307 | INFO | "gpu" : " 17 GB", | 20260505 12:38:33.307 | INFO | "huge" : "", | 20260505 12:38:33.307 | INFO | "huge_shared" : " 27 GB", | 20260505 12:38:33.307 | INFO | "malloc" : " 14 GB", | 20260505 12:38:33.307 | INFO | "pinned" : " 29 GB" | 20260505 12:38:33.307 | INFO | } | 20260505 12:38:33.307 | INFO | Perf report for the last 5.14286 seconds: | 20260505 12:38:33.307 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:38:33.307 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:33.307 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:33.307 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:33.307 | INFO | ANALYSIS : 63.1%, 3.0, 98.3ms, 0 | 20260505 12:38:33.307 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:33.307 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:33.307 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 12:38:33.307 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:33.307 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:33.307 | INFO | Uploaded 36GiB to GPU in 2.16173s (16.6533GiB/s) | 20260505 12:38:33.307 | INFO | Downloaded 34GiB to GPU in 2.12856s (15.9733GiB/s) | 20260505 12:38:38.440 | INFO | Analyzing chunk frames = [3270144,3270656) | 20260505 12:38:38.440 | INFO | MemInfo: { | 20260505 12:38:38.440 | INFO | "gpu" : " 17 GB", | 20260505 12:38:38.440 | INFO | "huge" : "", | 20260505 12:38:38.440 | INFO | "huge_shared" : " 27 GB", | 20260505 12:38:38.440 | INFO | "malloc" : " 14 GB", | 20260505 12:38:38.440 | INFO | "pinned" : " 29 GB" | 20260505 12:38:38.440 | INFO | } | 20260505 12:38:38.441 | INFO | Perf report for the last 5.13388 seconds: | 20260505 12:38:38.441 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:38:38.441 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:38.441 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:38.441 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:38.441 | INFO | ANALYSIS : 63.3%, 3.0, 98.2ms, 0 | 20260505 12:38:38.441 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:38.441 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:38.441 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 12:38:38.441 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:38.441 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:38.441 | INFO | Uploaded 36GiB to GPU in 2.2107s (16.2845GiB/s) | 20260505 12:38:38.441 | INFO | Downloaded 34GiB to GPU in 2.13384s (15.9337GiB/s) | 20260505 12:38:43.578 | INFO | Analyzing chunk frames = [3270656,3271168) | 20260505 12:38:43.578 | INFO | MemInfo: { | 20260505 12:38:43.578 | INFO | "gpu" : " 17 GB", | 20260505 12:38:43.578 | INFO | "huge" : "", | 20260505 12:38:43.578 | INFO | "huge_shared" : " 27 GB", | 20260505 12:38:43.578 | INFO | "malloc" : " 14 GB", | 20260505 12:38:43.578 | INFO | "pinned" : " 29 GB" | 20260505 12:38:43.578 | INFO | } | 20260505 12:38:43.578 | INFO | Perf report for the last 5.13729 seconds: | 20260505 12:38:43.578 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:38:43.578 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:43.578 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:43.578 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:43.578 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 12:38:43.578 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:43.578 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:43.578 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 12:38:43.578 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:43.578 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:43.578 | INFO | Uploaded 36GiB to GPU in 2.1772s (16.535GiB/s) | 20260505 12:38:43.578 | INFO | Downloaded 34GiB to GPU in 2.1283s (15.9752GiB/s) | 20260505 12:38:48.722 | INFO | Analyzing chunk frames = [3271168,3271680) | 20260505 12:38:48.722 | INFO | MemInfo: { | 20260505 12:38:48.722 | INFO | "gpu" : " 17 GB", | 20260505 12:38:48.722 | INFO | "huge" : "", | 20260505 12:38:48.722 | INFO | "huge_shared" : " 27 GB", | 20260505 12:38:48.722 | INFO | "malloc" : " 14 GB", | 20260505 12:38:48.722 | INFO | "pinned" : " 29 GB" | 20260505 12:38:48.722 | INFO | } | 20260505 12:38:48.722 | INFO | Perf report for the last 5.14364 seconds: | 20260505 12:38:48.722 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:38:48.722 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:48.722 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:48.722 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:48.722 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 12:38:48.722 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:48.722 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:48.722 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 12:38:48.722 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:48.722 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:48.722 | INFO | Uploaded 36GiB to GPU in 2.15238s (16.7257GiB/s) | 20260505 12:38:48.722 | INFO | Downloaded 34GiB to GPU in 2.11106s (16.1056GiB/s) | 20260505 12:38:53.861 | INFO | Analyzing chunk frames = [3271680,3272192) | 20260505 12:38:53.861 | INFO | MemInfo: { | 20260505 12:38:53.861 | INFO | "gpu" : " 17 GB", | 20260505 12:38:53.861 | INFO | "huge" : "", | 20260505 12:38:53.861 | INFO | "huge_shared" : " 27 GB", | 20260505 12:38:53.861 | INFO | "malloc" : " 14 GB", | 20260505 12:38:53.861 | INFO | "pinned" : " 29 GB" | 20260505 12:38:53.861 | INFO | } | 20260505 12:38:53.861 | INFO | Perf report for the last 5.13968 seconds: | 20260505 12:38:53.861 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:38:53.861 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:53.861 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:53.861 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:53.861 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 12:38:53.861 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:53.861 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:53.862 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 12:38:53.862 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:53.862 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:53.862 | INFO | Uploaded 36GiB to GPU in 2.18484s (16.4772GiB/s) | 20260505 12:38:53.862 | INFO | Downloaded 34GiB to GPU in 2.15991s (15.7414GiB/s) | 20260505 12:38:58.995 | INFO | Analyzing chunk frames = [3272192,3272704) | 20260505 12:38:58.995 | INFO | MemInfo: { | 20260505 12:38:58.995 | INFO | "gpu" : " 17 GB", | 20260505 12:38:58.995 | INFO | "huge" : "", | 20260505 12:38:58.995 | INFO | "huge_shared" : " 27 GB", | 20260505 12:38:58.995 | INFO | "malloc" : " 14 GB", | 20260505 12:38:58.995 | INFO | "pinned" : " 29 GB" | 20260505 12:38:58.995 | INFO | } | 20260505 12:38:58.995 | INFO | Perf report for the last 5.13335 seconds: | 20260505 12:38:58.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:38:58.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:58.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:58.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:58.995 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 12:38:58.995 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:58.995 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:58.995 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 12:38:58.995 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:58.995 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:38:58.995 | INFO | Uploaded 36GiB to GPU in 2.1643s (16.6336GiB/s) | 20260505 12:38:58.995 | INFO | Downloaded 34GiB to GPU in 2.11109s (16.1054GiB/s) | 20260505 12:38:59.132 | INFO | Starting a round of Baz IO | 20260505 12:39:02.381 | INFO | Populating RTMetrics report | 20260505 12:39:02.381 | INFO | RTMetrics report populated | 20260505 12:39:02.381 | INFO | Opening tmp RTMetrics json file | 20260505 12:39:02.422 | INFO | Serializing RTMetrics to json | 20260505 12:39:02.456 | INFO | Serialization done, swapping tmp file now... | 20260505 12:39:02.475 | INFO | RTMetrics done | 20260505 12:39:02.493 | INFO | Finished a round of Baz IO: Wrote 2.332 GiB in 3.36 seconds | 20260505 12:39:02.493 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.48% / 0.01% / 76.52% | 20260505 12:39:02.493 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.846684% / 1.58946e-05% / 7.54992e-05% | 20260505 12:39:02.493 | INFO | Baz IO overall: | 20260505 12:39:02.493 | INFO | BytesWritten : 1945879665587 | 20260505 12:39:02.493 | INFO | NumEvents : 926156942640 | 20260505 12:39:04.130 | INFO | Analyzing chunk frames = [3272704,3273216) | 20260505 12:39:04.130 | INFO | MemInfo: { | 20260505 12:39:04.130 | INFO | "gpu" : " 17 GB", | 20260505 12:39:04.130 | INFO | "huge" : "", | 20260505 12:39:04.130 | INFO | "huge_shared" : " 27 GB", | 20260505 12:39:04.130 | INFO | "malloc" : " 14 GB", | 20260505 12:39:04.130 | INFO | "pinned" : " 29 GB" | 20260505 12:39:04.130 | INFO | } | 20260505 12:39:04.130 | INFO | Perf report for the last 5.13577 seconds: | 20260505 12:39:04.130 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:39:04.131 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:04.131 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:04.131 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:04.131 | INFO | ANALYSIS : 64.8%, 3.0, 100.4ms, 0 | 20260505 12:39:04.131 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:04.131 | INFO | RT_METRICS : 47.2%, 1.0, 24.8ms, 0 | 20260505 12:39:04.131 | INFO | PRE_HQ : 61.1%, 1.8, 58.2ms, 0 | 20260505 12:39:04.131 | INFO | BAZWRITER : 64.8%, 4.0, 134.0ms, 0 | 20260505 12:39:04.131 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 12:39:04.131 | INFO | Uploaded 36GiB to GPU in 2.19876s (16.3729GiB/s) | 20260505 12:39:04.131 | INFO | Downloaded 38GiB to GPU in 2.30344s (16.497GiB/s) | 20260505 12:39:09.281 | INFO | Analyzing chunk frames = [3273216,3273728) | 20260505 12:39:09.281 | INFO | MemInfo: { | 20260505 12:39:09.281 | INFO | "gpu" : " 17 GB", | 20260505 12:39:09.281 | INFO | "huge" : "", | 20260505 12:39:09.281 | INFO | "huge_shared" : " 27 GB", | 20260505 12:39:09.281 | INFO | "malloc" : " 14 GB", | 20260505 12:39:09.281 | INFO | "pinned" : " 29 GB" | 20260505 12:39:09.281 | INFO | } | 20260505 12:39:09.281 | INFO | Perf report for the last 5.15024 seconds: | 20260505 12:39:09.281 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:39:09.281 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:09.281 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:09.281 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:09.281 | INFO | ANALYSIS : 64.0%, 3.0, 100.2ms, 0 | 20260505 12:39:09.281 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:09.281 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:09.281 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:39:09.281 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:09.281 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:09.281 | INFO | Uploaded 36GiB to GPU in 2.23545s (16.1041GiB/s) | 20260505 12:39:09.281 | INFO | Downloaded 34GiB to GPU in 2.10074s (16.1848GiB/s) | 20260505 12:39:14.423 | INFO | Analyzing chunk frames = [3273728,3274240) | 20260505 12:39:14.423 | INFO | MemInfo: { | 20260505 12:39:14.423 | INFO | "gpu" : " 17 GB", | 20260505 12:39:14.423 | INFO | "huge" : "", | 20260505 12:39:14.423 | INFO | "huge_shared" : " 27 GB", | 20260505 12:39:14.424 | INFO | "malloc" : " 14 GB", | 20260505 12:39:14.424 | INFO | "pinned" : " 29 GB" | 20260505 12:39:14.424 | INFO | } | 20260505 12:39:14.424 | INFO | Perf report for the last 5.14281 seconds: | 20260505 12:39:14.424 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:39:14.424 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:14.424 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:14.424 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:14.424 | INFO | ANALYSIS : 63.2%, 3.0, 98.4ms, 0 | 20260505 12:39:14.424 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:14.424 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:14.424 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 12:39:14.424 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:14.424 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:14.424 | INFO | Uploaded 36GiB to GPU in 2.18313s (16.4901GiB/s) | 20260505 12:39:14.424 | INFO | Downloaded 34GiB to GPU in 2.13677s (15.9119GiB/s) | 20260505 12:39:19.556 | INFO | Analyzing chunk frames = [3274240,3274752) | 20260505 12:39:19.556 | INFO | MemInfo: { | 20260505 12:39:19.556 | INFO | "gpu" : " 17 GB", | 20260505 12:39:19.556 | INFO | "huge" : "", | 20260505 12:39:19.556 | INFO | "huge_shared" : " 27 GB", | 20260505 12:39:19.556 | INFO | "malloc" : " 14 GB", | 20260505 12:39:19.556 | INFO | "pinned" : " 29 GB" | 20260505 12:39:19.556 | INFO | } | 20260505 12:39:19.556 | INFO | Perf report for the last 5.13221 seconds: | 20260505 12:39:19.556 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:39:19.556 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:19.556 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:19.556 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:19.556 | INFO | ANALYSIS : 63.1%, 3.0, 97.9ms, 0 | 20260505 12:39:19.556 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:19.556 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:19.556 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 12:39:19.556 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:19.556 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:19.556 | INFO | Uploaded 36GiB to GPU in 2.19803s (16.3783GiB/s) | 20260505 12:39:19.556 | INFO | Downloaded 34GiB to GPU in 2.12757s (15.9807GiB/s) | 20260505 12:39:24.690 | INFO | Analyzing chunk frames = [3274752,3275264) | 20260505 12:39:24.690 | INFO | MemInfo: { | 20260505 12:39:24.690 | INFO | "gpu" : " 17 GB", | 20260505 12:39:24.690 | INFO | "huge" : "", | 20260505 12:39:24.690 | INFO | "huge_shared" : " 27 GB", | 20260505 12:39:24.690 | INFO | "malloc" : " 14 GB", | 20260505 12:39:24.690 | INFO | "pinned" : " 29 GB" | 20260505 12:39:24.690 | INFO | } | 20260505 12:39:24.691 | INFO | Perf report for the last 5.13472 seconds: | 20260505 12:39:24.691 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:39:24.691 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:24.691 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:24.691 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:24.691 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 12:39:24.691 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:24.691 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:24.691 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 12:39:24.691 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:24.691 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:24.691 | INFO | Uploaded 36GiB to GPU in 2.17983s (16.515GiB/s) | 20260505 12:39:24.691 | INFO | Downloaded 34GiB to GPU in 2.13049s (15.9588GiB/s) | 20260505 12:39:29.835 | INFO | Analyzing chunk frames = [3275264,3275776) | 20260505 12:39:29.835 | INFO | MemInfo: { | 20260505 12:39:29.835 | INFO | "gpu" : " 17 GB", | 20260505 12:39:29.835 | INFO | "huge" : "", | 20260505 12:39:29.835 | INFO | "huge_shared" : " 27 GB", | 20260505 12:39:29.835 | INFO | "malloc" : " 14 GB", | 20260505 12:39:29.835 | INFO | "pinned" : " 29 GB" | 20260505 12:39:29.835 | INFO | } | 20260505 12:39:29.835 | INFO | Perf report for the last 5.14462 seconds: | 20260505 12:39:29.835 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:39:29.835 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:29.835 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:29.835 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:29.835 | INFO | ANALYSIS : 63.2%, 3.0, 98.8ms, 0 | 20260505 12:39:29.835 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:29.835 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:29.835 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:39:29.835 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:29.835 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:29.836 | INFO | Uploaded 36GiB to GPU in 2.17775s (16.5308GiB/s) | 20260505 12:39:29.836 | INFO | Downloaded 34GiB to GPU in 2.11314s (16.0898GiB/s) | 20260505 12:39:34.973 | INFO | Analyzing chunk frames = [3275776,3276288) | 20260505 12:39:34.973 | INFO | MemInfo: { | 20260505 12:39:34.973 | INFO | "gpu" : " 17 GB", | 20260505 12:39:34.973 | INFO | "huge" : "", | 20260505 12:39:34.973 | INFO | "huge_shared" : " 27 GB", | 20260505 12:39:34.973 | INFO | "malloc" : " 14 GB", | 20260505 12:39:34.973 | INFO | "pinned" : " 29 GB" | 20260505 12:39:34.973 | INFO | } | 20260505 12:39:34.973 | INFO | Perf report for the last 5.13793 seconds: | 20260505 12:39:34.973 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:39:34.973 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:34.973 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:34.973 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:34.973 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 12:39:34.973 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:34.973 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:34.973 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 12:39:34.973 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:34.973 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:34.973 | INFO | Uploaded 36GiB to GPU in 2.16915s (16.5964GiB/s) | 20260505 12:39:34.973 | INFO | Downloaded 34GiB to GPU in 2.12365s (16.0101GiB/s) | 20260505 12:39:40.113 | INFO | Analyzing chunk frames = [3276288,3276800) | 20260505 12:39:40.113 | INFO | MemInfo: { | 20260505 12:39:40.113 | INFO | "gpu" : " 17 GB", | 20260505 12:39:40.113 | INFO | "huge" : "", | 20260505 12:39:40.113 | INFO | "huge_shared" : " 27 GB", | 20260505 12:39:40.113 | INFO | "malloc" : " 14 GB", | 20260505 12:39:40.113 | INFO | "pinned" : " 29 GB" | 20260505 12:39:40.113 | INFO | } | 20260505 12:39:40.113 | INFO | Perf report for the last 5.13972 seconds: | 20260505 12:39:40.113 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:39:40.113 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:40.113 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:40.113 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:40.113 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 12:39:40.113 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:40.113 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:40.113 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 12:39:40.113 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:40.113 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:40.113 | INFO | Uploaded 36GiB to GPU in 2.16051s (16.6627GiB/s) | 20260505 12:39:40.113 | INFO | Downloaded 34GiB to GPU in 2.11739s (16.0575GiB/s) | 20260505 12:39:40.113 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 12:39:40.113 | INFO | Total mem usage: 15.3 GiB | 20260505 12:39:40.113 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 12:39:40.113 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -7.49%) | 20260505 12:39:40.113 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 12:39:40.113 | INFO | High Water Breakdown: | 20260505 12:39:40.113 | INFO | 60.9% : 8.69 GiB ( 91.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 12:39:40.113 | INFO | 38.9% : 5.55 GiB ( 83.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 12:39:40.114 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 12:39:40.114 | INFO | ------------------------------------------ | 20260505 12:39:40.114 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:39:40.114 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 12:39:40.114 | INFO | Total mem usage: 26.5 GiB | 20260505 12:39:40.114 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 12:39:40.114 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 12:39:40.114 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 12:39:40.114 | INFO | High Water Breakdown: | 20260505 12:39:40.114 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:39:40.114 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:39:40.114 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:39:40.114 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:39:40.114 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:39:40.114 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:39:40.114 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 12:39:40.114 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 12:39:40.114 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:39:40.114 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 12:39:40.114 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 12:39:40.114 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 12:39:40.114 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 12:39:40.114 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 12:39:40.114 | INFO | ------------------------------------------ | 20260505 12:39:40.114 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:39:40.114 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 12:39:40.114 | INFO | Total mem usage: 25.6 GiB | 20260505 12:39:40.114 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 12:39:40.114 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 12:39:40.114 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 12:39:40.114 | INFO | High Water Breakdown: | 20260505 12:39:40.114 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 12:39:40.114 | INFO | ------------------------------------------ | 20260505 12:39:40.114 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:39:40.114 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 12:39:40.114 | INFO | Total mem usage: 15.7 GiB | 20260505 12:39:40.114 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 12:39:40.114 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 12:39:40.114 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 12:39:40.114 | INFO | High Water Breakdown: | 20260505 12:39:40.114 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 12:39:40.114 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 12:39:40.114 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 12:39:40.114 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 12:39:40.114 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 12:39:40.114 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 12:39:40.114 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 12:39:40.114 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 12:39:40.114 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 12:39:40.114 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:39:40.114 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 12:39:40.114 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:39:40.114 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 12:39:40.114 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 12:39:40.114 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 12:39:40.114 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 12:39:40.114 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:39:40.114 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 12:39:40.114 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:39:40.114 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:39:40.114 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:39:40.114 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 12:39:40.114 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 12:39:40.114 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:39:40.114 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 12:39:40.114 | INFO | ------------------------------------------ | 20260505 12:39:40.114 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:39:40.114 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 12:39:40.114 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:39:40.114 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:39:40.114 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.72e+01, 1.38e+01 / 1.96e+00 | 20260505 12:39:40.115 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 9.89e-02, 1.02e-02 / 4.19e-03 | 20260505 12:39:40.115 | INFO | PerfReport: 11.74%, Baseline -- 3.75e+00 / 4.50e+00, 5.88e-01 / 1.36e-01 | 20260505 12:39:40.115 | INFO | PerfReport: 3.30%, Histogram -- 1.05e+00 / 1.50e+00, 2.01e-01 / 9.62e-02 | 20260505 12:39:40.115 | INFO | PerfReport: 16.07%, DME -- 4.11e+01 / 4.64e+01, 4.88e+00 / 2.05e+00 | 20260505 12:39:40.115 | INFO | PerfReport: 56.20%, FrameLabeling -- 1.80e+01 / 1.98e+01, 1.37e+00 / 1.85e+00 | 20260505 12:39:40.115 | INFO | PerfReport: 7.49%, PulseAccumulating -- 2.39e+00 / 2.94e+00, 4.55e-01 / 1.71e-01 | 20260505 12:39:40.115 | INFO | PerfReport: 3.63%, Metrics -- 1.16e+00 / 1.48e+00, 3.21e-01 / 1.62e-01 | 20260505 12:39:40.115 | INFO | PerfReport: 1.28%, PulseSerializing -- 4.08e-01 / 4.58e-01, 7.39e-02 / 3.16e-02 | 20260505 12:39:40.115 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 12:39:40.115 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:39:40.115 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:39:40.115 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260505 12:39:40.115 | INFO | PerfReport: 22.51%, Upload -- 2.27e+01 / 0.00e+00, 8.01e+00 / 0.00e+00 | 20260505 12:39:40.115 | INFO | PerfReport: 22.06%, Download -- 2.23e+01 / 0.00e+00, 8.15e+00 / 0.00e+00 | 20260505 12:39:40.115 | INFO | PerfReport: 31.63%, Compute -- 3.20e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 12:39:40.115 | INFO | PerfReport: 2.17%, LockUpload -- 2.19e+00 / 0.00e+00, 7.81e+00 / 0.00e+00 | 20260505 12:39:40.115 | INFO | PerfReport: 1.44%, LockDownload -- 1.46e+00 / 0.00e+00, 3.04e+00 / 0.00e+00 | 20260505 12:39:40.115 | INFO | PerfReport: 20.18%, LockCompute -- 2.04e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260505 12:39:40.115 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.60e-02 / 0.00e+00 | 20260505 12:39:40.243 | INFO | Starting a round of Baz IO | 20260505 12:39:43.488 | INFO | Populating RTMetrics report | 20260505 12:39:43.488 | INFO | RTMetrics report populated | 20260505 12:39:43.488 | INFO | Opening tmp RTMetrics json file | 20260505 12:39:43.507 | INFO | Serializing RTMetrics to json | 20260505 12:39:43.508 | INFO | Serialization done, swapping tmp file now... | 20260505 12:39:43.508 | INFO | RTMetrics done | 20260505 12:39:43.597 | INFO | Finished a round of Baz IO: Wrote 2.332 GiB in 3.353 seconds | 20260505 12:39:43.597 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.47% / 0.01% / 76.52% | 20260505 12:39:43.597 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.845496% / 3.97364e-06% / 8.74201e-05% | 20260505 12:39:43.597 | INFO | Baz IO overall: | 20260505 12:39:43.597 | INFO | BytesWritten : 1948383963239 | 20260505 12:39:43.597 | INFO | NumEvents : 927344577161 | 20260505 12:39:45.253 | INFO | Analyzing chunk frames = [3276800,3277312) | 20260505 12:39:45.253 | INFO | MemInfo: { | 20260505 12:39:45.253 | INFO | "gpu" : " 17 GB", | 20260505 12:39:45.253 | INFO | "huge" : "", | 20260505 12:39:45.253 | INFO | "huge_shared" : " 27 GB", | 20260505 12:39:45.253 | INFO | "malloc" : " 14 GB", | 20260505 12:39:45.253 | INFO | "pinned" : " 29 GB" | 20260505 12:39:45.253 | INFO | } | 20260505 12:39:45.253 | INFO | Perf report for the last 5.13976 seconds: | 20260505 12:39:45.253 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:39:45.253 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:45.253 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:45.253 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:45.253 | INFO | ANALYSIS : 64.6%, 3.0, 100.0ms, 0 | 20260505 12:39:45.253 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:45.253 | INFO | RT_METRICS : 45.5%, 1.0, 23.9ms, 0 | 20260505 12:39:45.253 | INFO | PRE_HQ : 61.1%, 1.8, 58.2ms, 0 | 20260505 12:39:45.253 | INFO | BAZWRITER : 64.6%, 4.0, 134.3ms, 0 | 20260505 12:39:45.253 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 12:39:45.253 | INFO | Uploaded 36GiB to GPU in 2.17313s (16.566GiB/s) | 20260505 12:39:45.253 | INFO | Downloaded 38GiB to GPU in 2.28686s (16.6167GiB/s) | 20260505 12:39:50.388 | INFO | Analyzing chunk frames = [3277312,3277824) | 20260505 12:39:50.388 | INFO | MemInfo: { | 20260505 12:39:50.388 | INFO | "gpu" : " 17 GB", | 20260505 12:39:50.388 | INFO | "huge" : "", | 20260505 12:39:50.388 | INFO | "huge_shared" : " 27 GB", | 20260505 12:39:50.388 | INFO | "malloc" : " 14 GB", | 20260505 12:39:50.388 | INFO | "pinned" : " 29 GB" | 20260505 12:39:50.388 | INFO | } | 20260505 12:39:50.388 | INFO | Perf report for the last 5.13527 seconds: | 20260505 12:39:50.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:39:50.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:50.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:50.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:50.388 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 12:39:50.388 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:50.388 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:50.388 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 12:39:50.388 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:50.388 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:50.388 | INFO | Uploaded 36GiB to GPU in 2.18028s (16.5116GiB/s) | 20260505 12:39:50.388 | INFO | Downloaded 34GiB to GPU in 2.05002s (16.5852GiB/s) | 20260505 12:39:55.530 | INFO | Analyzing chunk frames = [3277824,3278336) | 20260505 12:39:55.530 | INFO | MemInfo: { | 20260505 12:39:55.530 | INFO | "gpu" : " 17 GB", | 20260505 12:39:55.530 | INFO | "huge" : "", | 20260505 12:39:55.530 | INFO | "huge_shared" : " 27 GB", | 20260505 12:39:55.530 | INFO | "malloc" : " 14 GB", | 20260505 12:39:55.530 | INFO | "pinned" : " 29 GB" | 20260505 12:39:55.530 | INFO | } | 20260505 12:39:55.530 | INFO | Perf report for the last 5.14247 seconds: | 20260505 12:39:55.531 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:39:55.531 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:55.531 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:55.531 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:55.531 | INFO | ANALYSIS : 63.2%, 3.0, 98.4ms, 0 | 20260505 12:39:55.531 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:55.531 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:55.531 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:39:55.531 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:55.531 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:39:55.531 | INFO | Uploaded 36GiB to GPU in 2.16891s (16.5982GiB/s) | 20260505 12:39:55.531 | INFO | Downloaded 34GiB to GPU in 2.11548s (16.072GiB/s) | 20260505 12:40:00.673 | INFO | Analyzing chunk frames = [3278336,3278848) | 20260505 12:40:00.673 | INFO | MemInfo: { | 20260505 12:40:00.673 | INFO | "gpu" : " 17 GB", | 20260505 12:40:00.673 | INFO | "huge" : "", | 20260505 12:40:00.673 | INFO | "huge_shared" : " 27 GB", | 20260505 12:40:00.673 | INFO | "malloc" : " 14 GB", | 20260505 12:40:00.673 | INFO | "pinned" : " 29 GB" | 20260505 12:40:00.673 | INFO | } | 20260505 12:40:00.673 | INFO | Perf report for the last 5.14301 seconds: | 20260505 12:40:00.673 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:40:00.673 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:00.673 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:00.673 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:00.673 | INFO | ANALYSIS : 63.0%, 3.0, 98.0ms, 0 | 20260505 12:40:00.673 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:00.674 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:00.674 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 12:40:00.674 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:00.674 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:00.674 | INFO | Uploaded 36GiB to GPU in 2.20054s (16.3596GiB/s) | 20260505 12:40:00.674 | INFO | Downloaded 34GiB to GPU in 2.13032s (15.96GiB/s) | 20260505 12:40:05.805 | INFO | Analyzing chunk frames = [3278848,3279360) | 20260505 12:40:05.805 | INFO | MemInfo: { | 20260505 12:40:05.805 | INFO | "gpu" : " 17 GB", | 20260505 12:40:05.805 | INFO | "huge" : "", | 20260505 12:40:05.805 | INFO | "huge_shared" : " 27 GB", | 20260505 12:40:05.805 | INFO | "malloc" : " 14 GB", | 20260505 12:40:05.805 | INFO | "pinned" : " 29 GB" | 20260505 12:40:05.805 | INFO | } | 20260505 12:40:05.805 | INFO | Perf report for the last 5.13198 seconds: | 20260505 12:40:05.805 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:40:05.805 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:05.805 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:05.805 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:05.805 | INFO | ANALYSIS : 62.9%, 3.0, 98.0ms, 0 | 20260505 12:40:05.805 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:05.805 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:05.805 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:40:05.805 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:05.806 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:05.806 | INFO | Uploaded 36GiB to GPU in 2.14198s (16.8069GiB/s) | 20260505 12:40:05.806 | INFO | Downloaded 34GiB to GPU in 2.12198s (16.0227GiB/s) | 20260505 12:40:10.938 | INFO | Analyzing chunk frames = [3279360,3279872) | 20260505 12:40:10.938 | INFO | MemInfo: { | 20260505 12:40:10.938 | INFO | "gpu" : " 17 GB", | 20260505 12:40:10.938 | INFO | "huge" : "", | 20260505 12:40:10.939 | INFO | "huge_shared" : " 27 GB", | 20260505 12:40:10.939 | INFO | "malloc" : " 14 GB", | 20260505 12:40:10.939 | INFO | "pinned" : " 29 GB" | 20260505 12:40:10.939 | INFO | } | 20260505 12:40:10.939 | INFO | Perf report for the last 5.13328 seconds: | 20260505 12:40:10.939 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:40:10.939 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:10.939 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:10.939 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:10.939 | INFO | ANALYSIS : 63.4%, 3.0, 98.8ms, 0 | 20260505 12:40:10.939 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:10.939 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:10.939 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 12:40:10.939 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:10.939 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:10.939 | INFO | Uploaded 36GiB to GPU in 2.176s (16.5441GiB/s) | 20260505 12:40:10.939 | INFO | Downloaded 34GiB to GPU in 2.11448s (16.0796GiB/s) | 20260505 12:40:16.085 | INFO | Analyzing chunk frames = [3279872,3280384) | 20260505 12:40:16.085 | INFO | MemInfo: { | 20260505 12:40:16.085 | INFO | "gpu" : " 17 GB", | 20260505 12:40:16.085 | INFO | "huge" : "", | 20260505 12:40:16.085 | INFO | "huge_shared" : " 27 GB", | 20260505 12:40:16.085 | INFO | "malloc" : " 14 GB", | 20260505 12:40:16.085 | INFO | "pinned" : " 29 GB" | 20260505 12:40:16.085 | INFO | } | 20260505 12:40:16.085 | INFO | Perf report for the last 5.14627 seconds: | 20260505 12:40:16.085 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:40:16.085 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:16.085 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:16.085 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:16.085 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 12:40:16.085 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:16.085 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:16.085 | INFO | PRE_HQ : 29.5%, 1.0, 15.4ms, 0 | 20260505 12:40:16.085 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:16.085 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:16.085 | INFO | Uploaded 36GiB to GPU in 2.17775s (16.5308GiB/s) | 20260505 12:40:16.085 | INFO | Downloaded 34GiB to GPU in 2.13784s (15.9039GiB/s) | 20260505 12:40:21.228 | INFO | Analyzing chunk frames = [3280384,3280896) | 20260505 12:40:21.228 | INFO | MemInfo: { | 20260505 12:40:21.228 | INFO | "gpu" : " 17 GB", | 20260505 12:40:21.228 | INFO | "huge" : "", | 20260505 12:40:21.228 | INFO | "huge_shared" : " 27 GB", | 20260505 12:40:21.228 | INFO | "malloc" : " 14 GB", | 20260505 12:40:21.228 | INFO | "pinned" : " 29 GB" | 20260505 12:40:21.228 | INFO | } | 20260505 12:40:21.228 | INFO | Perf report for the last 5.14288 seconds: | 20260505 12:40:21.228 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:40:21.228 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:21.228 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:21.228 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:21.228 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 12:40:21.228 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:21.228 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:21.228 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:40:21.228 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:21.228 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:21.228 | INFO | Uploaded 36GiB to GPU in 2.12161s (16.9682GiB/s) | 20260505 12:40:21.229 | INFO | Downloaded 34GiB to GPU in 2.10597s (16.1446GiB/s) | 20260505 12:40:21.359 | INFO | Starting a round of Baz IO | 20260505 12:40:24.602 | INFO | Populating RTMetrics report | 20260505 12:40:24.602 | INFO | RTMetrics report populated | 20260505 12:40:24.602 | INFO | Opening tmp RTMetrics json file | 20260505 12:40:24.672 | INFO | Serializing RTMetrics to json | 20260505 12:40:24.696 | INFO | Serialization done, swapping tmp file now... | 20260505 12:40:24.696 | INFO | RTMetrics done | 20260505 12:40:24.712 | INFO | Finished a round of Baz IO: Wrote 2.328 GiB in 3.352 seconds | 20260505 12:40:24.712 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.47% / 0.01% / 76.53% | 20260505 12:40:24.712 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.839734% / 1.19209e-05% / 7.15256e-05% | 20260505 12:40:24.712 | INFO | Baz IO overall: | 20260505 12:40:24.712 | INFO | BytesWritten : 1950883379711 | 20260505 12:40:24.712 | INFO | NumEvents : 928528696955 | 20260505 12:40:26.373 | INFO | Analyzing chunk frames = [3280896,3281408) | 20260505 12:40:26.373 | INFO | MemInfo: { | 20260505 12:40:26.373 | INFO | "gpu" : " 17 GB", | 20260505 12:40:26.373 | INFO | "huge" : "", | 20260505 12:40:26.373 | INFO | "huge_shared" : " 27 GB", | 20260505 12:40:26.373 | INFO | "malloc" : " 14 GB", | 20260505 12:40:26.373 | INFO | "pinned" : " 29 GB" | 20260505 12:40:26.373 | INFO | } | 20260505 12:40:26.373 | INFO | Perf report for the last 5.14538 seconds: | 20260505 12:40:26.373 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:40:26.373 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:26.373 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:26.373 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:26.373 | INFO | ANALYSIS : 64.5%, 3.0, 100.0ms, 0 | 20260505 12:40:26.373 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:26.373 | INFO | RT_METRICS : 46.9%, 1.0, 24.6ms, 0 | 20260505 12:40:26.373 | INFO | PRE_HQ : 60.7%, 1.8, 58.0ms, 0 | 20260505 12:40:26.373 | INFO | BAZWRITER : 64.5%, 4.0, 133.9ms, 0 | 20260505 12:40:26.373 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 12:40:26.374 | INFO | Uploaded 36GiB to GPU in 2.17565s (16.5468GiB/s) | 20260505 12:40:26.374 | INFO | Downloaded 38GiB to GPU in 2.27018s (16.7388GiB/s) | 20260505 12:40:31.501 | INFO | Analyzing chunk frames = [3281408,3281920) | 20260505 12:40:31.501 | INFO | MemInfo: { | 20260505 12:40:31.501 | INFO | "gpu" : " 17 GB", | 20260505 12:40:31.501 | INFO | "huge" : "", | 20260505 12:40:31.501 | INFO | "huge_shared" : " 27 GB", | 20260505 12:40:31.501 | INFO | "malloc" : " 14 GB", | 20260505 12:40:31.501 | INFO | "pinned" : " 29 GB" | 20260505 12:40:31.501 | INFO | } | 20260505 12:40:31.501 | INFO | Perf report for the last 5.12796 seconds: | 20260505 12:40:31.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:40:31.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:31.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:31.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:31.501 | INFO | ANALYSIS : 63.8%, 3.0, 99.4ms, 0 | 20260505 12:40:31.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:31.501 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:31.501 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:40:31.501 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:31.501 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:31.501 | INFO | Uploaded 36GiB to GPU in 2.19403s (16.4082GiB/s) | 20260505 12:40:31.502 | INFO | Downloaded 34GiB to GPU in 2.04092s (16.6592GiB/s) | 20260505 12:40:36.644 | INFO | Analyzing chunk frames = [3281920,3282432) | 20260505 12:40:36.644 | INFO | MemInfo: { | 20260505 12:40:36.644 | INFO | "gpu" : " 17 GB", | 20260505 12:40:36.644 | INFO | "huge" : "", | 20260505 12:40:36.644 | INFO | "huge_shared" : " 27 GB", | 20260505 12:40:36.644 | INFO | "malloc" : " 14 GB", | 20260505 12:40:36.644 | INFO | "pinned" : " 29 GB" | 20260505 12:40:36.644 | INFO | } | 20260505 12:40:36.644 | INFO | Perf report for the last 5.14305 seconds: | 20260505 12:40:36.644 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:40:36.644 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:36.644 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:36.644 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:36.644 | INFO | ANALYSIS : 63.2%, 3.0, 98.4ms, 0 | 20260505 12:40:36.644 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:36.644 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:36.644 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:40:36.644 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:36.644 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:36.645 | INFO | Uploaded 36GiB to GPU in 2.21368s (16.2625GiB/s) | 20260505 12:40:36.645 | INFO | Downloaded 34GiB to GPU in 2.11926s (16.0433GiB/s) | 20260505 12:40:41.783 | INFO | Analyzing chunk frames = [3282432,3282944) | 20260505 12:40:41.783 | INFO | MemInfo: { | 20260505 12:40:41.783 | INFO | "gpu" : " 17 GB", | 20260505 12:40:41.783 | INFO | "huge" : "", | 20260505 12:40:41.783 | INFO | "huge_shared" : " 27 GB", | 20260505 12:40:41.783 | INFO | "malloc" : " 14 GB", | 20260505 12:40:41.783 | INFO | "pinned" : " 29 GB" | 20260505 12:40:41.783 | INFO | } | 20260505 12:40:41.783 | INFO | Perf report for the last 5.13881 seconds: | 20260505 12:40:41.783 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:40:41.783 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:41.783 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:41.783 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:41.783 | INFO | ANALYSIS : 63.3%, 3.0, 98.3ms, 0 | 20260505 12:40:41.783 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:41.783 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:41.783 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:40:41.783 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:41.783 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:41.783 | INFO | Uploaded 36GiB to GPU in 2.20797s (16.3045GiB/s) | 20260505 12:40:41.783 | INFO | Downloaded 34GiB to GPU in 2.13223s (15.9458GiB/s) | 20260505 12:40:46.925 | INFO | Analyzing chunk frames = [3282944,3283456) | 20260505 12:40:46.925 | INFO | MemInfo: { | 20260505 12:40:46.925 | INFO | "gpu" : " 17 GB", | 20260505 12:40:46.925 | INFO | "huge" : "", | 20260505 12:40:46.925 | INFO | "huge_shared" : " 27 GB", | 20260505 12:40:46.925 | INFO | "malloc" : " 14 GB", | 20260505 12:40:46.925 | INFO | "pinned" : " 29 GB" | 20260505 12:40:46.925 | INFO | } | 20260505 12:40:46.925 | INFO | Perf report for the last 5.14222 seconds: | 20260505 12:40:46.925 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:40:46.925 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:46.925 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:46.925 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:46.925 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 12:40:46.925 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:46.925 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:46.925 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:40:46.925 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:46.925 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:46.926 | INFO | Uploaded 36GiB to GPU in 2.15753s (16.6858GiB/s) | 20260505 12:40:46.926 | INFO | Downloaded 34GiB to GPU in 2.12404s (16.0073GiB/s) | 20260505 12:40:52.060 | INFO | Analyzing chunk frames = [3283456,3283968) | 20260505 12:40:52.060 | INFO | MemInfo: { | 20260505 12:40:52.060 | INFO | "gpu" : " 17 GB", | 20260505 12:40:52.060 | INFO | "huge" : "", | 20260505 12:40:52.060 | INFO | "huge_shared" : " 27 GB", | 20260505 12:40:52.060 | INFO | "malloc" : " 14 GB", | 20260505 12:40:52.060 | INFO | "pinned" : " 29 GB" | 20260505 12:40:52.060 | INFO | } | 20260505 12:40:52.060 | INFO | Perf report for the last 5.13446 seconds: | 20260505 12:40:52.060 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:40:52.060 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:52.060 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:52.060 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:52.060 | INFO | ANALYSIS : 63.4%, 3.0, 98.8ms, 0 | 20260505 12:40:52.060 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:52.060 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:52.060 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 12:40:52.060 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:52.060 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:52.060 | INFO | Uploaded 36GiB to GPU in 2.18006s (16.5133GiB/s) | 20260505 12:40:52.060 | INFO | Downloaded 34GiB to GPU in 2.10533s (16.1495GiB/s) | 20260505 12:40:57.198 | INFO | Analyzing chunk frames = [3283968,3284480) | 20260505 12:40:57.198 | INFO | MemInfo: { | 20260505 12:40:57.198 | INFO | "gpu" : " 17 GB", | 20260505 12:40:57.198 | INFO | "huge" : "", | 20260505 12:40:57.198 | INFO | "huge_shared" : " 27 GB", | 20260505 12:40:57.198 | INFO | "malloc" : " 14 GB", | 20260505 12:40:57.198 | INFO | "pinned" : " 29 GB" | 20260505 12:40:57.198 | INFO | } | 20260505 12:40:57.198 | INFO | Perf report for the last 5.13817 seconds: | 20260505 12:40:57.198 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:40:57.198 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:57.198 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:57.198 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:57.198 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 12:40:57.198 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:57.198 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:57.198 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 12:40:57.198 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:57.198 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:40:57.198 | INFO | Uploaded 36GiB to GPU in 2.16554s (16.624GiB/s) | 20260505 12:40:57.198 | INFO | Downloaded 34GiB to GPU in 2.14875s (15.8232GiB/s) | 20260505 12:41:02.346 | INFO | Analyzing chunk frames = [3284480,3284992) | 20260505 12:41:02.346 | INFO | MemInfo: { | 20260505 12:41:02.346 | INFO | "gpu" : " 17 GB", | 20260505 12:41:02.346 | INFO | "huge" : "", | 20260505 12:41:02.346 | INFO | "huge_shared" : " 27 GB", | 20260505 12:41:02.346 | INFO | "malloc" : " 14 GB", | 20260505 12:41:02.346 | INFO | "pinned" : " 29 GB" | 20260505 12:41:02.346 | INFO | } | 20260505 12:41:02.346 | INFO | Perf report for the last 5.14832 seconds: | 20260505 12:41:02.346 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:41:02.346 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:02.346 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:02.346 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:02.346 | INFO | ANALYSIS : 63.7%, 3.0, 99.7ms, 0 | 20260505 12:41:02.346 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:02.346 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:02.346 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 12:41:02.346 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:02.347 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:02.347 | INFO | Uploaded 36GiB to GPU in 2.21559s (16.2485GiB/s) | 20260505 12:41:02.347 | INFO | Downloaded 34GiB to GPU in 2.1046s (16.1551GiB/s) | 20260505 12:41:02.347 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 12:41:02.347 | INFO | Total mem usage: 15.3 GiB | 20260505 12:41:02.347 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 12:41:02.347 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.63%) | 20260505 12:41:02.347 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 12:41:02.347 | INFO | High Water Breakdown: | 20260505 12:41:02.347 | INFO | 61% : 8.69 GiB ( 91.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 12:41:02.347 | INFO | 38.9% : 5.53 GiB ( 83.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 12:41:02.347 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 12:41:02.347 | INFO | ------------------------------------------ | 20260505 12:41:02.347 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:41:02.347 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 12:41:02.347 | INFO | Total mem usage: 26.5 GiB | 20260505 12:41:02.347 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 12:41:02.347 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 12:41:02.347 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 12:41:02.347 | INFO | High Water Breakdown: | 20260505 12:41:02.347 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:41:02.347 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:41:02.347 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:41:02.347 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:41:02.347 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:41:02.347 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:41:02.347 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 12:41:02.347 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 12:41:02.347 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:41:02.347 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 12:41:02.348 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 12:41:02.348 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 12:41:02.348 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 12:41:02.348 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 12:41:02.348 | INFO | ------------------------------------------ | 20260505 12:41:02.348 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:41:02.348 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 12:41:02.348 | INFO | Total mem usage: 25.6 GiB | 20260505 12:41:02.348 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 12:41:02.348 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 12:41:02.348 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 12:41:02.348 | INFO | High Water Breakdown: | 20260505 12:41:02.348 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 12:41:02.348 | INFO | ------------------------------------------ | 20260505 12:41:02.348 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:41:02.348 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 12:41:02.348 | INFO | Total mem usage: 15.7 GiB | 20260505 12:41:02.348 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 12:41:02.348 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 12:41:02.348 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 12:41:02.348 | INFO | High Water Breakdown: | 20260505 12:41:02.348 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 12:41:02.348 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 12:41:02.348 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 12:41:02.348 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 12:41:02.348 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 12:41:02.348 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 12:41:02.348 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 12:41:02.348 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 12:41:02.348 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 12:41:02.348 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:41:02.348 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 12:41:02.348 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:41:02.348 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 12:41:02.348 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 12:41:02.348 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 12:41:02.348 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 12:41:02.348 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:41:02.348 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 12:41:02.348 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:41:02.348 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:41:02.348 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:41:02.348 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 12:41:02.348 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 12:41:02.348 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:41:02.348 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 12:41:02.348 | INFO | ------------------------------------------ | 20260505 12:41:02.348 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:41:02.348 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 12:41:02.348 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:41:02.348 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:41:02.348 | INFO | PerfReport: Full Execution -- 3.20e+01 / 7.68e+01, 1.39e+01 / 1.52e+00 | 20260505 12:41:02.348 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 1.01e-01, 1.02e-02 / 4.98e-03 | 20260505 12:41:02.348 | INFO | PerfReport: 11.77%, Baseline -- 3.77e+00 / 4.49e+00, 6.03e-01 / 1.71e-01 | 20260505 12:41:02.348 | INFO | PerfReport: 3.30%, Histogram -- 1.06e+00 / 1.49e+00, 2.02e-01 / 9.94e-02 | 20260505 12:41:02.348 | INFO | PerfReport: 15.87%, DME -- 4.06e+01 / 4.53e+01, 4.43e+00 / 1.98e+00 | 20260505 12:41:02.348 | INFO | PerfReport: 56.48%, FrameLabeling -- 1.81e+01 / 2.06e+01, 1.57e+00 / 1.91e+00 | 20260505 12:41:02.348 | INFO | PerfReport: 7.39%, PulseAccumulating -- 2.37e+00 / 2.86e+00, 4.37e-01 / 2.10e-01 | 20260505 12:41:02.348 | INFO | PerfReport: 3.61%, Metrics -- 1.15e+00 / 1.46e+00, 3.23e-01 / 1.40e-01 | 20260505 12:41:02.348 | INFO | PerfReport: 1.27%, PulseSerializing -- 4.06e-01 / 4.53e-01, 7.35e-02 / 3.06e-02 | 20260505 12:41:02.348 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 12:41:02.348 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:41:02.348 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:41:02.348 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260505 12:41:02.348 | INFO | PerfReport: 22.47%, Upload -- 2.27e+01 / 0.00e+00, 8.04e+00 / 0.00e+00 | 20260505 12:41:02.348 | INFO | PerfReport: 21.99%, Download -- 2.22e+01 / 0.00e+00, 8.13e+00 / 0.00e+00 | 20260505 12:41:02.348 | INFO | PerfReport: 31.70%, Compute -- 3.20e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 12:41:02.348 | INFO | PerfReport: 2.13%, LockUpload -- 2.15e+00 / 0.00e+00, 7.70e+00 / 0.00e+00 | 20260505 12:41:02.348 | INFO | PerfReport: 1.39%, LockDownload -- 1.41e+00 / 0.00e+00, 3.04e+00 / 0.00e+00 | 20260505 12:41:02.348 | INFO | PerfReport: 20.31%, LockCompute -- 2.05e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260505 12:41:02.348 | INFO | PerfReport: 0.01%, Misc -- 1.46e-02 / 0.00e+00, 3.44e-02 / 0.00e+00 | 20260505 12:41:02.485 | INFO | Starting a round of Baz IO | 20260505 12:41:05.702 | INFO | Populating RTMetrics report | 20260505 12:41:05.702 | INFO | RTMetrics report populated | 20260505 12:41:05.702 | INFO | Opening tmp RTMetrics json file | 20260505 12:41:05.703 | INFO | Serializing RTMetrics to json | 20260505 12:41:05.704 | INFO | Serialization done, swapping tmp file now... | 20260505 12:41:05.705 | INFO | RTMetrics done | 20260505 12:41:05.816 | INFO | Finished a round of Baz IO: Wrote 2.327 GiB in 3.33 seconds | 20260505 12:41:05.816 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.46% / 0.01% / 76.53% | 20260505 12:41:05.816 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.836706% / 7.94729e-06% / 5.96046e-05% | 20260505 12:41:05.816 | INFO | Baz IO overall: | 20260505 12:41:05.816 | INFO | BytesWritten : 1953382224705 | 20260505 12:41:05.816 | INFO | NumEvents : 929712911959 | 20260505 12:41:07.484 | INFO | Analyzing chunk frames = [3284992,3285504) | 20260505 12:41:07.484 | INFO | MemInfo: { | 20260505 12:41:07.484 | INFO | "gpu" : " 17 GB", | 20260505 12:41:07.484 | INFO | "huge" : "", | 20260505 12:41:07.484 | INFO | "huge_shared" : " 27 GB", | 20260505 12:41:07.484 | INFO | "malloc" : " 14 GB", | 20260505 12:41:07.484 | INFO | "pinned" : " 29 GB" | 20260505 12:41:07.484 | INFO | } | 20260505 12:41:07.484 | INFO | Perf report for the last 5.1374 seconds: | 20260505 12:41:07.484 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:41:07.484 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:07.484 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:07.484 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:07.484 | INFO | ANALYSIS : 64.2%, 3.0, 99.5ms, 0 | 20260505 12:41:07.484 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:07.484 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260505 12:41:07.484 | INFO | PRE_HQ : 60.9%, 1.8, 58.0ms, 0 | 20260505 12:41:07.484 | INFO | BAZWRITER : 64.2%, 4.1, 133.8ms, 0 | 20260505 12:41:07.484 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 12:41:07.484 | INFO | Uploaded 36GiB to GPU in 2.18224s (16.4968GiB/s) | 20260505 12:41:07.484 | INFO | Downloaded 38GiB to GPU in 2.25768s (16.8315GiB/s) | 20260505 12:41:12.622 | INFO | Analyzing chunk frames = [3285504,3286016) | 20260505 12:41:12.622 | INFO | MemInfo: { | 20260505 12:41:12.622 | INFO | "gpu" : " 17 GB", | 20260505 12:41:12.622 | INFO | "huge" : "", | 20260505 12:41:12.622 | INFO | "huge_shared" : " 27 GB", | 20260505 12:41:12.622 | INFO | "malloc" : " 14 GB", | 20260505 12:41:12.622 | INFO | "pinned" : " 29 GB" | 20260505 12:41:12.622 | INFO | } | 20260505 12:41:12.622 | INFO | Perf report for the last 5.13805 seconds: | 20260505 12:41:12.622 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:41:12.622 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:12.622 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:12.622 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:12.622 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 12:41:12.622 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:12.622 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:12.622 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:41:12.622 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:12.622 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:12.622 | INFO | Uploaded 36GiB to GPU in 2.1598s (16.6682GiB/s) | 20260505 12:41:12.622 | INFO | Downloaded 34GiB to GPU in 2.03477s (16.7095GiB/s) | 20260505 12:41:17.753 | INFO | Analyzing chunk frames = [3286016,3286528) | 20260505 12:41:17.753 | INFO | MemInfo: { | 20260505 12:41:17.753 | INFO | "gpu" : " 17 GB", | 20260505 12:41:17.753 | INFO | "huge" : "", | 20260505 12:41:17.753 | INFO | "huge_shared" : " 27 GB", | 20260505 12:41:17.753 | INFO | "malloc" : " 14 GB", | 20260505 12:41:17.753 | INFO | "pinned" : " 29 GB" | 20260505 12:41:17.754 | INFO | } | 20260505 12:41:17.754 | INFO | Perf report for the last 5.13177 seconds: | 20260505 12:41:17.754 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:41:17.754 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:17.754 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:17.754 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:17.754 | INFO | ANALYSIS : 63.3%, 3.0, 98.3ms, 0 | 20260505 12:41:17.754 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:17.754 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:17.754 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 12:41:17.754 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:17.754 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:17.754 | INFO | Uploaded 36GiB to GPU in 2.22193s (16.2022GiB/s) | 20260505 12:41:17.754 | INFO | Downloaded 34GiB to GPU in 2.12554s (15.996GiB/s) | 20260505 12:41:22.892 | INFO | Analyzing chunk frames = [3286528,3287040) | 20260505 12:41:22.892 | INFO | MemInfo: { | 20260505 12:41:22.892 | INFO | "gpu" : " 17 GB", | 20260505 12:41:22.892 | INFO | "huge" : "", | 20260505 12:41:22.892 | INFO | "huge_shared" : " 27 GB", | 20260505 12:41:22.892 | INFO | "malloc" : " 14 GB", | 20260505 12:41:22.892 | INFO | "pinned" : " 29 GB" | 20260505 12:41:22.892 | INFO | } | 20260505 12:41:22.892 | INFO | Perf report for the last 5.13857 seconds: | 20260505 12:41:22.892 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:41:22.892 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:22.892 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:22.892 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:22.892 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 12:41:22.892 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:22.892 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:22.892 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 12:41:22.892 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:22.892 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:22.892 | INFO | Uploaded 36GiB to GPU in 2.21356s (16.2634GiB/s) | 20260505 12:41:22.892 | INFO | Downloaded 34GiB to GPU in 2.13363s (15.9353GiB/s) | 20260505 12:41:28.039 | INFO | Analyzing chunk frames = [3287040,3287552) | 20260505 12:41:28.039 | INFO | MemInfo: { | 20260505 12:41:28.039 | INFO | "gpu" : " 17 GB", | 20260505 12:41:28.039 | INFO | "huge" : "", | 20260505 12:41:28.039 | INFO | "huge_shared" : " 27 GB", | 20260505 12:41:28.039 | INFO | "malloc" : " 14 GB", | 20260505 12:41:28.039 | INFO | "pinned" : " 29 GB" | 20260505 12:41:28.039 | INFO | } | 20260505 12:41:28.039 | INFO | Perf report for the last 5.14693 seconds: | 20260505 12:41:28.039 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:41:28.039 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:28.039 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:28.039 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:28.039 | INFO | ANALYSIS : 62.7%, 3.0, 98.0ms, 0 | 20260505 12:41:28.039 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:28.039 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:28.039 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 12:41:28.039 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:28.039 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:28.040 | INFO | Uploaded 36GiB to GPU in 2.12673s (16.9274GiB/s) | 20260505 12:41:28.040 | INFO | Downloaded 34GiB to GPU in 2.12507s (15.9995GiB/s) | 20260505 12:41:33.185 | INFO | Analyzing chunk frames = [3287552,3288064) | 20260505 12:41:33.185 | INFO | MemInfo: { | 20260505 12:41:33.185 | INFO | "gpu" : " 17 GB", | 20260505 12:41:33.185 | INFO | "huge" : "", | 20260505 12:41:33.185 | INFO | "huge_shared" : " 27 GB", | 20260505 12:41:33.185 | INFO | "malloc" : " 14 GB", | 20260505 12:41:33.185 | INFO | "pinned" : " 29 GB" | 20260505 12:41:33.185 | INFO | } | 20260505 12:41:33.185 | INFO | Perf report for the last 5.14595 seconds: | 20260505 12:41:33.185 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:41:33.185 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:33.185 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:33.185 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:33.185 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 12:41:33.185 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:33.185 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:33.185 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:41:33.185 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:33.185 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:33.185 | INFO | Uploaded 36GiB to GPU in 2.18372s (16.4856GiB/s) | 20260505 12:41:33.185 | INFO | Downloaded 34GiB to GPU in 2.11244s (16.0952GiB/s) | 20260505 12:41:38.317 | INFO | Analyzing chunk frames = [3288064,3288576) | 20260505 12:41:38.317 | INFO | MemInfo: { | 20260505 12:41:38.317 | INFO | "gpu" : " 17 GB", | 20260505 12:41:38.317 | INFO | "huge" : "", | 20260505 12:41:38.317 | INFO | "huge_shared" : " 27 GB", | 20260505 12:41:38.317 | INFO | "malloc" : " 14 GB", | 20260505 12:41:38.317 | INFO | "pinned" : " 29 GB" | 20260505 12:41:38.317 | INFO | } | 20260505 12:41:38.317 | INFO | Perf report for the last 5.13232 seconds: | 20260505 12:41:38.317 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:41:38.317 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:38.317 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:38.318 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:38.318 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 12:41:38.318 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:38.318 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:38.318 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260505 12:41:38.318 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:38.318 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:38.318 | INFO | Uploaded 36GiB to GPU in 2.14429s (16.7888GiB/s) | 20260505 12:41:38.318 | INFO | Downloaded 34GiB to GPU in 2.16797s (15.6829GiB/s) | 20260505 12:41:43.450 | INFO | Analyzing chunk frames = [3288576,3289088) | 20260505 12:41:43.450 | INFO | MemInfo: { | 20260505 12:41:43.450 | INFO | "gpu" : " 17 GB", | 20260505 12:41:43.450 | INFO | "huge" : "", | 20260505 12:41:43.450 | INFO | "huge_shared" : " 27 GB", | 20260505 12:41:43.450 | INFO | "malloc" : " 14 GB", | 20260505 12:41:43.450 | INFO | "pinned" : " 29 GB" | 20260505 12:41:43.450 | INFO | } | 20260505 12:41:43.450 | INFO | Perf report for the last 5.13296 seconds: | 20260505 12:41:43.450 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:41:43.450 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:43.450 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:43.450 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:43.450 | INFO | ANALYSIS : 63.9%, 3.0, 99.6ms, 0 | 20260505 12:41:43.450 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:43.450 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:43.450 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 12:41:43.450 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:43.451 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:43.451 | INFO | Uploaded 36GiB to GPU in 2.21287s (16.2685GiB/s) | 20260505 12:41:43.451 | INFO | Downloaded 34GiB to GPU in 2.09651s (16.2174GiB/s) | 20260505 12:41:43.586 | INFO | Starting a round of Baz IO | 20260505 12:41:46.840 | INFO | Populating RTMetrics report | 20260505 12:41:46.840 | INFO | RTMetrics report populated | 20260505 12:41:46.840 | INFO | Opening tmp RTMetrics json file | 20260505 12:41:46.859 | INFO | Serializing RTMetrics to json | 20260505 12:41:46.861 | INFO | Serialization done, swapping tmp file now... | 20260505 12:41:46.862 | INFO | RTMetrics done | 20260505 12:41:46.950 | INFO | Finished a round of Baz IO: Wrote 2.326 GiB in 3.363 seconds | 20260505 12:41:46.950 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.45% / 0.01% / 76.54% | 20260505 12:41:46.950 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.835812% / 7.94729e-06% / 6.35783e-05% | 20260505 12:41:46.950 | INFO | Baz IO overall: | 20260505 12:41:46.950 | INFO | BytesWritten : 1955879851327 | 20260505 12:41:46.950 | INFO | NumEvents : 930896591988 | 20260505 12:41:48.601 | INFO | Analyzing chunk frames = [3289088,3289600) | 20260505 12:41:48.601 | INFO | MemInfo: { | 20260505 12:41:48.601 | INFO | "gpu" : " 17 GB", | 20260505 12:41:48.601 | INFO | "huge" : "", | 20260505 12:41:48.601 | INFO | "huge_shared" : " 27 GB", | 20260505 12:41:48.601 | INFO | "malloc" : " 14 GB", | 20260505 12:41:48.601 | INFO | "pinned" : " 29 GB" | 20260505 12:41:48.601 | INFO | } | 20260505 12:41:48.601 | INFO | Perf report for the last 5.15075 seconds: | 20260505 12:41:48.601 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:41:48.601 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:48.601 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:48.601 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:48.601 | INFO | ANALYSIS : 64.7%, 3.0, 100.3ms, 0 | 20260505 12:41:48.601 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:48.601 | INFO | RT_METRICS : 45.5%, 1.0, 23.9ms, 0 | 20260505 12:41:48.601 | INFO | PRE_HQ : 60.8%, 1.8, 58.2ms, 0 | 20260505 12:41:48.601 | INFO | BAZWRITER : 64.6%, 4.1, 135.0ms, 0 | 20260505 12:41:48.601 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 12:41:48.601 | INFO | Uploaded 36GiB to GPU in 2.18147s (16.5026GiB/s) | 20260505 12:41:48.602 | INFO | Downloaded 38GiB to GPU in 2.30395s (16.4934GiB/s) | 20260505 12:41:53.737 | INFO | Analyzing chunk frames = [3289600,3290112) | 20260505 12:41:53.737 | INFO | MemInfo: { | 20260505 12:41:53.737 | INFO | "gpu" : " 17 GB", | 20260505 12:41:53.737 | INFO | "huge" : "", | 20260505 12:41:53.737 | INFO | "huge_shared" : " 27 GB", | 20260505 12:41:53.737 | INFO | "malloc" : " 14 GB", | 20260505 12:41:53.737 | INFO | "pinned" : " 29 GB" | 20260505 12:41:53.737 | INFO | } | 20260505 12:41:53.737 | INFO | Perf report for the last 5.13594 seconds: | 20260505 12:41:53.737 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:41:53.737 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:53.737 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:53.737 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:53.737 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 12:41:53.737 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:53.737 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:53.737 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:41:53.737 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:53.737 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:53.738 | INFO | Uploaded 36GiB to GPU in 2.19657s (16.3892GiB/s) | 20260505 12:41:53.738 | INFO | Downloaded 34GiB to GPU in 2.03528s (16.7053GiB/s) | 20260505 12:41:58.874 | INFO | Analyzing chunk frames = [3290112,3290624) | 20260505 12:41:58.874 | INFO | MemInfo: { | 20260505 12:41:58.874 | INFO | "gpu" : " 17 GB", | 20260505 12:41:58.874 | INFO | "huge" : "", | 20260505 12:41:58.874 | INFO | "huge_shared" : " 27 GB", | 20260505 12:41:58.874 | INFO | "malloc" : " 14 GB", | 20260505 12:41:58.874 | INFO | "pinned" : " 29 GB" | 20260505 12:41:58.874 | INFO | } | 20260505 12:41:58.874 | INFO | Perf report for the last 5.1367 seconds: | 20260505 12:41:58.874 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:41:58.874 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:58.874 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:58.874 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:58.874 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 12:41:58.874 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:58.874 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:58.874 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:41:58.874 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:58.874 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:41:58.874 | INFO | Uploaded 36GiB to GPU in 2.1669s (16.6136GiB/s) | 20260505 12:41:58.874 | INFO | Downloaded 34GiB to GPU in 2.13103s (15.9547GiB/s) | 20260505 12:42:04.015 | INFO | Analyzing chunk frames = [3290624,3291136) | 20260505 12:42:04.015 | INFO | MemInfo: { | 20260505 12:42:04.015 | INFO | "gpu" : " 17 GB", | 20260505 12:42:04.015 | INFO | "huge" : "", | 20260505 12:42:04.016 | INFO | "huge_shared" : " 27 GB", | 20260505 12:42:04.016 | INFO | "malloc" : " 14 GB", | 20260505 12:42:04.016 | INFO | "pinned" : " 29 GB" | 20260505 12:42:04.016 | INFO | } | 20260505 12:42:04.016 | INFO | Perf report for the last 5.14194 seconds: | 20260505 12:42:04.016 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:42:04.016 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:04.016 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:04.016 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:04.016 | INFO | ANALYSIS : 63.2%, 3.0, 98.3ms, 0 | 20260505 12:42:04.016 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:04.016 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:04.016 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:42:04.016 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:04.016 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:04.016 | INFO | Uploaded 36GiB to GPU in 2.20906s (16.2966GiB/s) | 20260505 12:42:04.016 | INFO | Downloaded 34GiB to GPU in 2.12966s (15.965GiB/s) | 20260505 12:42:09.151 | INFO | Analyzing chunk frames = [3291136,3291648) | 20260505 12:42:09.151 | INFO | MemInfo: { | 20260505 12:42:09.151 | INFO | "gpu" : " 17 GB", | 20260505 12:42:09.151 | INFO | "huge" : "", | 20260505 12:42:09.151 | INFO | "huge_shared" : " 27 GB", | 20260505 12:42:09.151 | INFO | "malloc" : " 14 GB", | 20260505 12:42:09.151 | INFO | "pinned" : " 29 GB" | 20260505 12:42:09.151 | INFO | } | 20260505 12:42:09.151 | INFO | Perf report for the last 5.13525 seconds: | 20260505 12:42:09.151 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:42:09.151 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:09.151 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:09.151 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:09.151 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 12:42:09.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:09.151 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:09.151 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:42:09.151 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:09.151 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:09.151 | INFO | Uploaded 36GiB to GPU in 2.17563s (16.5469GiB/s) | 20260505 12:42:09.151 | INFO | Downloaded 34GiB to GPU in 2.12612s (15.9915GiB/s) | 20260505 12:42:14.295 | INFO | Analyzing chunk frames = [3291648,3292160) | 20260505 12:42:14.295 | INFO | MemInfo: { | 20260505 12:42:14.295 | INFO | "gpu" : " 17 GB", | 20260505 12:42:14.295 | INFO | "huge" : "", | 20260505 12:42:14.295 | INFO | "huge_shared" : " 27 GB", | 20260505 12:42:14.295 | INFO | "malloc" : " 14 GB", | 20260505 12:42:14.295 | INFO | "pinned" : " 29 GB" | 20260505 12:42:14.295 | INFO | } | 20260505 12:42:14.295 | INFO | Perf report for the last 5.1437 seconds: | 20260505 12:42:14.295 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:42:14.295 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:14.295 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:14.295 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:14.295 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 12:42:14.295 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:14.295 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:14.295 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 12:42:14.295 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:14.295 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:14.295 | INFO | Uploaded 36GiB to GPU in 2.16709s (16.6122GiB/s) | 20260505 12:42:14.295 | INFO | Downloaded 34GiB to GPU in 2.10909s (16.1207GiB/s) | 20260505 12:42:19.434 | INFO | Analyzing chunk frames = [3292160,3292672) | 20260505 12:42:19.434 | INFO | MemInfo: { | 20260505 12:42:19.434 | INFO | "gpu" : " 17 GB", | 20260505 12:42:19.434 | INFO | "huge" : "", | 20260505 12:42:19.434 | INFO | "huge_shared" : " 27 GB", | 20260505 12:42:19.434 | INFO | "malloc" : " 14 GB", | 20260505 12:42:19.434 | INFO | "pinned" : " 29 GB" | 20260505 12:42:19.434 | INFO | } | 20260505 12:42:19.434 | INFO | Perf report for the last 5.13956 seconds: | 20260505 12:42:19.434 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:42:19.434 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:19.434 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:19.434 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:19.434 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 12:42:19.434 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:19.434 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:19.434 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 12:42:19.434 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:19.434 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:19.435 | INFO | Uploaded 36GiB to GPU in 2.1664s (16.6175GiB/s) | 20260505 12:42:19.435 | INFO | Downloaded 34GiB to GPU in 2.14494s (15.8513GiB/s) | 20260505 12:42:24.564 | INFO | Analyzing chunk frames = [3292672,3293184) | 20260505 12:42:24.564 | INFO | MemInfo: { | 20260505 12:42:24.564 | INFO | "gpu" : " 17 GB", | 20260505 12:42:24.564 | INFO | "huge" : "", | 20260505 12:42:24.564 | INFO | "huge_shared" : " 27 GB", | 20260505 12:42:24.564 | INFO | "malloc" : " 14 GB", | 20260505 12:42:24.564 | INFO | "pinned" : " 29 GB" | 20260505 12:42:24.564 | INFO | } | 20260505 12:42:24.564 | INFO | Perf report for the last 5.12984 seconds: | 20260505 12:42:24.564 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:42:24.564 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:24.564 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:24.564 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:24.564 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 12:42:24.564 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:24.564 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:24.564 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 12:42:24.564 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:24.564 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:24.565 | INFO | Uploaded 36GiB to GPU in 2.14028s (16.8202GiB/s) | 20260505 12:42:24.565 | INFO | Downloaded 34GiB to GPU in 2.10584s (16.1456GiB/s) | 20260505 12:42:24.565 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 12:42:24.565 | INFO | Total mem usage: 15.3 GiB | 20260505 12:42:24.565 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 12:42:24.565 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -6.92%) | 20260505 12:42:24.565 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 12:42:24.565 | INFO | High Water Breakdown: | 20260505 12:42:24.565 | INFO | 61.1% : 8.76 GiB ( 92% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 12:42:24.565 | INFO | 38.8% : 5.56 GiB ( 83.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 12:42:24.565 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 12:42:24.565 | INFO | ------------------------------------------ | 20260505 12:42:24.565 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:42:24.565 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 12:42:24.565 | INFO | Total mem usage: 26.5 GiB | 20260505 12:42:24.565 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 12:42:24.565 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 12:42:24.565 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 12:42:24.565 | INFO | High Water Breakdown: | 20260505 12:42:24.565 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:42:24.565 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:42:24.565 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:42:24.565 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:42:24.565 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:42:24.565 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:42:24.565 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 12:42:24.565 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 12:42:24.565 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:42:24.565 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 12:42:24.565 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 12:42:24.565 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 12:42:24.565 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 12:42:24.565 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 12:42:24.565 | INFO | ------------------------------------------ | 20260505 12:42:24.565 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:42:24.565 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 12:42:24.565 | INFO | Total mem usage: 25.6 GiB | 20260505 12:42:24.565 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 12:42:24.565 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 12:42:24.565 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 12:42:24.565 | INFO | High Water Breakdown: | 20260505 12:42:24.565 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 12:42:24.565 | INFO | ------------------------------------------ | 20260505 12:42:24.565 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:42:24.565 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 12:42:24.565 | INFO | Total mem usage: 15.7 GiB | 20260505 12:42:24.565 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 12:42:24.565 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 12:42:24.565 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 12:42:24.565 | INFO | High Water Breakdown: | 20260505 12:42:24.565 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 12:42:24.565 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 12:42:24.565 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 12:42:24.565 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 12:42:24.565 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 12:42:24.565 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 12:42:24.565 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 12:42:24.565 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 12:42:24.565 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 12:42:24.565 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:42:24.565 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 12:42:24.566 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:42:24.566 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 12:42:24.566 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 12:42:24.566 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 12:42:24.566 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 12:42:24.566 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:42:24.566 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 12:42:24.566 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:42:24.566 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:42:24.566 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:42:24.566 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 12:42:24.566 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 12:42:24.566 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:42:24.566 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 12:42:24.566 | INFO | ------------------------------------------ | 20260505 12:42:24.566 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:42:24.566 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 12:42:24.566 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:42:24.566 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:42:24.566 | INFO | PerfReport: Full Execution -- 3.20e+01 / 7.70e+01, 1.37e+01 / 1.58e+00 | 20260505 12:42:24.566 | INFO | PerfReport: 0.29%, AggregateStats -- 9.30e-02 / 9.96e-02, 1.05e-02 / 3.55e-03 | 20260505 12:42:24.566 | INFO | PerfReport: 11.77%, Baseline -- 3.76e+00 / 4.49e+00, 5.99e-01 / 1.94e-01 | 20260505 12:42:24.566 | INFO | PerfReport: 3.32%, Histogram -- 1.06e+00 / 1.48e+00, 2.04e-01 / 1.09e-01 | 20260505 12:42:24.566 | INFO | PerfReport: 15.92%, DME -- 4.07e+01 / 4.57e+01, 4.60e+00 / 2.23e+00 | 20260505 12:42:24.566 | INFO | PerfReport: 56.38%, FrameLabeling -- 1.80e+01 / 2.03e+01, 1.46e+00 / 1.91e+00 | 20260505 12:42:24.566 | INFO | PerfReport: 7.43%, PulseAccumulating -- 2.37e+00 / 2.90e+00, 4.45e-01 / 1.85e-01 | 20260505 12:42:24.566 | INFO | PerfReport: 3.62%, Metrics -- 1.16e+00 / 1.48e+00, 3.28e-01 / 1.52e-01 | 20260505 12:42:24.566 | INFO | PerfReport: 1.27%, PulseSerializing -- 4.07e-01 / 4.56e-01, 7.41e-02 / 2.97e-02 | 20260505 12:42:24.566 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 12:42:24.566 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:42:24.566 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:42:24.566 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260505 12:42:24.566 | INFO | PerfReport: 22.47%, Upload -- 2.27e+01 / 0.00e+00, 8.01e+00 / 0.00e+00 | 20260505 12:42:24.566 | INFO | PerfReport: 22.02%, Download -- 2.22e+01 / 0.00e+00, 8.13e+00 / 0.00e+00 | 20260505 12:42:24.566 | INFO | PerfReport: 31.68%, Compute -- 3.20e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 12:42:24.566 | INFO | PerfReport: 2.10%, LockUpload -- 2.12e+00 / 0.00e+00, 7.75e+00 / 0.00e+00 | 20260505 12:42:24.566 | INFO | PerfReport: 1.42%, LockDownload -- 1.43e+00 / 0.00e+00, 3.03e+00 / 0.00e+00 | 20260505 12:42:24.566 | INFO | PerfReport: 20.29%, LockCompute -- 2.05e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260505 12:42:24.566 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.78e-02 / 0.00e+00 | 20260505 12:42:24.699 | INFO | Starting a round of Baz IO | 20260505 12:42:27.952 | INFO | Populating RTMetrics report | 20260505 12:42:27.952 | INFO | RTMetrics report populated | 20260505 12:42:27.952 | INFO | Opening tmp RTMetrics json file | 20260505 12:42:27.953 | INFO | Serializing RTMetrics to json | 20260505 12:42:27.954 | INFO | Serialization done, swapping tmp file now... | 20260505 12:42:27.955 | INFO | RTMetrics done | 20260505 12:42:28.076 | INFO | Finished a round of Baz IO: Wrote 2.33 GiB in 3.377 seconds | 20260505 12:42:28.076 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.45% / 0.01% / 76.55% | 20260505 12:42:28.076 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.83398% / 3.97364e-06% / 7.54992e-05% | 20260505 12:42:28.076 | INFO | Baz IO overall: | 20260505 12:42:28.076 | INFO | BytesWritten : 1958381942274 | 20260505 12:42:28.076 | INFO | NumEvents : 932082620093 | 20260505 12:42:29.700 | INFO | Analyzing chunk frames = [3293184,3293696) | 20260505 12:42:29.700 | INFO | MemInfo: { | 20260505 12:42:29.700 | INFO | "gpu" : " 17 GB", | 20260505 12:42:29.700 | INFO | "huge" : "", | 20260505 12:42:29.700 | INFO | "huge_shared" : " 27 GB", | 20260505 12:42:29.700 | INFO | "malloc" : " 14 GB", | 20260505 12:42:29.700 | INFO | "pinned" : " 29 GB" | 20260505 12:42:29.700 | INFO | } | 20260505 12:42:29.700 | INFO | Perf report for the last 5.13623 seconds: | 20260505 12:42:29.700 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:42:29.700 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:29.700 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:29.700 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:29.700 | INFO | ANALYSIS : 64.8%, 3.0, 100.6ms, 0 | 20260505 12:42:29.701 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:29.701 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260505 12:42:29.701 | INFO | PRE_HQ : 61.1%, 1.8, 58.3ms, 0 | 20260505 12:42:29.701 | INFO | BAZWRITER : 65.1%, 4.0, 134.8ms, 0 | 20260505 12:42:29.701 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 12:42:29.701 | INFO | Uploaded 36GiB to GPU in 2.19095s (16.4312GiB/s) | 20260505 12:42:29.701 | INFO | Downloaded 38GiB to GPU in 2.28094s (16.6598GiB/s) | 20260505 12:42:34.845 | INFO | Analyzing chunk frames = [3293696,3294208) | 20260505 12:42:34.846 | INFO | MemInfo: { | 20260505 12:42:34.846 | INFO | "gpu" : " 17 GB", | 20260505 12:42:34.846 | INFO | "huge" : "", | 20260505 12:42:34.846 | INFO | "huge_shared" : " 27 GB", | 20260505 12:42:34.846 | INFO | "malloc" : " 14 GB", | 20260505 12:42:34.846 | INFO | "pinned" : " 29 GB" | 20260505 12:42:34.846 | INFO | } | 20260505 12:42:34.846 | INFO | Perf report for the last 5.14537 seconds: | 20260505 12:42:34.846 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:42:34.846 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:34.846 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:34.846 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:34.846 | INFO | ANALYSIS : 64.2%, 3.0, 100.4ms, 0 | 20260505 12:42:34.846 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:34.846 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:34.846 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 12:42:34.846 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:34.846 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:34.846 | INFO | Uploaded 36GiB to GPU in 2.24094s (16.0647GiB/s) | 20260505 12:42:34.846 | INFO | Downloaded 34GiB to GPU in 2.08411s (16.3139GiB/s) | 20260505 12:42:39.996 | INFO | Analyzing chunk frames = [3294208,3294720) | 20260505 12:42:39.996 | INFO | MemInfo: { | 20260505 12:42:39.996 | INFO | "gpu" : " 17 GB", | 20260505 12:42:39.996 | INFO | "huge" : "", | 20260505 12:42:39.996 | INFO | "huge_shared" : " 27 GB", | 20260505 12:42:39.996 | INFO | "malloc" : " 14 GB", | 20260505 12:42:39.996 | INFO | "pinned" : " 29 GB" | 20260505 12:42:39.996 | INFO | } | 20260505 12:42:39.996 | INFO | Perf report for the last 5.15008 seconds: | 20260505 12:42:39.996 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:42:39.996 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:39.996 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:39.996 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:39.996 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 12:42:39.996 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:39.996 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:39.996 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 12:42:39.996 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:39.996 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:39.996 | INFO | Uploaded 36GiB to GPU in 2.16085s (16.6601GiB/s) | 20260505 12:42:39.996 | INFO | Downloaded 34GiB to GPU in 2.13265s (15.9426GiB/s) | 20260505 12:42:45.129 | INFO | Analyzing chunk frames = [3294720,3295232) | 20260505 12:42:45.129 | INFO | MemInfo: { | 20260505 12:42:45.129 | INFO | "gpu" : " 17 GB", | 20260505 12:42:45.129 | INFO | "huge" : "", | 20260505 12:42:45.129 | INFO | "huge_shared" : " 27 GB", | 20260505 12:42:45.129 | INFO | "malloc" : " 14 GB", | 20260505 12:42:45.129 | INFO | "pinned" : " 29 GB" | 20260505 12:42:45.129 | INFO | } | 20260505 12:42:45.129 | INFO | Perf report for the last 5.13353 seconds: | 20260505 12:42:45.130 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:42:45.130 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:45.130 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:45.130 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:45.130 | INFO | ANALYSIS : 63.1%, 3.0, 98.0ms, 0 | 20260505 12:42:45.130 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:45.130 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:45.130 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 12:42:45.130 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:45.130 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:45.130 | INFO | Uploaded 36GiB to GPU in 2.18605s (16.468GiB/s) | 20260505 12:42:45.130 | INFO | Downloaded 34GiB to GPU in 2.12797s (15.9777GiB/s) | 20260505 12:42:50.262 | INFO | Analyzing chunk frames = [3295232,3295744) | 20260505 12:42:50.262 | INFO | MemInfo: { | 20260505 12:42:50.262 | INFO | "gpu" : " 17 GB", | 20260505 12:42:50.262 | INFO | "huge" : "", | 20260505 12:42:50.262 | INFO | "huge_shared" : " 27 GB", | 20260505 12:42:50.262 | INFO | "malloc" : " 14 GB", | 20260505 12:42:50.262 | INFO | "pinned" : " 29 GB" | 20260505 12:42:50.262 | INFO | } | 20260505 12:42:50.262 | INFO | Perf report for the last 5.13283 seconds: | 20260505 12:42:50.262 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:42:50.262 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:50.262 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:50.262 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:50.262 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 12:42:50.262 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:50.262 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:50.262 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 12:42:50.262 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:50.262 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:50.262 | INFO | Uploaded 36GiB to GPU in 2.15652s (16.6935GiB/s) | 20260505 12:42:50.263 | INFO | Downloaded 34GiB to GPU in 2.12233s (16.0201GiB/s) | 20260505 12:42:55.404 | INFO | Analyzing chunk frames = [3295744,3296256) | 20260505 12:42:55.405 | INFO | MemInfo: { | 20260505 12:42:55.405 | INFO | "gpu" : " 17 GB", | 20260505 12:42:55.405 | INFO | "huge" : "", | 20260505 12:42:55.405 | INFO | "huge_shared" : " 27 GB", | 20260505 12:42:55.405 | INFO | "malloc" : " 14 GB", | 20260505 12:42:55.405 | INFO | "pinned" : " 29 GB" | 20260505 12:42:55.405 | INFO | } | 20260505 12:42:55.405 | INFO | Perf report for the last 5.14234 seconds: | 20260505 12:42:55.405 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:42:55.405 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:55.405 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:55.405 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:55.405 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 12:42:55.405 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:55.405 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:55.405 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:42:55.405 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:55.405 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:42:55.405 | INFO | Uploaded 36GiB to GPU in 2.15823s (16.6803GiB/s) | 20260505 12:42:55.405 | INFO | Downloaded 34GiB to GPU in 2.11665s (16.0631GiB/s) | 20260505 12:43:00.547 | INFO | Analyzing chunk frames = [3296256,3296768) | 20260505 12:43:00.547 | INFO | MemInfo: { | 20260505 12:43:00.547 | INFO | "gpu" : " 17 GB", | 20260505 12:43:00.547 | INFO | "huge" : "", | 20260505 12:43:00.547 | INFO | "huge_shared" : " 27 GB", | 20260505 12:43:00.547 | INFO | "malloc" : " 14 GB", | 20260505 12:43:00.547 | INFO | "pinned" : " 29 GB" | 20260505 12:43:00.547 | INFO | } | 20260505 12:43:00.547 | INFO | Perf report for the last 5.14252 seconds: | 20260505 12:43:00.547 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:43:00.547 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:00.547 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:00.547 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:00.547 | INFO | ANALYSIS : 63.5%, 3.0, 98.2ms, 0 | 20260505 12:43:00.547 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:00.547 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:00.547 | INFO | PRE_HQ : 29.6%, 1.0, 15.2ms, 0 | 20260505 12:43:00.547 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:00.547 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:00.547 | INFO | Uploaded 36GiB to GPU in 2.15753s (16.6857GiB/s) | 20260505 12:43:00.547 | INFO | Downloaded 34GiB to GPU in 2.1623s (15.724GiB/s) | 20260505 12:43:05.689 | INFO | Analyzing chunk frames = [3296768,3297280) | 20260505 12:43:05.689 | INFO | MemInfo: { | 20260505 12:43:05.689 | INFO | "gpu" : " 17 GB", | 20260505 12:43:05.689 | INFO | "huge" : "", | 20260505 12:43:05.689 | INFO | "huge_shared" : " 27 GB", | 20260505 12:43:05.690 | INFO | "malloc" : " 14 GB", | 20260505 12:43:05.690 | INFO | "pinned" : " 29 GB" | 20260505 12:43:05.690 | INFO | } | 20260505 12:43:05.690 | INFO | Perf report for the last 5.1425 seconds: | 20260505 12:43:05.690 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:43:05.690 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:05.690 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:05.690 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:05.690 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 12:43:05.690 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:05.690 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:05.690 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 12:43:05.690 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:05.690 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:05.690 | INFO | Uploaded 36GiB to GPU in 2.17661s (16.5395GiB/s) | 20260505 12:43:05.690 | INFO | Downloaded 34GiB to GPU in 2.10711s (16.1358GiB/s) | 20260505 12:43:05.823 | INFO | Starting a round of Baz IO | 20260505 12:43:09.070 | INFO | Populating RTMetrics report | 20260505 12:43:09.070 | INFO | RTMetrics report populated | 20260505 12:43:09.070 | INFO | Opening tmp RTMetrics json file | 20260505 12:43:09.099 | INFO | Serializing RTMetrics to json | 20260505 12:43:09.100 | INFO | Serialization done, swapping tmp file now... | 20260505 12:43:09.100 | INFO | RTMetrics done | 20260505 12:43:09.201 | INFO | Finished a round of Baz IO: Wrote 2.347 GiB in 3.377 seconds | 20260505 12:43:09.201 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.44% / 0.01% / 76.55% | 20260505 12:43:09.201 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.837978% / 0% / 7.15256e-05% | 20260505 12:43:09.201 | INFO | Baz IO overall: | 20260505 12:43:09.201 | INFO | BytesWritten : 1960902159589 | 20260505 12:43:09.201 | INFO | NumEvents : 933274583095 | 20260505 12:43:10.823 | INFO | Analyzing chunk frames = [3297280,3297792) | 20260505 12:43:10.823 | INFO | MemInfo: { | 20260505 12:43:10.823 | INFO | "gpu" : " 17 GB", | 20260505 12:43:10.823 | INFO | "huge" : "", | 20260505 12:43:10.823 | INFO | "huge_shared" : " 27 GB", | 20260505 12:43:10.823 | INFO | "malloc" : " 14 GB", | 20260505 12:43:10.823 | INFO | "pinned" : " 29 GB" | 20260505 12:43:10.823 | INFO | } | 20260505 12:43:10.823 | INFO | Perf report for the last 5.13342 seconds: | 20260505 12:43:10.823 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:43:10.823 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:10.823 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:10.823 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:10.823 | INFO | ANALYSIS : 64.8%, 2.9, 99.8ms, 0 | 20260505 12:43:10.823 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:10.823 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260505 12:43:10.823 | INFO | PRE_HQ : 61.0%, 1.8, 58.3ms, 0 | 20260505 12:43:10.823 | INFO | BAZWRITER : 65.1%, 4.0, 135.3ms, 0 | 20260505 12:43:10.823 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 12:43:10.823 | INFO | Uploaded 36GiB to GPU in 2.20829s (16.3022GiB/s) | 20260505 12:43:10.824 | INFO | Downloaded 38GiB to GPU in 2.28955s (16.5971GiB/s) | 20260505 12:43:15.963 | INFO | Analyzing chunk frames = [3297792,3298304) | 20260505 12:43:15.963 | INFO | MemInfo: { | 20260505 12:43:15.963 | INFO | "gpu" : " 17 GB", | 20260505 12:43:15.963 | INFO | "huge" : "", | 20260505 12:43:15.963 | INFO | "huge_shared" : " 27 GB", | 20260505 12:43:15.963 | INFO | "malloc" : " 14 GB", | 20260505 12:43:15.963 | INFO | "pinned" : " 29 GB" | 20260505 12:43:15.963 | INFO | } | 20260505 12:43:15.963 | INFO | Perf report for the last 5.14 seconds: | 20260505 12:43:15.963 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:43:15.963 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:15.963 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:15.963 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:15.963 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 12:43:15.963 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:15.963 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:15.963 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:43:15.963 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:15.963 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:15.963 | INFO | Uploaded 36GiB to GPU in 2.23564s (16.1028GiB/s) | 20260505 12:43:15.963 | INFO | Downloaded 34GiB to GPU in 2.08242s (16.3271GiB/s) | 20260505 12:43:21.111 | INFO | Analyzing chunk frames = [3298304,3298816) | 20260505 12:43:21.111 | INFO | MemInfo: { | 20260505 12:43:21.111 | INFO | "gpu" : " 17 GB", | 20260505 12:43:21.111 | INFO | "huge" : "", | 20260505 12:43:21.111 | INFO | "huge_shared" : " 27 GB", | 20260505 12:43:21.111 | INFO | "malloc" : " 14 GB", | 20260505 12:43:21.111 | INFO | "pinned" : " 29 GB" | 20260505 12:43:21.111 | INFO | } | 20260505 12:43:21.111 | INFO | Perf report for the last 5.14779 seconds: | 20260505 12:43:21.111 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:43:21.111 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:21.111 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:21.111 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:21.111 | INFO | ANALYSIS : 63.1%, 3.0, 98.3ms, 0 | 20260505 12:43:21.111 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:21.111 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:21.111 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 12:43:21.111 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:21.111 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:21.111 | INFO | Uploaded 36GiB to GPU in 2.18603s (16.4682GiB/s) | 20260505 12:43:21.111 | INFO | Downloaded 34GiB to GPU in 2.11667s (16.0629GiB/s) | 20260505 12:43:26.243 | INFO | Analyzing chunk frames = [3298816,3299328) | 20260505 12:43:26.243 | INFO | MemInfo: { | 20260505 12:43:26.243 | INFO | "gpu" : " 17 GB", | 20260505 12:43:26.243 | INFO | "huge" : "", | 20260505 12:43:26.243 | INFO | "huge_shared" : " 27 GB", | 20260505 12:43:26.243 | INFO | "malloc" : " 14 GB", | 20260505 12:43:26.243 | INFO | "pinned" : " 29 GB" | 20260505 12:43:26.243 | INFO | } | 20260505 12:43:26.243 | INFO | Perf report for the last 5.13236 seconds: | 20260505 12:43:26.243 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:43:26.243 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:26.243 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:26.243 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:26.243 | INFO | ANALYSIS : 63.1%, 3.0, 97.8ms, 0 | 20260505 12:43:26.243 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:26.243 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:26.243 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 12:43:26.243 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:26.243 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:26.244 | INFO | Uploaded 36GiB to GPU in 2.19935s (16.3685GiB/s) | 20260505 12:43:26.244 | INFO | Downloaded 34GiB to GPU in 2.1289s (15.9707GiB/s) | 20260505 12:43:31.381 | INFO | Analyzing chunk frames = [3299328,3299840) | 20260505 12:43:31.381 | INFO | MemInfo: { | 20260505 12:43:31.381 | INFO | "gpu" : " 17 GB", | 20260505 12:43:31.381 | INFO | "huge" : "", | 20260505 12:43:31.381 | INFO | "huge_shared" : " 27 GB", | 20260505 12:43:31.381 | INFO | "malloc" : " 14 GB", | 20260505 12:43:31.382 | INFO | "pinned" : " 29 GB" | 20260505 12:43:31.382 | INFO | } | 20260505 12:43:31.382 | INFO | Perf report for the last 5.1384 seconds: | 20260505 12:43:31.382 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:43:31.382 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:31.382 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:31.382 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:31.382 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 12:43:31.382 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:31.382 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:31.382 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:43:31.382 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:31.382 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:31.382 | INFO | Uploaded 36GiB to GPU in 2.16162s (16.6542GiB/s) | 20260505 12:43:31.382 | INFO | Downloaded 34GiB to GPU in 2.1277s (15.9797GiB/s) | 20260505 12:43:36.514 | INFO | Analyzing chunk frames = [3299840,3300352) | 20260505 12:43:36.514 | INFO | MemInfo: { | 20260505 12:43:36.514 | INFO | "gpu" : " 17 GB", | 20260505 12:43:36.514 | INFO | "huge" : "", | 20260505 12:43:36.514 | INFO | "huge_shared" : " 27 GB", | 20260505 12:43:36.514 | INFO | "malloc" : " 14 GB", | 20260505 12:43:36.514 | INFO | "pinned" : " 29 GB" | 20260505 12:43:36.514 | INFO | } | 20260505 12:43:36.514 | INFO | Perf report for the last 5.1323 seconds: | 20260505 12:43:36.514 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:43:36.514 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:36.514 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:36.514 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:36.514 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 12:43:36.514 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:36.514 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:36.514 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 12:43:36.514 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:36.514 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:36.514 | INFO | Uploaded 36GiB to GPU in 2.16907s (16.5969GiB/s) | 20260505 12:43:36.514 | INFO | Downloaded 34GiB to GPU in 2.11117s (16.1048GiB/s) | 20260505 12:43:41.657 | INFO | Analyzing chunk frames = [3300352,3300864) | 20260505 12:43:41.657 | INFO | MemInfo: { | 20260505 12:43:41.657 | INFO | "gpu" : " 17 GB", | 20260505 12:43:41.657 | INFO | "huge" : "", | 20260505 12:43:41.657 | INFO | "huge_shared" : " 27 GB", | 20260505 12:43:41.657 | INFO | "malloc" : " 14 GB", | 20260505 12:43:41.657 | INFO | "pinned" : " 29 GB" | 20260505 12:43:41.657 | INFO | } | 20260505 12:43:41.657 | INFO | Perf report for the last 5.14315 seconds: | 20260505 12:43:41.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:43:41.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:41.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:41.657 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:41.657 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 12:43:41.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:41.657 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:41.657 | INFO | PRE_HQ : 29.7%, 1.0, 15.5ms, 0 | 20260505 12:43:41.657 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:41.657 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:41.657 | INFO | Uploaded 36GiB to GPU in 2.12547s (16.9374GiB/s) | 20260505 12:43:41.657 | INFO | Downloaded 34GiB to GPU in 2.15721s (15.7611GiB/s) | 20260505 12:43:46.799 | INFO | Analyzing chunk frames = [3300864,3301376) | 20260505 12:43:46.799 | INFO | MemInfo: { | 20260505 12:43:46.799 | INFO | "gpu" : " 17 GB", | 20260505 12:43:46.799 | INFO | "huge" : "", | 20260505 12:43:46.799 | INFO | "huge_shared" : " 27 GB", | 20260505 12:43:46.799 | INFO | "malloc" : " 14 GB", | 20260505 12:43:46.799 | INFO | "pinned" : " 29 GB" | 20260505 12:43:46.799 | INFO | } | 20260505 12:43:46.799 | INFO | Perf report for the last 5.14169 seconds: | 20260505 12:43:46.799 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:43:46.799 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:46.799 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:46.799 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:46.799 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 12:43:46.799 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:46.799 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:46.799 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 12:43:46.799 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:46.799 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:46.799 | INFO | Uploaded 36GiB to GPU in 2.14523s (16.7814GiB/s) | 20260505 12:43:46.799 | INFO | Downloaded 34GiB to GPU in 2.1086s (16.1244GiB/s) | 20260505 12:43:46.799 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 12:43:46.799 | INFO | Total mem usage: 15.3 GiB | 20260505 12:43:46.799 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 12:43:46.799 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -6.92%) | 20260505 12:43:46.799 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 12:43:46.799 | INFO | High Water Breakdown: | 20260505 12:43:46.799 | INFO | 61.1% : 8.76 GiB ( 92% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 12:43:46.799 | INFO | 38.8% : 5.56 GiB ( 83.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 12:43:46.799 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 12:43:46.800 | INFO | ------------------------------------------ | 20260505 12:43:46.800 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:43:46.800 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 12:43:46.800 | INFO | Total mem usage: 26.5 GiB | 20260505 12:43:46.800 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 12:43:46.800 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 12:43:46.800 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 12:43:46.800 | INFO | High Water Breakdown: | 20260505 12:43:46.800 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:43:46.800 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:43:46.800 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:43:46.800 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:43:46.800 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:43:46.800 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:43:46.800 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 12:43:46.800 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 12:43:46.800 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:43:46.800 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 12:43:46.800 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 12:43:46.800 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 12:43:46.800 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 12:43:46.800 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 12:43:46.800 | INFO | ------------------------------------------ | 20260505 12:43:46.800 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:43:46.800 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 12:43:46.800 | INFO | Total mem usage: 25.6 GiB | 20260505 12:43:46.800 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 12:43:46.800 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 12:43:46.800 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 12:43:46.800 | INFO | High Water Breakdown: | 20260505 12:43:46.800 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 12:43:46.800 | INFO | ------------------------------------------ | 20260505 12:43:46.800 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:43:46.800 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 12:43:46.800 | INFO | Total mem usage: 15.7 GiB | 20260505 12:43:46.800 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 12:43:46.800 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 12:43:46.800 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 12:43:46.800 | INFO | High Water Breakdown: | 20260505 12:43:46.800 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 12:43:46.800 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 12:43:46.800 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 12:43:46.800 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 12:43:46.800 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 12:43:46.800 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 12:43:46.800 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 12:43:46.800 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 12:43:46.800 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 12:43:46.800 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:43:46.800 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 12:43:46.800 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:43:46.800 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 12:43:46.800 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 12:43:46.800 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 12:43:46.800 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 12:43:46.800 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:43:46.800 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 12:43:46.800 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:43:46.800 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:43:46.800 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:43:46.800 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 12:43:46.800 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 12:43:46.800 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:43:46.800 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 12:43:46.800 | INFO | ------------------------------------------ | 20260505 12:43:46.800 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:43:46.800 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 12:43:46.800 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:43:46.800 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:43:46.800 | INFO | PerfReport: Full Execution -- 3.20e+01 / 7.71e+01, 1.40e+01 / 1.62e+00 | 20260505 12:43:46.800 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 9.90e-02, 1.02e-02 / 3.38e-03 | 20260505 12:43:46.800 | INFO | PerfReport: 11.75%, Baseline -- 3.76e+00 / 4.47e+00, 5.88e-01 / 2.04e-01 | 20260505 12:43:46.800 | INFO | PerfReport: 3.30%, Histogram -- 1.06e+00 / 1.50e+00, 2.03e-01 / 9.24e-02 | 20260505 12:43:46.800 | INFO | PerfReport: 16.04%, DME -- 4.10e+01 / 4.61e+01, 4.73e+00 / 2.41e+00 | 20260505 12:43:46.800 | INFO | PerfReport: 56.21%, FrameLabeling -- 1.80e+01 / 2.01e+01, 1.36e+00 / 1.84e+00 | 20260505 12:43:46.800 | INFO | PerfReport: 7.48%, PulseAccumulating -- 2.39e+00 / 2.93e+00, 4.49e-01 / 1.76e-01 | 20260505 12:43:46.800 | INFO | PerfReport: 3.65%, Metrics -- 1.17e+00 / 1.48e+00, 3.30e-01 / 1.49e-01 | 20260505 12:43:46.800 | INFO | PerfReport: 1.28%, PulseSerializing -- 4.09e-01 / 4.61e-01, 7.43e-02 / 3.78e-02 | 20260505 12:43:46.801 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 12:43:46.801 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:43:46.801 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:43:46.801 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260505 12:43:46.801 | INFO | PerfReport: 22.46%, Upload -- 2.27e+01 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260505 12:43:46.801 | INFO | PerfReport: 22.07%, Download -- 2.23e+01 / 0.00e+00, 8.13e+00 / 0.00e+00 | 20260505 12:43:46.801 | INFO | PerfReport: 31.65%, Compute -- 3.20e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260505 12:43:46.801 | INFO | PerfReport: 2.18%, LockUpload -- 2.20e+00 / 0.00e+00, 7.66e+00 / 0.00e+00 | 20260505 12:43:46.801 | INFO | PerfReport: 1.41%, LockDownload -- 1.42e+00 / 0.00e+00, 3.04e+00 / 0.00e+00 | 20260505 12:43:46.801 | INFO | PerfReport: 20.22%, LockCompute -- 2.04e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260505 12:43:46.801 | INFO | PerfReport: 0.01%, Misc -- 1.47e-02 / 0.00e+00, 2.67e-02 / 0.00e+00 | 20260505 12:43:46.933 | INFO | Starting a round of Baz IO | 20260505 12:43:50.164 | INFO | Populating RTMetrics report | 20260505 12:43:50.165 | INFO | RTMetrics report populated | 20260505 12:43:50.165 | INFO | Opening tmp RTMetrics json file | 20260505 12:43:50.174 | INFO | Serializing RTMetrics to json | 20260505 12:43:50.198 | INFO | Serialization done, swapping tmp file now... | 20260505 12:43:50.223 | INFO | RTMetrics done | 20260505 12:43:50.306 | INFO | Finished a round of Baz IO: Wrote 2.342 GiB in 3.372 seconds | 20260505 12:43:50.306 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.44% / 0.01% / 76.56% | 20260505 12:43:50.306 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.833698% / 7.94729e-06% / 7.54992e-05% | 20260505 12:43:50.306 | INFO | Baz IO overall: | 20260505 12:43:50.306 | INFO | BytesWritten : 1963416927310 | 20260505 12:43:50.306 | INFO | NumEvents : 934463179461 | 20260505 12:43:51.946 | INFO | Analyzing chunk frames = [3301376,3301888) | 20260505 12:43:51.946 | INFO | MemInfo: { | 20260505 12:43:51.946 | INFO | "gpu" : " 17 GB", | 20260505 12:43:51.946 | INFO | "huge" : "", | 20260505 12:43:51.946 | INFO | "huge_shared" : " 27 GB", | 20260505 12:43:51.946 | INFO | "malloc" : " 14 GB", | 20260505 12:43:51.946 | INFO | "pinned" : " 29 GB" | 20260505 12:43:51.946 | INFO | } | 20260505 12:43:51.946 | INFO | Perf report for the last 5.1468 seconds: | 20260505 12:43:51.946 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:43:51.946 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:51.946 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:51.946 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:51.946 | INFO | ANALYSIS : 64.3%, 2.9, 99.1ms, 0 | 20260505 12:43:51.946 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:51.946 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260505 12:43:51.946 | INFO | PRE_HQ : 61.0%, 1.8, 58.0ms, 0 | 20260505 12:43:51.946 | INFO | BAZWRITER : 64.8%, 4.1, 136.9ms, 0 | 20260505 12:43:51.946 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260505 12:43:51.946 | INFO | Uploaded 36GiB to GPU in 2.2024s (16.3458GiB/s) | 20260505 12:43:51.946 | INFO | Downloaded 38GiB to GPU in 2.27905s (16.6736GiB/s) | 20260505 12:43:57.080 | INFO | Analyzing chunk frames = [3301888,3302400) | 20260505 12:43:57.080 | INFO | MemInfo: { | 20260505 12:43:57.080 | INFO | "gpu" : " 17 GB", | 20260505 12:43:57.080 | INFO | "huge" : "", | 20260505 12:43:57.080 | INFO | "huge_shared" : " 27 GB", | 20260505 12:43:57.080 | INFO | "malloc" : " 14 GB", | 20260505 12:43:57.080 | INFO | "pinned" : " 29 GB" | 20260505 12:43:57.080 | INFO | } | 20260505 12:43:57.080 | INFO | Perf report for the last 5.13411 seconds: | 20260505 12:43:57.080 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:43:57.080 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:57.080 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:57.080 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:57.080 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 12:43:57.080 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:57.080 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:57.080 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 12:43:57.080 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:57.080 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:43:57.080 | INFO | Uploaded 36GiB to GPU in 2.19166s (16.4259GiB/s) | 20260505 12:43:57.080 | INFO | Downloaded 34GiB to GPU in 2.03167s (16.735GiB/s) | 20260505 12:44:02.215 | INFO | Analyzing chunk frames = [3302400,3302912) | 20260505 12:44:02.215 | INFO | MemInfo: { | 20260505 12:44:02.215 | INFO | "gpu" : " 17 GB", | 20260505 12:44:02.215 | INFO | "huge" : "", | 20260505 12:44:02.215 | INFO | "huge_shared" : " 27 GB", | 20260505 12:44:02.215 | INFO | "malloc" : " 14 GB", | 20260505 12:44:02.215 | INFO | "pinned" : " 29 GB" | 20260505 12:44:02.215 | INFO | } | 20260505 12:44:02.215 | INFO | Perf report for the last 5.13526 seconds: | 20260505 12:44:02.215 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:44:02.215 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:02.215 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:02.215 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:02.215 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 12:44:02.215 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:02.215 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:02.215 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 12:44:02.215 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:02.215 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:02.215 | INFO | Uploaded 36GiB to GPU in 2.16394s (16.6363GiB/s) | 20260505 12:44:02.216 | INFO | Downloaded 34GiB to GPU in 2.13238s (15.9446GiB/s) | 20260505 12:44:07.364 | INFO | Analyzing chunk frames = [3302912,3303424) | 20260505 12:44:07.364 | INFO | MemInfo: { | 20260505 12:44:07.364 | INFO | "gpu" : " 17 GB", | 20260505 12:44:07.364 | INFO | "huge" : "", | 20260505 12:44:07.364 | INFO | "huge_shared" : " 27 GB", | 20260505 12:44:07.364 | INFO | "malloc" : " 14 GB", | 20260505 12:44:07.364 | INFO | "pinned" : " 29 GB" | 20260505 12:44:07.364 | INFO | } | 20260505 12:44:07.364 | INFO | Perf report for the last 5.14895 seconds: | 20260505 12:44:07.364 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:44:07.364 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:07.364 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:07.364 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:07.364 | INFO | ANALYSIS : 63.2%, 3.0, 98.4ms, 0 | 20260505 12:44:07.364 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:07.364 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:07.364 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 12:44:07.364 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:07.364 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:07.364 | INFO | Uploaded 36GiB to GPU in 2.2124s (16.2719GiB/s) | 20260505 12:44:07.364 | INFO | Downloaded 34GiB to GPU in 2.1304s (15.9595GiB/s) | 20260505 12:44:12.499 | INFO | Analyzing chunk frames = [3303424,3303936) | 20260505 12:44:12.499 | INFO | MemInfo: { | 20260505 12:44:12.499 | INFO | "gpu" : " 17 GB", | 20260505 12:44:12.499 | INFO | "huge" : "", | 20260505 12:44:12.499 | INFO | "huge_shared" : " 27 GB", | 20260505 12:44:12.499 | INFO | "malloc" : " 14 GB", | 20260505 12:44:12.499 | INFO | "pinned" : " 29 GB" | 20260505 12:44:12.499 | INFO | } | 20260505 12:44:12.499 | INFO | Perf report for the last 5.13493 seconds: | 20260505 12:44:12.499 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:44:12.499 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:12.499 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:12.499 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:12.499 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 12:44:12.499 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:12.499 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:12.499 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 12:44:12.499 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:12.499 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:12.499 | INFO | Uploaded 36GiB to GPU in 2.14187s (16.8077GiB/s) | 20260505 12:44:12.499 | INFO | Downloaded 34GiB to GPU in 2.12032s (16.0353GiB/s) | 20260505 12:44:17.645 | INFO | Analyzing chunk frames = [3303936,3304448) | 20260505 12:44:17.645 | INFO | MemInfo: { | 20260505 12:44:17.645 | INFO | "gpu" : " 17 GB", | 20260505 12:44:17.645 | INFO | "huge" : "", | 20260505 12:44:17.645 | INFO | "huge_shared" : " 27 GB", | 20260505 12:44:17.645 | INFO | "malloc" : " 14 GB", | 20260505 12:44:17.645 | INFO | "pinned" : " 29 GB" | 20260505 12:44:17.645 | INFO | } | 20260505 12:44:17.645 | INFO | Perf report for the last 5.14629 seconds: | 20260505 12:44:17.645 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:44:17.645 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:17.646 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:17.646 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:17.646 | INFO | ANALYSIS : 63.2%, 3.0, 98.8ms, 0 | 20260505 12:44:17.646 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:17.646 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:17.646 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:44:17.646 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:17.646 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:17.646 | INFO | Uploaded 36GiB to GPU in 2.14634s (16.7727GiB/s) | 20260505 12:44:17.646 | INFO | Downloaded 34GiB to GPU in 2.10999s (16.1138GiB/s) | 20260505 12:44:22.774 | INFO | Analyzing chunk frames = [3304448,3304960) | 20260505 12:44:22.774 | INFO | MemInfo: { | 20260505 12:44:22.774 | INFO | "gpu" : " 17 GB", | 20260505 12:44:22.774 | INFO | "huge" : "", | 20260505 12:44:22.774 | INFO | "huge_shared" : " 27 GB", | 20260505 12:44:22.774 | INFO | "malloc" : " 14 GB", | 20260505 12:44:22.774 | INFO | "pinned" : " 29 GB" | 20260505 12:44:22.774 | INFO | } | 20260505 12:44:22.774 | INFO | Perf report for the last 5.12888 seconds: | 20260505 12:44:22.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:44:22.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:22.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:22.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:22.774 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 12:44:22.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:22.774 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:22.774 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 12:44:22.774 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:22.774 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:22.775 | INFO | Uploaded 36GiB to GPU in 2.13701s (16.846GiB/s) | 20260505 12:44:22.775 | INFO | Downloaded 34GiB to GPU in 2.18158s (15.585GiB/s) | 20260505 12:44:27.916 | INFO | Analyzing chunk frames = [3304960,3305472) | 20260505 12:44:27.916 | INFO | MemInfo: { | 20260505 12:44:27.916 | INFO | "gpu" : " 17 GB", | 20260505 12:44:27.916 | INFO | "huge" : "", | 20260505 12:44:27.916 | INFO | "huge_shared" : " 27 GB", | 20260505 12:44:27.916 | INFO | "malloc" : " 14 GB", | 20260505 12:44:27.916 | INFO | "pinned" : " 29 GB" | 20260505 12:44:27.916 | INFO | } | 20260505 12:44:27.916 | INFO | Perf report for the last 5.14185 seconds: | 20260505 12:44:27.916 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:44:27.916 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:27.916 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:27.916 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:27.916 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 12:44:27.916 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:27.916 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:27.916 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 12:44:27.916 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:27.916 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:27.916 | INFO | Uploaded 36GiB to GPU in 2.22238s (16.1989GiB/s) | 20260505 12:44:27.917 | INFO | Downloaded 34GiB to GPU in 2.1141s (16.0825GiB/s) | 20260505 12:44:28.051 | INFO | Starting a round of Baz IO | 20260505 12:44:31.285 | INFO | Populating RTMetrics report | 20260505 12:44:31.285 | INFO | RTMetrics report populated | 20260505 12:44:31.285 | INFO | Opening tmp RTMetrics json file | 20260505 12:44:31.307 | INFO | Serializing RTMetrics to json | 20260505 12:44:31.310 | INFO | Serialization done, swapping tmp file now... | 20260505 12:44:31.314 | INFO | RTMetrics done | 20260505 12:44:31.395 | INFO | Finished a round of Baz IO: Wrote 2.335 GiB in 3.343 seconds | 20260505 12:44:31.395 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.43% / 0.01% / 76.56% | 20260505 12:44:31.395 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.830857% / 1.19209e-05% / 5.96046e-05% | 20260505 12:44:31.395 | INFO | Baz IO overall: | 20260505 12:44:31.395 | INFO | BytesWritten : 1965924592756 | 20260505 12:44:31.395 | INFO | NumEvents : 935649806325 | 20260505 12:44:33.057 | INFO | Analyzing chunk frames = [3305472,3305984) | 20260505 12:44:33.057 | INFO | MemInfo: { | 20260505 12:44:33.057 | INFO | "gpu" : " 17 GB", | 20260505 12:44:33.057 | INFO | "huge" : "", | 20260505 12:44:33.057 | INFO | "huge_shared" : " 27 GB", | 20260505 12:44:33.057 | INFO | "malloc" : " 14 GB", | 20260505 12:44:33.057 | INFO | "pinned" : " 29 GB" | 20260505 12:44:33.057 | INFO | } | 20260505 12:44:33.057 | INFO | Perf report for the last 5.14107 seconds: | 20260505 12:44:33.057 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:44:33.057 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:33.057 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:33.057 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:33.057 | INFO | ANALYSIS : 64.4%, 3.0, 99.9ms, 0 | 20260505 12:44:33.057 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:33.057 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260505 12:44:33.057 | INFO | PRE_HQ : 61.1%, 1.8, 58.3ms, 0 | 20260505 12:44:33.057 | INFO | BAZWRITER : 64.3%, 4.1, 134.8ms, 0 | 20260505 12:44:33.057 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 12:44:33.057 | INFO | Uploaded 36GiB to GPU in 2.18266s (16.4937GiB/s) | 20260505 12:44:33.057 | INFO | Downloaded 38GiB to GPU in 2.27703s (16.6884GiB/s) | 20260505 12:44:38.193 | INFO | Analyzing chunk frames = [3305984,3306496) | 20260505 12:44:38.193 | INFO | MemInfo: { | 20260505 12:44:38.193 | INFO | "gpu" : " 17 GB", | 20260505 12:44:38.193 | INFO | "huge" : "", | 20260505 12:44:38.193 | INFO | "huge_shared" : " 27 GB", | 20260505 12:44:38.193 | INFO | "malloc" : " 14 GB", | 20260505 12:44:38.193 | INFO | "pinned" : " 29 GB" | 20260505 12:44:38.193 | INFO | } | 20260505 12:44:38.193 | INFO | Perf report for the last 5.13606 seconds: | 20260505 12:44:38.193 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:44:38.193 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:38.193 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:38.193 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:38.193 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 12:44:38.193 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:38.193 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:38.193 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:44:38.193 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:38.193 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:38.193 | INFO | Uploaded 36GiB to GPU in 2.19133s (16.4284GiB/s) | 20260505 12:44:38.194 | INFO | Downloaded 34GiB to GPU in 2.03214s (16.7311GiB/s) | 20260505 12:44:43.331 | INFO | Analyzing chunk frames = [3306496,3307008) | 20260505 12:44:43.331 | INFO | MemInfo: { | 20260505 12:44:43.331 | INFO | "gpu" : " 17 GB", | 20260505 12:44:43.331 | INFO | "huge" : "", | 20260505 12:44:43.331 | INFO | "huge_shared" : " 27 GB", | 20260505 12:44:43.331 | INFO | "malloc" : " 14 GB", | 20260505 12:44:43.331 | INFO | "pinned" : " 29 GB" | 20260505 12:44:43.331 | INFO | } | 20260505 12:44:43.331 | INFO | Perf report for the last 5.13787 seconds: | 20260505 12:44:43.331 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:44:43.331 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:43.331 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:43.331 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:43.331 | INFO | ANALYSIS : 63.4%, 3.0, 98.6ms, 0 | 20260505 12:44:43.331 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:43.331 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:43.331 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:44:43.331 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:43.331 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:43.331 | INFO | Uploaded 36GiB to GPU in 2.19531s (16.3986GiB/s) | 20260505 12:44:43.332 | INFO | Downloaded 34GiB to GPU in 2.11996s (16.038GiB/s) | 20260505 12:44:48.466 | INFO | Analyzing chunk frames = [3307008,3307520) | 20260505 12:44:48.466 | INFO | MemInfo: { | 20260505 12:44:48.466 | INFO | "gpu" : " 17 GB", | 20260505 12:44:48.466 | INFO | "huge" : "", | 20260505 12:44:48.466 | INFO | "huge_shared" : " 27 GB", | 20260505 12:44:48.466 | INFO | "malloc" : " 14 GB", | 20260505 12:44:48.466 | INFO | "pinned" : " 29 GB" | 20260505 12:44:48.466 | INFO | } | 20260505 12:44:48.466 | INFO | Perf report for the last 5.13456 seconds: | 20260505 12:44:48.466 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:44:48.466 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:48.466 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:48.466 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:48.466 | INFO | ANALYSIS : 63.4%, 3.0, 98.4ms, 0 | 20260505 12:44:48.466 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:48.466 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:48.466 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 12:44:48.466 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:48.466 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:48.466 | INFO | Uploaded 36GiB to GPU in 2.21174s (16.2768GiB/s) | 20260505 12:44:48.466 | INFO | Downloaded 34GiB to GPU in 2.1318s (15.9489GiB/s) | 20260505 12:44:53.613 | INFO | Analyzing chunk frames = [3307520,3308032) | 20260505 12:44:53.613 | INFO | MemInfo: { | 20260505 12:44:53.613 | INFO | "gpu" : " 17 GB", | 20260505 12:44:53.613 | INFO | "huge" : "", | 20260505 12:44:53.613 | INFO | "huge_shared" : " 27 GB", | 20260505 12:44:53.613 | INFO | "malloc" : " 14 GB", | 20260505 12:44:53.613 | INFO | "pinned" : " 29 GB" | 20260505 12:44:53.613 | INFO | } | 20260505 12:44:53.614 | INFO | Perf report for the last 5.14792 seconds: | 20260505 12:44:53.614 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:44:53.614 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:53.614 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:53.614 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:53.614 | INFO | ANALYSIS : 62.9%, 3.0, 98.4ms, 0 | 20260505 12:44:53.614 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:53.614 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:53.614 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 12:44:53.614 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:53.614 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:53.614 | INFO | Uploaded 36GiB to GPU in 2.18749s (16.4572GiB/s) | 20260505 12:44:53.614 | INFO | Downloaded 34GiB to GPU in 2.12992s (15.9631GiB/s) | 20260505 12:44:58.757 | INFO | Analyzing chunk frames = [3308032,3308544) | 20260505 12:44:58.757 | INFO | MemInfo: { | 20260505 12:44:58.757 | INFO | "gpu" : " 17 GB", | 20260505 12:44:58.757 | INFO | "huge" : "", | 20260505 12:44:58.757 | INFO | "huge_shared" : " 27 GB", | 20260505 12:44:58.757 | INFO | "malloc" : " 14 GB", | 20260505 12:44:58.757 | INFO | "pinned" : " 29 GB" | 20260505 12:44:58.757 | INFO | } | 20260505 12:44:58.757 | INFO | Perf report for the last 5.14319 seconds: | 20260505 12:44:58.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:44:58.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:58.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:58.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:58.757 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 12:44:58.757 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:58.757 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:58.757 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 12:44:58.757 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:58.757 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:44:58.757 | INFO | Uploaded 36GiB to GPU in 2.13271s (16.8799GiB/s) | 20260505 12:44:58.757 | INFO | Downloaded 34GiB to GPU in 2.10133s (16.1802GiB/s) | 20260505 12:45:03.889 | INFO | Analyzing chunk frames = [3308544,3309056) | 20260505 12:45:03.889 | INFO | MemInfo: { | 20260505 12:45:03.889 | INFO | "gpu" : " 17 GB", | 20260505 12:45:03.889 | INFO | "huge" : "", | 20260505 12:45:03.889 | INFO | "huge_shared" : " 27 GB", | 20260505 12:45:03.889 | INFO | "malloc" : " 14 GB", | 20260505 12:45:03.889 | INFO | "pinned" : " 29 GB" | 20260505 12:45:03.889 | INFO | } | 20260505 12:45:03.889 | INFO | Perf report for the last 5.13247 seconds: | 20260505 12:45:03.889 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:45:03.889 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:03.889 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:03.889 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:03.889 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 12:45:03.890 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:03.890 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:03.890 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 12:45:03.890 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:03.890 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:03.890 | INFO | Uploaded 36GiB to GPU in 2.13003s (16.9012GiB/s) | 20260505 12:45:03.890 | INFO | Downloaded 34GiB to GPU in 2.17134s (15.6585GiB/s) | 20260505 12:45:09.023 | INFO | Analyzing chunk frames = [3309056,3309568) | 20260505 12:45:09.023 | INFO | MemInfo: { | 20260505 12:45:09.023 | INFO | "gpu" : " 17 GB", | 20260505 12:45:09.023 | INFO | "huge" : "", | 20260505 12:45:09.023 | INFO | "huge_shared" : " 27 GB", | 20260505 12:45:09.023 | INFO | "malloc" : " 14 GB", | 20260505 12:45:09.023 | INFO | "pinned" : " 29 GB" | 20260505 12:45:09.023 | INFO | } | 20260505 12:45:09.023 | INFO | Perf report for the last 5.13369 seconds: | 20260505 12:45:09.023 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:45:09.023 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:09.023 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:09.023 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:09.023 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 12:45:09.023 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:09.023 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:09.023 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 12:45:09.023 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:09.023 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:09.023 | INFO | Uploaded 36GiB to GPU in 2.12099s (16.9732GiB/s) | 20260505 12:45:09.023 | INFO | Downloaded 34GiB to GPU in 2.10561s (16.1473GiB/s) | 20260505 12:45:09.023 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 12:45:09.023 | INFO | Total mem usage: 15.3 GiB | 20260505 12:45:09.023 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 12:45:09.023 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -6.9%) | 20260505 12:45:09.023 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 12:45:09.024 | INFO | High Water Breakdown: | 20260505 12:45:09.024 | INFO | 61.1% : 8.76 GiB ( 92.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 12:45:09.024 | INFO | 38.8% : 5.56 GiB ( 83.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 12:45:09.024 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 12:45:09.024 | INFO | ------------------------------------------ | 20260505 12:45:09.024 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:45:09.024 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 12:45:09.024 | INFO | Total mem usage: 26.5 GiB | 20260505 12:45:09.024 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 12:45:09.024 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 12:45:09.024 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 12:45:09.024 | INFO | High Water Breakdown: | 20260505 12:45:09.024 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:45:09.024 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:45:09.024 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:45:09.024 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:45:09.024 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:45:09.024 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:45:09.024 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 12:45:09.024 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 12:45:09.024 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:45:09.024 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 12:45:09.024 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 12:45:09.024 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 12:45:09.024 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 12:45:09.024 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 12:45:09.024 | INFO | ------------------------------------------ | 20260505 12:45:09.024 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:45:09.024 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 12:45:09.024 | INFO | Total mem usage: 25.6 GiB | 20260505 12:45:09.024 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 12:45:09.024 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 12:45:09.024 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 12:45:09.024 | INFO | High Water Breakdown: | 20260505 12:45:09.024 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 12:45:09.024 | INFO | ------------------------------------------ | 20260505 12:45:09.024 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:45:09.024 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 12:45:09.024 | INFO | Total mem usage: 15.7 GiB | 20260505 12:45:09.024 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 12:45:09.024 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 12:45:09.024 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 12:45:09.024 | INFO | High Water Breakdown: | 20260505 12:45:09.024 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 12:45:09.024 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 12:45:09.024 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 12:45:09.024 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 12:45:09.024 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 12:45:09.024 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 12:45:09.024 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 12:45:09.024 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 12:45:09.024 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 12:45:09.024 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:45:09.024 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 12:45:09.024 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:45:09.024 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 12:45:09.024 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 12:45:09.024 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 12:45:09.024 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 12:45:09.024 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:45:09.024 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 12:45:09.024 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:45:09.024 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:45:09.024 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:45:09.024 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 12:45:09.024 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 12:45:09.024 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:45:09.024 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 12:45:09.024 | INFO | ------------------------------------------ | 20260505 12:45:09.024 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:45:09.024 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 12:45:09.024 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:45:09.024 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:45:09.024 | INFO | PerfReport: Full Execution -- 3.20e+01 / 7.71e+01, 1.37e+01 / 1.77e+00 | 20260505 12:45:09.024 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 9.98e-02, 1.06e-02 / 4.85e-03 | 20260505 12:45:09.024 | INFO | PerfReport: 11.77%, Baseline -- 3.77e+00 / 4.53e+00, 5.98e-01 / 1.25e-01 | 20260505 12:45:09.024 | INFO | PerfReport: 3.31%, Histogram -- 1.06e+00 / 1.51e+00, 2.02e-01 / 9.58e-02 | 20260505 12:45:09.025 | INFO | PerfReport: 15.96%, DME -- 4.08e+01 / 4.60e+01, 4.50e+00 / 2.03e+00 | 20260505 12:45:09.025 | INFO | PerfReport: 56.33%, FrameLabeling -- 1.80e+01 / 2.01e+01, 1.46e+00 / 1.46e+00 | 20260505 12:45:09.025 | INFO | PerfReport: 7.43%, PulseAccumulating -- 2.38e+00 / 2.95e+00, 4.46e-01 / 1.44e-01 | 20260505 12:45:09.025 | INFO | PerfReport: 3.63%, Metrics -- 1.16e+00 / 1.48e+00, 3.27e-01 / 1.50e-01 | 20260505 12:45:09.025 | INFO | PerfReport: 1.27%, PulseSerializing -- 4.07e-01 / 4.57e-01, 7.36e-02 / 3.56e-02 | 20260505 12:45:09.025 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 12:45:09.025 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:45:09.025 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:45:09.025 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260505 12:45:09.025 | INFO | PerfReport: 22.43%, Upload -- 2.26e+01 / 0.00e+00, 7.88e+00 / 0.00e+00 | 20260505 12:45:09.025 | INFO | PerfReport: 22.04%, Download -- 2.22e+01 / 0.00e+00, 8.11e+00 / 0.00e+00 | 20260505 12:45:09.025 | INFO | PerfReport: 31.71%, Compute -- 3.20e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 12:45:09.025 | INFO | PerfReport: 2.10%, LockUpload -- 2.12e+00 / 0.00e+00, 7.84e+00 / 0.00e+00 | 20260505 12:45:09.025 | INFO | PerfReport: 1.46%, LockDownload -- 1.47e+00 / 0.00e+00, 3.06e+00 / 0.00e+00 | 20260505 12:45:09.025 | INFO | PerfReport: 20.25%, LockCompute -- 2.04e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260505 12:45:09.025 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.92e-02 / 0.00e+00 | 20260505 12:45:09.157 | INFO | Starting a round of Baz IO | 20260505 12:45:12.405 | INFO | Populating RTMetrics report | 20260505 12:45:12.405 | INFO | RTMetrics report populated | 20260505 12:45:12.405 | INFO | Opening tmp RTMetrics json file | 20260505 12:45:12.428 | INFO | Serializing RTMetrics to json | 20260505 12:45:12.450 | INFO | Serialization done, swapping tmp file now... | 20260505 12:45:12.469 | INFO | RTMetrics done | 20260505 12:45:12.513 | INFO | Finished a round of Baz IO: Wrote 2.333 GiB in 3.355 seconds | 20260505 12:45:12.513 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.43% / 0.01% / 76.57% | 20260505 12:45:12.513 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.831155% / 0% / 6.35783e-05% | 20260505 12:45:12.513 | INFO | Baz IO overall: | 20260505 12:45:12.513 | INFO | BytesWritten : 1968429872731 | 20260505 12:45:12.513 | INFO | NumEvents : 936836464330 | 20260505 12:45:14.167 | INFO | Analyzing chunk frames = [3309568,3310080) | 20260505 12:45:14.167 | INFO | MemInfo: { | 20260505 12:45:14.167 | INFO | "gpu" : " 17 GB", | 20260505 12:45:14.167 | INFO | "huge" : "", | 20260505 12:45:14.167 | INFO | "huge_shared" : " 27 GB", | 20260505 12:45:14.167 | INFO | "malloc" : " 14 GB", | 20260505 12:45:14.167 | INFO | "pinned" : " 29 GB" | 20260505 12:45:14.167 | INFO | } | 20260505 12:45:14.167 | INFO | Perf report for the last 5.14385 seconds: | 20260505 12:45:14.167 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:45:14.167 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:14.167 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:14.167 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:14.167 | INFO | ANALYSIS : 64.7%, 3.0, 100.3ms, 0 | 20260505 12:45:14.167 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:14.167 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260505 12:45:14.167 | INFO | PRE_HQ : 61.0%, 1.8, 58.2ms, 0 | 20260505 12:45:14.167 | INFO | BAZWRITER : 64.5%, 4.0, 134.3ms, 0 | 20260505 12:45:14.167 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 12:45:14.167 | INFO | Uploaded 36GiB to GPU in 2.20059s (16.3593GiB/s) | 20260505 12:45:14.167 | INFO | Downloaded 38GiB to GPU in 2.29136s (16.584GiB/s) | 20260505 12:45:19.307 | INFO | Analyzing chunk frames = [3310080,3310592) | 20260505 12:45:19.307 | INFO | MemInfo: { | 20260505 12:45:19.307 | INFO | "gpu" : " 17 GB", | 20260505 12:45:19.307 | INFO | "huge" : "", | 20260505 12:45:19.307 | INFO | "huge_shared" : " 27 GB", | 20260505 12:45:19.307 | INFO | "malloc" : " 14 GB", | 20260505 12:45:19.307 | INFO | "pinned" : " 29 GB" | 20260505 12:45:19.307 | INFO | } | 20260505 12:45:19.307 | INFO | Perf report for the last 5.14032 seconds: | 20260505 12:45:19.307 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:45:19.307 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:19.307 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:19.307 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:19.307 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 12:45:19.307 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:19.307 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:19.307 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:45:19.307 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:19.307 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:19.307 | INFO | Uploaded 36GiB to GPU in 2.15983s (16.668GiB/s) | 20260505 12:45:19.308 | INFO | Downloaded 34GiB to GPU in 2.02546s (16.7863GiB/s) | 20260505 12:45:24.449 | INFO | Analyzing chunk frames = [3310592,3311104) | 20260505 12:45:24.449 | INFO | MemInfo: { | 20260505 12:45:24.449 | INFO | "gpu" : " 17 GB", | 20260505 12:45:24.449 | INFO | "huge" : "", | 20260505 12:45:24.449 | INFO | "huge_shared" : " 27 GB", | 20260505 12:45:24.449 | INFO | "malloc" : " 14 GB", | 20260505 12:45:24.449 | INFO | "pinned" : " 29 GB" | 20260505 12:45:24.449 | INFO | } | 20260505 12:45:24.450 | INFO | Perf report for the last 5.14236 seconds: | 20260505 12:45:24.450 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:45:24.450 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:24.450 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:24.450 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:24.450 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 12:45:24.450 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:24.450 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:24.450 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 12:45:24.450 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:24.450 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:24.450 | INFO | Uploaded 36GiB to GPU in 2.20034s (16.3611GiB/s) | 20260505 12:45:24.450 | INFO | Downloaded 34GiB to GPU in 2.11948s (16.0417GiB/s) | 20260505 12:45:29.591 | INFO | Analyzing chunk frames = [3311104,3311616) | 20260505 12:45:29.591 | INFO | MemInfo: { | 20260505 12:45:29.591 | INFO | "gpu" : " 17 GB", | 20260505 12:45:29.591 | INFO | "huge" : "", | 20260505 12:45:29.591 | INFO | "huge_shared" : " 27 GB", | 20260505 12:45:29.591 | INFO | "malloc" : " 14 GB", | 20260505 12:45:29.591 | INFO | "pinned" : " 29 GB" | 20260505 12:45:29.591 | INFO | } | 20260505 12:45:29.591 | INFO | Perf report for the last 5.14156 seconds: | 20260505 12:45:29.591 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:45:29.591 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:29.591 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:29.591 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:29.591 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 12:45:29.591 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:29.591 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:29.591 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:45:29.591 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:29.591 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:29.591 | INFO | Uploaded 36GiB to GPU in 2.20841s (16.3013GiB/s) | 20260505 12:45:29.591 | INFO | Downloaded 34GiB to GPU in 2.12818s (15.9761GiB/s) | 20260505 12:45:34.723 | INFO | Analyzing chunk frames = [3311616,3312128) | 20260505 12:45:34.723 | INFO | MemInfo: { | 20260505 12:45:34.723 | INFO | "gpu" : " 17 GB", | 20260505 12:45:34.723 | INFO | "huge" : "", | 20260505 12:45:34.723 | INFO | "huge_shared" : " 27 GB", | 20260505 12:45:34.723 | INFO | "malloc" : " 14 GB", | 20260505 12:45:34.723 | INFO | "pinned" : " 29 GB" | 20260505 12:45:34.723 | INFO | } | 20260505 12:45:34.723 | INFO | Perf report for the last 5.13218 seconds: | 20260505 12:45:34.723 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:45:34.723 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:34.723 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:34.723 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:34.723 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 12:45:34.724 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:34.724 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:34.724 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 12:45:34.724 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:34.724 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:34.724 | INFO | Uploaded 36GiB to GPU in 2.16022s (16.6649GiB/s) | 20260505 12:45:34.724 | INFO | Downloaded 34GiB to GPU in 2.12753s (15.9809GiB/s) | 20260505 12:45:39.870 | INFO | Analyzing chunk frames = [3312128,3312640) | 20260505 12:45:39.870 | INFO | MemInfo: { | 20260505 12:45:39.870 | INFO | "gpu" : " 17 GB", | 20260505 12:45:39.870 | INFO | "huge" : "", | 20260505 12:45:39.870 | INFO | "huge_shared" : " 27 GB", | 20260505 12:45:39.870 | INFO | "malloc" : " 14 GB", | 20260505 12:45:39.870 | INFO | "pinned" : " 29 GB" | 20260505 12:45:39.870 | INFO | } | 20260505 12:45:39.870 | INFO | Perf report for the last 5.14644 seconds: | 20260505 12:45:39.870 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:45:39.870 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:39.870 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:39.870 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:39.870 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 12:45:39.870 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:39.870 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:39.870 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 12:45:39.870 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:39.870 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:39.870 | INFO | Uploaded 36GiB to GPU in 2.17778s (16.5306GiB/s) | 20260505 12:45:39.870 | INFO | Downloaded 34GiB to GPU in 2.09478s (16.2308GiB/s) | 20260505 12:45:45.011 | INFO | Analyzing chunk frames = [3312640,3313152) | 20260505 12:45:45.011 | INFO | MemInfo: { | 20260505 12:45:45.011 | INFO | "gpu" : " 17 GB", | 20260505 12:45:45.011 | INFO | "huge" : "", | 20260505 12:45:45.011 | INFO | "huge_shared" : " 27 GB", | 20260505 12:45:45.011 | INFO | "malloc" : " 14 GB", | 20260505 12:45:45.011 | INFO | "pinned" : " 29 GB" | 20260505 12:45:45.011 | INFO | } | 20260505 12:45:45.011 | INFO | Perf report for the last 5.14169 seconds: | 20260505 12:45:45.011 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:45:45.011 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:45.011 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:45.012 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:45.012 | INFO | ANALYSIS : 63.9%, 3.0, 99.9ms, 0 | 20260505 12:45:45.012 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:45.012 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:45.012 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 12:45:45.012 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:45.012 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:45.012 | INFO | Uploaded 36GiB to GPU in 2.14483s (16.7845GiB/s) | 20260505 12:45:45.012 | INFO | Downloaded 34GiB to GPU in 2.18088s (15.5901GiB/s) | 20260505 12:45:50.140 | INFO | Analyzing chunk frames = [3313152,3313664) | 20260505 12:45:50.140 | INFO | MemInfo: { | 20260505 12:45:50.140 | INFO | "gpu" : " 17 GB", | 20260505 12:45:50.140 | INFO | "huge" : "", | 20260505 12:45:50.140 | INFO | "huge_shared" : " 27 GB", | 20260505 12:45:50.141 | INFO | "malloc" : " 14 GB", | 20260505 12:45:50.141 | INFO | "pinned" : " 29 GB" | 20260505 12:45:50.141 | INFO | } | 20260505 12:45:50.141 | INFO | Perf report for the last 5.12917 seconds: | 20260505 12:45:50.141 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:45:50.141 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:50.141 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:50.141 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:50.141 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 12:45:50.141 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:50.141 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:50.141 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:45:50.141 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:50.141 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:50.141 | INFO | Uploaded 36GiB to GPU in 2.15843s (16.6788GiB/s) | 20260505 12:45:50.141 | INFO | Downloaded 34GiB to GPU in 2.1066s (16.1397GiB/s) | 20260505 12:45:50.276 | INFO | Starting a round of Baz IO | 20260505 12:45:53.522 | INFO | Populating RTMetrics report | 20260505 12:45:53.523 | INFO | RTMetrics report populated | 20260505 12:45:53.523 | INFO | Opening tmp RTMetrics json file | 20260505 12:45:53.560 | INFO | Serializing RTMetrics to json | 20260505 12:45:53.591 | INFO | Serialization done, swapping tmp file now... | 20260505 12:45:53.613 | INFO | RTMetrics done | 20260505 12:45:53.627 | INFO | Finished a round of Baz IO: Wrote 2.331 GiB in 3.351 seconds | 20260505 12:45:53.627 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.42% / 0.01% / 76.57% | 20260505 12:45:53.628 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.82496% / 3.97364e-06% / 6.35783e-05% | 20260505 12:45:53.628 | INFO | Baz IO overall: | 20260505 12:45:53.628 | INFO | BytesWritten : 1970932852987 | 20260505 12:45:53.628 | INFO | NumEvents : 938022437474 | 20260505 12:45:55.282 | INFO | Analyzing chunk frames = [3313664,3314176) | 20260505 12:45:55.282 | INFO | MemInfo: { | 20260505 12:45:55.282 | INFO | "gpu" : " 17 GB", | 20260505 12:45:55.282 | INFO | "huge" : "", | 20260505 12:45:55.282 | INFO | "huge_shared" : " 27 GB", | 20260505 12:45:55.282 | INFO | "malloc" : " 14 GB", | 20260505 12:45:55.282 | INFO | "pinned" : " 29 GB" | 20260505 12:45:55.282 | INFO | } | 20260505 12:45:55.282 | INFO | Perf report for the last 5.14171 seconds: | 20260505 12:45:55.282 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:45:55.282 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:55.282 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:55.282 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:55.282 | INFO | ANALYSIS : 64.7%, 3.0, 100.1ms, 0 | 20260505 12:45:55.282 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:45:55.282 | INFO | RT_METRICS : 46.9%, 1.0, 24.6ms, 0 | 20260505 12:45:55.282 | INFO | PRE_HQ : 60.9%, 1.8, 58.1ms, 0 | 20260505 12:45:55.282 | INFO | BAZWRITER : 64.5%, 4.0, 134.2ms, 0 | 20260505 12:45:55.282 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 12:45:55.283 | INFO | Uploaded 36GiB to GPU in 2.1989s (16.3718GiB/s) | 20260505 12:45:55.283 | INFO | Downloaded 38GiB to GPU in 2.28626s (16.621GiB/s) | 20260505 12:46:00.423 | INFO | Analyzing chunk frames = [3314176,3314688) | 20260505 12:46:00.423 | INFO | MemInfo: { | 20260505 12:46:00.423 | INFO | "gpu" : " 17 GB", | 20260505 12:46:00.423 | INFO | "huge" : "", | 20260505 12:46:00.423 | INFO | "huge_shared" : " 27 GB", | 20260505 12:46:00.423 | INFO | "malloc" : " 14 GB", | 20260505 12:46:00.423 | INFO | "pinned" : " 29 GB" | 20260505 12:46:00.423 | INFO | } | 20260505 12:46:00.423 | INFO | Perf report for the last 5.14054 seconds: | 20260505 12:46:00.423 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:46:00.423 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:00.423 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:00.423 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:00.423 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 12:46:00.423 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:00.423 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:00.423 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 12:46:00.423 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:00.423 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:00.423 | INFO | Uploaded 36GiB to GPU in 2.17665s (16.5392GiB/s) | 20260505 12:46:00.423 | INFO | Downloaded 34GiB to GPU in 2.04484s (16.6272GiB/s) | 20260505 12:46:05.567 | INFO | Analyzing chunk frames = [3314688,3315200) | 20260505 12:46:05.567 | INFO | MemInfo: { | 20260505 12:46:05.567 | INFO | "gpu" : " 17 GB", | 20260505 12:46:05.567 | INFO | "huge" : "", | 20260505 12:46:05.567 | INFO | "huge_shared" : " 27 GB", | 20260505 12:46:05.567 | INFO | "malloc" : " 14 GB", | 20260505 12:46:05.567 | INFO | "pinned" : " 29 GB" | 20260505 12:46:05.567 | INFO | } | 20260505 12:46:05.567 | INFO | Perf report for the last 5.144 seconds: | 20260505 12:46:05.567 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:46:05.567 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:05.567 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:05.567 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:05.567 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 12:46:05.567 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:05.567 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:05.567 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:46:05.567 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:05.567 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:05.567 | INFO | Uploaded 36GiB to GPU in 2.16071s (16.6612GiB/s) | 20260505 12:46:05.567 | INFO | Downloaded 34GiB to GPU in 2.13062s (15.9578GiB/s) | 20260505 12:46:10.706 | INFO | Analyzing chunk frames = [3315200,3315712) | 20260505 12:46:10.706 | INFO | MemInfo: { | 20260505 12:46:10.706 | INFO | "gpu" : " 17 GB", | 20260505 12:46:10.706 | INFO | "huge" : "", | 20260505 12:46:10.706 | INFO | "huge_shared" : " 27 GB", | 20260505 12:46:10.706 | INFO | "malloc" : " 14 GB", | 20260505 12:46:10.706 | INFO | "pinned" : " 29 GB" | 20260505 12:46:10.706 | INFO | } | 20260505 12:46:10.706 | INFO | Perf report for the last 5.13943 seconds: | 20260505 12:46:10.707 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:46:10.707 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:10.707 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:10.707 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:10.707 | INFO | ANALYSIS : 63.0%, 3.0, 97.8ms, 0 | 20260505 12:46:10.707 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:10.707 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:10.707 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 12:46:10.707 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:10.707 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:10.707 | INFO | Uploaded 36GiB to GPU in 2.19039s (16.4354GiB/s) | 20260505 12:46:10.707 | INFO | Downloaded 34GiB to GPU in 2.12998s (15.9626GiB/s) | 20260505 12:46:15.833 | INFO | Analyzing chunk frames = [3315712,3316224) | 20260505 12:46:15.833 | INFO | MemInfo: { | 20260505 12:46:15.833 | INFO | "gpu" : " 17 GB", | 20260505 12:46:15.833 | INFO | "huge" : "", | 20260505 12:46:15.833 | INFO | "huge_shared" : " 27 GB", | 20260505 12:46:15.833 | INFO | "malloc" : " 14 GB", | 20260505 12:46:15.833 | INFO | "pinned" : " 29 GB" | 20260505 12:46:15.833 | INFO | } | 20260505 12:46:15.833 | INFO | Perf report for the last 5.12656 seconds: | 20260505 12:46:15.833 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:46:15.833 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:15.833 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:15.833 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:15.833 | INFO | ANALYSIS : 63.1%, 3.0, 98.2ms, 0 | 20260505 12:46:15.833 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:15.833 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:15.833 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:46:15.833 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:15.833 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:15.833 | INFO | Uploaded 36GiB to GPU in 2.17092s (16.5828GiB/s) | 20260505 12:46:15.833 | INFO | Downloaded 34GiB to GPU in 2.12783s (15.9787GiB/s) | 20260505 12:46:20.974 | INFO | Analyzing chunk frames = [3316224,3316736) | 20260505 12:46:20.974 | INFO | MemInfo: { | 20260505 12:46:20.974 | INFO | "gpu" : " 17 GB", | 20260505 12:46:20.974 | INFO | "huge" : "", | 20260505 12:46:20.974 | INFO | "huge_shared" : " 27 GB", | 20260505 12:46:20.974 | INFO | "malloc" : " 14 GB", | 20260505 12:46:20.975 | INFO | "pinned" : " 29 GB" | 20260505 12:46:20.975 | INFO | } | 20260505 12:46:20.975 | INFO | Perf report for the last 5.14161 seconds: | 20260505 12:46:20.975 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:46:20.975 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:20.975 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:20.975 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:20.975 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 12:46:20.975 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:20.975 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:20.975 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 12:46:20.975 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:20.975 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:20.975 | INFO | Uploaded 36GiB to GPU in 2.19891s (16.3718GiB/s) | 20260505 12:46:20.975 | INFO | Downloaded 34GiB to GPU in 2.12383s (16.0088GiB/s) | 20260505 12:46:26.120 | INFO | Analyzing chunk frames = [3316736,3317248) | 20260505 12:46:26.120 | INFO | MemInfo: { | 20260505 12:46:26.120 | INFO | "gpu" : " 17 GB", | 20260505 12:46:26.120 | INFO | "huge" : "", | 20260505 12:46:26.120 | INFO | "huge_shared" : " 27 GB", | 20260505 12:46:26.120 | INFO | "malloc" : " 14 GB", | 20260505 12:46:26.120 | INFO | "pinned" : " 29 GB" | 20260505 12:46:26.120 | INFO | } | 20260505 12:46:26.120 | INFO | Perf report for the last 5.14569 seconds: | 20260505 12:46:26.120 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:46:26.120 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:26.120 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:26.120 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:26.120 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 12:46:26.120 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:26.120 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:26.120 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 12:46:26.120 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:26.120 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:26.120 | INFO | Uploaded 36GiB to GPU in 2.18004s (16.5135GiB/s) | 20260505 12:46:26.120 | INFO | Downloaded 34GiB to GPU in 2.14837s (15.826GiB/s) | 20260505 12:46:31.263 | INFO | Analyzing chunk frames = [3317248,3317760) | 20260505 12:46:31.263 | INFO | MemInfo: { | 20260505 12:46:31.263 | INFO | "gpu" : " 17 GB", | 20260505 12:46:31.263 | INFO | "huge" : "", | 20260505 12:46:31.263 | INFO | "huge_shared" : " 27 GB", | 20260505 12:46:31.263 | INFO | "malloc" : " 14 GB", | 20260505 12:46:31.263 | INFO | "pinned" : " 29 GB" | 20260505 12:46:31.263 | INFO | } | 20260505 12:46:31.263 | INFO | Perf report for the last 5.14295 seconds: | 20260505 12:46:31.263 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:46:31.263 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:31.263 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:31.263 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:31.263 | INFO | ANALYSIS : 63.5%, 3.0, 99.3ms, 0 | 20260505 12:46:31.263 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:31.263 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:31.263 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 12:46:31.263 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:31.263 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:31.263 | INFO | Uploaded 36GiB to GPU in 2.15454s (16.7089GiB/s) | 20260505 12:46:31.263 | INFO | Downloaded 34GiB to GPU in 2.10563s (16.1472GiB/s) | 20260505 12:46:31.264 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 12:46:31.264 | INFO | Total mem usage: 15.3 GiB | 20260505 12:46:31.264 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 12:46:31.264 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -6.91%) | 20260505 12:46:31.264 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 12:46:31.264 | INFO | High Water Breakdown: | 20260505 12:46:31.264 | INFO | 61.1% : 8.76 GiB ( 92% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 12:46:31.264 | INFO | 38.8% : 5.56 GiB ( 83.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 12:46:31.264 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 12:46:31.264 | INFO | ------------------------------------------ | 20260505 12:46:31.264 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:46:31.264 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 12:46:31.264 | INFO | Total mem usage: 26.5 GiB | 20260505 12:46:31.264 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 12:46:31.264 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 12:46:31.264 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 12:46:31.264 | INFO | High Water Breakdown: | 20260505 12:46:31.264 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:46:31.264 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:46:31.264 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:46:31.264 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:46:31.264 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:46:31.264 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:46:31.264 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 12:46:31.264 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 12:46:31.264 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:46:31.264 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 12:46:31.264 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 12:46:31.264 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 12:46:31.264 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 12:46:31.264 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 12:46:31.264 | INFO | ------------------------------------------ | 20260505 12:46:31.264 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:46:31.264 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 12:46:31.264 | INFO | Total mem usage: 25.6 GiB | 20260505 12:46:31.264 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 12:46:31.264 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 12:46:31.264 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 12:46:31.264 | INFO | High Water Breakdown: | 20260505 12:46:31.264 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 12:46:31.264 | INFO | ------------------------------------------ | 20260505 12:46:31.264 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:46:31.264 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 12:46:31.264 | INFO | Total mem usage: 15.7 GiB | 20260505 12:46:31.264 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 12:46:31.264 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 12:46:31.264 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 12:46:31.264 | INFO | High Water Breakdown: | 20260505 12:46:31.264 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 12:46:31.264 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 12:46:31.264 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 12:46:31.264 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 12:46:31.264 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 12:46:31.264 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 12:46:31.264 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 12:46:31.264 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 12:46:31.264 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 12:46:31.264 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:46:31.264 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 12:46:31.264 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:46:31.264 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 12:46:31.264 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 12:46:31.264 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 12:46:31.264 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 12:46:31.264 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:46:31.264 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 12:46:31.264 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:46:31.264 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:46:31.264 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:46:31.264 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 12:46:31.264 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 12:46:31.264 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:46:31.264 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 12:46:31.264 | INFO | ------------------------------------------ | 20260505 12:46:31.264 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:46:31.264 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 12:46:31.264 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:46:31.264 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:46:31.264 | INFO | PerfReport: Full Execution -- 3.20e+01 / 7.74e+01, 1.39e+01 / 1.82e+00 | 20260505 12:46:31.264 | INFO | PerfReport: 0.29%, AggregateStats -- 9.30e-02 / 9.88e-02, 1.05e-02 / 3.40e-03 | 20260505 12:46:31.264 | INFO | PerfReport: 11.78%, Baseline -- 3.77e+00 / 4.49e+00, 6.06e-01 / 1.44e-01 | 20260505 12:46:31.264 | INFO | PerfReport: 3.31%, Histogram -- 1.06e+00 / 1.50e+00, 2.04e-01 / 8.87e-02 | 20260505 12:46:31.264 | INFO | PerfReport: 16.06%, DME -- 4.11e+01 / 4.66e+01, 4.99e+00 / 2.20e+00 | 20260505 12:46:31.264 | INFO | PerfReport: 56.24%, FrameLabeling -- 1.80e+01 / 1.98e+01, 1.40e+00 / 1.87e+00 | 20260505 12:46:31.264 | INFO | PerfReport: 7.43%, PulseAccumulating -- 2.38e+00 / 2.96e+00, 4.52e-01 / 1.75e-01 | 20260505 12:46:31.264 | INFO | PerfReport: 3.60%, Metrics -- 1.15e+00 / 1.49e+00, 3.23e-01 / 1.82e-01 | 20260505 12:46:31.264 | INFO | PerfReport: 1.27%, PulseSerializing -- 4.07e-01 / 4.54e-01, 7.37e-02 / 3.68e-02 | 20260505 12:46:31.264 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 12:46:31.264 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:46:31.264 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:46:31.264 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.56e+01 / 0.00e+00 | 20260505 12:46:31.264 | INFO | PerfReport: 22.44%, Upload -- 2.27e+01 / 0.00e+00, 7.98e+00 / 0.00e+00 | 20260505 12:46:31.264 | INFO | PerfReport: 22.01%, Download -- 2.22e+01 / 0.00e+00, 8.11e+00 / 0.00e+00 | 20260505 12:46:31.264 | INFO | PerfReport: 31.68%, Compute -- 3.20e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 12:46:31.264 | INFO | PerfReport: 2.13%, LockUpload -- 2.15e+00 / 0.00e+00, 7.84e+00 / 0.00e+00 | 20260505 12:46:31.264 | INFO | PerfReport: 1.44%, LockDownload -- 1.46e+00 / 0.00e+00, 3.02e+00 / 0.00e+00 | 20260505 12:46:31.264 | INFO | PerfReport: 20.28%, LockCompute -- 2.05e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260505 12:46:31.264 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.73e-02 / 0.00e+00 | 20260505 12:46:31.401 | INFO | Starting a round of Baz IO | 20260505 12:46:34.622 | INFO | Populating RTMetrics report | 20260505 12:46:34.622 | INFO | RTMetrics report populated | 20260505 12:46:34.622 | INFO | Opening tmp RTMetrics json file | 20260505 12:46:34.623 | INFO | Serializing RTMetrics to json | 20260505 12:46:34.624 | INFO | Serialization done, swapping tmp file now... | 20260505 12:46:34.624 | INFO | RTMetrics done | 20260505 12:46:34.753 | INFO | Finished a round of Baz IO: Wrote 2.328 GiB in 3.351 seconds | 20260505 12:46:34.753 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.42% / 0.01% / 76.58% | 20260505 12:46:34.753 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.82169% / 1.58946e-05% / 7.94729e-05% | 20260505 12:46:34.753 | INFO | Baz IO overall: | 20260505 12:46:34.753 | INFO | BytesWritten : 1973432190689 | 20260505 12:46:34.753 | INFO | NumEvents : 939206164277 | 20260505 12:46:36.404 | INFO | Analyzing chunk frames = [3317760,3318272) | 20260505 12:46:36.404 | INFO | MemInfo: { | 20260505 12:46:36.404 | INFO | "gpu" : " 17 GB", | 20260505 12:46:36.404 | INFO | "huge" : "", | 20260505 12:46:36.404 | INFO | "huge_shared" : " 27 GB", | 20260505 12:46:36.404 | INFO | "malloc" : " 14 GB", | 20260505 12:46:36.404 | INFO | "pinned" : " 29 GB" | 20260505 12:46:36.404 | INFO | } | 20260505 12:46:36.404 | INFO | Perf report for the last 5.1408 seconds: | 20260505 12:46:36.404 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:46:36.404 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:36.404 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:36.404 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:36.404 | INFO | ANALYSIS : 64.2%, 3.0, 99.4ms, 0 | 20260505 12:46:36.404 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:36.404 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260505 12:46:36.404 | INFO | PRE_HQ : 61.2%, 1.8, 58.2ms, 0 | 20260505 12:46:36.404 | INFO | BAZWRITER : 64.6%, 4.1, 135.3ms, 0 | 20260505 12:46:36.404 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 12:46:36.404 | INFO | Uploaded 36GiB to GPU in 2.19156s (16.4266GiB/s) | 20260505 12:46:36.404 | INFO | Downloaded 38GiB to GPU in 2.26858s (16.7506GiB/s) | 20260505 12:46:41.534 | INFO | Analyzing chunk frames = [3318272,3318784) | 20260505 12:46:41.534 | INFO | MemInfo: { | 20260505 12:46:41.534 | INFO | "gpu" : " 17 GB", | 20260505 12:46:41.534 | INFO | "huge" : "", | 20260505 12:46:41.534 | INFO | "huge_shared" : " 27 GB", | 20260505 12:46:41.534 | INFO | "malloc" : " 14 GB", | 20260505 12:46:41.534 | INFO | "pinned" : " 29 GB" | 20260505 12:46:41.534 | INFO | } | 20260505 12:46:41.534 | INFO | Perf report for the last 5.13044 seconds: | 20260505 12:46:41.535 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:46:41.535 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:41.535 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:41.535 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:41.535 | INFO | ANALYSIS : 63.7%, 3.0, 99.2ms, 0 | 20260505 12:46:41.535 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:41.535 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:41.535 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:46:41.535 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:41.535 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:41.535 | INFO | Uploaded 36GiB to GPU in 2.16075s (16.6609GiB/s) | 20260505 12:46:41.535 | INFO | Downloaded 34GiB to GPU in 2.02603s (16.7815GiB/s) | 20260505 12:46:46.678 | INFO | Analyzing chunk frames = [3318784,3319296) | 20260505 12:46:46.678 | INFO | MemInfo: { | 20260505 12:46:46.678 | INFO | "gpu" : " 17 GB", | 20260505 12:46:46.678 | INFO | "huge" : "", | 20260505 12:46:46.679 | INFO | "huge_shared" : " 27 GB", | 20260505 12:46:46.679 | INFO | "malloc" : " 14 GB", | 20260505 12:46:46.679 | INFO | "pinned" : " 29 GB" | 20260505 12:46:46.679 | INFO | } | 20260505 12:46:46.679 | INFO | Perf report for the last 5.14412 seconds: | 20260505 12:46:46.679 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:46:46.679 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:46.679 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:46.679 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:46.679 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 12:46:46.679 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:46.679 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:46.679 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 12:46:46.679 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:46.679 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:46.679 | INFO | Uploaded 36GiB to GPU in 2.22509s (16.1792GiB/s) | 20260505 12:46:46.679 | INFO | Downloaded 34GiB to GPU in 2.12748s (15.9814GiB/s) | 20260505 12:46:51.812 | INFO | Analyzing chunk frames = [3319296,3319808) | 20260505 12:46:51.812 | INFO | MemInfo: { | 20260505 12:46:51.812 | INFO | "gpu" : " 17 GB", | 20260505 12:46:51.812 | INFO | "huge" : "", | 20260505 12:46:51.812 | INFO | "huge_shared" : " 27 GB", | 20260505 12:46:51.812 | INFO | "malloc" : " 14 GB", | 20260505 12:46:51.812 | INFO | "pinned" : " 29 GB" | 20260505 12:46:51.812 | INFO | } | 20260505 12:46:51.812 | INFO | Perf report for the last 5.13376 seconds: | 20260505 12:46:51.812 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:46:51.812 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:51.812 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:51.812 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:51.812 | INFO | ANALYSIS : 63.3%, 3.0, 98.2ms, 0 | 20260505 12:46:51.812 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:51.812 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:51.812 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 12:46:51.812 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:51.812 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:51.813 | INFO | Uploaded 36GiB to GPU in 2.20823s (16.3027GiB/s) | 20260505 12:46:51.813 | INFO | Downloaded 34GiB to GPU in 2.12979s (15.964GiB/s) | 20260505 12:46:56.955 | INFO | Analyzing chunk frames = [3319808,3320320) | 20260505 12:46:56.955 | INFO | MemInfo: { | 20260505 12:46:56.955 | INFO | "gpu" : " 17 GB", | 20260505 12:46:56.955 | INFO | "huge" : "", | 20260505 12:46:56.955 | INFO | "huge_shared" : " 27 GB", | 20260505 12:46:56.955 | INFO | "malloc" : " 14 GB", | 20260505 12:46:56.955 | INFO | "pinned" : " 29 GB" | 20260505 12:46:56.955 | INFO | } | 20260505 12:46:56.955 | INFO | Perf report for the last 5.14262 seconds: | 20260505 12:46:56.955 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:46:56.955 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:56.955 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:56.955 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:56.955 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 12:46:56.955 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:56.955 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:56.955 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:46:56.955 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:56.955 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:46:56.955 | INFO | Uploaded 36GiB to GPU in 2.17501s (16.5516GiB/s) | 20260505 12:46:56.955 | INFO | Downloaded 34GiB to GPU in 2.12721s (15.9834GiB/s) | 20260505 12:47:02.090 | INFO | Analyzing chunk frames = [3320320,3320832) | 20260505 12:47:02.090 | INFO | MemInfo: { | 20260505 12:47:02.090 | INFO | "gpu" : " 17 GB", | 20260505 12:47:02.090 | INFO | "huge" : "", | 20260505 12:47:02.090 | INFO | "huge_shared" : " 27 GB", | 20260505 12:47:02.090 | INFO | "malloc" : " 14 GB", | 20260505 12:47:02.090 | INFO | "pinned" : " 29 GB" | 20260505 12:47:02.090 | INFO | } | 20260505 12:47:02.090 | INFO | Perf report for the last 5.13544 seconds: | 20260505 12:47:02.090 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:47:02.090 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:02.090 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:02.090 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:02.090 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 12:47:02.091 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:02.091 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:02.091 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:47:02.091 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:02.091 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:02.091 | INFO | Uploaded 36GiB to GPU in 2.17887s (16.5224GiB/s) | 20260505 12:47:02.091 | INFO | Downloaded 34GiB to GPU in 2.11033s (16.1112GiB/s) | 20260505 12:47:07.231 | INFO | Analyzing chunk frames = [3320832,3321344) | 20260505 12:47:07.231 | INFO | MemInfo: { | 20260505 12:47:07.231 | INFO | "gpu" : " 17 GB", | 20260505 12:47:07.231 | INFO | "huge" : "", | 20260505 12:47:07.231 | INFO | "huge_shared" : " 27 GB", | 20260505 12:47:07.231 | INFO | "malloc" : " 14 GB", | 20260505 12:47:07.231 | INFO | "pinned" : " 29 GB" | 20260505 12:47:07.231 | INFO | } | 20260505 12:47:07.231 | INFO | Perf report for the last 5.14037 seconds: | 20260505 12:47:07.231 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:47:07.231 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:07.231 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:07.231 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:07.231 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 12:47:07.231 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:07.231 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:07.231 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260505 12:47:07.231 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:07.231 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:07.231 | INFO | Uploaded 36GiB to GPU in 2.14009s (16.8217GiB/s) | 20260505 12:47:07.231 | INFO | Downloaded 34GiB to GPU in 2.12843s (15.9742GiB/s) | 20260505 12:47:12.385 | INFO | Analyzing chunk frames = [3321344,3321856) | 20260505 12:47:12.385 | INFO | MemInfo: { | 20260505 12:47:12.385 | INFO | "gpu" : " 17 GB", | 20260505 12:47:12.385 | INFO | "huge" : "", | 20260505 12:47:12.385 | INFO | "huge_shared" : " 27 GB", | 20260505 12:47:12.385 | INFO | "malloc" : " 14 GB", | 20260505 12:47:12.385 | INFO | "pinned" : " 29 GB" | 20260505 12:47:12.385 | INFO | } | 20260505 12:47:12.385 | INFO | Perf report for the last 5.15405 seconds: | 20260505 12:47:12.385 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:47:12.385 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:12.385 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:12.385 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:12.385 | INFO | ANALYSIS : 63.6%, 3.0, 99.6ms, 0 | 20260505 12:47:12.385 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:12.385 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:12.385 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 12:47:12.385 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:12.385 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:12.385 | INFO | Uploaded 36GiB to GPU in 2.17382s (16.5607GiB/s) | 20260505 12:47:12.385 | INFO | Downloaded 34GiB to GPU in 2.11427s (16.0812GiB/s) | 20260505 12:47:12.521 | INFO | Starting a round of Baz IO | 20260505 12:47:15.743 | INFO | Populating RTMetrics report | 20260505 12:47:15.743 | INFO | RTMetrics report populated | 20260505 12:47:15.743 | INFO | Opening tmp RTMetrics json file | 20260505 12:47:15.765 | INFO | Serializing RTMetrics to json | 20260505 12:47:15.766 | INFO | Serialization done, swapping tmp file now... | 20260505 12:47:15.767 | INFO | RTMetrics done | 20260505 12:47:15.866 | INFO | Finished a round of Baz IO: Wrote 2.328 GiB in 3.344 seconds | 20260505 12:47:15.866 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.41% / 0.01% / 76.58% | 20260505 12:47:15.866 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.820768% / 3.97364e-06% / 6.75519e-05% | 20260505 12:47:15.866 | INFO | Baz IO overall: | 20260505 12:47:15.866 | INFO | BytesWritten : 1975932004426 | 20260505 12:47:15.866 | INFO | NumEvents : 940390631853 | 20260505 12:47:17.517 | INFO | Analyzing chunk frames = [3321856,3322368) | 20260505 12:47:17.517 | INFO | MemInfo: { | 20260505 12:47:17.517 | INFO | "gpu" : " 17 GB", | 20260505 12:47:17.517 | INFO | "huge" : "", | 20260505 12:47:17.517 | INFO | "huge_shared" : " 27 GB", | 20260505 12:47:17.517 | INFO | "malloc" : " 14 GB", | 20260505 12:47:17.518 | INFO | "pinned" : " 29 GB" | 20260505 12:47:17.518 | INFO | } | 20260505 12:47:17.518 | INFO | Perf report for the last 5.13263 seconds: | 20260505 12:47:17.518 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:47:17.518 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:17.518 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:17.518 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:17.518 | INFO | ANALYSIS : 64.3%, 2.9, 99.3ms, 0 | 20260505 12:47:17.518 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:17.518 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260505 12:47:17.518 | INFO | PRE_HQ : 61.3%, 1.8, 58.3ms, 0 | 20260505 12:47:17.518 | INFO | BAZWRITER : 64.5%, 4.1, 134.6ms, 0 | 20260505 12:47:17.518 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 12:47:17.518 | INFO | Uploaded 36GiB to GPU in 2.19595s (16.3938GiB/s) | 20260505 12:47:17.518 | INFO | Downloaded 38GiB to GPU in 2.26014s (16.8131GiB/s) | 20260505 12:47:22.653 | INFO | Analyzing chunk frames = [3322368,3322880) | 20260505 12:47:22.653 | INFO | MemInfo: { | 20260505 12:47:22.653 | INFO | "gpu" : " 17 GB", | 20260505 12:47:22.653 | INFO | "huge" : "", | 20260505 12:47:22.653 | INFO | "huge_shared" : " 27 GB", | 20260505 12:47:22.653 | INFO | "malloc" : " 14 GB", | 20260505 12:47:22.653 | INFO | "pinned" : " 29 GB" | 20260505 12:47:22.653 | INFO | } | 20260505 12:47:22.653 | INFO | Perf report for the last 5.13585 seconds: | 20260505 12:47:22.653 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:47:22.653 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:22.653 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:22.653 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:22.653 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 12:47:22.654 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:22.654 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:22.654 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 12:47:22.654 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:22.654 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:22.654 | INFO | Uploaded 36GiB to GPU in 2.15759s (16.6853GiB/s) | 20260505 12:47:22.654 | INFO | Downloaded 34GiB to GPU in 2.02758s (16.7688GiB/s) | 20260505 12:47:27.785 | INFO | Analyzing chunk frames = [3322880,3323392) | 20260505 12:47:27.785 | INFO | MemInfo: { | 20260505 12:47:27.785 | INFO | "gpu" : " 17 GB", | 20260505 12:47:27.785 | INFO | "huge" : "", | 20260505 12:47:27.785 | INFO | "huge_shared" : " 27 GB", | 20260505 12:47:27.785 | INFO | "malloc" : " 14 GB", | 20260505 12:47:27.785 | INFO | "pinned" : " 29 GB" | 20260505 12:47:27.785 | INFO | } | 20260505 12:47:27.785 | INFO | Perf report for the last 5.13193 seconds: | 20260505 12:47:27.785 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:47:27.785 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:27.785 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:27.785 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:27.785 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 12:47:27.785 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:27.785 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:27.785 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 12:47:27.785 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:27.785 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:27.786 | INFO | Uploaded 36GiB to GPU in 2.22355s (16.1903GiB/s) | 20260505 12:47:27.786 | INFO | Downloaded 34GiB to GPU in 2.12239s (16.0197GiB/s) | 20260505 12:47:32.928 | INFO | Analyzing chunk frames = [3323392,3323904) | 20260505 12:47:32.928 | INFO | MemInfo: { | 20260505 12:47:32.928 | INFO | "gpu" : " 17 GB", | 20260505 12:47:32.928 | INFO | "huge" : "", | 20260505 12:47:32.928 | INFO | "huge_shared" : " 27 GB", | 20260505 12:47:32.928 | INFO | "malloc" : " 14 GB", | 20260505 12:47:32.928 | INFO | "pinned" : " 29 GB" | 20260505 12:47:32.928 | INFO | } | 20260505 12:47:32.928 | INFO | Perf report for the last 5.14266 seconds: | 20260505 12:47:32.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:47:32.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:32.928 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:32.928 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:32.928 | INFO | ANALYSIS : 63.4%, 3.0, 98.6ms, 0 | 20260505 12:47:32.928 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:32.928 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:32.928 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 12:47:32.928 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:32.928 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:32.928 | INFO | Uploaded 36GiB to GPU in 2.21736s (16.2355GiB/s) | 20260505 12:47:32.928 | INFO | Downloaded 34GiB to GPU in 2.13297s (15.9402GiB/s) | 20260505 12:47:38.068 | INFO | Analyzing chunk frames = [3323904,3324416) | 20260505 12:47:38.068 | INFO | MemInfo: { | 20260505 12:47:38.068 | INFO | "gpu" : " 17 GB", | 20260505 12:47:38.068 | INFO | "huge" : "", | 20260505 12:47:38.068 | INFO | "huge_shared" : " 27 GB", | 20260505 12:47:38.068 | INFO | "malloc" : " 14 GB", | 20260505 12:47:38.068 | INFO | "pinned" : " 29 GB" | 20260505 12:47:38.068 | INFO | } | 20260505 12:47:38.068 | INFO | Perf report for the last 5.14024 seconds: | 20260505 12:47:38.068 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:47:38.068 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:38.068 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:38.068 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:38.068 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 12:47:38.068 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:38.068 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:38.068 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:47:38.068 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:38.068 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:38.069 | INFO | Uploaded 36GiB to GPU in 2.16896s (16.5978GiB/s) | 20260505 12:47:38.069 | INFO | Downloaded 34GiB to GPU in 2.12742s (15.9818GiB/s) | 20260505 12:47:43.215 | INFO | Analyzing chunk frames = [3324416,3324928) | 20260505 12:47:43.215 | INFO | MemInfo: { | 20260505 12:47:43.215 | INFO | "gpu" : " 17 GB", | 20260505 12:47:43.216 | INFO | "huge" : "", | 20260505 12:47:43.216 | INFO | "huge_shared" : " 27 GB", | 20260505 12:47:43.216 | INFO | "malloc" : " 14 GB", | 20260505 12:47:43.216 | INFO | "pinned" : " 29 GB" | 20260505 12:47:43.216 | INFO | } | 20260505 12:47:43.216 | INFO | Perf report for the last 5.14737 seconds: | 20260505 12:47:43.216 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:47:43.216 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:43.216 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:43.216 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:43.216 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 12:47:43.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:43.216 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:43.216 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 12:47:43.216 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:43.216 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:43.216 | INFO | Uploaded 36GiB to GPU in 2.15084s (16.7377GiB/s) | 20260505 12:47:43.216 | INFO | Downloaded 34GiB to GPU in 2.10857s (16.1247GiB/s) | 20260505 12:47:48.347 | INFO | Analyzing chunk frames = [3324928,3325440) | 20260505 12:47:48.347 | INFO | MemInfo: { | 20260505 12:47:48.347 | INFO | "gpu" : " 17 GB", | 20260505 12:47:48.347 | INFO | "huge" : "", | 20260505 12:47:48.347 | INFO | "huge_shared" : " 27 GB", | 20260505 12:47:48.347 | INFO | "malloc" : " 14 GB", | 20260505 12:47:48.347 | INFO | "pinned" : " 29 GB" | 20260505 12:47:48.347 | INFO | } | 20260505 12:47:48.347 | INFO | Perf report for the last 5.13098 seconds: | 20260505 12:47:48.347 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:47:48.347 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:48.347 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:48.347 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:48.347 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 12:47:48.347 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:48.347 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:48.347 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 12:47:48.347 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:48.347 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:48.347 | INFO | Uploaded 36GiB to GPU in 2.18915s (16.4447GiB/s) | 20260505 12:47:48.347 | INFO | Downloaded 34GiB to GPU in 2.15637s (15.7672GiB/s) | 20260505 12:47:53.483 | INFO | Analyzing chunk frames = [3325440,3325952) | 20260505 12:47:53.483 | INFO | MemInfo: { | 20260505 12:47:53.483 | INFO | "gpu" : " 17 GB", | 20260505 12:47:53.483 | INFO | "huge" : "", | 20260505 12:47:53.483 | INFO | "huge_shared" : " 27 GB", | 20260505 12:47:53.483 | INFO | "malloc" : " 14 GB", | 20260505 12:47:53.483 | INFO | "pinned" : " 29 GB" | 20260505 12:47:53.483 | INFO | } | 20260505 12:47:53.483 | INFO | Perf report for the last 5.13648 seconds: | 20260505 12:47:53.483 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:47:53.483 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:53.483 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:53.483 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:53.483 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 12:47:53.483 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:53.483 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:53.483 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 12:47:53.483 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:53.483 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:53.483 | INFO | Uploaded 36GiB to GPU in 2.21368s (16.2625GiB/s) | 20260505 12:47:53.483 | INFO | Downloaded 34GiB to GPU in 2.09496s (16.2294GiB/s) | 20260505 12:47:53.483 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 12:47:53.483 | INFO | Total mem usage: 15.3 GiB | 20260505 12:47:53.483 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 12:47:53.483 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -6.92%) | 20260505 12:47:53.484 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 12:47:53.484 | INFO | High Water Breakdown: | 20260505 12:47:53.484 | INFO | 61.1% : 8.76 GiB ( 92% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 12:47:53.484 | INFO | 38.8% : 5.56 GiB ( 83.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 12:47:53.484 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 12:47:53.484 | INFO | ------------------------------------------ | 20260505 12:47:53.484 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:47:53.484 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 12:47:53.484 | INFO | Total mem usage: 26.5 GiB | 20260505 12:47:53.484 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 12:47:53.484 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 12:47:53.484 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 12:47:53.484 | INFO | High Water Breakdown: | 20260505 12:47:53.484 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:47:53.484 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:47:53.484 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:47:53.484 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:47:53.484 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:47:53.484 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:47:53.484 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 12:47:53.484 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 12:47:53.484 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:47:53.484 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 12:47:53.484 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 12:47:53.484 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 12:47:53.484 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 12:47:53.484 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 12:47:53.484 | INFO | ------------------------------------------ | 20260505 12:47:53.484 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:47:53.484 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 12:47:53.484 | INFO | Total mem usage: 25.6 GiB | 20260505 12:47:53.484 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 12:47:53.484 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 12:47:53.484 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 12:47:53.484 | INFO | High Water Breakdown: | 20260505 12:47:53.484 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 12:47:53.484 | INFO | ------------------------------------------ | 20260505 12:47:53.484 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:47:53.484 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 12:47:53.484 | INFO | Total mem usage: 15.7 GiB | 20260505 12:47:53.484 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 12:47:53.484 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 12:47:53.484 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 12:47:53.484 | INFO | High Water Breakdown: | 20260505 12:47:53.484 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 12:47:53.484 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 12:47:53.484 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 12:47:53.484 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 12:47:53.484 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 12:47:53.484 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 12:47:53.484 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 12:47:53.484 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 12:47:53.484 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 12:47:53.484 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:47:53.484 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 12:47:53.484 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:47:53.484 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 12:47:53.484 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 12:47:53.484 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 12:47:53.484 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 12:47:53.484 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:47:53.484 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 12:47:53.484 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:47:53.484 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:47:53.484 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:47:53.484 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 12:47:53.484 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 12:47:53.484 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:47:53.484 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 12:47:53.484 | INFO | ------------------------------------------ | 20260505 12:47:53.484 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:47:53.484 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 12:47:53.484 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:47:53.484 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:47:53.484 | INFO | PerfReport: Full Execution -- 3.20e+01 / 7.68e+01, 1.38e+01 / 1.55e+00 | 20260505 12:47:53.484 | INFO | PerfReport: 0.29%, AggregateStats -- 9.30e-02 / 9.98e-02, 1.04e-02 / 4.34e-03 | 20260505 12:47:53.484 | INFO | PerfReport: 11.77%, Baseline -- 3.77e+00 / 4.49e+00, 6.08e-01 / 2.06e-01 | 20260505 12:47:53.484 | INFO | PerfReport: 3.32%, Histogram -- 1.06e+00 / 1.46e+00, 2.04e-01 / 1.05e-01 | 20260505 12:47:53.484 | INFO | PerfReport: 15.85%, DME -- 4.06e+01 / 4.52e+01, 4.45e+00 / 2.09e+00 | 20260505 12:47:53.484 | INFO | PerfReport: 56.50%, FrameLabeling -- 1.81e+01 / 2.08e+01, 1.56e+00 / 2.26e+00 | 20260505 12:47:53.484 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.36e+00 / 2.88e+00, 4.37e-01 / 2.16e-01 | 20260505 12:47:53.484 | INFO | PerfReport: 3.62%, Metrics -- 1.16e+00 / 1.45e+00, 3.30e-01 / 1.79e-01 | 20260505 12:47:53.484 | INFO | PerfReport: 1.27%, PulseSerializing -- 4.07e-01 / 4.54e-01, 7.45e-02 / 3.42e-02 | 20260505 12:47:53.484 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 12:47:53.484 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:47:53.484 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:47:53.484 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260505 12:47:53.484 | INFO | PerfReport: 22.56%, Upload -- 2.28e+01 / 0.00e+00, 8.12e+00 / 0.00e+00 | 20260505 12:47:53.484 | INFO | PerfReport: 21.97%, Download -- 2.22e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260505 12:47:53.484 | INFO | PerfReport: 31.73%, Compute -- 3.20e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 12:47:53.484 | INFO | PerfReport: 2.07%, LockUpload -- 2.09e+00 / 0.00e+00, 7.63e+00 / 0.00e+00 | 20260505 12:47:53.484 | INFO | PerfReport: 1.36%, LockDownload -- 1.37e+00 / 0.00e+00, 2.93e+00 / 0.00e+00 | 20260505 12:47:53.484 | INFO | PerfReport: 20.30%, LockCompute -- 2.05e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260505 12:47:53.485 | INFO | PerfReport: 0.01%, Misc -- 1.47e-02 / 0.00e+00, 2.46e-02 / 0.00e+00 | 20260505 12:47:53.616 | INFO | Starting a round of Baz IO | 20260505 12:47:56.863 | INFO | Populating RTMetrics report | 20260505 12:47:56.863 | INFO | RTMetrics report populated | 20260505 12:47:56.863 | INFO | Opening tmp RTMetrics json file | 20260505 12:47:56.887 | INFO | Serializing RTMetrics to json | 20260505 12:47:56.929 | INFO | Serialization done, swapping tmp file now... | 20260505 12:47:56.936 | INFO | RTMetrics done | 20260505 12:47:56.968 | INFO | Finished a round of Baz IO: Wrote 2.331 GiB in 3.352 seconds | 20260505 12:47:56.968 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.41% / 0.01% / 76.59% | 20260505 12:47:56.968 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.823228% / 3.97364e-06% / 6.35783e-05% | 20260505 12:47:56.968 | INFO | Baz IO overall: | 20260505 12:47:56.968 | INFO | BytesWritten : 1978435222783 | 20260505 12:47:56.968 | INFO | NumEvents : 941577932403 | 20260505 12:47:58.630 | INFO | Analyzing chunk frames = [3325952,3326464) | 20260505 12:47:58.630 | INFO | MemInfo: { | 20260505 12:47:58.630 | INFO | "gpu" : " 17 GB", | 20260505 12:47:58.630 | INFO | "huge" : "", | 20260505 12:47:58.630 | INFO | "huge_shared" : " 27 GB", | 20260505 12:47:58.630 | INFO | "malloc" : " 14 GB", | 20260505 12:47:58.630 | INFO | "pinned" : " 29 GB" | 20260505 12:47:58.630 | INFO | } | 20260505 12:47:58.630 | INFO | Perf report for the last 5.14705 seconds: | 20260505 12:47:58.630 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:47:58.630 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:58.630 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:58.630 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:58.630 | INFO | ANALYSIS : 64.6%, 3.0, 100.1ms, 0 | 20260505 12:47:58.630 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:47:58.630 | INFO | RT_METRICS : 46.5%, 1.0, 24.4ms, 0 | 20260505 12:47:58.630 | INFO | PRE_HQ : 60.7%, 1.8, 57.9ms, 0 | 20260505 12:47:58.630 | INFO | BAZWRITER : 64.4%, 4.0, 134.0ms, 0 | 20260505 12:47:58.630 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260505 12:47:58.630 | INFO | Uploaded 36GiB to GPU in 2.19244s (16.42GiB/s) | 20260505 12:47:58.630 | INFO | Downloaded 38GiB to GPU in 2.28558s (16.626GiB/s) | 20260505 12:48:03.770 | INFO | Analyzing chunk frames = [3326464,3326976) | 20260505 12:48:03.770 | INFO | MemInfo: { | 20260505 12:48:03.770 | INFO | "gpu" : " 17 GB", | 20260505 12:48:03.770 | INFO | "huge" : "", | 20260505 12:48:03.770 | INFO | "huge_shared" : " 27 GB", | 20260505 12:48:03.770 | INFO | "malloc" : " 14 GB", | 20260505 12:48:03.770 | INFO | "pinned" : " 29 GB" | 20260505 12:48:03.770 | INFO | } | 20260505 12:48:03.770 | INFO | Perf report for the last 5.13982 seconds: | 20260505 12:48:03.770 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:48:03.770 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:03.770 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:03.770 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:03.770 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 12:48:03.770 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:03.770 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:03.770 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:48:03.770 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:03.770 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:03.770 | INFO | Uploaded 36GiB to GPU in 2.17116s (16.581GiB/s) | 20260505 12:48:03.770 | INFO | Downloaded 34GiB to GPU in 2.02352s (16.8024GiB/s) | 20260505 12:48:08.907 | INFO | Analyzing chunk frames = [3326976,3327488) | 20260505 12:48:08.907 | INFO | MemInfo: { | 20260505 12:48:08.907 | INFO | "gpu" : " 17 GB", | 20260505 12:48:08.907 | INFO | "huge" : "", | 20260505 12:48:08.907 | INFO | "huge_shared" : " 27 GB", | 20260505 12:48:08.908 | INFO | "malloc" : " 14 GB", | 20260505 12:48:08.908 | INFO | "pinned" : " 29 GB" | 20260505 12:48:08.908 | INFO | } | 20260505 12:48:08.908 | INFO | Perf report for the last 5.13753 seconds: | 20260505 12:48:08.908 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:48:08.908 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:08.908 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:08.908 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:08.908 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 12:48:08.908 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:08.908 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:08.908 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:48:08.908 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:08.908 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:08.908 | INFO | Uploaded 36GiB to GPU in 2.18736s (16.4582GiB/s) | 20260505 12:48:08.908 | INFO | Downloaded 34GiB to GPU in 2.11786s (16.054GiB/s) | 20260505 12:48:14.042 | INFO | Analyzing chunk frames = [3327488,3328000) | 20260505 12:48:14.042 | INFO | MemInfo: { | 20260505 12:48:14.042 | INFO | "gpu" : " 17 GB", | 20260505 12:48:14.042 | INFO | "huge" : "", | 20260505 12:48:14.042 | INFO | "huge_shared" : " 27 GB", | 20260505 12:48:14.042 | INFO | "malloc" : " 14 GB", | 20260505 12:48:14.042 | INFO | "pinned" : " 29 GB" | 20260505 12:48:14.042 | INFO | } | 20260505 12:48:14.042 | INFO | Perf report for the last 5.13433 seconds: | 20260505 12:48:14.042 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:48:14.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:14.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:14.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:14.042 | INFO | ANALYSIS : 63.4%, 3.0, 98.4ms, 0 | 20260505 12:48:14.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:14.042 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:14.042 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:48:14.042 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:14.042 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:14.042 | INFO | Uploaded 36GiB to GPU in 2.21127s (16.2802GiB/s) | 20260505 12:48:14.042 | INFO | Downloaded 34GiB to GPU in 2.13412s (15.9316GiB/s) | 20260505 12:48:19.188 | INFO | Analyzing chunk frames = [3328000,3328512) | 20260505 12:48:19.188 | INFO | MemInfo: { | 20260505 12:48:19.188 | INFO | "gpu" : " 17 GB", | 20260505 12:48:19.188 | INFO | "huge" : "", | 20260505 12:48:19.188 | INFO | "huge_shared" : " 27 GB", | 20260505 12:48:19.188 | INFO | "malloc" : " 14 GB", | 20260505 12:48:19.188 | INFO | "pinned" : " 29 GB" | 20260505 12:48:19.188 | INFO | } | 20260505 12:48:19.188 | INFO | Perf report for the last 5.14603 seconds: | 20260505 12:48:19.188 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:48:19.188 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:19.188 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:19.188 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:19.188 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 12:48:19.188 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:19.188 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:19.188 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 12:48:19.188 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:19.188 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:19.188 | INFO | Uploaded 36GiB to GPU in 2.17017s (16.5885GiB/s) | 20260505 12:48:19.188 | INFO | Downloaded 34GiB to GPU in 2.12976s (15.9642GiB/s) | 20260505 12:48:24.329 | INFO | Analyzing chunk frames = [3328512,3329024) | 20260505 12:48:24.329 | INFO | MemInfo: { | 20260505 12:48:24.329 | INFO | "gpu" : " 17 GB", | 20260505 12:48:24.329 | INFO | "huge" : "", | 20260505 12:48:24.329 | INFO | "huge_shared" : " 27 GB", | 20260505 12:48:24.329 | INFO | "malloc" : " 14 GB", | 20260505 12:48:24.329 | INFO | "pinned" : " 29 GB" | 20260505 12:48:24.329 | INFO | } | 20260505 12:48:24.330 | INFO | Perf report for the last 5.14153 seconds: | 20260505 12:48:24.330 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:48:24.330 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:24.330 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:24.330 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:24.330 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 12:48:24.330 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:24.330 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:24.330 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 12:48:24.330 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:24.330 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:24.330 | INFO | Uploaded 36GiB to GPU in 2.187s (16.4609GiB/s) | 20260505 12:48:24.330 | INFO | Downloaded 34GiB to GPU in 2.11197s (16.0987GiB/s) | 20260505 12:48:29.463 | INFO | Analyzing chunk frames = [3329024,3329536) | 20260505 12:48:29.463 | INFO | MemInfo: { | 20260505 12:48:29.463 | INFO | "gpu" : " 17 GB", | 20260505 12:48:29.463 | INFO | "huge" : "", | 20260505 12:48:29.463 | INFO | "huge_shared" : " 27 GB", | 20260505 12:48:29.463 | INFO | "malloc" : " 14 GB", | 20260505 12:48:29.463 | INFO | "pinned" : " 29 GB" | 20260505 12:48:29.463 | INFO | } | 20260505 12:48:29.463 | INFO | Perf report for the last 5.13316 seconds: | 20260505 12:48:29.463 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:48:29.463 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:29.463 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:29.463 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:29.463 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 12:48:29.463 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:29.463 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:29.463 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 12:48:29.463 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:29.463 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:29.463 | INFO | Uploaded 36GiB to GPU in 2.14041s (16.8192GiB/s) | 20260505 12:48:29.463 | INFO | Downloaded 34GiB to GPU in 2.1689s (15.6762GiB/s) | 20260505 12:48:34.602 | INFO | Analyzing chunk frames = [3329536,3330048) | 20260505 12:48:34.602 | INFO | MemInfo: { | 20260505 12:48:34.602 | INFO | "gpu" : " 17 GB", | 20260505 12:48:34.602 | INFO | "huge" : "", | 20260505 12:48:34.602 | INFO | "huge_shared" : " 27 GB", | 20260505 12:48:34.602 | INFO | "malloc" : " 14 GB", | 20260505 12:48:34.602 | INFO | "pinned" : " 29 GB" | 20260505 12:48:34.602 | INFO | } | 20260505 12:48:34.602 | INFO | Perf report for the last 5.13964 seconds: | 20260505 12:48:34.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:48:34.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:34.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:34.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:34.602 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 12:48:34.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:34.602 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:34.602 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 12:48:34.603 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:34.603 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:34.603 | INFO | Uploaded 36GiB to GPU in 2.19236s (16.4206GiB/s) | 20260505 12:48:34.603 | INFO | Downloaded 34GiB to GPU in 2.11687s (16.0615GiB/s) | 20260505 12:48:34.732 | INFO | Starting a round of Baz IO | 20260505 12:48:37.982 | INFO | Populating RTMetrics report | 20260505 12:48:37.982 | INFO | RTMetrics report populated | 20260505 12:48:37.983 | INFO | Opening tmp RTMetrics json file | 20260505 12:48:37.984 | INFO | Serializing RTMetrics to json | 20260505 12:48:37.987 | INFO | Serialization done, swapping tmp file now... | 20260505 12:48:37.988 | INFO | RTMetrics done | 20260505 12:48:38.100 | INFO | Finished a round of Baz IO: Wrote 2.329 GiB in 3.367 seconds | 20260505 12:48:38.100 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.40% / 0.01% / 76.59% | 20260505 12:48:38.100 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.820704% / 3.97364e-06% / 9.93411e-05% | 20260505 12:48:38.100 | INFO | Baz IO overall: | 20260505 12:48:38.100 | INFO | BytesWritten : 1980935539690 | 20260505 12:48:38.100 | INFO | NumEvents : 942763319387 | 20260505 12:48:39.749 | INFO | Analyzing chunk frames = [3330048,3330560) | 20260505 12:48:39.749 | INFO | MemInfo: { | 20260505 12:48:39.749 | INFO | "gpu" : " 17 GB", | 20260505 12:48:39.749 | INFO | "huge" : "", | 20260505 12:48:39.749 | INFO | "huge_shared" : " 27 GB", | 20260505 12:48:39.749 | INFO | "malloc" : " 14 GB", | 20260505 12:48:39.749 | INFO | "pinned" : " 29 GB" | 20260505 12:48:39.749 | INFO | } | 20260505 12:48:39.749 | INFO | Perf report for the last 5.14652 seconds: | 20260505 12:48:39.749 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:48:39.749 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:39.749 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:39.749 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:39.749 | INFO | ANALYSIS : 64.6%, 2.9, 100.0ms, 0 | 20260505 12:48:39.749 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:39.749 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260505 12:48:39.749 | INFO | PRE_HQ : 60.7%, 1.8, 58.0ms, 0 | 20260505 12:48:39.749 | INFO | BAZWRITER : 64.8%, 4.1, 135.2ms, 0 | 20260505 12:48:39.749 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 12:48:39.749 | INFO | Uploaded 36GiB to GPU in 2.19697s (16.3862GiB/s) | 20260505 12:48:39.749 | INFO | Downloaded 38GiB to GPU in 2.28754s (16.6117GiB/s) | 20260505 12:48:44.880 | INFO | Analyzing chunk frames = [3330560,3331072) | 20260505 12:48:44.880 | INFO | MemInfo: { | 20260505 12:48:44.880 | INFO | "gpu" : " 17 GB", | 20260505 12:48:44.880 | INFO | "huge" : "", | 20260505 12:48:44.880 | INFO | "huge_shared" : " 27 GB", | 20260505 12:48:44.880 | INFO | "malloc" : " 14 GB", | 20260505 12:48:44.880 | INFO | "pinned" : " 29 GB" | 20260505 12:48:44.880 | INFO | } | 20260505 12:48:44.880 | INFO | Perf report for the last 5.13117 seconds: | 20260505 12:48:44.880 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:48:44.880 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:44.880 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:44.880 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:44.880 | INFO | ANALYSIS : 63.7%, 3.0, 99.2ms, 0 | 20260505 12:48:44.880 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:44.880 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:44.880 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 12:48:44.880 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:44.880 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:44.880 | INFO | Uploaded 36GiB to GPU in 2.16801s (16.6051GiB/s) | 20260505 12:48:44.880 | INFO | Downloaded 34GiB to GPU in 2.02366s (16.8013GiB/s) | 20260505 12:48:50.027 | INFO | Analyzing chunk frames = [3331072,3331584) | 20260505 12:48:50.027 | INFO | MemInfo: { | 20260505 12:48:50.027 | INFO | "gpu" : " 17 GB", | 20260505 12:48:50.027 | INFO | "huge" : "", | 20260505 12:48:50.028 | INFO | "huge_shared" : " 27 GB", | 20260505 12:48:50.028 | INFO | "malloc" : " 14 GB", | 20260505 12:48:50.028 | INFO | "pinned" : " 29 GB" | 20260505 12:48:50.028 | INFO | } | 20260505 12:48:50.028 | INFO | Perf report for the last 5.14756 seconds: | 20260505 12:48:50.028 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:48:50.028 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:50.028 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:50.028 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:50.028 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 12:48:50.028 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:50.028 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:50.028 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 12:48:50.028 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:50.028 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:50.028 | INFO | Uploaded 36GiB to GPU in 2.19137s (16.4281GiB/s) | 20260505 12:48:50.028 | INFO | Downloaded 34GiB to GPU in 2.13343s (15.9368GiB/s) | 20260505 12:48:55.163 | INFO | Analyzing chunk frames = [3331584,3332096) | 20260505 12:48:55.163 | INFO | MemInfo: { | 20260505 12:48:55.163 | INFO | "gpu" : " 17 GB", | 20260505 12:48:55.163 | INFO | "huge" : "", | 20260505 12:48:55.163 | INFO | "huge_shared" : " 27 GB", | 20260505 12:48:55.163 | INFO | "malloc" : " 14 GB", | 20260505 12:48:55.163 | INFO | "pinned" : " 29 GB" | 20260505 12:48:55.163 | INFO | } | 20260505 12:48:55.163 | INFO | Perf report for the last 5.13579 seconds: | 20260505 12:48:55.163 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:48:55.163 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:55.163 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:55.163 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:55.163 | INFO | ANALYSIS : 62.9%, 3.0, 97.6ms, 0 | 20260505 12:48:55.163 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:55.163 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:55.164 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:48:55.164 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:55.164 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:48:55.164 | INFO | Uploaded 36GiB to GPU in 2.17618s (16.5427GiB/s) | 20260505 12:48:55.164 | INFO | Downloaded 34GiB to GPU in 2.12905s (15.9696GiB/s) | 20260505 12:49:00.291 | INFO | Analyzing chunk frames = [3332096,3332608) | 20260505 12:49:00.291 | INFO | MemInfo: { | 20260505 12:49:00.291 | INFO | "gpu" : " 17 GB", | 20260505 12:49:00.291 | INFO | "huge" : "", | 20260505 12:49:00.292 | INFO | "huge_shared" : " 27 GB", | 20260505 12:49:00.292 | INFO | "malloc" : " 14 GB", | 20260505 12:49:00.292 | INFO | "pinned" : " 29 GB" | 20260505 12:49:00.292 | INFO | } | 20260505 12:49:00.292 | INFO | Perf report for the last 5.1282 seconds: | 20260505 12:49:00.292 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:49:00.292 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:00.292 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:00.292 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:00.292 | INFO | ANALYSIS : 63.1%, 3.0, 98.3ms, 0 | 20260505 12:49:00.292 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:00.292 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:00.292 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:49:00.292 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:00.292 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:00.292 | INFO | Uploaded 36GiB to GPU in 2.17536s (16.549GiB/s) | 20260505 12:49:00.292 | INFO | Downloaded 34GiB to GPU in 2.12574s (15.9944GiB/s) | 20260505 12:49:05.440 | INFO | Analyzing chunk frames = [3332608,3333120) | 20260505 12:49:05.440 | INFO | MemInfo: { | 20260505 12:49:05.440 | INFO | "gpu" : " 17 GB", | 20260505 12:49:05.440 | INFO | "huge" : "", | 20260505 12:49:05.440 | INFO | "huge_shared" : " 27 GB", | 20260505 12:49:05.440 | INFO | "malloc" : " 14 GB", | 20260505 12:49:05.440 | INFO | "pinned" : " 29 GB" | 20260505 12:49:05.440 | INFO | } | 20260505 12:49:05.440 | INFO | Perf report for the last 5.14848 seconds: | 20260505 12:49:05.440 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:49:05.440 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:05.440 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:05.440 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:05.440 | INFO | ANALYSIS : 63.2%, 3.0, 98.8ms, 0 | 20260505 12:49:05.440 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:05.440 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:05.440 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 12:49:05.440 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:05.440 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:05.440 | INFO | Uploaded 36GiB to GPU in 2.16878s (16.5992GiB/s) | 20260505 12:49:05.440 | INFO | Downloaded 34GiB to GPU in 2.11358s (16.0864GiB/s) | 20260505 12:49:10.580 | INFO | Analyzing chunk frames = [3333120,3333632) | 20260505 12:49:10.580 | INFO | MemInfo: { | 20260505 12:49:10.580 | INFO | "gpu" : " 17 GB", | 20260505 12:49:10.580 | INFO | "huge" : "", | 20260505 12:49:10.580 | INFO | "huge_shared" : " 27 GB", | 20260505 12:49:10.580 | INFO | "malloc" : " 14 GB", | 20260505 12:49:10.580 | INFO | "pinned" : " 29 GB" | 20260505 12:49:10.580 | INFO | } | 20260505 12:49:10.580 | INFO | Perf report for the last 5.13985 seconds: | 20260505 12:49:10.580 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:49:10.580 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:10.580 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:10.580 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:10.580 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 12:49:10.580 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:10.580 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:10.580 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 12:49:10.580 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:10.580 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:10.580 | INFO | Uploaded 36GiB to GPU in 2.17541s (16.5486GiB/s) | 20260505 12:49:10.580 | INFO | Downloaded 34GiB to GPU in 2.13189s (15.9483GiB/s) | 20260505 12:49:15.724 | INFO | Analyzing chunk frames = [3333632,3334144) | 20260505 12:49:15.724 | INFO | MemInfo: { | 20260505 12:49:15.724 | INFO | "gpu" : " 17 GB", | 20260505 12:49:15.724 | INFO | "huge" : "", | 20260505 12:49:15.725 | INFO | "huge_shared" : " 27 GB", | 20260505 12:49:15.725 | INFO | "malloc" : " 14 GB", | 20260505 12:49:15.725 | INFO | "pinned" : " 29 GB" | 20260505 12:49:15.725 | INFO | } | 20260505 12:49:15.725 | INFO | Perf report for the last 5.14458 seconds: | 20260505 12:49:15.725 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:49:15.725 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:15.725 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:15.725 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:15.725 | INFO | ANALYSIS : 63.5%, 3.0, 99.3ms, 0 | 20260505 12:49:15.725 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:15.725 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:15.725 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 12:49:15.725 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:15.725 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:15.725 | INFO | Uploaded 36GiB to GPU in 2.15414s (16.712GiB/s) | 20260505 12:49:15.725 | INFO | Downloaded 34GiB to GPU in 2.10423s (16.1579GiB/s) | 20260505 12:49:15.725 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 12:49:15.725 | INFO | Total mem usage: 15.3 GiB | 20260505 12:49:15.725 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 12:49:15.725 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -6.92%) | 20260505 12:49:15.725 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 12:49:15.725 | INFO | High Water Breakdown: | 20260505 12:49:15.725 | INFO | 61.1% : 8.76 GiB ( 92% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 12:49:15.725 | INFO | 38.8% : 5.56 GiB ( 83.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 12:49:15.725 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 12:49:15.725 | INFO | ------------------------------------------ | 20260505 12:49:15.725 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:49:15.725 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 12:49:15.725 | INFO | Total mem usage: 26.5 GiB | 20260505 12:49:15.725 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 12:49:15.725 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 12:49:15.725 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 12:49:15.725 | INFO | High Water Breakdown: | 20260505 12:49:15.725 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:49:15.725 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:49:15.725 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:49:15.725 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:49:15.725 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:49:15.725 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:49:15.725 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 12:49:15.725 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 12:49:15.725 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:49:15.725 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 12:49:15.725 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 12:49:15.725 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 12:49:15.725 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 12:49:15.725 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 12:49:15.725 | INFO | ------------------------------------------ | 20260505 12:49:15.725 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:49:15.725 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 12:49:15.725 | INFO | Total mem usage: 25.6 GiB | 20260505 12:49:15.725 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 12:49:15.725 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 12:49:15.725 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 12:49:15.725 | INFO | High Water Breakdown: | 20260505 12:49:15.725 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 12:49:15.725 | INFO | ------------------------------------------ | 20260505 12:49:15.725 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:49:15.725 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 12:49:15.725 | INFO | Total mem usage: 15.7 GiB | 20260505 12:49:15.725 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 12:49:15.725 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 12:49:15.725 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 12:49:15.725 | INFO | High Water Breakdown: | 20260505 12:49:15.725 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 12:49:15.725 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 12:49:15.725 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 12:49:15.725 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 12:49:15.725 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 12:49:15.725 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 12:49:15.725 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 12:49:15.726 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 12:49:15.726 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 12:49:15.726 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:49:15.726 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 12:49:15.726 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:49:15.726 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 12:49:15.726 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 12:49:15.726 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 12:49:15.726 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 12:49:15.726 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:49:15.726 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 12:49:15.726 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:49:15.726 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:49:15.726 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:49:15.726 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 12:49:15.726 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 12:49:15.726 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:49:15.726 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 12:49:15.726 | INFO | ------------------------------------------ | 20260505 12:49:15.726 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:49:15.726 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 12:49:15.726 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:49:15.726 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:49:15.726 | INFO | PerfReport: Full Execution -- 3.20e+01 / 7.72e+01, 1.36e+01 / 1.82e+00 | 20260505 12:49:15.726 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 9.94e-02, 1.05e-02 / 4.20e-03 | 20260505 12:49:15.726 | INFO | PerfReport: 11.76%, Baseline -- 3.76e+00 / 4.47e+00, 5.99e-01 / 1.74e-01 | 20260505 12:49:15.726 | INFO | PerfReport: 3.31%, Histogram -- 1.06e+00 / 1.50e+00, 2.02e-01 / 7.02e-02 | 20260505 12:49:15.726 | INFO | PerfReport: 16.06%, DME -- 4.10e+01 / 4.65e+01, 4.91e+00 / 2.10e+00 | 20260505 12:49:15.726 | INFO | PerfReport: 56.20%, FrameLabeling -- 1.80e+01 / 1.97e+01, 1.38e+00 / 1.70e+00 | 20260505 12:49:15.726 | INFO | PerfReport: 7.49%, PulseAccumulating -- 2.39e+00 / 2.95e+00, 4.58e-01 / 1.83e-01 | 20260505 12:49:15.726 | INFO | PerfReport: 3.61%, Metrics -- 1.15e+00 / 1.49e+00, 3.22e-01 / 1.33e-01 | 20260505 12:49:15.726 | INFO | PerfReport: 1.27%, PulseSerializing -- 4.06e-01 / 4.58e-01, 7.24e-02 / 2.46e-02 | 20260505 12:49:15.726 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 12:49:15.726 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:49:15.726 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:49:15.726 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260505 12:49:15.726 | INFO | PerfReport: 22.48%, Upload -- 2.27e+01 / 0.00e+00, 8.01e+00 / 0.00e+00 | 20260505 12:49:15.726 | INFO | PerfReport: 22.02%, Download -- 2.22e+01 / 0.00e+00, 8.14e+00 / 0.00e+00 | 20260505 12:49:15.726 | INFO | PerfReport: 31.66%, Compute -- 3.20e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 12:49:15.726 | INFO | PerfReport: 2.13%, LockUpload -- 2.15e+00 / 0.00e+00, 7.80e+00 / 0.00e+00 | 20260505 12:49:15.726 | INFO | PerfReport: 1.50%, LockDownload -- 1.52e+00 / 0.00e+00, 3.07e+00 / 0.00e+00 | 20260505 12:49:15.726 | INFO | PerfReport: 20.20%, LockCompute -- 2.04e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260505 12:49:15.726 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.59e-02 / 0.00e+00 | 20260505 12:49:15.862 | INFO | Starting a round of Baz IO | 20260505 12:49:19.105 | INFO | Populating RTMetrics report | 20260505 12:49:19.105 | INFO | RTMetrics report populated | 20260505 12:49:19.105 | INFO | Opening tmp RTMetrics json file | 20260505 12:49:19.106 | INFO | Serializing RTMetrics to json | 20260505 12:49:19.120 | INFO | Serialization done, swapping tmp file now... | 20260505 12:49:19.132 | INFO | RTMetrics done | 20260505 12:49:19.225 | INFO | Finished a round of Baz IO: Wrote 2.328 GiB in 3.362 seconds | 20260505 12:49:19.225 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.39% / 0.01% / 76.60% | 20260505 12:49:19.225 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.817362% / 7.94729e-06% / 8.34465e-05% | 20260505 12:49:19.225 | INFO | Baz IO overall: | 20260505 12:49:19.225 | INFO | BytesWritten : 1983435595529 | 20260505 12:49:19.225 | INFO | NumEvents : 943948572160 | 20260505 12:49:20.858 | INFO | Analyzing chunk frames = [3334144,3334656) | 20260505 12:49:20.858 | INFO | MemInfo: { | 20260505 12:49:20.858 | INFO | "gpu" : " 17 GB", | 20260505 12:49:20.858 | INFO | "huge" : "", | 20260505 12:49:20.858 | INFO | "huge_shared" : " 27 GB", | 20260505 12:49:20.858 | INFO | "malloc" : " 14 GB", | 20260505 12:49:20.858 | INFO | "pinned" : " 29 GB" | 20260505 12:49:20.858 | INFO | } | 20260505 12:49:20.858 | INFO | Perf report for the last 5.13356 seconds: | 20260505 12:49:20.858 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:49:20.858 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:20.858 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:20.858 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:20.858 | INFO | ANALYSIS : 64.7%, 3.0, 100.1ms, 0 | 20260505 12:49:20.858 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:20.858 | INFO | RT_METRICS : 45.9%, 1.0, 24.0ms, 0 | 20260505 12:49:20.858 | INFO | PRE_HQ : 60.8%, 1.8, 58.2ms, 0 | 20260505 12:49:20.858 | INFO | BAZWRITER : 64.9%, 4.0, 134.6ms, 0 | 20260505 12:49:20.858 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 12:49:20.858 | INFO | Uploaded 36GiB to GPU in 2.18497s (16.4762GiB/s) | 20260505 12:49:20.858 | INFO | Downloaded 38GiB to GPU in 2.28163s (16.6547GiB/s) | 20260505 12:49:25.993 | INFO | Analyzing chunk frames = [3334656,3335168) | 20260505 12:49:25.993 | INFO | MemInfo: { | 20260505 12:49:25.993 | INFO | "gpu" : " 17 GB", | 20260505 12:49:25.993 | INFO | "huge" : "", | 20260505 12:49:25.993 | INFO | "huge_shared" : " 27 GB", | 20260505 12:49:25.993 | INFO | "malloc" : " 14 GB", | 20260505 12:49:25.993 | INFO | "pinned" : " 29 GB" | 20260505 12:49:25.993 | INFO | } | 20260505 12:49:25.993 | INFO | Perf report for the last 5.1347 seconds: | 20260505 12:49:25.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:49:25.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:25.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:25.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:25.993 | INFO | ANALYSIS : 64.2%, 3.0, 100.2ms, 0 | 20260505 12:49:25.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:25.993 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:25.993 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 12:49:25.993 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:25.993 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:25.993 | INFO | Uploaded 36GiB to GPU in 2.23369s (16.1168GiB/s) | 20260505 12:49:25.993 | INFO | Downloaded 34GiB to GPU in 2.0813s (16.3359GiB/s) | 20260505 12:49:31.139 | INFO | Analyzing chunk frames = [3335168,3335680) | 20260505 12:49:31.139 | INFO | MemInfo: { | 20260505 12:49:31.139 | INFO | "gpu" : " 17 GB", | 20260505 12:49:31.139 | INFO | "huge" : "", | 20260505 12:49:31.139 | INFO | "huge_shared" : " 27 GB", | 20260505 12:49:31.139 | INFO | "malloc" : " 14 GB", | 20260505 12:49:31.139 | INFO | "pinned" : " 29 GB" | 20260505 12:49:31.139 | INFO | } | 20260505 12:49:31.139 | INFO | Perf report for the last 5.14638 seconds: | 20260505 12:49:31.139 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:49:31.139 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:31.139 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:31.139 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:31.139 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 12:49:31.139 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:31.139 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:31.139 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 12:49:31.139 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:31.139 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:31.140 | INFO | Uploaded 36GiB to GPU in 2.20248s (16.3452GiB/s) | 20260505 12:49:31.140 | INFO | Downloaded 34GiB to GPU in 2.12063s (16.0329GiB/s) | 20260505 12:49:36.279 | INFO | Analyzing chunk frames = [3335680,3336192) | 20260505 12:49:36.279 | INFO | MemInfo: { | 20260505 12:49:36.279 | INFO | "gpu" : " 17 GB", | 20260505 12:49:36.279 | INFO | "huge" : "", | 20260505 12:49:36.279 | INFO | "huge_shared" : " 27 GB", | 20260505 12:49:36.279 | INFO | "malloc" : " 14 GB", | 20260505 12:49:36.279 | INFO | "pinned" : " 29 GB" | 20260505 12:49:36.279 | INFO | } | 20260505 12:49:36.279 | INFO | Perf report for the last 5.13973 seconds: | 20260505 12:49:36.279 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:49:36.279 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:36.279 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:36.279 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:36.279 | INFO | ANALYSIS : 63.1%, 3.0, 98.0ms, 0 | 20260505 12:49:36.279 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:36.279 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:36.279 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:49:36.279 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:36.279 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:36.279 | INFO | Uploaded 36GiB to GPU in 2.19853s (16.3746GiB/s) | 20260505 12:49:36.279 | INFO | Downloaded 34GiB to GPU in 2.13468s (15.9274GiB/s) | 20260505 12:49:41.407 | INFO | Analyzing chunk frames = [3336192,3336704) | 20260505 12:49:41.407 | INFO | MemInfo: { | 20260505 12:49:41.407 | INFO | "gpu" : " 17 GB", | 20260505 12:49:41.407 | INFO | "huge" : "", | 20260505 12:49:41.408 | INFO | "huge_shared" : " 27 GB", | 20260505 12:49:41.408 | INFO | "malloc" : " 14 GB", | 20260505 12:49:41.408 | INFO | "pinned" : " 29 GB" | 20260505 12:49:41.408 | INFO | } | 20260505 12:49:41.408 | INFO | Perf report for the last 5.12858 seconds: | 20260505 12:49:41.408 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:49:41.408 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:41.408 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:41.408 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:41.408 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 12:49:41.408 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:41.408 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:41.408 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 12:49:41.408 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:41.408 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:41.408 | INFO | Uploaded 36GiB to GPU in 2.15023s (16.7424GiB/s) | 20260505 12:49:41.408 | INFO | Downloaded 34GiB to GPU in 2.12541s (15.9969GiB/s) | 20260505 12:49:46.548 | INFO | Analyzing chunk frames = [3336704,3337216) | 20260505 12:49:46.548 | INFO | MemInfo: { | 20260505 12:49:46.548 | INFO | "gpu" : " 17 GB", | 20260505 12:49:46.548 | INFO | "huge" : "", | 20260505 12:49:46.548 | INFO | "huge_shared" : " 27 GB", | 20260505 12:49:46.548 | INFO | "malloc" : " 14 GB", | 20260505 12:49:46.548 | INFO | "pinned" : " 29 GB" | 20260505 12:49:46.548 | INFO | } | 20260505 12:49:46.548 | INFO | Perf report for the last 5.14087 seconds: | 20260505 12:49:46.549 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:49:46.549 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:46.549 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:46.549 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:46.549 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 12:49:46.549 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:46.549 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:46.549 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:49:46.549 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:46.549 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:46.549 | INFO | Uploaded 36GiB to GPU in 2.19092s (16.4315GiB/s) | 20260505 12:49:46.549 | INFO | Downloaded 34GiB to GPU in 2.12122s (16.0285GiB/s) | 20260505 12:49:51.696 | INFO | Analyzing chunk frames = [3337216,3337728) | 20260505 12:49:51.696 | INFO | MemInfo: { | 20260505 12:49:51.696 | INFO | "gpu" : " 17 GB", | 20260505 12:49:51.696 | INFO | "huge" : "", | 20260505 12:49:51.696 | INFO | "huge_shared" : " 27 GB", | 20260505 12:49:51.696 | INFO | "malloc" : " 14 GB", | 20260505 12:49:51.696 | INFO | "pinned" : " 29 GB" | 20260505 12:49:51.696 | INFO | } | 20260505 12:49:51.696 | INFO | Perf report for the last 5.14793 seconds: | 20260505 12:49:51.696 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:49:51.696 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:51.696 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:51.696 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:51.696 | INFO | ANALYSIS : 63.3%, 3.0, 99.0ms, 0 | 20260505 12:49:51.696 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:51.697 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:51.697 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 12:49:51.697 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:51.697 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:51.697 | INFO | Uploaded 36GiB to GPU in 2.16835s (16.6025GiB/s) | 20260505 12:49:51.697 | INFO | Downloaded 34GiB to GPU in 2.14922s (15.8197GiB/s) | 20260505 12:49:56.837 | INFO | Analyzing chunk frames = [3337728,3338240) | 20260505 12:49:56.837 | INFO | MemInfo: { | 20260505 12:49:56.837 | INFO | "gpu" : " 17 GB", | 20260505 12:49:56.837 | INFO | "huge" : "", | 20260505 12:49:56.837 | INFO | "huge_shared" : " 27 GB", | 20260505 12:49:56.837 | INFO | "malloc" : " 14 GB", | 20260505 12:49:56.837 | INFO | "pinned" : " 29 GB" | 20260505 12:49:56.837 | INFO | } | 20260505 12:49:56.837 | INFO | Perf report for the last 5.14067 seconds: | 20260505 12:49:56.837 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:49:56.837 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:56.837 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:56.837 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:56.837 | INFO | ANALYSIS : 63.5%, 3.0, 98.1ms, 0 | 20260505 12:49:56.837 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:56.837 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:56.837 | INFO | PRE_HQ : 28.8%, 1.0, 14.8ms, 0 | 20260505 12:49:56.837 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:56.837 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:49:56.837 | INFO | Uploaded 36GiB to GPU in 2.11621s (17.0115GiB/s) | 20260505 12:49:56.837 | INFO | Downloaded 34GiB to GPU in 2.09687s (16.2146GiB/s) | 20260505 12:49:56.976 | INFO | Starting a round of Baz IO | 20260505 12:50:00.207 | INFO | Populating RTMetrics report | 20260505 12:50:00.207 | INFO | RTMetrics report populated | 20260505 12:50:00.207 | INFO | Opening tmp RTMetrics json file | 20260505 12:50:00.231 | INFO | Serializing RTMetrics to json | 20260505 12:50:00.232 | INFO | Serialization done, swapping tmp file now... | 20260505 12:50:00.232 | INFO | RTMetrics done | 20260505 12:50:00.326 | INFO | Finished a round of Baz IO: Wrote 2.326 GiB in 3.349 seconds | 20260505 12:50:00.326 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.39% / 0.01% / 76.61% | 20260505 12:50:00.326 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.81375% / 7.94729e-06% / 7.15256e-05% | 20260505 12:50:00.326 | INFO | Baz IO overall: | 20260505 12:50:00.326 | INFO | BytesWritten : 1985932852461 | 20260505 12:50:00.326 | INFO | NumEvents : 945131819039 | 20260505 12:50:01.973 | INFO | Analyzing chunk frames = [3338240,3338752) | 20260505 12:50:01.973 | INFO | MemInfo: { | 20260505 12:50:01.973 | INFO | "gpu" : " 17 GB", | 20260505 12:50:01.973 | INFO | "huge" : "", | 20260505 12:50:01.973 | INFO | "huge_shared" : " 27 GB", | 20260505 12:50:01.973 | INFO | "malloc" : " 14 GB", | 20260505 12:50:01.973 | INFO | "pinned" : " 29 GB" | 20260505 12:50:01.973 | INFO | } | 20260505 12:50:01.973 | INFO | Perf report for the last 5.13568 seconds: | 20260505 12:50:01.973 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:50:01.973 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:01.973 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:01.973 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:01.973 | INFO | ANALYSIS : 64.5%, 2.9, 99.6ms, 0 | 20260505 12:50:01.973 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:01.973 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260505 12:50:01.973 | INFO | PRE_HQ : 61.2%, 1.8, 58.1ms, 0 | 20260505 12:50:01.973 | INFO | BAZWRITER : 64.6%, 4.1, 134.6ms, 0 | 20260505 12:50:01.973 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260505 12:50:01.973 | INFO | Uploaded 36GiB to GPU in 2.18851s (16.4495GiB/s) | 20260505 12:50:01.973 | INFO | Downloaded 38GiB to GPU in 2.27113s (16.7318GiB/s) | 20260505 12:50:07.106 | INFO | Analyzing chunk frames = [3338752,3339264) | 20260505 12:50:07.106 | INFO | MemInfo: { | 20260505 12:50:07.106 | INFO | "gpu" : " 17 GB", | 20260505 12:50:07.106 | INFO | "huge" : "", | 20260505 12:50:07.106 | INFO | "huge_shared" : " 27 GB", | 20260505 12:50:07.106 | INFO | "malloc" : " 14 GB", | 20260505 12:50:07.106 | INFO | "pinned" : " 29 GB" | 20260505 12:50:07.106 | INFO | } | 20260505 12:50:07.107 | INFO | Perf report for the last 5.1337 seconds: | 20260505 12:50:07.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:50:07.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:07.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:07.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:07.107 | INFO | ANALYSIS : 63.8%, 3.0, 99.5ms, 0 | 20260505 12:50:07.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:07.107 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:07.107 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:50:07.107 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:07.107 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:07.107 | INFO | Uploaded 36GiB to GPU in 2.19921s (16.3695GiB/s) | 20260505 12:50:07.107 | INFO | Downloaded 34GiB to GPU in 2.05603s (16.5367GiB/s) | 20260505 12:50:12.247 | INFO | Analyzing chunk frames = [3339264,3339776) | 20260505 12:50:12.247 | INFO | MemInfo: { | 20260505 12:50:12.247 | INFO | "gpu" : " 17 GB", | 20260505 12:50:12.247 | INFO | "huge" : "", | 20260505 12:50:12.247 | INFO | "huge_shared" : " 27 GB", | 20260505 12:50:12.247 | INFO | "malloc" : " 14 GB", | 20260505 12:50:12.247 | INFO | "pinned" : " 29 GB" | 20260505 12:50:12.247 | INFO | } | 20260505 12:50:12.247 | INFO | Perf report for the last 5.14039 seconds: | 20260505 12:50:12.247 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:50:12.247 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:12.247 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:12.247 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:12.247 | INFO | ANALYSIS : 63.6%, 3.0, 99.0ms, 0 | 20260505 12:50:12.247 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:12.247 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:12.247 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:50:12.247 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:12.247 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:12.248 | INFO | Uploaded 36GiB to GPU in 2.22652s (16.1688GiB/s) | 20260505 12:50:12.248 | INFO | Downloaded 34GiB to GPU in 2.13116s (15.9537GiB/s) | 20260505 12:50:17.388 | INFO | Analyzing chunk frames = [3339776,3340288) | 20260505 12:50:17.388 | INFO | MemInfo: { | 20260505 12:50:17.388 | INFO | "gpu" : " 17 GB", | 20260505 12:50:17.388 | INFO | "huge" : "", | 20260505 12:50:17.389 | INFO | "huge_shared" : " 27 GB", | 20260505 12:50:17.389 | INFO | "malloc" : " 14 GB", | 20260505 12:50:17.389 | INFO | "pinned" : " 29 GB" | 20260505 12:50:17.389 | INFO | } | 20260505 12:50:17.389 | INFO | Perf report for the last 5.14169 seconds: | 20260505 12:50:17.389 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:50:17.389 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:17.389 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:17.389 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:17.389 | INFO | ANALYSIS : 63.3%, 3.0, 98.3ms, 0 | 20260505 12:50:17.389 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:17.389 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:17.389 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 12:50:17.389 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:17.389 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:17.389 | INFO | Uploaded 36GiB to GPU in 2.21265s (16.2701GiB/s) | 20260505 12:50:17.389 | INFO | Downloaded 34GiB to GPU in 2.13473s (15.9271GiB/s) | 20260505 12:50:22.534 | INFO | Analyzing chunk frames = [3340288,3340800) | 20260505 12:50:22.534 | INFO | MemInfo: { | 20260505 12:50:22.534 | INFO | "gpu" : " 17 GB", | 20260505 12:50:22.534 | INFO | "huge" : "", | 20260505 12:50:22.534 | INFO | "huge_shared" : " 27 GB", | 20260505 12:50:22.534 | INFO | "malloc" : " 14 GB", | 20260505 12:50:22.534 | INFO | "pinned" : " 29 GB" | 20260505 12:50:22.534 | INFO | } | 20260505 12:50:22.534 | INFO | Perf report for the last 5.14568 seconds: | 20260505 12:50:22.534 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:50:22.534 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:22.534 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:22.534 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:22.534 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 12:50:22.534 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:22.534 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:22.534 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 12:50:22.534 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:22.534 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:22.535 | INFO | Uploaded 36GiB to GPU in 2.15753s (16.6857GiB/s) | 20260505 12:50:22.535 | INFO | Downloaded 34GiB to GPU in 2.12735s (15.9824GiB/s) | 20260505 12:50:27.672 | INFO | Analyzing chunk frames = [3340800,3341312) | 20260505 12:50:27.672 | INFO | MemInfo: { | 20260505 12:50:27.673 | INFO | "gpu" : " 17 GB", | 20260505 12:50:27.673 | INFO | "huge" : "", | 20260505 12:50:27.673 | INFO | "huge_shared" : " 27 GB", | 20260505 12:50:27.673 | INFO | "malloc" : " 14 GB", | 20260505 12:50:27.673 | INFO | "pinned" : " 29 GB" | 20260505 12:50:27.673 | INFO | } | 20260505 12:50:27.673 | INFO | Perf report for the last 5.13833 seconds: | 20260505 12:50:27.673 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:50:27.673 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:27.673 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:27.673 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:27.673 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 12:50:27.673 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:27.673 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:27.673 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 12:50:27.673 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:27.673 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:27.673 | INFO | Uploaded 36GiB to GPU in 2.16351s (16.6396GiB/s) | 20260505 12:50:27.673 | INFO | Downloaded 34GiB to GPU in 2.10935s (16.1187GiB/s) | 20260505 12:50:32.807 | INFO | Analyzing chunk frames = [3341312,3341824) | 20260505 12:50:32.807 | INFO | MemInfo: { | 20260505 12:50:32.807 | INFO | "gpu" : " 17 GB", | 20260505 12:50:32.807 | INFO | "huge" : "", | 20260505 12:50:32.807 | INFO | "huge_shared" : " 27 GB", | 20260505 12:50:32.807 | INFO | "malloc" : " 14 GB", | 20260505 12:50:32.807 | INFO | "pinned" : " 29 GB" | 20260505 12:50:32.807 | INFO | } | 20260505 12:50:32.807 | INFO | Perf report for the last 5.13421 seconds: | 20260505 12:50:32.807 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:50:32.807 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:32.807 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:32.807 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:32.807 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 12:50:32.807 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:32.807 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:32.807 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 12:50:32.807 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:32.807 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:32.807 | INFO | Uploaded 36GiB to GPU in 2.17375s (16.5613GiB/s) | 20260505 12:50:32.807 | INFO | Downloaded 34GiB to GPU in 2.13573s (15.9196GiB/s) | 20260505 12:50:37.949 | INFO | Analyzing chunk frames = [3341824,3342336) | 20260505 12:50:37.949 | INFO | MemInfo: { | 20260505 12:50:37.949 | INFO | "gpu" : " 17 GB", | 20260505 12:50:37.949 | INFO | "huge" : "", | 20260505 12:50:37.949 | INFO | "huge_shared" : " 27 GB", | 20260505 12:50:37.949 | INFO | "malloc" : " 14 GB", | 20260505 12:50:37.949 | INFO | "pinned" : " 29 GB" | 20260505 12:50:37.949 | INFO | } | 20260505 12:50:37.949 | INFO | Perf report for the last 5.14204 seconds: | 20260505 12:50:37.949 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:50:37.949 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:37.949 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:37.949 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:37.949 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 12:50:37.949 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:37.949 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:37.949 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:50:37.949 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:37.949 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:37.949 | INFO | Uploaded 36GiB to GPU in 2.19196s (16.4237GiB/s) | 20260505 12:50:37.949 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 12:50:37.949 | INFO | Total mem usage: 15.3 GiB | 20260505 12:50:37.949 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 12:50:37.949 | INFO | Downloaded 34GiB to GPU in 2.10804s (16.1287GiB/s) | 20260505 12:50:37.949 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -6.92%) | 20260505 12:50:37.949 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 12:50:37.949 | INFO | High Water Breakdown: | 20260505 12:50:37.949 | INFO | 61.1% : 8.76 GiB ( 92% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 12:50:37.949 | INFO | 38.8% : 5.56 GiB ( 83.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 12:50:37.949 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 12:50:37.949 | INFO | ------------------------------------------ | 20260505 12:50:37.949 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:50:37.949 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 12:50:37.949 | INFO | Total mem usage: 26.5 GiB | 20260505 12:50:37.949 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 12:50:37.949 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 12:50:37.949 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 12:50:37.949 | INFO | High Water Breakdown: | 20260505 12:50:37.950 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:50:37.950 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:50:37.950 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:50:37.950 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:50:37.950 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:50:37.950 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:50:37.950 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 12:50:37.950 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 12:50:37.950 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:50:37.950 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 12:50:37.950 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 12:50:37.950 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 12:50:37.950 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 12:50:37.950 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 12:50:37.950 | INFO | ------------------------------------------ | 20260505 12:50:37.950 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:50:37.950 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 12:50:37.950 | INFO | Total mem usage: 25.6 GiB | 20260505 12:50:37.950 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 12:50:37.950 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 12:50:37.950 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 12:50:37.950 | INFO | High Water Breakdown: | 20260505 12:50:37.950 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 12:50:37.950 | INFO | ------------------------------------------ | 20260505 12:50:37.950 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:50:37.950 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 12:50:37.950 | INFO | Total mem usage: 15.7 GiB | 20260505 12:50:37.950 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 12:50:37.950 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 12:50:37.950 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 12:50:37.950 | INFO | High Water Breakdown: | 20260505 12:50:37.950 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 12:50:37.950 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 12:50:37.950 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 12:50:37.950 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 12:50:37.950 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 12:50:37.950 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 12:50:37.950 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 12:50:37.950 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 12:50:37.950 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 12:50:37.950 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:50:37.950 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 12:50:37.950 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:50:37.950 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 12:50:37.950 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 12:50:37.950 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 12:50:37.950 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 12:50:37.950 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:50:37.950 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 12:50:37.950 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:50:37.950 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:50:37.950 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:50:37.950 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 12:50:37.950 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 12:50:37.950 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:50:37.950 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 12:50:37.950 | INFO | ------------------------------------------ | 20260505 12:50:37.950 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:50:37.950 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 12:50:37.950 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:50:37.950 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:50:37.950 | INFO | PerfReport: Full Execution -- 3.20e+01 / 7.69e+01, 1.37e+01 / 1.67e+00 | 20260505 12:50:37.950 | INFO | PerfReport: 0.29%, AggregateStats -- 9.26e-02 / 1.01e-01, 1.07e-02 / 6.45e-03 | 20260505 12:50:37.950 | INFO | PerfReport: 11.74%, Baseline -- 3.76e+00 / 4.49e+00, 5.91e-01 / 1.96e-01 | 20260505 12:50:37.950 | INFO | PerfReport: 3.31%, Histogram -- 1.06e+00 / 1.48e+00, 2.03e-01 / 1.03e-01 | 20260505 12:50:37.950 | INFO | PerfReport: 15.87%, DME -- 4.06e+01 / 4.52e+01, 4.20e+00 / 2.04e+00 | 20260505 12:50:37.950 | INFO | PerfReport: 56.53%, FrameLabeling -- 1.81e+01 / 2.08e+01, 1.56e+00 / 2.05e+00 | 20260505 12:50:37.950 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.36e+00 / 2.87e+00, 4.31e-01 / 2.22e-01 | 20260505 12:50:37.950 | INFO | PerfReport: 3.61%, Metrics -- 1.16e+00 / 1.45e+00, 3.26e-01 / 1.52e-01 | 20260505 12:50:37.950 | INFO | PerfReport: 1.27%, PulseSerializing -- 4.06e-01 / 4.57e-01, 7.33e-02 / 2.76e-02 | 20260505 12:50:37.950 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 12:50:37.950 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:50:37.950 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:50:37.950 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260505 12:50:37.950 | INFO | PerfReport: 22.53%, Upload -- 2.28e+01 / 0.00e+00, 8.10e+00 / 0.00e+00 | 20260505 12:50:37.950 | INFO | PerfReport: 22.03%, Download -- 2.23e+01 / 0.00e+00, 8.16e+00 / 0.00e+00 | 20260505 12:50:37.950 | INFO | PerfReport: 31.68%, Compute -- 3.20e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 12:50:37.950 | INFO | PerfReport: 2.13%, LockUpload -- 2.15e+00 / 0.00e+00, 7.67e+00 / 0.00e+00 | 20260505 12:50:37.950 | INFO | PerfReport: 1.34%, LockDownload -- 1.36e+00 / 0.00e+00, 2.92e+00 / 0.00e+00 | 20260505 12:50:37.950 | INFO | PerfReport: 20.27%, LockCompute -- 2.05e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260505 12:50:37.950 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.89e-02 / 0.00e+00 | 20260505 12:50:38.084 | INFO | Starting a round of Baz IO | 20260505 12:50:41.327 | INFO | Populating RTMetrics report | 20260505 12:50:41.327 | INFO | RTMetrics report populated | 20260505 12:50:41.327 | INFO | Opening tmp RTMetrics json file | 20260505 12:50:41.351 | INFO | Serializing RTMetrics to json | 20260505 12:50:41.357 | INFO | Serialization done, swapping tmp file now... | 20260505 12:50:41.359 | INFO | RTMetrics done | 20260505 12:50:41.444 | INFO | Finished a round of Baz IO: Wrote 2.326 GiB in 3.36 seconds | 20260505 12:50:41.444 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.38% / 0.01% / 76.61% | 20260505 12:50:41.444 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.811867% / 1.19209e-05% / 8.74201e-05% | 20260505 12:50:41.445 | INFO | Baz IO overall: | 20260505 12:50:41.445 | INFO | BytesWritten : 1988430345406 | 20260505 12:50:41.445 | INFO | NumEvents : 946315249522 | 20260505 12:50:43.085 | INFO | Analyzing chunk frames = [3342336,3342848) | 20260505 12:50:43.085 | INFO | MemInfo: { | 20260505 12:50:43.085 | INFO | "gpu" : " 17 GB", | 20260505 12:50:43.085 | INFO | "huge" : "", | 20260505 12:50:43.085 | INFO | "huge_shared" : " 27 GB", | 20260505 12:50:43.085 | INFO | "malloc" : " 14 GB", | 20260505 12:50:43.085 | INFO | "pinned" : " 29 GB" | 20260505 12:50:43.085 | INFO | } | 20260505 12:50:43.085 | INFO | Perf report for the last 5.13648 seconds: | 20260505 12:50:43.085 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:50:43.085 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:43.085 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:43.085 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:43.086 | INFO | ANALYSIS : 64.7%, 3.0, 100.0ms, 0 | 20260505 12:50:43.086 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:43.086 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260505 12:50:43.086 | INFO | PRE_HQ : 60.9%, 1.8, 57.9ms, 0 | 20260505 12:50:43.086 | INFO | BAZWRITER : 64.8%, 4.1, 134.8ms, 0 | 20260505 12:50:43.086 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 12:50:43.086 | INFO | Uploaded 36GiB to GPU in 2.20134s (16.3537GiB/s) | 20260505 12:50:43.086 | INFO | Downloaded 38GiB to GPU in 2.28612s (16.6221GiB/s) | 20260505 12:50:48.227 | INFO | Analyzing chunk frames = [3342848,3343360) | 20260505 12:50:48.227 | INFO | MemInfo: { | 20260505 12:50:48.227 | INFO | "gpu" : " 17 GB", | 20260505 12:50:48.227 | INFO | "huge" : "", | 20260505 12:50:48.227 | INFO | "huge_shared" : " 27 GB", | 20260505 12:50:48.227 | INFO | "malloc" : " 14 GB", | 20260505 12:50:48.227 | INFO | "pinned" : " 29 GB" | 20260505 12:50:48.227 | INFO | } | 20260505 12:50:48.227 | INFO | Perf report for the last 5.14184 seconds: | 20260505 12:50:48.227 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:50:48.227 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:48.227 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:48.227 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:48.227 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 12:50:48.227 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:48.227 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:48.227 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:50:48.227 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:48.227 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:48.228 | INFO | Uploaded 36GiB to GPU in 2.18116s (16.505GiB/s) | 20260505 12:50:48.228 | INFO | Downloaded 34GiB to GPU in 2.05476s (16.5469GiB/s) | 20260505 12:50:53.361 | INFO | Analyzing chunk frames = [3343360,3343872) | 20260505 12:50:53.361 | INFO | MemInfo: { | 20260505 12:50:53.361 | INFO | "gpu" : " 17 GB", | 20260505 12:50:53.361 | INFO | "huge" : "", | 20260505 12:50:53.361 | INFO | "huge_shared" : " 27 GB", | 20260505 12:50:53.362 | INFO | "malloc" : " 14 GB", | 20260505 12:50:53.362 | INFO | "pinned" : " 29 GB" | 20260505 12:50:53.362 | INFO | } | 20260505 12:50:53.362 | INFO | Perf report for the last 5.13434 seconds: | 20260505 12:50:53.362 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:50:53.362 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:53.362 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:53.362 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:53.362 | INFO | ANALYSIS : 63.3%, 3.0, 98.3ms, 0 | 20260505 12:50:53.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:53.362 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:53.362 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 12:50:53.362 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:53.362 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:53.362 | INFO | Uploaded 36GiB to GPU in 2.16256s (16.6469GiB/s) | 20260505 12:50:53.362 | INFO | Downloaded 34GiB to GPU in 2.1216s (16.0257GiB/s) | 20260505 12:50:58.500 | INFO | Analyzing chunk frames = [3343872,3344384) | 20260505 12:50:58.500 | INFO | MemInfo: { | 20260505 12:50:58.500 | INFO | "gpu" : " 17 GB", | 20260505 12:50:58.500 | INFO | "huge" : "", | 20260505 12:50:58.500 | INFO | "huge_shared" : " 27 GB", | 20260505 12:50:58.500 | INFO | "malloc" : " 14 GB", | 20260505 12:50:58.500 | INFO | "pinned" : " 29 GB" | 20260505 12:50:58.500 | INFO | } | 20260505 12:50:58.500 | INFO | Perf report for the last 5.13866 seconds: | 20260505 12:50:58.500 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:50:58.500 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:58.500 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:58.500 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:58.500 | INFO | ANALYSIS : 63.4%, 3.0, 98.4ms, 0 | 20260505 12:50:58.500 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:58.500 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:58.500 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 12:50:58.500 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:58.500 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:50:58.501 | INFO | Uploaded 36GiB to GPU in 2.21171s (16.277GiB/s) | 20260505 12:50:58.501 | INFO | Downloaded 34GiB to GPU in 2.12943s (15.9667GiB/s) | 20260505 12:51:03.646 | INFO | Analyzing chunk frames = [3344384,3344896) | 20260505 12:51:03.646 | INFO | MemInfo: { | 20260505 12:51:03.646 | INFO | "gpu" : " 17 GB", | 20260505 12:51:03.646 | INFO | "huge" : "", | 20260505 12:51:03.646 | INFO | "huge_shared" : " 27 GB", | 20260505 12:51:03.646 | INFO | "malloc" : " 14 GB", | 20260505 12:51:03.646 | INFO | "pinned" : " 29 GB" | 20260505 12:51:03.646 | INFO | } | 20260505 12:51:03.646 | INFO | Perf report for the last 5.14549 seconds: | 20260505 12:51:03.646 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:51:03.646 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:03.646 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:03.646 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:03.646 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 12:51:03.646 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:03.646 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:03.646 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 12:51:03.646 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:03.646 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:03.646 | INFO | Uploaded 36GiB to GPU in 2.13575s (16.8559GiB/s) | 20260505 12:51:03.646 | INFO | Downloaded 34GiB to GPU in 2.12654s (15.9884GiB/s) | 20260505 12:51:08.789 | INFO | Analyzing chunk frames = [3344896,3345408) | 20260505 12:51:08.789 | INFO | MemInfo: { | 20260505 12:51:08.789 | INFO | "gpu" : " 17 GB", | 20260505 12:51:08.789 | INFO | "huge" : "", | 20260505 12:51:08.789 | INFO | "huge_shared" : " 27 GB", | 20260505 12:51:08.789 | INFO | "malloc" : " 14 GB", | 20260505 12:51:08.789 | INFO | "pinned" : " 29 GB" | 20260505 12:51:08.789 | INFO | } | 20260505 12:51:08.789 | INFO | Perf report for the last 5.14364 seconds: | 20260505 12:51:08.789 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:51:08.789 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:08.789 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:08.789 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:08.789 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 12:51:08.790 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:08.790 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:08.790 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 12:51:08.790 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:08.790 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:08.790 | INFO | Uploaded 36GiB to GPU in 2.13203s (16.8853GiB/s) | 20260505 12:51:08.790 | INFO | Downloaded 34GiB to GPU in 2.10407s (16.1591GiB/s) | 20260505 12:51:13.922 | INFO | Analyzing chunk frames = [3345408,3345920) | 20260505 12:51:13.922 | INFO | MemInfo: { | 20260505 12:51:13.922 | INFO | "gpu" : " 17 GB", | 20260505 12:51:13.922 | INFO | "huge" : "", | 20260505 12:51:13.922 | INFO | "huge_shared" : " 27 GB", | 20260505 12:51:13.922 | INFO | "malloc" : " 14 GB", | 20260505 12:51:13.922 | INFO | "pinned" : " 29 GB" | 20260505 12:51:13.922 | INFO | } | 20260505 12:51:13.922 | INFO | Perf report for the last 5.13304 seconds: | 20260505 12:51:13.923 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:51:13.923 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:13.923 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:13.923 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:13.923 | INFO | ANALYSIS : 63.5%, 3.0, 99.0ms, 0 | 20260505 12:51:13.923 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:13.923 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:13.923 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 12:51:13.923 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:13.923 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:13.923 | INFO | Uploaded 36GiB to GPU in 2.14689s (16.7685GiB/s) | 20260505 12:51:13.923 | INFO | Downloaded 34GiB to GPU in 2.17488s (15.633GiB/s) | 20260505 12:51:19.051 | INFO | Analyzing chunk frames = [3345920,3346432) | 20260505 12:51:19.051 | INFO | MemInfo: { | 20260505 12:51:19.051 | INFO | "gpu" : " 17 GB", | 20260505 12:51:19.051 | INFO | "huge" : "", | 20260505 12:51:19.051 | INFO | "huge_shared" : " 27 GB", | 20260505 12:51:19.051 | INFO | "malloc" : " 14 GB", | 20260505 12:51:19.051 | INFO | "pinned" : " 29 GB" | 20260505 12:51:19.051 | INFO | } | 20260505 12:51:19.051 | INFO | Perf report for the last 5.12884 seconds: | 20260505 12:51:19.051 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:51:19.051 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:19.051 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:19.051 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:19.051 | INFO | ANALYSIS : 64.0%, 3.0, 99.8ms, 0 | 20260505 12:51:19.051 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:19.051 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:19.051 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 12:51:19.051 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:19.051 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:19.052 | INFO | Uploaded 36GiB to GPU in 2.21614s (16.2444GiB/s) | 20260505 12:51:19.052 | INFO | Downloaded 34GiB to GPU in 2.09733s (16.2111GiB/s) | 20260505 12:51:19.190 | INFO | Starting a round of Baz IO | 20260505 12:51:22.420 | INFO | Populating RTMetrics report | 20260505 12:51:22.420 | INFO | RTMetrics report populated | 20260505 12:51:22.420 | INFO | Opening tmp RTMetrics json file | 20260505 12:51:22.451 | INFO | Serializing RTMetrics to json | 20260505 12:51:22.489 | INFO | Serialization done, swapping tmp file now... | 20260505 12:51:22.492 | INFO | RTMetrics done | 20260505 12:51:22.561 | INFO | Finished a round of Baz IO: Wrote 2.327 GiB in 3.371 seconds | 20260505 12:51:22.561 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.38% / 0.01% / 76.62% | 20260505 12:51:22.561 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.81203% / 0% / 7.94729e-05% | 20260505 12:51:22.561 | INFO | Baz IO overall: | 20260505 12:51:22.561 | INFO | BytesWritten : 1990928417486 | 20260505 12:51:22.561 | INFO | NumEvents : 947499553507 | 20260505 12:51:24.204 | INFO | Analyzing chunk frames = [3346432,3346944) | 20260505 12:51:24.204 | INFO | MemInfo: { | 20260505 12:51:24.204 | INFO | "gpu" : " 17 GB", | 20260505 12:51:24.204 | INFO | "huge" : "", | 20260505 12:51:24.204 | INFO | "huge_shared" : " 27 GB", | 20260505 12:51:24.204 | INFO | "malloc" : " 14 GB", | 20260505 12:51:24.204 | INFO | "pinned" : " 29 GB" | 20260505 12:51:24.204 | INFO | } | 20260505 12:51:24.204 | INFO | Perf report for the last 5.15309 seconds: | 20260505 12:51:24.204 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:51:24.204 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:24.204 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:24.204 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:24.204 | INFO | ANALYSIS : 64.8%, 3.0, 100.6ms, 0 | 20260505 12:51:24.204 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:24.205 | INFO | RT_METRICS : 46.3%, 1.0, 24.4ms, 0 | 20260505 12:51:24.205 | INFO | PRE_HQ : 60.8%, 1.8, 57.8ms, 0 | 20260505 12:51:24.205 | INFO | BAZWRITER : 64.7%, 4.0, 135.0ms, 0 | 20260505 12:51:24.205 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260505 12:51:24.205 | INFO | Uploaded 36GiB to GPU in 2.22583s (16.1737GiB/s) | 20260505 12:51:24.205 | INFO | Downloaded 38GiB to GPU in 2.29217s (16.5782GiB/s) | 20260505 12:51:29.339 | INFO | Analyzing chunk frames = [3346944,3347456) | 20260505 12:51:29.339 | INFO | MemInfo: { | 20260505 12:51:29.339 | INFO | "gpu" : " 17 GB", | 20260505 12:51:29.339 | INFO | "huge" : "", | 20260505 12:51:29.339 | INFO | "huge_shared" : " 27 GB", | 20260505 12:51:29.339 | INFO | "malloc" : " 14 GB", | 20260505 12:51:29.339 | INFO | "pinned" : " 29 GB" | 20260505 12:51:29.339 | INFO | } | 20260505 12:51:29.339 | INFO | Perf report for the last 5.13439 seconds: | 20260505 12:51:29.339 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:51:29.339 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:29.339 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:29.339 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:29.339 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 12:51:29.339 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:29.339 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:29.339 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:51:29.339 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:29.339 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:29.339 | INFO | Uploaded 36GiB to GPU in 2.18039s (16.5108GiB/s) | 20260505 12:51:29.339 | INFO | Downloaded 34GiB to GPU in 2.0319s (16.7331GiB/s) | 20260505 12:51:34.479 | INFO | Analyzing chunk frames = [3347456,3347968) | 20260505 12:51:34.479 | INFO | MemInfo: { | 20260505 12:51:34.479 | INFO | "gpu" : " 17 GB", | 20260505 12:51:34.479 | INFO | "huge" : "", | 20260505 12:51:34.479 | INFO | "huge_shared" : " 27 GB", | 20260505 12:51:34.479 | INFO | "malloc" : " 14 GB", | 20260505 12:51:34.479 | INFO | "pinned" : " 29 GB" | 20260505 12:51:34.479 | INFO | } | 20260505 12:51:34.479 | INFO | Perf report for the last 5.13986 seconds: | 20260505 12:51:34.479 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:51:34.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:34.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:34.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:34.479 | INFO | ANALYSIS : 63.4%, 3.0, 98.6ms, 0 | 20260505 12:51:34.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:34.479 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:34.479 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:51:34.479 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:34.479 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:34.479 | INFO | Uploaded 36GiB to GPU in 2.19589s (16.3942GiB/s) | 20260505 12:51:34.480 | INFO | Downloaded 34GiB to GPU in 2.12293s (16.0156GiB/s) | 20260505 12:51:39.616 | INFO | Analyzing chunk frames = [3347968,3348480) | 20260505 12:51:39.616 | INFO | MemInfo: { | 20260505 12:51:39.616 | INFO | "gpu" : " 17 GB", | 20260505 12:51:39.616 | INFO | "huge" : "", | 20260505 12:51:39.616 | INFO | "huge_shared" : " 27 GB", | 20260505 12:51:39.616 | INFO | "malloc" : " 14 GB", | 20260505 12:51:39.616 | INFO | "pinned" : " 29 GB" | 20260505 12:51:39.616 | INFO | } | 20260505 12:51:39.616 | INFO | Perf report for the last 5.13754 seconds: | 20260505 12:51:39.616 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:51:39.616 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:39.616 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:39.616 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:39.616 | INFO | ANALYSIS : 63.0%, 3.0, 97.9ms, 0 | 20260505 12:51:39.616 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:39.616 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:39.616 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:51:39.616 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:39.616 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:39.616 | INFO | Uploaded 36GiB to GPU in 2.17314s (16.5659GiB/s) | 20260505 12:51:39.617 | INFO | Downloaded 34GiB to GPU in 2.13193s (15.948GiB/s) | 20260505 12:51:44.757 | INFO | Analyzing chunk frames = [3348480,3348992) | 20260505 12:51:44.757 | INFO | MemInfo: { | 20260505 12:51:44.757 | INFO | "gpu" : " 17 GB", | 20260505 12:51:44.757 | INFO | "huge" : "", | 20260505 12:51:44.757 | INFO | "huge_shared" : " 27 GB", | 20260505 12:51:44.757 | INFO | "malloc" : " 14 GB", | 20260505 12:51:44.757 | INFO | "pinned" : " 29 GB" | 20260505 12:51:44.758 | INFO | } | 20260505 12:51:44.758 | INFO | Perf report for the last 5.14127 seconds: | 20260505 12:51:44.758 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:51:44.758 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:44.758 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:44.758 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:44.758 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 12:51:44.758 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:44.758 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:44.758 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 12:51:44.758 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:44.758 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:44.758 | INFO | Uploaded 36GiB to GPU in 2.19956s (16.3669GiB/s) | 20260505 12:51:44.758 | INFO | Downloaded 34GiB to GPU in 2.12996s (15.9628GiB/s) | 20260505 12:51:49.899 | INFO | Analyzing chunk frames = [3348992,3349504) | 20260505 12:51:49.899 | INFO | MemInfo: { | 20260505 12:51:49.899 | INFO | "gpu" : " 17 GB", | 20260505 12:51:49.899 | INFO | "huge" : "", | 20260505 12:51:49.899 | INFO | "huge_shared" : " 27 GB", | 20260505 12:51:49.899 | INFO | "malloc" : " 14 GB", | 20260505 12:51:49.899 | INFO | "pinned" : " 29 GB" | 20260505 12:51:49.899 | INFO | } | 20260505 12:51:49.899 | INFO | Perf report for the last 5.14163 seconds: | 20260505 12:51:49.899 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:51:49.899 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:49.899 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:49.899 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:49.899 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 12:51:49.899 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:49.899 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:49.899 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 12:51:49.899 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:49.899 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:49.900 | INFO | Uploaded 36GiB to GPU in 2.14388s (16.792GiB/s) | 20260505 12:51:49.900 | INFO | Downloaded 34GiB to GPU in 2.10466s (16.1546GiB/s) | 20260505 12:51:55.035 | INFO | Analyzing chunk frames = [3349504,3350016) | 20260505 12:51:55.035 | INFO | MemInfo: { | 20260505 12:51:55.035 | INFO | "gpu" : " 17 GB", | 20260505 12:51:55.035 | INFO | "huge" : "", | 20260505 12:51:55.035 | INFO | "huge_shared" : " 27 GB", | 20260505 12:51:55.035 | INFO | "malloc" : " 14 GB", | 20260505 12:51:55.035 | INFO | "pinned" : " 29 GB" | 20260505 12:51:55.035 | INFO | } | 20260505 12:51:55.035 | INFO | Perf report for the last 5.13596 seconds: | 20260505 12:51:55.035 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:51:55.035 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:55.035 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:55.035 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:55.035 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 12:51:55.035 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:55.035 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:55.035 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 12:51:55.035 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:55.035 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:51:55.035 | INFO | Uploaded 36GiB to GPU in 2.16206s (16.6508GiB/s) | 20260505 12:51:55.036 | INFO | Downloaded 34GiB to GPU in 2.14774s (15.8306GiB/s) | 20260505 12:52:00.180 | INFO | Analyzing chunk frames = [3350016,3350528) | 20260505 12:52:00.180 | INFO | MemInfo: { | 20260505 12:52:00.180 | INFO | "gpu" : " 17 GB", | 20260505 12:52:00.180 | INFO | "huge" : "", | 20260505 12:52:00.180 | INFO | "huge_shared" : " 27 GB", | 20260505 12:52:00.180 | INFO | "malloc" : " 14 GB", | 20260505 12:52:00.180 | INFO | "pinned" : " 29 GB" | 20260505 12:52:00.180 | INFO | } | 20260505 12:52:00.180 | INFO | Perf report for the last 5.14493 seconds: | 20260505 12:52:00.180 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:52:00.180 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:00.180 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:00.180 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:00.180 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 12:52:00.180 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:00.180 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:00.180 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 12:52:00.180 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:00.180 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:00.180 | INFO | Uploaded 36GiB to GPU in 2.19767s (16.381GiB/s) | 20260505 12:52:00.180 | INFO | Downloaded 34GiB to GPU in 2.11745s (16.057GiB/s) | 20260505 12:52:00.180 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 12:52:00.180 | INFO | Total mem usage: 15.3 GiB | 20260505 12:52:00.180 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 12:52:00.180 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -7.53%) | 20260505 12:52:00.180 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 12:52:00.180 | INFO | High Water Breakdown: | 20260505 12:52:00.180 | INFO | 60.9% : 8.69 GiB ( 91.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 12:52:00.180 | INFO | 38.9% : 5.55 GiB ( 83.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 12:52:00.180 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 12:52:00.180 | INFO | ------------------------------------------ | 20260505 12:52:00.180 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:52:00.180 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 12:52:00.181 | INFO | Total mem usage: 26.5 GiB | 20260505 12:52:00.181 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 12:52:00.181 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 12:52:00.181 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 12:52:00.181 | INFO | High Water Breakdown: | 20260505 12:52:00.181 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:52:00.181 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:52:00.181 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:52:00.181 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:52:00.181 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:52:00.181 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:52:00.181 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 12:52:00.181 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 12:52:00.181 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:52:00.181 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 12:52:00.181 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 12:52:00.181 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 12:52:00.181 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 12:52:00.181 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 12:52:00.181 | INFO | ------------------------------------------ | 20260505 12:52:00.181 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:52:00.181 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 12:52:00.181 | INFO | Total mem usage: 25.6 GiB | 20260505 12:52:00.181 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 12:52:00.181 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 12:52:00.181 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 12:52:00.181 | INFO | High Water Breakdown: | 20260505 12:52:00.181 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 12:52:00.181 | INFO | ------------------------------------------ | 20260505 12:52:00.181 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:52:00.181 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 12:52:00.181 | INFO | Total mem usage: 15.7 GiB | 20260505 12:52:00.181 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 12:52:00.181 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 12:52:00.181 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 12:52:00.181 | INFO | High Water Breakdown: | 20260505 12:52:00.181 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 12:52:00.181 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 12:52:00.181 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 12:52:00.181 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 12:52:00.181 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 12:52:00.181 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 12:52:00.181 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 12:52:00.181 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 12:52:00.181 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 12:52:00.181 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:52:00.181 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 12:52:00.181 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:52:00.181 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 12:52:00.181 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 12:52:00.181 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 12:52:00.181 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 12:52:00.181 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:52:00.181 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 12:52:00.181 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:52:00.181 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:52:00.181 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:52:00.181 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 12:52:00.181 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 12:52:00.181 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:52:00.181 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 12:52:00.181 | INFO | ------------------------------------------ | 20260505 12:52:00.181 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:52:00.181 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 12:52:00.181 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:52:00.181 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:52:00.181 | INFO | PerfReport: Full Execution -- 3.20e+01 / 7.75e+01, 1.40e+01 / 2.11e+00 | 20260505 12:52:00.181 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 9.89e-02, 1.07e-02 / 3.74e-03 | 20260505 12:52:00.181 | INFO | PerfReport: 11.75%, Baseline -- 3.76e+00 / 4.48e+00, 5.99e-01 / 2.30e-01 | 20260505 12:52:00.181 | INFO | PerfReport: 3.32%, Histogram -- 1.06e+00 / 1.50e+00, 2.06e-01 / 9.61e-02 | 20260505 12:52:00.181 | INFO | PerfReport: 16.01%, DME -- 4.10e+01 / 4.64e+01, 4.78e+00 / 2.25e+00 | 20260505 12:52:00.181 | INFO | PerfReport: 56.39%, FrameLabeling -- 1.81e+01 / 2.03e+01, 1.49e+00 / 1.87e+00 | 20260505 12:52:00.182 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.36e+00 / 2.92e+00, 4.42e-01 / 1.80e-01 | 20260505 12:52:00.182 | INFO | PerfReport: 3.59%, Metrics -- 1.15e+00 / 1.45e+00, 3.24e-01 / 1.69e-01 | 20260505 12:52:00.182 | INFO | PerfReport: 1.27%, PulseSerializing -- 4.06e-01 / 4.53e-01, 7.48e-02 / 3.45e-02 | 20260505 12:52:00.182 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 12:52:00.182 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:52:00.182 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:52:00.182 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260505 12:52:00.182 | INFO | PerfReport: 22.47%, Upload -- 2.27e+01 / 0.00e+00, 8.03e+00 / 0.00e+00 | 20260505 12:52:00.182 | INFO | PerfReport: 22.02%, Download -- 2.22e+01 / 0.00e+00, 8.18e+00 / 0.00e+00 | 20260505 12:52:00.182 | INFO | PerfReport: 31.70%, Compute -- 3.20e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 12:52:00.182 | INFO | PerfReport: 2.12%, LockUpload -- 2.14e+00 / 0.00e+00, 7.74e+00 / 0.00e+00 | 20260505 12:52:00.182 | INFO | PerfReport: 1.43%, LockDownload -- 1.45e+00 / 0.00e+00, 3.01e+00 / 0.00e+00 | 20260505 12:52:00.182 | INFO | PerfReport: 20.25%, LockCompute -- 2.05e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260505 12:52:00.182 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.91e-02 / 0.00e+00 | 20260505 12:52:00.312 | INFO | Starting a round of Baz IO | 20260505 12:52:03.557 | INFO | Populating RTMetrics report | 20260505 12:52:03.558 | INFO | RTMetrics report populated | 20260505 12:52:03.558 | INFO | Opening tmp RTMetrics json file | 20260505 12:52:03.577 | INFO | Serializing RTMetrics to json | 20260505 12:52:03.601 | INFO | Serialization done, swapping tmp file now... | 20260505 12:52:03.624 | INFO | RTMetrics done | 20260505 12:52:03.665 | INFO | Finished a round of Baz IO: Wrote 2.324 GiB in 3.353 seconds | 20260505 12:52:03.665 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.37% / 0.01% / 76.62% | 20260505 12:52:03.665 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.807365% / 1.19209e-05% / 7.15256e-05% | 20260505 12:52:03.665 | INFO | Baz IO overall: | 20260505 12:52:03.665 | INFO | BytesWritten : 1993423786176 | 20260505 12:52:03.665 | INFO | NumEvents : 948682126449 | 20260505 12:52:05.312 | INFO | Analyzing chunk frames = [3350528,3351040) | 20260505 12:52:05.312 | INFO | MemInfo: { | 20260505 12:52:05.312 | INFO | "gpu" : " 17 GB", | 20260505 12:52:05.312 | INFO | "huge" : "", | 20260505 12:52:05.312 | INFO | "huge_shared" : " 27 GB", | 20260505 12:52:05.312 | INFO | "malloc" : " 14 GB", | 20260505 12:52:05.312 | INFO | "pinned" : " 29 GB" | 20260505 12:52:05.312 | INFO | } | 20260505 12:52:05.312 | INFO | Perf report for the last 5.13197 seconds: | 20260505 12:52:05.312 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:52:05.312 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:05.312 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:05.312 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:05.312 | INFO | ANALYSIS : 64.7%, 3.0, 100.2ms, 0 | 20260505 12:52:05.312 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:05.312 | INFO | RT_METRICS : 46.6%, 1.0, 24.4ms, 0 | 20260505 12:52:05.312 | INFO | PRE_HQ : 60.8%, 1.8, 58.0ms, 0 | 20260505 12:52:05.312 | INFO | BAZWRITER : 64.6%, 4.0, 134.2ms, 0 | 20260505 12:52:05.312 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 12:52:05.312 | INFO | Uploaded 36GiB to GPU in 2.19115s (16.4297GiB/s) | 20260505 12:52:05.312 | INFO | Downloaded 38GiB to GPU in 2.27767s (16.6837GiB/s) | 20260505 12:52:10.455 | INFO | Analyzing chunk frames = [3351040,3351552) | 20260505 12:52:10.455 | INFO | MemInfo: { | 20260505 12:52:10.455 | INFO | "gpu" : " 17 GB", | 20260505 12:52:10.455 | INFO | "huge" : "", | 20260505 12:52:10.455 | INFO | "huge_shared" : " 27 GB", | 20260505 12:52:10.455 | INFO | "malloc" : " 14 GB", | 20260505 12:52:10.455 | INFO | "pinned" : " 29 GB" | 20260505 12:52:10.455 | INFO | } | 20260505 12:52:10.455 | INFO | Perf report for the last 5.14289 seconds: | 20260505 12:52:10.455 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:52:10.455 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:10.455 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:10.455 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:10.455 | INFO | ANALYSIS : 63.7%, 3.0, 99.6ms, 0 | 20260505 12:52:10.455 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:10.455 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:10.455 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 12:52:10.455 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:10.455 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:10.456 | INFO | Uploaded 36GiB to GPU in 2.17794s (16.5294GiB/s) | 20260505 12:52:10.456 | INFO | Downloaded 34GiB to GPU in 2.0526s (16.5644GiB/s) | 20260505 12:52:15.597 | INFO | Analyzing chunk frames = [3351552,3352064) | 20260505 12:52:15.597 | INFO | MemInfo: { | 20260505 12:52:15.597 | INFO | "gpu" : " 17 GB", | 20260505 12:52:15.597 | INFO | "huge" : "", | 20260505 12:52:15.597 | INFO | "huge_shared" : " 27 GB", | 20260505 12:52:15.597 | INFO | "malloc" : " 14 GB", | 20260505 12:52:15.597 | INFO | "pinned" : " 29 GB" | 20260505 12:52:15.597 | INFO | } | 20260505 12:52:15.597 | INFO | Perf report for the last 5.14191 seconds: | 20260505 12:52:15.597 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:52:15.597 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:15.597 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:15.597 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:15.597 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 12:52:15.597 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:15.597 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:15.597 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:52:15.597 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:15.597 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:15.597 | INFO | Uploaded 36GiB to GPU in 2.22001s (16.2161GiB/s) | 20260505 12:52:15.597 | INFO | Downloaded 34GiB to GPU in 2.11604s (16.0677GiB/s) | 20260505 12:52:20.731 | INFO | Analyzing chunk frames = [3352064,3352576) | 20260505 12:52:20.731 | INFO | MemInfo: { | 20260505 12:52:20.731 | INFO | "gpu" : " 17 GB", | 20260505 12:52:20.731 | INFO | "huge" : "", | 20260505 12:52:20.731 | INFO | "huge_shared" : " 27 GB", | 20260505 12:52:20.731 | INFO | "malloc" : " 14 GB", | 20260505 12:52:20.731 | INFO | "pinned" : " 29 GB" | 20260505 12:52:20.731 | INFO | } | 20260505 12:52:20.731 | INFO | Perf report for the last 5.13374 seconds: | 20260505 12:52:20.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:52:20.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:20.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:20.731 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:20.731 | INFO | ANALYSIS : 62.8%, 3.0, 97.5ms, 0 | 20260505 12:52:20.731 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:20.731 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:20.731 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 12:52:20.731 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:20.731 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:20.731 | INFO | Uploaded 36GiB to GPU in 2.14607s (16.7748GiB/s) | 20260505 12:52:20.731 | INFO | Downloaded 34GiB to GPU in 2.13685s (15.9113GiB/s) | 20260505 12:52:25.866 | INFO | Analyzing chunk frames = [3352576,3353088) | 20260505 12:52:25.866 | INFO | MemInfo: { | 20260505 12:52:25.866 | INFO | "gpu" : " 17 GB", | 20260505 12:52:25.866 | INFO | "huge" : "", | 20260505 12:52:25.866 | INFO | "huge_shared" : " 27 GB", | 20260505 12:52:25.866 | INFO | "malloc" : " 14 GB", | 20260505 12:52:25.866 | INFO | "pinned" : " 29 GB" | 20260505 12:52:25.866 | INFO | } | 20260505 12:52:25.866 | INFO | Perf report for the last 5.13531 seconds: | 20260505 12:52:25.866 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:52:25.866 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:25.866 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:25.866 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:25.866 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 12:52:25.866 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:25.866 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:25.866 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:52:25.866 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:25.866 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:25.866 | INFO | Uploaded 36GiB to GPU in 2.17728s (16.5344GiB/s) | 20260505 12:52:25.866 | INFO | Downloaded 34GiB to GPU in 2.12968s (15.9648GiB/s) | 20260505 12:52:31.008 | INFO | Analyzing chunk frames = [3353088,3353600) | 20260505 12:52:31.008 | INFO | MemInfo: { | 20260505 12:52:31.008 | INFO | "gpu" : " 17 GB", | 20260505 12:52:31.008 | INFO | "huge" : "", | 20260505 12:52:31.008 | INFO | "huge_shared" : " 27 GB", | 20260505 12:52:31.008 | INFO | "malloc" : " 14 GB", | 20260505 12:52:31.008 | INFO | "pinned" : " 29 GB" | 20260505 12:52:31.008 | INFO | } | 20260505 12:52:31.008 | INFO | Perf report for the last 5.14197 seconds: | 20260505 12:52:31.008 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:52:31.008 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:31.008 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:31.008 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:31.008 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 12:52:31.008 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:31.008 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:31.008 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 12:52:31.008 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:31.008 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:31.008 | INFO | Uploaded 36GiB to GPU in 2.16036s (16.6639GiB/s) | 20260505 12:52:31.008 | INFO | Downloaded 34GiB to GPU in 2.12332s (16.0127GiB/s) | 20260505 12:52:36.151 | INFO | Analyzing chunk frames = [3353600,3354112) | 20260505 12:52:36.151 | INFO | MemInfo: { | 20260505 12:52:36.151 | INFO | "gpu" : " 17 GB", | 20260505 12:52:36.151 | INFO | "huge" : "", | 20260505 12:52:36.151 | INFO | "huge_shared" : " 27 GB", | 20260505 12:52:36.151 | INFO | "malloc" : " 14 GB", | 20260505 12:52:36.151 | INFO | "pinned" : " 29 GB" | 20260505 12:52:36.151 | INFO | } | 20260505 12:52:36.151 | INFO | Perf report for the last 5.1429 seconds: | 20260505 12:52:36.151 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:52:36.151 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:36.151 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:36.151 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:36.151 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 12:52:36.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:36.151 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:36.151 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 12:52:36.151 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:36.151 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:36.151 | INFO | Uploaded 36GiB to GPU in 2.17099s (16.5823GiB/s) | 20260505 12:52:36.151 | INFO | Downloaded 34GiB to GPU in 2.13915s (15.8942GiB/s) | 20260505 12:52:41.294 | INFO | Analyzing chunk frames = [3354112,3354624) | 20260505 12:52:41.294 | INFO | MemInfo: { | 20260505 12:52:41.294 | INFO | "gpu" : " 17 GB", | 20260505 12:52:41.294 | INFO | "huge" : "", | 20260505 12:52:41.294 | INFO | "huge_shared" : " 27 GB", | 20260505 12:52:41.294 | INFO | "malloc" : " 14 GB", | 20260505 12:52:41.294 | INFO | "pinned" : " 29 GB" | 20260505 12:52:41.294 | INFO | } | 20260505 12:52:41.294 | INFO | Perf report for the last 5.1433 seconds: | 20260505 12:52:41.294 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:52:41.294 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:41.294 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:41.294 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:41.294 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 12:52:41.294 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:41.294 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:41.294 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 12:52:41.294 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:41.294 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:41.294 | INFO | Uploaded 36GiB to GPU in 2.13275s (16.8796GiB/s) | 20260505 12:52:41.294 | INFO | Downloaded 34GiB to GPU in 2.10925s (16.1195GiB/s) | 20260505 12:52:41.429 | INFO | Starting a round of Baz IO | 20260505 12:52:44.664 | INFO | Populating RTMetrics report | 20260505 12:52:44.665 | INFO | RTMetrics report populated | 20260505 12:52:44.665 | INFO | Opening tmp RTMetrics json file | 20260505 12:52:44.676 | INFO | Serializing RTMetrics to json | 20260505 12:52:44.717 | INFO | Serialization done, swapping tmp file now... | 20260505 12:52:44.720 | INFO | RTMetrics done | 20260505 12:52:44.795 | INFO | Finished a round of Baz IO: Wrote 2.324 GiB in 3.365 seconds | 20260505 12:52:44.795 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.37% / 0.01% / 76.63% | 20260505 12:52:44.795 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.803097% / 7.94729e-06% / 7.15256e-05% | 20260505 12:52:44.795 | INFO | Baz IO overall: | 20260505 12:52:44.795 | INFO | BytesWritten : 1995918754524 | 20260505 12:52:44.795 | INFO | NumEvents : 949864625063 | 20260505 12:52:46.429 | INFO | Analyzing chunk frames = [3354624,3355136) | 20260505 12:52:46.429 | INFO | MemInfo: { | 20260505 12:52:46.429 | INFO | "gpu" : " 17 GB", | 20260505 12:52:46.429 | INFO | "huge" : "", | 20260505 12:52:46.429 | INFO | "huge_shared" : " 27 GB", | 20260505 12:52:46.430 | INFO | "malloc" : " 14 GB", | 20260505 12:52:46.430 | INFO | "pinned" : " 29 GB" | 20260505 12:52:46.430 | INFO | } | 20260505 12:52:46.430 | INFO | Perf report for the last 5.13533 seconds: | 20260505 12:52:46.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:52:46.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:46.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:46.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:46.430 | INFO | ANALYSIS : 64.5%, 2.9, 99.3ms, 0 | 20260505 12:52:46.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:46.430 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260505 12:52:46.430 | INFO | PRE_HQ : 61.2%, 1.8, 58.2ms, 0 | 20260505 12:52:46.430 | INFO | BAZWRITER : 64.8%, 4.1, 136.2ms, 0 | 20260505 12:52:46.430 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 12:52:46.430 | INFO | Uploaded 36GiB to GPU in 2.19662s (16.3888GiB/s) | 20260505 12:52:46.430 | INFO | Downloaded 38GiB to GPU in 2.2883s (16.6062GiB/s) | 20260505 12:52:51.564 | INFO | Analyzing chunk frames = [3355136,3355648) | 20260505 12:52:51.564 | INFO | MemInfo: { | 20260505 12:52:51.564 | INFO | "gpu" : " 17 GB", | 20260505 12:52:51.564 | INFO | "huge" : "", | 20260505 12:52:51.564 | INFO | "huge_shared" : " 27 GB", | 20260505 12:52:51.564 | INFO | "malloc" : " 14 GB", | 20260505 12:52:51.564 | INFO | "pinned" : " 29 GB" | 20260505 12:52:51.564 | INFO | } | 20260505 12:52:51.564 | INFO | Perf report for the last 5.13416 seconds: | 20260505 12:52:51.564 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:52:51.564 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:51.564 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:51.564 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:51.564 | INFO | ANALYSIS : 64.3%, 3.0, 100.3ms, 0 | 20260505 12:52:51.564 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:51.564 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:51.564 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 12:52:51.564 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:51.564 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:51.564 | INFO | Uploaded 36GiB to GPU in 2.23774s (16.0877GiB/s) | 20260505 12:52:51.564 | INFO | Downloaded 34GiB to GPU in 2.09889s (16.199GiB/s) | 20260505 12:52:56.709 | INFO | Analyzing chunk frames = [3355648,3356160) | 20260505 12:52:56.709 | INFO | MemInfo: { | 20260505 12:52:56.709 | INFO | "gpu" : " 17 GB", | 20260505 12:52:56.709 | INFO | "huge" : "", | 20260505 12:52:56.709 | INFO | "huge_shared" : " 27 GB", | 20260505 12:52:56.709 | INFO | "malloc" : " 14 GB", | 20260505 12:52:56.709 | INFO | "pinned" : " 29 GB" | 20260505 12:52:56.709 | INFO | } | 20260505 12:52:56.709 | INFO | Perf report for the last 5.14559 seconds: | 20260505 12:52:56.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:52:56.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:56.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:56.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:56.709 | INFO | ANALYSIS : 63.2%, 3.0, 98.4ms, 0 | 20260505 12:52:56.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:56.709 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:56.709 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:52:56.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:56.709 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:52:56.710 | INFO | Uploaded 36GiB to GPU in 2.17881s (16.5228GiB/s) | 20260505 12:52:56.710 | INFO | Downloaded 34GiB to GPU in 2.1191s (16.0445GiB/s) | 20260505 12:53:01.852 | INFO | Analyzing chunk frames = [3356160,3356672) | 20260505 12:53:01.852 | INFO | MemInfo: { | 20260505 12:53:01.852 | INFO | "gpu" : " 17 GB", | 20260505 12:53:01.852 | INFO | "huge" : "", | 20260505 12:53:01.852 | INFO | "huge_shared" : " 27 GB", | 20260505 12:53:01.852 | INFO | "malloc" : " 14 GB", | 20260505 12:53:01.852 | INFO | "pinned" : " 29 GB" | 20260505 12:53:01.852 | INFO | } | 20260505 12:53:01.852 | INFO | Perf report for the last 5.14234 seconds: | 20260505 12:53:01.852 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:53:01.852 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:01.852 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:01.852 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:01.852 | INFO | ANALYSIS : 63.0%, 3.0, 97.9ms, 0 | 20260505 12:53:01.852 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:01.852 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:01.852 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 12:53:01.852 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:01.852 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:01.852 | INFO | Uploaded 36GiB to GPU in 2.20063s (16.359GiB/s) | 20260505 12:53:01.852 | INFO | Downloaded 34GiB to GPU in 2.13317s (15.9387GiB/s) | 20260505 12:53:06.988 | INFO | Analyzing chunk frames = [3356672,3357184) | 20260505 12:53:06.988 | INFO | MemInfo: { | 20260505 12:53:06.988 | INFO | "gpu" : " 17 GB", | 20260505 12:53:06.988 | INFO | "huge" : "", | 20260505 12:53:06.988 | INFO | "huge_shared" : " 27 GB", | 20260505 12:53:06.988 | INFO | "malloc" : " 14 GB", | 20260505 12:53:06.988 | INFO | "pinned" : " 29 GB" | 20260505 12:53:06.988 | INFO | } | 20260505 12:53:06.988 | INFO | Perf report for the last 5.13637 seconds: | 20260505 12:53:06.988 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:53:06.988 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:06.988 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:06.988 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:06.988 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 12:53:06.988 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:06.988 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:06.988 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 12:53:06.988 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:06.988 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:06.988 | INFO | Uploaded 36GiB to GPU in 2.1272s (16.9236GiB/s) | 20260505 12:53:06.988 | INFO | Downloaded 34GiB to GPU in 2.1042s (16.1582GiB/s) | 20260505 12:53:12.126 | INFO | Analyzing chunk frames = [3357184,3357696) | 20260505 12:53:12.126 | INFO | MemInfo: { | 20260505 12:53:12.126 | INFO | "gpu" : " 17 GB", | 20260505 12:53:12.126 | INFO | "huge" : "", | 20260505 12:53:12.126 | INFO | "huge_shared" : " 27 GB", | 20260505 12:53:12.127 | INFO | "malloc" : " 14 GB", | 20260505 12:53:12.127 | INFO | "pinned" : " 29 GB" | 20260505 12:53:12.127 | INFO | } | 20260505 12:53:12.127 | INFO | Perf report for the last 5.13848 seconds: | 20260505 12:53:12.127 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:53:12.127 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:12.127 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:12.127 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:12.127 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 12:53:12.127 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:12.127 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:12.127 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:53:12.127 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:12.127 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:12.127 | INFO | Uploaded 36GiB to GPU in 2.17952s (16.5174GiB/s) | 20260505 12:53:12.127 | INFO | Downloaded 34GiB to GPU in 2.1167s (16.0628GiB/s) | 20260505 12:53:17.268 | INFO | Analyzing chunk frames = [3357696,3358208) | 20260505 12:53:17.268 | INFO | MemInfo: { | 20260505 12:53:17.268 | INFO | "gpu" : " 17 GB", | 20260505 12:53:17.268 | INFO | "huge" : "", | 20260505 12:53:17.268 | INFO | "huge_shared" : " 27 GB", | 20260505 12:53:17.268 | INFO | "malloc" : " 14 GB", | 20260505 12:53:17.268 | INFO | "pinned" : " 29 GB" | 20260505 12:53:17.268 | INFO | } | 20260505 12:53:17.268 | INFO | Perf report for the last 5.14148 seconds: | 20260505 12:53:17.268 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:53:17.268 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:17.268 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:17.268 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:17.268 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 12:53:17.268 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:17.268 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:17.268 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 12:53:17.268 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:17.268 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:17.268 | INFO | Uploaded 36GiB to GPU in 2.15839s (16.6791GiB/s) | 20260505 12:53:17.268 | INFO | Downloaded 34GiB to GPU in 2.14734s (15.8336GiB/s) | 20260505 12:53:22.406 | INFO | Analyzing chunk frames = [3358208,3358720) | 20260505 12:53:22.406 | INFO | MemInfo: { | 20260505 12:53:22.406 | INFO | "gpu" : " 17 GB", | 20260505 12:53:22.406 | INFO | "huge" : "", | 20260505 12:53:22.406 | INFO | "huge_shared" : " 27 GB", | 20260505 12:53:22.406 | INFO | "malloc" : " 14 GB", | 20260505 12:53:22.406 | INFO | "pinned" : " 29 GB" | 20260505 12:53:22.406 | INFO | } | 20260505 12:53:22.406 | INFO | Perf report for the last 5.13788 seconds: | 20260505 12:53:22.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:53:22.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:22.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:22.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:22.406 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 12:53:22.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:22.406 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:22.406 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 12:53:22.406 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:22.406 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:22.406 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 12:53:22.406 | INFO | Total mem usage: 15.3 GiB | 20260505 12:53:22.406 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 12:53:22.406 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -6.95%) | 20260505 12:53:22.406 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 12:53:22.406 | INFO | High Water Breakdown: | 20260505 12:53:22.406 | INFO | 61.1% : 8.75 GiB ( 92% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 12:53:22.406 | INFO | 38.8% : 5.56 GiB ( 83.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 12:53:22.406 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 12:53:22.406 | INFO | ------------------------------------------ | 20260505 12:53:22.406 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:53:22.406 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 12:53:22.406 | INFO | Total mem usage: 26.5 GiB | 20260505 12:53:22.406 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 12:53:22.406 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 12:53:22.406 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 12:53:22.406 | INFO | High Water Breakdown: | 20260505 12:53:22.406 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:53:22.406 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:53:22.406 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:53:22.406 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:53:22.406 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:53:22.406 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:53:22.406 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 12:53:22.406 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 12:53:22.406 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:53:22.407 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 12:53:22.407 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 12:53:22.407 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 12:53:22.407 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 12:53:22.407 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 12:53:22.406 | INFO | Uploaded 36GiB to GPU in 2.20883s (16.2982GiB/s) | 20260505 12:53:22.407 | INFO | ------------------------------------------ | 20260505 12:53:22.407 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:53:22.407 | INFO | Downloaded 34GiB to GPU in 2.10466s (16.1546GiB/s) | 20260505 12:53:22.407 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 12:53:22.407 | INFO | Total mem usage: 25.6 GiB | 20260505 12:53:22.407 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 12:53:22.407 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 12:53:22.407 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 12:53:22.407 | INFO | High Water Breakdown: | 20260505 12:53:22.407 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 12:53:22.407 | INFO | ------------------------------------------ | 20260505 12:53:22.407 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:53:22.407 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 12:53:22.407 | INFO | Total mem usage: 15.7 GiB | 20260505 12:53:22.407 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 12:53:22.407 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 12:53:22.407 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 12:53:22.407 | INFO | High Water Breakdown: | 20260505 12:53:22.407 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 12:53:22.407 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 12:53:22.407 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 12:53:22.407 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 12:53:22.407 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 12:53:22.407 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 12:53:22.407 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 12:53:22.407 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 12:53:22.407 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 12:53:22.407 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:53:22.407 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 12:53:22.407 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:53:22.407 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 12:53:22.407 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 12:53:22.407 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 12:53:22.407 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 12:53:22.407 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:53:22.407 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 12:53:22.407 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:53:22.407 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:53:22.407 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:53:22.407 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 12:53:22.407 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 12:53:22.407 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:53:22.407 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 12:53:22.407 | INFO | ------------------------------------------ | 20260505 12:53:22.407 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:53:22.407 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 12:53:22.407 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:53:22.407 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:53:22.407 | INFO | PerfReport: Full Execution -- 3.20e+01 / 7.72e+01, 1.39e+01 / 1.69e+00 | 20260505 12:53:22.407 | INFO | PerfReport: 0.29%, AggregateStats -- 9.26e-02 / 1.00e-01, 1.05e-02 / 7.68e-03 | 20260505 12:53:22.407 | INFO | PerfReport: 11.74%, Baseline -- 3.75e+00 / 4.49e+00, 5.91e-01 / 1.65e-01 | 20260505 12:53:22.407 | INFO | PerfReport: 3.29%, Histogram -- 1.05e+00 / 1.49e+00, 2.01e-01 / 9.68e-02 | 20260505 12:53:22.407 | INFO | PerfReport: 16.07%, DME -- 4.11e+01 / 4.65e+01, 4.72e+00 / 2.04e+00 | 20260505 12:53:22.407 | INFO | PerfReport: 56.26%, FrameLabeling -- 1.80e+01 / 1.97e+01, 1.42e+00 / 1.69e+00 | 20260505 12:53:22.407 | INFO | PerfReport: 7.48%, PulseAccumulating -- 2.39e+00 / 2.94e+00, 4.55e-01 / 2.01e-01 | 20260505 12:53:22.407 | INFO | PerfReport: 3.61%, Metrics -- 1.15e+00 / 1.48e+00, 3.22e-01 / 1.98e-01 | 20260505 12:53:22.407 | INFO | PerfReport: 1.27%, PulseSerializing -- 4.06e-01 / 4.59e-01, 7.53e-02 / 3.89e-02 | 20260505 12:53:22.407 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 12:53:22.407 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:53:22.407 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:53:22.407 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260505 12:53:22.407 | INFO | PerfReport: 22.48%, Upload -- 2.27e+01 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260505 12:53:22.407 | INFO | PerfReport: 22.05%, Download -- 2.23e+01 / 0.00e+00, 8.11e+00 / 0.00e+00 | 20260505 12:53:22.407 | INFO | PerfReport: 31.66%, Compute -- 3.20e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 12:53:22.407 | INFO | PerfReport: 2.20%, LockUpload -- 2.22e+00 / 0.00e+00, 7.83e+00 / 0.00e+00 | 20260505 12:53:22.407 | INFO | PerfReport: 1.43%, LockDownload -- 1.45e+00 / 0.00e+00, 3.02e+00 / 0.00e+00 | 20260505 12:53:22.407 | INFO | PerfReport: 20.16%, LockCompute -- 2.04e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260505 12:53:22.407 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.01e-02 / 0.00e+00 | 20260505 12:53:22.541 | INFO | Starting a round of Baz IO | 20260505 12:53:25.771 | INFO | Populating RTMetrics report | 20260505 12:53:25.771 | INFO | RTMetrics report populated | 20260505 12:53:25.771 | INFO | Opening tmp RTMetrics json file | 20260505 12:53:25.772 | INFO | Serializing RTMetrics to json | 20260505 12:53:25.773 | INFO | Serialization done, swapping tmp file now... | 20260505 12:53:25.773 | INFO | RTMetrics done | 20260505 12:53:25.906 | INFO | Finished a round of Baz IO: Wrote 2.323 GiB in 3.364 seconds | 20260505 12:53:25.906 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.36% / 0.01% / 76.63% | 20260505 12:53:25.906 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.800645% / 7.94729e-06% / 8.34465e-05% | 20260505 12:53:25.906 | INFO | Baz IO overall: | 20260505 12:53:25.906 | INFO | BytesWritten : 1998412874520 | 20260505 12:53:25.906 | INFO | NumEvents : 951046611238 | 20260505 12:53:27.549 | INFO | Analyzing chunk frames = [3358720,3359232) | 20260505 12:53:27.549 | INFO | MemInfo: { | 20260505 12:53:27.550 | INFO | "gpu" : " 17 GB", | 20260505 12:53:27.550 | INFO | "huge" : "", | 20260505 12:53:27.550 | INFO | "huge_shared" : " 27 GB", | 20260505 12:53:27.550 | INFO | "malloc" : " 14 GB", | 20260505 12:53:27.550 | INFO | "pinned" : " 29 GB" | 20260505 12:53:27.550 | INFO | } | 20260505 12:53:27.550 | INFO | Perf report for the last 5.14358 seconds: | 20260505 12:53:27.550 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:53:27.550 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:27.550 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:27.550 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:27.550 | INFO | ANALYSIS : 64.3%, 2.9, 99.4ms, 0 | 20260505 12:53:27.550 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:27.550 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260505 12:53:27.550 | INFO | PRE_HQ : 61.3%, 1.8, 58.3ms, 0 | 20260505 12:53:27.550 | INFO | BAZWRITER : 64.7%, 4.1, 135.9ms, 0 | 20260505 12:53:27.550 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 12:53:27.550 | INFO | Uploaded 36GiB to GPU in 2.19528s (16.3988GiB/s) | 20260505 12:53:27.550 | INFO | Downloaded 38GiB to GPU in 2.28496s (16.6305GiB/s) | 20260505 12:53:32.680 | INFO | Analyzing chunk frames = [3359232,3359744) | 20260505 12:53:32.680 | INFO | MemInfo: { | 20260505 12:53:32.680 | INFO | "gpu" : " 17 GB", | 20260505 12:53:32.680 | INFO | "huge" : "", | 20260505 12:53:32.680 | INFO | "huge_shared" : " 27 GB", | 20260505 12:53:32.680 | INFO | "malloc" : " 14 GB", | 20260505 12:53:32.680 | INFO | "pinned" : " 29 GB" | 20260505 12:53:32.680 | INFO | } | 20260505 12:53:32.680 | INFO | Perf report for the last 5.13047 seconds: | 20260505 12:53:32.680 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:53:32.680 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:32.680 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:32.680 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:32.680 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 12:53:32.680 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:32.680 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:32.680 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 12:53:32.680 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:32.680 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:32.680 | INFO | Uploaded 36GiB to GPU in 2.19447s (16.4049GiB/s) | 20260505 12:53:32.680 | INFO | Downloaded 34GiB to GPU in 2.03298s (16.7243GiB/s) | 20260505 12:53:37.815 | INFO | Analyzing chunk frames = [3359744,3360256) | 20260505 12:53:37.815 | INFO | MemInfo: { | 20260505 12:53:37.815 | INFO | "gpu" : " 17 GB", | 20260505 12:53:37.815 | INFO | "huge" : "", | 20260505 12:53:37.815 | INFO | "huge_shared" : " 27 GB", | 20260505 12:53:37.815 | INFO | "malloc" : " 14 GB", | 20260505 12:53:37.815 | INFO | "pinned" : " 29 GB" | 20260505 12:53:37.815 | INFO | } | 20260505 12:53:37.815 | INFO | Perf report for the last 5.1354 seconds: | 20260505 12:53:37.815 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:53:37.815 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:37.815 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:37.815 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:37.815 | INFO | ANALYSIS : 63.4%, 3.0, 98.6ms, 0 | 20260505 12:53:37.815 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:37.815 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:37.815 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 12:53:37.815 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:37.816 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:37.816 | INFO | Uploaded 36GiB to GPU in 2.2099s (16.2903GiB/s) | 20260505 12:53:37.816 | INFO | Downloaded 34GiB to GPU in 2.11623s (16.0663GiB/s) | 20260505 12:53:42.966 | INFO | Analyzing chunk frames = [3360256,3360768) | 20260505 12:53:42.966 | INFO | MemInfo: { | 20260505 12:53:42.966 | INFO | "gpu" : " 17 GB", | 20260505 12:53:42.966 | INFO | "huge" : "", | 20260505 12:53:42.966 | INFO | "huge_shared" : " 27 GB", | 20260505 12:53:42.966 | INFO | "malloc" : " 14 GB", | 20260505 12:53:42.966 | INFO | "pinned" : " 29 GB" | 20260505 12:53:42.966 | INFO | } | 20260505 12:53:42.966 | INFO | Perf report for the last 5.1509 seconds: | 20260505 12:53:42.966 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:53:42.966 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:42.966 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:42.966 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:42.966 | INFO | ANALYSIS : 63.1%, 3.0, 98.3ms, 0 | 20260505 12:53:42.966 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:42.966 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:42.966 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 12:53:42.966 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:42.966 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:42.967 | INFO | Uploaded 36GiB to GPU in 2.21543s (16.2497GiB/s) | 20260505 12:53:42.967 | INFO | Downloaded 34GiB to GPU in 2.13168s (15.9499GiB/s) | 20260505 12:53:48.104 | INFO | Analyzing chunk frames = [3360768,3361280) | 20260505 12:53:48.104 | INFO | MemInfo: { | 20260505 12:53:48.104 | INFO | "gpu" : " 17 GB", | 20260505 12:53:48.104 | INFO | "huge" : "", | 20260505 12:53:48.104 | INFO | "huge_shared" : " 27 GB", | 20260505 12:53:48.104 | INFO | "malloc" : " 14 GB", | 20260505 12:53:48.104 | INFO | "pinned" : " 29 GB" | 20260505 12:53:48.104 | INFO | } | 20260505 12:53:48.104 | INFO | Perf report for the last 5.13803 seconds: | 20260505 12:53:48.104 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:53:48.104 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:48.104 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:48.104 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:48.104 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 12:53:48.104 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:48.104 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:48.104 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 12:53:48.104 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:48.104 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:48.105 | INFO | Uploaded 36GiB to GPU in 2.1602s (16.6651GiB/s) | 20260505 12:53:48.105 | INFO | Downloaded 34GiB to GPU in 2.12553s (15.996GiB/s) | 20260505 12:53:53.246 | INFO | Analyzing chunk frames = [3361280,3361792) | 20260505 12:53:53.246 | INFO | MemInfo: { | 20260505 12:53:53.246 | INFO | "gpu" : " 17 GB", | 20260505 12:53:53.246 | INFO | "huge" : "", | 20260505 12:53:53.246 | INFO | "huge_shared" : " 27 GB", | 20260505 12:53:53.246 | INFO | "malloc" : " 14 GB", | 20260505 12:53:53.246 | INFO | "pinned" : " 29 GB" | 20260505 12:53:53.246 | INFO | } | 20260505 12:53:53.246 | INFO | Perf report for the last 5.14174 seconds: | 20260505 12:53:53.246 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:53:53.246 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:53.246 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:53.246 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:53.246 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 12:53:53.246 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:53.246 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:53.246 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:53:53.246 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:53.246 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:53.246 | INFO | Uploaded 36GiB to GPU in 2.15851s (16.6782GiB/s) | 20260505 12:53:53.246 | INFO | Downloaded 34GiB to GPU in 2.10758s (16.1322GiB/s) | 20260505 12:53:58.378 | INFO | Analyzing chunk frames = [3361792,3362304) | 20260505 12:53:58.378 | INFO | MemInfo: { | 20260505 12:53:58.378 | INFO | "gpu" : " 17 GB", | 20260505 12:53:58.378 | INFO | "huge" : "", | 20260505 12:53:58.378 | INFO | "huge_shared" : " 27 GB", | 20260505 12:53:58.378 | INFO | "malloc" : " 14 GB", | 20260505 12:53:58.378 | INFO | "pinned" : " 29 GB" | 20260505 12:53:58.378 | INFO | } | 20260505 12:53:58.378 | INFO | Perf report for the last 5.13187 seconds: | 20260505 12:53:58.378 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:53:58.378 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:58.378 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:58.378 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:58.378 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 12:53:58.378 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:58.378 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:58.378 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 12:53:58.378 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:58.378 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:53:58.378 | INFO | Uploaded 36GiB to GPU in 2.14988s (16.7451GiB/s) | 20260505 12:53:58.378 | INFO | Downloaded 34GiB to GPU in 2.14837s (15.8259GiB/s) | 20260505 12:54:03.520 | INFO | Analyzing chunk frames = [3362304,3362816) | 20260505 12:54:03.520 | INFO | MemInfo: { | 20260505 12:54:03.520 | INFO | "gpu" : " 17 GB", | 20260505 12:54:03.520 | INFO | "huge" : "", | 20260505 12:54:03.520 | INFO | "huge_shared" : " 27 GB", | 20260505 12:54:03.520 | INFO | "malloc" : " 14 GB", | 20260505 12:54:03.520 | INFO | "pinned" : " 29 GB" | 20260505 12:54:03.520 | INFO | } | 20260505 12:54:03.520 | INFO | Perf report for the last 5.14212 seconds: | 20260505 12:54:03.520 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:54:03.520 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:03.520 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:03.520 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:03.520 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 12:54:03.520 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:03.520 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:03.520 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 12:54:03.520 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:03.520 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:03.520 | INFO | Uploaded 36GiB to GPU in 2.16977s (16.5916GiB/s) | 20260505 12:54:03.520 | INFO | Downloaded 34GiB to GPU in 2.11623s (16.0663GiB/s) | 20260505 12:54:03.654 | INFO | Starting a round of Baz IO | 20260505 12:54:06.887 | INFO | Populating RTMetrics report | 20260505 12:54:06.887 | INFO | RTMetrics report populated | 20260505 12:54:06.887 | INFO | Opening tmp RTMetrics json file | 20260505 12:54:06.936 | INFO | Serializing RTMetrics to json | 20260505 12:54:06.986 | INFO | Serialization done, swapping tmp file now... | 20260505 12:54:06.988 | INFO | RTMetrics done | 20260505 12:54:07.009 | INFO | Finished a round of Baz IO: Wrote 2.322 GiB in 3.354 seconds | 20260505 12:54:07.009 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.36% / 0.01% / 76.64% | 20260505 12:54:07.009 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.79968% / 7.94729e-06% / 8.34465e-05% | 20260505 12:54:07.009 | INFO | Baz IO overall: | 20260505 12:54:07.009 | INFO | BytesWritten : 2000906547558 | 20260505 12:54:07.009 | INFO | NumEvents : 952228527208 | 20260505 12:54:08.659 | INFO | Analyzing chunk frames = [3362816,3363328) | 20260505 12:54:08.659 | INFO | MemInfo: { | 20260505 12:54:08.659 | INFO | "gpu" : " 17 GB", | 20260505 12:54:08.659 | INFO | "huge" : "", | 20260505 12:54:08.659 | INFO | "huge_shared" : " 27 GB", | 20260505 12:54:08.659 | INFO | "malloc" : " 14 GB", | 20260505 12:54:08.659 | INFO | "pinned" : " 29 GB" | 20260505 12:54:08.659 | INFO | } | 20260505 12:54:08.659 | INFO | Perf report for the last 5.13853 seconds: | 20260505 12:54:08.659 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:54:08.659 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:08.659 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:08.659 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:08.659 | INFO | ANALYSIS : 64.4%, 2.9, 99.6ms, 0 | 20260505 12:54:08.659 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:08.659 | INFO | RT_METRICS : 47.5%, 1.0, 24.9ms, 0 | 20260505 12:54:08.659 | INFO | PRE_HQ : 61.2%, 1.8, 58.4ms, 0 | 20260505 12:54:08.659 | INFO | BAZWRITER : 64.6%, 4.1, 135.3ms, 0 | 20260505 12:54:08.659 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 12:54:08.659 | INFO | Uploaded 36GiB to GPU in 2.19759s (16.3815GiB/s) | 20260505 12:54:08.659 | INFO | Downloaded 38GiB to GPU in 2.28025s (16.6648GiB/s) | 20260505 12:54:13.800 | INFO | Analyzing chunk frames = [3363328,3363840) | 20260505 12:54:13.800 | INFO | MemInfo: { | 20260505 12:54:13.800 | INFO | "gpu" : " 17 GB", | 20260505 12:54:13.800 | INFO | "huge" : "", | 20260505 12:54:13.800 | INFO | "huge_shared" : " 27 GB", | 20260505 12:54:13.800 | INFO | "malloc" : " 14 GB", | 20260505 12:54:13.800 | INFO | "pinned" : " 29 GB" | 20260505 12:54:13.800 | INFO | } | 20260505 12:54:13.800 | INFO | Perf report for the last 5.14141 seconds: | 20260505 12:54:13.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:54:13.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:13.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:13.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:13.800 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 12:54:13.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:13.800 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:13.800 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 12:54:13.800 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:13.800 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:13.800 | INFO | Uploaded 36GiB to GPU in 2.1972s (16.3845GiB/s) | 20260505 12:54:13.800 | INFO | Downloaded 34GiB to GPU in 2.04486s (16.6271GiB/s) | 20260505 12:54:18.932 | INFO | Analyzing chunk frames = [3363840,3364352) | 20260505 12:54:18.932 | INFO | MemInfo: { | 20260505 12:54:18.932 | INFO | "gpu" : " 17 GB", | 20260505 12:54:18.932 | INFO | "huge" : "", | 20260505 12:54:18.932 | INFO | "huge_shared" : " 27 GB", | 20260505 12:54:18.932 | INFO | "malloc" : " 14 GB", | 20260505 12:54:18.932 | INFO | "pinned" : " 29 GB" | 20260505 12:54:18.932 | INFO | } | 20260505 12:54:18.932 | INFO | Perf report for the last 5.13217 seconds: | 20260505 12:54:18.932 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:54:18.932 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:18.932 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:18.932 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:18.932 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 12:54:18.932 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:18.932 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:18.932 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 12:54:18.932 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:18.932 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:18.933 | INFO | Uploaded 36GiB to GPU in 2.20623s (16.3174GiB/s) | 20260505 12:54:18.933 | INFO | Downloaded 34GiB to GPU in 2.12345s (16.0117GiB/s) | 20260505 12:54:24.072 | INFO | Analyzing chunk frames = [3364352,3364864) | 20260505 12:54:24.072 | INFO | MemInfo: { | 20260505 12:54:24.072 | INFO | "gpu" : " 17 GB", | 20260505 12:54:24.072 | INFO | "huge" : "", | 20260505 12:54:24.072 | INFO | "huge_shared" : " 27 GB", | 20260505 12:54:24.072 | INFO | "malloc" : " 14 GB", | 20260505 12:54:24.072 | INFO | "pinned" : " 29 GB" | 20260505 12:54:24.072 | INFO | } | 20260505 12:54:24.072 | INFO | Perf report for the last 5.13969 seconds: | 20260505 12:54:24.072 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:54:24.072 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:24.072 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:24.072 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:24.072 | INFO | ANALYSIS : 63.6%, 3.0, 98.8ms, 0 | 20260505 12:54:24.072 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:24.072 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:24.072 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 12:54:24.072 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:24.072 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:24.072 | INFO | Uploaded 36GiB to GPU in 2.22325s (16.1925GiB/s) | 20260505 12:54:24.072 | INFO | Downloaded 34GiB to GPU in 2.13332s (15.9376GiB/s) | 20260505 12:54:29.227 | INFO | Analyzing chunk frames = [3364864,3365376) | 20260505 12:54:29.227 | INFO | MemInfo: { | 20260505 12:54:29.227 | INFO | "gpu" : " 17 GB", | 20260505 12:54:29.227 | INFO | "huge" : "", | 20260505 12:54:29.227 | INFO | "huge_shared" : " 27 GB", | 20260505 12:54:29.227 | INFO | "malloc" : " 14 GB", | 20260505 12:54:29.227 | INFO | "pinned" : " 29 GB" | 20260505 12:54:29.227 | INFO | } | 20260505 12:54:29.227 | INFO | Perf report for the last 5.15506 seconds: | 20260505 12:54:29.227 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:54:29.227 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:29.227 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:29.227 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:29.227 | INFO | ANALYSIS : 62.9%, 3.0, 98.5ms, 0 | 20260505 12:54:29.227 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:29.227 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:29.227 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:54:29.227 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:29.227 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:29.227 | INFO | Uploaded 36GiB to GPU in 2.18078s (16.5078GiB/s) | 20260505 12:54:29.227 | INFO | Downloaded 34GiB to GPU in 2.13018s (15.9611GiB/s) | 20260505 12:54:34.364 | INFO | Analyzing chunk frames = [3365376,3365888) | 20260505 12:54:34.364 | INFO | MemInfo: { | 20260505 12:54:34.364 | INFO | "gpu" : " 17 GB", | 20260505 12:54:34.364 | INFO | "huge" : "", | 20260505 12:54:34.364 | INFO | "huge_shared" : " 27 GB", | 20260505 12:54:34.364 | INFO | "malloc" : " 14 GB", | 20260505 12:54:34.364 | INFO | "pinned" : " 29 GB" | 20260505 12:54:34.364 | INFO | } | 20260505 12:54:34.364 | INFO | Perf report for the last 5.137 seconds: | 20260505 12:54:34.364 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:54:34.364 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:34.364 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:34.364 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:34.364 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 12:54:34.364 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:34.364 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:34.364 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:54:34.364 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:34.364 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:34.364 | INFO | Uploaded 36GiB to GPU in 2.20399s (16.334GiB/s) | 20260505 12:54:34.364 | INFO | Downloaded 34GiB to GPU in 2.1115s (16.1023GiB/s) | 20260505 12:54:39.499 | INFO | Analyzing chunk frames = [3365888,3366400) | 20260505 12:54:39.499 | INFO | MemInfo: { | 20260505 12:54:39.499 | INFO | "gpu" : " 17 GB", | 20260505 12:54:39.499 | INFO | "huge" : "", | 20260505 12:54:39.499 | INFO | "huge_shared" : " 27 GB", | 20260505 12:54:39.499 | INFO | "malloc" : " 14 GB", | 20260505 12:54:39.499 | INFO | "pinned" : " 29 GB" | 20260505 12:54:39.499 | INFO | } | 20260505 12:54:39.499 | INFO | Perf report for the last 5.13523 seconds: | 20260505 12:54:39.499 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:54:39.499 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:39.499 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:39.499 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:39.499 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 12:54:39.499 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:39.499 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:39.499 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 12:54:39.499 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:39.499 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:39.500 | INFO | Uploaded 36GiB to GPU in 2.167s (16.6128GiB/s) | 20260505 12:54:39.500 | INFO | Downloaded 34GiB to GPU in 2.1598s (15.7422GiB/s) | 20260505 12:54:44.628 | INFO | Analyzing chunk frames = [3366400,3366912) | 20260505 12:54:44.628 | INFO | MemInfo: { | 20260505 12:54:44.628 | INFO | "gpu" : " 17 GB", | 20260505 12:54:44.628 | INFO | "huge" : "", | 20260505 12:54:44.628 | INFO | "huge_shared" : " 27 GB", | 20260505 12:54:44.628 | INFO | "malloc" : " 14 GB", | 20260505 12:54:44.628 | INFO | "pinned" : " 29 GB" | 20260505 12:54:44.628 | INFO | } | 20260505 12:54:44.628 | INFO | Perf report for the last 5.12877 seconds: | 20260505 12:54:44.628 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:54:44.628 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:44.628 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:44.628 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:44.628 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 12:54:44.628 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:44.628 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:44.628 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 12:54:44.628 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:44.628 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:44.628 | INFO | Uploaded 36GiB to GPU in 2.14613s (16.7744GiB/s) | 20260505 12:54:44.628 | INFO | Downloaded 34GiB to GPU in 2.10421s (16.1581GiB/s) | 20260505 12:54:44.628 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 12:54:44.629 | INFO | Total mem usage: 15.3 GiB | 20260505 12:54:44.629 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 12:54:44.629 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -6.96%) | 20260505 12:54:44.629 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 12:54:44.629 | INFO | High Water Breakdown: | 20260505 12:54:44.629 | INFO | 61.1% : 8.75 GiB ( 92% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 12:54:44.629 | INFO | 38.8% : 5.56 GiB ( 83.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 12:54:44.629 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 12:54:44.629 | INFO | ------------------------------------------ | 20260505 12:54:44.629 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:54:44.629 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 12:54:44.629 | INFO | Total mem usage: 26.5 GiB | 20260505 12:54:44.629 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 12:54:44.629 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 12:54:44.629 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 12:54:44.629 | INFO | High Water Breakdown: | 20260505 12:54:44.629 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:54:44.629 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:54:44.629 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:54:44.629 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:54:44.629 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:54:44.629 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:54:44.629 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 12:54:44.629 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 12:54:44.629 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:54:44.629 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 12:54:44.629 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 12:54:44.629 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 12:54:44.629 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 12:54:44.629 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 12:54:44.629 | INFO | ------------------------------------------ | 20260505 12:54:44.629 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:54:44.629 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 12:54:44.629 | INFO | Total mem usage: 25.6 GiB | 20260505 12:54:44.629 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 12:54:44.629 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 12:54:44.629 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 12:54:44.629 | INFO | High Water Breakdown: | 20260505 12:54:44.629 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 12:54:44.629 | INFO | ------------------------------------------ | 20260505 12:54:44.629 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:54:44.629 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 12:54:44.629 | INFO | Total mem usage: 15.7 GiB | 20260505 12:54:44.629 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 12:54:44.629 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 12:54:44.629 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 12:54:44.629 | INFO | High Water Breakdown: | 20260505 12:54:44.629 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 12:54:44.629 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 12:54:44.629 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 12:54:44.629 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 12:54:44.629 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 12:54:44.629 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 12:54:44.629 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 12:54:44.629 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 12:54:44.629 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 12:54:44.629 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:54:44.629 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 12:54:44.629 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:54:44.629 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 12:54:44.629 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 12:54:44.629 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 12:54:44.629 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 12:54:44.629 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:54:44.629 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 12:54:44.629 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:54:44.629 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:54:44.630 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:54:44.630 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 12:54:44.630 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 12:54:44.630 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:54:44.630 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 12:54:44.630 | INFO | ------------------------------------------ | 20260505 12:54:44.630 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:54:44.630 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 12:54:44.630 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:54:44.630 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:54:44.630 | INFO | PerfReport: Full Execution -- 3.20e+01 / 7.67e+01, 1.39e+01 / 1.79e+00 | 20260505 12:54:44.630 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 9.96e-02, 1.04e-02 / 5.03e-03 | 20260505 12:54:44.630 | INFO | PerfReport: 11.76%, Baseline -- 3.76e+00 / 4.47e+00, 5.95e-01 / 2.23e-01 | 20260505 12:54:44.630 | INFO | PerfReport: 3.29%, Histogram -- 1.05e+00 / 1.47e+00, 1.99e-01 / 1.17e-01 | 20260505 12:54:44.630 | INFO | PerfReport: 15.89%, DME -- 4.06e+01 / 4.52e+01, 4.36e+00 / 1.85e+00 | 20260505 12:54:44.630 | INFO | PerfReport: 56.46%, FrameLabeling -- 1.80e+01 / 2.07e+01, 1.51e+00 / 1.69e+00 | 20260505 12:54:44.630 | INFO | PerfReport: 7.43%, PulseAccumulating -- 2.38e+00 / 2.89e+00, 4.47e-01 / 2.07e-01 | 20260505 12:54:44.630 | INFO | PerfReport: 3.60%, Metrics -- 1.15e+00 / 1.47e+00, 3.24e-01 / 1.75e-01 | 20260505 12:54:44.630 | INFO | PerfReport: 1.27%, PulseSerializing -- 4.05e-01 / 4.53e-01, 7.43e-02 / 3.55e-02 | 20260505 12:54:44.630 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 12:54:44.630 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:54:44.630 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:54:44.630 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260505 12:54:44.630 | INFO | PerfReport: 22.56%, Upload -- 2.28e+01 / 0.00e+00, 8.07e+00 / 0.00e+00 | 20260505 12:54:44.630 | INFO | PerfReport: 22.03%, Download -- 2.22e+01 / 0.00e+00, 8.17e+00 / 0.00e+00 | 20260505 12:54:44.630 | INFO | PerfReport: 31.67%, Compute -- 3.20e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 12:54:44.630 | INFO | PerfReport: 2.16%, LockUpload -- 2.18e+00 / 0.00e+00, 7.83e+00 / 0.00e+00 | 20260505 12:54:44.630 | INFO | PerfReport: 1.37%, LockDownload -- 1.39e+00 / 0.00e+00, 2.95e+00 / 0.00e+00 | 20260505 12:54:44.630 | INFO | PerfReport: 20.20%, LockCompute -- 2.04e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260505 12:54:44.630 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.92e-02 / 0.00e+00 | 20260505 12:54:44.763 | INFO | Starting a round of Baz IO | 20260505 12:54:48.013 | INFO | Populating RTMetrics report | 20260505 12:54:48.014 | INFO | RTMetrics report populated | 20260505 12:54:48.014 | INFO | Opening tmp RTMetrics json file | 20260505 12:54:48.014 | INFO | Serializing RTMetrics to json | 20260505 12:54:48.015 | INFO | Serialization done, swapping tmp file now... | 20260505 12:54:48.016 | INFO | RTMetrics done | 20260505 12:54:48.129 | INFO | Finished a round of Baz IO: Wrote 2.321 GiB in 3.365 seconds | 20260505 12:54:48.129 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.35% / 0.01% / 76.64% | 20260505 12:54:48.129 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.796771% / 0% / 8.34465e-05% | 20260505 12:54:48.129 | INFO | Baz IO overall: | 20260505 12:54:48.129 | INFO | BytesWritten : 2003398192840 | 20260505 12:54:48.129 | INFO | NumEvents : 953409132246 | 20260505 12:54:49.770 | INFO | Analyzing chunk frames = [3366912,3367424) | 20260505 12:54:49.770 | INFO | MemInfo: { | 20260505 12:54:49.770 | INFO | "gpu" : " 17 GB", | 20260505 12:54:49.770 | INFO | "huge" : "", | 20260505 12:54:49.770 | INFO | "huge_shared" : " 27 GB", | 20260505 12:54:49.770 | INFO | "malloc" : " 14 GB", | 20260505 12:54:49.770 | INFO | "pinned" : " 29 GB" | 20260505 12:54:49.770 | INFO | } | 20260505 12:54:49.770 | INFO | Perf report for the last 5.14171 seconds: | 20260505 12:54:49.770 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:54:49.770 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:49.770 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:49.770 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:49.770 | INFO | ANALYSIS : 64.7%, 3.0, 100.4ms, 0 | 20260505 12:54:49.770 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:49.770 | INFO | RT_METRICS : 45.1%, 1.0, 23.7ms, 0 | 20260505 12:54:49.770 | INFO | PRE_HQ : 60.9%, 1.8, 57.8ms, 0 | 20260505 12:54:49.770 | INFO | BAZWRITER : 64.8%, 4.0, 133.8ms, 0 | 20260505 12:54:49.770 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 12:54:49.770 | INFO | Uploaded 36GiB to GPU in 2.18581s (16.4699GiB/s) | 20260505 12:54:49.770 | INFO | Downloaded 38GiB to GPU in 2.28121s (16.6578GiB/s) | 20260505 12:54:54.915 | INFO | Analyzing chunk frames = [3367424,3367936) | 20260505 12:54:54.915 | INFO | MemInfo: { | 20260505 12:54:54.915 | INFO | "gpu" : " 17 GB", | 20260505 12:54:54.915 | INFO | "huge" : "", | 20260505 12:54:54.915 | INFO | "huge_shared" : " 27 GB", | 20260505 12:54:54.915 | INFO | "malloc" : " 14 GB", | 20260505 12:54:54.915 | INFO | "pinned" : " 29 GB" | 20260505 12:54:54.915 | INFO | } | 20260505 12:54:54.915 | INFO | Perf report for the last 5.14514 seconds: | 20260505 12:54:54.915 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:54:54.915 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:54.915 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:54.915 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:54.915 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 12:54:54.915 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:54.915 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:54.915 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:54:54.915 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:54.915 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:54:54.915 | INFO | Uploaded 36GiB to GPU in 2.17905s (16.521GiB/s) | 20260505 12:54:54.915 | INFO | Downloaded 34GiB to GPU in 2.04934s (16.5907GiB/s) | 20260505 12:55:00.052 | INFO | Analyzing chunk frames = [3367936,3368448) | 20260505 12:55:00.052 | INFO | MemInfo: { | 20260505 12:55:00.052 | INFO | "gpu" : " 17 GB", | 20260505 12:55:00.052 | INFO | "huge" : "", | 20260505 12:55:00.052 | INFO | "huge_shared" : " 27 GB", | 20260505 12:55:00.052 | INFO | "malloc" : " 14 GB", | 20260505 12:55:00.052 | INFO | "pinned" : " 29 GB" | 20260505 12:55:00.052 | INFO | } | 20260505 12:55:00.053 | INFO | Perf report for the last 5.13753 seconds: | 20260505 12:55:00.053 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:55:00.053 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:00.053 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:00.053 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:00.053 | INFO | ANALYSIS : 63.4%, 3.0, 98.6ms, 0 | 20260505 12:55:00.053 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:00.053 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:00.053 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:55:00.053 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:00.053 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:00.053 | INFO | Uploaded 36GiB to GPU in 2.21921s (16.222GiB/s) | 20260505 12:55:00.053 | INFO | Downloaded 34GiB to GPU in 2.11282s (16.0922GiB/s) | 20260505 12:55:05.191 | INFO | Analyzing chunk frames = [3368448,3368960) | 20260505 12:55:05.191 | INFO | MemInfo: { | 20260505 12:55:05.191 | INFO | "gpu" : " 17 GB", | 20260505 12:55:05.191 | INFO | "huge" : "", | 20260505 12:55:05.191 | INFO | "huge_shared" : " 27 GB", | 20260505 12:55:05.191 | INFO | "malloc" : " 14 GB", | 20260505 12:55:05.191 | INFO | "pinned" : " 29 GB" | 20260505 12:55:05.191 | INFO | } | 20260505 12:55:05.191 | INFO | Perf report for the last 5.13879 seconds: | 20260505 12:55:05.191 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:55:05.191 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:05.191 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:05.191 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:05.191 | INFO | ANALYSIS : 63.1%, 3.0, 98.0ms, 0 | 20260505 12:55:05.191 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:05.191 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:05.191 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 12:55:05.191 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:05.191 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:05.192 | INFO | Uploaded 36GiB to GPU in 2.20291s (16.342GiB/s) | 20260505 12:55:05.192 | INFO | Downloaded 34GiB to GPU in 2.12845s (15.974GiB/s) | 20260505 12:55:10.326 | INFO | Analyzing chunk frames = [3368960,3369472) | 20260505 12:55:10.326 | INFO | MemInfo: { | 20260505 12:55:10.326 | INFO | "gpu" : " 17 GB", | 20260505 12:55:10.326 | INFO | "huge" : "", | 20260505 12:55:10.326 | INFO | "huge_shared" : " 27 GB", | 20260505 12:55:10.326 | INFO | "malloc" : " 14 GB", | 20260505 12:55:10.326 | INFO | "pinned" : " 29 GB" | 20260505 12:55:10.326 | INFO | } | 20260505 12:55:10.326 | INFO | Perf report for the last 5.13436 seconds: | 20260505 12:55:10.326 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:55:10.326 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:10.326 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:10.326 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:10.326 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 12:55:10.326 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:10.326 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:10.326 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 12:55:10.326 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:10.326 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:10.326 | INFO | Uploaded 36GiB to GPU in 2.15171s (16.7309GiB/s) | 20260505 12:55:10.326 | INFO | Downloaded 34GiB to GPU in 2.12686s (15.986GiB/s) | 20260505 12:55:15.476 | INFO | Analyzing chunk frames = [3369472,3369984) | 20260505 12:55:15.476 | INFO | MemInfo: { | 20260505 12:55:15.476 | INFO | "gpu" : " 17 GB", | 20260505 12:55:15.476 | INFO | "huge" : "", | 20260505 12:55:15.476 | INFO | "huge_shared" : " 27 GB", | 20260505 12:55:15.476 | INFO | "malloc" : " 14 GB", | 20260505 12:55:15.476 | INFO | "pinned" : " 29 GB" | 20260505 12:55:15.476 | INFO | } | 20260505 12:55:15.477 | INFO | Perf report for the last 5.1508 seconds: | 20260505 12:55:15.477 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:55:15.477 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:15.477 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:15.477 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:15.477 | INFO | ANALYSIS : 63.1%, 3.0, 98.8ms, 0 | 20260505 12:55:15.477 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:15.477 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:15.477 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 12:55:15.477 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:15.477 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:15.477 | INFO | Uploaded 36GiB to GPU in 2.16326s (16.6415GiB/s) | 20260505 12:55:15.477 | INFO | Downloaded 34GiB to GPU in 2.1145s (16.0794GiB/s) | 20260505 12:55:20.613 | INFO | Analyzing chunk frames = [3369984,3370496) | 20260505 12:55:20.613 | INFO | MemInfo: { | 20260505 12:55:20.613 | INFO | "gpu" : " 17 GB", | 20260505 12:55:20.613 | INFO | "huge" : "", | 20260505 12:55:20.613 | INFO | "huge_shared" : " 27 GB", | 20260505 12:55:20.613 | INFO | "malloc" : " 14 GB", | 20260505 12:55:20.613 | INFO | "pinned" : " 29 GB" | 20260505 12:55:20.613 | INFO | } | 20260505 12:55:20.613 | INFO | Perf report for the last 5.13632 seconds: | 20260505 12:55:20.613 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:55:20.613 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:20.613 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:20.613 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:20.613 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 12:55:20.613 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:20.613 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:20.613 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 12:55:20.613 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:20.613 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:20.613 | INFO | Uploaded 36GiB to GPU in 2.14538s (16.7803GiB/s) | 20260505 12:55:20.613 | INFO | Downloaded 34GiB to GPU in 2.17217s (15.6525GiB/s) | 20260505 12:55:25.746 | INFO | Analyzing chunk frames = [3370496,3371008) | 20260505 12:55:25.746 | INFO | MemInfo: { | 20260505 12:55:25.746 | INFO | "gpu" : " 17 GB", | 20260505 12:55:25.746 | INFO | "huge" : "", | 20260505 12:55:25.746 | INFO | "huge_shared" : " 27 GB", | 20260505 12:55:25.746 | INFO | "malloc" : " 14 GB", | 20260505 12:55:25.746 | INFO | "pinned" : " 29 GB" | 20260505 12:55:25.746 | INFO | } | 20260505 12:55:25.746 | INFO | Perf report for the last 5.13361 seconds: | 20260505 12:55:25.746 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:55:25.746 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:25.747 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:25.747 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:25.747 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 12:55:25.747 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:25.747 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:25.747 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 12:55:25.747 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:25.747 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:25.747 | INFO | Uploaded 36GiB to GPU in 2.12912s (16.9084GiB/s) | 20260505 12:55:25.747 | INFO | Downloaded 34GiB to GPU in 2.10247s (16.1715GiB/s) | 20260505 12:55:25.876 | INFO | Starting a round of Baz IO | 20260505 12:55:29.128 | INFO | Populating RTMetrics report | 20260505 12:55:29.128 | INFO | RTMetrics report populated | 20260505 12:55:29.128 | INFO | Opening tmp RTMetrics json file | 20260505 12:55:29.137 | INFO | Serializing RTMetrics to json | 20260505 12:55:29.141 | INFO | Serialization done, swapping tmp file now... | 20260505 12:55:29.141 | INFO | RTMetrics done | 20260505 12:55:29.271 | INFO | Finished a round of Baz IO: Wrote 2.321 GiB in 3.395 seconds | 20260505 12:55:29.271 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.35% / 0.01% / 76.65% | 20260505 12:55:29.271 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.794363% / 1.58946e-05% / 8.34465e-05% | 20260505 12:55:29.271 | INFO | Baz IO overall: | 20260505 12:55:29.271 | INFO | BytesWritten : 2005890441762 | 20260505 12:55:29.271 | INFO | NumEvents : 954589722168 | 20260505 12:55:30.884 | INFO | Analyzing chunk frames = [3371008,3371520) | 20260505 12:55:30.884 | INFO | MemInfo: { | 20260505 12:55:30.884 | INFO | "gpu" : " 17 GB", | 20260505 12:55:30.884 | INFO | "huge" : "", | 20260505 12:55:30.884 | INFO | "huge_shared" : " 27 GB", | 20260505 12:55:30.884 | INFO | "malloc" : " 14 GB", | 20260505 12:55:30.884 | INFO | "pinned" : " 29 GB" | 20260505 12:55:30.884 | INFO | } | 20260505 12:55:30.884 | INFO | Perf report for the last 5.13748 seconds: | 20260505 12:55:30.884 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:55:30.884 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:30.884 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:30.884 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:30.884 | INFO | ANALYSIS : 64.7%, 2.9, 99.2ms, 0 | 20260505 12:55:30.884 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:30.884 | INFO | RT_METRICS : 45.7%, 1.0, 23.9ms, 0 | 20260505 12:55:30.884 | INFO | PRE_HQ : 61.0%, 1.8, 58.2ms, 0 | 20260505 12:55:30.884 | INFO | BAZWRITER : 65.4%, 4.1, 138.7ms, 0 | 20260505 12:55:30.884 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 12:55:30.884 | INFO | Uploaded 36GiB to GPU in 2.18176s (16.5004GiB/s) | 20260505 12:55:30.884 | INFO | Downloaded 38GiB to GPU in 2.28117s (16.6581GiB/s) | 20260505 12:55:36.023 | INFO | Analyzing chunk frames = [3371520,3372032) | 20260505 12:55:36.023 | INFO | MemInfo: { | 20260505 12:55:36.023 | INFO | "gpu" : " 17 GB", | 20260505 12:55:36.023 | INFO | "huge" : "", | 20260505 12:55:36.023 | INFO | "huge_shared" : " 27 GB", | 20260505 12:55:36.023 | INFO | "malloc" : " 14 GB", | 20260505 12:55:36.023 | INFO | "pinned" : " 29 GB" | 20260505 12:55:36.023 | INFO | } | 20260505 12:55:36.023 | INFO | Perf report for the last 5.13938 seconds: | 20260505 12:55:36.023 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:55:36.023 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:36.023 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:36.023 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:36.023 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 12:55:36.023 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:36.023 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:36.023 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 12:55:36.023 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:36.024 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:36.024 | INFO | Uploaded 36GiB to GPU in 2.18745s (16.4575GiB/s) | 20260505 12:55:36.024 | INFO | Downloaded 34GiB to GPU in 2.05867s (16.5155GiB/s) | 20260505 12:55:41.172 | INFO | Analyzing chunk frames = [3372032,3372544) | 20260505 12:55:41.172 | INFO | MemInfo: { | 20260505 12:55:41.172 | INFO | "gpu" : " 17 GB", | 20260505 12:55:41.172 | INFO | "huge" : "", | 20260505 12:55:41.172 | INFO | "huge_shared" : " 27 GB", | 20260505 12:55:41.172 | INFO | "malloc" : " 14 GB", | 20260505 12:55:41.172 | INFO | "pinned" : " 29 GB" | 20260505 12:55:41.172 | INFO | } | 20260505 12:55:41.172 | INFO | Perf report for the last 5.14871 seconds: | 20260505 12:55:41.172 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:55:41.172 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:41.172 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:41.172 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:41.172 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 12:55:41.172 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:41.172 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:41.172 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:55:41.172 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:41.172 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:41.172 | INFO | Uploaded 36GiB to GPU in 2.22381s (16.1884GiB/s) | 20260505 12:55:41.172 | INFO | Downloaded 34GiB to GPU in 2.12055s (16.0336GiB/s) | 20260505 12:55:46.315 | INFO | Analyzing chunk frames = [3372544,3373056) | 20260505 12:55:46.315 | INFO | MemInfo: { | 20260505 12:55:46.315 | INFO | "gpu" : " 17 GB", | 20260505 12:55:46.315 | INFO | "huge" : "", | 20260505 12:55:46.315 | INFO | "huge_shared" : " 27 GB", | 20260505 12:55:46.315 | INFO | "malloc" : " 14 GB", | 20260505 12:55:46.315 | INFO | "pinned" : " 29 GB" | 20260505 12:55:46.315 | INFO | } | 20260505 12:55:46.315 | INFO | Perf report for the last 5.14305 seconds: | 20260505 12:55:46.315 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:55:46.315 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:46.315 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:46.315 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:46.315 | INFO | ANALYSIS : 63.1%, 3.0, 98.1ms, 0 | 20260505 12:55:46.315 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:46.315 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:46.315 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 12:55:46.315 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:46.315 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:46.315 | INFO | Uploaded 36GiB to GPU in 2.2061s (16.3184GiB/s) | 20260505 12:55:46.315 | INFO | Downloaded 34GiB to GPU in 2.13225s (15.9456GiB/s) | 20260505 12:55:51.448 | INFO | Analyzing chunk frames = [3373056,3373568) | 20260505 12:55:51.448 | INFO | MemInfo: { | 20260505 12:55:51.448 | INFO | "gpu" : " 17 GB", | 20260505 12:55:51.448 | INFO | "huge" : "", | 20260505 12:55:51.448 | INFO | "huge_shared" : " 27 GB", | 20260505 12:55:51.448 | INFO | "malloc" : " 14 GB", | 20260505 12:55:51.448 | INFO | "pinned" : " 29 GB" | 20260505 12:55:51.448 | INFO | } | 20260505 12:55:51.448 | INFO | Perf report for the last 5.13269 seconds: | 20260505 12:55:51.448 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:55:51.448 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:51.448 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:51.448 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:51.448 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260505 12:55:51.448 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:51.448 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:51.448 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 12:55:51.448 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:51.448 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:51.448 | INFO | Uploaded 36GiB to GPU in 2.16354s (16.6394GiB/s) | 20260505 12:55:51.448 | INFO | Downloaded 34GiB to GPU in 2.12566s (15.995GiB/s) | 20260505 12:55:56.586 | INFO | Analyzing chunk frames = [3373568,3374080) | 20260505 12:55:56.586 | INFO | MemInfo: { | 20260505 12:55:56.586 | INFO | "gpu" : " 17 GB", | 20260505 12:55:56.586 | INFO | "huge" : "", | 20260505 12:55:56.586 | INFO | "huge_shared" : " 27 GB", | 20260505 12:55:56.586 | INFO | "malloc" : " 14 GB", | 20260505 12:55:56.586 | INFO | "pinned" : " 29 GB" | 20260505 12:55:56.586 | INFO | } | 20260505 12:55:56.586 | INFO | Perf report for the last 5.13828 seconds: | 20260505 12:55:56.586 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:55:56.586 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:56.586 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:56.586 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:56.586 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 12:55:56.586 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:56.586 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:56.586 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 12:55:56.586 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:56.586 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:55:56.586 | INFO | Uploaded 36GiB to GPU in 2.17609s (16.5434GiB/s) | 20260505 12:55:56.586 | INFO | Downloaded 34GiB to GPU in 2.11039s (16.1108GiB/s) | 20260505 12:56:01.728 | INFO | Analyzing chunk frames = [3374080,3374592) | 20260505 12:56:01.728 | INFO | MemInfo: { | 20260505 12:56:01.728 | INFO | "gpu" : " 17 GB", | 20260505 12:56:01.728 | INFO | "huge" : "", | 20260505 12:56:01.728 | INFO | "huge_shared" : " 27 GB", | 20260505 12:56:01.728 | INFO | "malloc" : " 14 GB", | 20260505 12:56:01.728 | INFO | "pinned" : " 29 GB" | 20260505 12:56:01.728 | INFO | } | 20260505 12:56:01.728 | INFO | Perf report for the last 5.14217 seconds: | 20260505 12:56:01.728 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:56:01.728 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:01.728 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:01.728 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:01.728 | INFO | ANALYSIS : 63.2%, 3.0, 98.8ms, 0 | 20260505 12:56:01.728 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:01.728 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:01.728 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 12:56:01.728 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:01.728 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:01.729 | INFO | Uploaded 36GiB to GPU in 2.17304s (16.5666GiB/s) | 20260505 12:56:01.729 | INFO | Downloaded 34GiB to GPU in 2.15576s (15.7717GiB/s) | 20260505 12:56:06.875 | INFO | Analyzing chunk frames = [3374592,3375104) | 20260505 12:56:06.875 | INFO | MemInfo: { | 20260505 12:56:06.875 | INFO | "gpu" : " 17 GB", | 20260505 12:56:06.875 | INFO | "huge" : "", | 20260505 12:56:06.875 | INFO | "huge_shared" : " 27 GB", | 20260505 12:56:06.875 | INFO | "malloc" : " 14 GB", | 20260505 12:56:06.875 | INFO | "pinned" : " 29 GB" | 20260505 12:56:06.875 | INFO | } | 20260505 12:56:06.875 | INFO | Perf report for the last 5.14646 seconds: | 20260505 12:56:06.875 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:56:06.875 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:06.875 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:06.875 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:06.875 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 12:56:06.875 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:06.875 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:06.875 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 12:56:06.875 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:06.875 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:06.875 | INFO | Uploaded 36GiB to GPU in 2.20563s (16.3218GiB/s) | 20260505 12:56:06.875 | INFO | Downloaded 34GiB to GPU in 2.10826s (16.1271GiB/s) | 20260505 12:56:06.875 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 12:56:06.875 | INFO | Total mem usage: 15.3 GiB | 20260505 12:56:06.875 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 12:56:06.875 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -7%) | 20260505 12:56:06.875 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 12:56:06.875 | INFO | High Water Breakdown: | 20260505 12:56:06.875 | INFO | 61.1% : 8.75 GiB ( 91.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 12:56:06.875 | INFO | 38.8% : 5.56 GiB ( 83.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 12:56:06.875 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 12:56:06.875 | INFO | ------------------------------------------ | 20260505 12:56:06.875 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:56:06.875 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 12:56:06.875 | INFO | Total mem usage: 26.5 GiB | 20260505 12:56:06.875 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 12:56:06.875 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 12:56:06.875 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 12:56:06.875 | INFO | High Water Breakdown: | 20260505 12:56:06.875 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:56:06.875 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:56:06.875 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:56:06.875 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:56:06.875 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:56:06.875 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:56:06.875 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 12:56:06.875 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 12:56:06.875 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:56:06.875 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 12:56:06.875 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 12:56:06.875 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 12:56:06.875 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 12:56:06.875 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 12:56:06.875 | INFO | ------------------------------------------ | 20260505 12:56:06.875 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:56:06.875 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 12:56:06.875 | INFO | Total mem usage: 25.6 GiB | 20260505 12:56:06.875 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 12:56:06.875 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 12:56:06.875 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 12:56:06.875 | INFO | High Water Breakdown: | 20260505 12:56:06.876 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 12:56:06.876 | INFO | ------------------------------------------ | 20260505 12:56:06.876 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:56:06.876 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 12:56:06.876 | INFO | Total mem usage: 15.7 GiB | 20260505 12:56:06.876 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 12:56:06.876 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 12:56:06.876 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 12:56:06.876 | INFO | High Water Breakdown: | 20260505 12:56:06.876 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 12:56:06.876 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 12:56:06.876 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 12:56:06.876 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 12:56:06.876 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 12:56:06.876 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 12:56:06.876 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 12:56:06.876 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 12:56:06.876 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 12:56:06.876 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:56:06.876 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 12:56:06.876 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:56:06.876 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 12:56:06.876 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 12:56:06.876 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 12:56:06.876 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 12:56:06.876 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:56:06.876 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 12:56:06.876 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:56:06.876 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:56:06.876 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:56:06.876 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 12:56:06.876 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 12:56:06.876 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:56:06.876 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 12:56:06.876 | INFO | ------------------------------------------ | 20260505 12:56:06.876 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:56:06.876 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 12:56:06.876 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:56:06.876 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:56:06.876 | INFO | PerfReport: Full Execution -- 3.20e+01 / 7.72e+01, 1.38e+01 / 1.65e+00 | 20260505 12:56:06.876 | INFO | PerfReport: 0.29%, AggregateStats -- 9.31e-02 / 9.99e-02, 1.03e-02 / 4.33e-03 | 20260505 12:56:06.876 | INFO | PerfReport: 11.76%, Baseline -- 3.76e+00 / 4.47e+00, 5.99e-01 / 1.86e-01 | 20260505 12:56:06.876 | INFO | PerfReport: 3.32%, Histogram -- 1.06e+00 / 1.52e+00, 2.06e-01 / 7.44e-02 | 20260505 12:56:06.876 | INFO | PerfReport: 15.98%, DME -- 4.09e+01 / 4.60e+01, 4.52e+00 / 1.91e+00 | 20260505 12:56:06.876 | INFO | PerfReport: 56.32%, FrameLabeling -- 1.80e+01 / 2.02e+01, 1.45e+00 / 1.77e+00 | 20260505 12:56:06.876 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.38e+00 / 2.91e+00, 4.47e-01 / 1.66e-01 | 20260505 12:56:06.876 | INFO | PerfReport: 3.61%, Metrics -- 1.15e+00 / 1.50e+00, 3.26e-01 / 1.87e-01 | 20260505 12:56:06.876 | INFO | PerfReport: 1.27%, PulseSerializing -- 4.06e-01 / 4.57e-01, 7.39e-02 / 3.17e-02 | 20260505 12:56:06.876 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 12:56:06.876 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:56:06.876 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:56:06.876 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260505 12:56:06.876 | INFO | PerfReport: 22.50%, Upload -- 2.27e+01 / 0.00e+00, 8.06e+00 / 0.00e+00 | 20260505 12:56:06.876 | INFO | PerfReport: 22.04%, Download -- 2.23e+01 / 0.00e+00, 8.17e+00 / 0.00e+00 | 20260505 12:56:06.876 | INFO | PerfReport: 31.67%, Compute -- 3.20e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 12:56:06.876 | INFO | PerfReport: 2.19%, LockUpload -- 2.21e+00 / 0.00e+00, 7.82e+00 / 0.00e+00 | 20260505 12:56:06.876 | INFO | PerfReport: 1.42%, LockDownload -- 1.44e+00 / 0.00e+00, 3.03e+00 / 0.00e+00 | 20260505 12:56:06.876 | INFO | PerfReport: 20.18%, LockCompute -- 2.04e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260505 12:56:06.876 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.42e-02 / 0.00e+00 | 20260505 12:56:07.011 | INFO | Starting a round of Baz IO | 20260505 12:56:10.235 | INFO | Populating RTMetrics report | 20260505 12:56:10.235 | INFO | RTMetrics report populated | 20260505 12:56:10.235 | INFO | Opening tmp RTMetrics json file | 20260505 12:56:10.236 | INFO | Serializing RTMetrics to json | 20260505 12:56:10.237 | INFO | Serialization done, swapping tmp file now... | 20260505 12:56:10.238 | INFO | RTMetrics done | 20260505 12:56:10.355 | INFO | Finished a round of Baz IO: Wrote 2.322 GiB in 3.344 seconds | 20260505 12:56:10.355 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.34% / 0.01% / 76.65% | 20260505 12:56:10.355 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.794025% / 1.19209e-05% / 6.75519e-05% | 20260505 12:56:10.355 | INFO | Baz IO overall: | 20260505 12:56:10.355 | INFO | BytesWritten : 2008383616920 | 20260505 12:56:10.355 | INFO | NumEvents : 955771170765 | 20260505 12:56:12.007 | INFO | Analyzing chunk frames = [3375104,3375616) | 20260505 12:56:12.007 | INFO | MemInfo: { | 20260505 12:56:12.007 | INFO | "gpu" : " 17 GB", | 20260505 12:56:12.007 | INFO | "huge" : "", | 20260505 12:56:12.007 | INFO | "huge_shared" : " 27 GB", | 20260505 12:56:12.007 | INFO | "malloc" : " 14 GB", | 20260505 12:56:12.007 | INFO | "pinned" : " 29 GB" | 20260505 12:56:12.007 | INFO | } | 20260505 12:56:12.007 | INFO | Perf report for the last 5.13224 seconds: | 20260505 12:56:12.007 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:56:12.007 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:12.007 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:12.007 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:12.007 | INFO | ANALYSIS : 64.4%, 2.9, 99.3ms, 0 | 20260505 12:56:12.007 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:12.007 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260505 12:56:12.007 | INFO | PRE_HQ : 61.2%, 1.8, 57.7ms, 0 | 20260505 12:56:12.007 | INFO | BAZWRITER : 64.5%, 4.1, 135.8ms, 0 | 20260505 12:56:12.007 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 12:56:12.007 | INFO | Uploaded 36GiB to GPU in 2.19956s (16.3669GiB/s) | 20260505 12:56:12.007 | INFO | Downloaded 38GiB to GPU in 2.27376s (16.7124GiB/s) | 20260505 12:56:17.138 | INFO | Analyzing chunk frames = [3375616,3376128) | 20260505 12:56:17.138 | INFO | MemInfo: { | 20260505 12:56:17.138 | INFO | "gpu" : " 17 GB", | 20260505 12:56:17.138 | INFO | "huge" : "", | 20260505 12:56:17.138 | INFO | "huge_shared" : " 27 GB", | 20260505 12:56:17.138 | INFO | "malloc" : " 14 GB", | 20260505 12:56:17.138 | INFO | "pinned" : " 29 GB" | 20260505 12:56:17.138 | INFO | } | 20260505 12:56:17.138 | INFO | Perf report for the last 5.13102 seconds: | 20260505 12:56:17.138 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:56:17.138 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:17.138 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:17.138 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:17.138 | INFO | ANALYSIS : 63.7%, 3.0, 99.2ms, 0 | 20260505 12:56:17.138 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:17.138 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:17.138 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 12:56:17.138 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:17.138 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:17.138 | INFO | Uploaded 36GiB to GPU in 2.18472s (16.4781GiB/s) | 20260505 12:56:17.138 | INFO | Downloaded 34GiB to GPU in 2.03204s (16.732GiB/s) | 20260505 12:56:22.281 | INFO | Analyzing chunk frames = [3376128,3376640) | 20260505 12:56:22.281 | INFO | MemInfo: { | 20260505 12:56:22.281 | INFO | "gpu" : " 17 GB", | 20260505 12:56:22.281 | INFO | "huge" : "", | 20260505 12:56:22.281 | INFO | "huge_shared" : " 27 GB", | 20260505 12:56:22.281 | INFO | "malloc" : " 14 GB", | 20260505 12:56:22.281 | INFO | "pinned" : " 29 GB" | 20260505 12:56:22.281 | INFO | } | 20260505 12:56:22.281 | INFO | Perf report for the last 5.1428 seconds: | 20260505 12:56:22.281 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:56:22.281 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:22.281 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:22.281 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:22.281 | INFO | ANALYSIS : 63.2%, 3.0, 98.4ms, 0 | 20260505 12:56:22.281 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:22.281 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:22.281 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:56:22.281 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:22.281 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:22.281 | INFO | Uploaded 36GiB to GPU in 2.20908s (16.2964GiB/s) | 20260505 12:56:22.281 | INFO | Downloaded 34GiB to GPU in 2.11355s (16.0866GiB/s) | 20260505 12:56:27.424 | INFO | Analyzing chunk frames = [3376640,3377152) | 20260505 12:56:27.424 | INFO | MemInfo: { | 20260505 12:56:27.424 | INFO | "gpu" : " 17 GB", | 20260505 12:56:27.424 | INFO | "huge" : "", | 20260505 12:56:27.424 | INFO | "huge_shared" : " 27 GB", | 20260505 12:56:27.424 | INFO | "malloc" : " 14 GB", | 20260505 12:56:27.424 | INFO | "pinned" : " 29 GB" | 20260505 12:56:27.424 | INFO | } | 20260505 12:56:27.424 | INFO | Perf report for the last 5.14343 seconds: | 20260505 12:56:27.424 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:56:27.424 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:27.424 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:27.424 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:27.424 | INFO | ANALYSIS : 63.0%, 3.0, 98.0ms, 0 | 20260505 12:56:27.424 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:27.424 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:27.424 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:56:27.424 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:27.424 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:27.425 | INFO | Uploaded 36GiB to GPU in 2.20495s (16.3269GiB/s) | 20260505 12:56:27.425 | INFO | Downloaded 34GiB to GPU in 2.13131s (15.9526GiB/s) | 20260505 12:56:32.562 | INFO | Analyzing chunk frames = [3377152,3377664) | 20260505 12:56:32.562 | INFO | MemInfo: { | 20260505 12:56:32.562 | INFO | "gpu" : " 17 GB", | 20260505 12:56:32.562 | INFO | "huge" : "", | 20260505 12:56:32.562 | INFO | "huge_shared" : " 27 GB", | 20260505 12:56:32.562 | INFO | "malloc" : " 14 GB", | 20260505 12:56:32.562 | INFO | "pinned" : " 29 GB" | 20260505 12:56:32.562 | INFO | } | 20260505 12:56:32.562 | INFO | Perf report for the last 5.13792 seconds: | 20260505 12:56:32.562 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:56:32.562 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:32.562 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:32.562 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:32.562 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 12:56:32.562 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:32.562 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:32.562 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 12:56:32.562 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:32.562 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:32.562 | INFO | Uploaded 36GiB to GPU in 2.17473s (16.5538GiB/s) | 20260505 12:56:32.562 | INFO | Downloaded 34GiB to GPU in 2.12428s (16.0055GiB/s) | 20260505 12:56:37.690 | INFO | Analyzing chunk frames = [3377664,3378176) | 20260505 12:56:37.690 | INFO | MemInfo: { | 20260505 12:56:37.690 | INFO | "gpu" : " 17 GB", | 20260505 12:56:37.691 | INFO | "huge" : "", | 20260505 12:56:37.691 | INFO | "huge_shared" : " 27 GB", | 20260505 12:56:37.691 | INFO | "malloc" : " 14 GB", | 20260505 12:56:37.691 | INFO | "pinned" : " 29 GB" | 20260505 12:56:37.691 | INFO | } | 20260505 12:56:37.691 | INFO | Perf report for the last 5.12837 seconds: | 20260505 12:56:37.691 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:56:37.691 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:37.691 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:37.691 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:37.691 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 12:56:37.691 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:37.691 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:37.691 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 12:56:37.691 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:37.691 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:37.691 | INFO | Uploaded 36GiB to GPU in 2.12141s (16.9699GiB/s) | 20260505 12:56:37.691 | INFO | Downloaded 34GiB to GPU in 2.09764s (16.2087GiB/s) | 20260505 12:56:42.831 | INFO | Analyzing chunk frames = [3378176,3378688) | 20260505 12:56:42.831 | INFO | MemInfo: { | 20260505 12:56:42.831 | INFO | "gpu" : " 17 GB", | 20260505 12:56:42.831 | INFO | "huge" : "", | 20260505 12:56:42.831 | INFO | "huge_shared" : " 27 GB", | 20260505 12:56:42.831 | INFO | "malloc" : " 14 GB", | 20260505 12:56:42.831 | INFO | "pinned" : " 29 GB" | 20260505 12:56:42.831 | INFO | } | 20260505 12:56:42.831 | INFO | Perf report for the last 5.14033 seconds: | 20260505 12:56:42.831 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:56:42.831 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:42.831 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:42.831 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:42.831 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 12:56:42.831 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:42.831 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:42.831 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 12:56:42.831 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:42.831 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:42.831 | INFO | Uploaded 36GiB to GPU in 2.17428s (16.5572GiB/s) | 20260505 12:56:42.831 | INFO | Downloaded 34GiB to GPU in 2.15074s (15.8085GiB/s) | 20260505 12:56:47.981 | INFO | Analyzing chunk frames = [3378688,3379200) | 20260505 12:56:47.981 | INFO | MemInfo: { | 20260505 12:56:47.981 | INFO | "gpu" : " 17 GB", | 20260505 12:56:47.981 | INFO | "huge" : "", | 20260505 12:56:47.981 | INFO | "huge_shared" : " 27 GB", | 20260505 12:56:47.981 | INFO | "malloc" : " 14 GB", | 20260505 12:56:47.981 | INFO | "pinned" : " 29 GB" | 20260505 12:56:47.981 | INFO | } | 20260505 12:56:47.981 | INFO | Perf report for the last 5.15 seconds: | 20260505 12:56:47.981 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:56:47.981 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:47.981 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:47.981 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:47.981 | INFO | ANALYSIS : 63.7%, 3.0, 99.7ms, 0 | 20260505 12:56:47.981 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:47.981 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:47.981 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 12:56:47.981 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:47.981 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:47.981 | INFO | Uploaded 36GiB to GPU in 2.21425s (16.2583GiB/s) | 20260505 12:56:47.981 | INFO | Downloaded 34GiB to GPU in 2.09633s (16.2188GiB/s) | 20260505 12:56:48.115 | INFO | Starting a round of Baz IO | 20260505 12:56:51.361 | INFO | Populating RTMetrics report | 20260505 12:56:51.361 | INFO | RTMetrics report populated | 20260505 12:56:51.361 | INFO | Opening tmp RTMetrics json file | 20260505 12:56:51.385 | INFO | Serializing RTMetrics to json | 20260505 12:56:51.431 | INFO | Serialization done, swapping tmp file now... | 20260505 12:56:51.437 | INFO | RTMetrics done | 20260505 12:56:51.474 | INFO | Finished a round of Baz IO: Wrote 2.32 GiB in 3.358 seconds | 20260505 12:56:51.474 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.33% / 0.01% / 76.66% | 20260505 12:56:51.474 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.790246% / 3.97364e-06% / 9.53674e-05% | 20260505 12:56:51.474 | INFO | Baz IO overall: | 20260505 12:56:51.474 | INFO | BytesWritten : 2010874623224 | 20260505 12:56:51.474 | INFO | NumEvents : 956950910201 | 20260505 12:56:53.124 | INFO | Analyzing chunk frames = [3379200,3379712) | 20260505 12:56:53.124 | INFO | MemInfo: { | 20260505 12:56:53.124 | INFO | "gpu" : " 17 GB", | 20260505 12:56:53.124 | INFO | "huge" : "", | 20260505 12:56:53.124 | INFO | "huge_shared" : " 27 GB", | 20260505 12:56:53.124 | INFO | "malloc" : " 14 GB", | 20260505 12:56:53.124 | INFO | "pinned" : " 29 GB" | 20260505 12:56:53.124 | INFO | } | 20260505 12:56:53.124 | INFO | Perf report for the last 5.14341 seconds: | 20260505 12:56:53.124 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:56:53.124 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:53.124 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:53.124 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:53.125 | INFO | ANALYSIS : 64.6%, 3.0, 100.0ms, 0 | 20260505 12:56:53.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:53.125 | INFO | RT_METRICS : 46.7%, 1.0, 24.5ms, 0 | 20260505 12:56:53.125 | INFO | PRE_HQ : 60.7%, 1.8, 57.8ms, 0 | 20260505 12:56:53.125 | INFO | BAZWRITER : 64.6%, 4.0, 134.4ms, 0 | 20260505 12:56:53.125 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 12:56:53.125 | INFO | Uploaded 36GiB to GPU in 2.19538s (16.3981GiB/s) | 20260505 12:56:53.125 | INFO | Downloaded 38GiB to GPU in 2.27726s (16.6867GiB/s) | 20260505 12:56:58.261 | INFO | Analyzing chunk frames = [3379712,3380224) | 20260505 12:56:58.261 | INFO | MemInfo: { | 20260505 12:56:58.261 | INFO | "gpu" : " 17 GB", | 20260505 12:56:58.261 | INFO | "huge" : "", | 20260505 12:56:58.261 | INFO | "huge_shared" : " 27 GB", | 20260505 12:56:58.261 | INFO | "malloc" : " 14 GB", | 20260505 12:56:58.261 | INFO | "pinned" : " 29 GB" | 20260505 12:56:58.261 | INFO | } | 20260505 12:56:58.261 | INFO | Perf report for the last 5.13635 seconds: | 20260505 12:56:58.261 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:56:58.261 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:58.261 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:58.261 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:58.261 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 12:56:58.261 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:58.261 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:58.261 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 12:56:58.261 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:58.261 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:56:58.261 | INFO | Uploaded 36GiB to GPU in 2.17441s (16.5562GiB/s) | 20260505 12:56:58.261 | INFO | Downloaded 34GiB to GPU in 2.02829s (16.7629GiB/s) | 20260505 12:57:03.390 | INFO | Analyzing chunk frames = [3380224,3380736) | 20260505 12:57:03.390 | INFO | MemInfo: { | 20260505 12:57:03.390 | INFO | "gpu" : " 17 GB", | 20260505 12:57:03.390 | INFO | "huge" : "", | 20260505 12:57:03.390 | INFO | "huge_shared" : " 27 GB", | 20260505 12:57:03.390 | INFO | "malloc" : " 14 GB", | 20260505 12:57:03.390 | INFO | "pinned" : " 29 GB" | 20260505 12:57:03.390 | INFO | } | 20260505 12:57:03.391 | INFO | Perf report for the last 5.12973 seconds: | 20260505 12:57:03.391 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:57:03.391 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:03.391 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:03.391 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:03.391 | INFO | ANALYSIS : 63.6%, 3.0, 98.8ms, 0 | 20260505 12:57:03.391 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:03.391 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:03.391 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:57:03.391 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:03.391 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:03.391 | INFO | Uploaded 36GiB to GPU in 2.22741s (16.1622GiB/s) | 20260505 12:57:03.391 | INFO | Downloaded 34GiB to GPU in 2.10602s (16.1442GiB/s) | 20260505 12:57:08.537 | INFO | Analyzing chunk frames = [3380736,3381248) | 20260505 12:57:08.537 | INFO | MemInfo: { | 20260505 12:57:08.537 | INFO | "gpu" : " 17 GB", | 20260505 12:57:08.537 | INFO | "huge" : "", | 20260505 12:57:08.538 | INFO | "huge_shared" : " 27 GB", | 20260505 12:57:08.538 | INFO | "malloc" : " 14 GB", | 20260505 12:57:08.538 | INFO | "pinned" : " 29 GB" | 20260505 12:57:08.538 | INFO | } | 20260505 12:57:08.538 | INFO | Perf report for the last 5.14711 seconds: | 20260505 12:57:08.538 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:57:08.538 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:08.538 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:08.538 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:08.538 | INFO | ANALYSIS : 63.2%, 3.0, 98.3ms, 0 | 20260505 12:57:08.538 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:08.538 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:08.538 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:57:08.538 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:08.538 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:08.538 | INFO | Uploaded 36GiB to GPU in 2.20826s (16.3024GiB/s) | 20260505 12:57:08.538 | INFO | Downloaded 34GiB to GPU in 2.1351s (15.9243GiB/s) | 20260505 12:57:13.673 | INFO | Analyzing chunk frames = [3381248,3381760) | 20260505 12:57:13.673 | INFO | MemInfo: { | 20260505 12:57:13.673 | INFO | "gpu" : " 17 GB", | 20260505 12:57:13.673 | INFO | "huge" : "", | 20260505 12:57:13.673 | INFO | "huge_shared" : " 27 GB", | 20260505 12:57:13.673 | INFO | "malloc" : " 14 GB", | 20260505 12:57:13.673 | INFO | "pinned" : " 29 GB" | 20260505 12:57:13.673 | INFO | } | 20260505 12:57:13.673 | INFO | Perf report for the last 5.13546 seconds: | 20260505 12:57:13.673 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:57:13.673 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:13.673 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:13.673 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:13.673 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 12:57:13.673 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:13.673 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:13.673 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 12:57:13.673 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:13.673 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:13.673 | INFO | Uploaded 36GiB to GPU in 2.16787s (16.6062GiB/s) | 20260505 12:57:13.673 | INFO | Downloaded 34GiB to GPU in 2.12647s (15.9889GiB/s) | 20260505 12:57:18.823 | INFO | Analyzing chunk frames = [3381760,3382272) | 20260505 12:57:18.823 | INFO | MemInfo: { | 20260505 12:57:18.823 | INFO | "gpu" : " 17 GB", | 20260505 12:57:18.823 | INFO | "huge" : "", | 20260505 12:57:18.823 | INFO | "huge_shared" : " 27 GB", | 20260505 12:57:18.823 | INFO | "malloc" : " 14 GB", | 20260505 12:57:18.823 | INFO | "pinned" : " 29 GB" | 20260505 12:57:18.823 | INFO | } | 20260505 12:57:18.823 | INFO | Perf report for the last 5.14998 seconds: | 20260505 12:57:18.823 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:57:18.823 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:18.823 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:18.823 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:18.823 | INFO | ANALYSIS : 63.0%, 3.0, 98.6ms, 0 | 20260505 12:57:18.823 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:18.823 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:18.823 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:57:18.823 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:18.823 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:18.823 | INFO | Uploaded 36GiB to GPU in 2.16521s (16.6266GiB/s) | 20260505 12:57:18.823 | INFO | Downloaded 34GiB to GPU in 2.09818s (16.2045GiB/s) | 20260505 12:57:23.961 | INFO | Analyzing chunk frames = [3382272,3382784) | 20260505 12:57:23.961 | INFO | MemInfo: { | 20260505 12:57:23.961 | INFO | "gpu" : " 17 GB", | 20260505 12:57:23.961 | INFO | "huge" : "", | 20260505 12:57:23.961 | INFO | "huge_shared" : " 27 GB", | 20260505 12:57:23.961 | INFO | "malloc" : " 14 GB", | 20260505 12:57:23.961 | INFO | "pinned" : " 29 GB" | 20260505 12:57:23.961 | INFO | } | 20260505 12:57:23.961 | INFO | Perf report for the last 5.13788 seconds: | 20260505 12:57:23.961 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:57:23.961 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:23.961 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:23.961 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:23.961 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 12:57:23.961 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:23.961 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:23.961 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 12:57:23.961 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:23.961 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:23.961 | INFO | Uploaded 36GiB to GPU in 2.142s (16.8067GiB/s) | 20260505 12:57:23.961 | INFO | Downloaded 34GiB to GPU in 2.178s (15.6106GiB/s) | 20260505 12:57:29.091 | INFO | Analyzing chunk frames = [3382784,3383296) | 20260505 12:57:29.091 | INFO | MemInfo: { | 20260505 12:57:29.091 | INFO | "gpu" : " 17 GB", | 20260505 12:57:29.091 | INFO | "huge" : "", | 20260505 12:57:29.091 | INFO | "huge_shared" : " 27 GB", | 20260505 12:57:29.091 | INFO | "malloc" : " 14 GB", | 20260505 12:57:29.091 | INFO | "pinned" : " 29 GB" | 20260505 12:57:29.091 | INFO | } | 20260505 12:57:29.092 | INFO | Perf report for the last 5.13053 seconds: | 20260505 12:57:29.092 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:57:29.092 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:29.092 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:29.092 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:29.092 | INFO | ANALYSIS : 63.8%, 3.0, 99.4ms, 0 | 20260505 12:57:29.092 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:29.092 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:29.092 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 12:57:29.092 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:29.092 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:29.092 | INFO | Uploaded 36GiB to GPU in 2.17055s (16.5856GiB/s) | 20260505 12:57:29.092 | INFO | Downloaded 34GiB to GPU in 2.10649s (16.1406GiB/s) | 20260505 12:57:29.092 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 12:57:29.092 | INFO | Total mem usage: 15.3 GiB | 20260505 12:57:29.092 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 12:57:29.092 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -7.03%) | 20260505 12:57:29.092 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 12:57:29.092 | INFO | High Water Breakdown: | 20260505 12:57:29.092 | INFO | 61.1% : 8.75 GiB ( 92% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 12:57:29.092 | INFO | 38.7% : 5.55 GiB ( 83.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 12:57:29.092 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 12:57:29.092 | INFO | ------------------------------------------ | 20260505 12:57:29.092 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:57:29.092 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 12:57:29.092 | INFO | Total mem usage: 26.5 GiB | 20260505 12:57:29.092 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 12:57:29.092 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 12:57:29.092 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 12:57:29.092 | INFO | High Water Breakdown: | 20260505 12:57:29.092 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:57:29.092 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:57:29.092 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:57:29.092 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:57:29.092 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:57:29.092 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:57:29.092 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 12:57:29.092 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 12:57:29.092 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:57:29.092 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 12:57:29.092 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 12:57:29.092 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 12:57:29.092 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 12:57:29.092 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 12:57:29.092 | INFO | ------------------------------------------ | 20260505 12:57:29.092 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:57:29.092 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 12:57:29.092 | INFO | Total mem usage: 25.6 GiB | 20260505 12:57:29.092 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 12:57:29.093 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 12:57:29.093 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 12:57:29.093 | INFO | High Water Breakdown: | 20260505 12:57:29.093 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 12:57:29.093 | INFO | ------------------------------------------ | 20260505 12:57:29.093 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:57:29.093 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 12:57:29.093 | INFO | Total mem usage: 15.7 GiB | 20260505 12:57:29.093 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 12:57:29.093 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 12:57:29.093 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 12:57:29.093 | INFO | High Water Breakdown: | 20260505 12:57:29.093 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 12:57:29.093 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 12:57:29.093 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 12:57:29.093 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 12:57:29.093 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 12:57:29.093 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 12:57:29.093 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 12:57:29.093 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 12:57:29.093 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 12:57:29.093 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:57:29.093 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 12:57:29.093 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:57:29.093 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 12:57:29.093 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 12:57:29.093 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 12:57:29.093 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 12:57:29.093 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:57:29.093 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 12:57:29.093 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:57:29.093 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:57:29.093 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:57:29.093 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 12:57:29.093 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 12:57:29.093 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:57:29.093 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 12:57:29.093 | INFO | ------------------------------------------ | 20260505 12:57:29.093 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:57:29.093 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 12:57:29.093 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:57:29.093 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:57:29.093 | INFO | PerfReport: Full Execution -- 3.20e+01 / 7.68e+01, 1.38e+01 / 1.70e+00 | 20260505 12:57:29.093 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 1.00e-01, 1.04e-02 / 3.92e-03 | 20260505 12:57:29.093 | INFO | PerfReport: 11.77%, Baseline -- 3.76e+00 / 4.47e+00, 5.91e-01 / 2.33e-01 | 20260505 12:57:29.093 | INFO | PerfReport: 3.31%, Histogram -- 1.06e+00 / 1.48e+00, 2.02e-01 / 1.07e-01 | 20260505 12:57:29.093 | INFO | PerfReport: 15.90%, DME -- 4.07e+01 / 4.54e+01, 4.45e+00 / 2.18e+00 | 20260505 12:57:29.093 | INFO | PerfReport: 56.44%, FrameLabeling -- 1.80e+01 / 2.06e+01, 1.50e+00 / 1.97e+00 | 20260505 12:57:29.093 | INFO | PerfReport: 7.41%, PulseAccumulating -- 2.37e+00 / 2.87e+00, 4.36e-01 / 2.48e-01 | 20260505 12:57:29.093 | INFO | PerfReport: 3.61%, Metrics -- 1.15e+00 / 1.46e+00, 3.23e-01 / 1.83e-01 | 20260505 12:57:29.093 | INFO | PerfReport: 1.27%, PulseSerializing -- 4.05e-01 / 4.52e-01, 7.28e-02 / 3.88e-02 | 20260505 12:57:29.093 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 12:57:29.093 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:57:29.093 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:57:29.093 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260505 12:57:29.093 | INFO | PerfReport: 22.53%, Upload -- 2.27e+01 / 0.00e+00, 8.05e+00 / 0.00e+00 | 20260505 12:57:29.093 | INFO | PerfReport: 21.98%, Download -- 2.22e+01 / 0.00e+00, 8.01e+00 / 0.00e+00 | 20260505 12:57:29.093 | INFO | PerfReport: 31.67%, Compute -- 3.20e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 12:57:29.093 | INFO | PerfReport: 2.15%, LockUpload -- 2.17e+00 / 0.00e+00, 7.71e+00 / 0.00e+00 | 20260505 12:57:29.093 | INFO | PerfReport: 1.38%, LockDownload -- 1.39e+00 / 0.00e+00, 2.94e+00 / 0.00e+00 | 20260505 12:57:29.093 | INFO | PerfReport: 20.28%, LockCompute -- 2.05e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260505 12:57:29.093 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.65e-02 / 0.00e+00 | 20260505 12:57:29.223 | INFO | Starting a round of Baz IO | 20260505 12:57:32.473 | INFO | Populating RTMetrics report | 20260505 12:57:32.473 | INFO | RTMetrics report populated | 20260505 12:57:32.473 | INFO | Opening tmp RTMetrics json file | 20260505 12:57:32.476 | INFO | Serializing RTMetrics to json | 20260505 12:57:32.500 | INFO | Serialization done, swapping tmp file now... | 20260505 12:57:32.501 | INFO | RTMetrics done | 20260505 12:57:32.599 | INFO | Finished a round of Baz IO: Wrote 2.321 GiB in 3.375 seconds | 20260505 12:57:32.599 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.33% / 0.01% / 76.67% | 20260505 12:57:32.599 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.790842% / 7.94729e-06% / 9.13938e-05% | 20260505 12:57:32.599 | INFO | Baz IO overall: | 20260505 12:57:32.599 | INFO | BytesWritten : 2013366572713 | 20260505 12:57:32.599 | INFO | NumEvents : 958131693292 | 20260505 12:57:34.235 | INFO | Analyzing chunk frames = [3383296,3383808) | 20260505 12:57:34.235 | INFO | MemInfo: { | 20260505 12:57:34.235 | INFO | "gpu" : " 17 GB", | 20260505 12:57:34.235 | INFO | "huge" : "", | 20260505 12:57:34.235 | INFO | "huge_shared" : " 27 GB", | 20260505 12:57:34.235 | INFO | "malloc" : " 14 GB", | 20260505 12:57:34.235 | INFO | "pinned" : " 29 GB" | 20260505 12:57:34.235 | INFO | } | 20260505 12:57:34.235 | INFO | Perf report for the last 5.14323 seconds: | 20260505 12:57:34.235 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:57:34.235 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:34.235 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:34.235 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:34.235 | INFO | ANALYSIS : 64.6%, 2.9, 99.8ms, 0 | 20260505 12:57:34.235 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:34.235 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260505 12:57:34.235 | INFO | PRE_HQ : 61.0%, 1.8, 58.2ms, 0 | 20260505 12:57:34.235 | INFO | BAZWRITER : 64.9%, 4.1, 135.9ms, 0 | 20260505 12:57:34.235 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 12:57:34.235 | INFO | Uploaded 36GiB to GPU in 2.18799s (16.4535GiB/s) | 20260505 12:57:34.235 | INFO | Downloaded 38GiB to GPU in 2.27692s (16.6892GiB/s) | 20260505 12:57:39.376 | INFO | Analyzing chunk frames = [3383808,3384320) | 20260505 12:57:39.377 | INFO | MemInfo: { | 20260505 12:57:39.377 | INFO | "gpu" : " 17 GB", | 20260505 12:57:39.377 | INFO | "huge" : "", | 20260505 12:57:39.377 | INFO | "huge_shared" : " 27 GB", | 20260505 12:57:39.377 | INFO | "malloc" : " 14 GB", | 20260505 12:57:39.377 | INFO | "pinned" : " 29 GB" | 20260505 12:57:39.377 | INFO | } | 20260505 12:57:39.377 | INFO | Perf report for the last 5.14188 seconds: | 20260505 12:57:39.377 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:57:39.377 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:39.377 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:39.377 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:39.377 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 12:57:39.377 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:39.377 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:39.377 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:57:39.377 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:39.377 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:39.377 | INFO | Uploaded 36GiB to GPU in 2.19058s (16.434GiB/s) | 20260505 12:57:39.377 | INFO | Downloaded 34GiB to GPU in 2.03167s (16.735GiB/s) | 20260505 12:57:44.509 | INFO | Analyzing chunk frames = [3384320,3384832) | 20260505 12:57:44.509 | INFO | MemInfo: { | 20260505 12:57:44.509 | INFO | "gpu" : " 17 GB", | 20260505 12:57:44.509 | INFO | "huge" : "", | 20260505 12:57:44.509 | INFO | "huge_shared" : " 27 GB", | 20260505 12:57:44.509 | INFO | "malloc" : " 14 GB", | 20260505 12:57:44.509 | INFO | "pinned" : " 29 GB" | 20260505 12:57:44.509 | INFO | } | 20260505 12:57:44.509 | INFO | Perf report for the last 5.13252 seconds: | 20260505 12:57:44.509 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:57:44.509 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:44.509 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:44.509 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:44.509 | INFO | ANALYSIS : 63.7%, 3.0, 99.0ms, 0 | 20260505 12:57:44.509 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:44.509 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:44.509 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 12:57:44.509 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:44.509 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:44.509 | INFO | Uploaded 36GiB to GPU in 2.23861s (16.0814GiB/s) | 20260505 12:57:44.509 | INFO | Downloaded 34GiB to GPU in 2.11379s (16.0848GiB/s) | 20260505 12:57:49.643 | INFO | Analyzing chunk frames = [3384832,3385344) | 20260505 12:57:49.643 | INFO | MemInfo: { | 20260505 12:57:49.643 | INFO | "gpu" : " 17 GB", | 20260505 12:57:49.643 | INFO | "huge" : "", | 20260505 12:57:49.643 | INFO | "huge_shared" : " 27 GB", | 20260505 12:57:49.643 | INFO | "malloc" : " 14 GB", | 20260505 12:57:49.643 | INFO | "pinned" : " 29 GB" | 20260505 12:57:49.643 | INFO | } | 20260505 12:57:49.643 | INFO | Perf report for the last 5.13397 seconds: | 20260505 12:57:49.643 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:57:49.643 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:49.643 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:49.643 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:49.643 | INFO | ANALYSIS : 63.5%, 3.0, 98.5ms, 0 | 20260505 12:57:49.643 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:49.643 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:49.643 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 12:57:49.643 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:49.643 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:49.643 | INFO | Uploaded 36GiB to GPU in 2.21587s (16.2464GiB/s) | 20260505 12:57:49.643 | INFO | Downloaded 34GiB to GPU in 2.13272s (15.9421GiB/s) | 20260505 12:57:54.790 | INFO | Analyzing chunk frames = [3385344,3385856) | 20260505 12:57:54.790 | INFO | MemInfo: { | 20260505 12:57:54.790 | INFO | "gpu" : " 17 GB", | 20260505 12:57:54.790 | INFO | "huge" : "", | 20260505 12:57:54.790 | INFO | "huge_shared" : " 27 GB", | 20260505 12:57:54.790 | INFO | "malloc" : " 14 GB", | 20260505 12:57:54.790 | INFO | "pinned" : " 29 GB" | 20260505 12:57:54.790 | INFO | } | 20260505 12:57:54.790 | INFO | Perf report for the last 5.14728 seconds: | 20260505 12:57:54.790 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:57:54.790 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:54.790 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:54.790 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:54.790 | INFO | ANALYSIS : 62.8%, 3.0, 98.3ms, 0 | 20260505 12:57:54.791 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:54.791 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:54.791 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 12:57:54.791 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:54.791 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:54.791 | INFO | Uploaded 36GiB to GPU in 2.15105s (16.736GiB/s) | 20260505 12:57:54.791 | INFO | Downloaded 34GiB to GPU in 2.127s (15.985GiB/s) | 20260505 12:57:59.933 | INFO | Analyzing chunk frames = [3385856,3386368) | 20260505 12:57:59.933 | INFO | MemInfo: { | 20260505 12:57:59.933 | INFO | "gpu" : " 17 GB", | 20260505 12:57:59.933 | INFO | "huge" : "", | 20260505 12:57:59.933 | INFO | "huge_shared" : " 27 GB", | 20260505 12:57:59.933 | INFO | "malloc" : " 14 GB", | 20260505 12:57:59.934 | INFO | "pinned" : " 29 GB" | 20260505 12:57:59.934 | INFO | } | 20260505 12:57:59.934 | INFO | Perf report for the last 5.14317 seconds: | 20260505 12:57:59.934 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:57:59.934 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:59.934 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:59.934 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:59.934 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 12:57:59.934 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:59.934 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:59.934 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 12:57:59.934 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:59.934 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:57:59.934 | INFO | Uploaded 36GiB to GPU in 2.17174s (16.5766GiB/s) | 20260505 12:57:59.934 | INFO | Downloaded 34GiB to GPU in 2.10693s (16.1372GiB/s) | 20260505 12:58:05.076 | INFO | Analyzing chunk frames = [3386368,3386880) | 20260505 12:58:05.076 | INFO | MemInfo: { | 20260505 12:58:05.076 | INFO | "gpu" : " 17 GB", | 20260505 12:58:05.076 | INFO | "huge" : "", | 20260505 12:58:05.076 | INFO | "huge_shared" : " 27 GB", | 20260505 12:58:05.076 | INFO | "malloc" : " 14 GB", | 20260505 12:58:05.076 | INFO | "pinned" : " 29 GB" | 20260505 12:58:05.076 | INFO | } | 20260505 12:58:05.076 | INFO | Perf report for the last 5.14272 seconds: | 20260505 12:58:05.076 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:58:05.076 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:05.076 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:05.076 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:05.076 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 12:58:05.076 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:05.076 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:05.076 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 12:58:05.076 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:05.076 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:05.077 | INFO | Uploaded 36GiB to GPU in 2.13662s (16.849GiB/s) | 20260505 12:58:05.077 | INFO | Downloaded 34GiB to GPU in 2.14966s (15.8165GiB/s) | 20260505 12:58:10.206 | INFO | Analyzing chunk frames = [3386880,3387392) | 20260505 12:58:10.206 | INFO | MemInfo: { | 20260505 12:58:10.206 | INFO | "gpu" : " 17 GB", | 20260505 12:58:10.206 | INFO | "huge" : "", | 20260505 12:58:10.206 | INFO | "huge_shared" : " 27 GB", | 20260505 12:58:10.206 | INFO | "malloc" : " 14 GB", | 20260505 12:58:10.206 | INFO | "pinned" : " 29 GB" | 20260505 12:58:10.206 | INFO | } | 20260505 12:58:10.206 | INFO | Perf report for the last 5.12977 seconds: | 20260505 12:58:10.206 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:58:10.206 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:10.206 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:10.206 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:10.206 | INFO | ANALYSIS : 63.8%, 3.0, 99.5ms, 0 | 20260505 12:58:10.206 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:10.206 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:10.206 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 12:58:10.206 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:10.206 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:10.206 | INFO | Uploaded 36GiB to GPU in 2.17106s (16.5818GiB/s) | 20260505 12:58:10.206 | INFO | Downloaded 34GiB to GPU in 2.11387s (16.0842GiB/s) | 20260505 12:58:10.339 | INFO | Starting a round of Baz IO | 20260505 12:58:13.594 | INFO | Populating RTMetrics report | 20260505 12:58:13.594 | INFO | RTMetrics report populated | 20260505 12:58:13.594 | INFO | Opening tmp RTMetrics json file | 20260505 12:58:13.626 | INFO | Serializing RTMetrics to json | 20260505 12:58:13.668 | INFO | Serialization done, swapping tmp file now... | 20260505 12:58:13.685 | INFO | RTMetrics done | 20260505 12:58:13.710 | INFO | Finished a round of Baz IO: Wrote 2.318 GiB in 3.37 seconds | 20260505 12:58:13.710 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.32% / 0.01% / 76.67% | 20260505 12:58:13.710 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.78464% / 3.97364e-06% / 7.54992e-05% | 20260505 12:58:13.710 | INFO | Baz IO overall: | 20260505 12:58:13.710 | INFO | BytesWritten : 2015855966989 | 20260505 12:58:13.710 | INFO | NumEvents : 959310392665 | 20260505 12:58:15.343 | INFO | Analyzing chunk frames = [3387392,3387904) | 20260505 12:58:15.343 | INFO | MemInfo: { | 20260505 12:58:15.343 | INFO | "gpu" : " 17 GB", | 20260505 12:58:15.343 | INFO | "huge" : "", | 20260505 12:58:15.343 | INFO | "huge_shared" : " 27 GB", | 20260505 12:58:15.343 | INFO | "malloc" : " 14 GB", | 20260505 12:58:15.343 | INFO | "pinned" : " 29 GB" | 20260505 12:58:15.343 | INFO | } | 20260505 12:58:15.343 | INFO | Perf report for the last 5.13728 seconds: | 20260505 12:58:15.343 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:58:15.343 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:15.343 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:15.343 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:15.343 | INFO | ANALYSIS : 64.8%, 3.0, 100.7ms, 0 | 20260505 12:58:15.343 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:15.344 | INFO | RT_METRICS : 46.8%, 1.0, 24.6ms, 0 | 20260505 12:58:15.344 | INFO | PRE_HQ : 60.8%, 1.8, 57.8ms, 0 | 20260505 12:58:15.344 | INFO | BAZWRITER : 64.9%, 4.0, 132.9ms, 0 | 20260505 12:58:15.344 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 12:58:15.344 | INFO | Uploaded 36GiB to GPU in 2.19404s (16.4081GiB/s) | 20260505 12:58:15.344 | INFO | Downloaded 38GiB to GPU in 2.28046s (16.6633GiB/s) | 20260505 12:58:20.489 | INFO | Analyzing chunk frames = [3387904,3388416) | 20260505 12:58:20.489 | INFO | MemInfo: { | 20260505 12:58:20.489 | INFO | "gpu" : " 17 GB", | 20260505 12:58:20.490 | INFO | "huge" : "", | 20260505 12:58:20.490 | INFO | "huge_shared" : " 27 GB", | 20260505 12:58:20.490 | INFO | "malloc" : " 14 GB", | 20260505 12:58:20.490 | INFO | "pinned" : " 29 GB" | 20260505 12:58:20.490 | INFO | } | 20260505 12:58:20.490 | INFO | Perf report for the last 5.14617 seconds: | 20260505 12:58:20.490 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:58:20.490 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:20.490 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:20.490 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:20.490 | INFO | ANALYSIS : 63.7%, 3.0, 99.6ms, 0 | 20260505 12:58:20.490 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:20.490 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:20.490 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:58:20.490 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:20.490 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:20.490 | INFO | Uploaded 36GiB to GPU in 2.17785s (16.53GiB/s) | 20260505 12:58:20.490 | INFO | Downloaded 34GiB to GPU in 2.04502s (16.6257GiB/s) | 20260505 12:58:25.627 | INFO | Analyzing chunk frames = [3388416,3388928) | 20260505 12:58:25.627 | INFO | MemInfo: { | 20260505 12:58:25.627 | INFO | "gpu" : " 17 GB", | 20260505 12:58:25.627 | INFO | "huge" : "", | 20260505 12:58:25.627 | INFO | "huge_shared" : " 27 GB", | 20260505 12:58:25.627 | INFO | "malloc" : " 14 GB", | 20260505 12:58:25.627 | INFO | "pinned" : " 29 GB" | 20260505 12:58:25.627 | INFO | } | 20260505 12:58:25.627 | INFO | Perf report for the last 5.13775 seconds: | 20260505 12:58:25.627 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:58:25.627 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:25.627 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:25.627 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:25.627 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 12:58:25.627 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:25.627 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:25.627 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:58:25.627 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:25.627 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:25.628 | INFO | Uploaded 36GiB to GPU in 2.20756s (16.3076GiB/s) | 20260505 12:58:25.628 | INFO | Downloaded 34GiB to GPU in 2.08738s (16.2884GiB/s) | 20260505 12:58:30.767 | INFO | Analyzing chunk frames = [3388928,3389440) | 20260505 12:58:30.767 | INFO | MemInfo: { | 20260505 12:58:30.767 | INFO | "gpu" : " 17 GB", | 20260505 12:58:30.767 | INFO | "huge" : "", | 20260505 12:58:30.767 | INFO | "huge_shared" : " 27 GB", | 20260505 12:58:30.767 | INFO | "malloc" : " 14 GB", | 20260505 12:58:30.767 | INFO | "pinned" : " 29 GB" | 20260505 12:58:30.767 | INFO | } | 20260505 12:58:30.767 | INFO | Perf report for the last 5.13966 seconds: | 20260505 12:58:30.767 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:58:30.767 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:30.767 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:30.767 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:30.767 | INFO | ANALYSIS : 63.1%, 3.0, 98.0ms, 0 | 20260505 12:58:30.767 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:30.767 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:30.767 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 12:58:30.767 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:30.767 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:30.767 | INFO | Uploaded 36GiB to GPU in 2.20218s (16.3474GiB/s) | 20260505 12:58:30.767 | INFO | Downloaded 34GiB to GPU in 2.13089s (15.9558GiB/s) | 20260505 12:58:35.906 | INFO | Analyzing chunk frames = [3389440,3389952) | 20260505 12:58:35.906 | INFO | MemInfo: { | 20260505 12:58:35.906 | INFO | "gpu" : " 17 GB", | 20260505 12:58:35.906 | INFO | "huge" : "", | 20260505 12:58:35.906 | INFO | "huge_shared" : " 27 GB", | 20260505 12:58:35.906 | INFO | "malloc" : " 14 GB", | 20260505 12:58:35.906 | INFO | "pinned" : " 29 GB" | 20260505 12:58:35.906 | INFO | } | 20260505 12:58:35.906 | INFO | Perf report for the last 5.13897 seconds: | 20260505 12:58:35.906 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:58:35.906 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:35.906 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:35.906 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:35.906 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 12:58:35.906 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:35.906 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:35.906 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 12:58:35.906 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:35.906 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:35.906 | INFO | Uploaded 36GiB to GPU in 2.18079s (16.5078GiB/s) | 20260505 12:58:35.906 | INFO | Downloaded 34GiB to GPU in 2.13094s (15.9554GiB/s) | 20260505 12:58:41.047 | INFO | Analyzing chunk frames = [3389952,3390464) | 20260505 12:58:41.047 | INFO | MemInfo: { | 20260505 12:58:41.047 | INFO | "gpu" : " 17 GB", | 20260505 12:58:41.047 | INFO | "huge" : "", | 20260505 12:58:41.047 | INFO | "huge_shared" : " 27 GB", | 20260505 12:58:41.047 | INFO | "malloc" : " 14 GB", | 20260505 12:58:41.047 | INFO | "pinned" : " 29 GB" | 20260505 12:58:41.047 | INFO | } | 20260505 12:58:41.047 | INFO | Perf report for the last 5.14128 seconds: | 20260505 12:58:41.047 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:58:41.047 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:41.047 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:41.047 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:41.047 | INFO | ANALYSIS : 63.2%, 3.0, 98.8ms, 0 | 20260505 12:58:41.047 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:41.047 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:41.047 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 12:58:41.047 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:41.047 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:41.048 | INFO | Uploaded 36GiB to GPU in 2.17443s (16.5561GiB/s) | 20260505 12:58:41.048 | INFO | Downloaded 34GiB to GPU in 2.11205s (16.0981GiB/s) | 20260505 12:58:46.186 | INFO | Analyzing chunk frames = [3390464,3390976) | 20260505 12:58:46.186 | INFO | MemInfo: { | 20260505 12:58:46.186 | INFO | "gpu" : " 17 GB", | 20260505 12:58:46.186 | INFO | "huge" : "", | 20260505 12:58:46.186 | INFO | "huge_shared" : " 27 GB", | 20260505 12:58:46.186 | INFO | "malloc" : " 14 GB", | 20260505 12:58:46.186 | INFO | "pinned" : " 29 GB" | 20260505 12:58:46.186 | INFO | } | 20260505 12:58:46.186 | INFO | Perf report for the last 5.13907 seconds: | 20260505 12:58:46.186 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:58:46.186 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:46.186 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:46.186 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:46.186 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 12:58:46.186 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:46.186 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:46.186 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 12:58:46.186 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:46.187 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:46.187 | INFO | Uploaded 36GiB to GPU in 2.14447s (16.7874GiB/s) | 20260505 12:58:46.187 | INFO | Downloaded 34GiB to GPU in 2.17153s (15.6571GiB/s) | 20260505 12:58:51.323 | INFO | Analyzing chunk frames = [3390976,3391488) | 20260505 12:58:51.323 | INFO | MemInfo: { | 20260505 12:58:51.323 | INFO | "gpu" : " 17 GB", | 20260505 12:58:51.323 | INFO | "huge" : "", | 20260505 12:58:51.324 | INFO | "huge_shared" : " 27 GB", | 20260505 12:58:51.324 | INFO | "malloc" : " 14 GB", | 20260505 12:58:51.324 | INFO | "pinned" : " 29 GB" | 20260505 12:58:51.324 | INFO | } | 20260505 12:58:51.324 | INFO | Perf report for the last 5.13721 seconds: | 20260505 12:58:51.324 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:58:51.324 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:51.324 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:51.324 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:51.324 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 12:58:51.324 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:51.324 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:51.324 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 12:58:51.324 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:51.324 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:51.324 | INFO | Uploaded 36GiB to GPU in 2.15651s (16.6936GiB/s) | 20260505 12:58:51.324 | INFO | Downloaded 34GiB to GPU in 2.10381s (16.1612GiB/s) | 20260505 12:58:51.324 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 12:58:51.324 | INFO | Total mem usage: 15.3 GiB | 20260505 12:58:51.324 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 12:58:51.324 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -7.04%) | 20260505 12:58:51.324 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 12:58:51.324 | INFO | High Water Breakdown: | 20260505 12:58:51.324 | INFO | 61.1% : 8.74 GiB ( 91.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 12:58:51.324 | INFO | 38.8% : 5.56 GiB ( 83.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 12:58:51.324 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 12:58:51.324 | INFO | ------------------------------------------ | 20260505 12:58:51.324 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:58:51.324 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 12:58:51.324 | INFO | Total mem usage: 26.5 GiB | 20260505 12:58:51.324 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 12:58:51.324 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 12:58:51.324 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 12:58:51.324 | INFO | High Water Breakdown: | 20260505 12:58:51.324 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:58:51.324 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:58:51.324 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:58:51.324 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:58:51.324 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:58:51.324 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:58:51.324 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 12:58:51.324 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 12:58:51.324 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:58:51.324 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 12:58:51.324 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 12:58:51.324 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 12:58:51.324 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 12:58:51.324 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 12:58:51.324 | INFO | ------------------------------------------ | 20260505 12:58:51.324 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:58:51.324 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 12:58:51.324 | INFO | Total mem usage: 25.6 GiB | 20260505 12:58:51.324 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 12:58:51.324 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 12:58:51.324 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 12:58:51.324 | INFO | High Water Breakdown: | 20260505 12:58:51.324 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 12:58:51.324 | INFO | ------------------------------------------ | 20260505 12:58:51.324 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:58:51.324 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 12:58:51.324 | INFO | Total mem usage: 15.7 GiB | 20260505 12:58:51.324 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 12:58:51.324 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 12:58:51.324 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 12:58:51.324 | INFO | High Water Breakdown: | 20260505 12:58:51.324 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 12:58:51.324 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 12:58:51.324 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 12:58:51.324 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 12:58:51.324 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 12:58:51.324 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 12:58:51.324 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 12:58:51.324 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 12:58:51.324 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 12:58:51.324 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 12:58:51.324 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 12:58:51.324 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 12:58:51.324 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 12:58:51.325 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 12:58:51.325 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 12:58:51.325 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 12:58:51.325 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 12:58:51.325 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 12:58:51.325 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 12:58:51.325 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 12:58:51.325 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 12:58:51.325 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 12:58:51.325 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 12:58:51.325 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 12:58:51.325 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 12:58:51.325 | INFO | ------------------------------------------ | 20260505 12:58:51.325 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 12:58:51.325 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 12:58:51.325 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:58:51.325 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:58:51.325 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.70e+01, 1.36e+01 / 1.47e+00 | 20260505 12:58:51.325 | INFO | PerfReport: 0.29%, AggregateStats -- 9.32e-02 / 9.91e-02, 1.55e-02 / 4.73e-03 | 20260505 12:58:51.325 | INFO | PerfReport: 11.75%, Baseline -- 3.75e+00 / 4.51e+00, 5.88e-01 / 1.75e-01 | 20260505 12:58:51.325 | INFO | PerfReport: 3.30%, Histogram -- 1.05e+00 / 1.50e+00, 2.00e-01 / 9.69e-02 | 20260505 12:58:51.325 | INFO | PerfReport: 16.01%, DME -- 4.09e+01 / 4.59e+01, 4.52e+00 / 1.91e+00 | 20260505 12:58:51.325 | INFO | PerfReport: 56.29%, FrameLabeling -- 1.80e+01 / 2.00e+01, 1.42e+00 / 1.98e+00 | 20260505 12:58:51.325 | INFO | PerfReport: 7.47%, PulseAccumulating -- 2.39e+00 / 2.96e+00, 4.54e-01 / 1.70e-01 | 20260505 12:58:51.325 | INFO | PerfReport: 3.61%, Metrics -- 1.15e+00 / 1.47e+00, 3.24e-01 / 1.66e-01 | 20260505 12:58:51.325 | INFO | PerfReport: 1.27%, PulseSerializing -- 4.05e-01 / 4.57e-01, 7.38e-02 / 3.76e-02 | 20260505 12:58:51.325 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 12:58:51.325 | INFO | PerfReport: Gathered 1536 samples. | 20260505 12:58:51.325 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 12:58:51.325 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260505 12:58:51.325 | INFO | PerfReport: 22.48%, Upload -- 2.27e+01 / 0.00e+00, 8.04e+00 / 0.00e+00 | 20260505 12:58:51.325 | INFO | PerfReport: 21.98%, Download -- 2.22e+01 / 0.00e+00, 8.05e+00 / 0.00e+00 | 20260505 12:58:51.325 | INFO | PerfReport: 31.62%, Compute -- 3.20e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 12:58:51.325 | INFO | PerfReport: 2.29%, LockUpload -- 2.32e+00 / 0.00e+00, 8.02e+00 / 0.00e+00 | 20260505 12:58:51.325 | INFO | PerfReport: 1.39%, LockDownload -- 1.41e+00 / 0.00e+00, 3.00e+00 / 0.00e+00 | 20260505 12:58:51.325 | INFO | PerfReport: 20.23%, LockCompute -- 2.04e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260505 12:58:51.325 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.10e-02 / 0.00e+00 | 20260505 12:58:51.453 | INFO | Starting a round of Baz IO | 20260505 12:58:54.706 | INFO | Populating RTMetrics report | 20260505 12:58:54.707 | INFO | RTMetrics report populated | 20260505 12:58:54.707 | INFO | Opening tmp RTMetrics json file | 20260505 12:58:54.708 | INFO | Serializing RTMetrics to json | 20260505 12:58:54.730 | INFO | Serialization done, swapping tmp file now... | 20260505 12:58:54.740 | INFO | RTMetrics done | 20260505 12:58:54.820 | INFO | Finished a round of Baz IO: Wrote 2.32 GiB in 3.366 seconds | 20260505 12:58:54.820 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.32% / 0.01% / 76.68% | 20260505 12:58:54.820 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.784051% / 7.94729e-06% / 6.35783e-05% | 20260505 12:58:54.820 | INFO | Baz IO overall: | 20260505 12:58:54.820 | INFO | BytesWritten : 2018347450993 | 20260505 12:58:54.820 | INFO | NumEvents : 960491037578 | 20260505 12:58:56.454 | INFO | Analyzing chunk frames = [3391488,3392000) | 20260505 12:58:56.454 | INFO | MemInfo: { | 20260505 12:58:56.454 | INFO | "gpu" : " 17 GB", | 20260505 12:58:56.454 | INFO | "huge" : "", | 20260505 12:58:56.454 | INFO | "huge_shared" : " 27 GB", | 20260505 12:58:56.454 | INFO | "malloc" : " 14 GB", | 20260505 12:58:56.454 | INFO | "pinned" : " 29 GB" | 20260505 12:58:56.454 | INFO | } | 20260505 12:58:56.454 | INFO | Perf report for the last 5.13058 seconds: | 20260505 12:58:56.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:58:56.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:56.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:56.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:56.454 | INFO | ANALYSIS : 64.8%, 3.0, 100.3ms, 0 | 20260505 12:58:56.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:58:56.454 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260505 12:58:56.454 | INFO | PRE_HQ : 61.1%, 1.8, 57.9ms, 0 | 20260505 12:58:56.454 | INFO | BAZWRITER : 65.0%, 4.0, 133.9ms, 0 | 20260505 12:58:56.454 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 12:58:56.454 | INFO | Uploaded 36GiB to GPU in 2.21173s (16.2768GiB/s) | 20260505 12:58:56.454 | INFO | Downloaded 38GiB to GPU in 2.28788s (16.6093GiB/s) | 20260505 12:59:01.596 | INFO | Analyzing chunk frames = [3392000,3392512) | 20260505 12:59:01.596 | INFO | MemInfo: { | 20260505 12:59:01.596 | INFO | "gpu" : " 17 GB", | 20260505 12:59:01.596 | INFO | "huge" : "", | 20260505 12:59:01.596 | INFO | "huge_shared" : " 27 GB", | 20260505 12:59:01.596 | INFO | "malloc" : " 14 GB", | 20260505 12:59:01.596 | INFO | "pinned" : " 29 GB" | 20260505 12:59:01.596 | INFO | } | 20260505 12:59:01.596 | INFO | Perf report for the last 5.14182 seconds: | 20260505 12:59:01.596 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:59:01.596 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:01.596 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:01.596 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:01.596 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 12:59:01.596 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:01.596 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:01.596 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 12:59:01.596 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:01.596 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:01.596 | INFO | Uploaded 36GiB to GPU in 2.23503s (16.1071GiB/s) | 20260505 12:59:01.596 | INFO | Downloaded 34GiB to GPU in 2.08042s (16.3429GiB/s) | 20260505 12:59:06.742 | INFO | Analyzing chunk frames = [3392512,3393024) | 20260505 12:59:06.742 | INFO | MemInfo: { | 20260505 12:59:06.742 | INFO | "gpu" : " 17 GB", | 20260505 12:59:06.742 | INFO | "huge" : "", | 20260505 12:59:06.742 | INFO | "huge_shared" : " 27 GB", | 20260505 12:59:06.742 | INFO | "malloc" : " 14 GB", | 20260505 12:59:06.742 | INFO | "pinned" : " 29 GB" | 20260505 12:59:06.742 | INFO | } | 20260505 12:59:06.742 | INFO | Perf report for the last 5.14613 seconds: | 20260505 12:59:06.742 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:59:06.742 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:06.742 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:06.742 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:06.742 | INFO | ANALYSIS : 63.1%, 3.0, 98.3ms, 0 | 20260505 12:59:06.742 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:06.742 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:06.742 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:59:06.742 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:06.742 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:06.742 | INFO | Uploaded 36GiB to GPU in 2.21981s (16.2176GiB/s) | 20260505 12:59:06.742 | INFO | Downloaded 34GiB to GPU in 2.12816s (15.9762GiB/s) | 20260505 12:59:11.892 | INFO | Analyzing chunk frames = [3393024,3393536) | 20260505 12:59:11.892 | INFO | MemInfo: { | 20260505 12:59:11.892 | INFO | "gpu" : " 17 GB", | 20260505 12:59:11.892 | INFO | "huge" : "", | 20260505 12:59:11.892 | INFO | "huge_shared" : " 27 GB", | 20260505 12:59:11.892 | INFO | "malloc" : " 14 GB", | 20260505 12:59:11.892 | INFO | "pinned" : " 29 GB" | 20260505 12:59:11.892 | INFO | } | 20260505 12:59:11.892 | INFO | Perf report for the last 5.15023 seconds: | 20260505 12:59:11.892 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:59:11.892 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:11.892 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:11.893 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:11.893 | INFO | ANALYSIS : 62.9%, 3.0, 97.9ms, 0 | 20260505 12:59:11.893 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:11.893 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:11.893 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:59:11.893 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:11.893 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:11.893 | INFO | Uploaded 36GiB to GPU in 2.19849s (16.3749GiB/s) | 20260505 12:59:11.893 | INFO | Downloaded 34GiB to GPU in 2.1266s (15.988GiB/s) | 20260505 12:59:17.018 | INFO | Analyzing chunk frames = [3393536,3394048) | 20260505 12:59:17.018 | INFO | MemInfo: { | 20260505 12:59:17.018 | INFO | "gpu" : " 17 GB", | 20260505 12:59:17.018 | INFO | "huge" : "", | 20260505 12:59:17.018 | INFO | "huge_shared" : " 27 GB", | 20260505 12:59:17.018 | INFO | "malloc" : " 14 GB", | 20260505 12:59:17.018 | INFO | "pinned" : " 29 GB" | 20260505 12:59:17.018 | INFO | } | 20260505 12:59:17.018 | INFO | Perf report for the last 5.12578 seconds: | 20260505 12:59:17.018 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:59:17.018 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:17.018 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:17.018 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:17.018 | INFO | ANALYSIS : 62.9%, 3.0, 97.9ms, 0 | 20260505 12:59:17.018 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:17.018 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:17.018 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:59:17.018 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:17.018 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:17.018 | INFO | Uploaded 36GiB to GPU in 2.13464s (16.8646GiB/s) | 20260505 12:59:17.019 | INFO | Downloaded 34GiB to GPU in 2.12318s (16.0137GiB/s) | 20260505 12:59:22.154 | INFO | Analyzing chunk frames = [3394048,3394560) | 20260505 12:59:22.154 | INFO | MemInfo: { | 20260505 12:59:22.154 | INFO | "gpu" : " 17 GB", | 20260505 12:59:22.154 | INFO | "huge" : "", | 20260505 12:59:22.154 | INFO | "huge_shared" : " 27 GB", | 20260505 12:59:22.154 | INFO | "malloc" : " 14 GB", | 20260505 12:59:22.154 | INFO | "pinned" : " 29 GB" | 20260505 12:59:22.154 | INFO | } | 20260505 12:59:22.154 | INFO | Perf report for the last 5.13606 seconds: | 20260505 12:59:22.154 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:59:22.154 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:22.154 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:22.154 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:22.154 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 12:59:22.154 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:22.154 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:22.154 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 12:59:22.154 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:22.154 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:22.155 | INFO | Uploaded 36GiB to GPU in 2.1826s (16.4941GiB/s) | 20260505 12:59:22.155 | INFO | Downloaded 34GiB to GPU in 2.11522s (16.074GiB/s) | 20260505 12:59:27.299 | INFO | Analyzing chunk frames = [3394560,3395072) | 20260505 12:59:27.299 | INFO | MemInfo: { | 20260505 12:59:27.299 | INFO | "gpu" : " 17 GB", | 20260505 12:59:27.299 | INFO | "huge" : "", | 20260505 12:59:27.299 | INFO | "huge_shared" : " 27 GB", | 20260505 12:59:27.299 | INFO | "malloc" : " 14 GB", | 20260505 12:59:27.299 | INFO | "pinned" : " 29 GB" | 20260505 12:59:27.299 | INFO | } | 20260505 12:59:27.299 | INFO | Perf report for the last 5.14434 seconds: | 20260505 12:59:27.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:59:27.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:27.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:27.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:27.299 | INFO | ANALYSIS : 63.3%, 3.0, 99.0ms, 0 | 20260505 12:59:27.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:27.299 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:27.299 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 12:59:27.299 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:27.299 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:27.299 | INFO | Uploaded 36GiB to GPU in 2.16742s (16.6096GiB/s) | 20260505 12:59:27.299 | INFO | Downloaded 34GiB to GPU in 2.13281s (15.9414GiB/s) | 20260505 12:59:32.435 | INFO | Analyzing chunk frames = [3395072,3395584) | 20260505 12:59:32.435 | INFO | MemInfo: { | 20260505 12:59:32.435 | INFO | "gpu" : " 17 GB", | 20260505 12:59:32.435 | INFO | "huge" : "", | 20260505 12:59:32.435 | INFO | "huge_shared" : " 27 GB", | 20260505 12:59:32.435 | INFO | "malloc" : " 14 GB", | 20260505 12:59:32.435 | INFO | "pinned" : " 29 GB" | 20260505 12:59:32.435 | INFO | } | 20260505 12:59:32.435 | INFO | Perf report for the last 5.13679 seconds: | 20260505 12:59:32.435 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:59:32.435 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:32.435 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:32.435 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:32.436 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 12:59:32.436 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:32.436 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:32.436 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 12:59:32.436 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:32.436 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:32.436 | INFO | Uploaded 36GiB to GPU in 2.14027s (16.8203GiB/s) | 20260505 12:59:32.436 | INFO | Downloaded 34GiB to GPU in 2.10626s (16.1424GiB/s) | 20260505 12:59:32.570 | INFO | Starting a round of Baz IO | 20260505 12:59:35.815 | INFO | Populating RTMetrics report | 20260505 12:59:35.815 | INFO | RTMetrics report populated | 20260505 12:59:35.815 | INFO | Opening tmp RTMetrics json file | 20260505 12:59:35.816 | INFO | Serializing RTMetrics to json | 20260505 12:59:35.817 | INFO | Serialization done, swapping tmp file now... | 20260505 12:59:35.817 | INFO | RTMetrics done | 20260505 12:59:35.932 | INFO | Finished a round of Baz IO: Wrote 2.32 GiB in 3.362 seconds | 20260505 12:59:35.932 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.31% / 0.01% / 76.68% | 20260505 12:59:35.932 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.784826% / 7.94729e-06% / 6.75519e-05% | 20260505 12:59:35.932 | INFO | Baz IO overall: | 20260505 12:59:35.932 | INFO | BytesWritten : 2020838976389 | 20260505 12:59:35.932 | INFO | NumEvents : 961671775419 | 20260505 12:59:37.582 | INFO | Analyzing chunk frames = [3395584,3396096) | 20260505 12:59:37.582 | INFO | MemInfo: { | 20260505 12:59:37.582 | INFO | "gpu" : " 17 GB", | 20260505 12:59:37.582 | INFO | "huge" : "", | 20260505 12:59:37.582 | INFO | "huge_shared" : " 27 GB", | 20260505 12:59:37.582 | INFO | "malloc" : " 14 GB", | 20260505 12:59:37.582 | INFO | "pinned" : " 29 GB" | 20260505 12:59:37.582 | INFO | } | 20260505 12:59:37.582 | INFO | Perf report for the last 5.14672 seconds: | 20260505 12:59:37.582 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:59:37.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:37.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:37.582 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:37.582 | INFO | ANALYSIS : 64.6%, 2.9, 100.0ms, 0 | 20260505 12:59:37.582 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:37.582 | INFO | RT_METRICS : 45.1%, 1.0, 23.7ms, 0 | 20260505 12:59:37.582 | INFO | PRE_HQ : 60.9%, 1.8, 57.9ms, 0 | 20260505 12:59:37.582 | INFO | BAZWRITER : 64.6%, 4.0, 134.5ms, 0 | 20260505 12:59:37.582 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 12:59:37.582 | INFO | Uploaded 36GiB to GPU in 2.21259s (16.2705GiB/s) | 20260505 12:59:37.583 | INFO | Downloaded 38GiB to GPU in 2.27218s (16.724GiB/s) | 20260505 12:59:42.716 | INFO | Analyzing chunk frames = [3396096,3396608) | 20260505 12:59:42.716 | INFO | MemInfo: { | 20260505 12:59:42.716 | INFO | "gpu" : " 17 GB", | 20260505 12:59:42.716 | INFO | "huge" : "", | 20260505 12:59:42.716 | INFO | "huge_shared" : " 27 GB", | 20260505 12:59:42.716 | INFO | "malloc" : " 14 GB", | 20260505 12:59:42.716 | INFO | "pinned" : " 29 GB" | 20260505 12:59:42.716 | INFO | } | 20260505 12:59:42.716 | INFO | Perf report for the last 5.134 seconds: | 20260505 12:59:42.716 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:59:42.716 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:42.716 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:42.716 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:42.716 | INFO | ANALYSIS : 63.6%, 3.0, 99.1ms, 0 | 20260505 12:59:42.716 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:42.716 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:42.716 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:59:42.716 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:42.716 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:42.716 | INFO | Uploaded 36GiB to GPU in 2.19069s (16.4332GiB/s) | 20260505 12:59:42.717 | INFO | Downloaded 34GiB to GPU in 2.02869s (16.7595GiB/s) | 20260505 12:59:47.854 | INFO | Analyzing chunk frames = [3396608,3397120) | 20260505 12:59:47.854 | INFO | MemInfo: { | 20260505 12:59:47.854 | INFO | "gpu" : " 17 GB", | 20260505 12:59:47.854 | INFO | "huge" : "", | 20260505 12:59:47.854 | INFO | "huge_shared" : " 27 GB", | 20260505 12:59:47.854 | INFO | "malloc" : " 14 GB", | 20260505 12:59:47.854 | INFO | "pinned" : " 29 GB" | 20260505 12:59:47.854 | INFO | } | 20260505 12:59:47.854 | INFO | Perf report for the last 5.13761 seconds: | 20260505 12:59:47.854 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:59:47.854 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:47.854 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:47.854 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:47.854 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 12:59:47.854 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:47.854 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:47.854 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 12:59:47.854 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:47.854 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:47.854 | INFO | Uploaded 36GiB to GPU in 2.21546s (16.2495GiB/s) | 20260505 12:59:47.854 | INFO | Downloaded 34GiB to GPU in 2.12526s (15.998GiB/s) | 20260505 12:59:52.996 | INFO | Analyzing chunk frames = [3397120,3397632) | 20260505 12:59:52.996 | INFO | MemInfo: { | 20260505 12:59:52.996 | INFO | "gpu" : " 17 GB", | 20260505 12:59:52.996 | INFO | "huge" : "", | 20260505 12:59:52.997 | INFO | "huge_shared" : " 27 GB", | 20260505 12:59:52.997 | INFO | "malloc" : " 14 GB", | 20260505 12:59:52.997 | INFO | "pinned" : " 29 GB" | 20260505 12:59:52.997 | INFO | } | 20260505 12:59:52.997 | INFO | Perf report for the last 5.14278 seconds: | 20260505 12:59:52.997 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:59:52.997 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:52.997 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:52.997 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:52.997 | INFO | ANALYSIS : 63.0%, 3.0, 98.0ms, 0 | 20260505 12:59:52.997 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:52.997 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:52.997 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 12:59:52.997 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:52.997 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:52.997 | INFO | Uploaded 36GiB to GPU in 2.20143s (16.353GiB/s) | 20260505 12:59:52.997 | INFO | Downloaded 34GiB to GPU in 2.12903s (15.9697GiB/s) | 20260505 12:59:58.139 | INFO | Analyzing chunk frames = [3397632,3398144) | 20260505 12:59:58.139 | INFO | MemInfo: { | 20260505 12:59:58.139 | INFO | "gpu" : " 17 GB", | 20260505 12:59:58.139 | INFO | "huge" : "", | 20260505 12:59:58.139 | INFO | "huge_shared" : " 27 GB", | 20260505 12:59:58.139 | INFO | "malloc" : " 14 GB", | 20260505 12:59:58.139 | INFO | "pinned" : " 29 GB" | 20260505 12:59:58.139 | INFO | } | 20260505 12:59:58.139 | INFO | Perf report for the last 5.14271 seconds: | 20260505 12:59:58.139 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 12:59:58.139 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:58.139 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:58.139 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:58.139 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 12:59:58.139 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:58.139 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:58.139 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 12:59:58.139 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:58.139 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 12:59:58.140 | INFO | Uploaded 36GiB to GPU in 2.15168s (16.7311GiB/s) | 20260505 12:59:58.140 | INFO | Downloaded 34GiB to GPU in 2.12566s (15.9951GiB/s) | 20260505 13:00:03.267 | INFO | Analyzing chunk frames = [3398144,3398656) | 20260505 13:00:03.267 | INFO | MemInfo: { | 20260505 13:00:03.267 | INFO | "gpu" : " 17 GB", | 20260505 13:00:03.267 | INFO | "huge" : "", | 20260505 13:00:03.267 | INFO | "huge_shared" : " 27 GB", | 20260505 13:00:03.267 | INFO | "malloc" : " 14 GB", | 20260505 13:00:03.267 | INFO | "pinned" : " 29 GB" | 20260505 13:00:03.267 | INFO | } | 20260505 13:00:03.267 | INFO | Perf report for the last 5.12756 seconds: | 20260505 13:00:03.267 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:00:03.267 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:03.267 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:03.267 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:03.267 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 13:00:03.267 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:03.267 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:03.267 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 13:00:03.267 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:03.267 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:03.267 | INFO | Uploaded 36GiB to GPU in 2.17271s (16.5691GiB/s) | 20260505 13:00:03.267 | INFO | Downloaded 34GiB to GPU in 2.11204s (16.0982GiB/s) | 20260505 13:00:08.403 | INFO | Analyzing chunk frames = [3398656,3399168) | 20260505 13:00:08.403 | INFO | MemInfo: { | 20260505 13:00:08.403 | INFO | "gpu" : " 17 GB", | 20260505 13:00:08.403 | INFO | "huge" : "", | 20260505 13:00:08.403 | INFO | "huge_shared" : " 27 GB", | 20260505 13:00:08.403 | INFO | "malloc" : " 14 GB", | 20260505 13:00:08.403 | INFO | "pinned" : " 29 GB" | 20260505 13:00:08.403 | INFO | } | 20260505 13:00:08.403 | INFO | Perf report for the last 5.13651 seconds: | 20260505 13:00:08.403 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:00:08.403 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:08.403 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:08.403 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:08.403 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 13:00:08.404 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:08.404 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:08.404 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260505 13:00:08.404 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:08.404 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:08.404 | INFO | Uploaded 36GiB to GPU in 2.17465s (16.5544GiB/s) | 20260505 13:00:08.404 | INFO | Downloaded 34GiB to GPU in 2.15906s (15.7476GiB/s) | 20260505 13:00:13.552 | INFO | Analyzing chunk frames = [3399168,3399680) | 20260505 13:00:13.552 | INFO | MemInfo: { | 20260505 13:00:13.552 | INFO | "gpu" : " 17 GB", | 20260505 13:00:13.552 | INFO | "huge" : "", | 20260505 13:00:13.552 | INFO | "huge_shared" : " 27 GB", | 20260505 13:00:13.552 | INFO | "malloc" : " 14 GB", | 20260505 13:00:13.552 | INFO | "pinned" : " 29 GB" | 20260505 13:00:13.552 | INFO | } | 20260505 13:00:13.552 | INFO | Perf report for the last 5.1482 seconds: | 20260505 13:00:13.552 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:00:13.552 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:13.552 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:13.552 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:13.552 | INFO | ANALYSIS : 63.7%, 3.0, 99.6ms, 0 | 20260505 13:00:13.552 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:13.552 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:13.552 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 13:00:13.552 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:13.552 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:13.552 | INFO | Uploaded 36GiB to GPU in 2.21533s (16.2504GiB/s) | 20260505 13:00:13.552 | INFO | Downloaded 34GiB to GPU in 2.0844s (16.3116GiB/s) | 20260505 13:00:13.552 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 13:00:13.552 | INFO | Total mem usage: 15.3 GiB | 20260505 13:00:13.552 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 13:00:13.552 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.57%) | 20260505 13:00:13.552 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 13:00:13.552 | INFO | High Water Breakdown: | 20260505 13:00:13.552 | INFO | 60.9% : 8.68 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 13:00:13.552 | INFO | 38.9% : 5.55 GiB ( 83.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 13:00:13.552 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 13:00:13.552 | INFO | ------------------------------------------ | 20260505 13:00:13.552 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:00:13.552 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 13:00:13.552 | INFO | Total mem usage: 26.5 GiB | 20260505 13:00:13.553 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 13:00:13.553 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 13:00:13.553 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 13:00:13.553 | INFO | High Water Breakdown: | 20260505 13:00:13.553 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:00:13.553 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:00:13.553 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:00:13.553 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:00:13.553 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:00:13.553 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:00:13.553 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 13:00:13.553 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 13:00:13.553 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:00:13.553 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 13:00:13.553 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 13:00:13.553 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 13:00:13.553 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 13:00:13.553 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 13:00:13.553 | INFO | ------------------------------------------ | 20260505 13:00:13.553 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:00:13.553 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 13:00:13.553 | INFO | Total mem usage: 25.6 GiB | 20260505 13:00:13.553 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 13:00:13.553 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 13:00:13.553 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 13:00:13.553 | INFO | High Water Breakdown: | 20260505 13:00:13.553 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 13:00:13.553 | INFO | ------------------------------------------ | 20260505 13:00:13.553 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:00:13.553 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 13:00:13.553 | INFO | Total mem usage: 15.7 GiB | 20260505 13:00:13.553 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 13:00:13.553 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 13:00:13.553 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 13:00:13.553 | INFO | High Water Breakdown: | 20260505 13:00:13.553 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 13:00:13.553 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 13:00:13.553 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 13:00:13.553 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 13:00:13.553 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 13:00:13.553 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 13:00:13.553 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 13:00:13.553 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 13:00:13.553 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 13:00:13.553 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:00:13.553 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 13:00:13.553 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:00:13.553 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 13:00:13.553 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 13:00:13.553 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 13:00:13.553 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 13:00:13.553 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:00:13.553 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 13:00:13.553 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:00:13.553 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:00:13.553 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:00:13.553 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 13:00:13.553 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 13:00:13.553 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:00:13.553 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 13:00:13.553 | INFO | ------------------------------------------ | 20260505 13:00:13.554 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:00:13.554 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 13:00:13.554 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:00:13.554 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:00:13.554 | INFO | PerfReport: Full Execution -- 3.20e+01 / 7.69e+01, 1.36e+01 / 1.47e+00 | 20260505 13:00:13.554 | INFO | PerfReport: 0.29%, AggregateStats -- 9.30e-02 / 1.00e-01, 1.06e-02 / 5.07e-03 | 20260505 13:00:13.554 | INFO | PerfReport: 11.77%, Baseline -- 3.76e+00 / 4.49e+00, 5.92e-01 / 2.32e-01 | 20260505 13:00:13.554 | INFO | PerfReport: 3.32%, Histogram -- 1.06e+00 / 1.52e+00, 2.04e-01 / 8.40e-02 | 20260505 13:00:13.554 | INFO | PerfReport: 15.86%, DME -- 4.06e+01 / 4.51e+01, 4.34e+00 / 2.23e+00 | 20260505 13:00:13.554 | INFO | PerfReport: 56.43%, FrameLabeling -- 1.80e+01 / 2.10e+01, 1.49e+00 / 2.36e+00 | 20260505 13:00:13.554 | INFO | PerfReport: 7.43%, PulseAccumulating -- 2.37e+00 / 2.82e+00, 4.34e-01 / 2.47e-01 | 20260505 13:00:13.554 | INFO | PerfReport: 3.63%, Metrics -- 1.16e+00 / 1.47e+00, 3.30e-01 / 2.11e-01 | 20260505 13:00:13.554 | INFO | PerfReport: 1.27%, PulseSerializing -- 4.06e-01 / 4.60e-01, 7.42e-02 / 4.05e-02 | 20260505 13:00:13.554 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 13:00:13.554 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:00:13.554 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:00:13.554 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260505 13:00:13.554 | INFO | PerfReport: 22.58%, Upload -- 2.28e+01 / 0.00e+00, 8.14e+00 / 0.00e+00 | 20260505 13:00:13.554 | INFO | PerfReport: 22.01%, Download -- 2.22e+01 / 0.00e+00, 8.12e+00 / 0.00e+00 | 20260505 13:00:13.554 | INFO | PerfReport: 31.65%, Compute -- 3.20e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 13:00:13.554 | INFO | PerfReport: 2.19%, LockUpload -- 2.21e+00 / 0.00e+00, 7.70e+00 / 0.00e+00 | 20260505 13:00:13.554 | INFO | PerfReport: 1.32%, LockDownload -- 1.33e+00 / 0.00e+00, 2.83e+00 / 0.00e+00 | 20260505 13:00:13.554 | INFO | PerfReport: 20.24%, LockCompute -- 2.04e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260505 13:00:13.554 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.89e-02 / 0.00e+00 | 20260505 13:00:13.692 | INFO | Starting a round of Baz IO | 20260505 13:00:16.917 | INFO | Populating RTMetrics report | 20260505 13:00:16.917 | INFO | RTMetrics report populated | 20260505 13:00:16.917 | INFO | Opening tmp RTMetrics json file | 20260505 13:00:16.924 | INFO | Serializing RTMetrics to json | 20260505 13:00:16.941 | INFO | Serialization done, swapping tmp file now... | 20260505 13:00:16.941 | INFO | RTMetrics done | 20260505 13:00:17.034 | INFO | Finished a round of Baz IO: Wrote 2.319 GiB in 3.342 seconds | 20260505 13:00:17.034 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.31% / 0.01% / 76.69% | 20260505 13:00:17.034 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.780829% / 0% / 8.34465e-05% | 20260505 13:00:17.034 | INFO | Baz IO overall: | 20260505 13:00:17.034 | INFO | BytesWritten : 2023328609667 | 20260505 13:00:17.034 | INFO | NumEvents : 962850789915 | 20260505 13:00:18.697 | INFO | Analyzing chunk frames = [3399680,3400192) | 20260505 13:00:18.697 | INFO | MemInfo: { | 20260505 13:00:18.697 | INFO | "gpu" : " 17 GB", | 20260505 13:00:18.697 | INFO | "huge" : "", | 20260505 13:00:18.697 | INFO | "huge_shared" : " 27 GB", | 20260505 13:00:18.697 | INFO | "malloc" : " 14 GB", | 20260505 13:00:18.697 | INFO | "pinned" : " 29 GB" | 20260505 13:00:18.697 | INFO | } | 20260505 13:00:18.697 | INFO | Perf report for the last 5.14578 seconds: | 20260505 13:00:18.697 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:00:18.697 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:18.697 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:18.697 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:18.698 | INFO | ANALYSIS : 64.3%, 2.9, 99.5ms, 0 | 20260505 13:00:18.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:18.698 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260505 13:00:18.698 | INFO | PRE_HQ : 60.9%, 1.8, 57.7ms, 0 | 20260505 13:00:18.698 | INFO | BAZWRITER : 64.3%, 4.1, 135.0ms, 0 | 20260505 13:00:18.698 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 13:00:18.698 | INFO | Uploaded 36GiB to GPU in 2.1811s (16.5055GiB/s) | 20260505 13:00:18.698 | INFO | Downloaded 38GiB to GPU in 2.28091s (16.66GiB/s) | 20260505 13:00:23.837 | INFO | Analyzing chunk frames = [3400192,3400704) | 20260505 13:00:23.837 | INFO | MemInfo: { | 20260505 13:00:23.837 | INFO | "gpu" : " 17 GB", | 20260505 13:00:23.837 | INFO | "huge" : "", | 20260505 13:00:23.837 | INFO | "huge_shared" : " 27 GB", | 20260505 13:00:23.837 | INFO | "malloc" : " 14 GB", | 20260505 13:00:23.837 | INFO | "pinned" : " 29 GB" | 20260505 13:00:23.837 | INFO | } | 20260505 13:00:23.837 | INFO | Perf report for the last 5.13993 seconds: | 20260505 13:00:23.837 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:00:23.837 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:23.837 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:23.837 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:23.837 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 13:00:23.837 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:23.837 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:23.837 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:00:23.837 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:23.837 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:23.838 | INFO | Uploaded 36GiB to GPU in 2.19617s (16.3921GiB/s) | 20260505 13:00:23.838 | INFO | Downloaded 34GiB to GPU in 2.03036s (16.7458GiB/s) | 20260505 13:00:28.967 | INFO | Analyzing chunk frames = [3400704,3401216) | 20260505 13:00:28.967 | INFO | MemInfo: { | 20260505 13:00:28.967 | INFO | "gpu" : " 17 GB", | 20260505 13:00:28.967 | INFO | "huge" : "", | 20260505 13:00:28.967 | INFO | "huge_shared" : " 27 GB", | 20260505 13:00:28.967 | INFO | "malloc" : " 14 GB", | 20260505 13:00:28.967 | INFO | "pinned" : " 29 GB" | 20260505 13:00:28.967 | INFO | } | 20260505 13:00:28.967 | INFO | Perf report for the last 5.12993 seconds: | 20260505 13:00:28.967 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:00:28.967 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:28.967 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:28.967 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:28.967 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260505 13:00:28.967 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:28.967 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:28.967 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 13:00:28.967 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:28.967 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:28.967 | INFO | Uploaded 36GiB to GPU in 2.2025s (16.3451GiB/s) | 20260505 13:00:28.968 | INFO | Downloaded 34GiB to GPU in 2.10992s (16.1143GiB/s) | 20260505 13:00:34.110 | INFO | Analyzing chunk frames = [3401216,3401728) | 20260505 13:00:34.110 | INFO | MemInfo: { | 20260505 13:00:34.110 | INFO | "gpu" : " 17 GB", | 20260505 13:00:34.111 | INFO | "huge" : "", | 20260505 13:00:34.111 | INFO | "huge_shared" : " 27 GB", | 20260505 13:00:34.111 | INFO | "malloc" : " 14 GB", | 20260505 13:00:34.111 | INFO | "pinned" : " 29 GB" | 20260505 13:00:34.111 | INFO | } | 20260505 13:00:34.111 | INFO | Perf report for the last 5.14332 seconds: | 20260505 13:00:34.111 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:00:34.111 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:34.111 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:34.111 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:34.111 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 13:00:34.111 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:34.111 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:34.111 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 13:00:34.111 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:34.111 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:34.111 | INFO | Uploaded 36GiB to GPU in 2.21661s (16.241GiB/s) | 20260505 13:00:34.111 | INFO | Downloaded 34GiB to GPU in 2.13101s (15.9549GiB/s) | 20260505 13:00:39.251 | INFO | Analyzing chunk frames = [3401728,3402240) | 20260505 13:00:39.251 | INFO | MemInfo: { | 20260505 13:00:39.251 | INFO | "gpu" : " 17 GB", | 20260505 13:00:39.251 | INFO | "huge" : "", | 20260505 13:00:39.251 | INFO | "huge_shared" : " 27 GB", | 20260505 13:00:39.251 | INFO | "malloc" : " 14 GB", | 20260505 13:00:39.251 | INFO | "pinned" : " 29 GB" | 20260505 13:00:39.251 | INFO | } | 20260505 13:00:39.251 | INFO | Perf report for the last 5.14025 seconds: | 20260505 13:00:39.251 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:00:39.251 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:39.251 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:39.251 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:39.251 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 13:00:39.251 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:39.251 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:39.251 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 13:00:39.251 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:39.251 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:39.251 | INFO | Uploaded 36GiB to GPU in 2.15261s (16.7239GiB/s) | 20260505 13:00:39.251 | INFO | Downloaded 34GiB to GPU in 2.12355s (16.0109GiB/s) | 20260505 13:00:44.388 | INFO | Analyzing chunk frames = [3402240,3402752) | 20260505 13:00:44.388 | INFO | MemInfo: { | 20260505 13:00:44.388 | INFO | "gpu" : " 17 GB", | 20260505 13:00:44.388 | INFO | "huge" : "", | 20260505 13:00:44.388 | INFO | "huge_shared" : " 27 GB", | 20260505 13:00:44.388 | INFO | "malloc" : " 14 GB", | 20260505 13:00:44.388 | INFO | "pinned" : " 29 GB" | 20260505 13:00:44.388 | INFO | } | 20260505 13:00:44.388 | INFO | Perf report for the last 5.13706 seconds: | 20260505 13:00:44.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:00:44.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:44.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:44.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:44.388 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 13:00:44.388 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:44.388 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:44.388 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 13:00:44.388 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:44.388 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:44.388 | INFO | Uploaded 36GiB to GPU in 2.1841s (16.4828GiB/s) | 20260505 13:00:44.388 | INFO | Downloaded 34GiB to GPU in 2.11346s (16.0874GiB/s) | 20260505 13:00:49.529 | INFO | Analyzing chunk frames = [3402752,3403264) | 20260505 13:00:49.529 | INFO | MemInfo: { | 20260505 13:00:49.529 | INFO | "gpu" : " 17 GB", | 20260505 13:00:49.529 | INFO | "huge" : "", | 20260505 13:00:49.529 | INFO | "huge_shared" : " 27 GB", | 20260505 13:00:49.529 | INFO | "malloc" : " 14 GB", | 20260505 13:00:49.529 | INFO | "pinned" : " 29 GB" | 20260505 13:00:49.529 | INFO | } | 20260505 13:00:49.529 | INFO | Perf report for the last 5.141 seconds: | 20260505 13:00:49.529 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:00:49.529 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:49.529 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:49.529 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:49.529 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 13:00:49.529 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:49.529 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:49.529 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 13:00:49.529 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:49.529 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:49.529 | INFO | Uploaded 36GiB to GPU in 2.17294s (16.5674GiB/s) | 20260505 13:00:49.529 | INFO | Downloaded 34GiB to GPU in 2.14644s (15.8402GiB/s) | 20260505 13:00:54.664 | INFO | Analyzing chunk frames = [3403264,3403776) | 20260505 13:00:54.664 | INFO | MemInfo: { | 20260505 13:00:54.664 | INFO | "gpu" : " 17 GB", | 20260505 13:00:54.664 | INFO | "huge" : "", | 20260505 13:00:54.664 | INFO | "huge_shared" : " 27 GB", | 20260505 13:00:54.664 | INFO | "malloc" : " 14 GB", | 20260505 13:00:54.664 | INFO | "pinned" : " 29 GB" | 20260505 13:00:54.664 | INFO | } | 20260505 13:00:54.664 | INFO | Perf report for the last 5.13519 seconds: | 20260505 13:00:54.664 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:00:54.664 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:54.664 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:54.664 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:54.664 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 13:00:54.664 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:54.664 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:54.664 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 13:00:54.664 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:54.664 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:54.664 | INFO | Uploaded 36GiB to GPU in 2.15415s (16.7119GiB/s) | 20260505 13:00:54.664 | INFO | Downloaded 34GiB to GPU in 2.10874s (16.1234GiB/s) | 20260505 13:00:54.798 | INFO | Starting a round of Baz IO | 20260505 13:00:58.048 | INFO | Populating RTMetrics report | 20260505 13:00:58.049 | INFO | RTMetrics report populated | 20260505 13:00:58.049 | INFO | Opening tmp RTMetrics json file | 20260505 13:00:58.050 | INFO | Serializing RTMetrics to json | 20260505 13:00:58.051 | INFO | Serialization done, swapping tmp file now... | 20260505 13:00:58.051 | INFO | RTMetrics done | 20260505 13:00:58.174 | INFO | Finished a round of Baz IO: Wrote 2.318 GiB in 3.376 seconds | 20260505 13:00:58.174 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.30% / 0.00% / 76.69% | 20260505 13:00:58.175 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.778925% / 3.97364e-06% / 4.76837e-05% | 20260505 13:00:58.175 | INFO | Baz IO overall: | 20260505 13:00:58.175 | INFO | BytesWritten : 2025817556555 | 20260505 13:00:58.175 | INFO | NumEvents : 964029428826 | 20260505 13:00:59.811 | INFO | Analyzing chunk frames = [3403776,3404288) | 20260505 13:00:59.811 | INFO | MemInfo: { | 20260505 13:00:59.811 | INFO | "gpu" : " 17 GB", | 20260505 13:00:59.811 | INFO | "huge" : "", | 20260505 13:00:59.811 | INFO | "huge_shared" : " 27 GB", | 20260505 13:00:59.811 | INFO | "malloc" : " 14 GB", | 20260505 13:00:59.811 | INFO | "pinned" : " 29 GB" | 20260505 13:00:59.811 | INFO | } | 20260505 13:00:59.811 | INFO | Perf report for the last 5.14663 seconds: | 20260505 13:00:59.811 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:00:59.811 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:59.811 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:59.811 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:59.811 | INFO | ANALYSIS : 64.6%, 2.9, 99.9ms, 0 | 20260505 13:00:59.811 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:00:59.811 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260505 13:00:59.811 | INFO | PRE_HQ : 60.9%, 1.8, 58.1ms, 0 | 20260505 13:00:59.811 | INFO | BAZWRITER : 64.9%, 4.1, 135.9ms, 0 | 20260505 13:00:59.811 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 13:00:59.811 | INFO | Uploaded 36GiB to GPU in 2.19297s (16.4161GiB/s) | 20260505 13:00:59.811 | INFO | Downloaded 38GiB to GPU in 2.28609s (16.6222GiB/s) | 20260505 13:01:04.945 | INFO | Analyzing chunk frames = [3404288,3404800) | 20260505 13:01:04.945 | INFO | MemInfo: { | 20260505 13:01:04.945 | INFO | "gpu" : " 17 GB", | 20260505 13:01:04.945 | INFO | "huge" : "", | 20260505 13:01:04.945 | INFO | "huge_shared" : " 27 GB", | 20260505 13:01:04.945 | INFO | "malloc" : " 14 GB", | 20260505 13:01:04.945 | INFO | "pinned" : " 29 GB" | 20260505 13:01:04.945 | INFO | } | 20260505 13:01:04.945 | INFO | Perf report for the last 5.1346 seconds: | 20260505 13:01:04.945 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:01:04.945 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:04.945 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:04.945 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:04.945 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 13:01:04.946 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:04.946 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:04.946 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 13:01:04.946 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:04.946 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:04.946 | INFO | Uploaded 36GiB to GPU in 2.20156s (16.3521GiB/s) | 20260505 13:01:04.946 | INFO | Downloaded 34GiB to GPU in 2.04173s (16.6526GiB/s) | 20260505 13:01:10.086 | INFO | Analyzing chunk frames = [3404800,3405312) | 20260505 13:01:10.086 | INFO | MemInfo: { | 20260505 13:01:10.086 | INFO | "gpu" : " 17 GB", | 20260505 13:01:10.086 | INFO | "huge" : "", | 20260505 13:01:10.086 | INFO | "huge_shared" : " 27 GB", | 20260505 13:01:10.086 | INFO | "malloc" : " 14 GB", | 20260505 13:01:10.086 | INFO | "pinned" : " 29 GB" | 20260505 13:01:10.086 | INFO | } | 20260505 13:01:10.086 | INFO | Perf report for the last 5.14104 seconds: | 20260505 13:01:10.087 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:01:10.087 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:10.087 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:10.087 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:10.087 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 13:01:10.087 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:10.087 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:10.087 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:01:10.087 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:10.087 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:10.087 | INFO | Uploaded 36GiB to GPU in 2.18586s (16.4695GiB/s) | 20260505 13:01:10.087 | INFO | Downloaded 34GiB to GPU in 2.12765s (15.9801GiB/s) | 20260505 13:01:15.219 | INFO | Analyzing chunk frames = [3405312,3405824) | 20260505 13:01:15.219 | INFO | MemInfo: { | 20260505 13:01:15.219 | INFO | "gpu" : " 17 GB", | 20260505 13:01:15.219 | INFO | "huge" : "", | 20260505 13:01:15.219 | INFO | "huge_shared" : " 27 GB", | 20260505 13:01:15.219 | INFO | "malloc" : " 14 GB", | 20260505 13:01:15.219 | INFO | "pinned" : " 29 GB" | 20260505 13:01:15.219 | INFO | } | 20260505 13:01:15.219 | INFO | Perf report for the last 5.13231 seconds: | 20260505 13:01:15.219 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:01:15.219 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:15.219 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:15.219 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:15.219 | INFO | ANALYSIS : 63.2%, 3.0, 98.1ms, 0 | 20260505 13:01:15.219 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:15.219 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:15.219 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:01:15.219 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:15.219 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:15.219 | INFO | Uploaded 36GiB to GPU in 2.20704s (16.3114GiB/s) | 20260505 13:01:15.219 | INFO | Downloaded 34GiB to GPU in 2.13172s (15.9496GiB/s) | 20260505 13:01:20.357 | INFO | Analyzing chunk frames = [3405824,3406336) | 20260505 13:01:20.357 | INFO | MemInfo: { | 20260505 13:01:20.357 | INFO | "gpu" : " 17 GB", | 20260505 13:01:20.357 | INFO | "huge" : "", | 20260505 13:01:20.357 | INFO | "huge_shared" : " 27 GB", | 20260505 13:01:20.357 | INFO | "malloc" : " 14 GB", | 20260505 13:01:20.357 | INFO | "pinned" : " 29 GB" | 20260505 13:01:20.357 | INFO | } | 20260505 13:01:20.357 | INFO | Perf report for the last 5.13824 seconds: | 20260505 13:01:20.357 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:01:20.357 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:20.357 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:20.357 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:20.357 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 13:01:20.357 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:20.357 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:20.357 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 13:01:20.357 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:20.357 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:20.357 | INFO | Uploaded 36GiB to GPU in 2.17398s (16.5595GiB/s) | 20260505 13:01:20.357 | INFO | Downloaded 34GiB to GPU in 2.12984s (15.9637GiB/s) | 20260505 13:01:25.504 | INFO | Analyzing chunk frames = [3406336,3406848) | 20260505 13:01:25.504 | INFO | MemInfo: { | 20260505 13:01:25.504 | INFO | "gpu" : " 17 GB", | 20260505 13:01:25.504 | INFO | "huge" : "", | 20260505 13:01:25.504 | INFO | "huge_shared" : " 27 GB", | 20260505 13:01:25.504 | INFO | "malloc" : " 14 GB", | 20260505 13:01:25.504 | INFO | "pinned" : " 29 GB" | 20260505 13:01:25.504 | INFO | } | 20260505 13:01:25.504 | INFO | Perf report for the last 5.14689 seconds: | 20260505 13:01:25.504 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:01:25.504 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:25.504 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:25.504 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:25.504 | INFO | ANALYSIS : 63.2%, 3.0, 98.8ms, 0 | 20260505 13:01:25.504 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:25.504 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:25.504 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 13:01:25.504 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:25.504 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:25.504 | INFO | Uploaded 36GiB to GPU in 2.15366s (16.7157GiB/s) | 20260505 13:01:25.504 | INFO | Downloaded 34GiB to GPU in 2.10971s (16.1159GiB/s) | 20260505 13:01:30.652 | INFO | Analyzing chunk frames = [3406848,3407360) | 20260505 13:01:30.652 | INFO | MemInfo: { | 20260505 13:01:30.652 | INFO | "gpu" : " 17 GB", | 20260505 13:01:30.652 | INFO | "huge" : "", | 20260505 13:01:30.652 | INFO | "huge_shared" : " 27 GB", | 20260505 13:01:30.652 | INFO | "malloc" : " 14 GB", | 20260505 13:01:30.652 | INFO | "pinned" : " 29 GB" | 20260505 13:01:30.652 | INFO | } | 20260505 13:01:30.652 | INFO | Perf report for the last 5.14801 seconds: | 20260505 13:01:30.652 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:01:30.652 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:30.652 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:30.652 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:30.652 | INFO | ANALYSIS : 63.4%, 3.0, 99.2ms, 0 | 20260505 13:01:30.652 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:30.652 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:30.652 | INFO | PRE_HQ : 29.3%, 1.0, 15.3ms, 0 | 20260505 13:01:30.652 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:30.652 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:30.652 | INFO | Uploaded 36GiB to GPU in 2.13095s (16.8939GiB/s) | 20260505 13:01:30.652 | INFO | Downloaded 34GiB to GPU in 2.18174s (15.5839GiB/s) | 20260505 13:01:35.776 | INFO | Analyzing chunk frames = [3407360,3407872) | 20260505 13:01:35.776 | INFO | MemInfo: { | 20260505 13:01:35.776 | INFO | "gpu" : " 17 GB", | 20260505 13:01:35.776 | INFO | "huge" : "", | 20260505 13:01:35.776 | INFO | "huge_shared" : " 27 GB", | 20260505 13:01:35.776 | INFO | "malloc" : " 14 GB", | 20260505 13:01:35.776 | INFO | "pinned" : " 29 GB" | 20260505 13:01:35.776 | INFO | } | 20260505 13:01:35.777 | INFO | Perf report for the last 5.12452 seconds: | 20260505 13:01:35.777 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:01:35.777 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:35.777 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:35.777 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:35.777 | INFO | ANALYSIS : 63.8%, 3.0, 99.3ms, 0 | 20260505 13:01:35.777 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:35.777 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:35.777 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 13:01:35.777 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:35.777 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:35.777 | INFO | Uploaded 36GiB to GPU in 2.16933s (16.595GiB/s) | 20260505 13:01:35.777 | INFO | Downloaded 34GiB to GPU in 2.10555s (16.1478GiB/s) | 20260505 13:01:35.777 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 13:01:35.777 | INFO | Total mem usage: 15.3 GiB | 20260505 13:01:35.777 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 13:01:35.777 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -6.98%) | 20260505 13:01:35.777 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 13:01:35.777 | INFO | High Water Breakdown: | 20260505 13:01:35.777 | INFO | 61.1% : 8.75 GiB ( 92% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 13:01:35.777 | INFO | 38.8% : 5.56 GiB ( 83.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 13:01:35.777 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 13:01:35.777 | INFO | ------------------------------------------ | 20260505 13:01:35.777 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:01:35.777 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 13:01:35.777 | INFO | Total mem usage: 26.5 GiB | 20260505 13:01:35.777 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 13:01:35.777 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 13:01:35.777 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 13:01:35.777 | INFO | High Water Breakdown: | 20260505 13:01:35.777 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:01:35.777 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:01:35.777 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:01:35.777 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:01:35.777 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:01:35.777 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:01:35.777 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 13:01:35.777 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 13:01:35.777 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:01:35.777 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 13:01:35.777 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 13:01:35.777 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 13:01:35.777 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 13:01:35.777 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 13:01:35.777 | INFO | ------------------------------------------ | 20260505 13:01:35.777 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:01:35.778 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 13:01:35.778 | INFO | Total mem usage: 25.6 GiB | 20260505 13:01:35.778 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 13:01:35.778 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 13:01:35.778 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 13:01:35.778 | INFO | High Water Breakdown: | 20260505 13:01:35.778 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 13:01:35.778 | INFO | ------------------------------------------ | 20260505 13:01:35.778 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:01:35.778 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 13:01:35.778 | INFO | Total mem usage: 15.7 GiB | 20260505 13:01:35.778 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 13:01:35.778 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 13:01:35.778 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 13:01:35.778 | INFO | High Water Breakdown: | 20260505 13:01:35.778 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 13:01:35.778 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 13:01:35.778 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 13:01:35.778 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 13:01:35.778 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 13:01:35.778 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 13:01:35.778 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 13:01:35.778 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 13:01:35.778 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 13:01:35.778 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:01:35.778 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 13:01:35.778 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:01:35.778 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 13:01:35.778 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 13:01:35.778 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 13:01:35.778 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 13:01:35.778 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:01:35.778 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 13:01:35.778 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:01:35.778 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:01:35.778 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:01:35.778 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 13:01:35.778 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 13:01:35.778 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:01:35.778 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 13:01:35.778 | INFO | ------------------------------------------ | 20260505 13:01:35.778 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:01:35.778 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 13:01:35.778 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:01:35.778 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:01:35.778 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.70e+01, 1.36e+01 / 1.62e+00 | 20260505 13:01:35.778 | INFO | PerfReport: 0.29%, AggregateStats -- 9.27e-02 / 9.94e-02, 1.04e-02 / 3.66e-03 | 20260505 13:01:35.778 | INFO | PerfReport: 11.78%, Baseline -- 3.76e+00 / 4.50e+00, 5.96e-01 / 1.97e-01 | 20260505 13:01:35.778 | INFO | PerfReport: 3.31%, Histogram -- 1.06e+00 / 1.50e+00, 2.01e-01 / 9.20e-02 | 20260505 13:01:35.778 | INFO | PerfReport: 16.03%, DME -- 4.10e+01 / 4.62e+01, 4.75e+00 / 1.98e+00 | 20260505 13:01:35.778 | INFO | PerfReport: 56.23%, FrameLabeling -- 1.80e+01 / 1.99e+01, 1.35e+00 / 1.81e+00 | 20260505 13:01:35.778 | INFO | PerfReport: 7.48%, PulseAccumulating -- 2.39e+00 / 2.93e+00, 4.55e-01 / 2.13e-01 | 20260505 13:01:35.778 | INFO | PerfReport: 3.61%, Metrics -- 1.15e+00 / 1.46e+00, 3.22e-01 / 1.37e-01 | 20260505 13:01:35.778 | INFO | PerfReport: 1.27%, PulseSerializing -- 4.05e-01 / 4.55e-01, 7.29e-02 / 2.86e-02 | 20260505 13:01:35.778 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 13:01:35.778 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:01:35.778 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:01:35.778 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260505 13:01:35.778 | INFO | PerfReport: 22.49%, Upload -- 2.27e+01 / 0.00e+00, 7.95e+00 / 0.00e+00 | 20260505 13:01:35.778 | INFO | PerfReport: 22.03%, Download -- 2.22e+01 / 0.00e+00, 8.10e+00 / 0.00e+00 | 20260505 13:01:35.778 | INFO | PerfReport: 31.64%, Compute -- 3.19e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 13:01:35.778 | INFO | PerfReport: 2.19%, LockUpload -- 2.21e+00 / 0.00e+00, 7.97e+00 / 0.00e+00 | 20260505 13:01:35.778 | INFO | PerfReport: 1.47%, LockDownload -- 1.48e+00 / 0.00e+00, 3.07e+00 / 0.00e+00 | 20260505 13:01:35.778 | INFO | PerfReport: 20.17%, LockCompute -- 2.04e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260505 13:01:35.778 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.04e-02 / 0.00e+00 | 20260505 13:01:35.910 | INFO | Starting a round of Baz IO | 20260505 13:01:39.156 | INFO | Populating RTMetrics report | 20260505 13:01:39.156 | INFO | RTMetrics report populated | 20260505 13:01:39.156 | INFO | Opening tmp RTMetrics json file | 20260505 13:01:39.157 | INFO | Serializing RTMetrics to json | 20260505 13:01:39.158 | INFO | Serialization done, swapping tmp file now... | 20260505 13:01:39.159 | INFO | RTMetrics done | 20260505 13:01:39.278 | INFO | Finished a round of Baz IO: Wrote 2.321 GiB in 3.367 seconds | 20260505 13:01:39.278 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.30% / 0.01% / 76.70% | 20260505 13:01:39.278 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.778449% / 0% / 9.53674e-05% | 20260505 13:01:39.278 | INFO | Baz IO overall: | 20260505 13:01:39.278 | INFO | BytesWritten : 2028309654681 | 20260505 13:01:39.278 | INFO | NumEvents : 965210636018 | 20260505 13:01:40.913 | INFO | Analyzing chunk frames = [3407872,3408384) | 20260505 13:01:40.913 | INFO | MemInfo: { | 20260505 13:01:40.913 | INFO | "gpu" : " 17 GB", | 20260505 13:01:40.913 | INFO | "huge" : "", | 20260505 13:01:40.913 | INFO | "huge_shared" : " 27 GB", | 20260505 13:01:40.913 | INFO | "malloc" : " 14 GB", | 20260505 13:01:40.913 | INFO | "pinned" : " 29 GB" | 20260505 13:01:40.913 | INFO | } | 20260505 13:01:40.913 | INFO | Perf report for the last 5.13675 seconds: | 20260505 13:01:40.913 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:01:40.913 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:40.913 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:40.913 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:40.913 | INFO | ANALYSIS : 64.7%, 2.9, 99.9ms, 0 | 20260505 13:01:40.914 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:40.914 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260505 13:01:40.914 | INFO | PRE_HQ : 60.9%, 1.8, 57.8ms, 0 | 20260505 13:01:40.914 | INFO | BAZWRITER : 64.8%, 4.0, 134.3ms, 0 | 20260505 13:01:40.914 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 13:01:40.914 | INFO | Uploaded 36GiB to GPU in 2.19256s (16.4192GiB/s) | 20260505 13:01:40.914 | INFO | Downloaded 38GiB to GPU in 2.28346s (16.6414GiB/s) | 20260505 13:01:46.063 | INFO | Analyzing chunk frames = [3408384,3408896) | 20260505 13:01:46.063 | INFO | MemInfo: { | 20260505 13:01:46.063 | INFO | "gpu" : " 17 GB", | 20260505 13:01:46.063 | INFO | "huge" : "", | 20260505 13:01:46.063 | INFO | "huge_shared" : " 27 GB", | 20260505 13:01:46.063 | INFO | "malloc" : " 14 GB", | 20260505 13:01:46.063 | INFO | "pinned" : " 29 GB" | 20260505 13:01:46.063 | INFO | } | 20260505 13:01:46.063 | INFO | Perf report for the last 5.14939 seconds: | 20260505 13:01:46.063 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:01:46.063 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:46.063 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:46.063 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:46.063 | INFO | ANALYSIS : 63.9%, 3.0, 100.0ms, 0 | 20260505 13:01:46.063 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:46.063 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:46.063 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 13:01:46.063 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:46.063 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:46.063 | INFO | Uploaded 36GiB to GPU in 2.23346s (16.1185GiB/s) | 20260505 13:01:46.063 | INFO | Downloaded 34GiB to GPU in 2.07615s (16.3765GiB/s) | 20260505 13:01:51.202 | INFO | Analyzing chunk frames = [3408896,3409408) | 20260505 13:01:51.202 | INFO | MemInfo: { | 20260505 13:01:51.202 | INFO | "gpu" : " 17 GB", | 20260505 13:01:51.202 | INFO | "huge" : "", | 20260505 13:01:51.202 | INFO | "huge_shared" : " 27 GB", | 20260505 13:01:51.202 | INFO | "malloc" : " 14 GB", | 20260505 13:01:51.202 | INFO | "pinned" : " 29 GB" | 20260505 13:01:51.202 | INFO | } | 20260505 13:01:51.202 | INFO | Perf report for the last 5.139 seconds: | 20260505 13:01:51.202 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:01:51.202 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:51.202 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:51.202 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:51.202 | INFO | ANALYSIS : 63.2%, 3.0, 98.4ms, 0 | 20260505 13:01:51.202 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:51.202 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:51.202 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 13:01:51.202 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:51.202 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:51.202 | INFO | Uploaded 36GiB to GPU in 2.1959s (16.3942GiB/s) | 20260505 13:01:51.202 | INFO | Downloaded 34GiB to GPU in 2.12046s (16.0343GiB/s) | 20260505 13:01:56.340 | INFO | Analyzing chunk frames = [3409408,3409920) | 20260505 13:01:56.340 | INFO | MemInfo: { | 20260505 13:01:56.340 | INFO | "gpu" : " 17 GB", | 20260505 13:01:56.340 | INFO | "huge" : "", | 20260505 13:01:56.340 | INFO | "huge_shared" : " 27 GB", | 20260505 13:01:56.340 | INFO | "malloc" : " 14 GB", | 20260505 13:01:56.340 | INFO | "pinned" : " 29 GB" | 20260505 13:01:56.340 | INFO | } | 20260505 13:01:56.340 | INFO | Perf report for the last 5.13853 seconds: | 20260505 13:01:56.340 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:01:56.340 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:56.340 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:56.340 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:56.340 | INFO | ANALYSIS : 63.2%, 3.0, 98.1ms, 0 | 20260505 13:01:56.340 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:56.340 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:56.340 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:01:56.340 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:56.340 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:01:56.341 | INFO | Uploaded 36GiB to GPU in 2.2068s (16.3132GiB/s) | 20260505 13:01:56.341 | INFO | Downloaded 34GiB to GPU in 2.13245s (15.9441GiB/s) | 20260505 13:02:01.477 | INFO | Analyzing chunk frames = [3409920,3410432) | 20260505 13:02:01.477 | INFO | MemInfo: { | 20260505 13:02:01.477 | INFO | "gpu" : " 17 GB", | 20260505 13:02:01.477 | INFO | "huge" : "", | 20260505 13:02:01.477 | INFO | "huge_shared" : " 27 GB", | 20260505 13:02:01.477 | INFO | "malloc" : " 14 GB", | 20260505 13:02:01.477 | INFO | "pinned" : " 29 GB" | 20260505 13:02:01.477 | INFO | } | 20260505 13:02:01.477 | INFO | Perf report for the last 5.13697 seconds: | 20260505 13:02:01.477 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:02:01.477 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:01.477 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:01.477 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:01.477 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 13:02:01.477 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:01.477 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:01.477 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 13:02:01.477 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:01.477 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:01.478 | INFO | Uploaded 36GiB to GPU in 2.16322s (16.6418GiB/s) | 20260505 13:02:01.478 | INFO | Downloaded 34GiB to GPU in 2.12715s (15.9838GiB/s) | 20260505 13:02:06.617 | INFO | Analyzing chunk frames = [3410432,3410944) | 20260505 13:02:06.617 | INFO | MemInfo: { | 20260505 13:02:06.617 | INFO | "gpu" : " 17 GB", | 20260505 13:02:06.617 | INFO | "huge" : "", | 20260505 13:02:06.617 | INFO | "huge_shared" : " 27 GB", | 20260505 13:02:06.617 | INFO | "malloc" : " 14 GB", | 20260505 13:02:06.617 | INFO | "pinned" : " 29 GB" | 20260505 13:02:06.617 | INFO | } | 20260505 13:02:06.617 | INFO | Perf report for the last 5.14017 seconds: | 20260505 13:02:06.617 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:02:06.617 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:06.617 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:06.617 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:06.617 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 13:02:06.617 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:06.618 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:06.618 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 13:02:06.618 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:06.618 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:06.618 | INFO | Uploaded 36GiB to GPU in 2.14321s (16.7973GiB/s) | 20260505 13:02:06.618 | INFO | Downloaded 34GiB to GPU in 2.10291s (16.1681GiB/s) | 20260505 13:02:11.761 | INFO | Analyzing chunk frames = [3410944,3411456) | 20260505 13:02:11.761 | INFO | MemInfo: { | 20260505 13:02:11.761 | INFO | "gpu" : " 17 GB", | 20260505 13:02:11.761 | INFO | "huge" : "", | 20260505 13:02:11.761 | INFO | "huge_shared" : " 27 GB", | 20260505 13:02:11.761 | INFO | "malloc" : " 14 GB", | 20260505 13:02:11.761 | INFO | "pinned" : " 29 GB" | 20260505 13:02:11.761 | INFO | } | 20260505 13:02:11.761 | INFO | Perf report for the last 5.14337 seconds: | 20260505 13:02:11.761 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:02:11.761 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:11.761 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:11.761 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:11.761 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 13:02:11.761 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:11.761 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:11.761 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 13:02:11.761 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:11.761 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:11.761 | INFO | Uploaded 36GiB to GPU in 2.14633s (16.7728GiB/s) | 20260505 13:02:11.761 | INFO | Downloaded 34GiB to GPU in 2.15595s (15.7703GiB/s) | 20260505 13:02:16.900 | INFO | Analyzing chunk frames = [3411456,3411968) | 20260505 13:02:16.900 | INFO | MemInfo: { | 20260505 13:02:16.900 | INFO | "gpu" : " 17 GB", | 20260505 13:02:16.900 | INFO | "huge" : "", | 20260505 13:02:16.900 | INFO | "huge_shared" : " 27 GB", | 20260505 13:02:16.900 | INFO | "malloc" : " 14 GB", | 20260505 13:02:16.900 | INFO | "pinned" : " 29 GB" | 20260505 13:02:16.900 | INFO | } | 20260505 13:02:16.900 | INFO | Perf report for the last 5.13944 seconds: | 20260505 13:02:16.900 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:02:16.900 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:16.900 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:16.900 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:16.900 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 13:02:16.900 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:16.900 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:16.900 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 13:02:16.900 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:16.900 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:16.900 | INFO | Uploaded 36GiB to GPU in 2.21356s (16.2634GiB/s) | 20260505 13:02:16.901 | INFO | Downloaded 34GiB to GPU in 2.10232s (16.1726GiB/s) | 20260505 13:02:17.040 | INFO | Starting a round of Baz IO | 20260505 13:02:20.265 | INFO | Populating RTMetrics report | 20260505 13:02:20.265 | INFO | RTMetrics report populated | 20260505 13:02:20.265 | INFO | Opening tmp RTMetrics json file | 20260505 13:02:20.284 | INFO | Serializing RTMetrics to json | 20260505 13:02:20.286 | INFO | Serialization done, swapping tmp file now... | 20260505 13:02:20.289 | INFO | RTMetrics done | 20260505 13:02:20.382 | INFO | Finished a round of Baz IO: Wrote 2.319 GiB in 3.342 seconds | 20260505 13:02:20.383 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.29% / 0.01% / 76.70% | 20260505 13:02:20.383 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.775675% / 1.19209e-05% / 8.34465e-05% | 20260505 13:02:20.383 | INFO | Baz IO overall: | 20260505 13:02:20.383 | INFO | BytesWritten : 2030799483399 | 20260505 13:02:20.383 | INFO | NumEvents : 966390248377 | 20260505 13:02:22.028 | INFO | Analyzing chunk frames = [3411968,3412480) | 20260505 13:02:22.028 | INFO | MemInfo: { | 20260505 13:02:22.028 | INFO | "gpu" : " 17 GB", | 20260505 13:02:22.028 | INFO | "huge" : "", | 20260505 13:02:22.028 | INFO | "huge_shared" : " 27 GB", | 20260505 13:02:22.028 | INFO | "malloc" : " 14 GB", | 20260505 13:02:22.028 | INFO | "pinned" : " 29 GB" | 20260505 13:02:22.028 | INFO | } | 20260505 13:02:22.028 | INFO | Perf report for the last 5.12796 seconds: | 20260505 13:02:22.028 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:02:22.028 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:22.028 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:22.028 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:22.028 | INFO | ANALYSIS : 64.5%, 3.0, 99.7ms, 0 | 20260505 13:02:22.028 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:22.028 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260505 13:02:22.028 | INFO | PRE_HQ : 61.0%, 1.8, 57.7ms, 0 | 20260505 13:02:22.028 | INFO | BAZWRITER : 64.5%, 4.1, 134.1ms, 0 | 20260505 13:02:22.028 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 13:02:22.028 | INFO | Uploaded 36GiB to GPU in 2.18974s (16.4403GiB/s) | 20260505 13:02:22.029 | INFO | Downloaded 38GiB to GPU in 2.28336s (16.6422GiB/s) | 20260505 13:02:27.164 | INFO | Analyzing chunk frames = [3412480,3412992) | 20260505 13:02:27.164 | INFO | MemInfo: { | 20260505 13:02:27.164 | INFO | "gpu" : " 17 GB", | 20260505 13:02:27.164 | INFO | "huge" : "", | 20260505 13:02:27.164 | INFO | "huge_shared" : " 27 GB", | 20260505 13:02:27.164 | INFO | "malloc" : " 14 GB", | 20260505 13:02:27.164 | INFO | "pinned" : " 29 GB" | 20260505 13:02:27.164 | INFO | } | 20260505 13:02:27.164 | INFO | Perf report for the last 5.13553 seconds: | 20260505 13:02:27.164 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:02:27.164 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:27.164 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:27.164 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:27.164 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 13:02:27.164 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:27.164 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:27.164 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 13:02:27.164 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:27.164 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:27.164 | INFO | Uploaded 36GiB to GPU in 2.21855s (16.2268GiB/s) | 20260505 13:02:27.164 | INFO | Downloaded 34GiB to GPU in 2.08804s (16.2832GiB/s) | 20260505 13:02:32.313 | INFO | Analyzing chunk frames = [3412992,3413504) | 20260505 13:02:32.313 | INFO | MemInfo: { | 20260505 13:02:32.313 | INFO | "gpu" : " 17 GB", | 20260505 13:02:32.313 | INFO | "huge" : "", | 20260505 13:02:32.313 | INFO | "huge_shared" : " 27 GB", | 20260505 13:02:32.313 | INFO | "malloc" : " 14 GB", | 20260505 13:02:32.313 | INFO | "pinned" : " 29 GB" | 20260505 13:02:32.313 | INFO | } | 20260505 13:02:32.313 | INFO | Perf report for the last 5.14903 seconds: | 20260505 13:02:32.313 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:02:32.313 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:32.313 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:32.313 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:32.313 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 13:02:32.313 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:32.313 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:32.313 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 13:02:32.313 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:32.313 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:32.313 | INFO | Uploaded 36GiB to GPU in 2.21445s (16.2569GiB/s) | 20260505 13:02:32.313 | INFO | Downloaded 34GiB to GPU in 2.12507s (15.9995GiB/s) | 20260505 13:02:37.453 | INFO | Analyzing chunk frames = [3413504,3414016) | 20260505 13:02:37.453 | INFO | MemInfo: { | 20260505 13:02:37.453 | INFO | "gpu" : " 17 GB", | 20260505 13:02:37.453 | INFO | "huge" : "", | 20260505 13:02:37.453 | INFO | "huge_shared" : " 27 GB", | 20260505 13:02:37.453 | INFO | "malloc" : " 14 GB", | 20260505 13:02:37.453 | INFO | "pinned" : " 29 GB" | 20260505 13:02:37.453 | INFO | } | 20260505 13:02:37.453 | INFO | Perf report for the last 5.14062 seconds: | 20260505 13:02:37.453 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:02:37.453 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:37.453 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:37.453 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:37.453 | INFO | ANALYSIS : 63.1%, 3.0, 98.1ms, 0 | 20260505 13:02:37.453 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:37.453 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:37.453 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 13:02:37.453 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:37.453 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:37.454 | INFO | Uploaded 36GiB to GPU in 2.20137s (16.3534GiB/s) | 20260505 13:02:37.454 | INFO | Downloaded 34GiB to GPU in 2.13011s (15.9616GiB/s) | 20260505 13:02:42.595 | INFO | Analyzing chunk frames = [3414016,3414528) | 20260505 13:02:42.595 | INFO | MemInfo: { | 20260505 13:02:42.595 | INFO | "gpu" : " 17 GB", | 20260505 13:02:42.595 | INFO | "huge" : "", | 20260505 13:02:42.595 | INFO | "huge_shared" : " 27 GB", | 20260505 13:02:42.595 | INFO | "malloc" : " 14 GB", | 20260505 13:02:42.595 | INFO | "pinned" : " 29 GB" | 20260505 13:02:42.595 | INFO | } | 20260505 13:02:42.595 | INFO | Perf report for the last 5.14176 seconds: | 20260505 13:02:42.595 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:02:42.595 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:42.595 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:42.595 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:42.595 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 13:02:42.595 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:42.595 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:42.595 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 13:02:42.595 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:42.595 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:42.595 | INFO | Uploaded 36GiB to GPU in 2.15168s (16.7311GiB/s) | 20260505 13:02:42.596 | INFO | Downloaded 34GiB to GPU in 2.12475s (16.0019GiB/s) | 20260505 13:02:47.726 | INFO | Analyzing chunk frames = [3414528,3415040) | 20260505 13:02:47.726 | INFO | MemInfo: { | 20260505 13:02:47.726 | INFO | "gpu" : " 17 GB", | 20260505 13:02:47.726 | INFO | "huge" : "", | 20260505 13:02:47.726 | INFO | "huge_shared" : " 27 GB", | 20260505 13:02:47.727 | INFO | "malloc" : " 14 GB", | 20260505 13:02:47.727 | INFO | "pinned" : " 29 GB" | 20260505 13:02:47.727 | INFO | } | 20260505 13:02:47.727 | INFO | Perf report for the last 5.13138 seconds: | 20260505 13:02:47.727 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:02:47.727 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:47.727 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:47.727 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:47.727 | INFO | ANALYSIS : 63.4%, 3.0, 98.8ms, 0 | 20260505 13:02:47.727 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:47.727 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:47.727 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 13:02:47.727 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:47.727 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:47.727 | INFO | Uploaded 36GiB to GPU in 2.16832s (16.6027GiB/s) | 20260505 13:02:47.727 | INFO | Downloaded 34GiB to GPU in 2.10831s (16.1267GiB/s) | 20260505 13:02:52.870 | INFO | Analyzing chunk frames = [3415040,3415552) | 20260505 13:02:52.870 | INFO | MemInfo: { | 20260505 13:02:52.870 | INFO | "gpu" : " 17 GB", | 20260505 13:02:52.870 | INFO | "huge" : "", | 20260505 13:02:52.870 | INFO | "huge_shared" : " 27 GB", | 20260505 13:02:52.870 | INFO | "malloc" : " 14 GB", | 20260505 13:02:52.870 | INFO | "pinned" : " 29 GB" | 20260505 13:02:52.870 | INFO | } | 20260505 13:02:52.870 | INFO | Perf report for the last 5.14369 seconds: | 20260505 13:02:52.870 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:02:52.870 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:52.870 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:52.870 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:52.870 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 13:02:52.870 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:52.870 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:52.870 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 13:02:52.870 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:52.870 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:52.871 | INFO | Uploaded 36GiB to GPU in 2.19108s (16.4303GiB/s) | 20260505 13:02:52.871 | INFO | Downloaded 34GiB to GPU in 2.14669s (15.8383GiB/s) | 20260505 13:02:58.013 | INFO | Analyzing chunk frames = [3415552,3416064) | 20260505 13:02:58.013 | INFO | MemInfo: { | 20260505 13:02:58.013 | INFO | "gpu" : " 17 GB", | 20260505 13:02:58.013 | INFO | "huge" : "", | 20260505 13:02:58.013 | INFO | "huge_shared" : " 27 GB", | 20260505 13:02:58.013 | INFO | "malloc" : " 14 GB", | 20260505 13:02:58.013 | INFO | "pinned" : " 29 GB" | 20260505 13:02:58.013 | INFO | } | 20260505 13:02:58.013 | INFO | Perf report for the last 5.14307 seconds: | 20260505 13:02:58.013 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:02:58.013 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:58.013 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:58.013 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:58.013 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 13:02:58.013 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:58.013 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:58.013 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:02:58.014 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:58.014 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:02:58.014 | INFO | Uploaded 36GiB to GPU in 2.16519s (16.6267GiB/s) | 20260505 13:02:58.014 | INFO | Downloaded 34GiB to GPU in 2.10797s (16.1293GiB/s) | 20260505 13:02:58.014 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 13:02:58.014 | INFO | Total mem usage: 15.3 GiB | 20260505 13:02:58.014 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 13:02:58.014 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -6.99%) | 20260505 13:02:58.014 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 13:02:58.014 | INFO | High Water Breakdown: | 20260505 13:02:58.014 | INFO | 61.1% : 8.75 GiB ( 92% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 13:02:58.014 | INFO | 38.8% : 5.55 GiB ( 83.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 13:02:58.014 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 13:02:58.014 | INFO | ------------------------------------------ | 20260505 13:02:58.014 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:02:58.014 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 13:02:58.014 | INFO | Total mem usage: 26.5 GiB | 20260505 13:02:58.014 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 13:02:58.014 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 13:02:58.014 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 13:02:58.014 | INFO | High Water Breakdown: | 20260505 13:02:58.014 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:02:58.014 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:02:58.014 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:02:58.014 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:02:58.014 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:02:58.014 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:02:58.014 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 13:02:58.014 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 13:02:58.014 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:02:58.014 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 13:02:58.014 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 13:02:58.014 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 13:02:58.014 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 13:02:58.014 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 13:02:58.014 | INFO | ------------------------------------------ | 20260505 13:02:58.014 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:02:58.014 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 13:02:58.014 | INFO | Total mem usage: 25.6 GiB | 20260505 13:02:58.014 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 13:02:58.014 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 13:02:58.014 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 13:02:58.014 | INFO | High Water Breakdown: | 20260505 13:02:58.014 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 13:02:58.014 | INFO | ------------------------------------------ | 20260505 13:02:58.014 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:02:58.014 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 13:02:58.014 | INFO | Total mem usage: 15.7 GiB | 20260505 13:02:58.014 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 13:02:58.014 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 13:02:58.014 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 13:02:58.015 | INFO | High Water Breakdown: | 20260505 13:02:58.015 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 13:02:58.015 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 13:02:58.015 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 13:02:58.015 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 13:02:58.015 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 13:02:58.015 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 13:02:58.015 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 13:02:58.015 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 13:02:58.015 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 13:02:58.015 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:02:58.015 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 13:02:58.015 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:02:58.015 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 13:02:58.015 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 13:02:58.015 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 13:02:58.015 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 13:02:58.015 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:02:58.015 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 13:02:58.015 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:02:58.015 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:02:58.015 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:02:58.015 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 13:02:58.015 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 13:02:58.015 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:02:58.015 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 13:02:58.015 | INFO | ------------------------------------------ | 20260505 13:02:58.015 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:02:58.015 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 13:02:58.015 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:02:58.015 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:02:58.015 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.72e+01, 1.39e+01 / 1.36e+00 | 20260505 13:02:58.015 | INFO | PerfReport: 0.29%, AggregateStats -- 9.30e-02 / 9.96e-02, 1.08e-02 / 4.61e-03 | 20260505 13:02:58.015 | INFO | PerfReport: 11.76%, Baseline -- 3.76e+00 / 4.52e+00, 5.91e-01 / 1.67e-01 | 20260505 13:02:58.015 | INFO | PerfReport: 3.31%, Histogram -- 1.06e+00 / 1.50e+00, 2.02e-01 / 8.48e-02 | 20260505 13:02:58.015 | INFO | PerfReport: 16.03%, DME -- 4.10e+01 / 4.61e+01, 4.72e+00 / 1.73e+00 | 20260505 13:02:58.015 | INFO | PerfReport: 56.27%, FrameLabeling -- 1.80e+01 / 2.02e+01, 1.40e+00 / 1.93e+00 | 20260505 13:02:58.015 | INFO | PerfReport: 7.47%, PulseAccumulating -- 2.38e+00 / 2.93e+00, 4.49e-01 / 2.03e-01 | 20260505 13:02:58.015 | INFO | PerfReport: 3.61%, Metrics -- 1.15e+00 / 1.45e+00, 3.23e-01 / 1.20e-01 | 20260505 13:02:58.015 | INFO | PerfReport: 1.27%, PulseSerializing -- 4.05e-01 / 4.54e-01, 7.36e-02 / 2.64e-02 | 20260505 13:02:58.015 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 13:02:58.015 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:02:58.015 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:02:58.015 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260505 13:02:58.015 | INFO | PerfReport: 22.55%, Upload -- 2.28e+01 / 0.00e+00, 8.12e+00 / 0.00e+00 | 20260505 13:02:58.015 | INFO | PerfReport: 22.05%, Download -- 2.23e+01 / 0.00e+00, 8.12e+00 / 0.00e+00 | 20260505 13:02:58.015 | INFO | PerfReport: 31.62%, Compute -- 3.19e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 13:02:58.015 | INFO | PerfReport: 2.22%, LockUpload -- 2.24e+00 / 0.00e+00, 7.78e+00 / 0.00e+00 | 20260505 13:02:58.015 | INFO | PerfReport: 1.36%, LockDownload -- 1.38e+00 / 0.00e+00, 2.94e+00 / 0.00e+00 | 20260505 13:02:58.015 | INFO | PerfReport: 20.18%, LockCompute -- 2.04e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260505 13:02:58.015 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.80e-02 / 0.00e+00 | 20260505 13:02:58.153 | INFO | Starting a round of Baz IO | 20260505 13:03:01.378 | INFO | Populating RTMetrics report | 20260505 13:03:01.379 | INFO | RTMetrics report populated | 20260505 13:03:01.379 | INFO | Opening tmp RTMetrics json file | 20260505 13:03:01.398 | INFO | Serializing RTMetrics to json | 20260505 13:03:01.423 | INFO | Serialization done, swapping tmp file now... | 20260505 13:03:01.425 | INFO | RTMetrics done | 20260505 13:03:01.527 | INFO | Finished a round of Baz IO: Wrote 2.317 GiB in 3.373 seconds | 20260505 13:03:01.527 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.29% / 0.01% / 76.71% | 20260505 13:03:01.527 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.772667% / 7.94729e-06% / 0.000107288% | 20260505 13:03:01.527 | INFO | Baz IO overall: | 20260505 13:03:01.527 | INFO | BytesWritten : 2033286918740 | 20260505 13:03:01.527 | INFO | NumEvents : 967568373481 | 20260505 13:03:03.159 | INFO | Analyzing chunk frames = [3416064,3416576) | 20260505 13:03:03.159 | INFO | MemInfo: { | 20260505 13:03:03.159 | INFO | "gpu" : " 17 GB", | 20260505 13:03:03.159 | INFO | "huge" : "", | 20260505 13:03:03.159 | INFO | "huge_shared" : " 27 GB", | 20260505 13:03:03.159 | INFO | "malloc" : " 14 GB", | 20260505 13:03:03.159 | INFO | "pinned" : " 29 GB" | 20260505 13:03:03.159 | INFO | } | 20260505 13:03:03.159 | INFO | Perf report for the last 5.14558 seconds: | 20260505 13:03:03.159 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:03:03.159 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:03.159 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:03.159 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:03.159 | INFO | ANALYSIS : 64.3%, 2.9, 99.1ms, 0 | 20260505 13:03:03.159 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:03.159 | INFO | RT_METRICS : 46.7%, 1.0, 24.5ms, 0 | 20260505 13:03:03.159 | INFO | PRE_HQ : 60.9%, 1.8, 58.0ms, 0 | 20260505 13:03:03.159 | INFO | BAZWRITER : 64.9%, 4.1, 137.8ms, 0 | 20260505 13:03:03.159 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260505 13:03:03.159 | INFO | Uploaded 36GiB to GPU in 2.19475s (16.4028GiB/s) | 20260505 13:03:03.159 | INFO | Downloaded 38GiB to GPU in 2.28889s (16.602GiB/s) | 20260505 13:03:08.291 | INFO | Analyzing chunk frames = [3416576,3417088) | 20260505 13:03:08.291 | INFO | MemInfo: { | 20260505 13:03:08.291 | INFO | "gpu" : " 17 GB", | 20260505 13:03:08.291 | INFO | "huge" : "", | 20260505 13:03:08.292 | INFO | "huge_shared" : " 27 GB", | 20260505 13:03:08.292 | INFO | "malloc" : " 14 GB", | 20260505 13:03:08.292 | INFO | "pinned" : " 29 GB" | 20260505 13:03:08.292 | INFO | } | 20260505 13:03:08.292 | INFO | Perf report for the last 5.13263 seconds: | 20260505 13:03:08.292 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:03:08.292 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:08.292 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:08.292 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:08.292 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 13:03:08.292 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:08.292 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:08.292 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 13:03:08.292 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:08.292 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:08.292 | INFO | Uploaded 36GiB to GPU in 2.19379s (16.41GiB/s) | 20260505 13:03:08.292 | INFO | Downloaded 34GiB to GPU in 2.03567s (16.7021GiB/s) | 20260505 13:03:13.428 | INFO | Analyzing chunk frames = [3417088,3417600) | 20260505 13:03:13.428 | INFO | MemInfo: { | 20260505 13:03:13.428 | INFO | "gpu" : " 17 GB", | 20260505 13:03:13.428 | INFO | "huge" : "", | 20260505 13:03:13.428 | INFO | "huge_shared" : " 27 GB", | 20260505 13:03:13.428 | INFO | "malloc" : " 14 GB", | 20260505 13:03:13.428 | INFO | "pinned" : " 29 GB" | 20260505 13:03:13.428 | INFO | } | 20260505 13:03:13.428 | INFO | Perf report for the last 5.13623 seconds: | 20260505 13:03:13.428 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:03:13.428 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:13.428 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:13.428 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:13.428 | INFO | ANALYSIS : 63.4%, 3.0, 98.5ms, 0 | 20260505 13:03:13.428 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:13.428 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:13.428 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 13:03:13.428 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:13.428 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:13.428 | INFO | Uploaded 36GiB to GPU in 2.21068s (16.2846GiB/s) | 20260505 13:03:13.428 | INFO | Downloaded 34GiB to GPU in 2.12096s (16.0305GiB/s) | 20260505 13:03:18.568 | INFO | Analyzing chunk frames = [3417600,3418112) | 20260505 13:03:18.568 | INFO | MemInfo: { | 20260505 13:03:18.568 | INFO | "gpu" : " 17 GB", | 20260505 13:03:18.568 | INFO | "huge" : "", | 20260505 13:03:18.568 | INFO | "huge_shared" : " 27 GB", | 20260505 13:03:18.568 | INFO | "malloc" : " 14 GB", | 20260505 13:03:18.568 | INFO | "pinned" : " 29 GB" | 20260505 13:03:18.568 | INFO | } | 20260505 13:03:18.568 | INFO | Perf report for the last 5.13982 seconds: | 20260505 13:03:18.568 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:03:18.568 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:18.568 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:18.568 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:18.568 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 13:03:18.568 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:18.568 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:18.568 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 13:03:18.568 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:18.568 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:18.568 | INFO | Uploaded 36GiB to GPU in 2.20507s (16.326GiB/s) | 20260505 13:03:18.568 | INFO | Downloaded 34GiB to GPU in 2.13202s (15.9473GiB/s) | 20260505 13:03:23.713 | INFO | Analyzing chunk frames = [3418112,3418624) | 20260505 13:03:23.713 | INFO | MemInfo: { | 20260505 13:03:23.713 | INFO | "gpu" : " 17 GB", | 20260505 13:03:23.713 | INFO | "huge" : "", | 20260505 13:03:23.713 | INFO | "huge_shared" : " 27 GB", | 20260505 13:03:23.713 | INFO | "malloc" : " 14 GB", | 20260505 13:03:23.713 | INFO | "pinned" : " 29 GB" | 20260505 13:03:23.713 | INFO | } | 20260505 13:03:23.713 | INFO | Perf report for the last 5.14555 seconds: | 20260505 13:03:23.713 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:03:23.713 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:23.713 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:23.713 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:23.713 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 13:03:23.713 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:23.713 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:23.713 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:03:23.713 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:23.713 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:23.714 | INFO | Uploaded 36GiB to GPU in 2.14337s (16.796GiB/s) | 20260505 13:03:23.714 | INFO | Downloaded 34GiB to GPU in 2.12438s (16.0047GiB/s) | 20260505 13:03:28.856 | INFO | Analyzing chunk frames = [3418624,3419136) | 20260505 13:03:28.856 | INFO | MemInfo: { | 20260505 13:03:28.856 | INFO | "gpu" : " 17 GB", | 20260505 13:03:28.856 | INFO | "huge" : "", | 20260505 13:03:28.856 | INFO | "huge_shared" : " 27 GB", | 20260505 13:03:28.857 | INFO | "malloc" : " 14 GB", | 20260505 13:03:28.857 | INFO | "pinned" : " 29 GB" | 20260505 13:03:28.857 | INFO | } | 20260505 13:03:28.857 | INFO | Perf report for the last 5.14327 seconds: | 20260505 13:03:28.857 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:03:28.857 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:28.857 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:28.857 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:28.857 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 13:03:28.857 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:28.857 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:28.857 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 13:03:28.857 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:28.857 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:28.857 | INFO | Uploaded 36GiB to GPU in 2.18049s (16.51GiB/s) | 20260505 13:03:28.857 | INFO | Downloaded 34GiB to GPU in 2.11129s (16.1039GiB/s) | 20260505 13:03:33.978 | INFO | Analyzing chunk frames = [3419136,3419648) | 20260505 13:03:33.978 | INFO | MemInfo: { | 20260505 13:03:33.978 | INFO | "gpu" : " 17 GB", | 20260505 13:03:33.978 | INFO | "huge" : "", | 20260505 13:03:33.978 | INFO | "huge_shared" : " 27 GB", | 20260505 13:03:33.978 | INFO | "malloc" : " 14 GB", | 20260505 13:03:33.978 | INFO | "pinned" : " 29 GB" | 20260505 13:03:33.978 | INFO | } | 20260505 13:03:33.978 | INFO | Perf report for the last 5.1214 seconds: | 20260505 13:03:33.978 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:03:33.978 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:33.978 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:33.978 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:33.978 | INFO | ANALYSIS : 63.4%, 3.0, 98.6ms, 0 | 20260505 13:03:33.978 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:33.978 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:33.978 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 13:03:33.978 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:33.978 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:33.978 | INFO | Uploaded 36GiB to GPU in 2.1659s (16.6213GiB/s) | 20260505 13:03:33.978 | INFO | Downloaded 34GiB to GPU in 2.13486s (15.9261GiB/s) | 20260505 13:03:39.120 | INFO | Analyzing chunk frames = [3419648,3420160) | 20260505 13:03:39.120 | INFO | MemInfo: { | 20260505 13:03:39.120 | INFO | "gpu" : " 17 GB", | 20260505 13:03:39.120 | INFO | "huge" : "", | 20260505 13:03:39.120 | INFO | "huge_shared" : " 27 GB", | 20260505 13:03:39.120 | INFO | "malloc" : " 14 GB", | 20260505 13:03:39.120 | INFO | "pinned" : " 29 GB" | 20260505 13:03:39.120 | INFO | } | 20260505 13:03:39.120 | INFO | Perf report for the last 5.14222 seconds: | 20260505 13:03:39.120 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:03:39.120 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:39.120 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:39.120 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:39.120 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 13:03:39.120 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:39.120 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:39.120 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 13:03:39.120 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:39.120 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:39.120 | INFO | Uploaded 36GiB to GPU in 2.21993s (16.2168GiB/s) | 20260505 13:03:39.121 | INFO | Downloaded 34GiB to GPU in 2.09058s (16.2634GiB/s) | 20260505 13:03:39.253 | INFO | Starting a round of Baz IO | 20260505 13:03:42.499 | INFO | Populating RTMetrics report | 20260505 13:03:42.500 | INFO | RTMetrics report populated | 20260505 13:03:42.500 | INFO | Opening tmp RTMetrics json file | 20260505 13:03:42.501 | INFO | Serializing RTMetrics to json | 20260505 13:03:42.502 | INFO | Serialization done, swapping tmp file now... | 20260505 13:03:42.502 | INFO | RTMetrics done | 20260505 13:03:42.639 | INFO | Finished a round of Baz IO: Wrote 2.316 GiB in 3.385 seconds | 20260505 13:03:42.639 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.28% / 0.01% / 76.71% | 20260505 13:03:42.639 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.770028% / 1.19209e-05% / 9.13938e-05% | 20260505 13:03:42.639 | INFO | Baz IO overall: | 20260505 13:03:42.639 | INFO | BytesWritten : 2035773821347 | 20260505 13:03:42.639 | INFO | NumEvents : 968746492293 | 20260505 13:03:44.266 | INFO | Analyzing chunk frames = [3420160,3420672) | 20260505 13:03:44.266 | INFO | MemInfo: { | 20260505 13:03:44.266 | INFO | "gpu" : " 17 GB", | 20260505 13:03:44.266 | INFO | "huge" : "", | 20260505 13:03:44.266 | INFO | "huge_shared" : " 27 GB", | 20260505 13:03:44.266 | INFO | "malloc" : " 14 GB", | 20260505 13:03:44.266 | INFO | "pinned" : " 29 GB" | 20260505 13:03:44.266 | INFO | } | 20260505 13:03:44.266 | INFO | Perf report for the last 5.14607 seconds: | 20260505 13:03:44.266 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:03:44.266 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:44.266 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:44.266 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:44.266 | INFO | ANALYSIS : 64.6%, 2.9, 99.8ms, 0 | 20260505 13:03:44.266 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:44.266 | INFO | RT_METRICS : 45.3%, 1.0, 23.8ms, 0 | 20260505 13:03:44.266 | INFO | PRE_HQ : 60.7%, 1.8, 57.9ms, 0 | 20260505 13:03:44.266 | INFO | BAZWRITER : 65.1%, 4.1, 137.0ms, 0 | 20260505 13:03:44.266 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 13:03:44.267 | INFO | Uploaded 36GiB to GPU in 2.19262s (16.4187GiB/s) | 20260505 13:03:44.267 | INFO | Downloaded 38GiB to GPU in 2.28198s (16.6522GiB/s) | 20260505 13:03:49.412 | INFO | Analyzing chunk frames = [3420672,3421184) | 20260505 13:03:49.412 | INFO | MemInfo: { | 20260505 13:03:49.412 | INFO | "gpu" : " 17 GB", | 20260505 13:03:49.412 | INFO | "huge" : "", | 20260505 13:03:49.412 | INFO | "huge_shared" : " 27 GB", | 20260505 13:03:49.412 | INFO | "malloc" : " 14 GB", | 20260505 13:03:49.412 | INFO | "pinned" : " 29 GB" | 20260505 13:03:49.412 | INFO | } | 20260505 13:03:49.412 | INFO | Perf report for the last 5.14595 seconds: | 20260505 13:03:49.412 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:03:49.412 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:49.412 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:49.412 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:49.412 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 13:03:49.412 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:49.412 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:49.412 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:03:49.412 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:49.412 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:49.413 | INFO | Uploaded 36GiB to GPU in 2.19386s (16.4095GiB/s) | 20260505 13:03:49.413 | INFO | Downloaded 34GiB to GPU in 2.02178s (16.8168GiB/s) | 20260505 13:03:54.537 | INFO | Analyzing chunk frames = [3421184,3421696) | 20260505 13:03:54.537 | INFO | MemInfo: { | 20260505 13:03:54.537 | INFO | "gpu" : " 17 GB", | 20260505 13:03:54.537 | INFO | "huge" : "", | 20260505 13:03:54.537 | INFO | "huge_shared" : " 27 GB", | 20260505 13:03:54.537 | INFO | "malloc" : " 14 GB", | 20260505 13:03:54.537 | INFO | "pinned" : " 29 GB" | 20260505 13:03:54.537 | INFO | } | 20260505 13:03:54.537 | INFO | Perf report for the last 5.12522 seconds: | 20260505 13:03:54.537 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:03:54.537 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:54.537 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:54.538 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:54.538 | INFO | ANALYSIS : 63.8%, 3.0, 99.0ms, 0 | 20260505 13:03:54.538 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:54.538 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:54.538 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 13:03:54.538 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:54.538 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:54.538 | INFO | Uploaded 36GiB to GPU in 2.2406s (16.0671GiB/s) | 20260505 13:03:54.538 | INFO | Downloaded 34GiB to GPU in 2.10813s (16.128GiB/s) | 20260505 13:03:59.675 | INFO | Analyzing chunk frames = [3421696,3422208) | 20260505 13:03:59.675 | INFO | MemInfo: { | 20260505 13:03:59.675 | INFO | "gpu" : " 17 GB", | 20260505 13:03:59.675 | INFO | "huge" : "", | 20260505 13:03:59.675 | INFO | "huge_shared" : " 27 GB", | 20260505 13:03:59.675 | INFO | "malloc" : " 14 GB", | 20260505 13:03:59.675 | INFO | "pinned" : " 29 GB" | 20260505 13:03:59.675 | INFO | } | 20260505 13:03:59.675 | INFO | Perf report for the last 5.13743 seconds: | 20260505 13:03:59.675 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:03:59.675 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:59.675 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:59.675 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:59.675 | INFO | ANALYSIS : 63.3%, 3.0, 98.2ms, 0 | 20260505 13:03:59.675 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:59.675 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:59.675 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 13:03:59.675 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:59.675 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:03:59.675 | INFO | Uploaded 36GiB to GPU in 2.21304s (16.2672GiB/s) | 20260505 13:03:59.675 | INFO | Downloaded 34GiB to GPU in 2.12877s (15.9717GiB/s) | 20260505 13:04:04.821 | INFO | Analyzing chunk frames = [3422208,3422720) | 20260505 13:04:04.821 | INFO | MemInfo: { | 20260505 13:04:04.821 | INFO | "gpu" : " 17 GB", | 20260505 13:04:04.821 | INFO | "huge" : "", | 20260505 13:04:04.821 | INFO | "huge_shared" : " 27 GB", | 20260505 13:04:04.822 | INFO | "malloc" : " 14 GB", | 20260505 13:04:04.822 | INFO | "pinned" : " 29 GB" | 20260505 13:04:04.822 | INFO | } | 20260505 13:04:04.822 | INFO | Perf report for the last 5.1467 seconds: | 20260505 13:04:04.822 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:04:04.822 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:04.822 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:04.822 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:04.822 | INFO | ANALYSIS : 63.1%, 3.0, 98.7ms, 0 | 20260505 13:04:04.822 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:04.822 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:04.822 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:04:04.822 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:04.822 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:04.822 | INFO | Uploaded 36GiB to GPU in 2.19867s (16.3736GiB/s) | 20260505 13:04:04.822 | INFO | Downloaded 34GiB to GPU in 2.13058s (15.9581GiB/s) | 20260505 13:04:09.960 | INFO | Analyzing chunk frames = [3422720,3423232) | 20260505 13:04:09.960 | INFO | MemInfo: { | 20260505 13:04:09.960 | INFO | "gpu" : " 17 GB", | 20260505 13:04:09.960 | INFO | "huge" : "", | 20260505 13:04:09.960 | INFO | "huge_shared" : " 27 GB", | 20260505 13:04:09.960 | INFO | "malloc" : " 14 GB", | 20260505 13:04:09.960 | INFO | "pinned" : " 29 GB" | 20260505 13:04:09.960 | INFO | } | 20260505 13:04:09.960 | INFO | Perf report for the last 5.13861 seconds: | 20260505 13:04:09.960 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:04:09.960 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:09.960 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:09.960 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:09.960 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 13:04:09.960 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:09.960 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:09.960 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:04:09.960 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:09.960 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:09.961 | INFO | Uploaded 36GiB to GPU in 2.17408s (16.5588GiB/s) | 20260505 13:04:09.961 | INFO | Downloaded 34GiB to GPU in 2.10811s (16.1282GiB/s) | 20260505 13:04:15.102 | INFO | Analyzing chunk frames = [3423232,3423744) | 20260505 13:04:15.102 | INFO | MemInfo: { | 20260505 13:04:15.102 | INFO | "gpu" : " 17 GB", | 20260505 13:04:15.102 | INFO | "huge" : "", | 20260505 13:04:15.102 | INFO | "huge_shared" : " 27 GB", | 20260505 13:04:15.102 | INFO | "malloc" : " 14 GB", | 20260505 13:04:15.102 | INFO | "pinned" : " 29 GB" | 20260505 13:04:15.102 | INFO | } | 20260505 13:04:15.102 | INFO | Perf report for the last 5.14204 seconds: | 20260505 13:04:15.102 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:04:15.102 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:15.102 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:15.102 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:15.102 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 13:04:15.102 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:15.102 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:15.102 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 13:04:15.102 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:15.102 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:15.103 | INFO | Uploaded 36GiB to GPU in 2.17166s (16.5772GiB/s) | 20260505 13:04:15.103 | INFO | Downloaded 34GiB to GPU in 2.13743s (15.9069GiB/s) | 20260505 13:04:20.241 | INFO | Analyzing chunk frames = [3423744,3424256) | 20260505 13:04:20.241 | INFO | MemInfo: { | 20260505 13:04:20.241 | INFO | "gpu" : " 17 GB", | 20260505 13:04:20.241 | INFO | "huge" : "", | 20260505 13:04:20.241 | INFO | "huge_shared" : " 27 GB", | 20260505 13:04:20.241 | INFO | "malloc" : " 14 GB", | 20260505 13:04:20.241 | INFO | "pinned" : " 29 GB" | 20260505 13:04:20.241 | INFO | } | 20260505 13:04:20.241 | INFO | Perf report for the last 5.13866 seconds: | 20260505 13:04:20.241 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:04:20.241 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:20.241 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:20.241 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:20.241 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 13:04:20.241 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:20.241 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:20.241 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 13:04:20.241 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:20.241 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:20.241 | INFO | Uploaded 36GiB to GPU in 2.13529s (16.8595GiB/s) | 20260505 13:04:20.241 | INFO | Downloaded 34GiB to GPU in 2.10626s (16.1424GiB/s) | 20260505 13:04:20.241 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 13:04:20.241 | INFO | Total mem usage: 15.3 GiB | 20260505 13:04:20.241 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 13:04:20.241 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -6.97%) | 20260505 13:04:20.241 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 13:04:20.241 | INFO | High Water Breakdown: | 20260505 13:04:20.241 | INFO | 61.1% : 8.75 GiB ( 92% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 13:04:20.241 | INFO | 38.8% : 5.55 GiB ( 83.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 13:04:20.241 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 13:04:20.241 | INFO | ------------------------------------------ | 20260505 13:04:20.242 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:04:20.242 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 13:04:20.242 | INFO | Total mem usage: 26.5 GiB | 20260505 13:04:20.242 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 13:04:20.242 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 13:04:20.242 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 13:04:20.242 | INFO | High Water Breakdown: | 20260505 13:04:20.242 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:04:20.242 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:04:20.242 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:04:20.242 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:04:20.242 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:04:20.242 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:04:20.242 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 13:04:20.242 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 13:04:20.242 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:04:20.242 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 13:04:20.242 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 13:04:20.242 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 13:04:20.242 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 13:04:20.242 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 13:04:20.242 | INFO | ------------------------------------------ | 20260505 13:04:20.242 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:04:20.242 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 13:04:20.242 | INFO | Total mem usage: 25.6 GiB | 20260505 13:04:20.242 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 13:04:20.242 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 13:04:20.242 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 13:04:20.242 | INFO | High Water Breakdown: | 20260505 13:04:20.242 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 13:04:20.242 | INFO | ------------------------------------------ | 20260505 13:04:20.242 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:04:20.242 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 13:04:20.242 | INFO | Total mem usage: 15.7 GiB | 20260505 13:04:20.242 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 13:04:20.242 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 13:04:20.242 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 13:04:20.242 | INFO | High Water Breakdown: | 20260505 13:04:20.242 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 13:04:20.242 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 13:04:20.242 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 13:04:20.242 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 13:04:20.242 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 13:04:20.242 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 13:04:20.242 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 13:04:20.242 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 13:04:20.242 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 13:04:20.242 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:04:20.242 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 13:04:20.242 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:04:20.242 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 13:04:20.242 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 13:04:20.242 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 13:04:20.242 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 13:04:20.242 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:04:20.242 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 13:04:20.242 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:04:20.242 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:04:20.242 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:04:20.242 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 13:04:20.242 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 13:04:20.242 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:04:20.242 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 13:04:20.242 | INFO | ------------------------------------------ | 20260505 13:04:20.242 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:04:20.242 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 13:04:20.242 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:04:20.242 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:04:20.242 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.67e+01, 1.38e+01 / 1.62e+00 | 20260505 13:04:20.242 | INFO | PerfReport: 0.29%, AggregateStats -- 9.27e-02 / 1.01e-01, 1.03e-02 / 6.39e-03 | 20260505 13:04:20.242 | INFO | PerfReport: 11.73%, Baseline -- 3.75e+00 / 4.49e+00, 5.84e-01 / 2.04e-01 | 20260505 13:04:20.242 | INFO | PerfReport: 3.30%, Histogram -- 1.05e+00 / 1.51e+00, 2.00e-01 / 8.94e-02 | 20260505 13:04:20.243 | INFO | PerfReport: 15.87%, DME -- 4.06e+01 / 4.52e+01, 4.31e+00 / 1.78e+00 | 20260505 13:04:20.243 | INFO | PerfReport: 56.49%, FrameLabeling -- 1.80e+01 / 2.06e+01, 1.52e+00 / 2.19e+00 | 20260505 13:04:20.243 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.38e+00 / 2.91e+00, 4.43e-01 / 1.97e-01 | 20260505 13:04:20.243 | INFO | PerfReport: 3.61%, Metrics -- 1.15e+00 / 1.47e+00, 3.22e-01 / 1.81e-01 | 20260505 13:04:20.243 | INFO | PerfReport: 1.27%, PulseSerializing -- 4.04e-01 / 4.57e-01, 7.25e-02 / 3.18e-02 | 20260505 13:04:20.243 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 13:04:20.243 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:04:20.243 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:04:20.243 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260505 13:04:20.243 | INFO | PerfReport: 22.60%, Upload -- 2.28e+01 / 0.00e+00, 8.13e+00 / 0.00e+00 | 20260505 13:04:20.243 | INFO | PerfReport: 21.97%, Download -- 2.22e+01 / 0.00e+00, 8.12e+00 / 0.00e+00 | 20260505 13:04:20.243 | INFO | PerfReport: 31.66%, Compute -- 3.19e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 13:04:20.243 | INFO | PerfReport: 2.26%, LockUpload -- 2.28e+00 / 0.00e+00, 7.92e+00 / 0.00e+00 | 20260505 13:04:20.243 | INFO | PerfReport: 1.32%, LockDownload -- 1.33e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260505 13:04:20.243 | INFO | PerfReport: 20.18%, LockCompute -- 2.04e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260505 13:04:20.243 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.34e-02 / 0.00e+00 | 20260505 13:04:20.375 | INFO | Starting a round of Baz IO | 20260505 13:04:23.617 | INFO | Populating RTMetrics report | 20260505 13:04:23.617 | INFO | RTMetrics report populated | 20260505 13:04:23.617 | INFO | Opening tmp RTMetrics json file | 20260505 13:04:23.618 | INFO | Serializing RTMetrics to json | 20260505 13:04:23.619 | INFO | Serialization done, swapping tmp file now... | 20260505 13:04:23.619 | INFO | RTMetrics done | 20260505 13:04:23.746 | INFO | Finished a round of Baz IO: Wrote 2.317 GiB in 3.371 seconds | 20260505 13:04:23.746 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.28% / 0.00% / 76.72% | 20260505 13:04:23.746 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.769548% / 1.19209e-05% / 8.74201e-05% | 20260505 13:04:23.746 | INFO | Baz IO overall: | 20260505 13:04:23.746 | INFO | BytesWritten : 2038261255787 | 20260505 13:04:23.746 | INFO | NumEvents : 969925614245 | 20260505 13:04:25.384 | INFO | Analyzing chunk frames = [3424256,3424768) | 20260505 13:04:25.384 | INFO | MemInfo: { | 20260505 13:04:25.384 | INFO | "gpu" : " 17 GB", | 20260505 13:04:25.384 | INFO | "huge" : "", | 20260505 13:04:25.384 | INFO | "huge_shared" : " 27 GB", | 20260505 13:04:25.384 | INFO | "malloc" : " 14 GB", | 20260505 13:04:25.384 | INFO | "pinned" : " 29 GB" | 20260505 13:04:25.384 | INFO | } | 20260505 13:04:25.384 | INFO | Perf report for the last 5.14331 seconds: | 20260505 13:04:25.384 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:04:25.384 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:25.384 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:25.384 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:25.384 | INFO | ANALYSIS : 64.5%, 3.0, 100.0ms, 0 | 20260505 13:04:25.384 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:25.384 | INFO | RT_METRICS : 45.1%, 1.0, 23.7ms, 0 | 20260505 13:04:25.384 | INFO | PRE_HQ : 60.6%, 1.8, 58.1ms, 0 | 20260505 13:04:25.384 | INFO | BAZWRITER : 64.8%, 4.1, 135.5ms, 0 | 20260505 13:04:25.385 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 13:04:25.385 | INFO | Uploaded 36GiB to GPU in 2.18135s (16.5036GiB/s) | 20260505 13:04:25.385 | INFO | Downloaded 38GiB to GPU in 2.27978s (16.6683GiB/s) | 20260505 13:04:30.518 | INFO | Analyzing chunk frames = [3424768,3425280) | 20260505 13:04:30.518 | INFO | MemInfo: { | 20260505 13:04:30.518 | INFO | "gpu" : " 17 GB", | 20260505 13:04:30.518 | INFO | "huge" : "", | 20260505 13:04:30.518 | INFO | "huge_shared" : " 27 GB", | 20260505 13:04:30.518 | INFO | "malloc" : " 14 GB", | 20260505 13:04:30.518 | INFO | "pinned" : " 29 GB" | 20260505 13:04:30.518 | INFO | } | 20260505 13:04:30.518 | INFO | Perf report for the last 5.13407 seconds: | 20260505 13:04:30.518 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:04:30.518 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:30.518 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:30.518 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:30.518 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 13:04:30.518 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:30.518 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:30.518 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:04:30.518 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:30.518 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:30.519 | INFO | Uploaded 36GiB to GPU in 2.15614s (16.6965GiB/s) | 20260505 13:04:30.519 | INFO | Downloaded 34GiB to GPU in 2.03166s (16.7351GiB/s) | 20260505 13:04:35.657 | INFO | Analyzing chunk frames = [3425280,3425792) | 20260505 13:04:35.657 | INFO | MemInfo: { | 20260505 13:04:35.657 | INFO | "gpu" : " 17 GB", | 20260505 13:04:35.658 | INFO | "huge" : "", | 20260505 13:04:35.658 | INFO | "huge_shared" : " 27 GB", | 20260505 13:04:35.658 | INFO | "malloc" : " 14 GB", | 20260505 13:04:35.658 | INFO | "pinned" : " 29 GB" | 20260505 13:04:35.658 | INFO | } | 20260505 13:04:35.658 | INFO | Perf report for the last 5.13935 seconds: | 20260505 13:04:35.658 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:04:35.658 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:35.658 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:35.658 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:35.658 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 13:04:35.658 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:35.658 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:35.658 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:04:35.658 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:35.658 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:35.658 | INFO | Uploaded 36GiB to GPU in 2.22165s (16.2042GiB/s) | 20260505 13:04:35.658 | INFO | Downloaded 34GiB to GPU in 2.09551s (16.2252GiB/s) | 20260505 13:04:40.802 | INFO | Analyzing chunk frames = [3425792,3426304) | 20260505 13:04:40.802 | INFO | MemInfo: { | 20260505 13:04:40.802 | INFO | "gpu" : " 17 GB", | 20260505 13:04:40.802 | INFO | "huge" : "", | 20260505 13:04:40.802 | INFO | "huge_shared" : " 27 GB", | 20260505 13:04:40.802 | INFO | "malloc" : " 14 GB", | 20260505 13:04:40.802 | INFO | "pinned" : " 29 GB" | 20260505 13:04:40.802 | INFO | } | 20260505 13:04:40.802 | INFO | Perf report for the last 5.14448 seconds: | 20260505 13:04:40.802 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:04:40.802 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:40.802 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:40.802 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:40.802 | INFO | ANALYSIS : 63.1%, 3.0, 98.1ms, 0 | 20260505 13:04:40.802 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:40.802 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:40.802 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:04:40.802 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:40.802 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:40.803 | INFO | Uploaded 36GiB to GPU in 2.2072s (16.3103GiB/s) | 20260505 13:04:40.803 | INFO | Downloaded 34GiB to GPU in 2.13132s (15.9526GiB/s) | 20260505 13:04:45.929 | INFO | Analyzing chunk frames = [3426304,3426816) | 20260505 13:04:45.929 | INFO | MemInfo: { | 20260505 13:04:45.929 | INFO | "gpu" : " 17 GB", | 20260505 13:04:45.929 | INFO | "huge" : "", | 20260505 13:04:45.929 | INFO | "huge_shared" : " 27 GB", | 20260505 13:04:45.929 | INFO | "malloc" : " 14 GB", | 20260505 13:04:45.929 | INFO | "pinned" : " 29 GB" | 20260505 13:04:45.929 | INFO | } | 20260505 13:04:45.929 | INFO | Perf report for the last 5.12683 seconds: | 20260505 13:04:45.929 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:04:45.929 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:45.929 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:45.929 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:45.929 | INFO | ANALYSIS : 63.0%, 3.0, 98.1ms, 0 | 20260505 13:04:45.929 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:45.929 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:45.929 | INFO | PRE_HQ : 28.5%, 1.0, 14.7ms, 0 | 20260505 13:04:45.929 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:45.929 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:45.929 | INFO | Uploaded 36GiB to GPU in 2.15781s (16.6836GiB/s) | 20260505 13:04:45.929 | INFO | Downloaded 34GiB to GPU in 2.12533s (15.9976GiB/s) | 20260505 13:04:51.075 | INFO | Analyzing chunk frames = [3426816,3427328) | 20260505 13:04:51.075 | INFO | MemInfo: { | 20260505 13:04:51.075 | INFO | "gpu" : " 17 GB", | 20260505 13:04:51.075 | INFO | "huge" : "", | 20260505 13:04:51.075 | INFO | "huge_shared" : " 27 GB", | 20260505 13:04:51.075 | INFO | "malloc" : " 14 GB", | 20260505 13:04:51.075 | INFO | "pinned" : " 29 GB" | 20260505 13:04:51.075 | INFO | } | 20260505 13:04:51.075 | INFO | Perf report for the last 5.14635 seconds: | 20260505 13:04:51.075 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:04:51.075 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:51.075 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:51.075 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:51.075 | INFO | ANALYSIS : 63.1%, 3.0, 98.7ms, 0 | 20260505 13:04:51.076 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:51.076 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:51.076 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:04:51.076 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:51.076 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:51.076 | INFO | Uploaded 36GiB to GPU in 2.10684s (17.0872GiB/s) | 20260505 13:04:51.076 | INFO | Downloaded 34GiB to GPU in 2.11188s (16.0994GiB/s) | 20260505 13:04:56.226 | INFO | Analyzing chunk frames = [3427328,3427840) | 20260505 13:04:56.226 | INFO | MemInfo: { | 20260505 13:04:56.226 | INFO | "gpu" : " 17 GB", | 20260505 13:04:56.226 | INFO | "huge" : "", | 20260505 13:04:56.226 | INFO | "huge_shared" : " 27 GB", | 20260505 13:04:56.226 | INFO | "malloc" : " 14 GB", | 20260505 13:04:56.226 | INFO | "pinned" : " 29 GB" | 20260505 13:04:56.226 | INFO | } | 20260505 13:04:56.226 | INFO | Perf report for the last 5.15047 seconds: | 20260505 13:04:56.226 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:04:56.226 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:56.226 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:56.226 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:56.226 | INFO | ANALYSIS : 63.4%, 3.0, 99.2ms, 0 | 20260505 13:04:56.226 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:56.226 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:56.226 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 13:04:56.226 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:56.226 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:04:56.226 | INFO | Uploaded 36GiB to GPU in 2.11808s (16.9965GiB/s) | 20260505 13:04:56.226 | INFO | Downloaded 34GiB to GPU in 2.17973s (15.5983GiB/s) | 20260505 13:05:01.356 | INFO | Analyzing chunk frames = [3427840,3428352) | 20260505 13:05:01.356 | INFO | MemInfo: { | 20260505 13:05:01.356 | INFO | "gpu" : " 17 GB", | 20260505 13:05:01.356 | INFO | "huge" : "", | 20260505 13:05:01.356 | INFO | "huge_shared" : " 27 GB", | 20260505 13:05:01.356 | INFO | "malloc" : " 14 GB", | 20260505 13:05:01.357 | INFO | "pinned" : " 29 GB" | 20260505 13:05:01.357 | INFO | } | 20260505 13:05:01.357 | INFO | Perf report for the last 5.13068 seconds: | 20260505 13:05:01.357 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:05:01.357 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:01.357 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:01.357 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:01.357 | INFO | ANALYSIS : 63.8%, 3.0, 99.4ms, 0 | 20260505 13:05:01.357 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:01.357 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:01.357 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 13:05:01.357 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:01.357 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:01.357 | INFO | Uploaded 36GiB to GPU in 2.22066s (16.2114GiB/s) | 20260505 13:05:01.357 | INFO | Downloaded 34GiB to GPU in 2.10776s (16.1309GiB/s) | 20260505 13:05:01.491 | INFO | Starting a round of Baz IO | 20260505 13:05:04.733 | INFO | Populating RTMetrics report | 20260505 13:05:04.733 | INFO | RTMetrics report populated | 20260505 13:05:04.734 | INFO | Opening tmp RTMetrics json file | 20260505 13:05:04.757 | INFO | Serializing RTMetrics to json | 20260505 13:05:04.760 | INFO | Serialization done, swapping tmp file now... | 20260505 13:05:04.762 | INFO | RTMetrics done | 20260505 13:05:04.856 | INFO | Finished a round of Baz IO: Wrote 2.315 GiB in 3.365 seconds | 20260505 13:05:04.856 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.27% / 0.01% / 76.72% | 20260505 13:05:04.856 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.767227% / 3.97364e-06% / 8.34465e-05% | 20260505 13:05:04.856 | INFO | Baz IO overall: | 20260505 13:05:04.856 | INFO | BytesWritten : 2040746565795 | 20260505 13:05:04.856 | INFO | NumEvents : 971103087064 | 20260505 13:05:06.486 | INFO | Analyzing chunk frames = [3428352,3428864) | 20260505 13:05:06.486 | INFO | MemInfo: { | 20260505 13:05:06.486 | INFO | "gpu" : " 17 GB", | 20260505 13:05:06.486 | INFO | "huge" : "", | 20260505 13:05:06.486 | INFO | "huge_shared" : " 27 GB", | 20260505 13:05:06.486 | INFO | "malloc" : " 14 GB", | 20260505 13:05:06.486 | INFO | "pinned" : " 29 GB" | 20260505 13:05:06.486 | INFO | } | 20260505 13:05:06.486 | INFO | Perf report for the last 5.12937 seconds: | 20260505 13:05:06.486 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:05:06.486 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:06.486 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:06.486 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:06.486 | INFO | ANALYSIS : 64.7%, 2.9, 99.9ms, 0 | 20260505 13:05:06.486 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:06.486 | INFO | RT_METRICS : 45.7%, 1.0, 23.9ms, 0 | 20260505 13:05:06.486 | INFO | PRE_HQ : 61.0%, 1.8, 57.9ms, 0 | 20260505 13:05:06.486 | INFO | BAZWRITER : 64.9%, 4.1, 135.0ms, 0 | 20260505 13:05:06.486 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 13:05:06.486 | INFO | Uploaded 36GiB to GPU in 2.18975s (16.4402GiB/s) | 20260505 13:05:06.487 | INFO | Downloaded 38GiB to GPU in 2.28542s (16.6272GiB/s) | 20260505 13:05:11.630 | INFO | Analyzing chunk frames = [3428864,3429376) | 20260505 13:05:11.630 | INFO | MemInfo: { | 20260505 13:05:11.630 | INFO | "gpu" : " 17 GB", | 20260505 13:05:11.630 | INFO | "huge" : "", | 20260505 13:05:11.630 | INFO | "huge_shared" : " 27 GB", | 20260505 13:05:11.630 | INFO | "malloc" : " 14 GB", | 20260505 13:05:11.630 | INFO | "pinned" : " 29 GB" | 20260505 13:05:11.630 | INFO | } | 20260505 13:05:11.630 | INFO | Perf report for the last 5.1444 seconds: | 20260505 13:05:11.630 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:05:11.630 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:11.630 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:11.630 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:11.630 | INFO | ANALYSIS : 64.1%, 3.0, 100.2ms, 0 | 20260505 13:05:11.630 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:11.631 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:11.631 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 13:05:11.631 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:11.631 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:11.631 | INFO | Uploaded 36GiB to GPU in 2.23475s (16.1092GiB/s) | 20260505 13:05:11.631 | INFO | Downloaded 34GiB to GPU in 2.08642s (16.2959GiB/s) | 20260505 13:05:16.774 | INFO | Analyzing chunk frames = [3429376,3429888) | 20260505 13:05:16.774 | INFO | MemInfo: { | 20260505 13:05:16.774 | INFO | "gpu" : " 17 GB", | 20260505 13:05:16.774 | INFO | "huge" : "", | 20260505 13:05:16.774 | INFO | "huge_shared" : " 27 GB", | 20260505 13:05:16.774 | INFO | "malloc" : " 14 GB", | 20260505 13:05:16.774 | INFO | "pinned" : " 29 GB" | 20260505 13:05:16.774 | INFO | } | 20260505 13:05:16.774 | INFO | Perf report for the last 5.14404 seconds: | 20260505 13:05:16.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:05:16.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:16.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:16.775 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:16.775 | INFO | ANALYSIS : 63.1%, 3.0, 98.3ms, 0 | 20260505 13:05:16.775 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:16.775 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:16.775 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:05:16.775 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:16.775 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:16.775 | INFO | Uploaded 36GiB to GPU in 2.16987s (16.5908GiB/s) | 20260505 13:05:16.775 | INFO | Downloaded 34GiB to GPU in 2.1311s (15.9542GiB/s) | 20260505 13:05:21.913 | INFO | Analyzing chunk frames = [3429888,3430400) | 20260505 13:05:21.913 | INFO | MemInfo: { | 20260505 13:05:21.913 | INFO | "gpu" : " 17 GB", | 20260505 13:05:21.913 | INFO | "huge" : "", | 20260505 13:05:21.913 | INFO | "huge_shared" : " 27 GB", | 20260505 13:05:21.913 | INFO | "malloc" : " 14 GB", | 20260505 13:05:21.913 | INFO | "pinned" : " 29 GB" | 20260505 13:05:21.913 | INFO | } | 20260505 13:05:21.913 | INFO | Perf report for the last 5.13883 seconds: | 20260505 13:05:21.913 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:05:21.913 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:21.913 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:21.913 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:21.913 | INFO | ANALYSIS : 63.1%, 3.0, 97.9ms, 0 | 20260505 13:05:21.913 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:21.913 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:21.913 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 13:05:21.913 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:21.913 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:21.914 | INFO | Uploaded 36GiB to GPU in 2.19908s (16.3705GiB/s) | 20260505 13:05:21.914 | INFO | Downloaded 34GiB to GPU in 2.12911s (15.9691GiB/s) | 20260505 13:05:27.049 | INFO | Analyzing chunk frames = [3430400,3430912) | 20260505 13:05:27.049 | INFO | MemInfo: { | 20260505 13:05:27.049 | INFO | "gpu" : " 17 GB", | 20260505 13:05:27.049 | INFO | "huge" : "", | 20260505 13:05:27.049 | INFO | "huge_shared" : " 27 GB", | 20260505 13:05:27.049 | INFO | "malloc" : " 14 GB", | 20260505 13:05:27.049 | INFO | "pinned" : " 29 GB" | 20260505 13:05:27.049 | INFO | } | 20260505 13:05:27.049 | INFO | Perf report for the last 5.13601 seconds: | 20260505 13:05:27.049 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:05:27.049 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:27.049 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:27.049 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:27.049 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 13:05:27.049 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:27.049 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:27.049 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 13:05:27.049 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:27.049 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:27.050 | INFO | Uploaded 36GiB to GPU in 2.16551s (16.6242GiB/s) | 20260505 13:05:27.050 | INFO | Downloaded 34GiB to GPU in 2.12544s (15.9967GiB/s) | 20260505 13:05:32.185 | INFO | Analyzing chunk frames = [3430912,3431424) | 20260505 13:05:32.185 | INFO | MemInfo: { | 20260505 13:05:32.185 | INFO | "gpu" : " 17 GB", | 20260505 13:05:32.185 | INFO | "huge" : "", | 20260505 13:05:32.185 | INFO | "huge_shared" : " 27 GB", | 20260505 13:05:32.185 | INFO | "malloc" : " 14 GB", | 20260505 13:05:32.186 | INFO | "pinned" : " 29 GB" | 20260505 13:05:32.186 | INFO | } | 20260505 13:05:32.186 | INFO | Perf report for the last 5.13644 seconds: | 20260505 13:05:32.186 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:05:32.186 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:32.186 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:32.186 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:32.186 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 13:05:32.186 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:32.186 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:32.186 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 13:05:32.186 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:32.186 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:32.186 | INFO | Uploaded 36GiB to GPU in 2.15731s (16.6874GiB/s) | 20260505 13:05:32.186 | INFO | Downloaded 34GiB to GPU in 2.11165s (16.1012GiB/s) | 20260505 13:05:37.338 | INFO | Analyzing chunk frames = [3431424,3431936) | 20260505 13:05:37.338 | INFO | MemInfo: { | 20260505 13:05:37.338 | INFO | "gpu" : " 17 GB", | 20260505 13:05:37.338 | INFO | "huge" : "", | 20260505 13:05:37.338 | INFO | "huge_shared" : " 27 GB", | 20260505 13:05:37.338 | INFO | "malloc" : " 14 GB", | 20260505 13:05:37.338 | INFO | "pinned" : " 29 GB" | 20260505 13:05:37.338 | INFO | } | 20260505 13:05:37.338 | INFO | Perf report for the last 5.15196 seconds: | 20260505 13:05:37.338 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:05:37.338 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:37.338 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:37.338 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:37.338 | INFO | ANALYSIS : 63.1%, 3.0, 98.7ms, 0 | 20260505 13:05:37.338 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:37.338 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:37.338 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 13:05:37.338 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:37.338 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:37.338 | INFO | Uploaded 36GiB to GPU in 2.17307s (16.5664GiB/s) | 20260505 13:05:37.338 | INFO | Downloaded 34GiB to GPU in 2.14864s (15.8239GiB/s) | 20260505 13:05:42.469 | INFO | Analyzing chunk frames = [3431936,3432448) | 20260505 13:05:42.469 | INFO | MemInfo: { | 20260505 13:05:42.469 | INFO | "gpu" : " 17 GB", | 20260505 13:05:42.469 | INFO | "huge" : "", | 20260505 13:05:42.469 | INFO | "huge_shared" : " 27 GB", | 20260505 13:05:42.469 | INFO | "malloc" : " 14 GB", | 20260505 13:05:42.469 | INFO | "pinned" : " 29 GB" | 20260505 13:05:42.469 | INFO | } | 20260505 13:05:42.470 | INFO | Perf report for the last 5.13181 seconds: | 20260505 13:05:42.470 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:05:42.470 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:42.470 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:42.470 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:42.470 | INFO | ANALYSIS : 63.8%, 3.0, 99.5ms, 0 | 20260505 13:05:42.470 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:42.470 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:42.470 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 13:05:42.470 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:42.470 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:42.470 | INFO | Uploaded 36GiB to GPU in 2.21102s (16.282GiB/s) | 20260505 13:05:42.470 | INFO | Downloaded 34GiB to GPU in 2.10411s (16.1588GiB/s) | 20260505 13:05:42.470 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 13:05:42.470 | INFO | Total mem usage: 15.3 GiB | 20260505 13:05:42.470 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 13:05:42.470 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -7.07%) | 20260505 13:05:42.470 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 13:05:42.470 | INFO | High Water Breakdown: | 20260505 13:05:42.470 | INFO | 61.1% : 8.75 GiB ( 92% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 13:05:42.470 | INFO | 38.7% : 5.54 GiB ( 83.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 13:05:42.470 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 13:05:42.470 | INFO | ------------------------------------------ | 20260505 13:05:42.470 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:05:42.470 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 13:05:42.470 | INFO | Total mem usage: 26.5 GiB | 20260505 13:05:42.470 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 13:05:42.470 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 13:05:42.470 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 13:05:42.470 | INFO | High Water Breakdown: | 20260505 13:05:42.470 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:05:42.470 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:05:42.470 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:05:42.470 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:05:42.470 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:05:42.470 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:05:42.470 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 13:05:42.470 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 13:05:42.470 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:05:42.470 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 13:05:42.471 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 13:05:42.471 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 13:05:42.471 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 13:05:42.471 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 13:05:42.471 | INFO | ------------------------------------------ | 20260505 13:05:42.471 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:05:42.471 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 13:05:42.471 | INFO | Total mem usage: 25.6 GiB | 20260505 13:05:42.471 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 13:05:42.471 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 13:05:42.471 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 13:05:42.471 | INFO | High Water Breakdown: | 20260505 13:05:42.471 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 13:05:42.471 | INFO | ------------------------------------------ | 20260505 13:05:42.471 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:05:42.471 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 13:05:42.471 | INFO | Total mem usage: 15.7 GiB | 20260505 13:05:42.471 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 13:05:42.471 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 13:05:42.471 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 13:05:42.471 | INFO | High Water Breakdown: | 20260505 13:05:42.471 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 13:05:42.471 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 13:05:42.471 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 13:05:42.471 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 13:05:42.471 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 13:05:42.471 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 13:05:42.471 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 13:05:42.471 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 13:05:42.471 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 13:05:42.471 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:05:42.471 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 13:05:42.471 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:05:42.471 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 13:05:42.471 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 13:05:42.471 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 13:05:42.471 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 13:05:42.471 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:05:42.471 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 13:05:42.471 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:05:42.471 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:05:42.471 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:05:42.471 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 13:05:42.471 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 13:05:42.471 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:05:42.471 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 13:05:42.471 | INFO | ------------------------------------------ | 20260505 13:05:42.471 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:05:42.471 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 13:05:42.471 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:05:42.471 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:05:42.471 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.72e+01, 1.38e+01 / 1.49e+00 | 20260505 13:05:42.471 | INFO | PerfReport: 0.29%, AggregateStats -- 9.27e-02 / 9.99e-02, 1.11e-02 / 1.09e-02 | 20260505 13:05:42.471 | INFO | PerfReport: 11.76%, Baseline -- 3.75e+00 / 4.50e+00, 5.90e-01 / 1.60e-01 | 20260505 13:05:42.471 | INFO | PerfReport: 3.30%, Histogram -- 1.05e+00 / 1.49e+00, 2.01e-01 / 9.84e-02 | 20260505 13:05:42.471 | INFO | PerfReport: 16.09%, DME -- 4.10e+01 / 4.65e+01, 4.68e+00 / 2.05e+00 | 20260505 13:05:42.471 | INFO | PerfReport: 56.16%, FrameLabeling -- 1.79e+01 / 1.97e+01, 1.30e+00 / 1.92e+00 | 20260505 13:05:42.471 | INFO | PerfReport: 7.51%, PulseAccumulating -- 2.39e+00 / 2.94e+00, 4.57e-01 / 2.03e-01 | 20260505 13:05:42.471 | INFO | PerfReport: 3.62%, Metrics -- 1.15e+00 / 1.47e+00, 3.21e-01 / 1.53e-01 | 20260505 13:05:42.471 | INFO | PerfReport: 1.27%, PulseSerializing -- 4.05e-01 / 4.61e-01, 7.20e-02 / 2.79e-02 | 20260505 13:05:42.471 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 13:05:42.471 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:05:42.471 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:05:42.471 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.56e+01 / 0.00e+00 | 20260505 13:05:42.471 | INFO | PerfReport: 22.48%, Upload -- 2.27e+01 / 0.00e+00, 8.09e+00 / 0.00e+00 | 20260505 13:05:42.471 | INFO | PerfReport: 22.04%, Download -- 2.23e+01 / 0.00e+00, 8.13e+00 / 0.00e+00 | 20260505 13:05:42.471 | INFO | PerfReport: 31.58%, Compute -- 3.19e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 13:05:42.471 | INFO | PerfReport: 2.31%, LockUpload -- 2.33e+00 / 0.00e+00, 8.01e+00 / 0.00e+00 | 20260505 13:05:42.471 | INFO | PerfReport: 1.42%, LockDownload -- 1.43e+00 / 0.00e+00, 3.01e+00 / 0.00e+00 | 20260505 13:05:42.471 | INFO | PerfReport: 20.16%, LockCompute -- 2.04e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260505 13:05:42.471 | INFO | PerfReport: 0.01%, Misc -- 1.50e-02 / 0.00e+00, 2.79e-02 / 0.00e+00 | 20260505 13:05:42.612 | INFO | Starting a round of Baz IO | 20260505 13:05:45.840 | INFO | Populating RTMetrics report | 20260505 13:05:45.840 | INFO | RTMetrics report populated | 20260505 13:05:45.840 | INFO | Opening tmp RTMetrics json file | 20260505 13:05:45.843 | INFO | Serializing RTMetrics to json | 20260505 13:05:45.844 | INFO | Serialization done, swapping tmp file now... | 20260505 13:05:45.844 | INFO | RTMetrics done | 20260505 13:05:45.969 | INFO | Finished a round of Baz IO: Wrote 2.314 GiB in 3.357 seconds | 20260505 13:05:45.969 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.27% / 0.00% / 76.73% | 20260505 13:05:45.969 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.765336% / 3.97364e-06% / 9.53674e-05% | 20260505 13:05:45.969 | INFO | Baz IO overall: | 20260505 13:05:45.969 | INFO | BytesWritten : 2043231624434 | 20260505 13:05:45.969 | INFO | NumEvents : 972280542319 | 20260505 13:05:47.607 | INFO | Analyzing chunk frames = [3432448,3432960) | 20260505 13:05:47.607 | INFO | MemInfo: { | 20260505 13:05:47.607 | INFO | "gpu" : " 17 GB", | 20260505 13:05:47.607 | INFO | "huge" : "", | 20260505 13:05:47.607 | INFO | "huge_shared" : " 27 GB", | 20260505 13:05:47.607 | INFO | "malloc" : " 14 GB", | 20260505 13:05:47.607 | INFO | "pinned" : " 29 GB" | 20260505 13:05:47.607 | INFO | } | 20260505 13:05:47.607 | INFO | Perf report for the last 5.13761 seconds: | 20260505 13:05:47.607 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:05:47.607 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:47.607 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:47.607 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:47.607 | INFO | ANALYSIS : 64.5%, 2.9, 99.6ms, 0 | 20260505 13:05:47.607 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:47.607 | INFO | RT_METRICS : 45.3%, 1.0, 23.8ms, 0 | 20260505 13:05:47.607 | INFO | PRE_HQ : 61.1%, 1.8, 58.0ms, 0 | 20260505 13:05:47.607 | INFO | BAZWRITER : 64.7%, 4.1, 135.0ms, 0 | 20260505 13:05:47.607 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 13:05:47.608 | INFO | Uploaded 36GiB to GPU in 2.18486s (16.477GiB/s) | 20260505 13:05:47.608 | INFO | Downloaded 38GiB to GPU in 2.27528s (16.7012GiB/s) | 20260505 13:05:52.752 | INFO | Analyzing chunk frames = [3432960,3433472) | 20260505 13:05:52.752 | INFO | MemInfo: { | 20260505 13:05:52.752 | INFO | "gpu" : " 17 GB", | 20260505 13:05:52.752 | INFO | "huge" : "", | 20260505 13:05:52.752 | INFO | "huge_shared" : " 27 GB", | 20260505 13:05:52.752 | INFO | "malloc" : " 14 GB", | 20260505 13:05:52.752 | INFO | "pinned" : " 29 GB" | 20260505 13:05:52.752 | INFO | } | 20260505 13:05:52.752 | INFO | Perf report for the last 5.14492 seconds: | 20260505 13:05:52.752 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:05:52.752 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:52.752 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:52.752 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:52.752 | INFO | ANALYSIS : 63.7%, 3.0, 99.6ms, 0 | 20260505 13:05:52.752 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:52.752 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:52.752 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:05:52.752 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:52.752 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:52.752 | INFO | Uploaded 36GiB to GPU in 2.18861s (16.4488GiB/s) | 20260505 13:05:52.752 | INFO | Downloaded 34GiB to GPU in 2.05381s (16.5546GiB/s) | 20260505 13:05:57.886 | INFO | Analyzing chunk frames = [3433472,3433984) | 20260505 13:05:57.886 | INFO | MemInfo: { | 20260505 13:05:57.886 | INFO | "gpu" : " 17 GB", | 20260505 13:05:57.886 | INFO | "huge" : "", | 20260505 13:05:57.886 | INFO | "huge_shared" : " 27 GB", | 20260505 13:05:57.886 | INFO | "malloc" : " 14 GB", | 20260505 13:05:57.886 | INFO | "pinned" : " 29 GB" | 20260505 13:05:57.886 | INFO | } | 20260505 13:05:57.886 | INFO | Perf report for the last 5.13382 seconds: | 20260505 13:05:57.886 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:05:57.886 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:57.886 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:57.886 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:57.886 | INFO | ANALYSIS : 63.4%, 3.0, 98.6ms, 0 | 20260505 13:05:57.886 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:57.886 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:57.886 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 13:05:57.886 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:57.886 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:05:57.886 | INFO | Uploaded 36GiB to GPU in 2.20187s (16.3498GiB/s) | 20260505 13:05:57.886 | INFO | Downloaded 34GiB to GPU in 2.13252s (15.9436GiB/s) | 20260505 13:06:03.029 | INFO | Analyzing chunk frames = [3433984,3434496) | 20260505 13:06:03.029 | INFO | MemInfo: { | 20260505 13:06:03.029 | INFO | "gpu" : " 17 GB", | 20260505 13:06:03.029 | INFO | "huge" : "", | 20260505 13:06:03.029 | INFO | "huge_shared" : " 27 GB", | 20260505 13:06:03.029 | INFO | "malloc" : " 14 GB", | 20260505 13:06:03.029 | INFO | "pinned" : " 29 GB" | 20260505 13:06:03.029 | INFO | } | 20260505 13:06:03.029 | INFO | Perf report for the last 5.14286 seconds: | 20260505 13:06:03.029 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:06:03.029 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:03.029 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:03.029 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:03.029 | INFO | ANALYSIS : 63.1%, 3.0, 98.1ms, 0 | 20260505 13:06:03.029 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:03.029 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:03.029 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 13:06:03.029 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:03.029 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:03.029 | INFO | Uploaded 36GiB to GPU in 2.20735s (16.3092GiB/s) | 20260505 13:06:03.029 | INFO | Downloaded 34GiB to GPU in 2.12712s (15.9841GiB/s) | 20260505 13:06:08.177 | INFO | Analyzing chunk frames = [3434496,3435008) | 20260505 13:06:08.177 | INFO | MemInfo: { | 20260505 13:06:08.177 | INFO | "gpu" : " 17 GB", | 20260505 13:06:08.177 | INFO | "huge" : "", | 20260505 13:06:08.178 | INFO | "huge_shared" : " 27 GB", | 20260505 13:06:08.178 | INFO | "malloc" : " 14 GB", | 20260505 13:06:08.178 | INFO | "pinned" : " 29 GB" | 20260505 13:06:08.178 | INFO | } | 20260505 13:06:08.178 | INFO | Perf report for the last 5.14882 seconds: | 20260505 13:06:08.178 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:06:08.178 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:08.178 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:08.178 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:08.178 | INFO | ANALYSIS : 62.6%, 3.0, 97.9ms, 0 | 20260505 13:06:08.178 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:08.178 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:08.178 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 13:06:08.178 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:08.178 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:08.178 | INFO | Uploaded 36GiB to GPU in 2.14205s (16.8064GiB/s) | 20260505 13:06:08.178 | INFO | Downloaded 34GiB to GPU in 2.1189s (16.0461GiB/s) | 20260505 13:06:13.301 | INFO | Analyzing chunk frames = [3435008,3435520) | 20260505 13:06:13.301 | INFO | MemInfo: { | 20260505 13:06:13.301 | INFO | "gpu" : " 17 GB", | 20260505 13:06:13.301 | INFO | "huge" : "", | 20260505 13:06:13.301 | INFO | "huge_shared" : " 27 GB", | 20260505 13:06:13.301 | INFO | "malloc" : " 14 GB", | 20260505 13:06:13.301 | INFO | "pinned" : " 29 GB" | 20260505 13:06:13.301 | INFO | } | 20260505 13:06:13.301 | INFO | Perf report for the last 5.12338 seconds: | 20260505 13:06:13.301 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:06:13.301 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:13.301 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:13.301 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:13.301 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 13:06:13.301 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:13.301 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:13.301 | INFO | PRE_HQ : 28.3%, 1.0, 14.6ms, 0 | 20260505 13:06:13.301 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:13.301 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:13.301 | INFO | Uploaded 36GiB to GPU in 2.21309s (16.2668GiB/s) | 20260505 13:06:13.301 | INFO | Downloaded 34GiB to GPU in 2.10936s (16.1187GiB/s) | 20260505 13:06:18.436 | INFO | Analyzing chunk frames = [3435520,3436032) | 20260505 13:06:18.436 | INFO | MemInfo: { | 20260505 13:06:18.436 | INFO | "gpu" : " 17 GB", | 20260505 13:06:18.436 | INFO | "huge" : "", | 20260505 13:06:18.436 | INFO | "huge_shared" : " 27 GB", | 20260505 13:06:18.436 | INFO | "malloc" : " 14 GB", | 20260505 13:06:18.436 | INFO | "pinned" : " 29 GB" | 20260505 13:06:18.436 | INFO | } | 20260505 13:06:18.436 | INFO | Perf report for the last 5.13503 seconds: | 20260505 13:06:18.436 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:06:18.436 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:18.436 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:18.436 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:18.436 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 13:06:18.436 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:18.436 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:18.436 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 13:06:18.436 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:18.436 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:18.436 | INFO | Uploaded 36GiB to GPU in 2.17596s (16.5444GiB/s) | 20260505 13:06:18.437 | INFO | Downloaded 34GiB to GPU in 2.15003s (15.8137GiB/s) | 20260505 13:06:23.589 | INFO | Analyzing chunk frames = [3436032,3436544) | 20260505 13:06:23.589 | INFO | MemInfo: { | 20260505 13:06:23.589 | INFO | "gpu" : " 17 GB", | 20260505 13:06:23.589 | INFO | "huge" : "", | 20260505 13:06:23.589 | INFO | "huge_shared" : " 27 GB", | 20260505 13:06:23.589 | INFO | "malloc" : " 14 GB", | 20260505 13:06:23.589 | INFO | "pinned" : " 29 GB" | 20260505 13:06:23.589 | INFO | } | 20260505 13:06:23.589 | INFO | Perf report for the last 5.15333 seconds: | 20260505 13:06:23.589 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:06:23.589 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:23.589 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:23.589 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:23.589 | INFO | ANALYSIS : 63.8%, 3.0, 99.8ms, 0 | 20260505 13:06:23.589 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:23.590 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:23.590 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 13:06:23.590 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:23.590 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:23.590 | INFO | Uploaded 36GiB to GPU in 2.22775s (16.1598GiB/s) | 20260505 13:06:23.590 | INFO | Downloaded 34GiB to GPU in 2.09429s (16.2346GiB/s) | 20260505 13:06:23.726 | INFO | Starting a round of Baz IO | 20260505 13:06:26.953 | INFO | Populating RTMetrics report | 20260505 13:06:26.953 | INFO | RTMetrics report populated | 20260505 13:06:26.953 | INFO | Opening tmp RTMetrics json file | 20260505 13:06:27.013 | INFO | Serializing RTMetrics to json | 20260505 13:06:27.059 | INFO | Serialization done, swapping tmp file now... | 20260505 13:06:27.060 | INFO | RTMetrics done | 20260505 13:06:27.075 | INFO | Finished a round of Baz IO: Wrote 2.312 GiB in 3.348 seconds | 20260505 13:06:27.075 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.26% / 0.01% / 76.73% | 20260505 13:06:27.075 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.764267% / 1.19209e-05% / 7.94729e-05% | 20260505 13:06:27.075 | INFO | Baz IO overall: | 20260505 13:06:27.075 | INFO | BytesWritten : 2045714640757 | 20260505 13:06:27.075 | INFO | NumEvents : 973456539685 | 20260505 13:06:28.731 | INFO | Analyzing chunk frames = [3436544,3437056) | 20260505 13:06:28.731 | INFO | MemInfo: { | 20260505 13:06:28.731 | INFO | "gpu" : " 17 GB", | 20260505 13:06:28.731 | INFO | "huge" : "", | 20260505 13:06:28.731 | INFO | "huge_shared" : " 27 GB", | 20260505 13:06:28.731 | INFO | "malloc" : " 14 GB", | 20260505 13:06:28.731 | INFO | "pinned" : " 29 GB" | 20260505 13:06:28.731 | INFO | } | 20260505 13:06:28.731 | INFO | Perf report for the last 5.14161 seconds: | 20260505 13:06:28.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:06:28.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:28.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:28.731 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:28.731 | INFO | ANALYSIS : 64.3%, 2.9, 99.3ms, 0 | 20260505 13:06:28.731 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:28.731 | INFO | RT_METRICS : 47.1%, 1.0, 24.7ms, 0 | 20260505 13:06:28.731 | INFO | PRE_HQ : 61.1%, 1.8, 57.8ms, 0 | 20260505 13:06:28.731 | INFO | BAZWRITER : 64.5%, 4.1, 136.3ms, 0 | 20260505 13:06:28.731 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260505 13:06:28.731 | INFO | Uploaded 36GiB to GPU in 2.20174s (16.3507GiB/s) | 20260505 13:06:28.732 | INFO | Downloaded 38GiB to GPU in 2.27158s (16.7284GiB/s) | 20260505 13:06:33.864 | INFO | Analyzing chunk frames = [3437056,3437568) | 20260505 13:06:33.865 | INFO | MemInfo: { | 20260505 13:06:33.865 | INFO | "gpu" : " 17 GB", | 20260505 13:06:33.865 | INFO | "huge" : "", | 20260505 13:06:33.865 | INFO | "huge_shared" : " 27 GB", | 20260505 13:06:33.865 | INFO | "malloc" : " 14 GB", | 20260505 13:06:33.865 | INFO | "pinned" : " 29 GB" | 20260505 13:06:33.865 | INFO | } | 20260505 13:06:33.865 | INFO | Perf report for the last 5.13375 seconds: | 20260505 13:06:33.865 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:06:33.865 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:33.865 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:33.865 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:33.865 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 13:06:33.865 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:33.865 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:33.865 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:06:33.865 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:33.865 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:33.865 | INFO | Uploaded 36GiB to GPU in 2.19091s (16.4315GiB/s) | 20260505 13:06:33.865 | INFO | Downloaded 34GiB to GPU in 2.03652s (16.6952GiB/s) | 20260505 13:06:39.001 | INFO | Analyzing chunk frames = [3437568,3438080) | 20260505 13:06:39.001 | INFO | MemInfo: { | 20260505 13:06:39.001 | INFO | "gpu" : " 17 GB", | 20260505 13:06:39.001 | INFO | "huge" : "", | 20260505 13:06:39.001 | INFO | "huge_shared" : " 27 GB", | 20260505 13:06:39.001 | INFO | "malloc" : " 14 GB", | 20260505 13:06:39.001 | INFO | "pinned" : " 29 GB" | 20260505 13:06:39.001 | INFO | } | 20260505 13:06:39.001 | INFO | Perf report for the last 5.13632 seconds: | 20260505 13:06:39.001 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:06:39.001 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:39.001 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:39.001 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:39.001 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 13:06:39.001 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:39.001 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:39.001 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 13:06:39.001 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:39.001 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:39.002 | INFO | Uploaded 36GiB to GPU in 2.18027s (16.5117GiB/s) | 20260505 13:06:39.002 | INFO | Downloaded 34GiB to GPU in 2.13293s (15.9405GiB/s) | 20260505 13:06:44.141 | INFO | Analyzing chunk frames = [3438080,3438592) | 20260505 13:06:44.141 | INFO | MemInfo: { | 20260505 13:06:44.141 | INFO | "gpu" : " 17 GB", | 20260505 13:06:44.141 | INFO | "huge" : "", | 20260505 13:06:44.141 | INFO | "huge_shared" : " 27 GB", | 20260505 13:06:44.141 | INFO | "malloc" : " 14 GB", | 20260505 13:06:44.141 | INFO | "pinned" : " 29 GB" | 20260505 13:06:44.141 | INFO | } | 20260505 13:06:44.141 | INFO | Perf report for the last 5.14015 seconds: | 20260505 13:06:44.141 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:06:44.141 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:44.141 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:44.141 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:44.141 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 13:06:44.142 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:44.142 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:44.142 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:06:44.142 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:44.142 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:44.142 | INFO | Uploaded 36GiB to GPU in 2.20908s (16.2963GiB/s) | 20260505 13:06:44.142 | INFO | Downloaded 34GiB to GPU in 2.13441s (15.9295GiB/s) | 20260505 13:06:49.285 | INFO | Analyzing chunk frames = [3438592,3439104) | 20260505 13:06:49.285 | INFO | MemInfo: { | 20260505 13:06:49.285 | INFO | "gpu" : " 17 GB", | 20260505 13:06:49.285 | INFO | "huge" : "", | 20260505 13:06:49.285 | INFO | "huge_shared" : " 27 GB", | 20260505 13:06:49.285 | INFO | "malloc" : " 14 GB", | 20260505 13:06:49.285 | INFO | "pinned" : " 29 GB" | 20260505 13:06:49.286 | INFO | } | 20260505 13:06:49.286 | INFO | Perf report for the last 5.14428 seconds: | 20260505 13:06:49.286 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:06:49.286 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:49.286 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:49.286 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:49.286 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 13:06:49.286 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:49.286 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:49.286 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 13:06:49.286 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:49.286 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:49.286 | INFO | Uploaded 36GiB to GPU in 2.17604s (16.5438GiB/s) | 20260505 13:06:49.286 | INFO | Downloaded 34GiB to GPU in 2.12816s (15.9763GiB/s) | 20260505 13:06:54.420 | INFO | Analyzing chunk frames = [3439104,3439616) | 20260505 13:06:54.420 | INFO | MemInfo: { | 20260505 13:06:54.420 | INFO | "gpu" : " 17 GB", | 20260505 13:06:54.420 | INFO | "huge" : "", | 20260505 13:06:54.420 | INFO | "huge_shared" : " 27 GB", | 20260505 13:06:54.420 | INFO | "malloc" : " 14 GB", | 20260505 13:06:54.420 | INFO | "pinned" : " 29 GB" | 20260505 13:06:54.420 | INFO | } | 20260505 13:06:54.420 | INFO | Perf report for the last 5.13421 seconds: | 20260505 13:06:54.420 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:06:54.420 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:54.420 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:54.420 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:54.420 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 13:06:54.420 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:54.420 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:54.420 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:06:54.420 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:54.420 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:54.420 | INFO | Uploaded 36GiB to GPU in 2.19161s (16.4263GiB/s) | 20260505 13:06:54.420 | INFO | Downloaded 34GiB to GPU in 2.11407s (16.0828GiB/s) | 20260505 13:06:59.552 | INFO | Analyzing chunk frames = [3439616,3440128) | 20260505 13:06:59.552 | INFO | MemInfo: { | 20260505 13:06:59.552 | INFO | "gpu" : " 17 GB", | 20260505 13:06:59.552 | INFO | "huge" : "", | 20260505 13:06:59.552 | INFO | "huge_shared" : " 27 GB", | 20260505 13:06:59.552 | INFO | "malloc" : " 14 GB", | 20260505 13:06:59.552 | INFO | "pinned" : " 29 GB" | 20260505 13:06:59.552 | INFO | } | 20260505 13:06:59.552 | INFO | Perf report for the last 5.13247 seconds: | 20260505 13:06:59.552 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:06:59.552 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:59.552 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:59.552 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:59.552 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 13:06:59.552 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:59.552 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:59.552 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 13:06:59.552 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:59.552 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:06:59.553 | INFO | Uploaded 36GiB to GPU in 2.16192s (16.6518GiB/s) | 20260505 13:06:59.553 | INFO | Downloaded 34GiB to GPU in 2.1315s (15.9512GiB/s) | 20260505 13:07:04.695 | INFO | Analyzing chunk frames = [3440128,3440640) | 20260505 13:07:04.695 | INFO | MemInfo: { | 20260505 13:07:04.695 | INFO | "gpu" : " 17 GB", | 20260505 13:07:04.696 | INFO | "huge" : "", | 20260505 13:07:04.696 | INFO | "huge_shared" : " 27 GB", | 20260505 13:07:04.696 | INFO | "malloc" : " 14 GB", | 20260505 13:07:04.696 | INFO | "pinned" : " 29 GB" | 20260505 13:07:04.696 | INFO | } | 20260505 13:07:04.696 | INFO | Perf report for the last 5.14345 seconds: | 20260505 13:07:04.696 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:07:04.696 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:04.696 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:04.696 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:04.696 | INFO | ANALYSIS : 63.7%, 3.0, 99.6ms, 0 | 20260505 13:07:04.696 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:04.696 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:04.696 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:07:04.696 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:04.696 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:04.696 | INFO | Uploaded 36GiB to GPU in 2.22095s (16.2093GiB/s) | 20260505 13:07:04.696 | INFO | Downloaded 34GiB to GPU in 2.10513s (16.151GiB/s) | 20260505 13:07:04.696 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 13:07:04.696 | INFO | Total mem usage: 15.3 GiB | 20260505 13:07:04.696 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 13:07:04.696 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -7%) | 20260505 13:07:04.696 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 13:07:04.696 | INFO | High Water Breakdown: | 20260505 13:07:04.696 | INFO | 61.1% : 8.75 GiB ( 92% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 13:07:04.696 | INFO | 38.8% : 5.55 GiB ( 83.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 13:07:04.696 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 13:07:04.696 | INFO | ------------------------------------------ | 20260505 13:07:04.696 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:07:04.696 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 13:07:04.696 | INFO | Total mem usage: 26.5 GiB | 20260505 13:07:04.696 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 13:07:04.696 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 13:07:04.696 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 13:07:04.696 | INFO | High Water Breakdown: | 20260505 13:07:04.696 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:07:04.696 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:07:04.696 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:07:04.696 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:07:04.696 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:07:04.696 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:07:04.697 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 13:07:04.697 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 13:07:04.697 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:07:04.697 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 13:07:04.697 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 13:07:04.697 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 13:07:04.697 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 13:07:04.697 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 13:07:04.697 | INFO | ------------------------------------------ | 20260505 13:07:04.697 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:07:04.697 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 13:07:04.697 | INFO | Total mem usage: 25.6 GiB | 20260505 13:07:04.697 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 13:07:04.697 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 13:07:04.697 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 13:07:04.697 | INFO | High Water Breakdown: | 20260505 13:07:04.697 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 13:07:04.697 | INFO | ------------------------------------------ | 20260505 13:07:04.697 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:07:04.697 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 13:07:04.697 | INFO | Total mem usage: 15.7 GiB | 20260505 13:07:04.697 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 13:07:04.697 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 13:07:04.697 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 13:07:04.697 | INFO | High Water Breakdown: | 20260505 13:07:04.697 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 13:07:04.697 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 13:07:04.697 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 13:07:04.697 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 13:07:04.697 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 13:07:04.697 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 13:07:04.697 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 13:07:04.697 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 13:07:04.697 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 13:07:04.697 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:07:04.697 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 13:07:04.697 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:07:04.697 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 13:07:04.697 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 13:07:04.697 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 13:07:04.697 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 13:07:04.697 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:07:04.697 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 13:07:04.697 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:07:04.697 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:07:04.697 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:07:04.697 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 13:07:04.697 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 13:07:04.697 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:07:04.697 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 13:07:04.697 | INFO | ------------------------------------------ | 20260505 13:07:04.697 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:07:04.697 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 13:07:04.697 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:07:04.697 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:07:04.697 | INFO | PerfReport: Full Execution -- 3.20e+01 / 7.71e+01, 1.36e+01 / 1.57e+00 | 20260505 13:07:04.697 | INFO | PerfReport: 0.29%, AggregateStats -- 9.31e-02 / 1.01e-01, 1.06e-02 / 3.33e-03 | 20260505 13:07:04.697 | INFO | PerfReport: 11.75%, Baseline -- 3.76e+00 / 4.51e+00, 5.92e-01 / 1.27e-01 | 20260505 13:07:04.697 | INFO | PerfReport: 3.30%, Histogram -- 1.06e+00 / 1.50e+00, 2.00e-01 / 8.20e-02 | 20260505 13:07:04.697 | INFO | PerfReport: 15.86%, DME -- 4.06e+01 / 4.56e+01, 4.60e+00 / 1.94e+00 | 20260505 13:07:04.697 | INFO | PerfReport: 56.54%, FrameLabeling -- 1.81e+01 / 2.06e+01, 1.58e+00 / 2.16e+00 | 20260505 13:07:04.697 | INFO | PerfReport: 7.39%, PulseAccumulating -- 2.36e+00 / 2.89e+00, 4.39e-01 / 1.95e-01 | 20260505 13:07:04.697 | INFO | PerfReport: 3.60%, Metrics -- 1.15e+00 / 1.47e+00, 3.25e-01 / 1.55e-01 | 20260505 13:07:04.697 | INFO | PerfReport: 1.26%, PulseSerializing -- 4.04e-01 / 4.55e-01, 7.24e-02 / 2.23e-02 | 20260505 13:07:04.697 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 13:07:04.697 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:07:04.697 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:07:04.697 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260505 13:07:04.697 | INFO | PerfReport: 22.63%, Upload -- 2.28e+01 / 0.00e+00, 8.15e+00 / 0.00e+00 | 20260505 13:07:04.697 | INFO | PerfReport: 22.01%, Download -- 2.22e+01 / 0.00e+00, 8.17e+00 / 0.00e+00 | 20260505 13:07:04.697 | INFO | PerfReport: 31.70%, Compute -- 3.20e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 13:07:04.697 | INFO | PerfReport: 2.17%, LockUpload -- 2.19e+00 / 0.00e+00, 7.71e+00 / 0.00e+00 | 20260505 13:07:04.697 | INFO | PerfReport: 1.28%, LockDownload -- 1.29e+00 / 0.00e+00, 2.77e+00 / 0.00e+00 | 20260505 13:07:04.697 | INFO | PerfReport: 20.20%, LockCompute -- 2.04e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260505 13:07:04.697 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.24e-02 / 0.00e+00 | 20260505 13:07:04.829 | INFO | Starting a round of Baz IO | 20260505 13:07:08.077 | INFO | Populating RTMetrics report | 20260505 13:07:08.078 | INFO | RTMetrics report populated | 20260505 13:07:08.078 | INFO | Opening tmp RTMetrics json file | 20260505 13:07:08.090 | INFO | Serializing RTMetrics to json | 20260505 13:07:08.102 | INFO | Serialization done, swapping tmp file now... | 20260505 13:07:08.103 | INFO | RTMetrics done | 20260505 13:07:08.206 | INFO | Finished a round of Baz IO: Wrote 2.311 GiB in 3.376 seconds | 20260505 13:07:08.206 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.26% / 0.00% / 76.74% | 20260505 13:07:08.206 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.758417% / 7.94729e-06% / 0.000107288% | 20260505 13:07:08.206 | INFO | Baz IO overall: | 20260505 13:07:08.206 | INFO | BytesWritten : 2048196285822 | 20260505 13:07:08.206 | INFO | NumEvents : 974631548179 | 20260505 13:07:09.835 | INFO | Analyzing chunk frames = [3440640,3441152) | 20260505 13:07:09.835 | INFO | MemInfo: { | 20260505 13:07:09.835 | INFO | "gpu" : " 17 GB", | 20260505 13:07:09.835 | INFO | "huge" : "", | 20260505 13:07:09.835 | INFO | "huge_shared" : " 27 GB", | 20260505 13:07:09.836 | INFO | "malloc" : " 14 GB", | 20260505 13:07:09.836 | INFO | "pinned" : " 29 GB" | 20260505 13:07:09.836 | INFO | } | 20260505 13:07:09.836 | INFO | Perf report for the last 5.13986 seconds: | 20260505 13:07:09.836 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:07:09.836 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:09.836 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:09.836 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:09.836 | INFO | ANALYSIS : 64.7%, 3.0, 100.3ms, 0 | 20260505 13:07:09.836 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:09.836 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260505 13:07:09.836 | INFO | PRE_HQ : 60.8%, 1.8, 57.6ms, 0 | 20260505 13:07:09.836 | INFO | BAZWRITER : 65.0%, 4.0, 134.2ms, 0 | 20260505 13:07:09.836 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 13:07:09.836 | INFO | Uploaded 36GiB to GPU in 2.18793s (16.4539GiB/s) | 20260505 13:07:09.836 | INFO | Downloaded 38GiB to GPU in 2.27538s (16.7005GiB/s) | 20260505 13:07:14.987 | INFO | Analyzing chunk frames = [3441152,3441664) | 20260505 13:07:14.987 | INFO | MemInfo: { | 20260505 13:07:14.987 | INFO | "gpu" : " 17 GB", | 20260505 13:07:14.987 | INFO | "huge" : "", | 20260505 13:07:14.987 | INFO | "huge_shared" : " 27 GB", | 20260505 13:07:14.987 | INFO | "malloc" : " 14 GB", | 20260505 13:07:14.987 | INFO | "pinned" : " 29 GB" | 20260505 13:07:14.987 | INFO | } | 20260505 13:07:14.987 | INFO | Perf report for the last 5.15142 seconds: | 20260505 13:07:14.987 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:07:14.987 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:14.987 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:14.987 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:14.987 | INFO | ANALYSIS : 63.3%, 3.0, 99.1ms, 0 | 20260505 13:07:14.987 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:14.987 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:14.987 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 13:07:14.987 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:14.987 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:14.987 | INFO | Uploaded 36GiB to GPU in 2.18391s (16.4842GiB/s) | 20260505 13:07:14.987 | INFO | Downloaded 34GiB to GPU in 2.02752s (16.7693GiB/s) | 20260505 13:07:20.121 | INFO | Analyzing chunk frames = [3441664,3442176) | 20260505 13:07:20.121 | INFO | MemInfo: { | 20260505 13:07:20.121 | INFO | "gpu" : " 17 GB", | 20260505 13:07:20.121 | INFO | "huge" : "", | 20260505 13:07:20.121 | INFO | "huge_shared" : " 27 GB", | 20260505 13:07:20.121 | INFO | "malloc" : " 14 GB", | 20260505 13:07:20.121 | INFO | "pinned" : " 29 GB" | 20260505 13:07:20.121 | INFO | } | 20260505 13:07:20.121 | INFO | Perf report for the last 5.13423 seconds: | 20260505 13:07:20.121 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:07:20.121 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:20.121 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:20.121 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:20.121 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260505 13:07:20.121 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:20.121 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:20.121 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:07:20.121 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:20.121 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:20.122 | INFO | Uploaded 36GiB to GPU in 2.2356s (16.1031GiB/s) | 20260505 13:07:20.122 | INFO | Downloaded 34GiB to GPU in 2.09155s (16.2559GiB/s) | 20260505 13:07:25.247 | INFO | Analyzing chunk frames = [3442176,3442688) | 20260505 13:07:25.247 | INFO | MemInfo: { | 20260505 13:07:25.247 | INFO | "gpu" : " 17 GB", | 20260505 13:07:25.247 | INFO | "huge" : "", | 20260505 13:07:25.247 | INFO | "huge_shared" : " 27 GB", | 20260505 13:07:25.247 | INFO | "malloc" : " 14 GB", | 20260505 13:07:25.247 | INFO | "pinned" : " 29 GB" | 20260505 13:07:25.247 | INFO | } | 20260505 13:07:25.247 | INFO | Perf report for the last 5.12548 seconds: | 20260505 13:07:25.247 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:07:25.247 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:25.247 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:25.247 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:25.247 | INFO | ANALYSIS : 63.5%, 3.0, 98.3ms, 0 | 20260505 13:07:25.247 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:25.247 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:25.247 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:07:25.247 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:25.247 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:25.247 | INFO | Uploaded 36GiB to GPU in 2.21504s (16.2526GiB/s) | 20260505 13:07:25.247 | INFO | Downloaded 34GiB to GPU in 2.13293s (15.9405GiB/s) | 20260505 13:07:30.389 | INFO | Analyzing chunk frames = [3442688,3443200) | 20260505 13:07:30.390 | INFO | MemInfo: { | 20260505 13:07:30.390 | INFO | "gpu" : " 17 GB", | 20260505 13:07:30.390 | INFO | "huge" : "", | 20260505 13:07:30.390 | INFO | "huge_shared" : " 27 GB", | 20260505 13:07:30.390 | INFO | "malloc" : " 14 GB", | 20260505 13:07:30.390 | INFO | "pinned" : " 29 GB" | 20260505 13:07:30.390 | INFO | } | 20260505 13:07:30.390 | INFO | Perf report for the last 5.14298 seconds: | 20260505 13:07:30.390 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:07:30.390 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:30.390 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:30.390 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:30.390 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 13:07:30.390 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:30.390 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:30.390 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 13:07:30.390 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:30.390 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:30.390 | INFO | Uploaded 36GiB to GPU in 2.19614s (16.3924GiB/s) | 20260505 13:07:30.390 | INFO | Downloaded 34GiB to GPU in 2.1333s (15.9377GiB/s) | 20260505 13:07:35.535 | INFO | Analyzing chunk frames = [3443200,3443712) | 20260505 13:07:35.535 | INFO | MemInfo: { | 20260505 13:07:35.535 | INFO | "gpu" : " 17 GB", | 20260505 13:07:35.535 | INFO | "huge" : "", | 20260505 13:07:35.535 | INFO | "huge_shared" : " 27 GB", | 20260505 13:07:35.535 | INFO | "malloc" : " 14 GB", | 20260505 13:07:35.535 | INFO | "pinned" : " 29 GB" | 20260505 13:07:35.535 | INFO | } | 20260505 13:07:35.535 | INFO | Perf report for the last 5.14546 seconds: | 20260505 13:07:35.535 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:07:35.535 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:35.535 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:35.535 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:35.535 | INFO | ANALYSIS : 63.1%, 3.0, 98.7ms, 0 | 20260505 13:07:35.535 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:35.535 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:35.535 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 13:07:35.535 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:35.535 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:35.536 | INFO | Uploaded 36GiB to GPU in 2.17595s (16.5445GiB/s) | 20260505 13:07:35.536 | INFO | Downloaded 34GiB to GPU in 2.11123s (16.1044GiB/s) | 20260505 13:07:40.682 | INFO | Analyzing chunk frames = [3443712,3444224) | 20260505 13:07:40.682 | INFO | MemInfo: { | 20260505 13:07:40.682 | INFO | "gpu" : " 17 GB", | 20260505 13:07:40.682 | INFO | "huge" : "", | 20260505 13:07:40.682 | INFO | "huge_shared" : " 27 GB", | 20260505 13:07:40.682 | INFO | "malloc" : " 14 GB", | 20260505 13:07:40.682 | INFO | "pinned" : " 29 GB" | 20260505 13:07:40.682 | INFO | } | 20260505 13:07:40.682 | INFO | Perf report for the last 5.14698 seconds: | 20260505 13:07:40.682 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:07:40.682 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:40.682 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:40.682 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:40.682 | INFO | ANALYSIS : 63.3%, 3.0, 99.0ms, 0 | 20260505 13:07:40.682 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:40.682 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:40.682 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 13:07:40.682 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:40.682 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:40.683 | INFO | Uploaded 36GiB to GPU in 2.14332s (16.7964GiB/s) | 20260505 13:07:40.683 | INFO | Downloaded 34GiB to GPU in 2.1701s (15.6675GiB/s) | 20260505 13:07:45.811 | INFO | Analyzing chunk frames = [3444224,3444736) | 20260505 13:07:45.811 | INFO | MemInfo: { | 20260505 13:07:45.811 | INFO | "gpu" : " 17 GB", | 20260505 13:07:45.811 | INFO | "huge" : "", | 20260505 13:07:45.811 | INFO | "huge_shared" : " 27 GB", | 20260505 13:07:45.811 | INFO | "malloc" : " 14 GB", | 20260505 13:07:45.811 | INFO | "pinned" : " 29 GB" | 20260505 13:07:45.811 | INFO | } | 20260505 13:07:45.811 | INFO | Perf report for the last 5.1289 seconds: | 20260505 13:07:45.811 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:07:45.811 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:45.811 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:45.811 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:45.811 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 13:07:45.811 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:45.811 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:45.811 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 13:07:45.811 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:45.812 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:45.812 | INFO | Uploaded 36GiB to GPU in 2.21305s (16.2671GiB/s) | 20260505 13:07:45.812 | INFO | Downloaded 34GiB to GPU in 2.10705s (16.1363GiB/s) | 20260505 13:07:45.945 | INFO | Starting a round of Baz IO | 20260505 13:07:49.189 | INFO | Populating RTMetrics report | 20260505 13:07:49.189 | INFO | RTMetrics report populated | 20260505 13:07:49.189 | INFO | Opening tmp RTMetrics json file | 20260505 13:07:49.191 | INFO | Serializing RTMetrics to json | 20260505 13:07:49.192 | INFO | Serialization done, swapping tmp file now... | 20260505 13:07:49.192 | INFO | RTMetrics done | 20260505 13:07:49.309 | INFO | Finished a round of Baz IO: Wrote 2.313 GiB in 3.363 seconds | 20260505 13:07:49.309 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.25% / 0.01% / 76.74% | 20260505 13:07:49.309 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.758807% / 7.94729e-06% / 7.94729e-05% | 20260505 13:07:49.309 | INFO | Baz IO overall: | 20260505 13:07:49.309 | INFO | BytesWritten : 2050680157056 | 20260505 13:07:49.309 | INFO | NumEvents : 975808117718 | 20260505 13:07:50.952 | INFO | Analyzing chunk frames = [3444736,3445248) | 20260505 13:07:50.952 | INFO | MemInfo: { | 20260505 13:07:50.952 | INFO | "gpu" : " 17 GB", | 20260505 13:07:50.952 | INFO | "huge" : "", | 20260505 13:07:50.952 | INFO | "huge_shared" : " 27 GB", | 20260505 13:07:50.952 | INFO | "malloc" : " 14 GB", | 20260505 13:07:50.953 | INFO | "pinned" : " 29 GB" | 20260505 13:07:50.953 | INFO | } | 20260505 13:07:50.953 | INFO | Perf report for the last 5.14143 seconds: | 20260505 13:07:50.953 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:07:50.953 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:50.953 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:50.953 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:50.953 | INFO | ANALYSIS : 64.6%, 2.9, 99.8ms, 0 | 20260505 13:07:50.953 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:50.953 | INFO | RT_METRICS : 45.3%, 1.0, 23.8ms, 0 | 20260505 13:07:50.953 | INFO | PRE_HQ : 60.6%, 1.8, 57.9ms, 0 | 20260505 13:07:50.953 | INFO | BAZWRITER : 64.8%, 4.0, 134.6ms, 0 | 20260505 13:07:50.953 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 13:07:50.953 | INFO | Uploaded 36GiB to GPU in 2.17814s (16.5279GiB/s) | 20260505 13:07:50.953 | INFO | Downloaded 38GiB to GPU in 2.28561s (16.6258GiB/s) | 20260505 13:07:56.093 | INFO | Analyzing chunk frames = [3445248,3445760) | 20260505 13:07:56.093 | INFO | MemInfo: { | 20260505 13:07:56.093 | INFO | "gpu" : " 17 GB", | 20260505 13:07:56.093 | INFO | "huge" : "", | 20260505 13:07:56.093 | INFO | "huge_shared" : " 27 GB", | 20260505 13:07:56.093 | INFO | "malloc" : " 14 GB", | 20260505 13:07:56.093 | INFO | "pinned" : " 29 GB" | 20260505 13:07:56.093 | INFO | } | 20260505 13:07:56.093 | INFO | Perf report for the last 5.14034 seconds: | 20260505 13:07:56.093 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:07:56.093 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:56.093 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:56.093 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:56.093 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 13:07:56.093 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:56.093 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:56.093 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 13:07:56.093 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:56.093 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:07:56.093 | INFO | Uploaded 36GiB to GPU in 2.18011s (16.513GiB/s) | 20260505 13:07:56.093 | INFO | Downloaded 34GiB to GPU in 2.04927s (16.5913GiB/s) | 20260505 13:08:01.232 | INFO | Analyzing chunk frames = [3445760,3446272) | 20260505 13:08:01.232 | INFO | MemInfo: { | 20260505 13:08:01.232 | INFO | "gpu" : " 17 GB", | 20260505 13:08:01.232 | INFO | "huge" : "", | 20260505 13:08:01.232 | INFO | "huge_shared" : " 27 GB", | 20260505 13:08:01.232 | INFO | "malloc" : " 14 GB", | 20260505 13:08:01.232 | INFO | "pinned" : " 29 GB" | 20260505 13:08:01.232 | INFO | } | 20260505 13:08:01.232 | INFO | Perf report for the last 5.13905 seconds: | 20260505 13:08:01.232 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:08:01.232 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:01.232 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:01.232 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:01.232 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 13:08:01.232 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:01.232 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:01.232 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:08:01.232 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:01.232 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:01.232 | INFO | Uploaded 36GiB to GPU in 2.20534s (16.324GiB/s) | 20260505 13:08:01.232 | INFO | Downloaded 34GiB to GPU in 2.1212s (16.0286GiB/s) | 20260505 13:08:06.369 | INFO | Analyzing chunk frames = [3446272,3446784) | 20260505 13:08:06.369 | INFO | MemInfo: { | 20260505 13:08:06.369 | INFO | "gpu" : " 17 GB", | 20260505 13:08:06.369 | INFO | "huge" : "", | 20260505 13:08:06.369 | INFO | "huge_shared" : " 27 GB", | 20260505 13:08:06.369 | INFO | "malloc" : " 14 GB", | 20260505 13:08:06.369 | INFO | "pinned" : " 29 GB" | 20260505 13:08:06.369 | INFO | } | 20260505 13:08:06.370 | INFO | Perf report for the last 5.13751 seconds: | 20260505 13:08:06.370 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:08:06.370 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:06.370 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:06.370 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:06.370 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 13:08:06.370 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:06.370 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:06.370 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:08:06.370 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:06.370 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:06.370 | INFO | Uploaded 36GiB to GPU in 2.20342s (16.3382GiB/s) | 20260505 13:08:06.370 | INFO | Downloaded 34GiB to GPU in 2.12989s (15.9633GiB/s) | 20260505 13:08:11.502 | INFO | Analyzing chunk frames = [3446784,3447296) | 20260505 13:08:11.502 | INFO | MemInfo: { | 20260505 13:08:11.502 | INFO | "gpu" : " 17 GB", | 20260505 13:08:11.502 | INFO | "huge" : "", | 20260505 13:08:11.502 | INFO | "huge_shared" : " 27 GB", | 20260505 13:08:11.502 | INFO | "malloc" : " 14 GB", | 20260505 13:08:11.503 | INFO | "pinned" : " 29 GB" | 20260505 13:08:11.503 | INFO | } | 20260505 13:08:11.503 | INFO | Perf report for the last 5.13323 seconds: | 20260505 13:08:11.503 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:08:11.503 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:11.503 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:11.503 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:11.503 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 13:08:11.503 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:11.503 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:11.503 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:08:11.503 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:11.503 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:11.503 | INFO | Uploaded 36GiB to GPU in 2.18028s (16.5116GiB/s) | 20260505 13:08:11.503 | INFO | Downloaded 34GiB to GPU in 2.12802s (15.9773GiB/s) | 20260505 13:08:16.650 | INFO | Analyzing chunk frames = [3447296,3447808) | 20260505 13:08:16.650 | INFO | MemInfo: { | 20260505 13:08:16.650 | INFO | "gpu" : " 17 GB", | 20260505 13:08:16.650 | INFO | "huge" : "", | 20260505 13:08:16.650 | INFO | "huge_shared" : " 27 GB", | 20260505 13:08:16.650 | INFO | "malloc" : " 14 GB", | 20260505 13:08:16.650 | INFO | "pinned" : " 29 GB" | 20260505 13:08:16.650 | INFO | } | 20260505 13:08:16.650 | INFO | Perf report for the last 5.14717 seconds: | 20260505 13:08:16.650 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:08:16.650 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:16.650 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:16.650 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:16.650 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 13:08:16.650 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:16.650 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:16.650 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 13:08:16.650 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:16.650 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:16.650 | INFO | Uploaded 36GiB to GPU in 2.17303s (16.5667GiB/s) | 20260505 13:08:16.650 | INFO | Downloaded 34GiB to GPU in 2.11028s (16.1116GiB/s) | 20260505 13:08:21.789 | INFO | Analyzing chunk frames = [3447808,3448320) | 20260505 13:08:21.789 | INFO | MemInfo: { | 20260505 13:08:21.789 | INFO | "gpu" : " 17 GB", | 20260505 13:08:21.789 | INFO | "huge" : "", | 20260505 13:08:21.789 | INFO | "huge_shared" : " 27 GB", | 20260505 13:08:21.789 | INFO | "malloc" : " 14 GB", | 20260505 13:08:21.789 | INFO | "pinned" : " 29 GB" | 20260505 13:08:21.789 | INFO | } | 20260505 13:08:21.789 | INFO | Perf report for the last 5.13902 seconds: | 20260505 13:08:21.789 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:08:21.789 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:21.789 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:21.789 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:21.789 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 13:08:21.789 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:21.789 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:21.789 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 13:08:21.789 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:21.789 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:21.789 | INFO | Uploaded 36GiB to GPU in 2.14312s (16.7979GiB/s) | 20260505 13:08:21.789 | INFO | Downloaded 34GiB to GPU in 2.18195s (15.5824GiB/s) | 20260505 13:08:26.932 | INFO | Analyzing chunk frames = [3448320,3448832) | 20260505 13:08:26.932 | INFO | MemInfo: { | 20260505 13:08:26.932 | INFO | "gpu" : " 17 GB", | 20260505 13:08:26.932 | INFO | "huge" : "", | 20260505 13:08:26.932 | INFO | "huge_shared" : " 27 GB", | 20260505 13:08:26.932 | INFO | "malloc" : " 14 GB", | 20260505 13:08:26.932 | INFO | "pinned" : " 29 GB" | 20260505 13:08:26.932 | INFO | } | 20260505 13:08:26.933 | INFO | Perf report for the last 5.14354 seconds: | 20260505 13:08:26.933 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:08:26.933 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:26.933 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:26.933 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:26.933 | INFO | ANALYSIS : 63.5%, 3.0, 99.3ms, 0 | 20260505 13:08:26.933 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:26.933 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:26.933 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:08:26.933 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:26.933 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:26.933 | INFO | Uploaded 36GiB to GPU in 2.1584s (16.679GiB/s) | 20260505 13:08:26.933 | INFO | Downloaded 34GiB to GPU in 2.10424s (16.1578GiB/s) | 20260505 13:08:26.933 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 13:08:26.933 | INFO | Total mem usage: 15.3 GiB | 20260505 13:08:26.933 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 13:08:26.933 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -7.06%) | 20260505 13:08:26.933 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 13:08:26.933 | INFO | High Water Breakdown: | 20260505 13:08:26.933 | INFO | 61.1% : 8.75 GiB ( 92% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 13:08:26.933 | INFO | 38.7% : 5.54 GiB ( 83.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 13:08:26.933 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 13:08:26.933 | INFO | ------------------------------------------ | 20260505 13:08:26.933 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:08:26.933 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 13:08:26.933 | INFO | Total mem usage: 26.5 GiB | 20260505 13:08:26.933 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 13:08:26.933 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 13:08:26.933 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 13:08:26.933 | INFO | High Water Breakdown: | 20260505 13:08:26.933 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:08:26.933 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:08:26.933 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:08:26.933 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:08:26.933 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:08:26.933 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:08:26.933 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 13:08:26.933 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 13:08:26.933 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:08:26.933 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 13:08:26.933 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 13:08:26.933 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 13:08:26.933 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 13:08:26.934 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 13:08:26.934 | INFO | ------------------------------------------ | 20260505 13:08:26.934 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:08:26.934 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 13:08:26.934 | INFO | Total mem usage: 25.6 GiB | 20260505 13:08:26.934 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 13:08:26.934 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 13:08:26.934 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 13:08:26.934 | INFO | High Water Breakdown: | 20260505 13:08:26.934 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 13:08:26.934 | INFO | ------------------------------------------ | 20260505 13:08:26.934 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:08:26.934 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 13:08:26.934 | INFO | Total mem usage: 15.7 GiB | 20260505 13:08:26.934 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 13:08:26.934 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 13:08:26.934 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 13:08:26.934 | INFO | High Water Breakdown: | 20260505 13:08:26.934 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 13:08:26.934 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 13:08:26.934 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 13:08:26.934 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 13:08:26.934 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 13:08:26.934 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 13:08:26.934 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 13:08:26.934 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 13:08:26.934 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 13:08:26.934 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:08:26.934 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 13:08:26.934 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:08:26.934 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 13:08:26.934 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 13:08:26.934 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 13:08:26.934 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 13:08:26.934 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:08:26.934 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 13:08:26.934 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:08:26.934 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:08:26.934 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:08:26.934 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 13:08:26.934 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 13:08:26.934 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:08:26.934 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 13:08:26.934 | INFO | ------------------------------------------ | 20260505 13:08:26.934 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:08:26.934 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 13:08:26.934 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:08:26.934 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:08:26.934 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.70e+01, 1.39e+01 / 1.85e+00 | 20260505 13:08:26.934 | INFO | PerfReport: 0.29%, AggregateStats -- 9.27e-02 / 9.85e-02, 1.04e-02 / 3.65e-03 | 20260505 13:08:26.934 | INFO | PerfReport: 11.75%, Baseline -- 3.75e+00 / 4.48e+00, 5.90e-01 / 1.79e-01 | 20260505 13:08:26.934 | INFO | PerfReport: 3.31%, Histogram -- 1.05e+00 / 1.49e+00, 2.01e-01 / 1.03e-01 | 20260505 13:08:26.934 | INFO | PerfReport: 16.08%, DME -- 4.11e+01 / 4.63e+01, 4.82e+00 / 1.94e+00 | 20260505 13:08:26.934 | INFO | PerfReport: 56.27%, FrameLabeling -- 1.80e+01 / 1.97e+01, 1.39e+00 / 1.28e+00 | 20260505 13:08:26.934 | INFO | PerfReport: 7.44%, PulseAccumulating -- 2.37e+00 / 2.95e+00, 4.50e-01 / 1.50e-01 | 20260505 13:08:26.934 | INFO | PerfReport: 3.59%, Metrics -- 1.14e+00 / 1.48e+00, 3.21e-01 / 1.58e-01 | 20260505 13:08:26.934 | INFO | PerfReport: 1.26%, PulseSerializing -- 4.03e-01 / 4.51e-01, 7.29e-02 / 3.13e-02 | 20260505 13:08:26.934 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 13:08:26.934 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:08:26.934 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:08:26.934 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.57e+01 / 0.00e+00 | 20260505 13:08:26.934 | INFO | PerfReport: 22.53%, Upload -- 2.28e+01 / 0.00e+00, 8.08e+00 / 0.00e+00 | 20260505 13:08:26.934 | INFO | PerfReport: 22.01%, Download -- 2.22e+01 / 0.00e+00, 8.12e+00 / 0.00e+00 | 20260505 13:08:26.934 | INFO | PerfReport: 31.57%, Compute -- 3.19e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 13:08:26.934 | INFO | PerfReport: 2.32%, LockUpload -- 2.35e+00 / 0.00e+00, 8.04e+00 / 0.00e+00 | 20260505 13:08:26.934 | INFO | PerfReport: 1.39%, LockDownload -- 1.41e+00 / 0.00e+00, 2.98e+00 / 0.00e+00 | 20260505 13:08:26.934 | INFO | PerfReport: 20.16%, LockCompute -- 2.04e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 13:08:26.934 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 2.02e-02 / 0.00e+00 | 20260505 13:08:27.071 | INFO | Starting a round of Baz IO | 20260505 13:08:30.294 | INFO | Populating RTMetrics report | 20260505 13:08:30.294 | INFO | RTMetrics report populated | 20260505 13:08:30.295 | INFO | Opening tmp RTMetrics json file | 20260505 13:08:30.300 | INFO | Serializing RTMetrics to json | 20260505 13:08:30.328 | INFO | Serialization done, swapping tmp file now... | 20260505 13:08:30.342 | INFO | RTMetrics done | 20260505 13:08:30.403 | INFO | Finished a round of Baz IO: Wrote 2.31 GiB in 3.332 seconds | 20260505 13:08:30.403 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.25% / 0.00% / 76.75% | 20260505 13:08:30.403 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.752846% / 3.97364e-06% / 7.54992e-05% | 20260505 13:08:30.403 | INFO | Baz IO overall: | 20260505 13:08:30.403 | INFO | BytesWritten : 2053160516985 | 20260505 13:08:30.403 | INFO | NumEvents : 976982005878 | 20260505 13:08:32.069 | INFO | Analyzing chunk frames = [3448832,3449344) | 20260505 13:08:32.069 | INFO | MemInfo: { | 20260505 13:08:32.069 | INFO | "gpu" : " 17 GB", | 20260505 13:08:32.069 | INFO | "huge" : "", | 20260505 13:08:32.069 | INFO | "huge_shared" : " 27 GB", | 20260505 13:08:32.069 | INFO | "malloc" : " 14 GB", | 20260505 13:08:32.069 | INFO | "pinned" : " 29 GB" | 20260505 13:08:32.069 | INFO | } | 20260505 13:08:32.069 | INFO | Perf report for the last 5.13666 seconds: | 20260505 13:08:32.069 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:08:32.069 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:32.069 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:32.069 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:32.069 | INFO | ANALYSIS : 64.3%, 3.0, 99.7ms, 0 | 20260505 13:08:32.069 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:32.069 | INFO | RT_METRICS : 46.1%, 1.0, 24.1ms, 0 | 20260505 13:08:32.069 | INFO | PRE_HQ : 61.0%, 1.8, 57.8ms, 0 | 20260505 13:08:32.069 | INFO | BAZWRITER : 64.2%, 4.0, 133.2ms, 0 | 20260505 13:08:32.069 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 13:08:32.070 | INFO | Uploaded 36GiB to GPU in 2.16156s (16.6547GiB/s) | 20260505 13:08:32.070 | INFO | Downloaded 38GiB to GPU in 2.27518s (16.702GiB/s) | 20260505 13:08:37.195 | INFO | Analyzing chunk frames = [3449344,3449856) | 20260505 13:08:37.195 | INFO | MemInfo: { | 20260505 13:08:37.195 | INFO | "gpu" : " 17 GB", | 20260505 13:08:37.195 | INFO | "huge" : "", | 20260505 13:08:37.195 | INFO | "huge_shared" : " 27 GB", | 20260505 13:08:37.195 | INFO | "malloc" : " 14 GB", | 20260505 13:08:37.195 | INFO | "pinned" : " 29 GB" | 20260505 13:08:37.195 | INFO | } | 20260505 13:08:37.195 | INFO | Perf report for the last 5.12619 seconds: | 20260505 13:08:37.195 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:08:37.195 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:37.195 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:37.195 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:37.195 | INFO | ANALYSIS : 64.1%, 3.0, 99.9ms, 0 | 20260505 13:08:37.196 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:37.196 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:37.196 | INFO | PRE_HQ : 28.5%, 1.0, 14.7ms, 0 | 20260505 13:08:37.196 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:37.196 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:37.196 | INFO | Uploaded 36GiB to GPU in 2.20525s (16.3246GiB/s) | 20260505 13:08:37.196 | INFO | Downloaded 34GiB to GPU in 2.06232s (16.4863GiB/s) | 20260505 13:08:42.345 | INFO | Analyzing chunk frames = [3449856,3450368) | 20260505 13:08:42.345 | INFO | MemInfo: { | 20260505 13:08:42.345 | INFO | "gpu" : " 17 GB", | 20260505 13:08:42.345 | INFO | "huge" : "", | 20260505 13:08:42.345 | INFO | "huge_shared" : " 27 GB", | 20260505 13:08:42.345 | INFO | "malloc" : " 14 GB", | 20260505 13:08:42.345 | INFO | "pinned" : " 29 GB" | 20260505 13:08:42.345 | INFO | } | 20260505 13:08:42.345 | INFO | Perf report for the last 5.15002 seconds: | 20260505 13:08:42.346 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:08:42.346 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:42.346 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:42.346 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:42.346 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 13:08:42.346 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:42.346 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:42.346 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:08:42.346 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:42.346 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:42.346 | INFO | Uploaded 36GiB to GPU in 2.22739s (16.1624GiB/s) | 20260505 13:08:42.346 | INFO | Downloaded 34GiB to GPU in 2.12804s (15.9771GiB/s) | 20260505 13:08:47.489 | INFO | Analyzing chunk frames = [3450368,3450880) | 20260505 13:08:47.489 | INFO | MemInfo: { | 20260505 13:08:47.489 | INFO | "gpu" : " 17 GB", | 20260505 13:08:47.489 | INFO | "huge" : "", | 20260505 13:08:47.489 | INFO | "huge_shared" : " 27 GB", | 20260505 13:08:47.489 | INFO | "malloc" : " 14 GB", | 20260505 13:08:47.489 | INFO | "pinned" : " 29 GB" | 20260505 13:08:47.489 | INFO | } | 20260505 13:08:47.489 | INFO | Perf report for the last 5.14364 seconds: | 20260505 13:08:47.489 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:08:47.489 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:47.489 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:47.489 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:47.489 | INFO | ANALYSIS : 63.1%, 3.0, 98.1ms, 0 | 20260505 13:08:47.489 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:47.489 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:47.489 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:08:47.489 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:47.489 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:47.490 | INFO | Uploaded 36GiB to GPU in 2.20581s (16.3206GiB/s) | 20260505 13:08:47.490 | INFO | Downloaded 34GiB to GPU in 2.13433s (15.9301GiB/s) | 20260505 13:08:52.624 | INFO | Analyzing chunk frames = [3450880,3451392) | 20260505 13:08:52.624 | INFO | MemInfo: { | 20260505 13:08:52.624 | INFO | "gpu" : " 17 GB", | 20260505 13:08:52.624 | INFO | "huge" : "", | 20260505 13:08:52.624 | INFO | "huge_shared" : " 27 GB", | 20260505 13:08:52.624 | INFO | "malloc" : " 14 GB", | 20260505 13:08:52.624 | INFO | "pinned" : " 29 GB" | 20260505 13:08:52.624 | INFO | } | 20260505 13:08:52.624 | INFO | Perf report for the last 5.13521 seconds: | 20260505 13:08:52.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:08:52.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:52.625 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:52.625 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:52.625 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 13:08:52.625 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:52.625 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:52.625 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 13:08:52.625 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:52.625 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:52.625 | INFO | Uploaded 36GiB to GPU in 2.17279s (16.5685GiB/s) | 20260505 13:08:52.625 | INFO | Downloaded 34GiB to GPU in 2.12335s (16.0124GiB/s) | 20260505 13:08:57.759 | INFO | Analyzing chunk frames = [3451392,3451904) | 20260505 13:08:57.759 | INFO | MemInfo: { | 20260505 13:08:57.759 | INFO | "gpu" : " 17 GB", | 20260505 13:08:57.759 | INFO | "huge" : "", | 20260505 13:08:57.759 | INFO | "huge_shared" : " 27 GB", | 20260505 13:08:57.759 | INFO | "malloc" : " 14 GB", | 20260505 13:08:57.759 | INFO | "pinned" : " 29 GB" | 20260505 13:08:57.759 | INFO | } | 20260505 13:08:57.759 | INFO | Perf report for the last 5.13438 seconds: | 20260505 13:08:57.759 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:08:57.759 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:57.759 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:57.759 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:57.759 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 13:08:57.759 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:57.759 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:57.759 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 13:08:57.759 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:57.759 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:08:57.759 | INFO | Uploaded 36GiB to GPU in 2.169s (16.5975GiB/s) | 20260505 13:08:57.759 | INFO | Downloaded 34GiB to GPU in 2.11105s (16.1057GiB/s) | 20260505 13:09:02.907 | INFO | Analyzing chunk frames = [3451904,3452416) | 20260505 13:09:02.907 | INFO | MemInfo: { | 20260505 13:09:02.907 | INFO | "gpu" : " 17 GB", | 20260505 13:09:02.907 | INFO | "huge" : "", | 20260505 13:09:02.907 | INFO | "huge_shared" : " 27 GB", | 20260505 13:09:02.907 | INFO | "malloc" : " 14 GB", | 20260505 13:09:02.907 | INFO | "pinned" : " 29 GB" | 20260505 13:09:02.907 | INFO | } | 20260505 13:09:02.908 | INFO | Perf report for the last 5.1489 seconds: | 20260505 13:09:02.908 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:09:02.908 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:02.908 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:02.908 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:02.908 | INFO | ANALYSIS : 63.1%, 3.0, 98.7ms, 0 | 20260505 13:09:02.908 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:02.908 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:02.908 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 13:09:02.908 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:02.908 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:02.908 | INFO | Uploaded 36GiB to GPU in 2.18506s (16.4755GiB/s) | 20260505 13:09:02.908 | INFO | Downloaded 34GiB to GPU in 2.15632s (15.7676GiB/s) | 20260505 13:09:08.044 | INFO | Analyzing chunk frames = [3452416,3452928) | 20260505 13:09:08.044 | INFO | MemInfo: { | 20260505 13:09:08.044 | INFO | "gpu" : " 17 GB", | 20260505 13:09:08.044 | INFO | "huge" : "", | 20260505 13:09:08.044 | INFO | "huge_shared" : " 27 GB", | 20260505 13:09:08.044 | INFO | "malloc" : " 14 GB", | 20260505 13:09:08.044 | INFO | "pinned" : " 29 GB" | 20260505 13:09:08.044 | INFO | } | 20260505 13:09:08.044 | INFO | Perf report for the last 5.13676 seconds: | 20260505 13:09:08.044 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:09:08.044 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:08.044 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:08.044 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:08.044 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 13:09:08.044 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:08.044 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:08.045 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 13:09:08.045 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:08.045 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:08.045 | INFO | Uploaded 36GiB to GPU in 2.14923s (16.7501GiB/s) | 20260505 13:09:08.045 | INFO | Downloaded 34GiB to GPU in 2.10441s (16.1566GiB/s) | 20260505 13:09:08.179 | INFO | Starting a round of Baz IO | 20260505 13:09:11.416 | INFO | Populating RTMetrics report | 20260505 13:09:11.417 | INFO | RTMetrics report populated | 20260505 13:09:11.417 | INFO | Opening tmp RTMetrics json file | 20260505 13:09:11.434 | INFO | Serializing RTMetrics to json | 20260505 13:09:11.435 | INFO | Serialization done, swapping tmp file now... | 20260505 13:09:11.435 | INFO | RTMetrics done | 20260505 13:09:11.537 | INFO | Finished a round of Baz IO: Wrote 2.309 GiB in 3.358 seconds | 20260505 13:09:11.537 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.24% / 0.00% / 76.75% | 20260505 13:09:11.537 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.750065% / 7.94729e-06% / 7.15256e-05% | 20260505 13:09:11.537 | INFO | Baz IO overall: | 20260505 13:09:11.537 | INFO | BytesWritten : 2055639579766 | 20260505 13:09:11.537 | INFO | NumEvents : 978155014889 | 20260505 13:09:13.181 | INFO | Analyzing chunk frames = [3452928,3453440) | 20260505 13:09:13.181 | INFO | MemInfo: { | 20260505 13:09:13.181 | INFO | "gpu" : " 17 GB", | 20260505 13:09:13.181 | INFO | "huge" : "", | 20260505 13:09:13.181 | INFO | "huge_shared" : " 27 GB", | 20260505 13:09:13.181 | INFO | "malloc" : " 14 GB", | 20260505 13:09:13.181 | INFO | "pinned" : " 29 GB" | 20260505 13:09:13.181 | INFO | } | 20260505 13:09:13.181 | INFO | Perf report for the last 5.13667 seconds: | 20260505 13:09:13.181 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:09:13.181 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:13.181 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:13.181 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:13.181 | INFO | ANALYSIS : 64.5%, 3.0, 99.8ms, 0 | 20260505 13:09:13.181 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:13.181 | INFO | RT_METRICS : 45.5%, 1.0, 23.8ms, 0 | 20260505 13:09:13.181 | INFO | PRE_HQ : 61.1%, 1.8, 57.7ms, 0 | 20260505 13:09:13.181 | INFO | BAZWRITER : 64.7%, 4.0, 134.1ms, 0 | 20260505 13:09:13.181 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 13:09:13.181 | INFO | Uploaded 36GiB to GPU in 2.20371s (16.3361GiB/s) | 20260505 13:09:13.182 | INFO | Downloaded 38GiB to GPU in 2.27354s (16.714GiB/s) | 20260505 13:09:18.321 | INFO | Analyzing chunk frames = [3453440,3453952) | 20260505 13:09:18.321 | INFO | MemInfo: { | 20260505 13:09:18.321 | INFO | "gpu" : " 17 GB", | 20260505 13:09:18.321 | INFO | "huge" : "", | 20260505 13:09:18.321 | INFO | "huge_shared" : " 27 GB", | 20260505 13:09:18.321 | INFO | "malloc" : " 14 GB", | 20260505 13:09:18.321 | INFO | "pinned" : " 29 GB" | 20260505 13:09:18.321 | INFO | } | 20260505 13:09:18.321 | INFO | Perf report for the last 5.14019 seconds: | 20260505 13:09:18.321 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:09:18.321 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:18.321 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:18.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:18.321 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 13:09:18.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:18.321 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:18.321 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 13:09:18.321 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:18.321 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:18.322 | INFO | Uploaded 36GiB to GPU in 2.1786s (16.5244GiB/s) | 20260505 13:09:18.322 | INFO | Downloaded 34GiB to GPU in 2.04696s (16.61GiB/s) | 20260505 13:09:23.460 | INFO | Analyzing chunk frames = [3453952,3454464) | 20260505 13:09:23.460 | INFO | MemInfo: { | 20260505 13:09:23.460 | INFO | "gpu" : " 17 GB", | 20260505 13:09:23.460 | INFO | "huge" : "", | 20260505 13:09:23.460 | INFO | "huge_shared" : " 27 GB", | 20260505 13:09:23.460 | INFO | "malloc" : " 14 GB", | 20260505 13:09:23.460 | INFO | "pinned" : " 29 GB" | 20260505 13:09:23.460 | INFO | } | 20260505 13:09:23.460 | INFO | Perf report for the last 5.13891 seconds: | 20260505 13:09:23.460 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:09:23.460 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:23.460 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:23.460 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:23.460 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 13:09:23.460 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:23.461 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:23.461 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 13:09:23.461 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:23.461 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:23.461 | INFO | Uploaded 36GiB to GPU in 2.22393s (16.1875GiB/s) | 20260505 13:09:23.461 | INFO | Downloaded 34GiB to GPU in 2.12284s (16.0163GiB/s) | 20260505 13:09:28.602 | INFO | Analyzing chunk frames = [3454464,3454976) | 20260505 13:09:28.602 | INFO | MemInfo: { | 20260505 13:09:28.602 | INFO | "gpu" : " 17 GB", | 20260505 13:09:28.602 | INFO | "huge" : "", | 20260505 13:09:28.602 | INFO | "huge_shared" : " 27 GB", | 20260505 13:09:28.602 | INFO | "malloc" : " 14 GB", | 20260505 13:09:28.602 | INFO | "pinned" : " 29 GB" | 20260505 13:09:28.602 | INFO | } | 20260505 13:09:28.602 | INFO | Perf report for the last 5.14216 seconds: | 20260505 13:09:28.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:09:28.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:28.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:28.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:28.602 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 13:09:28.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:28.602 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:28.602 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 13:09:28.602 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:28.602 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:28.603 | INFO | Uploaded 36GiB to GPU in 2.21202s (16.2747GiB/s) | 20260505 13:09:28.603 | INFO | Downloaded 34GiB to GPU in 2.13462s (15.9279GiB/s) | 20260505 13:09:33.744 | INFO | Analyzing chunk frames = [3454976,3455488) | 20260505 13:09:33.744 | INFO | MemInfo: { | 20260505 13:09:33.744 | INFO | "gpu" : " 17 GB", | 20260505 13:09:33.744 | INFO | "huge" : "", | 20260505 13:09:33.744 | INFO | "huge_shared" : " 27 GB", | 20260505 13:09:33.744 | INFO | "malloc" : " 14 GB", | 20260505 13:09:33.744 | INFO | "pinned" : " 29 GB" | 20260505 13:09:33.744 | INFO | } | 20260505 13:09:33.744 | INFO | Perf report for the last 5.1418 seconds: | 20260505 13:09:33.744 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:09:33.744 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:33.744 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:33.744 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:33.744 | INFO | ANALYSIS : 62.7%, 3.0, 98.0ms, 0 | 20260505 13:09:33.744 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:33.744 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:33.744 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:09:33.744 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:33.744 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:33.744 | INFO | Uploaded 36GiB to GPU in 2.14124s (16.8127GiB/s) | 20260505 13:09:33.745 | INFO | Downloaded 34GiB to GPU in 2.12232s (16.0202GiB/s) | 20260505 13:09:38.879 | INFO | Analyzing chunk frames = [3455488,3456000) | 20260505 13:09:38.879 | INFO | MemInfo: { | 20260505 13:09:38.879 | INFO | "gpu" : " 17 GB", | 20260505 13:09:38.879 | INFO | "huge" : "", | 20260505 13:09:38.879 | INFO | "huge_shared" : " 27 GB", | 20260505 13:09:38.879 | INFO | "malloc" : " 14 GB", | 20260505 13:09:38.879 | INFO | "pinned" : " 29 GB" | 20260505 13:09:38.879 | INFO | } | 20260505 13:09:38.879 | INFO | Perf report for the last 5.13489 seconds: | 20260505 13:09:38.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:09:38.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:38.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:38.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:38.879 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 13:09:38.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:38.879 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:38.879 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 13:09:38.879 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:38.879 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:38.880 | INFO | Uploaded 36GiB to GPU in 2.19695s (16.3863GiB/s) | 20260505 13:09:38.880 | INFO | Downloaded 34GiB to GPU in 2.1133s (16.0886GiB/s) | 20260505 13:09:44.014 | INFO | Analyzing chunk frames = [3456000,3456512) | 20260505 13:09:44.014 | INFO | MemInfo: { | 20260505 13:09:44.014 | INFO | "gpu" : " 17 GB", | 20260505 13:09:44.014 | INFO | "huge" : "", | 20260505 13:09:44.014 | INFO | "huge_shared" : " 27 GB", | 20260505 13:09:44.014 | INFO | "malloc" : " 14 GB", | 20260505 13:09:44.014 | INFO | "pinned" : " 29 GB" | 20260505 13:09:44.014 | INFO | } | 20260505 13:09:44.014 | INFO | Perf report for the last 5.13488 seconds: | 20260505 13:09:44.014 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:09:44.014 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:44.014 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:44.014 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:44.014 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 13:09:44.014 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:44.014 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:44.014 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 13:09:44.014 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:44.014 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:44.014 | INFO | Uploaded 36GiB to GPU in 2.17496s (16.552GiB/s) | 20260505 13:09:44.014 | INFO | Downloaded 34GiB to GPU in 2.13844s (15.8995GiB/s) | 20260505 13:09:49.153 | INFO | Analyzing chunk frames = [3456512,3457024) | 20260505 13:09:49.153 | INFO | MemInfo: { | 20260505 13:09:49.153 | INFO | "gpu" : " 17 GB", | 20260505 13:09:49.153 | INFO | "huge" : "", | 20260505 13:09:49.153 | INFO | "huge_shared" : " 27 GB", | 20260505 13:09:49.153 | INFO | "malloc" : " 14 GB", | 20260505 13:09:49.153 | INFO | "pinned" : " 29 GB" | 20260505 13:09:49.153 | INFO | } | 20260505 13:09:49.153 | INFO | Perf report for the last 5.13902 seconds: | 20260505 13:09:49.153 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:09:49.153 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:49.153 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:49.153 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:49.153 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 13:09:49.153 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:49.153 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:49.153 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:09:49.153 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:49.153 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:49.153 | INFO | Uploaded 36GiB to GPU in 2.2264s (16.1696GiB/s) | 20260505 13:09:49.153 | INFO | Downloaded 34GiB to GPU in 2.10742s (16.1335GiB/s) | 20260505 13:09:49.153 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 13:09:49.153 | INFO | Total mem usage: 15.3 GiB | 20260505 13:09:49.153 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 13:09:49.153 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.71%) | 20260505 13:09:49.153 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 13:09:49.153 | INFO | High Water Breakdown: | 20260505 13:09:49.153 | INFO | 61% : 8.68 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 13:09:49.154 | INFO | 38.9% : 5.53 GiB ( 83.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 13:09:49.154 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 13:09:49.154 | INFO | ------------------------------------------ | 20260505 13:09:49.154 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:09:49.154 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 13:09:49.154 | INFO | Total mem usage: 26.5 GiB | 20260505 13:09:49.154 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 13:09:49.154 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 13:09:49.154 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 13:09:49.154 | INFO | High Water Breakdown: | 20260505 13:09:49.154 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:09:49.154 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:09:49.154 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:09:49.154 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:09:49.154 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:09:49.154 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:09:49.154 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 13:09:49.154 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 13:09:49.154 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:09:49.154 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 13:09:49.154 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 13:09:49.154 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 13:09:49.154 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 13:09:49.154 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 13:09:49.154 | INFO | ------------------------------------------ | 20260505 13:09:49.154 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:09:49.154 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 13:09:49.154 | INFO | Total mem usage: 25.6 GiB | 20260505 13:09:49.154 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 13:09:49.154 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 13:09:49.154 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 13:09:49.154 | INFO | High Water Breakdown: | 20260505 13:09:49.154 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 13:09:49.154 | INFO | ------------------------------------------ | 20260505 13:09:49.154 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:09:49.154 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 13:09:49.154 | INFO | Total mem usage: 15.7 GiB | 20260505 13:09:49.154 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 13:09:49.154 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 13:09:49.154 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 13:09:49.154 | INFO | High Water Breakdown: | 20260505 13:09:49.154 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 13:09:49.154 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 13:09:49.154 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 13:09:49.154 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 13:09:49.154 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 13:09:49.154 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 13:09:49.154 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 13:09:49.154 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 13:09:49.154 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 13:09:49.154 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:09:49.154 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 13:09:49.154 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:09:49.154 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 13:09:49.155 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 13:09:49.155 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 13:09:49.155 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 13:09:49.155 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:09:49.155 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 13:09:49.155 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:09:49.155 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:09:49.155 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:09:49.155 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 13:09:49.155 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 13:09:49.155 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:09:49.155 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 13:09:49.155 | INFO | ------------------------------------------ | 20260505 13:09:49.155 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:09:49.155 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 13:09:49.155 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:09:49.155 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:09:49.155 | INFO | PerfReport: Full Execution -- 3.20e+01 / 7.70e+01, 1.36e+01 / 1.51e+00 | 20260505 13:09:49.155 | INFO | PerfReport: 0.29%, AggregateStats -- 9.30e-02 / 1.00e-01, 1.04e-02 / 3.72e-03 | 20260505 13:09:49.155 | INFO | PerfReport: 11.73%, Baseline -- 3.75e+00 / 4.48e+00, 5.85e-01 / 2.17e-01 | 20260505 13:09:49.155 | INFO | PerfReport: 3.31%, Histogram -- 1.06e+00 / 1.49e+00, 2.01e-01 / 1.00e-01 | 20260505 13:09:49.155 | INFO | PerfReport: 15.86%, DME -- 4.06e+01 / 4.54e+01, 4.44e+00 / 1.92e+00 | 20260505 13:09:49.155 | INFO | PerfReport: 56.62%, FrameLabeling -- 1.81e+01 / 2.08e+01, 1.63e+00 / 1.97e+00 | 20260505 13:09:49.155 | INFO | PerfReport: 7.35%, PulseAccumulating -- 2.35e+00 / 2.84e+00, 4.28e-01 / 2.41e-01 | 20260505 13:09:49.155 | INFO | PerfReport: 3.58%, Metrics -- 1.15e+00 / 1.48e+00, 3.24e-01 / 1.67e-01 | 20260505 13:09:49.155 | INFO | PerfReport: 1.26%, PulseSerializing -- 4.03e-01 / 4.53e-01, 7.36e-02 / 2.61e-02 | 20260505 13:09:49.155 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 13:09:49.155 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:09:49.155 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:09:49.155 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260505 13:09:49.155 | INFO | PerfReport: 22.60%, Upload -- 2.28e+01 / 0.00e+00, 8.11e+00 / 0.00e+00 | 20260505 13:09:49.155 | INFO | PerfReport: 22.03%, Download -- 2.22e+01 / 0.00e+00, 8.12e+00 / 0.00e+00 | 20260505 13:09:49.155 | INFO | PerfReport: 31.70%, Compute -- 3.20e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 13:09:49.155 | INFO | PerfReport: 2.14%, LockUpload -- 2.16e+00 / 0.00e+00, 7.60e+00 / 0.00e+00 | 20260505 13:09:49.155 | INFO | PerfReport: 1.33%, LockDownload -- 1.34e+00 / 0.00e+00, 2.87e+00 / 0.00e+00 | 20260505 13:09:49.155 | INFO | PerfReport: 20.20%, LockCompute -- 2.04e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260505 13:09:49.155 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.75e-02 / 0.00e+00 | 20260505 13:09:49.283 | INFO | Starting a round of Baz IO | 20260505 13:09:52.537 | INFO | Populating RTMetrics report | 20260505 13:09:52.538 | INFO | RTMetrics report populated | 20260505 13:09:52.538 | INFO | Opening tmp RTMetrics json file | 20260505 13:09:52.539 | INFO | Serializing RTMetrics to json | 20260505 13:09:52.564 | INFO | Serialization done, swapping tmp file now... | 20260505 13:09:52.586 | INFO | RTMetrics done | 20260505 13:09:52.653 | INFO | Finished a round of Baz IO: Wrote 2.309 GiB in 3.369 seconds | 20260505 13:09:52.653 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.24% / 0.00% / 76.76% | 20260505 13:09:52.653 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.749016% / 0% / 9.13938e-05% | 20260505 13:09:52.653 | INFO | Baz IO overall: | 20260505 13:09:52.653 | INFO | BytesWritten : 2058118659941 | 20260505 13:09:52.653 | INFO | NumEvents : 979328461839 | 20260505 13:09:54.297 | INFO | Analyzing chunk frames = [3457024,3457536) | 20260505 13:09:54.297 | INFO | MemInfo: { | 20260505 13:09:54.297 | INFO | "gpu" : " 17 GB", | 20260505 13:09:54.297 | INFO | "huge" : "", | 20260505 13:09:54.297 | INFO | "huge_shared" : " 27 GB", | 20260505 13:09:54.297 | INFO | "malloc" : " 14 GB", | 20260505 13:09:54.297 | INFO | "pinned" : " 29 GB" | 20260505 13:09:54.297 | INFO | } | 20260505 13:09:54.297 | INFO | Perf report for the last 5.14381 seconds: | 20260505 13:09:54.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:09:54.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:54.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:54.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:54.297 | INFO | ANALYSIS : 64.7%, 3.0, 100.4ms, 0 | 20260505 13:09:54.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:54.297 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 13:09:54.297 | INFO | PRE_HQ : 60.9%, 1.8, 57.8ms, 0 | 20260505 13:09:54.297 | INFO | BAZWRITER : 64.8%, 4.0, 133.7ms, 0 | 20260505 13:09:54.297 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260505 13:09:54.297 | INFO | Uploaded 36GiB to GPU in 2.20931s (16.2947GiB/s) | 20260505 13:09:54.297 | INFO | Downloaded 38GiB to GPU in 2.29317s (16.571GiB/s) | 20260505 13:09:59.441 | INFO | Analyzing chunk frames = [3457536,3458048) | 20260505 13:09:59.441 | INFO | MemInfo: { | 20260505 13:09:59.441 | INFO | "gpu" : " 17 GB", | 20260505 13:09:59.441 | INFO | "huge" : "", | 20260505 13:09:59.441 | INFO | "huge_shared" : " 27 GB", | 20260505 13:09:59.441 | INFO | "malloc" : " 14 GB", | 20260505 13:09:59.441 | INFO | "pinned" : " 29 GB" | 20260505 13:09:59.441 | INFO | } | 20260505 13:09:59.441 | INFO | Perf report for the last 5.14436 seconds: | 20260505 13:09:59.441 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:09:59.441 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:59.441 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:59.441 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:59.441 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 13:09:59.441 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:59.441 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:59.441 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 13:09:59.442 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:59.442 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:09:59.442 | INFO | Uploaded 36GiB to GPU in 2.17067s (16.5848GiB/s) | 20260505 13:09:59.442 | INFO | Downloaded 34GiB to GPU in 2.02975s (16.7509GiB/s) | 20260505 13:10:04.571 | INFO | Analyzing chunk frames = [3458048,3458560) | 20260505 13:10:04.571 | INFO | MemInfo: { | 20260505 13:10:04.571 | INFO | "gpu" : " 17 GB", | 20260505 13:10:04.571 | INFO | "huge" : "", | 20260505 13:10:04.571 | INFO | "huge_shared" : " 27 GB", | 20260505 13:10:04.571 | INFO | "malloc" : " 14 GB", | 20260505 13:10:04.571 | INFO | "pinned" : " 29 GB" | 20260505 13:10:04.571 | INFO | } | 20260505 13:10:04.571 | INFO | Perf report for the last 5.13005 seconds: | 20260505 13:10:04.571 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:10:04.571 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:04.571 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:04.571 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:04.571 | INFO | ANALYSIS : 63.4%, 3.0, 98.5ms, 0 | 20260505 13:10:04.571 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:04.571 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:04.571 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:10:04.571 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:04.571 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:04.572 | INFO | Uploaded 36GiB to GPU in 2.19676s (16.3877GiB/s) | 20260505 13:10:04.572 | INFO | Downloaded 34GiB to GPU in 2.13303s (15.9398GiB/s) | 20260505 13:10:09.711 | INFO | Analyzing chunk frames = [3458560,3459072) | 20260505 13:10:09.711 | INFO | MemInfo: { | 20260505 13:10:09.711 | INFO | "gpu" : " 17 GB", | 20260505 13:10:09.711 | INFO | "huge" : "", | 20260505 13:10:09.711 | INFO | "huge_shared" : " 27 GB", | 20260505 13:10:09.711 | INFO | "malloc" : " 14 GB", | 20260505 13:10:09.711 | INFO | "pinned" : " 29 GB" | 20260505 13:10:09.711 | INFO | } | 20260505 13:10:09.711 | INFO | Perf report for the last 5.13995 seconds: | 20260505 13:10:09.711 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:10:09.711 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:09.711 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:09.711 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:09.711 | INFO | ANALYSIS : 63.2%, 3.0, 98.3ms, 0 | 20260505 13:10:09.711 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:09.711 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:09.711 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 13:10:09.711 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:09.711 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:09.712 | INFO | Uploaded 36GiB to GPU in 2.21522s (16.2512GiB/s) | 20260505 13:10:09.712 | INFO | Downloaded 34GiB to GPU in 2.13497s (15.9253GiB/s) | 20260505 13:10:14.859 | INFO | Analyzing chunk frames = [3459072,3459584) | 20260505 13:10:14.859 | INFO | MemInfo: { | 20260505 13:10:14.859 | INFO | "gpu" : " 17 GB", | 20260505 13:10:14.859 | INFO | "huge" : "", | 20260505 13:10:14.859 | INFO | "huge_shared" : " 27 GB", | 20260505 13:10:14.859 | INFO | "malloc" : " 14 GB", | 20260505 13:10:14.859 | INFO | "pinned" : " 29 GB" | 20260505 13:10:14.859 | INFO | } | 20260505 13:10:14.859 | INFO | Perf report for the last 5.14812 seconds: | 20260505 13:10:14.859 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:10:14.859 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:14.859 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:14.859 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:14.859 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 13:10:14.859 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:14.859 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:14.859 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 13:10:14.859 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:14.860 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:14.860 | INFO | Uploaded 36GiB to GPU in 2.17816s (16.5277GiB/s) | 20260505 13:10:14.860 | INFO | Downloaded 34GiB to GPU in 2.11562s (16.0709GiB/s) | 20260505 13:10:19.993 | INFO | Analyzing chunk frames = [3459584,3460096) | 20260505 13:10:19.993 | INFO | MemInfo: { | 20260505 13:10:19.993 | INFO | "gpu" : " 17 GB", | 20260505 13:10:19.993 | INFO | "huge" : "", | 20260505 13:10:19.993 | INFO | "huge_shared" : " 27 GB", | 20260505 13:10:19.993 | INFO | "malloc" : " 14 GB", | 20260505 13:10:19.993 | INFO | "pinned" : " 29 GB" | 20260505 13:10:19.993 | INFO | } | 20260505 13:10:19.994 | INFO | Perf report for the last 5.13416 seconds: | 20260505 13:10:19.994 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:10:19.994 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:19.994 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:19.994 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:19.994 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 13:10:19.994 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:19.994 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:19.994 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:10:19.994 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:19.994 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:19.994 | INFO | Uploaded 36GiB to GPU in 2.20394s (16.3344GiB/s) | 20260505 13:10:19.994 | INFO | Downloaded 34GiB to GPU in 2.11127s (16.104GiB/s) | 20260505 13:10:25.127 | INFO | Analyzing chunk frames = [3460096,3460608) | 20260505 13:10:25.127 | INFO | MemInfo: { | 20260505 13:10:25.127 | INFO | "gpu" : " 17 GB", | 20260505 13:10:25.127 | INFO | "huge" : "", | 20260505 13:10:25.127 | INFO | "huge_shared" : " 27 GB", | 20260505 13:10:25.127 | INFO | "malloc" : " 14 GB", | 20260505 13:10:25.127 | INFO | "pinned" : " 29 GB" | 20260505 13:10:25.127 | INFO | } | 20260505 13:10:25.127 | INFO | Perf report for the last 5.13374 seconds: | 20260505 13:10:25.127 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:10:25.127 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:25.127 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:25.127 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:25.127 | INFO | ANALYSIS : 63.5%, 3.0, 99.0ms, 0 | 20260505 13:10:25.127 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:25.127 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:25.127 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 13:10:25.127 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:25.127 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:25.128 | INFO | Uploaded 36GiB to GPU in 2.15844s (16.6787GiB/s) | 20260505 13:10:25.128 | INFO | Downloaded 34GiB to GPU in 2.15456s (15.7805GiB/s) | 20260505 13:10:30.273 | INFO | Analyzing chunk frames = [3460608,3461120) | 20260505 13:10:30.273 | INFO | MemInfo: { | 20260505 13:10:30.273 | INFO | "gpu" : " 17 GB", | 20260505 13:10:30.273 | INFO | "huge" : "", | 20260505 13:10:30.273 | INFO | "huge_shared" : " 27 GB", | 20260505 13:10:30.273 | INFO | "malloc" : " 14 GB", | 20260505 13:10:30.273 | INFO | "pinned" : " 29 GB" | 20260505 13:10:30.273 | INFO | } | 20260505 13:10:30.273 | INFO | Perf report for the last 5.14613 seconds: | 20260505 13:10:30.273 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:10:30.273 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:30.273 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:30.273 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:30.274 | INFO | ANALYSIS : 63.8%, 3.0, 99.8ms, 0 | 20260505 13:10:30.274 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:30.274 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:30.274 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 13:10:30.274 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:30.274 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:30.274 | INFO | Uploaded 36GiB to GPU in 2.22103s (16.2087GiB/s) | 20260505 13:10:30.274 | INFO | Downloaded 34GiB to GPU in 2.09778s (16.2076GiB/s) | 20260505 13:10:30.407 | INFO | Starting a round of Baz IO | 20260505 13:10:33.656 | INFO | Populating RTMetrics report | 20260505 13:10:33.656 | INFO | RTMetrics report populated | 20260505 13:10:33.656 | INFO | Opening tmp RTMetrics json file | 20260505 13:10:33.688 | INFO | Serializing RTMetrics to json | 20260505 13:10:33.714 | INFO | Serialization done, swapping tmp file now... | 20260505 13:10:33.729 | INFO | RTMetrics done | 20260505 13:10:33.763 | INFO | Finished a round of Baz IO: Wrote 2.309 GiB in 3.356 seconds | 20260505 13:10:33.763 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.23% / 0.00% / 76.76% | 20260505 13:10:33.763 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.748281% / 3.97364e-06% / 7.94729e-05% | 20260505 13:10:33.763 | INFO | Baz IO overall: | 20260505 13:10:33.763 | INFO | BytesWritten : 2060597611244 | 20260505 13:10:33.763 | INFO | NumEvents : 980501734495 | 20260505 13:10:35.417 | INFO | Analyzing chunk frames = [3461120,3461632) | 20260505 13:10:35.417 | INFO | MemInfo: { | 20260505 13:10:35.417 | INFO | "gpu" : " 17 GB", | 20260505 13:10:35.417 | INFO | "huge" : "", | 20260505 13:10:35.417 | INFO | "huge_shared" : " 27 GB", | 20260505 13:10:35.417 | INFO | "malloc" : " 14 GB", | 20260505 13:10:35.417 | INFO | "pinned" : " 29 GB" | 20260505 13:10:35.417 | INFO | } | 20260505 13:10:35.417 | INFO | Perf report for the last 5.14332 seconds: | 20260505 13:10:35.417 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:10:35.417 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:35.417 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:35.417 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:35.417 | INFO | ANALYSIS : 64.7%, 2.9, 100.0ms, 0 | 20260505 13:10:35.417 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:35.417 | INFO | RT_METRICS : 46.5%, 1.0, 24.4ms, 0 | 20260505 13:10:35.417 | INFO | PRE_HQ : 61.2%, 1.8, 58.2ms, 0 | 20260505 13:10:35.417 | INFO | BAZWRITER : 64.5%, 4.1, 135.7ms, 0 | 20260505 13:10:35.417 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 13:10:35.417 | INFO | Uploaded 36GiB to GPU in 2.19361s (16.4113GiB/s) | 20260505 13:10:35.417 | INFO | Downloaded 38GiB to GPU in 2.29456s (16.5609GiB/s) | 20260505 13:10:40.553 | INFO | Analyzing chunk frames = [3461632,3462144) | 20260505 13:10:40.553 | INFO | MemInfo: { | 20260505 13:10:40.553 | INFO | "gpu" : " 17 GB", | 20260505 13:10:40.553 | INFO | "huge" : "", | 20260505 13:10:40.553 | INFO | "huge_shared" : " 27 GB", | 20260505 13:10:40.553 | INFO | "malloc" : " 14 GB", | 20260505 13:10:40.553 | INFO | "pinned" : " 29 GB" | 20260505 13:10:40.553 | INFO | } | 20260505 13:10:40.553 | INFO | Perf report for the last 5.13645 seconds: | 20260505 13:10:40.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:10:40.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:40.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:40.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:40.553 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 13:10:40.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:40.553 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:40.553 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:10:40.553 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:40.553 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:40.554 | INFO | Uploaded 36GiB to GPU in 2.17976s (16.5156GiB/s) | 20260505 13:10:40.554 | INFO | Downloaded 34GiB to GPU in 2.0307s (16.743GiB/s) | 20260505 13:10:45.692 | INFO | Analyzing chunk frames = [3462144,3462656) | 20260505 13:10:45.692 | INFO | MemInfo: { | 20260505 13:10:45.692 | INFO | "gpu" : " 17 GB", | 20260505 13:10:45.692 | INFO | "huge" : "", | 20260505 13:10:45.692 | INFO | "huge_shared" : " 27 GB", | 20260505 13:10:45.692 | INFO | "malloc" : " 14 GB", | 20260505 13:10:45.692 | INFO | "pinned" : " 29 GB" | 20260505 13:10:45.692 | INFO | } | 20260505 13:10:45.692 | INFO | Perf report for the last 5.13878 seconds: | 20260505 13:10:45.692 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:10:45.692 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:45.692 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:45.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:45.692 | INFO | ANALYSIS : 63.6%, 3.0, 98.9ms, 0 | 20260505 13:10:45.692 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:45.692 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:45.692 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 13:10:45.692 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:45.692 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:45.692 | INFO | Uploaded 36GiB to GPU in 2.23873s (16.0806GiB/s) | 20260505 13:10:45.692 | INFO | Downloaded 34GiB to GPU in 2.09484s (16.2304GiB/s) | 20260505 13:10:50.822 | INFO | Analyzing chunk frames = [3462656,3463168) | 20260505 13:10:50.822 | INFO | MemInfo: { | 20260505 13:10:50.822 | INFO | "gpu" : " 17 GB", | 20260505 13:10:50.822 | INFO | "huge" : "", | 20260505 13:10:50.822 | INFO | "huge_shared" : " 27 GB", | 20260505 13:10:50.823 | INFO | "malloc" : " 14 GB", | 20260505 13:10:50.823 | INFO | "pinned" : " 29 GB" | 20260505 13:10:50.823 | INFO | } | 20260505 13:10:50.823 | INFO | Perf report for the last 5.13054 seconds: | 20260505 13:10:50.823 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:10:50.823 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:50.823 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:50.823 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:50.823 | INFO | ANALYSIS : 63.2%, 3.0, 98.0ms, 0 | 20260505 13:10:50.823 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:50.823 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:50.823 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 13:10:50.823 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:50.823 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:50.823 | INFO | Uploaded 36GiB to GPU in 2.203s (16.3413GiB/s) | 20260505 13:10:50.823 | INFO | Downloaded 34GiB to GPU in 2.13404s (15.9322GiB/s) | 20260505 13:10:55.966 | INFO | Analyzing chunk frames = [3463168,3463680) | 20260505 13:10:55.966 | INFO | MemInfo: { | 20260505 13:10:55.966 | INFO | "gpu" : " 17 GB", | 20260505 13:10:55.966 | INFO | "huge" : "", | 20260505 13:10:55.966 | INFO | "huge_shared" : " 27 GB", | 20260505 13:10:55.966 | INFO | "malloc" : " 14 GB", | 20260505 13:10:55.966 | INFO | "pinned" : " 29 GB" | 20260505 13:10:55.966 | INFO | } | 20260505 13:10:55.966 | INFO | Perf report for the last 5.14329 seconds: | 20260505 13:10:55.966 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:10:55.966 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:55.966 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:55.966 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:55.966 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 13:10:55.966 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:55.966 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:55.966 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 13:10:55.966 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:55.966 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:10:55.966 | INFO | Uploaded 36GiB to GPU in 2.18716s (16.4597GiB/s) | 20260505 13:10:55.966 | INFO | Downloaded 34GiB to GPU in 2.13202s (15.9473GiB/s) | 20260505 13:11:01.113 | INFO | Analyzing chunk frames = [3463680,3464192) | 20260505 13:11:01.113 | INFO | MemInfo: { | 20260505 13:11:01.113 | INFO | "gpu" : " 17 GB", | 20260505 13:11:01.113 | INFO | "huge" : "", | 20260505 13:11:01.113 | INFO | "huge_shared" : " 27 GB", | 20260505 13:11:01.113 | INFO | "malloc" : " 14 GB", | 20260505 13:11:01.113 | INFO | "pinned" : " 29 GB" | 20260505 13:11:01.113 | INFO | } | 20260505 13:11:01.113 | INFO | Perf report for the last 5.14739 seconds: | 20260505 13:11:01.113 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:11:01.113 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:01.113 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:01.113 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:01.113 | INFO | ANALYSIS : 63.1%, 3.0, 98.7ms, 0 | 20260505 13:11:01.113 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:01.113 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:01.113 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 13:11:01.113 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:01.113 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:01.114 | INFO | Uploaded 36GiB to GPU in 2.17842s (16.5257GiB/s) | 20260505 13:11:01.114 | INFO | Downloaded 34GiB to GPU in 2.11281s (16.0923GiB/s) | 20260505 13:11:06.249 | INFO | Analyzing chunk frames = [3464192,3464704) | 20260505 13:11:06.249 | INFO | MemInfo: { | 20260505 13:11:06.249 | INFO | "gpu" : " 17 GB", | 20260505 13:11:06.249 | INFO | "huge" : "", | 20260505 13:11:06.249 | INFO | "huge_shared" : " 27 GB", | 20260505 13:11:06.249 | INFO | "malloc" : " 14 GB", | 20260505 13:11:06.249 | INFO | "pinned" : " 29 GB" | 20260505 13:11:06.250 | INFO | } | 20260505 13:11:06.250 | INFO | Perf report for the last 5.13649 seconds: | 20260505 13:11:06.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:11:06.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:06.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:06.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:06.250 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 13:11:06.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:06.250 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:06.250 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 13:11:06.250 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:06.250 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:06.250 | INFO | Uploaded 36GiB to GPU in 2.14919s (16.7505GiB/s) | 20260505 13:11:06.250 | INFO | Downloaded 34GiB to GPU in 2.17995s (15.5967GiB/s) | 20260505 13:11:11.391 | INFO | Analyzing chunk frames = [3464704,3465216) | 20260505 13:11:11.391 | INFO | MemInfo: { | 20260505 13:11:11.391 | INFO | "gpu" : " 17 GB", | 20260505 13:11:11.391 | INFO | "huge" : "", | 20260505 13:11:11.391 | INFO | "huge_shared" : " 27 GB", | 20260505 13:11:11.391 | INFO | "malloc" : " 14 GB", | 20260505 13:11:11.391 | INFO | "pinned" : " 29 GB" | 20260505 13:11:11.391 | INFO | } | 20260505 13:11:11.391 | INFO | Perf report for the last 5.14127 seconds: | 20260505 13:11:11.391 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:11:11.391 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:11.391 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:11.391 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:11.391 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 13:11:11.391 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:11.391 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:11.391 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 13:11:11.391 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:11.391 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:11.391 | INFO | Uploaded 36GiB to GPU in 2.15864s (16.6772GiB/s) | 20260505 13:11:11.391 | INFO | Downloaded 34GiB to GPU in 2.1054s (16.149GiB/s) | 20260505 13:11:11.392 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 13:11:11.392 | INFO | Total mem usage: 15.3 GiB | 20260505 13:11:11.392 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 13:11:11.392 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -6.98%) | 20260505 13:11:11.392 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 13:11:11.392 | INFO | High Water Breakdown: | 20260505 13:11:11.392 | INFO | 61.1% : 8.75 GiB ( 92% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 13:11:11.392 | INFO | 38.8% : 5.55 GiB ( 83.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 13:11:11.392 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 13:11:11.392 | INFO | ------------------------------------------ | 20260505 13:11:11.392 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:11:11.392 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 13:11:11.392 | INFO | Total mem usage: 26.5 GiB | 20260505 13:11:11.392 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 13:11:11.392 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 13:11:11.392 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 13:11:11.392 | INFO | High Water Breakdown: | 20260505 13:11:11.392 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:11:11.392 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:11:11.392 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:11:11.392 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:11:11.392 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:11:11.392 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:11:11.392 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 13:11:11.392 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 13:11:11.392 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:11:11.392 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 13:11:11.392 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 13:11:11.392 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 13:11:11.392 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 13:11:11.392 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 13:11:11.392 | INFO | ------------------------------------------ | 20260505 13:11:11.392 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:11:11.392 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 13:11:11.392 | INFO | Total mem usage: 25.6 GiB | 20260505 13:11:11.392 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 13:11:11.392 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 13:11:11.392 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 13:11:11.392 | INFO | High Water Breakdown: | 20260505 13:11:11.392 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 13:11:11.392 | INFO | ------------------------------------------ | 20260505 13:11:11.392 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:11:11.392 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 13:11:11.393 | INFO | Total mem usage: 15.7 GiB | 20260505 13:11:11.393 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 13:11:11.393 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 13:11:11.393 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 13:11:11.393 | INFO | High Water Breakdown: | 20260505 13:11:11.393 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 13:11:11.393 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 13:11:11.393 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 13:11:11.393 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 13:11:11.393 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 13:11:11.393 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 13:11:11.393 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 13:11:11.393 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 13:11:11.393 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 13:11:11.393 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:11:11.393 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 13:11:11.393 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:11:11.393 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 13:11:11.393 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 13:11:11.393 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 13:11:11.393 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 13:11:11.393 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:11:11.393 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 13:11:11.393 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:11:11.393 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:11:11.393 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:11:11.393 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 13:11:11.393 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 13:11:11.393 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:11:11.393 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 13:11:11.393 | INFO | ------------------------------------------ | 20260505 13:11:11.393 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:11:11.393 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 13:11:11.393 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:11:11.393 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:11:11.393 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.71e+01, 1.39e+01 / 1.80e+00 | 20260505 13:11:11.393 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 1.01e-01, 1.04e-02 / 5.42e-03 | 20260505 13:11:11.393 | INFO | PerfReport: 11.75%, Baseline -- 3.75e+00 / 4.52e+00, 5.85e-01 / 1.16e-01 | 20260505 13:11:11.393 | INFO | PerfReport: 3.29%, Histogram -- 1.05e+00 / 1.50e+00, 2.00e-01 / 9.19e-02 | 20260505 13:11:11.393 | INFO | PerfReport: 15.99%, DME -- 4.08e+01 / 4.60e+01, 4.73e+00 / 2.15e+00 | 20260505 13:11:11.393 | INFO | PerfReport: 56.38%, FrameLabeling -- 1.80e+01 / 2.01e+01, 1.45e+00 / 1.96e+00 | 20260505 13:11:11.393 | INFO | PerfReport: 7.43%, PulseAccumulating -- 2.37e+00 / 2.93e+00, 4.47e-01 / 1.67e-01 | 20260505 13:11:11.393 | INFO | PerfReport: 3.60%, Metrics -- 1.15e+00 / 1.48e+00, 3.24e-01 / 1.72e-01 | 20260505 13:11:11.393 | INFO | PerfReport: 1.26%, PulseSerializing -- 4.04e-01 / 4.56e-01, 7.32e-02 / 3.09e-02 | 20260505 13:11:11.393 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 13:11:11.393 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:11:11.393 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:11:11.393 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260505 13:11:11.393 | INFO | PerfReport: 22.58%, Upload -- 2.28e+01 / 0.00e+00, 8.18e+00 / 0.00e+00 | 20260505 13:11:11.393 | INFO | PerfReport: 22.00%, Download -- 2.22e+01 / 0.00e+00, 8.16e+00 / 0.00e+00 | 20260505 13:11:11.393 | INFO | PerfReport: 31.60%, Compute -- 3.19e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 13:11:11.393 | INFO | PerfReport: 2.29%, LockUpload -- 2.32e+00 / 0.00e+00, 8.00e+00 / 0.00e+00 | 20260505 13:11:11.393 | INFO | PerfReport: 1.33%, LockDownload -- 1.34e+00 / 0.00e+00, 2.85e+00 / 0.00e+00 | 20260505 13:11:11.393 | INFO | PerfReport: 20.19%, LockCompute -- 2.04e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260505 13:11:11.393 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.60e-02 / 0.00e+00 | 20260505 13:11:11.525 | INFO | Starting a round of Baz IO | 20260505 13:11:14.769 | INFO | Populating RTMetrics report | 20260505 13:11:14.769 | INFO | RTMetrics report populated | 20260505 13:11:14.769 | INFO | Opening tmp RTMetrics json file | 20260505 13:11:14.769 | INFO | Serializing RTMetrics to json | 20260505 13:11:14.771 | INFO | Serialization done, swapping tmp file now... | 20260505 13:11:14.771 | INFO | RTMetrics done | 20260505 13:11:14.888 | INFO | Finished a round of Baz IO: Wrote 2.31 GiB in 3.362 seconds | 20260505 13:11:14.888 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.23% / 0.01% / 76.77% | 20260505 13:11:14.888 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.746171% / 3.97364e-06% / 7.94729e-05% | 20260505 13:11:14.888 | INFO | Baz IO overall: | 20260505 13:11:14.888 | INFO | BytesWritten : 2063078282887 | 20260505 13:11:14.888 | INFO | NumEvents : 981676324235 | 20260505 13:11:16.520 | INFO | Analyzing chunk frames = [3465216,3465728) | 20260505 13:11:16.520 | INFO | MemInfo: { | 20260505 13:11:16.520 | INFO | "gpu" : " 17 GB", | 20260505 13:11:16.520 | INFO | "huge" : "", | 20260505 13:11:16.520 | INFO | "huge_shared" : " 27 GB", | 20260505 13:11:16.520 | INFO | "malloc" : " 14 GB", | 20260505 13:11:16.520 | INFO | "pinned" : " 29 GB" | 20260505 13:11:16.520 | INFO | } | 20260505 13:11:16.520 | INFO | Perf report for the last 5.12933 seconds: | 20260505 13:11:16.520 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:11:16.520 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:16.520 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:16.521 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:16.521 | INFO | ANALYSIS : 64.7%, 2.9, 99.8ms, 0 | 20260505 13:11:16.521 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:16.521 | INFO | RT_METRICS : 45.4%, 1.0, 23.7ms, 0 | 20260505 13:11:16.521 | INFO | PRE_HQ : 61.0%, 1.8, 58.4ms, 0 | 20260505 13:11:16.521 | INFO | BAZWRITER : 64.9%, 4.1, 135.5ms, 0 | 20260505 13:11:16.521 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 13:11:16.521 | INFO | Uploaded 36GiB to GPU in 2.18282s (16.4924GiB/s) | 20260505 13:11:16.521 | INFO | Downloaded 38GiB to GPU in 2.29187s (16.5803GiB/s) | 20260505 13:11:21.668 | INFO | Analyzing chunk frames = [3465728,3466240) | 20260505 13:11:21.668 | INFO | MemInfo: { | 20260505 13:11:21.668 | INFO | "gpu" : " 17 GB", | 20260505 13:11:21.668 | INFO | "huge" : "", | 20260505 13:11:21.668 | INFO | "huge_shared" : " 27 GB", | 20260505 13:11:21.668 | INFO | "malloc" : " 14 GB", | 20260505 13:11:21.668 | INFO | "pinned" : " 29 GB" | 20260505 13:11:21.668 | INFO | } | 20260505 13:11:21.668 | INFO | Perf report for the last 5.14801 seconds: | 20260505 13:11:21.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:11:21.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:21.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:21.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:21.668 | INFO | ANALYSIS : 64.1%, 3.0, 100.2ms, 0 | 20260505 13:11:21.669 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:21.669 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:21.669 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 13:11:21.669 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:21.669 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:21.669 | INFO | Uploaded 36GiB to GPU in 2.23563s (16.1029GiB/s) | 20260505 13:11:21.669 | INFO | Downloaded 34GiB to GPU in 2.09951s (16.1942GiB/s) | 20260505 13:11:26.805 | INFO | Analyzing chunk frames = [3466240,3466752) | 20260505 13:11:26.805 | INFO | MemInfo: { | 20260505 13:11:26.805 | INFO | "gpu" : " 17 GB", | 20260505 13:11:26.805 | INFO | "huge" : "", | 20260505 13:11:26.805 | INFO | "huge_shared" : " 27 GB", | 20260505 13:11:26.805 | INFO | "malloc" : " 14 GB", | 20260505 13:11:26.805 | INFO | "pinned" : " 29 GB" | 20260505 13:11:26.805 | INFO | } | 20260505 13:11:26.805 | INFO | Perf report for the last 5.1365 seconds: | 20260505 13:11:26.805 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:11:26.805 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:26.805 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:26.805 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:26.805 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260505 13:11:26.805 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:26.805 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:26.805 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:11:26.805 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:26.805 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:26.805 | INFO | Uploaded 36GiB to GPU in 2.22822s (16.1564GiB/s) | 20260505 13:11:26.805 | INFO | Downloaded 34GiB to GPU in 2.09395s (16.2373GiB/s) | 20260505 13:11:31.949 | INFO | Analyzing chunk frames = [3466752,3467264) | 20260505 13:11:31.949 | INFO | MemInfo: { | 20260505 13:11:31.949 | INFO | "gpu" : " 17 GB", | 20260505 13:11:31.949 | INFO | "huge" : "", | 20260505 13:11:31.949 | INFO | "huge_shared" : " 27 GB", | 20260505 13:11:31.949 | INFO | "malloc" : " 14 GB", | 20260505 13:11:31.949 | INFO | "pinned" : " 29 GB" | 20260505 13:11:31.949 | INFO | } | 20260505 13:11:31.949 | INFO | Perf report for the last 5.14432 seconds: | 20260505 13:11:31.949 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:11:31.949 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:31.949 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:31.949 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:31.949 | INFO | ANALYSIS : 63.0%, 3.0, 97.9ms, 0 | 20260505 13:11:31.949 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:31.949 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:31.949 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 13:11:31.949 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:31.949 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:31.950 | INFO | Uploaded 36GiB to GPU in 2.20172s (16.3508GiB/s) | 20260505 13:11:31.950 | INFO | Downloaded 34GiB to GPU in 2.1345s (15.9288GiB/s) | 20260505 13:11:37.081 | INFO | Analyzing chunk frames = [3467264,3467776) | 20260505 13:11:37.081 | INFO | MemInfo: { | 20260505 13:11:37.081 | INFO | "gpu" : " 17 GB", | 20260505 13:11:37.081 | INFO | "huge" : "", | 20260505 13:11:37.081 | INFO | "huge_shared" : " 27 GB", | 20260505 13:11:37.081 | INFO | "malloc" : " 14 GB", | 20260505 13:11:37.081 | INFO | "pinned" : " 29 GB" | 20260505 13:11:37.081 | INFO | } | 20260505 13:11:37.081 | INFO | Perf report for the last 5.13196 seconds: | 20260505 13:11:37.081 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:11:37.081 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:37.081 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:37.081 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:37.081 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260505 13:11:37.081 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:37.081 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:37.081 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 13:11:37.081 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:37.081 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:37.082 | INFO | Uploaded 36GiB to GPU in 2.16298s (16.6437GiB/s) | 20260505 13:11:37.082 | INFO | Downloaded 34GiB to GPU in 2.1277s (15.9797GiB/s) | 20260505 13:11:42.221 | INFO | Analyzing chunk frames = [3467776,3468288) | 20260505 13:11:42.221 | INFO | MemInfo: { | 20260505 13:11:42.221 | INFO | "gpu" : " 17 GB", | 20260505 13:11:42.221 | INFO | "huge" : "", | 20260505 13:11:42.221 | INFO | "huge_shared" : " 27 GB", | 20260505 13:11:42.221 | INFO | "malloc" : " 14 GB", | 20260505 13:11:42.221 | INFO | "pinned" : " 29 GB" | 20260505 13:11:42.221 | INFO | } | 20260505 13:11:42.221 | INFO | Perf report for the last 5.1397 seconds: | 20260505 13:11:42.221 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:11:42.221 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:42.221 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:42.221 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:42.221 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 13:11:42.221 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:42.221 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:42.221 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 13:11:42.221 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:42.221 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:42.221 | INFO | Uploaded 36GiB to GPU in 2.15879s (16.676GiB/s) | 20260505 13:11:42.221 | INFO | Downloaded 34GiB to GPU in 2.11244s (16.0952GiB/s) | 20260505 13:11:47.369 | INFO | Analyzing chunk frames = [3468288,3468800) | 20260505 13:11:47.369 | INFO | MemInfo: { | 20260505 13:11:47.369 | INFO | "gpu" : " 17 GB", | 20260505 13:11:47.369 | INFO | "huge" : "", | 20260505 13:11:47.369 | INFO | "huge_shared" : " 27 GB", | 20260505 13:11:47.369 | INFO | "malloc" : " 14 GB", | 20260505 13:11:47.369 | INFO | "pinned" : " 29 GB" | 20260505 13:11:47.369 | INFO | } | 20260505 13:11:47.369 | INFO | Perf report for the last 5.14838 seconds: | 20260505 13:11:47.369 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:11:47.369 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:47.369 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:47.369 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:47.369 | INFO | ANALYSIS : 63.2%, 3.0, 98.9ms, 0 | 20260505 13:11:47.369 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:47.369 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:47.369 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 13:11:47.369 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:47.370 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:47.370 | INFO | Uploaded 36GiB to GPU in 2.16276s (16.6454GiB/s) | 20260505 13:11:47.370 | INFO | Downloaded 34GiB to GPU in 2.1441s (15.8575GiB/s) | 20260505 13:11:52.510 | INFO | Analyzing chunk frames = [3468800,3469312) | 20260505 13:11:52.510 | INFO | MemInfo: { | 20260505 13:11:52.510 | INFO | "gpu" : " 17 GB", | 20260505 13:11:52.510 | INFO | "huge" : "", | 20260505 13:11:52.510 | INFO | "huge_shared" : " 27 GB", | 20260505 13:11:52.510 | INFO | "malloc" : " 14 GB", | 20260505 13:11:52.510 | INFO | "pinned" : " 29 GB" | 20260505 13:11:52.510 | INFO | } | 20260505 13:11:52.510 | INFO | Perf report for the last 5.14035 seconds: | 20260505 13:11:52.510 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:11:52.510 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:52.510 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:52.510 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:52.510 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 13:11:52.510 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:52.510 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:52.510 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 13:11:52.510 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:52.510 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:52.510 | INFO | Uploaded 36GiB to GPU in 2.23025s (16.1417GiB/s) | 20260505 13:11:52.510 | INFO | Downloaded 34GiB to GPU in 2.09106s (16.2597GiB/s) | 20260505 13:11:52.646 | INFO | Starting a round of Baz IO | 20260505 13:11:55.868 | INFO | Populating RTMetrics report | 20260505 13:11:55.869 | INFO | RTMetrics report populated | 20260505 13:11:55.869 | INFO | Opening tmp RTMetrics json file | 20260505 13:11:55.891 | INFO | Serializing RTMetrics to json | 20260505 13:11:55.931 | INFO | Serialization done, swapping tmp file now... | 20260505 13:11:55.941 | INFO | RTMetrics done | 20260505 13:11:55.982 | INFO | Finished a round of Baz IO: Wrote 2.309 GiB in 3.336 seconds | 20260505 13:11:55.982 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.22% / 0.01% / 76.77% | 20260505 13:11:55.982 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.744172% / 0% / 5.96046e-05% | 20260505 13:11:55.982 | INFO | Baz IO overall: | 20260505 13:11:55.982 | INFO | BytesWritten : 2065557869026 | 20260505 13:11:55.982 | INFO | NumEvents : 982850457976 | 20260505 13:11:57.641 | INFO | Analyzing chunk frames = [3469312,3469824) | 20260505 13:11:57.641 | INFO | MemInfo: { | 20260505 13:11:57.641 | INFO | "gpu" : " 17 GB", | 20260505 13:11:57.641 | INFO | "huge" : "", | 20260505 13:11:57.641 | INFO | "huge_shared" : " 27 GB", | 20260505 13:11:57.641 | INFO | "malloc" : " 14 GB", | 20260505 13:11:57.641 | INFO | "pinned" : " 29 GB" | 20260505 13:11:57.641 | INFO | } | 20260505 13:11:57.641 | INFO | Perf report for the last 5.13145 seconds: | 20260505 13:11:57.641 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:11:57.641 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:57.641 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:57.641 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:57.641 | INFO | ANALYSIS : 64.3%, 2.9, 99.3ms, 0 | 20260505 13:11:57.641 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:11:57.641 | INFO | RT_METRICS : 46.6%, 1.0, 24.4ms, 0 | 20260505 13:11:57.642 | INFO | PRE_HQ : 61.4%, 1.8, 58.3ms, 0 | 20260505 13:11:57.642 | INFO | BAZWRITER : 64.1%, 4.1, 134.3ms, 0 | 20260505 13:11:57.642 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 13:11:57.642 | INFO | Uploaded 36GiB to GPU in 2.20146s (16.3528GiB/s) | 20260505 13:11:57.642 | INFO | Downloaded 38GiB to GPU in 2.25613s (16.843GiB/s) | 20260505 13:12:02.771 | INFO | Analyzing chunk frames = [3469824,3470336) | 20260505 13:12:02.771 | INFO | MemInfo: { | 20260505 13:12:02.771 | INFO | "gpu" : " 17 GB", | 20260505 13:12:02.771 | INFO | "huge" : "", | 20260505 13:12:02.771 | INFO | "huge_shared" : " 27 GB", | 20260505 13:12:02.771 | INFO | "malloc" : " 14 GB", | 20260505 13:12:02.771 | INFO | "pinned" : " 29 GB" | 20260505 13:12:02.771 | INFO | } | 20260505 13:12:02.771 | INFO | Perf report for the last 5.12949 seconds: | 20260505 13:12:02.771 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:12:02.771 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:02.771 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:02.771 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:02.771 | INFO | ANALYSIS : 63.8%, 3.0, 99.5ms, 0 | 20260505 13:12:02.771 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:02.771 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:02.771 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 13:12:02.771 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:02.771 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:02.771 | INFO | Uploaded 36GiB to GPU in 2.18196s (16.4989GiB/s) | 20260505 13:12:02.771 | INFO | Downloaded 34GiB to GPU in 2.04167s (16.653GiB/s) | 20260505 13:12:07.921 | INFO | Analyzing chunk frames = [3470336,3470848) | 20260505 13:12:07.921 | INFO | MemInfo: { | 20260505 13:12:07.921 | INFO | "gpu" : " 17 GB", | 20260505 13:12:07.921 | INFO | "huge" : "", | 20260505 13:12:07.921 | INFO | "huge_shared" : " 27 GB", | 20260505 13:12:07.921 | INFO | "malloc" : " 14 GB", | 20260505 13:12:07.921 | INFO | "pinned" : " 29 GB" | 20260505 13:12:07.921 | INFO | } | 20260505 13:12:07.922 | INFO | Perf report for the last 5.15086 seconds: | 20260505 13:12:07.922 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:12:07.922 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:07.922 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:07.922 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:07.922 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 13:12:07.922 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:07.922 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:07.922 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 13:12:07.922 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:07.922 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:07.922 | INFO | Uploaded 36GiB to GPU in 2.20977s (16.2913GiB/s) | 20260505 13:12:07.922 | INFO | Downloaded 34GiB to GPU in 2.11731s (16.0581GiB/s) | 20260505 13:12:13.069 | INFO | Analyzing chunk frames = [3470848,3471360) | 20260505 13:12:13.069 | INFO | MemInfo: { | 20260505 13:12:13.069 | INFO | "gpu" : " 17 GB", | 20260505 13:12:13.069 | INFO | "huge" : "", | 20260505 13:12:13.069 | INFO | "huge_shared" : " 27 GB", | 20260505 13:12:13.069 | INFO | "malloc" : " 14 GB", | 20260505 13:12:13.069 | INFO | "pinned" : " 29 GB" | 20260505 13:12:13.070 | INFO | } | 20260505 13:12:13.070 | INFO | Perf report for the last 5.14818 seconds: | 20260505 13:12:13.070 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:12:13.070 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:13.070 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:13.070 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:13.070 | INFO | ANALYSIS : 63.0%, 3.0, 98.1ms, 0 | 20260505 13:12:13.070 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:13.070 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:13.070 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:12:13.070 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:13.070 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:13.070 | INFO | Uploaded 36GiB to GPU in 2.20675s (16.3136GiB/s) | 20260505 13:12:13.070 | INFO | Downloaded 34GiB to GPU in 2.1324s (15.9445GiB/s) | 20260505 13:12:18.199 | INFO | Analyzing chunk frames = [3471360,3471872) | 20260505 13:12:18.199 | INFO | MemInfo: { | 20260505 13:12:18.199 | INFO | "gpu" : " 17 GB", | 20260505 13:12:18.199 | INFO | "huge" : "", | 20260505 13:12:18.199 | INFO | "huge_shared" : " 27 GB", | 20260505 13:12:18.199 | INFO | "malloc" : " 14 GB", | 20260505 13:12:18.199 | INFO | "pinned" : " 29 GB" | 20260505 13:12:18.199 | INFO | } | 20260505 13:12:18.199 | INFO | Perf report for the last 5.12924 seconds: | 20260505 13:12:18.199 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:12:18.199 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:18.199 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:18.199 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:18.199 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260505 13:12:18.199 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:18.199 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:18.199 | INFO | PRE_HQ : 28.7%, 1.0, 14.8ms, 0 | 20260505 13:12:18.199 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:18.199 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:18.199 | INFO | Uploaded 36GiB to GPU in 2.14534s (16.7805GiB/s) | 20260505 13:12:18.199 | INFO | Downloaded 34GiB to GPU in 2.12101s (16.0301GiB/s) | 20260505 13:12:23.335 | INFO | Analyzing chunk frames = [3471872,3472384) | 20260505 13:12:23.335 | INFO | MemInfo: { | 20260505 13:12:23.335 | INFO | "gpu" : " 17 GB", | 20260505 13:12:23.335 | INFO | "huge" : "", | 20260505 13:12:23.335 | INFO | "huge_shared" : " 27 GB", | 20260505 13:12:23.335 | INFO | "malloc" : " 14 GB", | 20260505 13:12:23.335 | INFO | "pinned" : " 29 GB" | 20260505 13:12:23.335 | INFO | } | 20260505 13:12:23.335 | INFO | Perf report for the last 5.13631 seconds: | 20260505 13:12:23.335 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:12:23.335 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:23.335 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:23.335 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:23.335 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 13:12:23.335 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:23.335 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:23.335 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 13:12:23.335 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:23.336 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:23.336 | INFO | Uploaded 36GiB to GPU in 2.17534s (16.5491GiB/s) | 20260505 13:12:23.336 | INFO | Downloaded 34GiB to GPU in 2.10657s (16.14GiB/s) | 20260505 13:12:28.473 | INFO | Analyzing chunk frames = [3472384,3472896) | 20260505 13:12:28.473 | INFO | MemInfo: { | 20260505 13:12:28.473 | INFO | "gpu" : " 17 GB", | 20260505 13:12:28.473 | INFO | "huge" : "", | 20260505 13:12:28.473 | INFO | "huge_shared" : " 27 GB", | 20260505 13:12:28.473 | INFO | "malloc" : " 14 GB", | 20260505 13:12:28.473 | INFO | "pinned" : " 29 GB" | 20260505 13:12:28.473 | INFO | } | 20260505 13:12:28.473 | INFO | Perf report for the last 5.13776 seconds: | 20260505 13:12:28.473 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:12:28.473 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:28.473 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:28.473 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:28.473 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 13:12:28.473 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:28.473 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:28.473 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 13:12:28.473 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:28.473 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:28.473 | INFO | Uploaded 36GiB to GPU in 2.18859s (16.4489GiB/s) | 20260505 13:12:28.474 | INFO | Downloaded 34GiB to GPU in 2.14758s (15.8318GiB/s) | 20260505 13:12:33.621 | INFO | Analyzing chunk frames = [3472896,3473408) | 20260505 13:12:33.621 | INFO | MemInfo: { | 20260505 13:12:33.621 | INFO | "gpu" : " 17 GB", | 20260505 13:12:33.621 | INFO | "huge" : "", | 20260505 13:12:33.621 | INFO | "huge_shared" : " 27 GB", | 20260505 13:12:33.621 | INFO | "malloc" : " 14 GB", | 20260505 13:12:33.621 | INFO | "pinned" : " 29 GB" | 20260505 13:12:33.621 | INFO | } | 20260505 13:12:33.621 | INFO | Perf report for the last 5.14777 seconds: | 20260505 13:12:33.621 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:12:33.621 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:33.621 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:33.621 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:33.621 | INFO | ANALYSIS : 63.5%, 3.0, 99.4ms, 0 | 20260505 13:12:33.621 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:33.621 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:33.621 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 13:12:33.621 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:33.621 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:33.621 | INFO | Uploaded 36GiB to GPU in 2.14043s (16.819GiB/s) | 20260505 13:12:33.621 | INFO | Downloaded 34GiB to GPU in 2.10604s (16.144GiB/s) | 20260505 13:12:33.621 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 13:12:33.621 | INFO | Total mem usage: 15.3 GiB | 20260505 13:12:33.621 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 13:12:33.621 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -7.02%) | 20260505 13:12:33.621 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 13:12:33.621 | INFO | High Water Breakdown: | 20260505 13:12:33.621 | INFO | 61.1% : 8.75 GiB ( 91.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 13:12:33.621 | INFO | 38.8% : 5.55 GiB ( 83.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 13:12:33.621 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 13:12:33.621 | INFO | ------------------------------------------ | 20260505 13:12:33.621 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:12:33.621 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 13:12:33.621 | INFO | Total mem usage: 26.5 GiB | 20260505 13:12:33.621 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 13:12:33.621 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 13:12:33.622 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 13:12:33.622 | INFO | High Water Breakdown: | 20260505 13:12:33.622 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:12:33.622 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:12:33.622 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:12:33.622 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:12:33.622 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:12:33.622 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:12:33.622 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 13:12:33.622 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 13:12:33.622 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:12:33.622 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 13:12:33.622 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 13:12:33.622 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 13:12:33.622 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 13:12:33.622 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 13:12:33.622 | INFO | ------------------------------------------ | 20260505 13:12:33.622 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:12:33.622 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 13:12:33.622 | INFO | Total mem usage: 25.6 GiB | 20260505 13:12:33.622 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 13:12:33.622 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 13:12:33.622 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 13:12:33.622 | INFO | High Water Breakdown: | 20260505 13:12:33.622 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 13:12:33.622 | INFO | ------------------------------------------ | 20260505 13:12:33.622 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:12:33.622 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 13:12:33.622 | INFO | Total mem usage: 15.7 GiB | 20260505 13:12:33.622 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 13:12:33.622 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 13:12:33.622 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 13:12:33.622 | INFO | High Water Breakdown: | 20260505 13:12:33.622 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 13:12:33.622 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 13:12:33.622 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 13:12:33.622 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 13:12:33.622 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 13:12:33.622 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 13:12:33.622 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 13:12:33.622 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 13:12:33.622 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 13:12:33.622 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:12:33.622 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 13:12:33.622 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:12:33.622 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 13:12:33.622 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 13:12:33.622 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 13:12:33.622 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 13:12:33.622 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:12:33.622 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 13:12:33.622 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:12:33.622 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:12:33.622 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:12:33.622 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 13:12:33.622 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 13:12:33.622 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:12:33.622 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 13:12:33.622 | INFO | ------------------------------------------ | 20260505 13:12:33.622 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:12:33.622 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 13:12:33.622 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:12:33.622 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:12:33.622 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.68e+01, 1.37e+01 / 1.38e+00 | 20260505 13:12:33.622 | INFO | PerfReport: 0.29%, AggregateStats -- 9.30e-02 / 9.96e-02, 1.07e-02 / 4.91e-03 | 20260505 13:12:33.622 | INFO | PerfReport: 11.75%, Baseline -- 3.75e+00 / 4.50e+00, 5.87e-01 / 1.95e-01 | 20260505 13:12:33.622 | INFO | PerfReport: 3.29%, Histogram -- 1.05e+00 / 1.48e+00, 1.96e-01 / 1.01e-01 | 20260505 13:12:33.622 | INFO | PerfReport: 15.98%, DME -- 4.08e+01 / 4.58e+01, 4.50e+00 / 1.70e+00 | 20260505 13:12:33.622 | INFO | PerfReport: 56.35%, FrameLabeling -- 1.80e+01 / 2.01e+01, 1.41e+00 / 2.17e+00 | 20260505 13:12:33.622 | INFO | PerfReport: 7.47%, PulseAccumulating -- 2.38e+00 / 2.92e+00, 4.48e-01 / 1.95e-01 | 20260505 13:12:33.622 | INFO | PerfReport: 3.60%, Metrics -- 1.15e+00 / 1.46e+00, 3.19e-01 / 1.79e-01 | 20260505 13:12:33.622 | INFO | PerfReport: 1.26%, PulseSerializing -- 4.03e-01 / 4.56e-01, 7.24e-02 / 3.09e-02 | 20260505 13:12:33.623 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 13:12:33.623 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:12:33.623 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:12:33.623 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260505 13:12:33.623 | INFO | PerfReport: 22.58%, Upload -- 2.28e+01 / 0.00e+00, 8.12e+00 / 0.00e+00 | 20260505 13:12:33.623 | INFO | PerfReport: 22.01%, Download -- 2.22e+01 / 0.00e+00, 8.11e+00 / 0.00e+00 | 20260505 13:12:33.623 | INFO | PerfReport: 31.60%, Compute -- 3.19e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 13:12:33.623 | INFO | PerfReport: 2.35%, LockUpload -- 2.38e+00 / 0.00e+00, 7.93e+00 / 0.00e+00 | 20260505 13:12:33.623 | INFO | PerfReport: 1.30%, LockDownload -- 1.31e+00 / 0.00e+00, 2.84e+00 / 0.00e+00 | 20260505 13:12:33.623 | INFO | PerfReport: 20.15%, LockCompute -- 2.03e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260505 13:12:33.623 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.45e-02 / 0.00e+00 | 20260505 13:12:33.757 | INFO | Starting a round of Baz IO | 20260505 13:12:36.976 | INFO | Populating RTMetrics report | 20260505 13:12:36.977 | INFO | RTMetrics report populated | 20260505 13:12:36.977 | INFO | Opening tmp RTMetrics json file | 20260505 13:12:36.991 | INFO | Serializing RTMetrics to json | 20260505 13:12:37.028 | INFO | Serialization done, swapping tmp file now... | 20260505 13:12:37.050 | INFO | RTMetrics done | 20260505 13:12:37.092 | INFO | Finished a round of Baz IO: Wrote 2.311 GiB in 3.335 seconds | 20260505 13:12:37.092 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.22% / 0.00% / 76.78% | 20260505 13:12:37.092 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.745451% / 7.94729e-06% / 7.54992e-05% | 20260505 13:12:37.092 | INFO | Baz IO overall: | 20260505 13:12:37.092 | INFO | BytesWritten : 2068039645527 | 20260505 13:12:37.092 | INFO | NumEvents : 984026063176 | 20260505 13:12:38.756 | INFO | Analyzing chunk frames = [3473408,3473920) | 20260505 13:12:38.756 | INFO | MemInfo: { | 20260505 13:12:38.756 | INFO | "gpu" : " 17 GB", | 20260505 13:12:38.756 | INFO | "huge" : "", | 20260505 13:12:38.756 | INFO | "huge_shared" : " 27 GB", | 20260505 13:12:38.756 | INFO | "malloc" : " 14 GB", | 20260505 13:12:38.756 | INFO | "pinned" : " 29 GB" | 20260505 13:12:38.756 | INFO | } | 20260505 13:12:38.756 | INFO | Perf report for the last 5.13512 seconds: | 20260505 13:12:38.756 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:12:38.756 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:38.756 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:38.756 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:38.756 | INFO | ANALYSIS : 64.2%, 2.9, 99.3ms, 0 | 20260505 13:12:38.756 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:38.756 | INFO | RT_METRICS : 46.5%, 1.0, 24.4ms, 0 | 20260505 13:12:38.756 | INFO | PRE_HQ : 61.0%, 1.8, 57.8ms, 0 | 20260505 13:12:38.756 | INFO | BAZWRITER : 64.3%, 4.1, 134.4ms, 0 | 20260505 13:12:38.756 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 13:12:38.756 | INFO | Uploaded 36GiB to GPU in 2.19312s (16.415GiB/s) | 20260505 13:12:38.756 | INFO | Downloaded 38GiB to GPU in 2.26693s (16.7628GiB/s) | 20260505 13:12:43.891 | INFO | Analyzing chunk frames = [3473920,3474432) | 20260505 13:12:43.891 | INFO | MemInfo: { | 20260505 13:12:43.891 | INFO | "gpu" : " 17 GB", | 20260505 13:12:43.891 | INFO | "huge" : "", | 20260505 13:12:43.891 | INFO | "huge_shared" : " 27 GB", | 20260505 13:12:43.891 | INFO | "malloc" : " 14 GB", | 20260505 13:12:43.891 | INFO | "pinned" : " 29 GB" | 20260505 13:12:43.891 | INFO | } | 20260505 13:12:43.891 | INFO | Perf report for the last 5.13504 seconds: | 20260505 13:12:43.891 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:12:43.891 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:43.891 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:43.891 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:43.891 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 13:12:43.891 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:43.891 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:43.891 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 13:12:43.891 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:43.891 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:43.891 | INFO | Uploaded 36GiB to GPU in 2.18459s (16.479GiB/s) | 20260505 13:12:43.891 | INFO | Downloaded 34GiB to GPU in 2.0399s (16.6675GiB/s) | 20260505 13:12:49.030 | INFO | Analyzing chunk frames = [3474432,3474944) | 20260505 13:12:49.030 | INFO | MemInfo: { | 20260505 13:12:49.030 | INFO | "gpu" : " 17 GB", | 20260505 13:12:49.030 | INFO | "huge" : "", | 20260505 13:12:49.030 | INFO | "huge_shared" : " 27 GB", | 20260505 13:12:49.030 | INFO | "malloc" : " 14 GB", | 20260505 13:12:49.030 | INFO | "pinned" : " 29 GB" | 20260505 13:12:49.030 | INFO | } | 20260505 13:12:49.030 | INFO | Perf report for the last 5.13874 seconds: | 20260505 13:12:49.030 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:12:49.030 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:49.030 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:49.030 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:49.030 | INFO | ANALYSIS : 63.6%, 3.0, 99.0ms, 0 | 20260505 13:12:49.030 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:49.030 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:49.030 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 13:12:49.030 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:49.030 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:49.030 | INFO | Uploaded 36GiB to GPU in 2.23192s (16.1296GiB/s) | 20260505 13:12:49.030 | INFO | Downloaded 34GiB to GPU in 2.12567s (15.995GiB/s) | 20260505 13:12:54.176 | INFO | Analyzing chunk frames = [3474944,3475456) | 20260505 13:12:54.176 | INFO | MemInfo: { | 20260505 13:12:54.176 | INFO | "gpu" : " 17 GB", | 20260505 13:12:54.176 | INFO | "huge" : "", | 20260505 13:12:54.176 | INFO | "huge_shared" : " 27 GB", | 20260505 13:12:54.176 | INFO | "malloc" : " 14 GB", | 20260505 13:12:54.176 | INFO | "pinned" : " 29 GB" | 20260505 13:12:54.176 | INFO | } | 20260505 13:12:54.176 | INFO | Perf report for the last 5.1465 seconds: | 20260505 13:12:54.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:12:54.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:54.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:54.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:54.176 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 13:12:54.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:54.176 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:54.176 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 13:12:54.176 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:54.176 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:54.177 | INFO | Uploaded 36GiB to GPU in 2.21503s (16.2526GiB/s) | 20260505 13:12:54.177 | INFO | Downloaded 34GiB to GPU in 2.12907s (15.9694GiB/s) | 20260505 13:12:59.322 | INFO | Analyzing chunk frames = [3475456,3475968) | 20260505 13:12:59.322 | INFO | MemInfo: { | 20260505 13:12:59.322 | INFO | "gpu" : " 17 GB", | 20260505 13:12:59.322 | INFO | "huge" : "", | 20260505 13:12:59.322 | INFO | "huge_shared" : " 27 GB", | 20260505 13:12:59.322 | INFO | "malloc" : " 14 GB", | 20260505 13:12:59.322 | INFO | "pinned" : " 29 GB" | 20260505 13:12:59.322 | INFO | } | 20260505 13:12:59.322 | INFO | Perf report for the last 5.14568 seconds: | 20260505 13:12:59.322 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:12:59.322 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:59.322 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:59.322 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:59.322 | INFO | ANALYSIS : 62.5%, 3.0, 97.7ms, 0 | 20260505 13:12:59.322 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:59.322 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:59.322 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 13:12:59.322 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:59.322 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:12:59.322 | INFO | Uploaded 36GiB to GPU in 2.11525s (17.0193GiB/s) | 20260505 13:12:59.322 | INFO | Downloaded 34GiB to GPU in 2.11798s (16.0531GiB/s) | 20260505 13:13:04.455 | INFO | Analyzing chunk frames = [3475968,3476480) | 20260505 13:13:04.455 | INFO | MemInfo: { | 20260505 13:13:04.455 | INFO | "gpu" : " 17 GB", | 20260505 13:13:04.455 | INFO | "huge" : "", | 20260505 13:13:04.455 | INFO | "huge_shared" : " 27 GB", | 20260505 13:13:04.455 | INFO | "malloc" : " 14 GB", | 20260505 13:13:04.455 | INFO | "pinned" : " 29 GB" | 20260505 13:13:04.455 | INFO | } | 20260505 13:13:04.455 | INFO | Perf report for the last 5.13276 seconds: | 20260505 13:13:04.455 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:13:04.455 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:04.455 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:04.455 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:04.455 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 13:13:04.455 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:04.455 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:04.455 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 13:13:04.455 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:04.455 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:04.455 | INFO | Uploaded 36GiB to GPU in 2.19s (16.4384GiB/s) | 20260505 13:13:04.455 | INFO | Downloaded 34GiB to GPU in 2.11105s (16.1057GiB/s) | 20260505 13:13:09.591 | INFO | Analyzing chunk frames = [3476480,3476992) | 20260505 13:13:09.591 | INFO | MemInfo: { | 20260505 13:13:09.591 | INFO | "gpu" : " 17 GB", | 20260505 13:13:09.591 | INFO | "huge" : "", | 20260505 13:13:09.591 | INFO | "huge_shared" : " 27 GB", | 20260505 13:13:09.591 | INFO | "malloc" : " 14 GB", | 20260505 13:13:09.591 | INFO | "pinned" : " 29 GB" | 20260505 13:13:09.591 | INFO | } | 20260505 13:13:09.591 | INFO | Perf report for the last 5.13593 seconds: | 20260505 13:13:09.591 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:13:09.591 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:09.591 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:09.591 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:09.591 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 13:13:09.591 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:09.591 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:09.591 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 13:13:09.591 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:09.591 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:09.591 | INFO | Uploaded 36GiB to GPU in 2.17442s (16.5562GiB/s) | 20260505 13:13:09.591 | INFO | Downloaded 34GiB to GPU in 2.14643s (15.8403GiB/s) | 20260505 13:13:14.729 | INFO | Analyzing chunk frames = [3476992,3477504) | 20260505 13:13:14.729 | INFO | MemInfo: { | 20260505 13:13:14.729 | INFO | "gpu" : " 17 GB", | 20260505 13:13:14.729 | INFO | "huge" : "", | 20260505 13:13:14.729 | INFO | "huge_shared" : " 27 GB", | 20260505 13:13:14.729 | INFO | "malloc" : " 14 GB", | 20260505 13:13:14.729 | INFO | "pinned" : " 29 GB" | 20260505 13:13:14.729 | INFO | } | 20260505 13:13:14.729 | INFO | Perf report for the last 5.13826 seconds: | 20260505 13:13:14.729 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:13:14.729 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:14.729 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:14.729 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:14.729 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 13:13:14.729 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:14.729 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:14.729 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 13:13:14.729 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:14.729 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:14.729 | INFO | Uploaded 36GiB to GPU in 2.20309s (16.3407GiB/s) | 20260505 13:13:14.729 | INFO | Downloaded 34GiB to GPU in 2.11713s (16.0595GiB/s) | 20260505 13:13:14.860 | INFO | Starting a round of Baz IO | 20260505 13:13:18.109 | INFO | Populating RTMetrics report | 20260505 13:13:18.110 | INFO | RTMetrics report populated | 20260505 13:13:18.110 | INFO | Opening tmp RTMetrics json file | 20260505 13:13:18.111 | INFO | Serializing RTMetrics to json | 20260505 13:13:18.125 | INFO | Serialization done, swapping tmp file now... | 20260505 13:13:18.136 | INFO | RTMetrics done | 20260505 13:13:18.226 | INFO | Finished a round of Baz IO: Wrote 2.309 GiB in 3.366 seconds | 20260505 13:13:18.226 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.21% / 0.01% / 76.78% | 20260505 13:13:18.226 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.741378% / 1.58946e-05% / 9.53674e-05% | 20260505 13:13:18.226 | INFO | Baz IO overall: | 20260505 13:13:18.226 | INFO | BytesWritten : 2070519156003 | 20260505 13:13:18.226 | INFO | NumEvents : 985200286939 | 20260505 13:13:19.875 | INFO | Analyzing chunk frames = [3477504,3478016) | 20260505 13:13:19.875 | INFO | MemInfo: { | 20260505 13:13:19.875 | INFO | "gpu" : " 17 GB", | 20260505 13:13:19.875 | INFO | "huge" : "", | 20260505 13:13:19.875 | INFO | "huge_shared" : " 27 GB", | 20260505 13:13:19.875 | INFO | "malloc" : " 14 GB", | 20260505 13:13:19.875 | INFO | "pinned" : " 29 GB" | 20260505 13:13:19.875 | INFO | } | 20260505 13:13:19.875 | INFO | Perf report for the last 5.14599 seconds: | 20260505 13:13:19.875 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:13:19.875 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:19.875 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:19.875 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:19.875 | INFO | ANALYSIS : 64.5%, 3.0, 100.1ms, 0 | 20260505 13:13:19.875 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:19.875 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260505 13:13:19.875 | INFO | PRE_HQ : 60.8%, 1.8, 58.2ms, 0 | 20260505 13:13:19.875 | INFO | BAZWRITER : 64.8%, 4.0, 134.6ms, 0 | 20260505 13:13:19.875 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 13:13:19.875 | INFO | Uploaded 36GiB to GPU in 2.19412s (16.4075GiB/s) | 20260505 13:13:19.875 | INFO | Downloaded 38GiB to GPU in 2.28454s (16.6336GiB/s) | 20260505 13:13:25.020 | INFO | Analyzing chunk frames = [3478016,3478528) | 20260505 13:13:25.020 | INFO | MemInfo: { | 20260505 13:13:25.020 | INFO | "gpu" : " 17 GB", | 20260505 13:13:25.020 | INFO | "huge" : "", | 20260505 13:13:25.020 | INFO | "huge_shared" : " 27 GB", | 20260505 13:13:25.020 | INFO | "malloc" : " 14 GB", | 20260505 13:13:25.020 | INFO | "pinned" : " 29 GB" | 20260505 13:13:25.020 | INFO | } | 20260505 13:13:25.020 | INFO | Perf report for the last 5.14496 seconds: | 20260505 13:13:25.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:13:25.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:25.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:25.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:25.020 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 13:13:25.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:25.020 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:25.020 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 13:13:25.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:25.020 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:25.020 | INFO | Uploaded 36GiB to GPU in 2.19441s (16.4053GiB/s) | 20260505 13:13:25.020 | INFO | Downloaded 34GiB to GPU in 2.03158s (16.7358GiB/s) | 20260505 13:13:30.146 | INFO | Analyzing chunk frames = [3478528,3479040) | 20260505 13:13:30.146 | INFO | MemInfo: { | 20260505 13:13:30.146 | INFO | "gpu" : " 17 GB", | 20260505 13:13:30.146 | INFO | "huge" : "", | 20260505 13:13:30.146 | INFO | "huge_shared" : " 27 GB", | 20260505 13:13:30.146 | INFO | "malloc" : " 14 GB", | 20260505 13:13:30.146 | INFO | "pinned" : " 29 GB" | 20260505 13:13:30.146 | INFO | } | 20260505 13:13:30.146 | INFO | Perf report for the last 5.12622 seconds: | 20260505 13:13:30.146 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:13:30.146 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:30.146 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:30.146 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:30.146 | INFO | ANALYSIS : 63.8%, 3.0, 99.0ms, 0 | 20260505 13:13:30.146 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:30.146 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:30.146 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 13:13:30.146 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:30.146 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:30.146 | INFO | Uploaded 36GiB to GPU in 2.23892s (16.0792GiB/s) | 20260505 13:13:30.146 | INFO | Downloaded 34GiB to GPU in 2.10877s (16.1231GiB/s) | 20260505 13:13:35.281 | INFO | Analyzing chunk frames = [3479040,3479552) | 20260505 13:13:35.281 | INFO | MemInfo: { | 20260505 13:13:35.281 | INFO | "gpu" : " 17 GB", | 20260505 13:13:35.281 | INFO | "huge" : "", | 20260505 13:13:35.281 | INFO | "huge_shared" : " 27 GB", | 20260505 13:13:35.281 | INFO | "malloc" : " 14 GB", | 20260505 13:13:35.281 | INFO | "pinned" : " 29 GB" | 20260505 13:13:35.281 | INFO | } | 20260505 13:13:35.281 | INFO | Perf report for the last 5.1351 seconds: | 20260505 13:13:35.281 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:13:35.281 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:35.281 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:35.281 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:35.281 | INFO | ANALYSIS : 63.2%, 3.0, 98.1ms, 0 | 20260505 13:13:35.281 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:35.281 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:35.281 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 13:13:35.281 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:35.281 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:35.282 | INFO | Uploaded 36GiB to GPU in 2.2092s (16.2955GiB/s) | 20260505 13:13:35.282 | INFO | Downloaded 34GiB to GPU in 2.13086s (15.956GiB/s) | 20260505 13:13:40.430 | INFO | Analyzing chunk frames = [3479552,3480064) | 20260505 13:13:40.430 | INFO | MemInfo: { | 20260505 13:13:40.430 | INFO | "gpu" : " 17 GB", | 20260505 13:13:40.430 | INFO | "huge" : "", | 20260505 13:13:40.430 | INFO | "huge_shared" : " 27 GB", | 20260505 13:13:40.430 | INFO | "malloc" : " 14 GB", | 20260505 13:13:40.431 | INFO | "pinned" : " 29 GB" | 20260505 13:13:40.431 | INFO | } | 20260505 13:13:40.431 | INFO | Perf report for the last 5.14929 seconds: | 20260505 13:13:40.431 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:13:40.431 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:40.431 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:40.431 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:40.431 | INFO | ANALYSIS : 62.9%, 3.0, 98.4ms, 0 | 20260505 13:13:40.431 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:40.431 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:40.431 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 13:13:40.431 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:40.431 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:40.431 | INFO | Uploaded 36GiB to GPU in 2.18548s (16.4724GiB/s) | 20260505 13:13:40.431 | INFO | Downloaded 34GiB to GPU in 2.12939s (15.967GiB/s) | 20260505 13:13:45.568 | INFO | Analyzing chunk frames = [3480064,3480576) | 20260505 13:13:45.568 | INFO | MemInfo: { | 20260505 13:13:45.568 | INFO | "gpu" : " 17 GB", | 20260505 13:13:45.568 | INFO | "huge" : "", | 20260505 13:13:45.568 | INFO | "huge_shared" : " 27 GB", | 20260505 13:13:45.568 | INFO | "malloc" : " 14 GB", | 20260505 13:13:45.568 | INFO | "pinned" : " 29 GB" | 20260505 13:13:45.568 | INFO | } | 20260505 13:13:45.568 | INFO | Perf report for the last 5.13729 seconds: | 20260505 13:13:45.568 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:13:45.568 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:45.568 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:45.568 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:45.568 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 13:13:45.568 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:45.568 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:45.568 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 13:13:45.568 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:45.568 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:45.568 | INFO | Uploaded 36GiB to GPU in 2.20183s (16.35GiB/s) | 20260505 13:13:45.568 | INFO | Downloaded 34GiB to GPU in 2.11074s (16.1081GiB/s) | 20260505 13:13:50.705 | INFO | Analyzing chunk frames = [3480576,3481088) | 20260505 13:13:50.705 | INFO | MemInfo: { | 20260505 13:13:50.705 | INFO | "gpu" : " 17 GB", | 20260505 13:13:50.705 | INFO | "huge" : "", | 20260505 13:13:50.705 | INFO | "huge_shared" : " 27 GB", | 20260505 13:13:50.705 | INFO | "malloc" : " 14 GB", | 20260505 13:13:50.705 | INFO | "pinned" : " 29 GB" | 20260505 13:13:50.705 | INFO | } | 20260505 13:13:50.705 | INFO | Perf report for the last 5.13734 seconds: | 20260505 13:13:50.705 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:13:50.705 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:50.705 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:50.705 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:50.705 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 13:13:50.705 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:50.705 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:50.705 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 13:13:50.705 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:50.705 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:50.706 | INFO | Uploaded 36GiB to GPU in 2.16029s (16.6644GiB/s) | 20260505 13:13:50.706 | INFO | Downloaded 34GiB to GPU in 2.14199s (15.8731GiB/s) | 20260505 13:13:55.837 | INFO | Analyzing chunk frames = [3481088,3481600) | 20260505 13:13:55.837 | INFO | MemInfo: { | 20260505 13:13:55.837 | INFO | "gpu" : " 17 GB", | 20260505 13:13:55.837 | INFO | "huge" : "", | 20260505 13:13:55.837 | INFO | "huge_shared" : " 27 GB", | 20260505 13:13:55.837 | INFO | "malloc" : " 14 GB", | 20260505 13:13:55.837 | INFO | "pinned" : " 29 GB" | 20260505 13:13:55.837 | INFO | } | 20260505 13:13:55.837 | INFO | Perf report for the last 5.13168 seconds: | 20260505 13:13:55.837 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:13:55.837 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:55.837 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:55.837 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:55.837 | INFO | ANALYSIS : 63.8%, 3.0, 99.5ms, 0 | 20260505 13:13:55.837 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:55.837 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:55.837 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 13:13:55.837 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:55.837 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:13:55.837 | INFO | Uploaded 36GiB to GPU in 2.18299s (16.4911GiB/s) | 20260505 13:13:55.837 | INFO | Downloaded 34GiB to GPU in 2.11566s (16.0706GiB/s) | 20260505 13:13:55.838 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 13:13:55.838 | INFO | Total mem usage: 15.3 GiB | 20260505 13:13:55.838 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 13:13:55.838 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.74%) | 20260505 13:13:55.838 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 13:13:55.838 | INFO | High Water Breakdown: | 20260505 13:13:55.838 | INFO | 61% : 8.68 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 13:13:55.838 | INFO | 38.9% : 5.53 GiB ( 83.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 13:13:55.838 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 13:13:55.838 | INFO | ------------------------------------------ | 20260505 13:13:55.838 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:13:55.838 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 13:13:55.838 | INFO | Total mem usage: 26.5 GiB | 20260505 13:13:55.838 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 13:13:55.838 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 13:13:55.838 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 13:13:55.838 | INFO | High Water Breakdown: | 20260505 13:13:55.838 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:13:55.838 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:13:55.838 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:13:55.838 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:13:55.838 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:13:55.838 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:13:55.838 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 13:13:55.838 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 13:13:55.838 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:13:55.838 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 13:13:55.838 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 13:13:55.838 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 13:13:55.838 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 13:13:55.838 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 13:13:55.838 | INFO | ------------------------------------------ | 20260505 13:13:55.838 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:13:55.838 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 13:13:55.838 | INFO | Total mem usage: 25.6 GiB | 20260505 13:13:55.838 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 13:13:55.838 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 13:13:55.838 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 13:13:55.838 | INFO | High Water Breakdown: | 20260505 13:13:55.838 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 13:13:55.838 | INFO | ------------------------------------------ | 20260505 13:13:55.838 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:13:55.838 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 13:13:55.838 | INFO | Total mem usage: 15.7 GiB | 20260505 13:13:55.838 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 13:13:55.838 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 13:13:55.838 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 13:13:55.838 | INFO | High Water Breakdown: | 20260505 13:13:55.838 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 13:13:55.838 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 13:13:55.838 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 13:13:55.838 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 13:13:55.838 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 13:13:55.838 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 13:13:55.838 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 13:13:55.838 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 13:13:55.838 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 13:13:55.838 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:13:55.838 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 13:13:55.838 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:13:55.838 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 13:13:55.838 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 13:13:55.838 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 13:13:55.838 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 13:13:55.838 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:13:55.838 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 13:13:55.838 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:13:55.838 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:13:55.838 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:13:55.838 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 13:13:55.838 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 13:13:55.838 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:13:55.838 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 13:13:55.839 | INFO | ------------------------------------------ | 20260505 13:13:55.839 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:13:55.839 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 13:13:55.839 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:13:55.839 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:13:55.839 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.69e+01, 1.39e+01 / 1.67e+00 | 20260505 13:13:55.839 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 1.01e-01, 1.03e-02 / 3.95e-03 | 20260505 13:13:55.839 | INFO | PerfReport: 11.76%, Baseline -- 3.76e+00 / 4.51e+00, 5.93e-01 / 1.73e-01 | 20260505 13:13:55.839 | INFO | PerfReport: 3.30%, Histogram -- 1.05e+00 / 1.50e+00, 2.01e-01 / 8.18e-02 | 20260505 13:13:55.839 | INFO | PerfReport: 15.94%, DME -- 4.07e+01 / 4.58e+01, 4.66e+00 / 1.99e+00 | 20260505 13:13:55.839 | INFO | PerfReport: 56.44%, FrameLabeling -- 1.80e+01 / 2.02e+01, 1.48e+00 / 1.92e+00 | 20260505 13:13:55.839 | INFO | PerfReport: 7.41%, PulseAccumulating -- 2.37e+00 / 2.93e+00, 4.46e-01 / 1.76e-01 | 20260505 13:13:55.839 | INFO | PerfReport: 3.59%, Metrics -- 1.15e+00 / 1.49e+00, 3.24e-01 / 1.72e-01 | 20260505 13:13:55.839 | INFO | PerfReport: 1.27%, PulseSerializing -- 4.04e-01 / 4.60e-01, 7.44e-02 / 2.70e-02 | 20260505 13:13:55.839 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 13:13:55.839 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:13:55.839 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:13:55.839 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260505 13:13:55.839 | INFO | PerfReport: 22.62%, Upload -- 2.28e+01 / 0.00e+00, 8.11e+00 / 0.00e+00 | 20260505 13:13:55.839 | INFO | PerfReport: 21.99%, Download -- 2.22e+01 / 0.00e+00, 8.11e+00 / 0.00e+00 | 20260505 13:13:55.839 | INFO | PerfReport: 31.64%, Compute -- 3.19e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 13:13:55.839 | INFO | PerfReport: 2.24%, LockUpload -- 2.26e+00 / 0.00e+00, 7.85e+00 / 0.00e+00 | 20260505 13:13:55.839 | INFO | PerfReport: 1.32%, LockDownload -- 1.34e+00 / 0.00e+00, 2.82e+00 / 0.00e+00 | 20260505 13:13:55.839 | INFO | PerfReport: 20.16%, LockCompute -- 2.04e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260505 13:13:55.839 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.96e-02 / 0.00e+00 | 20260505 13:13:55.973 | INFO | Starting a round of Baz IO | 20260505 13:13:59.221 | INFO | Populating RTMetrics report | 20260505 13:13:59.221 | INFO | RTMetrics report populated | 20260505 13:13:59.221 | INFO | Opening tmp RTMetrics json file | 20260505 13:13:59.238 | INFO | Serializing RTMetrics to json | 20260505 13:13:59.256 | INFO | Serialization done, swapping tmp file now... | 20260505 13:13:59.269 | INFO | RTMetrics done | 20260505 13:13:59.350 | INFO | Finished a round of Baz IO: Wrote 2.31 GiB in 3.377 seconds | 20260505 13:13:59.350 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.21% / 0.00% / 76.79% | 20260505 13:13:59.350 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.740631% / 3.97364e-06% / 0.000119209% | 20260505 13:13:59.350 | INFO | Baz IO overall: | 20260505 13:13:59.350 | INFO | BytesWritten : 2072999868324 | 20260505 13:13:59.350 | INFO | NumEvents : 986375505123 | 20260505 13:14:00.982 | INFO | Analyzing chunk frames = [3481600,3482112) | 20260505 13:14:00.982 | INFO | MemInfo: { | 20260505 13:14:00.982 | INFO | "gpu" : " 17 GB", | 20260505 13:14:00.982 | INFO | "huge" : "", | 20260505 13:14:00.982 | INFO | "huge_shared" : " 27 GB", | 20260505 13:14:00.982 | INFO | "malloc" : " 14 GB", | 20260505 13:14:00.982 | INFO | "pinned" : " 29 GB" | 20260505 13:14:00.982 | INFO | } | 20260505 13:14:00.982 | INFO | Perf report for the last 5.14525 seconds: | 20260505 13:14:00.982 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:14:00.982 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:00.982 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:00.982 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:00.982 | INFO | ANALYSIS : 64.7%, 2.9, 99.9ms, 0 | 20260505 13:14:00.982 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:00.982 | INFO | RT_METRICS : 46.0%, 1.0, 24.2ms, 0 | 20260505 13:14:00.982 | INFO | PRE_HQ : 61.2%, 1.8, 58.3ms, 0 | 20260505 13:14:00.983 | INFO | BAZWRITER : 64.9%, 4.1, 136.8ms, 0 | 20260505 13:14:00.983 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 13:14:00.983 | INFO | Uploaded 36GiB to GPU in 2.1923s (16.4211GiB/s) | 20260505 13:14:00.983 | INFO | Downloaded 38GiB to GPU in 2.30333s (16.4978GiB/s) | 20260505 13:14:06.127 | INFO | Analyzing chunk frames = [3482112,3482624) | 20260505 13:14:06.127 | INFO | MemInfo: { | 20260505 13:14:06.127 | INFO | "gpu" : " 17 GB", | 20260505 13:14:06.127 | INFO | "huge" : "", | 20260505 13:14:06.127 | INFO | "huge_shared" : " 27 GB", | 20260505 13:14:06.127 | INFO | "malloc" : " 14 GB", | 20260505 13:14:06.127 | INFO | "pinned" : " 29 GB" | 20260505 13:14:06.127 | INFO | } | 20260505 13:14:06.127 | INFO | Perf report for the last 5.14443 seconds: | 20260505 13:14:06.127 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:14:06.127 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:06.127 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:06.127 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:06.127 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 13:14:06.127 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:06.127 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:06.127 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 13:14:06.127 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:06.127 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:06.127 | INFO | Uploaded 36GiB to GPU in 2.16579s (16.6221GiB/s) | 20260505 13:14:06.127 | INFO | Downloaded 34GiB to GPU in 2.02786s (16.7665GiB/s) | 20260505 13:14:11.281 | INFO | Analyzing chunk frames = [3482624,3483136) | 20260505 13:14:11.281 | INFO | MemInfo: { | 20260505 13:14:11.281 | INFO | "gpu" : " 17 GB", | 20260505 13:14:11.281 | INFO | "huge" : "", | 20260505 13:14:11.281 | INFO | "huge_shared" : " 27 GB", | 20260505 13:14:11.281 | INFO | "malloc" : " 14 GB", | 20260505 13:14:11.281 | INFO | "pinned" : " 29 GB" | 20260505 13:14:11.281 | INFO | } | 20260505 13:14:11.281 | INFO | Perf report for the last 5.15406 seconds: | 20260505 13:14:11.281 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:14:11.281 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:11.281 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:11.281 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:11.281 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 13:14:11.281 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:11.281 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:11.281 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:14:11.281 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:11.281 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:11.281 | INFO | Uploaded 36GiB to GPU in 2.23037s (16.1408GiB/s) | 20260505 13:14:11.281 | INFO | Downloaded 34GiB to GPU in 2.09774s (16.2079GiB/s) | 20260505 13:14:16.403 | INFO | Analyzing chunk frames = [3483136,3483648) | 20260505 13:14:16.403 | INFO | MemInfo: { | 20260505 13:14:16.403 | INFO | "gpu" : " 17 GB", | 20260505 13:14:16.403 | INFO | "huge" : "", | 20260505 13:14:16.403 | INFO | "huge_shared" : " 27 GB", | 20260505 13:14:16.403 | INFO | "malloc" : " 14 GB", | 20260505 13:14:16.403 | INFO | "pinned" : " 29 GB" | 20260505 13:14:16.403 | INFO | } | 20260505 13:14:16.403 | INFO | Perf report for the last 5.12207 seconds: | 20260505 13:14:16.403 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:14:16.403 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:16.403 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:16.403 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:16.403 | INFO | ANALYSIS : 62.9%, 3.0, 97.4ms, 0 | 20260505 13:14:16.403 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:16.403 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:16.403 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 13:14:16.403 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:16.403 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:16.403 | INFO | Uploaded 36GiB to GPU in 2.18709s (16.4602GiB/s) | 20260505 13:14:16.403 | INFO | Downloaded 34GiB to GPU in 2.13182s (15.9488GiB/s) | 20260505 13:14:21.535 | INFO | Analyzing chunk frames = [3483648,3484160) | 20260505 13:14:21.535 | INFO | MemInfo: { | 20260505 13:14:21.535 | INFO | "gpu" : " 17 GB", | 20260505 13:14:21.535 | INFO | "huge" : "", | 20260505 13:14:21.535 | INFO | "huge_shared" : " 27 GB", | 20260505 13:14:21.535 | INFO | "malloc" : " 14 GB", | 20260505 13:14:21.535 | INFO | "pinned" : " 29 GB" | 20260505 13:14:21.535 | INFO | } | 20260505 13:14:21.535 | INFO | Perf report for the last 5.13204 seconds: | 20260505 13:14:21.535 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:14:21.535 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:21.535 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:21.535 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:21.535 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 13:14:21.535 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:21.535 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:21.535 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 13:14:21.535 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:21.535 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:21.535 | INFO | Uploaded 36GiB to GPU in 2.14092s (16.8152GiB/s) | 20260505 13:14:21.535 | INFO | Downloaded 34GiB to GPU in 2.12603s (15.9923GiB/s) | 20260505 13:14:26.684 | INFO | Analyzing chunk frames = [3484160,3484672) | 20260505 13:14:26.684 | INFO | MemInfo: { | 20260505 13:14:26.684 | INFO | "gpu" : " 17 GB", | 20260505 13:14:26.684 | INFO | "huge" : "", | 20260505 13:14:26.684 | INFO | "huge_shared" : " 27 GB", | 20260505 13:14:26.684 | INFO | "malloc" : " 14 GB", | 20260505 13:14:26.684 | INFO | "pinned" : " 29 GB" | 20260505 13:14:26.684 | INFO | } | 20260505 13:14:26.684 | INFO | Perf report for the last 5.14939 seconds: | 20260505 13:14:26.684 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:14:26.684 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:26.684 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:26.684 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:26.684 | INFO | ANALYSIS : 63.0%, 3.0, 98.6ms, 0 | 20260505 13:14:26.684 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:26.684 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:26.684 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 13:14:26.684 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:26.684 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:26.685 | INFO | Uploaded 36GiB to GPU in 2.14258s (16.8021GiB/s) | 20260505 13:14:26.685 | INFO | Downloaded 34GiB to GPU in 2.11113s (16.1051GiB/s) | 20260505 13:14:31.821 | INFO | Analyzing chunk frames = [3484672,3485184) | 20260505 13:14:31.821 | INFO | MemInfo: { | 20260505 13:14:31.821 | INFO | "gpu" : " 17 GB", | 20260505 13:14:31.821 | INFO | "huge" : "", | 20260505 13:14:31.821 | INFO | "huge_shared" : " 27 GB", | 20260505 13:14:31.821 | INFO | "malloc" : " 14 GB", | 20260505 13:14:31.821 | INFO | "pinned" : " 29 GB" | 20260505 13:14:31.821 | INFO | } | 20260505 13:14:31.821 | INFO | Perf report for the last 5.13699 seconds: | 20260505 13:14:31.821 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:14:31.821 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:31.821 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:31.821 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:31.821 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 13:14:31.821 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:31.821 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:31.821 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 13:14:31.821 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:31.821 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:31.822 | INFO | Uploaded 36GiB to GPU in 2.18309s (16.4904GiB/s) | 20260505 13:14:31.822 | INFO | Downloaded 34GiB to GPU in 2.15104s (15.8063GiB/s) | 20260505 13:14:36.962 | INFO | Analyzing chunk frames = [3485184,3485696) | 20260505 13:14:36.962 | INFO | MemInfo: { | 20260505 13:14:36.962 | INFO | "gpu" : " 17 GB", | 20260505 13:14:36.962 | INFO | "huge" : "", | 20260505 13:14:36.962 | INFO | "huge_shared" : " 27 GB", | 20260505 13:14:36.962 | INFO | "malloc" : " 14 GB", | 20260505 13:14:36.962 | INFO | "pinned" : " 29 GB" | 20260505 13:14:36.963 | INFO | } | 20260505 13:14:36.963 | INFO | Perf report for the last 5.14136 seconds: | 20260505 13:14:36.963 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:14:36.963 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:36.963 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:36.963 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:36.963 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 13:14:36.963 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:36.963 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:36.963 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 13:14:36.963 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:36.963 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:36.963 | INFO | Uploaded 36GiB to GPU in 2.15716s (16.6886GiB/s) | 20260505 13:14:36.963 | INFO | Downloaded 34GiB to GPU in 2.10646s (16.1408GiB/s) | 20260505 13:14:37.104 | INFO | Starting a round of Baz IO | 20260505 13:14:40.334 | INFO | Populating RTMetrics report | 20260505 13:14:40.334 | INFO | RTMetrics report populated | 20260505 13:14:40.334 | INFO | Opening tmp RTMetrics json file | 20260505 13:14:40.358 | INFO | Serializing RTMetrics to json | 20260505 13:14:40.363 | INFO | Serialization done, swapping tmp file now... | 20260505 13:14:40.365 | INFO | RTMetrics done | 20260505 13:14:40.456 | INFO | Finished a round of Baz IO: Wrote 2.308 GiB in 3.351 seconds | 20260505 13:14:40.456 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.20% / 0.00% / 76.79% | 20260505 13:14:40.456 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.736845% / 1.19209e-05% / 0.000103315% | 20260505 13:14:40.456 | INFO | Baz IO overall: | 20260505 13:14:40.456 | INFO | BytesWritten : 2075477805717 | 20260505 13:14:40.456 | INFO | NumEvents : 987548898111 | 20260505 13:14:42.100 | INFO | Analyzing chunk frames = [3485696,3486208) | 20260505 13:14:42.100 | INFO | MemInfo: { | 20260505 13:14:42.100 | INFO | "gpu" : " 17 GB", | 20260505 13:14:42.100 | INFO | "huge" : "", | 20260505 13:14:42.100 | INFO | "huge_shared" : " 27 GB", | 20260505 13:14:42.100 | INFO | "malloc" : " 14 GB", | 20260505 13:14:42.100 | INFO | "pinned" : " 29 GB" | 20260505 13:14:42.100 | INFO | } | 20260505 13:14:42.100 | INFO | Perf report for the last 5.13715 seconds: | 20260505 13:14:42.100 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:14:42.100 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:42.100 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:42.100 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:42.100 | INFO | ANALYSIS : 64.5%, 3.0, 100.0ms, 0 | 20260505 13:14:42.100 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:42.100 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260505 13:14:42.100 | INFO | PRE_HQ : 61.0%, 1.8, 57.7ms, 0 | 20260505 13:14:42.100 | INFO | BAZWRITER : 64.6%, 4.0, 134.3ms, 0 | 20260505 13:14:42.100 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 13:14:42.100 | INFO | Uploaded 36GiB to GPU in 2.19522s (16.3993GiB/s) | 20260505 13:14:42.100 | INFO | Downloaded 38GiB to GPU in 2.27825s (16.6794GiB/s) | 20260505 13:14:47.238 | INFO | Analyzing chunk frames = [3486208,3486720) | 20260505 13:14:47.238 | INFO | MemInfo: { | 20260505 13:14:47.238 | INFO | "gpu" : " 17 GB", | 20260505 13:14:47.238 | INFO | "huge" : "", | 20260505 13:14:47.238 | INFO | "huge_shared" : " 27 GB", | 20260505 13:14:47.238 | INFO | "malloc" : " 14 GB", | 20260505 13:14:47.238 | INFO | "pinned" : " 29 GB" | 20260505 13:14:47.238 | INFO | } | 20260505 13:14:47.238 | INFO | Perf report for the last 5.13837 seconds: | 20260505 13:14:47.238 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:14:47.238 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:47.238 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:47.238 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:47.238 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 13:14:47.238 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:47.238 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:47.238 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 13:14:47.238 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:47.238 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:47.238 | INFO | Uploaded 36GiB to GPU in 2.20059s (16.3593GiB/s) | 20260505 13:14:47.238 | INFO | Downloaded 34GiB to GPU in 2.03705s (16.6908GiB/s) | 20260505 13:14:52.382 | INFO | Analyzing chunk frames = [3486720,3487232) | 20260505 13:14:52.382 | INFO | MemInfo: { | 20260505 13:14:52.382 | INFO | "gpu" : " 17 GB", | 20260505 13:14:52.382 | INFO | "huge" : "", | 20260505 13:14:52.382 | INFO | "huge_shared" : " 27 GB", | 20260505 13:14:52.382 | INFO | "malloc" : " 14 GB", | 20260505 13:14:52.382 | INFO | "pinned" : " 29 GB" | 20260505 13:14:52.382 | INFO | } | 20260505 13:14:52.382 | INFO | Perf report for the last 5.14412 seconds: | 20260505 13:14:52.382 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:14:52.382 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:52.382 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:52.382 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:52.382 | INFO | ANALYSIS : 63.2%, 3.0, 98.4ms, 0 | 20260505 13:14:52.382 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:52.382 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:52.382 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:14:52.382 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:52.382 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:52.383 | INFO | Uploaded 36GiB to GPU in 2.16821s (16.6036GiB/s) | 20260505 13:14:52.383 | INFO | Downloaded 34GiB to GPU in 2.13339s (15.9371GiB/s) | 20260505 13:14:57.521 | INFO | Analyzing chunk frames = [3487232,3487744) | 20260505 13:14:57.521 | INFO | MemInfo: { | 20260505 13:14:57.521 | INFO | "gpu" : " 17 GB", | 20260505 13:14:57.521 | INFO | "huge" : "", | 20260505 13:14:57.521 | INFO | "huge_shared" : " 27 GB", | 20260505 13:14:57.521 | INFO | "malloc" : " 14 GB", | 20260505 13:14:57.521 | INFO | "pinned" : " 29 GB" | 20260505 13:14:57.521 | INFO | } | 20260505 13:14:57.521 | INFO | Perf report for the last 5.13901 seconds: | 20260505 13:14:57.521 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:14:57.521 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:57.521 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:57.521 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:57.521 | INFO | ANALYSIS : 63.0%, 3.0, 97.8ms, 0 | 20260505 13:14:57.521 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:57.521 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:57.521 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 13:14:57.521 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:57.521 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:14:57.522 | INFO | Uploaded 36GiB to GPU in 2.1911s (16.4301GiB/s) | 20260505 13:14:57.522 | INFO | Downloaded 34GiB to GPU in 2.13158s (15.9506GiB/s) | 20260505 13:15:02.652 | INFO | Analyzing chunk frames = [3487744,3488256) | 20260505 13:15:02.652 | INFO | MemInfo: { | 20260505 13:15:02.652 | INFO | "gpu" : " 17 GB", | 20260505 13:15:02.652 | INFO | "huge" : "", | 20260505 13:15:02.652 | INFO | "huge_shared" : " 27 GB", | 20260505 13:15:02.652 | INFO | "malloc" : " 14 GB", | 20260505 13:15:02.652 | INFO | "pinned" : " 29 GB" | 20260505 13:15:02.652 | INFO | } | 20260505 13:15:02.652 | INFO | Perf report for the last 5.1307 seconds: | 20260505 13:15:02.652 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:15:02.652 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:02.652 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:02.652 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:02.652 | INFO | ANALYSIS : 62.9%, 3.0, 98.0ms, 0 | 20260505 13:15:02.652 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:02.652 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:02.652 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 13:15:02.652 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:02.652 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:02.652 | INFO | Uploaded 36GiB to GPU in 2.14443s (16.7877GiB/s) | 20260505 13:15:02.652 | INFO | Downloaded 34GiB to GPU in 2.12199s (16.0227GiB/s) | 20260505 13:15:07.788 | INFO | Analyzing chunk frames = [3488256,3488768) | 20260505 13:15:07.788 | INFO | MemInfo: { | 20260505 13:15:07.788 | INFO | "gpu" : " 17 GB", | 20260505 13:15:07.788 | INFO | "huge" : "", | 20260505 13:15:07.788 | INFO | "huge_shared" : " 27 GB", | 20260505 13:15:07.788 | INFO | "malloc" : " 14 GB", | 20260505 13:15:07.788 | INFO | "pinned" : " 29 GB" | 20260505 13:15:07.788 | INFO | } | 20260505 13:15:07.788 | INFO | Perf report for the last 5.13586 seconds: | 20260505 13:15:07.788 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:15:07.788 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:07.788 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:07.788 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:07.788 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 13:15:07.788 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:07.788 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:07.788 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:15:07.788 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:07.788 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:07.788 | INFO | Uploaded 36GiB to GPU in 2.18462s (16.4789GiB/s) | 20260505 13:15:07.788 | INFO | Downloaded 34GiB to GPU in 2.11817s (16.0516GiB/s) | 20260505 13:15:12.937 | INFO | Analyzing chunk frames = [3488768,3489280) | 20260505 13:15:12.937 | INFO | MemInfo: { | 20260505 13:15:12.937 | INFO | "gpu" : " 17 GB", | 20260505 13:15:12.937 | INFO | "huge" : "", | 20260505 13:15:12.937 | INFO | "huge_shared" : " 27 GB", | 20260505 13:15:12.937 | INFO | "malloc" : " 14 GB", | 20260505 13:15:12.937 | INFO | "pinned" : " 29 GB" | 20260505 13:15:12.937 | INFO | } | 20260505 13:15:12.938 | INFO | Perf report for the last 5.14959 seconds: | 20260505 13:15:12.938 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:15:12.938 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:12.938 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:12.938 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:12.938 | INFO | ANALYSIS : 63.1%, 3.0, 98.7ms, 0 | 20260505 13:15:12.938 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:12.938 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:12.938 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 13:15:12.938 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:12.938 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:12.938 | INFO | Uploaded 36GiB to GPU in 2.16731s (16.6104GiB/s) | 20260505 13:15:12.938 | INFO | Downloaded 34GiB to GPU in 2.14893s (15.8218GiB/s) | 20260505 13:15:18.081 | INFO | Analyzing chunk frames = [3489280,3489792) | 20260505 13:15:18.081 | INFO | MemInfo: { | 20260505 13:15:18.081 | INFO | "gpu" : " 17 GB", | 20260505 13:15:18.081 | INFO | "huge" : "", | 20260505 13:15:18.081 | INFO | "huge_shared" : " 27 GB", | 20260505 13:15:18.081 | INFO | "malloc" : " 14 GB", | 20260505 13:15:18.081 | INFO | "pinned" : " 29 GB" | 20260505 13:15:18.081 | INFO | } | 20260505 13:15:18.081 | INFO | Perf report for the last 5.14369 seconds: | 20260505 13:15:18.081 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:15:18.081 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:18.081 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:18.081 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:18.081 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 13:15:18.081 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:18.081 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:18.081 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:15:18.081 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:18.081 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:18.081 | INFO | Uploaded 36GiB to GPU in 2.21591s (16.2461GiB/s) | 20260505 13:15:18.082 | INFO | Downloaded 34GiB to GPU in 2.10602s (16.1442GiB/s) | 20260505 13:15:18.082 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 13:15:18.082 | INFO | Total mem usage: 15.3 GiB | 20260505 13:15:18.082 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 13:15:18.082 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -7.03%) | 20260505 13:15:18.082 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 13:15:18.082 | INFO | High Water Breakdown: | 20260505 13:15:18.082 | INFO | 61.1% : 8.75 GiB ( 91.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 13:15:18.082 | INFO | 38.8% : 5.55 GiB ( 83.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 13:15:18.082 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 13:15:18.082 | INFO | ------------------------------------------ | 20260505 13:15:18.082 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:15:18.082 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 13:15:18.082 | INFO | Total mem usage: 26.5 GiB | 20260505 13:15:18.082 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 13:15:18.082 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 13:15:18.082 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 13:15:18.082 | INFO | High Water Breakdown: | 20260505 13:15:18.082 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:15:18.082 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:15:18.082 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:15:18.082 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:15:18.082 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:15:18.082 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:15:18.082 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 13:15:18.082 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 13:15:18.082 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:15:18.082 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 13:15:18.082 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 13:15:18.082 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 13:15:18.082 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 13:15:18.082 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 13:15:18.082 | INFO | ------------------------------------------ | 20260505 13:15:18.082 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:15:18.082 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 13:15:18.082 | INFO | Total mem usage: 25.6 GiB | 20260505 13:15:18.082 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 13:15:18.082 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 13:15:18.082 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 13:15:18.082 | INFO | High Water Breakdown: | 20260505 13:15:18.082 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 13:15:18.082 | INFO | ------------------------------------------ | 20260505 13:15:18.082 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:15:18.082 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 13:15:18.082 | INFO | Total mem usage: 15.7 GiB | 20260505 13:15:18.082 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 13:15:18.082 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 13:15:18.082 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 13:15:18.082 | INFO | High Water Breakdown: | 20260505 13:15:18.082 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 13:15:18.082 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 13:15:18.082 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 13:15:18.082 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 13:15:18.082 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 13:15:18.082 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 13:15:18.082 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 13:15:18.082 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 13:15:18.082 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 13:15:18.082 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:15:18.082 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 13:15:18.082 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:15:18.082 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 13:15:18.082 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 13:15:18.082 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 13:15:18.082 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 13:15:18.082 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:15:18.082 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 13:15:18.082 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:15:18.082 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:15:18.082 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:15:18.082 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 13:15:18.082 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 13:15:18.082 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:15:18.082 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 13:15:18.082 | INFO | ------------------------------------------ | 20260505 13:15:18.082 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:15:18.083 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 13:15:18.083 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:15:18.083 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:15:18.083 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.68e+01, 1.39e+01 / 1.47e+00 | 20260505 13:15:18.083 | INFO | PerfReport: 0.29%, AggregateStats -- 9.27e-02 / 9.87e-02, 1.03e-02 / 4.49e-03 | 20260505 13:15:18.083 | INFO | PerfReport: 11.79%, Baseline -- 3.76e+00 / 4.47e+00, 5.93e-01 / 1.94e-01 | 20260505 13:15:18.083 | INFO | PerfReport: 3.31%, Histogram -- 1.06e+00 / 1.48e+00, 2.02e-01 / 1.07e-01 | 20260505 13:15:18.083 | INFO | PerfReport: 16.02%, DME -- 4.09e+01 / 4.61e+01, 4.59e+00 / 2.11e+00 | 20260505 13:15:18.083 | INFO | PerfReport: 56.30%, FrameLabeling -- 1.80e+01 / 1.98e+01, 1.41e+00 / 1.89e+00 | 20260505 13:15:18.083 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.38e+00 / 2.91e+00, 4.50e-01 / 1.97e-01 | 20260505 13:15:18.083 | INFO | PerfReport: 3.59%, Metrics -- 1.15e+00 / 1.47e+00, 3.18e-01 / 1.44e-01 | 20260505 13:15:18.083 | INFO | PerfReport: 1.26%, PulseSerializing -- 4.02e-01 / 4.52e-01, 7.13e-02 / 2.77e-02 | 20260505 13:15:18.083 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 13:15:18.083 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:15:18.083 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:15:18.083 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260505 13:15:18.083 | INFO | PerfReport: 22.51%, Upload -- 2.27e+01 / 0.00e+00, 8.02e+00 / 0.00e+00 | 20260505 13:15:18.083 | INFO | PerfReport: 22.03%, Download -- 2.22e+01 / 0.00e+00, 8.08e+00 / 0.00e+00 | 20260505 13:15:18.083 | INFO | PerfReport: 31.66%, Compute -- 3.19e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 13:15:18.083 | INFO | PerfReport: 2.18%, LockUpload -- 2.20e+00 / 0.00e+00, 7.83e+00 / 0.00e+00 | 20260505 13:15:18.083 | INFO | PerfReport: 1.45%, LockDownload -- 1.47e+00 / 0.00e+00, 3.06e+00 / 0.00e+00 | 20260505 13:15:18.083 | INFO | PerfReport: 20.15%, LockCompute -- 2.03e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260505 13:15:18.083 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 2.45e-02 / 0.00e+00 | 20260505 13:15:18.218 | INFO | Starting a round of Baz IO | 20260505 13:15:21.439 | INFO | Populating RTMetrics report | 20260505 13:15:21.440 | INFO | RTMetrics report populated | 20260505 13:15:21.440 | INFO | Opening tmp RTMetrics json file | 20260505 13:15:21.441 | INFO | Serializing RTMetrics to json | 20260505 13:15:21.450 | INFO | Serialization done, swapping tmp file now... | 20260505 13:15:21.466 | INFO | RTMetrics done | 20260505 13:15:21.556 | INFO | Finished a round of Baz IO: Wrote 2.305 GiB in 3.337 seconds | 20260505 13:15:21.556 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.20% / 0.00% / 76.80% | 20260505 13:15:21.556 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.733395% / 7.94729e-06% / 7.54992e-05% | 20260505 13:15:21.556 | INFO | Baz IO overall: | 20260505 13:15:21.556 | INFO | BytesWritten : 2077952785658 | 20260505 13:15:21.556 | INFO | NumEvents : 988720154126 | 20260505 13:15:23.223 | INFO | Analyzing chunk frames = [3489792,3490304) | 20260505 13:15:23.223 | INFO | MemInfo: { | 20260505 13:15:23.223 | INFO | "gpu" : " 17 GB", | 20260505 13:15:23.223 | INFO | "huge" : "", | 20260505 13:15:23.223 | INFO | "huge_shared" : " 27 GB", | 20260505 13:15:23.223 | INFO | "malloc" : " 14 GB", | 20260505 13:15:23.223 | INFO | "pinned" : " 29 GB" | 20260505 13:15:23.223 | INFO | } | 20260505 13:15:23.223 | INFO | Perf report for the last 5.14136 seconds: | 20260505 13:15:23.223 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:15:23.223 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:23.223 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:23.223 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:23.223 | INFO | ANALYSIS : 64.2%, 3.0, 99.5ms, 0 | 20260505 13:15:23.223 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:23.223 | INFO | RT_METRICS : 45.5%, 1.0, 23.9ms, 0 | 20260505 13:15:23.223 | INFO | PRE_HQ : 60.8%, 1.8, 57.6ms, 0 | 20260505 13:15:23.223 | INFO | BAZWRITER : 64.3%, 4.1, 134.4ms, 0 | 20260505 13:15:23.223 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 13:15:23.223 | INFO | Uploaded 36GiB to GPU in 2.20187s (16.3497GiB/s) | 20260505 13:15:23.223 | INFO | Downloaded 38GiB to GPU in 2.26361s (16.7873GiB/s) | 20260505 13:15:28.350 | INFO | Analyzing chunk frames = [3490304,3490816) | 20260505 13:15:28.350 | INFO | MemInfo: { | 20260505 13:15:28.350 | INFO | "gpu" : " 17 GB", | 20260505 13:15:28.350 | INFO | "huge" : "", | 20260505 13:15:28.350 | INFO | "huge_shared" : " 27 GB", | 20260505 13:15:28.350 | INFO | "malloc" : " 14 GB", | 20260505 13:15:28.350 | INFO | "pinned" : " 29 GB" | 20260505 13:15:28.350 | INFO | } | 20260505 13:15:28.350 | INFO | Perf report for the last 5.127 seconds: | 20260505 13:15:28.350 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:15:28.350 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:28.350 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:28.350 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:28.350 | INFO | ANALYSIS : 63.6%, 3.0, 99.1ms, 0 | 20260505 13:15:28.350 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:28.350 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:28.350 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 13:15:28.350 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:28.350 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:28.350 | INFO | Uploaded 36GiB to GPU in 2.16988s (16.5908GiB/s) | 20260505 13:15:28.350 | INFO | Downloaded 34GiB to GPU in 2.02838s (16.7621GiB/s) | 20260505 13:15:33.491 | INFO | Analyzing chunk frames = [3490816,3491328) | 20260505 13:15:33.491 | INFO | MemInfo: { | 20260505 13:15:33.491 | INFO | "gpu" : " 17 GB", | 20260505 13:15:33.491 | INFO | "huge" : "", | 20260505 13:15:33.491 | INFO | "huge_shared" : " 27 GB", | 20260505 13:15:33.492 | INFO | "malloc" : " 14 GB", | 20260505 13:15:33.492 | INFO | "pinned" : " 29 GB" | 20260505 13:15:33.492 | INFO | } | 20260505 13:15:33.492 | INFO | Perf report for the last 5.14205 seconds: | 20260505 13:15:33.492 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:15:33.492 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:33.492 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:33.492 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:33.492 | INFO | ANALYSIS : 63.2%, 3.0, 98.4ms, 0 | 20260505 13:15:33.492 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:33.492 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:33.492 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 13:15:33.492 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:33.492 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:33.492 | INFO | Uploaded 36GiB to GPU in 2.21459s (16.2558GiB/s) | 20260505 13:15:33.492 | INFO | Downloaded 34GiB to GPU in 2.12717s (15.9837GiB/s) | 20260505 13:15:38.633 | INFO | Analyzing chunk frames = [3491328,3491840) | 20260505 13:15:38.633 | INFO | MemInfo: { | 20260505 13:15:38.633 | INFO | "gpu" : " 17 GB", | 20260505 13:15:38.633 | INFO | "huge" : "", | 20260505 13:15:38.633 | INFO | "huge_shared" : " 27 GB", | 20260505 13:15:38.633 | INFO | "malloc" : " 14 GB", | 20260505 13:15:38.633 | INFO | "pinned" : " 29 GB" | 20260505 13:15:38.633 | INFO | } | 20260505 13:15:38.633 | INFO | Perf report for the last 5.14169 seconds: | 20260505 13:15:38.633 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:15:38.633 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:38.633 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:38.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:38.633 | INFO | ANALYSIS : 63.1%, 3.0, 98.1ms, 0 | 20260505 13:15:38.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:38.633 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:38.633 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 13:15:38.633 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:38.633 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:38.634 | INFO | Uploaded 36GiB to GPU in 2.20351s (16.3376GiB/s) | 20260505 13:15:38.634 | INFO | Downloaded 34GiB to GPU in 2.13286s (15.941GiB/s) | 20260505 13:15:43.777 | INFO | Analyzing chunk frames = [3491840,3492352) | 20260505 13:15:43.777 | INFO | MemInfo: { | 20260505 13:15:43.777 | INFO | "gpu" : " 17 GB", | 20260505 13:15:43.777 | INFO | "huge" : "", | 20260505 13:15:43.777 | INFO | "huge_shared" : " 27 GB", | 20260505 13:15:43.777 | INFO | "malloc" : " 14 GB", | 20260505 13:15:43.777 | INFO | "pinned" : " 29 GB" | 20260505 13:15:43.777 | INFO | } | 20260505 13:15:43.777 | INFO | Perf report for the last 5.14359 seconds: | 20260505 13:15:43.777 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:15:43.777 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:43.777 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:43.777 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:43.777 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 13:15:43.777 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:43.777 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:43.777 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 13:15:43.777 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:43.777 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:43.777 | INFO | Uploaded 36GiB to GPU in 2.16527s (16.6261GiB/s) | 20260505 13:15:43.777 | INFO | Downloaded 34GiB to GPU in 2.12768s (15.9799GiB/s) | 20260505 13:15:48.909 | INFO | Analyzing chunk frames = [3492352,3492864) | 20260505 13:15:48.909 | INFO | MemInfo: { | 20260505 13:15:48.909 | INFO | "gpu" : " 17 GB", | 20260505 13:15:48.909 | INFO | "huge" : "", | 20260505 13:15:48.909 | INFO | "huge_shared" : " 27 GB", | 20260505 13:15:48.909 | INFO | "malloc" : " 14 GB", | 20260505 13:15:48.910 | INFO | "pinned" : " 29 GB" | 20260505 13:15:48.910 | INFO | } | 20260505 13:15:48.910 | INFO | Perf report for the last 5.1326 seconds: | 20260505 13:15:48.910 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:15:48.910 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:48.910 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:48.910 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:48.910 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 13:15:48.910 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:48.910 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:48.910 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 13:15:48.910 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:48.910 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:48.910 | INFO | Uploaded 36GiB to GPU in 2.16238s (16.6483GiB/s) | 20260505 13:15:48.910 | INFO | Downloaded 34GiB to GPU in 2.09943s (16.1949GiB/s) | 20260505 13:15:54.044 | INFO | Analyzing chunk frames = [3492864,3493376) | 20260505 13:15:54.044 | INFO | MemInfo: { | 20260505 13:15:54.044 | INFO | "gpu" : " 17 GB", | 20260505 13:15:54.044 | INFO | "huge" : "", | 20260505 13:15:54.044 | INFO | "huge_shared" : " 27 GB", | 20260505 13:15:54.044 | INFO | "malloc" : " 14 GB", | 20260505 13:15:54.044 | INFO | "pinned" : " 29 GB" | 20260505 13:15:54.044 | INFO | } | 20260505 13:15:54.044 | INFO | Perf report for the last 5.13479 seconds: | 20260505 13:15:54.044 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:15:54.044 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:54.044 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:54.044 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:54.044 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 13:15:54.044 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:54.044 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:54.045 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 13:15:54.045 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:54.045 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:54.045 | INFO | Uploaded 36GiB to GPU in 2.12823s (16.9155GiB/s) | 20260505 13:15:54.045 | INFO | Downloaded 34GiB to GPU in 2.15514s (15.7763GiB/s) | 20260505 13:15:59.195 | INFO | Analyzing chunk frames = [3493376,3493888) | 20260505 13:15:59.195 | INFO | MemInfo: { | 20260505 13:15:59.195 | INFO | "gpu" : " 17 GB", | 20260505 13:15:59.195 | INFO | "huge" : "", | 20260505 13:15:59.195 | INFO | "huge_shared" : " 27 GB", | 20260505 13:15:59.195 | INFO | "malloc" : " 14 GB", | 20260505 13:15:59.195 | INFO | "pinned" : " 29 GB" | 20260505 13:15:59.195 | INFO | } | 20260505 13:15:59.195 | INFO | Perf report for the last 5.15106 seconds: | 20260505 13:15:59.195 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:15:59.195 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:59.195 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:59.195 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:59.196 | INFO | ANALYSIS : 63.7%, 3.0, 99.6ms, 0 | 20260505 13:15:59.196 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:59.196 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:59.196 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 13:15:59.196 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:59.196 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:15:59.196 | INFO | Uploaded 36GiB to GPU in 2.20216s (16.3476GiB/s) | 20260505 13:15:59.196 | INFO | Downloaded 34GiB to GPU in 2.11696s (16.0607GiB/s) | 20260505 13:15:59.332 | INFO | Starting a round of Baz IO | 20260505 13:16:02.553 | INFO | Populating RTMetrics report | 20260505 13:16:02.553 | INFO | RTMetrics report populated | 20260505 13:16:02.553 | INFO | Opening tmp RTMetrics json file | 20260505 13:16:02.585 | INFO | Serializing RTMetrics to json | 20260505 13:16:02.605 | INFO | Serialization done, swapping tmp file now... | 20260505 13:16:02.626 | INFO | RTMetrics done | 20260505 13:16:02.663 | INFO | Finished a round of Baz IO: Wrote 2.305 GiB in 3.33 seconds | 20260505 13:16:02.663 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.19% / 0.00% / 76.80% | 20260505 13:16:02.663 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.73154% / 1.58946e-05% / 9.13938e-05% | 20260505 13:16:02.663 | INFO | Baz IO overall: | 20260505 13:16:02.663 | INFO | BytesWritten : 2080427317183 | 20260505 13:16:02.663 | INFO | NumEvents : 989891006597 | 20260505 13:16:04.333 | INFO | Analyzing chunk frames = [3493888,3494400) | 20260505 13:16:04.333 | INFO | MemInfo: { | 20260505 13:16:04.333 | INFO | "gpu" : " 17 GB", | 20260505 13:16:04.333 | INFO | "huge" : "", | 20260505 13:16:04.333 | INFO | "huge_shared" : " 27 GB", | 20260505 13:16:04.333 | INFO | "malloc" : " 14 GB", | 20260505 13:16:04.333 | INFO | "pinned" : " 29 GB" | 20260505 13:16:04.333 | INFO | } | 20260505 13:16:04.333 | INFO | Perf report for the last 5.13772 seconds: | 20260505 13:16:04.333 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:16:04.333 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:04.333 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:04.333 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:04.333 | INFO | ANALYSIS : 64.2%, 3.0, 99.5ms, 0 | 20260505 13:16:04.333 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:04.333 | INFO | RT_METRICS : 46.5%, 1.0, 24.4ms, 0 | 20260505 13:16:04.333 | INFO | PRE_HQ : 60.8%, 1.8, 57.7ms, 0 | 20260505 13:16:04.333 | INFO | BAZWRITER : 64.0%, 4.0, 132.7ms, 0 | 20260505 13:16:04.333 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 13:16:04.333 | INFO | Uploaded 36GiB to GPU in 2.20199s (16.3488GiB/s) | 20260505 13:16:04.334 | INFO | Downloaded 38GiB to GPU in 2.25857s (16.8248GiB/s) | 20260505 13:16:09.468 | INFO | Analyzing chunk frames = [3494400,3494912) | 20260505 13:16:09.468 | INFO | MemInfo: { | 20260505 13:16:09.468 | INFO | "gpu" : " 17 GB", | 20260505 13:16:09.468 | INFO | "huge" : "", | 20260505 13:16:09.468 | INFO | "huge_shared" : " 27 GB", | 20260505 13:16:09.468 | INFO | "malloc" : " 14 GB", | 20260505 13:16:09.468 | INFO | "pinned" : " 29 GB" | 20260505 13:16:09.468 | INFO | } | 20260505 13:16:09.468 | INFO | Perf report for the last 5.1347 seconds: | 20260505 13:16:09.468 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:16:09.468 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:09.468 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:09.468 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:09.468 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 13:16:09.468 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:09.468 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:09.468 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 13:16:09.468 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:09.468 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:09.469 | INFO | Uploaded 36GiB to GPU in 2.19927s (16.3691GiB/s) | 20260505 13:16:09.469 | INFO | Downloaded 34GiB to GPU in 2.04215s (16.6492GiB/s) | 20260505 13:16:14.600 | INFO | Analyzing chunk frames = [3494912,3495424) | 20260505 13:16:14.600 | INFO | MemInfo: { | 20260505 13:16:14.600 | INFO | "gpu" : " 17 GB", | 20260505 13:16:14.600 | INFO | "huge" : "", | 20260505 13:16:14.600 | INFO | "huge_shared" : " 27 GB", | 20260505 13:16:14.600 | INFO | "malloc" : " 14 GB", | 20260505 13:16:14.600 | INFO | "pinned" : " 29 GB" | 20260505 13:16:14.600 | INFO | } | 20260505 13:16:14.600 | INFO | Perf report for the last 5.13192 seconds: | 20260505 13:16:14.600 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:16:14.600 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:14.600 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:14.600 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:14.600 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 13:16:14.600 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:14.600 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:14.600 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 13:16:14.600 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:14.600 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:14.600 | INFO | Uploaded 36GiB to GPU in 2.19943s (16.3679GiB/s) | 20260505 13:16:14.600 | INFO | Downloaded 34GiB to GPU in 2.11861s (16.0482GiB/s) | 20260505 13:16:19.739 | INFO | Analyzing chunk frames = [3495424,3495936) | 20260505 13:16:19.739 | INFO | MemInfo: { | 20260505 13:16:19.739 | INFO | "gpu" : " 17 GB", | 20260505 13:16:19.739 | INFO | "huge" : "", | 20260505 13:16:19.739 | INFO | "huge_shared" : " 27 GB", | 20260505 13:16:19.739 | INFO | "malloc" : " 14 GB", | 20260505 13:16:19.739 | INFO | "pinned" : " 29 GB" | 20260505 13:16:19.739 | INFO | } | 20260505 13:16:19.739 | INFO | Perf report for the last 5.13943 seconds: | 20260505 13:16:19.739 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:16:19.739 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:19.739 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:19.739 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:19.739 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 13:16:19.739 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:19.739 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:19.739 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 13:16:19.739 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:19.739 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:19.739 | INFO | Uploaded 36GiB to GPU in 2.21361s (16.263GiB/s) | 20260505 13:16:19.740 | INFO | Downloaded 34GiB to GPU in 2.13178s (15.9491GiB/s) | 20260505 13:16:24.889 | INFO | Analyzing chunk frames = [3495936,3496448) | 20260505 13:16:24.889 | INFO | MemInfo: { | 20260505 13:16:24.889 | INFO | "gpu" : " 17 GB", | 20260505 13:16:24.889 | INFO | "huge" : "", | 20260505 13:16:24.889 | INFO | "huge_shared" : " 27 GB", | 20260505 13:16:24.889 | INFO | "malloc" : " 14 GB", | 20260505 13:16:24.889 | INFO | "pinned" : " 29 GB" | 20260505 13:16:24.889 | INFO | } | 20260505 13:16:24.889 | INFO | Perf report for the last 5.1502 seconds: | 20260505 13:16:24.889 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:16:24.889 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:24.889 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:24.890 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:24.890 | INFO | ANALYSIS : 62.7%, 3.0, 98.1ms, 0 | 20260505 13:16:24.890 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:24.890 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:24.890 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 13:16:24.890 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:24.890 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:24.890 | INFO | Uploaded 36GiB to GPU in 2.16468s (16.6306GiB/s) | 20260505 13:16:24.890 | INFO | Downloaded 34GiB to GPU in 2.12742s (15.9818GiB/s) | 20260505 13:16:30.034 | INFO | Analyzing chunk frames = [3496448,3496960) | 20260505 13:16:30.034 | INFO | MemInfo: { | 20260505 13:16:30.034 | INFO | "gpu" : " 17 GB", | 20260505 13:16:30.034 | INFO | "huge" : "", | 20260505 13:16:30.034 | INFO | "huge_shared" : " 27 GB", | 20260505 13:16:30.034 | INFO | "malloc" : " 14 GB", | 20260505 13:16:30.034 | INFO | "pinned" : " 29 GB" | 20260505 13:16:30.034 | INFO | } | 20260505 13:16:30.034 | INFO | Perf report for the last 5.14482 seconds: | 20260505 13:16:30.034 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:16:30.034 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:30.034 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:30.034 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:30.034 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 13:16:30.034 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:30.034 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:30.034 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 13:16:30.034 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:30.034 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:30.035 | INFO | Uploaded 36GiB to GPU in 2.17091s (16.5829GiB/s) | 20260505 13:16:30.035 | INFO | Downloaded 34GiB to GPU in 2.10959s (16.1169GiB/s) | 20260505 13:16:35.169 | INFO | Analyzing chunk frames = [3496960,3497472) | 20260505 13:16:35.169 | INFO | MemInfo: { | 20260505 13:16:35.169 | INFO | "gpu" : " 17 GB", | 20260505 13:16:35.169 | INFO | "huge" : "", | 20260505 13:16:35.169 | INFO | "huge_shared" : " 27 GB", | 20260505 13:16:35.169 | INFO | "malloc" : " 14 GB", | 20260505 13:16:35.169 | INFO | "pinned" : " 29 GB" | 20260505 13:16:35.169 | INFO | } | 20260505 13:16:35.169 | INFO | Perf report for the last 5.13483 seconds: | 20260505 13:16:35.169 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:16:35.169 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:35.169 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:35.169 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:35.169 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 13:16:35.169 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:35.169 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:35.169 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 13:16:35.169 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:35.169 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:35.169 | INFO | Uploaded 36GiB to GPU in 2.1261s (16.9324GiB/s) | 20260505 13:16:35.170 | INFO | Downloaded 34GiB to GPU in 2.17362s (15.6421GiB/s) | 20260505 13:16:40.301 | INFO | Analyzing chunk frames = [3497472,3497984) | 20260505 13:16:40.301 | INFO | MemInfo: { | 20260505 13:16:40.301 | INFO | "gpu" : " 17 GB", | 20260505 13:16:40.301 | INFO | "huge" : "", | 20260505 13:16:40.301 | INFO | "huge_shared" : " 27 GB", | 20260505 13:16:40.301 | INFO | "malloc" : " 14 GB", | 20260505 13:16:40.302 | INFO | "pinned" : " 29 GB" | 20260505 13:16:40.302 | INFO | } | 20260505 13:16:40.302 | INFO | Perf report for the last 5.13251 seconds: | 20260505 13:16:40.302 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:16:40.302 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:40.302 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:40.302 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:40.302 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 13:16:40.302 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:40.302 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:40.302 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:16:40.302 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:40.302 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:40.302 | INFO | Uploaded 36GiB to GPU in 2.14541s (16.78GiB/s) | 20260505 13:16:40.302 | INFO | Downloaded 34GiB to GPU in 2.11041s (16.1106GiB/s) | 20260505 13:16:40.302 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 13:16:40.302 | INFO | Total mem usage: 15.3 GiB | 20260505 13:16:40.302 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 13:16:40.302 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.74%) | 20260505 13:16:40.302 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 13:16:40.302 | INFO | High Water Breakdown: | 20260505 13:16:40.302 | INFO | 61% : 8.68 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 13:16:40.302 | INFO | 38.9% : 5.53 GiB ( 83.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 13:16:40.302 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 13:16:40.302 | INFO | ------------------------------------------ | 20260505 13:16:40.302 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:16:40.302 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 13:16:40.302 | INFO | Total mem usage: 26.5 GiB | 20260505 13:16:40.302 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 13:16:40.302 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 13:16:40.302 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 13:16:40.302 | INFO | High Water Breakdown: | 20260505 13:16:40.302 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:16:40.303 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:16:40.303 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:16:40.303 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:16:40.303 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:16:40.303 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:16:40.303 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 13:16:40.303 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 13:16:40.303 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:16:40.303 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 13:16:40.303 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 13:16:40.303 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 13:16:40.303 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 13:16:40.303 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 13:16:40.303 | INFO | ------------------------------------------ | 20260505 13:16:40.303 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:16:40.303 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 13:16:40.303 | INFO | Total mem usage: 25.6 GiB | 20260505 13:16:40.303 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 13:16:40.303 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 13:16:40.303 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 13:16:40.303 | INFO | High Water Breakdown: | 20260505 13:16:40.303 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 13:16:40.303 | INFO | ------------------------------------------ | 20260505 13:16:40.303 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:16:40.303 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 13:16:40.303 | INFO | Total mem usage: 15.7 GiB | 20260505 13:16:40.303 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 13:16:40.303 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 13:16:40.303 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 13:16:40.303 | INFO | High Water Breakdown: | 20260505 13:16:40.303 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 13:16:40.303 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 13:16:40.303 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 13:16:40.303 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 13:16:40.303 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 13:16:40.303 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 13:16:40.303 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 13:16:40.303 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 13:16:40.303 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 13:16:40.303 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:16:40.303 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 13:16:40.303 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:16:40.303 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 13:16:40.303 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 13:16:40.303 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 13:16:40.303 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 13:16:40.303 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:16:40.303 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 13:16:40.303 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:16:40.303 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:16:40.303 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:16:40.303 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 13:16:40.303 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 13:16:40.303 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:16:40.303 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 13:16:40.303 | INFO | ------------------------------------------ | 20260505 13:16:40.303 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:16:40.303 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 13:16:40.303 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:16:40.303 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:16:40.303 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.68e+01, 1.37e+01 / 1.72e+00 | 20260505 13:16:40.303 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 9.99e-02, 1.07e-02 / 4.07e-03 | 20260505 13:16:40.303 | INFO | PerfReport: 11.77%, Baseline -- 3.76e+00 / 4.49e+00, 5.91e-01 / 1.77e-01 | 20260505 13:16:40.303 | INFO | PerfReport: 3.31%, Histogram -- 1.06e+00 / 1.50e+00, 2.02e-01 / 8.08e-02 | 20260505 13:16:40.303 | INFO | PerfReport: 15.98%, DME -- 4.08e+01 / 4.59e+01, 4.66e+00 / 2.20e+00 | 20260505 13:16:40.303 | INFO | PerfReport: 56.34%, FrameLabeling -- 1.80e+01 / 2.00e+01, 1.41e+00 / 1.72e+00 | 20260505 13:16:40.303 | INFO | PerfReport: 7.44%, PulseAccumulating -- 2.38e+00 / 2.92e+00, 4.46e-01 / 1.77e-01 | 20260505 13:16:40.303 | INFO | PerfReport: 3.59%, Metrics -- 1.15e+00 / 1.46e+00, 3.20e-01 / 1.42e-01 | 20260505 13:16:40.304 | INFO | PerfReport: 1.26%, PulseSerializing -- 4.03e-01 / 4.53e-01, 7.29e-02 / 2.97e-02 | 20260505 13:16:40.304 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 13:16:40.304 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:16:40.304 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:16:40.304 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260505 13:16:40.304 | INFO | PerfReport: 22.52%, Upload -- 2.27e+01 / 0.00e+00, 7.98e+00 / 0.00e+00 | 20260505 13:16:40.304 | INFO | PerfReport: 22.04%, Download -- 2.22e+01 / 0.00e+00, 8.05e+00 / 0.00e+00 | 20260505 13:16:40.304 | INFO | PerfReport: 31.66%, Compute -- 3.19e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 13:16:40.304 | INFO | PerfReport: 2.14%, LockUpload -- 2.16e+00 / 0.00e+00, 7.83e+00 / 0.00e+00 | 20260505 13:16:40.304 | INFO | PerfReport: 1.46%, LockDownload -- 1.47e+00 / 0.00e+00, 3.05e+00 / 0.00e+00 | 20260505 13:16:40.304 | INFO | PerfReport: 20.16%, LockCompute -- 2.03e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260505 13:16:40.304 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.65e-02 / 0.00e+00 | 20260505 13:16:40.435 | INFO | Starting a round of Baz IO | 20260505 13:16:43.675 | INFO | Populating RTMetrics report | 20260505 13:16:43.676 | INFO | RTMetrics report populated | 20260505 13:16:43.676 | INFO | Opening tmp RTMetrics json file | 20260505 13:16:43.678 | INFO | Serializing RTMetrics to json | 20260505 13:16:43.700 | INFO | Serialization done, swapping tmp file now... | 20260505 13:16:43.715 | INFO | RTMetrics done | 20260505 13:16:43.788 | INFO | Finished a round of Baz IO: Wrote 2.305 GiB in 3.352 seconds | 20260505 13:16:43.788 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.19% / 0.00% / 76.81% | 20260505 13:16:43.788 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.729791% / 7.94729e-06% / 8.74201e-05% | 20260505 13:16:43.788 | INFO | Baz IO overall: | 20260505 13:16:43.788 | INFO | BytesWritten : 2082902218933 | 20260505 13:16:43.788 | INFO | NumEvents : 991062210386 | 20260505 13:16:45.448 | INFO | Analyzing chunk frames = [3497984,3498496) | 20260505 13:16:45.448 | INFO | MemInfo: { | 20260505 13:16:45.448 | INFO | "gpu" : " 17 GB", | 20260505 13:16:45.448 | INFO | "huge" : "", | 20260505 13:16:45.448 | INFO | "huge_shared" : " 27 GB", | 20260505 13:16:45.448 | INFO | "malloc" : " 14 GB", | 20260505 13:16:45.448 | INFO | "pinned" : " 29 GB" | 20260505 13:16:45.449 | INFO | } | 20260505 13:16:45.449 | INFO | Perf report for the last 5.14685 seconds: | 20260505 13:16:45.449 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:16:45.449 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:45.449 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:45.449 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:45.449 | INFO | ANALYSIS : 64.4%, 3.0, 99.9ms, 0 | 20260505 13:16:45.449 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:45.449 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260505 13:16:45.449 | INFO | PRE_HQ : 60.5%, 1.8, 57.8ms, 0 | 20260505 13:16:45.449 | INFO | BAZWRITER : 64.5%, 4.0, 133.8ms, 0 | 20260505 13:16:45.449 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 13:16:45.449 | INFO | Uploaded 36GiB to GPU in 2.16901s (16.5974GiB/s) | 20260505 13:16:45.449 | INFO | Downloaded 38GiB to GPU in 2.27982s (16.668GiB/s) | 20260505 13:16:50.582 | INFO | Analyzing chunk frames = [3498496,3499008) | 20260505 13:16:50.582 | INFO | MemInfo: { | 20260505 13:16:50.582 | INFO | "gpu" : " 17 GB", | 20260505 13:16:50.582 | INFO | "huge" : "", | 20260505 13:16:50.582 | INFO | "huge_shared" : " 27 GB", | 20260505 13:16:50.582 | INFO | "malloc" : " 14 GB", | 20260505 13:16:50.582 | INFO | "pinned" : " 29 GB" | 20260505 13:16:50.582 | INFO | } | 20260505 13:16:50.582 | INFO | Perf report for the last 5.13381 seconds: | 20260505 13:16:50.582 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:16:50.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:50.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:50.582 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:50.582 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 13:16:50.582 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:50.582 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:50.582 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 13:16:50.583 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:50.583 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:50.583 | INFO | Uploaded 36GiB to GPU in 2.19506s (16.4005GiB/s) | 20260505 13:16:50.583 | INFO | Downloaded 34GiB to GPU in 2.0369s (16.692GiB/s) | 20260505 13:16:55.720 | INFO | Analyzing chunk frames = [3499008,3499520) | 20260505 13:16:55.720 | INFO | MemInfo: { | 20260505 13:16:55.720 | INFO | "gpu" : " 17 GB", | 20260505 13:16:55.720 | INFO | "huge" : "", | 20260505 13:16:55.720 | INFO | "huge_shared" : " 27 GB", | 20260505 13:16:55.720 | INFO | "malloc" : " 14 GB", | 20260505 13:16:55.720 | INFO | "pinned" : " 29 GB" | 20260505 13:16:55.720 | INFO | } | 20260505 13:16:55.721 | INFO | Perf report for the last 5.13813 seconds: | 20260505 13:16:55.721 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:16:55.721 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:55.721 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:55.721 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:55.721 | INFO | ANALYSIS : 63.2%, 3.0, 98.3ms, 0 | 20260505 13:16:55.721 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:55.721 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:55.721 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 13:16:55.721 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:55.721 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:16:55.721 | INFO | Uploaded 36GiB to GPU in 2.16283s (16.6449GiB/s) | 20260505 13:16:55.721 | INFO | Downloaded 34GiB to GPU in 2.11476s (16.0774GiB/s) | 20260505 13:17:00.861 | INFO | Analyzing chunk frames = [3499520,3500032) | 20260505 13:17:00.861 | INFO | MemInfo: { | 20260505 13:17:00.861 | INFO | "gpu" : " 17 GB", | 20260505 13:17:00.861 | INFO | "huge" : "", | 20260505 13:17:00.861 | INFO | "huge_shared" : " 27 GB", | 20260505 13:17:00.861 | INFO | "malloc" : " 14 GB", | 20260505 13:17:00.861 | INFO | "pinned" : " 29 GB" | 20260505 13:17:00.861 | INFO | } | 20260505 13:17:00.861 | INFO | Perf report for the last 5.1408 seconds: | 20260505 13:17:00.861 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:17:00.861 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:00.861 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:00.861 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:00.861 | INFO | ANALYSIS : 62.9%, 3.0, 97.8ms, 0 | 20260505 13:17:00.861 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:00.861 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:00.861 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 13:17:00.861 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:00.861 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:00.861 | INFO | Uploaded 36GiB to GPU in 2.20136s (16.3535GiB/s) | 20260505 13:17:00.861 | INFO | Downloaded 34GiB to GPU in 2.13392s (15.9331GiB/s) | 20260505 13:17:06.008 | INFO | Analyzing chunk frames = [3500032,3500544) | 20260505 13:17:06.008 | INFO | MemInfo: { | 20260505 13:17:06.008 | INFO | "gpu" : " 17 GB", | 20260505 13:17:06.008 | INFO | "huge" : "", | 20260505 13:17:06.008 | INFO | "huge_shared" : " 27 GB", | 20260505 13:17:06.008 | INFO | "malloc" : " 14 GB", | 20260505 13:17:06.008 | INFO | "pinned" : " 29 GB" | 20260505 13:17:06.008 | INFO | } | 20260505 13:17:06.008 | INFO | Perf report for the last 5.1471 seconds: | 20260505 13:17:06.008 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:17:06.008 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:06.008 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:06.008 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:06.008 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 13:17:06.008 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:06.008 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:06.008 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:17:06.009 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:06.009 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:06.009 | INFO | Uploaded 36GiB to GPU in 2.15572s (16.6998GiB/s) | 20260505 13:17:06.009 | INFO | Downloaded 34GiB to GPU in 2.12438s (16.0046GiB/s) | 20260505 13:17:11.143 | INFO | Analyzing chunk frames = [3500544,3501056) | 20260505 13:17:11.143 | INFO | MemInfo: { | 20260505 13:17:11.143 | INFO | "gpu" : " 17 GB", | 20260505 13:17:11.143 | INFO | "huge" : "", | 20260505 13:17:11.143 | INFO | "huge_shared" : " 27 GB", | 20260505 13:17:11.143 | INFO | "malloc" : " 14 GB", | 20260505 13:17:11.143 | INFO | "pinned" : " 29 GB" | 20260505 13:17:11.143 | INFO | } | 20260505 13:17:11.143 | INFO | Perf report for the last 5.13445 seconds: | 20260505 13:17:11.143 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:17:11.143 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:11.143 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:11.143 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:11.143 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 13:17:11.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:11.143 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:11.143 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 13:17:11.143 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:11.143 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:11.143 | INFO | Uploaded 36GiB to GPU in 2.182s (16.4986GiB/s) | 20260505 13:17:11.143 | INFO | Downloaded 34GiB to GPU in 2.10896s (16.1217GiB/s) | 20260505 13:17:16.283 | INFO | Analyzing chunk frames = [3501056,3501568) | 20260505 13:17:16.283 | INFO | MemInfo: { | 20260505 13:17:16.283 | INFO | "gpu" : " 17 GB", | 20260505 13:17:16.283 | INFO | "huge" : "", | 20260505 13:17:16.283 | INFO | "huge_shared" : " 27 GB", | 20260505 13:17:16.283 | INFO | "malloc" : " 14 GB", | 20260505 13:17:16.283 | INFO | "pinned" : " 29 GB" | 20260505 13:17:16.283 | INFO | } | 20260505 13:17:16.283 | INFO | Perf report for the last 5.14003 seconds: | 20260505 13:17:16.283 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:17:16.283 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:16.283 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:16.283 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:16.283 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 13:17:16.283 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:16.283 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:16.283 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 13:17:16.283 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:16.283 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:16.283 | INFO | Uploaded 36GiB to GPU in 2.14786s (16.7609GiB/s) | 20260505 13:17:16.283 | INFO | Downloaded 34GiB to GPU in 2.17932s (15.6012GiB/s) | 20260505 13:17:21.415 | INFO | Analyzing chunk frames = [3501568,3502080) | 20260505 13:17:21.415 | INFO | MemInfo: { | 20260505 13:17:21.415 | INFO | "gpu" : " 17 GB", | 20260505 13:17:21.415 | INFO | "huge" : "", | 20260505 13:17:21.415 | INFO | "huge_shared" : " 27 GB", | 20260505 13:17:21.415 | INFO | "malloc" : " 14 GB", | 20260505 13:17:21.415 | INFO | "pinned" : " 29 GB" | 20260505 13:17:21.415 | INFO | } | 20260505 13:17:21.415 | INFO | Perf report for the last 5.1321 seconds: | 20260505 13:17:21.415 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:17:21.415 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:21.415 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:21.415 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:21.415 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 13:17:21.415 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:21.415 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:21.415 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 13:17:21.415 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:21.415 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:21.415 | INFO | Uploaded 36GiB to GPU in 2.18869s (16.4482GiB/s) | 20260505 13:17:21.415 | INFO | Downloaded 34GiB to GPU in 2.10361s (16.1627GiB/s) | 20260505 13:17:21.547 | INFO | Starting a round of Baz IO | 20260505 13:17:24.799 | INFO | Populating RTMetrics report | 20260505 13:17:24.799 | INFO | RTMetrics report populated | 20260505 13:17:24.799 | INFO | Opening tmp RTMetrics json file | 20260505 13:17:24.800 | INFO | Serializing RTMetrics to json | 20260505 13:17:24.803 | INFO | Serialization done, swapping tmp file now... | 20260505 13:17:24.805 | INFO | RTMetrics done | 20260505 13:17:24.923 | INFO | Finished a round of Baz IO: Wrote 2.307 GiB in 3.376 seconds | 20260505 13:17:24.923 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.19% / 0.00% / 76.81% | 20260505 13:17:24.923 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.730578% / 0% / 0.000111262% | 20260505 13:17:24.923 | INFO | Baz IO overall: | 20260505 13:17:24.923 | INFO | BytesWritten : 2085378844419 | 20260505 13:17:24.923 | INFO | NumEvents : 992234977477 | 20260505 13:17:26.550 | INFO | Analyzing chunk frames = [3502080,3502592) | 20260505 13:17:26.550 | INFO | MemInfo: { | 20260505 13:17:26.550 | INFO | "gpu" : " 17 GB", | 20260505 13:17:26.550 | INFO | "huge" : "", | 20260505 13:17:26.550 | INFO | "huge_shared" : " 27 GB", | 20260505 13:17:26.550 | INFO | "malloc" : " 14 GB", | 20260505 13:17:26.550 | INFO | "pinned" : " 29 GB" | 20260505 13:17:26.550 | INFO | } | 20260505 13:17:26.551 | INFO | Perf report for the last 5.13548 seconds: | 20260505 13:17:26.551 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:17:26.551 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:26.551 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:26.551 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:26.551 | INFO | ANALYSIS : 64.8%, 2.9, 100.2ms, 0 | 20260505 13:17:26.551 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:26.551 | INFO | RT_METRICS : 45.5%, 1.0, 23.8ms, 0 | 20260505 13:17:26.551 | INFO | PRE_HQ : 60.9%, 1.8, 57.9ms, 0 | 20260505 13:17:26.551 | INFO | BAZWRITER : 65.0%, 4.0, 134.8ms, 0 | 20260505 13:17:26.551 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260505 13:17:26.551 | INFO | Uploaded 36GiB to GPU in 2.19835s (16.3759GiB/s) | 20260505 13:17:26.551 | INFO | Downloaded 38GiB to GPU in 2.2941s (16.5642GiB/s) | 20260505 13:17:31.697 | INFO | Analyzing chunk frames = [3502592,3503104) | 20260505 13:17:31.697 | INFO | MemInfo: { | 20260505 13:17:31.697 | INFO | "gpu" : " 17 GB", | 20260505 13:17:31.697 | INFO | "huge" : "", | 20260505 13:17:31.697 | INFO | "huge_shared" : " 27 GB", | 20260505 13:17:31.697 | INFO | "malloc" : " 14 GB", | 20260505 13:17:31.697 | INFO | "pinned" : " 29 GB" | 20260505 13:17:31.697 | INFO | } | 20260505 13:17:31.697 | INFO | Perf report for the last 5.14654 seconds: | 20260505 13:17:31.697 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:17:31.697 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:31.697 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:31.697 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:31.697 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 13:17:31.697 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:31.697 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:31.697 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:17:31.697 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:31.697 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:31.697 | INFO | Uploaded 36GiB to GPU in 2.23222s (16.1275GiB/s) | 20260505 13:17:31.697 | INFO | Downloaded 34GiB to GPU in 2.1032s (16.1659GiB/s) | 20260505 13:17:36.848 | INFO | Analyzing chunk frames = [3503104,3503616) | 20260505 13:17:36.848 | INFO | MemInfo: { | 20260505 13:17:36.848 | INFO | "gpu" : " 17 GB", | 20260505 13:17:36.848 | INFO | "huge" : "", | 20260505 13:17:36.848 | INFO | "huge_shared" : " 27 GB", | 20260505 13:17:36.848 | INFO | "malloc" : " 14 GB", | 20260505 13:17:36.848 | INFO | "pinned" : " 29 GB" | 20260505 13:17:36.848 | INFO | } | 20260505 13:17:36.848 | INFO | Perf report for the last 5.15087 seconds: | 20260505 13:17:36.848 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:17:36.848 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:36.848 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:36.848 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:36.848 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 13:17:36.848 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:36.848 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:36.848 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 13:17:36.848 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:36.848 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:36.848 | INFO | Uploaded 36GiB to GPU in 2.22537s (16.1771GiB/s) | 20260505 13:17:36.848 | INFO | Downloaded 34GiB to GPU in 2.09837s (16.2031GiB/s) | 20260505 13:17:41.979 | INFO | Analyzing chunk frames = [3503616,3504128) | 20260505 13:17:41.979 | INFO | MemInfo: { | 20260505 13:17:41.979 | INFO | "gpu" : " 17 GB", | 20260505 13:17:41.979 | INFO | "huge" : "", | 20260505 13:17:41.979 | INFO | "huge_shared" : " 27 GB", | 20260505 13:17:41.979 | INFO | "malloc" : " 14 GB", | 20260505 13:17:41.979 | INFO | "pinned" : " 29 GB" | 20260505 13:17:41.979 | INFO | } | 20260505 13:17:41.979 | INFO | Perf report for the last 5.13126 seconds: | 20260505 13:17:41.979 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:17:41.979 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:41.979 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:41.979 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:41.979 | INFO | ANALYSIS : 62.7%, 3.0, 97.3ms, 0 | 20260505 13:17:41.979 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:41.979 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:41.979 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 13:17:41.979 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:41.979 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:41.979 | INFO | Uploaded 36GiB to GPU in 2.18018s (16.5124GiB/s) | 20260505 13:17:41.980 | INFO | Downloaded 34GiB to GPU in 2.1263s (15.9903GiB/s) | 20260505 13:17:47.107 | INFO | Analyzing chunk frames = [3504128,3504640) | 20260505 13:17:47.107 | INFO | MemInfo: { | 20260505 13:17:47.107 | INFO | "gpu" : " 17 GB", | 20260505 13:17:47.107 | INFO | "huge" : "", | 20260505 13:17:47.107 | INFO | "huge_shared" : " 27 GB", | 20260505 13:17:47.107 | INFO | "malloc" : " 14 GB", | 20260505 13:17:47.107 | INFO | "pinned" : " 29 GB" | 20260505 13:17:47.107 | INFO | } | 20260505 13:17:47.107 | INFO | Perf report for the last 5.12808 seconds: | 20260505 13:17:47.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:17:47.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:47.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:47.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:47.107 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 13:17:47.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:47.107 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:47.107 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 13:17:47.107 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:47.107 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:47.108 | INFO | Uploaded 36GiB to GPU in 2.15866s (16.677GiB/s) | 20260505 13:17:47.108 | INFO | Downloaded 34GiB to GPU in 2.12538s (15.9972GiB/s) | 20260505 13:17:52.254 | INFO | Analyzing chunk frames = [3504640,3505152) | 20260505 13:17:52.254 | INFO | MemInfo: { | 20260505 13:17:52.254 | INFO | "gpu" : " 17 GB", | 20260505 13:17:52.254 | INFO | "huge" : "", | 20260505 13:17:52.254 | INFO | "huge_shared" : " 27 GB", | 20260505 13:17:52.254 | INFO | "malloc" : " 14 GB", | 20260505 13:17:52.254 | INFO | "pinned" : " 29 GB" | 20260505 13:17:52.254 | INFO | } | 20260505 13:17:52.254 | INFO | Perf report for the last 5.14695 seconds: | 20260505 13:17:52.254 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:17:52.254 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:52.254 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:52.254 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:52.254 | INFO | ANALYSIS : 63.1%, 3.0, 98.7ms, 0 | 20260505 13:17:52.254 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:52.254 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:52.254 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 13:17:52.254 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:52.254 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:52.255 | INFO | Uploaded 36GiB to GPU in 2.16594s (16.621GiB/s) | 20260505 13:17:52.255 | INFO | Downloaded 34GiB to GPU in 2.11448s (16.0796GiB/s) | 20260505 13:17:57.395 | INFO | Analyzing chunk frames = [3505152,3505664) | 20260505 13:17:57.395 | INFO | MemInfo: { | 20260505 13:17:57.395 | INFO | "gpu" : " 17 GB", | 20260505 13:17:57.395 | INFO | "huge" : "", | 20260505 13:17:57.395 | INFO | "huge_shared" : " 27 GB", | 20260505 13:17:57.395 | INFO | "malloc" : " 14 GB", | 20260505 13:17:57.395 | INFO | "pinned" : " 29 GB" | 20260505 13:17:57.395 | INFO | } | 20260505 13:17:57.395 | INFO | Perf report for the last 5.14107 seconds: | 20260505 13:17:57.395 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:17:57.395 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:57.395 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:57.395 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:57.395 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 13:17:57.395 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:57.396 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:57.396 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260505 13:17:57.396 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:57.396 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:17:57.396 | INFO | Uploaded 36GiB to GPU in 2.17574s (16.5461GiB/s) | 20260505 13:17:57.396 | INFO | Downloaded 34GiB to GPU in 2.17305s (15.6462GiB/s) | 20260505 13:18:02.537 | INFO | Analyzing chunk frames = [3505664,3506176) | 20260505 13:18:02.537 | INFO | MemInfo: { | 20260505 13:18:02.537 | INFO | "gpu" : " 17 GB", | 20260505 13:18:02.537 | INFO | "huge" : "", | 20260505 13:18:02.537 | INFO | "huge_shared" : " 27 GB", | 20260505 13:18:02.537 | INFO | "malloc" : " 14 GB", | 20260505 13:18:02.537 | INFO | "pinned" : " 29 GB" | 20260505 13:18:02.537 | INFO | } | 20260505 13:18:02.537 | INFO | Perf report for the last 5.14189 seconds: | 20260505 13:18:02.537 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:18:02.537 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:02.537 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:02.537 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:02.537 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 13:18:02.537 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:02.537 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:02.537 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 13:18:02.537 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:02.537 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:02.538 | INFO | Uploaded 36GiB to GPU in 2.22131s (16.2066GiB/s) | 20260505 13:18:02.538 | INFO | Downloaded 34GiB to GPU in 2.10698s (16.1368GiB/s) | 20260505 13:18:02.538 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 13:18:02.538 | INFO | Total mem usage: 15.3 GiB | 20260505 13:18:02.538 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 13:18:02.538 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.62%) | 20260505 13:18:02.538 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 13:18:02.538 | INFO | High Water Breakdown: | 20260505 13:18:02.538 | INFO | 60.9% : 8.68 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 13:18:02.538 | INFO | 38.9% : 5.54 GiB ( 83.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 13:18:02.538 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 13:18:02.538 | INFO | ------------------------------------------ | 20260505 13:18:02.538 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:18:02.538 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 13:18:02.538 | INFO | Total mem usage: 26.5 GiB | 20260505 13:18:02.538 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 13:18:02.538 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 13:18:02.538 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 13:18:02.538 | INFO | High Water Breakdown: | 20260505 13:18:02.538 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:18:02.538 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:18:02.538 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:18:02.538 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:18:02.538 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:18:02.538 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:18:02.538 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 13:18:02.538 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 13:18:02.538 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:18:02.538 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 13:18:02.538 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 13:18:02.538 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 13:18:02.538 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 13:18:02.538 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 13:18:02.538 | INFO | ------------------------------------------ | 20260505 13:18:02.538 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:18:02.538 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 13:18:02.538 | INFO | Total mem usage: 25.6 GiB | 20260505 13:18:02.538 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 13:18:02.538 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 13:18:02.538 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 13:18:02.538 | INFO | High Water Breakdown: | 20260505 13:18:02.538 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 13:18:02.538 | INFO | ------------------------------------------ | 20260505 13:18:02.538 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:18:02.538 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 13:18:02.539 | INFO | Total mem usage: 15.7 GiB | 20260505 13:18:02.539 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 13:18:02.539 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 13:18:02.539 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 13:18:02.539 | INFO | High Water Breakdown: | 20260505 13:18:02.539 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 13:18:02.539 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 13:18:02.539 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 13:18:02.539 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 13:18:02.539 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 13:18:02.539 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 13:18:02.539 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 13:18:02.539 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 13:18:02.539 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 13:18:02.539 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:18:02.539 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 13:18:02.539 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:18:02.539 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 13:18:02.539 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 13:18:02.539 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 13:18:02.539 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 13:18:02.539 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:18:02.539 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 13:18:02.539 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:18:02.539 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:18:02.539 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:18:02.539 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 13:18:02.539 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 13:18:02.539 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:18:02.539 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 13:18:02.539 | INFO | ------------------------------------------ | 20260505 13:18:02.539 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:18:02.539 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 13:18:02.539 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:18:02.539 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:18:02.539 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.69e+01, 1.37e+01 / 1.60e+00 | 20260505 13:18:02.539 | INFO | PerfReport: 0.29%, AggregateStats -- 9.30e-02 / 1.00e-01, 1.08e-02 / 4.33e-03 | 20260505 13:18:02.539 | INFO | PerfReport: 11.77%, Baseline -- 3.75e+00 / 4.49e+00, 5.88e-01 / 1.63e-01 | 20260505 13:18:02.539 | INFO | PerfReport: 3.30%, Histogram -- 1.05e+00 / 1.51e+00, 2.01e-01 / 9.11e-02 | 20260505 13:18:02.539 | INFO | PerfReport: 16.08%, DME -- 4.10e+01 / 4.62e+01, 4.74e+00 / 1.93e+00 | 20260505 13:18:02.539 | INFO | PerfReport: 56.21%, FrameLabeling -- 1.79e+01 / 1.97e+01, 1.31e+00 / 1.65e+00 | 20260505 13:18:02.539 | INFO | PerfReport: 7.48%, PulseAccumulating -- 2.39e+00 / 2.90e+00, 4.56e-01 / 2.14e-01 | 20260505 13:18:02.539 | INFO | PerfReport: 3.60%, Metrics -- 1.15e+00 / 1.45e+00, 3.22e-01 / 1.34e-01 | 20260505 13:18:02.539 | INFO | PerfReport: 1.26%, PulseSerializing -- 4.01e-01 / 4.53e-01, 7.10e-02 / 2.96e-02 | 20260505 13:18:02.539 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 13:18:02.539 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:18:02.539 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:18:02.539 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260505 13:18:02.539 | INFO | PerfReport: 22.55%, Upload -- 2.28e+01 / 0.00e+00, 8.06e+00 / 0.00e+00 | 20260505 13:18:02.539 | INFO | PerfReport: 22.08%, Download -- 2.23e+01 / 0.00e+00, 8.14e+00 / 0.00e+00 | 20260505 13:18:02.539 | INFO | PerfReport: 31.58%, Compute -- 3.19e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 13:18:02.539 | INFO | PerfReport: 2.21%, LockUpload -- 2.23e+00 / 0.00e+00, 7.91e+00 / 0.00e+00 | 20260505 13:18:02.539 | INFO | PerfReport: 1.49%, LockDownload -- 1.51e+00 / 0.00e+00, 3.08e+00 / 0.00e+00 | 20260505 13:18:02.539 | INFO | PerfReport: 20.07%, LockCompute -- 2.03e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260505 13:18:02.539 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.64e-02 / 0.00e+00 | 20260505 13:18:02.677 | INFO | Starting a round of Baz IO | 20260505 13:18:05.899 | INFO | Populating RTMetrics report | 20260505 13:18:05.899 | INFO | RTMetrics report populated | 20260505 13:18:05.899 | INFO | Opening tmp RTMetrics json file | 20260505 13:18:05.900 | INFO | Serializing RTMetrics to json | 20260505 13:18:05.901 | INFO | Serialization done, swapping tmp file now... | 20260505 13:18:05.902 | INFO | RTMetrics done | 20260505 13:18:06.018 | INFO | Finished a round of Baz IO: Wrote 2.304 GiB in 3.341 seconds | 20260505 13:18:06.018 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.18% / 0.00% / 76.81% | 20260505 13:18:06.018 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.725102% / 3.97364e-06% / 9.13938e-05% | 20260505 13:18:06.018 | INFO | Baz IO overall: | 20260505 13:18:06.018 | INFO | BytesWritten : 2087852814806 | 20260505 13:18:06.018 | INFO | NumEvents : 993405596211 | 20260505 13:18:07.668 | INFO | Analyzing chunk frames = [3506176,3506688) | 20260505 13:18:07.668 | INFO | MemInfo: { | 20260505 13:18:07.668 | INFO | "gpu" : " 17 GB", | 20260505 13:18:07.668 | INFO | "huge" : "", | 20260505 13:18:07.668 | INFO | "huge_shared" : " 27 GB", | 20260505 13:18:07.668 | INFO | "malloc" : " 14 GB", | 20260505 13:18:07.668 | INFO | "pinned" : " 29 GB" | 20260505 13:18:07.668 | INFO | } | 20260505 13:18:07.668 | INFO | Perf report for the last 5.13054 seconds: | 20260505 13:18:07.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:18:07.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:07.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:07.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:07.668 | INFO | ANALYSIS : 64.4%, 3.0, 99.6ms, 0 | 20260505 13:18:07.668 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:07.668 | INFO | RT_METRICS : 45.3%, 1.0, 23.7ms, 0 | 20260505 13:18:07.668 | INFO | PRE_HQ : 61.1%, 1.8, 57.9ms, 0 | 20260505 13:18:07.668 | INFO | BAZWRITER : 64.5%, 4.1, 134.9ms, 0 | 20260505 13:18:07.668 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 13:18:07.668 | INFO | Uploaded 36GiB to GPU in 2.17462s (16.5546GiB/s) | 20260505 13:18:07.668 | INFO | Downloaded 38GiB to GPU in 2.28602s (16.6228GiB/s) | 20260505 13:18:12.809 | INFO | Analyzing chunk frames = [3506688,3507200) | 20260505 13:18:12.809 | INFO | MemInfo: { | 20260505 13:18:12.809 | INFO | "gpu" : " 17 GB", | 20260505 13:18:12.809 | INFO | "huge" : "", | 20260505 13:18:12.809 | INFO | "huge_shared" : " 27 GB", | 20260505 13:18:12.809 | INFO | "malloc" : " 14 GB", | 20260505 13:18:12.809 | INFO | "pinned" : " 29 GB" | 20260505 13:18:12.809 | INFO | } | 20260505 13:18:12.809 | INFO | Perf report for the last 5.14111 seconds: | 20260505 13:18:12.809 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:18:12.809 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:12.809 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:12.809 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:12.809 | INFO | ANALYSIS : 63.6%, 3.0, 97.9ms, 0 | 20260505 13:18:12.809 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:12.809 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:12.809 | INFO | PRE_HQ : 28.4%, 1.0, 14.6ms, 0 | 20260505 13:18:12.809 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:12.809 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:12.809 | INFO | Uploaded 36GiB to GPU in 2.19366s (16.4109GiB/s) | 20260505 13:18:12.809 | INFO | Downloaded 34GiB to GPU in 2.04387s (16.6351GiB/s) | 20260505 13:18:17.961 | INFO | Analyzing chunk frames = [3507200,3507712) | 20260505 13:18:17.961 | INFO | MemInfo: { | 20260505 13:18:17.961 | INFO | "gpu" : " 17 GB", | 20260505 13:18:17.961 | INFO | "huge" : "", | 20260505 13:18:17.961 | INFO | "huge_shared" : " 27 GB", | 20260505 13:18:17.961 | INFO | "malloc" : " 14 GB", | 20260505 13:18:17.961 | INFO | "pinned" : " 29 GB" | 20260505 13:18:17.961 | INFO | } | 20260505 13:18:17.961 | INFO | Perf report for the last 5.15185 seconds: | 20260505 13:18:17.961 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:18:17.961 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:17.961 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:17.961 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:17.961 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 13:18:17.961 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:17.961 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:17.961 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 13:18:17.961 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:17.961 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:17.961 | INFO | Uploaded 36GiB to GPU in 2.16356s (16.6393GiB/s) | 20260505 13:18:17.961 | INFO | Downloaded 34GiB to GPU in 2.12839s (15.9745GiB/s) | 20260505 13:18:23.095 | INFO | Analyzing chunk frames = [3507712,3508224) | 20260505 13:18:23.095 | INFO | MemInfo: { | 20260505 13:18:23.095 | INFO | "gpu" : " 17 GB", | 20260505 13:18:23.095 | INFO | "huge" : "", | 20260505 13:18:23.095 | INFO | "huge_shared" : " 27 GB", | 20260505 13:18:23.095 | INFO | "malloc" : " 14 GB", | 20260505 13:18:23.095 | INFO | "pinned" : " 29 GB" | 20260505 13:18:23.095 | INFO | } | 20260505 13:18:23.095 | INFO | Perf report for the last 5.13447 seconds: | 20260505 13:18:23.095 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:18:23.095 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:23.095 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:23.095 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:23.095 | INFO | ANALYSIS : 62.8%, 3.0, 97.4ms, 0 | 20260505 13:18:23.095 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:23.095 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:23.095 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 13:18:23.095 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:23.095 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:23.096 | INFO | Uploaded 36GiB to GPU in 2.17146s (16.5787GiB/s) | 20260505 13:18:23.096 | INFO | Downloaded 34GiB to GPU in 2.13334s (15.9375GiB/s) | 20260505 13:18:28.226 | INFO | Analyzing chunk frames = [3508224,3508736) | 20260505 13:18:28.226 | INFO | MemInfo: { | 20260505 13:18:28.226 | INFO | "gpu" : " 17 GB", | 20260505 13:18:28.226 | INFO | "huge" : "", | 20260505 13:18:28.226 | INFO | "huge_shared" : " 27 GB", | 20260505 13:18:28.226 | INFO | "malloc" : " 14 GB", | 20260505 13:18:28.226 | INFO | "pinned" : " 29 GB" | 20260505 13:18:28.226 | INFO | } | 20260505 13:18:28.226 | INFO | Perf report for the last 5.13093 seconds: | 20260505 13:18:28.226 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:18:28.226 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:28.226 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:28.226 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:28.226 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260505 13:18:28.226 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:28.226 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:28.226 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 13:18:28.226 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:28.226 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:28.227 | INFO | Uploaded 36GiB to GPU in 2.13593s (16.8545GiB/s) | 20260505 13:18:28.227 | INFO | Downloaded 34GiB to GPU in 2.12273s (16.0171GiB/s) | 20260505 13:18:33.362 | INFO | Analyzing chunk frames = [3508736,3509248) | 20260505 13:18:33.362 | INFO | MemInfo: { | 20260505 13:18:33.362 | INFO | "gpu" : " 17 GB", | 20260505 13:18:33.362 | INFO | "huge" : "", | 20260505 13:18:33.363 | INFO | "huge_shared" : " 27 GB", | 20260505 13:18:33.363 | INFO | "malloc" : " 14 GB", | 20260505 13:18:33.363 | INFO | "pinned" : " 29 GB" | 20260505 13:18:33.363 | INFO | } | 20260505 13:18:33.363 | INFO | Perf report for the last 5.1364 seconds: | 20260505 13:18:33.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:18:33.363 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:33.363 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:33.363 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:33.363 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 13:18:33.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:33.363 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:33.363 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 13:18:33.363 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:33.363 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:33.363 | INFO | Uploaded 36GiB to GPU in 2.19082s (16.4322GiB/s) | 20260505 13:18:33.363 | INFO | Downloaded 34GiB to GPU in 2.12259s (16.0182GiB/s) | 20260505 13:18:38.505 | INFO | Analyzing chunk frames = [3509248,3509760) | 20260505 13:18:38.505 | INFO | MemInfo: { | 20260505 13:18:38.505 | INFO | "gpu" : " 17 GB", | 20260505 13:18:38.505 | INFO | "huge" : "", | 20260505 13:18:38.505 | INFO | "huge_shared" : " 27 GB", | 20260505 13:18:38.506 | INFO | "malloc" : " 14 GB", | 20260505 13:18:38.506 | INFO | "pinned" : " 29 GB" | 20260505 13:18:38.506 | INFO | } | 20260505 13:18:38.506 | INFO | Perf report for the last 5.14294 seconds: | 20260505 13:18:38.506 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:18:38.506 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:38.506 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:38.506 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:38.506 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 13:18:38.506 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:38.506 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:38.506 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 13:18:38.506 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:38.506 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:38.506 | INFO | Uploaded 36GiB to GPU in 2.18513s (16.475GiB/s) | 20260505 13:18:38.506 | INFO | Downloaded 34GiB to GPU in 2.15635s (15.7674GiB/s) | 20260505 13:18:43.648 | INFO | Analyzing chunk frames = [3509760,3510272) | 20260505 13:18:43.648 | INFO | MemInfo: { | 20260505 13:18:43.649 | INFO | "gpu" : " 17 GB", | 20260505 13:18:43.649 | INFO | "huge" : "", | 20260505 13:18:43.649 | INFO | "huge_shared" : " 27 GB", | 20260505 13:18:43.649 | INFO | "malloc" : " 14 GB", | 20260505 13:18:43.649 | INFO | "pinned" : " 29 GB" | 20260505 13:18:43.649 | INFO | } | 20260505 13:18:43.649 | INFO | Perf report for the last 5.14313 seconds: | 20260505 13:18:43.649 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:18:43.649 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:43.649 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:43.649 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:43.649 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 13:18:43.649 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:43.649 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:43.649 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 13:18:43.649 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:43.649 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:43.649 | INFO | Uploaded 36GiB to GPU in 2.18502s (16.4758GiB/s) | 20260505 13:18:43.649 | INFO | Downloaded 34GiB to GPU in 2.10944s (16.118GiB/s) | 20260505 13:18:43.785 | INFO | Starting a round of Baz IO | 20260505 13:18:47.004 | INFO | Populating RTMetrics report | 20260505 13:18:47.005 | INFO | RTMetrics report populated | 20260505 13:18:47.005 | INFO | Opening tmp RTMetrics json file | 20260505 13:18:47.045 | INFO | Serializing RTMetrics to json | 20260505 13:18:47.052 | INFO | Serialization done, swapping tmp file now... | 20260505 13:18:47.055 | INFO | RTMetrics done | 20260505 13:18:47.119 | INFO | Finished a round of Baz IO: Wrote 2.302 GiB in 3.333 seconds | 20260505 13:18:47.119 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.18% / 0.00% / 76.82% | 20260505 13:18:47.119 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.719496% / 3.97364e-06% / 9.93411e-05% | 20260505 13:18:47.119 | INFO | Baz IO overall: | 20260505 13:18:47.119 | INFO | BytesWritten : 2090324841009 | 20260505 13:18:47.119 | INFO | NumEvents : 994574680800 | 20260505 13:18:48.795 | INFO | Analyzing chunk frames = [3510272,3510784) | 20260505 13:18:48.795 | INFO | MemInfo: { | 20260505 13:18:48.795 | INFO | "gpu" : " 17 GB", | 20260505 13:18:48.795 | INFO | "huge" : "", | 20260505 13:18:48.795 | INFO | "huge_shared" : " 27 GB", | 20260505 13:18:48.795 | INFO | "malloc" : " 14 GB", | 20260505 13:18:48.795 | INFO | "pinned" : " 29 GB" | 20260505 13:18:48.795 | INFO | } | 20260505 13:18:48.795 | INFO | Perf report for the last 5.14626 seconds: | 20260505 13:18:48.795 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:18:48.795 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:48.795 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:48.795 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:48.795 | INFO | ANALYSIS : 64.1%, 3.0, 99.3ms, 0 | 20260505 13:18:48.795 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:48.795 | INFO | RT_METRICS : 46.0%, 1.0, 24.2ms, 0 | 20260505 13:18:48.795 | INFO | PRE_HQ : 61.0%, 1.8, 58.1ms, 0 | 20260505 13:18:48.795 | INFO | BAZWRITER : 64.1%, 4.1, 134.4ms, 0 | 20260505 13:18:48.795 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260505 13:18:48.795 | INFO | Uploaded 36GiB to GPU in 2.21086s (16.2833GiB/s) | 20260505 13:18:48.795 | INFO | Downloaded 38GiB to GPU in 2.27369s (16.713GiB/s) | 20260505 13:18:53.929 | INFO | Analyzing chunk frames = [3510784,3511296) | 20260505 13:18:53.929 | INFO | MemInfo: { | 20260505 13:18:53.929 | INFO | "gpu" : " 17 GB", | 20260505 13:18:53.929 | INFO | "huge" : "", | 20260505 13:18:53.930 | INFO | "huge_shared" : " 27 GB", | 20260505 13:18:53.930 | INFO | "malloc" : " 14 GB", | 20260505 13:18:53.930 | INFO | "pinned" : " 29 GB" | 20260505 13:18:53.930 | INFO | } | 20260505 13:18:53.930 | INFO | Perf report for the last 5.13464 seconds: | 20260505 13:18:53.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:18:53.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:53.930 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:53.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:53.930 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 13:18:53.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:53.930 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:53.930 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:18:53.930 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:53.930 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:53.930 | INFO | Uploaded 36GiB to GPU in 2.20472s (16.3286GiB/s) | 20260505 13:18:53.930 | INFO | Downloaded 34GiB to GPU in 2.0414s (16.6552GiB/s) | 20260505 13:18:59.059 | INFO | Analyzing chunk frames = [3511296,3511808) | 20260505 13:18:59.059 | INFO | MemInfo: { | 20260505 13:18:59.059 | INFO | "gpu" : " 17 GB", | 20260505 13:18:59.059 | INFO | "huge" : "", | 20260505 13:18:59.059 | INFO | "huge_shared" : " 27 GB", | 20260505 13:18:59.059 | INFO | "malloc" : " 14 GB", | 20260505 13:18:59.059 | INFO | "pinned" : " 29 GB" | 20260505 13:18:59.059 | INFO | } | 20260505 13:18:59.059 | INFO | Perf report for the last 5.12929 seconds: | 20260505 13:18:59.059 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:18:59.059 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:59.059 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:59.059 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:59.059 | INFO | ANALYSIS : 63.4%, 3.0, 98.5ms, 0 | 20260505 13:18:59.059 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:59.059 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:59.059 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 13:18:59.059 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:59.059 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:18:59.059 | INFO | Uploaded 36GiB to GPU in 2.22212s (16.2008GiB/s) | 20260505 13:18:59.059 | INFO | Downloaded 34GiB to GPU in 2.1027s (16.1697GiB/s) | 20260505 13:19:04.211 | INFO | Analyzing chunk frames = [3511808,3512320) | 20260505 13:19:04.211 | INFO | MemInfo: { | 20260505 13:19:04.211 | INFO | "gpu" : " 17 GB", | 20260505 13:19:04.211 | INFO | "huge" : "", | 20260505 13:19:04.211 | INFO | "huge_shared" : " 27 GB", | 20260505 13:19:04.211 | INFO | "malloc" : " 14 GB", | 20260505 13:19:04.211 | INFO | "pinned" : " 29 GB" | 20260505 13:19:04.211 | INFO | } | 20260505 13:19:04.211 | INFO | Perf report for the last 5.1523 seconds: | 20260505 13:19:04.211 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:19:04.211 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:04.211 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:04.211 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:04.211 | INFO | ANALYSIS : 63.0%, 3.0, 98.0ms, 0 | 20260505 13:19:04.211 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:04.211 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:04.211 | INFO | PRE_HQ : 28.9%, 1.0, 15.1ms, 0 | 20260505 13:19:04.211 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:04.211 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:04.212 | INFO | Uploaded 36GiB to GPU in 2.20972s (16.2917GiB/s) | 20260505 13:19:04.212 | INFO | Downloaded 34GiB to GPU in 2.13107s (15.9544GiB/s) | 20260505 13:19:09.353 | INFO | Analyzing chunk frames = [3512320,3512832) | 20260505 13:19:09.353 | INFO | MemInfo: { | 20260505 13:19:09.353 | INFO | "gpu" : " 17 GB", | 20260505 13:19:09.353 | INFO | "huge" : "", | 20260505 13:19:09.353 | INFO | "huge_shared" : " 27 GB", | 20260505 13:19:09.353 | INFO | "malloc" : " 14 GB", | 20260505 13:19:09.353 | INFO | "pinned" : " 29 GB" | 20260505 13:19:09.353 | INFO | } | 20260505 13:19:09.353 | INFO | Perf report for the last 5.14202 seconds: | 20260505 13:19:09.353 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:19:09.353 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:09.353 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:09.353 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:09.353 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 13:19:09.353 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:09.353 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:09.353 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 13:19:09.353 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:09.354 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:09.354 | INFO | Uploaded 36GiB to GPU in 2.17072s (16.5844GiB/s) | 20260505 13:19:09.354 | INFO | Downloaded 34GiB to GPU in 2.12785s (15.9786GiB/s) | 20260505 13:19:14.483 | INFO | Analyzing chunk frames = [3512832,3513344) | 20260505 13:19:14.483 | INFO | MemInfo: { | 20260505 13:19:14.483 | INFO | "gpu" : " 17 GB", | 20260505 13:19:14.483 | INFO | "huge" : "", | 20260505 13:19:14.483 | INFO | "huge_shared" : " 27 GB", | 20260505 13:19:14.483 | INFO | "malloc" : " 14 GB", | 20260505 13:19:14.483 | INFO | "pinned" : " 29 GB" | 20260505 13:19:14.483 | INFO | } | 20260505 13:19:14.483 | INFO | Perf report for the last 5.12983 seconds: | 20260505 13:19:14.483 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:19:14.483 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:14.483 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:14.483 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:14.483 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 13:19:14.483 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:14.483 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:14.483 | INFO | PRE_HQ : 28.5%, 1.0, 14.7ms, 0 | 20260505 13:19:14.483 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:14.483 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:14.483 | INFO | Uploaded 36GiB to GPU in 2.17682s (16.5379GiB/s) | 20260505 13:19:14.483 | INFO | Downloaded 34GiB to GPU in 2.10862s (16.1243GiB/s) | 20260505 13:19:19.618 | INFO | Analyzing chunk frames = [3513344,3513856) | 20260505 13:19:19.618 | INFO | MemInfo: { | 20260505 13:19:19.618 | INFO | "gpu" : " 17 GB", | 20260505 13:19:19.618 | INFO | "huge" : "", | 20260505 13:19:19.618 | INFO | "huge_shared" : " 27 GB", | 20260505 13:19:19.618 | INFO | "malloc" : " 14 GB", | 20260505 13:19:19.618 | INFO | "pinned" : " 29 GB" | 20260505 13:19:19.618 | INFO | } | 20260505 13:19:19.618 | INFO | Perf report for the last 5.13518 seconds: | 20260505 13:19:19.618 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:19:19.618 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:19.618 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:19.618 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:19.618 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 13:19:19.619 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:19.619 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:19.619 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 13:19:19.619 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:19.619 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:19.619 | INFO | Uploaded 36GiB to GPU in 2.1436s (16.7942GiB/s) | 20260505 13:19:19.619 | INFO | Downloaded 34GiB to GPU in 2.17718s (15.6165GiB/s) | 20260505 13:19:24.765 | INFO | Analyzing chunk frames = [3513856,3514368) | 20260505 13:19:24.765 | INFO | MemInfo: { | 20260505 13:19:24.765 | INFO | "gpu" : " 17 GB", | 20260505 13:19:24.765 | INFO | "huge" : "", | 20260505 13:19:24.765 | INFO | "huge_shared" : " 27 GB", | 20260505 13:19:24.766 | INFO | "malloc" : " 14 GB", | 20260505 13:19:24.766 | INFO | "pinned" : " 29 GB" | 20260505 13:19:24.766 | INFO | } | 20260505 13:19:24.766 | INFO | Perf report for the last 5.14721 seconds: | 20260505 13:19:24.766 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:19:24.766 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:24.766 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:24.766 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:24.766 | INFO | ANALYSIS : 63.7%, 3.0, 99.6ms, 0 | 20260505 13:19:24.766 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:24.766 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:24.766 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 13:19:24.766 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:24.766 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:24.766 | INFO | Uploaded 36GiB to GPU in 2.21636s (16.2428GiB/s) | 20260505 13:19:24.766 | INFO | Downloaded 34GiB to GPU in 2.11423s (16.0815GiB/s) | 20260505 13:19:24.766 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 13:19:24.766 | INFO | Total mem usage: 15.3 GiB | 20260505 13:19:24.766 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 13:19:24.766 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -7.06%) | 20260505 13:19:24.766 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 13:19:24.766 | INFO | High Water Breakdown: | 20260505 13:19:24.766 | INFO | 61.1% : 8.74 GiB ( 91.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 13:19:24.766 | INFO | 38.8% : 5.55 GiB ( 83.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 13:19:24.766 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 13:19:24.766 | INFO | ------------------------------------------ | 20260505 13:19:24.766 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:19:24.766 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 13:19:24.766 | INFO | Total mem usage: 26.5 GiB | 20260505 13:19:24.766 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 13:19:24.766 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 13:19:24.766 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 13:19:24.766 | INFO | High Water Breakdown: | 20260505 13:19:24.766 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:19:24.766 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:19:24.766 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:19:24.766 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:19:24.767 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:19:24.767 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:19:24.767 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 13:19:24.767 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 13:19:24.767 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:19:24.767 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 13:19:24.767 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 13:19:24.767 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 13:19:24.767 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 13:19:24.767 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 13:19:24.767 | INFO | ------------------------------------------ | 20260505 13:19:24.767 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:19:24.767 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 13:19:24.767 | INFO | Total mem usage: 25.6 GiB | 20260505 13:19:24.767 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 13:19:24.767 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 13:19:24.767 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 13:19:24.767 | INFO | High Water Breakdown: | 20260505 13:19:24.767 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 13:19:24.767 | INFO | ------------------------------------------ | 20260505 13:19:24.767 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:19:24.767 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 13:19:24.767 | INFO | Total mem usage: 15.7 GiB | 20260505 13:19:24.767 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 13:19:24.767 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 13:19:24.767 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 13:19:24.767 | INFO | High Water Breakdown: | 20260505 13:19:24.767 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 13:19:24.767 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 13:19:24.767 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 13:19:24.767 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 13:19:24.767 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 13:19:24.767 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 13:19:24.767 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 13:19:24.767 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 13:19:24.767 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 13:19:24.767 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:19:24.767 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 13:19:24.767 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:19:24.767 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 13:19:24.767 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 13:19:24.767 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 13:19:24.767 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 13:19:24.767 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:19:24.767 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 13:19:24.767 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:19:24.767 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:19:24.767 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:19:24.767 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 13:19:24.767 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 13:19:24.767 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:19:24.767 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 13:19:24.767 | INFO | ------------------------------------------ | 20260505 13:19:24.767 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:19:24.767 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 13:19:24.767 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:19:24.767 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:19:24.767 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.68e+01, 1.39e+01 / 1.54e+00 | 20260505 13:19:24.767 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 1.00e-01, 1.05e-02 / 3.97e-03 | 20260505 13:19:24.767 | INFO | PerfReport: 11.80%, Baseline -- 3.76e+00 / 4.50e+00, 5.94e-01 / 1.63e-01 | 20260505 13:19:24.767 | INFO | PerfReport: 3.30%, Histogram -- 1.05e+00 / 1.50e+00, 2.01e-01 / 9.25e-02 | 20260505 13:19:24.767 | INFO | PerfReport: 16.00%, DME -- 4.08e+01 / 4.60e+01, 4.83e+00 / 1.90e+00 | 20260505 13:19:24.767 | INFO | PerfReport: 56.29%, FrameLabeling -- 1.80e+01 / 1.99e+01, 1.38e+00 / 1.45e+00 | 20260505 13:19:24.767 | INFO | PerfReport: 7.47%, PulseAccumulating -- 2.38e+00 / 2.91e+00, 4.57e-01 / 1.51e-01 | 20260505 13:19:24.767 | INFO | PerfReport: 3.59%, Metrics -- 1.15e+00 / 1.48e+00, 3.22e-01 / 1.45e-01 | 20260505 13:19:24.767 | INFO | PerfReport: 1.26%, PulseSerializing -- 4.01e-01 / 4.51e-01, 7.07e-02 / 2.91e-02 | 20260505 13:19:24.767 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 13:19:24.767 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:19:24.767 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:19:24.767 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260505 13:19:24.767 | INFO | PerfReport: 22.58%, Upload -- 2.28e+01 / 0.00e+00, 8.02e+00 / 0.00e+00 | 20260505 13:19:24.767 | INFO | PerfReport: 22.08%, Download -- 2.23e+01 / 0.00e+00, 8.08e+00 / 0.00e+00 | 20260505 13:19:24.767 | INFO | PerfReport: 31.66%, Compute -- 3.19e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 13:19:24.767 | INFO | PerfReport: 2.16%, LockUpload -- 2.18e+00 / 0.00e+00, 7.83e+00 / 0.00e+00 | 20260505 13:19:24.767 | INFO | PerfReport: 1.45%, LockDownload -- 1.47e+00 / 0.00e+00, 3.00e+00 / 0.00e+00 | 20260505 13:19:24.767 | INFO | PerfReport: 20.05%, LockCompute -- 2.02e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260505 13:19:24.767 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.44e-02 / 0.00e+00 | 20260505 13:19:24.903 | INFO | Starting a round of Baz IO | 20260505 13:19:28.126 | INFO | Populating RTMetrics report | 20260505 13:19:28.127 | INFO | RTMetrics report populated | 20260505 13:19:28.127 | INFO | Opening tmp RTMetrics json file | 20260505 13:19:28.143 | INFO | Serializing RTMetrics to json | 20260505 13:19:28.145 | INFO | Serialization done, swapping tmp file now... | 20260505 13:19:28.149 | INFO | RTMetrics done | 20260505 13:19:28.240 | INFO | Finished a round of Baz IO: Wrote 2.301 GiB in 3.337 seconds | 20260505 13:19:28.241 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.17% / 0.00% / 76.82% | 20260505 13:19:28.241 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.717986% / 7.94729e-06% / 7.15256e-05% | 20260505 13:19:28.241 | INFO | Baz IO overall: | 20260505 13:19:28.241 | INFO | BytesWritten : 2092795644366 | 20260505 13:19:28.241 | INFO | NumEvents : 995743140072 | 20260505 13:19:29.905 | INFO | Analyzing chunk frames = [3514368,3514880) | 20260505 13:19:29.905 | INFO | MemInfo: { | 20260505 13:19:29.905 | INFO | "gpu" : " 17 GB", | 20260505 13:19:29.905 | INFO | "huge" : "", | 20260505 13:19:29.905 | INFO | "huge_shared" : " 27 GB", | 20260505 13:19:29.905 | INFO | "malloc" : " 14 GB", | 20260505 13:19:29.905 | INFO | "pinned" : " 29 GB" | 20260505 13:19:29.905 | INFO | } | 20260505 13:19:29.905 | INFO | Perf report for the last 5.13933 seconds: | 20260505 13:19:29.905 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:19:29.905 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:29.905 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:29.905 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:29.905 | INFO | ANALYSIS : 64.3%, 3.0, 99.5ms, 0 | 20260505 13:19:29.905 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:29.905 | INFO | RT_METRICS : 45.5%, 1.0, 23.8ms, 0 | 20260505 13:19:29.905 | INFO | PRE_HQ : 61.0%, 1.8, 57.8ms, 0 | 20260505 13:19:29.905 | INFO | BAZWRITER : 64.2%, 4.0, 133.4ms, 0 | 20260505 13:19:29.905 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 13:19:29.905 | INFO | Uploaded 36GiB to GPU in 2.1863s (16.4662GiB/s) | 20260505 13:19:29.905 | INFO | Downloaded 38GiB to GPU in 2.27534s (16.7008GiB/s) | 20260505 13:19:35.043 | INFO | Analyzing chunk frames = [3514880,3515392) | 20260505 13:19:35.043 | INFO | MemInfo: { | 20260505 13:19:35.043 | INFO | "gpu" : " 17 GB", | 20260505 13:19:35.043 | INFO | "huge" : "", | 20260505 13:19:35.043 | INFO | "huge_shared" : " 27 GB", | 20260505 13:19:35.043 | INFO | "malloc" : " 14 GB", | 20260505 13:19:35.043 | INFO | "pinned" : " 29 GB" | 20260505 13:19:35.043 | INFO | } | 20260505 13:19:35.043 | INFO | Perf report for the last 5.13825 seconds: | 20260505 13:19:35.043 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:19:35.043 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:35.043 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:35.043 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:35.043 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 13:19:35.043 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:35.043 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:35.043 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:19:35.043 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:35.043 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:35.043 | INFO | Uploaded 36GiB to GPU in 2.17528s (16.5496GiB/s) | 20260505 13:19:35.044 | INFO | Downloaded 34GiB to GPU in 2.03234s (16.7295GiB/s) | 20260505 13:19:40.174 | INFO | Analyzing chunk frames = [3515392,3515904) | 20260505 13:19:40.174 | INFO | MemInfo: { | 20260505 13:19:40.174 | INFO | "gpu" : " 17 GB", | 20260505 13:19:40.174 | INFO | "huge" : "", | 20260505 13:19:40.174 | INFO | "huge_shared" : " 27 GB", | 20260505 13:19:40.174 | INFO | "malloc" : " 14 GB", | 20260505 13:19:40.174 | INFO | "pinned" : " 29 GB" | 20260505 13:19:40.174 | INFO | } | 20260505 13:19:40.174 | INFO | Perf report for the last 5.13062 seconds: | 20260505 13:19:40.174 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:19:40.174 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:40.174 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:40.174 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:40.174 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 13:19:40.174 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:40.174 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:40.174 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 13:19:40.174 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:40.174 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:40.174 | INFO | Uploaded 36GiB to GPU in 2.20129s (16.354GiB/s) | 20260505 13:19:40.174 | INFO | Downloaded 34GiB to GPU in 2.12311s (16.0142GiB/s) | 20260505 13:19:45.318 | INFO | Analyzing chunk frames = [3515904,3516416) | 20260505 13:19:45.318 | INFO | MemInfo: { | 20260505 13:19:45.318 | INFO | "gpu" : " 17 GB", | 20260505 13:19:45.318 | INFO | "huge" : "", | 20260505 13:19:45.318 | INFO | "huge_shared" : " 27 GB", | 20260505 13:19:45.319 | INFO | "malloc" : " 14 GB", | 20260505 13:19:45.319 | INFO | "pinned" : " 29 GB" | 20260505 13:19:45.319 | INFO | } | 20260505 13:19:45.319 | INFO | Perf report for the last 5.14486 seconds: | 20260505 13:19:45.319 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:19:45.319 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:45.319 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:45.319 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:45.319 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 13:19:45.319 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:45.319 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:45.319 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:19:45.319 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:45.319 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:45.319 | INFO | Uploaded 36GiB to GPU in 2.21451s (16.2564GiB/s) | 20260505 13:19:45.319 | INFO | Downloaded 34GiB to GPU in 2.13574s (15.9195GiB/s) | 20260505 13:19:50.460 | INFO | Analyzing chunk frames = [3516416,3516928) | 20260505 13:19:50.460 | INFO | MemInfo: { | 20260505 13:19:50.460 | INFO | "gpu" : " 17 GB", | 20260505 13:19:50.460 | INFO | "huge" : "", | 20260505 13:19:50.460 | INFO | "huge_shared" : " 27 GB", | 20260505 13:19:50.460 | INFO | "malloc" : " 14 GB", | 20260505 13:19:50.460 | INFO | "pinned" : " 29 GB" | 20260505 13:19:50.460 | INFO | } | 20260505 13:19:50.460 | INFO | Perf report for the last 5.14109 seconds: | 20260505 13:19:50.460 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:19:50.460 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:50.460 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:50.460 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:50.460 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 13:19:50.460 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:50.460 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:50.460 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 13:19:50.460 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:50.460 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:50.460 | INFO | Uploaded 36GiB to GPU in 2.15575s (16.6995GiB/s) | 20260505 13:19:50.460 | INFO | Downloaded 34GiB to GPU in 2.12596s (15.9927GiB/s) | 20260505 13:19:55.603 | INFO | Analyzing chunk frames = [3516928,3517440) | 20260505 13:19:55.603 | INFO | MemInfo: { | 20260505 13:19:55.603 | INFO | "gpu" : " 17 GB", | 20260505 13:19:55.603 | INFO | "huge" : "", | 20260505 13:19:55.604 | INFO | "huge_shared" : " 27 GB", | 20260505 13:19:55.604 | INFO | "malloc" : " 14 GB", | 20260505 13:19:55.604 | INFO | "pinned" : " 29 GB" | 20260505 13:19:55.604 | INFO | } | 20260505 13:19:55.604 | INFO | Perf report for the last 5.14381 seconds: | 20260505 13:19:55.604 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:19:55.604 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:55.604 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:55.604 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:55.604 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 13:19:55.604 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:55.604 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:55.604 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 13:19:55.604 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:55.604 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:19:55.604 | INFO | Uploaded 36GiB to GPU in 2.13224s (16.8836GiB/s) | 20260505 13:19:55.604 | INFO | Downloaded 34GiB to GPU in 2.10348s (16.1637GiB/s) | 20260505 13:20:00.741 | INFO | Analyzing chunk frames = [3517440,3517952) | 20260505 13:20:00.742 | INFO | MemInfo: { | 20260505 13:20:00.742 | INFO | "gpu" : " 17 GB", | 20260505 13:20:00.742 | INFO | "huge" : "", | 20260505 13:20:00.742 | INFO | "huge_shared" : " 27 GB", | 20260505 13:20:00.742 | INFO | "malloc" : " 14 GB", | 20260505 13:20:00.742 | INFO | "pinned" : " 29 GB" | 20260505 13:20:00.742 | INFO | } | 20260505 13:20:00.742 | INFO | Perf report for the last 5.13808 seconds: | 20260505 13:20:00.742 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:20:00.742 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:00.742 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:00.742 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:00.742 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 13:20:00.742 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:00.742 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:00.742 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 13:20:00.742 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:00.742 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:00.742 | INFO | Uploaded 36GiB to GPU in 2.16742s (16.6096GiB/s) | 20260505 13:20:00.742 | INFO | Downloaded 34GiB to GPU in 2.19207s (15.5104GiB/s) | 20260505 13:20:05.871 | INFO | Analyzing chunk frames = [3517952,3518464) | 20260505 13:20:05.871 | INFO | MemInfo: { | 20260505 13:20:05.871 | INFO | "gpu" : " 17 GB", | 20260505 13:20:05.871 | INFO | "huge" : "", | 20260505 13:20:05.871 | INFO | "huge_shared" : " 27 GB", | 20260505 13:20:05.872 | INFO | "malloc" : " 14 GB", | 20260505 13:20:05.872 | INFO | "pinned" : " 29 GB" | 20260505 13:20:05.872 | INFO | } | 20260505 13:20:05.872 | INFO | Perf report for the last 5.12992 seconds: | 20260505 13:20:05.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:20:05.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:05.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:05.872 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:05.872 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 13:20:05.872 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:05.872 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:05.872 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 13:20:05.872 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:05.872 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:05.872 | INFO | Uploaded 36GiB to GPU in 2.14965s (16.7469GiB/s) | 20260505 13:20:05.872 | INFO | Downloaded 34GiB to GPU in 2.10603s (16.1441GiB/s) | 20260505 13:20:06.022 | INFO | Starting a round of Baz IO | 20260505 13:20:09.246 | INFO | Populating RTMetrics report | 20260505 13:20:09.247 | INFO | RTMetrics report populated | 20260505 13:20:09.247 | INFO | Opening tmp RTMetrics json file | 20260505 13:20:09.264 | INFO | Serializing RTMetrics to json | 20260505 13:20:09.271 | INFO | Serialization done, swapping tmp file now... | 20260505 13:20:09.288 | INFO | RTMetrics done | 20260505 13:20:09.359 | INFO | Finished a round of Baz IO: Wrote 2.301 GiB in 3.337 seconds | 20260505 13:20:09.359 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.17% / 0.00% / 76.83% | 20260505 13:20:09.359 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.717354% / 0% / 8.74201e-05% | 20260505 13:20:09.359 | INFO | Baz IO overall: | 20260505 13:20:09.359 | INFO | BytesWritten : 2095265949297 | 20260505 13:20:09.359 | INFO | NumEvents : 996911261125 | 20260505 13:20:11.017 | INFO | Analyzing chunk frames = [3518464,3518976) | 20260505 13:20:11.017 | INFO | MemInfo: { | 20260505 13:20:11.017 | INFO | "gpu" : " 17 GB", | 20260505 13:20:11.017 | INFO | "huge" : "", | 20260505 13:20:11.017 | INFO | "huge_shared" : " 27 GB", | 20260505 13:20:11.017 | INFO | "malloc" : " 14 GB", | 20260505 13:20:11.017 | INFO | "pinned" : " 29 GB" | 20260505 13:20:11.018 | INFO | } | 20260505 13:20:11.018 | INFO | Perf report for the last 5.14588 seconds: | 20260505 13:20:11.018 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:20:11.018 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:11.018 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:11.018 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:11.018 | INFO | ANALYSIS : 64.5%, 3.0, 100.1ms, 0 | 20260505 13:20:11.018 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:11.018 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260505 13:20:11.018 | INFO | PRE_HQ : 60.7%, 1.8, 57.9ms, 0 | 20260505 13:20:11.018 | INFO | BAZWRITER : 64.9%, 4.0, 132.8ms, 0 | 20260505 13:20:11.018 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 13:20:11.018 | INFO | Uploaded 36GiB to GPU in 2.19307s (16.4153GiB/s) | 20260505 13:20:11.018 | INFO | Downloaded 38GiB to GPU in 2.28651s (16.6192GiB/s) | 20260505 13:20:16.160 | INFO | Analyzing chunk frames = [3518976,3519488) | 20260505 13:20:16.160 | INFO | MemInfo: { | 20260505 13:20:16.160 | INFO | "gpu" : " 17 GB", | 20260505 13:20:16.160 | INFO | "huge" : "", | 20260505 13:20:16.160 | INFO | "huge_shared" : " 27 GB", | 20260505 13:20:16.160 | INFO | "malloc" : " 14 GB", | 20260505 13:20:16.160 | INFO | "pinned" : " 29 GB" | 20260505 13:20:16.160 | INFO | } | 20260505 13:20:16.160 | INFO | Perf report for the last 5.14225 seconds: | 20260505 13:20:16.160 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:20:16.160 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:16.160 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:16.160 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:16.160 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 13:20:16.160 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:16.160 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:16.160 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:20:16.160 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:16.160 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:16.160 | INFO | Uploaded 36GiB to GPU in 2.1765s (16.5404GiB/s) | 20260505 13:20:16.160 | INFO | Downloaded 34GiB to GPU in 2.03432s (16.7132GiB/s) | 20260505 13:20:21.301 | INFO | Analyzing chunk frames = [3519488,3520000) | 20260505 13:20:21.301 | INFO | MemInfo: { | 20260505 13:20:21.301 | INFO | "gpu" : " 17 GB", | 20260505 13:20:21.301 | INFO | "huge" : "", | 20260505 13:20:21.301 | INFO | "huge_shared" : " 27 GB", | 20260505 13:20:21.301 | INFO | "malloc" : " 14 GB", | 20260505 13:20:21.301 | INFO | "pinned" : " 29 GB" | 20260505 13:20:21.301 | INFO | } | 20260505 13:20:21.302 | INFO | Perf report for the last 5.14181 seconds: | 20260505 13:20:21.302 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:20:21.302 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:21.302 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:21.302 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:21.302 | INFO | ANALYSIS : 63.4%, 3.0, 98.6ms, 0 | 20260505 13:20:21.302 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:21.302 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:21.302 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:20:21.302 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:21.302 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:21.302 | INFO | Uploaded 36GiB to GPU in 2.23276s (16.1236GiB/s) | 20260505 13:20:21.302 | INFO | Downloaded 34GiB to GPU in 2.09339s (16.2416GiB/s) | 20260505 13:20:26.432 | INFO | Analyzing chunk frames = [3520000,3520512) | 20260505 13:20:26.432 | INFO | MemInfo: { | 20260505 13:20:26.432 | INFO | "gpu" : " 17 GB", | 20260505 13:20:26.432 | INFO | "huge" : "", | 20260505 13:20:26.432 | INFO | "huge_shared" : " 27 GB", | 20260505 13:20:26.432 | INFO | "malloc" : " 14 GB", | 20260505 13:20:26.432 | INFO | "pinned" : " 29 GB" | 20260505 13:20:26.432 | INFO | } | 20260505 13:20:26.433 | INFO | Perf report for the last 5.13096 seconds: | 20260505 13:20:26.433 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:20:26.433 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:26.433 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:26.433 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:26.433 | INFO | ANALYSIS : 63.0%, 3.0, 97.8ms, 0 | 20260505 13:20:26.433 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:26.433 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:26.433 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 13:20:26.433 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:26.433 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:26.433 | INFO | Uploaded 36GiB to GPU in 2.19705s (16.3856GiB/s) | 20260505 13:20:26.433 | INFO | Downloaded 34GiB to GPU in 2.13423s (15.9308GiB/s) | 20260505 13:20:31.570 | INFO | Analyzing chunk frames = [3520512,3521024) | 20260505 13:20:31.570 | INFO | MemInfo: { | 20260505 13:20:31.570 | INFO | "gpu" : " 17 GB", | 20260505 13:20:31.570 | INFO | "huge" : "", | 20260505 13:20:31.570 | INFO | "huge_shared" : " 27 GB", | 20260505 13:20:31.570 | INFO | "malloc" : " 14 GB", | 20260505 13:20:31.570 | INFO | "pinned" : " 29 GB" | 20260505 13:20:31.570 | INFO | } | 20260505 13:20:31.570 | INFO | Perf report for the last 5.13706 seconds: | 20260505 13:20:31.570 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:20:31.570 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:31.570 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:31.570 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:31.570 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 13:20:31.570 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:31.570 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:31.570 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 13:20:31.570 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:31.570 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:31.570 | INFO | Uploaded 36GiB to GPU in 2.18754s (16.4568GiB/s) | 20260505 13:20:31.570 | INFO | Downloaded 34GiB to GPU in 2.12969s (15.9648GiB/s) | 20260505 13:20:36.720 | INFO | Analyzing chunk frames = [3521024,3521536) | 20260505 13:20:36.720 | INFO | MemInfo: { | 20260505 13:20:36.720 | INFO | "gpu" : " 17 GB", | 20260505 13:20:36.720 | INFO | "huge" : "", | 20260505 13:20:36.720 | INFO | "huge_shared" : " 27 GB", | 20260505 13:20:36.720 | INFO | "malloc" : " 14 GB", | 20260505 13:20:36.720 | INFO | "pinned" : " 29 GB" | 20260505 13:20:36.720 | INFO | } | 20260505 13:20:36.720 | INFO | Perf report for the last 5.1504 seconds: | 20260505 13:20:36.720 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:20:36.720 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:36.720 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:36.720 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:36.720 | INFO | ANALYSIS : 63.0%, 3.0, 98.7ms, 0 | 20260505 13:20:36.720 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:36.720 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:36.720 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 13:20:36.720 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:36.720 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:36.720 | INFO | Uploaded 36GiB to GPU in 2.18005s (16.5134GiB/s) | 20260505 13:20:36.720 | INFO | Downloaded 34GiB to GPU in 2.10707s (16.1362GiB/s) | 20260505 13:20:41.857 | INFO | Analyzing chunk frames = [3521536,3522048) | 20260505 13:20:41.857 | INFO | MemInfo: { | 20260505 13:20:41.857 | INFO | "gpu" : " 17 GB", | 20260505 13:20:41.857 | INFO | "huge" : "", | 20260505 13:20:41.857 | INFO | "huge_shared" : " 27 GB", | 20260505 13:20:41.857 | INFO | "malloc" : " 14 GB", | 20260505 13:20:41.857 | INFO | "pinned" : " 29 GB" | 20260505 13:20:41.857 | INFO | } | 20260505 13:20:41.857 | INFO | Perf report for the last 5.13724 seconds: | 20260505 13:20:41.857 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:20:41.857 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:41.857 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:41.857 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:41.857 | INFO | ANALYSIS : 64.0%, 3.0, 99.8ms, 0 | 20260505 13:20:41.857 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:41.857 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:41.857 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 13:20:41.857 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:41.857 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:41.858 | INFO | Uploaded 36GiB to GPU in 2.15619s (16.6961GiB/s) | 20260505 13:20:41.858 | INFO | Downloaded 34GiB to GPU in 2.19391s (15.4975GiB/s) | 20260505 13:20:46.991 | INFO | Analyzing chunk frames = [3522048,3522560) | 20260505 13:20:46.991 | INFO | MemInfo: { | 20260505 13:20:46.991 | INFO | "gpu" : " 17 GB", | 20260505 13:20:46.991 | INFO | "huge" : "", | 20260505 13:20:46.991 | INFO | "huge_shared" : " 27 GB", | 20260505 13:20:46.991 | INFO | "malloc" : " 14 GB", | 20260505 13:20:46.991 | INFO | "pinned" : " 29 GB" | 20260505 13:20:46.991 | INFO | } | 20260505 13:20:46.991 | INFO | Perf report for the last 5.13398 seconds: | 20260505 13:20:46.991 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:20:46.991 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:46.991 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:46.991 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:46.992 | INFO | ANALYSIS : 63.6%, 3.0, 99.1ms, 0 | 20260505 13:20:46.992 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:46.992 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:46.992 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 13:20:46.992 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:46.992 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:46.992 | INFO | Uploaded 36GiB to GPU in 2.15626s (16.6956GiB/s) | 20260505 13:20:46.992 | INFO | Downloaded 34GiB to GPU in 2.10346s (16.1639GiB/s) | 20260505 13:20:46.992 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 13:20:46.992 | INFO | Total mem usage: 15.3 GiB | 20260505 13:20:46.992 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 13:20:46.992 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.61%) | 20260505 13:20:46.992 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 13:20:46.992 | INFO | High Water Breakdown: | 20260505 13:20:46.992 | INFO | 60.9% : 8.68 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 13:20:46.992 | INFO | 38.9% : 5.54 GiB ( 83.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 13:20:46.992 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 13:20:46.992 | INFO | ------------------------------------------ | 20260505 13:20:46.992 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:20:46.992 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 13:20:46.992 | INFO | Total mem usage: 26.5 GiB | 20260505 13:20:46.992 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 13:20:46.992 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 13:20:46.992 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 13:20:46.992 | INFO | High Water Breakdown: | 20260505 13:20:46.992 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:20:46.992 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:20:46.992 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:20:46.992 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:20:46.992 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:20:46.992 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:20:46.992 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 13:20:46.992 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 13:20:46.992 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:20:46.992 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 13:20:46.992 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 13:20:46.992 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 13:20:46.992 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 13:20:46.992 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 13:20:46.992 | INFO | ------------------------------------------ | 20260505 13:20:46.992 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:20:46.992 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 13:20:46.992 | INFO | Total mem usage: 25.6 GiB | 20260505 13:20:46.992 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 13:20:46.992 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 13:20:46.992 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 13:20:46.992 | INFO | High Water Breakdown: | 20260505 13:20:46.992 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 13:20:46.992 | INFO | ------------------------------------------ | 20260505 13:20:46.992 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:20:46.992 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 13:20:46.992 | INFO | Total mem usage: 15.7 GiB | 20260505 13:20:46.993 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 13:20:46.993 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 13:20:46.993 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 13:20:46.993 | INFO | High Water Breakdown: | 20260505 13:20:46.993 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 13:20:46.993 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 13:20:46.993 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 13:20:46.993 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 13:20:46.993 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 13:20:46.993 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 13:20:46.993 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 13:20:46.993 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 13:20:46.993 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 13:20:46.993 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:20:46.993 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 13:20:46.993 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:20:46.993 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 13:20:46.993 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 13:20:46.993 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 13:20:46.993 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 13:20:46.993 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:20:46.993 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 13:20:46.993 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:20:46.993 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:20:46.993 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:20:46.993 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 13:20:46.993 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 13:20:46.993 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:20:46.993 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 13:20:46.993 | INFO | ------------------------------------------ | 20260505 13:20:46.993 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:20:46.993 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 13:20:46.993 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:20:46.993 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:20:46.993 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.75e+01, 1.39e+01 / 1.99e+00 | 20260505 13:20:46.993 | INFO | PerfReport: 0.29%, AggregateStats -- 9.31e-02 / 1.00e-01, 1.09e-02 / 6.39e-03 | 20260505 13:20:46.993 | INFO | PerfReport: 11.78%, Baseline -- 3.76e+00 / 4.48e+00, 5.97e-01 / 1.31e-01 | 20260505 13:20:46.993 | INFO | PerfReport: 3.30%, Histogram -- 1.05e+00 / 1.48e+00, 2.01e-01 / 9.30e-02 | 20260505 13:20:46.993 | INFO | PerfReport: 16.15%, DME -- 4.12e+01 / 4.70e+01, 4.96e+00 / 2.30e+00 | 20260505 13:20:46.993 | INFO | PerfReport: 56.23%, FrameLabeling -- 1.79e+01 / 1.95e+01, 1.33e+00 / 1.64e+00 | 20260505 13:20:46.993 | INFO | PerfReport: 7.42%, PulseAccumulating -- 2.37e+00 / 2.95e+00, 4.50e-01 / 1.81e-01 | 20260505 13:20:46.993 | INFO | PerfReport: 3.57%, Metrics -- 1.14e+00 / 1.47e+00, 3.20e-01 / 1.73e-01 | 20260505 13:20:46.993 | INFO | PerfReport: 1.26%, PulseSerializing -- 4.01e-01 / 4.45e-01, 7.01e-02 / 3.05e-02 | 20260505 13:20:46.993 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 13:20:46.993 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:20:46.993 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:20:46.993 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.58e+01 / 0.00e+00 | 20260505 13:20:46.993 | INFO | PerfReport: 22.47%, Upload -- 2.27e+01 / 0.00e+00, 8.03e+00 / 0.00e+00 | 20260505 13:20:46.993 | INFO | PerfReport: 22.03%, Download -- 2.23e+01 / 0.00e+00, 8.15e+00 / 0.00e+00 | 20260505 13:20:46.993 | INFO | PerfReport: 31.61%, Compute -- 3.19e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 13:20:46.993 | INFO | PerfReport: 2.21%, LockUpload -- 2.23e+00 / 0.00e+00, 8.03e+00 / 0.00e+00 | 20260505 13:20:46.993 | INFO | PerfReport: 1.51%, LockDownload -- 1.53e+00 / 0.00e+00, 3.09e+00 / 0.00e+00 | 20260505 13:20:46.993 | INFO | PerfReport: 20.16%, LockCompute -- 2.04e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 13:20:46.993 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.90e-02 / 0.00e+00 | 20260505 13:20:47.122 | INFO | Starting a round of Baz IO | 20260505 13:20:50.366 | INFO | Populating RTMetrics report | 20260505 13:20:50.366 | INFO | RTMetrics report populated | 20260505 13:20:50.366 | INFO | Opening tmp RTMetrics json file | 20260505 13:20:50.386 | INFO | Serializing RTMetrics to json | 20260505 13:20:50.400 | INFO | Serialization done, swapping tmp file now... | 20260505 13:20:50.402 | INFO | RTMetrics done | 20260505 13:20:50.485 | INFO | Finished a round of Baz IO: Wrote 2.301 GiB in 3.362 seconds | 20260505 13:20:50.485 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.16% / 0.00% / 76.83% | 20260505 13:20:50.485 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.717966% / 3.97364e-06% / 6.75519e-05% | 20260505 13:20:50.485 | INFO | Baz IO overall: | 20260505 13:20:50.485 | INFO | BytesWritten : 2097737146466 | 20260505 13:20:50.485 | INFO | NumEvents : 998080313299 | 20260505 13:20:52.133 | INFO | Analyzing chunk frames = [3522560,3523072) | 20260505 13:20:52.133 | INFO | MemInfo: { | 20260505 13:20:52.133 | INFO | "gpu" : " 17 GB", | 20260505 13:20:52.133 | INFO | "huge" : "", | 20260505 13:20:52.133 | INFO | "huge_shared" : " 27 GB", | 20260505 13:20:52.133 | INFO | "malloc" : " 14 GB", | 20260505 13:20:52.133 | INFO | "pinned" : " 29 GB" | 20260505 13:20:52.133 | INFO | } | 20260505 13:20:52.133 | INFO | Perf report for the last 5.14196 seconds: | 20260505 13:20:52.133 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:20:52.133 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:52.133 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:52.133 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:52.133 | INFO | ANALYSIS : 64.5%, 3.0, 100.1ms, 0 | 20260505 13:20:52.133 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:52.134 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260505 13:20:52.134 | INFO | PRE_HQ : 60.8%, 1.8, 58.0ms, 0 | 20260505 13:20:52.134 | INFO | BAZWRITER : 64.7%, 4.0, 134.2ms, 0 | 20260505 13:20:52.134 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 13:20:52.134 | INFO | Uploaded 36GiB to GPU in 2.17304s (16.5667GiB/s) | 20260505 13:20:52.134 | INFO | Downloaded 38GiB to GPU in 2.27635s (16.6934GiB/s) | 20260505 13:20:57.275 | INFO | Analyzing chunk frames = [3523072,3523584) | 20260505 13:20:57.275 | INFO | MemInfo: { | 20260505 13:20:57.275 | INFO | "gpu" : " 17 GB", | 20260505 13:20:57.275 | INFO | "huge" : "", | 20260505 13:20:57.275 | INFO | "huge_shared" : " 27 GB", | 20260505 13:20:57.275 | INFO | "malloc" : " 14 GB", | 20260505 13:20:57.275 | INFO | "pinned" : " 29 GB" | 20260505 13:20:57.275 | INFO | } | 20260505 13:20:57.275 | INFO | Perf report for the last 5.14126 seconds: | 20260505 13:20:57.275 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:20:57.275 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:57.275 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:57.275 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:57.275 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 13:20:57.275 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:57.275 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:57.275 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 13:20:57.275 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:57.275 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:20:57.275 | INFO | Uploaded 36GiB to GPU in 2.17726s (16.5346GiB/s) | 20260505 13:20:57.275 | INFO | Downloaded 34GiB to GPU in 2.02812s (16.7643GiB/s) | 20260505 13:21:02.412 | INFO | Analyzing chunk frames = [3523584,3524096) | 20260505 13:21:02.412 | INFO | MemInfo: { | 20260505 13:21:02.412 | INFO | "gpu" : " 17 GB", | 20260505 13:21:02.412 | INFO | "huge" : "", | 20260505 13:21:02.412 | INFO | "huge_shared" : " 27 GB", | 20260505 13:21:02.412 | INFO | "malloc" : " 14 GB", | 20260505 13:21:02.412 | INFO | "pinned" : " 29 GB" | 20260505 13:21:02.412 | INFO | } | 20260505 13:21:02.412 | INFO | Perf report for the last 5.13725 seconds: | 20260505 13:21:02.412 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:21:02.412 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:02.412 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:02.412 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:02.412 | INFO | ANALYSIS : 63.4%, 3.0, 98.6ms, 0 | 20260505 13:21:02.412 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:02.412 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:02.412 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:21:02.412 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:02.412 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:02.412 | INFO | Uploaded 36GiB to GPU in 2.22842s (16.1549GiB/s) | 20260505 13:21:02.412 | INFO | Downloaded 34GiB to GPU in 2.10241s (16.1719GiB/s) | 20260505 13:21:07.555 | INFO | Analyzing chunk frames = [3524096,3524608) | 20260505 13:21:07.555 | INFO | MemInfo: { | 20260505 13:21:07.555 | INFO | "gpu" : " 17 GB", | 20260505 13:21:07.555 | INFO | "huge" : "", | 20260505 13:21:07.555 | INFO | "huge_shared" : " 27 GB", | 20260505 13:21:07.555 | INFO | "malloc" : " 14 GB", | 20260505 13:21:07.555 | INFO | "pinned" : " 29 GB" | 20260505 13:21:07.555 | INFO | } | 20260505 13:21:07.555 | INFO | Perf report for the last 5.1433 seconds: | 20260505 13:21:07.555 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:21:07.555 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:07.555 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:07.555 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:07.555 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260505 13:21:07.555 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:07.555 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:07.555 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 13:21:07.555 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:07.556 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:07.556 | INFO | Uploaded 36GiB to GPU in 2.19942s (16.3679GiB/s) | 20260505 13:21:07.556 | INFO | Downloaded 34GiB to GPU in 2.13296s (15.9403GiB/s) | 20260505 13:21:12.683 | INFO | Analyzing chunk frames = [3524608,3525120) | 20260505 13:21:12.683 | INFO | MemInfo: { | 20260505 13:21:12.683 | INFO | "gpu" : " 17 GB", | 20260505 13:21:12.683 | INFO | "huge" : "", | 20260505 13:21:12.683 | INFO | "huge_shared" : " 27 GB", | 20260505 13:21:12.683 | INFO | "malloc" : " 14 GB", | 20260505 13:21:12.683 | INFO | "pinned" : " 29 GB" | 20260505 13:21:12.683 | INFO | } | 20260505 13:21:12.683 | INFO | Perf report for the last 5.12772 seconds: | 20260505 13:21:12.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:21:12.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:12.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:12.683 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:12.683 | INFO | ANALYSIS : 63.0%, 3.0, 98.1ms, 0 | 20260505 13:21:12.683 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:12.683 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:12.683 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 13:21:12.683 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:12.683 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:12.683 | INFO | Uploaded 36GiB to GPU in 2.15195s (16.729GiB/s) | 20260505 13:21:12.683 | INFO | Downloaded 34GiB to GPU in 2.12118s (16.0288GiB/s) | 20260505 13:21:17.822 | INFO | Analyzing chunk frames = [3525120,3525632) | 20260505 13:21:17.822 | INFO | MemInfo: { | 20260505 13:21:17.822 | INFO | "gpu" : " 17 GB", | 20260505 13:21:17.822 | INFO | "huge" : "", | 20260505 13:21:17.822 | INFO | "huge_shared" : " 27 GB", | 20260505 13:21:17.822 | INFO | "malloc" : " 14 GB", | 20260505 13:21:17.822 | INFO | "pinned" : " 29 GB" | 20260505 13:21:17.822 | INFO | } | 20260505 13:21:17.822 | INFO | Perf report for the last 5.13895 seconds: | 20260505 13:21:17.822 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:21:17.822 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:17.822 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:17.822 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:17.822 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 13:21:17.822 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:17.822 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:17.822 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:21:17.822 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:17.822 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:17.822 | INFO | Uploaded 36GiB to GPU in 2.15908s (16.6738GiB/s) | 20260505 13:21:17.822 | INFO | Downloaded 34GiB to GPU in 2.11184s (16.0997GiB/s) | 20260505 13:21:22.968 | INFO | Analyzing chunk frames = [3525632,3526144) | 20260505 13:21:22.968 | INFO | MemInfo: { | 20260505 13:21:22.968 | INFO | "gpu" : " 17 GB", | 20260505 13:21:22.968 | INFO | "huge" : "", | 20260505 13:21:22.968 | INFO | "huge_shared" : " 27 GB", | 20260505 13:21:22.968 | INFO | "malloc" : " 14 GB", | 20260505 13:21:22.969 | INFO | "pinned" : " 29 GB" | 20260505 13:21:22.969 | INFO | } | 20260505 13:21:22.969 | INFO | Perf report for the last 5.14667 seconds: | 20260505 13:21:22.969 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:21:22.969 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:22.969 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:22.969 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:22.969 | INFO | ANALYSIS : 63.3%, 3.0, 99.0ms, 0 | 20260505 13:21:22.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:22.969 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:22.969 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 13:21:22.969 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:22.969 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:22.969 | INFO | Uploaded 36GiB to GPU in 2.16276s (16.6454GiB/s) | 20260505 13:21:22.969 | INFO | Downloaded 34GiB to GPU in 2.13923s (15.8935GiB/s) | 20260505 13:21:28.118 | INFO | Analyzing chunk frames = [3526144,3526656) | 20260505 13:21:28.118 | INFO | MemInfo: { | 20260505 13:21:28.118 | INFO | "gpu" : " 17 GB", | 20260505 13:21:28.118 | INFO | "huge" : "", | 20260505 13:21:28.118 | INFO | "huge_shared" : " 27 GB", | 20260505 13:21:28.118 | INFO | "malloc" : " 14 GB", | 20260505 13:21:28.118 | INFO | "pinned" : " 29 GB" | 20260505 13:21:28.118 | INFO | } | 20260505 13:21:28.118 | INFO | Perf report for the last 5.14976 seconds: | 20260505 13:21:28.118 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:21:28.118 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:28.118 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:28.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:28.118 | INFO | ANALYSIS : 63.4%, 3.0, 99.3ms, 0 | 20260505 13:21:28.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:28.118 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:28.118 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 13:21:28.119 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:28.119 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:28.119 | INFO | Uploaded 36GiB to GPU in 2.20652s (16.3153GiB/s) | 20260505 13:21:28.119 | INFO | Downloaded 34GiB to GPU in 2.10169s (16.1774GiB/s) | 20260505 13:21:28.256 | INFO | Starting a round of Baz IO | 20260505 13:21:31.485 | INFO | Populating RTMetrics report | 20260505 13:21:31.485 | INFO | RTMetrics report populated | 20260505 13:21:31.485 | INFO | Opening tmp RTMetrics json file | 20260505 13:21:31.486 | INFO | Serializing RTMetrics to json | 20260505 13:21:31.487 | INFO | Serialization done, swapping tmp file now... | 20260505 13:21:31.488 | INFO | RTMetrics done | 20260505 13:21:31.604 | INFO | Finished a round of Baz IO: Wrote 2.3 GiB in 3.348 seconds | 20260505 13:21:31.604 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.16% / 0.00% / 76.84% | 20260505 13:21:31.604 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.715232% / 3.97364e-06% / 7.15256e-05% | 20260505 13:21:31.604 | INFO | Baz IO overall: | 20260505 13:21:31.605 | INFO | BytesWritten : 2100206591321 | 20260505 13:21:31.605 | INFO | NumEvents : 999247913109 | 20260505 13:21:33.249 | INFO | Analyzing chunk frames = [3526656,3527168) | 20260505 13:21:33.249 | INFO | MemInfo: { | 20260505 13:21:33.249 | INFO | "gpu" : " 17 GB", | 20260505 13:21:33.249 | INFO | "huge" : "", | 20260505 13:21:33.249 | INFO | "huge_shared" : " 27 GB", | 20260505 13:21:33.249 | INFO | "malloc" : " 14 GB", | 20260505 13:21:33.249 | INFO | "pinned" : " 29 GB" | 20260505 13:21:33.249 | INFO | } | 20260505 13:21:33.250 | INFO | Perf report for the last 5.1312 seconds: | 20260505 13:21:33.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:21:33.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:33.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:33.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:33.250 | INFO | ANALYSIS : 64.5%, 3.0, 100.0ms, 0 | 20260505 13:21:33.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:33.250 | INFO | RT_METRICS : 45.3%, 1.0, 23.7ms, 0 | 20260505 13:21:33.250 | INFO | PRE_HQ : 61.1%, 1.8, 57.9ms, 0 | 20260505 13:21:33.250 | INFO | BAZWRITER : 64.6%, 4.0, 133.5ms, 0 | 20260505 13:21:33.250 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 13:21:33.250 | INFO | Uploaded 36GiB to GPU in 2.19356s (16.4117GiB/s) | 20260505 13:21:33.250 | INFO | Downloaded 38GiB to GPU in 2.25842s (16.8259GiB/s) | 20260505 13:21:38.383 | INFO | Analyzing chunk frames = [3527168,3527680) | 20260505 13:21:38.383 | INFO | MemInfo: { | 20260505 13:21:38.383 | INFO | "gpu" : " 17 GB", | 20260505 13:21:38.383 | INFO | "huge" : "", | 20260505 13:21:38.383 | INFO | "huge_shared" : " 27 GB", | 20260505 13:21:38.383 | INFO | "malloc" : " 14 GB", | 20260505 13:21:38.383 | INFO | "pinned" : " 29 GB" | 20260505 13:21:38.383 | INFO | } | 20260505 13:21:38.383 | INFO | Perf report for the last 5.1336 seconds: | 20260505 13:21:38.383 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:21:38.383 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:38.383 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:38.383 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:38.383 | INFO | ANALYSIS : 63.6%, 3.0, 99.1ms, 0 | 20260505 13:21:38.383 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:38.383 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:38.383 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 13:21:38.383 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:38.383 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:38.383 | INFO | Uploaded 36GiB to GPU in 2.17404s (16.559GiB/s) | 20260505 13:21:38.383 | INFO | Downloaded 34GiB to GPU in 2.028s (16.7653GiB/s) | 20260505 13:21:43.528 | INFO | Analyzing chunk frames = [3527680,3528192) | 20260505 13:21:43.528 | INFO | MemInfo: { | 20260505 13:21:43.528 | INFO | "gpu" : " 17 GB", | 20260505 13:21:43.528 | INFO | "huge" : "", | 20260505 13:21:43.528 | INFO | "huge_shared" : " 27 GB", | 20260505 13:21:43.528 | INFO | "malloc" : " 14 GB", | 20260505 13:21:43.528 | INFO | "pinned" : " 29 GB" | 20260505 13:21:43.528 | INFO | } | 20260505 13:21:43.529 | INFO | Perf report for the last 5.14535 seconds: | 20260505 13:21:43.529 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:21:43.529 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:43.529 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:43.529 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:43.529 | INFO | ANALYSIS : 63.1%, 3.0, 98.3ms, 0 | 20260505 13:21:43.529 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:43.529 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:43.529 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:21:43.529 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:43.529 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:43.529 | INFO | Uploaded 36GiB to GPU in 2.20921s (16.2954GiB/s) | 20260505 13:21:43.529 | INFO | Downloaded 34GiB to GPU in 2.11951s (16.0414GiB/s) | 20260505 13:21:48.665 | INFO | Analyzing chunk frames = [3528192,3528704) | 20260505 13:21:48.665 | INFO | MemInfo: { | 20260505 13:21:48.665 | INFO | "gpu" : " 17 GB", | 20260505 13:21:48.665 | INFO | "huge" : "", | 20260505 13:21:48.665 | INFO | "huge_shared" : " 27 GB", | 20260505 13:21:48.665 | INFO | "malloc" : " 14 GB", | 20260505 13:21:48.665 | INFO | "pinned" : " 29 GB" | 20260505 13:21:48.665 | INFO | } | 20260505 13:21:48.665 | INFO | Perf report for the last 5.13688 seconds: | 20260505 13:21:48.666 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:21:48.666 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:48.666 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:48.666 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:48.666 | INFO | ANALYSIS : 62.8%, 3.0, 97.5ms, 0 | 20260505 13:21:48.666 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:48.666 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:48.666 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 13:21:48.666 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:48.666 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:48.666 | INFO | Uploaded 36GiB to GPU in 2.17962s (16.5166GiB/s) | 20260505 13:21:48.666 | INFO | Downloaded 34GiB to GPU in 2.12936s (15.9672GiB/s) | 20260505 13:21:53.802 | INFO | Analyzing chunk frames = [3528704,3529216) | 20260505 13:21:53.802 | INFO | MemInfo: { | 20260505 13:21:53.802 | INFO | "gpu" : " 17 GB", | 20260505 13:21:53.802 | INFO | "huge" : "", | 20260505 13:21:53.802 | INFO | "huge_shared" : " 27 GB", | 20260505 13:21:53.802 | INFO | "malloc" : " 14 GB", | 20260505 13:21:53.802 | INFO | "pinned" : " 29 GB" | 20260505 13:21:53.802 | INFO | } | 20260505 13:21:53.802 | INFO | Perf report for the last 5.13649 seconds: | 20260505 13:21:53.802 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:21:53.802 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:53.802 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:53.802 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:53.802 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 13:21:53.802 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:53.802 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:53.802 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 13:21:53.802 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:53.802 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:53.802 | INFO | Uploaded 36GiB to GPU in 2.17253s (16.5706GiB/s) | 20260505 13:21:53.802 | INFO | Downloaded 34GiB to GPU in 2.12584s (15.9937GiB/s) | 20260505 13:21:58.938 | INFO | Analyzing chunk frames = [3529216,3529728) | 20260505 13:21:58.938 | INFO | MemInfo: { | 20260505 13:21:58.938 | INFO | "gpu" : " 17 GB", | 20260505 13:21:58.938 | INFO | "huge" : "", | 20260505 13:21:58.938 | INFO | "huge_shared" : " 27 GB", | 20260505 13:21:58.938 | INFO | "malloc" : " 14 GB", | 20260505 13:21:58.938 | INFO | "pinned" : " 29 GB" | 20260505 13:21:58.938 | INFO | } | 20260505 13:21:58.938 | INFO | Perf report for the last 5.13658 seconds: | 20260505 13:21:58.938 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:21:58.938 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:58.938 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:58.939 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:58.939 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 13:21:58.939 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:58.939 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:58.939 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 13:21:58.939 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:58.939 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:21:58.939 | INFO | Uploaded 36GiB to GPU in 2.13969s (16.8249GiB/s) | 20260505 13:21:58.939 | INFO | Downloaded 34GiB to GPU in 2.1043s (16.1574GiB/s) | 20260505 13:22:04.080 | INFO | Analyzing chunk frames = [3529728,3530240) | 20260505 13:22:04.080 | INFO | MemInfo: { | 20260505 13:22:04.080 | INFO | "gpu" : " 17 GB", | 20260505 13:22:04.080 | INFO | "huge" : "", | 20260505 13:22:04.081 | INFO | "huge_shared" : " 27 GB", | 20260505 13:22:04.081 | INFO | "malloc" : " 14 GB", | 20260505 13:22:04.081 | INFO | "pinned" : " 29 GB" | 20260505 13:22:04.081 | INFO | } | 20260505 13:22:04.081 | INFO | Perf report for the last 5.14228 seconds: | 20260505 13:22:04.081 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:22:04.081 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:04.081 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:04.081 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:04.081 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 13:22:04.081 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:04.081 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:04.081 | INFO | PRE_HQ : 29.7%, 1.0, 15.5ms, 0 | 20260505 13:22:04.081 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:04.081 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:04.081 | INFO | Uploaded 36GiB to GPU in 2.1871s (16.4601GiB/s) | 20260505 13:22:04.081 | INFO | Downloaded 34GiB to GPU in 2.14628s (15.8414GiB/s) | 20260505 13:22:09.231 | INFO | Analyzing chunk frames = [3530240,3530752) | 20260505 13:22:09.231 | INFO | MemInfo: { | 20260505 13:22:09.231 | INFO | "gpu" : " 17 GB", | 20260505 13:22:09.231 | INFO | "huge" : "", | 20260505 13:22:09.231 | INFO | "huge_shared" : " 27 GB", | 20260505 13:22:09.231 | INFO | "malloc" : " 14 GB", | 20260505 13:22:09.231 | INFO | "pinned" : " 29 GB" | 20260505 13:22:09.231 | INFO | } | 20260505 13:22:09.231 | INFO | Perf report for the last 5.15035 seconds: | 20260505 13:22:09.231 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:22:09.231 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:09.231 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:09.231 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:09.231 | INFO | ANALYSIS : 63.4%, 3.0, 99.2ms, 0 | 20260505 13:22:09.231 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:09.231 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:09.231 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 13:22:09.231 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:09.231 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:09.231 | INFO | Uploaded 36GiB to GPU in 2.13716s (16.8448GiB/s) | 20260505 13:22:09.232 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 13:22:09.232 | INFO | Total mem usage: 15.3 GiB | 20260505 13:22:09.232 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 13:22:09.232 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.75%) | 20260505 13:22:09.232 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 13:22:09.232 | INFO | High Water Breakdown: | 20260505 13:22:09.232 | INFO | 61% : 8.68 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 13:22:09.232 | INFO | 38.9% : 5.53 GiB ( 83.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 13:22:09.232 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 13:22:09.232 | INFO | ------------------------------------------ | 20260505 13:22:09.232 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:22:09.232 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 13:22:09.232 | INFO | Total mem usage: 26.5 GiB | 20260505 13:22:09.232 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 13:22:09.232 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 13:22:09.232 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 13:22:09.232 | INFO | High Water Breakdown: | 20260505 13:22:09.232 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:22:09.232 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:22:09.232 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:22:09.232 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:22:09.232 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:22:09.232 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:22:09.232 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 13:22:09.232 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 13:22:09.232 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:22:09.232 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 13:22:09.232 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 13:22:09.232 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 13:22:09.232 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 13:22:09.232 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 13:22:09.232 | INFO | ------------------------------------------ | 20260505 13:22:09.232 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:22:09.232 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 13:22:09.232 | INFO | Total mem usage: 25.6 GiB | 20260505 13:22:09.232 | INFO | Downloaded 34GiB to GPU in 2.10188s (16.176GiB/s) | 20260505 13:22:09.232 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 13:22:09.232 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 13:22:09.232 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 13:22:09.232 | INFO | High Water Breakdown: | 20260505 13:22:09.232 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 13:22:09.232 | INFO | ------------------------------------------ | 20260505 13:22:09.232 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:22:09.232 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 13:22:09.232 | INFO | Total mem usage: 15.7 GiB | 20260505 13:22:09.232 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 13:22:09.232 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 13:22:09.232 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 13:22:09.232 | INFO | High Water Breakdown: | 20260505 13:22:09.232 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 13:22:09.232 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 13:22:09.232 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 13:22:09.232 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 13:22:09.232 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 13:22:09.232 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 13:22:09.232 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 13:22:09.232 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 13:22:09.232 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 13:22:09.232 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:22:09.232 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 13:22:09.232 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:22:09.232 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 13:22:09.232 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 13:22:09.232 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 13:22:09.233 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 13:22:09.233 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:22:09.233 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 13:22:09.233 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:22:09.233 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:22:09.233 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:22:09.233 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 13:22:09.233 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 13:22:09.233 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:22:09.233 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 13:22:09.233 | INFO | ------------------------------------------ | 20260505 13:22:09.233 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:22:09.233 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 13:22:09.233 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:22:09.233 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:22:09.233 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.66e+01, 1.37e+01 / 1.66e+00 | 20260505 13:22:09.233 | INFO | PerfReport: 0.29%, AggregateStats -- 9.31e-02 / 9.99e-02, 1.08e-02 / 3.77e-03 | 20260505 13:22:09.233 | INFO | PerfReport: 11.78%, Baseline -- 3.76e+00 / 4.48e+00, 5.92e-01 / 2.15e-01 | 20260505 13:22:09.233 | INFO | PerfReport: 3.31%, Histogram -- 1.06e+00 / 1.51e+00, 2.03e-01 / 8.16e-02 | 20260505 13:22:09.233 | INFO | PerfReport: 15.99%, DME -- 4.08e+01 / 4.57e+01, 4.60e+00 / 1.70e+00 | 20260505 13:22:09.233 | INFO | PerfReport: 56.32%, FrameLabeling -- 1.80e+01 / 2.00e+01, 1.42e+00 / 1.81e+00 | 20260505 13:22:09.233 | INFO | PerfReport: 7.46%, PulseAccumulating -- 2.38e+00 / 2.91e+00, 4.48e-01 / 2.06e-01 | 20260505 13:22:09.233 | INFO | PerfReport: 3.59%, Metrics -- 1.15e+00 / 1.46e+00, 3.20e-01 / 1.64e-01 | 20260505 13:22:09.233 | INFO | PerfReport: 1.26%, PulseSerializing -- 4.01e-01 / 4.49e-01, 7.05e-02 / 2.63e-02 | 20260505 13:22:09.233 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 13:22:09.233 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:22:09.233 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:22:09.233 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260505 13:22:09.233 | INFO | PerfReport: 22.50%, Upload -- 2.27e+01 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260505 13:22:09.233 | INFO | PerfReport: 21.97%, Download -- 2.22e+01 / 0.00e+00, 8.07e+00 / 0.00e+00 | 20260505 13:22:09.233 | INFO | PerfReport: 31.65%, Compute -- 3.19e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 13:22:09.233 | INFO | PerfReport: 2.14%, LockUpload -- 2.16e+00 / 0.00e+00, 7.78e+00 / 0.00e+00 | 20260505 13:22:09.233 | INFO | PerfReport: 1.44%, LockDownload -- 1.45e+00 / 0.00e+00, 3.01e+00 / 0.00e+00 | 20260505 13:22:09.233 | INFO | PerfReport: 20.28%, LockCompute -- 2.04e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260505 13:22:09.233 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 2.34e-02 / 0.00e+00 | 20260505 13:22:09.362 | INFO | Starting a round of Baz IO | 20260505 13:22:12.586 | INFO | Populating RTMetrics report | 20260505 13:22:12.587 | INFO | RTMetrics report populated | 20260505 13:22:12.587 | INFO | Opening tmp RTMetrics json file | 20260505 13:22:12.587 | INFO | Serializing RTMetrics to json | 20260505 13:22:12.589 | INFO | Serialization done, swapping tmp file now... | 20260505 13:22:12.589 | INFO | RTMetrics done | 20260505 13:22:12.695 | INFO | Finished a round of Baz IO: Wrote 2.302 GiB in 3.332 seconds | 20260505 13:22:12.695 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.15% / 0.00% / 76.84% | 20260505 13:22:12.695 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.716527% / 1.58946e-05% / 9.93411e-05% | 20260505 13:22:12.695 | INFO | Baz IO overall: | 20260505 13:22:12.695 | INFO | BytesWritten : 2102678550722 | 20260505 13:22:12.695 | INFO | NumEvents : 1000417498868 | 20260505 13:22:14.368 | INFO | Analyzing chunk frames = [3530752,3531264) | 20260505 13:22:14.368 | INFO | MemInfo: { | 20260505 13:22:14.368 | INFO | "gpu" : " 17 GB", | 20260505 13:22:14.368 | INFO | "huge" : "", | 20260505 13:22:14.368 | INFO | "huge_shared" : " 27 GB", | 20260505 13:22:14.368 | INFO | "malloc" : " 14 GB", | 20260505 13:22:14.368 | INFO | "pinned" : " 29 GB" | 20260505 13:22:14.368 | INFO | } | 20260505 13:22:14.368 | INFO | Perf report for the last 5.13724 seconds: | 20260505 13:22:14.369 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:22:14.369 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:14.369 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:14.369 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:14.369 | INFO | ANALYSIS : 64.2%, 3.0, 99.3ms, 0 | 20260505 13:22:14.369 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:14.369 | INFO | RT_METRICS : 45.1%, 1.0, 23.6ms, 0 | 20260505 13:22:14.369 | INFO | PRE_HQ : 61.3%, 1.8, 57.7ms, 0 | 20260505 13:22:14.369 | INFO | BAZWRITER : 64.2%, 4.0, 133.5ms, 0 | 20260505 13:22:14.369 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 13:22:14.369 | INFO | Uploaded 36GiB to GPU in 2.21508s (16.2523GiB/s) | 20260505 13:22:14.369 | INFO | Downloaded 38GiB to GPU in 2.27058s (16.7358GiB/s) | 20260505 13:22:19.503 | INFO | Analyzing chunk frames = [3531264,3531776) | 20260505 13:22:19.503 | INFO | MemInfo: { | 20260505 13:22:19.503 | INFO | "gpu" : " 17 GB", | 20260505 13:22:19.503 | INFO | "huge" : "", | 20260505 13:22:19.503 | INFO | "huge_shared" : " 27 GB", | 20260505 13:22:19.503 | INFO | "malloc" : " 14 GB", | 20260505 13:22:19.503 | INFO | "pinned" : " 29 GB" | 20260505 13:22:19.503 | INFO | } | 20260505 13:22:19.503 | INFO | Perf report for the last 5.13475 seconds: | 20260505 13:22:19.503 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:22:19.503 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:19.503 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:19.503 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:19.503 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 13:22:19.503 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:19.503 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:19.503 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:22:19.503 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:19.503 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:19.504 | INFO | Uploaded 36GiB to GPU in 2.20076s (16.358GiB/s) | 20260505 13:22:19.504 | INFO | Downloaded 34GiB to GPU in 2.03727s (16.689GiB/s) | 20260505 13:22:24.630 | INFO | Analyzing chunk frames = [3531776,3532288) | 20260505 13:22:24.630 | INFO | MemInfo: { | 20260505 13:22:24.630 | INFO | "gpu" : " 17 GB", | 20260505 13:22:24.630 | INFO | "huge" : "", | 20260505 13:22:24.630 | INFO | "huge_shared" : " 27 GB", | 20260505 13:22:24.630 | INFO | "malloc" : " 14 GB", | 20260505 13:22:24.630 | INFO | "pinned" : " 29 GB" | 20260505 13:22:24.630 | INFO | } | 20260505 13:22:24.630 | INFO | Perf report for the last 5.12715 seconds: | 20260505 13:22:24.630 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:22:24.630 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:24.631 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:24.631 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:24.631 | INFO | ANALYSIS : 63.6%, 3.0, 98.7ms, 0 | 20260505 13:22:24.631 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:24.631 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:24.631 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 13:22:24.631 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:24.631 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:24.631 | INFO | Uploaded 36GiB to GPU in 2.23268s (16.1241GiB/s) | 20260505 13:22:24.631 | INFO | Downloaded 34GiB to GPU in 2.10924s (16.1195GiB/s) | 20260505 13:22:29.779 | INFO | Analyzing chunk frames = [3532288,3532800) | 20260505 13:22:29.779 | INFO | MemInfo: { | 20260505 13:22:29.779 | INFO | "gpu" : " 17 GB", | 20260505 13:22:29.779 | INFO | "huge" : "", | 20260505 13:22:29.779 | INFO | "huge_shared" : " 27 GB", | 20260505 13:22:29.779 | INFO | "malloc" : " 14 GB", | 20260505 13:22:29.779 | INFO | "pinned" : " 29 GB" | 20260505 13:22:29.779 | INFO | } | 20260505 13:22:29.779 | INFO | Perf report for the last 5.14882 seconds: | 20260505 13:22:29.779 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:22:29.779 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:29.779 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:29.779 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:29.779 | INFO | ANALYSIS : 63.1%, 3.0, 98.3ms, 0 | 20260505 13:22:29.779 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:29.779 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:29.779 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:22:29.779 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:29.779 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:29.779 | INFO | Uploaded 36GiB to GPU in 2.21372s (16.2622GiB/s) | 20260505 13:22:29.779 | INFO | Downloaded 34GiB to GPU in 2.13543s (15.9218GiB/s) | 20260505 13:22:34.920 | INFO | Analyzing chunk frames = [3532800,3533312) | 20260505 13:22:34.920 | INFO | MemInfo: { | 20260505 13:22:34.920 | INFO | "gpu" : " 17 GB", | 20260505 13:22:34.920 | INFO | "huge" : "", | 20260505 13:22:34.920 | INFO | "huge_shared" : " 27 GB", | 20260505 13:22:34.920 | INFO | "malloc" : " 14 GB", | 20260505 13:22:34.920 | INFO | "pinned" : " 29 GB" | 20260505 13:22:34.920 | INFO | } | 20260505 13:22:34.920 | INFO | Perf report for the last 5.14064 seconds: | 20260505 13:22:34.920 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:22:34.920 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:34.920 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:34.920 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:34.920 | INFO | ANALYSIS : 62.7%, 3.0, 98.0ms, 0 | 20260505 13:22:34.920 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:34.920 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:34.920 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:22:34.920 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:34.920 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:34.920 | INFO | Uploaded 36GiB to GPU in 2.14855s (16.7555GiB/s) | 20260505 13:22:34.920 | INFO | Downloaded 34GiB to GPU in 2.12269s (16.0174GiB/s) | 20260505 13:22:40.066 | INFO | Analyzing chunk frames = [3533312,3533824) | 20260505 13:22:40.066 | INFO | MemInfo: { | 20260505 13:22:40.066 | INFO | "gpu" : " 17 GB", | 20260505 13:22:40.066 | INFO | "huge" : "", | 20260505 13:22:40.066 | INFO | "huge_shared" : " 27 GB", | 20260505 13:22:40.066 | INFO | "malloc" : " 14 GB", | 20260505 13:22:40.066 | INFO | "pinned" : " 29 GB" | 20260505 13:22:40.066 | INFO | } | 20260505 13:22:40.066 | INFO | Perf report for the last 5.14666 seconds: | 20260505 13:22:40.067 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:22:40.067 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:40.067 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:40.067 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:40.067 | INFO | ANALYSIS : 62.9%, 3.0, 98.4ms, 0 | 20260505 13:22:40.067 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:40.067 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:40.067 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 13:22:40.067 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:40.067 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:40.067 | INFO | Uploaded 36GiB to GPU in 2.1354s (16.8587GiB/s) | 20260505 13:22:40.067 | INFO | Downloaded 34GiB to GPU in 2.10559s (16.1475GiB/s) | 20260505 13:22:45.201 | INFO | Analyzing chunk frames = [3533824,3534336) | 20260505 13:22:45.201 | INFO | MemInfo: { | 20260505 13:22:45.201 | INFO | "gpu" : " 17 GB", | 20260505 13:22:45.201 | INFO | "huge" : "", | 20260505 13:22:45.201 | INFO | "huge_shared" : " 27 GB", | 20260505 13:22:45.201 | INFO | "malloc" : " 14 GB", | 20260505 13:22:45.201 | INFO | "pinned" : " 29 GB" | 20260505 13:22:45.201 | INFO | } | 20260505 13:22:45.201 | INFO | Perf report for the last 5.13454 seconds: | 20260505 13:22:45.201 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:22:45.201 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:45.201 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:45.201 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:45.201 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 13:22:45.201 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:45.201 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:45.201 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 13:22:45.201 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:45.201 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:45.201 | INFO | Uploaded 36GiB to GPU in 2.15475s (16.7072GiB/s) | 20260505 13:22:45.202 | INFO | Downloaded 34GiB to GPU in 2.18198s (15.5822GiB/s) | 20260505 13:22:50.334 | INFO | Analyzing chunk frames = [3534336,3534848) | 20260505 13:22:50.334 | INFO | MemInfo: { | 20260505 13:22:50.334 | INFO | "gpu" : " 17 GB", | 20260505 13:22:50.334 | INFO | "huge" : "", | 20260505 13:22:50.334 | INFO | "huge_shared" : " 27 GB", | 20260505 13:22:50.334 | INFO | "malloc" : " 14 GB", | 20260505 13:22:50.334 | INFO | "pinned" : " 29 GB" | 20260505 13:22:50.334 | INFO | } | 20260505 13:22:50.334 | INFO | Perf report for the last 5.13322 seconds: | 20260505 13:22:50.334 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:22:50.334 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:50.334 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:50.334 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:50.335 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 13:22:50.335 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:50.335 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:50.335 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 13:22:50.335 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:50.335 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:50.335 | INFO | Uploaded 36GiB to GPU in 2.12337s (16.9542GiB/s) | 20260505 13:22:50.335 | INFO | Downloaded 34GiB to GPU in 2.10084s (16.184GiB/s) | 20260505 13:22:50.467 | INFO | Starting a round of Baz IO | 20260505 13:22:53.706 | INFO | Populating RTMetrics report | 20260505 13:22:53.707 | INFO | RTMetrics report populated | 20260505 13:22:53.707 | INFO | Opening tmp RTMetrics json file | 20260505 13:22:53.726 | INFO | Serializing RTMetrics to json | 20260505 13:22:53.728 | INFO | Serialization done, swapping tmp file now... | 20260505 13:22:53.731 | INFO | RTMetrics done | 20260505 13:22:53.827 | INFO | Finished a round of Baz IO: Wrote 2.301 GiB in 3.359 seconds | 20260505 13:22:53.827 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.15% / 0.00% / 76.85% | 20260505 13:22:53.827 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.713849% / 0% / 8.74201e-05% | 20260505 13:22:53.827 | INFO | Baz IO overall: | 20260505 13:22:53.827 | INFO | BytesWritten : 2105149565615 | 20260505 13:22:53.827 | INFO | NumEvents : 1001586426754 | 20260505 13:22:55.483 | INFO | Analyzing chunk frames = [3534848,3535360) | 20260505 13:22:55.483 | INFO | MemInfo: { | 20260505 13:22:55.483 | INFO | "gpu" : " 17 GB", | 20260505 13:22:55.483 | INFO | "huge" : "", | 20260505 13:22:55.483 | INFO | "huge_shared" : " 27 GB", | 20260505 13:22:55.483 | INFO | "malloc" : " 14 GB", | 20260505 13:22:55.483 | INFO | "pinned" : " 29 GB" | 20260505 13:22:55.483 | INFO | } | 20260505 13:22:55.483 | INFO | Perf report for the last 5.14896 seconds: | 20260505 13:22:55.483 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:22:55.483 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:55.483 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:55.483 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:55.483 | INFO | ANALYSIS : 64.4%, 3.0, 99.9ms, 0 | 20260505 13:22:55.483 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:22:55.483 | INFO | RT_METRICS : 45.4%, 1.0, 23.9ms, 0 | 20260505 13:22:55.483 | INFO | PRE_HQ : 60.7%, 1.8, 57.7ms, 0 | 20260505 13:22:55.483 | INFO | BAZWRITER : 64.6%, 4.0, 133.5ms, 0 | 20260505 13:22:55.483 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 13:22:55.484 | INFO | Uploaded 36GiB to GPU in 2.18631s (16.4661GiB/s) | 20260505 13:22:55.484 | INFO | Downloaded 38GiB to GPU in 2.27302s (16.7178GiB/s) | 20260505 13:23:00.616 | INFO | Analyzing chunk frames = [3535360,3535872) | 20260505 13:23:00.616 | INFO | MemInfo: { | 20260505 13:23:00.616 | INFO | "gpu" : " 17 GB", | 20260505 13:23:00.616 | INFO | "huge" : "", | 20260505 13:23:00.616 | INFO | "huge_shared" : " 27 GB", | 20260505 13:23:00.616 | INFO | "malloc" : " 14 GB", | 20260505 13:23:00.616 | INFO | "pinned" : " 29 GB" | 20260505 13:23:00.616 | INFO | } | 20260505 13:23:00.616 | INFO | Perf report for the last 5.13273 seconds: | 20260505 13:23:00.616 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:23:00.616 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:00.616 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:00.616 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:00.616 | INFO | ANALYSIS : 63.6%, 3.0, 99.1ms, 0 | 20260505 13:23:00.616 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:00.616 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:00.616 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:23:00.616 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:00.616 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:00.616 | INFO | Uploaded 36GiB to GPU in 2.19529s (16.3988GiB/s) | 20260505 13:23:00.617 | INFO | Downloaded 34GiB to GPU in 2.03592s (16.7001GiB/s) | 20260505 13:23:05.751 | INFO | Analyzing chunk frames = [3535872,3536384) | 20260505 13:23:05.751 | INFO | MemInfo: { | 20260505 13:23:05.751 | INFO | "gpu" : " 17 GB", | 20260505 13:23:05.751 | INFO | "huge" : "", | 20260505 13:23:05.751 | INFO | "huge_shared" : " 27 GB", | 20260505 13:23:05.751 | INFO | "malloc" : " 14 GB", | 20260505 13:23:05.751 | INFO | "pinned" : " 29 GB" | 20260505 13:23:05.751 | INFO | } | 20260505 13:23:05.751 | INFO | Perf report for the last 5.13547 seconds: | 20260505 13:23:05.752 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:23:05.752 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:05.752 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:05.752 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:05.752 | INFO | ANALYSIS : 63.2%, 3.0, 98.3ms, 0 | 20260505 13:23:05.752 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:05.752 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:05.752 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 13:23:05.752 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:05.752 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:05.752 | INFO | Uploaded 36GiB to GPU in 2.17156s (16.5779GiB/s) | 20260505 13:23:05.752 | INFO | Downloaded 34GiB to GPU in 2.12057s (16.0335GiB/s) | 20260505 13:23:10.889 | INFO | Analyzing chunk frames = [3536384,3536896) | 20260505 13:23:10.889 | INFO | MemInfo: { | 20260505 13:23:10.889 | INFO | "gpu" : " 17 GB", | 20260505 13:23:10.889 | INFO | "huge" : "", | 20260505 13:23:10.889 | INFO | "huge_shared" : " 27 GB", | 20260505 13:23:10.889 | INFO | "malloc" : " 14 GB", | 20260505 13:23:10.889 | INFO | "pinned" : " 29 GB" | 20260505 13:23:10.889 | INFO | } | 20260505 13:23:10.889 | INFO | Perf report for the last 5.1372 seconds: | 20260505 13:23:10.889 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:23:10.889 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:10.889 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:10.889 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:10.889 | INFO | ANALYSIS : 63.0%, 3.0, 97.8ms, 0 | 20260505 13:23:10.889 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:10.889 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:10.889 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:23:10.889 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:10.889 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:10.889 | INFO | Uploaded 36GiB to GPU in 2.18979s (16.4399GiB/s) | 20260505 13:23:10.889 | INFO | Downloaded 34GiB to GPU in 2.1335s (15.9363GiB/s) | 20260505 13:23:16.036 | INFO | Analyzing chunk frames = [3536896,3537408) | 20260505 13:23:16.036 | INFO | MemInfo: { | 20260505 13:23:16.036 | INFO | "gpu" : " 17 GB", | 20260505 13:23:16.036 | INFO | "huge" : "", | 20260505 13:23:16.036 | INFO | "huge_shared" : " 27 GB", | 20260505 13:23:16.036 | INFO | "malloc" : " 14 GB", | 20260505 13:23:16.036 | INFO | "pinned" : " 29 GB" | 20260505 13:23:16.036 | INFO | } | 20260505 13:23:16.036 | INFO | Perf report for the last 5.14726 seconds: | 20260505 13:23:16.036 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:23:16.036 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:16.036 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:16.036 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:16.036 | INFO | ANALYSIS : 62.6%, 3.0, 97.9ms, 0 | 20260505 13:23:16.036 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:16.036 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:16.036 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:23:16.036 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:16.036 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:16.036 | INFO | Uploaded 36GiB to GPU in 2.14602s (16.7752GiB/s) | 20260505 13:23:16.036 | INFO | Downloaded 34GiB to GPU in 2.12588s (15.9934GiB/s) | 20260505 13:23:21.177 | INFO | Analyzing chunk frames = [3537408,3537920) | 20260505 13:23:21.177 | INFO | MemInfo: { | 20260505 13:23:21.177 | INFO | "gpu" : " 17 GB", | 20260505 13:23:21.177 | INFO | "huge" : "", | 20260505 13:23:21.177 | INFO | "huge_shared" : " 27 GB", | 20260505 13:23:21.177 | INFO | "malloc" : " 14 GB", | 20260505 13:23:21.177 | INFO | "pinned" : " 29 GB" | 20260505 13:23:21.177 | INFO | } | 20260505 13:23:21.177 | INFO | Perf report for the last 5.1408 seconds: | 20260505 13:23:21.177 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:23:21.177 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:21.177 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:21.177 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:21.177 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 13:23:21.177 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:21.177 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:21.177 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:23:21.177 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:21.177 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:21.177 | INFO | Uploaded 36GiB to GPU in 2.19556s (16.3967GiB/s) | 20260505 13:23:21.177 | INFO | Downloaded 34GiB to GPU in 2.11198s (16.0987GiB/s) | 20260505 13:23:26.314 | INFO | Analyzing chunk frames = [3537920,3538432) | 20260505 13:23:26.314 | INFO | MemInfo: { | 20260505 13:23:26.314 | INFO | "gpu" : " 17 GB", | 20260505 13:23:26.314 | INFO | "huge" : "", | 20260505 13:23:26.314 | INFO | "huge_shared" : " 27 GB", | 20260505 13:23:26.314 | INFO | "malloc" : " 14 GB", | 20260505 13:23:26.314 | INFO | "pinned" : " 29 GB" | 20260505 13:23:26.314 | INFO | } | 20260505 13:23:26.314 | INFO | Perf report for the last 5.13754 seconds: | 20260505 13:23:26.315 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:23:26.315 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:26.315 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:26.315 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:26.315 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 13:23:26.315 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:26.315 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:26.315 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 13:23:26.315 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:26.315 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:26.315 | INFO | Uploaded 36GiB to GPU in 2.17075s (16.5841GiB/s) | 20260505 13:23:26.315 | INFO | Downloaded 34GiB to GPU in 2.14968s (15.8163GiB/s) | 20260505 13:23:31.446 | INFO | Analyzing chunk frames = [3538432,3538944) | 20260505 13:23:31.446 | INFO | MemInfo: { | 20260505 13:23:31.446 | INFO | "gpu" : " 17 GB", | 20260505 13:23:31.446 | INFO | "huge" : "", | 20260505 13:23:31.446 | INFO | "huge_shared" : " 27 GB", | 20260505 13:23:31.446 | INFO | "malloc" : " 14 GB", | 20260505 13:23:31.446 | INFO | "pinned" : " 29 GB" | 20260505 13:23:31.446 | INFO | } | 20260505 13:23:31.446 | INFO | Perf report for the last 5.13212 seconds: | 20260505 13:23:31.446 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:23:31.446 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:31.446 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:31.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:31.447 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 13:23:31.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:31.447 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:31.447 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 13:23:31.447 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:31.447 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:31.447 | INFO | Uploaded 36GiB to GPU in 2.1499s (16.7449GiB/s) | 20260505 13:23:31.447 | INFO | Downloaded 34GiB to GPU in 2.10727s (16.1346GiB/s) | 20260505 13:23:31.447 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 13:23:31.447 | INFO | Total mem usage: 15.3 GiB | 20260505 13:23:31.447 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 13:23:31.447 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.63%) | 20260505 13:23:31.447 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 13:23:31.447 | INFO | High Water Breakdown: | 20260505 13:23:31.447 | INFO | 60.9% : 8.68 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 13:23:31.447 | INFO | 38.9% : 5.54 GiB ( 83.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 13:23:31.447 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 13:23:31.447 | INFO | ------------------------------------------ | 20260505 13:23:31.447 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:23:31.447 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 13:23:31.447 | INFO | Total mem usage: 26.5 GiB | 20260505 13:23:31.447 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 13:23:31.447 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 13:23:31.447 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 13:23:31.447 | INFO | High Water Breakdown: | 20260505 13:23:31.447 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:23:31.447 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:23:31.447 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:23:31.447 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:23:31.447 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:23:31.447 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:23:31.447 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 13:23:31.447 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 13:23:31.447 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:23:31.447 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 13:23:31.447 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 13:23:31.447 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 13:23:31.447 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 13:23:31.448 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 13:23:31.448 | INFO | ------------------------------------------ | 20260505 13:23:31.448 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:23:31.448 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 13:23:31.448 | INFO | Total mem usage: 25.6 GiB | 20260505 13:23:31.448 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 13:23:31.448 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 13:23:31.448 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 13:23:31.448 | INFO | High Water Breakdown: | 20260505 13:23:31.448 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 13:23:31.448 | INFO | ------------------------------------------ | 20260505 13:23:31.448 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:23:31.448 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 13:23:31.448 | INFO | Total mem usage: 15.7 GiB | 20260505 13:23:31.448 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 13:23:31.448 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 13:23:31.448 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 13:23:31.448 | INFO | High Water Breakdown: | 20260505 13:23:31.448 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 13:23:31.448 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 13:23:31.448 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 13:23:31.448 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 13:23:31.448 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 13:23:31.448 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 13:23:31.448 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 13:23:31.448 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 13:23:31.448 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 13:23:31.448 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:23:31.448 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 13:23:31.448 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:23:31.448 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 13:23:31.448 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 13:23:31.448 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 13:23:31.448 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 13:23:31.448 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:23:31.448 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 13:23:31.448 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:23:31.448 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:23:31.448 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:23:31.448 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 13:23:31.448 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 13:23:31.448 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:23:31.448 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 13:23:31.448 | INFO | ------------------------------------------ | 20260505 13:23:31.448 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:23:31.448 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 13:23:31.448 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:23:31.448 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:23:31.448 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.71e+01, 1.39e+01 / 2.25e+00 | 20260505 13:23:31.448 | INFO | PerfReport: 0.29%, AggregateStats -- 9.32e-02 / 1.01e-01, 1.10e-02 / 4.36e-03 | 20260505 13:23:31.448 | INFO | PerfReport: 11.78%, Baseline -- 3.76e+00 / 4.51e+00, 5.96e-01 / 1.44e-01 | 20260505 13:23:31.448 | INFO | PerfReport: 3.29%, Histogram -- 1.05e+00 / 1.50e+00, 2.00e-01 / 7.73e-02 | 20260505 13:23:31.448 | INFO | PerfReport: 16.07%, DME -- 4.10e+01 / 4.66e+01, 5.06e+00 / 2.14e+00 | 20260505 13:23:31.448 | INFO | PerfReport: 56.27%, FrameLabeling -- 1.80e+01 / 1.96e+01, 1.37e+00 / 1.22e+00 | 20260505 13:23:31.448 | INFO | PerfReport: 7.46%, PulseAccumulating -- 2.38e+00 / 2.95e+00, 4.52e-01 / 1.22e-01 | 20260505 13:23:31.448 | INFO | PerfReport: 3.58%, Metrics -- 1.14e+00 / 1.46e+00, 3.21e-01 / 1.70e-01 | 20260505 13:23:31.448 | INFO | PerfReport: 1.26%, PulseSerializing -- 4.01e-01 / 4.53e-01, 7.11e-02 / 2.74e-02 | 20260505 13:23:31.448 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 13:23:31.448 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:23:31.448 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:23:31.448 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260505 13:23:31.448 | INFO | PerfReport: 22.48%, Upload -- 2.27e+01 / 0.00e+00, 8.01e+00 / 0.00e+00 | 20260505 13:23:31.448 | INFO | PerfReport: 22.03%, Download -- 2.22e+01 / 0.00e+00, 8.09e+00 / 0.00e+00 | 20260505 13:23:31.449 | INFO | PerfReport: 31.63%, Compute -- 3.19e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 13:23:31.449 | INFO | PerfReport: 2.17%, LockUpload -- 2.19e+00 / 0.00e+00, 7.92e+00 / 0.00e+00 | 20260505 13:23:31.449 | INFO | PerfReport: 1.48%, LockDownload -- 1.49e+00 / 0.00e+00, 3.01e+00 / 0.00e+00 | 20260505 13:23:31.449 | INFO | PerfReport: 20.19%, LockCompute -- 2.04e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 13:23:31.449 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 2.38e-02 / 0.00e+00 | 20260505 13:23:31.581 | INFO | Starting a round of Baz IO | 20260505 13:23:34.826 | INFO | Populating RTMetrics report | 20260505 13:23:34.827 | INFO | RTMetrics report populated | 20260505 13:23:34.827 | INFO | Opening tmp RTMetrics json file | 20260505 13:23:34.835 | INFO | Serializing RTMetrics to json | 20260505 13:23:34.850 | INFO | Serialization done, swapping tmp file now... | 20260505 13:23:34.851 | INFO | RTMetrics done | 20260505 13:23:34.942 | INFO | Finished a round of Baz IO: Wrote 2.3 GiB in 3.36 seconds | 20260505 13:23:34.942 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.14% / 0.00% / 76.85% | 20260505 13:23:34.942 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.710551% / 1.19209e-05% / 9.53674e-05% | 20260505 13:23:34.942 | INFO | Baz IO overall: | 20260505 13:23:34.942 | INFO | BytesWritten : 2107619540491 | 20260505 13:23:34.942 | INFO | NumEvents : 1002754628051 | 20260505 13:23:36.584 | INFO | Analyzing chunk frames = [3538944,3539456) | 20260505 13:23:36.584 | INFO | MemInfo: { | 20260505 13:23:36.584 | INFO | "gpu" : " 17 GB", | 20260505 13:23:36.584 | INFO | "huge" : "", | 20260505 13:23:36.584 | INFO | "huge_shared" : " 27 GB", | 20260505 13:23:36.584 | INFO | "malloc" : " 14 GB", | 20260505 13:23:36.584 | INFO | "pinned" : " 29 GB" | 20260505 13:23:36.584 | INFO | } | 20260505 13:23:36.584 | INFO | Perf report for the last 5.13722 seconds: | 20260505 13:23:36.584 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:23:36.584 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:36.584 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:36.584 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:36.584 | INFO | ANALYSIS : 64.7%, 3.0, 100.0ms, 0 | 20260505 13:23:36.584 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:36.584 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260505 13:23:36.584 | INFO | PRE_HQ : 60.8%, 1.8, 58.2ms, 0 | 20260505 13:23:36.584 | INFO | BAZWRITER : 64.8%, 4.0, 134.7ms, 0 | 20260505 13:23:36.584 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 13:23:36.584 | INFO | Uploaded 36GiB to GPU in 2.20135s (16.3536GiB/s) | 20260505 13:23:36.584 | INFO | Downloaded 38GiB to GPU in 2.29107s (16.5861GiB/s) | 20260505 13:23:41.728 | INFO | Analyzing chunk frames = [3539456,3539968) | 20260505 13:23:41.728 | INFO | MemInfo: { | 20260505 13:23:41.728 | INFO | "gpu" : " 17 GB", | 20260505 13:23:41.728 | INFO | "huge" : "", | 20260505 13:23:41.728 | INFO | "huge_shared" : " 27 GB", | 20260505 13:23:41.728 | INFO | "malloc" : " 14 GB", | 20260505 13:23:41.728 | INFO | "pinned" : " 29 GB" | 20260505 13:23:41.728 | INFO | } | 20260505 13:23:41.728 | INFO | Perf report for the last 5.14453 seconds: | 20260505 13:23:41.728 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:23:41.728 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:41.728 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:41.728 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:41.728 | INFO | ANALYSIS : 63.9%, 3.0, 99.9ms, 0 | 20260505 13:23:41.728 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:41.728 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:41.728 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 13:23:41.728 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:41.728 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:41.728 | INFO | Uploaded 36GiB to GPU in 2.2287s (16.1529GiB/s) | 20260505 13:23:41.729 | INFO | Downloaded 34GiB to GPU in 2.09987s (16.1915GiB/s) | 20260505 13:23:46.875 | INFO | Analyzing chunk frames = [3539968,3540480) | 20260505 13:23:46.875 | INFO | MemInfo: { | 20260505 13:23:46.875 | INFO | "gpu" : " 17 GB", | 20260505 13:23:46.875 | INFO | "huge" : "", | 20260505 13:23:46.875 | INFO | "huge_shared" : " 27 GB", | 20260505 13:23:46.875 | INFO | "malloc" : " 14 GB", | 20260505 13:23:46.875 | INFO | "pinned" : " 29 GB" | 20260505 13:23:46.875 | INFO | } | 20260505 13:23:46.875 | INFO | Perf report for the last 5.14701 seconds: | 20260505 13:23:46.875 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:23:46.875 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:46.875 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:46.875 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:46.875 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 13:23:46.875 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:46.875 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:46.875 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:23:46.876 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:46.876 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:46.876 | INFO | Uploaded 36GiB to GPU in 2.21876s (16.2253GiB/s) | 20260505 13:23:46.876 | INFO | Downloaded 34GiB to GPU in 2.10464s (16.1548GiB/s) | 20260505 13:23:52.005 | INFO | Analyzing chunk frames = [3540480,3540992) | 20260505 13:23:52.005 | INFO | MemInfo: { | 20260505 13:23:52.005 | INFO | "gpu" : " 17 GB", | 20260505 13:23:52.005 | INFO | "huge" : "", | 20260505 13:23:52.005 | INFO | "huge_shared" : " 27 GB", | 20260505 13:23:52.005 | INFO | "malloc" : " 14 GB", | 20260505 13:23:52.005 | INFO | "pinned" : " 29 GB" | 20260505 13:23:52.005 | INFO | } | 20260505 13:23:52.005 | INFO | Perf report for the last 5.12977 seconds: | 20260505 13:23:52.005 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:23:52.005 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:52.005 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:52.005 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:52.005 | INFO | ANALYSIS : 62.8%, 3.0, 97.4ms, 0 | 20260505 13:23:52.005 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:52.005 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:52.005 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 13:23:52.005 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:52.005 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:52.005 | INFO | Uploaded 36GiB to GPU in 2.165s (16.6282GiB/s) | 20260505 13:23:52.005 | INFO | Downloaded 34GiB to GPU in 2.12924s (15.9681GiB/s) | 20260505 13:23:57.141 | INFO | Analyzing chunk frames = [3540992,3541504) | 20260505 13:23:57.141 | INFO | MemInfo: { | 20260505 13:23:57.141 | INFO | "gpu" : " 17 GB", | 20260505 13:23:57.141 | INFO | "huge" : "", | 20260505 13:23:57.141 | INFO | "huge_shared" : " 27 GB", | 20260505 13:23:57.142 | INFO | "malloc" : " 14 GB", | 20260505 13:23:57.142 | INFO | "pinned" : " 29 GB" | 20260505 13:23:57.142 | INFO | } | 20260505 13:23:57.142 | INFO | Perf report for the last 5.13673 seconds: | 20260505 13:23:57.142 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:23:57.142 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:57.142 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:57.142 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:57.142 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 13:23:57.142 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:57.142 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:57.142 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:23:57.142 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:57.142 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:23:57.142 | INFO | Uploaded 36GiB to GPU in 2.17586s (16.5452GiB/s) | 20260505 13:23:57.142 | INFO | Downloaded 34GiB to GPU in 2.13019s (15.961GiB/s) | 20260505 13:24:02.289 | INFO | Analyzing chunk frames = [3541504,3542016) | 20260505 13:24:02.289 | INFO | MemInfo: { | 20260505 13:24:02.289 | INFO | "gpu" : " 17 GB", | 20260505 13:24:02.289 | INFO | "huge" : "", | 20260505 13:24:02.289 | INFO | "huge_shared" : " 27 GB", | 20260505 13:24:02.289 | INFO | "malloc" : " 14 GB", | 20260505 13:24:02.289 | INFO | "pinned" : " 29 GB" | 20260505 13:24:02.289 | INFO | } | 20260505 13:24:02.289 | INFO | Perf report for the last 5.1469 seconds: | 20260505 13:24:02.289 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:24:02.289 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:02.289 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:02.289 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:02.289 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 13:24:02.289 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:02.289 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:02.289 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 13:24:02.289 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:02.289 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:02.289 | INFO | Uploaded 36GiB to GPU in 2.16753s (16.6088GiB/s) | 20260505 13:24:02.289 | INFO | Downloaded 34GiB to GPU in 2.11318s (16.0895GiB/s) | 20260505 13:24:07.433 | INFO | Analyzing chunk frames = [3542016,3542528) | 20260505 13:24:07.433 | INFO | MemInfo: { | 20260505 13:24:07.433 | INFO | "gpu" : " 17 GB", | 20260505 13:24:07.433 | INFO | "huge" : "", | 20260505 13:24:07.433 | INFO | "huge_shared" : " 27 GB", | 20260505 13:24:07.433 | INFO | "malloc" : " 14 GB", | 20260505 13:24:07.433 | INFO | "pinned" : " 29 GB" | 20260505 13:24:07.433 | INFO | } | 20260505 13:24:07.433 | INFO | Perf report for the last 5.14414 seconds: | 20260505 13:24:07.433 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:24:07.433 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:07.433 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:07.433 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:07.433 | INFO | ANALYSIS : 63.7%, 3.0, 99.6ms, 0 | 20260505 13:24:07.433 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:07.433 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:07.433 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 13:24:07.433 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:07.433 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:07.433 | INFO | Uploaded 36GiB to GPU in 2.15171s (16.7309GiB/s) | 20260505 13:24:07.433 | INFO | Downloaded 34GiB to GPU in 2.17966s (15.5988GiB/s) | 20260505 13:24:12.569 | INFO | Analyzing chunk frames = [3542528,3543040) | 20260505 13:24:12.569 | INFO | MemInfo: { | 20260505 13:24:12.569 | INFO | "gpu" : " 17 GB", | 20260505 13:24:12.569 | INFO | "huge" : "", | 20260505 13:24:12.569 | INFO | "huge_shared" : " 27 GB", | 20260505 13:24:12.569 | INFO | "malloc" : " 14 GB", | 20260505 13:24:12.569 | INFO | "pinned" : " 29 GB" | 20260505 13:24:12.569 | INFO | } | 20260505 13:24:12.569 | INFO | Perf report for the last 5.13584 seconds: | 20260505 13:24:12.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:24:12.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:12.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:12.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:12.569 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 13:24:12.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:12.569 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:12.569 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 13:24:12.569 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:12.569 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:12.569 | INFO | Uploaded 36GiB to GPU in 2.21149s (16.2786GiB/s) | 20260505 13:24:12.569 | INFO | Downloaded 34GiB to GPU in 2.10154s (16.1786GiB/s) | 20260505 13:24:12.703 | INFO | Starting a round of Baz IO | 20260505 13:24:15.936 | INFO | Populating RTMetrics report | 20260505 13:24:15.937 | INFO | RTMetrics report populated | 20260505 13:24:15.937 | INFO | Opening tmp RTMetrics json file | 20260505 13:24:15.938 | INFO | Serializing RTMetrics to json | 20260505 13:24:15.939 | INFO | Serialization done, swapping tmp file now... | 20260505 13:24:15.940 | INFO | RTMetrics done | 20260505 13:24:16.051 | INFO | Finished a round of Baz IO: Wrote 2.301 GiB in 3.347 seconds | 20260505 13:24:16.051 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.14% / 0.00% / 76.86% | 20260505 13:24:16.051 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.710396% / 7.94729e-06% / 9.53674e-05% | 20260505 13:24:16.051 | INFO | Baz IO overall: | 20260505 13:24:16.051 | INFO | BytesWritten : 2110090419100 | 20260505 13:24:16.051 | INFO | NumEvents : 1003923557976 | 20260505 13:24:17.702 | INFO | Analyzing chunk frames = [3543040,3543552) | 20260505 13:24:17.702 | INFO | MemInfo: { | 20260505 13:24:17.702 | INFO | "gpu" : " 17 GB", | 20260505 13:24:17.702 | INFO | "huge" : "", | 20260505 13:24:17.702 | INFO | "huge_shared" : " 27 GB", | 20260505 13:24:17.702 | INFO | "malloc" : " 14 GB", | 20260505 13:24:17.702 | INFO | "pinned" : " 29 GB" | 20260505 13:24:17.702 | INFO | } | 20260505 13:24:17.702 | INFO | Perf report for the last 5.1332 seconds: | 20260505 13:24:17.702 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:24:17.702 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:17.702 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:17.702 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:17.702 | INFO | ANALYSIS : 64.5%, 3.0, 99.9ms, 0 | 20260505 13:24:17.702 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:17.702 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260505 13:24:17.702 | INFO | PRE_HQ : 61.3%, 1.8, 58.1ms, 0 | 20260505 13:24:17.702 | INFO | BAZWRITER : 64.5%, 4.0, 133.6ms, 0 | 20260505 13:24:17.702 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 13:24:17.702 | INFO | Uploaded 36GiB to GPU in 2.19182s (16.4247GiB/s) | 20260505 13:24:17.702 | INFO | Downloaded 38GiB to GPU in 2.27441s (16.7076GiB/s) | 20260505 13:24:22.845 | INFO | Analyzing chunk frames = [3543552,3544064) | 20260505 13:24:22.845 | INFO | MemInfo: { | 20260505 13:24:22.845 | INFO | "gpu" : " 17 GB", | 20260505 13:24:22.845 | INFO | "huge" : "", | 20260505 13:24:22.845 | INFO | "huge_shared" : " 27 GB", | 20260505 13:24:22.845 | INFO | "malloc" : " 14 GB", | 20260505 13:24:22.845 | INFO | "pinned" : " 29 GB" | 20260505 13:24:22.845 | INFO | } | 20260505 13:24:22.845 | INFO | Perf report for the last 5.14268 seconds: | 20260505 13:24:22.845 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:24:22.845 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:22.845 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:22.845 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:22.845 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 13:24:22.845 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:22.845 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:22.845 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 13:24:22.845 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:22.845 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:22.845 | INFO | Uploaded 36GiB to GPU in 2.15581s (16.6991GiB/s) | 20260505 13:24:22.845 | INFO | Downloaded 34GiB to GPU in 2.03253s (16.728GiB/s) | 20260505 13:24:27.988 | INFO | Analyzing chunk frames = [3544064,3544576) | 20260505 13:24:27.988 | INFO | MemInfo: { | 20260505 13:24:27.988 | INFO | "gpu" : " 17 GB", | 20260505 13:24:27.988 | INFO | "huge" : "", | 20260505 13:24:27.988 | INFO | "huge_shared" : " 27 GB", | 20260505 13:24:27.988 | INFO | "malloc" : " 14 GB", | 20260505 13:24:27.988 | INFO | "pinned" : " 29 GB" | 20260505 13:24:27.988 | INFO | } | 20260505 13:24:27.988 | INFO | Perf report for the last 5.14341 seconds: | 20260505 13:24:27.988 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:24:27.988 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:27.988 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:27.988 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:27.988 | INFO | ANALYSIS : 63.2%, 3.0, 98.4ms, 0 | 20260505 13:24:27.988 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:27.988 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:27.988 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 13:24:27.988 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:27.988 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:27.988 | INFO | Uploaded 36GiB to GPU in 2.21724s (16.2364GiB/s) | 20260505 13:24:27.988 | INFO | Downloaded 34GiB to GPU in 2.10308s (16.1668GiB/s) | 20260505 13:24:33.126 | INFO | Analyzing chunk frames = [3544576,3545088) | 20260505 13:24:33.126 | INFO | MemInfo: { | 20260505 13:24:33.126 | INFO | "gpu" : " 17 GB", | 20260505 13:24:33.126 | INFO | "huge" : "", | 20260505 13:24:33.126 | INFO | "huge_shared" : " 27 GB", | 20260505 13:24:33.126 | INFO | "malloc" : " 14 GB", | 20260505 13:24:33.126 | INFO | "pinned" : " 29 GB" | 20260505 13:24:33.126 | INFO | } | 20260505 13:24:33.126 | INFO | Perf report for the last 5.13835 seconds: | 20260505 13:24:33.126 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:24:33.126 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:33.126 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:33.126 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:33.126 | INFO | ANALYSIS : 63.0%, 3.0, 97.8ms, 0 | 20260505 13:24:33.126 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:33.126 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:33.126 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:24:33.126 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:33.127 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:33.127 | INFO | Uploaded 36GiB to GPU in 2.19002s (16.4382GiB/s) | 20260505 13:24:33.127 | INFO | Downloaded 34GiB to GPU in 2.13701s (15.9101GiB/s) | 20260505 13:24:38.263 | INFO | Analyzing chunk frames = [3545088,3545600) | 20260505 13:24:38.263 | INFO | MemInfo: { | 20260505 13:24:38.263 | INFO | "gpu" : " 17 GB", | 20260505 13:24:38.263 | INFO | "huge" : "", | 20260505 13:24:38.263 | INFO | "huge_shared" : " 27 GB", | 20260505 13:24:38.263 | INFO | "malloc" : " 14 GB", | 20260505 13:24:38.263 | INFO | "pinned" : " 29 GB" | 20260505 13:24:38.263 | INFO | } | 20260505 13:24:38.263 | INFO | Perf report for the last 5.13712 seconds: | 20260505 13:24:38.264 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:24:38.264 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:38.264 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:38.264 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:38.264 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 13:24:38.264 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:38.264 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:38.264 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:24:38.264 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:38.264 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:38.264 | INFO | Uploaded 36GiB to GPU in 2.14289s (16.7997GiB/s) | 20260505 13:24:38.264 | INFO | Downloaded 34GiB to GPU in 2.12265s (16.0177GiB/s) | 20260505 13:24:43.393 | INFO | Analyzing chunk frames = [3545600,3546112) | 20260505 13:24:43.393 | INFO | MemInfo: { | 20260505 13:24:43.393 | INFO | "gpu" : " 17 GB", | 20260505 13:24:43.393 | INFO | "huge" : "", | 20260505 13:24:43.393 | INFO | "huge_shared" : " 27 GB", | 20260505 13:24:43.393 | INFO | "malloc" : " 14 GB", | 20260505 13:24:43.393 | INFO | "pinned" : " 29 GB" | 20260505 13:24:43.393 | INFO | } | 20260505 13:24:43.393 | INFO | Perf report for the last 5.12974 seconds: | 20260505 13:24:43.393 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:24:43.393 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:43.393 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:43.393 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:43.393 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 13:24:43.393 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:43.393 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:43.393 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 13:24:43.393 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:43.393 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:43.393 | INFO | Uploaded 36GiB to GPU in 2.1451s (16.7824GiB/s) | 20260505 13:24:43.393 | INFO | Downloaded 34GiB to GPU in 2.10921s (16.1198GiB/s) | 20260505 13:24:48.535 | INFO | Analyzing chunk frames = [3546112,3546624) | 20260505 13:24:48.535 | INFO | MemInfo: { | 20260505 13:24:48.535 | INFO | "gpu" : " 17 GB", | 20260505 13:24:48.535 | INFO | "huge" : "", | 20260505 13:24:48.535 | INFO | "huge_shared" : " 27 GB", | 20260505 13:24:48.535 | INFO | "malloc" : " 14 GB", | 20260505 13:24:48.535 | INFO | "pinned" : " 29 GB" | 20260505 13:24:48.535 | INFO | } | 20260505 13:24:48.535 | INFO | Perf report for the last 5.14152 seconds: | 20260505 13:24:48.535 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:24:48.535 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:48.535 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:48.535 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:48.535 | INFO | ANALYSIS : 63.2%, 3.0, 98.8ms, 0 | 20260505 13:24:48.535 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:48.535 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:48.535 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 13:24:48.535 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:48.535 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:48.535 | INFO | Uploaded 36GiB to GPU in 2.17898s (16.5215GiB/s) | 20260505 13:24:48.535 | INFO | Downloaded 34GiB to GPU in 2.16032s (15.7384GiB/s) | 20260505 13:24:53.679 | INFO | Analyzing chunk frames = [3546624,3547136) | 20260505 13:24:53.679 | INFO | MemInfo: { | 20260505 13:24:53.679 | INFO | "gpu" : " 17 GB", | 20260505 13:24:53.679 | INFO | "huge" : "", | 20260505 13:24:53.679 | INFO | "huge_shared" : " 27 GB", | 20260505 13:24:53.679 | INFO | "malloc" : " 14 GB", | 20260505 13:24:53.679 | INFO | "pinned" : " 29 GB" | 20260505 13:24:53.679 | INFO | } | 20260505 13:24:53.679 | INFO | Perf report for the last 5.14391 seconds: | 20260505 13:24:53.679 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:24:53.679 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:53.679 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:53.679 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:53.679 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 13:24:53.679 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:53.679 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:53.679 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 13:24:53.679 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:53.679 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:53.679 | INFO | Uploaded 36GiB to GPU in 2.17264s (16.5697GiB/s) | 20260505 13:24:53.679 | INFO | Downloaded 34GiB to GPU in 2.10689s (16.1376GiB/s) | 20260505 13:24:53.679 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 13:24:53.679 | INFO | Total mem usage: 15.3 GiB | 20260505 13:24:53.679 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 13:24:53.679 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -7.1%) | 20260505 13:24:53.679 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 13:24:53.679 | INFO | High Water Breakdown: | 20260505 13:24:53.679 | INFO | 61.1% : 8.75 GiB ( 91.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 13:24:53.679 | INFO | 38.7% : 5.54 GiB ( 83.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 13:24:53.679 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 13:24:53.679 | INFO | ------------------------------------------ | 20260505 13:24:53.679 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:24:53.679 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 13:24:53.679 | INFO | Total mem usage: 26.5 GiB | 20260505 13:24:53.679 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 13:24:53.679 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 13:24:53.679 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 13:24:53.679 | INFO | High Water Breakdown: | 20260505 13:24:53.679 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:24:53.679 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:24:53.679 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:24:53.679 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:24:53.679 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:24:53.679 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:24:53.679 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 13:24:53.679 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 13:24:53.679 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:24:53.679 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 13:24:53.679 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 13:24:53.679 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 13:24:53.679 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 13:24:53.679 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 13:24:53.679 | INFO | ------------------------------------------ | 20260505 13:24:53.679 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:24:53.679 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 13:24:53.679 | INFO | Total mem usage: 25.6 GiB | 20260505 13:24:53.679 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 13:24:53.680 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 13:24:53.680 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 13:24:53.680 | INFO | High Water Breakdown: | 20260505 13:24:53.680 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 13:24:53.680 | INFO | ------------------------------------------ | 20260505 13:24:53.680 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:24:53.680 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 13:24:53.680 | INFO | Total mem usage: 15.7 GiB | 20260505 13:24:53.680 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 13:24:53.680 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 13:24:53.680 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 13:24:53.680 | INFO | High Water Breakdown: | 20260505 13:24:53.680 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 13:24:53.680 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 13:24:53.680 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 13:24:53.680 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 13:24:53.680 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 13:24:53.680 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 13:24:53.680 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 13:24:53.680 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 13:24:53.680 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 13:24:53.680 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:24:53.680 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 13:24:53.680 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:24:53.680 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 13:24:53.680 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 13:24:53.680 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 13:24:53.680 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 13:24:53.680 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:24:53.680 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 13:24:53.680 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:24:53.680 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:24:53.680 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:24:53.680 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 13:24:53.680 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 13:24:53.680 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:24:53.680 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 13:24:53.680 | INFO | ------------------------------------------ | 20260505 13:24:53.680 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:24:53.680 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 13:24:53.680 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:24:53.680 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:24:53.680 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.70e+01, 1.39e+01 / 1.80e+00 | 20260505 13:24:53.680 | INFO | PerfReport: 0.29%, AggregateStats -- 9.33e-02 / 9.93e-02, 1.08e-02 / 4.17e-03 | 20260505 13:24:53.680 | INFO | PerfReport: 11.78%, Baseline -- 3.76e+00 / 4.49e+00, 5.87e-01 / 1.74e-01 | 20260505 13:24:53.680 | INFO | PerfReport: 3.30%, Histogram -- 1.05e+00 / 1.51e+00, 2.01e-01 / 9.67e-02 | 20260505 13:24:53.680 | INFO | PerfReport: 16.07%, DME -- 4.10e+01 / 4.64e+01, 4.89e+00 / 1.83e+00 | 20260505 13:24:53.680 | INFO | PerfReport: 56.25%, FrameLabeling -- 1.79e+01 / 1.97e+01, 1.37e+00 / 1.60e+00 | 20260505 13:24:53.680 | INFO | PerfReport: 7.46%, PulseAccumulating -- 2.38e+00 / 2.92e+00, 4.51e-01 / 1.53e-01 | 20260505 13:24:53.680 | INFO | PerfReport: 3.58%, Metrics -- 1.14e+00 / 1.48e+00, 3.22e-01 / 1.88e-01 | 20260505 13:24:53.680 | INFO | PerfReport: 1.26%, PulseSerializing -- 4.01e-01 / 4.52e-01, 7.20e-02 / 3.43e-02 | 20260505 13:24:53.680 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 13:24:53.680 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:24:53.680 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:24:53.680 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260505 13:24:53.680 | INFO | PerfReport: 22.53%, Upload -- 2.27e+01 / 0.00e+00, 8.03e+00 / 0.00e+00 | 20260505 13:24:53.680 | INFO | PerfReport: 22.06%, Download -- 2.23e+01 / 0.00e+00, 8.08e+00 / 0.00e+00 | 20260505 13:24:53.680 | INFO | PerfReport: 31.61%, Compute -- 3.19e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 13:24:53.680 | INFO | PerfReport: 2.21%, LockUpload -- 2.23e+00 / 0.00e+00, 7.95e+00 / 0.00e+00 | 20260505 13:24:53.680 | INFO | PerfReport: 1.49%, LockDownload -- 1.50e+00 / 0.00e+00, 3.01e+00 / 0.00e+00 | 20260505 13:24:53.680 | INFO | PerfReport: 20.09%, LockCompute -- 2.03e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260505 13:24:53.680 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 2.40e-02 / 0.00e+00 | 20260505 13:24:53.813 | INFO | Starting a round of Baz IO | 20260505 13:24:57.047 | INFO | Populating RTMetrics report | 20260505 13:24:57.048 | INFO | RTMetrics report populated | 20260505 13:24:57.048 | INFO | Opening tmp RTMetrics json file | 20260505 13:24:57.066 | INFO | Serializing RTMetrics to json | 20260505 13:24:57.068 | INFO | Serialization done, swapping tmp file now... | 20260505 13:24:57.073 | INFO | RTMetrics done | 20260505 13:24:57.166 | INFO | Finished a round of Baz IO: Wrote 2.303 GiB in 3.353 seconds | 20260505 13:24:57.166 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.13% / 0.00% / 76.86% | 20260505 13:24:57.166 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.710098% / 0% / 7.15256e-05% | 20260505 13:24:57.166 | INFO | Baz IO overall: | 20260505 13:24:57.166 | INFO | BytesWritten : 2112563667407 | 20260505 13:24:57.166 | INFO | NumEvents : 1005094831812 | 20260505 13:24:58.824 | INFO | Analyzing chunk frames = [3547136,3547648) | 20260505 13:24:58.824 | INFO | MemInfo: { | 20260505 13:24:58.824 | INFO | "gpu" : " 17 GB", | 20260505 13:24:58.824 | INFO | "huge" : "", | 20260505 13:24:58.824 | INFO | "huge_shared" : " 27 GB", | 20260505 13:24:58.824 | INFO | "malloc" : " 14 GB", | 20260505 13:24:58.824 | INFO | "pinned" : " 29 GB" | 20260505 13:24:58.824 | INFO | } | 20260505 13:24:58.824 | INFO | Perf report for the last 5.14539 seconds: | 20260505 13:24:58.824 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:24:58.824 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:58.824 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:58.824 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:58.824 | INFO | ANALYSIS : 64.4%, 3.0, 100.0ms, 0 | 20260505 13:24:58.824 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:24:58.824 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260505 13:24:58.824 | INFO | PRE_HQ : 61.0%, 1.8, 58.0ms, 0 | 20260505 13:24:58.824 | INFO | BAZWRITER : 64.4%, 4.0, 133.8ms, 0 | 20260505 13:24:58.824 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 13:24:58.824 | INFO | Uploaded 36GiB to GPU in 2.19104s (16.4305GiB/s) | 20260505 13:24:58.824 | INFO | Downloaded 38GiB to GPU in 2.27486s (16.7043GiB/s) | 20260505 13:25:03.956 | INFO | Analyzing chunk frames = [3547648,3548160) | 20260505 13:25:03.956 | INFO | MemInfo: { | 20260505 13:25:03.956 | INFO | "gpu" : " 17 GB", | 20260505 13:25:03.956 | INFO | "huge" : "", | 20260505 13:25:03.956 | INFO | "huge_shared" : " 27 GB", | 20260505 13:25:03.956 | INFO | "malloc" : " 14 GB", | 20260505 13:25:03.956 | INFO | "pinned" : " 29 GB" | 20260505 13:25:03.956 | INFO | } | 20260505 13:25:03.956 | INFO | Perf report for the last 5.13176 seconds: | 20260505 13:25:03.956 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:25:03.956 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:03.956 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:03.956 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:03.956 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 13:25:03.956 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:03.956 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:03.956 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 13:25:03.956 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:03.956 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:03.956 | INFO | Uploaded 36GiB to GPU in 2.207s (16.3117GiB/s) | 20260505 13:25:03.956 | INFO | Downloaded 34GiB to GPU in 2.0379s (16.6838GiB/s) | 20260505 13:25:09.095 | INFO | Analyzing chunk frames = [3548160,3548672) | 20260505 13:25:09.095 | INFO | MemInfo: { | 20260505 13:25:09.095 | INFO | "gpu" : " 17 GB", | 20260505 13:25:09.095 | INFO | "huge" : "", | 20260505 13:25:09.095 | INFO | "huge_shared" : " 27 GB", | 20260505 13:25:09.095 | INFO | "malloc" : " 14 GB", | 20260505 13:25:09.095 | INFO | "pinned" : " 29 GB" | 20260505 13:25:09.095 | INFO | } | 20260505 13:25:09.095 | INFO | Perf report for the last 5.13953 seconds: | 20260505 13:25:09.095 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:25:09.095 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:09.095 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:09.095 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:09.095 | INFO | ANALYSIS : 63.2%, 3.0, 98.4ms, 0 | 20260505 13:25:09.095 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:09.096 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:09.096 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:25:09.096 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:09.096 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:09.096 | INFO | Uploaded 36GiB to GPU in 2.21731s (16.2359GiB/s) | 20260505 13:25:09.096 | INFO | Downloaded 34GiB to GPU in 2.12224s (16.0208GiB/s) | 20260505 13:25:14.237 | INFO | Analyzing chunk frames = [3548672,3549184) | 20260505 13:25:14.237 | INFO | MemInfo: { | 20260505 13:25:14.237 | INFO | "gpu" : " 17 GB", | 20260505 13:25:14.237 | INFO | "huge" : "", | 20260505 13:25:14.237 | INFO | "huge_shared" : " 27 GB", | 20260505 13:25:14.237 | INFO | "malloc" : " 14 GB", | 20260505 13:25:14.237 | INFO | "pinned" : " 29 GB" | 20260505 13:25:14.237 | INFO | } | 20260505 13:25:14.237 | INFO | Perf report for the last 5.14172 seconds: | 20260505 13:25:14.237 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:25:14.237 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:14.237 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:14.237 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:14.237 | INFO | ANALYSIS : 63.0%, 3.0, 97.9ms, 0 | 20260505 13:25:14.237 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:14.237 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:14.237 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 13:25:14.237 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:14.237 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:14.237 | INFO | Uploaded 36GiB to GPU in 2.20334s (16.3389GiB/s) | 20260505 13:25:14.237 | INFO | Downloaded 34GiB to GPU in 2.13254s (15.9434GiB/s) | 20260505 13:25:19.382 | INFO | Analyzing chunk frames = [3549184,3549696) | 20260505 13:25:19.382 | INFO | MemInfo: { | 20260505 13:25:19.382 | INFO | "gpu" : " 17 GB", | 20260505 13:25:19.382 | INFO | "huge" : "", | 20260505 13:25:19.382 | INFO | "huge_shared" : " 27 GB", | 20260505 13:25:19.382 | INFO | "malloc" : " 14 GB", | 20260505 13:25:19.382 | INFO | "pinned" : " 29 GB" | 20260505 13:25:19.382 | INFO | } | 20260505 13:25:19.382 | INFO | Perf report for the last 5.14515 seconds: | 20260505 13:25:19.382 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:25:19.382 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:19.382 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:19.382 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:19.382 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 13:25:19.382 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:19.382 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:19.383 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:25:19.383 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:19.383 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:19.383 | INFO | Uploaded 36GiB to GPU in 2.17405s (16.5589GiB/s) | 20260505 13:25:19.383 | INFO | Downloaded 34GiB to GPU in 2.12679s (15.9866GiB/s) | 20260505 13:25:24.515 | INFO | Analyzing chunk frames = [3549696,3550208) | 20260505 13:25:24.515 | INFO | MemInfo: { | 20260505 13:25:24.515 | INFO | "gpu" : " 17 GB", | 20260505 13:25:24.515 | INFO | "huge" : "", | 20260505 13:25:24.515 | INFO | "huge_shared" : " 27 GB", | 20260505 13:25:24.515 | INFO | "malloc" : " 14 GB", | 20260505 13:25:24.515 | INFO | "pinned" : " 29 GB" | 20260505 13:25:24.515 | INFO | } | 20260505 13:25:24.516 | INFO | Perf report for the last 5.13329 seconds: | 20260505 13:25:24.516 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:25:24.516 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:24.516 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:24.516 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:24.516 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 13:25:24.516 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:24.516 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:24.516 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 13:25:24.516 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:24.516 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:24.516 | INFO | Uploaded 36GiB to GPU in 2.17082s (16.5836GiB/s) | 20260505 13:25:24.516 | INFO | Downloaded 34GiB to GPU in 2.10564s (16.1471GiB/s) | 20260505 13:25:29.649 | INFO | Analyzing chunk frames = [3550208,3550720) | 20260505 13:25:29.649 | INFO | MemInfo: { | 20260505 13:25:29.649 | INFO | "gpu" : " 17 GB", | 20260505 13:25:29.649 | INFO | "huge" : "", | 20260505 13:25:29.649 | INFO | "huge_shared" : " 27 GB", | 20260505 13:25:29.649 | INFO | "malloc" : " 14 GB", | 20260505 13:25:29.649 | INFO | "pinned" : " 29 GB" | 20260505 13:25:29.649 | INFO | } | 20260505 13:25:29.649 | INFO | Perf report for the last 5.13362 seconds: | 20260505 13:25:29.649 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:25:29.649 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:29.649 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:29.649 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:29.649 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 13:25:29.649 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:29.649 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:29.649 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 13:25:29.649 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:29.649 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:29.650 | INFO | Uploaded 36GiB to GPU in 2.15382s (16.7145GiB/s) | 20260505 13:25:29.650 | INFO | Downloaded 34GiB to GPU in 2.13994s (15.8883GiB/s) | 20260505 13:25:34.801 | INFO | Analyzing chunk frames = [3550720,3551232) | 20260505 13:25:34.801 | INFO | MemInfo: { | 20260505 13:25:34.801 | INFO | "gpu" : " 17 GB", | 20260505 13:25:34.801 | INFO | "huge" : "", | 20260505 13:25:34.801 | INFO | "huge_shared" : " 27 GB", | 20260505 13:25:34.801 | INFO | "malloc" : " 14 GB", | 20260505 13:25:34.801 | INFO | "pinned" : " 29 GB" | 20260505 13:25:34.802 | INFO | } | 20260505 13:25:34.802 | INFO | Perf report for the last 5.15228 seconds: | 20260505 13:25:34.802 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:25:34.802 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:34.802 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:34.802 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:34.802 | INFO | ANALYSIS : 63.5%, 3.0, 99.3ms, 0 | 20260505 13:25:34.802 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:34.802 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:34.802 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 13:25:34.802 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:34.802 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:34.802 | INFO | Uploaded 36GiB to GPU in 2.14543s (16.7799GiB/s) | 20260505 13:25:34.802 | INFO | Downloaded 34GiB to GPU in 2.10257s (16.1707GiB/s) | 20260505 13:25:34.938 | INFO | Starting a round of Baz IO | 20260505 13:25:38.158 | INFO | Populating RTMetrics report | 20260505 13:25:38.158 | INFO | RTMetrics report populated | 20260505 13:25:38.158 | INFO | Opening tmp RTMetrics json file | 20260505 13:25:38.202 | INFO | Serializing RTMetrics to json | 20260505 13:25:38.223 | INFO | Serialization done, swapping tmp file now... | 20260505 13:25:38.229 | INFO | RTMetrics done | 20260505 13:25:38.260 | INFO | Finished a round of Baz IO: Wrote 2.299 GiB in 3.322 seconds | 20260505 13:25:38.261 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.13% / 0.00% / 76.87% | 20260505 13:25:38.261 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.705568% / 3.97364e-06% / 8.34465e-05% | 20260505 13:25:38.261 | INFO | Baz IO overall: | 20260505 13:25:38.261 | INFO | BytesWritten : 2115032254309 | 20260505 13:25:38.261 | INFO | NumEvents : 1006262678302 | 20260505 13:25:39.942 | INFO | Analyzing chunk frames = [3551232,3551744) | 20260505 13:25:39.942 | INFO | MemInfo: { | 20260505 13:25:39.942 | INFO | "gpu" : " 17 GB", | 20260505 13:25:39.942 | INFO | "huge" : "", | 20260505 13:25:39.942 | INFO | "huge_shared" : " 27 GB", | 20260505 13:25:39.942 | INFO | "malloc" : " 14 GB", | 20260505 13:25:39.942 | INFO | "pinned" : " 29 GB" | 20260505 13:25:39.942 | INFO | } | 20260505 13:25:39.942 | INFO | Perf report for the last 5.14009 seconds: | 20260505 13:25:39.942 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:25:39.942 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:39.942 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:39.942 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:39.942 | INFO | ANALYSIS : 64.2%, 3.0, 99.6ms, 0 | 20260505 13:25:39.942 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:39.942 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260505 13:25:39.942 | INFO | PRE_HQ : 60.9%, 1.8, 58.1ms, 0 | 20260505 13:25:39.942 | INFO | BAZWRITER : 64.0%, 4.0, 132.5ms, 0 | 20260505 13:25:39.942 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 13:25:39.942 | INFO | Uploaded 36GiB to GPU in 2.20465s (16.3291GiB/s) | 20260505 13:25:39.942 | INFO | Downloaded 38GiB to GPU in 2.25703s (16.8363GiB/s) | 20260505 13:25:45.072 | INFO | Analyzing chunk frames = [3551744,3552256) | 20260505 13:25:45.072 | INFO | MemInfo: { | 20260505 13:25:45.072 | INFO | "gpu" : " 17 GB", | 20260505 13:25:45.072 | INFO | "huge" : "", | 20260505 13:25:45.072 | INFO | "huge_shared" : " 27 GB", | 20260505 13:25:45.072 | INFO | "malloc" : " 14 GB", | 20260505 13:25:45.072 | INFO | "pinned" : " 29 GB" | 20260505 13:25:45.072 | INFO | } | 20260505 13:25:45.072 | INFO | Perf report for the last 5.13068 seconds: | 20260505 13:25:45.073 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:25:45.073 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:45.073 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:45.073 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:45.073 | INFO | ANALYSIS : 63.7%, 3.0, 99.2ms, 0 | 20260505 13:25:45.073 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:45.073 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:45.073 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:25:45.073 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:45.073 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:45.073 | INFO | Uploaded 36GiB to GPU in 2.20257s (16.3445GiB/s) | 20260505 13:25:45.073 | INFO | Downloaded 34GiB to GPU in 2.03805s (16.6826GiB/s) | 20260505 13:25:50.208 | INFO | Analyzing chunk frames = [3552256,3552768) | 20260505 13:25:50.208 | INFO | MemInfo: { | 20260505 13:25:50.208 | INFO | "gpu" : " 17 GB", | 20260505 13:25:50.208 | INFO | "huge" : "", | 20260505 13:25:50.208 | INFO | "huge_shared" : " 27 GB", | 20260505 13:25:50.208 | INFO | "malloc" : " 14 GB", | 20260505 13:25:50.208 | INFO | "pinned" : " 29 GB" | 20260505 13:25:50.208 | INFO | } | 20260505 13:25:50.208 | INFO | Perf report for the last 5.13596 seconds: | 20260505 13:25:50.208 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:25:50.208 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:50.208 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:50.208 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:50.208 | INFO | ANALYSIS : 63.1%, 3.0, 98.2ms, 0 | 20260505 13:25:50.208 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:50.208 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:50.208 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 13:25:50.208 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:50.208 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:50.209 | INFO | Uploaded 36GiB to GPU in 2.21564s (16.2482GiB/s) | 20260505 13:25:50.209 | INFO | Downloaded 34GiB to GPU in 2.12068s (16.0326GiB/s) | 20260505 13:25:55.349 | INFO | Analyzing chunk frames = [3552768,3553280) | 20260505 13:25:55.349 | INFO | MemInfo: { | 20260505 13:25:55.349 | INFO | "gpu" : " 17 GB", | 20260505 13:25:55.349 | INFO | "huge" : "", | 20260505 13:25:55.349 | INFO | "huge_shared" : " 27 GB", | 20260505 13:25:55.349 | INFO | "malloc" : " 14 GB", | 20260505 13:25:55.349 | INFO | "pinned" : " 29 GB" | 20260505 13:25:55.349 | INFO | } | 20260505 13:25:55.350 | INFO | Perf report for the last 5.1413 seconds: | 20260505 13:25:55.350 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:25:55.350 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:55.350 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:55.350 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:55.350 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 13:25:55.350 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:55.350 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:55.350 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 13:25:55.350 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:55.350 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:25:55.350 | INFO | Uploaded 36GiB to GPU in 2.21637s (16.2428GiB/s) | 20260505 13:25:55.350 | INFO | Downloaded 34GiB to GPU in 2.13304s (15.9397GiB/s) | 20260505 13:26:00.491 | INFO | Analyzing chunk frames = [3553280,3553792) | 20260505 13:26:00.491 | INFO | MemInfo: { | 20260505 13:26:00.491 | INFO | "gpu" : " 17 GB", | 20260505 13:26:00.491 | INFO | "huge" : "", | 20260505 13:26:00.491 | INFO | "huge_shared" : " 27 GB", | 20260505 13:26:00.491 | INFO | "malloc" : " 14 GB", | 20260505 13:26:00.491 | INFO | "pinned" : " 29 GB" | 20260505 13:26:00.491 | INFO | } | 20260505 13:26:00.491 | INFO | Perf report for the last 5.14166 seconds: | 20260505 13:26:00.491 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:26:00.491 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:00.491 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:00.491 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:00.492 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 13:26:00.492 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:00.492 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:00.492 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 13:26:00.492 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:00.492 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:00.492 | INFO | Uploaded 36GiB to GPU in 2.15659s (16.693GiB/s) | 20260505 13:26:00.492 | INFO | Downloaded 34GiB to GPU in 2.12528s (15.9979GiB/s) | 20260505 13:26:05.629 | INFO | Analyzing chunk frames = [3553792,3554304) | 20260505 13:26:05.629 | INFO | MemInfo: { | 20260505 13:26:05.629 | INFO | "gpu" : " 17 GB", | 20260505 13:26:05.629 | INFO | "huge" : "", | 20260505 13:26:05.629 | INFO | "huge_shared" : " 27 GB", | 20260505 13:26:05.629 | INFO | "malloc" : " 14 GB", | 20260505 13:26:05.629 | INFO | "pinned" : " 29 GB" | 20260505 13:26:05.629 | INFO | } | 20260505 13:26:05.629 | INFO | Perf report for the last 5.13787 seconds: | 20260505 13:26:05.629 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:26:05.629 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:05.629 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:05.629 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:05.629 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 13:26:05.629 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:05.629 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:05.629 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 13:26:05.629 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:05.629 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:05.629 | INFO | Uploaded 36GiB to GPU in 2.14633s (16.7728GiB/s) | 20260505 13:26:05.629 | INFO | Downloaded 34GiB to GPU in 2.1049s (16.1528GiB/s) | 20260505 13:26:10.768 | INFO | Analyzing chunk frames = [3554304,3554816) | 20260505 13:26:10.768 | INFO | MemInfo: { | 20260505 13:26:10.768 | INFO | "gpu" : " 17 GB", | 20260505 13:26:10.768 | INFO | "huge" : "", | 20260505 13:26:10.768 | INFO | "huge_shared" : " 27 GB", | 20260505 13:26:10.768 | INFO | "malloc" : " 14 GB", | 20260505 13:26:10.768 | INFO | "pinned" : " 29 GB" | 20260505 13:26:10.769 | INFO | } | 20260505 13:26:10.769 | INFO | Perf report for the last 5.13954 seconds: | 20260505 13:26:10.769 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:26:10.769 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:10.769 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:10.769 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:10.769 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 13:26:10.769 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:10.769 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:10.769 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 13:26:10.769 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:10.769 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:10.769 | INFO | Uploaded 36GiB to GPU in 2.15678s (16.6915GiB/s) | 20260505 13:26:10.769 | INFO | Downloaded 34GiB to GPU in 2.17625s (15.6232GiB/s) | 20260505 13:26:15.902 | INFO | Analyzing chunk frames = [3554816,3555328) | 20260505 13:26:15.902 | INFO | MemInfo: { | 20260505 13:26:15.902 | INFO | "gpu" : " 17 GB", | 20260505 13:26:15.902 | INFO | "huge" : "", | 20260505 13:26:15.902 | INFO | "huge_shared" : " 27 GB", | 20260505 13:26:15.902 | INFO | "malloc" : " 14 GB", | 20260505 13:26:15.902 | INFO | "pinned" : " 29 GB" | 20260505 13:26:15.902 | INFO | } | 20260505 13:26:15.902 | INFO | Perf report for the last 5.1333 seconds: | 20260505 13:26:15.902 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:26:15.902 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:15.902 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:15.902 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:15.902 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 13:26:15.902 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:15.902 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:15.902 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 13:26:15.902 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:15.902 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:15.902 | INFO | Uploaded 36GiB to GPU in 2.20265s (16.3439GiB/s) | 20260505 13:26:15.902 | INFO | Downloaded 34GiB to GPU in 2.10736s (16.134GiB/s) | 20260505 13:26:15.903 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 13:26:15.903 | INFO | Total mem usage: 15.3 GiB | 20260505 13:26:15.903 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 13:26:15.903 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.64%) | 20260505 13:26:15.903 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 13:26:15.903 | INFO | High Water Breakdown: | 20260505 13:26:15.903 | INFO | 60.9% : 8.68 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 13:26:15.903 | INFO | 38.9% : 5.54 GiB ( 83.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 13:26:15.903 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 13:26:15.903 | INFO | ------------------------------------------ | 20260505 13:26:15.903 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:26:15.903 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 13:26:15.903 | INFO | Total mem usage: 26.5 GiB | 20260505 13:26:15.903 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 13:26:15.903 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 13:26:15.903 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 13:26:15.903 | INFO | High Water Breakdown: | 20260505 13:26:15.903 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:26:15.903 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:26:15.903 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:26:15.903 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:26:15.903 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:26:15.903 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:26:15.903 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 13:26:15.903 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 13:26:15.903 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:26:15.903 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 13:26:15.903 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 13:26:15.903 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 13:26:15.903 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 13:26:15.903 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 13:26:15.903 | INFO | ------------------------------------------ | 20260505 13:26:15.903 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:26:15.903 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 13:26:15.903 | INFO | Total mem usage: 25.6 GiB | 20260505 13:26:15.903 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 13:26:15.903 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 13:26:15.903 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 13:26:15.903 | INFO | High Water Breakdown: | 20260505 13:26:15.903 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 13:26:15.903 | INFO | ------------------------------------------ | 20260505 13:26:15.903 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:26:15.903 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 13:26:15.903 | INFO | Total mem usage: 15.7 GiB | 20260505 13:26:15.903 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 13:26:15.903 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 13:26:15.903 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 13:26:15.903 | INFO | High Water Breakdown: | 20260505 13:26:15.903 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 13:26:15.903 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 13:26:15.903 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 13:26:15.903 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 13:26:15.903 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 13:26:15.903 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 13:26:15.903 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 13:26:15.903 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 13:26:15.903 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 13:26:15.903 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:26:15.903 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 13:26:15.903 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:26:15.903 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 13:26:15.903 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 13:26:15.903 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 13:26:15.903 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 13:26:15.903 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:26:15.903 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 13:26:15.903 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:26:15.903 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:26:15.903 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:26:15.903 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 13:26:15.904 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 13:26:15.904 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:26:15.904 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 13:26:15.904 | INFO | ------------------------------------------ | 20260505 13:26:15.904 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:26:15.904 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 13:26:15.904 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:26:15.904 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:26:15.904 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.66e+01, 1.36e+01 / 1.72e+00 | 20260505 13:26:15.904 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 1.00e-01, 1.06e-02 / 5.13e-03 | 20260505 13:26:15.904 | INFO | PerfReport: 11.78%, Baseline -- 3.76e+00 / 4.49e+00, 5.90e-01 / 2.11e-01 | 20260505 13:26:15.904 | INFO | PerfReport: 3.30%, Histogram -- 1.05e+00 / 1.51e+00, 2.01e-01 / 9.71e-02 | 20260505 13:26:15.904 | INFO | PerfReport: 15.91%, DME -- 4.06e+01 / 4.54e+01, 4.38e+00 / 1.95e+00 | 20260505 13:26:15.904 | INFO | PerfReport: 56.46%, FrameLabeling -- 1.80e+01 / 2.04e+01, 1.47e+00 / 1.96e+00 | 20260505 13:26:15.904 | INFO | PerfReport: 7.43%, PulseAccumulating -- 2.37e+00 / 2.88e+00, 4.41e-01 / 2.05e-01 | 20260505 13:26:15.904 | INFO | PerfReport: 3.58%, Metrics -- 1.14e+00 / 1.45e+00, 3.19e-01 / 1.78e-01 | 20260505 13:26:15.904 | INFO | PerfReport: 1.25%, PulseSerializing -- 4.00e-01 / 4.50e-01, 7.12e-02 / 3.40e-02 | 20260505 13:26:15.904 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 13:26:15.904 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:26:15.904 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:26:15.904 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260505 13:26:15.904 | INFO | PerfReport: 22.57%, Upload -- 2.28e+01 / 0.00e+00, 8.03e+00 / 0.00e+00 | 20260505 13:26:15.904 | INFO | PerfReport: 22.02%, Download -- 2.22e+01 / 0.00e+00, 8.11e+00 / 0.00e+00 | 20260505 13:26:15.904 | INFO | PerfReport: 31.65%, Compute -- 3.19e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 13:26:15.904 | INFO | PerfReport: 2.16%, LockUpload -- 2.18e+00 / 0.00e+00, 7.87e+00 / 0.00e+00 | 20260505 13:26:15.904 | INFO | PerfReport: 1.38%, LockDownload -- 1.40e+00 / 0.00e+00, 2.94e+00 / 0.00e+00 | 20260505 13:26:15.904 | INFO | PerfReport: 20.20%, LockCompute -- 2.04e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260505 13:26:15.904 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.29e-02 / 0.00e+00 | 20260505 13:26:16.034 | INFO | Starting a round of Baz IO | 20260505 13:26:19.275 | INFO | Populating RTMetrics report | 20260505 13:26:19.275 | INFO | RTMetrics report populated | 20260505 13:26:19.275 | INFO | Opening tmp RTMetrics json file | 20260505 13:26:19.301 | INFO | Serializing RTMetrics to json | 20260505 13:26:19.324 | INFO | Serialization done, swapping tmp file now... | 20260505 13:26:19.324 | INFO | RTMetrics done | 20260505 13:26:19.397 | INFO | Finished a round of Baz IO: Wrote 2.298 GiB in 3.363 seconds | 20260505 13:26:19.397 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.12% / 0.00% / 76.87% | 20260505 13:26:19.397 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.704475% / 1.19209e-05% / 7.15256e-05% | 20260505 13:26:19.397 | INFO | Baz IO overall: | 20260505 13:26:19.397 | INFO | BytesWritten : 2117499395035 | 20260505 13:26:19.397 | INFO | NumEvents : 1007429657899 | 20260505 13:26:21.054 | INFO | Analyzing chunk frames = [3555328,3555840) | 20260505 13:26:21.054 | INFO | MemInfo: { | 20260505 13:26:21.054 | INFO | "gpu" : " 17 GB", | 20260505 13:26:21.054 | INFO | "huge" : "", | 20260505 13:26:21.054 | INFO | "huge_shared" : " 27 GB", | 20260505 13:26:21.054 | INFO | "malloc" : " 14 GB", | 20260505 13:26:21.054 | INFO | "pinned" : " 29 GB" | 20260505 13:26:21.054 | INFO | } | 20260505 13:26:21.054 | INFO | Perf report for the last 5.15169 seconds: | 20260505 13:26:21.054 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:26:21.054 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:21.054 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:21.054 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:21.054 | INFO | ANALYSIS : 64.4%, 3.0, 99.9ms, 0 | 20260505 13:26:21.054 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:21.054 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260505 13:26:21.054 | INFO | PRE_HQ : 60.6%, 1.8, 57.8ms, 0 | 20260505 13:26:21.054 | INFO | BAZWRITER : 64.6%, 4.0, 132.8ms, 0 | 20260505 13:26:21.054 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 13:26:21.054 | INFO | Uploaded 36GiB to GPU in 2.19052s (16.4345GiB/s) | 20260505 13:26:21.054 | INFO | Downloaded 38GiB to GPU in 2.28507s (16.6297GiB/s) | 20260505 13:26:26.188 | INFO | Analyzing chunk frames = [3555840,3556352) | 20260505 13:26:26.188 | INFO | MemInfo: { | 20260505 13:26:26.188 | INFO | "gpu" : " 17 GB", | 20260505 13:26:26.188 | INFO | "huge" : "", | 20260505 13:26:26.188 | INFO | "huge_shared" : " 27 GB", | 20260505 13:26:26.188 | INFO | "malloc" : " 14 GB", | 20260505 13:26:26.188 | INFO | "pinned" : " 29 GB" | 20260505 13:26:26.188 | INFO | } | 20260505 13:26:26.188 | INFO | Perf report for the last 5.13405 seconds: | 20260505 13:26:26.188 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:26:26.188 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:26.188 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:26.188 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:26.188 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 13:26:26.188 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:26.188 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:26.188 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 13:26:26.188 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:26.188 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:26.188 | INFO | Uploaded 36GiB to GPU in 2.20422s (16.3323GiB/s) | 20260505 13:26:26.188 | INFO | Downloaded 34GiB to GPU in 2.04168s (16.653GiB/s) | 20260505 13:26:31.325 | INFO | Analyzing chunk frames = [3556352,3556864) | 20260505 13:26:31.325 | INFO | MemInfo: { | 20260505 13:26:31.325 | INFO | "gpu" : " 17 GB", | 20260505 13:26:31.325 | INFO | "huge" : "", | 20260505 13:26:31.326 | INFO | "huge_shared" : " 27 GB", | 20260505 13:26:31.326 | INFO | "malloc" : " 14 GB", | 20260505 13:26:31.326 | INFO | "pinned" : " 29 GB" | 20260505 13:26:31.326 | INFO | } | 20260505 13:26:31.326 | INFO | Perf report for the last 5.13794 seconds: | 20260505 13:26:31.326 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:26:31.326 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:31.326 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:31.326 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:31.326 | INFO | ANALYSIS : 63.2%, 3.0, 98.4ms, 0 | 20260505 13:26:31.326 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:31.326 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:31.326 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 13:26:31.326 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:31.326 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:31.326 | INFO | Uploaded 36GiB to GPU in 2.17952s (16.5174GiB/s) | 20260505 13:26:31.326 | INFO | Downloaded 34GiB to GPU in 2.12814s (15.9764GiB/s) | 20260505 13:26:36.465 | INFO | Analyzing chunk frames = [3556864,3557376) | 20260505 13:26:36.465 | INFO | MemInfo: { | 20260505 13:26:36.465 | INFO | "gpu" : " 17 GB", | 20260505 13:26:36.465 | INFO | "huge" : "", | 20260505 13:26:36.465 | INFO | "huge_shared" : " 27 GB", | 20260505 13:26:36.465 | INFO | "malloc" : " 14 GB", | 20260505 13:26:36.465 | INFO | "pinned" : " 29 GB" | 20260505 13:26:36.465 | INFO | } | 20260505 13:26:36.465 | INFO | Perf report for the last 5.13955 seconds: | 20260505 13:26:36.465 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:26:36.465 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:36.465 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:36.465 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:36.465 | INFO | ANALYSIS : 63.2%, 3.0, 98.1ms, 0 | 20260505 13:26:36.465 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:36.465 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:36.465 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 13:26:36.465 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:36.466 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:36.466 | INFO | Uploaded 36GiB to GPU in 2.21033s (16.2871GiB/s) | 20260505 13:26:36.466 | INFO | Downloaded 34GiB to GPU in 2.13431s (15.9302GiB/s) | 20260505 13:26:41.605 | INFO | Analyzing chunk frames = [3557376,3557888) | 20260505 13:26:41.605 | INFO | MemInfo: { | 20260505 13:26:41.605 | INFO | "gpu" : " 17 GB", | 20260505 13:26:41.605 | INFO | "huge" : "", | 20260505 13:26:41.605 | INFO | "huge_shared" : " 27 GB", | 20260505 13:26:41.605 | INFO | "malloc" : " 14 GB", | 20260505 13:26:41.605 | INFO | "pinned" : " 29 GB" | 20260505 13:26:41.605 | INFO | } | 20260505 13:26:41.605 | INFO | Perf report for the last 5.13992 seconds: | 20260505 13:26:41.605 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:26:41.605 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:41.605 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:41.605 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:41.605 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 13:26:41.605 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:41.605 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:41.605 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 13:26:41.605 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:41.605 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:41.606 | INFO | Uploaded 36GiB to GPU in 2.17202s (16.5745GiB/s) | 20260505 13:26:41.606 | INFO | Downloaded 34GiB to GPU in 2.12508s (15.9994GiB/s) | 20260505 13:26:46.750 | INFO | Analyzing chunk frames = [3557888,3558400) | 20260505 13:26:46.750 | INFO | MemInfo: { | 20260505 13:26:46.750 | INFO | "gpu" : " 17 GB", | 20260505 13:26:46.750 | INFO | "huge" : "", | 20260505 13:26:46.750 | INFO | "huge_shared" : " 27 GB", | 20260505 13:26:46.750 | INFO | "malloc" : " 14 GB", | 20260505 13:26:46.750 | INFO | "pinned" : " 29 GB" | 20260505 13:26:46.750 | INFO | } | 20260505 13:26:46.750 | INFO | Perf report for the last 5.14458 seconds: | 20260505 13:26:46.750 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:26:46.750 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:46.750 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:46.750 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:46.750 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 13:26:46.750 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:46.750 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:46.750 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 13:26:46.750 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:46.751 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:46.751 | INFO | Uploaded 36GiB to GPU in 2.17373s (16.5614GiB/s) | 20260505 13:26:46.751 | INFO | Downloaded 34GiB to GPU in 2.10792s (16.1296GiB/s) | 20260505 13:26:51.892 | INFO | Analyzing chunk frames = [3558400,3558912) | 20260505 13:26:51.892 | INFO | MemInfo: { | 20260505 13:26:51.892 | INFO | "gpu" : " 17 GB", | 20260505 13:26:51.892 | INFO | "huge" : "", | 20260505 13:26:51.892 | INFO | "huge_shared" : " 27 GB", | 20260505 13:26:51.892 | INFO | "malloc" : " 14 GB", | 20260505 13:26:51.892 | INFO | "pinned" : " 29 GB" | 20260505 13:26:51.892 | INFO | } | 20260505 13:26:51.892 | INFO | Perf report for the last 5.14209 seconds: | 20260505 13:26:51.892 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:26:51.892 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:51.892 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:51.892 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:51.892 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 13:26:51.892 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:51.892 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:51.892 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 13:26:51.892 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:51.892 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:51.892 | INFO | Uploaded 36GiB to GPU in 2.15549s (16.7015GiB/s) | 20260505 13:26:51.892 | INFO | Downloaded 34GiB to GPU in 2.17991s (15.597GiB/s) | 20260505 13:26:57.022 | INFO | Analyzing chunk frames = [3558912,3559424) | 20260505 13:26:57.022 | INFO | MemInfo: { | 20260505 13:26:57.022 | INFO | "gpu" : " 17 GB", | 20260505 13:26:57.022 | INFO | "huge" : "", | 20260505 13:26:57.022 | INFO | "huge_shared" : " 27 GB", | 20260505 13:26:57.022 | INFO | "malloc" : " 14 GB", | 20260505 13:26:57.022 | INFO | "pinned" : " 29 GB" | 20260505 13:26:57.023 | INFO | } | 20260505 13:26:57.023 | INFO | Perf report for the last 5.1306 seconds: | 20260505 13:26:57.023 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:26:57.023 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:57.023 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:57.023 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:57.023 | INFO | ANALYSIS : 63.8%, 3.0, 99.4ms, 0 | 20260505 13:26:57.023 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:57.023 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:57.023 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 13:26:57.023 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:57.023 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:26:57.023 | INFO | Uploaded 36GiB to GPU in 2.19889s (16.3719GiB/s) | 20260505 13:26:57.023 | INFO | Downloaded 34GiB to GPU in 2.10163s (16.1779GiB/s) | 20260505 13:26:57.152 | INFO | Starting a round of Baz IO | 20260505 13:27:00.399 | INFO | Populating RTMetrics report | 20260505 13:27:00.399 | INFO | RTMetrics report populated | 20260505 13:27:00.399 | INFO | Opening tmp RTMetrics json file | 20260505 13:27:00.428 | INFO | Serializing RTMetrics to json | 20260505 13:27:00.442 | INFO | Serialization done, swapping tmp file now... | 20260505 13:27:00.464 | INFO | RTMetrics done | 20260505 13:27:00.503 | INFO | Finished a round of Baz IO: Wrote 2.298 GiB in 3.35 seconds | 20260505 13:27:00.503 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.12% / 0.00% / 76.88% | 20260505 13:27:00.503 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.701054% / 7.94729e-06% / 7.54992e-05% | 20260505 13:27:00.503 | INFO | Baz IO overall: | 20260505 13:27:00.503 | INFO | BytesWritten : 2119966471193 | 20260505 13:27:00.503 | INFO | NumEvents : 1008596990156 | 20260505 13:27:02.159 | INFO | Analyzing chunk frames = [3559424,3559936) | 20260505 13:27:02.159 | INFO | MemInfo: { | 20260505 13:27:02.159 | INFO | "gpu" : " 17 GB", | 20260505 13:27:02.159 | INFO | "huge" : "", | 20260505 13:27:02.159 | INFO | "huge_shared" : " 27 GB", | 20260505 13:27:02.159 | INFO | "malloc" : " 14 GB", | 20260505 13:27:02.159 | INFO | "pinned" : " 29 GB" | 20260505 13:27:02.159 | INFO | } | 20260505 13:27:02.159 | INFO | Perf report for the last 5.13643 seconds: | 20260505 13:27:02.159 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:27:02.159 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:02.159 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:02.159 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:02.159 | INFO | ANALYSIS : 64.6%, 3.0, 100.1ms, 0 | 20260505 13:27:02.159 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:02.159 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260505 13:27:02.159 | INFO | PRE_HQ : 61.0%, 1.8, 58.0ms, 0 | 20260505 13:27:02.159 | INFO | BAZWRITER : 64.5%, 4.0, 132.5ms, 0 | 20260505 13:27:02.159 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 13:27:02.159 | INFO | Uploaded 36GiB to GPU in 2.19716s (16.3848GiB/s) | 20260505 13:27:02.159 | INFO | Downloaded 38GiB to GPU in 2.28505s (16.6298GiB/s) | 20260505 13:27:07.305 | INFO | Analyzing chunk frames = [3559936,3560448) | 20260505 13:27:07.305 | INFO | MemInfo: { | 20260505 13:27:07.305 | INFO | "gpu" : " 17 GB", | 20260505 13:27:07.305 | INFO | "huge" : "", | 20260505 13:27:07.305 | INFO | "huge_shared" : " 27 GB", | 20260505 13:27:07.305 | INFO | "malloc" : " 14 GB", | 20260505 13:27:07.305 | INFO | "pinned" : " 29 GB" | 20260505 13:27:07.305 | INFO | } | 20260505 13:27:07.305 | INFO | Perf report for the last 5.14622 seconds: | 20260505 13:27:07.305 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:27:07.305 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:07.305 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:07.305 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:07.305 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 13:27:07.305 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:07.305 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:07.305 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:27:07.305 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:07.305 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:07.306 | INFO | Uploaded 36GiB to GPU in 2.22898s (16.1509GiB/s) | 20260505 13:27:07.306 | INFO | Downloaded 34GiB to GPU in 2.10494s (16.1525GiB/s) | 20260505 13:27:12.449 | INFO | Analyzing chunk frames = [3560448,3560960) | 20260505 13:27:12.449 | INFO | MemInfo: { | 20260505 13:27:12.449 | INFO | "gpu" : " 17 GB", | 20260505 13:27:12.449 | INFO | "huge" : "", | 20260505 13:27:12.449 | INFO | "huge_shared" : " 27 GB", | 20260505 13:27:12.449 | INFO | "malloc" : " 14 GB", | 20260505 13:27:12.449 | INFO | "pinned" : " 29 GB" | 20260505 13:27:12.449 | INFO | } | 20260505 13:27:12.449 | INFO | Perf report for the last 5.14362 seconds: | 20260505 13:27:12.449 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:27:12.449 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:12.449 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:12.449 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:12.449 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 13:27:12.449 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:12.449 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:12.449 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 13:27:12.449 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:12.449 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:12.449 | INFO | Uploaded 36GiB to GPU in 2.22073s (16.2109GiB/s) | 20260505 13:27:12.449 | INFO | Downloaded 34GiB to GPU in 2.09225s (16.2504GiB/s) | 20260505 13:27:17.583 | INFO | Analyzing chunk frames = [3560960,3561472) | 20260505 13:27:17.583 | INFO | MemInfo: { | 20260505 13:27:17.583 | INFO | "gpu" : " 17 GB", | 20260505 13:27:17.583 | INFO | "huge" : "", | 20260505 13:27:17.583 | INFO | "huge_shared" : " 27 GB", | 20260505 13:27:17.583 | INFO | "malloc" : " 14 GB", | 20260505 13:27:17.583 | INFO | "pinned" : " 29 GB" | 20260505 13:27:17.583 | INFO | } | 20260505 13:27:17.583 | INFO | Perf report for the last 5.13395 seconds: | 20260505 13:27:17.583 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:27:17.583 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:17.583 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:17.583 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:17.583 | INFO | ANALYSIS : 62.8%, 3.0, 97.4ms, 0 | 20260505 13:27:17.583 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:17.583 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:17.583 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 13:27:17.583 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:17.583 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:17.583 | INFO | Uploaded 36GiB to GPU in 2.17914s (16.5203GiB/s) | 20260505 13:27:17.583 | INFO | Downloaded 34GiB to GPU in 2.12935s (15.9673GiB/s) | 20260505 13:27:22.718 | INFO | Analyzing chunk frames = [3561472,3561984) | 20260505 13:27:22.718 | INFO | MemInfo: { | 20260505 13:27:22.718 | INFO | "gpu" : " 17 GB", | 20260505 13:27:22.718 | INFO | "huge" : "", | 20260505 13:27:22.718 | INFO | "huge_shared" : " 27 GB", | 20260505 13:27:22.718 | INFO | "malloc" : " 14 GB", | 20260505 13:27:22.718 | INFO | "pinned" : " 29 GB" | 20260505 13:27:22.718 | INFO | } | 20260505 13:27:22.718 | INFO | Perf report for the last 5.13511 seconds: | 20260505 13:27:22.718 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:27:22.718 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:22.718 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:22.718 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:22.718 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 13:27:22.718 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:22.718 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:22.718 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:27:22.718 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:22.718 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:22.718 | INFO | Uploaded 36GiB to GPU in 2.18692s (16.4615GiB/s) | 20260505 13:27:22.718 | INFO | Downloaded 34GiB to GPU in 2.13043s (15.9592GiB/s) | 20260505 13:27:27.857 | INFO | Analyzing chunk frames = [3561984,3562496) | 20260505 13:27:27.857 | INFO | MemInfo: { | 20260505 13:27:27.857 | INFO | "gpu" : " 17 GB", | 20260505 13:27:27.857 | INFO | "huge" : "", | 20260505 13:27:27.857 | INFO | "huge_shared" : " 27 GB", | 20260505 13:27:27.857 | INFO | "malloc" : " 14 GB", | 20260505 13:27:27.857 | INFO | "pinned" : " 29 GB" | 20260505 13:27:27.857 | INFO | } | 20260505 13:27:27.857 | INFO | Perf report for the last 5.13938 seconds: | 20260505 13:27:27.857 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:27:27.857 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:27.857 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:27.857 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:27.857 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 13:27:27.857 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:27.857 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:27.857 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 13:27:27.857 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:27.857 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:27.858 | INFO | Uploaded 36GiB to GPU in 2.17336s (16.5642GiB/s) | 20260505 13:27:27.858 | INFO | Downloaded 34GiB to GPU in 2.10997s (16.114GiB/s) | 20260505 13:27:33.005 | INFO | Analyzing chunk frames = [3562496,3563008) | 20260505 13:27:33.005 | INFO | MemInfo: { | 20260505 13:27:33.005 | INFO | "gpu" : " 17 GB", | 20260505 13:27:33.005 | INFO | "huge" : "", | 20260505 13:27:33.005 | INFO | "huge_shared" : " 27 GB", | 20260505 13:27:33.005 | INFO | "malloc" : " 14 GB", | 20260505 13:27:33.005 | INFO | "pinned" : " 29 GB" | 20260505 13:27:33.005 | INFO | } | 20260505 13:27:33.005 | INFO | Perf report for the last 5.14785 seconds: | 20260505 13:27:33.005 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:27:33.005 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:33.005 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:33.005 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:33.005 | INFO | ANALYSIS : 63.2%, 3.0, 98.9ms, 0 | 20260505 13:27:33.005 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:33.005 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:33.005 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 13:27:33.005 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:33.005 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:33.006 | INFO | Uploaded 36GiB to GPU in 2.15413s (16.7121GiB/s) | 20260505 13:27:33.006 | INFO | Downloaded 34GiB to GPU in 2.13991s (15.8885GiB/s) | 20260505 13:27:38.143 | INFO | Analyzing chunk frames = [3563008,3563520) | 20260505 13:27:38.143 | INFO | MemInfo: { | 20260505 13:27:38.143 | INFO | "gpu" : " 17 GB", | 20260505 13:27:38.143 | INFO | "huge" : "", | 20260505 13:27:38.143 | INFO | "huge_shared" : " 27 GB", | 20260505 13:27:38.143 | INFO | "malloc" : " 14 GB", | 20260505 13:27:38.143 | INFO | "pinned" : " 29 GB" | 20260505 13:27:38.143 | INFO | } | 20260505 13:27:38.143 | INFO | Perf report for the last 5.13779 seconds: | 20260505 13:27:38.143 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:27:38.143 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:38.143 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:38.143 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:38.143 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 13:27:38.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:38.143 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:38.143 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 13:27:38.143 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:38.143 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:38.143 | INFO | Uploaded 36GiB to GPU in 2.21301s (16.2674GiB/s) | 20260505 13:27:38.143 | INFO | Downloaded 34GiB to GPU in 2.1062s (16.1428GiB/s) | 20260505 13:27:38.143 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 13:27:38.143 | INFO | Total mem usage: 15.3 GiB | 20260505 13:27:38.143 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 13:27:38.143 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -7.06%) | 20260505 13:27:38.143 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 13:27:38.143 | INFO | High Water Breakdown: | 20260505 13:27:38.143 | INFO | 61.1% : 8.75 GiB ( 91.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 13:27:38.143 | INFO | 38.8% : 5.55 GiB ( 83.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 13:27:38.143 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 13:27:38.144 | INFO | ------------------------------------------ | 20260505 13:27:38.144 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:27:38.144 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 13:27:38.144 | INFO | Total mem usage: 26.5 GiB | 20260505 13:27:38.144 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 13:27:38.144 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 13:27:38.144 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 13:27:38.144 | INFO | High Water Breakdown: | 20260505 13:27:38.144 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:27:38.144 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:27:38.144 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:27:38.144 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:27:38.144 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:27:38.144 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:27:38.144 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 13:27:38.144 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 13:27:38.144 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:27:38.144 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 13:27:38.144 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 13:27:38.144 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 13:27:38.144 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 13:27:38.144 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 13:27:38.144 | INFO | ------------------------------------------ | 20260505 13:27:38.144 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:27:38.144 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 13:27:38.144 | INFO | Total mem usage: 25.6 GiB | 20260505 13:27:38.144 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 13:27:38.144 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 13:27:38.144 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 13:27:38.144 | INFO | High Water Breakdown: | 20260505 13:27:38.144 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 13:27:38.144 | INFO | ------------------------------------------ | 20260505 13:27:38.144 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:27:38.144 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 13:27:38.144 | INFO | Total mem usage: 15.7 GiB | 20260505 13:27:38.144 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 13:27:38.144 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 13:27:38.144 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 13:27:38.144 | INFO | High Water Breakdown: | 20260505 13:27:38.144 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 13:27:38.144 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 13:27:38.145 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 13:27:38.145 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 13:27:38.145 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 13:27:38.145 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 13:27:38.145 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 13:27:38.145 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 13:27:38.145 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 13:27:38.145 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:27:38.145 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 13:27:38.145 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:27:38.145 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 13:27:38.145 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 13:27:38.145 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 13:27:38.145 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 13:27:38.145 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:27:38.145 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 13:27:38.145 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:27:38.145 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:27:38.145 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:27:38.145 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 13:27:38.145 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 13:27:38.145 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:27:38.145 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 13:27:38.145 | INFO | ------------------------------------------ | 20260505 13:27:38.145 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:27:38.145 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 13:27:38.145 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:27:38.145 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:27:38.145 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.73e+01, 1.38e+01 / 1.71e+00 | 20260505 13:27:38.145 | INFO | PerfReport: 0.29%, AggregateStats -- 9.27e-02 / 1.00e-01, 1.07e-02 / 4.64e-03 | 20260505 13:27:38.145 | INFO | PerfReport: 11.74%, Baseline -- 3.74e+00 / 4.50e+00, 5.79e-01 / 1.22e-01 | 20260505 13:27:38.145 | INFO | PerfReport: 3.29%, Histogram -- 1.05e+00 / 1.50e+00, 2.01e-01 / 9.07e-02 | 20260505 13:27:38.145 | INFO | PerfReport: 16.13%, DME -- 4.11e+01 / 4.66e+01, 4.84e+00 / 2.18e+00 | 20260505 13:27:38.145 | INFO | PerfReport: 56.22%, FrameLabeling -- 1.79e+01 / 1.97e+01, 1.27e+00 / 1.94e+00 | 20260505 13:27:38.145 | INFO | PerfReport: 7.49%, PulseAccumulating -- 2.38e+00 / 2.95e+00, 4.51e-01 / 1.77e-01 | 20260505 13:27:38.145 | INFO | PerfReport: 3.58%, Metrics -- 1.14e+00 / 1.45e+00, 3.15e-01 / 1.36e-01 | 20260505 13:27:38.145 | INFO | PerfReport: 1.26%, PulseSerializing -- 4.00e-01 / 4.52e-01, 7.12e-02 / 2.97e-02 | 20260505 13:27:38.145 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 13:27:38.145 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:27:38.145 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:27:38.145 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260505 13:27:38.145 | INFO | PerfReport: 22.59%, Upload -- 2.28e+01 / 0.00e+00, 8.15e+00 / 0.00e+00 | 20260505 13:27:38.145 | INFO | PerfReport: 22.05%, Download -- 2.23e+01 / 0.00e+00, 8.12e+00 / 0.00e+00 | 20260505 13:27:38.145 | INFO | PerfReport: 31.53%, Compute -- 3.18e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 13:27:38.145 | INFO | PerfReport: 2.34%, LockUpload -- 2.37e+00 / 0.00e+00, 8.04e+00 / 0.00e+00 | 20260505 13:27:38.145 | INFO | PerfReport: 1.40%, LockDownload -- 1.42e+00 / 0.00e+00, 2.93e+00 / 0.00e+00 | 20260505 13:27:38.145 | INFO | PerfReport: 20.08%, LockCompute -- 2.03e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260505 13:27:38.145 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.70e-02 / 0.00e+00 | 20260505 13:27:38.277 | INFO | Starting a round of Baz IO | 20260505 13:27:41.518 | INFO | Populating RTMetrics report | 20260505 13:27:41.519 | INFO | RTMetrics report populated | 20260505 13:27:41.519 | INFO | Opening tmp RTMetrics json file | 20260505 13:27:41.541 | INFO | Serializing RTMetrics to json | 20260505 13:27:41.548 | INFO | Serialization done, swapping tmp file now... | 20260505 13:27:41.566 | INFO | RTMetrics done | 20260505 13:27:41.628 | INFO | Finished a round of Baz IO: Wrote 2.297 GiB in 3.351 seconds | 20260505 13:27:41.628 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.11% / 0.00% / 76.88% | 20260505 13:27:41.628 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.699313% / 7.94729e-06% / 9.13938e-05% | 20260505 13:27:41.628 | INFO | Baz IO overall: | 20260505 13:27:41.628 | INFO | BytesWritten : 2122432833048 | 20260505 13:27:41.628 | INFO | NumEvents : 1009763810413 | 20260505 13:27:43.270 | INFO | Analyzing chunk frames = [3563520,3564032) | 20260505 13:27:43.270 | INFO | MemInfo: { | 20260505 13:27:43.270 | INFO | "gpu" : " 17 GB", | 20260505 13:27:43.270 | INFO | "huge" : "", | 20260505 13:27:43.270 | INFO | "huge_shared" : " 27 GB", | 20260505 13:27:43.270 | INFO | "malloc" : " 14 GB", | 20260505 13:27:43.270 | INFO | "pinned" : " 29 GB" | 20260505 13:27:43.270 | INFO | } | 20260505 13:27:43.270 | INFO | Perf report for the last 5.12678 seconds: | 20260505 13:27:43.270 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:27:43.270 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:43.270 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:43.270 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:43.270 | INFO | ANALYSIS : 64.7%, 3.0, 100.0ms, 0 | 20260505 13:27:43.270 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:43.270 | INFO | RT_METRICS : 46.1%, 1.0, 24.1ms, 0 | 20260505 13:27:43.270 | INFO | PRE_HQ : 61.0%, 1.8, 58.1ms, 0 | 20260505 13:27:43.270 | INFO | BAZWRITER : 64.7%, 4.0, 132.5ms, 0 | 20260505 13:27:43.270 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 13:27:43.270 | INFO | Uploaded 36GiB to GPU in 2.20741s (16.3087GiB/s) | 20260505 13:27:43.270 | INFO | Downloaded 38GiB to GPU in 2.2857s (16.6251GiB/s) | 20260505 13:27:48.410 | INFO | Analyzing chunk frames = [3564032,3564544) | 20260505 13:27:48.410 | INFO | MemInfo: { | 20260505 13:27:48.410 | INFO | "gpu" : " 17 GB", | 20260505 13:27:48.410 | INFO | "huge" : "", | 20260505 13:27:48.410 | INFO | "huge_shared" : " 27 GB", | 20260505 13:27:48.410 | INFO | "malloc" : " 14 GB", | 20260505 13:27:48.410 | INFO | "pinned" : " 29 GB" | 20260505 13:27:48.410 | INFO | } | 20260505 13:27:48.410 | INFO | Perf report for the last 5.1404 seconds: | 20260505 13:27:48.410 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:27:48.410 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:48.410 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:48.410 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:48.410 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 13:27:48.410 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:48.410 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:48.410 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:27:48.410 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:48.410 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:48.410 | INFO | Uploaded 36GiB to GPU in 2.17902s (16.5212GiB/s) | 20260505 13:27:48.411 | INFO | Downloaded 34GiB to GPU in 2.05797s (16.5211GiB/s) | 20260505 13:27:53.560 | INFO | Analyzing chunk frames = [3564544,3565056) | 20260505 13:27:53.560 | INFO | MemInfo: { | 20260505 13:27:53.560 | INFO | "gpu" : " 17 GB", | 20260505 13:27:53.560 | INFO | "huge" : "", | 20260505 13:27:53.560 | INFO | "huge_shared" : " 27 GB", | 20260505 13:27:53.560 | INFO | "malloc" : " 14 GB", | 20260505 13:27:53.560 | INFO | "pinned" : " 29 GB" | 20260505 13:27:53.560 | INFO | } | 20260505 13:27:53.560 | INFO | Perf report for the last 5.14997 seconds: | 20260505 13:27:53.560 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:27:53.560 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:53.560 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:53.560 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:53.560 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 13:27:53.560 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:53.560 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:53.560 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 13:27:53.560 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:53.560 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:53.560 | INFO | Uploaded 36GiB to GPU in 2.18467s (16.4785GiB/s) | 20260505 13:27:53.561 | INFO | Downloaded 34GiB to GPU in 2.11655s (16.0639GiB/s) | 20260505 13:27:58.705 | INFO | Analyzing chunk frames = [3565056,3565568) | 20260505 13:27:58.705 | INFO | MemInfo: { | 20260505 13:27:58.705 | INFO | "gpu" : " 17 GB", | 20260505 13:27:58.705 | INFO | "huge" : "", | 20260505 13:27:58.705 | INFO | "huge_shared" : " 27 GB", | 20260505 13:27:58.705 | INFO | "malloc" : " 14 GB", | 20260505 13:27:58.706 | INFO | "pinned" : " 29 GB" | 20260505 13:27:58.706 | INFO | } | 20260505 13:27:58.706 | INFO | Perf report for the last 5.14545 seconds: | 20260505 13:27:58.706 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:27:58.706 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:58.706 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:58.706 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:58.706 | INFO | ANALYSIS : 62.6%, 3.0, 97.3ms, 0 | 20260505 13:27:58.706 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:58.706 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:58.706 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:27:58.706 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:58.706 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:27:58.706 | INFO | Uploaded 36GiB to GPU in 2.16935s (16.5948GiB/s) | 20260505 13:27:58.706 | INFO | Downloaded 34GiB to GPU in 2.12749s (15.9813GiB/s) | 20260505 13:28:03.838 | INFO | Analyzing chunk frames = [3565568,3566080) | 20260505 13:28:03.838 | INFO | MemInfo: { | 20260505 13:28:03.838 | INFO | "gpu" : " 17 GB", | 20260505 13:28:03.838 | INFO | "huge" : "", | 20260505 13:28:03.838 | INFO | "huge_shared" : " 27 GB", | 20260505 13:28:03.838 | INFO | "malloc" : " 14 GB", | 20260505 13:28:03.838 | INFO | "pinned" : " 29 GB" | 20260505 13:28:03.838 | INFO | } | 20260505 13:28:03.838 | INFO | Perf report for the last 5.13253 seconds: | 20260505 13:28:03.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:28:03.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:03.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:03.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:03.838 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 13:28:03.838 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:03.838 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:03.838 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:28:03.838 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:03.838 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:03.838 | INFO | Uploaded 36GiB to GPU in 2.17838s (16.5261GiB/s) | 20260505 13:28:03.839 | INFO | Downloaded 34GiB to GPU in 2.13251s (15.9437GiB/s) | 20260505 13:28:08.971 | INFO | Analyzing chunk frames = [3566080,3566592) | 20260505 13:28:08.971 | INFO | MemInfo: { | 20260505 13:28:08.971 | INFO | "gpu" : " 17 GB", | 20260505 13:28:08.971 | INFO | "huge" : "", | 20260505 13:28:08.971 | INFO | "huge_shared" : " 27 GB", | 20260505 13:28:08.971 | INFO | "malloc" : " 14 GB", | 20260505 13:28:08.971 | INFO | "pinned" : " 29 GB" | 20260505 13:28:08.971 | INFO | } | 20260505 13:28:08.971 | INFO | Perf report for the last 5.13262 seconds: | 20260505 13:28:08.971 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:28:08.971 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:08.971 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:08.971 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:08.971 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 13:28:08.971 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:08.971 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:08.971 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 13:28:08.971 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:08.971 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:08.971 | INFO | Uploaded 36GiB to GPU in 2.16263s (16.6464GiB/s) | 20260505 13:28:08.971 | INFO | Downloaded 34GiB to GPU in 2.106s (16.1443GiB/s) | 20260505 13:28:14.115 | INFO | Analyzing chunk frames = [3566592,3567104) | 20260505 13:28:14.115 | INFO | MemInfo: { | 20260505 13:28:14.115 | INFO | "gpu" : " 17 GB", | 20260505 13:28:14.115 | INFO | "huge" : "", | 20260505 13:28:14.115 | INFO | "huge_shared" : " 27 GB", | 20260505 13:28:14.116 | INFO | "malloc" : " 14 GB", | 20260505 13:28:14.116 | INFO | "pinned" : " 29 GB" | 20260505 13:28:14.116 | INFO | } | 20260505 13:28:14.116 | INFO | Perf report for the last 5.14476 seconds: | 20260505 13:28:14.116 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:28:14.116 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:14.116 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:14.116 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:14.116 | INFO | ANALYSIS : 63.2%, 3.0, 98.8ms, 0 | 20260505 13:28:14.116 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:14.116 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:14.116 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 13:28:14.116 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:14.116 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:14.116 | INFO | Uploaded 36GiB to GPU in 2.17511s (16.5509GiB/s) | 20260505 13:28:14.116 | INFO | Downloaded 34GiB to GPU in 2.1309s (15.9557GiB/s) | 20260505 13:28:19.253 | INFO | Analyzing chunk frames = [3567104,3567616) | 20260505 13:28:19.254 | INFO | MemInfo: { | 20260505 13:28:19.254 | INFO | "gpu" : " 17 GB", | 20260505 13:28:19.254 | INFO | "huge" : "", | 20260505 13:28:19.254 | INFO | "huge_shared" : " 27 GB", | 20260505 13:28:19.254 | INFO | "malloc" : " 14 GB", | 20260505 13:28:19.254 | INFO | "pinned" : " 29 GB" | 20260505 13:28:19.254 | INFO | } | 20260505 13:28:19.254 | INFO | Perf report for the last 5.13808 seconds: | 20260505 13:28:19.254 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:28:19.254 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:19.254 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:19.254 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:19.254 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 13:28:19.254 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:19.254 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:19.254 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 13:28:19.254 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:19.254 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:19.254 | INFO | Uploaded 36GiB to GPU in 2.18477s (16.4777GiB/s) | 20260505 13:28:19.254 | INFO | Downloaded 34GiB to GPU in 2.10436s (16.1569GiB/s) | 20260505 13:28:19.395 | INFO | Starting a round of Baz IO | 20260505 13:28:22.623 | INFO | Populating RTMetrics report | 20260505 13:28:22.623 | INFO | RTMetrics report populated | 20260505 13:28:22.623 | INFO | Opening tmp RTMetrics json file | 20260505 13:28:22.683 | INFO | Serializing RTMetrics to json | 20260505 13:28:22.718 | INFO | Serialization done, swapping tmp file now... | 20260505 13:28:22.722 | INFO | RTMetrics done | 20260505 13:28:22.741 | INFO | Finished a round of Baz IO: Wrote 2.298 GiB in 3.345 seconds | 20260505 13:28:22.741 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.11% / 0.00% / 76.89% | 20260505 13:28:22.741 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.698129% / 3.97364e-06% / 5.96046e-05% | 20260505 13:28:22.741 | INFO | Baz IO overall: | 20260505 13:28:22.741 | INFO | BytesWritten : 2124899765488 | 20260505 13:28:22.741 | INFO | NumEvents : 1010931274151 | 20260505 13:28:24.395 | INFO | Analyzing chunk frames = [3567616,3568128) | 20260505 13:28:24.395 | INFO | MemInfo: { | 20260505 13:28:24.395 | INFO | "gpu" : " 17 GB", | 20260505 13:28:24.395 | INFO | "huge" : "", | 20260505 13:28:24.395 | INFO | "huge_shared" : " 27 GB", | 20260505 13:28:24.395 | INFO | "malloc" : " 14 GB", | 20260505 13:28:24.396 | INFO | "pinned" : " 29 GB" | 20260505 13:28:24.396 | INFO | } | 20260505 13:28:24.396 | INFO | Perf report for the last 5.14186 seconds: | 20260505 13:28:24.396 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:28:24.396 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:24.396 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:24.396 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:24.396 | INFO | ANALYSIS : 64.4%, 3.0, 99.7ms, 0 | 20260505 13:28:24.396 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:24.396 | INFO | RT_METRICS : 47.0%, 1.0, 24.7ms, 0 | 20260505 13:28:24.396 | INFO | PRE_HQ : 61.0%, 1.8, 57.9ms, 0 | 20260505 13:28:24.396 | INFO | BAZWRITER : 64.4%, 4.1, 134.7ms, 0 | 20260505 13:28:24.396 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260505 13:28:24.396 | INFO | Uploaded 36GiB to GPU in 2.19136s (16.4282GiB/s) | 20260505 13:28:24.396 | INFO | Downloaded 38GiB to GPU in 2.28831s (16.6061GiB/s) | 20260505 13:28:29.529 | INFO | Analyzing chunk frames = [3568128,3568640) | 20260505 13:28:29.529 | INFO | MemInfo: { | 20260505 13:28:29.529 | INFO | "gpu" : " 17 GB", | 20260505 13:28:29.529 | INFO | "huge" : "", | 20260505 13:28:29.529 | INFO | "huge_shared" : " 27 GB", | 20260505 13:28:29.529 | INFO | "malloc" : " 14 GB", | 20260505 13:28:29.529 | INFO | "pinned" : " 29 GB" | 20260505 13:28:29.529 | INFO | } | 20260505 13:28:29.529 | INFO | Perf report for the last 5.13343 seconds: | 20260505 13:28:29.529 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:28:29.529 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:29.529 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:29.529 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:29.529 | INFO | ANALYSIS : 63.6%, 3.0, 99.1ms, 0 | 20260505 13:28:29.529 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:29.529 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:29.529 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 13:28:29.529 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:29.529 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:29.529 | INFO | Uploaded 36GiB to GPU in 2.17832s (16.5265GiB/s) | 20260505 13:28:29.529 | INFO | Downloaded 34GiB to GPU in 2.03169s (16.7349GiB/s) | 20260505 13:28:34.668 | INFO | Analyzing chunk frames = [3568640,3569152) | 20260505 13:28:34.668 | INFO | MemInfo: { | 20260505 13:28:34.668 | INFO | "gpu" : " 17 GB", | 20260505 13:28:34.668 | INFO | "huge" : "", | 20260505 13:28:34.668 | INFO | "huge_shared" : " 27 GB", | 20260505 13:28:34.668 | INFO | "malloc" : " 14 GB", | 20260505 13:28:34.668 | INFO | "pinned" : " 29 GB" | 20260505 13:28:34.668 | INFO | } | 20260505 13:28:34.668 | INFO | Perf report for the last 5.13936 seconds: | 20260505 13:28:34.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:28:34.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:34.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:34.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:34.668 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 13:28:34.668 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:34.669 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:34.669 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 13:28:34.669 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:34.669 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:34.669 | INFO | Uploaded 36GiB to GPU in 2.23159s (16.132GiB/s) | 20260505 13:28:34.669 | INFO | Downloaded 34GiB to GPU in 2.12961s (15.9653GiB/s) | 20260505 13:28:39.813 | INFO | Analyzing chunk frames = [3569152,3569664) | 20260505 13:28:39.813 | INFO | MemInfo: { | 20260505 13:28:39.813 | INFO | "gpu" : " 17 GB", | 20260505 13:28:39.813 | INFO | "huge" : "", | 20260505 13:28:39.813 | INFO | "huge_shared" : " 27 GB", | 20260505 13:28:39.813 | INFO | "malloc" : " 14 GB", | 20260505 13:28:39.813 | INFO | "pinned" : " 29 GB" | 20260505 13:28:39.813 | INFO | } | 20260505 13:28:39.814 | INFO | Perf report for the last 5.14513 seconds: | 20260505 13:28:39.814 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:28:39.814 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:39.814 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:39.814 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:39.814 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260505 13:28:39.814 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:39.814 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:39.814 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 13:28:39.814 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:39.814 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:39.814 | INFO | Uploaded 36GiB to GPU in 2.19732s (16.3836GiB/s) | 20260505 13:28:39.814 | INFO | Downloaded 34GiB to GPU in 2.13353s (15.936GiB/s) | 20260505 13:28:44.959 | INFO | Analyzing chunk frames = [3569664,3570176) | 20260505 13:28:44.959 | INFO | MemInfo: { | 20260505 13:28:44.959 | INFO | "gpu" : " 17 GB", | 20260505 13:28:44.959 | INFO | "huge" : "", | 20260505 13:28:44.959 | INFO | "huge_shared" : " 27 GB", | 20260505 13:28:44.959 | INFO | "malloc" : " 14 GB", | 20260505 13:28:44.959 | INFO | "pinned" : " 29 GB" | 20260505 13:28:44.959 | INFO | } | 20260505 13:28:44.959 | INFO | Perf report for the last 5.1454 seconds: | 20260505 13:28:44.959 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:28:44.959 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:44.959 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:44.959 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:44.959 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 13:28:44.959 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:44.959 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:44.959 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 13:28:44.959 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:44.959 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:44.959 | INFO | Uploaded 36GiB to GPU in 2.13478s (16.8636GiB/s) | 20260505 13:28:44.959 | INFO | Downloaded 34GiB to GPU in 2.11933s (16.0428GiB/s) | 20260505 13:28:50.089 | INFO | Analyzing chunk frames = [3570176,3570688) | 20260505 13:28:50.089 | INFO | MemInfo: { | 20260505 13:28:50.089 | INFO | "gpu" : " 17 GB", | 20260505 13:28:50.089 | INFO | "huge" : "", | 20260505 13:28:50.089 | INFO | "huge_shared" : " 27 GB", | 20260505 13:28:50.089 | INFO | "malloc" : " 14 GB", | 20260505 13:28:50.089 | INFO | "pinned" : " 29 GB" | 20260505 13:28:50.089 | INFO | } | 20260505 13:28:50.090 | INFO | Perf report for the last 5.13057 seconds: | 20260505 13:28:50.090 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:28:50.090 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:50.090 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:50.090 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:50.090 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 13:28:50.090 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:50.090 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:50.090 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 13:28:50.090 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:50.090 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:50.090 | INFO | Uploaded 36GiB to GPU in 2.18919s (16.4445GiB/s) | 20260505 13:28:50.090 | INFO | Downloaded 34GiB to GPU in 2.10976s (16.1156GiB/s) | 20260505 13:28:55.218 | INFO | Analyzing chunk frames = [3570688,3571200) | 20260505 13:28:55.218 | INFO | MemInfo: { | 20260505 13:28:55.218 | INFO | "gpu" : " 17 GB", | 20260505 13:28:55.218 | INFO | "huge" : "", | 20260505 13:28:55.218 | INFO | "huge_shared" : " 27 GB", | 20260505 13:28:55.218 | INFO | "malloc" : " 14 GB", | 20260505 13:28:55.218 | INFO | "pinned" : " 29 GB" | 20260505 13:28:55.218 | INFO | } | 20260505 13:28:55.218 | INFO | Perf report for the last 5.1285 seconds: | 20260505 13:28:55.218 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:28:55.218 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:55.218 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:55.218 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:55.218 | INFO | ANALYSIS : 63.4%, 3.0, 98.8ms, 0 | 20260505 13:28:55.218 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:55.218 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:55.218 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 13:28:55.218 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:55.218 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:28:55.218 | INFO | Uploaded 36GiB to GPU in 2.18304s (16.4907GiB/s) | 20260505 13:28:55.218 | INFO | Downloaded 34GiB to GPU in 2.14636s (15.8407GiB/s) | 20260505 13:29:00.368 | INFO | Analyzing chunk frames = [3571200,3571712) | 20260505 13:29:00.368 | INFO | MemInfo: { | 20260505 13:29:00.368 | INFO | "gpu" : " 17 GB", | 20260505 13:29:00.368 | INFO | "huge" : "", | 20260505 13:29:00.368 | INFO | "huge_shared" : " 27 GB", | 20260505 13:29:00.368 | INFO | "malloc" : " 14 GB", | 20260505 13:29:00.368 | INFO | "pinned" : " 29 GB" | 20260505 13:29:00.368 | INFO | } | 20260505 13:29:00.368 | INFO | Perf report for the last 5.14978 seconds: | 20260505 13:29:00.368 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:29:00.368 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:00.368 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:00.368 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:00.368 | INFO | ANALYSIS : 63.7%, 3.0, 99.6ms, 0 | 20260505 13:29:00.368 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:00.368 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:00.368 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:29:00.368 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:00.368 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:00.368 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 13:29:00.368 | INFO | Uploaded 36GiB to GPU in 2.23068s (16.1386GiB/s) | 20260505 13:29:00.368 | INFO | Total mem usage: 15.3 GiB | 20260505 13:29:00.368 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 13:29:00.368 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -7.12%) | 20260505 13:29:00.368 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 13:29:00.368 | INFO | High Water Breakdown: | 20260505 13:29:00.368 | INFO | Downloaded 34GiB to GPU in 2.09067s (16.2627GiB/s) | 20260505 13:29:00.368 | INFO | 61.1% : 8.75 GiB ( 91.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 13:29:00.368 | INFO | 38.7% : 5.54 GiB ( 83.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 13:29:00.368 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 13:29:00.368 | INFO | ------------------------------------------ | 20260505 13:29:00.368 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:29:00.368 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 13:29:00.368 | INFO | Total mem usage: 26.5 GiB | 20260505 13:29:00.368 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 13:29:00.368 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 13:29:00.368 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 13:29:00.368 | INFO | High Water Breakdown: | 20260505 13:29:00.369 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:29:00.369 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:29:00.369 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:29:00.369 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:29:00.369 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:29:00.369 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:29:00.369 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 13:29:00.369 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 13:29:00.369 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:29:00.369 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 13:29:00.369 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 13:29:00.369 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 13:29:00.369 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 13:29:00.369 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 13:29:00.369 | INFO | ------------------------------------------ | 20260505 13:29:00.369 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:29:00.369 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 13:29:00.369 | INFO | Total mem usage: 25.6 GiB | 20260505 13:29:00.369 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 13:29:00.369 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 13:29:00.369 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 13:29:00.369 | INFO | High Water Breakdown: | 20260505 13:29:00.369 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 13:29:00.369 | INFO | ------------------------------------------ | 20260505 13:29:00.369 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:29:00.369 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 13:29:00.369 | INFO | Total mem usage: 15.7 GiB | 20260505 13:29:00.369 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 13:29:00.369 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 13:29:00.369 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 13:29:00.369 | INFO | High Water Breakdown: | 20260505 13:29:00.369 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 13:29:00.369 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 13:29:00.369 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 13:29:00.369 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 13:29:00.369 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 13:29:00.369 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 13:29:00.369 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 13:29:00.369 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 13:29:00.369 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 13:29:00.369 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:29:00.369 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 13:29:00.369 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:29:00.369 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 13:29:00.369 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 13:29:00.369 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 13:29:00.369 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 13:29:00.369 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:29:00.369 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 13:29:00.369 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:29:00.369 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:29:00.369 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:29:00.369 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 13:29:00.369 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 13:29:00.369 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:29:00.369 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 13:29:00.369 | INFO | ------------------------------------------ | 20260505 13:29:00.369 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:29:00.369 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 13:29:00.369 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:29:00.369 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:29:00.369 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.68e+01, 1.35e+01 / 1.58e+00 | 20260505 13:29:00.369 | INFO | PerfReport: 0.29%, AggregateStats -- 9.31e-02 / 1.04e-01, 1.07e-02 / 1.31e-02 | 20260505 13:29:00.369 | INFO | PerfReport: 11.78%, Baseline -- 3.76e+00 / 4.47e+00, 5.86e-01 / 1.87e-01 | 20260505 13:29:00.369 | INFO | PerfReport: 3.32%, Histogram -- 1.06e+00 / 1.51e+00, 2.01e-01 / 8.55e-02 | 20260505 13:29:00.369 | INFO | PerfReport: 15.87%, DME -- 4.05e+01 / 4.53e+01, 4.45e+00 / 2.17e+00 | 20260505 13:29:00.369 | INFO | PerfReport: 56.51%, FrameLabeling -- 1.80e+01 / 2.07e+01, 1.51e+00 / 2.07e+00 | 20260505 13:29:00.369 | INFO | PerfReport: 7.42%, PulseAccumulating -- 2.37e+00 / 2.83e+00, 4.43e-01 / 2.33e-01 | 20260505 13:29:00.369 | INFO | PerfReport: 3.56%, Metrics -- 1.14e+00 / 1.42e+00, 3.16e-01 / 1.43e-01 | 20260505 13:29:00.369 | INFO | PerfReport: 1.25%, PulseSerializing -- 4.00e-01 / 4.48e-01, 7.25e-02 / 2.84e-02 | 20260505 13:29:00.369 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 13:29:00.369 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:29:00.369 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:29:00.369 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260505 13:29:00.369 | INFO | PerfReport: 22.60%, Upload -- 2.28e+01 / 0.00e+00, 8.02e+00 / 0.00e+00 | 20260505 13:29:00.369 | INFO | PerfReport: 22.04%, Download -- 2.22e+01 / 0.00e+00, 8.15e+00 / 0.00e+00 | 20260505 13:29:00.369 | INFO | PerfReport: 31.69%, Compute -- 3.19e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 13:29:00.369 | INFO | PerfReport: 2.09%, LockUpload -- 2.11e+00 / 0.00e+00, 7.57e+00 / 0.00e+00 | 20260505 13:29:00.369 | INFO | PerfReport: 1.43%, LockDownload -- 1.44e+00 / 0.00e+00, 2.95e+00 / 0.00e+00 | 20260505 13:29:00.369 | INFO | PerfReport: 20.14%, LockCompute -- 2.03e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260505 13:29:00.369 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.43e-02 / 0.00e+00 | 20260505 13:29:00.503 | INFO | Starting a round of Baz IO | 20260505 13:29:03.734 | INFO | Populating RTMetrics report | 20260505 13:29:03.734 | INFO | RTMetrics report populated | 20260505 13:29:03.735 | INFO | Opening tmp RTMetrics json file | 20260505 13:29:03.735 | INFO | Serializing RTMetrics to json | 20260505 13:29:03.736 | INFO | Serialization done, swapping tmp file now... | 20260505 13:29:03.737 | INFO | RTMetrics done | 20260505 13:29:03.851 | INFO | Finished a round of Baz IO: Wrote 2.298 GiB in 3.347 seconds | 20260505 13:29:03.851 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.11% / 0.00% / 76.89% | 20260505 13:29:03.851 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.69745% / 3.97364e-06% / 9.53674e-05% | 20260505 13:29:03.851 | INFO | Baz IO overall: | 20260505 13:29:03.851 | INFO | BytesWritten : 2127367052679 | 20260505 13:29:03.851 | INFO | NumEvents : 1012098547982 | 20260505 13:29:05.511 | INFO | Analyzing chunk frames = [3571712,3572224) | 20260505 13:29:05.511 | INFO | MemInfo: { | 20260505 13:29:05.511 | INFO | "gpu" : " 17 GB", | 20260505 13:29:05.511 | INFO | "huge" : "", | 20260505 13:29:05.511 | INFO | "huge_shared" : " 27 GB", | 20260505 13:29:05.511 | INFO | "malloc" : " 14 GB", | 20260505 13:29:05.511 | INFO | "pinned" : " 29 GB" | 20260505 13:29:05.511 | INFO | } | 20260505 13:29:05.511 | INFO | Perf report for the last 5.14294 seconds: | 20260505 13:29:05.511 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:29:05.511 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:05.511 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:05.511 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:05.511 | INFO | ANALYSIS : 64.3%, 3.0, 99.8ms, 0 | 20260505 13:29:05.511 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:05.511 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260505 13:29:05.511 | INFO | PRE_HQ : 60.5%, 1.8, 57.5ms, 0 | 20260505 13:29:05.511 | INFO | BAZWRITER : 64.4%, 4.1, 134.4ms, 0 | 20260505 13:29:05.511 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 13:29:05.511 | INFO | Uploaded 36GiB to GPU in 2.19776s (16.3803GiB/s) | 20260505 13:29:05.511 | INFO | Downloaded 38GiB to GPU in 2.28107s (16.6589GiB/s) | 20260505 13:29:10.656 | INFO | Analyzing chunk frames = [3572224,3572736) | 20260505 13:29:10.656 | INFO | MemInfo: { | 20260505 13:29:10.656 | INFO | "gpu" : " 17 GB", | 20260505 13:29:10.656 | INFO | "huge" : "", | 20260505 13:29:10.656 | INFO | "huge_shared" : " 27 GB", | 20260505 13:29:10.656 | INFO | "malloc" : " 14 GB", | 20260505 13:29:10.656 | INFO | "pinned" : " 29 GB" | 20260505 13:29:10.656 | INFO | } | 20260505 13:29:10.656 | INFO | Perf report for the last 5.14517 seconds: | 20260505 13:29:10.656 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:29:10.656 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:10.656 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:10.656 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:10.656 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 13:29:10.656 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:10.656 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:10.656 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 13:29:10.656 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:10.656 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:10.656 | INFO | Uploaded 36GiB to GPU in 2.19622s (16.3918GiB/s) | 20260505 13:29:10.656 | INFO | Downloaded 34GiB to GPU in 2.04058s (16.6619GiB/s) | 20260505 13:29:15.789 | INFO | Analyzing chunk frames = [3572736,3573248) | 20260505 13:29:15.789 | INFO | MemInfo: { | 20260505 13:29:15.789 | INFO | "gpu" : " 17 GB", | 20260505 13:29:15.789 | INFO | "huge" : "", | 20260505 13:29:15.789 | INFO | "huge_shared" : " 27 GB", | 20260505 13:29:15.789 | INFO | "malloc" : " 14 GB", | 20260505 13:29:15.789 | INFO | "pinned" : " 29 GB" | 20260505 13:29:15.789 | INFO | } | 20260505 13:29:15.789 | INFO | Perf report for the last 5.13314 seconds: | 20260505 13:29:15.789 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:29:15.789 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:15.789 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:15.789 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:15.789 | INFO | ANALYSIS : 63.0%, 3.0, 97.9ms, 0 | 20260505 13:29:15.789 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:15.789 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:15.789 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 13:29:15.789 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:15.789 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:15.789 | INFO | Uploaded 36GiB to GPU in 2.14877s (16.7538GiB/s) | 20260505 13:29:15.790 | INFO | Downloaded 34GiB to GPU in 2.05277s (16.563GiB/s) | 20260505 13:29:20.920 | INFO | Analyzing chunk frames = [3573248,3573760) | 20260505 13:29:20.920 | INFO | MemInfo: { | 20260505 13:29:20.920 | INFO | "gpu" : " 17 GB", | 20260505 13:29:20.920 | INFO | "huge" : "", | 20260505 13:29:20.920 | INFO | "huge_shared" : " 27 GB", | 20260505 13:29:20.920 | INFO | "malloc" : " 14 GB", | 20260505 13:29:20.920 | INFO | "pinned" : " 29 GB" | 20260505 13:29:20.920 | INFO | } | 20260505 13:29:20.920 | INFO | Perf report for the last 5.13108 seconds: | 20260505 13:29:20.920 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:29:20.920 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:20.920 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:20.920 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:20.920 | INFO | ANALYSIS : 63.1%, 3.0, 97.9ms, 0 | 20260505 13:29:20.920 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:20.920 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:20.920 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:29:20.920 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:20.920 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:20.920 | INFO | Uploaded 36GiB to GPU in 2.19933s (16.3686GiB/s) | 20260505 13:29:20.920 | INFO | Downloaded 34GiB to GPU in 2.1316s (15.9505GiB/s) | 20260505 13:29:26.067 | INFO | Analyzing chunk frames = [3573760,3574272) | 20260505 13:29:26.067 | INFO | MemInfo: { | 20260505 13:29:26.067 | INFO | "gpu" : " 17 GB", | 20260505 13:29:26.067 | INFO | "huge" : "", | 20260505 13:29:26.067 | INFO | "huge_shared" : " 27 GB", | 20260505 13:29:26.067 | INFO | "malloc" : " 14 GB", | 20260505 13:29:26.067 | INFO | "pinned" : " 29 GB" | 20260505 13:29:26.067 | INFO | } | 20260505 13:29:26.067 | INFO | Perf report for the last 5.14647 seconds: | 20260505 13:29:26.067 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:29:26.067 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:26.067 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:26.067 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:26.067 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 13:29:26.067 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:26.067 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:26.067 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 13:29:26.067 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:26.067 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:26.067 | INFO | Uploaded 36GiB to GPU in 2.17086s (16.5833GiB/s) | 20260505 13:29:26.067 | INFO | Downloaded 34GiB to GPU in 2.1285s (15.9737GiB/s) | 20260505 13:29:31.214 | INFO | Analyzing chunk frames = [3574272,3574784) | 20260505 13:29:31.214 | INFO | MemInfo: { | 20260505 13:29:31.214 | INFO | "gpu" : " 17 GB", | 20260505 13:29:31.214 | INFO | "huge" : "", | 20260505 13:29:31.214 | INFO | "huge_shared" : " 27 GB", | 20260505 13:29:31.214 | INFO | "malloc" : " 14 GB", | 20260505 13:29:31.214 | INFO | "pinned" : " 29 GB" | 20260505 13:29:31.214 | INFO | } | 20260505 13:29:31.214 | INFO | Perf report for the last 5.14743 seconds: | 20260505 13:29:31.214 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:29:31.214 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:31.214 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:31.214 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:31.214 | INFO | ANALYSIS : 63.0%, 3.0, 98.6ms, 0 | 20260505 13:29:31.214 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:31.214 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:31.214 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:29:31.214 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:31.214 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:31.214 | INFO | Uploaded 36GiB to GPU in 2.17066s (16.5849GiB/s) | 20260505 13:29:31.214 | INFO | Downloaded 34GiB to GPU in 2.11143s (16.1029GiB/s) | 20260505 13:29:36.341 | INFO | Analyzing chunk frames = [3574784,3575296) | 20260505 13:29:36.341 | INFO | MemInfo: { | 20260505 13:29:36.341 | INFO | "gpu" : " 17 GB", | 20260505 13:29:36.341 | INFO | "huge" : "", | 20260505 13:29:36.341 | INFO | "huge_shared" : " 27 GB", | 20260505 13:29:36.341 | INFO | "malloc" : " 14 GB", | 20260505 13:29:36.341 | INFO | "pinned" : " 29 GB" | 20260505 13:29:36.341 | INFO | } | 20260505 13:29:36.341 | INFO | Perf report for the last 5.12662 seconds: | 20260505 13:29:36.341 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:29:36.341 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:36.341 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:36.341 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:36.341 | INFO | ANALYSIS : 63.6%, 3.0, 99.0ms, 0 | 20260505 13:29:36.341 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:36.341 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:36.341 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260505 13:29:36.341 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:36.341 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:36.341 | INFO | Uploaded 36GiB to GPU in 2.15988s (16.6676GiB/s) | 20260505 13:29:36.341 | INFO | Downloaded 34GiB to GPU in 2.17266s (15.649GiB/s) | 20260505 13:29:41.476 | INFO | Analyzing chunk frames = [3575296,3575808) | 20260505 13:29:41.476 | INFO | MemInfo: { | 20260505 13:29:41.476 | INFO | "gpu" : " 17 GB", | 20260505 13:29:41.476 | INFO | "huge" : "", | 20260505 13:29:41.476 | INFO | "huge_shared" : " 27 GB", | 20260505 13:29:41.476 | INFO | "malloc" : " 14 GB", | 20260505 13:29:41.476 | INFO | "pinned" : " 29 GB" | 20260505 13:29:41.476 | INFO | } | 20260505 13:29:41.476 | INFO | Perf report for the last 5.13564 seconds: | 20260505 13:29:41.476 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:29:41.476 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:41.476 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:41.476 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:41.476 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 13:29:41.476 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:41.477 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:41.477 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 13:29:41.477 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:41.477 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:41.477 | INFO | Uploaded 36GiB to GPU in 2.20175s (16.3507GiB/s) | 20260505 13:29:41.477 | INFO | Downloaded 34GiB to GPU in 2.11616s (16.0668GiB/s) | 20260505 13:29:41.610 | INFO | Starting a round of Baz IO | 20260505 13:29:44.844 | INFO | Populating RTMetrics report | 20260505 13:29:44.845 | INFO | RTMetrics report populated | 20260505 13:29:44.845 | INFO | Opening tmp RTMetrics json file | 20260505 13:29:44.870 | INFO | Serializing RTMetrics to json | 20260505 13:29:44.872 | INFO | Serialization done, swapping tmp file now... | 20260505 13:29:44.877 | INFO | RTMetrics done | 20260505 13:29:44.959 | INFO | Finished a round of Baz IO: Wrote 2.295 GiB in 3.348 seconds | 20260505 13:29:44.959 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.10% / 0.00% / 76.89% | 20260505 13:29:44.959 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.693321% / 3.97364e-06% / 8.34465e-05% | 20260505 13:29:44.959 | INFO | Baz IO overall: | 20260505 13:29:44.959 | INFO | BytesWritten : 2129831742748 | 20260505 13:29:44.959 | INFO | NumEvents : 1013264174728 | 20260505 13:29:46.624 | INFO | Analyzing chunk frames = [3575808,3576320) | 20260505 13:29:46.624 | INFO | MemInfo: { | 20260505 13:29:46.624 | INFO | "gpu" : " 17 GB", | 20260505 13:29:46.624 | INFO | "huge" : "", | 20260505 13:29:46.624 | INFO | "huge_shared" : " 27 GB", | 20260505 13:29:46.624 | INFO | "malloc" : " 14 GB", | 20260505 13:29:46.624 | INFO | "pinned" : " 29 GB" | 20260505 13:29:46.624 | INFO | } | 20260505 13:29:46.624 | INFO | Perf report for the last 5.14771 seconds: | 20260505 13:29:46.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:29:46.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:46.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:46.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:46.624 | INFO | ANALYSIS : 64.3%, 3.0, 99.8ms, 0 | 20260505 13:29:46.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:46.624 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260505 13:29:46.624 | INFO | PRE_HQ : 60.5%, 1.8, 57.9ms, 0 | 20260505 13:29:46.624 | INFO | BAZWRITER : 64.2%, 4.0, 133.5ms, 0 | 20260505 13:29:46.624 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 13:29:46.624 | INFO | Uploaded 36GiB to GPU in 2.177s (16.5365GiB/s) | 20260505 13:29:46.624 | INFO | Downloaded 38GiB to GPU in 2.28275s (16.6466GiB/s) | 20260505 13:29:51.766 | INFO | Analyzing chunk frames = [3576320,3576832) | 20260505 13:29:51.766 | INFO | MemInfo: { | 20260505 13:29:51.766 | INFO | "gpu" : " 17 GB", | 20260505 13:29:51.766 | INFO | "huge" : "", | 20260505 13:29:51.766 | INFO | "huge_shared" : " 27 GB", | 20260505 13:29:51.766 | INFO | "malloc" : " 14 GB", | 20260505 13:29:51.766 | INFO | "pinned" : " 29 GB" | 20260505 13:29:51.766 | INFO | } | 20260505 13:29:51.766 | INFO | Perf report for the last 5.14179 seconds: | 20260505 13:29:51.766 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:29:51.766 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:51.766 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:51.766 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:51.766 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 13:29:51.766 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:51.766 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:51.766 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 13:29:51.766 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:51.766 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:51.766 | INFO | Uploaded 36GiB to GPU in 2.19441s (16.4053GiB/s) | 20260505 13:29:51.766 | INFO | Downloaded 34GiB to GPU in 2.03436s (16.7129GiB/s) | 20260505 13:29:56.906 | INFO | Analyzing chunk frames = [3576832,3577344) | 20260505 13:29:56.906 | INFO | MemInfo: { | 20260505 13:29:56.906 | INFO | "gpu" : " 17 GB", | 20260505 13:29:56.906 | INFO | "huge" : "", | 20260505 13:29:56.906 | INFO | "huge_shared" : " 27 GB", | 20260505 13:29:56.906 | INFO | "malloc" : " 14 GB", | 20260505 13:29:56.906 | INFO | "pinned" : " 29 GB" | 20260505 13:29:56.906 | INFO | } | 20260505 13:29:56.906 | INFO | Perf report for the last 5.14038 seconds: | 20260505 13:29:56.906 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:29:56.906 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:56.906 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:56.906 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:56.906 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 13:29:56.906 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:56.906 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:56.906 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 13:29:56.906 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:56.906 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:29:56.907 | INFO | Uploaded 36GiB to GPU in 2.23539s (16.1045GiB/s) | 20260505 13:29:56.907 | INFO | Downloaded 34GiB to GPU in 2.12295s (16.0155GiB/s) | 20260505 13:30:02.031 | INFO | Analyzing chunk frames = [3577344,3577856) | 20260505 13:30:02.031 | INFO | MemInfo: { | 20260505 13:30:02.031 | INFO | "gpu" : " 17 GB", | 20260505 13:30:02.031 | INFO | "huge" : "", | 20260505 13:30:02.031 | INFO | "huge_shared" : " 27 GB", | 20260505 13:30:02.032 | INFO | "malloc" : " 14 GB", | 20260505 13:30:02.032 | INFO | "pinned" : " 29 GB" | 20260505 13:30:02.032 | INFO | } | 20260505 13:30:02.032 | INFO | Perf report for the last 5.12537 seconds: | 20260505 13:30:02.032 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:30:02.032 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:02.032 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:02.032 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:02.032 | INFO | ANALYSIS : 63.1%, 3.0, 97.8ms, 0 | 20260505 13:30:02.032 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:02.032 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:02.032 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 13:30:02.032 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:02.032 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:02.032 | INFO | Uploaded 36GiB to GPU in 2.20189s (16.3496GiB/s) | 20260505 13:30:02.032 | INFO | Downloaded 34GiB to GPU in 2.13397s (15.9328GiB/s) | 20260505 13:30:07.171 | INFO | Analyzing chunk frames = [3577856,3578368) | 20260505 13:30:07.171 | INFO | MemInfo: { | 20260505 13:30:07.171 | INFO | "gpu" : " 17 GB", | 20260505 13:30:07.171 | INFO | "huge" : "", | 20260505 13:30:07.171 | INFO | "huge_shared" : " 27 GB", | 20260505 13:30:07.171 | INFO | "malloc" : " 14 GB", | 20260505 13:30:07.171 | INFO | "pinned" : " 29 GB" | 20260505 13:30:07.171 | INFO | } | 20260505 13:30:07.171 | INFO | Perf report for the last 5.13955 seconds: | 20260505 13:30:07.171 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:30:07.171 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:07.171 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:07.171 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:07.171 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 13:30:07.171 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:07.171 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:07.171 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 13:30:07.171 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:07.171 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:07.172 | INFO | Uploaded 36GiB to GPU in 2.19779s (16.3801GiB/s) | 20260505 13:30:07.172 | INFO | Downloaded 34GiB to GPU in 2.12864s (15.9726GiB/s) | 20260505 13:30:12.322 | INFO | Analyzing chunk frames = [3578368,3578880) | 20260505 13:30:12.322 | INFO | MemInfo: { | 20260505 13:30:12.322 | INFO | "gpu" : " 17 GB", | 20260505 13:30:12.322 | INFO | "huge" : "", | 20260505 13:30:12.322 | INFO | "huge_shared" : " 27 GB", | 20260505 13:30:12.322 | INFO | "malloc" : " 14 GB", | 20260505 13:30:12.322 | INFO | "pinned" : " 29 GB" | 20260505 13:30:12.322 | INFO | } | 20260505 13:30:12.322 | INFO | Perf report for the last 5.15108 seconds: | 20260505 13:30:12.322 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:30:12.322 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:12.322 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:12.322 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:12.322 | INFO | ANALYSIS : 63.1%, 3.0, 98.7ms, 0 | 20260505 13:30:12.322 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:12.322 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:12.322 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 13:30:12.322 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:12.322 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:12.323 | INFO | Uploaded 36GiB to GPU in 2.16158s (16.6545GiB/s) | 20260505 13:30:12.323 | INFO | Downloaded 34GiB to GPU in 2.11262s (16.0938GiB/s) | 20260505 13:30:17.467 | INFO | Analyzing chunk frames = [3578880,3579392) | 20260505 13:30:17.467 | INFO | MemInfo: { | 20260505 13:30:17.467 | INFO | "gpu" : " 17 GB", | 20260505 13:30:17.467 | INFO | "huge" : "", | 20260505 13:30:17.467 | INFO | "huge_shared" : " 27 GB", | 20260505 13:30:17.467 | INFO | "malloc" : " 14 GB", | 20260505 13:30:17.467 | INFO | "pinned" : " 29 GB" | 20260505 13:30:17.467 | INFO | } | 20260505 13:30:17.467 | INFO | Perf report for the last 5.1444 seconds: | 20260505 13:30:17.467 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:30:17.467 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:17.467 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:17.467 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:17.467 | INFO | ANALYSIS : 63.7%, 3.0, 99.6ms, 0 | 20260505 13:30:17.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:17.467 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:17.467 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 13:30:17.467 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:17.467 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:17.467 | INFO | Uploaded 36GiB to GPU in 2.1506s (16.7395GiB/s) | 20260505 13:30:17.467 | INFO | Downloaded 34GiB to GPU in 2.18472s (15.5626GiB/s) | 20260505 13:30:22.601 | INFO | Analyzing chunk frames = [3579392,3579904) | 20260505 13:30:22.601 | INFO | MemInfo: { | 20260505 13:30:22.601 | INFO | "gpu" : " 17 GB", | 20260505 13:30:22.601 | INFO | "huge" : "", | 20260505 13:30:22.601 | INFO | "huge_shared" : " 27 GB", | 20260505 13:30:22.601 | INFO | "malloc" : " 14 GB", | 20260505 13:30:22.601 | INFO | "pinned" : " 29 GB" | 20260505 13:30:22.601 | INFO | } | 20260505 13:30:22.601 | INFO | Perf report for the last 5.13444 seconds: | 20260505 13:30:22.601 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:30:22.601 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:22.601 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:22.601 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:22.601 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 13:30:22.601 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:22.601 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:22.601 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 13:30:22.601 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:22.601 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:22.602 | INFO | Uploaded 36GiB to GPU in 2.2175s (16.2345GiB/s) | 20260505 13:30:22.602 | INFO | Downloaded 34GiB to GPU in 2.09515s (16.2279GiB/s) | 20260505 13:30:22.602 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 13:30:22.602 | INFO | Total mem usage: 15.3 GiB | 20260505 13:30:22.602 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 13:30:22.602 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -7.12%) | 20260505 13:30:22.602 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 13:30:22.602 | INFO | High Water Breakdown: | 20260505 13:30:22.602 | INFO | 61.1% : 8.75 GiB ( 91.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 13:30:22.602 | INFO | 38.7% : 5.54 GiB ( 83.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 13:30:22.602 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 13:30:22.602 | INFO | ------------------------------------------ | 20260505 13:30:22.602 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:30:22.602 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 13:30:22.602 | INFO | Total mem usage: 26.5 GiB | 20260505 13:30:22.602 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 13:30:22.602 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 13:30:22.602 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 13:30:22.602 | INFO | High Water Breakdown: | 20260505 13:30:22.602 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:30:22.602 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:30:22.602 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:30:22.602 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:30:22.602 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:30:22.602 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:30:22.602 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 13:30:22.602 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 13:30:22.602 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:30:22.602 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 13:30:22.602 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 13:30:22.602 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 13:30:22.602 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 13:30:22.602 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 13:30:22.602 | INFO | ------------------------------------------ | 20260505 13:30:22.602 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:30:22.602 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 13:30:22.602 | INFO | Total mem usage: 25.6 GiB | 20260505 13:30:22.602 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 13:30:22.602 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 13:30:22.602 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 13:30:22.602 | INFO | High Water Breakdown: | 20260505 13:30:22.602 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 13:30:22.602 | INFO | ------------------------------------------ | 20260505 13:30:22.602 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:30:22.602 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 13:30:22.602 | INFO | Total mem usage: 15.7 GiB | 20260505 13:30:22.602 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 13:30:22.602 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 13:30:22.602 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 13:30:22.602 | INFO | High Water Breakdown: | 20260505 13:30:22.602 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 13:30:22.602 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 13:30:22.602 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 13:30:22.602 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 13:30:22.602 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 13:30:22.602 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 13:30:22.602 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 13:30:22.602 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 13:30:22.602 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 13:30:22.602 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:30:22.602 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 13:30:22.602 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:30:22.602 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 13:30:22.602 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 13:30:22.602 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 13:30:22.602 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 13:30:22.602 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:30:22.602 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 13:30:22.602 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:30:22.602 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:30:22.602 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:30:22.602 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 13:30:22.602 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 13:30:22.603 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:30:22.603 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 13:30:22.603 | INFO | ------------------------------------------ | 20260505 13:30:22.603 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:30:22.603 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 13:30:22.603 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:30:22.603 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:30:22.603 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.67e+01, 1.39e+01 / 2.00e+00 | 20260505 13:30:22.603 | INFO | PerfReport: 0.29%, AggregateStats -- 9.26e-02 / 1.01e-01, 1.04e-02 / 5.24e-03 | 20260505 13:30:22.603 | INFO | PerfReport: 11.75%, Baseline -- 3.74e+00 / 4.47e+00, 5.79e-01 / 1.84e-01 | 20260505 13:30:22.603 | INFO | PerfReport: 3.29%, Histogram -- 1.05e+00 / 1.50e+00, 1.98e-01 / 9.69e-02 | 20260505 13:30:22.603 | INFO | PerfReport: 16.04%, DME -- 4.09e+01 / 4.60e+01, 4.82e+00 / 2.16e+00 | 20260505 13:30:22.603 | INFO | PerfReport: 56.35%, FrameLabeling -- 1.80e+01 / 1.98e+01, 1.37e+00 / 1.70e+00 | 20260505 13:30:22.603 | INFO | PerfReport: 7.46%, PulseAccumulating -- 2.38e+00 / 2.95e+00, 4.55e-01 / 1.52e-01 | 20260505 13:30:22.603 | INFO | PerfReport: 3.57%, Metrics -- 1.14e+00 / 1.46e+00, 3.20e-01 / 1.49e-01 | 20260505 13:30:22.603 | INFO | PerfReport: 1.25%, PulseSerializing -- 4.00e-01 / 4.52e-01, 7.30e-02 / 3.62e-02 | 20260505 13:30:22.603 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 13:30:22.603 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:30:22.603 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:30:22.603 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260505 13:30:22.603 | INFO | PerfReport: 22.57%, Upload -- 2.28e+01 / 0.00e+00, 8.04e+00 / 0.00e+00 | 20260505 13:30:22.603 | INFO | PerfReport: 22.02%, Download -- 2.22e+01 / 0.00e+00, 8.04e+00 / 0.00e+00 | 20260505 13:30:22.603 | INFO | PerfReport: 31.59%, Compute -- 3.19e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 13:30:22.603 | INFO | PerfReport: 2.25%, LockUpload -- 2.27e+00 / 0.00e+00, 8.01e+00 / 0.00e+00 | 20260505 13:30:22.603 | INFO | PerfReport: 1.41%, LockDownload -- 1.42e+00 / 0.00e+00, 2.93e+00 / 0.00e+00 | 20260505 13:30:22.603 | INFO | PerfReport: 20.15%, LockCompute -- 2.03e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260505 13:30:22.603 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.07e-02 / 0.00e+00 | 20260505 13:30:22.740 | INFO | Starting a round of Baz IO | 20260505 13:30:25.962 | INFO | Populating RTMetrics report | 20260505 13:30:25.962 | INFO | RTMetrics report populated | 20260505 13:30:25.962 | INFO | Opening tmp RTMetrics json file | 20260505 13:30:25.983 | INFO | Serializing RTMetrics to json | 20260505 13:30:26.006 | INFO | Serialization done, swapping tmp file now... | 20260505 13:30:26.009 | INFO | RTMetrics done | 20260505 13:30:26.085 | INFO | Finished a round of Baz IO: Wrote 2.295 GiB in 3.345 seconds | 20260505 13:30:26.086 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.10% / 0.01% / 76.90% | 20260505 13:30:26.086 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.690933% / 0% / 9.13938e-05% | 20260505 13:30:26.086 | INFO | Baz IO overall: | 20260505 13:30:26.086 | INFO | BytesWritten : 2132296301986 | 20260505 13:30:26.086 | INFO | NumEvents : 1014429501694 | 20260505 13:30:27.734 | INFO | Analyzing chunk frames = [3579904,3580416) | 20260505 13:30:27.734 | INFO | MemInfo: { | 20260505 13:30:27.734 | INFO | "gpu" : " 17 GB", | 20260505 13:30:27.734 | INFO | "huge" : "", | 20260505 13:30:27.734 | INFO | "huge_shared" : " 27 GB", | 20260505 13:30:27.734 | INFO | "malloc" : " 14 GB", | 20260505 13:30:27.734 | INFO | "pinned" : " 29 GB" | 20260505 13:30:27.734 | INFO | } | 20260505 13:30:27.734 | INFO | Perf report for the last 5.13284 seconds: | 20260505 13:30:27.734 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:30:27.734 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:27.734 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:27.734 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:27.734 | INFO | ANALYSIS : 64.4%, 3.0, 99.7ms, 0 | 20260505 13:30:27.734 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:27.734 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 13:30:27.734 | INFO | PRE_HQ : 60.7%, 1.8, 57.5ms, 0 | 20260505 13:30:27.734 | INFO | BAZWRITER : 64.5%, 4.0, 132.4ms, 0 | 20260505 13:30:27.734 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260505 13:30:27.734 | INFO | Uploaded 36GiB to GPU in 2.17668s (16.5389GiB/s) | 20260505 13:30:27.734 | INFO | Downloaded 38GiB to GPU in 2.28165s (16.6546GiB/s) | 20260505 13:30:32.876 | INFO | Analyzing chunk frames = [3580416,3580928) | 20260505 13:30:32.876 | INFO | MemInfo: { | 20260505 13:30:32.876 | INFO | "gpu" : " 17 GB", | 20260505 13:30:32.876 | INFO | "huge" : "", | 20260505 13:30:32.876 | INFO | "huge_shared" : " 27 GB", | 20260505 13:30:32.876 | INFO | "malloc" : " 14 GB", | 20260505 13:30:32.876 | INFO | "pinned" : " 29 GB" | 20260505 13:30:32.876 | INFO | } | 20260505 13:30:32.876 | INFO | Perf report for the last 5.14226 seconds: | 20260505 13:30:32.876 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:30:32.876 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:32.876 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:32.876 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:32.876 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 13:30:32.876 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:32.876 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:32.876 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:30:32.876 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:32.877 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:32.877 | INFO | Uploaded 36GiB to GPU in 2.18652s (16.4646GiB/s) | 20260505 13:30:32.877 | INFO | Downloaded 34GiB to GPU in 2.05425s (16.5511GiB/s) | 20260505 13:30:38.019 | INFO | Analyzing chunk frames = [3580928,3581440) | 20260505 13:30:38.019 | INFO | MemInfo: { | 20260505 13:30:38.019 | INFO | "gpu" : " 17 GB", | 20260505 13:30:38.019 | INFO | "huge" : "", | 20260505 13:30:38.019 | INFO | "huge_shared" : " 27 GB", | 20260505 13:30:38.019 | INFO | "malloc" : " 14 GB", | 20260505 13:30:38.019 | INFO | "pinned" : " 29 GB" | 20260505 13:30:38.019 | INFO | } | 20260505 13:30:38.019 | INFO | Perf report for the last 5.1427 seconds: | 20260505 13:30:38.019 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:30:38.019 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:38.019 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:38.019 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:38.019 | INFO | ANALYSIS : 63.2%, 3.0, 98.4ms, 0 | 20260505 13:30:38.019 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:38.019 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:38.019 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 13:30:38.019 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:38.019 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:38.019 | INFO | Uploaded 36GiB to GPU in 2.19911s (16.3702GiB/s) | 20260505 13:30:38.019 | INFO | Downloaded 34GiB to GPU in 2.12346s (16.0116GiB/s) | 20260505 13:30:43.166 | INFO | Analyzing chunk frames = [3581440,3581952) | 20260505 13:30:43.166 | INFO | MemInfo: { | 20260505 13:30:43.166 | INFO | "gpu" : " 17 GB", | 20260505 13:30:43.166 | INFO | "huge" : "", | 20260505 13:30:43.166 | INFO | "huge_shared" : " 27 GB", | 20260505 13:30:43.166 | INFO | "malloc" : " 14 GB", | 20260505 13:30:43.166 | INFO | "pinned" : " 29 GB" | 20260505 13:30:43.166 | INFO | } | 20260505 13:30:43.166 | INFO | Perf report for the last 5.14688 seconds: | 20260505 13:30:43.166 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:30:43.166 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:43.166 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:43.166 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:43.166 | INFO | ANALYSIS : 62.9%, 3.0, 97.9ms, 0 | 20260505 13:30:43.166 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:43.166 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:43.166 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 13:30:43.166 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:43.166 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:43.166 | INFO | Uploaded 36GiB to GPU in 2.20223s (16.3471GiB/s) | 20260505 13:30:43.166 | INFO | Downloaded 34GiB to GPU in 2.13362s (15.9354GiB/s) | 20260505 13:30:48.288 | INFO | Analyzing chunk frames = [3581952,3582464) | 20260505 13:30:48.288 | INFO | MemInfo: { | 20260505 13:30:48.288 | INFO | "gpu" : " 17 GB", | 20260505 13:30:48.288 | INFO | "huge" : "", | 20260505 13:30:48.288 | INFO | "huge_shared" : " 27 GB", | 20260505 13:30:48.288 | INFO | "malloc" : " 14 GB", | 20260505 13:30:48.288 | INFO | "pinned" : " 29 GB" | 20260505 13:30:48.288 | INFO | } | 20260505 13:30:48.288 | INFO | Perf report for the last 5.12244 seconds: | 20260505 13:30:48.288 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:30:48.288 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:48.288 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:48.288 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:48.289 | INFO | ANALYSIS : 62.9%, 3.0, 97.9ms, 0 | 20260505 13:30:48.289 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:48.289 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:48.289 | INFO | PRE_HQ : 28.7%, 1.0, 14.8ms, 0 | 20260505 13:30:48.289 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:48.289 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:48.289 | INFO | Uploaded 36GiB to GPU in 2.13811s (16.8373GiB/s) | 20260505 13:30:48.289 | INFO | Downloaded 34GiB to GPU in 2.12068s (16.0326GiB/s) | 20260505 13:30:53.428 | INFO | Analyzing chunk frames = [3582464,3582976) | 20260505 13:30:53.428 | INFO | MemInfo: { | 20260505 13:30:53.428 | INFO | "gpu" : " 17 GB", | 20260505 13:30:53.428 | INFO | "huge" : "", | 20260505 13:30:53.428 | INFO | "huge_shared" : " 27 GB", | 20260505 13:30:53.429 | INFO | "malloc" : " 14 GB", | 20260505 13:30:53.429 | INFO | "pinned" : " 29 GB" | 20260505 13:30:53.429 | INFO | } | 20260505 13:30:53.429 | INFO | Perf report for the last 5.1401 seconds: | 20260505 13:30:53.429 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:30:53.429 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:53.429 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:53.429 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:53.429 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 13:30:53.429 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:53.429 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:53.429 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:30:53.429 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:53.429 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:53.429 | INFO | Uploaded 36GiB to GPU in 2.1586s (16.6774GiB/s) | 20260505 13:30:53.429 | INFO | Downloaded 34GiB to GPU in 2.10933s (16.1189GiB/s) | 20260505 13:30:58.576 | INFO | Analyzing chunk frames = [3582976,3583488) | 20260505 13:30:58.576 | INFO | MemInfo: { | 20260505 13:30:58.576 | INFO | "gpu" : " 17 GB", | 20260505 13:30:58.576 | INFO | "huge" : "", | 20260505 13:30:58.577 | INFO | "huge_shared" : " 27 GB", | 20260505 13:30:58.577 | INFO | "malloc" : " 14 GB", | 20260505 13:30:58.577 | INFO | "pinned" : " 29 GB" | 20260505 13:30:58.577 | INFO | } | 20260505 13:30:58.577 | INFO | Perf report for the last 5.14804 seconds: | 20260505 13:30:58.577 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:30:58.577 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:58.577 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:58.577 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:58.577 | INFO | ANALYSIS : 63.1%, 3.0, 98.7ms, 0 | 20260505 13:30:58.577 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:58.577 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:58.577 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 13:30:58.577 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:58.577 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:30:58.577 | INFO | Uploaded 36GiB to GPU in 2.17302s (16.5668GiB/s) | 20260505 13:30:58.577 | INFO | Downloaded 34GiB to GPU in 2.15139s (15.8037GiB/s) | 20260505 13:31:03.713 | INFO | Analyzing chunk frames = [3583488,3584000) | 20260505 13:31:03.713 | INFO | MemInfo: { | 20260505 13:31:03.713 | INFO | "gpu" : " 17 GB", | 20260505 13:31:03.713 | INFO | "huge" : "", | 20260505 13:31:03.713 | INFO | "huge_shared" : " 27 GB", | 20260505 13:31:03.713 | INFO | "malloc" : " 14 GB", | 20260505 13:31:03.713 | INFO | "pinned" : " 29 GB" | 20260505 13:31:03.713 | INFO | } | 20260505 13:31:03.713 | INFO | Perf report for the last 5.13686 seconds: | 20260505 13:31:03.713 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:31:03.713 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:03.713 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:03.714 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:03.714 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 13:31:03.714 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:03.714 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:03.714 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 13:31:03.714 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:03.714 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:03.714 | INFO | Uploaded 36GiB to GPU in 2.21076s (16.284GiB/s) | 20260505 13:31:03.714 | INFO | Downloaded 34GiB to GPU in 2.10621s (16.1427GiB/s) | 20260505 13:31:03.853 | INFO | Starting a round of Baz IO | 20260505 13:31:07.070 | INFO | Populating RTMetrics report | 20260505 13:31:07.070 | INFO | RTMetrics report populated | 20260505 13:31:07.070 | INFO | Opening tmp RTMetrics json file | 20260505 13:31:07.111 | INFO | Serializing RTMetrics to json | 20260505 13:31:07.117 | INFO | Serialization done, swapping tmp file now... | 20260505 13:31:07.130 | INFO | RTMetrics done | 20260505 13:31:07.171 | INFO | Finished a round of Baz IO: Wrote 2.295 GiB in 3.317 seconds | 20260505 13:31:07.171 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.09% / 0.00% / 76.90% | 20260505 13:31:07.171 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.690381% / 3.97364e-06% / 5.16574e-05% | 20260505 13:31:07.171 | INFO | Baz IO overall: | 20260505 13:31:07.171 | INFO | BytesWritten : 2134760296441 | 20260505 13:31:07.171 | INFO | NumEvents : 1015594680915 | 20260505 13:31:08.849 | INFO | Analyzing chunk frames = [3584000,3584512) | 20260505 13:31:08.849 | INFO | MemInfo: { | 20260505 13:31:08.849 | INFO | "gpu" : " 17 GB", | 20260505 13:31:08.849 | INFO | "huge" : "", | 20260505 13:31:08.849 | INFO | "huge_shared" : " 27 GB", | 20260505 13:31:08.849 | INFO | "malloc" : " 14 GB", | 20260505 13:31:08.849 | INFO | "pinned" : " 29 GB" | 20260505 13:31:08.849 | INFO | } | 20260505 13:31:08.849 | INFO | Perf report for the last 5.13558 seconds: | 20260505 13:31:08.849 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:31:08.849 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:08.849 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:08.849 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:08.849 | INFO | ANALYSIS : 64.2%, 3.0, 99.4ms, 0 | 20260505 13:31:08.849 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:08.849 | INFO | RT_METRICS : 46.5%, 1.0, 24.3ms, 0 | 20260505 13:31:08.849 | INFO | PRE_HQ : 61.0%, 1.8, 57.8ms, 0 | 20260505 13:31:08.849 | INFO | BAZWRITER : 63.9%, 4.1, 133.8ms, 0 | 20260505 13:31:08.849 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 13:31:08.849 | INFO | Uploaded 36GiB to GPU in 2.17855s (16.5247GiB/s) | 20260505 13:31:08.849 | INFO | Downloaded 38GiB to GPU in 2.27139s (16.7299GiB/s) | 20260505 13:31:13.985 | INFO | Analyzing chunk frames = [3584512,3585024) | 20260505 13:31:13.985 | INFO | MemInfo: { | 20260505 13:31:13.985 | INFO | "gpu" : " 17 GB", | 20260505 13:31:13.985 | INFO | "huge" : "", | 20260505 13:31:13.985 | INFO | "huge_shared" : " 27 GB", | 20260505 13:31:13.985 | INFO | "malloc" : " 14 GB", | 20260505 13:31:13.985 | INFO | "pinned" : " 29 GB" | 20260505 13:31:13.985 | INFO | } | 20260505 13:31:13.985 | INFO | Perf report for the last 5.13627 seconds: | 20260505 13:31:13.985 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:31:13.985 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:13.985 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:13.985 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:13.985 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 13:31:13.985 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:13.985 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:13.985 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:31:13.985 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:13.985 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:13.986 | INFO | Uploaded 36GiB to GPU in 2.18665s (16.4635GiB/s) | 20260505 13:31:13.986 | INFO | Downloaded 34GiB to GPU in 2.05272s (16.5634GiB/s) | 20260505 13:31:19.130 | INFO | Analyzing chunk frames = [3585024,3585536) | 20260505 13:31:19.130 | INFO | MemInfo: { | 20260505 13:31:19.130 | INFO | "gpu" : " 17 GB", | 20260505 13:31:19.130 | INFO | "huge" : "", | 20260505 13:31:19.130 | INFO | "huge_shared" : " 27 GB", | 20260505 13:31:19.130 | INFO | "malloc" : " 14 GB", | 20260505 13:31:19.130 | INFO | "pinned" : " 29 GB" | 20260505 13:31:19.131 | INFO | } | 20260505 13:31:19.131 | INFO | Perf report for the last 5.14521 seconds: | 20260505 13:31:19.131 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:31:19.131 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:19.131 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:19.131 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:19.131 | INFO | ANALYSIS : 63.1%, 3.0, 98.2ms, 0 | 20260505 13:31:19.131 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:19.131 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:19.131 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:31:19.131 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:19.131 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:19.131 | INFO | Uploaded 36GiB to GPU in 2.19889s (16.3719GiB/s) | 20260505 13:31:19.131 | INFO | Downloaded 34GiB to GPU in 2.11959s (16.0408GiB/s) | 20260505 13:31:24.273 | INFO | Analyzing chunk frames = [3585536,3586048) | 20260505 13:31:24.273 | INFO | MemInfo: { | 20260505 13:31:24.273 | INFO | "gpu" : " 17 GB", | 20260505 13:31:24.273 | INFO | "huge" : "", | 20260505 13:31:24.273 | INFO | "huge_shared" : " 27 GB", | 20260505 13:31:24.273 | INFO | "malloc" : " 14 GB", | 20260505 13:31:24.273 | INFO | "pinned" : " 29 GB" | 20260505 13:31:24.273 | INFO | } | 20260505 13:31:24.273 | INFO | Perf report for the last 5.14287 seconds: | 20260505 13:31:24.273 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:31:24.273 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:24.273 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:24.274 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:24.274 | INFO | ANALYSIS : 62.8%, 3.0, 97.6ms, 0 | 20260505 13:31:24.274 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:24.274 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:24.274 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:31:24.274 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:24.274 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:24.274 | INFO | Uploaded 36GiB to GPU in 2.19044s (16.435GiB/s) | 20260505 13:31:24.274 | INFO | Downloaded 34GiB to GPU in 2.13352s (15.9361GiB/s) | 20260505 13:31:29.419 | INFO | Analyzing chunk frames = [3586048,3586560) | 20260505 13:31:29.419 | INFO | MemInfo: { | 20260505 13:31:29.419 | INFO | "gpu" : " 17 GB", | 20260505 13:31:29.419 | INFO | "huge" : "", | 20260505 13:31:29.419 | INFO | "huge_shared" : " 27 GB", | 20260505 13:31:29.419 | INFO | "malloc" : " 14 GB", | 20260505 13:31:29.419 | INFO | "pinned" : " 29 GB" | 20260505 13:31:29.419 | INFO | } | 20260505 13:31:29.420 | INFO | Perf report for the last 5.14606 seconds: | 20260505 13:31:29.420 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:31:29.420 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:29.420 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:29.420 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:29.420 | INFO | ANALYSIS : 62.7%, 3.0, 98.0ms, 0 | 20260505 13:31:29.420 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:29.420 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:29.420 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 13:31:29.420 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:29.420 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:29.420 | INFO | Uploaded 36GiB to GPU in 2.16315s (16.6424GiB/s) | 20260505 13:31:29.420 | INFO | Downloaded 34GiB to GPU in 2.12296s (16.0154GiB/s) | 20260505 13:31:34.546 | INFO | Analyzing chunk frames = [3586560,3587072) | 20260505 13:31:34.546 | INFO | MemInfo: { | 20260505 13:31:34.546 | INFO | "gpu" : " 17 GB", | 20260505 13:31:34.546 | INFO | "huge" : "", | 20260505 13:31:34.546 | INFO | "huge_shared" : " 27 GB", | 20260505 13:31:34.546 | INFO | "malloc" : " 14 GB", | 20260505 13:31:34.546 | INFO | "pinned" : " 29 GB" | 20260505 13:31:34.546 | INFO | } | 20260505 13:31:34.546 | INFO | Perf report for the last 5.12682 seconds: | 20260505 13:31:34.546 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:31:34.546 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:34.546 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:34.546 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:34.546 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 13:31:34.546 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:34.546 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:34.546 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 13:31:34.547 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:34.547 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:34.547 | INFO | Uploaded 36GiB to GPU in 2.20483s (16.3278GiB/s) | 20260505 13:31:34.547 | INFO | Downloaded 34GiB to GPU in 2.11196s (16.0988GiB/s) | 20260505 13:31:39.683 | INFO | Analyzing chunk frames = [3587072,3587584) | 20260505 13:31:39.683 | INFO | MemInfo: { | 20260505 13:31:39.683 | INFO | "gpu" : " 17 GB", | 20260505 13:31:39.683 | INFO | "huge" : "", | 20260505 13:31:39.683 | INFO | "huge_shared" : " 27 GB", | 20260505 13:31:39.683 | INFO | "malloc" : " 14 GB", | 20260505 13:31:39.684 | INFO | "pinned" : " 29 GB" | 20260505 13:31:39.684 | INFO | } | 20260505 13:31:39.684 | INFO | Perf report for the last 5.13719 seconds: | 20260505 13:31:39.684 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:31:39.684 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:39.684 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:39.684 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:39.684 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 13:31:39.684 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:39.684 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:39.684 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 13:31:39.684 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:39.684 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:39.684 | INFO | Uploaded 36GiB to GPU in 2.18452s (16.4796GiB/s) | 20260505 13:31:39.684 | INFO | Downloaded 34GiB to GPU in 2.13921s (15.8937GiB/s) | 20260505 13:31:44.831 | INFO | Analyzing chunk frames = [3587584,3588096) | 20260505 13:31:44.831 | INFO | MemInfo: { | 20260505 13:31:44.831 | INFO | "gpu" : " 17 GB", | 20260505 13:31:44.831 | INFO | "huge" : "", | 20260505 13:31:44.832 | INFO | "huge_shared" : " 27 GB", | 20260505 13:31:44.832 | INFO | "malloc" : " 14 GB", | 20260505 13:31:44.832 | INFO | "pinned" : " 29 GB" | 20260505 13:31:44.832 | INFO | } | 20260505 13:31:44.832 | INFO | Perf report for the last 5.14814 seconds: | 20260505 13:31:44.832 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:31:44.832 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:44.832 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:44.832 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:44.832 | INFO | ANALYSIS : 63.5%, 3.0, 99.3ms, 0 | 20260505 13:31:44.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:44.832 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:44.832 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 13:31:44.832 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:44.832 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:44.832 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 13:31:44.832 | INFO | Total mem usage: 15.3 GiB | 20260505 13:31:44.832 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 13:31:44.832 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -7.15%) | 20260505 13:31:44.832 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 13:31:44.832 | INFO | High Water Breakdown: | 20260505 13:31:44.832 | INFO | 61.1% : 8.74 GiB ( 91.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 13:31:44.832 | INFO | 38.7% : 5.54 GiB ( 83.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 13:31:44.832 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 13:31:44.832 | INFO | ------------------------------------------ | 20260505 13:31:44.832 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:31:44.832 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 13:31:44.832 | INFO | Total mem usage: 26.5 GiB | 20260505 13:31:44.832 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 13:31:44.832 | INFO | Uploaded 36GiB to GPU in 2.16879s (16.5991GiB/s) | 20260505 13:31:44.832 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 13:31:44.832 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 13:31:44.832 | INFO | High Water Breakdown: | 20260505 13:31:44.832 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:31:44.832 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:31:44.832 | INFO | Downloaded 34GiB to GPU in 2.10426s (16.1577GiB/s) | 20260505 13:31:44.832 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:31:44.832 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:31:44.832 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:31:44.832 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:31:44.832 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 13:31:44.832 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 13:31:44.833 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:31:44.833 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 13:31:44.833 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 13:31:44.833 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 13:31:44.833 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 13:31:44.833 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 13:31:44.833 | INFO | ------------------------------------------ | 20260505 13:31:44.833 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:31:44.833 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 13:31:44.833 | INFO | Total mem usage: 25.6 GiB | 20260505 13:31:44.833 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 13:31:44.833 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 13:31:44.833 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 13:31:44.833 | INFO | High Water Breakdown: | 20260505 13:31:44.833 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 13:31:44.833 | INFO | ------------------------------------------ | 20260505 13:31:44.833 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:31:44.833 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 13:31:44.833 | INFO | Total mem usage: 15.7 GiB | 20260505 13:31:44.833 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 13:31:44.833 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 13:31:44.833 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 13:31:44.833 | INFO | High Water Breakdown: | 20260505 13:31:44.833 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 13:31:44.833 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 13:31:44.833 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 13:31:44.833 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 13:31:44.833 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 13:31:44.833 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 13:31:44.833 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 13:31:44.833 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 13:31:44.833 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 13:31:44.833 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:31:44.833 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 13:31:44.833 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:31:44.833 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 13:31:44.833 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 13:31:44.833 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 13:31:44.833 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 13:31:44.833 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:31:44.833 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 13:31:44.833 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:31:44.833 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:31:44.833 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:31:44.833 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 13:31:44.833 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 13:31:44.833 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:31:44.833 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 13:31:44.833 | INFO | ------------------------------------------ | 20260505 13:31:44.833 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:31:44.833 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 13:31:44.833 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:31:44.833 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:31:44.833 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.68e+01, 1.37e+01 / 1.58e+00 | 20260505 13:31:44.834 | INFO | PerfReport: 0.29%, AggregateStats -- 9.30e-02 / 1.00e-01, 1.09e-02 / 3.56e-03 | 20260505 13:31:44.834 | INFO | PerfReport: 11.77%, Baseline -- 3.76e+00 / 4.47e+00, 5.97e-01 / 2.20e-01 | 20260505 13:31:44.834 | INFO | PerfReport: 3.31%, Histogram -- 1.06e+00 / 1.50e+00, 2.04e-01 / 8.24e-02 | 20260505 13:31:44.834 | INFO | PerfReport: 15.92%, DME -- 4.06e+01 / 4.56e+01, 4.69e+00 / 1.91e+00 | 20260505 13:31:44.834 | INFO | PerfReport: 56.46%, FrameLabeling -- 1.80e+01 / 2.03e+01, 1.48e+00 / 1.92e+00 | 20260505 13:31:44.834 | INFO | PerfReport: 7.41%, PulseAccumulating -- 2.37e+00 / 2.89e+00, 4.48e-01 / 2.02e-01 | 20260505 13:31:44.834 | INFO | PerfReport: 3.58%, Metrics -- 1.14e+00 / 1.47e+00, 3.22e-01 / 1.68e-01 | 20260505 13:31:44.834 | INFO | PerfReport: 1.25%, PulseSerializing -- 4.00e-01 / 4.54e-01, 7.23e-02 / 3.09e-02 | 20260505 13:31:44.834 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 13:31:44.834 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:31:44.834 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:31:44.834 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260505 13:31:44.834 | INFO | PerfReport: 22.56%, Upload -- 2.27e+01 / 0.00e+00, 8.06e+00 / 0.00e+00 | 20260505 13:31:44.834 | INFO | PerfReport: 22.06%, Download -- 2.22e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260505 13:31:44.834 | INFO | PerfReport: 31.68%, Compute -- 3.19e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 13:31:44.834 | INFO | PerfReport: 2.12%, LockUpload -- 2.13e+00 / 0.00e+00, 7.69e+00 / 0.00e+00 | 20260505 13:31:44.834 | INFO | PerfReport: 1.44%, LockDownload -- 1.45e+00 / 0.00e+00, 2.99e+00 / 0.00e+00 | 20260505 13:31:44.834 | INFO | PerfReport: 20.13%, LockCompute -- 2.03e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260505 13:31:44.834 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.49e-02 / 0.00e+00 | 20260505 13:31:44.969 | INFO | Starting a round of Baz IO | 20260505 13:31:48.194 | INFO | Populating RTMetrics report | 20260505 13:31:48.194 | INFO | RTMetrics report populated | 20260505 13:31:48.194 | INFO | Opening tmp RTMetrics json file | 20260505 13:31:48.215 | INFO | Serializing RTMetrics to json | 20260505 13:31:48.257 | INFO | Serialization done, swapping tmp file now... | 20260505 13:31:48.281 | INFO | RTMetrics done | 20260505 13:31:48.297 | INFO | Finished a round of Baz IO: Wrote 2.293 GiB in 3.328 seconds | 20260505 13:31:48.297 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.09% / 0.00% / 76.91% | 20260505 13:31:48.298 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.686594% / 1.58946e-05% / 6.75519e-05% | 20260505 13:31:48.298 | INFO | Baz IO overall: | 20260505 13:31:48.298 | INFO | BytesWritten : 2137222852612 | 20260505 13:31:48.298 | INFO | NumEvents : 1016758737122 | 20260505 13:31:49.970 | INFO | Analyzing chunk frames = [3588096,3588608) | 20260505 13:31:49.970 | INFO | MemInfo: { | 20260505 13:31:49.970 | INFO | "gpu" : " 17 GB", | 20260505 13:31:49.970 | INFO | "huge" : "", | 20260505 13:31:49.970 | INFO | "huge_shared" : " 27 GB", | 20260505 13:31:49.970 | INFO | "malloc" : " 14 GB", | 20260505 13:31:49.970 | INFO | "pinned" : " 29 GB" | 20260505 13:31:49.970 | INFO | } | 20260505 13:31:49.970 | INFO | Perf report for the last 5.1386 seconds: | 20260505 13:31:49.970 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:31:49.970 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:49.970 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:49.970 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:49.970 | INFO | ANALYSIS : 64.3%, 3.0, 99.6ms, 0 | 20260505 13:31:49.970 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:49.971 | INFO | RT_METRICS : 46.8%, 1.0, 24.5ms, 0 | 20260505 13:31:49.971 | INFO | PRE_HQ : 60.9%, 1.8, 57.6ms, 0 | 20260505 13:31:49.971 | INFO | BAZWRITER : 64.1%, 4.0, 132.8ms, 0 | 20260505 13:31:49.971 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260505 13:31:49.971 | INFO | Uploaded 36GiB to GPU in 2.18503s (16.4757GiB/s) | 20260505 13:31:49.971 | INFO | Downloaded 38GiB to GPU in 2.26859s (16.7505GiB/s) | 20260505 13:31:55.109 | INFO | Analyzing chunk frames = [3588608,3589120) | 20260505 13:31:55.109 | INFO | MemInfo: { | 20260505 13:31:55.109 | INFO | "gpu" : " 17 GB", | 20260505 13:31:55.109 | INFO | "huge" : "", | 20260505 13:31:55.109 | INFO | "huge_shared" : " 27 GB", | 20260505 13:31:55.109 | INFO | "malloc" : " 14 GB", | 20260505 13:31:55.109 | INFO | "pinned" : " 29 GB" | 20260505 13:31:55.109 | INFO | } | 20260505 13:31:55.109 | INFO | Perf report for the last 5.13891 seconds: | 20260505 13:31:55.109 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:31:55.109 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:55.109 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:55.109 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:55.109 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 13:31:55.109 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:55.109 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:55.109 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:31:55.109 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:55.109 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:31:55.109 | INFO | Uploaded 36GiB to GPU in 2.17112s (16.5813GiB/s) | 20260505 13:31:55.110 | INFO | Downloaded 34GiB to GPU in 2.03514s (16.7064GiB/s) | 20260505 13:32:00.239 | INFO | Analyzing chunk frames = [3589120,3589632) | 20260505 13:32:00.239 | INFO | MemInfo: { | 20260505 13:32:00.239 | INFO | "gpu" : " 17 GB", | 20260505 13:32:00.239 | INFO | "huge" : "", | 20260505 13:32:00.239 | INFO | "huge_shared" : " 27 GB", | 20260505 13:32:00.239 | INFO | "malloc" : " 14 GB", | 20260505 13:32:00.239 | INFO | "pinned" : " 29 GB" | 20260505 13:32:00.239 | INFO | } | 20260505 13:32:00.239 | INFO | Perf report for the last 5.13017 seconds: | 20260505 13:32:00.239 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:32:00.239 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:00.239 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:00.240 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:00.240 | INFO | ANALYSIS : 63.4%, 3.0, 98.5ms, 0 | 20260505 13:32:00.240 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:00.240 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:00.240 | INFO | PRE_HQ : 28.5%, 1.0, 14.7ms, 0 | 20260505 13:32:00.240 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:00.240 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:00.240 | INFO | Uploaded 36GiB to GPU in 2.22054s (16.2123GiB/s) | 20260505 13:32:00.240 | INFO | Downloaded 34GiB to GPU in 2.11466s (16.0782GiB/s) | 20260505 13:32:05.389 | INFO | Analyzing chunk frames = [3589632,3590144) | 20260505 13:32:05.389 | INFO | MemInfo: { | 20260505 13:32:05.389 | INFO | "gpu" : " 17 GB", | 20260505 13:32:05.389 | INFO | "huge" : "", | 20260505 13:32:05.389 | INFO | "huge_shared" : " 27 GB", | 20260505 13:32:05.389 | INFO | "malloc" : " 14 GB", | 20260505 13:32:05.389 | INFO | "pinned" : " 29 GB" | 20260505 13:32:05.389 | INFO | } | 20260505 13:32:05.389 | INFO | Perf report for the last 5.14993 seconds: | 20260505 13:32:05.390 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:32:05.390 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:05.390 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:05.390 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:05.390 | INFO | ANALYSIS : 62.9%, 3.0, 98.0ms, 0 | 20260505 13:32:05.390 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:05.390 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:05.390 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 13:32:05.390 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:05.390 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:05.390 | INFO | Uploaded 36GiB to GPU in 2.20235s (16.3462GiB/s) | 20260505 13:32:05.390 | INFO | Downloaded 34GiB to GPU in 2.13658s (15.9133GiB/s) | 20260505 13:32:10.527 | INFO | Analyzing chunk frames = [3590144,3590656) | 20260505 13:32:10.527 | INFO | MemInfo: { | 20260505 13:32:10.527 | INFO | "gpu" : " 17 GB", | 20260505 13:32:10.527 | INFO | "huge" : "", | 20260505 13:32:10.527 | INFO | "huge_shared" : " 27 GB", | 20260505 13:32:10.527 | INFO | "malloc" : " 14 GB", | 20260505 13:32:10.527 | INFO | "pinned" : " 29 GB" | 20260505 13:32:10.527 | INFO | } | 20260505 13:32:10.527 | INFO | Perf report for the last 5.13803 seconds: | 20260505 13:32:10.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:32:10.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:10.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:10.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:10.528 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 13:32:10.528 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:10.528 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:10.528 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:32:10.528 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:10.528 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:10.528 | INFO | Uploaded 36GiB to GPU in 2.1446s (16.7864GiB/s) | 20260505 13:32:10.528 | INFO | Downloaded 34GiB to GPU in 2.1232s (16.0136GiB/s) | 20260505 13:32:15.662 | INFO | Analyzing chunk frames = [3590656,3591168) | 20260505 13:32:15.662 | INFO | MemInfo: { | 20260505 13:32:15.662 | INFO | "gpu" : " 17 GB", | 20260505 13:32:15.662 | INFO | "huge" : "", | 20260505 13:32:15.662 | INFO | "huge_shared" : " 27 GB", | 20260505 13:32:15.662 | INFO | "malloc" : " 14 GB", | 20260505 13:32:15.662 | INFO | "pinned" : " 29 GB" | 20260505 13:32:15.662 | INFO | } | 20260505 13:32:15.662 | INFO | Perf report for the last 5.13453 seconds: | 20260505 13:32:15.662 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:32:15.662 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:15.662 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:15.662 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:15.662 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 13:32:15.662 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:15.662 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:15.662 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 13:32:15.662 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:15.662 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:15.662 | INFO | Uploaded 36GiB to GPU in 2.18728s (16.4588GiB/s) | 20260505 13:32:15.662 | INFO | Downloaded 34GiB to GPU in 2.11381s (16.0847GiB/s) | 20260505 13:32:20.793 | INFO | Analyzing chunk frames = [3591168,3591680) | 20260505 13:32:20.793 | INFO | MemInfo: { | 20260505 13:32:20.793 | INFO | "gpu" : " 17 GB", | 20260505 13:32:20.793 | INFO | "huge" : "", | 20260505 13:32:20.793 | INFO | "huge_shared" : " 27 GB", | 20260505 13:32:20.793 | INFO | "malloc" : " 14 GB", | 20260505 13:32:20.793 | INFO | "pinned" : " 29 GB" | 20260505 13:32:20.793 | INFO | } | 20260505 13:32:20.793 | INFO | Perf report for the last 5.13098 seconds: | 20260505 13:32:20.793 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:32:20.793 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:20.793 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:20.793 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:20.793 | INFO | ANALYSIS : 63.4%, 3.0, 98.8ms, 0 | 20260505 13:32:20.793 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:20.793 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:20.793 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 13:32:20.793 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:20.793 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:20.794 | INFO | Uploaded 36GiB to GPU in 2.16954s (16.5934GiB/s) | 20260505 13:32:20.794 | INFO | Downloaded 34GiB to GPU in 2.15304s (15.7916GiB/s) | 20260505 13:32:25.935 | INFO | Analyzing chunk frames = [3591680,3592192) | 20260505 13:32:25.935 | INFO | MemInfo: { | 20260505 13:32:25.935 | INFO | "gpu" : " 17 GB", | 20260505 13:32:25.935 | INFO | "huge" : "", | 20260505 13:32:25.935 | INFO | "huge_shared" : " 27 GB", | 20260505 13:32:25.935 | INFO | "malloc" : " 14 GB", | 20260505 13:32:25.935 | INFO | "pinned" : " 29 GB" | 20260505 13:32:25.935 | INFO | } | 20260505 13:32:25.935 | INFO | Perf report for the last 5.14203 seconds: | 20260505 13:32:25.935 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:32:25.935 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:25.935 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:25.935 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:25.935 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 13:32:25.935 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:25.935 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:25.935 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 13:32:25.935 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:25.935 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:25.935 | INFO | Uploaded 36GiB to GPU in 2.22177s (16.2033GiB/s) | 20260505 13:32:25.935 | INFO | Downloaded 34GiB to GPU in 2.10975s (16.1157GiB/s) | 20260505 13:32:26.065 | INFO | Starting a round of Baz IO | 20260505 13:32:29.308 | INFO | Populating RTMetrics report | 20260505 13:32:29.309 | INFO | RTMetrics report populated | 20260505 13:32:29.309 | INFO | Opening tmp RTMetrics json file | 20260505 13:32:29.324 | INFO | Serializing RTMetrics to json | 20260505 13:32:29.325 | INFO | Serialization done, swapping tmp file now... | 20260505 13:32:29.325 | INFO | RTMetrics done | 20260505 13:32:29.447 | INFO | Finished a round of Baz IO: Wrote 2.294 GiB in 3.381 seconds | 20260505 13:32:29.447 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.08% / 0.00% / 76.91% | 20260505 13:32:29.447 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.68839% / 0% / 7.54992e-05% | 20260505 13:32:29.447 | INFO | Baz IO overall: | 20260505 13:32:29.447 | INFO | BytesWritten : 2139685523518 | 20260505 13:32:29.447 | INFO | NumEvents : 1017923084780 | 20260505 13:32:31.079 | INFO | Analyzing chunk frames = [3592192,3592704) | 20260505 13:32:31.079 | INFO | MemInfo: { | 20260505 13:32:31.079 | INFO | "gpu" : " 17 GB", | 20260505 13:32:31.079 | INFO | "huge" : "", | 20260505 13:32:31.079 | INFO | "huge_shared" : " 27 GB", | 20260505 13:32:31.079 | INFO | "malloc" : " 14 GB", | 20260505 13:32:31.079 | INFO | "pinned" : " 29 GB" | 20260505 13:32:31.079 | INFO | } | 20260505 13:32:31.079 | INFO | Perf report for the last 5.14395 seconds: | 20260505 13:32:31.079 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:32:31.079 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:31.079 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:31.079 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:31.079 | INFO | ANALYSIS : 64.5%, 3.0, 100.0ms, 0 | 20260505 13:32:31.079 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:31.079 | INFO | RT_METRICS : 45.5%, 1.0, 23.9ms, 0 | 20260505 13:32:31.079 | INFO | PRE_HQ : 60.9%, 1.8, 58.5ms, 0 | 20260505 13:32:31.079 | INFO | BAZWRITER : 65.0%, 4.0, 135.0ms, 0 | 20260505 13:32:31.079 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 13:32:31.079 | INFO | Uploaded 36GiB to GPU in 2.1926s (16.4189GiB/s) | 20260505 13:32:31.079 | INFO | Downloaded 38GiB to GPU in 2.29552s (16.554GiB/s) | 20260505 13:32:36.228 | INFO | Analyzing chunk frames = [3592704,3593216) | 20260505 13:32:36.228 | INFO | MemInfo: { | 20260505 13:32:36.228 | INFO | "gpu" : " 17 GB", | 20260505 13:32:36.228 | INFO | "huge" : "", | 20260505 13:32:36.228 | INFO | "huge_shared" : " 27 GB", | 20260505 13:32:36.228 | INFO | "malloc" : " 14 GB", | 20260505 13:32:36.228 | INFO | "pinned" : " 29 GB" | 20260505 13:32:36.228 | INFO | } | 20260505 13:32:36.228 | INFO | Perf report for the last 5.14891 seconds: | 20260505 13:32:36.228 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:32:36.228 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:36.228 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:36.228 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:36.228 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 13:32:36.228 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:36.228 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:36.228 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:32:36.228 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:36.228 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:36.228 | INFO | Uploaded 36GiB to GPU in 2.16546s (16.6247GiB/s) | 20260505 13:32:36.228 | INFO | Downloaded 34GiB to GPU in 2.03105s (16.7401GiB/s) | 20260505 13:32:41.370 | INFO | Analyzing chunk frames = [3593216,3593728) | 20260505 13:32:41.370 | INFO | MemInfo: { | 20260505 13:32:41.370 | INFO | "gpu" : " 17 GB", | 20260505 13:32:41.370 | INFO | "huge" : "", | 20260505 13:32:41.370 | INFO | "huge_shared" : " 27 GB", | 20260505 13:32:41.370 | INFO | "malloc" : " 14 GB", | 20260505 13:32:41.370 | INFO | "pinned" : " 29 GB" | 20260505 13:32:41.370 | INFO | } | 20260505 13:32:41.370 | INFO | Perf report for the last 5.14261 seconds: | 20260505 13:32:41.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:32:41.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:41.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:41.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:41.371 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 13:32:41.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:41.371 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:41.371 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 13:32:41.371 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:41.371 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:41.371 | INFO | Uploaded 36GiB to GPU in 2.23354s (16.1179GiB/s) | 20260505 13:32:41.371 | INFO | Downloaded 34GiB to GPU in 2.12446s (16.004GiB/s) | 20260505 13:32:46.496 | INFO | Analyzing chunk frames = [3593728,3594240) | 20260505 13:32:46.496 | INFO | MemInfo: { | 20260505 13:32:46.496 | INFO | "gpu" : " 17 GB", | 20260505 13:32:46.496 | INFO | "huge" : "", | 20260505 13:32:46.496 | INFO | "huge_shared" : " 27 GB", | 20260505 13:32:46.496 | INFO | "malloc" : " 14 GB", | 20260505 13:32:46.496 | INFO | "pinned" : " 29 GB" | 20260505 13:32:46.496 | INFO | } | 20260505 13:32:46.496 | INFO | Perf report for the last 5.1259 seconds: | 20260505 13:32:46.496 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:32:46.496 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:46.496 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:46.496 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:46.496 | INFO | ANALYSIS : 63.0%, 3.0, 97.7ms, 0 | 20260505 13:32:46.496 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:46.497 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:46.497 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 13:32:46.497 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:46.497 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:46.497 | INFO | Uploaded 36GiB to GPU in 2.19406s (16.4079GiB/s) | 20260505 13:32:46.497 | INFO | Downloaded 34GiB to GPU in 2.13191s (15.9481GiB/s) | 20260505 13:32:51.642 | INFO | Analyzing chunk frames = [3594240,3594752) | 20260505 13:32:51.642 | INFO | MemInfo: { | 20260505 13:32:51.642 | INFO | "gpu" : " 17 GB", | 20260505 13:32:51.642 | INFO | "huge" : "", | 20260505 13:32:51.642 | INFO | "huge_shared" : " 27 GB", | 20260505 13:32:51.642 | INFO | "malloc" : " 14 GB", | 20260505 13:32:51.642 | INFO | "pinned" : " 29 GB" | 20260505 13:32:51.642 | INFO | } | 20260505 13:32:51.642 | INFO | Perf report for the last 5.14602 seconds: | 20260505 13:32:51.642 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:32:51.642 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:51.642 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:51.642 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:51.642 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 13:32:51.642 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:51.643 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:51.643 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:32:51.643 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:51.643 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:51.643 | INFO | Uploaded 36GiB to GPU in 2.16449s (16.6321GiB/s) | 20260505 13:32:51.643 | INFO | Downloaded 34GiB to GPU in 2.12628s (15.9904GiB/s) | 20260505 13:32:56.779 | INFO | Analyzing chunk frames = [3594752,3595264) | 20260505 13:32:56.779 | INFO | MemInfo: { | 20260505 13:32:56.779 | INFO | "gpu" : " 17 GB", | 20260505 13:32:56.779 | INFO | "huge" : "", | 20260505 13:32:56.779 | INFO | "huge_shared" : " 27 GB", | 20260505 13:32:56.779 | INFO | "malloc" : " 14 GB", | 20260505 13:32:56.779 | INFO | "pinned" : " 29 GB" | 20260505 13:32:56.779 | INFO | } | 20260505 13:32:56.779 | INFO | Perf report for the last 5.13625 seconds: | 20260505 13:32:56.779 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:32:56.779 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:56.779 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:56.779 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:56.779 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 13:32:56.779 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:56.779 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:56.779 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 13:32:56.779 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:56.779 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:32:56.779 | INFO | Uploaded 36GiB to GPU in 2.18563s (16.4713GiB/s) | 20260505 13:32:56.779 | INFO | Downloaded 34GiB to GPU in 2.11188s (16.0994GiB/s) | 20260505 13:33:01.920 | INFO | Analyzing chunk frames = [3595264,3595776) | 20260505 13:33:01.920 | INFO | MemInfo: { | 20260505 13:33:01.920 | INFO | "gpu" : " 17 GB", | 20260505 13:33:01.920 | INFO | "huge" : "", | 20260505 13:33:01.920 | INFO | "huge_shared" : " 27 GB", | 20260505 13:33:01.920 | INFO | "malloc" : " 14 GB", | 20260505 13:33:01.920 | INFO | "pinned" : " 29 GB" | 20260505 13:33:01.920 | INFO | } | 20260505 13:33:01.920 | INFO | Perf report for the last 5.14177 seconds: | 20260505 13:33:01.921 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:33:01.921 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:01.921 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:01.921 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:01.921 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 13:33:01.921 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:01.921 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:01.921 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 13:33:01.921 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:01.921 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:01.921 | INFO | Uploaded 36GiB to GPU in 2.13308s (16.877GiB/s) | 20260505 13:33:01.921 | INFO | Downloaded 34GiB to GPU in 2.16889s (15.6762GiB/s) | 20260505 13:33:07.052 | INFO | Analyzing chunk frames = [3595776,3596288) | 20260505 13:33:07.052 | INFO | MemInfo: { | 20260505 13:33:07.052 | INFO | "gpu" : " 17 GB", | 20260505 13:33:07.052 | INFO | "huge" : "", | 20260505 13:33:07.052 | INFO | "huge_shared" : " 27 GB", | 20260505 13:33:07.052 | INFO | "malloc" : " 14 GB", | 20260505 13:33:07.052 | INFO | "pinned" : " 29 GB" | 20260505 13:33:07.052 | INFO | } | 20260505 13:33:07.052 | INFO | Perf report for the last 5.13135 seconds: | 20260505 13:33:07.052 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:33:07.052 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:07.052 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:07.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:07.052 | INFO | ANALYSIS : 63.7%, 3.0, 99.2ms, 0 | 20260505 13:33:07.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:07.052 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:07.052 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 13:33:07.052 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:07.052 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:07.052 | INFO | Uploaded 36GiB to GPU in 2.16152s (16.655GiB/s) | 20260505 13:33:07.052 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 13:33:07.053 | INFO | Total mem usage: 15.3 GiB | 20260505 13:33:07.053 | INFO | Downloaded 34GiB to GPU in 2.10187s (16.176GiB/s) | 20260505 13:33:07.053 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 13:33:07.053 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -7.48%) | 20260505 13:33:07.053 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 13:33:07.053 | INFO | High Water Breakdown: | 20260505 13:33:07.053 | INFO | 61.1% : 8.71 GiB ( 91.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 13:33:07.053 | INFO | 38.8% : 5.53 GiB ( 83.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 13:33:07.053 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 13:33:07.053 | INFO | ------------------------------------------ | 20260505 13:33:07.053 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:33:07.053 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 13:33:07.053 | INFO | Total mem usage: 26.5 GiB | 20260505 13:33:07.053 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 13:33:07.053 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 13:33:07.053 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 13:33:07.053 | INFO | High Water Breakdown: | 20260505 13:33:07.053 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:33:07.053 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:33:07.053 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:33:07.053 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:33:07.053 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:33:07.053 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:33:07.053 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 13:33:07.053 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 13:33:07.053 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:33:07.053 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 13:33:07.053 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 13:33:07.053 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 13:33:07.053 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 13:33:07.053 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 13:33:07.053 | INFO | ------------------------------------------ | 20260505 13:33:07.053 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:33:07.053 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 13:33:07.053 | INFO | Total mem usage: 25.6 GiB | 20260505 13:33:07.053 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 13:33:07.053 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 13:33:07.053 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 13:33:07.053 | INFO | High Water Breakdown: | 20260505 13:33:07.053 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 13:33:07.053 | INFO | ------------------------------------------ | 20260505 13:33:07.053 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:33:07.053 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 13:33:07.053 | INFO | Total mem usage: 15.7 GiB | 20260505 13:33:07.053 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 13:33:07.053 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 13:33:07.053 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260505 13:33:07.053 | INFO | High Water Breakdown: | 20260505 13:33:07.053 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 13:33:07.053 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 13:33:07.053 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 13:33:07.053 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 13:33:07.053 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 13:33:07.053 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 13:33:07.053 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 13:33:07.053 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 13:33:07.053 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 13:33:07.053 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:33:07.053 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 13:33:07.053 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:33:07.053 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 13:33:07.053 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 13:33:07.053 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 13:33:07.053 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 13:33:07.053 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:33:07.053 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 13:33:07.053 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:33:07.053 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:33:07.053 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:33:07.053 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 13:33:07.053 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 13:33:07.053 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:33:07.054 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 13:33:07.054 | INFO | ------------------------------------------ | 20260505 13:33:07.054 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:33:07.054 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 13:33:07.054 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:33:07.054 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:33:07.054 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.67e+01, 1.37e+01 / 1.38e+00 | 20260505 13:33:07.054 | INFO | PerfReport: 0.29%, AggregateStats -- 9.30e-02 / 1.01e-01, 1.08e-02 / 3.12e-03 | 20260505 13:33:07.054 | INFO | PerfReport: 11.78%, Baseline -- 3.76e+00 / 4.49e+00, 5.93e-01 / 1.84e-01 | 20260505 13:33:07.054 | INFO | PerfReport: 3.31%, Histogram -- 1.06e+00 / 1.50e+00, 2.03e-01 / 8.31e-02 | 20260505 13:33:07.054 | INFO | PerfReport: 15.93%, DME -- 4.06e+01 / 4.58e+01, 4.63e+00 / 1.61e+00 | 20260505 13:33:07.054 | INFO | PerfReport: 56.38%, FrameLabeling -- 1.80e+01 / 2.01e+01, 1.44e+00 / 1.80e+00 | 20260505 13:33:07.054 | INFO | PerfReport: 7.47%, PulseAccumulating -- 2.38e+00 / 2.88e+00, 4.55e-01 / 1.70e-01 | 20260505 13:33:07.054 | INFO | PerfReport: 3.58%, Metrics -- 1.14e+00 / 1.48e+00, 3.22e-01 / 1.78e-01 | 20260505 13:33:07.054 | INFO | PerfReport: 1.25%, PulseSerializing -- 3.99e-01 / 4.50e-01, 7.14e-02 / 2.88e-02 | 20260505 13:33:07.054 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 13:33:07.054 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:33:07.054 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:33:07.054 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260505 13:33:07.054 | INFO | PerfReport: 22.56%, Upload -- 2.27e+01 / 0.00e+00, 8.09e+00 / 0.00e+00 | 20260505 13:33:07.054 | INFO | PerfReport: 22.06%, Download -- 2.22e+01 / 0.00e+00, 8.17e+00 / 0.00e+00 | 20260505 13:33:07.054 | INFO | PerfReport: 31.63%, Compute -- 3.19e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 13:33:07.054 | INFO | PerfReport: 2.19%, LockUpload -- 2.21e+00 / 0.00e+00, 7.88e+00 / 0.00e+00 | 20260505 13:33:07.054 | INFO | PerfReport: 1.43%, LockDownload -- 1.44e+00 / 0.00e+00, 2.98e+00 / 0.00e+00 | 20260505 13:33:07.054 | INFO | PerfReport: 20.11%, LockCompute -- 2.03e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260505 13:33:07.054 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 2.65e-02 / 0.00e+00 | 20260505 13:33:07.186 | INFO | Starting a round of Baz IO | 20260505 13:33:10.423 | INFO | Populating RTMetrics report | 20260505 13:33:10.423 | INFO | RTMetrics report populated | 20260505 13:33:10.423 | INFO | Opening tmp RTMetrics json file | 20260505 13:33:10.445 | INFO | Serializing RTMetrics to json | 20260505 13:33:10.466 | INFO | Serialization done, swapping tmp file now... | 20260505 13:33:10.489 | INFO | RTMetrics done | 20260505 13:33:10.529 | INFO | Finished a round of Baz IO: Wrote 2.294 GiB in 3.342 seconds | 20260505 13:33:10.529 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.08% / 0.00% / 76.92% | 20260505 13:33:10.529 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.685628% / 3.97364e-06% / 8.74201e-05% | 20260505 13:33:10.529 | INFO | Baz IO overall: | 20260505 13:33:10.529 | INFO | BytesWritten : 2142148710634 | 20260505 13:33:10.529 | INFO | NumEvents : 1019087894304 | 20260505 13:33:12.193 | INFO | Analyzing chunk frames = [3596288,3596800) | 20260505 13:33:12.193 | INFO | MemInfo: { | 20260505 13:33:12.193 | INFO | "gpu" : " 17 GB", | 20260505 13:33:12.193 | INFO | "huge" : "", | 20260505 13:33:12.193 | INFO | "huge_shared" : " 27 GB", | 20260505 13:33:12.193 | INFO | "malloc" : " 14 GB", | 20260505 13:33:12.193 | INFO | "pinned" : " 29 GB" | 20260505 13:33:12.193 | INFO | } | 20260505 13:33:12.193 | INFO | Perf report for the last 5.14085 seconds: | 20260505 13:33:12.193 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:33:12.193 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:12.193 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:12.193 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:12.193 | INFO | ANALYSIS : 64.5%, 3.0, 100.1ms, 0 | 20260505 13:33:12.193 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:12.193 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260505 13:33:12.193 | INFO | PRE_HQ : 60.5%, 1.8, 57.6ms, 0 | 20260505 13:33:12.193 | INFO | BAZWRITER : 64.3%, 4.0, 131.0ms, 0 | 20260505 13:33:12.193 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260505 13:33:12.193 | INFO | Uploaded 36GiB to GPU in 2.18931s (16.4435GiB/s) | 20260505 13:33:12.193 | INFO | Downloaded 38GiB to GPU in 2.28756s (16.6116GiB/s) | 20260505 13:33:17.337 | INFO | Analyzing chunk frames = [3596800,3597312) | 20260505 13:33:17.337 | INFO | MemInfo: { | 20260505 13:33:17.337 | INFO | "gpu" : " 17 GB", | 20260505 13:33:17.337 | INFO | "huge" : "", | 20260505 13:33:17.337 | INFO | "huge_shared" : " 27 GB", | 20260505 13:33:17.337 | INFO | "malloc" : " 14 GB", | 20260505 13:33:17.337 | INFO | "pinned" : " 29 GB" | 20260505 13:33:17.337 | INFO | } | 20260505 13:33:17.337 | INFO | Perf report for the last 5.144 seconds: | 20260505 13:33:17.337 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:33:17.337 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:17.337 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:17.337 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:17.337 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 13:33:17.337 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:17.337 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:17.337 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:33:17.337 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:17.337 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:17.337 | INFO | Uploaded 36GiB to GPU in 2.18176s (16.5005GiB/s) | 20260505 13:33:17.337 | INFO | Downloaded 34GiB to GPU in 2.03602s (16.6992GiB/s) | 20260505 13:33:22.482 | INFO | Analyzing chunk frames = [3597312,3597824) | 20260505 13:33:22.482 | INFO | MemInfo: { | 20260505 13:33:22.482 | INFO | "gpu" : " 17 GB", | 20260505 13:33:22.482 | INFO | "huge" : "", | 20260505 13:33:22.482 | INFO | "huge_shared" : " 27 GB", | 20260505 13:33:22.482 | INFO | "malloc" : " 14 GB", | 20260505 13:33:22.482 | INFO | "pinned" : " 29 GB" | 20260505 13:33:22.482 | INFO | } | 20260505 13:33:22.482 | INFO | Perf report for the last 5.14496 seconds: | 20260505 13:33:22.482 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:33:22.482 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:22.482 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:22.482 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:22.482 | INFO | ANALYSIS : 63.2%, 3.0, 98.4ms, 0 | 20260505 13:33:22.482 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:22.482 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:22.482 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:33:22.482 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:22.482 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:22.482 | INFO | Uploaded 36GiB to GPU in 2.22245s (16.1983GiB/s) | 20260505 13:33:22.482 | INFO | Downloaded 34GiB to GPU in 2.09464s (16.2319GiB/s) | 20260505 13:33:27.610 | INFO | Analyzing chunk frames = [3597824,3598336) | 20260505 13:33:27.610 | INFO | MemInfo: { | 20260505 13:33:27.610 | INFO | "gpu" : " 17 GB", | 20260505 13:33:27.610 | INFO | "huge" : "", | 20260505 13:33:27.610 | INFO | "huge_shared" : " 27 GB", | 20260505 13:33:27.610 | INFO | "malloc" : " 14 GB", | 20260505 13:33:27.610 | INFO | "pinned" : " 29 GB" | 20260505 13:33:27.610 | INFO | } | 20260505 13:33:27.610 | INFO | Perf report for the last 5.1286 seconds: | 20260505 13:33:27.610 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:33:27.610 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:27.610 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:27.610 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:27.610 | INFO | ANALYSIS : 63.0%, 3.0, 97.7ms, 0 | 20260505 13:33:27.610 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:27.610 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:27.610 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 13:33:27.610 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:27.611 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:27.611 | INFO | Uploaded 36GiB to GPU in 2.19225s (16.4215GiB/s) | 20260505 13:33:27.611 | INFO | Downloaded 34GiB to GPU in 2.13186s (15.9485GiB/s) | 20260505 13:33:32.744 | INFO | Analyzing chunk frames = [3598336,3598848) | 20260505 13:33:32.744 | INFO | MemInfo: { | 20260505 13:33:32.744 | INFO | "gpu" : " 17 GB", | 20260505 13:33:32.744 | INFO | "huge" : "", | 20260505 13:33:32.744 | INFO | "huge_shared" : " 27 GB", | 20260505 13:33:32.744 | INFO | "malloc" : " 14 GB", | 20260505 13:33:32.744 | INFO | "pinned" : " 29 GB" | 20260505 13:33:32.744 | INFO | } | 20260505 13:33:32.744 | INFO | Perf report for the last 5.1339 seconds: | 20260505 13:33:32.744 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:33:32.744 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:32.744 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:32.744 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:32.744 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 13:33:32.744 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:32.744 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:32.744 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 13:33:32.744 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:32.744 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:32.744 | INFO | Uploaded 36GiB to GPU in 2.18643s (16.4652GiB/s) | 20260505 13:33:32.745 | INFO | Downloaded 34GiB to GPU in 2.1322s (15.946GiB/s) | 20260505 13:33:37.894 | INFO | Analyzing chunk frames = [3598848,3599360) | 20260505 13:33:37.894 | INFO | MemInfo: { | 20260505 13:33:37.894 | INFO | "gpu" : " 17 GB", | 20260505 13:33:37.894 | INFO | "huge" : "", | 20260505 13:33:37.894 | INFO | "huge_shared" : " 27 GB", | 20260505 13:33:37.894 | INFO | "malloc" : " 14 GB", | 20260505 13:33:37.894 | INFO | "pinned" : " 29 GB" | 20260505 13:33:37.894 | INFO | } | 20260505 13:33:37.894 | INFO | Perf report for the last 5.14986 seconds: | 20260505 13:33:37.894 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:33:37.894 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:37.894 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:37.894 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:37.894 | INFO | ANALYSIS : 63.0%, 3.0, 98.6ms, 0 | 20260505 13:33:37.894 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:37.894 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:37.894 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:33:37.894 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:37.894 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:37.894 | INFO | Uploaded 36GiB to GPU in 2.15099s (16.7365GiB/s) | 20260505 13:33:37.894 | INFO | Downloaded 34GiB to GPU in 2.10902s (16.1212GiB/s) | 20260505 13:33:43.037 | INFO | Analyzing chunk frames = [3599360,3599872) | 20260505 13:33:43.037 | INFO | MemInfo: { | 20260505 13:33:43.037 | INFO | "gpu" : " 17 GB", | 20260505 13:33:43.037 | INFO | "huge" : "", | 20260505 13:33:43.037 | INFO | "huge_shared" : " 27 GB", | 20260505 13:33:43.037 | INFO | "malloc" : " 14 GB", | 20260505 13:33:43.037 | INFO | "pinned" : " 29 GB" | 20260505 13:33:43.037 | INFO | } | 20260505 13:33:43.037 | INFO | Perf report for the last 5.14279 seconds: | 20260505 13:33:43.037 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:33:43.037 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:43.037 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:43.037 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:43.037 | INFO | ANALYSIS : 63.2%, 3.0, 98.8ms, 0 | 20260505 13:33:43.037 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:43.037 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:43.037 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 13:33:43.037 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:43.037 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:43.037 | INFO | Uploaded 36GiB to GPU in 2.16948s (16.5938GiB/s) | 20260505 13:33:43.037 | INFO | Downloaded 34GiB to GPU in 2.14846s (15.8253GiB/s) | 20260505 13:33:48.175 | INFO | Analyzing chunk frames = [3599872,3600384) | 20260505 13:33:48.175 | INFO | MemInfo: { | 20260505 13:33:48.175 | INFO | "gpu" : " 17 GB", | 20260505 13:33:48.175 | INFO | "huge" : "", | 20260505 13:33:48.175 | INFO | "huge_shared" : " 27 GB", | 20260505 13:33:48.175 | INFO | "malloc" : " 14 GB", | 20260505 13:33:48.175 | INFO | "pinned" : " 29 GB" | 20260505 13:33:48.175 | INFO | } | 20260505 13:33:48.175 | INFO | Perf report for the last 5.13776 seconds: | 20260505 13:33:48.175 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:33:48.175 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:48.175 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:48.175 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:48.175 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 13:33:48.175 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:48.175 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:48.175 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:33:48.175 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:48.175 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:48.175 | INFO | Uploaded 36GiB to GPU in 2.20954s (16.293GiB/s) | 20260505 13:33:48.175 | INFO | Downloaded 34GiB to GPU in 2.10309s (16.1667GiB/s) | 20260505 13:33:48.309 | INFO | Starting a round of Baz IO | 20260505 13:33:51.547 | INFO | Populating RTMetrics report | 20260505 13:33:51.547 | INFO | RTMetrics report populated | 20260505 13:33:51.547 | INFO | Opening tmp RTMetrics json file | 20260505 13:33:51.590 | INFO | Serializing RTMetrics to json | 20260505 13:33:51.623 | INFO | Serialization done, swapping tmp file now... | 20260505 13:33:51.633 | INFO | RTMetrics done | 20260505 13:33:51.650 | INFO | Finished a round of Baz IO: Wrote 2.295 GiB in 3.341 seconds | 20260505 13:33:51.650 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.07% / 0.00% / 76.92% | 20260505 13:33:51.650 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.684635% / 1.58946e-05% / 6.35783e-05% | 20260505 13:33:51.650 | INFO | Baz IO overall: | 20260505 13:33:51.650 | INFO | BytesWritten : 2144612629795 | 20260505 13:33:51.650 | INFO | NumEvents : 1020253002040 | 20260505 13:33:53.306 | INFO | Analyzing chunk frames = [3600384,3600896) | 20260505 13:33:53.306 | INFO | MemInfo: { | 20260505 13:33:53.306 | INFO | "gpu" : " 17 GB", | 20260505 13:33:53.306 | INFO | "huge" : "", | 20260505 13:33:53.306 | INFO | "huge_shared" : " 27 GB", | 20260505 13:33:53.306 | INFO | "malloc" : " 14 GB", | 20260505 13:33:53.306 | INFO | "pinned" : " 29 GB" | 20260505 13:33:53.306 | INFO | } | 20260505 13:33:53.306 | INFO | Perf report for the last 5.13116 seconds: | 20260505 13:33:53.306 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:33:53.306 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:53.306 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:53.306 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:53.306 | INFO | ANALYSIS : 64.6%, 3.0, 100.1ms, 0 | 20260505 13:33:53.306 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:53.306 | INFO | RT_METRICS : 46.8%, 1.0, 24.5ms, 0 | 20260505 13:33:53.306 | INFO | PRE_HQ : 60.7%, 1.8, 57.5ms, 0 | 20260505 13:33:53.306 | INFO | BAZWRITER : 64.4%, 4.0, 130.9ms, 0 | 20260505 13:33:53.306 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 13:33:53.306 | INFO | Uploaded 36GiB to GPU in 2.17369s (16.5617GiB/s) | 20260505 13:33:53.306 | INFO | Downloaded 38GiB to GPU in 2.27753s (16.6847GiB/s) | 20260505 13:33:58.445 | INFO | Analyzing chunk frames = [3600896,3601408) | 20260505 13:33:58.445 | INFO | MemInfo: { | 20260505 13:33:58.445 | INFO | "gpu" : " 17 GB", | 20260505 13:33:58.445 | INFO | "huge" : "", | 20260505 13:33:58.445 | INFO | "huge_shared" : " 27 GB", | 20260505 13:33:58.445 | INFO | "malloc" : " 14 GB", | 20260505 13:33:58.445 | INFO | "pinned" : " 29 GB" | 20260505 13:33:58.445 | INFO | } | 20260505 13:33:58.445 | INFO | Perf report for the last 5.13943 seconds: | 20260505 13:33:58.445 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:33:58.445 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:58.445 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:58.445 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:58.445 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 13:33:58.445 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:58.445 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:58.445 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:33:58.445 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:58.445 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:33:58.446 | INFO | Uploaded 36GiB to GPU in 2.23275s (16.1236GiB/s) | 20260505 13:33:58.446 | INFO | Downloaded 34GiB to GPU in 2.07808s (16.3613GiB/s) | 20260505 13:34:03.593 | INFO | Analyzing chunk frames = [3601408,3601920) | 20260505 13:34:03.593 | INFO | MemInfo: { | 20260505 13:34:03.593 | INFO | "gpu" : " 17 GB", | 20260505 13:34:03.593 | INFO | "huge" : "", | 20260505 13:34:03.593 | INFO | "huge_shared" : " 27 GB", | 20260505 13:34:03.593 | INFO | "malloc" : " 14 GB", | 20260505 13:34:03.593 | INFO | "pinned" : " 29 GB" | 20260505 13:34:03.593 | INFO | } | 20260505 13:34:03.593 | INFO | Perf report for the last 5.14768 seconds: | 20260505 13:34:03.593 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:34:03.593 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:03.593 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:03.593 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:03.593 | INFO | ANALYSIS : 63.1%, 3.0, 98.3ms, 0 | 20260505 13:34:03.593 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:03.593 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:03.593 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:34:03.593 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:03.593 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:03.594 | INFO | Uploaded 36GiB to GPU in 2.19732s (16.3836GiB/s) | 20260505 13:34:03.594 | INFO | Downloaded 34GiB to GPU in 2.12139s (16.0272GiB/s) | 20260505 13:34:08.733 | INFO | Analyzing chunk frames = [3601920,3602432) | 20260505 13:34:08.733 | INFO | MemInfo: { | 20260505 13:34:08.733 | INFO | "gpu" : " 17 GB", | 20260505 13:34:08.733 | INFO | "huge" : "", | 20260505 13:34:08.733 | INFO | "huge_shared" : " 27 GB", | 20260505 13:34:08.733 | INFO | "malloc" : " 14 GB", | 20260505 13:34:08.733 | INFO | "pinned" : " 29 GB" | 20260505 13:34:08.733 | INFO | } | 20260505 13:34:08.734 | INFO | Perf report for the last 5.14053 seconds: | 20260505 13:34:08.734 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:34:08.734 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:08.734 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:08.734 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:08.734 | INFO | ANALYSIS : 62.8%, 3.0, 97.5ms, 0 | 20260505 13:34:08.734 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:08.734 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:08.734 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:34:08.734 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:08.734 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:08.734 | INFO | Uploaded 36GiB to GPU in 2.18021s (16.5122GiB/s) | 20260505 13:34:08.734 | INFO | Downloaded 34GiB to GPU in 2.12931s (15.9676GiB/s) | 20260505 13:34:13.865 | INFO | Analyzing chunk frames = [3602432,3602944) | 20260505 13:34:13.865 | INFO | MemInfo: { | 20260505 13:34:13.865 | INFO | "gpu" : " 17 GB", | 20260505 13:34:13.865 | INFO | "huge" : "", | 20260505 13:34:13.865 | INFO | "huge_shared" : " 27 GB", | 20260505 13:34:13.865 | INFO | "malloc" : " 14 GB", | 20260505 13:34:13.865 | INFO | "pinned" : " 29 GB" | 20260505 13:34:13.865 | INFO | } | 20260505 13:34:13.865 | INFO | Perf report for the last 5.13158 seconds: | 20260505 13:34:13.865 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:34:13.865 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:13.865 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:13.865 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:13.865 | INFO | ANALYSIS : 62.9%, 3.0, 98.0ms, 0 | 20260505 13:34:13.865 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:13.865 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:13.865 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 13:34:13.865 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:13.865 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:13.865 | INFO | Uploaded 36GiB to GPU in 2.15411s (16.7122GiB/s) | 20260505 13:34:13.865 | INFO | Downloaded 34GiB to GPU in 2.12137s (16.0274GiB/s) | 20260505 13:34:19.005 | INFO | Analyzing chunk frames = [3602944,3603456) | 20260505 13:34:19.005 | INFO | MemInfo: { | 20260505 13:34:19.005 | INFO | "gpu" : " 17 GB", | 20260505 13:34:19.005 | INFO | "huge" : "", | 20260505 13:34:19.005 | INFO | "huge_shared" : " 27 GB", | 20260505 13:34:19.005 | INFO | "malloc" : " 14 GB", | 20260505 13:34:19.005 | INFO | "pinned" : " 29 GB" | 20260505 13:34:19.005 | INFO | } | 20260505 13:34:19.005 | INFO | Perf report for the last 5.13964 seconds: | 20260505 13:34:19.005 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:34:19.005 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:19.005 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:19.005 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:19.005 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 13:34:19.005 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:19.005 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:19.005 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 13:34:19.005 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:19.005 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:19.005 | INFO | Uploaded 36GiB to GPU in 2.14647s (16.7717GiB/s) | 20260505 13:34:19.005 | INFO | Downloaded 34GiB to GPU in 2.10853s (16.125GiB/s) | 20260505 13:34:24.151 | INFO | Analyzing chunk frames = [3603456,3603968) | 20260505 13:34:24.151 | INFO | MemInfo: { | 20260505 13:34:24.151 | INFO | "gpu" : " 17 GB", | 20260505 13:34:24.151 | INFO | "huge" : "", | 20260505 13:34:24.151 | INFO | "huge_shared" : " 27 GB", | 20260505 13:34:24.151 | INFO | "malloc" : " 14 GB", | 20260505 13:34:24.151 | INFO | "pinned" : " 29 GB" | 20260505 13:34:24.151 | INFO | } | 20260505 13:34:24.151 | INFO | Perf report for the last 5.1465 seconds: | 20260505 13:34:24.151 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:34:24.151 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:24.151 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:24.151 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:24.151 | INFO | ANALYSIS : 63.0%, 3.0, 98.6ms, 0 | 20260505 13:34:24.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:24.151 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:24.151 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 13:34:24.151 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:24.151 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:24.152 | INFO | Uploaded 36GiB to GPU in 2.1732s (16.5654GiB/s) | 20260505 13:34:24.152 | INFO | Downloaded 34GiB to GPU in 2.1535s (15.7882GiB/s) | 20260505 13:34:29.288 | INFO | Analyzing chunk frames = [3603968,3604480) | 20260505 13:34:29.288 | INFO | MemInfo: { | 20260505 13:34:29.288 | INFO | "gpu" : " 17 GB", | 20260505 13:34:29.288 | INFO | "huge" : "", | 20260505 13:34:29.288 | INFO | "huge_shared" : " 27 GB", | 20260505 13:34:29.288 | INFO | "malloc" : " 14 GB", | 20260505 13:34:29.288 | INFO | "pinned" : " 29 GB" | 20260505 13:34:29.288 | INFO | } | 20260505 13:34:29.289 | INFO | Perf report for the last 5.13732 seconds: | 20260505 13:34:29.289 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:34:29.289 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:29.289 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:29.289 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:29.289 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 13:34:29.289 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:29.289 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:29.289 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 13:34:29.289 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:29.289 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:29.289 | INFO | Uploaded 36GiB to GPU in 2.21481s (16.2542GiB/s) | 20260505 13:34:29.289 | INFO | Downloaded 34GiB to GPU in 2.10206s (16.1746GiB/s) | 20260505 13:34:29.289 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 13:34:29.289 | INFO | Total mem usage: 15.3 GiB | 20260505 13:34:29.289 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 13:34:29.289 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -7.05%) | 20260505 13:34:29.289 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 13:34:29.289 | INFO | High Water Breakdown: | 20260505 13:34:29.289 | INFO | 61.1% : 8.75 GiB ( 91.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 13:34:29.289 | INFO | 38.8% : 5.55 GiB ( 83.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 13:34:29.289 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 13:34:29.289 | INFO | ------------------------------------------ | 20260505 13:34:29.289 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:34:29.289 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 13:34:29.289 | INFO | Total mem usage: 26.5 GiB | 20260505 13:34:29.289 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 13:34:29.289 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 13:34:29.289 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 13:34:29.289 | INFO | High Water Breakdown: | 20260505 13:34:29.289 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:34:29.289 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:34:29.289 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:34:29.289 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:34:29.289 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:34:29.289 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:34:29.289 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 13:34:29.289 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 13:34:29.290 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:34:29.290 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 13:34:29.290 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 13:34:29.290 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 13:34:29.290 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 13:34:29.290 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 13:34:29.290 | INFO | ------------------------------------------ | 20260505 13:34:29.290 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:34:29.290 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 13:34:29.290 | INFO | Total mem usage: 25.6 GiB | 20260505 13:34:29.290 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 13:34:29.290 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 13:34:29.290 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 13:34:29.290 | INFO | High Water Breakdown: | 20260505 13:34:29.290 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 13:34:29.290 | INFO | ------------------------------------------ | 20260505 13:34:29.290 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:34:29.290 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 13:34:29.290 | INFO | Total mem usage: 15.7 GiB | 20260505 13:34:29.290 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 13:34:29.290 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 13:34:29.290 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 13:34:29.290 | INFO | High Water Breakdown: | 20260505 13:34:29.290 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 13:34:29.290 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 13:34:29.290 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 13:34:29.290 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 13:34:29.290 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 13:34:29.290 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 13:34:29.290 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 13:34:29.290 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 13:34:29.290 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 13:34:29.290 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:34:29.290 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 13:34:29.290 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:34:29.290 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 13:34:29.290 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 13:34:29.290 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 13:34:29.290 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 13:34:29.290 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:34:29.290 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 13:34:29.290 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:34:29.290 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:34:29.290 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:34:29.290 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 13:34:29.290 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 13:34:29.290 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:34:29.290 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 13:34:29.290 | INFO | ------------------------------------------ | 20260505 13:34:29.290 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:34:29.290 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 13:34:29.290 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:34:29.290 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:34:29.290 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.66e+01, 1.37e+01 / 1.36e+00 | 20260505 13:34:29.290 | INFO | PerfReport: 0.29%, AggregateStats -- 9.27e-02 / 9.92e-02, 1.05e-02 / 3.69e-03 | 20260505 13:34:29.290 | INFO | PerfReport: 11.77%, Baseline -- 3.75e+00 / 4.49e+00, 5.83e-01 / 1.60e-01 | 20260505 13:34:29.290 | INFO | PerfReport: 3.31%, Histogram -- 1.05e+00 / 1.50e+00, 2.01e-01 / 9.29e-02 | 20260505 13:34:29.290 | INFO | PerfReport: 16.04%, DME -- 4.08e+01 / 4.59e+01, 4.49e+00 / 1.68e+00 | 20260505 13:34:29.290 | INFO | PerfReport: 56.26%, FrameLabeling -- 1.79e+01 / 1.98e+01, 1.33e+00 / 1.70e+00 | 20260505 13:34:29.290 | INFO | PerfReport: 7.50%, PulseAccumulating -- 2.39e+00 / 2.95e+00, 4.52e-01 / 1.45e-01 | 20260505 13:34:29.290 | INFO | PerfReport: 3.58%, Metrics -- 1.14e+00 / 1.46e+00, 3.18e-01 / 1.38e-01 | 20260505 13:34:29.290 | INFO | PerfReport: 1.26%, PulseSerializing -- 4.00e-01 / 4.48e-01, 7.24e-02 / 2.81e-02 | 20260505 13:34:29.290 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 13:34:29.290 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:34:29.290 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:34:29.290 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260505 13:34:29.290 | INFO | PerfReport: 22.58%, Upload -- 2.28e+01 / 0.00e+00, 8.04e+00 / 0.00e+00 | 20260505 13:34:29.290 | INFO | PerfReport: 22.04%, Download -- 2.22e+01 / 0.00e+00, 8.09e+00 / 0.00e+00 | 20260505 13:34:29.290 | INFO | PerfReport: 31.57%, Compute -- 3.18e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 13:34:29.290 | INFO | PerfReport: 2.27%, LockUpload -- 2.29e+00 / 0.00e+00, 7.95e+00 / 0.00e+00 | 20260505 13:34:29.290 | INFO | PerfReport: 1.43%, LockDownload -- 1.44e+00 / 0.00e+00, 2.96e+00 / 0.00e+00 | 20260505 13:34:29.290 | INFO | PerfReport: 20.11%, LockCompute -- 2.03e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260505 13:34:29.290 | INFO | PerfReport: 0.02%, Misc -- 1.52e-02 / 0.00e+00, 3.19e-02 / 0.00e+00 | 20260505 13:34:29.430 | INFO | Starting a round of Baz IO | 20260505 13:34:32.655 | INFO | Populating RTMetrics report | 20260505 13:34:32.656 | INFO | RTMetrics report populated | 20260505 13:34:32.656 | INFO | Opening tmp RTMetrics json file | 20260505 13:34:32.660 | INFO | Serializing RTMetrics to json | 20260505 13:34:32.679 | INFO | Serialization done, swapping tmp file now... | 20260505 13:34:32.680 | INFO | RTMetrics done | 20260505 13:34:32.766 | INFO | Finished a round of Baz IO: Wrote 2.291 GiB in 3.336 seconds | 20260505 13:34:32.766 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.07% / 0.00% / 76.93% | 20260505 13:34:32.766 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.680101% / 7.94729e-06% / 7.94729e-05% | 20260505 13:34:32.766 | INFO | Baz IO overall: | 20260505 13:34:32.766 | INFO | BytesWritten : 2147072816335 | 20260505 13:34:32.766 | INFO | NumEvents : 1021415607866 | 20260505 13:34:34.424 | INFO | Analyzing chunk frames = [3604480,3604992) | 20260505 13:34:34.424 | INFO | MemInfo: { | 20260505 13:34:34.424 | INFO | "gpu" : " 17 GB", | 20260505 13:34:34.424 | INFO | "huge" : "", | 20260505 13:34:34.424 | INFO | "huge_shared" : " 27 GB", | 20260505 13:34:34.424 | INFO | "malloc" : " 14 GB", | 20260505 13:34:34.424 | INFO | "pinned" : " 29 GB" | 20260505 13:34:34.424 | INFO | } | 20260505 13:34:34.424 | INFO | Perf report for the last 5.13536 seconds: | 20260505 13:34:34.424 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:34:34.424 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:34.424 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:34.424 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:34.424 | INFO | ANALYSIS : 64.5%, 3.0, 99.9ms, 0 | 20260505 13:34:34.424 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:34.424 | INFO | RT_METRICS : 45.5%, 1.0, 23.9ms, 0 | 20260505 13:34:34.424 | INFO | PRE_HQ : 61.0%, 1.8, 57.7ms, 0 | 20260505 13:34:34.424 | INFO | BAZWRITER : 64.3%, 4.0, 131.7ms, 0 | 20260505 13:34:34.424 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 13:34:34.424 | INFO | Uploaded 36GiB to GPU in 2.18281s (16.4925GiB/s) | 20260505 13:34:34.424 | INFO | Downloaded 38GiB to GPU in 2.27313s (16.717GiB/s) | 20260505 13:34:39.555 | INFO | Analyzing chunk frames = [3604992,3605504) | 20260505 13:34:39.555 | INFO | MemInfo: { | 20260505 13:34:39.555 | INFO | "gpu" : " 17 GB", | 20260505 13:34:39.555 | INFO | "huge" : "", | 20260505 13:34:39.555 | INFO | "huge_shared" : " 27 GB", | 20260505 13:34:39.555 | INFO | "malloc" : " 14 GB", | 20260505 13:34:39.555 | INFO | "pinned" : " 29 GB" | 20260505 13:34:39.555 | INFO | } | 20260505 13:34:39.555 | INFO | Perf report for the last 5.13145 seconds: | 20260505 13:34:39.555 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:34:39.555 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:39.555 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:39.556 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:39.556 | INFO | ANALYSIS : 63.6%, 3.0, 99.1ms, 0 | 20260505 13:34:39.556 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:39.556 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:39.556 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:34:39.556 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:39.556 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:39.556 | INFO | Uploaded 36GiB to GPU in 2.16601s (16.6204GiB/s) | 20260505 13:34:39.556 | INFO | Downloaded 34GiB to GPU in 2.03367s (16.7185GiB/s) | 20260505 13:34:44.699 | INFO | Analyzing chunk frames = [3605504,3606016) | 20260505 13:34:44.699 | INFO | MemInfo: { | 20260505 13:34:44.699 | INFO | "gpu" : " 17 GB", | 20260505 13:34:44.699 | INFO | "huge" : "", | 20260505 13:34:44.699 | INFO | "huge_shared" : " 27 GB", | 20260505 13:34:44.699 | INFO | "malloc" : " 14 GB", | 20260505 13:34:44.699 | INFO | "pinned" : " 29 GB" | 20260505 13:34:44.699 | INFO | } | 20260505 13:34:44.700 | INFO | Perf report for the last 5.14483 seconds: | 20260505 13:34:44.700 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:34:44.700 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:44.700 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:44.700 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:44.700 | INFO | ANALYSIS : 63.1%, 3.0, 98.2ms, 0 | 20260505 13:34:44.700 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:44.700 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:44.700 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:34:44.700 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:44.700 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:44.700 | INFO | Uploaded 36GiB to GPU in 2.21364s (16.2628GiB/s) | 20260505 13:34:44.700 | INFO | Downloaded 34GiB to GPU in 2.12203s (16.0224GiB/s) | 20260505 13:34:49.845 | INFO | Analyzing chunk frames = [3606016,3606528) | 20260505 13:34:49.845 | INFO | MemInfo: { | 20260505 13:34:49.845 | INFO | "gpu" : " 17 GB", | 20260505 13:34:49.845 | INFO | "huge" : "", | 20260505 13:34:49.845 | INFO | "huge_shared" : " 27 GB", | 20260505 13:34:49.845 | INFO | "malloc" : " 14 GB", | 20260505 13:34:49.845 | INFO | "pinned" : " 29 GB" | 20260505 13:34:49.845 | INFO | } | 20260505 13:34:49.845 | INFO | Perf report for the last 5.14461 seconds: | 20260505 13:34:49.845 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:34:49.845 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:49.845 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:49.845 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:49.845 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 13:34:49.845 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:49.845 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:49.845 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 13:34:49.845 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:49.845 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:49.845 | INFO | Uploaded 36GiB to GPU in 2.18779s (16.455GiB/s) | 20260505 13:34:49.845 | INFO | Downloaded 34GiB to GPU in 2.13457s (15.9283GiB/s) | 20260505 13:34:54.993 | INFO | Analyzing chunk frames = [3606528,3607040) | 20260505 13:34:54.993 | INFO | MemInfo: { | 20260505 13:34:54.993 | INFO | "gpu" : " 17 GB", | 20260505 13:34:54.993 | INFO | "huge" : "", | 20260505 13:34:54.993 | INFO | "huge_shared" : " 27 GB", | 20260505 13:34:54.993 | INFO | "malloc" : " 14 GB", | 20260505 13:34:54.993 | INFO | "pinned" : " 29 GB" | 20260505 13:34:54.993 | INFO | } | 20260505 13:34:54.994 | INFO | Perf report for the last 5.14859 seconds: | 20260505 13:34:54.994 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:34:54.994 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:54.994 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:54.994 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:54.994 | INFO | ANALYSIS : 62.6%, 3.0, 98.0ms, 0 | 20260505 13:34:54.994 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:54.994 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:54.994 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 13:34:54.994 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:54.994 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:34:54.994 | INFO | Uploaded 36GiB to GPU in 2.15044s (16.7407GiB/s) | 20260505 13:34:54.994 | INFO | Downloaded 34GiB to GPU in 2.12987s (15.9634GiB/s) | 20260505 13:35:00.120 | INFO | Analyzing chunk frames = [3607040,3607552) | 20260505 13:35:00.120 | INFO | MemInfo: { | 20260505 13:35:00.120 | INFO | "gpu" : " 17 GB", | 20260505 13:35:00.120 | INFO | "huge" : "", | 20260505 13:35:00.120 | INFO | "huge_shared" : " 27 GB", | 20260505 13:35:00.120 | INFO | "malloc" : " 14 GB", | 20260505 13:35:00.120 | INFO | "pinned" : " 29 GB" | 20260505 13:35:00.120 | INFO | } | 20260505 13:35:00.120 | INFO | Perf report for the last 5.12683 seconds: | 20260505 13:35:00.120 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:35:00.120 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:00.120 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:00.120 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:00.120 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 13:35:00.120 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:00.120 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:00.120 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 13:35:00.120 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:00.121 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:00.121 | INFO | Uploaded 36GiB to GPU in 2.1126s (17.0406GiB/s) | 20260505 13:35:00.121 | INFO | Downloaded 34GiB to GPU in 2.0302s (16.7472GiB/s) | 20260505 13:35:05.251 | INFO | Analyzing chunk frames = [3607552,3608064) | 20260505 13:35:05.251 | INFO | MemInfo: { | 20260505 13:35:05.251 | INFO | "gpu" : " 17 GB", | 20260505 13:35:05.251 | INFO | "huge" : "", | 20260505 13:35:05.251 | INFO | "huge_shared" : " 27 GB", | 20260505 13:35:05.251 | INFO | "malloc" : " 14 GB", | 20260505 13:35:05.251 | INFO | "pinned" : " 29 GB" | 20260505 13:35:05.252 | INFO | } | 20260505 13:35:05.252 | INFO | Perf report for the last 5.13131 seconds: | 20260505 13:35:05.252 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:35:05.252 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:05.252 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:05.252 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:05.252 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 13:35:05.252 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:05.252 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:05.252 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 13:35:05.252 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:05.252 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:05.252 | INFO | Uploaded 36GiB to GPU in 2.19975s (16.3655GiB/s) | 20260505 13:35:05.252 | INFO | Downloaded 34GiB to GPU in 2.17972s (15.5983GiB/s) | 20260505 13:35:10.402 | INFO | Analyzing chunk frames = [3608064,3608576) | 20260505 13:35:10.402 | INFO | MemInfo: { | 20260505 13:35:10.402 | INFO | "gpu" : " 17 GB", | 20260505 13:35:10.402 | INFO | "huge" : "", | 20260505 13:35:10.402 | INFO | "huge_shared" : " 27 GB", | 20260505 13:35:10.402 | INFO | "malloc" : " 14 GB", | 20260505 13:35:10.402 | INFO | "pinned" : " 29 GB" | 20260505 13:35:10.402 | INFO | } | 20260505 13:35:10.402 | INFO | Perf report for the last 5.15041 seconds: | 20260505 13:35:10.402 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:35:10.402 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:10.402 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:10.402 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:10.402 | INFO | ANALYSIS : 63.5%, 3.0, 99.4ms, 0 | 20260505 13:35:10.402 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:10.402 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:10.402 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 13:35:10.402 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:10.402 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:10.402 | INFO | Uploaded 36GiB to GPU in 2.21096s (16.2825GiB/s) | 20260505 13:35:10.402 | INFO | Downloaded 34GiB to GPU in 2.09797s (16.2061GiB/s) | 20260505 13:35:10.540 | INFO | Starting a round of Baz IO | 20260505 13:35:13.766 | INFO | Populating RTMetrics report | 20260505 13:35:13.767 | INFO | RTMetrics report populated | 20260505 13:35:13.767 | INFO | Opening tmp RTMetrics json file | 20260505 13:35:13.798 | INFO | Serializing RTMetrics to json | 20260505 13:35:13.838 | INFO | Serialization done, swapping tmp file now... | 20260505 13:35:13.856 | INFO | RTMetrics done | 20260505 13:35:13.872 | INFO | Finished a round of Baz IO: Wrote 2.291 GiB in 3.331 seconds | 20260505 13:35:13.872 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.06% / 0.00% / 76.93% | 20260505 13:35:13.872 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.679056% / 3.97364e-06% / 7.15256e-05% | 20260505 13:35:13.872 | INFO | Baz IO overall: | 20260505 13:35:13.872 | INFO | BytesWritten : 2149532823010 | 20260505 13:35:13.872 | INFO | NumEvents : 1022578151971 | 20260505 13:35:15.542 | INFO | Analyzing chunk frames = [3608576,3609088) | 20260505 13:35:15.542 | INFO | MemInfo: { | 20260505 13:35:15.542 | INFO | "gpu" : " 17 GB", | 20260505 13:35:15.542 | INFO | "huge" : "", | 20260505 13:35:15.542 | INFO | "huge_shared" : " 27 GB", | 20260505 13:35:15.542 | INFO | "malloc" : " 14 GB", | 20260505 13:35:15.543 | INFO | "pinned" : " 29 GB" | 20260505 13:35:15.543 | INFO | } | 20260505 13:35:15.543 | INFO | Perf report for the last 5.14054 seconds: | 20260505 13:35:15.543 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:35:15.543 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:15.543 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:15.543 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:15.543 | INFO | ANALYSIS : 64.3%, 3.0, 100.0ms, 0 | 20260505 13:35:15.543 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:15.543 | INFO | RT_METRICS : 46.8%, 1.0, 24.5ms, 0 | 20260505 13:35:15.543 | INFO | PRE_HQ : 60.9%, 1.8, 57.6ms, 0 | 20260505 13:35:15.543 | INFO | BAZWRITER : 64.2%, 4.0, 131.7ms, 0 | 20260505 13:35:15.543 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 13:35:15.543 | INFO | Uploaded 36GiB to GPU in 2.18449s (16.4799GiB/s) | 20260505 13:35:15.543 | INFO | Downloaded 38GiB to GPU in 2.26907s (16.7469GiB/s) | 20260505 13:35:20.683 | INFO | Analyzing chunk frames = [3609088,3609600) | 20260505 13:35:20.683 | INFO | MemInfo: { | 20260505 13:35:20.683 | INFO | "gpu" : " 17 GB", | 20260505 13:35:20.683 | INFO | "huge" : "", | 20260505 13:35:20.683 | INFO | "huge_shared" : " 27 GB", | 20260505 13:35:20.683 | INFO | "malloc" : " 14 GB", | 20260505 13:35:20.683 | INFO | "pinned" : " 29 GB" | 20260505 13:35:20.683 | INFO | } | 20260505 13:35:20.683 | INFO | Perf report for the last 5.14069 seconds: | 20260505 13:35:20.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:35:20.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:20.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:20.683 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:20.683 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 13:35:20.683 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:20.684 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:20.684 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:35:20.684 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:20.684 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:20.684 | INFO | Uploaded 36GiB to GPU in 2.19312s (16.4149GiB/s) | 20260505 13:35:20.684 | INFO | Downloaded 34GiB to GPU in 2.03043s (16.7452GiB/s) | 20260505 13:35:25.819 | INFO | Analyzing chunk frames = [3609600,3610112) | 20260505 13:35:25.819 | INFO | MemInfo: { | 20260505 13:35:25.819 | INFO | "gpu" : " 17 GB", | 20260505 13:35:25.819 | INFO | "huge" : "", | 20260505 13:35:25.819 | INFO | "huge_shared" : " 27 GB", | 20260505 13:35:25.819 | INFO | "malloc" : " 14 GB", | 20260505 13:35:25.819 | INFO | "pinned" : " 29 GB" | 20260505 13:35:25.819 | INFO | } | 20260505 13:35:25.819 | INFO | Perf report for the last 5.13567 seconds: | 20260505 13:35:25.819 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:35:25.819 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:25.819 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:25.819 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:25.819 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 13:35:25.819 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:25.819 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:25.819 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 13:35:25.819 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:25.819 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:25.819 | INFO | Uploaded 36GiB to GPU in 2.22302s (16.1942GiB/s) | 20260505 13:35:25.819 | INFO | Downloaded 34GiB to GPU in 2.10381s (16.1611GiB/s) | 20260505 13:35:30.956 | INFO | Analyzing chunk frames = [3610112,3610624) | 20260505 13:35:30.956 | INFO | MemInfo: { | 20260505 13:35:30.956 | INFO | "gpu" : " 17 GB", | 20260505 13:35:30.956 | INFO | "huge" : "", | 20260505 13:35:30.956 | INFO | "huge_shared" : " 27 GB", | 20260505 13:35:30.956 | INFO | "malloc" : " 14 GB", | 20260505 13:35:30.956 | INFO | "pinned" : " 29 GB" | 20260505 13:35:30.956 | INFO | } | 20260505 13:35:30.956 | INFO | Perf report for the last 5.1368 seconds: | 20260505 13:35:30.956 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:35:30.956 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:30.956 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:30.956 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:30.956 | INFO | ANALYSIS : 62.9%, 3.0, 97.7ms, 0 | 20260505 13:35:30.956 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:30.956 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:30.956 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:35:30.956 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:30.956 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:30.956 | INFO | Uploaded 36GiB to GPU in 2.19512s (16.4GiB/s) | 20260505 13:35:30.956 | INFO | Downloaded 34GiB to GPU in 2.1361s (15.9169GiB/s) | 20260505 13:35:36.107 | INFO | Analyzing chunk frames = [3610624,3611136) | 20260505 13:35:36.107 | INFO | MemInfo: { | 20260505 13:35:36.107 | INFO | "gpu" : " 17 GB", | 20260505 13:35:36.107 | INFO | "huge" : "", | 20260505 13:35:36.107 | INFO | "huge_shared" : " 27 GB", | 20260505 13:35:36.107 | INFO | "malloc" : " 14 GB", | 20260505 13:35:36.107 | INFO | "pinned" : " 29 GB" | 20260505 13:35:36.107 | INFO | } | 20260505 13:35:36.107 | INFO | Perf report for the last 5.15161 seconds: | 20260505 13:35:36.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:35:36.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:36.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:36.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:36.108 | INFO | ANALYSIS : 62.6%, 3.0, 98.0ms, 0 | 20260505 13:35:36.108 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:36.108 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:36.108 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 13:35:36.108 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:36.108 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:36.108 | INFO | Uploaded 36GiB to GPU in 2.14677s (16.7694GiB/s) | 20260505 13:35:36.108 | INFO | Downloaded 34GiB to GPU in 2.12415s (16.0064GiB/s) | 20260505 13:35:41.241 | INFO | Analyzing chunk frames = [3611136,3611648) | 20260505 13:35:41.241 | INFO | MemInfo: { | 20260505 13:35:41.241 | INFO | "gpu" : " 17 GB", | 20260505 13:35:41.241 | INFO | "huge" : "", | 20260505 13:35:41.241 | INFO | "huge_shared" : " 27 GB", | 20260505 13:35:41.241 | INFO | "malloc" : " 14 GB", | 20260505 13:35:41.241 | INFO | "pinned" : " 29 GB" | 20260505 13:35:41.241 | INFO | } | 20260505 13:35:41.241 | INFO | Perf report for the last 5.13337 seconds: | 20260505 13:35:41.241 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:35:41.241 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:41.241 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:41.241 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:41.241 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 13:35:41.241 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:41.241 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:41.241 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 13:35:41.241 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:41.241 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:41.241 | INFO | Uploaded 36GiB to GPU in 2.19396s (16.4087GiB/s) | 20260505 13:35:41.241 | INFO | Downloaded 34GiB to GPU in 2.11281s (16.0923GiB/s) | 20260505 13:35:46.369 | INFO | Analyzing chunk frames = [3611648,3612160) | 20260505 13:35:46.369 | INFO | MemInfo: { | 20260505 13:35:46.369 | INFO | "gpu" : " 17 GB", | 20260505 13:35:46.369 | INFO | "huge" : "", | 20260505 13:35:46.369 | INFO | "huge_shared" : " 27 GB", | 20260505 13:35:46.369 | INFO | "malloc" : " 14 GB", | 20260505 13:35:46.369 | INFO | "pinned" : " 29 GB" | 20260505 13:35:46.369 | INFO | } | 20260505 13:35:46.369 | INFO | Perf report for the last 5.12847 seconds: | 20260505 13:35:46.369 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:35:46.369 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:46.369 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:46.369 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:46.369 | INFO | ANALYSIS : 64.2%, 3.0, 100.0ms, 0 | 20260505 13:35:46.369 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:46.369 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:46.369 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 13:35:46.369 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:46.369 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:46.370 | INFO | Uploaded 36GiB to GPU in 2.16s (16.6666GiB/s) | 20260505 13:35:46.370 | INFO | Downloaded 34GiB to GPU in 2.19753s (15.4719GiB/s) | 20260505 13:35:51.510 | INFO | Analyzing chunk frames = [3612160,3612672) | 20260505 13:35:51.510 | INFO | MemInfo: { | 20260505 13:35:51.510 | INFO | "gpu" : " 17 GB", | 20260505 13:35:51.510 | INFO | "huge" : "", | 20260505 13:35:51.511 | INFO | "huge_shared" : " 27 GB", | 20260505 13:35:51.511 | INFO | "malloc" : " 14 GB", | 20260505 13:35:51.511 | INFO | "pinned" : " 29 GB" | 20260505 13:35:51.511 | INFO | } | 20260505 13:35:51.511 | INFO | Perf report for the last 5.14127 seconds: | 20260505 13:35:51.511 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:35:51.511 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:51.511 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:51.511 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:51.511 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 13:35:51.511 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:51.511 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:51.511 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 13:35:51.511 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:51.511 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:51.511 | INFO | Uploaded 36GiB to GPU in 2.21898s (16.2237GiB/s) | 20260505 13:35:51.511 | INFO | Downloaded 34GiB to GPU in 2.08834s (16.2809GiB/s) | 20260505 13:35:51.511 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 13:35:51.511 | INFO | Total mem usage: 15.3 GiB | 20260505 13:35:51.511 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 13:35:51.511 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.77%) | 20260505 13:35:51.511 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 13:35:51.511 | INFO | High Water Breakdown: | 20260505 13:35:51.511 | INFO | 61% : 8.68 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 13:35:51.511 | INFO | 38.9% : 5.53 GiB ( 83.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 13:35:51.511 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 13:35:51.511 | INFO | ------------------------------------------ | 20260505 13:35:51.511 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:35:51.511 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 13:35:51.511 | INFO | Total mem usage: 26.5 GiB | 20260505 13:35:51.511 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 13:35:51.511 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 13:35:51.512 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 13:35:51.512 | INFO | High Water Breakdown: | 20260505 13:35:51.512 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:35:51.512 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:35:51.512 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:35:51.512 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:35:51.512 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:35:51.512 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:35:51.512 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 13:35:51.512 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 13:35:51.512 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:35:51.512 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 13:35:51.512 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 13:35:51.512 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 13:35:51.512 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 13:35:51.512 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 13:35:51.512 | INFO | ------------------------------------------ | 20260505 13:35:51.512 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:35:51.512 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 13:35:51.512 | INFO | Total mem usage: 25.6 GiB | 20260505 13:35:51.512 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 13:35:51.512 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 13:35:51.512 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 13:35:51.512 | INFO | High Water Breakdown: | 20260505 13:35:51.512 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 13:35:51.512 | INFO | ------------------------------------------ | 20260505 13:35:51.512 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:35:51.512 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 13:35:51.512 | INFO | Total mem usage: 15.7 GiB | 20260505 13:35:51.512 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 13:35:51.512 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 13:35:51.512 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260505 13:35:51.512 | INFO | High Water Breakdown: | 20260505 13:35:51.512 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 13:35:51.512 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 13:35:51.512 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 13:35:51.512 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 13:35:51.512 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 13:35:51.512 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 13:35:51.512 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 13:35:51.512 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 13:35:51.512 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 13:35:51.512 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:35:51.512 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 13:35:51.512 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:35:51.512 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 13:35:51.512 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 13:35:51.512 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 13:35:51.512 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 13:35:51.512 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:35:51.512 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 13:35:51.512 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:35:51.512 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:35:51.512 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:35:51.512 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 13:35:51.512 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 13:35:51.512 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:35:51.512 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 13:35:51.512 | INFO | ------------------------------------------ | 20260505 13:35:51.512 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:35:51.512 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 13:35:51.512 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:35:51.512 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:35:51.512 | INFO | PerfReport: Full Execution -- 3.20e+01 / 7.68e+01, 1.38e+01 / 2.01e+00 | 20260505 13:35:51.512 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 1.01e-01, 1.07e-02 / 4.93e-03 | 20260505 13:35:51.512 | INFO | PerfReport: 11.77%, Baseline -- 3.76e+00 / 4.45e+00, 6.01e-01 / 1.92e-01 | 20260505 13:35:51.512 | INFO | PerfReport: 3.31%, Histogram -- 1.06e+00 / 1.50e+00, 2.05e-01 / 9.31e-02 | 20260505 13:35:51.512 | INFO | PerfReport: 15.84%, DME -- 4.05e+01 / 4.51e+01, 4.59e+00 / 2.34e+00 | 20260505 13:35:51.512 | INFO | PerfReport: 56.62%, FrameLabeling -- 1.81e+01 / 2.09e+01, 1.60e+00 / 2.30e+00 | 20260505 13:35:51.512 | INFO | PerfReport: 7.36%, PulseAccumulating -- 2.35e+00 / 2.83e+00, 4.37e-01 / 2.38e-01 | 20260505 13:35:51.512 | INFO | PerfReport: 3.57%, Metrics -- 1.14e+00 / 1.45e+00, 3.28e-01 / 2.06e-01 | 20260505 13:35:51.512 | INFO | PerfReport: 1.25%, PulseSerializing -- 4.00e-01 / 4.49e-01, 7.22e-02 / 3.20e-02 | 20260505 13:35:51.512 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 13:35:51.512 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:35:51.512 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:35:51.512 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260505 13:35:51.512 | INFO | PerfReport: 22.56%, Upload -- 2.27e+01 / 0.00e+00, 8.14e+00 / 0.00e+00 | 20260505 13:35:51.512 | INFO | PerfReport: 21.99%, Download -- 2.22e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260505 13:35:51.512 | INFO | PerfReport: 31.70%, Compute -- 3.20e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 13:35:51.512 | INFO | PerfReport: 2.04%, LockUpload -- 2.06e+00 / 0.00e+00, 7.65e+00 / 0.00e+00 | 20260505 13:35:51.512 | INFO | PerfReport: 1.41%, LockDownload -- 1.42e+00 / 0.00e+00, 2.95e+00 / 0.00e+00 | 20260505 13:35:51.512 | INFO | PerfReport: 20.29%, LockCompute -- 2.05e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260505 13:35:51.512 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.90e-02 / 0.00e+00 | 20260505 13:35:51.643 | INFO | Starting a round of Baz IO | 20260505 13:35:54.884 | INFO | Populating RTMetrics report | 20260505 13:35:54.884 | INFO | RTMetrics report populated | 20260505 13:35:54.884 | INFO | Opening tmp RTMetrics json file | 20260505 13:35:54.891 | INFO | Serializing RTMetrics to json | 20260505 13:35:54.933 | INFO | Serialization done, swapping tmp file now... | 20260505 13:35:54.966 | INFO | RTMetrics done | 20260505 13:35:54.990 | INFO | Finished a round of Baz IO: Wrote 2.291 GiB in 3.346 seconds | 20260505 13:35:54.990 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.06% / 0.00% / 76.94% | 20260505 13:35:54.990 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.677399% / 7.94729e-06% / 8.74201e-05% | 20260505 13:35:54.990 | INFO | Baz IO overall: | 20260505 13:35:54.990 | INFO | BytesWritten : 2151992730085 | 20260505 13:35:54.990 | INFO | NumEvents : 1023740883365 | 20260505 13:35:56.654 | INFO | Analyzing chunk frames = [3612672,3613184) | 20260505 13:35:56.654 | INFO | MemInfo: { | 20260505 13:35:56.654 | INFO | "gpu" : " 17 GB", | 20260505 13:35:56.654 | INFO | "huge" : "", | 20260505 13:35:56.654 | INFO | "huge_shared" : " 27 GB", | 20260505 13:35:56.654 | INFO | "malloc" : " 14 GB", | 20260505 13:35:56.654 | INFO | "pinned" : " 29 GB" | 20260505 13:35:56.654 | INFO | } | 20260505 13:35:56.654 | INFO | Perf report for the last 5.14319 seconds: | 20260505 13:35:56.654 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:35:56.654 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:56.654 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:56.654 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:56.654 | INFO | ANALYSIS : 64.5%, 3.0, 100.2ms, 0 | 20260505 13:35:56.654 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:35:56.654 | INFO | RT_METRICS : 46.6%, 1.0, 24.4ms, 0 | 20260505 13:35:56.654 | INFO | PRE_HQ : 60.7%, 1.8, 57.5ms, 0 | 20260505 13:35:56.654 | INFO | BAZWRITER : 64.4%, 4.0, 131.9ms, 0 | 20260505 13:35:56.654 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 13:35:56.654 | INFO | Uploaded 36GiB to GPU in 2.20093s (16.3567GiB/s) | 20260505 13:35:56.655 | INFO | Downloaded 38GiB to GPU in 2.26794s (16.7553GiB/s) | 20260505 13:36:01.800 | INFO | Analyzing chunk frames = [3613184,3613696) | 20260505 13:36:01.800 | INFO | MemInfo: { | 20260505 13:36:01.800 | INFO | "gpu" : " 17 GB", | 20260505 13:36:01.800 | INFO | "huge" : "", | 20260505 13:36:01.800 | INFO | "huge_shared" : " 27 GB", | 20260505 13:36:01.800 | INFO | "malloc" : " 14 GB", | 20260505 13:36:01.800 | INFO | "pinned" : " 29 GB" | 20260505 13:36:01.800 | INFO | } | 20260505 13:36:01.800 | INFO | Perf report for the last 5.14605 seconds: | 20260505 13:36:01.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:36:01.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:01.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:01.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:01.800 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 13:36:01.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:01.800 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:01.800 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 13:36:01.800 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:01.800 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:01.800 | INFO | Uploaded 36GiB to GPU in 2.19167s (16.4258GiB/s) | 20260505 13:36:01.800 | INFO | Downloaded 34GiB to GPU in 2.0297s (16.7512GiB/s) | 20260505 13:36:06.939 | INFO | Analyzing chunk frames = [3613696,3614208) | 20260505 13:36:06.939 | INFO | MemInfo: { | 20260505 13:36:06.939 | INFO | "gpu" : " 17 GB", | 20260505 13:36:06.939 | INFO | "huge" : "", | 20260505 13:36:06.939 | INFO | "huge_shared" : " 27 GB", | 20260505 13:36:06.939 | INFO | "malloc" : " 14 GB", | 20260505 13:36:06.939 | INFO | "pinned" : " 29 GB" | 20260505 13:36:06.939 | INFO | } | 20260505 13:36:06.939 | INFO | Perf report for the last 5.13956 seconds: | 20260505 13:36:06.939 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:36:06.939 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:06.939 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:06.939 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:06.940 | INFO | ANALYSIS : 63.1%, 3.0, 98.2ms, 0 | 20260505 13:36:06.940 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:06.940 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:06.940 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:36:06.940 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:06.940 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:06.940 | INFO | Uploaded 36GiB to GPU in 2.1802s (16.5123GiB/s) | 20260505 13:36:06.940 | INFO | Downloaded 34GiB to GPU in 2.06086s (16.498GiB/s) | 20260505 13:36:12.069 | INFO | Analyzing chunk frames = [3614208,3614720) | 20260505 13:36:12.069 | INFO | MemInfo: { | 20260505 13:36:12.069 | INFO | "gpu" : " 17 GB", | 20260505 13:36:12.069 | INFO | "huge" : "", | 20260505 13:36:12.069 | INFO | "huge_shared" : " 27 GB", | 20260505 13:36:12.069 | INFO | "malloc" : " 14 GB", | 20260505 13:36:12.069 | INFO | "pinned" : " 29 GB" | 20260505 13:36:12.069 | INFO | } | 20260505 13:36:12.069 | INFO | Perf report for the last 5.12981 seconds: | 20260505 13:36:12.069 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:36:12.069 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:12.069 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:12.069 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:12.069 | INFO | ANALYSIS : 62.9%, 3.0, 97.6ms, 0 | 20260505 13:36:12.069 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:12.069 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:12.069 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 13:36:12.069 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:12.069 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:12.070 | INFO | Uploaded 36GiB to GPU in 2.18552s (16.4721GiB/s) | 20260505 13:36:12.070 | INFO | Downloaded 34GiB to GPU in 2.12961s (15.9653GiB/s) | 20260505 13:36:17.210 | INFO | Analyzing chunk frames = [3614720,3615232) | 20260505 13:36:17.210 | INFO | MemInfo: { | 20260505 13:36:17.210 | INFO | "gpu" : " 17 GB", | 20260505 13:36:17.210 | INFO | "huge" : "", | 20260505 13:36:17.210 | INFO | "huge_shared" : " 27 GB", | 20260505 13:36:17.210 | INFO | "malloc" : " 14 GB", | 20260505 13:36:17.210 | INFO | "pinned" : " 29 GB" | 20260505 13:36:17.210 | INFO | } | 20260505 13:36:17.210 | INFO | Perf report for the last 5.14037 seconds: | 20260505 13:36:17.210 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:36:17.210 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:17.210 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:17.210 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:17.210 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 13:36:17.210 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:17.210 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:17.210 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 13:36:17.210 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:17.210 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:17.210 | INFO | Uploaded 36GiB to GPU in 2.17568s (16.5466GiB/s) | 20260505 13:36:17.210 | INFO | Downloaded 34GiB to GPU in 2.12738s (15.9821GiB/s) | 20260505 13:36:22.352 | INFO | Analyzing chunk frames = [3615232,3615744) | 20260505 13:36:22.352 | INFO | MemInfo: { | 20260505 13:36:22.352 | INFO | "gpu" : " 17 GB", | 20260505 13:36:22.352 | INFO | "huge" : "", | 20260505 13:36:22.352 | INFO | "huge_shared" : " 27 GB", | 20260505 13:36:22.352 | INFO | "malloc" : " 14 GB", | 20260505 13:36:22.352 | INFO | "pinned" : " 29 GB" | 20260505 13:36:22.352 | INFO | } | 20260505 13:36:22.352 | INFO | Perf report for the last 5.14244 seconds: | 20260505 13:36:22.352 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:36:22.352 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:22.352 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:22.352 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:22.352 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 13:36:22.352 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:22.352 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:22.352 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 13:36:22.352 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:22.352 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:22.352 | INFO | Uploaded 36GiB to GPU in 2.16034s (16.6641GiB/s) | 20260505 13:36:22.352 | INFO | Downloaded 34GiB to GPU in 2.10803s (16.1288GiB/s) | 20260505 13:36:27.497 | INFO | Analyzing chunk frames = [3615744,3616256) | 20260505 13:36:27.497 | INFO | MemInfo: { | 20260505 13:36:27.497 | INFO | "gpu" : " 17 GB", | 20260505 13:36:27.497 | INFO | "huge" : "", | 20260505 13:36:27.497 | INFO | "huge_shared" : " 27 GB", | 20260505 13:36:27.497 | INFO | "malloc" : " 14 GB", | 20260505 13:36:27.497 | INFO | "pinned" : " 29 GB" | 20260505 13:36:27.497 | INFO | } | 20260505 13:36:27.497 | INFO | Perf report for the last 5.14523 seconds: | 20260505 13:36:27.497 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:36:27.497 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:27.497 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:27.497 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:27.497 | INFO | ANALYSIS : 64.0%, 3.0, 100.1ms, 0 | 20260505 13:36:27.497 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:27.497 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:27.498 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 13:36:27.498 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:27.498 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:27.498 | INFO | Uploaded 36GiB to GPU in 2.1513s (16.7341GiB/s) | 20260505 13:36:27.498 | INFO | Downloaded 34GiB to GPU in 2.19023s (15.5235GiB/s) | 20260505 13:36:32.630 | INFO | Analyzing chunk frames = [3616256,3616768) | 20260505 13:36:32.630 | INFO | MemInfo: { | 20260505 13:36:32.630 | INFO | "gpu" : " 17 GB", | 20260505 13:36:32.630 | INFO | "huge" : "", | 20260505 13:36:32.630 | INFO | "huge_shared" : " 27 GB", | 20260505 13:36:32.630 | INFO | "malloc" : " 14 GB", | 20260505 13:36:32.630 | INFO | "pinned" : " 29 GB" | 20260505 13:36:32.630 | INFO | } | 20260505 13:36:32.630 | INFO | Perf report for the last 5.13288 seconds: | 20260505 13:36:32.630 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:36:32.630 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:32.630 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:32.630 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:32.630 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 13:36:32.630 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:32.630 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:32.630 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 13:36:32.630 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:32.630 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:32.631 | INFO | Uploaded 36GiB to GPU in 2.20932s (16.2946GiB/s) | 20260505 13:36:32.631 | INFO | Downloaded 34GiB to GPU in 2.10486s (16.1531GiB/s) | 20260505 13:36:32.765 | INFO | Starting a round of Baz IO | 20260505 13:36:35.999 | INFO | Populating RTMetrics report | 20260505 13:36:35.999 | INFO | RTMetrics report populated | 20260505 13:36:35.999 | INFO | Opening tmp RTMetrics json file | 20260505 13:36:36.015 | INFO | Serializing RTMetrics to json | 20260505 13:36:36.016 | INFO | Serialization done, swapping tmp file now... | 20260505 13:36:36.016 | INFO | RTMetrics done | 20260505 13:36:36.101 | INFO | Finished a round of Baz IO: Wrote 2.29 GiB in 3.336 seconds | 20260505 13:36:36.101 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.05% / 0.00% / 76.94% | 20260505 13:36:36.101 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.675535% / 3.97364e-06% / 9.53674e-05% | 20260505 13:36:36.101 | INFO | Baz IO overall: | 20260505 13:36:36.101 | INFO | BytesWritten : 2154452039861 | 20260505 13:36:36.101 | INFO | NumEvents : 1024903419112 | 20260505 13:36:37.763 | INFO | Analyzing chunk frames = [3616768,3617280) | 20260505 13:36:37.763 | INFO | MemInfo: { | 20260505 13:36:37.763 | INFO | "gpu" : " 17 GB", | 20260505 13:36:37.763 | INFO | "huge" : "", | 20260505 13:36:37.763 | INFO | "huge_shared" : " 27 GB", | 20260505 13:36:37.763 | INFO | "malloc" : " 14 GB", | 20260505 13:36:37.763 | INFO | "pinned" : " 29 GB" | 20260505 13:36:37.763 | INFO | } | 20260505 13:36:37.763 | INFO | Perf report for the last 5.13308 seconds: | 20260505 13:36:37.763 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:36:37.763 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:37.763 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:37.763 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:37.763 | INFO | ANALYSIS : 64.5%, 3.0, 100.0ms, 0 | 20260505 13:36:37.763 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:37.763 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260505 13:36:37.763 | INFO | PRE_HQ : 60.6%, 1.8, 57.5ms, 0 | 20260505 13:36:37.764 | INFO | BAZWRITER : 64.3%, 4.0, 131.3ms, 0 | 20260505 13:36:37.764 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 13:36:37.764 | INFO | Uploaded 36GiB to GPU in 2.19457s (16.4041GiB/s) | 20260505 13:36:37.764 | INFO | Downloaded 38GiB to GPU in 2.27061s (16.7356GiB/s) | 20260505 13:36:42.912 | INFO | Analyzing chunk frames = [3617280,3617792) | 20260505 13:36:42.912 | INFO | MemInfo: { | 20260505 13:36:42.912 | INFO | "gpu" : " 17 GB", | 20260505 13:36:42.912 | INFO | "huge" : "", | 20260505 13:36:42.912 | INFO | "huge_shared" : " 27 GB", | 20260505 13:36:42.912 | INFO | "malloc" : " 14 GB", | 20260505 13:36:42.912 | INFO | "pinned" : " 29 GB" | 20260505 13:36:42.912 | INFO | } | 20260505 13:36:42.912 | INFO | Perf report for the last 5.14911 seconds: | 20260505 13:36:42.912 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:36:42.912 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:42.913 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:42.913 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:42.913 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 13:36:42.913 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:42.913 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:42.913 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 13:36:42.913 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:42.913 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:42.913 | INFO | Uploaded 36GiB to GPU in 2.18598s (16.4686GiB/s) | 20260505 13:36:42.913 | INFO | Downloaded 34GiB to GPU in 2.05128s (16.575GiB/s) | 20260505 13:36:48.052 | INFO | Analyzing chunk frames = [3617792,3618304) | 20260505 13:36:48.052 | INFO | MemInfo: { | 20260505 13:36:48.052 | INFO | "gpu" : " 17 GB", | 20260505 13:36:48.052 | INFO | "huge" : "", | 20260505 13:36:48.052 | INFO | "huge_shared" : " 27 GB", | 20260505 13:36:48.052 | INFO | "malloc" : " 14 GB", | 20260505 13:36:48.052 | INFO | "pinned" : " 29 GB" | 20260505 13:36:48.052 | INFO | } | 20260505 13:36:48.053 | INFO | Perf report for the last 5.14008 seconds: | 20260505 13:36:48.053 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:36:48.053 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:48.053 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:48.053 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:48.053 | INFO | ANALYSIS : 63.0%, 3.0, 98.1ms, 0 | 20260505 13:36:48.053 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:48.053 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:48.053 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 13:36:48.053 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:48.053 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:48.053 | INFO | Uploaded 36GiB to GPU in 2.17607s (16.5436GiB/s) | 20260505 13:36:48.053 | INFO | Downloaded 34GiB to GPU in 2.05777s (16.5227GiB/s) | 20260505 13:36:53.187 | INFO | Analyzing chunk frames = [3618304,3618816) | 20260505 13:36:53.187 | INFO | MemInfo: { | 20260505 13:36:53.187 | INFO | "gpu" : " 17 GB", | 20260505 13:36:53.187 | INFO | "huge" : "", | 20260505 13:36:53.187 | INFO | "huge_shared" : " 27 GB", | 20260505 13:36:53.187 | INFO | "malloc" : " 14 GB", | 20260505 13:36:53.187 | INFO | "pinned" : " 29 GB" | 20260505 13:36:53.187 | INFO | } | 20260505 13:36:53.187 | INFO | Perf report for the last 5.13463 seconds: | 20260505 13:36:53.187 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:36:53.187 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:53.187 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:53.187 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:53.187 | INFO | ANALYSIS : 63.0%, 3.0, 97.8ms, 0 | 20260505 13:36:53.187 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:53.187 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:53.187 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 13:36:53.187 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:53.187 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:53.187 | INFO | Uploaded 36GiB to GPU in 2.20093s (16.3567GiB/s) | 20260505 13:36:53.188 | INFO | Downloaded 34GiB to GPU in 2.13529s (15.9229GiB/s) | 20260505 13:36:58.322 | INFO | Analyzing chunk frames = [3618816,3619328) | 20260505 13:36:58.322 | INFO | MemInfo: { | 20260505 13:36:58.322 | INFO | "gpu" : " 17 GB", | 20260505 13:36:58.322 | INFO | "huge" : "", | 20260505 13:36:58.322 | INFO | "huge_shared" : " 27 GB", | 20260505 13:36:58.322 | INFO | "malloc" : " 14 GB", | 20260505 13:36:58.322 | INFO | "pinned" : " 29 GB" | 20260505 13:36:58.322 | INFO | } | 20260505 13:36:58.322 | INFO | Perf report for the last 5.13455 seconds: | 20260505 13:36:58.322 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:36:58.322 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:58.322 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:58.322 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:58.322 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 13:36:58.322 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:58.322 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:58.322 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 13:36:58.322 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:58.322 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:36:58.322 | INFO | Uploaded 36GiB to GPU in 2.1728s (16.5685GiB/s) | 20260505 13:36:58.322 | INFO | Downloaded 34GiB to GPU in 2.12899s (15.97GiB/s) | 20260505 13:37:03.466 | INFO | Analyzing chunk frames = [3619328,3619840) | 20260505 13:37:03.466 | INFO | MemInfo: { | 20260505 13:37:03.466 | INFO | "gpu" : " 17 GB", | 20260505 13:37:03.466 | INFO | "huge" : "", | 20260505 13:37:03.466 | INFO | "huge_shared" : " 27 GB", | 20260505 13:37:03.466 | INFO | "malloc" : " 14 GB", | 20260505 13:37:03.466 | INFO | "pinned" : " 29 GB" | 20260505 13:37:03.466 | INFO | } | 20260505 13:37:03.466 | INFO | Perf report for the last 5.1442 seconds: | 20260505 13:37:03.466 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:37:03.466 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:03.466 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:03.466 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:03.466 | INFO | ANALYSIS : 63.1%, 3.0, 98.7ms, 0 | 20260505 13:37:03.466 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:03.466 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:03.466 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 13:37:03.466 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:03.466 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:03.466 | INFO | Uploaded 36GiB to GPU in 2.16066s (16.6616GiB/s) | 20260505 13:37:03.466 | INFO | Downloaded 34GiB to GPU in 2.11066s (16.1087GiB/s) | 20260505 13:37:08.608 | INFO | Analyzing chunk frames = [3619840,3620352) | 20260505 13:37:08.608 | INFO | MemInfo: { | 20260505 13:37:08.608 | INFO | "gpu" : " 17 GB", | 20260505 13:37:08.608 | INFO | "huge" : "", | 20260505 13:37:08.608 | INFO | "huge_shared" : " 27 GB", | 20260505 13:37:08.608 | INFO | "malloc" : " 14 GB", | 20260505 13:37:08.608 | INFO | "pinned" : " 29 GB" | 20260505 13:37:08.608 | INFO | } | 20260505 13:37:08.608 | INFO | Perf report for the last 5.14235 seconds: | 20260505 13:37:08.608 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:37:08.608 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:08.608 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:08.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:08.608 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 13:37:08.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:08.608 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:08.608 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 13:37:08.608 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:08.608 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:08.609 | INFO | Uploaded 36GiB to GPU in 2.16762s (16.6081GiB/s) | 20260505 13:37:08.609 | INFO | Downloaded 34GiB to GPU in 2.19897s (15.4618GiB/s) | 20260505 13:37:13.747 | INFO | Analyzing chunk frames = [3620352,3620864) | 20260505 13:37:13.747 | INFO | MemInfo: { | 20260505 13:37:13.747 | INFO | "gpu" : " 17 GB", | 20260505 13:37:13.747 | INFO | "huge" : "", | 20260505 13:37:13.747 | INFO | "huge_shared" : " 27 GB", | 20260505 13:37:13.747 | INFO | "malloc" : " 14 GB", | 20260505 13:37:13.747 | INFO | "pinned" : " 29 GB" | 20260505 13:37:13.747 | INFO | } | 20260505 13:37:13.747 | INFO | Perf report for the last 5.13888 seconds: | 20260505 13:37:13.747 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:37:13.747 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:13.747 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:13.747 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:13.747 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 13:37:13.747 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:13.747 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:13.747 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 13:37:13.747 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:13.747 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:13.748 | INFO | Uploaded 36GiB to GPU in 2.16403s (16.6357GiB/s) | 20260505 13:37:13.748 | INFO | Downloaded 34GiB to GPU in 2.09914s (16.1971GiB/s) | 20260505 13:37:13.748 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 13:37:13.748 | INFO | Total mem usage: 15.3 GiB | 20260505 13:37:13.748 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 13:37:13.748 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -7.06%) | 20260505 13:37:13.748 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 13:37:13.748 | INFO | High Water Breakdown: | 20260505 13:37:13.748 | INFO | 61.1% : 8.75 GiB ( 91.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 13:37:13.748 | INFO | 38.8% : 5.55 GiB ( 83.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 13:37:13.748 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 13:37:13.748 | INFO | ------------------------------------------ | 20260505 13:37:13.748 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:37:13.748 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 13:37:13.748 | INFO | Total mem usage: 26.5 GiB | 20260505 13:37:13.748 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 13:37:13.748 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 13:37:13.748 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 13:37:13.748 | INFO | High Water Breakdown: | 20260505 13:37:13.748 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:37:13.748 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:37:13.748 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:37:13.748 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:37:13.748 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:37:13.748 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:37:13.748 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 13:37:13.748 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 13:37:13.748 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:37:13.748 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 13:37:13.748 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 13:37:13.748 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 13:37:13.748 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 13:37:13.748 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 13:37:13.748 | INFO | ------------------------------------------ | 20260505 13:37:13.748 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:37:13.748 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 13:37:13.748 | INFO | Total mem usage: 25.6 GiB | 20260505 13:37:13.748 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 13:37:13.748 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 13:37:13.748 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 13:37:13.748 | INFO | High Water Breakdown: | 20260505 13:37:13.748 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 13:37:13.748 | INFO | ------------------------------------------ | 20260505 13:37:13.748 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:37:13.748 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 13:37:13.748 | INFO | Total mem usage: 15.7 GiB | 20260505 13:37:13.748 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 13:37:13.748 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 13:37:13.748 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 13:37:13.748 | INFO | High Water Breakdown: | 20260505 13:37:13.748 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 13:37:13.748 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 13:37:13.748 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 13:37:13.749 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 13:37:13.749 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 13:37:13.749 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 13:37:13.749 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 13:37:13.749 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 13:37:13.749 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 13:37:13.749 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:37:13.749 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 13:37:13.749 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:37:13.749 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 13:37:13.749 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 13:37:13.749 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 13:37:13.749 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 13:37:13.749 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:37:13.749 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 13:37:13.749 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:37:13.749 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:37:13.749 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:37:13.749 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 13:37:13.749 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 13:37:13.749 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:37:13.749 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 13:37:13.749 | INFO | ------------------------------------------ | 20260505 13:37:13.749 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:37:13.749 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 13:37:13.749 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:37:13.749 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:37:13.749 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.71e+01, 1.39e+01 / 2.30e+00 | 20260505 13:37:13.749 | INFO | PerfReport: 0.29%, AggregateStats -- 9.27e-02 / 1.00e-01, 1.03e-02 / 4.15e-03 | 20260505 13:37:13.749 | INFO | PerfReport: 11.75%, Baseline -- 3.75e+00 / 4.44e+00, 5.90e-01 / 1.81e-01 | 20260505 13:37:13.749 | INFO | PerfReport: 3.31%, Histogram -- 1.06e+00 / 1.49e+00, 2.01e-01 / 1.01e-01 | 20260505 13:37:13.749 | INFO | PerfReport: 16.07%, DME -- 4.11e+01 / 4.65e+01, 4.79e+00 / 2.63e+00 | 20260505 13:37:13.749 | INFO | PerfReport: 56.39%, FrameLabeling -- 1.80e+01 / 1.98e+01, 1.47e+00 / 1.71e+00 | 20260505 13:37:13.749 | INFO | PerfReport: 7.39%, PulseAccumulating -- 2.36e+00 / 2.97e+00, 4.46e-01 / 1.49e-01 | 20260505 13:37:13.749 | INFO | PerfReport: 3.55%, Metrics -- 1.13e+00 / 1.46e+00, 3.21e-01 / 1.97e-01 | 20260505 13:37:13.749 | INFO | PerfReport: 1.25%, PulseSerializing -- 3.99e-01 / 4.49e-01, 7.21e-02 / 3.08e-02 | 20260505 13:37:13.749 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 13:37:13.749 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:37:13.749 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:37:13.749 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.57e+01 / 0.00e+00 | 20260505 13:37:13.749 | INFO | PerfReport: 22.49%, Upload -- 2.27e+01 / 0.00e+00, 8.01e+00 / 0.00e+00 | 20260505 13:37:13.749 | INFO | PerfReport: 21.98%, Download -- 2.22e+01 / 0.00e+00, 8.06e+00 / 0.00e+00 | 20260505 13:37:13.749 | INFO | PerfReport: 31.65%, Compute -- 3.19e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 13:37:13.749 | INFO | PerfReport: 2.17%, LockUpload -- 2.19e+00 / 0.00e+00, 7.94e+00 / 0.00e+00 | 20260505 13:37:13.749 | INFO | PerfReport: 1.47%, LockDownload -- 1.49e+00 / 0.00e+00, 3.00e+00 / 0.00e+00 | 20260505 13:37:13.749 | INFO | PerfReport: 20.22%, LockCompute -- 2.04e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 13:37:13.749 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.96e-02 / 0.00e+00 | 20260505 13:37:13.892 | INFO | Starting a round of Baz IO | 20260505 13:37:17.108 | INFO | Populating RTMetrics report | 20260505 13:37:17.109 | INFO | RTMetrics report populated | 20260505 13:37:17.109 | INFO | Opening tmp RTMetrics json file | 20260505 13:37:17.118 | INFO | Serializing RTMetrics to json | 20260505 13:37:17.151 | INFO | Serialization done, swapping tmp file now... | 20260505 13:37:17.176 | INFO | RTMetrics done | 20260505 13:37:17.215 | INFO | Finished a round of Baz IO: Wrote 2.292 GiB in 3.323 seconds | 20260505 13:37:17.215 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.05% / 0.00% / 76.95% | 20260505 13:37:17.216 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.676517% / 2.38419e-05% / 8.34465e-05% | 20260505 13:37:17.216 | INFO | Baz IO overall: | 20260505 13:37:17.216 | INFO | BytesWritten : 2156913345592 | 20260505 13:37:17.216 | INFO | NumEvents : 1026067640781 | 20260505 13:37:18.882 | INFO | Analyzing chunk frames = [3620864,3621376) | 20260505 13:37:18.882 | INFO | MemInfo: { | 20260505 13:37:18.882 | INFO | "gpu" : " 17 GB", | 20260505 13:37:18.882 | INFO | "huge" : "", | 20260505 13:37:18.882 | INFO | "huge_shared" : " 27 GB", | 20260505 13:37:18.882 | INFO | "malloc" : " 14 GB", | 20260505 13:37:18.882 | INFO | "pinned" : " 29 GB" | 20260505 13:37:18.882 | INFO | } | 20260505 13:37:18.882 | INFO | Perf report for the last 5.13441 seconds: | 20260505 13:37:18.882 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:37:18.882 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:18.882 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:18.882 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:18.882 | INFO | ANALYSIS : 64.4%, 3.0, 99.9ms, 0 | 20260505 13:37:18.882 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:18.882 | INFO | RT_METRICS : 46.5%, 1.0, 24.4ms, 0 | 20260505 13:37:18.882 | INFO | PRE_HQ : 60.8%, 1.8, 57.7ms, 0 | 20260505 13:37:18.882 | INFO | BAZWRITER : 64.1%, 4.0, 131.6ms, 0 | 20260505 13:37:18.882 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260505 13:37:18.882 | INFO | Uploaded 36GiB to GPU in 2.16728s (16.6107GiB/s) | 20260505 13:37:18.882 | INFO | Downloaded 38GiB to GPU in 2.27295s (16.7184GiB/s) | 20260505 13:37:24.017 | INFO | Analyzing chunk frames = [3621376,3621888) | 20260505 13:37:24.017 | INFO | MemInfo: { | 20260505 13:37:24.017 | INFO | "gpu" : " 17 GB", | 20260505 13:37:24.017 | INFO | "huge" : "", | 20260505 13:37:24.017 | INFO | "huge_shared" : " 27 GB", | 20260505 13:37:24.017 | INFO | "malloc" : " 14 GB", | 20260505 13:37:24.017 | INFO | "pinned" : " 29 GB" | 20260505 13:37:24.017 | INFO | } | 20260505 13:37:24.017 | INFO | Perf report for the last 5.13515 seconds: | 20260505 13:37:24.017 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:37:24.017 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:24.017 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:24.017 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:24.017 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 13:37:24.017 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:24.017 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:24.017 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:37:24.017 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:24.017 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:24.017 | INFO | Uploaded 36GiB to GPU in 2.19522s (16.3993GiB/s) | 20260505 13:37:24.017 | INFO | Downloaded 34GiB to GPU in 2.05002s (16.5852GiB/s) | 20260505 13:37:29.164 | INFO | Analyzing chunk frames = [3621888,3622400) | 20260505 13:37:29.164 | INFO | MemInfo: { | 20260505 13:37:29.164 | INFO | "gpu" : " 17 GB", | 20260505 13:37:29.164 | INFO | "huge" : "", | 20260505 13:37:29.164 | INFO | "huge_shared" : " 27 GB", | 20260505 13:37:29.164 | INFO | "malloc" : " 14 GB", | 20260505 13:37:29.164 | INFO | "pinned" : " 29 GB" | 20260505 13:37:29.164 | INFO | } | 20260505 13:37:29.165 | INFO | Perf report for the last 5.14772 seconds: | 20260505 13:37:29.165 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:37:29.165 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:29.165 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:29.165 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:29.165 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 13:37:29.165 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:29.165 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:29.165 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:37:29.165 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:29.165 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:29.165 | INFO | Uploaded 36GiB to GPU in 2.21712s (16.2373GiB/s) | 20260505 13:37:29.165 | INFO | Downloaded 34GiB to GPU in 2.11675s (16.0624GiB/s) | 20260505 13:37:34.308 | INFO | Analyzing chunk frames = [3622400,3622912) | 20260505 13:37:34.308 | INFO | MemInfo: { | 20260505 13:37:34.308 | INFO | "gpu" : " 17 GB", | 20260505 13:37:34.308 | INFO | "huge" : "", | 20260505 13:37:34.308 | INFO | "huge_shared" : " 27 GB", | 20260505 13:37:34.308 | INFO | "malloc" : " 14 GB", | 20260505 13:37:34.308 | INFO | "pinned" : " 29 GB" | 20260505 13:37:34.308 | INFO | } | 20260505 13:37:34.308 | INFO | Perf report for the last 5.1435 seconds: | 20260505 13:37:34.308 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:37:34.308 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:34.308 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:34.308 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:34.308 | INFO | ANALYSIS : 62.8%, 3.0, 97.6ms, 0 | 20260505 13:37:34.308 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:34.308 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:34.308 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 13:37:34.308 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:34.308 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:34.308 | INFO | Uploaded 36GiB to GPU in 2.18354s (16.487GiB/s) | 20260505 13:37:34.308 | INFO | Downloaded 34GiB to GPU in 2.12929s (15.9678GiB/s) | 20260505 13:37:39.449 | INFO | Analyzing chunk frames = [3622912,3623424) | 20260505 13:37:39.449 | INFO | MemInfo: { | 20260505 13:37:39.449 | INFO | "gpu" : " 17 GB", | 20260505 13:37:39.449 | INFO | "huge" : "", | 20260505 13:37:39.449 | INFO | "huge_shared" : " 27 GB", | 20260505 13:37:39.449 | INFO | "malloc" : " 14 GB", | 20260505 13:37:39.449 | INFO | "pinned" : " 29 GB" | 20260505 13:37:39.449 | INFO | } | 20260505 13:37:39.449 | INFO | Perf report for the last 5.14132 seconds: | 20260505 13:37:39.449 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:37:39.449 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:39.449 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:39.449 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:39.449 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 13:37:39.449 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:39.450 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:39.450 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:37:39.450 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:39.450 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:39.450 | INFO | Uploaded 36GiB to GPU in 2.16113s (16.658GiB/s) | 20260505 13:37:39.450 | INFO | Downloaded 34GiB to GPU in 2.12814s (15.9764GiB/s) | 20260505 13:37:44.581 | INFO | Analyzing chunk frames = [3623424,3623936) | 20260505 13:37:44.581 | INFO | MemInfo: { | 20260505 13:37:44.581 | INFO | "gpu" : " 17 GB", | 20260505 13:37:44.581 | INFO | "huge" : "", | 20260505 13:37:44.581 | INFO | "huge_shared" : " 27 GB", | 20260505 13:37:44.581 | INFO | "malloc" : " 14 GB", | 20260505 13:37:44.581 | INFO | "pinned" : " 29 GB" | 20260505 13:37:44.581 | INFO | } | 20260505 13:37:44.581 | INFO | Perf report for the last 5.13169 seconds: | 20260505 13:37:44.581 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:37:44.581 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:44.581 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:44.581 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:44.581 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 13:37:44.581 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:44.581 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:44.581 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 13:37:44.581 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:44.581 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:44.581 | INFO | Uploaded 36GiB to GPU in 2.14225s (16.8048GiB/s) | 20260505 13:37:44.581 | INFO | Downloaded 34GiB to GPU in 2.10295s (16.1678GiB/s) | 20260505 13:37:49.720 | INFO | Analyzing chunk frames = [3623936,3624448) | 20260505 13:37:49.720 | INFO | MemInfo: { | 20260505 13:37:49.720 | INFO | "gpu" : " 17 GB", | 20260505 13:37:49.720 | INFO | "huge" : "", | 20260505 13:37:49.720 | INFO | "huge_shared" : " 27 GB", | 20260505 13:37:49.720 | INFO | "malloc" : " 14 GB", | 20260505 13:37:49.720 | INFO | "pinned" : " 29 GB" | 20260505 13:37:49.720 | INFO | } | 20260505 13:37:49.720 | INFO | Perf report for the last 5.13897 seconds: | 20260505 13:37:49.720 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:37:49.720 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:49.720 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:49.720 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:49.720 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 13:37:49.720 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:49.720 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:49.720 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 13:37:49.720 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:49.720 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:49.720 | INFO | Uploaded 36GiB to GPU in 2.18579s (16.47GiB/s) | 20260505 13:37:49.720 | INFO | Downloaded 34GiB to GPU in 2.14883s (15.8226GiB/s) | 20260505 13:37:54.863 | INFO | Analyzing chunk frames = [3624448,3624960) | 20260505 13:37:54.864 | INFO | MemInfo: { | 20260505 13:37:54.864 | INFO | "gpu" : " 17 GB", | 20260505 13:37:54.864 | INFO | "huge" : "", | 20260505 13:37:54.864 | INFO | "huge_shared" : " 27 GB", | 20260505 13:37:54.864 | INFO | "malloc" : " 14 GB", | 20260505 13:37:54.864 | INFO | "pinned" : " 29 GB" | 20260505 13:37:54.864 | INFO | } | 20260505 13:37:54.864 | INFO | Perf report for the last 5.14368 seconds: | 20260505 13:37:54.864 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:37:54.864 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:54.864 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:54.864 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:54.864 | INFO | ANALYSIS : 63.5%, 3.0, 99.3ms, 0 | 20260505 13:37:54.864 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:54.864 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:54.864 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:37:54.864 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:54.864 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:37:54.864 | INFO | Uploaded 36GiB to GPU in 2.20555s (16.3224GiB/s) | 20260505 13:37:54.864 | INFO | Downloaded 34GiB to GPU in 2.10567s (16.1469GiB/s) | 20260505 13:37:55.000 | INFO | Starting a round of Baz IO | 20260505 13:37:58.224 | INFO | Populating RTMetrics report | 20260505 13:37:58.224 | INFO | RTMetrics report populated | 20260505 13:37:58.224 | INFO | Opening tmp RTMetrics json file | 20260505 13:37:58.243 | INFO | Serializing RTMetrics to json | 20260505 13:37:58.266 | INFO | Serialization done, swapping tmp file now... | 20260505 13:37:58.289 | INFO | RTMetrics done | 20260505 13:37:58.328 | INFO | Finished a round of Baz IO: Wrote 2.293 GiB in 3.328 seconds | 20260505 13:37:58.328 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.05% / 0.00% / 76.95% | 20260505 13:37:58.328 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.675094% / 3.97364e-06% / 7.94729e-05% | 20260505 13:37:58.328 | INFO | Baz IO overall: | 20260505 13:37:58.328 | INFO | BytesWritten : 2159374919839 | 20260505 13:37:58.328 | INFO | NumEvents : 1027232368776 | 20260505 13:38:00.003 | INFO | Analyzing chunk frames = [3624960,3625472) | 20260505 13:38:00.003 | INFO | MemInfo: { | 20260505 13:38:00.003 | INFO | "gpu" : " 17 GB", | 20260505 13:38:00.003 | INFO | "huge" : "", | 20260505 13:38:00.003 | INFO | "huge_shared" : " 27 GB", | 20260505 13:38:00.003 | INFO | "malloc" : " 14 GB", | 20260505 13:38:00.003 | INFO | "pinned" : " 29 GB" | 20260505 13:38:00.003 | INFO | } | 20260505 13:38:00.003 | INFO | Perf report for the last 5.13925 seconds: | 20260505 13:38:00.003 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:38:00.003 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:00.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:00.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:00.003 | INFO | ANALYSIS : 64.3%, 3.0, 99.9ms, 0 | 20260505 13:38:00.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:00.003 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260505 13:38:00.003 | INFO | PRE_HQ : 61.0%, 1.8, 57.6ms, 0 | 20260505 13:38:00.003 | INFO | BAZWRITER : 64.1%, 4.0, 131.1ms, 0 | 20260505 13:38:00.003 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 13:38:00.003 | INFO | Uploaded 36GiB to GPU in 2.20139s (16.3533GiB/s) | 20260505 13:38:00.003 | INFO | Downloaded 38GiB to GPU in 2.26418s (16.7831GiB/s) | 20260505 13:38:05.136 | INFO | Analyzing chunk frames = [3625472,3625984) | 20260505 13:38:05.136 | INFO | MemInfo: { | 20260505 13:38:05.136 | INFO | "gpu" : " 17 GB", | 20260505 13:38:05.136 | INFO | "huge" : "", | 20260505 13:38:05.136 | INFO | "huge_shared" : " 27 GB", | 20260505 13:38:05.136 | INFO | "malloc" : " 14 GB", | 20260505 13:38:05.136 | INFO | "pinned" : " 29 GB" | 20260505 13:38:05.136 | INFO | } | 20260505 13:38:05.136 | INFO | Perf report for the last 5.13317 seconds: | 20260505 13:38:05.136 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:38:05.136 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:05.136 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:05.136 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:05.136 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 13:38:05.136 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:05.136 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:05.136 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:38:05.136 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:05.136 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:05.136 | INFO | Uploaded 36GiB to GPU in 2.20468s (16.3289GiB/s) | 20260505 13:38:05.136 | INFO | Downloaded 34GiB to GPU in 2.04563s (16.6208GiB/s) | 20260505 13:38:10.274 | INFO | Analyzing chunk frames = [3625984,3626496) | 20260505 13:38:10.274 | INFO | MemInfo: { | 20260505 13:38:10.274 | INFO | "gpu" : " 17 GB", | 20260505 13:38:10.274 | INFO | "huge" : "", | 20260505 13:38:10.274 | INFO | "huge_shared" : " 27 GB", | 20260505 13:38:10.274 | INFO | "malloc" : " 14 GB", | 20260505 13:38:10.274 | INFO | "pinned" : " 29 GB" | 20260505 13:38:10.274 | INFO | } | 20260505 13:38:10.274 | INFO | Perf report for the last 5.13813 seconds: | 20260505 13:38:10.274 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:38:10.274 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:10.274 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:10.274 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:10.274 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 13:38:10.274 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:10.274 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:10.274 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 13:38:10.274 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:10.274 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:10.275 | INFO | Uploaded 36GiB to GPU in 2.21433s (16.2578GiB/s) | 20260505 13:38:10.275 | INFO | Downloaded 34GiB to GPU in 2.10652s (16.1403GiB/s) | 20260505 13:38:15.423 | INFO | Analyzing chunk frames = [3626496,3627008) | 20260505 13:38:15.423 | INFO | MemInfo: { | 20260505 13:38:15.423 | INFO | "gpu" : " 17 GB", | 20260505 13:38:15.423 | INFO | "huge" : "", | 20260505 13:38:15.423 | INFO | "huge_shared" : " 27 GB", | 20260505 13:38:15.423 | INFO | "malloc" : " 14 GB", | 20260505 13:38:15.423 | INFO | "pinned" : " 29 GB" | 20260505 13:38:15.423 | INFO | } | 20260505 13:38:15.423 | INFO | Perf report for the last 5.14834 seconds: | 20260505 13:38:15.423 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:38:15.423 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:15.423 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:15.423 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:15.423 | INFO | ANALYSIS : 62.9%, 3.0, 97.9ms, 0 | 20260505 13:38:15.423 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:15.423 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:15.423 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:38:15.423 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:15.423 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:15.423 | INFO | Uploaded 36GiB to GPU in 2.20861s (16.2998GiB/s) | 20260505 13:38:15.423 | INFO | Downloaded 34GiB to GPU in 2.13219s (15.9461GiB/s) | 20260505 13:38:20.564 | INFO | Analyzing chunk frames = [3627008,3627520) | 20260505 13:38:20.564 | INFO | MemInfo: { | 20260505 13:38:20.564 | INFO | "gpu" : " 17 GB", | 20260505 13:38:20.564 | INFO | "huge" : "", | 20260505 13:38:20.564 | INFO | "huge_shared" : " 27 GB", | 20260505 13:38:20.564 | INFO | "malloc" : " 14 GB", | 20260505 13:38:20.564 | INFO | "pinned" : " 29 GB" | 20260505 13:38:20.564 | INFO | } | 20260505 13:38:20.564 | INFO | Perf report for the last 5.14114 seconds: | 20260505 13:38:20.564 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:38:20.564 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:20.564 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:20.564 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:20.564 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 13:38:20.564 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:20.564 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:20.564 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 13:38:20.564 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:20.564 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:20.564 | INFO | Uploaded 36GiB to GPU in 2.15182s (16.73GiB/s) | 20260505 13:38:20.564 | INFO | Downloaded 34GiB to GPU in 2.12447s (16.004GiB/s) | 20260505 13:38:25.694 | INFO | Analyzing chunk frames = [3627520,3628032) | 20260505 13:38:25.694 | INFO | MemInfo: { | 20260505 13:38:25.695 | INFO | "gpu" : " 17 GB", | 20260505 13:38:25.695 | INFO | "huge" : "", | 20260505 13:38:25.695 | INFO | "huge_shared" : " 27 GB", | 20260505 13:38:25.695 | INFO | "malloc" : " 14 GB", | 20260505 13:38:25.695 | INFO | "pinned" : " 29 GB" | 20260505 13:38:25.695 | INFO | } | 20260505 13:38:25.695 | INFO | Perf report for the last 5.13091 seconds: | 20260505 13:38:25.695 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:38:25.695 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:25.695 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:25.695 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:25.695 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 13:38:25.695 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:25.695 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:25.695 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 13:38:25.695 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:25.695 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:25.695 | INFO | Uploaded 36GiB to GPU in 2.19848s (16.3749GiB/s) | 20260505 13:38:25.695 | INFO | Downloaded 34GiB to GPU in 2.11267s (16.0934GiB/s) | 20260505 13:38:30.829 | INFO | Analyzing chunk frames = [3628032,3628544) | 20260505 13:38:30.829 | INFO | MemInfo: { | 20260505 13:38:30.829 | INFO | "gpu" : " 17 GB", | 20260505 13:38:30.829 | INFO | "huge" : "", | 20260505 13:38:30.829 | INFO | "huge_shared" : " 27 GB", | 20260505 13:38:30.829 | INFO | "malloc" : " 14 GB", | 20260505 13:38:30.829 | INFO | "pinned" : " 29 GB" | 20260505 13:38:30.829 | INFO | } | 20260505 13:38:30.829 | INFO | Perf report for the last 5.13458 seconds: | 20260505 13:38:30.829 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:38:30.829 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:30.829 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:30.829 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:30.829 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 13:38:30.829 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:30.829 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:30.830 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 13:38:30.830 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:30.830 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:30.830 | INFO | Uploaded 36GiB to GPU in 2.1711s (16.5814GiB/s) | 20260505 13:38:30.830 | INFO | Downloaded 34GiB to GPU in 2.15201s (15.7992GiB/s) | 20260505 13:38:35.972 | INFO | Analyzing chunk frames = [3628544,3629056) | 20260505 13:38:35.972 | INFO | MemInfo: { | 20260505 13:38:35.972 | INFO | "gpu" : " 17 GB", | 20260505 13:38:35.972 | INFO | "huge" : "", | 20260505 13:38:35.972 | INFO | "huge_shared" : " 27 GB", | 20260505 13:38:35.972 | INFO | "malloc" : " 14 GB", | 20260505 13:38:35.972 | INFO | "pinned" : " 29 GB" | 20260505 13:38:35.972 | INFO | } | 20260505 13:38:35.972 | INFO | Perf report for the last 5.14256 seconds: | 20260505 13:38:35.972 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:38:35.972 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:35.972 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:35.972 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:35.972 | INFO | ANALYSIS : 63.7%, 3.0, 99.6ms, 0 | 20260505 13:38:35.972 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:35.972 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:35.972 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:38:35.972 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:35.972 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:35.972 | INFO | Uploaded 36GiB to GPU in 2.20693s (16.3122GiB/s) | 20260505 13:38:35.972 | INFO | Downloaded 34GiB to GPU in 2.11421s (16.0817GiB/s) | 20260505 13:38:35.972 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 13:38:35.973 | INFO | Total mem usage: 15.3 GiB | 20260505 13:38:35.973 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 13:38:35.973 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -7.03%) | 20260505 13:38:35.973 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 13:38:35.973 | INFO | High Water Breakdown: | 20260505 13:38:35.973 | INFO | 61.1% : 8.75 GiB ( 91.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 13:38:35.973 | INFO | 38.8% : 5.55 GiB ( 83.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 13:38:35.973 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 13:38:35.973 | INFO | ------------------------------------------ | 20260505 13:38:35.973 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:38:35.973 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 13:38:35.973 | INFO | Total mem usage: 26.5 GiB | 20260505 13:38:35.973 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 13:38:35.973 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 13:38:35.973 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 13:38:35.973 | INFO | High Water Breakdown: | 20260505 13:38:35.973 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:38:35.973 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:38:35.973 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:38:35.973 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:38:35.973 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:38:35.973 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:38:35.973 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 13:38:35.973 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 13:38:35.973 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:38:35.973 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 13:38:35.973 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 13:38:35.973 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 13:38:35.973 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 13:38:35.973 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 13:38:35.973 | INFO | ------------------------------------------ | 20260505 13:38:35.973 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:38:35.973 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 13:38:35.973 | INFO | Total mem usage: 25.6 GiB | 20260505 13:38:35.973 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 13:38:35.973 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 13:38:35.973 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 13:38:35.973 | INFO | High Water Breakdown: | 20260505 13:38:35.973 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 13:38:35.973 | INFO | ------------------------------------------ | 20260505 13:38:35.973 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:38:35.973 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 13:38:35.973 | INFO | Total mem usage: 15.7 GiB | 20260505 13:38:35.973 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 13:38:35.973 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 13:38:35.973 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 13:38:35.973 | INFO | High Water Breakdown: | 20260505 13:38:35.973 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 13:38:35.973 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 13:38:35.973 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 13:38:35.973 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 13:38:35.973 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 13:38:35.974 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 13:38:35.974 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 13:38:35.974 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 13:38:35.974 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 13:38:35.974 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:38:35.974 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 13:38:35.974 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:38:35.974 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 13:38:35.974 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 13:38:35.974 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 13:38:35.974 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 13:38:35.974 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:38:35.974 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 13:38:35.974 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:38:35.974 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:38:35.974 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:38:35.974 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 13:38:35.974 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 13:38:35.974 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:38:35.974 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 13:38:35.974 | INFO | ------------------------------------------ | 20260505 13:38:35.974 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:38:35.974 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 13:38:35.974 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:38:35.974 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:38:35.974 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.66e+01, 1.37e+01 / 1.53e+00 | 20260505 13:38:35.974 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 9.99e-02, 1.07e-02 / 4.60e-03 | 20260505 13:38:35.974 | INFO | PerfReport: 11.77%, Baseline -- 3.75e+00 / 4.49e+00, 5.85e-01 / 1.79e-01 | 20260505 13:38:35.974 | INFO | PerfReport: 3.31%, Histogram -- 1.06e+00 / 1.49e+00, 2.02e-01 / 1.05e-01 | 20260505 13:38:35.974 | INFO | PerfReport: 15.92%, DME -- 4.06e+01 / 4.56e+01, 4.47e+00 / 1.74e+00 | 20260505 13:38:35.974 | INFO | PerfReport: 56.40%, FrameLabeling -- 1.80e+01 / 2.02e+01, 1.45e+00 / 1.69e+00 | 20260505 13:38:35.974 | INFO | PerfReport: 7.48%, PulseAccumulating -- 2.38e+00 / 2.89e+00, 4.49e-01 / 1.95e-01 | 20260505 13:38:35.974 | INFO | PerfReport: 3.57%, Metrics -- 1.14e+00 / 1.47e+00, 3.17e-01 / 1.78e-01 | 20260505 13:38:35.974 | INFO | PerfReport: 1.25%, PulseSerializing -- 4.00e-01 / 4.57e-01, 7.25e-02 / 3.09e-02 | 20260505 13:38:35.974 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 13:38:35.974 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:38:35.974 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:38:35.974 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260505 13:38:35.974 | INFO | PerfReport: 22.62%, Upload -- 2.28e+01 / 0.00e+00, 8.03e+00 / 0.00e+00 | 20260505 13:38:35.974 | INFO | PerfReport: 22.03%, Download -- 2.22e+01 / 0.00e+00, 8.04e+00 / 0.00e+00 | 20260505 13:38:35.974 | INFO | PerfReport: 31.63%, Compute -- 3.19e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 13:38:35.974 | INFO | PerfReport: 2.18%, LockUpload -- 2.20e+00 / 0.00e+00, 7.80e+00 / 0.00e+00 | 20260505 13:38:35.974 | INFO | PerfReport: 1.38%, LockDownload -- 1.39e+00 / 0.00e+00, 2.90e+00 / 0.00e+00 | 20260505 13:38:35.974 | INFO | PerfReport: 20.14%, LockCompute -- 2.03e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260505 13:38:35.974 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 2.11e-02 / 0.00e+00 | 20260505 13:38:36.112 | INFO | Starting a round of Baz IO | 20260505 13:38:39.335 | INFO | Populating RTMetrics report | 20260505 13:38:39.335 | INFO | RTMetrics report populated | 20260505 13:38:39.335 | INFO | Opening tmp RTMetrics json file | 20260505 13:38:39.356 | INFO | Serializing RTMetrics to json | 20260505 13:38:39.378 | INFO | Serialization done, swapping tmp file now... | 20260505 13:38:39.399 | INFO | RTMetrics done | 20260505 13:38:39.439 | INFO | Finished a round of Baz IO: Wrote 2.292 GiB in 3.327 seconds | 20260505 13:38:39.439 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.04% / 0.00% / 76.95% | 20260505 13:38:39.439 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.673143% / 3.97364e-06% / 8.34465e-05% | 20260505 13:38:39.439 | INFO | Baz IO overall: | 20260505 13:38:39.439 | INFO | BytesWritten : 2161835686296 | 20260505 13:38:39.439 | INFO | NumEvents : 1028396407613 | 20260505 13:38:41.116 | INFO | Analyzing chunk frames = [3629056,3629568) | 20260505 13:38:41.116 | INFO | MemInfo: { | 20260505 13:38:41.116 | INFO | "gpu" : " 17 GB", | 20260505 13:38:41.116 | INFO | "huge" : "", | 20260505 13:38:41.116 | INFO | "huge_shared" : " 27 GB", | 20260505 13:38:41.116 | INFO | "malloc" : " 14 GB", | 20260505 13:38:41.116 | INFO | "pinned" : " 29 GB" | 20260505 13:38:41.116 | INFO | } | 20260505 13:38:41.116 | INFO | Perf report for the last 5.14377 seconds: | 20260505 13:38:41.116 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:38:41.116 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:41.116 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:41.116 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:41.116 | INFO | ANALYSIS : 64.3%, 3.0, 99.8ms, 0 | 20260505 13:38:41.116 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:41.116 | INFO | RT_METRICS : 46.4%, 1.0, 24.4ms, 0 | 20260505 13:38:41.116 | INFO | PRE_HQ : 60.8%, 1.8, 57.6ms, 0 | 20260505 13:38:41.116 | INFO | BAZWRITER : 64.0%, 4.0, 131.4ms, 0 | 20260505 13:38:41.116 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 13:38:41.116 | INFO | Uploaded 36GiB to GPU in 2.17516s (16.5505GiB/s) | 20260505 13:38:41.116 | INFO | Downloaded 38GiB to GPU in 2.27028s (16.7381GiB/s) | 20260505 13:38:46.262 | INFO | Analyzing chunk frames = [3629568,3630080) | 20260505 13:38:46.262 | INFO | MemInfo: { | 20260505 13:38:46.262 | INFO | "gpu" : " 17 GB", | 20260505 13:38:46.262 | INFO | "huge" : "", | 20260505 13:38:46.262 | INFO | "huge_shared" : " 27 GB", | 20260505 13:38:46.262 | INFO | "malloc" : " 14 GB", | 20260505 13:38:46.262 | INFO | "pinned" : " 29 GB" | 20260505 13:38:46.262 | INFO | } | 20260505 13:38:46.262 | INFO | Perf report for the last 5.14628 seconds: | 20260505 13:38:46.262 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:38:46.262 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:46.262 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:46.262 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:46.262 | INFO | ANALYSIS : 63.3%, 3.0, 99.0ms, 0 | 20260505 13:38:46.262 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:46.262 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:46.262 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 13:38:46.262 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:46.262 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:46.262 | INFO | Uploaded 36GiB to GPU in 2.19651s (16.3896GiB/s) | 20260505 13:38:46.262 | INFO | Downloaded 34GiB to GPU in 2.02715s (16.7723GiB/s) | 20260505 13:38:51.397 | INFO | Analyzing chunk frames = [3630080,3630592) | 20260505 13:38:51.397 | INFO | MemInfo: { | 20260505 13:38:51.397 | INFO | "gpu" : " 17 GB", | 20260505 13:38:51.397 | INFO | "huge" : "", | 20260505 13:38:51.397 | INFO | "huge_shared" : " 27 GB", | 20260505 13:38:51.397 | INFO | "malloc" : " 14 GB", | 20260505 13:38:51.397 | INFO | "pinned" : " 29 GB" | 20260505 13:38:51.397 | INFO | } | 20260505 13:38:51.397 | INFO | Perf report for the last 5.13515 seconds: | 20260505 13:38:51.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:38:51.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:51.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:51.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:51.397 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260505 13:38:51.397 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:51.397 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:51.397 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:38:51.397 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:51.397 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:51.397 | INFO | Uploaded 36GiB to GPU in 2.23487s (16.1083GiB/s) | 20260505 13:38:51.398 | INFO | Downloaded 34GiB to GPU in 2.11733s (16.0579GiB/s) | 20260505 13:38:56.525 | INFO | Analyzing chunk frames = [3630592,3631104) | 20260505 13:38:56.525 | INFO | MemInfo: { | 20260505 13:38:56.525 | INFO | "gpu" : " 17 GB", | 20260505 13:38:56.525 | INFO | "huge" : "", | 20260505 13:38:56.525 | INFO | "huge_shared" : " 27 GB", | 20260505 13:38:56.525 | INFO | "malloc" : " 14 GB", | 20260505 13:38:56.525 | INFO | "pinned" : " 29 GB" | 20260505 13:38:56.525 | INFO | } | 20260505 13:38:56.525 | INFO | Perf report for the last 5.12819 seconds: | 20260505 13:38:56.525 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:38:56.525 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:56.525 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:56.525 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:56.525 | INFO | ANALYSIS : 63.2%, 3.0, 97.9ms, 0 | 20260505 13:38:56.525 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:56.525 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:56.525 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 13:38:56.525 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:56.525 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:38:56.526 | INFO | Uploaded 36GiB to GPU in 2.20852s (16.3005GiB/s) | 20260505 13:38:56.526 | INFO | Downloaded 34GiB to GPU in 2.13228s (15.9454GiB/s) | 20260505 13:39:01.678 | INFO | Analyzing chunk frames = [3631104,3631616) | 20260505 13:39:01.678 | INFO | MemInfo: { | 20260505 13:39:01.678 | INFO | "gpu" : " 17 GB", | 20260505 13:39:01.678 | INFO | "huge" : "", | 20260505 13:39:01.678 | INFO | "huge_shared" : " 27 GB", | 20260505 13:39:01.678 | INFO | "malloc" : " 14 GB", | 20260505 13:39:01.678 | INFO | "pinned" : " 29 GB" | 20260505 13:39:01.678 | INFO | } | 20260505 13:39:01.678 | INFO | Perf report for the last 5.15298 seconds: | 20260505 13:39:01.678 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:39:01.678 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:01.678 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:01.678 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:01.678 | INFO | ANALYSIS : 62.7%, 3.0, 98.2ms, 0 | 20260505 13:39:01.678 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:01.678 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:01.678 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 13:39:01.678 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:01.678 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:01.679 | INFO | Uploaded 36GiB to GPU in 2.15983s (16.668GiB/s) | 20260505 13:39:01.679 | INFO | Downloaded 34GiB to GPU in 2.12751s (15.9811GiB/s) | 20260505 13:39:06.813 | INFO | Analyzing chunk frames = [3631616,3632128) | 20260505 13:39:06.813 | INFO | MemInfo: { | 20260505 13:39:06.813 | INFO | "gpu" : " 17 GB", | 20260505 13:39:06.813 | INFO | "huge" : "", | 20260505 13:39:06.813 | INFO | "huge_shared" : " 27 GB", | 20260505 13:39:06.813 | INFO | "malloc" : " 14 GB", | 20260505 13:39:06.813 | INFO | "pinned" : " 29 GB" | 20260505 13:39:06.813 | INFO | } | 20260505 13:39:06.813 | INFO | Perf report for the last 5.13444 seconds: | 20260505 13:39:06.813 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:39:06.813 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:06.813 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:06.813 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:06.813 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 13:39:06.813 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:06.813 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:06.813 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:39:06.813 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:06.813 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:06.813 | INFO | Uploaded 36GiB to GPU in 2.16234s (16.6487GiB/s) | 20260505 13:39:06.813 | INFO | Downloaded 34GiB to GPU in 2.03734s (16.6884GiB/s) | 20260505 13:39:11.951 | INFO | Analyzing chunk frames = [3632128,3632640) | 20260505 13:39:11.951 | INFO | MemInfo: { | 20260505 13:39:11.951 | INFO | "gpu" : " 17 GB", | 20260505 13:39:11.951 | INFO | "huge" : "", | 20260505 13:39:11.951 | INFO | "huge_shared" : " 27 GB", | 20260505 13:39:11.951 | INFO | "malloc" : " 14 GB", | 20260505 13:39:11.951 | INFO | "pinned" : " 29 GB" | 20260505 13:39:11.951 | INFO | } | 20260505 13:39:11.951 | INFO | Perf report for the last 5.13806 seconds: | 20260505 13:39:11.951 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:39:11.951 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:11.951 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:11.951 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:11.951 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 13:39:11.951 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:11.951 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:11.951 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 13:39:11.951 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:11.951 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:11.951 | INFO | Uploaded 36GiB to GPU in 2.15876s (16.6762GiB/s) | 20260505 13:39:11.951 | INFO | Downloaded 34GiB to GPU in 2.15715s (15.7616GiB/s) | 20260505 13:39:17.086 | INFO | Analyzing chunk frames = [3632640,3633152) | 20260505 13:39:17.086 | INFO | MemInfo: { | 20260505 13:39:17.086 | INFO | "gpu" : " 17 GB", | 20260505 13:39:17.086 | INFO | "huge" : "", | 20260505 13:39:17.086 | INFO | "huge_shared" : " 27 GB", | 20260505 13:39:17.086 | INFO | "malloc" : " 14 GB", | 20260505 13:39:17.086 | INFO | "pinned" : " 29 GB" | 20260505 13:39:17.086 | INFO | } | 20260505 13:39:17.086 | INFO | Perf report for the last 5.13543 seconds: | 20260505 13:39:17.086 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:39:17.086 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:17.087 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:17.087 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:17.087 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 13:39:17.087 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:17.087 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:17.087 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 13:39:17.087 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:17.087 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:17.087 | INFO | Uploaded 36GiB to GPU in 2.15002s (16.744GiB/s) | 20260505 13:39:17.087 | INFO | Downloaded 34GiB to GPU in 2.10171s (16.1773GiB/s) | 20260505 13:39:17.227 | INFO | Starting a round of Baz IO | 20260505 13:39:20.448 | INFO | Populating RTMetrics report | 20260505 13:39:20.448 | INFO | RTMetrics report populated | 20260505 13:39:20.448 | INFO | Opening tmp RTMetrics json file | 20260505 13:39:20.451 | INFO | Serializing RTMetrics to json | 20260505 13:39:20.453 | INFO | Serialization done, swapping tmp file now... | 20260505 13:39:20.454 | INFO | RTMetrics done | 20260505 13:39:20.565 | INFO | Finished a round of Baz IO: Wrote 2.29 GiB in 3.337 seconds | 20260505 13:39:20.565 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.04% / 0.00% / 76.96% | 20260505 13:39:20.565 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.668248% / 1.58946e-05% / 8.34465e-05% | 20260505 13:39:20.565 | INFO | Baz IO overall: | 20260505 13:39:20.565 | INFO | BytesWritten : 2164294391303 | 20260505 13:39:20.565 | INFO | NumEvents : 1029559117450 | 20260505 13:39:22.228 | INFO | Analyzing chunk frames = [3633152,3633664) | 20260505 13:39:22.228 | INFO | MemInfo: { | 20260505 13:39:22.228 | INFO | "gpu" : " 17 GB", | 20260505 13:39:22.228 | INFO | "huge" : "", | 20260505 13:39:22.228 | INFO | "huge_shared" : " 27 GB", | 20260505 13:39:22.228 | INFO | "malloc" : " 14 GB", | 20260505 13:39:22.228 | INFO | "pinned" : " 29 GB" | 20260505 13:39:22.228 | INFO | } | 20260505 13:39:22.228 | INFO | Perf report for the last 5.1417 seconds: | 20260505 13:39:22.228 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:39:22.228 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:22.228 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:22.228 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:22.228 | INFO | ANALYSIS : 64.3%, 3.0, 99.6ms, 0 | 20260505 13:39:22.228 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:22.228 | INFO | RT_METRICS : 45.1%, 1.0, 23.7ms, 0 | 20260505 13:39:22.228 | INFO | PRE_HQ : 60.9%, 1.8, 57.7ms, 0 | 20260505 13:39:22.228 | INFO | BAZWRITER : 64.2%, 4.0, 133.4ms, 0 | 20260505 13:39:22.228 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 13:39:22.228 | INFO | Uploaded 36GiB to GPU in 2.18497s (16.4762GiB/s) | 20260505 13:39:22.229 | INFO | Downloaded 38GiB to GPU in 2.27749s (16.6851GiB/s) | 20260505 13:39:27.374 | INFO | Analyzing chunk frames = [3633664,3634176) | 20260505 13:39:27.374 | INFO | MemInfo: { | 20260505 13:39:27.374 | INFO | "gpu" : " 17 GB", | 20260505 13:39:27.374 | INFO | "huge" : "", | 20260505 13:39:27.374 | INFO | "huge_shared" : " 27 GB", | 20260505 13:39:27.374 | INFO | "malloc" : " 14 GB", | 20260505 13:39:27.374 | INFO | "pinned" : " 29 GB" | 20260505 13:39:27.374 | INFO | } | 20260505 13:39:27.374 | INFO | Perf report for the last 5.1461 seconds: | 20260505 13:39:27.374 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:39:27.374 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:27.374 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:27.374 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:27.374 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 13:39:27.374 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:27.374 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:27.374 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:39:27.374 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:27.374 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:27.375 | INFO | Uploaded 36GiB to GPU in 2.1986s (16.374GiB/s) | 20260505 13:39:27.375 | INFO | Downloaded 34GiB to GPU in 2.03595s (16.6998GiB/s) | 20260505 13:39:32.512 | INFO | Analyzing chunk frames = [3634176,3634688) | 20260505 13:39:32.512 | INFO | MemInfo: { | 20260505 13:39:32.512 | INFO | "gpu" : " 17 GB", | 20260505 13:39:32.512 | INFO | "huge" : "", | 20260505 13:39:32.512 | INFO | "huge_shared" : " 27 GB", | 20260505 13:39:32.512 | INFO | "malloc" : " 14 GB", | 20260505 13:39:32.512 | INFO | "pinned" : " 29 GB" | 20260505 13:39:32.512 | INFO | } | 20260505 13:39:32.512 | INFO | Perf report for the last 5.13745 seconds: | 20260505 13:39:32.512 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:39:32.512 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:32.512 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:32.512 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:32.512 | INFO | ANALYSIS : 63.1%, 3.0, 98.1ms, 0 | 20260505 13:39:32.512 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:32.512 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:32.512 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:39:32.512 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:32.512 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:32.512 | INFO | Uploaded 36GiB to GPU in 2.18275s (16.4929GiB/s) | 20260505 13:39:32.512 | INFO | Downloaded 34GiB to GPU in 2.05751s (16.5249GiB/s) | 20260505 13:39:37.639 | INFO | Analyzing chunk frames = [3634688,3635200) | 20260505 13:39:37.640 | INFO | MemInfo: { | 20260505 13:39:37.640 | INFO | "gpu" : " 17 GB", | 20260505 13:39:37.640 | INFO | "huge" : "", | 20260505 13:39:37.640 | INFO | "huge_shared" : " 27 GB", | 20260505 13:39:37.640 | INFO | "malloc" : " 14 GB", | 20260505 13:39:37.640 | INFO | "pinned" : " 29 GB" | 20260505 13:39:37.640 | INFO | } | 20260505 13:39:37.640 | INFO | Perf report for the last 5.12816 seconds: | 20260505 13:39:37.640 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:39:37.640 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:37.640 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:37.640 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:37.640 | INFO | ANALYSIS : 63.1%, 3.0, 97.8ms, 0 | 20260505 13:39:37.640 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:37.640 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:37.640 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:39:37.640 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:37.640 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:37.640 | INFO | Uploaded 36GiB to GPU in 2.19125s (16.429GiB/s) | 20260505 13:39:37.640 | INFO | Downloaded 34GiB to GPU in 2.13296s (15.9403GiB/s) | 20260505 13:39:42.777 | INFO | Analyzing chunk frames = [3635200,3635712) | 20260505 13:39:42.777 | INFO | MemInfo: { | 20260505 13:39:42.777 | INFO | "gpu" : " 17 GB", | 20260505 13:39:42.777 | INFO | "huge" : "", | 20260505 13:39:42.777 | INFO | "huge_shared" : " 27 GB", | 20260505 13:39:42.777 | INFO | "malloc" : " 14 GB", | 20260505 13:39:42.777 | INFO | "pinned" : " 29 GB" | 20260505 13:39:42.777 | INFO | } | 20260505 13:39:42.777 | INFO | Perf report for the last 5.13749 seconds: | 20260505 13:39:42.777 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:39:42.777 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:42.777 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:42.777 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:42.777 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 13:39:42.777 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:42.777 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:42.777 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:39:42.777 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:42.777 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:42.778 | INFO | Uploaded 36GiB to GPU in 2.18234s (16.4961GiB/s) | 20260505 13:39:42.778 | INFO | Downloaded 34GiB to GPU in 2.12902s (15.9698GiB/s) | 20260505 13:39:47.924 | INFO | Analyzing chunk frames = [3635712,3636224) | 20260505 13:39:47.924 | INFO | MemInfo: { | 20260505 13:39:47.924 | INFO | "gpu" : " 17 GB", | 20260505 13:39:47.924 | INFO | "huge" : "", | 20260505 13:39:47.924 | INFO | "huge_shared" : " 27 GB", | 20260505 13:39:47.924 | INFO | "malloc" : " 14 GB", | 20260505 13:39:47.924 | INFO | "pinned" : " 29 GB" | 20260505 13:39:47.924 | INFO | } | 20260505 13:39:47.924 | INFO | Perf report for the last 5.14686 seconds: | 20260505 13:39:47.924 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:39:47.924 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:47.924 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:47.924 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:47.924 | INFO | ANALYSIS : 63.1%, 3.0, 98.7ms, 0 | 20260505 13:39:47.924 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:47.924 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:47.924 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 13:39:47.924 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:47.924 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:47.925 | INFO | Uploaded 36GiB to GPU in 2.17175s (16.5765GiB/s) | 20260505 13:39:47.925 | INFO | Downloaded 34GiB to GPU in 2.11549s (16.072GiB/s) | 20260505 13:39:53.071 | INFO | Analyzing chunk frames = [3636224,3636736) | 20260505 13:39:53.071 | INFO | MemInfo: { | 20260505 13:39:53.071 | INFO | "gpu" : " 17 GB", | 20260505 13:39:53.071 | INFO | "huge" : "", | 20260505 13:39:53.071 | INFO | "huge_shared" : " 27 GB", | 20260505 13:39:53.071 | INFO | "malloc" : " 14 GB", | 20260505 13:39:53.071 | INFO | "pinned" : " 29 GB" | 20260505 13:39:53.071 | INFO | } | 20260505 13:39:53.071 | INFO | Perf report for the last 5.14695 seconds: | 20260505 13:39:53.071 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:39:53.071 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:53.071 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:53.071 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:53.071 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 13:39:53.071 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:53.071 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:53.071 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 13:39:53.071 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:53.071 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:53.071 | INFO | Uploaded 36GiB to GPU in 2.15523s (16.7035GiB/s) | 20260505 13:39:53.072 | INFO | Downloaded 34GiB to GPU in 2.18236s (15.5794GiB/s) | 20260505 13:39:58.213 | INFO | Analyzing chunk frames = [3636736,3637248) | 20260505 13:39:58.213 | INFO | MemInfo: { | 20260505 13:39:58.213 | INFO | "gpu" : " 17 GB", | 20260505 13:39:58.213 | INFO | "huge" : "", | 20260505 13:39:58.213 | INFO | "huge_shared" : " 27 GB", | 20260505 13:39:58.213 | INFO | "malloc" : " 14 GB", | 20260505 13:39:58.213 | INFO | "pinned" : " 29 GB" | 20260505 13:39:58.213 | INFO | } | 20260505 13:39:58.213 | INFO | Perf report for the last 5.14213 seconds: | 20260505 13:39:58.213 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:39:58.213 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:58.213 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:58.213 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:58.213 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 13:39:58.213 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:58.213 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:58.213 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:39:58.213 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:58.213 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:39:58.214 | INFO | Uploaded 36GiB to GPU in 2.2283s (16.1558GiB/s) | 20260505 13:39:58.214 | INFO | Downloaded 34GiB to GPU in 2.09908s (16.1976GiB/s) | 20260505 13:39:58.214 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 13:39:58.214 | INFO | Total mem usage: 15.3 GiB | 20260505 13:39:58.214 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 13:39:58.214 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.64%) | 20260505 13:39:58.214 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 13:39:58.214 | INFO | High Water Breakdown: | 20260505 13:39:58.214 | INFO | 60.9% : 8.68 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 13:39:58.214 | INFO | 38.9% : 5.54 GiB ( 83.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 13:39:58.214 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 13:39:58.214 | INFO | ------------------------------------------ | 20260505 13:39:58.214 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:39:58.214 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 13:39:58.214 | INFO | Total mem usage: 26.5 GiB | 20260505 13:39:58.214 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 13:39:58.214 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 13:39:58.214 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 13:39:58.214 | INFO | High Water Breakdown: | 20260505 13:39:58.214 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:39:58.214 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:39:58.214 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:39:58.214 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:39:58.214 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:39:58.214 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:39:58.214 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 13:39:58.214 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 13:39:58.214 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:39:58.214 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 13:39:58.214 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 13:39:58.214 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 13:39:58.214 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 13:39:58.214 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 13:39:58.214 | INFO | ------------------------------------------ | 20260505 13:39:58.214 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:39:58.214 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 13:39:58.214 | INFO | Total mem usage: 25.6 GiB | 20260505 13:39:58.214 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 13:39:58.214 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 13:39:58.214 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 13:39:58.214 | INFO | High Water Breakdown: | 20260505 13:39:58.214 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 13:39:58.214 | INFO | ------------------------------------------ | 20260505 13:39:58.214 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:39:58.214 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 13:39:58.214 | INFO | Total mem usage: 15.7 GiB | 20260505 13:39:58.214 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 13:39:58.214 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 13:39:58.214 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 13:39:58.214 | INFO | High Water Breakdown: | 20260505 13:39:58.214 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 13:39:58.214 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 13:39:58.214 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 13:39:58.214 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 13:39:58.215 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 13:39:58.215 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 13:39:58.215 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 13:39:58.215 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 13:39:58.215 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 13:39:58.215 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:39:58.215 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 13:39:58.215 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:39:58.215 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 13:39:58.215 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 13:39:58.215 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 13:39:58.215 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 13:39:58.215 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:39:58.215 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 13:39:58.215 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:39:58.215 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:39:58.215 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:39:58.215 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 13:39:58.215 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 13:39:58.215 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:39:58.215 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 13:39:58.215 | INFO | ------------------------------------------ | 20260505 13:39:58.215 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:39:58.215 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 13:39:58.215 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:39:58.215 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:39:58.215 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.66e+01, 1.38e+01 / 2.46e+00 | 20260505 13:39:58.215 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 1.01e-01, 1.05e-02 / 4.66e-03 | 20260505 13:39:58.215 | INFO | PerfReport: 11.75%, Baseline -- 3.75e+00 / 4.48e+00, 5.89e-01 / 1.87e-01 | 20260505 13:39:58.215 | INFO | PerfReport: 3.29%, Histogram -- 1.05e+00 / 1.50e+00, 2.00e-01 / 1.02e-01 | 20260505 13:39:58.215 | INFO | PerfReport: 16.04%, DME -- 4.09e+01 / 4.61e+01, 4.37e+00 / 2.66e+00 | 20260505 13:39:58.215 | INFO | PerfReport: 56.41%, FrameLabeling -- 1.80e+01 / 1.96e+01, 1.45e+00 / 1.88e+00 | 20260505 13:39:58.215 | INFO | PerfReport: 7.42%, PulseAccumulating -- 2.37e+00 / 2.94e+00, 4.53e-01 / 1.83e-01 | 20260505 13:39:58.215 | INFO | PerfReport: 3.56%, Metrics -- 1.13e+00 / 1.44e+00, 3.21e-01 / 1.50e-01 | 20260505 13:39:58.215 | INFO | PerfReport: 1.25%, PulseSerializing -- 3.99e-01 / 4.49e-01, 7.18e-02 / 3.34e-02 | 20260505 13:39:58.215 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 13:39:58.215 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:39:58.215 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:39:58.215 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.58e+01 / 0.00e+00 | 20260505 13:39:58.215 | INFO | PerfReport: 22.56%, Upload -- 2.27e+01 / 0.00e+00, 8.03e+00 / 0.00e+00 | 20260505 13:39:58.215 | INFO | PerfReport: 21.95%, Download -- 2.21e+01 / 0.00e+00, 8.06e+00 / 0.00e+00 | 20260505 13:39:58.215 | INFO | PerfReport: 31.65%, Compute -- 3.19e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 13:39:58.215 | INFO | PerfReport: 2.23%, LockUpload -- 2.25e+00 / 0.00e+00, 8.04e+00 / 0.00e+00 | 20260505 13:39:58.215 | INFO | PerfReport: 1.40%, LockDownload -- 1.41e+00 / 0.00e+00, 2.92e+00 / 0.00e+00 | 20260505 13:39:58.215 | INFO | PerfReport: 20.21%, LockCompute -- 2.04e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 13:39:58.215 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.12e-02 / 0.00e+00 | 20260505 13:39:58.349 | INFO | Starting a round of Baz IO | 20260505 13:40:01.571 | INFO | Populating RTMetrics report | 20260505 13:40:01.571 | INFO | RTMetrics report populated | 20260505 13:40:01.571 | INFO | Opening tmp RTMetrics json file | 20260505 13:40:01.587 | INFO | Serializing RTMetrics to json | 20260505 13:40:01.590 | INFO | Serialization done, swapping tmp file now... | 20260505 13:40:01.591 | INFO | RTMetrics done | 20260505 13:40:01.676 | INFO | Finished a round of Baz IO: Wrote 2.291 GiB in 3.327 seconds | 20260505 13:40:01.677 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.03% / 0.00% / 76.96% | 20260505 13:40:01.677 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.667266% / 3.97364e-06% / 7.54992e-05% | 20260505 13:40:01.677 | INFO | Baz IO overall: | 20260505 13:40:01.677 | INFO | BytesWritten : 2166754253149 | 20260505 13:40:01.677 | INFO | NumEvents : 1030722842320 | 20260505 13:40:03.340 | INFO | Analyzing chunk frames = [3637248,3637760) | 20260505 13:40:03.340 | INFO | MemInfo: { | 20260505 13:40:03.340 | INFO | "gpu" : " 17 GB", | 20260505 13:40:03.340 | INFO | "huge" : "", | 20260505 13:40:03.340 | INFO | "huge_shared" : " 27 GB", | 20260505 13:40:03.340 | INFO | "malloc" : " 14 GB", | 20260505 13:40:03.340 | INFO | "pinned" : " 29 GB" | 20260505 13:40:03.340 | INFO | } | 20260505 13:40:03.340 | INFO | Perf report for the last 5.12676 seconds: | 20260505 13:40:03.340 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:40:03.340 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:03.340 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:03.340 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:03.340 | INFO | ANALYSIS : 64.4%, 3.0, 99.8ms, 0 | 20260505 13:40:03.340 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:03.340 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260505 13:40:03.340 | INFO | PRE_HQ : 61.2%, 1.8, 58.0ms, 0 | 20260505 13:40:03.340 | INFO | BAZWRITER : 64.2%, 4.0, 131.7ms, 0 | 20260505 13:40:03.340 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260505 13:40:03.340 | INFO | Uploaded 36GiB to GPU in 2.17405s (16.559GiB/s) | 20260505 13:40:03.340 | INFO | Downloaded 38GiB to GPU in 2.26321s (16.7903GiB/s) | 20260505 13:40:08.484 | INFO | Analyzing chunk frames = [3637760,3638272) | 20260505 13:40:08.484 | INFO | MemInfo: { | 20260505 13:40:08.484 | INFO | "gpu" : " 17 GB", | 20260505 13:40:08.485 | INFO | "huge" : "", | 20260505 13:40:08.485 | INFO | "huge_shared" : " 27 GB", | 20260505 13:40:08.485 | INFO | "malloc" : " 14 GB", | 20260505 13:40:08.485 | INFO | "pinned" : " 29 GB" | 20260505 13:40:08.485 | INFO | } | 20260505 13:40:08.485 | INFO | Perf report for the last 5.14453 seconds: | 20260505 13:40:08.485 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:40:08.485 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:08.485 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:08.485 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:08.485 | INFO | ANALYSIS : 63.5%, 3.0, 99.3ms, 0 | 20260505 13:40:08.485 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:08.485 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:08.485 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 13:40:08.485 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:08.485 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:08.485 | INFO | Uploaded 36GiB to GPU in 2.17338s (16.5641GiB/s) | 20260505 13:40:08.485 | INFO | Downloaded 34GiB to GPU in 2.0479s (16.6024GiB/s) | 20260505 13:40:13.626 | INFO | Analyzing chunk frames = [3638272,3638784) | 20260505 13:40:13.626 | INFO | MemInfo: { | 20260505 13:40:13.626 | INFO | "gpu" : " 17 GB", | 20260505 13:40:13.626 | INFO | "huge" : "", | 20260505 13:40:13.626 | INFO | "huge_shared" : " 27 GB", | 20260505 13:40:13.626 | INFO | "malloc" : " 14 GB", | 20260505 13:40:13.626 | INFO | "pinned" : " 29 GB" | 20260505 13:40:13.626 | INFO | } | 20260505 13:40:13.626 | INFO | Perf report for the last 5.14101 seconds: | 20260505 13:40:13.626 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:40:13.626 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:13.626 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:13.626 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:13.626 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 13:40:13.626 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:13.626 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:13.626 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 13:40:13.626 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:13.626 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:13.626 | INFO | Uploaded 36GiB to GPU in 2.23483s (16.1086GiB/s) | 20260505 13:40:13.626 | INFO | Downloaded 34GiB to GPU in 2.11718s (16.0591GiB/s) | 20260505 13:40:18.766 | INFO | Analyzing chunk frames = [3638784,3639296) | 20260505 13:40:18.766 | INFO | MemInfo: { | 20260505 13:40:18.766 | INFO | "gpu" : " 17 GB", | 20260505 13:40:18.766 | INFO | "huge" : "", | 20260505 13:40:18.766 | INFO | "huge_shared" : " 27 GB", | 20260505 13:40:18.766 | INFO | "malloc" : " 14 GB", | 20260505 13:40:18.766 | INFO | "pinned" : " 29 GB" | 20260505 13:40:18.766 | INFO | } | 20260505 13:40:18.766 | INFO | Perf report for the last 5.14038 seconds: | 20260505 13:40:18.766 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:40:18.766 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:18.766 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:18.766 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:18.766 | INFO | ANALYSIS : 62.7%, 3.0, 97.5ms, 0 | 20260505 13:40:18.766 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:18.766 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:18.766 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 13:40:18.766 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:18.766 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:18.766 | INFO | Uploaded 36GiB to GPU in 2.18304s (16.4907GiB/s) | 20260505 13:40:18.766 | INFO | Downloaded 34GiB to GPU in 2.13205s (15.9471GiB/s) | 20260505 13:40:23.893 | INFO | Analyzing chunk frames = [3639296,3639808) | 20260505 13:40:23.893 | INFO | MemInfo: { | 20260505 13:40:23.893 | INFO | "gpu" : " 17 GB", | 20260505 13:40:23.893 | INFO | "huge" : "", | 20260505 13:40:23.893 | INFO | "huge_shared" : " 27 GB", | 20260505 13:40:23.893 | INFO | "malloc" : " 14 GB", | 20260505 13:40:23.893 | INFO | "pinned" : " 29 GB" | 20260505 13:40:23.893 | INFO | } | 20260505 13:40:23.893 | INFO | Perf report for the last 5.12674 seconds: | 20260505 13:40:23.893 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:40:23.893 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:23.893 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:23.893 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:23.893 | INFO | ANALYSIS : 62.9%, 3.0, 98.0ms, 0 | 20260505 13:40:23.893 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:23.893 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:23.893 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 13:40:23.893 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:23.893 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:23.893 | INFO | Uploaded 36GiB to GPU in 2.15076s (16.7383GiB/s) | 20260505 13:40:23.893 | INFO | Downloaded 34GiB to GPU in 2.12269s (16.0174GiB/s) | 20260505 13:40:29.036 | INFO | Analyzing chunk frames = [3639808,3640320) | 20260505 13:40:29.036 | INFO | MemInfo: { | 20260505 13:40:29.036 | INFO | "gpu" : " 17 GB", | 20260505 13:40:29.036 | INFO | "huge" : "", | 20260505 13:40:29.036 | INFO | "huge_shared" : " 27 GB", | 20260505 13:40:29.036 | INFO | "malloc" : " 14 GB", | 20260505 13:40:29.036 | INFO | "pinned" : " 29 GB" | 20260505 13:40:29.036 | INFO | } | 20260505 13:40:29.036 | INFO | Perf report for the last 5.14302 seconds: | 20260505 13:40:29.036 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:40:29.036 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:29.036 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:29.036 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:29.036 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 13:40:29.036 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:29.036 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:29.036 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 13:40:29.036 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:29.036 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:29.036 | INFO | Uploaded 36GiB to GPU in 2.19763s (16.3813GiB/s) | 20260505 13:40:29.036 | INFO | Downloaded 34GiB to GPU in 2.11861s (16.0482GiB/s) | 20260505 13:40:34.188 | INFO | Analyzing chunk frames = [3640320,3640832) | 20260505 13:40:34.188 | INFO | MemInfo: { | 20260505 13:40:34.188 | INFO | "gpu" : " 17 GB", | 20260505 13:40:34.188 | INFO | "huge" : "", | 20260505 13:40:34.188 | INFO | "huge_shared" : " 27 GB", | 20260505 13:40:34.188 | INFO | "malloc" : " 14 GB", | 20260505 13:40:34.188 | INFO | "pinned" : " 29 GB" | 20260505 13:40:34.188 | INFO | } | 20260505 13:40:34.188 | INFO | Perf report for the last 5.15248 seconds: | 20260505 13:40:34.188 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:40:34.188 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:34.188 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:34.188 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:34.188 | INFO | ANALYSIS : 63.2%, 3.0, 98.9ms, 0 | 20260505 13:40:34.188 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:34.188 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:34.189 | INFO | PRE_HQ : 29.7%, 1.0, 15.5ms, 0 | 20260505 13:40:34.189 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:34.189 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:34.189 | INFO | Uploaded 36GiB to GPU in 2.14351s (16.7949GiB/s) | 20260505 13:40:34.189 | INFO | Downloaded 34GiB to GPU in 2.18111s (15.5884GiB/s) | 20260505 13:40:39.325 | INFO | Analyzing chunk frames = [3640832,3641344) | 20260505 13:40:39.325 | INFO | MemInfo: { | 20260505 13:40:39.325 | INFO | "gpu" : " 17 GB", | 20260505 13:40:39.325 | INFO | "huge" : "", | 20260505 13:40:39.325 | INFO | "huge_shared" : " 27 GB", | 20260505 13:40:39.325 | INFO | "malloc" : " 14 GB", | 20260505 13:40:39.325 | INFO | "pinned" : " 29 GB" | 20260505 13:40:39.325 | INFO | } | 20260505 13:40:39.325 | INFO | Perf report for the last 5.13673 seconds: | 20260505 13:40:39.325 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:40:39.325 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:39.325 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:39.325 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:39.325 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 13:40:39.325 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:39.325 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:39.325 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 13:40:39.325 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:39.325 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:39.325 | INFO | Uploaded 36GiB to GPU in 2.21652s (16.2417GiB/s) | 20260505 13:40:39.326 | INFO | Downloaded 34GiB to GPU in 2.10134s (16.1801GiB/s) | 20260505 13:40:39.464 | INFO | Starting a round of Baz IO | 20260505 13:40:42.683 | INFO | Populating RTMetrics report | 20260505 13:40:42.684 | INFO | RTMetrics report populated | 20260505 13:40:42.684 | INFO | Opening tmp RTMetrics json file | 20260505 13:40:42.726 | INFO | Serializing RTMetrics to json | 20260505 13:40:42.769 | INFO | Serialization done, swapping tmp file now... | 20260505 13:40:42.773 | INFO | RTMetrics done | 20260505 13:40:42.790 | INFO | Finished a round of Baz IO: Wrote 2.29 GiB in 3.325 seconds | 20260505 13:40:42.790 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.03% / 0.00% / 76.97% | 20260505 13:40:42.790 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.666233% / 7.94729e-06% / 8.34465e-05% | 20260505 13:40:42.790 | INFO | Baz IO overall: | 20260505 13:40:42.790 | INFO | BytesWritten : 2169212628330 | 20260505 13:40:42.790 | INFO | NumEvents : 1031885313102 | 20260505 13:40:44.457 | INFO | Analyzing chunk frames = [3641344,3641856) | 20260505 13:40:44.457 | INFO | MemInfo: { | 20260505 13:40:44.457 | INFO | "gpu" : " 17 GB", | 20260505 13:40:44.457 | INFO | "huge" : "", | 20260505 13:40:44.457 | INFO | "huge_shared" : " 27 GB", | 20260505 13:40:44.457 | INFO | "malloc" : " 14 GB", | 20260505 13:40:44.457 | INFO | "pinned" : " 29 GB" | 20260505 13:40:44.457 | INFO | } | 20260505 13:40:44.457 | INFO | Perf report for the last 5.13195 seconds: | 20260505 13:40:44.457 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:40:44.457 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:44.457 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:44.457 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:44.457 | INFO | ANALYSIS : 64.3%, 3.0, 99.6ms, 0 | 20260505 13:40:44.457 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:44.457 | INFO | RT_METRICS : 47.0%, 1.0, 24.6ms, 0 | 20260505 13:40:44.457 | INFO | PRE_HQ : 60.9%, 1.8, 58.0ms, 0 | 20260505 13:40:44.457 | INFO | BAZWRITER : 64.2%, 4.0, 132.1ms, 0 | 20260505 13:40:44.457 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 13:40:44.457 | INFO | Uploaded 36GiB to GPU in 2.20131s (16.3539GiB/s) | 20260505 13:40:44.457 | INFO | Downloaded 38GiB to GPU in 2.26142s (16.8036GiB/s) | 20260505 13:40:49.591 | INFO | Analyzing chunk frames = [3641856,3642368) | 20260505 13:40:49.591 | INFO | MemInfo: { | 20260505 13:40:49.591 | INFO | "gpu" : " 17 GB", | 20260505 13:40:49.591 | INFO | "huge" : "", | 20260505 13:40:49.591 | INFO | "huge_shared" : " 27 GB", | 20260505 13:40:49.591 | INFO | "malloc" : " 14 GB", | 20260505 13:40:49.591 | INFO | "pinned" : " 29 GB" | 20260505 13:40:49.591 | INFO | } | 20260505 13:40:49.592 | INFO | Perf report for the last 5.13449 seconds: | 20260505 13:40:49.592 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:40:49.592 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:49.592 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:49.592 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:49.592 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 13:40:49.592 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:49.592 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:49.592 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 13:40:49.592 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:49.592 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:49.592 | INFO | Uploaded 36GiB to GPU in 2.1851s (16.4752GiB/s) | 20260505 13:40:49.592 | INFO | Downloaded 34GiB to GPU in 2.0503s (16.5829GiB/s) | 20260505 13:40:54.731 | INFO | Analyzing chunk frames = [3642368,3642880) | 20260505 13:40:54.731 | INFO | MemInfo: { | 20260505 13:40:54.731 | INFO | "gpu" : " 17 GB", | 20260505 13:40:54.731 | INFO | "huge" : "", | 20260505 13:40:54.731 | INFO | "huge_shared" : " 27 GB", | 20260505 13:40:54.731 | INFO | "malloc" : " 14 GB", | 20260505 13:40:54.731 | INFO | "pinned" : " 29 GB" | 20260505 13:40:54.731 | INFO | } | 20260505 13:40:54.732 | INFO | Perf report for the last 5.14 seconds: | 20260505 13:40:54.732 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:40:54.732 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:54.732 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:54.732 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:54.732 | INFO | ANALYSIS : 63.1%, 3.0, 98.2ms, 0 | 20260505 13:40:54.732 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:54.732 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:54.732 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 13:40:54.732 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:54.732 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:54.732 | INFO | Uploaded 36GiB to GPU in 2.20636s (16.3165GiB/s) | 20260505 13:40:54.732 | INFO | Downloaded 34GiB to GPU in 2.11926s (16.0433GiB/s) | 20260505 13:40:59.876 | INFO | Analyzing chunk frames = [3642880,3643392) | 20260505 13:40:59.876 | INFO | MemInfo: { | 20260505 13:40:59.876 | INFO | "gpu" : " 17 GB", | 20260505 13:40:59.876 | INFO | "huge" : "", | 20260505 13:40:59.876 | INFO | "huge_shared" : " 27 GB", | 20260505 13:40:59.876 | INFO | "malloc" : " 14 GB", | 20260505 13:40:59.876 | INFO | "pinned" : " 29 GB" | 20260505 13:40:59.876 | INFO | } | 20260505 13:40:59.877 | INFO | Perf report for the last 5.14506 seconds: | 20260505 13:40:59.877 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:40:59.877 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:59.877 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:59.877 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:59.877 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260505 13:40:59.877 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:59.877 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:59.877 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 13:40:59.877 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:59.877 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:40:59.877 | INFO | Uploaded 36GiB to GPU in 2.19559s (16.3965GiB/s) | 20260505 13:40:59.877 | INFO | Downloaded 34GiB to GPU in 2.13405s (15.9322GiB/s) | 20260505 13:41:05.019 | INFO | Analyzing chunk frames = [3643392,3643904) | 20260505 13:41:05.019 | INFO | MemInfo: { | 20260505 13:41:05.019 | INFO | "gpu" : " 17 GB", | 20260505 13:41:05.019 | INFO | "huge" : "", | 20260505 13:41:05.019 | INFO | "huge_shared" : " 27 GB", | 20260505 13:41:05.019 | INFO | "malloc" : " 14 GB", | 20260505 13:41:05.019 | INFO | "pinned" : " 29 GB" | 20260505 13:41:05.019 | INFO | } | 20260505 13:41:05.019 | INFO | Perf report for the last 5.14257 seconds: | 20260505 13:41:05.019 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:41:05.019 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:05.019 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:05.019 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:05.019 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260505 13:41:05.019 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:05.019 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:05.019 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 13:41:05.019 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:05.019 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:05.019 | INFO | Uploaded 36GiB to GPU in 2.14371s (16.7933GiB/s) | 20260505 13:41:05.019 | INFO | Downloaded 34GiB to GPU in 2.12603s (15.9923GiB/s) | 20260505 13:41:10.155 | INFO | Analyzing chunk frames = [3643904,3644416) | 20260505 13:41:10.155 | INFO | MemInfo: { | 20260505 13:41:10.155 | INFO | "gpu" : " 17 GB", | 20260505 13:41:10.155 | INFO | "huge" : "", | 20260505 13:41:10.155 | INFO | "huge_shared" : " 27 GB", | 20260505 13:41:10.155 | INFO | "malloc" : " 14 GB", | 20260505 13:41:10.155 | INFO | "pinned" : " 29 GB" | 20260505 13:41:10.155 | INFO | } | 20260505 13:41:10.156 | INFO | Perf report for the last 5.13637 seconds: | 20260505 13:41:10.156 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:41:10.156 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:10.156 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:10.156 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:10.156 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 13:41:10.156 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:10.156 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:10.156 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 13:41:10.156 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:10.156 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:10.156 | INFO | Uploaded 36GiB to GPU in 2.20579s (16.3207GiB/s) | 20260505 13:41:10.156 | INFO | Downloaded 34GiB to GPU in 2.11452s (16.0793GiB/s) | 20260505 13:41:15.290 | INFO | Analyzing chunk frames = [3644416,3644928) | 20260505 13:41:15.290 | INFO | MemInfo: { | 20260505 13:41:15.290 | INFO | "gpu" : " 17 GB", | 20260505 13:41:15.290 | INFO | "huge" : "", | 20260505 13:41:15.290 | INFO | "huge_shared" : " 27 GB", | 20260505 13:41:15.290 | INFO | "malloc" : " 14 GB", | 20260505 13:41:15.290 | INFO | "pinned" : " 29 GB" | 20260505 13:41:15.290 | INFO | } | 20260505 13:41:15.290 | INFO | Perf report for the last 5.1344 seconds: | 20260505 13:41:15.290 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:41:15.290 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:15.290 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:15.290 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:15.290 | INFO | ANALYSIS : 63.5%, 3.0, 99.0ms, 0 | 20260505 13:41:15.290 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:15.290 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:15.290 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 13:41:15.290 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:15.290 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:15.290 | INFO | Uploaded 36GiB to GPU in 2.14374s (16.7931GiB/s) | 20260505 13:41:15.290 | INFO | Downloaded 34GiB to GPU in 2.17603s (15.6248GiB/s) | 20260505 13:41:20.437 | INFO | Analyzing chunk frames = [3644928,3645440) | 20260505 13:41:20.437 | INFO | MemInfo: { | 20260505 13:41:20.437 | INFO | "gpu" : " 17 GB", | 20260505 13:41:20.437 | INFO | "huge" : "", | 20260505 13:41:20.437 | INFO | "huge_shared" : " 27 GB", | 20260505 13:41:20.437 | INFO | "malloc" : " 14 GB", | 20260505 13:41:20.437 | INFO | "pinned" : " 29 GB" | 20260505 13:41:20.437 | INFO | } | 20260505 13:41:20.437 | INFO | Perf report for the last 5.1475 seconds: | 20260505 13:41:20.437 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:41:20.437 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:20.437 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:20.437 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:20.438 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 13:41:20.438 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:20.438 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:20.438 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:41:20.438 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:20.438 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:20.438 | INFO | Uploaded 36GiB to GPU in 2.18297s (16.4913GiB/s) | 20260505 13:41:20.438 | INFO | Downloaded 34GiB to GPU in 2.10656s (16.1401GiB/s) | 20260505 13:41:20.438 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 13:41:20.438 | INFO | Total mem usage: 15.3 GiB | 20260505 13:41:20.438 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 13:41:20.438 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.65%) | 20260505 13:41:20.438 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 13:41:20.438 | INFO | High Water Breakdown: | 20260505 13:41:20.438 | INFO | 61% : 8.68 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 13:41:20.438 | INFO | 38.9% : 5.54 GiB ( 83.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 13:41:20.438 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 13:41:20.438 | INFO | ------------------------------------------ | 20260505 13:41:20.438 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:41:20.438 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 13:41:20.438 | INFO | Total mem usage: 26.5 GiB | 20260505 13:41:20.438 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 13:41:20.438 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 13:41:20.438 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 13:41:20.438 | INFO | High Water Breakdown: | 20260505 13:41:20.438 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:41:20.438 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:41:20.438 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:41:20.438 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:41:20.438 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:41:20.438 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:41:20.438 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 13:41:20.438 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 13:41:20.438 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:41:20.438 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 13:41:20.438 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 13:41:20.439 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 13:41:20.439 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 13:41:20.439 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 13:41:20.439 | INFO | ------------------------------------------ | 20260505 13:41:20.439 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:41:20.439 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 13:41:20.439 | INFO | Total mem usage: 25.6 GiB | 20260505 13:41:20.439 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 13:41:20.439 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 13:41:20.439 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 13:41:20.439 | INFO | High Water Breakdown: | 20260505 13:41:20.439 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 13:41:20.439 | INFO | ------------------------------------------ | 20260505 13:41:20.439 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:41:20.439 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 13:41:20.439 | INFO | Total mem usage: 15.7 GiB | 20260505 13:41:20.439 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 13:41:20.439 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 13:41:20.439 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 13:41:20.439 | INFO | High Water Breakdown: | 20260505 13:41:20.439 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 13:41:20.439 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 13:41:20.439 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 13:41:20.439 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 13:41:20.439 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 13:41:20.439 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 13:41:20.439 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 13:41:20.439 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 13:41:20.439 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 13:41:20.439 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:41:20.439 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 13:41:20.439 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:41:20.439 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 13:41:20.439 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 13:41:20.439 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 13:41:20.439 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 13:41:20.439 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:41:20.439 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 13:41:20.439 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:41:20.439 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:41:20.439 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:41:20.439 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 13:41:20.439 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 13:41:20.439 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:41:20.439 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 13:41:20.439 | INFO | ------------------------------------------ | 20260505 13:41:20.439 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:41:20.439 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 13:41:20.439 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:41:20.439 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:41:20.439 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.67e+01, 1.38e+01 / 1.46e+00 | 20260505 13:41:20.439 | INFO | PerfReport: 0.29%, AggregateStats -- 9.31e-02 / 1.01e-01, 1.14e-02 / 3.90e-03 | 20260505 13:41:20.439 | INFO | PerfReport: 11.78%, Baseline -- 3.76e+00 / 4.49e+00, 5.93e-01 / 1.85e-01 | 20260505 13:41:20.439 | INFO | PerfReport: 3.31%, Histogram -- 1.05e+00 / 1.51e+00, 2.01e-01 / 7.78e-02 | 20260505 13:41:20.439 | INFO | PerfReport: 15.95%, DME -- 4.07e+01 / 4.57e+01, 4.67e+00 / 1.72e+00 | 20260505 13:41:20.439 | INFO | PerfReport: 56.43%, FrameLabeling -- 1.80e+01 / 2.02e+01, 1.46e+00 / 1.31e+00 | 20260505 13:41:20.439 | INFO | PerfReport: 7.42%, PulseAccumulating -- 2.37e+00 / 2.88e+00, 4.48e-01 / 1.75e-01 | 20260505 13:41:20.439 | INFO | PerfReport: 3.56%, Metrics -- 1.14e+00 / 1.43e+00, 3.17e-01 / 1.58e-01 | 20260505 13:41:20.439 | INFO | PerfReport: 1.25%, PulseSerializing -- 3.99e-01 / 4.49e-01, 7.18e-02 / 2.58e-02 | 20260505 13:41:20.439 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 13:41:20.439 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:41:20.439 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:41:20.439 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260505 13:41:20.439 | INFO | PerfReport: 22.56%, Upload -- 2.27e+01 / 0.00e+00, 8.04e+00 / 0.00e+00 | 20260505 13:41:20.439 | INFO | PerfReport: 22.07%, Download -- 2.22e+01 / 0.00e+00, 8.10e+00 / 0.00e+00 | 20260505 13:41:20.439 | INFO | PerfReport: 31.64%, Compute -- 3.19e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 13:41:20.439 | INFO | PerfReport: 2.16%, LockUpload -- 2.18e+00 / 0.00e+00, 7.77e+00 / 0.00e+00 | 20260505 13:41:20.439 | INFO | PerfReport: 1.40%, LockDownload -- 1.41e+00 / 0.00e+00, 2.92e+00 / 0.00e+00 | 20260505 13:41:20.439 | INFO | PerfReport: 20.15%, LockCompute -- 2.03e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260505 13:41:20.439 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.55e-02 / 0.00e+00 | 20260505 13:41:20.573 | INFO | Starting a round of Baz IO | 20260505 13:41:23.792 | INFO | Populating RTMetrics report | 20260505 13:41:23.792 | INFO | RTMetrics report populated | 20260505 13:41:23.792 | INFO | Opening tmp RTMetrics json file | 20260505 13:41:23.809 | INFO | Serializing RTMetrics to json | 20260505 13:41:23.816 | INFO | Serialization done, swapping tmp file now... | 20260505 13:41:23.833 | INFO | RTMetrics done | 20260505 13:41:23.894 | INFO | Finished a round of Baz IO: Wrote 2.288 GiB in 3.321 seconds | 20260505 13:41:23.894 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.02% / 0.00% / 76.97% | 20260505 13:41:23.894 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.663682% / 7.94729e-06% / 0.000115236% | 20260505 13:41:23.894 | INFO | Baz IO overall: | 20260505 13:41:23.894 | INFO | BytesWritten : 2171669581540 | 20260505 13:41:23.894 | INFO | NumEvents : 1033046686391 | 20260505 13:41:25.579 | INFO | Analyzing chunk frames = [3645440,3645952) | 20260505 13:41:25.579 | INFO | MemInfo: { | 20260505 13:41:25.579 | INFO | "gpu" : " 17 GB", | 20260505 13:41:25.579 | INFO | "huge" : "", | 20260505 13:41:25.579 | INFO | "huge_shared" : " 27 GB", | 20260505 13:41:25.579 | INFO | "malloc" : " 14 GB", | 20260505 13:41:25.579 | INFO | "pinned" : " 29 GB" | 20260505 13:41:25.579 | INFO | } | 20260505 13:41:25.579 | INFO | Perf report for the last 5.1413 seconds: | 20260505 13:41:25.579 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:41:25.579 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:25.579 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:25.579 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:25.579 | INFO | ANALYSIS : 64.1%, 3.0, 99.8ms, 0 | 20260505 13:41:25.579 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:25.579 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260505 13:41:25.579 | INFO | PRE_HQ : 60.9%, 1.8, 57.5ms, 0 | 20260505 13:41:25.579 | INFO | BAZWRITER : 63.7%, 4.0, 131.4ms, 0 | 20260505 13:41:25.579 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 13:41:25.579 | INFO | Uploaded 36GiB to GPU in 2.18215s (16.4975GiB/s) | 20260505 13:41:25.579 | INFO | Downloaded 38GiB to GPU in 2.25504s (16.8512GiB/s) | 20260505 13:41:30.712 | INFO | Analyzing chunk frames = [3645952,3646464) | 20260505 13:41:30.712 | INFO | MemInfo: { | 20260505 13:41:30.712 | INFO | "gpu" : " 17 GB", | 20260505 13:41:30.712 | INFO | "huge" : "", | 20260505 13:41:30.712 | INFO | "huge_shared" : " 27 GB", | 20260505 13:41:30.712 | INFO | "malloc" : " 14 GB", | 20260505 13:41:30.712 | INFO | "pinned" : " 29 GB" | 20260505 13:41:30.712 | INFO | } | 20260505 13:41:30.712 | INFO | Perf report for the last 5.13361 seconds: | 20260505 13:41:30.712 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:41:30.712 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:30.712 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:30.712 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:30.712 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 13:41:30.712 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:30.712 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:30.713 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 13:41:30.713 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:30.713 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:30.713 | INFO | Uploaded 36GiB to GPU in 2.19729s (16.3838GiB/s) | 20260505 13:41:30.713 | INFO | Downloaded 34GiB to GPU in 2.04024s (16.6647GiB/s) | 20260505 13:41:35.846 | INFO | Analyzing chunk frames = [3646464,3646976) | 20260505 13:41:35.846 | INFO | MemInfo: { | 20260505 13:41:35.846 | INFO | "gpu" : " 17 GB", | 20260505 13:41:35.846 | INFO | "huge" : "", | 20260505 13:41:35.846 | INFO | "huge_shared" : " 27 GB", | 20260505 13:41:35.846 | INFO | "malloc" : " 14 GB", | 20260505 13:41:35.846 | INFO | "pinned" : " 29 GB" | 20260505 13:41:35.846 | INFO | } | 20260505 13:41:35.846 | INFO | Perf report for the last 5.1335 seconds: | 20260505 13:41:35.846 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:41:35.846 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:35.846 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:35.846 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:35.846 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 13:41:35.846 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:35.846 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:35.846 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:41:35.846 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:35.846 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:35.846 | INFO | Uploaded 36GiB to GPU in 2.21781s (16.2322GiB/s) | 20260505 13:41:35.846 | INFO | Downloaded 34GiB to GPU in 2.11361s (16.0862GiB/s) | 20260505 13:41:40.993 | INFO | Analyzing chunk frames = [3646976,3647488) | 20260505 13:41:40.993 | INFO | MemInfo: { | 20260505 13:41:40.993 | INFO | "gpu" : " 17 GB", | 20260505 13:41:40.993 | INFO | "huge" : "", | 20260505 13:41:40.993 | INFO | "huge_shared" : " 27 GB", | 20260505 13:41:40.993 | INFO | "malloc" : " 14 GB", | 20260505 13:41:40.993 | INFO | "pinned" : " 29 GB" | 20260505 13:41:40.993 | INFO | } | 20260505 13:41:40.993 | INFO | Perf report for the last 5.14718 seconds: | 20260505 13:41:40.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:41:40.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:40.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:40.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:40.993 | INFO | ANALYSIS : 62.9%, 3.0, 97.8ms, 0 | 20260505 13:41:40.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:40.993 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:40.993 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 13:41:40.993 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:40.993 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:40.993 | INFO | Uploaded 36GiB to GPU in 2.20216s (16.3476GiB/s) | 20260505 13:41:40.993 | INFO | Downloaded 34GiB to GPU in 2.13328s (15.9379GiB/s) | 20260505 13:41:46.133 | INFO | Analyzing chunk frames = [3647488,3648000) | 20260505 13:41:46.133 | INFO | MemInfo: { | 20260505 13:41:46.133 | INFO | "gpu" : " 17 GB", | 20260505 13:41:46.133 | INFO | "huge" : "", | 20260505 13:41:46.133 | INFO | "huge_shared" : " 27 GB", | 20260505 13:41:46.133 | INFO | "malloc" : " 14 GB", | 20260505 13:41:46.133 | INFO | "pinned" : " 29 GB" | 20260505 13:41:46.134 | INFO | } | 20260505 13:41:46.134 | INFO | Perf report for the last 5.14058 seconds: | 20260505 13:41:46.134 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:41:46.134 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:46.134 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:46.134 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:46.134 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 13:41:46.134 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:46.134 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:46.134 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 13:41:46.134 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:46.134 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:46.134 | INFO | Uploaded 36GiB to GPU in 2.16389s (16.6367GiB/s) | 20260505 13:41:46.134 | INFO | Downloaded 34GiB to GPU in 2.12768s (15.9798GiB/s) | 20260505 13:41:51.278 | INFO | Analyzing chunk frames = [3648000,3648512) | 20260505 13:41:51.278 | INFO | MemInfo: { | 20260505 13:41:51.278 | INFO | "gpu" : " 17 GB", | 20260505 13:41:51.278 | INFO | "huge" : "", | 20260505 13:41:51.278 | INFO | "huge_shared" : " 27 GB", | 20260505 13:41:51.278 | INFO | "malloc" : " 14 GB", | 20260505 13:41:51.278 | INFO | "pinned" : " 29 GB" | 20260505 13:41:51.278 | INFO | } | 20260505 13:41:51.278 | INFO | Perf report for the last 5.14445 seconds: | 20260505 13:41:51.278 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:41:51.278 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:51.278 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:51.278 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:51.278 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 13:41:51.278 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:51.278 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:51.278 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 13:41:51.278 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:51.278 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:51.278 | INFO | Uploaded 36GiB to GPU in 2.17052s (16.5859GiB/s) | 20260505 13:41:51.278 | INFO | Downloaded 34GiB to GPU in 2.10841s (16.1259GiB/s) | 20260505 13:41:56.410 | INFO | Analyzing chunk frames = [3648512,3649024) | 20260505 13:41:56.410 | INFO | MemInfo: { | 20260505 13:41:56.410 | INFO | "gpu" : " 17 GB", | 20260505 13:41:56.410 | INFO | "huge" : "", | 20260505 13:41:56.410 | INFO | "huge_shared" : " 27 GB", | 20260505 13:41:56.410 | INFO | "malloc" : " 14 GB", | 20260505 13:41:56.410 | INFO | "pinned" : " 29 GB" | 20260505 13:41:56.410 | INFO | } | 20260505 13:41:56.410 | INFO | Perf report for the last 5.13162 seconds: | 20260505 13:41:56.410 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:41:56.410 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:56.410 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:56.410 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:56.410 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 13:41:56.410 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:56.410 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:56.410 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 13:41:56.410 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:56.410 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:41:56.410 | INFO | Uploaded 36GiB to GPU in 2.1723s (16.5723GiB/s) | 20260505 13:41:56.410 | INFO | Downloaded 34GiB to GPU in 2.16865s (15.6779GiB/s) | 20260505 13:42:01.543 | INFO | Analyzing chunk frames = [3649024,3649536) | 20260505 13:42:01.543 | INFO | MemInfo: { | 20260505 13:42:01.543 | INFO | "gpu" : " 17 GB", | 20260505 13:42:01.543 | INFO | "huge" : "", | 20260505 13:42:01.543 | INFO | "huge_shared" : " 27 GB", | 20260505 13:42:01.543 | INFO | "malloc" : " 14 GB", | 20260505 13:42:01.543 | INFO | "pinned" : " 29 GB" | 20260505 13:42:01.543 | INFO | } | 20260505 13:42:01.543 | INFO | Perf report for the last 5.13362 seconds: | 20260505 13:42:01.543 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:42:01.543 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:01.543 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:01.543 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:01.543 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 13:42:01.544 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:01.544 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:01.544 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 13:42:01.544 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:01.544 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:01.544 | INFO | Uploaded 36GiB to GPU in 2.16275s (16.6455GiB/s) | 20260505 13:42:01.544 | INFO | Downloaded 34GiB to GPU in 2.10443s (16.1564GiB/s) | 20260505 13:42:01.674 | INFO | Starting a round of Baz IO | 20260505 13:42:04.919 | INFO | Populating RTMetrics report | 20260505 13:42:04.919 | INFO | RTMetrics report populated | 20260505 13:42:04.919 | INFO | Opening tmp RTMetrics json file | 20260505 13:42:05.005 | INFO | Serializing RTMetrics to json | 20260505 13:42:05.017 | INFO | Serialization done, swapping tmp file now... | 20260505 13:42:05.018 | INFO | RTMetrics done | 20260505 13:42:05.029 | INFO | Finished a round of Baz IO: Wrote 2.289 GiB in 3.354 seconds | 20260505 13:42:05.029 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.02% / 0.00% / 76.98% | 20260505 13:42:05.029 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.663225% / 1.19209e-05% / 0.000103315% | 20260505 13:42:05.029 | INFO | Baz IO overall: | 20260505 13:42:05.029 | INFO | BytesWritten : 2174127129717 | 20260505 13:42:05.029 | INFO | NumEvents : 1034208289545 | 20260505 13:42:06.687 | INFO | Analyzing chunk frames = [3649536,3650048) | 20260505 13:42:06.687 | INFO | MemInfo: { | 20260505 13:42:06.687 | INFO | "gpu" : " 17 GB", | 20260505 13:42:06.687 | INFO | "huge" : "", | 20260505 13:42:06.687 | INFO | "huge_shared" : " 27 GB", | 20260505 13:42:06.687 | INFO | "malloc" : " 14 GB", | 20260505 13:42:06.687 | INFO | "pinned" : " 29 GB" | 20260505 13:42:06.687 | INFO | } | 20260505 13:42:06.687 | INFO | Perf report for the last 5.14392 seconds: | 20260505 13:42:06.687 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:42:06.687 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:06.687 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:06.687 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:06.687 | INFO | ANALYSIS : 64.5%, 3.0, 100.2ms, 0 | 20260505 13:42:06.687 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:06.687 | INFO | RT_METRICS : 47.0%, 1.0, 24.6ms, 0 | 20260505 13:42:06.687 | INFO | PRE_HQ : 60.6%, 1.8, 57.7ms, 0 | 20260505 13:42:06.687 | INFO | BAZWRITER : 64.5%, 4.0, 132.3ms, 0 | 20260505 13:42:06.687 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260505 13:42:06.688 | INFO | Uploaded 36GiB to GPU in 2.17988s (16.5147GiB/s) | 20260505 13:42:06.688 | INFO | Downloaded 38GiB to GPU in 2.28207s (16.6515GiB/s) | 20260505 13:42:11.834 | INFO | Analyzing chunk frames = [3650048,3650560) | 20260505 13:42:11.834 | INFO | MemInfo: { | 20260505 13:42:11.834 | INFO | "gpu" : " 17 GB", | 20260505 13:42:11.834 | INFO | "huge" : "", | 20260505 13:42:11.834 | INFO | "huge_shared" : " 27 GB", | 20260505 13:42:11.834 | INFO | "malloc" : " 14 GB", | 20260505 13:42:11.834 | INFO | "pinned" : " 29 GB" | 20260505 13:42:11.834 | INFO | } | 20260505 13:42:11.834 | INFO | Perf report for the last 5.14676 seconds: | 20260505 13:42:11.834 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:42:11.834 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:11.834 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:11.834 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:11.834 | INFO | ANALYSIS : 63.5%, 3.0, 99.3ms, 0 | 20260505 13:42:11.834 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:11.834 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:11.834 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 13:42:11.834 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:11.834 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:11.835 | INFO | Uploaded 36GiB to GPU in 2.20342s (16.3382GiB/s) | 20260505 13:42:11.835 | INFO | Downloaded 34GiB to GPU in 2.04139s (16.6554GiB/s) | 20260505 13:42:16.968 | INFO | Analyzing chunk frames = [3650560,3651072) | 20260505 13:42:16.968 | INFO | MemInfo: { | 20260505 13:42:16.968 | INFO | "gpu" : " 17 GB", | 20260505 13:42:16.968 | INFO | "huge" : "", | 20260505 13:42:16.968 | INFO | "huge_shared" : " 27 GB", | 20260505 13:42:16.968 | INFO | "malloc" : " 14 GB", | 20260505 13:42:16.968 | INFO | "pinned" : " 29 GB" | 20260505 13:42:16.968 | INFO | } | 20260505 13:42:16.968 | INFO | Perf report for the last 5.13396 seconds: | 20260505 13:42:16.968 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:42:16.968 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:16.968 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:16.968 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:16.968 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260505 13:42:16.968 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:16.968 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:16.968 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:42:16.968 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:16.968 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:16.969 | INFO | Uploaded 36GiB to GPU in 2.23532s (16.1051GiB/s) | 20260505 13:42:16.969 | INFO | Downloaded 34GiB to GPU in 2.12344s (16.0118GiB/s) | 20260505 13:42:22.100 | INFO | Analyzing chunk frames = [3651072,3651584) | 20260505 13:42:22.100 | INFO | MemInfo: { | 20260505 13:42:22.100 | INFO | "gpu" : " 17 GB", | 20260505 13:42:22.100 | INFO | "huge" : "", | 20260505 13:42:22.100 | INFO | "huge_shared" : " 27 GB", | 20260505 13:42:22.100 | INFO | "malloc" : " 14 GB", | 20260505 13:42:22.101 | INFO | "pinned" : " 29 GB" | 20260505 13:42:22.101 | INFO | } | 20260505 13:42:22.101 | INFO | Perf report for the last 5.13257 seconds: | 20260505 13:42:22.101 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:42:22.101 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:22.101 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:22.101 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:22.101 | INFO | ANALYSIS : 63.0%, 3.0, 97.7ms, 0 | 20260505 13:42:22.101 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:22.101 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:22.101 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 13:42:22.101 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:22.101 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:22.101 | INFO | Uploaded 36GiB to GPU in 2.19838s (16.3757GiB/s) | 20260505 13:42:22.101 | INFO | Downloaded 34GiB to GPU in 2.13103s (15.9547GiB/s) | 20260505 13:42:27.250 | INFO | Analyzing chunk frames = [3651584,3652096) | 20260505 13:42:27.250 | INFO | MemInfo: { | 20260505 13:42:27.250 | INFO | "gpu" : " 17 GB", | 20260505 13:42:27.250 | INFO | "huge" : "", | 20260505 13:42:27.250 | INFO | "huge_shared" : " 27 GB", | 20260505 13:42:27.250 | INFO | "malloc" : " 14 GB", | 20260505 13:42:27.250 | INFO | "pinned" : " 29 GB" | 20260505 13:42:27.250 | INFO | } | 20260505 13:42:27.250 | INFO | Perf report for the last 5.14924 seconds: | 20260505 13:42:27.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:42:27.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:27.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:27.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:27.250 | INFO | ANALYSIS : 62.7%, 3.0, 98.2ms, 0 | 20260505 13:42:27.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:27.250 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:27.250 | INFO | PRE_HQ : 28.7%, 1.0, 15.0ms, 0 | 20260505 13:42:27.250 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:27.250 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:27.250 | INFO | Uploaded 36GiB to GPU in 2.15285s (16.722GiB/s) | 20260505 13:42:27.250 | INFO | Downloaded 34GiB to GPU in 2.12584s (15.9937GiB/s) | 20260505 13:42:32.389 | INFO | Analyzing chunk frames = [3652096,3652608) | 20260505 13:42:32.389 | INFO | MemInfo: { | 20260505 13:42:32.389 | INFO | "gpu" : " 17 GB", | 20260505 13:42:32.389 | INFO | "huge" : "", | 20260505 13:42:32.389 | INFO | "huge_shared" : " 27 GB", | 20260505 13:42:32.389 | INFO | "malloc" : " 14 GB", | 20260505 13:42:32.389 | INFO | "pinned" : " 29 GB" | 20260505 13:42:32.389 | INFO | } | 20260505 13:42:32.389 | INFO | Perf report for the last 5.13928 seconds: | 20260505 13:42:32.389 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:42:32.389 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:32.389 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:32.389 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:32.389 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 13:42:32.389 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:32.389 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:32.389 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 13:42:32.389 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:32.389 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:32.390 | INFO | Uploaded 36GiB to GPU in 2.1952s (16.3994GiB/s) | 20260505 13:42:32.390 | INFO | Downloaded 34GiB to GPU in 2.11353s (16.0868GiB/s) | 20260505 13:42:37.525 | INFO | Analyzing chunk frames = [3652608,3653120) | 20260505 13:42:37.525 | INFO | MemInfo: { | 20260505 13:42:37.525 | INFO | "gpu" : " 17 GB", | 20260505 13:42:37.525 | INFO | "huge" : "", | 20260505 13:42:37.526 | INFO | "huge_shared" : " 27 GB", | 20260505 13:42:37.526 | INFO | "malloc" : " 14 GB", | 20260505 13:42:37.526 | INFO | "pinned" : " 29 GB" | 20260505 13:42:37.526 | INFO | } | 20260505 13:42:37.526 | INFO | Perf report for the last 5.13634 seconds: | 20260505 13:42:37.526 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:42:37.526 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:37.526 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:37.526 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:37.526 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 13:42:37.526 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:37.526 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:37.526 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 13:42:37.526 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:37.526 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:37.526 | INFO | Uploaded 36GiB to GPU in 2.21212s (16.274GiB/s) | 20260505 13:42:37.526 | INFO | Downloaded 34GiB to GPU in 2.15529s (15.7752GiB/s) | 20260505 13:42:42.655 | INFO | Analyzing chunk frames = [3653120,3653632) | 20260505 13:42:42.655 | INFO | MemInfo: { | 20260505 13:42:42.655 | INFO | "gpu" : " 17 GB", | 20260505 13:42:42.655 | INFO | "huge" : "", | 20260505 13:42:42.655 | INFO | "huge_shared" : " 27 GB", | 20260505 13:42:42.655 | INFO | "malloc" : " 14 GB", | 20260505 13:42:42.655 | INFO | "pinned" : " 29 GB" | 20260505 13:42:42.655 | INFO | } | 20260505 13:42:42.655 | INFO | Perf report for the last 5.12985 seconds: | 20260505 13:42:42.655 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:42:42.655 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:42.655 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:42.655 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:42.656 | INFO | ANALYSIS : 63.6%, 3.0, 99.1ms, 0 | 20260505 13:42:42.656 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:42.656 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:42.656 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 13:42:42.656 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:42.656 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:42.656 | INFO | Uploaded 36GiB to GPU in 2.13711s (16.8452GiB/s) | 20260505 13:42:42.656 | INFO | Downloaded 34GiB to GPU in 2.09715s (16.2125GiB/s) | 20260505 13:42:42.656 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 13:42:42.656 | INFO | Total mem usage: 15.3 GiB | 20260505 13:42:42.656 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 13:42:42.656 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.78%) | 20260505 13:42:42.656 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 13:42:42.656 | INFO | High Water Breakdown: | 20260505 13:42:42.656 | INFO | 61% : 8.68 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 13:42:42.656 | INFO | 38.8% : 5.52 GiB ( 83.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 13:42:42.656 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 13:42:42.656 | INFO | ------------------------------------------ | 20260505 13:42:42.656 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:42:42.656 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 13:42:42.656 | INFO | Total mem usage: 26.5 GiB | 20260505 13:42:42.656 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 13:42:42.656 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 13:42:42.656 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 13:42:42.656 | INFO | High Water Breakdown: | 20260505 13:42:42.656 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:42:42.656 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:42:42.656 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:42:42.656 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:42:42.656 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:42:42.656 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:42:42.656 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 13:42:42.656 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 13:42:42.656 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:42:42.656 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 13:42:42.656 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 13:42:42.656 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 13:42:42.656 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 13:42:42.656 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 13:42:42.656 | INFO | ------------------------------------------ | 20260505 13:42:42.656 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:42:42.656 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 13:42:42.656 | INFO | Total mem usage: 25.6 GiB | 20260505 13:42:42.656 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 13:42:42.656 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 13:42:42.656 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 13:42:42.656 | INFO | High Water Breakdown: | 20260505 13:42:42.656 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 13:42:42.656 | INFO | ------------------------------------------ | 20260505 13:42:42.656 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:42:42.656 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 13:42:42.656 | INFO | Total mem usage: 15.7 GiB | 20260505 13:42:42.657 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 13:42:42.657 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 13:42:42.657 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 13:42:42.657 | INFO | High Water Breakdown: | 20260505 13:42:42.657 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 13:42:42.657 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 13:42:42.657 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 13:42:42.657 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 13:42:42.657 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 13:42:42.657 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 13:42:42.657 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 13:42:42.657 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 13:42:42.657 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 13:42:42.657 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:42:42.657 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 13:42:42.657 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:42:42.657 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 13:42:42.657 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 13:42:42.657 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 13:42:42.657 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 13:42:42.657 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:42:42.657 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 13:42:42.657 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:42:42.657 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:42:42.657 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:42:42.657 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 13:42:42.657 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 13:42:42.657 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:42:42.657 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 13:42:42.657 | INFO | ------------------------------------------ | 20260505 13:42:42.657 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:42:42.657 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 13:42:42.657 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:42:42.657 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:42:42.657 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.67e+01, 1.39e+01 / 1.63e+00 | 20260505 13:42:42.657 | INFO | PerfReport: 0.29%, AggregateStats -- 9.32e-02 / 1.00e-01, 1.10e-02 / 4.28e-03 | 20260505 13:42:42.657 | INFO | PerfReport: 11.76%, Baseline -- 3.75e+00 / 4.50e+00, 5.85e-01 / 1.70e-01 | 20260505 13:42:42.657 | INFO | PerfReport: 3.30%, Histogram -- 1.05e+00 / 1.50e+00, 2.00e-01 / 7.46e-02 | 20260505 13:42:42.657 | INFO | PerfReport: 16.00%, DME -- 4.08e+01 / 4.60e+01, 4.74e+00 / 1.58e+00 | 20260505 13:42:42.657 | INFO | PerfReport: 56.32%, FrameLabeling -- 1.79e+01 / 1.98e+01, 1.40e+00 / 1.80e+00 | 20260505 13:42:42.657 | INFO | PerfReport: 7.49%, PulseAccumulating -- 2.39e+00 / 2.94e+00, 4.56e-01 / 1.53e-01 | 20260505 13:42:42.657 | INFO | PerfReport: 3.58%, Metrics -- 1.14e+00 / 1.45e+00, 3.18e-01 / 1.80e-01 | 20260505 13:42:42.657 | INFO | PerfReport: 1.25%, PulseSerializing -- 3.99e-01 / 4.45e-01, 7.10e-02 / 3.32e-02 | 20260505 13:42:42.657 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 13:42:42.657 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:42:42.657 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:42:42.657 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260505 13:42:42.657 | INFO | PerfReport: 22.57%, Upload -- 2.28e+01 / 0.00e+00, 8.12e+00 / 0.00e+00 | 20260505 13:42:42.657 | INFO | PerfReport: 22.02%, Download -- 2.22e+01 / 0.00e+00, 8.09e+00 / 0.00e+00 | 20260505 13:42:42.657 | INFO | PerfReport: 31.58%, Compute -- 3.19e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 13:42:42.657 | INFO | PerfReport: 2.22%, LockUpload -- 2.24e+00 / 0.00e+00, 7.94e+00 / 0.00e+00 | 20260505 13:42:42.657 | INFO | PerfReport: 1.42%, LockDownload -- 1.44e+00 / 0.00e+00, 2.94e+00 / 0.00e+00 | 20260505 13:42:42.657 | INFO | PerfReport: 20.17%, LockCompute -- 2.04e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 13:42:42.657 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.14e-02 / 0.00e+00 | 20260505 13:42:42.788 | INFO | Starting a round of Baz IO | 20260505 13:42:46.028 | INFO | Populating RTMetrics report | 20260505 13:42:46.028 | INFO | RTMetrics report populated | 20260505 13:42:46.028 | INFO | Opening tmp RTMetrics json file | 20260505 13:42:46.045 | INFO | Serializing RTMetrics to json | 20260505 13:42:46.050 | INFO | Serialization done, swapping tmp file now... | 20260505 13:42:46.052 | INFO | RTMetrics done | 20260505 13:42:46.142 | INFO | Finished a round of Baz IO: Wrote 2.301 GiB in 3.354 seconds | 20260505 13:42:46.142 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.02% / 0.00% / 76.98% | 20260505 13:42:46.142 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.663277% / 7.94729e-06% / 7.54992e-05% | 20260505 13:42:46.142 | INFO | Baz IO overall: | 20260505 13:42:46.142 | INFO | BytesWritten : 2176597899564 | 20260505 13:42:46.142 | INFO | NumEvents : 1035373666738 | 20260505 13:42:47.797 | INFO | Analyzing chunk frames = [3653632,3654144) | 20260505 13:42:47.797 | INFO | MemInfo: { | 20260505 13:42:47.797 | INFO | "gpu" : " 17 GB", | 20260505 13:42:47.797 | INFO | "huge" : "", | 20260505 13:42:47.797 | INFO | "huge_shared" : " 27 GB", | 20260505 13:42:47.797 | INFO | "malloc" : " 14 GB", | 20260505 13:42:47.797 | INFO | "pinned" : " 29 GB" | 20260505 13:42:47.797 | INFO | } | 20260505 13:42:47.797 | INFO | Perf report for the last 5.14131 seconds: | 20260505 13:42:47.797 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:42:47.797 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:47.797 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:47.797 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:47.797 | INFO | ANALYSIS : 64.5%, 3.0, 100.3ms, 0 | 20260505 13:42:47.797 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:47.797 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260505 13:42:47.797 | INFO | PRE_HQ : 60.7%, 1.8, 57.9ms, 0 | 20260505 13:42:47.797 | INFO | BAZWRITER : 64.5%, 4.0, 132.5ms, 0 | 20260505 13:42:47.797 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 13:42:47.797 | INFO | Uploaded 36GiB to GPU in 2.17834s (16.5264GiB/s) | 20260505 13:42:47.797 | INFO | Downloaded 38GiB to GPU in 2.28605s (16.6226GiB/s) | 20260505 13:42:52.950 | INFO | Analyzing chunk frames = [3654144,3654656) | 20260505 13:42:52.950 | INFO | MemInfo: { | 20260505 13:42:52.950 | INFO | "gpu" : " 17 GB", | 20260505 13:42:52.950 | INFO | "huge" : "", | 20260505 13:42:52.950 | INFO | "huge_shared" : " 27 GB", | 20260505 13:42:52.950 | INFO | "malloc" : " 14 GB", | 20260505 13:42:52.950 | INFO | "pinned" : " 29 GB" | 20260505 13:42:52.950 | INFO | } | 20260505 13:42:52.950 | INFO | Perf report for the last 5.15355 seconds: | 20260505 13:42:52.950 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:42:52.950 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:52.950 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:52.950 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:52.950 | INFO | ANALYSIS : 63.4%, 3.0, 99.3ms, 0 | 20260505 13:42:52.950 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:52.950 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:52.950 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:42:52.950 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:52.950 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:52.951 | INFO | Uploaded 36GiB to GPU in 2.17807s (16.5284GiB/s) | 20260505 13:42:52.951 | INFO | Downloaded 34GiB to GPU in 2.04165s (16.6532GiB/s) | 20260505 13:42:58.087 | INFO | Analyzing chunk frames = [3654656,3655168) | 20260505 13:42:58.087 | INFO | MemInfo: { | 20260505 13:42:58.087 | INFO | "gpu" : " 17 GB", | 20260505 13:42:58.087 | INFO | "huge" : "", | 20260505 13:42:58.087 | INFO | "huge_shared" : " 27 GB", | 20260505 13:42:58.087 | INFO | "malloc" : " 14 GB", | 20260505 13:42:58.087 | INFO | "pinned" : " 29 GB" | 20260505 13:42:58.087 | INFO | } | 20260505 13:42:58.087 | INFO | Perf report for the last 5.13712 seconds: | 20260505 13:42:58.087 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:42:58.087 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:58.087 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:58.087 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:58.088 | INFO | ANALYSIS : 63.0%, 3.0, 98.0ms, 0 | 20260505 13:42:58.088 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:58.088 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:58.088 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 13:42:58.088 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:58.088 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:42:58.088 | INFO | Uploaded 36GiB to GPU in 2.17891s (16.522GiB/s) | 20260505 13:42:58.088 | INFO | Downloaded 34GiB to GPU in 2.04173s (16.6526GiB/s) | 20260505 13:43:03.225 | INFO | Analyzing chunk frames = [3655168,3655680) | 20260505 13:43:03.225 | INFO | MemInfo: { | 20260505 13:43:03.225 | INFO | "gpu" : " 17 GB", | 20260505 13:43:03.225 | INFO | "huge" : "", | 20260505 13:43:03.225 | INFO | "huge_shared" : " 27 GB", | 20260505 13:43:03.225 | INFO | "malloc" : " 14 GB", | 20260505 13:43:03.225 | INFO | "pinned" : " 29 GB" | 20260505 13:43:03.225 | INFO | } | 20260505 13:43:03.225 | INFO | Perf report for the last 5.13731 seconds: | 20260505 13:43:03.225 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:43:03.225 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:03.225 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:03.225 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:03.225 | INFO | ANALYSIS : 62.9%, 3.0, 97.6ms, 0 | 20260505 13:43:03.225 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:03.225 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:03.225 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 13:43:03.225 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:03.225 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:03.225 | INFO | Uploaded 36GiB to GPU in 2.1866s (16.4639GiB/s) | 20260505 13:43:03.225 | INFO | Downloaded 34GiB to GPU in 2.12939s (15.967GiB/s) | 20260505 13:43:08.358 | INFO | Analyzing chunk frames = [3655680,3656192) | 20260505 13:43:08.358 | INFO | MemInfo: { | 20260505 13:43:08.358 | INFO | "gpu" : " 17 GB", | 20260505 13:43:08.358 | INFO | "huge" : "", | 20260505 13:43:08.358 | INFO | "huge_shared" : " 27 GB", | 20260505 13:43:08.358 | INFO | "malloc" : " 14 GB", | 20260505 13:43:08.358 | INFO | "pinned" : " 29 GB" | 20260505 13:43:08.358 | INFO | } | 20260505 13:43:08.358 | INFO | Perf report for the last 5.13356 seconds: | 20260505 13:43:08.358 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:43:08.358 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:08.358 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:08.358 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:08.358 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260505 13:43:08.358 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:08.358 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:08.358 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 13:43:08.358 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:08.358 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:08.359 | INFO | Uploaded 36GiB to GPU in 2.14635s (16.7726GiB/s) | 20260505 13:43:08.359 | INFO | Downloaded 34GiB to GPU in 2.12256s (16.0184GiB/s) | 20260505 13:43:13.501 | INFO | Analyzing chunk frames = [3656192,3656704) | 20260505 13:43:13.501 | INFO | MemInfo: { | 20260505 13:43:13.501 | INFO | "gpu" : " 17 GB", | 20260505 13:43:13.501 | INFO | "huge" : "", | 20260505 13:43:13.501 | INFO | "huge_shared" : " 27 GB", | 20260505 13:43:13.501 | INFO | "malloc" : " 14 GB", | 20260505 13:43:13.501 | INFO | "pinned" : " 29 GB" | 20260505 13:43:13.502 | INFO | } | 20260505 13:43:13.502 | INFO | Perf report for the last 5.14325 seconds: | 20260505 13:43:13.502 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:43:13.502 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:13.502 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:13.502 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:13.502 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 13:43:13.502 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:13.502 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:13.502 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:43:13.502 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:13.502 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:13.502 | INFO | Uploaded 36GiB to GPU in 2.16256s (16.6469GiB/s) | 20260505 13:43:13.502 | INFO | Downloaded 34GiB to GPU in 2.11009s (16.113GiB/s) | 20260505 13:43:18.637 | INFO | Analyzing chunk frames = [3656704,3657216) | 20260505 13:43:18.637 | INFO | MemInfo: { | 20260505 13:43:18.637 | INFO | "gpu" : " 17 GB", | 20260505 13:43:18.637 | INFO | "huge" : "", | 20260505 13:43:18.637 | INFO | "huge_shared" : " 27 GB", | 20260505 13:43:18.637 | INFO | "malloc" : " 14 GB", | 20260505 13:43:18.637 | INFO | "pinned" : " 29 GB" | 20260505 13:43:18.637 | INFO | } | 20260505 13:43:18.638 | INFO | Perf report for the last 5.13606 seconds: | 20260505 13:43:18.638 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:43:18.638 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:18.638 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:18.638 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:18.638 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 13:43:18.638 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:18.638 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:18.638 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 13:43:18.638 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:18.638 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:18.638 | INFO | Uploaded 36GiB to GPU in 2.19184s (16.4245GiB/s) | 20260505 13:43:18.638 | INFO | Downloaded 34GiB to GPU in 2.17546s (15.6289GiB/s) | 20260505 13:43:23.780 | INFO | Analyzing chunk frames = [3657216,3657728) | 20260505 13:43:23.780 | INFO | MemInfo: { | 20260505 13:43:23.780 | INFO | "gpu" : " 17 GB", | 20260505 13:43:23.780 | INFO | "huge" : "", | 20260505 13:43:23.781 | INFO | "huge_shared" : " 27 GB", | 20260505 13:43:23.781 | INFO | "malloc" : " 14 GB", | 20260505 13:43:23.781 | INFO | "pinned" : " 29 GB" | 20260505 13:43:23.781 | INFO | } | 20260505 13:43:23.781 | INFO | Perf report for the last 5.14299 seconds: | 20260505 13:43:23.781 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:43:23.781 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:23.781 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:23.781 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:23.781 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 13:43:23.781 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:23.781 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:23.781 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 13:43:23.781 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:23.781 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:23.781 | INFO | Uploaded 36GiB to GPU in 2.13189s (16.8865GiB/s) | 20260505 13:43:23.781 | INFO | Downloaded 34GiB to GPU in 2.09806s (16.2055GiB/s) | 20260505 13:43:23.922 | INFO | Starting a round of Baz IO | 20260505 13:43:27.145 | INFO | Populating RTMetrics report | 20260505 13:43:27.146 | INFO | RTMetrics report populated | 20260505 13:43:27.146 | INFO | Opening tmp RTMetrics json file | 20260505 13:43:27.148 | INFO | Serializing RTMetrics to json | 20260505 13:43:27.149 | INFO | Serialization done, swapping tmp file now... | 20260505 13:43:27.150 | INFO | RTMetrics done | 20260505 13:43:27.269 | INFO | Finished a round of Baz IO: Wrote 2.301 GiB in 3.346 seconds | 20260505 13:43:27.269 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.01% / 0.00% / 76.98% | 20260505 13:43:27.269 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.662363% / 7.94729e-06% / 7.15256e-05% | 20260505 13:43:27.269 | INFO | Baz IO overall: | 20260505 13:43:27.269 | INFO | BytesWritten : 2179068208831 | 20260505 13:43:27.269 | INFO | NumEvents : 1036537676010 | 20260505 13:43:28.915 | INFO | Analyzing chunk frames = [3657728,3658240) | 20260505 13:43:28.915 | INFO | MemInfo: { | 20260505 13:43:28.915 | INFO | "gpu" : " 17 GB", | 20260505 13:43:28.915 | INFO | "huge" : "", | 20260505 13:43:28.915 | INFO | "huge_shared" : " 27 GB", | 20260505 13:43:28.915 | INFO | "malloc" : " 14 GB", | 20260505 13:43:28.915 | INFO | "pinned" : " 29 GB" | 20260505 13:43:28.915 | INFO | } | 20260505 13:43:28.915 | INFO | Perf report for the last 5.13469 seconds: | 20260505 13:43:28.915 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:43:28.915 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:28.915 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:28.915 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:28.915 | INFO | ANALYSIS : 64.4%, 3.0, 99.6ms, 0 | 20260505 13:43:28.915 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:28.915 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260505 13:43:28.915 | INFO | PRE_HQ : 60.9%, 1.8, 57.9ms, 0 | 20260505 13:43:28.915 | INFO | BAZWRITER : 64.5%, 4.1, 135.0ms, 0 | 20260505 13:43:28.915 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 13:43:28.916 | INFO | Uploaded 36GiB to GPU in 2.19325s (16.414GiB/s) | 20260505 13:43:28.916 | INFO | Downloaded 38GiB to GPU in 2.28707s (16.6151GiB/s) | 20260505 13:43:34.052 | INFO | Analyzing chunk frames = [3658240,3658752) | 20260505 13:43:34.052 | INFO | MemInfo: { | 20260505 13:43:34.052 | INFO | "gpu" : " 17 GB", | 20260505 13:43:34.052 | INFO | "huge" : "", | 20260505 13:43:34.052 | INFO | "huge_shared" : " 27 GB", | 20260505 13:43:34.052 | INFO | "malloc" : " 14 GB", | 20260505 13:43:34.052 | INFO | "pinned" : " 29 GB" | 20260505 13:43:34.052 | INFO | } | 20260505 13:43:34.052 | INFO | Perf report for the last 5.13708 seconds: | 20260505 13:43:34.052 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:43:34.052 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:34.052 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:34.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:34.052 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 13:43:34.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:34.052 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:34.052 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 13:43:34.052 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:34.052 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:34.053 | INFO | Uploaded 36GiB to GPU in 2.17491s (16.5524GiB/s) | 20260505 13:43:34.053 | INFO | Downloaded 34GiB to GPU in 2.03289s (16.725GiB/s) | 20260505 13:43:39.204 | INFO | Analyzing chunk frames = [3658752,3659264) | 20260505 13:43:39.204 | INFO | MemInfo: { | 20260505 13:43:39.204 | INFO | "gpu" : " 17 GB", | 20260505 13:43:39.204 | INFO | "huge" : "", | 20260505 13:43:39.204 | INFO | "huge_shared" : " 27 GB", | 20260505 13:43:39.204 | INFO | "malloc" : " 14 GB", | 20260505 13:43:39.204 | INFO | "pinned" : " 29 GB" | 20260505 13:43:39.204 | INFO | } | 20260505 13:43:39.204 | INFO | Perf report for the last 5.15192 seconds: | 20260505 13:43:39.204 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:43:39.204 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:39.204 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:39.204 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:39.204 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 13:43:39.204 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:39.204 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:39.205 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 13:43:39.205 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:39.205 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:39.205 | INFO | Uploaded 36GiB to GPU in 2.21748s (16.2346GiB/s) | 20260505 13:43:39.205 | INFO | Downloaded 34GiB to GPU in 2.10874s (16.1234GiB/s) | 20260505 13:43:44.343 | INFO | Analyzing chunk frames = [3659264,3659776) | 20260505 13:43:44.343 | INFO | MemInfo: { | 20260505 13:43:44.343 | INFO | "gpu" : " 17 GB", | 20260505 13:43:44.343 | INFO | "huge" : "", | 20260505 13:43:44.343 | INFO | "huge_shared" : " 27 GB", | 20260505 13:43:44.343 | INFO | "malloc" : " 14 GB", | 20260505 13:43:44.343 | INFO | "pinned" : " 29 GB" | 20260505 13:43:44.343 | INFO | } | 20260505 13:43:44.343 | INFO | Perf report for the last 5.13823 seconds: | 20260505 13:43:44.343 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:43:44.343 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:44.343 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:44.343 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:44.343 | INFO | ANALYSIS : 62.6%, 3.0, 97.2ms, 0 | 20260505 13:43:44.343 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:44.343 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:44.343 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 13:43:44.343 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:44.343 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:44.343 | INFO | Uploaded 36GiB to GPU in 2.17323s (16.5652GiB/s) | 20260505 13:43:44.343 | INFO | Downloaded 34GiB to GPU in 2.10997s (16.1139GiB/s) | 20260505 13:43:49.469 | INFO | Analyzing chunk frames = [3659776,3660288) | 20260505 13:43:49.469 | INFO | MemInfo: { | 20260505 13:43:49.469 | INFO | "gpu" : " 17 GB", | 20260505 13:43:49.469 | INFO | "huge" : "", | 20260505 13:43:49.469 | INFO | "huge_shared" : " 27 GB", | 20260505 13:43:49.469 | INFO | "malloc" : " 14 GB", | 20260505 13:43:49.469 | INFO | "pinned" : " 29 GB" | 20260505 13:43:49.469 | INFO | } | 20260505 13:43:49.469 | INFO | Perf report for the last 5.12679 seconds: | 20260505 13:43:49.469 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:43:49.469 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:49.469 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:49.469 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:49.470 | INFO | ANALYSIS : 62.9%, 3.0, 97.9ms, 0 | 20260505 13:43:49.470 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:49.470 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:49.470 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:43:49.470 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:49.470 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:49.470 | INFO | Uploaded 36GiB to GPU in 2.1573s (16.6875GiB/s) | 20260505 13:43:49.470 | INFO | Downloaded 34GiB to GPU in 2.12641s (15.9894GiB/s) | 20260505 13:43:54.609 | INFO | Analyzing chunk frames = [3660288,3660800) | 20260505 13:43:54.609 | INFO | MemInfo: { | 20260505 13:43:54.609 | INFO | "gpu" : " 17 GB", | 20260505 13:43:54.609 | INFO | "huge" : "", | 20260505 13:43:54.609 | INFO | "huge_shared" : " 27 GB", | 20260505 13:43:54.609 | INFO | "malloc" : " 14 GB", | 20260505 13:43:54.609 | INFO | "pinned" : " 29 GB" | 20260505 13:43:54.609 | INFO | } | 20260505 13:43:54.609 | INFO | Perf report for the last 5.13956 seconds: | 20260505 13:43:54.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:43:54.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:54.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:54.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:54.609 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 13:43:54.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:54.609 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:54.609 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 13:43:54.609 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:54.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:54.609 | INFO | Uploaded 36GiB to GPU in 2.17123s (16.5805GiB/s) | 20260505 13:43:54.609 | INFO | Downloaded 34GiB to GPU in 2.11109s (16.1054GiB/s) | 20260505 13:43:59.754 | INFO | Analyzing chunk frames = [3660800,3661312) | 20260505 13:43:59.754 | INFO | MemInfo: { | 20260505 13:43:59.754 | INFO | "gpu" : " 17 GB", | 20260505 13:43:59.754 | INFO | "huge" : "", | 20260505 13:43:59.754 | INFO | "huge_shared" : " 27 GB", | 20260505 13:43:59.754 | INFO | "malloc" : " 14 GB", | 20260505 13:43:59.755 | INFO | "pinned" : " 29 GB" | 20260505 13:43:59.755 | INFO | } | 20260505 13:43:59.755 | INFO | Perf report for the last 5.14554 seconds: | 20260505 13:43:59.755 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:43:59.755 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:59.755 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:59.755 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:59.755 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 13:43:59.755 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:59.755 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:59.755 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 13:43:59.755 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:59.755 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:43:59.755 | INFO | Uploaded 36GiB to GPU in 2.15295s (16.7213GiB/s) | 20260505 13:43:59.755 | INFO | Downloaded 34GiB to GPU in 2.16114s (15.7324GiB/s) | 20260505 13:44:04.900 | INFO | Analyzing chunk frames = [3661312,3661824) | 20260505 13:44:04.900 | INFO | MemInfo: { | 20260505 13:44:04.900 | INFO | "gpu" : " 17 GB", | 20260505 13:44:04.900 | INFO | "huge" : "", | 20260505 13:44:04.900 | INFO | "huge_shared" : " 27 GB", | 20260505 13:44:04.900 | INFO | "malloc" : " 14 GB", | 20260505 13:44:04.900 | INFO | "pinned" : " 29 GB" | 20260505 13:44:04.900 | INFO | } | 20260505 13:44:04.900 | INFO | Perf report for the last 5.14566 seconds: | 20260505 13:44:04.900 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:44:04.900 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:04.900 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:04.900 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:04.900 | INFO | ANALYSIS : 63.5%, 3.0, 99.3ms, 0 | 20260505 13:44:04.900 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:04.900 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:04.900 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 13:44:04.900 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:04.900 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:04.901 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 13:44:04.900 | INFO | Uploaded 36GiB to GPU in 2.21541s (16.2498GiB/s) | 20260505 13:44:04.901 | INFO | Total mem usage: 15.3 GiB | 20260505 13:44:04.901 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 13:44:04.901 | INFO | Downloaded 34GiB to GPU in 2.10431s (16.1573GiB/s) | 20260505 13:44:04.901 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -7.05%) | 20260505 13:44:04.901 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 13:44:04.901 | INFO | High Water Breakdown: | 20260505 13:44:04.901 | INFO | 61.1% : 8.75 GiB ( 91.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 13:44:04.901 | INFO | 38.8% : 5.55 GiB ( 83.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 13:44:04.901 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 13:44:04.901 | INFO | ------------------------------------------ | 20260505 13:44:04.901 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:44:04.901 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 13:44:04.901 | INFO | Total mem usage: 26.5 GiB | 20260505 13:44:04.901 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 13:44:04.901 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 13:44:04.901 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 13:44:04.901 | INFO | High Water Breakdown: | 20260505 13:44:04.901 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:44:04.901 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:44:04.901 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:44:04.901 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:44:04.901 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:44:04.901 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:44:04.901 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 13:44:04.901 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 13:44:04.901 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:44:04.901 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 13:44:04.901 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 13:44:04.901 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 13:44:04.901 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 13:44:04.901 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 13:44:04.901 | INFO | ------------------------------------------ | 20260505 13:44:04.901 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:44:04.901 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 13:44:04.901 | INFO | Total mem usage: 25.6 GiB | 20260505 13:44:04.901 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 13:44:04.901 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 13:44:04.901 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 13:44:04.901 | INFO | High Water Breakdown: | 20260505 13:44:04.901 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 13:44:04.901 | INFO | ------------------------------------------ | 20260505 13:44:04.901 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:44:04.901 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 13:44:04.901 | INFO | Total mem usage: 15.7 GiB | 20260505 13:44:04.901 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 13:44:04.901 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 13:44:04.901 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 13:44:04.901 | INFO | High Water Breakdown: | 20260505 13:44:04.901 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 13:44:04.901 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 13:44:04.901 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 13:44:04.901 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 13:44:04.901 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 13:44:04.901 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 13:44:04.901 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 13:44:04.901 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 13:44:04.901 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 13:44:04.901 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:44:04.902 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 13:44:04.902 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:44:04.902 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 13:44:04.902 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 13:44:04.902 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 13:44:04.902 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 13:44:04.902 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:44:04.902 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 13:44:04.902 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:44:04.902 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:44:04.902 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:44:04.902 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 13:44:04.902 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 13:44:04.902 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:44:04.902 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 13:44:04.902 | INFO | ------------------------------------------ | 20260505 13:44:04.902 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:44:04.902 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 13:44:04.902 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:44:04.902 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:44:04.902 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.66e+01, 1.37e+01 / 1.44e+00 | 20260505 13:44:04.902 | INFO | PerfReport: 0.29%, AggregateStats -- 9.30e-02 / 1.01e-01, 1.12e-02 / 5.48e-03 | 20260505 13:44:04.902 | INFO | PerfReport: 11.79%, Baseline -- 3.76e+00 / 4.51e+00, 5.98e-01 / 1.90e-01 | 20260505 13:44:04.902 | INFO | PerfReport: 3.31%, Histogram -- 1.06e+00 / 1.49e+00, 2.02e-01 / 9.59e-02 | 20260505 13:44:04.902 | INFO | PerfReport: 15.98%, DME -- 4.08e+01 / 4.59e+01, 4.57e+00 / 1.82e+00 | 20260505 13:44:04.902 | INFO | PerfReport: 56.36%, FrameLabeling -- 1.80e+01 / 1.98e+01, 1.43e+00 / 1.89e+00 | 20260505 13:44:04.902 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.38e+00 / 2.89e+00, 4.54e-01 / 2.20e-01 | 20260505 13:44:04.902 | INFO | PerfReport: 3.57%, Metrics -- 1.14e+00 / 1.49e+00, 3.21e-01 / 1.49e-01 | 20260505 13:44:04.902 | INFO | PerfReport: 1.25%, PulseSerializing -- 4.00e-01 / 4.53e-01, 7.27e-02 / 3.07e-02 | 20260505 13:44:04.902 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 13:44:04.902 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:44:04.902 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:44:04.902 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260505 13:44:04.902 | INFO | PerfReport: 22.49%, Upload -- 2.27e+01 / 0.00e+00, 7.95e+00 / 0.00e+00 | 20260505 13:44:04.902 | INFO | PerfReport: 21.99%, Download -- 2.22e+01 / 0.00e+00, 8.05e+00 / 0.00e+00 | 20260505 13:44:04.902 | INFO | PerfReport: 31.66%, Compute -- 3.19e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 13:44:04.902 | INFO | PerfReport: 2.16%, LockUpload -- 2.17e+00 / 0.00e+00, 7.84e+00 / 0.00e+00 | 20260505 13:44:04.902 | INFO | PerfReport: 1.49%, LockDownload -- 1.51e+00 / 0.00e+00, 3.09e+00 / 0.00e+00 | 20260505 13:44:04.902 | INFO | PerfReport: 20.19%, LockCompute -- 2.04e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260505 13:44:04.902 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.16e-02 / 0.00e+00 | 20260505 13:44:05.033 | INFO | Starting a round of Baz IO | 20260505 13:44:08.258 | INFO | Populating RTMetrics report | 20260505 13:44:08.258 | INFO | RTMetrics report populated | 20260505 13:44:08.258 | INFO | Opening tmp RTMetrics json file | 20260505 13:44:08.280 | INFO | Serializing RTMetrics to json | 20260505 13:44:08.302 | INFO | Serialization done, swapping tmp file now... | 20260505 13:44:08.323 | INFO | RTMetrics done | 20260505 13:44:08.366 | INFO | Finished a round of Baz IO: Wrote 2.296 GiB in 3.332 seconds | 20260505 13:44:08.366 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.01% / 0.00% / 76.99% | 20260505 13:44:08.366 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.661902% / 3.97364e-06% / 7.54992e-05% | 20260505 13:44:08.366 | INFO | Baz IO overall: | 20260505 13:44:08.366 | INFO | BytesWritten : 2181533145726 | 20260505 13:44:08.366 | INFO | NumEvents : 1037700548557 | 20260505 13:44:10.039 | INFO | Analyzing chunk frames = [3661824,3662336) | 20260505 13:44:10.039 | INFO | MemInfo: { | 20260505 13:44:10.039 | INFO | "gpu" : " 17 GB", | 20260505 13:44:10.039 | INFO | "huge" : "", | 20260505 13:44:10.039 | INFO | "huge_shared" : " 27 GB", | 20260505 13:44:10.039 | INFO | "malloc" : " 14 GB", | 20260505 13:44:10.039 | INFO | "pinned" : " 29 GB" | 20260505 13:44:10.039 | INFO | } | 20260505 13:44:10.039 | INFO | Perf report for the last 5.13832 seconds: | 20260505 13:44:10.039 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:44:10.039 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:10.039 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:10.039 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:10.039 | INFO | ANALYSIS : 64.3%, 3.0, 99.8ms, 0 | 20260505 13:44:10.039 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:10.039 | INFO | RT_METRICS : 46.2%, 1.0, 24.2ms, 0 | 20260505 13:44:10.039 | INFO | PRE_HQ : 61.2%, 1.8, 57.4ms, 0 | 20260505 13:44:10.039 | INFO | BAZWRITER : 64.2%, 4.0, 131.6ms, 0 | 20260505 13:44:10.039 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 13:44:10.039 | INFO | Uploaded 36GiB to GPU in 2.21462s (16.2556GiB/s) | 20260505 13:44:10.039 | INFO | Downloaded 38GiB to GPU in 2.26811s (16.7541GiB/s) | 20260505 13:44:15.164 | INFO | Analyzing chunk frames = [3662336,3662848) | 20260505 13:44:15.164 | INFO | MemInfo: { | 20260505 13:44:15.164 | INFO | "gpu" : " 17 GB", | 20260505 13:44:15.164 | INFO | "huge" : "", | 20260505 13:44:15.164 | INFO | "huge_shared" : " 27 GB", | 20260505 13:44:15.164 | INFO | "malloc" : " 14 GB", | 20260505 13:44:15.164 | INFO | "pinned" : " 29 GB" | 20260505 13:44:15.164 | INFO | } | 20260505 13:44:15.165 | INFO | Perf report for the last 5.12607 seconds: | 20260505 13:44:15.165 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:44:15.165 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:15.165 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:15.165 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:15.165 | INFO | ANALYSIS : 63.7%, 3.0, 99.1ms, 0 | 20260505 13:44:15.165 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:15.165 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:15.165 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:44:15.165 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:15.165 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:15.165 | INFO | Uploaded 36GiB to GPU in 2.19842s (16.3754GiB/s) | 20260505 13:44:15.165 | INFO | Downloaded 34GiB to GPU in 2.02797s (16.7656GiB/s) | 20260505 13:44:20.306 | INFO | Analyzing chunk frames = [3662848,3663360) | 20260505 13:44:20.306 | INFO | MemInfo: { | 20260505 13:44:20.306 | INFO | "gpu" : " 17 GB", | 20260505 13:44:20.306 | INFO | "huge" : "", | 20260505 13:44:20.306 | INFO | "huge_shared" : " 27 GB", | 20260505 13:44:20.306 | INFO | "malloc" : " 14 GB", | 20260505 13:44:20.306 | INFO | "pinned" : " 29 GB" | 20260505 13:44:20.307 | INFO | } | 20260505 13:44:20.307 | INFO | Perf report for the last 5.14195 seconds: | 20260505 13:44:20.307 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:44:20.307 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:20.307 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:20.307 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:20.307 | INFO | ANALYSIS : 63.2%, 3.0, 98.3ms, 0 | 20260505 13:44:20.307 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:20.307 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:20.307 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 13:44:20.307 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:20.307 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:20.307 | INFO | Uploaded 36GiB to GPU in 2.21434s (16.2576GiB/s) | 20260505 13:44:20.307 | INFO | Downloaded 34GiB to GPU in 2.11904s (16.045GiB/s) | 20260505 13:44:25.454 | INFO | Analyzing chunk frames = [3663360,3663872) | 20260505 13:44:25.454 | INFO | MemInfo: { | 20260505 13:44:25.454 | INFO | "gpu" : " 17 GB", | 20260505 13:44:25.454 | INFO | "huge" : "", | 20260505 13:44:25.454 | INFO | "huge_shared" : " 27 GB", | 20260505 13:44:25.454 | INFO | "malloc" : " 14 GB", | 20260505 13:44:25.454 | INFO | "pinned" : " 29 GB" | 20260505 13:44:25.454 | INFO | } | 20260505 13:44:25.454 | INFO | Perf report for the last 5.14765 seconds: | 20260505 13:44:25.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:44:25.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:25.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:25.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:25.454 | INFO | ANALYSIS : 62.9%, 3.0, 97.8ms, 0 | 20260505 13:44:25.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:25.454 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:25.454 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 13:44:25.454 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:25.454 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:25.454 | INFO | Uploaded 36GiB to GPU in 2.19577s (16.3952GiB/s) | 20260505 13:44:25.455 | INFO | Downloaded 34GiB to GPU in 2.13056s (15.9582GiB/s) | 20260505 13:44:30.595 | INFO | Analyzing chunk frames = [3663872,3664384) | 20260505 13:44:30.595 | INFO | MemInfo: { | 20260505 13:44:30.595 | INFO | "gpu" : " 17 GB", | 20260505 13:44:30.595 | INFO | "huge" : "", | 20260505 13:44:30.595 | INFO | "huge_shared" : " 27 GB", | 20260505 13:44:30.595 | INFO | "malloc" : " 14 GB", | 20260505 13:44:30.595 | INFO | "pinned" : " 29 GB" | 20260505 13:44:30.595 | INFO | } | 20260505 13:44:30.595 | INFO | Perf report for the last 5.14075 seconds: | 20260505 13:44:30.595 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:44:30.595 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:30.595 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:30.595 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:30.595 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 13:44:30.595 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:30.595 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:30.595 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:44:30.595 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:30.595 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:30.595 | INFO | Uploaded 36GiB to GPU in 2.17246s (16.571GiB/s) | 20260505 13:44:30.595 | INFO | Downloaded 34GiB to GPU in 2.12572s (15.9945GiB/s) | 20260505 13:44:35.728 | INFO | Analyzing chunk frames = [3664384,3664896) | 20260505 13:44:35.728 | INFO | MemInfo: { | 20260505 13:44:35.728 | INFO | "gpu" : " 17 GB", | 20260505 13:44:35.728 | INFO | "huge" : "", | 20260505 13:44:35.728 | INFO | "huge_shared" : " 27 GB", | 20260505 13:44:35.728 | INFO | "malloc" : " 14 GB", | 20260505 13:44:35.728 | INFO | "pinned" : " 29 GB" | 20260505 13:44:35.728 | INFO | } | 20260505 13:44:35.728 | INFO | Perf report for the last 5.13272 seconds: | 20260505 13:44:35.728 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:44:35.728 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:35.728 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:35.728 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:35.728 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 13:44:35.728 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:35.728 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:35.728 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 13:44:35.728 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:35.728 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:35.728 | INFO | Uploaded 36GiB to GPU in 2.16462s (16.6311GiB/s) | 20260505 13:44:35.728 | INFO | Downloaded 34GiB to GPU in 2.10718s (16.1353GiB/s) | 20260505 13:44:40.864 | INFO | Analyzing chunk frames = [3664896,3665408) | 20260505 13:44:40.864 | INFO | MemInfo: { | 20260505 13:44:40.864 | INFO | "gpu" : " 17 GB", | 20260505 13:44:40.864 | INFO | "huge" : "", | 20260505 13:44:40.864 | INFO | "huge_shared" : " 27 GB", | 20260505 13:44:40.864 | INFO | "malloc" : " 14 GB", | 20260505 13:44:40.864 | INFO | "pinned" : " 29 GB" | 20260505 13:44:40.864 | INFO | } | 20260505 13:44:40.864 | INFO | Perf report for the last 5.13601 seconds: | 20260505 13:44:40.864 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:44:40.864 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:40.864 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:40.864 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:40.864 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 13:44:40.864 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:40.864 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:40.864 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 13:44:40.864 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:40.864 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:40.864 | INFO | Uploaded 36GiB to GPU in 2.16986s (16.5909GiB/s) | 20260505 13:44:40.864 | INFO | Downloaded 34GiB to GPU in 2.14973s (15.816GiB/s) | 20260505 13:44:46.007 | INFO | Analyzing chunk frames = [3665408,3665920) | 20260505 13:44:46.007 | INFO | MemInfo: { | 20260505 13:44:46.007 | INFO | "gpu" : " 17 GB", | 20260505 13:44:46.007 | INFO | "huge" : "", | 20260505 13:44:46.007 | INFO | "huge_shared" : " 27 GB", | 20260505 13:44:46.007 | INFO | "malloc" : " 14 GB", | 20260505 13:44:46.007 | INFO | "pinned" : " 29 GB" | 20260505 13:44:46.007 | INFO | } | 20260505 13:44:46.007 | INFO | Perf report for the last 5.1431 seconds: | 20260505 13:44:46.007 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:44:46.007 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:46.007 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:46.007 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:46.007 | INFO | ANALYSIS : 63.5%, 3.0, 99.3ms, 0 | 20260505 13:44:46.007 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:46.007 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:46.007 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 13:44:46.007 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:46.007 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:46.007 | INFO | Uploaded 36GiB to GPU in 2.17063s (16.585GiB/s) | 20260505 13:44:46.007 | INFO | Downloaded 34GiB to GPU in 2.10128s (16.1806GiB/s) | 20260505 13:44:46.146 | INFO | Starting a round of Baz IO | 20260505 13:44:49.370 | INFO | Populating RTMetrics report | 20260505 13:44:49.370 | INFO | RTMetrics report populated | 20260505 13:44:49.370 | INFO | Opening tmp RTMetrics json file | 20260505 13:44:49.391 | INFO | Serializing RTMetrics to json | 20260505 13:44:49.410 | INFO | Serialization done, swapping tmp file now... | 20260505 13:44:49.425 | INFO | RTMetrics done | 20260505 13:44:49.474 | INFO | Finished a round of Baz IO: Wrote 2.295 GiB in 3.328 seconds | 20260505 13:44:49.474 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.00% / 0.00% / 76.99% | 20260505 13:44:49.474 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.660904% / 0% / 5.16574e-05% | 20260505 13:44:49.474 | INFO | Baz IO overall: | 20260505 13:44:49.474 | INFO | BytesWritten : 2183997465309 | 20260505 13:44:49.474 | INFO | NumEvents : 1038864742445 | 20260505 13:44:51.151 | INFO | Analyzing chunk frames = [3665920,3666432) | 20260505 13:44:51.151 | INFO | MemInfo: { | 20260505 13:44:51.151 | INFO | "gpu" : " 17 GB", | 20260505 13:44:51.151 | INFO | "huge" : "", | 20260505 13:44:51.151 | INFO | "huge_shared" : " 27 GB", | 20260505 13:44:51.151 | INFO | "malloc" : " 14 GB", | 20260505 13:44:51.151 | INFO | "pinned" : " 29 GB" | 20260505 13:44:51.151 | INFO | } | 20260505 13:44:51.151 | INFO | Perf report for the last 5.14392 seconds: | 20260505 13:44:51.151 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:44:51.151 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:51.151 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:51.151 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:51.151 | INFO | ANALYSIS : 64.3%, 3.0, 99.9ms, 0 | 20260505 13:44:51.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:51.151 | INFO | RT_METRICS : 46.2%, 1.0, 24.2ms, 0 | 20260505 13:44:51.151 | INFO | PRE_HQ : 60.9%, 1.8, 57.5ms, 0 | 20260505 13:44:51.151 | INFO | BAZWRITER : 64.1%, 4.0, 132.2ms, 0 | 20260505 13:44:51.151 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 13:44:51.151 | INFO | Uploaded 36GiB to GPU in 2.18339s (16.4881GiB/s) | 20260505 13:44:51.151 | INFO | Downloaded 38GiB to GPU in 2.27197s (16.7256GiB/s) | 20260505 13:44:56.286 | INFO | Analyzing chunk frames = [3666432,3666944) | 20260505 13:44:56.286 | INFO | MemInfo: { | 20260505 13:44:56.286 | INFO | "gpu" : " 17 GB", | 20260505 13:44:56.286 | INFO | "huge" : "", | 20260505 13:44:56.286 | INFO | "huge_shared" : " 27 GB", | 20260505 13:44:56.286 | INFO | "malloc" : " 14 GB", | 20260505 13:44:56.286 | INFO | "pinned" : " 29 GB" | 20260505 13:44:56.286 | INFO | } | 20260505 13:44:56.286 | INFO | Perf report for the last 5.13557 seconds: | 20260505 13:44:56.286 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:44:56.286 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:56.286 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:56.286 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:56.287 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 13:44:56.287 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:56.287 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:56.287 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:44:56.287 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:56.287 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:44:56.287 | INFO | Uploaded 36GiB to GPU in 2.20081s (16.3576GiB/s) | 20260505 13:44:56.287 | INFO | Downloaded 34GiB to GPU in 2.03681s (16.6928GiB/s) | 20260505 13:45:01.419 | INFO | Analyzing chunk frames = [3666944,3667456) | 20260505 13:45:01.419 | INFO | MemInfo: { | 20260505 13:45:01.419 | INFO | "gpu" : " 17 GB", | 20260505 13:45:01.419 | INFO | "huge" : "", | 20260505 13:45:01.419 | INFO | "huge_shared" : " 27 GB", | 20260505 13:45:01.419 | INFO | "malloc" : " 14 GB", | 20260505 13:45:01.419 | INFO | "pinned" : " 29 GB" | 20260505 13:45:01.419 | INFO | } | 20260505 13:45:01.419 | INFO | Perf report for the last 5.13299 seconds: | 20260505 13:45:01.419 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:45:01.419 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:01.420 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:01.420 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:01.420 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 13:45:01.420 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:01.420 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:01.420 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:45:01.420 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:01.420 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:01.420 | INFO | Uploaded 36GiB to GPU in 2.19392s (16.409GiB/s) | 20260505 13:45:01.420 | INFO | Downloaded 34GiB to GPU in 2.13359s (15.9356GiB/s) | 20260505 13:45:06.561 | INFO | Analyzing chunk frames = [3667456,3667968) | 20260505 13:45:06.561 | INFO | MemInfo: { | 20260505 13:45:06.561 | INFO | "gpu" : " 17 GB", | 20260505 13:45:06.561 | INFO | "huge" : "", | 20260505 13:45:06.561 | INFO | "huge_shared" : " 27 GB", | 20260505 13:45:06.561 | INFO | "malloc" : " 14 GB", | 20260505 13:45:06.561 | INFO | "pinned" : " 29 GB" | 20260505 13:45:06.561 | INFO | } | 20260505 13:45:06.561 | INFO | Perf report for the last 5.14134 seconds: | 20260505 13:45:06.561 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:45:06.561 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:06.561 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:06.561 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:06.561 | INFO | ANALYSIS : 63.2%, 3.0, 98.3ms, 0 | 20260505 13:45:06.561 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:06.561 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:06.561 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 13:45:06.561 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:06.561 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:06.561 | INFO | Uploaded 36GiB to GPU in 2.21551s (16.2491GiB/s) | 20260505 13:45:06.561 | INFO | Downloaded 34GiB to GPU in 2.13549s (15.9214GiB/s) | 20260505 13:45:11.706 | INFO | Analyzing chunk frames = [3667968,3668480) | 20260505 13:45:11.706 | INFO | MemInfo: { | 20260505 13:45:11.706 | INFO | "gpu" : " 17 GB", | 20260505 13:45:11.706 | INFO | "huge" : "", | 20260505 13:45:11.706 | INFO | "huge_shared" : " 27 GB", | 20260505 13:45:11.706 | INFO | "malloc" : " 14 GB", | 20260505 13:45:11.706 | INFO | "pinned" : " 29 GB" | 20260505 13:45:11.706 | INFO | } | 20260505 13:45:11.706 | INFO | Perf report for the last 5.14562 seconds: | 20260505 13:45:11.706 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:45:11.706 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:11.706 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:11.706 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:11.707 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 13:45:11.707 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:11.707 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:11.707 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:45:11.707 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:11.707 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:11.707 | INFO | Uploaded 36GiB to GPU in 2.16341s (16.6404GiB/s) | 20260505 13:45:11.707 | INFO | Downloaded 34GiB to GPU in 2.12517s (15.9987GiB/s) | 20260505 13:45:16.848 | INFO | Analyzing chunk frames = [3668480,3668992) | 20260505 13:45:16.848 | INFO | MemInfo: { | 20260505 13:45:16.848 | INFO | "gpu" : " 17 GB", | 20260505 13:45:16.848 | INFO | "huge" : "", | 20260505 13:45:16.848 | INFO | "huge_shared" : " 27 GB", | 20260505 13:45:16.848 | INFO | "malloc" : " 14 GB", | 20260505 13:45:16.849 | INFO | "pinned" : " 29 GB" | 20260505 13:45:16.849 | INFO | } | 20260505 13:45:16.849 | INFO | Perf report for the last 5.14231 seconds: | 20260505 13:45:16.849 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:45:16.849 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:16.849 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:16.849 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:16.849 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 13:45:16.849 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:16.849 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:16.849 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:45:16.849 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:16.849 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:16.849 | INFO | Uploaded 36GiB to GPU in 2.20008s (16.3631GiB/s) | 20260505 13:45:16.849 | INFO | Downloaded 34GiB to GPU in 2.11253s (16.0945GiB/s) | 20260505 13:45:21.977 | INFO | Analyzing chunk frames = [3668992,3669504) | 20260505 13:45:21.977 | INFO | MemInfo: { | 20260505 13:45:21.977 | INFO | "gpu" : " 17 GB", | 20260505 13:45:21.977 | INFO | "huge" : "", | 20260505 13:45:21.977 | INFO | "huge_shared" : " 27 GB", | 20260505 13:45:21.978 | INFO | "malloc" : " 14 GB", | 20260505 13:45:21.978 | INFO | "pinned" : " 29 GB" | 20260505 13:45:21.978 | INFO | } | 20260505 13:45:21.978 | INFO | Perf report for the last 5.12888 seconds: | 20260505 13:45:21.978 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:45:21.978 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:21.978 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:21.978 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:21.978 | INFO | ANALYSIS : 64.2%, 3.0, 100.1ms, 0 | 20260505 13:45:21.978 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:21.978 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:21.978 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 13:45:21.978 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:21.978 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:21.978 | INFO | Uploaded 36GiB to GPU in 2.1582s (16.6806GiB/s) | 20260505 13:45:21.978 | INFO | Downloaded 34GiB to GPU in 2.18683s (15.5476GiB/s) | 20260505 13:45:27.114 | INFO | Analyzing chunk frames = [3669504,3670016) | 20260505 13:45:27.114 | INFO | MemInfo: { | 20260505 13:45:27.114 | INFO | "gpu" : " 17 GB", | 20260505 13:45:27.114 | INFO | "huge" : "", | 20260505 13:45:27.114 | INFO | "huge_shared" : " 27 GB", | 20260505 13:45:27.114 | INFO | "malloc" : " 14 GB", | 20260505 13:45:27.114 | INFO | "pinned" : " 29 GB" | 20260505 13:45:27.114 | INFO | } | 20260505 13:45:27.114 | INFO | Perf report for the last 5.13647 seconds: | 20260505 13:45:27.114 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:45:27.114 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:27.114 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:27.114 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:27.114 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 13:45:27.114 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:27.114 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:27.114 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 13:45:27.114 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:27.114 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:27.114 | INFO | Uploaded 36GiB to GPU in 2.14763s (16.7627GiB/s) | 20260505 13:45:27.114 | INFO | Downloaded 34GiB to GPU in 2.10407s (16.1591GiB/s) | 20260505 13:45:27.114 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 13:45:27.115 | INFO | Total mem usage: 15.3 GiB | 20260505 13:45:27.115 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 13:45:27.115 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -7.11%) | 20260505 13:45:27.115 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 13:45:27.115 | INFO | High Water Breakdown: | 20260505 13:45:27.115 | INFO | 61.1% : 8.75 GiB ( 91.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 13:45:27.115 | INFO | 38.7% : 5.54 GiB ( 83.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 13:45:27.115 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 13:45:27.115 | INFO | ------------------------------------------ | 20260505 13:45:27.115 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:45:27.115 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 13:45:27.115 | INFO | Total mem usage: 26.5 GiB | 20260505 13:45:27.115 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 13:45:27.115 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 13:45:27.115 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 13:45:27.115 | INFO | High Water Breakdown: | 20260505 13:45:27.115 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:45:27.115 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:45:27.115 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:45:27.115 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:45:27.115 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:45:27.115 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:45:27.115 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 13:45:27.115 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 13:45:27.115 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:45:27.115 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 13:45:27.115 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 13:45:27.115 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 13:45:27.115 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 13:45:27.115 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 13:45:27.115 | INFO | ------------------------------------------ | 20260505 13:45:27.115 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:45:27.115 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 13:45:27.115 | INFO | Total mem usage: 25.6 GiB | 20260505 13:45:27.115 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 13:45:27.115 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 13:45:27.115 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 13:45:27.115 | INFO | High Water Breakdown: | 20260505 13:45:27.115 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 13:45:27.115 | INFO | ------------------------------------------ | 20260505 13:45:27.115 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:45:27.115 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 13:45:27.115 | INFO | Total mem usage: 15.7 GiB | 20260505 13:45:27.115 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 13:45:27.115 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 13:45:27.115 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 13:45:27.115 | INFO | High Water Breakdown: | 20260505 13:45:27.115 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 13:45:27.115 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 13:45:27.115 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 13:45:27.115 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 13:45:27.115 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 13:45:27.115 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 13:45:27.115 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 13:45:27.115 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 13:45:27.115 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 13:45:27.115 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:45:27.115 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 13:45:27.115 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:45:27.115 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 13:45:27.115 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 13:45:27.115 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 13:45:27.115 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 13:45:27.115 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:45:27.115 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 13:45:27.115 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:45:27.115 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:45:27.115 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:45:27.115 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 13:45:27.115 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 13:45:27.115 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:45:27.116 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 13:45:27.116 | INFO | ------------------------------------------ | 20260505 13:45:27.116 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:45:27.116 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 13:45:27.116 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:45:27.116 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:45:27.116 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.69e+01, 1.37e+01 / 2.00e+00 | 20260505 13:45:27.116 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 1.01e-01, 1.04e-02 / 4.41e-03 | 20260505 13:45:27.116 | INFO | PerfReport: 11.78%, Baseline -- 3.76e+00 / 4.48e+00, 5.90e-01 / 1.91e-01 | 20260505 13:45:27.116 | INFO | PerfReport: 3.31%, Histogram -- 1.06e+00 / 1.48e+00, 2.00e-01 / 1.03e-01 | 20260505 13:45:27.116 | INFO | PerfReport: 16.02%, DME -- 4.09e+01 / 4.62e+01, 4.76e+00 / 2.40e+00 | 20260505 13:45:27.116 | INFO | PerfReport: 56.34%, FrameLabeling -- 1.80e+01 / 1.99e+01, 1.39e+00 / 1.88e+00 | 20260505 13:45:27.116 | INFO | PerfReport: 7.44%, PulseAccumulating -- 2.37e+00 / 2.92e+00, 4.44e-01 / 1.80e-01 | 20260505 13:45:27.116 | INFO | PerfReport: 3.57%, Metrics -- 1.14e+00 / 1.46e+00, 3.19e-01 / 1.55e-01 | 20260505 13:45:27.116 | INFO | PerfReport: 1.25%, PulseSerializing -- 4.00e-01 / 4.51e-01, 7.07e-02 / 2.88e-02 | 20260505 13:45:27.116 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 13:45:27.116 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:45:27.116 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:45:27.116 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260505 13:45:27.116 | INFO | PerfReport: 22.56%, Upload -- 2.28e+01 / 0.00e+00, 8.03e+00 / 0.00e+00 | 20260505 13:45:27.116 | INFO | PerfReport: 22.03%, Download -- 2.22e+01 / 0.00e+00, 8.10e+00 / 0.00e+00 | 20260505 13:45:27.116 | INFO | PerfReport: 31.63%, Compute -- 3.19e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 13:45:27.116 | INFO | PerfReport: 2.15%, LockUpload -- 2.17e+00 / 0.00e+00, 7.85e+00 / 0.00e+00 | 20260505 13:45:27.116 | INFO | PerfReport: 1.43%, LockDownload -- 1.44e+00 / 0.00e+00, 2.95e+00 / 0.00e+00 | 20260505 13:45:27.116 | INFO | PerfReport: 20.18%, LockCompute -- 2.04e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260505 13:45:27.116 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.11e-02 / 0.00e+00 | 20260505 13:45:27.247 | INFO | Starting a round of Baz IO | 20260505 13:45:30.487 | INFO | Populating RTMetrics report | 20260505 13:45:30.488 | INFO | RTMetrics report populated | 20260505 13:45:30.488 | INFO | Opening tmp RTMetrics json file | 20260505 13:45:30.508 | INFO | Serializing RTMetrics to json | 20260505 13:45:30.531 | INFO | Serialization done, swapping tmp file now... | 20260505 13:45:30.553 | INFO | RTMetrics done | 20260505 13:45:30.592 | INFO | Finished a round of Baz IO: Wrote 2.29 GiB in 3.345 seconds | 20260505 13:45:30.592 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 23.00% / 0.00% / 77.00% | 20260505 13:45:30.592 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.657137% / 3.97364e-06% / 4.37101e-05% | 20260505 13:45:30.592 | INFO | Baz IO overall: | 20260505 13:45:30.592 | INFO | BytesWritten : 2186456761372 | 20260505 13:45:30.592 | INFO | NumEvents : 1040026082732 | 20260505 13:45:32.265 | INFO | Analyzing chunk frames = [3670016,3670528) | 20260505 13:45:32.265 | INFO | MemInfo: { | 20260505 13:45:32.265 | INFO | "gpu" : " 17 GB", | 20260505 13:45:32.265 | INFO | "huge" : "", | 20260505 13:45:32.265 | INFO | "huge_shared" : " 27 GB", | 20260505 13:45:32.265 | INFO | "malloc" : " 14 GB", | 20260505 13:45:32.265 | INFO | "pinned" : " 29 GB" | 20260505 13:45:32.265 | INFO | } | 20260505 13:45:32.265 | INFO | Perf report for the last 5.15105 seconds: | 20260505 13:45:32.265 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:45:32.265 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:32.265 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:32.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:32.265 | INFO | ANALYSIS : 64.4%, 3.0, 100.2ms, 0 | 20260505 13:45:32.265 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:32.265 | INFO | RT_METRICS : 46.2%, 1.0, 24.3ms, 0 | 20260505 13:45:32.265 | INFO | PRE_HQ : 60.7%, 1.8, 57.9ms, 0 | 20260505 13:45:32.265 | INFO | BAZWRITER : 64.3%, 4.0, 131.3ms, 0 | 20260505 13:45:32.265 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 13:45:32.266 | INFO | Uploaded 36GiB to GPU in 2.18117s (16.5049GiB/s) | 20260505 13:45:32.266 | INFO | Downloaded 38GiB to GPU in 2.2841s (16.6368GiB/s) | 20260505 13:45:37.403 | INFO | Analyzing chunk frames = [3670528,3671040) | 20260505 13:45:37.403 | INFO | MemInfo: { | 20260505 13:45:37.403 | INFO | "gpu" : " 17 GB", | 20260505 13:45:37.403 | INFO | "huge" : "", | 20260505 13:45:37.403 | INFO | "huge_shared" : " 27 GB", | 20260505 13:45:37.403 | INFO | "malloc" : " 14 GB", | 20260505 13:45:37.403 | INFO | "pinned" : " 29 GB" | 20260505 13:45:37.403 | INFO | } | 20260505 13:45:37.403 | INFO | Perf report for the last 5.13772 seconds: | 20260505 13:45:37.403 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:45:37.403 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:37.403 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:37.403 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:37.403 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 13:45:37.403 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:37.403 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:37.403 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:45:37.403 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:37.403 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:37.403 | INFO | Uploaded 36GiB to GPU in 2.19976s (16.3654GiB/s) | 20260505 13:45:37.403 | INFO | Downloaded 34GiB to GPU in 2.04172s (16.6526GiB/s) | 20260505 13:45:42.545 | INFO | Analyzing chunk frames = [3671040,3671552) | 20260505 13:45:42.545 | INFO | MemInfo: { | 20260505 13:45:42.545 | INFO | "gpu" : " 17 GB", | 20260505 13:45:42.545 | INFO | "huge" : "", | 20260505 13:45:42.545 | INFO | "huge_shared" : " 27 GB", | 20260505 13:45:42.545 | INFO | "malloc" : " 14 GB", | 20260505 13:45:42.545 | INFO | "pinned" : " 29 GB" | 20260505 13:45:42.545 | INFO | } | 20260505 13:45:42.545 | INFO | Perf report for the last 5.14188 seconds: | 20260505 13:45:42.545 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:45:42.545 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:42.545 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:42.545 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:42.545 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 13:45:42.545 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:42.545 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:42.545 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 13:45:42.545 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:42.545 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:42.545 | INFO | Uploaded 36GiB to GPU in 2.22166s (16.2041GiB/s) | 20260505 13:45:42.545 | INFO | Downloaded 34GiB to GPU in 2.11155s (16.1019GiB/s) | 20260505 13:45:47.676 | INFO | Analyzing chunk frames = [3671552,3672064) | 20260505 13:45:47.676 | INFO | MemInfo: { | 20260505 13:45:47.676 | INFO | "gpu" : " 17 GB", | 20260505 13:45:47.676 | INFO | "huge" : "", | 20260505 13:45:47.676 | INFO | "huge_shared" : " 27 GB", | 20260505 13:45:47.676 | INFO | "malloc" : " 14 GB", | 20260505 13:45:47.676 | INFO | "pinned" : " 29 GB" | 20260505 13:45:47.676 | INFO | } | 20260505 13:45:47.677 | INFO | Perf report for the last 5.13167 seconds: | 20260505 13:45:47.677 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:45:47.677 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:47.677 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:47.677 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:47.677 | INFO | ANALYSIS : 62.9%, 3.0, 97.6ms, 0 | 20260505 13:45:47.677 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:47.677 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:47.677 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:45:47.677 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:47.677 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:47.677 | INFO | Uploaded 36GiB to GPU in 2.19221s (16.4218GiB/s) | 20260505 13:45:47.677 | INFO | Downloaded 34GiB to GPU in 2.13673s (15.9122GiB/s) | 20260505 13:45:52.813 | INFO | Analyzing chunk frames = [3672064,3672576) | 20260505 13:45:52.813 | INFO | MemInfo: { | 20260505 13:45:52.813 | INFO | "gpu" : " 17 GB", | 20260505 13:45:52.813 | INFO | "huge" : "", | 20260505 13:45:52.813 | INFO | "huge_shared" : " 27 GB", | 20260505 13:45:52.813 | INFO | "malloc" : " 14 GB", | 20260505 13:45:52.813 | INFO | "pinned" : " 29 GB" | 20260505 13:45:52.813 | INFO | } | 20260505 13:45:52.813 | INFO | Perf report for the last 5.13694 seconds: | 20260505 13:45:52.813 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:45:52.813 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:52.813 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:52.814 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:52.814 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 13:45:52.814 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:52.814 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:52.814 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:45:52.814 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:52.814 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:52.814 | INFO | Uploaded 36GiB to GPU in 2.16709s (16.6122GiB/s) | 20260505 13:45:52.814 | INFO | Downloaded 34GiB to GPU in 2.11419s (16.0818GiB/s) | 20260505 13:45:57.961 | INFO | Analyzing chunk frames = [3672576,3673088) | 20260505 13:45:57.961 | INFO | MemInfo: { | 20260505 13:45:57.961 | INFO | "gpu" : " 17 GB", | 20260505 13:45:57.961 | INFO | "huge" : "", | 20260505 13:45:57.961 | INFO | "huge_shared" : " 27 GB", | 20260505 13:45:57.961 | INFO | "malloc" : " 14 GB", | 20260505 13:45:57.961 | INFO | "pinned" : " 29 GB" | 20260505 13:45:57.962 | INFO | } | 20260505 13:45:57.962 | INFO | Perf report for the last 5.1482 seconds: | 20260505 13:45:57.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:45:57.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:57.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:57.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:57.962 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 13:45:57.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:57.962 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:57.962 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 13:45:57.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:57.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:45:57.962 | INFO | Uploaded 36GiB to GPU in 2.166s (16.6205GiB/s) | 20260505 13:45:57.962 | INFO | Downloaded 34GiB to GPU in 2.11169s (16.1008GiB/s) | 20260505 13:46:03.102 | INFO | Analyzing chunk frames = [3673088,3673600) | 20260505 13:46:03.102 | INFO | MemInfo: { | 20260505 13:46:03.102 | INFO | "gpu" : " 17 GB", | 20260505 13:46:03.102 | INFO | "huge" : "", | 20260505 13:46:03.102 | INFO | "huge_shared" : " 27 GB", | 20260505 13:46:03.103 | INFO | "malloc" : " 14 GB", | 20260505 13:46:03.103 | INFO | "pinned" : " 29 GB" | 20260505 13:46:03.103 | INFO | } | 20260505 13:46:03.103 | INFO | Perf report for the last 5.14106 seconds: | 20260505 13:46:03.103 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:46:03.103 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:03.103 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:03.103 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:03.103 | INFO | ANALYSIS : 64.0%, 3.0, 100.0ms, 0 | 20260505 13:46:03.103 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:03.103 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:03.103 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 13:46:03.103 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:03.103 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:03.103 | INFO | Uploaded 36GiB to GPU in 2.15697s (16.69GiB/s) | 20260505 13:46:03.103 | INFO | Downloaded 34GiB to GPU in 2.20042s (15.4516GiB/s) | 20260505 13:46:08.235 | INFO | Analyzing chunk frames = [3673600,3674112) | 20260505 13:46:08.235 | INFO | MemInfo: { | 20260505 13:46:08.235 | INFO | "gpu" : " 17 GB", | 20260505 13:46:08.235 | INFO | "huge" : "", | 20260505 13:46:08.235 | INFO | "huge_shared" : " 27 GB", | 20260505 13:46:08.235 | INFO | "malloc" : " 14 GB", | 20260505 13:46:08.235 | INFO | "pinned" : " 29 GB" | 20260505 13:46:08.235 | INFO | } | 20260505 13:46:08.235 | INFO | Perf report for the last 5.13264 seconds: | 20260505 13:46:08.235 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:46:08.235 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:08.235 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:08.235 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:08.235 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 13:46:08.235 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:08.235 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:08.235 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 13:46:08.236 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:08.236 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:08.236 | INFO | Uploaded 36GiB to GPU in 2.21411s (16.2593GiB/s) | 20260505 13:46:08.236 | INFO | Downloaded 34GiB to GPU in 2.10584s (16.1456GiB/s) | 20260505 13:46:08.366 | INFO | Starting a round of Baz IO | 20260505 13:46:11.606 | INFO | Populating RTMetrics report | 20260505 13:46:11.607 | INFO | RTMetrics report populated | 20260505 13:46:11.607 | INFO | Opening tmp RTMetrics json file | 20260505 13:46:11.627 | INFO | Serializing RTMetrics to json | 20260505 13:46:11.637 | INFO | Serialization done, swapping tmp file now... | 20260505 13:46:11.652 | INFO | RTMetrics done | 20260505 13:46:11.711 | INFO | Finished a round of Baz IO: Wrote 2.289 GiB in 3.345 seconds | 20260505 13:46:11.711 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.99% / 0.00% / 77.00% | 20260505 13:46:11.711 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.65612% / 0% / 8.34465e-05% | 20260505 13:46:11.711 | INFO | Baz IO overall: | 20260505 13:46:11.711 | INFO | BytesWritten : 2188915043869 | 20260505 13:46:11.711 | INFO | NumEvents : 1041187194041 | 20260505 13:46:13.369 | INFO | Analyzing chunk frames = [3674112,3674624) | 20260505 13:46:13.369 | INFO | MemInfo: { | 20260505 13:46:13.369 | INFO | "gpu" : " 17 GB", | 20260505 13:46:13.369 | INFO | "huge" : "", | 20260505 13:46:13.369 | INFO | "huge_shared" : " 27 GB", | 20260505 13:46:13.369 | INFO | "malloc" : " 14 GB", | 20260505 13:46:13.369 | INFO | "pinned" : " 29 GB" | 20260505 13:46:13.369 | INFO | } | 20260505 13:46:13.369 | INFO | Perf report for the last 5.13395 seconds: | 20260505 13:46:13.369 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:46:13.369 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:13.369 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:13.369 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:13.369 | INFO | ANALYSIS : 64.6%, 3.0, 100.0ms, 0 | 20260505 13:46:13.369 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:13.369 | INFO | RT_METRICS : 45.9%, 1.0, 24.0ms, 0 | 20260505 13:46:13.369 | INFO | PRE_HQ : 60.8%, 1.8, 57.4ms, 0 | 20260505 13:46:13.369 | INFO | BAZWRITER : 64.5%, 4.0, 131.4ms, 0 | 20260505 13:46:13.370 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 13:46:13.370 | INFO | Uploaded 36GiB to GPU in 2.17589s (16.545GiB/s) | 20260505 13:46:13.370 | INFO | Downloaded 38GiB to GPU in 2.2764s (16.6931GiB/s) | 20260505 13:46:18.523 | INFO | Analyzing chunk frames = [3674624,3675136) | 20260505 13:46:18.523 | INFO | MemInfo: { | 20260505 13:46:18.523 | INFO | "gpu" : " 17 GB", | 20260505 13:46:18.523 | INFO | "huge" : "", | 20260505 13:46:18.523 | INFO | "huge_shared" : " 27 GB", | 20260505 13:46:18.523 | INFO | "malloc" : " 14 GB", | 20260505 13:46:18.523 | INFO | "pinned" : " 29 GB" | 20260505 13:46:18.523 | INFO | } | 20260505 13:46:18.523 | INFO | Perf report for the last 5.15348 seconds: | 20260505 13:46:18.523 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:46:18.523 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:18.523 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:18.523 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:18.523 | INFO | ANALYSIS : 63.5%, 3.0, 99.4ms, 0 | 20260505 13:46:18.523 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:18.523 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:18.523 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 13:46:18.523 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:18.523 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:18.523 | INFO | Uploaded 36GiB to GPU in 2.18619s (16.467GiB/s) | 20260505 13:46:18.523 | INFO | Downloaded 34GiB to GPU in 2.05676s (16.5308GiB/s) | 20260505 13:46:23.662 | INFO | Analyzing chunk frames = [3675136,3675648) | 20260505 13:46:23.662 | INFO | MemInfo: { | 20260505 13:46:23.662 | INFO | "gpu" : " 17 GB", | 20260505 13:46:23.662 | INFO | "huge" : "", | 20260505 13:46:23.662 | INFO | "huge_shared" : " 27 GB", | 20260505 13:46:23.662 | INFO | "malloc" : " 14 GB", | 20260505 13:46:23.662 | INFO | "pinned" : " 29 GB" | 20260505 13:46:23.662 | INFO | } | 20260505 13:46:23.662 | INFO | Perf report for the last 5.1392 seconds: | 20260505 13:46:23.662 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:46:23.662 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:23.662 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:23.662 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:23.662 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 13:46:23.662 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:23.662 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:23.662 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:46:23.662 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:23.662 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:23.662 | INFO | Uploaded 36GiB to GPU in 2.21515s (16.2518GiB/s) | 20260505 13:46:23.662 | INFO | Downloaded 34GiB to GPU in 2.10463s (16.1549GiB/s) | 20260505 13:46:28.795 | INFO | Analyzing chunk frames = [3675648,3676160) | 20260505 13:46:28.795 | INFO | MemInfo: { | 20260505 13:46:28.795 | INFO | "gpu" : " 17 GB", | 20260505 13:46:28.795 | INFO | "huge" : "", | 20260505 13:46:28.795 | INFO | "huge_shared" : " 27 GB", | 20260505 13:46:28.795 | INFO | "malloc" : " 14 GB", | 20260505 13:46:28.795 | INFO | "pinned" : " 29 GB" | 20260505 13:46:28.795 | INFO | } | 20260505 13:46:28.796 | INFO | Perf report for the last 5.13344 seconds: | 20260505 13:46:28.796 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:46:28.796 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:28.796 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:28.796 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:28.796 | INFO | ANALYSIS : 62.8%, 3.0, 97.4ms, 0 | 20260505 13:46:28.796 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:28.796 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:28.796 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 13:46:28.796 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:28.796 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:28.796 | INFO | Uploaded 36GiB to GPU in 2.17474s (16.5537GiB/s) | 20260505 13:46:28.796 | INFO | Downloaded 34GiB to GPU in 2.12827s (15.9754GiB/s) | 20260505 13:46:33.929 | INFO | Analyzing chunk frames = [3676160,3676672) | 20260505 13:46:33.929 | INFO | MemInfo: { | 20260505 13:46:33.929 | INFO | "gpu" : " 17 GB", | 20260505 13:46:33.929 | INFO | "huge" : "", | 20260505 13:46:33.929 | INFO | "huge_shared" : " 27 GB", | 20260505 13:46:33.929 | INFO | "malloc" : " 14 GB", | 20260505 13:46:33.930 | INFO | "pinned" : " 29 GB" | 20260505 13:46:33.930 | INFO | } | 20260505 13:46:33.930 | INFO | Perf report for the last 5.13407 seconds: | 20260505 13:46:33.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:46:33.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:33.930 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:33.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:33.930 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 13:46:33.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:33.930 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:33.930 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:46:33.930 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:33.930 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:33.930 | INFO | Uploaded 36GiB to GPU in 2.15408s (16.7124GiB/s) | 20260505 13:46:33.930 | INFO | Downloaded 34GiB to GPU in 2.1259s (15.9932GiB/s) | 20260505 13:46:39.067 | INFO | Analyzing chunk frames = [3676672,3677184) | 20260505 13:46:39.067 | INFO | MemInfo: { | 20260505 13:46:39.067 | INFO | "gpu" : " 17 GB", | 20260505 13:46:39.067 | INFO | "huge" : "", | 20260505 13:46:39.067 | INFO | "huge_shared" : " 27 GB", | 20260505 13:46:39.067 | INFO | "malloc" : " 14 GB", | 20260505 13:46:39.067 | INFO | "pinned" : " 29 GB" | 20260505 13:46:39.067 | INFO | } | 20260505 13:46:39.067 | INFO | Perf report for the last 5.1375 seconds: | 20260505 13:46:39.067 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:46:39.067 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:39.067 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:39.067 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:39.067 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 13:46:39.067 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:39.067 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:39.067 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:46:39.067 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:39.067 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:39.067 | INFO | Uploaded 36GiB to GPU in 2.1681s (16.6044GiB/s) | 20260505 13:46:39.067 | INFO | Downloaded 34GiB to GPU in 2.11032s (16.1113GiB/s) | 20260505 13:46:44.218 | INFO | Analyzing chunk frames = [3677184,3677696) | 20260505 13:46:44.218 | INFO | MemInfo: { | 20260505 13:46:44.218 | INFO | "gpu" : " 17 GB", | 20260505 13:46:44.218 | INFO | "huge" : "", | 20260505 13:46:44.218 | INFO | "huge_shared" : " 27 GB", | 20260505 13:46:44.218 | INFO | "malloc" : " 14 GB", | 20260505 13:46:44.218 | INFO | "pinned" : " 29 GB" | 20260505 13:46:44.218 | INFO | } | 20260505 13:46:44.218 | INFO | Perf report for the last 5.15079 seconds: | 20260505 13:46:44.218 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:46:44.218 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:44.218 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:44.218 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:44.218 | INFO | ANALYSIS : 63.5%, 3.0, 99.4ms, 0 | 20260505 13:46:44.218 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:44.218 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:44.218 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 13:46:44.218 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:44.218 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:44.218 | INFO | Uploaded 36GiB to GPU in 2.18391s (16.4842GiB/s) | 20260505 13:46:44.218 | INFO | Downloaded 34GiB to GPU in 2.17829s (15.6085GiB/s) | 20260505 13:46:49.360 | INFO | Analyzing chunk frames = [3677696,3678208) | 20260505 13:46:49.360 | INFO | MemInfo: { | 20260505 13:46:49.360 | INFO | "gpu" : " 17 GB", | 20260505 13:46:49.360 | INFO | "huge" : "", | 20260505 13:46:49.360 | INFO | "huge_shared" : " 27 GB", | 20260505 13:46:49.361 | INFO | "malloc" : " 14 GB", | 20260505 13:46:49.361 | INFO | "pinned" : " 29 GB" | 20260505 13:46:49.361 | INFO | } | 20260505 13:46:49.361 | INFO | Perf report for the last 5.14268 seconds: | 20260505 13:46:49.361 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:46:49.361 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:49.361 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:49.361 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:49.361 | INFO | ANALYSIS : 63.5%, 3.0, 99.3ms, 0 | 20260505 13:46:49.361 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:49.361 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:49.361 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:46:49.361 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:49.361 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:49.361 | INFO | Uploaded 36GiB to GPU in 2.21321s (16.266GiB/s) | 20260505 13:46:49.361 | INFO | Downloaded 34GiB to GPU in 2.10597s (16.1446GiB/s) | 20260505 13:46:49.361 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 13:46:49.361 | INFO | Total mem usage: 15.3 GiB | 20260505 13:46:49.361 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 13:46:49.361 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.73%) | 20260505 13:46:49.361 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 13:46:49.361 | INFO | High Water Breakdown: | 20260505 13:46:49.361 | INFO | 61% : 8.68 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 13:46:49.361 | INFO | 38.8% : 5.52 GiB ( 83.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 13:46:49.361 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 13:46:49.361 | INFO | ------------------------------------------ | 20260505 13:46:49.361 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:46:49.361 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 13:46:49.361 | INFO | Total mem usage: 26.5 GiB | 20260505 13:46:49.361 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 13:46:49.361 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 13:46:49.361 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 13:46:49.361 | INFO | High Water Breakdown: | 20260505 13:46:49.361 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:46:49.361 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:46:49.361 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:46:49.361 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:46:49.361 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:46:49.361 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:46:49.361 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 13:46:49.361 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 13:46:49.361 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:46:49.361 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 13:46:49.361 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 13:46:49.361 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 13:46:49.361 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 13:46:49.361 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 13:46:49.361 | INFO | ------------------------------------------ | 20260505 13:46:49.361 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:46:49.362 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 13:46:49.362 | INFO | Total mem usage: 25.6 GiB | 20260505 13:46:49.362 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 13:46:49.362 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 13:46:49.362 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 13:46:49.362 | INFO | High Water Breakdown: | 20260505 13:46:49.362 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 13:46:49.362 | INFO | ------------------------------------------ | 20260505 13:46:49.362 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:46:49.362 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 13:46:49.362 | INFO | Total mem usage: 15.7 GiB | 20260505 13:46:49.362 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 13:46:49.362 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 13:46:49.362 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 13:46:49.362 | INFO | High Water Breakdown: | 20260505 13:46:49.362 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 13:46:49.362 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 13:46:49.362 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 13:46:49.362 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 13:46:49.362 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 13:46:49.362 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 13:46:49.362 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 13:46:49.362 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 13:46:49.362 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 13:46:49.362 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:46:49.362 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 13:46:49.362 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:46:49.362 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 13:46:49.362 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 13:46:49.362 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 13:46:49.362 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 13:46:49.362 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:46:49.362 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 13:46:49.362 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:46:49.362 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:46:49.362 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:46:49.362 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 13:46:49.362 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 13:46:49.362 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:46:49.362 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 13:46:49.362 | INFO | ------------------------------------------ | 20260505 13:46:49.362 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:46:49.362 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 13:46:49.362 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:46:49.362 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:46:49.362 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.70e+01, 1.38e+01 / 2.00e+00 | 20260505 13:46:49.362 | INFO | PerfReport: 0.29%, AggregateStats -- 9.27e-02 / 1.00e-01, 1.07e-02 / 4.10e-03 | 20260505 13:46:49.362 | INFO | PerfReport: 11.78%, Baseline -- 3.75e+00 / 4.45e+00, 5.86e-01 / 1.98e-01 | 20260505 13:46:49.362 | INFO | PerfReport: 3.30%, Histogram -- 1.05e+00 / 1.51e+00, 2.01e-01 / 8.93e-02 | 20260505 13:46:49.362 | INFO | PerfReport: 16.06%, DME -- 4.10e+01 / 4.63e+01, 4.70e+00 / 2.35e+00 | 20260505 13:46:49.362 | INFO | PerfReport: 56.30%, FrameLabeling -- 1.79e+01 / 1.98e+01, 1.37e+00 / 1.68e+00 | 20260505 13:46:49.362 | INFO | PerfReport: 7.46%, PulseAccumulating -- 2.38e+00 / 2.93e+00, 4.49e-01 / 1.72e-01 | 20260505 13:46:49.362 | INFO | PerfReport: 3.56%, Metrics -- 1.13e+00 / 1.45e+00, 3.16e-01 / 1.76e-01 | 20260505 13:46:49.362 | INFO | PerfReport: 1.25%, PulseSerializing -- 3.99e-01 / 4.51e-01, 7.15e-02 / 3.37e-02 | 20260505 13:46:49.362 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 13:46:49.362 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:46:49.362 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:46:49.362 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.57e+01 / 0.00e+00 | 20260505 13:46:49.362 | INFO | PerfReport: 22.56%, Upload -- 2.28e+01 / 0.00e+00, 8.06e+00 / 0.00e+00 | 20260505 13:46:49.362 | INFO | PerfReport: 22.06%, Download -- 2.23e+01 / 0.00e+00, 8.11e+00 / 0.00e+00 | 20260505 13:46:49.362 | INFO | PerfReport: 31.59%, Compute -- 3.19e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 13:46:49.362 | INFO | PerfReport: 2.20%, LockUpload -- 2.22e+00 / 0.00e+00, 7.93e+00 / 0.00e+00 | 20260505 13:46:49.362 | INFO | PerfReport: 1.50%, LockDownload -- 1.52e+00 / 0.00e+00, 3.05e+00 / 0.00e+00 | 20260505 13:46:49.362 | INFO | PerfReport: 20.08%, LockCompute -- 2.03e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 13:46:49.362 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.86e-02 / 0.00e+00 | 20260505 13:46:49.496 | INFO | Starting a round of Baz IO | 20260505 13:46:52.717 | INFO | Populating RTMetrics report | 20260505 13:46:52.717 | INFO | RTMetrics report populated | 20260505 13:46:52.717 | INFO | Opening tmp RTMetrics json file | 20260505 13:46:52.730 | INFO | Serializing RTMetrics to json | 20260505 13:46:52.731 | INFO | Serialization done, swapping tmp file now... | 20260505 13:46:52.733 | INFO | RTMetrics done | 20260505 13:46:52.827 | INFO | Finished a round of Baz IO: Wrote 2.289 GiB in 3.33 seconds | 20260505 13:46:52.827 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.99% / 0.00% / 77.01% | 20260505 13:46:52.827 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.65376% / 0% / 6.35783e-05% | 20260505 13:46:52.827 | INFO | Baz IO overall: | 20260505 13:46:52.827 | INFO | BytesWritten : 2191372939113 | 20260505 13:46:52.827 | INFO | NumEvents : 1042348518208 | 20260505 13:46:54.493 | INFO | Analyzing chunk frames = [3678208,3678720) | 20260505 13:46:54.493 | INFO | MemInfo: { | 20260505 13:46:54.493 | INFO | "gpu" : " 17 GB", | 20260505 13:46:54.493 | INFO | "huge" : "", | 20260505 13:46:54.493 | INFO | "huge_shared" : " 27 GB", | 20260505 13:46:54.493 | INFO | "malloc" : " 14 GB", | 20260505 13:46:54.493 | INFO | "pinned" : " 29 GB" | 20260505 13:46:54.493 | INFO | } | 20260505 13:46:54.493 | INFO | Perf report for the last 5.13254 seconds: | 20260505 13:46:54.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:46:54.493 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:54.493 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:54.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:54.493 | INFO | ANALYSIS : 64.3%, 2.9, 99.2ms, 0 | 20260505 13:46:54.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:54.493 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260505 13:46:54.493 | INFO | PRE_HQ : 61.2%, 1.8, 57.5ms, 0 | 20260505 13:46:54.493 | INFO | BAZWRITER : 64.2%, 4.0, 132.7ms, 0 | 20260505 13:46:54.493 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 13:46:54.494 | INFO | Uploaded 36GiB to GPU in 2.19968s (16.366GiB/s) | 20260505 13:46:54.494 | INFO | Downloaded 38GiB to GPU in 2.27478s (16.7049GiB/s) | 20260505 13:46:59.626 | INFO | Analyzing chunk frames = [3678720,3679232) | 20260505 13:46:59.626 | INFO | MemInfo: { | 20260505 13:46:59.626 | INFO | "gpu" : " 17 GB", | 20260505 13:46:59.626 | INFO | "huge" : "", | 20260505 13:46:59.626 | INFO | "huge_shared" : " 27 GB", | 20260505 13:46:59.626 | INFO | "malloc" : " 14 GB", | 20260505 13:46:59.626 | INFO | "pinned" : " 29 GB" | 20260505 13:46:59.626 | INFO | } | 20260505 13:46:59.626 | INFO | Perf report for the last 5.13313 seconds: | 20260505 13:46:59.626 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:46:59.626 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:59.626 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:59.626 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:59.626 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 13:46:59.626 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:59.626 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:59.626 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 13:46:59.626 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:59.626 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:46:59.627 | INFO | Uploaded 36GiB to GPU in 2.19026s (16.4364GiB/s) | 20260505 13:46:59.627 | INFO | Downloaded 34GiB to GPU in 2.0351s (16.7068GiB/s) | 20260505 13:47:04.774 | INFO | Analyzing chunk frames = [3679232,3679744) | 20260505 13:47:04.774 | INFO | MemInfo: { | 20260505 13:47:04.774 | INFO | "gpu" : " 17 GB", | 20260505 13:47:04.774 | INFO | "huge" : "", | 20260505 13:47:04.774 | INFO | "huge_shared" : " 27 GB", | 20260505 13:47:04.774 | INFO | "malloc" : " 14 GB", | 20260505 13:47:04.774 | INFO | "pinned" : " 29 GB" | 20260505 13:47:04.774 | INFO | } | 20260505 13:47:04.774 | INFO | Perf report for the last 5.14807 seconds: | 20260505 13:47:04.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:47:04.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:04.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:04.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:04.774 | INFO | ANALYSIS : 63.1%, 3.0, 98.3ms, 0 | 20260505 13:47:04.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:04.774 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:04.774 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 13:47:04.775 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:04.775 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:04.775 | INFO | Uploaded 36GiB to GPU in 2.18979s (16.4399GiB/s) | 20260505 13:47:04.775 | INFO | Downloaded 34GiB to GPU in 2.13167s (15.9499GiB/s) | 20260505 13:47:09.912 | INFO | Analyzing chunk frames = [3679744,3680256) | 20260505 13:47:09.912 | INFO | MemInfo: { | 20260505 13:47:09.912 | INFO | "gpu" : " 17 GB", | 20260505 13:47:09.912 | INFO | "huge" : "", | 20260505 13:47:09.912 | INFO | "huge_shared" : " 27 GB", | 20260505 13:47:09.912 | INFO | "malloc" : " 14 GB", | 20260505 13:47:09.912 | INFO | "pinned" : " 29 GB" | 20260505 13:47:09.912 | INFO | } | 20260505 13:47:09.912 | INFO | Perf report for the last 5.13736 seconds: | 20260505 13:47:09.912 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:47:09.912 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:09.912 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:09.912 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:09.912 | INFO | ANALYSIS : 62.9%, 3.0, 97.6ms, 0 | 20260505 13:47:09.912 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:09.912 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:09.912 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:47:09.912 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:09.912 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:09.912 | INFO | Uploaded 36GiB to GPU in 2.18194s (16.4991GiB/s) | 20260505 13:47:09.912 | INFO | Downloaded 34GiB to GPU in 2.13315s (15.9389GiB/s) | 20260505 13:47:15.049 | INFO | Analyzing chunk frames = [3680256,3680768) | 20260505 13:47:15.049 | INFO | MemInfo: { | 20260505 13:47:15.049 | INFO | "gpu" : " 17 GB", | 20260505 13:47:15.049 | INFO | "huge" : "", | 20260505 13:47:15.049 | INFO | "huge_shared" : " 27 GB", | 20260505 13:47:15.049 | INFO | "malloc" : " 14 GB", | 20260505 13:47:15.049 | INFO | "pinned" : " 29 GB" | 20260505 13:47:15.049 | INFO | } | 20260505 13:47:15.049 | INFO | Perf report for the last 5.1374 seconds: | 20260505 13:47:15.049 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:47:15.049 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:15.049 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:15.049 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:15.049 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 13:47:15.049 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:15.049 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:15.049 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:47:15.049 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:15.049 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:15.049 | INFO | Uploaded 36GiB to GPU in 2.16171s (16.6535GiB/s) | 20260505 13:47:15.050 | INFO | Downloaded 34GiB to GPU in 2.12409s (16.0069GiB/s) | 20260505 13:47:20.182 | INFO | Analyzing chunk frames = [3680768,3681280) | 20260505 13:47:20.182 | INFO | MemInfo: { | 20260505 13:47:20.182 | INFO | "gpu" : " 17 GB", | 20260505 13:47:20.182 | INFO | "huge" : "", | 20260505 13:47:20.182 | INFO | "huge_shared" : " 27 GB", | 20260505 13:47:20.182 | INFO | "malloc" : " 14 GB", | 20260505 13:47:20.182 | INFO | "pinned" : " 29 GB" | 20260505 13:47:20.182 | INFO | } | 20260505 13:47:20.182 | INFO | Perf report for the last 5.13309 seconds: | 20260505 13:47:20.182 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:47:20.182 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:20.182 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:20.182 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:20.182 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 13:47:20.182 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:20.182 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:20.182 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 13:47:20.182 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:20.182 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:20.182 | INFO | Uploaded 36GiB to GPU in 2.16273s (16.6457GiB/s) | 20260505 13:47:20.183 | INFO | Downloaded 34GiB to GPU in 2.11023s (16.112GiB/s) | 20260505 13:47:25.323 | INFO | Analyzing chunk frames = [3681280,3681792) | 20260505 13:47:25.323 | INFO | MemInfo: { | 20260505 13:47:25.323 | INFO | "gpu" : " 17 GB", | 20260505 13:47:25.323 | INFO | "huge" : "", | 20260505 13:47:25.323 | INFO | "huge_shared" : " 27 GB", | 20260505 13:47:25.323 | INFO | "malloc" : " 14 GB", | 20260505 13:47:25.323 | INFO | "pinned" : " 29 GB" | 20260505 13:47:25.323 | INFO | } | 20260505 13:47:25.323 | INFO | Perf report for the last 5.14104 seconds: | 20260505 13:47:25.323 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:47:25.323 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:25.323 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:25.323 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:25.323 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 13:47:25.323 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:25.323 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:25.323 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 13:47:25.323 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:25.323 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:25.324 | INFO | Uploaded 36GiB to GPU in 2.16758s (16.6084GiB/s) | 20260505 13:47:25.324 | INFO | Downloaded 34GiB to GPU in 2.14835s (15.8261GiB/s) | 20260505 13:47:30.472 | INFO | Analyzing chunk frames = [3681792,3682304) | 20260505 13:47:30.472 | INFO | MemInfo: { | 20260505 13:47:30.472 | INFO | "gpu" : " 17 GB", | 20260505 13:47:30.472 | INFO | "huge" : "", | 20260505 13:47:30.472 | INFO | "huge_shared" : " 27 GB", | 20260505 13:47:30.472 | INFO | "malloc" : " 14 GB", | 20260505 13:47:30.472 | INFO | "pinned" : " 29 GB" | 20260505 13:47:30.472 | INFO | } | 20260505 13:47:30.473 | INFO | Perf report for the last 5.14921 seconds: | 20260505 13:47:30.473 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:47:30.473 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:30.473 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:30.473 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:30.473 | INFO | ANALYSIS : 63.5%, 3.0, 99.4ms, 0 | 20260505 13:47:30.473 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:30.473 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:30.473 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 13:47:30.473 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:30.473 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:30.473 | INFO | Uploaded 36GiB to GPU in 2.19066s (16.4334GiB/s) | 20260505 13:47:30.473 | INFO | Downloaded 34GiB to GPU in 2.1104s (16.1107GiB/s) | 20260505 13:47:30.608 | INFO | Starting a round of Baz IO | 20260505 13:47:33.827 | INFO | Populating RTMetrics report | 20260505 13:47:33.828 | INFO | RTMetrics report populated | 20260505 13:47:33.828 | INFO | Opening tmp RTMetrics json file | 20260505 13:47:33.845 | INFO | Serializing RTMetrics to json | 20260505 13:47:33.860 | INFO | Serialization done, swapping tmp file now... | 20260505 13:47:33.870 | INFO | RTMetrics done | 20260505 13:47:33.926 | INFO | Finished a round of Baz IO: Wrote 2.287 GiB in 3.317 seconds | 20260505 13:47:33.926 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.99% / 0.00% / 77.01% | 20260505 13:47:33.926 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.651693% / 0% / 9.53674e-05% | 20260505 13:47:33.926 | INFO | Baz IO overall: | 20260505 13:47:33.926 | INFO | BytesWritten : 2193828701824 | 20260505 13:47:33.926 | INFO | NumEvents : 1043508782797 | 20260505 13:47:35.617 | INFO | Analyzing chunk frames = [3682304,3682816) | 20260505 13:47:35.617 | INFO | MemInfo: { | 20260505 13:47:35.617 | INFO | "gpu" : " 17 GB", | 20260505 13:47:35.617 | INFO | "huge" : "", | 20260505 13:47:35.617 | INFO | "huge_shared" : " 27 GB", | 20260505 13:47:35.618 | INFO | "malloc" : " 14 GB", | 20260505 13:47:35.618 | INFO | "pinned" : " 29 GB" | 20260505 13:47:35.618 | INFO | } | 20260505 13:47:35.618 | INFO | Perf report for the last 5.14506 seconds: | 20260505 13:47:35.618 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:47:35.618 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:35.618 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:35.618 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:35.618 | INFO | ANALYSIS : 64.1%, 3.0, 99.7ms, 0 | 20260505 13:47:35.618 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:35.618 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260505 13:47:35.618 | INFO | PRE_HQ : 60.9%, 1.8, 57.6ms, 0 | 20260505 13:47:35.618 | INFO | BAZWRITER : 63.8%, 4.0, 132.2ms, 0 | 20260505 13:47:35.618 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 13:47:35.618 | INFO | Uploaded 36GiB to GPU in 2.19959s (16.3667GiB/s) | 20260505 13:47:35.618 | INFO | Downloaded 38GiB to GPU in 2.2612s (16.8052GiB/s) | 20260505 13:47:40.743 | INFO | Analyzing chunk frames = [3682816,3683328) | 20260505 13:47:40.743 | INFO | MemInfo: { | 20260505 13:47:40.743 | INFO | "gpu" : " 17 GB", | 20260505 13:47:40.743 | INFO | "huge" : "", | 20260505 13:47:40.743 | INFO | "huge_shared" : " 27 GB", | 20260505 13:47:40.743 | INFO | "malloc" : " 14 GB", | 20260505 13:47:40.743 | INFO | "pinned" : " 29 GB" | 20260505 13:47:40.743 | INFO | } | 20260505 13:47:40.743 | INFO | Perf report for the last 5.12512 seconds: | 20260505 13:47:40.743 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:47:40.743 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:40.743 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:40.743 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:40.743 | INFO | ANALYSIS : 63.6%, 3.0, 99.0ms, 0 | 20260505 13:47:40.743 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:40.743 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:40.743 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:47:40.743 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:40.743 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:40.743 | INFO | Uploaded 36GiB to GPU in 2.19s (16.4384GiB/s) | 20260505 13:47:40.743 | INFO | Downloaded 34GiB to GPU in 2.02955s (16.7525GiB/s) | 20260505 13:47:45.879 | INFO | Analyzing chunk frames = [3683328,3683840) | 20260505 13:47:45.879 | INFO | MemInfo: { | 20260505 13:47:45.879 | INFO | "gpu" : " 17 GB", | 20260505 13:47:45.879 | INFO | "huge" : "", | 20260505 13:47:45.879 | INFO | "huge_shared" : " 27 GB", | 20260505 13:47:45.879 | INFO | "malloc" : " 14 GB", | 20260505 13:47:45.879 | INFO | "pinned" : " 29 GB" | 20260505 13:47:45.879 | INFO | } | 20260505 13:47:45.879 | INFO | Perf report for the last 5.13624 seconds: | 20260505 13:47:45.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:47:45.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:45.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:45.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:45.879 | INFO | ANALYSIS : 63.2%, 3.0, 98.3ms, 0 | 20260505 13:47:45.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:45.879 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:45.879 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:47:45.879 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:45.879 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:45.879 | INFO | Uploaded 36GiB to GPU in 2.21237s (16.2721GiB/s) | 20260505 13:47:45.879 | INFO | Downloaded 34GiB to GPU in 2.1186s (16.0483GiB/s) | 20260505 13:47:51.025 | INFO | Analyzing chunk frames = [3683840,3684352) | 20260505 13:47:51.025 | INFO | MemInfo: { | 20260505 13:47:51.025 | INFO | "gpu" : " 17 GB", | 20260505 13:47:51.025 | INFO | "huge" : "", | 20260505 13:47:51.025 | INFO | "huge_shared" : " 27 GB", | 20260505 13:47:51.025 | INFO | "malloc" : " 14 GB", | 20260505 13:47:51.025 | INFO | "pinned" : " 29 GB" | 20260505 13:47:51.025 | INFO | } | 20260505 13:47:51.025 | INFO | Perf report for the last 5.14596 seconds: | 20260505 13:47:51.025 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:47:51.025 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:51.025 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:51.025 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:51.025 | INFO | ANALYSIS : 62.9%, 3.0, 97.8ms, 0 | 20260505 13:47:51.025 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:51.025 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:51.025 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:47:51.025 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:51.025 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:51.025 | INFO | Uploaded 36GiB to GPU in 2.20005s (16.3633GiB/s) | 20260505 13:47:51.026 | INFO | Downloaded 34GiB to GPU in 2.13213s (15.9465GiB/s) | 20260505 13:47:56.168 | INFO | Analyzing chunk frames = [3684352,3684864) | 20260505 13:47:56.168 | INFO | MemInfo: { | 20260505 13:47:56.168 | INFO | "gpu" : " 17 GB", | 20260505 13:47:56.168 | INFO | "huge" : "", | 20260505 13:47:56.168 | INFO | "huge_shared" : " 27 GB", | 20260505 13:47:56.168 | INFO | "malloc" : " 14 GB", | 20260505 13:47:56.168 | INFO | "pinned" : " 29 GB" | 20260505 13:47:56.168 | INFO | } | 20260505 13:47:56.168 | INFO | Perf report for the last 5.14351 seconds: | 20260505 13:47:56.168 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:47:56.168 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:56.168 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:56.169 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:56.169 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260505 13:47:56.169 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:56.169 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:56.169 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:47:56.169 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:56.169 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:47:56.169 | INFO | Uploaded 36GiB to GPU in 2.1478s (16.7613GiB/s) | 20260505 13:47:56.169 | INFO | Downloaded 34GiB to GPU in 2.12738s (15.9821GiB/s) | 20260505 13:48:01.307 | INFO | Analyzing chunk frames = [3684864,3685376) | 20260505 13:48:01.307 | INFO | MemInfo: { | 20260505 13:48:01.307 | INFO | "gpu" : " 17 GB", | 20260505 13:48:01.307 | INFO | "huge" : "", | 20260505 13:48:01.307 | INFO | "huge_shared" : " 27 GB", | 20260505 13:48:01.307 | INFO | "malloc" : " 14 GB", | 20260505 13:48:01.307 | INFO | "pinned" : " 29 GB" | 20260505 13:48:01.307 | INFO | } | 20260505 13:48:01.307 | INFO | Perf report for the last 5.13852 seconds: | 20260505 13:48:01.307 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:48:01.307 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:01.307 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:01.307 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:01.307 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 13:48:01.307 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:01.307 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:01.307 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:48:01.307 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:01.307 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:01.307 | INFO | Uploaded 36GiB to GPU in 2.20641s (16.3161GiB/s) | 20260505 13:48:01.307 | INFO | Downloaded 34GiB to GPU in 2.11394s (16.0837GiB/s) | 20260505 13:48:06.440 | INFO | Analyzing chunk frames = [3685376,3685888) | 20260505 13:48:06.440 | INFO | MemInfo: { | 20260505 13:48:06.440 | INFO | "gpu" : " 17 GB", | 20260505 13:48:06.440 | INFO | "huge" : "", | 20260505 13:48:06.440 | INFO | "huge_shared" : " 27 GB", | 20260505 13:48:06.440 | INFO | "malloc" : " 14 GB", | 20260505 13:48:06.440 | INFO | "pinned" : " 29 GB" | 20260505 13:48:06.440 | INFO | } | 20260505 13:48:06.440 | INFO | Perf report for the last 5.13335 seconds: | 20260505 13:48:06.440 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:48:06.440 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:06.440 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:06.440 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:06.440 | INFO | ANALYSIS : 63.5%, 3.0, 99.0ms, 0 | 20260505 13:48:06.440 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:06.440 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:06.440 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 13:48:06.441 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:06.441 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:06.441 | INFO | Uploaded 36GiB to GPU in 2.15862s (16.6774GiB/s) | 20260505 13:48:06.441 | INFO | Downloaded 34GiB to GPU in 2.18271s (15.5769GiB/s) | 20260505 13:48:11.582 | INFO | Analyzing chunk frames = [3685888,3686400) | 20260505 13:48:11.582 | INFO | MemInfo: { | 20260505 13:48:11.582 | INFO | "gpu" : " 17 GB", | 20260505 13:48:11.582 | INFO | "huge" : "", | 20260505 13:48:11.582 | INFO | "huge_shared" : " 27 GB", | 20260505 13:48:11.582 | INFO | "malloc" : " 14 GB", | 20260505 13:48:11.582 | INFO | "pinned" : " 29 GB" | 20260505 13:48:11.582 | INFO | } | 20260505 13:48:11.583 | INFO | Perf report for the last 5.14214 seconds: | 20260505 13:48:11.583 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:48:11.583 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:11.583 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:11.583 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:11.583 | INFO | ANALYSIS : 63.5%, 3.0, 99.3ms, 0 | 20260505 13:48:11.583 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:11.583 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:11.583 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 13:48:11.583 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:11.583 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:11.583 | INFO | Uploaded 36GiB to GPU in 2.17387s (16.5603GiB/s) | 20260505 13:48:11.583 | INFO | Downloaded 34GiB to GPU in 2.1053s (16.1497GiB/s) | 20260505 13:48:11.583 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 13:48:11.583 | INFO | Total mem usage: 15.3 GiB | 20260505 13:48:11.583 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 13:48:11.583 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -7.05%) | 20260505 13:48:11.583 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 13:48:11.583 | INFO | High Water Breakdown: | 20260505 13:48:11.583 | INFO | 61.1% : 8.75 GiB ( 91.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 13:48:11.583 | INFO | 38.8% : 5.55 GiB ( 83.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 13:48:11.583 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 13:48:11.584 | INFO | ------------------------------------------ | 20260505 13:48:11.584 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:48:11.584 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 13:48:11.584 | INFO | Total mem usage: 26.5 GiB | 20260505 13:48:11.584 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 13:48:11.584 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 13:48:11.584 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 13:48:11.584 | INFO | High Water Breakdown: | 20260505 13:48:11.584 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:48:11.584 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:48:11.584 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:48:11.584 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:48:11.584 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:48:11.584 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:48:11.584 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 13:48:11.584 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 13:48:11.584 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:48:11.584 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 13:48:11.584 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 13:48:11.584 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 13:48:11.584 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 13:48:11.584 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 13:48:11.584 | INFO | ------------------------------------------ | 20260505 13:48:11.584 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:48:11.584 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 13:48:11.584 | INFO | Total mem usage: 25.6 GiB | 20260505 13:48:11.584 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 13:48:11.584 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 13:48:11.584 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 13:48:11.584 | INFO | High Water Breakdown: | 20260505 13:48:11.584 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 13:48:11.584 | INFO | ------------------------------------------ | 20260505 13:48:11.584 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:48:11.584 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 13:48:11.584 | INFO | Total mem usage: 15.7 GiB | 20260505 13:48:11.584 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 13:48:11.584 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 13:48:11.584 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 13:48:11.584 | INFO | High Water Breakdown: | 20260505 13:48:11.584 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 13:48:11.584 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 13:48:11.584 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 13:48:11.584 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 13:48:11.584 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 13:48:11.584 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 13:48:11.584 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 13:48:11.584 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 13:48:11.584 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 13:48:11.584 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:48:11.584 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 13:48:11.584 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:48:11.584 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 13:48:11.584 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 13:48:11.584 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 13:48:11.584 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 13:48:11.584 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:48:11.584 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 13:48:11.584 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:48:11.584 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:48:11.584 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:48:11.584 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 13:48:11.584 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 13:48:11.584 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:48:11.584 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 13:48:11.584 | INFO | ------------------------------------------ | 20260505 13:48:11.584 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:48:11.584 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 13:48:11.584 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:48:11.584 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:48:11.584 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.64e+01, 1.37e+01 / 1.51e+00 | 20260505 13:48:11.584 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 1.01e-01, 1.05e-02 / 4.19e-03 | 20260505 13:48:11.584 | INFO | PerfReport: 11.78%, Baseline -- 3.75e+00 / 4.48e+00, 5.88e-01 / 2.20e-01 | 20260505 13:48:11.584 | INFO | PerfReport: 3.30%, Histogram -- 1.05e+00 / 1.50e+00, 1.98e-01 / 1.04e-01 | 20260505 13:48:11.584 | INFO | PerfReport: 15.94%, DME -- 4.06e+01 / 4.56e+01, 4.53e+00 / 1.73e+00 | 20260505 13:48:11.584 | INFO | PerfReport: 56.41%, FrameLabeling -- 1.80e+01 / 1.99e+01, 1.41e+00 / 1.78e+00 | 20260505 13:48:11.584 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.37e+00 / 2.92e+00, 4.48e-01 / 1.82e-01 | 20260505 13:48:11.584 | INFO | PerfReport: 3.57%, Metrics -- 1.14e+00 / 1.44e+00, 3.14e-01 / 1.38e-01 | 20260505 13:48:11.584 | INFO | PerfReport: 1.25%, PulseSerializing -- 3.98e-01 / 4.48e-01, 7.06e-02 / 2.91e-02 | 20260505 13:48:11.584 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 13:48:11.584 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:48:11.584 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:48:11.584 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260505 13:48:11.584 | INFO | PerfReport: 22.58%, Upload -- 2.27e+01 / 0.00e+00, 8.05e+00 / 0.00e+00 | 20260505 13:48:11.584 | INFO | PerfReport: 22.07%, Download -- 2.22e+01 / 0.00e+00, 8.03e+00 / 0.00e+00 | 20260505 13:48:11.584 | INFO | PerfReport: 31.65%, Compute -- 3.19e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 13:48:11.584 | INFO | PerfReport: 2.13%, LockUpload -- 2.15e+00 / 0.00e+00, 7.74e+00 / 0.00e+00 | 20260505 13:48:11.584 | INFO | PerfReport: 1.44%, LockDownload -- 1.45e+00 / 0.00e+00, 2.97e+00 / 0.00e+00 | 20260505 13:48:11.584 | INFO | PerfReport: 20.11%, LockCompute -- 2.03e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260505 13:48:11.584 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 2.09e-02 / 0.00e+00 | 20260505 13:48:11.721 | INFO | Starting a round of Baz IO | 20260505 13:48:14.944 | INFO | Populating RTMetrics report | 20260505 13:48:14.944 | INFO | RTMetrics report populated | 20260505 13:48:14.944 | INFO | Opening tmp RTMetrics json file | 20260505 13:48:14.955 | INFO | Serializing RTMetrics to json | 20260505 13:48:14.957 | INFO | Serialization done, swapping tmp file now... | 20260505 13:48:14.958 | INFO | RTMetrics done | 20260505 13:48:15.049 | INFO | Finished a round of Baz IO: Wrote 2.284 GiB in 3.327 seconds | 20260505 13:48:15.049 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.98% / 0.00% / 77.01% | 20260505 13:48:15.049 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.647748% / 1.19209e-05% / 5.96046e-05% | 20260505 13:48:15.049 | INFO | Baz IO overall: | 20260505 13:48:15.049 | INFO | BytesWritten : 2196281311934 | 20260505 13:48:15.049 | INFO | NumEvents : 1044667500839 | 20260505 13:48:16.726 | INFO | Analyzing chunk frames = [3686400,3686912) | 20260505 13:48:16.726 | INFO | MemInfo: { | 20260505 13:48:16.726 | INFO | "gpu" : " 17 GB", | 20260505 13:48:16.726 | INFO | "huge" : "", | 20260505 13:48:16.726 | INFO | "huge_shared" : " 27 GB", | 20260505 13:48:16.726 | INFO | "malloc" : " 14 GB", | 20260505 13:48:16.726 | INFO | "pinned" : " 29 GB" | 20260505 13:48:16.726 | INFO | } | 20260505 13:48:16.726 | INFO | Perf report for the last 5.14346 seconds: | 20260505 13:48:16.726 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:48:16.726 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:16.726 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:16.726 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:16.726 | INFO | ANALYSIS : 64.2%, 3.0, 99.6ms, 0 | 20260505 13:48:16.726 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:16.726 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260505 13:48:16.726 | INFO | PRE_HQ : 61.1%, 1.8, 58.0ms, 0 | 20260505 13:48:16.726 | INFO | BAZWRITER : 64.0%, 4.0, 132.1ms, 0 | 20260505 13:48:16.726 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 13:48:16.726 | INFO | Uploaded 36GiB to GPU in 2.18147s (16.5026GiB/s) | 20260505 13:48:16.727 | INFO | Downloaded 38GiB to GPU in 2.2689s (16.7482GiB/s) | 20260505 13:48:21.860 | INFO | Analyzing chunk frames = [3686912,3687424) | 20260505 13:48:21.860 | INFO | MemInfo: { | 20260505 13:48:21.860 | INFO | "gpu" : " 17 GB", | 20260505 13:48:21.860 | INFO | "huge" : "", | 20260505 13:48:21.860 | INFO | "huge_shared" : " 27 GB", | 20260505 13:48:21.860 | INFO | "malloc" : " 14 GB", | 20260505 13:48:21.860 | INFO | "pinned" : " 29 GB" | 20260505 13:48:21.860 | INFO | } | 20260505 13:48:21.860 | INFO | Perf report for the last 5.13405 seconds: | 20260505 13:48:21.860 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:48:21.860 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:21.860 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:21.860 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:21.860 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 13:48:21.860 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:21.860 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:21.860 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 13:48:21.860 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:21.860 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:21.860 | INFO | Uploaded 36GiB to GPU in 2.20347s (16.3378GiB/s) | 20260505 13:48:21.860 | INFO | Downloaded 34GiB to GPU in 2.04026s (16.6646GiB/s) | 20260505 13:48:26.993 | INFO | Analyzing chunk frames = [3687424,3687936) | 20260505 13:48:26.993 | INFO | MemInfo: { | 20260505 13:48:26.993 | INFO | "gpu" : " 17 GB", | 20260505 13:48:26.993 | INFO | "huge" : "", | 20260505 13:48:26.993 | INFO | "huge_shared" : " 27 GB", | 20260505 13:48:26.993 | INFO | "malloc" : " 14 GB", | 20260505 13:48:26.993 | INFO | "pinned" : " 29 GB" | 20260505 13:48:26.993 | INFO | } | 20260505 13:48:26.993 | INFO | Perf report for the last 5.13309 seconds: | 20260505 13:48:26.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:48:26.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:26.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:26.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:26.993 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 13:48:26.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:26.993 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:26.993 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:48:26.993 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:26.993 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:26.994 | INFO | Uploaded 36GiB to GPU in 2.18708s (16.4603GiB/s) | 20260505 13:48:26.994 | INFO | Downloaded 34GiB to GPU in 2.11737s (16.0576GiB/s) | 20260505 13:48:32.132 | INFO | Analyzing chunk frames = [3687936,3688448) | 20260505 13:48:32.132 | INFO | MemInfo: { | 20260505 13:48:32.132 | INFO | "gpu" : " 17 GB", | 20260505 13:48:32.132 | INFO | "huge" : "", | 20260505 13:48:32.132 | INFO | "huge_shared" : " 27 GB", | 20260505 13:48:32.133 | INFO | "malloc" : " 14 GB", | 20260505 13:48:32.133 | INFO | "pinned" : " 29 GB" | 20260505 13:48:32.133 | INFO | } | 20260505 13:48:32.133 | INFO | Perf report for the last 5.13953 seconds: | 20260505 13:48:32.133 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:48:32.133 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:32.133 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:32.133 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:32.133 | INFO | ANALYSIS : 62.9%, 3.0, 97.7ms, 0 | 20260505 13:48:32.133 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:32.133 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:32.133 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 13:48:32.133 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:32.133 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:32.133 | INFO | Uploaded 36GiB to GPU in 2.20001s (16.3636GiB/s) | 20260505 13:48:32.133 | INFO | Downloaded 34GiB to GPU in 2.13223s (15.9457GiB/s) | 20260505 13:48:37.282 | INFO | Analyzing chunk frames = [3688448,3688960) | 20260505 13:48:37.282 | INFO | MemInfo: { | 20260505 13:48:37.282 | INFO | "gpu" : " 17 GB", | 20260505 13:48:37.282 | INFO | "huge" : "", | 20260505 13:48:37.282 | INFO | "huge_shared" : " 27 GB", | 20260505 13:48:37.283 | INFO | "malloc" : " 14 GB", | 20260505 13:48:37.283 | INFO | "pinned" : " 29 GB" | 20260505 13:48:37.283 | INFO | } | 20260505 13:48:37.283 | INFO | Perf report for the last 5.15001 seconds: | 20260505 13:48:37.283 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:48:37.283 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:37.283 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:37.283 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:37.283 | INFO | ANALYSIS : 62.7%, 3.0, 98.1ms, 0 | 20260505 13:48:37.283 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:37.283 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:37.283 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:48:37.283 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:37.283 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:37.283 | INFO | Uploaded 36GiB to GPU in 2.15349s (16.717GiB/s) | 20260505 13:48:37.283 | INFO | Downloaded 34GiB to GPU in 2.12398s (16.0077GiB/s) | 20260505 13:48:42.424 | INFO | Analyzing chunk frames = [3688960,3689472) | 20260505 13:48:42.424 | INFO | MemInfo: { | 20260505 13:48:42.424 | INFO | "gpu" : " 17 GB", | 20260505 13:48:42.424 | INFO | "huge" : "", | 20260505 13:48:42.424 | INFO | "huge_shared" : " 27 GB", | 20260505 13:48:42.424 | INFO | "malloc" : " 14 GB", | 20260505 13:48:42.424 | INFO | "pinned" : " 29 GB" | 20260505 13:48:42.424 | INFO | } | 20260505 13:48:42.425 | INFO | Perf report for the last 5.14175 seconds: | 20260505 13:48:42.425 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:48:42.425 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:42.425 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:42.425 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:42.425 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 13:48:42.425 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:42.425 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:42.425 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 13:48:42.425 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:42.425 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:42.425 | INFO | Uploaded 36GiB to GPU in 2.16986s (16.591GiB/s) | 20260505 13:48:42.425 | INFO | Downloaded 34GiB to GPU in 2.0361s (16.6986GiB/s) | 20260505 13:48:47.559 | INFO | Analyzing chunk frames = [3689472,3689984) | 20260505 13:48:47.559 | INFO | MemInfo: { | 20260505 13:48:47.559 | INFO | "gpu" : " 17 GB", | 20260505 13:48:47.559 | INFO | "huge" : "", | 20260505 13:48:47.559 | INFO | "huge_shared" : " 27 GB", | 20260505 13:48:47.559 | INFO | "malloc" : " 14 GB", | 20260505 13:48:47.559 | INFO | "pinned" : " 29 GB" | 20260505 13:48:47.559 | INFO | } | 20260505 13:48:47.560 | INFO | Perf report for the last 5.135 seconds: | 20260505 13:48:47.560 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:48:47.560 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:47.560 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:47.560 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:47.560 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 13:48:47.560 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:47.560 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:47.560 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 13:48:47.560 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:47.560 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:47.560 | INFO | Uploaded 36GiB to GPU in 2.11944s (16.9856GiB/s) | 20260505 13:48:47.560 | INFO | Downloaded 34GiB to GPU in 2.17344s (15.6434GiB/s) | 20260505 13:48:52.688 | INFO | Analyzing chunk frames = [3689984,3690496) | 20260505 13:48:52.688 | INFO | MemInfo: { | 20260505 13:48:52.688 | INFO | "gpu" : " 17 GB", | 20260505 13:48:52.688 | INFO | "huge" : "", | 20260505 13:48:52.688 | INFO | "huge_shared" : " 27 GB", | 20260505 13:48:52.688 | INFO | "malloc" : " 14 GB", | 20260505 13:48:52.688 | INFO | "pinned" : " 29 GB" | 20260505 13:48:52.688 | INFO | } | 20260505 13:48:52.689 | INFO | Perf report for the last 5.12901 seconds: | 20260505 13:48:52.689 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:48:52.689 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:52.689 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:52.689 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:52.689 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 13:48:52.689 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:52.689 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:52.689 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:48:52.689 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:52.689 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:52.689 | INFO | Uploaded 36GiB to GPU in 2.1876s (16.4564GiB/s) | 20260505 13:48:52.689 | INFO | Downloaded 34GiB to GPU in 2.10358s (16.1629GiB/s) | 20260505 13:48:52.823 | INFO | Starting a round of Baz IO | 20260505 13:48:56.066 | INFO | Populating RTMetrics report | 20260505 13:48:56.066 | INFO | RTMetrics report populated | 20260505 13:48:56.066 | INFO | Opening tmp RTMetrics json file | 20260505 13:48:56.083 | INFO | Serializing RTMetrics to json | 20260505 13:48:56.085 | INFO | Serialization done, swapping tmp file now... | 20260505 13:48:56.086 | INFO | RTMetrics done | 20260505 13:48:56.169 | INFO | Finished a round of Baz IO: Wrote 2.288 GiB in 3.346 seconds | 20260505 13:48:56.170 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.98% / 0.00% / 77.02% | 20260505 13:48:56.170 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.649385% / 3.97364e-06% / 6.75519e-05% | 20260505 13:48:56.170 | INFO | Baz IO overall: | 20260505 13:48:56.170 | INFO | BytesWritten : 2198737596470 | 20260505 13:48:56.170 | INFO | NumEvents : 1045829295616 | 20260505 13:48:57.834 | INFO | Analyzing chunk frames = [3690496,3691008) | 20260505 13:48:57.835 | INFO | MemInfo: { | 20260505 13:48:57.835 | INFO | "gpu" : " 17 GB", | 20260505 13:48:57.835 | INFO | "huge" : "", | 20260505 13:48:57.835 | INFO | "huge_shared" : " 27 GB", | 20260505 13:48:57.835 | INFO | "malloc" : " 14 GB", | 20260505 13:48:57.835 | INFO | "pinned" : " 29 GB" | 20260505 13:48:57.835 | INFO | } | 20260505 13:48:57.835 | INFO | Perf report for the last 5.14616 seconds: | 20260505 13:48:57.835 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:48:57.835 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:57.835 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:57.835 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:57.835 | INFO | ANALYSIS : 64.5%, 3.0, 100.5ms, 0 | 20260505 13:48:57.835 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:48:57.835 | INFO | RT_METRICS : 45.3%, 1.0, 23.8ms, 0 | 20260505 13:48:57.835 | INFO | PRE_HQ : 60.7%, 1.8, 57.7ms, 0 | 20260505 13:48:57.835 | INFO | BAZWRITER : 64.4%, 4.0, 131.1ms, 0 | 20260505 13:48:57.835 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 13:48:57.835 | INFO | Uploaded 36GiB to GPU in 2.17812s (16.528GiB/s) | 20260505 13:48:57.835 | INFO | Downloaded 38GiB to GPU in 2.27858s (16.677GiB/s) | 20260505 13:49:02.978 | INFO | Analyzing chunk frames = [3691008,3691520) | 20260505 13:49:02.978 | INFO | MemInfo: { | 20260505 13:49:02.978 | INFO | "gpu" : " 17 GB", | 20260505 13:49:02.978 | INFO | "huge" : "", | 20260505 13:49:02.978 | INFO | "huge_shared" : " 27 GB", | 20260505 13:49:02.978 | INFO | "malloc" : " 14 GB", | 20260505 13:49:02.978 | INFO | "pinned" : " 29 GB" | 20260505 13:49:02.978 | INFO | } | 20260505 13:49:02.978 | INFO | Perf report for the last 5.14326 seconds: | 20260505 13:49:02.978 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:49:02.978 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:02.978 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:02.978 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:02.978 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 13:49:02.978 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:02.978 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:02.978 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:49:02.978 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:02.978 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:02.978 | INFO | Uploaded 36GiB to GPU in 2.1948s (16.4024GiB/s) | 20260505 13:49:02.978 | INFO | Downloaded 34GiB to GPU in 2.03053s (16.7444GiB/s) | 20260505 13:49:08.121 | INFO | Analyzing chunk frames = [3691520,3692032) | 20260505 13:49:08.121 | INFO | MemInfo: { | 20260505 13:49:08.121 | INFO | "gpu" : " 17 GB", | 20260505 13:49:08.121 | INFO | "huge" : "", | 20260505 13:49:08.121 | INFO | "huge_shared" : " 27 GB", | 20260505 13:49:08.121 | INFO | "malloc" : " 14 GB", | 20260505 13:49:08.122 | INFO | "pinned" : " 29 GB" | 20260505 13:49:08.122 | INFO | } | 20260505 13:49:08.122 | INFO | Perf report for the last 5.14367 seconds: | 20260505 13:49:08.122 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:49:08.122 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:08.122 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:08.122 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:08.122 | INFO | ANALYSIS : 63.1%, 3.0, 98.2ms, 0 | 20260505 13:49:08.122 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:08.122 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:08.122 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:49:08.122 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:08.122 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:08.122 | INFO | Uploaded 36GiB to GPU in 2.18053s (16.5098GiB/s) | 20260505 13:49:08.122 | INFO | Downloaded 34GiB to GPU in 2.05437s (16.5501GiB/s) | 20260505 13:49:13.250 | INFO | Analyzing chunk frames = [3692032,3692544) | 20260505 13:49:13.250 | INFO | MemInfo: { | 20260505 13:49:13.250 | INFO | "gpu" : " 17 GB", | 20260505 13:49:13.250 | INFO | "huge" : "", | 20260505 13:49:13.250 | INFO | "huge_shared" : " 27 GB", | 20260505 13:49:13.250 | INFO | "malloc" : " 14 GB", | 20260505 13:49:13.250 | INFO | "pinned" : " 29 GB" | 20260505 13:49:13.250 | INFO | } | 20260505 13:49:13.250 | INFO | Perf report for the last 5.12852 seconds: | 20260505 13:49:13.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:49:13.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:13.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:13.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:13.250 | INFO | ANALYSIS : 62.8%, 3.0, 97.3ms, 0 | 20260505 13:49:13.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:13.250 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:13.250 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:49:13.250 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:13.250 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:13.250 | INFO | Uploaded 36GiB to GPU in 2.16967s (16.5924GiB/s) | 20260505 13:49:13.250 | INFO | Downloaded 34GiB to GPU in 2.12896s (15.9702GiB/s) | 20260505 13:49:18.386 | INFO | Analyzing chunk frames = [3692544,3693056) | 20260505 13:49:18.386 | INFO | MemInfo: { | 20260505 13:49:18.386 | INFO | "gpu" : " 17 GB", | 20260505 13:49:18.386 | INFO | "huge" : "", | 20260505 13:49:18.386 | INFO | "huge_shared" : " 27 GB", | 20260505 13:49:18.386 | INFO | "malloc" : " 14 GB", | 20260505 13:49:18.386 | INFO | "pinned" : " 29 GB" | 20260505 13:49:18.386 | INFO | } | 20260505 13:49:18.386 | INFO | Perf report for the last 5.13557 seconds: | 20260505 13:49:18.386 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:49:18.386 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:18.386 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:18.386 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:18.386 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 13:49:18.386 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:18.386 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:18.386 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 13:49:18.386 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:18.386 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:18.386 | INFO | Uploaded 36GiB to GPU in 2.18457s (16.4792GiB/s) | 20260505 13:49:18.386 | INFO | Downloaded 34GiB to GPU in 2.12824s (15.9757GiB/s) | 20260505 13:49:23.538 | INFO | Analyzing chunk frames = [3693056,3693568) | 20260505 13:49:23.538 | INFO | MemInfo: { | 20260505 13:49:23.538 | INFO | "gpu" : " 17 GB", | 20260505 13:49:23.538 | INFO | "huge" : "", | 20260505 13:49:23.538 | INFO | "huge_shared" : " 27 GB", | 20260505 13:49:23.538 | INFO | "malloc" : " 14 GB", | 20260505 13:49:23.538 | INFO | "pinned" : " 29 GB" | 20260505 13:49:23.538 | INFO | } | 20260505 13:49:23.538 | INFO | Perf report for the last 5.15253 seconds: | 20260505 13:49:23.538 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:49:23.538 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:23.538 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:23.538 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:23.538 | INFO | ANALYSIS : 62.9%, 3.0, 98.5ms, 0 | 20260505 13:49:23.538 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:23.538 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:23.538 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 13:49:23.538 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:23.538 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:23.539 | INFO | Uploaded 36GiB to GPU in 2.17205s (16.5742GiB/s) | 20260505 13:49:23.539 | INFO | Downloaded 34GiB to GPU in 2.09367s (16.2394GiB/s) | 20260505 13:49:28.682 | INFO | Analyzing chunk frames = [3693568,3694080) | 20260505 13:49:28.682 | INFO | MemInfo: { | 20260505 13:49:28.682 | INFO | "gpu" : " 17 GB", | 20260505 13:49:28.682 | INFO | "huge" : "", | 20260505 13:49:28.682 | INFO | "huge_shared" : " 27 GB", | 20260505 13:49:28.682 | INFO | "malloc" : " 14 GB", | 20260505 13:49:28.683 | INFO | "pinned" : " 29 GB" | 20260505 13:49:28.683 | INFO | } | 20260505 13:49:28.683 | INFO | Perf report for the last 5.14426 seconds: | 20260505 13:49:28.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:49:28.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:28.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:28.683 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:28.683 | INFO | ANALYSIS : 64.2%, 3.0, 100.3ms, 0 | 20260505 13:49:28.683 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:28.683 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:28.683 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 13:49:28.683 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:28.683 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:28.683 | INFO | Uploaded 36GiB to GPU in 2.19239s (16.4205GiB/s) | 20260505 13:49:28.683 | INFO | Downloaded 34GiB to GPU in 2.19962s (15.4572GiB/s) | 20260505 13:49:33.810 | INFO | Analyzing chunk frames = [3694080,3694592) | 20260505 13:49:33.810 | INFO | MemInfo: { | 20260505 13:49:33.810 | INFO | "gpu" : " 17 GB", | 20260505 13:49:33.810 | INFO | "huge" : "", | 20260505 13:49:33.810 | INFO | "huge_shared" : " 27 GB", | 20260505 13:49:33.810 | INFO | "malloc" : " 14 GB", | 20260505 13:49:33.810 | INFO | "pinned" : " 29 GB" | 20260505 13:49:33.810 | INFO | } | 20260505 13:49:33.810 | INFO | Perf report for the last 5.12774 seconds: | 20260505 13:49:33.810 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:49:33.810 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:33.810 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:33.810 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:33.810 | INFO | ANALYSIS : 63.9%, 3.0, 99.5ms, 0 | 20260505 13:49:33.810 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:33.810 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:33.810 | INFO | PRE_HQ : 28.5%, 1.0, 14.7ms, 0 | 20260505 13:49:33.810 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:33.810 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:33.811 | INFO | Uploaded 36GiB to GPU in 2.23362s (16.1174GiB/s) | 20260505 13:49:33.811 | INFO | Downloaded 34GiB to GPU in 2.09835s (16.2032GiB/s) | 20260505 13:49:33.811 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 13:49:33.811 | INFO | Total mem usage: 15.3 GiB | 20260505 13:49:33.811 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 13:49:33.811 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.65%) | 20260505 13:49:33.811 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 13:49:33.811 | INFO | High Water Breakdown: | 20260505 13:49:33.811 | INFO | 61% : 8.68 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 13:49:33.811 | INFO | 38.9% : 5.54 GiB ( 83.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 13:49:33.811 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 13:49:33.811 | INFO | ------------------------------------------ | 20260505 13:49:33.811 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:49:33.811 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 13:49:33.811 | INFO | Total mem usage: 26.5 GiB | 20260505 13:49:33.811 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 13:49:33.811 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 13:49:33.811 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 13:49:33.811 | INFO | High Water Breakdown: | 20260505 13:49:33.811 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:49:33.811 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:49:33.811 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:49:33.811 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:49:33.811 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:49:33.811 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:49:33.811 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 13:49:33.811 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 13:49:33.811 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:49:33.811 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 13:49:33.811 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 13:49:33.811 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 13:49:33.811 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 13:49:33.811 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 13:49:33.811 | INFO | ------------------------------------------ | 20260505 13:49:33.811 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:49:33.811 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 13:49:33.811 | INFO | Total mem usage: 25.6 GiB | 20260505 13:49:33.811 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 13:49:33.811 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 13:49:33.811 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 13:49:33.811 | INFO | High Water Breakdown: | 20260505 13:49:33.811 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 13:49:33.811 | INFO | ------------------------------------------ | 20260505 13:49:33.811 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:49:33.812 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 13:49:33.812 | INFO | Total mem usage: 15.7 GiB | 20260505 13:49:33.812 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 13:49:33.812 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 13:49:33.812 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 13:49:33.812 | INFO | High Water Breakdown: | 20260505 13:49:33.812 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 13:49:33.812 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 13:49:33.812 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 13:49:33.812 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 13:49:33.812 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 13:49:33.812 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 13:49:33.812 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 13:49:33.812 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 13:49:33.812 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 13:49:33.812 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:49:33.812 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 13:49:33.812 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:49:33.812 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 13:49:33.812 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 13:49:33.812 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 13:49:33.812 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 13:49:33.812 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:49:33.812 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 13:49:33.812 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:49:33.812 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:49:33.812 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:49:33.812 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 13:49:33.812 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 13:49:33.812 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:49:33.812 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 13:49:33.812 | INFO | ------------------------------------------ | 20260505 13:49:33.812 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:49:33.812 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 13:49:33.812 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:49:33.812 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:49:33.812 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.69e+01, 1.37e+01 / 2.14e+00 | 20260505 13:49:33.812 | INFO | PerfReport: 0.29%, AggregateStats -- 9.31e-02 / 1.02e-01, 1.12e-02 / 4.17e-03 | 20260505 13:49:33.812 | INFO | PerfReport: 11.76%, Baseline -- 3.75e+00 / 4.49e+00, 5.87e-01 / 1.40e-01 | 20260505 13:49:33.812 | INFO | PerfReport: 3.30%, Histogram -- 1.05e+00 / 1.51e+00, 1.98e-01 / 9.26e-02 | 20260505 13:49:33.812 | INFO | PerfReport: 16.03%, DME -- 4.09e+01 / 4.62e+01, 4.77e+00 / 2.56e+00 | 20260505 13:49:33.812 | INFO | PerfReport: 56.40%, FrameLabeling -- 1.80e+01 / 1.97e+01, 1.45e+00 / 1.77e+00 | 20260505 13:49:33.812 | INFO | PerfReport: 7.42%, PulseAccumulating -- 2.37e+00 / 2.93e+00, 4.45e-01 / 2.00e-01 | 20260505 13:49:33.812 | INFO | PerfReport: 3.55%, Metrics -- 1.13e+00 / 1.48e+00, 3.21e-01 / 1.89e-01 | 20260505 13:49:33.812 | INFO | PerfReport: 1.25%, PulseSerializing -- 3.98e-01 / 4.50e-01, 7.10e-02 / 3.11e-02 | 20260505 13:49:33.812 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 13:49:33.812 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:49:33.812 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:49:33.812 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.56e+01 / 0.00e+00 | 20260505 13:49:33.812 | INFO | PerfReport: 22.53%, Upload -- 2.27e+01 / 0.00e+00, 8.01e+00 / 0.00e+00 | 20260505 13:49:33.812 | INFO | PerfReport: 21.95%, Download -- 2.21e+01 / 0.00e+00, 8.02e+00 / 0.00e+00 | 20260505 13:49:33.812 | INFO | PerfReport: 31.63%, Compute -- 3.19e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 13:49:33.812 | INFO | PerfReport: 2.23%, LockUpload -- 2.25e+00 / 0.00e+00, 7.88e+00 / 0.00e+00 | 20260505 13:49:33.812 | INFO | PerfReport: 1.43%, LockDownload -- 1.44e+00 / 0.00e+00, 2.98e+00 / 0.00e+00 | 20260505 13:49:33.812 | INFO | PerfReport: 20.22%, LockCompute -- 2.04e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 13:49:33.812 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 2.22e-02 / 0.00e+00 | 20260505 13:49:33.944 | INFO | Starting a round of Baz IO | 20260505 13:49:37.182 | INFO | Populating RTMetrics report | 20260505 13:49:37.182 | INFO | RTMetrics report populated | 20260505 13:49:37.182 | INFO | Opening tmp RTMetrics json file | 20260505 13:49:37.201 | INFO | Serializing RTMetrics to json | 20260505 13:49:37.224 | INFO | Serialization done, swapping tmp file now... | 20260505 13:49:37.248 | INFO | RTMetrics done | 20260505 13:49:37.287 | INFO | Finished a round of Baz IO: Wrote 2.284 GiB in 3.343 seconds | 20260505 13:49:37.287 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.97% / 0.00% / 77.02% | 20260505 13:49:37.287 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.647803% / 7.94729e-06% / 5.5631e-05% | 20260505 13:49:37.287 | INFO | Baz IO overall: | 20260505 13:49:37.287 | INFO | BytesWritten : 2201190384255 | 20260505 13:49:37.287 | INFO | NumEvents : 1046988500000 | 20260505 13:49:38.944 | INFO | Analyzing chunk frames = [3694592,3695104) | 20260505 13:49:38.944 | INFO | MemInfo: { | 20260505 13:49:38.944 | INFO | "gpu" : " 17 GB", | 20260505 13:49:38.944 | INFO | "huge" : "", | 20260505 13:49:38.944 | INFO | "huge_shared" : " 27 GB", | 20260505 13:49:38.944 | INFO | "malloc" : " 14 GB", | 20260505 13:49:38.945 | INFO | "pinned" : " 29 GB" | 20260505 13:49:38.945 | INFO | } | 20260505 13:49:38.945 | INFO | Perf report for the last 5.13426 seconds: | 20260505 13:49:38.945 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:49:38.945 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:38.945 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:38.945 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:38.945 | INFO | ANALYSIS : 64.6%, 3.0, 100.2ms, 0 | 20260505 13:49:38.945 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:38.945 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260505 13:49:38.945 | INFO | PRE_HQ : 60.8%, 1.8, 57.6ms, 0 | 20260505 13:49:38.945 | INFO | BAZWRITER : 64.4%, 4.0, 131.8ms, 0 | 20260505 13:49:38.945 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 13:49:38.945 | INFO | Uploaded 36GiB to GPU in 2.18476s (16.4778GiB/s) | 20260505 13:49:38.945 | INFO | Downloaded 38GiB to GPU in 2.27729s (16.6865GiB/s) | 20260505 13:49:44.087 | INFO | Analyzing chunk frames = [3695104,3695616) | 20260505 13:49:44.087 | INFO | MemInfo: { | 20260505 13:49:44.087 | INFO | "gpu" : " 17 GB", | 20260505 13:49:44.087 | INFO | "huge" : "", | 20260505 13:49:44.087 | INFO | "huge_shared" : " 27 GB", | 20260505 13:49:44.087 | INFO | "malloc" : " 14 GB", | 20260505 13:49:44.087 | INFO | "pinned" : " 29 GB" | 20260505 13:49:44.087 | INFO | } | 20260505 13:49:44.088 | INFO | Perf report for the last 5.1429 seconds: | 20260505 13:49:44.088 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:49:44.088 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:44.088 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:44.088 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:44.088 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 13:49:44.088 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:44.088 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:44.088 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:49:44.088 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:44.088 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:44.088 | INFO | Uploaded 36GiB to GPU in 2.1798s (16.5153GiB/s) | 20260505 13:49:44.088 | INFO | Downloaded 34GiB to GPU in 2.05409s (16.5524GiB/s) | 20260505 13:49:49.234 | INFO | Analyzing chunk frames = [3695616,3696128) | 20260505 13:49:49.234 | INFO | MemInfo: { | 20260505 13:49:49.234 | INFO | "gpu" : " 17 GB", | 20260505 13:49:49.234 | INFO | "huge" : "", | 20260505 13:49:49.234 | INFO | "huge_shared" : " 27 GB", | 20260505 13:49:49.234 | INFO | "malloc" : " 14 GB", | 20260505 13:49:49.234 | INFO | "pinned" : " 29 GB" | 20260505 13:49:49.234 | INFO | } | 20260505 13:49:49.234 | INFO | Perf report for the last 5.14689 seconds: | 20260505 13:49:49.234 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:49:49.234 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:49.234 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:49.234 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:49.235 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 13:49:49.235 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:49.235 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:49.235 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:49:49.235 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:49.235 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:49.235 | INFO | Uploaded 36GiB to GPU in 2.231s (16.1363GiB/s) | 20260505 13:49:49.235 | INFO | Downloaded 34GiB to GPU in 2.096s (16.2214GiB/s) | 20260505 13:49:54.371 | INFO | Analyzing chunk frames = [3696128,3696640) | 20260505 13:49:54.371 | INFO | MemInfo: { | 20260505 13:49:54.371 | INFO | "gpu" : " 17 GB", | 20260505 13:49:54.371 | INFO | "huge" : "", | 20260505 13:49:54.371 | INFO | "huge_shared" : " 27 GB", | 20260505 13:49:54.371 | INFO | "malloc" : " 14 GB", | 20260505 13:49:54.371 | INFO | "pinned" : " 29 GB" | 20260505 13:49:54.371 | INFO | } | 20260505 13:49:54.371 | INFO | Perf report for the last 5.13644 seconds: | 20260505 13:49:54.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:49:54.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:54.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:54.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:54.371 | INFO | ANALYSIS : 62.7%, 3.0, 97.3ms, 0 | 20260505 13:49:54.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:54.371 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:54.371 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 13:49:54.371 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:54.371 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:54.371 | INFO | Uploaded 36GiB to GPU in 2.17031s (16.5875GiB/s) | 20260505 13:49:54.371 | INFO | Downloaded 34GiB to GPU in 2.12883s (15.9712GiB/s) | 20260505 13:49:59.509 | INFO | Analyzing chunk frames = [3696640,3697152) | 20260505 13:49:59.509 | INFO | MemInfo: { | 20260505 13:49:59.509 | INFO | "gpu" : " 17 GB", | 20260505 13:49:59.509 | INFO | "huge" : "", | 20260505 13:49:59.509 | INFO | "huge_shared" : " 27 GB", | 20260505 13:49:59.509 | INFO | "malloc" : " 14 GB", | 20260505 13:49:59.509 | INFO | "pinned" : " 29 GB" | 20260505 13:49:59.509 | INFO | } | 20260505 13:49:59.509 | INFO | Perf report for the last 5.13859 seconds: | 20260505 13:49:59.509 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:49:59.509 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:59.509 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:59.509 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:59.510 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 13:49:59.510 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:59.510 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:59.510 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 13:49:59.510 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:59.510 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:49:59.510 | INFO | Uploaded 36GiB to GPU in 2.16986s (16.5909GiB/s) | 20260505 13:49:59.510 | INFO | Downloaded 34GiB to GPU in 2.12867s (15.9724GiB/s) | 20260505 13:50:04.639 | INFO | Analyzing chunk frames = [3697152,3697664) | 20260505 13:50:04.639 | INFO | MemInfo: { | 20260505 13:50:04.639 | INFO | "gpu" : " 17 GB", | 20260505 13:50:04.639 | INFO | "huge" : "", | 20260505 13:50:04.639 | INFO | "huge_shared" : " 27 GB", | 20260505 13:50:04.639 | INFO | "malloc" : " 14 GB", | 20260505 13:50:04.639 | INFO | "pinned" : " 29 GB" | 20260505 13:50:04.639 | INFO | } | 20260505 13:50:04.639 | INFO | Perf report for the last 5.12969 seconds: | 20260505 13:50:04.639 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:50:04.639 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:04.639 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:04.639 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:04.639 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 13:50:04.639 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:04.639 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:04.639 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:50:04.639 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:04.639 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:04.640 | INFO | Uploaded 36GiB to GPU in 2.15237s (16.7258GiB/s) | 20260505 13:50:04.640 | INFO | Downloaded 34GiB to GPU in 2.11021s (16.1121GiB/s) | 20260505 13:50:09.790 | INFO | Analyzing chunk frames = [3697664,3698176) | 20260505 13:50:09.790 | INFO | MemInfo: { | 20260505 13:50:09.790 | INFO | "gpu" : " 17 GB", | 20260505 13:50:09.790 | INFO | "huge" : "", | 20260505 13:50:09.790 | INFO | "huge_shared" : " 27 GB", | 20260505 13:50:09.790 | INFO | "malloc" : " 14 GB", | 20260505 13:50:09.790 | INFO | "pinned" : " 29 GB" | 20260505 13:50:09.790 | INFO | } | 20260505 13:50:09.791 | INFO | Perf report for the last 5.1514 seconds: | 20260505 13:50:09.791 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:50:09.791 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:09.791 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:09.791 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:09.791 | INFO | ANALYSIS : 63.1%, 3.0, 98.7ms, 0 | 20260505 13:50:09.791 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:09.791 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:09.791 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 13:50:09.791 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:09.791 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:09.791 | INFO | Uploaded 36GiB to GPU in 2.17511s (16.5509GiB/s) | 20260505 13:50:09.791 | INFO | Downloaded 34GiB to GPU in 2.1514s (15.8037GiB/s) | 20260505 13:50:14.934 | INFO | Analyzing chunk frames = [3698176,3698688) | 20260505 13:50:14.934 | INFO | MemInfo: { | 20260505 13:50:14.934 | INFO | "gpu" : " 17 GB", | 20260505 13:50:14.934 | INFO | "huge" : "", | 20260505 13:50:14.934 | INFO | "huge_shared" : " 27 GB", | 20260505 13:50:14.934 | INFO | "malloc" : " 14 GB", | 20260505 13:50:14.934 | INFO | "pinned" : " 29 GB" | 20260505 13:50:14.934 | INFO | } | 20260505 13:50:14.934 | INFO | Perf report for the last 5.14353 seconds: | 20260505 13:50:14.934 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:50:14.934 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:14.934 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:14.934 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:14.934 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 13:50:14.934 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:14.934 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:14.934 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 13:50:14.934 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:14.934 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:14.934 | INFO | Uploaded 36GiB to GPU in 2.22054s (16.2123GiB/s) | 20260505 13:50:14.934 | INFO | Downloaded 34GiB to GPU in 2.09517s (16.2278GiB/s) | 20260505 13:50:15.068 | INFO | Starting a round of Baz IO | 20260505 13:50:18.289 | INFO | Populating RTMetrics report | 20260505 13:50:18.290 | INFO | RTMetrics report populated | 20260505 13:50:18.290 | INFO | Opening tmp RTMetrics json file | 20260505 13:50:18.291 | INFO | Serializing RTMetrics to json | 20260505 13:50:18.292 | INFO | Serialization done, swapping tmp file now... | 20260505 13:50:18.292 | INFO | RTMetrics done | 20260505 13:50:18.426 | INFO | Finished a round of Baz IO: Wrote 2.287 GiB in 3.358 seconds | 20260505 13:50:18.426 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.97% / 0.00% / 77.03% | 20260505 13:50:18.426 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.648006% / 0% / 7.54992e-05% | 20260505 13:50:18.426 | INFO | Baz IO overall: | 20260505 13:50:18.426 | INFO | BytesWritten : 2203646268520 | 20260505 13:50:18.426 | INFO | NumEvents : 1048150100707 | 20260505 13:50:20.070 | INFO | Analyzing chunk frames = [3698688,3699200) | 20260505 13:50:20.070 | INFO | MemInfo: { | 20260505 13:50:20.070 | INFO | "gpu" : " 17 GB", | 20260505 13:50:20.070 | INFO | "huge" : "", | 20260505 13:50:20.070 | INFO | "huge_shared" : " 27 GB", | 20260505 13:50:20.070 | INFO | "malloc" : " 14 GB", | 20260505 13:50:20.070 | INFO | "pinned" : " 29 GB" | 20260505 13:50:20.070 | INFO | } | 20260505 13:50:20.070 | INFO | Perf report for the last 5.13619 seconds: | 20260505 13:50:20.070 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:50:20.070 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:20.070 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:20.070 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:20.070 | INFO | ANALYSIS : 64.2%, 3.0, 99.7ms, 0 | 20260505 13:50:20.070 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:20.070 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260505 13:50:20.070 | INFO | PRE_HQ : 61.3%, 1.8, 57.6ms, 0 | 20260505 13:50:20.071 | INFO | BAZWRITER : 64.5%, 4.0, 133.6ms, 0 | 20260505 13:50:20.071 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 13:50:20.071 | INFO | Uploaded 36GiB to GPU in 2.20327s (16.3394GiB/s) | 20260505 13:50:20.071 | INFO | Downloaded 38GiB to GPU in 2.2545s (16.8552GiB/s) | 20260505 13:50:25.200 | INFO | Analyzing chunk frames = [3699200,3699712) | 20260505 13:50:25.200 | INFO | MemInfo: { | 20260505 13:50:25.200 | INFO | "gpu" : " 17 GB", | 20260505 13:50:25.200 | INFO | "huge" : "", | 20260505 13:50:25.200 | INFO | "huge_shared" : " 27 GB", | 20260505 13:50:25.200 | INFO | "malloc" : " 14 GB", | 20260505 13:50:25.200 | INFO | "pinned" : " 29 GB" | 20260505 13:50:25.200 | INFO | } | 20260505 13:50:25.201 | INFO | Perf report for the last 5.13021 seconds: | 20260505 13:50:25.201 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:50:25.201 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:25.201 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:25.201 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:25.201 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 13:50:25.201 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:25.201 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:25.201 | INFO | PRE_HQ : 28.5%, 1.0, 14.7ms, 0 | 20260505 13:50:25.201 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:25.201 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:25.201 | INFO | Uploaded 36GiB to GPU in 2.20023s (16.3619GiB/s) | 20260505 13:50:25.201 | INFO | Downloaded 34GiB to GPU in 2.03916s (16.6736GiB/s) | 20260505 13:50:30.341 | INFO | Analyzing chunk frames = [3699712,3700224) | 20260505 13:50:30.341 | INFO | MemInfo: { | 20260505 13:50:30.341 | INFO | "gpu" : " 17 GB", | 20260505 13:50:30.341 | INFO | "huge" : "", | 20260505 13:50:30.341 | INFO | "huge_shared" : " 27 GB", | 20260505 13:50:30.341 | INFO | "malloc" : " 14 GB", | 20260505 13:50:30.341 | INFO | "pinned" : " 29 GB" | 20260505 13:50:30.341 | INFO | } | 20260505 13:50:30.341 | INFO | Perf report for the last 5.14016 seconds: | 20260505 13:50:30.341 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:50:30.341 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:30.341 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:30.341 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:30.341 | INFO | ANALYSIS : 63.2%, 3.0, 98.3ms, 0 | 20260505 13:50:30.341 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:30.341 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:30.341 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 13:50:30.341 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:30.341 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:30.341 | INFO | Uploaded 36GiB to GPU in 2.17784s (16.5301GiB/s) | 20260505 13:50:30.341 | INFO | Downloaded 34GiB to GPU in 2.13134s (15.9524GiB/s) | 20260505 13:50:35.485 | INFO | Analyzing chunk frames = [3700224,3700736) | 20260505 13:50:35.485 | INFO | MemInfo: { | 20260505 13:50:35.485 | INFO | "gpu" : " 17 GB", | 20260505 13:50:35.485 | INFO | "huge" : "", | 20260505 13:50:35.485 | INFO | "huge_shared" : " 27 GB", | 20260505 13:50:35.485 | INFO | "malloc" : " 14 GB", | 20260505 13:50:35.485 | INFO | "pinned" : " 29 GB" | 20260505 13:50:35.485 | INFO | } | 20260505 13:50:35.485 | INFO | Perf report for the last 5.14429 seconds: | 20260505 13:50:35.485 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:50:35.485 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:35.485 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:35.485 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:35.485 | INFO | ANALYSIS : 62.9%, 3.0, 97.7ms, 0 | 20260505 13:50:35.485 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:35.485 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:35.485 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 13:50:35.485 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:35.485 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:35.485 | INFO | Uploaded 36GiB to GPU in 2.20124s (16.3545GiB/s) | 20260505 13:50:35.485 | INFO | Downloaded 34GiB to GPU in 2.13056s (15.9582GiB/s) | 20260505 13:50:40.635 | INFO | Analyzing chunk frames = [3700736,3701248) | 20260505 13:50:40.635 | INFO | MemInfo: { | 20260505 13:50:40.635 | INFO | "gpu" : " 17 GB", | 20260505 13:50:40.635 | INFO | "huge" : "", | 20260505 13:50:40.635 | INFO | "huge_shared" : " 27 GB", | 20260505 13:50:40.635 | INFO | "malloc" : " 14 GB", | 20260505 13:50:40.635 | INFO | "pinned" : " 29 GB" | 20260505 13:50:40.635 | INFO | } | 20260505 13:50:40.635 | INFO | Perf report for the last 5.15033 seconds: | 20260505 13:50:40.635 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:50:40.635 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:40.635 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:40.635 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:40.635 | INFO | ANALYSIS : 62.5%, 3.0, 97.8ms, 0 | 20260505 13:50:40.635 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:40.635 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:40.636 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 13:50:40.636 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:40.636 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:40.636 | INFO | Uploaded 36GiB to GPU in 2.13886s (16.8314GiB/s) | 20260505 13:50:40.636 | INFO | Downloaded 34GiB to GPU in 2.12299s (16.0151GiB/s) | 20260505 13:50:45.756 | INFO | Analyzing chunk frames = [3701248,3701760) | 20260505 13:50:45.756 | INFO | MemInfo: { | 20260505 13:50:45.756 | INFO | "gpu" : " 17 GB", | 20260505 13:50:45.756 | INFO | "huge" : "", | 20260505 13:50:45.756 | INFO | "huge_shared" : " 27 GB", | 20260505 13:50:45.756 | INFO | "malloc" : " 14 GB", | 20260505 13:50:45.757 | INFO | "pinned" : " 29 GB" | 20260505 13:50:45.757 | INFO | } | 20260505 13:50:45.757 | INFO | Perf report for the last 5.12121 seconds: | 20260505 13:50:45.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:50:45.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:45.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:45.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:45.757 | INFO | ANALYSIS : 62.9%, 3.0, 97.8ms, 0 | 20260505 13:50:45.757 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:45.757 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:45.757 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 13:50:45.757 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:45.757 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:45.757 | INFO | Uploaded 36GiB to GPU in 2.13752s (16.842GiB/s) | 20260505 13:50:45.757 | INFO | Downloaded 34GiB to GPU in 2.02912s (16.756GiB/s) | 20260505 13:50:50.898 | INFO | Analyzing chunk frames = [3701760,3702272) | 20260505 13:50:50.898 | INFO | MemInfo: { | 20260505 13:50:50.898 | INFO | "gpu" : " 17 GB", | 20260505 13:50:50.898 | INFO | "huge" : "", | 20260505 13:50:50.898 | INFO | "huge_shared" : " 27 GB", | 20260505 13:50:50.898 | INFO | "malloc" : " 14 GB", | 20260505 13:50:50.898 | INFO | "pinned" : " 29 GB" | 20260505 13:50:50.898 | INFO | } | 20260505 13:50:50.898 | INFO | Perf report for the last 5.14135 seconds: | 20260505 13:50:50.898 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:50:50.898 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:50.898 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:50.898 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:50.898 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 13:50:50.898 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:50.898 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:50.898 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 13:50:50.898 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:50.898 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:50.898 | INFO | Uploaded 36GiB to GPU in 2.16358s (16.6391GiB/s) | 20260505 13:50:50.898 | INFO | Downloaded 34GiB to GPU in 2.16363s (15.7143GiB/s) | 20260505 13:50:56.044 | INFO | Analyzing chunk frames = [3702272,3702784) | 20260505 13:50:56.044 | INFO | MemInfo: { | 20260505 13:50:56.044 | INFO | "gpu" : " 17 GB", | 20260505 13:50:56.044 | INFO | "huge" : "", | 20260505 13:50:56.044 | INFO | "huge_shared" : " 27 GB", | 20260505 13:50:56.044 | INFO | "malloc" : " 14 GB", | 20260505 13:50:56.044 | INFO | "pinned" : " 29 GB" | 20260505 13:50:56.044 | INFO | } | 20260505 13:50:56.045 | INFO | Perf report for the last 5.14658 seconds: | 20260505 13:50:56.045 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:50:56.045 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:56.045 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:56.045 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:56.045 | INFO | ANALYSIS : 63.5%, 3.0, 99.4ms, 0 | 20260505 13:50:56.045 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:56.045 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:56.045 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:50:56.045 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:56.045 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:50:56.045 | INFO | Uploaded 36GiB to GPU in 2.17374s (16.5613GiB/s) | 20260505 13:50:56.045 | INFO | Downloaded 34GiB to GPU in 2.10363s (16.1626GiB/s) | 20260505 13:50:56.045 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 13:50:56.045 | INFO | Total mem usage: 15.3 GiB | 20260505 13:50:56.045 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 13:50:56.045 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -7.12%) | 20260505 13:50:56.045 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 13:50:56.045 | INFO | High Water Breakdown: | 20260505 13:50:56.045 | INFO | 61.1% : 8.75 GiB ( 91.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 13:50:56.045 | INFO | 38.7% : 5.54 GiB ( 83.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 13:50:56.045 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 13:50:56.045 | INFO | ------------------------------------------ | 20260505 13:50:56.045 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:50:56.045 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 13:50:56.045 | INFO | Total mem usage: 26.5 GiB | 20260505 13:50:56.045 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 13:50:56.045 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 13:50:56.045 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 13:50:56.045 | INFO | High Water Breakdown: | 20260505 13:50:56.045 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:50:56.045 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:50:56.045 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:50:56.045 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:50:56.045 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:50:56.045 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:50:56.045 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 13:50:56.045 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 13:50:56.045 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:50:56.045 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 13:50:56.045 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 13:50:56.045 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 13:50:56.045 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 13:50:56.045 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 13:50:56.045 | INFO | ------------------------------------------ | 20260505 13:50:56.045 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:50:56.045 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 13:50:56.045 | INFO | Total mem usage: 25.6 GiB | 20260505 13:50:56.045 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 13:50:56.045 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 13:50:56.045 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 13:50:56.045 | INFO | High Water Breakdown: | 20260505 13:50:56.045 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 13:50:56.045 | INFO | ------------------------------------------ | 20260505 13:50:56.045 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:50:56.045 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 13:50:56.045 | INFO | Total mem usage: 15.7 GiB | 20260505 13:50:56.045 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 13:50:56.045 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 13:50:56.045 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 13:50:56.046 | INFO | High Water Breakdown: | 20260505 13:50:56.046 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 13:50:56.046 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 13:50:56.046 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 13:50:56.046 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 13:50:56.046 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 13:50:56.046 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 13:50:56.046 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 13:50:56.046 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 13:50:56.046 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 13:50:56.046 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:50:56.046 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 13:50:56.046 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:50:56.046 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 13:50:56.046 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 13:50:56.046 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 13:50:56.046 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 13:50:56.046 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:50:56.046 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 13:50:56.046 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:50:56.046 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:50:56.046 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:50:56.046 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 13:50:56.046 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 13:50:56.046 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:50:56.046 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 13:50:56.046 | INFO | ------------------------------------------ | 20260505 13:50:56.046 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:50:56.046 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 13:50:56.046 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:50:56.046 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:50:56.046 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.65e+01, 1.37e+01 / 1.59e+00 | 20260505 13:50:56.046 | INFO | PerfReport: 0.29%, AggregateStats -- 9.31e-02 / 1.01e-01, 1.07e-02 / 4.24e-03 | 20260505 13:50:56.046 | INFO | PerfReport: 11.78%, Baseline -- 3.76e+00 / 4.49e+00, 5.93e-01 / 2.00e-01 | 20260505 13:50:56.046 | INFO | PerfReport: 3.30%, Histogram -- 1.05e+00 / 1.52e+00, 2.02e-01 / 8.51e-02 | 20260505 13:50:56.046 | INFO | PerfReport: 15.95%, DME -- 4.07e+01 / 4.57e+01, 4.57e+00 / 1.70e+00 | 20260505 13:50:56.046 | INFO | PerfReport: 56.38%, FrameLabeling -- 1.80e+01 / 1.99e+01, 1.44e+00 / 1.89e+00 | 20260505 13:50:56.046 | INFO | PerfReport: 7.47%, PulseAccumulating -- 2.38e+00 / 2.88e+00, 4.52e-01 / 2.53e-01 | 20260505 13:50:56.046 | INFO | PerfReport: 3.57%, Metrics -- 1.14e+00 / 1.47e+00, 3.14e-01 / 1.55e-01 | 20260505 13:50:56.046 | INFO | PerfReport: 1.25%, PulseSerializing -- 3.99e-01 / 4.58e-01, 7.12e-02 / 3.03e-02 | 20260505 13:50:56.046 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 13:50:56.046 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:50:56.046 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:50:56.046 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260505 13:50:56.046 | INFO | PerfReport: 22.55%, Upload -- 2.27e+01 / 0.00e+00, 8.06e+00 / 0.00e+00 | 20260505 13:50:56.046 | INFO | PerfReport: 21.99%, Download -- 2.21e+01 / 0.00e+00, 8.05e+00 / 0.00e+00 | 20260505 13:50:56.046 | INFO | PerfReport: 31.64%, Compute -- 3.19e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 13:50:56.046 | INFO | PerfReport: 2.22%, LockUpload -- 2.23e+00 / 0.00e+00, 7.90e+00 / 0.00e+00 | 20260505 13:50:56.046 | INFO | PerfReport: 1.40%, LockDownload -- 1.41e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260505 13:50:56.046 | INFO | PerfReport: 20.20%, LockCompute -- 2.03e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260505 13:50:56.046 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.93e-02 / 0.00e+00 | 20260505 13:50:56.178 | INFO | Starting a round of Baz IO | 20260505 13:50:59.391 | INFO | Populating RTMetrics report | 20260505 13:50:59.392 | INFO | RTMetrics report populated | 20260505 13:50:59.392 | INFO | Opening tmp RTMetrics json file | 20260505 13:50:59.412 | INFO | Serializing RTMetrics to json | 20260505 13:50:59.419 | INFO | Serialization done, swapping tmp file now... | 20260505 13:50:59.435 | INFO | RTMetrics done | 20260505 13:50:59.497 | INFO | Finished a round of Baz IO: Wrote 2.288 GiB in 3.319 seconds | 20260505 13:50:59.497 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.96% / 0.00% / 77.03% | 20260505 13:50:59.497 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.647716% / 0% / 6.75519e-05% | 20260505 13:50:59.497 | INFO | Baz IO overall: | 20260505 13:50:59.497 | INFO | BytesWritten : 2206102585305 | 20260505 13:50:59.497 | INFO | NumEvents : 1049312290984 | 20260505 13:51:01.184 | INFO | Analyzing chunk frames = [3702784,3703296) | 20260505 13:51:01.184 | INFO | MemInfo: { | 20260505 13:51:01.184 | INFO | "gpu" : " 17 GB", | 20260505 13:51:01.184 | INFO | "huge" : "", | 20260505 13:51:01.184 | INFO | "huge_shared" : " 27 GB", | 20260505 13:51:01.184 | INFO | "malloc" : " 14 GB", | 20260505 13:51:01.184 | INFO | "pinned" : " 29 GB" | 20260505 13:51:01.184 | INFO | } | 20260505 13:51:01.184 | INFO | Perf report for the last 5.1392 seconds: | 20260505 13:51:01.184 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:51:01.184 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:01.184 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:01.184 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:01.184 | INFO | ANALYSIS : 64.0%, 3.0, 99.2ms, 0 | 20260505 13:51:01.184 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:01.184 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260505 13:51:01.184 | INFO | PRE_HQ : 60.9%, 1.8, 57.5ms, 0 | 20260505 13:51:01.184 | INFO | BAZWRITER : 63.9%, 4.0, 131.4ms, 0 | 20260505 13:51:01.184 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 13:51:01.185 | INFO | Uploaded 36GiB to GPU in 2.17456s (16.5551GiB/s) | 20260505 13:51:01.185 | INFO | Downloaded 38GiB to GPU in 2.26193s (16.7998GiB/s) | 20260505 13:51:06.315 | INFO | Analyzing chunk frames = [3703296,3703808) | 20260505 13:51:06.315 | INFO | MemInfo: { | 20260505 13:51:06.315 | INFO | "gpu" : " 17 GB", | 20260505 13:51:06.315 | INFO | "huge" : "", | 20260505 13:51:06.315 | INFO | "huge_shared" : " 27 GB", | 20260505 13:51:06.315 | INFO | "malloc" : " 14 GB", | 20260505 13:51:06.315 | INFO | "pinned" : " 29 GB" | 20260505 13:51:06.315 | INFO | } | 20260505 13:51:06.315 | INFO | Perf report for the last 5.13164 seconds: | 20260505 13:51:06.315 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:51:06.315 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:06.315 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:06.315 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:06.315 | INFO | ANALYSIS : 63.7%, 3.0, 99.2ms, 0 | 20260505 13:51:06.315 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:06.315 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:06.316 | INFO | PRE_HQ : 28.3%, 1.0, 14.6ms, 0 | 20260505 13:51:06.316 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:06.316 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:06.316 | INFO | Uploaded 36GiB to GPU in 2.20032s (16.3613GiB/s) | 20260505 13:51:06.316 | INFO | Downloaded 34GiB to GPU in 2.04028s (16.6644GiB/s) | 20260505 13:51:11.452 | INFO | Analyzing chunk frames = [3703808,3704320) | 20260505 13:51:11.452 | INFO | MemInfo: { | 20260505 13:51:11.452 | INFO | "gpu" : " 17 GB", | 20260505 13:51:11.452 | INFO | "huge" : "", | 20260505 13:51:11.452 | INFO | "huge_shared" : " 27 GB", | 20260505 13:51:11.452 | INFO | "malloc" : " 14 GB", | 20260505 13:51:11.452 | INFO | "pinned" : " 29 GB" | 20260505 13:51:11.452 | INFO | } | 20260505 13:51:11.452 | INFO | Perf report for the last 5.13686 seconds: | 20260505 13:51:11.452 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:51:11.452 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:11.452 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:11.452 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:11.452 | INFO | ANALYSIS : 63.1%, 3.0, 98.2ms, 0 | 20260505 13:51:11.452 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:11.452 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:11.452 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:51:11.452 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:11.452 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:11.453 | INFO | Uploaded 36GiB to GPU in 2.17027s (16.5878GiB/s) | 20260505 13:51:11.453 | INFO | Downloaded 34GiB to GPU in 2.11845s (16.0495GiB/s) | 20260505 13:51:16.597 | INFO | Analyzing chunk frames = [3704320,3704832) | 20260505 13:51:16.597 | INFO | MemInfo: { | 20260505 13:51:16.597 | INFO | "gpu" : " 17 GB", | 20260505 13:51:16.597 | INFO | "huge" : "", | 20260505 13:51:16.597 | INFO | "huge_shared" : " 27 GB", | 20260505 13:51:16.597 | INFO | "malloc" : " 14 GB", | 20260505 13:51:16.597 | INFO | "pinned" : " 29 GB" | 20260505 13:51:16.597 | INFO | } | 20260505 13:51:16.597 | INFO | Perf report for the last 5.14455 seconds: | 20260505 13:51:16.597 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:51:16.597 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:16.597 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:16.597 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:16.597 | INFO | ANALYSIS : 63.0%, 3.0, 98.0ms, 0 | 20260505 13:51:16.597 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:16.597 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:16.597 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 13:51:16.597 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:16.597 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:16.597 | INFO | Uploaded 36GiB to GPU in 2.20983s (16.2909GiB/s) | 20260505 13:51:16.597 | INFO | Downloaded 34GiB to GPU in 2.13135s (15.9524GiB/s) | 20260505 13:51:21.742 | INFO | Analyzing chunk frames = [3704832,3705344) | 20260505 13:51:21.742 | INFO | MemInfo: { | 20260505 13:51:21.742 | INFO | "gpu" : " 17 GB", | 20260505 13:51:21.742 | INFO | "huge" : "", | 20260505 13:51:21.742 | INFO | "huge_shared" : " 27 GB", | 20260505 13:51:21.742 | INFO | "malloc" : " 14 GB", | 20260505 13:51:21.742 | INFO | "pinned" : " 29 GB" | 20260505 13:51:21.742 | INFO | } | 20260505 13:51:21.742 | INFO | Perf report for the last 5.14555 seconds: | 20260505 13:51:21.742 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:51:21.742 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:21.742 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:21.742 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:21.742 | INFO | ANALYSIS : 62.7%, 3.0, 98.0ms, 0 | 20260505 13:51:21.742 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:21.742 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:21.742 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 13:51:21.742 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:21.743 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:21.743 | INFO | Uploaded 36GiB to GPU in 2.15786s (16.6832GiB/s) | 20260505 13:51:21.743 | INFO | Downloaded 34GiB to GPU in 2.12391s (16.0082GiB/s) | 20260505 13:51:26.884 | INFO | Analyzing chunk frames = [3705344,3705856) | 20260505 13:51:26.884 | INFO | MemInfo: { | 20260505 13:51:26.884 | INFO | "gpu" : " 17 GB", | 20260505 13:51:26.884 | INFO | "huge" : "", | 20260505 13:51:26.884 | INFO | "huge_shared" : " 27 GB", | 20260505 13:51:26.884 | INFO | "malloc" : " 14 GB", | 20260505 13:51:26.884 | INFO | "pinned" : " 29 GB" | 20260505 13:51:26.884 | INFO | } | 20260505 13:51:26.885 | INFO | Perf report for the last 5.1421 seconds: | 20260505 13:51:26.885 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:51:26.885 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:26.885 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:26.885 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:26.885 | INFO | ANALYSIS : 62.7%, 3.0, 98.0ms, 0 | 20260505 13:51:26.885 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:26.885 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:26.885 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 13:51:26.885 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:26.885 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:26.885 | INFO | Uploaded 36GiB to GPU in 2.14587s (16.7764GiB/s) | 20260505 13:51:26.885 | INFO | Downloaded 34GiB to GPU in 2.03338s (16.7209GiB/s) | 20260505 13:51:32.010 | INFO | Analyzing chunk frames = [3705856,3706368) | 20260505 13:51:32.010 | INFO | MemInfo: { | 20260505 13:51:32.010 | INFO | "gpu" : " 17 GB", | 20260505 13:51:32.010 | INFO | "huge" : "", | 20260505 13:51:32.010 | INFO | "huge_shared" : " 27 GB", | 20260505 13:51:32.010 | INFO | "malloc" : " 14 GB", | 20260505 13:51:32.010 | INFO | "pinned" : " 29 GB" | 20260505 13:51:32.010 | INFO | } | 20260505 13:51:32.010 | INFO | Perf report for the last 5.12584 seconds: | 20260505 13:51:32.010 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:51:32.010 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:32.010 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:32.010 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:32.010 | INFO | ANALYSIS : 63.9%, 3.0, 99.4ms, 0 | 20260505 13:51:32.010 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:32.010 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:32.010 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 13:51:32.010 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:32.010 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:32.011 | INFO | Uploaded 36GiB to GPU in 2.11788s (16.9981GiB/s) | 20260505 13:51:32.011 | INFO | Downloaded 34GiB to GPU in 2.18105s (15.5888GiB/s) | 20260505 13:51:37.147 | INFO | Analyzing chunk frames = [3706368,3706880) | 20260505 13:51:37.147 | INFO | MemInfo: { | 20260505 13:51:37.147 | INFO | "gpu" : " 17 GB", | 20260505 13:51:37.147 | INFO | "huge" : "", | 20260505 13:51:37.147 | INFO | "huge_shared" : " 27 GB", | 20260505 13:51:37.147 | INFO | "malloc" : " 14 GB", | 20260505 13:51:37.147 | INFO | "pinned" : " 29 GB" | 20260505 13:51:37.147 | INFO | } | 20260505 13:51:37.147 | INFO | Perf report for the last 5.13701 seconds: | 20260505 13:51:37.147 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:51:37.147 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:37.147 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:37.147 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:37.147 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 13:51:37.147 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:37.147 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:37.147 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 13:51:37.147 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:37.148 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:37.148 | INFO | Uploaded 36GiB to GPU in 2.17139s (16.5793GiB/s) | 20260505 13:51:37.148 | INFO | Downloaded 34GiB to GPU in 2.10633s (16.1418GiB/s) | 20260505 13:51:37.280 | INFO | Starting a round of Baz IO | 20260505 13:51:40.518 | INFO | Populating RTMetrics report | 20260505 13:51:40.518 | INFO | RTMetrics report populated | 20260505 13:51:40.518 | INFO | Opening tmp RTMetrics json file | 20260505 13:51:40.539 | INFO | Serializing RTMetrics to json | 20260505 13:51:40.560 | INFO | Serialization done, swapping tmp file now... | 20260505 13:51:40.583 | INFO | RTMetrics done | 20260505 13:51:40.620 | INFO | Finished a round of Baz IO: Wrote 2.282 GiB in 3.34 seconds | 20260505 13:51:40.620 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.96% / 0.00% / 77.04% | 20260505 13:51:40.620 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.641521% / 7.94729e-06% / 6.75519e-05% | 20260505 13:51:40.620 | INFO | Baz IO overall: | 20260505 13:51:40.620 | INFO | BytesWritten : 2208552503802 | 20260505 13:51:40.620 | INFO | NumEvents : 1050469906023 | 20260505 13:51:42.298 | INFO | Analyzing chunk frames = [3706880,3707392) | 20260505 13:51:42.298 | INFO | MemInfo: { | 20260505 13:51:42.298 | INFO | "gpu" : " 17 GB", | 20260505 13:51:42.298 | INFO | "huge" : "", | 20260505 13:51:42.298 | INFO | "huge_shared" : " 27 GB", | 20260505 13:51:42.298 | INFO | "malloc" : " 14 GB", | 20260505 13:51:42.298 | INFO | "pinned" : " 29 GB" | 20260505 13:51:42.298 | INFO | } | 20260505 13:51:42.298 | INFO | Perf report for the last 5.15076 seconds: | 20260505 13:51:42.298 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:51:42.298 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:42.298 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:42.298 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:42.298 | INFO | ANALYSIS : 64.3%, 3.0, 100.1ms, 0 | 20260505 13:51:42.298 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:42.298 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260505 13:51:42.298 | INFO | PRE_HQ : 60.4%, 1.8, 57.4ms, 0 | 20260505 13:51:42.298 | INFO | BAZWRITER : 64.2%, 4.0, 131.1ms, 0 | 20260505 13:51:42.298 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 13:51:42.298 | INFO | Uploaded 36GiB to GPU in 2.1669s (16.6136GiB/s) | 20260505 13:51:42.299 | INFO | Downloaded 38GiB to GPU in 2.27463s (16.706GiB/s) | 20260505 13:51:47.436 | INFO | Analyzing chunk frames = [3707392,3707904) | 20260505 13:51:47.436 | INFO | MemInfo: { | 20260505 13:51:47.436 | INFO | "gpu" : " 17 GB", | 20260505 13:51:47.436 | INFO | "huge" : "", | 20260505 13:51:47.436 | INFO | "huge_shared" : " 27 GB", | 20260505 13:51:47.436 | INFO | "malloc" : " 14 GB", | 20260505 13:51:47.437 | INFO | "pinned" : " 29 GB" | 20260505 13:51:47.437 | INFO | } | 20260505 13:51:47.437 | INFO | Perf report for the last 5.13845 seconds: | 20260505 13:51:47.437 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:51:47.437 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:47.437 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:47.437 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:47.437 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 13:51:47.437 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:47.437 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:47.437 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 13:51:47.437 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:47.437 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:47.437 | INFO | Uploaded 36GiB to GPU in 2.1995s (16.3674GiB/s) | 20260505 13:51:47.437 | INFO | Downloaded 34GiB to GPU in 2.03765s (16.6859GiB/s) | 20260505 13:51:52.574 | INFO | Analyzing chunk frames = [3707904,3708416) | 20260505 13:51:52.574 | INFO | MemInfo: { | 20260505 13:51:52.574 | INFO | "gpu" : " 17 GB", | 20260505 13:51:52.574 | INFO | "huge" : "", | 20260505 13:51:52.574 | INFO | "huge_shared" : " 27 GB", | 20260505 13:51:52.574 | INFO | "malloc" : " 14 GB", | 20260505 13:51:52.574 | INFO | "pinned" : " 29 GB" | 20260505 13:51:52.574 | INFO | } | 20260505 13:51:52.574 | INFO | Perf report for the last 5.1372 seconds: | 20260505 13:51:52.574 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:51:52.574 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:52.574 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:52.574 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:52.574 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260505 13:51:52.574 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:52.574 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:52.574 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:51:52.574 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:52.574 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:52.574 | INFO | Uploaded 36GiB to GPU in 2.23253s (16.1252GiB/s) | 20260505 13:51:52.574 | INFO | Downloaded 34GiB to GPU in 2.10355s (16.1632GiB/s) | 20260505 13:51:57.709 | INFO | Analyzing chunk frames = [3708416,3708928) | 20260505 13:51:57.709 | INFO | MemInfo: { | 20260505 13:51:57.709 | INFO | "gpu" : " 17 GB", | 20260505 13:51:57.709 | INFO | "huge" : "", | 20260505 13:51:57.709 | INFO | "huge_shared" : " 27 GB", | 20260505 13:51:57.709 | INFO | "malloc" : " 14 GB", | 20260505 13:51:57.709 | INFO | "pinned" : " 29 GB" | 20260505 13:51:57.709 | INFO | } | 20260505 13:51:57.709 | INFO | Perf report for the last 5.1352 seconds: | 20260505 13:51:57.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:51:57.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:57.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:57.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:57.709 | INFO | ANALYSIS : 62.8%, 3.0, 97.4ms, 0 | 20260505 13:51:57.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:57.709 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:57.709 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 13:51:57.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:57.709 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:51:57.709 | INFO | Uploaded 36GiB to GPU in 2.17329s (16.5648GiB/s) | 20260505 13:51:57.709 | INFO | Downloaded 34GiB to GPU in 2.13261s (15.9429GiB/s) | 20260505 13:52:02.855 | INFO | Analyzing chunk frames = [3708928,3709440) | 20260505 13:52:02.855 | INFO | MemInfo: { | 20260505 13:52:02.855 | INFO | "gpu" : " 17 GB", | 20260505 13:52:02.855 | INFO | "huge" : "", | 20260505 13:52:02.855 | INFO | "huge_shared" : " 27 GB", | 20260505 13:52:02.855 | INFO | "malloc" : " 14 GB", | 20260505 13:52:02.855 | INFO | "pinned" : " 29 GB" | 20260505 13:52:02.855 | INFO | } | 20260505 13:52:02.856 | INFO | Perf report for the last 5.14648 seconds: | 20260505 13:52:02.856 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:52:02.856 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:02.856 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:02.856 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:02.856 | INFO | ANALYSIS : 62.6%, 3.0, 98.0ms, 0 | 20260505 13:52:02.856 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:02.856 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:02.856 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:52:02.856 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:02.856 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:02.856 | INFO | Uploaded 36GiB to GPU in 2.14603s (16.7751GiB/s) | 20260505 13:52:02.856 | INFO | Downloaded 34GiB to GPU in 2.12383s (16.0088GiB/s) | 20260505 13:52:07.993 | INFO | Analyzing chunk frames = [3709440,3709952) | 20260505 13:52:07.993 | INFO | MemInfo: { | 20260505 13:52:07.993 | INFO | "gpu" : " 17 GB", | 20260505 13:52:07.993 | INFO | "huge" : "", | 20260505 13:52:07.993 | INFO | "huge_shared" : " 27 GB", | 20260505 13:52:07.993 | INFO | "malloc" : " 14 GB", | 20260505 13:52:07.993 | INFO | "pinned" : " 29 GB" | 20260505 13:52:07.993 | INFO | } | 20260505 13:52:07.993 | INFO | Perf report for the last 5.13728 seconds: | 20260505 13:52:07.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:52:07.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:07.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:07.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:07.993 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 13:52:07.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:07.993 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:07.993 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 13:52:07.993 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:07.993 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:07.993 | INFO | Uploaded 36GiB to GPU in 2.19175s (16.4252GiB/s) | 20260505 13:52:07.993 | INFO | Downloaded 34GiB to GPU in 2.11017s (16.1125GiB/s) | 20260505 13:52:13.131 | INFO | Analyzing chunk frames = [3709952,3710464) | 20260505 13:52:13.131 | INFO | MemInfo: { | 20260505 13:52:13.131 | INFO | "gpu" : " 17 GB", | 20260505 13:52:13.131 | INFO | "huge" : "", | 20260505 13:52:13.131 | INFO | "huge_shared" : " 27 GB", | 20260505 13:52:13.131 | INFO | "malloc" : " 14 GB", | 20260505 13:52:13.131 | INFO | "pinned" : " 29 GB" | 20260505 13:52:13.131 | INFO | } | 20260505 13:52:13.131 | INFO | Perf report for the last 5.1378 seconds: | 20260505 13:52:13.131 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:52:13.131 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:13.131 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:13.131 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:13.131 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 13:52:13.131 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:13.131 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:13.131 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 13:52:13.131 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:13.131 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:13.131 | INFO | Uploaded 36GiB to GPU in 2.1674s (16.6097GiB/s) | 20260505 13:52:13.131 | INFO | Downloaded 34GiB to GPU in 2.15337s (15.7892GiB/s) | 20260505 13:52:18.264 | INFO | Analyzing chunk frames = [3710464,3710976) | 20260505 13:52:18.264 | INFO | MemInfo: { | 20260505 13:52:18.264 | INFO | "gpu" : " 17 GB", | 20260505 13:52:18.264 | INFO | "huge" : "", | 20260505 13:52:18.264 | INFO | "huge_shared" : " 27 GB", | 20260505 13:52:18.264 | INFO | "malloc" : " 14 GB", | 20260505 13:52:18.264 | INFO | "pinned" : " 29 GB" | 20260505 13:52:18.264 | INFO | } | 20260505 13:52:18.264 | INFO | Perf report for the last 5.13365 seconds: | 20260505 13:52:18.264 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:52:18.264 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:18.264 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:18.264 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:18.264 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 13:52:18.264 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:18.264 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:18.264 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 13:52:18.264 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:18.264 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:18.265 | INFO | Uploaded 36GiB to GPU in 2.15759s (16.6853GiB/s) | 20260505 13:52:18.265 | INFO | Downloaded 34GiB to GPU in 2.10721s (16.1351GiB/s) | 20260505 13:52:18.265 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 13:52:18.265 | INFO | Total mem usage: 15.3 GiB | 20260505 13:52:18.265 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 13:52:18.265 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.74%) | 20260505 13:52:18.265 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 13:52:18.265 | INFO | High Water Breakdown: | 20260505 13:52:18.265 | INFO | 61% : 8.68 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 13:52:18.265 | INFO | 38.8% : 5.52 GiB ( 83.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 13:52:18.265 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 13:52:18.265 | INFO | ------------------------------------------ | 20260505 13:52:18.265 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:52:18.265 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 13:52:18.265 | INFO | Total mem usage: 26.5 GiB | 20260505 13:52:18.265 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 13:52:18.265 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 13:52:18.265 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 13:52:18.265 | INFO | High Water Breakdown: | 20260505 13:52:18.265 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:52:18.265 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:52:18.265 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:52:18.265 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:52:18.265 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:52:18.265 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:52:18.265 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 13:52:18.265 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 13:52:18.265 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:52:18.265 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 13:52:18.265 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 13:52:18.265 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 13:52:18.265 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 13:52:18.265 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 13:52:18.265 | INFO | ------------------------------------------ | 20260505 13:52:18.265 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:52:18.265 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 13:52:18.265 | INFO | Total mem usage: 25.6 GiB | 20260505 13:52:18.265 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 13:52:18.265 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 13:52:18.265 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 13:52:18.265 | INFO | High Water Breakdown: | 20260505 13:52:18.265 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 13:52:18.265 | INFO | ------------------------------------------ | 20260505 13:52:18.265 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:52:18.265 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 13:52:18.265 | INFO | Total mem usage: 15.7 GiB | 20260505 13:52:18.265 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 13:52:18.265 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 13:52:18.265 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 13:52:18.265 | INFO | High Water Breakdown: | 20260505 13:52:18.265 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 13:52:18.265 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 13:52:18.265 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 13:52:18.265 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 13:52:18.265 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 13:52:18.265 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 13:52:18.265 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 13:52:18.265 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 13:52:18.265 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 13:52:18.265 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:52:18.265 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 13:52:18.265 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:52:18.265 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 13:52:18.265 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 13:52:18.265 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 13:52:18.265 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 13:52:18.265 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:52:18.265 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 13:52:18.265 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:52:18.265 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:52:18.265 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:52:18.265 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 13:52:18.265 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 13:52:18.265 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:52:18.266 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 13:52:18.266 | INFO | ------------------------------------------ | 20260505 13:52:18.266 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:52:18.266 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 13:52:18.266 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:52:18.266 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:52:18.266 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.66e+01, 1.39e+01 / 1.97e+00 | 20260505 13:52:18.266 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 1.00e-01, 1.04e-02 / 4.52e-03 | 20260505 13:52:18.266 | INFO | PerfReport: 11.77%, Baseline -- 3.75e+00 / 4.49e+00, 5.92e-01 / 1.81e-01 | 20260505 13:52:18.266 | INFO | PerfReport: 3.29%, Histogram -- 1.05e+00 / 1.50e+00, 2.00e-01 / 8.24e-02 | 20260505 13:52:18.266 | INFO | PerfReport: 16.01%, DME -- 4.08e+01 / 4.58e+01, 4.76e+00 / 2.27e+00 | 20260505 13:52:18.266 | INFO | PerfReport: 56.36%, FrameLabeling -- 1.79e+01 / 1.98e+01, 1.38e+00 / 1.96e+00 | 20260505 13:52:18.266 | INFO | PerfReport: 7.46%, PulseAccumulating -- 2.38e+00 / 2.93e+00, 4.52e-01 / 1.73e-01 | 20260505 13:52:18.266 | INFO | PerfReport: 3.56%, Metrics -- 1.13e+00 / 1.46e+00, 3.17e-01 / 1.77e-01 | 20260505 13:52:18.266 | INFO | PerfReport: 1.25%, PulseSerializing -- 3.98e-01 / 4.50e-01, 7.07e-02 / 3.70e-02 | 20260505 13:52:18.266 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 13:52:18.266 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:52:18.266 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:52:18.266 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260505 13:52:18.266 | INFO | PerfReport: 22.48%, Upload -- 2.26e+01 / 0.00e+00, 7.98e+00 / 0.00e+00 | 20260505 13:52:18.266 | INFO | PerfReport: 22.00%, Download -- 2.22e+01 / 0.00e+00, 8.10e+00 / 0.00e+00 | 20260505 13:52:18.266 | INFO | PerfReport: 31.61%, Compute -- 3.19e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 13:52:18.266 | INFO | PerfReport: 2.21%, LockUpload -- 2.22e+00 / 0.00e+00, 7.94e+00 / 0.00e+00 | 20260505 13:52:18.266 | INFO | PerfReport: 1.50%, LockDownload -- 1.51e+00 / 0.00e+00, 3.14e+00 / 0.00e+00 | 20260505 13:52:18.266 | INFO | PerfReport: 20.20%, LockCompute -- 2.04e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260505 13:52:18.266 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.07e-02 / 0.00e+00 | 20260505 13:52:18.398 | INFO | Starting a round of Baz IO | 20260505 13:52:21.634 | INFO | Populating RTMetrics report | 20260505 13:52:21.634 | INFO | RTMetrics report populated | 20260505 13:52:21.634 | INFO | Opening tmp RTMetrics json file | 20260505 13:52:21.640 | INFO | Serializing RTMetrics to json | 20260505 13:52:21.671 | INFO | Serialization done, swapping tmp file now... | 20260505 13:52:21.696 | INFO | RTMetrics done | 20260505 13:52:21.736 | INFO | Finished a round of Baz IO: Wrote 2.284 GiB in 3.338 seconds | 20260505 13:52:21.736 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.96% / 0.00% / 77.04% | 20260505 13:52:21.736 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.641203% / 3.97364e-06% / 7.15256e-05% | 20260505 13:52:21.736 | INFO | Baz IO overall: | 20260505 13:52:21.736 | INFO | BytesWritten : 2211005182242 | 20260505 13:52:21.736 | INFO | NumEvents : 1051629417542 | 20260505 13:52:23.402 | INFO | Analyzing chunk frames = [3710976,3711488) | 20260505 13:52:23.402 | INFO | MemInfo: { | 20260505 13:52:23.402 | INFO | "gpu" : " 17 GB", | 20260505 13:52:23.402 | INFO | "huge" : "", | 20260505 13:52:23.402 | INFO | "huge_shared" : " 27 GB", | 20260505 13:52:23.402 | INFO | "malloc" : " 14 GB", | 20260505 13:52:23.402 | INFO | "pinned" : " 29 GB" | 20260505 13:52:23.402 | INFO | } | 20260505 13:52:23.402 | INFO | Perf report for the last 5.13809 seconds: | 20260505 13:52:23.402 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:52:23.402 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:23.402 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:23.402 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:23.402 | INFO | ANALYSIS : 64.5%, 3.0, 100.2ms, 0 | 20260505 13:52:23.402 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:23.402 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260505 13:52:23.402 | INFO | PRE_HQ : 60.6%, 1.8, 57.7ms, 0 | 20260505 13:52:23.402 | INFO | BAZWRITER : 64.2%, 4.0, 133.3ms, 0 | 20260505 13:52:23.403 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 13:52:23.403 | INFO | Uploaded 36GiB to GPU in 2.18466s (16.4786GiB/s) | 20260505 13:52:23.403 | INFO | Downloaded 38GiB to GPU in 2.28406s (16.637GiB/s) | 20260505 13:52:28.556 | INFO | Analyzing chunk frames = [3711488,3712000) | 20260505 13:52:28.556 | INFO | MemInfo: { | 20260505 13:52:28.556 | INFO | "gpu" : " 17 GB", | 20260505 13:52:28.556 | INFO | "huge" : "", | 20260505 13:52:28.556 | INFO | "huge_shared" : " 27 GB", | 20260505 13:52:28.556 | INFO | "malloc" : " 14 GB", | 20260505 13:52:28.556 | INFO | "pinned" : " 29 GB" | 20260505 13:52:28.556 | INFO | } | 20260505 13:52:28.556 | INFO | Perf report for the last 5.15351 seconds: | 20260505 13:52:28.556 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:52:28.556 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:28.556 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:28.556 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:28.556 | INFO | ANALYSIS : 63.4%, 3.0, 99.3ms, 0 | 20260505 13:52:28.556 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:28.556 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:28.556 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 13:52:28.556 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:28.556 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:28.556 | INFO | Uploaded 36GiB to GPU in 2.17367s (16.5619GiB/s) | 20260505 13:52:28.556 | INFO | Downloaded 34GiB to GPU in 2.03502s (16.7075GiB/s) | 20260505 13:52:33.695 | INFO | Analyzing chunk frames = [3712000,3712512) | 20260505 13:52:33.695 | INFO | MemInfo: { | 20260505 13:52:33.695 | INFO | "gpu" : " 17 GB", | 20260505 13:52:33.695 | INFO | "huge" : "", | 20260505 13:52:33.695 | INFO | "huge_shared" : " 27 GB", | 20260505 13:52:33.695 | INFO | "malloc" : " 14 GB", | 20260505 13:52:33.695 | INFO | "pinned" : " 29 GB" | 20260505 13:52:33.695 | INFO | } | 20260505 13:52:33.695 | INFO | Perf report for the last 5.13911 seconds: | 20260505 13:52:33.695 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:52:33.695 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:33.695 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:33.695 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:33.695 | INFO | ANALYSIS : 62.8%, 3.0, 97.8ms, 0 | 20260505 13:52:33.695 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:33.695 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:33.695 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 13:52:33.695 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:33.695 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:33.695 | INFO | Uploaded 36GiB to GPU in 2.16514s (16.6271GiB/s) | 20260505 13:52:33.695 | INFO | Downloaded 34GiB to GPU in 2.0351s (16.7068GiB/s) | 20260505 13:52:38.830 | INFO | Analyzing chunk frames = [3712512,3713024) | 20260505 13:52:38.830 | INFO | MemInfo: { | 20260505 13:52:38.830 | INFO | "gpu" : " 17 GB", | 20260505 13:52:38.830 | INFO | "huge" : "", | 20260505 13:52:38.830 | INFO | "huge_shared" : " 27 GB", | 20260505 13:52:38.830 | INFO | "malloc" : " 14 GB", | 20260505 13:52:38.830 | INFO | "pinned" : " 29 GB" | 20260505 13:52:38.830 | INFO | } | 20260505 13:52:38.830 | INFO | Perf report for the last 5.13491 seconds: | 20260505 13:52:38.830 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:52:38.830 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:38.830 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:38.830 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:38.830 | INFO | ANALYSIS : 62.7%, 3.0, 97.3ms, 0 | 20260505 13:52:38.830 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:38.830 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:38.830 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 13:52:38.830 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:38.830 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:38.830 | INFO | Uploaded 36GiB to GPU in 2.18344s (16.4878GiB/s) | 20260505 13:52:38.830 | INFO | Downloaded 34GiB to GPU in 2.12844s (15.9741GiB/s) | 20260505 13:52:43.957 | INFO | Analyzing chunk frames = [3713024,3713536) | 20260505 13:52:43.957 | INFO | MemInfo: { | 20260505 13:52:43.957 | INFO | "gpu" : " 17 GB", | 20260505 13:52:43.957 | INFO | "huge" : "", | 20260505 13:52:43.957 | INFO | "huge_shared" : " 27 GB", | 20260505 13:52:43.957 | INFO | "malloc" : " 14 GB", | 20260505 13:52:43.957 | INFO | "pinned" : " 29 GB" | 20260505 13:52:43.957 | INFO | } | 20260505 13:52:43.957 | INFO | Perf report for the last 5.12719 seconds: | 20260505 13:52:43.957 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:52:43.957 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:43.957 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:43.957 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:43.957 | INFO | ANALYSIS : 62.9%, 3.0, 98.0ms, 0 | 20260505 13:52:43.957 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:43.957 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:43.957 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 13:52:43.958 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:43.958 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:43.958 | INFO | Uploaded 36GiB to GPU in 2.12362s (16.9522GiB/s) | 20260505 13:52:43.958 | INFO | Downloaded 34GiB to GPU in 2.12256s (16.0184GiB/s) | 20260505 13:52:49.103 | INFO | Analyzing chunk frames = [3713536,3714048) | 20260505 13:52:49.103 | INFO | MemInfo: { | 20260505 13:52:49.103 | INFO | "gpu" : " 17 GB", | 20260505 13:52:49.103 | INFO | "huge" : "", | 20260505 13:52:49.103 | INFO | "huge_shared" : " 27 GB", | 20260505 13:52:49.104 | INFO | "malloc" : " 14 GB", | 20260505 13:52:49.104 | INFO | "pinned" : " 29 GB" | 20260505 13:52:49.104 | INFO | } | 20260505 13:52:49.104 | INFO | Perf report for the last 5.14641 seconds: | 20260505 13:52:49.104 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:52:49.104 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:49.104 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:49.104 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:49.104 | INFO | ANALYSIS : 63.1%, 3.0, 98.7ms, 0 | 20260505 13:52:49.104 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:49.104 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:49.104 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 13:52:49.104 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:49.104 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:49.104 | INFO | Uploaded 36GiB to GPU in 2.18464s (16.4787GiB/s) | 20260505 13:52:49.104 | INFO | Downloaded 34GiB to GPU in 2.11735s (16.0578GiB/s) | 20260505 13:52:54.256 | INFO | Analyzing chunk frames = [3714048,3714560) | 20260505 13:52:54.256 | INFO | MemInfo: { | 20260505 13:52:54.256 | INFO | "gpu" : " 17 GB", | 20260505 13:52:54.256 | INFO | "huge" : "", | 20260505 13:52:54.256 | INFO | "huge_shared" : " 27 GB", | 20260505 13:52:54.256 | INFO | "malloc" : " 14 GB", | 20260505 13:52:54.256 | INFO | "pinned" : " 29 GB" | 20260505 13:52:54.256 | INFO | } | 20260505 13:52:54.256 | INFO | Perf report for the last 5.15253 seconds: | 20260505 13:52:54.256 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:52:54.256 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:54.256 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:54.256 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:54.256 | INFO | ANALYSIS : 63.9%, 3.0, 100.0ms, 0 | 20260505 13:52:54.256 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:54.256 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:54.256 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 13:52:54.256 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:54.256 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:54.256 | INFO | Uploaded 36GiB to GPU in 2.15s (16.7442GiB/s) | 20260505 13:52:54.256 | INFO | Downloaded 34GiB to GPU in 2.18395s (15.5681GiB/s) | 20260505 13:52:59.388 | INFO | Analyzing chunk frames = [3714560,3715072) | 20260505 13:52:59.388 | INFO | MemInfo: { | 20260505 13:52:59.388 | INFO | "gpu" : " 17 GB", | 20260505 13:52:59.388 | INFO | "huge" : "", | 20260505 13:52:59.388 | INFO | "huge_shared" : " 27 GB", | 20260505 13:52:59.388 | INFO | "malloc" : " 14 GB", | 20260505 13:52:59.388 | INFO | "pinned" : " 29 GB" | 20260505 13:52:59.388 | INFO | } | 20260505 13:52:59.388 | INFO | Perf report for the last 5.13206 seconds: | 20260505 13:52:59.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:52:59.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:59.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:59.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:59.388 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 13:52:59.388 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:59.388 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:59.388 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 13:52:59.388 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:59.388 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:52:59.388 | INFO | Uploaded 36GiB to GPU in 2.14701s (16.7675GiB/s) | 20260505 13:52:59.389 | INFO | Downloaded 34GiB to GPU in 2.03646s (16.6957GiB/s) | 20260505 13:52:59.526 | INFO | Starting a round of Baz IO | 20260505 13:53:02.752 | INFO | Populating RTMetrics report | 20260505 13:53:02.752 | INFO | RTMetrics report populated | 20260505 13:53:02.752 | INFO | Opening tmp RTMetrics json file | 20260505 13:53:02.771 | INFO | Serializing RTMetrics to json | 20260505 13:53:02.789 | INFO | Serialization done, swapping tmp file now... | 20260505 13:53:02.795 | INFO | RTMetrics done | 20260505 13:53:02.859 | INFO | Finished a round of Baz IO: Wrote 2.283 GiB in 3.332 seconds | 20260505 13:53:02.859 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.95% / 0.00% / 77.04% | 20260505 13:53:02.859 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.639387% / 3.97364e-06% / 5.5631e-05% | 20260505 13:53:02.859 | INFO | Baz IO overall: | 20260505 13:53:02.859 | INFO | BytesWritten : 2213456476592 | 20260505 13:53:02.859 | INFO | NumEvents : 1052787789951 | 20260505 13:53:04.523 | INFO | Analyzing chunk frames = [3715072,3715584) | 20260505 13:53:04.523 | INFO | MemInfo: { | 20260505 13:53:04.523 | INFO | "gpu" : " 17 GB", | 20260505 13:53:04.523 | INFO | "huge" : "", | 20260505 13:53:04.523 | INFO | "huge_shared" : " 27 GB", | 20260505 13:53:04.523 | INFO | "malloc" : " 14 GB", | 20260505 13:53:04.523 | INFO | "pinned" : " 29 GB" | 20260505 13:53:04.523 | INFO | } | 20260505 13:53:04.523 | INFO | Perf report for the last 5.13479 seconds: | 20260505 13:53:04.523 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:53:04.523 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:04.523 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:04.523 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:04.523 | INFO | ANALYSIS : 64.4%, 3.0, 99.8ms, 0 | 20260505 13:53:04.523 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:04.523 | INFO | RT_METRICS : 46.1%, 1.0, 24.1ms, 0 | 20260505 13:53:04.523 | INFO | PRE_HQ : 61.2%, 1.8, 58.0ms, 0 | 20260505 13:53:04.523 | INFO | BAZWRITER : 64.2%, 4.0, 132.8ms, 0 | 20260505 13:53:04.523 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 13:53:04.523 | INFO | Uploaded 36GiB to GPU in 2.17259s (16.57GiB/s) | 20260505 13:53:04.523 | INFO | Downloaded 38GiB to GPU in 2.27665s (16.6912GiB/s) | 20260505 13:53:09.661 | INFO | Analyzing chunk frames = [3715584,3716096) | 20260505 13:53:09.661 | INFO | MemInfo: { | 20260505 13:53:09.661 | INFO | "gpu" : " 17 GB", | 20260505 13:53:09.661 | INFO | "huge" : "", | 20260505 13:53:09.661 | INFO | "huge_shared" : " 27 GB", | 20260505 13:53:09.661 | INFO | "malloc" : " 14 GB", | 20260505 13:53:09.661 | INFO | "pinned" : " 29 GB" | 20260505 13:53:09.661 | INFO | } | 20260505 13:53:09.661 | INFO | Perf report for the last 5.13831 seconds: | 20260505 13:53:09.661 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:53:09.661 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:09.661 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:09.661 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:09.661 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 13:53:09.661 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:09.661 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:09.661 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:53:09.662 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:09.662 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:09.662 | INFO | Uploaded 36GiB to GPU in 2.17661s (16.5394GiB/s) | 20260505 13:53:09.662 | INFO | Downloaded 34GiB to GPU in 2.03774s (16.6851GiB/s) | 20260505 13:53:14.807 | INFO | Analyzing chunk frames = [3716096,3716608) | 20260505 13:53:14.807 | INFO | MemInfo: { | 20260505 13:53:14.807 | INFO | "gpu" : " 17 GB", | 20260505 13:53:14.807 | INFO | "huge" : "", | 20260505 13:53:14.807 | INFO | "huge_shared" : " 27 GB", | 20260505 13:53:14.807 | INFO | "malloc" : " 14 GB", | 20260505 13:53:14.807 | INFO | "pinned" : " 29 GB" | 20260505 13:53:14.807 | INFO | } | 20260505 13:53:14.807 | INFO | Perf report for the last 5.14579 seconds: | 20260505 13:53:14.807 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:53:14.807 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:14.807 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:14.807 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:14.807 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 13:53:14.807 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:14.807 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:14.807 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:53:14.807 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:14.807 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:14.807 | INFO | Uploaded 36GiB to GPU in 2.22034s (16.2137GiB/s) | 20260505 13:53:14.807 | INFO | Downloaded 34GiB to GPU in 2.11507s (16.0751GiB/s) | 20260505 13:53:19.946 | INFO | Analyzing chunk frames = [3716608,3717120) | 20260505 13:53:19.946 | INFO | MemInfo: { | 20260505 13:53:19.946 | INFO | "gpu" : " 17 GB", | 20260505 13:53:19.946 | INFO | "huge" : "", | 20260505 13:53:19.946 | INFO | "huge_shared" : " 27 GB", | 20260505 13:53:19.946 | INFO | "malloc" : " 14 GB", | 20260505 13:53:19.946 | INFO | "pinned" : " 29 GB" | 20260505 13:53:19.946 | INFO | } | 20260505 13:53:19.946 | INFO | Perf report for the last 5.1392 seconds: | 20260505 13:53:19.946 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:53:19.946 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:19.946 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:19.946 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:19.946 | INFO | ANALYSIS : 62.7%, 3.0, 97.5ms, 0 | 20260505 13:53:19.946 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:19.946 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:19.946 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 13:53:19.947 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:19.947 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:19.947 | INFO | Uploaded 36GiB to GPU in 2.185s (16.476GiB/s) | 20260505 13:53:19.947 | INFO | Downloaded 34GiB to GPU in 2.13234s (15.9449GiB/s) | 20260505 13:53:25.078 | INFO | Analyzing chunk frames = [3717120,3717632) | 20260505 13:53:25.078 | INFO | MemInfo: { | 20260505 13:53:25.078 | INFO | "gpu" : " 17 GB", | 20260505 13:53:25.078 | INFO | "huge" : "", | 20260505 13:53:25.078 | INFO | "huge_shared" : " 27 GB", | 20260505 13:53:25.078 | INFO | "malloc" : " 14 GB", | 20260505 13:53:25.078 | INFO | "pinned" : " 29 GB" | 20260505 13:53:25.078 | INFO | } | 20260505 13:53:25.078 | INFO | Perf report for the last 5.1313 seconds: | 20260505 13:53:25.078 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:53:25.078 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:25.078 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:25.078 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:25.078 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260505 13:53:25.078 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:25.078 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:25.078 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 13:53:25.078 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:25.078 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:25.078 | INFO | Uploaded 36GiB to GPU in 2.14846s (16.7562GiB/s) | 20260505 13:53:25.078 | INFO | Downloaded 34GiB to GPU in 2.12395s (16.0079GiB/s) | 20260505 13:53:30.215 | INFO | Analyzing chunk frames = [3717632,3718144) | 20260505 13:53:30.215 | INFO | MemInfo: { | 20260505 13:53:30.215 | INFO | "gpu" : " 17 GB", | 20260505 13:53:30.215 | INFO | "huge" : "", | 20260505 13:53:30.215 | INFO | "huge_shared" : " 27 GB", | 20260505 13:53:30.215 | INFO | "malloc" : " 14 GB", | 20260505 13:53:30.215 | INFO | "pinned" : " 29 GB" | 20260505 13:53:30.215 | INFO | } | 20260505 13:53:30.215 | INFO | Perf report for the last 5.13749 seconds: | 20260505 13:53:30.215 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:53:30.215 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:30.215 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:30.215 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:30.215 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 13:53:30.215 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:30.215 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:30.215 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:53:30.215 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:30.215 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:30.215 | INFO | Uploaded 36GiB to GPU in 2.15941s (16.6712GiB/s) | 20260505 13:53:30.215 | INFO | Downloaded 34GiB to GPU in 2.11319s (16.0894GiB/s) | 20260505 13:53:35.358 | INFO | Analyzing chunk frames = [3718144,3718656) | 20260505 13:53:35.358 | INFO | MemInfo: { | 20260505 13:53:35.358 | INFO | "gpu" : " 17 GB", | 20260505 13:53:35.358 | INFO | "huge" : "", | 20260505 13:53:35.358 | INFO | "huge_shared" : " 27 GB", | 20260505 13:53:35.358 | INFO | "malloc" : " 14 GB", | 20260505 13:53:35.358 | INFO | "pinned" : " 29 GB" | 20260505 13:53:35.358 | INFO | } | 20260505 13:53:35.358 | INFO | Perf report for the last 5.14322 seconds: | 20260505 13:53:35.358 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:53:35.358 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:35.358 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:35.358 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:35.358 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 13:53:35.358 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:35.358 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:35.358 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 13:53:35.359 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:35.359 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:35.359 | INFO | Uploaded 36GiB to GPU in 2.16983s (16.5912GiB/s) | 20260505 13:53:35.359 | INFO | Downloaded 34GiB to GPU in 2.13955s (15.8912GiB/s) | 20260505 13:53:40.501 | INFO | Analyzing chunk frames = [3718656,3719168) | 20260505 13:53:40.501 | INFO | MemInfo: { | 20260505 13:53:40.501 | INFO | "gpu" : " 17 GB", | 20260505 13:53:40.501 | INFO | "huge" : "", | 20260505 13:53:40.502 | INFO | "huge_shared" : " 27 GB", | 20260505 13:53:40.502 | INFO | "malloc" : " 14 GB", | 20260505 13:53:40.502 | INFO | "pinned" : " 29 GB" | 20260505 13:53:40.502 | INFO | } | 20260505 13:53:40.502 | INFO | Perf report for the last 5.14316 seconds: | 20260505 13:53:40.502 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:53:40.502 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:40.502 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:40.502 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:40.502 | INFO | ANALYSIS : 63.5%, 3.0, 99.3ms, 0 | 20260505 13:53:40.502 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:40.502 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:40.502 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 13:53:40.502 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:40.502 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:40.502 | INFO | Uploaded 36GiB to GPU in 2.21473s (16.2548GiB/s) | 20260505 13:53:40.502 | INFO | Downloaded 34GiB to GPU in 2.10423s (16.1579GiB/s) | 20260505 13:53:40.502 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 13:53:40.502 | INFO | Total mem usage: 15.3 GiB | 20260505 13:53:40.502 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 13:53:40.502 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -7.03%) | 20260505 13:53:40.502 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 13:53:40.502 | INFO | High Water Breakdown: | 20260505 13:53:40.502 | INFO | 61.1% : 8.75 GiB ( 92% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 13:53:40.502 | INFO | 38.7% : 5.55 GiB ( 83.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 13:53:40.502 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 13:53:40.502 | INFO | ------------------------------------------ | 20260505 13:53:40.502 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:53:40.502 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 13:53:40.502 | INFO | Total mem usage: 26.5 GiB | 20260505 13:53:40.502 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 13:53:40.502 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 13:53:40.502 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 13:53:40.502 | INFO | High Water Breakdown: | 20260505 13:53:40.502 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:53:40.502 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:53:40.502 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:53:40.502 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:53:40.502 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:53:40.502 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:53:40.502 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 13:53:40.502 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 13:53:40.502 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:53:40.502 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 13:53:40.502 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 13:53:40.502 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 13:53:40.502 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 13:53:40.502 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 13:53:40.502 | INFO | ------------------------------------------ | 20260505 13:53:40.502 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:53:40.502 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 13:53:40.502 | INFO | Total mem usage: 25.6 GiB | 20260505 13:53:40.502 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 13:53:40.502 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 13:53:40.502 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 13:53:40.502 | INFO | High Water Breakdown: | 20260505 13:53:40.502 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 13:53:40.502 | INFO | ------------------------------------------ | 20260505 13:53:40.502 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:53:40.502 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 13:53:40.502 | INFO | Total mem usage: 15.7 GiB | 20260505 13:53:40.502 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 13:53:40.502 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 13:53:40.502 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 13:53:40.502 | INFO | High Water Breakdown: | 20260505 13:53:40.502 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 13:53:40.502 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 13:53:40.502 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 13:53:40.502 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 13:53:40.502 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 13:53:40.502 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 13:53:40.502 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 13:53:40.502 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 13:53:40.502 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 13:53:40.502 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:53:40.502 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 13:53:40.502 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:53:40.502 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 13:53:40.502 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 13:53:40.502 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 13:53:40.502 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 13:53:40.502 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:53:40.502 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 13:53:40.502 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:53:40.502 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:53:40.502 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:53:40.503 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 13:53:40.503 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 13:53:40.503 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:53:40.503 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 13:53:40.503 | INFO | ------------------------------------------ | 20260505 13:53:40.503 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:53:40.503 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 13:53:40.503 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:53:40.503 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:53:40.503 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.64e+01, 1.38e+01 / 1.72e+00 | 20260505 13:53:40.503 | INFO | PerfReport: 0.29%, AggregateStats -- 9.31e-02 / 1.01e-01, 1.08e-02 / 3.74e-03 | 20260505 13:53:40.503 | INFO | PerfReport: 11.78%, Baseline -- 3.76e+00 / 4.47e+00, 5.96e-01 / 1.80e-01 | 20260505 13:53:40.503 | INFO | PerfReport: 3.31%, Histogram -- 1.06e+00 / 1.49e+00, 2.03e-01 / 1.05e-01 | 20260505 13:53:40.503 | INFO | PerfReport: 15.97%, DME -- 4.08e+01 / 4.57e+01, 4.39e+00 / 2.13e+00 | 20260505 13:53:40.503 | INFO | PerfReport: 56.43%, FrameLabeling -- 1.80e+01 / 1.98e+01, 1.47e+00 / 1.83e+00 | 20260505 13:53:40.503 | INFO | PerfReport: 7.42%, PulseAccumulating -- 2.37e+00 / 2.92e+00, 4.53e-01 / 2.17e-01 | 20260505 13:53:40.503 | INFO | PerfReport: 3.55%, Metrics -- 1.13e+00 / 1.45e+00, 3.17e-01 / 1.52e-01 | 20260505 13:53:40.503 | INFO | PerfReport: 1.25%, PulseSerializing -- 3.98e-01 / 4.48e-01, 7.08e-02 / 2.84e-02 | 20260505 13:53:40.503 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 13:53:40.503 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:53:40.503 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:53:40.503 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260505 13:53:40.503 | INFO | PerfReport: 22.46%, Upload -- 2.26e+01 / 0.00e+00, 7.95e+00 / 0.00e+00 | 20260505 13:53:40.503 | INFO | PerfReport: 21.96%, Download -- 2.21e+01 / 0.00e+00, 8.11e+00 / 0.00e+00 | 20260505 13:53:40.503 | INFO | PerfReport: 31.67%, Compute -- 3.19e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 13:53:40.503 | INFO | PerfReport: 2.15%, LockUpload -- 2.16e+00 / 0.00e+00, 7.87e+00 / 0.00e+00 | 20260505 13:53:40.503 | INFO | PerfReport: 1.51%, LockDownload -- 1.52e+00 / 0.00e+00, 3.11e+00 / 0.00e+00 | 20260505 13:53:40.503 | INFO | PerfReport: 20.24%, LockCompute -- 2.04e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 13:53:40.503 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.56e-02 / 0.00e+00 | 20260505 13:53:40.637 | INFO | Starting a round of Baz IO | 20260505 13:53:43.861 | INFO | Populating RTMetrics report | 20260505 13:53:43.861 | INFO | RTMetrics report populated | 20260505 13:53:43.861 | INFO | Opening tmp RTMetrics json file | 20260505 13:53:43.882 | INFO | Serializing RTMetrics to json | 20260505 13:53:43.883 | INFO | Serialization done, swapping tmp file now... | 20260505 13:53:43.883 | INFO | RTMetrics done | 20260505 13:53:43.969 | INFO | Finished a round of Baz IO: Wrote 2.284 GiB in 3.331 seconds | 20260505 13:53:43.969 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.95% / 0.00% / 77.05% | 20260505 13:53:43.969 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.637301% / 7.94729e-06% / 6.35783e-05% | 20260505 13:53:43.969 | INFO | Baz IO overall: | 20260505 13:53:43.969 | INFO | BytesWritten : 2215908441436 | 20260505 13:53:43.969 | INFO | NumEvents : 1053946929436 | 20260505 13:53:45.643 | INFO | Analyzing chunk frames = [3719168,3719680) | 20260505 13:53:45.643 | INFO | MemInfo: { | 20260505 13:53:45.643 | INFO | "gpu" : " 17 GB", | 20260505 13:53:45.643 | INFO | "huge" : "", | 20260505 13:53:45.643 | INFO | "huge_shared" : " 27 GB", | 20260505 13:53:45.643 | INFO | "malloc" : " 14 GB", | 20260505 13:53:45.643 | INFO | "pinned" : " 29 GB" | 20260505 13:53:45.643 | INFO | } | 20260505 13:53:45.643 | INFO | Perf report for the last 5.1416 seconds: | 20260505 13:53:45.643 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:53:45.643 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:45.643 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:45.643 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:45.643 | INFO | ANALYSIS : 64.2%, 3.0, 99.6ms, 0 | 20260505 13:53:45.643 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:45.643 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260505 13:53:45.643 | INFO | PRE_HQ : 60.9%, 1.8, 57.5ms, 0 | 20260505 13:53:45.643 | INFO | BAZWRITER : 64.1%, 4.0, 132.3ms, 0 | 20260505 13:53:45.643 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 13:53:45.643 | INFO | Uploaded 36GiB to GPU in 2.1984s (16.3755GiB/s) | 20260505 13:53:45.644 | INFO | Downloaded 38GiB to GPU in 2.26972s (16.7422GiB/s) | 20260505 13:53:50.770 | INFO | Analyzing chunk frames = [3719680,3720192) | 20260505 13:53:50.770 | INFO | MemInfo: { | 20260505 13:53:50.770 | INFO | "gpu" : " 17 GB", | 20260505 13:53:50.770 | INFO | "huge" : "", | 20260505 13:53:50.770 | INFO | "huge_shared" : " 27 GB", | 20260505 13:53:50.770 | INFO | "malloc" : " 14 GB", | 20260505 13:53:50.770 | INFO | "pinned" : " 29 GB" | 20260505 13:53:50.770 | INFO | } | 20260505 13:53:50.770 | INFO | Perf report for the last 5.12665 seconds: | 20260505 13:53:50.770 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:53:50.770 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:50.770 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:50.770 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:50.770 | INFO | ANALYSIS : 63.6%, 3.0, 99.1ms, 0 | 20260505 13:53:50.770 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:50.770 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:50.770 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:53:50.770 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:50.770 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:50.770 | INFO | Uploaded 36GiB to GPU in 2.19056s (16.4341GiB/s) | 20260505 13:53:50.770 | INFO | Downloaded 34GiB to GPU in 2.03683s (16.6926GiB/s) | 20260505 13:53:55.911 | INFO | Analyzing chunk frames = [3720192,3720704) | 20260505 13:53:55.911 | INFO | MemInfo: { | 20260505 13:53:55.911 | INFO | "gpu" : " 17 GB", | 20260505 13:53:55.911 | INFO | "huge" : "", | 20260505 13:53:55.911 | INFO | "huge_shared" : " 27 GB", | 20260505 13:53:55.911 | INFO | "malloc" : " 14 GB", | 20260505 13:53:55.911 | INFO | "pinned" : " 29 GB" | 20260505 13:53:55.911 | INFO | } | 20260505 13:53:55.911 | INFO | Perf report for the last 5.14137 seconds: | 20260505 13:53:55.911 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:53:55.911 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:55.911 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:55.911 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:55.911 | INFO | ANALYSIS : 63.1%, 3.0, 98.2ms, 0 | 20260505 13:53:55.911 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:55.911 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:55.911 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 13:53:55.911 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:55.911 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:53:55.912 | INFO | Uploaded 36GiB to GPU in 2.2075s (16.3081GiB/s) | 20260505 13:53:55.912 | INFO | Downloaded 34GiB to GPU in 2.11988s (16.0386GiB/s) | 20260505 13:54:01.060 | INFO | Analyzing chunk frames = [3720704,3721216) | 20260505 13:54:01.061 | INFO | MemInfo: { | 20260505 13:54:01.061 | INFO | "gpu" : " 17 GB", | 20260505 13:54:01.061 | INFO | "huge" : "", | 20260505 13:54:01.061 | INFO | "huge_shared" : " 27 GB", | 20260505 13:54:01.061 | INFO | "malloc" : " 14 GB", | 20260505 13:54:01.061 | INFO | "pinned" : " 29 GB" | 20260505 13:54:01.061 | INFO | } | 20260505 13:54:01.061 | INFO | Perf report for the last 5.14943 seconds: | 20260505 13:54:01.061 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:54:01.061 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:01.061 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:01.061 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:01.061 | INFO | ANALYSIS : 62.9%, 3.0, 98.0ms, 0 | 20260505 13:54:01.061 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:01.061 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:01.061 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 13:54:01.061 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:01.061 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:01.061 | INFO | Uploaded 36GiB to GPU in 2.20523s (16.3248GiB/s) | 20260505 13:54:01.061 | INFO | Downloaded 34GiB to GPU in 2.13742s (15.9071GiB/s) | 20260505 13:54:06.209 | INFO | Analyzing chunk frames = [3721216,3721728) | 20260505 13:54:06.209 | INFO | MemInfo: { | 20260505 13:54:06.209 | INFO | "gpu" : " 17 GB", | 20260505 13:54:06.209 | INFO | "huge" : "", | 20260505 13:54:06.209 | INFO | "huge_shared" : " 27 GB", | 20260505 13:54:06.209 | INFO | "malloc" : " 14 GB", | 20260505 13:54:06.209 | INFO | "pinned" : " 29 GB" | 20260505 13:54:06.209 | INFO | } | 20260505 13:54:06.209 | INFO | Perf report for the last 5.1487 seconds: | 20260505 13:54:06.209 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:54:06.209 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:06.209 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:06.209 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:06.209 | INFO | ANALYSIS : 62.5%, 3.0, 97.8ms, 0 | 20260505 13:54:06.209 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:06.209 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:06.209 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 13:54:06.209 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:06.209 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:06.210 | INFO | Uploaded 36GiB to GPU in 2.16464s (16.6309GiB/s) | 20260505 13:54:06.210 | INFO | Downloaded 34GiB to GPU in 2.13008s (15.9618GiB/s) | 20260505 13:54:11.334 | INFO | Analyzing chunk frames = [3721728,3722240) | 20260505 13:54:11.334 | INFO | MemInfo: { | 20260505 13:54:11.334 | INFO | "gpu" : " 17 GB", | 20260505 13:54:11.334 | INFO | "huge" : "", | 20260505 13:54:11.334 | INFO | "huge_shared" : " 27 GB", | 20260505 13:54:11.334 | INFO | "malloc" : " 14 GB", | 20260505 13:54:11.334 | INFO | "pinned" : " 29 GB" | 20260505 13:54:11.334 | INFO | } | 20260505 13:54:11.334 | INFO | Perf report for the last 5.12437 seconds: | 20260505 13:54:11.334 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:54:11.334 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:11.334 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:11.334 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:11.334 | INFO | ANALYSIS : 63.0%, 3.0, 98.0ms, 0 | 20260505 13:54:11.334 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:11.334 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:11.334 | INFO | PRE_HQ : 28.3%, 1.0, 14.6ms, 0 | 20260505 13:54:11.334 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:11.334 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:11.334 | INFO | Uploaded 36GiB to GPU in 2.1539s (16.7139GiB/s) | 20260505 13:54:11.334 | INFO | Downloaded 34GiB to GPU in 2.04011s (16.6658GiB/s) | 20260505 13:54:16.469 | INFO | Analyzing chunk frames = [3722240,3722752) | 20260505 13:54:16.469 | INFO | MemInfo: { | 20260505 13:54:16.469 | INFO | "gpu" : " 17 GB", | 20260505 13:54:16.469 | INFO | "huge" : "", | 20260505 13:54:16.469 | INFO | "huge_shared" : " 27 GB", | 20260505 13:54:16.469 | INFO | "malloc" : " 14 GB", | 20260505 13:54:16.469 | INFO | "pinned" : " 29 GB" | 20260505 13:54:16.469 | INFO | } | 20260505 13:54:16.469 | INFO | Perf report for the last 5.13548 seconds: | 20260505 13:54:16.469 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:54:16.469 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:16.469 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:16.469 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:16.469 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 13:54:16.469 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:16.469 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:16.469 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 13:54:16.469 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:16.469 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:16.470 | INFO | Uploaded 36GiB to GPU in 2.16948s (16.5938GiB/s) | 20260505 13:54:16.470 | INFO | Downloaded 34GiB to GPU in 2.1512s (15.8051GiB/s) | 20260505 13:54:21.614 | INFO | Analyzing chunk frames = [3722752,3723264) | 20260505 13:54:21.614 | INFO | MemInfo: { | 20260505 13:54:21.614 | INFO | "gpu" : " 17 GB", | 20260505 13:54:21.614 | INFO | "huge" : "", | 20260505 13:54:21.614 | INFO | "huge_shared" : " 27 GB", | 20260505 13:54:21.614 | INFO | "malloc" : " 14 GB", | 20260505 13:54:21.614 | INFO | "pinned" : " 29 GB" | 20260505 13:54:21.614 | INFO | } | 20260505 13:54:21.614 | INFO | Perf report for the last 5.14454 seconds: | 20260505 13:54:21.614 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:54:21.614 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:21.614 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:21.614 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:21.614 | INFO | ANALYSIS : 63.7%, 3.0, 99.6ms, 0 | 20260505 13:54:21.614 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:21.614 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:21.614 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 13:54:21.614 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:21.614 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:21.614 | INFO | Uploaded 36GiB to GPU in 2.1843s (16.4812GiB/s) | 20260505 13:54:21.614 | INFO | Downloaded 34GiB to GPU in 2.11337s (16.088GiB/s) | 20260505 13:54:21.752 | INFO | Starting a round of Baz IO | 20260505 13:54:24.971 | INFO | Populating RTMetrics report | 20260505 13:54:24.972 | INFO | RTMetrics report populated | 20260505 13:54:24.972 | INFO | Opening tmp RTMetrics json file | 20260505 13:54:25.017 | INFO | Serializing RTMetrics to json | 20260505 13:54:25.060 | INFO | Serialization done, swapping tmp file now... | 20260505 13:54:25.061 | INFO | RTMetrics done | 20260505 13:54:25.073 | INFO | Finished a round of Baz IO: Wrote 2.284 GiB in 3.321 seconds | 20260505 13:54:25.073 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.94% / 0.00% / 77.05% | 20260505 13:54:25.073 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.635139% / 7.94729e-06% / 6.75519e-05% | 20260505 13:54:25.073 | INFO | Baz IO overall: | 20260505 13:54:25.073 | INFO | BytesWritten : 2218361252190 | 20260505 13:54:25.073 | INFO | NumEvents : 1055106910195 | 20260505 13:54:26.758 | INFO | Analyzing chunk frames = [3723264,3723776) | 20260505 13:54:26.758 | INFO | MemInfo: { | 20260505 13:54:26.758 | INFO | "gpu" : " 17 GB", | 20260505 13:54:26.758 | INFO | "huge" : "", | 20260505 13:54:26.758 | INFO | "huge_shared" : " 27 GB", | 20260505 13:54:26.759 | INFO | "malloc" : " 14 GB", | 20260505 13:54:26.759 | INFO | "pinned" : " 29 GB" | 20260505 13:54:26.759 | INFO | } | 20260505 13:54:26.759 | INFO | Perf report for the last 5.14473 seconds: | 20260505 13:54:26.759 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:54:26.759 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:26.759 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:26.759 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:26.759 | INFO | ANALYSIS : 64.2%, 3.0, 99.6ms, 0 | 20260505 13:54:26.759 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:26.759 | INFO | RT_METRICS : 46.7%, 1.0, 24.5ms, 0 | 20260505 13:54:26.759 | INFO | PRE_HQ : 60.8%, 1.8, 57.4ms, 0 | 20260505 13:54:26.759 | INFO | BAZWRITER : 63.9%, 4.0, 132.7ms, 0 | 20260505 13:54:26.759 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 13:54:26.759 | INFO | Uploaded 36GiB to GPU in 2.20175s (16.3507GiB/s) | 20260505 13:54:26.759 | INFO | Downloaded 38GiB to GPU in 2.26589s (16.7705GiB/s) | 20260505 13:54:31.894 | INFO | Analyzing chunk frames = [3723776,3724288) | 20260505 13:54:31.894 | INFO | MemInfo: { | 20260505 13:54:31.894 | INFO | "gpu" : " 17 GB", | 20260505 13:54:31.894 | INFO | "huge" : "", | 20260505 13:54:31.894 | INFO | "huge_shared" : " 27 GB", | 20260505 13:54:31.894 | INFO | "malloc" : " 14 GB", | 20260505 13:54:31.894 | INFO | "pinned" : " 29 GB" | 20260505 13:54:31.894 | INFO | } | 20260505 13:54:31.894 | INFO | Perf report for the last 5.13536 seconds: | 20260505 13:54:31.894 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:54:31.894 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:31.894 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:31.894 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:31.894 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 13:54:31.894 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:31.894 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:31.894 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:54:31.894 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:31.894 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:31.894 | INFO | Uploaded 36GiB to GPU in 2.19475s (16.4028GiB/s) | 20260505 13:54:31.894 | INFO | Downloaded 34GiB to GPU in 2.03877s (16.6767GiB/s) | 20260505 13:54:37.024 | INFO | Analyzing chunk frames = [3724288,3724800) | 20260505 13:54:37.024 | INFO | MemInfo: { | 20260505 13:54:37.024 | INFO | "gpu" : " 17 GB", | 20260505 13:54:37.024 | INFO | "huge" : "", | 20260505 13:54:37.024 | INFO | "huge_shared" : " 27 GB", | 20260505 13:54:37.024 | INFO | "malloc" : " 14 GB", | 20260505 13:54:37.024 | INFO | "pinned" : " 29 GB" | 20260505 13:54:37.024 | INFO | } | 20260505 13:54:37.024 | INFO | Perf report for the last 5.12996 seconds: | 20260505 13:54:37.024 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:54:37.024 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:37.024 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:37.024 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:37.024 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260505 13:54:37.024 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:37.024 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:37.024 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:54:37.024 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:37.024 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:37.024 | INFO | Uploaded 36GiB to GPU in 2.23408s (16.114GiB/s) | 20260505 13:54:37.024 | INFO | Downloaded 34GiB to GPU in 2.10198s (16.1752GiB/s) | 20260505 13:54:42.169 | INFO | Analyzing chunk frames = [3724800,3725312) | 20260505 13:54:42.169 | INFO | MemInfo: { | 20260505 13:54:42.169 | INFO | "gpu" : " 17 GB", | 20260505 13:54:42.169 | INFO | "huge" : "", | 20260505 13:54:42.169 | INFO | "huge_shared" : " 27 GB", | 20260505 13:54:42.169 | INFO | "malloc" : " 14 GB", | 20260505 13:54:42.169 | INFO | "pinned" : " 29 GB" | 20260505 13:54:42.169 | INFO | } | 20260505 13:54:42.169 | INFO | Perf report for the last 5.14497 seconds: | 20260505 13:54:42.169 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:54:42.169 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:42.169 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:42.169 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:42.169 | INFO | ANALYSIS : 63.1%, 3.0, 98.2ms, 0 | 20260505 13:54:42.169 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:42.169 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:42.169 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 13:54:42.169 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:42.169 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:42.169 | INFO | Uploaded 36GiB to GPU in 2.21329s (16.2654GiB/s) | 20260505 13:54:42.169 | INFO | Downloaded 34GiB to GPU in 2.13643s (15.9144GiB/s) | 20260505 13:54:47.311 | INFO | Analyzing chunk frames = [3725312,3725824) | 20260505 13:54:47.311 | INFO | MemInfo: { | 20260505 13:54:47.311 | INFO | "gpu" : " 17 GB", | 20260505 13:54:47.311 | INFO | "huge" : "", | 20260505 13:54:47.311 | INFO | "huge_shared" : " 27 GB", | 20260505 13:54:47.311 | INFO | "malloc" : " 14 GB", | 20260505 13:54:47.311 | INFO | "pinned" : " 29 GB" | 20260505 13:54:47.311 | INFO | } | 20260505 13:54:47.311 | INFO | Perf report for the last 5.14226 seconds: | 20260505 13:54:47.311 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:54:47.311 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:47.311 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:47.311 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:47.311 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 13:54:47.311 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:47.311 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:47.311 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 13:54:47.311 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:47.311 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:47.311 | INFO | Uploaded 36GiB to GPU in 2.15588s (16.6985GiB/s) | 20260505 13:54:47.312 | INFO | Downloaded 34GiB to GPU in 2.12749s (15.9813GiB/s) | 20260505 13:54:52.456 | INFO | Analyzing chunk frames = [3725824,3726336) | 20260505 13:54:52.456 | INFO | MemInfo: { | 20260505 13:54:52.456 | INFO | "gpu" : " 17 GB", | 20260505 13:54:52.456 | INFO | "huge" : "", | 20260505 13:54:52.456 | INFO | "huge_shared" : " 27 GB", | 20260505 13:54:52.456 | INFO | "malloc" : " 14 GB", | 20260505 13:54:52.456 | INFO | "pinned" : " 29 GB" | 20260505 13:54:52.456 | INFO | } | 20260505 13:54:52.456 | INFO | Perf report for the last 5.14463 seconds: | 20260505 13:54:52.456 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:54:52.456 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:52.456 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:52.456 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:52.456 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 13:54:52.456 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:52.456 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:52.456 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:54:52.456 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:52.456 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:52.456 | INFO | Uploaded 36GiB to GPU in 2.1744s (16.5563GiB/s) | 20260505 13:54:52.456 | INFO | Downloaded 34GiB to GPU in 2.10966s (16.1163GiB/s) | 20260505 13:54:57.589 | INFO | Analyzing chunk frames = [3726336,3726848) | 20260505 13:54:57.589 | INFO | MemInfo: { | 20260505 13:54:57.589 | INFO | "gpu" : " 17 GB", | 20260505 13:54:57.589 | INFO | "huge" : "", | 20260505 13:54:57.589 | INFO | "huge_shared" : " 27 GB", | 20260505 13:54:57.589 | INFO | "malloc" : " 14 GB", | 20260505 13:54:57.589 | INFO | "pinned" : " 29 GB" | 20260505 13:54:57.589 | INFO | } | 20260505 13:54:57.589 | INFO | Perf report for the last 5.13336 seconds: | 20260505 13:54:57.589 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:54:57.589 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:57.589 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:57.589 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:57.589 | INFO | ANALYSIS : 63.5%, 3.0, 99.0ms, 0 | 20260505 13:54:57.589 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:57.589 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:57.589 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 13:54:57.589 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:57.589 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:54:57.589 | INFO | Uploaded 36GiB to GPU in 2.17242s (16.5714GiB/s) | 20260505 13:54:57.589 | INFO | Downloaded 34GiB to GPU in 2.16013s (15.7398GiB/s) | 20260505 13:55:02.722 | INFO | Analyzing chunk frames = [3726848,3727360) | 20260505 13:55:02.722 | INFO | MemInfo: { | 20260505 13:55:02.722 | INFO | "gpu" : " 17 GB", | 20260505 13:55:02.722 | INFO | "huge" : "", | 20260505 13:55:02.722 | INFO | "huge_shared" : " 27 GB", | 20260505 13:55:02.722 | INFO | "malloc" : " 14 GB", | 20260505 13:55:02.722 | INFO | "pinned" : " 29 GB" | 20260505 13:55:02.722 | INFO | } | 20260505 13:55:02.722 | INFO | Perf report for the last 5.13262 seconds: | 20260505 13:55:02.722 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:55:02.722 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:02.722 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:02.722 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:02.722 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 13:55:02.722 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:02.722 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:02.722 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 13:55:02.722 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:02.722 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:02.722 | INFO | Uploaded 36GiB to GPU in 2.16177s (16.653GiB/s) | 20260505 13:55:02.722 | INFO | Downloaded 34GiB to GPU in 2.10475s (16.1539GiB/s) | 20260505 13:55:02.722 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 13:55:02.722 | INFO | Total mem usage: 15.3 GiB | 20260505 13:55:02.722 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 13:55:02.722 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.65%) | 20260505 13:55:02.722 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 13:55:02.722 | INFO | High Water Breakdown: | 20260505 13:55:02.722 | INFO | 61% : 8.68 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 13:55:02.722 | INFO | 38.9% : 5.54 GiB ( 83.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 13:55:02.722 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 13:55:02.722 | INFO | ------------------------------------------ | 20260505 13:55:02.722 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:55:02.722 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 13:55:02.722 | INFO | Total mem usage: 26.5 GiB | 20260505 13:55:02.722 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 13:55:02.722 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 13:55:02.722 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 13:55:02.722 | INFO | High Water Breakdown: | 20260505 13:55:02.722 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:55:02.722 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:55:02.722 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:55:02.722 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:55:02.722 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:55:02.722 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:55:02.723 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 13:55:02.723 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 13:55:02.723 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:55:02.723 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 13:55:02.723 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 13:55:02.723 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 13:55:02.723 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 13:55:02.723 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 13:55:02.723 | INFO | ------------------------------------------ | 20260505 13:55:02.723 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:55:02.723 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 13:55:02.723 | INFO | Total mem usage: 25.6 GiB | 20260505 13:55:02.723 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 13:55:02.723 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 13:55:02.723 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 13:55:02.723 | INFO | High Water Breakdown: | 20260505 13:55:02.723 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 13:55:02.723 | INFO | ------------------------------------------ | 20260505 13:55:02.723 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:55:02.723 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 13:55:02.723 | INFO | Total mem usage: 15.7 GiB | 20260505 13:55:02.723 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 13:55:02.723 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 13:55:02.723 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 13:55:02.723 | INFO | High Water Breakdown: | 20260505 13:55:02.723 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 13:55:02.723 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 13:55:02.723 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 13:55:02.723 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 13:55:02.723 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 13:55:02.723 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 13:55:02.723 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 13:55:02.723 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 13:55:02.723 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 13:55:02.723 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:55:02.723 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 13:55:02.723 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:55:02.723 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 13:55:02.723 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 13:55:02.723 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 13:55:02.723 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 13:55:02.723 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:55:02.723 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 13:55:02.723 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:55:02.723 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:55:02.723 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:55:02.723 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 13:55:02.723 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 13:55:02.723 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:55:02.723 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 13:55:02.723 | INFO | ------------------------------------------ | 20260505 13:55:02.723 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:55:02.723 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 13:55:02.723 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:55:02.723 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:55:02.723 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.64e+01, 1.36e+01 / 1.89e+00 | 20260505 13:55:02.723 | INFO | PerfReport: 0.29%, AggregateStats -- 9.27e-02 / 1.02e-01, 1.03e-02 / 5.03e-03 | 20260505 13:55:02.723 | INFO | PerfReport: 11.77%, Baseline -- 3.75e+00 / 4.48e+00, 5.94e-01 / 2.13e-01 | 20260505 13:55:02.723 | INFO | PerfReport: 3.30%, Histogram -- 1.05e+00 / 1.51e+00, 1.99e-01 / 8.93e-02 | 20260505 13:55:02.723 | INFO | PerfReport: 15.92%, DME -- 4.06e+01 / 4.54e+01, 4.47e+00 / 2.21e+00 | 20260505 13:55:02.723 | INFO | PerfReport: 56.48%, FrameLabeling -- 1.80e+01 / 2.01e+01, 1.49e+00 / 2.00e+00 | 20260505 13:55:02.723 | INFO | PerfReport: 7.41%, PulseAccumulating -- 2.36e+00 / 2.90e+00, 4.40e-01 / 2.07e-01 | 20260505 13:55:02.723 | INFO | PerfReport: 3.57%, Metrics -- 1.14e+00 / 1.47e+00, 3.25e-01 / 2.05e-01 | 20260505 13:55:02.723 | INFO | PerfReport: 1.25%, PulseSerializing -- 3.98e-01 / 4.47e-01, 7.10e-02 / 3.05e-02 | 20260505 13:55:02.723 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 13:55:02.723 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:55:02.723 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:55:02.723 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260505 13:55:02.723 | INFO | PerfReport: 22.61%, Upload -- 2.28e+01 / 0.00e+00, 8.11e+00 / 0.00e+00 | 20260505 13:55:02.723 | INFO | PerfReport: 22.00%, Download -- 2.22e+01 / 0.00e+00, 8.09e+00 / 0.00e+00 | 20260505 13:55:02.723 | INFO | PerfReport: 31.64%, Compute -- 3.19e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 13:55:02.723 | INFO | PerfReport: 2.17%, LockUpload -- 2.19e+00 / 0.00e+00, 7.85e+00 / 0.00e+00 | 20260505 13:55:02.723 | INFO | PerfReport: 1.38%, LockDownload -- 1.39e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260505 13:55:02.723 | INFO | PerfReport: 20.19%, LockCompute -- 2.03e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260505 13:55:02.723 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.65e-02 / 0.00e+00 | 20260505 13:55:02.853 | INFO | Starting a round of Baz IO | 20260505 13:55:06.094 | INFO | Populating RTMetrics report | 20260505 13:55:06.095 | INFO | RTMetrics report populated | 20260505 13:55:06.095 | INFO | Opening tmp RTMetrics json file | 20260505 13:55:06.098 | INFO | Serializing RTMetrics to json | 20260505 13:55:06.101 | INFO | Serialization done, swapping tmp file now... | 20260505 13:55:06.103 | INFO | RTMetrics done | 20260505 13:55:06.208 | INFO | Finished a round of Baz IO: Wrote 2.281 GiB in 3.355 seconds | 20260505 13:55:06.209 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.94% / 0.00% / 77.06% | 20260505 13:55:06.209 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.632886% / 3.97364e-06% / 7.94729e-05% | 20260505 13:55:06.209 | INFO | Baz IO overall: | 20260505 13:55:06.209 | INFO | BytesWritten : 2220809943336 | 20260505 13:55:06.209 | INFO | NumEvents : 1056263665240 | 20260505 13:55:07.866 | INFO | Analyzing chunk frames = [3727360,3727872) | 20260505 13:55:07.866 | INFO | MemInfo: { | 20260505 13:55:07.866 | INFO | "gpu" : " 17 GB", | 20260505 13:55:07.866 | INFO | "huge" : "", | 20260505 13:55:07.866 | INFO | "huge_shared" : " 27 GB", | 20260505 13:55:07.866 | INFO | "malloc" : " 14 GB", | 20260505 13:55:07.866 | INFO | "pinned" : " 29 GB" | 20260505 13:55:07.867 | INFO | } | 20260505 13:55:07.867 | INFO | Perf report for the last 5.14473 seconds: | 20260505 13:55:07.867 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:55:07.867 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:07.867 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:07.867 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:07.867 | INFO | ANALYSIS : 64.4%, 3.0, 100.0ms, 0 | 20260505 13:55:07.867 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:07.867 | INFO | RT_METRICS : 45.1%, 1.0, 23.7ms, 0 | 20260505 13:55:07.867 | INFO | PRE_HQ : 60.7%, 1.8, 57.3ms, 0 | 20260505 13:55:07.867 | INFO | BAZWRITER : 64.5%, 4.0, 132.2ms, 0 | 20260505 13:55:07.867 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 13:55:07.867 | INFO | Uploaded 36GiB to GPU in 2.18009s (16.5131GiB/s) | 20260505 13:55:07.867 | INFO | Downloaded 38GiB to GPU in 2.28929s (16.5991GiB/s) | 20260505 13:55:13.009 | INFO | Analyzing chunk frames = [3727872,3728384) | 20260505 13:55:13.009 | INFO | MemInfo: { | 20260505 13:55:13.009 | INFO | "gpu" : " 17 GB", | 20260505 13:55:13.009 | INFO | "huge" : "", | 20260505 13:55:13.009 | INFO | "huge_shared" : " 27 GB", | 20260505 13:55:13.009 | INFO | "malloc" : " 14 GB", | 20260505 13:55:13.009 | INFO | "pinned" : " 29 GB" | 20260505 13:55:13.009 | INFO | } | 20260505 13:55:13.009 | INFO | Perf report for the last 5.14282 seconds: | 20260505 13:55:13.009 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:55:13.009 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:13.009 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:13.009 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:13.009 | INFO | ANALYSIS : 63.5%, 3.0, 99.3ms, 0 | 20260505 13:55:13.009 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:13.009 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:13.010 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 13:55:13.010 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:13.010 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:13.010 | INFO | Uploaded 36GiB to GPU in 2.2029s (16.3421GiB/s) | 20260505 13:55:13.010 | INFO | Downloaded 34GiB to GPU in 2.03848s (16.6791GiB/s) | 20260505 13:55:18.151 | INFO | Analyzing chunk frames = [3728384,3728896) | 20260505 13:55:18.151 | INFO | MemInfo: { | 20260505 13:55:18.151 | INFO | "gpu" : " 17 GB", | 20260505 13:55:18.151 | INFO | "huge" : "", | 20260505 13:55:18.151 | INFO | "huge_shared" : " 27 GB", | 20260505 13:55:18.151 | INFO | "malloc" : " 14 GB", | 20260505 13:55:18.151 | INFO | "pinned" : " 29 GB" | 20260505 13:55:18.151 | INFO | } | 20260505 13:55:18.151 | INFO | Perf report for the last 5.14197 seconds: | 20260505 13:55:18.151 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:55:18.151 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:18.151 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:18.151 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:18.151 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 13:55:18.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:18.151 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:18.151 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 13:55:18.151 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:18.151 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:18.152 | INFO | Uploaded 36GiB to GPU in 2.23132s (16.1339GiB/s) | 20260505 13:55:18.152 | INFO | Downloaded 34GiB to GPU in 2.11328s (16.0887GiB/s) | 20260505 13:55:23.284 | INFO | Analyzing chunk frames = [3728896,3729408) | 20260505 13:55:23.284 | INFO | MemInfo: { | 20260505 13:55:23.284 | INFO | "gpu" : " 17 GB", | 20260505 13:55:23.284 | INFO | "huge" : "", | 20260505 13:55:23.284 | INFO | "huge_shared" : " 27 GB", | 20260505 13:55:23.284 | INFO | "malloc" : " 14 GB", | 20260505 13:55:23.284 | INFO | "pinned" : " 29 GB" | 20260505 13:55:23.284 | INFO | } | 20260505 13:55:23.284 | INFO | Perf report for the last 5.13283 seconds: | 20260505 13:55:23.284 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:55:23.284 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:23.284 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:23.284 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:23.284 | INFO | ANALYSIS : 62.9%, 3.0, 97.6ms, 0 | 20260505 13:55:23.284 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:23.284 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:23.284 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 13:55:23.284 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:23.284 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:23.284 | INFO | Uploaded 36GiB to GPU in 2.1876s (16.4564GiB/s) | 20260505 13:55:23.284 | INFO | Downloaded 34GiB to GPU in 2.1343s (15.9303GiB/s) | 20260505 13:55:28.423 | INFO | Analyzing chunk frames = [3729408,3729920) | 20260505 13:55:28.423 | INFO | MemInfo: { | 20260505 13:55:28.423 | INFO | "gpu" : " 17 GB", | 20260505 13:55:28.423 | INFO | "huge" : "", | 20260505 13:55:28.423 | INFO | "huge_shared" : " 27 GB", | 20260505 13:55:28.423 | INFO | "malloc" : " 14 GB", | 20260505 13:55:28.423 | INFO | "pinned" : " 29 GB" | 20260505 13:55:28.423 | INFO | } | 20260505 13:55:28.423 | INFO | Perf report for the last 5.13902 seconds: | 20260505 13:55:28.423 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:55:28.423 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:28.423 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:28.423 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:28.423 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 13:55:28.423 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:28.423 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:28.423 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 13:55:28.423 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:28.423 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:28.424 | INFO | Uploaded 36GiB to GPU in 2.1571s (16.6891GiB/s) | 20260505 13:55:28.424 | INFO | Downloaded 34GiB to GPU in 2.12626s (15.9905GiB/s) | 20260505 13:55:33.573 | INFO | Analyzing chunk frames = [3729920,3730432) | 20260505 13:55:33.573 | INFO | MemInfo: { | 20260505 13:55:33.573 | INFO | "gpu" : " 17 GB", | 20260505 13:55:33.573 | INFO | "huge" : "", | 20260505 13:55:33.573 | INFO | "huge_shared" : " 27 GB", | 20260505 13:55:33.573 | INFO | "malloc" : " 14 GB", | 20260505 13:55:33.573 | INFO | "pinned" : " 29 GB" | 20260505 13:55:33.573 | INFO | } | 20260505 13:55:33.573 | INFO | Perf report for the last 5.1499 seconds: | 20260505 13:55:33.573 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:55:33.573 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:33.573 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:33.573 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:33.573 | INFO | ANALYSIS : 62.8%, 3.0, 98.3ms, 0 | 20260505 13:55:33.573 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:33.573 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:33.573 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 13:55:33.573 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:33.573 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:33.573 | INFO | Uploaded 36GiB to GPU in 2.13346s (16.874GiB/s) | 20260505 13:55:33.573 | INFO | Downloaded 34GiB to GPU in 2.10532s (16.1496GiB/s) | 20260505 13:55:38.710 | INFO | Analyzing chunk frames = [3730432,3730944) | 20260505 13:55:38.710 | INFO | MemInfo: { | 20260505 13:55:38.710 | INFO | "gpu" : " 17 GB", | 20260505 13:55:38.710 | INFO | "huge" : "", | 20260505 13:55:38.710 | INFO | "huge_shared" : " 27 GB", | 20260505 13:55:38.710 | INFO | "malloc" : " 14 GB", | 20260505 13:55:38.710 | INFO | "pinned" : " 29 GB" | 20260505 13:55:38.710 | INFO | } | 20260505 13:55:38.711 | INFO | Perf report for the last 5.13734 seconds: | 20260505 13:55:38.711 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:55:38.711 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:38.711 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:38.711 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:38.711 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 13:55:38.711 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:38.711 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:38.711 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 13:55:38.711 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:38.711 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:38.711 | INFO | Uploaded 36GiB to GPU in 2.13342s (16.8743GiB/s) | 20260505 13:55:38.711 | INFO | Downloaded 34GiB to GPU in 2.1553s (15.775GiB/s) | 20260505 13:55:43.842 | INFO | Analyzing chunk frames = [3730944,3731456) | 20260505 13:55:43.842 | INFO | MemInfo: { | 20260505 13:55:43.842 | INFO | "gpu" : " 17 GB", | 20260505 13:55:43.842 | INFO | "huge" : "", | 20260505 13:55:43.842 | INFO | "huge_shared" : " 27 GB", | 20260505 13:55:43.842 | INFO | "malloc" : " 14 GB", | 20260505 13:55:43.842 | INFO | "pinned" : " 29 GB" | 20260505 13:55:43.842 | INFO | } | 20260505 13:55:43.842 | INFO | Perf report for the last 5.13168 seconds: | 20260505 13:55:43.842 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:55:43.842 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:43.842 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:43.842 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:43.842 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 13:55:43.842 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:43.842 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:43.842 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 13:55:43.842 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:43.842 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:43.842 | INFO | Uploaded 36GiB to GPU in 2.21318s (16.2662GiB/s) | 20260505 13:55:43.842 | INFO | Downloaded 34GiB to GPU in 2.10547s (16.1484GiB/s) | 20260505 13:55:43.974 | INFO | Starting a round of Baz IO | 20260505 13:55:47.217 | INFO | Populating RTMetrics report | 20260505 13:55:47.217 | INFO | RTMetrics report populated | 20260505 13:55:47.217 | INFO | Opening tmp RTMetrics json file | 20260505 13:55:47.259 | INFO | Serializing RTMetrics to json | 20260505 13:55:47.296 | INFO | Serialization done, swapping tmp file now... | 20260505 13:55:47.304 | INFO | RTMetrics done | 20260505 13:55:47.321 | INFO | Finished a round of Baz IO: Wrote 2.281 GiB in 3.346 seconds | 20260505 13:55:47.321 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.93% / 0.00% / 77.06% | 20260505 13:55:47.321 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.631416% / 0% / 8.34465e-05% | 20260505 13:55:47.321 | INFO | Baz IO overall: | 20260505 13:55:47.321 | INFO | BytesWritten : 2223259282747 | 20260505 13:55:47.321 | INFO | NumEvents : 1057421069823 | 20260505 13:55:48.979 | INFO | Analyzing chunk frames = [3731456,3731968) | 20260505 13:55:48.979 | INFO | MemInfo: { | 20260505 13:55:48.979 | INFO | "gpu" : " 17 GB", | 20260505 13:55:48.979 | INFO | "huge" : "", | 20260505 13:55:48.979 | INFO | "huge_shared" : " 27 GB", | 20260505 13:55:48.979 | INFO | "malloc" : " 14 GB", | 20260505 13:55:48.979 | INFO | "pinned" : " 29 GB" | 20260505 13:55:48.979 | INFO | } | 20260505 13:55:48.979 | INFO | Perf report for the last 5.13699 seconds: | 20260505 13:55:48.979 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:55:48.979 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:48.979 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:48.979 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:48.979 | INFO | ANALYSIS : 64.6%, 3.0, 100.4ms, 0 | 20260505 13:55:48.979 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:48.979 | INFO | RT_METRICS : 46.7%, 1.0, 24.5ms, 0 | 20260505 13:55:48.979 | INFO | PRE_HQ : 60.7%, 1.8, 57.6ms, 0 | 20260505 13:55:48.979 | INFO | BAZWRITER : 64.4%, 4.0, 131.4ms, 0 | 20260505 13:55:48.979 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 13:55:48.979 | INFO | Uploaded 36GiB to GPU in 2.17281s (16.5684GiB/s) | 20260505 13:55:48.979 | INFO | Downloaded 38GiB to GPU in 2.28352s (16.641GiB/s) | 20260505 13:55:54.123 | INFO | Analyzing chunk frames = [3731968,3732480) | 20260505 13:55:54.123 | INFO | MemInfo: { | 20260505 13:55:54.123 | INFO | "gpu" : " 17 GB", | 20260505 13:55:54.123 | INFO | "huge" : "", | 20260505 13:55:54.123 | INFO | "huge_shared" : " 27 GB", | 20260505 13:55:54.123 | INFO | "malloc" : " 14 GB", | 20260505 13:55:54.123 | INFO | "pinned" : " 29 GB" | 20260505 13:55:54.123 | INFO | } | 20260505 13:55:54.123 | INFO | Perf report for the last 5.1441 seconds: | 20260505 13:55:54.123 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:55:54.123 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:54.123 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:54.123 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:54.123 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 13:55:54.123 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:54.123 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:54.123 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 13:55:54.123 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:54.123 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:54.124 | INFO | Uploaded 36GiB to GPU in 2.1714s (16.5792GiB/s) | 20260505 13:55:54.124 | INFO | Downloaded 34GiB to GPU in 2.04204s (16.65GiB/s) | 20260505 13:55:59.264 | INFO | Analyzing chunk frames = [3732480,3732992) | 20260505 13:55:59.264 | INFO | MemInfo: { | 20260505 13:55:59.264 | INFO | "gpu" : " 17 GB", | 20260505 13:55:59.264 | INFO | "huge" : "", | 20260505 13:55:59.264 | INFO | "huge_shared" : " 27 GB", | 20260505 13:55:59.264 | INFO | "malloc" : " 14 GB", | 20260505 13:55:59.264 | INFO | "pinned" : " 29 GB" | 20260505 13:55:59.265 | INFO | } | 20260505 13:55:59.265 | INFO | Perf report for the last 5.14121 seconds: | 20260505 13:55:59.265 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:55:59.265 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:59.265 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:59.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:59.265 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 13:55:59.265 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:59.265 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:59.265 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 13:55:59.265 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:59.265 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:55:59.265 | INFO | Uploaded 36GiB to GPU in 2.2243s (16.1849GiB/s) | 20260505 13:55:59.265 | INFO | Downloaded 34GiB to GPU in 2.09915s (16.197GiB/s) | 20260505 13:56:04.406 | INFO | Analyzing chunk frames = [3732992,3733504) | 20260505 13:56:04.406 | INFO | MemInfo: { | 20260505 13:56:04.406 | INFO | "gpu" : " 17 GB", | 20260505 13:56:04.406 | INFO | "huge" : "", | 20260505 13:56:04.406 | INFO | "huge_shared" : " 27 GB", | 20260505 13:56:04.406 | INFO | "malloc" : " 14 GB", | 20260505 13:56:04.406 | INFO | "pinned" : " 29 GB" | 20260505 13:56:04.406 | INFO | } | 20260505 13:56:04.406 | INFO | Perf report for the last 5.14171 seconds: | 20260505 13:56:04.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:56:04.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:04.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:04.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:04.406 | INFO | ANALYSIS : 62.6%, 3.0, 97.3ms, 0 | 20260505 13:56:04.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:04.406 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:04.406 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:56:04.406 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:04.406 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:04.406 | INFO | Uploaded 36GiB to GPU in 2.18221s (16.4971GiB/s) | 20260505 13:56:04.406 | INFO | Downloaded 34GiB to GPU in 2.12798s (15.9776GiB/s) | 20260505 13:56:09.538 | INFO | Analyzing chunk frames = [3733504,3734016) | 20260505 13:56:09.538 | INFO | MemInfo: { | 20260505 13:56:09.538 | INFO | "gpu" : " 17 GB", | 20260505 13:56:09.538 | INFO | "huge" : "", | 20260505 13:56:09.538 | INFO | "huge_shared" : " 27 GB", | 20260505 13:56:09.538 | INFO | "malloc" : " 14 GB", | 20260505 13:56:09.538 | INFO | "pinned" : " 29 GB" | 20260505 13:56:09.538 | INFO | } | 20260505 13:56:09.538 | INFO | Perf report for the last 5.13151 seconds: | 20260505 13:56:09.538 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:56:09.538 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:09.538 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:09.538 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:09.538 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 13:56:09.538 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:09.538 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:09.538 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:56:09.538 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:09.538 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:09.538 | INFO | Uploaded 36GiB to GPU in 2.16281s (16.645GiB/s) | 20260505 13:56:09.538 | INFO | Downloaded 34GiB to GPU in 2.12942s (15.9668GiB/s) | 20260505 13:56:14.673 | INFO | Analyzing chunk frames = [3734016,3734528) | 20260505 13:56:14.673 | INFO | MemInfo: { | 20260505 13:56:14.673 | INFO | "gpu" : " 17 GB", | 20260505 13:56:14.673 | INFO | "huge" : "", | 20260505 13:56:14.673 | INFO | "huge_shared" : " 27 GB", | 20260505 13:56:14.673 | INFO | "malloc" : " 14 GB", | 20260505 13:56:14.673 | INFO | "pinned" : " 29 GB" | 20260505 13:56:14.673 | INFO | } | 20260505 13:56:14.674 | INFO | Perf report for the last 5.1357 seconds: | 20260505 13:56:14.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:56:14.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:14.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:14.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:14.674 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 13:56:14.674 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:14.674 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:14.674 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:56:14.674 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:14.674 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:14.674 | INFO | Uploaded 36GiB to GPU in 2.16865s (16.6002GiB/s) | 20260505 13:56:14.674 | INFO | Downloaded 34GiB to GPU in 2.11504s (16.0753GiB/s) | 20260505 13:56:19.821 | INFO | Analyzing chunk frames = [3734528,3735040) | 20260505 13:56:19.821 | INFO | MemInfo: { | 20260505 13:56:19.821 | INFO | "gpu" : " 17 GB", | 20260505 13:56:19.821 | INFO | "huge" : "", | 20260505 13:56:19.821 | INFO | "huge_shared" : " 27 GB", | 20260505 13:56:19.821 | INFO | "malloc" : " 14 GB", | 20260505 13:56:19.821 | INFO | "pinned" : " 29 GB" | 20260505 13:56:19.821 | INFO | } | 20260505 13:56:19.821 | INFO | Perf report for the last 5.14784 seconds: | 20260505 13:56:19.821 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:56:19.821 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:19.821 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:19.821 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:19.821 | INFO | ANALYSIS : 63.1%, 3.0, 98.7ms, 0 | 20260505 13:56:19.821 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:19.821 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:19.821 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 13:56:19.821 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:19.821 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:19.822 | INFO | Uploaded 36GiB to GPU in 2.17101s (16.5821GiB/s) | 20260505 13:56:19.822 | INFO | Downloaded 34GiB to GPU in 2.14371s (15.8603GiB/s) | 20260505 13:56:24.962 | INFO | Analyzing chunk frames = [3735040,3735552) | 20260505 13:56:24.962 | INFO | MemInfo: { | 20260505 13:56:24.962 | INFO | "gpu" : " 17 GB", | 20260505 13:56:24.962 | INFO | "huge" : "", | 20260505 13:56:24.962 | INFO | "huge_shared" : " 27 GB", | 20260505 13:56:24.962 | INFO | "malloc" : " 14 GB", | 20260505 13:56:24.962 | INFO | "pinned" : " 29 GB" | 20260505 13:56:24.962 | INFO | } | 20260505 13:56:24.962 | INFO | Perf report for the last 5.14105 seconds: | 20260505 13:56:24.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:56:24.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:24.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:24.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:24.962 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 13:56:24.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:24.962 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:24.962 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 13:56:24.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:24.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:24.963 | INFO | Uploaded 36GiB to GPU in 2.21413s (16.2592GiB/s) | 20260505 13:56:24.963 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 13:56:24.963 | INFO | Total mem usage: 15.3 GiB | 20260505 13:56:24.963 | INFO | Downloaded 34GiB to GPU in 2.10004s (16.1901GiB/s) | 20260505 13:56:24.963 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 13:56:24.963 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.76%) | 20260505 13:56:24.963 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 13:56:24.963 | INFO | High Water Breakdown: | 20260505 13:56:24.963 | INFO | 61% : 8.68 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 13:56:24.963 | INFO | 38.8% : 5.52 GiB ( 83.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 13:56:24.963 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 13:56:24.963 | INFO | ------------------------------------------ | 20260505 13:56:24.963 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:56:24.963 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 13:56:24.963 | INFO | Total mem usage: 26.5 GiB | 20260505 13:56:24.963 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 13:56:24.963 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 13:56:24.963 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 13:56:24.963 | INFO | High Water Breakdown: | 20260505 13:56:24.963 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:56:24.963 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:56:24.963 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:56:24.963 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:56:24.963 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:56:24.963 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:56:24.963 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 13:56:24.963 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 13:56:24.963 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:56:24.963 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 13:56:24.963 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 13:56:24.963 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 13:56:24.963 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 13:56:24.963 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 13:56:24.963 | INFO | ------------------------------------------ | 20260505 13:56:24.963 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:56:24.963 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 13:56:24.963 | INFO | Total mem usage: 25.6 GiB | 20260505 13:56:24.963 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 13:56:24.963 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 13:56:24.963 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 13:56:24.963 | INFO | High Water Breakdown: | 20260505 13:56:24.963 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 13:56:24.963 | INFO | ------------------------------------------ | 20260505 13:56:24.963 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:56:24.963 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 13:56:24.963 | INFO | Total mem usage: 15.7 GiB | 20260505 13:56:24.963 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 13:56:24.963 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 13:56:24.963 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 13:56:24.963 | INFO | High Water Breakdown: | 20260505 13:56:24.963 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 13:56:24.963 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 13:56:24.963 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 13:56:24.963 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 13:56:24.963 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 13:56:24.963 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 13:56:24.963 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 13:56:24.963 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 13:56:24.963 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 13:56:24.963 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:56:24.963 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 13:56:24.963 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:56:24.963 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 13:56:24.963 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 13:56:24.963 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 13:56:24.963 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 13:56:24.963 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:56:24.963 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 13:56:24.963 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:56:24.963 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:56:24.963 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:56:24.963 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 13:56:24.963 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 13:56:24.963 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:56:24.963 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 13:56:24.963 | INFO | ------------------------------------------ | 20260505 13:56:24.963 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:56:24.963 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 13:56:24.963 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:56:24.963 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:56:24.963 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.64e+01, 1.37e+01 / 1.49e+00 | 20260505 13:56:24.963 | INFO | PerfReport: 0.29%, AggregateStats -- 9.26e-02 / 1.00e-01, 1.06e-02 / 4.47e-03 | 20260505 13:56:24.963 | INFO | PerfReport: 11.78%, Baseline -- 3.75e+00 / 4.50e+00, 5.85e-01 / 1.50e-01 | 20260505 13:56:24.963 | INFO | PerfReport: 3.29%, Histogram -- 1.05e+00 / 1.51e+00, 1.99e-01 / 8.52e-02 | 20260505 13:56:24.963 | INFO | PerfReport: 15.99%, DME -- 4.07e+01 / 4.57e+01, 4.67e+00 / 1.66e+00 | 20260505 13:56:24.963 | INFO | PerfReport: 56.33%, FrameLabeling -- 1.79e+01 / 1.97e+01, 1.36e+00 / 1.94e+00 | 20260505 13:56:24.963 | INFO | PerfReport: 7.49%, PulseAccumulating -- 2.38e+00 / 2.94e+00, 4.52e-01 / 1.45e-01 | 20260505 13:56:24.964 | INFO | PerfReport: 3.57%, Metrics -- 1.14e+00 / 1.48e+00, 3.18e-01 / 1.50e-01 | 20260505 13:56:24.964 | INFO | PerfReport: 1.25%, PulseSerializing -- 3.98e-01 / 4.50e-01, 7.14e-02 / 2.30e-02 | 20260505 13:56:24.964 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 13:56:24.964 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:56:24.964 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:56:24.964 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260505 13:56:24.964 | INFO | PerfReport: 22.55%, Upload -- 2.27e+01 / 0.00e+00, 8.05e+00 / 0.00e+00 | 20260505 13:56:24.964 | INFO | PerfReport: 22.04%, Download -- 2.22e+01 / 0.00e+00, 8.11e+00 / 0.00e+00 | 20260505 13:56:24.964 | INFO | PerfReport: 31.57%, Compute -- 3.18e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 13:56:24.964 | INFO | PerfReport: 2.27%, LockUpload -- 2.29e+00 / 0.00e+00, 8.01e+00 / 0.00e+00 | 20260505 13:56:24.964 | INFO | PerfReport: 1.47%, LockDownload -- 1.48e+00 / 0.00e+00, 3.08e+00 / 0.00e+00 | 20260505 13:56:24.964 | INFO | PerfReport: 20.09%, LockCompute -- 2.02e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260505 13:56:24.964 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.73e-02 / 0.00e+00 | 20260505 13:56:25.099 | INFO | Starting a round of Baz IO | 20260505 13:56:28.315 | INFO | Populating RTMetrics report | 20260505 13:56:28.316 | INFO | RTMetrics report populated | 20260505 13:56:28.316 | INFO | Opening tmp RTMetrics json file | 20260505 13:56:28.329 | INFO | Serializing RTMetrics to json | 20260505 13:56:28.331 | INFO | Serialization done, swapping tmp file now... | 20260505 13:56:28.333 | INFO | RTMetrics done | 20260505 13:56:28.419 | INFO | Finished a round of Baz IO: Wrote 2.28 GiB in 3.319 seconds | 20260505 13:56:28.419 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.93% / 0.00% / 77.07% | 20260505 13:56:28.419 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.628781% / 3.97364e-06% / 7.54992e-05% | 20260505 13:56:28.419 | INFO | Baz IO overall: | 20260505 13:56:28.419 | INFO | BytesWritten : 2225707155539 | 20260505 13:56:28.419 | INFO | NumEvents : 1058577636273 | 20260505 13:56:30.095 | INFO | Analyzing chunk frames = [3735552,3736064) | 20260505 13:56:30.095 | INFO | MemInfo: { | 20260505 13:56:30.095 | INFO | "gpu" : " 17 GB", | 20260505 13:56:30.095 | INFO | "huge" : "", | 20260505 13:56:30.095 | INFO | "huge_shared" : " 27 GB", | 20260505 13:56:30.095 | INFO | "malloc" : " 14 GB", | 20260505 13:56:30.095 | INFO | "pinned" : " 29 GB" | 20260505 13:56:30.095 | INFO | } | 20260505 13:56:30.095 | INFO | Perf report for the last 5.13272 seconds: | 20260505 13:56:30.095 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:56:30.095 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:30.095 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:30.095 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:30.095 | INFO | ANALYSIS : 64.2%, 3.0, 99.5ms, 0 | 20260505 13:56:30.095 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:30.095 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260505 13:56:30.095 | INFO | PRE_HQ : 61.1%, 1.8, 57.6ms, 0 | 20260505 13:56:30.095 | INFO | BAZWRITER : 64.0%, 4.0, 131.3ms, 0 | 20260505 13:56:30.095 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 13:56:30.096 | INFO | Uploaded 36GiB to GPU in 2.19603s (16.3932GiB/s) | 20260505 13:56:30.096 | INFO | Downloaded 38GiB to GPU in 2.26541s (16.774GiB/s) | 20260505 13:56:35.232 | INFO | Analyzing chunk frames = [3736064,3736576) | 20260505 13:56:35.232 | INFO | MemInfo: { | 20260505 13:56:35.232 | INFO | "gpu" : " 17 GB", | 20260505 13:56:35.232 | INFO | "huge" : "", | 20260505 13:56:35.232 | INFO | "huge_shared" : " 27 GB", | 20260505 13:56:35.232 | INFO | "malloc" : " 14 GB", | 20260505 13:56:35.232 | INFO | "pinned" : " 29 GB" | 20260505 13:56:35.232 | INFO | } | 20260505 13:56:35.232 | INFO | Perf report for the last 5.1366 seconds: | 20260505 13:56:35.232 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:56:35.232 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:35.232 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:35.232 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:35.232 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 13:56:35.232 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:35.232 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:35.232 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:56:35.232 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:35.232 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:35.232 | INFO | Uploaded 36GiB to GPU in 2.20225s (16.3469GiB/s) | 20260505 13:56:35.232 | INFO | Downloaded 34GiB to GPU in 2.04098s (16.6587GiB/s) | 20260505 13:56:40.380 | INFO | Analyzing chunk frames = [3736576,3737088) | 20260505 13:56:40.380 | INFO | MemInfo: { | 20260505 13:56:40.380 | INFO | "gpu" : " 17 GB", | 20260505 13:56:40.380 | INFO | "huge" : "", | 20260505 13:56:40.380 | INFO | "huge_shared" : " 27 GB", | 20260505 13:56:40.380 | INFO | "malloc" : " 14 GB", | 20260505 13:56:40.380 | INFO | "pinned" : " 29 GB" | 20260505 13:56:40.380 | INFO | } | 20260505 13:56:40.380 | INFO | Perf report for the last 5.14818 seconds: | 20260505 13:56:40.380 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:56:40.380 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:40.380 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:40.380 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:40.380 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 13:56:40.380 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:40.380 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:40.380 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 13:56:40.380 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:40.380 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:40.380 | INFO | Uploaded 36GiB to GPU in 2.22982s (16.1448GiB/s) | 20260505 13:56:40.380 | INFO | Downloaded 34GiB to GPU in 2.09704s (16.2133GiB/s) | 20260505 13:56:45.519 | INFO | Analyzing chunk frames = [3737088,3737600) | 20260505 13:56:45.519 | INFO | MemInfo: { | 20260505 13:56:45.519 | INFO | "gpu" : " 17 GB", | 20260505 13:56:45.519 | INFO | "huge" : "", | 20260505 13:56:45.519 | INFO | "huge_shared" : " 27 GB", | 20260505 13:56:45.519 | INFO | "malloc" : " 14 GB", | 20260505 13:56:45.519 | INFO | "pinned" : " 29 GB" | 20260505 13:56:45.519 | INFO | } | 20260505 13:56:45.519 | INFO | Perf report for the last 5.13882 seconds: | 20260505 13:56:45.519 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:56:45.519 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:45.519 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:45.519 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:45.519 | INFO | ANALYSIS : 62.6%, 3.0, 97.3ms, 0 | 20260505 13:56:45.519 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:45.519 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:45.519 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 13:56:45.519 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:45.519 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:45.519 | INFO | Uploaded 36GiB to GPU in 2.18106s (16.5058GiB/s) | 20260505 13:56:45.519 | INFO | Downloaded 34GiB to GPU in 2.12716s (15.9837GiB/s) | 20260505 13:56:50.654 | INFO | Analyzing chunk frames = [3737600,3738112) | 20260505 13:56:50.654 | INFO | MemInfo: { | 20260505 13:56:50.654 | INFO | "gpu" : " 17 GB", | 20260505 13:56:50.654 | INFO | "huge" : "", | 20260505 13:56:50.654 | INFO | "huge_shared" : " 27 GB", | 20260505 13:56:50.654 | INFO | "malloc" : " 14 GB", | 20260505 13:56:50.654 | INFO | "pinned" : " 29 GB" | 20260505 13:56:50.654 | INFO | } | 20260505 13:56:50.654 | INFO | Perf report for the last 5.13501 seconds: | 20260505 13:56:50.654 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:56:50.654 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:50.654 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:50.654 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:50.654 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 13:56:50.654 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:50.654 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:50.654 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 13:56:50.654 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:50.654 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:50.654 | INFO | Uploaded 36GiB to GPU in 2.15538s (16.7024GiB/s) | 20260505 13:56:50.654 | INFO | Downloaded 34GiB to GPU in 2.1248s (16.0015GiB/s) | 20260505 13:56:55.789 | INFO | Analyzing chunk frames = [3738112,3738624) | 20260505 13:56:55.789 | INFO | MemInfo: { | 20260505 13:56:55.789 | INFO | "gpu" : " 17 GB", | 20260505 13:56:55.789 | INFO | "huge" : "", | 20260505 13:56:55.789 | INFO | "huge_shared" : " 27 GB", | 20260505 13:56:55.789 | INFO | "malloc" : " 14 GB", | 20260505 13:56:55.789 | INFO | "pinned" : " 29 GB" | 20260505 13:56:55.789 | INFO | } | 20260505 13:56:55.789 | INFO | Perf report for the last 5.13548 seconds: | 20260505 13:56:55.789 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:56:55.789 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:55.789 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:55.789 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:55.789 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 13:56:55.789 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:55.789 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:55.789 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:56:55.789 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:55.789 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:56:55.790 | INFO | Uploaded 36GiB to GPU in 2.15259s (16.7241GiB/s) | 20260505 13:56:55.790 | INFO | Downloaded 34GiB to GPU in 2.09314s (16.2435GiB/s) | 20260505 13:57:00.929 | INFO | Analyzing chunk frames = [3738624,3739136) | 20260505 13:57:00.929 | INFO | MemInfo: { | 20260505 13:57:00.929 | INFO | "gpu" : " 17 GB", | 20260505 13:57:00.929 | INFO | "huge" : "", | 20260505 13:57:00.929 | INFO | "huge_shared" : " 27 GB", | 20260505 13:57:00.929 | INFO | "malloc" : " 14 GB", | 20260505 13:57:00.929 | INFO | "pinned" : " 29 GB" | 20260505 13:57:00.930 | INFO | } | 20260505 13:57:00.930 | INFO | Perf report for the last 5.14023 seconds: | 20260505 13:57:00.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:57:00.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:00.930 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:00.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:00.930 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 13:57:00.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:00.930 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:00.930 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 13:57:00.930 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:00.930 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:00.930 | INFO | Uploaded 36GiB to GPU in 2.15081s (16.7379GiB/s) | 20260505 13:57:00.930 | INFO | Downloaded 34GiB to GPU in 2.14134s (15.8779GiB/s) | 20260505 13:57:06.077 | INFO | Analyzing chunk frames = [3739136,3739648) | 20260505 13:57:06.077 | INFO | MemInfo: { | 20260505 13:57:06.077 | INFO | "gpu" : " 17 GB", | 20260505 13:57:06.077 | INFO | "huge" : "", | 20260505 13:57:06.077 | INFO | "huge_shared" : " 27 GB", | 20260505 13:57:06.077 | INFO | "malloc" : " 14 GB", | 20260505 13:57:06.077 | INFO | "pinned" : " 29 GB" | 20260505 13:57:06.077 | INFO | } | 20260505 13:57:06.077 | INFO | Perf report for the last 5.14727 seconds: | 20260505 13:57:06.077 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:57:06.077 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:06.077 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:06.077 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:06.077 | INFO | ANALYSIS : 63.6%, 3.0, 99.5ms, 0 | 20260505 13:57:06.077 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:06.077 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:06.077 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 13:57:06.077 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:06.077 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:06.077 | INFO | Uploaded 36GiB to GPU in 2.18341s (16.488GiB/s) | 20260505 13:57:06.077 | INFO | Downloaded 34GiB to GPU in 2.1115s (16.1023GiB/s) | 20260505 13:57:06.226 | INFO | Starting a round of Baz IO | 20260505 13:57:09.431 | INFO | Populating RTMetrics report | 20260505 13:57:09.432 | INFO | RTMetrics report populated | 20260505 13:57:09.432 | INFO | Opening tmp RTMetrics json file | 20260505 13:57:09.475 | INFO | Serializing RTMetrics to json | 20260505 13:57:09.501 | INFO | Serialization done, swapping tmp file now... | 20260505 13:57:09.502 | INFO | RTMetrics done | 20260505 13:57:09.542 | INFO | Finished a round of Baz IO: Wrote 2.28 GiB in 3.316 seconds | 20260505 13:57:09.542 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.93% / 0.00% / 77.07% | 20260505 13:57:09.542 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.626914% / 0% / 9.13938e-05% | 20260505 13:57:09.543 | INFO | Baz IO overall: | 20260505 13:57:09.543 | INFO | BytesWritten : 2228154797739 | 20260505 13:57:09.543 | INFO | NumEvents : 1059733989035 | 20260505 13:57:11.216 | INFO | Analyzing chunk frames = [3739648,3740160) | 20260505 13:57:11.216 | INFO | MemInfo: { | 20260505 13:57:11.216 | INFO | "gpu" : " 17 GB", | 20260505 13:57:11.216 | INFO | "huge" : "", | 20260505 13:57:11.216 | INFO | "huge_shared" : " 27 GB", | 20260505 13:57:11.216 | INFO | "malloc" : " 14 GB", | 20260505 13:57:11.216 | INFO | "pinned" : " 29 GB" | 20260505 13:57:11.216 | INFO | } | 20260505 13:57:11.216 | INFO | Perf report for the last 5.1391 seconds: | 20260505 13:57:11.216 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:57:11.216 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:11.216 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:11.216 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:11.216 | INFO | ANALYSIS : 64.2%, 3.0, 99.6ms, 0 | 20260505 13:57:11.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:11.216 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260505 13:57:11.216 | INFO | PRE_HQ : 61.0%, 1.8, 57.6ms, 0 | 20260505 13:57:11.216 | INFO | BAZWRITER : 64.5%, 4.0, 131.9ms, 0 | 20260505 13:57:11.216 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 13:57:11.216 | INFO | Uploaded 36GiB to GPU in 2.19329s (16.4137GiB/s) | 20260505 13:57:11.216 | INFO | Downloaded 38GiB to GPU in 2.25482s (16.8528GiB/s) | 20260505 13:57:16.346 | INFO | Analyzing chunk frames = [3740160,3740672) | 20260505 13:57:16.346 | INFO | MemInfo: { | 20260505 13:57:16.346 | INFO | "gpu" : " 17 GB", | 20260505 13:57:16.346 | INFO | "huge" : "", | 20260505 13:57:16.346 | INFO | "huge_shared" : " 27 GB", | 20260505 13:57:16.346 | INFO | "malloc" : " 14 GB", | 20260505 13:57:16.346 | INFO | "pinned" : " 29 GB" | 20260505 13:57:16.346 | INFO | } | 20260505 13:57:16.346 | INFO | Perf report for the last 5.13026 seconds: | 20260505 13:57:16.346 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:57:16.346 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:16.346 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:16.346 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:16.346 | INFO | ANALYSIS : 63.6%, 3.0, 99.1ms, 0 | 20260505 13:57:16.346 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:16.346 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:16.346 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 13:57:16.346 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:16.346 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:16.346 | INFO | Uploaded 36GiB to GPU in 2.19221s (16.4218GiB/s) | 20260505 13:57:16.347 | INFO | Downloaded 34GiB to GPU in 2.02888s (16.758GiB/s) | 20260505 13:57:21.483 | INFO | Analyzing chunk frames = [3740672,3741184) | 20260505 13:57:21.483 | INFO | MemInfo: { | 20260505 13:57:21.483 | INFO | "gpu" : " 17 GB", | 20260505 13:57:21.483 | INFO | "huge" : "", | 20260505 13:57:21.483 | INFO | "huge_shared" : " 27 GB", | 20260505 13:57:21.483 | INFO | "malloc" : " 14 GB", | 20260505 13:57:21.483 | INFO | "pinned" : " 29 GB" | 20260505 13:57:21.483 | INFO | } | 20260505 13:57:21.483 | INFO | Perf report for the last 5.13686 seconds: | 20260505 13:57:21.483 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:57:21.483 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:21.483 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:21.483 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:21.483 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 13:57:21.483 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:21.483 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:21.483 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:57:21.483 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:21.483 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:21.483 | INFO | Uploaded 36GiB to GPU in 2.20962s (16.2924GiB/s) | 20260505 13:57:21.483 | INFO | Downloaded 34GiB to GPU in 2.12274s (16.0171GiB/s) | 20260505 13:57:26.633 | INFO | Analyzing chunk frames = [3741184,3741696) | 20260505 13:57:26.633 | INFO | MemInfo: { | 20260505 13:57:26.633 | INFO | "gpu" : " 17 GB", | 20260505 13:57:26.633 | INFO | "huge" : "", | 20260505 13:57:26.633 | INFO | "huge_shared" : " 27 GB", | 20260505 13:57:26.633 | INFO | "malloc" : " 14 GB", | 20260505 13:57:26.633 | INFO | "pinned" : " 29 GB" | 20260505 13:57:26.633 | INFO | } | 20260505 13:57:26.633 | INFO | Perf report for the last 5.14967 seconds: | 20260505 13:57:26.633 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:57:26.633 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:26.633 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:26.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:26.633 | INFO | ANALYSIS : 62.9%, 3.0, 98.0ms, 0 | 20260505 13:57:26.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:26.633 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:26.633 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:57:26.633 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:26.633 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:26.633 | INFO | Uploaded 36GiB to GPU in 2.20458s (16.3296GiB/s) | 20260505 13:57:26.633 | INFO | Downloaded 34GiB to GPU in 2.13188s (15.9484GiB/s) | 20260505 13:57:31.772 | INFO | Analyzing chunk frames = [3741696,3742208) | 20260505 13:57:31.772 | INFO | MemInfo: { | 20260505 13:57:31.772 | INFO | "gpu" : " 17 GB", | 20260505 13:57:31.772 | INFO | "huge" : "", | 20260505 13:57:31.772 | INFO | "huge_shared" : " 27 GB", | 20260505 13:57:31.772 | INFO | "malloc" : " 14 GB", | 20260505 13:57:31.772 | INFO | "pinned" : " 29 GB" | 20260505 13:57:31.772 | INFO | } | 20260505 13:57:31.772 | INFO | Perf report for the last 5.1395 seconds: | 20260505 13:57:31.772 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:57:31.772 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:31.772 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:31.772 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:31.772 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 13:57:31.772 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:31.772 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:31.772 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 13:57:31.772 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:31.772 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:31.773 | INFO | Uploaded 36GiB to GPU in 2.1545s (16.7092GiB/s) | 20260505 13:57:31.773 | INFO | Downloaded 34GiB to GPU in 2.12553s (15.996GiB/s) | 20260505 13:57:36.913 | INFO | Analyzing chunk frames = [3742208,3742720) | 20260505 13:57:36.913 | INFO | MemInfo: { | 20260505 13:57:36.913 | INFO | "gpu" : " 17 GB", | 20260505 13:57:36.913 | INFO | "huge" : "", | 20260505 13:57:36.913 | INFO | "huge_shared" : " 27 GB", | 20260505 13:57:36.913 | INFO | "malloc" : " 14 GB", | 20260505 13:57:36.913 | INFO | "pinned" : " 29 GB" | 20260505 13:57:36.913 | INFO | } | 20260505 13:57:36.913 | INFO | Perf report for the last 5.1405 seconds: | 20260505 13:57:36.913 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:57:36.913 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:36.913 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:36.913 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:36.913 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 13:57:36.913 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:36.913 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:36.913 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:57:36.913 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:36.913 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:36.913 | INFO | Uploaded 36GiB to GPU in 2.13085s (16.8947GiB/s) | 20260505 13:57:36.913 | INFO | Downloaded 34GiB to GPU in 2.10323s (16.1656GiB/s) | 20260505 13:57:42.051 | INFO | Analyzing chunk frames = [3742720,3743232) | 20260505 13:57:42.051 | INFO | MemInfo: { | 20260505 13:57:42.051 | INFO | "gpu" : " 17 GB", | 20260505 13:57:42.051 | INFO | "huge" : "", | 20260505 13:57:42.051 | INFO | "huge_shared" : " 27 GB", | 20260505 13:57:42.051 | INFO | "malloc" : " 14 GB", | 20260505 13:57:42.051 | INFO | "pinned" : " 29 GB" | 20260505 13:57:42.051 | INFO | } | 20260505 13:57:42.051 | INFO | Perf report for the last 5.13836 seconds: | 20260505 13:57:42.051 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:57:42.051 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:42.051 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:42.051 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:42.051 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 13:57:42.051 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:42.051 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:42.051 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 13:57:42.051 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:42.051 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:42.051 | INFO | Uploaded 36GiB to GPU in 2.19361s (16.4113GiB/s) | 20260505 13:57:42.051 | INFO | Downloaded 34GiB to GPU in 2.15166s (15.8018GiB/s) | 20260505 13:57:47.190 | INFO | Analyzing chunk frames = [3743232,3743744) | 20260505 13:57:47.190 | INFO | MemInfo: { | 20260505 13:57:47.190 | INFO | "gpu" : " 17 GB", | 20260505 13:57:47.190 | INFO | "huge" : "", | 20260505 13:57:47.190 | INFO | "huge_shared" : " 27 GB", | 20260505 13:57:47.190 | INFO | "malloc" : " 14 GB", | 20260505 13:57:47.190 | INFO | "pinned" : " 29 GB" | 20260505 13:57:47.190 | INFO | } | 20260505 13:57:47.190 | INFO | Perf report for the last 5.13911 seconds: | 20260505 13:57:47.190 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:57:47.190 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:47.190 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:47.190 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:47.190 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 13:57:47.190 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:47.190 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:47.190 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 13:57:47.190 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:47.190 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:47.190 | INFO | Uploaded 36GiB to GPU in 2.21727s (16.2362GiB/s) | 20260505 13:57:47.191 | INFO | Downloaded 34GiB to GPU in 2.10683s (16.138GiB/s) | 20260505 13:57:47.191 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 13:57:47.191 | INFO | Total mem usage: 15.3 GiB | 20260505 13:57:47.191 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 13:57:47.191 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.64%) | 20260505 13:57:47.191 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 13:57:47.191 | INFO | High Water Breakdown: | 20260505 13:57:47.191 | INFO | 61% : 8.68 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 13:57:47.191 | INFO | 38.9% : 5.54 GiB ( 83.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 13:57:47.191 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 13:57:47.191 | INFO | ------------------------------------------ | 20260505 13:57:47.191 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:57:47.191 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 13:57:47.191 | INFO | Total mem usage: 26.5 GiB | 20260505 13:57:47.191 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 13:57:47.191 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 13:57:47.191 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 13:57:47.191 | INFO | High Water Breakdown: | 20260505 13:57:47.191 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:57:47.191 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:57:47.191 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:57:47.191 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:57:47.191 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:57:47.191 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:57:47.191 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 13:57:47.191 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 13:57:47.191 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:57:47.191 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 13:57:47.191 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 13:57:47.191 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 13:57:47.191 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 13:57:47.191 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 13:57:47.191 | INFO | ------------------------------------------ | 20260505 13:57:47.191 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:57:47.191 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 13:57:47.191 | INFO | Total mem usage: 25.6 GiB | 20260505 13:57:47.191 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 13:57:47.191 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 13:57:47.191 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 13:57:47.191 | INFO | High Water Breakdown: | 20260505 13:57:47.191 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 13:57:47.191 | INFO | ------------------------------------------ | 20260505 13:57:47.191 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:57:47.191 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 13:57:47.191 | INFO | Total mem usage: 15.7 GiB | 20260505 13:57:47.191 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 13:57:47.191 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 13:57:47.191 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 13:57:47.191 | INFO | High Water Breakdown: | 20260505 13:57:47.191 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 13:57:47.191 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 13:57:47.191 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 13:57:47.191 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 13:57:47.191 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 13:57:47.191 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 13:57:47.191 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 13:57:47.191 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 13:57:47.191 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 13:57:47.191 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:57:47.191 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 13:57:47.191 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:57:47.191 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 13:57:47.191 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 13:57:47.191 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 13:57:47.191 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 13:57:47.191 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:57:47.191 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 13:57:47.191 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:57:47.191 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:57:47.191 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:57:47.191 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 13:57:47.191 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 13:57:47.191 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:57:47.191 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 13:57:47.191 | INFO | ------------------------------------------ | 20260505 13:57:47.191 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:57:47.191 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 13:57:47.191 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:57:47.191 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:57:47.191 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.67e+01, 1.38e+01 / 1.60e+00 | 20260505 13:57:47.191 | INFO | PerfReport: 0.29%, AggregateStats -- 9.32e-02 / 1.01e-01, 1.09e-02 / 4.86e-03 | 20260505 13:57:47.191 | INFO | PerfReport: 11.79%, Baseline -- 3.75e+00 / 4.48e+00, 5.88e-01 / 2.03e-01 | 20260505 13:57:47.191 | INFO | PerfReport: 3.30%, Histogram -- 1.05e+00 / 1.52e+00, 2.00e-01 / 8.53e-02 | 20260505 13:57:47.191 | INFO | PerfReport: 15.99%, DME -- 4.07e+01 / 4.59e+01, 4.63e+00 / 1.73e+00 | 20260505 13:57:47.191 | INFO | PerfReport: 56.31%, FrameLabeling -- 1.79e+01 / 1.99e+01, 1.38e+00 / 1.60e+00 | 20260505 13:57:47.191 | INFO | PerfReport: 7.50%, PulseAccumulating -- 2.39e+00 / 2.92e+00, 4.54e-01 / 1.86e-01 | 20260505 13:57:47.191 | INFO | PerfReport: 3.57%, Metrics -- 1.14e+00 / 1.46e+00, 3.21e-01 / 1.76e-01 | 20260505 13:57:47.191 | INFO | PerfReport: 1.25%, PulseSerializing -- 3.98e-01 / 4.52e-01, 7.09e-02 / 2.94e-02 | 20260505 13:57:47.191 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 13:57:47.191 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:57:47.191 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:57:47.191 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260505 13:57:47.191 | INFO | PerfReport: 22.58%, Upload -- 2.28e+01 / 0.00e+00, 8.07e+00 / 0.00e+00 | 20260505 13:57:47.191 | INFO | PerfReport: 21.99%, Download -- 2.22e+01 / 0.00e+00, 8.06e+00 / 0.00e+00 | 20260505 13:57:47.191 | INFO | PerfReport: 31.61%, Compute -- 3.19e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 13:57:47.191 | INFO | PerfReport: 2.23%, LockUpload -- 2.25e+00 / 0.00e+00, 7.97e+00 / 0.00e+00 | 20260505 13:57:47.191 | INFO | PerfReport: 1.41%, LockDownload -- 1.42e+00 / 0.00e+00, 2.87e+00 / 0.00e+00 | 20260505 13:57:47.191 | INFO | PerfReport: 20.16%, LockCompute -- 2.03e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260505 13:57:47.191 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.54e-02 / 0.00e+00 | 20260505 13:57:47.326 | INFO | Starting a round of Baz IO | 20260505 13:57:50.549 | INFO | Populating RTMetrics report | 20260505 13:57:50.549 | INFO | RTMetrics report populated | 20260505 13:57:50.549 | INFO | Opening tmp RTMetrics json file | 20260505 13:57:50.569 | INFO | Serializing RTMetrics to json | 20260505 13:57:50.572 | INFO | Serialization done, swapping tmp file now... | 20260505 13:57:50.573 | INFO | RTMetrics done | 20260505 13:57:50.660 | INFO | Finished a round of Baz IO: Wrote 2.28 GiB in 3.334 seconds | 20260505 13:57:50.660 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.92% / 0.00% / 77.07% | 20260505 13:57:50.660 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.626703% / 3.97364e-06% / 0.000103315% | 20260505 13:57:50.660 | INFO | Baz IO overall: | 20260505 13:57:50.660 | INFO | BytesWritten : 2230603307337 | 20260505 13:57:50.660 | INFO | NumEvents : 1060891197612 | 20260505 13:57:52.329 | INFO | Analyzing chunk frames = [3743744,3744256) | 20260505 13:57:52.329 | INFO | MemInfo: { | 20260505 13:57:52.329 | INFO | "gpu" : " 17 GB", | 20260505 13:57:52.329 | INFO | "huge" : "", | 20260505 13:57:52.329 | INFO | "huge_shared" : " 27 GB", | 20260505 13:57:52.329 | INFO | "malloc" : " 14 GB", | 20260505 13:57:52.329 | INFO | "pinned" : " 29 GB" | 20260505 13:57:52.329 | INFO | } | 20260505 13:57:52.329 | INFO | Perf report for the last 5.13886 seconds: | 20260505 13:57:52.329 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:57:52.329 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:52.329 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:52.329 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:52.329 | INFO | ANALYSIS : 64.3%, 3.0, 99.4ms, 0 | 20260505 13:57:52.329 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:52.329 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260505 13:57:52.329 | INFO | PRE_HQ : 61.1%, 1.8, 57.8ms, 0 | 20260505 13:57:52.329 | INFO | BAZWRITER : 64.2%, 4.0, 132.8ms, 0 | 20260505 13:57:52.329 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 13:57:52.330 | INFO | Uploaded 36GiB to GPU in 2.19242s (16.4202GiB/s) | 20260505 13:57:52.330 | INFO | Downloaded 38GiB to GPU in 2.2532s (16.8649GiB/s) | 20260505 13:57:57.466 | INFO | Analyzing chunk frames = [3744256,3744768) | 20260505 13:57:57.466 | INFO | MemInfo: { | 20260505 13:57:57.466 | INFO | "gpu" : " 17 GB", | 20260505 13:57:57.466 | INFO | "huge" : "", | 20260505 13:57:57.466 | INFO | "huge_shared" : " 27 GB", | 20260505 13:57:57.466 | INFO | "malloc" : " 14 GB", | 20260505 13:57:57.466 | INFO | "pinned" : " 29 GB" | 20260505 13:57:57.466 | INFO | } | 20260505 13:57:57.466 | INFO | Perf report for the last 5.1366 seconds: | 20260505 13:57:57.466 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:57:57.466 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:57.466 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:57.466 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:57.466 | INFO | ANALYSIS : 63.5%, 3.0, 99.0ms, 0 | 20260505 13:57:57.466 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:57.466 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:57.466 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 13:57:57.466 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:57.466 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:57:57.466 | INFO | Uploaded 36GiB to GPU in 2.19111s (16.43GiB/s) | 20260505 13:57:57.466 | INFO | Downloaded 34GiB to GPU in 2.03229s (16.7299GiB/s) | 20260505 13:58:02.609 | INFO | Analyzing chunk frames = [3744768,3745280) | 20260505 13:58:02.609 | INFO | MemInfo: { | 20260505 13:58:02.609 | INFO | "gpu" : " 17 GB", | 20260505 13:58:02.609 | INFO | "huge" : "", | 20260505 13:58:02.609 | INFO | "huge_shared" : " 27 GB", | 20260505 13:58:02.609 | INFO | "malloc" : " 14 GB", | 20260505 13:58:02.609 | INFO | "pinned" : " 29 GB" | 20260505 13:58:02.609 | INFO | } | 20260505 13:58:02.609 | INFO | Perf report for the last 5.14304 seconds: | 20260505 13:58:02.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:58:02.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:02.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:02.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:02.609 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 13:58:02.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:02.609 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:02.609 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 13:58:02.609 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:02.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:02.609 | INFO | Uploaded 36GiB to GPU in 2.21554s (16.2489GiB/s) | 20260505 13:58:02.609 | INFO | Downloaded 34GiB to GPU in 2.11015s (16.1126GiB/s) | 20260505 13:58:07.739 | INFO | Analyzing chunk frames = [3745280,3745792) | 20260505 13:58:07.739 | INFO | MemInfo: { | 20260505 13:58:07.739 | INFO | "gpu" : " 17 GB", | 20260505 13:58:07.739 | INFO | "huge" : "", | 20260505 13:58:07.739 | INFO | "huge_shared" : " 27 GB", | 20260505 13:58:07.739 | INFO | "malloc" : " 14 GB", | 20260505 13:58:07.739 | INFO | "pinned" : " 29 GB" | 20260505 13:58:07.739 | INFO | } | 20260505 13:58:07.739 | INFO | Perf report for the last 5.1299 seconds: | 20260505 13:58:07.739 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:58:07.739 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:07.739 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:07.739 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:07.739 | INFO | ANALYSIS : 63.2%, 3.0, 97.9ms, 0 | 20260505 13:58:07.739 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:07.739 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:07.739 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 13:58:07.739 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:07.739 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:07.739 | INFO | Uploaded 36GiB to GPU in 2.20192s (16.3493GiB/s) | 20260505 13:58:07.739 | INFO | Downloaded 34GiB to GPU in 2.13732s (15.9078GiB/s) | 20260505 13:58:12.885 | INFO | Analyzing chunk frames = [3745792,3746304) | 20260505 13:58:12.885 | INFO | MemInfo: { | 20260505 13:58:12.885 | INFO | "gpu" : " 17 GB", | 20260505 13:58:12.885 | INFO | "huge" : "", | 20260505 13:58:12.885 | INFO | "huge_shared" : " 27 GB", | 20260505 13:58:12.885 | INFO | "malloc" : " 14 GB", | 20260505 13:58:12.885 | INFO | "pinned" : " 29 GB" | 20260505 13:58:12.885 | INFO | } | 20260505 13:58:12.885 | INFO | Perf report for the last 5.14657 seconds: | 20260505 13:58:12.885 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:58:12.885 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:12.885 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:12.885 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:12.885 | INFO | ANALYSIS : 62.7%, 3.0, 98.1ms, 0 | 20260505 13:58:12.885 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:12.885 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:12.885 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 13:58:12.885 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:12.885 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:12.886 | INFO | Uploaded 36GiB to GPU in 2.16262s (16.6465GiB/s) | 20260505 13:58:12.886 | INFO | Downloaded 34GiB to GPU in 2.12806s (15.977GiB/s) | 20260505 13:58:18.026 | INFO | Analyzing chunk frames = [3746304,3746816) | 20260505 13:58:18.026 | INFO | MemInfo: { | 20260505 13:58:18.026 | INFO | "gpu" : " 17 GB", | 20260505 13:58:18.026 | INFO | "huge" : "", | 20260505 13:58:18.026 | INFO | "huge_shared" : " 27 GB", | 20260505 13:58:18.026 | INFO | "malloc" : " 14 GB", | 20260505 13:58:18.026 | INFO | "pinned" : " 29 GB" | 20260505 13:58:18.026 | INFO | } | 20260505 13:58:18.026 | INFO | Perf report for the last 5.141 seconds: | 20260505 13:58:18.026 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:58:18.026 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:18.026 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:18.026 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:18.026 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 13:58:18.026 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:18.026 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:18.026 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:58:18.026 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:18.026 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:18.027 | INFO | Uploaded 36GiB to GPU in 2.19376s (16.4102GiB/s) | 20260505 13:58:18.027 | INFO | Downloaded 34GiB to GPU in 2.11393s (16.0838GiB/s) | 20260505 13:58:23.166 | INFO | Analyzing chunk frames = [3746816,3747328) | 20260505 13:58:23.166 | INFO | MemInfo: { | 20260505 13:58:23.166 | INFO | "gpu" : " 17 GB", | 20260505 13:58:23.166 | INFO | "huge" : "", | 20260505 13:58:23.166 | INFO | "huge_shared" : " 27 GB", | 20260505 13:58:23.166 | INFO | "malloc" : " 14 GB", | 20260505 13:58:23.166 | INFO | "pinned" : " 29 GB" | 20260505 13:58:23.166 | INFO | } | 20260505 13:58:23.166 | INFO | Perf report for the last 5.14006 seconds: | 20260505 13:58:23.166 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:58:23.166 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:23.166 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:23.166 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:23.166 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 13:58:23.166 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:23.166 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:23.166 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 13:58:23.166 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:23.166 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:23.167 | INFO | Uploaded 36GiB to GPU in 2.15671s (16.6921GiB/s) | 20260505 13:58:23.167 | INFO | Downloaded 34GiB to GPU in 2.16041s (15.7377GiB/s) | 20260505 13:58:28.298 | INFO | Analyzing chunk frames = [3747328,3747840) | 20260505 13:58:28.298 | INFO | MemInfo: { | 20260505 13:58:28.298 | INFO | "gpu" : " 17 GB", | 20260505 13:58:28.298 | INFO | "huge" : "", | 20260505 13:58:28.298 | INFO | "huge_shared" : " 27 GB", | 20260505 13:58:28.298 | INFO | "malloc" : " 14 GB", | 20260505 13:58:28.298 | INFO | "pinned" : " 29 GB" | 20260505 13:58:28.298 | INFO | } | 20260505 13:58:28.298 | INFO | Perf report for the last 5.13124 seconds: | 20260505 13:58:28.298 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:58:28.298 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:28.298 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:28.298 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:28.298 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 13:58:28.298 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:28.298 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:28.298 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 13:58:28.298 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:28.298 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:28.298 | INFO | Uploaded 36GiB to GPU in 2.20828s (16.3023GiB/s) | 20260505 13:58:28.298 | INFO | Downloaded 34GiB to GPU in 2.10338s (16.1645GiB/s) | 20260505 13:58:28.430 | INFO | Starting a round of Baz IO | 20260505 13:58:31.673 | INFO | Populating RTMetrics report | 20260505 13:58:31.673 | INFO | RTMetrics report populated | 20260505 13:58:31.673 | INFO | Opening tmp RTMetrics json file | 20260505 13:58:31.688 | INFO | Serializing RTMetrics to json | 20260505 13:58:31.711 | INFO | Serialization done, swapping tmp file now... | 20260505 13:58:31.733 | INFO | RTMetrics done | 20260505 13:58:31.769 | INFO | Finished a round of Baz IO: Wrote 2.279 GiB in 3.339 seconds | 20260505 13:58:31.770 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.92% / 0.00% / 77.08% | 20260505 13:58:31.770 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.623731% / 0% / 6.75519e-05% | 20260505 13:58:31.770 | INFO | Baz IO overall: | 20260505 13:58:31.770 | INFO | BytesWritten : 2233050789839 | 20260505 13:58:31.770 | INFO | NumEvents : 1062047502895 | 20260505 13:58:33.438 | INFO | Analyzing chunk frames = [3747840,3748352) | 20260505 13:58:33.438 | INFO | MemInfo: { | 20260505 13:58:33.438 | INFO | "gpu" : " 17 GB", | 20260505 13:58:33.438 | INFO | "huge" : "", | 20260505 13:58:33.438 | INFO | "huge_shared" : " 27 GB", | 20260505 13:58:33.438 | INFO | "malloc" : " 14 GB", | 20260505 13:58:33.438 | INFO | "pinned" : " 29 GB" | 20260505 13:58:33.438 | INFO | } | 20260505 13:58:33.438 | INFO | Perf report for the last 5.1407 seconds: | 20260505 13:58:33.438 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:58:33.438 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:33.438 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:33.438 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:33.438 | INFO | ANALYSIS : 64.5%, 3.0, 100.1ms, 0 | 20260505 13:58:33.438 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:33.438 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260505 13:58:33.438 | INFO | PRE_HQ : 60.7%, 1.9, 58.4ms, 0 | 20260505 13:58:33.439 | INFO | BAZWRITER : 64.2%, 4.0, 132.4ms, 0 | 20260505 13:58:33.439 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 13:58:33.439 | INFO | Uploaded 36GiB to GPU in 2.18369s (16.4858GiB/s) | 20260505 13:58:33.439 | INFO | Downloaded 38GiB to GPU in 2.28363s (16.6401GiB/s) | 20260505 13:58:38.582 | INFO | Analyzing chunk frames = [3748352,3748864) | 20260505 13:58:38.582 | INFO | MemInfo: { | 20260505 13:58:38.582 | INFO | "gpu" : " 17 GB", | 20260505 13:58:38.582 | INFO | "huge" : "", | 20260505 13:58:38.582 | INFO | "huge_shared" : " 27 GB", | 20260505 13:58:38.582 | INFO | "malloc" : " 14 GB", | 20260505 13:58:38.582 | INFO | "pinned" : " 29 GB" | 20260505 13:58:38.582 | INFO | } | 20260505 13:58:38.582 | INFO | Perf report for the last 5.14348 seconds: | 20260505 13:58:38.582 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:58:38.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:38.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:38.582 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:38.582 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 13:58:38.582 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:38.582 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:38.582 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:58:38.582 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:38.582 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:38.582 | INFO | Uploaded 36GiB to GPU in 2.17453s (16.5553GiB/s) | 20260505 13:58:38.582 | INFO | Downloaded 34GiB to GPU in 2.02368s (16.8011GiB/s) | 20260505 13:58:43.724 | INFO | Analyzing chunk frames = [3748864,3749376) | 20260505 13:58:43.724 | INFO | MemInfo: { | 20260505 13:58:43.724 | INFO | "gpu" : " 17 GB", | 20260505 13:58:43.724 | INFO | "huge" : "", | 20260505 13:58:43.724 | INFO | "huge_shared" : " 27 GB", | 20260505 13:58:43.724 | INFO | "malloc" : " 14 GB", | 20260505 13:58:43.724 | INFO | "pinned" : " 29 GB" | 20260505 13:58:43.724 | INFO | } | 20260505 13:58:43.724 | INFO | Perf report for the last 5.14186 seconds: | 20260505 13:58:43.724 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:58:43.724 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:43.724 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:43.724 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:43.724 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 13:58:43.724 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:43.724 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:43.724 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:58:43.724 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:43.724 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:43.724 | INFO | Uploaded 36GiB to GPU in 2.22991s (16.1441GiB/s) | 20260505 13:58:43.724 | INFO | Downloaded 34GiB to GPU in 2.1009s (16.1836GiB/s) | 20260505 13:58:48.858 | INFO | Analyzing chunk frames = [3749376,3749888) | 20260505 13:58:48.858 | INFO | MemInfo: { | 20260505 13:58:48.858 | INFO | "gpu" : " 17 GB", | 20260505 13:58:48.858 | INFO | "huge" : "", | 20260505 13:58:48.858 | INFO | "huge_shared" : " 27 GB", | 20260505 13:58:48.858 | INFO | "malloc" : " 14 GB", | 20260505 13:58:48.858 | INFO | "pinned" : " 29 GB" | 20260505 13:58:48.858 | INFO | } | 20260505 13:58:48.858 | INFO | Perf report for the last 5.13419 seconds: | 20260505 13:58:48.858 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:58:48.858 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:48.858 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:48.858 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:48.858 | INFO | ANALYSIS : 62.9%, 3.0, 97.5ms, 0 | 20260505 13:58:48.858 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:48.858 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:48.858 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 13:58:48.858 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:48.858 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:48.858 | INFO | Uploaded 36GiB to GPU in 2.18898s (16.446GiB/s) | 20260505 13:58:48.858 | INFO | Downloaded 34GiB to GPU in 2.13265s (15.9426GiB/s) | 20260505 13:58:53.992 | INFO | Analyzing chunk frames = [3749888,3750400) | 20260505 13:58:53.992 | INFO | MemInfo: { | 20260505 13:58:53.992 | INFO | "gpu" : " 17 GB", | 20260505 13:58:53.992 | INFO | "huge" : "", | 20260505 13:58:53.992 | INFO | "huge_shared" : " 27 GB", | 20260505 13:58:53.992 | INFO | "malloc" : " 14 GB", | 20260505 13:58:53.992 | INFO | "pinned" : " 29 GB" | 20260505 13:58:53.993 | INFO | } | 20260505 13:58:53.993 | INFO | Perf report for the last 5.13459 seconds: | 20260505 13:58:53.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:58:53.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:53.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:53.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:53.993 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260505 13:58:53.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:53.993 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:53.993 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 13:58:53.993 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:53.993 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:53.993 | INFO | Uploaded 36GiB to GPU in 2.15293s (16.7214GiB/s) | 20260505 13:58:53.993 | INFO | Downloaded 34GiB to GPU in 2.12529s (15.9978GiB/s) | 20260505 13:58:59.145 | INFO | Analyzing chunk frames = [3750400,3750912) | 20260505 13:58:59.145 | INFO | MemInfo: { | 20260505 13:58:59.145 | INFO | "gpu" : " 17 GB", | 20260505 13:58:59.145 | INFO | "huge" : "", | 20260505 13:58:59.145 | INFO | "huge_shared" : " 27 GB", | 20260505 13:58:59.145 | INFO | "malloc" : " 14 GB", | 20260505 13:58:59.145 | INFO | "pinned" : " 29 GB" | 20260505 13:58:59.145 | INFO | } | 20260505 13:58:59.145 | INFO | Perf report for the last 5.15213 seconds: | 20260505 13:58:59.145 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:58:59.145 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:59.145 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:59.145 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:59.145 | INFO | ANALYSIS : 62.8%, 3.0, 98.4ms, 0 | 20260505 13:58:59.145 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:59.145 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:59.145 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 13:58:59.145 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:59.145 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:58:59.145 | INFO | Uploaded 36GiB to GPU in 2.13292s (16.8783GiB/s) | 20260505 13:58:59.145 | INFO | Downloaded 34GiB to GPU in 2.10357s (16.163GiB/s) | 20260505 13:59:04.284 | INFO | Analyzing chunk frames = [3750912,3751424) | 20260505 13:59:04.284 | INFO | MemInfo: { | 20260505 13:59:04.284 | INFO | "gpu" : " 17 GB", | 20260505 13:59:04.284 | INFO | "huge" : "", | 20260505 13:59:04.284 | INFO | "huge_shared" : " 27 GB", | 20260505 13:59:04.284 | INFO | "malloc" : " 14 GB", | 20260505 13:59:04.284 | INFO | "pinned" : " 29 GB" | 20260505 13:59:04.284 | INFO | } | 20260505 13:59:04.284 | INFO | Perf report for the last 5.13909 seconds: | 20260505 13:59:04.284 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:59:04.284 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:04.284 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:04.284 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:04.284 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 13:59:04.284 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:04.284 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:04.284 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260505 13:59:04.284 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:04.284 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:04.284 | INFO | Uploaded 36GiB to GPU in 2.137s (16.846GiB/s) | 20260505 13:59:04.284 | INFO | Downloaded 34GiB to GPU in 2.17602s (15.6248GiB/s) | 20260505 13:59:09.414 | INFO | Analyzing chunk frames = [3751424,3751936) | 20260505 13:59:09.414 | INFO | MemInfo: { | 20260505 13:59:09.414 | INFO | "gpu" : " 17 GB", | 20260505 13:59:09.414 | INFO | "huge" : "", | 20260505 13:59:09.414 | INFO | "huge_shared" : " 27 GB", | 20260505 13:59:09.414 | INFO | "malloc" : " 14 GB", | 20260505 13:59:09.414 | INFO | "pinned" : " 29 GB" | 20260505 13:59:09.414 | INFO | } | 20260505 13:59:09.414 | INFO | Perf report for the last 5.13035 seconds: | 20260505 13:59:09.414 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:59:09.414 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:09.414 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:09.414 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:09.414 | INFO | ANALYSIS : 63.7%, 3.0, 99.2ms, 0 | 20260505 13:59:09.414 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:09.414 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:09.414 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 13:59:09.414 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:09.414 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:09.414 | INFO | Uploaded 36GiB to GPU in 2.21346s (16.2641GiB/s) | 20260505 13:59:09.414 | INFO | Downloaded 34GiB to GPU in 2.09626s (16.2193GiB/s) | 20260505 13:59:09.414 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 13:59:09.414 | INFO | Total mem usage: 15.3 GiB | 20260505 13:59:09.414 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 13:59:09.415 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.65%) | 20260505 13:59:09.415 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 13:59:09.415 | INFO | High Water Breakdown: | 20260505 13:59:09.415 | INFO | 61% : 8.68 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 13:59:09.415 | INFO | 38.9% : 5.54 GiB ( 83.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 13:59:09.415 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 13:59:09.415 | INFO | ------------------------------------------ | 20260505 13:59:09.415 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:59:09.415 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 13:59:09.415 | INFO | Total mem usage: 26.5 GiB | 20260505 13:59:09.415 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 13:59:09.415 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 13:59:09.415 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 13:59:09.415 | INFO | High Water Breakdown: | 20260505 13:59:09.415 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:59:09.415 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:59:09.415 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:59:09.415 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:59:09.415 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:59:09.415 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:59:09.415 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 13:59:09.415 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 13:59:09.415 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:59:09.415 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 13:59:09.415 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 13:59:09.415 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 13:59:09.415 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 13:59:09.415 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 13:59:09.415 | INFO | ------------------------------------------ | 20260505 13:59:09.415 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:59:09.415 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 13:59:09.415 | INFO | Total mem usage: 25.6 GiB | 20260505 13:59:09.415 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 13:59:09.415 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 13:59:09.415 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 13:59:09.415 | INFO | High Water Breakdown: | 20260505 13:59:09.415 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 13:59:09.415 | INFO | ------------------------------------------ | 20260505 13:59:09.415 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:59:09.415 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 13:59:09.415 | INFO | Total mem usage: 15.7 GiB | 20260505 13:59:09.415 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 13:59:09.415 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 13:59:09.415 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 13:59:09.415 | INFO | High Water Breakdown: | 20260505 13:59:09.415 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 13:59:09.415 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 13:59:09.415 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 13:59:09.415 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 13:59:09.415 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 13:59:09.415 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 13:59:09.415 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 13:59:09.415 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 13:59:09.415 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 13:59:09.415 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 13:59:09.415 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 13:59:09.415 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 13:59:09.415 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 13:59:09.415 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 13:59:09.415 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 13:59:09.415 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 13:59:09.415 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 13:59:09.415 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 13:59:09.415 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 13:59:09.415 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 13:59:09.415 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 13:59:09.415 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 13:59:09.415 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 13:59:09.415 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 13:59:09.415 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 13:59:09.415 | INFO | ------------------------------------------ | 20260505 13:59:09.415 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 13:59:09.415 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 13:59:09.415 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:59:09.415 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:59:09.415 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.65e+01, 1.37e+01 / 1.83e+00 | 20260505 13:59:09.415 | INFO | PerfReport: 0.29%, AggregateStats -- 9.32e-02 / 9.98e-02, 1.08e-02 / 3.92e-03 | 20260505 13:59:09.415 | INFO | PerfReport: 11.80%, Baseline -- 3.76e+00 / 4.46e+00, 5.96e-01 / 2.34e-01 | 20260505 13:59:09.415 | INFO | PerfReport: 3.31%, Histogram -- 1.05e+00 / 1.50e+00, 2.01e-01 / 1.03e-01 | 20260505 13:59:09.415 | INFO | PerfReport: 16.05%, DME -- 4.09e+01 / 4.60e+01, 4.65e+00 / 1.94e+00 | 20260505 13:59:09.415 | INFO | PerfReport: 56.30%, FrameLabeling -- 1.79e+01 / 1.97e+01, 1.35e+00 / 1.91e+00 | 20260505 13:59:09.415 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.37e+00 / 2.93e+00, 4.52e-01 / 1.98e-01 | 20260505 13:59:09.415 | INFO | PerfReport: 3.56%, Metrics -- 1.13e+00 / 1.46e+00, 3.20e-01 / 1.77e-01 | 20260505 13:59:09.415 | INFO | PerfReport: 1.25%, PulseSerializing -- 3.97e-01 / 4.46e-01, 7.04e-02 / 3.19e-02 | 20260505 13:59:09.415 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 13:59:09.415 | INFO | PerfReport: Gathered 1536 samples. | 20260505 13:59:09.415 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 13:59:09.416 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260505 13:59:09.416 | INFO | PerfReport: 22.56%, Upload -- 2.27e+01 / 0.00e+00, 8.12e+00 / 0.00e+00 | 20260505 13:59:09.416 | INFO | PerfReport: 22.00%, Download -- 2.22e+01 / 0.00e+00, 8.09e+00 / 0.00e+00 | 20260505 13:59:09.416 | INFO | PerfReport: 31.58%, Compute -- 3.18e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 13:59:09.416 | INFO | PerfReport: 2.27%, LockUpload -- 2.29e+00 / 0.00e+00, 8.06e+00 / 0.00e+00 | 20260505 13:59:09.416 | INFO | PerfReport: 1.46%, LockDownload -- 1.48e+00 / 0.00e+00, 2.99e+00 / 0.00e+00 | 20260505 13:59:09.416 | INFO | PerfReport: 20.11%, LockCompute -- 2.03e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260505 13:59:09.416 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.07e-02 / 0.00e+00 | 20260505 13:59:09.549 | INFO | Starting a round of Baz IO | 20260505 13:59:12.785 | INFO | Populating RTMetrics report | 20260505 13:59:12.785 | INFO | RTMetrics report populated | 20260505 13:59:12.785 | INFO | Opening tmp RTMetrics json file | 20260505 13:59:12.811 | INFO | Serializing RTMetrics to json | 20260505 13:59:12.856 | INFO | Serialization done, swapping tmp file now... | 20260505 13:59:12.878 | INFO | RTMetrics done | 20260505 13:59:12.892 | INFO | Finished a round of Baz IO: Wrote 2.278 GiB in 3.342 seconds | 20260505 13:59:12.892 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.91% / 0.00% / 77.08% | 20260505 13:59:12.892 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.621955% / 7.94729e-06% / 9.13938e-05% | 20260505 13:59:12.892 | INFO | Baz IO overall: | 20260505 13:59:12.892 | INFO | BytesWritten : 2235496751455 | 20260505 13:59:12.892 | INFO | NumEvents : 1063202832698 | 20260505 13:59:14.552 | INFO | Analyzing chunk frames = [3751936,3752448) | 20260505 13:59:14.552 | INFO | MemInfo: { | 20260505 13:59:14.552 | INFO | "gpu" : " 17 GB", | 20260505 13:59:14.552 | INFO | "huge" : "", | 20260505 13:59:14.552 | INFO | "huge_shared" : " 27 GB", | 20260505 13:59:14.552 | INFO | "malloc" : " 14 GB", | 20260505 13:59:14.553 | INFO | "pinned" : " 29 GB" | 20260505 13:59:14.553 | INFO | } | 20260505 13:59:14.553 | INFO | Perf report for the last 5.13845 seconds: | 20260505 13:59:14.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:59:14.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:14.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:14.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:14.553 | INFO | ANALYSIS : 64.5%, 3.0, 100.1ms, 0 | 20260505 13:59:14.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:14.553 | INFO | RT_METRICS : 47.0%, 1.0, 24.7ms, 0 | 20260505 13:59:14.553 | INFO | PRE_HQ : 60.6%, 1.8, 57.9ms, 0 | 20260505 13:59:14.553 | INFO | BAZWRITER : 64.3%, 4.0, 132.9ms, 0 | 20260505 13:59:14.553 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260505 13:59:14.553 | INFO | Uploaded 36GiB to GPU in 2.20502s (16.3264GiB/s) | 20260505 13:59:14.553 | INFO | Downloaded 38GiB to GPU in 2.27677s (16.6903GiB/s) | 20260505 13:59:19.693 | INFO | Analyzing chunk frames = [3752448,3752960) | 20260505 13:59:19.693 | INFO | MemInfo: { | 20260505 13:59:19.693 | INFO | "gpu" : " 17 GB", | 20260505 13:59:19.693 | INFO | "huge" : "", | 20260505 13:59:19.693 | INFO | "huge_shared" : " 27 GB", | 20260505 13:59:19.693 | INFO | "malloc" : " 14 GB", | 20260505 13:59:19.693 | INFO | "pinned" : " 29 GB" | 20260505 13:59:19.693 | INFO | } | 20260505 13:59:19.693 | INFO | Perf report for the last 5.14017 seconds: | 20260505 13:59:19.693 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:59:19.693 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:19.693 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:19.693 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:19.693 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 13:59:19.693 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:19.693 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:19.693 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 13:59:19.693 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:19.693 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:19.693 | INFO | Uploaded 36GiB to GPU in 2.17223s (16.5729GiB/s) | 20260505 13:59:19.693 | INFO | Downloaded 34GiB to GPU in 2.03659s (16.6946GiB/s) | 20260505 13:59:24.838 | INFO | Analyzing chunk frames = [3752960,3753472) | 20260505 13:59:24.838 | INFO | MemInfo: { | 20260505 13:59:24.838 | INFO | "gpu" : " 17 GB", | 20260505 13:59:24.838 | INFO | "huge" : "", | 20260505 13:59:24.838 | INFO | "huge_shared" : " 27 GB", | 20260505 13:59:24.838 | INFO | "malloc" : " 14 GB", | 20260505 13:59:24.838 | INFO | "pinned" : " 29 GB" | 20260505 13:59:24.838 | INFO | } | 20260505 13:59:24.838 | INFO | Perf report for the last 5.14496 seconds: | 20260505 13:59:24.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:59:24.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:24.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:24.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:24.838 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 13:59:24.838 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:24.838 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:24.838 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 13:59:24.838 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:24.838 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:24.838 | INFO | Uploaded 36GiB to GPU in 2.23219s (16.1277GiB/s) | 20260505 13:59:24.838 | INFO | Downloaded 34GiB to GPU in 2.10219s (16.1736GiB/s) | 20260505 13:59:29.979 | INFO | Analyzing chunk frames = [3753472,3753984) | 20260505 13:59:29.979 | INFO | MemInfo: { | 20260505 13:59:29.979 | INFO | "gpu" : " 17 GB", | 20260505 13:59:29.979 | INFO | "huge" : "", | 20260505 13:59:29.979 | INFO | "huge_shared" : " 27 GB", | 20260505 13:59:29.979 | INFO | "malloc" : " 14 GB", | 20260505 13:59:29.979 | INFO | "pinned" : " 29 GB" | 20260505 13:59:29.979 | INFO | } | 20260505 13:59:29.979 | INFO | Perf report for the last 5.14117 seconds: | 20260505 13:59:29.979 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:59:29.979 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:29.979 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:29.979 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:29.979 | INFO | ANALYSIS : 62.7%, 3.0, 97.4ms, 0 | 20260505 13:59:29.979 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:29.979 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:29.979 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 13:59:29.979 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:29.979 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:29.979 | INFO | Uploaded 36GiB to GPU in 2.1761s (16.5434GiB/s) | 20260505 13:59:29.979 | INFO | Downloaded 34GiB to GPU in 2.12843s (15.9742GiB/s) | 20260505 13:59:35.113 | INFO | Analyzing chunk frames = [3753984,3754496) | 20260505 13:59:35.113 | INFO | MemInfo: { | 20260505 13:59:35.113 | INFO | "gpu" : " 17 GB", | 20260505 13:59:35.113 | INFO | "huge" : "", | 20260505 13:59:35.113 | INFO | "huge_shared" : " 27 GB", | 20260505 13:59:35.113 | INFO | "malloc" : " 14 GB", | 20260505 13:59:35.113 | INFO | "pinned" : " 29 GB" | 20260505 13:59:35.114 | INFO | } | 20260505 13:59:35.114 | INFO | Perf report for the last 5.13458 seconds: | 20260505 13:59:35.114 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:59:35.114 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:35.114 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:35.114 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:35.114 | INFO | ANALYSIS : 62.8%, 3.0, 97.0ms, 0 | 20260505 13:59:35.114 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:35.114 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:35.114 | INFO | PRE_HQ : 28.4%, 1.0, 14.6ms, 0 | 20260505 13:59:35.114 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:35.114 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:35.114 | INFO | Uploaded 36GiB to GPU in 2.16123s (16.6572GiB/s) | 20260505 13:59:35.114 | INFO | Downloaded 34GiB to GPU in 2.12536s (15.9973GiB/s) | 20260505 13:59:40.245 | INFO | Analyzing chunk frames = [3754496,3755008) | 20260505 13:59:40.245 | INFO | MemInfo: { | 20260505 13:59:40.245 | INFO | "gpu" : " 17 GB", | 20260505 13:59:40.245 | INFO | "huge" : "", | 20260505 13:59:40.245 | INFO | "huge_shared" : " 27 GB", | 20260505 13:59:40.245 | INFO | "malloc" : " 14 GB", | 20260505 13:59:40.245 | INFO | "pinned" : " 29 GB" | 20260505 13:59:40.245 | INFO | } | 20260505 13:59:40.245 | INFO | Perf report for the last 5.1318 seconds: | 20260505 13:59:40.245 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:59:40.245 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:40.245 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:40.245 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:40.245 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 13:59:40.245 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:40.245 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:40.245 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 13:59:40.245 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:40.245 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:40.246 | INFO | Uploaded 36GiB to GPU in 2.18296s (16.4914GiB/s) | 20260505 13:59:40.246 | INFO | Downloaded 34GiB to GPU in 2.10751s (16.1327GiB/s) | 20260505 13:59:45.390 | INFO | Analyzing chunk frames = [3755008,3755520) | 20260505 13:59:45.390 | INFO | MemInfo: { | 20260505 13:59:45.390 | INFO | "gpu" : " 17 GB", | 20260505 13:59:45.390 | INFO | "huge" : "", | 20260505 13:59:45.390 | INFO | "huge_shared" : " 27 GB", | 20260505 13:59:45.390 | INFO | "malloc" : " 14 GB", | 20260505 13:59:45.390 | INFO | "pinned" : " 29 GB" | 20260505 13:59:45.390 | INFO | } | 20260505 13:59:45.390 | INFO | Perf report for the last 5.14494 seconds: | 20260505 13:59:45.390 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:59:45.390 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:45.390 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:45.390 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:45.390 | INFO | ANALYSIS : 62.7%, 3.0, 98.1ms, 0 | 20260505 13:59:45.390 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:45.390 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:45.390 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260505 13:59:45.390 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:45.390 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:45.391 | INFO | Uploaded 36GiB to GPU in 2.12644s (16.9297GiB/s) | 20260505 13:59:45.391 | INFO | Downloaded 34GiB to GPU in 2.13683s (15.9114GiB/s) | 20260505 13:59:50.535 | INFO | Analyzing chunk frames = [3755520,3756032) | 20260505 13:59:50.535 | INFO | MemInfo: { | 20260505 13:59:50.535 | INFO | "gpu" : " 17 GB", | 20260505 13:59:50.535 | INFO | "huge" : "", | 20260505 13:59:50.535 | INFO | "huge_shared" : " 27 GB", | 20260505 13:59:50.535 | INFO | "malloc" : " 14 GB", | 20260505 13:59:50.535 | INFO | "pinned" : " 29 GB" | 20260505 13:59:50.535 | INFO | } | 20260505 13:59:50.535 | INFO | Perf report for the last 5.14473 seconds: | 20260505 13:59:50.535 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:59:50.535 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:50.535 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:50.535 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:50.535 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 13:59:50.535 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:50.535 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:50.535 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 13:59:50.535 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:50.535 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:50.535 | INFO | Uploaded 36GiB to GPU in 2.21629s (16.2433GiB/s) | 20260505 13:59:50.535 | INFO | Downloaded 34GiB to GPU in 2.10549s (16.1483GiB/s) | 20260505 13:59:50.675 | INFO | Starting a round of Baz IO | 20260505 13:59:53.898 | INFO | Populating RTMetrics report | 20260505 13:59:53.898 | INFO | RTMetrics report populated | 20260505 13:59:53.898 | INFO | Opening tmp RTMetrics json file | 20260505 13:59:53.944 | INFO | Serializing RTMetrics to json | 20260505 13:59:53.969 | INFO | Serialization done, swapping tmp file now... | 20260505 13:59:53.970 | INFO | RTMetrics done | 20260505 13:59:54.003 | INFO | Finished a round of Baz IO: Wrote 2.273 GiB in 3.328 seconds | 20260505 13:59:54.003 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.91% / 0.00% / 77.09% | 20260505 13:59:54.003 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.616503% / 1.19209e-05% / 7.94729e-05% | 20260505 13:59:54.003 | INFO | Baz IO overall: | 20260505 13:59:54.003 | INFO | BytesWritten : 2237937579139 | 20260505 13:59:54.003 | INFO | NumEvents : 1064354567754 | 20260505 13:59:55.675 | INFO | Analyzing chunk frames = [3756032,3756544) | 20260505 13:59:55.675 | INFO | MemInfo: { | 20260505 13:59:55.675 | INFO | "gpu" : " 17 GB", | 20260505 13:59:55.675 | INFO | "huge" : "", | 20260505 13:59:55.675 | INFO | "huge_shared" : " 27 GB", | 20260505 13:59:55.675 | INFO | "malloc" : " 14 GB", | 20260505 13:59:55.675 | INFO | "pinned" : " 29 GB" | 20260505 13:59:55.675 | INFO | } | 20260505 13:59:55.675 | INFO | Perf report for the last 5.14027 seconds: | 20260505 13:59:55.675 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 13:59:55.675 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:55.675 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:55.675 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:55.675 | INFO | ANALYSIS : 64.3%, 3.0, 100.0ms, 0 | 20260505 13:59:55.675 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 13:59:55.675 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260505 13:59:55.675 | INFO | PRE_HQ : 61.0%, 1.8, 57.6ms, 0 | 20260505 13:59:55.675 | INFO | BAZWRITER : 64.1%, 4.0, 132.2ms, 0 | 20260505 13:59:55.675 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260505 13:59:55.676 | INFO | Uploaded 36GiB to GPU in 2.19668s (16.3884GiB/s) | 20260505 13:59:55.676 | INFO | Downloaded 38GiB to GPU in 2.2612s (16.8053GiB/s) | 20260505 14:00:00.810 | INFO | Analyzing chunk frames = [3756544,3757056) | 20260505 14:00:00.810 | INFO | MemInfo: { | 20260505 14:00:00.810 | INFO | "gpu" : " 17 GB", | 20260505 14:00:00.810 | INFO | "huge" : "", | 20260505 14:00:00.810 | INFO | "huge_shared" : " 27 GB", | 20260505 14:00:00.810 | INFO | "malloc" : " 14 GB", | 20260505 14:00:00.810 | INFO | "pinned" : " 29 GB" | 20260505 14:00:00.810 | INFO | } | 20260505 14:00:00.810 | INFO | Perf report for the last 5.13481 seconds: | 20260505 14:00:00.810 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:00:00.810 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:00.810 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:00.810 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:00.810 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 14:00:00.810 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:00.810 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:00.810 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 14:00:00.810 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:00.810 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:00.810 | INFO | Uploaded 36GiB to GPU in 2.20369s (16.3362GiB/s) | 20260505 14:00:00.810 | INFO | Downloaded 34GiB to GPU in 2.0428s (16.6438GiB/s) | 20260505 14:00:05.950 | INFO | Analyzing chunk frames = [3757056,3757568) | 20260505 14:00:05.950 | INFO | MemInfo: { | 20260505 14:00:05.950 | INFO | "gpu" : " 17 GB", | 20260505 14:00:05.951 | INFO | "huge" : "", | 20260505 14:00:05.951 | INFO | "huge_shared" : " 27 GB", | 20260505 14:00:05.951 | INFO | "malloc" : " 14 GB", | 20260505 14:00:05.951 | INFO | "pinned" : " 29 GB" | 20260505 14:00:05.951 | INFO | } | 20260505 14:00:05.951 | INFO | Perf report for the last 5.14051 seconds: | 20260505 14:00:05.951 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:00:05.951 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:05.951 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:05.951 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:05.951 | INFO | ANALYSIS : 63.2%, 3.0, 98.4ms, 0 | 20260505 14:00:05.951 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:05.951 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:05.951 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 14:00:05.951 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:05.951 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:05.951 | INFO | Uploaded 36GiB to GPU in 2.20169s (16.3511GiB/s) | 20260505 14:00:05.951 | INFO | Downloaded 34GiB to GPU in 2.12096s (16.0305GiB/s) | 20260505 14:00:11.097 | INFO | Analyzing chunk frames = [3757568,3758080) | 20260505 14:00:11.097 | INFO | MemInfo: { | 20260505 14:00:11.097 | INFO | "gpu" : " 17 GB", | 20260505 14:00:11.097 | INFO | "huge" : "", | 20260505 14:00:11.098 | INFO | "huge_shared" : " 27 GB", | 20260505 14:00:11.098 | INFO | "malloc" : " 14 GB", | 20260505 14:00:11.098 | INFO | "pinned" : " 29 GB" | 20260505 14:00:11.098 | INFO | } | 20260505 14:00:11.098 | INFO | Perf report for the last 5.14694 seconds: | 20260505 14:00:11.098 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:00:11.098 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:11.098 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:11.098 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:11.098 | INFO | ANALYSIS : 62.8%, 3.0, 97.6ms, 0 | 20260505 14:00:11.098 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:11.098 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:11.098 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 14:00:11.098 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:11.098 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:11.098 | INFO | Uploaded 36GiB to GPU in 2.19699s (16.3861GiB/s) | 20260505 14:00:11.098 | INFO | Downloaded 34GiB to GPU in 2.13756s (15.906GiB/s) | 20260505 14:00:16.234 | INFO | Analyzing chunk frames = [3758080,3758592) | 20260505 14:00:16.234 | INFO | MemInfo: { | 20260505 14:00:16.234 | INFO | "gpu" : " 17 GB", | 20260505 14:00:16.234 | INFO | "huge" : "", | 20260505 14:00:16.235 | INFO | "huge_shared" : " 27 GB", | 20260505 14:00:16.235 | INFO | "malloc" : " 14 GB", | 20260505 14:00:16.235 | INFO | "pinned" : " 29 GB" | 20260505 14:00:16.235 | INFO | } | 20260505 14:00:16.235 | INFO | Perf report for the last 5.13705 seconds: | 20260505 14:00:16.235 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:00:16.235 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:16.235 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:16.235 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:16.235 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260505 14:00:16.235 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:16.235 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:16.235 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:00:16.235 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:16.235 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:16.235 | INFO | Uploaded 36GiB to GPU in 2.13419s (16.8682GiB/s) | 20260505 14:00:16.235 | INFO | Downloaded 34GiB to GPU in 2.05177s (16.571GiB/s) | 20260505 14:00:21.362 | INFO | Analyzing chunk frames = [3758592,3759104) | 20260505 14:00:21.362 | INFO | MemInfo: { | 20260505 14:00:21.362 | INFO | "gpu" : " 17 GB", | 20260505 14:00:21.362 | INFO | "huge" : "", | 20260505 14:00:21.362 | INFO | "huge_shared" : " 27 GB", | 20260505 14:00:21.362 | INFO | "malloc" : " 14 GB", | 20260505 14:00:21.362 | INFO | "pinned" : " 29 GB" | 20260505 14:00:21.362 | INFO | } | 20260505 14:00:21.362 | INFO | Perf report for the last 5.12745 seconds: | 20260505 14:00:21.362 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:00:21.362 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:21.362 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:21.362 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:21.362 | INFO | ANALYSIS : 63.2%, 3.0, 98.4ms, 0 | 20260505 14:00:21.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:21.362 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:21.362 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:00:21.362 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:21.362 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:21.362 | INFO | Uploaded 36GiB to GPU in 2.19243s (16.4201GiB/s) | 20260505 14:00:21.363 | INFO | Downloaded 34GiB to GPU in 2.11186s (16.0995GiB/s) | 20260505 14:00:26.501 | INFO | Analyzing chunk frames = [3759104,3759616) | 20260505 14:00:26.501 | INFO | MemInfo: { | 20260505 14:00:26.501 | INFO | "gpu" : " 17 GB", | 20260505 14:00:26.501 | INFO | "huge" : "", | 20260505 14:00:26.501 | INFO | "huge_shared" : " 27 GB", | 20260505 14:00:26.501 | INFO | "malloc" : " 14 GB", | 20260505 14:00:26.501 | INFO | "pinned" : " 29 GB" | 20260505 14:00:26.501 | INFO | } | 20260505 14:00:26.502 | INFO | Perf report for the last 5.13931 seconds: | 20260505 14:00:26.502 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:00:26.502 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:26.502 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:26.502 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:26.502 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 14:00:26.502 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:26.502 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:26.502 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 14:00:26.502 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:26.502 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:26.502 | INFO | Uploaded 36GiB to GPU in 2.17918s (16.52GiB/s) | 20260505 14:00:26.502 | INFO | Downloaded 34GiB to GPU in 2.15002s (15.8138GiB/s) | 20260505 14:00:31.654 | INFO | Analyzing chunk frames = [3759616,3760128) | 20260505 14:00:31.654 | INFO | MemInfo: { | 20260505 14:00:31.654 | INFO | "gpu" : " 17 GB", | 20260505 14:00:31.654 | INFO | "huge" : "", | 20260505 14:00:31.654 | INFO | "huge_shared" : " 27 GB", | 20260505 14:00:31.654 | INFO | "malloc" : " 14 GB", | 20260505 14:00:31.654 | INFO | "pinned" : " 29 GB" | 20260505 14:00:31.654 | INFO | } | 20260505 14:00:31.654 | INFO | Perf report for the last 5.15295 seconds: | 20260505 14:00:31.654 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:00:31.654 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:31.654 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:31.654 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:31.654 | INFO | ANALYSIS : 63.4%, 3.0, 99.2ms, 0 | 20260505 14:00:31.655 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:31.655 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:31.655 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 14:00:31.655 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:31.655 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:31.655 | INFO | Uploaded 36GiB to GPU in 2.12502s (16.941GiB/s) | 20260505 14:00:31.655 | INFO | Downloaded 34GiB to GPU in 2.10253s (16.171GiB/s) | 20260505 14:00:31.655 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 14:00:31.655 | INFO | Total mem usage: 15.3 GiB | 20260505 14:00:31.655 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 14:00:31.655 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.66%) | 20260505 14:00:31.655 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 14:00:31.655 | INFO | High Water Breakdown: | 20260505 14:00:31.655 | INFO | 61% : 8.68 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 14:00:31.655 | INFO | 38.9% : 5.54 GiB ( 83.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 14:00:31.655 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 14:00:31.655 | INFO | ------------------------------------------ | 20260505 14:00:31.655 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:00:31.655 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 14:00:31.655 | INFO | Total mem usage: 26.5 GiB | 20260505 14:00:31.655 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 14:00:31.655 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 14:00:31.655 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 14:00:31.655 | INFO | High Water Breakdown: | 20260505 14:00:31.655 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:00:31.655 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:00:31.655 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:00:31.655 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:00:31.655 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:00:31.655 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:00:31.655 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 14:00:31.655 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 14:00:31.655 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:00:31.655 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 14:00:31.655 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 14:00:31.655 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 14:00:31.655 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 14:00:31.655 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 14:00:31.655 | INFO | ------------------------------------------ | 20260505 14:00:31.655 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:00:31.655 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 14:00:31.655 | INFO | Total mem usage: 25.6 GiB | 20260505 14:00:31.655 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 14:00:31.655 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 14:00:31.655 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 14:00:31.655 | INFO | High Water Breakdown: | 20260505 14:00:31.655 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 14:00:31.655 | INFO | ------------------------------------------ | 20260505 14:00:31.655 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:00:31.655 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 14:00:31.655 | INFO | Total mem usage: 15.7 GiB | 20260505 14:00:31.656 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 14:00:31.656 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 14:00:31.656 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260505 14:00:31.656 | INFO | High Water Breakdown: | 20260505 14:00:31.656 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 14:00:31.656 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 14:00:31.656 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 14:00:31.656 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 14:00:31.656 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 14:00:31.656 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 14:00:31.656 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 14:00:31.656 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 14:00:31.656 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 14:00:31.656 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:00:31.656 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 14:00:31.656 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:00:31.656 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 14:00:31.656 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 14:00:31.656 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 14:00:31.656 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 14:00:31.656 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:00:31.656 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 14:00:31.656 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:00:31.656 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:00:31.656 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:00:31.656 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 14:00:31.656 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 14:00:31.656 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:00:31.656 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 14:00:31.656 | INFO | ------------------------------------------ | 20260505 14:00:31.656 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:00:31.656 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 14:00:31.656 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:00:31.656 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:00:31.656 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.65e+01, 1.36e+01 / 1.48e+00 | 20260505 14:00:31.656 | INFO | PerfReport: 0.29%, AggregateStats -- 9.34e-02 / 1.01e-01, 1.10e-02 / 4.91e-03 | 20260505 14:00:31.656 | INFO | PerfReport: 11.78%, Baseline -- 3.75e+00 / 4.50e+00, 5.83e-01 / 1.48e-01 | 20260505 14:00:31.656 | INFO | PerfReport: 3.31%, Histogram -- 1.05e+00 / 1.51e+00, 1.98e-01 / 7.07e-02 | 20260505 14:00:31.656 | INFO | PerfReport: 15.98%, DME -- 4.07e+01 / 4.58e+01, 4.46e+00 / 1.97e+00 | 20260505 14:00:31.656 | INFO | PerfReport: 56.37%, FrameLabeling -- 1.80e+01 / 1.97e+01, 1.41e+00 / 2.07e+00 | 20260505 14:00:31.656 | INFO | PerfReport: 7.47%, PulseAccumulating -- 2.38e+00 / 2.94e+00, 4.55e-01 / 1.85e-01 | 20260505 14:00:31.656 | INFO | PerfReport: 3.56%, Metrics -- 1.13e+00 / 1.45e+00, 3.14e-01 / 1.68e-01 | 20260505 14:00:31.656 | INFO | PerfReport: 1.24%, PulseSerializing -- 3.97e-01 / 4.54e-01, 7.05e-02 / 3.36e-02 | 20260505 14:00:31.656 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 14:00:31.656 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:00:31.656 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:00:31.656 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260505 14:00:31.656 | INFO | PerfReport: 22.57%, Upload -- 2.27e+01 / 0.00e+00, 8.05e+00 / 0.00e+00 | 20260505 14:00:31.656 | INFO | PerfReport: 21.98%, Download -- 2.21e+01 / 0.00e+00, 8.01e+00 / 0.00e+00 | 20260505 14:00:31.656 | INFO | PerfReport: 31.63%, Compute -- 3.19e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 14:00:31.656 | INFO | PerfReport: 2.22%, LockUpload -- 2.23e+00 / 0.00e+00, 7.79e+00 / 0.00e+00 | 20260505 14:00:31.656 | INFO | PerfReport: 1.39%, LockDownload -- 1.40e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260505 14:00:31.656 | INFO | PerfReport: 20.19%, LockCompute -- 2.03e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260505 14:00:31.656 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.82e-02 / 0.00e+00 | 20260505 14:00:31.785 | INFO | Starting a round of Baz IO | 20260505 14:00:35.000 | INFO | Populating RTMetrics report | 20260505 14:00:35.001 | INFO | RTMetrics report populated | 20260505 14:00:35.001 | INFO | Opening tmp RTMetrics json file | 20260505 14:00:35.019 | INFO | Serializing RTMetrics to json | 20260505 14:00:35.042 | INFO | Serialization done, swapping tmp file now... | 20260505 14:00:35.062 | INFO | RTMetrics done | 20260505 14:00:35.103 | INFO | Finished a round of Baz IO: Wrote 2.273 GiB in 3.317 seconds | 20260505 14:00:35.103 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.91% / 0.00% / 77.09% | 20260505 14:00:35.103 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.615072% / 3.97364e-06% / 9.13938e-05% | 20260505 14:00:35.103 | INFO | Baz IO overall: | 20260505 14:00:35.103 | INFO | BytesWritten : 2240378542144 | 20260505 14:00:35.103 | INFO | NumEvents : 1065506542797 | 20260505 14:00:36.795 | INFO | Analyzing chunk frames = [3760128,3760640) | 20260505 14:00:36.795 | INFO | MemInfo: { | 20260505 14:00:36.795 | INFO | "gpu" : " 17 GB", | 20260505 14:00:36.795 | INFO | "huge" : "", | 20260505 14:00:36.795 | INFO | "huge_shared" : " 27 GB", | 20260505 14:00:36.795 | INFO | "malloc" : " 14 GB", | 20260505 14:00:36.795 | INFO | "pinned" : " 29 GB" | 20260505 14:00:36.795 | INFO | } | 20260505 14:00:36.795 | INFO | Perf report for the last 5.14023 seconds: | 20260505 14:00:36.795 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:00:36.795 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:36.795 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:36.795 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:36.795 | INFO | ANALYSIS : 64.0%, 3.0, 99.5ms, 0 | 20260505 14:00:36.795 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:36.795 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260505 14:00:36.795 | INFO | PRE_HQ : 61.0%, 1.8, 57.4ms, 0 | 20260505 14:00:36.795 | INFO | BAZWRITER : 64.0%, 4.0, 131.8ms, 0 | 20260505 14:00:36.795 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 14:00:36.795 | INFO | Uploaded 36GiB to GPU in 2.19926s (16.3691GiB/s) | 20260505 14:00:36.795 | INFO | Downloaded 38GiB to GPU in 2.26359s (16.7875GiB/s) | 20260505 14:00:41.929 | INFO | Analyzing chunk frames = [3760640,3761152) | 20260505 14:00:41.929 | INFO | MemInfo: { | 20260505 14:00:41.929 | INFO | "gpu" : " 17 GB", | 20260505 14:00:41.929 | INFO | "huge" : "", | 20260505 14:00:41.929 | INFO | "huge_shared" : " 27 GB", | 20260505 14:00:41.929 | INFO | "malloc" : " 14 GB", | 20260505 14:00:41.929 | INFO | "pinned" : " 29 GB" | 20260505 14:00:41.929 | INFO | } | 20260505 14:00:41.930 | INFO | Perf report for the last 5.13481 seconds: | 20260505 14:00:41.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:00:41.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:41.930 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:41.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:41.930 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 14:00:41.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:41.930 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:41.930 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:00:41.930 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:41.930 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:41.930 | INFO | Uploaded 36GiB to GPU in 2.20368s (16.3363GiB/s) | 20260505 14:00:41.930 | INFO | Downloaded 34GiB to GPU in 2.04152s (16.6543GiB/s) | 20260505 14:00:47.057 | INFO | Analyzing chunk frames = [3761152,3761664) | 20260505 14:00:47.057 | INFO | MemInfo: { | 20260505 14:00:47.057 | INFO | "gpu" : " 17 GB", | 20260505 14:00:47.057 | INFO | "huge" : "", | 20260505 14:00:47.057 | INFO | "huge_shared" : " 27 GB", | 20260505 14:00:47.057 | INFO | "malloc" : " 14 GB", | 20260505 14:00:47.057 | INFO | "pinned" : " 29 GB" | 20260505 14:00:47.057 | INFO | } | 20260505 14:00:47.057 | INFO | Perf report for the last 5.12785 seconds: | 20260505 14:00:47.057 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:00:47.057 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:47.057 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:47.057 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:47.057 | INFO | ANALYSIS : 63.5%, 3.0, 98.6ms, 0 | 20260505 14:00:47.057 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:47.058 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:47.058 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:00:47.058 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:47.058 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:47.058 | INFO | Uploaded 36GiB to GPU in 2.23471s (16.1095GiB/s) | 20260505 14:00:47.058 | INFO | Downloaded 34GiB to GPU in 2.10036s (16.1877GiB/s) | 20260505 14:00:52.200 | INFO | Analyzing chunk frames = [3761664,3762176) | 20260505 14:00:52.200 | INFO | MemInfo: { | 20260505 14:00:52.200 | INFO | "gpu" : " 17 GB", | 20260505 14:00:52.200 | INFO | "huge" : "", | 20260505 14:00:52.200 | INFO | "huge_shared" : " 27 GB", | 20260505 14:00:52.200 | INFO | "malloc" : " 14 GB", | 20260505 14:00:52.200 | INFO | "pinned" : " 29 GB" | 20260505 14:00:52.200 | INFO | } | 20260505 14:00:52.200 | INFO | Perf report for the last 5.14293 seconds: | 20260505 14:00:52.200 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:00:52.200 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:52.200 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:52.200 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:52.200 | INFO | ANALYSIS : 63.0%, 3.0, 98.0ms, 0 | 20260505 14:00:52.200 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:52.200 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:52.200 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 14:00:52.200 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:52.200 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:52.201 | INFO | Uploaded 36GiB to GPU in 2.20238s (16.346GiB/s) | 20260505 14:00:52.201 | INFO | Downloaded 34GiB to GPU in 2.13307s (15.9395GiB/s) | 20260505 14:00:57.344 | INFO | Analyzing chunk frames = [3762176,3762688) | 20260505 14:00:57.344 | INFO | MemInfo: { | 20260505 14:00:57.344 | INFO | "gpu" : " 17 GB", | 20260505 14:00:57.344 | INFO | "huge" : "", | 20260505 14:00:57.344 | INFO | "huge_shared" : " 27 GB", | 20260505 14:00:57.344 | INFO | "malloc" : " 14 GB", | 20260505 14:00:57.344 | INFO | "pinned" : " 29 GB" | 20260505 14:00:57.344 | INFO | } | 20260505 14:00:57.345 | INFO | Perf report for the last 5.14419 seconds: | 20260505 14:00:57.345 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:00:57.345 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:57.345 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:57.345 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:57.345 | INFO | ANALYSIS : 62.7%, 3.0, 98.0ms, 0 | 20260505 14:00:57.345 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:57.345 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:57.345 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:00:57.345 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:57.345 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:00:57.345 | INFO | Uploaded 36GiB to GPU in 2.16551s (16.6242GiB/s) | 20260505 14:00:57.345 | INFO | Downloaded 34GiB to GPU in 2.12423s (16.0058GiB/s) | 20260505 14:01:02.483 | INFO | Analyzing chunk frames = [3762688,3763200) | 20260505 14:01:02.483 | INFO | MemInfo: { | 20260505 14:01:02.483 | INFO | "gpu" : " 17 GB", | 20260505 14:01:02.483 | INFO | "huge" : "", | 20260505 14:01:02.483 | INFO | "huge_shared" : " 27 GB", | 20260505 14:01:02.483 | INFO | "malloc" : " 14 GB", | 20260505 14:01:02.483 | INFO | "pinned" : " 29 GB" | 20260505 14:01:02.483 | INFO | } | 20260505 14:01:02.483 | INFO | Perf report for the last 5.13818 seconds: | 20260505 14:01:02.483 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:01:02.483 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:02.483 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:02.483 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:02.483 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 14:01:02.483 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:02.483 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:02.483 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:01:02.483 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:02.483 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:02.483 | INFO | Uploaded 36GiB to GPU in 2.19188s (16.4242GiB/s) | 20260505 14:01:02.483 | INFO | Downloaded 34GiB to GPU in 2.10996s (16.1141GiB/s) | 20260505 14:01:07.619 | INFO | Analyzing chunk frames = [3763200,3763712) | 20260505 14:01:07.619 | INFO | MemInfo: { | 20260505 14:01:07.619 | INFO | "gpu" : " 17 GB", | 20260505 14:01:07.619 | INFO | "huge" : "", | 20260505 14:01:07.619 | INFO | "huge_shared" : " 27 GB", | 20260505 14:01:07.619 | INFO | "malloc" : " 14 GB", | 20260505 14:01:07.619 | INFO | "pinned" : " 29 GB" | 20260505 14:01:07.619 | INFO | } | 20260505 14:01:07.619 | INFO | Perf report for the last 5.13677 seconds: | 20260505 14:01:07.620 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:01:07.620 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:07.620 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:07.620 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:07.620 | INFO | ANALYSIS : 64.1%, 3.0, 100.0ms, 0 | 20260505 14:01:07.620 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:07.620 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:07.620 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 14:01:07.620 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:07.620 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:07.620 | INFO | Uploaded 36GiB to GPU in 2.16361s (16.6389GiB/s) | 20260505 14:01:07.620 | INFO | Downloaded 34GiB to GPU in 2.18475s (15.5624GiB/s) | 20260505 14:01:12.758 | INFO | Analyzing chunk frames = [3763712,3764224) | 20260505 14:01:12.758 | INFO | MemInfo: { | 20260505 14:01:12.758 | INFO | "gpu" : " 17 GB", | 20260505 14:01:12.758 | INFO | "huge" : "", | 20260505 14:01:12.758 | INFO | "huge_shared" : " 27 GB", | 20260505 14:01:12.758 | INFO | "malloc" : " 14 GB", | 20260505 14:01:12.758 | INFO | "pinned" : " 29 GB" | 20260505 14:01:12.758 | INFO | } | 20260505 14:01:12.758 | INFO | Perf report for the last 5.13853 seconds: | 20260505 14:01:12.758 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:01:12.758 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:12.758 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:12.758 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:12.758 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 14:01:12.758 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:12.758 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:12.758 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 14:01:12.758 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:12.758 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:12.758 | INFO | Uploaded 36GiB to GPU in 2.14191s (16.8075GiB/s) | 20260505 14:01:12.758 | INFO | Downloaded 34GiB to GPU in 2.10074s (16.1848GiB/s) | 20260505 14:01:12.890 | INFO | Starting a round of Baz IO | 20260505 14:01:16.130 | INFO | Populating RTMetrics report | 20260505 14:01:16.130 | INFO | RTMetrics report populated | 20260505 14:01:16.130 | INFO | Opening tmp RTMetrics json file | 20260505 14:01:16.201 | INFO | Serializing RTMetrics to json | 20260505 14:01:16.221 | INFO | Serialization done, swapping tmp file now... | 20260505 14:01:16.224 | INFO | RTMetrics done | 20260505 14:01:16.241 | INFO | Finished a round of Baz IO: Wrote 2.276 GiB in 3.35 seconds | 20260505 14:01:16.241 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.90% / 0.00% / 77.09% | 20260505 14:01:16.241 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.61487% / 1.58946e-05% / 7.54992e-05% | 20260505 14:01:16.241 | INFO | Baz IO overall: | 20260505 14:01:16.241 | INFO | BytesWritten : 2242822485731 | 20260505 14:01:16.241 | INFO | NumEvents : 1066661110843 | 20260505 14:01:17.904 | INFO | Analyzing chunk frames = [3764224,3764736) | 20260505 14:01:17.904 | INFO | MemInfo: { | 20260505 14:01:17.904 | INFO | "gpu" : " 17 GB", | 20260505 14:01:17.904 | INFO | "huge" : "", | 20260505 14:01:17.904 | INFO | "huge_shared" : " 27 GB", | 20260505 14:01:17.904 | INFO | "malloc" : " 14 GB", | 20260505 14:01:17.904 | INFO | "pinned" : " 29 GB" | 20260505 14:01:17.904 | INFO | } | 20260505 14:01:17.904 | INFO | Perf report for the last 5.14629 seconds: | 20260505 14:01:17.904 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:01:17.904 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:17.904 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:17.904 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:17.904 | INFO | ANALYSIS : 64.4%, 3.0, 100.1ms, 0 | 20260505 14:01:17.904 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:17.904 | INFO | RT_METRICS : 46.7%, 1.0, 24.5ms, 0 | 20260505 14:01:17.904 | INFO | PRE_HQ : 60.6%, 1.8, 57.4ms, 0 | 20260505 14:01:17.904 | INFO | BAZWRITER : 64.2%, 4.0, 132.0ms, 0 | 20260505 14:01:17.904 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 14:01:17.905 | INFO | Uploaded 36GiB to GPU in 2.19216s (16.4222GiB/s) | 20260505 14:01:17.905 | INFO | Downloaded 38GiB to GPU in 2.27331s (16.7157GiB/s) | 20260505 14:01:23.042 | INFO | Analyzing chunk frames = [3764736,3765248) | 20260505 14:01:23.042 | INFO | MemInfo: { | 20260505 14:01:23.042 | INFO | "gpu" : " 17 GB", | 20260505 14:01:23.042 | INFO | "huge" : "", | 20260505 14:01:23.042 | INFO | "huge_shared" : " 27 GB", | 20260505 14:01:23.042 | INFO | "malloc" : " 14 GB", | 20260505 14:01:23.042 | INFO | "pinned" : " 29 GB" | 20260505 14:01:23.042 | INFO | } | 20260505 14:01:23.042 | INFO | Perf report for the last 5.13762 seconds: | 20260505 14:01:23.042 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:01:23.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:23.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:23.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:23.042 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 14:01:23.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:23.042 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:23.042 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:01:23.042 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:23.042 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:23.042 | INFO | Uploaded 36GiB to GPU in 2.20396s (16.3342GiB/s) | 20260505 14:01:23.042 | INFO | Downloaded 34GiB to GPU in 2.04171s (16.6527GiB/s) | 20260505 14:01:28.179 | INFO | Analyzing chunk frames = [3765248,3765760) | 20260505 14:01:28.179 | INFO | MemInfo: { | 20260505 14:01:28.179 | INFO | "gpu" : " 17 GB", | 20260505 14:01:28.179 | INFO | "huge" : "", | 20260505 14:01:28.179 | INFO | "huge_shared" : " 27 GB", | 20260505 14:01:28.179 | INFO | "malloc" : " 14 GB", | 20260505 14:01:28.179 | INFO | "pinned" : " 29 GB" | 20260505 14:01:28.179 | INFO | } | 20260505 14:01:28.179 | INFO | Perf report for the last 5.137 seconds: | 20260505 14:01:28.179 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:01:28.179 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:28.179 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:28.179 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:28.179 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 14:01:28.179 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:28.179 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:28.179 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:01:28.179 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:28.179 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:28.179 | INFO | Uploaded 36GiB to GPU in 2.23328s (16.1198GiB/s) | 20260505 14:01:28.179 | INFO | Downloaded 34GiB to GPU in 2.12411s (16.0067GiB/s) | 20260505 14:01:33.313 | INFO | Analyzing chunk frames = [3765760,3766272) | 20260505 14:01:33.313 | INFO | MemInfo: { | 20260505 14:01:33.313 | INFO | "gpu" : " 17 GB", | 20260505 14:01:33.313 | INFO | "huge" : "", | 20260505 14:01:33.313 | INFO | "huge_shared" : " 27 GB", | 20260505 14:01:33.313 | INFO | "malloc" : " 14 GB", | 20260505 14:01:33.313 | INFO | "pinned" : " 29 GB" | 20260505 14:01:33.313 | INFO | } | 20260505 14:01:33.313 | INFO | Perf report for the last 5.13422 seconds: | 20260505 14:01:33.313 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:01:33.313 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:33.313 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:33.313 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:33.313 | INFO | ANALYSIS : 63.0%, 3.0, 97.8ms, 0 | 20260505 14:01:33.313 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:33.313 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:33.313 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 14:01:33.313 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:33.313 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:33.314 | INFO | Uploaded 36GiB to GPU in 2.20223s (16.3471GiB/s) | 20260505 14:01:33.314 | INFO | Downloaded 34GiB to GPU in 2.13562s (15.9204GiB/s) | 20260505 14:01:38.457 | INFO | Analyzing chunk frames = [3766272,3766784) | 20260505 14:01:38.457 | INFO | MemInfo: { | 20260505 14:01:38.457 | INFO | "gpu" : " 17 GB", | 20260505 14:01:38.457 | INFO | "huge" : "", | 20260505 14:01:38.457 | INFO | "huge_shared" : " 27 GB", | 20260505 14:01:38.457 | INFO | "malloc" : " 14 GB", | 20260505 14:01:38.457 | INFO | "pinned" : " 29 GB" | 20260505 14:01:38.457 | INFO | } | 20260505 14:01:38.457 | INFO | Perf report for the last 5.14354 seconds: | 20260505 14:01:38.457 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:01:38.457 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:38.457 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:38.457 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:38.457 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 14:01:38.457 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:38.457 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:38.457 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:01:38.457 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:38.457 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:38.457 | INFO | Uploaded 36GiB to GPU in 2.17023s (16.5881GiB/s) | 20260505 14:01:38.457 | INFO | Downloaded 34GiB to GPU in 2.12691s (15.9856GiB/s) | 20260505 14:01:43.597 | INFO | Analyzing chunk frames = [3766784,3767296) | 20260505 14:01:43.597 | INFO | MemInfo: { | 20260505 14:01:43.597 | INFO | "gpu" : " 17 GB", | 20260505 14:01:43.597 | INFO | "huge" : "", | 20260505 14:01:43.597 | INFO | "huge_shared" : " 27 GB", | 20260505 14:01:43.597 | INFO | "malloc" : " 14 GB", | 20260505 14:01:43.597 | INFO | "pinned" : " 29 GB" | 20260505 14:01:43.597 | INFO | } | 20260505 14:01:43.597 | INFO | Perf report for the last 5.1405 seconds: | 20260505 14:01:43.597 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:01:43.597 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:43.597 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:43.597 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:43.597 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 14:01:43.597 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:43.597 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:43.597 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 14:01:43.597 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:43.597 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:43.597 | INFO | Uploaded 36GiB to GPU in 2.19653s (16.3895GiB/s) | 20260505 14:01:43.597 | INFO | Downloaded 34GiB to GPU in 2.11313s (16.0899GiB/s) | 20260505 14:01:48.741 | INFO | Analyzing chunk frames = [3767296,3767808) | 20260505 14:01:48.741 | INFO | MemInfo: { | 20260505 14:01:48.741 | INFO | "gpu" : " 17 GB", | 20260505 14:01:48.741 | INFO | "huge" : "", | 20260505 14:01:48.741 | INFO | "huge_shared" : " 27 GB", | 20260505 14:01:48.741 | INFO | "malloc" : " 14 GB", | 20260505 14:01:48.741 | INFO | "pinned" : " 29 GB" | 20260505 14:01:48.741 | INFO | } | 20260505 14:01:48.741 | INFO | Perf report for the last 5.14367 seconds: | 20260505 14:01:48.741 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:01:48.741 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:48.741 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:48.741 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:48.741 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 14:01:48.741 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:48.741 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:48.741 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 14:01:48.741 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:48.741 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:48.741 | INFO | Uploaded 36GiB to GPU in 2.14944s (16.7485GiB/s) | 20260505 14:01:48.741 | INFO | Downloaded 34GiB to GPU in 2.18664s (15.5489GiB/s) | 20260505 14:01:53.876 | INFO | Analyzing chunk frames = [3767808,3768320) | 20260505 14:01:53.876 | INFO | MemInfo: { | 20260505 14:01:53.876 | INFO | "gpu" : " 17 GB", | 20260505 14:01:53.876 | INFO | "huge" : "", | 20260505 14:01:53.876 | INFO | "huge_shared" : " 27 GB", | 20260505 14:01:53.876 | INFO | "malloc" : " 14 GB", | 20260505 14:01:53.876 | INFO | "pinned" : " 29 GB" | 20260505 14:01:53.876 | INFO | } | 20260505 14:01:53.876 | INFO | Perf report for the last 5.13517 seconds: | 20260505 14:01:53.876 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:01:53.876 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:53.876 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:53.876 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:53.876 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 14:01:53.876 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:53.876 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:53.876 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:01:53.876 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:53.876 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:53.876 | INFO | Uploaded 36GiB to GPU in 2.21461s (16.2557GiB/s) | 20260505 14:01:53.877 | INFO | Downloaded 34GiB to GPU in 2.10184s (16.1763GiB/s) | 20260505 14:01:53.877 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 14:01:53.877 | INFO | Total mem usage: 15.3 GiB | 20260505 14:01:53.877 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 14:01:53.877 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -7.05%) | 20260505 14:01:53.877 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 14:01:53.877 | INFO | High Water Breakdown: | 20260505 14:01:53.877 | INFO | 61.1% : 8.75 GiB ( 91.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 14:01:53.877 | INFO | 38.8% : 5.55 GiB ( 83.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 14:01:53.877 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 14:01:53.877 | INFO | ------------------------------------------ | 20260505 14:01:53.877 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:01:53.877 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 14:01:53.877 | INFO | Total mem usage: 26.5 GiB | 20260505 14:01:53.877 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 14:01:53.877 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 14:01:53.877 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 14:01:53.877 | INFO | High Water Breakdown: | 20260505 14:01:53.877 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:01:53.877 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:01:53.877 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:01:53.877 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:01:53.877 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:01:53.877 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:01:53.877 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 14:01:53.877 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 14:01:53.877 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:01:53.877 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 14:01:53.877 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 14:01:53.877 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 14:01:53.877 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 14:01:53.877 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 14:01:53.877 | INFO | ------------------------------------------ | 20260505 14:01:53.877 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:01:53.877 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 14:01:53.877 | INFO | Total mem usage: 25.6 GiB | 20260505 14:01:53.877 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 14:01:53.877 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 14:01:53.877 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 14:01:53.877 | INFO | High Water Breakdown: | 20260505 14:01:53.877 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 14:01:53.877 | INFO | ------------------------------------------ | 20260505 14:01:53.877 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:01:53.877 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 14:01:53.877 | INFO | Total mem usage: 15.7 GiB | 20260505 14:01:53.877 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 14:01:53.877 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 14:01:53.877 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 14:01:53.877 | INFO | High Water Breakdown: | 20260505 14:01:53.877 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 14:01:53.877 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 14:01:53.877 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 14:01:53.877 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 14:01:53.877 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 14:01:53.877 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 14:01:53.877 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 14:01:53.877 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 14:01:53.877 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 14:01:53.877 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:01:53.877 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 14:01:53.877 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:01:53.877 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 14:01:53.877 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 14:01:53.877 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 14:01:53.877 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 14:01:53.877 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:01:53.877 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 14:01:53.877 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:01:53.877 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:01:53.877 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:01:53.877 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 14:01:53.877 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 14:01:53.877 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:01:53.877 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 14:01:53.877 | INFO | ------------------------------------------ | 20260505 14:01:53.877 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:01:53.877 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 14:01:53.877 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:01:53.877 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:01:53.877 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.71e+01, 1.39e+01 / 2.20e+00 | 20260505 14:01:53.877 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 1.02e-01, 1.16e-02 / 1.86e-02 | 20260505 14:01:53.877 | INFO | PerfReport: 11.75%, Baseline -- 3.74e+00 / 4.50e+00, 5.74e-01 / 1.44e-01 | 20260505 14:01:53.877 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.48e+00, 1.95e-01 / 9.96e-02 | 20260505 14:01:53.877 | INFO | PerfReport: 16.12%, DME -- 4.10e+01 / 4.65e+01, 4.79e+00 / 2.33e+00 | 20260505 14:01:53.877 | INFO | PerfReport: 56.28%, FrameLabeling -- 1.79e+01 / 1.96e+01, 1.28e+00 / 1.53e+00 | 20260505 14:01:53.877 | INFO | PerfReport: 7.48%, PulseAccumulating -- 2.38e+00 / 2.97e+00, 4.51e-01 / 1.19e-01 | 20260505 14:01:53.877 | INFO | PerfReport: 3.54%, Metrics -- 1.13e+00 / 1.45e+00, 3.12e-01 / 1.68e-01 | 20260505 14:01:53.877 | INFO | PerfReport: 1.24%, PulseSerializing -- 3.96e-01 / 4.43e-01, 6.99e-02 / 2.70e-02 | 20260505 14:01:53.877 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 14:01:53.877 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:01:53.877 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:01:53.877 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.56e+01 / 0.00e+00 | 20260505 14:01:53.877 | INFO | PerfReport: 22.62%, Upload -- 2.28e+01 / 0.00e+00, 8.04e+00 / 0.00e+00 | 20260505 14:01:53.877 | INFO | PerfReport: 22.04%, Download -- 2.22e+01 / 0.00e+00, 8.07e+00 / 0.00e+00 | 20260505 14:01:53.877 | INFO | PerfReport: 31.51%, Compute -- 3.18e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 14:01:53.877 | INFO | PerfReport: 2.33%, LockUpload -- 2.35e+00 / 0.00e+00, 8.10e+00 / 0.00e+00 | 20260505 14:01:53.877 | INFO | PerfReport: 1.41%, LockDownload -- 1.43e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260505 14:01:53.878 | INFO | PerfReport: 20.07%, LockCompute -- 2.03e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 14:01:53.878 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.87e-02 / 0.00e+00 | 20260505 14:01:54.017 | INFO | Starting a round of Baz IO | 20260505 14:01:57.242 | INFO | Populating RTMetrics report | 20260505 14:01:57.242 | INFO | RTMetrics report populated | 20260505 14:01:57.242 | INFO | Opening tmp RTMetrics json file | 20260505 14:01:57.261 | INFO | Serializing RTMetrics to json | 20260505 14:01:57.262 | INFO | Serialization done, swapping tmp file now... | 20260505 14:01:57.263 | INFO | RTMetrics done | 20260505 14:01:57.350 | INFO | Finished a round of Baz IO: Wrote 2.272 GiB in 3.333 seconds | 20260505 14:01:57.350 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.90% / 0.00% / 77.10% | 20260505 14:01:57.350 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.612891% / 3.97364e-06% / 9.13938e-05% | 20260505 14:01:57.350 | INFO | Baz IO overall: | 20260505 14:01:57.350 | INFO | BytesWritten : 2245262182142 | 20260505 14:01:57.350 | INFO | NumEvents : 1067812568693 | 20260505 14:01:59.008 | INFO | Analyzing chunk frames = [3768320,3768832) | 20260505 14:01:59.008 | INFO | MemInfo: { | 20260505 14:01:59.008 | INFO | "gpu" : " 17 GB", | 20260505 14:01:59.008 | INFO | "huge" : "", | 20260505 14:01:59.008 | INFO | "huge_shared" : " 27 GB", | 20260505 14:01:59.008 | INFO | "malloc" : " 14 GB", | 20260505 14:01:59.008 | INFO | "pinned" : " 29 GB" | 20260505 14:01:59.008 | INFO | } | 20260505 14:01:59.008 | INFO | Perf report for the last 5.13156 seconds: | 20260505 14:01:59.008 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:01:59.008 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:59.008 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:59.008 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:59.008 | INFO | ANALYSIS : 64.5%, 3.0, 100.0ms, 0 | 20260505 14:01:59.008 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:01:59.008 | INFO | RT_METRICS : 45.5%, 1.0, 23.8ms, 0 | 20260505 14:01:59.008 | INFO | PRE_HQ : 60.8%, 1.8, 57.3ms, 0 | 20260505 14:01:59.008 | INFO | BAZWRITER : 64.3%, 4.0, 131.6ms, 0 | 20260505 14:01:59.008 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 14:01:59.008 | INFO | Uploaded 36GiB to GPU in 2.18303s (16.4908GiB/s) | 20260505 14:01:59.008 | INFO | Downloaded 38GiB to GPU in 2.27051s (16.7363GiB/s) | 20260505 14:02:04.158 | INFO | Analyzing chunk frames = [3768832,3769344) | 20260505 14:02:04.158 | INFO | MemInfo: { | 20260505 14:02:04.158 | INFO | "gpu" : " 17 GB", | 20260505 14:02:04.158 | INFO | "huge" : "", | 20260505 14:02:04.158 | INFO | "huge_shared" : " 27 GB", | 20260505 14:02:04.158 | INFO | "malloc" : " 14 GB", | 20260505 14:02:04.158 | INFO | "pinned" : " 29 GB" | 20260505 14:02:04.158 | INFO | } | 20260505 14:02:04.158 | INFO | Perf report for the last 5.15072 seconds: | 20260505 14:02:04.158 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:02:04.158 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:04.158 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:04.158 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:04.159 | INFO | ANALYSIS : 63.4%, 3.0, 99.3ms, 0 | 20260505 14:02:04.159 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:04.159 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:04.159 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:02:04.159 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:04.159 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:04.159 | INFO | Uploaded 36GiB to GPU in 2.16196s (16.6515GiB/s) | 20260505 14:02:04.159 | INFO | Downloaded 34GiB to GPU in 2.03716s (16.6899GiB/s) | 20260505 14:02:09.296 | INFO | Analyzing chunk frames = [3769344,3769856) | 20260505 14:02:09.296 | INFO | MemInfo: { | 20260505 14:02:09.296 | INFO | "gpu" : " 17 GB", | 20260505 14:02:09.296 | INFO | "huge" : "", | 20260505 14:02:09.296 | INFO | "huge_shared" : " 27 GB", | 20260505 14:02:09.296 | INFO | "malloc" : " 14 GB", | 20260505 14:02:09.296 | INFO | "pinned" : " 29 GB" | 20260505 14:02:09.296 | INFO | } | 20260505 14:02:09.296 | INFO | Perf report for the last 5.1376 seconds: | 20260505 14:02:09.296 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:02:09.296 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:09.296 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:09.296 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:09.296 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 14:02:09.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:09.296 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:09.296 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:02:09.296 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:09.296 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:09.296 | INFO | Uploaded 36GiB to GPU in 2.23433s (16.1122GiB/s) | 20260505 14:02:09.296 | INFO | Downloaded 34GiB to GPU in 2.12088s (16.0311GiB/s) | 20260505 14:02:14.435 | INFO | Analyzing chunk frames = [3769856,3770368) | 20260505 14:02:14.435 | INFO | MemInfo: { | 20260505 14:02:14.435 | INFO | "gpu" : " 17 GB", | 20260505 14:02:14.435 | INFO | "huge" : "", | 20260505 14:02:14.435 | INFO | "huge_shared" : " 27 GB", | 20260505 14:02:14.435 | INFO | "malloc" : " 14 GB", | 20260505 14:02:14.435 | INFO | "pinned" : " 29 GB" | 20260505 14:02:14.435 | INFO | } | 20260505 14:02:14.435 | INFO | Perf report for the last 5.13875 seconds: | 20260505 14:02:14.435 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:02:14.435 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:14.435 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:14.435 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:14.435 | INFO | ANALYSIS : 62.9%, 3.0, 97.7ms, 0 | 20260505 14:02:14.435 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:14.435 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:14.435 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:02:14.435 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:14.435 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:14.435 | INFO | Uploaded 36GiB to GPU in 2.20103s (16.356GiB/s) | 20260505 14:02:14.435 | INFO | Downloaded 34GiB to GPU in 2.13364s (15.9352GiB/s) | 20260505 14:02:19.569 | INFO | Analyzing chunk frames = [3770368,3770880) | 20260505 14:02:19.569 | INFO | MemInfo: { | 20260505 14:02:19.569 | INFO | "gpu" : " 17 GB", | 20260505 14:02:19.569 | INFO | "huge" : "", | 20260505 14:02:19.569 | INFO | "huge_shared" : " 27 GB", | 20260505 14:02:19.569 | INFO | "malloc" : " 14 GB", | 20260505 14:02:19.569 | INFO | "pinned" : " 29 GB" | 20260505 14:02:19.569 | INFO | } | 20260505 14:02:19.569 | INFO | Perf report for the last 5.13432 seconds: | 20260505 14:02:19.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:02:19.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:19.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:19.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:19.569 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 14:02:19.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:19.569 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:19.569 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:02:19.569 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:19.569 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:19.569 | INFO | Uploaded 36GiB to GPU in 2.16143s (16.6557GiB/s) | 20260505 14:02:19.569 | INFO | Downloaded 34GiB to GPU in 2.12317s (16.0138GiB/s) | 20260505 14:02:24.714 | INFO | Analyzing chunk frames = [3770880,3771392) | 20260505 14:02:24.714 | INFO | MemInfo: { | 20260505 14:02:24.714 | INFO | "gpu" : " 17 GB", | 20260505 14:02:24.714 | INFO | "huge" : "", | 20260505 14:02:24.714 | INFO | "huge_shared" : " 27 GB", | 20260505 14:02:24.714 | INFO | "malloc" : " 14 GB", | 20260505 14:02:24.714 | INFO | "pinned" : " 29 GB" | 20260505 14:02:24.714 | INFO | } | 20260505 14:02:24.714 | INFO | Perf report for the last 5.14478 seconds: | 20260505 14:02:24.714 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:02:24.714 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:24.714 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:24.714 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:24.714 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 14:02:24.714 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:24.714 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:24.714 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 14:02:24.714 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:24.714 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:24.714 | INFO | Uploaded 36GiB to GPU in 2.1752s (16.5502GiB/s) | 20260505 14:02:24.714 | INFO | Downloaded 34GiB to GPU in 2.11066s (16.1087GiB/s) | 20260505 14:02:29.853 | INFO | Analyzing chunk frames = [3771392,3771904) | 20260505 14:02:29.853 | INFO | MemInfo: { | 20260505 14:02:29.853 | INFO | "gpu" : " 17 GB", | 20260505 14:02:29.853 | INFO | "huge" : "", | 20260505 14:02:29.853 | INFO | "huge_shared" : " 27 GB", | 20260505 14:02:29.853 | INFO | "malloc" : " 14 GB", | 20260505 14:02:29.853 | INFO | "pinned" : " 29 GB" | 20260505 14:02:29.853 | INFO | } | 20260505 14:02:29.853 | INFO | Perf report for the last 5.13915 seconds: | 20260505 14:02:29.853 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:02:29.853 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:29.853 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:29.853 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:29.853 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 14:02:29.853 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:29.853 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:29.853 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 14:02:29.853 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:29.853 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:29.853 | INFO | Uploaded 36GiB to GPU in 2.13181s (16.8871GiB/s) | 20260505 14:02:29.853 | INFO | Downloaded 34GiB to GPU in 2.15966s (15.7433GiB/s) | 20260505 14:02:34.990 | INFO | Analyzing chunk frames = [3771904,3772416) | 20260505 14:02:34.990 | INFO | MemInfo: { | 20260505 14:02:34.990 | INFO | "gpu" : " 17 GB", | 20260505 14:02:34.990 | INFO | "huge" : "", | 20260505 14:02:34.990 | INFO | "huge_shared" : " 27 GB", | 20260505 14:02:34.990 | INFO | "malloc" : " 14 GB", | 20260505 14:02:34.990 | INFO | "pinned" : " 29 GB" | 20260505 14:02:34.990 | INFO | } | 20260505 14:02:34.990 | INFO | Perf report for the last 5.13712 seconds: | 20260505 14:02:34.990 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:02:34.990 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:34.990 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:34.990 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:34.990 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 14:02:34.990 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:34.990 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:34.990 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 14:02:34.990 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:34.990 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:34.991 | INFO | Uploaded 36GiB to GPU in 2.21195s (16.2753GiB/s) | 20260505 14:02:34.991 | INFO | Downloaded 34GiB to GPU in 2.10264s (16.1702GiB/s) | 20260505 14:02:35.124 | INFO | Starting a round of Baz IO | 20260505 14:02:38.363 | INFO | Populating RTMetrics report | 20260505 14:02:38.364 | INFO | RTMetrics report populated | 20260505 14:02:38.364 | INFO | Opening tmp RTMetrics json file | 20260505 14:02:38.381 | INFO | Serializing RTMetrics to json | 20260505 14:02:38.385 | INFO | Serialization done, swapping tmp file now... | 20260505 14:02:38.394 | INFO | RTMetrics done | 20260505 14:02:38.465 | INFO | Finished a round of Baz IO: Wrote 2.274 GiB in 3.34 seconds | 20260505 14:02:38.465 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.89% / 0.00% / 77.10% | 20260505 14:02:38.465 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.612386% / 3.97364e-06% / 7.54992e-05% | 20260505 14:02:38.465 | INFO | Baz IO overall: | 20260505 14:02:38.465 | INFO | BytesWritten : 2247703336924 | 20260505 14:02:38.465 | INFO | NumEvents : 1068965031189 | 20260505 14:02:40.126 | INFO | Analyzing chunk frames = [3772416,3772928) | 20260505 14:02:40.126 | INFO | MemInfo: { | 20260505 14:02:40.126 | INFO | "gpu" : " 17 GB", | 20260505 14:02:40.126 | INFO | "huge" : "", | 20260505 14:02:40.126 | INFO | "huge_shared" : " 27 GB", | 20260505 14:02:40.126 | INFO | "malloc" : " 14 GB", | 20260505 14:02:40.126 | INFO | "pinned" : " 29 GB" | 20260505 14:02:40.126 | INFO | } | 20260505 14:02:40.126 | INFO | Perf report for the last 5.1362 seconds: | 20260505 14:02:40.126 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:02:40.126 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:40.126 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:40.126 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:40.126 | INFO | ANALYSIS : 64.6%, 3.0, 100.2ms, 0 | 20260505 14:02:40.126 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:40.127 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260505 14:02:40.127 | INFO | PRE_HQ : 60.7%, 1.8, 57.5ms, 0 | 20260505 14:02:40.127 | INFO | BAZWRITER : 64.3%, 4.0, 131.7ms, 0 | 20260505 14:02:40.127 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 14:02:40.127 | INFO | Uploaded 36GiB to GPU in 2.19046s (16.4349GiB/s) | 20260505 14:02:40.127 | INFO | Downloaded 38GiB to GPU in 2.28071s (16.6615GiB/s) | 20260505 14:02:45.265 | INFO | Analyzing chunk frames = [3772928,3773440) | 20260505 14:02:45.265 | INFO | MemInfo: { | 20260505 14:02:45.265 | INFO | "gpu" : " 17 GB", | 20260505 14:02:45.265 | INFO | "huge" : "", | 20260505 14:02:45.265 | INFO | "huge_shared" : " 27 GB", | 20260505 14:02:45.265 | INFO | "malloc" : " 14 GB", | 20260505 14:02:45.265 | INFO | "pinned" : " 29 GB" | 20260505 14:02:45.265 | INFO | } | 20260505 14:02:45.265 | INFO | Perf report for the last 5.13884 seconds: | 20260505 14:02:45.265 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:02:45.265 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:45.265 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:45.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:45.265 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 14:02:45.265 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:45.265 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:45.265 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:02:45.265 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:45.265 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:45.266 | INFO | Uploaded 36GiB to GPU in 2.18085s (16.5073GiB/s) | 20260505 14:02:45.266 | INFO | Downloaded 34GiB to GPU in 2.03856s (16.6784GiB/s) | 20260505 14:02:50.416 | INFO | Analyzing chunk frames = [3773440,3773952) | 20260505 14:02:50.416 | INFO | MemInfo: { | 20260505 14:02:50.416 | INFO | "gpu" : " 17 GB", | 20260505 14:02:50.416 | INFO | "huge" : "", | 20260505 14:02:50.416 | INFO | "huge_shared" : " 27 GB", | 20260505 14:02:50.416 | INFO | "malloc" : " 14 GB", | 20260505 14:02:50.416 | INFO | "pinned" : " 29 GB" | 20260505 14:02:50.416 | INFO | } | 20260505 14:02:50.416 | INFO | Perf report for the last 5.15031 seconds: | 20260505 14:02:50.416 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:02:50.416 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:50.416 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:50.416 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:50.416 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 14:02:50.416 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:50.416 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:50.416 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:02:50.416 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:50.416 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:50.416 | INFO | Uploaded 36GiB to GPU in 2.22705s (16.1649GiB/s) | 20260505 14:02:50.416 | INFO | Downloaded 34GiB to GPU in 2.09861s (16.2012GiB/s) | 20260505 14:02:55.555 | INFO | Analyzing chunk frames = [3773952,3774464) | 20260505 14:02:55.555 | INFO | MemInfo: { | 20260505 14:02:55.555 | INFO | "gpu" : " 17 GB", | 20260505 14:02:55.555 | INFO | "huge" : "", | 20260505 14:02:55.555 | INFO | "huge_shared" : " 27 GB", | 20260505 14:02:55.555 | INFO | "malloc" : " 14 GB", | 20260505 14:02:55.555 | INFO | "pinned" : " 29 GB" | 20260505 14:02:55.555 | INFO | } | 20260505 14:02:55.555 | INFO | Perf report for the last 5.13903 seconds: | 20260505 14:02:55.555 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:02:55.555 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:55.555 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:55.555 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:55.555 | INFO | ANALYSIS : 62.5%, 3.0, 97.0ms, 0 | 20260505 14:02:55.555 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:55.555 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:55.555 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:02:55.555 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:55.555 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:02:55.555 | INFO | Uploaded 36GiB to GPU in 2.17678s (16.5382GiB/s) | 20260505 14:02:55.555 | INFO | Downloaded 34GiB to GPU in 2.13132s (15.9526GiB/s) | 20260505 14:03:00.684 | INFO | Analyzing chunk frames = [3774464,3774976) | 20260505 14:03:00.684 | INFO | MemInfo: { | 20260505 14:03:00.684 | INFO | "gpu" : " 17 GB", | 20260505 14:03:00.684 | INFO | "huge" : "", | 20260505 14:03:00.684 | INFO | "huge_shared" : " 27 GB", | 20260505 14:03:00.684 | INFO | "malloc" : " 14 GB", | 20260505 14:03:00.684 | INFO | "pinned" : " 29 GB" | 20260505 14:03:00.684 | INFO | } | 20260505 14:03:00.684 | INFO | Perf report for the last 5.12905 seconds: | 20260505 14:03:00.684 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:03:00.684 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:00.684 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:00.684 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:00.684 | INFO | ANALYSIS : 62.9%, 3.0, 98.0ms, 0 | 20260505 14:03:00.684 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:00.684 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:00.684 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:03:00.684 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:00.684 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:00.684 | INFO | Uploaded 36GiB to GPU in 2.18362s (16.4864GiB/s) | 20260505 14:03:00.684 | INFO | Downloaded 34GiB to GPU in 2.12941s (15.9669GiB/s) | 20260505 14:03:05.821 | INFO | Analyzing chunk frames = [3774976,3775488) | 20260505 14:03:05.821 | INFO | MemInfo: { | 20260505 14:03:05.821 | INFO | "gpu" : " 17 GB", | 20260505 14:03:05.821 | INFO | "huge" : "", | 20260505 14:03:05.821 | INFO | "huge_shared" : " 27 GB", | 20260505 14:03:05.821 | INFO | "malloc" : " 14 GB", | 20260505 14:03:05.821 | INFO | "pinned" : " 29 GB" | 20260505 14:03:05.821 | INFO | } | 20260505 14:03:05.821 | INFO | Perf report for the last 5.13696 seconds: | 20260505 14:03:05.821 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:03:05.821 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:05.821 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:05.821 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:05.821 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 14:03:05.821 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:05.821 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:05.821 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:03:05.821 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:05.821 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:05.821 | INFO | Uploaded 36GiB to GPU in 2.16341s (16.6404GiB/s) | 20260505 14:03:05.821 | INFO | Downloaded 34GiB to GPU in 2.10667s (16.1392GiB/s) | 20260505 14:03:10.962 | INFO | Analyzing chunk frames = [3775488,3776000) | 20260505 14:03:10.962 | INFO | MemInfo: { | 20260505 14:03:10.962 | INFO | "gpu" : " 17 GB", | 20260505 14:03:10.962 | INFO | "huge" : "", | 20260505 14:03:10.962 | INFO | "huge_shared" : " 27 GB", | 20260505 14:03:10.962 | INFO | "malloc" : " 14 GB", | 20260505 14:03:10.962 | INFO | "pinned" : " 29 GB" | 20260505 14:03:10.962 | INFO | } | 20260505 14:03:10.962 | INFO | Perf report for the last 5.1413 seconds: | 20260505 14:03:10.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:03:10.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:10.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:10.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:10.962 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 14:03:10.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:10.962 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:10.962 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 14:03:10.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:10.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:10.962 | INFO | Uploaded 36GiB to GPU in 2.14927s (16.7498GiB/s) | 20260505 14:03:10.962 | INFO | Downloaded 34GiB to GPU in 2.12845s (15.9741GiB/s) | 20260505 14:03:16.106 | INFO | Analyzing chunk frames = [3776000,3776512) | 20260505 14:03:16.106 | INFO | MemInfo: { | 20260505 14:03:16.106 | INFO | "gpu" : " 17 GB", | 20260505 14:03:16.106 | INFO | "huge" : "", | 20260505 14:03:16.106 | INFO | "huge_shared" : " 27 GB", | 20260505 14:03:16.106 | INFO | "malloc" : " 14 GB", | 20260505 14:03:16.106 | INFO | "pinned" : " 29 GB" | 20260505 14:03:16.106 | INFO | } | 20260505 14:03:16.106 | INFO | Perf report for the last 5.14444 seconds: | 20260505 14:03:16.106 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:03:16.106 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:16.106 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:16.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:16.107 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 14:03:16.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:16.107 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:16.107 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 14:03:16.107 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:16.107 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:16.107 | INFO | Uploaded 36GiB to GPU in 2.15687s (16.6909GiB/s) | 20260505 14:03:16.107 | INFO | Downloaded 34GiB to GPU in 2.10137s (16.1799GiB/s) | 20260505 14:03:16.107 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 14:03:16.107 | INFO | Total mem usage: 15.3 GiB | 20260505 14:03:16.107 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 14:03:16.107 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.65%) | 20260505 14:03:16.107 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 14:03:16.107 | INFO | High Water Breakdown: | 20260505 14:03:16.107 | INFO | 61% : 8.68 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 14:03:16.107 | INFO | 38.9% : 5.54 GiB ( 83.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 14:03:16.107 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 14:03:16.107 | INFO | ------------------------------------------ | 20260505 14:03:16.107 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:03:16.107 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 14:03:16.107 | INFO | Total mem usage: 26.5 GiB | 20260505 14:03:16.107 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 14:03:16.107 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 14:03:16.107 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 14:03:16.107 | INFO | High Water Breakdown: | 20260505 14:03:16.107 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:03:16.107 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:03:16.107 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:03:16.107 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:03:16.107 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:03:16.107 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:03:16.107 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 14:03:16.107 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 14:03:16.107 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:03:16.107 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 14:03:16.107 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 14:03:16.107 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 14:03:16.107 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 14:03:16.107 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 14:03:16.107 | INFO | ------------------------------------------ | 20260505 14:03:16.107 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:03:16.107 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 14:03:16.107 | INFO | Total mem usage: 25.6 GiB | 20260505 14:03:16.107 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 14:03:16.107 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 14:03:16.107 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 14:03:16.107 | INFO | High Water Breakdown: | 20260505 14:03:16.107 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 14:03:16.107 | INFO | ------------------------------------------ | 20260505 14:03:16.107 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:03:16.107 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 14:03:16.107 | INFO | Total mem usage: 15.7 GiB | 20260505 14:03:16.107 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 14:03:16.107 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 14:03:16.107 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 14:03:16.107 | INFO | High Water Breakdown: | 20260505 14:03:16.107 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 14:03:16.107 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 14:03:16.107 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 14:03:16.107 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 14:03:16.107 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 14:03:16.107 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 14:03:16.107 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 14:03:16.107 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 14:03:16.107 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 14:03:16.107 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:03:16.107 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 14:03:16.107 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:03:16.107 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 14:03:16.107 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 14:03:16.107 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 14:03:16.107 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 14:03:16.107 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:03:16.107 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 14:03:16.107 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:03:16.107 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:03:16.107 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:03:16.107 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 14:03:16.107 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 14:03:16.107 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:03:16.107 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 14:03:16.107 | INFO | ------------------------------------------ | 20260505 14:03:16.107 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:03:16.108 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 14:03:16.108 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:03:16.108 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:03:16.108 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.64e+01, 1.38e+01 / 1.31e+00 | 20260505 14:03:16.108 | INFO | PerfReport: 0.29%, AggregateStats -- 9.30e-02 / 9.90e-02, 1.11e-02 / 4.21e-03 | 20260505 14:03:16.108 | INFO | PerfReport: 11.79%, Baseline -- 3.75e+00 / 4.49e+00, 5.88e-01 / 1.82e-01 | 20260505 14:03:16.108 | INFO | PerfReport: 3.31%, Histogram -- 1.05e+00 / 1.50e+00, 2.01e-01 / 9.49e-02 | 20260505 14:03:16.108 | INFO | PerfReport: 15.99%, DME -- 4.07e+01 / 4.57e+01, 4.59e+00 / 1.79e+00 | 20260505 14:03:16.108 | INFO | PerfReport: 56.38%, FrameLabeling -- 1.79e+01 / 1.99e+01, 1.39e+00 / 1.79e+00 | 20260505 14:03:16.108 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.37e+00 / 2.89e+00, 4.51e-01 / 2.23e-01 | 20260505 14:03:16.108 | INFO | PerfReport: 3.54%, Metrics -- 1.13e+00 / 1.44e+00, 3.14e-01 / 2.04e-01 | 20260505 14:03:16.108 | INFO | PerfReport: 1.24%, PulseSerializing -- 3.96e-01 / 4.47e-01, 7.11e-02 / 3.69e-02 | 20260505 14:03:16.108 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 14:03:16.108 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:03:16.108 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:03:16.108 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260505 14:03:16.108 | INFO | PerfReport: 22.54%, Upload -- 2.27e+01 / 0.00e+00, 8.03e+00 / 0.00e+00 | 20260505 14:03:16.108 | INFO | PerfReport: 22.02%, Download -- 2.22e+01 / 0.00e+00, 8.07e+00 / 0.00e+00 | 20260505 14:03:16.108 | INFO | PerfReport: 31.59%, Compute -- 3.18e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 14:03:16.108 | INFO | PerfReport: 2.26%, LockUpload -- 2.27e+00 / 0.00e+00, 7.92e+00 / 0.00e+00 | 20260505 14:03:16.108 | INFO | PerfReport: 1.45%, LockDownload -- 1.46e+00 / 0.00e+00, 2.99e+00 / 0.00e+00 | 20260505 14:03:16.108 | INFO | PerfReport: 20.13%, LockCompute -- 2.03e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260505 14:03:16.108 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.69e-02 / 0.00e+00 | 20260505 14:03:16.244 | INFO | Starting a round of Baz IO | 20260505 14:03:19.466 | INFO | Populating RTMetrics report | 20260505 14:03:19.467 | INFO | RTMetrics report populated | 20260505 14:03:19.467 | INFO | Opening tmp RTMetrics json file | 20260505 14:03:19.475 | INFO | Serializing RTMetrics to json | 20260505 14:03:19.513 | INFO | Serialization done, swapping tmp file now... | 20260505 14:03:19.519 | INFO | RTMetrics done | 20260505 14:03:19.572 | INFO | Finished a round of Baz IO: Wrote 2.274 GiB in 3.328 seconds | 20260505 14:03:19.572 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.89% / 0.00% / 77.11% | 20260505 14:03:19.572 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.612601% / 3.97364e-06% / 7.94729e-05% | 20260505 14:03:19.572 | INFO | Baz IO overall: | 20260505 14:03:19.572 | INFO | BytesWritten : 2250145065204 | 20260505 14:03:19.572 | INFO | NumEvents : 1070118300193 | 20260505 14:03:21.256 | INFO | Analyzing chunk frames = [3776512,3777024) | 20260505 14:03:21.256 | INFO | MemInfo: { | 20260505 14:03:21.256 | INFO | "gpu" : " 17 GB", | 20260505 14:03:21.256 | INFO | "huge" : "", | 20260505 14:03:21.256 | INFO | "huge_shared" : " 27 GB", | 20260505 14:03:21.256 | INFO | "malloc" : " 14 GB", | 20260505 14:03:21.256 | INFO | "pinned" : " 29 GB" | 20260505 14:03:21.256 | INFO | } | 20260505 14:03:21.256 | INFO | Perf report for the last 5.14939 seconds: | 20260505 14:03:21.256 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:03:21.256 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:21.256 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:21.256 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:21.256 | INFO | ANALYSIS : 64.2%, 3.0, 99.8ms, 0 | 20260505 14:03:21.256 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:21.256 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260505 14:03:21.256 | INFO | PRE_HQ : 60.9%, 1.8, 57.4ms, 0 | 20260505 14:03:21.256 | INFO | BAZWRITER : 64.0%, 4.0, 131.5ms, 0 | 20260505 14:03:21.256 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 14:03:21.256 | INFO | Uploaded 36GiB to GPU in 2.17708s (16.5359GiB/s) | 20260505 14:03:21.256 | INFO | Downloaded 38GiB to GPU in 2.26366s (16.787GiB/s) | 20260505 14:03:26.385 | INFO | Analyzing chunk frames = [3777024,3777536) | 20260505 14:03:26.385 | INFO | MemInfo: { | 20260505 14:03:26.385 | INFO | "gpu" : " 17 GB", | 20260505 14:03:26.385 | INFO | "huge" : "", | 20260505 14:03:26.385 | INFO | "huge_shared" : " 27 GB", | 20260505 14:03:26.385 | INFO | "malloc" : " 14 GB", | 20260505 14:03:26.385 | INFO | "pinned" : " 29 GB" | 20260505 14:03:26.385 | INFO | } | 20260505 14:03:26.385 | INFO | Perf report for the last 5.12881 seconds: | 20260505 14:03:26.385 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:03:26.385 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:26.385 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:26.385 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:26.385 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 14:03:26.385 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:26.385 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:26.385 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:03:26.385 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:26.385 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:26.385 | INFO | Uploaded 36GiB to GPU in 2.20814s (16.3033GiB/s) | 20260505 14:03:26.385 | INFO | Downloaded 34GiB to GPU in 2.0571s (16.5281GiB/s) | 20260505 14:03:31.517 | INFO | Analyzing chunk frames = [3777536,3778048) | 20260505 14:03:31.517 | INFO | MemInfo: { | 20260505 14:03:31.517 | INFO | "gpu" : " 17 GB", | 20260505 14:03:31.517 | INFO | "huge" : "", | 20260505 14:03:31.518 | INFO | "huge_shared" : " 27 GB", | 20260505 14:03:31.518 | INFO | "malloc" : " 14 GB", | 20260505 14:03:31.518 | INFO | "pinned" : " 29 GB" | 20260505 14:03:31.518 | INFO | } | 20260505 14:03:31.518 | INFO | Perf report for the last 5.1329 seconds: | 20260505 14:03:31.518 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:03:31.518 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:31.518 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:31.518 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:31.518 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 14:03:31.518 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:31.518 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:31.518 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:03:31.518 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:31.518 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:31.518 | INFO | Uploaded 36GiB to GPU in 2.21113s (16.2813GiB/s) | 20260505 14:03:31.518 | INFO | Downloaded 34GiB to GPU in 2.11205s (16.0981GiB/s) | 20260505 14:03:36.671 | INFO | Analyzing chunk frames = [3778048,3778560) | 20260505 14:03:36.671 | INFO | MemInfo: { | 20260505 14:03:36.671 | INFO | "gpu" : " 17 GB", | 20260505 14:03:36.671 | INFO | "huge" : "", | 20260505 14:03:36.671 | INFO | "huge_shared" : " 27 GB", | 20260505 14:03:36.671 | INFO | "malloc" : " 14 GB", | 20260505 14:03:36.671 | INFO | "pinned" : " 29 GB" | 20260505 14:03:36.671 | INFO | } | 20260505 14:03:36.671 | INFO | Perf report for the last 5.15361 seconds: | 20260505 14:03:36.671 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:03:36.671 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:36.671 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:36.671 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:36.671 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260505 14:03:36.671 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:36.671 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:36.671 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 14:03:36.671 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:36.671 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:36.671 | INFO | Uploaded 36GiB to GPU in 2.20012s (16.3627GiB/s) | 20260505 14:03:36.672 | INFO | Downloaded 34GiB to GPU in 2.13446s (15.9291GiB/s) | 20260505 14:03:41.808 | INFO | Analyzing chunk frames = [3778560,3779072) | 20260505 14:03:41.808 | INFO | MemInfo: { | 20260505 14:03:41.808 | INFO | "gpu" : " 17 GB", | 20260505 14:03:41.808 | INFO | "huge" : "", | 20260505 14:03:41.808 | INFO | "huge_shared" : " 27 GB", | 20260505 14:03:41.808 | INFO | "malloc" : " 14 GB", | 20260505 14:03:41.808 | INFO | "pinned" : " 29 GB" | 20260505 14:03:41.808 | INFO | } | 20260505 14:03:41.808 | INFO | Perf report for the last 5.13644 seconds: | 20260505 14:03:41.808 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:03:41.808 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:41.808 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:41.808 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:41.808 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 14:03:41.808 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:41.808 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:41.808 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 14:03:41.808 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:41.808 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:41.808 | INFO | Uploaded 36GiB to GPU in 2.167s (16.6128GiB/s) | 20260505 14:03:41.808 | INFO | Downloaded 34GiB to GPU in 2.12947s (15.9664GiB/s) | 20260505 14:03:46.938 | INFO | Analyzing chunk frames = [3779072,3779584) | 20260505 14:03:46.938 | INFO | MemInfo: { | 20260505 14:03:46.938 | INFO | "gpu" : " 17 GB", | 20260505 14:03:46.938 | INFO | "huge" : "", | 20260505 14:03:46.938 | INFO | "huge_shared" : " 27 GB", | 20260505 14:03:46.938 | INFO | "malloc" : " 14 GB", | 20260505 14:03:46.938 | INFO | "pinned" : " 29 GB" | 20260505 14:03:46.938 | INFO | } | 20260505 14:03:46.938 | INFO | Perf report for the last 5.13034 seconds: | 20260505 14:03:46.938 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:03:46.938 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:46.938 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:46.938 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:46.938 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 14:03:46.938 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:46.938 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:46.938 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:03:46.938 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:46.938 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:46.938 | INFO | Uploaded 36GiB to GPU in 2.20194s (16.3492GiB/s) | 20260505 14:03:46.938 | INFO | Downloaded 34GiB to GPU in 2.11201s (16.0984GiB/s) | 20260505 14:03:52.080 | INFO | Analyzing chunk frames = [3779584,3780096) | 20260505 14:03:52.080 | INFO | MemInfo: { | 20260505 14:03:52.080 | INFO | "gpu" : " 17 GB", | 20260505 14:03:52.080 | INFO | "huge" : "", | 20260505 14:03:52.080 | INFO | "huge_shared" : " 27 GB", | 20260505 14:03:52.080 | INFO | "malloc" : " 14 GB", | 20260505 14:03:52.080 | INFO | "pinned" : " 29 GB" | 20260505 14:03:52.080 | INFO | } | 20260505 14:03:52.080 | INFO | Perf report for the last 5.14236 seconds: | 20260505 14:03:52.080 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:03:52.080 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:52.080 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:52.080 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:52.080 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 14:03:52.080 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:52.080 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:52.080 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 14:03:52.080 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:52.080 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:52.081 | INFO | Uploaded 36GiB to GPU in 2.17789s (16.5297GiB/s) | 20260505 14:03:52.081 | INFO | Downloaded 34GiB to GPU in 2.14651s (15.8396GiB/s) | 20260505 14:03:57.222 | INFO | Analyzing chunk frames = [3780096,3780608) | 20260505 14:03:57.222 | INFO | MemInfo: { | 20260505 14:03:57.222 | INFO | "gpu" : " 17 GB", | 20260505 14:03:57.222 | INFO | "huge" : "", | 20260505 14:03:57.222 | INFO | "huge_shared" : " 27 GB", | 20260505 14:03:57.222 | INFO | "malloc" : " 14 GB", | 20260505 14:03:57.222 | INFO | "pinned" : " 29 GB" | 20260505 14:03:57.222 | INFO | } | 20260505 14:03:57.222 | INFO | Perf report for the last 5.14186 seconds: | 20260505 14:03:57.222 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:03:57.222 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:57.222 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:57.222 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:57.222 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 14:03:57.222 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:57.222 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:57.222 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 14:03:57.222 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:57.222 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:03:57.223 | INFO | Uploaded 36GiB to GPU in 2.18869s (16.4482GiB/s) | 20260505 14:03:57.223 | INFO | Downloaded 34GiB to GPU in 2.11118s (16.1047GiB/s) | 20260505 14:03:57.359 | INFO | Starting a round of Baz IO | 20260505 14:04:00.580 | INFO | Populating RTMetrics report | 20260505 14:04:00.580 | INFO | RTMetrics report populated | 20260505 14:04:00.580 | INFO | Opening tmp RTMetrics json file | 20260505 14:04:00.592 | INFO | Serializing RTMetrics to json | 20260505 14:04:00.594 | INFO | Serialization done, swapping tmp file now... | 20260505 14:04:00.594 | INFO | RTMetrics done | 20260505 14:04:00.683 | INFO | Finished a round of Baz IO: Wrote 2.276 GiB in 3.324 seconds | 20260505 14:04:00.683 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.88% / 0.00% / 77.11% | 20260505 14:04:00.683 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.612068% / 3.97364e-06% / 9.13938e-05% | 20260505 14:04:00.683 | INFO | Baz IO overall: | 20260505 14:04:00.683 | INFO | BytesWritten : 2252588509578 | 20260505 14:04:00.683 | INFO | NumEvents : 1071272970661 | 20260505 14:04:02.365 | INFO | Analyzing chunk frames = [3780608,3781120) | 20260505 14:04:02.365 | INFO | MemInfo: { | 20260505 14:04:02.365 | INFO | "gpu" : " 17 GB", | 20260505 14:04:02.365 | INFO | "huge" : "", | 20260505 14:04:02.365 | INFO | "huge_shared" : " 27 GB", | 20260505 14:04:02.365 | INFO | "malloc" : " 14 GB", | 20260505 14:04:02.365 | INFO | "pinned" : " 29 GB" | 20260505 14:04:02.365 | INFO | } | 20260505 14:04:02.365 | INFO | Perf report for the last 5.14289 seconds: | 20260505 14:04:02.365 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:04:02.365 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:02.365 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:02.365 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:02.365 | INFO | ANALYSIS : 64.2%, 3.0, 99.4ms, 0 | 20260505 14:04:02.365 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:02.365 | INFO | RT_METRICS : 45.3%, 1.0, 23.8ms, 0 | 20260505 14:04:02.365 | INFO | PRE_HQ : 60.9%, 1.8, 57.6ms, 0 | 20260505 14:04:02.365 | INFO | BAZWRITER : 64.0%, 4.0, 133.1ms, 0 | 20260505 14:04:02.365 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 14:04:02.365 | INFO | Uploaded 36GiB to GPU in 2.20276s (16.3431GiB/s) | 20260505 14:04:02.365 | INFO | Downloaded 38GiB to GPU in 2.2605s (16.8105GiB/s) | 20260505 14:04:07.503 | INFO | Analyzing chunk frames = [3781120,3781632) | 20260505 14:04:07.503 | INFO | MemInfo: { | 20260505 14:04:07.503 | INFO | "gpu" : " 17 GB", | 20260505 14:04:07.503 | INFO | "huge" : "", | 20260505 14:04:07.503 | INFO | "huge_shared" : " 27 GB", | 20260505 14:04:07.503 | INFO | "malloc" : " 14 GB", | 20260505 14:04:07.503 | INFO | "pinned" : " 29 GB" | 20260505 14:04:07.503 | INFO | } | 20260505 14:04:07.503 | INFO | Perf report for the last 5.1379 seconds: | 20260505 14:04:07.503 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:04:07.503 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:07.503 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:07.503 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:07.503 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 14:04:07.503 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:07.503 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:07.503 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:04:07.503 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:07.503 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:07.503 | INFO | Uploaded 36GiB to GPU in 2.20474s (16.3284GiB/s) | 20260505 14:04:07.503 | INFO | Downloaded 34GiB to GPU in 2.04211s (16.6494GiB/s) | 20260505 14:04:12.631 | INFO | Analyzing chunk frames = [3781632,3782144) | 20260505 14:04:12.631 | INFO | MemInfo: { | 20260505 14:04:12.631 | INFO | "gpu" : " 17 GB", | 20260505 14:04:12.631 | INFO | "huge" : "", | 20260505 14:04:12.631 | INFO | "huge_shared" : " 27 GB", | 20260505 14:04:12.631 | INFO | "malloc" : " 14 GB", | 20260505 14:04:12.631 | INFO | "pinned" : " 29 GB" | 20260505 14:04:12.631 | INFO | } | 20260505 14:04:12.631 | INFO | Perf report for the last 5.12801 seconds: | 20260505 14:04:12.631 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:04:12.631 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:12.631 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:12.631 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:12.631 | INFO | ANALYSIS : 63.4%, 3.0, 98.5ms, 0 | 20260505 14:04:12.631 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:12.631 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:12.631 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:04:12.631 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:12.631 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:12.631 | INFO | Uploaded 36GiB to GPU in 2.22972s (16.1455GiB/s) | 20260505 14:04:12.631 | INFO | Downloaded 34GiB to GPU in 2.0967s (16.2159GiB/s) | 20260505 14:04:17.771 | INFO | Analyzing chunk frames = [3782144,3782656) | 20260505 14:04:17.771 | INFO | MemInfo: { | 20260505 14:04:17.771 | INFO | "gpu" : " 17 GB", | 20260505 14:04:17.771 | INFO | "huge" : "", | 20260505 14:04:17.771 | INFO | "huge_shared" : " 27 GB", | 20260505 14:04:17.771 | INFO | "malloc" : " 14 GB", | 20260505 14:04:17.771 | INFO | "pinned" : " 29 GB" | 20260505 14:04:17.771 | INFO | } | 20260505 14:04:17.771 | INFO | Perf report for the last 5.14028 seconds: | 20260505 14:04:17.771 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:04:17.771 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:17.771 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:17.771 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:17.771 | INFO | ANALYSIS : 63.1%, 3.0, 98.0ms, 0 | 20260505 14:04:17.771 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:17.772 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:17.772 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 14:04:17.772 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:17.772 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:17.772 | INFO | Uploaded 36GiB to GPU in 2.21s (16.2896GiB/s) | 20260505 14:04:17.772 | INFO | Downloaded 34GiB to GPU in 2.13187s (15.9484GiB/s) | 20260505 14:04:22.921 | INFO | Analyzing chunk frames = [3782656,3783168) | 20260505 14:04:22.922 | INFO | MemInfo: { | 20260505 14:04:22.922 | INFO | "gpu" : " 17 GB", | 20260505 14:04:22.922 | INFO | "huge" : "", | 20260505 14:04:22.922 | INFO | "huge_shared" : " 27 GB", | 20260505 14:04:22.922 | INFO | "malloc" : " 14 GB", | 20260505 14:04:22.922 | INFO | "pinned" : " 29 GB" | 20260505 14:04:22.922 | INFO | } | 20260505 14:04:22.922 | INFO | Perf report for the last 5.1502 seconds: | 20260505 14:04:22.922 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:04:22.922 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:22.922 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:22.922 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:22.922 | INFO | ANALYSIS : 62.7%, 3.0, 98.1ms, 0 | 20260505 14:04:22.922 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:22.922 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:22.922 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:04:22.922 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:22.922 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:22.922 | INFO | Uploaded 36GiB to GPU in 2.16234s (16.6486GiB/s) | 20260505 14:04:22.922 | INFO | Downloaded 34GiB to GPU in 2.12126s (16.0282GiB/s) | 20260505 14:04:28.061 | INFO | Analyzing chunk frames = [3783168,3783680) | 20260505 14:04:28.061 | INFO | MemInfo: { | 20260505 14:04:28.061 | INFO | "gpu" : " 17 GB", | 20260505 14:04:28.061 | INFO | "huge" : "", | 20260505 14:04:28.061 | INFO | "huge_shared" : " 27 GB", | 20260505 14:04:28.061 | INFO | "malloc" : " 14 GB", | 20260505 14:04:28.061 | INFO | "pinned" : " 29 GB" | 20260505 14:04:28.061 | INFO | } | 20260505 14:04:28.061 | INFO | Perf report for the last 5.13952 seconds: | 20260505 14:04:28.061 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:04:28.061 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:28.061 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:28.061 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:28.061 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 14:04:28.061 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:28.061 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:28.061 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 14:04:28.061 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:28.061 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:28.061 | INFO | Uploaded 36GiB to GPU in 2.20965s (16.2922GiB/s) | 20260505 14:04:28.061 | INFO | Downloaded 34GiB to GPU in 2.11228s (16.0963GiB/s) | 20260505 14:04:33.194 | INFO | Analyzing chunk frames = [3783680,3784192) | 20260505 14:04:33.194 | INFO | MemInfo: { | 20260505 14:04:33.194 | INFO | "gpu" : " 17 GB", | 20260505 14:04:33.194 | INFO | "huge" : "", | 20260505 14:04:33.194 | INFO | "huge_shared" : " 27 GB", | 20260505 14:04:33.194 | INFO | "malloc" : " 14 GB", | 20260505 14:04:33.194 | INFO | "pinned" : " 29 GB" | 20260505 14:04:33.194 | INFO | } | 20260505 14:04:33.194 | INFO | Perf report for the last 5.13272 seconds: | 20260505 14:04:33.194 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:04:33.194 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:33.194 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:33.194 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:33.194 | INFO | ANALYSIS : 64.3%, 3.0, 100.3ms, 0 | 20260505 14:04:33.194 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:33.194 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:33.194 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 14:04:33.194 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:33.194 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:33.194 | INFO | Uploaded 36GiB to GPU in 2.18008s (16.5131GiB/s) | 20260505 14:04:33.194 | INFO | Downloaded 34GiB to GPU in 2.19376s (15.4985GiB/s) | 20260505 14:04:38.329 | INFO | Analyzing chunk frames = [3784192,3784704) | 20260505 14:04:38.329 | INFO | MemInfo: { | 20260505 14:04:38.329 | INFO | "gpu" : " 17 GB", | 20260505 14:04:38.329 | INFO | "huge" : "", | 20260505 14:04:38.329 | INFO | "huge_shared" : " 27 GB", | 20260505 14:04:38.329 | INFO | "malloc" : " 14 GB", | 20260505 14:04:38.329 | INFO | "pinned" : " 29 GB" | 20260505 14:04:38.329 | INFO | } | 20260505 14:04:38.329 | INFO | Perf report for the last 5.1348 seconds: | 20260505 14:04:38.329 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:04:38.329 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:38.329 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:38.329 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:38.329 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 14:04:38.329 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:38.329 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:38.329 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 14:04:38.329 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:38.329 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:38.329 | INFO | Uploaded 36GiB to GPU in 2.12448s (16.9453GiB/s) | 20260505 14:04:38.329 | INFO | Downloaded 34GiB to GPU in 2.09979s (16.1921GiB/s) | 20260505 14:04:38.329 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 14:04:38.329 | INFO | Total mem usage: 15.3 GiB | 20260505 14:04:38.329 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 14:04:38.329 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.25%) | 20260505 14:04:38.329 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 14:04:38.329 | INFO | High Water Breakdown: | 20260505 14:04:38.329 | INFO | 61.2% : 8.83 GiB ( 92.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 14:04:38.329 | INFO | 38.6% : 5.57 GiB ( 83.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 14:04:38.329 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 14:04:38.329 | INFO | ------------------------------------------ | 20260505 14:04:38.329 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:04:38.329 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 14:04:38.329 | INFO | Total mem usage: 26.5 GiB | 20260505 14:04:38.329 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 14:04:38.329 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 14:04:38.329 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 14:04:38.329 | INFO | High Water Breakdown: | 20260505 14:04:38.329 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:04:38.329 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:04:38.329 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:04:38.329 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:04:38.329 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:04:38.329 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:04:38.329 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 14:04:38.329 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 14:04:38.329 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:04:38.329 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 14:04:38.329 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 14:04:38.329 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 14:04:38.329 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 14:04:38.329 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 14:04:38.329 | INFO | ------------------------------------------ | 20260505 14:04:38.329 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:04:38.329 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 14:04:38.329 | INFO | Total mem usage: 25.6 GiB | 20260505 14:04:38.329 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 14:04:38.329 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 14:04:38.329 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 14:04:38.329 | INFO | High Water Breakdown: | 20260505 14:04:38.329 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 14:04:38.329 | INFO | ------------------------------------------ | 20260505 14:04:38.329 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:04:38.329 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 14:04:38.329 | INFO | Total mem usage: 15.7 GiB | 20260505 14:04:38.329 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 14:04:38.329 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 14:04:38.329 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 14:04:38.330 | INFO | High Water Breakdown: | 20260505 14:04:38.330 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 14:04:38.330 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 14:04:38.330 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 14:04:38.330 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 14:04:38.330 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 14:04:38.330 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 14:04:38.330 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 14:04:38.330 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 14:04:38.330 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 14:04:38.330 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:04:38.330 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 14:04:38.330 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:04:38.330 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 14:04:38.330 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 14:04:38.330 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 14:04:38.330 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 14:04:38.330 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:04:38.330 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 14:04:38.330 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:04:38.330 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:04:38.330 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:04:38.330 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 14:04:38.330 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 14:04:38.330 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:04:38.330 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 14:04:38.330 | INFO | ------------------------------------------ | 20260505 14:04:38.330 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:04:38.330 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 14:04:38.330 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:04:38.330 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:04:38.330 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.71e+01, 1.39e+01 / 2.05e+00 | 20260505 14:04:38.330 | INFO | PerfReport: 0.29%, AggregateStats -- 9.26e-02 / 1.01e-01, 1.06e-02 / 3.94e-03 | 20260505 14:04:38.330 | INFO | PerfReport: 11.78%, Baseline -- 3.75e+00 / 4.50e+00, 5.86e-01 / 1.20e-01 | 20260505 14:04:38.330 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.51e+00, 1.96e-01 / 8.25e-02 | 20260505 14:04:38.330 | INFO | PerfReport: 16.07%, DME -- 4.09e+01 / 4.65e+01, 4.93e+00 / 2.40e+00 | 20260505 14:04:38.330 | INFO | PerfReport: 56.33%, FrameLabeling -- 1.79e+01 / 1.96e+01, 1.36e+00 / 1.60e+00 | 20260505 14:04:38.330 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.37e+00 / 2.93e+00, 4.52e-01 / 1.98e-01 | 20260505 14:04:38.330 | INFO | PerfReport: 3.56%, Metrics -- 1.13e+00 / 1.47e+00, 3.17e-01 / 1.57e-01 | 20260505 14:04:38.330 | INFO | PerfReport: 1.24%, PulseSerializing -- 3.96e-01 / 4.52e-01, 7.08e-02 / 2.95e-02 | 20260505 14:04:38.330 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 14:04:38.330 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:04:38.330 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:04:38.330 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260505 14:04:38.330 | INFO | PerfReport: 22.63%, Upload -- 2.28e+01 / 0.00e+00, 8.13e+00 / 0.00e+00 | 20260505 14:04:38.330 | INFO | PerfReport: 22.03%, Download -- 2.22e+01 / 0.00e+00, 8.11e+00 / 0.00e+00 | 20260505 14:04:38.330 | INFO | PerfReport: 31.56%, Compute -- 3.18e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 14:04:38.330 | INFO | PerfReport: 2.33%, LockUpload -- 2.35e+00 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260505 14:04:38.330 | INFO | PerfReport: 1.37%, LockDownload -- 1.38e+00 / 0.00e+00, 2.84e+00 / 0.00e+00 | 20260505 14:04:38.330 | INFO | PerfReport: 20.07%, LockCompute -- 2.02e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 14:04:38.330 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.74e-02 / 0.00e+00 | 20260505 14:04:38.463 | INFO | Starting a round of Baz IO | 20260505 14:04:41.696 | INFO | Populating RTMetrics report | 20260505 14:04:41.696 | INFO | RTMetrics report populated | 20260505 14:04:41.696 | INFO | Opening tmp RTMetrics json file | 20260505 14:04:41.698 | INFO | Serializing RTMetrics to json | 20260505 14:04:41.721 | INFO | Serialization done, swapping tmp file now... | 20260505 14:04:41.740 | INFO | RTMetrics done | 20260505 14:04:41.807 | INFO | Finished a round of Baz IO: Wrote 2.274 GiB in 3.344 seconds | 20260505 14:04:41.807 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.88% / 0.00% / 77.12% | 20260505 14:04:41.807 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.609883% / 7.94729e-06% / 8.34465e-05% | 20260505 14:04:41.807 | INFO | Baz IO overall: | 20260505 14:04:41.807 | INFO | BytesWritten : 2255030037851 | 20260505 14:04:41.807 | INFO | NumEvents : 1072426042559 | 20260505 14:04:43.475 | INFO | Analyzing chunk frames = [3784704,3785216) | 20260505 14:04:43.475 | INFO | MemInfo: { | 20260505 14:04:43.475 | INFO | "gpu" : " 17 GB", | 20260505 14:04:43.475 | INFO | "huge" : "", | 20260505 14:04:43.476 | INFO | "huge_shared" : " 27 GB", | 20260505 14:04:43.476 | INFO | "malloc" : " 14 GB", | 20260505 14:04:43.476 | INFO | "pinned" : " 29 GB" | 20260505 14:04:43.476 | INFO | } | 20260505 14:04:43.476 | INFO | Perf report for the last 5.14685 seconds: | 20260505 14:04:43.476 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:04:43.476 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:43.476 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:43.476 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:43.476 | INFO | ANALYSIS : 64.3%, 3.0, 99.8ms, 0 | 20260505 14:04:43.476 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:43.476 | INFO | RT_METRICS : 46.0%, 1.0, 24.2ms, 0 | 20260505 14:04:43.476 | INFO | PRE_HQ : 60.6%, 1.8, 58.1ms, 0 | 20260505 14:04:43.476 | INFO | BAZWRITER : 64.3%, 4.0, 132.3ms, 0 | 20260505 14:04:43.476 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 14:04:43.476 | INFO | Uploaded 36GiB to GPU in 2.17998s (16.5139GiB/s) | 20260505 14:04:43.476 | INFO | Downloaded 38GiB to GPU in 2.28749s (16.6121GiB/s) | 20260505 14:04:48.613 | INFO | Analyzing chunk frames = [3785216,3785728) | 20260505 14:04:48.613 | INFO | MemInfo: { | 20260505 14:04:48.613 | INFO | "gpu" : " 17 GB", | 20260505 14:04:48.613 | INFO | "huge" : "", | 20260505 14:04:48.613 | INFO | "huge_shared" : " 27 GB", | 20260505 14:04:48.613 | INFO | "malloc" : " 14 GB", | 20260505 14:04:48.613 | INFO | "pinned" : " 29 GB" | 20260505 14:04:48.613 | INFO | } | 20260505 14:04:48.613 | INFO | Perf report for the last 5.13761 seconds: | 20260505 14:04:48.613 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:04:48.613 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:48.613 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:48.613 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:48.613 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 14:04:48.613 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:48.613 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:48.613 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 14:04:48.613 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:48.613 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:48.613 | INFO | Uploaded 36GiB to GPU in 2.20146s (16.3528GiB/s) | 20260505 14:04:48.613 | INFO | Downloaded 34GiB to GPU in 2.03952s (16.6706GiB/s) | 20260505 14:04:53.755 | INFO | Analyzing chunk frames = [3785728,3786240) | 20260505 14:04:53.755 | INFO | MemInfo: { | 20260505 14:04:53.755 | INFO | "gpu" : " 17 GB", | 20260505 14:04:53.755 | INFO | "huge" : "", | 20260505 14:04:53.755 | INFO | "huge_shared" : " 27 GB", | 20260505 14:04:53.756 | INFO | "malloc" : " 14 GB", | 20260505 14:04:53.756 | INFO | "pinned" : " 29 GB" | 20260505 14:04:53.756 | INFO | } | 20260505 14:04:53.756 | INFO | Perf report for the last 5.14244 seconds: | 20260505 14:04:53.756 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:04:53.756 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:53.756 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:53.756 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:53.756 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 14:04:53.756 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:53.756 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:53.756 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 14:04:53.756 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:53.756 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:53.756 | INFO | Uploaded 36GiB to GPU in 2.23285s (16.1229GiB/s) | 20260505 14:04:53.756 | INFO | Downloaded 34GiB to GPU in 2.11303s (16.0907GiB/s) | 20260505 14:04:58.896 | INFO | Analyzing chunk frames = [3786240,3786752) | 20260505 14:04:58.896 | INFO | MemInfo: { | 20260505 14:04:58.896 | INFO | "gpu" : " 17 GB", | 20260505 14:04:58.896 | INFO | "huge" : "", | 20260505 14:04:58.896 | INFO | "huge_shared" : " 27 GB", | 20260505 14:04:58.896 | INFO | "malloc" : " 14 GB", | 20260505 14:04:58.896 | INFO | "pinned" : " 29 GB" | 20260505 14:04:58.896 | INFO | } | 20260505 14:04:58.896 | INFO | Perf report for the last 5.14007 seconds: | 20260505 14:04:58.896 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:04:58.896 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:58.896 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:58.896 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:58.896 | INFO | ANALYSIS : 62.7%, 3.0, 97.5ms, 0 | 20260505 14:04:58.896 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:58.896 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:58.896 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:04:58.896 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:58.896 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:04:58.896 | INFO | Uploaded 36GiB to GPU in 2.17473s (16.5538GiB/s) | 20260505 14:04:58.896 | INFO | Downloaded 34GiB to GPU in 2.1318s (15.949GiB/s) | 20260505 14:05:04.028 | INFO | Analyzing chunk frames = [3786752,3787264) | 20260505 14:05:04.028 | INFO | MemInfo: { | 20260505 14:05:04.028 | INFO | "gpu" : " 17 GB", | 20260505 14:05:04.028 | INFO | "huge" : "", | 20260505 14:05:04.028 | INFO | "huge_shared" : " 27 GB", | 20260505 14:05:04.028 | INFO | "malloc" : " 14 GB", | 20260505 14:05:04.028 | INFO | "pinned" : " 29 GB" | 20260505 14:05:04.028 | INFO | } | 20260505 14:05:04.028 | INFO | Perf report for the last 5.13238 seconds: | 20260505 14:05:04.028 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:05:04.028 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:04.028 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:04.028 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:04.028 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 14:05:04.028 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:04.028 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:04.028 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:05:04.028 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:04.028 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:04.028 | INFO | Uploaded 36GiB to GPU in 2.15242s (16.7253GiB/s) | 20260505 14:05:04.028 | INFO | Downloaded 34GiB to GPU in 2.12698s (15.9851GiB/s) | 20260505 14:05:09.177 | INFO | Analyzing chunk frames = [3787264,3787776) | 20260505 14:05:09.177 | INFO | MemInfo: { | 20260505 14:05:09.177 | INFO | "gpu" : " 17 GB", | 20260505 14:05:09.178 | INFO | "huge" : "", | 20260505 14:05:09.178 | INFO | "huge_shared" : " 27 GB", | 20260505 14:05:09.178 | INFO | "malloc" : " 14 GB", | 20260505 14:05:09.178 | INFO | "pinned" : " 29 GB" | 20260505 14:05:09.178 | INFO | } | 20260505 14:05:09.178 | INFO | Perf report for the last 5.14946 seconds: | 20260505 14:05:09.178 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:05:09.178 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:09.178 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:09.178 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:09.178 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 14:05:09.178 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:09.178 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:09.178 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 14:05:09.178 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:09.178 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:09.178 | INFO | Uploaded 36GiB to GPU in 2.16204s (16.651GiB/s) | 20260505 14:05:09.178 | INFO | Downloaded 34GiB to GPU in 2.10746s (16.1331GiB/s) | 20260505 14:05:14.316 | INFO | Analyzing chunk frames = [3787776,3788288) | 20260505 14:05:14.316 | INFO | MemInfo: { | 20260505 14:05:14.316 | INFO | "gpu" : " 17 GB", | 20260505 14:05:14.316 | INFO | "huge" : "", | 20260505 14:05:14.316 | INFO | "huge_shared" : " 27 GB", | 20260505 14:05:14.317 | INFO | "malloc" : " 14 GB", | 20260505 14:05:14.317 | INFO | "pinned" : " 29 GB" | 20260505 14:05:14.317 | INFO | } | 20260505 14:05:14.317 | INFO | Perf report for the last 5.13899 seconds: | 20260505 14:05:14.317 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:05:14.317 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:14.317 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:14.317 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:14.317 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 14:05:14.317 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:14.317 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:14.317 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 14:05:14.317 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:14.317 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:14.317 | INFO | Uploaded 36GiB to GPU in 2.11379s (17.031GiB/s) | 20260505 14:05:14.317 | INFO | Downloaded 34GiB to GPU in 2.15265s (15.7945GiB/s) | 20260505 14:05:19.450 | INFO | Analyzing chunk frames = [3788288,3788800) | 20260505 14:05:19.450 | INFO | MemInfo: { | 20260505 14:05:19.450 | INFO | "gpu" : " 17 GB", | 20260505 14:05:19.450 | INFO | "huge" : "", | 20260505 14:05:19.450 | INFO | "huge_shared" : " 27 GB", | 20260505 14:05:19.450 | INFO | "malloc" : " 14 GB", | 20260505 14:05:19.450 | INFO | "pinned" : " 29 GB" | 20260505 14:05:19.450 | INFO | } | 20260505 14:05:19.450 | INFO | Perf report for the last 5.13382 seconds: | 20260505 14:05:19.450 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:05:19.450 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:19.450 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:19.450 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:19.450 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 14:05:19.451 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:19.451 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:19.451 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 14:05:19.451 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:19.451 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:19.451 | INFO | Uploaded 36GiB to GPU in 2.22139s (16.206GiB/s) | 20260505 14:05:19.451 | INFO | Downloaded 34GiB to GPU in 2.09391s (16.2376GiB/s) | 20260505 14:05:19.588 | INFO | Starting a round of Baz IO | 20260505 14:05:22.811 | INFO | Populating RTMetrics report | 20260505 14:05:22.811 | INFO | RTMetrics report populated | 20260505 14:05:22.811 | INFO | Opening tmp RTMetrics json file | 20260505 14:05:22.813 | INFO | Serializing RTMetrics to json | 20260505 14:05:22.835 | INFO | Serialization done, swapping tmp file now... | 20260505 14:05:22.861 | INFO | RTMetrics done | 20260505 14:05:22.922 | INFO | Finished a round of Baz IO: Wrote 2.273 GiB in 3.334 seconds | 20260505 14:05:22.922 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.88% / 0.00% / 77.12% | 20260505 14:05:22.922 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.607332% / 1.19209e-05% / 8.34465e-05% | 20260505 14:05:22.922 | INFO | Baz IO overall: | 20260505 14:05:22.922 | INFO | BytesWritten : 2257470608602 | 20260505 14:05:22.922 | INFO | NumEvents : 1073578480084 | 20260505 14:05:24.579 | INFO | Analyzing chunk frames = [3788800,3789312) | 20260505 14:05:24.579 | INFO | MemInfo: { | 20260505 14:05:24.579 | INFO | "gpu" : " 17 GB", | 20260505 14:05:24.579 | INFO | "huge" : "", | 20260505 14:05:24.579 | INFO | "huge_shared" : " 27 GB", | 20260505 14:05:24.579 | INFO | "malloc" : " 14 GB", | 20260505 14:05:24.579 | INFO | "pinned" : " 29 GB" | 20260505 14:05:24.579 | INFO | } | 20260505 14:05:24.579 | INFO | Perf report for the last 5.12903 seconds: | 20260505 14:05:24.579 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:05:24.579 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:24.579 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:24.579 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:24.580 | INFO | ANALYSIS : 64.4%, 3.0, 99.5ms, 0 | 20260505 14:05:24.580 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:24.580 | INFO | RT_METRICS : 46.1%, 1.0, 24.1ms, 0 | 20260505 14:05:24.580 | INFO | PRE_HQ : 61.1%, 1.8, 57.6ms, 0 | 20260505 14:05:24.580 | INFO | BAZWRITER : 64.3%, 4.0, 132.7ms, 0 | 20260505 14:05:24.580 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 14:05:24.580 | INFO | Uploaded 36GiB to GPU in 2.19833s (16.3761GiB/s) | 20260505 14:05:24.580 | INFO | Downloaded 38GiB to GPU in 2.26457s (16.7802GiB/s) | 20260505 14:05:29.729 | INFO | Analyzing chunk frames = [3789312,3789824) | 20260505 14:05:29.729 | INFO | MemInfo: { | 20260505 14:05:29.729 | INFO | "gpu" : " 17 GB", | 20260505 14:05:29.729 | INFO | "huge" : "", | 20260505 14:05:29.729 | INFO | "huge_shared" : " 27 GB", | 20260505 14:05:29.729 | INFO | "malloc" : " 14 GB", | 20260505 14:05:29.729 | INFO | "pinned" : " 29 GB" | 20260505 14:05:29.729 | INFO | } | 20260505 14:05:29.729 | INFO | Perf report for the last 5.14977 seconds: | 20260505 14:05:29.729 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:05:29.729 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:29.729 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:29.729 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:29.729 | INFO | ANALYSIS : 63.5%, 3.0, 99.3ms, 0 | 20260505 14:05:29.729 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:29.729 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:29.729 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:05:29.729 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:29.729 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:29.730 | INFO | Uploaded 36GiB to GPU in 2.17002s (16.5897GiB/s) | 20260505 14:05:29.730 | INFO | Downloaded 34GiB to GPU in 2.03608s (16.6988GiB/s) | 20260505 14:05:34.873 | INFO | Analyzing chunk frames = [3789824,3790336) | 20260505 14:05:34.873 | INFO | MemInfo: { | 20260505 14:05:34.873 | INFO | "gpu" : " 17 GB", | 20260505 14:05:34.873 | INFO | "huge" : "", | 20260505 14:05:34.873 | INFO | "huge_shared" : " 27 GB", | 20260505 14:05:34.873 | INFO | "malloc" : " 14 GB", | 20260505 14:05:34.873 | INFO | "pinned" : " 29 GB" | 20260505 14:05:34.873 | INFO | } | 20260505 14:05:34.873 | INFO | Perf report for the last 5.14399 seconds: | 20260505 14:05:34.873 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:05:34.873 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:34.873 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:34.873 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:34.873 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 14:05:34.873 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:34.873 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:34.873 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:05:34.873 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:34.873 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:34.873 | INFO | Uploaded 36GiB to GPU in 2.23258s (16.1249GiB/s) | 20260505 14:05:34.874 | INFO | Downloaded 34GiB to GPU in 2.12255s (16.0185GiB/s) | 20260505 14:05:40.011 | INFO | Analyzing chunk frames = [3790336,3790848) | 20260505 14:05:40.011 | INFO | MemInfo: { | 20260505 14:05:40.011 | INFO | "gpu" : " 17 GB", | 20260505 14:05:40.011 | INFO | "huge" : "", | 20260505 14:05:40.011 | INFO | "huge_shared" : " 27 GB", | 20260505 14:05:40.011 | INFO | "malloc" : " 14 GB", | 20260505 14:05:40.011 | INFO | "pinned" : " 29 GB" | 20260505 14:05:40.011 | INFO | } | 20260505 14:05:40.011 | INFO | Perf report for the last 5.1375 seconds: | 20260505 14:05:40.011 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:05:40.011 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:40.011 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:40.011 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:40.011 | INFO | ANALYSIS : 62.7%, 3.0, 97.4ms, 0 | 20260505 14:05:40.011 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:40.011 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:40.011 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:05:40.011 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:40.011 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:40.011 | INFO | Uploaded 36GiB to GPU in 2.1807s (16.5085GiB/s) | 20260505 14:05:40.011 | INFO | Downloaded 34GiB to GPU in 2.13066s (15.9575GiB/s) | 20260505 14:05:45.141 | INFO | Analyzing chunk frames = [3790848,3791360) | 20260505 14:05:45.141 | INFO | MemInfo: { | 20260505 14:05:45.141 | INFO | "gpu" : " 17 GB", | 20260505 14:05:45.141 | INFO | "huge" : "", | 20260505 14:05:45.141 | INFO | "huge_shared" : " 27 GB", | 20260505 14:05:45.141 | INFO | "malloc" : " 14 GB", | 20260505 14:05:45.141 | INFO | "pinned" : " 29 GB" | 20260505 14:05:45.141 | INFO | } | 20260505 14:05:45.142 | INFO | Perf report for the last 5.13079 seconds: | 20260505 14:05:45.142 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:05:45.142 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:45.142 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:45.142 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:45.142 | INFO | ANALYSIS : 62.9%, 3.0, 98.0ms, 0 | 20260505 14:05:45.142 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:45.142 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:45.142 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:05:45.142 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:45.142 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:45.142 | INFO | Uploaded 36GiB to GPU in 2.16042s (16.6634GiB/s) | 20260505 14:05:45.142 | INFO | Downloaded 34GiB to GPU in 2.12423s (16.0058GiB/s) | 20260505 14:05:50.276 | INFO | Analyzing chunk frames = [3791360,3791872) | 20260505 14:05:50.276 | INFO | MemInfo: { | 20260505 14:05:50.276 | INFO | "gpu" : " 17 GB", | 20260505 14:05:50.277 | INFO | "huge" : "", | 20260505 14:05:50.277 | INFO | "huge_shared" : " 27 GB", | 20260505 14:05:50.277 | INFO | "malloc" : " 14 GB", | 20260505 14:05:50.277 | INFO | "pinned" : " 29 GB" | 20260505 14:05:50.277 | INFO | } | 20260505 14:05:50.277 | INFO | Perf report for the last 5.13504 seconds: | 20260505 14:05:50.277 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:05:50.277 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:50.277 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:50.277 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:50.277 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 14:05:50.277 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:50.277 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:50.277 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:05:50.277 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:50.277 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:50.277 | INFO | Uploaded 36GiB to GPU in 2.15761s (16.6851GiB/s) | 20260505 14:05:50.277 | INFO | Downloaded 34GiB to GPU in 2.10528s (16.1499GiB/s) | 20260505 14:05:55.426 | INFO | Analyzing chunk frames = [3791872,3792384) | 20260505 14:05:55.426 | INFO | MemInfo: { | 20260505 14:05:55.426 | INFO | "gpu" : " 17 GB", | 20260505 14:05:55.426 | INFO | "huge" : "", | 20260505 14:05:55.426 | INFO | "huge_shared" : " 27 GB", | 20260505 14:05:55.426 | INFO | "malloc" : " 14 GB", | 20260505 14:05:55.426 | INFO | "pinned" : " 29 GB" | 20260505 14:05:55.426 | INFO | } | 20260505 14:05:55.426 | INFO | Perf report for the last 5.14968 seconds: | 20260505 14:05:55.426 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:05:55.426 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:55.426 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:55.426 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:55.426 | INFO | ANALYSIS : 63.1%, 3.0, 98.8ms, 0 | 20260505 14:05:55.426 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:55.426 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:55.426 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 14:05:55.426 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:55.426 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:05:55.427 | INFO | Uploaded 36GiB to GPU in 2.18861s (16.4488GiB/s) | 20260505 14:05:55.427 | INFO | Downloaded 34GiB to GPU in 2.15036s (15.8113GiB/s) | 20260505 14:06:00.567 | INFO | Analyzing chunk frames = [3792384,3792896) | 20260505 14:06:00.567 | INFO | MemInfo: { | 20260505 14:06:00.567 | INFO | "gpu" : " 17 GB", | 20260505 14:06:00.567 | INFO | "huge" : "", | 20260505 14:06:00.567 | INFO | "huge_shared" : " 27 GB", | 20260505 14:06:00.567 | INFO | "malloc" : " 14 GB", | 20260505 14:06:00.567 | INFO | "pinned" : " 29 GB" | 20260505 14:06:00.567 | INFO | } | 20260505 14:06:00.567 | INFO | Perf report for the last 5.14095 seconds: | 20260505 14:06:00.567 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:06:00.567 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:00.567 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:00.567 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:00.567 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 14:06:00.567 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:00.567 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:00.567 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 14:06:00.567 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:00.567 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:00.567 | INFO | Uploaded 36GiB to GPU in 2.21088s (16.2831GiB/s) | 20260505 14:06:00.568 | INFO | Downloaded 34GiB to GPU in 2.10526s (16.15GiB/s) | 20260505 14:06:00.568 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 14:06:00.568 | INFO | Total mem usage: 15.3 GiB | 20260505 14:06:00.568 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 14:06:00.568 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -7.04%) | 20260505 14:06:00.568 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 14:06:00.568 | INFO | High Water Breakdown: | 20260505 14:06:00.568 | INFO | 61.1% : 8.75 GiB ( 92% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 14:06:00.568 | INFO | 38.7% : 5.55 GiB ( 83.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 14:06:00.568 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 14:06:00.568 | INFO | ------------------------------------------ | 20260505 14:06:00.568 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:06:00.568 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 14:06:00.568 | INFO | Total mem usage: 26.5 GiB | 20260505 14:06:00.568 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 14:06:00.568 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 14:06:00.568 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 14:06:00.568 | INFO | High Water Breakdown: | 20260505 14:06:00.568 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:06:00.568 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:06:00.568 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:06:00.568 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:06:00.568 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:06:00.568 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:06:00.568 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 14:06:00.568 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 14:06:00.568 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:06:00.568 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 14:06:00.568 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 14:06:00.568 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 14:06:00.568 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 14:06:00.568 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 14:06:00.568 | INFO | ------------------------------------------ | 20260505 14:06:00.568 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:06:00.568 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 14:06:00.568 | INFO | Total mem usage: 25.6 GiB | 20260505 14:06:00.568 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 14:06:00.568 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 14:06:00.568 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 14:06:00.568 | INFO | High Water Breakdown: | 20260505 14:06:00.568 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 14:06:00.568 | INFO | ------------------------------------------ | 20260505 14:06:00.568 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:06:00.568 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 14:06:00.568 | INFO | Total mem usage: 15.7 GiB | 20260505 14:06:00.568 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 14:06:00.568 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 14:06:00.568 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 14:06:00.568 | INFO | High Water Breakdown: | 20260505 14:06:00.568 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 14:06:00.568 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 14:06:00.568 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 14:06:00.568 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 14:06:00.568 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 14:06:00.568 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 14:06:00.568 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 14:06:00.568 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 14:06:00.568 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 14:06:00.568 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:06:00.568 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 14:06:00.568 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:06:00.568 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 14:06:00.568 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 14:06:00.568 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 14:06:00.568 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 14:06:00.568 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:06:00.568 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 14:06:00.568 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:06:00.568 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:06:00.568 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:06:00.568 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 14:06:00.568 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 14:06:00.568 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:06:00.568 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 14:06:00.568 | INFO | ------------------------------------------ | 20260505 14:06:00.568 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:06:00.568 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 14:06:00.568 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:06:00.568 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:06:00.568 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.64e+01, 1.36e+01 / 1.54e+00 | 20260505 14:06:00.568 | INFO | PerfReport: 0.29%, AggregateStats -- 9.31e-02 / 1.00e-01, 1.14e-02 / 4.08e-03 | 20260505 14:06:00.568 | INFO | PerfReport: 11.75%, Baseline -- 3.74e+00 / 4.48e+00, 5.76e-01 / 1.91e-01 | 20260505 14:06:00.568 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.52e+00, 1.97e-01 / 7.98e-02 | 20260505 14:06:00.568 | INFO | PerfReport: 16.02%, DME -- 4.08e+01 / 4.58e+01, 4.68e+00 / 1.81e+00 | 20260505 14:06:00.568 | INFO | PerfReport: 56.36%, FrameLabeling -- 1.79e+01 / 1.97e+01, 1.34e+00 / 1.99e+00 | 20260505 14:06:00.568 | INFO | PerfReport: 7.49%, PulseAccumulating -- 2.38e+00 / 2.91e+00, 4.52e-01 / 2.03e-01 | 20260505 14:06:00.568 | INFO | PerfReport: 3.55%, Metrics -- 1.13e+00 / 1.45e+00, 3.15e-01 / 1.43e-01 | 20260505 14:06:00.568 | INFO | PerfReport: 1.25%, PulseSerializing -- 3.96e-01 / 4.55e-01, 7.04e-02 / 3.43e-02 | 20260505 14:06:00.568 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 14:06:00.568 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:06:00.568 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:06:00.568 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260505 14:06:00.568 | INFO | PerfReport: 22.57%, Upload -- 2.27e+01 / 0.00e+00, 8.12e+00 / 0.00e+00 | 20260505 14:06:00.568 | INFO | PerfReport: 22.02%, Download -- 2.22e+01 / 0.00e+00, 8.10e+00 / 0.00e+00 | 20260505 14:06:00.568 | INFO | PerfReport: 31.55%, Compute -- 3.18e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 14:06:00.568 | INFO | PerfReport: 2.34%, LockUpload -- 2.36e+00 / 0.00e+00, 8.09e+00 / 0.00e+00 | 20260505 14:06:00.568 | INFO | PerfReport: 1.42%, LockDownload -- 1.43e+00 / 0.00e+00, 2.96e+00 / 0.00e+00 | 20260505 14:06:00.568 | INFO | PerfReport: 20.09%, LockCompute -- 2.02e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260505 14:06:00.568 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.95e-02 / 0.00e+00 | 20260505 14:06:00.704 | INFO | Starting a round of Baz IO | 20260505 14:06:03.926 | INFO | Populating RTMetrics report | 20260505 14:06:03.927 | INFO | RTMetrics report populated | 20260505 14:06:03.927 | INFO | Opening tmp RTMetrics json file | 20260505 14:06:03.943 | INFO | Serializing RTMetrics to json | 20260505 14:06:03.949 | INFO | Serialization done, swapping tmp file now... | 20260505 14:06:03.965 | INFO | RTMetrics done | 20260505 14:06:04.029 | INFO | Finished a round of Baz IO: Wrote 2.274 GiB in 3.325 seconds | 20260505 14:06:04.029 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.87% / 0.00% / 77.12% | 20260505 14:06:04.029 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.608595% / 7.94729e-06% / 0.000103315% | 20260505 14:06:04.030 | INFO | Baz IO overall: | 20260505 14:06:04.030 | INFO | BytesWritten : 2259911801500 | 20260505 14:06:04.030 | INFO | NumEvents : 1074731440134 | 20260505 14:06:05.702 | INFO | Analyzing chunk frames = [3792896,3793408) | 20260505 14:06:05.702 | INFO | MemInfo: { | 20260505 14:06:05.702 | INFO | "gpu" : " 17 GB", | 20260505 14:06:05.702 | INFO | "huge" : "", | 20260505 14:06:05.702 | INFO | "huge_shared" : " 27 GB", | 20260505 14:06:05.702 | INFO | "malloc" : " 14 GB", | 20260505 14:06:05.702 | INFO | "pinned" : " 29 GB" | 20260505 14:06:05.702 | INFO | } | 20260505 14:06:05.702 | INFO | Perf report for the last 5.13481 seconds: | 20260505 14:06:05.702 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:06:05.702 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:05.702 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:05.702 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:05.702 | INFO | ANALYSIS : 64.3%, 3.0, 99.9ms, 0 | 20260505 14:06:05.702 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:05.702 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260505 14:06:05.702 | INFO | PRE_HQ : 61.0%, 1.8, 57.4ms, 0 | 20260505 14:06:05.702 | INFO | BAZWRITER : 64.1%, 4.0, 130.5ms, 0 | 20260505 14:06:05.702 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 14:06:05.702 | INFO | Uploaded 36GiB to GPU in 2.19251s (16.4195GiB/s) | 20260505 14:06:05.702 | INFO | Downloaded 38GiB to GPU in 2.26543s (16.7739GiB/s) | 20260505 14:06:10.838 | INFO | Analyzing chunk frames = [3793408,3793920) | 20260505 14:06:10.838 | INFO | MemInfo: { | 20260505 14:06:10.838 | INFO | "gpu" : " 17 GB", | 20260505 14:06:10.838 | INFO | "huge" : "", | 20260505 14:06:10.838 | INFO | "huge_shared" : " 27 GB", | 20260505 14:06:10.838 | INFO | "malloc" : " 14 GB", | 20260505 14:06:10.838 | INFO | "pinned" : " 29 GB" | 20260505 14:06:10.838 | INFO | } | 20260505 14:06:10.838 | INFO | Perf report for the last 5.1361 seconds: | 20260505 14:06:10.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:06:10.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:10.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:10.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:10.838 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 14:06:10.838 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:10.838 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:10.838 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:06:10.838 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:10.838 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:10.839 | INFO | Uploaded 36GiB to GPU in 2.16726s (16.6108GiB/s) | 20260505 14:06:10.839 | INFO | Downloaded 34GiB to GPU in 2.03051s (16.7445GiB/s) | 20260505 14:06:15.989 | INFO | Analyzing chunk frames = [3793920,3794432) | 20260505 14:06:15.989 | INFO | MemInfo: { | 20260505 14:06:15.989 | INFO | "gpu" : " 17 GB", | 20260505 14:06:15.989 | INFO | "huge" : "", | 20260505 14:06:15.989 | INFO | "huge_shared" : " 27 GB", | 20260505 14:06:15.989 | INFO | "malloc" : " 14 GB", | 20260505 14:06:15.989 | INFO | "pinned" : " 29 GB" | 20260505 14:06:15.989 | INFO | } | 20260505 14:06:15.989 | INFO | Perf report for the last 5.15116 seconds: | 20260505 14:06:15.989 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:06:15.989 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:15.989 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:15.989 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:15.989 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 14:06:15.989 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:15.989 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:15.989 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:06:15.989 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:15.989 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:15.990 | INFO | Uploaded 36GiB to GPU in 2.22288s (16.1952GiB/s) | 20260505 14:06:15.990 | INFO | Downloaded 34GiB to GPU in 2.10669s (16.139GiB/s) | 20260505 14:06:21.125 | INFO | Analyzing chunk frames = [3794432,3794944) | 20260505 14:06:21.125 | INFO | MemInfo: { | 20260505 14:06:21.125 | INFO | "gpu" : " 17 GB", | 20260505 14:06:21.125 | INFO | "huge" : "", | 20260505 14:06:21.125 | INFO | "huge_shared" : " 27 GB", | 20260505 14:06:21.125 | INFO | "malloc" : " 14 GB", | 20260505 14:06:21.125 | INFO | "pinned" : " 29 GB" | 20260505 14:06:21.125 | INFO | } | 20260505 14:06:21.125 | INFO | Perf report for the last 5.13533 seconds: | 20260505 14:06:21.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:06:21.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:21.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:21.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:21.125 | INFO | ANALYSIS : 62.6%, 3.0, 97.2ms, 0 | 20260505 14:06:21.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:21.125 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:21.125 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 14:06:21.125 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:21.125 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:21.125 | INFO | Uploaded 36GiB to GPU in 2.19302s (16.4157GiB/s) | 20260505 14:06:21.125 | INFO | Downloaded 34GiB to GPU in 2.13332s (15.9376GiB/s) | 20260505 14:06:26.262 | INFO | Analyzing chunk frames = [3794944,3795456) | 20260505 14:06:26.262 | INFO | MemInfo: { | 20260505 14:06:26.262 | INFO | "gpu" : " 17 GB", | 20260505 14:06:26.262 | INFO | "huge" : "", | 20260505 14:06:26.262 | INFO | "huge_shared" : " 27 GB", | 20260505 14:06:26.262 | INFO | "malloc" : " 14 GB", | 20260505 14:06:26.262 | INFO | "pinned" : " 29 GB" | 20260505 14:06:26.262 | INFO | } | 20260505 14:06:26.262 | INFO | Perf report for the last 5.13747 seconds: | 20260505 14:06:26.262 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:06:26.262 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:26.262 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:26.262 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:26.262 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 14:06:26.262 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:26.262 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:26.262 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:06:26.262 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:26.262 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:26.262 | INFO | Uploaded 36GiB to GPU in 2.15811s (16.6812GiB/s) | 20260505 14:06:26.262 | INFO | Downloaded 34GiB to GPU in 2.12824s (15.9756GiB/s) | 20260505 14:06:31.402 | INFO | Analyzing chunk frames = [3795456,3795968) | 20260505 14:06:31.402 | INFO | MemInfo: { | 20260505 14:06:31.402 | INFO | "gpu" : " 17 GB", | 20260505 14:06:31.402 | INFO | "huge" : "", | 20260505 14:06:31.402 | INFO | "huge_shared" : " 27 GB", | 20260505 14:06:31.402 | INFO | "malloc" : " 14 GB", | 20260505 14:06:31.402 | INFO | "pinned" : " 29 GB" | 20260505 14:06:31.402 | INFO | } | 20260505 14:06:31.402 | INFO | Perf report for the last 5.13977 seconds: | 20260505 14:06:31.402 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:06:31.402 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:31.402 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:31.402 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:31.402 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 14:06:31.402 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:31.402 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:31.402 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:06:31.402 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:31.402 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:31.402 | INFO | Uploaded 36GiB to GPU in 2.17514s (16.5507GiB/s) | 20260505 14:06:31.402 | INFO | Downloaded 34GiB to GPU in 2.11142s (16.1029GiB/s) | 20260505 14:06:36.538 | INFO | Analyzing chunk frames = [3795968,3796480) | 20260505 14:06:36.538 | INFO | MemInfo: { | 20260505 14:06:36.538 | INFO | "gpu" : " 17 GB", | 20260505 14:06:36.538 | INFO | "huge" : "", | 20260505 14:06:36.538 | INFO | "huge_shared" : " 27 GB", | 20260505 14:06:36.538 | INFO | "malloc" : " 14 GB", | 20260505 14:06:36.538 | INFO | "pinned" : " 29 GB" | 20260505 14:06:36.538 | INFO | } | 20260505 14:06:36.538 | INFO | Perf report for the last 5.13625 seconds: | 20260505 14:06:36.538 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:06:36.538 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:36.538 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:36.538 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:36.538 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 14:06:36.538 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:36.538 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:36.538 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 14:06:36.538 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:36.538 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:36.538 | INFO | Uploaded 36GiB to GPU in 2.13244s (16.8821GiB/s) | 20260505 14:06:36.539 | INFO | Downloaded 34GiB to GPU in 2.13444s (15.9292GiB/s) | 20260505 14:06:41.681 | INFO | Analyzing chunk frames = [3796480,3796992) | 20260505 14:06:41.681 | INFO | MemInfo: { | 20260505 14:06:41.681 | INFO | "gpu" : " 17 GB", | 20260505 14:06:41.681 | INFO | "huge" : "", | 20260505 14:06:41.681 | INFO | "huge_shared" : " 27 GB", | 20260505 14:06:41.681 | INFO | "malloc" : " 14 GB", | 20260505 14:06:41.681 | INFO | "pinned" : " 29 GB" | 20260505 14:06:41.681 | INFO | } | 20260505 14:06:41.681 | INFO | Perf report for the last 5.14319 seconds: | 20260505 14:06:41.681 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:06:41.681 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:41.681 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:41.681 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:41.681 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 14:06:41.681 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:41.682 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:41.682 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 14:06:41.682 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:41.682 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:41.682 | INFO | Uploaded 36GiB to GPU in 2.21497s (16.2531GiB/s) | 20260505 14:06:41.682 | INFO | Downloaded 34GiB to GPU in 2.10333s (16.1649GiB/s) | 20260505 14:06:41.819 | INFO | Starting a round of Baz IO | 20260505 14:06:45.041 | INFO | Populating RTMetrics report | 20260505 14:06:45.042 | INFO | RTMetrics report populated | 20260505 14:06:45.042 | INFO | Opening tmp RTMetrics json file | 20260505 14:06:45.056 | INFO | Serializing RTMetrics to json | 20260505 14:06:45.059 | INFO | Serialization done, swapping tmp file now... | 20260505 14:06:45.062 | INFO | RTMetrics done | 20260505 14:06:45.143 | INFO | Finished a round of Baz IO: Wrote 2.273 GiB in 3.323 seconds | 20260505 14:06:45.143 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.87% / 0.00% / 77.13% | 20260505 14:06:45.143 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.60637% / 3.97364e-06% / 6.75519e-05% | 20260505 14:06:45.143 | INFO | Baz IO overall: | 20260505 14:06:45.143 | INFO | BytesWritten : 2262352891945 | 20260505 14:06:45.143 | INFO | NumEvents : 1075884042527 | 20260505 14:06:46.828 | INFO | Analyzing chunk frames = [3796992,3797504) | 20260505 14:06:46.828 | INFO | MemInfo: { | 20260505 14:06:46.828 | INFO | "gpu" : " 17 GB", | 20260505 14:06:46.828 | INFO | "huge" : "", | 20260505 14:06:46.828 | INFO | "huge_shared" : " 27 GB", | 20260505 14:06:46.828 | INFO | "malloc" : " 14 GB", | 20260505 14:06:46.828 | INFO | "pinned" : " 29 GB" | 20260505 14:06:46.828 | INFO | } | 20260505 14:06:46.828 | INFO | Perf report for the last 5.14638 seconds: | 20260505 14:06:46.828 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:06:46.828 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:46.828 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:46.828 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:46.828 | INFO | ANALYSIS : 64.2%, 3.0, 99.7ms, 0 | 20260505 14:06:46.828 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:46.828 | INFO | RT_METRICS : 45.3%, 1.0, 23.8ms, 0 | 20260505 14:06:46.828 | INFO | PRE_HQ : 60.8%, 1.8, 57.5ms, 0 | 20260505 14:06:46.828 | INFO | BAZWRITER : 64.0%, 4.0, 130.7ms, 0 | 20260505 14:06:46.828 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 14:06:46.828 | INFO | Uploaded 36GiB to GPU in 2.20291s (16.3421GiB/s) | 20260505 14:06:46.828 | INFO | Downloaded 38GiB to GPU in 2.26224s (16.7975GiB/s) | 20260505 14:06:51.953 | INFO | Analyzing chunk frames = [3797504,3798016) | 20260505 14:06:51.953 | INFO | MemInfo: { | 20260505 14:06:51.953 | INFO | "gpu" : " 17 GB", | 20260505 14:06:51.953 | INFO | "huge" : "", | 20260505 14:06:51.953 | INFO | "huge_shared" : " 27 GB", | 20260505 14:06:51.953 | INFO | "malloc" : " 14 GB", | 20260505 14:06:51.953 | INFO | "pinned" : " 29 GB" | 20260505 14:06:51.953 | INFO | } | 20260505 14:06:51.953 | INFO | Perf report for the last 5.12562 seconds: | 20260505 14:06:51.953 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:06:51.953 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:51.953 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:51.953 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:51.954 | INFO | ANALYSIS : 63.7%, 3.0, 99.2ms, 0 | 20260505 14:06:51.954 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:51.954 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:51.954 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:06:51.954 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:51.954 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:51.954 | INFO | Uploaded 36GiB to GPU in 2.20492s (16.3271GiB/s) | 20260505 14:06:51.954 | INFO | Downloaded 34GiB to GPU in 2.05744s (16.5254GiB/s) | 20260505 14:06:57.087 | INFO | Analyzing chunk frames = [3798016,3798528) | 20260505 14:06:57.087 | INFO | MemInfo: { | 20260505 14:06:57.087 | INFO | "gpu" : " 17 GB", | 20260505 14:06:57.087 | INFO | "huge" : "", | 20260505 14:06:57.087 | INFO | "huge_shared" : " 27 GB", | 20260505 14:06:57.087 | INFO | "malloc" : " 14 GB", | 20260505 14:06:57.087 | INFO | "pinned" : " 29 GB" | 20260505 14:06:57.087 | INFO | } | 20260505 14:06:57.088 | INFO | Perf report for the last 5.13405 seconds: | 20260505 14:06:57.088 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:06:57.088 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:57.088 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:57.088 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:57.088 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 14:06:57.088 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:57.088 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:57.088 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:06:57.088 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:57.088 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:06:57.088 | INFO | Uploaded 36GiB to GPU in 2.20113s (16.3553GiB/s) | 20260505 14:06:57.088 | INFO | Downloaded 34GiB to GPU in 2.12183s (16.0239GiB/s) | 20260505 14:07:02.236 | INFO | Analyzing chunk frames = [3798528,3799040) | 20260505 14:07:02.236 | INFO | MemInfo: { | 20260505 14:07:02.236 | INFO | "gpu" : " 17 GB", | 20260505 14:07:02.236 | INFO | "huge" : "", | 20260505 14:07:02.236 | INFO | "huge_shared" : " 27 GB", | 20260505 14:07:02.236 | INFO | "malloc" : " 14 GB", | 20260505 14:07:02.236 | INFO | "pinned" : " 29 GB" | 20260505 14:07:02.236 | INFO | } | 20260505 14:07:02.236 | INFO | Perf report for the last 5.14837 seconds: | 20260505 14:07:02.236 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:07:02.236 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:02.236 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:02.236 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:02.236 | INFO | ANALYSIS : 63.0%, 3.0, 98.0ms, 0 | 20260505 14:07:02.236 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:02.236 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:02.236 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 14:07:02.236 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:02.236 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:02.236 | INFO | Uploaded 36GiB to GPU in 2.20878s (16.2986GiB/s) | 20260505 14:07:02.236 | INFO | Downloaded 34GiB to GPU in 2.13618s (15.9162GiB/s) | 20260505 14:07:07.379 | INFO | Analyzing chunk frames = [3799040,3799552) | 20260505 14:07:07.379 | INFO | MemInfo: { | 20260505 14:07:07.379 | INFO | "gpu" : " 17 GB", | 20260505 14:07:07.379 | INFO | "huge" : "", | 20260505 14:07:07.379 | INFO | "huge_shared" : " 27 GB", | 20260505 14:07:07.379 | INFO | "malloc" : " 14 GB", | 20260505 14:07:07.379 | INFO | "pinned" : " 29 GB" | 20260505 14:07:07.379 | INFO | } | 20260505 14:07:07.379 | INFO | Perf report for the last 5.14354 seconds: | 20260505 14:07:07.379 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:07:07.379 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:07.379 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:07.379 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:07.380 | INFO | ANALYSIS : 62.6%, 3.0, 97.9ms, 0 | 20260505 14:07:07.380 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:07.380 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:07.380 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 14:07:07.380 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:07.380 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:07.380 | INFO | Uploaded 36GiB to GPU in 2.14674s (16.7696GiB/s) | 20260505 14:07:07.380 | INFO | Downloaded 34GiB to GPU in 2.12274s (16.017GiB/s) | 20260505 14:07:12.519 | INFO | Analyzing chunk frames = [3799552,3800064) | 20260505 14:07:12.519 | INFO | MemInfo: { | 20260505 14:07:12.519 | INFO | "gpu" : " 17 GB", | 20260505 14:07:12.519 | INFO | "huge" : "", | 20260505 14:07:12.519 | INFO | "huge_shared" : " 27 GB", | 20260505 14:07:12.519 | INFO | "malloc" : " 14 GB", | 20260505 14:07:12.519 | INFO | "pinned" : " 29 GB" | 20260505 14:07:12.519 | INFO | } | 20260505 14:07:12.519 | INFO | Perf report for the last 5.13948 seconds: | 20260505 14:07:12.519 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:07:12.519 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:12.519 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:12.519 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:12.519 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 14:07:12.519 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:12.519 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:12.519 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:07:12.519 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:12.519 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:12.519 | INFO | Uploaded 36GiB to GPU in 2.19168s (16.4258GiB/s) | 20260505 14:07:12.519 | INFO | Downloaded 34GiB to GPU in 2.11131s (16.1037GiB/s) | 20260505 14:07:17.649 | INFO | Analyzing chunk frames = [3800064,3800576) | 20260505 14:07:17.649 | INFO | MemInfo: { | 20260505 14:07:17.649 | INFO | "gpu" : " 17 GB", | 20260505 14:07:17.649 | INFO | "huge" : "", | 20260505 14:07:17.649 | INFO | "huge_shared" : " 27 GB", | 20260505 14:07:17.649 | INFO | "malloc" : " 14 GB", | 20260505 14:07:17.649 | INFO | "pinned" : " 29 GB" | 20260505 14:07:17.649 | INFO | } | 20260505 14:07:17.649 | INFO | Perf report for the last 5.12973 seconds: | 20260505 14:07:17.649 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:07:17.649 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:17.649 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:17.649 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:17.649 | INFO | ANALYSIS : 64.4%, 3.0, 100.3ms, 0 | 20260505 14:07:17.649 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:17.649 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:17.649 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 14:07:17.649 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:17.649 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:17.649 | INFO | Uploaded 36GiB to GPU in 2.17435s (16.5567GiB/s) | 20260505 14:07:17.649 | INFO | Downloaded 34GiB to GPU in 2.20612s (15.4116GiB/s) | 20260505 14:07:22.788 | INFO | Analyzing chunk frames = [3800576,3801088) | 20260505 14:07:22.788 | INFO | MemInfo: { | 20260505 14:07:22.788 | INFO | "gpu" : " 17 GB", | 20260505 14:07:22.788 | INFO | "huge" : "", | 20260505 14:07:22.788 | INFO | "huge_shared" : " 27 GB", | 20260505 14:07:22.788 | INFO | "malloc" : " 14 GB", | 20260505 14:07:22.788 | INFO | "pinned" : " 29 GB" | 20260505 14:07:22.788 | INFO | } | 20260505 14:07:22.788 | INFO | Perf report for the last 5.13941 seconds: | 20260505 14:07:22.788 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:07:22.788 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:22.788 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:22.788 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:22.788 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 14:07:22.788 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:22.788 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:22.788 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 14:07:22.788 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:22.788 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:22.788 | INFO | Uploaded 36GiB to GPU in 2.17764s (16.5316GiB/s) | 20260505 14:07:22.788 | INFO | Downloaded 34GiB to GPU in 2.11207s (16.0979GiB/s) | 20260505 14:07:22.788 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 14:07:22.788 | INFO | Total mem usage: 15.3 GiB | 20260505 14:07:22.788 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 14:07:22.788 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.76%) | 20260505 14:07:22.788 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 14:07:22.788 | INFO | High Water Breakdown: | 20260505 14:07:22.788 | INFO | 61% : 8.68 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 14:07:22.788 | INFO | 38.8% : 5.52 GiB ( 83.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 14:07:22.788 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 14:07:22.789 | INFO | ------------------------------------------ | 20260505 14:07:22.789 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:07:22.789 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 14:07:22.789 | INFO | Total mem usage: 26.5 GiB | 20260505 14:07:22.789 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 14:07:22.789 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 14:07:22.789 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 14:07:22.789 | INFO | High Water Breakdown: | 20260505 14:07:22.789 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:07:22.789 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:07:22.789 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:07:22.789 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:07:22.789 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:07:22.789 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:07:22.789 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 14:07:22.789 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 14:07:22.789 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:07:22.789 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 14:07:22.789 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 14:07:22.789 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 14:07:22.789 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 14:07:22.789 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 14:07:22.789 | INFO | ------------------------------------------ | 20260505 14:07:22.789 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:07:22.789 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 14:07:22.789 | INFO | Total mem usage: 25.6 GiB | 20260505 14:07:22.789 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 14:07:22.789 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 14:07:22.789 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 14:07:22.789 | INFO | High Water Breakdown: | 20260505 14:07:22.789 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 14:07:22.789 | INFO | ------------------------------------------ | 20260505 14:07:22.789 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:07:22.789 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 14:07:22.789 | INFO | Total mem usage: 15.7 GiB | 20260505 14:07:22.789 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 14:07:22.789 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 14:07:22.789 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 14:07:22.789 | INFO | High Water Breakdown: | 20260505 14:07:22.789 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 14:07:22.789 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 14:07:22.789 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 14:07:22.789 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 14:07:22.789 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 14:07:22.789 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 14:07:22.789 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 14:07:22.789 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 14:07:22.789 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 14:07:22.789 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:07:22.789 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 14:07:22.789 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:07:22.789 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 14:07:22.789 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 14:07:22.789 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 14:07:22.789 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 14:07:22.789 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:07:22.789 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 14:07:22.789 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:07:22.789 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:07:22.789 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:07:22.789 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 14:07:22.789 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 14:07:22.789 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:07:22.789 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 14:07:22.789 | INFO | ------------------------------------------ | 20260505 14:07:22.789 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:07:22.789 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 14:07:22.789 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:07:22.789 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:07:22.789 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.72e+01, 1.38e+01 / 2.11e+00 | 20260505 14:07:22.789 | INFO | PerfReport: 0.29%, AggregateStats -- 9.30e-02 / 9.93e-02, 1.10e-02 / 4.26e-03 | 20260505 14:07:22.789 | INFO | PerfReport: 11.76%, Baseline -- 3.75e+00 / 4.47e+00, 5.81e-01 / 1.77e-01 | 20260505 14:07:22.789 | INFO | PerfReport: 3.31%, Histogram -- 1.05e+00 / 1.50e+00, 2.00e-01 / 9.11e-02 | 20260505 14:07:22.789 | INFO | PerfReport: 16.10%, DME -- 4.10e+01 / 4.67e+01, 4.88e+00 / 2.43e+00 | 20260505 14:07:22.789 | INFO | PerfReport: 56.32%, FrameLabeling -- 1.79e+01 / 1.96e+01, 1.38e+00 / 1.61e+00 | 20260505 14:07:22.789 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.37e+00 / 2.95e+00, 4.54e-01 / 1.35e-01 | 20260505 14:07:22.789 | INFO | PerfReport: 3.53%, Metrics -- 1.13e+00 / 1.43e+00, 3.14e-01 / 1.37e-01 | 20260505 14:07:22.789 | INFO | PerfReport: 1.24%, PulseSerializing -- 3.95e-01 / 4.41e-01, 6.97e-02 / 3.47e-02 | 20260505 14:07:22.789 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 14:07:22.789 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:07:22.789 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:07:22.789 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260505 14:07:22.789 | INFO | PerfReport: 22.58%, Upload -- 2.28e+01 / 0.00e+00, 8.04e+00 / 0.00e+00 | 20260505 14:07:22.789 | INFO | PerfReport: 22.05%, Download -- 2.22e+01 / 0.00e+00, 8.11e+00 / 0.00e+00 | 20260505 14:07:22.789 | INFO | PerfReport: 31.60%, Compute -- 3.19e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 14:07:22.789 | INFO | PerfReport: 2.16%, LockUpload -- 2.18e+00 / 0.00e+00, 7.87e+00 / 0.00e+00 | 20260505 14:07:22.789 | INFO | PerfReport: 1.49%, LockDownload -- 1.50e+00 / 0.00e+00, 3.01e+00 / 0.00e+00 | 20260505 14:07:22.789 | INFO | PerfReport: 20.10%, LockCompute -- 2.03e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260505 14:07:22.789 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.18e-02 / 0.00e+00 | 20260505 14:07:22.926 | INFO | Starting a round of Baz IO | 20260505 14:07:26.146 | INFO | Populating RTMetrics report | 20260505 14:07:26.147 | INFO | RTMetrics report populated | 20260505 14:07:26.147 | INFO | Opening tmp RTMetrics json file | 20260505 14:07:26.159 | INFO | Serializing RTMetrics to json | 20260505 14:07:26.160 | INFO | Serialization done, swapping tmp file now... | 20260505 14:07:26.160 | INFO | RTMetrics done | 20260505 14:07:26.249 | INFO | Finished a round of Baz IO: Wrote 2.27 GiB in 3.322 seconds | 20260505 14:07:26.249 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.87% / 0.00% / 77.13% | 20260505 14:07:26.249 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.604653% / 1.58946e-05% / 7.15256e-05% | 20260505 14:07:26.249 | INFO | Baz IO overall: | 20260505 14:07:26.249 | INFO | BytesWritten : 2264790806073 | 20260505 14:07:26.249 | INFO | NumEvents : 1077034331739 | 20260505 14:07:27.938 | INFO | Analyzing chunk frames = [3801088,3801600) | 20260505 14:07:27.938 | INFO | MemInfo: { | 20260505 14:07:27.938 | INFO | "gpu" : " 17 GB", | 20260505 14:07:27.938 | INFO | "huge" : "", | 20260505 14:07:27.938 | INFO | "huge_shared" : " 27 GB", | 20260505 14:07:27.938 | INFO | "malloc" : " 14 GB", | 20260505 14:07:27.938 | INFO | "pinned" : " 29 GB" | 20260505 14:07:27.938 | INFO | } | 20260505 14:07:27.938 | INFO | Perf report for the last 5.1499 seconds: | 20260505 14:07:27.938 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:07:27.938 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:27.938 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:27.938 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:27.938 | INFO | ANALYSIS : 64.1%, 3.0, 99.8ms, 0 | 20260505 14:07:27.938 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:27.938 | INFO | RT_METRICS : 45.1%, 1.0, 23.7ms, 0 | 20260505 14:07:27.938 | INFO | PRE_HQ : 60.6%, 1.8, 57.2ms, 0 | 20260505 14:07:27.938 | INFO | BAZWRITER : 63.9%, 4.0, 131.3ms, 0 | 20260505 14:07:27.938 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 14:07:27.938 | INFO | Uploaded 36GiB to GPU in 2.17854s (16.5248GiB/s) | 20260505 14:07:27.938 | INFO | Downloaded 38GiB to GPU in 2.26883s (16.7487GiB/s) | 20260505 14:07:33.072 | INFO | Analyzing chunk frames = [3801600,3802112) | 20260505 14:07:33.072 | INFO | MemInfo: { | 20260505 14:07:33.072 | INFO | "gpu" : " 17 GB", | 20260505 14:07:33.072 | INFO | "huge" : "", | 20260505 14:07:33.072 | INFO | "huge_shared" : " 27 GB", | 20260505 14:07:33.072 | INFO | "malloc" : " 14 GB", | 20260505 14:07:33.072 | INFO | "pinned" : " 29 GB" | 20260505 14:07:33.072 | INFO | } | 20260505 14:07:33.072 | INFO | Perf report for the last 5.13403 seconds: | 20260505 14:07:33.072 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:07:33.072 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:33.072 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:33.072 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:33.072 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 14:07:33.072 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:33.072 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:33.072 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:07:33.072 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:33.072 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:33.072 | INFO | Uploaded 36GiB to GPU in 2.19918s (16.3697GiB/s) | 20260505 14:07:33.072 | INFO | Downloaded 34GiB to GPU in 2.03947s (16.671GiB/s) | 20260505 14:07:38.211 | INFO | Analyzing chunk frames = [3802112,3802624) | 20260505 14:07:38.211 | INFO | MemInfo: { | 20260505 14:07:38.211 | INFO | "gpu" : " 17 GB", | 20260505 14:07:38.211 | INFO | "huge" : "", | 20260505 14:07:38.211 | INFO | "huge_shared" : " 27 GB", | 20260505 14:07:38.211 | INFO | "malloc" : " 14 GB", | 20260505 14:07:38.211 | INFO | "pinned" : " 29 GB" | 20260505 14:07:38.211 | INFO | } | 20260505 14:07:38.211 | INFO | Perf report for the last 5.1386 seconds: | 20260505 14:07:38.211 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:07:38.211 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:38.211 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:38.211 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:38.211 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 14:07:38.211 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:38.211 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:38.211 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:07:38.211 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:38.211 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:38.211 | INFO | Uploaded 36GiB to GPU in 2.22896s (16.151GiB/s) | 20260505 14:07:38.211 | INFO | Downloaded 34GiB to GPU in 2.10119s (16.1813GiB/s) | 20260505 14:07:43.351 | INFO | Analyzing chunk frames = [3802624,3803136) | 20260505 14:07:43.351 | INFO | MemInfo: { | 20260505 14:07:43.351 | INFO | "gpu" : " 17 GB", | 20260505 14:07:43.351 | INFO | "huge" : "", | 20260505 14:07:43.351 | INFO | "huge_shared" : " 27 GB", | 20260505 14:07:43.351 | INFO | "malloc" : " 14 GB", | 20260505 14:07:43.351 | INFO | "pinned" : " 29 GB" | 20260505 14:07:43.351 | INFO | } | 20260505 14:07:43.351 | INFO | Perf report for the last 5.14037 seconds: | 20260505 14:07:43.351 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:07:43.351 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:43.351 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:43.351 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:43.351 | INFO | ANALYSIS : 62.9%, 3.0, 97.8ms, 0 | 20260505 14:07:43.351 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:43.351 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:43.351 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:07:43.351 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:43.351 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:43.351 | INFO | Uploaded 36GiB to GPU in 2.20371s (16.3361GiB/s) | 20260505 14:07:43.351 | INFO | Downloaded 34GiB to GPU in 2.13194s (15.9479GiB/s) | 20260505 14:07:48.495 | INFO | Analyzing chunk frames = [3803136,3803648) | 20260505 14:07:48.495 | INFO | MemInfo: { | 20260505 14:07:48.495 | INFO | "gpu" : " 17 GB", | 20260505 14:07:48.495 | INFO | "huge" : "", | 20260505 14:07:48.495 | INFO | "huge_shared" : " 27 GB", | 20260505 14:07:48.495 | INFO | "malloc" : " 14 GB", | 20260505 14:07:48.495 | INFO | "pinned" : " 29 GB" | 20260505 14:07:48.495 | INFO | } | 20260505 14:07:48.495 | INFO | Perf report for the last 5.14408 seconds: | 20260505 14:07:48.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:07:48.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:48.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:48.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:48.495 | INFO | ANALYSIS : 62.7%, 3.0, 98.0ms, 0 | 20260505 14:07:48.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:48.495 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:48.495 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:07:48.495 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:48.495 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:48.495 | INFO | Uploaded 36GiB to GPU in 2.16049s (16.6629GiB/s) | 20260505 14:07:48.495 | INFO | Downloaded 34GiB to GPU in 2.12569s (15.9948GiB/s) | 20260505 14:07:53.636 | INFO | Analyzing chunk frames = [3803648,3804160) | 20260505 14:07:53.636 | INFO | MemInfo: { | 20260505 14:07:53.636 | INFO | "gpu" : " 17 GB", | 20260505 14:07:53.636 | INFO | "huge" : "", | 20260505 14:07:53.636 | INFO | "huge_shared" : " 27 GB", | 20260505 14:07:53.636 | INFO | "malloc" : " 14 GB", | 20260505 14:07:53.636 | INFO | "pinned" : " 29 GB" | 20260505 14:07:53.636 | INFO | } | 20260505 14:07:53.636 | INFO | Perf report for the last 5.14066 seconds: | 20260505 14:07:53.636 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:07:53.636 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:53.636 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:53.636 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:53.636 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 14:07:53.636 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:53.636 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:53.636 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 14:07:53.636 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:53.636 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:53.636 | INFO | Uploaded 36GiB to GPU in 2.20226s (16.3468GiB/s) | 20260505 14:07:53.636 | INFO | Downloaded 34GiB to GPU in 2.11273s (16.0929GiB/s) | 20260505 14:07:58.772 | INFO | Analyzing chunk frames = [3804160,3804672) | 20260505 14:07:58.772 | INFO | MemInfo: { | 20260505 14:07:58.772 | INFO | "gpu" : " 17 GB", | 20260505 14:07:58.772 | INFO | "huge" : "", | 20260505 14:07:58.772 | INFO | "huge_shared" : " 27 GB", | 20260505 14:07:58.772 | INFO | "malloc" : " 14 GB", | 20260505 14:07:58.772 | INFO | "pinned" : " 29 GB" | 20260505 14:07:58.772 | INFO | } | 20260505 14:07:58.772 | INFO | Perf report for the last 5.13636 seconds: | 20260505 14:07:58.772 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:07:58.772 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:58.772 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:58.772 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:58.772 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 14:07:58.772 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:58.772 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:58.772 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 14:07:58.772 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:58.772 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:07:58.772 | INFO | Uploaded 36GiB to GPU in 2.17967s (16.5162GiB/s) | 20260505 14:07:58.773 | INFO | Downloaded 34GiB to GPU in 2.14507s (15.8503GiB/s) | 20260505 14:08:03.901 | INFO | Analyzing chunk frames = [3804672,3805184) | 20260505 14:08:03.901 | INFO | MemInfo: { | 20260505 14:08:03.901 | INFO | "gpu" : " 17 GB", | 20260505 14:08:03.901 | INFO | "huge" : "", | 20260505 14:08:03.901 | INFO | "huge_shared" : " 27 GB", | 20260505 14:08:03.901 | INFO | "malloc" : " 14 GB", | 20260505 14:08:03.901 | INFO | "pinned" : " 29 GB" | 20260505 14:08:03.901 | INFO | } | 20260505 14:08:03.901 | INFO | Perf report for the last 5.12907 seconds: | 20260505 14:08:03.901 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:08:03.901 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:03.901 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:03.901 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:03.901 | INFO | ANALYSIS : 63.8%, 3.0, 99.4ms, 0 | 20260505 14:08:03.901 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:03.901 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:03.901 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 14:08:03.901 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:03.901 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:03.902 | INFO | Uploaded 36GiB to GPU in 2.21622s (16.2439GiB/s) | 20260505 14:08:03.902 | INFO | Downloaded 34GiB to GPU in 2.10986s (16.1148GiB/s) | 20260505 14:08:04.030 | INFO | Starting a round of Baz IO | 20260505 14:08:07.275 | INFO | Populating RTMetrics report | 20260505 14:08:07.276 | INFO | RTMetrics report populated | 20260505 14:08:07.276 | INFO | Opening tmp RTMetrics json file | 20260505 14:08:07.290 | INFO | Serializing RTMetrics to json | 20260505 14:08:07.291 | INFO | Serialization done, swapping tmp file now... | 20260505 14:08:07.291 | INFO | RTMetrics done | 20260505 14:08:07.376 | INFO | Finished a round of Baz IO: Wrote 2.272 GiB in 3.346 seconds | 20260505 14:08:07.376 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.86% / 0.00% / 77.13% | 20260505 14:08:07.376 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.602098% / 7.94729e-06% / 8.74201e-05% | 20260505 14:08:07.376 | INFO | Baz IO overall: | 20260505 14:08:07.376 | INFO | BytesWritten : 2267230414105 | 20260505 14:08:07.376 | INFO | NumEvents : 1078185744732 | 20260505 14:08:09.044 | INFO | Analyzing chunk frames = [3805184,3805696) | 20260505 14:08:09.044 | INFO | MemInfo: { | 20260505 14:08:09.044 | INFO | "gpu" : " 17 GB", | 20260505 14:08:09.044 | INFO | "huge" : "", | 20260505 14:08:09.044 | INFO | "huge_shared" : " 27 GB", | 20260505 14:08:09.044 | INFO | "malloc" : " 14 GB", | 20260505 14:08:09.044 | INFO | "pinned" : " 29 GB" | 20260505 14:08:09.044 | INFO | } | 20260505 14:08:09.045 | INFO | Perf report for the last 5.14323 seconds: | 20260505 14:08:09.045 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:08:09.045 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:09.045 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:09.045 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:09.045 | INFO | ANALYSIS : 64.5%, 3.0, 100.0ms, 0 | 20260505 14:08:09.045 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:09.045 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260505 14:08:09.045 | INFO | PRE_HQ : 60.6%, 1.8, 57.5ms, 0 | 20260505 14:08:09.045 | INFO | BAZWRITER : 64.4%, 4.0, 132.0ms, 0 | 20260505 14:08:09.045 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 14:08:09.045 | INFO | Uploaded 36GiB to GPU in 2.18127s (16.5041GiB/s) | 20260505 14:08:09.045 | INFO | Downloaded 38GiB to GPU in 2.26925s (16.7456GiB/s) | 20260505 14:08:14.192 | INFO | Analyzing chunk frames = [3805696,3806208) | 20260505 14:08:14.192 | INFO | MemInfo: { | 20260505 14:08:14.192 | INFO | "gpu" : " 17 GB", | 20260505 14:08:14.192 | INFO | "huge" : "", | 20260505 14:08:14.192 | INFO | "huge_shared" : " 27 GB", | 20260505 14:08:14.192 | INFO | "malloc" : " 14 GB", | 20260505 14:08:14.192 | INFO | "pinned" : " 29 GB" | 20260505 14:08:14.192 | INFO | } | 20260505 14:08:14.192 | INFO | Perf report for the last 5.14774 seconds: | 20260505 14:08:14.192 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:08:14.192 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:14.192 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:14.192 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:14.192 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 14:08:14.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:14.192 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:14.192 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:08:14.192 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:14.192 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:14.193 | INFO | Uploaded 36GiB to GPU in 2.19548s (16.3973GiB/s) | 20260505 14:08:14.193 | INFO | Downloaded 34GiB to GPU in 2.03273s (16.7263GiB/s) | 20260505 14:08:19.333 | INFO | Analyzing chunk frames = [3806208,3806720) | 20260505 14:08:19.333 | INFO | MemInfo: { | 20260505 14:08:19.333 | INFO | "gpu" : " 17 GB", | 20260505 14:08:19.333 | INFO | "huge" : "", | 20260505 14:08:19.333 | INFO | "huge_shared" : " 27 GB", | 20260505 14:08:19.333 | INFO | "malloc" : " 14 GB", | 20260505 14:08:19.333 | INFO | "pinned" : " 29 GB" | 20260505 14:08:19.333 | INFO | } | 20260505 14:08:19.333 | INFO | Perf report for the last 5.14088 seconds: | 20260505 14:08:19.333 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:08:19.333 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:19.333 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:19.333 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:19.333 | INFO | ANALYSIS : 63.0%, 3.0, 98.1ms, 0 | 20260505 14:08:19.333 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:19.333 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:19.333 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:08:19.333 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:19.333 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:19.333 | INFO | Uploaded 36GiB to GPU in 2.17227s (16.5725GiB/s) | 20260505 14:08:19.333 | INFO | Downloaded 34GiB to GPU in 2.05719s (16.5274GiB/s) | 20260505 14:08:24.467 | INFO | Analyzing chunk frames = [3806720,3807232) | 20260505 14:08:24.467 | INFO | MemInfo: { | 20260505 14:08:24.467 | INFO | "gpu" : " 17 GB", | 20260505 14:08:24.467 | INFO | "huge" : "", | 20260505 14:08:24.467 | INFO | "huge_shared" : " 27 GB", | 20260505 14:08:24.467 | INFO | "malloc" : " 14 GB", | 20260505 14:08:24.467 | INFO | "pinned" : " 29 GB" | 20260505 14:08:24.467 | INFO | } | 20260505 14:08:24.467 | INFO | Perf report for the last 5.13374 seconds: | 20260505 14:08:24.467 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:08:24.467 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:24.467 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:24.467 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:24.467 | INFO | ANALYSIS : 62.7%, 3.0, 97.3ms, 0 | 20260505 14:08:24.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:24.467 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:24.467 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 14:08:24.467 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:24.467 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:24.467 | INFO | Uploaded 36GiB to GPU in 2.18765s (16.456GiB/s) | 20260505 14:08:24.467 | INFO | Downloaded 34GiB to GPU in 2.12801s (15.9774GiB/s) | 20260505 14:08:29.599 | INFO | Analyzing chunk frames = [3807232,3807744) | 20260505 14:08:29.599 | INFO | MemInfo: { | 20260505 14:08:29.599 | INFO | "gpu" : " 17 GB", | 20260505 14:08:29.599 | INFO | "huge" : "", | 20260505 14:08:29.599 | INFO | "huge_shared" : " 27 GB", | 20260505 14:08:29.599 | INFO | "malloc" : " 14 GB", | 20260505 14:08:29.599 | INFO | "pinned" : " 29 GB" | 20260505 14:08:29.599 | INFO | } | 20260505 14:08:29.599 | INFO | Perf report for the last 5.13183 seconds: | 20260505 14:08:29.599 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:08:29.599 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:29.599 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:29.599 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:29.599 | INFO | ANALYSIS : 62.9%, 3.0, 98.0ms, 0 | 20260505 14:08:29.599 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:29.599 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:29.599 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:08:29.599 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:29.599 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:29.599 | INFO | Uploaded 36GiB to GPU in 2.16544s (16.6248GiB/s) | 20260505 14:08:29.599 | INFO | Downloaded 34GiB to GPU in 2.12473s (16.0021GiB/s) | 20260505 14:08:34.747 | INFO | Analyzing chunk frames = [3807744,3808256) | 20260505 14:08:34.747 | INFO | MemInfo: { | 20260505 14:08:34.747 | INFO | "gpu" : " 17 GB", | 20260505 14:08:34.747 | INFO | "huge" : "", | 20260505 14:08:34.747 | INFO | "huge_shared" : " 27 GB", | 20260505 14:08:34.748 | INFO | "malloc" : " 14 GB", | 20260505 14:08:34.748 | INFO | "pinned" : " 29 GB" | 20260505 14:08:34.748 | INFO | } | 20260505 14:08:34.748 | INFO | Perf report for the last 5.14879 seconds: | 20260505 14:08:34.748 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:08:34.748 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:34.748 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:34.748 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:34.748 | INFO | ANALYSIS : 63.0%, 3.0, 98.6ms, 0 | 20260505 14:08:34.748 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:34.748 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:34.748 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 14:08:34.748 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:34.748 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:34.748 | INFO | Uploaded 36GiB to GPU in 2.17033s (16.5873GiB/s) | 20260505 14:08:34.748 | INFO | Downloaded 34GiB to GPU in 2.10488s (16.1529GiB/s) | 20260505 14:08:39.885 | INFO | Analyzing chunk frames = [3808256,3808768) | 20260505 14:08:39.885 | INFO | MemInfo: { | 20260505 14:08:39.885 | INFO | "gpu" : " 17 GB", | 20260505 14:08:39.885 | INFO | "huge" : "", | 20260505 14:08:39.885 | INFO | "huge_shared" : " 27 GB", | 20260505 14:08:39.885 | INFO | "malloc" : " 14 GB", | 20260505 14:08:39.885 | INFO | "pinned" : " 29 GB" | 20260505 14:08:39.885 | INFO | } | 20260505 14:08:39.885 | INFO | Perf report for the last 5.13788 seconds: | 20260505 14:08:39.885 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:08:39.886 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:39.886 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:39.886 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:39.886 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 14:08:39.886 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:39.886 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:39.886 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 14:08:39.886 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:39.886 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:39.886 | INFO | Uploaded 36GiB to GPU in 2.12397s (16.9494GiB/s) | 20260505 14:08:39.886 | INFO | Downloaded 34GiB to GPU in 2.16827s (15.6807GiB/s) | 20260505 14:08:45.020 | INFO | Analyzing chunk frames = [3808768,3809280) | 20260505 14:08:45.020 | INFO | MemInfo: { | 20260505 14:08:45.020 | INFO | "gpu" : " 17 GB", | 20260505 14:08:45.020 | INFO | "huge" : "", | 20260505 14:08:45.020 | INFO | "huge_shared" : " 27 GB", | 20260505 14:08:45.020 | INFO | "malloc" : " 14 GB", | 20260505 14:08:45.020 | INFO | "pinned" : " 29 GB" | 20260505 14:08:45.020 | INFO | } | 20260505 14:08:45.020 | INFO | Perf report for the last 5.13486 seconds: | 20260505 14:08:45.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:08:45.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:45.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:45.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:45.020 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 14:08:45.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:45.020 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:45.020 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 14:08:45.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:45.021 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:45.021 | INFO | Uploaded 36GiB to GPU in 2.22643s (16.1694GiB/s) | 20260505 14:08:45.021 | INFO | Downloaded 34GiB to GPU in 2.10485s (16.1532GiB/s) | 20260505 14:08:45.021 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 14:08:45.021 | INFO | Total mem usage: 15.3 GiB | 20260505 14:08:45.021 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 14:08:45.021 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -7.43%) | 20260505 14:08:45.021 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 14:08:45.021 | INFO | High Water Breakdown: | 20260505 14:08:45.021 | INFO | 61.1% : 8.72 GiB ( 91.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 14:08:45.021 | INFO | 38.7% : 5.53 GiB ( 83.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 14:08:45.021 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 14:08:45.021 | INFO | ------------------------------------------ | 20260505 14:08:45.021 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:08:45.021 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 14:08:45.021 | INFO | Total mem usage: 26.5 GiB | 20260505 14:08:45.021 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 14:08:45.021 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 14:08:45.021 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 14:08:45.021 | INFO | High Water Breakdown: | 20260505 14:08:45.021 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:08:45.021 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:08:45.021 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:08:45.021 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:08:45.021 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:08:45.021 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:08:45.021 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 14:08:45.021 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 14:08:45.021 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:08:45.021 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 14:08:45.021 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 14:08:45.021 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 14:08:45.021 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 14:08:45.021 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 14:08:45.021 | INFO | ------------------------------------------ | 20260505 14:08:45.021 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:08:45.021 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 14:08:45.021 | INFO | Total mem usage: 25.6 GiB | 20260505 14:08:45.021 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 14:08:45.021 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 14:08:45.021 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 14:08:45.021 | INFO | High Water Breakdown: | 20260505 14:08:45.021 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 14:08:45.021 | INFO | ------------------------------------------ | 20260505 14:08:45.021 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:08:45.021 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 14:08:45.021 | INFO | Total mem usage: 15.7 GiB | 20260505 14:08:45.021 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 14:08:45.021 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 14:08:45.021 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 14:08:45.021 | INFO | High Water Breakdown: | 20260505 14:08:45.021 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 14:08:45.021 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 14:08:45.021 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 14:08:45.021 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 14:08:45.021 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 14:08:45.021 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 14:08:45.021 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 14:08:45.021 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 14:08:45.021 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 14:08:45.021 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:08:45.021 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 14:08:45.021 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:08:45.021 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 14:08:45.021 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 14:08:45.021 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 14:08:45.021 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 14:08:45.021 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:08:45.021 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 14:08:45.021 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:08:45.021 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:08:45.021 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:08:45.021 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 14:08:45.021 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 14:08:45.021 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:08:45.021 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 14:08:45.021 | INFO | ------------------------------------------ | 20260505 14:08:45.022 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:08:45.022 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 14:08:45.022 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:08:45.022 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:08:45.022 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.64e+01, 1.37e+01 / 1.83e+00 | 20260505 14:08:45.022 | INFO | PerfReport: 0.29%, AggregateStats -- 9.27e-02 / 9.99e-02, 1.07e-02 / 3.65e-03 | 20260505 14:08:45.022 | INFO | PerfReport: 11.77%, Baseline -- 3.75e+00 / 4.51e+00, 5.83e-01 / 1.38e-01 | 20260505 14:08:45.022 | INFO | PerfReport: 3.29%, Histogram -- 1.05e+00 / 1.50e+00, 1.97e-01 / 8.80e-02 | 20260505 14:08:45.022 | INFO | PerfReport: 15.99%, DME -- 4.07e+01 / 4.57e+01, 4.67e+00 / 1.84e+00 | 20260505 14:08:45.022 | INFO | PerfReport: 56.43%, FrameLabeling -- 1.80e+01 / 1.97e+01, 1.42e+00 / 1.73e+00 | 20260505 14:08:45.022 | INFO | PerfReport: 7.44%, PulseAccumulating -- 2.37e+00 / 2.91e+00, 4.47e-01 / 1.83e-01 | 20260505 14:08:45.022 | INFO | PerfReport: 3.55%, Metrics -- 1.13e+00 / 1.46e+00, 3.17e-01 / 1.54e-01 | 20260505 14:08:45.022 | INFO | PerfReport: 1.24%, PulseSerializing -- 3.95e-01 / 4.48e-01, 7.07e-02 / 3.13e-02 | 20260505 14:08:45.022 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 14:08:45.022 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:08:45.022 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:08:45.022 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260505 14:08:45.022 | INFO | PerfReport: 22.60%, Upload -- 2.28e+01 / 0.00e+00, 8.06e+00 / 0.00e+00 | 20260505 14:08:45.022 | INFO | PerfReport: 21.97%, Download -- 2.22e+01 / 0.00e+00, 8.08e+00 / 0.00e+00 | 20260505 14:08:45.022 | INFO | PerfReport: 31.58%, Compute -- 3.18e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 14:08:45.022 | INFO | PerfReport: 2.31%, LockUpload -- 2.33e+00 / 0.00e+00, 8.06e+00 / 0.00e+00 | 20260505 14:08:45.022 | INFO | PerfReport: 1.40%, LockDownload -- 1.41e+00 / 0.00e+00, 2.97e+00 / 0.00e+00 | 20260505 14:08:45.022 | INFO | PerfReport: 20.12%, LockCompute -- 2.03e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260505 14:08:45.022 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 2.25e-02 / 0.00e+00 | 20260505 14:08:45.160 | INFO | Starting a round of Baz IO | 20260505 14:08:48.383 | INFO | Populating RTMetrics report | 20260505 14:08:48.383 | INFO | RTMetrics report populated | 20260505 14:08:48.383 | INFO | Opening tmp RTMetrics json file | 20260505 14:08:48.405 | INFO | Serializing RTMetrics to json | 20260505 14:08:48.443 | INFO | Serialization done, swapping tmp file now... | 20260505 14:08:48.471 | INFO | RTMetrics done | 20260505 14:08:48.488 | INFO | Finished a round of Baz IO: Wrote 2.272 GiB in 3.327 seconds | 20260505 14:08:48.488 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.86% / 0.00% / 77.14% | 20260505 14:08:48.488 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.601002% / 0% / 7.15256e-05% | 20260505 14:08:48.488 | INFO | Baz IO overall: | 20260505 14:08:48.488 | INFO | BytesWritten : 2269669771465 | 20260505 14:08:48.488 | INFO | NumEvents : 1079337119223 | 20260505 14:08:50.156 | INFO | Analyzing chunk frames = [3809280,3809792) | 20260505 14:08:50.156 | INFO | MemInfo: { | 20260505 14:08:50.156 | INFO | "gpu" : " 17 GB", | 20260505 14:08:50.156 | INFO | "huge" : "", | 20260505 14:08:50.156 | INFO | "huge_shared" : " 27 GB", | 20260505 14:08:50.156 | INFO | "malloc" : " 14 GB", | 20260505 14:08:50.156 | INFO | "pinned" : " 29 GB" | 20260505 14:08:50.156 | INFO | } | 20260505 14:08:50.156 | INFO | Perf report for the last 5.13537 seconds: | 20260505 14:08:50.156 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:08:50.156 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:50.156 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:50.156 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:50.156 | INFO | ANALYSIS : 64.4%, 3.0, 99.9ms, 0 | 20260505 14:08:50.156 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:50.156 | INFO | RT_METRICS : 46.7%, 1.0, 24.5ms, 0 | 20260505 14:08:50.156 | INFO | PRE_HQ : 60.8%, 1.8, 57.4ms, 0 | 20260505 14:08:50.156 | INFO | BAZWRITER : 64.1%, 4.0, 130.5ms, 0 | 20260505 14:08:50.156 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260505 14:08:50.156 | INFO | Uploaded 36GiB to GPU in 2.17899s (16.5214GiB/s) | 20260505 14:08:50.156 | INFO | Downloaded 38GiB to GPU in 2.27015s (16.739GiB/s) | 20260505 14:08:55.299 | INFO | Analyzing chunk frames = [3809792,3810304) | 20260505 14:08:55.299 | INFO | MemInfo: { | 20260505 14:08:55.299 | INFO | "gpu" : " 17 GB", | 20260505 14:08:55.299 | INFO | "huge" : "", | 20260505 14:08:55.299 | INFO | "huge_shared" : " 27 GB", | 20260505 14:08:55.299 | INFO | "malloc" : " 14 GB", | 20260505 14:08:55.299 | INFO | "pinned" : " 29 GB" | 20260505 14:08:55.299 | INFO | } | 20260505 14:08:55.299 | INFO | Perf report for the last 5.14333 seconds: | 20260505 14:08:55.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:08:55.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:55.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:55.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:55.299 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 14:08:55.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:55.299 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:55.299 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 14:08:55.299 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:55.299 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:08:55.299 | INFO | Uploaded 36GiB to GPU in 2.17995s (16.5141GiB/s) | 20260505 14:08:55.299 | INFO | Downloaded 34GiB to GPU in 2.05122s (16.5755GiB/s) | 20260505 14:09:00.448 | INFO | Analyzing chunk frames = [3810304,3810816) | 20260505 14:09:00.448 | INFO | MemInfo: { | 20260505 14:09:00.448 | INFO | "gpu" : " 17 GB", | 20260505 14:09:00.448 | INFO | "huge" : "", | 20260505 14:09:00.448 | INFO | "huge_shared" : " 27 GB", | 20260505 14:09:00.448 | INFO | "malloc" : " 14 GB", | 20260505 14:09:00.448 | INFO | "pinned" : " 29 GB" | 20260505 14:09:00.448 | INFO | } | 20260505 14:09:00.448 | INFO | Perf report for the last 5.14856 seconds: | 20260505 14:09:00.448 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:09:00.448 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:00.448 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:00.448 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:00.448 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260505 14:09:00.448 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:00.448 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:00.448 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:09:00.448 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:00.448 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:00.448 | INFO | Uploaded 36GiB to GPU in 2.16753s (16.6088GiB/s) | 20260505 14:09:00.448 | INFO | Downloaded 34GiB to GPU in 2.10488s (16.1529GiB/s) | 20260505 14:09:05.588 | INFO | Analyzing chunk frames = [3810816,3811328) | 20260505 14:09:05.588 | INFO | MemInfo: { | 20260505 14:09:05.588 | INFO | "gpu" : " 17 GB", | 20260505 14:09:05.588 | INFO | "huge" : "", | 20260505 14:09:05.588 | INFO | "huge_shared" : " 27 GB", | 20260505 14:09:05.588 | INFO | "malloc" : " 14 GB", | 20260505 14:09:05.588 | INFO | "pinned" : " 29 GB" | 20260505 14:09:05.588 | INFO | } | 20260505 14:09:05.588 | INFO | Perf report for the last 5.14078 seconds: | 20260505 14:09:05.588 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:09:05.588 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:05.588 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:05.588 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:05.588 | INFO | ANALYSIS : 62.6%, 3.0, 97.3ms, 0 | 20260505 14:09:05.588 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:05.589 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:05.589 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:09:05.589 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:05.589 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:05.589 | INFO | Uploaded 36GiB to GPU in 2.1865s (16.4646GiB/s) | 20260505 14:09:05.589 | INFO | Downloaded 34GiB to GPU in 2.13191s (15.9482GiB/s) | 20260505 14:09:10.719 | INFO | Analyzing chunk frames = [3811328,3811840) | 20260505 14:09:10.719 | INFO | MemInfo: { | 20260505 14:09:10.719 | INFO | "gpu" : " 17 GB", | 20260505 14:09:10.719 | INFO | "huge" : "", | 20260505 14:09:10.719 | INFO | "huge_shared" : " 27 GB", | 20260505 14:09:10.719 | INFO | "malloc" : " 14 GB", | 20260505 14:09:10.719 | INFO | "pinned" : " 29 GB" | 20260505 14:09:10.719 | INFO | } | 20260505 14:09:10.719 | INFO | Perf report for the last 5.131 seconds: | 20260505 14:09:10.719 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:09:10.719 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:10.719 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:10.719 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:10.719 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260505 14:09:10.719 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:10.720 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:10.720 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 14:09:10.720 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:10.720 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:10.720 | INFO | Uploaded 36GiB to GPU in 2.16813s (16.6042GiB/s) | 20260505 14:09:10.720 | INFO | Downloaded 34GiB to GPU in 2.12665s (15.9876GiB/s) | 20260505 14:09:15.849 | INFO | Analyzing chunk frames = [3811840,3812352) | 20260505 14:09:15.849 | INFO | MemInfo: { | 20260505 14:09:15.849 | INFO | "gpu" : " 17 GB", | 20260505 14:09:15.849 | INFO | "huge" : "", | 20260505 14:09:15.849 | INFO | "huge_shared" : " 27 GB", | 20260505 14:09:15.849 | INFO | "malloc" : " 14 GB", | 20260505 14:09:15.849 | INFO | "pinned" : " 29 GB" | 20260505 14:09:15.849 | INFO | } | 20260505 14:09:15.849 | INFO | Perf report for the last 5.12977 seconds: | 20260505 14:09:15.849 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:09:15.849 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:15.849 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:15.849 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:15.849 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 14:09:15.849 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:15.849 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:15.849 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:09:15.849 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:15.849 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:15.849 | INFO | Uploaded 36GiB to GPU in 2.18987s (16.4393GiB/s) | 20260505 14:09:15.850 | INFO | Downloaded 34GiB to GPU in 2.11198s (16.0986GiB/s) | 20260505 14:09:21.001 | INFO | Analyzing chunk frames = [3812352,3812864) | 20260505 14:09:21.001 | INFO | MemInfo: { | 20260505 14:09:21.001 | INFO | "gpu" : " 17 GB", | 20260505 14:09:21.001 | INFO | "huge" : "", | 20260505 14:09:21.001 | INFO | "huge_shared" : " 27 GB", | 20260505 14:09:21.001 | INFO | "malloc" : " 14 GB", | 20260505 14:09:21.001 | INFO | "pinned" : " 29 GB" | 20260505 14:09:21.001 | INFO | } | 20260505 14:09:21.001 | INFO | Perf report for the last 5.15202 seconds: | 20260505 14:09:21.001 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:09:21.001 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:21.001 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:21.001 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:21.001 | INFO | ANALYSIS : 63.0%, 3.0, 98.7ms, 0 | 20260505 14:09:21.001 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:21.001 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:21.001 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 14:09:21.001 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:21.001 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:21.002 | INFO | Uploaded 36GiB to GPU in 2.17242s (16.5714GiB/s) | 20260505 14:09:21.002 | INFO | Downloaded 34GiB to GPU in 2.15003s (15.8137GiB/s) | 20260505 14:09:26.148 | INFO | Analyzing chunk frames = [3812864,3813376) | 20260505 14:09:26.148 | INFO | MemInfo: { | 20260505 14:09:26.148 | INFO | "gpu" : " 17 GB", | 20260505 14:09:26.148 | INFO | "huge" : "", | 20260505 14:09:26.148 | INFO | "huge_shared" : " 27 GB", | 20260505 14:09:26.148 | INFO | "malloc" : " 14 GB", | 20260505 14:09:26.148 | INFO | "pinned" : " 29 GB" | 20260505 14:09:26.148 | INFO | } | 20260505 14:09:26.148 | INFO | Perf report for the last 5.14685 seconds: | 20260505 14:09:26.148 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:09:26.148 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:26.148 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:26.148 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:26.148 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 14:09:26.148 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:26.148 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:26.148 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:09:26.148 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:26.148 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:26.148 | INFO | Uploaded 36GiB to GPU in 2.21054s (16.2856GiB/s) | 20260505 14:09:26.148 | INFO | Downloaded 34GiB to GPU in 2.10056s (16.1862GiB/s) | 20260505 14:09:26.282 | INFO | Starting a round of Baz IO | 20260505 14:09:29.503 | INFO | Populating RTMetrics report | 20260505 14:09:29.503 | INFO | RTMetrics report populated | 20260505 14:09:29.503 | INFO | Opening tmp RTMetrics json file | 20260505 14:09:29.521 | INFO | Serializing RTMetrics to json | 20260505 14:09:29.537 | INFO | Serialization done, swapping tmp file now... | 20260505 14:09:29.547 | INFO | RTMetrics done | 20260505 14:09:29.603 | INFO | Finished a round of Baz IO: Wrote 2.27 GiB in 3.321 seconds | 20260505 14:09:29.603 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.85% / 0.00% / 77.14% | 20260505 14:09:29.604 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.599213% / 3.97364e-06% / 4.76837e-05% | 20260505 14:09:29.604 | INFO | Baz IO overall: | 20260505 14:09:29.604 | INFO | BytesWritten : 2272106645461 | 20260505 14:09:29.604 | INFO | NumEvents : 1080487349373 | 20260505 14:09:31.278 | INFO | Analyzing chunk frames = [3813376,3813888) | 20260505 14:09:31.278 | INFO | MemInfo: { | 20260505 14:09:31.278 | INFO | "gpu" : " 17 GB", | 20260505 14:09:31.279 | INFO | "huge" : "", | 20260505 14:09:31.279 | INFO | "huge_shared" : " 27 GB", | 20260505 14:09:31.279 | INFO | "malloc" : " 14 GB", | 20260505 14:09:31.279 | INFO | "pinned" : " 29 GB" | 20260505 14:09:31.279 | INFO | } | 20260505 14:09:31.279 | INFO | Perf report for the last 5.13047 seconds: | 20260505 14:09:31.279 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:09:31.279 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:31.279 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:31.279 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:31.279 | INFO | ANALYSIS : 64.3%, 3.0, 99.6ms, 0 | 20260505 14:09:31.279 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:31.279 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 14:09:31.279 | INFO | PRE_HQ : 61.3%, 1.8, 57.4ms, 0 | 20260505 14:09:31.279 | INFO | BAZWRITER : 64.0%, 4.0, 131.4ms, 0 | 20260505 14:09:31.279 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 14:09:31.279 | INFO | Uploaded 36GiB to GPU in 2.20914s (16.2959GiB/s) | 20260505 14:09:31.279 | INFO | Downloaded 38GiB to GPU in 2.26237s (16.7966GiB/s) | 20260505 14:09:36.413 | INFO | Analyzing chunk frames = [3813888,3814400) | 20260505 14:09:36.413 | INFO | MemInfo: { | 20260505 14:09:36.413 | INFO | "gpu" : " 17 GB", | 20260505 14:09:36.413 | INFO | "huge" : "", | 20260505 14:09:36.413 | INFO | "huge_shared" : " 27 GB", | 20260505 14:09:36.413 | INFO | "malloc" : " 14 GB", | 20260505 14:09:36.413 | INFO | "pinned" : " 29 GB" | 20260505 14:09:36.413 | INFO | } | 20260505 14:09:36.413 | INFO | Perf report for the last 5.13461 seconds: | 20260505 14:09:36.413 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:09:36.413 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:36.413 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:36.413 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:36.413 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 14:09:36.413 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:36.413 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:36.413 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:09:36.413 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:36.413 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:36.413 | INFO | Uploaded 36GiB to GPU in 2.20695s (16.3121GiB/s) | 20260505 14:09:36.414 | INFO | Downloaded 34GiB to GPU in 2.04308s (16.6416GiB/s) | 20260505 14:09:41.556 | INFO | Analyzing chunk frames = [3814400,3814912) | 20260505 14:09:41.556 | INFO | MemInfo: { | 20260505 14:09:41.556 | INFO | "gpu" : " 17 GB", | 20260505 14:09:41.556 | INFO | "huge" : "", | 20260505 14:09:41.556 | INFO | "huge_shared" : " 27 GB", | 20260505 14:09:41.556 | INFO | "malloc" : " 14 GB", | 20260505 14:09:41.556 | INFO | "pinned" : " 29 GB" | 20260505 14:09:41.556 | INFO | } | 20260505 14:09:41.556 | INFO | Perf report for the last 5.14258 seconds: | 20260505 14:09:41.556 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:09:41.556 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:41.556 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:41.556 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:41.556 | INFO | ANALYSIS : 63.2%, 3.0, 98.4ms, 0 | 20260505 14:09:41.556 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:41.556 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:41.556 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:09:41.556 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:41.556 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:41.556 | INFO | Uploaded 36GiB to GPU in 2.20916s (16.2958GiB/s) | 20260505 14:09:41.556 | INFO | Downloaded 34GiB to GPU in 2.13025s (15.9606GiB/s) | 20260505 14:09:46.700 | INFO | Analyzing chunk frames = [3814912,3815424) | 20260505 14:09:46.700 | INFO | MemInfo: { | 20260505 14:09:46.700 | INFO | "gpu" : " 17 GB", | 20260505 14:09:46.701 | INFO | "huge" : "", | 20260505 14:09:46.701 | INFO | "huge_shared" : " 27 GB", | 20260505 14:09:46.701 | INFO | "malloc" : " 14 GB", | 20260505 14:09:46.701 | INFO | "pinned" : " 29 GB" | 20260505 14:09:46.701 | INFO | } | 20260505 14:09:46.701 | INFO | Perf report for the last 5.14482 seconds: | 20260505 14:09:46.701 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:09:46.701 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:46.701 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:46.701 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:46.701 | INFO | ANALYSIS : 62.7%, 3.0, 97.5ms, 0 | 20260505 14:09:46.701 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:46.701 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:46.701 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:09:46.701 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:46.701 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:46.701 | INFO | Uploaded 36GiB to GPU in 2.18943s (16.4427GiB/s) | 20260505 14:09:46.701 | INFO | Downloaded 34GiB to GPU in 2.12977s (15.9642GiB/s) | 20260505 14:09:51.836 | INFO | Analyzing chunk frames = [3815424,3815936) | 20260505 14:09:51.836 | INFO | MemInfo: { | 20260505 14:09:51.836 | INFO | "gpu" : " 17 GB", | 20260505 14:09:51.836 | INFO | "huge" : "", | 20260505 14:09:51.836 | INFO | "huge_shared" : " 27 GB", | 20260505 14:09:51.836 | INFO | "malloc" : " 14 GB", | 20260505 14:09:51.836 | INFO | "pinned" : " 29 GB" | 20260505 14:09:51.836 | INFO | } | 20260505 14:09:51.836 | INFO | Perf report for the last 5.13527 seconds: | 20260505 14:09:51.836 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:09:51.836 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:51.836 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:51.836 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:51.836 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260505 14:09:51.836 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:51.836 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:51.836 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:09:51.836 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:51.836 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:51.837 | INFO | Uploaded 36GiB to GPU in 2.17433s (16.5568GiB/s) | 20260505 14:09:51.837 | INFO | Downloaded 34GiB to GPU in 2.12839s (15.9745GiB/s) | 20260505 14:09:56.973 | INFO | Analyzing chunk frames = [3815936,3816448) | 20260505 14:09:56.973 | INFO | MemInfo: { | 20260505 14:09:56.973 | INFO | "gpu" : " 17 GB", | 20260505 14:09:56.973 | INFO | "huge" : "", | 20260505 14:09:56.973 | INFO | "huge_shared" : " 27 GB", | 20260505 14:09:56.973 | INFO | "malloc" : " 14 GB", | 20260505 14:09:56.973 | INFO | "pinned" : " 29 GB" | 20260505 14:09:56.973 | INFO | } | 20260505 14:09:56.973 | INFO | Perf report for the last 5.13699 seconds: | 20260505 14:09:56.973 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:09:56.973 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:56.973 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:56.973 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:56.973 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 14:09:56.973 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:56.973 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:56.973 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:09:56.973 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:56.973 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:09:56.973 | INFO | Uploaded 36GiB to GPU in 2.17711s (16.5357GiB/s) | 20260505 14:09:56.973 | INFO | Downloaded 34GiB to GPU in 2.1104s (16.1107GiB/s) | 20260505 14:10:02.108 | INFO | Analyzing chunk frames = [3816448,3816960) | 20260505 14:10:02.108 | INFO | MemInfo: { | 20260505 14:10:02.108 | INFO | "gpu" : " 17 GB", | 20260505 14:10:02.108 | INFO | "huge" : "", | 20260505 14:10:02.108 | INFO | "huge_shared" : " 27 GB", | 20260505 14:10:02.108 | INFO | "malloc" : " 14 GB", | 20260505 14:10:02.108 | INFO | "pinned" : " 29 GB" | 20260505 14:10:02.108 | INFO | } | 20260505 14:10:02.108 | INFO | Perf report for the last 5.13518 seconds: | 20260505 14:10:02.108 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:10:02.108 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:02.108 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:02.108 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:02.108 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 14:10:02.108 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:02.108 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:02.108 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 14:10:02.108 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:02.108 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:02.108 | INFO | Uploaded 36GiB to GPU in 2.16664s (16.6156GiB/s) | 20260505 14:10:02.108 | INFO | Downloaded 34GiB to GPU in 2.14617s (15.8422GiB/s) | 20260505 14:10:07.256 | INFO | Analyzing chunk frames = [3816960,3817472) | 20260505 14:10:07.256 | INFO | MemInfo: { | 20260505 14:10:07.256 | INFO | "gpu" : " 17 GB", | 20260505 14:10:07.256 | INFO | "huge" : "", | 20260505 14:10:07.256 | INFO | "huge_shared" : " 27 GB", | 20260505 14:10:07.256 | INFO | "malloc" : " 14 GB", | 20260505 14:10:07.256 | INFO | "pinned" : " 29 GB" | 20260505 14:10:07.256 | INFO | } | 20260505 14:10:07.256 | INFO | Perf report for the last 5.14833 seconds: | 20260505 14:10:07.256 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:10:07.256 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:07.257 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:07.257 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:07.257 | INFO | ANALYSIS : 63.4%, 3.0, 99.2ms, 0 | 20260505 14:10:07.257 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:07.257 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:07.257 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:10:07.257 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:07.257 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:07.257 | INFO | Uploaded 36GiB to GPU in 2.14423s (16.7893GiB/s) | 20260505 14:10:07.257 | INFO | Downloaded 34GiB to GPU in 2.10192s (16.1757GiB/s) | 20260505 14:10:07.257 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 14:10:07.257 | INFO | Total mem usage: 15.3 GiB | 20260505 14:10:07.257 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 14:10:07.257 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.66%) | 20260505 14:10:07.257 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 14:10:07.257 | INFO | High Water Breakdown: | 20260505 14:10:07.257 | INFO | 61% : 8.68 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 14:10:07.257 | INFO | 38.9% : 5.54 GiB ( 83.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 14:10:07.257 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 14:10:07.257 | INFO | ------------------------------------------ | 20260505 14:10:07.257 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:10:07.257 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 14:10:07.257 | INFO | Total mem usage: 26.5 GiB | 20260505 14:10:07.257 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 14:10:07.257 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 14:10:07.257 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 14:10:07.257 | INFO | High Water Breakdown: | 20260505 14:10:07.257 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:10:07.257 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:10:07.257 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:10:07.257 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:10:07.257 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:10:07.257 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:10:07.257 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 14:10:07.257 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 14:10:07.257 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:10:07.257 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 14:10:07.257 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 14:10:07.257 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 14:10:07.257 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 14:10:07.257 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 14:10:07.257 | INFO | ------------------------------------------ | 20260505 14:10:07.257 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:10:07.257 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 14:10:07.257 | INFO | Total mem usage: 25.6 GiB | 20260505 14:10:07.257 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 14:10:07.257 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 14:10:07.257 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 14:10:07.257 | INFO | High Water Breakdown: | 20260505 14:10:07.257 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 14:10:07.257 | INFO | ------------------------------------------ | 20260505 14:10:07.257 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:10:07.257 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 14:10:07.257 | INFO | Total mem usage: 15.7 GiB | 20260505 14:10:07.257 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 14:10:07.257 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 14:10:07.257 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 14:10:07.257 | INFO | High Water Breakdown: | 20260505 14:10:07.257 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 14:10:07.257 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 14:10:07.257 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 14:10:07.257 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 14:10:07.257 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 14:10:07.257 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 14:10:07.257 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 14:10:07.257 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 14:10:07.257 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 14:10:07.257 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:10:07.257 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 14:10:07.257 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:10:07.257 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 14:10:07.257 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 14:10:07.257 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 14:10:07.257 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 14:10:07.257 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:10:07.257 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 14:10:07.257 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:10:07.257 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:10:07.257 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:10:07.257 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 14:10:07.257 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 14:10:07.257 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:10:07.257 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 14:10:07.257 | INFO | ------------------------------------------ | 20260505 14:10:07.257 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:10:07.258 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 14:10:07.258 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:10:07.258 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:10:07.258 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.65e+01, 1.38e+01 / 1.30e+00 | 20260505 14:10:07.258 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 1.01e-01, 1.14e-02 / 5.90e-03 | 20260505 14:10:07.258 | INFO | PerfReport: 11.78%, Baseline -- 3.75e+00 / 4.46e+00, 5.87e-01 / 2.32e-01 | 20260505 14:10:07.258 | INFO | PerfReport: 3.31%, Histogram -- 1.05e+00 / 1.50e+00, 2.02e-01 / 8.97e-02 | 20260505 14:10:07.258 | INFO | PerfReport: 15.95%, DME -- 4.06e+01 / 4.56e+01, 4.60e+00 / 2.23e+00 | 20260505 14:10:07.258 | INFO | PerfReport: 56.42%, FrameLabeling -- 1.80e+01 / 2.01e+01, 1.45e+00 / 2.07e+00 | 20260505 14:10:07.258 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.37e+00 / 2.89e+00, 4.53e-01 / 2.31e-01 | 20260505 14:10:07.258 | INFO | PerfReport: 3.55%, Metrics -- 1.13e+00 / 1.44e+00, 3.19e-01 / 1.64e-01 | 20260505 14:10:07.258 | INFO | PerfReport: 1.24%, PulseSerializing -- 3.96e-01 / 4.48e-01, 7.14e-02 / 3.46e-02 | 20260505 14:10:07.258 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 14:10:07.258 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:10:07.258 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:10:07.258 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260505 14:10:07.258 | INFO | PerfReport: 22.59%, Upload -- 2.27e+01 / 0.00e+00, 8.06e+00 / 0.00e+00 | 20260505 14:10:07.258 | INFO | PerfReport: 22.06%, Download -- 2.22e+01 / 0.00e+00, 8.09e+00 / 0.00e+00 | 20260505 14:10:07.258 | INFO | PerfReport: 31.65%, Compute -- 3.19e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 14:10:07.258 | INFO | PerfReport: 2.15%, LockUpload -- 2.17e+00 / 0.00e+00, 7.73e+00 / 0.00e+00 | 20260505 14:10:07.258 | INFO | PerfReport: 1.44%, LockDownload -- 1.45e+00 / 0.00e+00, 2.96e+00 / 0.00e+00 | 20260505 14:10:07.258 | INFO | PerfReport: 20.09%, LockCompute -- 2.02e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260505 14:10:07.258 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.92e-02 / 0.00e+00 | 20260505 14:10:07.388 | INFO | Starting a round of Baz IO | 20260505 14:10:10.610 | INFO | Populating RTMetrics report | 20260505 14:10:10.610 | INFO | RTMetrics report populated | 20260505 14:10:10.610 | INFO | Opening tmp RTMetrics json file | 20260505 14:10:10.628 | INFO | Serializing RTMetrics to json | 20260505 14:10:10.651 | INFO | Serialization done, swapping tmp file now... | 20260505 14:10:10.675 | INFO | RTMetrics done | 20260505 14:10:10.712 | INFO | Finished a round of Baz IO: Wrote 2.27 GiB in 3.323 seconds | 20260505 14:10:10.712 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.85% / 0.00% / 77.15% | 20260505 14:10:10.712 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.598828% / 3.97364e-06% / 7.15256e-05% | 20260505 14:10:10.712 | INFO | Baz IO overall: | 20260505 14:10:10.712 | INFO | BytesWritten : 2274544346050 | 20260505 14:10:10.712 | INFO | NumEvents : 1081638560679 | 20260505 14:10:12.399 | INFO | Analyzing chunk frames = [3817472,3817984) | 20260505 14:10:12.399 | INFO | MemInfo: { | 20260505 14:10:12.399 | INFO | "gpu" : " 17 GB", | 20260505 14:10:12.399 | INFO | "huge" : "", | 20260505 14:10:12.399 | INFO | "huge_shared" : " 27 GB", | 20260505 14:10:12.399 | INFO | "malloc" : " 14 GB", | 20260505 14:10:12.399 | INFO | "pinned" : " 29 GB" | 20260505 14:10:12.399 | INFO | } | 20260505 14:10:12.399 | INFO | Perf report for the last 5.14252 seconds: | 20260505 14:10:12.399 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:10:12.399 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:12.399 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:12.399 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:12.399 | INFO | ANALYSIS : 64.1%, 3.0, 99.6ms, 0 | 20260505 14:10:12.399 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:12.399 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260505 14:10:12.399 | INFO | PRE_HQ : 61.0%, 1.8, 57.2ms, 0 | 20260505 14:10:12.399 | INFO | BAZWRITER : 63.9%, 4.0, 130.6ms, 0 | 20260505 14:10:12.399 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 14:10:12.399 | INFO | Uploaded 36GiB to GPU in 2.19622s (16.3918GiB/s) | 20260505 14:10:12.399 | INFO | Downloaded 38GiB to GPU in 2.25995s (16.8145GiB/s) | 20260505 14:10:17.539 | INFO | Analyzing chunk frames = [3817984,3818496) | 20260505 14:10:17.539 | INFO | MemInfo: { | 20260505 14:10:17.539 | INFO | "gpu" : " 17 GB", | 20260505 14:10:17.539 | INFO | "huge" : "", | 20260505 14:10:17.539 | INFO | "huge_shared" : " 27 GB", | 20260505 14:10:17.539 | INFO | "malloc" : " 14 GB", | 20260505 14:10:17.539 | INFO | "pinned" : " 29 GB" | 20260505 14:10:17.539 | INFO | } | 20260505 14:10:17.539 | INFO | Perf report for the last 5.14034 seconds: | 20260505 14:10:17.539 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:10:17.539 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:17.539 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:17.539 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:17.539 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 14:10:17.539 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:17.539 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:17.539 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:10:17.539 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:17.539 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:17.540 | INFO | Uploaded 36GiB to GPU in 2.20678s (16.3133GiB/s) | 20260505 14:10:17.540 | INFO | Downloaded 34GiB to GPU in 2.06425s (16.4709GiB/s) | 20260505 14:10:22.666 | INFO | Analyzing chunk frames = [3818496,3819008) | 20260505 14:10:22.666 | INFO | MemInfo: { | 20260505 14:10:22.666 | INFO | "gpu" : " 17 GB", | 20260505 14:10:22.666 | INFO | "huge" : "", | 20260505 14:10:22.666 | INFO | "huge_shared" : " 27 GB", | 20260505 14:10:22.666 | INFO | "malloc" : " 14 GB", | 20260505 14:10:22.666 | INFO | "pinned" : " 29 GB" | 20260505 14:10:22.666 | INFO | } | 20260505 14:10:22.666 | INFO | Perf report for the last 5.12651 seconds: | 20260505 14:10:22.666 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:10:22.666 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:22.666 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:22.666 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:22.666 | INFO | ANALYSIS : 63.5%, 3.0, 98.6ms, 0 | 20260505 14:10:22.666 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:22.666 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:22.666 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:10:22.666 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:22.666 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:22.666 | INFO | Uploaded 36GiB to GPU in 2.23265s (16.1243GiB/s) | 20260505 14:10:22.666 | INFO | Downloaded 34GiB to GPU in 2.11253s (16.0945GiB/s) | 20260505 14:10:27.809 | INFO | Analyzing chunk frames = [3819008,3819520) | 20260505 14:10:27.809 | INFO | MemInfo: { | 20260505 14:10:27.809 | INFO | "gpu" : " 17 GB", | 20260505 14:10:27.809 | INFO | "huge" : "", | 20260505 14:10:27.809 | INFO | "huge_shared" : " 27 GB", | 20260505 14:10:27.809 | INFO | "malloc" : " 14 GB", | 20260505 14:10:27.809 | INFO | "pinned" : " 29 GB" | 20260505 14:10:27.809 | INFO | } | 20260505 14:10:27.809 | INFO | Perf report for the last 5.14331 seconds: | 20260505 14:10:27.809 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:10:27.809 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:27.809 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:27.809 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:27.809 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260505 14:10:27.809 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:27.809 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:27.809 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:10:27.809 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:27.809 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:27.809 | INFO | Uploaded 36GiB to GPU in 2.1994s (16.3681GiB/s) | 20260505 14:10:27.809 | INFO | Downloaded 34GiB to GPU in 2.13223s (15.9457GiB/s) | 20260505 14:10:32.951 | INFO | Analyzing chunk frames = [3819520,3820032) | 20260505 14:10:32.951 | INFO | MemInfo: { | 20260505 14:10:32.951 | INFO | "gpu" : " 17 GB", | 20260505 14:10:32.951 | INFO | "huge" : "", | 20260505 14:10:32.951 | INFO | "huge_shared" : " 27 GB", | 20260505 14:10:32.951 | INFO | "malloc" : " 14 GB", | 20260505 14:10:32.951 | INFO | "pinned" : " 29 GB" | 20260505 14:10:32.951 | INFO | } | 20260505 14:10:32.951 | INFO | Perf report for the last 5.14194 seconds: | 20260505 14:10:32.951 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:10:32.951 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:32.951 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:32.951 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:32.951 | INFO | ANALYSIS : 62.7%, 3.0, 98.0ms, 0 | 20260505 14:10:32.951 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:32.951 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:32.951 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:10:32.951 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:32.951 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:32.951 | INFO | Uploaded 36GiB to GPU in 2.14736s (16.7647GiB/s) | 20260505 14:10:32.951 | INFO | Downloaded 34GiB to GPU in 2.12209s (16.022GiB/s) | 20260505 14:10:38.103 | INFO | Analyzing chunk frames = [3820032,3820544) | 20260505 14:10:38.103 | INFO | MemInfo: { | 20260505 14:10:38.103 | INFO | "gpu" : " 17 GB", | 20260505 14:10:38.103 | INFO | "huge" : "", | 20260505 14:10:38.104 | INFO | "huge_shared" : " 27 GB", | 20260505 14:10:38.104 | INFO | "malloc" : " 14 GB", | 20260505 14:10:38.104 | INFO | "pinned" : " 29 GB" | 20260505 14:10:38.104 | INFO | } | 20260505 14:10:38.104 | INFO | Perf report for the last 5.15256 seconds: | 20260505 14:10:38.104 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:10:38.104 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:38.104 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:38.104 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:38.104 | INFO | ANALYSIS : 63.0%, 3.0, 98.6ms, 0 | 20260505 14:10:38.104 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:38.104 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:38.104 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:10:38.104 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:38.104 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:38.104 | INFO | Uploaded 36GiB to GPU in 2.20119s (16.3548GiB/s) | 20260505 14:10:38.104 | INFO | Downloaded 34GiB to GPU in 2.11348s (16.0872GiB/s) | 20260505 14:10:43.224 | INFO | Analyzing chunk frames = [3820544,3821056) | 20260505 14:10:43.224 | INFO | MemInfo: { | 20260505 14:10:43.224 | INFO | "gpu" : " 17 GB", | 20260505 14:10:43.224 | INFO | "huge" : "", | 20260505 14:10:43.224 | INFO | "huge_shared" : " 27 GB", | 20260505 14:10:43.224 | INFO | "malloc" : " 14 GB", | 20260505 14:10:43.224 | INFO | "pinned" : " 29 GB" | 20260505 14:10:43.224 | INFO | } | 20260505 14:10:43.224 | INFO | Perf report for the last 5.12063 seconds: | 20260505 14:10:43.224 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:10:43.224 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:43.224 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:43.224 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:43.224 | INFO | ANALYSIS : 63.8%, 3.0, 99.2ms, 0 | 20260505 14:10:43.224 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:43.224 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:43.225 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 14:10:43.225 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:43.225 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:43.225 | INFO | Uploaded 36GiB to GPU in 2.11919s (16.9877GiB/s) | 20260505 14:10:43.225 | INFO | Downloaded 34GiB to GPU in 2.18337s (15.5723GiB/s) | 20260505 14:10:48.361 | INFO | Analyzing chunk frames = [3821056,3821568) | 20260505 14:10:48.361 | INFO | MemInfo: { | 20260505 14:10:48.361 | INFO | "gpu" : " 17 GB", | 20260505 14:10:48.361 | INFO | "huge" : "", | 20260505 14:10:48.361 | INFO | "huge_shared" : " 27 GB", | 20260505 14:10:48.361 | INFO | "malloc" : " 14 GB", | 20260505 14:10:48.361 | INFO | "pinned" : " 29 GB" | 20260505 14:10:48.361 | INFO | } | 20260505 14:10:48.361 | INFO | Perf report for the last 5.13644 seconds: | 20260505 14:10:48.361 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:10:48.361 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:48.361 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:48.361 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:48.361 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 14:10:48.361 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:48.361 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:48.361 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 14:10:48.361 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:48.361 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:48.361 | INFO | Uploaded 36GiB to GPU in 2.1341s (16.8689GiB/s) | 20260505 14:10:48.361 | INFO | Downloaded 34GiB to GPU in 2.10328s (16.1652GiB/s) | 20260505 14:10:48.501 | INFO | Starting a round of Baz IO | 20260505 14:10:51.718 | INFO | Populating RTMetrics report | 20260505 14:10:51.718 | INFO | RTMetrics report populated | 20260505 14:10:51.718 | INFO | Opening tmp RTMetrics json file | 20260505 14:10:51.752 | INFO | Serializing RTMetrics to json | 20260505 14:10:51.783 | INFO | Serialization done, swapping tmp file now... | 20260505 14:10:51.805 | INFO | RTMetrics done | 20260505 14:10:51.821 | INFO | Finished a round of Baz IO: Wrote 2.269 GiB in 3.32 seconds | 20260505 14:10:51.822 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.85% / 0.00% / 77.15% | 20260505 14:10:51.822 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.596682% / 0% / 6.75519e-05% | 20260505 14:10:51.822 | INFO | Baz IO overall: | 20260505 14:10:51.822 | INFO | BytesWritten : 2276980380688 | 20260505 14:10:51.822 | INFO | NumEvents : 1082788760459 | 20260505 14:10:53.513 | INFO | Analyzing chunk frames = [3821568,3822080) | 20260505 14:10:53.513 | INFO | MemInfo: { | 20260505 14:10:53.513 | INFO | "gpu" : " 17 GB", | 20260505 14:10:53.513 | INFO | "huge" : "", | 20260505 14:10:53.513 | INFO | "huge_shared" : " 27 GB", | 20260505 14:10:53.513 | INFO | "malloc" : " 14 GB", | 20260505 14:10:53.513 | INFO | "pinned" : " 29 GB" | 20260505 14:10:53.513 | INFO | } | 20260505 14:10:53.513 | INFO | Perf report for the last 5.152 seconds: | 20260505 14:10:53.513 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:10:53.513 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:53.513 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:53.513 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:53.513 | INFO | ANALYSIS : 64.1%, 3.0, 99.8ms, 0 | 20260505 14:10:53.513 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:53.513 | INFO | RT_METRICS : 46.5%, 1.0, 24.4ms, 0 | 20260505 14:10:53.513 | INFO | PRE_HQ : 60.6%, 1.8, 57.2ms, 0 | 20260505 14:10:53.513 | INFO | BAZWRITER : 63.8%, 4.0, 130.7ms, 0 | 20260505 14:10:53.513 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 14:10:53.513 | INFO | Uploaded 36GiB to GPU in 2.16718s (16.6115GiB/s) | 20260505 14:10:53.513 | INFO | Downloaded 38GiB to GPU in 2.27609s (16.6953GiB/s) | 20260505 14:10:58.651 | INFO | Analyzing chunk frames = [3822080,3822592) | 20260505 14:10:58.651 | INFO | MemInfo: { | 20260505 14:10:58.651 | INFO | "gpu" : " 17 GB", | 20260505 14:10:58.651 | INFO | "huge" : "", | 20260505 14:10:58.651 | INFO | "huge_shared" : " 27 GB", | 20260505 14:10:58.651 | INFO | "malloc" : " 14 GB", | 20260505 14:10:58.651 | INFO | "pinned" : " 29 GB" | 20260505 14:10:58.651 | INFO | } | 20260505 14:10:58.651 | INFO | Perf report for the last 5.1381 seconds: | 20260505 14:10:58.651 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:10:58.651 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:58.651 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:58.651 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:58.651 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 14:10:58.651 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:58.651 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:58.651 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:10:58.651 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:58.651 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:10:58.651 | INFO | Uploaded 36GiB to GPU in 2.20471s (16.3286GiB/s) | 20260505 14:10:58.651 | INFO | Downloaded 34GiB to GPU in 2.05858s (16.5162GiB/s) | 20260505 14:11:03.784 | INFO | Analyzing chunk frames = [3822592,3823104) | 20260505 14:11:03.784 | INFO | MemInfo: { | 20260505 14:11:03.784 | INFO | "gpu" : " 17 GB", | 20260505 14:11:03.784 | INFO | "huge" : "", | 20260505 14:11:03.784 | INFO | "huge_shared" : " 27 GB", | 20260505 14:11:03.784 | INFO | "malloc" : " 14 GB", | 20260505 14:11:03.784 | INFO | "pinned" : " 29 GB" | 20260505 14:11:03.784 | INFO | } | 20260505 14:11:03.784 | INFO | Perf report for the last 5.13315 seconds: | 20260505 14:11:03.784 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:11:03.784 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:03.784 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:03.784 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:03.784 | INFO | ANALYSIS : 63.0%, 3.0, 98.0ms, 0 | 20260505 14:11:03.784 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:03.784 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:03.784 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 14:11:03.784 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:03.784 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:03.784 | INFO | Uploaded 36GiB to GPU in 2.17415s (16.5582GiB/s) | 20260505 14:11:03.784 | INFO | Downloaded 34GiB to GPU in 2.06147s (16.4931GiB/s) | 20260505 14:11:08.918 | INFO | Analyzing chunk frames = [3823104,3823616) | 20260505 14:11:08.918 | INFO | MemInfo: { | 20260505 14:11:08.918 | INFO | "gpu" : " 17 GB", | 20260505 14:11:08.918 | INFO | "huge" : "", | 20260505 14:11:08.918 | INFO | "huge_shared" : " 27 GB", | 20260505 14:11:08.918 | INFO | "malloc" : " 14 GB", | 20260505 14:11:08.918 | INFO | "pinned" : " 29 GB" | 20260505 14:11:08.918 | INFO | } | 20260505 14:11:08.918 | INFO | Perf report for the last 5.13406 seconds: | 20260505 14:11:08.918 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:11:08.918 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:08.918 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:08.918 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:08.918 | INFO | ANALYSIS : 63.0%, 3.0, 97.7ms, 0 | 20260505 14:11:08.918 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:08.918 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:08.918 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:11:08.918 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:08.918 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:08.919 | INFO | Uploaded 36GiB to GPU in 2.19744s (16.3827GiB/s) | 20260505 14:11:08.919 | INFO | Downloaded 34GiB to GPU in 2.13527s (15.923GiB/s) | 20260505 14:11:14.060 | INFO | Analyzing chunk frames = [3823616,3824128) | 20260505 14:11:14.060 | INFO | MemInfo: { | 20260505 14:11:14.060 | INFO | "gpu" : " 17 GB", | 20260505 14:11:14.060 | INFO | "huge" : "", | 20260505 14:11:14.060 | INFO | "huge_shared" : " 27 GB", | 20260505 14:11:14.060 | INFO | "malloc" : " 14 GB", | 20260505 14:11:14.060 | INFO | "pinned" : " 29 GB" | 20260505 14:11:14.060 | INFO | } | 20260505 14:11:14.060 | INFO | Perf report for the last 5.1421 seconds: | 20260505 14:11:14.060 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:11:14.060 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:14.060 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:14.060 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:14.060 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 14:11:14.060 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:14.060 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:14.060 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 14:11:14.060 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:14.060 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:14.061 | INFO | Uploaded 36GiB to GPU in 2.15136s (16.7336GiB/s) | 20260505 14:11:14.061 | INFO | Downloaded 34GiB to GPU in 2.12513s (15.999GiB/s) | 20260505 14:11:19.206 | INFO | Analyzing chunk frames = [3824128,3824640) | 20260505 14:11:19.206 | INFO | MemInfo: { | 20260505 14:11:19.206 | INFO | "gpu" : " 17 GB", | 20260505 14:11:19.206 | INFO | "huge" : "", | 20260505 14:11:19.206 | INFO | "huge_shared" : " 27 GB", | 20260505 14:11:19.206 | INFO | "malloc" : " 14 GB", | 20260505 14:11:19.206 | INFO | "pinned" : " 29 GB" | 20260505 14:11:19.206 | INFO | } | 20260505 14:11:19.206 | INFO | Perf report for the last 5.1459 seconds: | 20260505 14:11:19.206 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:11:19.206 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:19.206 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:19.206 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:19.206 | INFO | ANALYSIS : 62.9%, 3.0, 98.4ms, 0 | 20260505 14:11:19.206 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:19.206 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:19.206 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 14:11:19.206 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:19.206 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:19.206 | INFO | Uploaded 36GiB to GPU in 2.15767s (16.6846GiB/s) | 20260505 14:11:19.207 | INFO | Downloaded 34GiB to GPU in 2.09609s (16.2207GiB/s) | 20260505 14:11:24.351 | INFO | Analyzing chunk frames = [3824640,3825152) | 20260505 14:11:24.351 | INFO | MemInfo: { | 20260505 14:11:24.351 | INFO | "gpu" : " 17 GB", | 20260505 14:11:24.351 | INFO | "huge" : "", | 20260505 14:11:24.351 | INFO | "huge_shared" : " 27 GB", | 20260505 14:11:24.351 | INFO | "malloc" : " 14 GB", | 20260505 14:11:24.351 | INFO | "pinned" : " 29 GB" | 20260505 14:11:24.351 | INFO | } | 20260505 14:11:24.351 | INFO | Perf report for the last 5.14515 seconds: | 20260505 14:11:24.351 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:11:24.351 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:24.351 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:24.351 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:24.351 | INFO | ANALYSIS : 63.5%, 3.0, 99.3ms, 0 | 20260505 14:11:24.351 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:24.351 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:24.351 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 14:11:24.352 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:24.352 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:24.352 | INFO | Uploaded 36GiB to GPU in 2.11815s (16.996GiB/s) | 20260505 14:11:24.352 | INFO | Downloaded 34GiB to GPU in 2.1752s (15.6307GiB/s) | 20260505 14:11:29.481 | INFO | Analyzing chunk frames = [3825152,3825664) | 20260505 14:11:29.481 | INFO | MemInfo: { | 20260505 14:11:29.481 | INFO | "gpu" : " 17 GB", | 20260505 14:11:29.481 | INFO | "huge" : "", | 20260505 14:11:29.481 | INFO | "huge_shared" : " 27 GB", | 20260505 14:11:29.481 | INFO | "malloc" : " 14 GB", | 20260505 14:11:29.481 | INFO | "pinned" : " 29 GB" | 20260505 14:11:29.481 | INFO | } | 20260505 14:11:29.481 | INFO | Perf report for the last 5.12961 seconds: | 20260505 14:11:29.481 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:11:29.481 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:29.481 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:29.481 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:29.481 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 14:11:29.481 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:29.481 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:29.481 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 14:11:29.481 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:29.481 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:29.481 | INFO | Uploaded 36GiB to GPU in 2.2183s (16.2286GiB/s) | 20260505 14:11:29.481 | INFO | Downloaded 34GiB to GPU in 2.09407s (16.2363GiB/s) | 20260505 14:11:29.482 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 14:11:29.482 | INFO | Total mem usage: 15.3 GiB | 20260505 14:11:29.482 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 14:11:29.482 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.67%) | 20260505 14:11:29.482 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 14:11:29.482 | INFO | High Water Breakdown: | 20260505 14:11:29.482 | INFO | 61% : 8.68 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 14:11:29.482 | INFO | 38.9% : 5.54 GiB ( 83.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 14:11:29.482 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 14:11:29.482 | INFO | ------------------------------------------ | 20260505 14:11:29.482 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:11:29.482 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 14:11:29.482 | INFO | Total mem usage: 26.5 GiB | 20260505 14:11:29.482 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 14:11:29.482 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 14:11:29.482 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 14:11:29.482 | INFO | High Water Breakdown: | 20260505 14:11:29.482 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:11:29.482 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:11:29.482 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:11:29.482 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:11:29.482 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:11:29.482 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:11:29.482 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 14:11:29.482 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 14:11:29.482 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:11:29.482 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 14:11:29.482 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 14:11:29.482 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 14:11:29.482 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 14:11:29.482 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 14:11:29.482 | INFO | ------------------------------------------ | 20260505 14:11:29.482 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:11:29.482 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 14:11:29.482 | INFO | Total mem usage: 25.6 GiB | 20260505 14:11:29.482 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 14:11:29.482 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 14:11:29.482 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 14:11:29.482 | INFO | High Water Breakdown: | 20260505 14:11:29.482 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 14:11:29.482 | INFO | ------------------------------------------ | 20260505 14:11:29.482 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:11:29.482 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 14:11:29.482 | INFO | Total mem usage: 15.7 GiB | 20260505 14:11:29.482 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 14:11:29.482 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 14:11:29.482 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 14:11:29.483 | INFO | High Water Breakdown: | 20260505 14:11:29.483 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 14:11:29.483 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 14:11:29.483 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 14:11:29.483 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 14:11:29.483 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 14:11:29.483 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 14:11:29.483 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 14:11:29.483 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 14:11:29.483 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 14:11:29.483 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:11:29.483 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 14:11:29.483 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:11:29.483 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 14:11:29.483 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 14:11:29.483 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 14:11:29.483 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 14:11:29.483 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:11:29.483 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 14:11:29.483 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:11:29.483 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:11:29.483 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:11:29.483 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 14:11:29.483 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 14:11:29.483 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:11:29.483 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 14:11:29.483 | INFO | ------------------------------------------ | 20260505 14:11:29.483 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:11:29.483 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 14:11:29.483 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:11:29.483 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:11:29.483 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.66e+01, 1.37e+01 / 1.84e+00 | 20260505 14:11:29.483 | INFO | PerfReport: 0.29%, AggregateStats -- 9.27e-02 / 9.96e-02, 1.08e-02 / 3.06e-03 | 20260505 14:11:29.483 | INFO | PerfReport: 11.77%, Baseline -- 3.75e+00 / 4.49e+00, 5.78e-01 / 1.69e-01 | 20260505 14:11:29.483 | INFO | PerfReport: 3.29%, Histogram -- 1.05e+00 / 1.49e+00, 1.95e-01 / 1.16e-01 | 20260505 14:11:29.483 | INFO | PerfReport: 16.01%, DME -- 4.08e+01 / 4.59e+01, 4.72e+00 / 2.08e+00 | 20260505 14:11:29.483 | INFO | PerfReport: 56.43%, FrameLabeling -- 1.80e+01 / 1.98e+01, 1.37e+00 / 1.52e+00 | 20260505 14:11:29.483 | INFO | PerfReport: 7.44%, PulseAccumulating -- 2.37e+00 / 2.96e+00, 4.45e-01 / 1.43e-01 | 20260505 14:11:29.483 | INFO | PerfReport: 3.53%, Metrics -- 1.12e+00 / 1.43e+00, 3.10e-01 / 1.43e-01 | 20260505 14:11:29.483 | INFO | PerfReport: 1.24%, PulseSerializing -- 3.94e-01 / 4.46e-01, 6.96e-02 / 2.87e-02 | 20260505 14:11:29.483 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 14:11:29.483 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:11:29.483 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:11:29.483 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260505 14:11:29.483 | INFO | PerfReport: 22.49%, Upload -- 2.27e+01 / 0.00e+00, 7.93e+00 / 0.00e+00 | 20260505 14:11:29.483 | INFO | PerfReport: 22.04%, Download -- 2.22e+01 / 0.00e+00, 7.94e+00 / 0.00e+00 | 20260505 14:11:29.483 | INFO | PerfReport: 31.58%, Compute -- 3.18e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 14:11:29.483 | INFO | PerfReport: 2.23%, LockUpload -- 2.25e+00 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260505 14:11:29.483 | INFO | PerfReport: 1.50%, LockDownload -- 1.51e+00 / 0.00e+00, 3.09e+00 / 0.00e+00 | 20260505 14:11:29.483 | INFO | PerfReport: 20.14%, LockCompute -- 2.03e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260505 14:11:29.483 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 2.09e-02 / 0.00e+00 | 20260505 14:11:29.618 | INFO | Starting a round of Baz IO | 20260505 14:11:32.836 | INFO | Populating RTMetrics report | 20260505 14:11:32.836 | INFO | RTMetrics report populated | 20260505 14:11:32.836 | INFO | Opening tmp RTMetrics json file | 20260505 14:11:32.849 | INFO | Serializing RTMetrics to json | 20260505 14:11:32.850 | INFO | Serialization done, swapping tmp file now... | 20260505 14:11:32.851 | INFO | RTMetrics done | 20260505 14:11:32.937 | INFO | Finished a round of Baz IO: Wrote 2.266 GiB in 3.318 seconds | 20260505 14:11:32.937 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.84% / 0.00% / 77.15% | 20260505 14:11:32.937 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.592895% / 7.94729e-06% / 8.34465e-05% | 20260505 14:11:32.937 | INFO | Baz IO overall: | 20260505 14:11:32.937 | INFO | BytesWritten : 2279413530027 | 20260505 14:11:32.937 | INFO | NumEvents : 1083937205407 | 20260505 14:11:34.613 | INFO | Analyzing chunk frames = [3825664,3826176) | 20260505 14:11:34.613 | INFO | MemInfo: { | 20260505 14:11:34.613 | INFO | "gpu" : " 17 GB", | 20260505 14:11:34.613 | INFO | "huge" : "", | 20260505 14:11:34.613 | INFO | "huge_shared" : " 27 GB", | 20260505 14:11:34.613 | INFO | "malloc" : " 14 GB", | 20260505 14:11:34.613 | INFO | "pinned" : " 29 GB" | 20260505 14:11:34.613 | INFO | } | 20260505 14:11:34.613 | INFO | Perf report for the last 5.13183 seconds: | 20260505 14:11:34.613 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:11:34.613 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:34.613 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:34.613 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:34.613 | INFO | ANALYSIS : 64.3%, 3.0, 99.5ms, 0 | 20260505 14:11:34.613 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:34.613 | INFO | RT_METRICS : 45.3%, 1.0, 23.7ms, 0 | 20260505 14:11:34.613 | INFO | PRE_HQ : 60.7%, 1.8, 57.2ms, 0 | 20260505 14:11:34.613 | INFO | BAZWRITER : 64.0%, 4.0, 131.2ms, 0 | 20260505 14:11:34.613 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 14:11:34.613 | INFO | Uploaded 36GiB to GPU in 2.15831s (16.6797GiB/s) | 20260505 14:11:34.613 | INFO | Downloaded 38GiB to GPU in 2.28201s (16.652GiB/s) | 20260505 14:11:39.763 | INFO | Analyzing chunk frames = [3826176,3826688) | 20260505 14:11:39.763 | INFO | MemInfo: { | 20260505 14:11:39.763 | INFO | "gpu" : " 17 GB", | 20260505 14:11:39.763 | INFO | "huge" : "", | 20260505 14:11:39.763 | INFO | "huge_shared" : " 27 GB", | 20260505 14:11:39.763 | INFO | "malloc" : " 14 GB", | 20260505 14:11:39.763 | INFO | "pinned" : " 29 GB" | 20260505 14:11:39.763 | INFO | } | 20260505 14:11:39.763 | INFO | Perf report for the last 5.15044 seconds: | 20260505 14:11:39.763 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:11:39.763 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:39.763 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:39.763 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:39.763 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 14:11:39.763 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:39.763 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:39.763 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 14:11:39.763 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:39.763 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:39.764 | INFO | Uploaded 36GiB to GPU in 2.17093s (16.5827GiB/s) | 20260505 14:11:39.764 | INFO | Downloaded 34GiB to GPU in 2.02852s (16.761GiB/s) | 20260505 14:11:44.903 | INFO | Analyzing chunk frames = [3826688,3827200) | 20260505 14:11:44.903 | INFO | MemInfo: { | 20260505 14:11:44.903 | INFO | "gpu" : " 17 GB", | 20260505 14:11:44.903 | INFO | "huge" : "", | 20260505 14:11:44.903 | INFO | "huge_shared" : " 27 GB", | 20260505 14:11:44.903 | INFO | "malloc" : " 14 GB", | 20260505 14:11:44.903 | INFO | "pinned" : " 29 GB" | 20260505 14:11:44.903 | INFO | } | 20260505 14:11:44.903 | INFO | Perf report for the last 5.13998 seconds: | 20260505 14:11:44.903 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:11:44.903 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:44.905 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:44.905 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:44.905 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 14:11:44.905 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:44.905 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:44.905 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:11:44.905 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:44.905 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:44.905 | INFO | Uploaded 36GiB to GPU in 2.23235s (16.1265GiB/s) | 20260505 14:11:44.905 | INFO | Downloaded 34GiB to GPU in 2.11966s (16.0403GiB/s) | 20260505 14:11:50.043 | INFO | Analyzing chunk frames = [3827200,3827712) | 20260505 14:11:50.043 | INFO | MemInfo: { | 20260505 14:11:50.043 | INFO | "gpu" : " 17 GB", | 20260505 14:11:50.043 | INFO | "huge" : "", | 20260505 14:11:50.043 | INFO | "huge_shared" : " 27 GB", | 20260505 14:11:50.043 | INFO | "malloc" : " 14 GB", | 20260505 14:11:50.044 | INFO | "pinned" : " 29 GB" | 20260505 14:11:50.044 | INFO | } | 20260505 14:11:50.044 | INFO | Perf report for the last 5.14024 seconds: | 20260505 14:11:50.044 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:11:50.044 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:50.044 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:50.044 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:50.044 | INFO | ANALYSIS : 62.8%, 3.0, 97.5ms, 0 | 20260505 14:11:50.044 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:50.044 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:50.044 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:11:50.044 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:50.044 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:50.044 | INFO | Uploaded 36GiB to GPU in 2.18451s (16.4797GiB/s) | 20260505 14:11:50.044 | INFO | Downloaded 34GiB to GPU in 2.1269s (15.9857GiB/s) | 20260505 14:11:55.178 | INFO | Analyzing chunk frames = [3827712,3828224) | 20260505 14:11:55.178 | INFO | MemInfo: { | 20260505 14:11:55.178 | INFO | "gpu" : " 17 GB", | 20260505 14:11:55.178 | INFO | "huge" : "", | 20260505 14:11:55.179 | INFO | "huge_shared" : " 27 GB", | 20260505 14:11:55.179 | INFO | "malloc" : " 14 GB", | 20260505 14:11:55.179 | INFO | "pinned" : " 29 GB" | 20260505 14:11:55.179 | INFO | } | 20260505 14:11:55.179 | INFO | Perf report for the last 5.13501 seconds: | 20260505 14:11:55.179 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:11:55.179 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:55.179 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:55.179 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:55.179 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 14:11:55.179 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:55.179 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:55.179 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:11:55.179 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:55.179 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:11:55.180 | INFO | Uploaded 36GiB to GPU in 2.16088s (16.6599GiB/s) | 20260505 14:11:55.180 | INFO | Downloaded 34GiB to GPU in 2.12442s (16.0044GiB/s) | 20260505 14:12:00.319 | INFO | Analyzing chunk frames = [3828224,3828736) | 20260505 14:12:00.319 | INFO | MemInfo: { | 20260505 14:12:00.319 | INFO | "gpu" : " 17 GB", | 20260505 14:12:00.319 | INFO | "huge" : "", | 20260505 14:12:00.319 | INFO | "huge_shared" : " 27 GB", | 20260505 14:12:00.319 | INFO | "malloc" : " 14 GB", | 20260505 14:12:00.319 | INFO | "pinned" : " 29 GB" | 20260505 14:12:00.319 | INFO | } | 20260505 14:12:00.319 | INFO | Perf report for the last 5.14052 seconds: | 20260505 14:12:00.319 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:12:00.319 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:00.319 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:00.319 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:00.319 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 14:12:00.319 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:00.319 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:00.319 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 14:12:00.319 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:00.319 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:00.319 | INFO | Uploaded 36GiB to GPU in 2.17711s (16.5357GiB/s) | 20260505 14:12:00.319 | INFO | Downloaded 34GiB to GPU in 2.11174s (16.1005GiB/s) | 20260505 14:12:05.457 | INFO | Analyzing chunk frames = [3828736,3829248) | 20260505 14:12:05.457 | INFO | MemInfo: { | 20260505 14:12:05.457 | INFO | "gpu" : " 17 GB", | 20260505 14:12:05.457 | INFO | "huge" : "", | 20260505 14:12:05.457 | INFO | "huge_shared" : " 27 GB", | 20260505 14:12:05.457 | INFO | "malloc" : " 14 GB", | 20260505 14:12:05.457 | INFO | "pinned" : " 29 GB" | 20260505 14:12:05.457 | INFO | } | 20260505 14:12:05.457 | INFO | Perf report for the last 5.13826 seconds: | 20260505 14:12:05.457 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:12:05.457 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:05.457 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:05.457 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:05.457 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 14:12:05.457 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:05.457 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:05.457 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 14:12:05.458 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:05.458 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:05.458 | INFO | Uploaded 36GiB to GPU in 2.12539s (16.9381GiB/s) | 20260505 14:12:05.458 | INFO | Downloaded 34GiB to GPU in 2.17917s (15.6023GiB/s) | 20260505 14:12:10.599 | INFO | Analyzing chunk frames = [3829248,3829760) | 20260505 14:12:10.599 | INFO | MemInfo: { | 20260505 14:12:10.599 | INFO | "gpu" : " 17 GB", | 20260505 14:12:10.599 | INFO | "huge" : "", | 20260505 14:12:10.599 | INFO | "huge_shared" : " 27 GB", | 20260505 14:12:10.599 | INFO | "malloc" : " 14 GB", | 20260505 14:12:10.599 | INFO | "pinned" : " 29 GB" | 20260505 14:12:10.599 | INFO | } | 20260505 14:12:10.599 | INFO | Perf report for the last 5.14189 seconds: | 20260505 14:12:10.599 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:12:10.599 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:10.599 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:10.599 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:10.599 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 14:12:10.599 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:10.599 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:10.599 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 14:12:10.599 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:10.599 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:10.599 | INFO | Uploaded 36GiB to GPU in 2.21668s (16.2405GiB/s) | 20260505 14:12:10.600 | INFO | Downloaded 34GiB to GPU in 2.10541s (16.1488GiB/s) | 20260505 14:12:10.734 | INFO | Starting a round of Baz IO | 20260505 14:12:13.959 | INFO | Populating RTMetrics report | 20260505 14:12:13.959 | INFO | RTMetrics report populated | 20260505 14:12:13.959 | INFO | Opening tmp RTMetrics json file | 20260505 14:12:13.998 | INFO | Serializing RTMetrics to json | 20260505 14:12:14.007 | INFO | Serialization done, swapping tmp file now... | 20260505 14:12:14.013 | INFO | RTMetrics done | 20260505 14:12:14.065 | INFO | Finished a round of Baz IO: Wrote 2.267 GiB in 3.331 seconds | 20260505 14:12:14.065 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.84% / 0.00% / 77.16% | 20260505 14:12:14.065 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.592188% / 0% / 7.54992e-05% | 20260505 14:12:14.065 | INFO | Baz IO overall: | 20260505 14:12:14.065 | INFO | BytesWritten : 2281848035670 | 20260505 14:12:14.065 | INFO | NumEvents : 1085086787783 | 20260505 14:12:15.727 | INFO | Analyzing chunk frames = [3829760,3830272) | 20260505 14:12:15.727 | INFO | MemInfo: { | 20260505 14:12:15.727 | INFO | "gpu" : " 17 GB", | 20260505 14:12:15.727 | INFO | "huge" : "", | 20260505 14:12:15.728 | INFO | "huge_shared" : " 27 GB", | 20260505 14:12:15.728 | INFO | "malloc" : " 14 GB", | 20260505 14:12:15.728 | INFO | "pinned" : " 29 GB" | 20260505 14:12:15.728 | INFO | } | 20260505 14:12:15.728 | INFO | Perf report for the last 5.12828 seconds: | 20260505 14:12:15.728 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:12:15.728 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:15.728 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:15.728 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:15.728 | INFO | ANALYSIS : 64.3%, 3.0, 99.7ms, 0 | 20260505 14:12:15.728 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:15.728 | INFO | RT_METRICS : 46.2%, 1.0, 24.2ms, 0 | 20260505 14:12:15.728 | INFO | PRE_HQ : 61.0%, 1.8, 57.6ms, 0 | 20260505 14:12:15.728 | INFO | BAZWRITER : 64.3%, 4.0, 131.0ms, 0 | 20260505 14:12:15.728 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 14:12:15.728 | INFO | Uploaded 36GiB to GPU in 2.19689s (16.3868GiB/s) | 20260505 14:12:15.728 | INFO | Downloaded 38GiB to GPU in 2.26258s (16.795GiB/s) | 20260505 14:12:20.866 | INFO | Analyzing chunk frames = [3830272,3830784) | 20260505 14:12:20.866 | INFO | MemInfo: { | 20260505 14:12:20.866 | INFO | "gpu" : " 17 GB", | 20260505 14:12:20.866 | INFO | "huge" : "", | 20260505 14:12:20.866 | INFO | "huge_shared" : " 27 GB", | 20260505 14:12:20.866 | INFO | "malloc" : " 14 GB", | 20260505 14:12:20.866 | INFO | "pinned" : " 29 GB" | 20260505 14:12:20.866 | INFO | } | 20260505 14:12:20.867 | INFO | Perf report for the last 5.13888 seconds: | 20260505 14:12:20.867 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:12:20.867 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:20.867 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:20.867 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:20.867 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 14:12:20.867 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:20.867 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:20.867 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:12:20.867 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:20.867 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:20.867 | INFO | Uploaded 36GiB to GPU in 2.17422s (16.5577GiB/s) | 20260505 14:12:20.867 | INFO | Downloaded 34GiB to GPU in 2.02965s (16.7517GiB/s) | 20260505 14:12:26.020 | INFO | Analyzing chunk frames = [3830784,3831296) | 20260505 14:12:26.020 | INFO | MemInfo: { | 20260505 14:12:26.020 | INFO | "gpu" : " 17 GB", | 20260505 14:12:26.020 | INFO | "huge" : "", | 20260505 14:12:26.020 | INFO | "huge_shared" : " 27 GB", | 20260505 14:12:26.020 | INFO | "malloc" : " 14 GB", | 20260505 14:12:26.020 | INFO | "pinned" : " 29 GB" | 20260505 14:12:26.020 | INFO | } | 20260505 14:12:26.020 | INFO | Perf report for the last 5.15376 seconds: | 20260505 14:12:26.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:12:26.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:26.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:26.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:26.020 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 14:12:26.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:26.020 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:26.020 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:12:26.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:26.020 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:26.021 | INFO | Uploaded 36GiB to GPU in 2.22467s (16.1822GiB/s) | 20260505 14:12:26.021 | INFO | Downloaded 34GiB to GPU in 2.10511s (16.1512GiB/s) | 20260505 14:12:31.164 | INFO | Analyzing chunk frames = [3831296,3831808) | 20260505 14:12:31.164 | INFO | MemInfo: { | 20260505 14:12:31.164 | INFO | "gpu" : " 17 GB", | 20260505 14:12:31.164 | INFO | "huge" : "", | 20260505 14:12:31.164 | INFO | "huge_shared" : " 27 GB", | 20260505 14:12:31.164 | INFO | "malloc" : " 14 GB", | 20260505 14:12:31.164 | INFO | "pinned" : " 29 GB" | 20260505 14:12:31.164 | INFO | } | 20260505 14:12:31.164 | INFO | Perf report for the last 5.14406 seconds: | 20260505 14:12:31.164 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:12:31.164 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:31.164 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:31.164 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:31.164 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260505 14:12:31.164 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:31.165 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:31.165 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 14:12:31.165 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:31.165 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:31.165 | INFO | Uploaded 36GiB to GPU in 2.18629s (16.4662GiB/s) | 20260505 14:12:31.165 | INFO | Downloaded 34GiB to GPU in 2.13429s (15.9304GiB/s) | 20260505 14:12:36.296 | INFO | Analyzing chunk frames = [3831808,3832320) | 20260505 14:12:36.296 | INFO | MemInfo: { | 20260505 14:12:36.296 | INFO | "gpu" : " 17 GB", | 20260505 14:12:36.296 | INFO | "huge" : "", | 20260505 14:12:36.296 | INFO | "huge_shared" : " 27 GB", | 20260505 14:12:36.296 | INFO | "malloc" : " 14 GB", | 20260505 14:12:36.296 | INFO | "pinned" : " 29 GB" | 20260505 14:12:36.296 | INFO | } | 20260505 14:12:36.296 | INFO | Perf report for the last 5.13162 seconds: | 20260505 14:12:36.296 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:12:36.296 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:36.296 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:36.296 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:36.296 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 14:12:36.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:36.296 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:36.296 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:12:36.296 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:36.296 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:36.296 | INFO | Uploaded 36GiB to GPU in 2.13377s (16.8716GiB/s) | 20260505 14:12:36.296 | INFO | Downloaded 34GiB to GPU in 2.1259s (15.9932GiB/s) | 20260505 14:12:41.424 | INFO | Analyzing chunk frames = [3832320,3832832) | 20260505 14:12:41.424 | INFO | MemInfo: { | 20260505 14:12:41.424 | INFO | "gpu" : " 17 GB", | 20260505 14:12:41.424 | INFO | "huge" : "", | 20260505 14:12:41.424 | INFO | "huge_shared" : " 27 GB", | 20260505 14:12:41.424 | INFO | "malloc" : " 14 GB", | 20260505 14:12:41.424 | INFO | "pinned" : " 29 GB" | 20260505 14:12:41.424 | INFO | } | 20260505 14:12:41.424 | INFO | Perf report for the last 5.12794 seconds: | 20260505 14:12:41.424 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:12:41.424 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:41.424 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:41.424 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:41.424 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 14:12:41.424 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:41.424 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:41.424 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 14:12:41.424 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:41.424 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:41.424 | INFO | Uploaded 36GiB to GPU in 2.17151s (16.5783GiB/s) | 20260505 14:12:41.424 | INFO | Downloaded 34GiB to GPU in 2.10834s (16.1264GiB/s) | 20260505 14:12:46.572 | INFO | Analyzing chunk frames = [3832832,3833344) | 20260505 14:12:46.572 | INFO | MemInfo: { | 20260505 14:12:46.572 | INFO | "gpu" : " 17 GB", | 20260505 14:12:46.572 | INFO | "huge" : "", | 20260505 14:12:46.572 | INFO | "huge_shared" : " 27 GB", | 20260505 14:12:46.572 | INFO | "malloc" : " 14 GB", | 20260505 14:12:46.572 | INFO | "pinned" : " 29 GB" | 20260505 14:12:46.572 | INFO | } | 20260505 14:12:46.572 | INFO | Perf report for the last 5.14804 seconds: | 20260505 14:12:46.572 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:12:46.572 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:46.572 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:46.572 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:46.572 | INFO | ANALYSIS : 63.2%, 3.0, 98.8ms, 0 | 20260505 14:12:46.572 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:46.572 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:46.572 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 14:12:46.572 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:46.572 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:46.572 | INFO | Uploaded 36GiB to GPU in 2.1615s (16.6551GiB/s) | 20260505 14:12:46.572 | INFO | Downloaded 34GiB to GPU in 2.14662s (15.8389GiB/s) | 20260505 14:12:51.714 | INFO | Analyzing chunk frames = [3833344,3833856) | 20260505 14:12:51.714 | INFO | MemInfo: { | 20260505 14:12:51.714 | INFO | "gpu" : " 17 GB", | 20260505 14:12:51.714 | INFO | "huge" : "", | 20260505 14:12:51.714 | INFO | "huge_shared" : " 27 GB", | 20260505 14:12:51.714 | INFO | "malloc" : " 14 GB", | 20260505 14:12:51.714 | INFO | "pinned" : " 29 GB" | 20260505 14:12:51.714 | INFO | } | 20260505 14:12:51.714 | INFO | Perf report for the last 5.14179 seconds: | 20260505 14:12:51.714 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:12:51.714 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:51.714 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:51.714 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:51.714 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 14:12:51.714 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:51.714 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:51.714 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:12:51.714 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:51.714 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:51.714 | INFO | Uploaded 36GiB to GPU in 2.20993s (16.2901GiB/s) | 20260505 14:12:51.714 | INFO | Downloaded 34GiB to GPU in 2.10704s (16.1364GiB/s) | 20260505 14:12:51.714 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 14:12:51.714 | INFO | Total mem usage: 15.3 GiB | 20260505 14:12:51.714 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 14:12:51.714 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.8%) | 20260505 14:12:51.714 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 14:12:51.714 | INFO | High Water Breakdown: | 20260505 14:12:51.714 | INFO | 61% : 8.68 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 14:12:51.714 | INFO | 38.8% : 5.52 GiB ( 83.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 14:12:51.714 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 14:12:51.714 | INFO | ------------------------------------------ | 20260505 14:12:51.714 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:12:51.714 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 14:12:51.715 | INFO | Total mem usage: 26.5 GiB | 20260505 14:12:51.715 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 14:12:51.715 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 14:12:51.715 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 14:12:51.715 | INFO | High Water Breakdown: | 20260505 14:12:51.715 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:12:51.715 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:12:51.715 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:12:51.715 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:12:51.715 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:12:51.715 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:12:51.715 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 14:12:51.715 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 14:12:51.715 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:12:51.715 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 14:12:51.715 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 14:12:51.715 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 14:12:51.715 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 14:12:51.715 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 14:12:51.715 | INFO | ------------------------------------------ | 20260505 14:12:51.715 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:12:51.715 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 14:12:51.715 | INFO | Total mem usage: 25.6 GiB | 20260505 14:12:51.715 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 14:12:51.715 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 14:12:51.715 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 14:12:51.715 | INFO | High Water Breakdown: | 20260505 14:12:51.715 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 14:12:51.715 | INFO | ------------------------------------------ | 20260505 14:12:51.715 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:12:51.715 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 14:12:51.715 | INFO | Total mem usage: 15.7 GiB | 20260505 14:12:51.715 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 14:12:51.715 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 14:12:51.715 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 14:12:51.715 | INFO | High Water Breakdown: | 20260505 14:12:51.715 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 14:12:51.715 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 14:12:51.715 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 14:12:51.715 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 14:12:51.715 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 14:12:51.715 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 14:12:51.715 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 14:12:51.715 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 14:12:51.715 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 14:12:51.715 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:12:51.715 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 14:12:51.715 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:12:51.715 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 14:12:51.715 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 14:12:51.715 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 14:12:51.715 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 14:12:51.715 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:12:51.715 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 14:12:51.715 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:12:51.715 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:12:51.715 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:12:51.715 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 14:12:51.715 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 14:12:51.715 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:12:51.715 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 14:12:51.715 | INFO | ------------------------------------------ | 20260505 14:12:51.715 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:12:51.715 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 14:12:51.715 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:12:51.715 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:12:51.715 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.65e+01, 1.36e+01 / 1.59e+00 | 20260505 14:12:51.715 | INFO | PerfReport: 0.29%, AggregateStats -- 9.30e-02 / 1.01e-01, 1.11e-02 / 4.85e-03 | 20260505 14:12:51.715 | INFO | PerfReport: 11.79%, Baseline -- 3.75e+00 / 4.48e+00, 5.89e-01 / 1.48e-01 | 20260505 14:12:51.715 | INFO | PerfReport: 3.30%, Histogram -- 1.05e+00 / 1.50e+00, 2.00e-01 / 1.02e-01 | 20260505 14:12:51.715 | INFO | PerfReport: 16.01%, DME -- 4.08e+01 / 4.59e+01, 4.71e+00 / 1.89e+00 | 20260505 14:12:51.715 | INFO | PerfReport: 56.35%, FrameLabeling -- 1.79e+01 / 1.97e+01, 1.36e+00 / 1.95e+00 | 20260505 14:12:51.715 | INFO | PerfReport: 7.47%, PulseAccumulating -- 2.38e+00 / 2.94e+00, 4.57e-01 / 1.87e-01 | 20260505 14:12:51.715 | INFO | PerfReport: 3.54%, Metrics -- 1.13e+00 / 1.45e+00, 3.14e-01 / 1.67e-01 | 20260505 14:12:51.715 | INFO | PerfReport: 1.24%, PulseSerializing -- 3.95e-01 / 4.45e-01, 6.93e-02 / 3.33e-02 | 20260505 14:12:51.715 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 14:12:51.715 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:12:51.715 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:12:51.715 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260505 14:12:51.715 | INFO | PerfReport: 22.55%, Upload -- 2.27e+01 / 0.00e+00, 8.04e+00 / 0.00e+00 | 20260505 14:12:51.715 | INFO | PerfReport: 22.04%, Download -- 2.22e+01 / 0.00e+00, 8.09e+00 / 0.00e+00 | 20260505 14:12:51.715 | INFO | PerfReport: 31.61%, Compute -- 3.18e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 14:12:51.715 | INFO | PerfReport: 2.21%, LockUpload -- 2.23e+00 / 0.00e+00, 7.90e+00 / 0.00e+00 | 20260505 14:12:51.715 | INFO | PerfReport: 1.46%, LockDownload -- 1.47e+00 / 0.00e+00, 3.00e+00 / 0.00e+00 | 20260505 14:12:51.715 | INFO | PerfReport: 20.11%, LockCompute -- 2.03e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260505 14:12:51.715 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.86e-02 / 0.00e+00 | 20260505 14:12:51.850 | INFO | Starting a round of Baz IO | 20260505 14:12:55.067 | INFO | Populating RTMetrics report | 20260505 14:12:55.067 | INFO | RTMetrics report populated | 20260505 14:12:55.067 | INFO | Opening tmp RTMetrics json file | 20260505 14:12:55.080 | INFO | Serializing RTMetrics to json | 20260505 14:12:55.103 | INFO | Serialization done, swapping tmp file now... | 20260505 14:12:55.125 | INFO | RTMetrics done | 20260505 14:12:55.168 | INFO | Finished a round of Baz IO: Wrote 2.268 GiB in 3.317 seconds | 20260505 14:12:55.168 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.83% / 0.00% / 77.16% | 20260505 14:12:55.168 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.591195% / 0% / 9.13938e-05% | 20260505 14:12:55.168 | INFO | Baz IO overall: | 20260505 14:12:55.168 | INFO | BytesWritten : 2284283118743 | 20260505 14:12:55.168 | INFO | NumEvents : 1086236781586 | 20260505 14:12:56.855 | INFO | Analyzing chunk frames = [3833856,3834368) | 20260505 14:12:56.855 | INFO | MemInfo: { | 20260505 14:12:56.855 | INFO | "gpu" : " 17 GB", | 20260505 14:12:56.855 | INFO | "huge" : "", | 20260505 14:12:56.855 | INFO | "huge_shared" : " 27 GB", | 20260505 14:12:56.855 | INFO | "malloc" : " 14 GB", | 20260505 14:12:56.855 | INFO | "pinned" : " 29 GB" | 20260505 14:12:56.855 | INFO | } | 20260505 14:12:56.855 | INFO | Perf report for the last 5.14094 seconds: | 20260505 14:12:56.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:12:56.855 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:56.855 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:56.855 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:56.855 | INFO | ANALYSIS : 64.1%, 3.0, 99.5ms, 0 | 20260505 14:12:56.855 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:12:56.855 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260505 14:12:56.855 | INFO | PRE_HQ : 60.7%, 1.8, 57.1ms, 0 | 20260505 14:12:56.855 | INFO | BAZWRITER : 63.9%, 4.0, 131.0ms, 0 | 20260505 14:12:56.855 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260505 14:12:56.855 | INFO | Uploaded 36GiB to GPU in 2.1893s (16.4436GiB/s) | 20260505 14:12:56.855 | INFO | Downloaded 38GiB to GPU in 2.26519s (16.7756GiB/s) | 20260505 14:13:01.991 | INFO | Analyzing chunk frames = [3834368,3834880) | 20260505 14:13:01.991 | INFO | MemInfo: { | 20260505 14:13:01.991 | INFO | "gpu" : " 17 GB", | 20260505 14:13:01.991 | INFO | "huge" : "", | 20260505 14:13:01.991 | INFO | "huge_shared" : " 27 GB", | 20260505 14:13:01.991 | INFO | "malloc" : " 14 GB", | 20260505 14:13:01.991 | INFO | "pinned" : " 29 GB" | 20260505 14:13:01.991 | INFO | } | 20260505 14:13:01.991 | INFO | Perf report for the last 5.13664 seconds: | 20260505 14:13:01.991 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:13:01.991 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:01.991 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:01.991 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:01.991 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 14:13:01.991 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:01.991 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:01.991 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:13:01.991 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:01.992 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:01.992 | INFO | Uploaded 36GiB to GPU in 2.1947s (16.4032GiB/s) | 20260505 14:13:01.992 | INFO | Downloaded 34GiB to GPU in 2.03515s (16.7064GiB/s) | 20260505 14:13:07.124 | INFO | Analyzing chunk frames = [3834880,3835392) | 20260505 14:13:07.124 | INFO | MemInfo: { | 20260505 14:13:07.124 | INFO | "gpu" : " 17 GB", | 20260505 14:13:07.124 | INFO | "huge" : "", | 20260505 14:13:07.124 | INFO | "huge_shared" : " 27 GB", | 20260505 14:13:07.124 | INFO | "malloc" : " 14 GB", | 20260505 14:13:07.124 | INFO | "pinned" : " 29 GB" | 20260505 14:13:07.124 | INFO | } | 20260505 14:13:07.124 | INFO | Perf report for the last 5.13289 seconds: | 20260505 14:13:07.124 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:13:07.124 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:07.124 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:07.124 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:07.124 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 14:13:07.124 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:07.124 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:07.124 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:13:07.124 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:07.124 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:07.125 | INFO | Uploaded 36GiB to GPU in 2.20121s (16.3546GiB/s) | 20260505 14:13:07.125 | INFO | Downloaded 34GiB to GPU in 2.12192s (16.0232GiB/s) | 20260505 14:13:12.275 | INFO | Analyzing chunk frames = [3835392,3835904) | 20260505 14:13:12.275 | INFO | MemInfo: { | 20260505 14:13:12.275 | INFO | "gpu" : " 17 GB", | 20260505 14:13:12.275 | INFO | "huge" : "", | 20260505 14:13:12.275 | INFO | "huge_shared" : " 27 GB", | 20260505 14:13:12.275 | INFO | "malloc" : " 14 GB", | 20260505 14:13:12.275 | INFO | "pinned" : " 29 GB" | 20260505 14:13:12.275 | INFO | } | 20260505 14:13:12.275 | INFO | Perf report for the last 5.15094 seconds: | 20260505 14:13:12.275 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:13:12.275 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:12.275 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:12.275 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:12.275 | INFO | ANALYSIS : 62.8%, 3.0, 97.8ms, 0 | 20260505 14:13:12.275 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:12.275 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:12.275 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:13:12.275 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:12.275 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:12.275 | INFO | Uploaded 36GiB to GPU in 2.20247s (16.3453GiB/s) | 20260505 14:13:12.276 | INFO | Downloaded 34GiB to GPU in 2.13476s (15.9268GiB/s) | 20260505 14:13:17.413 | INFO | Analyzing chunk frames = [3835904,3836416) | 20260505 14:13:17.413 | INFO | MemInfo: { | 20260505 14:13:17.413 | INFO | "gpu" : " 17 GB", | 20260505 14:13:17.413 | INFO | "huge" : "", | 20260505 14:13:17.413 | INFO | "huge_shared" : " 27 GB", | 20260505 14:13:17.413 | INFO | "malloc" : " 14 GB", | 20260505 14:13:17.413 | INFO | "pinned" : " 29 GB" | 20260505 14:13:17.413 | INFO | } | 20260505 14:13:17.413 | INFO | Perf report for the last 5.13817 seconds: | 20260505 14:13:17.413 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:13:17.413 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:17.413 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:17.413 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:17.414 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 14:13:17.414 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:17.414 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:17.414 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 14:13:17.414 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:17.414 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:17.414 | INFO | Uploaded 36GiB to GPU in 2.16635s (16.6178GiB/s) | 20260505 14:13:17.414 | INFO | Downloaded 34GiB to GPU in 2.1306s (15.9579GiB/s) | 20260505 14:13:22.544 | INFO | Analyzing chunk frames = [3836416,3836928) | 20260505 14:13:22.544 | INFO | MemInfo: { | 20260505 14:13:22.544 | INFO | "gpu" : " 17 GB", | 20260505 14:13:22.544 | INFO | "huge" : "", | 20260505 14:13:22.545 | INFO | "huge_shared" : " 27 GB", | 20260505 14:13:22.545 | INFO | "malloc" : " 14 GB", | 20260505 14:13:22.545 | INFO | "pinned" : " 29 GB" | 20260505 14:13:22.545 | INFO | } | 20260505 14:13:22.545 | INFO | Perf report for the last 5.13115 seconds: | 20260505 14:13:22.545 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:13:22.545 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:22.545 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:22.545 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:22.545 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 14:13:22.545 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:22.545 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:22.545 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:13:22.545 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:22.545 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:22.545 | INFO | Uploaded 36GiB to GPU in 2.20724s (16.3099GiB/s) | 20260505 14:13:22.545 | INFO | Downloaded 34GiB to GPU in 2.11391s (16.0839GiB/s) | 20260505 14:13:27.681 | INFO | Analyzing chunk frames = [3836928,3837440) | 20260505 14:13:27.681 | INFO | MemInfo: { | 20260505 14:13:27.681 | INFO | "gpu" : " 17 GB", | 20260505 14:13:27.681 | INFO | "huge" : "", | 20260505 14:13:27.681 | INFO | "huge_shared" : " 27 GB", | 20260505 14:13:27.681 | INFO | "malloc" : " 14 GB", | 20260505 14:13:27.681 | INFO | "pinned" : " 29 GB" | 20260505 14:13:27.681 | INFO | } | 20260505 14:13:27.681 | INFO | Perf report for the last 5.13621 seconds: | 20260505 14:13:27.681 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:13:27.681 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:27.681 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:27.681 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:27.681 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 14:13:27.681 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:27.681 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:27.681 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 14:13:27.681 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:27.681 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:27.682 | INFO | Uploaded 36GiB to GPU in 2.14815s (16.7586GiB/s) | 20260505 14:13:27.682 | INFO | Downloaded 34GiB to GPU in 2.17741s (15.6149GiB/s) | 20260505 14:13:32.823 | INFO | Analyzing chunk frames = [3837440,3837952) | 20260505 14:13:32.823 | INFO | MemInfo: { | 20260505 14:13:32.823 | INFO | "gpu" : " 17 GB", | 20260505 14:13:32.823 | INFO | "huge" : "", | 20260505 14:13:32.823 | INFO | "huge_shared" : " 27 GB", | 20260505 14:13:32.823 | INFO | "malloc" : " 14 GB", | 20260505 14:13:32.823 | INFO | "pinned" : " 29 GB" | 20260505 14:13:32.823 | INFO | } | 20260505 14:13:32.823 | INFO | Perf report for the last 5.14189 seconds: | 20260505 14:13:32.823 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:13:32.823 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:32.823 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:32.823 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:32.823 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 14:13:32.823 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:32.823 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:32.823 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 14:13:32.823 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:32.823 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:32.823 | INFO | Uploaded 36GiB to GPU in 2.14884s (16.7532GiB/s) | 20260505 14:13:32.823 | INFO | Downloaded 34GiB to GPU in 2.10299s (16.1675GiB/s) | 20260505 14:13:32.960 | INFO | Starting a round of Baz IO | 20260505 14:13:36.181 | INFO | Populating RTMetrics report | 20260505 14:13:36.181 | INFO | RTMetrics report populated | 20260505 14:13:36.181 | INFO | Opening tmp RTMetrics json file | 20260505 14:13:36.221 | INFO | Serializing RTMetrics to json | 20260505 14:13:36.264 | INFO | Serialization done, swapping tmp file now... | 20260505 14:13:36.265 | INFO | RTMetrics done | 20260505 14:13:36.285 | INFO | Finished a round of Baz IO: Wrote 2.267 GiB in 3.325 seconds | 20260505 14:13:36.285 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.83% / 0.00% / 77.17% | 20260505 14:13:36.285 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.589685% / 0% / 8.34465e-05% | 20260505 14:13:36.285 | INFO | Baz IO overall: | 20260505 14:13:36.285 | INFO | BytesWritten : 2286717455375 | 20260505 14:13:36.285 | INFO | NumEvents : 1087386168197 | 20260505 14:13:37.967 | INFO | Analyzing chunk frames = [3837952,3838464) | 20260505 14:13:37.967 | INFO | MemInfo: { | 20260505 14:13:37.967 | INFO | "gpu" : " 17 GB", | 20260505 14:13:37.967 | INFO | "huge" : "", | 20260505 14:13:37.967 | INFO | "huge_shared" : " 27 GB", | 20260505 14:13:37.967 | INFO | "malloc" : " 14 GB", | 20260505 14:13:37.967 | INFO | "pinned" : " 29 GB" | 20260505 14:13:37.967 | INFO | } | 20260505 14:13:37.967 | INFO | Perf report for the last 5.14455 seconds: | 20260505 14:13:37.967 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:13:37.967 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:37.967 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:37.967 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:37.967 | INFO | ANALYSIS : 64.2%, 3.0, 100.0ms, 0 | 20260505 14:13:37.967 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:37.967 | INFO | RT_METRICS : 46.6%, 1.0, 24.4ms, 0 | 20260505 14:13:37.967 | INFO | PRE_HQ : 60.7%, 1.8, 57.1ms, 0 | 20260505 14:13:37.967 | INFO | BAZWRITER : 64.0%, 4.0, 130.4ms, 0 | 20260505 14:13:37.967 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 14:13:37.968 | INFO | Uploaded 36GiB to GPU in 2.17828s (16.5268GiB/s) | 20260505 14:13:37.968 | INFO | Downloaded 38GiB to GPU in 2.26444s (16.7812GiB/s) | 20260505 14:13:43.110 | INFO | Analyzing chunk frames = [3838464,3838976) | 20260505 14:13:43.110 | INFO | MemInfo: { | 20260505 14:13:43.110 | INFO | "gpu" : " 17 GB", | 20260505 14:13:43.110 | INFO | "huge" : "", | 20260505 14:13:43.110 | INFO | "huge_shared" : " 27 GB", | 20260505 14:13:43.110 | INFO | "malloc" : " 14 GB", | 20260505 14:13:43.110 | INFO | "pinned" : " 29 GB" | 20260505 14:13:43.110 | INFO | } | 20260505 14:13:43.110 | INFO | Perf report for the last 5.14313 seconds: | 20260505 14:13:43.110 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:13:43.110 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:43.110 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:43.110 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:43.110 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 14:13:43.110 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:43.110 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:43.111 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:13:43.111 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:43.111 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:43.111 | INFO | Uploaded 36GiB to GPU in 2.20092s (16.3568GiB/s) | 20260505 14:13:43.111 | INFO | Downloaded 34GiB to GPU in 2.04487s (16.627GiB/s) | 20260505 14:13:48.240 | INFO | Analyzing chunk frames = [3838976,3839488) | 20260505 14:13:48.240 | INFO | MemInfo: { | 20260505 14:13:48.240 | INFO | "gpu" : " 17 GB", | 20260505 14:13:48.240 | INFO | "huge" : "", | 20260505 14:13:48.240 | INFO | "huge_shared" : " 27 GB", | 20260505 14:13:48.240 | INFO | "malloc" : " 14 GB", | 20260505 14:13:48.240 | INFO | "pinned" : " 29 GB" | 20260505 14:13:48.240 | INFO | } | 20260505 14:13:48.240 | INFO | Perf report for the last 5.12943 seconds: | 20260505 14:13:48.240 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:13:48.240 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:48.240 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:48.240 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:48.240 | INFO | ANALYSIS : 63.6%, 3.0, 98.8ms, 0 | 20260505 14:13:48.240 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:48.240 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:48.240 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:13:48.240 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:48.240 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:48.240 | INFO | Uploaded 36GiB to GPU in 2.23686s (16.094GiB/s) | 20260505 14:13:48.240 | INFO | Downloaded 34GiB to GPU in 2.12264s (16.0178GiB/s) | 20260505 14:13:53.379 | INFO | Analyzing chunk frames = [3839488,3840000) | 20260505 14:13:53.379 | INFO | MemInfo: { | 20260505 14:13:53.379 | INFO | "gpu" : " 17 GB", | 20260505 14:13:53.379 | INFO | "huge" : "", | 20260505 14:13:53.379 | INFO | "huge_shared" : " 27 GB", | 20260505 14:13:53.379 | INFO | "malloc" : " 14 GB", | 20260505 14:13:53.379 | INFO | "pinned" : " 29 GB" | 20260505 14:13:53.379 | INFO | } | 20260505 14:13:53.379 | INFO | Perf report for the last 5.13955 seconds: | 20260505 14:13:53.379 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:13:53.379 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:53.379 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:53.379 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:53.379 | INFO | ANALYSIS : 62.9%, 3.0, 97.7ms, 0 | 20260505 14:13:53.380 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:53.380 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:53.380 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:13:53.380 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:53.380 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:53.380 | INFO | Uploaded 36GiB to GPU in 2.19941s (16.368GiB/s) | 20260505 14:13:53.380 | INFO | Downloaded 34GiB to GPU in 2.13698s (15.9103GiB/s) | 20260505 14:13:58.529 | INFO | Analyzing chunk frames = [3840000,3840512) | 20260505 14:13:58.529 | INFO | MemInfo: { | 20260505 14:13:58.529 | INFO | "gpu" : " 17 GB", | 20260505 14:13:58.529 | INFO | "huge" : "", | 20260505 14:13:58.529 | INFO | "huge_shared" : " 27 GB", | 20260505 14:13:58.529 | INFO | "malloc" : " 14 GB", | 20260505 14:13:58.529 | INFO | "pinned" : " 29 GB" | 20260505 14:13:58.529 | INFO | } | 20260505 14:13:58.529 | INFO | Perf report for the last 5.14958 seconds: | 20260505 14:13:58.529 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:13:58.529 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:58.529 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:58.529 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:58.529 | INFO | ANALYSIS : 62.6%, 3.0, 97.9ms, 0 | 20260505 14:13:58.529 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:58.529 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:58.529 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 14:13:58.529 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:58.529 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:13:58.529 | INFO | Uploaded 36GiB to GPU in 2.15408s (16.7125GiB/s) | 20260505 14:13:58.529 | INFO | Downloaded 34GiB to GPU in 2.12298s (16.0152GiB/s) | 20260505 14:14:03.671 | INFO | Analyzing chunk frames = [3840512,3841024) | 20260505 14:14:03.671 | INFO | MemInfo: { | 20260505 14:14:03.671 | INFO | "gpu" : " 17 GB", | 20260505 14:14:03.671 | INFO | "huge" : "", | 20260505 14:14:03.671 | INFO | "huge_shared" : " 27 GB", | 20260505 14:14:03.671 | INFO | "malloc" : " 14 GB", | 20260505 14:14:03.671 | INFO | "pinned" : " 29 GB" | 20260505 14:14:03.671 | INFO | } | 20260505 14:14:03.671 | INFO | Perf report for the last 5.14217 seconds: | 20260505 14:14:03.671 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:14:03.671 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:03.671 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:03.671 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:03.671 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 14:14:03.671 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:03.671 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:03.671 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 14:14:03.671 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:03.671 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:03.671 | INFO | Uploaded 36GiB to GPU in 2.15965s (16.6693GiB/s) | 20260505 14:14:03.672 | INFO | Downloaded 34GiB to GPU in 2.03263s (16.7271GiB/s) | 20260505 14:14:08.804 | INFO | Analyzing chunk frames = [3841024,3841536) | 20260505 14:14:08.804 | INFO | MemInfo: { | 20260505 14:14:08.804 | INFO | "gpu" : " 17 GB", | 20260505 14:14:08.804 | INFO | "huge" : "", | 20260505 14:14:08.804 | INFO | "huge_shared" : " 27 GB", | 20260505 14:14:08.804 | INFO | "malloc" : " 14 GB", | 20260505 14:14:08.804 | INFO | "pinned" : " 29 GB" | 20260505 14:14:08.804 | INFO | } | 20260505 14:14:08.804 | INFO | Perf report for the last 5.13297 seconds: | 20260505 14:14:08.804 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:14:08.804 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:08.804 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:08.804 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:08.804 | INFO | ANALYSIS : 64.1%, 3.0, 100.0ms, 0 | 20260505 14:14:08.804 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:08.804 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:08.804 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 14:14:08.804 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:08.804 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:08.804 | INFO | Uploaded 36GiB to GPU in 2.1513s (16.734GiB/s) | 20260505 14:14:08.804 | INFO | Downloaded 34GiB to GPU in 2.17081s (15.6624GiB/s) | 20260505 14:14:13.935 | INFO | Analyzing chunk frames = [3841536,3842048) | 20260505 14:14:13.935 | INFO | MemInfo: { | 20260505 14:14:13.935 | INFO | "gpu" : " 17 GB", | 20260505 14:14:13.935 | INFO | "huge" : "", | 20260505 14:14:13.935 | INFO | "huge_shared" : " 27 GB", | 20260505 14:14:13.935 | INFO | "malloc" : " 14 GB", | 20260505 14:14:13.935 | INFO | "pinned" : " 29 GB" | 20260505 14:14:13.935 | INFO | } | 20260505 14:14:13.935 | INFO | Perf report for the last 5.1313 seconds: | 20260505 14:14:13.936 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:14:13.936 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:13.936 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:13.936 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:13.936 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 14:14:13.936 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:13.936 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:13.936 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:14:13.936 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:13.936 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:13.936 | INFO | Uploaded 36GiB to GPU in 2.21301s (16.2674GiB/s) | 20260505 14:14:13.936 | INFO | Downloaded 34GiB to GPU in 2.10498s (16.1522GiB/s) | 20260505 14:14:13.936 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 14:14:13.936 | INFO | Total mem usage: 15.3 GiB | 20260505 14:14:13.936 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 14:14:13.936 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.81%) | 20260505 14:14:13.936 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 14:14:13.936 | INFO | High Water Breakdown: | 20260505 14:14:13.936 | INFO | 61% : 8.67 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 14:14:13.936 | INFO | 38.8% : 5.52 GiB ( 83.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 14:14:13.936 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 14:14:13.936 | INFO | ------------------------------------------ | 20260505 14:14:13.936 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:14:13.936 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 14:14:13.936 | INFO | Total mem usage: 26.5 GiB | 20260505 14:14:13.936 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 14:14:13.936 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 14:14:13.936 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 14:14:13.936 | INFO | High Water Breakdown: | 20260505 14:14:13.936 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:14:13.936 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:14:13.936 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:14:13.936 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:14:13.936 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:14:13.936 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:14:13.936 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 14:14:13.936 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 14:14:13.936 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:14:13.936 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 14:14:13.936 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 14:14:13.936 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 14:14:13.936 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 14:14:13.936 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 14:14:13.936 | INFO | ------------------------------------------ | 20260505 14:14:13.936 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:14:13.936 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 14:14:13.936 | INFO | Total mem usage: 25.6 GiB | 20260505 14:14:13.936 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 14:14:13.936 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 14:14:13.936 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 14:14:13.936 | INFO | High Water Breakdown: | 20260505 14:14:13.936 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 14:14:13.936 | INFO | ------------------------------------------ | 20260505 14:14:13.936 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:14:13.936 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 14:14:13.936 | INFO | Total mem usage: 15.7 GiB | 20260505 14:14:13.936 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 14:14:13.936 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 14:14:13.936 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 14:14:13.936 | INFO | High Water Breakdown: | 20260505 14:14:13.936 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 14:14:13.936 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 14:14:13.936 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 14:14:13.936 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 14:14:13.936 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 14:14:13.936 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 14:14:13.936 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 14:14:13.936 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 14:14:13.936 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 14:14:13.936 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:14:13.936 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 14:14:13.936 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:14:13.936 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 14:14:13.936 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 14:14:13.936 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 14:14:13.936 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 14:14:13.936 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:14:13.936 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 14:14:13.937 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:14:13.937 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:14:13.937 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:14:13.937 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 14:14:13.937 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 14:14:13.937 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:14:13.937 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 14:14:13.937 | INFO | ------------------------------------------ | 20260505 14:14:13.937 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:14:13.937 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 14:14:13.937 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:14:13.937 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:14:13.937 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.67e+01, 1.38e+01 / 2.02e+00 | 20260505 14:14:13.937 | INFO | PerfReport: 0.29%, AggregateStats -- 9.27e-02 / 1.01e-01, 1.09e-02 / 3.63e-03 | 20260505 14:14:13.937 | INFO | PerfReport: 11.77%, Baseline -- 3.75e+00 / 4.51e+00, 5.85e-01 / 1.08e-01 | 20260505 14:14:13.937 | INFO | PerfReport: 3.29%, Histogram -- 1.05e+00 / 1.51e+00, 2.00e-01 / 7.92e-02 | 20260505 14:14:13.937 | INFO | PerfReport: 16.04%, DME -- 4.08e+01 / 4.61e+01, 4.84e+00 / 2.18e+00 | 20260505 14:14:13.937 | INFO | PerfReport: 56.36%, FrameLabeling -- 1.79e+01 / 1.96e+01, 1.36e+00 / 1.43e+00 | 20260505 14:14:13.937 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.37e+00 / 2.96e+00, 4.50e-01 / 1.33e-01 | 20260505 14:14:13.937 | INFO | PerfReport: 3.55%, Metrics -- 1.13e+00 / 1.47e+00, 3.19e-01 / 1.81e-01 | 20260505 14:14:13.937 | INFO | PerfReport: 1.24%, PulseSerializing -- 3.95e-01 / 4.47e-01, 6.93e-02 / 3.42e-02 | 20260505 14:14:13.937 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 14:14:13.937 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:14:13.937 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:14:13.937 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260505 14:14:13.937 | INFO | PerfReport: 22.58%, Upload -- 2.28e+01 / 0.00e+00, 8.08e+00 / 0.00e+00 | 20260505 14:14:13.937 | INFO | PerfReport: 22.02%, Download -- 2.22e+01 / 0.00e+00, 8.10e+00 / 0.00e+00 | 20260505 14:14:13.937 | INFO | PerfReport: 31.57%, Compute -- 3.18e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 14:14:13.937 | INFO | PerfReport: 2.28%, LockUpload -- 2.30e+00 / 0.00e+00, 7.89e+00 / 0.00e+00 | 20260505 14:14:13.937 | INFO | PerfReport: 1.38%, LockDownload -- 1.39e+00 / 0.00e+00, 2.87e+00 / 0.00e+00 | 20260505 14:14:13.937 | INFO | PerfReport: 20.16%, LockCompute -- 2.03e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 14:14:13.937 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.73e-02 / 0.00e+00 | 20260505 14:14:14.076 | INFO | Starting a round of Baz IO | 20260505 14:14:17.302 | INFO | Populating RTMetrics report | 20260505 14:14:17.302 | INFO | RTMetrics report populated | 20260505 14:14:17.302 | INFO | Opening tmp RTMetrics json file | 20260505 14:14:17.314 | INFO | Serializing RTMetrics to json | 20260505 14:14:17.341 | INFO | Serialization done, swapping tmp file now... | 20260505 14:14:17.361 | INFO | RTMetrics done | 20260505 14:14:17.407 | INFO | Finished a round of Baz IO: Wrote 2.266 GiB in 3.331 seconds | 20260505 14:14:17.408 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.83% / 0.00% / 77.17% | 20260505 14:14:17.408 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.588846% / 1.19209e-05% / 0.000119209% | 20260505 14:14:17.408 | INFO | Baz IO overall: | 20260505 14:14:17.408 | INFO | BytesWritten : 2289151074522 | 20260505 14:14:17.408 | INFO | NumEvents : 1088534889177 | 20260505 14:14:19.086 | INFO | Analyzing chunk frames = [3842048,3842560) | 20260505 14:14:19.086 | INFO | MemInfo: { | 20260505 14:14:19.086 | INFO | "gpu" : " 17 GB", | 20260505 14:14:19.086 | INFO | "huge" : "", | 20260505 14:14:19.086 | INFO | "huge_shared" : " 27 GB", | 20260505 14:14:19.086 | INFO | "malloc" : " 14 GB", | 20260505 14:14:19.086 | INFO | "pinned" : " 29 GB" | 20260505 14:14:19.086 | INFO | } | 20260505 14:14:19.087 | INFO | Perf report for the last 5.15103 seconds: | 20260505 14:14:19.087 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:14:19.087 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:19.087 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:19.087 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:19.087 | INFO | ANALYSIS : 64.3%, 3.0, 100.2ms, 0 | 20260505 14:14:19.087 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:19.087 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260505 14:14:19.087 | INFO | PRE_HQ : 60.7%, 1.8, 57.3ms, 0 | 20260505 14:14:19.087 | INFO | BAZWRITER : 64.1%, 4.0, 131.2ms, 0 | 20260505 14:14:19.087 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 14:14:19.087 | INFO | Uploaded 36GiB to GPU in 2.18962s (16.4412GiB/s) | 20260505 14:14:19.087 | INFO | Downloaded 38GiB to GPU in 2.2759s (16.6967GiB/s) | 20260505 14:14:24.219 | INFO | Analyzing chunk frames = [3842560,3843072) | 20260505 14:14:24.219 | INFO | MemInfo: { | 20260505 14:14:24.220 | INFO | "gpu" : " 17 GB", | 20260505 14:14:24.220 | INFO | "huge" : "", | 20260505 14:14:24.220 | INFO | "huge_shared" : " 27 GB", | 20260505 14:14:24.220 | INFO | "malloc" : " 14 GB", | 20260505 14:14:24.220 | INFO | "pinned" : " 29 GB" | 20260505 14:14:24.220 | INFO | } | 20260505 14:14:24.220 | INFO | Perf report for the last 5.13311 seconds: | 20260505 14:14:24.220 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:14:24.220 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:24.220 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:24.220 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:24.220 | INFO | ANALYSIS : 63.6%, 3.0, 99.1ms, 0 | 20260505 14:14:24.220 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:24.220 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:24.220 | INFO | PRE_HQ : 28.1%, 1.0, 14.5ms, 0 | 20260505 14:14:24.220 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:24.220 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:24.220 | INFO | Uploaded 36GiB to GPU in 2.19681s (16.3874GiB/s) | 20260505 14:14:24.220 | INFO | Downloaded 34GiB to GPU in 2.04186s (16.6514GiB/s) | 20260505 14:14:29.360 | INFO | Analyzing chunk frames = [3843072,3843584) | 20260505 14:14:29.360 | INFO | MemInfo: { | 20260505 14:14:29.360 | INFO | "gpu" : " 17 GB", | 20260505 14:14:29.362 | INFO | "huge" : "", | 20260505 14:14:29.362 | INFO | "huge_shared" : " 27 GB", | 20260505 14:14:29.362 | INFO | "malloc" : " 14 GB", | 20260505 14:14:29.362 | INFO | "pinned" : " 29 GB" | 20260505 14:14:29.362 | INFO | } | 20260505 14:14:29.362 | INFO | Perf report for the last 5.14195 seconds: | 20260505 14:14:29.362 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:14:29.362 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:29.362 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:29.362 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:29.362 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 14:14:29.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:29.362 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:29.362 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:14:29.362 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:29.362 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:29.362 | INFO | Uploaded 36GiB to GPU in 2.23116s (16.1351GiB/s) | 20260505 14:14:29.362 | INFO | Downloaded 34GiB to GPU in 2.12508s (15.9994GiB/s) | 20260505 14:14:34.492 | INFO | Analyzing chunk frames = [3843584,3844096) | 20260505 14:14:34.492 | INFO | MemInfo: { | 20260505 14:14:34.492 | INFO | "gpu" : " 17 GB", | 20260505 14:14:34.492 | INFO | "huge" : "", | 20260505 14:14:34.492 | INFO | "huge_shared" : " 27 GB", | 20260505 14:14:34.492 | INFO | "malloc" : " 14 GB", | 20260505 14:14:34.492 | INFO | "pinned" : " 29 GB" | 20260505 14:14:34.492 | INFO | } | 20260505 14:14:34.492 | INFO | Perf report for the last 5.13061 seconds: | 20260505 14:14:34.492 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:14:34.492 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:34.492 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:34.492 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:34.492 | INFO | ANALYSIS : 63.0%, 3.0, 97.7ms, 0 | 20260505 14:14:34.492 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:34.492 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:34.492 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:14:34.492 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:34.492 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:34.492 | INFO | Uploaded 36GiB to GPU in 2.19936s (16.3684GiB/s) | 20260505 14:14:34.493 | INFO | Downloaded 34GiB to GPU in 2.13912s (15.8944GiB/s) | 20260505 14:14:39.631 | INFO | Analyzing chunk frames = [3844096,3844608) | 20260505 14:14:39.632 | INFO | MemInfo: { | 20260505 14:14:39.632 | INFO | "gpu" : " 17 GB", | 20260505 14:14:39.632 | INFO | "huge" : "", | 20260505 14:14:39.632 | INFO | "huge_shared" : " 27 GB", | 20260505 14:14:39.632 | INFO | "malloc" : " 14 GB", | 20260505 14:14:39.632 | INFO | "pinned" : " 29 GB" | 20260505 14:14:39.632 | INFO | } | 20260505 14:14:39.632 | INFO | Perf report for the last 5.13941 seconds: | 20260505 14:14:39.632 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:14:39.632 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:39.632 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:39.632 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:39.632 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 14:14:39.632 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:39.632 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:39.632 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 14:14:39.632 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:39.632 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:39.632 | INFO | Uploaded 36GiB to GPU in 2.17708s (16.5359GiB/s) | 20260505 14:14:39.632 | INFO | Downloaded 34GiB to GPU in 2.12926s (15.968GiB/s) | 20260505 14:14:44.784 | INFO | Analyzing chunk frames = [3844608,3845120) | 20260505 14:14:44.784 | INFO | MemInfo: { | 20260505 14:14:44.784 | INFO | "gpu" : " 17 GB", | 20260505 14:14:44.784 | INFO | "huge" : "", | 20260505 14:14:44.784 | INFO | "huge_shared" : " 27 GB", | 20260505 14:14:44.784 | INFO | "malloc" : " 14 GB", | 20260505 14:14:44.784 | INFO | "pinned" : " 29 GB" | 20260505 14:14:44.784 | INFO | } | 20260505 14:14:44.784 | INFO | Perf report for the last 5.15263 seconds: | 20260505 14:14:44.784 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:14:44.784 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:44.784 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:44.784 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:44.784 | INFO | ANALYSIS : 62.9%, 3.0, 98.5ms, 0 | 20260505 14:14:44.784 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:44.784 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:44.784 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 14:14:44.784 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:44.784 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:44.784 | INFO | Uploaded 36GiB to GPU in 2.15212s (16.7277GiB/s) | 20260505 14:14:44.785 | INFO | Downloaded 34GiB to GPU in 2.10456s (16.1554GiB/s) | 20260505 14:14:49.922 | INFO | Analyzing chunk frames = [3845120,3845632) | 20260505 14:14:49.922 | INFO | MemInfo: { | 20260505 14:14:49.922 | INFO | "gpu" : " 17 GB", | 20260505 14:14:49.922 | INFO | "huge" : "", | 20260505 14:14:49.922 | INFO | "huge_shared" : " 27 GB", | 20260505 14:14:49.922 | INFO | "malloc" : " 14 GB", | 20260505 14:14:49.922 | INFO | "pinned" : " 29 GB" | 20260505 14:14:49.922 | INFO | } | 20260505 14:14:49.923 | INFO | Perf report for the last 5.13821 seconds: | 20260505 14:14:49.923 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:14:49.923 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:49.923 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:49.923 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:49.923 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 14:14:49.923 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:49.923 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:49.923 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 14:14:49.923 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:49.923 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:49.923 | INFO | Uploaded 36GiB to GPU in 2.135s (16.8618GiB/s) | 20260505 14:14:49.923 | INFO | Downloaded 34GiB to GPU in 2.17106s (15.6605GiB/s) | 20260505 14:14:55.054 | INFO | Analyzing chunk frames = [3845632,3846144) | 20260505 14:14:55.054 | INFO | MemInfo: { | 20260505 14:14:55.054 | INFO | "gpu" : " 17 GB", | 20260505 14:14:55.054 | INFO | "huge" : "", | 20260505 14:14:55.054 | INFO | "huge_shared" : " 27 GB", | 20260505 14:14:55.054 | INFO | "malloc" : " 14 GB", | 20260505 14:14:55.054 | INFO | "pinned" : " 29 GB" | 20260505 14:14:55.054 | INFO | } | 20260505 14:14:55.054 | INFO | Perf report for the last 5.13165 seconds: | 20260505 14:14:55.054 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:14:55.054 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:55.054 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:55.054 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:55.054 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 14:14:55.054 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:55.054 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:55.054 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:14:55.054 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:55.054 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:14:55.054 | INFO | Uploaded 36GiB to GPU in 2.215s (16.2528GiB/s) | 20260505 14:14:55.054 | INFO | Downloaded 34GiB to GPU in 2.10218s (16.1737GiB/s) | 20260505 14:14:55.196 | INFO | Starting a round of Baz IO | 20260505 14:14:58.415 | INFO | Populating RTMetrics report | 20260505 14:14:58.416 | INFO | RTMetrics report populated | 20260505 14:14:58.416 | INFO | Opening tmp RTMetrics json file | 20260505 14:14:58.433 | INFO | Serializing RTMetrics to json | 20260505 14:14:58.455 | INFO | Serialization done, swapping tmp file now... | 20260505 14:14:58.471 | INFO | RTMetrics done | 20260505 14:14:58.514 | INFO | Finished a round of Baz IO: Wrote 2.265 GiB in 3.318 seconds | 20260505 14:14:58.514 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.82% / 0.00% / 77.17% | 20260505 14:14:58.514 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.584618% / 1.58946e-05% / 9.93411e-05% | 20260505 14:14:58.514 | INFO | Baz IO overall: | 20260505 14:14:58.514 | INFO | BytesWritten : 2291582705835 | 20260505 14:14:58.514 | INFO | NumEvents : 1089682518572 | 20260505 14:15:00.185 | INFO | Analyzing chunk frames = [3846144,3846656) | 20260505 14:15:00.185 | INFO | MemInfo: { | 20260505 14:15:00.185 | INFO | "gpu" : " 17 GB", | 20260505 14:15:00.185 | INFO | "huge" : "", | 20260505 14:15:00.185 | INFO | "huge_shared" : " 27 GB", | 20260505 14:15:00.185 | INFO | "malloc" : " 14 GB", | 20260505 14:15:00.185 | INFO | "pinned" : " 29 GB" | 20260505 14:15:00.185 | INFO | } | 20260505 14:15:00.185 | INFO | Perf report for the last 5.1309 seconds: | 20260505 14:15:00.185 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:15:00.185 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:00.185 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:00.185 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:00.185 | INFO | ANALYSIS : 64.4%, 3.0, 100.0ms, 0 | 20260505 14:15:00.185 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:00.185 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 14:15:00.185 | INFO | PRE_HQ : 60.7%, 1.8, 57.4ms, 0 | 20260505 14:15:00.185 | INFO | BAZWRITER : 64.0%, 4.0, 129.9ms, 0 | 20260505 14:15:00.185 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 14:15:00.185 | INFO | Uploaded 36GiB to GPU in 2.1773s (16.5342GiB/s) | 20260505 14:15:00.185 | INFO | Downloaded 38GiB to GPU in 2.28232s (16.6497GiB/s) | 20260505 14:15:05.333 | INFO | Analyzing chunk frames = [3846656,3847168) | 20260505 14:15:05.333 | INFO | MemInfo: { | 20260505 14:15:05.333 | INFO | "gpu" : " 17 GB", | 20260505 14:15:05.333 | INFO | "huge" : "", | 20260505 14:15:05.333 | INFO | "huge_shared" : " 27 GB", | 20260505 14:15:05.333 | INFO | "malloc" : " 14 GB", | 20260505 14:15:05.333 | INFO | "pinned" : " 29 GB" | 20260505 14:15:05.333 | INFO | } | 20260505 14:15:05.333 | INFO | Perf report for the last 5.14805 seconds: | 20260505 14:15:05.333 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:15:05.333 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:05.333 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:05.333 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:05.333 | INFO | ANALYSIS : 63.6%, 3.0, 99.5ms, 0 | 20260505 14:15:05.333 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:05.333 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:05.333 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 14:15:05.333 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:05.333 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:05.333 | INFO | Uploaded 36GiB to GPU in 2.18932s (16.4435GiB/s) | 20260505 14:15:05.333 | INFO | Downloaded 34GiB to GPU in 2.05216s (16.5679GiB/s) | 20260505 14:15:10.475 | INFO | Analyzing chunk frames = [3847168,3847680) | 20260505 14:15:10.475 | INFO | MemInfo: { | 20260505 14:15:10.475 | INFO | "gpu" : " 17 GB", | 20260505 14:15:10.475 | INFO | "huge" : "", | 20260505 14:15:10.475 | INFO | "huge_shared" : " 27 GB", | 20260505 14:15:10.475 | INFO | "malloc" : " 14 GB", | 20260505 14:15:10.475 | INFO | "pinned" : " 29 GB" | 20260505 14:15:10.475 | INFO | } | 20260505 14:15:10.475 | INFO | Perf report for the last 5.14228 seconds: | 20260505 14:15:10.475 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:15:10.475 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:10.475 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:10.475 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:10.476 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 14:15:10.476 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:10.476 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:10.476 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:15:10.476 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:10.476 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:10.476 | INFO | Uploaded 36GiB to GPU in 2.23487s (16.1084GiB/s) | 20260505 14:15:10.476 | INFO | Downloaded 34GiB to GPU in 2.11873s (16.0473GiB/s) | 20260505 14:15:15.622 | INFO | Analyzing chunk frames = [3847680,3848192) | 20260505 14:15:15.622 | INFO | MemInfo: { | 20260505 14:15:15.622 | INFO | "gpu" : " 17 GB", | 20260505 14:15:15.622 | INFO | "huge" : "", | 20260505 14:15:15.622 | INFO | "huge_shared" : " 27 GB", | 20260505 14:15:15.622 | INFO | "malloc" : " 14 GB", | 20260505 14:15:15.622 | INFO | "pinned" : " 29 GB" | 20260505 14:15:15.622 | INFO | } | 20260505 14:15:15.622 | INFO | Perf report for the last 5.14653 seconds: | 20260505 14:15:15.622 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:15:15.622 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:15.622 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:15.622 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:15.622 | INFO | ANALYSIS : 62.7%, 3.0, 97.5ms, 0 | 20260505 14:15:15.622 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:15.622 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:15.622 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 14:15:15.622 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:15.622 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:15.622 | INFO | Uploaded 36GiB to GPU in 2.18422s (16.4819GiB/s) | 20260505 14:15:15.622 | INFO | Downloaded 34GiB to GPU in 2.12835s (15.9748GiB/s) | 20260505 14:15:20.754 | INFO | Analyzing chunk frames = [3848192,3848704) | 20260505 14:15:20.754 | INFO | MemInfo: { | 20260505 14:15:20.754 | INFO | "gpu" : " 17 GB", | 20260505 14:15:20.754 | INFO | "huge" : "", | 20260505 14:15:20.754 | INFO | "huge_shared" : " 27 GB", | 20260505 14:15:20.754 | INFO | "malloc" : " 14 GB", | 20260505 14:15:20.754 | INFO | "pinned" : " 29 GB" | 20260505 14:15:20.754 | INFO | } | 20260505 14:15:20.754 | INFO | Perf report for the last 5.13237 seconds: | 20260505 14:15:20.754 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:15:20.754 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:20.754 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:20.754 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:20.754 | INFO | ANALYSIS : 62.9%, 3.0, 98.0ms, 0 | 20260505 14:15:20.754 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:20.754 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:20.754 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:15:20.755 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:20.755 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:20.755 | INFO | Uploaded 36GiB to GPU in 2.17393s (16.5599GiB/s) | 20260505 14:15:20.755 | INFO | Downloaded 34GiB to GPU in 2.13055s (15.9584GiB/s) | 20260505 14:15:25.888 | INFO | Analyzing chunk frames = [3848704,3849216) | 20260505 14:15:25.888 | INFO | MemInfo: { | 20260505 14:15:25.888 | INFO | "gpu" : " 17 GB", | 20260505 14:15:25.888 | INFO | "huge" : "", | 20260505 14:15:25.888 | INFO | "huge_shared" : " 27 GB", | 20260505 14:15:25.888 | INFO | "malloc" : " 14 GB", | 20260505 14:15:25.888 | INFO | "pinned" : " 29 GB" | 20260505 14:15:25.888 | INFO | } | 20260505 14:15:25.888 | INFO | Perf report for the last 5.1336 seconds: | 20260505 14:15:25.888 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:15:25.888 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:25.888 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:25.888 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:25.888 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 14:15:25.888 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:25.888 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:25.888 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:15:25.888 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:25.888 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:25.888 | INFO | Uploaded 36GiB to GPU in 2.20972s (16.2917GiB/s) | 20260505 14:15:25.888 | INFO | Downloaded 34GiB to GPU in 2.11043s (16.1104GiB/s) | 20260505 14:15:31.036 | INFO | Analyzing chunk frames = [3849216,3849728) | 20260505 14:15:31.036 | INFO | MemInfo: { | 20260505 14:15:31.036 | INFO | "gpu" : " 17 GB", | 20260505 14:15:31.036 | INFO | "huge" : "", | 20260505 14:15:31.036 | INFO | "huge_shared" : " 27 GB", | 20260505 14:15:31.036 | INFO | "malloc" : " 14 GB", | 20260505 14:15:31.036 | INFO | "pinned" : " 29 GB" | 20260505 14:15:31.036 | INFO | } | 20260505 14:15:31.036 | INFO | Perf report for the last 5.14777 seconds: | 20260505 14:15:31.036 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:15:31.036 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:31.036 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:31.036 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:31.036 | INFO | ANALYSIS : 63.3%, 3.0, 99.0ms, 0 | 20260505 14:15:31.036 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:31.036 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:31.036 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 14:15:31.036 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:31.036 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:31.036 | INFO | Uploaded 36GiB to GPU in 2.18117s (16.5049GiB/s) | 20260505 14:15:31.036 | INFO | Downloaded 34GiB to GPU in 2.15124s (15.8048GiB/s) | 20260505 14:15:36.175 | INFO | Analyzing chunk frames = [3849728,3850240) | 20260505 14:15:36.175 | INFO | MemInfo: { | 20260505 14:15:36.175 | INFO | "gpu" : " 17 GB", | 20260505 14:15:36.175 | INFO | "huge" : "", | 20260505 14:15:36.175 | INFO | "huge_shared" : " 27 GB", | 20260505 14:15:36.175 | INFO | "malloc" : " 14 GB", | 20260505 14:15:36.175 | INFO | "pinned" : " 29 GB" | 20260505 14:15:36.175 | INFO | } | 20260505 14:15:36.175 | INFO | Perf report for the last 5.13966 seconds: | 20260505 14:15:36.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:15:36.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:36.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:36.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:36.176 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 14:15:36.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:36.176 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:36.176 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:15:36.176 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:36.176 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:36.176 | INFO | Uploaded 36GiB to GPU in 2.21478s (16.2545GiB/s) | 20260505 14:15:36.176 | INFO | Downloaded 34GiB to GPU in 2.09793s (16.2064GiB/s) | 20260505 14:15:36.176 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 14:15:36.176 | INFO | Total mem usage: 15.3 GiB | 20260505 14:15:36.176 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 14:15:36.176 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.68%) | 20260505 14:15:36.176 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 14:15:36.176 | INFO | High Water Breakdown: | 20260505 14:15:36.176 | INFO | 61% : 8.68 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 14:15:36.176 | INFO | 38.9% : 5.54 GiB ( 83.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 14:15:36.176 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 14:15:36.176 | INFO | ------------------------------------------ | 20260505 14:15:36.176 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:15:36.176 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 14:15:36.176 | INFO | Total mem usage: 26.5 GiB | 20260505 14:15:36.176 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 14:15:36.176 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 14:15:36.176 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 14:15:36.176 | INFO | High Water Breakdown: | 20260505 14:15:36.176 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:15:36.176 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:15:36.176 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:15:36.176 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:15:36.176 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:15:36.176 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:15:36.176 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 14:15:36.176 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 14:15:36.176 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:15:36.176 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 14:15:36.176 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 14:15:36.176 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 14:15:36.177 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 14:15:36.177 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 14:15:36.177 | INFO | ------------------------------------------ | 20260505 14:15:36.177 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:15:36.177 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 14:15:36.177 | INFO | Total mem usage: 25.6 GiB | 20260505 14:15:36.177 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 14:15:36.177 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 14:15:36.177 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 14:15:36.177 | INFO | High Water Breakdown: | 20260505 14:15:36.177 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 14:15:36.177 | INFO | ------------------------------------------ | 20260505 14:15:36.177 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:15:36.177 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 14:15:36.177 | INFO | Total mem usage: 15.7 GiB | 20260505 14:15:36.177 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 14:15:36.177 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 14:15:36.177 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260505 14:15:36.177 | INFO | High Water Breakdown: | 20260505 14:15:36.177 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 14:15:36.177 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 14:15:36.177 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 14:15:36.177 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 14:15:36.177 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 14:15:36.177 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 14:15:36.177 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 14:15:36.177 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 14:15:36.177 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 14:15:36.177 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:15:36.177 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 14:15:36.177 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:15:36.177 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 14:15:36.177 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 14:15:36.177 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 14:15:36.177 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 14:15:36.177 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:15:36.177 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 14:15:36.177 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:15:36.177 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:15:36.177 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:15:36.177 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 14:15:36.177 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 14:15:36.177 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:15:36.177 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 14:15:36.177 | INFO | ------------------------------------------ | 20260505 14:15:36.177 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:15:36.177 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 14:15:36.177 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:15:36.177 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:15:36.177 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.64e+01, 1.38e+01 / 1.90e+00 | 20260505 14:15:36.177 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 9.92e-02, 1.08e-02 / 4.10e-03 | 20260505 14:15:36.177 | INFO | PerfReport: 11.74%, Baseline -- 3.74e+00 / 4.49e+00, 5.73e-01 / 1.81e-01 | 20260505 14:15:36.177 | INFO | PerfReport: 3.29%, Histogram -- 1.05e+00 / 1.49e+00, 1.95e-01 / 7.48e-02 | 20260505 14:15:36.177 | INFO | PerfReport: 15.99%, DME -- 4.07e+01 / 4.56e+01, 4.67e+00 / 2.02e+00 | 20260505 14:15:36.177 | INFO | PerfReport: 56.49%, FrameLabeling -- 1.80e+01 / 1.99e+01, 1.44e+00 / 2.11e+00 | 20260505 14:15:36.177 | INFO | PerfReport: 7.44%, PulseAccumulating -- 2.37e+00 / 2.92e+00, 4.50e-01 / 1.71e-01 | 20260505 14:15:36.177 | INFO | PerfReport: 3.53%, Metrics -- 1.12e+00 / 1.43e+00, 3.12e-01 / 1.66e-01 | 20260505 14:15:36.177 | INFO | PerfReport: 1.24%, PulseSerializing -- 3.94e-01 / 4.43e-01, 7.03e-02 / 2.47e-02 | 20260505 14:15:36.177 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 14:15:36.177 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:15:36.177 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:15:36.177 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260505 14:15:36.177 | INFO | PerfReport: 22.62%, Upload -- 2.28e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260505 14:15:36.177 | INFO | PerfReport: 22.04%, Download -- 2.22e+01 / 0.00e+00, 8.10e+00 / 0.00e+00 | 20260505 14:15:36.177 | INFO | PerfReport: 31.55%, Compute -- 3.18e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 14:15:36.177 | INFO | PerfReport: 2.34%, LockUpload -- 2.36e+00 / 0.00e+00, 7.98e+00 / 0.00e+00 | 20260505 14:15:36.177 | INFO | PerfReport: 1.33%, LockDownload -- 1.35e+00 / 0.00e+00, 2.85e+00 / 0.00e+00 | 20260505 14:15:36.177 | INFO | PerfReport: 20.10%, LockCompute -- 2.03e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260505 14:15:36.177 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.84e-02 / 0.00e+00 | 20260505 14:15:36.318 | INFO | Starting a round of Baz IO | 20260505 14:15:39.529 | INFO | Populating RTMetrics report | 20260505 14:15:39.529 | INFO | RTMetrics report populated | 20260505 14:15:39.529 | INFO | Opening tmp RTMetrics json file | 20260505 14:15:39.538 | INFO | Serializing RTMetrics to json | 20260505 14:15:39.573 | INFO | Serialization done, swapping tmp file now... | 20260505 14:15:39.596 | INFO | RTMetrics done | 20260505 14:15:39.633 | INFO | Finished a round of Baz IO: Wrote 2.266 GiB in 3.315 seconds | 20260505 14:15:39.633 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.82% / 0.00% / 77.18% | 20260505 14:15:39.633 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.585461% / 0% / 9.13938e-05% | 20260505 14:15:39.633 | INFO | Baz IO overall: | 20260505 14:15:39.633 | INFO | BytesWritten : 2294015660116 | 20260505 14:15:39.633 | INFO | NumEvents : 1090830965863 | 20260505 14:15:41.314 | INFO | Analyzing chunk frames = [3850240,3850752) | 20260505 14:15:41.314 | INFO | MemInfo: { | 20260505 14:15:41.314 | INFO | "gpu" : " 17 GB", | 20260505 14:15:41.314 | INFO | "huge" : "", | 20260505 14:15:41.314 | INFO | "huge_shared" : " 27 GB", | 20260505 14:15:41.314 | INFO | "malloc" : " 14 GB", | 20260505 14:15:41.314 | INFO | "pinned" : " 29 GB" | 20260505 14:15:41.314 | INFO | } | 20260505 14:15:41.314 | INFO | Perf report for the last 5.13839 seconds: | 20260505 14:15:41.314 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:15:41.314 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:41.314 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:41.314 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:41.314 | INFO | ANALYSIS : 64.1%, 3.0, 99.4ms, 0 | 20260505 14:15:41.314 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:41.314 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260505 14:15:41.314 | INFO | PRE_HQ : 61.1%, 1.8, 57.7ms, 0 | 20260505 14:15:41.314 | INFO | BAZWRITER : 64.3%, 4.0, 131.6ms, 0 | 20260505 14:15:41.314 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260505 14:15:41.314 | INFO | Uploaded 36GiB to GPU in 2.20419s (16.3325GiB/s) | 20260505 14:15:41.314 | INFO | Downloaded 38GiB to GPU in 2.26008s (16.8136GiB/s) | 20260505 14:15:46.445 | INFO | Analyzing chunk frames = [3850752,3851264) | 20260505 14:15:46.445 | INFO | MemInfo: { | 20260505 14:15:46.445 | INFO | "gpu" : " 17 GB", | 20260505 14:15:46.445 | INFO | "huge" : "", | 20260505 14:15:46.445 | INFO | "huge_shared" : " 27 GB", | 20260505 14:15:46.445 | INFO | "malloc" : " 14 GB", | 20260505 14:15:46.445 | INFO | "pinned" : " 29 GB" | 20260505 14:15:46.445 | INFO | } | 20260505 14:15:46.446 | INFO | Perf report for the last 5.13165 seconds: | 20260505 14:15:46.446 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:15:46.446 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:46.446 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:46.446 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:46.446 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 14:15:46.446 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:46.446 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:46.446 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 14:15:46.446 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:46.446 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:46.446 | INFO | Uploaded 36GiB to GPU in 2.20177s (16.3505GiB/s) | 20260505 14:15:46.446 | INFO | Downloaded 34GiB to GPU in 2.03738s (16.6881GiB/s) | 20260505 14:15:51.590 | INFO | Analyzing chunk frames = [3851264,3851776) | 20260505 14:15:51.590 | INFO | MemInfo: { | 20260505 14:15:51.590 | INFO | "gpu" : " 17 GB", | 20260505 14:15:51.590 | INFO | "huge" : "", | 20260505 14:15:51.590 | INFO | "huge_shared" : " 27 GB", | 20260505 14:15:51.590 | INFO | "malloc" : " 14 GB", | 20260505 14:15:51.590 | INFO | "pinned" : " 29 GB" | 20260505 14:15:51.590 | INFO | } | 20260505 14:15:51.590 | INFO | Perf report for the last 5.14447 seconds: | 20260505 14:15:51.590 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:15:51.590 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:51.590 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:51.590 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:51.590 | INFO | ANALYSIS : 63.1%, 3.0, 98.3ms, 0 | 20260505 14:15:51.590 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:51.590 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:51.590 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 14:15:51.590 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:51.590 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:51.590 | INFO | Uploaded 36GiB to GPU in 2.20225s (16.3469GiB/s) | 20260505 14:15:51.590 | INFO | Downloaded 34GiB to GPU in 2.13082s (15.9563GiB/s) | 20260505 14:15:56.732 | INFO | Analyzing chunk frames = [3851776,3852288) | 20260505 14:15:56.732 | INFO | MemInfo: { | 20260505 14:15:56.732 | INFO | "gpu" : " 17 GB", | 20260505 14:15:56.732 | INFO | "huge" : "", | 20260505 14:15:56.732 | INFO | "huge_shared" : " 27 GB", | 20260505 14:15:56.732 | INFO | "malloc" : " 14 GB", | 20260505 14:15:56.732 | INFO | "pinned" : " 29 GB" | 20260505 14:15:56.732 | INFO | } | 20260505 14:15:56.732 | INFO | Perf report for the last 5.14239 seconds: | 20260505 14:15:56.732 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:15:56.732 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:56.732 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:56.732 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:56.732 | INFO | ANALYSIS : 62.5%, 3.0, 97.1ms, 0 | 20260505 14:15:56.732 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:56.732 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:56.732 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:15:56.732 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:56.732 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:15:56.733 | INFO | Uploaded 36GiB to GPU in 2.17049s (16.5861GiB/s) | 20260505 14:15:56.733 | INFO | Downloaded 34GiB to GPU in 2.12986s (15.9635GiB/s) | 20260505 14:16:01.870 | INFO | Analyzing chunk frames = [3852288,3852800) | 20260505 14:16:01.870 | INFO | MemInfo: { | 20260505 14:16:01.870 | INFO | "gpu" : " 17 GB", | 20260505 14:16:01.870 | INFO | "huge" : "", | 20260505 14:16:01.870 | INFO | "huge_shared" : " 27 GB", | 20260505 14:16:01.870 | INFO | "malloc" : " 14 GB", | 20260505 14:16:01.870 | INFO | "pinned" : " 29 GB" | 20260505 14:16:01.870 | INFO | } | 20260505 14:16:01.870 | INFO | Perf report for the last 5.13754 seconds: | 20260505 14:16:01.870 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:16:01.870 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:01.870 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:01.870 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:01.870 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 14:16:01.870 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:01.870 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:01.870 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:16:01.870 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:01.870 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:01.870 | INFO | Uploaded 36GiB to GPU in 2.17452s (16.5554GiB/s) | 20260505 14:16:01.870 | INFO | Downloaded 34GiB to GPU in 2.12993s (15.963GiB/s) | 20260505 14:16:06.999 | INFO | Analyzing chunk frames = [3852800,3853312) | 20260505 14:16:06.999 | INFO | MemInfo: { | 20260505 14:16:06.999 | INFO | "gpu" : " 17 GB", | 20260505 14:16:06.999 | INFO | "huge" : "", | 20260505 14:16:06.999 | INFO | "huge_shared" : " 27 GB", | 20260505 14:16:06.999 | INFO | "malloc" : " 14 GB", | 20260505 14:16:06.999 | INFO | "pinned" : " 29 GB" | 20260505 14:16:06.999 | INFO | } | 20260505 14:16:06.999 | INFO | Perf report for the last 5.12892 seconds: | 20260505 14:16:06.999 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:16:06.999 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:06.999 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:06.999 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:06.999 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 14:16:06.999 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:06.999 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:06.999 | INFO | PRE_HQ : 28.1%, 1.0, 14.5ms, 0 | 20260505 14:16:06.999 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:06.999 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:06.999 | INFO | Uploaded 36GiB to GPU in 2.17961s (16.5167GiB/s) | 20260505 14:16:06.999 | INFO | Downloaded 34GiB to GPU in 2.1113s (16.1038GiB/s) | 20260505 14:16:12.140 | INFO | Analyzing chunk frames = [3853312,3853824) | 20260505 14:16:12.140 | INFO | MemInfo: { | 20260505 14:16:12.140 | INFO | "gpu" : " 17 GB", | 20260505 14:16:12.140 | INFO | "huge" : "", | 20260505 14:16:12.140 | INFO | "huge_shared" : " 27 GB", | 20260505 14:16:12.140 | INFO | "malloc" : " 14 GB", | 20260505 14:16:12.140 | INFO | "pinned" : " 29 GB" | 20260505 14:16:12.140 | INFO | } | 20260505 14:16:12.140 | INFO | Perf report for the last 5.14112 seconds: | 20260505 14:16:12.140 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:16:12.140 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:12.140 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:12.140 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:12.140 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 14:16:12.140 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:12.140 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:12.140 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 14:16:12.140 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:12.140 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:12.140 | INFO | Uploaded 36GiB to GPU in 2.17518s (16.5503GiB/s) | 20260505 14:16:12.140 | INFO | Downloaded 34GiB to GPU in 2.15052s (15.8101GiB/s) | 20260505 14:16:17.292 | INFO | Analyzing chunk frames = [3853824,3854336) | 20260505 14:16:17.292 | INFO | MemInfo: { | 20260505 14:16:17.292 | INFO | "gpu" : " 17 GB", | 20260505 14:16:17.292 | INFO | "huge" : "", | 20260505 14:16:17.292 | INFO | "huge_shared" : " 27 GB", | 20260505 14:16:17.292 | INFO | "malloc" : " 14 GB", | 20260505 14:16:17.292 | INFO | "pinned" : " 29 GB" | 20260505 14:16:17.292 | INFO | } | 20260505 14:16:17.292 | INFO | Perf report for the last 5.15174 seconds: | 20260505 14:16:17.292 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:16:17.292 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:17.292 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:17.292 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:17.292 | INFO | ANALYSIS : 63.4%, 3.0, 99.2ms, 0 | 20260505 14:16:17.292 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:17.292 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:17.292 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:16:17.292 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:17.292 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:17.292 | INFO | Uploaded 36GiB to GPU in 2.20604s (16.3188GiB/s) | 20260505 14:16:17.292 | INFO | Downloaded 34GiB to GPU in 2.10423s (16.158GiB/s) | 20260505 14:16:17.422 | INFO | Starting a round of Baz IO | 20260505 14:16:20.641 | INFO | Populating RTMetrics report | 20260505 14:16:20.641 | INFO | RTMetrics report populated | 20260505 14:16:20.641 | INFO | Opening tmp RTMetrics json file | 20260505 14:16:20.669 | INFO | Serializing RTMetrics to json | 20260505 14:16:20.707 | INFO | Serialization done, swapping tmp file now... | 20260505 14:16:20.733 | INFO | RTMetrics done | 20260505 14:16:20.748 | INFO | Finished a round of Baz IO: Wrote 2.267 GiB in 3.325 seconds | 20260505 14:16:20.748 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.81% / 0.00% / 77.18% | 20260505 14:16:20.748 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.584225% / 0% / 6.75519e-05% | 20260505 14:16:20.748 | INFO | Baz IO overall: | 20260505 14:16:20.748 | INFO | BytesWritten : 2296449556618 | 20260505 14:16:20.748 | INFO | NumEvents : 1091980009449 | 20260505 14:16:22.435 | INFO | Analyzing chunk frames = [3854336,3854848) | 20260505 14:16:22.435 | INFO | MemInfo: { | 20260505 14:16:22.435 | INFO | "gpu" : " 17 GB", | 20260505 14:16:22.435 | INFO | "huge" : "", | 20260505 14:16:22.435 | INFO | "huge_shared" : " 27 GB", | 20260505 14:16:22.435 | INFO | "malloc" : " 14 GB", | 20260505 14:16:22.435 | INFO | "pinned" : " 29 GB" | 20260505 14:16:22.435 | INFO | } | 20260505 14:16:22.435 | INFO | Perf report for the last 5.14326 seconds: | 20260505 14:16:22.435 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:16:22.435 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:22.435 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:22.435 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:22.435 | INFO | ANALYSIS : 64.0%, 3.0, 99.6ms, 0 | 20260505 14:16:22.435 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:22.435 | INFO | RT_METRICS : 46.7%, 1.0, 24.5ms, 0 | 20260505 14:16:22.435 | INFO | PRE_HQ : 61.0%, 1.8, 57.3ms, 0 | 20260505 14:16:22.435 | INFO | BAZWRITER : 64.0%, 4.0, 130.4ms, 0 | 20260505 14:16:22.435 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260505 14:16:22.435 | INFO | Uploaded 36GiB to GPU in 2.19168s (16.4258GiB/s) | 20260505 14:16:22.435 | INFO | Downloaded 38GiB to GPU in 2.2549s (16.8522GiB/s) | 20260505 14:16:27.566 | INFO | Analyzing chunk frames = [3854848,3855360) | 20260505 14:16:27.566 | INFO | MemInfo: { | 20260505 14:16:27.566 | INFO | "gpu" : " 17 GB", | 20260505 14:16:27.566 | INFO | "huge" : "", | 20260505 14:16:27.566 | INFO | "huge_shared" : " 27 GB", | 20260505 14:16:27.566 | INFO | "malloc" : " 14 GB", | 20260505 14:16:27.566 | INFO | "pinned" : " 29 GB" | 20260505 14:16:27.566 | INFO | } | 20260505 14:16:27.566 | INFO | Perf report for the last 5.13096 seconds: | 20260505 14:16:27.566 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:16:27.566 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:27.566 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:27.566 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:27.566 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 14:16:27.566 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:27.566 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:27.566 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:16:27.566 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:27.566 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:27.566 | INFO | Uploaded 36GiB to GPU in 2.20885s (16.2981GiB/s) | 20260505 14:16:27.566 | INFO | Downloaded 34GiB to GPU in 2.06346s (16.4772GiB/s) | 20260505 14:16:32.696 | INFO | Analyzing chunk frames = [3855360,3855872) | 20260505 14:16:32.696 | INFO | MemInfo: { | 20260505 14:16:32.696 | INFO | "gpu" : " 17 GB", | 20260505 14:16:32.696 | INFO | "huge" : "", | 20260505 14:16:32.696 | INFO | "huge_shared" : " 27 GB", | 20260505 14:16:32.696 | INFO | "malloc" : " 14 GB", | 20260505 14:16:32.696 | INFO | "pinned" : " 29 GB" | 20260505 14:16:32.696 | INFO | } | 20260505 14:16:32.696 | INFO | Perf report for the last 5.13021 seconds: | 20260505 14:16:32.696 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:16:32.696 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:32.696 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:32.696 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:32.696 | INFO | ANALYSIS : 63.5%, 3.0, 98.6ms, 0 | 20260505 14:16:32.696 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:32.696 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:32.696 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 14:16:32.696 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:32.696 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:32.696 | INFO | Uploaded 36GiB to GPU in 2.22755s (16.1613GiB/s) | 20260505 14:16:32.697 | INFO | Downloaded 34GiB to GPU in 2.09379s (16.2385GiB/s) | 20260505 14:16:37.844 | INFO | Analyzing chunk frames = [3855872,3856384) | 20260505 14:16:37.844 | INFO | MemInfo: { | 20260505 14:16:37.844 | INFO | "gpu" : " 17 GB", | 20260505 14:16:37.844 | INFO | "huge" : "", | 20260505 14:16:37.844 | INFO | "huge_shared" : " 27 GB", | 20260505 14:16:37.844 | INFO | "malloc" : " 14 GB", | 20260505 14:16:37.844 | INFO | "pinned" : " 29 GB" | 20260505 14:16:37.844 | INFO | } | 20260505 14:16:37.844 | INFO | Perf report for the last 5.14745 seconds: | 20260505 14:16:37.844 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:16:37.844 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:37.844 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:37.844 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:37.844 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260505 14:16:37.844 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:37.844 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:37.844 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 14:16:37.844 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:37.844 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:37.844 | INFO | Uploaded 36GiB to GPU in 2.20241s (16.3457GiB/s) | 20260505 14:16:37.844 | INFO | Downloaded 34GiB to GPU in 2.13578s (15.9193GiB/s) | 20260505 14:16:42.982 | INFO | Analyzing chunk frames = [3856384,3856896) | 20260505 14:16:42.982 | INFO | MemInfo: { | 20260505 14:16:42.982 | INFO | "gpu" : " 17 GB", | 20260505 14:16:42.982 | INFO | "huge" : "", | 20260505 14:16:42.982 | INFO | "huge_shared" : " 27 GB", | 20260505 14:16:42.982 | INFO | "malloc" : " 14 GB", | 20260505 14:16:42.982 | INFO | "pinned" : " 29 GB" | 20260505 14:16:42.982 | INFO | } | 20260505 14:16:42.982 | INFO | Perf report for the last 5.13822 seconds: | 20260505 14:16:42.982 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:16:42.982 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:42.982 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:42.982 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:42.982 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 14:16:42.982 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:42.982 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:42.982 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:16:42.982 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:42.982 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:42.982 | INFO | Uploaded 36GiB to GPU in 2.15076s (16.7382GiB/s) | 20260505 14:16:42.982 | INFO | Downloaded 34GiB to GPU in 2.12597s (15.9927GiB/s) | 20260505 14:16:48.124 | INFO | Analyzing chunk frames = [3856896,3857408) | 20260505 14:16:48.124 | INFO | MemInfo: { | 20260505 14:16:48.124 | INFO | "gpu" : " 17 GB", | 20260505 14:16:48.124 | INFO | "huge" : "", | 20260505 14:16:48.124 | INFO | "huge_shared" : " 27 GB", | 20260505 14:16:48.124 | INFO | "malloc" : " 14 GB", | 20260505 14:16:48.124 | INFO | "pinned" : " 29 GB" | 20260505 14:16:48.124 | INFO | } | 20260505 14:16:48.124 | INFO | Perf report for the last 5.14237 seconds: | 20260505 14:16:48.124 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:16:48.124 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:48.124 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:48.124 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:48.124 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 14:16:48.124 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:48.124 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:48.124 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:16:48.124 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:48.124 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:48.125 | INFO | Uploaded 36GiB to GPU in 2.20667s (16.3142GiB/s) | 20260505 14:16:48.125 | INFO | Downloaded 34GiB to GPU in 2.11308s (16.0903GiB/s) | 20260505 14:16:53.263 | INFO | Analyzing chunk frames = [3857408,3857920) | 20260505 14:16:53.263 | INFO | MemInfo: { | 20260505 14:16:53.263 | INFO | "gpu" : " 17 GB", | 20260505 14:16:53.263 | INFO | "huge" : "", | 20260505 14:16:53.263 | INFO | "huge_shared" : " 27 GB", | 20260505 14:16:53.263 | INFO | "malloc" : " 14 GB", | 20260505 14:16:53.263 | INFO | "pinned" : " 29 GB" | 20260505 14:16:53.263 | INFO | } | 20260505 14:16:53.263 | INFO | Perf report for the last 5.13827 seconds: | 20260505 14:16:53.263 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:16:53.263 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:53.263 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:53.263 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:53.263 | INFO | ANALYSIS : 64.0%, 3.0, 99.9ms, 0 | 20260505 14:16:53.263 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:53.263 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:53.263 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260505 14:16:53.263 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:53.263 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:53.263 | INFO | Uploaded 36GiB to GPU in 2.15077s (16.7382GiB/s) | 20260505 14:16:53.263 | INFO | Downloaded 34GiB to GPU in 2.17514s (15.6312GiB/s) | 20260505 14:16:58.396 | INFO | Analyzing chunk frames = [3857920,3858432) | 20260505 14:16:58.396 | INFO | MemInfo: { | 20260505 14:16:58.396 | INFO | "gpu" : " 17 GB", | 20260505 14:16:58.396 | INFO | "huge" : "", | 20260505 14:16:58.396 | INFO | "huge_shared" : " 27 GB", | 20260505 14:16:58.396 | INFO | "malloc" : " 14 GB", | 20260505 14:16:58.396 | INFO | "pinned" : " 29 GB" | 20260505 14:16:58.396 | INFO | } | 20260505 14:16:58.397 | INFO | Perf report for the last 5.13398 seconds: | 20260505 14:16:58.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:16:58.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:58.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:58.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:58.397 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 14:16:58.397 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:58.397 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:58.397 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 14:16:58.397 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:58.397 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:16:58.397 | INFO | Uploaded 36GiB to GPU in 2.21368s (16.2625GiB/s) | 20260505 14:16:58.397 | INFO | Downloaded 34GiB to GPU in 2.10601s (16.1443GiB/s) | 20260505 14:16:58.397 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 14:16:58.397 | INFO | Total mem usage: 15.3 GiB | 20260505 14:16:58.397 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 14:16:58.397 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -7.14%) | 20260505 14:16:58.397 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 14:16:58.397 | INFO | High Water Breakdown: | 20260505 14:16:58.397 | INFO | 61.1% : 8.74 GiB ( 91.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 14:16:58.397 | INFO | 38.8% : 5.55 GiB ( 83.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 14:16:58.397 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 14:16:58.397 | INFO | ------------------------------------------ | 20260505 14:16:58.397 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:16:58.397 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 14:16:58.397 | INFO | Total mem usage: 26.5 GiB | 20260505 14:16:58.397 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 14:16:58.398 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 14:16:58.398 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 14:16:58.398 | INFO | High Water Breakdown: | 20260505 14:16:58.398 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:16:58.398 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:16:58.398 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:16:58.398 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:16:58.398 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:16:58.398 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:16:58.398 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 14:16:58.398 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 14:16:58.398 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:16:58.398 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 14:16:58.398 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 14:16:58.398 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 14:16:58.398 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 14:16:58.398 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 14:16:58.398 | INFO | ------------------------------------------ | 20260505 14:16:58.398 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:16:58.398 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 14:16:58.398 | INFO | Total mem usage: 25.6 GiB | 20260505 14:16:58.398 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 14:16:58.398 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 14:16:58.398 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 14:16:58.398 | INFO | High Water Breakdown: | 20260505 14:16:58.398 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 14:16:58.398 | INFO | ------------------------------------------ | 20260505 14:16:58.398 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:16:58.398 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 14:16:58.398 | INFO | Total mem usage: 15.7 GiB | 20260505 14:16:58.398 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 14:16:58.398 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 14:16:58.398 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 14:16:58.398 | INFO | High Water Breakdown: | 20260505 14:16:58.398 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 14:16:58.398 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 14:16:58.398 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 14:16:58.398 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 14:16:58.398 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 14:16:58.398 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 14:16:58.398 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 14:16:58.398 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 14:16:58.398 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 14:16:58.398 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:16:58.398 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 14:16:58.398 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:16:58.398 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 14:16:58.398 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 14:16:58.398 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 14:16:58.398 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 14:16:58.398 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:16:58.398 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 14:16:58.398 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:16:58.398 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:16:58.398 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:16:58.398 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 14:16:58.398 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 14:16:58.398 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:16:58.398 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 14:16:58.398 | INFO | ------------------------------------------ | 20260505 14:16:58.398 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:16:58.398 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 14:16:58.398 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:16:58.398 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:16:58.398 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.67e+01, 1.37e+01 / 1.96e+00 | 20260505 14:16:58.398 | INFO | PerfReport: 0.29%, AggregateStats -- 9.31e-02 / 9.99e-02, 1.15e-02 / 4.77e-03 | 20260505 14:16:58.398 | INFO | PerfReport: 11.78%, Baseline -- 3.74e+00 / 4.50e+00, 5.80e-01 / 1.69e-01 | 20260505 14:16:58.398 | INFO | PerfReport: 3.29%, Histogram -- 1.05e+00 / 1.50e+00, 1.98e-01 / 1.10e-01 | 20260505 14:16:58.398 | INFO | PerfReport: 16.05%, DME -- 4.08e+01 / 4.62e+01, 4.80e+00 / 2.22e+00 | 20260505 14:16:58.398 | INFO | PerfReport: 56.29%, FrameLabeling -- 1.79e+01 / 1.96e+01, 1.31e+00 / 1.90e+00 | 20260505 14:16:58.398 | INFO | PerfReport: 7.50%, PulseAccumulating -- 2.38e+00 / 2.92e+00, 4.57e-01 / 1.92e-01 | 20260505 14:16:58.398 | INFO | PerfReport: 3.55%, Metrics -- 1.13e+00 / 1.46e+00, 3.11e-01 / 1.54e-01 | 20260505 14:16:58.399 | INFO | PerfReport: 1.24%, PulseSerializing -- 3.94e-01 / 4.46e-01, 6.90e-02 / 2.63e-02 | 20260505 14:16:58.399 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 14:16:58.399 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:16:58.399 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:16:58.399 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260505 14:16:58.399 | INFO | PerfReport: 22.66%, Upload -- 2.28e+01 / 0.00e+00, 8.15e+00 / 0.00e+00 | 20260505 14:16:58.399 | INFO | PerfReport: 22.05%, Download -- 2.22e+01 / 0.00e+00, 8.09e+00 / 0.00e+00 | 20260505 14:16:58.399 | INFO | PerfReport: 31.55%, Compute -- 3.18e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 14:16:58.399 | INFO | PerfReport: 2.26%, LockUpload -- 2.28e+00 / 0.00e+00, 7.95e+00 / 0.00e+00 | 20260505 14:16:58.399 | INFO | PerfReport: 1.44%, LockDownload -- 1.45e+00 / 0.00e+00, 2.90e+00 / 0.00e+00 | 20260505 14:16:58.399 | INFO | PerfReport: 20.01%, LockCompute -- 2.02e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 14:16:58.399 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.68e-02 / 0.00e+00 | 20260505 14:16:58.531 | INFO | Starting a round of Baz IO | 20260505 14:17:01.766 | INFO | Populating RTMetrics report | 20260505 14:17:01.766 | INFO | RTMetrics report populated | 20260505 14:17:01.766 | INFO | Opening tmp RTMetrics json file | 20260505 14:17:01.776 | INFO | Serializing RTMetrics to json | 20260505 14:17:01.809 | INFO | Serialization done, swapping tmp file now... | 20260505 14:17:01.831 | INFO | RTMetrics done | 20260505 14:17:01.868 | INFO | Finished a round of Baz IO: Wrote 2.264 GiB in 3.337 seconds | 20260505 14:17:01.868 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.81% / 0.00% / 77.19% | 20260505 14:17:01.868 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.581356% / 0% / 6.75519e-05% | 20260505 14:17:01.868 | INFO | Baz IO overall: | 20260505 14:17:01.868 | INFO | BytesWritten : 2298880576736 | 20260505 14:17:01.868 | INFO | NumEvents : 1093127231120 | 20260505 14:17:03.545 | INFO | Analyzing chunk frames = [3858432,3858944) | 20260505 14:17:03.545 | INFO | MemInfo: { | 20260505 14:17:03.545 | INFO | "gpu" : " 17 GB", | 20260505 14:17:03.545 | INFO | "huge" : "", | 20260505 14:17:03.545 | INFO | "huge_shared" : " 27 GB", | 20260505 14:17:03.545 | INFO | "malloc" : " 14 GB", | 20260505 14:17:03.545 | INFO | "pinned" : " 29 GB" | 20260505 14:17:03.545 | INFO | } | 20260505 14:17:03.545 | INFO | Perf report for the last 5.14868 seconds: | 20260505 14:17:03.545 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:17:03.545 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:03.545 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:03.545 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:03.545 | INFO | ANALYSIS : 64.3%, 3.0, 100.3ms, 0 | 20260505 14:17:03.545 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:03.545 | INFO | RT_METRICS : 46.2%, 1.0, 24.3ms, 0 | 20260505 14:17:03.545 | INFO | PRE_HQ : 60.6%, 1.8, 57.6ms, 0 | 20260505 14:17:03.546 | INFO | BAZWRITER : 64.2%, 3.9, 130.3ms, 0 | 20260505 14:17:03.546 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 14:17:03.546 | INFO | Uploaded 36GiB to GPU in 2.19363s (16.4111GiB/s) | 20260505 14:17:03.546 | INFO | Downloaded 38GiB to GPU in 2.27575s (16.6978GiB/s) | 20260505 14:17:08.683 | INFO | Analyzing chunk frames = [3858944,3859456) | 20260505 14:17:08.683 | INFO | MemInfo: { | 20260505 14:17:08.683 | INFO | "gpu" : " 17 GB", | 20260505 14:17:08.683 | INFO | "huge" : "", | 20260505 14:17:08.683 | INFO | "huge_shared" : " 27 GB", | 20260505 14:17:08.683 | INFO | "malloc" : " 14 GB", | 20260505 14:17:08.683 | INFO | "pinned" : " 29 GB" | 20260505 14:17:08.683 | INFO | } | 20260505 14:17:08.683 | INFO | Perf report for the last 5.13816 seconds: | 20260505 14:17:08.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:17:08.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:08.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:08.683 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:08.683 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 14:17:08.684 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:08.684 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:08.684 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:17:08.684 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:08.684 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:08.684 | INFO | Uploaded 36GiB to GPU in 2.20889s (16.2978GiB/s) | 20260505 14:17:08.684 | INFO | Downloaded 34GiB to GPU in 2.06931s (16.4306GiB/s) | 20260505 14:17:13.817 | INFO | Analyzing chunk frames = [3859456,3859968) | 20260505 14:17:13.817 | INFO | MemInfo: { | 20260505 14:17:13.817 | INFO | "gpu" : " 17 GB", | 20260505 14:17:13.817 | INFO | "huge" : "", | 20260505 14:17:13.817 | INFO | "huge_shared" : " 27 GB", | 20260505 14:17:13.817 | INFO | "malloc" : " 14 GB", | 20260505 14:17:13.817 | INFO | "pinned" : " 29 GB" | 20260505 14:17:13.817 | INFO | } | 20260505 14:17:13.818 | INFO | Perf report for the last 5.13412 seconds: | 20260505 14:17:13.818 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:17:13.818 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:13.818 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:13.818 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:13.818 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260505 14:17:13.818 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:13.818 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:13.818 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 14:17:13.818 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:13.818 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:13.818 | INFO | Uploaded 36GiB to GPU in 2.23053s (16.1397GiB/s) | 20260505 14:17:13.818 | INFO | Downloaded 34GiB to GPU in 2.12257s (16.0183GiB/s) | 20260505 14:17:18.948 | INFO | Analyzing chunk frames = [3859968,3860480) | 20260505 14:17:18.948 | INFO | MemInfo: { | 20260505 14:17:18.948 | INFO | "gpu" : " 17 GB", | 20260505 14:17:18.948 | INFO | "huge" : "", | 20260505 14:17:18.948 | INFO | "huge_shared" : " 27 GB", | 20260505 14:17:18.948 | INFO | "malloc" : " 14 GB", | 20260505 14:17:18.948 | INFO | "pinned" : " 29 GB" | 20260505 14:17:18.948 | INFO | } | 20260505 14:17:18.948 | INFO | Perf report for the last 5.13089 seconds: | 20260505 14:17:18.949 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:17:18.949 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:18.949 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:18.949 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:18.949 | INFO | ANALYSIS : 63.0%, 3.0, 97.7ms, 0 | 20260505 14:17:18.949 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:18.949 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:18.949 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:17:18.949 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:18.949 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:18.949 | INFO | Uploaded 36GiB to GPU in 2.19288s (16.4168GiB/s) | 20260505 14:17:18.949 | INFO | Downloaded 34GiB to GPU in 2.13762s (15.9056GiB/s) | 20260505 14:17:24.093 | INFO | Analyzing chunk frames = [3860480,3860992) | 20260505 14:17:24.093 | INFO | MemInfo: { | 20260505 14:17:24.093 | INFO | "gpu" : " 17 GB", | 20260505 14:17:24.093 | INFO | "huge" : "", | 20260505 14:17:24.093 | INFO | "huge_shared" : " 27 GB", | 20260505 14:17:24.093 | INFO | "malloc" : " 14 GB", | 20260505 14:17:24.093 | INFO | "pinned" : " 29 GB" | 20260505 14:17:24.093 | INFO | } | 20260505 14:17:24.093 | INFO | Perf report for the last 5.14465 seconds: | 20260505 14:17:24.093 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:17:24.093 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:24.093 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:24.093 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:24.093 | INFO | ANALYSIS : 62.7%, 3.0, 98.1ms, 0 | 20260505 14:17:24.093 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:24.093 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:24.093 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:17:24.093 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:24.093 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:24.093 | INFO | Uploaded 36GiB to GPU in 2.15028s (16.742GiB/s) | 20260505 14:17:24.093 | INFO | Downloaded 34GiB to GPU in 2.12764s (15.9801GiB/s) | 20260505 14:17:29.244 | INFO | Analyzing chunk frames = [3860992,3861504) | 20260505 14:17:29.244 | INFO | MemInfo: { | 20260505 14:17:29.244 | INFO | "gpu" : " 17 GB", | 20260505 14:17:29.244 | INFO | "huge" : "", | 20260505 14:17:29.244 | INFO | "huge_shared" : " 27 GB", | 20260505 14:17:29.244 | INFO | "malloc" : " 14 GB", | 20260505 14:17:29.244 | INFO | "pinned" : " 29 GB" | 20260505 14:17:29.244 | INFO | } | 20260505 14:17:29.244 | INFO | Perf report for the last 5.15087 seconds: | 20260505 14:17:29.244 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:17:29.244 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:29.244 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:29.244 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:29.244 | INFO | ANALYSIS : 62.7%, 3.0, 98.1ms, 0 | 20260505 14:17:29.244 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:29.244 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:29.244 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 14:17:29.244 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:29.244 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:29.244 | INFO | Uploaded 36GiB to GPU in 2.17918s (16.5199GiB/s) | 20260505 14:17:29.244 | INFO | Downloaded 34GiB to GPU in 2.11129s (16.1039GiB/s) | 20260505 14:17:34.378 | INFO | Analyzing chunk frames = [3861504,3862016) | 20260505 14:17:34.378 | INFO | MemInfo: { | 20260505 14:17:34.378 | INFO | "gpu" : " 17 GB", | 20260505 14:17:34.378 | INFO | "huge" : "", | 20260505 14:17:34.378 | INFO | "huge_shared" : " 27 GB", | 20260505 14:17:34.378 | INFO | "malloc" : " 14 GB", | 20260505 14:17:34.378 | INFO | "pinned" : " 29 GB" | 20260505 14:17:34.378 | INFO | } | 20260505 14:17:34.378 | INFO | Perf report for the last 5.13393 seconds: | 20260505 14:17:34.378 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:17:34.378 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:34.378 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:34.378 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:34.378 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 14:17:34.378 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:34.378 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:34.378 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 14:17:34.378 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:34.378 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:34.378 | INFO | Uploaded 36GiB to GPU in 2.10914s (17.0686GiB/s) | 20260505 14:17:34.378 | INFO | Downloaded 34GiB to GPU in 2.17723s (15.6162GiB/s) | 20260505 14:17:39.508 | INFO | Analyzing chunk frames = [3862016,3862528) | 20260505 14:17:39.508 | INFO | MemInfo: { | 20260505 14:17:39.508 | INFO | "gpu" : " 17 GB", | 20260505 14:17:39.508 | INFO | "huge" : "", | 20260505 14:17:39.508 | INFO | "huge_shared" : " 27 GB", | 20260505 14:17:39.508 | INFO | "malloc" : " 14 GB", | 20260505 14:17:39.508 | INFO | "pinned" : " 29 GB" | 20260505 14:17:39.508 | INFO | } | 20260505 14:17:39.508 | INFO | Perf report for the last 5.13024 seconds: | 20260505 14:17:39.508 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:17:39.508 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:39.508 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:39.508 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:39.508 | INFO | ANALYSIS : 63.6%, 3.0, 99.1ms, 0 | 20260505 14:17:39.508 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:39.508 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:39.508 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:17:39.508 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:39.508 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:39.509 | INFO | Uploaded 36GiB to GPU in 2.21332s (16.2652GiB/s) | 20260505 14:17:39.509 | INFO | Downloaded 34GiB to GPU in 2.10433s (16.1572GiB/s) | 20260505 14:17:39.640 | INFO | Starting a round of Baz IO | 20260505 14:17:42.882 | INFO | Populating RTMetrics report | 20260505 14:17:42.882 | INFO | RTMetrics report populated | 20260505 14:17:42.882 | INFO | Opening tmp RTMetrics json file | 20260505 14:17:42.915 | INFO | Serializing RTMetrics to json | 20260505 14:17:42.947 | INFO | Serialization done, swapping tmp file now... | 20260505 14:17:42.970 | INFO | RTMetrics done | 20260505 14:17:42.986 | INFO | Finished a round of Baz IO: Wrote 2.264 GiB in 3.346 seconds | 20260505 14:17:42.986 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.81% / 0.00% / 77.19% | 20260505 14:17:42.986 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.58047% / 7.94729e-06% / 6.35783e-05% | 20260505 14:17:42.986 | INFO | Baz IO overall: | 20260505 14:17:42.986 | INFO | BytesWritten : 2301311779810 | 20260505 14:17:42.986 | INFO | NumEvents : 1094274426361 | 20260505 14:17:44.651 | INFO | Analyzing chunk frames = [3862528,3863040) | 20260505 14:17:44.651 | INFO | MemInfo: { | 20260505 14:17:44.651 | INFO | "gpu" : " 17 GB", | 20260505 14:17:44.651 | INFO | "huge" : "", | 20260505 14:17:44.651 | INFO | "huge_shared" : " 27 GB", | 20260505 14:17:44.651 | INFO | "malloc" : " 14 GB", | 20260505 14:17:44.651 | INFO | "pinned" : " 29 GB" | 20260505 14:17:44.651 | INFO | } | 20260505 14:17:44.651 | INFO | Perf report for the last 5.14261 seconds: | 20260505 14:17:44.651 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:17:44.651 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:44.651 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:44.651 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:44.651 | INFO | ANALYSIS : 64.5%, 3.0, 100.1ms, 0 | 20260505 14:17:44.651 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:44.651 | INFO | RT_METRICS : 46.6%, 1.0, 24.5ms, 0 | 20260505 14:17:44.651 | INFO | PRE_HQ : 60.8%, 1.8, 57.5ms, 0 | 20260505 14:17:44.651 | INFO | BAZWRITER : 64.4%, 4.0, 132.9ms, 0 | 20260505 14:17:44.651 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 14:17:44.651 | INFO | Uploaded 36GiB to GPU in 2.19609s (16.3928GiB/s) | 20260505 14:17:44.651 | INFO | Downloaded 38GiB to GPU in 2.26802s (16.7547GiB/s) | 20260505 14:17:49.793 | INFO | Analyzing chunk frames = [3863040,3863552) | 20260505 14:17:49.793 | INFO | MemInfo: { | 20260505 14:17:49.793 | INFO | "gpu" : " 17 GB", | 20260505 14:17:49.793 | INFO | "huge" : "", | 20260505 14:17:49.793 | INFO | "huge_shared" : " 27 GB", | 20260505 14:17:49.793 | INFO | "malloc" : " 14 GB", | 20260505 14:17:49.793 | INFO | "pinned" : " 29 GB" | 20260505 14:17:49.793 | INFO | } | 20260505 14:17:49.793 | INFO | Perf report for the last 5.14252 seconds: | 20260505 14:17:49.793 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:17:49.793 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:49.793 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:49.793 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:49.793 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 14:17:49.793 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:49.793 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:49.793 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:17:49.793 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:49.793 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:49.794 | INFO | Uploaded 36GiB to GPU in 2.20528s (16.3244GiB/s) | 20260505 14:17:49.794 | INFO | Downloaded 34GiB to GPU in 2.0368s (16.6929GiB/s) | 20260505 14:17:54.934 | INFO | Analyzing chunk frames = [3863552,3864064) | 20260505 14:17:54.934 | INFO | MemInfo: { | 20260505 14:17:54.934 | INFO | "gpu" : " 17 GB", | 20260505 14:17:54.934 | INFO | "huge" : "", | 20260505 14:17:54.934 | INFO | "huge_shared" : " 27 GB", | 20260505 14:17:54.934 | INFO | "malloc" : " 14 GB", | 20260505 14:17:54.934 | INFO | "pinned" : " 29 GB" | 20260505 14:17:54.934 | INFO | } | 20260505 14:17:54.935 | INFO | Perf report for the last 5.14113 seconds: | 20260505 14:17:54.935 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:17:54.935 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:54.935 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:54.935 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:54.935 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 14:17:54.935 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:54.935 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:54.935 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:17:54.935 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:54.935 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:17:54.935 | INFO | Uploaded 36GiB to GPU in 2.23712s (16.0921GiB/s) | 20260505 14:17:54.935 | INFO | Downloaded 34GiB to GPU in 2.12618s (15.9911GiB/s) | 20260505 14:18:00.071 | INFO | Analyzing chunk frames = [3864064,3864576) | 20260505 14:18:00.071 | INFO | MemInfo: { | 20260505 14:18:00.071 | INFO | "gpu" : " 17 GB", | 20260505 14:18:00.071 | INFO | "huge" : "", | 20260505 14:18:00.071 | INFO | "huge_shared" : " 27 GB", | 20260505 14:18:00.071 | INFO | "malloc" : " 14 GB", | 20260505 14:18:00.072 | INFO | "pinned" : " 29 GB" | 20260505 14:18:00.072 | INFO | } | 20260505 14:18:00.072 | INFO | Perf report for the last 5.13714 seconds: | 20260505 14:18:00.072 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:18:00.072 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:00.072 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:00.072 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:00.072 | INFO | ANALYSIS : 62.7%, 3.0, 97.3ms, 0 | 20260505 14:18:00.072 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:00.072 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:00.072 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 14:18:00.072 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:00.072 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:00.072 | INFO | Uploaded 36GiB to GPU in 2.17369s (16.5617GiB/s) | 20260505 14:18:00.072 | INFO | Downloaded 34GiB to GPU in 2.13097s (15.9552GiB/s) | 20260505 14:18:05.207 | INFO | Analyzing chunk frames = [3864576,3865088) | 20260505 14:18:05.207 | INFO | MemInfo: { | 20260505 14:18:05.207 | INFO | "gpu" : " 17 GB", | 20260505 14:18:05.207 | INFO | "huge" : "", | 20260505 14:18:05.207 | INFO | "huge_shared" : " 27 GB", | 20260505 14:18:05.207 | INFO | "malloc" : " 14 GB", | 20260505 14:18:05.207 | INFO | "pinned" : " 29 GB" | 20260505 14:18:05.207 | INFO | } | 20260505 14:18:05.207 | INFO | Perf report for the last 5.13557 seconds: | 20260505 14:18:05.207 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:18:05.207 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:05.207 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:05.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:05.207 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 14:18:05.207 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:05.207 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:05.207 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:18:05.207 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:05.207 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:05.207 | INFO | Uploaded 36GiB to GPU in 2.13692s (16.8467GiB/s) | 20260505 14:18:05.208 | INFO | Downloaded 34GiB to GPU in 2.12287s (16.0161GiB/s) | 20260505 14:18:10.354 | INFO | Analyzing chunk frames = [3865088,3865600) | 20260505 14:18:10.354 | INFO | MemInfo: { | 20260505 14:18:10.354 | INFO | "gpu" : " 17 GB", | 20260505 14:18:10.354 | INFO | "huge" : "", | 20260505 14:18:10.354 | INFO | "huge_shared" : " 27 GB", | 20260505 14:18:10.354 | INFO | "malloc" : " 14 GB", | 20260505 14:18:10.354 | INFO | "pinned" : " 29 GB" | 20260505 14:18:10.354 | INFO | } | 20260505 14:18:10.354 | INFO | Perf report for the last 5.14692 seconds: | 20260505 14:18:10.354 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:18:10.354 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:10.354 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:10.354 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:10.354 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 14:18:10.354 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:10.354 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:10.354 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:18:10.354 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:10.354 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:10.355 | INFO | Uploaded 36GiB to GPU in 2.15446s (16.7095GiB/s) | 20260505 14:18:10.355 | INFO | Downloaded 34GiB to GPU in 2.09434s (16.2342GiB/s) | 20260505 14:18:15.491 | INFO | Analyzing chunk frames = [3865600,3866112) | 20260505 14:18:15.491 | INFO | MemInfo: { | 20260505 14:18:15.491 | INFO | "gpu" : " 17 GB", | 20260505 14:18:15.491 | INFO | "huge" : "", | 20260505 14:18:15.491 | INFO | "huge_shared" : " 27 GB", | 20260505 14:18:15.491 | INFO | "malloc" : " 14 GB", | 20260505 14:18:15.491 | INFO | "pinned" : " 29 GB" | 20260505 14:18:15.491 | INFO | } | 20260505 14:18:15.491 | INFO | Perf report for the last 5.13732 seconds: | 20260505 14:18:15.491 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:18:15.491 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:15.491 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:15.491 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:15.491 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 14:18:15.492 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:15.492 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:15.492 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 14:18:15.492 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:15.492 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:15.492 | INFO | Uploaded 36GiB to GPU in 2.14878s (16.7537GiB/s) | 20260505 14:18:15.492 | INFO | Downloaded 34GiB to GPU in 2.17899s (15.6035GiB/s) | 20260505 14:18:20.629 | INFO | Analyzing chunk frames = [3866112,3866624) | 20260505 14:18:20.629 | INFO | MemInfo: { | 20260505 14:18:20.629 | INFO | "gpu" : " 17 GB", | 20260505 14:18:20.629 | INFO | "huge" : "", | 20260505 14:18:20.629 | INFO | "huge_shared" : " 27 GB", | 20260505 14:18:20.629 | INFO | "malloc" : " 14 GB", | 20260505 14:18:20.629 | INFO | "pinned" : " 29 GB" | 20260505 14:18:20.629 | INFO | } | 20260505 14:18:20.630 | INFO | Perf report for the last 5.13806 seconds: | 20260505 14:18:20.630 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:18:20.630 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:20.630 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:20.630 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:20.630 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 14:18:20.630 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:20.630 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:20.630 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 14:18:20.630 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:20.630 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:20.630 | INFO | Uploaded 36GiB to GPU in 2.20969s (16.2919GiB/s) | 20260505 14:18:20.630 | INFO | Downloaded 34GiB to GPU in 2.10232s (16.1726GiB/s) | 20260505 14:18:20.630 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 14:18:20.630 | INFO | Total mem usage: 15.3 GiB | 20260505 14:18:20.630 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 14:18:20.630 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -7.11%) | 20260505 14:18:20.630 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 14:18:20.630 | INFO | High Water Breakdown: | 20260505 14:18:20.630 | INFO | 61.1% : 8.74 GiB ( 91.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 14:18:20.630 | INFO | 38.8% : 5.55 GiB ( 83.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 14:18:20.630 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 14:18:20.630 | INFO | ------------------------------------------ | 20260505 14:18:20.630 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:18:20.630 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 14:18:20.630 | INFO | Total mem usage: 26.5 GiB | 20260505 14:18:20.630 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 14:18:20.630 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 14:18:20.630 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 14:18:20.630 | INFO | High Water Breakdown: | 20260505 14:18:20.630 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:18:20.630 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:18:20.630 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:18:20.630 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:18:20.630 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:18:20.630 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:18:20.630 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 14:18:20.630 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 14:18:20.630 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:18:20.630 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 14:18:20.630 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 14:18:20.630 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 14:18:20.630 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 14:18:20.630 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 14:18:20.630 | INFO | ------------------------------------------ | 20260505 14:18:20.630 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:18:20.630 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 14:18:20.631 | INFO | Total mem usage: 25.6 GiB | 20260505 14:18:20.631 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 14:18:20.631 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 14:18:20.631 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 14:18:20.631 | INFO | High Water Breakdown: | 20260505 14:18:20.631 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 14:18:20.631 | INFO | ------------------------------------------ | 20260505 14:18:20.631 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:18:20.631 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 14:18:20.631 | INFO | Total mem usage: 15.7 GiB | 20260505 14:18:20.631 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 14:18:20.631 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 14:18:20.631 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260505 14:18:20.631 | INFO | High Water Breakdown: | 20260505 14:18:20.631 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 14:18:20.631 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 14:18:20.631 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 14:18:20.631 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 14:18:20.631 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 14:18:20.631 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 14:18:20.631 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 14:18:20.631 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 14:18:20.631 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 14:18:20.631 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:18:20.631 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 14:18:20.631 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:18:20.631 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 14:18:20.631 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 14:18:20.631 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 14:18:20.631 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 14:18:20.631 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:18:20.631 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 14:18:20.631 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:18:20.631 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:18:20.631 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:18:20.631 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 14:18:20.631 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 14:18:20.631 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:18:20.631 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 14:18:20.631 | INFO | ------------------------------------------ | 20260505 14:18:20.631 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:18:20.631 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 14:18:20.631 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:18:20.631 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:18:20.631 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.68e+01, 1.38e+01 / 2.22e+00 | 20260505 14:18:20.631 | INFO | PerfReport: 0.29%, AggregateStats -- 9.27e-02 / 9.95e-02, 1.07e-02 / 4.12e-03 | 20260505 14:18:20.631 | INFO | PerfReport: 11.75%, Baseline -- 3.73e+00 / 4.51e+00, 5.76e-01 / 1.43e-01 | 20260505 14:18:20.631 | INFO | PerfReport: 3.29%, Histogram -- 1.04e+00 / 1.48e+00, 1.96e-01 / 1.00e-01 | 20260505 14:18:20.631 | INFO | PerfReport: 16.12%, DME -- 4.10e+01 / 4.64e+01, 4.95e+00 / 2.45e+00 | 20260505 14:18:20.631 | INFO | PerfReport: 56.30%, FrameLabeling -- 1.79e+01 / 1.95e+01, 1.26e+00 / 1.21e+00 | 20260505 14:18:20.631 | INFO | PerfReport: 7.48%, PulseAccumulating -- 2.38e+00 / 2.96e+00, 4.50e-01 / 1.07e-01 | 20260505 14:18:20.631 | INFO | PerfReport: 3.53%, Metrics -- 1.12e+00 / 1.43e+00, 3.10e-01 / 1.58e-01 | 20260505 14:18:20.631 | INFO | PerfReport: 1.24%, PulseSerializing -- 3.94e-01 / 4.44e-01, 6.90e-02 / 3.34e-02 | 20260505 14:18:20.631 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 14:18:20.631 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:18:20.631 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:18:20.631 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260505 14:18:20.631 | INFO | PerfReport: 22.55%, Upload -- 2.27e+01 / 0.00e+00, 8.10e+00 / 0.00e+00 | 20260505 14:18:20.631 | INFO | PerfReport: 22.07%, Download -- 2.23e+01 / 0.00e+00, 8.14e+00 / 0.00e+00 | 20260505 14:18:20.631 | INFO | PerfReport: 31.50%, Compute -- 3.18e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 14:18:20.631 | INFO | PerfReport: 2.31%, LockUpload -- 2.33e+00 / 0.00e+00, 8.07e+00 / 0.00e+00 | 20260505 14:18:20.631 | INFO | PerfReport: 1.48%, LockDownload -- 1.49e+00 / 0.00e+00, 3.04e+00 / 0.00e+00 | 20260505 14:18:20.631 | INFO | PerfReport: 20.07%, LockCompute -- 2.02e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 14:18:20.631 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.49e-02 / 0.00e+00 | 20260505 14:18:20.767 | INFO | Starting a round of Baz IO | 20260505 14:18:23.988 | INFO | Populating RTMetrics report | 20260505 14:18:23.988 | INFO | RTMetrics report populated | 20260505 14:18:23.988 | INFO | Opening tmp RTMetrics json file | 20260505 14:18:24.005 | INFO | Serializing RTMetrics to json | 20260505 14:18:24.028 | INFO | Serialization done, swapping tmp file now... | 20260505 14:18:24.031 | INFO | RTMetrics done | 20260505 14:18:24.092 | INFO | Finished a round of Baz IO: Wrote 2.262 GiB in 3.324 seconds | 20260505 14:18:24.092 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.80% / 0.00% / 77.19% | 20260505 14:18:24.092 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.577255% / 0% / 7.94729e-05% | 20260505 14:18:24.092 | INFO | Baz IO overall: | 20260505 14:18:24.092 | INFO | BytesWritten : 2303740995125 | 20260505 14:18:24.092 | INFO | NumEvents : 1095420379031 | 20260505 14:18:25.765 | INFO | Analyzing chunk frames = [3866624,3867136) | 20260505 14:18:25.765 | INFO | MemInfo: { | 20260505 14:18:25.765 | INFO | "gpu" : " 17 GB", | 20260505 14:18:25.765 | INFO | "huge" : "", | 20260505 14:18:25.765 | INFO | "huge_shared" : " 27 GB", | 20260505 14:18:25.765 | INFO | "malloc" : " 14 GB", | 20260505 14:18:25.765 | INFO | "pinned" : " 29 GB" | 20260505 14:18:25.765 | INFO | } | 20260505 14:18:25.765 | INFO | Perf report for the last 5.13515 seconds: | 20260505 14:18:25.765 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:18:25.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:25.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:25.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:25.765 | INFO | ANALYSIS : 64.3%, 3.0, 99.7ms, 0 | 20260505 14:18:25.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:25.765 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260505 14:18:25.765 | INFO | PRE_HQ : 61.1%, 1.8, 57.7ms, 0 | 20260505 14:18:25.765 | INFO | BAZWRITER : 64.1%, 4.0, 130.2ms, 0 | 20260505 14:18:25.765 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 14:18:25.765 | INFO | Uploaded 36GiB to GPU in 2.20065s (16.3588GiB/s) | 20260505 14:18:25.765 | INFO | Downloaded 38GiB to GPU in 2.26465s (16.7796GiB/s) | 20260505 14:18:30.904 | INFO | Analyzing chunk frames = [3867136,3867648) | 20260505 14:18:30.904 | INFO | MemInfo: { | 20260505 14:18:30.904 | INFO | "gpu" : " 17 GB", | 20260505 14:18:30.904 | INFO | "huge" : "", | 20260505 14:18:30.904 | INFO | "huge_shared" : " 27 GB", | 20260505 14:18:30.904 | INFO | "malloc" : " 14 GB", | 20260505 14:18:30.904 | INFO | "pinned" : " 29 GB" | 20260505 14:18:30.904 | INFO | } | 20260505 14:18:30.905 | INFO | Perf report for the last 5.13978 seconds: | 20260505 14:18:30.905 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:18:30.905 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:30.905 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:30.905 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:30.905 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 14:18:30.905 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:30.905 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:30.905 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:18:30.905 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:30.905 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:30.905 | INFO | Uploaded 36GiB to GPU in 2.19859s (16.3741GiB/s) | 20260505 14:18:30.905 | INFO | Downloaded 34GiB to GPU in 2.04193s (16.6509GiB/s) | 20260505 14:18:36.049 | INFO | Analyzing chunk frames = [3867648,3868160) | 20260505 14:18:36.049 | INFO | MemInfo: { | 20260505 14:18:36.049 | INFO | "gpu" : " 17 GB", | 20260505 14:18:36.049 | INFO | "huge" : "", | 20260505 14:18:36.049 | INFO | "huge_shared" : " 27 GB", | 20260505 14:18:36.049 | INFO | "malloc" : " 14 GB", | 20260505 14:18:36.049 | INFO | "pinned" : " 29 GB" | 20260505 14:18:36.049 | INFO | } | 20260505 14:18:36.049 | INFO | Perf report for the last 5.14427 seconds: | 20260505 14:18:36.049 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:18:36.049 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:36.049 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:36.049 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:36.049 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 14:18:36.049 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:36.049 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:36.049 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:18:36.049 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:36.049 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:36.049 | INFO | Uploaded 36GiB to GPU in 2.23155s (16.1323GiB/s) | 20260505 14:18:36.049 | INFO | Downloaded 34GiB to GPU in 2.10369s (16.1621GiB/s) | 20260505 14:18:41.195 | INFO | Analyzing chunk frames = [3868160,3868672) | 20260505 14:18:41.195 | INFO | MemInfo: { | 20260505 14:18:41.195 | INFO | "gpu" : " 17 GB", | 20260505 14:18:41.195 | INFO | "huge" : "", | 20260505 14:18:41.195 | INFO | "huge_shared" : " 27 GB", | 20260505 14:18:41.195 | INFO | "malloc" : " 14 GB", | 20260505 14:18:41.195 | INFO | "pinned" : " 29 GB" | 20260505 14:18:41.195 | INFO | } | 20260505 14:18:41.195 | INFO | Perf report for the last 5.14668 seconds: | 20260505 14:18:41.196 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:18:41.196 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:41.196 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:41.196 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:41.196 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260505 14:18:41.196 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:41.196 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:41.196 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:18:41.196 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:41.196 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:41.196 | INFO | Uploaded 36GiB to GPU in 2.17475s (16.5536GiB/s) | 20260505 14:18:41.196 | INFO | Downloaded 34GiB to GPU in 2.12715s (15.9838GiB/s) | 20260505 14:18:46.319 | INFO | Analyzing chunk frames = [3868672,3869184) | 20260505 14:18:46.319 | INFO | MemInfo: { | 20260505 14:18:46.319 | INFO | "gpu" : " 17 GB", | 20260505 14:18:46.319 | INFO | "huge" : "", | 20260505 14:18:46.319 | INFO | "huge_shared" : " 27 GB", | 20260505 14:18:46.319 | INFO | "malloc" : " 14 GB", | 20260505 14:18:46.319 | INFO | "pinned" : " 29 GB" | 20260505 14:18:46.319 | INFO | } | 20260505 14:18:46.319 | INFO | Perf report for the last 5.12392 seconds: | 20260505 14:18:46.320 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:18:46.320 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:46.320 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:46.320 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:46.320 | INFO | ANALYSIS : 62.8%, 3.0, 97.8ms, 0 | 20260505 14:18:46.320 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:46.320 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:46.320 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:18:46.320 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:46.320 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:46.320 | INFO | Uploaded 36GiB to GPU in 2.16813s (16.6041GiB/s) | 20260505 14:18:46.320 | INFO | Downloaded 34GiB to GPU in 2.12933s (15.9675GiB/s) | 20260505 14:18:51.457 | INFO | Analyzing chunk frames = [3869184,3869696) | 20260505 14:18:51.457 | INFO | MemInfo: { | 20260505 14:18:51.457 | INFO | "gpu" : " 17 GB", | 20260505 14:18:51.457 | INFO | "huge" : "", | 20260505 14:18:51.457 | INFO | "huge_shared" : " 27 GB", | 20260505 14:18:51.457 | INFO | "malloc" : " 14 GB", | 20260505 14:18:51.457 | INFO | "pinned" : " 29 GB" | 20260505 14:18:51.457 | INFO | } | 20260505 14:18:51.457 | INFO | Perf report for the last 5.1377 seconds: | 20260505 14:18:51.457 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:18:51.457 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:51.457 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:51.457 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:51.457 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 14:18:51.457 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:51.457 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:51.457 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 14:18:51.457 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:51.457 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:51.458 | INFO | Uploaded 36GiB to GPU in 2.17095s (16.5826GiB/s) | 20260505 14:18:51.458 | INFO | Downloaded 34GiB to GPU in 2.11112s (16.1052GiB/s) | 20260505 14:18:56.609 | INFO | Analyzing chunk frames = [3869696,3870208) | 20260505 14:18:56.609 | INFO | MemInfo: { | 20260505 14:18:56.609 | INFO | "gpu" : " 17 GB", | 20260505 14:18:56.609 | INFO | "huge" : "", | 20260505 14:18:56.609 | INFO | "huge_shared" : " 27 GB", | 20260505 14:18:56.609 | INFO | "malloc" : " 14 GB", | 20260505 14:18:56.609 | INFO | "pinned" : " 29 GB" | 20260505 14:18:56.609 | INFO | } | 20260505 14:18:56.609 | INFO | Perf report for the last 5.15185 seconds: | 20260505 14:18:56.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:18:56.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:56.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:56.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:56.609 | INFO | ANALYSIS : 63.0%, 3.0, 98.6ms, 0 | 20260505 14:18:56.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:56.609 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:56.609 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 14:18:56.609 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:56.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:18:56.609 | INFO | Uploaded 36GiB to GPU in 2.132s (16.8856GiB/s) | 20260505 14:18:56.609 | INFO | Downloaded 34GiB to GPU in 2.13529s (15.9229GiB/s) | 20260505 14:19:01.750 | INFO | Analyzing chunk frames = [3870208,3870720) | 20260505 14:19:01.750 | INFO | MemInfo: { | 20260505 14:19:01.750 | INFO | "gpu" : " 17 GB", | 20260505 14:19:01.750 | INFO | "huge" : "", | 20260505 14:19:01.750 | INFO | "huge_shared" : " 27 GB", | 20260505 14:19:01.750 | INFO | "malloc" : " 14 GB", | 20260505 14:19:01.750 | INFO | "pinned" : " 29 GB" | 20260505 14:19:01.750 | INFO | } | 20260505 14:19:01.750 | INFO | Perf report for the last 5.14134 seconds: | 20260505 14:19:01.751 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:19:01.751 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:01.751 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:01.751 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:01.751 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 14:19:01.751 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:01.751 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:01.751 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 14:19:01.751 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:01.751 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:01.751 | INFO | Uploaded 36GiB to GPU in 2.23272s (16.1238GiB/s) | 20260505 14:19:01.751 | INFO | Downloaded 34GiB to GPU in 2.10313s (16.1663GiB/s) | 20260505 14:19:01.882 | INFO | Starting a round of Baz IO | 20260505 14:19:05.101 | INFO | Populating RTMetrics report | 20260505 14:19:05.102 | INFO | RTMetrics report populated | 20260505 14:19:05.102 | INFO | Opening tmp RTMetrics json file | 20260505 14:19:05.121 | INFO | Serializing RTMetrics to json | 20260505 14:19:05.123 | INFO | Serialization done, swapping tmp file now... | 20260505 14:19:05.124 | INFO | RTMetrics done | 20260505 14:19:05.225 | INFO | Finished a round of Baz IO: Wrote 2.262 GiB in 3.342 seconds | 20260505 14:19:05.225 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.80% / 0.00% / 77.20% | 20260505 14:19:05.225 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.575515% / 1.58946e-05% / 7.94729e-05% | 20260505 14:19:05.225 | INFO | Baz IO overall: | 20260505 14:19:05.225 | INFO | BytesWritten : 2306169450161 | 20260505 14:19:05.225 | INFO | NumEvents : 1096565587256 | 20260505 14:19:06.889 | INFO | Analyzing chunk frames = [3870720,3871232) | 20260505 14:19:06.889 | INFO | MemInfo: { | 20260505 14:19:06.889 | INFO | "gpu" : " 17 GB", | 20260505 14:19:06.889 | INFO | "huge" : "", | 20260505 14:19:06.889 | INFO | "huge_shared" : " 27 GB", | 20260505 14:19:06.889 | INFO | "malloc" : " 14 GB", | 20260505 14:19:06.889 | INFO | "pinned" : " 29 GB" | 20260505 14:19:06.889 | INFO | } | 20260505 14:19:06.890 | INFO | Perf report for the last 5.13916 seconds: | 20260505 14:19:06.890 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:19:06.890 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:06.890 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:06.890 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:06.890 | INFO | ANALYSIS : 64.1%, 3.0, 99.6ms, 0 | 20260505 14:19:06.890 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:06.890 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260505 14:19:06.890 | INFO | PRE_HQ : 61.0%, 1.8, 57.3ms, 0 | 20260505 14:19:06.890 | INFO | BAZWRITER : 64.4%, 4.0, 131.4ms, 0 | 20260505 14:19:06.890 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 14:19:06.890 | INFO | Uploaded 36GiB to GPU in 2.19124s (16.429GiB/s) | 20260505 14:19:06.890 | INFO | Downloaded 38GiB to GPU in 2.26452s (16.7806GiB/s) | 20260505 14:19:12.021 | INFO | Analyzing chunk frames = [3871232,3871744) | 20260505 14:19:12.021 | INFO | MemInfo: { | 20260505 14:19:12.021 | INFO | "gpu" : " 17 GB", | 20260505 14:19:12.021 | INFO | "huge" : "", | 20260505 14:19:12.021 | INFO | "huge_shared" : " 27 GB", | 20260505 14:19:12.021 | INFO | "malloc" : " 14 GB", | 20260505 14:19:12.021 | INFO | "pinned" : " 29 GB" | 20260505 14:19:12.021 | INFO | } | 20260505 14:19:12.021 | INFO | Perf report for the last 5.13184 seconds: | 20260505 14:19:12.022 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:19:12.022 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:12.022 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:12.022 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:12.022 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 14:19:12.022 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:12.022 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:12.022 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:19:12.022 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:12.022 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:12.022 | INFO | Uploaded 36GiB to GPU in 2.20335s (16.3387GiB/s) | 20260505 14:19:12.022 | INFO | Downloaded 34GiB to GPU in 2.0359s (16.7003GiB/s) | 20260505 14:19:17.156 | INFO | Analyzing chunk frames = [3871744,3872256) | 20260505 14:19:17.156 | INFO | MemInfo: { | 20260505 14:19:17.156 | INFO | "gpu" : " 17 GB", | 20260505 14:19:17.156 | INFO | "huge" : "", | 20260505 14:19:17.156 | INFO | "huge_shared" : " 27 GB", | 20260505 14:19:17.156 | INFO | "malloc" : " 14 GB", | 20260505 14:19:17.156 | INFO | "pinned" : " 29 GB" | 20260505 14:19:17.156 | INFO | } | 20260505 14:19:17.157 | INFO | Perf report for the last 5.13518 seconds: | 20260505 14:19:17.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:19:17.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:17.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:17.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:17.157 | INFO | ANALYSIS : 63.4%, 3.0, 98.6ms, 0 | 20260505 14:19:17.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:17.157 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:17.157 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:19:17.157 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:17.157 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:17.157 | INFO | Uploaded 36GiB to GPU in 2.23383s (16.1158GiB/s) | 20260505 14:19:17.157 | INFO | Downloaded 34GiB to GPU in 2.10187s (16.1761GiB/s) | 20260505 14:19:22.306 | INFO | Analyzing chunk frames = [3872256,3872768) | 20260505 14:19:22.306 | INFO | MemInfo: { | 20260505 14:19:22.306 | INFO | "gpu" : " 17 GB", | 20260505 14:19:22.306 | INFO | "huge" : "", | 20260505 14:19:22.306 | INFO | "huge_shared" : " 27 GB", | 20260505 14:19:22.306 | INFO | "malloc" : " 14 GB", | 20260505 14:19:22.307 | INFO | "pinned" : " 29 GB" | 20260505 14:19:22.307 | INFO | } | 20260505 14:19:22.307 | INFO | Perf report for the last 5.15012 seconds: | 20260505 14:19:22.307 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:19:22.307 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:22.307 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:22.307 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:22.307 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260505 14:19:22.307 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:22.307 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:22.307 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 14:19:22.307 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:22.307 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:22.307 | INFO | Uploaded 36GiB to GPU in 2.20088s (16.3571GiB/s) | 20260505 14:19:22.307 | INFO | Downloaded 34GiB to GPU in 2.13203s (15.9473GiB/s) | 20260505 14:19:27.445 | INFO | Analyzing chunk frames = [3872768,3873280) | 20260505 14:19:27.445 | INFO | MemInfo: { | 20260505 14:19:27.445 | INFO | "gpu" : " 17 GB", | 20260505 14:19:27.445 | INFO | "huge" : "", | 20260505 14:19:27.445 | INFO | "huge_shared" : " 27 GB", | 20260505 14:19:27.445 | INFO | "malloc" : " 14 GB", | 20260505 14:19:27.445 | INFO | "pinned" : " 29 GB" | 20260505 14:19:27.445 | INFO | } | 20260505 14:19:27.445 | INFO | Perf report for the last 5.1382 seconds: | 20260505 14:19:27.445 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:19:27.445 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:27.445 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:27.445 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:27.445 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 14:19:27.445 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:27.445 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:27.445 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:19:27.445 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:27.445 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:27.445 | INFO | Uploaded 36GiB to GPU in 2.17789s (16.5297GiB/s) | 20260505 14:19:27.445 | INFO | Downloaded 34GiB to GPU in 2.12946s (15.9665GiB/s) | 20260505 14:19:32.583 | INFO | Analyzing chunk frames = [3873280,3873792) | 20260505 14:19:32.583 | INFO | MemInfo: { | 20260505 14:19:32.583 | INFO | "gpu" : " 17 GB", | 20260505 14:19:32.583 | INFO | "huge" : "", | 20260505 14:19:32.583 | INFO | "huge_shared" : " 27 GB", | 20260505 14:19:32.583 | INFO | "malloc" : " 14 GB", | 20260505 14:19:32.583 | INFO | "pinned" : " 29 GB" | 20260505 14:19:32.583 | INFO | } | 20260505 14:19:32.583 | INFO | Perf report for the last 5.13853 seconds: | 20260505 14:19:32.584 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:19:32.584 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:32.584 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:32.584 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:32.584 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 14:19:32.584 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:32.584 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:32.584 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:19:32.584 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:32.584 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:32.584 | INFO | Uploaded 36GiB to GPU in 2.20946s (16.2936GiB/s) | 20260505 14:19:32.584 | INFO | Downloaded 34GiB to GPU in 2.11271s (16.0931GiB/s) | 20260505 14:19:37.717 | INFO | Analyzing chunk frames = [3873792,3874304) | 20260505 14:19:37.717 | INFO | MemInfo: { | 20260505 14:19:37.717 | INFO | "gpu" : " 17 GB", | 20260505 14:19:37.717 | INFO | "huge" : "", | 20260505 14:19:37.717 | INFO | "huge_shared" : " 27 GB", | 20260505 14:19:37.718 | INFO | "malloc" : " 14 GB", | 20260505 14:19:37.718 | INFO | "pinned" : " 29 GB" | 20260505 14:19:37.718 | INFO | } | 20260505 14:19:37.718 | INFO | Perf report for the last 5.13416 seconds: | 20260505 14:19:37.718 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:19:37.718 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:37.718 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:37.718 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:37.718 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 14:19:37.718 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:37.718 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:37.718 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 14:19:37.718 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:37.718 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:37.718 | INFO | Uploaded 36GiB to GPU in 2.16614s (16.6195GiB/s) | 20260505 14:19:37.718 | INFO | Downloaded 34GiB to GPU in 2.18004s (15.596GiB/s) | 20260505 14:19:42.862 | INFO | Analyzing chunk frames = [3874304,3874816) | 20260505 14:19:42.862 | INFO | MemInfo: { | 20260505 14:19:42.862 | INFO | "gpu" : " 17 GB", | 20260505 14:19:42.862 | INFO | "huge" : "", | 20260505 14:19:42.862 | INFO | "huge_shared" : " 27 GB", | 20260505 14:19:42.862 | INFO | "malloc" : " 14 GB", | 20260505 14:19:42.862 | INFO | "pinned" : " 29 GB" | 20260505 14:19:42.862 | INFO | } | 20260505 14:19:42.863 | INFO | Perf report for the last 5.1449 seconds: | 20260505 14:19:42.863 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:19:42.863 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:42.863 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:42.863 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:42.863 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 14:19:42.863 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:42.863 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:42.863 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:19:42.863 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:42.863 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:42.863 | INFO | Uploaded 36GiB to GPU in 2.16203s (16.651GiB/s) | 20260505 14:19:42.863 | INFO | Downloaded 34GiB to GPU in 2.10685s (16.1379GiB/s) | 20260505 14:19:42.863 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 14:19:42.863 | INFO | Total mem usage: 15.3 GiB | 20260505 14:19:42.863 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 14:19:42.863 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.81%) | 20260505 14:19:42.863 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 14:19:42.863 | INFO | High Water Breakdown: | 20260505 14:19:42.863 | INFO | 61% : 8.68 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 14:19:42.863 | INFO | 38.8% : 5.52 GiB ( 83.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 14:19:42.863 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 14:19:42.863 | INFO | ------------------------------------------ | 20260505 14:19:42.863 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:19:42.863 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 14:19:42.863 | INFO | Total mem usage: 26.5 GiB | 20260505 14:19:42.863 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 14:19:42.863 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 14:19:42.863 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 14:19:42.863 | INFO | High Water Breakdown: | 20260505 14:19:42.863 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:19:42.863 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:19:42.863 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:19:42.863 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:19:42.863 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:19:42.863 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:19:42.864 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 14:19:42.864 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 14:19:42.864 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:19:42.864 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 14:19:42.864 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 14:19:42.864 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 14:19:42.864 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 14:19:42.864 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 14:19:42.864 | INFO | ------------------------------------------ | 20260505 14:19:42.864 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:19:42.864 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 14:19:42.864 | INFO | Total mem usage: 25.6 GiB | 20260505 14:19:42.864 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 14:19:42.864 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 14:19:42.864 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 14:19:42.864 | INFO | High Water Breakdown: | 20260505 14:19:42.864 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 14:19:42.864 | INFO | ------------------------------------------ | 20260505 14:19:42.864 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:19:42.864 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 14:19:42.864 | INFO | Total mem usage: 15.7 GiB | 20260505 14:19:42.864 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 14:19:42.864 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 14:19:42.864 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 14:19:42.864 | INFO | High Water Breakdown: | 20260505 14:19:42.864 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 14:19:42.864 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 14:19:42.864 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 14:19:42.864 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 14:19:42.864 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 14:19:42.864 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 14:19:42.864 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 14:19:42.864 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 14:19:42.864 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 14:19:42.864 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:19:42.864 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 14:19:42.864 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:19:42.864 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 14:19:42.864 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 14:19:42.864 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 14:19:42.864 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 14:19:42.864 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:19:42.864 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 14:19:42.864 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:19:42.864 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:19:42.864 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:19:42.864 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 14:19:42.864 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 14:19:42.864 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:19:42.864 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 14:19:42.864 | INFO | ------------------------------------------ | 20260505 14:19:42.864 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:19:42.864 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 14:19:42.864 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:19:42.864 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:19:42.864 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.64e+01, 1.38e+01 / 1.42e+00 | 20260505 14:19:42.864 | INFO | PerfReport: 0.29%, AggregateStats -- 9.30e-02 / 9.99e-02, 1.16e-02 / 4.00e-03 | 20260505 14:19:42.864 | INFO | PerfReport: 11.77%, Baseline -- 3.74e+00 / 4.51e+00, 5.77e-01 / 1.52e-01 | 20260505 14:19:42.864 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.51e+00, 1.94e-01 / 1.06e-01 | 20260505 14:19:42.864 | INFO | PerfReport: 16.00%, DME -- 4.07e+01 / 4.56e+01, 4.69e+00 / 2.01e+00 | 20260505 14:19:42.864 | INFO | PerfReport: 56.40%, FrameLabeling -- 1.79e+01 / 1.98e+01, 1.38e+00 / 1.87e+00 | 20260505 14:19:42.864 | INFO | PerfReport: 7.47%, PulseAccumulating -- 2.38e+00 / 2.89e+00, 4.54e-01 / 2.04e-01 | 20260505 14:19:42.864 | INFO | PerfReport: 3.54%, Metrics -- 1.13e+00 / 1.44e+00, 3.13e-01 / 1.48e-01 | 20260505 14:19:42.864 | INFO | PerfReport: 1.24%, PulseSerializing -- 3.94e-01 / 4.43e-01, 7.01e-02 / 2.98e-02 | 20260505 14:19:42.864 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 14:19:42.864 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:19:42.864 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:19:42.864 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260505 14:19:42.864 | INFO | PerfReport: 22.65%, Upload -- 2.28e+01 / 0.00e+00, 8.16e+00 / 0.00e+00 | 20260505 14:19:42.864 | INFO | PerfReport: 22.02%, Download -- 2.22e+01 / 0.00e+00, 8.09e+00 / 0.00e+00 | 20260505 14:19:42.864 | INFO | PerfReport: 31.55%, Compute -- 3.18e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 14:19:42.864 | INFO | PerfReport: 2.34%, LockUpload -- 2.35e+00 / 0.00e+00, 7.92e+00 / 0.00e+00 | 20260505 14:19:42.864 | INFO | PerfReport: 1.36%, LockDownload -- 1.37e+00 / 0.00e+00, 2.87e+00 / 0.00e+00 | 20260505 14:19:42.864 | INFO | PerfReport: 20.07%, LockCompute -- 2.02e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260505 14:19:42.864 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 2.08e-02 / 0.00e+00 | 20260505 14:19:42.995 | INFO | Starting a round of Baz IO | 20260505 14:19:46.208 | INFO | Populating RTMetrics report | 20260505 14:19:46.209 | INFO | RTMetrics report populated | 20260505 14:19:46.209 | INFO | Opening tmp RTMetrics json file | 20260505 14:19:46.227 | INFO | Serializing RTMetrics to json | 20260505 14:19:46.230 | INFO | Serialization done, swapping tmp file now... | 20260505 14:19:46.233 | INFO | RTMetrics done | 20260505 14:19:46.313 | INFO | Finished a round of Baz IO: Wrote 2.261 GiB in 3.317 seconds | 20260505 14:19:46.313 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.80% / 0.00% / 77.20% | 20260505 14:19:46.313 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.573397% / 0% / 0.000107288% | 20260505 14:19:46.313 | INFO | Baz IO overall: | 20260505 14:19:46.313 | INFO | BytesWritten : 2308597218329 | 20260505 14:19:46.313 | INFO | NumEvents : 1097710608235 | 20260505 14:19:47.998 | INFO | Analyzing chunk frames = [3874816,3875328) | 20260505 14:19:47.998 | INFO | MemInfo: { | 20260505 14:19:47.998 | INFO | "gpu" : " 17 GB", | 20260505 14:19:47.998 | INFO | "huge" : "", | 20260505 14:19:47.998 | INFO | "huge_shared" : " 27 GB", | 20260505 14:19:47.998 | INFO | "malloc" : " 14 GB", | 20260505 14:19:47.998 | INFO | "pinned" : " 29 GB" | 20260505 14:19:47.998 | INFO | } | 20260505 14:19:47.998 | INFO | Perf report for the last 5.13537 seconds: | 20260505 14:19:47.998 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:19:47.998 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:47.998 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:47.998 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:47.998 | INFO | ANALYSIS : 64.1%, 3.0, 99.4ms, 0 | 20260505 14:19:47.998 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:47.998 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260505 14:19:47.998 | INFO | PRE_HQ : 61.0%, 1.8, 57.4ms, 0 | 20260505 14:19:47.998 | INFO | BAZWRITER : 63.9%, 4.0, 132.0ms, 0 | 20260505 14:19:47.998 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 14:19:47.998 | INFO | Uploaded 36GiB to GPU in 2.16533s (16.6257GiB/s) | 20260505 14:19:47.998 | INFO | Downloaded 38GiB to GPU in 2.25764s (16.8317GiB/s) | 20260505 14:19:53.137 | INFO | Analyzing chunk frames = [3875328,3875840) | 20260505 14:19:53.137 | INFO | MemInfo: { | 20260505 14:19:53.137 | INFO | "gpu" : " 17 GB", | 20260505 14:19:53.137 | INFO | "huge" : "", | 20260505 14:19:53.137 | INFO | "huge_shared" : " 27 GB", | 20260505 14:19:53.137 | INFO | "malloc" : " 14 GB", | 20260505 14:19:53.137 | INFO | "pinned" : " 29 GB" | 20260505 14:19:53.138 | INFO | } | 20260505 14:19:53.138 | INFO | Perf report for the last 5.13977 seconds: | 20260505 14:19:53.138 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:19:53.138 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:53.138 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:53.138 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:53.138 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 14:19:53.138 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:53.138 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:53.138 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:19:53.138 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:53.138 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:53.138 | INFO | Uploaded 36GiB to GPU in 2.20034s (16.3611GiB/s) | 20260505 14:19:53.138 | INFO | Downloaded 34GiB to GPU in 2.03782s (16.6845GiB/s) | 20260505 14:19:58.271 | INFO | Analyzing chunk frames = [3875840,3876352) | 20260505 14:19:58.271 | INFO | MemInfo: { | 20260505 14:19:58.271 | INFO | "gpu" : " 17 GB", | 20260505 14:19:58.271 | INFO | "huge" : "", | 20260505 14:19:58.271 | INFO | "huge_shared" : " 27 GB", | 20260505 14:19:58.271 | INFO | "malloc" : " 14 GB", | 20260505 14:19:58.271 | INFO | "pinned" : " 29 GB" | 20260505 14:19:58.271 | INFO | } | 20260505 14:19:58.271 | INFO | Perf report for the last 5.13309 seconds: | 20260505 14:19:58.271 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:19:58.271 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:58.271 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:58.271 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:58.271 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260505 14:19:58.271 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:58.271 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:58.271 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:19:58.271 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:58.271 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:19:58.271 | INFO | Uploaded 36GiB to GPU in 2.23091s (16.1369GiB/s) | 20260505 14:19:58.271 | INFO | Downloaded 34GiB to GPU in 2.10232s (16.1726GiB/s) | 20260505 14:20:03.411 | INFO | Analyzing chunk frames = [3876352,3876864) | 20260505 14:20:03.411 | INFO | MemInfo: { | 20260505 14:20:03.411 | INFO | "gpu" : " 17 GB", | 20260505 14:20:03.411 | INFO | "huge" : "", | 20260505 14:20:03.411 | INFO | "huge_shared" : " 27 GB", | 20260505 14:20:03.411 | INFO | "malloc" : " 14 GB", | 20260505 14:20:03.411 | INFO | "pinned" : " 29 GB" | 20260505 14:20:03.411 | INFO | } | 20260505 14:20:03.411 | INFO | Perf report for the last 5.14033 seconds: | 20260505 14:20:03.411 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:20:03.411 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:03.411 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:03.411 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:03.411 | INFO | ANALYSIS : 63.0%, 3.0, 97.9ms, 0 | 20260505 14:20:03.411 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:03.411 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:03.411 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:20:03.411 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:03.411 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:03.411 | INFO | Uploaded 36GiB to GPU in 2.2056s (16.3221GiB/s) | 20260505 14:20:03.412 | INFO | Downloaded 34GiB to GPU in 2.13689s (15.911GiB/s) | 20260505 14:20:08.560 | INFO | Analyzing chunk frames = [3876864,3877376) | 20260505 14:20:08.560 | INFO | MemInfo: { | 20260505 14:20:08.560 | INFO | "gpu" : " 17 GB", | 20260505 14:20:08.560 | INFO | "huge" : "", | 20260505 14:20:08.560 | INFO | "huge_shared" : " 27 GB", | 20260505 14:20:08.560 | INFO | "malloc" : " 14 GB", | 20260505 14:20:08.560 | INFO | "pinned" : " 29 GB" | 20260505 14:20:08.560 | INFO | } | 20260505 14:20:08.560 | INFO | Perf report for the last 5.14932 seconds: | 20260505 14:20:08.561 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:20:08.561 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:08.561 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:08.561 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:08.561 | INFO | ANALYSIS : 62.7%, 3.0, 98.1ms, 0 | 20260505 14:20:08.561 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:08.561 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:08.561 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:20:08.561 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:08.561 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:08.561 | INFO | Uploaded 36GiB to GPU in 2.16856s (16.6009GiB/s) | 20260505 14:20:08.561 | INFO | Downloaded 34GiB to GPU in 2.12505s (15.9996GiB/s) | 20260505 14:20:13.706 | INFO | Analyzing chunk frames = [3877376,3877888) | 20260505 14:20:13.706 | INFO | MemInfo: { | 20260505 14:20:13.706 | INFO | "gpu" : " 17 GB", | 20260505 14:20:13.706 | INFO | "huge" : "", | 20260505 14:20:13.706 | INFO | "huge_shared" : " 27 GB", | 20260505 14:20:13.706 | INFO | "malloc" : " 14 GB", | 20260505 14:20:13.706 | INFO | "pinned" : " 29 GB" | 20260505 14:20:13.706 | INFO | } | 20260505 14:20:13.706 | INFO | Perf report for the last 5.14544 seconds: | 20260505 14:20:13.706 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:20:13.706 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:13.706 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:13.706 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:13.706 | INFO | ANALYSIS : 62.6%, 3.0, 97.9ms, 0 | 20260505 14:20:13.706 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:13.706 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:13.706 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 14:20:13.706 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:13.706 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:13.706 | INFO | Uploaded 36GiB to GPU in 2.1639s (16.6367GiB/s) | 20260505 14:20:13.706 | INFO | Downloaded 34GiB to GPU in 2.03543s (16.7041GiB/s) | 20260505 14:20:18.842 | INFO | Analyzing chunk frames = [3877888,3878400) | 20260505 14:20:18.842 | INFO | MemInfo: { | 20260505 14:20:18.842 | INFO | "gpu" : " 17 GB", | 20260505 14:20:18.842 | INFO | "huge" : "", | 20260505 14:20:18.842 | INFO | "huge_shared" : " 27 GB", | 20260505 14:20:18.842 | INFO | "malloc" : " 14 GB", | 20260505 14:20:18.842 | INFO | "pinned" : " 29 GB" | 20260505 14:20:18.842 | INFO | } | 20260505 14:20:18.842 | INFO | Perf report for the last 5.13602 seconds: | 20260505 14:20:18.842 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:20:18.842 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:18.842 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:18.842 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:18.842 | INFO | ANALYSIS : 64.1%, 3.0, 100.1ms, 0 | 20260505 14:20:18.842 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:18.842 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:18.842 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 14:20:18.842 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:18.842 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:18.842 | INFO | Uploaded 36GiB to GPU in 2.18028s (16.5116GiB/s) | 20260505 14:20:18.843 | INFO | Downloaded 34GiB to GPU in 2.20652s (15.4089GiB/s) | 20260505 14:20:23.971 | INFO | Analyzing chunk frames = [3878400,3878912) | 20260505 14:20:23.971 | INFO | MemInfo: { | 20260505 14:20:23.971 | INFO | "gpu" : " 17 GB", | 20260505 14:20:23.971 | INFO | "huge" : "", | 20260505 14:20:23.971 | INFO | "huge_shared" : " 27 GB", | 20260505 14:20:23.971 | INFO | "malloc" : " 14 GB", | 20260505 14:20:23.971 | INFO | "pinned" : " 29 GB" | 20260505 14:20:23.971 | INFO | } | 20260505 14:20:23.971 | INFO | Perf report for the last 5.12945 seconds: | 20260505 14:20:23.971 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:20:23.971 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:23.971 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:23.971 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:23.971 | INFO | ANALYSIS : 63.7%, 3.0, 99.2ms, 0 | 20260505 14:20:23.972 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:23.972 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:23.972 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 14:20:23.972 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:23.972 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:23.972 | INFO | Uploaded 36GiB to GPU in 2.22194s (16.2021GiB/s) | 20260505 14:20:23.972 | INFO | Downloaded 34GiB to GPU in 2.10088s (16.1837GiB/s) | 20260505 14:20:24.109 | INFO | Starting a round of Baz IO | 20260505 14:20:27.332 | INFO | Populating RTMetrics report | 20260505 14:20:27.332 | INFO | RTMetrics report populated | 20260505 14:20:27.332 | INFO | Opening tmp RTMetrics json file | 20260505 14:20:27.355 | INFO | Serializing RTMetrics to json | 20260505 14:20:27.377 | INFO | Serialization done, swapping tmp file now... | 20260505 14:20:27.380 | INFO | RTMetrics done | 20260505 14:20:27.446 | INFO | Finished a round of Baz IO: Wrote 2.262 GiB in 3.336 seconds | 20260505 14:20:27.446 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.79% / 0.00% / 77.20% | 20260505 14:20:27.446 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.573989% / 0% / 7.54992e-05% | 20260505 14:20:27.446 | INFO | Baz IO overall: | 20260505 14:20:27.446 | INFO | BytesWritten : 2311025965554 | 20260505 14:20:27.446 | INFO | NumEvents : 1098856671407 | 20260505 14:20:29.114 | INFO | Analyzing chunk frames = [3878912,3879424) | 20260505 14:20:29.114 | INFO | MemInfo: { | 20260505 14:20:29.114 | INFO | "gpu" : " 17 GB", | 20260505 14:20:29.114 | INFO | "huge" : "", | 20260505 14:20:29.114 | INFO | "huge_shared" : " 27 GB", | 20260505 14:20:29.114 | INFO | "malloc" : " 14 GB", | 20260505 14:20:29.114 | INFO | "pinned" : " 29 GB" | 20260505 14:20:29.114 | INFO | } | 20260505 14:20:29.114 | INFO | Perf report for the last 5.14266 seconds: | 20260505 14:20:29.114 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:20:29.114 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:29.114 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:29.114 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:29.114 | INFO | ANALYSIS : 64.2%, 3.0, 99.8ms, 0 | 20260505 14:20:29.114 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:29.114 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260505 14:20:29.114 | INFO | PRE_HQ : 60.7%, 1.8, 57.2ms, 0 | 20260505 14:20:29.114 | INFO | BAZWRITER : 64.3%, 4.0, 131.3ms, 0 | 20260505 14:20:29.114 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 14:20:29.114 | INFO | Uploaded 36GiB to GPU in 2.16445s (16.6324GiB/s) | 20260505 14:20:29.114 | INFO | Downloaded 38GiB to GPU in 2.2824s (16.6491GiB/s) | 20260505 14:20:34.253 | INFO | Analyzing chunk frames = [3879424,3879936) | 20260505 14:20:34.253 | INFO | MemInfo: { | 20260505 14:20:34.253 | INFO | "gpu" : " 17 GB", | 20260505 14:20:34.253 | INFO | "huge" : "", | 20260505 14:20:34.253 | INFO | "huge_shared" : " 27 GB", | 20260505 14:20:34.253 | INFO | "malloc" : " 14 GB", | 20260505 14:20:34.253 | INFO | "pinned" : " 29 GB" | 20260505 14:20:34.253 | INFO | } | 20260505 14:20:34.253 | INFO | Perf report for the last 5.13898 seconds: | 20260505 14:20:34.253 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:20:34.253 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:34.253 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:34.253 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:34.253 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 14:20:34.253 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:34.253 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:34.253 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:20:34.253 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:34.253 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:34.254 | INFO | Uploaded 36GiB to GPU in 2.2008s (16.3577GiB/s) | 20260505 14:20:34.254 | INFO | Downloaded 34GiB to GPU in 2.04411s (16.6331GiB/s) | 20260505 14:20:39.392 | INFO | Analyzing chunk frames = [3879936,3880448) | 20260505 14:20:39.392 | INFO | MemInfo: { | 20260505 14:20:39.392 | INFO | "gpu" : " 17 GB", | 20260505 14:20:39.392 | INFO | "huge" : "", | 20260505 14:20:39.392 | INFO | "huge_shared" : " 27 GB", | 20260505 14:20:39.392 | INFO | "malloc" : " 14 GB", | 20260505 14:20:39.392 | INFO | "pinned" : " 29 GB" | 20260505 14:20:39.392 | INFO | } | 20260505 14:20:39.392 | INFO | Perf report for the last 5.13852 seconds: | 20260505 14:20:39.392 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:20:39.392 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:39.392 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:39.392 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:39.392 | INFO | ANALYSIS : 63.4%, 3.0, 98.6ms, 0 | 20260505 14:20:39.392 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:39.392 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:39.392 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:20:39.392 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:39.392 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:39.392 | INFO | Uploaded 36GiB to GPU in 2.23167s (16.1314GiB/s) | 20260505 14:20:39.392 | INFO | Downloaded 34GiB to GPU in 2.11944s (16.042GiB/s) | 20260505 14:20:44.527 | INFO | Analyzing chunk frames = [3880448,3880960) | 20260505 14:20:44.527 | INFO | MemInfo: { | 20260505 14:20:44.527 | INFO | "gpu" : " 17 GB", | 20260505 14:20:44.527 | INFO | "huge" : "", | 20260505 14:20:44.527 | INFO | "huge_shared" : " 27 GB", | 20260505 14:20:44.527 | INFO | "malloc" : " 14 GB", | 20260505 14:20:44.527 | INFO | "pinned" : " 29 GB" | 20260505 14:20:44.527 | INFO | } | 20260505 14:20:44.527 | INFO | Perf report for the last 5.1352 seconds: | 20260505 14:20:44.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:20:44.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:44.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:44.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:44.527 | INFO | ANALYSIS : 62.7%, 3.0, 97.3ms, 0 | 20260505 14:20:44.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:44.527 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:44.527 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:20:44.527 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:44.527 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:44.528 | INFO | Uploaded 36GiB to GPU in 2.16621s (16.6188GiB/s) | 20260505 14:20:44.528 | INFO | Downloaded 34GiB to GPU in 2.13323s (15.9383GiB/s) | 20260505 14:20:49.668 | INFO | Analyzing chunk frames = [3880960,3881472) | 20260505 14:20:49.668 | INFO | MemInfo: { | 20260505 14:20:49.668 | INFO | "gpu" : " 17 GB", | 20260505 14:20:49.668 | INFO | "huge" : "", | 20260505 14:20:49.668 | INFO | "huge_shared" : " 27 GB", | 20260505 14:20:49.668 | INFO | "malloc" : " 14 GB", | 20260505 14:20:49.668 | INFO | "pinned" : " 29 GB" | 20260505 14:20:49.668 | INFO | } | 20260505 14:20:49.668 | INFO | Perf report for the last 5.1414 seconds: | 20260505 14:20:49.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:20:49.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:49.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:49.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:49.668 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 14:20:49.668 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:49.668 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:49.668 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 14:20:49.668 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:49.668 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:49.669 | INFO | Uploaded 36GiB to GPU in 2.15935s (16.6717GiB/s) | 20260505 14:20:49.669 | INFO | Downloaded 34GiB to GPU in 2.12401s (16.0074GiB/s) | 20260505 14:20:54.815 | INFO | Analyzing chunk frames = [3881472,3881984) | 20260505 14:20:54.815 | INFO | MemInfo: { | 20260505 14:20:54.815 | INFO | "gpu" : " 17 GB", | 20260505 14:20:54.815 | INFO | "huge" : "", | 20260505 14:20:54.815 | INFO | "huge_shared" : " 27 GB", | 20260505 14:20:54.815 | INFO | "malloc" : " 14 GB", | 20260505 14:20:54.815 | INFO | "pinned" : " 29 GB" | 20260505 14:20:54.815 | INFO | } | 20260505 14:20:54.815 | INFO | Perf report for the last 5.14717 seconds: | 20260505 14:20:54.815 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:20:54.815 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:54.815 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:54.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:54.816 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 14:20:54.816 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:54.816 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:54.816 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:20:54.816 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:54.816 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:54.816 | INFO | Uploaded 36GiB to GPU in 2.20385s (16.335GiB/s) | 20260505 14:20:54.816 | INFO | Downloaded 34GiB to GPU in 2.1146s (16.0787GiB/s) | 20260505 14:20:59.958 | INFO | Analyzing chunk frames = [3881984,3882496) | 20260505 14:20:59.958 | INFO | MemInfo: { | 20260505 14:20:59.958 | INFO | "gpu" : " 17 GB", | 20260505 14:20:59.958 | INFO | "huge" : "", | 20260505 14:20:59.958 | INFO | "huge_shared" : " 27 GB", | 20260505 14:20:59.958 | INFO | "malloc" : " 14 GB", | 20260505 14:20:59.958 | INFO | "pinned" : " 29 GB" | 20260505 14:20:59.958 | INFO | } | 20260505 14:20:59.958 | INFO | Perf report for the last 5.1428 seconds: | 20260505 14:20:59.958 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:20:59.958 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:59.958 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:59.958 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:59.958 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 14:20:59.958 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:59.958 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:59.958 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 14:20:59.958 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:59.958 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:20:59.959 | INFO | Uploaded 36GiB to GPU in 2.1259s (16.934GiB/s) | 20260505 14:20:59.959 | INFO | Downloaded 34GiB to GPU in 2.17027s (15.6663GiB/s) | 20260505 14:21:05.084 | INFO | Analyzing chunk frames = [3882496,3883008) | 20260505 14:21:05.084 | INFO | MemInfo: { | 20260505 14:21:05.084 | INFO | "gpu" : " 17 GB", | 20260505 14:21:05.084 | INFO | "huge" : "", | 20260505 14:21:05.084 | INFO | "huge_shared" : " 27 GB", | 20260505 14:21:05.084 | INFO | "malloc" : " 14 GB", | 20260505 14:21:05.084 | INFO | "pinned" : " 29 GB" | 20260505 14:21:05.084 | INFO | } | 20260505 14:21:05.084 | INFO | Perf report for the last 5.12556 seconds: | 20260505 14:21:05.084 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:21:05.084 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:05.084 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:05.084 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:05.084 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 14:21:05.084 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:05.084 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:05.084 | INFO | PRE_HQ : 28.3%, 1.0, 14.6ms, 0 | 20260505 14:21:05.084 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:05.084 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:05.084 | INFO | Uploaded 36GiB to GPU in 2.22493s (16.1803GiB/s) | 20260505 14:21:05.084 | INFO | Downloaded 34GiB to GPU in 2.10427s (16.1576GiB/s) | 20260505 14:21:05.084 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 14:21:05.084 | INFO | Total mem usage: 15.3 GiB | 20260505 14:21:05.084 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 14:21:05.084 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.8%) | 20260505 14:21:05.084 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 14:21:05.084 | INFO | High Water Breakdown: | 20260505 14:21:05.084 | INFO | 61% : 8.68 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 14:21:05.084 | INFO | 38.8% : 5.52 GiB ( 83.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 14:21:05.084 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 14:21:05.084 | INFO | ------------------------------------------ | 20260505 14:21:05.084 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:21:05.084 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 14:21:05.084 | INFO | Total mem usage: 26.5 GiB | 20260505 14:21:05.085 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 14:21:05.085 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 14:21:05.085 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 14:21:05.085 | INFO | High Water Breakdown: | 20260505 14:21:05.085 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:21:05.085 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:21:05.085 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:21:05.085 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:21:05.085 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:21:05.085 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:21:05.085 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 14:21:05.085 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 14:21:05.085 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:21:05.085 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 14:21:05.085 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 14:21:05.085 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 14:21:05.085 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 14:21:05.085 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 14:21:05.085 | INFO | ------------------------------------------ | 20260505 14:21:05.085 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:21:05.085 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 14:21:05.085 | INFO | Total mem usage: 25.6 GiB | 20260505 14:21:05.085 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 14:21:05.085 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 14:21:05.085 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 14:21:05.085 | INFO | High Water Breakdown: | 20260505 14:21:05.085 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 14:21:05.085 | INFO | ------------------------------------------ | 20260505 14:21:05.085 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:21:05.085 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 14:21:05.085 | INFO | Total mem usage: 15.7 GiB | 20260505 14:21:05.085 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 14:21:05.085 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 14:21:05.085 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 14:21:05.085 | INFO | High Water Breakdown: | 20260505 14:21:05.085 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 14:21:05.085 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 14:21:05.085 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 14:21:05.085 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 14:21:05.085 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 14:21:05.085 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 14:21:05.085 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 14:21:05.085 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 14:21:05.085 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 14:21:05.085 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:21:05.085 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 14:21:05.085 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:21:05.085 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 14:21:05.085 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 14:21:05.085 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 14:21:05.085 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 14:21:05.085 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:21:05.085 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 14:21:05.085 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:21:05.085 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:21:05.085 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:21:05.085 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 14:21:05.085 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 14:21:05.085 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:21:05.085 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 14:21:05.085 | INFO | ------------------------------------------ | 20260505 14:21:05.085 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:21:05.085 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 14:21:05.085 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:21:05.085 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:21:05.085 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.66e+01, 1.36e+01 / 2.69e+00 | 20260505 14:21:05.085 | INFO | PerfReport: 0.29%, AggregateStats -- 9.31e-02 / 1.02e-01, 1.22e-02 / 6.06e-03 | 20260505 14:21:05.085 | INFO | PerfReport: 11.76%, Baseline -- 3.74e+00 / 4.51e+00, 5.77e-01 / 1.67e-01 | 20260505 14:21:05.085 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.49e+00, 1.96e-01 / 1.01e-01 | 20260505 14:21:05.085 | INFO | PerfReport: 16.05%, DME -- 4.08e+01 / 4.59e+01, 4.71e+00 / 2.70e+00 | 20260505 14:21:05.085 | INFO | PerfReport: 56.42%, FrameLabeling -- 1.79e+01 / 1.97e+01, 1.38e+00 / 1.60e+00 | 20260505 14:21:05.085 | INFO | PerfReport: 7.43%, PulseAccumulating -- 2.36e+00 / 2.94e+00, 4.44e-01 / 1.47e-01 | 20260505 14:21:05.085 | INFO | PerfReport: 3.53%, Metrics -- 1.12e+00 / 1.46e+00, 3.16e-01 / 2.01e-01 | 20260505 14:21:05.085 | INFO | PerfReport: 1.24%, PulseSerializing -- 3.94e-01 / 4.44e-01, 6.98e-02 / 3.31e-02 | 20260505 14:21:05.085 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 14:21:05.085 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:21:05.085 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:21:05.085 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.58e+01 / 0.00e+00 | 20260505 14:21:05.085 | INFO | PerfReport: 22.60%, Upload -- 2.28e+01 / 0.00e+00, 8.12e+00 / 0.00e+00 | 20260505 14:21:05.085 | INFO | PerfReport: 22.01%, Download -- 2.22e+01 / 0.00e+00, 8.07e+00 / 0.00e+00 | 20260505 14:21:05.085 | INFO | PerfReport: 31.55%, Compute -- 3.18e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 14:21:05.085 | INFO | PerfReport: 2.34%, LockUpload -- 2.36e+00 / 0.00e+00, 8.06e+00 / 0.00e+00 | 20260505 14:21:05.085 | INFO | PerfReport: 1.37%, LockDownload -- 1.38e+00 / 0.00e+00, 2.90e+00 / 0.00e+00 | 20260505 14:21:05.085 | INFO | PerfReport: 20.10%, LockCompute -- 2.03e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 14:21:05.085 | INFO | PerfReport: 0.01%, Misc -- 1.46e-02 / 0.00e+00, 2.85e-02 / 0.00e+00 | 20260505 14:21:05.224 | INFO | Starting a round of Baz IO | 20260505 14:21:08.443 | INFO | Populating RTMetrics report | 20260505 14:21:08.444 | INFO | RTMetrics report populated | 20260505 14:21:08.444 | INFO | Opening tmp RTMetrics json file | 20260505 14:21:08.444 | INFO | Serializing RTMetrics to json | 20260505 14:21:08.445 | INFO | Serialization done, swapping tmp file now... | 20260505 14:21:08.445 | INFO | RTMetrics done | 20260505 14:21:08.575 | INFO | Finished a round of Baz IO: Wrote 2.263 GiB in 3.35 seconds | 20260505 14:21:08.575 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.79% / 0.00% / 77.21% | 20260505 14:21:08.575 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.57253% / 1.58946e-05% / 9.93411e-05% | 20260505 14:21:08.575 | INFO | Baz IO overall: | 20260505 14:21:08.575 | INFO | BytesWritten : 2313455678714 | 20260505 14:21:08.575 | INFO | NumEvents : 1100003302539 | 20260505 14:21:10.218 | INFO | Analyzing chunk frames = [3883008,3883520) | 20260505 14:21:10.218 | INFO | MemInfo: { | 20260505 14:21:10.218 | INFO | "gpu" : " 17 GB", | 20260505 14:21:10.218 | INFO | "huge" : "", | 20260505 14:21:10.218 | INFO | "huge_shared" : " 27 GB", | 20260505 14:21:10.218 | INFO | "malloc" : " 14 GB", | 20260505 14:21:10.218 | INFO | "pinned" : " 29 GB" | 20260505 14:21:10.218 | INFO | } | 20260505 14:21:10.218 | INFO | Perf report for the last 5.13464 seconds: | 20260505 14:21:10.219 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:21:10.219 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:10.219 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:10.219 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:10.219 | INFO | ANALYSIS : 64.3%, 3.0, 99.6ms, 0 | 20260505 14:21:10.219 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:10.219 | INFO | RT_METRICS : 44.9%, 1.0, 23.5ms, 0 | 20260505 14:21:10.219 | INFO | PRE_HQ : 60.7%, 1.8, 57.1ms, 0 | 20260505 14:21:10.219 | INFO | BAZWRITER : 64.6%, 4.0, 134.0ms, 0 | 20260505 14:21:10.219 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 14:21:10.219 | INFO | Uploaded 36GiB to GPU in 2.17366s (16.5619GiB/s) | 20260505 14:21:10.219 | INFO | Downloaded 38GiB to GPU in 2.27991s (16.6673GiB/s) | 20260505 14:21:15.366 | INFO | Analyzing chunk frames = [3883520,3884032) | 20260505 14:21:15.366 | INFO | MemInfo: { | 20260505 14:21:15.366 | INFO | "gpu" : " 17 GB", | 20260505 14:21:15.366 | INFO | "huge" : "", | 20260505 14:21:15.366 | INFO | "huge_shared" : " 27 GB", | 20260505 14:21:15.366 | INFO | "malloc" : " 14 GB", | 20260505 14:21:15.366 | INFO | "pinned" : " 29 GB" | 20260505 14:21:15.366 | INFO | } | 20260505 14:21:15.366 | INFO | Perf report for the last 5.14745 seconds: | 20260505 14:21:15.366 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:21:15.366 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:15.366 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:15.366 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:15.366 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 14:21:15.366 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:15.366 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:15.366 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:21:15.366 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:15.366 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:15.366 | INFO | Uploaded 36GiB to GPU in 2.17106s (16.5818GiB/s) | 20260505 14:21:15.366 | INFO | Downloaded 34GiB to GPU in 2.02955s (16.7525GiB/s) | 20260505 14:21:20.511 | INFO | Analyzing chunk frames = [3884032,3884544) | 20260505 14:21:20.511 | INFO | MemInfo: { | 20260505 14:21:20.511 | INFO | "gpu" : " 17 GB", | 20260505 14:21:20.511 | INFO | "huge" : "", | 20260505 14:21:20.511 | INFO | "huge_shared" : " 27 GB", | 20260505 14:21:20.511 | INFO | "malloc" : " 14 GB", | 20260505 14:21:20.511 | INFO | "pinned" : " 29 GB" | 20260505 14:21:20.511 | INFO | } | 20260505 14:21:20.511 | INFO | Perf report for the last 5.14521 seconds: | 20260505 14:21:20.511 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:21:20.511 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:20.511 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:20.511 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:20.511 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 14:21:20.511 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:20.511 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:20.511 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:21:20.511 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:20.511 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:20.512 | INFO | Uploaded 36GiB to GPU in 2.23732s (16.0907GiB/s) | 20260505 14:21:20.512 | INFO | Downloaded 34GiB to GPU in 2.12204s (16.0223GiB/s) | 20260505 14:21:25.650 | INFO | Analyzing chunk frames = [3884544,3885056) | 20260505 14:21:25.650 | INFO | MemInfo: { | 20260505 14:21:25.650 | INFO | "gpu" : " 17 GB", | 20260505 14:21:25.650 | INFO | "huge" : "", | 20260505 14:21:25.650 | INFO | "huge_shared" : " 27 GB", | 20260505 14:21:25.650 | INFO | "malloc" : " 14 GB", | 20260505 14:21:25.650 | INFO | "pinned" : " 29 GB" | 20260505 14:21:25.650 | INFO | } | 20260505 14:21:25.650 | INFO | Perf report for the last 5.13877 seconds: | 20260505 14:21:25.650 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:21:25.650 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:25.650 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:25.650 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:25.650 | INFO | ANALYSIS : 62.7%, 3.0, 97.4ms, 0 | 20260505 14:21:25.650 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:25.650 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:25.650 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 14:21:25.650 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:25.650 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:25.650 | INFO | Uploaded 36GiB to GPU in 2.18264s (16.4938GiB/s) | 20260505 14:21:25.650 | INFO | Downloaded 34GiB to GPU in 2.12629s (15.9903GiB/s) | 20260505 14:21:30.785 | INFO | Analyzing chunk frames = [3885056,3885568) | 20260505 14:21:30.785 | INFO | MemInfo: { | 20260505 14:21:30.785 | INFO | "gpu" : " 17 GB", | 20260505 14:21:30.785 | INFO | "huge" : "", | 20260505 14:21:30.785 | INFO | "huge_shared" : " 27 GB", | 20260505 14:21:30.785 | INFO | "malloc" : " 14 GB", | 20260505 14:21:30.785 | INFO | "pinned" : " 29 GB" | 20260505 14:21:30.785 | INFO | } | 20260505 14:21:30.785 | INFO | Perf report for the last 5.13538 seconds: | 20260505 14:21:30.785 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:21:30.785 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:30.785 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:30.785 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:30.785 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 14:21:30.785 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:30.785 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:30.785 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:21:30.785 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:30.786 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:30.786 | INFO | Uploaded 36GiB to GPU in 2.15403s (16.7129GiB/s) | 20260505 14:21:30.786 | INFO | Downloaded 34GiB to GPU in 2.127s (15.9849GiB/s) | 20260505 14:21:35.930 | INFO | Analyzing chunk frames = [3885568,3886080) | 20260505 14:21:35.930 | INFO | MemInfo: { | 20260505 14:21:35.930 | INFO | "gpu" : " 17 GB", | 20260505 14:21:35.930 | INFO | "huge" : "", | 20260505 14:21:35.930 | INFO | "huge_shared" : " 27 GB", | 20260505 14:21:35.930 | INFO | "malloc" : " 14 GB", | 20260505 14:21:35.930 | INFO | "pinned" : " 29 GB" | 20260505 14:21:35.930 | INFO | } | 20260505 14:21:35.930 | INFO | Perf report for the last 5.14481 seconds: | 20260505 14:21:35.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:21:35.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:35.930 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:35.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:35.930 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 14:21:35.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:35.930 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:35.930 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 14:21:35.930 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:35.930 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:35.931 | INFO | Uploaded 36GiB to GPU in 2.16913s (16.5965GiB/s) | 20260505 14:21:35.931 | INFO | Downloaded 34GiB to GPU in 2.10361s (16.1627GiB/s) | 20260505 14:21:41.067 | INFO | Analyzing chunk frames = [3886080,3886592) | 20260505 14:21:41.067 | INFO | MemInfo: { | 20260505 14:21:41.067 | INFO | "gpu" : " 17 GB", | 20260505 14:21:41.067 | INFO | "huge" : "", | 20260505 14:21:41.067 | INFO | "huge_shared" : " 27 GB", | 20260505 14:21:41.067 | INFO | "malloc" : " 14 GB", | 20260505 14:21:41.067 | INFO | "pinned" : " 29 GB" | 20260505 14:21:41.067 | INFO | } | 20260505 14:21:41.067 | INFO | Perf report for the last 5.13687 seconds: | 20260505 14:21:41.067 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:21:41.067 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:41.067 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:41.067 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:41.067 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 14:21:41.067 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:41.067 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:41.067 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 14:21:41.067 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:41.067 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:41.067 | INFO | Uploaded 36GiB to GPU in 2.15099s (16.7365GiB/s) | 20260505 14:21:41.067 | INFO | Downloaded 34GiB to GPU in 2.14122s (15.8788GiB/s) | 20260505 14:21:46.206 | INFO | Analyzing chunk frames = [3886592,3887104) | 20260505 14:21:46.206 | INFO | MemInfo: { | 20260505 14:21:46.206 | INFO | "gpu" : " 17 GB", | 20260505 14:21:46.206 | INFO | "huge" : "", | 20260505 14:21:46.206 | INFO | "huge_shared" : " 27 GB", | 20260505 14:21:46.206 | INFO | "malloc" : " 14 GB", | 20260505 14:21:46.206 | INFO | "pinned" : " 29 GB" | 20260505 14:21:46.206 | INFO | } | 20260505 14:21:46.206 | INFO | Perf report for the last 5.13884 seconds: | 20260505 14:21:46.206 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:21:46.206 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:46.206 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:46.206 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:46.206 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 14:21:46.206 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:46.206 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:46.206 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 14:21:46.206 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:46.206 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:46.206 | INFO | Uploaded 36GiB to GPU in 2.21048s (16.286GiB/s) | 20260505 14:21:46.206 | INFO | Downloaded 34GiB to GPU in 2.09318s (16.2432GiB/s) | 20260505 14:21:46.341 | INFO | Starting a round of Baz IO | 20260505 14:21:49.555 | INFO | Populating RTMetrics report | 20260505 14:21:49.555 | INFO | RTMetrics report populated | 20260505 14:21:49.555 | INFO | Opening tmp RTMetrics json file | 20260505 14:21:49.597 | INFO | Serializing RTMetrics to json | 20260505 14:21:49.623 | INFO | Serialization done, swapping tmp file now... | 20260505 14:21:49.631 | INFO | RTMetrics done | 20260505 14:21:49.658 | INFO | Finished a round of Baz IO: Wrote 2.26 GiB in 3.317 seconds | 20260505 14:21:49.658 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.78% / 0.00% / 77.21% | 20260505 14:21:49.658 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.572797% / 0% / 8.34465e-05% | 20260505 14:21:49.658 | INFO | Baz IO overall: | 20260505 14:21:49.658 | INFO | BytesWritten : 2315882814791 | 20260505 14:21:49.659 | INFO | NumEvents : 1101148541827 | 20260505 14:21:51.344 | INFO | Analyzing chunk frames = [3887104,3887616) | 20260505 14:21:51.344 | INFO | MemInfo: { | 20260505 14:21:51.344 | INFO | "gpu" : " 17 GB", | 20260505 14:21:51.344 | INFO | "huge" : "", | 20260505 14:21:51.344 | INFO | "huge_shared" : " 27 GB", | 20260505 14:21:51.344 | INFO | "malloc" : " 14 GB", | 20260505 14:21:51.344 | INFO | "pinned" : " 29 GB" | 20260505 14:21:51.344 | INFO | } | 20260505 14:21:51.344 | INFO | Perf report for the last 5.13792 seconds: | 20260505 14:21:51.344 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:21:51.344 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:51.344 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:51.344 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:51.344 | INFO | ANALYSIS : 64.1%, 3.0, 99.4ms, 0 | 20260505 14:21:51.344 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:51.344 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260505 14:21:51.344 | INFO | PRE_HQ : 60.7%, 1.8, 57.1ms, 0 | 20260505 14:21:51.344 | INFO | BAZWRITER : 63.9%, 4.0, 131.0ms, 0 | 20260505 14:21:51.344 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 14:21:51.344 | INFO | Uploaded 36GiB to GPU in 2.17642s (16.5409GiB/s) | 20260505 14:21:51.344 | INFO | Downloaded 38GiB to GPU in 2.26146s (16.8033GiB/s) | 20260505 14:21:56.476 | INFO | Analyzing chunk frames = [3887616,3888128) | 20260505 14:21:56.476 | INFO | MemInfo: { | 20260505 14:21:56.476 | INFO | "gpu" : " 17 GB", | 20260505 14:21:56.476 | INFO | "huge" : "", | 20260505 14:21:56.476 | INFO | "huge_shared" : " 27 GB", | 20260505 14:21:56.476 | INFO | "malloc" : " 14 GB", | 20260505 14:21:56.476 | INFO | "pinned" : " 29 GB" | 20260505 14:21:56.476 | INFO | } | 20260505 14:21:56.476 | INFO | Perf report for the last 5.13256 seconds: | 20260505 14:21:56.476 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:21:56.476 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:56.476 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:56.476 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:56.476 | INFO | ANALYSIS : 63.5%, 3.0, 99.0ms, 0 | 20260505 14:21:56.476 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:56.476 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:56.476 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:21:56.477 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:56.477 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:21:56.477 | INFO | Uploaded 36GiB to GPU in 2.19238s (16.4205GiB/s) | 20260505 14:21:56.477 | INFO | Downloaded 34GiB to GPU in 2.03223s (16.7304GiB/s) | 20260505 14:22:01.626 | INFO | Analyzing chunk frames = [3888128,3888640) | 20260505 14:22:01.626 | INFO | MemInfo: { | 20260505 14:22:01.626 | INFO | "gpu" : " 17 GB", | 20260505 14:22:01.626 | INFO | "huge" : "", | 20260505 14:22:01.626 | INFO | "huge_shared" : " 27 GB", | 20260505 14:22:01.626 | INFO | "malloc" : " 14 GB", | 20260505 14:22:01.626 | INFO | "pinned" : " 29 GB" | 20260505 14:22:01.626 | INFO | } | 20260505 14:22:01.626 | INFO | Perf report for the last 5.14991 seconds: | 20260505 14:22:01.627 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:22:01.627 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:01.627 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:01.627 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:01.627 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 14:22:01.627 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:01.627 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:01.627 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 14:22:01.627 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:01.627 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:01.627 | INFO | Uploaded 36GiB to GPU in 2.22216s (16.2005GiB/s) | 20260505 14:22:01.627 | INFO | Downloaded 34GiB to GPU in 2.10453s (16.1556GiB/s) | 20260505 14:22:06.765 | INFO | Analyzing chunk frames = [3888640,3889152) | 20260505 14:22:06.765 | INFO | MemInfo: { | 20260505 14:22:06.765 | INFO | "gpu" : " 17 GB", | 20260505 14:22:06.765 | INFO | "huge" : "", | 20260505 14:22:06.765 | INFO | "huge_shared" : " 27 GB", | 20260505 14:22:06.765 | INFO | "malloc" : " 14 GB", | 20260505 14:22:06.765 | INFO | "pinned" : " 29 GB" | 20260505 14:22:06.765 | INFO | } | 20260505 14:22:06.765 | INFO | Perf report for the last 5.13843 seconds: | 20260505 14:22:06.765 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:22:06.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:06.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:06.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:06.765 | INFO | ANALYSIS : 62.6%, 3.0, 97.3ms, 0 | 20260505 14:22:06.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:06.765 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:06.765 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:22:06.765 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:06.765 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:06.765 | INFO | Uploaded 36GiB to GPU in 2.1877s (16.4556GiB/s) | 20260505 14:22:06.765 | INFO | Downloaded 34GiB to GPU in 2.13061s (15.9578GiB/s) | 20260505 14:22:11.897 | INFO | Analyzing chunk frames = [3889152,3889664) | 20260505 14:22:11.897 | INFO | MemInfo: { | 20260505 14:22:11.897 | INFO | "gpu" : " 17 GB", | 20260505 14:22:11.897 | INFO | "huge" : "", | 20260505 14:22:11.897 | INFO | "huge_shared" : " 27 GB", | 20260505 14:22:11.897 | INFO | "malloc" : " 14 GB", | 20260505 14:22:11.897 | INFO | "pinned" : " 29 GB" | 20260505 14:22:11.897 | INFO | } | 20260505 14:22:11.898 | INFO | Perf report for the last 5.13288 seconds: | 20260505 14:22:11.898 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:22:11.898 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:11.898 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:11.898 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:11.898 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260505 14:22:11.898 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:11.898 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:11.898 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:22:11.898 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:11.898 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:11.898 | INFO | Uploaded 36GiB to GPU in 2.18347s (16.4875GiB/s) | 20260505 14:22:11.898 | INFO | Downloaded 34GiB to GPU in 2.13097s (15.9551GiB/s) | 20260505 14:22:17.028 | INFO | Analyzing chunk frames = [3889664,3890176) | 20260505 14:22:17.028 | INFO | MemInfo: { | 20260505 14:22:17.028 | INFO | "gpu" : " 17 GB", | 20260505 14:22:17.028 | INFO | "huge" : "", | 20260505 14:22:17.028 | INFO | "huge_shared" : " 27 GB", | 20260505 14:22:17.028 | INFO | "malloc" : " 14 GB", | 20260505 14:22:17.028 | INFO | "pinned" : " 29 GB" | 20260505 14:22:17.028 | INFO | } | 20260505 14:22:17.028 | INFO | Perf report for the last 5.1307 seconds: | 20260505 14:22:17.028 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:22:17.028 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:17.028 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:17.028 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:17.028 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 14:22:17.028 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:17.028 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:17.028 | INFO | PRE_HQ : 28.3%, 1.0, 14.6ms, 0 | 20260505 14:22:17.028 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:17.029 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:17.029 | INFO | Uploaded 36GiB to GPU in 2.16553s (16.6241GiB/s) | 20260505 14:22:17.029 | INFO | Downloaded 34GiB to GPU in 2.10181s (16.1765GiB/s) | 20260505 14:22:22.175 | INFO | Analyzing chunk frames = [3890176,3890688) | 20260505 14:22:22.175 | INFO | MemInfo: { | 20260505 14:22:22.175 | INFO | "gpu" : " 17 GB", | 20260505 14:22:22.175 | INFO | "huge" : "", | 20260505 14:22:22.175 | INFO | "huge_shared" : " 27 GB", | 20260505 14:22:22.175 | INFO | "malloc" : " 14 GB", | 20260505 14:22:22.175 | INFO | "pinned" : " 29 GB" | 20260505 14:22:22.175 | INFO | } | 20260505 14:22:22.175 | INFO | Perf report for the last 5.14652 seconds: | 20260505 14:22:22.175 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:22:22.175 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:22.175 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:22.175 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:22.175 | INFO | ANALYSIS : 63.1%, 3.0, 98.7ms, 0 | 20260505 14:22:22.175 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:22.175 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:22.175 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 14:22:22.175 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:22.175 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:22.175 | INFO | Uploaded 36GiB to GPU in 2.17987s (16.5148GiB/s) | 20260505 14:22:22.175 | INFO | Downloaded 34GiB to GPU in 2.14865s (15.8239GiB/s) | 20260505 14:22:27.320 | INFO | Analyzing chunk frames = [3890688,3891200) | 20260505 14:22:27.320 | INFO | MemInfo: { | 20260505 14:22:27.320 | INFO | "gpu" : " 17 GB", | 20260505 14:22:27.320 | INFO | "huge" : "", | 20260505 14:22:27.320 | INFO | "huge_shared" : " 27 GB", | 20260505 14:22:27.320 | INFO | "malloc" : " 14 GB", | 20260505 14:22:27.320 | INFO | "pinned" : " 29 GB" | 20260505 14:22:27.320 | INFO | } | 20260505 14:22:27.320 | INFO | Perf report for the last 5.14532 seconds: | 20260505 14:22:27.320 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:22:27.320 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:27.320 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:27.320 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:27.320 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 14:22:27.320 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:27.320 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:27.320 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:22:27.320 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:27.320 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:27.321 | INFO | Uploaded 36GiB to GPU in 2.21254s (16.2709GiB/s) | 20260505 14:22:27.321 | INFO | Downloaded 34GiB to GPU in 2.10595s (16.1448GiB/s) | 20260505 14:22:27.321 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 14:22:27.321 | INFO | Total mem usage: 15.3 GiB | 20260505 14:22:27.321 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 14:22:27.321 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.79%) | 20260505 14:22:27.321 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 14:22:27.321 | INFO | High Water Breakdown: | 20260505 14:22:27.321 | INFO | 61% : 8.68 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 14:22:27.321 | INFO | 38.8% : 5.52 GiB ( 83.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 14:22:27.321 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 14:22:27.321 | INFO | ------------------------------------------ | 20260505 14:22:27.321 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:22:27.321 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 14:22:27.321 | INFO | Total mem usage: 26.5 GiB | 20260505 14:22:27.321 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 14:22:27.321 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 14:22:27.321 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 14:22:27.321 | INFO | High Water Breakdown: | 20260505 14:22:27.321 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:22:27.321 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:22:27.321 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:22:27.321 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:22:27.321 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:22:27.321 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:22:27.321 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 14:22:27.321 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 14:22:27.321 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:22:27.321 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 14:22:27.321 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 14:22:27.321 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 14:22:27.321 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 14:22:27.321 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 14:22:27.321 | INFO | ------------------------------------------ | 20260505 14:22:27.321 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:22:27.321 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 14:22:27.321 | INFO | Total mem usage: 25.6 GiB | 20260505 14:22:27.321 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 14:22:27.321 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 14:22:27.321 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 14:22:27.321 | INFO | High Water Breakdown: | 20260505 14:22:27.321 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 14:22:27.321 | INFO | ------------------------------------------ | 20260505 14:22:27.321 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:22:27.321 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 14:22:27.321 | INFO | Total mem usage: 15.7 GiB | 20260505 14:22:27.321 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 14:22:27.321 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 14:22:27.321 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 14:22:27.321 | INFO | High Water Breakdown: | 20260505 14:22:27.321 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 14:22:27.321 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 14:22:27.321 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 14:22:27.321 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 14:22:27.321 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 14:22:27.321 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 14:22:27.322 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 14:22:27.322 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 14:22:27.322 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 14:22:27.322 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:22:27.322 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 14:22:27.322 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:22:27.322 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 14:22:27.322 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 14:22:27.322 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 14:22:27.322 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 14:22:27.322 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:22:27.322 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 14:22:27.322 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:22:27.322 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:22:27.322 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:22:27.322 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 14:22:27.322 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 14:22:27.322 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:22:27.322 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 14:22:27.322 | INFO | ------------------------------------------ | 20260505 14:22:27.322 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:22:27.322 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 14:22:27.322 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:22:27.322 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:22:27.322 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.62e+01, 1.37e+01 / 1.40e+00 | 20260505 14:22:27.322 | INFO | PerfReport: 0.29%, AggregateStats -- 9.30e-02 / 9.93e-02, 1.10e-02 / 4.68e-03 | 20260505 14:22:27.322 | INFO | PerfReport: 11.82%, Baseline -- 3.75e+00 / 4.49e+00, 5.90e-01 / 1.63e-01 | 20260505 14:22:27.322 | INFO | PerfReport: 3.30%, Histogram -- 1.05e+00 / 1.51e+00, 1.98e-01 / 9.53e-02 | 20260505 14:22:27.322 | INFO | PerfReport: 15.96%, DME -- 4.05e+01 / 4.55e+01, 4.61e+00 / 1.78e+00 | 20260505 14:22:27.322 | INFO | PerfReport: 56.34%, FrameLabeling -- 1.79e+01 / 1.98e+01, 1.34e+00 / 1.84e+00 | 20260505 14:22:27.322 | INFO | PerfReport: 7.49%, PulseAccumulating -- 2.38e+00 / 2.90e+00, 4.60e-01 / 2.08e-01 | 20260505 14:22:27.322 | INFO | PerfReport: 3.55%, Metrics -- 1.13e+00 / 1.50e+00, 3.16e-01 / 1.97e-01 | 20260505 14:22:27.322 | INFO | PerfReport: 1.24%, PulseSerializing -- 3.94e-01 / 4.48e-01, 6.98e-02 / 2.94e-02 | 20260505 14:22:27.322 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 14:22:27.322 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:22:27.322 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:22:27.322 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260505 14:22:27.322 | INFO | PerfReport: 22.63%, Upload -- 2.28e+01 / 0.00e+00, 8.12e+00 / 0.00e+00 | 20260505 14:22:27.322 | INFO | PerfReport: 22.03%, Download -- 2.22e+01 / 0.00e+00, 8.10e+00 / 0.00e+00 | 20260505 14:22:27.322 | INFO | PerfReport: 31.57%, Compute -- 3.18e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 14:22:27.322 | INFO | PerfReport: 2.27%, LockUpload -- 2.28e+00 / 0.00e+00, 7.96e+00 / 0.00e+00 | 20260505 14:22:27.322 | INFO | PerfReport: 1.45%, LockDownload -- 1.46e+00 / 0.00e+00, 2.99e+00 / 0.00e+00 | 20260505 14:22:27.322 | INFO | PerfReport: 20.05%, LockCompute -- 2.02e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260505 14:22:27.322 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.86e-02 / 0.00e+00 | 20260505 14:22:27.452 | INFO | Starting a round of Baz IO | 20260505 14:22:30.670 | INFO | Populating RTMetrics report | 20260505 14:22:30.671 | INFO | RTMetrics report populated | 20260505 14:22:30.671 | INFO | Opening tmp RTMetrics json file | 20260505 14:22:30.683 | INFO | Serializing RTMetrics to json | 20260505 14:22:30.686 | INFO | Serialization done, swapping tmp file now... | 20260505 14:22:30.686 | INFO | RTMetrics done | 20260505 14:22:30.773 | INFO | Finished a round of Baz IO: Wrote 2.26 GiB in 3.32 seconds | 20260505 14:22:30.773 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.78% / 0.00% / 77.22% | 20260505 14:22:30.773 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.569709% / 3.97364e-06% / 9.13938e-05% | 20260505 14:22:30.773 | INFO | Baz IO overall: | 20260505 14:22:30.773 | INFO | BytesWritten : 2318309861054 | 20260505 14:22:30.773 | INFO | NumEvents : 1102293782295 | 20260505 14:22:32.465 | INFO | Analyzing chunk frames = [3891200,3891712) | 20260505 14:22:32.465 | INFO | MemInfo: { | 20260505 14:22:32.465 | INFO | "gpu" : " 17 GB", | 20260505 14:22:32.465 | INFO | "huge" : "", | 20260505 14:22:32.465 | INFO | "huge_shared" : " 27 GB", | 20260505 14:22:32.465 | INFO | "malloc" : " 14 GB", | 20260505 14:22:32.465 | INFO | "pinned" : " 29 GB" | 20260505 14:22:32.465 | INFO | } | 20260505 14:22:32.465 | INFO | Perf report for the last 5.14446 seconds: | 20260505 14:22:32.465 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:22:32.465 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:32.465 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:32.465 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:32.465 | INFO | ANALYSIS : 64.0%, 3.0, 99.3ms, 0 | 20260505 14:22:32.465 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:32.465 | INFO | RT_METRICS : 45.3%, 1.0, 23.8ms, 0 | 20260505 14:22:32.465 | INFO | PRE_HQ : 60.9%, 1.8, 57.4ms, 0 | 20260505 14:22:32.465 | INFO | BAZWRITER : 63.9%, 4.1, 133.6ms, 0 | 20260505 14:22:32.465 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 14:22:32.465 | INFO | Uploaded 36GiB to GPU in 2.18393s (16.484GiB/s) | 20260505 14:22:32.465 | INFO | Downloaded 38GiB to GPU in 2.25908s (16.821GiB/s) | 20260505 14:22:37.595 | INFO | Analyzing chunk frames = [3891712,3892224) | 20260505 14:22:37.595 | INFO | MemInfo: { | 20260505 14:22:37.595 | INFO | "gpu" : " 17 GB", | 20260505 14:22:37.595 | INFO | "huge" : "", | 20260505 14:22:37.595 | INFO | "huge_shared" : " 27 GB", | 20260505 14:22:37.596 | INFO | "malloc" : " 14 GB", | 20260505 14:22:37.596 | INFO | "pinned" : " 29 GB" | 20260505 14:22:37.596 | INFO | } | 20260505 14:22:37.596 | INFO | Perf report for the last 5.13082 seconds: | 20260505 14:22:37.596 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:22:37.596 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:37.596 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:37.596 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:37.596 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 14:22:37.596 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:37.596 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:37.596 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:22:37.596 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:37.596 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:37.596 | INFO | Uploaded 36GiB to GPU in 2.19069s (16.4332GiB/s) | 20260505 14:22:37.597 | INFO | Downloaded 34GiB to GPU in 2.03375s (16.7179GiB/s) | 20260505 14:22:42.728 | INFO | Analyzing chunk frames = [3892224,3892736) | 20260505 14:22:42.728 | INFO | MemInfo: { | 20260505 14:22:42.728 | INFO | "gpu" : " 17 GB", | 20260505 14:22:42.728 | INFO | "huge" : "", | 20260505 14:22:42.728 | INFO | "huge_shared" : " 27 GB", | 20260505 14:22:42.728 | INFO | "malloc" : " 14 GB", | 20260505 14:22:42.728 | INFO | "pinned" : " 29 GB" | 20260505 14:22:42.728 | INFO | } | 20260505 14:22:42.728 | INFO | Perf report for the last 5.13293 seconds: | 20260505 14:22:42.729 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:22:42.729 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:42.729 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:42.729 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:42.729 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260505 14:22:42.729 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:42.729 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:42.729 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:22:42.729 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:42.729 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:42.729 | INFO | Uploaded 36GiB to GPU in 2.23646s (16.0968GiB/s) | 20260505 14:22:42.729 | INFO | Downloaded 34GiB to GPU in 2.11846s (16.0494GiB/s) | 20260505 14:22:47.878 | INFO | Analyzing chunk frames = [3892736,3893248) | 20260505 14:22:47.878 | INFO | MemInfo: { | 20260505 14:22:47.878 | INFO | "gpu" : " 17 GB", | 20260505 14:22:47.878 | INFO | "huge" : "", | 20260505 14:22:47.878 | INFO | "huge_shared" : " 27 GB", | 20260505 14:22:47.878 | INFO | "malloc" : " 14 GB", | 20260505 14:22:47.878 | INFO | "pinned" : " 29 GB" | 20260505 14:22:47.878 | INFO | } | 20260505 14:22:47.878 | INFO | Perf report for the last 5.14979 seconds: | 20260505 14:22:47.878 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:22:47.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:47.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:47.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:47.879 | INFO | ANALYSIS : 62.9%, 3.0, 97.9ms, 0 | 20260505 14:22:47.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:47.879 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:47.879 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 14:22:47.879 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:47.879 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:47.879 | INFO | Uploaded 36GiB to GPU in 2.20029s (16.3615GiB/s) | 20260505 14:22:47.879 | INFO | Downloaded 34GiB to GPU in 2.13393s (15.933GiB/s) | 20260505 14:22:53.019 | INFO | Analyzing chunk frames = [3893248,3893760) | 20260505 14:22:53.019 | INFO | MemInfo: { | 20260505 14:22:53.019 | INFO | "gpu" : " 17 GB", | 20260505 14:22:53.019 | INFO | "huge" : "", | 20260505 14:22:53.019 | INFO | "huge_shared" : " 27 GB", | 20260505 14:22:53.019 | INFO | "malloc" : " 14 GB", | 20260505 14:22:53.019 | INFO | "pinned" : " 29 GB" | 20260505 14:22:53.019 | INFO | } | 20260505 14:22:53.019 | INFO | Perf report for the last 5.14087 seconds: | 20260505 14:22:53.019 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:22:53.019 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:53.019 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:53.019 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:53.019 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 14:22:53.019 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:53.019 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:53.019 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:22:53.019 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:53.019 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:53.019 | INFO | Uploaded 36GiB to GPU in 2.15602s (16.6975GiB/s) | 20260505 14:22:53.019 | INFO | Downloaded 34GiB to GPU in 2.12924s (15.9682GiB/s) | 20260505 14:22:58.155 | INFO | Analyzing chunk frames = [3893760,3894272) | 20260505 14:22:58.155 | INFO | MemInfo: { | 20260505 14:22:58.155 | INFO | "gpu" : " 17 GB", | 20260505 14:22:58.155 | INFO | "huge" : "", | 20260505 14:22:58.155 | INFO | "huge_shared" : " 27 GB", | 20260505 14:22:58.155 | INFO | "malloc" : " 14 GB", | 20260505 14:22:58.155 | INFO | "pinned" : " 29 GB" | 20260505 14:22:58.155 | INFO | } | 20260505 14:22:58.155 | INFO | Perf report for the last 5.13628 seconds: | 20260505 14:22:58.156 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:22:58.156 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:58.156 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:58.156 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:58.156 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 14:22:58.156 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:58.156 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:58.156 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:22:58.156 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:58.156 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:22:58.156 | INFO | Uploaded 36GiB to GPU in 2.21012s (16.2887GiB/s) | 20260505 14:22:58.156 | INFO | Downloaded 34GiB to GPU in 2.1126s (16.0939GiB/s) | 20260505 14:23:03.298 | INFO | Analyzing chunk frames = [3894272,3894784) | 20260505 14:23:03.298 | INFO | MemInfo: { | 20260505 14:23:03.298 | INFO | "gpu" : " 17 GB", | 20260505 14:23:03.298 | INFO | "huge" : "", | 20260505 14:23:03.298 | INFO | "huge_shared" : " 27 GB", | 20260505 14:23:03.298 | INFO | "malloc" : " 14 GB", | 20260505 14:23:03.298 | INFO | "pinned" : " 29 GB" | 20260505 14:23:03.298 | INFO | } | 20260505 14:23:03.298 | INFO | Perf report for the last 5.14224 seconds: | 20260505 14:23:03.298 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:23:03.298 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:03.298 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:03.298 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:03.298 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 14:23:03.298 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:03.298 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:03.298 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 14:23:03.298 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:03.298 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:03.298 | INFO | Uploaded 36GiB to GPU in 2.12148s (16.9692GiB/s) | 20260505 14:23:03.298 | INFO | Downloaded 34GiB to GPU in 2.16681s (15.6913GiB/s) | 20260505 14:23:08.433 | INFO | Analyzing chunk frames = [3894784,3895296) | 20260505 14:23:08.433 | INFO | MemInfo: { | 20260505 14:23:08.433 | INFO | "gpu" : " 17 GB", | 20260505 14:23:08.433 | INFO | "huge" : "", | 20260505 14:23:08.433 | INFO | "huge_shared" : " 27 GB", | 20260505 14:23:08.433 | INFO | "malloc" : " 14 GB", | 20260505 14:23:08.433 | INFO | "pinned" : " 29 GB" | 20260505 14:23:08.433 | INFO | } | 20260505 14:23:08.433 | INFO | Perf report for the last 5.13538 seconds: | 20260505 14:23:08.433 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:23:08.433 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:08.433 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:08.433 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:08.433 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 14:23:08.433 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:08.433 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:08.433 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:23:08.433 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:08.433 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:08.433 | INFO | Uploaded 36GiB to GPU in 2.21662s (16.2409GiB/s) | 20260505 14:23:08.433 | INFO | Downloaded 34GiB to GPU in 2.09705s (16.2132GiB/s) | 20260505 14:23:08.569 | INFO | Starting a round of Baz IO | 20260505 14:23:11.790 | INFO | Populating RTMetrics report | 20260505 14:23:11.790 | INFO | RTMetrics report populated | 20260505 14:23:11.790 | INFO | Opening tmp RTMetrics json file | 20260505 14:23:11.813 | INFO | Serializing RTMetrics to json | 20260505 14:23:11.814 | INFO | Serialization done, swapping tmp file now... | 20260505 14:23:11.815 | INFO | RTMetrics done | 20260505 14:23:11.902 | INFO | Finished a round of Baz IO: Wrote 2.26 GiB in 3.333 seconds | 20260505 14:23:11.902 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.78% / 0.00% / 77.22% | 20260505 14:23:11.902 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.567317% / 3.97364e-06% / 8.34465e-05% | 20260505 14:23:11.902 | INFO | Baz IO overall: | 20260505 14:23:11.902 | INFO | BytesWritten : 2320736827071 | 20260505 14:23:11.902 | INFO | NumEvents : 1103439072036 | 20260505 14:23:13.576 | INFO | Analyzing chunk frames = [3895296,3895808) | 20260505 14:23:13.576 | INFO | MemInfo: { | 20260505 14:23:13.576 | INFO | "gpu" : " 17 GB", | 20260505 14:23:13.576 | INFO | "huge" : "", | 20260505 14:23:13.576 | INFO | "huge_shared" : " 27 GB", | 20260505 14:23:13.576 | INFO | "malloc" : " 14 GB", | 20260505 14:23:13.576 | INFO | "pinned" : " 29 GB" | 20260505 14:23:13.576 | INFO | } | 20260505 14:23:13.576 | INFO | Perf report for the last 5.14322 seconds: | 20260505 14:23:13.576 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:23:13.576 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:13.576 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:13.576 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:13.576 | INFO | ANALYSIS : 64.2%, 3.0, 99.5ms, 0 | 20260505 14:23:13.576 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:13.576 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260505 14:23:13.576 | INFO | PRE_HQ : 60.8%, 1.8, 57.1ms, 0 | 20260505 14:23:13.576 | INFO | BAZWRITER : 64.2%, 4.0, 132.5ms, 0 | 20260505 14:23:13.576 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 14:23:13.577 | INFO | Uploaded 36GiB to GPU in 2.20891s (16.2976GiB/s) | 20260505 14:23:13.577 | INFO | Downloaded 38GiB to GPU in 2.26446s (16.781GiB/s) | 20260505 14:23:18.719 | INFO | Analyzing chunk frames = [3895808,3896320) | 20260505 14:23:18.719 | INFO | MemInfo: { | 20260505 14:23:18.719 | INFO | "gpu" : " 17 GB", | 20260505 14:23:18.719 | INFO | "huge" : "", | 20260505 14:23:18.719 | INFO | "huge_shared" : " 27 GB", | 20260505 14:23:18.719 | INFO | "malloc" : " 14 GB", | 20260505 14:23:18.719 | INFO | "pinned" : " 29 GB" | 20260505 14:23:18.719 | INFO | } | 20260505 14:23:18.719 | INFO | Perf report for the last 5.14296 seconds: | 20260505 14:23:18.719 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:23:18.719 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:18.719 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:18.719 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:18.719 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 14:23:18.719 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:18.719 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:18.719 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:23:18.719 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:18.719 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:18.720 | INFO | Uploaded 36GiB to GPU in 2.13043s (16.898GiB/s) | 20260505 14:23:18.720 | INFO | Downloaded 34GiB to GPU in 2.03984s (16.668GiB/s) | 20260505 14:23:23.848 | INFO | Analyzing chunk frames = [3896320,3896832) | 20260505 14:23:23.848 | INFO | MemInfo: { | 20260505 14:23:23.848 | INFO | "gpu" : " 17 GB", | 20260505 14:23:23.848 | INFO | "huge" : "", | 20260505 14:23:23.848 | INFO | "huge_shared" : " 27 GB", | 20260505 14:23:23.848 | INFO | "malloc" : " 14 GB", | 20260505 14:23:23.848 | INFO | "pinned" : " 29 GB" | 20260505 14:23:23.848 | INFO | } | 20260505 14:23:23.848 | INFO | Perf report for the last 5.12886 seconds: | 20260505 14:23:23.848 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:23:23.848 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:23.848 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:23.848 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:23.848 | INFO | ANALYSIS : 63.1%, 3.0, 98.1ms, 0 | 20260505 14:23:23.848 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:23.848 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:23.848 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:23:23.848 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:23.848 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:23.848 | INFO | Uploaded 36GiB to GPU in 2.18104s (16.5059GiB/s) | 20260505 14:23:23.848 | INFO | Downloaded 34GiB to GPU in 2.05312s (16.5601GiB/s) | 20260505 14:23:28.980 | INFO | Analyzing chunk frames = [3896832,3897344) | 20260505 14:23:28.980 | INFO | MemInfo: { | 20260505 14:23:28.980 | INFO | "gpu" : " 17 GB", | 20260505 14:23:28.980 | INFO | "huge" : "", | 20260505 14:23:28.980 | INFO | "huge_shared" : " 27 GB", | 20260505 14:23:28.980 | INFO | "malloc" : " 14 GB", | 20260505 14:23:28.980 | INFO | "pinned" : " 29 GB" | 20260505 14:23:28.980 | INFO | } | 20260505 14:23:28.980 | INFO | Perf report for the last 5.13175 seconds: | 20260505 14:23:28.980 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:23:28.980 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:28.980 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:28.980 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:28.980 | INFO | ANALYSIS : 63.1%, 3.0, 97.8ms, 0 | 20260505 14:23:28.980 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:28.980 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:28.980 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:23:28.980 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:28.980 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:28.980 | INFO | Uploaded 36GiB to GPU in 2.20251s (16.345GiB/s) | 20260505 14:23:28.980 | INFO | Downloaded 34GiB to GPU in 2.13607s (15.9171GiB/s) | 20260505 14:23:34.131 | INFO | Analyzing chunk frames = [3897344,3897856) | 20260505 14:23:34.131 | INFO | MemInfo: { | 20260505 14:23:34.131 | INFO | "gpu" : " 17 GB", | 20260505 14:23:34.131 | INFO | "huge" : "", | 20260505 14:23:34.131 | INFO | "huge_shared" : " 27 GB", | 20260505 14:23:34.131 | INFO | "malloc" : " 14 GB", | 20260505 14:23:34.131 | INFO | "pinned" : " 29 GB" | 20260505 14:23:34.131 | INFO | } | 20260505 14:23:34.132 | INFO | Perf report for the last 5.15157 seconds: | 20260505 14:23:34.132 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:23:34.132 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:34.132 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:34.132 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:34.132 | INFO | ANALYSIS : 62.7%, 3.0, 98.1ms, 0 | 20260505 14:23:34.132 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:34.132 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:34.132 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:23:34.132 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:34.132 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:34.132 | INFO | Uploaded 36GiB to GPU in 2.16004s (16.6663GiB/s) | 20260505 14:23:34.132 | INFO | Downloaded 34GiB to GPU in 2.12813s (15.9765GiB/s) | 20260505 14:23:39.276 | INFO | Analyzing chunk frames = [3897856,3898368) | 20260505 14:23:39.276 | INFO | MemInfo: { | 20260505 14:23:39.276 | INFO | "gpu" : " 17 GB", | 20260505 14:23:39.276 | INFO | "huge" : "", | 20260505 14:23:39.276 | INFO | "huge_shared" : " 27 GB", | 20260505 14:23:39.276 | INFO | "malloc" : " 14 GB", | 20260505 14:23:39.276 | INFO | "pinned" : " 29 GB" | 20260505 14:23:39.276 | INFO | } | 20260505 14:23:39.276 | INFO | Perf report for the last 5.14467 seconds: | 20260505 14:23:39.276 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:23:39.276 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:39.276 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:39.276 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:39.276 | INFO | ANALYSIS : 62.5%, 3.0, 97.8ms, 0 | 20260505 14:23:39.276 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:39.276 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:39.276 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:23:39.276 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:39.276 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:39.276 | INFO | Uploaded 36GiB to GPU in 2.13503s (16.8616GiB/s) | 20260505 14:23:39.277 | INFO | Downloaded 34GiB to GPU in 2.02939s (16.7538GiB/s) | 20260505 14:23:44.408 | INFO | Analyzing chunk frames = [3898368,3898880) | 20260505 14:23:44.408 | INFO | MemInfo: { | 20260505 14:23:44.408 | INFO | "gpu" : " 17 GB", | 20260505 14:23:44.408 | INFO | "huge" : "", | 20260505 14:23:44.408 | INFO | "huge_shared" : " 27 GB", | 20260505 14:23:44.408 | INFO | "malloc" : " 14 GB", | 20260505 14:23:44.408 | INFO | "pinned" : " 29 GB" | 20260505 14:23:44.408 | INFO | } | 20260505 14:23:44.408 | INFO | Perf report for the last 5.13202 seconds: | 20260505 14:23:44.408 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:23:44.408 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:44.408 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:44.408 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:44.408 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 14:23:44.408 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:44.408 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:44.408 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260505 14:23:44.408 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:44.408 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:44.409 | INFO | Uploaded 36GiB to GPU in 2.13128s (16.8913GiB/s) | 20260505 14:23:44.409 | INFO | Downloaded 34GiB to GPU in 2.16275s (15.7207GiB/s) | 20260505 14:23:49.540 | INFO | Analyzing chunk frames = [3898880,3899392) | 20260505 14:23:49.540 | INFO | MemInfo: { | 20260505 14:23:49.540 | INFO | "gpu" : " 17 GB", | 20260505 14:23:49.540 | INFO | "huge" : "", | 20260505 14:23:49.540 | INFO | "huge_shared" : " 27 GB", | 20260505 14:23:49.540 | INFO | "malloc" : " 14 GB", | 20260505 14:23:49.540 | INFO | "pinned" : " 29 GB" | 20260505 14:23:49.540 | INFO | } | 20260505 14:23:49.540 | INFO | Perf report for the last 5.13197 seconds: | 20260505 14:23:49.540 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:23:49.540 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:49.540 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:49.540 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:49.540 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 14:23:49.540 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:49.540 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:49.540 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 14:23:49.540 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:49.540 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:49.540 | INFO | Uploaded 36GiB to GPU in 2.20915s (16.2958GiB/s) | 20260505 14:23:49.540 | INFO | Downloaded 34GiB to GPU in 2.10744s (16.1333GiB/s) | 20260505 14:23:49.541 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 14:23:49.541 | INFO | Total mem usage: 15.3 GiB | 20260505 14:23:49.541 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 14:23:49.541 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.31%) | 20260505 14:23:49.541 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 14:23:49.541 | INFO | High Water Breakdown: | 20260505 14:23:49.541 | INFO | 61.2% : 8.83 GiB ( 92.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 14:23:49.541 | INFO | 38.6% : 5.57 GiB ( 83.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 14:23:49.541 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 14:23:49.541 | INFO | ------------------------------------------ | 20260505 14:23:49.541 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:23:49.541 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 14:23:49.541 | INFO | Total mem usage: 26.5 GiB | 20260505 14:23:49.541 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 14:23:49.541 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 14:23:49.541 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 14:23:49.541 | INFO | High Water Breakdown: | 20260505 14:23:49.541 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:23:49.541 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:23:49.541 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:23:49.541 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:23:49.541 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:23:49.541 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:23:49.541 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 14:23:49.541 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 14:23:49.541 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:23:49.541 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 14:23:49.541 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 14:23:49.541 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 14:23:49.541 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 14:23:49.541 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 14:23:49.541 | INFO | ------------------------------------------ | 20260505 14:23:49.541 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:23:49.541 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 14:23:49.541 | INFO | Total mem usage: 25.6 GiB | 20260505 14:23:49.541 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 14:23:49.541 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 14:23:49.541 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 14:23:49.541 | INFO | High Water Breakdown: | 20260505 14:23:49.541 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 14:23:49.541 | INFO | ------------------------------------------ | 20260505 14:23:49.541 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:23:49.541 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 14:23:49.541 | INFO | Total mem usage: 15.7 GiB | 20260505 14:23:49.541 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 14:23:49.541 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 14:23:49.541 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 14:23:49.541 | INFO | High Water Breakdown: | 20260505 14:23:49.541 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 14:23:49.541 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 14:23:49.541 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 14:23:49.541 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 14:23:49.541 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 14:23:49.541 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 14:23:49.541 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 14:23:49.541 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 14:23:49.541 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 14:23:49.541 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:23:49.541 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 14:23:49.541 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:23:49.541 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 14:23:49.541 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 14:23:49.541 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 14:23:49.541 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 14:23:49.541 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:23:49.541 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 14:23:49.541 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:23:49.541 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:23:49.541 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:23:49.541 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 14:23:49.541 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 14:23:49.541 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:23:49.541 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 14:23:49.541 | INFO | ------------------------------------------ | 20260505 14:23:49.541 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:23:49.541 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 14:23:49.541 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:23:49.541 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:23:49.541 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.64e+01, 1.36e+01 / 1.63e+00 | 20260505 14:23:49.541 | INFO | PerfReport: 0.29%, AggregateStats -- 9.31e-02 / 9.99e-02, 1.20e-02 / 4.29e-03 | 20260505 14:23:49.541 | INFO | PerfReport: 11.78%, Baseline -- 3.75e+00 / 4.52e+00, 5.90e-01 / 1.21e-01 | 20260505 14:23:49.541 | INFO | PerfReport: 3.31%, Histogram -- 1.05e+00 / 1.53e+00, 2.01e-01 / 7.81e-02 | 20260505 14:23:49.541 | INFO | PerfReport: 15.95%, DME -- 4.06e+01 / 4.56e+01, 4.59e+00 / 1.72e+00 | 20260505 14:23:49.541 | INFO | PerfReport: 56.48%, FrameLabeling -- 1.80e+01 / 1.98e+01, 1.44e+00 / 2.00e+00 | 20260505 14:23:49.541 | INFO | PerfReport: 7.43%, PulseAccumulating -- 2.36e+00 / 2.96e+00, 4.52e-01 / 1.41e-01 | 20260505 14:23:49.541 | INFO | PerfReport: 3.53%, Metrics -- 1.12e+00 / 1.45e+00, 3.14e-01 / 1.50e-01 | 20260505 14:23:49.541 | INFO | PerfReport: 1.24%, PulseSerializing -- 3.94e-01 / 4.46e-01, 6.97e-02 / 2.98e-02 | 20260505 14:23:49.541 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 14:23:49.541 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:23:49.541 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:23:49.541 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260505 14:23:49.541 | INFO | PerfReport: 22.56%, Upload -- 2.27e+01 / 0.00e+00, 8.07e+00 / 0.00e+00 | 20260505 14:23:49.541 | INFO | PerfReport: 21.97%, Download -- 2.21e+01 / 0.00e+00, 8.03e+00 / 0.00e+00 | 20260505 14:23:49.541 | INFO | PerfReport: 31.63%, Compute -- 3.18e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 14:23:49.541 | INFO | PerfReport: 2.28%, LockUpload -- 2.30e+00 / 0.00e+00, 8.00e+00 / 0.00e+00 | 20260505 14:23:49.541 | INFO | PerfReport: 1.36%, LockDownload -- 1.37e+00 / 0.00e+00, 2.85e+00 / 0.00e+00 | 20260505 14:23:49.542 | INFO | PerfReport: 20.18%, LockCompute -- 2.03e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 14:23:49.542 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 2.20e-02 / 0.00e+00 | 20260505 14:23:49.681 | INFO | Starting a round of Baz IO | 20260505 14:23:52.902 | INFO | Populating RTMetrics report | 20260505 14:23:52.902 | INFO | RTMetrics report populated | 20260505 14:23:52.902 | INFO | Opening tmp RTMetrics json file | 20260505 14:23:52.917 | INFO | Serializing RTMetrics to json | 20260505 14:23:52.921 | INFO | Serialization done, swapping tmp file now... | 20260505 14:23:52.926 | INFO | RTMetrics done | 20260505 14:23:53.007 | INFO | Finished a round of Baz IO: Wrote 2.257 GiB in 3.325 seconds | 20260505 14:23:53.007 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.77% / 0.00% / 77.22% | 20260505 14:23:53.007 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.564055% / 0% / 7.15256e-05% | 20260505 14:23:53.007 | INFO | Baz IO overall: | 20260505 14:23:53.007 | INFO | BytesWritten : 2323159901561 | 20260505 14:23:53.007 | INFO | NumEvents : 1104581493801 | 20260505 14:23:54.686 | INFO | Analyzing chunk frames = [3899392,3899904) | 20260505 14:23:54.686 | INFO | MemInfo: { | 20260505 14:23:54.686 | INFO | "gpu" : " 17 GB", | 20260505 14:23:54.686 | INFO | "huge" : "", | 20260505 14:23:54.686 | INFO | "huge_shared" : " 27 GB", | 20260505 14:23:54.686 | INFO | "malloc" : " 14 GB", | 20260505 14:23:54.686 | INFO | "pinned" : " 29 GB" | 20260505 14:23:54.686 | INFO | } | 20260505 14:23:54.686 | INFO | Perf report for the last 5.14556 seconds: | 20260505 14:23:54.686 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:23:54.686 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:54.686 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:54.686 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:54.686 | INFO | ANALYSIS : 64.2%, 3.0, 99.8ms, 0 | 20260505 14:23:54.686 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:54.686 | INFO | RT_METRICS : 45.3%, 1.0, 23.8ms, 0 | 20260505 14:23:54.686 | INFO | PRE_HQ : 60.6%, 1.8, 57.3ms, 0 | 20260505 14:23:54.686 | INFO | BAZWRITER : 64.0%, 4.0, 131.5ms, 0 | 20260505 14:23:54.686 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 14:23:54.686 | INFO | Uploaded 36GiB to GPU in 2.19045s (16.435GiB/s) | 20260505 14:23:54.686 | INFO | Downloaded 38GiB to GPU in 2.28058s (16.6624GiB/s) | 20260505 14:23:59.826 | INFO | Analyzing chunk frames = [3899904,3900416) | 20260505 14:23:59.826 | INFO | MemInfo: { | 20260505 14:23:59.826 | INFO | "gpu" : " 17 GB", | 20260505 14:23:59.826 | INFO | "huge" : "", | 20260505 14:23:59.826 | INFO | "huge_shared" : " 27 GB", | 20260505 14:23:59.826 | INFO | "malloc" : " 14 GB", | 20260505 14:23:59.826 | INFO | "pinned" : " 29 GB" | 20260505 14:23:59.826 | INFO | } | 20260505 14:23:59.826 | INFO | Perf report for the last 5.14062 seconds: | 20260505 14:23:59.826 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:23:59.826 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:59.826 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:59.826 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:59.826 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 14:23:59.827 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:59.827 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:59.827 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:23:59.827 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:59.827 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:23:59.827 | INFO | Uploaded 36GiB to GPU in 2.19459s (16.404GiB/s) | 20260505 14:23:59.827 | INFO | Downloaded 34GiB to GPU in 2.03736s (16.6883GiB/s) | 20260505 14:24:04.968 | INFO | Analyzing chunk frames = [3900416,3900928) | 20260505 14:24:04.968 | INFO | MemInfo: { | 20260505 14:24:04.968 | INFO | "gpu" : " 17 GB", | 20260505 14:24:04.968 | INFO | "huge" : "", | 20260505 14:24:04.968 | INFO | "huge_shared" : " 27 GB", | 20260505 14:24:04.969 | INFO | "malloc" : " 14 GB", | 20260505 14:24:04.969 | INFO | "pinned" : " 29 GB" | 20260505 14:24:04.969 | INFO | } | 20260505 14:24:04.969 | INFO | Perf report for the last 5.1423 seconds: | 20260505 14:24:04.969 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:24:04.969 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:04.969 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:04.969 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:04.969 | INFO | ANALYSIS : 63.4%, 3.0, 98.8ms, 0 | 20260505 14:24:04.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:04.969 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:04.969 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:24:04.969 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:04.969 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:04.969 | INFO | Uploaded 36GiB to GPU in 2.23747s (16.0896GiB/s) | 20260505 14:24:04.969 | INFO | Downloaded 34GiB to GPU in 2.1012s (16.1812GiB/s) | 20260505 14:24:10.103 | INFO | Analyzing chunk frames = [3900928,3901440) | 20260505 14:24:10.103 | INFO | MemInfo: { | 20260505 14:24:10.103 | INFO | "gpu" : " 17 GB", | 20260505 14:24:10.103 | INFO | "huge" : "", | 20260505 14:24:10.103 | INFO | "huge_shared" : " 27 GB", | 20260505 14:24:10.103 | INFO | "malloc" : " 14 GB", | 20260505 14:24:10.103 | INFO | "pinned" : " 29 GB" | 20260505 14:24:10.103 | INFO | } | 20260505 14:24:10.103 | INFO | Perf report for the last 5.13454 seconds: | 20260505 14:24:10.103 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:24:10.103 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:10.103 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:10.103 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:10.103 | INFO | ANALYSIS : 62.8%, 3.0, 97.5ms, 0 | 20260505 14:24:10.103 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:10.103 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:10.103 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:24:10.103 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:10.103 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:10.104 | INFO | Uploaded 36GiB to GPU in 2.18583s (16.4697GiB/s) | 20260505 14:24:10.104 | INFO | Downloaded 34GiB to GPU in 2.12601s (15.9924GiB/s) | 20260505 14:24:15.244 | INFO | Analyzing chunk frames = [3901440,3901952) | 20260505 14:24:15.244 | INFO | MemInfo: { | 20260505 14:24:15.244 | INFO | "gpu" : " 17 GB", | 20260505 14:24:15.244 | INFO | "huge" : "", | 20260505 14:24:15.244 | INFO | "huge_shared" : " 27 GB", | 20260505 14:24:15.244 | INFO | "malloc" : " 14 GB", | 20260505 14:24:15.244 | INFO | "pinned" : " 29 GB" | 20260505 14:24:15.244 | INFO | } | 20260505 14:24:15.244 | INFO | Perf report for the last 5.14032 seconds: | 20260505 14:24:15.244 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:24:15.244 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:15.244 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:15.244 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:15.244 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 14:24:15.244 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:15.244 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:15.244 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:24:15.244 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:15.244 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:15.244 | INFO | Uploaded 36GiB to GPU in 2.15425s (16.7112GiB/s) | 20260505 14:24:15.244 | INFO | Downloaded 34GiB to GPU in 2.12614s (15.9914GiB/s) | 20260505 14:24:20.383 | INFO | Analyzing chunk frames = [3901952,3902464) | 20260505 14:24:20.383 | INFO | MemInfo: { | 20260505 14:24:20.383 | INFO | "gpu" : " 17 GB", | 20260505 14:24:20.384 | INFO | "huge" : "", | 20260505 14:24:20.384 | INFO | "huge_shared" : " 27 GB", | 20260505 14:24:20.384 | INFO | "malloc" : " 14 GB", | 20260505 14:24:20.384 | INFO | "pinned" : " 29 GB" | 20260505 14:24:20.384 | INFO | } | 20260505 14:24:20.384 | INFO | Perf report for the last 5.1401 seconds: | 20260505 14:24:20.384 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:24:20.384 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:20.384 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:20.384 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:20.384 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 14:24:20.384 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:20.384 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:20.384 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 14:24:20.384 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:20.384 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:20.384 | INFO | Uploaded 36GiB to GPU in 2.17589s (16.545GiB/s) | 20260505 14:24:20.384 | INFO | Downloaded 34GiB to GPU in 2.11341s (16.0877GiB/s) | 20260505 14:24:25.524 | INFO | Analyzing chunk frames = [3902464,3902976) | 20260505 14:24:25.524 | INFO | MemInfo: { | 20260505 14:24:25.524 | INFO | "gpu" : " 17 GB", | 20260505 14:24:25.524 | INFO | "huge" : "", | 20260505 14:24:25.524 | INFO | "huge_shared" : " 27 GB", | 20260505 14:24:25.524 | INFO | "malloc" : " 14 GB", | 20260505 14:24:25.524 | INFO | "pinned" : " 29 GB" | 20260505 14:24:25.524 | INFO | } | 20260505 14:24:25.524 | INFO | Perf report for the last 5.14027 seconds: | 20260505 14:24:25.524 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:24:25.524 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:25.524 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:25.524 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:25.524 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 14:24:25.524 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:25.524 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:25.524 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 14:24:25.524 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:25.524 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:25.524 | INFO | Uploaded 36GiB to GPU in 2.17892s (16.5219GiB/s) | 20260505 14:24:25.524 | INFO | Downloaded 34GiB to GPU in 2.15136s (15.8039GiB/s) | 20260505 14:24:30.664 | INFO | Analyzing chunk frames = [3902976,3903488) | 20260505 14:24:30.664 | INFO | MemInfo: { | 20260505 14:24:30.664 | INFO | "gpu" : " 17 GB", | 20260505 14:24:30.664 | INFO | "huge" : "", | 20260505 14:24:30.664 | INFO | "huge_shared" : " 27 GB", | 20260505 14:24:30.664 | INFO | "malloc" : " 14 GB", | 20260505 14:24:30.664 | INFO | "pinned" : " 29 GB" | 20260505 14:24:30.664 | INFO | } | 20260505 14:24:30.664 | INFO | Perf report for the last 5.14035 seconds: | 20260505 14:24:30.664 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:24:30.664 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:30.664 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:30.664 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:30.664 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 14:24:30.664 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:30.664 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:30.665 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 14:24:30.665 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:30.665 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:30.665 | INFO | Uploaded 36GiB to GPU in 2.21848s (16.2273GiB/s) | 20260505 14:24:30.665 | INFO | Downloaded 34GiB to GPU in 2.10601s (16.1443GiB/s) | 20260505 14:24:30.803 | INFO | Starting a round of Baz IO | 20260505 14:24:34.030 | INFO | Populating RTMetrics report | 20260505 14:24:34.030 | INFO | RTMetrics report populated | 20260505 14:24:34.030 | INFO | Opening tmp RTMetrics json file | 20260505 14:24:34.052 | INFO | Serializing RTMetrics to json | 20260505 14:24:34.072 | INFO | Serialization done, swapping tmp file now... | 20260505 14:24:34.094 | INFO | RTMetrics done | 20260505 14:24:34.134 | INFO | Finished a round of Baz IO: Wrote 2.257 GiB in 3.33 seconds | 20260505 14:24:34.134 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.77% / 0.00% / 77.23% | 20260505 14:24:34.134 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.562779% / 0% / 8.34465e-05% | 20260505 14:24:34.134 | INFO | Baz IO overall: | 20260505 14:24:34.134 | INFO | BytesWritten : 2325583280055 | 20260505 14:24:34.134 | INFO | NumEvents : 1105724331339 | 20260505 14:24:35.794 | INFO | Analyzing chunk frames = [3903488,3904000) | 20260505 14:24:35.794 | INFO | MemInfo: { | 20260505 14:24:35.794 | INFO | "gpu" : " 17 GB", | 20260505 14:24:35.794 | INFO | "huge" : "", | 20260505 14:24:35.794 | INFO | "huge_shared" : " 27 GB", | 20260505 14:24:35.794 | INFO | "malloc" : " 14 GB", | 20260505 14:24:35.794 | INFO | "pinned" : " 29 GB" | 20260505 14:24:35.794 | INFO | } | 20260505 14:24:35.794 | INFO | Perf report for the last 5.12953 seconds: | 20260505 14:24:35.794 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:24:35.794 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:35.794 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:35.794 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:35.794 | INFO | ANALYSIS : 64.5%, 3.0, 100.0ms, 0 | 20260505 14:24:35.794 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:35.794 | INFO | RT_METRICS : 46.2%, 1.0, 24.2ms, 0 | 20260505 14:24:35.794 | INFO | PRE_HQ : 60.9%, 1.8, 57.1ms, 0 | 20260505 14:24:35.794 | INFO | BAZWRITER : 64.2%, 4.0, 131.0ms, 0 | 20260505 14:24:35.794 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 14:24:35.794 | INFO | Uploaded 36GiB to GPU in 2.19555s (16.3968GiB/s) | 20260505 14:24:35.794 | INFO | Downloaded 38GiB to GPU in 2.27965s (16.6692GiB/s) | 20260505 14:24:40.935 | INFO | Analyzing chunk frames = [3904000,3904512) | 20260505 14:24:40.935 | INFO | MemInfo: { | 20260505 14:24:40.935 | INFO | "gpu" : " 17 GB", | 20260505 14:24:40.935 | INFO | "huge" : "", | 20260505 14:24:40.935 | INFO | "huge_shared" : " 27 GB", | 20260505 14:24:40.935 | INFO | "malloc" : " 14 GB", | 20260505 14:24:40.935 | INFO | "pinned" : " 29 GB" | 20260505 14:24:40.935 | INFO | } | 20260505 14:24:40.935 | INFO | Perf report for the last 5.14126 seconds: | 20260505 14:24:40.935 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:24:40.935 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:40.935 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:40.935 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:40.935 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 14:24:40.935 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:40.935 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:40.935 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:24:40.935 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:40.935 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:40.935 | INFO | Uploaded 36GiB to GPU in 2.1858s (16.4699GiB/s) | 20260505 14:24:40.935 | INFO | Downloaded 34GiB to GPU in 2.04052s (16.6624GiB/s) | 20260505 14:24:46.079 | INFO | Analyzing chunk frames = [3904512,3905024) | 20260505 14:24:46.080 | INFO | MemInfo: { | 20260505 14:24:46.080 | INFO | "gpu" : " 17 GB", | 20260505 14:24:46.080 | INFO | "huge" : "", | 20260505 14:24:46.080 | INFO | "huge_shared" : " 27 GB", | 20260505 14:24:46.080 | INFO | "malloc" : " 14 GB", | 20260505 14:24:46.080 | INFO | "pinned" : " 29 GB" | 20260505 14:24:46.080 | INFO | } | 20260505 14:24:46.080 | INFO | Perf report for the last 5.14455 seconds: | 20260505 14:24:46.080 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:24:46.080 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:46.080 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:46.080 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:46.080 | INFO | ANALYSIS : 63.6%, 3.0, 99.0ms, 0 | 20260505 14:24:46.080 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:46.080 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:46.080 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:24:46.080 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:46.080 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:46.080 | INFO | Uploaded 36GiB to GPU in 2.24043s (16.0683GiB/s) | 20260505 14:24:46.080 | INFO | Downloaded 34GiB to GPU in 2.10284s (16.1686GiB/s) | 20260505 14:24:51.224 | INFO | Analyzing chunk frames = [3905024,3905536) | 20260505 14:24:51.224 | INFO | MemInfo: { | 20260505 14:24:51.224 | INFO | "gpu" : " 17 GB", | 20260505 14:24:51.224 | INFO | "huge" : "", | 20260505 14:24:51.224 | INFO | "huge_shared" : " 27 GB", | 20260505 14:24:51.224 | INFO | "malloc" : " 14 GB", | 20260505 14:24:51.224 | INFO | "pinned" : " 29 GB" | 20260505 14:24:51.224 | INFO | } | 20260505 14:24:51.224 | INFO | Perf report for the last 5.14468 seconds: | 20260505 14:24:51.224 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:24:51.224 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:51.224 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:51.224 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:51.224 | INFO | ANALYSIS : 62.6%, 3.0, 97.3ms, 0 | 20260505 14:24:51.225 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:51.225 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:51.225 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:24:51.225 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:51.225 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:51.225 | INFO | Uploaded 36GiB to GPU in 2.16787s (16.6062GiB/s) | 20260505 14:24:51.225 | INFO | Downloaded 34GiB to GPU in 2.12747s (15.9814GiB/s) | 20260505 14:24:56.352 | INFO | Analyzing chunk frames = [3905536,3906048) | 20260505 14:24:56.352 | INFO | MemInfo: { | 20260505 14:24:56.352 | INFO | "gpu" : " 17 GB", | 20260505 14:24:56.352 | INFO | "huge" : "", | 20260505 14:24:56.353 | INFO | "huge_shared" : " 27 GB", | 20260505 14:24:56.353 | INFO | "malloc" : " 14 GB", | 20260505 14:24:56.353 | INFO | "pinned" : " 29 GB" | 20260505 14:24:56.353 | INFO | } | 20260505 14:24:56.353 | INFO | Perf report for the last 5.12821 seconds: | 20260505 14:24:56.353 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:24:56.353 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:56.353 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:56.353 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:56.353 | INFO | ANALYSIS : 62.9%, 3.0, 97.9ms, 0 | 20260505 14:24:56.353 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:56.353 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:56.353 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:24:56.353 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:56.353 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:24:56.353 | INFO | Uploaded 36GiB to GPU in 2.17553s (16.5477GiB/s) | 20260505 14:24:56.353 | INFO | Downloaded 34GiB to GPU in 2.13166s (15.95GiB/s) | 20260505 14:25:01.492 | INFO | Analyzing chunk frames = [3906048,3906560) | 20260505 14:25:01.492 | INFO | MemInfo: { | 20260505 14:25:01.492 | INFO | "gpu" : " 17 GB", | 20260505 14:25:01.492 | INFO | "huge" : "", | 20260505 14:25:01.492 | INFO | "huge_shared" : " 27 GB", | 20260505 14:25:01.492 | INFO | "malloc" : " 14 GB", | 20260505 14:25:01.492 | INFO | "pinned" : " 29 GB" | 20260505 14:25:01.492 | INFO | } | 20260505 14:25:01.492 | INFO | Perf report for the last 5.13957 seconds: | 20260505 14:25:01.492 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:25:01.492 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:01.492 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:01.492 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:01.492 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 14:25:01.492 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:01.492 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:01.492 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:25:01.492 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:01.492 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:01.493 | INFO | Uploaded 36GiB to GPU in 2.16347s (16.6399GiB/s) | 20260505 14:25:01.493 | INFO | Downloaded 34GiB to GPU in 2.11027s (16.1117GiB/s) | 20260505 14:25:06.641 | INFO | Analyzing chunk frames = [3906560,3907072) | 20260505 14:25:06.641 | INFO | MemInfo: { | 20260505 14:25:06.641 | INFO | "gpu" : " 17 GB", | 20260505 14:25:06.641 | INFO | "huge" : "", | 20260505 14:25:06.641 | INFO | "huge_shared" : " 27 GB", | 20260505 14:25:06.641 | INFO | "malloc" : " 14 GB", | 20260505 14:25:06.641 | INFO | "pinned" : " 29 GB" | 20260505 14:25:06.641 | INFO | } | 20260505 14:25:06.641 | INFO | Perf report for the last 5.14882 seconds: | 20260505 14:25:06.641 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:25:06.641 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:06.641 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:06.641 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:06.641 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 14:25:06.641 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:06.641 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:06.641 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 14:25:06.641 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:06.641 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:06.641 | INFO | Uploaded 36GiB to GPU in 2.16843s (16.6019GiB/s) | 20260505 14:25:06.641 | INFO | Downloaded 34GiB to GPU in 2.15318s (15.7906GiB/s) | 20260505 14:25:11.780 | INFO | Analyzing chunk frames = [3907072,3907584) | 20260505 14:25:11.780 | INFO | MemInfo: { | 20260505 14:25:11.780 | INFO | "gpu" : " 17 GB", | 20260505 14:25:11.780 | INFO | "huge" : "", | 20260505 14:25:11.780 | INFO | "huge_shared" : " 27 GB", | 20260505 14:25:11.780 | INFO | "malloc" : " 14 GB", | 20260505 14:25:11.780 | INFO | "pinned" : " 29 GB" | 20260505 14:25:11.780 | INFO | } | 20260505 14:25:11.780 | INFO | Perf report for the last 5.13923 seconds: | 20260505 14:25:11.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:25:11.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:11.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:11.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:11.780 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 14:25:11.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:11.780 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:11.780 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 14:25:11.780 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:11.780 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:11.781 | INFO | Uploaded 36GiB to GPU in 2.22622s (16.1709GiB/s) | 20260505 14:25:11.781 | INFO | Downloaded 34GiB to GPU in 2.09387s (16.2378GiB/s) | 20260505 14:25:11.781 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 14:25:11.781 | INFO | Total mem usage: 15.3 GiB | 20260505 14:25:11.781 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 14:25:11.781 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.7%) | 20260505 14:25:11.781 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 14:25:11.781 | INFO | High Water Breakdown: | 20260505 14:25:11.781 | INFO | 61% : 8.68 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 14:25:11.781 | INFO | 38.9% : 5.53 GiB ( 83.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 14:25:11.781 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 14:25:11.781 | INFO | ------------------------------------------ | 20260505 14:25:11.781 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:25:11.781 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 14:25:11.781 | INFO | Total mem usage: 26.5 GiB | 20260505 14:25:11.781 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 14:25:11.781 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 14:25:11.781 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 14:25:11.781 | INFO | High Water Breakdown: | 20260505 14:25:11.781 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:25:11.781 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:25:11.781 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:25:11.781 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:25:11.781 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:25:11.781 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:25:11.781 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 14:25:11.781 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 14:25:11.781 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:25:11.781 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 14:25:11.781 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 14:25:11.781 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 14:25:11.781 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 14:25:11.781 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 14:25:11.781 | INFO | ------------------------------------------ | 20260505 14:25:11.781 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:25:11.781 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 14:25:11.781 | INFO | Total mem usage: 25.6 GiB | 20260505 14:25:11.781 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 14:25:11.781 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 14:25:11.781 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 14:25:11.781 | INFO | High Water Breakdown: | 20260505 14:25:11.781 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 14:25:11.781 | INFO | ------------------------------------------ | 20260505 14:25:11.781 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:25:11.781 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 14:25:11.781 | INFO | Total mem usage: 15.7 GiB | 20260505 14:25:11.781 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 14:25:11.781 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 14:25:11.782 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 14:25:11.782 | INFO | High Water Breakdown: | 20260505 14:25:11.782 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 14:25:11.782 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 14:25:11.782 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 14:25:11.782 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 14:25:11.782 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 14:25:11.782 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 14:25:11.782 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 14:25:11.782 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 14:25:11.782 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 14:25:11.782 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:25:11.782 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 14:25:11.782 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:25:11.782 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 14:25:11.782 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 14:25:11.782 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 14:25:11.782 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 14:25:11.782 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:25:11.782 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 14:25:11.782 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:25:11.782 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:25:11.782 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:25:11.782 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 14:25:11.782 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 14:25:11.782 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:25:11.782 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 14:25:11.782 | INFO | ------------------------------------------ | 20260505 14:25:11.782 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:25:11.782 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 14:25:11.782 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:25:11.782 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:25:11.782 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.63e+01, 1.36e+01 / 1.42e+00 | 20260505 14:25:11.782 | INFO | PerfReport: 0.29%, AggregateStats -- 9.27e-02 / 9.81e-02, 1.04e-02 / 4.08e-03 | 20260505 14:25:11.782 | INFO | PerfReport: 11.78%, Baseline -- 3.74e+00 / 4.51e+00, 5.79e-01 / 1.79e-01 | 20260505 14:25:11.782 | INFO | PerfReport: 3.29%, Histogram -- 1.05e+00 / 1.49e+00, 1.95e-01 / 1.11e-01 | 20260505 14:25:11.782 | INFO | PerfReport: 15.98%, DME -- 4.06e+01 / 4.55e+01, 4.43e+00 / 1.85e+00 | 20260505 14:25:11.782 | INFO | PerfReport: 56.39%, FrameLabeling -- 1.79e+01 / 1.99e+01, 1.39e+00 / 1.99e+00 | 20260505 14:25:11.782 | INFO | PerfReport: 7.48%, PulseAccumulating -- 2.38e+00 / 2.90e+00, 4.54e-01 / 2.13e-01 | 20260505 14:25:11.782 | INFO | PerfReport: 3.55%, Metrics -- 1.13e+00 / 1.44e+00, 3.13e-01 / 1.45e-01 | 20260505 14:25:11.782 | INFO | PerfReport: 1.24%, PulseSerializing -- 3.94e-01 / 4.44e-01, 7.01e-02 / 3.00e-02 | 20260505 14:25:11.782 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 14:25:11.782 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:25:11.782 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:25:11.782 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260505 14:25:11.782 | INFO | PerfReport: 22.65%, Upload -- 2.28e+01 / 0.00e+00, 8.15e+00 / 0.00e+00 | 20260505 14:25:11.782 | INFO | PerfReport: 22.02%, Download -- 2.22e+01 / 0.00e+00, 8.07e+00 / 0.00e+00 | 20260505 14:25:11.782 | INFO | PerfReport: 31.53%, Compute -- 3.18e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 14:25:11.782 | INFO | PerfReport: 2.39%, LockUpload -- 2.41e+00 / 0.00e+00, 8.04e+00 / 0.00e+00 | 20260505 14:25:11.782 | INFO | PerfReport: 1.34%, LockDownload -- 1.35e+00 / 0.00e+00, 2.83e+00 / 0.00e+00 | 20260505 14:25:11.782 | INFO | PerfReport: 20.05%, LockCompute -- 2.02e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260505 14:25:11.782 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.84e-02 / 0.00e+00 | 20260505 14:25:11.916 | INFO | Starting a round of Baz IO | 20260505 14:25:15.136 | INFO | Populating RTMetrics report | 20260505 14:25:15.136 | INFO | RTMetrics report populated | 20260505 14:25:15.136 | INFO | Opening tmp RTMetrics json file | 20260505 14:25:15.152 | INFO | Serializing RTMetrics to json | 20260505 14:25:15.168 | INFO | Serialization done, swapping tmp file now... | 20260505 14:25:15.182 | INFO | RTMetrics done | 20260505 14:25:15.245 | INFO | Finished a round of Baz IO: Wrote 2.258 GiB in 3.328 seconds | 20260505 14:25:15.245 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.77% / 0.00% / 77.23% | 20260505 14:25:15.245 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.562155% / 0% / 8.74201e-05% | 20260505 14:25:15.245 | INFO | Baz IO overall: | 20260505 14:25:15.245 | INFO | BytesWritten : 2328008299130 | 20260505 14:25:15.245 | INFO | NumEvents : 1106868348168 | 20260505 14:25:16.918 | INFO | Analyzing chunk frames = [3907584,3908096) | 20260505 14:25:16.918 | INFO | MemInfo: { | 20260505 14:25:16.918 | INFO | "gpu" : " 17 GB", | 20260505 14:25:16.918 | INFO | "huge" : "", | 20260505 14:25:16.918 | INFO | "huge_shared" : " 27 GB", | 20260505 14:25:16.918 | INFO | "malloc" : " 14 GB", | 20260505 14:25:16.918 | INFO | "pinned" : " 29 GB" | 20260505 14:25:16.918 | INFO | } | 20260505 14:25:16.918 | INFO | Perf report for the last 5.13785 seconds: | 20260505 14:25:16.918 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:25:16.918 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:16.918 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:16.918 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:16.918 | INFO | ANALYSIS : 64.2%, 3.0, 99.5ms, 0 | 20260505 14:25:16.918 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:16.918 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 14:25:16.919 | INFO | PRE_HQ : 60.9%, 1.8, 57.5ms, 0 | 20260505 14:25:16.919 | INFO | BAZWRITER : 64.1%, 4.0, 132.0ms, 0 | 20260505 14:25:16.919 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260505 14:25:16.919 | INFO | Uploaded 36GiB to GPU in 2.20146s (16.3528GiB/s) | 20260505 14:25:16.919 | INFO | Downloaded 38GiB to GPU in 2.26273s (16.7939GiB/s) | 20260505 14:25:22.049 | INFO | Analyzing chunk frames = [3908096,3908608) | 20260505 14:25:22.049 | INFO | MemInfo: { | 20260505 14:25:22.049 | INFO | "gpu" : " 17 GB", | 20260505 14:25:22.049 | INFO | "huge" : "", | 20260505 14:25:22.049 | INFO | "huge_shared" : " 27 GB", | 20260505 14:25:22.049 | INFO | "malloc" : " 14 GB", | 20260505 14:25:22.049 | INFO | "pinned" : " 29 GB" | 20260505 14:25:22.050 | INFO | } | 20260505 14:25:22.050 | INFO | Perf report for the last 5.13155 seconds: | 20260505 14:25:22.050 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:25:22.050 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:22.050 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:22.050 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:22.050 | INFO | ANALYSIS : 63.6%, 3.0, 99.1ms, 0 | 20260505 14:25:22.050 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:22.050 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:22.050 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:25:22.050 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:22.050 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:22.050 | INFO | Uploaded 36GiB to GPU in 2.20283s (16.3426GiB/s) | 20260505 14:25:22.050 | INFO | Downloaded 34GiB to GPU in 2.0382s (16.6814GiB/s) | 20260505 14:25:27.196 | INFO | Analyzing chunk frames = [3908608,3909120) | 20260505 14:25:27.196 | INFO | MemInfo: { | 20260505 14:25:27.196 | INFO | "gpu" : " 17 GB", | 20260505 14:25:27.196 | INFO | "huge" : "", | 20260505 14:25:27.197 | INFO | "huge_shared" : " 27 GB", | 20260505 14:25:27.197 | INFO | "malloc" : " 14 GB", | 20260505 14:25:27.197 | INFO | "pinned" : " 29 GB" | 20260505 14:25:27.197 | INFO | } | 20260505 14:25:27.197 | INFO | Perf report for the last 5.14687 seconds: | 20260505 14:25:27.197 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:25:27.197 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:27.197 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:27.197 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:27.197 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 14:25:27.197 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:27.197 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:27.197 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 14:25:27.197 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:27.197 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:27.197 | INFO | Uploaded 36GiB to GPU in 2.23801s (16.0857GiB/s) | 20260505 14:25:27.197 | INFO | Downloaded 34GiB to GPU in 2.09675s (16.2156GiB/s) | 20260505 14:25:32.337 | INFO | Analyzing chunk frames = [3909120,3909632) | 20260505 14:25:32.337 | INFO | MemInfo: { | 20260505 14:25:32.337 | INFO | "gpu" : " 17 GB", | 20260505 14:25:32.337 | INFO | "huge" : "", | 20260505 14:25:32.337 | INFO | "huge_shared" : " 27 GB", | 20260505 14:25:32.337 | INFO | "malloc" : " 14 GB", | 20260505 14:25:32.337 | INFO | "pinned" : " 29 GB" | 20260505 14:25:32.337 | INFO | } | 20260505 14:25:32.337 | INFO | Perf report for the last 5.14032 seconds: | 20260505 14:25:32.337 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:25:32.337 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:32.337 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:32.337 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:32.337 | INFO | ANALYSIS : 62.8%, 3.0, 97.5ms, 0 | 20260505 14:25:32.337 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:32.337 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:32.337 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 14:25:32.337 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:32.337 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:32.337 | INFO | Uploaded 36GiB to GPU in 2.18628s (16.4663GiB/s) | 20260505 14:25:32.337 | INFO | Downloaded 34GiB to GPU in 2.12766s (15.98GiB/s) | 20260505 14:25:37.476 | INFO | Analyzing chunk frames = [3909632,3910144) | 20260505 14:25:37.476 | INFO | MemInfo: { | 20260505 14:25:37.476 | INFO | "gpu" : " 17 GB", | 20260505 14:25:37.476 | INFO | "huge" : "", | 20260505 14:25:37.476 | INFO | "huge_shared" : " 27 GB", | 20260505 14:25:37.476 | INFO | "malloc" : " 14 GB", | 20260505 14:25:37.476 | INFO | "pinned" : " 29 GB" | 20260505 14:25:37.476 | INFO | } | 20260505 14:25:37.476 | INFO | Perf report for the last 5.13933 seconds: | 20260505 14:25:37.476 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:25:37.476 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:37.476 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:37.476 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:37.476 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 14:25:37.476 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:37.476 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:37.476 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:25:37.476 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:37.476 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:37.477 | INFO | Uploaded 36GiB to GPU in 2.17464s (16.5545GiB/s) | 20260505 14:25:37.477 | INFO | Downloaded 34GiB to GPU in 2.13043s (15.9592GiB/s) | 20260505 14:25:42.612 | INFO | Analyzing chunk frames = [3910144,3910656) | 20260505 14:25:42.612 | INFO | MemInfo: { | 20260505 14:25:42.612 | INFO | "gpu" : " 17 GB", | 20260505 14:25:42.612 | INFO | "huge" : "", | 20260505 14:25:42.612 | INFO | "huge_shared" : " 27 GB", | 20260505 14:25:42.612 | INFO | "malloc" : " 14 GB", | 20260505 14:25:42.612 | INFO | "pinned" : " 29 GB" | 20260505 14:25:42.612 | INFO | } | 20260505 14:25:42.612 | INFO | Perf report for the last 5.13564 seconds: | 20260505 14:25:42.612 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:25:42.612 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:42.612 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:42.612 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:42.612 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 14:25:42.612 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:42.612 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:42.612 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 14:25:42.612 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:42.612 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:42.612 | INFO | Uploaded 36GiB to GPU in 2.2145s (16.2565GiB/s) | 20260505 14:25:42.612 | INFO | Downloaded 34GiB to GPU in 2.11174s (16.1004GiB/s) | 20260505 14:25:47.744 | INFO | Analyzing chunk frames = [3910656,3911168) | 20260505 14:25:47.744 | INFO | MemInfo: { | 20260505 14:25:47.744 | INFO | "gpu" : " 17 GB", | 20260505 14:25:47.744 | INFO | "huge" : "", | 20260505 14:25:47.744 | INFO | "huge_shared" : " 27 GB", | 20260505 14:25:47.744 | INFO | "malloc" : " 14 GB", | 20260505 14:25:47.744 | INFO | "pinned" : " 29 GB" | 20260505 14:25:47.744 | INFO | } | 20260505 14:25:47.744 | INFO | Perf report for the last 5.13184 seconds: | 20260505 14:25:47.744 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:25:47.744 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:47.744 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:47.744 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:47.744 | INFO | ANALYSIS : 63.8%, 3.0, 99.5ms, 0 | 20260505 14:25:47.744 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:47.744 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:47.744 | INFO | PRE_HQ : 29.5%, 1.0, 15.4ms, 0 | 20260505 14:25:47.744 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:47.744 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:47.744 | INFO | Uploaded 36GiB to GPU in 2.15028s (16.742GiB/s) | 20260505 14:25:47.744 | INFO | Downloaded 34GiB to GPU in 2.18222s (15.5805GiB/s) | 20260505 14:25:52.891 | INFO | Analyzing chunk frames = [3911168,3911680) | 20260505 14:25:52.891 | INFO | MemInfo: { | 20260505 14:25:52.891 | INFO | "gpu" : " 17 GB", | 20260505 14:25:52.891 | INFO | "huge" : "", | 20260505 14:25:52.891 | INFO | "huge_shared" : " 27 GB", | 20260505 14:25:52.891 | INFO | "malloc" : " 14 GB", | 20260505 14:25:52.891 | INFO | "pinned" : " 29 GB" | 20260505 14:25:52.891 | INFO | } | 20260505 14:25:52.891 | INFO | Perf report for the last 5.1474 seconds: | 20260505 14:25:52.891 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:25:52.891 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:52.891 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:52.891 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:52.891 | INFO | ANALYSIS : 63.5%, 3.0, 99.3ms, 0 | 20260505 14:25:52.891 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:52.891 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:52.891 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 14:25:52.891 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:52.891 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:52.891 | INFO | Uploaded 36GiB to GPU in 2.16198s (16.6514GiB/s) | 20260505 14:25:52.891 | INFO | Downloaded 34GiB to GPU in 2.10772s (16.1312GiB/s) | 20260505 14:25:53.027 | INFO | Starting a round of Baz IO | 20260505 14:25:56.243 | INFO | Populating RTMetrics report | 20260505 14:25:56.244 | INFO | RTMetrics report populated | 20260505 14:25:56.244 | INFO | Opening tmp RTMetrics json file | 20260505 14:25:56.280 | INFO | Serializing RTMetrics to json | 20260505 14:25:56.307 | INFO | Serialization done, swapping tmp file now... | 20260505 14:25:56.317 | INFO | RTMetrics done | 20260505 14:25:56.346 | INFO | Finished a round of Baz IO: Wrote 2.256 GiB in 3.318 seconds | 20260505 14:25:56.346 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.76% / 0.00% / 77.23% | 20260505 14:25:56.347 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.559564% / 0% / 9.13938e-05% | 20260505 14:25:56.347 | INFO | Baz IO overall: | 20260505 14:25:56.347 | INFO | BytesWritten : 2330431049171 | 20260505 14:25:56.347 | INFO | NumEvents : 1108010624325 | 20260505 14:25:58.032 | INFO | Analyzing chunk frames = [3911680,3912192) | 20260505 14:25:58.032 | INFO | MemInfo: { | 20260505 14:25:58.032 | INFO | "gpu" : " 17 GB", | 20260505 14:25:58.032 | INFO | "huge" : "", | 20260505 14:25:58.032 | INFO | "huge_shared" : " 27 GB", | 20260505 14:25:58.032 | INFO | "malloc" : " 14 GB", | 20260505 14:25:58.032 | INFO | "pinned" : " 29 GB" | 20260505 14:25:58.032 | INFO | } | 20260505 14:25:58.032 | INFO | Perf report for the last 5.14118 seconds: | 20260505 14:25:58.032 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:25:58.032 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:58.032 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:58.032 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:58.032 | INFO | ANALYSIS : 64.1%, 3.0, 99.6ms, 0 | 20260505 14:25:58.032 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:25:58.033 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260505 14:25:58.033 | INFO | PRE_HQ : 60.8%, 1.8, 57.2ms, 0 | 20260505 14:25:58.033 | INFO | BAZWRITER : 63.9%, 4.0, 130.6ms, 0 | 20260505 14:25:58.033 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 14:25:58.033 | INFO | Uploaded 36GiB to GPU in 2.18802s (16.4532GiB/s) | 20260505 14:25:58.033 | INFO | Downloaded 38GiB to GPU in 2.2668s (16.7637GiB/s) | 20260505 14:26:03.170 | INFO | Analyzing chunk frames = [3912192,3912704) | 20260505 14:26:03.170 | INFO | MemInfo: { | 20260505 14:26:03.170 | INFO | "gpu" : " 17 GB", | 20260505 14:26:03.170 | INFO | "huge" : "", | 20260505 14:26:03.170 | INFO | "huge_shared" : " 27 GB", | 20260505 14:26:03.170 | INFO | "malloc" : " 14 GB", | 20260505 14:26:03.170 | INFO | "pinned" : " 29 GB" | 20260505 14:26:03.170 | INFO | } | 20260505 14:26:03.170 | INFO | Perf report for the last 5.13752 seconds: | 20260505 14:26:03.170 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:26:03.170 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:03.170 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:03.170 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:03.170 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 14:26:03.170 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:03.170 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:03.170 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 14:26:03.170 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:03.170 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:03.170 | INFO | Uploaded 36GiB to GPU in 2.19515s (16.3998GiB/s) | 20260505 14:26:03.170 | INFO | Downloaded 34GiB to GPU in 2.03768s (16.6856GiB/s) | 20260505 14:26:08.303 | INFO | Analyzing chunk frames = [3912704,3913216) | 20260505 14:26:08.303 | INFO | MemInfo: { | 20260505 14:26:08.303 | INFO | "gpu" : " 17 GB", | 20260505 14:26:08.304 | INFO | "huge" : "", | 20260505 14:26:08.304 | INFO | "huge_shared" : " 27 GB", | 20260505 14:26:08.304 | INFO | "malloc" : " 14 GB", | 20260505 14:26:08.304 | INFO | "pinned" : " 29 GB" | 20260505 14:26:08.304 | INFO | } | 20260505 14:26:08.304 | INFO | Perf report for the last 5.13372 seconds: | 20260505 14:26:08.304 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:26:08.304 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:08.304 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:08.304 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:08.304 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 14:26:08.304 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:08.304 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:08.304 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:26:08.304 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:08.304 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:08.304 | INFO | Uploaded 36GiB to GPU in 2.22805s (16.1576GiB/s) | 20260505 14:26:08.304 | INFO | Downloaded 34GiB to GPU in 2.09116s (16.259GiB/s) | 20260505 14:26:13.450 | INFO | Analyzing chunk frames = [3913216,3913728) | 20260505 14:26:13.450 | INFO | MemInfo: { | 20260505 14:26:13.450 | INFO | "gpu" : " 17 GB", | 20260505 14:26:13.450 | INFO | "huge" : "", | 20260505 14:26:13.450 | INFO | "huge_shared" : " 27 GB", | 20260505 14:26:13.450 | INFO | "malloc" : " 14 GB", | 20260505 14:26:13.451 | INFO | "pinned" : " 29 GB" | 20260505 14:26:13.451 | INFO | } | 20260505 14:26:13.451 | INFO | Perf report for the last 5.14701 seconds: | 20260505 14:26:13.451 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:26:13.451 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:13.451 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:13.451 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:13.451 | INFO | ANALYSIS : 62.9%, 3.0, 97.9ms, 0 | 20260505 14:26:13.451 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:13.451 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:13.451 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 14:26:13.451 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:13.451 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:13.451 | INFO | Uploaded 36GiB to GPU in 2.20041s (16.3606GiB/s) | 20260505 14:26:13.451 | INFO | Downloaded 34GiB to GPU in 2.132s (15.9475GiB/s) | 20260505 14:26:18.589 | INFO | Analyzing chunk frames = [3913728,3914240) | 20260505 14:26:18.589 | INFO | MemInfo: { | 20260505 14:26:18.589 | INFO | "gpu" : " 17 GB", | 20260505 14:26:18.589 | INFO | "huge" : "", | 20260505 14:26:18.589 | INFO | "huge_shared" : " 27 GB", | 20260505 14:26:18.589 | INFO | "malloc" : " 14 GB", | 20260505 14:26:18.589 | INFO | "pinned" : " 29 GB" | 20260505 14:26:18.589 | INFO | } | 20260505 14:26:18.590 | INFO | Perf report for the last 5.1388 seconds: | 20260505 14:26:18.590 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:26:18.590 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:18.590 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:18.590 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:18.590 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 14:26:18.590 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:18.590 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:18.590 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:26:18.590 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:18.590 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:18.590 | INFO | Uploaded 36GiB to GPU in 2.1838s (16.4851GiB/s) | 20260505 14:26:18.590 | INFO | Downloaded 34GiB to GPU in 2.1313s (15.9527GiB/s) | 20260505 14:26:23.730 | INFO | Analyzing chunk frames = [3914240,3914752) | 20260505 14:26:23.730 | INFO | MemInfo: { | 20260505 14:26:23.730 | INFO | "gpu" : " 17 GB", | 20260505 14:26:23.730 | INFO | "huge" : "", | 20260505 14:26:23.730 | INFO | "huge_shared" : " 27 GB", | 20260505 14:26:23.730 | INFO | "malloc" : " 14 GB", | 20260505 14:26:23.730 | INFO | "pinned" : " 29 GB" | 20260505 14:26:23.730 | INFO | } | 20260505 14:26:23.730 | INFO | Perf report for the last 5.14044 seconds: | 20260505 14:26:23.730 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:26:23.730 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:23.730 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:23.730 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:23.730 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 14:26:23.730 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:23.730 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:23.730 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 14:26:23.730 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:23.730 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:23.730 | INFO | Uploaded 36GiB to GPU in 2.20169s (16.3511GiB/s) | 20260505 14:26:23.730 | INFO | Downloaded 34GiB to GPU in 2.1126s (16.0939GiB/s) | 20260505 14:26:28.870 | INFO | Analyzing chunk frames = [3914752,3915264) | 20260505 14:26:28.870 | INFO | MemInfo: { | 20260505 14:26:28.870 | INFO | "gpu" : " 17 GB", | 20260505 14:26:28.870 | INFO | "huge" : "", | 20260505 14:26:28.870 | INFO | "huge_shared" : " 27 GB", | 20260505 14:26:28.870 | INFO | "malloc" : " 14 GB", | 20260505 14:26:28.870 | INFO | "pinned" : " 29 GB" | 20260505 14:26:28.870 | INFO | } | 20260505 14:26:28.870 | INFO | Perf report for the last 5.14048 seconds: | 20260505 14:26:28.870 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:26:28.870 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:28.870 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:28.870 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:28.870 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 14:26:28.871 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:28.871 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:28.871 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 14:26:28.871 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:28.871 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:28.871 | INFO | Uploaded 36GiB to GPU in 2.13928s (16.8281GiB/s) | 20260505 14:26:28.871 | INFO | Downloaded 34GiB to GPU in 2.17225s (15.652GiB/s) | 20260505 14:26:34.003 | INFO | Analyzing chunk frames = [3915264,3915776) | 20260505 14:26:34.003 | INFO | MemInfo: { | 20260505 14:26:34.003 | INFO | "gpu" : " 17 GB", | 20260505 14:26:34.003 | INFO | "huge" : "", | 20260505 14:26:34.003 | INFO | "huge_shared" : " 27 GB", | 20260505 14:26:34.003 | INFO | "malloc" : " 14 GB", | 20260505 14:26:34.003 | INFO | "pinned" : " 29 GB" | 20260505 14:26:34.003 | INFO | } | 20260505 14:26:34.003 | INFO | Perf report for the last 5.13242 seconds: | 20260505 14:26:34.003 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:26:34.003 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:34.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:34.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:34.003 | INFO | ANALYSIS : 63.6%, 3.0, 99.1ms, 0 | 20260505 14:26:34.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:34.003 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:34.003 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:26:34.003 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:34.003 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:34.003 | INFO | Uploaded 36GiB to GPU in 2.20785s (16.3054GiB/s) | 20260505 14:26:34.003 | INFO | Downloaded 34GiB to GPU in 2.10134s (16.1801GiB/s) | 20260505 14:26:34.003 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 14:26:34.003 | INFO | Total mem usage: 15.3 GiB | 20260505 14:26:34.003 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 14:26:34.003 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.7%) | 20260505 14:26:34.003 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 14:26:34.003 | INFO | High Water Breakdown: | 20260505 14:26:34.003 | INFO | 61% : 8.68 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 14:26:34.003 | INFO | 38.9% : 5.53 GiB ( 83.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 14:26:34.003 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 14:26:34.003 | INFO | ------------------------------------------ | 20260505 14:26:34.003 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:26:34.003 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 14:26:34.003 | INFO | Total mem usage: 26.5 GiB | 20260505 14:26:34.003 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 14:26:34.003 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 14:26:34.003 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 14:26:34.003 | INFO | High Water Breakdown: | 20260505 14:26:34.003 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:26:34.003 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:26:34.003 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:26:34.003 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:26:34.003 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:26:34.004 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:26:34.004 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 14:26:34.004 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 14:26:34.004 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:26:34.004 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 14:26:34.004 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 14:26:34.004 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 14:26:34.004 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 14:26:34.004 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 14:26:34.004 | INFO | ------------------------------------------ | 20260505 14:26:34.004 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:26:34.004 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 14:26:34.004 | INFO | Total mem usage: 25.6 GiB | 20260505 14:26:34.004 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 14:26:34.004 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 14:26:34.004 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 14:26:34.004 | INFO | High Water Breakdown: | 20260505 14:26:34.004 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 14:26:34.004 | INFO | ------------------------------------------ | 20260505 14:26:34.004 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:26:34.004 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 14:26:34.004 | INFO | Total mem usage: 15.7 GiB | 20260505 14:26:34.004 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 14:26:34.004 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 14:26:34.004 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 14:26:34.004 | INFO | High Water Breakdown: | 20260505 14:26:34.004 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 14:26:34.004 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 14:26:34.004 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 14:26:34.004 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 14:26:34.004 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 14:26:34.004 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 14:26:34.004 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 14:26:34.004 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 14:26:34.004 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 14:26:34.004 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:26:34.004 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 14:26:34.004 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:26:34.004 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 14:26:34.004 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 14:26:34.004 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 14:26:34.004 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 14:26:34.004 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:26:34.004 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 14:26:34.004 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:26:34.004 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:26:34.004 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:26:34.004 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 14:26:34.004 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 14:26:34.004 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:26:34.004 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 14:26:34.004 | INFO | ------------------------------------------ | 20260505 14:26:34.004 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:26:34.004 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 14:26:34.004 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:26:34.004 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:26:34.004 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.67e+01, 1.38e+01 / 1.66e+00 | 20260505 14:26:34.004 | INFO | PerfReport: 0.29%, AggregateStats -- 9.26e-02 / 9.92e-02, 1.07e-02 / 3.68e-03 | 20260505 14:26:34.004 | INFO | PerfReport: 11.78%, Baseline -- 3.75e+00 / 4.49e+00, 5.83e-01 / 1.53e-01 | 20260505 14:26:34.004 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.48e+00, 1.93e-01 / 9.28e-02 | 20260505 14:26:34.004 | INFO | PerfReport: 16.05%, DME -- 4.08e+01 / 4.62e+01, 4.87e+00 / 2.28e+00 | 20260505 14:26:34.004 | INFO | PerfReport: 56.37%, FrameLabeling -- 1.79e+01 / 1.96e+01, 1.35e+00 / 1.72e+00 | 20260505 14:26:34.004 | INFO | PerfReport: 7.46%, PulseAccumulating -- 2.37e+00 / 2.94e+00, 4.56e-01 / 1.53e-01 | 20260505 14:26:34.004 | INFO | PerfReport: 3.53%, Metrics -- 1.12e+00 / 1.44e+00, 3.14e-01 / 1.56e-01 | 20260505 14:26:34.004 | INFO | PerfReport: 1.24%, PulseSerializing -- 3.94e-01 / 4.42e-01, 6.94e-02 / 3.00e-02 | 20260505 14:26:34.004 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 14:26:34.004 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:26:34.004 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:26:34.004 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.57e+01 / 0.00e+00 | 20260505 14:26:34.004 | INFO | PerfReport: 22.64%, Upload -- 2.28e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260505 14:26:34.004 | INFO | PerfReport: 22.02%, Download -- 2.22e+01 / 0.00e+00, 8.07e+00 / 0.00e+00 | 20260505 14:26:34.004 | INFO | PerfReport: 31.53%, Compute -- 3.18e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 14:26:34.005 | INFO | PerfReport: 2.37%, LockUpload -- 2.39e+00 / 0.00e+00, 8.03e+00 / 0.00e+00 | 20260505 14:26:34.005 | INFO | PerfReport: 1.35%, LockDownload -- 1.36e+00 / 0.00e+00, 2.80e+00 / 0.00e+00 | 20260505 14:26:34.005 | INFO | PerfReport: 20.08%, LockCompute -- 2.02e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260505 14:26:34.005 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.27e-02 / 0.00e+00 | 20260505 14:26:34.140 | INFO | Starting a round of Baz IO | 20260505 14:26:37.360 | INFO | Populating RTMetrics report | 20260505 14:26:37.360 | INFO | RTMetrics report populated | 20260505 14:26:37.360 | INFO | Opening tmp RTMetrics json file | 20260505 14:26:37.377 | INFO | Serializing RTMetrics to json | 20260505 14:26:37.387 | INFO | Serialization done, swapping tmp file now... | 20260505 14:26:37.405 | INFO | RTMetrics done | 20260505 14:26:37.467 | INFO | Finished a round of Baz IO: Wrote 2.257 GiB in 3.326 seconds | 20260505 14:26:37.467 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.76% / 0.00% / 77.24% | 20260505 14:26:37.467 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.559505% / 3.97364e-06% / 7.15256e-05% | 20260505 14:26:37.467 | INFO | Baz IO overall: | 20260505 14:26:37.467 | INFO | BytesWritten : 2332854343462 | 20260505 14:26:37.467 | INFO | NumEvents : 1109153568070 | 20260505 14:26:39.154 | INFO | Analyzing chunk frames = [3915776,3916288) | 20260505 14:26:39.154 | INFO | MemInfo: { | 20260505 14:26:39.154 | INFO | "gpu" : " 17 GB", | 20260505 14:26:39.154 | INFO | "huge" : "", | 20260505 14:26:39.154 | INFO | "huge_shared" : " 27 GB", | 20260505 14:26:39.154 | INFO | "malloc" : " 14 GB", | 20260505 14:26:39.154 | INFO | "pinned" : " 29 GB" | 20260505 14:26:39.154 | INFO | } | 20260505 14:26:39.154 | INFO | Perf report for the last 5.15076 seconds: | 20260505 14:26:39.154 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:26:39.154 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:39.154 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:39.154 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:39.154 | INFO | ANALYSIS : 64.1%, 3.0, 99.5ms, 0 | 20260505 14:26:39.154 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:39.154 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260505 14:26:39.154 | INFO | PRE_HQ : 60.6%, 1.8, 57.1ms, 0 | 20260505 14:26:39.154 | INFO | BAZWRITER : 63.9%, 4.0, 131.4ms, 0 | 20260505 14:26:39.154 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 14:26:39.154 | INFO | Uploaded 36GiB to GPU in 2.17238s (16.5717GiB/s) | 20260505 14:26:39.154 | INFO | Downloaded 38GiB to GPU in 2.27461s (16.7062GiB/s) | 20260505 14:26:44.291 | INFO | Analyzing chunk frames = [3916288,3916800) | 20260505 14:26:44.291 | INFO | MemInfo: { | 20260505 14:26:44.291 | INFO | "gpu" : " 17 GB", | 20260505 14:26:44.291 | INFO | "huge" : "", | 20260505 14:26:44.291 | INFO | "huge_shared" : " 27 GB", | 20260505 14:26:44.291 | INFO | "malloc" : " 14 GB", | 20260505 14:26:44.291 | INFO | "pinned" : " 29 GB" | 20260505 14:26:44.291 | INFO | } | 20260505 14:26:44.292 | INFO | Perf report for the last 5.13787 seconds: | 20260505 14:26:44.292 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:26:44.292 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:44.292 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:44.292 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:44.292 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 14:26:44.292 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:44.292 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:44.292 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:26:44.292 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:44.292 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:44.292 | INFO | Uploaded 36GiB to GPU in 2.13047s (16.8977GiB/s) | 20260505 14:26:44.292 | INFO | Downloaded 34GiB to GPU in 2.04108s (16.6579GiB/s) | 20260505 14:26:49.417 | INFO | Analyzing chunk frames = [3916800,3917312) | 20260505 14:26:49.417 | INFO | MemInfo: { | 20260505 14:26:49.417 | INFO | "gpu" : " 17 GB", | 20260505 14:26:49.417 | INFO | "huge" : "", | 20260505 14:26:49.417 | INFO | "huge_shared" : " 27 GB", | 20260505 14:26:49.417 | INFO | "malloc" : " 14 GB", | 20260505 14:26:49.417 | INFO | "pinned" : " 29 GB" | 20260505 14:26:49.417 | INFO | } | 20260505 14:26:49.417 | INFO | Perf report for the last 5.12564 seconds: | 20260505 14:26:49.417 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:26:49.417 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:49.417 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:49.417 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:49.417 | INFO | ANALYSIS : 63.2%, 3.0, 98.1ms, 0 | 20260505 14:26:49.417 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:49.417 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:49.417 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:26:49.417 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:49.417 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:49.417 | INFO | Uploaded 36GiB to GPU in 2.195s (16.4009GiB/s) | 20260505 14:26:49.417 | INFO | Downloaded 34GiB to GPU in 2.0458s (16.6194GiB/s) | 20260505 14:26:54.556 | INFO | Analyzing chunk frames = [3917312,3917824) | 20260505 14:26:54.556 | INFO | MemInfo: { | 20260505 14:26:54.556 | INFO | "gpu" : " 17 GB", | 20260505 14:26:54.556 | INFO | "huge" : "", | 20260505 14:26:54.556 | INFO | "huge_shared" : " 27 GB", | 20260505 14:26:54.556 | INFO | "malloc" : " 14 GB", | 20260505 14:26:54.556 | INFO | "pinned" : " 29 GB" | 20260505 14:26:54.556 | INFO | } | 20260505 14:26:54.556 | INFO | Perf report for the last 5.13895 seconds: | 20260505 14:26:54.556 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:26:54.556 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:54.556 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:54.556 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:54.556 | INFO | ANALYSIS : 63.0%, 3.0, 97.8ms, 0 | 20260505 14:26:54.556 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:54.556 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:54.556 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:26:54.556 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:54.556 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:54.557 | INFO | Uploaded 36GiB to GPU in 2.19949s (16.3674GiB/s) | 20260505 14:26:54.557 | INFO | Downloaded 34GiB to GPU in 2.13752s (15.9063GiB/s) | 20260505 14:26:59.702 | INFO | Analyzing chunk frames = [3917824,3918336) | 20260505 14:26:59.702 | INFO | MemInfo: { | 20260505 14:26:59.702 | INFO | "gpu" : " 17 GB", | 20260505 14:26:59.702 | INFO | "huge" : "", | 20260505 14:26:59.702 | INFO | "huge_shared" : " 27 GB", | 20260505 14:26:59.702 | INFO | "malloc" : " 14 GB", | 20260505 14:26:59.702 | INFO | "pinned" : " 29 GB" | 20260505 14:26:59.702 | INFO | } | 20260505 14:26:59.702 | INFO | Perf report for the last 5.14553 seconds: | 20260505 14:26:59.702 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:26:59.702 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:59.702 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:59.702 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:59.702 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 14:26:59.702 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:59.702 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:59.702 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 14:26:59.702 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:59.702 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:26:59.702 | INFO | Uploaded 36GiB to GPU in 2.18185s (16.4998GiB/s) | 20260505 14:26:59.702 | INFO | Downloaded 34GiB to GPU in 2.13475s (15.9269GiB/s) | 20260505 14:27:04.846 | INFO | Analyzing chunk frames = [3918336,3918848) | 20260505 14:27:04.846 | INFO | MemInfo: { | 20260505 14:27:04.846 | INFO | "gpu" : " 17 GB", | 20260505 14:27:04.846 | INFO | "huge" : "", | 20260505 14:27:04.846 | INFO | "huge_shared" : " 27 GB", | 20260505 14:27:04.846 | INFO | "malloc" : " 14 GB", | 20260505 14:27:04.846 | INFO | "pinned" : " 29 GB" | 20260505 14:27:04.846 | INFO | } | 20260505 14:27:04.846 | INFO | Perf report for the last 5.14427 seconds: | 20260505 14:27:04.846 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:27:04.846 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:04.846 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:04.846 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:04.846 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 14:27:04.846 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:04.846 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:04.846 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 14:27:04.846 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:04.846 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:04.846 | INFO | Uploaded 36GiB to GPU in 2.20469s (16.3289GiB/s) | 20260505 14:27:04.846 | INFO | Downloaded 34GiB to GPU in 2.11246s (16.095GiB/s) | 20260505 14:27:09.991 | INFO | Analyzing chunk frames = [3918848,3919360) | 20260505 14:27:09.991 | INFO | MemInfo: { | 20260505 14:27:09.991 | INFO | "gpu" : " 17 GB", | 20260505 14:27:09.991 | INFO | "huge" : "", | 20260505 14:27:09.991 | INFO | "huge_shared" : " 27 GB", | 20260505 14:27:09.991 | INFO | "malloc" : " 14 GB", | 20260505 14:27:09.991 | INFO | "pinned" : " 29 GB" | 20260505 14:27:09.991 | INFO | } | 20260505 14:27:09.991 | INFO | Perf report for the last 5.14512 seconds: | 20260505 14:27:09.991 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:27:09.991 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:09.991 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:09.991 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:09.991 | INFO | ANALYSIS : 63.5%, 3.0, 99.3ms, 0 | 20260505 14:27:09.991 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:09.991 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:09.991 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 14:27:09.991 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:09.991 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:09.992 | INFO | Uploaded 36GiB to GPU in 2.12244s (16.9616GiB/s) | 20260505 14:27:09.992 | INFO | Downloaded 34GiB to GPU in 2.17273s (15.6485GiB/s) | 20260505 14:27:15.119 | INFO | Analyzing chunk frames = [3919360,3919872) | 20260505 14:27:15.119 | INFO | MemInfo: { | 20260505 14:27:15.119 | INFO | "gpu" : " 17 GB", | 20260505 14:27:15.119 | INFO | "huge" : "", | 20260505 14:27:15.119 | INFO | "huge_shared" : " 27 GB", | 20260505 14:27:15.119 | INFO | "malloc" : " 14 GB", | 20260505 14:27:15.119 | INFO | "pinned" : " 29 GB" | 20260505 14:27:15.119 | INFO | } | 20260505 14:27:15.119 | INFO | Perf report for the last 5.12747 seconds: | 20260505 14:27:15.119 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:27:15.119 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:15.119 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:15.119 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:15.119 | INFO | ANALYSIS : 63.8%, 3.0, 99.4ms, 0 | 20260505 14:27:15.119 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:15.119 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:15.119 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:27:15.119 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:15.119 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:15.119 | INFO | Uploaded 36GiB to GPU in 2.22474s (16.1817GiB/s) | 20260505 14:27:15.119 | INFO | Downloaded 34GiB to GPU in 2.099s (16.1982GiB/s) | 20260505 14:27:15.259 | INFO | Starting a round of Baz IO | 20260505 14:27:18.481 | INFO | Populating RTMetrics report | 20260505 14:27:18.481 | INFO | RTMetrics report populated | 20260505 14:27:18.481 | INFO | Opening tmp RTMetrics json file | 20260505 14:27:18.497 | INFO | Serializing RTMetrics to json | 20260505 14:27:18.498 | INFO | Serialization done, swapping tmp file now... | 20260505 14:27:18.499 | INFO | RTMetrics done | 20260505 14:27:18.589 | INFO | Finished a round of Baz IO: Wrote 2.255 GiB in 3.33 seconds | 20260505 14:27:18.589 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.75% / 0.00% / 77.24% | 20260505 14:27:18.589 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.558825% / 3.97364e-06% / 8.74201e-05% | 20260505 14:27:18.589 | INFO | Baz IO overall: | 20260505 14:27:18.589 | INFO | BytesWritten : 2335275485146 | 20260505 14:27:18.589 | INFO | NumEvents : 1110294726084 | 20260505 14:27:20.254 | INFO | Analyzing chunk frames = [3919872,3920384) | 20260505 14:27:20.254 | INFO | MemInfo: { | 20260505 14:27:20.254 | INFO | "gpu" : " 17 GB", | 20260505 14:27:20.254 | INFO | "huge" : "", | 20260505 14:27:20.254 | INFO | "huge_shared" : " 27 GB", | 20260505 14:27:20.254 | INFO | "malloc" : " 14 GB", | 20260505 14:27:20.254 | INFO | "pinned" : " 29 GB" | 20260505 14:27:20.254 | INFO | } | 20260505 14:27:20.254 | INFO | Perf report for the last 5.13536 seconds: | 20260505 14:27:20.254 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:27:20.254 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:20.254 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:20.254 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:20.254 | INFO | ANALYSIS : 64.4%, 3.0, 100.0ms, 0 | 20260505 14:27:20.254 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:20.254 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260505 14:27:20.254 | INFO | PRE_HQ : 60.8%, 1.8, 57.2ms, 0 | 20260505 14:27:20.254 | INFO | BAZWRITER : 64.2%, 4.0, 131.3ms, 0 | 20260505 14:27:20.254 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 14:27:20.254 | INFO | Uploaded 36GiB to GPU in 2.17721s (16.5349GiB/s) | 20260505 14:27:20.254 | INFO | Downloaded 38GiB to GPU in 2.27431s (16.7083GiB/s) | 20260505 14:27:25.404 | INFO | Analyzing chunk frames = [3920384,3920896) | 20260505 14:27:25.404 | INFO | MemInfo: { | 20260505 14:27:25.404 | INFO | "gpu" : " 17 GB", | 20260505 14:27:25.404 | INFO | "huge" : "", | 20260505 14:27:25.404 | INFO | "huge_shared" : " 27 GB", | 20260505 14:27:25.404 | INFO | "malloc" : " 14 GB", | 20260505 14:27:25.404 | INFO | "pinned" : " 29 GB" | 20260505 14:27:25.404 | INFO | } | 20260505 14:27:25.404 | INFO | Perf report for the last 5.14997 seconds: | 20260505 14:27:25.404 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:27:25.404 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:25.404 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:25.404 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:25.404 | INFO | ANALYSIS : 63.3%, 3.0, 99.1ms, 0 | 20260505 14:27:25.404 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:25.404 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:25.404 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:27:25.404 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:25.404 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:25.404 | INFO | Uploaded 36GiB to GPU in 2.19333s (16.4134GiB/s) | 20260505 14:27:25.404 | INFO | Downloaded 34GiB to GPU in 2.03551s (16.7034GiB/s) | 20260505 14:27:30.543 | INFO | Analyzing chunk frames = [3920896,3921408) | 20260505 14:27:30.543 | INFO | MemInfo: { | 20260505 14:27:30.543 | INFO | "gpu" : " 17 GB", | 20260505 14:27:30.543 | INFO | "huge" : "", | 20260505 14:27:30.543 | INFO | "huge_shared" : " 27 GB", | 20260505 14:27:30.543 | INFO | "malloc" : " 14 GB", | 20260505 14:27:30.543 | INFO | "pinned" : " 29 GB" | 20260505 14:27:30.543 | INFO | } | 20260505 14:27:30.543 | INFO | Perf report for the last 5.13916 seconds: | 20260505 14:27:30.543 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:27:30.543 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:30.543 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:30.543 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:30.543 | INFO | ANALYSIS : 62.9%, 3.0, 97.9ms, 0 | 20260505 14:27:30.543 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:30.543 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:30.543 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:27:30.543 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:30.543 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:30.543 | INFO | Uploaded 36GiB to GPU in 2.17619s (16.5427GiB/s) | 20260505 14:27:30.543 | INFO | Downloaded 34GiB to GPU in 2.03611s (16.6985GiB/s) | 20260505 14:27:35.677 | INFO | Analyzing chunk frames = [3921408,3921920) | 20260505 14:27:35.677 | INFO | MemInfo: { | 20260505 14:27:35.677 | INFO | "gpu" : " 17 GB", | 20260505 14:27:35.677 | INFO | "huge" : "", | 20260505 14:27:35.677 | INFO | "huge_shared" : " 27 GB", | 20260505 14:27:35.677 | INFO | "malloc" : " 14 GB", | 20260505 14:27:35.677 | INFO | "pinned" : " 29 GB" | 20260505 14:27:35.677 | INFO | } | 20260505 14:27:35.677 | INFO | Perf report for the last 5.13406 seconds: | 20260505 14:27:35.677 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:27:35.677 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:35.677 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:35.677 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:35.677 | INFO | ANALYSIS : 62.8%, 3.0, 97.5ms, 0 | 20260505 14:27:35.677 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:35.677 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:35.677 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:27:35.677 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:35.677 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:35.677 | INFO | Uploaded 36GiB to GPU in 2.17602s (16.544GiB/s) | 20260505 14:27:35.678 | INFO | Downloaded 34GiB to GPU in 2.13217s (15.9462GiB/s) | 20260505 14:27:40.816 | INFO | Analyzing chunk frames = [3921920,3922432) | 20260505 14:27:40.816 | INFO | MemInfo: { | 20260505 14:27:40.816 | INFO | "gpu" : " 17 GB", | 20260505 14:27:40.816 | INFO | "huge" : "", | 20260505 14:27:40.816 | INFO | "huge_shared" : " 27 GB", | 20260505 14:27:40.816 | INFO | "malloc" : " 14 GB", | 20260505 14:27:40.816 | INFO | "pinned" : " 29 GB" | 20260505 14:27:40.816 | INFO | } | 20260505 14:27:40.816 | INFO | Perf report for the last 5.1389 seconds: | 20260505 14:27:40.816 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:27:40.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:40.816 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:40.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:40.816 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 14:27:40.816 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:40.816 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:40.816 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:27:40.816 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:40.816 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:40.816 | INFO | Uploaded 36GiB to GPU in 2.17527s (16.5497GiB/s) | 20260505 14:27:40.816 | INFO | Downloaded 34GiB to GPU in 2.12856s (15.9733GiB/s) | 20260505 14:27:45.962 | INFO | Analyzing chunk frames = [3922432,3922944) | 20260505 14:27:45.962 | INFO | MemInfo: { | 20260505 14:27:45.962 | INFO | "gpu" : " 17 GB", | 20260505 14:27:45.962 | INFO | "huge" : "", | 20260505 14:27:45.962 | INFO | "huge_shared" : " 27 GB", | 20260505 14:27:45.962 | INFO | "malloc" : " 14 GB", | 20260505 14:27:45.962 | INFO | "pinned" : " 29 GB" | 20260505 14:27:45.962 | INFO | } | 20260505 14:27:45.962 | INFO | Perf report for the last 5.14601 seconds: | 20260505 14:27:45.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:27:45.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:45.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:45.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:45.962 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 14:27:45.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:45.962 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:45.962 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 14:27:45.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:45.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:45.962 | INFO | Uploaded 36GiB to GPU in 2.16693s (16.6133GiB/s) | 20260505 14:27:45.962 | INFO | Downloaded 34GiB to GPU in 2.11242s (16.0953GiB/s) | 20260505 14:27:51.104 | INFO | Analyzing chunk frames = [3922944,3923456) | 20260505 14:27:51.104 | INFO | MemInfo: { | 20260505 14:27:51.104 | INFO | "gpu" : " 17 GB", | 20260505 14:27:51.104 | INFO | "huge" : "", | 20260505 14:27:51.104 | INFO | "huge_shared" : " 27 GB", | 20260505 14:27:51.104 | INFO | "malloc" : " 14 GB", | 20260505 14:27:51.105 | INFO | "pinned" : " 29 GB" | 20260505 14:27:51.105 | INFO | } | 20260505 14:27:51.105 | INFO | Perf report for the last 5.14249 seconds: | 20260505 14:27:51.105 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:27:51.105 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:51.105 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:51.105 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:51.105 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 14:27:51.105 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:51.105 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:51.105 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 14:27:51.105 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:51.105 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:51.105 | INFO | Uploaded 36GiB to GPU in 2.12373s (16.9513GiB/s) | 20260505 14:27:51.105 | INFO | Downloaded 34GiB to GPU in 2.1823s (15.5799GiB/s) | 20260505 14:27:56.235 | INFO | Analyzing chunk frames = [3923456,3923968) | 20260505 14:27:56.235 | INFO | MemInfo: { | 20260505 14:27:56.235 | INFO | "gpu" : " 17 GB", | 20260505 14:27:56.235 | INFO | "huge" : "", | 20260505 14:27:56.235 | INFO | "huge_shared" : " 27 GB", | 20260505 14:27:56.235 | INFO | "malloc" : " 14 GB", | 20260505 14:27:56.235 | INFO | "pinned" : " 29 GB" | 20260505 14:27:56.235 | INFO | } | 20260505 14:27:56.235 | INFO | Perf report for the last 5.13081 seconds: | 20260505 14:27:56.235 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:27:56.235 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:56.235 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:56.235 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:56.236 | INFO | ANALYSIS : 63.4%, 3.0, 98.8ms, 0 | 20260505 14:27:56.236 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:56.236 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:56.236 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:27:56.236 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:56.236 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:27:56.236 | INFO | Uploaded 36GiB to GPU in 2.16046s (16.6631GiB/s) | 20260505 14:27:56.236 | INFO | Downloaded 34GiB to GPU in 2.03236s (16.7293GiB/s) | 20260505 14:27:56.236 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 14:27:56.236 | INFO | Total mem usage: 15.3 GiB | 20260505 14:27:56.236 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 14:27:56.236 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.81%) | 20260505 14:27:56.236 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 14:27:56.236 | INFO | High Water Breakdown: | 20260505 14:27:56.236 | INFO | 61% : 8.68 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 14:27:56.236 | INFO | 38.8% : 5.52 GiB ( 83.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 14:27:56.236 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 14:27:56.236 | INFO | ------------------------------------------ | 20260505 14:27:56.236 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:27:56.236 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 14:27:56.236 | INFO | Total mem usage: 26.5 GiB | 20260505 14:27:56.236 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 14:27:56.236 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 14:27:56.236 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 14:27:56.236 | INFO | High Water Breakdown: | 20260505 14:27:56.236 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:27:56.236 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:27:56.236 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:27:56.236 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:27:56.236 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:27:56.236 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:27:56.236 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 14:27:56.236 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 14:27:56.236 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:27:56.236 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 14:27:56.236 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 14:27:56.236 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 14:27:56.236 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 14:27:56.236 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 14:27:56.236 | INFO | ------------------------------------------ | 20260505 14:27:56.236 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:27:56.236 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 14:27:56.236 | INFO | Total mem usage: 25.6 GiB | 20260505 14:27:56.236 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 14:27:56.236 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 14:27:56.236 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 14:27:56.236 | INFO | High Water Breakdown: | 20260505 14:27:56.236 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 14:27:56.236 | INFO | ------------------------------------------ | 20260505 14:27:56.236 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:27:56.236 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 14:27:56.236 | INFO | Total mem usage: 15.7 GiB | 20260505 14:27:56.236 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 14:27:56.236 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 14:27:56.236 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 14:27:56.237 | INFO | High Water Breakdown: | 20260505 14:27:56.237 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 14:27:56.237 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 14:27:56.237 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 14:27:56.237 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 14:27:56.237 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 14:27:56.237 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 14:27:56.237 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 14:27:56.237 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 14:27:56.237 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 14:27:56.237 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:27:56.237 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 14:27:56.237 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:27:56.237 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 14:27:56.237 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 14:27:56.237 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 14:27:56.237 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 14:27:56.237 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:27:56.237 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 14:27:56.237 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:27:56.237 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:27:56.237 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:27:56.237 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 14:27:56.237 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 14:27:56.237 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:27:56.237 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 14:27:56.237 | INFO | ------------------------------------------ | 20260505 14:27:56.237 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:27:56.237 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 14:27:56.237 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:27:56.237 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:27:56.237 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.60e+01, 1.38e+01 / 1.94e+00 | 20260505 14:27:56.237 | INFO | PerfReport: 0.29%, AggregateStats -- 9.27e-02 / 1.01e-01, 1.05e-02 / 5.20e-03 | 20260505 14:27:56.237 | INFO | PerfReport: 11.78%, Baseline -- 3.75e+00 / 4.48e+00, 5.85e-01 / 1.84e-01 | 20260505 14:27:56.237 | INFO | PerfReport: 3.29%, Histogram -- 1.05e+00 / 1.47e+00, 1.97e-01 / 1.16e-01 | 20260505 14:27:56.237 | INFO | PerfReport: 15.95%, DME -- 4.06e+01 / 4.52e+01, 4.45e+00 / 2.00e+00 | 20260505 14:27:56.237 | INFO | PerfReport: 56.55%, FrameLabeling -- 1.80e+01 / 1.99e+01, 1.48e+00 / 1.63e+00 | 20260505 14:27:56.237 | INFO | PerfReport: 7.39%, PulseAccumulating -- 2.35e+00 / 2.91e+00, 4.48e-01 / 1.91e-01 | 20260505 14:27:56.237 | INFO | PerfReport: 3.51%, Metrics -- 1.12e+00 / 1.44e+00, 3.10e-01 / 1.32e-01 | 20260505 14:27:56.237 | INFO | PerfReport: 1.24%, PulseSerializing -- 3.94e-01 / 4.47e-01, 7.06e-02 / 3.88e-02 | 20260505 14:27:56.237 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 14:27:56.237 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:27:56.237 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:27:56.237 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260505 14:27:56.237 | INFO | PerfReport: 22.48%, Upload -- 2.26e+01 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260505 14:27:56.237 | INFO | PerfReport: 21.95%, Download -- 2.21e+01 / 0.00e+00, 7.97e+00 / 0.00e+00 | 20260505 14:27:56.237 | INFO | PerfReport: 31.61%, Compute -- 3.18e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 14:27:56.237 | INFO | PerfReport: 2.27%, LockUpload -- 2.29e+00 / 0.00e+00, 8.03e+00 / 0.00e+00 | 20260505 14:27:56.237 | INFO | PerfReport: 1.46%, LockDownload -- 1.47e+00 / 0.00e+00, 3.08e+00 / 0.00e+00 | 20260505 14:27:56.237 | INFO | PerfReport: 20.22%, LockCompute -- 2.04e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 14:27:56.237 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.05e-02 / 0.00e+00 | 20260505 14:27:56.373 | INFO | Starting a round of Baz IO | 20260505 14:27:59.595 | INFO | Populating RTMetrics report | 20260505 14:27:59.596 | INFO | RTMetrics report populated | 20260505 14:27:59.596 | INFO | Opening tmp RTMetrics json file | 20260505 14:27:59.619 | INFO | Serializing RTMetrics to json | 20260505 14:27:59.640 | INFO | Serialization done, swapping tmp file now... | 20260505 14:27:59.640 | INFO | RTMetrics done | 20260505 14:27:59.707 | INFO | Finished a round of Baz IO: Wrote 2.256 GiB in 3.334 seconds | 20260505 14:27:59.707 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.75% / 0.00% / 77.25% | 20260505 14:27:59.707 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.556811% / 3.97364e-06% / 9.13938e-05% | 20260505 14:27:59.707 | INFO | Baz IO overall: | 20260505 14:27:59.708 | INFO | BytesWritten : 2337697863906 | 20260505 14:27:59.708 | INFO | NumEvents : 1111436882064 | 20260505 14:28:01.369 | INFO | Analyzing chunk frames = [3923968,3924480) | 20260505 14:28:01.369 | INFO | MemInfo: { | 20260505 14:28:01.369 | INFO | "gpu" : " 17 GB", | 20260505 14:28:01.369 | INFO | "huge" : "", | 20260505 14:28:01.369 | INFO | "huge_shared" : " 27 GB", | 20260505 14:28:01.369 | INFO | "malloc" : " 14 GB", | 20260505 14:28:01.369 | INFO | "pinned" : " 29 GB" | 20260505 14:28:01.370 | INFO | } | 20260505 14:28:01.370 | INFO | Perf report for the last 5.13412 seconds: | 20260505 14:28:01.370 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:28:01.370 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:01.370 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:01.370 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:01.370 | INFO | ANALYSIS : 64.3%, 3.0, 99.6ms, 0 | 20260505 14:28:01.370 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:01.370 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260505 14:28:01.370 | INFO | PRE_HQ : 60.9%, 1.8, 57.5ms, 0 | 20260505 14:28:01.370 | INFO | BAZWRITER : 64.3%, 4.0, 131.4ms, 0 | 20260505 14:28:01.370 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 14:28:01.370 | INFO | Uploaded 36GiB to GPU in 2.18173s (16.5007GiB/s) | 20260505 14:28:01.370 | INFO | Downloaded 38GiB to GPU in 2.26929s (16.7453GiB/s) | 20260505 14:28:06.510 | INFO | Analyzing chunk frames = [3924480,3924992) | 20260505 14:28:06.510 | INFO | MemInfo: { | 20260505 14:28:06.510 | INFO | "gpu" : " 17 GB", | 20260505 14:28:06.510 | INFO | "huge" : "", | 20260505 14:28:06.510 | INFO | "huge_shared" : " 27 GB", | 20260505 14:28:06.510 | INFO | "malloc" : " 14 GB", | 20260505 14:28:06.510 | INFO | "pinned" : " 29 GB" | 20260505 14:28:06.510 | INFO | } | 20260505 14:28:06.510 | INFO | Perf report for the last 5.14045 seconds: | 20260505 14:28:06.510 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:28:06.510 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:06.510 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:06.510 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:06.510 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 14:28:06.510 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:06.510 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:06.510 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:28:06.510 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:06.510 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:06.510 | INFO | Uploaded 36GiB to GPU in 2.16794s (16.6056GiB/s) | 20260505 14:28:06.510 | INFO | Downloaded 34GiB to GPU in 2.02804s (16.765GiB/s) | 20260505 14:28:11.656 | INFO | Analyzing chunk frames = [3924992,3925504) | 20260505 14:28:11.656 | INFO | MemInfo: { | 20260505 14:28:11.656 | INFO | "gpu" : " 17 GB", | 20260505 14:28:11.656 | INFO | "huge" : "", | 20260505 14:28:11.656 | INFO | "huge_shared" : " 27 GB", | 20260505 14:28:11.656 | INFO | "malloc" : " 14 GB", | 20260505 14:28:11.656 | INFO | "pinned" : " 29 GB" | 20260505 14:28:11.656 | INFO | } | 20260505 14:28:11.656 | INFO | Perf report for the last 5.14556 seconds: | 20260505 14:28:11.656 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:28:11.656 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:11.656 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:11.656 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:11.656 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 14:28:11.656 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:11.656 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:11.656 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:28:11.656 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:11.656 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:11.656 | INFO | Uploaded 36GiB to GPU in 2.22877s (16.1524GiB/s) | 20260505 14:28:11.656 | INFO | Downloaded 34GiB to GPU in 2.0956s (16.2245GiB/s) | 20260505 14:28:16.801 | INFO | Analyzing chunk frames = [3925504,3926016) | 20260505 14:28:16.801 | INFO | MemInfo: { | 20260505 14:28:16.801 | INFO | "gpu" : " 17 GB", | 20260505 14:28:16.801 | INFO | "huge" : "", | 20260505 14:28:16.801 | INFO | "huge_shared" : " 27 GB", | 20260505 14:28:16.801 | INFO | "malloc" : " 14 GB", | 20260505 14:28:16.801 | INFO | "pinned" : " 29 GB" | 20260505 14:28:16.801 | INFO | } | 20260505 14:28:16.801 | INFO | Perf report for the last 5.1458 seconds: | 20260505 14:28:16.801 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:28:16.801 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:16.801 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:16.801 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:16.801 | INFO | ANALYSIS : 62.6%, 3.0, 97.3ms, 0 | 20260505 14:28:16.801 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:16.802 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:16.802 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:28:16.802 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:16.802 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:16.802 | INFO | Uploaded 36GiB to GPU in 2.18089s (16.507GiB/s) | 20260505 14:28:16.802 | INFO | Downloaded 34GiB to GPU in 2.131s (15.9549GiB/s) | 20260505 14:28:21.938 | INFO | Analyzing chunk frames = [3926016,3926528) | 20260505 14:28:21.938 | INFO | MemInfo: { | 20260505 14:28:21.938 | INFO | "gpu" : " 17 GB", | 20260505 14:28:21.938 | INFO | "huge" : "", | 20260505 14:28:21.938 | INFO | "huge_shared" : " 27 GB", | 20260505 14:28:21.938 | INFO | "malloc" : " 14 GB", | 20260505 14:28:21.938 | INFO | "pinned" : " 29 GB" | 20260505 14:28:21.938 | INFO | } | 20260505 14:28:21.938 | INFO | Perf report for the last 5.13689 seconds: | 20260505 14:28:21.938 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:28:21.938 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:21.938 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:21.938 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:21.938 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 14:28:21.938 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:21.938 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:21.938 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:28:21.938 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:21.939 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:21.939 | INFO | Uploaded 36GiB to GPU in 2.18103s (16.5059GiB/s) | 20260505 14:28:21.939 | INFO | Downloaded 34GiB to GPU in 2.13074s (15.9569GiB/s) | 20260505 14:28:27.065 | INFO | Analyzing chunk frames = [3926528,3927040) | 20260505 14:28:27.065 | INFO | MemInfo: { | 20260505 14:28:27.065 | INFO | "gpu" : " 17 GB", | 20260505 14:28:27.065 | INFO | "huge" : "", | 20260505 14:28:27.065 | INFO | "huge_shared" : " 27 GB", | 20260505 14:28:27.065 | INFO | "malloc" : " 14 GB", | 20260505 14:28:27.065 | INFO | "pinned" : " 29 GB" | 20260505 14:28:27.065 | INFO | } | 20260505 14:28:27.065 | INFO | Perf report for the last 5.12713 seconds: | 20260505 14:28:27.065 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:28:27.066 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:27.066 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:27.066 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:27.066 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 14:28:27.066 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:27.066 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:27.066 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:28:27.066 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:27.066 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:27.066 | INFO | Uploaded 36GiB to GPU in 2.19248s (16.4198GiB/s) | 20260505 14:28:27.066 | INFO | Downloaded 34GiB to GPU in 2.11372s (16.0854GiB/s) | 20260505 14:28:32.212 | INFO | Analyzing chunk frames = [3927040,3927552) | 20260505 14:28:32.212 | INFO | MemInfo: { | 20260505 14:28:32.212 | INFO | "gpu" : " 17 GB", | 20260505 14:28:32.212 | INFO | "huge" : "", | 20260505 14:28:32.212 | INFO | "huge_shared" : " 27 GB", | 20260505 14:28:32.212 | INFO | "malloc" : " 14 GB", | 20260505 14:28:32.212 | INFO | "pinned" : " 29 GB" | 20260505 14:28:32.212 | INFO | } | 20260505 14:28:32.212 | INFO | Perf report for the last 5.14674 seconds: | 20260505 14:28:32.212 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:28:32.212 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:32.212 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:32.212 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:32.212 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 14:28:32.212 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:32.213 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:32.213 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 14:28:32.213 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:32.213 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:32.213 | INFO | Uploaded 36GiB to GPU in 2.16008s (16.666GiB/s) | 20260505 14:28:32.213 | INFO | Downloaded 34GiB to GPU in 2.18201s (15.582GiB/s) | 20260505 14:28:37.351 | INFO | Analyzing chunk frames = [3927552,3928064) | 20260505 14:28:37.351 | INFO | MemInfo: { | 20260505 14:28:37.351 | INFO | "gpu" : " 17 GB", | 20260505 14:28:37.351 | INFO | "huge" : "", | 20260505 14:28:37.351 | INFO | "huge_shared" : " 27 GB", | 20260505 14:28:37.351 | INFO | "malloc" : " 14 GB", | 20260505 14:28:37.351 | INFO | "pinned" : " 29 GB" | 20260505 14:28:37.351 | INFO | } | 20260505 14:28:37.351 | INFO | Perf report for the last 5.13853 seconds: | 20260505 14:28:37.351 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:28:37.351 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:37.351 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:37.351 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:37.351 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 14:28:37.351 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:37.351 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:37.351 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 14:28:37.351 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:37.351 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:37.351 | INFO | Uploaded 36GiB to GPU in 2.20921s (16.2954GiB/s) | 20260505 14:28:37.351 | INFO | Downloaded 34GiB to GPU in 2.09241s (16.2492GiB/s) | 20260505 14:28:37.485 | INFO | Starting a round of Baz IO | 20260505 14:28:40.700 | INFO | Populating RTMetrics report | 20260505 14:28:40.700 | INFO | RTMetrics report populated | 20260505 14:28:40.700 | INFO | Opening tmp RTMetrics json file | 20260505 14:28:40.718 | INFO | Serializing RTMetrics to json | 20260505 14:28:40.725 | INFO | Serialization done, swapping tmp file now... | 20260505 14:28:40.742 | INFO | RTMetrics done | 20260505 14:28:40.811 | INFO | Finished a round of Baz IO: Wrote 2.256 GiB in 3.326 seconds | 20260505 14:28:40.811 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.75% / 0.00% / 77.25% | 20260505 14:28:40.811 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.556306% / 3.97364e-06% / 7.15256e-05% | 20260505 14:28:40.811 | INFO | Baz IO overall: | 20260505 14:28:40.811 | INFO | BytesWritten : 2340119817149 | 20260505 14:28:40.811 | INFO | NumEvents : 1112578564561 | 20260505 14:28:42.494 | INFO | Analyzing chunk frames = [3928064,3928576) | 20260505 14:28:42.494 | INFO | MemInfo: { | 20260505 14:28:42.494 | INFO | "gpu" : " 17 GB", | 20260505 14:28:42.494 | INFO | "huge" : "", | 20260505 14:28:42.494 | INFO | "huge_shared" : " 27 GB", | 20260505 14:28:42.494 | INFO | "malloc" : " 14 GB", | 20260505 14:28:42.494 | INFO | "pinned" : " 29 GB" | 20260505 14:28:42.494 | INFO | } | 20260505 14:28:42.494 | INFO | Perf report for the last 5.14333 seconds: | 20260505 14:28:42.494 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:28:42.494 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:42.494 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:42.494 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:42.494 | INFO | ANALYSIS : 64.0%, 3.0, 99.3ms, 0 | 20260505 14:28:42.494 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:42.494 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260505 14:28:42.494 | INFO | PRE_HQ : 60.8%, 1.8, 57.1ms, 0 | 20260505 14:28:42.494 | INFO | BAZWRITER : 63.9%, 4.0, 131.7ms, 0 | 20260505 14:28:42.494 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260505 14:28:42.494 | INFO | Uploaded 36GiB to GPU in 2.20242s (16.3457GiB/s) | 20260505 14:28:42.494 | INFO | Downloaded 38GiB to GPU in 2.26326s (16.7899GiB/s) | 20260505 14:28:47.627 | INFO | Analyzing chunk frames = [3928576,3929088) | 20260505 14:28:47.627 | INFO | MemInfo: { | 20260505 14:28:47.627 | INFO | "gpu" : " 17 GB", | 20260505 14:28:47.627 | INFO | "huge" : "", | 20260505 14:28:47.627 | INFO | "huge_shared" : " 27 GB", | 20260505 14:28:47.627 | INFO | "malloc" : " 14 GB", | 20260505 14:28:47.627 | INFO | "pinned" : " 29 GB" | 20260505 14:28:47.627 | INFO | } | 20260505 14:28:47.627 | INFO | Perf report for the last 5.13276 seconds: | 20260505 14:28:47.627 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:28:47.627 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:47.627 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:47.627 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:47.627 | INFO | ANALYSIS : 63.6%, 3.0, 99.1ms, 0 | 20260505 14:28:47.627 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:47.627 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:47.627 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:28:47.627 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:47.627 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:47.627 | INFO | Uploaded 36GiB to GPU in 2.20296s (16.3417GiB/s) | 20260505 14:28:47.627 | INFO | Downloaded 34GiB to GPU in 2.05911s (16.512GiB/s) | 20260505 14:28:52.766 | INFO | Analyzing chunk frames = [3929088,3929600) | 20260505 14:28:52.766 | INFO | MemInfo: { | 20260505 14:28:52.766 | INFO | "gpu" : " 17 GB", | 20260505 14:28:52.766 | INFO | "huge" : "", | 20260505 14:28:52.766 | INFO | "huge_shared" : " 27 GB", | 20260505 14:28:52.766 | INFO | "malloc" : " 14 GB", | 20260505 14:28:52.767 | INFO | "pinned" : " 29 GB" | 20260505 14:28:52.767 | INFO | } | 20260505 14:28:52.767 | INFO | Perf report for the last 5.13984 seconds: | 20260505 14:28:52.767 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:28:52.767 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:52.767 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:52.767 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:52.767 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 14:28:52.767 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:52.767 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:52.767 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:28:52.767 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:52.767 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:52.767 | INFO | Uploaded 36GiB to GPU in 2.23846s (16.0825GiB/s) | 20260505 14:28:52.767 | INFO | Downloaded 34GiB to GPU in 2.10166s (16.1777GiB/s) | 20260505 14:28:57.913 | INFO | Analyzing chunk frames = [3929600,3930112) | 20260505 14:28:57.913 | INFO | MemInfo: { | 20260505 14:28:57.913 | INFO | "gpu" : " 17 GB", | 20260505 14:28:57.913 | INFO | "huge" : "", | 20260505 14:28:57.913 | INFO | "huge_shared" : " 27 GB", | 20260505 14:28:57.913 | INFO | "malloc" : " 14 GB", | 20260505 14:28:57.913 | INFO | "pinned" : " 29 GB" | 20260505 14:28:57.913 | INFO | } | 20260505 14:28:57.913 | INFO | Perf report for the last 5.14669 seconds: | 20260505 14:28:57.913 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:28:57.913 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:57.913 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:57.914 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:57.914 | INFO | ANALYSIS : 62.9%, 3.0, 97.9ms, 0 | 20260505 14:28:57.914 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:57.914 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:57.914 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 14:28:57.914 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:57.914 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:28:57.914 | INFO | Uploaded 36GiB to GPU in 2.20296s (16.3417GiB/s) | 20260505 14:28:57.914 | INFO | Downloaded 34GiB to GPU in 2.13172s (15.9495GiB/s) | 20260505 14:29:03.053 | INFO | Analyzing chunk frames = [3930112,3930624) | 20260505 14:29:03.053 | INFO | MemInfo: { | 20260505 14:29:03.053 | INFO | "gpu" : " 17 GB", | 20260505 14:29:03.053 | INFO | "huge" : "", | 20260505 14:29:03.053 | INFO | "huge_shared" : " 27 GB", | 20260505 14:29:03.053 | INFO | "malloc" : " 14 GB", | 20260505 14:29:03.053 | INFO | "pinned" : " 29 GB" | 20260505 14:29:03.053 | INFO | } | 20260505 14:29:03.053 | INFO | Perf report for the last 5.13958 seconds: | 20260505 14:29:03.053 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:29:03.053 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:03.053 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:03.053 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:03.053 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 14:29:03.053 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:03.053 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:03.053 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:29:03.053 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:03.053 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:03.053 | INFO | Uploaded 36GiB to GPU in 2.17438s (16.5564GiB/s) | 20260505 14:29:03.053 | INFO | Downloaded 34GiB to GPU in 2.13034s (15.9599GiB/s) | 20260505 14:29:08.183 | INFO | Analyzing chunk frames = [3930624,3931136) | 20260505 14:29:08.183 | INFO | MemInfo: { | 20260505 14:29:08.183 | INFO | "gpu" : " 17 GB", | 20260505 14:29:08.183 | INFO | "huge" : "", | 20260505 14:29:08.183 | INFO | "huge_shared" : " 27 GB", | 20260505 14:29:08.183 | INFO | "malloc" : " 14 GB", | 20260505 14:29:08.183 | INFO | "pinned" : " 29 GB" | 20260505 14:29:08.183 | INFO | } | 20260505 14:29:08.183 | INFO | Perf report for the last 5.12983 seconds: | 20260505 14:29:08.183 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:29:08.183 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:08.183 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:08.183 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:08.183 | INFO | ANALYSIS : 62.9%, 3.0, 98.0ms, 0 | 20260505 14:29:08.183 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:08.183 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:08.183 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:29:08.183 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:08.183 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:08.183 | INFO | Uploaded 36GiB to GPU in 2.16954s (16.5934GiB/s) | 20260505 14:29:08.183 | INFO | Downloaded 34GiB to GPU in 2.03647s (16.6955GiB/s) | 20260505 14:29:13.316 | INFO | Analyzing chunk frames = [3931136,3931648) | 20260505 14:29:13.316 | INFO | MemInfo: { | 20260505 14:29:13.316 | INFO | "gpu" : " 17 GB", | 20260505 14:29:13.316 | INFO | "huge" : "", | 20260505 14:29:13.316 | INFO | "huge_shared" : " 27 GB", | 20260505 14:29:13.316 | INFO | "malloc" : " 14 GB", | 20260505 14:29:13.316 | INFO | "pinned" : " 29 GB" | 20260505 14:29:13.316 | INFO | } | 20260505 14:29:13.317 | INFO | Perf report for the last 5.13378 seconds: | 20260505 14:29:13.317 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:29:13.317 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:13.317 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:13.317 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:13.317 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 14:29:13.317 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:13.317 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:13.317 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 14:29:13.317 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:13.317 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:13.317 | INFO | Uploaded 36GiB to GPU in 2.16592s (16.6211GiB/s) | 20260505 14:29:13.317 | INFO | Downloaded 34GiB to GPU in 2.15669s (15.7649GiB/s) | 20260505 14:29:18.463 | INFO | Analyzing chunk frames = [3931648,3932160) | 20260505 14:29:18.463 | INFO | MemInfo: { | 20260505 14:29:18.463 | INFO | "gpu" : " 17 GB", | 20260505 14:29:18.463 | INFO | "huge" : "", | 20260505 14:29:18.463 | INFO | "huge_shared" : " 27 GB", | 20260505 14:29:18.464 | INFO | "malloc" : " 14 GB", | 20260505 14:29:18.464 | INFO | "pinned" : " 29 GB" | 20260505 14:29:18.464 | INFO | } | 20260505 14:29:18.464 | INFO | Perf report for the last 5.1469 seconds: | 20260505 14:29:18.464 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:29:18.464 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:18.464 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:18.464 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:18.464 | INFO | ANALYSIS : 63.4%, 3.0, 99.2ms, 0 | 20260505 14:29:18.464 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:18.464 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:18.464 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 14:29:18.464 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:18.464 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:18.464 | INFO | Uploaded 36GiB to GPU in 2.16399s (16.6359GiB/s) | 20260505 14:29:18.464 | INFO | Downloaded 34GiB to GPU in 2.10336s (16.1646GiB/s) | 20260505 14:29:18.464 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 14:29:18.464 | INFO | Total mem usage: 15.3 GiB | 20260505 14:29:18.464 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 14:29:18.464 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.82%) | 20260505 14:29:18.464 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 14:29:18.464 | INFO | High Water Breakdown: | 20260505 14:29:18.464 | INFO | 61% : 8.68 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 14:29:18.464 | INFO | 38.8% : 5.52 GiB ( 83.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 14:29:18.464 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 14:29:18.464 | INFO | ------------------------------------------ | 20260505 14:29:18.464 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:29:18.464 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 14:29:18.464 | INFO | Total mem usage: 26.5 GiB | 20260505 14:29:18.464 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 14:29:18.464 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 14:29:18.464 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 14:29:18.464 | INFO | High Water Breakdown: | 20260505 14:29:18.464 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:29:18.464 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:29:18.464 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:29:18.464 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:29:18.464 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:29:18.464 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:29:18.464 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 14:29:18.464 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 14:29:18.464 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:29:18.464 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 14:29:18.464 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 14:29:18.465 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 14:29:18.465 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 14:29:18.465 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 14:29:18.465 | INFO | ------------------------------------------ | 20260505 14:29:18.465 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:29:18.465 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 14:29:18.465 | INFO | Total mem usage: 25.6 GiB | 20260505 14:29:18.465 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 14:29:18.465 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 14:29:18.465 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 14:29:18.465 | INFO | High Water Breakdown: | 20260505 14:29:18.465 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 14:29:18.465 | INFO | ------------------------------------------ | 20260505 14:29:18.465 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:29:18.465 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 14:29:18.465 | INFO | Total mem usage: 15.7 GiB | 20260505 14:29:18.465 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 14:29:18.465 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 14:29:18.465 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 14:29:18.465 | INFO | High Water Breakdown: | 20260505 14:29:18.465 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 14:29:18.465 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 14:29:18.465 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 14:29:18.465 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 14:29:18.465 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 14:29:18.465 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 14:29:18.465 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 14:29:18.465 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 14:29:18.465 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 14:29:18.465 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:29:18.465 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 14:29:18.465 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:29:18.465 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 14:29:18.465 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 14:29:18.465 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 14:29:18.465 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 14:29:18.465 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:29:18.465 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 14:29:18.465 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:29:18.465 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:29:18.465 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:29:18.465 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 14:29:18.465 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 14:29:18.465 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:29:18.465 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 14:29:18.465 | INFO | ------------------------------------------ | 20260505 14:29:18.465 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:29:18.465 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 14:29:18.465 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:29:18.465 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:29:18.465 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.61e+01, 1.37e+01 / 1.52e+00 | 20260505 14:29:18.465 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 1.00e-01, 1.06e-02 / 5.39e-03 | 20260505 14:29:18.465 | INFO | PerfReport: 11.80%, Baseline -- 3.75e+00 / 4.51e+00, 5.83e-01 / 1.43e-01 | 20260505 14:29:18.465 | INFO | PerfReport: 3.29%, Histogram -- 1.05e+00 / 1.51e+00, 1.97e-01 / 1.01e-01 | 20260505 14:29:18.465 | INFO | PerfReport: 15.98%, DME -- 4.06e+01 / 4.55e+01, 4.62e+00 / 1.89e+00 | 20260505 14:29:18.465 | INFO | PerfReport: 56.41%, FrameLabeling -- 1.79e+01 / 1.97e+01, 1.37e+00 / 1.67e+00 | 20260505 14:29:18.465 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.37e+00 / 2.90e+00, 4.54e-01 / 1.71e-01 | 20260505 14:29:18.465 | INFO | PerfReport: 3.54%, Metrics -- 1.13e+00 / 1.45e+00, 3.15e-01 / 1.76e-01 | 20260505 14:29:18.465 | INFO | PerfReport: 1.24%, PulseSerializing -- 3.94e-01 / 4.46e-01, 7.00e-02 / 3.01e-02 | 20260505 14:29:18.465 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 14:29:18.465 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:29:18.465 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:29:18.465 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260505 14:29:18.465 | INFO | PerfReport: 22.65%, Upload -- 2.28e+01 / 0.00e+00, 8.10e+00 / 0.00e+00 | 20260505 14:29:18.465 | INFO | PerfReport: 22.01%, Download -- 2.22e+01 / 0.00e+00, 8.03e+00 / 0.00e+00 | 20260505 14:29:18.465 | INFO | PerfReport: 31.58%, Compute -- 3.18e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 14:29:18.465 | INFO | PerfReport: 2.30%, LockUpload -- 2.31e+00 / 0.00e+00, 7.86e+00 / 0.00e+00 | 20260505 14:29:18.465 | INFO | PerfReport: 1.36%, LockDownload -- 1.37e+00 / 0.00e+00, 2.80e+00 / 0.00e+00 | 20260505 14:29:18.465 | INFO | PerfReport: 20.10%, LockCompute -- 2.02e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260505 14:29:18.465 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.29e-02 / 0.00e+00 | 20260505 14:29:18.598 | INFO | Starting a round of Baz IO | 20260505 14:29:21.814 | INFO | Populating RTMetrics report | 20260505 14:29:21.814 | INFO | RTMetrics report populated | 20260505 14:29:21.814 | INFO | Opening tmp RTMetrics json file | 20260505 14:29:21.830 | INFO | Serializing RTMetrics to json | 20260505 14:29:21.847 | INFO | Serialization done, swapping tmp file now... | 20260505 14:29:21.855 | INFO | RTMetrics done | 20260505 14:29:21.917 | INFO | Finished a round of Baz IO: Wrote 2.253 GiB in 3.319 seconds | 20260505 14:29:21.917 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.74% / 0.00% / 77.25% | 20260505 14:29:21.917 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.554132% / 3.97364e-06% / 9.13938e-05% | 20260505 14:29:21.917 | INFO | Baz IO overall: | 20260505 14:29:21.917 | INFO | BytesWritten : 2342539443994 | 20260505 14:29:21.917 | INFO | NumEvents : 1113718893341 | 20260505 14:29:23.607 | INFO | Analyzing chunk frames = [3932160,3932672) | 20260505 14:29:23.607 | INFO | MemInfo: { | 20260505 14:29:23.607 | INFO | "gpu" : " 17 GB", | 20260505 14:29:23.607 | INFO | "huge" : "", | 20260505 14:29:23.607 | INFO | "huge_shared" : " 27 GB", | 20260505 14:29:23.607 | INFO | "malloc" : " 14 GB", | 20260505 14:29:23.607 | INFO | "pinned" : " 29 GB" | 20260505 14:29:23.607 | INFO | } | 20260505 14:29:23.607 | INFO | Perf report for the last 5.14375 seconds: | 20260505 14:29:23.607 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:29:23.607 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:23.607 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:23.607 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:23.607 | INFO | ANALYSIS : 64.0%, 3.0, 99.6ms, 0 | 20260505 14:29:23.607 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:23.608 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260505 14:29:23.608 | INFO | PRE_HQ : 60.7%, 1.8, 57.1ms, 0 | 20260505 14:29:23.608 | INFO | BAZWRITER : 63.9%, 4.0, 131.6ms, 0 | 20260505 14:29:23.608 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 14:29:23.608 | INFO | Uploaded 36GiB to GPU in 2.19949s (16.3675GiB/s) | 20260505 14:29:23.608 | INFO | Downloaded 38GiB to GPU in 2.26327s (16.7899GiB/s) | 20260505 14:29:28.746 | INFO | Analyzing chunk frames = [3932672,3933184) | 20260505 14:29:28.746 | INFO | MemInfo: { | 20260505 14:29:28.746 | INFO | "gpu" : " 17 GB", | 20260505 14:29:28.746 | INFO | "huge" : "", | 20260505 14:29:28.746 | INFO | "huge_shared" : " 27 GB", | 20260505 14:29:28.746 | INFO | "malloc" : " 14 GB", | 20260505 14:29:28.746 | INFO | "pinned" : " 29 GB" | 20260505 14:29:28.746 | INFO | } | 20260505 14:29:28.746 | INFO | Perf report for the last 5.13899 seconds: | 20260505 14:29:28.746 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:29:28.746 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:28.746 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:28.746 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:28.746 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 14:29:28.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:28.746 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:28.746 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:29:28.747 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:28.747 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:28.747 | INFO | Uploaded 36GiB to GPU in 2.19173s (16.4254GiB/s) | 20260505 14:29:28.747 | INFO | Downloaded 34GiB to GPU in 2.02984s (16.7501GiB/s) | 20260505 14:29:33.882 | INFO | Analyzing chunk frames = [3933184,3933696) | 20260505 14:29:33.882 | INFO | MemInfo: { | 20260505 14:29:33.882 | INFO | "gpu" : " 17 GB", | 20260505 14:29:33.882 | INFO | "huge" : "", | 20260505 14:29:33.882 | INFO | "huge_shared" : " 27 GB", | 20260505 14:29:33.882 | INFO | "malloc" : " 14 GB", | 20260505 14:29:33.882 | INFO | "pinned" : " 29 GB" | 20260505 14:29:33.882 | INFO | } | 20260505 14:29:33.882 | INFO | Perf report for the last 5.13575 seconds: | 20260505 14:29:33.882 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:29:33.882 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:33.882 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:33.882 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:33.882 | INFO | ANALYSIS : 63.6%, 3.0, 98.9ms, 0 | 20260505 14:29:33.882 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:33.882 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:33.882 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:29:33.882 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:33.882 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:33.882 | INFO | Uploaded 36GiB to GPU in 2.24228s (16.0551GiB/s) | 20260505 14:29:33.883 | INFO | Downloaded 34GiB to GPU in 2.11873s (16.0474GiB/s) | 20260505 14:29:39.016 | INFO | Analyzing chunk frames = [3933696,3934208) | 20260505 14:29:39.016 | INFO | MemInfo: { | 20260505 14:29:39.016 | INFO | "gpu" : " 17 GB", | 20260505 14:29:39.016 | INFO | "huge" : "", | 20260505 14:29:39.016 | INFO | "huge_shared" : " 27 GB", | 20260505 14:29:39.016 | INFO | "malloc" : " 14 GB", | 20260505 14:29:39.016 | INFO | "pinned" : " 29 GB" | 20260505 14:29:39.016 | INFO | } | 20260505 14:29:39.016 | INFO | Perf report for the last 5.13388 seconds: | 20260505 14:29:39.016 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:29:39.016 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:39.016 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:39.016 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:39.016 | INFO | ANALYSIS : 63.1%, 3.0, 98.0ms, 0 | 20260505 14:29:39.016 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:39.016 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:39.016 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:29:39.016 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:39.016 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:39.016 | INFO | Uploaded 36GiB to GPU in 2.20463s (16.3292GiB/s) | 20260505 14:29:39.016 | INFO | Downloaded 34GiB to GPU in 2.13118s (15.9536GiB/s) | 20260505 14:29:44.166 | INFO | Analyzing chunk frames = [3934208,3934720) | 20260505 14:29:44.166 | INFO | MemInfo: { | 20260505 14:29:44.166 | INFO | "gpu" : " 17 GB", | 20260505 14:29:44.166 | INFO | "huge" : "", | 20260505 14:29:44.166 | INFO | "huge_shared" : " 27 GB", | 20260505 14:29:44.166 | INFO | "malloc" : " 14 GB", | 20260505 14:29:44.166 | INFO | "pinned" : " 29 GB" | 20260505 14:29:44.166 | INFO | } | 20260505 14:29:44.166 | INFO | Perf report for the last 5.14991 seconds: | 20260505 14:29:44.166 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:29:44.166 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:44.166 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:44.166 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:44.166 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 14:29:44.166 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:44.166 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:44.166 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:29:44.166 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:44.166 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:44.166 | INFO | Uploaded 36GiB to GPU in 2.18278s (16.4928GiB/s) | 20260505 14:29:44.166 | INFO | Downloaded 34GiB to GPU in 2.12533s (15.9976GiB/s) | 20260505 14:29:49.305 | INFO | Analyzing chunk frames = [3934720,3935232) | 20260505 14:29:49.305 | INFO | MemInfo: { | 20260505 14:29:49.305 | INFO | "gpu" : " 17 GB", | 20260505 14:29:49.305 | INFO | "huge" : "", | 20260505 14:29:49.305 | INFO | "huge_shared" : " 27 GB", | 20260505 14:29:49.305 | INFO | "malloc" : " 14 GB", | 20260505 14:29:49.305 | INFO | "pinned" : " 29 GB" | 20260505 14:29:49.305 | INFO | } | 20260505 14:29:49.305 | INFO | Perf report for the last 5.13895 seconds: | 20260505 14:29:49.305 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:29:49.305 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:49.305 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:49.305 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:49.305 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 14:29:49.305 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:49.305 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:49.305 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 14:29:49.305 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:49.305 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:49.305 | INFO | Uploaded 36GiB to GPU in 2.21298s (16.2676GiB/s) | 20260505 14:29:49.305 | INFO | Downloaded 34GiB to GPU in 2.11011s (16.1129GiB/s) | 20260505 14:29:54.440 | INFO | Analyzing chunk frames = [3935232,3935744) | 20260505 14:29:54.440 | INFO | MemInfo: { | 20260505 14:29:54.440 | INFO | "gpu" : " 17 GB", | 20260505 14:29:54.440 | INFO | "huge" : "", | 20260505 14:29:54.440 | INFO | "huge_shared" : " 27 GB", | 20260505 14:29:54.440 | INFO | "malloc" : " 14 GB", | 20260505 14:29:54.440 | INFO | "pinned" : " 29 GB" | 20260505 14:29:54.440 | INFO | } | 20260505 14:29:54.440 | INFO | Perf report for the last 5.1355 seconds: | 20260505 14:29:54.440 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:29:54.440 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:54.440 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:54.440 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:54.441 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 14:29:54.441 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:54.441 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:54.441 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 14:29:54.441 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:54.441 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:54.441 | INFO | Uploaded 36GiB to GPU in 2.14224s (16.8048GiB/s) | 20260505 14:29:54.441 | INFO | Downloaded 34GiB to GPU in 2.16912s (15.6746GiB/s) | 20260505 14:29:59.574 | INFO | Analyzing chunk frames = [3935744,3936256) | 20260505 14:29:59.574 | INFO | MemInfo: { | 20260505 14:29:59.574 | INFO | "gpu" : " 17 GB", | 20260505 14:29:59.574 | INFO | "huge" : "", | 20260505 14:29:59.574 | INFO | "huge_shared" : " 27 GB", | 20260505 14:29:59.574 | INFO | "malloc" : " 14 GB", | 20260505 14:29:59.574 | INFO | "pinned" : " 29 GB" | 20260505 14:29:59.574 | INFO | } | 20260505 14:29:59.574 | INFO | Perf report for the last 5.13393 seconds: | 20260505 14:29:59.574 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:29:59.574 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:59.574 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:59.574 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:59.574 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 14:29:59.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:59.575 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:59.575 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:29:59.575 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:59.575 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:29:59.575 | INFO | Uploaded 36GiB to GPU in 2.20962s (16.2924GiB/s) | 20260505 14:29:59.575 | INFO | Downloaded 34GiB to GPU in 2.10447s (16.1561GiB/s) | 20260505 14:29:59.708 | INFO | Starting a round of Baz IO | 20260505 14:30:02.940 | INFO | Populating RTMetrics report | 20260505 14:30:02.940 | INFO | RTMetrics report populated | 20260505 14:30:02.940 | INFO | Opening tmp RTMetrics json file | 20260505 14:30:02.951 | INFO | Serializing RTMetrics to json | 20260505 14:30:02.954 | INFO | Serialization done, swapping tmp file now... | 20260505 14:30:02.956 | INFO | RTMetrics done | 20260505 14:30:03.041 | INFO | Finished a round of Baz IO: Wrote 2.252 GiB in 3.333 seconds | 20260505 14:30:03.041 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.74% / 0.00% / 77.26% | 20260505 14:30:03.041 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.551112% / 0% / 7.54992e-05% | 20260505 14:30:03.041 | INFO | Baz IO overall: | 20260505 14:30:03.041 | INFO | BytesWritten : 2344956996654 | 20260505 14:30:03.041 | INFO | NumEvents : 1114857758936 | 20260505 14:30:04.723 | INFO | Analyzing chunk frames = [3936256,3936768) | 20260505 14:30:04.723 | INFO | MemInfo: { | 20260505 14:30:04.723 | INFO | "gpu" : " 17 GB", | 20260505 14:30:04.723 | INFO | "huge" : "", | 20260505 14:30:04.723 | INFO | "huge_shared" : " 27 GB", | 20260505 14:30:04.724 | INFO | "malloc" : " 14 GB", | 20260505 14:30:04.724 | INFO | "pinned" : " 29 GB" | 20260505 14:30:04.724 | INFO | } | 20260505 14:30:04.724 | INFO | Perf report for the last 5.14924 seconds: | 20260505 14:30:04.724 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:30:04.724 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:04.724 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:04.724 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:04.724 | INFO | ANALYSIS : 64.3%, 3.0, 100.1ms, 0 | 20260505 14:30:04.724 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:04.724 | INFO | RT_METRICS : 45.1%, 1.0, 23.7ms, 0 | 20260505 14:30:04.724 | INFO | PRE_HQ : 60.7%, 1.8, 57.2ms, 0 | 20260505 14:30:04.724 | INFO | BAZWRITER : 64.0%, 4.0, 130.8ms, 0 | 20260505 14:30:04.724 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 14:30:04.724 | INFO | Uploaded 36GiB to GPU in 2.1909s (16.4316GiB/s) | 20260505 14:30:04.724 | INFO | Downloaded 38GiB to GPU in 2.27836s (16.6787GiB/s) | 20260505 14:30:09.861 | INFO | Analyzing chunk frames = [3936768,3937280) | 20260505 14:30:09.861 | INFO | MemInfo: { | 20260505 14:30:09.861 | INFO | "gpu" : " 17 GB", | 20260505 14:30:09.861 | INFO | "huge" : "", | 20260505 14:30:09.861 | INFO | "huge_shared" : " 27 GB", | 20260505 14:30:09.861 | INFO | "malloc" : " 14 GB", | 20260505 14:30:09.861 | INFO | "pinned" : " 29 GB" | 20260505 14:30:09.861 | INFO | } | 20260505 14:30:09.861 | INFO | Perf report for the last 5.13729 seconds: | 20260505 14:30:09.861 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:30:09.861 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:09.861 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:09.861 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:09.861 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 14:30:09.861 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:09.861 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:09.861 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:30:09.861 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:09.861 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:09.861 | INFO | Uploaded 36GiB to GPU in 2.20393s (16.3344GiB/s) | 20260505 14:30:09.861 | INFO | Downloaded 34GiB to GPU in 2.05657s (16.5324GiB/s) | 20260505 14:30:14.998 | INFO | Analyzing chunk frames = [3937280,3937792) | 20260505 14:30:14.998 | INFO | MemInfo: { | 20260505 14:30:14.998 | INFO | "gpu" : " 17 GB", | 20260505 14:30:14.998 | INFO | "huge" : "", | 20260505 14:30:14.998 | INFO | "huge_shared" : " 27 GB", | 20260505 14:30:14.998 | INFO | "malloc" : " 14 GB", | 20260505 14:30:14.998 | INFO | "pinned" : " 29 GB" | 20260505 14:30:14.998 | INFO | } | 20260505 14:30:14.998 | INFO | Perf report for the last 5.13697 seconds: | 20260505 14:30:14.998 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:30:14.998 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:14.998 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:14.998 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:14.998 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260505 14:30:14.998 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:14.998 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:14.998 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:30:14.998 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:14.998 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:14.998 | INFO | Uploaded 36GiB to GPU in 2.1879s (16.4541GiB/s) | 20260505 14:30:14.998 | INFO | Downloaded 34GiB to GPU in 2.05487s (16.5461GiB/s) | 20260505 14:30:20.133 | INFO | Analyzing chunk frames = [3937792,3938304) | 20260505 14:30:20.133 | INFO | MemInfo: { | 20260505 14:30:20.133 | INFO | "gpu" : " 17 GB", | 20260505 14:30:20.133 | INFO | "huge" : "", | 20260505 14:30:20.133 | INFO | "huge_shared" : " 27 GB", | 20260505 14:30:20.133 | INFO | "malloc" : " 14 GB", | 20260505 14:30:20.133 | INFO | "pinned" : " 29 GB" | 20260505 14:30:20.133 | INFO | } | 20260505 14:30:20.133 | INFO | Perf report for the last 5.13549 seconds: | 20260505 14:30:20.134 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:30:20.134 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:20.134 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:20.134 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:20.134 | INFO | ANALYSIS : 62.8%, 3.0, 97.4ms, 0 | 20260505 14:30:20.134 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:20.134 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:20.134 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:30:20.134 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:20.134 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:20.134 | INFO | Uploaded 36GiB to GPU in 2.18202s (16.4985GiB/s) | 20260505 14:30:20.134 | INFO | Downloaded 34GiB to GPU in 2.1307s (15.9572GiB/s) | 20260505 14:30:25.270 | INFO | Analyzing chunk frames = [3938304,3938816) | 20260505 14:30:25.270 | INFO | MemInfo: { | 20260505 14:30:25.270 | INFO | "gpu" : " 17 GB", | 20260505 14:30:25.270 | INFO | "huge" : "", | 20260505 14:30:25.270 | INFO | "huge_shared" : " 27 GB", | 20260505 14:30:25.270 | INFO | "malloc" : " 14 GB", | 20260505 14:30:25.270 | INFO | "pinned" : " 29 GB" | 20260505 14:30:25.270 | INFO | } | 20260505 14:30:25.271 | INFO | Perf report for the last 5.13709 seconds: | 20260505 14:30:25.271 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:30:25.271 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:25.271 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:25.271 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:25.271 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260505 14:30:25.271 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:25.271 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:25.271 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:30:25.271 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:25.271 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:25.271 | INFO | Uploaded 36GiB to GPU in 2.16725s (16.6109GiB/s) | 20260505 14:30:25.271 | INFO | Downloaded 34GiB to GPU in 2.12855s (15.9733GiB/s) | 20260505 14:30:30.419 | INFO | Analyzing chunk frames = [3938816,3939328) | 20260505 14:30:30.419 | INFO | MemInfo: { | 20260505 14:30:30.419 | INFO | "gpu" : " 17 GB", | 20260505 14:30:30.419 | INFO | "huge" : "", | 20260505 14:30:30.419 | INFO | "huge_shared" : " 27 GB", | 20260505 14:30:30.419 | INFO | "malloc" : " 14 GB", | 20260505 14:30:30.419 | INFO | "pinned" : " 29 GB" | 20260505 14:30:30.419 | INFO | } | 20260505 14:30:30.419 | INFO | Perf report for the last 5.14881 seconds: | 20260505 14:30:30.420 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:30:30.420 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:30.420 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:30.420 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:30.420 | INFO | ANALYSIS : 62.9%, 3.0, 98.4ms, 0 | 20260505 14:30:30.420 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:30.420 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:30.420 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:30:30.420 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:30.420 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:30.420 | INFO | Uploaded 36GiB to GPU in 2.1939s (16.4091GiB/s) | 20260505 14:30:30.420 | INFO | Downloaded 34GiB to GPU in 2.1115s (16.1023GiB/s) | 20260505 14:30:35.560 | INFO | Analyzing chunk frames = [3939328,3939840) | 20260505 14:30:35.560 | INFO | MemInfo: { | 20260505 14:30:35.560 | INFO | "gpu" : " 17 GB", | 20260505 14:30:35.560 | INFO | "huge" : "", | 20260505 14:30:35.560 | INFO | "huge_shared" : " 27 GB", | 20260505 14:30:35.560 | INFO | "malloc" : " 14 GB", | 20260505 14:30:35.560 | INFO | "pinned" : " 29 GB" | 20260505 14:30:35.560 | INFO | } | 20260505 14:30:35.560 | INFO | Perf report for the last 5.14095 seconds: | 20260505 14:30:35.560 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:30:35.560 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:35.560 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:35.560 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:35.560 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 14:30:35.560 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:35.560 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:35.560 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 14:30:35.560 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:35.560 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:35.561 | INFO | Uploaded 36GiB to GPU in 2.11397s (17.0296GiB/s) | 20260505 14:30:35.561 | INFO | Downloaded 34GiB to GPU in 2.17207s (15.6533GiB/s) | 20260505 14:30:40.690 | INFO | Analyzing chunk frames = [3939840,3940352) | 20260505 14:30:40.690 | INFO | MemInfo: { | 20260505 14:30:40.690 | INFO | "gpu" : " 17 GB", | 20260505 14:30:40.690 | INFO | "huge" : "", | 20260505 14:30:40.690 | INFO | "huge_shared" : " 27 GB", | 20260505 14:30:40.690 | INFO | "malloc" : " 14 GB", | 20260505 14:30:40.690 | INFO | "pinned" : " 29 GB" | 20260505 14:30:40.690 | INFO | } | 20260505 14:30:40.691 | INFO | Perf report for the last 5.13018 seconds: | 20260505 14:30:40.691 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:30:40.691 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:40.691 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:40.691 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:40.691 | INFO | ANALYSIS : 63.8%, 3.0, 99.4ms, 0 | 20260505 14:30:40.691 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:40.691 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:40.691 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:30:40.691 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:40.691 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:40.691 | INFO | Uploaded 36GiB to GPU in 2.22975s (16.1453GiB/s) | 20260505 14:30:40.691 | INFO | Downloaded 34GiB to GPU in 2.09445s (16.2334GiB/s) | 20260505 14:30:40.691 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 14:30:40.691 | INFO | Total mem usage: 15.3 GiB | 20260505 14:30:40.691 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 14:30:40.691 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.72%) | 20260505 14:30:40.691 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 14:30:40.691 | INFO | High Water Breakdown: | 20260505 14:30:40.691 | INFO | 61% : 8.67 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 14:30:40.691 | INFO | 38.9% : 5.54 GiB ( 83.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 14:30:40.691 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 14:30:40.691 | INFO | ------------------------------------------ | 20260505 14:30:40.691 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:30:40.691 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 14:30:40.691 | INFO | Total mem usage: 26.5 GiB | 20260505 14:30:40.691 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 14:30:40.691 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 14:30:40.691 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 14:30:40.691 | INFO | High Water Breakdown: | 20260505 14:30:40.691 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:30:40.691 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:30:40.691 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:30:40.691 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:30:40.691 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:30:40.691 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:30:40.691 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 14:30:40.691 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 14:30:40.691 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:30:40.691 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 14:30:40.691 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 14:30:40.691 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 14:30:40.691 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 14:30:40.691 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 14:30:40.691 | INFO | ------------------------------------------ | 20260505 14:30:40.691 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:30:40.691 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 14:30:40.691 | INFO | Total mem usage: 25.6 GiB | 20260505 14:30:40.691 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 14:30:40.691 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 14:30:40.692 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 14:30:40.692 | INFO | High Water Breakdown: | 20260505 14:30:40.692 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 14:30:40.692 | INFO | ------------------------------------------ | 20260505 14:30:40.692 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:30:40.692 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 14:30:40.692 | INFO | Total mem usage: 15.7 GiB | 20260505 14:30:40.692 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 14:30:40.692 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 14:30:40.692 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 14:30:40.692 | INFO | High Water Breakdown: | 20260505 14:30:40.692 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 14:30:40.692 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 14:30:40.692 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 14:30:40.692 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 14:30:40.692 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 14:30:40.692 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 14:30:40.692 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 14:30:40.692 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 14:30:40.692 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 14:30:40.692 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:30:40.692 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 14:30:40.692 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:30:40.692 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 14:30:40.692 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 14:30:40.692 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 14:30:40.692 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 14:30:40.692 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:30:40.692 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 14:30:40.692 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:30:40.692 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:30:40.692 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:30:40.692 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 14:30:40.692 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 14:30:40.692 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:30:40.692 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 14:30:40.692 | INFO | ------------------------------------------ | 20260505 14:30:40.692 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:30:40.692 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 14:30:40.692 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:30:40.692 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:30:40.692 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.61e+01, 1.38e+01 / 1.97e+00 | 20260505 14:30:40.692 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 1.00e-01, 1.09e-02 / 3.82e-03 | 20260505 14:30:40.692 | INFO | PerfReport: 11.74%, Baseline -- 3.73e+00 / 4.49e+00, 5.68e-01 / 1.69e-01 | 20260505 14:30:40.692 | INFO | PerfReport: 3.29%, Histogram -- 1.04e+00 / 1.50e+00, 1.95e-01 / 9.52e-02 | 20260505 14:30:40.692 | INFO | PerfReport: 16.01%, DME -- 4.07e+01 / 4.55e+01, 4.53e+00 / 2.05e+00 | 20260505 14:30:40.692 | INFO | PerfReport: 56.46%, FrameLabeling -- 1.79e+01 / 1.98e+01, 1.38e+00 / 1.80e+00 | 20260505 14:30:40.692 | INFO | PerfReport: 7.44%, PulseAccumulating -- 2.36e+00 / 2.93e+00, 4.49e-01 / 1.80e-01 | 20260505 14:30:40.692 | INFO | PerfReport: 3.52%, Metrics -- 1.12e+00 / 1.42e+00, 3.10e-01 / 1.22e-01 | 20260505 14:30:40.692 | INFO | PerfReport: 1.24%, PulseSerializing -- 3.93e-01 / 4.41e-01, 6.94e-02 / 2.88e-02 | 20260505 14:30:40.692 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 14:30:40.692 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:30:40.692 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:30:40.692 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.57e+01 / 0.00e+00 | 20260505 14:30:40.692 | INFO | PerfReport: 22.63%, Upload -- 2.28e+01 / 0.00e+00, 8.16e+00 / 0.00e+00 | 20260505 14:30:40.692 | INFO | PerfReport: 22.00%, Download -- 2.22e+01 / 0.00e+00, 8.08e+00 / 0.00e+00 | 20260505 14:30:40.692 | INFO | PerfReport: 31.48%, Compute -- 3.18e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 14:30:40.692 | INFO | PerfReport: 2.41%, LockUpload -- 2.43e+00 / 0.00e+00, 8.08e+00 / 0.00e+00 | 20260505 14:30:40.692 | INFO | PerfReport: 1.36%, LockDownload -- 1.37e+00 / 0.00e+00, 2.84e+00 / 0.00e+00 | 20260505 14:30:40.692 | INFO | PerfReport: 20.11%, LockCompute -- 2.03e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 14:30:40.692 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.48e-02 / 0.00e+00 | 20260505 14:30:40.828 | INFO | Starting a round of Baz IO | 20260505 14:30:44.052 | INFO | Populating RTMetrics report | 20260505 14:30:44.052 | INFO | RTMetrics report populated | 20260505 14:30:44.052 | INFO | Opening tmp RTMetrics json file | 20260505 14:30:44.059 | INFO | Serializing RTMetrics to json | 20260505 14:30:44.075 | INFO | Serialization done, swapping tmp file now... | 20260505 14:30:44.077 | INFO | RTMetrics done | 20260505 14:30:44.165 | INFO | Finished a round of Baz IO: Wrote 2.253 GiB in 3.336 seconds | 20260505 14:30:44.165 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.74% / 0.00% / 77.26% | 20260505 14:30:44.165 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.551514% / 7.94729e-06% / 9.93411e-05% | 20260505 14:30:44.165 | INFO | Baz IO overall: | 20260505 14:30:44.165 | INFO | BytesWritten : 2347376188687 | 20260505 14:30:44.165 | INFO | NumEvents : 1115998010361 | 20260505 14:30:45.827 | INFO | Analyzing chunk frames = [3940352,3940864) | 20260505 14:30:45.827 | INFO | MemInfo: { | 20260505 14:30:45.827 | INFO | "gpu" : " 17 GB", | 20260505 14:30:45.827 | INFO | "huge" : "", | 20260505 14:30:45.827 | INFO | "huge_shared" : " 27 GB", | 20260505 14:30:45.827 | INFO | "malloc" : " 14 GB", | 20260505 14:30:45.827 | INFO | "pinned" : " 29 GB" | 20260505 14:30:45.827 | INFO | } | 20260505 14:30:45.827 | INFO | Perf report for the last 5.13687 seconds: | 20260505 14:30:45.827 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:30:45.827 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:45.827 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:45.828 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:45.828 | INFO | ANALYSIS : 64.3%, 3.0, 99.8ms, 0 | 20260505 14:30:45.828 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:45.828 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260505 14:30:45.828 | INFO | PRE_HQ : 61.0%, 1.8, 57.4ms, 0 | 20260505 14:30:45.828 | INFO | BAZWRITER : 64.3%, 4.0, 132.0ms, 0 | 20260505 14:30:45.828 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260505 14:30:45.828 | INFO | Uploaded 36GiB to GPU in 2.17898s (16.5215GiB/s) | 20260505 14:30:45.828 | INFO | Downloaded 38GiB to GPU in 2.27219s (16.724GiB/s) | 20260505 14:30:50.973 | INFO | Analyzing chunk frames = [3940864,3941376) | 20260505 14:30:50.973 | INFO | MemInfo: { | 20260505 14:30:50.973 | INFO | "gpu" : " 17 GB", | 20260505 14:30:50.973 | INFO | "huge" : "", | 20260505 14:30:50.973 | INFO | "huge_shared" : " 27 GB", | 20260505 14:30:50.973 | INFO | "malloc" : " 14 GB", | 20260505 14:30:50.973 | INFO | "pinned" : " 29 GB" | 20260505 14:30:50.973 | INFO | } | 20260505 14:30:50.973 | INFO | Perf report for the last 5.14547 seconds: | 20260505 14:30:50.973 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:30:50.973 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:50.973 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:50.973 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:50.973 | INFO | ANALYSIS : 63.3%, 3.0, 99.0ms, 0 | 20260505 14:30:50.973 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:50.973 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:50.973 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 14:30:50.973 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:50.973 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:50.973 | INFO | Uploaded 36GiB to GPU in 2.16347s (16.64GiB/s) | 20260505 14:30:50.973 | INFO | Downloaded 34GiB to GPU in 2.02652s (16.7775GiB/s) | 20260505 14:30:56.114 | INFO | Analyzing chunk frames = [3941376,3941888) | 20260505 14:30:56.114 | INFO | MemInfo: { | 20260505 14:30:56.114 | INFO | "gpu" : " 17 GB", | 20260505 14:30:56.114 | INFO | "huge" : "", | 20260505 14:30:56.114 | INFO | "huge_shared" : " 27 GB", | 20260505 14:30:56.114 | INFO | "malloc" : " 14 GB", | 20260505 14:30:56.114 | INFO | "pinned" : " 29 GB" | 20260505 14:30:56.114 | INFO | } | 20260505 14:30:56.114 | INFO | Perf report for the last 5.14115 seconds: | 20260505 14:30:56.114 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:30:56.114 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:56.114 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:56.114 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:56.114 | INFO | ANALYSIS : 63.6%, 3.0, 99.0ms, 0 | 20260505 14:30:56.114 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:56.114 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:56.114 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 14:30:56.114 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:56.114 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:30:56.114 | INFO | Uploaded 36GiB to GPU in 2.24248s (16.0536GiB/s) | 20260505 14:30:56.114 | INFO | Downloaded 34GiB to GPU in 2.10955s (16.1172GiB/s) | 20260505 14:31:01.249 | INFO | Analyzing chunk frames = [3941888,3942400) | 20260505 14:31:01.249 | INFO | MemInfo: { | 20260505 14:31:01.249 | INFO | "gpu" : " 17 GB", | 20260505 14:31:01.249 | INFO | "huge" : "", | 20260505 14:31:01.249 | INFO | "huge_shared" : " 27 GB", | 20260505 14:31:01.249 | INFO | "malloc" : " 14 GB", | 20260505 14:31:01.249 | INFO | "pinned" : " 29 GB" | 20260505 14:31:01.249 | INFO | } | 20260505 14:31:01.249 | INFO | Perf report for the last 5.13531 seconds: | 20260505 14:31:01.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:31:01.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:01.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:01.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:01.250 | INFO | ANALYSIS : 62.9%, 3.0, 97.7ms, 0 | 20260505 14:31:01.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:01.250 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:01.250 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:31:01.250 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:01.250 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:01.250 | INFO | Uploaded 36GiB to GPU in 2.1913s (16.4286GiB/s) | 20260505 14:31:01.250 | INFO | Downloaded 34GiB to GPU in 2.13544s (15.9218GiB/s) | 20260505 14:31:06.383 | INFO | Analyzing chunk frames = [3942400,3942912) | 20260505 14:31:06.383 | INFO | MemInfo: { | 20260505 14:31:06.383 | INFO | "gpu" : " 17 GB", | 20260505 14:31:06.383 | INFO | "huge" : "", | 20260505 14:31:06.383 | INFO | "huge_shared" : " 27 GB", | 20260505 14:31:06.383 | INFO | "malloc" : " 14 GB", | 20260505 14:31:06.383 | INFO | "pinned" : " 29 GB" | 20260505 14:31:06.383 | INFO | } | 20260505 14:31:06.383 | INFO | Perf report for the last 5.13398 seconds: | 20260505 14:31:06.383 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:31:06.384 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:06.384 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:06.384 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:06.384 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260505 14:31:06.384 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:06.384 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:06.384 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:31:06.384 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:06.384 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:06.384 | INFO | Uploaded 36GiB to GPU in 2.16602s (16.6203GiB/s) | 20260505 14:31:06.384 | INFO | Downloaded 34GiB to GPU in 2.12619s (15.9911GiB/s) | 20260505 14:31:11.527 | INFO | Analyzing chunk frames = [3942912,3943424) | 20260505 14:31:11.527 | INFO | MemInfo: { | 20260505 14:31:11.527 | INFO | "gpu" : " 17 GB", | 20260505 14:31:11.527 | INFO | "huge" : "", | 20260505 14:31:11.527 | INFO | "huge_shared" : " 27 GB", | 20260505 14:31:11.527 | INFO | "malloc" : " 14 GB", | 20260505 14:31:11.527 | INFO | "pinned" : " 29 GB" | 20260505 14:31:11.527 | INFO | } | 20260505 14:31:11.528 | INFO | Perf report for the last 5.14423 seconds: | 20260505 14:31:11.528 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:31:11.528 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:11.528 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:11.528 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:11.528 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 14:31:11.528 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:11.528 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:11.528 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 14:31:11.528 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:11.528 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:11.528 | INFO | Uploaded 36GiB to GPU in 2.17561s (16.5471GiB/s) | 20260505 14:31:11.528 | INFO | Downloaded 34GiB to GPU in 2.09854s (16.2017GiB/s) | 20260505 14:31:16.677 | INFO | Analyzing chunk frames = [3943424,3943936) | 20260505 14:31:16.677 | INFO | MemInfo: { | 20260505 14:31:16.677 | INFO | "gpu" : " 17 GB", | 20260505 14:31:16.677 | INFO | "huge" : "", | 20260505 14:31:16.677 | INFO | "huge_shared" : " 27 GB", | 20260505 14:31:16.678 | INFO | "malloc" : " 14 GB", | 20260505 14:31:16.678 | INFO | "pinned" : " 29 GB" | 20260505 14:31:16.678 | INFO | } | 20260505 14:31:16.678 | INFO | Perf report for the last 5.15021 seconds: | 20260505 14:31:16.678 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:31:16.678 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:16.678 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:16.678 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:16.678 | INFO | ANALYSIS : 63.3%, 3.0, 99.1ms, 0 | 20260505 14:31:16.678 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:16.678 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:16.678 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 14:31:16.678 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:16.678 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:16.678 | INFO | Uploaded 36GiB to GPU in 2.12202s (16.9649GiB/s) | 20260505 14:31:16.678 | INFO | Downloaded 34GiB to GPU in 2.17819s (15.6093GiB/s) | 20260505 14:31:21.813 | INFO | Analyzing chunk frames = [3943936,3944448) | 20260505 14:31:21.813 | INFO | MemInfo: { | 20260505 14:31:21.813 | INFO | "gpu" : " 17 GB", | 20260505 14:31:21.813 | INFO | "huge" : "", | 20260505 14:31:21.813 | INFO | "huge_shared" : " 27 GB", | 20260505 14:31:21.813 | INFO | "malloc" : " 14 GB", | 20260505 14:31:21.813 | INFO | "pinned" : " 29 GB" | 20260505 14:31:21.813 | INFO | } | 20260505 14:31:21.813 | INFO | Perf report for the last 5.13568 seconds: | 20260505 14:31:21.813 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:31:21.813 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:21.813 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:21.813 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:21.813 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 14:31:21.813 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:21.814 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:21.814 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:31:21.814 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:21.814 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:21.814 | INFO | Uploaded 36GiB to GPU in 2.15678s (16.6915GiB/s) | 20260505 14:31:21.814 | INFO | Downloaded 34GiB to GPU in 2.02708s (16.7729GiB/s) | 20260505 14:31:21.952 | INFO | Starting a round of Baz IO | 20260505 14:31:25.171 | INFO | Populating RTMetrics report | 20260505 14:31:25.171 | INFO | RTMetrics report populated | 20260505 14:31:25.171 | INFO | Opening tmp RTMetrics json file | 20260505 14:31:25.222 | INFO | Serializing RTMetrics to json | 20260505 14:31:25.264 | INFO | Serialization done, swapping tmp file now... | 20260505 14:31:25.266 | INFO | RTMetrics done | 20260505 14:31:25.282 | INFO | Finished a round of Baz IO: Wrote 2.252 GiB in 3.329 seconds | 20260505 14:31:25.282 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.73% / 0.00% / 77.26% | 20260505 14:31:25.282 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.548549% / 0% / 0.000103315% | 20260505 14:31:25.282 | INFO | Baz IO overall: | 20260505 14:31:25.282 | INFO | BytesWritten : 2349794184367 | 20260505 14:31:25.282 | INFO | NumEvents : 1117137713293 | 20260505 14:31:26.939 | INFO | Analyzing chunk frames = [3944448,3944960) | 20260505 14:31:26.939 | INFO | MemInfo: { | 20260505 14:31:26.939 | INFO | "gpu" : " 17 GB", | 20260505 14:31:26.939 | INFO | "huge" : "", | 20260505 14:31:26.939 | INFO | "huge_shared" : " 27 GB", | 20260505 14:31:26.939 | INFO | "malloc" : " 14 GB", | 20260505 14:31:26.939 | INFO | "pinned" : " 29 GB" | 20260505 14:31:26.939 | INFO | } | 20260505 14:31:26.940 | INFO | Perf report for the last 5.12601 seconds: | 20260505 14:31:26.940 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:31:26.940 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:26.940 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:26.940 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:26.940 | INFO | ANALYSIS : 64.4%, 3.0, 99.8ms, 0 | 20260505 14:31:26.940 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:26.940 | INFO | RT_METRICS : 46.9%, 1.0, 24.5ms, 0 | 20260505 14:31:26.940 | INFO | PRE_HQ : 61.1%, 1.8, 57.3ms, 0 | 20260505 14:31:26.940 | INFO | BAZWRITER : 64.3%, 4.0, 132.8ms, 0 | 20260505 14:31:26.940 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260505 14:31:26.940 | INFO | Uploaded 36GiB to GPU in 2.20445s (16.3306GiB/s) | 20260505 14:31:26.940 | INFO | Downloaded 38GiB to GPU in 2.26633s (16.7672GiB/s) | 20260505 14:31:32.084 | INFO | Analyzing chunk frames = [3944960,3945472) | 20260505 14:31:32.084 | INFO | MemInfo: { | 20260505 14:31:32.084 | INFO | "gpu" : " 17 GB", | 20260505 14:31:32.084 | INFO | "huge" : "", | 20260505 14:31:32.084 | INFO | "huge_shared" : " 27 GB", | 20260505 14:31:32.084 | INFO | "malloc" : " 14 GB", | 20260505 14:31:32.084 | INFO | "pinned" : " 29 GB" | 20260505 14:31:32.084 | INFO | } | 20260505 14:31:32.084 | INFO | Perf report for the last 5.14478 seconds: | 20260505 14:31:32.084 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:31:32.084 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:32.084 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:32.084 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:32.084 | INFO | ANALYSIS : 63.5%, 3.0, 99.3ms, 0 | 20260505 14:31:32.084 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:32.084 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:32.084 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:31:32.084 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:32.084 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:32.085 | INFO | Uploaded 36GiB to GPU in 2.19672s (16.3881GiB/s) | 20260505 14:31:32.085 | INFO | Downloaded 34GiB to GPU in 2.04015s (16.6654GiB/s) | 20260505 14:31:37.229 | INFO | Analyzing chunk frames = [3945472,3945984) | 20260505 14:31:37.229 | INFO | MemInfo: { | 20260505 14:31:37.229 | INFO | "gpu" : " 17 GB", | 20260505 14:31:37.229 | INFO | "huge" : "", | 20260505 14:31:37.229 | INFO | "huge_shared" : " 27 GB", | 20260505 14:31:37.229 | INFO | "malloc" : " 14 GB", | 20260505 14:31:37.229 | INFO | "pinned" : " 29 GB" | 20260505 14:31:37.229 | INFO | } | 20260505 14:31:37.230 | INFO | Perf report for the last 5.14537 seconds: | 20260505 14:31:37.230 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:31:37.230 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:37.230 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:37.230 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:37.230 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 14:31:37.230 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:37.230 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:37.230 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:31:37.230 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:37.230 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:37.230 | INFO | Uploaded 36GiB to GPU in 2.2333s (16.1197GiB/s) | 20260505 14:31:37.230 | INFO | Downloaded 34GiB to GPU in 2.10665s (16.1394GiB/s) | 20260505 14:31:42.371 | INFO | Analyzing chunk frames = [3945984,3946496) | 20260505 14:31:42.371 | INFO | MemInfo: { | 20260505 14:31:42.371 | INFO | "gpu" : " 17 GB", | 20260505 14:31:42.371 | INFO | "huge" : "", | 20260505 14:31:42.371 | INFO | "huge_shared" : " 27 GB", | 20260505 14:31:42.371 | INFO | "malloc" : " 14 GB", | 20260505 14:31:42.371 | INFO | "pinned" : " 29 GB" | 20260505 14:31:42.371 | INFO | } | 20260505 14:31:42.371 | INFO | Perf report for the last 5.14117 seconds: | 20260505 14:31:42.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:31:42.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:42.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:42.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:42.371 | INFO | ANALYSIS : 62.7%, 3.0, 97.4ms, 0 | 20260505 14:31:42.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:42.371 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:42.371 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:31:42.371 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:42.371 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:42.371 | INFO | Uploaded 36GiB to GPU in 2.19879s (16.3726GiB/s) | 20260505 14:31:42.371 | INFO | Downloaded 34GiB to GPU in 2.13148s (15.9514GiB/s) | 20260505 14:31:47.511 | INFO | Analyzing chunk frames = [3946496,3947008) | 20260505 14:31:47.511 | INFO | MemInfo: { | 20260505 14:31:47.511 | INFO | "gpu" : " 17 GB", | 20260505 14:31:47.511 | INFO | "huge" : "", | 20260505 14:31:47.511 | INFO | "huge_shared" : " 27 GB", | 20260505 14:31:47.511 | INFO | "malloc" : " 14 GB", | 20260505 14:31:47.511 | INFO | "pinned" : " 29 GB" | 20260505 14:31:47.511 | INFO | } | 20260505 14:31:47.511 | INFO | Perf report for the last 5.14037 seconds: | 20260505 14:31:47.511 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:31:47.511 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:47.511 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:47.511 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:47.511 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 14:31:47.511 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:47.511 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:47.511 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:31:47.511 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:47.511 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:47.512 | INFO | Uploaded 36GiB to GPU in 2.17773s (16.5309GiB/s) | 20260505 14:31:47.512 | INFO | Downloaded 34GiB to GPU in 2.1301s (15.9617GiB/s) | 20260505 14:31:52.639 | INFO | Analyzing chunk frames = [3947008,3947520) | 20260505 14:31:52.639 | INFO | MemInfo: { | 20260505 14:31:52.639 | INFO | "gpu" : " 17 GB", | 20260505 14:31:52.639 | INFO | "huge" : "", | 20260505 14:31:52.639 | INFO | "huge_shared" : " 27 GB", | 20260505 14:31:52.639 | INFO | "malloc" : " 14 GB", | 20260505 14:31:52.639 | INFO | "pinned" : " 29 GB" | 20260505 14:31:52.639 | INFO | } | 20260505 14:31:52.639 | INFO | Perf report for the last 5.12817 seconds: | 20260505 14:31:52.640 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:31:52.640 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:52.640 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:52.640 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:52.640 | INFO | ANALYSIS : 63.4%, 3.0, 98.8ms, 0 | 20260505 14:31:52.640 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:52.640 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:52.640 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 14:31:52.640 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:52.640 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:52.640 | INFO | Uploaded 36GiB to GPU in 2.2162s (16.244GiB/s) | 20260505 14:31:52.640 | INFO | Downloaded 34GiB to GPU in 2.11109s (16.1054GiB/s) | 20260505 14:31:57.783 | INFO | Analyzing chunk frames = [3947520,3948032) | 20260505 14:31:57.783 | INFO | MemInfo: { | 20260505 14:31:57.783 | INFO | "gpu" : " 17 GB", | 20260505 14:31:57.783 | INFO | "huge" : "", | 20260505 14:31:57.783 | INFO | "huge_shared" : " 27 GB", | 20260505 14:31:57.783 | INFO | "malloc" : " 14 GB", | 20260505 14:31:57.783 | INFO | "pinned" : " 29 GB" | 20260505 14:31:57.783 | INFO | } | 20260505 14:31:57.783 | INFO | Perf report for the last 5.14347 seconds: | 20260505 14:31:57.783 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:31:57.783 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:57.783 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:57.783 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:57.783 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 14:31:57.783 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:57.783 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:57.783 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 14:31:57.783 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:57.783 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:31:57.783 | INFO | Uploaded 36GiB to GPU in 2.15142s (16.7331GiB/s) | 20260505 14:31:57.783 | INFO | Downloaded 34GiB to GPU in 2.18444s (15.5646GiB/s) | 20260505 14:32:02.924 | INFO | Analyzing chunk frames = [3948032,3948544) | 20260505 14:32:02.924 | INFO | MemInfo: { | 20260505 14:32:02.924 | INFO | "gpu" : " 17 GB", | 20260505 14:32:02.924 | INFO | "huge" : "", | 20260505 14:32:02.924 | INFO | "huge_shared" : " 27 GB", | 20260505 14:32:02.924 | INFO | "malloc" : " 14 GB", | 20260505 14:32:02.924 | INFO | "pinned" : " 29 GB" | 20260505 14:32:02.924 | INFO | } | 20260505 14:32:02.924 | INFO | Perf report for the last 5.14101 seconds: | 20260505 14:32:02.924 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:32:02.924 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:02.924 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:02.924 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:02.924 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 14:32:02.924 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:02.924 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:02.924 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 14:32:02.924 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:02.924 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:02.924 | INFO | Uploaded 36GiB to GPU in 2.21s (16.2896GiB/s) | 20260505 14:32:02.924 | INFO | Downloaded 34GiB to GPU in 2.11016s (16.1125GiB/s) | 20260505 14:32:02.925 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 14:32:02.925 | INFO | Total mem usage: 15.3 GiB | 20260505 14:32:02.925 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 14:32:02.925 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.82%) | 20260505 14:32:02.925 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 14:32:02.925 | INFO | High Water Breakdown: | 20260505 14:32:02.925 | INFO | 61% : 8.68 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 14:32:02.925 | INFO | 38.8% : 5.52 GiB ( 83.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 14:32:02.925 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 14:32:02.925 | INFO | ------------------------------------------ | 20260505 14:32:02.925 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:32:02.925 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 14:32:02.925 | INFO | Total mem usage: 26.5 GiB | 20260505 14:32:02.925 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 14:32:02.925 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 14:32:02.925 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 14:32:02.925 | INFO | High Water Breakdown: | 20260505 14:32:02.925 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:32:02.925 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:32:02.925 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:32:02.925 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:32:02.925 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:32:02.925 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:32:02.925 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 14:32:02.925 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 14:32:02.925 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:32:02.925 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 14:32:02.925 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 14:32:02.925 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 14:32:02.925 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 14:32:02.925 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 14:32:02.925 | INFO | ------------------------------------------ | 20260505 14:32:02.925 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:32:02.925 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 14:32:02.925 | INFO | Total mem usage: 25.6 GiB | 20260505 14:32:02.925 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 14:32:02.925 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 14:32:02.925 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 14:32:02.925 | INFO | High Water Breakdown: | 20260505 14:32:02.925 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 14:32:02.925 | INFO | ------------------------------------------ | 20260505 14:32:02.925 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:32:02.925 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 14:32:02.925 | INFO | Total mem usage: 15.7 GiB | 20260505 14:32:02.925 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 14:32:02.925 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 14:32:02.925 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 14:32:02.925 | INFO | High Water Breakdown: | 20260505 14:32:02.925 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 14:32:02.925 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 14:32:02.925 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 14:32:02.925 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 14:32:02.925 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 14:32:02.925 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 14:32:02.925 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 14:32:02.925 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 14:32:02.925 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 14:32:02.925 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:32:02.925 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 14:32:02.925 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:32:02.925 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 14:32:02.925 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 14:32:02.925 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 14:32:02.925 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 14:32:02.925 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:32:02.925 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 14:32:02.925 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:32:02.925 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:32:02.925 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:32:02.925 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 14:32:02.925 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 14:32:02.925 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:32:02.925 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 14:32:02.925 | INFO | ------------------------------------------ | 20260505 14:32:02.925 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:32:02.926 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 14:32:02.926 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:32:02.926 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:32:02.926 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.65e+01, 1.38e+01 / 1.89e+00 | 20260505 14:32:02.926 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 9.81e-02, 1.08e-02 / 4.93e-03 | 20260505 14:32:02.926 | INFO | PerfReport: 11.75%, Baseline -- 3.74e+00 / 4.45e+00, 5.80e-01 / 2.07e-01 | 20260505 14:32:02.926 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.46e+00, 1.93e-01 / 1.29e-01 | 20260505 14:32:02.926 | INFO | PerfReport: 16.02%, DME -- 4.08e+01 / 4.59e+01, 4.66e+00 / 2.36e+00 | 20260505 14:32:02.926 | INFO | PerfReport: 56.51%, FrameLabeling -- 1.80e+01 / 1.98e+01, 1.42e+00 / 1.78e+00 | 20260505 14:32:02.926 | INFO | PerfReport: 7.40%, PulseAccumulating -- 2.35e+00 / 2.93e+00, 4.51e-01 / 2.02e-01 | 20260505 14:32:02.926 | INFO | PerfReport: 3.50%, Metrics -- 1.11e+00 / 1.43e+00, 3.09e-01 / 1.71e-01 | 20260505 14:32:02.926 | INFO | PerfReport: 1.23%, PulseSerializing -- 3.92e-01 / 4.40e-01, 6.92e-02 / 3.28e-02 | 20260505 14:32:02.926 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 14:32:02.926 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:32:02.926 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:32:02.926 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.56e+01 / 0.00e+00 | 20260505 14:32:02.926 | INFO | PerfReport: 22.59%, Upload -- 2.28e+01 / 0.00e+00, 8.16e+00 / 0.00e+00 | 20260505 14:32:02.926 | INFO | PerfReport: 21.99%, Download -- 2.22e+01 / 0.00e+00, 8.10e+00 / 0.00e+00 | 20260505 14:32:02.926 | INFO | PerfReport: 31.54%, Compute -- 3.18e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 14:32:02.926 | INFO | PerfReport: 2.36%, LockUpload -- 2.38e+00 / 0.00e+00, 7.98e+00 / 0.00e+00 | 20260505 14:32:02.926 | INFO | PerfReport: 1.40%, LockDownload -- 1.41e+00 / 0.00e+00, 2.97e+00 / 0.00e+00 | 20260505 14:32:02.926 | INFO | PerfReport: 20.12%, LockCompute -- 2.03e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 14:32:02.926 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.64e-02 / 0.00e+00 | 20260505 14:32:03.059 | INFO | Starting a round of Baz IO | 20260505 14:32:06.276 | INFO | Populating RTMetrics report | 20260505 14:32:06.276 | INFO | RTMetrics report populated | 20260505 14:32:06.276 | INFO | Opening tmp RTMetrics json file | 20260505 14:32:06.296 | INFO | Serializing RTMetrics to json | 20260505 14:32:06.318 | INFO | Serialization done, swapping tmp file now... | 20260505 14:32:06.328 | INFO | RTMetrics done | 20260505 14:32:06.384 | INFO | Finished a round of Baz IO: Wrote 2.252 GiB in 3.324 seconds | 20260505 14:32:06.384 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.73% / 0.00% / 77.27% | 20260505 14:32:06.384 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.547608% / 0% / 9.53674e-05% | 20260505 14:32:06.384 | INFO | Baz IO overall: | 20260505 14:32:06.384 | INFO | BytesWritten : 2352211998555 | 20260505 14:32:06.384 | INFO | NumEvents : 1118277590955 | 20260505 14:32:08.070 | INFO | Analyzing chunk frames = [3948544,3949056) | 20260505 14:32:08.070 | INFO | MemInfo: { | 20260505 14:32:08.070 | INFO | "gpu" : " 17 GB", | 20260505 14:32:08.070 | INFO | "huge" : "", | 20260505 14:32:08.070 | INFO | "huge_shared" : " 27 GB", | 20260505 14:32:08.070 | INFO | "malloc" : " 14 GB", | 20260505 14:32:08.070 | INFO | "pinned" : " 29 GB" | 20260505 14:32:08.070 | INFO | } | 20260505 14:32:08.070 | INFO | Perf report for the last 5.14574 seconds: | 20260505 14:32:08.070 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:32:08.070 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:08.070 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:08.070 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:08.070 | INFO | ANALYSIS : 64.0%, 3.0, 98.7ms, 0 | 20260505 14:32:08.070 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:08.070 | INFO | RT_METRICS : 45.8%, 1.0, 23.8ms, 0 | 20260505 14:32:08.070 | INFO | PRE_HQ : 60.7%, 1.8, 56.5ms, 0 | 20260505 14:32:08.070 | INFO | BAZWRITER : 63.9%, 4.0, 130.3ms, 0 | 20260505 14:32:08.070 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.5ms, 0 | 20260505 14:32:08.070 | INFO | Uploaded 36GiB to GPU in 2.18943s (16.4426GiB/s) | 20260505 14:32:08.070 | INFO | Downloaded 38GiB to GPU in 2.25992s (16.8148GiB/s) | 20260505 14:32:13.198 | INFO | Analyzing chunk frames = [3949056,3949568) | 20260505 14:32:13.198 | INFO | MemInfo: { | 20260505 14:32:13.198 | INFO | "gpu" : " 17 GB", | 20260505 14:32:13.198 | INFO | "huge" : "", | 20260505 14:32:13.198 | INFO | "huge_shared" : " 27 GB", | 20260505 14:32:13.198 | INFO | "malloc" : " 14 GB", | 20260505 14:32:13.198 | INFO | "pinned" : " 29 GB" | 20260505 14:32:13.198 | INFO | } | 20260505 14:32:13.198 | INFO | Perf report for the last 5.12818 seconds: | 20260505 14:32:13.198 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:32:13.198 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:13.198 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:13.198 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:13.198 | INFO | ANALYSIS : 63.6%, 3.0, 99.1ms, 0 | 20260505 14:32:13.198 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:13.198 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:13.198 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:32:13.198 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:13.198 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:13.198 | INFO | Uploaded 36GiB to GPU in 2.19888s (16.372GiB/s) | 20260505 14:32:13.198 | INFO | Downloaded 34GiB to GPU in 2.04358s (16.6375GiB/s) | 20260505 14:32:18.335 | INFO | Analyzing chunk frames = [3949568,3950080) | 20260505 14:32:18.335 | INFO | MemInfo: { | 20260505 14:32:18.335 | INFO | "gpu" : " 17 GB", | 20260505 14:32:18.335 | INFO | "huge" : "", | 20260505 14:32:18.335 | INFO | "huge_shared" : " 27 GB", | 20260505 14:32:18.335 | INFO | "malloc" : " 14 GB", | 20260505 14:32:18.335 | INFO | "pinned" : " 29 GB" | 20260505 14:32:18.335 | INFO | } | 20260505 14:32:18.335 | INFO | Perf report for the last 5.13692 seconds: | 20260505 14:32:18.335 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:32:18.335 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:18.335 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:18.335 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:18.335 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260505 14:32:18.335 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:18.335 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:18.335 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:32:18.335 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:18.335 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:18.335 | INFO | Uploaded 36GiB to GPU in 2.23694s (16.0934GiB/s) | 20260505 14:32:18.335 | INFO | Downloaded 34GiB to GPU in 2.09817s (16.2046GiB/s) | 20260505 14:32:23.485 | INFO | Analyzing chunk frames = [3950080,3950592) | 20260505 14:32:23.485 | INFO | MemInfo: { | 20260505 14:32:23.485 | INFO | "gpu" : " 17 GB", | 20260505 14:32:23.485 | INFO | "huge" : "", | 20260505 14:32:23.485 | INFO | "huge_shared" : " 27 GB", | 20260505 14:32:23.485 | INFO | "malloc" : " 14 GB", | 20260505 14:32:23.485 | INFO | "pinned" : " 29 GB" | 20260505 14:32:23.485 | INFO | } | 20260505 14:32:23.485 | INFO | Perf report for the last 5.15017 seconds: | 20260505 14:32:23.485 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:32:23.485 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:23.485 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:23.485 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:23.485 | INFO | ANALYSIS : 62.9%, 3.0, 97.9ms, 0 | 20260505 14:32:23.485 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:23.485 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:23.485 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:32:23.485 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:23.485 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:23.485 | INFO | Uploaded 36GiB to GPU in 2.20383s (16.3352GiB/s) | 20260505 14:32:23.486 | INFO | Downloaded 34GiB to GPU in 2.13431s (15.9302GiB/s) | 20260505 14:32:28.626 | INFO | Analyzing chunk frames = [3950592,3951104) | 20260505 14:32:28.626 | INFO | MemInfo: { | 20260505 14:32:28.626 | INFO | "gpu" : " 17 GB", | 20260505 14:32:28.626 | INFO | "huge" : "", | 20260505 14:32:28.626 | INFO | "huge_shared" : " 27 GB", | 20260505 14:32:28.626 | INFO | "malloc" : " 14 GB", | 20260505 14:32:28.626 | INFO | "pinned" : " 29 GB" | 20260505 14:32:28.626 | INFO | } | 20260505 14:32:28.626 | INFO | Perf report for the last 5.14098 seconds: | 20260505 14:32:28.626 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:32:28.626 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:28.626 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:28.626 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:28.626 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260505 14:32:28.626 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:28.626 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:28.626 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 14:32:28.626 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:28.626 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:28.626 | INFO | Uploaded 36GiB to GPU in 2.13147s (16.8898GiB/s) | 20260505 14:32:28.626 | INFO | Downloaded 34GiB to GPU in 2.05411s (16.5521GiB/s) | 20260505 14:32:33.762 | INFO | Analyzing chunk frames = [3951104,3951616) | 20260505 14:32:33.762 | INFO | MemInfo: { | 20260505 14:32:33.762 | INFO | "gpu" : " 17 GB", | 20260505 14:32:33.762 | INFO | "huge" : "", | 20260505 14:32:33.762 | INFO | "huge_shared" : " 27 GB", | 20260505 14:32:33.762 | INFO | "malloc" : " 14 GB", | 20260505 14:32:33.762 | INFO | "pinned" : " 29 GB" | 20260505 14:32:33.762 | INFO | } | 20260505 14:32:33.762 | INFO | Perf report for the last 5.136 seconds: | 20260505 14:32:33.762 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:32:33.762 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:33.762 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:33.762 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:33.762 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 14:32:33.762 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:33.762 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:33.762 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:32:33.762 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:33.762 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:33.762 | INFO | Uploaded 36GiB to GPU in 2.16305s (16.6432GiB/s) | 20260505 14:32:33.762 | INFO | Downloaded 34GiB to GPU in 2.03533s (16.7049GiB/s) | 20260505 14:32:38.892 | INFO | Analyzing chunk frames = [3951616,3952128) | 20260505 14:32:38.892 | INFO | MemInfo: { | 20260505 14:32:38.892 | INFO | "gpu" : " 17 GB", | 20260505 14:32:38.892 | INFO | "huge" : "", | 20260505 14:32:38.892 | INFO | "huge_shared" : " 27 GB", | 20260505 14:32:38.892 | INFO | "malloc" : " 14 GB", | 20260505 14:32:38.892 | INFO | "pinned" : " 29 GB" | 20260505 14:32:38.892 | INFO | } | 20260505 14:32:38.892 | INFO | Perf report for the last 5.12959 seconds: | 20260505 14:32:38.892 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:32:38.892 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:38.892 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:38.892 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:38.892 | INFO | ANALYSIS : 64.0%, 3.0, 99.8ms, 0 | 20260505 14:32:38.892 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:38.892 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:38.892 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 14:32:38.892 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:38.892 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:38.892 | INFO | Uploaded 36GiB to GPU in 2.13935s (16.8276GiB/s) | 20260505 14:32:38.892 | INFO | Downloaded 34GiB to GPU in 2.18197s (15.5823GiB/s) | 20260505 14:32:44.037 | INFO | Analyzing chunk frames = [3952128,3952640) | 20260505 14:32:44.037 | INFO | MemInfo: { | 20260505 14:32:44.037 | INFO | "gpu" : " 17 GB", | 20260505 14:32:44.037 | INFO | "huge" : "", | 20260505 14:32:44.037 | INFO | "huge_shared" : " 27 GB", | 20260505 14:32:44.037 | INFO | "malloc" : " 14 GB", | 20260505 14:32:44.037 | INFO | "pinned" : " 29 GB" | 20260505 14:32:44.037 | INFO | } | 20260505 14:32:44.037 | INFO | Perf report for the last 5.14573 seconds: | 20260505 14:32:44.037 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:32:44.037 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:44.037 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:44.037 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:44.037 | INFO | ANALYSIS : 63.7%, 3.0, 99.6ms, 0 | 20260505 14:32:44.037 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:44.037 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:44.037 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:32:44.037 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:44.037 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:44.038 | INFO | Uploaded 36GiB to GPU in 2.21685s (16.2392GiB/s) | 20260505 14:32:44.038 | INFO | Downloaded 34GiB to GPU in 2.12149s (16.0264GiB/s) | 20260505 14:32:44.173 | INFO | Starting a round of Baz IO | 20260505 14:32:47.389 | INFO | Populating RTMetrics report | 20260505 14:32:47.389 | INFO | RTMetrics report populated | 20260505 14:32:47.389 | INFO | Opening tmp RTMetrics json file | 20260505 14:32:47.437 | INFO | Serializing RTMetrics to json | 20260505 14:32:47.478 | INFO | Serialization done, swapping tmp file now... | 20260505 14:32:47.480 | INFO | RTMetrics done | 20260505 14:32:47.495 | INFO | Finished a round of Baz IO: Wrote 2.251 GiB in 3.321 seconds | 20260505 14:32:47.495 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.73% / 0.00% / 77.27% | 20260505 14:32:47.495 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.545049% / 1.19209e-05% / 6.75519e-05% | 20260505 14:32:47.495 | INFO | Baz IO overall: | 20260505 14:32:47.495 | INFO | BytesWritten : 2354628965482 | 20260505 14:32:47.495 | INFO | NumEvents : 1119417060540 | 20260505 14:32:49.184 | INFO | Analyzing chunk frames = [3952640,3953152) | 20260505 14:32:49.184 | INFO | MemInfo: { | 20260505 14:32:49.184 | INFO | "gpu" : " 17 GB", | 20260505 14:32:49.184 | INFO | "huge" : "", | 20260505 14:32:49.184 | INFO | "huge_shared" : " 27 GB", | 20260505 14:32:49.184 | INFO | "malloc" : " 14 GB", | 20260505 14:32:49.184 | INFO | "pinned" : " 29 GB" | 20260505 14:32:49.184 | INFO | } | 20260505 14:32:49.184 | INFO | Perf report for the last 5.1471 seconds: | 20260505 14:32:49.184 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:32:49.184 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:49.184 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:49.185 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:49.185 | INFO | ANALYSIS : 64.0%, 3.0, 99.5ms, 0 | 20260505 14:32:49.185 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:49.185 | INFO | RT_METRICS : 46.5%, 1.0, 24.4ms, 0 | 20260505 14:32:49.185 | INFO | PRE_HQ : 60.7%, 1.8, 57.0ms, 0 | 20260505 14:32:49.185 | INFO | BAZWRITER : 63.9%, 4.0, 131.5ms, 0 | 20260505 14:32:49.185 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260505 14:32:49.185 | INFO | Uploaded 36GiB to GPU in 2.20821s (16.3028GiB/s) | 20260505 14:32:49.185 | INFO | Downloaded 38GiB to GPU in 2.26978s (16.7417GiB/s) | 20260505 14:32:54.326 | INFO | Analyzing chunk frames = [3953152,3953664) | 20260505 14:32:54.326 | INFO | MemInfo: { | 20260505 14:32:54.326 | INFO | "gpu" : " 17 GB", | 20260505 14:32:54.326 | INFO | "huge" : "", | 20260505 14:32:54.326 | INFO | "huge_shared" : " 27 GB", | 20260505 14:32:54.326 | INFO | "malloc" : " 14 GB", | 20260505 14:32:54.326 | INFO | "pinned" : " 29 GB" | 20260505 14:32:54.326 | INFO | } | 20260505 14:32:54.326 | INFO | Perf report for the last 5.14202 seconds: | 20260505 14:32:54.326 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:32:54.326 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:54.327 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:54.327 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:54.327 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 14:32:54.327 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:54.327 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:54.327 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 14:32:54.327 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:54.327 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:54.327 | INFO | Uploaded 36GiB to GPU in 2.12877s (16.9112GiB/s) | 20260505 14:32:54.327 | INFO | Downloaded 34GiB to GPU in 2.0312s (16.7389GiB/s) | 20260505 14:32:59.454 | INFO | Analyzing chunk frames = [3953664,3954176) | 20260505 14:32:59.454 | INFO | MemInfo: { | 20260505 14:32:59.454 | INFO | "gpu" : " 17 GB", | 20260505 14:32:59.454 | INFO | "huge" : "", | 20260505 14:32:59.454 | INFO | "huge_shared" : " 27 GB", | 20260505 14:32:59.454 | INFO | "malloc" : " 14 GB", | 20260505 14:32:59.454 | INFO | "pinned" : " 29 GB" | 20260505 14:32:59.454 | INFO | } | 20260505 14:32:59.454 | INFO | Perf report for the last 5.1277 seconds: | 20260505 14:32:59.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:32:59.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:59.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:59.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:59.454 | INFO | ANALYSIS : 63.0%, 3.0, 97.8ms, 0 | 20260505 14:32:59.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:59.454 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:59.454 | INFO | PRE_HQ : 28.3%, 1.0, 14.6ms, 0 | 20260505 14:32:59.454 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:59.454 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:32:59.454 | INFO | Uploaded 36GiB to GPU in 2.18479s (16.4776GiB/s) | 20260505 14:32:59.455 | INFO | Downloaded 34GiB to GPU in 2.0258s (16.7835GiB/s) | 20260505 14:33:04.588 | INFO | Analyzing chunk frames = [3954176,3954688) | 20260505 14:33:04.588 | INFO | MemInfo: { | 20260505 14:33:04.588 | INFO | "gpu" : " 17 GB", | 20260505 14:33:04.588 | INFO | "huge" : "", | 20260505 14:33:04.588 | INFO | "huge_shared" : " 27 GB", | 20260505 14:33:04.588 | INFO | "malloc" : " 14 GB", | 20260505 14:33:04.588 | INFO | "pinned" : " 29 GB" | 20260505 14:33:04.588 | INFO | } | 20260505 14:33:04.589 | INFO | Perf report for the last 5.1345 seconds: | 20260505 14:33:04.589 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:33:04.589 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:04.589 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:04.589 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:04.589 | INFO | ANALYSIS : 63.1%, 3.0, 97.9ms, 0 | 20260505 14:33:04.589 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:04.589 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:04.589 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:33:04.589 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:04.589 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:04.589 | INFO | Uploaded 36GiB to GPU in 2.20351s (16.3376GiB/s) | 20260505 14:33:04.589 | INFO | Downloaded 34GiB to GPU in 2.13467s (15.9276GiB/s) | 20260505 14:33:09.740 | INFO | Analyzing chunk frames = [3954688,3955200) | 20260505 14:33:09.740 | INFO | MemInfo: { | 20260505 14:33:09.740 | INFO | "gpu" : " 17 GB", | 20260505 14:33:09.740 | INFO | "huge" : "", | 20260505 14:33:09.740 | INFO | "huge_shared" : " 27 GB", | 20260505 14:33:09.740 | INFO | "malloc" : " 14 GB", | 20260505 14:33:09.740 | INFO | "pinned" : " 29 GB" | 20260505 14:33:09.740 | INFO | } | 20260505 14:33:09.741 | INFO | Perf report for the last 5.15183 seconds: | 20260505 14:33:09.741 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:33:09.741 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:09.741 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:09.741 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:09.741 | INFO | ANALYSIS : 62.6%, 3.0, 97.9ms, 0 | 20260505 14:33:09.741 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:09.741 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:09.741 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 14:33:09.741 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:09.741 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:09.741 | INFO | Uploaded 36GiB to GPU in 2.15626s (16.6956GiB/s) | 20260505 14:33:09.741 | INFO | Downloaded 34GiB to GPU in 2.12567s (15.9949GiB/s) | 20260505 14:33:14.882 | INFO | Analyzing chunk frames = [3955200,3955712) | 20260505 14:33:14.882 | INFO | MemInfo: { | 20260505 14:33:14.882 | INFO | "gpu" : " 17 GB", | 20260505 14:33:14.882 | INFO | "huge" : "", | 20260505 14:33:14.882 | INFO | "huge_shared" : " 27 GB", | 20260505 14:33:14.882 | INFO | "malloc" : " 14 GB", | 20260505 14:33:14.882 | INFO | "pinned" : " 29 GB" | 20260505 14:33:14.882 | INFO | } | 20260505 14:33:14.882 | INFO | Perf report for the last 5.14154 seconds: | 20260505 14:33:14.882 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:33:14.882 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:14.882 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:14.882 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:14.882 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260505 14:33:14.882 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:14.882 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:14.882 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:33:14.882 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:14.882 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:14.882 | INFO | Uploaded 36GiB to GPU in 2.13464s (16.8647GiB/s) | 20260505 14:33:14.882 | INFO | Downloaded 34GiB to GPU in 2.03328s (16.7218GiB/s) | 20260505 14:33:20.013 | INFO | Analyzing chunk frames = [3955712,3956224) | 20260505 14:33:20.013 | INFO | MemInfo: { | 20260505 14:33:20.013 | INFO | "gpu" : " 17 GB", | 20260505 14:33:20.013 | INFO | "huge" : "", | 20260505 14:33:20.013 | INFO | "huge_shared" : " 27 GB", | 20260505 14:33:20.013 | INFO | "malloc" : " 14 GB", | 20260505 14:33:20.013 | INFO | "pinned" : " 29 GB" | 20260505 14:33:20.013 | INFO | } | 20260505 14:33:20.014 | INFO | Perf report for the last 5.13142 seconds: | 20260505 14:33:20.014 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:33:20.014 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:20.014 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:20.014 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:20.014 | INFO | ANALYSIS : 63.5%, 3.0, 99.0ms, 0 | 20260505 14:33:20.014 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:20.014 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:20.014 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 14:33:20.014 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:20.014 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:20.014 | INFO | Uploaded 36GiB to GPU in 2.17385s (16.5605GiB/s) | 20260505 14:33:20.014 | INFO | Downloaded 34GiB to GPU in 2.16731s (15.6877GiB/s) | 20260505 14:33:25.149 | INFO | Analyzing chunk frames = [3956224,3956736) | 20260505 14:33:25.149 | INFO | MemInfo: { | 20260505 14:33:25.149 | INFO | "gpu" : " 17 GB", | 20260505 14:33:25.149 | INFO | "huge" : "", | 20260505 14:33:25.149 | INFO | "huge_shared" : " 27 GB", | 20260505 14:33:25.149 | INFO | "malloc" : " 14 GB", | 20260505 14:33:25.149 | INFO | "pinned" : " 29 GB" | 20260505 14:33:25.149 | INFO | } | 20260505 14:33:25.149 | INFO | Perf report for the last 5.13573 seconds: | 20260505 14:33:25.149 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:33:25.149 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:25.149 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:25.149 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:25.149 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 14:33:25.149 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:25.149 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:25.149 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:33:25.149 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:25.149 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:25.150 | INFO | Uploaded 36GiB to GPU in 2.2119s (16.2756GiB/s) | 20260505 14:33:25.150 | INFO | Downloaded 34GiB to GPU in 2.10308s (16.1668GiB/s) | 20260505 14:33:25.150 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 14:33:25.150 | INFO | Total mem usage: 15.3 GiB | 20260505 14:33:25.150 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 14:33:25.150 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.82%) | 20260505 14:33:25.150 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 14:33:25.150 | INFO | High Water Breakdown: | 20260505 14:33:25.150 | INFO | 61% : 8.68 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 14:33:25.150 | INFO | 38.8% : 5.52 GiB ( 83.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 14:33:25.150 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 14:33:25.150 | INFO | ------------------------------------------ | 20260505 14:33:25.150 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:33:25.150 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 14:33:25.150 | INFO | Total mem usage: 26.5 GiB | 20260505 14:33:25.150 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 14:33:25.150 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 14:33:25.150 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 14:33:25.150 | INFO | High Water Breakdown: | 20260505 14:33:25.150 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:33:25.150 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:33:25.150 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:33:25.150 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:33:25.150 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:33:25.150 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:33:25.150 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 14:33:25.150 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 14:33:25.150 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:33:25.150 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 14:33:25.150 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 14:33:25.150 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 14:33:25.150 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 14:33:25.150 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 14:33:25.150 | INFO | ------------------------------------------ | 20260505 14:33:25.150 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:33:25.150 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 14:33:25.150 | INFO | Total mem usage: 25.6 GiB | 20260505 14:33:25.150 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 14:33:25.150 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 14:33:25.150 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 14:33:25.150 | INFO | High Water Breakdown: | 20260505 14:33:25.150 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 14:33:25.150 | INFO | ------------------------------------------ | 20260505 14:33:25.150 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:33:25.150 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 14:33:25.150 | INFO | Total mem usage: 15.7 GiB | 20260505 14:33:25.150 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 14:33:25.150 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 14:33:25.150 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 14:33:25.150 | INFO | High Water Breakdown: | 20260505 14:33:25.150 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 14:33:25.150 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 14:33:25.150 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 14:33:25.150 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 14:33:25.150 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 14:33:25.150 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 14:33:25.150 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 14:33:25.150 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 14:33:25.150 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 14:33:25.150 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:33:25.150 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 14:33:25.150 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:33:25.150 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 14:33:25.150 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 14:33:25.150 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 14:33:25.150 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 14:33:25.150 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:33:25.150 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 14:33:25.150 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:33:25.150 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:33:25.150 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:33:25.150 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 14:33:25.150 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 14:33:25.150 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:33:25.151 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 14:33:25.151 | INFO | ------------------------------------------ | 20260505 14:33:25.151 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:33:25.151 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 14:33:25.151 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:33:25.151 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:33:25.151 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.63e+01, 1.37e+01 / 2.53e+00 | 20260505 14:33:25.151 | INFO | PerfReport: 0.29%, AggregateStats -- 9.27e-02 / 1.00e-01, 1.04e-02 / 4.27e-03 | 20260505 14:33:25.151 | INFO | PerfReport: 11.77%, Baseline -- 3.75e+00 / 4.48e+00, 5.89e-01 / 2.00e-01 | 20260505 14:33:25.151 | INFO | PerfReport: 3.29%, Histogram -- 1.05e+00 / 1.49e+00, 1.98e-01 / 1.11e-01 | 20260505 14:33:25.151 | INFO | PerfReport: 16.00%, DME -- 4.08e+01 / 4.57e+01, 4.65e+00 / 2.22e+00 | 20260505 14:33:25.151 | INFO | PerfReport: 56.55%, FrameLabeling -- 1.80e+01 / 1.96e+01, 1.47e+00 / 1.63e+00 | 20260505 14:33:25.151 | INFO | PerfReport: 7.37%, PulseAccumulating -- 2.35e+00 / 2.96e+00, 4.45e-01 / 1.41e-01 | 20260505 14:33:25.151 | INFO | PerfReport: 3.50%, Metrics -- 1.11e+00 / 1.43e+00, 3.08e-01 / 1.39e-01 | 20260505 14:33:25.151 | INFO | PerfReport: 1.23%, PulseSerializing -- 3.92e-01 / 4.38e-01, 6.93e-02 / 3.28e-02 | 20260505 14:33:25.151 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 14:33:25.151 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:33:25.151 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:33:25.151 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.56e+01 / 0.00e+00 | 20260505 14:33:25.151 | INFO | PerfReport: 22.56%, Upload -- 2.27e+01 / 0.00e+00, 8.01e+00 / 0.00e+00 | 20260505 14:33:25.151 | INFO | PerfReport: 21.87%, Download -- 2.20e+01 / 0.00e+00, 7.98e+00 / 0.00e+00 | 20260505 14:33:25.151 | INFO | PerfReport: 31.65%, Compute -- 3.19e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 14:33:25.151 | INFO | PerfReport: 2.22%, LockUpload -- 2.23e+00 / 0.00e+00, 7.90e+00 / 0.00e+00 | 20260505 14:33:25.151 | INFO | PerfReport: 1.36%, LockDownload -- 1.37e+00 / 0.00e+00, 2.87e+00 / 0.00e+00 | 20260505 14:33:25.151 | INFO | PerfReport: 20.33%, LockCompute -- 2.05e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 14:33:25.151 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.51e-02 / 0.00e+00 | 20260505 14:33:25.286 | INFO | Starting a round of Baz IO | 20260505 14:33:28.511 | INFO | Populating RTMetrics report | 20260505 14:33:28.512 | INFO | RTMetrics report populated | 20260505 14:33:28.512 | INFO | Opening tmp RTMetrics json file | 20260505 14:33:28.535 | INFO | Serializing RTMetrics to json | 20260505 14:33:28.577 | INFO | Serialization done, swapping tmp file now... | 20260505 14:33:28.583 | INFO | RTMetrics done | 20260505 14:33:28.626 | INFO | Finished a round of Baz IO: Wrote 2.25 GiB in 3.339 seconds | 20260505 14:33:28.626 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.72% / 0.00% / 77.27% | 20260505 14:33:28.626 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.544043% / 0% / 5.96046e-05% | 20260505 14:33:28.626 | INFO | Baz IO overall: | 20260505 14:33:28.626 | INFO | BytesWritten : 2357044973067 | 20260505 14:33:28.626 | INFO | NumEvents : 1120555881573 | 20260505 14:33:30.292 | INFO | Analyzing chunk frames = [3956736,3957248) | 20260505 14:33:30.292 | INFO | MemInfo: { | 20260505 14:33:30.292 | INFO | "gpu" : " 17 GB", | 20260505 14:33:30.292 | INFO | "huge" : "", | 20260505 14:33:30.292 | INFO | "huge_shared" : " 27 GB", | 20260505 14:33:30.292 | INFO | "malloc" : " 14 GB", | 20260505 14:33:30.292 | INFO | "pinned" : " 29 GB" | 20260505 14:33:30.292 | INFO | } | 20260505 14:33:30.292 | INFO | Perf report for the last 5.14274 seconds: | 20260505 14:33:30.292 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:33:30.292 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:30.292 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:30.292 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:30.292 | INFO | ANALYSIS : 64.3%, 3.0, 100.1ms, 0 | 20260505 14:33:30.292 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:30.292 | INFO | RT_METRICS : 46.2%, 1.0, 24.2ms, 0 | 20260505 14:33:30.292 | INFO | PRE_HQ : 60.8%, 1.8, 57.1ms, 0 | 20260505 14:33:30.292 | INFO | BAZWRITER : 64.3%, 3.9, 130.2ms, 0 | 20260505 14:33:30.292 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260505 14:33:30.292 | INFO | Uploaded 36GiB to GPU in 2.1778s (16.5304GiB/s) | 20260505 14:33:30.292 | INFO | Downloaded 38GiB to GPU in 2.27129s (16.7306GiB/s) | 20260505 14:33:35.432 | INFO | Analyzing chunk frames = [3957248,3957760) | 20260505 14:33:35.432 | INFO | MemInfo: { | 20260505 14:33:35.432 | INFO | "gpu" : " 17 GB", | 20260505 14:33:35.432 | INFO | "huge" : "", | 20260505 14:33:35.432 | INFO | "huge_shared" : " 27 GB", | 20260505 14:33:35.432 | INFO | "malloc" : " 14 GB", | 20260505 14:33:35.432 | INFO | "pinned" : " 29 GB" | 20260505 14:33:35.432 | INFO | } | 20260505 14:33:35.432 | INFO | Perf report for the last 5.14028 seconds: | 20260505 14:33:35.432 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:33:35.432 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:35.432 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:35.432 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:35.433 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 14:33:35.433 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:35.433 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:35.433 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:33:35.433 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:35.433 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:35.433 | INFO | Uploaded 36GiB to GPU in 2.19594s (16.3939GiB/s) | 20260505 14:33:35.433 | INFO | Downloaded 34GiB to GPU in 2.04072s (16.6608GiB/s) | 20260505 14:33:40.573 | INFO | Analyzing chunk frames = [3957760,3958272) | 20260505 14:33:40.573 | INFO | MemInfo: { | 20260505 14:33:40.573 | INFO | "gpu" : " 17 GB", | 20260505 14:33:40.573 | INFO | "huge" : "", | 20260505 14:33:40.573 | INFO | "huge_shared" : " 27 GB", | 20260505 14:33:40.573 | INFO | "malloc" : " 14 GB", | 20260505 14:33:40.573 | INFO | "pinned" : " 29 GB" | 20260505 14:33:40.573 | INFO | } | 20260505 14:33:40.573 | INFO | Perf report for the last 5.14097 seconds: | 20260505 14:33:40.573 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:33:40.574 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:40.574 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:40.574 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:40.574 | INFO | ANALYSIS : 63.1%, 3.0, 98.2ms, 0 | 20260505 14:33:40.574 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:40.574 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:40.574 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:33:40.574 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:40.574 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:40.574 | INFO | Uploaded 36GiB to GPU in 2.18388s (16.4844GiB/s) | 20260505 14:33:40.574 | INFO | Downloaded 34GiB to GPU in 2.06186s (16.49GiB/s) | 20260505 14:33:45.702 | INFO | Analyzing chunk frames = [3958272,3958784) | 20260505 14:33:45.702 | INFO | MemInfo: { | 20260505 14:33:45.702 | INFO | "gpu" : " 17 GB", | 20260505 14:33:45.702 | INFO | "huge" : "", | 20260505 14:33:45.702 | INFO | "huge_shared" : " 27 GB", | 20260505 14:33:45.702 | INFO | "malloc" : " 14 GB", | 20260505 14:33:45.702 | INFO | "pinned" : " 29 GB" | 20260505 14:33:45.702 | INFO | } | 20260505 14:33:45.702 | INFO | Perf report for the last 5.1289 seconds: | 20260505 14:33:45.702 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:33:45.702 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:45.702 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:45.702 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:45.702 | INFO | ANALYSIS : 63.0%, 3.0, 97.6ms, 0 | 20260505 14:33:45.702 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:45.702 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:45.702 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:33:45.702 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:45.702 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:45.703 | INFO | Uploaded 36GiB to GPU in 2.18311s (16.4902GiB/s) | 20260505 14:33:45.703 | INFO | Downloaded 34GiB to GPU in 2.13135s (15.9523GiB/s) | 20260505 14:33:50.841 | INFO | Analyzing chunk frames = [3958784,3959296) | 20260505 14:33:50.841 | INFO | MemInfo: { | 20260505 14:33:50.841 | INFO | "gpu" : " 17 GB", | 20260505 14:33:50.841 | INFO | "huge" : "", | 20260505 14:33:50.841 | INFO | "huge_shared" : " 27 GB", | 20260505 14:33:50.841 | INFO | "malloc" : " 14 GB", | 20260505 14:33:50.842 | INFO | "pinned" : " 29 GB" | 20260505 14:33:50.842 | INFO | } | 20260505 14:33:50.842 | INFO | Perf report for the last 5.1395 seconds: | 20260505 14:33:50.842 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:33:50.842 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:50.842 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:50.842 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:50.842 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 14:33:50.842 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:50.842 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:50.842 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:33:50.842 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:50.842 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:50.842 | INFO | Uploaded 36GiB to GPU in 2.19767s (16.381GiB/s) | 20260505 14:33:50.842 | INFO | Downloaded 34GiB to GPU in 2.13401s (15.9324GiB/s) | 20260505 14:33:55.993 | INFO | Analyzing chunk frames = [3959296,3959808) | 20260505 14:33:55.993 | INFO | MemInfo: { | 20260505 14:33:55.993 | INFO | "gpu" : " 17 GB", | 20260505 14:33:55.993 | INFO | "huge" : "", | 20260505 14:33:55.993 | INFO | "huge_shared" : " 27 GB", | 20260505 14:33:55.993 | INFO | "malloc" : " 14 GB", | 20260505 14:33:55.993 | INFO | "pinned" : " 29 GB" | 20260505 14:33:55.993 | INFO | } | 20260505 14:33:55.993 | INFO | Perf report for the last 5.15151 seconds: | 20260505 14:33:55.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:33:55.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:55.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:55.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:55.993 | INFO | ANALYSIS : 62.9%, 3.0, 98.5ms, 0 | 20260505 14:33:55.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:55.993 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:55.993 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 14:33:55.993 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:55.993 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:33:55.994 | INFO | Uploaded 36GiB to GPU in 2.1715s (16.5784GiB/s) | 20260505 14:33:55.994 | INFO | Downloaded 34GiB to GPU in 2.10939s (16.1184GiB/s) | 20260505 14:34:01.141 | INFO | Analyzing chunk frames = [3959808,3960320) | 20260505 14:34:01.141 | INFO | MemInfo: { | 20260505 14:34:01.141 | INFO | "gpu" : " 17 GB", | 20260505 14:34:01.141 | INFO | "huge" : "", | 20260505 14:34:01.141 | INFO | "huge_shared" : " 27 GB", | 20260505 14:34:01.141 | INFO | "malloc" : " 14 GB", | 20260505 14:34:01.141 | INFO | "pinned" : " 29 GB" | 20260505 14:34:01.141 | INFO | } | 20260505 14:34:01.141 | INFO | Perf report for the last 5.14812 seconds: | 20260505 14:34:01.142 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:34:01.142 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:01.142 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:01.142 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:01.142 | INFO | ANALYSIS : 63.5%, 3.0, 99.4ms, 0 | 20260505 14:34:01.142 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:01.142 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:01.142 | INFO | PRE_HQ : 29.3%, 1.0, 15.3ms, 0 | 20260505 14:34:01.142 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:01.142 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:01.142 | INFO | Uploaded 36GiB to GPU in 2.13187s (16.8866GiB/s) | 20260505 14:34:01.142 | INFO | Downloaded 34GiB to GPU in 2.15085s (15.8077GiB/s) | 20260505 14:34:06.272 | INFO | Analyzing chunk frames = [3960320,3960832) | 20260505 14:34:06.272 | INFO | MemInfo: { | 20260505 14:34:06.272 | INFO | "gpu" : " 17 GB", | 20260505 14:34:06.272 | INFO | "huge" : "", | 20260505 14:34:06.272 | INFO | "huge_shared" : " 27 GB", | 20260505 14:34:06.272 | INFO | "malloc" : " 14 GB", | 20260505 14:34:06.272 | INFO | "pinned" : " 29 GB" | 20260505 14:34:06.272 | INFO | } | 20260505 14:34:06.272 | INFO | Perf report for the last 5.13082 seconds: | 20260505 14:34:06.272 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:34:06.272 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:06.272 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:06.272 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:06.272 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 14:34:06.272 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:06.273 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:06.273 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 14:34:06.273 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:06.273 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:06.273 | INFO | Uploaded 36GiB to GPU in 2.1652s (16.6266GiB/s) | 20260505 14:34:06.273 | INFO | Downloaded 34GiB to GPU in 2.01787s (16.8495GiB/s) | 20260505 14:34:06.409 | INFO | Starting a round of Baz IO | 20260505 14:34:09.628 | INFO | Populating RTMetrics report | 20260505 14:34:09.628 | INFO | RTMetrics report populated | 20260505 14:34:09.628 | INFO | Opening tmp RTMetrics json file | 20260505 14:34:09.640 | INFO | Serializing RTMetrics to json | 20260505 14:34:09.642 | INFO | Serialization done, swapping tmp file now... | 20260505 14:34:09.642 | INFO | RTMetrics done | 20260505 14:34:09.729 | INFO | Finished a round of Baz IO: Wrote 2.25 GiB in 3.32 seconds | 20260505 14:34:09.729 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.72% / 0.00% / 77.28% | 20260505 14:34:09.729 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.543114% / 0% / 7.54992e-05% | 20260505 14:34:09.729 | INFO | Baz IO overall: | 20260505 14:34:09.729 | INFO | BytesWritten : 2359460537896 | 20260505 14:34:09.729 | INFO | NumEvents : 1121694334462 | 20260505 14:34:11.404 | INFO | Analyzing chunk frames = [3960832,3961344) | 20260505 14:34:11.404 | INFO | MemInfo: { | 20260505 14:34:11.404 | INFO | "gpu" : " 17 GB", | 20260505 14:34:11.404 | INFO | "huge" : "", | 20260505 14:34:11.404 | INFO | "huge_shared" : " 27 GB", | 20260505 14:34:11.404 | INFO | "malloc" : " 14 GB", | 20260505 14:34:11.404 | INFO | "pinned" : " 29 GB" | 20260505 14:34:11.404 | INFO | } | 20260505 14:34:11.404 | INFO | Perf report for the last 5.13159 seconds: | 20260505 14:34:11.404 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:34:11.404 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:11.404 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:11.404 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:11.404 | INFO | ANALYSIS : 64.3%, 3.0, 99.8ms, 0 | 20260505 14:34:11.404 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:11.404 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260505 14:34:11.404 | INFO | PRE_HQ : 60.9%, 1.8, 57.3ms, 0 | 20260505 14:34:11.404 | INFO | BAZWRITER : 64.0%, 4.0, 130.0ms, 0 | 20260505 14:34:11.404 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 14:34:11.404 | INFO | Uploaded 36GiB to GPU in 2.20477s (16.3283GiB/s) | 20260505 14:34:11.404 | INFO | Downloaded 38GiB to GPU in 2.26751s (16.7585GiB/s) | 20260505 14:34:16.544 | INFO | Analyzing chunk frames = [3961344,3961856) | 20260505 14:34:16.544 | INFO | MemInfo: { | 20260505 14:34:16.544 | INFO | "gpu" : " 17 GB", | 20260505 14:34:16.544 | INFO | "huge" : "", | 20260505 14:34:16.544 | INFO | "huge_shared" : " 27 GB", | 20260505 14:34:16.544 | INFO | "malloc" : " 14 GB", | 20260505 14:34:16.544 | INFO | "pinned" : " 29 GB" | 20260505 14:34:16.544 | INFO | } | 20260505 14:34:16.544 | INFO | Perf report for the last 5.14009 seconds: | 20260505 14:34:16.544 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:34:16.544 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:16.544 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:16.544 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:16.544 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 14:34:16.544 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:16.544 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:16.544 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:34:16.544 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:16.544 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:16.544 | INFO | Uploaded 36GiB to GPU in 2.18544s (16.4726GiB/s) | 20260505 14:34:16.544 | INFO | Downloaded 34GiB to GPU in 2.02791s (16.766GiB/s) | 20260505 14:34:21.689 | INFO | Analyzing chunk frames = [3961856,3962368) | 20260505 14:34:21.689 | INFO | MemInfo: { | 20260505 14:34:21.689 | INFO | "gpu" : " 17 GB", | 20260505 14:34:21.689 | INFO | "huge" : "", | 20260505 14:34:21.689 | INFO | "huge_shared" : " 27 GB", | 20260505 14:34:21.689 | INFO | "malloc" : " 14 GB", | 20260505 14:34:21.689 | INFO | "pinned" : " 29 GB" | 20260505 14:34:21.689 | INFO | } | 20260505 14:34:21.689 | INFO | Perf report for the last 5.14504 seconds: | 20260505 14:34:21.689 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:34:21.689 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:21.689 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:21.689 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:21.689 | INFO | ANALYSIS : 63.4%, 3.0, 98.8ms, 0 | 20260505 14:34:21.689 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:21.689 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:21.689 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:34:21.689 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:21.689 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:21.689 | INFO | Uploaded 36GiB to GPU in 2.241s (16.0642GiB/s) | 20260505 14:34:21.689 | INFO | Downloaded 34GiB to GPU in 2.09894s (16.1987GiB/s) | 20260505 14:34:26.830 | INFO | Analyzing chunk frames = [3962368,3962880) | 20260505 14:34:26.830 | INFO | MemInfo: { | 20260505 14:34:26.830 | INFO | "gpu" : " 17 GB", | 20260505 14:34:26.830 | INFO | "huge" : "", | 20260505 14:34:26.830 | INFO | "huge_shared" : " 27 GB", | 20260505 14:34:26.830 | INFO | "malloc" : " 14 GB", | 20260505 14:34:26.830 | INFO | "pinned" : " 29 GB" | 20260505 14:34:26.830 | INFO | } | 20260505 14:34:26.830 | INFO | Perf report for the last 5.14118 seconds: | 20260505 14:34:26.830 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:34:26.830 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:26.830 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:26.830 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:26.830 | INFO | ANALYSIS : 62.7%, 3.0, 97.4ms, 0 | 20260505 14:34:26.830 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:26.830 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:26.830 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:34:26.830 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:26.830 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:26.830 | INFO | Uploaded 36GiB to GPU in 2.18397s (16.4837GiB/s) | 20260505 14:34:26.831 | INFO | Downloaded 34GiB to GPU in 2.1285s (15.9737GiB/s) | 20260505 14:34:31.960 | INFO | Analyzing chunk frames = [3962880,3963392) | 20260505 14:34:31.960 | INFO | MemInfo: { | 20260505 14:34:31.960 | INFO | "gpu" : " 17 GB", | 20260505 14:34:31.960 | INFO | "huge" : "", | 20260505 14:34:31.960 | INFO | "huge_shared" : " 27 GB", | 20260505 14:34:31.960 | INFO | "malloc" : " 14 GB", | 20260505 14:34:31.960 | INFO | "pinned" : " 29 GB" | 20260505 14:34:31.960 | INFO | } | 20260505 14:34:31.960 | INFO | Perf report for the last 5.13 seconds: | 20260505 14:34:31.960 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:34:31.960 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:31.960 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:31.960 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:31.960 | INFO | ANALYSIS : 62.8%, 3.0, 97.8ms, 0 | 20260505 14:34:31.960 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:31.960 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:31.960 | INFO | PRE_HQ : 28.5%, 1.0, 14.7ms, 0 | 20260505 14:34:31.960 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:31.960 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:31.960 | INFO | Uploaded 36GiB to GPU in 2.14154s (16.8103GiB/s) | 20260505 14:34:31.961 | INFO | Downloaded 34GiB to GPU in 2.12741s (15.9819GiB/s) | 20260505 14:34:37.096 | INFO | Analyzing chunk frames = [3963392,3963904) | 20260505 14:34:37.096 | INFO | MemInfo: { | 20260505 14:34:37.096 | INFO | "gpu" : " 17 GB", | 20260505 14:34:37.097 | INFO | "huge" : "", | 20260505 14:34:37.097 | INFO | "huge_shared" : " 27 GB", | 20260505 14:34:37.097 | INFO | "malloc" : " 14 GB", | 20260505 14:34:37.097 | INFO | "pinned" : " 29 GB" | 20260505 14:34:37.097 | INFO | } | 20260505 14:34:37.097 | INFO | Perf report for the last 5.13655 seconds: | 20260505 14:34:37.097 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:34:37.097 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:37.097 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:37.097 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:37.097 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 14:34:37.097 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:37.097 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:37.097 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:34:37.097 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:37.097 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:37.097 | INFO | Uploaded 36GiB to GPU in 2.17818s (16.5276GiB/s) | 20260505 14:34:37.097 | INFO | Downloaded 34GiB to GPU in 2.10942s (16.1181GiB/s) | 20260505 14:34:42.248 | INFO | Analyzing chunk frames = [3963904,3964416) | 20260505 14:34:42.248 | INFO | MemInfo: { | 20260505 14:34:42.248 | INFO | "gpu" : " 17 GB", | 20260505 14:34:42.248 | INFO | "huge" : "", | 20260505 14:34:42.248 | INFO | "huge_shared" : " 27 GB", | 20260505 14:34:42.248 | INFO | "malloc" : " 14 GB", | 20260505 14:34:42.248 | INFO | "pinned" : " 29 GB" | 20260505 14:34:42.248 | INFO | } | 20260505 14:34:42.248 | INFO | Perf report for the last 5.15129 seconds: | 20260505 14:34:42.248 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:34:42.248 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:42.248 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:42.248 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:42.248 | INFO | ANALYSIS : 63.3%, 3.0, 99.1ms, 0 | 20260505 14:34:42.248 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:42.248 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:42.248 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 14:34:42.248 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:42.248 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:42.248 | INFO | Uploaded 36GiB to GPU in 2.18087s (16.5072GiB/s) | 20260505 14:34:42.248 | INFO | Downloaded 34GiB to GPU in 2.17346s (15.6433GiB/s) | 20260505 14:34:47.390 | INFO | Analyzing chunk frames = [3964416,3964928) | 20260505 14:34:47.390 | INFO | MemInfo: { | 20260505 14:34:47.390 | INFO | "gpu" : " 17 GB", | 20260505 14:34:47.390 | INFO | "huge" : "", | 20260505 14:34:47.390 | INFO | "huge_shared" : " 27 GB", | 20260505 14:34:47.390 | INFO | "malloc" : " 14 GB", | 20260505 14:34:47.390 | INFO | "pinned" : " 29 GB" | 20260505 14:34:47.390 | INFO | } | 20260505 14:34:47.390 | INFO | Perf report for the last 5.14213 seconds: | 20260505 14:34:47.390 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:34:47.390 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:47.390 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:47.390 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:47.390 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 14:34:47.390 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:47.390 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:47.390 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 14:34:47.390 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:47.390 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:47.390 | INFO | Uploaded 36GiB to GPU in 2.15794s (16.6826GiB/s) | 20260505 14:34:47.391 | INFO | Downloaded 34GiB to GPU in 2.02994s (16.7493GiB/s) | 20260505 14:34:47.391 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 14:34:47.391 | INFO | Total mem usage: 15.3 GiB | 20260505 14:34:47.391 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 14:34:47.391 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.82%) | 20260505 14:34:47.391 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 14:34:47.391 | INFO | High Water Breakdown: | 20260505 14:34:47.391 | INFO | 61% : 8.68 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 14:34:47.391 | INFO | 38.8% : 5.52 GiB ( 83.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 14:34:47.391 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 14:34:47.391 | INFO | ------------------------------------------ | 20260505 14:34:47.391 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:34:47.391 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 14:34:47.391 | INFO | Total mem usage: 26.5 GiB | 20260505 14:34:47.391 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 14:34:47.391 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 14:34:47.391 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 14:34:47.391 | INFO | High Water Breakdown: | 20260505 14:34:47.391 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:34:47.391 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:34:47.391 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:34:47.391 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:34:47.391 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:34:47.391 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:34:47.391 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 14:34:47.391 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 14:34:47.391 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:34:47.391 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 14:34:47.391 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 14:34:47.391 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 14:34:47.391 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 14:34:47.391 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 14:34:47.391 | INFO | ------------------------------------------ | 20260505 14:34:47.391 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:34:47.391 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 14:34:47.391 | INFO | Total mem usage: 25.6 GiB | 20260505 14:34:47.391 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 14:34:47.392 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 14:34:47.392 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 14:34:47.392 | INFO | High Water Breakdown: | 20260505 14:34:47.392 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 14:34:47.392 | INFO | ------------------------------------------ | 20260505 14:34:47.392 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:34:47.392 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 14:34:47.392 | INFO | Total mem usage: 15.7 GiB | 20260505 14:34:47.392 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 14:34:47.392 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 14:34:47.392 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 14:34:47.392 | INFO | High Water Breakdown: | 20260505 14:34:47.392 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 14:34:47.392 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 14:34:47.392 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 14:34:47.392 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 14:34:47.392 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 14:34:47.392 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 14:34:47.392 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 14:34:47.392 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 14:34:47.392 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 14:34:47.392 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:34:47.392 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 14:34:47.392 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:34:47.392 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 14:34:47.392 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 14:34:47.392 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 14:34:47.392 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 14:34:47.392 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:34:47.392 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 14:34:47.392 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:34:47.392 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:34:47.392 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:34:47.392 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 14:34:47.392 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 14:34:47.392 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:34:47.392 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 14:34:47.392 | INFO | ------------------------------------------ | 20260505 14:34:47.392 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:34:47.392 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 14:34:47.392 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:34:47.392 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:34:47.392 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.60e+01, 1.36e+01 / 1.87e+00 | 20260505 14:34:47.392 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 9.94e-02, 1.03e-02 / 4.65e-03 | 20260505 14:34:47.392 | INFO | PerfReport: 11.75%, Baseline -- 3.74e+00 / 4.47e+00, 5.80e-01 / 2.21e-01 | 20260505 14:34:47.392 | INFO | PerfReport: 3.30%, Histogram -- 1.05e+00 / 1.49e+00, 1.99e-01 / 1.05e-01 | 20260505 14:34:47.392 | INFO | PerfReport: 15.95%, DME -- 4.06e+01 / 4.53e+01, 4.39e+00 / 1.77e+00 | 20260505 14:34:47.392 | INFO | PerfReport: 56.57%, FrameLabeling -- 1.80e+01 / 1.99e+01, 1.48e+00 / 1.91e+00 | 20260505 14:34:47.392 | INFO | PerfReport: 7.39%, PulseAccumulating -- 2.35e+00 / 2.91e+00, 4.48e-01 / 2.23e-01 | 20260505 14:34:47.392 | INFO | PerfReport: 3.51%, Metrics -- 1.12e+00 / 1.42e+00, 3.11e-01 / 1.46e-01 | 20260505 14:34:47.392 | INFO | PerfReport: 1.23%, PulseSerializing -- 3.92e-01 / 4.39e-01, 6.98e-02 / 3.48e-02 | 20260505 14:34:47.392 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 14:34:47.392 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:34:47.392 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:34:47.392 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.56e+01 / 0.00e+00 | 20260505 14:34:47.392 | INFO | PerfReport: 22.54%, Upload -- 2.27e+01 / 0.00e+00, 8.04e+00 / 0.00e+00 | 20260505 14:34:47.392 | INFO | PerfReport: 21.90%, Download -- 2.21e+01 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260505 14:34:47.392 | INFO | PerfReport: 31.60%, Compute -- 3.18e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 14:34:47.392 | INFO | PerfReport: 2.29%, LockUpload -- 2.31e+00 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260505 14:34:47.392 | INFO | PerfReport: 1.41%, LockDownload -- 1.42e+00 / 0.00e+00, 3.01e+00 / 0.00e+00 | 20260505 14:34:47.392 | INFO | PerfReport: 20.24%, LockCompute -- 2.04e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260505 14:34:47.392 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.66e-02 / 0.00e+00 | 20260505 14:34:47.520 | INFO | Starting a round of Baz IO | 20260505 14:34:50.737 | INFO | Populating RTMetrics report | 20260505 14:34:50.738 | INFO | RTMetrics report populated | 20260505 14:34:50.738 | INFO | Opening tmp RTMetrics json file | 20260505 14:34:50.754 | INFO | Serializing RTMetrics to json | 20260505 14:34:50.774 | INFO | Serialization done, swapping tmp file now... | 20260505 14:34:50.793 | INFO | RTMetrics done | 20260505 14:34:50.845 | INFO | Finished a round of Baz IO: Wrote 2.251 GiB in 3.325 seconds | 20260505 14:34:50.845 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.72% / 0.00% / 77.28% | 20260505 14:34:50.845 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.543368% / 7.94729e-06% / 6.35783e-05% | 20260505 14:34:50.845 | INFO | Baz IO overall: | 20260505 14:34:50.845 | INFO | BytesWritten : 2361877911613 | 20260505 14:34:50.845 | INFO | NumEvents : 1122834159915 | 20260505 14:34:52.520 | INFO | Analyzing chunk frames = [3964928,3965440) | 20260505 14:34:52.520 | INFO | MemInfo: { | 20260505 14:34:52.520 | INFO | "gpu" : " 17 GB", | 20260505 14:34:52.520 | INFO | "huge" : "", | 20260505 14:34:52.520 | INFO | "huge_shared" : " 27 GB", | 20260505 14:34:52.520 | INFO | "malloc" : " 14 GB", | 20260505 14:34:52.520 | INFO | "pinned" : " 29 GB" | 20260505 14:34:52.520 | INFO | } | 20260505 14:34:52.520 | INFO | Perf report for the last 5.12994 seconds: | 20260505 14:34:52.520 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:34:52.520 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:52.520 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:52.520 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:52.520 | INFO | ANALYSIS : 64.1%, 2.9, 99.0ms, 0 | 20260505 14:34:52.520 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:52.520 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 14:34:52.520 | INFO | PRE_HQ : 61.2%, 1.8, 57.1ms, 0 | 20260505 14:34:52.520 | INFO | BAZWRITER : 64.1%, 4.1, 133.7ms, 0 | 20260505 14:34:52.520 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 14:34:52.520 | INFO | Uploaded 36GiB to GPU in 2.17854s (16.5248GiB/s) | 20260505 14:34:52.520 | INFO | Downloaded 38GiB to GPU in 2.26819s (16.7534GiB/s) | 20260505 14:34:57.652 | INFO | Analyzing chunk frames = [3965440,3965952) | 20260505 14:34:57.652 | INFO | MemInfo: { | 20260505 14:34:57.652 | INFO | "gpu" : " 17 GB", | 20260505 14:34:57.652 | INFO | "huge" : "", | 20260505 14:34:57.652 | INFO | "huge_shared" : " 27 GB", | 20260505 14:34:57.652 | INFO | "malloc" : " 14 GB", | 20260505 14:34:57.652 | INFO | "pinned" : " 29 GB" | 20260505 14:34:57.652 | INFO | } | 20260505 14:34:57.652 | INFO | Perf report for the last 5.13207 seconds: | 20260505 14:34:57.652 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:34:57.652 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:57.652 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:57.652 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:57.652 | INFO | ANALYSIS : 63.6%, 3.0, 99.1ms, 0 | 20260505 14:34:57.652 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:57.652 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:57.652 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 14:34:57.652 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:57.652 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:34:57.653 | INFO | Uploaded 36GiB to GPU in 2.18249s (16.4949GiB/s) | 20260505 14:34:57.653 | INFO | Downloaded 34GiB to GPU in 2.03318s (16.7226GiB/s) | 20260505 14:35:02.798 | INFO | Analyzing chunk frames = [3965952,3966464) | 20260505 14:35:02.798 | INFO | MemInfo: { | 20260505 14:35:02.798 | INFO | "gpu" : " 17 GB", | 20260505 14:35:02.798 | INFO | "huge" : "", | 20260505 14:35:02.798 | INFO | "huge_shared" : " 27 GB", | 20260505 14:35:02.798 | INFO | "malloc" : " 14 GB", | 20260505 14:35:02.798 | INFO | "pinned" : " 29 GB" | 20260505 14:35:02.798 | INFO | } | 20260505 14:35:02.798 | INFO | Perf report for the last 5.14561 seconds: | 20260505 14:35:02.798 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:35:02.798 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:02.798 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:02.798 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:02.798 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 14:35:02.798 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:02.798 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:02.798 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:35:02.798 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:02.798 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:02.798 | INFO | Uploaded 36GiB to GPU in 2.21034s (16.2871GiB/s) | 20260505 14:35:02.798 | INFO | Downloaded 34GiB to GPU in 2.13503s (15.9248GiB/s) | 20260505 14:35:07.943 | INFO | Analyzing chunk frames = [3966464,3966976) | 20260505 14:35:07.943 | INFO | MemInfo: { | 20260505 14:35:07.943 | INFO | "gpu" : " 17 GB", | 20260505 14:35:07.943 | INFO | "huge" : "", | 20260505 14:35:07.943 | INFO | "huge_shared" : " 27 GB", | 20260505 14:35:07.943 | INFO | "malloc" : " 14 GB", | 20260505 14:35:07.943 | INFO | "pinned" : " 29 GB" | 20260505 14:35:07.943 | INFO | } | 20260505 14:35:07.943 | INFO | Perf report for the last 5.14536 seconds: | 20260505 14:35:07.943 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:35:07.943 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:07.943 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:07.943 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:07.943 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 14:35:07.943 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:07.943 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:07.943 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:35:07.943 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:07.943 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:07.943 | INFO | Uploaded 36GiB to GPU in 2.18724s (16.4591GiB/s) | 20260505 14:35:07.944 | INFO | Downloaded 34GiB to GPU in 2.12976s (15.9642GiB/s) | 20260505 14:35:13.093 | INFO | Analyzing chunk frames = [3966976,3967488) | 20260505 14:35:13.093 | INFO | MemInfo: { | 20260505 14:35:13.093 | INFO | "gpu" : " 17 GB", | 20260505 14:35:13.093 | INFO | "huge" : "", | 20260505 14:35:13.093 | INFO | "huge_shared" : " 27 GB", | 20260505 14:35:13.093 | INFO | "malloc" : " 14 GB", | 20260505 14:35:13.093 | INFO | "pinned" : " 29 GB" | 20260505 14:35:13.093 | INFO | } | 20260505 14:35:13.093 | INFO | Perf report for the last 5.15022 seconds: | 20260505 14:35:13.093 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:35:13.093 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:13.094 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:13.094 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:13.094 | INFO | ANALYSIS : 62.7%, 3.0, 98.1ms, 0 | 20260505 14:35:13.094 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:13.094 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:13.094 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 14:35:13.094 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:13.094 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:13.094 | INFO | Uploaded 36GiB to GPU in 2.18085s (16.5073GiB/s) | 20260505 14:35:13.094 | INFO | Downloaded 34GiB to GPU in 2.13043s (15.9592GiB/s) | 20260505 14:35:18.221 | INFO | Analyzing chunk frames = [3967488,3968000) | 20260505 14:35:18.221 | INFO | MemInfo: { | 20260505 14:35:18.221 | INFO | "gpu" : " 17 GB", | 20260505 14:35:18.221 | INFO | "huge" : "", | 20260505 14:35:18.221 | INFO | "huge_shared" : " 27 GB", | 20260505 14:35:18.221 | INFO | "malloc" : " 14 GB", | 20260505 14:35:18.222 | INFO | "pinned" : " 29 GB" | 20260505 14:35:18.222 | INFO | } | 20260505 14:35:18.222 | INFO | Perf report for the last 5.12813 seconds: | 20260505 14:35:18.222 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:35:18.222 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:18.222 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:18.222 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:18.222 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260505 14:35:18.222 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:18.222 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:18.222 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 14:35:18.222 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:18.222 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:18.222 | INFO | Uploaded 36GiB to GPU in 2.13595s (16.8543GiB/s) | 20260505 14:35:18.222 | INFO | Downloaded 34GiB to GPU in 2.02656s (16.7772GiB/s) | 20260505 14:35:23.353 | INFO | Analyzing chunk frames = [3968000,3968512) | 20260505 14:35:23.353 | INFO | MemInfo: { | 20260505 14:35:23.353 | INFO | "gpu" : " 17 GB", | 20260505 14:35:23.353 | INFO | "huge" : "", | 20260505 14:35:23.353 | INFO | "huge_shared" : " 27 GB", | 20260505 14:35:23.353 | INFO | "malloc" : " 14 GB", | 20260505 14:35:23.353 | INFO | "pinned" : " 29 GB" | 20260505 14:35:23.353 | INFO | } | 20260505 14:35:23.353 | INFO | Perf report for the last 5.13127 seconds: | 20260505 14:35:23.353 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:35:23.353 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:23.353 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:23.353 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:23.353 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 14:35:23.353 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:23.353 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:23.353 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 14:35:23.353 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:23.353 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:23.353 | INFO | Uploaded 36GiB to GPU in 2.13579s (16.8556GiB/s) | 20260505 14:35:23.353 | INFO | Downloaded 34GiB to GPU in 2.17973s (15.5983GiB/s) | 20260505 14:35:28.501 | INFO | Analyzing chunk frames = [3968512,3969024) | 20260505 14:35:28.501 | INFO | MemInfo: { | 20260505 14:35:28.501 | INFO | "gpu" : " 17 GB", | 20260505 14:35:28.501 | INFO | "huge" : "", | 20260505 14:35:28.501 | INFO | "huge_shared" : " 27 GB", | 20260505 14:35:28.501 | INFO | "malloc" : " 14 GB", | 20260505 14:35:28.501 | INFO | "pinned" : " 29 GB" | 20260505 14:35:28.501 | INFO | } | 20260505 14:35:28.501 | INFO | Perf report for the last 5.14789 seconds: | 20260505 14:35:28.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:35:28.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:28.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:28.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:28.501 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 14:35:28.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:28.501 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:28.501 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 14:35:28.501 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:28.501 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:28.501 | INFO | Uploaded 36GiB to GPU in 2.15302s (16.7207GiB/s) | 20260505 14:35:28.501 | INFO | Downloaded 34GiB to GPU in 2.1029s (16.1682GiB/s) | 20260505 14:35:28.634 | INFO | Starting a round of Baz IO | 20260505 14:35:31.855 | INFO | Populating RTMetrics report | 20260505 14:35:31.856 | INFO | RTMetrics report populated | 20260505 14:35:31.856 | INFO | Opening tmp RTMetrics json file | 20260505 14:35:31.881 | INFO | Serializing RTMetrics to json | 20260505 14:35:31.896 | INFO | Serialization done, swapping tmp file now... | 20260505 14:35:31.923 | INFO | RTMetrics done | 20260505 14:35:31.964 | INFO | Finished a round of Baz IO: Wrote 2.254 GiB in 3.329 seconds | 20260505 14:35:31.964 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.71% / 0.00% / 77.28% | 20260505 14:35:31.964 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.54361% / 1.19209e-05% / 6.35783e-05% | 20260505 14:35:31.964 | INFO | Baz IO overall: | 20260505 14:35:31.964 | INFO | BytesWritten : 2364297895763 | 20260505 14:35:31.964 | INFO | NumEvents : 1123976124351 | 20260505 14:35:33.641 | INFO | Analyzing chunk frames = [3969024,3969536) | 20260505 14:35:33.641 | INFO | MemInfo: { | 20260505 14:35:33.641 | INFO | "gpu" : " 17 GB", | 20260505 14:35:33.641 | INFO | "huge" : "", | 20260505 14:35:33.641 | INFO | "huge_shared" : " 27 GB", | 20260505 14:35:33.641 | INFO | "malloc" : " 14 GB", | 20260505 14:35:33.641 | INFO | "pinned" : " 29 GB" | 20260505 14:35:33.641 | INFO | } | 20260505 14:35:33.641 | INFO | Perf report for the last 5.14037 seconds: | 20260505 14:35:33.641 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:35:33.641 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:33.641 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:33.641 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:33.641 | INFO | ANALYSIS : 64.2%, 3.0, 99.8ms, 0 | 20260505 14:35:33.641 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:33.641 | INFO | RT_METRICS : 46.2%, 1.0, 24.2ms, 0 | 20260505 14:35:33.641 | INFO | PRE_HQ : 61.1%, 1.8, 57.4ms, 0 | 20260505 14:35:33.641 | INFO | BAZWRITER : 64.1%, 4.0, 131.6ms, 0 | 20260505 14:35:33.641 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 14:35:33.642 | INFO | Uploaded 36GiB to GPU in 2.20913s (16.296GiB/s) | 20260505 14:35:33.642 | INFO | Downloaded 38GiB to GPU in 2.26468s (16.7794GiB/s) | 20260505 14:35:38.778 | INFO | Analyzing chunk frames = [3969536,3970048) | 20260505 14:35:38.778 | INFO | MemInfo: { | 20260505 14:35:38.778 | INFO | "gpu" : " 17 GB", | 20260505 14:35:38.778 | INFO | "huge" : "", | 20260505 14:35:38.778 | INFO | "huge_shared" : " 27 GB", | 20260505 14:35:38.778 | INFO | "malloc" : " 14 GB", | 20260505 14:35:38.778 | INFO | "pinned" : " 29 GB" | 20260505 14:35:38.778 | INFO | } | 20260505 14:35:38.778 | INFO | Perf report for the last 5.13703 seconds: | 20260505 14:35:38.778 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:35:38.778 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:38.778 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:38.778 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:38.778 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 14:35:38.778 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:38.778 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:38.778 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:35:38.778 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:38.778 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:38.778 | INFO | Uploaded 36GiB to GPU in 2.19558s (16.3966GiB/s) | 20260505 14:35:38.779 | INFO | Downloaded 34GiB to GPU in 2.03742s (16.6878GiB/s) | 20260505 14:35:43.915 | INFO | Analyzing chunk frames = [3970048,3970560) | 20260505 14:35:43.915 | INFO | MemInfo: { | 20260505 14:35:43.915 | INFO | "gpu" : " 17 GB", | 20260505 14:35:43.915 | INFO | "huge" : "", | 20260505 14:35:43.915 | INFO | "huge_shared" : " 27 GB", | 20260505 14:35:43.915 | INFO | "malloc" : " 14 GB", | 20260505 14:35:43.916 | INFO | "pinned" : " 29 GB" | 20260505 14:35:43.916 | INFO | } | 20260505 14:35:43.916 | INFO | Perf report for the last 5.13736 seconds: | 20260505 14:35:43.916 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:35:43.916 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:43.916 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:43.916 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:43.916 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260505 14:35:43.916 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:43.916 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:43.916 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:35:43.916 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:43.916 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:43.916 | INFO | Uploaded 36GiB to GPU in 2.24051s (16.0678GiB/s) | 20260505 14:35:43.916 | INFO | Downloaded 34GiB to GPU in 2.12036s (16.035GiB/s) | 20260505 14:35:49.055 | INFO | Analyzing chunk frames = [3970560,3971072) | 20260505 14:35:49.055 | INFO | MemInfo: { | 20260505 14:35:49.055 | INFO | "gpu" : " 17 GB", | 20260505 14:35:49.055 | INFO | "huge" : "", | 20260505 14:35:49.055 | INFO | "huge_shared" : " 27 GB", | 20260505 14:35:49.055 | INFO | "malloc" : " 14 GB", | 20260505 14:35:49.056 | INFO | "pinned" : " 29 GB" | 20260505 14:35:49.056 | INFO | } | 20260505 14:35:49.056 | INFO | Perf report for the last 5.14 seconds: | 20260505 14:35:49.056 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:35:49.056 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:49.056 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:49.056 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:49.056 | INFO | ANALYSIS : 62.9%, 3.0, 97.8ms, 0 | 20260505 14:35:49.056 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:49.056 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:49.056 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:35:49.056 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:49.056 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:49.056 | INFO | Uploaded 36GiB to GPU in 2.19103s (16.4306GiB/s) | 20260505 14:35:49.056 | INFO | Downloaded 34GiB to GPU in 2.13311s (15.9392GiB/s) | 20260505 14:35:54.197 | INFO | Analyzing chunk frames = [3971072,3971584) | 20260505 14:35:54.197 | INFO | MemInfo: { | 20260505 14:35:54.197 | INFO | "gpu" : " 17 GB", | 20260505 14:35:54.197 | INFO | "huge" : "", | 20260505 14:35:54.197 | INFO | "huge_shared" : " 27 GB", | 20260505 14:35:54.197 | INFO | "malloc" : " 14 GB", | 20260505 14:35:54.197 | INFO | "pinned" : " 29 GB" | 20260505 14:35:54.197 | INFO | } | 20260505 14:35:54.197 | INFO | Perf report for the last 5.14181 seconds: | 20260505 14:35:54.197 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:35:54.197 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:54.197 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:54.197 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:54.197 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260505 14:35:54.198 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:54.198 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:54.198 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 14:35:54.198 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:54.198 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:54.198 | INFO | Uploaded 36GiB to GPU in 2.14339s (16.7959GiB/s) | 20260505 14:35:54.198 | INFO | Downloaded 34GiB to GPU in 2.12718s (15.9836GiB/s) | 20260505 14:35:59.335 | INFO | Analyzing chunk frames = [3971584,3972096) | 20260505 14:35:59.335 | INFO | MemInfo: { | 20260505 14:35:59.335 | INFO | "gpu" : " 17 GB", | 20260505 14:35:59.335 | INFO | "huge" : "", | 20260505 14:35:59.335 | INFO | "huge_shared" : " 27 GB", | 20260505 14:35:59.335 | INFO | "malloc" : " 14 GB", | 20260505 14:35:59.335 | INFO | "pinned" : " 29 GB" | 20260505 14:35:59.335 | INFO | } | 20260505 14:35:59.335 | INFO | Perf report for the last 5.13777 seconds: | 20260505 14:35:59.335 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:35:59.335 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:59.335 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:59.335 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:59.335 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 14:35:59.335 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:59.335 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:59.335 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 14:35:59.335 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:59.335 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:35:59.335 | INFO | Uploaded 36GiB to GPU in 2.17169s (16.577GiB/s) | 20260505 14:35:59.336 | INFO | Downloaded 34GiB to GPU in 2.0311s (16.7397GiB/s) | 20260505 14:36:04.466 | INFO | Analyzing chunk frames = [3972096,3972608) | 20260505 14:36:04.466 | INFO | MemInfo: { | 20260505 14:36:04.466 | INFO | "gpu" : " 17 GB", | 20260505 14:36:04.466 | INFO | "huge" : "", | 20260505 14:36:04.466 | INFO | "huge_shared" : " 27 GB", | 20260505 14:36:04.466 | INFO | "malloc" : " 14 GB", | 20260505 14:36:04.466 | INFO | "pinned" : " 29 GB" | 20260505 14:36:04.466 | INFO | } | 20260505 14:36:04.467 | INFO | Perf report for the last 5.13133 seconds: | 20260505 14:36:04.467 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:36:04.467 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:04.467 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:04.467 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:04.467 | INFO | ANALYSIS : 63.9%, 3.0, 99.6ms, 0 | 20260505 14:36:04.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:04.467 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:04.467 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 14:36:04.467 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:04.467 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:04.467 | INFO | Uploaded 36GiB to GPU in 2.13523s (16.86GiB/s) | 20260505 14:36:04.467 | INFO | Downloaded 34GiB to GPU in 2.19138s (15.5153GiB/s) | 20260505 14:36:09.604 | INFO | Analyzing chunk frames = [3972608,3973120) | 20260505 14:36:09.604 | INFO | MemInfo: { | 20260505 14:36:09.604 | INFO | "gpu" : " 17 GB", | 20260505 14:36:09.604 | INFO | "huge" : "", | 20260505 14:36:09.604 | INFO | "huge_shared" : " 27 GB", | 20260505 14:36:09.604 | INFO | "malloc" : " 14 GB", | 20260505 14:36:09.604 | INFO | "pinned" : " 29 GB" | 20260505 14:36:09.604 | INFO | } | 20260505 14:36:09.604 | INFO | Perf report for the last 5.13751 seconds: | 20260505 14:36:09.604 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:36:09.604 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:09.604 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:09.604 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:09.604 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 14:36:09.604 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:09.604 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:09.604 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 14:36:09.604 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:09.604 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:09.604 | INFO | Uploaded 36GiB to GPU in 2.14241s (16.8035GiB/s) | 20260505 14:36:09.604 | INFO | Downloaded 34GiB to GPU in 2.10487s (16.153GiB/s) | 20260505 14:36:09.605 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 14:36:09.605 | INFO | Total mem usage: 15.3 GiB | 20260505 14:36:09.605 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 14:36:09.605 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.33%) | 20260505 14:36:09.605 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 14:36:09.605 | INFO | High Water Breakdown: | 20260505 14:36:09.605 | INFO | 61.2% : 8.82 GiB ( 92.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 14:36:09.605 | INFO | 38.6% : 5.57 GiB ( 83.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 14:36:09.605 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 14:36:09.605 | INFO | ------------------------------------------ | 20260505 14:36:09.605 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:36:09.605 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 14:36:09.605 | INFO | Total mem usage: 26.5 GiB | 20260505 14:36:09.605 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 14:36:09.605 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 14:36:09.605 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 14:36:09.605 | INFO | High Water Breakdown: | 20260505 14:36:09.605 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:36:09.605 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:36:09.605 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:36:09.605 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:36:09.605 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:36:09.605 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:36:09.605 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 14:36:09.605 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 14:36:09.605 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:36:09.605 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 14:36:09.605 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 14:36:09.605 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 14:36:09.605 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 14:36:09.605 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 14:36:09.605 | INFO | ------------------------------------------ | 20260505 14:36:09.605 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:36:09.605 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 14:36:09.605 | INFO | Total mem usage: 25.6 GiB | 20260505 14:36:09.605 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 14:36:09.605 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 14:36:09.605 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 14:36:09.605 | INFO | High Water Breakdown: | 20260505 14:36:09.605 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 14:36:09.605 | INFO | ------------------------------------------ | 20260505 14:36:09.605 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:36:09.605 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 14:36:09.605 | INFO | Total mem usage: 15.7 GiB | 20260505 14:36:09.605 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 14:36:09.605 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 14:36:09.605 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 14:36:09.605 | INFO | High Water Breakdown: | 20260505 14:36:09.605 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 14:36:09.605 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 14:36:09.605 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 14:36:09.605 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 14:36:09.605 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 14:36:09.605 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 14:36:09.605 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 14:36:09.605 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 14:36:09.605 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 14:36:09.605 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:36:09.605 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 14:36:09.605 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:36:09.605 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 14:36:09.605 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 14:36:09.605 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 14:36:09.605 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 14:36:09.605 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:36:09.605 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 14:36:09.605 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:36:09.605 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:36:09.605 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:36:09.605 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 14:36:09.605 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 14:36:09.605 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:36:09.605 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 14:36:09.605 | INFO | ------------------------------------------ | 20260505 14:36:09.605 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:36:09.605 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 14:36:09.605 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:36:09.605 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:36:09.605 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.65e+01, 1.38e+01 / 2.20e+00 | 20260505 14:36:09.605 | INFO | PerfReport: 0.29%, AggregateStats -- 9.26e-02 / 1.01e-01, 1.04e-02 / 5.78e-03 | 20260505 14:36:09.605 | INFO | PerfReport: 11.79%, Baseline -- 3.75e+00 / 4.50e+00, 5.93e-01 / 1.54e-01 | 20260505 14:36:09.605 | INFO | PerfReport: 3.29%, Histogram -- 1.05e+00 / 1.50e+00, 1.98e-01 / 9.25e-02 | 20260505 14:36:09.605 | INFO | PerfReport: 16.03%, DME -- 4.08e+01 / 4.61e+01, 4.93e+00 / 2.30e+00 | 20260505 14:36:09.605 | INFO | PerfReport: 56.43%, FrameLabeling -- 1.80e+01 / 1.96e+01, 1.39e+00 / 1.49e+00 | 20260505 14:36:09.605 | INFO | PerfReport: 7.42%, PulseAccumulating -- 2.36e+00 / 2.94e+00, 4.52e-01 / 1.61e-01 | 20260505 14:36:09.605 | INFO | PerfReport: 3.51%, Metrics -- 1.12e+00 / 1.42e+00, 3.12e-01 / 1.47e-01 | 20260505 14:36:09.605 | INFO | PerfReport: 1.24%, PulseSerializing -- 3.93e-01 / 4.47e-01, 6.98e-02 / 3.47e-02 | 20260505 14:36:09.605 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 14:36:09.605 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:36:09.605 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:36:09.606 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.56e+01 / 0.00e+00 | 20260505 14:36:09.606 | INFO | PerfReport: 22.50%, Upload -- 2.27e+01 / 0.00e+00, 8.03e+00 / 0.00e+00 | 20260505 14:36:09.606 | INFO | PerfReport: 21.99%, Download -- 2.21e+01 / 0.00e+00, 8.09e+00 / 0.00e+00 | 20260505 14:36:09.606 | INFO | PerfReport: 31.61%, Compute -- 3.18e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 14:36:09.606 | INFO | PerfReport: 2.20%, LockUpload -- 2.22e+00 / 0.00e+00, 7.85e+00 / 0.00e+00 | 20260505 14:36:09.606 | INFO | PerfReport: 1.46%, LockDownload -- 1.47e+00 / 0.00e+00, 3.04e+00 / 0.00e+00 | 20260505 14:36:09.606 | INFO | PerfReport: 20.22%, LockCompute -- 2.04e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 14:36:09.606 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.93e-02 / 0.00e+00 | 20260505 14:36:09.745 | INFO | Starting a round of Baz IO | 20260505 14:36:12.966 | INFO | Populating RTMetrics report | 20260505 14:36:12.966 | INFO | RTMetrics report populated | 20260505 14:36:12.966 | INFO | Opening tmp RTMetrics json file | 20260505 14:36:12.977 | INFO | Serializing RTMetrics to json | 20260505 14:36:12.994 | INFO | Serialization done, swapping tmp file now... | 20260505 14:36:13.007 | INFO | RTMetrics done | 20260505 14:36:13.077 | INFO | Finished a round of Baz IO: Wrote 2.252 GiB in 3.332 seconds | 20260505 14:36:13.077 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.71% / 0.00% / 77.29% | 20260505 14:36:13.077 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.541671% / 1.58946e-05% / 5.5631e-05% | 20260505 14:36:13.077 | INFO | Baz IO overall: | 20260505 14:36:13.077 | INFO | BytesWritten : 2366715546608 | 20260505 14:36:13.077 | INFO | NumEvents : 1125116480290 | 20260505 14:36:14.750 | INFO | Analyzing chunk frames = [3973120,3973632) | 20260505 14:36:14.750 | INFO | MemInfo: { | 20260505 14:36:14.750 | INFO | "gpu" : " 17 GB", | 20260505 14:36:14.750 | INFO | "huge" : "", | 20260505 14:36:14.750 | INFO | "huge_shared" : " 27 GB", | 20260505 14:36:14.750 | INFO | "malloc" : " 14 GB", | 20260505 14:36:14.750 | INFO | "pinned" : " 29 GB" | 20260505 14:36:14.750 | INFO | } | 20260505 14:36:14.750 | INFO | Perf report for the last 5.14573 seconds: | 20260505 14:36:14.750 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:36:14.750 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:14.750 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:14.750 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:14.750 | INFO | ANALYSIS : 64.2%, 3.0, 99.9ms, 0 | 20260505 14:36:14.750 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:14.750 | INFO | RT_METRICS : 45.6%, 1.0, 24.0ms, 0 | 20260505 14:36:14.750 | INFO | PRE_HQ : 60.6%, 1.8, 57.6ms, 0 | 20260505 14:36:14.750 | INFO | BAZWRITER : 64.1%, 4.0, 133.5ms, 0 | 20260505 14:36:14.750 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 14:36:14.750 | INFO | Uploaded 36GiB to GPU in 2.2014s (16.3532GiB/s) | 20260505 14:36:14.750 | INFO | Downloaded 38GiB to GPU in 2.27905s (16.6736GiB/s) | 20260505 14:36:19.901 | INFO | Analyzing chunk frames = [3973632,3974144) | 20260505 14:36:19.901 | INFO | MemInfo: { | 20260505 14:36:19.901 | INFO | "gpu" : " 17 GB", | 20260505 14:36:19.901 | INFO | "huge" : "", | 20260505 14:36:19.902 | INFO | "huge_shared" : " 27 GB", | 20260505 14:36:19.902 | INFO | "malloc" : " 14 GB", | 20260505 14:36:19.902 | INFO | "pinned" : " 29 GB" | 20260505 14:36:19.902 | INFO | } | 20260505 14:36:19.902 | INFO | Perf report for the last 5.15192 seconds: | 20260505 14:36:19.902 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:36:19.902 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:19.902 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:19.902 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:19.902 | INFO | ANALYSIS : 63.2%, 3.0, 99.0ms, 0 | 20260505 14:36:19.902 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:19.902 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:19.902 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:36:19.902 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:19.902 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:19.902 | INFO | Uploaded 36GiB to GPU in 2.1908s (16.4324GiB/s) | 20260505 14:36:19.902 | INFO | Downloaded 34GiB to GPU in 2.03316s (16.7228GiB/s) | 20260505 14:36:25.036 | INFO | Analyzing chunk frames = [3974144,3974656) | 20260505 14:36:25.036 | INFO | MemInfo: { | 20260505 14:36:25.036 | INFO | "gpu" : " 17 GB", | 20260505 14:36:25.036 | INFO | "huge" : "", | 20260505 14:36:25.036 | INFO | "huge_shared" : " 27 GB", | 20260505 14:36:25.036 | INFO | "malloc" : " 14 GB", | 20260505 14:36:25.036 | INFO | "pinned" : " 29 GB" | 20260505 14:36:25.036 | INFO | } | 20260505 14:36:25.036 | INFO | Perf report for the last 5.13432 seconds: | 20260505 14:36:25.036 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:36:25.036 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:25.036 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:25.036 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:25.036 | INFO | ANALYSIS : 62.9%, 3.0, 97.8ms, 0 | 20260505 14:36:25.036 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:25.036 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:25.036 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:36:25.036 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:25.036 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:25.036 | INFO | Uploaded 36GiB to GPU in 2.18217s (16.4974GiB/s) | 20260505 14:36:25.036 | INFO | Downloaded 34GiB to GPU in 2.03441s (16.7125GiB/s) | 20260505 14:36:30.164 | INFO | Analyzing chunk frames = [3974656,3975168) | 20260505 14:36:30.164 | INFO | MemInfo: { | 20260505 14:36:30.164 | INFO | "gpu" : " 17 GB", | 20260505 14:36:30.164 | INFO | "huge" : "", | 20260505 14:36:30.164 | INFO | "huge_shared" : " 27 GB", | 20260505 14:36:30.164 | INFO | "malloc" : " 14 GB", | 20260505 14:36:30.165 | INFO | "pinned" : " 29 GB" | 20260505 14:36:30.165 | INFO | } | 20260505 14:36:30.165 | INFO | Perf report for the last 5.12851 seconds: | 20260505 14:36:30.165 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:36:30.165 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:30.165 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:30.165 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:30.165 | INFO | ANALYSIS : 63.0%, 3.0, 97.7ms, 0 | 20260505 14:36:30.165 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:30.165 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:30.165 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 14:36:30.165 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:30.165 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:30.165 | INFO | Uploaded 36GiB to GPU in 2.19118s (16.4295GiB/s) | 20260505 14:36:30.165 | INFO | Downloaded 34GiB to GPU in 2.13257s (15.9432GiB/s) | 20260505 14:36:35.311 | INFO | Analyzing chunk frames = [3975168,3975680) | 20260505 14:36:35.311 | INFO | MemInfo: { | 20260505 14:36:35.311 | INFO | "gpu" : " 17 GB", | 20260505 14:36:35.311 | INFO | "huge" : "", | 20260505 14:36:35.311 | INFO | "huge_shared" : " 27 GB", | 20260505 14:36:35.311 | INFO | "malloc" : " 14 GB", | 20260505 14:36:35.311 | INFO | "pinned" : " 29 GB" | 20260505 14:36:35.311 | INFO | } | 20260505 14:36:35.311 | INFO | Perf report for the last 5.14658 seconds: | 20260505 14:36:35.311 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:36:35.311 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:35.311 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:35.311 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:35.311 | INFO | ANALYSIS : 62.6%, 3.0, 97.9ms, 0 | 20260505 14:36:35.311 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:35.311 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:35.311 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 14:36:35.311 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:35.311 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:35.311 | INFO | Uploaded 36GiB to GPU in 2.15895s (16.6747GiB/s) | 20260505 14:36:35.311 | INFO | Downloaded 34GiB to GPU in 2.12405s (16.0072GiB/s) | 20260505 14:36:40.451 | INFO | Analyzing chunk frames = [3975680,3976192) | 20260505 14:36:40.451 | INFO | MemInfo: { | 20260505 14:36:40.451 | INFO | "gpu" : " 17 GB", | 20260505 14:36:40.451 | INFO | "huge" : "", | 20260505 14:36:40.451 | INFO | "huge_shared" : " 27 GB", | 20260505 14:36:40.451 | INFO | "malloc" : " 14 GB", | 20260505 14:36:40.451 | INFO | "pinned" : " 29 GB" | 20260505 14:36:40.451 | INFO | } | 20260505 14:36:40.451 | INFO | Perf report for the last 5.13965 seconds: | 20260505 14:36:40.451 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:36:40.451 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:40.451 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:40.451 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:40.451 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 14:36:40.451 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:40.451 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:40.451 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:36:40.451 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:40.451 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:40.451 | INFO | Uploaded 36GiB to GPU in 2.21223s (16.2731GiB/s) | 20260505 14:36:40.451 | INFO | Downloaded 34GiB to GPU in 2.113s (16.0908GiB/s) | 20260505 14:36:45.589 | INFO | Analyzing chunk frames = [3976192,3976704) | 20260505 14:36:45.589 | INFO | MemInfo: { | 20260505 14:36:45.589 | INFO | "gpu" : " 17 GB", | 20260505 14:36:45.589 | INFO | "huge" : "", | 20260505 14:36:45.589 | INFO | "huge_shared" : " 27 GB", | 20260505 14:36:45.589 | INFO | "malloc" : " 14 GB", | 20260505 14:36:45.589 | INFO | "pinned" : " 29 GB" | 20260505 14:36:45.589 | INFO | } | 20260505 14:36:45.589 | INFO | Perf report for the last 5.1383 seconds: | 20260505 14:36:45.589 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:36:45.589 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:45.589 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:45.589 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:45.589 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 14:36:45.589 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:45.589 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:45.589 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 14:36:45.589 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:45.589 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:45.589 | INFO | Uploaded 36GiB to GPU in 2.11687s (17.0062GiB/s) | 20260505 14:36:45.590 | INFO | Downloaded 34GiB to GPU in 2.16675s (15.6917GiB/s) | 20260505 14:36:50.729 | INFO | Analyzing chunk frames = [3976704,3977216) | 20260505 14:36:50.729 | INFO | MemInfo: { | 20260505 14:36:50.729 | INFO | "gpu" : " 17 GB", | 20260505 14:36:50.729 | INFO | "huge" : "", | 20260505 14:36:50.729 | INFO | "huge_shared" : " 27 GB", | 20260505 14:36:50.729 | INFO | "malloc" : " 14 GB", | 20260505 14:36:50.729 | INFO | "pinned" : " 29 GB" | 20260505 14:36:50.729 | INFO | } | 20260505 14:36:50.729 | INFO | Perf report for the last 5.1397 seconds: | 20260505 14:36:50.729 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:36:50.729 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:50.729 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:50.729 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:50.729 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 14:36:50.729 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:50.729 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:50.729 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:36:50.729 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:50.729 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:50.729 | INFO | Uploaded 36GiB to GPU in 2.21593s (16.246GiB/s) | 20260505 14:36:50.729 | INFO | Downloaded 34GiB to GPU in 2.09715s (16.2125GiB/s) | 20260505 14:36:50.868 | INFO | Starting a round of Baz IO | 20260505 14:36:54.087 | INFO | Populating RTMetrics report | 20260505 14:36:54.087 | INFO | RTMetrics report populated | 20260505 14:36:54.088 | INFO | Opening tmp RTMetrics json file | 20260505 14:36:54.103 | INFO | Serializing RTMetrics to json | 20260505 14:36:54.115 | INFO | Serialization done, swapping tmp file now... | 20260505 14:36:54.132 | INFO | RTMetrics done | 20260505 14:36:54.190 | INFO | Finished a round of Baz IO: Wrote 2.249 GiB in 3.321 seconds | 20260505 14:36:54.190 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.71% / 0.00% / 77.29% | 20260505 14:36:54.190 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.53918% / 0% / 7.54992e-05% | 20260505 14:36:54.190 | INFO | Baz IO overall: | 20260505 14:36:54.190 | INFO | BytesWritten : 2369130190155 | 20260505 14:36:54.190 | INFO | NumEvents : 1126254732071 | 20260505 14:36:55.863 | INFO | Analyzing chunk frames = [3977216,3977728) | 20260505 14:36:55.863 | INFO | MemInfo: { | 20260505 14:36:55.863 | INFO | "gpu" : " 17 GB", | 20260505 14:36:55.863 | INFO | "huge" : "", | 20260505 14:36:55.863 | INFO | "huge_shared" : " 27 GB", | 20260505 14:36:55.863 | INFO | "malloc" : " 14 GB", | 20260505 14:36:55.863 | INFO | "pinned" : " 29 GB" | 20260505 14:36:55.863 | INFO | } | 20260505 14:36:55.863 | INFO | Perf report for the last 5.13421 seconds: | 20260505 14:36:55.863 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:36:55.863 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:55.863 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:55.863 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:55.863 | INFO | ANALYSIS : 64.3%, 3.0, 99.8ms, 0 | 20260505 14:36:55.863 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:36:55.863 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260505 14:36:55.863 | INFO | PRE_HQ : 60.9%, 1.8, 57.1ms, 0 | 20260505 14:36:55.863 | INFO | BAZWRITER : 64.1%, 4.0, 131.5ms, 0 | 20260505 14:36:55.863 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 14:36:55.863 | INFO | Uploaded 36GiB to GPU in 2.19808s (16.3779GiB/s) | 20260505 14:36:55.863 | INFO | Downloaded 38GiB to GPU in 2.26577s (16.7713GiB/s) | 20260505 14:37:01.014 | INFO | Analyzing chunk frames = [3977728,3978240) | 20260505 14:37:01.015 | INFO | MemInfo: { | 20260505 14:37:01.015 | INFO | "gpu" : " 17 GB", | 20260505 14:37:01.015 | INFO | "huge" : "", | 20260505 14:37:01.015 | INFO | "huge_shared" : " 27 GB", | 20260505 14:37:01.015 | INFO | "malloc" : " 14 GB", | 20260505 14:37:01.015 | INFO | "pinned" : " 29 GB" | 20260505 14:37:01.015 | INFO | } | 20260505 14:37:01.015 | INFO | Perf report for the last 5.15163 seconds: | 20260505 14:37:01.015 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:37:01.015 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:01.015 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:01.015 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:01.015 | INFO | ANALYSIS : 63.3%, 3.0, 99.0ms, 0 | 20260505 14:37:01.015 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:01.015 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:01.015 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:37:01.015 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:01.015 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:01.015 | INFO | Uploaded 36GiB to GPU in 2.19808s (16.3779GiB/s) | 20260505 14:37:01.015 | INFO | Downloaded 34GiB to GPU in 2.04528s (16.6236GiB/s) | 20260505 14:37:06.148 | INFO | Analyzing chunk frames = [3978240,3978752) | 20260505 14:37:06.148 | INFO | MemInfo: { | 20260505 14:37:06.148 | INFO | "gpu" : " 17 GB", | 20260505 14:37:06.148 | INFO | "huge" : "", | 20260505 14:37:06.148 | INFO | "huge_shared" : " 27 GB", | 20260505 14:37:06.148 | INFO | "malloc" : " 14 GB", | 20260505 14:37:06.148 | INFO | "pinned" : " 29 GB" | 20260505 14:37:06.148 | INFO | } | 20260505 14:37:06.148 | INFO | Perf report for the last 5.13341 seconds: | 20260505 14:37:06.148 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:37:06.148 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:06.148 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:06.148 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:06.148 | INFO | ANALYSIS : 62.9%, 3.0, 97.7ms, 0 | 20260505 14:37:06.148 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:06.148 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:06.148 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:37:06.148 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:06.148 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:06.148 | INFO | Uploaded 36GiB to GPU in 2.17475s (16.5537GiB/s) | 20260505 14:37:06.148 | INFO | Downloaded 34GiB to GPU in 2.02143s (16.8198GiB/s) | 20260505 14:37:11.280 | INFO | Analyzing chunk frames = [3978752,3979264) | 20260505 14:37:11.280 | INFO | MemInfo: { | 20260505 14:37:11.280 | INFO | "gpu" : " 17 GB", | 20260505 14:37:11.280 | INFO | "huge" : "", | 20260505 14:37:11.280 | INFO | "huge_shared" : " 27 GB", | 20260505 14:37:11.280 | INFO | "malloc" : " 14 GB", | 20260505 14:37:11.280 | INFO | "pinned" : " 29 GB" | 20260505 14:37:11.280 | INFO | } | 20260505 14:37:11.280 | INFO | Perf report for the last 5.13161 seconds: | 20260505 14:37:11.280 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:37:11.280 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:11.280 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:11.280 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:11.280 | INFO | ANALYSIS : 62.8%, 3.0, 97.3ms, 0 | 20260505 14:37:11.280 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:11.280 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:11.280 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 14:37:11.280 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:11.280 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:11.280 | INFO | Uploaded 36GiB to GPU in 2.18759s (16.4564GiB/s) | 20260505 14:37:11.281 | INFO | Downloaded 34GiB to GPU in 2.128s (15.9775GiB/s) | 20260505 14:37:16.416 | INFO | Analyzing chunk frames = [3979264,3979776) | 20260505 14:37:16.416 | INFO | MemInfo: { | 20260505 14:37:16.416 | INFO | "gpu" : " 17 GB", | 20260505 14:37:16.416 | INFO | "huge" : "", | 20260505 14:37:16.416 | INFO | "huge_shared" : " 27 GB", | 20260505 14:37:16.416 | INFO | "malloc" : " 14 GB", | 20260505 14:37:16.416 | INFO | "pinned" : " 29 GB" | 20260505 14:37:16.416 | INFO | } | 20260505 14:37:16.416 | INFO | Perf report for the last 5.13614 seconds: | 20260505 14:37:16.416 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:37:16.416 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:16.416 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:16.416 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:16.416 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260505 14:37:16.416 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:16.416 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:16.416 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:37:16.416 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:16.416 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:16.416 | INFO | Uploaded 36GiB to GPU in 2.15657s (16.6932GiB/s) | 20260505 14:37:16.416 | INFO | Downloaded 34GiB to GPU in 2.12732s (15.9825GiB/s) | 20260505 14:37:21.559 | INFO | Analyzing chunk frames = [3979776,3980288) | 20260505 14:37:21.559 | INFO | MemInfo: { | 20260505 14:37:21.559 | INFO | "gpu" : " 17 GB", | 20260505 14:37:21.559 | INFO | "huge" : "", | 20260505 14:37:21.559 | INFO | "huge_shared" : " 27 GB", | 20260505 14:37:21.559 | INFO | "malloc" : " 14 GB", | 20260505 14:37:21.559 | INFO | "pinned" : " 29 GB" | 20260505 14:37:21.559 | INFO | } | 20260505 14:37:21.560 | INFO | Perf report for the last 5.14353 seconds: | 20260505 14:37:21.560 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:37:21.560 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:21.560 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:21.560 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:21.560 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 14:37:21.560 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:21.560 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:21.560 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 14:37:21.560 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:21.560 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:21.560 | INFO | Uploaded 36GiB to GPU in 2.163s (16.6435GiB/s) | 20260505 14:37:21.560 | INFO | Downloaded 34GiB to GPU in 2.10912s (16.1204GiB/s) | 20260505 14:37:26.703 | INFO | Analyzing chunk frames = [3980288,3980800) | 20260505 14:37:26.703 | INFO | MemInfo: { | 20260505 14:37:26.703 | INFO | "gpu" : " 17 GB", | 20260505 14:37:26.703 | INFO | "huge" : "", | 20260505 14:37:26.703 | INFO | "huge_shared" : " 27 GB", | 20260505 14:37:26.703 | INFO | "malloc" : " 14 GB", | 20260505 14:37:26.703 | INFO | "pinned" : " 29 GB" | 20260505 14:37:26.703 | INFO | } | 20260505 14:37:26.703 | INFO | Perf report for the last 5.14397 seconds: | 20260505 14:37:26.703 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:37:26.703 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:26.704 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:26.704 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:26.704 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 14:37:26.704 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:26.704 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:26.704 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 14:37:26.704 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:26.704 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:26.704 | INFO | Uploaded 36GiB to GPU in 2.11955s (16.9848GiB/s) | 20260505 14:37:26.704 | INFO | Downloaded 34GiB to GPU in 2.17909s (15.6028GiB/s) | 20260505 14:37:31.850 | INFO | Analyzing chunk frames = [3980800,3981312) | 20260505 14:37:31.850 | INFO | MemInfo: { | 20260505 14:37:31.850 | INFO | "gpu" : " 17 GB", | 20260505 14:37:31.850 | INFO | "huge" : "", | 20260505 14:37:31.850 | INFO | "huge_shared" : " 27 GB", | 20260505 14:37:31.850 | INFO | "malloc" : " 14 GB", | 20260505 14:37:31.850 | INFO | "pinned" : " 29 GB" | 20260505 14:37:31.850 | INFO | } | 20260505 14:37:31.850 | INFO | Perf report for the last 5.14659 seconds: | 20260505 14:37:31.850 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:37:31.850 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:31.850 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:31.850 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:31.850 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 14:37:31.850 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:31.850 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:31.850 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 14:37:31.850 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:31.850 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:31.850 | INFO | Uploaded 36GiB to GPU in 2.20914s (16.2959GiB/s) | 20260505 14:37:31.850 | INFO | Downloaded 34GiB to GPU in 2.09704s (16.2133GiB/s) | 20260505 14:37:31.851 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 14:37:31.851 | INFO | Total mem usage: 15.3 GiB | 20260505 14:37:31.851 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 14:37:31.851 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -7.13%) | 20260505 14:37:31.851 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 14:37:31.851 | INFO | High Water Breakdown: | 20260505 14:37:31.851 | INFO | 61.1% : 8.74 GiB ( 91.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 14:37:31.851 | INFO | 38.8% : 5.54 GiB ( 83.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 14:37:31.851 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 14:37:31.851 | INFO | ------------------------------------------ | 20260505 14:37:31.851 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:37:31.851 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 14:37:31.851 | INFO | Total mem usage: 26.5 GiB | 20260505 14:37:31.851 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 14:37:31.851 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 14:37:31.851 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 14:37:31.851 | INFO | High Water Breakdown: | 20260505 14:37:31.851 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:37:31.851 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:37:31.851 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:37:31.851 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:37:31.851 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:37:31.851 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:37:31.851 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 14:37:31.851 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 14:37:31.851 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:37:31.851 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 14:37:31.851 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 14:37:31.851 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 14:37:31.851 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 14:37:31.851 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 14:37:31.851 | INFO | ------------------------------------------ | 20260505 14:37:31.851 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:37:31.851 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 14:37:31.851 | INFO | Total mem usage: 25.6 GiB | 20260505 14:37:31.851 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 14:37:31.851 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 14:37:31.851 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 14:37:31.851 | INFO | High Water Breakdown: | 20260505 14:37:31.851 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 14:37:31.851 | INFO | ------------------------------------------ | 20260505 14:37:31.851 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:37:31.851 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 14:37:31.851 | INFO | Total mem usage: 15.7 GiB | 20260505 14:37:31.851 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 14:37:31.851 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 14:37:31.851 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260505 14:37:31.851 | INFO | High Water Breakdown: | 20260505 14:37:31.851 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 14:37:31.851 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 14:37:31.851 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 14:37:31.851 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 14:37:31.851 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 14:37:31.851 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 14:37:31.851 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 14:37:31.851 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 14:37:31.851 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 14:37:31.851 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:37:31.851 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 14:37:31.851 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:37:31.851 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 14:37:31.851 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 14:37:31.851 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 14:37:31.851 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 14:37:31.851 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:37:31.851 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 14:37:31.851 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:37:31.851 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:37:31.851 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:37:31.851 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 14:37:31.851 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 14:37:31.851 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:37:31.851 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 14:37:31.851 | INFO | ------------------------------------------ | 20260505 14:37:31.851 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:37:31.851 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 14:37:31.851 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:37:31.851 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:37:31.851 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.62e+01, 1.37e+01 / 1.82e+00 | 20260505 14:37:31.851 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 1.00e-01, 1.08e-02 / 3.81e-03 | 20260505 14:37:31.851 | INFO | PerfReport: 11.79%, Baseline -- 3.75e+00 / 4.49e+00, 5.86e-01 / 1.60e-01 | 20260505 14:37:31.851 | INFO | PerfReport: 3.30%, Histogram -- 1.05e+00 / 1.49e+00, 1.97e-01 / 1.06e-01 | 20260505 14:37:31.851 | INFO | PerfReport: 15.99%, DME -- 4.07e+01 / 4.57e+01, 4.49e+00 / 1.74e+00 | 20260505 14:37:31.851 | INFO | PerfReport: 56.41%, FrameLabeling -- 1.79e+01 / 1.96e+01, 1.38e+00 / 1.77e+00 | 20260505 14:37:31.851 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.37e+00 / 2.94e+00, 4.55e-01 / 1.65e-01 | 20260505 14:37:31.851 | INFO | PerfReport: 3.53%, Metrics -- 1.12e+00 / 1.43e+00, 3.11e-01 / 1.40e-01 | 20260505 14:37:31.851 | INFO | PerfReport: 1.23%, PulseSerializing -- 3.92e-01 / 4.38e-01, 6.80e-02 / 2.96e-02 | 20260505 14:37:31.852 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 14:37:31.852 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:37:31.852 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:37:31.852 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260505 14:37:31.852 | INFO | PerfReport: 22.55%, Upload -- 2.27e+01 / 0.00e+00, 8.06e+00 / 0.00e+00 | 20260505 14:37:31.852 | INFO | PerfReport: 21.95%, Download -- 2.21e+01 / 0.00e+00, 8.02e+00 / 0.00e+00 | 20260505 14:37:31.852 | INFO | PerfReport: 31.58%, Compute -- 3.18e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 14:37:31.852 | INFO | PerfReport: 2.28%, LockUpload -- 2.30e+00 / 0.00e+00, 8.11e+00 / 0.00e+00 | 20260505 14:37:31.852 | INFO | PerfReport: 1.47%, LockDownload -- 1.48e+00 / 0.00e+00, 3.03e+00 / 0.00e+00 | 20260505 14:37:31.852 | INFO | PerfReport: 20.15%, LockCompute -- 2.03e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 14:37:31.852 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.81e-02 / 0.00e+00 | 20260505 14:37:31.982 | INFO | Starting a round of Baz IO | 20260505 14:37:35.201 | INFO | Populating RTMetrics report | 20260505 14:37:35.201 | INFO | RTMetrics report populated | 20260505 14:37:35.201 | INFO | Opening tmp RTMetrics json file | 20260505 14:37:35.207 | INFO | Serializing RTMetrics to json | 20260505 14:37:35.235 | INFO | Serialization done, swapping tmp file now... | 20260505 14:37:35.249 | INFO | RTMetrics done | 20260505 14:37:35.307 | INFO | Finished a round of Baz IO: Wrote 2.25 GiB in 3.325 seconds | 20260505 14:37:35.307 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.70% / 0.00% / 77.29% | 20260505 14:37:35.307 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.537864% / 7.94729e-06% / 7.54992e-05% | 20260505 14:37:35.308 | INFO | Baz IO overall: | 20260505 14:37:35.308 | INFO | BytesWritten : 2371545694221 | 20260505 14:37:35.308 | INFO | NumEvents : 1127393556637 | 20260505 14:37:36.982 | INFO | Analyzing chunk frames = [3981312,3981824) | 20260505 14:37:36.982 | INFO | MemInfo: { | 20260505 14:37:36.982 | INFO | "gpu" : " 17 GB", | 20260505 14:37:36.982 | INFO | "huge" : "", | 20260505 14:37:36.982 | INFO | "huge_shared" : " 27 GB", | 20260505 14:37:36.982 | INFO | "malloc" : " 14 GB", | 20260505 14:37:36.982 | INFO | "pinned" : " 29 GB" | 20260505 14:37:36.982 | INFO | } | 20260505 14:37:36.982 | INFO | Perf report for the last 5.13218 seconds: | 20260505 14:37:36.982 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:37:36.982 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:36.982 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:36.982 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:36.982 | INFO | ANALYSIS : 64.2%, 3.0, 99.5ms, 0 | 20260505 14:37:36.982 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:36.982 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 14:37:36.982 | INFO | PRE_HQ : 61.3%, 1.8, 57.7ms, 0 | 20260505 14:37:36.982 | INFO | BAZWRITER : 64.1%, 4.0, 131.6ms, 0 | 20260505 14:37:36.982 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 14:37:36.983 | INFO | Uploaded 36GiB to GPU in 2.20989s (16.2904GiB/s) | 20260505 14:37:36.983 | INFO | Downloaded 38GiB to GPU in 2.26888s (16.7483GiB/s) | 20260505 14:37:42.115 | INFO | Analyzing chunk frames = [3981824,3982336) | 20260505 14:37:42.115 | INFO | MemInfo: { | 20260505 14:37:42.115 | INFO | "gpu" : " 17 GB", | 20260505 14:37:42.115 | INFO | "huge" : "", | 20260505 14:37:42.115 | INFO | "huge_shared" : " 27 GB", | 20260505 14:37:42.115 | INFO | "malloc" : " 14 GB", | 20260505 14:37:42.115 | INFO | "pinned" : " 29 GB" | 20260505 14:37:42.115 | INFO | } | 20260505 14:37:42.115 | INFO | Perf report for the last 5.13254 seconds: | 20260505 14:37:42.115 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:37:42.115 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:42.115 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:42.115 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:42.115 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 14:37:42.115 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:42.115 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:42.115 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:37:42.115 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:42.115 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:42.115 | INFO | Uploaded 36GiB to GPU in 2.19762s (16.3813GiB/s) | 20260505 14:37:42.115 | INFO | Downloaded 34GiB to GPU in 2.03275s (16.7261GiB/s) | 20260505 14:37:47.266 | INFO | Analyzing chunk frames = [3982336,3982848) | 20260505 14:37:47.266 | INFO | MemInfo: { | 20260505 14:37:47.266 | INFO | "gpu" : " 17 GB", | 20260505 14:37:47.266 | INFO | "huge" : "", | 20260505 14:37:47.266 | INFO | "huge_shared" : " 27 GB", | 20260505 14:37:47.266 | INFO | "malloc" : " 14 GB", | 20260505 14:37:47.266 | INFO | "pinned" : " 29 GB" | 20260505 14:37:47.266 | INFO | } | 20260505 14:37:47.266 | INFO | Perf report for the last 5.1512 seconds: | 20260505 14:37:47.266 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:37:47.266 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:47.266 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:47.266 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:47.266 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 14:37:47.266 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:47.266 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:47.266 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 14:37:47.266 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:47.266 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:47.266 | INFO | Uploaded 36GiB to GPU in 2.24178s (16.0586GiB/s) | 20260505 14:37:47.266 | INFO | Downloaded 34GiB to GPU in 2.10367s (16.1622GiB/s) | 20260505 14:37:52.406 | INFO | Analyzing chunk frames = [3982848,3983360) | 20260505 14:37:52.406 | INFO | MemInfo: { | 20260505 14:37:52.406 | INFO | "gpu" : " 17 GB", | 20260505 14:37:52.406 | INFO | "huge" : "", | 20260505 14:37:52.406 | INFO | "huge_shared" : " 27 GB", | 20260505 14:37:52.406 | INFO | "malloc" : " 14 GB", | 20260505 14:37:52.406 | INFO | "pinned" : " 29 GB" | 20260505 14:37:52.406 | INFO | } | 20260505 14:37:52.407 | INFO | Perf report for the last 5.14051 seconds: | 20260505 14:37:52.407 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:37:52.407 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:52.407 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:52.407 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:52.407 | INFO | ANALYSIS : 62.6%, 3.0, 97.3ms, 0 | 20260505 14:37:52.407 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:52.407 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:52.407 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:37:52.407 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:52.407 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:52.407 | INFO | Uploaded 36GiB to GPU in 2.19293s (16.4164GiB/s) | 20260505 14:37:52.407 | INFO | Downloaded 34GiB to GPU in 2.13305s (15.9396GiB/s) | 20260505 14:37:57.536 | INFO | Analyzing chunk frames = [3983360,3983872) | 20260505 14:37:57.536 | INFO | MemInfo: { | 20260505 14:37:57.536 | INFO | "gpu" : " 17 GB", | 20260505 14:37:57.536 | INFO | "huge" : "", | 20260505 14:37:57.536 | INFO | "huge_shared" : " 27 GB", | 20260505 14:37:57.536 | INFO | "malloc" : " 14 GB", | 20260505 14:37:57.536 | INFO | "pinned" : " 29 GB" | 20260505 14:37:57.536 | INFO | } | 20260505 14:37:57.536 | INFO | Perf report for the last 5.12969 seconds: | 20260505 14:37:57.536 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:37:57.536 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:57.536 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:57.536 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:57.536 | INFO | ANALYSIS : 62.9%, 3.0, 98.0ms, 0 | 20260505 14:37:57.536 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:57.536 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:57.536 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:37:57.536 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:57.536 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:37:57.537 | INFO | Uploaded 36GiB to GPU in 2.18746s (16.4575GiB/s) | 20260505 14:37:57.537 | INFO | Downloaded 34GiB to GPU in 2.13155s (15.9508GiB/s) | 20260505 14:38:02.676 | INFO | Analyzing chunk frames = [3983872,3984384) | 20260505 14:38:02.676 | INFO | MemInfo: { | 20260505 14:38:02.676 | INFO | "gpu" : " 17 GB", | 20260505 14:38:02.676 | INFO | "huge" : "", | 20260505 14:38:02.676 | INFO | "huge_shared" : " 27 GB", | 20260505 14:38:02.676 | INFO | "malloc" : " 14 GB", | 20260505 14:38:02.676 | INFO | "pinned" : " 29 GB" | 20260505 14:38:02.676 | INFO | } | 20260505 14:38:02.676 | INFO | Perf report for the last 5.13953 seconds: | 20260505 14:38:02.676 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:38:02.676 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:02.676 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:02.676 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:02.676 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 14:38:02.676 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:02.676 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:02.676 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 14:38:02.676 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:02.676 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:02.676 | INFO | Uploaded 36GiB to GPU in 2.18876s (16.4477GiB/s) | 20260505 14:38:02.676 | INFO | Downloaded 34GiB to GPU in 2.10774s (16.131GiB/s) | 20260505 14:38:07.823 | INFO | Analyzing chunk frames = [3984384,3984896) | 20260505 14:38:07.823 | INFO | MemInfo: { | 20260505 14:38:07.823 | INFO | "gpu" : " 17 GB", | 20260505 14:38:07.823 | INFO | "huge" : "", | 20260505 14:38:07.823 | INFO | "huge_shared" : " 27 GB", | 20260505 14:38:07.823 | INFO | "malloc" : " 14 GB", | 20260505 14:38:07.823 | INFO | "pinned" : " 29 GB" | 20260505 14:38:07.823 | INFO | } | 20260505 14:38:07.823 | INFO | Perf report for the last 5.14766 seconds: | 20260505 14:38:07.824 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:38:07.824 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:07.824 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:07.824 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:07.824 | INFO | ANALYSIS : 63.1%, 3.0, 98.7ms, 0 | 20260505 14:38:07.824 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:07.824 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:07.824 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 14:38:07.824 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:07.824 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:07.824 | INFO | Uploaded 36GiB to GPU in 2.1385s (16.8342GiB/s) | 20260505 14:38:07.824 | INFO | Downloaded 34GiB to GPU in 2.1804s (15.5935GiB/s) | 20260505 14:38:12.961 | INFO | Analyzing chunk frames = [3984896,3985408) | 20260505 14:38:12.961 | INFO | MemInfo: { | 20260505 14:38:12.961 | INFO | "gpu" : " 17 GB", | 20260505 14:38:12.961 | INFO | "huge" : "", | 20260505 14:38:12.961 | INFO | "huge_shared" : " 27 GB", | 20260505 14:38:12.961 | INFO | "malloc" : " 14 GB", | 20260505 14:38:12.961 | INFO | "pinned" : " 29 GB" | 20260505 14:38:12.961 | INFO | } | 20260505 14:38:12.961 | INFO | Perf report for the last 5.13716 seconds: | 20260505 14:38:12.961 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:38:12.961 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:12.961 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:12.961 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:12.961 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 14:38:12.961 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:12.961 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:12.961 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 14:38:12.961 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:12.961 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:12.961 | INFO | Uploaded 36GiB to GPU in 2.2343s (16.1124GiB/s) | 20260505 14:38:12.961 | INFO | Downloaded 34GiB to GPU in 2.10346s (16.1638GiB/s) | 20260505 14:38:13.096 | INFO | Starting a round of Baz IO | 20260505 14:38:16.311 | INFO | Populating RTMetrics report | 20260505 14:38:16.312 | INFO | RTMetrics report populated | 20260505 14:38:16.312 | INFO | Opening tmp RTMetrics json file | 20260505 14:38:16.330 | INFO | Serializing RTMetrics to json | 20260505 14:38:16.332 | INFO | Serialization done, swapping tmp file now... | 20260505 14:38:16.332 | INFO | RTMetrics done | 20260505 14:38:16.420 | INFO | Finished a round of Baz IO: Wrote 2.25 GiB in 3.324 seconds | 20260505 14:38:16.420 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.70% / 0.00% / 77.30% | 20260505 14:38:16.420 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.536009% / 3.97364e-06% / 7.54992e-05% | 20260505 14:38:16.420 | INFO | Baz IO overall: | 20260505 14:38:16.420 | INFO | BytesWritten : 2373961328127 | 20260505 14:38:16.420 | INFO | NumEvents : 1128532447612 | 20260505 14:38:18.097 | INFO | Analyzing chunk frames = [3985408,3985920) | 20260505 14:38:18.097 | INFO | MemInfo: { | 20260505 14:38:18.097 | INFO | "gpu" : " 17 GB", | 20260505 14:38:18.097 | INFO | "huge" : "", | 20260505 14:38:18.097 | INFO | "huge_shared" : " 27 GB", | 20260505 14:38:18.097 | INFO | "malloc" : " 14 GB", | 20260505 14:38:18.097 | INFO | "pinned" : " 29 GB" | 20260505 14:38:18.097 | INFO | } | 20260505 14:38:18.097 | INFO | Perf report for the last 5.13621 seconds: | 20260505 14:38:18.097 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:38:18.097 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:18.097 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:18.097 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:18.097 | INFO | ANALYSIS : 64.1%, 3.0, 99.7ms, 0 | 20260505 14:38:18.097 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:18.097 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260505 14:38:18.097 | INFO | PRE_HQ : 61.1%, 1.8, 57.2ms, 0 | 20260505 14:38:18.097 | INFO | BAZWRITER : 64.1%, 4.0, 130.8ms, 0 | 20260505 14:38:18.097 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 14:38:18.097 | INFO | Uploaded 36GiB to GPU in 2.2059s (16.3199GiB/s) | 20260505 14:38:18.097 | INFO | Downloaded 38GiB to GPU in 2.26489s (16.7779GiB/s) | 20260505 14:38:23.231 | INFO | Analyzing chunk frames = [3985920,3986432) | 20260505 14:38:23.231 | INFO | MemInfo: { | 20260505 14:38:23.231 | INFO | "gpu" : " 17 GB", | 20260505 14:38:23.231 | INFO | "huge" : "", | 20260505 14:38:23.231 | INFO | "huge_shared" : " 27 GB", | 20260505 14:38:23.231 | INFO | "malloc" : " 14 GB", | 20260505 14:38:23.231 | INFO | "pinned" : " 29 GB" | 20260505 14:38:23.231 | INFO | } | 20260505 14:38:23.231 | INFO | Perf report for the last 5.13443 seconds: | 20260505 14:38:23.231 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:38:23.231 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:23.231 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:23.231 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:23.231 | INFO | ANALYSIS : 63.5%, 3.0, 99.0ms, 0 | 20260505 14:38:23.231 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:23.231 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:23.231 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:38:23.231 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:23.231 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:23.232 | INFO | Uploaded 36GiB to GPU in 2.1923s (16.4211GiB/s) | 20260505 14:38:23.232 | INFO | Downloaded 34GiB to GPU in 2.02916s (16.7557GiB/s) | 20260505 14:38:28.371 | INFO | Analyzing chunk frames = [3986432,3986944) | 20260505 14:38:28.371 | INFO | MemInfo: { | 20260505 14:38:28.371 | INFO | "gpu" : " 17 GB", | 20260505 14:38:28.371 | INFO | "huge" : "", | 20260505 14:38:28.371 | INFO | "huge_shared" : " 27 GB", | 20260505 14:38:28.371 | INFO | "malloc" : " 14 GB", | 20260505 14:38:28.371 | INFO | "pinned" : " 29 GB" | 20260505 14:38:28.371 | INFO | } | 20260505 14:38:28.371 | INFO | Perf report for the last 5.13958 seconds: | 20260505 14:38:28.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:38:28.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:28.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:28.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:28.371 | INFO | ANALYSIS : 63.2%, 3.0, 98.4ms, 0 | 20260505 14:38:28.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:28.371 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:28.371 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:38:28.371 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:28.371 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:28.371 | INFO | Uploaded 36GiB to GPU in 2.20195s (16.3492GiB/s) | 20260505 14:38:28.371 | INFO | Downloaded 34GiB to GPU in 2.12439s (16.0046GiB/s) | 20260505 14:38:33.517 | INFO | Analyzing chunk frames = [3986944,3987456) | 20260505 14:38:33.517 | INFO | MemInfo: { | 20260505 14:38:33.517 | INFO | "gpu" : " 17 GB", | 20260505 14:38:33.517 | INFO | "huge" : "", | 20260505 14:38:33.517 | INFO | "huge_shared" : " 27 GB", | 20260505 14:38:33.517 | INFO | "malloc" : " 14 GB", | 20260505 14:38:33.517 | INFO | "pinned" : " 29 GB" | 20260505 14:38:33.517 | INFO | } | 20260505 14:38:33.517 | INFO | Perf report for the last 5.14618 seconds: | 20260505 14:38:33.517 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:38:33.517 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:33.517 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:33.517 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:33.517 | INFO | ANALYSIS : 62.7%, 3.0, 97.5ms, 0 | 20260505 14:38:33.517 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:33.517 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:33.517 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:38:33.517 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:33.517 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:33.517 | INFO | Uploaded 36GiB to GPU in 2.16826s (16.6031GiB/s) | 20260505 14:38:33.517 | INFO | Downloaded 34GiB to GPU in 2.13524s (15.9233GiB/s) | 20260505 14:38:38.659 | INFO | Analyzing chunk frames = [3987456,3987968) | 20260505 14:38:38.659 | INFO | MemInfo: { | 20260505 14:38:38.659 | INFO | "gpu" : " 17 GB", | 20260505 14:38:38.659 | INFO | "huge" : "", | 20260505 14:38:38.659 | INFO | "huge_shared" : " 27 GB", | 20260505 14:38:38.659 | INFO | "malloc" : " 14 GB", | 20260505 14:38:38.659 | INFO | "pinned" : " 29 GB" | 20260505 14:38:38.659 | INFO | } | 20260505 14:38:38.659 | INFO | Perf report for the last 5.14212 seconds: | 20260505 14:38:38.659 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:38:38.659 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:38.659 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:38.659 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:38.659 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260505 14:38:38.659 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:38.659 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:38.659 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:38:38.659 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:38.659 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:38.660 | INFO | Uploaded 36GiB to GPU in 2.13732s (16.8436GiB/s) | 20260505 14:38:38.660 | INFO | Downloaded 34GiB to GPU in 2.05092s (16.578GiB/s) | 20260505 14:38:43.794 | INFO | Analyzing chunk frames = [3987968,3988480) | 20260505 14:38:43.794 | INFO | MemInfo: { | 20260505 14:38:43.794 | INFO | "gpu" : " 17 GB", | 20260505 14:38:43.794 | INFO | "huge" : "", | 20260505 14:38:43.794 | INFO | "huge_shared" : " 27 GB", | 20260505 14:38:43.794 | INFO | "malloc" : " 14 GB", | 20260505 14:38:43.794 | INFO | "pinned" : " 29 GB" | 20260505 14:38:43.794 | INFO | } | 20260505 14:38:43.794 | INFO | Perf report for the last 5.1352 seconds: | 20260505 14:38:43.795 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:38:43.795 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:43.795 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:43.795 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:43.795 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 14:38:43.795 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:43.795 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:43.795 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 14:38:43.795 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:43.795 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:43.795 | INFO | Uploaded 36GiB to GPU in 2.20245s (16.3454GiB/s) | 20260505 14:38:43.795 | INFO | Downloaded 34GiB to GPU in 2.11333s (16.0883GiB/s) | 20260505 14:38:48.923 | INFO | Analyzing chunk frames = [3988480,3988992) | 20260505 14:38:48.924 | INFO | MemInfo: { | 20260505 14:38:48.924 | INFO | "gpu" : " 17 GB", | 20260505 14:38:48.924 | INFO | "huge" : "", | 20260505 14:38:48.924 | INFO | "huge_shared" : " 27 GB", | 20260505 14:38:48.924 | INFO | "malloc" : " 14 GB", | 20260505 14:38:48.924 | INFO | "pinned" : " 29 GB" | 20260505 14:38:48.924 | INFO | } | 20260505 14:38:48.924 | INFO | Perf report for the last 5.12911 seconds: | 20260505 14:38:48.924 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:38:48.924 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:48.924 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:48.924 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:48.924 | INFO | ANALYSIS : 63.4%, 3.0, 98.8ms, 0 | 20260505 14:38:48.924 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:48.924 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:48.924 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 14:38:48.924 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:48.924 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:48.924 | INFO | Uploaded 36GiB to GPU in 2.17012s (16.5889GiB/s) | 20260505 14:38:48.924 | INFO | Downloaded 34GiB to GPU in 2.15703s (15.7624GiB/s) | 20260505 14:38:54.072 | INFO | Analyzing chunk frames = [3988992,3989504) | 20260505 14:38:54.072 | INFO | MemInfo: { | 20260505 14:38:54.072 | INFO | "gpu" : " 17 GB", | 20260505 14:38:54.072 | INFO | "huge" : "", | 20260505 14:38:54.072 | INFO | "huge_shared" : " 27 GB", | 20260505 14:38:54.072 | INFO | "malloc" : " 14 GB", | 20260505 14:38:54.072 | INFO | "pinned" : " 29 GB" | 20260505 14:38:54.072 | INFO | } | 20260505 14:38:54.073 | INFO | Perf report for the last 5.14892 seconds: | 20260505 14:38:54.073 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:38:54.073 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:54.073 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:54.073 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:54.073 | INFO | ANALYSIS : 63.4%, 3.0, 99.2ms, 0 | 20260505 14:38:54.073 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:54.073 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:54.073 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 14:38:54.073 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:54.073 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:54.073 | INFO | Uploaded 36GiB to GPU in 2.13936s (16.8275GiB/s) | 20260505 14:38:54.073 | INFO | Downloaded 34GiB to GPU in 2.10664s (16.1394GiB/s) | 20260505 14:38:54.073 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 14:38:54.073 | INFO | Total mem usage: 15.3 GiB | 20260505 14:38:54.073 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 14:38:54.073 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.7%) | 20260505 14:38:54.073 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 14:38:54.073 | INFO | High Water Breakdown: | 20260505 14:38:54.073 | INFO | 61% : 8.68 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 14:38:54.073 | INFO | 38.9% : 5.53 GiB ( 83.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 14:38:54.073 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 14:38:54.073 | INFO | ------------------------------------------ | 20260505 14:38:54.073 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:38:54.073 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 14:38:54.073 | INFO | Total mem usage: 26.5 GiB | 20260505 14:38:54.073 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 14:38:54.073 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 14:38:54.073 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 14:38:54.073 | INFO | High Water Breakdown: | 20260505 14:38:54.073 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:38:54.073 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:38:54.073 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:38:54.073 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:38:54.073 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:38:54.073 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:38:54.073 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 14:38:54.073 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 14:38:54.073 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:38:54.073 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 14:38:54.073 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 14:38:54.074 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 14:38:54.074 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 14:38:54.074 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 14:38:54.074 | INFO | ------------------------------------------ | 20260505 14:38:54.074 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:38:54.074 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 14:38:54.074 | INFO | Total mem usage: 25.6 GiB | 20260505 14:38:54.074 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 14:38:54.074 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 14:38:54.074 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 14:38:54.074 | INFO | High Water Breakdown: | 20260505 14:38:54.074 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 14:38:54.074 | INFO | ------------------------------------------ | 20260505 14:38:54.074 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:38:54.074 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 14:38:54.074 | INFO | Total mem usage: 15.7 GiB | 20260505 14:38:54.074 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 14:38:54.074 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 14:38:54.074 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 14:38:54.074 | INFO | High Water Breakdown: | 20260505 14:38:54.074 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 14:38:54.074 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 14:38:54.074 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 14:38:54.074 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 14:38:54.074 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 14:38:54.074 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 14:38:54.074 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 14:38:54.074 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 14:38:54.074 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 14:38:54.074 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:38:54.074 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 14:38:54.074 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:38:54.074 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 14:38:54.074 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 14:38:54.074 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 14:38:54.074 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 14:38:54.074 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:38:54.074 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 14:38:54.074 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:38:54.074 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:38:54.074 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:38:54.074 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 14:38:54.074 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 14:38:54.074 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:38:54.074 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 14:38:54.074 | INFO | ------------------------------------------ | 20260505 14:38:54.074 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:38:54.074 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 14:38:54.074 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:38:54.074 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:38:54.074 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.61e+01, 1.36e+01 / 1.75e+00 | 20260505 14:38:54.074 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 9.98e-02, 1.05e-02 / 5.30e-03 | 20260505 14:38:54.074 | INFO | PerfReport: 11.77%, Baseline -- 3.74e+00 / 4.51e+00, 5.69e-01 / 1.63e-01 | 20260505 14:38:54.074 | INFO | PerfReport: 3.29%, Histogram -- 1.04e+00 / 1.50e+00, 1.94e-01 / 9.26e-02 | 20260505 14:38:54.074 | INFO | PerfReport: 15.99%, DME -- 4.06e+01 / 4.55e+01, 4.38e+00 / 1.83e+00 | 20260505 14:38:54.074 | INFO | PerfReport: 56.42%, FrameLabeling -- 1.79e+01 / 1.97e+01, 1.34e+00 / 1.66e+00 | 20260505 14:38:54.074 | INFO | PerfReport: 7.46%, PulseAccumulating -- 2.37e+00 / 2.90e+00, 4.54e-01 / 2.03e-01 | 20260505 14:38:54.074 | INFO | PerfReport: 3.54%, Metrics -- 1.12e+00 / 1.44e+00, 3.13e-01 / 1.90e-01 | 20260505 14:38:54.074 | INFO | PerfReport: 1.24%, PulseSerializing -- 3.93e-01 / 4.42e-01, 7.04e-02 / 3.47e-02 | 20260505 14:38:54.074 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 14:38:54.074 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:38:54.074 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:38:54.074 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260505 14:38:54.074 | INFO | PerfReport: 22.64%, Upload -- 2.28e+01 / 0.00e+00, 8.11e+00 / 0.00e+00 | 20260505 14:38:54.074 | INFO | PerfReport: 22.02%, Download -- 2.22e+01 / 0.00e+00, 8.02e+00 / 0.00e+00 | 20260505 14:38:54.074 | INFO | PerfReport: 31.53%, Compute -- 3.17e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 14:38:54.074 | INFO | PerfReport: 2.33%, LockUpload -- 2.34e+00 / 0.00e+00, 7.90e+00 / 0.00e+00 | 20260505 14:38:54.074 | INFO | PerfReport: 1.35%, LockDownload -- 1.36e+00 / 0.00e+00, 2.81e+00 / 0.00e+00 | 20260505 14:38:54.074 | INFO | PerfReport: 20.11%, LockCompute -- 2.02e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260505 14:38:54.074 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.71e-02 / 0.00e+00 | 20260505 14:38:54.204 | INFO | Starting a round of Baz IO | 20260505 14:38:57.422 | INFO | Populating RTMetrics report | 20260505 14:38:57.422 | INFO | RTMetrics report populated | 20260505 14:38:57.422 | INFO | Opening tmp RTMetrics json file | 20260505 14:38:57.437 | INFO | Serializing RTMetrics to json | 20260505 14:38:57.438 | INFO | Serialization done, swapping tmp file now... | 20260505 14:38:57.439 | INFO | RTMetrics done | 20260505 14:38:57.528 | INFO | Finished a round of Baz IO: Wrote 2.248 GiB in 3.323 seconds | 20260505 14:38:57.528 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.69% / 0.00% / 77.30% | 20260505 14:38:57.528 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.534336% / 1.19209e-05% / 7.54992e-05% | 20260505 14:38:57.528 | INFO | Baz IO overall: | 20260505 14:38:57.528 | INFO | BytesWritten : 2376375008559 | 20260505 14:38:57.528 | INFO | NumEvents : 1129670069987 | 20260505 14:38:59.212 | INFO | Analyzing chunk frames = [3989504,3990016) | 20260505 14:38:59.212 | INFO | MemInfo: { | 20260505 14:38:59.212 | INFO | "gpu" : " 17 GB", | 20260505 14:38:59.212 | INFO | "huge" : "", | 20260505 14:38:59.212 | INFO | "huge_shared" : " 27 GB", | 20260505 14:38:59.212 | INFO | "malloc" : " 14 GB", | 20260505 14:38:59.212 | INFO | "pinned" : " 29 GB" | 20260505 14:38:59.212 | INFO | } | 20260505 14:38:59.212 | INFO | Perf report for the last 5.13983 seconds: | 20260505 14:38:59.212 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:38:59.212 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:59.212 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:59.212 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:59.212 | INFO | ANALYSIS : 64.1%, 3.0, 99.6ms, 0 | 20260505 14:38:59.212 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:38:59.213 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260505 14:38:59.213 | INFO | PRE_HQ : 61.0%, 1.8, 57.0ms, 0 | 20260505 14:38:59.213 | INFO | BAZWRITER : 64.0%, 4.0, 130.6ms, 0 | 20260505 14:38:59.213 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 14:38:59.213 | INFO | Uploaded 36GiB to GPU in 2.20396s (16.3342GiB/s) | 20260505 14:38:59.213 | INFO | Downloaded 38GiB to GPU in 2.2718s (16.7268GiB/s) | 20260505 14:39:04.355 | INFO | Analyzing chunk frames = [3990016,3990528) | 20260505 14:39:04.355 | INFO | MemInfo: { | 20260505 14:39:04.355 | INFO | "gpu" : " 17 GB", | 20260505 14:39:04.355 | INFO | "huge" : "", | 20260505 14:39:04.355 | INFO | "huge_shared" : " 27 GB", | 20260505 14:39:04.355 | INFO | "malloc" : " 14 GB", | 20260505 14:39:04.355 | INFO | "pinned" : " 29 GB" | 20260505 14:39:04.355 | INFO | } | 20260505 14:39:04.355 | INFO | Perf report for the last 5.14308 seconds: | 20260505 14:39:04.355 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:39:04.355 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:04.355 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:04.355 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:04.355 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 14:39:04.356 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:04.356 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:04.356 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 14:39:04.356 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:04.356 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:04.356 | INFO | Uploaded 36GiB to GPU in 2.19982s (16.365GiB/s) | 20260505 14:39:04.356 | INFO | Downloaded 34GiB to GPU in 2.04485s (16.6271GiB/s) | 20260505 14:39:09.490 | INFO | Analyzing chunk frames = [3990528,3991040) | 20260505 14:39:09.490 | INFO | MemInfo: { | 20260505 14:39:09.490 | INFO | "gpu" : " 17 GB", | 20260505 14:39:09.490 | INFO | "huge" : "", | 20260505 14:39:09.490 | INFO | "huge_shared" : " 27 GB", | 20260505 14:39:09.490 | INFO | "malloc" : " 14 GB", | 20260505 14:39:09.490 | INFO | "pinned" : " 29 GB" | 20260505 14:39:09.490 | INFO | } | 20260505 14:39:09.490 | INFO | Perf report for the last 5.13497 seconds: | 20260505 14:39:09.490 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:39:09.490 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:09.490 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:09.491 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:09.491 | INFO | ANALYSIS : 63.1%, 3.0, 98.2ms, 0 | 20260505 14:39:09.491 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:09.491 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:09.491 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:39:09.491 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:09.491 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:09.491 | INFO | Uploaded 36GiB to GPU in 2.19249s (16.4197GiB/s) | 20260505 14:39:09.491 | INFO | Downloaded 34GiB to GPU in 2.05953s (16.5087GiB/s) | 20260505 14:39:14.626 | INFO | Analyzing chunk frames = [3991040,3991552) | 20260505 14:39:14.626 | INFO | MemInfo: { | 20260505 14:39:14.626 | INFO | "gpu" : " 17 GB", | 20260505 14:39:14.627 | INFO | "huge" : "", | 20260505 14:39:14.627 | INFO | "huge_shared" : " 27 GB", | 20260505 14:39:14.627 | INFO | "malloc" : " 14 GB", | 20260505 14:39:14.627 | INFO | "pinned" : " 29 GB" | 20260505 14:39:14.627 | INFO | } | 20260505 14:39:14.627 | INFO | Perf report for the last 5.13616 seconds: | 20260505 14:39:14.627 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:39:14.627 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:14.627 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:14.627 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:14.627 | INFO | ANALYSIS : 63.0%, 3.0, 97.8ms, 0 | 20260505 14:39:14.627 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:14.627 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:14.627 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 14:39:14.627 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:14.627 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:14.627 | INFO | Uploaded 36GiB to GPU in 2.19946s (16.3677GiB/s) | 20260505 14:39:14.627 | INFO | Downloaded 34GiB to GPU in 2.13469s (15.9274GiB/s) | 20260505 14:39:19.773 | INFO | Analyzing chunk frames = [3991552,3992064) | 20260505 14:39:19.773 | INFO | MemInfo: { | 20260505 14:39:19.773 | INFO | "gpu" : " 17 GB", | 20260505 14:39:19.773 | INFO | "huge" : "", | 20260505 14:39:19.773 | INFO | "huge_shared" : " 27 GB", | 20260505 14:39:19.773 | INFO | "malloc" : " 14 GB", | 20260505 14:39:19.773 | INFO | "pinned" : " 29 GB" | 20260505 14:39:19.773 | INFO | } | 20260505 14:39:19.774 | INFO | Perf report for the last 5.14692 seconds: | 20260505 14:39:19.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:39:19.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:19.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:19.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:19.774 | INFO | ANALYSIS : 62.7%, 3.0, 98.1ms, 0 | 20260505 14:39:19.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:19.774 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:19.774 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 14:39:19.774 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:19.774 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:19.774 | INFO | Uploaded 36GiB to GPU in 2.17728s (16.5344GiB/s) | 20260505 14:39:19.774 | INFO | Downloaded 34GiB to GPU in 2.13004s (15.9621GiB/s) | 20260505 14:39:24.913 | INFO | Analyzing chunk frames = [3992064,3992576) | 20260505 14:39:24.913 | INFO | MemInfo: { | 20260505 14:39:24.913 | INFO | "gpu" : " 17 GB", | 20260505 14:39:24.913 | INFO | "huge" : "", | 20260505 14:39:24.913 | INFO | "huge_shared" : " 27 GB", | 20260505 14:39:24.914 | INFO | "malloc" : " 14 GB", | 20260505 14:39:24.914 | INFO | "pinned" : " 29 GB" | 20260505 14:39:24.914 | INFO | } | 20260505 14:39:24.914 | INFO | Perf report for the last 5.14065 seconds: | 20260505 14:39:24.914 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:39:24.914 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:24.914 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:24.914 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:24.914 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 14:39:24.914 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:24.914 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:24.914 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 14:39:24.914 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:24.914 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:24.914 | INFO | Uploaded 36GiB to GPU in 2.15416s (16.7119GiB/s) | 20260505 14:39:24.914 | INFO | Downloaded 34GiB to GPU in 2.04075s (16.6605GiB/s) | 20260505 14:39:30.041 | INFO | Analyzing chunk frames = [3992576,3993088) | 20260505 14:39:30.041 | INFO | MemInfo: { | 20260505 14:39:30.041 | INFO | "gpu" : " 17 GB", | 20260505 14:39:30.041 | INFO | "huge" : "", | 20260505 14:39:30.041 | INFO | "huge_shared" : " 27 GB", | 20260505 14:39:30.041 | INFO | "malloc" : " 14 GB", | 20260505 14:39:30.041 | INFO | "pinned" : " 29 GB" | 20260505 14:39:30.042 | INFO | } | 20260505 14:39:30.042 | INFO | Perf report for the last 5.12747 seconds: | 20260505 14:39:30.042 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:39:30.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:30.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:30.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:30.042 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 14:39:30.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:30.042 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:30.042 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 14:39:30.042 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:30.042 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:30.042 | INFO | Uploaded 36GiB to GPU in 2.11007s (17.061GiB/s) | 20260505 14:39:30.042 | INFO | Downloaded 34GiB to GPU in 2.17s (15.6682GiB/s) | 20260505 14:39:35.180 | INFO | Analyzing chunk frames = [3993088,3993600) | 20260505 14:39:35.180 | INFO | MemInfo: { | 20260505 14:39:35.180 | INFO | "gpu" : " 17 GB", | 20260505 14:39:35.180 | INFO | "huge" : "", | 20260505 14:39:35.180 | INFO | "huge_shared" : " 27 GB", | 20260505 14:39:35.180 | INFO | "malloc" : " 14 GB", | 20260505 14:39:35.180 | INFO | "pinned" : " 29 GB" | 20260505 14:39:35.180 | INFO | } | 20260505 14:39:35.180 | INFO | Perf report for the last 5.13841 seconds: | 20260505 14:39:35.180 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:39:35.180 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:35.180 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:35.180 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:35.180 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 14:39:35.180 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:35.180 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:35.180 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:39:35.180 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:35.180 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:35.180 | INFO | Uploaded 36GiB to GPU in 2.17892s (16.522GiB/s) | 20260505 14:39:35.180 | INFO | Downloaded 34GiB to GPU in 2.11812s (16.052GiB/s) | 20260505 14:39:35.315 | INFO | Starting a round of Baz IO | 20260505 14:39:38.555 | INFO | Populating RTMetrics report | 20260505 14:39:38.555 | INFO | RTMetrics report populated | 20260505 14:39:38.555 | INFO | Opening tmp RTMetrics json file | 20260505 14:39:38.596 | INFO | Serializing RTMetrics to json | 20260505 14:39:38.639 | INFO | Serialization done, swapping tmp file now... | 20260505 14:39:38.644 | INFO | RTMetrics done | 20260505 14:39:38.660 | INFO | Finished a round of Baz IO: Wrote 2.246 GiB in 3.344 seconds | 20260505 14:39:38.660 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.69% / 0.00% / 77.31% | 20260505 14:39:38.660 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.532619% / 0% / 0.000107288% | 20260505 14:39:38.660 | INFO | Baz IO overall: | 20260505 14:39:38.660 | INFO | BytesWritten : 2378787096461 | 20260505 14:39:38.660 | INFO | NumEvents : 1130806573853 | 20260505 14:39:40.325 | INFO | Analyzing chunk frames = [3993600,3994112) | 20260505 14:39:40.325 | INFO | MemInfo: { | 20260505 14:39:40.325 | INFO | "gpu" : " 17 GB", | 20260505 14:39:40.325 | INFO | "huge" : "", | 20260505 14:39:40.325 | INFO | "huge_shared" : " 27 GB", | 20260505 14:39:40.325 | INFO | "malloc" : " 14 GB", | 20260505 14:39:40.325 | INFO | "pinned" : " 29 GB" | 20260505 14:39:40.325 | INFO | } | 20260505 14:39:40.325 | INFO | Perf report for the last 5.14493 seconds: | 20260505 14:39:40.325 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:39:40.325 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:40.325 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:40.325 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:40.325 | INFO | ANALYSIS : 64.5%, 3.0, 100.3ms, 0 | 20260505 14:39:40.325 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:40.325 | INFO | RT_METRICS : 46.6%, 1.0, 24.4ms, 0 | 20260505 14:39:40.325 | INFO | PRE_HQ : 60.5%, 1.8, 57.1ms, 0 | 20260505 14:39:40.325 | INFO | BAZWRITER : 64.3%, 4.0, 131.4ms, 0 | 20260505 14:39:40.325 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 14:39:40.325 | INFO | Uploaded 36GiB to GPU in 2.18693s (16.4614GiB/s) | 20260505 14:39:40.325 | INFO | Downloaded 38GiB to GPU in 2.28408s (16.6369GiB/s) | 20260505 14:39:45.471 | INFO | Analyzing chunk frames = [3994112,3994624) | 20260505 14:39:45.471 | INFO | MemInfo: { | 20260505 14:39:45.471 | INFO | "gpu" : " 17 GB", | 20260505 14:39:45.471 | INFO | "huge" : "", | 20260505 14:39:45.471 | INFO | "huge_shared" : " 27 GB", | 20260505 14:39:45.471 | INFO | "malloc" : " 14 GB", | 20260505 14:39:45.471 | INFO | "pinned" : " 29 GB" | 20260505 14:39:45.471 | INFO | } | 20260505 14:39:45.471 | INFO | Perf report for the last 5.14586 seconds: | 20260505 14:39:45.471 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:39:45.471 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:45.471 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:45.471 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:45.471 | INFO | ANALYSIS : 63.3%, 3.0, 99.0ms, 0 | 20260505 14:39:45.471 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:45.471 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:45.471 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 14:39:45.471 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:45.471 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:45.471 | INFO | Uploaded 36GiB to GPU in 2.19232s (16.4209GiB/s) | 20260505 14:39:45.471 | INFO | Downloaded 34GiB to GPU in 2.03661s (16.6944GiB/s) | 20260505 14:39:50.611 | INFO | Analyzing chunk frames = [3994624,3995136) | 20260505 14:39:50.611 | INFO | MemInfo: { | 20260505 14:39:50.611 | INFO | "gpu" : " 17 GB", | 20260505 14:39:50.611 | INFO | "huge" : "", | 20260505 14:39:50.611 | INFO | "huge_shared" : " 27 GB", | 20260505 14:39:50.611 | INFO | "malloc" : " 14 GB", | 20260505 14:39:50.611 | INFO | "pinned" : " 29 GB" | 20260505 14:39:50.611 | INFO | } | 20260505 14:39:50.611 | INFO | Perf report for the last 5.14048 seconds: | 20260505 14:39:50.611 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:39:50.611 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:50.611 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:50.611 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:50.611 | INFO | ANALYSIS : 62.8%, 3.0, 97.8ms, 0 | 20260505 14:39:50.611 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:50.611 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:50.611 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:39:50.611 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:50.611 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:50.612 | INFO | Uploaded 36GiB to GPU in 2.17466s (16.5543GiB/s) | 20260505 14:39:50.612 | INFO | Downloaded 34GiB to GPU in 2.02893s (16.7576GiB/s) | 20260505 14:39:55.740 | INFO | Analyzing chunk frames = [3995136,3995648) | 20260505 14:39:55.740 | INFO | MemInfo: { | 20260505 14:39:55.740 | INFO | "gpu" : " 17 GB", | 20260505 14:39:55.740 | INFO | "huge" : "", | 20260505 14:39:55.740 | INFO | "huge_shared" : " 27 GB", | 20260505 14:39:55.740 | INFO | "malloc" : " 14 GB", | 20260505 14:39:55.740 | INFO | "pinned" : " 29 GB" | 20260505 14:39:55.740 | INFO | } | 20260505 14:39:55.740 | INFO | Perf report for the last 5.12901 seconds: | 20260505 14:39:55.740 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:39:55.740 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:55.740 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:55.740 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:55.740 | INFO | ANALYSIS : 62.8%, 3.0, 97.4ms, 0 | 20260505 14:39:55.740 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:55.740 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:55.741 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:39:55.741 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:55.741 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:39:55.741 | INFO | Uploaded 36GiB to GPU in 2.18379s (16.4851GiB/s) | 20260505 14:39:55.741 | INFO | Downloaded 34GiB to GPU in 2.12993s (15.9629GiB/s) | 20260505 14:40:00.878 | INFO | Analyzing chunk frames = [3995648,3996160) | 20260505 14:40:00.878 | INFO | MemInfo: { | 20260505 14:40:00.878 | INFO | "gpu" : " 17 GB", | 20260505 14:40:00.878 | INFO | "huge" : "", | 20260505 14:40:00.878 | INFO | "huge_shared" : " 27 GB", | 20260505 14:40:00.878 | INFO | "malloc" : " 14 GB", | 20260505 14:40:00.878 | INFO | "pinned" : " 29 GB" | 20260505 14:40:00.878 | INFO | } | 20260505 14:40:00.878 | INFO | Perf report for the last 5.13741 seconds: | 20260505 14:40:00.878 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:40:00.878 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:00.878 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:00.878 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:00.878 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 14:40:00.878 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:00.878 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:00.878 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:40:00.878 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:00.878 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:00.878 | INFO | Uploaded 36GiB to GPU in 2.16634s (16.6179GiB/s) | 20260505 14:40:00.878 | INFO | Downloaded 34GiB to GPU in 2.12838s (15.9746GiB/s) | 20260505 14:40:06.026 | INFO | Analyzing chunk frames = [3996160,3996672) | 20260505 14:40:06.026 | INFO | MemInfo: { | 20260505 14:40:06.026 | INFO | "gpu" : " 17 GB", | 20260505 14:40:06.026 | INFO | "huge" : "", | 20260505 14:40:06.026 | INFO | "huge_shared" : " 27 GB", | 20260505 14:40:06.026 | INFO | "malloc" : " 14 GB", | 20260505 14:40:06.026 | INFO | "pinned" : " 29 GB" | 20260505 14:40:06.026 | INFO | } | 20260505 14:40:06.026 | INFO | Perf report for the last 5.14814 seconds: | 20260505 14:40:06.026 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:40:06.026 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:06.026 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:06.026 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:06.026 | INFO | ANALYSIS : 62.9%, 3.0, 98.4ms, 0 | 20260505 14:40:06.026 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:06.026 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:06.026 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 14:40:06.026 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:06.026 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:06.026 | INFO | Uploaded 36GiB to GPU in 2.16618s (16.6191GiB/s) | 20260505 14:40:06.026 | INFO | Downloaded 34GiB to GPU in 2.10173s (16.1772GiB/s) | 20260505 14:40:11.170 | INFO | Analyzing chunk frames = [3996672,3997184) | 20260505 14:40:11.170 | INFO | MemInfo: { | 20260505 14:40:11.170 | INFO | "gpu" : " 17 GB", | 20260505 14:40:11.170 | INFO | "huge" : "", | 20260505 14:40:11.170 | INFO | "huge_shared" : " 27 GB", | 20260505 14:40:11.170 | INFO | "malloc" : " 14 GB", | 20260505 14:40:11.170 | INFO | "pinned" : " 29 GB" | 20260505 14:40:11.170 | INFO | } | 20260505 14:40:11.170 | INFO | Perf report for the last 5.14382 seconds: | 20260505 14:40:11.170 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:40:11.170 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:11.170 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:11.170 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:11.170 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 14:40:11.170 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:11.170 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:11.170 | INFO | PRE_HQ : 28.9%, 1.0, 15.1ms, 0 | 20260505 14:40:11.170 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:11.170 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:11.170 | INFO | Uploaded 36GiB to GPU in 2.11319s (17.0359GiB/s) | 20260505 14:40:11.170 | INFO | Downloaded 34GiB to GPU in 2.18304s (15.5746GiB/s) | 20260505 14:40:16.298 | INFO | Analyzing chunk frames = [3997184,3997696) | 20260505 14:40:16.298 | INFO | MemInfo: { | 20260505 14:40:16.298 | INFO | "gpu" : " 17 GB", | 20260505 14:40:16.298 | INFO | "huge" : "", | 20260505 14:40:16.298 | INFO | "huge_shared" : " 27 GB", | 20260505 14:40:16.298 | INFO | "malloc" : " 14 GB", | 20260505 14:40:16.298 | INFO | "pinned" : " 29 GB" | 20260505 14:40:16.299 | INFO | } | 20260505 14:40:16.299 | INFO | Perf report for the last 5.12872 seconds: | 20260505 14:40:16.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:40:16.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:16.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:16.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:16.299 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 14:40:16.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:16.299 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:16.299 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 14:40:16.299 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:16.299 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:16.299 | INFO | Uploaded 36GiB to GPU in 2.17101s (16.5822GiB/s) | 20260505 14:40:16.299 | INFO | Downloaded 34GiB to GPU in 2.03612s (16.6984GiB/s) | 20260505 14:40:16.299 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 14:40:16.299 | INFO | Total mem usage: 15.3 GiB | 20260505 14:40:16.299 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 14:40:16.299 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.82%) | 20260505 14:40:16.299 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 14:40:16.299 | INFO | High Water Breakdown: | 20260505 14:40:16.299 | INFO | 61% : 8.68 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 14:40:16.299 | INFO | 38.8% : 5.52 GiB ( 83.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 14:40:16.299 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 14:40:16.299 | INFO | ------------------------------------------ | 20260505 14:40:16.299 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:40:16.299 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 14:40:16.299 | INFO | Total mem usage: 26.5 GiB | 20260505 14:40:16.299 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 14:40:16.299 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 14:40:16.299 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 14:40:16.299 | INFO | High Water Breakdown: | 20260505 14:40:16.299 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:40:16.299 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:40:16.299 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:40:16.299 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:40:16.299 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:40:16.299 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:40:16.299 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 14:40:16.299 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 14:40:16.299 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:40:16.299 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 14:40:16.299 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 14:40:16.300 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 14:40:16.300 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 14:40:16.300 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 14:40:16.300 | INFO | ------------------------------------------ | 20260505 14:40:16.300 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:40:16.300 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 14:40:16.300 | INFO | Total mem usage: 25.6 GiB | 20260505 14:40:16.300 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 14:40:16.300 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 14:40:16.300 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 14:40:16.300 | INFO | High Water Breakdown: | 20260505 14:40:16.300 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 14:40:16.300 | INFO | ------------------------------------------ | 20260505 14:40:16.300 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:40:16.300 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 14:40:16.300 | INFO | Total mem usage: 15.7 GiB | 20260505 14:40:16.300 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 14:40:16.300 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 14:40:16.300 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 14:40:16.300 | INFO | High Water Breakdown: | 20260505 14:40:16.300 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 14:40:16.300 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 14:40:16.300 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 14:40:16.300 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 14:40:16.300 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 14:40:16.300 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 14:40:16.300 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 14:40:16.300 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 14:40:16.300 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 14:40:16.300 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:40:16.300 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 14:40:16.300 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:40:16.300 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 14:40:16.300 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 14:40:16.300 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 14:40:16.300 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 14:40:16.300 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:40:16.300 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 14:40:16.300 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:40:16.300 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:40:16.300 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:40:16.300 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 14:40:16.300 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 14:40:16.300 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:40:16.300 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 14:40:16.300 | INFO | ------------------------------------------ | 20260505 14:40:16.300 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:40:16.300 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 14:40:16.300 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:40:16.300 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:40:16.300 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.62e+01, 1.36e+01 / 2.39e+00 | 20260505 14:40:16.300 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 1.02e-01, 1.07e-02 / 5.48e-03 | 20260505 14:40:16.300 | INFO | PerfReport: 11.74%, Baseline -- 3.74e+00 / 4.47e+00, 5.78e-01 / 2.03e-01 | 20260505 14:40:16.300 | INFO | PerfReport: 3.29%, Histogram -- 1.05e+00 / 1.49e+00, 1.97e-01 / 1.07e-01 | 20260505 14:40:16.300 | INFO | PerfReport: 15.95%, DME -- 4.06e+01 / 4.54e+01, 4.61e+00 / 2.43e+00 | 20260505 14:40:16.300 | INFO | PerfReport: 56.61%, FrameLabeling -- 1.80e+01 / 1.99e+01, 1.50e+00 / 1.44e+00 | 20260505 14:40:16.300 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.35e+00 / 2.94e+00, 4.48e-01 / 1.47e-01 | 20260505 14:40:16.300 | INFO | PerfReport: 3.49%, Metrics -- 1.11e+00 / 1.43e+00, 3.11e-01 / 1.46e-01 | 20260505 14:40:16.300 | INFO | PerfReport: 1.23%, PulseSerializing -- 3.91e-01 / 4.40e-01, 6.95e-02 / 3.77e-02 | 20260505 14:40:16.300 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 14:40:16.300 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:40:16.300 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:40:16.301 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260505 14:40:16.301 | INFO | PerfReport: 22.48%, Upload -- 2.26e+01 / 0.00e+00, 8.00e+00 / 0.00e+00 | 20260505 14:40:16.301 | INFO | PerfReport: 21.92%, Download -- 2.21e+01 / 0.00e+00, 8.05e+00 / 0.00e+00 | 20260505 14:40:16.301 | INFO | PerfReport: 31.61%, Compute -- 3.18e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 14:40:16.301 | INFO | PerfReport: 2.22%, LockUpload -- 2.24e+00 / 0.00e+00, 7.92e+00 / 0.00e+00 | 20260505 14:40:16.301 | INFO | PerfReport: 1.48%, LockDownload -- 1.49e+00 / 0.00e+00, 3.10e+00 / 0.00e+00 | 20260505 14:40:16.301 | INFO | PerfReport: 20.29%, LockCompute -- 2.04e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 14:40:16.301 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.91e-02 / 0.00e+00 | 20260505 14:40:16.436 | INFO | Starting a round of Baz IO | 20260505 14:40:19.655 | INFO | Populating RTMetrics report | 20260505 14:40:19.656 | INFO | RTMetrics report populated | 20260505 14:40:19.656 | INFO | Opening tmp RTMetrics json file | 20260505 14:40:19.673 | INFO | Serializing RTMetrics to json | 20260505 14:40:19.696 | INFO | Serialization done, swapping tmp file now... | 20260505 14:40:19.700 | INFO | RTMetrics done | 20260505 14:40:19.762 | INFO | Finished a round of Baz IO: Wrote 2.245 GiB in 3.325 seconds | 20260505 14:40:19.762 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.69% / 0.00% / 77.31% | 20260505 14:40:19.762 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.530354% / 3.97364e-06% / 8.34465e-05% | 20260505 14:40:19.762 | INFO | Baz IO overall: | 20260505 14:40:19.762 | INFO | BytesWritten : 2381197969221 | 20260505 14:40:19.762 | INFO | NumEvents : 1131942502896 | 20260505 14:40:21.434 | INFO | Analyzing chunk frames = [3997696,3998208) | 20260505 14:40:21.434 | INFO | MemInfo: { | 20260505 14:40:21.434 | INFO | "gpu" : " 17 GB", | 20260505 14:40:21.434 | INFO | "huge" : "", | 20260505 14:40:21.434 | INFO | "huge_shared" : " 27 GB", | 20260505 14:40:21.434 | INFO | "malloc" : " 14 GB", | 20260505 14:40:21.434 | INFO | "pinned" : " 29 GB" | 20260505 14:40:21.434 | INFO | } | 20260505 14:40:21.434 | INFO | Perf report for the last 5.13553 seconds: | 20260505 14:40:21.434 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:40:21.434 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:21.434 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:21.434 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:21.434 | INFO | ANALYSIS : 64.3%, 3.0, 99.6ms, 0 | 20260505 14:40:21.434 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:21.434 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260505 14:40:21.434 | INFO | PRE_HQ : 60.6%, 1.8, 57.2ms, 0 | 20260505 14:40:21.434 | INFO | BAZWRITER : 64.1%, 4.0, 131.1ms, 0 | 20260505 14:40:21.434 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 14:40:21.435 | INFO | Uploaded 36GiB to GPU in 2.16017s (16.6654GiB/s) | 20260505 14:40:21.435 | INFO | Downloaded 38GiB to GPU in 2.27661s (16.6915GiB/s) | 20260505 14:40:26.580 | INFO | Analyzing chunk frames = [3998208,3998720) | 20260505 14:40:26.580 | INFO | MemInfo: { | 20260505 14:40:26.580 | INFO | "gpu" : " 17 GB", | 20260505 14:40:26.580 | INFO | "huge" : "", | 20260505 14:40:26.580 | INFO | "huge_shared" : " 27 GB", | 20260505 14:40:26.580 | INFO | "malloc" : " 14 GB", | 20260505 14:40:26.580 | INFO | "pinned" : " 29 GB" | 20260505 14:40:26.580 | INFO | } | 20260505 14:40:26.580 | INFO | Perf report for the last 5.14584 seconds: | 20260505 14:40:26.580 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:40:26.580 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:26.580 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:26.580 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:26.580 | INFO | ANALYSIS : 63.4%, 3.0, 99.2ms, 0 | 20260505 14:40:26.580 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:26.580 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:26.580 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 14:40:26.580 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:26.580 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:26.580 | INFO | Uploaded 36GiB to GPU in 2.20246s (16.3454GiB/s) | 20260505 14:40:26.580 | INFO | Downloaded 34GiB to GPU in 2.04251s (16.6462GiB/s) | 20260505 14:40:31.723 | INFO | Analyzing chunk frames = [3998720,3999232) | 20260505 14:40:31.723 | INFO | MemInfo: { | 20260505 14:40:31.723 | INFO | "gpu" : " 17 GB", | 20260505 14:40:31.723 | INFO | "huge" : "", | 20260505 14:40:31.723 | INFO | "huge_shared" : " 27 GB", | 20260505 14:40:31.723 | INFO | "malloc" : " 14 GB", | 20260505 14:40:31.723 | INFO | "pinned" : " 29 GB" | 20260505 14:40:31.723 | INFO | } | 20260505 14:40:31.723 | INFO | Perf report for the last 5.14324 seconds: | 20260505 14:40:31.723 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:40:31.723 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:31.723 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:31.723 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:31.723 | INFO | ANALYSIS : 63.0%, 3.0, 98.1ms, 0 | 20260505 14:40:31.723 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:31.723 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:31.723 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:40:31.723 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:31.723 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:31.724 | INFO | Uploaded 36GiB to GPU in 2.18699s (16.461GiB/s) | 20260505 14:40:31.724 | INFO | Downloaded 34GiB to GPU in 2.04986s (16.5865GiB/s) | 20260505 14:40:36.859 | INFO | Analyzing chunk frames = [3999232,3999744) | 20260505 14:40:36.859 | INFO | MemInfo: { | 20260505 14:40:36.859 | INFO | "gpu" : " 17 GB", | 20260505 14:40:36.859 | INFO | "huge" : "", | 20260505 14:40:36.859 | INFO | "huge_shared" : " 27 GB", | 20260505 14:40:36.859 | INFO | "malloc" : " 14 GB", | 20260505 14:40:36.859 | INFO | "pinned" : " 29 GB" | 20260505 14:40:36.859 | INFO | } | 20260505 14:40:36.859 | INFO | Perf report for the last 5.13619 seconds: | 20260505 14:40:36.859 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:40:36.859 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:36.859 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:36.859 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:36.859 | INFO | ANALYSIS : 62.6%, 3.0, 97.2ms, 0 | 20260505 14:40:36.859 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:36.859 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:36.860 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:40:36.860 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:36.860 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:36.860 | INFO | Uploaded 36GiB to GPU in 2.18403s (16.4833GiB/s) | 20260505 14:40:36.860 | INFO | Downloaded 34GiB to GPU in 2.13484s (15.9262GiB/s) | 20260505 14:40:41.992 | INFO | Analyzing chunk frames = [3999744,4000256) | 20260505 14:40:41.992 | INFO | MemInfo: { | 20260505 14:40:41.993 | INFO | "gpu" : " 17 GB", | 20260505 14:40:41.993 | INFO | "huge" : "", | 20260505 14:40:41.993 | INFO | "huge_shared" : " 27 GB", | 20260505 14:40:41.993 | INFO | "malloc" : " 14 GB", | 20260505 14:40:41.993 | INFO | "pinned" : " 29 GB" | 20260505 14:40:41.993 | INFO | } | 20260505 14:40:41.993 | INFO | Perf report for the last 5.13316 seconds: | 20260505 14:40:41.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:40:41.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:41.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:41.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:41.993 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260505 14:40:41.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:41.993 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:41.993 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 14:40:41.993 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:41.993 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:41.993 | INFO | Uploaded 36GiB to GPU in 2.14152s (16.8105GiB/s) | 20260505 14:40:41.993 | INFO | Downloaded 34GiB to GPU in 2.12359s (16.0106GiB/s) | 20260505 14:40:47.131 | INFO | Analyzing chunk frames = [4000256,4000768) | 20260505 14:40:47.131 | INFO | MemInfo: { | 20260505 14:40:47.131 | INFO | "gpu" : " 17 GB", | 20260505 14:40:47.131 | INFO | "huge" : "", | 20260505 14:40:47.131 | INFO | "huge_shared" : " 27 GB", | 20260505 14:40:47.131 | INFO | "malloc" : " 14 GB", | 20260505 14:40:47.131 | INFO | "pinned" : " 29 GB" | 20260505 14:40:47.131 | INFO | } | 20260505 14:40:47.131 | INFO | Perf report for the last 5.13829 seconds: | 20260505 14:40:47.131 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:40:47.131 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:47.131 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:47.131 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:47.131 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 14:40:47.131 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:47.131 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:47.131 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:40:47.131 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:47.131 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:47.131 | INFO | Uploaded 36GiB to GPU in 2.17238s (16.5717GiB/s) | 20260505 14:40:47.131 | INFO | Downloaded 34GiB to GPU in 2.11304s (16.0906GiB/s) | 20260505 14:40:52.277 | INFO | Analyzing chunk frames = [4000768,4001280) | 20260505 14:40:52.277 | INFO | MemInfo: { | 20260505 14:40:52.277 | INFO | "gpu" : " 17 GB", | 20260505 14:40:52.277 | INFO | "huge" : "", | 20260505 14:40:52.277 | INFO | "huge_shared" : " 27 GB", | 20260505 14:40:52.277 | INFO | "malloc" : " 14 GB", | 20260505 14:40:52.277 | INFO | "pinned" : " 29 GB" | 20260505 14:40:52.277 | INFO | } | 20260505 14:40:52.277 | INFO | Perf report for the last 5.14596 seconds: | 20260505 14:40:52.277 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:40:52.277 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:52.277 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:52.277 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:52.277 | INFO | ANALYSIS : 63.6%, 3.0, 99.5ms, 0 | 20260505 14:40:52.277 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:52.277 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:52.277 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260505 14:40:52.277 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:52.277 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:52.277 | INFO | Uploaded 36GiB to GPU in 2.14656s (16.771GiB/s) | 20260505 14:40:52.277 | INFO | Downloaded 34GiB to GPU in 2.19375s (15.4986GiB/s) | 20260505 14:40:57.423 | INFO | Analyzing chunk frames = [4001280,4001792) | 20260505 14:40:57.423 | INFO | MemInfo: { | 20260505 14:40:57.423 | INFO | "gpu" : " 17 GB", | 20260505 14:40:57.423 | INFO | "huge" : "", | 20260505 14:40:57.423 | INFO | "huge_shared" : " 27 GB", | 20260505 14:40:57.423 | INFO | "malloc" : " 14 GB", | 20260505 14:40:57.423 | INFO | "pinned" : " 29 GB" | 20260505 14:40:57.423 | INFO | } | 20260505 14:40:57.423 | INFO | Perf report for the last 5.14611 seconds: | 20260505 14:40:57.423 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:40:57.423 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:57.423 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:57.423 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:57.423 | INFO | ANALYSIS : 63.5%, 3.0, 99.3ms, 0 | 20260505 14:40:57.423 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:57.423 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:57.423 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 14:40:57.423 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:57.423 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:40:57.423 | INFO | Uploaded 36GiB to GPU in 2.22083s (16.2102GiB/s) | 20260505 14:40:57.423 | INFO | Downloaded 34GiB to GPU in 2.09597s (16.2216GiB/s) | 20260505 14:40:57.553 | INFO | Starting a round of Baz IO | 20260505 14:41:00.769 | INFO | Populating RTMetrics report | 20260505 14:41:00.769 | INFO | RTMetrics report populated | 20260505 14:41:00.769 | INFO | Opening tmp RTMetrics json file | 20260505 14:41:00.788 | INFO | Serializing RTMetrics to json | 20260505 14:41:00.795 | INFO | Serialization done, swapping tmp file now... | 20260505 14:41:00.812 | INFO | RTMetrics done | 20260505 14:41:00.874 | INFO | Finished a round of Baz IO: Wrote 2.246 GiB in 3.321 seconds | 20260505 14:41:00.874 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.68% / 0.00% / 77.31% | 20260505 14:41:00.874 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.529238% / 3.97364e-06% / 7.94729e-05% | 20260505 14:41:00.874 | INFO | Baz IO overall: | 20260505 14:41:00.875 | INFO | BytesWritten : 2383609110850 | 20260505 14:41:00.875 | INFO | NumEvents : 1133078373010 | 20260505 14:41:02.552 | INFO | Analyzing chunk frames = [4001792,4002304) | 20260505 14:41:02.552 | INFO | MemInfo: { | 20260505 14:41:02.552 | INFO | "gpu" : " 17 GB", | 20260505 14:41:02.552 | INFO | "huge" : "", | 20260505 14:41:02.552 | INFO | "huge_shared" : " 27 GB", | 20260505 14:41:02.552 | INFO | "malloc" : " 14 GB", | 20260505 14:41:02.552 | INFO | "pinned" : " 29 GB" | 20260505 14:41:02.552 | INFO | } | 20260505 14:41:02.552 | INFO | Perf report for the last 5.12874 seconds: | 20260505 14:41:02.552 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:41:02.552 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:02.552 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:02.552 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:02.552 | INFO | ANALYSIS : 64.1%, 3.0, 99.5ms, 0 | 20260505 14:41:02.552 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:02.552 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260505 14:41:02.552 | INFO | PRE_HQ : 61.0%, 1.8, 57.3ms, 0 | 20260505 14:41:02.552 | INFO | BAZWRITER : 64.1%, 4.0, 131.0ms, 0 | 20260505 14:41:02.552 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260505 14:41:02.552 | INFO | Uploaded 36GiB to GPU in 2.18626s (16.4665GiB/s) | 20260505 14:41:02.552 | INFO | Downloaded 38GiB to GPU in 2.2677s (16.7571GiB/s) | 20260505 14:41:07.691 | INFO | Analyzing chunk frames = [4002304,4002816) | 20260505 14:41:07.691 | INFO | MemInfo: { | 20260505 14:41:07.691 | INFO | "gpu" : " 17 GB", | 20260505 14:41:07.691 | INFO | "huge" : "", | 20260505 14:41:07.691 | INFO | "huge_shared" : " 27 GB", | 20260505 14:41:07.691 | INFO | "malloc" : " 14 GB", | 20260505 14:41:07.691 | INFO | "pinned" : " 29 GB" | 20260505 14:41:07.691 | INFO | } | 20260505 14:41:07.691 | INFO | Perf report for the last 5.13929 seconds: | 20260505 14:41:07.691 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:41:07.691 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:07.691 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:07.691 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:07.691 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 14:41:07.691 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:07.691 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:07.691 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:41:07.691 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:07.691 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:07.691 | INFO | Uploaded 36GiB to GPU in 2.19676s (16.3878GiB/s) | 20260505 14:41:07.691 | INFO | Downloaded 34GiB to GPU in 2.02896s (16.7574GiB/s) | 20260505 14:41:12.836 | INFO | Analyzing chunk frames = [4002816,4003328) | 20260505 14:41:12.836 | INFO | MemInfo: { | 20260505 14:41:12.836 | INFO | "gpu" : " 17 GB", | 20260505 14:41:12.836 | INFO | "huge" : "", | 20260505 14:41:12.836 | INFO | "huge_shared" : " 27 GB", | 20260505 14:41:12.836 | INFO | "malloc" : " 14 GB", | 20260505 14:41:12.836 | INFO | "pinned" : " 29 GB" | 20260505 14:41:12.836 | INFO | } | 20260505 14:41:12.836 | INFO | Perf report for the last 5.14532 seconds: | 20260505 14:41:12.836 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:41:12.836 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:12.836 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:12.836 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:12.836 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 14:41:12.836 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:12.836 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:12.836 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 14:41:12.836 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:12.836 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:12.837 | INFO | Uploaded 36GiB to GPU in 2.23897s (16.0788GiB/s) | 20260505 14:41:12.837 | INFO | Downloaded 34GiB to GPU in 2.09567s (16.2239GiB/s) | 20260505 14:41:17.975 | INFO | Analyzing chunk frames = [4003328,4003840) | 20260505 14:41:17.975 | INFO | MemInfo: { | 20260505 14:41:17.975 | INFO | "gpu" : " 17 GB", | 20260505 14:41:17.975 | INFO | "huge" : "", | 20260505 14:41:17.975 | INFO | "huge_shared" : " 27 GB", | 20260505 14:41:17.976 | INFO | "malloc" : " 14 GB", | 20260505 14:41:17.976 | INFO | "pinned" : " 29 GB" | 20260505 14:41:17.976 | INFO | } | 20260505 14:41:17.976 | INFO | Perf report for the last 5.13926 seconds: | 20260505 14:41:17.976 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:41:17.976 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:17.976 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:17.976 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:17.976 | INFO | ANALYSIS : 62.7%, 3.0, 97.4ms, 0 | 20260505 14:41:17.976 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:17.976 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:17.976 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:41:17.976 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:17.976 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:17.976 | INFO | Uploaded 36GiB to GPU in 2.19285s (16.417GiB/s) | 20260505 14:41:17.976 | INFO | Downloaded 34GiB to GPU in 2.13128s (15.9529GiB/s) | 20260505 14:41:23.118 | INFO | Analyzing chunk frames = [4003840,4004352) | 20260505 14:41:23.118 | INFO | MemInfo: { | 20260505 14:41:23.118 | INFO | "gpu" : " 17 GB", | 20260505 14:41:23.118 | INFO | "huge" : "", | 20260505 14:41:23.118 | INFO | "huge_shared" : " 27 GB", | 20260505 14:41:23.118 | INFO | "malloc" : " 14 GB", | 20260505 14:41:23.118 | INFO | "pinned" : " 29 GB" | 20260505 14:41:23.118 | INFO | } | 20260505 14:41:23.118 | INFO | Perf report for the last 5.14216 seconds: | 20260505 14:41:23.118 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:41:23.118 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:23.118 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:23.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:23.118 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 14:41:23.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:23.118 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:23.118 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:41:23.118 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:23.118 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:23.118 | INFO | Uploaded 36GiB to GPU in 2.17204s (16.5742GiB/s) | 20260505 14:41:23.118 | INFO | Downloaded 34GiB to GPU in 2.12855s (15.9733GiB/s) | 20260505 14:41:28.251 | INFO | Analyzing chunk frames = [4004352,4004864) | 20260505 14:41:28.251 | INFO | MemInfo: { | 20260505 14:41:28.251 | INFO | "gpu" : " 17 GB", | 20260505 14:41:28.251 | INFO | "huge" : "", | 20260505 14:41:28.251 | INFO | "huge_shared" : " 27 GB", | 20260505 14:41:28.251 | INFO | "malloc" : " 14 GB", | 20260505 14:41:28.251 | INFO | "pinned" : " 29 GB" | 20260505 14:41:28.251 | INFO | } | 20260505 14:41:28.251 | INFO | Perf report for the last 5.13317 seconds: | 20260505 14:41:28.251 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:41:28.251 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:28.251 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:28.251 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:28.251 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260505 14:41:28.251 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:28.251 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:28.251 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 14:41:28.251 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:28.251 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:28.251 | INFO | Uploaded 36GiB to GPU in 2.16177s (16.653GiB/s) | 20260505 14:41:28.251 | INFO | Downloaded 34GiB to GPU in 2.03829s (16.6806GiB/s) | 20260505 14:41:33.392 | INFO | Analyzing chunk frames = [4004864,4005376) | 20260505 14:41:33.392 | INFO | MemInfo: { | 20260505 14:41:33.392 | INFO | "gpu" : " 17 GB", | 20260505 14:41:33.392 | INFO | "huge" : "", | 20260505 14:41:33.392 | INFO | "huge_shared" : " 27 GB", | 20260505 14:41:33.393 | INFO | "malloc" : " 14 GB", | 20260505 14:41:33.393 | INFO | "pinned" : " 29 GB" | 20260505 14:41:33.393 | INFO | } | 20260505 14:41:33.393 | INFO | Perf report for the last 5.14164 seconds: | 20260505 14:41:33.393 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:41:33.393 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:33.393 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:33.393 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:33.393 | INFO | ANALYSIS : 63.9%, 3.0, 99.8ms, 0 | 20260505 14:41:33.393 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:33.393 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:33.393 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 14:41:33.393 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:33.393 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:33.393 | INFO | Uploaded 36GiB to GPU in 2.13609s (16.8533GiB/s) | 20260505 14:41:33.393 | INFO | Downloaded 34GiB to GPU in 2.18695s (15.5468GiB/s) | 20260505 14:41:38.531 | INFO | Analyzing chunk frames = [4005376,4005888) | 20260505 14:41:38.531 | INFO | MemInfo: { | 20260505 14:41:38.531 | INFO | "gpu" : " 17 GB", | 20260505 14:41:38.531 | INFO | "huge" : "", | 20260505 14:41:38.531 | INFO | "huge_shared" : " 27 GB", | 20260505 14:41:38.531 | INFO | "malloc" : " 14 GB", | 20260505 14:41:38.531 | INFO | "pinned" : " 29 GB" | 20260505 14:41:38.531 | INFO | } | 20260505 14:41:38.531 | INFO | Perf report for the last 5.13863 seconds: | 20260505 14:41:38.531 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:41:38.531 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:38.531 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:38.531 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:38.531 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 14:41:38.531 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:38.531 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:38.531 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:41:38.531 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:38.531 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:38.532 | INFO | Uploaded 36GiB to GPU in 2.21994s (16.2167GiB/s) | 20260505 14:41:38.532 | INFO | Downloaded 34GiB to GPU in 2.10418s (16.1583GiB/s) | 20260505 14:41:38.532 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 14:41:38.532 | INFO | Total mem usage: 15.3 GiB | 20260505 14:41:38.532 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 14:41:38.532 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.85%) | 20260505 14:41:38.532 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 14:41:38.532 | INFO | High Water Breakdown: | 20260505 14:41:38.532 | INFO | 61% : 8.67 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 14:41:38.532 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 14:41:38.532 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 14:41:38.532 | INFO | ------------------------------------------ | 20260505 14:41:38.532 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:41:38.532 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 14:41:38.532 | INFO | Total mem usage: 26.5 GiB | 20260505 14:41:38.532 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 14:41:38.532 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 14:41:38.532 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 14:41:38.532 | INFO | High Water Breakdown: | 20260505 14:41:38.532 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:41:38.532 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:41:38.532 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:41:38.532 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:41:38.532 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:41:38.532 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:41:38.532 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 14:41:38.532 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 14:41:38.532 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:41:38.532 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 14:41:38.532 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 14:41:38.532 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 14:41:38.532 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 14:41:38.532 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 14:41:38.532 | INFO | ------------------------------------------ | 20260505 14:41:38.532 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:41:38.532 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 14:41:38.532 | INFO | Total mem usage: 25.6 GiB | 20260505 14:41:38.532 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 14:41:38.532 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 14:41:38.532 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 14:41:38.532 | INFO | High Water Breakdown: | 20260505 14:41:38.532 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 14:41:38.532 | INFO | ------------------------------------------ | 20260505 14:41:38.532 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:41:38.532 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 14:41:38.532 | INFO | Total mem usage: 15.7 GiB | 20260505 14:41:38.532 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 14:41:38.532 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 14:41:38.532 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 14:41:38.532 | INFO | High Water Breakdown: | 20260505 14:41:38.532 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 14:41:38.532 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 14:41:38.532 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 14:41:38.532 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 14:41:38.532 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 14:41:38.532 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 14:41:38.532 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 14:41:38.532 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 14:41:38.532 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 14:41:38.532 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:41:38.532 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 14:41:38.532 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:41:38.532 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 14:41:38.532 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 14:41:38.532 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 14:41:38.532 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 14:41:38.532 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:41:38.532 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 14:41:38.532 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:41:38.532 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:41:38.532 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:41:38.532 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 14:41:38.532 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 14:41:38.532 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:41:38.532 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 14:41:38.532 | INFO | ------------------------------------------ | 20260505 14:41:38.532 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:41:38.532 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 14:41:38.532 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:41:38.532 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:41:38.532 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.63e+01, 1.37e+01 / 2.09e+00 | 20260505 14:41:38.532 | INFO | PerfReport: 0.29%, AggregateStats -- 9.30e-02 / 1.00e-01, 1.10e-02 / 4.31e-03 | 20260505 14:41:38.532 | INFO | PerfReport: 11.77%, Baseline -- 3.74e+00 / 4.46e+00, 5.75e-01 / 1.98e-01 | 20260505 14:41:38.533 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.49e+00, 1.93e-01 / 1.01e-01 | 20260505 14:41:38.533 | INFO | PerfReport: 16.00%, DME -- 4.07e+01 / 4.56e+01, 4.63e+00 / 2.33e+00 | 20260505 14:41:38.533 | INFO | PerfReport: 56.49%, FrameLabeling -- 1.80e+01 / 1.99e+01, 1.42e+00 / 1.75e+00 | 20260505 14:41:38.533 | INFO | PerfReport: 7.43%, PulseAccumulating -- 2.36e+00 / 2.91e+00, 4.51e-01 / 2.02e-01 | 20260505 14:41:38.533 | INFO | PerfReport: 3.51%, Metrics -- 1.12e+00 / 1.43e+00, 3.10e-01 / 1.74e-01 | 20260505 14:41:38.533 | INFO | PerfReport: 1.23%, PulseSerializing -- 3.91e-01 / 4.37e-01, 6.94e-02 / 2.98e-02 | 20260505 14:41:38.533 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 14:41:38.533 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:41:38.533 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:41:38.533 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.58e+01 / 0.00e+00 | 20260505 14:41:38.533 | INFO | PerfReport: 22.56%, Upload -- 2.27e+01 / 0.00e+00, 7.97e+00 / 0.00e+00 | 20260505 14:41:38.533 | INFO | PerfReport: 21.98%, Download -- 2.21e+01 / 0.00e+00, 8.04e+00 / 0.00e+00 | 20260505 14:41:38.533 | INFO | PerfReport: 31.57%, Compute -- 3.18e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 14:41:38.533 | INFO | PerfReport: 2.32%, LockUpload -- 2.34e+00 / 0.00e+00, 8.02e+00 / 0.00e+00 | 20260505 14:41:38.533 | INFO | PerfReport: 1.41%, LockDownload -- 1.42e+00 / 0.00e+00, 2.98e+00 / 0.00e+00 | 20260505 14:41:38.533 | INFO | PerfReport: 20.14%, LockCompute -- 2.03e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 14:41:38.533 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.66e-02 / 0.00e+00 | 20260505 14:41:38.663 | INFO | Starting a round of Baz IO | 20260505 14:41:41.874 | INFO | Populating RTMetrics report | 20260505 14:41:41.874 | INFO | RTMetrics report populated | 20260505 14:41:41.874 | INFO | Opening tmp RTMetrics json file | 20260505 14:41:41.890 | INFO | Serializing RTMetrics to json | 20260505 14:41:41.902 | INFO | Serialization done, swapping tmp file now... | 20260505 14:41:41.917 | INFO | RTMetrics done | 20260505 14:41:41.970 | INFO | Finished a round of Baz IO: Wrote 2.248 GiB in 3.306 seconds | 20260505 14:41:41.970 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.68% / 0.00% / 77.32% | 20260505 14:41:41.970 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.529059% / 7.94729e-06% / 5.5631e-05% | 20260505 14:41:41.970 | INFO | Baz IO overall: | 20260505 14:41:41.970 | INFO | BytesWritten : 2386022349556 | 20260505 14:41:41.970 | INFO | NumEvents : 1134215754933 | 20260505 14:41:43.671 | INFO | Analyzing chunk frames = [4005888,4006400) | 20260505 14:41:43.671 | INFO | MemInfo: { | 20260505 14:41:43.671 | INFO | "gpu" : " 17 GB", | 20260505 14:41:43.671 | INFO | "huge" : "", | 20260505 14:41:43.671 | INFO | "huge_shared" : " 27 GB", | 20260505 14:41:43.671 | INFO | "malloc" : " 14 GB", | 20260505 14:41:43.671 | INFO | "pinned" : " 29 GB" | 20260505 14:41:43.671 | INFO | } | 20260505 14:41:43.671 | INFO | Perf report for the last 5.1397 seconds: | 20260505 14:41:43.671 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:41:43.671 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:43.671 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:43.671 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:43.671 | INFO | ANALYSIS : 63.9%, 3.0, 99.3ms, 0 | 20260505 14:41:43.671 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:43.671 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260505 14:41:43.671 | INFO | PRE_HQ : 60.9%, 1.8, 57.1ms, 0 | 20260505 14:41:43.671 | INFO | BAZWRITER : 63.6%, 4.0, 130.7ms, 0 | 20260505 14:41:43.671 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260505 14:41:43.671 | INFO | Uploaded 36GiB to GPU in 2.16498s (16.6283GiB/s) | 20260505 14:41:43.671 | INFO | Downloaded 38GiB to GPU in 2.25587s (16.8449GiB/s) | 20260505 14:41:48.806 | INFO | Analyzing chunk frames = [4006400,4006912) | 20260505 14:41:48.806 | INFO | MemInfo: { | 20260505 14:41:48.806 | INFO | "gpu" : " 17 GB", | 20260505 14:41:48.806 | INFO | "huge" : "", | 20260505 14:41:48.806 | INFO | "huge_shared" : " 27 GB", | 20260505 14:41:48.806 | INFO | "malloc" : " 14 GB", | 20260505 14:41:48.806 | INFO | "pinned" : " 29 GB" | 20260505 14:41:48.806 | INFO | } | 20260505 14:41:48.806 | INFO | Perf report for the last 5.13502 seconds: | 20260505 14:41:48.806 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:41:48.806 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:48.806 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:48.806 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:48.806 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 14:41:48.806 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:48.806 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:48.806 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:41:48.806 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:48.806 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:48.806 | INFO | Uploaded 36GiB to GPU in 2.19789s (16.3794GiB/s) | 20260505 14:41:48.806 | INFO | Downloaded 34GiB to GPU in 2.04606s (16.6173GiB/s) | 20260505 14:41:53.942 | INFO | Analyzing chunk frames = [4006912,4007424) | 20260505 14:41:53.942 | INFO | MemInfo: { | 20260505 14:41:53.942 | INFO | "gpu" : " 17 GB", | 20260505 14:41:53.942 | INFO | "huge" : "", | 20260505 14:41:53.942 | INFO | "huge_shared" : " 27 GB", | 20260505 14:41:53.942 | INFO | "malloc" : " 14 GB", | 20260505 14:41:53.942 | INFO | "pinned" : " 29 GB" | 20260505 14:41:53.942 | INFO | } | 20260505 14:41:53.942 | INFO | Perf report for the last 5.13617 seconds: | 20260505 14:41:53.942 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:41:53.942 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:53.942 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:53.942 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:53.942 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 14:41:53.942 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:53.942 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:53.942 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:41:53.942 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:53.942 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:53.942 | INFO | Uploaded 36GiB to GPU in 2.19704s (16.3857GiB/s) | 20260505 14:41:53.942 | INFO | Downloaded 34GiB to GPU in 2.12278s (16.0167GiB/s) | 20260505 14:41:59.090 | INFO | Analyzing chunk frames = [4007424,4007936) | 20260505 14:41:59.090 | INFO | MemInfo: { | 20260505 14:41:59.090 | INFO | "gpu" : " 17 GB", | 20260505 14:41:59.090 | INFO | "huge" : "", | 20260505 14:41:59.090 | INFO | "huge_shared" : " 27 GB", | 20260505 14:41:59.090 | INFO | "malloc" : " 14 GB", | 20260505 14:41:59.090 | INFO | "pinned" : " 29 GB" | 20260505 14:41:59.090 | INFO | } | 20260505 14:41:59.090 | INFO | Perf report for the last 5.148 seconds: | 20260505 14:41:59.090 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:41:59.090 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:59.090 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:59.090 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:59.090 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260505 14:41:59.090 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:59.090 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:59.090 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:41:59.090 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:59.090 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:41:59.091 | INFO | Uploaded 36GiB to GPU in 2.19881s (16.3725GiB/s) | 20260505 14:41:59.091 | INFO | Downloaded 34GiB to GPU in 2.13446s (15.9291GiB/s) | 20260505 14:42:04.233 | INFO | Analyzing chunk frames = [4007936,4008448) | 20260505 14:42:04.233 | INFO | MemInfo: { | 20260505 14:42:04.233 | INFO | "gpu" : " 17 GB", | 20260505 14:42:04.233 | INFO | "huge" : "", | 20260505 14:42:04.233 | INFO | "huge_shared" : " 27 GB", | 20260505 14:42:04.233 | INFO | "malloc" : " 14 GB", | 20260505 14:42:04.233 | INFO | "pinned" : " 29 GB" | 20260505 14:42:04.233 | INFO | } | 20260505 14:42:04.233 | INFO | Perf report for the last 5.14301 seconds: | 20260505 14:42:04.233 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:42:04.233 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:04.233 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:04.233 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:04.233 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260505 14:42:04.233 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:04.233 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:04.233 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:42:04.233 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:04.233 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:04.234 | INFO | Uploaded 36GiB to GPU in 2.13756s (16.8416GiB/s) | 20260505 14:42:04.234 | INFO | Downloaded 34GiB to GPU in 2.05803s (16.5207GiB/s) | 20260505 14:42:09.368 | INFO | Analyzing chunk frames = [4008448,4008960) | 20260505 14:42:09.368 | INFO | MemInfo: { | 20260505 14:42:09.368 | INFO | "gpu" : " 17 GB", | 20260505 14:42:09.368 | INFO | "huge" : "", | 20260505 14:42:09.368 | INFO | "huge_shared" : " 27 GB", | 20260505 14:42:09.369 | INFO | "malloc" : " 14 GB", | 20260505 14:42:09.369 | INFO | "pinned" : " 29 GB" | 20260505 14:42:09.369 | INFO | } | 20260505 14:42:09.369 | INFO | Perf report for the last 5.1355 seconds: | 20260505 14:42:09.369 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:42:09.369 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:09.369 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:09.369 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:09.369 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260505 14:42:09.369 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:09.369 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:09.369 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:42:09.369 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:09.369 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:09.369 | INFO | Uploaded 36GiB to GPU in 2.15547s (16.7017GiB/s) | 20260505 14:42:09.369 | INFO | Downloaded 34GiB to GPU in 2.03607s (16.6988GiB/s) | 20260505 14:42:14.500 | INFO | Analyzing chunk frames = [4008960,4009472) | 20260505 14:42:14.500 | INFO | MemInfo: { | 20260505 14:42:14.500 | INFO | "gpu" : " 17 GB", | 20260505 14:42:14.500 | INFO | "huge" : "", | 20260505 14:42:14.500 | INFO | "huge_shared" : " 27 GB", | 20260505 14:42:14.500 | INFO | "malloc" : " 14 GB", | 20260505 14:42:14.500 | INFO | "pinned" : " 29 GB" | 20260505 14:42:14.500 | INFO | } | 20260505 14:42:14.500 | INFO | Perf report for the last 5.13127 seconds: | 20260505 14:42:14.500 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:42:14.500 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:14.500 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:14.500 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:14.500 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 14:42:14.500 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:14.500 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:14.500 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 14:42:14.500 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:14.500 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:14.500 | INFO | Uploaded 36GiB to GPU in 2.1232s (16.9555GiB/s) | 20260505 14:42:14.500 | INFO | Downloaded 34GiB to GPU in 2.17212s (15.6529GiB/s) | 20260505 14:42:19.641 | INFO | Analyzing chunk frames = [4009472,4009984) | 20260505 14:42:19.641 | INFO | MemInfo: { | 20260505 14:42:19.641 | INFO | "gpu" : " 17 GB", | 20260505 14:42:19.641 | INFO | "huge" : "", | 20260505 14:42:19.641 | INFO | "huge_shared" : " 27 GB", | 20260505 14:42:19.641 | INFO | "malloc" : " 14 GB", | 20260505 14:42:19.641 | INFO | "pinned" : " 29 GB" | 20260505 14:42:19.641 | INFO | } | 20260505 14:42:19.641 | INFO | Perf report for the last 5.14116 seconds: | 20260505 14:42:19.641 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:42:19.641 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:19.641 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:19.641 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:19.641 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 14:42:19.641 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:19.641 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:19.641 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:42:19.641 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:19.641 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:19.641 | INFO | Uploaded 36GiB to GPU in 2.1309s (16.8942GiB/s) | 20260505 14:42:19.642 | INFO | Downloaded 34GiB to GPU in 2.10145s (16.1793GiB/s) | 20260505 14:42:19.777 | INFO | Starting a round of Baz IO | 20260505 14:42:22.999 | INFO | Populating RTMetrics report | 20260505 14:42:22.999 | INFO | RTMetrics report populated | 20260505 14:42:22.999 | INFO | Opening tmp RTMetrics json file | 20260505 14:42:23.033 | INFO | Serializing RTMetrics to json | 20260505 14:42:23.064 | INFO | Serialization done, swapping tmp file now... | 20260505 14:42:23.085 | INFO | RTMetrics done | 20260505 14:42:23.102 | INFO | Finished a round of Baz IO: Wrote 2.249 GiB in 3.324 seconds | 20260505 14:42:23.102 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.68% / 0.00% / 77.32% | 20260505 14:42:23.102 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.529357% / 0% / 8.34465e-05% | 20260505 14:42:23.102 | INFO | Baz IO overall: | 20260505 14:42:23.102 | INFO | BytesWritten : 2388436687887 | 20260505 14:42:23.102 | INFO | NumEvents : 1135354416573 | 20260505 14:42:24.787 | INFO | Analyzing chunk frames = [4009984,4010496) | 20260505 14:42:24.787 | INFO | MemInfo: { | 20260505 14:42:24.787 | INFO | "gpu" : " 17 GB", | 20260505 14:42:24.787 | INFO | "huge" : "", | 20260505 14:42:24.787 | INFO | "huge_shared" : " 27 GB", | 20260505 14:42:24.787 | INFO | "malloc" : " 14 GB", | 20260505 14:42:24.787 | INFO | "pinned" : " 29 GB" | 20260505 14:42:24.787 | INFO | } | 20260505 14:42:24.787 | INFO | Perf report for the last 5.14591 seconds: | 20260505 14:42:24.787 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:42:24.787 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:24.787 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:24.787 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:24.787 | INFO | ANALYSIS : 64.1%, 3.0, 99.8ms, 0 | 20260505 14:42:24.787 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:24.787 | INFO | RT_METRICS : 46.5%, 1.0, 24.4ms, 0 | 20260505 14:42:24.787 | INFO | PRE_HQ : 60.8%, 1.8, 57.1ms, 0 | 20260505 14:42:24.787 | INFO | BAZWRITER : 64.0%, 4.0, 131.2ms, 0 | 20260505 14:42:24.787 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 14:42:24.787 | INFO | Uploaded 36GiB to GPU in 2.21971s (16.2184GiB/s) | 20260505 14:42:24.787 | INFO | Downloaded 38GiB to GPU in 2.26716s (16.7611GiB/s) | 20260505 14:42:29.926 | INFO | Analyzing chunk frames = [4010496,4011008) | 20260505 14:42:29.926 | INFO | MemInfo: { | 20260505 14:42:29.926 | INFO | "gpu" : " 17 GB", | 20260505 14:42:29.926 | INFO | "huge" : "", | 20260505 14:42:29.926 | INFO | "huge_shared" : " 27 GB", | 20260505 14:42:29.926 | INFO | "malloc" : " 14 GB", | 20260505 14:42:29.926 | INFO | "pinned" : " 29 GB" | 20260505 14:42:29.926 | INFO | } | 20260505 14:42:29.926 | INFO | Perf report for the last 5.13909 seconds: | 20260505 14:42:29.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:42:29.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:29.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:29.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:29.926 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 14:42:29.926 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:29.926 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:29.926 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:42:29.926 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:29.926 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:29.926 | INFO | Uploaded 36GiB to GPU in 2.20171s (16.3509GiB/s) | 20260505 14:42:29.927 | INFO | Downloaded 34GiB to GPU in 2.05994s (16.5054GiB/s) | 20260505 14:42:35.062 | INFO | Analyzing chunk frames = [4011008,4011520) | 20260505 14:42:35.062 | INFO | MemInfo: { | 20260505 14:42:35.062 | INFO | "gpu" : " 17 GB", | 20260505 14:42:35.062 | INFO | "huge" : "", | 20260505 14:42:35.062 | INFO | "huge_shared" : " 27 GB", | 20260505 14:42:35.062 | INFO | "malloc" : " 14 GB", | 20260505 14:42:35.062 | INFO | "pinned" : " 29 GB" | 20260505 14:42:35.062 | INFO | } | 20260505 14:42:35.062 | INFO | Perf report for the last 5.13583 seconds: | 20260505 14:42:35.062 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:42:35.062 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:35.062 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:35.062 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:35.062 | INFO | ANALYSIS : 63.1%, 3.0, 98.2ms, 0 | 20260505 14:42:35.062 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:35.062 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:35.062 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:42:35.062 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:35.062 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:35.062 | INFO | Uploaded 36GiB to GPU in 2.18433s (16.481GiB/s) | 20260505 14:42:35.062 | INFO | Downloaded 34GiB to GPU in 2.06217s (16.4875GiB/s) | 20260505 14:42:40.200 | INFO | Analyzing chunk frames = [4011520,4012032) | 20260505 14:42:40.200 | INFO | MemInfo: { | 20260505 14:42:40.200 | INFO | "gpu" : " 17 GB", | 20260505 14:42:40.200 | INFO | "huge" : "", | 20260505 14:42:40.200 | INFO | "huge_shared" : " 27 GB", | 20260505 14:42:40.200 | INFO | "malloc" : " 14 GB", | 20260505 14:42:40.200 | INFO | "pinned" : " 29 GB" | 20260505 14:42:40.200 | INFO | } | 20260505 14:42:40.200 | INFO | Perf report for the last 5.13769 seconds: | 20260505 14:42:40.200 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:42:40.200 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:40.200 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:40.200 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:40.200 | INFO | ANALYSIS : 63.0%, 3.0, 97.8ms, 0 | 20260505 14:42:40.200 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:40.200 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:40.200 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:42:40.200 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:40.200 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:40.200 | INFO | Uploaded 36GiB to GPU in 2.19479s (16.4025GiB/s) | 20260505 14:42:40.200 | INFO | Downloaded 34GiB to GPU in 2.13402s (15.9324GiB/s) | 20260505 14:42:45.350 | INFO | Analyzing chunk frames = [4012032,4012544) | 20260505 14:42:45.350 | INFO | MemInfo: { | 20260505 14:42:45.350 | INFO | "gpu" : " 17 GB", | 20260505 14:42:45.350 | INFO | "huge" : "", | 20260505 14:42:45.350 | INFO | "huge_shared" : " 27 GB", | 20260505 14:42:45.350 | INFO | "malloc" : " 14 GB", | 20260505 14:42:45.350 | INFO | "pinned" : " 29 GB" | 20260505 14:42:45.350 | INFO | } | 20260505 14:42:45.350 | INFO | Perf report for the last 5.15048 seconds: | 20260505 14:42:45.350 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:42:45.350 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:45.350 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:45.350 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:45.350 | INFO | ANALYSIS : 62.7%, 3.0, 98.1ms, 0 | 20260505 14:42:45.350 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:45.350 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:45.351 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 14:42:45.351 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:45.351 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:45.351 | INFO | Uploaded 36GiB to GPU in 2.17278s (16.5686GiB/s) | 20260505 14:42:45.351 | INFO | Downloaded 34GiB to GPU in 2.13036s (15.9598GiB/s) | 20260505 14:42:50.482 | INFO | Analyzing chunk frames = [4012544,4013056) | 20260505 14:42:50.482 | INFO | MemInfo: { | 20260505 14:42:50.482 | INFO | "gpu" : " 17 GB", | 20260505 14:42:50.482 | INFO | "huge" : "", | 20260505 14:42:50.482 | INFO | "huge_shared" : " 27 GB", | 20260505 14:42:50.482 | INFO | "malloc" : " 14 GB", | 20260505 14:42:50.483 | INFO | "pinned" : " 29 GB" | 20260505 14:42:50.483 | INFO | } | 20260505 14:42:50.483 | INFO | Perf report for the last 5.13227 seconds: | 20260505 14:42:50.483 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:42:50.483 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:50.483 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:50.483 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:50.483 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 14:42:50.483 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:50.483 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:50.483 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 14:42:50.483 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:50.483 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:50.483 | INFO | Uploaded 36GiB to GPU in 2.13513s (16.8608GiB/s) | 20260505 14:42:50.483 | INFO | Downloaded 34GiB to GPU in 2.02874s (16.7592GiB/s) | 20260505 14:42:55.618 | INFO | Analyzing chunk frames = [4013056,4013568) | 20260505 14:42:55.618 | INFO | MemInfo: { | 20260505 14:42:55.618 | INFO | "gpu" : " 17 GB", | 20260505 14:42:55.618 | INFO | "huge" : "", | 20260505 14:42:55.618 | INFO | "huge_shared" : " 27 GB", | 20260505 14:42:55.618 | INFO | "malloc" : " 14 GB", | 20260505 14:42:55.618 | INFO | "pinned" : " 29 GB" | 20260505 14:42:55.618 | INFO | } | 20260505 14:42:55.618 | INFO | Perf report for the last 5.13585 seconds: | 20260505 14:42:55.618 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:42:55.618 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:55.618 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:55.619 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:55.619 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 14:42:55.619 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:55.619 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:55.619 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 14:42:55.619 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:55.619 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:42:55.619 | INFO | Uploaded 36GiB to GPU in 2.10788s (17.0788GiB/s) | 20260505 14:42:55.619 | INFO | Downloaded 34GiB to GPU in 2.17358s (15.6424GiB/s) | 20260505 14:43:00.759 | INFO | Analyzing chunk frames = [4013568,4014080) | 20260505 14:43:00.759 | INFO | MemInfo: { | 20260505 14:43:00.759 | INFO | "gpu" : " 17 GB", | 20260505 14:43:00.759 | INFO | "huge" : "", | 20260505 14:43:00.759 | INFO | "huge_shared" : " 27 GB", | 20260505 14:43:00.759 | INFO | "malloc" : " 14 GB", | 20260505 14:43:00.759 | INFO | "pinned" : " 29 GB" | 20260505 14:43:00.759 | INFO | } | 20260505 14:43:00.759 | INFO | Perf report for the last 5.14037 seconds: | 20260505 14:43:00.759 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:43:00.759 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:00.759 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:00.759 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:00.759 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 14:43:00.759 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:00.759 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:00.759 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:43:00.759 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:00.759 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:00.759 | INFO | Uploaded 36GiB to GPU in 2.20015s (16.3625GiB/s) | 20260505 14:43:00.759 | INFO | Downloaded 34GiB to GPU in 2.10618s (16.143GiB/s) | 20260505 14:43:00.759 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 14:43:00.759 | INFO | Total mem usage: 15.3 GiB | 20260505 14:43:00.759 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 14:43:00.759 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.72%) | 20260505 14:43:00.759 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 14:43:00.759 | INFO | High Water Breakdown: | 20260505 14:43:00.759 | INFO | 61% : 8.67 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 14:43:00.759 | INFO | 38.9% : 5.53 GiB ( 83.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 14:43:00.759 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 14:43:00.759 | INFO | ------------------------------------------ | 20260505 14:43:00.759 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:43:00.760 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 14:43:00.760 | INFO | Total mem usage: 26.5 GiB | 20260505 14:43:00.760 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 14:43:00.760 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 14:43:00.760 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 14:43:00.760 | INFO | High Water Breakdown: | 20260505 14:43:00.760 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:43:00.760 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:43:00.760 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:43:00.760 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:43:00.760 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:43:00.760 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:43:00.760 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 14:43:00.760 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 14:43:00.760 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:43:00.760 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 14:43:00.760 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 14:43:00.760 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 14:43:00.760 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 14:43:00.760 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 14:43:00.760 | INFO | ------------------------------------------ | 20260505 14:43:00.760 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:43:00.760 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 14:43:00.760 | INFO | Total mem usage: 25.6 GiB | 20260505 14:43:00.760 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 14:43:00.760 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 14:43:00.760 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 14:43:00.760 | INFO | High Water Breakdown: | 20260505 14:43:00.760 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 14:43:00.760 | INFO | ------------------------------------------ | 20260505 14:43:00.760 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:43:00.760 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 14:43:00.760 | INFO | Total mem usage: 15.7 GiB | 20260505 14:43:00.760 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 14:43:00.760 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 14:43:00.760 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260505 14:43:00.760 | INFO | High Water Breakdown: | 20260505 14:43:00.760 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 14:43:00.760 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 14:43:00.760 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 14:43:00.760 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 14:43:00.760 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 14:43:00.760 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 14:43:00.760 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 14:43:00.760 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 14:43:00.760 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 14:43:00.760 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:43:00.760 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 14:43:00.760 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:43:00.760 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 14:43:00.760 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 14:43:00.760 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 14:43:00.760 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 14:43:00.760 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:43:00.760 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 14:43:00.760 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:43:00.760 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:43:00.760 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:43:00.760 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 14:43:00.760 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 14:43:00.760 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:43:00.761 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 14:43:00.761 | INFO | ------------------------------------------ | 20260505 14:43:00.761 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:43:00.761 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 14:43:00.761 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:43:00.761 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:43:00.761 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.60e+01, 1.36e+01 / 1.90e+00 | 20260505 14:43:00.761 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 1.01e-01, 1.09e-02 / 3.54e-03 | 20260505 14:43:00.761 | INFO | PerfReport: 11.78%, Baseline -- 3.75e+00 / 4.50e+00, 5.83e-01 / 1.78e-01 | 20260505 14:43:00.761 | INFO | PerfReport: 3.30%, Histogram -- 1.05e+00 / 1.51e+00, 1.97e-01 / 8.90e-02 | 20260505 14:43:00.761 | INFO | PerfReport: 15.96%, DME -- 4.06e+01 / 4.53e+01, 4.35e+00 / 2.09e+00 | 20260505 14:43:00.761 | INFO | PerfReport: 56.52%, FrameLabeling -- 1.80e+01 / 1.98e+01, 1.46e+00 / 1.91e+00 | 20260505 14:43:00.761 | INFO | PerfReport: 7.39%, PulseAccumulating -- 2.35e+00 / 2.92e+00, 4.42e-01 / 1.59e-01 | 20260505 14:43:00.761 | INFO | PerfReport: 3.52%, Metrics -- 1.12e+00 / 1.44e+00, 3.12e-01 / 1.57e-01 | 20260505 14:43:00.761 | INFO | PerfReport: 1.23%, PulseSerializing -- 3.92e-01 / 4.49e-01, 6.96e-02 / 3.80e-02 | 20260505 14:43:00.761 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 14:43:00.761 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:43:00.761 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:43:00.761 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260505 14:43:00.761 | INFO | PerfReport: 22.47%, Upload -- 2.26e+01 / 0.00e+00, 7.88e+00 / 0.00e+00 | 20260505 14:43:00.761 | INFO | PerfReport: 21.93%, Download -- 2.21e+01 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260505 14:43:00.761 | INFO | PerfReport: 31.64%, Compute -- 3.18e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 14:43:00.761 | INFO | PerfReport: 2.18%, LockUpload -- 2.19e+00 / 0.00e+00, 7.82e+00 / 0.00e+00 | 20260505 14:43:00.761 | INFO | PerfReport: 1.44%, LockDownload -- 1.45e+00 / 0.00e+00, 2.98e+00 / 0.00e+00 | 20260505 14:43:00.761 | INFO | PerfReport: 20.32%, LockCompute -- 2.04e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260505 14:43:00.761 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.85e-02 / 0.00e+00 | 20260505 14:43:00.894 | INFO | Starting a round of Baz IO | 20260505 14:43:04.113 | INFO | Populating RTMetrics report | 20260505 14:43:04.113 | INFO | RTMetrics report populated | 20260505 14:43:04.116 | INFO | Opening tmp RTMetrics json file | 20260505 14:43:04.126 | INFO | Serializing RTMetrics to json | 20260505 14:43:04.170 | INFO | Serialization done, swapping tmp file now... | 20260505 14:43:04.191 | INFO | RTMetrics done | 20260505 14:43:04.230 | INFO | Finished a round of Baz IO: Wrote 2.253 GiB in 3.335 seconds | 20260505 14:43:04.230 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.67% / 0.00% / 77.32% | 20260505 14:43:04.230 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.524251% / 7.94729e-06% / 8.34465e-05% | 20260505 14:43:04.230 | INFO | Baz IO overall: | 20260505 14:43:04.230 | INFO | BytesWritten : 2390855714481 | 20260505 14:43:04.230 | INFO | NumEvents : 1136493617869 | 20260505 14:43:05.896 | INFO | Analyzing chunk frames = [4014080,4014592) | 20260505 14:43:05.896 | INFO | MemInfo: { | 20260505 14:43:05.896 | INFO | "gpu" : " 17 GB", | 20260505 14:43:05.896 | INFO | "huge" : "", | 20260505 14:43:05.896 | INFO | "huge_shared" : " 27 GB", | 20260505 14:43:05.896 | INFO | "malloc" : " 14 GB", | 20260505 14:43:05.896 | INFO | "pinned" : " 29 GB" | 20260505 14:43:05.896 | INFO | } | 20260505 14:43:05.896 | INFO | Perf report for the last 5.13753 seconds: | 20260505 14:43:05.896 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:43:05.896 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:05.896 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:05.896 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:05.896 | INFO | ANALYSIS : 64.2%, 3.0, 99.4ms, 0 | 20260505 14:43:05.896 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:05.897 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260505 14:43:05.897 | INFO | PRE_HQ : 60.8%, 1.8, 57.0ms, 0 | 20260505 14:43:05.897 | INFO | BAZWRITER : 64.1%, 4.0, 132.4ms, 0 | 20260505 14:43:05.897 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 14:43:05.897 | INFO | Uploaded 36GiB to GPU in 2.21768s (16.2332GiB/s) | 20260505 14:43:05.897 | INFO | Downloaded 38GiB to GPU in 2.26695s (16.7626GiB/s) | 20260505 14:43:11.046 | INFO | Analyzing chunk frames = [4014592,4015104) | 20260505 14:43:11.046 | INFO | MemInfo: { | 20260505 14:43:11.046 | INFO | "gpu" : " 17 GB", | 20260505 14:43:11.046 | INFO | "huge" : "", | 20260505 14:43:11.046 | INFO | "huge_shared" : " 27 GB", | 20260505 14:43:11.046 | INFO | "malloc" : " 14 GB", | 20260505 14:43:11.046 | INFO | "pinned" : " 29 GB" | 20260505 14:43:11.046 | INFO | } | 20260505 14:43:11.046 | INFO | Perf report for the last 5.15003 seconds: | 20260505 14:43:11.046 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:43:11.046 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:11.046 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:11.046 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:11.046 | INFO | ANALYSIS : 63.3%, 3.0, 99.0ms, 0 | 20260505 14:43:11.046 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:11.046 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:11.046 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 14:43:11.046 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:11.046 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:11.047 | INFO | Uploaded 36GiB to GPU in 2.1854s (16.473GiB/s) | 20260505 14:43:11.047 | INFO | Downloaded 34GiB to GPU in 2.037s (16.6912GiB/s) | 20260505 14:43:16.189 | INFO | Analyzing chunk frames = [4015104,4015616) | 20260505 14:43:16.189 | INFO | MemInfo: { | 20260505 14:43:16.189 | INFO | "gpu" : " 17 GB", | 20260505 14:43:16.189 | INFO | "huge" : "", | 20260505 14:43:16.189 | INFO | "huge_shared" : " 27 GB", | 20260505 14:43:16.189 | INFO | "malloc" : " 14 GB", | 20260505 14:43:16.189 | INFO | "pinned" : " 29 GB" | 20260505 14:43:16.189 | INFO | } | 20260505 14:43:16.190 | INFO | Perf report for the last 5.14313 seconds: | 20260505 14:43:16.190 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:43:16.190 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:16.190 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:16.190 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:16.190 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260505 14:43:16.190 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:16.190 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:16.190 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:43:16.190 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:16.190 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:16.190 | INFO | Uploaded 36GiB to GPU in 2.20343s (16.3382GiB/s) | 20260505 14:43:16.190 | INFO | Downloaded 34GiB to GPU in 2.04401s (16.6339GiB/s) | 20260505 14:43:21.314 | INFO | Analyzing chunk frames = [4015616,4016128) | 20260505 14:43:21.314 | INFO | MemInfo: { | 20260505 14:43:21.314 | INFO | "gpu" : " 17 GB", | 20260505 14:43:21.314 | INFO | "huge" : "", | 20260505 14:43:21.314 | INFO | "huge_shared" : " 27 GB", | 20260505 14:43:21.314 | INFO | "malloc" : " 14 GB", | 20260505 14:43:21.314 | INFO | "pinned" : " 29 GB" | 20260505 14:43:21.314 | INFO | } | 20260505 14:43:21.314 | INFO | Perf report for the last 5.1245 seconds: | 20260505 14:43:21.314 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:43:21.314 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:21.314 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:21.314 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:21.314 | INFO | ANALYSIS : 62.8%, 3.0, 97.2ms, 0 | 20260505 14:43:21.314 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:21.314 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:21.314 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:43:21.314 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:21.314 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:21.314 | INFO | Uploaded 36GiB to GPU in 2.18539s (16.4731GiB/s) | 20260505 14:43:21.314 | INFO | Downloaded 34GiB to GPU in 2.12874s (15.9719GiB/s) | 20260505 14:43:26.448 | INFO | Analyzing chunk frames = [4016128,4016640) | 20260505 14:43:26.448 | INFO | MemInfo: { | 20260505 14:43:26.448 | INFO | "gpu" : " 17 GB", | 20260505 14:43:26.448 | INFO | "huge" : "", | 20260505 14:43:26.448 | INFO | "huge_shared" : " 27 GB", | 20260505 14:43:26.448 | INFO | "malloc" : " 14 GB", | 20260505 14:43:26.448 | INFO | "pinned" : " 29 GB" | 20260505 14:43:26.448 | INFO | } | 20260505 14:43:26.448 | INFO | Perf report for the last 5.13397 seconds: | 20260505 14:43:26.448 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:43:26.448 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:26.448 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:26.448 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:26.448 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260505 14:43:26.448 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:26.448 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:26.448 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:43:26.448 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:26.448 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:26.448 | INFO | Uploaded 36GiB to GPU in 2.16473s (16.6303GiB/s) | 20260505 14:43:26.448 | INFO | Downloaded 34GiB to GPU in 2.12823s (15.9757GiB/s) | 20260505 14:43:31.596 | INFO | Analyzing chunk frames = [4016640,4017152) | 20260505 14:43:31.596 | INFO | MemInfo: { | 20260505 14:43:31.596 | INFO | "gpu" : " 17 GB", | 20260505 14:43:31.596 | INFO | "huge" : "", | 20260505 14:43:31.596 | INFO | "huge_shared" : " 27 GB", | 20260505 14:43:31.596 | INFO | "malloc" : " 14 GB", | 20260505 14:43:31.596 | INFO | "pinned" : " 29 GB" | 20260505 14:43:31.597 | INFO | } | 20260505 14:43:31.597 | INFO | Perf report for the last 5.14851 seconds: | 20260505 14:43:31.597 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:43:31.597 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:31.597 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:31.597 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:31.597 | INFO | ANALYSIS : 62.9%, 3.0, 98.4ms, 0 | 20260505 14:43:31.597 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:31.597 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:31.597 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:43:31.597 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:31.597 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:31.597 | INFO | Uploaded 36GiB to GPU in 2.17189s (16.5754GiB/s) | 20260505 14:43:31.597 | INFO | Downloaded 34GiB to GPU in 2.09921s (16.1966GiB/s) | 20260505 14:43:36.737 | INFO | Analyzing chunk frames = [4017152,4017664) | 20260505 14:43:36.737 | INFO | MemInfo: { | 20260505 14:43:36.737 | INFO | "gpu" : " 17 GB", | 20260505 14:43:36.737 | INFO | "huge" : "", | 20260505 14:43:36.737 | INFO | "huge_shared" : " 27 GB", | 20260505 14:43:36.737 | INFO | "malloc" : " 14 GB", | 20260505 14:43:36.737 | INFO | "pinned" : " 29 GB" | 20260505 14:43:36.737 | INFO | } | 20260505 14:43:36.738 | INFO | Perf report for the last 5.14108 seconds: | 20260505 14:43:36.738 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:43:36.738 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:36.738 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:36.738 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:36.738 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 14:43:36.738 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:36.738 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:36.738 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 14:43:36.738 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:36.738 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:36.738 | INFO | Uploaded 36GiB to GPU in 2.13902s (16.8301GiB/s) | 20260505 14:43:36.738 | INFO | Downloaded 34GiB to GPU in 2.18642s (15.5505GiB/s) | 20260505 14:43:41.876 | INFO | Analyzing chunk frames = [4017664,4018176) | 20260505 14:43:41.876 | INFO | MemInfo: { | 20260505 14:43:41.876 | INFO | "gpu" : " 17 GB", | 20260505 14:43:41.876 | INFO | "huge" : "", | 20260505 14:43:41.876 | INFO | "huge_shared" : " 27 GB", | 20260505 14:43:41.876 | INFO | "malloc" : " 14 GB", | 20260505 14:43:41.876 | INFO | "pinned" : " 29 GB" | 20260505 14:43:41.876 | INFO | } | 20260505 14:43:41.876 | INFO | Perf report for the last 5.13827 seconds: | 20260505 14:43:41.876 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:43:41.876 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:41.876 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:41.876 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:41.876 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 14:43:41.876 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:41.876 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:41.876 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:43:41.876 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:41.876 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:41.876 | INFO | Uploaded 36GiB to GPU in 2.2206s (16.2119GiB/s) | 20260505 14:43:41.876 | INFO | Downloaded 34GiB to GPU in 2.09363s (16.2397GiB/s) | 20260505 14:43:42.011 | INFO | Starting a round of Baz IO | 20260505 14:43:45.220 | INFO | Populating RTMetrics report | 20260505 14:43:45.220 | INFO | RTMetrics report populated | 20260505 14:43:45.220 | INFO | Opening tmp RTMetrics json file | 20260505 14:43:45.237 | INFO | Serializing RTMetrics to json | 20260505 14:43:45.242 | INFO | Serialization done, swapping tmp file now... | 20260505 14:43:45.251 | INFO | RTMetrics done | 20260505 14:43:45.324 | INFO | Finished a round of Baz IO: Wrote 2.252 GiB in 3.313 seconds | 20260505 14:43:45.324 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.67% / 0.00% / 77.33% | 20260505 14:43:45.324 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.526178% / 3.97364e-06% / 6.35783e-05% | 20260505 14:43:45.324 | INFO | Baz IO overall: | 20260505 14:43:45.324 | INFO | BytesWritten : 2393273794463 | 20260505 14:43:45.324 | INFO | NumEvents : 1137632234077 | 20260505 14:43:47.013 | INFO | Analyzing chunk frames = [4018176,4018688) | 20260505 14:43:47.013 | INFO | MemInfo: { | 20260505 14:43:47.013 | INFO | "gpu" : " 17 GB", | 20260505 14:43:47.013 | INFO | "huge" : "", | 20260505 14:43:47.013 | INFO | "huge_shared" : " 27 GB", | 20260505 14:43:47.013 | INFO | "malloc" : " 14 GB", | 20260505 14:43:47.013 | INFO | "pinned" : " 29 GB" | 20260505 14:43:47.013 | INFO | } | 20260505 14:43:47.013 | INFO | Perf report for the last 5.13703 seconds: | 20260505 14:43:47.013 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:43:47.013 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:47.013 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:47.013 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:47.013 | INFO | ANALYSIS : 64.0%, 3.0, 99.3ms, 0 | 20260505 14:43:47.013 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:47.013 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260505 14:43:47.013 | INFO | PRE_HQ : 61.0%, 1.8, 57.1ms, 0 | 20260505 14:43:47.013 | INFO | BAZWRITER : 63.8%, 4.0, 131.9ms, 0 | 20260505 14:43:47.013 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 14:43:47.013 | INFO | Uploaded 36GiB to GPU in 2.15996s (16.6669GiB/s) | 20260505 14:43:47.013 | INFO | Downloaded 38GiB to GPU in 2.25388s (16.8598GiB/s) | 20260505 14:43:52.151 | INFO | Analyzing chunk frames = [4018688,4019200) | 20260505 14:43:52.151 | INFO | MemInfo: { | 20260505 14:43:52.151 | INFO | "gpu" : " 17 GB", | 20260505 14:43:52.151 | INFO | "huge" : "", | 20260505 14:43:52.151 | INFO | "huge_shared" : " 27 GB", | 20260505 14:43:52.151 | INFO | "malloc" : " 14 GB", | 20260505 14:43:52.151 | INFO | "pinned" : " 29 GB" | 20260505 14:43:52.151 | INFO | } | 20260505 14:43:52.152 | INFO | Perf report for the last 5.13858 seconds: | 20260505 14:43:52.152 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:43:52.152 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:52.152 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:52.152 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:52.152 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 14:43:52.152 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:52.152 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:52.152 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:43:52.152 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:52.152 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:52.152 | INFO | Uploaded 36GiB to GPU in 2.19768s (16.3809GiB/s) | 20260505 14:43:52.152 | INFO | Downloaded 34GiB to GPU in 2.04118s (16.657GiB/s) | 20260505 14:43:57.295 | INFO | Analyzing chunk frames = [4019200,4019712) | 20260505 14:43:57.295 | INFO | MemInfo: { | 20260505 14:43:57.295 | INFO | "gpu" : " 17 GB", | 20260505 14:43:57.295 | INFO | "huge" : "", | 20260505 14:43:57.295 | INFO | "huge_shared" : " 27 GB", | 20260505 14:43:57.295 | INFO | "malloc" : " 14 GB", | 20260505 14:43:57.295 | INFO | "pinned" : " 29 GB" | 20260505 14:43:57.295 | INFO | } | 20260505 14:43:57.295 | INFO | Perf report for the last 5.14368 seconds: | 20260505 14:43:57.295 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:43:57.295 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:57.295 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:57.295 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:57.295 | INFO | ANALYSIS : 63.5%, 3.0, 99.0ms, 0 | 20260505 14:43:57.295 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:57.295 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:57.295 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:43:57.295 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:57.295 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:43:57.296 | INFO | Uploaded 36GiB to GPU in 2.23844s (16.0826GiB/s) | 20260505 14:43:57.296 | INFO | Downloaded 34GiB to GPU in 2.11608s (16.0674GiB/s) | 20260505 14:44:02.433 | INFO | Analyzing chunk frames = [4019712,4020224) | 20260505 14:44:02.433 | INFO | MemInfo: { | 20260505 14:44:02.433 | INFO | "gpu" : " 17 GB", | 20260505 14:44:02.433 | INFO | "huge" : "", | 20260505 14:44:02.433 | INFO | "huge_shared" : " 27 GB", | 20260505 14:44:02.433 | INFO | "malloc" : " 14 GB", | 20260505 14:44:02.433 | INFO | "pinned" : " 29 GB" | 20260505 14:44:02.433 | INFO | } | 20260505 14:44:02.434 | INFO | Perf report for the last 5.13826 seconds: | 20260505 14:44:02.434 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:44:02.434 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:02.434 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:02.434 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:02.434 | INFO | ANALYSIS : 62.6%, 3.0, 97.2ms, 0 | 20260505 14:44:02.434 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:02.434 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:02.434 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:44:02.434 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:02.434 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:02.435 | INFO | Uploaded 36GiB to GPU in 2.17797s (16.5292GiB/s) | 20260505 14:44:02.435 | INFO | Downloaded 34GiB to GPU in 2.13092s (15.9555GiB/s) | 20260505 14:44:07.568 | INFO | Analyzing chunk frames = [4020224,4020736) | 20260505 14:44:07.568 | INFO | MemInfo: { | 20260505 14:44:07.568 | INFO | "gpu" : " 17 GB", | 20260505 14:44:07.568 | INFO | "huge" : "", | 20260505 14:44:07.568 | INFO | "huge_shared" : " 27 GB", | 20260505 14:44:07.568 | INFO | "malloc" : " 14 GB", | 20260505 14:44:07.568 | INFO | "pinned" : " 29 GB" | 20260505 14:44:07.568 | INFO | } | 20260505 14:44:07.568 | INFO | Perf report for the last 5.13459 seconds: | 20260505 14:44:07.568 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:44:07.568 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:07.568 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:07.568 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:07.568 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 14:44:07.568 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:07.568 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:07.568 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:44:07.568 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:07.568 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:07.568 | INFO | Uploaded 36GiB to GPU in 2.17118s (16.5808GiB/s) | 20260505 14:44:07.568 | INFO | Downloaded 34GiB to GPU in 2.1313s (15.9527GiB/s) | 20260505 14:44:12.705 | INFO | Analyzing chunk frames = [4020736,4021248) | 20260505 14:44:12.705 | INFO | MemInfo: { | 20260505 14:44:12.705 | INFO | "gpu" : " 17 GB", | 20260505 14:44:12.705 | INFO | "huge" : "", | 20260505 14:44:12.705 | INFO | "huge_shared" : " 27 GB", | 20260505 14:44:12.705 | INFO | "malloc" : " 14 GB", | 20260505 14:44:12.706 | INFO | "pinned" : " 29 GB" | 20260505 14:44:12.706 | INFO | } | 20260505 14:44:12.706 | INFO | Perf report for the last 5.13755 seconds: | 20260505 14:44:12.706 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:44:12.706 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:12.706 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:12.706 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:12.706 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 14:44:12.706 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:12.706 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:12.706 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 14:44:12.706 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:12.706 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:12.706 | INFO | Uploaded 36GiB to GPU in 2.16071s (16.6612GiB/s) | 20260505 14:44:12.706 | INFO | Downloaded 34GiB to GPU in 2.11079s (16.1077GiB/s) | 20260505 14:44:17.854 | INFO | Analyzing chunk frames = [4021248,4021760) | 20260505 14:44:17.854 | INFO | MemInfo: { | 20260505 14:44:17.854 | INFO | "gpu" : " 17 GB", | 20260505 14:44:17.854 | INFO | "huge" : "", | 20260505 14:44:17.854 | INFO | "huge_shared" : " 27 GB", | 20260505 14:44:17.854 | INFO | "malloc" : " 14 GB", | 20260505 14:44:17.854 | INFO | "pinned" : " 29 GB" | 20260505 14:44:17.854 | INFO | } | 20260505 14:44:17.854 | INFO | Perf report for the last 5.14833 seconds: | 20260505 14:44:17.854 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:44:17.854 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:17.854 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:17.854 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:17.854 | INFO | ANALYSIS : 63.3%, 3.0, 99.0ms, 0 | 20260505 14:44:17.854 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:17.854 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:17.854 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 14:44:17.854 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:17.854 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:17.854 | INFO | Uploaded 36GiB to GPU in 2.1675s (16.609GiB/s) | 20260505 14:44:17.854 | INFO | Downloaded 34GiB to GPU in 2.16837s (15.68GiB/s) | 20260505 14:44:22.994 | INFO | Analyzing chunk frames = [4021760,4022272) | 20260505 14:44:22.994 | INFO | MemInfo: { | 20260505 14:44:22.994 | INFO | "gpu" : " 17 GB", | 20260505 14:44:22.994 | INFO | "huge" : "", | 20260505 14:44:22.994 | INFO | "huge_shared" : " 27 GB", | 20260505 14:44:22.994 | INFO | "malloc" : " 14 GB", | 20260505 14:44:22.994 | INFO | "pinned" : " 29 GB" | 20260505 14:44:22.994 | INFO | } | 20260505 14:44:22.994 | INFO | Perf report for the last 5.1403 seconds: | 20260505 14:44:22.994 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:44:22.994 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:22.994 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:22.994 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:22.994 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 14:44:22.994 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:22.994 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:22.994 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:44:22.994 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:22.994 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:22.995 | INFO | Uploaded 36GiB to GPU in 2.15655s (16.6933GiB/s) | 20260505 14:44:22.995 | INFO | Downloaded 34GiB to GPU in 2.03657s (16.6947GiB/s) | 20260505 14:44:22.995 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 14:44:22.995 | INFO | Total mem usage: 15.3 GiB | 20260505 14:44:22.995 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 14:44:22.995 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.76%) | 20260505 14:44:22.995 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 14:44:22.995 | INFO | High Water Breakdown: | 20260505 14:44:22.995 | INFO | 61% : 8.68 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 14:44:22.995 | INFO | 38.8% : 5.52 GiB ( 83.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 14:44:22.995 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 14:44:22.995 | INFO | ------------------------------------------ | 20260505 14:44:22.995 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:44:22.995 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 14:44:22.995 | INFO | Total mem usage: 26.5 GiB | 20260505 14:44:22.995 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 14:44:22.995 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 14:44:22.995 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 14:44:22.995 | INFO | High Water Breakdown: | 20260505 14:44:22.995 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:44:22.995 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:44:22.995 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:44:22.995 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:44:22.995 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:44:22.995 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:44:22.995 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 14:44:22.995 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 14:44:22.995 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:44:22.995 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 14:44:22.995 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 14:44:22.995 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 14:44:22.995 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 14:44:22.995 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 14:44:22.995 | INFO | ------------------------------------------ | 20260505 14:44:22.995 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:44:22.995 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 14:44:22.995 | INFO | Total mem usage: 25.6 GiB | 20260505 14:44:22.995 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 14:44:22.995 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 14:44:22.995 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 14:44:22.995 | INFO | High Water Breakdown: | 20260505 14:44:22.995 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 14:44:22.995 | INFO | ------------------------------------------ | 20260505 14:44:22.995 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:44:22.995 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 14:44:22.996 | INFO | Total mem usage: 15.7 GiB | 20260505 14:44:22.996 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 14:44:22.996 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 14:44:22.996 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 14:44:22.996 | INFO | High Water Breakdown: | 20260505 14:44:22.996 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 14:44:22.996 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 14:44:22.996 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 14:44:22.996 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 14:44:22.996 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 14:44:22.996 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 14:44:22.996 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 14:44:22.996 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 14:44:22.996 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 14:44:22.996 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:44:22.996 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 14:44:22.996 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:44:22.996 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 14:44:22.996 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 14:44:22.996 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 14:44:22.996 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 14:44:22.996 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:44:22.996 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 14:44:22.996 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:44:22.996 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:44:22.996 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:44:22.996 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 14:44:22.996 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 14:44:22.996 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:44:22.996 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 14:44:22.996 | INFO | ------------------------------------------ | 20260505 14:44:22.996 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:44:22.996 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 14:44:22.996 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:44:22.996 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:44:22.996 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.59e+01, 1.36e+01 / 1.99e+00 | 20260505 14:44:22.996 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 9.98e-02, 1.09e-02 / 4.08e-03 | 20260505 14:44:22.996 | INFO | PerfReport: 11.76%, Baseline -- 3.73e+00 / 4.49e+00, 5.68e-01 / 1.98e-01 | 20260505 14:44:22.996 | INFO | PerfReport: 3.29%, Histogram -- 1.05e+00 / 1.52e+00, 1.96e-01 / 9.16e-02 | 20260505 14:44:22.996 | INFO | PerfReport: 15.97%, DME -- 4.06e+01 / 4.52e+01, 4.45e+00 / 2.23e+00 | 20260505 14:44:22.996 | INFO | PerfReport: 56.44%, FrameLabeling -- 1.79e+01 / 1.98e+01, 1.35e+00 / 1.73e+00 | 20260505 14:44:22.996 | INFO | PerfReport: 7.47%, PulseAccumulating -- 2.37e+00 / 2.92e+00, 4.53e-01 / 1.88e-01 | 20260505 14:44:22.996 | INFO | PerfReport: 3.53%, Metrics -- 1.12e+00 / 1.45e+00, 3.10e-01 / 1.60e-01 | 20260505 14:44:22.996 | INFO | PerfReport: 1.23%, PulseSerializing -- 3.92e-01 / 4.48e-01, 6.99e-02 / 3.52e-02 | 20260505 14:44:22.996 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 14:44:22.996 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:44:22.996 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:44:22.996 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260505 14:44:22.996 | INFO | PerfReport: 22.59%, Upload -- 2.27e+01 / 0.00e+00, 7.95e+00 / 0.00e+00 | 20260505 14:44:22.996 | INFO | PerfReport: 21.98%, Download -- 2.21e+01 / 0.00e+00, 7.96e+00 / 0.00e+00 | 20260505 14:44:22.996 | INFO | PerfReport: 31.56%, Compute -- 3.18e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 14:44:22.996 | INFO | PerfReport: 2.32%, LockUpload -- 2.34e+00 / 0.00e+00, 8.07e+00 / 0.00e+00 | 20260505 14:44:22.996 | INFO | PerfReport: 1.42%, LockDownload -- 1.42e+00 / 0.00e+00, 2.95e+00 / 0.00e+00 | 20260505 14:44:22.996 | INFO | PerfReport: 20.11%, LockCompute -- 2.02e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260505 14:44:22.996 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 2.36e-02 / 0.00e+00 | 20260505 14:44:23.132 | INFO | Starting a round of Baz IO | 20260505 14:44:26.346 | INFO | Populating RTMetrics report | 20260505 14:44:26.346 | INFO | RTMetrics report populated | 20260505 14:44:26.346 | INFO | Opening tmp RTMetrics json file | 20260505 14:44:26.363 | INFO | Serializing RTMetrics to json | 20260505 14:44:26.364 | INFO | Serialization done, swapping tmp file now... | 20260505 14:44:26.368 | INFO | RTMetrics done | 20260505 14:44:26.463 | INFO | Finished a round of Baz IO: Wrote 2.248 GiB in 3.33 seconds | 20260505 14:44:26.463 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.67% / 0.00% / 77.33% | 20260505 14:44:26.463 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.523674% / 7.94729e-06% / 8.34465e-05% | 20260505 14:44:26.463 | INFO | Baz IO overall: | 20260505 14:44:26.463 | INFO | BytesWritten : 2395688027550 | 20260505 14:44:26.463 | INFO | NumEvents : 1138769535994 | 20260505 14:44:28.127 | INFO | Analyzing chunk frames = [4022272,4022784) | 20260505 14:44:28.127 | INFO | MemInfo: { | 20260505 14:44:28.127 | INFO | "gpu" : " 17 GB", | 20260505 14:44:28.127 | INFO | "huge" : "", | 20260505 14:44:28.127 | INFO | "huge_shared" : " 27 GB", | 20260505 14:44:28.127 | INFO | "malloc" : " 14 GB", | 20260505 14:44:28.127 | INFO | "pinned" : " 29 GB" | 20260505 14:44:28.127 | INFO | } | 20260505 14:44:28.127 | INFO | Perf report for the last 5.13259 seconds: | 20260505 14:44:28.127 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:44:28.127 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:28.127 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:28.127 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:28.127 | INFO | ANALYSIS : 64.2%, 3.0, 99.5ms, 0 | 20260505 14:44:28.127 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:28.127 | INFO | RT_METRICS : 45.5%, 1.0, 23.8ms, 0 | 20260505 14:44:28.127 | INFO | PRE_HQ : 60.9%, 1.8, 57.3ms, 0 | 20260505 14:44:28.127 | INFO | BAZWRITER : 64.1%, 4.0, 131.8ms, 0 | 20260505 14:44:28.127 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 14:44:28.127 | INFO | Uploaded 36GiB to GPU in 2.20374s (16.3358GiB/s) | 20260505 14:44:28.127 | INFO | Downloaded 38GiB to GPU in 2.26003s (16.8139GiB/s) | 20260505 14:44:33.259 | INFO | Analyzing chunk frames = [4022784,4023296) | 20260505 14:44:33.259 | INFO | MemInfo: { | 20260505 14:44:33.259 | INFO | "gpu" : " 17 GB", | 20260505 14:44:33.259 | INFO | "huge" : "", | 20260505 14:44:33.259 | INFO | "huge_shared" : " 27 GB", | 20260505 14:44:33.259 | INFO | "malloc" : " 14 GB", | 20260505 14:44:33.259 | INFO | "pinned" : " 29 GB" | 20260505 14:44:33.260 | INFO | } | 20260505 14:44:33.260 | INFO | Perf report for the last 5.13269 seconds: | 20260505 14:44:33.260 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:44:33.260 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:33.260 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:33.260 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:33.260 | INFO | ANALYSIS : 63.5%, 3.0, 99.0ms, 0 | 20260505 14:44:33.260 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:33.260 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:33.260 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:44:33.260 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:33.260 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:33.260 | INFO | Uploaded 36GiB to GPU in 2.19418s (16.4071GiB/s) | 20260505 14:44:33.260 | INFO | Downloaded 34GiB to GPU in 2.02736s (16.7706GiB/s) | 20260505 14:44:38.403 | INFO | Analyzing chunk frames = [4023296,4023808) | 20260505 14:44:38.403 | INFO | MemInfo: { | 20260505 14:44:38.403 | INFO | "gpu" : " 17 GB", | 20260505 14:44:38.403 | INFO | "huge" : "", | 20260505 14:44:38.403 | INFO | "huge_shared" : " 27 GB", | 20260505 14:44:38.403 | INFO | "malloc" : " 14 GB", | 20260505 14:44:38.403 | INFO | "pinned" : " 29 GB" | 20260505 14:44:38.403 | INFO | } | 20260505 14:44:38.404 | INFO | Perf report for the last 5.14397 seconds: | 20260505 14:44:38.404 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:44:38.404 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:38.404 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:38.404 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:38.404 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 14:44:38.404 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:38.404 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:38.404 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:44:38.404 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:38.404 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:38.404 | INFO | Uploaded 36GiB to GPU in 2.21869s (16.2258GiB/s) | 20260505 14:44:38.404 | INFO | Downloaded 34GiB to GPU in 2.12162s (16.0255GiB/s) | 20260505 14:44:43.550 | INFO | Analyzing chunk frames = [4023808,4024320) | 20260505 14:44:43.550 | INFO | MemInfo: { | 20260505 14:44:43.550 | INFO | "gpu" : " 17 GB", | 20260505 14:44:43.550 | INFO | "huge" : "", | 20260505 14:44:43.550 | INFO | "huge_shared" : " 27 GB", | 20260505 14:44:43.550 | INFO | "malloc" : " 14 GB", | 20260505 14:44:43.550 | INFO | "pinned" : " 29 GB" | 20260505 14:44:43.550 | INFO | } | 20260505 14:44:43.550 | INFO | Perf report for the last 5.14616 seconds: | 20260505 14:44:43.550 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:44:43.550 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:43.550 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:43.550 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:43.550 | INFO | ANALYSIS : 62.6%, 3.0, 97.4ms, 0 | 20260505 14:44:43.550 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:43.550 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:43.550 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:44:43.550 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:43.550 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:43.550 | INFO | Uploaded 36GiB to GPU in 2.18418s (16.4821GiB/s) | 20260505 14:44:43.550 | INFO | Downloaded 34GiB to GPU in 2.12814s (15.9764GiB/s) | 20260505 14:44:48.690 | INFO | Analyzing chunk frames = [4024320,4024832) | 20260505 14:44:48.690 | INFO | MemInfo: { | 20260505 14:44:48.690 | INFO | "gpu" : " 17 GB", | 20260505 14:44:48.690 | INFO | "huge" : "", | 20260505 14:44:48.690 | INFO | "huge_shared" : " 27 GB", | 20260505 14:44:48.690 | INFO | "malloc" : " 14 GB", | 20260505 14:44:48.690 | INFO | "pinned" : " 29 GB" | 20260505 14:44:48.690 | INFO | } | 20260505 14:44:48.690 | INFO | Perf report for the last 5.14066 seconds: | 20260505 14:44:48.690 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:44:48.690 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:48.690 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:48.691 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:48.691 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 14:44:48.691 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:48.691 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:48.691 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:44:48.691 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:48.691 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:48.691 | INFO | Uploaded 36GiB to GPU in 2.18091s (16.5069GiB/s) | 20260505 14:44:48.691 | INFO | Downloaded 34GiB to GPU in 2.12804s (15.9771GiB/s) | 20260505 14:44:53.824 | INFO | Analyzing chunk frames = [4024832,4025344) | 20260505 14:44:53.824 | INFO | MemInfo: { | 20260505 14:44:53.824 | INFO | "gpu" : " 17 GB", | 20260505 14:44:53.824 | INFO | "huge" : "", | 20260505 14:44:53.824 | INFO | "huge_shared" : " 27 GB", | 20260505 14:44:53.824 | INFO | "malloc" : " 14 GB", | 20260505 14:44:53.824 | INFO | "pinned" : " 29 GB" | 20260505 14:44:53.824 | INFO | } | 20260505 14:44:53.824 | INFO | Perf report for the last 5.13348 seconds: | 20260505 14:44:53.824 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:44:53.824 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:53.824 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:53.824 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:53.824 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 14:44:53.824 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:53.824 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:53.824 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:44:53.824 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:53.824 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:53.824 | INFO | Uploaded 36GiB to GPU in 2.14034s (16.8198GiB/s) | 20260505 14:44:53.824 | INFO | Downloaded 34GiB to GPU in 2.03094s (16.741GiB/s) | 20260505 14:44:58.960 | INFO | Analyzing chunk frames = [4025344,4025856) | 20260505 14:44:58.960 | INFO | MemInfo: { | 20260505 14:44:58.960 | INFO | "gpu" : " 17 GB", | 20260505 14:44:58.960 | INFO | "huge" : "", | 20260505 14:44:58.960 | INFO | "huge_shared" : " 27 GB", | 20260505 14:44:58.960 | INFO | "malloc" : " 14 GB", | 20260505 14:44:58.960 | INFO | "pinned" : " 29 GB" | 20260505 14:44:58.960 | INFO | } | 20260505 14:44:58.960 | INFO | Perf report for the last 5.13633 seconds: | 20260505 14:44:58.960 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:44:58.960 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:58.960 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:58.960 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:58.960 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 14:44:58.960 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:58.960 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:58.960 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 14:44:58.960 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:58.960 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:44:58.961 | INFO | Uploaded 36GiB to GPU in 2.10561s (17.0972GiB/s) | 20260505 14:44:58.961 | INFO | Downloaded 34GiB to GPU in 2.17616s (15.6238GiB/s) | 20260505 14:45:04.111 | INFO | Analyzing chunk frames = [4025856,4026368) | 20260505 14:45:04.111 | INFO | MemInfo: { | 20260505 14:45:04.111 | INFO | "gpu" : " 17 GB", | 20260505 14:45:04.111 | INFO | "huge" : "", | 20260505 14:45:04.111 | INFO | "huge_shared" : " 27 GB", | 20260505 14:45:04.111 | INFO | "malloc" : " 14 GB", | 20260505 14:45:04.111 | INFO | "pinned" : " 29 GB" | 20260505 14:45:04.111 | INFO | } | 20260505 14:45:04.111 | INFO | Perf report for the last 5.15115 seconds: | 20260505 14:45:04.111 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:45:04.111 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:04.112 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:04.112 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:04.112 | INFO | ANALYSIS : 63.3%, 3.0, 99.2ms, 0 | 20260505 14:45:04.112 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:04.112 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:04.112 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 14:45:04.112 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:04.112 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:04.112 | INFO | Uploaded 36GiB to GPU in 2.2084s (16.3014GiB/s) | 20260505 14:45:04.112 | INFO | Downloaded 34GiB to GPU in 2.10309s (16.1667GiB/s) | 20260505 14:45:04.244 | INFO | Starting a round of Baz IO | 20260505 14:45:07.460 | INFO | Populating RTMetrics report | 20260505 14:45:07.460 | INFO | RTMetrics report populated | 20260505 14:45:07.460 | INFO | Opening tmp RTMetrics json file | 20260505 14:45:07.492 | INFO | Serializing RTMetrics to json | 20260505 14:45:07.527 | INFO | Serialization done, swapping tmp file now... | 20260505 14:45:07.549 | INFO | RTMetrics done | 20260505 14:45:07.564 | INFO | Finished a round of Baz IO: Wrote 2.244 GiB in 3.319 seconds | 20260505 14:45:07.564 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.66% / 0.00% / 77.33% | 20260505 14:45:07.564 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.521541% / 0% / 8.74201e-05% | 20260505 14:45:07.564 | INFO | Baz IO overall: | 20260505 14:45:07.564 | INFO | BytesWritten : 2398098024260 | 20260505 14:45:07.564 | INFO | NumEvents : 1139904596750 | 20260505 14:45:09.241 | INFO | Analyzing chunk frames = [4026368,4026880) | 20260505 14:45:09.241 | INFO | MemInfo: { | 20260505 14:45:09.241 | INFO | "gpu" : " 17 GB", | 20260505 14:45:09.241 | INFO | "huge" : "", | 20260505 14:45:09.241 | INFO | "huge_shared" : " 27 GB", | 20260505 14:45:09.241 | INFO | "malloc" : " 14 GB", | 20260505 14:45:09.241 | INFO | "pinned" : " 29 GB" | 20260505 14:45:09.241 | INFO | } | 20260505 14:45:09.241 | INFO | Perf report for the last 5.12976 seconds: | 20260505 14:45:09.241 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:45:09.241 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:09.241 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:09.241 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:09.241 | INFO | ANALYSIS : 64.2%, 3.0, 99.5ms, 0 | 20260505 14:45:09.241 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:09.241 | INFO | RT_METRICS : 46.7%, 1.0, 24.4ms, 0 | 20260505 14:45:09.241 | INFO | PRE_HQ : 61.1%, 1.8, 57.4ms, 0 | 20260505 14:45:09.241 | INFO | BAZWRITER : 64.0%, 4.0, 131.0ms, 0 | 20260505 14:45:09.241 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 14:45:09.242 | INFO | Uploaded 36GiB to GPU in 2.1967s (16.3882GiB/s) | 20260505 14:45:09.242 | INFO | Downloaded 38GiB to GPU in 2.25942s (16.8185GiB/s) | 20260505 14:45:14.390 | INFO | Analyzing chunk frames = [4026880,4027392) | 20260505 14:45:14.390 | INFO | MemInfo: { | 20260505 14:45:14.390 | INFO | "gpu" : " 17 GB", | 20260505 14:45:14.390 | INFO | "huge" : "", | 20260505 14:45:14.390 | INFO | "huge_shared" : " 27 GB", | 20260505 14:45:14.390 | INFO | "malloc" : " 14 GB", | 20260505 14:45:14.390 | INFO | "pinned" : " 29 GB" | 20260505 14:45:14.390 | INFO | } | 20260505 14:45:14.390 | INFO | Perf report for the last 5.14869 seconds: | 20260505 14:45:14.390 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:45:14.390 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:14.390 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:14.390 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:14.390 | INFO | ANALYSIS : 63.1%, 3.0, 98.8ms, 0 | 20260505 14:45:14.390 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:14.390 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:14.390 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 14:45:14.390 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:14.390 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:14.390 | INFO | Uploaded 36GiB to GPU in 2.19147s (16.4273GiB/s) | 20260505 14:45:14.390 | INFO | Downloaded 34GiB to GPU in 2.02962s (16.7519GiB/s) | 20260505 14:45:19.518 | INFO | Analyzing chunk frames = [4027392,4027904) | 20260505 14:45:19.518 | INFO | MemInfo: { | 20260505 14:45:19.518 | INFO | "gpu" : " 17 GB", | 20260505 14:45:19.518 | INFO | "huge" : "", | 20260505 14:45:19.518 | INFO | "huge_shared" : " 27 GB", | 20260505 14:45:19.518 | INFO | "malloc" : " 14 GB", | 20260505 14:45:19.518 | INFO | "pinned" : " 29 GB" | 20260505 14:45:19.518 | INFO | } | 20260505 14:45:19.518 | INFO | Perf report for the last 5.12808 seconds: | 20260505 14:45:19.518 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:45:19.518 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:19.518 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:19.518 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:19.518 | INFO | ANALYSIS : 63.0%, 3.0, 97.8ms, 0 | 20260505 14:45:19.518 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:19.518 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:19.518 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:45:19.518 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:19.518 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:19.518 | INFO | Uploaded 36GiB to GPU in 2.16236s (16.6485GiB/s) | 20260505 14:45:19.518 | INFO | Downloaded 34GiB to GPU in 2.02985s (16.75GiB/s) | 20260505 14:45:24.661 | INFO | Analyzing chunk frames = [4027904,4028416) | 20260505 14:45:24.661 | INFO | MemInfo: { | 20260505 14:45:24.661 | INFO | "gpu" : " 17 GB", | 20260505 14:45:24.661 | INFO | "huge" : "", | 20260505 14:45:24.661 | INFO | "huge_shared" : " 27 GB", | 20260505 14:45:24.661 | INFO | "malloc" : " 14 GB", | 20260505 14:45:24.661 | INFO | "pinned" : " 29 GB" | 20260505 14:45:24.661 | INFO | } | 20260505 14:45:24.661 | INFO | Perf report for the last 5.14274 seconds: | 20260505 14:45:24.661 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:45:24.661 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:24.661 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:24.661 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:24.661 | INFO | ANALYSIS : 62.9%, 3.0, 97.8ms, 0 | 20260505 14:45:24.661 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:24.661 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:24.661 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 14:45:24.661 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:24.661 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:24.661 | INFO | Uploaded 36GiB to GPU in 2.20314s (16.3403GiB/s) | 20260505 14:45:24.661 | INFO | Downloaded 34GiB to GPU in 2.13412s (15.9317GiB/s) | 20260505 14:45:29.803 | INFO | Analyzing chunk frames = [4028416,4028928) | 20260505 14:45:29.803 | INFO | MemInfo: { | 20260505 14:45:29.803 | INFO | "gpu" : " 17 GB", | 20260505 14:45:29.803 | INFO | "huge" : "", | 20260505 14:45:29.803 | INFO | "huge_shared" : " 27 GB", | 20260505 14:45:29.803 | INFO | "malloc" : " 14 GB", | 20260505 14:45:29.803 | INFO | "pinned" : " 29 GB" | 20260505 14:45:29.803 | INFO | } | 20260505 14:45:29.803 | INFO | Perf report for the last 5.14261 seconds: | 20260505 14:45:29.803 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:45:29.803 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:29.803 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:29.804 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:29.804 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 14:45:29.804 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:29.804 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:29.804 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:45:29.804 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:29.804 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:29.804 | INFO | Uploaded 36GiB to GPU in 2.18501s (16.4759GiB/s) | 20260505 14:45:29.804 | INFO | Downloaded 34GiB to GPU in 2.12992s (15.9631GiB/s) | 20260505 14:45:34.942 | INFO | Analyzing chunk frames = [4028928,4029440) | 20260505 14:45:34.942 | INFO | MemInfo: { | 20260505 14:45:34.942 | INFO | "gpu" : " 17 GB", | 20260505 14:45:34.942 | INFO | "huge" : "", | 20260505 14:45:34.943 | INFO | "huge_shared" : " 27 GB", | 20260505 14:45:34.943 | INFO | "malloc" : " 14 GB", | 20260505 14:45:34.943 | INFO | "pinned" : " 29 GB" | 20260505 14:45:34.943 | INFO | } | 20260505 14:45:34.943 | INFO | Perf report for the last 5.13928 seconds: | 20260505 14:45:34.943 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:45:34.943 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:34.943 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:34.943 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:34.943 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 14:45:34.943 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:34.943 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:34.943 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 14:45:34.943 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:34.943 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:34.943 | INFO | Uploaded 36GiB to GPU in 2.15484s (16.7066GiB/s) | 20260505 14:45:34.943 | INFO | Downloaded 34GiB to GPU in 2.0365s (16.6953GiB/s) | 20260505 14:45:40.072 | INFO | Analyzing chunk frames = [4029440,4029952) | 20260505 14:45:40.072 | INFO | MemInfo: { | 20260505 14:45:40.072 | INFO | "gpu" : " 17 GB", | 20260505 14:45:40.072 | INFO | "huge" : "", | 20260505 14:45:40.072 | INFO | "huge_shared" : " 27 GB", | 20260505 14:45:40.072 | INFO | "malloc" : " 14 GB", | 20260505 14:45:40.072 | INFO | "pinned" : " 29 GB" | 20260505 14:45:40.072 | INFO | } | 20260505 14:45:40.072 | INFO | Perf report for the last 5.12933 seconds: | 20260505 14:45:40.072 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:45:40.072 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:40.072 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:40.072 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:40.072 | INFO | ANALYSIS : 63.6%, 3.0, 99.0ms, 0 | 20260505 14:45:40.072 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:40.072 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:40.072 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 14:45:40.072 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:40.072 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:40.072 | INFO | Uploaded 36GiB to GPU in 2.18142s (16.503GiB/s) | 20260505 14:45:40.072 | INFO | Downloaded 34GiB to GPU in 2.15733s (15.7602GiB/s) | 20260505 14:45:45.208 | INFO | Analyzing chunk frames = [4029952,4030464) | 20260505 14:45:45.208 | INFO | MemInfo: { | 20260505 14:45:45.208 | INFO | "gpu" : " 17 GB", | 20260505 14:45:45.208 | INFO | "huge" : "", | 20260505 14:45:45.208 | INFO | "huge_shared" : " 27 GB", | 20260505 14:45:45.208 | INFO | "malloc" : " 14 GB", | 20260505 14:45:45.208 | INFO | "pinned" : " 29 GB" | 20260505 14:45:45.208 | INFO | } | 20260505 14:45:45.209 | INFO | Perf report for the last 5.13645 seconds: | 20260505 14:45:45.209 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:45:45.209 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:45.209 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:45.209 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:45.209 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 14:45:45.209 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:45.209 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:45.209 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 14:45:45.209 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:45.209 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:45.209 | INFO | Uploaded 36GiB to GPU in 2.16791s (16.6058GiB/s) | 20260505 14:45:45.209 | INFO | Downloaded 34GiB to GPU in 2.10596s (16.1447GiB/s) | 20260505 14:45:45.209 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 14:45:45.209 | INFO | Total mem usage: 15.3 GiB | 20260505 14:45:45.209 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 14:45:45.209 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -7.12%) | 20260505 14:45:45.209 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 14:45:45.209 | INFO | High Water Breakdown: | 20260505 14:45:45.209 | INFO | 61.1% : 8.74 GiB ( 91.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 14:45:45.209 | INFO | 38.7% : 5.54 GiB ( 83.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 14:45:45.209 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 14:45:45.209 | INFO | ------------------------------------------ | 20260505 14:45:45.209 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:45:45.209 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 14:45:45.209 | INFO | Total mem usage: 26.5 GiB | 20260505 14:45:45.209 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 14:45:45.209 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 14:45:45.209 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 14:45:45.209 | INFO | High Water Breakdown: | 20260505 14:45:45.209 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:45:45.209 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:45:45.209 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:45:45.209 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:45:45.209 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:45:45.209 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:45:45.209 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 14:45:45.209 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 14:45:45.209 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:45:45.209 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 14:45:45.209 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 14:45:45.209 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 14:45:45.209 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 14:45:45.209 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 14:45:45.209 | INFO | ------------------------------------------ | 20260505 14:45:45.209 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:45:45.210 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 14:45:45.210 | INFO | Total mem usage: 25.6 GiB | 20260505 14:45:45.210 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 14:45:45.210 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 14:45:45.210 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 14:45:45.210 | INFO | High Water Breakdown: | 20260505 14:45:45.210 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 14:45:45.210 | INFO | ------------------------------------------ | 20260505 14:45:45.210 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:45:45.210 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 14:45:45.210 | INFO | Total mem usage: 15.7 GiB | 20260505 14:45:45.210 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 14:45:45.210 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 14:45:45.210 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 14:45:45.210 | INFO | High Water Breakdown: | 20260505 14:45:45.210 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 14:45:45.210 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 14:45:45.210 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 14:45:45.210 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 14:45:45.210 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 14:45:45.210 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 14:45:45.210 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 14:45:45.210 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 14:45:45.210 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 14:45:45.210 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:45:45.210 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 14:45:45.210 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:45:45.210 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 14:45:45.210 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 14:45:45.210 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 14:45:45.210 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 14:45:45.210 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:45:45.210 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 14:45:45.210 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:45:45.210 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:45:45.210 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:45:45.210 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 14:45:45.210 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 14:45:45.210 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:45:45.210 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 14:45:45.210 | INFO | ------------------------------------------ | 20260505 14:45:45.210 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:45:45.210 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 14:45:45.210 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:45:45.210 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:45:45.210 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.58e+01, 1.37e+01 / 1.88e+00 | 20260505 14:45:45.210 | INFO | PerfReport: 0.29%, AggregateStats -- 9.31e-02 / 1.00e-01, 1.04e-02 / 4.33e-03 | 20260505 14:45:45.210 | INFO | PerfReport: 11.81%, Baseline -- 3.75e+00 / 4.49e+00, 5.87e-01 / 2.07e-01 | 20260505 14:45:45.210 | INFO | PerfReport: 3.29%, Histogram -- 1.05e+00 / 1.49e+00, 1.95e-01 / 1.14e-01 | 20260505 14:45:45.210 | INFO | PerfReport: 15.95%, DME -- 4.05e+01 / 4.53e+01, 4.55e+00 / 2.03e+00 | 20260505 14:45:45.210 | INFO | PerfReport: 56.48%, FrameLabeling -- 1.79e+01 / 1.97e+01, 1.43e+00 / 1.79e+00 | 20260505 14:45:45.210 | INFO | PerfReport: 7.43%, PulseAccumulating -- 2.36e+00 / 2.92e+00, 4.55e-01 / 1.78e-01 | 20260505 14:45:45.210 | INFO | PerfReport: 3.51%, Metrics -- 1.12e+00 / 1.44e+00, 3.13e-01 / 1.53e-01 | 20260505 14:45:45.210 | INFO | PerfReport: 1.23%, PulseSerializing -- 3.92e-01 / 4.37e-01, 6.95e-02 / 2.67e-02 | 20260505 14:45:45.210 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 14:45:45.210 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:45:45.210 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:45:45.210 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260505 14:45:45.210 | INFO | PerfReport: 22.58%, Upload -- 2.27e+01 / 0.00e+00, 8.07e+00 / 0.00e+00 | 20260505 14:45:45.210 | INFO | PerfReport: 21.92%, Download -- 2.20e+01 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260505 14:45:45.210 | INFO | PerfReport: 31.60%, Compute -- 3.18e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 14:45:45.210 | INFO | PerfReport: 2.21%, LockUpload -- 2.23e+00 / 0.00e+00, 7.85e+00 / 0.00e+00 | 20260505 14:45:45.210 | INFO | PerfReport: 1.43%, LockDownload -- 1.44e+00 / 0.00e+00, 2.93e+00 / 0.00e+00 | 20260505 14:45:45.210 | INFO | PerfReport: 20.25%, LockCompute -- 2.04e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 14:45:45.210 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.68e-02 / 0.00e+00 | 20260505 14:45:45.340 | INFO | Starting a round of Baz IO | 20260505 14:45:48.577 | INFO | Populating RTMetrics report | 20260505 14:45:48.577 | INFO | RTMetrics report populated | 20260505 14:45:48.577 | INFO | Opening tmp RTMetrics json file | 20260505 14:45:48.590 | INFO | Serializing RTMetrics to json | 20260505 14:45:48.591 | INFO | Serialization done, swapping tmp file now... | 20260505 14:45:48.592 | INFO | RTMetrics done | 20260505 14:45:48.684 | INFO | Finished a round of Baz IO: Wrote 2.242 GiB in 3.344 seconds | 20260505 14:45:48.684 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.66% / 0.00% / 77.34% | 20260505 14:45:48.684 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.520806% / 0% / 5.96046e-05% | 20260505 14:45:48.684 | INFO | Baz IO overall: | 20260505 14:45:48.685 | INFO | BytesWritten : 2400505662045 | 20260505 14:45:48.685 | INFO | NumEvents : 1141038325263 | 20260505 14:45:50.363 | INFO | Analyzing chunk frames = [4030464,4030976) | 20260505 14:45:50.363 | INFO | MemInfo: { | 20260505 14:45:50.363 | INFO | "gpu" : " 17 GB", | 20260505 14:45:50.363 | INFO | "huge" : "", | 20260505 14:45:50.363 | INFO | "huge_shared" : " 27 GB", | 20260505 14:45:50.363 | INFO | "malloc" : " 14 GB", | 20260505 14:45:50.363 | INFO | "pinned" : " 29 GB" | 20260505 14:45:50.363 | INFO | } | 20260505 14:45:50.363 | INFO | Perf report for the last 5.15433 seconds: | 20260505 14:45:50.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:45:50.363 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:50.363 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:50.363 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:50.363 | INFO | ANALYSIS : 64.2%, 3.0, 100.0ms, 0 | 20260505 14:45:50.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:50.363 | INFO | RT_METRICS : 45.0%, 1.0, 23.7ms, 0 | 20260505 14:45:50.363 | INFO | PRE_HQ : 60.4%, 1.8, 57.3ms, 0 | 20260505 14:45:50.363 | INFO | BAZWRITER : 64.2%, 3.9, 130.1ms, 0 | 20260505 14:45:50.363 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 14:45:50.363 | INFO | Uploaded 36GiB to GPU in 2.17787s (16.5299GiB/s) | 20260505 14:45:50.363 | INFO | Downloaded 38GiB to GPU in 2.28231s (16.6498GiB/s) | 20260505 14:45:55.511 | INFO | Analyzing chunk frames = [4030976,4031488) | 20260505 14:45:55.511 | INFO | MemInfo: { | 20260505 14:45:55.511 | INFO | "gpu" : " 17 GB", | 20260505 14:45:55.511 | INFO | "huge" : "", | 20260505 14:45:55.511 | INFO | "huge_shared" : " 27 GB", | 20260505 14:45:55.511 | INFO | "malloc" : " 14 GB", | 20260505 14:45:55.511 | INFO | "pinned" : " 29 GB" | 20260505 14:45:55.511 | INFO | } | 20260505 14:45:55.511 | INFO | Perf report for the last 5.14823 seconds: | 20260505 14:45:55.511 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:45:55.511 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:55.511 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:55.511 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:55.511 | INFO | ANALYSIS : 63.4%, 3.0, 99.2ms, 0 | 20260505 14:45:55.511 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:55.511 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:55.511 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:45:55.511 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:55.511 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:45:55.511 | INFO | Uploaded 36GiB to GPU in 2.203s (16.3413GiB/s) | 20260505 14:45:55.511 | INFO | Downloaded 34GiB to GPU in 2.0634s (16.4776GiB/s) | 20260505 14:46:00.634 | INFO | Analyzing chunk frames = [4031488,4032000) | 20260505 14:46:00.634 | INFO | MemInfo: { | 20260505 14:46:00.634 | INFO | "gpu" : " 17 GB", | 20260505 14:46:00.634 | INFO | "huge" : "", | 20260505 14:46:00.634 | INFO | "huge_shared" : " 27 GB", | 20260505 14:46:00.634 | INFO | "malloc" : " 14 GB", | 20260505 14:46:00.634 | INFO | "pinned" : " 29 GB" | 20260505 14:46:00.634 | INFO | } | 20260505 14:46:00.634 | INFO | Perf report for the last 5.12313 seconds: | 20260505 14:46:00.634 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:46:00.634 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:00.634 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:00.634 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:00.634 | INFO | ANALYSIS : 63.0%, 3.0, 97.8ms, 0 | 20260505 14:46:00.634 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:00.634 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:00.634 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:46:00.634 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:00.634 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:00.635 | INFO | Uploaded 36GiB to GPU in 2.20244s (16.3455GiB/s) | 20260505 14:46:00.635 | INFO | Downloaded 34GiB to GPU in 2.04043s (16.6631GiB/s) | 20260505 14:46:05.769 | INFO | Analyzing chunk frames = [4032000,4032512) | 20260505 14:46:05.769 | INFO | MemInfo: { | 20260505 14:46:05.769 | INFO | "gpu" : " 17 GB", | 20260505 14:46:05.769 | INFO | "huge" : "", | 20260505 14:46:05.769 | INFO | "huge_shared" : " 27 GB", | 20260505 14:46:05.769 | INFO | "malloc" : " 14 GB", | 20260505 14:46:05.769 | INFO | "pinned" : " 29 GB" | 20260505 14:46:05.769 | INFO | } | 20260505 14:46:05.769 | INFO | Perf report for the last 5.13506 seconds: | 20260505 14:46:05.769 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:46:05.769 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:05.769 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:05.769 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:05.769 | INFO | ANALYSIS : 63.0%, 3.0, 97.7ms, 0 | 20260505 14:46:05.769 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:05.769 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:05.769 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 14:46:05.769 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:05.769 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:05.770 | INFO | Uploaded 36GiB to GPU in 2.18838s (16.4505GiB/s) | 20260505 14:46:05.770 | INFO | Downloaded 34GiB to GPU in 2.13155s (15.9508GiB/s) | 20260505 14:46:10.914 | INFO | Analyzing chunk frames = [4032512,4033024) | 20260505 14:46:10.914 | INFO | MemInfo: { | 20260505 14:46:10.914 | INFO | "gpu" : " 17 GB", | 20260505 14:46:10.914 | INFO | "huge" : "", | 20260505 14:46:10.914 | INFO | "huge_shared" : " 27 GB", | 20260505 14:46:10.914 | INFO | "malloc" : " 14 GB", | 20260505 14:46:10.915 | INFO | "pinned" : " 29 GB" | 20260505 14:46:10.915 | INFO | } | 20260505 14:46:10.915 | INFO | Perf report for the last 5.14525 seconds: | 20260505 14:46:10.915 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:46:10.915 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:10.915 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:10.915 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:10.915 | INFO | ANALYSIS : 62.7%, 3.0, 98.0ms, 0 | 20260505 14:46:10.915 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:10.915 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:10.915 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 14:46:10.915 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:10.915 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:10.915 | INFO | Uploaded 36GiB to GPU in 2.1676s (16.6082GiB/s) | 20260505 14:46:10.915 | INFO | Downloaded 34GiB to GPU in 2.12642s (15.9893GiB/s) | 20260505 14:46:16.055 | INFO | Analyzing chunk frames = [4033024,4033536) | 20260505 14:46:16.055 | INFO | MemInfo: { | 20260505 14:46:16.055 | INFO | "gpu" : " 17 GB", | 20260505 14:46:16.055 | INFO | "huge" : "", | 20260505 14:46:16.055 | INFO | "huge_shared" : " 27 GB", | 20260505 14:46:16.055 | INFO | "malloc" : " 14 GB", | 20260505 14:46:16.055 | INFO | "pinned" : " 29 GB" | 20260505 14:46:16.055 | INFO | } | 20260505 14:46:16.055 | INFO | Perf report for the last 5.1409 seconds: | 20260505 14:46:16.055 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:46:16.056 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:16.056 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:16.056 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:16.056 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 14:46:16.056 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:16.056 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:16.056 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:46:16.056 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:16.056 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:16.056 | INFO | Uploaded 36GiB to GPU in 2.19466s (16.4034GiB/s) | 20260505 14:46:16.056 | INFO | Downloaded 34GiB to GPU in 2.11217s (16.0972GiB/s) | 20260505 14:46:21.196 | INFO | Analyzing chunk frames = [4033536,4034048) | 20260505 14:46:21.196 | INFO | MemInfo: { | 20260505 14:46:21.196 | INFO | "gpu" : " 17 GB", | 20260505 14:46:21.196 | INFO | "huge" : "", | 20260505 14:46:21.196 | INFO | "huge_shared" : " 27 GB", | 20260505 14:46:21.196 | INFO | "malloc" : " 14 GB", | 20260505 14:46:21.196 | INFO | "pinned" : " 29 GB" | 20260505 14:46:21.196 | INFO | } | 20260505 14:46:21.196 | INFO | Perf report for the last 5.1406 seconds: | 20260505 14:46:21.196 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:46:21.196 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:21.196 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:21.196 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:21.196 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 14:46:21.196 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:21.196 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:21.196 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 14:46:21.196 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:21.196 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:21.196 | INFO | Uploaded 36GiB to GPU in 2.11896s (16.9895GiB/s) | 20260505 14:46:21.197 | INFO | Downloaded 34GiB to GPU in 2.17938s (15.6008GiB/s) | 20260505 14:46:26.339 | INFO | Analyzing chunk frames = [4034048,4034560) | 20260505 14:46:26.339 | INFO | MemInfo: { | 20260505 14:46:26.339 | INFO | "gpu" : " 17 GB", | 20260505 14:46:26.339 | INFO | "huge" : "", | 20260505 14:46:26.339 | INFO | "huge_shared" : " 27 GB", | 20260505 14:46:26.340 | INFO | "malloc" : " 14 GB", | 20260505 14:46:26.340 | INFO | "pinned" : " 29 GB" | 20260505 14:46:26.340 | INFO | } | 20260505 14:46:26.340 | INFO | Perf report for the last 5.14351 seconds: | 20260505 14:46:26.340 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:46:26.340 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:26.340 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:26.340 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:26.340 | INFO | ANALYSIS : 63.5%, 3.0, 99.3ms, 0 | 20260505 14:46:26.340 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:26.340 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:26.340 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 14:46:26.340 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:26.340 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:26.340 | INFO | Uploaded 36GiB to GPU in 2.21438s (16.2574GiB/s) | 20260505 14:46:26.340 | INFO | Downloaded 34GiB to GPU in 2.1065s (16.1405GiB/s) | 20260505 14:46:26.475 | INFO | Starting a round of Baz IO | 20260505 14:46:29.694 | INFO | Populating RTMetrics report | 20260505 14:46:29.695 | INFO | RTMetrics report populated | 20260505 14:46:29.695 | INFO | Opening tmp RTMetrics json file | 20260505 14:46:29.745 | INFO | Serializing RTMetrics to json | 20260505 14:46:29.748 | INFO | Serialization done, swapping tmp file now... | 20260505 14:46:29.750 | INFO | RTMetrics done | 20260505 14:46:29.817 | INFO | Finished a round of Baz IO: Wrote 2.243 GiB in 3.341 seconds | 20260505 14:46:29.817 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.66% / 0.00% / 77.34% | 20260505 14:46:29.817 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.520384% / 3.97364e-06% / 7.94729e-05% | 20260505 14:46:29.817 | INFO | Baz IO overall: | 20260505 14:46:29.817 | INFO | BytesWritten : 2402914006939 | 20260505 14:46:29.817 | INFO | NumEvents : 1142172731493 | 20260505 14:46:31.470 | INFO | Analyzing chunk frames = [4034560,4035072) | 20260505 14:46:31.470 | INFO | MemInfo: { | 20260505 14:46:31.470 | INFO | "gpu" : " 17 GB", | 20260505 14:46:31.470 | INFO | "huge" : "", | 20260505 14:46:31.470 | INFO | "huge_shared" : " 27 GB", | 20260505 14:46:31.470 | INFO | "malloc" : " 14 GB", | 20260505 14:46:31.470 | INFO | "pinned" : " 29 GB" | 20260505 14:46:31.470 | INFO | } | 20260505 14:46:31.470 | INFO | Perf report for the last 5.13044 seconds: | 20260505 14:46:31.470 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:46:31.470 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:31.470 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:31.470 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:31.470 | INFO | ANALYSIS : 64.3%, 3.0, 99.4ms, 0 | 20260505 14:46:31.470 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:31.470 | INFO | RT_METRICS : 46.1%, 1.0, 24.1ms, 0 | 20260505 14:46:31.470 | INFO | PRE_HQ : 60.9%, 1.8, 57.2ms, 0 | 20260505 14:46:31.470 | INFO | BAZWRITER : 64.5%, 4.0, 131.8ms, 0 | 20260505 14:46:31.470 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260505 14:46:31.470 | INFO | Uploaded 36GiB to GPU in 2.19818s (16.3772GiB/s) | 20260505 14:46:31.470 | INFO | Downloaded 38GiB to GPU in 2.26778s (16.7565GiB/s) | 20260505 14:46:36.618 | INFO | Analyzing chunk frames = [4035072,4035584) | 20260505 14:46:36.618 | INFO | MemInfo: { | 20260505 14:46:36.618 | INFO | "gpu" : " 17 GB", | 20260505 14:46:36.618 | INFO | "huge" : "", | 20260505 14:46:36.618 | INFO | "huge_shared" : " 27 GB", | 20260505 14:46:36.618 | INFO | "malloc" : " 14 GB", | 20260505 14:46:36.618 | INFO | "pinned" : " 29 GB" | 20260505 14:46:36.618 | INFO | } | 20260505 14:46:36.619 | INFO | Perf report for the last 5.14842 seconds: | 20260505 14:46:36.619 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:46:36.619 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:36.619 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:36.619 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:36.619 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 14:46:36.619 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:36.619 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:36.619 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:46:36.619 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:36.619 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:36.619 | INFO | Uploaded 36GiB to GPU in 2.17731s (16.5341GiB/s) | 20260505 14:46:36.619 | INFO | Downloaded 34GiB to GPU in 2.02588s (16.7828GiB/s) | 20260505 14:46:41.756 | INFO | Analyzing chunk frames = [4035584,4036096) | 20260505 14:46:41.756 | INFO | MemInfo: { | 20260505 14:46:41.756 | INFO | "gpu" : " 17 GB", | 20260505 14:46:41.756 | INFO | "huge" : "", | 20260505 14:46:41.756 | INFO | "huge_shared" : " 27 GB", | 20260505 14:46:41.756 | INFO | "malloc" : " 14 GB", | 20260505 14:46:41.756 | INFO | "pinned" : " 29 GB" | 20260505 14:46:41.756 | INFO | } | 20260505 14:46:41.756 | INFO | Perf report for the last 5.13763 seconds: | 20260505 14:46:41.756 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:46:41.756 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:41.756 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:41.756 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:41.756 | INFO | ANALYSIS : 62.9%, 3.0, 97.8ms, 0 | 20260505 14:46:41.756 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:41.756 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:41.756 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:46:41.756 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:41.756 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:41.756 | INFO | Uploaded 36GiB to GPU in 2.19815s (16.3774GiB/s) | 20260505 14:46:41.756 | INFO | Downloaded 34GiB to GPU in 2.03853s (16.6787GiB/s) | 20260505 14:46:46.887 | INFO | Analyzing chunk frames = [4036096,4036608) | 20260505 14:46:46.887 | INFO | MemInfo: { | 20260505 14:46:46.887 | INFO | "gpu" : " 17 GB", | 20260505 14:46:46.887 | INFO | "huge" : "", | 20260505 14:46:46.887 | INFO | "huge_shared" : " 27 GB", | 20260505 14:46:46.887 | INFO | "malloc" : " 14 GB", | 20260505 14:46:46.888 | INFO | "pinned" : " 29 GB" | 20260505 14:46:46.888 | INFO | } | 20260505 14:46:46.888 | INFO | Perf report for the last 5.13142 seconds: | 20260505 14:46:46.888 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:46:46.888 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:46.888 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:46.888 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:46.888 | INFO | ANALYSIS : 62.7%, 3.0, 97.3ms, 0 | 20260505 14:46:46.888 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:46.888 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:46.888 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 14:46:46.888 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:46.888 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:46.888 | INFO | Uploaded 36GiB to GPU in 2.18279s (16.4926GiB/s) | 20260505 14:46:46.888 | INFO | Downloaded 34GiB to GPU in 2.13181s (15.9489GiB/s) | 20260505 14:46:52.023 | INFO | Analyzing chunk frames = [4036608,4037120) | 20260505 14:46:52.023 | INFO | MemInfo: { | 20260505 14:46:52.023 | INFO | "gpu" : " 17 GB", | 20260505 14:46:52.023 | INFO | "huge" : "", | 20260505 14:46:52.023 | INFO | "huge_shared" : " 27 GB", | 20260505 14:46:52.023 | INFO | "malloc" : " 14 GB", | 20260505 14:46:52.023 | INFO | "pinned" : " 29 GB" | 20260505 14:46:52.023 | INFO | } | 20260505 14:46:52.023 | INFO | Perf report for the last 5.13525 seconds: | 20260505 14:46:52.023 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:46:52.023 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:52.023 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:52.023 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:52.023 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 14:46:52.023 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:52.023 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:52.023 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 14:46:52.023 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:52.023 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:52.023 | INFO | Uploaded 36GiB to GPU in 2.15655s (16.6933GiB/s) | 20260505 14:46:52.023 | INFO | Downloaded 34GiB to GPU in 2.1266s (15.988GiB/s) | 20260505 14:46:57.167 | INFO | Analyzing chunk frames = [4037120,4037632) | 20260505 14:46:57.167 | INFO | MemInfo: { | 20260505 14:46:57.167 | INFO | "gpu" : " 17 GB", | 20260505 14:46:57.167 | INFO | "huge" : "", | 20260505 14:46:57.167 | INFO | "huge_shared" : " 27 GB", | 20260505 14:46:57.167 | INFO | "malloc" : " 14 GB", | 20260505 14:46:57.167 | INFO | "pinned" : " 29 GB" | 20260505 14:46:57.167 | INFO | } | 20260505 14:46:57.167 | INFO | Perf report for the last 5.14395 seconds: | 20260505 14:46:57.167 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:46:57.167 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:57.167 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:57.167 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:57.167 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 14:46:57.167 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:57.167 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:57.167 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 14:46:57.167 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:57.167 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:46:57.167 | INFO | Uploaded 36GiB to GPU in 2.17709s (16.5358GiB/s) | 20260505 14:46:57.167 | INFO | Downloaded 34GiB to GPU in 2.11261s (16.0938GiB/s) | 20260505 14:47:02.312 | INFO | Analyzing chunk frames = [4037632,4038144) | 20260505 14:47:02.312 | INFO | MemInfo: { | 20260505 14:47:02.312 | INFO | "gpu" : " 17 GB", | 20260505 14:47:02.312 | INFO | "huge" : "", | 20260505 14:47:02.312 | INFO | "huge_shared" : " 27 GB", | 20260505 14:47:02.312 | INFO | "malloc" : " 14 GB", | 20260505 14:47:02.312 | INFO | "pinned" : " 29 GB" | 20260505 14:47:02.312 | INFO | } | 20260505 14:47:02.312 | INFO | Perf report for the last 5.14536 seconds: | 20260505 14:47:02.312 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:47:02.312 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:02.312 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:02.312 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:02.312 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 14:47:02.312 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:02.312 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:02.312 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 14:47:02.312 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:02.312 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:02.313 | INFO | Uploaded 36GiB to GPU in 2.1468s (16.7692GiB/s) | 20260505 14:47:02.313 | INFO | Downloaded 34GiB to GPU in 2.17237s (15.6511GiB/s) | 20260505 14:47:07.453 | INFO | Analyzing chunk frames = [4038144,4038656) | 20260505 14:47:07.454 | INFO | MemInfo: { | 20260505 14:47:07.454 | INFO | "gpu" : " 17 GB", | 20260505 14:47:07.454 | INFO | "huge" : "", | 20260505 14:47:07.454 | INFO | "huge_shared" : " 27 GB", | 20260505 14:47:07.454 | INFO | "malloc" : " 14 GB", | 20260505 14:47:07.454 | INFO | "pinned" : " 29 GB" | 20260505 14:47:07.454 | INFO | } | 20260505 14:47:07.454 | INFO | Perf report for the last 5.14156 seconds: | 20260505 14:47:07.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:47:07.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:07.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:07.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:07.454 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 14:47:07.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:07.454 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:07.454 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 14:47:07.454 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:07.454 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:07.454 | INFO | Uploaded 36GiB to GPU in 2.22493s (16.1803GiB/s) | 20260505 14:47:07.454 | INFO | Downloaded 34GiB to GPU in 2.10567s (16.1469GiB/s) | 20260505 14:47:07.454 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 14:47:07.454 | INFO | Total mem usage: 15.3 GiB | 20260505 14:47:07.454 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 14:47:07.454 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.86%) | 20260505 14:47:07.454 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 14:47:07.454 | INFO | High Water Breakdown: | 20260505 14:47:07.454 | INFO | 61% : 8.67 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 14:47:07.454 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 14:47:07.454 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 14:47:07.454 | INFO | ------------------------------------------ | 20260505 14:47:07.454 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:47:07.454 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 14:47:07.454 | INFO | Total mem usage: 26.5 GiB | 20260505 14:47:07.454 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 14:47:07.454 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 14:47:07.454 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 14:47:07.454 | INFO | High Water Breakdown: | 20260505 14:47:07.454 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:47:07.454 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:47:07.454 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:47:07.454 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:47:07.454 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:47:07.454 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:47:07.454 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 14:47:07.454 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 14:47:07.454 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:47:07.454 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 14:47:07.454 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 14:47:07.454 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 14:47:07.455 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 14:47:07.455 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 14:47:07.455 | INFO | ------------------------------------------ | 20260505 14:47:07.455 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:47:07.455 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 14:47:07.455 | INFO | Total mem usage: 25.6 GiB | 20260505 14:47:07.455 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 14:47:07.455 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 14:47:07.455 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 14:47:07.455 | INFO | High Water Breakdown: | 20260505 14:47:07.455 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 14:47:07.455 | INFO | ------------------------------------------ | 20260505 14:47:07.455 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:47:07.455 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 14:47:07.455 | INFO | Total mem usage: 15.7 GiB | 20260505 14:47:07.455 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 14:47:07.455 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 14:47:07.455 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 14:47:07.455 | INFO | High Water Breakdown: | 20260505 14:47:07.455 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 14:47:07.455 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 14:47:07.455 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 14:47:07.455 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 14:47:07.455 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 14:47:07.455 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 14:47:07.455 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 14:47:07.455 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 14:47:07.455 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 14:47:07.455 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:47:07.455 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 14:47:07.455 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:47:07.455 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 14:47:07.455 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 14:47:07.455 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 14:47:07.455 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 14:47:07.455 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:47:07.455 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 14:47:07.455 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:47:07.455 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:47:07.455 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:47:07.455 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 14:47:07.455 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 14:47:07.455 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:47:07.455 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 14:47:07.455 | INFO | ------------------------------------------ | 20260505 14:47:07.455 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:47:07.455 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 14:47:07.455 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:47:07.455 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:47:07.455 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.62e+01, 1.38e+01 / 1.87e+00 | 20260505 14:47:07.455 | INFO | PerfReport: 0.29%, AggregateStats -- 9.30e-02 / 9.92e-02, 1.07e-02 / 3.06e-03 | 20260505 14:47:07.455 | INFO | PerfReport: 11.78%, Baseline -- 3.74e+00 / 4.48e+00, 5.78e-01 / 1.97e-01 | 20260505 14:47:07.455 | INFO | PerfReport: 3.29%, Histogram -- 1.04e+00 / 1.50e+00, 1.94e-01 / 8.98e-02 | 20260505 14:47:07.455 | INFO | PerfReport: 15.99%, DME -- 4.06e+01 / 4.55e+01, 4.41e+00 / 1.93e+00 | 20260505 14:47:07.455 | INFO | PerfReport: 56.40%, FrameLabeling -- 1.79e+01 / 1.98e+01, 1.32e+00 / 1.53e+00 | 20260505 14:47:07.455 | INFO | PerfReport: 7.48%, PulseAccumulating -- 2.37e+00 / 2.92e+00, 4.54e-01 / 1.09e-01 | 20260505 14:47:07.455 | INFO | PerfReport: 3.52%, Metrics -- 1.12e+00 / 1.42e+00, 3.12e-01 / 1.60e-01 | 20260505 14:47:07.455 | INFO | PerfReport: 1.23%, PulseSerializing -- 3.92e-01 / 4.47e-01, 6.95e-02 / 3.53e-02 | 20260505 14:47:07.455 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 14:47:07.455 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:47:07.455 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:47:07.455 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260505 14:47:07.455 | INFO | PerfReport: 22.59%, Upload -- 2.27e+01 / 0.00e+00, 8.08e+00 / 0.00e+00 | 20260505 14:47:07.455 | INFO | PerfReport: 22.01%, Download -- 2.22e+01 / 0.00e+00, 8.05e+00 / 0.00e+00 | 20260505 14:47:07.455 | INFO | PerfReport: 31.53%, Compute -- 3.17e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 14:47:07.455 | INFO | PerfReport: 2.30%, LockUpload -- 2.31e+00 / 0.00e+00, 8.11e+00 / 0.00e+00 | 20260505 14:47:07.455 | INFO | PerfReport: 1.49%, LockDownload -- 1.50e+00 / 0.00e+00, 3.05e+00 / 0.00e+00 | 20260505 14:47:07.455 | INFO | PerfReport: 20.07%, LockCompute -- 2.02e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 14:47:07.455 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.97e-02 / 0.00e+00 | 20260505 14:47:07.588 | INFO | Starting a round of Baz IO | 20260505 14:47:10.798 | INFO | Populating RTMetrics report | 20260505 14:47:10.799 | INFO | RTMetrics report populated | 20260505 14:47:10.799 | INFO | Opening tmp RTMetrics json file | 20260505 14:47:10.818 | INFO | Serializing RTMetrics to json | 20260505 14:47:10.841 | INFO | Serialization done, swapping tmp file now... | 20260505 14:47:10.863 | INFO | RTMetrics done | 20260505 14:47:10.903 | INFO | Finished a round of Baz IO: Wrote 2.242 GiB in 3.314 seconds | 20260505 14:47:10.903 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.65% / 0.00% / 77.34% | 20260505 14:47:10.903 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.51686% / 7.94729e-06% / 5.96046e-05% | 20260505 14:47:10.903 | INFO | Baz IO overall: | 20260505 14:47:10.903 | INFO | BytesWritten : 2405320924236 | 20260505 14:47:10.903 | INFO | NumEvents : 1143306105151 | 20260505 14:47:12.582 | INFO | Analyzing chunk frames = [4038656,4039168) | 20260505 14:47:12.582 | INFO | MemInfo: { | 20260505 14:47:12.582 | INFO | "gpu" : " 17 GB", | 20260505 14:47:12.582 | INFO | "huge" : "", | 20260505 14:47:12.582 | INFO | "huge_shared" : " 27 GB", | 20260505 14:47:12.582 | INFO | "malloc" : " 14 GB", | 20260505 14:47:12.582 | INFO | "pinned" : " 29 GB" | 20260505 14:47:12.582 | INFO | } | 20260505 14:47:12.582 | INFO | Perf report for the last 5.12823 seconds: | 20260505 14:47:12.582 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:47:12.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:12.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:12.582 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:12.582 | INFO | ANALYSIS : 64.1%, 3.0, 99.5ms, 0 | 20260505 14:47:12.582 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:12.582 | INFO | RT_METRICS : 46.2%, 1.0, 24.2ms, 0 | 20260505 14:47:12.582 | INFO | PRE_HQ : 61.2%, 1.8, 57.5ms, 0 | 20260505 14:47:12.582 | INFO | BAZWRITER : 64.0%, 4.0, 130.6ms, 0 | 20260505 14:47:12.582 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 14:47:12.582 | INFO | Uploaded 36GiB to GPU in 2.19166s (16.4259GiB/s) | 20260505 14:47:12.582 | INFO | Downloaded 38GiB to GPU in 2.26676s (16.764GiB/s) | 20260505 14:47:17.720 | INFO | Analyzing chunk frames = [4039168,4039680) | 20260505 14:47:17.720 | INFO | MemInfo: { | 20260505 14:47:17.720 | INFO | "gpu" : " 17 GB", | 20260505 14:47:17.720 | INFO | "huge" : "", | 20260505 14:47:17.720 | INFO | "huge_shared" : " 27 GB", | 20260505 14:47:17.720 | INFO | "malloc" : " 14 GB", | 20260505 14:47:17.721 | INFO | "pinned" : " 29 GB" | 20260505 14:47:17.721 | INFO | } | 20260505 14:47:17.721 | INFO | Perf report for the last 5.13872 seconds: | 20260505 14:47:17.721 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:47:17.721 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:17.721 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:17.721 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:17.721 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 14:47:17.721 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:17.721 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:17.721 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:47:17.721 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:17.721 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:17.721 | INFO | Uploaded 36GiB to GPU in 2.196s (16.3934GiB/s) | 20260505 14:47:17.721 | INFO | Downloaded 34GiB to GPU in 2.03214s (16.7312GiB/s) | 20260505 14:47:22.867 | INFO | Analyzing chunk frames = [4039680,4040192) | 20260505 14:47:22.867 | INFO | MemInfo: { | 20260505 14:47:22.867 | INFO | "gpu" : " 17 GB", | 20260505 14:47:22.867 | INFO | "huge" : "", | 20260505 14:47:22.867 | INFO | "huge_shared" : " 27 GB", | 20260505 14:47:22.867 | INFO | "malloc" : " 14 GB", | 20260505 14:47:22.867 | INFO | "pinned" : " 29 GB" | 20260505 14:47:22.867 | INFO | } | 20260505 14:47:22.867 | INFO | Perf report for the last 5.14638 seconds: | 20260505 14:47:22.867 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:47:22.867 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:22.867 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:22.867 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:22.867 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 14:47:22.867 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:22.867 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:22.867 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 14:47:22.867 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:22.867 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:22.867 | INFO | Uploaded 36GiB to GPU in 2.2395s (16.075GiB/s) | 20260505 14:47:22.867 | INFO | Downloaded 34GiB to GPU in 2.09398s (16.237GiB/s) | 20260505 14:47:28.009 | INFO | Analyzing chunk frames = [4040192,4040704) | 20260505 14:47:28.009 | INFO | MemInfo: { | 20260505 14:47:28.009 | INFO | "gpu" : " 17 GB", | 20260505 14:47:28.009 | INFO | "huge" : "", | 20260505 14:47:28.009 | INFO | "huge_shared" : " 27 GB", | 20260505 14:47:28.009 | INFO | "malloc" : " 14 GB", | 20260505 14:47:28.009 | INFO | "pinned" : " 29 GB" | 20260505 14:47:28.009 | INFO | } | 20260505 14:47:28.009 | INFO | Perf report for the last 5.14184 seconds: | 20260505 14:47:28.009 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:47:28.009 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:28.009 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:28.009 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:28.009 | INFO | ANALYSIS : 62.6%, 3.0, 97.3ms, 0 | 20260505 14:47:28.009 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:28.009 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:28.009 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:47:28.009 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:28.009 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:28.009 | INFO | Uploaded 36GiB to GPU in 2.18502s (16.4758GiB/s) | 20260505 14:47:28.009 | INFO | Downloaded 34GiB to GPU in 2.12704s (15.9847GiB/s) | 20260505 14:47:33.143 | INFO | Analyzing chunk frames = [4040704,4041216) | 20260505 14:47:33.143 | INFO | MemInfo: { | 20260505 14:47:33.143 | INFO | "gpu" : " 17 GB", | 20260505 14:47:33.143 | INFO | "huge" : "", | 20260505 14:47:33.143 | INFO | "huge_shared" : " 27 GB", | 20260505 14:47:33.143 | INFO | "malloc" : " 14 GB", | 20260505 14:47:33.143 | INFO | "pinned" : " 29 GB" | 20260505 14:47:33.143 | INFO | } | 20260505 14:47:33.144 | INFO | Perf report for the last 5.13455 seconds: | 20260505 14:47:33.144 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:47:33.144 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:33.144 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:33.144 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:33.144 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260505 14:47:33.144 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:33.144 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:33.144 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:47:33.144 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:33.144 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:33.144 | INFO | Uploaded 36GiB to GPU in 2.177s (16.5365GiB/s) | 20260505 14:47:33.144 | INFO | Downloaded 34GiB to GPU in 2.13303s (15.9397GiB/s) | 20260505 14:47:38.282 | INFO | Analyzing chunk frames = [4041216,4041728) | 20260505 14:47:38.282 | INFO | MemInfo: { | 20260505 14:47:38.282 | INFO | "gpu" : " 17 GB", | 20260505 14:47:38.282 | INFO | "huge" : "", | 20260505 14:47:38.282 | INFO | "huge_shared" : " 27 GB", | 20260505 14:47:38.282 | INFO | "malloc" : " 14 GB", | 20260505 14:47:38.282 | INFO | "pinned" : " 29 GB" | 20260505 14:47:38.282 | INFO | } | 20260505 14:47:38.282 | INFO | Perf report for the last 5.13847 seconds: | 20260505 14:47:38.282 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:47:38.282 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:38.282 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:38.282 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:38.282 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 14:47:38.282 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:38.282 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:38.282 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 14:47:38.282 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:38.282 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:38.282 | INFO | Uploaded 36GiB to GPU in 2.19906s (16.3707GiB/s) | 20260505 14:47:38.282 | INFO | Downloaded 34GiB to GPU in 2.11375s (16.0852GiB/s) | 20260505 14:47:43.424 | INFO | Analyzing chunk frames = [4041728,4042240) | 20260505 14:47:43.424 | INFO | MemInfo: { | 20260505 14:47:43.424 | INFO | "gpu" : " 17 GB", | 20260505 14:47:43.424 | INFO | "huge" : "", | 20260505 14:47:43.424 | INFO | "huge_shared" : " 27 GB", | 20260505 14:47:43.424 | INFO | "malloc" : " 14 GB", | 20260505 14:47:43.425 | INFO | "pinned" : " 29 GB" | 20260505 14:47:43.425 | INFO | } | 20260505 14:47:43.425 | INFO | Perf report for the last 5.14263 seconds: | 20260505 14:47:43.425 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:47:43.425 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:43.425 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:43.425 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:43.425 | INFO | ANALYSIS : 63.2%, 3.0, 98.8ms, 0 | 20260505 14:47:43.425 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:43.425 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:43.425 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 14:47:43.425 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:43.425 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:43.425 | INFO | Uploaded 36GiB to GPU in 2.14906s (16.7515GiB/s) | 20260505 14:47:43.425 | INFO | Downloaded 34GiB to GPU in 2.17254s (15.6499GiB/s) | 20260505 14:47:48.567 | INFO | Analyzing chunk frames = [4042240,4042752) | 20260505 14:47:48.567 | INFO | MemInfo: { | 20260505 14:47:48.567 | INFO | "gpu" : " 17 GB", | 20260505 14:47:48.567 | INFO | "huge" : "", | 20260505 14:47:48.567 | INFO | "huge_shared" : " 27 GB", | 20260505 14:47:48.567 | INFO | "malloc" : " 14 GB", | 20260505 14:47:48.567 | INFO | "pinned" : " 29 GB" | 20260505 14:47:48.567 | INFO | } | 20260505 14:47:48.567 | INFO | Perf report for the last 5.14228 seconds: | 20260505 14:47:48.567 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:47:48.567 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:48.567 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:48.567 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:48.567 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 14:47:48.567 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:48.567 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:48.567 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 14:47:48.567 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:48.567 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:48.567 | INFO | Uploaded 36GiB to GPU in 2.21652s (16.2417GiB/s) | 20260505 14:47:48.567 | INFO | Downloaded 34GiB to GPU in 2.09547s (16.2255GiB/s) | 20260505 14:47:48.697 | INFO | Starting a round of Baz IO | 20260505 14:47:51.917 | INFO | Populating RTMetrics report | 20260505 14:47:51.917 | INFO | RTMetrics report populated | 20260505 14:47:51.917 | INFO | Opening tmp RTMetrics json file | 20260505 14:47:51.921 | INFO | Serializing RTMetrics to json | 20260505 14:47:51.924 | INFO | Serialization done, swapping tmp file now... | 20260505 14:47:51.927 | INFO | RTMetrics done | 20260505 14:47:52.027 | INFO | Finished a round of Baz IO: Wrote 2.241 GiB in 3.33 seconds | 20260505 14:47:52.027 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.65% / 0.00% / 77.35% | 20260505 14:47:52.027 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.515294% / 1.58946e-05% / 8.74201e-05% | 20260505 14:47:52.027 | INFO | Baz IO overall: | 20260505 14:47:52.027 | INFO | BytesWritten : 2407727538994 | 20260505 14:47:52.027 | INFO | NumEvents : 1144439360379 | 20260505 14:47:53.705 | INFO | Analyzing chunk frames = [4042752,4043264) | 20260505 14:47:53.705 | INFO | MemInfo: { | 20260505 14:47:53.705 | INFO | "gpu" : " 17 GB", | 20260505 14:47:53.705 | INFO | "huge" : "", | 20260505 14:47:53.705 | INFO | "huge_shared" : " 27 GB", | 20260505 14:47:53.705 | INFO | "malloc" : " 14 GB", | 20260505 14:47:53.705 | INFO | "pinned" : " 29 GB" | 20260505 14:47:53.705 | INFO | } | 20260505 14:47:53.705 | INFO | Perf report for the last 5.13786 seconds: | 20260505 14:47:53.705 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:47:53.705 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:53.705 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:53.705 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:53.705 | INFO | ANALYSIS : 64.1%, 3.0, 99.4ms, 0 | 20260505 14:47:53.705 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:53.705 | INFO | RT_METRICS : 45.1%, 1.0, 23.7ms, 0 | 20260505 14:47:53.705 | INFO | PRE_HQ : 61.1%, 1.8, 57.0ms, 0 | 20260505 14:47:53.705 | INFO | BAZWRITER : 64.1%, 4.0, 131.6ms, 0 | 20260505 14:47:53.705 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 14:47:53.705 | INFO | Uploaded 36GiB to GPU in 2.19049s (16.4346GiB/s) | 20260505 14:47:53.705 | INFO | Downloaded 38GiB to GPU in 2.26583s (16.7709GiB/s) | 20260505 14:47:58.836 | INFO | Analyzing chunk frames = [4043264,4043776) | 20260505 14:47:58.836 | INFO | MemInfo: { | 20260505 14:47:58.836 | INFO | "gpu" : " 17 GB", | 20260505 14:47:58.836 | INFO | "huge" : "", | 20260505 14:47:58.836 | INFO | "huge_shared" : " 27 GB", | 20260505 14:47:58.836 | INFO | "malloc" : " 14 GB", | 20260505 14:47:58.836 | INFO | "pinned" : " 29 GB" | 20260505 14:47:58.836 | INFO | } | 20260505 14:47:58.836 | INFO | Perf report for the last 5.1313 seconds: | 20260505 14:47:58.836 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:47:58.836 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:58.836 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:58.836 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:58.836 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 14:47:58.836 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:58.836 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:58.836 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:47:58.836 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:58.836 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:47:58.836 | INFO | Uploaded 36GiB to GPU in 2.19298s (16.416GiB/s) | 20260505 14:47:58.836 | INFO | Downloaded 34GiB to GPU in 2.03376s (16.7178GiB/s) | 20260505 14:48:03.974 | INFO | Analyzing chunk frames = [4043776,4044288) | 20260505 14:48:03.974 | INFO | MemInfo: { | 20260505 14:48:03.974 | INFO | "gpu" : " 17 GB", | 20260505 14:48:03.974 | INFO | "huge" : "", | 20260505 14:48:03.974 | INFO | "huge_shared" : " 27 GB", | 20260505 14:48:03.974 | INFO | "malloc" : " 14 GB", | 20260505 14:48:03.974 | INFO | "pinned" : " 29 GB" | 20260505 14:48:03.974 | INFO | } | 20260505 14:48:03.974 | INFO | Perf report for the last 5.13814 seconds: | 20260505 14:48:03.974 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:48:03.974 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:03.974 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:03.974 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:03.974 | INFO | ANALYSIS : 63.2%, 3.0, 98.3ms, 0 | 20260505 14:48:03.974 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:03.974 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:03.974 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:48:03.974 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:03.974 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:03.975 | INFO | Uploaded 36GiB to GPU in 2.18917s (16.4446GiB/s) | 20260505 14:48:03.975 | INFO | Downloaded 34GiB to GPU in 2.1231s (16.0143GiB/s) | 20260505 14:48:09.127 | INFO | Analyzing chunk frames = [4044288,4044800) | 20260505 14:48:09.127 | INFO | MemInfo: { | 20260505 14:48:09.127 | INFO | "gpu" : " 17 GB", | 20260505 14:48:09.127 | INFO | "huge" : "", | 20260505 14:48:09.127 | INFO | "huge_shared" : " 27 GB", | 20260505 14:48:09.128 | INFO | "malloc" : " 14 GB", | 20260505 14:48:09.128 | INFO | "pinned" : " 29 GB" | 20260505 14:48:09.128 | INFO | } | 20260505 14:48:09.128 | INFO | Perf report for the last 5.1533 seconds: | 20260505 14:48:09.128 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:48:09.128 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:09.128 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:09.128 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:09.128 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 14:48:09.128 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:09.128 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:09.128 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 14:48:09.128 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:09.128 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:09.128 | INFO | Uploaded 36GiB to GPU in 2.18624s (16.4666GiB/s) | 20260505 14:48:09.128 | INFO | Downloaded 34GiB to GPU in 2.13756s (15.906GiB/s) | 20260505 14:48:14.268 | INFO | Analyzing chunk frames = [4044800,4045312) | 20260505 14:48:14.268 | INFO | MemInfo: { | 20260505 14:48:14.268 | INFO | "gpu" : " 17 GB", | 20260505 14:48:14.268 | INFO | "huge" : "", | 20260505 14:48:14.268 | INFO | "huge_shared" : " 27 GB", | 20260505 14:48:14.268 | INFO | "malloc" : " 14 GB", | 20260505 14:48:14.268 | INFO | "pinned" : " 29 GB" | 20260505 14:48:14.268 | INFO | } | 20260505 14:48:14.268 | INFO | Perf report for the last 5.14054 seconds: | 20260505 14:48:14.268 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:48:14.268 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:14.268 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:14.268 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:14.268 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 14:48:14.268 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:14.268 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:14.268 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:48:14.268 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:14.268 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:14.268 | INFO | Uploaded 36GiB to GPU in 2.13701s (16.8459GiB/s) | 20260505 14:48:14.268 | INFO | Downloaded 34GiB to GPU in 2.05202s (16.5691GiB/s) | 20260505 14:48:19.399 | INFO | Analyzing chunk frames = [4045312,4045824) | 20260505 14:48:19.399 | INFO | MemInfo: { | 20260505 14:48:19.399 | INFO | "gpu" : " 17 GB", | 20260505 14:48:19.399 | INFO | "huge" : "", | 20260505 14:48:19.399 | INFO | "huge_shared" : " 27 GB", | 20260505 14:48:19.399 | INFO | "malloc" : " 14 GB", | 20260505 14:48:19.400 | INFO | "pinned" : " 29 GB" | 20260505 14:48:19.400 | INFO | } | 20260505 14:48:19.400 | INFO | Perf report for the last 5.13141 seconds: | 20260505 14:48:19.400 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:48:19.400 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:19.400 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:19.400 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:19.400 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 14:48:19.400 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:19.400 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:19.400 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 14:48:19.400 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:19.400 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:19.400 | INFO | Uploaded 36GiB to GPU in 2.1381s (16.8374GiB/s) | 20260505 14:48:19.400 | INFO | Downloaded 34GiB to GPU in 2.03732s (16.6886GiB/s) | 20260505 14:48:24.529 | INFO | Analyzing chunk frames = [4045824,4046336) | 20260505 14:48:24.529 | INFO | MemInfo: { | 20260505 14:48:24.529 | INFO | "gpu" : " 17 GB", | 20260505 14:48:24.529 | INFO | "huge" : "", | 20260505 14:48:24.529 | INFO | "huge_shared" : " 27 GB", | 20260505 14:48:24.529 | INFO | "malloc" : " 14 GB", | 20260505 14:48:24.529 | INFO | "pinned" : " 29 GB" | 20260505 14:48:24.529 | INFO | } | 20260505 14:48:24.530 | INFO | Perf report for the last 5.12997 seconds: | 20260505 14:48:24.530 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:48:24.530 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:24.530 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:24.530 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:24.530 | INFO | ANALYSIS : 63.7%, 3.0, 99.2ms, 0 | 20260505 14:48:24.530 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:24.530 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:24.530 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 14:48:24.530 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:24.530 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:24.530 | INFO | Uploaded 36GiB to GPU in 2.13819s (16.8367GiB/s) | 20260505 14:48:24.530 | INFO | Downloaded 34GiB to GPU in 2.18498s (15.5608GiB/s) | 20260505 14:48:29.678 | INFO | Analyzing chunk frames = [4046336,4046848) | 20260505 14:48:29.678 | INFO | MemInfo: { | 20260505 14:48:29.678 | INFO | "gpu" : " 17 GB", | 20260505 14:48:29.678 | INFO | "huge" : "", | 20260505 14:48:29.678 | INFO | "huge_shared" : " 27 GB", | 20260505 14:48:29.678 | INFO | "malloc" : " 14 GB", | 20260505 14:48:29.678 | INFO | "pinned" : " 29 GB" | 20260505 14:48:29.678 | INFO | } | 20260505 14:48:29.678 | INFO | Perf report for the last 5.14886 seconds: | 20260505 14:48:29.678 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:48:29.678 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:29.678 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:29.678 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:29.678 | INFO | ANALYSIS : 63.4%, 3.0, 99.2ms, 0 | 20260505 14:48:29.678 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:29.679 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:29.679 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:48:29.679 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:29.679 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:29.679 | INFO | Uploaded 36GiB to GPU in 2.15892s (16.675GiB/s) | 20260505 14:48:29.679 | INFO | Downloaded 34GiB to GPU in 2.11065s (16.1088GiB/s) | 20260505 14:48:29.679 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 14:48:29.679 | INFO | Total mem usage: 15.3 GiB | 20260505 14:48:29.679 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 14:48:29.679 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -7.19%) | 20260505 14:48:29.679 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 14:48:29.679 | INFO | High Water Breakdown: | 20260505 14:48:29.679 | INFO | 61.2% : 8.74 GiB ( 91.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 14:48:29.679 | INFO | 38.7% : 5.53 GiB ( 83.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 14:48:29.679 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 14:48:29.679 | INFO | ------------------------------------------ | 20260505 14:48:29.679 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:48:29.679 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 14:48:29.679 | INFO | Total mem usage: 26.5 GiB | 20260505 14:48:29.679 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 14:48:29.679 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 14:48:29.679 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 14:48:29.679 | INFO | High Water Breakdown: | 20260505 14:48:29.679 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:48:29.679 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:48:29.679 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:48:29.679 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:48:29.679 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:48:29.679 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:48:29.679 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 14:48:29.679 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 14:48:29.679 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:48:29.679 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 14:48:29.679 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 14:48:29.679 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 14:48:29.679 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 14:48:29.679 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 14:48:29.679 | INFO | ------------------------------------------ | 20260505 14:48:29.679 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:48:29.679 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 14:48:29.679 | INFO | Total mem usage: 25.6 GiB | 20260505 14:48:29.679 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 14:48:29.679 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 14:48:29.679 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 14:48:29.679 | INFO | High Water Breakdown: | 20260505 14:48:29.679 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 14:48:29.679 | INFO | ------------------------------------------ | 20260505 14:48:29.679 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:48:29.679 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 14:48:29.679 | INFO | Total mem usage: 15.7 GiB | 20260505 14:48:29.679 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 14:48:29.679 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 14:48:29.679 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 14:48:29.679 | INFO | High Water Breakdown: | 20260505 14:48:29.679 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 14:48:29.679 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 14:48:29.679 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 14:48:29.679 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 14:48:29.679 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 14:48:29.679 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 14:48:29.679 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 14:48:29.679 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 14:48:29.679 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 14:48:29.679 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:48:29.679 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 14:48:29.679 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:48:29.679 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 14:48:29.679 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 14:48:29.679 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 14:48:29.679 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 14:48:29.679 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:48:29.679 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 14:48:29.679 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:48:29.679 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:48:29.680 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:48:29.680 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 14:48:29.680 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 14:48:29.680 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:48:29.680 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 14:48:29.680 | INFO | ------------------------------------------ | 20260505 14:48:29.680 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:48:29.680 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 14:48:29.680 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:48:29.680 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:48:29.680 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.64e+01, 1.37e+01 / 1.90e+00 | 20260505 14:48:29.680 | INFO | PerfReport: 0.29%, AggregateStats -- 9.32e-02 / 1.00e-01, 1.19e-02 / 5.81e-03 | 20260505 14:48:29.680 | INFO | PerfReport: 11.79%, Baseline -- 3.75e+00 / 4.50e+00, 5.80e-01 / 1.88e-01 | 20260505 14:48:29.680 | INFO | PerfReport: 3.29%, Histogram -- 1.05e+00 / 1.50e+00, 1.96e-01 / 1.18e-01 | 20260505 14:48:29.680 | INFO | PerfReport: 15.99%, DME -- 4.07e+01 / 4.57e+01, 4.58e+00 / 2.02e+00 | 20260505 14:48:29.680 | INFO | PerfReport: 56.46%, FrameLabeling -- 1.79e+01 / 1.97e+01, 1.39e+00 / 1.39e+00 | 20260505 14:48:29.680 | INFO | PerfReport: 7.43%, PulseAccumulating -- 2.36e+00 / 2.91e+00, 4.53e-01 / 1.31e-01 | 20260505 14:48:29.680 | INFO | PerfReport: 3.51%, Metrics -- 1.12e+00 / 1.43e+00, 3.13e-01 / 1.57e-01 | 20260505 14:48:29.680 | INFO | PerfReport: 1.23%, PulseSerializing -- 3.92e-01 / 4.43e-01, 7.00e-02 / 3.53e-02 | 20260505 14:48:29.680 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 14:48:29.680 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:48:29.680 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:48:29.680 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260505 14:48:29.680 | INFO | PerfReport: 22.58%, Upload -- 2.27e+01 / 0.00e+00, 8.05e+00 / 0.00e+00 | 20260505 14:48:29.680 | INFO | PerfReport: 21.99%, Download -- 2.21e+01 / 0.00e+00, 8.03e+00 / 0.00e+00 | 20260505 14:48:29.680 | INFO | PerfReport: 31.59%, Compute -- 3.18e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 14:48:29.680 | INFO | PerfReport: 2.24%, LockUpload -- 2.25e+00 / 0.00e+00, 7.76e+00 / 0.00e+00 | 20260505 14:48:29.680 | INFO | PerfReport: 1.42%, LockDownload -- 1.42e+00 / 0.00e+00, 2.96e+00 / 0.00e+00 | 20260505 14:48:29.680 | INFO | PerfReport: 20.17%, LockCompute -- 2.03e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260505 14:48:29.680 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.70e-02 / 0.00e+00 | 20260505 14:48:29.812 | INFO | Starting a round of Baz IO | 20260505 14:48:33.033 | INFO | Populating RTMetrics report | 20260505 14:48:33.034 | INFO | RTMetrics report populated | 20260505 14:48:33.034 | INFO | Opening tmp RTMetrics json file | 20260505 14:48:33.048 | INFO | Serializing RTMetrics to json | 20260505 14:48:33.049 | INFO | Serialization done, swapping tmp file now... | 20260505 14:48:33.050 | INFO | RTMetrics done | 20260505 14:48:33.135 | INFO | Finished a round of Baz IO: Wrote 2.243 GiB in 3.322 seconds | 20260505 14:48:33.135 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.65% / 0.00% / 77.35% | 20260505 14:48:33.135 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.51655% / 1.19209e-05% / 8.74201e-05% | 20260505 14:48:33.135 | INFO | Baz IO overall: | 20260505 14:48:33.135 | INFO | BytesWritten : 2410136292588 | 20260505 14:48:33.135 | INFO | NumEvents : 1145574417304 | 20260505 14:48:34.816 | INFO | Analyzing chunk frames = [4046848,4047360) | 20260505 14:48:34.816 | INFO | MemInfo: { | 20260505 14:48:34.816 | INFO | "gpu" : " 17 GB", | 20260505 14:48:34.816 | INFO | "huge" : "", | 20260505 14:48:34.816 | INFO | "huge_shared" : " 27 GB", | 20260505 14:48:34.816 | INFO | "malloc" : " 14 GB", | 20260505 14:48:34.816 | INFO | "pinned" : " 29 GB" | 20260505 14:48:34.816 | INFO | } | 20260505 14:48:34.816 | INFO | Perf report for the last 5.13768 seconds: | 20260505 14:48:34.816 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:48:34.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:34.816 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:34.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:34.816 | INFO | ANALYSIS : 64.2%, 3.0, 99.9ms, 0 | 20260505 14:48:34.816 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:34.816 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260505 14:48:34.816 | INFO | PRE_HQ : 61.0%, 1.8, 57.0ms, 0 | 20260505 14:48:34.816 | INFO | BAZWRITER : 64.0%, 4.0, 131.5ms, 0 | 20260505 14:48:34.816 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 14:48:34.817 | INFO | Uploaded 36GiB to GPU in 2.20752s (16.3079GiB/s) | 20260505 14:48:34.817 | INFO | Downloaded 38GiB to GPU in 2.26429s (16.7823GiB/s) | 20260505 14:48:39.958 | INFO | Analyzing chunk frames = [4047360,4047872) | 20260505 14:48:39.958 | INFO | MemInfo: { | 20260505 14:48:39.958 | INFO | "gpu" : " 17 GB", | 20260505 14:48:39.958 | INFO | "huge" : "", | 20260505 14:48:39.958 | INFO | "huge_shared" : " 27 GB", | 20260505 14:48:39.958 | INFO | "malloc" : " 14 GB", | 20260505 14:48:39.958 | INFO | "pinned" : " 29 GB" | 20260505 14:48:39.958 | INFO | } | 20260505 14:48:39.958 | INFO | Perf report for the last 5.14237 seconds: | 20260505 14:48:39.959 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:48:39.959 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:39.959 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:39.959 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:39.959 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 14:48:39.959 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:39.959 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:39.959 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:48:39.959 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:39.959 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:39.959 | INFO | Uploaded 36GiB to GPU in 2.20066s (16.3587GiB/s) | 20260505 14:48:39.959 | INFO | Downloaded 34GiB to GPU in 2.05059s (16.5806GiB/s) | 20260505 14:48:45.095 | INFO | Analyzing chunk frames = [4047872,4048384) | 20260505 14:48:45.095 | INFO | MemInfo: { | 20260505 14:48:45.095 | INFO | "gpu" : " 17 GB", | 20260505 14:48:45.095 | INFO | "huge" : "", | 20260505 14:48:45.095 | INFO | "huge_shared" : " 27 GB", | 20260505 14:48:45.095 | INFO | "malloc" : " 14 GB", | 20260505 14:48:45.095 | INFO | "pinned" : " 29 GB" | 20260505 14:48:45.095 | INFO | } | 20260505 14:48:45.096 | INFO | Perf report for the last 5.13704 seconds: | 20260505 14:48:45.096 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:48:45.096 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:45.096 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:45.096 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:45.096 | INFO | ANALYSIS : 63.1%, 3.0, 98.1ms, 0 | 20260505 14:48:45.096 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:45.096 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:45.096 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:48:45.096 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:45.096 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:45.096 | INFO | Uploaded 36GiB to GPU in 2.18097s (16.5064GiB/s) | 20260505 14:48:45.096 | INFO | Downloaded 34GiB to GPU in 2.05867s (16.5156GiB/s) | 20260505 14:48:50.232 | INFO | Analyzing chunk frames = [4048384,4048896) | 20260505 14:48:50.232 | INFO | MemInfo: { | 20260505 14:48:50.232 | INFO | "gpu" : " 17 GB", | 20260505 14:48:50.232 | INFO | "huge" : "", | 20260505 14:48:50.232 | INFO | "huge_shared" : " 27 GB", | 20260505 14:48:50.232 | INFO | "malloc" : " 14 GB", | 20260505 14:48:50.232 | INFO | "pinned" : " 29 GB" | 20260505 14:48:50.232 | INFO | } | 20260505 14:48:50.232 | INFO | Perf report for the last 5.13678 seconds: | 20260505 14:48:50.232 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:48:50.232 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:50.232 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:50.232 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:50.232 | INFO | ANALYSIS : 63.0%, 3.0, 97.8ms, 0 | 20260505 14:48:50.232 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:50.232 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:50.232 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:48:50.232 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:50.232 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:50.233 | INFO | Uploaded 36GiB to GPU in 2.19833s (16.3761GiB/s) | 20260505 14:48:50.233 | INFO | Downloaded 34GiB to GPU in 2.13918s (15.894GiB/s) | 20260505 14:48:55.380 | INFO | Analyzing chunk frames = [4048896,4049408) | 20260505 14:48:55.380 | INFO | MemInfo: { | 20260505 14:48:55.380 | INFO | "gpu" : " 17 GB", | 20260505 14:48:55.380 | INFO | "huge" : "", | 20260505 14:48:55.380 | INFO | "huge_shared" : " 27 GB", | 20260505 14:48:55.380 | INFO | "malloc" : " 14 GB", | 20260505 14:48:55.380 | INFO | "pinned" : " 29 GB" | 20260505 14:48:55.380 | INFO | } | 20260505 14:48:55.380 | INFO | Perf report for the last 5.14741 seconds: | 20260505 14:48:55.380 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:48:55.380 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:55.380 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:55.380 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:55.380 | INFO | ANALYSIS : 62.6%, 3.0, 97.9ms, 0 | 20260505 14:48:55.380 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:55.380 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:55.380 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 14:48:55.380 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:55.380 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:48:55.380 | INFO | Uploaded 36GiB to GPU in 2.15876s (16.6762GiB/s) | 20260505 14:48:55.380 | INFO | Downloaded 34GiB to GPU in 2.12912s (15.969GiB/s) | 20260505 14:49:00.514 | INFO | Analyzing chunk frames = [4049408,4049920) | 20260505 14:49:00.514 | INFO | MemInfo: { | 20260505 14:49:00.514 | INFO | "gpu" : " 17 GB", | 20260505 14:49:00.514 | INFO | "huge" : "", | 20260505 14:49:00.514 | INFO | "huge_shared" : " 27 GB", | 20260505 14:49:00.514 | INFO | "malloc" : " 14 GB", | 20260505 14:49:00.514 | INFO | "pinned" : " 29 GB" | 20260505 14:49:00.514 | INFO | } | 20260505 14:49:00.514 | INFO | Perf report for the last 5.13461 seconds: | 20260505 14:49:00.514 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:49:00.514 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:00.514 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:00.514 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:00.514 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 14:49:00.515 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:00.515 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:00.515 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:49:00.515 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:00.515 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:00.515 | INFO | Uploaded 36GiB to GPU in 2.15506s (16.7049GiB/s) | 20260505 14:49:00.515 | INFO | Downloaded 34GiB to GPU in 2.04126s (16.6563GiB/s) | 20260505 14:49:05.651 | INFO | Analyzing chunk frames = [4049920,4050432) | 20260505 14:49:05.652 | INFO | MemInfo: { | 20260505 14:49:05.652 | INFO | "gpu" : " 17 GB", | 20260505 14:49:05.652 | INFO | "huge" : "", | 20260505 14:49:05.652 | INFO | "huge_shared" : " 27 GB", | 20260505 14:49:05.652 | INFO | "malloc" : " 14 GB", | 20260505 14:49:05.652 | INFO | "pinned" : " 29 GB" | 20260505 14:49:05.652 | INFO | } | 20260505 14:49:05.652 | INFO | Perf report for the last 5.13728 seconds: | 20260505 14:49:05.652 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:49:05.652 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:05.652 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:05.652 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:05.652 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 14:49:05.652 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:05.652 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:05.652 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 14:49:05.652 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:05.652 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:05.652 | INFO | Uploaded 36GiB to GPU in 2.11544s (17.0177GiB/s) | 20260505 14:49:05.652 | INFO | Downloaded 34GiB to GPU in 2.18158s (15.585GiB/s) | 20260505 14:49:10.785 | INFO | Analyzing chunk frames = [4050432,4050944) | 20260505 14:49:10.785 | INFO | MemInfo: { | 20260505 14:49:10.785 | INFO | "gpu" : " 17 GB", | 20260505 14:49:10.785 | INFO | "huge" : "", | 20260505 14:49:10.785 | INFO | "huge_shared" : " 27 GB", | 20260505 14:49:10.785 | INFO | "malloc" : " 14 GB", | 20260505 14:49:10.785 | INFO | "pinned" : " 29 GB" | 20260505 14:49:10.785 | INFO | } | 20260505 14:49:10.785 | INFO | Perf report for the last 5.13329 seconds: | 20260505 14:49:10.785 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:49:10.785 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:10.785 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:10.785 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:10.785 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 14:49:10.785 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:10.785 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:10.785 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:49:10.785 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:10.785 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:10.785 | INFO | Uploaded 36GiB to GPU in 2.21174s (16.2768GiB/s) | 20260505 14:49:10.785 | INFO | Downloaded 34GiB to GPU in 2.10862s (16.1243GiB/s) | 20260505 14:49:10.922 | INFO | Starting a round of Baz IO | 20260505 14:49:14.145 | INFO | Populating RTMetrics report | 20260505 14:49:14.145 | INFO | RTMetrics report populated | 20260505 14:49:14.145 | INFO | Opening tmp RTMetrics json file | 20260505 14:49:14.174 | INFO | Serializing RTMetrics to json | 20260505 14:49:14.217 | INFO | Serialization done, swapping tmp file now... | 20260505 14:49:14.225 | INFO | RTMetrics done | 20260505 14:49:14.259 | INFO | Finished a round of Baz IO: Wrote 2.241 GiB in 3.336 seconds | 20260505 14:49:14.259 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.64% / 0.00% / 77.35% | 20260505 14:49:14.259 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.514972% / 7.94729e-06% / 7.15256e-05% | 20260505 14:49:14.259 | INFO | Baz IO overall: | 20260505 14:49:14.259 | INFO | BytesWritten : 2412542940846 | 20260505 14:49:14.259 | INFO | NumEvents : 1146707723969 | 20260505 14:49:15.933 | INFO | Analyzing chunk frames = [4050944,4051456) | 20260505 14:49:15.933 | INFO | MemInfo: { | 20260505 14:49:15.933 | INFO | "gpu" : " 17 GB", | 20260505 14:49:15.933 | INFO | "huge" : "", | 20260505 14:49:15.933 | INFO | "huge_shared" : " 27 GB", | 20260505 14:49:15.933 | INFO | "malloc" : " 14 GB", | 20260505 14:49:15.933 | INFO | "pinned" : " 29 GB" | 20260505 14:49:15.933 | INFO | } | 20260505 14:49:15.933 | INFO | Perf report for the last 5.14783 seconds: | 20260505 14:49:15.933 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:49:15.933 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:15.933 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:15.933 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:15.933 | INFO | ANALYSIS : 64.2%, 3.0, 99.9ms, 0 | 20260505 14:49:15.933 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:15.933 | INFO | RT_METRICS : 46.5%, 1.0, 24.4ms, 0 | 20260505 14:49:15.933 | INFO | PRE_HQ : 60.6%, 1.8, 57.1ms, 0 | 20260505 14:49:15.933 | INFO | BAZWRITER : 64.2%, 3.9, 130.3ms, 0 | 20260505 14:49:15.933 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 14:49:15.933 | INFO | Uploaded 36GiB to GPU in 2.17343s (16.5637GiB/s) | 20260505 14:49:15.933 | INFO | Downloaded 38GiB to GPU in 2.28306s (16.6443GiB/s) | 20260505 14:49:21.078 | INFO | Analyzing chunk frames = [4051456,4051968) | 20260505 14:49:21.078 | INFO | MemInfo: { | 20260505 14:49:21.078 | INFO | "gpu" : " 17 GB", | 20260505 14:49:21.078 | INFO | "huge" : "", | 20260505 14:49:21.078 | INFO | "huge_shared" : " 27 GB", | 20260505 14:49:21.078 | INFO | "malloc" : " 14 GB", | 20260505 14:49:21.078 | INFO | "pinned" : " 29 GB" | 20260505 14:49:21.078 | INFO | } | 20260505 14:49:21.079 | INFO | Perf report for the last 5.14574 seconds: | 20260505 14:49:21.079 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:49:21.079 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:21.079 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:21.079 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:21.079 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 14:49:21.079 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:21.079 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:21.079 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:49:21.079 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:21.079 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:21.079 | INFO | Uploaded 36GiB to GPU in 2.20072s (16.3583GiB/s) | 20260505 14:49:21.079 | INFO | Downloaded 34GiB to GPU in 2.05827s (16.5187GiB/s) | 20260505 14:49:26.215 | INFO | Analyzing chunk frames = [4051968,4052480) | 20260505 14:49:26.215 | INFO | MemInfo: { | 20260505 14:49:26.215 | INFO | "gpu" : " 17 GB", | 20260505 14:49:26.215 | INFO | "huge" : "", | 20260505 14:49:26.215 | INFO | "huge_shared" : " 27 GB", | 20260505 14:49:26.216 | INFO | "malloc" : " 14 GB", | 20260505 14:49:26.216 | INFO | "pinned" : " 29 GB" | 20260505 14:49:26.216 | INFO | } | 20260505 14:49:26.216 | INFO | Perf report for the last 5.13705 seconds: | 20260505 14:49:26.216 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:49:26.216 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:26.216 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:26.216 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:26.216 | INFO | ANALYSIS : 62.8%, 3.0, 97.6ms, 0 | 20260505 14:49:26.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:26.216 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:26.216 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:49:26.216 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:26.216 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:26.216 | INFO | Uploaded 36GiB to GPU in 2.16748s (16.6092GiB/s) | 20260505 14:49:26.216 | INFO | Downloaded 34GiB to GPU in 2.03259s (16.7274GiB/s) | 20260505 14:49:31.340 | INFO | Analyzing chunk frames = [4052480,4052992) | 20260505 14:49:31.340 | INFO | MemInfo: { | 20260505 14:49:31.340 | INFO | "gpu" : " 17 GB", | 20260505 14:49:31.340 | INFO | "huge" : "", | 20260505 14:49:31.340 | INFO | "huge_shared" : " 27 GB", | 20260505 14:49:31.340 | INFO | "malloc" : " 14 GB", | 20260505 14:49:31.341 | INFO | "pinned" : " 29 GB" | 20260505 14:49:31.341 | INFO | } | 20260505 14:49:31.341 | INFO | Perf report for the last 5.12498 seconds: | 20260505 14:49:31.341 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:49:31.341 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:31.341 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:31.341 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:31.341 | INFO | ANALYSIS : 62.7%, 3.0, 97.1ms, 0 | 20260505 14:49:31.341 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:31.341 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:31.341 | INFO | PRE_HQ : 28.3%, 1.0, 14.6ms, 0 | 20260505 14:49:31.341 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:31.341 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:31.341 | INFO | Uploaded 36GiB to GPU in 2.1725s (16.5708GiB/s) | 20260505 14:49:31.341 | INFO | Downloaded 34GiB to GPU in 2.12681s (15.9864GiB/s) | 20260505 14:49:36.482 | INFO | Analyzing chunk frames = [4052992,4053504) | 20260505 14:49:36.482 | INFO | MemInfo: { | 20260505 14:49:36.482 | INFO | "gpu" : " 17 GB", | 20260505 14:49:36.482 | INFO | "huge" : "", | 20260505 14:49:36.482 | INFO | "huge_shared" : " 27 GB", | 20260505 14:49:36.482 | INFO | "malloc" : " 14 GB", | 20260505 14:49:36.482 | INFO | "pinned" : " 29 GB" | 20260505 14:49:36.482 | INFO | } | 20260505 14:49:36.482 | INFO | Perf report for the last 5.14181 seconds: | 20260505 14:49:36.482 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:49:36.482 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:36.482 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:36.482 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:36.483 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 14:49:36.483 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:36.483 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:36.483 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:49:36.483 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:36.483 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:36.483 | INFO | Uploaded 36GiB to GPU in 2.17445s (16.5559GiB/s) | 20260505 14:49:36.483 | INFO | Downloaded 34GiB to GPU in 2.12774s (15.9794GiB/s) | 20260505 14:49:41.632 | INFO | Analyzing chunk frames = [4053504,4054016) | 20260505 14:49:41.632 | INFO | MemInfo: { | 20260505 14:49:41.632 | INFO | "gpu" : " 17 GB", | 20260505 14:49:41.632 | INFO | "huge" : "", | 20260505 14:49:41.632 | INFO | "huge_shared" : " 27 GB", | 20260505 14:49:41.632 | INFO | "malloc" : " 14 GB", | 20260505 14:49:41.632 | INFO | "pinned" : " 29 GB" | 20260505 14:49:41.632 | INFO | } | 20260505 14:49:41.632 | INFO | Perf report for the last 5.14984 seconds: | 20260505 14:49:41.632 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:49:41.632 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:41.632 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:41.632 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:41.632 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 14:49:41.632 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:41.632 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:41.632 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 14:49:41.632 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:41.632 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:41.633 | INFO | Uploaded 36GiB to GPU in 2.16454s (16.6317GiB/s) | 20260505 14:49:41.633 | INFO | Downloaded 34GiB to GPU in 2.10773s (16.1311GiB/s) | 20260505 14:49:46.772 | INFO | Analyzing chunk frames = [4054016,4054528) | 20260505 14:49:46.772 | INFO | MemInfo: { | 20260505 14:49:46.772 | INFO | "gpu" : " 17 GB", | 20260505 14:49:46.772 | INFO | "huge" : "", | 20260505 14:49:46.772 | INFO | "huge_shared" : " 27 GB", | 20260505 14:49:46.772 | INFO | "malloc" : " 14 GB", | 20260505 14:49:46.772 | INFO | "pinned" : " 29 GB" | 20260505 14:49:46.772 | INFO | } | 20260505 14:49:46.772 | INFO | Perf report for the last 5.1395 seconds: | 20260505 14:49:46.772 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:49:46.772 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:46.772 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:46.772 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:46.772 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 14:49:46.772 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:46.772 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:46.772 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 14:49:46.772 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:46.772 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:46.772 | INFO | Uploaded 36GiB to GPU in 2.11868s (16.9917GiB/s) | 20260505 14:49:46.772 | INFO | Downloaded 34GiB to GPU in 2.1759s (15.6257GiB/s) | 20260505 14:49:51.903 | INFO | Analyzing chunk frames = [4054528,4055040) | 20260505 14:49:51.903 | INFO | MemInfo: { | 20260505 14:49:51.903 | INFO | "gpu" : " 17 GB", | 20260505 14:49:51.903 | INFO | "huge" : "", | 20260505 14:49:51.903 | INFO | "huge_shared" : " 27 GB", | 20260505 14:49:51.903 | INFO | "malloc" : " 14 GB", | 20260505 14:49:51.903 | INFO | "pinned" : " 29 GB" | 20260505 14:49:51.903 | INFO | } | 20260505 14:49:51.903 | INFO | Perf report for the last 5.13167 seconds: | 20260505 14:49:51.904 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:49:51.904 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:51.904 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:51.904 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:51.904 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 14:49:51.904 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:51.904 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:51.904 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:49:51.904 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:51.904 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:51.904 | INFO | Uploaded 36GiB to GPU in 2.21091s (16.2829GiB/s) | 20260505 14:49:51.904 | INFO | Downloaded 34GiB to GPU in 2.09549s (16.2253GiB/s) | 20260505 14:49:51.904 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 14:49:51.904 | INFO | Total mem usage: 15.3 GiB | 20260505 14:49:51.904 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 14:49:51.904 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.72%) | 20260505 14:49:51.904 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 14:49:51.904 | INFO | High Water Breakdown: | 20260505 14:49:51.904 | INFO | 61% : 8.67 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 14:49:51.904 | INFO | 38.9% : 5.53 GiB ( 83.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 14:49:51.904 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 14:49:51.904 | INFO | ------------------------------------------ | 20260505 14:49:51.904 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:49:51.904 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 14:49:51.904 | INFO | Total mem usage: 26.5 GiB | 20260505 14:49:51.904 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 14:49:51.904 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 14:49:51.904 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 14:49:51.904 | INFO | High Water Breakdown: | 20260505 14:49:51.904 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:49:51.904 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:49:51.904 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:49:51.904 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:49:51.904 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:49:51.904 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:49:51.904 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 14:49:51.904 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 14:49:51.904 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:49:51.904 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 14:49:51.904 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 14:49:51.904 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 14:49:51.904 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 14:49:51.904 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 14:49:51.904 | INFO | ------------------------------------------ | 20260505 14:49:51.904 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:49:51.904 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 14:49:51.904 | INFO | Total mem usage: 25.6 GiB | 20260505 14:49:51.904 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 14:49:51.904 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 14:49:51.904 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 14:49:51.904 | INFO | High Water Breakdown: | 20260505 14:49:51.904 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 14:49:51.904 | INFO | ------------------------------------------ | 20260505 14:49:51.904 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:49:51.904 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 14:49:51.904 | INFO | Total mem usage: 15.7 GiB | 20260505 14:49:51.904 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 14:49:51.904 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 14:49:51.904 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 14:49:51.904 | INFO | High Water Breakdown: | 20260505 14:49:51.904 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 14:49:51.904 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 14:49:51.904 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 14:49:51.904 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 14:49:51.904 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 14:49:51.904 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 14:49:51.904 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 14:49:51.904 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 14:49:51.904 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 14:49:51.904 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:49:51.904 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 14:49:51.905 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:49:51.905 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 14:49:51.905 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 14:49:51.905 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 14:49:51.905 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 14:49:51.905 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:49:51.905 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 14:49:51.905 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:49:51.905 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:49:51.905 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:49:51.905 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 14:49:51.905 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 14:49:51.905 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:49:51.905 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 14:49:51.905 | INFO | ------------------------------------------ | 20260505 14:49:51.905 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:49:51.905 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 14:49:51.905 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:49:51.905 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:49:51.905 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.62e+01, 1.36e+01 / 2.13e+00 | 20260505 14:49:51.905 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 1.00e-01, 1.06e-02 / 4.28e-03 | 20260505 14:49:51.905 | INFO | PerfReport: 11.78%, Baseline -- 3.75e+00 / 4.48e+00, 5.82e-01 / 2.18e-01 | 20260505 14:49:51.905 | INFO | PerfReport: 3.29%, Histogram -- 1.05e+00 / 1.50e+00, 1.97e-01 / 1.00e-01 | 20260505 14:49:51.905 | INFO | PerfReport: 15.97%, DME -- 4.06e+01 / 4.55e+01, 4.57e+00 / 2.27e+00 | 20260505 14:49:51.905 | INFO | PerfReport: 56.53%, FrameLabeling -- 1.80e+01 / 1.99e+01, 1.44e+00 / 1.55e+00 | 20260505 14:49:51.905 | INFO | PerfReport: 7.40%, PulseAccumulating -- 2.35e+00 / 2.91e+00, 4.47e-01 / 1.40e-01 | 20260505 14:49:51.905 | INFO | PerfReport: 3.50%, Metrics -- 1.11e+00 / 1.44e+00, 3.09e-01 / 1.77e-01 | 20260505 14:49:51.905 | INFO | PerfReport: 1.23%, PulseSerializing -- 3.91e-01 / 4.44e-01, 6.89e-02 / 3.31e-02 | 20260505 14:49:51.905 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 14:49:51.905 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:49:51.905 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:49:51.905 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260505 14:49:51.905 | INFO | PerfReport: 22.51%, Upload -- 2.27e+01 / 0.00e+00, 8.00e+00 / 0.00e+00 | 20260505 14:49:51.905 | INFO | PerfReport: 21.97%, Download -- 2.21e+01 / 0.00e+00, 8.00e+00 / 0.00e+00 | 20260505 14:49:51.905 | INFO | PerfReport: 31.59%, Compute -- 3.18e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 14:49:51.905 | INFO | PerfReport: 2.23%, LockUpload -- 2.24e+00 / 0.00e+00, 7.97e+00 / 0.00e+00 | 20260505 14:49:51.905 | INFO | PerfReport: 1.48%, LockDownload -- 1.49e+00 / 0.00e+00, 3.07e+00 / 0.00e+00 | 20260505 14:49:51.905 | INFO | PerfReport: 20.21%, LockCompute -- 2.03e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260505 14:49:51.905 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.85e-02 / 0.00e+00 | 20260505 14:49:52.040 | INFO | Starting a round of Baz IO | 20260505 14:49:55.257 | INFO | Populating RTMetrics report | 20260505 14:49:55.258 | INFO | RTMetrics report populated | 20260505 14:49:55.258 | INFO | Opening tmp RTMetrics json file | 20260505 14:49:55.275 | INFO | Serializing RTMetrics to json | 20260505 14:49:55.301 | INFO | Serialization done, swapping tmp file now... | 20260505 14:49:55.322 | INFO | RTMetrics done | 20260505 14:49:55.365 | INFO | Finished a round of Baz IO: Wrote 2.242 GiB in 3.324 seconds | 20260505 14:49:55.365 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.64% / 0.00% / 77.36% | 20260505 14:49:55.365 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.515552% / 0% / 7.15256e-05% | 20260505 14:49:55.365 | INFO | Baz IO overall: | 20260505 14:49:55.365 | INFO | BytesWritten : 2414950010703 | 20260505 14:49:55.365 | INFO | NumEvents : 1147841618794 | 20260505 14:49:57.042 | INFO | Analyzing chunk frames = [4055040,4055552) | 20260505 14:49:57.042 | INFO | MemInfo: { | 20260505 14:49:57.042 | INFO | "gpu" : " 17 GB", | 20260505 14:49:57.042 | INFO | "huge" : "", | 20260505 14:49:57.042 | INFO | "huge_shared" : " 27 GB", | 20260505 14:49:57.042 | INFO | "malloc" : " 14 GB", | 20260505 14:49:57.042 | INFO | "pinned" : " 29 GB" | 20260505 14:49:57.042 | INFO | } | 20260505 14:49:57.042 | INFO | Perf report for the last 5.13876 seconds: | 20260505 14:49:57.043 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:49:57.043 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:57.043 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:57.043 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:57.043 | INFO | ANALYSIS : 64.2%, 3.0, 99.8ms, 0 | 20260505 14:49:57.043 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:49:57.043 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 14:49:57.043 | INFO | PRE_HQ : 60.9%, 1.8, 57.1ms, 0 | 20260505 14:49:57.043 | INFO | BAZWRITER : 64.0%, 4.0, 130.2ms, 0 | 20260505 14:49:57.043 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 14:49:57.043 | INFO | Uploaded 36GiB to GPU in 2.21079s (16.2838GiB/s) | 20260505 14:49:57.043 | INFO | Downloaded 38GiB to GPU in 2.26531s (16.7748GiB/s) | 20260505 14:50:02.190 | INFO | Analyzing chunk frames = [4055552,4056064) | 20260505 14:50:02.190 | INFO | MemInfo: { | 20260505 14:50:02.190 | INFO | "gpu" : " 17 GB", | 20260505 14:50:02.190 | INFO | "huge" : "", | 20260505 14:50:02.190 | INFO | "huge_shared" : " 27 GB", | 20260505 14:50:02.190 | INFO | "malloc" : " 14 GB", | 20260505 14:50:02.190 | INFO | "pinned" : " 29 GB" | 20260505 14:50:02.190 | INFO | } | 20260505 14:50:02.190 | INFO | Perf report for the last 5.14787 seconds: | 20260505 14:50:02.190 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:50:02.190 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:02.190 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:02.190 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:02.190 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 14:50:02.190 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:02.190 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:02.190 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:50:02.190 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:02.190 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:02.190 | INFO | Uploaded 36GiB to GPU in 2.19751s (16.3822GiB/s) | 20260505 14:50:02.190 | INFO | Downloaded 34GiB to GPU in 2.0452s (16.6243GiB/s) | 20260505 14:50:07.325 | INFO | Analyzing chunk frames = [4056064,4056576) | 20260505 14:50:07.325 | INFO | MemInfo: { | 20260505 14:50:07.325 | INFO | "gpu" : " 17 GB", | 20260505 14:50:07.325 | INFO | "huge" : "", | 20260505 14:50:07.325 | INFO | "huge_shared" : " 27 GB", | 20260505 14:50:07.325 | INFO | "malloc" : " 14 GB", | 20260505 14:50:07.325 | INFO | "pinned" : " 29 GB" | 20260505 14:50:07.325 | INFO | } | 20260505 14:50:07.325 | INFO | Perf report for the last 5.13493 seconds: | 20260505 14:50:07.325 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:50:07.325 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:07.325 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:07.325 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:07.325 | INFO | ANALYSIS : 62.9%, 3.0, 97.7ms, 0 | 20260505 14:50:07.325 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:07.325 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:07.325 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:50:07.325 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:07.325 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:07.325 | INFO | Uploaded 36GiB to GPU in 2.17154s (16.5781GiB/s) | 20260505 14:50:07.325 | INFO | Downloaded 34GiB to GPU in 2.03412s (16.7148GiB/s) | 20260505 14:50:12.463 | INFO | Analyzing chunk frames = [4056576,4057088) | 20260505 14:50:12.463 | INFO | MemInfo: { | 20260505 14:50:12.463 | INFO | "gpu" : " 17 GB", | 20260505 14:50:12.463 | INFO | "huge" : "", | 20260505 14:50:12.463 | INFO | "huge_shared" : " 27 GB", | 20260505 14:50:12.463 | INFO | "malloc" : " 14 GB", | 20260505 14:50:12.463 | INFO | "pinned" : " 29 GB" | 20260505 14:50:12.463 | INFO | } | 20260505 14:50:12.463 | INFO | Perf report for the last 5.13773 seconds: | 20260505 14:50:12.463 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:50:12.463 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:12.463 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:12.463 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:12.463 | INFO | ANALYSIS : 62.9%, 3.0, 97.7ms, 0 | 20260505 14:50:12.463 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:12.463 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:12.463 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:50:12.463 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:12.463 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:12.463 | INFO | Uploaded 36GiB to GPU in 2.18536s (16.4733GiB/s) | 20260505 14:50:12.463 | INFO | Downloaded 34GiB to GPU in 2.13368s (15.9349GiB/s) | 20260505 14:50:17.601 | INFO | Analyzing chunk frames = [4057088,4057600) | 20260505 14:50:17.601 | INFO | MemInfo: { | 20260505 14:50:17.601 | INFO | "gpu" : " 17 GB", | 20260505 14:50:17.601 | INFO | "huge" : "", | 20260505 14:50:17.601 | INFO | "huge_shared" : " 27 GB", | 20260505 14:50:17.601 | INFO | "malloc" : " 14 GB", | 20260505 14:50:17.601 | INFO | "pinned" : " 29 GB" | 20260505 14:50:17.601 | INFO | } | 20260505 14:50:17.602 | INFO | Perf report for the last 5.1387 seconds: | 20260505 14:50:17.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:50:17.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:17.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:17.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:17.602 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 14:50:17.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:17.602 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:17.602 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 14:50:17.602 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:17.602 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:17.602 | INFO | Uploaded 36GiB to GPU in 2.15717s (16.6885GiB/s) | 20260505 14:50:17.602 | INFO | Downloaded 34GiB to GPU in 2.12944s (15.9666GiB/s) | 20260505 14:50:22.738 | INFO | Analyzing chunk frames = [4057600,4058112) | 20260505 14:50:22.738 | INFO | MemInfo: { | 20260505 14:50:22.738 | INFO | "gpu" : " 17 GB", | 20260505 14:50:22.738 | INFO | "huge" : "", | 20260505 14:50:22.738 | INFO | "huge_shared" : " 27 GB", | 20260505 14:50:22.738 | INFO | "malloc" : " 14 GB", | 20260505 14:50:22.738 | INFO | "pinned" : " 29 GB" | 20260505 14:50:22.738 | INFO | } | 20260505 14:50:22.738 | INFO | Perf report for the last 5.13668 seconds: | 20260505 14:50:22.738 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:50:22.738 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:22.738 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:22.738 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:22.738 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 14:50:22.738 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:22.738 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:22.738 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 14:50:22.738 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:22.738 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:22.739 | INFO | Uploaded 36GiB to GPU in 2.18922s (16.4442GiB/s) | 20260505 14:50:22.739 | INFO | Downloaded 34GiB to GPU in 2.11582s (16.0694GiB/s) | 20260505 14:50:27.886 | INFO | Analyzing chunk frames = [4058112,4058624) | 20260505 14:50:27.886 | INFO | MemInfo: { | 20260505 14:50:27.886 | INFO | "gpu" : " 17 GB", | 20260505 14:50:27.886 | INFO | "huge" : "", | 20260505 14:50:27.886 | INFO | "huge_shared" : " 27 GB", | 20260505 14:50:27.886 | INFO | "malloc" : " 14 GB", | 20260505 14:50:27.886 | INFO | "pinned" : " 29 GB" | 20260505 14:50:27.886 | INFO | } | 20260505 14:50:27.886 | INFO | Perf report for the last 5.14803 seconds: | 20260505 14:50:27.886 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:50:27.886 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:27.886 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:27.886 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:27.886 | INFO | ANALYSIS : 63.3%, 3.0, 99.0ms, 0 | 20260505 14:50:27.886 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:27.886 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:27.886 | INFO | PRE_HQ : 28.9%, 1.0, 15.1ms, 0 | 20260505 14:50:27.886 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:27.886 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:27.887 | INFO | Uploaded 36GiB to GPU in 2.11793s (16.9978GiB/s) | 20260505 14:50:27.887 | INFO | Downloaded 34GiB to GPU in 2.17914s (15.6025GiB/s) | 20260505 14:50:33.034 | INFO | Analyzing chunk frames = [4058624,4059136) | 20260505 14:50:33.034 | INFO | MemInfo: { | 20260505 14:50:33.034 | INFO | "gpu" : " 17 GB", | 20260505 14:50:33.034 | INFO | "huge" : "", | 20260505 14:50:33.034 | INFO | "huge_shared" : " 27 GB", | 20260505 14:50:33.034 | INFO | "malloc" : " 14 GB", | 20260505 14:50:33.034 | INFO | "pinned" : " 29 GB" | 20260505 14:50:33.034 | INFO | } | 20260505 14:50:33.034 | INFO | Perf report for the last 5.14818 seconds: | 20260505 14:50:33.034 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:50:33.034 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:33.034 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:33.035 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:33.035 | INFO | ANALYSIS : 63.2%, 3.0, 98.8ms, 0 | 20260505 14:50:33.035 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:33.035 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:33.035 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 14:50:33.035 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:33.035 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:33.035 | INFO | Uploaded 36GiB to GPU in 2.17491s (16.5524GiB/s) | 20260505 14:50:33.035 | INFO | Downloaded 34GiB to GPU in 2.03381s (16.7174GiB/s) | 20260505 14:50:33.168 | INFO | Starting a round of Baz IO | 20260505 14:50:36.378 | INFO | Populating RTMetrics report | 20260505 14:50:36.378 | INFO | RTMetrics report populated | 20260505 14:50:36.378 | INFO | Opening tmp RTMetrics json file | 20260505 14:50:36.423 | INFO | Serializing RTMetrics to json | 20260505 14:50:36.431 | INFO | Serialization done, swapping tmp file now... | 20260505 14:50:36.455 | INFO | RTMetrics done | 20260505 14:50:36.482 | INFO | Finished a round of Baz IO: Wrote 2.239 GiB in 3.314 seconds | 20260505 14:50:36.483 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.64% / 0.00% / 77.36% | 20260505 14:50:36.483 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.511714% / 7.94729e-06% / 9.53674e-05% | 20260505 14:50:36.483 | INFO | Baz IO overall: | 20260505 14:50:36.483 | INFO | BytesWritten : 2417353667201 | 20260505 14:50:36.483 | INFO | NumEvents : 1148972859640 | 20260505 14:50:38.159 | INFO | Analyzing chunk frames = [4059136,4059648) | 20260505 14:50:38.159 | INFO | MemInfo: { | 20260505 14:50:38.159 | INFO | "gpu" : " 17 GB", | 20260505 14:50:38.159 | INFO | "huge" : "", | 20260505 14:50:38.159 | INFO | "huge_shared" : " 27 GB", | 20260505 14:50:38.159 | INFO | "malloc" : " 14 GB", | 20260505 14:50:38.159 | INFO | "pinned" : " 29 GB" | 20260505 14:50:38.159 | INFO | } | 20260505 14:50:38.159 | INFO | Perf report for the last 5.12438 seconds: | 20260505 14:50:38.159 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:50:38.159 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:38.159 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:38.159 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:38.159 | INFO | ANALYSIS : 64.1%, 3.0, 99.4ms, 0 | 20260505 14:50:38.159 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:38.159 | INFO | RT_METRICS : 46.6%, 1.0, 24.3ms, 0 | 20260505 14:50:38.159 | INFO | PRE_HQ : 61.4%, 1.8, 57.4ms, 0 | 20260505 14:50:38.159 | INFO | BAZWRITER : 64.0%, 4.0, 131.3ms, 0 | 20260505 14:50:38.159 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260505 14:50:38.159 | INFO | Uploaded 36GiB to GPU in 2.19429s (16.4062GiB/s) | 20260505 14:50:38.159 | INFO | Downloaded 38GiB to GPU in 2.2603s (16.8119GiB/s) | 20260505 14:50:43.292 | INFO | Analyzing chunk frames = [4059648,4060160) | 20260505 14:50:43.292 | INFO | MemInfo: { | 20260505 14:50:43.292 | INFO | "gpu" : " 17 GB", | 20260505 14:50:43.292 | INFO | "huge" : "", | 20260505 14:50:43.292 | INFO | "huge_shared" : " 27 GB", | 20260505 14:50:43.292 | INFO | "malloc" : " 14 GB", | 20260505 14:50:43.292 | INFO | "pinned" : " 29 GB" | 20260505 14:50:43.292 | INFO | } | 20260505 14:50:43.292 | INFO | Perf report for the last 5.13348 seconds: | 20260505 14:50:43.292 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:50:43.292 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:43.292 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:43.292 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:43.292 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 14:50:43.292 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:43.292 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:43.292 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:50:43.292 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:43.292 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:43.293 | INFO | Uploaded 36GiB to GPU in 2.20185s (16.3499GiB/s) | 20260505 14:50:43.293 | INFO | Downloaded 34GiB to GPU in 2.04011s (16.6657GiB/s) | 20260505 14:50:48.440 | INFO | Analyzing chunk frames = [4060160,4060672) | 20260505 14:50:48.440 | INFO | MemInfo: { | 20260505 14:50:48.440 | INFO | "gpu" : " 17 GB", | 20260505 14:50:48.440 | INFO | "huge" : "", | 20260505 14:50:48.440 | INFO | "huge_shared" : " 27 GB", | 20260505 14:50:48.440 | INFO | "malloc" : " 14 GB", | 20260505 14:50:48.441 | INFO | "pinned" : " 29 GB" | 20260505 14:50:48.441 | INFO | } | 20260505 14:50:48.441 | INFO | Perf report for the last 5.1482 seconds: | 20260505 14:50:48.441 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:50:48.441 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:48.441 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:48.441 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:48.441 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 14:50:48.441 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:48.441 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:48.441 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:50:48.441 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:48.441 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:48.441 | INFO | Uploaded 36GiB to GPU in 2.20957s (16.2928GiB/s) | 20260505 14:50:48.441 | INFO | Downloaded 34GiB to GPU in 2.13481s (15.9264GiB/s) | 20260505 14:50:53.581 | INFO | Analyzing chunk frames = [4060672,4061184) | 20260505 14:50:53.581 | INFO | MemInfo: { | 20260505 14:50:53.581 | INFO | "gpu" : " 17 GB", | 20260505 14:50:53.581 | INFO | "huge" : "", | 20260505 14:50:53.581 | INFO | "huge_shared" : " 27 GB", | 20260505 14:50:53.581 | INFO | "malloc" : " 14 GB", | 20260505 14:50:53.581 | INFO | "pinned" : " 29 GB" | 20260505 14:50:53.581 | INFO | } | 20260505 14:50:53.581 | INFO | Perf report for the last 5.14085 seconds: | 20260505 14:50:53.581 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:50:53.581 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:53.581 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:53.581 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:53.581 | INFO | ANALYSIS : 62.8%, 3.0, 97.6ms, 0 | 20260505 14:50:53.582 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:53.582 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:53.582 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:50:53.582 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:53.582 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:53.582 | INFO | Uploaded 36GiB to GPU in 2.18649s (16.4647GiB/s) | 20260505 14:50:53.582 | INFO | Downloaded 34GiB to GPU in 2.12735s (15.9823GiB/s) | 20260505 14:50:58.722 | INFO | Analyzing chunk frames = [4061184,4061696) | 20260505 14:50:58.722 | INFO | MemInfo: { | 20260505 14:50:58.722 | INFO | "gpu" : " 17 GB", | 20260505 14:50:58.722 | INFO | "huge" : "", | 20260505 14:50:58.722 | INFO | "huge_shared" : " 27 GB", | 20260505 14:50:58.722 | INFO | "malloc" : " 14 GB", | 20260505 14:50:58.722 | INFO | "pinned" : " 29 GB" | 20260505 14:50:58.722 | INFO | } | 20260505 14:50:58.722 | INFO | Perf report for the last 5.14084 seconds: | 20260505 14:50:58.722 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:50:58.722 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:58.722 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:58.722 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:58.722 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 14:50:58.722 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:58.722 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:58.722 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:50:58.722 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:58.722 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:50:58.723 | INFO | Uploaded 36GiB to GPU in 2.19484s (16.4021GiB/s) | 20260505 14:50:58.723 | INFO | Downloaded 34GiB to GPU in 2.1299s (15.9632GiB/s) | 20260505 14:51:03.852 | INFO | Analyzing chunk frames = [4061696,4062208) | 20260505 14:51:03.852 | INFO | MemInfo: { | 20260505 14:51:03.852 | INFO | "gpu" : " 17 GB", | 20260505 14:51:03.852 | INFO | "huge" : "", | 20260505 14:51:03.852 | INFO | "huge_shared" : " 27 GB", | 20260505 14:51:03.852 | INFO | "malloc" : " 14 GB", | 20260505 14:51:03.852 | INFO | "pinned" : " 29 GB" | 20260505 14:51:03.852 | INFO | } | 20260505 14:51:03.852 | INFO | Perf report for the last 5.12986 seconds: | 20260505 14:51:03.852 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:51:03.852 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:03.852 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:03.852 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:03.852 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 14:51:03.852 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:03.852 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:03.852 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 14:51:03.852 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:03.852 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:03.852 | INFO | Uploaded 36GiB to GPU in 2.21284s (16.2687GiB/s) | 20260505 14:51:03.852 | INFO | Downloaded 34GiB to GPU in 2.11175s (16.1004GiB/s) | 20260505 14:51:08.993 | INFO | Analyzing chunk frames = [4062208,4062720) | 20260505 14:51:08.993 | INFO | MemInfo: { | 20260505 14:51:08.993 | INFO | "gpu" : " 17 GB", | 20260505 14:51:08.993 | INFO | "huge" : "", | 20260505 14:51:08.993 | INFO | "huge_shared" : " 27 GB", | 20260505 14:51:08.993 | INFO | "malloc" : " 14 GB", | 20260505 14:51:08.993 | INFO | "pinned" : " 29 GB" | 20260505 14:51:08.993 | INFO | } | 20260505 14:51:08.994 | INFO | Perf report for the last 5.14135 seconds: | 20260505 14:51:08.994 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:51:08.994 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:08.994 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:08.994 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:08.994 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 14:51:08.994 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:08.994 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:08.994 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 14:51:08.994 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:08.994 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:08.994 | INFO | Uploaded 36GiB to GPU in 2.15904s (16.6741GiB/s) | 20260505 14:51:08.994 | INFO | Downloaded 34GiB to GPU in 2.18462s (15.5634GiB/s) | 20260505 14:51:14.139 | INFO | Analyzing chunk frames = [4062720,4063232) | 20260505 14:51:14.139 | INFO | MemInfo: { | 20260505 14:51:14.139 | INFO | "gpu" : " 17 GB", | 20260505 14:51:14.139 | INFO | "huge" : "", | 20260505 14:51:14.139 | INFO | "huge_shared" : " 27 GB", | 20260505 14:51:14.139 | INFO | "malloc" : " 14 GB", | 20260505 14:51:14.139 | INFO | "pinned" : " 29 GB" | 20260505 14:51:14.139 | INFO | } | 20260505 14:51:14.140 | INFO | Perf report for the last 5.14602 seconds: | 20260505 14:51:14.140 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:51:14.140 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:14.140 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:14.140 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:14.140 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 14:51:14.140 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:14.140 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:14.140 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 14:51:14.140 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:14.140 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:14.140 | INFO | Uploaded 36GiB to GPU in 2.22929s (16.1486GiB/s) | 20260505 14:51:14.140 | INFO | Downloaded 34GiB to GPU in 2.1014s (16.1797GiB/s) | 20260505 14:51:14.140 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 14:51:14.140 | INFO | Total mem usage: 15.3 GiB | 20260505 14:51:14.140 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 14:51:14.140 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.85%) | 20260505 14:51:14.140 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 14:51:14.140 | INFO | High Water Breakdown: | 20260505 14:51:14.140 | INFO | 61% : 8.67 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 14:51:14.140 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 14:51:14.140 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 14:51:14.140 | INFO | ------------------------------------------ | 20260505 14:51:14.140 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:51:14.140 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 14:51:14.140 | INFO | Total mem usage: 26.5 GiB | 20260505 14:51:14.140 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 14:51:14.140 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 14:51:14.140 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 14:51:14.140 | INFO | High Water Breakdown: | 20260505 14:51:14.140 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:51:14.140 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:51:14.140 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:51:14.140 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:51:14.140 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:51:14.140 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:51:14.140 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 14:51:14.140 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 14:51:14.140 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:51:14.140 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 14:51:14.140 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 14:51:14.140 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 14:51:14.140 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 14:51:14.141 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 14:51:14.141 | INFO | ------------------------------------------ | 20260505 14:51:14.141 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:51:14.141 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 14:51:14.141 | INFO | Total mem usage: 25.6 GiB | 20260505 14:51:14.141 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 14:51:14.141 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 14:51:14.141 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 14:51:14.141 | INFO | High Water Breakdown: | 20260505 14:51:14.141 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 14:51:14.141 | INFO | ------------------------------------------ | 20260505 14:51:14.141 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:51:14.141 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 14:51:14.141 | INFO | Total mem usage: 15.7 GiB | 20260505 14:51:14.141 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 14:51:14.141 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 14:51:14.141 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 14:51:14.141 | INFO | High Water Breakdown: | 20260505 14:51:14.141 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 14:51:14.141 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 14:51:14.141 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 14:51:14.141 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 14:51:14.141 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 14:51:14.141 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 14:51:14.141 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 14:51:14.141 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 14:51:14.141 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 14:51:14.141 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:51:14.141 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 14:51:14.141 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:51:14.141 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 14:51:14.141 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 14:51:14.141 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 14:51:14.141 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 14:51:14.141 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:51:14.141 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 14:51:14.141 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:51:14.141 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:51:14.141 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:51:14.141 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 14:51:14.141 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 14:51:14.141 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:51:14.141 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 14:51:14.141 | INFO | ------------------------------------------ | 20260505 14:51:14.141 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:51:14.141 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 14:51:14.141 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:51:14.141 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:51:14.141 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.61e+01, 1.36e+01 / 1.84e+00 | 20260505 14:51:14.141 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 1.00e-01, 1.07e-02 / 3.47e-03 | 20260505 14:51:14.141 | INFO | PerfReport: 11.77%, Baseline -- 3.74e+00 / 4.46e+00, 5.75e-01 / 2.29e-01 | 20260505 14:51:14.141 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.49e+00, 1.92e-01 / 1.09e-01 | 20260505 14:51:14.141 | INFO | PerfReport: 15.99%, DME -- 4.06e+01 / 4.55e+01, 4.55e+00 / 1.85e+00 | 20260505 14:51:14.141 | INFO | PerfReport: 56.49%, FrameLabeling -- 1.79e+01 / 1.97e+01, 1.39e+00 / 1.40e+00 | 20260505 14:51:14.141 | INFO | PerfReport: 7.43%, PulseAccumulating -- 2.36e+00 / 2.94e+00, 4.53e-01 / 1.08e-01 | 20260505 14:51:14.141 | INFO | PerfReport: 3.52%, Metrics -- 1.12e+00 / 1.43e+00, 3.12e-01 / 1.39e-01 | 20260505 14:51:14.141 | INFO | PerfReport: 1.23%, PulseSerializing -- 3.91e-01 / 4.41e-01, 6.90e-02 / 3.46e-02 | 20260505 14:51:14.141 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 14:51:14.141 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:51:14.141 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:51:14.141 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260505 14:51:14.141 | INFO | PerfReport: 22.62%, Upload -- 2.28e+01 / 0.00e+00, 8.11e+00 / 0.00e+00 | 20260505 14:51:14.141 | INFO | PerfReport: 22.00%, Download -- 2.22e+01 / 0.00e+00, 8.09e+00 / 0.00e+00 | 20260505 14:51:14.141 | INFO | PerfReport: 31.54%, Compute -- 3.18e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 14:51:14.141 | INFO | PerfReport: 2.30%, LockUpload -- 2.32e+00 / 0.00e+00, 7.92e+00 / 0.00e+00 | 20260505 14:51:14.141 | INFO | PerfReport: 1.40%, LockDownload -- 1.41e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260505 14:51:14.141 | INFO | PerfReport: 20.13%, LockCompute -- 2.03e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260505 14:51:14.141 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.49e-02 / 0.00e+00 | 20260505 14:51:14.273 | INFO | Starting a round of Baz IO | 20260505 14:51:17.495 | INFO | Populating RTMetrics report | 20260505 14:51:17.495 | INFO | RTMetrics report populated | 20260505 14:51:17.495 | INFO | Opening tmp RTMetrics json file | 20260505 14:51:17.506 | INFO | Serializing RTMetrics to json | 20260505 14:51:17.507 | INFO | Serialization done, swapping tmp file now... | 20260505 14:51:17.508 | INFO | RTMetrics done | 20260505 14:51:17.594 | INFO | Finished a round of Baz IO: Wrote 2.238 GiB in 3.32 seconds | 20260505 14:51:17.594 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.63% / 0.00% / 77.36% | 20260505 14:51:17.594 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.510852% / 3.97364e-06% / 8.74201e-05% | 20260505 14:51:17.594 | INFO | Baz IO overall: | 20260505 14:51:17.594 | INFO | BytesWritten : 2419756946143 | 20260505 14:51:17.594 | INFO | NumEvents : 1150104260532 | 20260505 14:51:19.279 | INFO | Analyzing chunk frames = [4063232,4063744) | 20260505 14:51:19.279 | INFO | MemInfo: { | 20260505 14:51:19.279 | INFO | "gpu" : " 17 GB", | 20260505 14:51:19.279 | INFO | "huge" : "", | 20260505 14:51:19.279 | INFO | "huge_shared" : " 27 GB", | 20260505 14:51:19.279 | INFO | "malloc" : " 14 GB", | 20260505 14:51:19.279 | INFO | "pinned" : " 29 GB" | 20260505 14:51:19.280 | INFO | } | 20260505 14:51:19.280 | INFO | Perf report for the last 5.14 seconds: | 20260505 14:51:19.280 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:51:19.280 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:19.280 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:19.280 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:19.280 | INFO | ANALYSIS : 64.2%, 3.0, 99.9ms, 0 | 20260505 14:51:19.280 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:19.280 | INFO | RT_METRICS : 45.1%, 1.0, 23.6ms, 0 | 20260505 14:51:19.280 | INFO | PRE_HQ : 61.0%, 1.8, 57.0ms, 0 | 20260505 14:51:19.280 | INFO | BAZWRITER : 63.9%, 4.0, 130.1ms, 0 | 20260505 14:51:19.280 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 14:51:19.280 | INFO | Uploaded 36GiB to GPU in 2.20114s (16.3552GiB/s) | 20260505 14:51:19.280 | INFO | Downloaded 38GiB to GPU in 2.25602s (16.8438GiB/s) | 20260505 14:51:24.409 | INFO | Analyzing chunk frames = [4063744,4064256) | 20260505 14:51:24.410 | INFO | MemInfo: { | 20260505 14:51:24.410 | INFO | "gpu" : " 17 GB", | 20260505 14:51:24.410 | INFO | "huge" : "", | 20260505 14:51:24.410 | INFO | "huge_shared" : " 27 GB", | 20260505 14:51:24.410 | INFO | "malloc" : " 14 GB", | 20260505 14:51:24.410 | INFO | "pinned" : " 29 GB" | 20260505 14:51:24.410 | INFO | } | 20260505 14:51:24.410 | INFO | Perf report for the last 5.13017 seconds: | 20260505 14:51:24.410 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:51:24.410 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:24.410 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:24.410 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:24.410 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 14:51:24.410 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:24.410 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:24.410 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 14:51:24.410 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:24.410 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:24.410 | INFO | Uploaded 36GiB to GPU in 2.19224s (16.4215GiB/s) | 20260505 14:51:24.410 | INFO | Downloaded 34GiB to GPU in 2.03562s (16.7025GiB/s) | 20260505 14:51:29.549 | INFO | Analyzing chunk frames = [4064256,4064768) | 20260505 14:51:29.549 | INFO | MemInfo: { | 20260505 14:51:29.549 | INFO | "gpu" : " 17 GB", | 20260505 14:51:29.549 | INFO | "huge" : "", | 20260505 14:51:29.549 | INFO | "huge_shared" : " 27 GB", | 20260505 14:51:29.549 | INFO | "malloc" : " 14 GB", | 20260505 14:51:29.549 | INFO | "pinned" : " 29 GB" | 20260505 14:51:29.550 | INFO | } | 20260505 14:51:29.550 | INFO | Perf report for the last 5.13999 seconds: | 20260505 14:51:29.550 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:51:29.550 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:29.550 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:29.550 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:29.550 | INFO | ANALYSIS : 63.2%, 3.0, 98.3ms, 0 | 20260505 14:51:29.550 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:29.550 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:29.550 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:51:29.550 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:29.550 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:29.550 | INFO | Uploaded 36GiB to GPU in 2.19436s (16.4057GiB/s) | 20260505 14:51:29.550 | INFO | Downloaded 34GiB to GPU in 2.13778s (15.9044GiB/s) | 20260505 14:51:34.698 | INFO | Analyzing chunk frames = [4064768,4065280) | 20260505 14:51:34.698 | INFO | MemInfo: { | 20260505 14:51:34.698 | INFO | "gpu" : " 17 GB", | 20260505 14:51:34.698 | INFO | "huge" : "", | 20260505 14:51:34.698 | INFO | "huge_shared" : " 27 GB", | 20260505 14:51:34.698 | INFO | "malloc" : " 14 GB", | 20260505 14:51:34.699 | INFO | "pinned" : " 29 GB" | 20260505 14:51:34.699 | INFO | } | 20260505 14:51:34.699 | INFO | Perf report for the last 5.14884 seconds: | 20260505 14:51:34.699 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:51:34.699 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:34.699 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:34.699 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:34.699 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260505 14:51:34.699 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:34.699 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:34.699 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:51:34.699 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:34.699 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:34.699 | INFO | Uploaded 36GiB to GPU in 2.19229s (16.4212GiB/s) | 20260505 14:51:34.699 | INFO | Downloaded 34GiB to GPU in 2.13734s (15.9076GiB/s) | 20260505 14:51:39.839 | INFO | Analyzing chunk frames = [4065280,4065792) | 20260505 14:51:39.839 | INFO | MemInfo: { | 20260505 14:51:39.839 | INFO | "gpu" : " 17 GB", | 20260505 14:51:39.839 | INFO | "huge" : "", | 20260505 14:51:39.839 | INFO | "huge_shared" : " 27 GB", | 20260505 14:51:39.839 | INFO | "malloc" : " 14 GB", | 20260505 14:51:39.839 | INFO | "pinned" : " 29 GB" | 20260505 14:51:39.839 | INFO | } | 20260505 14:51:39.839 | INFO | Perf report for the last 5.14019 seconds: | 20260505 14:51:39.839 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:51:39.839 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:39.839 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:39.839 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:39.839 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260505 14:51:39.839 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:39.839 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:39.839 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 14:51:39.839 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:39.839 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:39.839 | INFO | Uploaded 36GiB to GPU in 2.13131s (16.8911GiB/s) | 20260505 14:51:39.839 | INFO | Downloaded 34GiB to GPU in 2.05377s (16.5549GiB/s) | 20260505 14:51:44.973 | INFO | Analyzing chunk frames = [4065792,4066304) | 20260505 14:51:44.973 | INFO | MemInfo: { | 20260505 14:51:44.973 | INFO | "gpu" : " 17 GB", | 20260505 14:51:44.973 | INFO | "huge" : "", | 20260505 14:51:44.973 | INFO | "huge_shared" : " 27 GB", | 20260505 14:51:44.973 | INFO | "malloc" : " 14 GB", | 20260505 14:51:44.974 | INFO | "pinned" : " 29 GB" | 20260505 14:51:44.974 | INFO | } | 20260505 14:51:44.974 | INFO | Perf report for the last 5.13493 seconds: | 20260505 14:51:44.974 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:51:44.974 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:44.974 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:44.974 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:44.974 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 14:51:44.974 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:44.974 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:44.974 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 14:51:44.974 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:44.974 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:44.974 | INFO | Uploaded 36GiB to GPU in 2.13565s (16.8567GiB/s) | 20260505 14:51:44.974 | INFO | Downloaded 34GiB to GPU in 2.0358s (16.7011GiB/s) | 20260505 14:51:50.108 | INFO | Analyzing chunk frames = [4066304,4066816) | 20260505 14:51:50.108 | INFO | MemInfo: { | 20260505 14:51:50.108 | INFO | "gpu" : " 17 GB", | 20260505 14:51:50.108 | INFO | "huge" : "", | 20260505 14:51:50.108 | INFO | "huge_shared" : " 27 GB", | 20260505 14:51:50.108 | INFO | "malloc" : " 14 GB", | 20260505 14:51:50.108 | INFO | "pinned" : " 29 GB" | 20260505 14:51:50.108 | INFO | } | 20260505 14:51:50.108 | INFO | Perf report for the last 5.13409 seconds: | 20260505 14:51:50.108 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:51:50.108 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:50.108 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:50.108 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:50.108 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 14:51:50.108 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:50.108 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:50.108 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 14:51:50.108 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:50.108 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:50.108 | INFO | Uploaded 36GiB to GPU in 2.12777s (16.9191GiB/s) | 20260505 14:51:50.108 | INFO | Downloaded 34GiB to GPU in 2.1718s (15.6552GiB/s) | 20260505 14:51:55.246 | INFO | Analyzing chunk frames = [4066816,4067328) | 20260505 14:51:55.246 | INFO | MemInfo: { | 20260505 14:51:55.246 | INFO | "gpu" : " 17 GB", | 20260505 14:51:55.246 | INFO | "huge" : "", | 20260505 14:51:55.246 | INFO | "huge_shared" : " 27 GB", | 20260505 14:51:55.246 | INFO | "malloc" : " 14 GB", | 20260505 14:51:55.246 | INFO | "pinned" : " 29 GB" | 20260505 14:51:55.246 | INFO | } | 20260505 14:51:55.246 | INFO | Perf report for the last 5.13798 seconds: | 20260505 14:51:55.246 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:51:55.246 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:55.246 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:55.246 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:55.246 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 14:51:55.246 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:55.246 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:55.246 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:51:55.246 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:55.246 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:51:55.246 | INFO | Uploaded 36GiB to GPU in 2.17577s (16.5458GiB/s) | 20260505 14:51:55.246 | INFO | Downloaded 34GiB to GPU in 2.11045s (16.1103GiB/s) | 20260505 14:51:55.384 | INFO | Starting a round of Baz IO | 20260505 14:51:58.605 | INFO | Populating RTMetrics report | 20260505 14:51:58.606 | INFO | RTMetrics report populated | 20260505 14:51:58.606 | INFO | Opening tmp RTMetrics json file | 20260505 14:51:58.640 | INFO | Serializing RTMetrics to json | 20260505 14:51:58.670 | INFO | Serialization done, swapping tmp file now... | 20260505 14:51:58.688 | INFO | RTMetrics done | 20260505 14:51:58.707 | INFO | Finished a round of Baz IO: Wrote 2.238 GiB in 3.323 seconds | 20260505 14:51:58.707 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.63% / 0.00% / 77.37% | 20260505 14:51:58.707 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.5114% / 7.94729e-06% / 0.000103315% | 20260505 14:51:58.707 | INFO | Baz IO overall: | 20260505 14:51:58.707 | INFO | BytesWritten : 2422159920365 | 20260505 14:51:58.707 | INFO | NumEvents : 1151235618696 | 20260505 14:52:00.390 | INFO | Analyzing chunk frames = [4067328,4067840) | 20260505 14:52:00.390 | INFO | MemInfo: { | 20260505 14:52:00.390 | INFO | "gpu" : " 17 GB", | 20260505 14:52:00.390 | INFO | "huge" : "", | 20260505 14:52:00.390 | INFO | "huge_shared" : " 27 GB", | 20260505 14:52:00.390 | INFO | "malloc" : " 14 GB", | 20260505 14:52:00.390 | INFO | "pinned" : " 29 GB" | 20260505 14:52:00.390 | INFO | } | 20260505 14:52:00.390 | INFO | Perf report for the last 5.14433 seconds: | 20260505 14:52:00.390 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:52:00.390 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:00.390 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:00.390 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:00.390 | INFO | ANALYSIS : 64.2%, 3.0, 100.0ms, 0 | 20260505 14:52:00.390 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:00.390 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260505 14:52:00.390 | INFO | PRE_HQ : 60.8%, 1.8, 57.1ms, 0 | 20260505 14:52:00.390 | INFO | BAZWRITER : 64.0%, 3.9, 129.4ms, 0 | 20260505 14:52:00.390 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 14:52:00.390 | INFO | Uploaded 36GiB to GPU in 2.16956s (16.5932GiB/s) | 20260505 14:52:00.390 | INFO | Downloaded 38GiB to GPU in 2.27163s (16.7281GiB/s) | 20260505 14:52:05.536 | INFO | Analyzing chunk frames = [4067840,4068352) | 20260505 14:52:05.536 | INFO | MemInfo: { | 20260505 14:52:05.536 | INFO | "gpu" : " 17 GB", | 20260505 14:52:05.536 | INFO | "huge" : "", | 20260505 14:52:05.536 | INFO | "huge_shared" : " 27 GB", | 20260505 14:52:05.536 | INFO | "malloc" : " 14 GB", | 20260505 14:52:05.536 | INFO | "pinned" : " 29 GB" | 20260505 14:52:05.536 | INFO | } | 20260505 14:52:05.536 | INFO | Perf report for the last 5.14585 seconds: | 20260505 14:52:05.536 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:52:05.536 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:05.536 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:05.536 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:05.536 | INFO | ANALYSIS : 63.3%, 3.0, 99.0ms, 0 | 20260505 14:52:05.536 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:05.536 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:05.536 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 14:52:05.536 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:05.536 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:05.536 | INFO | Uploaded 36GiB to GPU in 2.20057s (16.3594GiB/s) | 20260505 14:52:05.536 | INFO | Downloaded 34GiB to GPU in 2.06324s (16.479GiB/s) | 20260505 14:52:10.671 | INFO | Analyzing chunk frames = [4068352,4068864) | 20260505 14:52:10.671 | INFO | MemInfo: { | 20260505 14:52:10.671 | INFO | "gpu" : " 17 GB", | 20260505 14:52:10.671 | INFO | "huge" : "", | 20260505 14:52:10.671 | INFO | "huge_shared" : " 27 GB", | 20260505 14:52:10.671 | INFO | "malloc" : " 14 GB", | 20260505 14:52:10.671 | INFO | "pinned" : " 29 GB" | 20260505 14:52:10.671 | INFO | } | 20260505 14:52:10.671 | INFO | Perf report for the last 5.13478 seconds: | 20260505 14:52:10.671 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:52:10.671 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:10.671 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:10.671 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:10.671 | INFO | ANALYSIS : 62.9%, 3.0, 97.8ms, 0 | 20260505 14:52:10.671 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:10.671 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:10.671 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:52:10.671 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:10.671 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:10.671 | INFO | Uploaded 36GiB to GPU in 2.17956s (16.5171GiB/s) | 20260505 14:52:10.671 | INFO | Downloaded 34GiB to GPU in 2.03584s (16.7007GiB/s) | 20260505 14:52:15.804 | INFO | Analyzing chunk frames = [4068864,4069376) | 20260505 14:52:15.804 | INFO | MemInfo: { | 20260505 14:52:15.804 | INFO | "gpu" : " 17 GB", | 20260505 14:52:15.804 | INFO | "huge" : "", | 20260505 14:52:15.804 | INFO | "huge_shared" : " 27 GB", | 20260505 14:52:15.804 | INFO | "malloc" : " 14 GB", | 20260505 14:52:15.804 | INFO | "pinned" : " 29 GB" | 20260505 14:52:15.804 | INFO | } | 20260505 14:52:15.804 | INFO | Perf report for the last 5.13314 seconds: | 20260505 14:52:15.804 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:52:15.804 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:15.804 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:15.804 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:15.804 | INFO | ANALYSIS : 62.9%, 3.0, 97.6ms, 0 | 20260505 14:52:15.804 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:15.804 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:15.804 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:52:15.804 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:15.804 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:15.804 | INFO | Uploaded 36GiB to GPU in 2.18892s (16.4465GiB/s) | 20260505 14:52:15.804 | INFO | Downloaded 34GiB to GPU in 2.13343s (15.9368GiB/s) | 20260505 14:52:20.947 | INFO | Analyzing chunk frames = [4069376,4069888) | 20260505 14:52:20.947 | INFO | MemInfo: { | 20260505 14:52:20.947 | INFO | "gpu" : " 17 GB", | 20260505 14:52:20.947 | INFO | "huge" : "", | 20260505 14:52:20.947 | INFO | "huge_shared" : " 27 GB", | 20260505 14:52:20.947 | INFO | "malloc" : " 14 GB", | 20260505 14:52:20.947 | INFO | "pinned" : " 29 GB" | 20260505 14:52:20.947 | INFO | } | 20260505 14:52:20.947 | INFO | Perf report for the last 5.14305 seconds: | 20260505 14:52:20.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:52:20.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:20.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:20.947 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:20.947 | INFO | ANALYSIS : 62.6%, 3.0, 97.9ms, 0 | 20260505 14:52:20.947 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:20.947 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:20.947 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 14:52:20.947 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:20.947 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:20.947 | INFO | Uploaded 36GiB to GPU in 2.14437s (16.7881GiB/s) | 20260505 14:52:20.947 | INFO | Downloaded 34GiB to GPU in 2.12561s (15.9954GiB/s) | 20260505 14:52:26.088 | INFO | Analyzing chunk frames = [4069888,4070400) | 20260505 14:52:26.088 | INFO | MemInfo: { | 20260505 14:52:26.088 | INFO | "gpu" : " 17 GB", | 20260505 14:52:26.088 | INFO | "huge" : "", | 20260505 14:52:26.088 | INFO | "huge_shared" : " 27 GB", | 20260505 14:52:26.088 | INFO | "malloc" : " 14 GB", | 20260505 14:52:26.089 | INFO | "pinned" : " 29 GB" | 20260505 14:52:26.089 | INFO | } | 20260505 14:52:26.089 | INFO | Perf report for the last 5.14166 seconds: | 20260505 14:52:26.089 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:52:26.089 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:26.089 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:26.089 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:26.089 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 14:52:26.089 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:26.089 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:26.089 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 14:52:26.089 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:26.089 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:26.089 | INFO | Uploaded 36GiB to GPU in 2.16923s (16.5957GiB/s) | 20260505 14:52:26.089 | INFO | Downloaded 34GiB to GPU in 2.0369s (16.692GiB/s) | 20260505 14:52:31.227 | INFO | Analyzing chunk frames = [4070400,4070912) | 20260505 14:52:31.227 | INFO | MemInfo: { | 20260505 14:52:31.227 | INFO | "gpu" : " 17 GB", | 20260505 14:52:31.227 | INFO | "huge" : "", | 20260505 14:52:31.227 | INFO | "huge_shared" : " 27 GB", | 20260505 14:52:31.227 | INFO | "malloc" : " 14 GB", | 20260505 14:52:31.227 | INFO | "pinned" : " 29 GB" | 20260505 14:52:31.227 | INFO | } | 20260505 14:52:31.227 | INFO | Perf report for the last 5.13815 seconds: | 20260505 14:52:31.227 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:52:31.227 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:31.227 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:31.227 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:31.227 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 14:52:31.227 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:31.227 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:31.227 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 14:52:31.227 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:31.227 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:31.227 | INFO | Uploaded 36GiB to GPU in 2.13315s (16.8765GiB/s) | 20260505 14:52:31.227 | INFO | Downloaded 34GiB to GPU in 2.16573s (15.6991GiB/s) | 20260505 14:52:36.361 | INFO | Analyzing chunk frames = [4070912,4071424) | 20260505 14:52:36.361 | INFO | MemInfo: { | 20260505 14:52:36.361 | INFO | "gpu" : " 17 GB", | 20260505 14:52:36.361 | INFO | "huge" : "", | 20260505 14:52:36.361 | INFO | "huge_shared" : " 27 GB", | 20260505 14:52:36.361 | INFO | "malloc" : " 14 GB", | 20260505 14:52:36.361 | INFO | "pinned" : " 29 GB" | 20260505 14:52:36.361 | INFO | } | 20260505 14:52:36.361 | INFO | Perf report for the last 5.13416 seconds: | 20260505 14:52:36.361 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:52:36.361 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:36.361 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:36.361 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:36.361 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 14:52:36.361 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:36.361 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:36.361 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 14:52:36.361 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:36.361 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:36.361 | INFO | Uploaded 36GiB to GPU in 2.21469s (16.2551GiB/s) | 20260505 14:52:36.361 | INFO | Downloaded 34GiB to GPU in 2.10507s (16.1514GiB/s) | 20260505 14:52:36.362 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 14:52:36.362 | INFO | Total mem usage: 15.3 GiB | 20260505 14:52:36.362 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 14:52:36.362 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.83%) | 20260505 14:52:36.362 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 14:52:36.362 | INFO | High Water Breakdown: | 20260505 14:52:36.362 | INFO | 61% : 8.67 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 14:52:36.362 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 14:52:36.362 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 14:52:36.362 | INFO | ------------------------------------------ | 20260505 14:52:36.362 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:52:36.362 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 14:52:36.362 | INFO | Total mem usage: 26.5 GiB | 20260505 14:52:36.362 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 14:52:36.362 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 14:52:36.362 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 14:52:36.362 | INFO | High Water Breakdown: | 20260505 14:52:36.362 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:52:36.362 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:52:36.362 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:52:36.362 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:52:36.362 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:52:36.362 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:52:36.362 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 14:52:36.362 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 14:52:36.362 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:52:36.362 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 14:52:36.362 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 14:52:36.362 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 14:52:36.362 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 14:52:36.362 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 14:52:36.362 | INFO | ------------------------------------------ | 20260505 14:52:36.362 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:52:36.362 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 14:52:36.362 | INFO | Total mem usage: 25.6 GiB | 20260505 14:52:36.362 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 14:52:36.362 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 14:52:36.362 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 14:52:36.362 | INFO | High Water Breakdown: | 20260505 14:52:36.362 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 14:52:36.362 | INFO | ------------------------------------------ | 20260505 14:52:36.362 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:52:36.362 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 14:52:36.362 | INFO | Total mem usage: 15.7 GiB | 20260505 14:52:36.362 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 14:52:36.362 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 14:52:36.362 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260505 14:52:36.362 | INFO | High Water Breakdown: | 20260505 14:52:36.362 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 14:52:36.362 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 14:52:36.362 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 14:52:36.362 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 14:52:36.362 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 14:52:36.362 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 14:52:36.362 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 14:52:36.362 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 14:52:36.362 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 14:52:36.362 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:52:36.362 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 14:52:36.362 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:52:36.362 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 14:52:36.362 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 14:52:36.362 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 14:52:36.362 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 14:52:36.362 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:52:36.362 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 14:52:36.362 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:52:36.362 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:52:36.362 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:52:36.362 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 14:52:36.362 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 14:52:36.362 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:52:36.362 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 14:52:36.362 | INFO | ------------------------------------------ | 20260505 14:52:36.362 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:52:36.362 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 14:52:36.362 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:52:36.362 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:52:36.362 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.59e+01, 1.36e+01 / 2.19e+00 | 20260505 14:52:36.362 | INFO | PerfReport: 0.29%, AggregateStats -- 9.25e-02 / 1.00e-01, 1.03e-02 / 3.97e-03 | 20260505 14:52:36.362 | INFO | PerfReport: 11.78%, Baseline -- 3.75e+00 / 4.48e+00, 5.84e-01 / 1.87e-01 | 20260505 14:52:36.362 | INFO | PerfReport: 3.30%, Histogram -- 1.05e+00 / 1.50e+00, 1.99e-01 / 1.08e-01 | 20260505 14:52:36.362 | INFO | PerfReport: 15.96%, DME -- 4.06e+01 / 4.53e+01, 4.54e+00 / 1.97e+00 | 20260505 14:52:36.362 | INFO | PerfReport: 56.59%, FrameLabeling -- 1.80e+01 / 1.97e+01, 1.47e+00 / 1.09e+00 | 20260505 14:52:36.362 | INFO | PerfReport: 7.36%, PulseAccumulating -- 2.34e+00 / 2.93e+00, 4.41e-01 / 1.32e-01 | 20260505 14:52:36.362 | INFO | PerfReport: 3.49%, Metrics -- 1.11e+00 / 1.43e+00, 3.08e-01 / 1.42e-01 | 20260505 14:52:36.362 | INFO | PerfReport: 1.23%, PulseSerializing -- 3.90e-01 / 4.39e-01, 6.97e-02 / 3.50e-02 | 20260505 14:52:36.362 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 14:52:36.362 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:52:36.362 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:52:36.362 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260505 14:52:36.362 | INFO | PerfReport: 22.49%, Upload -- 2.26e+01 / 0.00e+00, 7.95e+00 / 0.00e+00 | 20260505 14:52:36.363 | INFO | PerfReport: 21.92%, Download -- 2.21e+01 / 0.00e+00, 7.97e+00 / 0.00e+00 | 20260505 14:52:36.363 | INFO | PerfReport: 31.63%, Compute -- 3.18e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 14:52:36.363 | INFO | PerfReport: 2.15%, LockUpload -- 2.16e+00 / 0.00e+00, 7.77e+00 / 0.00e+00 | 20260505 14:52:36.363 | INFO | PerfReport: 1.44%, LockDownload -- 1.45e+00 / 0.00e+00, 2.98e+00 / 0.00e+00 | 20260505 14:52:36.363 | INFO | PerfReport: 20.37%, LockCompute -- 2.05e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 14:52:36.363 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.07e-02 / 0.00e+00 | 20260505 14:52:36.501 | INFO | Starting a round of Baz IO | 20260505 14:52:39.716 | INFO | Populating RTMetrics report | 20260505 14:52:39.716 | INFO | RTMetrics report populated | 20260505 14:52:39.716 | INFO | Opening tmp RTMetrics json file | 20260505 14:52:39.734 | INFO | Serializing RTMetrics to json | 20260505 14:52:39.735 | INFO | Serialization done, swapping tmp file now... | 20260505 14:52:39.736 | INFO | RTMetrics done | 20260505 14:52:39.822 | INFO | Finished a round of Baz IO: Wrote 2.237 GiB in 3.32 seconds | 20260505 14:52:39.822 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.63% / 0.00% / 77.37% | 20260505 14:52:39.822 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.50935% / 1.19209e-05% / 9.13938e-05% | 20260505 14:52:39.822 | INFO | Baz IO overall: | 20260505 14:52:39.822 | INFO | BytesWritten : 2424562392930 | 20260505 14:52:39.822 | INFO | NumEvents : 1152366656926 | 20260505 14:52:41.501 | INFO | Analyzing chunk frames = [4071424,4071936) | 20260505 14:52:41.501 | INFO | MemInfo: { | 20260505 14:52:41.501 | INFO | "gpu" : " 17 GB", | 20260505 14:52:41.501 | INFO | "huge" : "", | 20260505 14:52:41.501 | INFO | "huge_shared" : " 27 GB", | 20260505 14:52:41.501 | INFO | "malloc" : " 14 GB", | 20260505 14:52:41.501 | INFO | "pinned" : " 29 GB" | 20260505 14:52:41.501 | INFO | } | 20260505 14:52:41.501 | INFO | Perf report for the last 5.14009 seconds: | 20260505 14:52:41.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:52:41.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:41.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:41.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:41.501 | INFO | ANALYSIS : 64.2%, 3.0, 99.7ms, 0 | 20260505 14:52:41.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:41.501 | INFO | RT_METRICS : 45.3%, 1.0, 23.7ms, 0 | 20260505 14:52:41.501 | INFO | PRE_HQ : 60.7%, 1.8, 57.5ms, 0 | 20260505 14:52:41.501 | INFO | BAZWRITER : 64.0%, 4.0, 131.2ms, 0 | 20260505 14:52:41.501 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 14:52:41.502 | INFO | Uploaded 36GiB to GPU in 2.18814s (16.4523GiB/s) | 20260505 14:52:41.502 | INFO | Downloaded 38GiB to GPU in 2.27821s (16.6797GiB/s) | 20260505 14:52:46.653 | INFO | Analyzing chunk frames = [4071936,4072448) | 20260505 14:52:46.653 | INFO | MemInfo: { | 20260505 14:52:46.653 | INFO | "gpu" : " 17 GB", | 20260505 14:52:46.653 | INFO | "huge" : "", | 20260505 14:52:46.653 | INFO | "huge_shared" : " 27 GB", | 20260505 14:52:46.653 | INFO | "malloc" : " 14 GB", | 20260505 14:52:46.653 | INFO | "pinned" : " 29 GB" | 20260505 14:52:46.653 | INFO | } | 20260505 14:52:46.653 | INFO | Perf report for the last 5.15217 seconds: | 20260505 14:52:46.653 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:52:46.653 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:46.653 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:46.653 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:46.653 | INFO | ANALYSIS : 63.4%, 3.0, 99.2ms, 0 | 20260505 14:52:46.653 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:46.653 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:46.653 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 14:52:46.653 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:46.653 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:46.654 | INFO | Uploaded 36GiB to GPU in 2.20205s (16.3484GiB/s) | 20260505 14:52:46.654 | INFO | Downloaded 34GiB to GPU in 2.04205s (16.6499GiB/s) | 20260505 14:52:51.793 | INFO | Analyzing chunk frames = [4072448,4072960) | 20260505 14:52:51.793 | INFO | MemInfo: { | 20260505 14:52:51.793 | INFO | "gpu" : " 17 GB", | 20260505 14:52:51.793 | INFO | "huge" : "", | 20260505 14:52:51.793 | INFO | "huge_shared" : " 27 GB", | 20260505 14:52:51.793 | INFO | "malloc" : " 14 GB", | 20260505 14:52:51.793 | INFO | "pinned" : " 29 GB" | 20260505 14:52:51.793 | INFO | } | 20260505 14:52:51.793 | INFO | Perf report for the last 5.13946 seconds: | 20260505 14:52:51.793 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:52:51.793 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:51.793 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:51.793 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:51.793 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260505 14:52:51.793 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:51.793 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:51.793 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 14:52:51.793 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:51.793 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:51.793 | INFO | Uploaded 36GiB to GPU in 2.18556s (16.4718GiB/s) | 20260505 14:52:51.793 | INFO | Downloaded 34GiB to GPU in 2.02954s (16.7526GiB/s) | 20260505 14:52:56.917 | INFO | Analyzing chunk frames = [4072960,4073472) | 20260505 14:52:56.917 | INFO | MemInfo: { | 20260505 14:52:56.917 | INFO | "gpu" : " 17 GB", | 20260505 14:52:56.917 | INFO | "huge" : "", | 20260505 14:52:56.917 | INFO | "huge_shared" : " 27 GB", | 20260505 14:52:56.917 | INFO | "malloc" : " 14 GB", | 20260505 14:52:56.917 | INFO | "pinned" : " 29 GB" | 20260505 14:52:56.917 | INFO | } | 20260505 14:52:56.917 | INFO | Perf report for the last 5.12427 seconds: | 20260505 14:52:56.917 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:52:56.917 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:56.917 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:56.917 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:56.917 | INFO | ANALYSIS : 62.7%, 3.0, 97.2ms, 0 | 20260505 14:52:56.917 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:56.917 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:56.917 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:52:56.917 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:56.917 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:52:56.917 | INFO | Uploaded 36GiB to GPU in 2.18214s (16.4976GiB/s) | 20260505 14:52:56.917 | INFO | Downloaded 34GiB to GPU in 2.12816s (15.9762GiB/s) | 20260505 14:53:02.055 | INFO | Analyzing chunk frames = [4073472,4073984) | 20260505 14:53:02.055 | INFO | MemInfo: { | 20260505 14:53:02.055 | INFO | "gpu" : " 17 GB", | 20260505 14:53:02.055 | INFO | "huge" : "", | 20260505 14:53:02.055 | INFO | "huge_shared" : " 27 GB", | 20260505 14:53:02.055 | INFO | "malloc" : " 14 GB", | 20260505 14:53:02.055 | INFO | "pinned" : " 29 GB" | 20260505 14:53:02.055 | INFO | } | 20260505 14:53:02.055 | INFO | Perf report for the last 5.13772 seconds: | 20260505 14:53:02.055 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:53:02.055 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:02.055 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:02.055 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:02.055 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 14:53:02.055 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:02.055 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:02.055 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 14:53:02.055 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:02.055 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:02.055 | INFO | Uploaded 36GiB to GPU in 2.16721s (16.6112GiB/s) | 20260505 14:53:02.055 | INFO | Downloaded 34GiB to GPU in 2.12862s (15.9728GiB/s) | 20260505 14:53:07.206 | INFO | Analyzing chunk frames = [4073984,4074496) | 20260505 14:53:07.206 | INFO | MemInfo: { | 20260505 14:53:07.206 | INFO | "gpu" : " 17 GB", | 20260505 14:53:07.206 | INFO | "huge" : "", | 20260505 14:53:07.206 | INFO | "huge_shared" : " 27 GB", | 20260505 14:53:07.206 | INFO | "malloc" : " 14 GB", | 20260505 14:53:07.206 | INFO | "pinned" : " 29 GB" | 20260505 14:53:07.206 | INFO | } | 20260505 14:53:07.206 | INFO | Perf report for the last 5.15104 seconds: | 20260505 14:53:07.206 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:53:07.206 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:07.206 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:07.206 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:07.206 | INFO | ANALYSIS : 62.9%, 3.0, 98.5ms, 0 | 20260505 14:53:07.206 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:07.206 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:07.206 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 14:53:07.206 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:07.206 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:07.206 | INFO | Uploaded 36GiB to GPU in 2.17154s (16.5781GiB/s) | 20260505 14:53:07.206 | INFO | Downloaded 34GiB to GPU in 2.09408s (16.2362GiB/s) | 20260505 14:53:12.348 | INFO | Analyzing chunk frames = [4074496,4075008) | 20260505 14:53:12.348 | INFO | MemInfo: { | 20260505 14:53:12.348 | INFO | "gpu" : " 17 GB", | 20260505 14:53:12.348 | INFO | "huge" : "", | 20260505 14:53:12.348 | INFO | "huge_shared" : " 27 GB", | 20260505 14:53:12.349 | INFO | "malloc" : " 14 GB", | 20260505 14:53:12.349 | INFO | "pinned" : " 29 GB" | 20260505 14:53:12.349 | INFO | } | 20260505 14:53:12.349 | INFO | Perf report for the last 5.14278 seconds: | 20260505 14:53:12.349 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:53:12.349 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:12.349 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:12.349 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:12.349 | INFO | ANALYSIS : 63.2%, 3.0, 98.8ms, 0 | 20260505 14:53:12.349 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:12.349 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:12.349 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 14:53:12.349 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:12.349 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:12.349 | INFO | Uploaded 36GiB to GPU in 2.10566s (17.0968GiB/s) | 20260505 14:53:12.349 | INFO | Downloaded 34GiB to GPU in 2.16927s (15.6735GiB/s) | 20260505 14:53:17.487 | INFO | Analyzing chunk frames = [4075008,4075520) | 20260505 14:53:17.487 | INFO | MemInfo: { | 20260505 14:53:17.487 | INFO | "gpu" : " 17 GB", | 20260505 14:53:17.487 | INFO | "huge" : "", | 20260505 14:53:17.487 | INFO | "huge_shared" : " 27 GB", | 20260505 14:53:17.487 | INFO | "malloc" : " 14 GB", | 20260505 14:53:17.487 | INFO | "pinned" : " 29 GB" | 20260505 14:53:17.487 | INFO | } | 20260505 14:53:17.487 | INFO | Perf report for the last 5.13872 seconds: | 20260505 14:53:17.487 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:53:17.487 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:17.487 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:17.487 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:17.487 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 14:53:17.487 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:17.487 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:17.488 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 14:53:17.488 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:17.488 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:17.488 | INFO | Uploaded 36GiB to GPU in 2.16401s (16.6358GiB/s) | 20260505 14:53:17.488 | INFO | Downloaded 34GiB to GPU in 2.02942s (16.7535GiB/s) | 20260505 14:53:17.619 | INFO | Starting a round of Baz IO | 20260505 14:53:20.834 | INFO | Populating RTMetrics report | 20260505 14:53:20.834 | INFO | RTMetrics report populated | 20260505 14:53:20.834 | INFO | Opening tmp RTMetrics json file | 20260505 14:53:20.849 | INFO | Serializing RTMetrics to json | 20260505 14:53:20.851 | INFO | Serialization done, swapping tmp file now... | 20260505 14:53:20.852 | INFO | RTMetrics done | 20260505 14:53:20.939 | INFO | Finished a round of Baz IO: Wrote 2.236 GiB in 3.319 seconds | 20260505 14:53:20.939 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.62% / 0.00% / 77.37% | 20260505 14:53:20.939 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.506731% / 3.97364e-06% / 7.54992e-05% | 20260505 14:53:20.939 | INFO | Baz IO overall: | 20260505 14:53:20.939 | INFO | BytesWritten : 2426963660393 | 20260505 14:53:20.939 | INFO | NumEvents : 1153497299213 | 20260505 14:53:22.618 | INFO | Analyzing chunk frames = [4075520,4076032) | 20260505 14:53:22.618 | INFO | MemInfo: { | 20260505 14:53:22.618 | INFO | "gpu" : " 17 GB", | 20260505 14:53:22.618 | INFO | "huge" : "", | 20260505 14:53:22.618 | INFO | "huge_shared" : " 27 GB", | 20260505 14:53:22.618 | INFO | "malloc" : " 14 GB", | 20260505 14:53:22.618 | INFO | "pinned" : " 29 GB" | 20260505 14:53:22.618 | INFO | } | 20260505 14:53:22.618 | INFO | Perf report for the last 5.1309 seconds: | 20260505 14:53:22.618 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:53:22.618 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:22.618 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:22.618 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:22.618 | INFO | ANALYSIS : 64.1%, 3.0, 99.4ms, 0 | 20260505 14:53:22.618 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:22.618 | INFO | RT_METRICS : 45.3%, 1.0, 23.7ms, 0 | 20260505 14:53:22.618 | INFO | PRE_HQ : 61.0%, 1.8, 57.4ms, 0 | 20260505 14:53:22.618 | INFO | BAZWRITER : 64.0%, 4.0, 130.8ms, 0 | 20260505 14:53:22.618 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 14:53:22.619 | INFO | Uploaded 36GiB to GPU in 2.19696s (16.3863GiB/s) | 20260505 14:53:22.619 | INFO | Downloaded 38GiB to GPU in 2.24421s (16.9324GiB/s) | 20260505 14:53:27.753 | INFO | Analyzing chunk frames = [4076032,4076544) | 20260505 14:53:27.753 | INFO | MemInfo: { | 20260505 14:53:27.753 | INFO | "gpu" : " 17 GB", | 20260505 14:53:27.753 | INFO | "huge" : "", | 20260505 14:53:27.753 | INFO | "huge_shared" : " 27 GB", | 20260505 14:53:27.753 | INFO | "malloc" : " 14 GB", | 20260505 14:53:27.753 | INFO | "pinned" : " 29 GB" | 20260505 14:53:27.753 | INFO | } | 20260505 14:53:27.753 | INFO | Perf report for the last 5.1349 seconds: | 20260505 14:53:27.753 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:53:27.753 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:27.753 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:27.753 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:27.753 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 14:53:27.753 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:27.753 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:27.753 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:53:27.753 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:27.753 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:27.753 | INFO | Uploaded 36GiB to GPU in 2.17722s (16.5349GiB/s) | 20260505 14:53:27.754 | INFO | Downloaded 34GiB to GPU in 2.03265s (16.7269GiB/s) | 20260505 14:53:32.901 | INFO | Analyzing chunk frames = [4076544,4077056) | 20260505 14:53:32.901 | INFO | MemInfo: { | 20260505 14:53:32.901 | INFO | "gpu" : " 17 GB", | 20260505 14:53:32.901 | INFO | "huge" : "", | 20260505 14:53:32.901 | INFO | "huge_shared" : " 27 GB", | 20260505 14:53:32.901 | INFO | "malloc" : " 14 GB", | 20260505 14:53:32.901 | INFO | "pinned" : " 29 GB" | 20260505 14:53:32.901 | INFO | } | 20260505 14:53:32.901 | INFO | Perf report for the last 5.14774 seconds: | 20260505 14:53:32.901 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:53:32.901 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:32.901 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:32.901 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:32.901 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 14:53:32.901 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:32.901 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:32.901 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 14:53:32.901 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:32.901 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:32.901 | INFO | Uploaded 36GiB to GPU in 2.241s (16.0642GiB/s) | 20260505 14:53:32.901 | INFO | Downloaded 34GiB to GPU in 2.11389s (16.0841GiB/s) | 20260505 14:53:38.041 | INFO | Analyzing chunk frames = [4077056,4077568) | 20260505 14:53:38.041 | INFO | MemInfo: { | 20260505 14:53:38.041 | INFO | "gpu" : " 17 GB", | 20260505 14:53:38.041 | INFO | "huge" : "", | 20260505 14:53:38.041 | INFO | "huge_shared" : " 27 GB", | 20260505 14:53:38.041 | INFO | "malloc" : " 14 GB", | 20260505 14:53:38.041 | INFO | "pinned" : " 29 GB" | 20260505 14:53:38.041 | INFO | } | 20260505 14:53:38.041 | INFO | Perf report for the last 5.14022 seconds: | 20260505 14:53:38.041 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:53:38.041 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:38.041 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:38.041 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:38.041 | INFO | ANALYSIS : 62.5%, 3.0, 97.1ms, 0 | 20260505 14:53:38.041 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:38.041 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:38.041 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:53:38.041 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:38.041 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:38.042 | INFO | Uploaded 36GiB to GPU in 2.17638s (16.5413GiB/s) | 20260505 14:53:38.042 | INFO | Downloaded 34GiB to GPU in 2.13285s (15.9411GiB/s) | 20260505 14:53:43.172 | INFO | Analyzing chunk frames = [4077568,4078080) | 20260505 14:53:43.172 | INFO | MemInfo: { | 20260505 14:53:43.172 | INFO | "gpu" : " 17 GB", | 20260505 14:53:43.172 | INFO | "huge" : "", | 20260505 14:53:43.172 | INFO | "huge_shared" : " 27 GB", | 20260505 14:53:43.172 | INFO | "malloc" : " 14 GB", | 20260505 14:53:43.172 | INFO | "pinned" : " 29 GB" | 20260505 14:53:43.172 | INFO | } | 20260505 14:53:43.172 | INFO | Perf report for the last 5.131 seconds: | 20260505 14:53:43.172 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:53:43.172 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:43.172 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:43.172 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:43.172 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260505 14:53:43.172 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:43.172 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:43.172 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:53:43.172 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:43.172 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:43.173 | INFO | Uploaded 36GiB to GPU in 2.16908s (16.5969GiB/s) | 20260505 14:53:43.173 | INFO | Downloaded 34GiB to GPU in 2.12985s (15.9636GiB/s) | 20260505 14:53:48.312 | INFO | Analyzing chunk frames = [4078080,4078592) | 20260505 14:53:48.312 | INFO | MemInfo: { | 20260505 14:53:48.312 | INFO | "gpu" : " 17 GB", | 20260505 14:53:48.312 | INFO | "huge" : "", | 20260505 14:53:48.312 | INFO | "huge_shared" : " 27 GB", | 20260505 14:53:48.312 | INFO | "malloc" : " 14 GB", | 20260505 14:53:48.312 | INFO | "pinned" : " 29 GB" | 20260505 14:53:48.312 | INFO | } | 20260505 14:53:48.312 | INFO | Perf report for the last 5.14016 seconds: | 20260505 14:53:48.312 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:53:48.312 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:48.312 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:48.312 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:48.312 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 14:53:48.313 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:48.313 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:48.313 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 14:53:48.313 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:48.313 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:48.313 | INFO | Uploaded 36GiB to GPU in 2.21008s (16.289GiB/s) | 20260505 14:53:48.313 | INFO | Downloaded 34GiB to GPU in 2.11376s (16.0851GiB/s) | 20260505 14:53:53.459 | INFO | Analyzing chunk frames = [4078592,4079104) | 20260505 14:53:53.459 | INFO | MemInfo: { | 20260505 14:53:53.459 | INFO | "gpu" : " 17 GB", | 20260505 14:53:53.459 | INFO | "huge" : "", | 20260505 14:53:53.459 | INFO | "huge_shared" : " 27 GB", | 20260505 14:53:53.459 | INFO | "malloc" : " 14 GB", | 20260505 14:53:53.459 | INFO | "pinned" : " 29 GB" | 20260505 14:53:53.459 | INFO | } | 20260505 14:53:53.459 | INFO | Perf report for the last 5.14686 seconds: | 20260505 14:53:53.459 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:53:53.459 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:53.459 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:53.459 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:53.459 | INFO | ANALYSIS : 63.4%, 3.0, 99.2ms, 0 | 20260505 14:53:53.459 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:53.459 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:53.459 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 14:53:53.459 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:53.459 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:53.460 | INFO | Uploaded 36GiB to GPU in 2.12034s (16.9784GiB/s) | 20260505 14:53:53.460 | INFO | Downloaded 34GiB to GPU in 2.16919s (15.6741GiB/s) | 20260505 14:53:58.602 | INFO | Analyzing chunk frames = [4079104,4079616) | 20260505 14:53:58.602 | INFO | MemInfo: { | 20260505 14:53:58.602 | INFO | "gpu" : " 17 GB", | 20260505 14:53:58.602 | INFO | "huge" : "", | 20260505 14:53:58.602 | INFO | "huge_shared" : " 27 GB", | 20260505 14:53:58.602 | INFO | "malloc" : " 14 GB", | 20260505 14:53:58.602 | INFO | "pinned" : " 29 GB" | 20260505 14:53:58.602 | INFO | } | 20260505 14:53:58.602 | INFO | Perf report for the last 5.14319 seconds: | 20260505 14:53:58.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:53:58.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:58.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:58.603 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:58.603 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 14:53:58.603 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:58.603 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:58.603 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 14:53:58.603 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:58.603 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:53:58.603 | INFO | Uploaded 36GiB to GPU in 2.21109s (16.2816GiB/s) | 20260505 14:53:58.603 | INFO | Downloaded 34GiB to GPU in 2.10293s (16.1679GiB/s) | 20260505 14:53:58.603 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 14:53:58.603 | INFO | Total mem usage: 15.3 GiB | 20260505 14:53:58.603 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 14:53:58.603 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.84%) | 20260505 14:53:58.603 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 14:53:58.603 | INFO | High Water Breakdown: | 20260505 14:53:58.603 | INFO | 61% : 8.67 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 14:53:58.603 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 14:53:58.603 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 14:53:58.603 | INFO | ------------------------------------------ | 20260505 14:53:58.603 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:53:58.603 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 14:53:58.603 | INFO | Total mem usage: 26.5 GiB | 20260505 14:53:58.603 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 14:53:58.603 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 14:53:58.603 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 14:53:58.603 | INFO | High Water Breakdown: | 20260505 14:53:58.603 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:53:58.603 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:53:58.603 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:53:58.603 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:53:58.603 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:53:58.603 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:53:58.603 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 14:53:58.603 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 14:53:58.603 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:53:58.603 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 14:53:58.603 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 14:53:58.603 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 14:53:58.603 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 14:53:58.603 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 14:53:58.603 | INFO | ------------------------------------------ | 20260505 14:53:58.603 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:53:58.603 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 14:53:58.603 | INFO | Total mem usage: 25.6 GiB | 20260505 14:53:58.603 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 14:53:58.603 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 14:53:58.603 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 14:53:58.603 | INFO | High Water Breakdown: | 20260505 14:53:58.603 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 14:53:58.603 | INFO | ------------------------------------------ | 20260505 14:53:58.603 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:53:58.604 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 14:53:58.604 | INFO | Total mem usage: 15.7 GiB | 20260505 14:53:58.604 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 14:53:58.604 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 14:53:58.604 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 14:53:58.604 | INFO | High Water Breakdown: | 20260505 14:53:58.604 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 14:53:58.604 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 14:53:58.604 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 14:53:58.604 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 14:53:58.604 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 14:53:58.604 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 14:53:58.604 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 14:53:58.604 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 14:53:58.604 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 14:53:58.604 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:53:58.604 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 14:53:58.604 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:53:58.604 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 14:53:58.604 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 14:53:58.604 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 14:53:58.604 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 14:53:58.604 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:53:58.604 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 14:53:58.604 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:53:58.604 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:53:58.604 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:53:58.604 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 14:53:58.604 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 14:53:58.604 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:53:58.604 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 14:53:58.604 | INFO | ------------------------------------------ | 20260505 14:53:58.604 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:53:58.604 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 14:53:58.604 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:53:58.604 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:53:58.604 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.59e+01, 1.39e+01 / 1.59e+00 | 20260505 14:53:58.604 | INFO | PerfReport: 0.29%, AggregateStats -- 9.27e-02 / 1.01e-01, 1.16e-02 / 5.11e-03 | 20260505 14:53:58.604 | INFO | PerfReport: 11.77%, Baseline -- 3.73e+00 / 4.46e+00, 5.74e-01 / 2.12e-01 | 20260505 14:53:58.604 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.50e+00, 1.95e-01 / 1.10e-01 | 20260505 14:53:58.604 | INFO | PerfReport: 15.99%, DME -- 4.06e+01 / 4.53e+01, 4.50e+00 / 1.65e+00 | 20260505 14:53:58.604 | INFO | PerfReport: 56.46%, FrameLabeling -- 1.79e+01 / 1.97e+01, 1.34e+00 / 1.48e+00 | 20260505 14:53:58.604 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.36e+00 / 2.92e+00, 4.51e-01 / 1.76e-01 | 20260505 14:53:58.604 | INFO | PerfReport: 3.51%, Metrics -- 1.11e+00 / 1.46e+00, 3.10e-01 / 1.64e-01 | 20260505 14:53:58.604 | INFO | PerfReport: 1.23%, PulseSerializing -- 3.90e-01 / 4.41e-01, 6.92e-02 / 3.14e-02 | 20260505 14:53:58.604 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 14:53:58.604 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:53:58.604 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:53:58.604 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260505 14:53:58.604 | INFO | PerfReport: 22.56%, Upload -- 2.27e+01 / 0.00e+00, 7.97e+00 / 0.00e+00 | 20260505 14:53:58.604 | INFO | PerfReport: 21.96%, Download -- 2.21e+01 / 0.00e+00, 7.98e+00 / 0.00e+00 | 20260505 14:53:58.604 | INFO | PerfReport: 31.53%, Compute -- 3.17e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260505 14:53:58.604 | INFO | PerfReport: 2.31%, LockUpload -- 2.33e+00 / 0.00e+00, 8.01e+00 / 0.00e+00 | 20260505 14:53:58.604 | INFO | PerfReport: 1.48%, LockDownload -- 1.49e+00 / 0.00e+00, 3.03e+00 / 0.00e+00 | 20260505 14:53:58.604 | INFO | PerfReport: 20.13%, LockCompute -- 2.03e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 14:53:58.604 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.79e-02 / 0.00e+00 | 20260505 14:53:58.733 | INFO | Starting a round of Baz IO | 20260505 14:54:01.954 | INFO | Populating RTMetrics report | 20260505 14:54:01.954 | INFO | RTMetrics report populated | 20260505 14:54:01.954 | INFO | Opening tmp RTMetrics json file | 20260505 14:54:01.973 | INFO | Serializing RTMetrics to json | 20260505 14:54:01.996 | INFO | Serialization done, swapping tmp file now... | 20260505 14:54:01.998 | INFO | RTMetrics done | 20260505 14:54:02.062 | INFO | Finished a round of Baz IO: Wrote 2.235 GiB in 3.328 seconds | 20260505 14:54:02.062 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.62% / 0.00% / 77.38% | 20260505 14:54:02.062 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.504533% / 3.97364e-06% / 7.15256e-05% | 20260505 14:54:02.062 | INFO | Baz IO overall: | 20260505 14:54:02.062 | INFO | BytesWritten : 2429363712420 | 20260505 14:54:02.062 | INFO | NumEvents : 1154627367257 | 20260505 14:54:03.734 | INFO | Analyzing chunk frames = [4079616,4080128) | 20260505 14:54:03.735 | INFO | MemInfo: { | 20260505 14:54:03.735 | INFO | "gpu" : " 17 GB", | 20260505 14:54:03.735 | INFO | "huge" : "", | 20260505 14:54:03.735 | INFO | "huge_shared" : " 27 GB", | 20260505 14:54:03.735 | INFO | "malloc" : " 14 GB", | 20260505 14:54:03.735 | INFO | "pinned" : " 29 GB" | 20260505 14:54:03.735 | INFO | } | 20260505 14:54:03.735 | INFO | Perf report for the last 5.1321 seconds: | 20260505 14:54:03.735 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:54:03.735 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:03.735 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:03.735 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:03.735 | INFO | ANALYSIS : 64.2%, 3.0, 99.8ms, 0 | 20260505 14:54:03.735 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:03.735 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260505 14:54:03.735 | INFO | PRE_HQ : 61.3%, 1.8, 57.4ms, 0 | 20260505 14:54:03.735 | INFO | BAZWRITER : 64.2%, 3.9, 129.6ms, 0 | 20260505 14:54:03.735 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 14:54:03.735 | INFO | Uploaded 36GiB to GPU in 2.21129s (16.2801GiB/s) | 20260505 14:54:03.735 | INFO | Downloaded 38GiB to GPU in 2.261s (16.8067GiB/s) | 20260505 14:54:08.865 | INFO | Analyzing chunk frames = [4080128,4080640) | 20260505 14:54:08.865 | INFO | MemInfo: { | 20260505 14:54:08.865 | INFO | "gpu" : " 17 GB", | 20260505 14:54:08.865 | INFO | "huge" : "", | 20260505 14:54:08.865 | INFO | "huge_shared" : " 27 GB", | 20260505 14:54:08.866 | INFO | "malloc" : " 14 GB", | 20260505 14:54:08.866 | INFO | "pinned" : " 29 GB" | 20260505 14:54:08.866 | INFO | } | 20260505 14:54:08.866 | INFO | Perf report for the last 5.13113 seconds: | 20260505 14:54:08.866 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:54:08.866 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:08.866 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:08.866 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:08.866 | INFO | ANALYSIS : 63.4%, 3.0, 98.8ms, 0 | 20260505 14:54:08.866 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:08.866 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:08.866 | INFO | PRE_HQ : 28.3%, 1.0, 14.6ms, 0 | 20260505 14:54:08.866 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:08.866 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:08.866 | INFO | Uploaded 36GiB to GPU in 2.1912s (16.4293GiB/s) | 20260505 14:54:08.866 | INFO | Downloaded 34GiB to GPU in 2.03587s (16.7005GiB/s) | 20260505 14:54:14.010 | INFO | Analyzing chunk frames = [4080640,4081152) | 20260505 14:54:14.010 | INFO | MemInfo: { | 20260505 14:54:14.010 | INFO | "gpu" : " 17 GB", | 20260505 14:54:14.010 | INFO | "huge" : "", | 20260505 14:54:14.011 | INFO | "huge_shared" : " 27 GB", | 20260505 14:54:14.011 | INFO | "malloc" : " 14 GB", | 20260505 14:54:14.011 | INFO | "pinned" : " 29 GB" | 20260505 14:54:14.011 | INFO | } | 20260505 14:54:14.011 | INFO | Perf report for the last 5.14494 seconds: | 20260505 14:54:14.011 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:54:14.011 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:14.011 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:14.011 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:14.011 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 14:54:14.011 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:14.011 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:14.011 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 14:54:14.011 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:14.011 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:14.011 | INFO | Uploaded 36GiB to GPU in 2.23073s (16.1382GiB/s) | 20260505 14:54:14.011 | INFO | Downloaded 34GiB to GPU in 2.10187s (16.1761GiB/s) | 20260505 14:54:19.153 | INFO | Analyzing chunk frames = [4081152,4081664) | 20260505 14:54:19.153 | INFO | MemInfo: { | 20260505 14:54:19.153 | INFO | "gpu" : " 17 GB", | 20260505 14:54:19.154 | INFO | "huge" : "", | 20260505 14:54:19.154 | INFO | "huge_shared" : " 27 GB", | 20260505 14:54:19.154 | INFO | "malloc" : " 14 GB", | 20260505 14:54:19.154 | INFO | "pinned" : " 29 GB" | 20260505 14:54:19.154 | INFO | } | 20260505 14:54:19.154 | INFO | Perf report for the last 5.1431 seconds: | 20260505 14:54:19.154 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:54:19.154 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:19.154 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:19.154 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:19.154 | INFO | ANALYSIS : 62.7%, 3.0, 97.4ms, 0 | 20260505 14:54:19.154 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:19.154 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:19.154 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:54:19.154 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:19.154 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:19.154 | INFO | Uploaded 36GiB to GPU in 2.17662s (16.5394GiB/s) | 20260505 14:54:19.154 | INFO | Downloaded 34GiB to GPU in 2.12997s (15.9627GiB/s) | 20260505 14:54:24.303 | INFO | Analyzing chunk frames = [4081664,4082176) | 20260505 14:54:24.303 | INFO | MemInfo: { | 20260505 14:54:24.303 | INFO | "gpu" : " 17 GB", | 20260505 14:54:24.303 | INFO | "huge" : "", | 20260505 14:54:24.303 | INFO | "huge_shared" : " 27 GB", | 20260505 14:54:24.303 | INFO | "malloc" : " 14 GB", | 20260505 14:54:24.303 | INFO | "pinned" : " 29 GB" | 20260505 14:54:24.303 | INFO | } | 20260505 14:54:24.303 | INFO | Perf report for the last 5.1497 seconds: | 20260505 14:54:24.304 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:54:24.304 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:24.304 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:24.304 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:24.304 | INFO | ANALYSIS : 62.6%, 3.0, 97.9ms, 0 | 20260505 14:54:24.304 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:24.304 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:24.304 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:54:24.304 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:24.304 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:24.304 | INFO | Uploaded 36GiB to GPU in 2.18405s (16.4832GiB/s) | 20260505 14:54:24.304 | INFO | Downloaded 34GiB to GPU in 2.12864s (15.9726GiB/s) | 20260505 14:54:29.433 | INFO | Analyzing chunk frames = [4082176,4082688) | 20260505 14:54:29.433 | INFO | MemInfo: { | 20260505 14:54:29.433 | INFO | "gpu" : " 17 GB", | 20260505 14:54:29.433 | INFO | "huge" : "", | 20260505 14:54:29.433 | INFO | "huge_shared" : " 27 GB", | 20260505 14:54:29.433 | INFO | "malloc" : " 14 GB", | 20260505 14:54:29.433 | INFO | "pinned" : " 29 GB" | 20260505 14:54:29.433 | INFO | } | 20260505 14:54:29.433 | INFO | Perf report for the last 5.12995 seconds: | 20260505 14:54:29.433 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:54:29.433 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:29.433 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:29.434 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:29.434 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 14:54:29.434 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:29.434 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:29.434 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:54:29.434 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:29.434 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:29.434 | INFO | Uploaded 36GiB to GPU in 2.13617s (16.8526GiB/s) | 20260505 14:54:29.434 | INFO | Downloaded 34GiB to GPU in 2.03264s (16.727GiB/s) | 20260505 14:54:34.566 | INFO | Analyzing chunk frames = [4082688,4083200) | 20260505 14:54:34.566 | INFO | MemInfo: { | 20260505 14:54:34.566 | INFO | "gpu" : " 17 GB", | 20260505 14:54:34.566 | INFO | "huge" : "", | 20260505 14:54:34.566 | INFO | "huge_shared" : " 27 GB", | 20260505 14:54:34.566 | INFO | "malloc" : " 14 GB", | 20260505 14:54:34.566 | INFO | "pinned" : " 29 GB" | 20260505 14:54:34.566 | INFO | } | 20260505 14:54:34.566 | INFO | Perf report for the last 5.13243 seconds: | 20260505 14:54:34.566 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:54:34.566 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:34.566 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:34.566 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:34.566 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 14:54:34.566 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:34.566 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:34.566 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 14:54:34.566 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:34.566 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:34.566 | INFO | Uploaded 36GiB to GPU in 2.11472s (17.0236GiB/s) | 20260505 14:54:34.566 | INFO | Downloaded 34GiB to GPU in 2.16573s (15.6991GiB/s) | 20260505 14:54:39.716 | INFO | Analyzing chunk frames = [4083200,4083712) | 20260505 14:54:39.716 | INFO | MemInfo: { | 20260505 14:54:39.716 | INFO | "gpu" : " 17 GB", | 20260505 14:54:39.716 | INFO | "huge" : "", | 20260505 14:54:39.716 | INFO | "huge_shared" : " 27 GB", | 20260505 14:54:39.716 | INFO | "malloc" : " 14 GB", | 20260505 14:54:39.716 | INFO | "pinned" : " 29 GB" | 20260505 14:54:39.716 | INFO | } | 20260505 14:54:39.716 | INFO | Perf report for the last 5.15017 seconds: | 20260505 14:54:39.716 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:54:39.716 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:39.716 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:39.716 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:39.716 | INFO | ANALYSIS : 63.3%, 3.0, 99.1ms, 0 | 20260505 14:54:39.716 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:39.716 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:39.716 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 14:54:39.716 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:39.716 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:39.716 | INFO | Uploaded 36GiB to GPU in 2.17595s (16.5445GiB/s) | 20260505 14:54:39.716 | INFO | Downloaded 34GiB to GPU in 2.10802s (16.1289GiB/s) | 20260505 14:54:39.846 | INFO | Starting a round of Baz IO | 20260505 14:54:43.060 | INFO | Populating RTMetrics report | 20260505 14:54:43.060 | INFO | RTMetrics report populated | 20260505 14:54:43.060 | INFO | Opening tmp RTMetrics json file | 20260505 14:54:43.076 | INFO | Serializing RTMetrics to json | 20260505 14:54:43.081 | INFO | Serialization done, swapping tmp file now... | 20260505 14:54:43.090 | INFO | RTMetrics done | 20260505 14:54:43.165 | INFO | Finished a round of Baz IO: Wrote 2.236 GiB in 3.319 seconds | 20260505 14:54:43.165 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.62% / 0.00% / 77.38% | 20260505 14:54:43.165 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.504927% / 0% / 7.54992e-05% | 20260505 14:54:43.165 | INFO | Baz IO overall: | 20260505 14:54:43.165 | INFO | BytesWritten : 2431764270540 | 20260505 14:54:43.165 | INFO | NumEvents : 1155758103338 | 20260505 14:54:44.852 | INFO | Analyzing chunk frames = [4083712,4084224) | 20260505 14:54:44.852 | INFO | MemInfo: { | 20260505 14:54:44.852 | INFO | "gpu" : " 17 GB", | 20260505 14:54:44.852 | INFO | "huge" : "", | 20260505 14:54:44.852 | INFO | "huge_shared" : " 27 GB", | 20260505 14:54:44.852 | INFO | "malloc" : " 14 GB", | 20260505 14:54:44.852 | INFO | "pinned" : " 29 GB" | 20260505 14:54:44.852 | INFO | } | 20260505 14:54:44.852 | INFO | Perf report for the last 5.13572 seconds: | 20260505 14:54:44.852 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:54:44.852 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:44.852 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:44.852 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:44.852 | INFO | ANALYSIS : 64.0%, 3.0, 99.5ms, 0 | 20260505 14:54:44.852 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:44.852 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260505 14:54:44.852 | INFO | PRE_HQ : 61.0%, 1.8, 57.0ms, 0 | 20260505 14:54:44.852 | INFO | BAZWRITER : 63.8%, 4.0, 130.7ms, 0 | 20260505 14:54:44.852 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 14:54:44.852 | INFO | Uploaded 36GiB to GPU in 2.17313s (16.566GiB/s) | 20260505 14:54:44.852 | INFO | Downloaded 38GiB to GPU in 2.2624s (16.7963GiB/s) | 20260505 14:54:49.987 | INFO | Analyzing chunk frames = [4084224,4084736) | 20260505 14:54:49.987 | INFO | MemInfo: { | 20260505 14:54:49.987 | INFO | "gpu" : " 17 GB", | 20260505 14:54:49.987 | INFO | "huge" : "", | 20260505 14:54:49.988 | INFO | "huge_shared" : " 27 GB", | 20260505 14:54:49.988 | INFO | "malloc" : " 14 GB", | 20260505 14:54:49.988 | INFO | "pinned" : " 29 GB" | 20260505 14:54:49.988 | INFO | } | 20260505 14:54:49.988 | INFO | Perf report for the last 5.13581 seconds: | 20260505 14:54:49.988 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:54:49.988 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:49.988 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:49.988 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:49.988 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 14:54:49.988 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:49.988 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:49.988 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:54:49.988 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:49.988 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:49.988 | INFO | Uploaded 36GiB to GPU in 2.20044s (16.3604GiB/s) | 20260505 14:54:49.988 | INFO | Downloaded 34GiB to GPU in 2.05661s (16.5321GiB/s) | 20260505 14:54:55.121 | INFO | Analyzing chunk frames = [4084736,4085248) | 20260505 14:54:55.121 | INFO | MemInfo: { | 20260505 14:54:55.121 | INFO | "gpu" : " 17 GB", | 20260505 14:54:55.121 | INFO | "huge" : "", | 20260505 14:54:55.121 | INFO | "huge_shared" : " 27 GB", | 20260505 14:54:55.121 | INFO | "malloc" : " 14 GB", | 20260505 14:54:55.121 | INFO | "pinned" : " 29 GB" | 20260505 14:54:55.121 | INFO | } | 20260505 14:54:55.121 | INFO | Perf report for the last 5.13342 seconds: | 20260505 14:54:55.121 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:54:55.121 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:55.121 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:55.121 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:55.121 | INFO | ANALYSIS : 63.7%, 3.0, 99.0ms, 0 | 20260505 14:54:55.121 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:55.121 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:55.121 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 14:54:55.121 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:55.121 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:54:55.121 | INFO | Uploaded 36GiB to GPU in 2.23961s (16.0742GiB/s) | 20260505 14:54:55.121 | INFO | Downloaded 34GiB to GPU in 2.1206s (16.0332GiB/s) | 20260505 14:55:00.266 | INFO | Analyzing chunk frames = [4085248,4085760) | 20260505 14:55:00.266 | INFO | MemInfo: { | 20260505 14:55:00.266 | INFO | "gpu" : " 17 GB", | 20260505 14:55:00.266 | INFO | "huge" : "", | 20260505 14:55:00.266 | INFO | "huge_shared" : " 27 GB", | 20260505 14:55:00.266 | INFO | "malloc" : " 14 GB", | 20260505 14:55:00.266 | INFO | "pinned" : " 29 GB" | 20260505 14:55:00.266 | INFO | } | 20260505 14:55:00.266 | INFO | Perf report for the last 5.14454 seconds: | 20260505 14:55:00.266 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:55:00.266 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:00.266 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:00.266 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:00.266 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260505 14:55:00.266 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:00.266 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:00.266 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:55:00.266 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:00.266 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:00.266 | INFO | Uploaded 36GiB to GPU in 2.19578s (16.3951GiB/s) | 20260505 14:55:00.266 | INFO | Downloaded 34GiB to GPU in 2.13726s (15.9082GiB/s) | 20260505 14:55:05.412 | INFO | Analyzing chunk frames = [4085760,4086272) | 20260505 14:55:05.412 | INFO | MemInfo: { | 20260505 14:55:05.412 | INFO | "gpu" : " 17 GB", | 20260505 14:55:05.412 | INFO | "huge" : "", | 20260505 14:55:05.412 | INFO | "huge_shared" : " 27 GB", | 20260505 14:55:05.412 | INFO | "malloc" : " 14 GB", | 20260505 14:55:05.412 | INFO | "pinned" : " 29 GB" | 20260505 14:55:05.412 | INFO | } | 20260505 14:55:05.412 | INFO | Perf report for the last 5.14676 seconds: | 20260505 14:55:05.412 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:55:05.412 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:05.412 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:05.412 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:05.412 | INFO | ANALYSIS : 62.7%, 3.0, 98.0ms, 0 | 20260505 14:55:05.412 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:05.412 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:05.412 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:55:05.413 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:05.413 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:05.413 | INFO | Uploaded 36GiB to GPU in 2.17691s (16.5372GiB/s) | 20260505 14:55:05.413 | INFO | Downloaded 34GiB to GPU in 2.12883s (15.9712GiB/s) | 20260505 14:55:10.552 | INFO | Analyzing chunk frames = [4086272,4086784) | 20260505 14:55:10.552 | INFO | MemInfo: { | 20260505 14:55:10.552 | INFO | "gpu" : " 17 GB", | 20260505 14:55:10.552 | INFO | "huge" : "", | 20260505 14:55:10.552 | INFO | "huge_shared" : " 27 GB", | 20260505 14:55:10.552 | INFO | "malloc" : " 14 GB", | 20260505 14:55:10.552 | INFO | "pinned" : " 29 GB" | 20260505 14:55:10.552 | INFO | } | 20260505 14:55:10.552 | INFO | Perf report for the last 5.13946 seconds: | 20260505 14:55:10.552 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:55:10.552 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:10.552 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:10.552 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:10.552 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 14:55:10.552 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:10.552 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:10.552 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 14:55:10.552 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:10.552 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:10.552 | INFO | Uploaded 36GiB to GPU in 2.13123s (16.8917GiB/s) | 20260505 14:55:10.552 | INFO | Downloaded 34GiB to GPU in 2.03354s (16.7196GiB/s) | 20260505 14:55:15.681 | INFO | Analyzing chunk frames = [4086784,4087296) | 20260505 14:55:15.681 | INFO | MemInfo: { | 20260505 14:55:15.681 | INFO | "gpu" : " 17 GB", | 20260505 14:55:15.681 | INFO | "huge" : "", | 20260505 14:55:15.681 | INFO | "huge_shared" : " 27 GB", | 20260505 14:55:15.681 | INFO | "malloc" : " 14 GB", | 20260505 14:55:15.681 | INFO | "pinned" : " 29 GB" | 20260505 14:55:15.681 | INFO | } | 20260505 14:55:15.681 | INFO | Perf report for the last 5.12885 seconds: | 20260505 14:55:15.681 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:55:15.681 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:15.681 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:15.681 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:15.681 | INFO | ANALYSIS : 63.5%, 3.0, 99.0ms, 0 | 20260505 14:55:15.681 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:15.681 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:15.681 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 14:55:15.681 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:15.681 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:15.681 | INFO | Uploaded 36GiB to GPU in 2.12053s (16.9769GiB/s) | 20260505 14:55:15.681 | INFO | Downloaded 34GiB to GPU in 2.17211s (15.653GiB/s) | 20260505 14:55:20.817 | INFO | Analyzing chunk frames = [4087296,4087808) | 20260505 14:55:20.817 | INFO | MemInfo: { | 20260505 14:55:20.817 | INFO | "gpu" : " 17 GB", | 20260505 14:55:20.817 | INFO | "huge" : "", | 20260505 14:55:20.817 | INFO | "huge_shared" : " 27 GB", | 20260505 14:55:20.817 | INFO | "malloc" : " 14 GB", | 20260505 14:55:20.817 | INFO | "pinned" : " 29 GB" | 20260505 14:55:20.817 | INFO | } | 20260505 14:55:20.818 | INFO | Perf report for the last 5.13679 seconds: | 20260505 14:55:20.818 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:55:20.818 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:20.818 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:20.818 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:20.818 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 14:55:20.818 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:20.818 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:20.818 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:55:20.818 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:20.818 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:20.818 | INFO | Uploaded 36GiB to GPU in 2.21139s (16.2794GiB/s) | 20260505 14:55:20.818 | INFO | Downloaded 34GiB to GPU in 2.10419s (16.1583GiB/s) | 20260505 14:55:20.818 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 14:55:20.818 | INFO | Total mem usage: 15.3 GiB | 20260505 14:55:20.818 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 14:55:20.818 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.84%) | 20260505 14:55:20.818 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 14:55:20.818 | INFO | High Water Breakdown: | 20260505 14:55:20.818 | INFO | 61% : 8.67 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 14:55:20.818 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 14:55:20.818 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 14:55:20.818 | INFO | ------------------------------------------ | 20260505 14:55:20.818 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:55:20.818 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 14:55:20.818 | INFO | Total mem usage: 26.5 GiB | 20260505 14:55:20.818 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 14:55:20.818 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 14:55:20.818 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 14:55:20.818 | INFO | High Water Breakdown: | 20260505 14:55:20.818 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:55:20.818 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:55:20.818 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:55:20.818 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:55:20.818 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:55:20.818 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:55:20.818 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 14:55:20.818 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 14:55:20.818 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:55:20.818 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 14:55:20.818 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 14:55:20.818 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 14:55:20.818 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 14:55:20.818 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 14:55:20.818 | INFO | ------------------------------------------ | 20260505 14:55:20.818 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:55:20.818 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 14:55:20.818 | INFO | Total mem usage: 25.6 GiB | 20260505 14:55:20.818 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 14:55:20.818 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 14:55:20.818 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 14:55:20.819 | INFO | High Water Breakdown: | 20260505 14:55:20.819 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 14:55:20.819 | INFO | ------------------------------------------ | 20260505 14:55:20.819 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:55:20.819 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 14:55:20.819 | INFO | Total mem usage: 15.7 GiB | 20260505 14:55:20.819 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 14:55:20.819 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 14:55:20.819 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 14:55:20.819 | INFO | High Water Breakdown: | 20260505 14:55:20.819 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 14:55:20.819 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 14:55:20.819 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 14:55:20.819 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 14:55:20.819 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 14:55:20.819 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 14:55:20.819 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 14:55:20.819 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 14:55:20.819 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 14:55:20.819 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:55:20.819 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 14:55:20.819 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:55:20.819 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 14:55:20.819 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 14:55:20.819 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 14:55:20.819 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 14:55:20.819 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:55:20.819 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 14:55:20.819 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:55:20.819 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:55:20.819 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:55:20.819 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 14:55:20.819 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 14:55:20.819 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:55:20.819 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 14:55:20.819 | INFO | ------------------------------------------ | 20260505 14:55:20.819 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:55:20.819 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 14:55:20.819 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:55:20.819 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:55:20.819 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.59e+01, 1.38e+01 / 1.62e+00 | 20260505 14:55:20.819 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 9.98e-02, 1.14e-02 / 3.73e-03 | 20260505 14:55:20.819 | INFO | PerfReport: 11.80%, Baseline -- 3.74e+00 / 4.52e+00, 5.75e-01 / 1.11e-01 | 20260505 14:55:20.819 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.50e+00, 1.92e-01 / 1.07e-01 | 20260505 14:55:20.819 | INFO | PerfReport: 15.97%, DME -- 4.05e+01 / 4.52e+01, 4.36e+00 / 1.81e+00 | 20260505 14:55:20.819 | INFO | PerfReport: 56.45%, FrameLabeling -- 1.79e+01 / 1.97e+01, 1.34e+00 / 1.76e+00 | 20260505 14:55:20.819 | INFO | PerfReport: 7.46%, PulseAccumulating -- 2.37e+00 / 2.92e+00, 4.50e-01 / 1.64e-01 | 20260505 14:55:20.819 | INFO | PerfReport: 3.52%, Metrics -- 1.12e+00 / 1.42e+00, 3.11e-01 / 1.57e-01 | 20260505 14:55:20.819 | INFO | PerfReport: 1.23%, PulseSerializing -- 3.91e-01 / 4.49e-01, 6.98e-02 / 3.90e-02 | 20260505 14:55:20.819 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 14:55:20.819 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:55:20.819 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:55:20.819 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260505 14:55:20.819 | INFO | PerfReport: 22.57%, Upload -- 2.27e+01 / 0.00e+00, 8.07e+00 / 0.00e+00 | 20260505 14:55:20.819 | INFO | PerfReport: 21.99%, Download -- 2.21e+01 / 0.00e+00, 7.94e+00 / 0.00e+00 | 20260505 14:55:20.819 | INFO | PerfReport: 31.55%, Compute -- 3.17e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 14:55:20.819 | INFO | PerfReport: 2.28%, LockUpload -- 2.30e+00 / 0.00e+00, 7.84e+00 / 0.00e+00 | 20260505 14:55:20.819 | INFO | PerfReport: 1.41%, LockDownload -- 1.42e+00 / 0.00e+00, 2.93e+00 / 0.00e+00 | 20260505 14:55:20.819 | INFO | PerfReport: 20.19%, LockCompute -- 2.03e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 14:55:20.819 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 2.51e-02 / 0.00e+00 | 20260505 14:55:20.957 | INFO | Starting a round of Baz IO | 20260505 14:55:24.184 | INFO | Populating RTMetrics report | 20260505 14:55:24.185 | INFO | RTMetrics report populated | 20260505 14:55:24.185 | INFO | Opening tmp RTMetrics json file | 20260505 14:55:24.188 | INFO | Serializing RTMetrics to json | 20260505 14:55:24.211 | INFO | Serialization done, swapping tmp file now... | 20260505 14:55:24.234 | INFO | RTMetrics done | 20260505 14:55:24.296 | INFO | Finished a round of Baz IO: Wrote 2.236 GiB in 3.338 seconds | 20260505 14:55:24.296 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.61% / 0.00% / 77.38% | 20260505 14:55:24.296 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.502026% / 1.19209e-05% / 8.34465e-05% | 20260505 14:55:24.296 | INFO | Baz IO overall: | 20260505 14:55:24.296 | INFO | BytesWritten : 2434164967035 | 20260505 14:55:24.296 | INFO | NumEvents : 1156888961363 | 20260505 14:55:25.963 | INFO | Analyzing chunk frames = [4087808,4088320) | 20260505 14:55:25.963 | INFO | MemInfo: { | 20260505 14:55:25.963 | INFO | "gpu" : " 17 GB", | 20260505 14:55:25.963 | INFO | "huge" : "", | 20260505 14:55:25.963 | INFO | "huge_shared" : " 27 GB", | 20260505 14:55:25.963 | INFO | "malloc" : " 14 GB", | 20260505 14:55:25.963 | INFO | "pinned" : " 29 GB" | 20260505 14:55:25.963 | INFO | } | 20260505 14:55:25.963 | INFO | Perf report for the last 5.14511 seconds: | 20260505 14:55:25.963 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:55:25.963 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:25.963 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:25.963 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:25.963 | INFO | ANALYSIS : 64.3%, 3.0, 100.2ms, 0 | 20260505 14:55:25.963 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:25.963 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260505 14:55:25.963 | INFO | PRE_HQ : 60.6%, 1.8, 56.9ms, 0 | 20260505 14:55:25.963 | INFO | BAZWRITER : 64.2%, 3.9, 130.5ms, 0 | 20260505 14:55:25.963 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 14:55:25.963 | INFO | Uploaded 36GiB to GPU in 2.19949s (16.3674GiB/s) | 20260505 14:55:25.963 | INFO | Downloaded 38GiB to GPU in 2.27618s (16.6946GiB/s) | 20260505 14:55:31.108 | INFO | Analyzing chunk frames = [4088320,4088832) | 20260505 14:55:31.108 | INFO | MemInfo: { | 20260505 14:55:31.108 | INFO | "gpu" : " 17 GB", | 20260505 14:55:31.108 | INFO | "huge" : "", | 20260505 14:55:31.108 | INFO | "huge_shared" : " 27 GB", | 20260505 14:55:31.108 | INFO | "malloc" : " 14 GB", | 20260505 14:55:31.108 | INFO | "pinned" : " 29 GB" | 20260505 14:55:31.108 | INFO | } | 20260505 14:55:31.108 | INFO | Perf report for the last 5.14577 seconds: | 20260505 14:55:31.108 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:55:31.108 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:31.108 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:31.108 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:31.108 | INFO | ANALYSIS : 63.3%, 3.0, 99.0ms, 0 | 20260505 14:55:31.108 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:31.108 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:31.108 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:55:31.109 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:31.109 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:31.109 | INFO | Uploaded 36GiB to GPU in 2.20082s (16.3575GiB/s) | 20260505 14:55:31.109 | INFO | Downloaded 34GiB to GPU in 2.0619s (16.4896GiB/s) | 20260505 14:55:36.245 | INFO | Analyzing chunk frames = [4088832,4089344) | 20260505 14:55:36.245 | INFO | MemInfo: { | 20260505 14:55:36.245 | INFO | "gpu" : " 17 GB", | 20260505 14:55:36.245 | INFO | "huge" : "", | 20260505 14:55:36.245 | INFO | "huge_shared" : " 27 GB", | 20260505 14:55:36.245 | INFO | "malloc" : " 14 GB", | 20260505 14:55:36.245 | INFO | "pinned" : " 29 GB" | 20260505 14:55:36.245 | INFO | } | 20260505 14:55:36.245 | INFO | Perf report for the last 5.13681 seconds: | 20260505 14:55:36.245 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:55:36.245 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:36.245 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:36.245 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:36.245 | INFO | ANALYSIS : 62.9%, 3.0, 97.8ms, 0 | 20260505 14:55:36.245 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:36.245 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:36.246 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 14:55:36.246 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:36.246 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:36.246 | INFO | Uploaded 36GiB to GPU in 2.17321s (16.5654GiB/s) | 20260505 14:55:36.246 | INFO | Downloaded 34GiB to GPU in 2.02955s (16.7525GiB/s) | 20260505 14:55:41.379 | INFO | Analyzing chunk frames = [4089344,4089856) | 20260505 14:55:41.379 | INFO | MemInfo: { | 20260505 14:55:41.379 | INFO | "gpu" : " 17 GB", | 20260505 14:55:41.379 | INFO | "huge" : "", | 20260505 14:55:41.379 | INFO | "huge_shared" : " 27 GB", | 20260505 14:55:41.379 | INFO | "malloc" : " 14 GB", | 20260505 14:55:41.379 | INFO | "pinned" : " 29 GB" | 20260505 14:55:41.379 | INFO | } | 20260505 14:55:41.379 | INFO | Perf report for the last 5.13394 seconds: | 20260505 14:55:41.379 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:55:41.379 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:41.379 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:41.379 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:41.379 | INFO | ANALYSIS : 62.6%, 3.0, 97.2ms, 0 | 20260505 14:55:41.379 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:41.379 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:41.379 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:55:41.379 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:41.379 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:41.380 | INFO | Uploaded 36GiB to GPU in 2.17809s (16.5283GiB/s) | 20260505 14:55:41.380 | INFO | Downloaded 34GiB to GPU in 2.13207s (15.9469GiB/s) | 20260505 14:55:46.519 | INFO | Analyzing chunk frames = [4089856,4090368) | 20260505 14:55:46.519 | INFO | MemInfo: { | 20260505 14:55:46.519 | INFO | "gpu" : " 17 GB", | 20260505 14:55:46.519 | INFO | "huge" : "", | 20260505 14:55:46.519 | INFO | "huge_shared" : " 27 GB", | 20260505 14:55:46.519 | INFO | "malloc" : " 14 GB", | 20260505 14:55:46.519 | INFO | "pinned" : " 29 GB" | 20260505 14:55:46.519 | INFO | } | 20260505 14:55:46.519 | INFO | Perf report for the last 5.14024 seconds: | 20260505 14:55:46.520 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:55:46.520 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:46.520 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:46.520 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:46.520 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 14:55:46.520 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:46.520 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:46.520 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:55:46.520 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:46.520 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:46.520 | INFO | Uploaded 36GiB to GPU in 2.16585s (16.6216GiB/s) | 20260505 14:55:46.520 | INFO | Downloaded 34GiB to GPU in 2.12465s (16.0026GiB/s) | 20260505 14:55:51.664 | INFO | Analyzing chunk frames = [4090368,4090880) | 20260505 14:55:51.664 | INFO | MemInfo: { | 20260505 14:55:51.664 | INFO | "gpu" : " 17 GB", | 20260505 14:55:51.664 | INFO | "huge" : "", | 20260505 14:55:51.664 | INFO | "huge_shared" : " 27 GB", | 20260505 14:55:51.664 | INFO | "malloc" : " 14 GB", | 20260505 14:55:51.664 | INFO | "pinned" : " 29 GB" | 20260505 14:55:51.664 | INFO | } | 20260505 14:55:51.664 | INFO | Perf report for the last 5.14451 seconds: | 20260505 14:55:51.664 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:55:51.664 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:51.664 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:51.664 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:51.664 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 14:55:51.664 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:51.664 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:51.664 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 14:55:51.664 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:51.664 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:51.664 | INFO | Uploaded 36GiB to GPU in 2.20654s (16.3151GiB/s) | 20260505 14:55:51.664 | INFO | Downloaded 34GiB to GPU in 2.11296s (16.0912GiB/s) | 20260505 14:55:56.804 | INFO | Analyzing chunk frames = [4090880,4091392) | 20260505 14:55:56.804 | INFO | MemInfo: { | 20260505 14:55:56.804 | INFO | "gpu" : " 17 GB", | 20260505 14:55:56.804 | INFO | "huge" : "", | 20260505 14:55:56.804 | INFO | "huge_shared" : " 27 GB", | 20260505 14:55:56.804 | INFO | "malloc" : " 14 GB", | 20260505 14:55:56.804 | INFO | "pinned" : " 29 GB" | 20260505 14:55:56.804 | INFO | } | 20260505 14:55:56.804 | INFO | Perf report for the last 5.14046 seconds: | 20260505 14:55:56.804 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:55:56.804 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:56.804 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:56.804 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:56.805 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 14:55:56.805 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:56.805 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:56.805 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 14:55:56.805 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:56.805 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:55:56.805 | INFO | Uploaded 36GiB to GPU in 2.17247s (16.571GiB/s) | 20260505 14:55:56.805 | INFO | Downloaded 34GiB to GPU in 2.14914s (15.8203GiB/s) | 20260505 14:56:01.933 | INFO | Analyzing chunk frames = [4091392,4091904) | 20260505 14:56:01.933 | INFO | MemInfo: { | 20260505 14:56:01.933 | INFO | "gpu" : " 17 GB", | 20260505 14:56:01.933 | INFO | "huge" : "", | 20260505 14:56:01.933 | INFO | "huge_shared" : " 27 GB", | 20260505 14:56:01.933 | INFO | "malloc" : " 14 GB", | 20260505 14:56:01.933 | INFO | "pinned" : " 29 GB" | 20260505 14:56:01.933 | INFO | } | 20260505 14:56:01.933 | INFO | Perf report for the last 5.12876 seconds: | 20260505 14:56:01.933 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:56:01.933 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:01.933 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:01.933 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:01.933 | INFO | ANALYSIS : 63.6%, 3.0, 99.1ms, 0 | 20260505 14:56:01.933 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:01.933 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:01.933 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 14:56:01.933 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:01.933 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:01.933 | INFO | Uploaded 36GiB to GPU in 2.20134s (16.3537GiB/s) | 20260505 14:56:01.934 | INFO | Downloaded 34GiB to GPU in 2.10832s (16.1266GiB/s) | 20260505 14:56:02.069 | INFO | Starting a round of Baz IO | 20260505 14:56:05.286 | INFO | Populating RTMetrics report | 20260505 14:56:05.286 | INFO | RTMetrics report populated | 20260505 14:56:05.286 | INFO | Opening tmp RTMetrics json file | 20260505 14:56:05.302 | INFO | Serializing RTMetrics to json | 20260505 14:56:05.314 | INFO | Serialization done, swapping tmp file now... | 20260505 14:56:05.327 | INFO | RTMetrics done | 20260505 14:56:05.386 | INFO | Finished a round of Baz IO: Wrote 2.235 GiB in 3.316 seconds | 20260505 14:56:05.386 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.61% / 0.00% / 77.39% | 20260505 14:56:05.386 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.500051% / 7.94729e-06% / 7.94729e-05% | 20260505 14:56:05.386 | INFO | Baz IO overall: | 20260505 14:56:05.386 | INFO | BytesWritten : 2436564476942 | 20260505 14:56:05.386 | INFO | NumEvents : 1158019069255 | 20260505 14:56:07.073 | INFO | Analyzing chunk frames = [4091904,4092416) | 20260505 14:56:07.073 | INFO | MemInfo: { | 20260505 14:56:07.073 | INFO | "gpu" : " 17 GB", | 20260505 14:56:07.073 | INFO | "huge" : "", | 20260505 14:56:07.073 | INFO | "huge_shared" : " 27 GB", | 20260505 14:56:07.073 | INFO | "malloc" : " 14 GB", | 20260505 14:56:07.073 | INFO | "pinned" : " 29 GB" | 20260505 14:56:07.073 | INFO | } | 20260505 14:56:07.073 | INFO | Perf report for the last 5.13986 seconds: | 20260505 14:56:07.073 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:56:07.073 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:07.074 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:07.074 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:07.074 | INFO | ANALYSIS : 64.1%, 3.0, 99.6ms, 0 | 20260505 14:56:07.074 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:07.074 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 14:56:07.074 | INFO | PRE_HQ : 60.9%, 1.8, 57.5ms, 0 | 20260505 14:56:07.074 | INFO | BAZWRITER : 63.9%, 4.0, 131.1ms, 0 | 20260505 14:56:07.074 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 14:56:07.074 | INFO | Uploaded 36GiB to GPU in 2.18953s (16.4419GiB/s) | 20260505 14:56:07.074 | INFO | Downloaded 38GiB to GPU in 2.25988s (16.815GiB/s) | 20260505 14:56:12.220 | INFO | Analyzing chunk frames = [4092416,4092928) | 20260505 14:56:12.220 | INFO | MemInfo: { | 20260505 14:56:12.220 | INFO | "gpu" : " 17 GB", | 20260505 14:56:12.220 | INFO | "huge" : "", | 20260505 14:56:12.220 | INFO | "huge_shared" : " 27 GB", | 20260505 14:56:12.220 | INFO | "malloc" : " 14 GB", | 20260505 14:56:12.220 | INFO | "pinned" : " 29 GB" | 20260505 14:56:12.220 | INFO | } | 20260505 14:56:12.220 | INFO | Perf report for the last 5.14721 seconds: | 20260505 14:56:12.220 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:56:12.220 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:12.220 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:12.220 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:12.220 | INFO | ANALYSIS : 63.2%, 3.0, 98.8ms, 0 | 20260505 14:56:12.220 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:12.220 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:12.220 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 14:56:12.220 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:12.220 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:12.221 | INFO | Uploaded 36GiB to GPU in 2.18801s (16.4533GiB/s) | 20260505 14:56:12.221 | INFO | Downloaded 34GiB to GPU in 2.0354s (16.7044GiB/s) | 20260505 14:56:17.362 | INFO | Analyzing chunk frames = [4092928,4093440) | 20260505 14:56:17.362 | INFO | MemInfo: { | 20260505 14:56:17.362 | INFO | "gpu" : " 17 GB", | 20260505 14:56:17.362 | INFO | "huge" : "", | 20260505 14:56:17.362 | INFO | "huge_shared" : " 27 GB", | 20260505 14:56:17.363 | INFO | "malloc" : " 14 GB", | 20260505 14:56:17.363 | INFO | "pinned" : " 29 GB" | 20260505 14:56:17.363 | INFO | } | 20260505 14:56:17.363 | INFO | Perf report for the last 5.14229 seconds: | 20260505 14:56:17.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:56:17.363 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:17.363 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:17.363 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:17.363 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260505 14:56:17.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:17.363 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:17.363 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:56:17.363 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:17.363 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:17.363 | INFO | Uploaded 36GiB to GPU in 2.17364s (16.5621GiB/s) | 20260505 14:56:17.363 | INFO | Downloaded 34GiB to GPU in 2.03537s (16.7045GiB/s) | 20260505 14:56:22.495 | INFO | Analyzing chunk frames = [4093440,4093952) | 20260505 14:56:22.495 | INFO | MemInfo: { | 20260505 14:56:22.495 | INFO | "gpu" : " 17 GB", | 20260505 14:56:22.495 | INFO | "huge" : "", | 20260505 14:56:22.495 | INFO | "huge_shared" : " 27 GB", | 20260505 14:56:22.495 | INFO | "malloc" : " 14 GB", | 20260505 14:56:22.495 | INFO | "pinned" : " 29 GB" | 20260505 14:56:22.495 | INFO | } | 20260505 14:56:22.495 | INFO | Perf report for the last 5.13223 seconds: | 20260505 14:56:22.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:56:22.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:22.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:22.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:22.495 | INFO | ANALYSIS : 62.7%, 3.0, 97.3ms, 0 | 20260505 14:56:22.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:22.495 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:22.495 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:56:22.495 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:22.495 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:22.495 | INFO | Uploaded 36GiB to GPU in 2.19276s (16.4177GiB/s) | 20260505 14:56:22.495 | INFO | Downloaded 34GiB to GPU in 2.13303s (15.9398GiB/s) | 20260505 14:56:27.631 | INFO | Analyzing chunk frames = [4093952,4094464) | 20260505 14:56:27.631 | INFO | MemInfo: { | 20260505 14:56:27.631 | INFO | "gpu" : " 17 GB", | 20260505 14:56:27.631 | INFO | "huge" : "", | 20260505 14:56:27.631 | INFO | "huge_shared" : " 27 GB", | 20260505 14:56:27.631 | INFO | "malloc" : " 14 GB", | 20260505 14:56:27.631 | INFO | "pinned" : " 29 GB" | 20260505 14:56:27.631 | INFO | } | 20260505 14:56:27.631 | INFO | Perf report for the last 5.13588 seconds: | 20260505 14:56:27.631 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:56:27.631 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:27.631 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:27.631 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:27.631 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 14:56:27.631 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:27.631 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:27.631 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 14:56:27.631 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:27.631 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:27.631 | INFO | Uploaded 36GiB to GPU in 2.14101s (16.8145GiB/s) | 20260505 14:56:27.631 | INFO | Downloaded 34GiB to GPU in 2.12417s (16.0063GiB/s) | 20260505 14:56:32.774 | INFO | Analyzing chunk frames = [4094464,4094976) | 20260505 14:56:32.774 | INFO | MemInfo: { | 20260505 14:56:32.774 | INFO | "gpu" : " 17 GB", | 20260505 14:56:32.774 | INFO | "huge" : "", | 20260505 14:56:32.774 | INFO | "huge_shared" : " 27 GB", | 20260505 14:56:32.774 | INFO | "malloc" : " 14 GB", | 20260505 14:56:32.774 | INFO | "pinned" : " 29 GB" | 20260505 14:56:32.774 | INFO | } | 20260505 14:56:32.774 | INFO | Perf report for the last 5.14308 seconds: | 20260505 14:56:32.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:56:32.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:32.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:32.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:32.774 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 14:56:32.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:32.774 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:32.774 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 14:56:32.774 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:32.774 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:32.774 | INFO | Uploaded 36GiB to GPU in 2.16898s (16.5976GiB/s) | 20260505 14:56:32.774 | INFO | Downloaded 34GiB to GPU in 2.11077s (16.1079GiB/s) | 20260505 14:56:37.915 | INFO | Analyzing chunk frames = [4094976,4095488) | 20260505 14:56:37.915 | INFO | MemInfo: { | 20260505 14:56:37.915 | INFO | "gpu" : " 17 GB", | 20260505 14:56:37.915 | INFO | "huge" : "", | 20260505 14:56:37.915 | INFO | "huge_shared" : " 27 GB", | 20260505 14:56:37.915 | INFO | "malloc" : " 14 GB", | 20260505 14:56:37.915 | INFO | "pinned" : " 29 GB" | 20260505 14:56:37.915 | INFO | } | 20260505 14:56:37.915 | INFO | Perf report for the last 5.14157 seconds: | 20260505 14:56:37.915 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:56:37.915 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:37.915 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:37.915 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:37.916 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 14:56:37.916 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:37.916 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:37.916 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 14:56:37.916 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:37.916 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:37.916 | INFO | Uploaded 36GiB to GPU in 2.1167s (17.0076GiB/s) | 20260505 14:56:37.916 | INFO | Downloaded 34GiB to GPU in 2.17876s (15.6052GiB/s) | 20260505 14:56:43.063 | INFO | Analyzing chunk frames = [4095488,4096000) | 20260505 14:56:43.063 | INFO | MemInfo: { | 20260505 14:56:43.063 | INFO | "gpu" : " 17 GB", | 20260505 14:56:43.063 | INFO | "huge" : "", | 20260505 14:56:43.063 | INFO | "huge_shared" : " 27 GB", | 20260505 14:56:43.063 | INFO | "malloc" : " 14 GB", | 20260505 14:56:43.063 | INFO | "pinned" : " 29 GB" | 20260505 14:56:43.063 | INFO | } | 20260505 14:56:43.063 | INFO | Perf report for the last 5.14778 seconds: | 20260505 14:56:43.063 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:56:43.063 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:43.063 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:43.063 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:43.063 | INFO | ANALYSIS : 63.4%, 3.0, 99.2ms, 0 | 20260505 14:56:43.063 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:43.063 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:43.063 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:56:43.063 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:43.063 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:43.063 | INFO | Uploaded 36GiB to GPU in 2.16321s (16.6419GiB/s) | 20260505 14:56:43.064 | INFO | Downloaded 34GiB to GPU in 2.03383s (16.7172GiB/s) | 20260505 14:56:43.064 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 14:56:43.064 | INFO | Total mem usage: 15.3 GiB | 20260505 14:56:43.064 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 14:56:43.064 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.73%) | 20260505 14:56:43.064 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 14:56:43.064 | INFO | High Water Breakdown: | 20260505 14:56:43.064 | INFO | 61% : 8.67 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 14:56:43.064 | INFO | 38.9% : 5.53 GiB ( 83.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 14:56:43.064 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 14:56:43.064 | INFO | ------------------------------------------ | 20260505 14:56:43.064 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:56:43.064 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 14:56:43.064 | INFO | Total mem usage: 26.5 GiB | 20260505 14:56:43.064 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 14:56:43.064 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 14:56:43.064 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 14:56:43.064 | INFO | High Water Breakdown: | 20260505 14:56:43.064 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:56:43.064 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:56:43.064 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:56:43.064 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:56:43.064 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:56:43.064 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:56:43.064 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 14:56:43.064 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 14:56:43.064 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:56:43.064 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 14:56:43.064 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 14:56:43.064 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 14:56:43.064 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 14:56:43.064 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 14:56:43.064 | INFO | ------------------------------------------ | 20260505 14:56:43.064 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:56:43.064 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 14:56:43.064 | INFO | Total mem usage: 25.6 GiB | 20260505 14:56:43.064 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 14:56:43.064 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 14:56:43.064 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 14:56:43.064 | INFO | High Water Breakdown: | 20260505 14:56:43.064 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 14:56:43.064 | INFO | ------------------------------------------ | 20260505 14:56:43.064 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:56:43.064 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 14:56:43.064 | INFO | Total mem usage: 15.7 GiB | 20260505 14:56:43.064 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 14:56:43.064 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 14:56:43.064 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260505 14:56:43.064 | INFO | High Water Breakdown: | 20260505 14:56:43.064 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 14:56:43.064 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 14:56:43.064 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 14:56:43.064 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 14:56:43.064 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 14:56:43.064 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 14:56:43.064 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 14:56:43.064 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 14:56:43.064 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 14:56:43.065 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:56:43.065 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 14:56:43.065 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:56:43.065 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 14:56:43.065 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 14:56:43.065 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 14:56:43.065 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 14:56:43.065 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:56:43.065 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 14:56:43.065 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:56:43.065 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:56:43.065 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:56:43.065 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 14:56:43.065 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 14:56:43.065 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:56:43.065 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 14:56:43.065 | INFO | ------------------------------------------ | 20260505 14:56:43.065 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:56:43.065 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 14:56:43.065 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:56:43.065 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:56:43.065 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.59e+01, 1.38e+01 / 1.90e+00 | 20260505 14:56:43.065 | INFO | PerfReport: 0.29%, AggregateStats -- 9.26e-02 / 9.92e-02, 1.04e-02 / 4.33e-03 | 20260505 14:56:43.065 | INFO | PerfReport: 11.77%, Baseline -- 3.74e+00 / 4.46e+00, 5.83e-01 / 1.95e-01 | 20260505 14:56:43.065 | INFO | PerfReport: 3.29%, Histogram -- 1.04e+00 / 1.48e+00, 1.96e-01 / 1.07e-01 | 20260505 14:56:43.065 | INFO | PerfReport: 15.95%, DME -- 4.06e+01 / 4.53e+01, 4.52e+00 / 2.18e+00 | 20260505 14:56:43.065 | INFO | PerfReport: 56.59%, FrameLabeling -- 1.80e+01 / 1.98e+01, 1.48e+00 / 1.89e+00 | 20260505 14:56:43.065 | INFO | PerfReport: 7.39%, PulseAccumulating -- 2.35e+00 / 2.91e+00, 4.48e-01 / 1.92e-01 | 20260505 14:56:43.065 | INFO | PerfReport: 3.50%, Metrics -- 1.11e+00 / 1.45e+00, 3.11e-01 / 1.63e-01 | 20260505 14:56:43.065 | INFO | PerfReport: 1.23%, PulseSerializing -- 3.90e-01 / 4.46e-01, 6.92e-02 / 3.04e-02 | 20260505 14:56:43.065 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 14:56:43.065 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:56:43.065 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:56:43.065 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260505 14:56:43.065 | INFO | PerfReport: 22.54%, Upload -- 2.27e+01 / 0.00e+00, 8.02e+00 / 0.00e+00 | 20260505 14:56:43.065 | INFO | PerfReport: 21.94%, Download -- 2.21e+01 / 0.00e+00, 8.05e+00 / 0.00e+00 | 20260505 14:56:43.065 | INFO | PerfReport: 31.61%, Compute -- 3.18e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 14:56:43.065 | INFO | PerfReport: 2.23%, LockUpload -- 2.24e+00 / 0.00e+00, 7.94e+00 / 0.00e+00 | 20260505 14:56:43.065 | INFO | PerfReport: 1.46%, LockDownload -- 1.47e+00 / 0.00e+00, 3.00e+00 / 0.00e+00 | 20260505 14:56:43.065 | INFO | PerfReport: 20.21%, LockCompute -- 2.03e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 14:56:43.065 | INFO | PerfReport: 0.01%, Misc -- 1.25e-02 / 0.00e+00, 1.43e-02 / 0.00e+00 | 20260505 14:56:43.197 | INFO | Starting a round of Baz IO | 20260505 14:56:46.408 | INFO | Populating RTMetrics report | 20260505 14:56:46.408 | INFO | RTMetrics report populated | 20260505 14:56:46.408 | INFO | Opening tmp RTMetrics json file | 20260505 14:56:46.429 | INFO | Serializing RTMetrics to json | 20260505 14:56:46.451 | INFO | Serialization done, swapping tmp file now... | 20260505 14:56:46.473 | INFO | RTMetrics done | 20260505 14:56:46.514 | INFO | Finished a round of Baz IO: Wrote 2.233 GiB in 3.316 seconds | 20260505 14:56:46.514 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.61% / 0.00% / 77.39% | 20260505 14:56:46.514 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.499602% / 1.19209e-05% / 9.13938e-05% | 20260505 14:56:46.514 | INFO | Baz IO overall: | 20260505 14:56:46.514 | INFO | BytesWritten : 2438962410992 | 20260505 14:56:46.514 | INFO | NumEvents : 1159148174765 | 20260505 14:56:48.190 | INFO | Analyzing chunk frames = [4096000,4096512) | 20260505 14:56:48.190 | INFO | MemInfo: { | 20260505 14:56:48.190 | INFO | "gpu" : " 17 GB", | 20260505 14:56:48.190 | INFO | "huge" : "", | 20260505 14:56:48.190 | INFO | "huge_shared" : " 27 GB", | 20260505 14:56:48.190 | INFO | "malloc" : " 14 GB", | 20260505 14:56:48.190 | INFO | "pinned" : " 29 GB" | 20260505 14:56:48.190 | INFO | } | 20260505 14:56:48.191 | INFO | Perf report for the last 5.12727 seconds: | 20260505 14:56:48.191 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:56:48.191 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:48.191 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:48.191 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:48.191 | INFO | ANALYSIS : 64.1%, 3.0, 99.2ms, 0 | 20260505 14:56:48.191 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:48.191 | INFO | RT_METRICS : 46.3%, 1.0, 24.2ms, 0 | 20260505 14:56:48.191 | INFO | PRE_HQ : 61.3%, 1.8, 57.4ms, 0 | 20260505 14:56:48.191 | INFO | BAZWRITER : 64.0%, 4.0, 130.7ms, 0 | 20260505 14:56:48.191 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 14:56:48.191 | INFO | Uploaded 36GiB to GPU in 2.21479s (16.2543GiB/s) | 20260505 14:56:48.191 | INFO | Downloaded 38GiB to GPU in 2.27088s (16.7336GiB/s) | 20260505 14:56:53.324 | INFO | Analyzing chunk frames = [4096512,4097024) | 20260505 14:56:53.325 | INFO | MemInfo: { | 20260505 14:56:53.325 | INFO | "gpu" : " 17 GB", | 20260505 14:56:53.325 | INFO | "huge" : "", | 20260505 14:56:53.325 | INFO | "huge_shared" : " 27 GB", | 20260505 14:56:53.325 | INFO | "malloc" : " 14 GB", | 20260505 14:56:53.325 | INFO | "pinned" : " 29 GB" | 20260505 14:56:53.325 | INFO | } | 20260505 14:56:53.325 | INFO | Perf report for the last 5.13422 seconds: | 20260505 14:56:53.325 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:56:53.325 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:53.325 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:53.325 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:53.325 | INFO | ANALYSIS : 63.5%, 3.0, 99.0ms, 0 | 20260505 14:56:53.325 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:53.325 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:53.325 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:56:53.325 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:53.325 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:53.325 | INFO | Uploaded 36GiB to GPU in 2.17829s (16.5267GiB/s) | 20260505 14:56:53.325 | INFO | Downloaded 34GiB to GPU in 2.02832s (16.7626GiB/s) | 20260505 14:56:58.474 | INFO | Analyzing chunk frames = [4097024,4097536) | 20260505 14:56:58.474 | INFO | MemInfo: { | 20260505 14:56:58.474 | INFO | "gpu" : " 17 GB", | 20260505 14:56:58.474 | INFO | "huge" : "", | 20260505 14:56:58.474 | INFO | "huge_shared" : " 27 GB", | 20260505 14:56:58.474 | INFO | "malloc" : " 14 GB", | 20260505 14:56:58.474 | INFO | "pinned" : " 29 GB" | 20260505 14:56:58.474 | INFO | } | 20260505 14:56:58.474 | INFO | Perf report for the last 5.14912 seconds: | 20260505 14:56:58.474 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:56:58.474 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:58.474 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:58.474 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:58.474 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 14:56:58.474 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:58.474 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:58.474 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:56:58.474 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:58.474 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:56:58.474 | INFO | Uploaded 36GiB to GPU in 2.23471s (16.1095GiB/s) | 20260505 14:56:58.474 | INFO | Downloaded 34GiB to GPU in 2.09281s (16.2461GiB/s) | 20260505 14:57:03.617 | INFO | Analyzing chunk frames = [4097536,4098048) | 20260505 14:57:03.617 | INFO | MemInfo: { | 20260505 14:57:03.617 | INFO | "gpu" : " 17 GB", | 20260505 14:57:03.617 | INFO | "huge" : "", | 20260505 14:57:03.617 | INFO | "huge_shared" : " 27 GB", | 20260505 14:57:03.617 | INFO | "malloc" : " 14 GB", | 20260505 14:57:03.617 | INFO | "pinned" : " 29 GB" | 20260505 14:57:03.617 | INFO | } | 20260505 14:57:03.617 | INFO | Perf report for the last 5.14326 seconds: | 20260505 14:57:03.617 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:57:03.617 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:03.617 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:03.617 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:03.617 | INFO | ANALYSIS : 62.6%, 3.0, 97.4ms, 0 | 20260505 14:57:03.617 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:03.617 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:03.617 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:57:03.617 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:03.617 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:03.617 | INFO | Uploaded 36GiB to GPU in 2.19316s (16.4147GiB/s) | 20260505 14:57:03.617 | INFO | Downloaded 34GiB to GPU in 2.13096s (15.9553GiB/s) | 20260505 14:57:08.750 | INFO | Analyzing chunk frames = [4098048,4098560) | 20260505 14:57:08.750 | INFO | MemInfo: { | 20260505 14:57:08.750 | INFO | "gpu" : " 17 GB", | 20260505 14:57:08.750 | INFO | "huge" : "", | 20260505 14:57:08.750 | INFO | "huge_shared" : " 27 GB", | 20260505 14:57:08.750 | INFO | "malloc" : " 14 GB", | 20260505 14:57:08.750 | INFO | "pinned" : " 29 GB" | 20260505 14:57:08.750 | INFO | } | 20260505 14:57:08.750 | INFO | Perf report for the last 5.1332 seconds: | 20260505 14:57:08.750 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:57:08.750 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:08.750 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:08.750 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:08.750 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 14:57:08.750 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:08.751 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:08.751 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:57:08.751 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:08.751 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:08.751 | INFO | Uploaded 36GiB to GPU in 2.18103s (16.5059GiB/s) | 20260505 14:57:08.751 | INFO | Downloaded 34GiB to GPU in 2.12929s (15.9678GiB/s) | 20260505 14:57:13.883 | INFO | Analyzing chunk frames = [4098560,4099072) | 20260505 14:57:13.883 | INFO | MemInfo: { | 20260505 14:57:13.883 | INFO | "gpu" : " 17 GB", | 20260505 14:57:13.883 | INFO | "huge" : "", | 20260505 14:57:13.883 | INFO | "huge_shared" : " 27 GB", | 20260505 14:57:13.883 | INFO | "malloc" : " 14 GB", | 20260505 14:57:13.883 | INFO | "pinned" : " 29 GB" | 20260505 14:57:13.883 | INFO | } | 20260505 14:57:13.883 | INFO | Perf report for the last 5.13295 seconds: | 20260505 14:57:13.883 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:57:13.883 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:13.883 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:13.883 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:13.883 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 14:57:13.883 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:13.883 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:13.883 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:57:13.883 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:13.883 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:13.883 | INFO | Uploaded 36GiB to GPU in 2.18004s (16.5135GiB/s) | 20260505 14:57:13.883 | INFO | Downloaded 34GiB to GPU in 2.11173s (16.1005GiB/s) | 20260505 14:57:19.028 | INFO | Analyzing chunk frames = [4099072,4099584) | 20260505 14:57:19.028 | INFO | MemInfo: { | 20260505 14:57:19.028 | INFO | "gpu" : " 17 GB", | 20260505 14:57:19.028 | INFO | "huge" : "", | 20260505 14:57:19.028 | INFO | "huge_shared" : " 27 GB", | 20260505 14:57:19.028 | INFO | "malloc" : " 14 GB", | 20260505 14:57:19.028 | INFO | "pinned" : " 29 GB" | 20260505 14:57:19.028 | INFO | } | 20260505 14:57:19.028 | INFO | Perf report for the last 5.14493 seconds: | 20260505 14:57:19.028 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:57:19.028 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:19.028 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:19.028 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:19.028 | INFO | ANALYSIS : 63.1%, 3.0, 98.7ms, 0 | 20260505 14:57:19.028 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:19.028 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:19.028 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 14:57:19.028 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:19.028 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:19.029 | INFO | Uploaded 36GiB to GPU in 2.14236s (16.8039GiB/s) | 20260505 14:57:19.029 | INFO | Downloaded 34GiB to GPU in 2.1759s (15.6257GiB/s) | 20260505 14:57:24.174 | INFO | Analyzing chunk frames = [4099584,4100096) | 20260505 14:57:24.174 | INFO | MemInfo: { | 20260505 14:57:24.174 | INFO | "gpu" : " 17 GB", | 20260505 14:57:24.174 | INFO | "huge" : "", | 20260505 14:57:24.174 | INFO | "huge_shared" : " 27 GB", | 20260505 14:57:24.174 | INFO | "malloc" : " 14 GB", | 20260505 14:57:24.174 | INFO | "pinned" : " 29 GB" | 20260505 14:57:24.174 | INFO | } | 20260505 14:57:24.174 | INFO | Perf report for the last 5.1457 seconds: | 20260505 14:57:24.174 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:57:24.174 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:24.174 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:24.174 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:24.174 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 14:57:24.174 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:24.174 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:24.174 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 14:57:24.174 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:24.174 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:24.174 | INFO | Uploaded 36GiB to GPU in 2.22828s (16.1559GiB/s) | 20260505 14:57:24.174 | INFO | Downloaded 34GiB to GPU in 2.09662s (16.2165GiB/s) | 20260505 14:57:24.303 | INFO | Starting a round of Baz IO | 20260505 14:57:27.523 | INFO | Populating RTMetrics report | 20260505 14:57:27.524 | INFO | RTMetrics report populated | 20260505 14:57:27.524 | INFO | Opening tmp RTMetrics json file | 20260505 14:57:27.537 | INFO | Serializing RTMetrics to json | 20260505 14:57:27.554 | INFO | Serialization done, swapping tmp file now... | 20260505 14:57:27.560 | INFO | RTMetrics done | 20260505 14:57:27.618 | INFO | Finished a round of Baz IO: Wrote 2.234 GiB in 3.315 seconds | 20260505 14:57:27.618 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.60% / 0.00% / 77.39% | 20260505 14:57:27.618 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.498382% / 7.94729e-06% / 7.94729e-05% | 20260505 14:57:27.618 | INFO | Baz IO overall: | 20260505 14:57:27.618 | INFO | BytesWritten : 2441360654840 | 20260505 14:57:27.618 | INFO | NumEvents : 1160277480856 | 20260505 14:57:29.309 | INFO | Analyzing chunk frames = [4100096,4100608) | 20260505 14:57:29.309 | INFO | MemInfo: { | 20260505 14:57:29.309 | INFO | "gpu" : " 17 GB", | 20260505 14:57:29.309 | INFO | "huge" : "", | 20260505 14:57:29.309 | INFO | "huge_shared" : " 27 GB", | 20260505 14:57:29.309 | INFO | "malloc" : " 14 GB", | 20260505 14:57:29.309 | INFO | "pinned" : " 29 GB" | 20260505 14:57:29.309 | INFO | } | 20260505 14:57:29.309 | INFO | Perf report for the last 5.13535 seconds: | 20260505 14:57:29.309 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:57:29.309 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:29.309 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:29.309 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:29.309 | INFO | ANALYSIS : 64.0%, 3.0, 99.4ms, 0 | 20260505 14:57:29.309 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:29.309 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260505 14:57:29.309 | INFO | PRE_HQ : 61.2%, 1.8, 57.1ms, 0 | 20260505 14:57:29.309 | INFO | BAZWRITER : 63.9%, 4.0, 130.1ms, 0 | 20260505 14:57:29.309 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 14:57:29.310 | INFO | Uploaded 36GiB to GPU in 2.19702s (16.3859GiB/s) | 20260505 14:57:29.310 | INFO | Downloaded 38GiB to GPU in 2.25506s (16.851GiB/s) | 20260505 14:57:34.447 | INFO | Analyzing chunk frames = [4100608,4101120) | 20260505 14:57:34.447 | INFO | MemInfo: { | 20260505 14:57:34.447 | INFO | "gpu" : " 17 GB", | 20260505 14:57:34.447 | INFO | "huge" : "", | 20260505 14:57:34.447 | INFO | "huge_shared" : " 27 GB", | 20260505 14:57:34.447 | INFO | "malloc" : " 14 GB", | 20260505 14:57:34.447 | INFO | "pinned" : " 29 GB" | 20260505 14:57:34.447 | INFO | } | 20260505 14:57:34.447 | INFO | Perf report for the last 5.13779 seconds: | 20260505 14:57:34.447 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:57:34.447 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:34.447 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:34.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:34.447 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 14:57:34.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:34.447 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:34.447 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:57:34.447 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:34.447 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:34.447 | INFO | Uploaded 36GiB to GPU in 2.1923s (16.4211GiB/s) | 20260505 14:57:34.448 | INFO | Downloaded 34GiB to GPU in 2.03371s (16.7182GiB/s) | 20260505 14:57:39.580 | INFO | Analyzing chunk frames = [4101120,4101632) | 20260505 14:57:39.580 | INFO | MemInfo: { | 20260505 14:57:39.580 | INFO | "gpu" : " 17 GB", | 20260505 14:57:39.580 | INFO | "huge" : "", | 20260505 14:57:39.580 | INFO | "huge_shared" : " 27 GB", | 20260505 14:57:39.580 | INFO | "malloc" : " 14 GB", | 20260505 14:57:39.580 | INFO | "pinned" : " 29 GB" | 20260505 14:57:39.580 | INFO | } | 20260505 14:57:39.581 | INFO | Perf report for the last 5.13337 seconds: | 20260505 14:57:39.581 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:57:39.581 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:39.581 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:39.581 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:39.581 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 14:57:39.581 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:39.581 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:39.581 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:57:39.581 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:39.581 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:39.581 | INFO | Uploaded 36GiB to GPU in 2.2086s (16.3GiB/s) | 20260505 14:57:39.581 | INFO | Downloaded 34GiB to GPU in 2.10965s (16.1164GiB/s) | 20260505 14:57:44.730 | INFO | Analyzing chunk frames = [4101632,4102144) | 20260505 14:57:44.730 | INFO | MemInfo: { | 20260505 14:57:44.730 | INFO | "gpu" : " 17 GB", | 20260505 14:57:44.730 | INFO | "huge" : "", | 20260505 14:57:44.730 | INFO | "huge_shared" : " 27 GB", | 20260505 14:57:44.730 | INFO | "malloc" : " 14 GB", | 20260505 14:57:44.730 | INFO | "pinned" : " 29 GB" | 20260505 14:57:44.730 | INFO | } | 20260505 14:57:44.730 | INFO | Perf report for the last 5.14926 seconds: | 20260505 14:57:44.730 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:57:44.730 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:44.730 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:44.730 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:44.730 | INFO | ANALYSIS : 62.9%, 3.0, 97.9ms, 0 | 20260505 14:57:44.730 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:44.730 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:44.730 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 14:57:44.730 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:44.730 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:44.730 | INFO | Uploaded 36GiB to GPU in 2.20553s (16.3226GiB/s) | 20260505 14:57:44.730 | INFO | Downloaded 34GiB to GPU in 2.13658s (15.9133GiB/s) | 20260505 14:57:49.870 | INFO | Analyzing chunk frames = [4102144,4102656) | 20260505 14:57:49.870 | INFO | MemInfo: { | 20260505 14:57:49.870 | INFO | "gpu" : " 17 GB", | 20260505 14:57:49.870 | INFO | "huge" : "", | 20260505 14:57:49.870 | INFO | "huge_shared" : " 27 GB", | 20260505 14:57:49.870 | INFO | "malloc" : " 14 GB", | 20260505 14:57:49.870 | INFO | "pinned" : " 29 GB" | 20260505 14:57:49.870 | INFO | } | 20260505 14:57:49.870 | INFO | Perf report for the last 5.14071 seconds: | 20260505 14:57:49.870 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:57:49.870 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:49.871 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:49.871 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:49.871 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 14:57:49.871 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:49.871 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:49.871 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 14:57:49.871 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:49.871 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:49.871 | INFO | Uploaded 36GiB to GPU in 2.17751s (16.5326GiB/s) | 20260505 14:57:49.871 | INFO | Downloaded 34GiB to GPU in 2.13072s (15.9571GiB/s) | 20260505 14:57:55.012 | INFO | Analyzing chunk frames = [4102656,4103168) | 20260505 14:57:55.012 | INFO | MemInfo: { | 20260505 14:57:55.012 | INFO | "gpu" : " 17 GB", | 20260505 14:57:55.012 | INFO | "huge" : "", | 20260505 14:57:55.012 | INFO | "huge_shared" : " 27 GB", | 20260505 14:57:55.012 | INFO | "malloc" : " 14 GB", | 20260505 14:57:55.012 | INFO | "pinned" : " 29 GB" | 20260505 14:57:55.012 | INFO | } | 20260505 14:57:55.012 | INFO | Perf report for the last 5.14146 seconds: | 20260505 14:57:55.012 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:57:55.012 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:55.012 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:55.012 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:55.012 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260505 14:57:55.012 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:55.012 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:55.012 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 14:57:55.012 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:55.012 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:57:55.012 | INFO | Uploaded 36GiB to GPU in 2.13913s (16.8293GiB/s) | 20260505 14:57:55.012 | INFO | Downloaded 34GiB to GPU in 2.03558s (16.7029GiB/s) | 20260505 14:58:00.137 | INFO | Analyzing chunk frames = [4103168,4103680) | 20260505 14:58:00.137 | INFO | MemInfo: { | 20260505 14:58:00.137 | INFO | "gpu" : " 17 GB", | 20260505 14:58:00.137 | INFO | "huge" : "", | 20260505 14:58:00.137 | INFO | "huge_shared" : " 27 GB", | 20260505 14:58:00.137 | INFO | "malloc" : " 14 GB", | 20260505 14:58:00.137 | INFO | "pinned" : " 29 GB" | 20260505 14:58:00.137 | INFO | } | 20260505 14:58:00.137 | INFO | Perf report for the last 5.12545 seconds: | 20260505 14:58:00.137 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:58:00.137 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:00.137 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:00.137 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:00.137 | INFO | ANALYSIS : 63.6%, 3.0, 99.1ms, 0 | 20260505 14:58:00.137 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:00.137 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:00.137 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 14:58:00.137 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:00.138 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:00.138 | INFO | Uploaded 36GiB to GPU in 2.10098s (17.1348GiB/s) | 20260505 14:58:00.138 | INFO | Downloaded 34GiB to GPU in 2.17338s (15.6439GiB/s) | 20260505 14:58:05.281 | INFO | Analyzing chunk frames = [4103680,4104192) | 20260505 14:58:05.281 | INFO | MemInfo: { | 20260505 14:58:05.281 | INFO | "gpu" : " 17 GB", | 20260505 14:58:05.281 | INFO | "huge" : "", | 20260505 14:58:05.281 | INFO | "huge_shared" : " 27 GB", | 20260505 14:58:05.281 | INFO | "malloc" : " 14 GB", | 20260505 14:58:05.281 | INFO | "pinned" : " 29 GB" | 20260505 14:58:05.281 | INFO | } | 20260505 14:58:05.281 | INFO | Perf report for the last 5.14375 seconds: | 20260505 14:58:05.281 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:58:05.281 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:05.281 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:05.281 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:05.281 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 14:58:05.281 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:05.281 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:05.281 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 14:58:05.281 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:05.281 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:05.281 | INFO | Uploaded 36GiB to GPU in 2.14796s (16.7601GiB/s) | 20260505 14:58:05.281 | INFO | Downloaded 34GiB to GPU in 2.10457s (16.1553GiB/s) | 20260505 14:58:05.282 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 14:58:05.282 | INFO | Total mem usage: 15.3 GiB | 20260505 14:58:05.282 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 14:58:05.282 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -7.21%) | 20260505 14:58:05.282 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 14:58:05.282 | INFO | High Water Breakdown: | 20260505 14:58:05.282 | INFO | 61.2% : 8.74 GiB ( 91.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 14:58:05.282 | INFO | 38.7% : 5.53 GiB ( 83.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 14:58:05.282 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 14:58:05.282 | INFO | ------------------------------------------ | 20260505 14:58:05.282 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:58:05.282 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 14:58:05.282 | INFO | Total mem usage: 26.5 GiB | 20260505 14:58:05.282 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 14:58:05.282 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 14:58:05.282 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 14:58:05.282 | INFO | High Water Breakdown: | 20260505 14:58:05.282 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:58:05.282 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:58:05.282 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:58:05.282 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:58:05.282 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:58:05.282 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:58:05.282 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 14:58:05.282 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 14:58:05.282 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:58:05.282 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 14:58:05.282 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 14:58:05.282 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 14:58:05.282 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 14:58:05.282 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 14:58:05.282 | INFO | ------------------------------------------ | 20260505 14:58:05.282 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:58:05.282 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 14:58:05.282 | INFO | Total mem usage: 25.6 GiB | 20260505 14:58:05.282 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 14:58:05.282 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 14:58:05.282 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 14:58:05.282 | INFO | High Water Breakdown: | 20260505 14:58:05.282 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 14:58:05.282 | INFO | ------------------------------------------ | 20260505 14:58:05.282 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:58:05.282 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 14:58:05.282 | INFO | Total mem usage: 15.7 GiB | 20260505 14:58:05.282 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 14:58:05.282 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 14:58:05.282 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 14:58:05.282 | INFO | High Water Breakdown: | 20260505 14:58:05.282 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 14:58:05.282 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 14:58:05.282 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 14:58:05.282 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 14:58:05.282 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 14:58:05.282 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 14:58:05.282 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 14:58:05.282 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 14:58:05.282 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 14:58:05.282 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:58:05.282 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 14:58:05.282 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:58:05.282 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 14:58:05.282 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 14:58:05.282 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 14:58:05.282 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 14:58:05.282 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:58:05.282 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 14:58:05.282 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:58:05.282 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:58:05.282 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:58:05.282 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 14:58:05.282 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 14:58:05.282 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:58:05.282 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 14:58:05.282 | INFO | ------------------------------------------ | 20260505 14:58:05.282 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:58:05.282 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 14:58:05.282 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:58:05.282 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:58:05.282 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.62e+01, 1.35e+01 / 1.92e+00 | 20260505 14:58:05.282 | INFO | PerfReport: 0.29%, AggregateStats -- 9.31e-02 / 1.00e-01, 1.06e-02 / 5.26e-03 | 20260505 14:58:05.282 | INFO | PerfReport: 11.80%, Baseline -- 3.74e+00 / 4.52e+00, 5.77e-01 / 1.49e-01 | 20260505 14:58:05.282 | INFO | PerfReport: 3.29%, Histogram -- 1.05e+00 / 1.50e+00, 1.94e-01 / 9.11e-02 | 20260505 14:58:05.282 | INFO | PerfReport: 16.03%, DME -- 4.07e+01 / 4.58e+01, 4.66e+00 / 1.96e+00 | 20260505 14:58:05.282 | INFO | PerfReport: 56.42%, FrameLabeling -- 1.79e+01 / 1.95e+01, 1.34e+00 / 1.67e+00 | 20260505 14:58:05.283 | INFO | PerfReport: 7.43%, PulseAccumulating -- 2.36e+00 / 2.95e+00, 4.48e-01 / 1.37e-01 | 20260505 14:58:05.283 | INFO | PerfReport: 3.50%, Metrics -- 1.11e+00 / 1.41e+00, 3.06e-01 / 1.44e-01 | 20260505 14:58:05.283 | INFO | PerfReport: 1.23%, PulseSerializing -- 3.90e-01 / 4.38e-01, 6.96e-02 / 3.25e-02 | 20260505 14:58:05.283 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 14:58:05.283 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:58:05.283 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:58:05.283 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260505 14:58:05.283 | INFO | PerfReport: 22.60%, Upload -- 2.27e+01 / 0.00e+00, 7.98e+00 / 0.00e+00 | 20260505 14:58:05.283 | INFO | PerfReport: 22.01%, Download -- 2.21e+01 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260505 14:58:05.283 | INFO | PerfReport: 31.56%, Compute -- 3.17e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 14:58:05.283 | INFO | PerfReport: 2.28%, LockUpload -- 2.30e+00 / 0.00e+00, 7.83e+00 / 0.00e+00 | 20260505 14:58:05.283 | INFO | PerfReport: 1.44%, LockDownload -- 1.45e+00 / 0.00e+00, 2.94e+00 / 0.00e+00 | 20260505 14:58:05.283 | INFO | PerfReport: 20.10%, LockCompute -- 2.02e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 14:58:05.283 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.71e-02 / 0.00e+00 | 20260505 14:58:05.415 | INFO | Starting a round of Baz IO | 20260505 14:58:08.635 | INFO | Populating RTMetrics report | 20260505 14:58:08.636 | INFO | RTMetrics report populated | 20260505 14:58:08.636 | INFO | Opening tmp RTMetrics json file | 20260505 14:58:08.658 | INFO | Serializing RTMetrics to json | 20260505 14:58:08.700 | INFO | Serialization done, swapping tmp file now... | 20260505 14:58:08.723 | INFO | RTMetrics done | 20260505 14:58:08.743 | INFO | Finished a round of Baz IO: Wrote 2.233 GiB in 3.327 seconds | 20260505 14:58:08.743 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.60% / 0.00% / 77.40% | 20260505 14:58:08.743 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.497262% / 3.97364e-06% / 8.34465e-05% | 20260505 14:58:08.743 | INFO | Baz IO overall: | 20260505 14:58:08.743 | INFO | BytesWritten : 2443757917970 | 20260505 14:58:08.743 | INFO | NumEvents : 1161406229286 | 20260505 14:58:10.425 | INFO | Analyzing chunk frames = [4104192,4104704) | 20260505 14:58:10.425 | INFO | MemInfo: { | 20260505 14:58:10.425 | INFO | "gpu" : " 17 GB", | 20260505 14:58:10.425 | INFO | "huge" : "", | 20260505 14:58:10.425 | INFO | "huge_shared" : " 27 GB", | 20260505 14:58:10.425 | INFO | "malloc" : " 14 GB", | 20260505 14:58:10.425 | INFO | "pinned" : " 29 GB" | 20260505 14:58:10.425 | INFO | } | 20260505 14:58:10.425 | INFO | Perf report for the last 5.14398 seconds: | 20260505 14:58:10.425 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:58:10.425 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:10.425 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:10.425 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:10.425 | INFO | ANALYSIS : 64.1%, 3.0, 99.6ms, 0 | 20260505 14:58:10.425 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:10.425 | INFO | RT_METRICS : 46.5%, 1.0, 24.4ms, 0 | 20260505 14:58:10.425 | INFO | PRE_HQ : 60.9%, 1.8, 57.4ms, 0 | 20260505 14:58:10.425 | INFO | BAZWRITER : 64.0%, 4.0, 130.9ms, 0 | 20260505 14:58:10.425 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260505 14:58:10.426 | INFO | Uploaded 36GiB to GPU in 2.19984s (16.3648GiB/s) | 20260505 14:58:10.426 | INFO | Downloaded 38GiB to GPU in 2.2572s (16.835GiB/s) | 20260505 14:58:15.569 | INFO | Analyzing chunk frames = [4104704,4105216) | 20260505 14:58:15.569 | INFO | MemInfo: { | 20260505 14:58:15.569 | INFO | "gpu" : " 17 GB", | 20260505 14:58:15.569 | INFO | "huge" : "", | 20260505 14:58:15.569 | INFO | "huge_shared" : " 27 GB", | 20260505 14:58:15.569 | INFO | "malloc" : " 14 GB", | 20260505 14:58:15.569 | INFO | "pinned" : " 29 GB" | 20260505 14:58:15.569 | INFO | } | 20260505 14:58:15.569 | INFO | Perf report for the last 5.14413 seconds: | 20260505 14:58:15.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:58:15.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:15.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:15.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:15.569 | INFO | ANALYSIS : 63.3%, 3.0, 99.0ms, 0 | 20260505 14:58:15.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:15.569 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:15.569 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 14:58:15.569 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:15.569 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:15.570 | INFO | Uploaded 36GiB to GPU in 2.19549s (16.3972GiB/s) | 20260505 14:58:15.570 | INFO | Downloaded 34GiB to GPU in 2.04036s (16.6637GiB/s) | 20260505 14:58:20.697 | INFO | Analyzing chunk frames = [4105216,4105728) | 20260505 14:58:20.697 | INFO | MemInfo: { | 20260505 14:58:20.697 | INFO | "gpu" : " 17 GB", | 20260505 14:58:20.697 | INFO | "huge" : "", | 20260505 14:58:20.697 | INFO | "huge_shared" : " 27 GB", | 20260505 14:58:20.697 | INFO | "malloc" : " 14 GB", | 20260505 14:58:20.697 | INFO | "pinned" : " 29 GB" | 20260505 14:58:20.697 | INFO | } | 20260505 14:58:20.697 | INFO | Perf report for the last 5.12793 seconds: | 20260505 14:58:20.697 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:58:20.697 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:20.697 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:20.697 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:20.697 | INFO | ANALYSIS : 63.1%, 3.0, 98.0ms, 0 | 20260505 14:58:20.697 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:20.697 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:20.697 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 14:58:20.697 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:20.697 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:20.697 | INFO | Uploaded 36GiB to GPU in 2.18181s (16.5GiB/s) | 20260505 14:58:20.698 | INFO | Downloaded 34GiB to GPU in 2.05273s (16.5633GiB/s) | 20260505 14:58:25.829 | INFO | Analyzing chunk frames = [4105728,4106240) | 20260505 14:58:25.829 | INFO | MemInfo: { | 20260505 14:58:25.829 | INFO | "gpu" : " 17 GB", | 20260505 14:58:25.829 | INFO | "huge" : "", | 20260505 14:58:25.829 | INFO | "huge_shared" : " 27 GB", | 20260505 14:58:25.829 | INFO | "malloc" : " 14 GB", | 20260505 14:58:25.829 | INFO | "pinned" : " 29 GB" | 20260505 14:58:25.829 | INFO | } | 20260505 14:58:25.829 | INFO | Perf report for the last 5.1322 seconds: | 20260505 14:58:25.830 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:58:25.830 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:25.830 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:25.830 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:25.830 | INFO | ANALYSIS : 63.1%, 3.0, 97.9ms, 0 | 20260505 14:58:25.830 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:25.830 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:25.830 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:58:25.830 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:25.830 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:25.830 | INFO | Uploaded 36GiB to GPU in 2.20196s (16.3491GiB/s) | 20260505 14:58:25.830 | INFO | Downloaded 34GiB to GPU in 2.13933s (15.8929GiB/s) | 20260505 14:58:30.983 | INFO | Analyzing chunk frames = [4106240,4106752) | 20260505 14:58:30.983 | INFO | MemInfo: { | 20260505 14:58:30.983 | INFO | "gpu" : " 17 GB", | 20260505 14:58:30.983 | INFO | "huge" : "", | 20260505 14:58:30.983 | INFO | "huge_shared" : " 27 GB", | 20260505 14:58:30.983 | INFO | "malloc" : " 14 GB", | 20260505 14:58:30.983 | INFO | "pinned" : " 29 GB" | 20260505 14:58:30.983 | INFO | } | 20260505 14:58:30.983 | INFO | Perf report for the last 5.1537 seconds: | 20260505 14:58:30.983 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:58:30.983 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:30.983 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:30.983 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:30.983 | INFO | ANALYSIS : 62.6%, 3.0, 98.0ms, 0 | 20260505 14:58:30.983 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:30.983 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:30.983 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:58:30.983 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:30.983 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:30.984 | INFO | Uploaded 36GiB to GPU in 2.16517s (16.6269GiB/s) | 20260505 14:58:30.984 | INFO | Downloaded 34GiB to GPU in 2.1281s (15.9767GiB/s) | 20260505 14:58:36.125 | INFO | Analyzing chunk frames = [4106752,4107264) | 20260505 14:58:36.125 | INFO | MemInfo: { | 20260505 14:58:36.125 | INFO | "gpu" : " 17 GB", | 20260505 14:58:36.125 | INFO | "huge" : "", | 20260505 14:58:36.125 | INFO | "huge_shared" : " 27 GB", | 20260505 14:58:36.125 | INFO | "malloc" : " 14 GB", | 20260505 14:58:36.125 | INFO | "pinned" : " 29 GB" | 20260505 14:58:36.125 | INFO | } | 20260505 14:58:36.125 | INFO | Perf report for the last 5.14156 seconds: | 20260505 14:58:36.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:58:36.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:36.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:36.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:36.125 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260505 14:58:36.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:36.125 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:36.125 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 14:58:36.125 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:36.125 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:36.125 | INFO | Uploaded 36GiB to GPU in 2.13749s (16.8422GiB/s) | 20260505 14:58:36.125 | INFO | Downloaded 34GiB to GPU in 2.03426s (16.7137GiB/s) | 20260505 14:58:41.253 | INFO | Analyzing chunk frames = [4107264,4107776) | 20260505 14:58:41.253 | INFO | MemInfo: { | 20260505 14:58:41.253 | INFO | "gpu" : " 17 GB", | 20260505 14:58:41.253 | INFO | "huge" : "", | 20260505 14:58:41.253 | INFO | "huge_shared" : " 27 GB", | 20260505 14:58:41.253 | INFO | "malloc" : " 14 GB", | 20260505 14:58:41.253 | INFO | "pinned" : " 29 GB" | 20260505 14:58:41.254 | INFO | } | 20260505 14:58:41.254 | INFO | Perf report for the last 5.12886 seconds: | 20260505 14:58:41.254 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:58:41.254 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:41.254 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:41.254 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:41.254 | INFO | ANALYSIS : 63.2%, 3.0, 98.4ms, 0 | 20260505 14:58:41.254 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:41.254 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:41.254 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 14:58:41.254 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:41.254 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:41.254 | INFO | Uploaded 36GiB to GPU in 2.11925s (16.9872GiB/s) | 20260505 14:58:41.254 | INFO | Downloaded 34GiB to GPU in 2.15184s (15.8004GiB/s) | 20260505 14:58:46.392 | INFO | Analyzing chunk frames = [4107776,4108288) | 20260505 14:58:46.392 | INFO | MemInfo: { | 20260505 14:58:46.392 | INFO | "gpu" : " 17 GB", | 20260505 14:58:46.392 | INFO | "huge" : "", | 20260505 14:58:46.392 | INFO | "huge_shared" : " 27 GB", | 20260505 14:58:46.392 | INFO | "malloc" : " 14 GB", | 20260505 14:58:46.392 | INFO | "pinned" : " 29 GB" | 20260505 14:58:46.392 | INFO | } | 20260505 14:58:46.392 | INFO | Perf report for the last 5.13822 seconds: | 20260505 14:58:46.392 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:58:46.392 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:46.392 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:46.392 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:46.392 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 14:58:46.392 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:46.392 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:46.392 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:58:46.392 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:46.392 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:46.392 | INFO | Uploaded 36GiB to GPU in 2.18336s (16.4884GiB/s) | 20260505 14:58:46.392 | INFO | Downloaded 34GiB to GPU in 2.10691s (16.1374GiB/s) | 20260505 14:58:46.531 | INFO | Starting a round of Baz IO | 20260505 14:58:49.753 | INFO | Populating RTMetrics report | 20260505 14:58:49.754 | INFO | RTMetrics report populated | 20260505 14:58:49.754 | INFO | Opening tmp RTMetrics json file | 20260505 14:58:49.772 | INFO | Serializing RTMetrics to json | 20260505 14:58:49.774 | INFO | Serialization done, swapping tmp file now... | 20260505 14:58:49.778 | INFO | RTMetrics done | 20260505 14:58:49.856 | INFO | Finished a round of Baz IO: Wrote 2.234 GiB in 3.324 seconds | 20260505 14:58:49.856 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.60% / 0.00% / 77.40% | 20260505 14:58:49.856 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.496153% / 1.19209e-05% / 9.13938e-05% | 20260505 14:58:49.856 | INFO | Baz IO overall: | 20260505 14:58:49.856 | INFO | BytesWritten : 2446157054413 | 20260505 14:58:49.856 | INFO | NumEvents : 1162536356494 | 20260505 14:58:51.539 | INFO | Analyzing chunk frames = [4108288,4108800) | 20260505 14:58:51.539 | INFO | MemInfo: { | 20260505 14:58:51.539 | INFO | "gpu" : " 17 GB", | 20260505 14:58:51.539 | INFO | "huge" : "", | 20260505 14:58:51.539 | INFO | "huge_shared" : " 27 GB", | 20260505 14:58:51.539 | INFO | "malloc" : " 14 GB", | 20260505 14:58:51.539 | INFO | "pinned" : " 29 GB" | 20260505 14:58:51.539 | INFO | } | 20260505 14:58:51.539 | INFO | Perf report for the last 5.14741 seconds: | 20260505 14:58:51.539 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:58:51.539 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:51.539 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:51.539 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:51.539 | INFO | ANALYSIS : 64.2%, 3.0, 100.0ms, 0 | 20260505 14:58:51.539 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:51.539 | INFO | RT_METRICS : 45.3%, 1.0, 23.8ms, 0 | 20260505 14:58:51.539 | INFO | PRE_HQ : 60.5%, 1.8, 56.8ms, 0 | 20260505 14:58:51.539 | INFO | BAZWRITER : 63.9%, 4.0, 130.6ms, 0 | 20260505 14:58:51.539 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 14:58:51.540 | INFO | Uploaded 36GiB to GPU in 2.16778s (16.6069GiB/s) | 20260505 14:58:51.540 | INFO | Downloaded 38GiB to GPU in 2.27262s (16.7208GiB/s) | 20260505 14:58:56.681 | INFO | Analyzing chunk frames = [4108800,4109312) | 20260505 14:58:56.681 | INFO | MemInfo: { | 20260505 14:58:56.681 | INFO | "gpu" : " 17 GB", | 20260505 14:58:56.681 | INFO | "huge" : "", | 20260505 14:58:56.681 | INFO | "huge_shared" : " 27 GB", | 20260505 14:58:56.681 | INFO | "malloc" : " 14 GB", | 20260505 14:58:56.681 | INFO | "pinned" : " 29 GB" | 20260505 14:58:56.681 | INFO | } | 20260505 14:58:56.681 | INFO | Perf report for the last 5.14175 seconds: | 20260505 14:58:56.681 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:58:56.681 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:56.681 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:56.681 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:56.681 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 14:58:56.681 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:56.681 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:56.681 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:58:56.681 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:56.681 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:58:56.681 | INFO | Uploaded 36GiB to GPU in 2.19725s (16.3841GiB/s) | 20260505 14:58:56.681 | INFO | Downloaded 34GiB to GPU in 2.04226s (16.6482GiB/s) | 20260505 14:59:01.820 | INFO | Analyzing chunk frames = [4109312,4109824) | 20260505 14:59:01.820 | INFO | MemInfo: { | 20260505 14:59:01.820 | INFO | "gpu" : " 17 GB", | 20260505 14:59:01.820 | INFO | "huge" : "", | 20260505 14:59:01.820 | INFO | "huge_shared" : " 27 GB", | 20260505 14:59:01.820 | INFO | "malloc" : " 14 GB", | 20260505 14:59:01.820 | INFO | "pinned" : " 29 GB" | 20260505 14:59:01.820 | INFO | } | 20260505 14:59:01.820 | INFO | Perf report for the last 5.13913 seconds: | 20260505 14:59:01.820 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:59:01.820 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:01.820 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:01.820 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:01.820 | INFO | ANALYSIS : 63.0%, 3.0, 98.1ms, 0 | 20260505 14:59:01.820 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:01.820 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:01.820 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 14:59:01.821 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:01.821 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:01.821 | INFO | Uploaded 36GiB to GPU in 2.18097s (16.5064GiB/s) | 20260505 14:59:01.821 | INFO | Downloaded 34GiB to GPU in 2.05856s (16.5164GiB/s) | 20260505 14:59:06.955 | INFO | Analyzing chunk frames = [4109824,4110336) | 20260505 14:59:06.955 | INFO | MemInfo: { | 20260505 14:59:06.955 | INFO | "gpu" : " 17 GB", | 20260505 14:59:06.955 | INFO | "huge" : "", | 20260505 14:59:06.955 | INFO | "huge_shared" : " 27 GB", | 20260505 14:59:06.955 | INFO | "malloc" : " 14 GB", | 20260505 14:59:06.955 | INFO | "pinned" : " 29 GB" | 20260505 14:59:06.955 | INFO | } | 20260505 14:59:06.955 | INFO | Perf report for the last 5.13511 seconds: | 20260505 14:59:06.955 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:59:06.955 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:06.955 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:06.955 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:06.955 | INFO | ANALYSIS : 62.7%, 3.0, 97.3ms, 0 | 20260505 14:59:06.955 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:06.955 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:06.955 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 14:59:06.955 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:06.956 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:06.956 | INFO | Uploaded 36GiB to GPU in 2.18992s (16.4389GiB/s) | 20260505 14:59:06.956 | INFO | Downloaded 34GiB to GPU in 2.13154s (15.9509GiB/s) | 20260505 14:59:12.092 | INFO | Analyzing chunk frames = [4110336,4110848) | 20260505 14:59:12.092 | INFO | MemInfo: { | 20260505 14:59:12.092 | INFO | "gpu" : " 17 GB", | 20260505 14:59:12.092 | INFO | "huge" : "", | 20260505 14:59:12.092 | INFO | "huge_shared" : " 27 GB", | 20260505 14:59:12.092 | INFO | "malloc" : " 14 GB", | 20260505 14:59:12.092 | INFO | "pinned" : " 29 GB" | 20260505 14:59:12.092 | INFO | } | 20260505 14:59:12.092 | INFO | Perf report for the last 5.13715 seconds: | 20260505 14:59:12.093 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:59:12.093 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:12.093 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:12.093 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:12.093 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 14:59:12.093 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:12.093 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:12.093 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 14:59:12.093 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:12.093 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:12.093 | INFO | Uploaded 36GiB to GPU in 2.17194s (16.5751GiB/s) | 20260505 14:59:12.093 | INFO | Downloaded 34GiB to GPU in 2.12915s (15.9688GiB/s) | 20260505 14:59:17.240 | INFO | Analyzing chunk frames = [4110848,4111360) | 20260505 14:59:17.240 | INFO | MemInfo: { | 20260505 14:59:17.240 | INFO | "gpu" : " 17 GB", | 20260505 14:59:17.240 | INFO | "huge" : "", | 20260505 14:59:17.240 | INFO | "huge_shared" : " 27 GB", | 20260505 14:59:17.240 | INFO | "malloc" : " 14 GB", | 20260505 14:59:17.240 | INFO | "pinned" : " 29 GB" | 20260505 14:59:17.240 | INFO | } | 20260505 14:59:17.240 | INFO | Perf report for the last 5.1473 seconds: | 20260505 14:59:17.240 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:59:17.240 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:17.240 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:17.240 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:17.240 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 14:59:17.240 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:17.240 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:17.240 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:59:17.240 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:17.240 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:17.240 | INFO | Uploaded 36GiB to GPU in 2.2s (16.3637GiB/s) | 20260505 14:59:17.240 | INFO | Downloaded 34GiB to GPU in 2.11382s (16.0846GiB/s) | 20260505 14:59:22.377 | INFO | Analyzing chunk frames = [4111360,4111872) | 20260505 14:59:22.377 | INFO | MemInfo: { | 20260505 14:59:22.377 | INFO | "gpu" : " 17 GB", | 20260505 14:59:22.377 | INFO | "huge" : "", | 20260505 14:59:22.377 | INFO | "huge_shared" : " 27 GB", | 20260505 14:59:22.377 | INFO | "malloc" : " 14 GB", | 20260505 14:59:22.377 | INFO | "pinned" : " 29 GB" | 20260505 14:59:22.377 | INFO | } | 20260505 14:59:22.377 | INFO | Perf report for the last 5.13765 seconds: | 20260505 14:59:22.377 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:59:22.377 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:22.377 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:22.377 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:22.377 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 14:59:22.377 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:22.377 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:22.377 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 14:59:22.378 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:22.378 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:22.378 | INFO | Uploaded 36GiB to GPU in 2.11766s (16.9999GiB/s) | 20260505 14:59:22.378 | INFO | Downloaded 34GiB to GPU in 2.18404s (15.5675GiB/s) | 20260505 14:59:27.516 | INFO | Analyzing chunk frames = [4111872,4112384) | 20260505 14:59:27.516 | INFO | MemInfo: { | 20260505 14:59:27.516 | INFO | "gpu" : " 17 GB", | 20260505 14:59:27.516 | INFO | "huge" : "", | 20260505 14:59:27.516 | INFO | "huge_shared" : " 27 GB", | 20260505 14:59:27.516 | INFO | "malloc" : " 14 GB", | 20260505 14:59:27.516 | INFO | "pinned" : " 29 GB" | 20260505 14:59:27.516 | INFO | } | 20260505 14:59:27.517 | INFO | Perf report for the last 5.13908 seconds: | 20260505 14:59:27.517 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:59:27.517 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:27.517 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:27.517 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:27.517 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 14:59:27.517 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:27.517 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:27.517 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 14:59:27.517 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:27.517 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:27.517 | INFO | Uploaded 36GiB to GPU in 2.2174s (16.2352GiB/s) | 20260505 14:59:27.517 | INFO | Downloaded 34GiB to GPU in 2.09618s (16.2199GiB/s) | 20260505 14:59:27.517 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 14:59:27.517 | INFO | Total mem usage: 15.3 GiB | 20260505 14:59:27.517 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 14:59:27.517 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.84%) | 20260505 14:59:27.517 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 14:59:27.518 | INFO | High Water Breakdown: | 20260505 14:59:27.518 | INFO | 61% : 8.67 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 14:59:27.518 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 14:59:27.518 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 14:59:27.518 | INFO | ------------------------------------------ | 20260505 14:59:27.518 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:59:27.518 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 14:59:27.518 | INFO | Total mem usage: 26.5 GiB | 20260505 14:59:27.518 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 14:59:27.518 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 14:59:27.518 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 14:59:27.518 | INFO | High Water Breakdown: | 20260505 14:59:27.518 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:59:27.518 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:59:27.518 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:59:27.518 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:59:27.518 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:59:27.518 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:59:27.518 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 14:59:27.518 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 14:59:27.518 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:59:27.518 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 14:59:27.518 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 14:59:27.518 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 14:59:27.518 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 14:59:27.518 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 14:59:27.518 | INFO | ------------------------------------------ | 20260505 14:59:27.518 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:59:27.518 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 14:59:27.518 | INFO | Total mem usage: 25.6 GiB | 20260505 14:59:27.518 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 14:59:27.518 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 14:59:27.518 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 14:59:27.518 | INFO | High Water Breakdown: | 20260505 14:59:27.518 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 14:59:27.518 | INFO | ------------------------------------------ | 20260505 14:59:27.518 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:59:27.518 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 14:59:27.518 | INFO | Total mem usage: 15.7 GiB | 20260505 14:59:27.518 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 14:59:27.518 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 14:59:27.518 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 14:59:27.518 | INFO | High Water Breakdown: | 20260505 14:59:27.518 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 14:59:27.518 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 14:59:27.518 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 14:59:27.518 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 14:59:27.518 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 14:59:27.518 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 14:59:27.518 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 14:59:27.518 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 14:59:27.518 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 14:59:27.518 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 14:59:27.518 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 14:59:27.518 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 14:59:27.518 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 14:59:27.518 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 14:59:27.518 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 14:59:27.518 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 14:59:27.518 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 14:59:27.518 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 14:59:27.518 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 14:59:27.518 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 14:59:27.518 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 14:59:27.518 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 14:59:27.518 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 14:59:27.518 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 14:59:27.518 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 14:59:27.518 | INFO | ------------------------------------------ | 20260505 14:59:27.518 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 14:59:27.518 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 14:59:27.518 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:59:27.518 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:59:27.518 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.62e+01, 1.36e+01 / 1.83e+00 | 20260505 14:59:27.518 | INFO | PerfReport: 0.29%, AggregateStats -- 9.27e-02 / 1.00e-01, 1.05e-02 / 5.40e-03 | 20260505 14:59:27.518 | INFO | PerfReport: 11.79%, Baseline -- 3.75e+00 / 4.47e+00, 5.83e-01 / 2.14e-01 | 20260505 14:59:27.518 | INFO | PerfReport: 3.29%, Histogram -- 1.05e+00 / 1.51e+00, 1.96e-01 / 9.06e-02 | 20260505 14:59:27.518 | INFO | PerfReport: 15.93%, DME -- 4.05e+01 / 4.53e+01, 4.57e+00 / 2.24e+00 | 20260505 14:59:27.518 | INFO | PerfReport: 56.63%, FrameLabeling -- 1.80e+01 / 2.00e+01, 1.50e+00 / 2.02e+00 | 20260505 14:59:27.518 | INFO | PerfReport: 7.36%, PulseAccumulating -- 2.34e+00 / 2.90e+00, 4.42e-01 / 1.78e-01 | 20260505 14:59:27.518 | INFO | PerfReport: 3.48%, Metrics -- 1.11e+00 / 1.42e+00, 3.10e-01 / 1.23e-01 | 20260505 14:59:27.518 | INFO | PerfReport: 1.23%, PulseSerializing -- 3.90e-01 / 4.39e-01, 7.11e-02 / 2.88e-02 | 20260505 14:59:27.518 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 14:59:27.518 | INFO | PerfReport: Gathered 1536 samples. | 20260505 14:59:27.518 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 14:59:27.519 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260505 14:59:27.519 | INFO | PerfReport: 22.53%, Upload -- 2.27e+01 / 0.00e+00, 8.00e+00 / 0.00e+00 | 20260505 14:59:27.519 | INFO | PerfReport: 21.96%, Download -- 2.21e+01 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260505 14:59:27.519 | INFO | PerfReport: 31.61%, Compute -- 3.18e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 14:59:27.519 | INFO | PerfReport: 2.25%, LockUpload -- 2.27e+00 / 0.00e+00, 7.92e+00 / 0.00e+00 | 20260505 14:59:27.519 | INFO | PerfReport: 1.44%, LockDownload -- 1.45e+00 / 0.00e+00, 3.05e+00 / 0.00e+00 | 20260505 14:59:27.519 | INFO | PerfReport: 20.19%, LockCompute -- 2.03e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260505 14:59:27.519 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.78e-02 / 0.00e+00 | 20260505 14:59:27.652 | INFO | Starting a round of Baz IO | 20260505 14:59:30.871 | INFO | Populating RTMetrics report | 20260505 14:59:30.871 | INFO | RTMetrics report populated | 20260505 14:59:30.871 | INFO | Opening tmp RTMetrics json file | 20260505 14:59:30.887 | INFO | Serializing RTMetrics to json | 20260505 14:59:30.888 | INFO | Serialization done, swapping tmp file now... | 20260505 14:59:30.891 | INFO | RTMetrics done | 20260505 14:59:30.968 | INFO | Finished a round of Baz IO: Wrote 2.233 GiB in 3.315 seconds | 20260505 14:59:30.968 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.59% / 0.00% / 77.40% | 20260505 14:59:30.968 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.493566% / 7.94729e-06% / 9.13938e-05% | 20260505 14:59:30.968 | INFO | Baz IO overall: | 20260505 14:59:30.968 | INFO | BytesWritten : 2448554199680 | 20260505 14:59:30.968 | INFO | NumEvents : 1163664861452 | 20260505 14:59:32.651 | INFO | Analyzing chunk frames = [4112384,4112896) | 20260505 14:59:32.651 | INFO | MemInfo: { | 20260505 14:59:32.651 | INFO | "gpu" : " 17 GB", | 20260505 14:59:32.651 | INFO | "huge" : "", | 20260505 14:59:32.651 | INFO | "huge_shared" : " 27 GB", | 20260505 14:59:32.651 | INFO | "malloc" : " 14 GB", | 20260505 14:59:32.651 | INFO | "pinned" : " 29 GB" | 20260505 14:59:32.651 | INFO | } | 20260505 14:59:32.651 | INFO | Perf report for the last 5.13482 seconds: | 20260505 14:59:32.652 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:59:32.652 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:32.652 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:32.652 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:32.652 | INFO | ANALYSIS : 64.2%, 3.0, 99.8ms, 0 | 20260505 14:59:32.652 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:32.652 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260505 14:59:32.652 | INFO | PRE_HQ : 60.9%, 1.8, 57.4ms, 0 | 20260505 14:59:32.652 | INFO | BAZWRITER : 63.9%, 4.0, 129.9ms, 0 | 20260505 14:59:32.652 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 14:59:32.652 | INFO | Uploaded 36GiB to GPU in 2.20034s (16.3611GiB/s) | 20260505 14:59:32.652 | INFO | Downloaded 38GiB to GPU in 2.25989s (16.815GiB/s) | 20260505 14:59:37.793 | INFO | Analyzing chunk frames = [4112896,4113408) | 20260505 14:59:37.793 | INFO | MemInfo: { | 20260505 14:59:37.793 | INFO | "gpu" : " 17 GB", | 20260505 14:59:37.793 | INFO | "huge" : "", | 20260505 14:59:37.793 | INFO | "huge_shared" : " 27 GB", | 20260505 14:59:37.793 | INFO | "malloc" : " 14 GB", | 20260505 14:59:37.793 | INFO | "pinned" : " 29 GB" | 20260505 14:59:37.793 | INFO | } | 20260505 14:59:37.793 | INFO | Perf report for the last 5.14153 seconds: | 20260505 14:59:37.793 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:59:37.793 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:37.793 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:37.793 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:37.793 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 14:59:37.793 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:37.793 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:37.793 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 14:59:37.793 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:37.793 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:37.793 | INFO | Uploaded 36GiB to GPU in 2.19434s (16.4058GiB/s) | 20260505 14:59:37.793 | INFO | Downloaded 34GiB to GPU in 2.03309s (16.7233GiB/s) | 20260505 14:59:42.934 | INFO | Analyzing chunk frames = [4113408,4113920) | 20260505 14:59:42.934 | INFO | MemInfo: { | 20260505 14:59:42.934 | INFO | "gpu" : " 17 GB", | 20260505 14:59:42.934 | INFO | "huge" : "", | 20260505 14:59:42.934 | INFO | "huge_shared" : " 27 GB", | 20260505 14:59:42.934 | INFO | "malloc" : " 14 GB", | 20260505 14:59:42.934 | INFO | "pinned" : " 29 GB" | 20260505 14:59:42.934 | INFO | } | 20260505 14:59:42.934 | INFO | Perf report for the last 5.14091 seconds: | 20260505 14:59:42.934 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:59:42.934 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:42.934 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:42.934 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:42.934 | INFO | ANALYSIS : 62.9%, 3.0, 98.0ms, 0 | 20260505 14:59:42.934 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:42.934 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:42.934 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 14:59:42.934 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:42.934 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:42.934 | INFO | Uploaded 36GiB to GPU in 2.18837s (16.4506GiB/s) | 20260505 14:59:42.934 | INFO | Downloaded 34GiB to GPU in 2.04223s (16.6485GiB/s) | 20260505 14:59:48.072 | INFO | Analyzing chunk frames = [4113920,4114432) | 20260505 14:59:48.072 | INFO | MemInfo: { | 20260505 14:59:48.072 | INFO | "gpu" : " 17 GB", | 20260505 14:59:48.072 | INFO | "huge" : "", | 20260505 14:59:48.072 | INFO | "huge_shared" : " 27 GB", | 20260505 14:59:48.072 | INFO | "malloc" : " 14 GB", | 20260505 14:59:48.072 | INFO | "pinned" : " 29 GB" | 20260505 14:59:48.072 | INFO | } | 20260505 14:59:48.072 | INFO | Perf report for the last 5.13866 seconds: | 20260505 14:59:48.073 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:59:48.073 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:48.073 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:48.073 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:48.073 | INFO | ANALYSIS : 62.6%, 3.0, 97.3ms, 0 | 20260505 14:59:48.073 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:48.073 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:48.073 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:59:48.073 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:48.073 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:48.073 | INFO | Uploaded 36GiB to GPU in 2.18967s (16.4408GiB/s) | 20260505 14:59:48.073 | INFO | Downloaded 34GiB to GPU in 2.12881s (15.9714GiB/s) | 20260505 14:59:53.201 | INFO | Analyzing chunk frames = [4114432,4114944) | 20260505 14:59:53.201 | INFO | MemInfo: { | 20260505 14:59:53.201 | INFO | "gpu" : " 17 GB", | 20260505 14:59:53.201 | INFO | "huge" : "", | 20260505 14:59:53.201 | INFO | "huge_shared" : " 27 GB", | 20260505 14:59:53.201 | INFO | "malloc" : " 14 GB", | 20260505 14:59:53.201 | INFO | "pinned" : " 29 GB" | 20260505 14:59:53.201 | INFO | } | 20260505 14:59:53.201 | INFO | Perf report for the last 5.12867 seconds: | 20260505 14:59:53.201 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:59:53.201 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:53.201 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:53.201 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:53.201 | INFO | ANALYSIS : 62.8%, 3.0, 97.8ms, 0 | 20260505 14:59:53.201 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:53.201 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:53.202 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 14:59:53.202 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:53.202 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:53.202 | INFO | Uploaded 36GiB to GPU in 2.14779s (16.7614GiB/s) | 20260505 14:59:53.202 | INFO | Downloaded 34GiB to GPU in 2.12765s (15.9801GiB/s) | 20260505 14:59:58.341 | INFO | Analyzing chunk frames = [4114944,4115456) | 20260505 14:59:58.341 | INFO | MemInfo: { | 20260505 14:59:58.341 | INFO | "gpu" : " 17 GB", | 20260505 14:59:58.341 | INFO | "huge" : "", | 20260505 14:59:58.341 | INFO | "huge_shared" : " 27 GB", | 20260505 14:59:58.341 | INFO | "malloc" : " 14 GB", | 20260505 14:59:58.341 | INFO | "pinned" : " 29 GB" | 20260505 14:59:58.341 | INFO | } | 20260505 14:59:58.341 | INFO | Perf report for the last 5.14013 seconds: | 20260505 14:59:58.341 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 14:59:58.341 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:58.341 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:58.341 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:58.341 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 14:59:58.341 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:58.341 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:58.341 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 14:59:58.341 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:58.341 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 14:59:58.342 | INFO | Uploaded 36GiB to GPU in 2.14236s (16.8039GiB/s) | 20260505 14:59:58.342 | INFO | Downloaded 34GiB to GPU in 2.10438s (16.1568GiB/s) | 20260505 15:00:03.493 | INFO | Analyzing chunk frames = [4115456,4115968) | 20260505 15:00:03.493 | INFO | MemInfo: { | 20260505 15:00:03.493 | INFO | "gpu" : " 17 GB", | 20260505 15:00:03.493 | INFO | "huge" : "", | 20260505 15:00:03.493 | INFO | "huge_shared" : " 27 GB", | 20260505 15:00:03.493 | INFO | "malloc" : " 14 GB", | 20260505 15:00:03.493 | INFO | "pinned" : " 29 GB" | 20260505 15:00:03.493 | INFO | } | 20260505 15:00:03.493 | INFO | Perf report for the last 5.15169 seconds: | 20260505 15:00:03.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:00:03.493 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:03.493 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:03.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:03.493 | INFO | ANALYSIS : 63.0%, 3.0, 98.6ms, 0 | 20260505 15:00:03.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:03.493 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:03.493 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 15:00:03.493 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:03.493 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:03.493 | INFO | Uploaded 36GiB to GPU in 2.18596s (16.4687GiB/s) | 20260505 15:00:03.493 | INFO | Downloaded 34GiB to GPU in 2.14751s (15.8323GiB/s) | 20260505 15:00:08.636 | INFO | Analyzing chunk frames = [4115968,4116480) | 20260505 15:00:08.636 | INFO | MemInfo: { | 20260505 15:00:08.636 | INFO | "gpu" : " 17 GB", | 20260505 15:00:08.636 | INFO | "huge" : "", | 20260505 15:00:08.636 | INFO | "huge_shared" : " 27 GB", | 20260505 15:00:08.636 | INFO | "malloc" : " 14 GB", | 20260505 15:00:08.636 | INFO | "pinned" : " 29 GB" | 20260505 15:00:08.636 | INFO | } | 20260505 15:00:08.636 | INFO | Perf report for the last 5.14306 seconds: | 20260505 15:00:08.636 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:00:08.636 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:08.636 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:08.636 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:08.636 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 15:00:08.636 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:08.636 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:08.636 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 15:00:08.636 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:08.636 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:08.636 | INFO | Uploaded 36GiB to GPU in 2.16309s (16.6429GiB/s) | 20260505 15:00:08.636 | INFO | Downloaded 34GiB to GPU in 2.036s (16.6994GiB/s) | 20260505 15:00:08.770 | INFO | Starting a round of Baz IO | 20260505 15:00:11.971 | INFO | Populating RTMetrics report | 20260505 15:00:11.971 | INFO | RTMetrics report populated | 20260505 15:00:11.971 | INFO | Opening tmp RTMetrics json file | 20260505 15:00:12.015 | INFO | Serializing RTMetrics to json | 20260505 15:00:12.053 | INFO | Serialization done, swapping tmp file now... | 20260505 15:00:12.056 | INFO | RTMetrics done | 20260505 15:00:12.077 | INFO | Finished a round of Baz IO: Wrote 2.231 GiB in 3.306 seconds | 20260505 15:00:12.077 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.59% / 0.00% / 77.41% | 20260505 15:00:12.077 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.493201% / 0% / 5.96046e-05% | 20260505 15:00:12.077 | INFO | Baz IO overall: | 20260505 15:00:12.077 | INFO | BytesWritten : 2450949598482 | 20260505 15:00:12.077 | INFO | NumEvents : 1164792065685 | 20260505 15:00:13.768 | INFO | Analyzing chunk frames = [4116480,4116992) | 20260505 15:00:13.768 | INFO | MemInfo: { | 20260505 15:00:13.768 | INFO | "gpu" : " 17 GB", | 20260505 15:00:13.768 | INFO | "huge" : "", | 20260505 15:00:13.768 | INFO | "huge_shared" : " 27 GB", | 20260505 15:00:13.768 | INFO | "malloc" : " 14 GB", | 20260505 15:00:13.768 | INFO | "pinned" : " 29 GB" | 20260505 15:00:13.768 | INFO | } | 20260505 15:00:13.769 | INFO | Perf report for the last 5.13245 seconds: | 20260505 15:00:13.769 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:00:13.769 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:13.769 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:13.769 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:13.769 | INFO | ANALYSIS : 63.9%, 3.0, 99.0ms, 0 | 20260505 15:00:13.769 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:13.769 | INFO | RT_METRICS : 46.6%, 1.0, 24.4ms, 0 | 20260505 15:00:13.769 | INFO | PRE_HQ : 61.2%, 1.8, 57.3ms, 0 | 20260505 15:00:13.769 | INFO | BAZWRITER : 63.7%, 4.0, 132.0ms, 0 | 20260505 15:00:13.769 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 15:00:13.769 | INFO | Uploaded 36GiB to GPU in 2.15946s (16.6708GiB/s) | 20260505 15:00:13.769 | INFO | Downloaded 38GiB to GPU in 2.25856s (16.8249GiB/s) | 20260505 15:00:18.898 | INFO | Analyzing chunk frames = [4116992,4117504) | 20260505 15:00:18.898 | INFO | MemInfo: { | 20260505 15:00:18.898 | INFO | "gpu" : " 17 GB", | 20260505 15:00:18.898 | INFO | "huge" : "", | 20260505 15:00:18.898 | INFO | "huge_shared" : " 27 GB", | 20260505 15:00:18.898 | INFO | "malloc" : " 14 GB", | 20260505 15:00:18.898 | INFO | "pinned" : " 29 GB" | 20260505 15:00:18.898 | INFO | } | 20260505 15:00:18.898 | INFO | Perf report for the last 5.12976 seconds: | 20260505 15:00:18.898 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:00:18.898 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:18.898 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:18.898 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:18.898 | INFO | ANALYSIS : 63.6%, 3.0, 99.1ms, 0 | 20260505 15:00:18.898 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:18.898 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:18.898 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 15:00:18.898 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:18.899 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:18.899 | INFO | Uploaded 36GiB to GPU in 2.1965s (16.3897GiB/s) | 20260505 15:00:18.899 | INFO | Downloaded 34GiB to GPU in 2.04704s (16.6093GiB/s) | 20260505 15:00:24.045 | INFO | Analyzing chunk frames = [4117504,4118016) | 20260505 15:00:24.045 | INFO | MemInfo: { | 20260505 15:00:24.045 | INFO | "gpu" : " 17 GB", | 20260505 15:00:24.045 | INFO | "huge" : "", | 20260505 15:00:24.045 | INFO | "huge_shared" : " 27 GB", | 20260505 15:00:24.045 | INFO | "malloc" : " 14 GB", | 20260505 15:00:24.045 | INFO | "pinned" : " 29 GB" | 20260505 15:00:24.045 | INFO | } | 20260505 15:00:24.045 | INFO | Perf report for the last 5.14719 seconds: | 20260505 15:00:24.045 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:00:24.046 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:24.046 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:24.046 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:24.046 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 15:00:24.046 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:24.046 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:24.046 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 15:00:24.046 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:24.046 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:24.046 | INFO | Uploaded 36GiB to GPU in 2.20429s (16.3318GiB/s) | 20260505 15:00:24.046 | INFO | Downloaded 34GiB to GPU in 2.11969s (16.0401GiB/s) | 20260505 15:00:29.185 | INFO | Analyzing chunk frames = [4118016,4118528) | 20260505 15:00:29.185 | INFO | MemInfo: { | 20260505 15:00:29.185 | INFO | "gpu" : " 17 GB", | 20260505 15:00:29.185 | INFO | "huge" : "", | 20260505 15:00:29.185 | INFO | "huge_shared" : " 27 GB", | 20260505 15:00:29.185 | INFO | "malloc" : " 14 GB", | 20260505 15:00:29.185 | INFO | "pinned" : " 29 GB" | 20260505 15:00:29.185 | INFO | } | 20260505 15:00:29.185 | INFO | Perf report for the last 5.13977 seconds: | 20260505 15:00:29.185 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:00:29.185 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:29.185 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:29.185 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:29.185 | INFO | ANALYSIS : 62.8%, 3.0, 97.5ms, 0 | 20260505 15:00:29.185 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:29.185 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:29.185 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 15:00:29.185 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:29.185 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:29.186 | INFO | Uploaded 36GiB to GPU in 2.18596s (16.4687GiB/s) | 20260505 15:00:29.186 | INFO | Downloaded 34GiB to GPU in 2.13205s (15.9471GiB/s) | 20260505 15:00:34.329 | INFO | Analyzing chunk frames = [4118528,4119040) | 20260505 15:00:34.329 | INFO | MemInfo: { | 20260505 15:00:34.329 | INFO | "gpu" : " 17 GB", | 20260505 15:00:34.329 | INFO | "huge" : "", | 20260505 15:00:34.329 | INFO | "huge_shared" : " 27 GB", | 20260505 15:00:34.329 | INFO | "malloc" : " 14 GB", | 20260505 15:00:34.329 | INFO | "pinned" : " 29 GB" | 20260505 15:00:34.329 | INFO | } | 20260505 15:00:34.329 | INFO | Perf report for the last 5.14399 seconds: | 20260505 15:00:34.329 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:00:34.330 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:34.330 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:34.330 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:34.330 | INFO | ANALYSIS : 62.5%, 3.0, 97.7ms, 0 | 20260505 15:00:34.330 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:34.330 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:34.330 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 15:00:34.330 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:34.330 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:34.330 | INFO | Uploaded 36GiB to GPU in 2.15142s (16.7332GiB/s) | 20260505 15:00:34.330 | INFO | Downloaded 34GiB to GPU in 2.1311s (15.9542GiB/s) | 20260505 15:00:39.458 | INFO | Analyzing chunk frames = [4119040,4119552) | 20260505 15:00:39.459 | INFO | MemInfo: { | 20260505 15:00:39.459 | INFO | "gpu" : " 17 GB", | 20260505 15:00:39.459 | INFO | "huge" : "", | 20260505 15:00:39.459 | INFO | "huge_shared" : " 27 GB", | 20260505 15:00:39.459 | INFO | "malloc" : " 14 GB", | 20260505 15:00:39.459 | INFO | "pinned" : " 29 GB" | 20260505 15:00:39.459 | INFO | } | 20260505 15:00:39.459 | INFO | Perf report for the last 5.12948 seconds: | 20260505 15:00:39.459 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:00:39.459 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:39.459 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:39.459 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:39.459 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260505 15:00:39.459 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:39.459 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:39.459 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 15:00:39.459 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:39.459 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:39.459 | INFO | Uploaded 36GiB to GPU in 2.15983s (16.6679GiB/s) | 20260505 15:00:39.459 | INFO | Downloaded 34GiB to GPU in 2.0561s (16.5362GiB/s) | 20260505 15:00:44.598 | INFO | Analyzing chunk frames = [4119552,4120064) | 20260505 15:00:44.598 | INFO | MemInfo: { | 20260505 15:00:44.598 | INFO | "gpu" : " 17 GB", | 20260505 15:00:44.598 | INFO | "huge" : "", | 20260505 15:00:44.598 | INFO | "huge_shared" : " 27 GB", | 20260505 15:00:44.598 | INFO | "malloc" : " 14 GB", | 20260505 15:00:44.598 | INFO | "pinned" : " 29 GB" | 20260505 15:00:44.598 | INFO | } | 20260505 15:00:44.598 | INFO | Perf report for the last 5.13918 seconds: | 20260505 15:00:44.598 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:00:44.598 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:44.598 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:44.598 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:44.598 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 15:00:44.598 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:44.598 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:44.598 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 15:00:44.598 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:44.598 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:44.598 | INFO | Uploaded 36GiB to GPU in 2.1302s (16.8998GiB/s) | 20260505 15:00:44.599 | INFO | Downloaded 34GiB to GPU in 2.18758s (15.5423GiB/s) | 20260505 15:00:49.752 | INFO | Analyzing chunk frames = [4120064,4120576) | 20260505 15:00:49.752 | INFO | MemInfo: { | 20260505 15:00:49.752 | INFO | "gpu" : " 17 GB", | 20260505 15:00:49.752 | INFO | "huge" : "", | 20260505 15:00:49.752 | INFO | "huge_shared" : " 27 GB", | 20260505 15:00:49.752 | INFO | "malloc" : " 14 GB", | 20260505 15:00:49.752 | INFO | "pinned" : " 29 GB" | 20260505 15:00:49.752 | INFO | } | 20260505 15:00:49.752 | INFO | Perf report for the last 5.15424 seconds: | 20260505 15:00:49.752 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:00:49.752 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:49.752 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:49.752 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:49.752 | INFO | ANALYSIS : 63.3%, 3.0, 99.1ms, 0 | 20260505 15:00:49.752 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:49.752 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:49.752 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 15:00:49.752 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:49.752 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:49.752 | INFO | Uploaded 36GiB to GPU in 2.12345s (16.9535GiB/s) | 20260505 15:00:49.752 | INFO | Downloaded 34GiB to GPU in 2.10464s (16.1548GiB/s) | 20260505 15:00:49.753 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 15:00:49.753 | INFO | Total mem usage: 15.3 GiB | 20260505 15:00:49.753 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 15:00:49.753 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.35%) | 20260505 15:00:49.753 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 15:00:49.753 | INFO | High Water Breakdown: | 20260505 15:00:49.753 | INFO | 61.2% : 8.82 GiB ( 92.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 15:00:49.753 | INFO | 38.6% : 5.57 GiB ( 83.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 15:00:49.753 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 15:00:49.753 | INFO | ------------------------------------------ | 20260505 15:00:49.753 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:00:49.753 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 15:00:49.753 | INFO | Total mem usage: 26.5 GiB | 20260505 15:00:49.753 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 15:00:49.753 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 15:00:49.753 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 15:00:49.753 | INFO | High Water Breakdown: | 20260505 15:00:49.753 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:00:49.753 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:00:49.753 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:00:49.753 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:00:49.753 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:00:49.753 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:00:49.753 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 15:00:49.753 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 15:00:49.753 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:00:49.753 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 15:00:49.753 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 15:00:49.753 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 15:00:49.753 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 15:00:49.753 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 15:00:49.753 | INFO | ------------------------------------------ | 20260505 15:00:49.753 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:00:49.753 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 15:00:49.753 | INFO | Total mem usage: 25.6 GiB | 20260505 15:00:49.753 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 15:00:49.753 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 15:00:49.753 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 15:00:49.753 | INFO | High Water Breakdown: | 20260505 15:00:49.753 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 15:00:49.753 | INFO | ------------------------------------------ | 20260505 15:00:49.753 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:00:49.753 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 15:00:49.753 | INFO | Total mem usage: 15.7 GiB | 20260505 15:00:49.753 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 15:00:49.753 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 15:00:49.753 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 15:00:49.753 | INFO | High Water Breakdown: | 20260505 15:00:49.753 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 15:00:49.753 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 15:00:49.753 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 15:00:49.753 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 15:00:49.753 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 15:00:49.753 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 15:00:49.753 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 15:00:49.753 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 15:00:49.753 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 15:00:49.753 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:00:49.753 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 15:00:49.753 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:00:49.753 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 15:00:49.753 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 15:00:49.753 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 15:00:49.753 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 15:00:49.753 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:00:49.753 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 15:00:49.753 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:00:49.753 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:00:49.753 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:00:49.753 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 15:00:49.753 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 15:00:49.753 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:00:49.753 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 15:00:49.753 | INFO | ------------------------------------------ | 20260505 15:00:49.753 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:00:49.753 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 15:00:49.753 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:00:49.753 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:00:49.753 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.59e+01, 1.37e+01 / 1.98e+00 | 20260505 15:00:49.753 | INFO | PerfReport: 0.29%, AggregateStats -- 9.30e-02 / 1.00e-01, 1.11e-02 / 4.66e-03 | 20260505 15:00:49.753 | INFO | PerfReport: 11.80%, Baseline -- 3.75e+00 / 4.50e+00, 5.82e-01 / 1.73e-01 | 20260505 15:00:49.753 | INFO | PerfReport: 3.31%, Histogram -- 1.05e+00 / 1.49e+00, 1.99e-01 / 1.04e-01 | 20260505 15:00:49.754 | INFO | PerfReport: 15.94%, DME -- 4.05e+01 / 4.52e+01, 4.35e+00 / 2.02e+00 | 20260505 15:00:49.754 | INFO | PerfReport: 56.54%, FrameLabeling -- 1.80e+01 / 1.99e+01, 1.45e+00 / 1.51e+00 | 20260505 15:00:49.754 | INFO | PerfReport: 7.40%, PulseAccumulating -- 2.35e+00 / 2.92e+00, 4.46e-01 / 1.64e-01 | 20260505 15:00:49.754 | INFO | PerfReport: 3.50%, Metrics -- 1.11e+00 / 1.42e+00, 3.09e-01 / 1.72e-01 | 20260505 15:00:49.754 | INFO | PerfReport: 1.23%, PulseSerializing -- 3.90e-01 / 4.47e-01, 7.03e-02 / 3.13e-02 | 20260505 15:00:49.754 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 15:00:49.754 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:00:49.754 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:00:49.754 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260505 15:00:49.754 | INFO | PerfReport: 22.49%, Upload -- 2.26e+01 / 0.00e+00, 7.91e+00 / 0.00e+00 | 20260505 15:00:49.754 | INFO | PerfReport: 21.97%, Download -- 2.21e+01 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260505 15:00:49.754 | INFO | PerfReport: 31.64%, Compute -- 3.18e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 15:00:49.754 | INFO | PerfReport: 2.11%, LockUpload -- 2.12e+00 / 0.00e+00, 7.78e+00 / 0.00e+00 | 20260505 15:00:49.754 | INFO | PerfReport: 1.50%, LockDownload -- 1.51e+00 / 0.00e+00, 3.07e+00 / 0.00e+00 | 20260505 15:00:49.754 | INFO | PerfReport: 20.27%, LockCompute -- 2.04e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260505 15:00:49.754 | INFO | PerfReport: 0.01%, Misc -- 1.27e-02 / 0.00e+00, 1.53e-02 / 0.00e+00 | 20260505 15:00:49.885 | INFO | Starting a round of Baz IO | 20260505 15:00:53.098 | INFO | Populating RTMetrics report | 20260505 15:00:53.098 | INFO | RTMetrics report populated | 20260505 15:00:53.098 | INFO | Opening tmp RTMetrics json file | 20260505 15:00:53.116 | INFO | Serializing RTMetrics to json | 20260505 15:00:53.140 | INFO | Serialization done, swapping tmp file now... | 20260505 15:00:53.162 | INFO | RTMetrics done | 20260505 15:00:53.199 | INFO | Finished a round of Baz IO: Wrote 2.232 GiB in 3.313 seconds | 20260505 15:00:53.199 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.59% / 0.00% / 77.41% | 20260505 15:00:53.199 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.49146% / 3.97364e-06% / 5.96046e-05% | 20260505 15:00:53.199 | INFO | Baz IO overall: | 20260505 15:00:53.199 | INFO | BytesWritten : 2453345841653 | 20260505 15:00:53.199 | INFO | NumEvents : 1165919864700 | 20260505 15:00:54.883 | INFO | Analyzing chunk frames = [4120576,4121088) | 20260505 15:00:54.883 | INFO | MemInfo: { | 20260505 15:00:54.883 | INFO | "gpu" : " 17 GB", | 20260505 15:00:54.883 | INFO | "huge" : "", | 20260505 15:00:54.883 | INFO | "huge_shared" : " 27 GB", | 20260505 15:00:54.883 | INFO | "malloc" : " 14 GB", | 20260505 15:00:54.883 | INFO | "pinned" : " 29 GB" | 20260505 15:00:54.883 | INFO | } | 20260505 15:00:54.883 | INFO | Perf report for the last 5.13096 seconds: | 20260505 15:00:54.883 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:00:54.883 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:54.883 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:54.883 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:54.883 | INFO | ANALYSIS : 64.1%, 3.0, 99.5ms, 0 | 20260505 15:00:54.883 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:00:54.883 | INFO | RT_METRICS : 46.1%, 1.0, 24.1ms, 0 | 20260505 15:00:54.883 | INFO | PRE_HQ : 61.2%, 1.8, 57.2ms, 0 | 20260505 15:00:54.883 | INFO | BAZWRITER : 63.9%, 4.0, 130.1ms, 0 | 20260505 15:00:54.883 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 15:00:54.884 | INFO | Uploaded 36GiB to GPU in 2.19757s (16.3817GiB/s) | 20260505 15:00:54.884 | INFO | Downloaded 38GiB to GPU in 2.26461s (16.78GiB/s) | 20260505 15:01:00.018 | INFO | Analyzing chunk frames = [4121088,4121600) | 20260505 15:01:00.018 | INFO | MemInfo: { | 20260505 15:01:00.018 | INFO | "gpu" : " 17 GB", | 20260505 15:01:00.018 | INFO | "huge" : "", | 20260505 15:01:00.018 | INFO | "huge_shared" : " 27 GB", | 20260505 15:01:00.018 | INFO | "malloc" : " 14 GB", | 20260505 15:01:00.019 | INFO | "pinned" : " 29 GB" | 20260505 15:01:00.019 | INFO | } | 20260505 15:01:00.019 | INFO | Perf report for the last 5.13535 seconds: | 20260505 15:01:00.019 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:01:00.019 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:00.019 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:00.019 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:00.019 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 15:01:00.019 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:00.019 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:00.019 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 15:01:00.019 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:00.019 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:00.019 | INFO | Uploaded 36GiB to GPU in 2.20143s (16.353GiB/s) | 20260505 15:01:00.019 | INFO | Downloaded 34GiB to GPU in 2.04879s (16.5951GiB/s) | 20260505 15:01:05.152 | INFO | Analyzing chunk frames = [4121600,4122112) | 20260505 15:01:05.152 | INFO | MemInfo: { | 20260505 15:01:05.152 | INFO | "gpu" : " 17 GB", | 20260505 15:01:05.152 | INFO | "huge" : "", | 20260505 15:01:05.152 | INFO | "huge_shared" : " 27 GB", | 20260505 15:01:05.152 | INFO | "malloc" : " 14 GB", | 20260505 15:01:05.152 | INFO | "pinned" : " 29 GB" | 20260505 15:01:05.152 | INFO | } | 20260505 15:01:05.152 | INFO | Perf report for the last 5.13369 seconds: | 20260505 15:01:05.152 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:01:05.152 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:05.152 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:05.152 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:05.152 | INFO | ANALYSIS : 63.3%, 3.0, 98.3ms, 0 | 20260505 15:01:05.152 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:05.152 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:05.153 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 15:01:05.153 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:05.153 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:05.153 | INFO | Uploaded 36GiB to GPU in 2.1982s (16.377GiB/s) | 20260505 15:01:05.153 | INFO | Downloaded 34GiB to GPU in 2.1205s (16.034GiB/s) | 20260505 15:01:10.301 | INFO | Analyzing chunk frames = [4122112,4122624) | 20260505 15:01:10.301 | INFO | MemInfo: { | 20260505 15:01:10.301 | INFO | "gpu" : " 17 GB", | 20260505 15:01:10.301 | INFO | "huge" : "", | 20260505 15:01:10.301 | INFO | "huge_shared" : " 27 GB", | 20260505 15:01:10.301 | INFO | "malloc" : " 14 GB", | 20260505 15:01:10.301 | INFO | "pinned" : " 29 GB" | 20260505 15:01:10.301 | INFO | } | 20260505 15:01:10.301 | INFO | Perf report for the last 5.14865 seconds: | 20260505 15:01:10.301 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:01:10.301 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:10.301 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:10.301 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:10.301 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 15:01:10.301 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:10.301 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:10.301 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 15:01:10.301 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:10.301 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:10.301 | INFO | Uploaded 36GiB to GPU in 2.18162s (16.5015GiB/s) | 20260505 15:01:10.301 | INFO | Downloaded 34GiB to GPU in 2.13168s (15.9498GiB/s) | 20260505 15:01:15.442 | INFO | Analyzing chunk frames = [4122624,4123136) | 20260505 15:01:15.442 | INFO | MemInfo: { | 20260505 15:01:15.442 | INFO | "gpu" : " 17 GB", | 20260505 15:01:15.442 | INFO | "huge" : "", | 20260505 15:01:15.442 | INFO | "huge_shared" : " 27 GB", | 20260505 15:01:15.442 | INFO | "malloc" : " 14 GB", | 20260505 15:01:15.442 | INFO | "pinned" : " 29 GB" | 20260505 15:01:15.442 | INFO | } | 20260505 15:01:15.442 | INFO | Perf report for the last 5.14081 seconds: | 20260505 15:01:15.442 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:01:15.442 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:15.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:15.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:15.442 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 15:01:15.442 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:15.442 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:15.442 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 15:01:15.442 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:15.442 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:15.442 | INFO | Uploaded 36GiB to GPU in 2.19773s (16.3805GiB/s) | 20260505 15:01:15.442 | INFO | Downloaded 34GiB to GPU in 2.13141s (15.9519GiB/s) | 20260505 15:01:20.582 | INFO | Analyzing chunk frames = [4123136,4123648) | 20260505 15:01:20.582 | INFO | MemInfo: { | 20260505 15:01:20.582 | INFO | "gpu" : " 17 GB", | 20260505 15:01:20.582 | INFO | "huge" : "", | 20260505 15:01:20.582 | INFO | "huge_shared" : " 27 GB", | 20260505 15:01:20.582 | INFO | "malloc" : " 14 GB", | 20260505 15:01:20.582 | INFO | "pinned" : " 29 GB" | 20260505 15:01:20.582 | INFO | } | 20260505 15:01:20.582 | INFO | Perf report for the last 5.14019 seconds: | 20260505 15:01:20.582 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:01:20.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:20.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:20.582 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:20.582 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260505 15:01:20.582 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:20.582 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:20.582 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 15:01:20.582 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:20.582 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:20.582 | INFO | Uploaded 36GiB to GPU in 2.13961s (16.8255GiB/s) | 20260505 15:01:20.582 | INFO | Downloaded 34GiB to GPU in 2.03067s (16.7433GiB/s) | 20260505 15:01:25.716 | INFO | Analyzing chunk frames = [4123648,4124160) | 20260505 15:01:25.716 | INFO | MemInfo: { | 20260505 15:01:25.716 | INFO | "gpu" : " 17 GB", | 20260505 15:01:25.716 | INFO | "huge" : "", | 20260505 15:01:25.716 | INFO | "huge_shared" : " 27 GB", | 20260505 15:01:25.716 | INFO | "malloc" : " 14 GB", | 20260505 15:01:25.716 | INFO | "pinned" : " 29 GB" | 20260505 15:01:25.716 | INFO | } | 20260505 15:01:25.716 | INFO | Perf report for the last 5.1342 seconds: | 20260505 15:01:25.716 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:01:25.716 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:25.716 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:25.716 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:25.716 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 15:01:25.716 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:25.716 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:25.716 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 15:01:25.716 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:25.716 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:25.716 | INFO | Uploaded 36GiB to GPU in 2.11627s (17.0111GiB/s) | 20260505 15:01:25.717 | INFO | Downloaded 34GiB to GPU in 2.15432s (15.7823GiB/s) | 20260505 15:01:30.854 | INFO | Analyzing chunk frames = [4124160,4124672) | 20260505 15:01:30.854 | INFO | MemInfo: { | 20260505 15:01:30.854 | INFO | "gpu" : " 17 GB", | 20260505 15:01:30.854 | INFO | "huge" : "", | 20260505 15:01:30.854 | INFO | "huge_shared" : " 27 GB", | 20260505 15:01:30.854 | INFO | "malloc" : " 14 GB", | 20260505 15:01:30.854 | INFO | "pinned" : " 29 GB" | 20260505 15:01:30.854 | INFO | } | 20260505 15:01:30.854 | INFO | Perf report for the last 5.1381 seconds: | 20260505 15:01:30.854 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:01:30.854 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:30.854 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:30.854 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:30.854 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 15:01:30.854 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:30.854 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:30.854 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 15:01:30.854 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:30.854 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:30.855 | INFO | Uploaded 36GiB to GPU in 2.12372s (16.9514GiB/s) | 20260505 15:01:30.855 | INFO | Downloaded 34GiB to GPU in 2.10198s (16.1752GiB/s) | 20260505 15:01:30.990 | INFO | Starting a round of Baz IO | 20260505 15:01:34.205 | INFO | Populating RTMetrics report | 20260505 15:01:34.205 | INFO | RTMetrics report populated | 20260505 15:01:34.205 | INFO | Opening tmp RTMetrics json file | 20260505 15:01:34.222 | INFO | Serializing RTMetrics to json | 20260505 15:01:34.224 | INFO | Serialization done, swapping tmp file now... | 20260505 15:01:34.227 | INFO | RTMetrics done | 20260505 15:01:34.320 | INFO | Finished a round of Baz IO: Wrote 2.231 GiB in 3.329 seconds | 20260505 15:01:34.320 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.58% / 0.00% / 77.41% | 20260505 15:01:34.320 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.491198% / 1.19209e-05% / 9.13938e-05% | 20260505 15:01:34.320 | INFO | Baz IO overall: | 20260505 15:01:34.320 | INFO | BytesWritten : 2455741743412 | 20260505 15:01:34.320 | INFO | NumEvents : 1167047164375 | 20260505 15:01:36.000 | INFO | Analyzing chunk frames = [4124672,4125184) | 20260505 15:01:36.000 | INFO | MemInfo: { | 20260505 15:01:36.000 | INFO | "gpu" : " 17 GB", | 20260505 15:01:36.000 | INFO | "huge" : "", | 20260505 15:01:36.000 | INFO | "huge_shared" : " 27 GB", | 20260505 15:01:36.000 | INFO | "malloc" : " 14 GB", | 20260505 15:01:36.001 | INFO | "pinned" : " 29 GB" | 20260505 15:01:36.001 | INFO | } | 20260505 15:01:36.001 | INFO | Perf report for the last 5.14625 seconds: | 20260505 15:01:36.001 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:01:36.001 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:36.001 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:36.001 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:36.001 | INFO | ANALYSIS : 64.0%, 3.0, 99.6ms, 0 | 20260505 15:01:36.001 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:36.001 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260505 15:01:36.001 | INFO | PRE_HQ : 60.6%, 1.8, 57.2ms, 0 | 20260505 15:01:36.001 | INFO | BAZWRITER : 64.1%, 4.0, 131.4ms, 0 | 20260505 15:01:36.001 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260505 15:01:36.001 | INFO | Uploaded 36GiB to GPU in 2.19521s (16.3994GiB/s) | 20260505 15:01:36.001 | INFO | Downloaded 38GiB to GPU in 2.26128s (16.8046GiB/s) | 20260505 15:01:41.141 | INFO | Analyzing chunk frames = [4125184,4125696) | 20260505 15:01:41.141 | INFO | MemInfo: { | 20260505 15:01:41.141 | INFO | "gpu" : " 17 GB", | 20260505 15:01:41.141 | INFO | "huge" : "", | 20260505 15:01:41.141 | INFO | "huge_shared" : " 27 GB", | 20260505 15:01:41.141 | INFO | "malloc" : " 14 GB", | 20260505 15:01:41.141 | INFO | "pinned" : " 29 GB" | 20260505 15:01:41.141 | INFO | } | 20260505 15:01:41.142 | INFO | Perf report for the last 5.14097 seconds: | 20260505 15:01:41.142 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:01:41.142 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:41.142 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:41.142 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:41.142 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 15:01:41.142 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:41.142 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:41.142 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:01:41.142 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:41.142 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:41.142 | INFO | Uploaded 36GiB to GPU in 2.13467s (16.8644GiB/s) | 20260505 15:01:41.142 | INFO | Downloaded 34GiB to GPU in 2.03989s (16.6676GiB/s) | 20260505 15:01:46.275 | INFO | Analyzing chunk frames = [4125696,4126208) | 20260505 15:01:46.275 | INFO | MemInfo: { | 20260505 15:01:46.275 | INFO | "gpu" : " 17 GB", | 20260505 15:01:46.275 | INFO | "huge" : "", | 20260505 15:01:46.275 | INFO | "huge_shared" : " 27 GB", | 20260505 15:01:46.275 | INFO | "malloc" : " 14 GB", | 20260505 15:01:46.275 | INFO | "pinned" : " 29 GB" | 20260505 15:01:46.275 | INFO | } | 20260505 15:01:46.275 | INFO | Perf report for the last 5.13377 seconds: | 20260505 15:01:46.275 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:01:46.275 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:46.275 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:46.275 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:46.275 | INFO | ANALYSIS : 63.1%, 3.0, 98.1ms, 0 | 20260505 15:01:46.275 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:46.275 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:46.275 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 15:01:46.275 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:46.275 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:46.276 | INFO | Uploaded 36GiB to GPU in 2.18369s (16.4858GiB/s) | 20260505 15:01:46.276 | INFO | Downloaded 34GiB to GPU in 2.05868s (16.5154GiB/s) | 20260505 15:01:51.406 | INFO | Analyzing chunk frames = [4126208,4126720) | 20260505 15:01:51.406 | INFO | MemInfo: { | 20260505 15:01:51.406 | INFO | "gpu" : " 17 GB", | 20260505 15:01:51.406 | INFO | "huge" : "", | 20260505 15:01:51.406 | INFO | "huge_shared" : " 27 GB", | 20260505 15:01:51.406 | INFO | "malloc" : " 14 GB", | 20260505 15:01:51.406 | INFO | "pinned" : " 29 GB" | 20260505 15:01:51.406 | INFO | } | 20260505 15:01:51.406 | INFO | Perf report for the last 5.13101 seconds: | 20260505 15:01:51.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:01:51.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:51.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:51.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:51.406 | INFO | ANALYSIS : 62.8%, 3.0, 97.5ms, 0 | 20260505 15:01:51.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:51.406 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:51.406 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 15:01:51.406 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:51.407 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:51.407 | INFO | Uploaded 36GiB to GPU in 2.18035s (16.5111GiB/s) | 20260505 15:01:51.407 | INFO | Downloaded 34GiB to GPU in 2.1345s (15.9288GiB/s) | 20260505 15:01:56.556 | INFO | Analyzing chunk frames = [4126720,4127232) | 20260505 15:01:56.556 | INFO | MemInfo: { | 20260505 15:01:56.556 | INFO | "gpu" : " 17 GB", | 20260505 15:01:56.556 | INFO | "huge" : "", | 20260505 15:01:56.556 | INFO | "huge_shared" : " 27 GB", | 20260505 15:01:56.556 | INFO | "malloc" : " 14 GB", | 20260505 15:01:56.556 | INFO | "pinned" : " 29 GB" | 20260505 15:01:56.556 | INFO | } | 20260505 15:01:56.556 | INFO | Perf report for the last 5.14942 seconds: | 20260505 15:01:56.556 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:01:56.556 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:56.556 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:56.556 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:56.556 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 15:01:56.556 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:56.556 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:56.556 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 15:01:56.556 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:56.556 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:01:56.556 | INFO | Uploaded 36GiB to GPU in 2.17883s (16.5226GiB/s) | 20260505 15:01:56.556 | INFO | Downloaded 34GiB to GPU in 2.12667s (15.9874GiB/s) | 20260505 15:02:01.696 | INFO | Analyzing chunk frames = [4127232,4127744) | 20260505 15:02:01.696 | INFO | MemInfo: { | 20260505 15:02:01.696 | INFO | "gpu" : " 17 GB", | 20260505 15:02:01.696 | INFO | "huge" : "", | 20260505 15:02:01.696 | INFO | "huge_shared" : " 27 GB", | 20260505 15:02:01.696 | INFO | "malloc" : " 14 GB", | 20260505 15:02:01.696 | INFO | "pinned" : " 29 GB" | 20260505 15:02:01.696 | INFO | } | 20260505 15:02:01.696 | INFO | Perf report for the last 5.14063 seconds: | 20260505 15:02:01.696 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:02:01.696 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:01.696 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:01.696 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:01.696 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 15:02:01.696 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:01.696 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:01.696 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:02:01.696 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:01.697 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:01.697 | INFO | Uploaded 36GiB to GPU in 2.20877s (16.2987GiB/s) | 20260505 15:02:01.697 | INFO | Downloaded 34GiB to GPU in 2.11016s (16.1125GiB/s) | 20260505 15:02:06.832 | INFO | Analyzing chunk frames = [4127744,4128256) | 20260505 15:02:06.832 | INFO | MemInfo: { | 20260505 15:02:06.832 | INFO | "gpu" : " 17 GB", | 20260505 15:02:06.832 | INFO | "huge" : "", | 20260505 15:02:06.832 | INFO | "huge_shared" : " 27 GB", | 20260505 15:02:06.832 | INFO | "malloc" : " 14 GB", | 20260505 15:02:06.832 | INFO | "pinned" : " 29 GB" | 20260505 15:02:06.832 | INFO | } | 20260505 15:02:06.832 | INFO | Perf report for the last 5.13596 seconds: | 20260505 15:02:06.832 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:02:06.832 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:06.832 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:06.832 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:06.832 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 15:02:06.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:06.832 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:06.832 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 15:02:06.832 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:06.832 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:06.833 | INFO | Uploaded 36GiB to GPU in 2.10687s (17.0869GiB/s) | 20260505 15:02:06.833 | INFO | Downloaded 34GiB to GPU in 2.17984s (15.5975GiB/s) | 20260505 15:02:11.966 | INFO | Analyzing chunk frames = [4128256,4128768) | 20260505 15:02:11.966 | INFO | MemInfo: { | 20260505 15:02:11.966 | INFO | "gpu" : " 17 GB", | 20260505 15:02:11.966 | INFO | "huge" : "", | 20260505 15:02:11.966 | INFO | "huge_shared" : " 27 GB", | 20260505 15:02:11.966 | INFO | "malloc" : " 14 GB", | 20260505 15:02:11.966 | INFO | "pinned" : " 29 GB" | 20260505 15:02:11.966 | INFO | } | 20260505 15:02:11.966 | INFO | Perf report for the last 5.13407 seconds: | 20260505 15:02:11.966 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:02:11.966 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:11.967 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:11.967 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:11.967 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 15:02:11.967 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:11.967 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:11.967 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 15:02:11.967 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:11.967 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:11.967 | INFO | Uploaded 36GiB to GPU in 2.20228s (16.3467GiB/s) | 20260505 15:02:11.967 | INFO | Downloaded 34GiB to GPU in 2.10649s (16.1406GiB/s) | 20260505 15:02:11.967 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 15:02:11.967 | INFO | Total mem usage: 15.3 GiB | 20260505 15:02:11.967 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 15:02:11.967 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.88%) | 20260505 15:02:11.967 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 15:02:11.967 | INFO | High Water Breakdown: | 20260505 15:02:11.967 | INFO | 61% : 8.67 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 15:02:11.967 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 15:02:11.967 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 15:02:11.967 | INFO | ------------------------------------------ | 20260505 15:02:11.967 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:02:11.967 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 15:02:11.967 | INFO | Total mem usage: 26.5 GiB | 20260505 15:02:11.967 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 15:02:11.967 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 15:02:11.967 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 15:02:11.967 | INFO | High Water Breakdown: | 20260505 15:02:11.967 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:02:11.967 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:02:11.967 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:02:11.967 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:02:11.967 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:02:11.967 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:02:11.967 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 15:02:11.967 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 15:02:11.967 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:02:11.967 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 15:02:11.967 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 15:02:11.967 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 15:02:11.967 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 15:02:11.967 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 15:02:11.967 | INFO | ------------------------------------------ | 20260505 15:02:11.967 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:02:11.967 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 15:02:11.967 | INFO | Total mem usage: 25.6 GiB | 20260505 15:02:11.967 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 15:02:11.967 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 15:02:11.967 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 15:02:11.967 | INFO | High Water Breakdown: | 20260505 15:02:11.968 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 15:02:11.968 | INFO | ------------------------------------------ | 20260505 15:02:11.968 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:02:11.968 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 15:02:11.968 | INFO | Total mem usage: 15.7 GiB | 20260505 15:02:11.968 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 15:02:11.968 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 15:02:11.968 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 15:02:11.968 | INFO | High Water Breakdown: | 20260505 15:02:11.968 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 15:02:11.968 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 15:02:11.968 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 15:02:11.968 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 15:02:11.968 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 15:02:11.968 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 15:02:11.968 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 15:02:11.968 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 15:02:11.968 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 15:02:11.968 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:02:11.968 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 15:02:11.968 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:02:11.968 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 15:02:11.968 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 15:02:11.968 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 15:02:11.968 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 15:02:11.968 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:02:11.968 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 15:02:11.968 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:02:11.968 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:02:11.968 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:02:11.968 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 15:02:11.968 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 15:02:11.968 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:02:11.968 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 15:02:11.968 | INFO | ------------------------------------------ | 20260505 15:02:11.968 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:02:11.968 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 15:02:11.968 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:02:11.968 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:02:11.968 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.60e+01, 1.36e+01 / 1.61e+00 | 20260505 15:02:11.968 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 1.01e-01, 1.05e-02 / 5.84e-03 | 20260505 15:02:11.968 | INFO | PerfReport: 11.79%, Baseline -- 3.74e+00 / 4.51e+00, 5.76e-01 / 1.62e-01 | 20260505 15:02:11.968 | INFO | PerfReport: 3.29%, Histogram -- 1.05e+00 / 1.50e+00, 1.94e-01 / 9.69e-02 | 20260505 15:02:11.968 | INFO | PerfReport: 15.97%, DME -- 4.06e+01 / 4.54e+01, 4.55e+00 / 1.84e+00 | 20260505 15:02:11.968 | INFO | PerfReport: 56.52%, FrameLabeling -- 1.80e+01 / 1.97e+01, 1.40e+00 / 1.85e+00 | 20260505 15:02:11.968 | INFO | PerfReport: 7.41%, PulseAccumulating -- 2.35e+00 / 2.90e+00, 4.46e-01 / 1.87e-01 | 20260505 15:02:11.968 | INFO | PerfReport: 3.50%, Metrics -- 1.11e+00 / 1.42e+00, 3.09e-01 / 1.38e-01 | 20260505 15:02:11.968 | INFO | PerfReport: 1.23%, PulseSerializing -- 3.90e-01 / 4.39e-01, 6.99e-02 / 3.17e-02 | 20260505 15:02:11.968 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 15:02:11.968 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:02:11.968 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:02:11.968 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260505 15:02:11.968 | INFO | PerfReport: 22.49%, Upload -- 2.26e+01 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260505 15:02:11.968 | INFO | PerfReport: 22.01%, Download -- 2.21e+01 / 0.00e+00, 7.94e+00 / 0.00e+00 | 20260505 15:02:11.968 | INFO | PerfReport: 31.59%, Compute -- 3.18e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 15:02:11.968 | INFO | PerfReport: 2.28%, LockUpload -- 2.29e+00 / 0.00e+00, 7.84e+00 / 0.00e+00 | 20260505 15:02:11.968 | INFO | PerfReport: 1.38%, LockDownload -- 1.39e+00 / 0.00e+00, 2.95e+00 / 0.00e+00 | 20260505 15:02:11.968 | INFO | PerfReport: 20.23%, LockCompute -- 2.03e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260505 15:02:11.968 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.27e-02 / 0.00e+00 | 20260505 15:02:12.100 | INFO | Starting a round of Baz IO | 20260505 15:02:15.338 | INFO | Populating RTMetrics report | 20260505 15:02:15.339 | INFO | RTMetrics report populated | 20260505 15:02:15.339 | INFO | Opening tmp RTMetrics json file | 20260505 15:02:15.354 | INFO | Serializing RTMetrics to json | 20260505 15:02:15.373 | INFO | Serialization done, swapping tmp file now... | 20260505 15:02:15.383 | INFO | RTMetrics done | 20260505 15:02:15.437 | INFO | Finished a round of Baz IO: Wrote 2.23 GiB in 3.336 seconds | 20260505 15:02:15.437 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.58% / 0.00% / 77.42% | 20260505 15:02:15.437 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.489664% / 3.97364e-06% / 7.54992e-05% | 20260505 15:02:15.437 | INFO | Baz IO overall: | 20260505 15:02:15.437 | INFO | BytesWritten : 2458136381300 | 20260505 15:02:15.437 | INFO | NumEvents : 1168174099749 | 20260505 15:02:17.106 | INFO | Analyzing chunk frames = [4128768,4129280) | 20260505 15:02:17.107 | INFO | MemInfo: { | 20260505 15:02:17.107 | INFO | "gpu" : " 17 GB", | 20260505 15:02:17.107 | INFO | "huge" : "", | 20260505 15:02:17.107 | INFO | "huge_shared" : " 27 GB", | 20260505 15:02:17.107 | INFO | "malloc" : " 14 GB", | 20260505 15:02:17.107 | INFO | "pinned" : " 29 GB" | 20260505 15:02:17.107 | INFO | } | 20260505 15:02:17.107 | INFO | Perf report for the last 5.14008 seconds: | 20260505 15:02:17.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:02:17.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:17.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:17.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:17.107 | INFO | ANALYSIS : 64.5%, 3.0, 100.2ms, 0 | 20260505 15:02:17.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:17.107 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260505 15:02:17.107 | INFO | PRE_HQ : 60.5%, 1.8, 57.0ms, 0 | 20260505 15:02:17.107 | INFO | BAZWRITER : 64.2%, 3.9, 129.3ms, 0 | 20260505 15:02:17.107 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 15:02:17.107 | INFO | Uploaded 36GiB to GPU in 2.17851s (16.525GiB/s) | 20260505 15:02:17.107 | INFO | Downloaded 38GiB to GPU in 2.28101s (16.6593GiB/s) | 20260505 15:02:22.257 | INFO | Analyzing chunk frames = [4129280,4129792) | 20260505 15:02:22.257 | INFO | MemInfo: { | 20260505 15:02:22.257 | INFO | "gpu" : " 17 GB", | 20260505 15:02:22.257 | INFO | "huge" : "", | 20260505 15:02:22.257 | INFO | "huge_shared" : " 27 GB", | 20260505 15:02:22.257 | INFO | "malloc" : " 14 GB", | 20260505 15:02:22.257 | INFO | "pinned" : " 29 GB" | 20260505 15:02:22.257 | INFO | } | 20260505 15:02:22.257 | INFO | Perf report for the last 5.15062 seconds: | 20260505 15:02:22.257 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:02:22.257 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:22.257 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:22.257 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:22.257 | INFO | ANALYSIS : 63.2%, 3.0, 98.8ms, 0 | 20260505 15:02:22.257 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:22.257 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:22.257 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 15:02:22.257 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:22.257 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:22.257 | INFO | Uploaded 36GiB to GPU in 2.19285s (16.417GiB/s) | 20260505 15:02:22.257 | INFO | Downloaded 34GiB to GPU in 2.03299s (16.7241GiB/s) | 20260505 15:02:27.393 | INFO | Analyzing chunk frames = [4129792,4130304) | 20260505 15:02:27.393 | INFO | MemInfo: { | 20260505 15:02:27.393 | INFO | "gpu" : " 17 GB", | 20260505 15:02:27.393 | INFO | "huge" : "", | 20260505 15:02:27.393 | INFO | "huge_shared" : " 27 GB", | 20260505 15:02:27.393 | INFO | "malloc" : " 14 GB", | 20260505 15:02:27.393 | INFO | "pinned" : " 29 GB" | 20260505 15:02:27.393 | INFO | } | 20260505 15:02:27.393 | INFO | Perf report for the last 5.13542 seconds: | 20260505 15:02:27.393 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:02:27.393 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:27.393 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:27.393 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:27.393 | INFO | ANALYSIS : 62.9%, 3.0, 97.8ms, 0 | 20260505 15:02:27.393 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:27.393 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:27.393 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:02:27.393 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:27.393 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:27.393 | INFO | Uploaded 36GiB to GPU in 2.2039s (16.3347GiB/s) | 20260505 15:02:27.393 | INFO | Downloaded 34GiB to GPU in 2.03733s (16.6885GiB/s) | 20260505 15:02:32.523 | INFO | Analyzing chunk frames = [4130304,4130816) | 20260505 15:02:32.523 | INFO | MemInfo: { | 20260505 15:02:32.523 | INFO | "gpu" : " 17 GB", | 20260505 15:02:32.523 | INFO | "huge" : "", | 20260505 15:02:32.523 | INFO | "huge_shared" : " 27 GB", | 20260505 15:02:32.523 | INFO | "malloc" : " 14 GB", | 20260505 15:02:32.523 | INFO | "pinned" : " 29 GB" | 20260505 15:02:32.523 | INFO | } | 20260505 15:02:32.523 | INFO | Perf report for the last 5.13056 seconds: | 20260505 15:02:32.523 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:02:32.523 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:32.523 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:32.523 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:32.523 | INFO | ANALYSIS : 62.7%, 3.0, 97.3ms, 0 | 20260505 15:02:32.523 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:32.523 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:32.523 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 15:02:32.523 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:32.523 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:32.523 | INFO | Uploaded 36GiB to GPU in 2.18822s (16.4518GiB/s) | 20260505 15:02:32.524 | INFO | Downloaded 34GiB to GPU in 2.12699s (15.985GiB/s) | 20260505 15:02:37.658 | INFO | Analyzing chunk frames = [4130816,4131328) | 20260505 15:02:37.658 | INFO | MemInfo: { | 20260505 15:02:37.658 | INFO | "gpu" : " 17 GB", | 20260505 15:02:37.658 | INFO | "huge" : "", | 20260505 15:02:37.658 | INFO | "huge_shared" : " 27 GB", | 20260505 15:02:37.658 | INFO | "malloc" : " 14 GB", | 20260505 15:02:37.658 | INFO | "pinned" : " 29 GB" | 20260505 15:02:37.658 | INFO | } | 20260505 15:02:37.658 | INFO | Perf report for the last 5.13501 seconds: | 20260505 15:02:37.658 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:02:37.658 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:37.658 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:37.658 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:37.658 | INFO | ANALYSIS : 62.9%, 3.0, 98.0ms, 0 | 20260505 15:02:37.658 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:37.658 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:37.658 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 15:02:37.658 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:37.658 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:37.658 | INFO | Uploaded 36GiB to GPU in 2.16238s (16.6483GiB/s) | 20260505 15:02:37.659 | INFO | Downloaded 34GiB to GPU in 2.12527s (15.9979GiB/s) | 20260505 15:02:42.810 | INFO | Analyzing chunk frames = [4131328,4131840) | 20260505 15:02:42.810 | INFO | MemInfo: { | 20260505 15:02:42.810 | INFO | "gpu" : " 17 GB", | 20260505 15:02:42.810 | INFO | "huge" : "", | 20260505 15:02:42.810 | INFO | "huge_shared" : " 27 GB", | 20260505 15:02:42.810 | INFO | "malloc" : " 14 GB", | 20260505 15:02:42.810 | INFO | "pinned" : " 29 GB" | 20260505 15:02:42.810 | INFO | } | 20260505 15:02:42.810 | INFO | Perf report for the last 5.15218 seconds: | 20260505 15:02:42.810 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:02:42.810 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:42.810 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:42.810 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:42.810 | INFO | ANALYSIS : 62.9%, 3.0, 98.5ms, 0 | 20260505 15:02:42.810 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:42.810 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:42.811 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 15:02:42.811 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:42.811 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:42.811 | INFO | Uploaded 36GiB to GPU in 2.17119s (16.5808GiB/s) | 20260505 15:02:42.811 | INFO | Downloaded 34GiB to GPU in 2.10926s (16.1194GiB/s) | 20260505 15:02:47.954 | INFO | Analyzing chunk frames = [4131840,4132352) | 20260505 15:02:47.954 | INFO | MemInfo: { | 20260505 15:02:47.954 | INFO | "gpu" : " 17 GB", | 20260505 15:02:47.954 | INFO | "huge" : "", | 20260505 15:02:47.954 | INFO | "huge_shared" : " 27 GB", | 20260505 15:02:47.954 | INFO | "malloc" : " 14 GB", | 20260505 15:02:47.954 | INFO | "pinned" : " 29 GB" | 20260505 15:02:47.954 | INFO | } | 20260505 15:02:47.954 | INFO | Perf report for the last 5.14408 seconds: | 20260505 15:02:47.954 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:02:47.955 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:47.955 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:47.955 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:47.955 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 15:02:47.955 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:47.955 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:47.955 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 15:02:47.955 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:47.955 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:47.955 | INFO | Uploaded 36GiB to GPU in 2.11598s (17.0134GiB/s) | 20260505 15:02:47.955 | INFO | Downloaded 34GiB to GPU in 2.18121s (15.5877GiB/s) | 20260505 15:02:53.094 | INFO | Analyzing chunk frames = [4132352,4132864) | 20260505 15:02:53.094 | INFO | MemInfo: { | 20260505 15:02:53.094 | INFO | "gpu" : " 17 GB", | 20260505 15:02:53.094 | INFO | "huge" : "", | 20260505 15:02:53.094 | INFO | "huge_shared" : " 27 GB", | 20260505 15:02:53.094 | INFO | "malloc" : " 14 GB", | 20260505 15:02:53.094 | INFO | "pinned" : " 29 GB" | 20260505 15:02:53.094 | INFO | } | 20260505 15:02:53.094 | INFO | Perf report for the last 5.13952 seconds: | 20260505 15:02:53.094 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:02:53.094 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:53.094 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:53.094 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:53.094 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 15:02:53.094 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:53.094 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:53.094 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 15:02:53.094 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:53.094 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:53.094 | INFO | Uploaded 36GiB to GPU in 2.15226s (16.7266GiB/s) | 20260505 15:02:53.094 | INFO | Downloaded 34GiB to GPU in 2.03367s (16.7186GiB/s) | 20260505 15:02:53.226 | INFO | Starting a round of Baz IO | 20260505 15:02:56.437 | INFO | Populating RTMetrics report | 20260505 15:02:56.437 | INFO | RTMetrics report populated | 20260505 15:02:56.437 | INFO | Opening tmp RTMetrics json file | 20260505 15:02:56.456 | INFO | Serializing RTMetrics to json | 20260505 15:02:56.478 | INFO | Serialization done, swapping tmp file now... | 20260505 15:02:56.500 | INFO | RTMetrics done | 20260505 15:02:56.545 | INFO | Finished a round of Baz IO: Wrote 2.23 GiB in 3.319 seconds | 20260505 15:02:56.545 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.58% / 0.00% / 77.42% | 20260505 15:02:56.545 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.487868% / 7.94729e-06% / 9.93411e-05% | 20260505 15:02:56.545 | INFO | Baz IO overall: | 20260505 15:02:56.545 | INFO | BytesWritten : 2460531057733 | 20260505 15:02:56.545 | INFO | NumEvents : 1169301250839 | 20260505 15:02:58.228 | INFO | Analyzing chunk frames = [4132864,4133376) | 20260505 15:02:58.228 | INFO | MemInfo: { | 20260505 15:02:58.228 | INFO | "gpu" : " 17 GB", | 20260505 15:02:58.228 | INFO | "huge" : "", | 20260505 15:02:58.228 | INFO | "huge_shared" : " 27 GB", | 20260505 15:02:58.228 | INFO | "malloc" : " 14 GB", | 20260505 15:02:58.228 | INFO | "pinned" : " 29 GB" | 20260505 15:02:58.228 | INFO | } | 20260505 15:02:58.228 | INFO | Perf report for the last 5.13379 seconds: | 20260505 15:02:58.228 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:02:58.228 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:58.228 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:58.228 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:58.228 | INFO | ANALYSIS : 64.0%, 3.0, 99.4ms, 0 | 20260505 15:02:58.228 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:02:58.228 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260505 15:02:58.228 | INFO | PRE_HQ : 60.9%, 1.8, 56.9ms, 0 | 20260505 15:02:58.228 | INFO | BAZWRITER : 64.0%, 4.0, 130.2ms, 0 | 20260505 15:02:58.228 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 15:02:58.228 | INFO | Uploaded 36GiB to GPU in 2.16216s (16.65GiB/s) | 20260505 15:02:58.228 | INFO | Downloaded 38GiB to GPU in 2.24947s (16.8929GiB/s) | 20260505 15:03:03.360 | INFO | Analyzing chunk frames = [4133376,4133888) | 20260505 15:03:03.360 | INFO | MemInfo: { | 20260505 15:03:03.360 | INFO | "gpu" : " 17 GB", | 20260505 15:03:03.360 | INFO | "huge" : "", | 20260505 15:03:03.360 | INFO | "huge_shared" : " 27 GB", | 20260505 15:03:03.360 | INFO | "malloc" : " 14 GB", | 20260505 15:03:03.360 | INFO | "pinned" : " 29 GB" | 20260505 15:03:03.360 | INFO | } | 20260505 15:03:03.360 | INFO | Perf report for the last 5.13218 seconds: | 20260505 15:03:03.360 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:03:03.360 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:03.360 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:03.360 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:03.360 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 15:03:03.360 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:03.360 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:03.360 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:03:03.360 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:03.360 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:03.360 | INFO | Uploaded 36GiB to GPU in 2.19785s (16.3797GiB/s) | 20260505 15:03:03.361 | INFO | Downloaded 34GiB to GPU in 2.04224s (16.6484GiB/s) | 20260505 15:03:08.509 | INFO | Analyzing chunk frames = [4133888,4134400) | 20260505 15:03:08.509 | INFO | MemInfo: { | 20260505 15:03:08.509 | INFO | "gpu" : " 17 GB", | 20260505 15:03:08.509 | INFO | "huge" : "", | 20260505 15:03:08.509 | INFO | "huge_shared" : " 27 GB", | 20260505 15:03:08.509 | INFO | "malloc" : " 14 GB", | 20260505 15:03:08.509 | INFO | "pinned" : " 29 GB" | 20260505 15:03:08.509 | INFO | } | 20260505 15:03:08.509 | INFO | Perf report for the last 5.14881 seconds: | 20260505 15:03:08.509 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:03:08.509 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:08.509 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:08.509 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:08.509 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 15:03:08.509 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:08.509 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:08.509 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 15:03:08.509 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:08.509 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:08.509 | INFO | Uploaded 36GiB to GPU in 2.22905s (16.1504GiB/s) | 20260505 15:03:08.509 | INFO | Downloaded 34GiB to GPU in 2.10206s (16.1746GiB/s) | 20260505 15:03:13.650 | INFO | Analyzing chunk frames = [4134400,4134912) | 20260505 15:03:13.650 | INFO | MemInfo: { | 20260505 15:03:13.650 | INFO | "gpu" : " 17 GB", | 20260505 15:03:13.650 | INFO | "huge" : "", | 20260505 15:03:13.650 | INFO | "huge_shared" : " 27 GB", | 20260505 15:03:13.650 | INFO | "malloc" : " 14 GB", | 20260505 15:03:13.650 | INFO | "pinned" : " 29 GB" | 20260505 15:03:13.650 | INFO | } | 20260505 15:03:13.650 | INFO | Perf report for the last 5.14076 seconds: | 20260505 15:03:13.650 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:03:13.650 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:13.650 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:13.650 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:13.650 | INFO | ANALYSIS : 62.6%, 3.0, 97.2ms, 0 | 20260505 15:03:13.650 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:13.650 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:13.650 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:03:13.650 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:13.650 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:13.650 | INFO | Uploaded 36GiB to GPU in 2.18226s (16.4967GiB/s) | 20260505 15:03:13.650 | INFO | Downloaded 34GiB to GPU in 2.1329s (15.9408GiB/s) | 20260505 15:03:18.781 | INFO | Analyzing chunk frames = [4134912,4135424) | 20260505 15:03:18.781 | INFO | MemInfo: { | 20260505 15:03:18.781 | INFO | "gpu" : " 17 GB", | 20260505 15:03:18.781 | INFO | "huge" : "", | 20260505 15:03:18.781 | INFO | "huge_shared" : " 27 GB", | 20260505 15:03:18.781 | INFO | "malloc" : " 14 GB", | 20260505 15:03:18.782 | INFO | "pinned" : " 29 GB" | 20260505 15:03:18.782 | INFO | } | 20260505 15:03:18.782 | INFO | Perf report for the last 5.13205 seconds: | 20260505 15:03:18.782 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:03:18.782 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:18.782 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:18.782 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:18.782 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260505 15:03:18.782 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:18.782 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:18.782 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 15:03:18.782 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:18.782 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:18.782 | INFO | Uploaded 36GiB to GPU in 2.1347s (16.8642GiB/s) | 20260505 15:03:18.782 | INFO | Downloaded 34GiB to GPU in 2.05808s (16.5202GiB/s) | 20260505 15:03:23.915 | INFO | Analyzing chunk frames = [4135424,4135936) | 20260505 15:03:23.915 | INFO | MemInfo: { | 20260505 15:03:23.915 | INFO | "gpu" : " 17 GB", | 20260505 15:03:23.915 | INFO | "huge" : "", | 20260505 15:03:23.915 | INFO | "huge_shared" : " 27 GB", | 20260505 15:03:23.915 | INFO | "malloc" : " 14 GB", | 20260505 15:03:23.915 | INFO | "pinned" : " 29 GB" | 20260505 15:03:23.915 | INFO | } | 20260505 15:03:23.915 | INFO | Perf report for the last 5.13378 seconds: | 20260505 15:03:23.916 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:03:23.916 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:23.916 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:23.916 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:23.916 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 15:03:23.916 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:23.916 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:23.916 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:03:23.916 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:23.916 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:23.916 | INFO | Uploaded 36GiB to GPU in 2.19787s (16.3795GiB/s) | 20260505 15:03:23.916 | INFO | Downloaded 34GiB to GPU in 2.11374s (16.0852GiB/s) | 20260505 15:03:29.062 | INFO | Analyzing chunk frames = [4135936,4136448) | 20260505 15:03:29.062 | INFO | MemInfo: { | 20260505 15:03:29.062 | INFO | "gpu" : " 17 GB", | 20260505 15:03:29.062 | INFO | "huge" : "", | 20260505 15:03:29.062 | INFO | "huge_shared" : " 27 GB", | 20260505 15:03:29.062 | INFO | "malloc" : " 14 GB", | 20260505 15:03:29.062 | INFO | "pinned" : " 29 GB" | 20260505 15:03:29.062 | INFO | } | 20260505 15:03:29.062 | INFO | Perf report for the last 5.14672 seconds: | 20260505 15:03:29.062 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:03:29.062 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:29.062 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:29.062 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:29.062 | INFO | ANALYSIS : 63.1%, 3.0, 98.7ms, 0 | 20260505 15:03:29.062 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:29.062 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:29.063 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 15:03:29.063 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:29.063 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:29.063 | INFO | Uploaded 36GiB to GPU in 2.13873s (16.8324GiB/s) | 20260505 15:03:29.063 | INFO | Downloaded 34GiB to GPU in 2.17765s (15.6131GiB/s) | 20260505 15:03:34.206 | INFO | Analyzing chunk frames = [4136448,4136960) | 20260505 15:03:34.206 | INFO | MemInfo: { | 20260505 15:03:34.206 | INFO | "gpu" : " 17 GB", | 20260505 15:03:34.206 | INFO | "huge" : "", | 20260505 15:03:34.206 | INFO | "huge_shared" : " 27 GB", | 20260505 15:03:34.206 | INFO | "malloc" : " 14 GB", | 20260505 15:03:34.206 | INFO | "pinned" : " 29 GB" | 20260505 15:03:34.206 | INFO | } | 20260505 15:03:34.207 | INFO | Perf report for the last 5.1443 seconds: | 20260505 15:03:34.207 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:03:34.207 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:34.207 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:34.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:34.207 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 15:03:34.207 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:34.207 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:34.207 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 15:03:34.207 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:34.207 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:34.207 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 15:03:34.207 | INFO | Total mem usage: 15.3 GiB | 20260505 15:03:34.207 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 15:03:34.207 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.85%) | 20260505 15:03:34.207 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 15:03:34.207 | INFO | High Water Breakdown: | 20260505 15:03:34.207 | INFO | 61% : 8.67 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 15:03:34.207 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 15:03:34.207 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 15:03:34.207 | INFO | ------------------------------------------ | 20260505 15:03:34.207 | INFO | Uploaded 36GiB to GPU in 2.20862s (16.2998GiB/s) | 20260505 15:03:34.207 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:03:34.207 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 15:03:34.207 | INFO | Total mem usage: 26.5 GiB | 20260505 15:03:34.207 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 15:03:34.207 | INFO | Downloaded 34GiB to GPU in 2.0927s (16.2469GiB/s) | 20260505 15:03:34.207 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 15:03:34.207 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 15:03:34.207 | INFO | High Water Breakdown: | 20260505 15:03:34.207 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:03:34.207 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:03:34.207 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:03:34.207 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:03:34.207 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:03:34.207 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:03:34.207 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 15:03:34.207 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 15:03:34.207 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:03:34.207 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 15:03:34.207 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 15:03:34.207 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 15:03:34.207 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 15:03:34.208 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 15:03:34.208 | INFO | ------------------------------------------ | 20260505 15:03:34.208 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:03:34.208 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 15:03:34.208 | INFO | Total mem usage: 25.6 GiB | 20260505 15:03:34.208 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 15:03:34.208 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 15:03:34.208 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 15:03:34.208 | INFO | High Water Breakdown: | 20260505 15:03:34.208 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 15:03:34.208 | INFO | ------------------------------------------ | 20260505 15:03:34.208 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:03:34.208 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 15:03:34.208 | INFO | Total mem usage: 15.7 GiB | 20260505 15:03:34.208 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 15:03:34.208 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 15:03:34.208 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260505 15:03:34.208 | INFO | High Water Breakdown: | 20260505 15:03:34.208 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 15:03:34.208 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 15:03:34.208 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 15:03:34.208 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 15:03:34.208 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 15:03:34.208 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 15:03:34.208 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 15:03:34.208 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 15:03:34.208 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 15:03:34.208 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:03:34.208 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 15:03:34.208 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:03:34.208 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 15:03:34.208 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 15:03:34.208 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 15:03:34.208 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 15:03:34.208 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:03:34.208 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 15:03:34.208 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:03:34.208 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:03:34.208 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:03:34.208 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 15:03:34.208 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 15:03:34.208 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:03:34.208 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 15:03:34.208 | INFO | ------------------------------------------ | 20260505 15:03:34.208 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:03:34.208 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 15:03:34.208 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:03:34.208 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:03:34.208 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.58e+01, 1.37e+01 / 2.08e+00 | 20260505 15:03:34.208 | INFO | PerfReport: 0.29%, AggregateStats -- 9.26e-02 / 1.00e-01, 1.04e-02 / 4.45e-03 | 20260505 15:03:34.208 | INFO | PerfReport: 11.79%, Baseline -- 3.75e+00 / 4.48e+00, 5.82e-01 / 1.64e-01 | 20260505 15:03:34.208 | INFO | PerfReport: 3.29%, Histogram -- 1.05e+00 / 1.51e+00, 1.98e-01 / 9.85e-02 | 20260505 15:03:34.208 | INFO | PerfReport: 15.96%, DME -- 4.06e+01 / 4.52e+01, 4.29e+00 / 2.07e+00 | 20260505 15:03:34.208 | INFO | PerfReport: 56.53%, FrameLabeling -- 1.80e+01 / 1.97e+01, 1.43e+00 / 1.34e+00 | 20260505 15:03:34.208 | INFO | PerfReport: 7.42%, PulseAccumulating -- 2.36e+00 / 2.89e+00, 4.50e-01 / 1.60e-01 | 20260505 15:03:34.208 | INFO | PerfReport: 3.49%, Metrics -- 1.11e+00 / 1.44e+00, 3.10e-01 / 1.57e-01 | 20260505 15:03:34.209 | INFO | PerfReport: 1.22%, PulseSerializing -- 3.89e-01 / 4.40e-01, 6.93e-02 / 3.43e-02 | 20260505 15:03:34.209 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 15:03:34.209 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:03:34.209 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:03:34.209 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260505 15:03:34.209 | INFO | PerfReport: 22.53%, Upload -- 2.27e+01 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260505 15:03:34.209 | INFO | PerfReport: 21.94%, Download -- 2.21e+01 / 0.00e+00, 8.02e+00 / 0.00e+00 | 20260505 15:03:34.209 | INFO | PerfReport: 31.59%, Compute -- 3.18e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 15:03:34.209 | INFO | PerfReport: 2.28%, LockUpload -- 2.29e+00 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260505 15:03:34.209 | INFO | PerfReport: 1.46%, LockDownload -- 1.47e+00 / 0.00e+00, 3.05e+00 / 0.00e+00 | 20260505 15:03:34.209 | INFO | PerfReport: 20.19%, LockCompute -- 2.03e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 15:03:34.209 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.08e-02 / 0.00e+00 | 20260505 15:03:34.336 | INFO | Starting a round of Baz IO | 20260505 15:03:37.560 | INFO | Populating RTMetrics report | 20260505 15:03:37.560 | INFO | RTMetrics report populated | 20260505 15:03:37.560 | INFO | Opening tmp RTMetrics json file | 20260505 15:03:37.573 | INFO | Serializing RTMetrics to json | 20260505 15:03:37.575 | INFO | Serialization done, swapping tmp file now... | 20260505 15:03:37.575 | INFO | RTMetrics done | 20260505 15:03:37.663 | INFO | Finished a round of Baz IO: Wrote 2.227 GiB in 3.327 seconds | 20260505 15:03:37.663 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.58% / 0.00% / 77.42% | 20260505 15:03:37.663 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.485325% / 3.97364e-06% / 0.000103315% | 20260505 15:03:37.663 | INFO | Baz IO overall: | 20260505 15:03:37.664 | INFO | BytesWritten : 2462922611415 | 20260505 15:03:37.664 | INFO | NumEvents : 1170426072666 | 20260505 15:03:39.341 | INFO | Analyzing chunk frames = [4136960,4137472) | 20260505 15:03:39.341 | INFO | MemInfo: { | 20260505 15:03:39.341 | INFO | "gpu" : " 17 GB", | 20260505 15:03:39.341 | INFO | "huge" : "", | 20260505 15:03:39.341 | INFO | "huge_shared" : " 27 GB", | 20260505 15:03:39.341 | INFO | "malloc" : " 14 GB", | 20260505 15:03:39.341 | INFO | "pinned" : " 29 GB" | 20260505 15:03:39.341 | INFO | } | 20260505 15:03:39.341 | INFO | Perf report for the last 5.13405 seconds: | 20260505 15:03:39.341 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:03:39.341 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:39.341 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:39.341 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:39.341 | INFO | ANALYSIS : 64.2%, 3.0, 99.8ms, 0 | 20260505 15:03:39.341 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:39.341 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260505 15:03:39.341 | INFO | PRE_HQ : 61.0%, 1.8, 57.0ms, 0 | 20260505 15:03:39.341 | INFO | BAZWRITER : 64.2%, 3.9, 129.8ms, 0 | 20260505 15:03:39.341 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 15:03:39.341 | INFO | Uploaded 36GiB to GPU in 2.18888s (16.4468GiB/s) | 20260505 15:03:39.341 | INFO | Downloaded 38GiB to GPU in 2.25431s (16.8566GiB/s) | 20260505 15:03:44.471 | INFO | Analyzing chunk frames = [4137472,4137984) | 20260505 15:03:44.471 | INFO | MemInfo: { | 20260505 15:03:44.471 | INFO | "gpu" : " 17 GB", | 20260505 15:03:44.471 | INFO | "huge" : "", | 20260505 15:03:44.471 | INFO | "huge_shared" : " 27 GB", | 20260505 15:03:44.471 | INFO | "malloc" : " 14 GB", | 20260505 15:03:44.471 | INFO | "pinned" : " 29 GB" | 20260505 15:03:44.471 | INFO | } | 20260505 15:03:44.471 | INFO | Perf report for the last 5.13083 seconds: | 20260505 15:03:44.472 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:03:44.472 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:44.472 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:44.472 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:44.472 | INFO | ANALYSIS : 63.4%, 3.0, 98.8ms, 0 | 20260505 15:03:44.472 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:44.472 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:44.472 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:03:44.472 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:44.472 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:44.472 | INFO | Uploaded 36GiB to GPU in 2.19443s (16.4051GiB/s) | 20260505 15:03:44.472 | INFO | Downloaded 34GiB to GPU in 2.03597s (16.6996GiB/s) | 20260505 15:03:49.614 | INFO | Analyzing chunk frames = [4137984,4138496) | 20260505 15:03:49.614 | INFO | MemInfo: { | 20260505 15:03:49.614 | INFO | "gpu" : " 17 GB", | 20260505 15:03:49.614 | INFO | "huge" : "", | 20260505 15:03:49.614 | INFO | "huge_shared" : " 27 GB", | 20260505 15:03:49.614 | INFO | "malloc" : " 14 GB", | 20260505 15:03:49.614 | INFO | "pinned" : " 29 GB" | 20260505 15:03:49.614 | INFO | } | 20260505 15:03:49.614 | INFO | Perf report for the last 5.14281 seconds: | 20260505 15:03:49.614 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:03:49.614 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:49.614 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:49.614 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:49.614 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 15:03:49.614 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:49.614 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:49.614 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:03:49.614 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:49.614 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:49.614 | INFO | Uploaded 36GiB to GPU in 2.23549s (16.1038GiB/s) | 20260505 15:03:49.614 | INFO | Downloaded 34GiB to GPU in 2.09092s (16.2608GiB/s) | 20260505 15:03:54.759 | INFO | Analyzing chunk frames = [4138496,4139008) | 20260505 15:03:54.759 | INFO | MemInfo: { | 20260505 15:03:54.759 | INFO | "gpu" : " 17 GB", | 20260505 15:03:54.759 | INFO | "huge" : "", | 20260505 15:03:54.759 | INFO | "huge_shared" : " 27 GB", | 20260505 15:03:54.759 | INFO | "malloc" : " 14 GB", | 20260505 15:03:54.759 | INFO | "pinned" : " 29 GB" | 20260505 15:03:54.759 | INFO | } | 20260505 15:03:54.759 | INFO | Perf report for the last 5.14504 seconds: | 20260505 15:03:54.759 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:03:54.759 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:54.759 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:54.760 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:54.760 | INFO | ANALYSIS : 62.6%, 3.0, 97.3ms, 0 | 20260505 15:03:54.760 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:54.760 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:54.760 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:03:54.760 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:54.760 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:54.760 | INFO | Uploaded 36GiB to GPU in 2.17805s (16.5286GiB/s) | 20260505 15:03:54.760 | INFO | Downloaded 34GiB to GPU in 2.12798s (15.9776GiB/s) | 20260505 15:03:59.905 | INFO | Analyzing chunk frames = [4139008,4139520) | 20260505 15:03:59.905 | INFO | MemInfo: { | 20260505 15:03:59.905 | INFO | "gpu" : " 17 GB", | 20260505 15:03:59.905 | INFO | "huge" : "", | 20260505 15:03:59.905 | INFO | "huge_shared" : " 27 GB", | 20260505 15:03:59.905 | INFO | "malloc" : " 14 GB", | 20260505 15:03:59.905 | INFO | "pinned" : " 29 GB" | 20260505 15:03:59.905 | INFO | } | 20260505 15:03:59.905 | INFO | Perf report for the last 5.14598 seconds: | 20260505 15:03:59.905 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:03:59.905 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:59.905 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:59.905 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:59.906 | INFO | ANALYSIS : 62.3%, 3.0, 97.4ms, 0 | 20260505 15:03:59.906 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:59.906 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:59.906 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 15:03:59.906 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:59.906 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:03:59.906 | INFO | Uploaded 36GiB to GPU in 2.12812s (16.9164GiB/s) | 20260505 15:03:59.906 | INFO | Downloaded 34GiB to GPU in 2.04931s (16.5909GiB/s) | 20260505 15:04:05.039 | INFO | Analyzing chunk frames = [4139520,4140032) | 20260505 15:04:05.039 | INFO | MemInfo: { | 20260505 15:04:05.039 | INFO | "gpu" : " 17 GB", | 20260505 15:04:05.039 | INFO | "huge" : "", | 20260505 15:04:05.039 | INFO | "huge_shared" : " 27 GB", | 20260505 15:04:05.039 | INFO | "malloc" : " 14 GB", | 20260505 15:04:05.039 | INFO | "pinned" : " 29 GB" | 20260505 15:04:05.039 | INFO | } | 20260505 15:04:05.039 | INFO | Perf report for the last 5.13365 seconds: | 20260505 15:04:05.039 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:04:05.039 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:05.039 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:05.039 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:05.039 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 15:04:05.039 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:05.039 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:05.039 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 15:04:05.039 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:05.039 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:05.039 | INFO | Uploaded 36GiB to GPU in 2.14502s (16.783GiB/s) | 20260505 15:04:05.039 | INFO | Downloaded 34GiB to GPU in 2.03793s (16.6836GiB/s) | 20260505 15:04:10.175 | INFO | Analyzing chunk frames = [4140032,4140544) | 20260505 15:04:10.175 | INFO | MemInfo: { | 20260505 15:04:10.175 | INFO | "gpu" : " 17 GB", | 20260505 15:04:10.175 | INFO | "huge" : "", | 20260505 15:04:10.175 | INFO | "huge_shared" : " 27 GB", | 20260505 15:04:10.175 | INFO | "malloc" : " 14 GB", | 20260505 15:04:10.175 | INFO | "pinned" : " 29 GB" | 20260505 15:04:10.175 | INFO | } | 20260505 15:04:10.175 | INFO | Perf report for the last 5.13634 seconds: | 20260505 15:04:10.175 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:04:10.175 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:10.175 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:10.175 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:10.175 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 15:04:10.175 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:10.175 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:10.175 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 15:04:10.175 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:10.175 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:10.176 | INFO | Uploaded 36GiB to GPU in 2.13248s (16.8817GiB/s) | 20260505 15:04:10.176 | INFO | Downloaded 34GiB to GPU in 2.17611s (15.6242GiB/s) | 20260505 15:04:15.321 | INFO | Analyzing chunk frames = [4140544,4141056) | 20260505 15:04:15.321 | INFO | MemInfo: { | 20260505 15:04:15.321 | INFO | "gpu" : " 17 GB", | 20260505 15:04:15.321 | INFO | "huge" : "", | 20260505 15:04:15.321 | INFO | "huge_shared" : " 27 GB", | 20260505 15:04:15.321 | INFO | "malloc" : " 14 GB", | 20260505 15:04:15.321 | INFO | "pinned" : " 29 GB" | 20260505 15:04:15.321 | INFO | } | 20260505 15:04:15.321 | INFO | Perf report for the last 5.14592 seconds: | 20260505 15:04:15.321 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:04:15.321 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:15.321 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:15.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:15.321 | INFO | ANALYSIS : 63.5%, 3.0, 99.3ms, 0 | 20260505 15:04:15.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:15.321 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:15.322 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 15:04:15.322 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:15.322 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:15.322 | INFO | Uploaded 36GiB to GPU in 2.21473s (16.2548GiB/s) | 20260505 15:04:15.322 | INFO | Downloaded 34GiB to GPU in 2.10455s (16.1555GiB/s) | 20260505 15:04:15.453 | INFO | Starting a round of Baz IO | 20260505 15:04:18.671 | INFO | Populating RTMetrics report | 20260505 15:04:18.671 | INFO | RTMetrics report populated | 20260505 15:04:18.671 | INFO | Opening tmp RTMetrics json file | 20260505 15:04:18.699 | INFO | Serializing RTMetrics to json | 20260505 15:04:18.735 | INFO | Serialization done, swapping tmp file now... | 20260505 15:04:18.757 | INFO | RTMetrics done | 20260505 15:04:18.777 | INFO | Finished a round of Baz IO: Wrote 2.229 GiB in 3.323 seconds | 20260505 15:04:18.777 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.57% / 0.00% / 77.42% | 20260505 15:04:18.777 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.484828% / 3.97364e-06% / 0.000107288% | 20260505 15:04:18.777 | INFO | Baz IO overall: | 20260505 15:04:18.777 | INFO | BytesWritten : 2465315576456 | 20260505 15:04:18.777 | INFO | NumEvents : 1171552083666 | 20260505 15:04:20.454 | INFO | Analyzing chunk frames = [4141056,4141568) | 20260505 15:04:20.454 | INFO | MemInfo: { | 20260505 15:04:20.454 | INFO | "gpu" : " 17 GB", | 20260505 15:04:20.454 | INFO | "huge" : "", | 20260505 15:04:20.454 | INFO | "huge_shared" : " 27 GB", | 20260505 15:04:20.454 | INFO | "malloc" : " 14 GB", | 20260505 15:04:20.455 | INFO | "pinned" : " 29 GB" | 20260505 15:04:20.455 | INFO | } | 20260505 15:04:20.455 | INFO | Perf report for the last 5.13343 seconds: | 20260505 15:04:20.455 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:04:20.455 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:20.455 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:20.455 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:20.455 | INFO | ANALYSIS : 64.1%, 3.0, 99.6ms, 0 | 20260505 15:04:20.455 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:20.455 | INFO | RT_METRICS : 46.5%, 1.0, 24.4ms, 0 | 20260505 15:04:20.455 | INFO | PRE_HQ : 61.2%, 1.8, 57.0ms, 0 | 20260505 15:04:20.455 | INFO | BAZWRITER : 64.1%, 4.0, 130.5ms, 0 | 20260505 15:04:20.455 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 15:04:20.455 | INFO | Uploaded 36GiB to GPU in 2.20033s (16.3612GiB/s) | 20260505 15:04:20.455 | INFO | Downloaded 38GiB to GPU in 2.26377s (16.7862GiB/s) | 20260505 15:04:25.600 | INFO | Analyzing chunk frames = [4141568,4142080) | 20260505 15:04:25.600 | INFO | MemInfo: { | 20260505 15:04:25.600 | INFO | "gpu" : " 17 GB", | 20260505 15:04:25.600 | INFO | "huge" : "", | 20260505 15:04:25.600 | INFO | "huge_shared" : " 27 GB", | 20260505 15:04:25.601 | INFO | "malloc" : " 14 GB", | 20260505 15:04:25.601 | INFO | "pinned" : " 29 GB" | 20260505 15:04:25.601 | INFO | } | 20260505 15:04:25.601 | INFO | Perf report for the last 5.14602 seconds: | 20260505 15:04:25.601 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:04:25.601 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:25.601 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:25.601 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:25.601 | INFO | ANALYSIS : 63.3%, 3.0, 99.0ms, 0 | 20260505 15:04:25.601 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:25.601 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:25.601 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 15:04:25.601 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:25.601 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:25.601 | INFO | Uploaded 36GiB to GPU in 2.2014s (16.3532GiB/s) | 20260505 15:04:25.601 | INFO | Downloaded 34GiB to GPU in 2.05225s (16.5672GiB/s) | 20260505 15:04:30.730 | INFO | Analyzing chunk frames = [4142080,4142592) | 20260505 15:04:30.730 | INFO | MemInfo: { | 20260505 15:04:30.730 | INFO | "gpu" : " 17 GB", | 20260505 15:04:30.730 | INFO | "huge" : "", | 20260505 15:04:30.730 | INFO | "huge_shared" : " 27 GB", | 20260505 15:04:30.730 | INFO | "malloc" : " 14 GB", | 20260505 15:04:30.730 | INFO | "pinned" : " 29 GB" | 20260505 15:04:30.731 | INFO | } | 20260505 15:04:30.731 | INFO | Perf report for the last 5.12987 seconds: | 20260505 15:04:30.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:04:30.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:30.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:30.731 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:30.731 | INFO | ANALYSIS : 62.9%, 3.0, 97.6ms, 0 | 20260505 15:04:30.731 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:30.731 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:30.731 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 15:04:30.731 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:30.731 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:30.731 | INFO | Uploaded 36GiB to GPU in 2.16248s (16.6476GiB/s) | 20260505 15:04:30.731 | INFO | Downloaded 34GiB to GPU in 2.02878s (16.7589GiB/s) | 20260505 15:04:35.873 | INFO | Analyzing chunk frames = [4142592,4143104) | 20260505 15:04:35.873 | INFO | MemInfo: { | 20260505 15:04:35.873 | INFO | "gpu" : " 17 GB", | 20260505 15:04:35.873 | INFO | "huge" : "", | 20260505 15:04:35.873 | INFO | "huge_shared" : " 27 GB", | 20260505 15:04:35.873 | INFO | "malloc" : " 14 GB", | 20260505 15:04:35.873 | INFO | "pinned" : " 29 GB" | 20260505 15:04:35.873 | INFO | } | 20260505 15:04:35.873 | INFO | Perf report for the last 5.14221 seconds: | 20260505 15:04:35.873 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:04:35.873 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:35.873 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:35.873 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:35.873 | INFO | ANALYSIS : 62.8%, 3.0, 97.6ms, 0 | 20260505 15:04:35.873 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:35.873 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:35.873 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 15:04:35.873 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:35.873 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:35.873 | INFO | Uploaded 36GiB to GPU in 2.18508s (16.4754GiB/s) | 20260505 15:04:35.873 | INFO | Downloaded 34GiB to GPU in 2.13483s (15.9264GiB/s) | 20260505 15:04:41.020 | INFO | Analyzing chunk frames = [4143104,4143616) | 20260505 15:04:41.020 | INFO | MemInfo: { | 20260505 15:04:41.020 | INFO | "gpu" : " 17 GB", | 20260505 15:04:41.020 | INFO | "huge" : "", | 20260505 15:04:41.020 | INFO | "huge_shared" : " 27 GB", | 20260505 15:04:41.020 | INFO | "malloc" : " 14 GB", | 20260505 15:04:41.020 | INFO | "pinned" : " 29 GB" | 20260505 15:04:41.020 | INFO | } | 20260505 15:04:41.020 | INFO | Perf report for the last 5.14729 seconds: | 20260505 15:04:41.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:04:41.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:41.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:41.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:41.020 | INFO | ANALYSIS : 62.7%, 3.0, 98.0ms, 0 | 20260505 15:04:41.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:41.020 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:41.020 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:04:41.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:41.020 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:41.020 | INFO | Uploaded 36GiB to GPU in 2.18689s (16.4617GiB/s) | 20260505 15:04:41.020 | INFO | Downloaded 34GiB to GPU in 2.13039s (15.9595GiB/s) | 20260505 15:04:46.155 | INFO | Analyzing chunk frames = [4143616,4144128) | 20260505 15:04:46.155 | INFO | MemInfo: { | 20260505 15:04:46.155 | INFO | "gpu" : " 17 GB", | 20260505 15:04:46.155 | INFO | "huge" : "", | 20260505 15:04:46.155 | INFO | "huge_shared" : " 27 GB", | 20260505 15:04:46.155 | INFO | "malloc" : " 14 GB", | 20260505 15:04:46.155 | INFO | "pinned" : " 29 GB" | 20260505 15:04:46.155 | INFO | } | 20260505 15:04:46.155 | INFO | Perf report for the last 5.13502 seconds: | 20260505 15:04:46.155 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:04:46.155 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:46.155 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:46.155 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:46.155 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 15:04:46.155 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:46.155 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:46.155 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:04:46.155 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:46.155 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:46.155 | INFO | Uploaded 36GiB to GPU in 2.14126s (16.8126GiB/s) | 20260505 15:04:46.155 | INFO | Downloaded 34GiB to GPU in 2.03422s (16.714GiB/s) | 20260505 15:04:51.282 | INFO | Analyzing chunk frames = [4144128,4144640) | 20260505 15:04:51.282 | INFO | MemInfo: { | 20260505 15:04:51.282 | INFO | "gpu" : " 17 GB", | 20260505 15:04:51.282 | INFO | "huge" : "", | 20260505 15:04:51.282 | INFO | "huge_shared" : " 27 GB", | 20260505 15:04:51.282 | INFO | "malloc" : " 14 GB", | 20260505 15:04:51.282 | INFO | "pinned" : " 29 GB" | 20260505 15:04:51.282 | INFO | } | 20260505 15:04:51.283 | INFO | Perf report for the last 5.12736 seconds: | 20260505 15:04:51.283 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:04:51.283 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:51.283 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:51.283 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:51.283 | INFO | ANALYSIS : 63.1%, 3.0, 98.3ms, 0 | 20260505 15:04:51.283 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:51.283 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:51.283 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 15:04:51.283 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:51.283 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:51.283 | INFO | Uploaded 36GiB to GPU in 2.14872s (16.7542GiB/s) | 20260505 15:04:51.283 | INFO | Downloaded 34GiB to GPU in 2.15471s (15.7794GiB/s) | 20260505 15:04:56.421 | INFO | Analyzing chunk frames = [4144640,4145152) | 20260505 15:04:56.421 | INFO | MemInfo: { | 20260505 15:04:56.421 | INFO | "gpu" : " 17 GB", | 20260505 15:04:56.421 | INFO | "huge" : "", | 20260505 15:04:56.421 | INFO | "huge_shared" : " 27 GB", | 20260505 15:04:56.421 | INFO | "malloc" : " 14 GB", | 20260505 15:04:56.421 | INFO | "pinned" : " 29 GB" | 20260505 15:04:56.421 | INFO | } | 20260505 15:04:56.421 | INFO | Perf report for the last 5.13891 seconds: | 20260505 15:04:56.421 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:04:56.421 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:56.421 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:56.421 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:56.422 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 15:04:56.422 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:56.422 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:56.422 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 15:04:56.422 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:56.422 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:04:56.422 | INFO | Uploaded 36GiB to GPU in 2.13618s (16.8525GiB/s) | 20260505 15:04:56.422 | INFO | Downloaded 34GiB to GPU in 2.10427s (16.1576GiB/s) | 20260505 15:04:56.422 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 15:04:56.422 | INFO | Total mem usage: 15.3 GiB | 20260505 15:04:56.422 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 15:04:56.422 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.85%) | 20260505 15:04:56.422 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 15:04:56.422 | INFO | High Water Breakdown: | 20260505 15:04:56.422 | INFO | 61% : 8.67 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 15:04:56.422 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 15:04:56.422 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 15:04:56.422 | INFO | ------------------------------------------ | 20260505 15:04:56.422 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:04:56.422 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 15:04:56.422 | INFO | Total mem usage: 26.5 GiB | 20260505 15:04:56.422 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 15:04:56.422 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 15:04:56.422 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 15:04:56.422 | INFO | High Water Breakdown: | 20260505 15:04:56.422 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:04:56.422 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:04:56.422 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:04:56.422 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:04:56.422 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:04:56.422 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:04:56.422 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 15:04:56.422 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 15:04:56.422 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:04:56.422 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 15:04:56.422 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 15:04:56.422 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 15:04:56.422 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 15:04:56.422 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 15:04:56.422 | INFO | ------------------------------------------ | 20260505 15:04:56.422 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:04:56.422 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 15:04:56.423 | INFO | Total mem usage: 25.6 GiB | 20260505 15:04:56.423 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 15:04:56.423 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 15:04:56.423 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 15:04:56.423 | INFO | High Water Breakdown: | 20260505 15:04:56.423 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 15:04:56.423 | INFO | ------------------------------------------ | 20260505 15:04:56.423 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:04:56.423 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 15:04:56.423 | INFO | Total mem usage: 15.7 GiB | 20260505 15:04:56.423 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 15:04:56.423 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 15:04:56.423 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 15:04:56.423 | INFO | High Water Breakdown: | 20260505 15:04:56.423 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 15:04:56.423 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 15:04:56.423 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 15:04:56.423 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 15:04:56.423 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 15:04:56.423 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 15:04:56.423 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 15:04:56.423 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 15:04:56.423 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 15:04:56.423 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:04:56.423 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 15:04:56.423 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:04:56.423 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 15:04:56.423 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 15:04:56.423 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 15:04:56.423 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 15:04:56.423 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:04:56.423 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 15:04:56.423 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:04:56.423 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:04:56.423 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:04:56.423 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 15:04:56.423 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 15:04:56.423 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:04:56.423 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 15:04:56.423 | INFO | ------------------------------------------ | 20260505 15:04:56.423 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:04:56.423 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 15:04:56.423 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:04:56.423 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:04:56.423 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.57e+01, 1.37e+01 / 1.97e+00 | 20260505 15:04:56.423 | INFO | PerfReport: 0.29%, AggregateStats -- 9.24e-02 / 1.01e-01, 1.07e-02 / 5.19e-03 | 20260505 15:04:56.423 | INFO | PerfReport: 11.80%, Baseline -- 3.75e+00 / 4.48e+00, 5.83e-01 / 2.03e-01 | 20260505 15:04:56.423 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.50e+00, 1.96e-01 / 1.22e-01 | 20260505 15:04:56.423 | INFO | PerfReport: 15.91%, DME -- 4.04e+01 / 4.49e+01, 4.29e+00 / 1.98e+00 | 20260505 15:04:56.423 | INFO | PerfReport: 56.59%, FrameLabeling -- 1.80e+01 / 1.99e+01, 1.45e+00 / 1.74e+00 | 20260505 15:04:56.423 | INFO | PerfReport: 7.40%, PulseAccumulating -- 2.35e+00 / 2.91e+00, 4.46e-01 / 1.64e-01 | 20260505 15:04:56.423 | INFO | PerfReport: 3.49%, Metrics -- 1.11e+00 / 1.42e+00, 3.08e-01 / 1.47e-01 | 20260505 15:04:56.423 | INFO | PerfReport: 1.22%, PulseSerializing -- 3.89e-01 / 4.43e-01, 6.87e-02 / 3.81e-02 | 20260505 15:04:56.423 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 15:04:56.423 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:04:56.423 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:04:56.423 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260505 15:04:56.423 | INFO | PerfReport: 22.53%, Upload -- 2.26e+01 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260505 15:04:56.423 | INFO | PerfReport: 21.89%, Download -- 2.20e+01 / 0.00e+00, 7.93e+00 / 0.00e+00 | 20260505 15:04:56.423 | INFO | PerfReport: 31.62%, Compute -- 3.18e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 15:04:56.423 | INFO | PerfReport: 2.22%, LockUpload -- 2.23e+00 / 0.00e+00, 7.78e+00 / 0.00e+00 | 20260505 15:04:56.423 | INFO | PerfReport: 1.41%, LockDownload -- 1.42e+00 / 0.00e+00, 2.92e+00 / 0.00e+00 | 20260505 15:04:56.423 | INFO | PerfReport: 20.32%, LockCompute -- 2.04e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 15:04:56.423 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.70e-02 / 0.00e+00 | 20260505 15:04:56.561 | INFO | Starting a round of Baz IO | 20260505 15:04:59.775 | INFO | Populating RTMetrics report | 20260505 15:04:59.775 | INFO | RTMetrics report populated | 20260505 15:04:59.775 | INFO | Opening tmp RTMetrics json file | 20260505 15:04:59.791 | INFO | Serializing RTMetrics to json | 20260505 15:04:59.796 | INFO | Serialization done, swapping tmp file now... | 20260505 15:04:59.807 | INFO | RTMetrics done | 20260505 15:04:59.877 | INFO | Finished a round of Baz IO: Wrote 2.228 GiB in 3.316 seconds | 20260505 15:04:59.878 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.57% / 0.00% / 77.43% | 20260505 15:04:59.878 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.484542% / 3.97364e-06% / 9.93411e-05% | 20260505 15:04:59.878 | INFO | Baz IO overall: | 20260505 15:04:59.878 | INFO | BytesWritten : 2467708335980 | 20260505 15:04:59.878 | INFO | NumEvents : 1172678239262 | 20260505 15:05:01.566 | INFO | Analyzing chunk frames = [4145152,4145664) | 20260505 15:05:01.566 | INFO | MemInfo: { | 20260505 15:05:01.566 | INFO | "gpu" : " 17 GB", | 20260505 15:05:01.566 | INFO | "huge" : "", | 20260505 15:05:01.566 | INFO | "huge_shared" : " 27 GB", | 20260505 15:05:01.566 | INFO | "malloc" : " 14 GB", | 20260505 15:05:01.566 | INFO | "pinned" : " 29 GB" | 20260505 15:05:01.566 | INFO | } | 20260505 15:05:01.566 | INFO | Perf report for the last 5.14488 seconds: | 20260505 15:05:01.566 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:05:01.566 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:01.567 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:01.567 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:01.567 | INFO | ANALYSIS : 64.1%, 3.0, 99.6ms, 0 | 20260505 15:05:01.567 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:01.567 | INFO | RT_METRICS : 45.5%, 1.0, 23.9ms, 0 | 20260505 15:05:01.567 | INFO | PRE_HQ : 60.8%, 1.8, 57.3ms, 0 | 20260505 15:05:01.567 | INFO | BAZWRITER : 63.8%, 4.0, 130.2ms, 0 | 20260505 15:05:01.567 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 15:05:01.567 | INFO | Uploaded 36GiB to GPU in 2.17123s (16.5805GiB/s) | 20260505 15:05:01.567 | INFO | Downloaded 38GiB to GPU in 2.28097s (16.6596GiB/s) | 20260505 15:05:06.716 | INFO | Analyzing chunk frames = [4145664,4146176) | 20260505 15:05:06.716 | INFO | MemInfo: { | 20260505 15:05:06.716 | INFO | "gpu" : " 17 GB", | 20260505 15:05:06.716 | INFO | "huge" : "", | 20260505 15:05:06.716 | INFO | "huge_shared" : " 27 GB", | 20260505 15:05:06.716 | INFO | "malloc" : " 14 GB", | 20260505 15:05:06.716 | INFO | "pinned" : " 29 GB" | 20260505 15:05:06.716 | INFO | } | 20260505 15:05:06.716 | INFO | Perf report for the last 5.14974 seconds: | 20260505 15:05:06.716 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:05:06.716 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:06.716 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:06.716 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:06.716 | INFO | ANALYSIS : 63.3%, 3.0, 99.1ms, 0 | 20260505 15:05:06.716 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:06.716 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:06.716 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:05:06.716 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:06.716 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:06.716 | INFO | Uploaded 36GiB to GPU in 2.20226s (16.3469GiB/s) | 20260505 15:05:06.716 | INFO | Downloaded 34GiB to GPU in 2.0608s (16.4985GiB/s) | 20260505 15:05:11.855 | INFO | Analyzing chunk frames = [4146176,4146688) | 20260505 15:05:11.855 | INFO | MemInfo: { | 20260505 15:05:11.855 | INFO | "gpu" : " 17 GB", | 20260505 15:05:11.855 | INFO | "huge" : "", | 20260505 15:05:11.855 | INFO | "huge_shared" : " 27 GB", | 20260505 15:05:11.855 | INFO | "malloc" : " 14 GB", | 20260505 15:05:11.855 | INFO | "pinned" : " 29 GB" | 20260505 15:05:11.855 | INFO | } | 20260505 15:05:11.855 | INFO | Perf report for the last 5.13913 seconds: | 20260505 15:05:11.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:05:11.855 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:11.855 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:11.855 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:11.855 | INFO | ANALYSIS : 62.8%, 3.0, 97.8ms, 0 | 20260505 15:05:11.855 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:11.855 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:11.855 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 15:05:11.855 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:11.855 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:11.856 | INFO | Uploaded 36GiB to GPU in 2.19209s (16.4227GiB/s) | 20260505 15:05:11.856 | INFO | Downloaded 34GiB to GPU in 2.03208s (16.7316GiB/s) | 20260505 15:05:16.984 | INFO | Analyzing chunk frames = [4146688,4147200) | 20260505 15:05:16.984 | INFO | MemInfo: { | 20260505 15:05:16.984 | INFO | "gpu" : " 17 GB", | 20260505 15:05:16.984 | INFO | "huge" : "", | 20260505 15:05:16.984 | INFO | "huge_shared" : " 27 GB", | 20260505 15:05:16.984 | INFO | "malloc" : " 14 GB", | 20260505 15:05:16.984 | INFO | "pinned" : " 29 GB" | 20260505 15:05:16.984 | INFO | } | 20260505 15:05:16.984 | INFO | Perf report for the last 5.12858 seconds: | 20260505 15:05:16.984 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:05:16.984 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:16.984 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:16.984 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:16.984 | INFO | ANALYSIS : 62.7%, 3.0, 97.1ms, 0 | 20260505 15:05:16.984 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:16.984 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:16.984 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 15:05:16.984 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:16.984 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:16.984 | INFO | Uploaded 36GiB to GPU in 2.17492s (16.5523GiB/s) | 20260505 15:05:16.984 | INFO | Downloaded 34GiB to GPU in 2.13231s (15.9452GiB/s) | 20260505 15:05:22.126 | INFO | Analyzing chunk frames = [4147200,4147712) | 20260505 15:05:22.126 | INFO | MemInfo: { | 20260505 15:05:22.126 | INFO | "gpu" : " 17 GB", | 20260505 15:05:22.126 | INFO | "huge" : "", | 20260505 15:05:22.126 | INFO | "huge_shared" : " 27 GB", | 20260505 15:05:22.126 | INFO | "malloc" : " 14 GB", | 20260505 15:05:22.126 | INFO | "pinned" : " 29 GB" | 20260505 15:05:22.126 | INFO | } | 20260505 15:05:22.126 | INFO | Perf report for the last 5.14247 seconds: | 20260505 15:05:22.126 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:05:22.126 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:22.126 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:22.126 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:22.126 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 15:05:22.126 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:22.126 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:22.126 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 15:05:22.126 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:22.126 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:22.127 | INFO | Uploaded 36GiB to GPU in 2.16466s (16.6308GiB/s) | 20260505 15:05:22.127 | INFO | Downloaded 34GiB to GPU in 2.12616s (15.9913GiB/s) | 20260505 15:05:27.273 | INFO | Analyzing chunk frames = [4147712,4148224) | 20260505 15:05:27.273 | INFO | MemInfo: { | 20260505 15:05:27.273 | INFO | "gpu" : " 17 GB", | 20260505 15:05:27.273 | INFO | "huge" : "", | 20260505 15:05:27.273 | INFO | "huge_shared" : " 27 GB", | 20260505 15:05:27.273 | INFO | "malloc" : " 14 GB", | 20260505 15:05:27.273 | INFO | "pinned" : " 29 GB" | 20260505 15:05:27.273 | INFO | } | 20260505 15:05:27.274 | INFO | Perf report for the last 5.14722 seconds: | 20260505 15:05:27.274 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:05:27.274 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:27.274 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:27.274 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:27.274 | INFO | ANALYSIS : 62.6%, 3.0, 98.0ms, 0 | 20260505 15:05:27.274 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:27.274 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:27.274 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 15:05:27.274 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:27.274 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:27.274 | INFO | Uploaded 36GiB to GPU in 2.16809s (16.6045GiB/s) | 20260505 15:05:27.274 | INFO | Downloaded 34GiB to GPU in 2.0383s (16.6805GiB/s) | 20260505 15:05:32.407 | INFO | Analyzing chunk frames = [4148224,4148736) | 20260505 15:05:32.407 | INFO | MemInfo: { | 20260505 15:05:32.407 | INFO | "gpu" : " 17 GB", | 20260505 15:05:32.407 | INFO | "huge" : "", | 20260505 15:05:32.407 | INFO | "huge_shared" : " 27 GB", | 20260505 15:05:32.407 | INFO | "malloc" : " 14 GB", | 20260505 15:05:32.407 | INFO | "pinned" : " 29 GB" | 20260505 15:05:32.407 | INFO | } | 20260505 15:05:32.407 | INFO | Perf report for the last 5.13365 seconds: | 20260505 15:05:32.407 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:05:32.407 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:32.407 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:32.407 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:32.407 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 15:05:32.407 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:32.407 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:32.407 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 15:05:32.407 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:32.407 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:32.408 | INFO | Uploaded 36GiB to GPU in 2.1158s (17.0148GiB/s) | 20260505 15:05:32.408 | INFO | Downloaded 34GiB to GPU in 2.17752s (15.6141GiB/s) | 20260505 15:05:37.543 | INFO | Analyzing chunk frames = [4148736,4149248) | 20260505 15:05:37.543 | INFO | MemInfo: { | 20260505 15:05:37.543 | INFO | "gpu" : " 17 GB", | 20260505 15:05:37.543 | INFO | "huge" : "", | 20260505 15:05:37.543 | INFO | "huge_shared" : " 27 GB", | 20260505 15:05:37.544 | INFO | "malloc" : " 14 GB", | 20260505 15:05:37.544 | INFO | "pinned" : " 29 GB" | 20260505 15:05:37.544 | INFO | } | 20260505 15:05:37.544 | INFO | Perf report for the last 5.13651 seconds: | 20260505 15:05:37.544 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:05:37.544 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:37.544 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:37.544 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:37.544 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 15:05:37.544 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:37.544 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:37.544 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 15:05:37.544 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:37.544 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:37.544 | INFO | Uploaded 36GiB to GPU in 2.22588s (16.1734GiB/s) | 20260505 15:05:37.544 | INFO | Downloaded 34GiB to GPU in 2.1018s (16.1766GiB/s) | 20260505 15:05:37.679 | INFO | Starting a round of Baz IO | 20260505 15:05:40.898 | INFO | Populating RTMetrics report | 20260505 15:05:40.899 | INFO | RTMetrics report populated | 20260505 15:05:40.899 | INFO | Opening tmp RTMetrics json file | 20260505 15:05:40.915 | INFO | Serializing RTMetrics to json | 20260505 15:05:40.932 | INFO | Serialization done, swapping tmp file now... | 20260505 15:05:40.942 | INFO | RTMetrics done | 20260505 15:05:41.000 | INFO | Finished a round of Baz IO: Wrote 2.229 GiB in 3.321 seconds | 20260505 15:05:41.000 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.57% / 0.00% / 77.43% | 20260505 15:05:41.000 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.485126% / 0% / 6.35783e-05% | 20260505 15:05:41.000 | INFO | Baz IO overall: | 20260505 15:05:41.000 | INFO | BytesWritten : 2470101735175 | 20260505 15:05:41.000 | INFO | NumEvents : 1173805136550 | 20260505 15:05:42.680 | INFO | Analyzing chunk frames = [4149248,4149760) | 20260505 15:05:42.680 | INFO | MemInfo: { | 20260505 15:05:42.680 | INFO | "gpu" : " 17 GB", | 20260505 15:05:42.680 | INFO | "huge" : "", | 20260505 15:05:42.680 | INFO | "huge_shared" : " 27 GB", | 20260505 15:05:42.680 | INFO | "malloc" : " 14 GB", | 20260505 15:05:42.680 | INFO | "pinned" : " 29 GB" | 20260505 15:05:42.680 | INFO | } | 20260505 15:05:42.680 | INFO | Perf report for the last 5.13605 seconds: | 20260505 15:05:42.680 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:05:42.680 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:42.680 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:42.680 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:42.680 | INFO | ANALYSIS : 64.2%, 3.0, 99.8ms, 0 | 20260505 15:05:42.680 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:42.680 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260505 15:05:42.680 | INFO | PRE_HQ : 60.7%, 1.8, 57.0ms, 0 | 20260505 15:05:42.680 | INFO | BAZWRITER : 64.0%, 3.9, 129.6ms, 0 | 20260505 15:05:42.680 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 15:05:42.680 | INFO | Uploaded 36GiB to GPU in 2.18548s (16.4724GiB/s) | 20260505 15:05:42.680 | INFO | Downloaded 38GiB to GPU in 2.26059s (16.8098GiB/s) | 20260505 15:05:47.830 | INFO | Analyzing chunk frames = [4149760,4150272) | 20260505 15:05:47.830 | INFO | MemInfo: { | 20260505 15:05:47.830 | INFO | "gpu" : " 17 GB", | 20260505 15:05:47.830 | INFO | "huge" : "", | 20260505 15:05:47.830 | INFO | "huge_shared" : " 27 GB", | 20260505 15:05:47.830 | INFO | "malloc" : " 14 GB", | 20260505 15:05:47.830 | INFO | "pinned" : " 29 GB" | 20260505 15:05:47.831 | INFO | } | 20260505 15:05:47.831 | INFO | Perf report for the last 5.15083 seconds: | 20260505 15:05:47.831 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:05:47.831 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:47.831 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:47.831 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:47.831 | INFO | ANALYSIS : 63.3%, 3.0, 99.1ms, 0 | 20260505 15:05:47.831 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:47.831 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:47.831 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 15:05:47.831 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:47.831 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:47.831 | INFO | Uploaded 36GiB to GPU in 2.20048s (16.3601GiB/s) | 20260505 15:05:47.831 | INFO | Downloaded 34GiB to GPU in 2.04579s (16.6195GiB/s) | 20260505 15:05:52.967 | INFO | Analyzing chunk frames = [4150272,4150784) | 20260505 15:05:52.967 | INFO | MemInfo: { | 20260505 15:05:52.967 | INFO | "gpu" : " 17 GB", | 20260505 15:05:52.967 | INFO | "huge" : "", | 20260505 15:05:52.967 | INFO | "huge_shared" : " 27 GB", | 20260505 15:05:52.967 | INFO | "malloc" : " 14 GB", | 20260505 15:05:52.967 | INFO | "pinned" : " 29 GB" | 20260505 15:05:52.967 | INFO | } | 20260505 15:05:52.967 | INFO | Perf report for the last 5.13632 seconds: | 20260505 15:05:52.967 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:05:52.967 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:52.967 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:52.967 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:52.967 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260505 15:05:52.967 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:52.967 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:52.967 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 15:05:52.967 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:52.967 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:52.967 | INFO | Uploaded 36GiB to GPU in 2.18192s (16.4993GiB/s) | 20260505 15:05:52.967 | INFO | Downloaded 34GiB to GPU in 2.02692s (16.7742GiB/s) | 20260505 15:05:58.099 | INFO | Analyzing chunk frames = [4150784,4151296) | 20260505 15:05:58.099 | INFO | MemInfo: { | 20260505 15:05:58.099 | INFO | "gpu" : " 17 GB", | 20260505 15:05:58.099 | INFO | "huge" : "", | 20260505 15:05:58.099 | INFO | "huge_shared" : " 27 GB", | 20260505 15:05:58.099 | INFO | "malloc" : " 14 GB", | 20260505 15:05:58.099 | INFO | "pinned" : " 29 GB" | 20260505 15:05:58.099 | INFO | } | 20260505 15:05:58.099 | INFO | Perf report for the last 5.13215 seconds: | 20260505 15:05:58.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:05:58.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:58.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:58.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:58.099 | INFO | ANALYSIS : 62.6%, 3.0, 97.0ms, 0 | 20260505 15:05:58.100 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:58.100 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:58.100 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 15:05:58.100 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:58.100 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:05:58.100 | INFO | Uploaded 36GiB to GPU in 2.18051s (16.5099GiB/s) | 20260505 15:05:58.100 | INFO | Downloaded 34GiB to GPU in 2.13228s (15.9454GiB/s) | 20260505 15:06:03.233 | INFO | Analyzing chunk frames = [4151296,4151808) | 20260505 15:06:03.233 | INFO | MemInfo: { | 20260505 15:06:03.233 | INFO | "gpu" : " 17 GB", | 20260505 15:06:03.233 | INFO | "huge" : "", | 20260505 15:06:03.233 | INFO | "huge_shared" : " 27 GB", | 20260505 15:06:03.233 | INFO | "malloc" : " 14 GB", | 20260505 15:06:03.233 | INFO | "pinned" : " 29 GB" | 20260505 15:06:03.234 | INFO | } | 20260505 15:06:03.234 | INFO | Perf report for the last 5.13446 seconds: | 20260505 15:06:03.234 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:06:03.234 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:03.234 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:03.234 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:03.234 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 15:06:03.234 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:03.234 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:03.234 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 15:06:03.234 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:03.234 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:03.234 | INFO | Uploaded 36GiB to GPU in 2.16835s (16.6025GiB/s) | 20260505 15:06:03.234 | INFO | Downloaded 34GiB to GPU in 2.12732s (15.9825GiB/s) | 20260505 15:06:08.376 | INFO | Analyzing chunk frames = [4151808,4152320) | 20260505 15:06:08.376 | INFO | MemInfo: { | 20260505 15:06:08.376 | INFO | "gpu" : " 17 GB", | 20260505 15:06:08.376 | INFO | "huge" : "", | 20260505 15:06:08.376 | INFO | "huge_shared" : " 27 GB", | 20260505 15:06:08.376 | INFO | "malloc" : " 14 GB", | 20260505 15:06:08.376 | INFO | "pinned" : " 29 GB" | 20260505 15:06:08.376 | INFO | } | 20260505 15:06:08.376 | INFO | Perf report for the last 5.14218 seconds: | 20260505 15:06:08.376 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:06:08.376 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:08.376 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:08.376 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:08.376 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 15:06:08.376 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:08.376 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:08.376 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 15:06:08.376 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:08.376 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:08.376 | INFO | Uploaded 36GiB to GPU in 2.16769s (16.6076GiB/s) | 20260505 15:06:08.376 | INFO | Downloaded 34GiB to GPU in 2.10941s (16.1182GiB/s) | 20260505 15:06:13.520 | INFO | Analyzing chunk frames = [4152320,4152832) | 20260505 15:06:13.520 | INFO | MemInfo: { | 20260505 15:06:13.520 | INFO | "gpu" : " 17 GB", | 20260505 15:06:13.520 | INFO | "huge" : "", | 20260505 15:06:13.520 | INFO | "huge_shared" : " 27 GB", | 20260505 15:06:13.520 | INFO | "malloc" : " 14 GB", | 20260505 15:06:13.520 | INFO | "pinned" : " 29 GB" | 20260505 15:06:13.520 | INFO | } | 20260505 15:06:13.520 | INFO | Perf report for the last 5.14449 seconds: | 20260505 15:06:13.520 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:06:13.520 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:13.520 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:13.520 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:13.520 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 15:06:13.520 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:13.520 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:13.520 | INFO | PRE_HQ : 29.1%, 1.0, 15.2ms, 0 | 20260505 15:06:13.520 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:13.521 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:13.521 | INFO | Uploaded 36GiB to GPU in 2.11586s (17.0143GiB/s) | 20260505 15:06:13.521 | INFO | Downloaded 34GiB to GPU in 2.17871s (15.6056GiB/s) | 20260505 15:06:18.671 | INFO | Analyzing chunk frames = [4152832,4153344) | 20260505 15:06:18.671 | INFO | MemInfo: { | 20260505 15:06:18.671 | INFO | "gpu" : " 17 GB", | 20260505 15:06:18.671 | INFO | "huge" : "", | 20260505 15:06:18.671 | INFO | "huge_shared" : " 27 GB", | 20260505 15:06:18.671 | INFO | "malloc" : " 14 GB", | 20260505 15:06:18.671 | INFO | "pinned" : " 29 GB" | 20260505 15:06:18.671 | INFO | } | 20260505 15:06:18.671 | INFO | Perf report for the last 5.15079 seconds: | 20260505 15:06:18.671 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:06:18.671 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:18.671 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:18.671 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:18.671 | INFO | ANALYSIS : 63.6%, 3.0, 99.5ms, 0 | 20260505 15:06:18.671 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:18.671 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:18.671 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 15:06:18.671 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:18.671 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:18.671 | INFO | Uploaded 36GiB to GPU in 2.23698s (16.0931GiB/s) | 20260505 15:06:18.671 | INFO | Downloaded 34GiB to GPU in 2.10311s (16.1665GiB/s) | 20260505 15:06:18.672 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 15:06:18.672 | INFO | Total mem usage: 15.3 GiB | 20260505 15:06:18.672 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 15:06:18.672 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.84%) | 20260505 15:06:18.672 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 15:06:18.672 | INFO | High Water Breakdown: | 20260505 15:06:18.672 | INFO | 61% : 8.67 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 15:06:18.672 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 15:06:18.672 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 15:06:18.672 | INFO | ------------------------------------------ | 20260505 15:06:18.672 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:06:18.672 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 15:06:18.672 | INFO | Total mem usage: 26.5 GiB | 20260505 15:06:18.672 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 15:06:18.672 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 15:06:18.672 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 15:06:18.672 | INFO | High Water Breakdown: | 20260505 15:06:18.672 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:06:18.672 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:06:18.672 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:06:18.672 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:06:18.672 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:06:18.672 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:06:18.672 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 15:06:18.672 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 15:06:18.672 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:06:18.672 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 15:06:18.672 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 15:06:18.672 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 15:06:18.672 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 15:06:18.672 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 15:06:18.672 | INFO | ------------------------------------------ | 20260505 15:06:18.672 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:06:18.672 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 15:06:18.672 | INFO | Total mem usage: 25.6 GiB | 20260505 15:06:18.672 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 15:06:18.672 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 15:06:18.672 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 15:06:18.672 | INFO | High Water Breakdown: | 20260505 15:06:18.672 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 15:06:18.672 | INFO | ------------------------------------------ | 20260505 15:06:18.672 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:06:18.672 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 15:06:18.672 | INFO | Total mem usage: 15.7 GiB | 20260505 15:06:18.672 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 15:06:18.672 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 15:06:18.672 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 15:06:18.672 | INFO | High Water Breakdown: | 20260505 15:06:18.672 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 15:06:18.672 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 15:06:18.672 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 15:06:18.672 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 15:06:18.672 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 15:06:18.672 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 15:06:18.672 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 15:06:18.672 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 15:06:18.672 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 15:06:18.672 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:06:18.673 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 15:06:18.673 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:06:18.673 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 15:06:18.673 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 15:06:18.673 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 15:06:18.673 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 15:06:18.673 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:06:18.673 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 15:06:18.673 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:06:18.673 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:06:18.673 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:06:18.673 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 15:06:18.673 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 15:06:18.673 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:06:18.673 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 15:06:18.673 | INFO | ------------------------------------------ | 20260505 15:06:18.673 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:06:18.673 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 15:06:18.673 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:06:18.673 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:06:18.673 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.60e+01, 1.37e+01 / 1.99e+00 | 20260505 15:06:18.673 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 1.01e-01, 1.08e-02 / 4.75e-03 | 20260505 15:06:18.673 | INFO | PerfReport: 11.76%, Baseline -- 3.73e+00 / 4.50e+00, 5.72e-01 / 1.79e-01 | 20260505 15:06:18.673 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.49e+00, 1.92e-01 / 1.12e-01 | 20260505 15:06:18.673 | INFO | PerfReport: 15.97%, DME -- 4.05e+01 / 4.53e+01, 4.57e+00 / 2.61e+00 | 20260505 15:06:18.673 | INFO | PerfReport: 56.54%, FrameLabeling -- 1.79e+01 / 1.98e+01, 1.38e+00 / 1.73e+00 | 20260505 15:06:18.673 | INFO | PerfReport: 7.43%, PulseAccumulating -- 2.36e+00 / 2.92e+00, 4.49e-01 / 2.07e-01 | 20260505 15:06:18.673 | INFO | PerfReport: 3.49%, Metrics -- 1.11e+00 / 1.42e+00, 3.09e-01 / 1.69e-01 | 20260505 15:06:18.673 | INFO | PerfReport: 1.23%, PulseSerializing -- 3.89e-01 / 4.45e-01, 6.88e-02 / 3.96e-02 | 20260505 15:06:18.673 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 15:06:18.673 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:06:18.673 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:06:18.673 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.56e+01 / 0.00e+00 | 20260505 15:06:18.673 | INFO | PerfReport: 22.55%, Upload -- 2.27e+01 / 0.00e+00, 8.02e+00 / 0.00e+00 | 20260505 15:06:18.673 | INFO | PerfReport: 21.95%, Download -- 2.21e+01 / 0.00e+00, 8.03e+00 / 0.00e+00 | 20260505 15:06:18.673 | INFO | PerfReport: 31.52%, Compute -- 3.17e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 15:06:18.673 | INFO | PerfReport: 2.35%, LockUpload -- 2.36e+00 / 0.00e+00, 8.10e+00 / 0.00e+00 | 20260505 15:06:18.673 | INFO | PerfReport: 1.45%, LockDownload -- 1.46e+00 / 0.00e+00, 3.05e+00 / 0.00e+00 | 20260505 15:06:18.673 | INFO | PerfReport: 20.16%, LockCompute -- 2.03e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 15:06:18.673 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.66e-02 / 0.00e+00 | 20260505 15:06:18.811 | INFO | Starting a round of Baz IO | 20260505 15:06:22.005 | INFO | Populating RTMetrics report | 20260505 15:06:22.005 | INFO | RTMetrics report populated | 20260505 15:06:22.005 | INFO | Opening tmp RTMetrics json file | 20260505 15:06:22.023 | INFO | Serializing RTMetrics to json | 20260505 15:06:22.024 | INFO | Serialization done, swapping tmp file now... | 20260505 15:06:22.025 | INFO | RTMetrics done | 20260505 15:06:22.118 | INFO | Finished a round of Baz IO: Wrote 2.228 GiB in 3.306 seconds | 20260505 15:06:22.118 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.56% / 0.00% / 77.43% | 20260505 15:06:22.118 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.482241% / 3.97364e-06% / 9.13938e-05% | 20260505 15:06:22.118 | INFO | Baz IO overall: | 20260505 15:06:22.118 | INFO | BytesWritten : 2472493562425 | 20260505 15:06:22.118 | INFO | NumEvents : 1174930864860 | 20260505 15:06:23.805 | INFO | Analyzing chunk frames = [4153344,4153856) | 20260505 15:06:23.805 | INFO | MemInfo: { | 20260505 15:06:23.805 | INFO | "gpu" : " 17 GB", | 20260505 15:06:23.805 | INFO | "huge" : "", | 20260505 15:06:23.805 | INFO | "huge_shared" : " 27 GB", | 20260505 15:06:23.805 | INFO | "malloc" : " 14 GB", | 20260505 15:06:23.805 | INFO | "pinned" : " 29 GB" | 20260505 15:06:23.805 | INFO | } | 20260505 15:06:23.805 | INFO | Perf report for the last 5.13396 seconds: | 20260505 15:06:23.805 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:06:23.805 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:23.805 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:23.805 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:23.805 | INFO | ANALYSIS : 63.8%, 3.0, 99.1ms, 0 | 20260505 15:06:23.805 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:23.805 | INFO | RT_METRICS : 45.3%, 1.0, 23.7ms, 0 | 20260505 15:06:23.805 | INFO | PRE_HQ : 60.9%, 1.8, 57.1ms, 0 | 20260505 15:06:23.805 | INFO | BAZWRITER : 63.8%, 4.0, 130.4ms, 0 | 20260505 15:06:23.805 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 15:06:23.806 | INFO | Uploaded 36GiB to GPU in 2.17648s (16.5404GiB/s) | 20260505 15:06:23.806 | INFO | Downloaded 38GiB to GPU in 2.26627s (16.7676GiB/s) | 20260505 15:06:28.933 | INFO | Analyzing chunk frames = [4153856,4154368) | 20260505 15:06:28.933 | INFO | MemInfo: { | 20260505 15:06:28.933 | INFO | "gpu" : " 17 GB", | 20260505 15:06:28.933 | INFO | "huge" : "", | 20260505 15:06:28.933 | INFO | "huge_shared" : " 27 GB", | 20260505 15:06:28.933 | INFO | "malloc" : " 14 GB", | 20260505 15:06:28.933 | INFO | "pinned" : " 29 GB" | 20260505 15:06:28.933 | INFO | } | 20260505 15:06:28.933 | INFO | Perf report for the last 5.12764 seconds: | 20260505 15:06:28.933 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:06:28.933 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:28.933 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:28.933 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:28.933 | INFO | ANALYSIS : 63.4%, 3.0, 98.8ms, 0 | 20260505 15:06:28.933 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:28.933 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:28.933 | INFO | PRE_HQ : 28.1%, 1.0, 14.5ms, 0 | 20260505 15:06:28.933 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:28.933 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:28.933 | INFO | Uploaded 36GiB to GPU in 2.19342s (16.4127GiB/s) | 20260505 15:06:28.933 | INFO | Downloaded 34GiB to GPU in 2.03499s (16.7077GiB/s) | 20260505 15:06:34.083 | INFO | Analyzing chunk frames = [4154368,4154880) | 20260505 15:06:34.083 | INFO | MemInfo: { | 20260505 15:06:34.083 | INFO | "gpu" : " 17 GB", | 20260505 15:06:34.083 | INFO | "huge" : "", | 20260505 15:06:34.083 | INFO | "huge_shared" : " 27 GB", | 20260505 15:06:34.083 | INFO | "malloc" : " 14 GB", | 20260505 15:06:34.083 | INFO | "pinned" : " 29 GB" | 20260505 15:06:34.083 | INFO | } | 20260505 15:06:34.083 | INFO | Perf report for the last 5.15029 seconds: | 20260505 15:06:34.083 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:06:34.083 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:34.083 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:34.083 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:34.083 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 15:06:34.083 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:34.083 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:34.083 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:06:34.083 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:34.083 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:34.083 | INFO | Uploaded 36GiB to GPU in 2.24031s (16.0692GiB/s) | 20260505 15:06:34.083 | INFO | Downloaded 34GiB to GPU in 2.09682s (16.215GiB/s) | 20260505 15:06:39.223 | INFO | Analyzing chunk frames = [4154880,4155392) | 20260505 15:06:39.223 | INFO | MemInfo: { | 20260505 15:06:39.223 | INFO | "gpu" : " 17 GB", | 20260505 15:06:39.223 | INFO | "huge" : "", | 20260505 15:06:39.223 | INFO | "huge_shared" : " 27 GB", | 20260505 15:06:39.224 | INFO | "malloc" : " 14 GB", | 20260505 15:06:39.224 | INFO | "pinned" : " 29 GB" | 20260505 15:06:39.224 | INFO | } | 20260505 15:06:39.224 | INFO | Perf report for the last 5.14064 seconds: | 20260505 15:06:39.224 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:06:39.224 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:39.224 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:39.224 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:39.224 | INFO | ANALYSIS : 62.5%, 3.0, 97.1ms, 0 | 20260505 15:06:39.224 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:39.224 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:39.224 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 15:06:39.224 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:39.224 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:39.224 | INFO | Uploaded 36GiB to GPU in 2.17543s (16.5485GiB/s) | 20260505 15:06:39.224 | INFO | Downloaded 34GiB to GPU in 2.1324s (15.9445GiB/s) | 20260505 15:06:44.354 | INFO | Analyzing chunk frames = [4155392,4155904) | 20260505 15:06:44.354 | INFO | MemInfo: { | 20260505 15:06:44.354 | INFO | "gpu" : " 17 GB", | 20260505 15:06:44.355 | INFO | "huge" : "", | 20260505 15:06:44.355 | INFO | "huge_shared" : " 27 GB", | 20260505 15:06:44.355 | INFO | "malloc" : " 14 GB", | 20260505 15:06:44.355 | INFO | "pinned" : " 29 GB" | 20260505 15:06:44.355 | INFO | } | 20260505 15:06:44.355 | INFO | Perf report for the last 5.13088 seconds: | 20260505 15:06:44.355 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:06:44.355 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:44.355 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:44.355 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:44.355 | INFO | ANALYSIS : 62.9%, 3.0, 98.0ms, 0 | 20260505 15:06:44.355 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:44.355 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:44.355 | INFO | PRE_HQ : 27.9%, 1.0, 14.4ms, 0 | 20260505 15:06:44.355 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:44.355 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:44.355 | INFO | Uploaded 36GiB to GPU in 2.18989s (16.4392GiB/s) | 20260505 15:06:44.355 | INFO | Downloaded 34GiB to GPU in 2.12982s (15.9638GiB/s) | 20260505 15:06:49.491 | INFO | Analyzing chunk frames = [4155904,4156416) | 20260505 15:06:49.491 | INFO | MemInfo: { | 20260505 15:06:49.491 | INFO | "gpu" : " 17 GB", | 20260505 15:06:49.491 | INFO | "huge" : "", | 20260505 15:06:49.491 | INFO | "huge_shared" : " 27 GB", | 20260505 15:06:49.491 | INFO | "malloc" : " 14 GB", | 20260505 15:06:49.491 | INFO | "pinned" : " 29 GB" | 20260505 15:06:49.491 | INFO | } | 20260505 15:06:49.491 | INFO | Perf report for the last 5.13642 seconds: | 20260505 15:06:49.491 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:06:49.491 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:49.491 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:49.491 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:49.491 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 15:06:49.491 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:49.491 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:49.491 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:06:49.491 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:49.491 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:49.491 | INFO | Uploaded 36GiB to GPU in 2.19967s (16.3661GiB/s) | 20260505 15:06:49.491 | INFO | Downloaded 34GiB to GPU in 2.1109s (16.1069GiB/s) | 20260505 15:06:54.636 | INFO | Analyzing chunk frames = [4156416,4156928) | 20260505 15:06:54.636 | INFO | MemInfo: { | 20260505 15:06:54.636 | INFO | "gpu" : " 17 GB", | 20260505 15:06:54.636 | INFO | "huge" : "", | 20260505 15:06:54.636 | INFO | "huge_shared" : " 27 GB", | 20260505 15:06:54.637 | INFO | "malloc" : " 14 GB", | 20260505 15:06:54.637 | INFO | "pinned" : " 29 GB" | 20260505 15:06:54.637 | INFO | } | 20260505 15:06:54.637 | INFO | Perf report for the last 5.14566 seconds: | 20260505 15:06:54.637 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:06:54.637 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:54.637 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:54.637 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:54.637 | INFO | ANALYSIS : 63.4%, 3.0, 99.2ms, 0 | 20260505 15:06:54.637 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:54.637 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:54.637 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 15:06:54.637 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:54.637 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:54.637 | INFO | Uploaded 36GiB to GPU in 2.13785s (16.8393GiB/s) | 20260505 15:06:54.637 | INFO | Downloaded 34GiB to GPU in 2.18445s (15.5646GiB/s) | 20260505 15:06:59.776 | INFO | Analyzing chunk frames = [4156928,4157440) | 20260505 15:06:59.776 | INFO | MemInfo: { | 20260505 15:06:59.776 | INFO | "gpu" : " 17 GB", | 20260505 15:06:59.777 | INFO | "huge" : "", | 20260505 15:06:59.777 | INFO | "huge_shared" : " 27 GB", | 20260505 15:06:59.777 | INFO | "malloc" : " 14 GB", | 20260505 15:06:59.777 | INFO | "pinned" : " 29 GB" | 20260505 15:06:59.777 | INFO | } | 20260505 15:06:59.777 | INFO | Perf report for the last 5.13992 seconds: | 20260505 15:06:59.777 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:06:59.777 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:59.777 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:59.777 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:59.777 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 15:06:59.777 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:59.777 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:59.777 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:06:59.777 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:59.777 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:06:59.777 | INFO | Uploaded 36GiB to GPU in 2.22438s (16.1843GiB/s) | 20260505 15:06:59.777 | INFO | Downloaded 34GiB to GPU in 2.09919s (16.1967GiB/s) | 20260505 15:06:59.908 | INFO | Starting a round of Baz IO | 20260505 15:07:03.125 | INFO | Populating RTMetrics report | 20260505 15:07:03.125 | INFO | RTMetrics report populated | 20260505 15:07:03.125 | INFO | Opening tmp RTMetrics json file | 20260505 15:07:03.141 | INFO | Serializing RTMetrics to json | 20260505 15:07:03.156 | INFO | Serialization done, swapping tmp file now... | 20260505 15:07:03.166 | INFO | RTMetrics done | 20260505 15:07:03.223 | INFO | Finished a round of Baz IO: Wrote 2.228 GiB in 3.315 seconds | 20260505 15:07:03.223 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.56% / 0.00% / 77.44% | 20260505 15:07:03.223 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.480874% / 3.97364e-06% / 7.94729e-05% | 20260505 15:07:03.223 | INFO | Baz IO overall: | 20260505 15:07:03.223 | INFO | BytesWritten : 2474885658443 | 20260505 15:07:03.223 | INFO | NumEvents : 1176057051885 | 20260505 15:07:04.919 | INFO | Analyzing chunk frames = [4157440,4157952) | 20260505 15:07:04.919 | INFO | MemInfo: { | 20260505 15:07:04.919 | INFO | "gpu" : " 17 GB", | 20260505 15:07:04.919 | INFO | "huge" : "", | 20260505 15:07:04.919 | INFO | "huge_shared" : " 27 GB", | 20260505 15:07:04.919 | INFO | "malloc" : " 14 GB", | 20260505 15:07:04.919 | INFO | "pinned" : " 29 GB" | 20260505 15:07:04.919 | INFO | } | 20260505 15:07:04.919 | INFO | Perf report for the last 5.1425 seconds: | 20260505 15:07:04.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:07:04.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:04.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:04.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:04.919 | INFO | ANALYSIS : 64.0%, 3.0, 99.6ms, 0 | 20260505 15:07:04.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:04.919 | INFO | RT_METRICS : 45.5%, 1.0, 23.9ms, 0 | 20260505 15:07:04.919 | INFO | PRE_HQ : 61.0%, 1.8, 57.0ms, 0 | 20260505 15:07:04.919 | INFO | BAZWRITER : 63.7%, 4.0, 129.9ms, 0 | 20260505 15:07:04.919 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 15:07:04.919 | INFO | Uploaded 36GiB to GPU in 2.20379s (16.3355GiB/s) | 20260505 15:07:04.919 | INFO | Downloaded 38GiB to GPU in 2.25976s (16.8159GiB/s) | 20260505 15:07:10.053 | INFO | Analyzing chunk frames = [4157952,4158464) | 20260505 15:07:10.053 | INFO | MemInfo: { | 20260505 15:07:10.053 | INFO | "gpu" : " 17 GB", | 20260505 15:07:10.053 | INFO | "huge" : "", | 20260505 15:07:10.053 | INFO | "huge_shared" : " 27 GB", | 20260505 15:07:10.053 | INFO | "malloc" : " 14 GB", | 20260505 15:07:10.053 | INFO | "pinned" : " 29 GB" | 20260505 15:07:10.053 | INFO | } | 20260505 15:07:10.053 | INFO | Perf report for the last 5.13378 seconds: | 20260505 15:07:10.053 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:07:10.053 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:10.053 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:10.053 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:10.053 | INFO | ANALYSIS : 63.5%, 3.0, 99.0ms, 0 | 20260505 15:07:10.053 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:10.053 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:10.053 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:07:10.053 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:10.053 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:10.053 | INFO | Uploaded 36GiB to GPU in 2.20497s (16.3267GiB/s) | 20260505 15:07:10.053 | INFO | Downloaded 34GiB to GPU in 2.06498s (16.465GiB/s) | 20260505 15:07:15.188 | INFO | Analyzing chunk frames = [4158464,4158976) | 20260505 15:07:15.188 | INFO | MemInfo: { | 20260505 15:07:15.188 | INFO | "gpu" : " 17 GB", | 20260505 15:07:15.188 | INFO | "huge" : "", | 20260505 15:07:15.188 | INFO | "huge_shared" : " 27 GB", | 20260505 15:07:15.188 | INFO | "malloc" : " 14 GB", | 20260505 15:07:15.188 | INFO | "pinned" : " 29 GB" | 20260505 15:07:15.188 | INFO | } | 20260505 15:07:15.188 | INFO | Perf report for the last 5.13529 seconds: | 20260505 15:07:15.188 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:07:15.188 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:15.188 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:15.188 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:15.188 | INFO | ANALYSIS : 63.4%, 3.0, 98.6ms, 0 | 20260505 15:07:15.188 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:15.188 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:15.188 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:07:15.188 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:15.188 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:15.188 | INFO | Uploaded 36GiB to GPU in 2.23741s (16.0901GiB/s) | 20260505 15:07:15.189 | INFO | Downloaded 34GiB to GPU in 2.09865s (16.2009GiB/s) | 20260505 15:07:20.330 | INFO | Analyzing chunk frames = [4158976,4159488) | 20260505 15:07:20.330 | INFO | MemInfo: { | 20260505 15:07:20.331 | INFO | "gpu" : " 17 GB", | 20260505 15:07:20.331 | INFO | "huge" : "", | 20260505 15:07:20.331 | INFO | "huge_shared" : " 27 GB", | 20260505 15:07:20.331 | INFO | "malloc" : " 14 GB", | 20260505 15:07:20.331 | INFO | "pinned" : " 29 GB" | 20260505 15:07:20.331 | INFO | } | 20260505 15:07:20.331 | INFO | Perf report for the last 5.1424 seconds: | 20260505 15:07:20.331 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:07:20.331 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:20.331 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:20.331 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:20.331 | INFO | ANALYSIS : 63.0%, 3.0, 97.9ms, 0 | 20260505 15:07:20.331 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:20.331 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:20.331 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 15:07:20.331 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:20.331 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:20.331 | INFO | Uploaded 36GiB to GPU in 2.2074s (16.3088GiB/s) | 20260505 15:07:20.331 | INFO | Downloaded 34GiB to GPU in 2.13322s (15.9383GiB/s) | 20260505 15:07:25.477 | INFO | Analyzing chunk frames = [4159488,4160000) | 20260505 15:07:25.477 | INFO | MemInfo: { | 20260505 15:07:25.477 | INFO | "gpu" : " 17 GB", | 20260505 15:07:25.477 | INFO | "huge" : "", | 20260505 15:07:25.477 | INFO | "huge_shared" : " 27 GB", | 20260505 15:07:25.477 | INFO | "malloc" : " 14 GB", | 20260505 15:07:25.477 | INFO | "pinned" : " 29 GB" | 20260505 15:07:25.477 | INFO | } | 20260505 15:07:25.477 | INFO | Perf report for the last 5.1467 seconds: | 20260505 15:07:25.477 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:07:25.477 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:25.477 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:25.477 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:25.477 | INFO | ANALYSIS : 62.6%, 3.0, 97.9ms, 0 | 20260505 15:07:25.477 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:25.477 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:25.478 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 15:07:25.478 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:25.478 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:25.478 | INFO | Uploaded 36GiB to GPU in 2.1799s (16.5145GiB/s) | 20260505 15:07:25.478 | INFO | Downloaded 34GiB to GPU in 2.12872s (15.972GiB/s) | 20260505 15:07:30.614 | INFO | Analyzing chunk frames = [4160000,4160512) | 20260505 15:07:30.614 | INFO | MemInfo: { | 20260505 15:07:30.614 | INFO | "gpu" : " 17 GB", | 20260505 15:07:30.614 | INFO | "huge" : "", | 20260505 15:07:30.614 | INFO | "huge_shared" : " 27 GB", | 20260505 15:07:30.614 | INFO | "malloc" : " 14 GB", | 20260505 15:07:30.614 | INFO | "pinned" : " 29 GB" | 20260505 15:07:30.614 | INFO | } | 20260505 15:07:30.614 | INFO | Perf report for the last 5.13648 seconds: | 20260505 15:07:30.614 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:07:30.614 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:30.614 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:30.614 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:30.614 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 15:07:30.614 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:30.614 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:30.614 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:07:30.614 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:30.614 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:30.614 | INFO | Uploaded 36GiB to GPU in 2.13759s (16.8414GiB/s) | 20260505 15:07:30.614 | INFO | Downloaded 34GiB to GPU in 2.03388s (16.7169GiB/s) | 20260505 15:07:35.744 | INFO | Analyzing chunk frames = [4160512,4161024) | 20260505 15:07:35.744 | INFO | MemInfo: { | 20260505 15:07:35.744 | INFO | "gpu" : " 17 GB", | 20260505 15:07:35.744 | INFO | "huge" : "", | 20260505 15:07:35.744 | INFO | "huge_shared" : " 27 GB", | 20260505 15:07:35.744 | INFO | "malloc" : " 14 GB", | 20260505 15:07:35.744 | INFO | "pinned" : " 29 GB" | 20260505 15:07:35.744 | INFO | } | 20260505 15:07:35.744 | INFO | Perf report for the last 5.13033 seconds: | 20260505 15:07:35.744 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:07:35.744 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:35.744 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:35.744 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:35.744 | INFO | ANALYSIS : 63.5%, 3.0, 99.0ms, 0 | 20260505 15:07:35.744 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:35.744 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:35.744 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 15:07:35.744 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:35.744 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:35.744 | INFO | Uploaded 36GiB to GPU in 2.11354s (17.0331GiB/s) | 20260505 15:07:35.744 | INFO | Downloaded 34GiB to GPU in 2.17014s (15.6672GiB/s) | 20260505 15:07:40.890 | INFO | Analyzing chunk frames = [4161024,4161536) | 20260505 15:07:40.890 | INFO | MemInfo: { | 20260505 15:07:40.890 | INFO | "gpu" : " 17 GB", | 20260505 15:07:40.890 | INFO | "huge" : "", | 20260505 15:07:40.890 | INFO | "huge_shared" : " 27 GB", | 20260505 15:07:40.890 | INFO | "malloc" : " 14 GB", | 20260505 15:07:40.891 | INFO | "pinned" : " 29 GB" | 20260505 15:07:40.891 | INFO | } | 20260505 15:07:40.891 | INFO | Perf report for the last 5.14634 seconds: | 20260505 15:07:40.891 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:07:40.891 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:40.891 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:40.891 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:40.891 | INFO | ANALYSIS : 63.5%, 3.0, 99.3ms, 0 | 20260505 15:07:40.891 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:40.891 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:40.891 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 15:07:40.891 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:40.891 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:40.891 | INFO | Uploaded 36GiB to GPU in 2.18407s (16.483GiB/s) | 20260505 15:07:40.891 | INFO | Downloaded 34GiB to GPU in 2.10882s (16.1227GiB/s) | 20260505 15:07:40.891 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 15:07:40.891 | INFO | Total mem usage: 15.3 GiB | 20260505 15:07:40.891 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 15:07:40.891 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.84%) | 20260505 15:07:40.891 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 15:07:40.891 | INFO | High Water Breakdown: | 20260505 15:07:40.891 | INFO | 61% : 8.67 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 15:07:40.891 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 15:07:40.891 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 15:07:40.891 | INFO | ------------------------------------------ | 20260505 15:07:40.891 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:07:40.891 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 15:07:40.891 | INFO | Total mem usage: 26.5 GiB | 20260505 15:07:40.891 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 15:07:40.891 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 15:07:40.891 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 15:07:40.891 | INFO | High Water Breakdown: | 20260505 15:07:40.891 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:07:40.891 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:07:40.891 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:07:40.891 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:07:40.891 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:07:40.891 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:07:40.891 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 15:07:40.891 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 15:07:40.891 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:07:40.891 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 15:07:40.891 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 15:07:40.891 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 15:07:40.891 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 15:07:40.891 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 15:07:40.891 | INFO | ------------------------------------------ | 20260505 15:07:40.891 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:07:40.891 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 15:07:40.891 | INFO | Total mem usage: 25.6 GiB | 20260505 15:07:40.891 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 15:07:40.891 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 15:07:40.891 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 15:07:40.891 | INFO | High Water Breakdown: | 20260505 15:07:40.891 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 15:07:40.891 | INFO | ------------------------------------------ | 20260505 15:07:40.892 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:07:40.892 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 15:07:40.892 | INFO | Total mem usage: 15.7 GiB | 20260505 15:07:40.892 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 15:07:40.892 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 15:07:40.892 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 15:07:40.892 | INFO | High Water Breakdown: | 20260505 15:07:40.892 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 15:07:40.892 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 15:07:40.892 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 15:07:40.892 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 15:07:40.892 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 15:07:40.892 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 15:07:40.892 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 15:07:40.892 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 15:07:40.892 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 15:07:40.892 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:07:40.892 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 15:07:40.892 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:07:40.892 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 15:07:40.892 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 15:07:40.892 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 15:07:40.892 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 15:07:40.892 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:07:40.892 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 15:07:40.892 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:07:40.892 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:07:40.892 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:07:40.892 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 15:07:40.892 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 15:07:40.892 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:07:40.892 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 15:07:40.892 | INFO | ------------------------------------------ | 20260505 15:07:40.892 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:07:40.892 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 15:07:40.892 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:07:40.892 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:07:40.892 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.60e+01, 1.36e+01 / 2.07e+00 | 20260505 15:07:40.892 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 9.91e-02, 1.03e-02 / 4.77e-03 | 20260505 15:07:40.892 | INFO | PerfReport: 11.77%, Baseline -- 3.73e+00 / 4.47e+00, 5.67e-01 / 1.93e-01 | 20260505 15:07:40.892 | INFO | PerfReport: 3.26%, Histogram -- 1.03e+00 / 1.47e+00, 1.88e-01 / 1.25e-01 | 20260505 15:07:40.892 | INFO | PerfReport: 16.02%, DME -- 4.06e+01 / 4.55e+01, 4.68e+00 / 2.04e+00 | 20260505 15:07:40.892 | INFO | PerfReport: 56.47%, FrameLabeling -- 1.79e+01 / 1.96e+01, 1.31e+00 / 1.76e+00 | 20260505 15:07:40.892 | INFO | PerfReport: 7.46%, PulseAccumulating -- 2.36e+00 / 2.94e+00, 4.53e-01 / 1.57e-01 | 20260505 15:07:40.892 | INFO | PerfReport: 3.50%, Metrics -- 1.11e+00 / 1.42e+00, 3.07e-01 / 1.70e-01 | 20260505 15:07:40.892 | INFO | PerfReport: 1.23%, PulseSerializing -- 3.89e-01 / 4.40e-01, 6.87e-02 / 3.67e-02 | 20260505 15:07:40.892 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 15:07:40.892 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:07:40.892 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:07:40.892 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260505 15:07:40.892 | INFO | PerfReport: 22.64%, Upload -- 2.28e+01 / 0.00e+00, 8.11e+00 / 0.00e+00 | 20260505 15:07:40.892 | INFO | PerfReport: 22.02%, Download -- 2.22e+01 / 0.00e+00, 8.07e+00 / 0.00e+00 | 20260505 15:07:40.892 | INFO | PerfReport: 31.48%, Compute -- 3.17e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 15:07:40.892 | INFO | PerfReport: 2.38%, LockUpload -- 2.40e+00 / 0.00e+00, 8.02e+00 / 0.00e+00 | 20260505 15:07:40.892 | INFO | PerfReport: 1.39%, LockDownload -- 1.40e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260505 15:07:40.892 | INFO | PerfReport: 20.07%, LockCompute -- 2.02e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260505 15:07:40.892 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.70e-02 / 0.00e+00 | 20260505 15:07:41.021 | INFO | Starting a round of Baz IO | 20260505 15:07:44.241 | INFO | Populating RTMetrics report | 20260505 15:07:44.241 | INFO | RTMetrics report populated | 20260505 15:07:44.241 | INFO | Opening tmp RTMetrics json file | 20260505 15:07:44.257 | INFO | Serializing RTMetrics to json | 20260505 15:07:44.276 | INFO | Serialization done, swapping tmp file now... | 20260505 15:07:44.291 | INFO | RTMetrics done | 20260505 15:07:44.342 | INFO | Finished a round of Baz IO: Wrote 2.227 GiB in 3.321 seconds | 20260505 15:07:44.343 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.56% / 0.00% / 77.44% | 20260505 15:07:44.343 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.479849% / 3.97364e-06% / 8.74201e-05% | 20260505 15:07:44.343 | INFO | Baz IO overall: | 20260505 15:07:44.343 | INFO | BytesWritten : 2477276777730 | 20260505 15:07:44.343 | INFO | NumEvents : 1177182241485 | 20260505 15:07:46.035 | INFO | Analyzing chunk frames = [4161536,4162048) | 20260505 15:07:46.035 | INFO | MemInfo: { | 20260505 15:07:46.035 | INFO | "gpu" : " 17 GB", | 20260505 15:07:46.035 | INFO | "huge" : "", | 20260505 15:07:46.035 | INFO | "huge_shared" : " 27 GB", | 20260505 15:07:46.035 | INFO | "malloc" : " 14 GB", | 20260505 15:07:46.035 | INFO | "pinned" : " 29 GB" | 20260505 15:07:46.035 | INFO | } | 20260505 15:07:46.035 | INFO | Perf report for the last 5.14443 seconds: | 20260505 15:07:46.035 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:07:46.035 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:46.035 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:46.035 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:46.035 | INFO | ANALYSIS : 64.0%, 3.0, 99.7ms, 0 | 20260505 15:07:46.035 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:46.035 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260505 15:07:46.035 | INFO | PRE_HQ : 60.9%, 1.8, 56.9ms, 0 | 20260505 15:07:46.035 | INFO | BAZWRITER : 63.9%, 3.9, 129.7ms, 0 | 20260505 15:07:46.035 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 15:07:46.035 | INFO | Uploaded 36GiB to GPU in 2.19764s (16.3812GiB/s) | 20260505 15:07:46.035 | INFO | Downloaded 38GiB to GPU in 2.26794s (16.7553GiB/s) | 20260505 15:07:51.171 | INFO | Analyzing chunk frames = [4162048,4162560) | 20260505 15:07:51.171 | INFO | MemInfo: { | 20260505 15:07:51.171 | INFO | "gpu" : " 17 GB", | 20260505 15:07:51.171 | INFO | "huge" : "", | 20260505 15:07:51.171 | INFO | "huge_shared" : " 27 GB", | 20260505 15:07:51.171 | INFO | "malloc" : " 14 GB", | 20260505 15:07:51.172 | INFO | "pinned" : " 29 GB" | 20260505 15:07:51.172 | INFO | } | 20260505 15:07:51.172 | INFO | Perf report for the last 5.13668 seconds: | 20260505 15:07:51.172 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:07:51.172 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:51.172 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:51.172 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:51.172 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 15:07:51.172 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:51.172 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:51.172 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:07:51.172 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:51.172 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:51.172 | INFO | Uploaded 36GiB to GPU in 2.13665s (16.8488GiB/s) | 20260505 15:07:51.172 | INFO | Downloaded 34GiB to GPU in 2.0391s (16.674GiB/s) | 20260505 15:07:56.302 | INFO | Analyzing chunk frames = [4162560,4163072) | 20260505 15:07:56.302 | INFO | MemInfo: { | 20260505 15:07:56.302 | INFO | "gpu" : " 17 GB", | 20260505 15:07:56.302 | INFO | "huge" : "", | 20260505 15:07:56.302 | INFO | "huge_shared" : " 27 GB", | 20260505 15:07:56.302 | INFO | "malloc" : " 14 GB", | 20260505 15:07:56.302 | INFO | "pinned" : " 29 GB" | 20260505 15:07:56.302 | INFO | } | 20260505 15:07:56.302 | INFO | Perf report for the last 5.13048 seconds: | 20260505 15:07:56.302 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:07:56.302 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:56.302 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:56.302 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:56.302 | INFO | ANALYSIS : 63.1%, 3.0, 98.1ms, 0 | 20260505 15:07:56.302 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:56.302 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:56.302 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 15:07:56.302 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:56.302 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:07:56.302 | INFO | Uploaded 36GiB to GPU in 2.18373s (16.4856GiB/s) | 20260505 15:07:56.302 | INFO | Downloaded 34GiB to GPU in 2.05394s (16.5536GiB/s) | 20260505 15:08:01.443 | INFO | Analyzing chunk frames = [4163072,4163584) | 20260505 15:08:01.443 | INFO | MemInfo: { | 20260505 15:08:01.443 | INFO | "gpu" : " 17 GB", | 20260505 15:08:01.443 | INFO | "huge" : "", | 20260505 15:08:01.443 | INFO | "huge_shared" : " 27 GB", | 20260505 15:08:01.443 | INFO | "malloc" : " 14 GB", | 20260505 15:08:01.443 | INFO | "pinned" : " 29 GB" | 20260505 15:08:01.443 | INFO | } | 20260505 15:08:01.443 | INFO | Perf report for the last 5.14114 seconds: | 20260505 15:08:01.443 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:08:01.443 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:01.443 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:01.443 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:01.443 | INFO | ANALYSIS : 62.9%, 3.0, 97.8ms, 0 | 20260505 15:08:01.443 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:01.443 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:01.443 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 15:08:01.444 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:01.444 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:01.444 | INFO | Uploaded 36GiB to GPU in 2.1937s (16.4106GiB/s) | 20260505 15:08:01.444 | INFO | Downloaded 34GiB to GPU in 2.13366s (15.935GiB/s) | 20260505 15:08:06.595 | INFO | Analyzing chunk frames = [4163584,4164096) | 20260505 15:08:06.595 | INFO | MemInfo: { | 20260505 15:08:06.595 | INFO | "gpu" : " 17 GB", | 20260505 15:08:06.595 | INFO | "huge" : "", | 20260505 15:08:06.595 | INFO | "huge_shared" : " 27 GB", | 20260505 15:08:06.595 | INFO | "malloc" : " 14 GB", | 20260505 15:08:06.595 | INFO | "pinned" : " 29 GB" | 20260505 15:08:06.595 | INFO | } | 20260505 15:08:06.595 | INFO | Perf report for the last 5.15149 seconds: | 20260505 15:08:06.595 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:08:06.595 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:06.595 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:06.595 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:06.595 | INFO | ANALYSIS : 62.6%, 3.0, 98.0ms, 0 | 20260505 15:08:06.595 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:06.595 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:06.595 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 15:08:06.595 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:06.595 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:06.595 | INFO | Uploaded 36GiB to GPU in 2.1636s (16.639GiB/s) | 20260505 15:08:06.595 | INFO | Downloaded 34GiB to GPU in 2.12479s (16.0016GiB/s) | 20260505 15:08:11.733 | INFO | Analyzing chunk frames = [4164096,4164608) | 20260505 15:08:11.733 | INFO | MemInfo: { | 20260505 15:08:11.733 | INFO | "gpu" : " 17 GB", | 20260505 15:08:11.733 | INFO | "huge" : "", | 20260505 15:08:11.733 | INFO | "huge_shared" : " 27 GB", | 20260505 15:08:11.733 | INFO | "malloc" : " 14 GB", | 20260505 15:08:11.733 | INFO | "pinned" : " 29 GB" | 20260505 15:08:11.733 | INFO | } | 20260505 15:08:11.733 | INFO | Perf report for the last 5.13789 seconds: | 20260505 15:08:11.733 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:08:11.733 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:11.733 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:11.733 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:11.733 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260505 15:08:11.733 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:11.733 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:11.733 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:08:11.733 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:11.733 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:11.733 | INFO | Uploaded 36GiB to GPU in 2.13942s (16.827GiB/s) | 20260505 15:08:11.733 | INFO | Downloaded 34GiB to GPU in 2.03336s (16.7211GiB/s) | 20260505 15:08:16.867 | INFO | Analyzing chunk frames = [4164608,4165120) | 20260505 15:08:16.867 | INFO | MemInfo: { | 20260505 15:08:16.867 | INFO | "gpu" : " 17 GB", | 20260505 15:08:16.867 | INFO | "huge" : "", | 20260505 15:08:16.867 | INFO | "huge_shared" : " 27 GB", | 20260505 15:08:16.867 | INFO | "malloc" : " 14 GB", | 20260505 15:08:16.867 | INFO | "pinned" : " 29 GB" | 20260505 15:08:16.867 | INFO | } | 20260505 15:08:16.867 | INFO | Perf report for the last 5.1341 seconds: | 20260505 15:08:16.867 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:08:16.867 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:16.867 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:16.867 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:16.867 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 15:08:16.867 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:16.867 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:16.867 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 15:08:16.867 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:16.867 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:16.867 | INFO | Uploaded 36GiB to GPU in 2.11788s (16.9981GiB/s) | 20260505 15:08:16.867 | INFO | Downloaded 34GiB to GPU in 2.17766s (15.6131GiB/s) | 20260505 15:08:22.001 | INFO | Analyzing chunk frames = [4165120,4165632) | 20260505 15:08:22.001 | INFO | MemInfo: { | 20260505 15:08:22.001 | INFO | "gpu" : " 17 GB", | 20260505 15:08:22.002 | INFO | "huge" : "", | 20260505 15:08:22.002 | INFO | "huge_shared" : " 27 GB", | 20260505 15:08:22.002 | INFO | "malloc" : " 14 GB", | 20260505 15:08:22.002 | INFO | "pinned" : " 29 GB" | 20260505 15:08:22.002 | INFO | } | 20260505 15:08:22.002 | INFO | Perf report for the last 5.13483 seconds: | 20260505 15:08:22.002 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:08:22.002 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:22.002 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:22.002 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:22.002 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 15:08:22.002 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:22.002 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:22.002 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 15:08:22.002 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:22.002 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:22.002 | INFO | Uploaded 36GiB to GPU in 2.21993s (16.2167GiB/s) | 20260505 15:08:22.002 | INFO | Downloaded 34GiB to GPU in 2.09577s (16.2232GiB/s) | 20260505 15:08:22.139 | INFO | Starting a round of Baz IO | 20260505 15:08:25.360 | INFO | Populating RTMetrics report | 20260505 15:08:25.361 | INFO | RTMetrics report populated | 20260505 15:08:25.361 | INFO | Opening tmp RTMetrics json file | 20260505 15:08:25.376 | INFO | Serializing RTMetrics to json | 20260505 15:08:25.383 | INFO | Serialization done, swapping tmp file now... | 20260505 15:08:25.400 | INFO | RTMetrics done | 20260505 15:08:25.461 | INFO | Finished a round of Baz IO: Wrote 2.227 GiB in 3.322 seconds | 20260505 15:08:25.461 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.55% / 0.00% / 77.44% | 20260505 15:08:25.461 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.480358% / 3.97364e-06% / 5.5631e-05% | 20260505 15:08:25.461 | INFO | Baz IO overall: | 20260505 15:08:25.461 | INFO | BytesWritten : 2479668399245 | 20260505 15:08:25.461 | INFO | NumEvents : 1178308086799 | 20260505 15:08:27.145 | INFO | Analyzing chunk frames = [4165632,4166144) | 20260505 15:08:27.145 | INFO | MemInfo: { | 20260505 15:08:27.145 | INFO | "gpu" : " 17 GB", | 20260505 15:08:27.145 | INFO | "huge" : "", | 20260505 15:08:27.145 | INFO | "huge_shared" : " 27 GB", | 20260505 15:08:27.145 | INFO | "malloc" : " 14 GB", | 20260505 15:08:27.145 | INFO | "pinned" : " 29 GB" | 20260505 15:08:27.145 | INFO | } | 20260505 15:08:27.145 | INFO | Perf report for the last 5.14307 seconds: | 20260505 15:08:27.145 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:08:27.145 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:27.145 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:27.145 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:27.145 | INFO | ANALYSIS : 64.2%, 3.0, 99.8ms, 0 | 20260505 15:08:27.145 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:27.145 | INFO | RT_METRICS : 45.5%, 1.0, 23.9ms, 0 | 20260505 15:08:27.145 | INFO | PRE_HQ : 60.7%, 1.8, 56.7ms, 0 | 20260505 15:08:27.145 | INFO | BAZWRITER : 64.0%, 3.9, 129.0ms, 0 | 20260505 15:08:27.145 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 15:08:27.145 | INFO | Uploaded 36GiB to GPU in 2.19099s (16.4309GiB/s) | 20260505 15:08:27.145 | INFO | Downloaded 38GiB to GPU in 2.27123s (16.7311GiB/s) | 20260505 15:08:32.284 | INFO | Analyzing chunk frames = [4166144,4166656) | 20260505 15:08:32.284 | INFO | MemInfo: { | 20260505 15:08:32.284 | INFO | "gpu" : " 17 GB", | 20260505 15:08:32.284 | INFO | "huge" : "", | 20260505 15:08:32.284 | INFO | "huge_shared" : " 27 GB", | 20260505 15:08:32.284 | INFO | "malloc" : " 14 GB", | 20260505 15:08:32.284 | INFO | "pinned" : " 29 GB" | 20260505 15:08:32.285 | INFO | } | 20260505 15:08:32.285 | INFO | Perf report for the last 5.13975 seconds: | 20260505 15:08:32.285 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:08:32.285 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:32.285 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:32.285 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:32.285 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 15:08:32.285 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:32.285 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:32.285 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 15:08:32.285 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:32.285 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:32.285 | INFO | Uploaded 36GiB to GPU in 2.20234s (16.3463GiB/s) | 20260505 15:08:32.285 | INFO | Downloaded 34GiB to GPU in 2.06579s (16.4586GiB/s) | 20260505 15:08:37.426 | INFO | Analyzing chunk frames = [4166656,4167168) | 20260505 15:08:37.426 | INFO | MemInfo: { | 20260505 15:08:37.426 | INFO | "gpu" : " 17 GB", | 20260505 15:08:37.426 | INFO | "huge" : "", | 20260505 15:08:37.426 | INFO | "huge_shared" : " 27 GB", | 20260505 15:08:37.426 | INFO | "malloc" : " 14 GB", | 20260505 15:08:37.426 | INFO | "pinned" : " 29 GB" | 20260505 15:08:37.426 | INFO | } | 20260505 15:08:37.426 | INFO | Perf report for the last 5.14182 seconds: | 20260505 15:08:37.427 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:08:37.427 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:37.427 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:37.427 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:37.427 | INFO | ANALYSIS : 62.9%, 3.0, 97.9ms, 0 | 20260505 15:08:37.427 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:37.427 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:37.427 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 15:08:37.427 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:37.427 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:37.427 | INFO | Uploaded 36GiB to GPU in 2.18318s (16.4897GiB/s) | 20260505 15:08:37.427 | INFO | Downloaded 34GiB to GPU in 2.04047s (16.6628GiB/s) | 20260505 15:08:42.564 | INFO | Analyzing chunk frames = [4167168,4167680) | 20260505 15:08:42.564 | INFO | MemInfo: { | 20260505 15:08:42.564 | INFO | "gpu" : " 17 GB", | 20260505 15:08:42.564 | INFO | "huge" : "", | 20260505 15:08:42.564 | INFO | "huge_shared" : " 27 GB", | 20260505 15:08:42.564 | INFO | "malloc" : " 14 GB", | 20260505 15:08:42.564 | INFO | "pinned" : " 29 GB" | 20260505 15:08:42.564 | INFO | } | 20260505 15:08:42.565 | INFO | Perf report for the last 5.1381 seconds: | 20260505 15:08:42.565 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:08:42.565 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:42.565 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:42.565 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:42.565 | INFO | ANALYSIS : 62.6%, 3.0, 97.2ms, 0 | 20260505 15:08:42.565 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:42.565 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:42.565 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 15:08:42.565 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:42.565 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:42.565 | INFO | Uploaded 36GiB to GPU in 2.17026s (16.5878GiB/s) | 20260505 15:08:42.565 | INFO | Downloaded 34GiB to GPU in 2.13253s (15.9435GiB/s) | 20260505 15:08:47.695 | INFO | Analyzing chunk frames = [4167680,4168192) | 20260505 15:08:47.695 | INFO | MemInfo: { | 20260505 15:08:47.695 | INFO | "gpu" : " 17 GB", | 20260505 15:08:47.695 | INFO | "huge" : "", | 20260505 15:08:47.695 | INFO | "huge_shared" : " 27 GB", | 20260505 15:08:47.695 | INFO | "malloc" : " 14 GB", | 20260505 15:08:47.695 | INFO | "pinned" : " 29 GB" | 20260505 15:08:47.695 | INFO | } | 20260505 15:08:47.695 | INFO | Perf report for the last 5.13072 seconds: | 20260505 15:08:47.695 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:08:47.695 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:47.695 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:47.695 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:47.695 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260505 15:08:47.695 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:47.695 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:47.695 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 15:08:47.695 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:47.695 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:47.695 | INFO | Uploaded 36GiB to GPU in 2.18427s (16.4814GiB/s) | 20260505 15:08:47.695 | INFO | Downloaded 34GiB to GPU in 2.13204s (15.9472GiB/s) | 20260505 15:08:52.843 | INFO | Analyzing chunk frames = [4168192,4168704) | 20260505 15:08:52.843 | INFO | MemInfo: { | 20260505 15:08:52.843 | INFO | "gpu" : " 17 GB", | 20260505 15:08:52.843 | INFO | "huge" : "", | 20260505 15:08:52.843 | INFO | "huge_shared" : " 27 GB", | 20260505 15:08:52.843 | INFO | "malloc" : " 14 GB", | 20260505 15:08:52.843 | INFO | "pinned" : " 29 GB" | 20260505 15:08:52.843 | INFO | } | 20260505 15:08:52.843 | INFO | Perf report for the last 5.1482 seconds: | 20260505 15:08:52.843 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:08:52.843 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:52.843 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:52.843 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:52.843 | INFO | ANALYSIS : 62.9%, 3.0, 98.4ms, 0 | 20260505 15:08:52.843 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:52.843 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:52.843 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 15:08:52.843 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:52.843 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:52.844 | INFO | Uploaded 36GiB to GPU in 2.19933s (16.3686GiB/s) | 20260505 15:08:52.844 | INFO | Downloaded 34GiB to GPU in 2.11005s (16.1134GiB/s) | 20260505 15:08:57.984 | INFO | Analyzing chunk frames = [4168704,4169216) | 20260505 15:08:57.984 | INFO | MemInfo: { | 20260505 15:08:57.984 | INFO | "gpu" : " 17 GB", | 20260505 15:08:57.984 | INFO | "huge" : "", | 20260505 15:08:57.984 | INFO | "huge_shared" : " 27 GB", | 20260505 15:08:57.984 | INFO | "malloc" : " 14 GB", | 20260505 15:08:57.984 | INFO | "pinned" : " 29 GB" | 20260505 15:08:57.984 | INFO | } | 20260505 15:08:57.984 | INFO | Perf report for the last 5.14038 seconds: | 20260505 15:08:57.984 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:08:57.984 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:57.984 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:57.984 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:57.984 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 15:08:57.984 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:57.984 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:57.984 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 15:08:57.984 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:57.984 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:08:57.984 | INFO | Uploaded 36GiB to GPU in 2.12742s (16.9219GiB/s) | 20260505 15:08:57.984 | INFO | Downloaded 34GiB to GPU in 2.17343s (15.6434GiB/s) | 20260505 15:09:03.116 | INFO | Analyzing chunk frames = [4169216,4169728) | 20260505 15:09:03.116 | INFO | MemInfo: { | 20260505 15:09:03.116 | INFO | "gpu" : " 17 GB", | 20260505 15:09:03.116 | INFO | "huge" : "", | 20260505 15:09:03.116 | INFO | "huge_shared" : " 27 GB", | 20260505 15:09:03.116 | INFO | "malloc" : " 14 GB", | 20260505 15:09:03.116 | INFO | "pinned" : " 29 GB" | 20260505 15:09:03.116 | INFO | } | 20260505 15:09:03.116 | INFO | Perf report for the last 5.13259 seconds: | 20260505 15:09:03.116 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:09:03.116 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:03.116 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:03.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:03.117 | INFO | ANALYSIS : 63.8%, 3.0, 99.4ms, 0 | 20260505 15:09:03.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:03.117 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:03.117 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 15:09:03.117 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:03.117 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:03.117 | INFO | Uploaded 36GiB to GPU in 2.23164s (16.1317GiB/s) | 20260505 15:09:03.117 | INFO | Downloaded 34GiB to GPU in 2.10721s (16.1351GiB/s) | 20260505 15:09:03.117 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 15:09:03.117 | INFO | Total mem usage: 15.3 GiB | 20260505 15:09:03.117 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 15:09:03.117 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.88%) | 20260505 15:09:03.117 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 15:09:03.117 | INFO | High Water Breakdown: | 20260505 15:09:03.117 | INFO | 61% : 8.67 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 15:09:03.117 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 15:09:03.117 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 15:09:03.117 | INFO | ------------------------------------------ | 20260505 15:09:03.117 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:09:03.117 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 15:09:03.117 | INFO | Total mem usage: 26.5 GiB | 20260505 15:09:03.117 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 15:09:03.117 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 15:09:03.117 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 15:09:03.117 | INFO | High Water Breakdown: | 20260505 15:09:03.117 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:09:03.117 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:09:03.117 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:09:03.117 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:09:03.117 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:09:03.117 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:09:03.117 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 15:09:03.117 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 15:09:03.117 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:09:03.117 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 15:09:03.117 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 15:09:03.117 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 15:09:03.117 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 15:09:03.117 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 15:09:03.117 | INFO | ------------------------------------------ | 20260505 15:09:03.117 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:09:03.117 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 15:09:03.117 | INFO | Total mem usage: 25.6 GiB | 20260505 15:09:03.117 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 15:09:03.117 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 15:09:03.117 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 15:09:03.117 | INFO | High Water Breakdown: | 20260505 15:09:03.117 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 15:09:03.117 | INFO | ------------------------------------------ | 20260505 15:09:03.117 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:09:03.117 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 15:09:03.117 | INFO | Total mem usage: 15.7 GiB | 20260505 15:09:03.117 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 15:09:03.117 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 15:09:03.117 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 15:09:03.117 | INFO | High Water Breakdown: | 20260505 15:09:03.117 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 15:09:03.117 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 15:09:03.117 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 15:09:03.117 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 15:09:03.117 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 15:09:03.117 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 15:09:03.118 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 15:09:03.118 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 15:09:03.118 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 15:09:03.118 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:09:03.118 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 15:09:03.118 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:09:03.118 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 15:09:03.118 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 15:09:03.118 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 15:09:03.118 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 15:09:03.118 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:09:03.118 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 15:09:03.118 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:09:03.118 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:09:03.118 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:09:03.118 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 15:09:03.118 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 15:09:03.118 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:09:03.118 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 15:09:03.118 | INFO | ------------------------------------------ | 20260505 15:09:03.118 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:09:03.118 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 15:09:03.118 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:09:03.118 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:09:03.118 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.61e+01, 1.37e+01 / 2.31e+00 | 20260505 15:09:03.118 | INFO | PerfReport: 0.29%, AggregateStats -- 9.31e-02 / 1.02e-01, 1.18e-02 / 6.18e-03 | 20260505 15:09:03.118 | INFO | PerfReport: 11.76%, Baseline -- 3.74e+00 / 4.45e+00, 5.82e-01 / 2.37e-01 | 20260505 15:09:03.118 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.49e+00, 1.96e-01 / 1.13e-01 | 20260505 15:09:03.118 | INFO | PerfReport: 15.96%, DME -- 4.06e+01 / 4.53e+01, 4.46e+00 / 2.35e+00 | 20260505 15:09:03.118 | INFO | PerfReport: 56.64%, FrameLabeling -- 1.80e+01 / 2.00e+01, 1.48e+00 / 1.94e+00 | 20260505 15:09:03.118 | INFO | PerfReport: 7.37%, PulseAccumulating -- 2.34e+00 / 2.90e+00, 4.46e-01 / 2.20e-01 | 20260505 15:09:03.118 | INFO | PerfReport: 3.47%, Metrics -- 1.10e+00 / 1.41e+00, 3.08e-01 / 1.24e-01 | 20260505 15:09:03.118 | INFO | PerfReport: 1.22%, PulseSerializing -- 3.88e-01 / 4.44e-01, 6.99e-02 / 3.82e-02 | 20260505 15:09:03.118 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 15:09:03.118 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:09:03.118 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:09:03.118 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260505 15:09:03.118 | INFO | PerfReport: 22.55%, Upload -- 2.27e+01 / 0.00e+00, 8.04e+00 / 0.00e+00 | 20260505 15:09:03.118 | INFO | PerfReport: 21.98%, Download -- 2.21e+01 / 0.00e+00, 8.09e+00 / 0.00e+00 | 20260505 15:09:03.118 | INFO | PerfReport: 31.60%, Compute -- 3.18e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 15:09:03.118 | INFO | PerfReport: 2.25%, LockUpload -- 2.27e+00 / 0.00e+00, 7.87e+00 / 0.00e+00 | 20260505 15:09:03.118 | INFO | PerfReport: 1.41%, LockDownload -- 1.42e+00 / 0.00e+00, 2.98e+00 / 0.00e+00 | 20260505 15:09:03.118 | INFO | PerfReport: 20.20%, LockCompute -- 2.03e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 15:09:03.118 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.98e-02 / 0.00e+00 | 20260505 15:09:03.261 | INFO | Starting a round of Baz IO | 20260505 15:09:06.476 | INFO | Populating RTMetrics report | 20260505 15:09:06.476 | INFO | RTMetrics report populated | 20260505 15:09:06.476 | INFO | Opening tmp RTMetrics json file | 20260505 15:09:06.495 | INFO | Serializing RTMetrics to json | 20260505 15:09:06.517 | INFO | Serialization done, swapping tmp file now... | 20260505 15:09:06.537 | INFO | RTMetrics done | 20260505 15:09:06.578 | INFO | Finished a round of Baz IO: Wrote 2.224 GiB in 3.316 seconds | 20260505 15:09:06.578 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.55% / 0.00% / 77.45% | 20260505 15:09:06.578 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.476293% / 7.94729e-06% / 7.94729e-05% | 20260505 15:09:06.578 | INFO | Baz IO overall: | 20260505 15:09:06.578 | INFO | BytesWritten : 2482056049000 | 20260505 15:09:06.578 | INFO | NumEvents : 1179431033648 | 20260505 15:09:08.250 | INFO | Analyzing chunk frames = [4169728,4170240) | 20260505 15:09:08.250 | INFO | MemInfo: { | 20260505 15:09:08.250 | INFO | "gpu" : " 17 GB", | 20260505 15:09:08.250 | INFO | "huge" : "", | 20260505 15:09:08.250 | INFO | "huge_shared" : " 27 GB", | 20260505 15:09:08.250 | INFO | "malloc" : " 14 GB", | 20260505 15:09:08.250 | INFO | "pinned" : " 29 GB" | 20260505 15:09:08.250 | INFO | } | 20260505 15:09:08.250 | INFO | Perf report for the last 5.13367 seconds: | 20260505 15:09:08.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:09:08.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:08.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:08.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:08.250 | INFO | ANALYSIS : 64.3%, 3.0, 99.9ms, 0 | 20260505 15:09:08.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:08.250 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260505 15:09:08.250 | INFO | PRE_HQ : 61.0%, 1.8, 57.5ms, 0 | 20260505 15:09:08.250 | INFO | BAZWRITER : 64.3%, 3.9, 129.9ms, 0 | 20260505 15:09:08.250 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 15:09:08.251 | INFO | Uploaded 36GiB to GPU in 2.18873s (16.4479GiB/s) | 20260505 15:09:08.251 | INFO | Downloaded 38GiB to GPU in 2.26112s (16.8058GiB/s) | 20260505 15:09:13.396 | INFO | Analyzing chunk frames = [4170240,4170752) | 20260505 15:09:13.396 | INFO | MemInfo: { | 20260505 15:09:13.396 | INFO | "gpu" : " 17 GB", | 20260505 15:09:13.396 | INFO | "huge" : "", | 20260505 15:09:13.396 | INFO | "huge_shared" : " 27 GB", | 20260505 15:09:13.396 | INFO | "malloc" : " 14 GB", | 20260505 15:09:13.396 | INFO | "pinned" : " 29 GB" | 20260505 15:09:13.396 | INFO | } | 20260505 15:09:13.396 | INFO | Perf report for the last 5.1462 seconds: | 20260505 15:09:13.396 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:09:13.396 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:13.396 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:13.396 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:13.396 | INFO | ANALYSIS : 63.2%, 3.0, 98.9ms, 0 | 20260505 15:09:13.396 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:13.396 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:13.396 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 15:09:13.396 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:13.397 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:13.397 | INFO | Uploaded 36GiB to GPU in 2.19358s (16.4116GiB/s) | 20260505 15:09:13.397 | INFO | Downloaded 34GiB to GPU in 2.03358s (16.7193GiB/s) | 20260505 15:09:18.542 | INFO | Analyzing chunk frames = [4170752,4171264) | 20260505 15:09:18.542 | INFO | MemInfo: { | 20260505 15:09:18.542 | INFO | "gpu" : " 17 GB", | 20260505 15:09:18.542 | INFO | "huge" : "", | 20260505 15:09:18.542 | INFO | "huge_shared" : " 27 GB", | 20260505 15:09:18.542 | INFO | "malloc" : " 14 GB", | 20260505 15:09:18.542 | INFO | "pinned" : " 29 GB" | 20260505 15:09:18.542 | INFO | } | 20260505 15:09:18.542 | INFO | Perf report for the last 5.14612 seconds: | 20260505 15:09:18.542 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:09:18.542 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:18.542 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:18.542 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:18.542 | INFO | ANALYSIS : 63.0%, 3.0, 98.1ms, 0 | 20260505 15:09:18.542 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:18.543 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:18.543 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 15:09:18.543 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:18.543 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:18.543 | INFO | Uploaded 36GiB to GPU in 2.18585s (16.4696GiB/s) | 20260505 15:09:18.543 | INFO | Downloaded 34GiB to GPU in 2.06469s (16.4673GiB/s) | 20260505 15:09:23.680 | INFO | Analyzing chunk frames = [4171264,4171776) | 20260505 15:09:23.680 | INFO | MemInfo: { | 20260505 15:09:23.680 | INFO | "gpu" : " 17 GB", | 20260505 15:09:23.680 | INFO | "huge" : "", | 20260505 15:09:23.680 | INFO | "huge_shared" : " 27 GB", | 20260505 15:09:23.680 | INFO | "malloc" : " 14 GB", | 20260505 15:09:23.680 | INFO | "pinned" : " 29 GB" | 20260505 15:09:23.680 | INFO | } | 20260505 15:09:23.680 | INFO | Perf report for the last 5.1373 seconds: | 20260505 15:09:23.680 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:09:23.680 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:23.680 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:23.680 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:23.680 | INFO | ANALYSIS : 62.7%, 3.0, 97.4ms, 0 | 20260505 15:09:23.680 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:23.680 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:23.680 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 15:09:23.680 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:23.680 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:23.680 | INFO | Uploaded 36GiB to GPU in 2.16919s (16.5961GiB/s) | 20260505 15:09:23.680 | INFO | Downloaded 34GiB to GPU in 2.14262s (15.8684GiB/s) | 20260505 15:09:28.812 | INFO | Analyzing chunk frames = [4171776,4172288) | 20260505 15:09:28.812 | INFO | MemInfo: { | 20260505 15:09:28.812 | INFO | "gpu" : " 17 GB", | 20260505 15:09:28.812 | INFO | "huge" : "", | 20260505 15:09:28.812 | INFO | "huge_shared" : " 27 GB", | 20260505 15:09:28.812 | INFO | "malloc" : " 14 GB", | 20260505 15:09:28.812 | INFO | "pinned" : " 29 GB" | 20260505 15:09:28.812 | INFO | } | 20260505 15:09:28.812 | INFO | Perf report for the last 5.13254 seconds: | 20260505 15:09:28.812 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:09:28.812 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:28.812 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:28.812 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:28.812 | INFO | ANALYSIS : 62.8%, 3.0, 96.9ms, 0 | 20260505 15:09:28.812 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:28.812 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:28.812 | INFO | PRE_HQ : 28.2%, 1.0, 14.5ms, 0 | 20260505 15:09:28.813 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:28.813 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:28.813 | INFO | Uploaded 36GiB to GPU in 2.17936s (16.5186GiB/s) | 20260505 15:09:28.813 | INFO | Downloaded 34GiB to GPU in 2.1292s (15.9685GiB/s) | 20260505 15:09:33.949 | INFO | Analyzing chunk frames = [4172288,4172800) | 20260505 15:09:33.949 | INFO | MemInfo: { | 20260505 15:09:33.949 | INFO | "gpu" : " 17 GB", | 20260505 15:09:33.949 | INFO | "huge" : "", | 20260505 15:09:33.949 | INFO | "huge_shared" : " 27 GB", | 20260505 15:09:33.949 | INFO | "malloc" : " 14 GB", | 20260505 15:09:33.949 | INFO | "pinned" : " 29 GB" | 20260505 15:09:33.949 | INFO | } | 20260505 15:09:33.949 | INFO | Perf report for the last 5.13674 seconds: | 20260505 15:09:33.949 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:09:33.949 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:33.949 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:33.949 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:33.949 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 15:09:33.949 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:33.949 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:33.949 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 15:09:33.949 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:33.949 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:33.949 | INFO | Uploaded 36GiB to GPU in 2.1412s (16.813GiB/s) | 20260505 15:09:33.949 | INFO | Downloaded 34GiB to GPU in 2.109s (16.1214GiB/s) | 20260505 15:09:39.094 | INFO | Analyzing chunk frames = [4172800,4173312) | 20260505 15:09:39.094 | INFO | MemInfo: { | 20260505 15:09:39.094 | INFO | "gpu" : " 17 GB", | 20260505 15:09:39.094 | INFO | "huge" : "", | 20260505 15:09:39.094 | INFO | "huge_shared" : " 27 GB", | 20260505 15:09:39.094 | INFO | "malloc" : " 14 GB", | 20260505 15:09:39.094 | INFO | "pinned" : " 29 GB" | 20260505 15:09:39.094 | INFO | } | 20260505 15:09:39.094 | INFO | Perf report for the last 5.14524 seconds: | 20260505 15:09:39.094 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:09:39.094 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:39.094 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:39.095 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:39.095 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 15:09:39.095 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:39.095 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:39.095 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 15:09:39.095 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:39.095 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:39.095 | INFO | Uploaded 36GiB to GPU in 2.15462s (16.7083GiB/s) | 20260505 15:09:39.095 | INFO | Downloaded 34GiB to GPU in 2.15319s (15.7905GiB/s) | 20260505 15:09:44.238 | INFO | Analyzing chunk frames = [4173312,4173824) | 20260505 15:09:44.238 | INFO | MemInfo: { | 20260505 15:09:44.238 | INFO | "gpu" : " 17 GB", | 20260505 15:09:44.238 | INFO | "huge" : "", | 20260505 15:09:44.238 | INFO | "huge_shared" : " 27 GB", | 20260505 15:09:44.238 | INFO | "malloc" : " 14 GB", | 20260505 15:09:44.238 | INFO | "pinned" : " 29 GB" | 20260505 15:09:44.238 | INFO | } | 20260505 15:09:44.238 | INFO | Perf report for the last 5.14353 seconds: | 20260505 15:09:44.238 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:09:44.238 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:44.238 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:44.238 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:44.238 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 15:09:44.238 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:44.238 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:44.238 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 15:09:44.238 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:44.238 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:44.238 | INFO | Uploaded 36GiB to GPU in 2.22557s (16.1757GiB/s) | 20260505 15:09:44.238 | INFO | Downloaded 34GiB to GPU in 2.10594s (16.1448GiB/s) | 20260505 15:09:44.385 | INFO | Starting a round of Baz IO | 20260505 15:09:47.584 | INFO | Populating RTMetrics report | 20260505 15:09:47.585 | INFO | RTMetrics report populated | 20260505 15:09:47.585 | INFO | Opening tmp RTMetrics json file | 20260505 15:09:47.601 | INFO | Serializing RTMetrics to json | 20260505 15:09:47.602 | INFO | Serialization done, swapping tmp file now... | 20260505 15:09:47.603 | INFO | RTMetrics done | 20260505 15:09:47.688 | INFO | Finished a round of Baz IO: Wrote 2.226 GiB in 3.303 seconds | 20260505 15:09:47.688 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.55% / 0.00% / 77.45% | 20260505 15:09:47.689 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.477556% / 0% / 8.34465e-05% | 20260505 15:09:47.689 | INFO | Baz IO overall: | 20260505 15:09:47.689 | INFO | BytesWritten : 2484446481795 | 20260505 15:09:47.689 | INFO | NumEvents : 1180556038608 | 20260505 15:09:49.376 | INFO | Analyzing chunk frames = [4173824,4174336) | 20260505 15:09:49.376 | INFO | MemInfo: { | 20260505 15:09:49.376 | INFO | "gpu" : " 17 GB", | 20260505 15:09:49.376 | INFO | "huge" : "", | 20260505 15:09:49.376 | INFO | "huge_shared" : " 27 GB", | 20260505 15:09:49.376 | INFO | "malloc" : " 14 GB", | 20260505 15:09:49.376 | INFO | "pinned" : " 29 GB" | 20260505 15:09:49.376 | INFO | } | 20260505 15:09:49.376 | INFO | Perf report for the last 5.13832 seconds: | 20260505 15:09:49.376 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:09:49.376 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:49.376 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:49.376 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:49.376 | INFO | ANALYSIS : 64.0%, 3.0, 99.6ms, 0 | 20260505 15:09:49.376 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:49.376 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260505 15:09:49.376 | INFO | PRE_HQ : 61.1%, 1.8, 57.2ms, 0 | 20260505 15:09:49.376 | INFO | BAZWRITER : 64.2%, 3.9, 129.0ms, 0 | 20260505 15:09:49.376 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 15:09:49.376 | INFO | Uploaded 36GiB to GPU in 2.19888s (16.372GiB/s) | 20260505 15:09:49.377 | INFO | Downloaded 38GiB to GPU in 2.26893s (16.748GiB/s) | 20260505 15:09:54.511 | INFO | Analyzing chunk frames = [4174336,4174848) | 20260505 15:09:54.511 | INFO | MemInfo: { | 20260505 15:09:54.511 | INFO | "gpu" : " 17 GB", | 20260505 15:09:54.511 | INFO | "huge" : "", | 20260505 15:09:54.511 | INFO | "huge_shared" : " 27 GB", | 20260505 15:09:54.511 | INFO | "malloc" : " 14 GB", | 20260505 15:09:54.511 | INFO | "pinned" : " 29 GB" | 20260505 15:09:54.512 | INFO | } | 20260505 15:09:54.512 | INFO | Perf report for the last 5.13541 seconds: | 20260505 15:09:54.512 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:09:54.512 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:54.512 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:54.512 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:54.512 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 15:09:54.512 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:54.512 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:54.512 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:09:54.512 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:54.512 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:54.512 | INFO | Uploaded 36GiB to GPU in 2.19684s (16.3872GiB/s) | 20260505 15:09:54.512 | INFO | Downloaded 34GiB to GPU in 2.04386s (16.6352GiB/s) | 20260505 15:09:59.653 | INFO | Analyzing chunk frames = [4174848,4175360) | 20260505 15:09:59.653 | INFO | MemInfo: { | 20260505 15:09:59.653 | INFO | "gpu" : " 17 GB", | 20260505 15:09:59.653 | INFO | "huge" : "", | 20260505 15:09:59.653 | INFO | "huge_shared" : " 27 GB", | 20260505 15:09:59.653 | INFO | "malloc" : " 14 GB", | 20260505 15:09:59.653 | INFO | "pinned" : " 29 GB" | 20260505 15:09:59.653 | INFO | } | 20260505 15:09:59.653 | INFO | Perf report for the last 5.14159 seconds: | 20260505 15:09:59.653 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:09:59.653 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:59.653 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:59.653 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:59.653 | INFO | ANALYSIS : 63.0%, 3.0, 98.1ms, 0 | 20260505 15:09:59.653 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:59.653 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:59.653 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 15:09:59.653 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:59.653 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:09:59.654 | INFO | Uploaded 36GiB to GPU in 2.18159s (16.5017GiB/s) | 20260505 15:09:59.654 | INFO | Downloaded 34GiB to GPU in 2.06098s (16.497GiB/s) | 20260505 15:10:04.794 | INFO | Analyzing chunk frames = [4175360,4175872) | 20260505 15:10:04.794 | INFO | MemInfo: { | 20260505 15:10:04.794 | INFO | "gpu" : " 17 GB", | 20260505 15:10:04.794 | INFO | "huge" : "", | 20260505 15:10:04.794 | INFO | "huge_shared" : " 27 GB", | 20260505 15:10:04.794 | INFO | "malloc" : " 14 GB", | 20260505 15:10:04.794 | INFO | "pinned" : " 29 GB" | 20260505 15:10:04.794 | INFO | } | 20260505 15:10:04.794 | INFO | Perf report for the last 5.14086 seconds: | 20260505 15:10:04.794 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:10:04.794 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:04.794 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:04.794 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:04.794 | INFO | ANALYSIS : 62.6%, 3.0, 97.2ms, 0 | 20260505 15:10:04.794 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:04.794 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:04.794 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 15:10:04.794 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:04.794 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:04.794 | INFO | Uploaded 36GiB to GPU in 2.18305s (16.4907GiB/s) | 20260505 15:10:04.794 | INFO | Downloaded 34GiB to GPU in 2.12798s (15.9776GiB/s) | 20260505 15:10:09.931 | INFO | Analyzing chunk frames = [4175872,4176384) | 20260505 15:10:09.931 | INFO | MemInfo: { | 20260505 15:10:09.931 | INFO | "gpu" : " 17 GB", | 20260505 15:10:09.931 | INFO | "huge" : "", | 20260505 15:10:09.931 | INFO | "huge_shared" : " 27 GB", | 20260505 15:10:09.931 | INFO | "malloc" : " 14 GB", | 20260505 15:10:09.931 | INFO | "pinned" : " 29 GB" | 20260505 15:10:09.931 | INFO | } | 20260505 15:10:09.931 | INFO | Perf report for the last 5.13735 seconds: | 20260505 15:10:09.931 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:10:09.931 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:09.931 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:09.931 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:09.932 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 15:10:09.932 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:09.932 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:09.932 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 15:10:09.932 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:09.932 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:09.932 | INFO | Uploaded 36GiB to GPU in 2.16889s (16.5983GiB/s) | 20260505 15:10:09.932 | INFO | Downloaded 34GiB to GPU in 2.13086s (15.956GiB/s) | 20260505 15:10:15.066 | INFO | Analyzing chunk frames = [4176384,4176896) | 20260505 15:10:15.066 | INFO | MemInfo: { | 20260505 15:10:15.066 | INFO | "gpu" : " 17 GB", | 20260505 15:10:15.066 | INFO | "huge" : "", | 20260505 15:10:15.066 | INFO | "huge_shared" : " 27 GB", | 20260505 15:10:15.066 | INFO | "malloc" : " 14 GB", | 20260505 15:10:15.066 | INFO | "pinned" : " 29 GB" | 20260505 15:10:15.066 | INFO | } | 20260505 15:10:15.066 | INFO | Perf report for the last 5.13497 seconds: | 20260505 15:10:15.066 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:10:15.066 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:15.066 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:15.067 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:15.067 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 15:10:15.067 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:15.067 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:15.067 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 15:10:15.067 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:15.067 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:15.067 | INFO | Uploaded 36GiB to GPU in 2.21041s (16.2865GiB/s) | 20260505 15:10:15.067 | INFO | Downloaded 34GiB to GPU in 2.11355s (16.0867GiB/s) | 20260505 15:10:20.204 | INFO | Analyzing chunk frames = [4176896,4177408) | 20260505 15:10:20.204 | INFO | MemInfo: { | 20260505 15:10:20.204 | INFO | "gpu" : " 17 GB", | 20260505 15:10:20.204 | INFO | "huge" : "", | 20260505 15:10:20.204 | INFO | "huge_shared" : " 27 GB", | 20260505 15:10:20.204 | INFO | "malloc" : " 14 GB", | 20260505 15:10:20.204 | INFO | "pinned" : " 29 GB" | 20260505 15:10:20.204 | INFO | } | 20260505 15:10:20.204 | INFO | Perf report for the last 5.13751 seconds: | 20260505 15:10:20.204 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:10:20.204 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:20.204 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:20.204 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:20.204 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 15:10:20.204 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:20.204 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:20.204 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 15:10:20.204 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:20.204 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:20.204 | INFO | Uploaded 36GiB to GPU in 2.16451s (16.6319GiB/s) | 20260505 15:10:20.204 | INFO | Downloaded 34GiB to GPU in 2.17399s (15.6394GiB/s) | 20260505 15:10:25.350 | INFO | Analyzing chunk frames = [4177408,4177920) | 20260505 15:10:25.350 | INFO | MemInfo: { | 20260505 15:10:25.350 | INFO | "gpu" : " 17 GB", | 20260505 15:10:25.350 | INFO | "huge" : "", | 20260505 15:10:25.350 | INFO | "huge_shared" : " 27 GB", | 20260505 15:10:25.350 | INFO | "malloc" : " 14 GB", | 20260505 15:10:25.350 | INFO | "pinned" : " 29 GB" | 20260505 15:10:25.350 | INFO | } | 20260505 15:10:25.350 | INFO | Perf report for the last 5.14616 seconds: | 20260505 15:10:25.350 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:10:25.350 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:25.350 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:25.350 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:25.350 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 15:10:25.350 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:25.350 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:25.350 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 15:10:25.350 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:25.350 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:25.350 | INFO | Uploaded 36GiB to GPU in 2.20765s (16.3069GiB/s) | 20260505 15:10:25.350 | INFO | Downloaded 34GiB to GPU in 2.10266s (16.17GiB/s) | 20260505 15:10:25.351 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 15:10:25.351 | INFO | Total mem usage: 15.3 GiB | 20260505 15:10:25.351 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 15:10:25.351 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.86%) | 20260505 15:10:25.351 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 15:10:25.351 | INFO | High Water Breakdown: | 20260505 15:10:25.351 | INFO | 61% : 8.67 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 15:10:25.351 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 15:10:25.351 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 15:10:25.351 | INFO | ------------------------------------------ | 20260505 15:10:25.351 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:10:25.351 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 15:10:25.351 | INFO | Total mem usage: 26.5 GiB | 20260505 15:10:25.351 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 15:10:25.351 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 15:10:25.351 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 15:10:25.351 | INFO | High Water Breakdown: | 20260505 15:10:25.351 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:10:25.351 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:10:25.351 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:10:25.351 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:10:25.351 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:10:25.351 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:10:25.351 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 15:10:25.351 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 15:10:25.351 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:10:25.351 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 15:10:25.351 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 15:10:25.351 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 15:10:25.351 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 15:10:25.351 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 15:10:25.351 | INFO | ------------------------------------------ | 20260505 15:10:25.351 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:10:25.351 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 15:10:25.351 | INFO | Total mem usage: 25.6 GiB | 20260505 15:10:25.351 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 15:10:25.351 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 15:10:25.351 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 15:10:25.351 | INFO | High Water Breakdown: | 20260505 15:10:25.351 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 15:10:25.351 | INFO | ------------------------------------------ | 20260505 15:10:25.351 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:10:25.351 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 15:10:25.351 | INFO | Total mem usage: 15.7 GiB | 20260505 15:10:25.351 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 15:10:25.351 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 15:10:25.351 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 15:10:25.351 | INFO | High Water Breakdown: | 20260505 15:10:25.351 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 15:10:25.351 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 15:10:25.351 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 15:10:25.351 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 15:10:25.351 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 15:10:25.351 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 15:10:25.351 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 15:10:25.351 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 15:10:25.351 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 15:10:25.351 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:10:25.351 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 15:10:25.351 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:10:25.351 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 15:10:25.351 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 15:10:25.351 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 15:10:25.351 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 15:10:25.351 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:10:25.351 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 15:10:25.351 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:10:25.351 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:10:25.351 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:10:25.351 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 15:10:25.351 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 15:10:25.351 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:10:25.351 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 15:10:25.351 | INFO | ------------------------------------------ | 20260505 15:10:25.351 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:10:25.352 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 15:10:25.352 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:10:25.352 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:10:25.352 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.60e+01, 1.36e+01 / 1.81e+00 | 20260505 15:10:25.352 | INFO | PerfReport: 0.29%, AggregateStats -- 9.30e-02 / 9.92e-02, 1.05e-02 / 3.93e-03 | 20260505 15:10:25.352 | INFO | PerfReport: 11.77%, Baseline -- 3.74e+00 / 4.48e+00, 5.69e-01 / 1.96e-01 | 20260505 15:10:25.352 | INFO | PerfReport: 3.27%, Histogram -- 1.04e+00 / 1.50e+00, 1.92e-01 / 1.03e-01 | 20260505 15:10:25.352 | INFO | PerfReport: 16.01%, DME -- 4.07e+01 / 4.57e+01, 4.66e+00 / 2.10e+00 | 20260505 15:10:25.352 | INFO | PerfReport: 56.50%, FrameLabeling -- 1.79e+01 / 1.94e+01, 1.41e+00 / 1.80e+00 | 20260505 15:10:25.352 | INFO | PerfReport: 7.44%, PulseAccumulating -- 2.36e+00 / 2.95e+00, 4.55e-01 / 1.36e-01 | 20260505 15:10:25.352 | INFO | PerfReport: 3.49%, Metrics -- 1.11e+00 / 1.43e+00, 3.09e-01 / 1.62e-01 | 20260505 15:10:25.352 | INFO | PerfReport: 1.22%, PulseSerializing -- 3.89e-01 / 4.38e-01, 6.95e-02 / 3.43e-02 | 20260505 15:10:25.352 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 15:10:25.352 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:10:25.352 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:10:25.352 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260505 15:10:25.352 | INFO | PerfReport: 22.62%, Upload -- 2.28e+01 / 0.00e+00, 8.04e+00 / 0.00e+00 | 20260505 15:10:25.352 | INFO | PerfReport: 22.02%, Download -- 2.22e+01 / 0.00e+00, 8.00e+00 / 0.00e+00 | 20260505 15:10:25.352 | INFO | PerfReport: 31.57%, Compute -- 3.18e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 15:10:25.352 | INFO | PerfReport: 2.28%, LockUpload -- 2.29e+00 / 0.00e+00, 7.88e+00 / 0.00e+00 | 20260505 15:10:25.352 | INFO | PerfReport: 1.43%, LockDownload -- 1.44e+00 / 0.00e+00, 2.90e+00 / 0.00e+00 | 20260505 15:10:25.352 | INFO | PerfReport: 20.07%, LockCompute -- 2.02e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260505 15:10:25.352 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.70e-02 / 0.00e+00 | 20260505 15:10:25.487 | INFO | Starting a round of Baz IO | 20260505 15:10:28.693 | INFO | Populating RTMetrics report | 20260505 15:10:28.693 | INFO | RTMetrics report populated | 20260505 15:10:28.693 | INFO | Opening tmp RTMetrics json file | 20260505 15:10:28.709 | INFO | Serializing RTMetrics to json | 20260505 15:10:28.714 | INFO | Serialization done, swapping tmp file now... | 20260505 15:10:28.714 | INFO | RTMetrics done | 20260505 15:10:28.800 | INFO | Finished a round of Baz IO: Wrote 2.224 GiB in 3.312 seconds | 20260505 15:10:28.800 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.54% / 0.00% / 77.45% | 20260505 15:10:28.800 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.475522% / 0% / 8.74201e-05% | 20260505 15:10:28.800 | INFO | Baz IO overall: | 20260505 15:10:28.800 | INFO | BytesWritten : 2486834931217 | 20260505 15:10:28.800 | INFO | NumEvents : 1181679486423 | 20260505 15:10:30.495 | INFO | Analyzing chunk frames = [4177920,4178432) | 20260505 15:10:30.495 | INFO | MemInfo: { | 20260505 15:10:30.495 | INFO | "gpu" : " 17 GB", | 20260505 15:10:30.495 | INFO | "huge" : "", | 20260505 15:10:30.495 | INFO | "huge_shared" : " 27 GB", | 20260505 15:10:30.495 | INFO | "malloc" : " 14 GB", | 20260505 15:10:30.495 | INFO | "pinned" : " 29 GB" | 20260505 15:10:30.495 | INFO | } | 20260505 15:10:30.495 | INFO | Perf report for the last 5.14482 seconds: | 20260505 15:10:30.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:10:30.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:30.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:30.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:30.495 | INFO | ANALYSIS : 63.9%, 3.0, 99.2ms, 0 | 20260505 15:10:30.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:30.495 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260505 15:10:30.495 | INFO | PRE_HQ : 60.8%, 1.8, 57.4ms, 0 | 20260505 15:10:30.495 | INFO | BAZWRITER : 63.7%, 4.0, 131.6ms, 0 | 20260505 15:10:30.495 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 15:10:30.495 | INFO | Uploaded 36GiB to GPU in 2.19437s (16.4057GiB/s) | 20260505 15:10:30.495 | INFO | Downloaded 38GiB to GPU in 2.2621s (16.7985GiB/s) | 20260505 15:10:35.630 | INFO | Analyzing chunk frames = [4178432,4178944) | 20260505 15:10:35.630 | INFO | MemInfo: { | 20260505 15:10:35.630 | INFO | "gpu" : " 17 GB", | 20260505 15:10:35.630 | INFO | "huge" : "", | 20260505 15:10:35.630 | INFO | "huge_shared" : " 27 GB", | 20260505 15:10:35.630 | INFO | "malloc" : " 14 GB", | 20260505 15:10:35.630 | INFO | "pinned" : " 29 GB" | 20260505 15:10:35.630 | INFO | } | 20260505 15:10:35.630 | INFO | Perf report for the last 5.13505 seconds: | 20260505 15:10:35.630 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:10:35.630 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:35.630 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:35.630 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:35.630 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 15:10:35.630 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:35.630 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:35.630 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 15:10:35.630 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:35.630 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:35.630 | INFO | Uploaded 36GiB to GPU in 2.13246s (16.8819GiB/s) | 20260505 15:10:35.630 | INFO | Downloaded 34GiB to GPU in 2.03797s (16.6833GiB/s) | 20260505 15:10:40.759 | INFO | Analyzing chunk frames = [4178944,4179456) | 20260505 15:10:40.759 | INFO | MemInfo: { | 20260505 15:10:40.759 | INFO | "gpu" : " 17 GB", | 20260505 15:10:40.759 | INFO | "huge" : "", | 20260505 15:10:40.759 | INFO | "huge_shared" : " 27 GB", | 20260505 15:10:40.759 | INFO | "malloc" : " 14 GB", | 20260505 15:10:40.759 | INFO | "pinned" : " 29 GB" | 20260505 15:10:40.759 | INFO | } | 20260505 15:10:40.759 | INFO | Perf report for the last 5.12916 seconds: | 20260505 15:10:40.759 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:10:40.759 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:40.759 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:40.759 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:40.759 | INFO | ANALYSIS : 63.1%, 3.0, 98.1ms, 0 | 20260505 15:10:40.759 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:40.759 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:40.759 | INFO | PRE_HQ : 28.3%, 1.0, 14.6ms, 0 | 20260505 15:10:40.759 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:40.759 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:40.759 | INFO | Uploaded 36GiB to GPU in 2.1938s (16.4099GiB/s) | 20260505 15:10:40.760 | INFO | Downloaded 34GiB to GPU in 2.06167s (16.4914GiB/s) | 20260505 15:10:45.904 | INFO | Analyzing chunk frames = [4179456,4179968) | 20260505 15:10:45.904 | INFO | MemInfo: { | 20260505 15:10:45.904 | INFO | "gpu" : " 17 GB", | 20260505 15:10:45.904 | INFO | "huge" : "", | 20260505 15:10:45.904 | INFO | "huge_shared" : " 27 GB", | 20260505 15:10:45.904 | INFO | "malloc" : " 14 GB", | 20260505 15:10:45.904 | INFO | "pinned" : " 29 GB" | 20260505 15:10:45.904 | INFO | } | 20260505 15:10:45.904 | INFO | Perf report for the last 5.14469 seconds: | 20260505 15:10:45.904 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:10:45.904 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:45.904 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:45.904 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:45.904 | INFO | ANALYSIS : 62.9%, 3.0, 97.9ms, 0 | 20260505 15:10:45.904 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:45.904 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:45.904 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 15:10:45.904 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:45.904 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:45.904 | INFO | Uploaded 36GiB to GPU in 2.20772s (16.3064GiB/s) | 20260505 15:10:45.904 | INFO | Downloaded 34GiB to GPU in 2.13838s (15.8999GiB/s) | 20260505 15:10:51.045 | INFO | Analyzing chunk frames = [4179968,4180480) | 20260505 15:10:51.046 | INFO | MemInfo: { | 20260505 15:10:51.046 | INFO | "gpu" : " 17 GB", | 20260505 15:10:51.046 | INFO | "huge" : "", | 20260505 15:10:51.046 | INFO | "huge_shared" : " 27 GB", | 20260505 15:10:51.046 | INFO | "malloc" : " 14 GB", | 20260505 15:10:51.046 | INFO | "pinned" : " 29 GB" | 20260505 15:10:51.046 | INFO | } | 20260505 15:10:51.046 | INFO | Perf report for the last 5.1418 seconds: | 20260505 15:10:51.046 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:10:51.046 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:51.046 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:51.046 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:51.046 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 15:10:51.046 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:51.046 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:51.046 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:10:51.046 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:51.046 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:51.046 | INFO | Uploaded 36GiB to GPU in 2.17529s (16.5495GiB/s) | 20260505 15:10:51.046 | INFO | Downloaded 34GiB to GPU in 2.12947s (15.9664GiB/s) | 20260505 15:10:56.188 | INFO | Analyzing chunk frames = [4180480,4180992) | 20260505 15:10:56.188 | INFO | MemInfo: { | 20260505 15:10:56.188 | INFO | "gpu" : " 17 GB", | 20260505 15:10:56.188 | INFO | "huge" : "", | 20260505 15:10:56.188 | INFO | "huge_shared" : " 27 GB", | 20260505 15:10:56.188 | INFO | "malloc" : " 14 GB", | 20260505 15:10:56.188 | INFO | "pinned" : " 29 GB" | 20260505 15:10:56.188 | INFO | } | 20260505 15:10:56.188 | INFO | Perf report for the last 5.14213 seconds: | 20260505 15:10:56.188 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:10:56.188 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:56.188 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:56.188 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:56.188 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 15:10:56.188 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:56.188 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:56.188 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 15:10:56.188 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:56.188 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:10:56.188 | INFO | Uploaded 36GiB to GPU in 2.20776s (16.3061GiB/s) | 20260505 15:10:56.188 | INFO | Downloaded 34GiB to GPU in 2.1132s (16.0894GiB/s) | 20260505 15:11:01.321 | INFO | Analyzing chunk frames = [4180992,4181504) | 20260505 15:11:01.321 | INFO | MemInfo: { | 20260505 15:11:01.321 | INFO | "gpu" : " 17 GB", | 20260505 15:11:01.321 | INFO | "huge" : "", | 20260505 15:11:01.321 | INFO | "huge_shared" : " 27 GB", | 20260505 15:11:01.321 | INFO | "malloc" : " 14 GB", | 20260505 15:11:01.321 | INFO | "pinned" : " 29 GB" | 20260505 15:11:01.321 | INFO | } | 20260505 15:11:01.321 | INFO | Perf report for the last 5.1334 seconds: | 20260505 15:11:01.321 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:11:01.321 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:01.321 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:01.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:01.321 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 15:11:01.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:01.321 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:01.321 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 15:11:01.321 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:01.321 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:01.322 | INFO | Uploaded 36GiB to GPU in 2.12946s (16.9057GiB/s) | 20260505 15:11:01.322 | INFO | Downloaded 34GiB to GPU in 2.17618s (15.6237GiB/s) | 20260505 15:11:06.458 | INFO | Analyzing chunk frames = [4181504,4182016) | 20260505 15:11:06.458 | INFO | MemInfo: { | 20260505 15:11:06.458 | INFO | "gpu" : " 17 GB", | 20260505 15:11:06.458 | INFO | "huge" : "", | 20260505 15:11:06.458 | INFO | "huge_shared" : " 27 GB", | 20260505 15:11:06.458 | INFO | "malloc" : " 14 GB", | 20260505 15:11:06.458 | INFO | "pinned" : " 29 GB" | 20260505 15:11:06.458 | INFO | } | 20260505 15:11:06.458 | INFO | Perf report for the last 5.1369 seconds: | 20260505 15:11:06.458 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:11:06.458 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:06.458 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:06.458 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:06.458 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 15:11:06.458 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:06.458 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:06.458 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 15:11:06.458 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:06.458 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:06.459 | INFO | Uploaded 36GiB to GPU in 2.21045s (16.2863GiB/s) | 20260505 15:11:06.459 | INFO | Downloaded 34GiB to GPU in 2.09871s (16.2004GiB/s) | 20260505 15:11:06.595 | INFO | Starting a round of Baz IO | 20260505 15:11:09.807 | INFO | Populating RTMetrics report | 20260505 15:11:09.807 | INFO | RTMetrics report populated | 20260505 15:11:09.807 | INFO | Opening tmp RTMetrics json file | 20260505 15:11:09.818 | INFO | Serializing RTMetrics to json | 20260505 15:11:09.819 | INFO | Serialization done, swapping tmp file now... | 20260505 15:11:09.820 | INFO | RTMetrics done | 20260505 15:11:09.910 | INFO | Finished a round of Baz IO: Wrote 2.223 GiB in 3.314 seconds | 20260505 15:11:09.910 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.54% / 0.00% / 77.46% | 20260505 15:11:09.910 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.474846% / 3.97364e-06% / 0.000103315% | 20260505 15:11:09.910 | INFO | Baz IO overall: | 20260505 15:11:09.910 | INFO | BytesWritten : 2489221941593 | 20260505 15:11:09.910 | INFO | NumEvents : 1182802491750 | 20260505 15:11:11.610 | INFO | Analyzing chunk frames = [4182016,4182528) | 20260505 15:11:11.610 | INFO | MemInfo: { | 20260505 15:11:11.610 | INFO | "gpu" : " 17 GB", | 20260505 15:11:11.610 | INFO | "huge" : "", | 20260505 15:11:11.610 | INFO | "huge_shared" : " 27 GB", | 20260505 15:11:11.610 | INFO | "malloc" : " 14 GB", | 20260505 15:11:11.610 | INFO | "pinned" : " 29 GB" | 20260505 15:11:11.610 | INFO | } | 20260505 15:11:11.610 | INFO | Perf report for the last 5.15161 seconds: | 20260505 15:11:11.610 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:11:11.610 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:11.610 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:11.610 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:11.610 | INFO | ANALYSIS : 63.9%, 3.0, 99.6ms, 0 | 20260505 15:11:11.610 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:11.610 | INFO | RT_METRICS : 44.9%, 1.0, 23.6ms, 0 | 20260505 15:11:11.610 | INFO | PRE_HQ : 60.5%, 1.8, 57.0ms, 0 | 20260505 15:11:11.610 | INFO | BAZWRITER : 63.7%, 4.0, 129.8ms, 0 | 20260505 15:11:11.610 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 15:11:11.610 | INFO | Uploaded 36GiB to GPU in 2.2022s (16.3473GiB/s) | 20260505 15:11:11.610 | INFO | Downloaded 38GiB to GPU in 2.26381s (16.7859GiB/s) | 20260505 15:11:16.745 | INFO | Analyzing chunk frames = [4182528,4183040) | 20260505 15:11:16.745 | INFO | MemInfo: { | 20260505 15:11:16.745 | INFO | "gpu" : " 17 GB", | 20260505 15:11:16.745 | INFO | "huge" : "", | 20260505 15:11:16.745 | INFO | "huge_shared" : " 27 GB", | 20260505 15:11:16.745 | INFO | "malloc" : " 14 GB", | 20260505 15:11:16.745 | INFO | "pinned" : " 29 GB" | 20260505 15:11:16.745 | INFO | } | 20260505 15:11:16.745 | INFO | Perf report for the last 5.13527 seconds: | 20260505 15:11:16.745 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:11:16.745 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:16.745 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:16.745 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:16.745 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 15:11:16.745 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:16.745 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:16.745 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:11:16.745 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:16.745 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:16.745 | INFO | Uploaded 36GiB to GPU in 2.12503s (16.9409GiB/s) | 20260505 15:11:16.745 | INFO | Downloaded 34GiB to GPU in 2.02872s (16.7593GiB/s) | 20260505 15:11:21.879 | INFO | Analyzing chunk frames = [4183040,4183552) | 20260505 15:11:21.879 | INFO | MemInfo: { | 20260505 15:11:21.879 | INFO | "gpu" : " 17 GB", | 20260505 15:11:21.879 | INFO | "huge" : "", | 20260505 15:11:21.879 | INFO | "huge_shared" : " 27 GB", | 20260505 15:11:21.879 | INFO | "malloc" : " 14 GB", | 20260505 15:11:21.879 | INFO | "pinned" : " 29 GB" | 20260505 15:11:21.879 | INFO | } | 20260505 15:11:21.879 | INFO | Perf report for the last 5.13376 seconds: | 20260505 15:11:21.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:11:21.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:21.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:21.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:21.879 | INFO | ANALYSIS : 63.0%, 3.0, 98.0ms, 0 | 20260505 15:11:21.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:21.879 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:21.879 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 15:11:21.879 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:21.879 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:21.879 | INFO | Uploaded 36GiB to GPU in 2.19095s (16.4312GiB/s) | 20260505 15:11:21.879 | INFO | Downloaded 34GiB to GPU in 2.05724s (16.527GiB/s) | 20260505 15:11:27.013 | INFO | Analyzing chunk frames = [4183552,4184064) | 20260505 15:11:27.013 | INFO | MemInfo: { | 20260505 15:11:27.013 | INFO | "gpu" : " 17 GB", | 20260505 15:11:27.013 | INFO | "huge" : "", | 20260505 15:11:27.013 | INFO | "huge_shared" : " 27 GB", | 20260505 15:11:27.013 | INFO | "malloc" : " 14 GB", | 20260505 15:11:27.013 | INFO | "pinned" : " 29 GB" | 20260505 15:11:27.013 | INFO | } | 20260505 15:11:27.013 | INFO | Perf report for the last 5.13462 seconds: | 20260505 15:11:27.014 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:11:27.014 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:27.014 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:27.014 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:27.014 | INFO | ANALYSIS : 62.9%, 3.0, 97.7ms, 0 | 20260505 15:11:27.014 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:27.014 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:27.014 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 15:11:27.014 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:27.014 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:27.014 | INFO | Uploaded 36GiB to GPU in 2.19477s (16.4026GiB/s) | 20260505 15:11:27.014 | INFO | Downloaded 34GiB to GPU in 2.13575s (15.9195GiB/s) | 20260505 15:11:32.164 | INFO | Analyzing chunk frames = [4184064,4184576) | 20260505 15:11:32.164 | INFO | MemInfo: { | 20260505 15:11:32.164 | INFO | "gpu" : " 17 GB", | 20260505 15:11:32.164 | INFO | "huge" : "", | 20260505 15:11:32.164 | INFO | "huge_shared" : " 27 GB", | 20260505 15:11:32.164 | INFO | "malloc" : " 14 GB", | 20260505 15:11:32.164 | INFO | "pinned" : " 29 GB" | 20260505 15:11:32.164 | INFO | } | 20260505 15:11:32.164 | INFO | Perf report for the last 5.15041 seconds: | 20260505 15:11:32.164 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:11:32.164 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:32.164 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:32.164 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:32.164 | INFO | ANALYSIS : 62.6%, 3.0, 97.9ms, 0 | 20260505 15:11:32.164 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:32.164 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:32.164 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 15:11:32.164 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:32.164 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:32.164 | INFO | Uploaded 36GiB to GPU in 2.15283s (16.7222GiB/s) | 20260505 15:11:32.164 | INFO | Downloaded 34GiB to GPU in 2.12737s (15.9822GiB/s) | 20260505 15:11:37.303 | INFO | Analyzing chunk frames = [4184576,4185088) | 20260505 15:11:37.303 | INFO | MemInfo: { | 20260505 15:11:37.303 | INFO | "gpu" : " 17 GB", | 20260505 15:11:37.303 | INFO | "huge" : "", | 20260505 15:11:37.303 | INFO | "huge_shared" : " 27 GB", | 20260505 15:11:37.303 | INFO | "malloc" : " 14 GB", | 20260505 15:11:37.303 | INFO | "pinned" : " 29 GB" | 20260505 15:11:37.303 | INFO | } | 20260505 15:11:37.303 | INFO | Perf report for the last 5.13938 seconds: | 20260505 15:11:37.303 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:11:37.303 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:37.303 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:37.303 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:37.303 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260505 15:11:37.303 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:37.303 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:37.303 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 15:11:37.303 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:37.303 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:37.304 | INFO | Uploaded 36GiB to GPU in 2.13559s (16.8572GiB/s) | 20260505 15:11:37.304 | INFO | Downloaded 34GiB to GPU in 2.03576s (16.7013GiB/s) | 20260505 15:11:42.441 | INFO | Analyzing chunk frames = [4185088,4185600) | 20260505 15:11:42.441 | INFO | MemInfo: { | 20260505 15:11:42.441 | INFO | "gpu" : " 17 GB", | 20260505 15:11:42.441 | INFO | "huge" : "", | 20260505 15:11:42.441 | INFO | "huge_shared" : " 27 GB", | 20260505 15:11:42.441 | INFO | "malloc" : " 14 GB", | 20260505 15:11:42.441 | INFO | "pinned" : " 29 GB" | 20260505 15:11:42.441 | INFO | } | 20260505 15:11:42.441 | INFO | Perf report for the last 5.13762 seconds: | 20260505 15:11:42.441 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:11:42.441 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:42.441 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:42.441 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:42.441 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 15:11:42.441 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:42.441 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:42.441 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 15:11:42.441 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:42.441 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:42.441 | INFO | Uploaded 36GiB to GPU in 2.12176s (16.967GiB/s) | 20260505 15:11:42.441 | INFO | Downloaded 34GiB to GPU in 2.17355s (15.6426GiB/s) | 20260505 15:11:47.572 | INFO | Analyzing chunk frames = [4185600,4186112) | 20260505 15:11:47.572 | INFO | MemInfo: { | 20260505 15:11:47.572 | INFO | "gpu" : " 17 GB", | 20260505 15:11:47.572 | INFO | "huge" : "", | 20260505 15:11:47.572 | INFO | "huge_shared" : " 27 GB", | 20260505 15:11:47.572 | INFO | "malloc" : " 14 GB", | 20260505 15:11:47.572 | INFO | "pinned" : " 29 GB" | 20260505 15:11:47.572 | INFO | } | 20260505 15:11:47.572 | INFO | Perf report for the last 5.13104 seconds: | 20260505 15:11:47.572 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:11:47.572 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:47.572 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:47.572 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:47.572 | INFO | ANALYSIS : 63.8%, 3.0, 99.5ms, 0 | 20260505 15:11:47.572 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:47.572 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:47.572 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 15:11:47.572 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:47.572 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:47.572 | INFO | Uploaded 36GiB to GPU in 2.23292s (16.1224GiB/s) | 20260505 15:11:47.572 | INFO | Downloaded 34GiB to GPU in 2.10622s (16.1427GiB/s) | 20260505 15:11:47.573 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 15:11:47.573 | INFO | Total mem usage: 15.3 GiB | 20260505 15:11:47.573 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 15:11:47.573 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.85%) | 20260505 15:11:47.573 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 15:11:47.573 | INFO | High Water Breakdown: | 20260505 15:11:47.573 | INFO | 61% : 8.67 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 15:11:47.573 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 15:11:47.573 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 15:11:47.573 | INFO | ------------------------------------------ | 20260505 15:11:47.573 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:11:47.573 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 15:11:47.573 | INFO | Total mem usage: 26.5 GiB | 20260505 15:11:47.573 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 15:11:47.573 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 15:11:47.573 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 15:11:47.573 | INFO | High Water Breakdown: | 20260505 15:11:47.573 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:11:47.573 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:11:47.573 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:11:47.573 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:11:47.573 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:11:47.573 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:11:47.573 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 15:11:47.573 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 15:11:47.573 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:11:47.573 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 15:11:47.573 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 15:11:47.573 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 15:11:47.573 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 15:11:47.573 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 15:11:47.573 | INFO | ------------------------------------------ | 20260505 15:11:47.573 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:11:47.573 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 15:11:47.573 | INFO | Total mem usage: 25.6 GiB | 20260505 15:11:47.573 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 15:11:47.573 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 15:11:47.573 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 15:11:47.573 | INFO | High Water Breakdown: | 20260505 15:11:47.573 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 15:11:47.573 | INFO | ------------------------------------------ | 20260505 15:11:47.573 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:11:47.573 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 15:11:47.573 | INFO | Total mem usage: 15.7 GiB | 20260505 15:11:47.573 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 15:11:47.573 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 15:11:47.573 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260505 15:11:47.573 | INFO | High Water Breakdown: | 20260505 15:11:47.573 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 15:11:47.573 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 15:11:47.573 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 15:11:47.573 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 15:11:47.573 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 15:11:47.573 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 15:11:47.573 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 15:11:47.573 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 15:11:47.573 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 15:11:47.573 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:11:47.573 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 15:11:47.573 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:11:47.573 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 15:11:47.573 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 15:11:47.573 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 15:11:47.573 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 15:11:47.573 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:11:47.573 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 15:11:47.573 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:11:47.573 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:11:47.573 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:11:47.573 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 15:11:47.573 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 15:11:47.573 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:11:47.573 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 15:11:47.573 | INFO | ------------------------------------------ | 20260505 15:11:47.573 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:11:47.573 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 15:11:47.573 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:11:47.573 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:11:47.573 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.63e+01, 1.37e+01 / 2.22e+00 | 20260505 15:11:47.573 | INFO | PerfReport: 0.29%, AggregateStats -- 9.31e-02 / 1.01e-01, 1.09e-02 / 4.53e-03 | 20260505 15:11:47.573 | INFO | PerfReport: 11.76%, Baseline -- 3.74e+00 / 4.47e+00, 5.77e-01 / 2.11e-01 | 20260505 15:11:47.573 | INFO | PerfReport: 3.29%, Histogram -- 1.05e+00 / 1.47e+00, 1.93e-01 / 1.22e-01 | 20260505 15:11:47.573 | INFO | PerfReport: 15.97%, DME -- 4.06e+01 / 4.56e+01, 4.61e+00 / 2.45e+00 | 20260505 15:11:47.573 | INFO | PerfReport: 56.69%, FrameLabeling -- 1.80e+01 / 1.99e+01, 1.51e+00 / 1.75e+00 | 20260505 15:11:47.573 | INFO | PerfReport: 7.32%, PulseAccumulating -- 2.33e+00 / 2.92e+00, 4.37e-01 / 1.57e-01 | 20260505 15:11:47.573 | INFO | PerfReport: 3.46%, Metrics -- 1.10e+00 / 1.42e+00, 3.06e-01 / 1.31e-01 | 20260505 15:11:47.573 | INFO | PerfReport: 1.22%, PulseSerializing -- 3.87e-01 / 4.35e-01, 6.91e-02 / 2.98e-02 | 20260505 15:11:47.574 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 15:11:47.574 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:11:47.574 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:11:47.574 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.56e+01 / 0.00e+00 | 20260505 15:11:47.574 | INFO | PerfReport: 22.53%, Upload -- 2.27e+01 / 0.00e+00, 7.97e+00 / 0.00e+00 | 20260505 15:11:47.574 | INFO | PerfReport: 21.97%, Download -- 2.21e+01 / 0.00e+00, 7.97e+00 / 0.00e+00 | 20260505 15:11:47.574 | INFO | PerfReport: 31.63%, Compute -- 3.18e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 15:11:47.574 | INFO | PerfReport: 2.24%, LockUpload -- 2.25e+00 / 0.00e+00, 7.91e+00 / 0.00e+00 | 20260505 15:11:47.574 | INFO | PerfReport: 1.40%, LockDownload -- 1.41e+00 / 0.00e+00, 2.90e+00 / 0.00e+00 | 20260505 15:11:47.574 | INFO | PerfReport: 20.22%, LockCompute -- 2.03e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 15:11:47.574 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.57e-02 / 0.00e+00 | 20260505 15:11:47.713 | INFO | Starting a round of Baz IO | 20260505 15:11:50.934 | INFO | Populating RTMetrics report | 20260505 15:11:50.934 | INFO | RTMetrics report populated | 20260505 15:11:50.934 | INFO | Opening tmp RTMetrics json file | 20260505 15:11:50.953 | INFO | Serializing RTMetrics to json | 20260505 15:11:50.987 | INFO | Serialization done, swapping tmp file now... | 20260505 15:11:51.004 | INFO | RTMetrics done | 20260505 15:11:51.040 | INFO | Finished a round of Baz IO: Wrote 2.221 GiB in 3.327 seconds | 20260505 15:11:51.040 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.54% / 0.00% / 77.46% | 20260505 15:11:51.040 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.471143% / 3.97364e-06% / 7.54992e-05% | 20260505 15:11:51.040 | INFO | Baz IO overall: | 20260505 15:11:51.040 | INFO | BytesWritten : 2491606617479 | 20260505 15:11:51.040 | INFO | NumEvents : 1183923854146 | 20260505 15:11:52.711 | INFO | Analyzing chunk frames = [4186112,4186624) | 20260505 15:11:52.711 | INFO | MemInfo: { | 20260505 15:11:52.711 | INFO | "gpu" : " 17 GB", | 20260505 15:11:52.711 | INFO | "huge" : "", | 20260505 15:11:52.711 | INFO | "huge_shared" : " 27 GB", | 20260505 15:11:52.711 | INFO | "malloc" : " 14 GB", | 20260505 15:11:52.711 | INFO | "pinned" : " 29 GB" | 20260505 15:11:52.711 | INFO | } | 20260505 15:11:52.711 | INFO | Perf report for the last 5.139 seconds: | 20260505 15:11:52.711 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:11:52.711 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:52.711 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:52.711 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:52.711 | INFO | ANALYSIS : 64.3%, 3.0, 98.9ms, 0 | 20260505 15:11:52.711 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:52.711 | INFO | RT_METRICS : 46.2%, 1.0, 24.0ms, 0 | 20260505 15:11:52.711 | INFO | PRE_HQ : 60.6%, 1.8, 56.4ms, 0 | 20260505 15:11:52.711 | INFO | BAZWRITER : 64.1%, 4.0, 128.9ms, 0 | 20260505 15:11:52.711 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 15:11:52.711 | INFO | Uploaded 36GiB to GPU in 2.18875s (16.4477GiB/s) | 20260505 15:11:52.711 | INFO | Downloaded 38GiB to GPU in 2.27063s (16.7354GiB/s) | 20260505 15:11:57.862 | INFO | Analyzing chunk frames = [4186624,4187136) | 20260505 15:11:57.862 | INFO | MemInfo: { | 20260505 15:11:57.862 | INFO | "gpu" : " 17 GB", | 20260505 15:11:57.862 | INFO | "huge" : "", | 20260505 15:11:57.862 | INFO | "huge_shared" : " 27 GB", | 20260505 15:11:57.862 | INFO | "malloc" : " 14 GB", | 20260505 15:11:57.862 | INFO | "pinned" : " 29 GB" | 20260505 15:11:57.862 | INFO | } | 20260505 15:11:57.862 | INFO | Perf report for the last 5.15128 seconds: | 20260505 15:11:57.862 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:11:57.862 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:57.862 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:57.862 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:57.862 | INFO | ANALYSIS : 63.3%, 3.0, 99.0ms, 0 | 20260505 15:11:57.862 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:57.862 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:57.862 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:11:57.862 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:57.862 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:11:57.863 | INFO | Uploaded 36GiB to GPU in 2.19916s (16.3699GiB/s) | 20260505 15:11:57.863 | INFO | Downloaded 34GiB to GPU in 2.04044s (16.6631GiB/s) | 20260505 15:12:03.001 | INFO | Analyzing chunk frames = [4187136,4187648) | 20260505 15:12:03.001 | INFO | MemInfo: { | 20260505 15:12:03.001 | INFO | "gpu" : " 17 GB", | 20260505 15:12:03.001 | INFO | "huge" : "", | 20260505 15:12:03.001 | INFO | "huge_shared" : " 27 GB", | 20260505 15:12:03.001 | INFO | "malloc" : " 14 GB", | 20260505 15:12:03.001 | INFO | "pinned" : " 29 GB" | 20260505 15:12:03.001 | INFO | } | 20260505 15:12:03.001 | INFO | Perf report for the last 5.13895 seconds: | 20260505 15:12:03.001 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:12:03.001 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:03.001 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:03.001 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:03.001 | INFO | ANALYSIS : 62.8%, 3.0, 97.8ms, 0 | 20260505 15:12:03.001 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:03.001 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:03.001 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 15:12:03.001 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:03.001 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:03.002 | INFO | Uploaded 36GiB to GPU in 2.18327s (16.489GiB/s) | 20260505 15:12:03.002 | INFO | Downloaded 34GiB to GPU in 2.03115s (16.7393GiB/s) | 20260505 15:12:08.138 | INFO | Analyzing chunk frames = [4187648,4188160) | 20260505 15:12:08.138 | INFO | MemInfo: { | 20260505 15:12:08.138 | INFO | "gpu" : " 17 GB", | 20260505 15:12:08.138 | INFO | "huge" : "", | 20260505 15:12:08.138 | INFO | "huge_shared" : " 27 GB", | 20260505 15:12:08.138 | INFO | "malloc" : " 14 GB", | 20260505 15:12:08.138 | INFO | "pinned" : " 29 GB" | 20260505 15:12:08.138 | INFO | } | 20260505 15:12:08.139 | INFO | Perf report for the last 5.1373 seconds: | 20260505 15:12:08.139 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:12:08.139 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:08.139 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:08.139 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:08.139 | INFO | ANALYSIS : 62.5%, 3.0, 97.1ms, 0 | 20260505 15:12:08.139 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:08.139 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:08.139 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 15:12:08.139 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:08.139 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:08.139 | INFO | Uploaded 36GiB to GPU in 2.18063s (16.509GiB/s) | 20260505 15:12:08.139 | INFO | Downloaded 34GiB to GPU in 2.1335s (15.9363GiB/s) | 20260505 15:12:13.269 | INFO | Analyzing chunk frames = [4188160,4188672) | 20260505 15:12:13.269 | INFO | MemInfo: { | 20260505 15:12:13.269 | INFO | "gpu" : " 17 GB", | 20260505 15:12:13.269 | INFO | "huge" : "", | 20260505 15:12:13.269 | INFO | "huge_shared" : " 27 GB", | 20260505 15:12:13.269 | INFO | "malloc" : " 14 GB", | 20260505 15:12:13.269 | INFO | "pinned" : " 29 GB" | 20260505 15:12:13.269 | INFO | } | 20260505 15:12:13.269 | INFO | Perf report for the last 5.13059 seconds: | 20260505 15:12:13.269 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:12:13.269 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:13.269 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:13.269 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:13.269 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260505 15:12:13.269 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:13.269 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:13.269 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 15:12:13.269 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:13.269 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:13.269 | INFO | Uploaded 36GiB to GPU in 2.1729s (16.5677GiB/s) | 20260505 15:12:13.269 | INFO | Downloaded 34GiB to GPU in 2.13292s (15.9406GiB/s) | 20260505 15:12:18.413 | INFO | Analyzing chunk frames = [4188672,4189184) | 20260505 15:12:18.413 | INFO | MemInfo: { | 20260505 15:12:18.413 | INFO | "gpu" : " 17 GB", | 20260505 15:12:18.413 | INFO | "huge" : "", | 20260505 15:12:18.413 | INFO | "huge_shared" : " 27 GB", | 20260505 15:12:18.413 | INFO | "malloc" : " 14 GB", | 20260505 15:12:18.413 | INFO | "pinned" : " 29 GB" | 20260505 15:12:18.413 | INFO | } | 20260505 15:12:18.413 | INFO | Perf report for the last 5.1443 seconds: | 20260505 15:12:18.413 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:12:18.413 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:18.414 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:18.414 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:18.414 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 15:12:18.414 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:18.414 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:18.414 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:12:18.414 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:18.414 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:18.414 | INFO | Uploaded 36GiB to GPU in 2.20813s (16.3034GiB/s) | 20260505 15:12:18.414 | INFO | Downloaded 34GiB to GPU in 2.11581s (16.0695GiB/s) | 20260505 15:12:23.551 | INFO | Analyzing chunk frames = [4189184,4189696) | 20260505 15:12:23.552 | INFO | MemInfo: { | 20260505 15:12:23.552 | INFO | "gpu" : " 17 GB", | 20260505 15:12:23.552 | INFO | "huge" : "", | 20260505 15:12:23.552 | INFO | "huge_shared" : " 27 GB", | 20260505 15:12:23.552 | INFO | "malloc" : " 14 GB", | 20260505 15:12:23.552 | INFO | "pinned" : " 29 GB" | 20260505 15:12:23.552 | INFO | } | 20260505 15:12:23.552 | INFO | Perf report for the last 5.13828 seconds: | 20260505 15:12:23.552 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:12:23.552 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:23.552 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:23.552 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:23.552 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 15:12:23.552 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:23.552 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:23.552 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 15:12:23.552 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:23.552 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:23.552 | INFO | Uploaded 36GiB to GPU in 2.1157s (17.0156GiB/s) | 20260505 15:12:23.552 | INFO | Downloaded 34GiB to GPU in 2.17578s (15.6266GiB/s) | 20260505 15:12:28.694 | INFO | Analyzing chunk frames = [4189696,4190208) | 20260505 15:12:28.694 | INFO | MemInfo: { | 20260505 15:12:28.694 | INFO | "gpu" : " 17 GB", | 20260505 15:12:28.694 | INFO | "huge" : "", | 20260505 15:12:28.694 | INFO | "huge_shared" : " 27 GB", | 20260505 15:12:28.694 | INFO | "malloc" : " 14 GB", | 20260505 15:12:28.694 | INFO | "pinned" : " 29 GB" | 20260505 15:12:28.694 | INFO | } | 20260505 15:12:28.694 | INFO | Perf report for the last 5.14225 seconds: | 20260505 15:12:28.694 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:12:28.694 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:28.694 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:28.694 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:28.694 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 15:12:28.694 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:28.694 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:28.694 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 15:12:28.694 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:28.694 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:28.694 | INFO | Uploaded 36GiB to GPU in 2.23472s (16.1094GiB/s) | 20260505 15:12:28.694 | INFO | Downloaded 34GiB to GPU in 2.10531s (16.1497GiB/s) | 20260505 15:12:28.829 | INFO | Starting a round of Baz IO | 20260505 15:12:32.047 | INFO | Populating RTMetrics report | 20260505 15:12:32.048 | INFO | RTMetrics report populated | 20260505 15:12:32.048 | INFO | Opening tmp RTMetrics json file | 20260505 15:12:32.062 | INFO | Serializing RTMetrics to json | 20260505 15:12:32.064 | INFO | Serialization done, swapping tmp file now... | 20260505 15:12:32.064 | INFO | RTMetrics done | 20260505 15:12:32.149 | INFO | Finished a round of Baz IO: Wrote 2.22 GiB in 3.32 seconds | 20260505 15:12:32.149 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.53% / 0.00% / 77.46% | 20260505 15:12:32.149 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.469347% / 1.19209e-05% / 5.96046e-05% | 20260505 15:12:32.150 | INFO | Baz IO overall: | 20260505 15:12:32.150 | INFO | BytesWritten : 2493989813989 | 20260505 15:12:32.150 | INFO | NumEvents : 1185043730462 | 20260505 15:12:33.828 | INFO | Analyzing chunk frames = [4190208,4190720) | 20260505 15:12:33.828 | INFO | MemInfo: { | 20260505 15:12:33.828 | INFO | "gpu" : " 17 GB", | 20260505 15:12:33.828 | INFO | "huge" : "", | 20260505 15:12:33.828 | INFO | "huge_shared" : " 27 GB", | 20260505 15:12:33.828 | INFO | "malloc" : " 14 GB", | 20260505 15:12:33.828 | INFO | "pinned" : " 29 GB" | 20260505 15:12:33.828 | INFO | } | 20260505 15:12:33.829 | INFO | Perf report for the last 5.13448 seconds: | 20260505 15:12:33.829 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:12:33.829 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:33.829 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:33.829 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:33.829 | INFO | ANALYSIS : 64.2%, 3.0, 99.8ms, 0 | 20260505 15:12:33.829 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:33.829 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260505 15:12:33.829 | INFO | PRE_HQ : 61.1%, 1.8, 56.9ms, 0 | 20260505 15:12:33.829 | INFO | BAZWRITER : 64.0%, 4.0, 130.0ms, 0 | 20260505 15:12:33.829 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 15:12:33.829 | INFO | Uploaded 36GiB to GPU in 2.21156s (16.2781GiB/s) | 20260505 15:12:33.829 | INFO | Downloaded 38GiB to GPU in 2.26366s (16.787GiB/s) | 20260505 15:12:38.968 | INFO | Analyzing chunk frames = [4190720,4191232) | 20260505 15:12:38.968 | INFO | MemInfo: { | 20260505 15:12:38.968 | INFO | "gpu" : " 17 GB", | 20260505 15:12:38.968 | INFO | "huge" : "", | 20260505 15:12:38.968 | INFO | "huge_shared" : " 27 GB", | 20260505 15:12:38.968 | INFO | "malloc" : " 14 GB", | 20260505 15:12:38.968 | INFO | "pinned" : " 29 GB" | 20260505 15:12:38.968 | INFO | } | 20260505 15:12:38.968 | INFO | Perf report for the last 5.13988 seconds: | 20260505 15:12:38.968 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:12:38.968 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:38.968 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:38.968 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:38.969 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 15:12:38.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:38.969 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:38.969 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:12:38.969 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:38.969 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:38.969 | INFO | Uploaded 36GiB to GPU in 2.19885s (16.3722GiB/s) | 20260505 15:12:38.969 | INFO | Downloaded 34GiB to GPU in 2.04135s (16.6557GiB/s) | 20260505 15:12:44.119 | INFO | Analyzing chunk frames = [4191232,4191744) | 20260505 15:12:44.119 | INFO | MemInfo: { | 20260505 15:12:44.119 | INFO | "gpu" : " 17 GB", | 20260505 15:12:44.119 | INFO | "huge" : "", | 20260505 15:12:44.119 | INFO | "huge_shared" : " 27 GB", | 20260505 15:12:44.119 | INFO | "malloc" : " 14 GB", | 20260505 15:12:44.119 | INFO | "pinned" : " 29 GB" | 20260505 15:12:44.119 | INFO | } | 20260505 15:12:44.119 | INFO | Perf report for the last 5.15058 seconds: | 20260505 15:12:44.119 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:12:44.119 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:44.119 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:44.119 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:44.119 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 15:12:44.119 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:44.119 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:44.119 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:12:44.119 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:44.119 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:44.119 | INFO | Uploaded 36GiB to GPU in 2.23821s (16.0843GiB/s) | 20260505 15:12:44.119 | INFO | Downloaded 34GiB to GPU in 2.1048s (16.1536GiB/s) | 20260505 15:12:49.258 | INFO | Analyzing chunk frames = [4191744,4192256) | 20260505 15:12:49.258 | INFO | MemInfo: { | 20260505 15:12:49.258 | INFO | "gpu" : " 17 GB", | 20260505 15:12:49.258 | INFO | "huge" : "", | 20260505 15:12:49.258 | INFO | "huge_shared" : " 27 GB", | 20260505 15:12:49.258 | INFO | "malloc" : " 14 GB", | 20260505 15:12:49.258 | INFO | "pinned" : " 29 GB" | 20260505 15:12:49.258 | INFO | } | 20260505 15:12:49.258 | INFO | Perf report for the last 5.1393 seconds: | 20260505 15:12:49.258 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:12:49.258 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:49.258 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:49.258 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:49.258 | INFO | ANALYSIS : 62.6%, 3.0, 97.3ms, 0 | 20260505 15:12:49.258 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:49.258 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:49.258 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:12:49.258 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:49.258 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:49.259 | INFO | Uploaded 36GiB to GPU in 2.19166s (16.4259GiB/s) | 20260505 15:12:49.259 | INFO | Downloaded 34GiB to GPU in 2.13588s (15.9185GiB/s) | 20260505 15:12:54.386 | INFO | Analyzing chunk frames = [4192256,4192768) | 20260505 15:12:54.386 | INFO | MemInfo: { | 20260505 15:12:54.386 | INFO | "gpu" : " 17 GB", | 20260505 15:12:54.386 | INFO | "huge" : "", | 20260505 15:12:54.386 | INFO | "huge_shared" : " 27 GB", | 20260505 15:12:54.386 | INFO | "malloc" : " 14 GB", | 20260505 15:12:54.386 | INFO | "pinned" : " 29 GB" | 20260505 15:12:54.387 | INFO | } | 20260505 15:12:54.387 | INFO | Perf report for the last 5.12837 seconds: | 20260505 15:12:54.387 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:12:54.387 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:54.387 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:54.387 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:54.387 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260505 15:12:54.387 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:54.387 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:54.387 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:12:54.387 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:54.387 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:54.387 | INFO | Uploaded 36GiB to GPU in 2.13315s (16.8765GiB/s) | 20260505 15:12:54.387 | INFO | Downloaded 34GiB to GPU in 2.05187s (16.5703GiB/s) | 20260505 15:12:59.522 | INFO | Analyzing chunk frames = [4192768,4193280) | 20260505 15:12:59.522 | INFO | MemInfo: { | 20260505 15:12:59.522 | INFO | "gpu" : " 17 GB", | 20260505 15:12:59.522 | INFO | "huge" : "", | 20260505 15:12:59.522 | INFO | "huge_shared" : " 27 GB", | 20260505 15:12:59.522 | INFO | "malloc" : " 14 GB", | 20260505 15:12:59.522 | INFO | "pinned" : " 29 GB" | 20260505 15:12:59.522 | INFO | } | 20260505 15:12:59.522 | INFO | Perf report for the last 5.13526 seconds: | 20260505 15:12:59.522 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:12:59.522 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:59.522 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:59.522 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:59.522 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 15:12:59.522 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:59.522 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:59.522 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 15:12:59.522 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:59.522 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:12:59.522 | INFO | Uploaded 36GiB to GPU in 2.18948s (16.4422GiB/s) | 20260505 15:12:59.522 | INFO | Downloaded 34GiB to GPU in 2.1146s (16.0787GiB/s) | 20260505 15:13:04.667 | INFO | Analyzing chunk frames = [4193280,4193792) | 20260505 15:13:04.667 | INFO | MemInfo: { | 20260505 15:13:04.667 | INFO | "gpu" : " 17 GB", | 20260505 15:13:04.667 | INFO | "huge" : "", | 20260505 15:13:04.667 | INFO | "huge_shared" : " 27 GB", | 20260505 15:13:04.667 | INFO | "malloc" : " 14 GB", | 20260505 15:13:04.667 | INFO | "pinned" : " 29 GB" | 20260505 15:13:04.667 | INFO | } | 20260505 15:13:04.668 | INFO | Perf report for the last 5.14557 seconds: | 20260505 15:13:04.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:13:04.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:04.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:04.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:04.668 | INFO | ANALYSIS : 63.2%, 3.0, 97.7ms, 0 | 20260505 15:13:04.668 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:04.668 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:04.668 | INFO | PRE_HQ : 28.9%, 1.0, 14.9ms, 0 | 20260505 15:13:04.668 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:04.668 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:04.668 | INFO | Uploaded 36GiB to GPU in 2.13936s (16.8275GiB/s) | 20260505 15:13:04.668 | INFO | Downloaded 34GiB to GPU in 2.18155s (15.5853GiB/s) | 20260505 15:13:09.812 | INFO | Analyzing chunk frames = [4193792,4194304) | 20260505 15:13:09.812 | INFO | MemInfo: { | 20260505 15:13:09.812 | INFO | "gpu" : " 17 GB", | 20260505 15:13:09.812 | INFO | "huge" : "", | 20260505 15:13:09.812 | INFO | "huge_shared" : " 27 GB", | 20260505 15:13:09.812 | INFO | "malloc" : " 14 GB", | 20260505 15:13:09.812 | INFO | "pinned" : " 29 GB" | 20260505 15:13:09.812 | INFO | } | 20260505 15:13:09.812 | INFO | Perf report for the last 5.14452 seconds: | 20260505 15:13:09.812 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:13:09.812 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:09.812 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:09.812 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:09.812 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 15:13:09.812 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:09.812 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:09.812 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 15:13:09.812 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:09.812 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:09.812 | INFO | Uploaded 36GiB to GPU in 2.1602s (16.6651GiB/s) | 20260505 15:13:09.812 | INFO | Downloaded 34GiB to GPU in 2.03468s (16.7103GiB/s) | 20260505 15:13:09.812 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 15:13:09.812 | INFO | Total mem usage: 15.3 GiB | 20260505 15:13:09.812 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 15:13:09.812 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.85%) | 20260505 15:13:09.812 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 15:13:09.812 | INFO | High Water Breakdown: | 20260505 15:13:09.812 | INFO | 61% : 8.67 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 15:13:09.813 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 15:13:09.813 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 15:13:09.813 | INFO | ------------------------------------------ | 20260505 15:13:09.813 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:13:09.813 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 15:13:09.813 | INFO | Total mem usage: 26.5 GiB | 20260505 15:13:09.813 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 15:13:09.813 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 15:13:09.813 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 15:13:09.813 | INFO | High Water Breakdown: | 20260505 15:13:09.813 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:13:09.813 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:13:09.813 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:13:09.813 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:13:09.813 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:13:09.813 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:13:09.813 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 15:13:09.813 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 15:13:09.813 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:13:09.813 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 15:13:09.813 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 15:13:09.813 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 15:13:09.813 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 15:13:09.813 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 15:13:09.813 | INFO | ------------------------------------------ | 20260505 15:13:09.813 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:13:09.813 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 15:13:09.813 | INFO | Total mem usage: 25.6 GiB | 20260505 15:13:09.813 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 15:13:09.813 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 15:13:09.813 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 15:13:09.813 | INFO | High Water Breakdown: | 20260505 15:13:09.813 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 15:13:09.813 | INFO | ------------------------------------------ | 20260505 15:13:09.813 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:13:09.813 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 15:13:09.813 | INFO | Total mem usage: 15.7 GiB | 20260505 15:13:09.813 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 15:13:09.813 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 15:13:09.813 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 15:13:09.813 | INFO | High Water Breakdown: | 20260505 15:13:09.813 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 15:13:09.813 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 15:13:09.813 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 15:13:09.813 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 15:13:09.813 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 15:13:09.813 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 15:13:09.813 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 15:13:09.813 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 15:13:09.813 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 15:13:09.813 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:13:09.813 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 15:13:09.813 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:13:09.813 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 15:13:09.813 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 15:13:09.813 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 15:13:09.813 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 15:13:09.813 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:13:09.813 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 15:13:09.813 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:13:09.813 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:13:09.813 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:13:09.813 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 15:13:09.813 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 15:13:09.813 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:13:09.813 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 15:13:09.813 | INFO | ------------------------------------------ | 20260505 15:13:09.813 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:13:09.813 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 15:13:09.813 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:13:09.813 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:13:09.814 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.59e+01, 1.38e+01 / 1.84e+00 | 20260505 15:13:09.814 | INFO | PerfReport: 0.29%, AggregateStats -- 9.27e-02 / 1.01e-01, 1.07e-02 / 4.73e-03 | 20260505 15:13:09.814 | INFO | PerfReport: 11.75%, Baseline -- 3.73e+00 / 4.49e+00, 5.65e-01 / 1.55e-01 | 20260505 15:13:09.814 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.49e+00, 1.94e-01 / 9.07e-02 | 20260505 15:13:09.814 | INFO | PerfReport: 15.96%, DME -- 4.05e+01 / 4.53e+01, 4.39e+00 / 1.82e+00 | 20260505 15:13:09.814 | INFO | PerfReport: 56.60%, FrameLabeling -- 1.80e+01 / 1.98e+01, 1.46e+00 / 1.62e+00 | 20260505 15:13:09.814 | INFO | PerfReport: 7.42%, PulseAccumulating -- 2.35e+00 / 2.91e+00, 4.48e-01 / 1.54e-01 | 20260505 15:13:09.814 | INFO | PerfReport: 3.48%, Metrics -- 1.10e+00 / 1.43e+00, 3.07e-01 / 1.68e-01 | 20260505 15:13:09.814 | INFO | PerfReport: 1.22%, PulseSerializing -- 3.88e-01 / 4.45e-01, 6.90e-02 / 4.49e-02 | 20260505 15:13:09.814 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 15:13:09.814 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:13:09.814 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:13:09.814 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.56e+01 / 0.00e+00 | 20260505 15:13:09.814 | INFO | PerfReport: 22.61%, Upload -- 2.28e+01 / 0.00e+00, 8.10e+00 / 0.00e+00 | 20260505 15:13:09.814 | INFO | PerfReport: 21.96%, Download -- 2.21e+01 / 0.00e+00, 8.04e+00 / 0.00e+00 | 20260505 15:13:09.814 | INFO | PerfReport: 31.54%, Compute -- 3.17e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 15:13:09.814 | INFO | PerfReport: 2.33%, LockUpload -- 2.35e+00 / 0.00e+00, 7.97e+00 / 0.00e+00 | 20260505 15:13:09.814 | INFO | PerfReport: 1.40%, LockDownload -- 1.41e+00 / 0.00e+00, 2.92e+00 / 0.00e+00 | 20260505 15:13:09.814 | INFO | PerfReport: 20.14%, LockCompute -- 2.03e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 15:13:09.814 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.93e-02 / 0.00e+00 | 20260505 15:13:09.946 | INFO | Starting a round of Baz IO | 20260505 15:13:13.158 | INFO | Populating RTMetrics report | 20260505 15:13:13.158 | INFO | RTMetrics report populated | 20260505 15:13:13.158 | INFO | Opening tmp RTMetrics json file | 20260505 15:13:13.171 | INFO | Serializing RTMetrics to json | 20260505 15:13:13.172 | INFO | Serialization done, swapping tmp file now... | 20260505 15:13:13.172 | INFO | RTMetrics done | 20260505 15:13:13.260 | INFO | Finished a round of Baz IO: Wrote 2.22 GiB in 3.313 seconds | 20260505 15:13:13.260 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.53% / 0.00% / 77.47% | 20260505 15:13:13.260 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.468163% / 0% / 7.54992e-05% | 20260505 15:13:13.260 | INFO | Baz IO overall: | 20260505 15:13:13.260 | INFO | BytesWritten : 2496373230935 | 20260505 15:13:13.260 | INFO | NumEvents : 1186164323188 | 20260505 15:13:14.950 | INFO | Analyzing chunk frames = [4194304,4194816) | 20260505 15:13:14.950 | INFO | MemInfo: { | 20260505 15:13:14.950 | INFO | "gpu" : " 17 GB", | 20260505 15:13:14.950 | INFO | "huge" : "", | 20260505 15:13:14.950 | INFO | "huge_shared" : " 27 GB", | 20260505 15:13:14.950 | INFO | "malloc" : " 14 GB", | 20260505 15:13:14.950 | INFO | "pinned" : " 29 GB" | 20260505 15:13:14.950 | INFO | } | 20260505 15:13:14.950 | INFO | Perf report for the last 5.13802 seconds: | 20260505 15:13:14.950 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:13:14.950 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:14.950 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:14.950 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:14.950 | INFO | ANALYSIS : 64.0%, 3.0, 99.6ms, 0 | 20260505 15:13:14.950 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:14.950 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260505 15:13:14.950 | INFO | PRE_HQ : 61.0%, 1.8, 57.0ms, 0 | 20260505 15:13:14.950 | INFO | BAZWRITER : 63.8%, 3.9, 129.3ms, 0 | 20260505 15:13:14.950 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 15:13:14.951 | INFO | Uploaded 36GiB to GPU in 2.18349s (16.4874GiB/s) | 20260505 15:13:14.951 | INFO | Downloaded 38GiB to GPU in 2.27316s (16.7168GiB/s) | 20260505 15:13:20.083 | INFO | Analyzing chunk frames = [4194816,4195328) | 20260505 15:13:20.083 | INFO | MemInfo: { | 20260505 15:13:20.083 | INFO | "gpu" : " 17 GB", | 20260505 15:13:20.083 | INFO | "huge" : "", | 20260505 15:13:20.083 | INFO | "huge_shared" : " 27 GB", | 20260505 15:13:20.083 | INFO | "malloc" : " 14 GB", | 20260505 15:13:20.083 | INFO | "pinned" : " 29 GB" | 20260505 15:13:20.083 | INFO | } | 20260505 15:13:20.083 | INFO | Perf report for the last 5.13298 seconds: | 20260505 15:13:20.083 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:13:20.083 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:20.083 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:20.083 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:20.083 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 15:13:20.083 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:20.083 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:20.083 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:13:20.083 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:20.083 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:20.083 | INFO | Uploaded 36GiB to GPU in 2.19793s (16.379GiB/s) | 20260505 15:13:20.083 | INFO | Downloaded 34GiB to GPU in 2.05269s (16.5636GiB/s) | 20260505 15:13:25.223 | INFO | Analyzing chunk frames = [4195328,4195840) | 20260505 15:13:25.223 | INFO | MemInfo: { | 20260505 15:13:25.223 | INFO | "gpu" : " 17 GB", | 20260505 15:13:25.223 | INFO | "huge" : "", | 20260505 15:13:25.223 | INFO | "huge_shared" : " 27 GB", | 20260505 15:13:25.223 | INFO | "malloc" : " 14 GB", | 20260505 15:13:25.223 | INFO | "pinned" : " 29 GB" | 20260505 15:13:25.223 | INFO | } | 20260505 15:13:25.223 | INFO | Perf report for the last 5.13988 seconds: | 20260505 15:13:25.223 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:13:25.223 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:25.223 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:25.223 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:25.223 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 15:13:25.223 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:25.223 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:25.223 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:13:25.223 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:25.223 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:25.223 | INFO | Uploaded 36GiB to GPU in 2.23379s (16.1161GiB/s) | 20260505 15:13:25.223 | INFO | Downloaded 34GiB to GPU in 2.10647s (16.1407GiB/s) | 20260505 15:13:30.366 | INFO | Analyzing chunk frames = [4195840,4196352) | 20260505 15:13:30.366 | INFO | MemInfo: { | 20260505 15:13:30.366 | INFO | "gpu" : " 17 GB", | 20260505 15:13:30.366 | INFO | "huge" : "", | 20260505 15:13:30.366 | INFO | "huge_shared" : " 27 GB", | 20260505 15:13:30.366 | INFO | "malloc" : " 14 GB", | 20260505 15:13:30.366 | INFO | "pinned" : " 29 GB" | 20260505 15:13:30.366 | INFO | } | 20260505 15:13:30.366 | INFO | Perf report for the last 5.14296 seconds: | 20260505 15:13:30.366 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:13:30.366 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:30.366 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:30.366 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:30.366 | INFO | ANALYSIS : 62.7%, 3.0, 97.4ms, 0 | 20260505 15:13:30.366 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:30.366 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:30.366 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 15:13:30.366 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:30.366 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:30.366 | INFO | Uploaded 36GiB to GPU in 2.18365s (16.4861GiB/s) | 20260505 15:13:30.366 | INFO | Downloaded 34GiB to GPU in 2.13337s (15.9372GiB/s) | 20260505 15:13:35.508 | INFO | Analyzing chunk frames = [4196352,4196864) | 20260505 15:13:35.508 | INFO | MemInfo: { | 20260505 15:13:35.508 | INFO | "gpu" : " 17 GB", | 20260505 15:13:35.508 | INFO | "huge" : "", | 20260505 15:13:35.508 | INFO | "huge_shared" : " 27 GB", | 20260505 15:13:35.508 | INFO | "malloc" : " 14 GB", | 20260505 15:13:35.508 | INFO | "pinned" : " 29 GB" | 20260505 15:13:35.508 | INFO | } | 20260505 15:13:35.508 | INFO | Perf report for the last 5.14237 seconds: | 20260505 15:13:35.508 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:13:35.508 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:35.508 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:35.508 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:35.508 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 15:13:35.508 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:35.508 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:35.508 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 15:13:35.508 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:35.509 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:35.509 | INFO | Uploaded 36GiB to GPU in 2.13589s (16.8548GiB/s) | 20260505 15:13:35.509 | INFO | Downloaded 34GiB to GPU in 2.05183s (16.5706GiB/s) | 20260505 15:13:40.643 | INFO | Analyzing chunk frames = [4196864,4197376) | 20260505 15:13:40.643 | INFO | MemInfo: { | 20260505 15:13:40.643 | INFO | "gpu" : " 17 GB", | 20260505 15:13:40.643 | INFO | "huge" : "", | 20260505 15:13:40.643 | INFO | "huge_shared" : " 27 GB", | 20260505 15:13:40.643 | INFO | "malloc" : " 14 GB", | 20260505 15:13:40.643 | INFO | "pinned" : " 29 GB" | 20260505 15:13:40.643 | INFO | } | 20260505 15:13:40.643 | INFO | Perf report for the last 5.13491 seconds: | 20260505 15:13:40.643 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:13:40.643 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:40.643 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:40.643 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:40.643 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 15:13:40.643 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:40.643 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:40.643 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 15:13:40.643 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:40.643 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:40.644 | INFO | Uploaded 36GiB to GPU in 2.13447s (16.866GiB/s) | 20260505 15:13:40.644 | INFO | Downloaded 34GiB to GPU in 2.03163s (16.7353GiB/s) | 20260505 15:13:45.777 | INFO | Analyzing chunk frames = [4197376,4197888) | 20260505 15:13:45.777 | INFO | MemInfo: { | 20260505 15:13:45.777 | INFO | "gpu" : " 17 GB", | 20260505 15:13:45.777 | INFO | "huge" : "", | 20260505 15:13:45.777 | INFO | "huge_shared" : " 27 GB", | 20260505 15:13:45.777 | INFO | "malloc" : " 14 GB", | 20260505 15:13:45.777 | INFO | "pinned" : " 29 GB" | 20260505 15:13:45.777 | INFO | } | 20260505 15:13:45.777 | INFO | Perf report for the last 5.13386 seconds: | 20260505 15:13:45.777 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:13:45.777 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:45.777 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:45.777 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:45.777 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 15:13:45.777 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:45.777 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:45.777 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 15:13:45.777 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:45.777 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:45.777 | INFO | Uploaded 36GiB to GPU in 2.16044s (16.6633GiB/s) | 20260505 15:13:45.778 | INFO | Downloaded 34GiB to GPU in 2.15327s (15.7899GiB/s) | 20260505 15:13:50.927 | INFO | Analyzing chunk frames = [4197888,4198400) | 20260505 15:13:50.927 | INFO | MemInfo: { | 20260505 15:13:50.927 | INFO | "gpu" : " 17 GB", | 20260505 15:13:50.927 | INFO | "huge" : "", | 20260505 15:13:50.927 | INFO | "huge_shared" : " 27 GB", | 20260505 15:13:50.927 | INFO | "malloc" : " 14 GB", | 20260505 15:13:50.927 | INFO | "pinned" : " 29 GB" | 20260505 15:13:50.927 | INFO | } | 20260505 15:13:50.927 | INFO | Perf report for the last 5.14995 seconds: | 20260505 15:13:50.927 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:13:50.927 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:50.927 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:50.927 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:50.927 | INFO | ANALYSIS : 63.5%, 3.0, 99.3ms, 0 | 20260505 15:13:50.927 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:50.927 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:50.927 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 15:13:50.927 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:50.927 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:50.927 | INFO | Uploaded 36GiB to GPU in 2.22047s (16.2128GiB/s) | 20260505 15:13:50.927 | INFO | Downloaded 34GiB to GPU in 2.10611s (16.1435GiB/s) | 20260505 15:13:51.060 | INFO | Starting a round of Baz IO | 20260505 15:13:54.265 | INFO | Populating RTMetrics report | 20260505 15:13:54.265 | INFO | RTMetrics report populated | 20260505 15:13:54.265 | INFO | Opening tmp RTMetrics json file | 20260505 15:13:54.295 | INFO | Serializing RTMetrics to json | 20260505 15:13:54.311 | INFO | Serialization done, swapping tmp file now... | 20260505 15:13:54.319 | INFO | RTMetrics done | 20260505 15:13:54.371 | INFO | Finished a round of Baz IO: Wrote 2.221 GiB in 3.31 seconds | 20260505 15:13:54.371 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.53% / 0.00% / 77.47% | 20260505 15:13:54.371 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.468012% / 0% / 0.000103315% | 20260505 15:13:54.371 | INFO | Baz IO overall: | 20260505 15:13:54.371 | INFO | BytesWritten : 2498758047237 | 20260505 15:13:54.371 | INFO | NumEvents : 1187286068023 | 20260505 15:13:56.067 | INFO | Analyzing chunk frames = [4198400,4198912) | 20260505 15:13:56.067 | INFO | MemInfo: { | 20260505 15:13:56.067 | INFO | "gpu" : " 17 GB", | 20260505 15:13:56.067 | INFO | "huge" : "", | 20260505 15:13:56.067 | INFO | "huge_shared" : " 27 GB", | 20260505 15:13:56.067 | INFO | "malloc" : " 14 GB", | 20260505 15:13:56.067 | INFO | "pinned" : " 29 GB" | 20260505 15:13:56.067 | INFO | } | 20260505 15:13:56.067 | INFO | Perf report for the last 5.13968 seconds: | 20260505 15:13:56.067 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:13:56.067 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:56.067 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:56.067 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:56.067 | INFO | ANALYSIS : 63.8%, 3.0, 99.2ms, 0 | 20260505 15:13:56.067 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:13:56.067 | INFO | RT_METRICS : 45.9%, 1.0, 24.0ms, 0 | 20260505 15:13:56.067 | INFO | PRE_HQ : 61.0%, 1.8, 57.1ms, 0 | 20260505 15:13:56.067 | INFO | BAZWRITER : 63.7%, 4.0, 130.6ms, 0 | 20260505 15:13:56.067 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 15:13:56.067 | INFO | Uploaded 36GiB to GPU in 2.16177s (16.6531GiB/s) | 20260505 15:13:56.067 | INFO | Downloaded 38GiB to GPU in 2.2604s (16.8112GiB/s) | 20260505 15:14:01.202 | INFO | Analyzing chunk frames = [4198912,4199424) | 20260505 15:14:01.202 | INFO | MemInfo: { | 20260505 15:14:01.202 | INFO | "gpu" : " 17 GB", | 20260505 15:14:01.202 | INFO | "huge" : "", | 20260505 15:14:01.202 | INFO | "huge_shared" : " 27 GB", | 20260505 15:14:01.202 | INFO | "malloc" : " 14 GB", | 20260505 15:14:01.202 | INFO | "pinned" : " 29 GB" | 20260505 15:14:01.202 | INFO | } | 20260505 15:14:01.202 | INFO | Perf report for the last 5.13511 seconds: | 20260505 15:14:01.202 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:14:01.202 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:01.202 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:01.202 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:01.202 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 15:14:01.202 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:01.202 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:01.202 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 15:14:01.202 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:01.202 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:01.202 | INFO | Uploaded 36GiB to GPU in 2.20261s (16.3442GiB/s) | 20260505 15:14:01.202 | INFO | Downloaded 34GiB to GPU in 2.05854s (16.5165GiB/s) | 20260505 15:14:06.334 | INFO | Analyzing chunk frames = [4199424,4199936) | 20260505 15:14:06.334 | INFO | MemInfo: { | 20260505 15:14:06.334 | INFO | "gpu" : " 17 GB", | 20260505 15:14:06.334 | INFO | "huge" : "", | 20260505 15:14:06.334 | INFO | "huge_shared" : " 27 GB", | 20260505 15:14:06.334 | INFO | "malloc" : " 14 GB", | 20260505 15:14:06.334 | INFO | "pinned" : " 29 GB" | 20260505 15:14:06.334 | INFO | } | 20260505 15:14:06.334 | INFO | Perf report for the last 5.13228 seconds: | 20260505 15:14:06.334 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:14:06.334 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:06.334 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:06.334 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:06.334 | INFO | ANALYSIS : 63.0%, 3.0, 97.9ms, 0 | 20260505 15:14:06.334 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:06.334 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:06.334 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 15:14:06.334 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:06.334 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:06.335 | INFO | Uploaded 36GiB to GPU in 2.18893s (16.4464GiB/s) | 20260505 15:14:06.335 | INFO | Downloaded 34GiB to GPU in 2.04657s (16.6131GiB/s) | 20260505 15:14:11.478 | INFO | Analyzing chunk frames = [4199936,4200448) | 20260505 15:14:11.478 | INFO | MemInfo: { | 20260505 15:14:11.478 | INFO | "gpu" : " 17 GB", | 20260505 15:14:11.478 | INFO | "huge" : "", | 20260505 15:14:11.478 | INFO | "huge_shared" : " 27 GB", | 20260505 15:14:11.478 | INFO | "malloc" : " 14 GB", | 20260505 15:14:11.478 | INFO | "pinned" : " 29 GB" | 20260505 15:14:11.478 | INFO | } | 20260505 15:14:11.478 | INFO | Perf report for the last 5.14363 seconds: | 20260505 15:14:11.478 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:14:11.478 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:11.478 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:11.478 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:11.478 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260505 15:14:11.478 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:11.478 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:11.478 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:14:11.478 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:11.478 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:11.478 | INFO | Uploaded 36GiB to GPU in 2.18667s (16.4634GiB/s) | 20260505 15:14:11.478 | INFO | Downloaded 34GiB to GPU in 2.13481s (15.9264GiB/s) | 20260505 15:14:16.621 | INFO | Analyzing chunk frames = [4200448,4200960) | 20260505 15:14:16.621 | INFO | MemInfo: { | 20260505 15:14:16.621 | INFO | "gpu" : " 17 GB", | 20260505 15:14:16.621 | INFO | "huge" : "", | 20260505 15:14:16.621 | INFO | "huge_shared" : " 27 GB", | 20260505 15:14:16.621 | INFO | "malloc" : " 14 GB", | 20260505 15:14:16.621 | INFO | "pinned" : " 29 GB" | 20260505 15:14:16.621 | INFO | } | 20260505 15:14:16.621 | INFO | Perf report for the last 5.14349 seconds: | 20260505 15:14:16.621 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:14:16.621 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:16.621 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:16.621 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:16.621 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260505 15:14:16.621 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:16.621 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:16.621 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 15:14:16.622 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:16.622 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:16.622 | INFO | Uploaded 36GiB to GPU in 2.17227s (16.5725GiB/s) | 20260505 15:14:16.622 | INFO | Downloaded 34GiB to GPU in 2.13311s (15.9392GiB/s) | 20260505 15:14:21.764 | INFO | Analyzing chunk frames = [4200960,4201472) | 20260505 15:14:21.764 | INFO | MemInfo: { | 20260505 15:14:21.764 | INFO | "gpu" : " 17 GB", | 20260505 15:14:21.764 | INFO | "huge" : "", | 20260505 15:14:21.764 | INFO | "huge_shared" : " 27 GB", | 20260505 15:14:21.764 | INFO | "malloc" : " 14 GB", | 20260505 15:14:21.764 | INFO | "pinned" : " 29 GB" | 20260505 15:14:21.764 | INFO | } | 20260505 15:14:21.764 | INFO | Perf report for the last 5.14276 seconds: | 20260505 15:14:21.764 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:14:21.764 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:21.764 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:21.764 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:21.764 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260505 15:14:21.764 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:21.764 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:21.764 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 15:14:21.764 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:21.764 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:21.764 | INFO | Uploaded 36GiB to GPU in 2.14295s (16.7993GiB/s) | 20260505 15:14:21.764 | INFO | Downloaded 34GiB to GPU in 2.03397s (16.7161GiB/s) | 20260505 15:14:26.896 | INFO | Analyzing chunk frames = [4201472,4201984) | 20260505 15:14:26.896 | INFO | MemInfo: { | 20260505 15:14:26.896 | INFO | "gpu" : " 17 GB", | 20260505 15:14:26.896 | INFO | "huge" : "", | 20260505 15:14:26.896 | INFO | "huge_shared" : " 27 GB", | 20260505 15:14:26.896 | INFO | "malloc" : " 14 GB", | 20260505 15:14:26.896 | INFO | "pinned" : " 29 GB" | 20260505 15:14:26.896 | INFO | } | 20260505 15:14:26.896 | INFO | Perf report for the last 5.13231 seconds: | 20260505 15:14:26.896 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:14:26.896 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:26.896 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:26.896 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:26.897 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 15:14:26.897 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:26.897 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:26.897 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 15:14:26.897 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:26.897 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:26.897 | INFO | Uploaded 36GiB to GPU in 2.1119s (17.0462GiB/s) | 20260505 15:14:26.897 | INFO | Downloaded 34GiB to GPU in 2.16625s (15.6953GiB/s) | 20260505 15:14:32.031 | INFO | Analyzing chunk frames = [4201984,4202496) | 20260505 15:14:32.031 | INFO | MemInfo: { | 20260505 15:14:32.031 | INFO | "gpu" : " 17 GB", | 20260505 15:14:32.031 | INFO | "huge" : "", | 20260505 15:14:32.031 | INFO | "huge_shared" : " 27 GB", | 20260505 15:14:32.031 | INFO | "malloc" : " 14 GB", | 20260505 15:14:32.031 | INFO | "pinned" : " 29 GB" | 20260505 15:14:32.031 | INFO | } | 20260505 15:14:32.031 | INFO | Perf report for the last 5.13441 seconds: | 20260505 15:14:32.031 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:14:32.031 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:32.031 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:32.031 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:32.031 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 15:14:32.031 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:32.031 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:32.031 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 15:14:32.031 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:32.031 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:32.031 | INFO | Uploaded 36GiB to GPU in 2.21516s (16.2517GiB/s) | 20260505 15:14:32.031 | INFO | Downloaded 34GiB to GPU in 2.10797s (16.1293GiB/s) | 20260505 15:14:32.031 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 15:14:32.031 | INFO | Total mem usage: 15.3 GiB | 20260505 15:14:32.031 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 15:14:32.031 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.85%) | 20260505 15:14:32.031 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 15:14:32.031 | INFO | High Water Breakdown: | 20260505 15:14:32.031 | INFO | 61% : 8.67 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 15:14:32.031 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 15:14:32.031 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 15:14:32.031 | INFO | ------------------------------------------ | 20260505 15:14:32.031 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:14:32.031 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 15:14:32.031 | INFO | Total mem usage: 26.5 GiB | 20260505 15:14:32.031 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 15:14:32.032 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 15:14:32.032 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 15:14:32.032 | INFO | High Water Breakdown: | 20260505 15:14:32.032 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:14:32.032 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:14:32.032 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:14:32.032 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:14:32.032 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:14:32.032 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:14:32.032 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 15:14:32.032 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 15:14:32.032 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:14:32.032 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 15:14:32.032 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 15:14:32.032 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 15:14:32.032 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 15:14:32.032 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 15:14:32.032 | INFO | ------------------------------------------ | 20260505 15:14:32.032 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:14:32.032 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 15:14:32.032 | INFO | Total mem usage: 25.6 GiB | 20260505 15:14:32.032 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 15:14:32.032 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 15:14:32.032 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 15:14:32.032 | INFO | High Water Breakdown: | 20260505 15:14:32.032 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 15:14:32.032 | INFO | ------------------------------------------ | 20260505 15:14:32.032 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:14:32.032 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 15:14:32.032 | INFO | Total mem usage: 15.7 GiB | 20260505 15:14:32.032 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 15:14:32.032 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 15:14:32.032 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 15:14:32.032 | INFO | High Water Breakdown: | 20260505 15:14:32.032 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 15:14:32.032 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 15:14:32.032 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 15:14:32.032 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 15:14:32.032 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 15:14:32.032 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 15:14:32.032 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 15:14:32.032 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 15:14:32.032 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 15:14:32.032 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:14:32.032 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 15:14:32.032 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:14:32.032 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 15:14:32.032 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 15:14:32.032 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 15:14:32.032 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 15:14:32.032 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:14:32.032 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 15:14:32.032 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:14:32.032 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:14:32.032 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:14:32.032 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 15:14:32.032 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 15:14:32.032 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:14:32.032 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 15:14:32.032 | INFO | ------------------------------------------ | 20260505 15:14:32.032 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:14:32.032 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 15:14:32.032 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:14:32.032 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:14:32.032 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.57e+01, 1.34e+01 / 1.75e+00 | 20260505 15:14:32.033 | INFO | PerfReport: 0.29%, AggregateStats -- 9.26e-02 / 1.01e-01, 1.04e-02 / 5.46e-03 | 20260505 15:14:32.033 | INFO | PerfReport: 11.77%, Baseline -- 3.74e+00 / 4.48e+00, 5.80e-01 / 2.15e-01 | 20260505 15:14:32.033 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.50e+00, 1.95e-01 / 1.12e-01 | 20260505 15:14:32.033 | INFO | PerfReport: 15.91%, DME -- 4.04e+01 / 4.49e+01, 4.33e+00 / 1.97e+00 | 20260505 15:14:32.033 | INFO | PerfReport: 56.64%, FrameLabeling -- 1.80e+01 / 2.00e+01, 1.48e+00 / 1.80e+00 | 20260505 15:14:32.033 | INFO | PerfReport: 7.39%, PulseAccumulating -- 2.35e+00 / 2.88e+00, 4.43e-01 / 2.02e-01 | 20260505 15:14:32.033 | INFO | PerfReport: 3.49%, Metrics -- 1.11e+00 / 1.43e+00, 3.08e-01 / 1.62e-01 | 20260505 15:14:32.033 | INFO | PerfReport: 1.22%, PulseSerializing -- 3.88e-01 / 4.45e-01, 7.00e-02 / 3.19e-02 | 20260505 15:14:32.033 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 15:14:32.033 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:14:32.033 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:14:32.033 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260505 15:14:32.033 | INFO | PerfReport: 22.57%, Upload -- 2.27e+01 / 0.00e+00, 8.01e+00 / 0.00e+00 | 20260505 15:14:32.033 | INFO | PerfReport: 21.94%, Download -- 2.20e+01 / 0.00e+00, 8.00e+00 / 0.00e+00 | 20260505 15:14:32.033 | INFO | PerfReport: 31.61%, Compute -- 3.18e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 15:14:32.033 | INFO | PerfReport: 2.27%, LockUpload -- 2.28e+00 / 0.00e+00, 7.86e+00 / 0.00e+00 | 20260505 15:14:32.033 | INFO | PerfReport: 1.38%, LockDownload -- 1.39e+00 / 0.00e+00, 2.90e+00 / 0.00e+00 | 20260505 15:14:32.033 | INFO | PerfReport: 20.22%, LockCompute -- 2.03e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260505 15:14:32.033 | INFO | PerfReport: 0.01%, Misc -- 1.27e-02 / 0.00e+00, 1.41e-02 / 0.00e+00 | 20260505 15:14:32.167 | INFO | Starting a round of Baz IO | 20260505 15:14:35.384 | INFO | Populating RTMetrics report | 20260505 15:14:35.385 | INFO | RTMetrics report populated | 20260505 15:14:35.385 | INFO | Opening tmp RTMetrics json file | 20260505 15:14:35.394 | INFO | Serializing RTMetrics to json | 20260505 15:14:35.423 | INFO | Serialization done, swapping tmp file now... | 20260505 15:14:35.441 | INFO | RTMetrics done | 20260505 15:14:35.487 | INFO | Finished a round of Baz IO: Wrote 2.22 GiB in 3.319 seconds | 20260505 15:14:35.487 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.53% / 0.00% / 77.47% | 20260505 15:14:35.487 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.466657% / 1.19209e-05% / 7.15256e-05% | 20260505 15:14:35.487 | INFO | Baz IO overall: | 20260505 15:14:35.487 | INFO | BytesWritten : 2501142076196 | 20260505 15:14:35.487 | INFO | NumEvents : 1188407143952 | 20260505 15:14:37.178 | INFO | Analyzing chunk frames = [4202496,4203008) | 20260505 15:14:37.178 | INFO | MemInfo: { | 20260505 15:14:37.178 | INFO | "gpu" : " 17 GB", | 20260505 15:14:37.178 | INFO | "huge" : "", | 20260505 15:14:37.178 | INFO | "huge_shared" : " 27 GB", | 20260505 15:14:37.178 | INFO | "malloc" : " 14 GB", | 20260505 15:14:37.178 | INFO | "pinned" : " 29 GB" | 20260505 15:14:37.178 | INFO | } | 20260505 15:14:37.178 | INFO | Perf report for the last 5.14737 seconds: | 20260505 15:14:37.178 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:14:37.178 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:37.178 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:37.178 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:37.178 | INFO | ANALYSIS : 64.1%, 3.0, 99.8ms, 0 | 20260505 15:14:37.178 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:37.178 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260505 15:14:37.178 | INFO | PRE_HQ : 60.7%, 1.8, 57.1ms, 0 | 20260505 15:14:37.178 | INFO | BAZWRITER : 63.8%, 4.0, 129.8ms, 0 | 20260505 15:14:37.178 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 15:14:37.179 | INFO | Uploaded 36GiB to GPU in 2.18513s (16.475GiB/s) | 20260505 15:14:37.179 | INFO | Downloaded 38GiB to GPU in 2.25999s (16.8142GiB/s) | 20260505 15:14:42.316 | INFO | Analyzing chunk frames = [4203008,4203520) | 20260505 15:14:42.316 | INFO | MemInfo: { | 20260505 15:14:42.316 | INFO | "gpu" : " 17 GB", | 20260505 15:14:42.316 | INFO | "huge" : "", | 20260505 15:14:42.316 | INFO | "huge_shared" : " 27 GB", | 20260505 15:14:42.316 | INFO | "malloc" : " 14 GB", | 20260505 15:14:42.317 | INFO | "pinned" : " 29 GB" | 20260505 15:14:42.317 | INFO | } | 20260505 15:14:42.317 | INFO | Perf report for the last 5.13841 seconds: | 20260505 15:14:42.317 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:14:42.317 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:42.317 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:42.317 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:42.317 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 15:14:42.317 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:42.317 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:42.317 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:14:42.317 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:42.317 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:42.317 | INFO | Uploaded 36GiB to GPU in 2.20495s (16.3269GiB/s) | 20260505 15:14:42.317 | INFO | Downloaded 34GiB to GPU in 2.06068s (16.4994GiB/s) | 20260505 15:14:47.459 | INFO | Analyzing chunk frames = [4203520,4204032) | 20260505 15:14:47.459 | INFO | MemInfo: { | 20260505 15:14:47.459 | INFO | "gpu" : " 17 GB", | 20260505 15:14:47.459 | INFO | "huge" : "", | 20260505 15:14:47.459 | INFO | "huge_shared" : " 27 GB", | 20260505 15:14:47.459 | INFO | "malloc" : " 14 GB", | 20260505 15:14:47.459 | INFO | "pinned" : " 29 GB" | 20260505 15:14:47.459 | INFO | } | 20260505 15:14:47.459 | INFO | Perf report for the last 5.14278 seconds: | 20260505 15:14:47.459 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:14:47.459 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:47.459 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:47.459 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:47.460 | INFO | ANALYSIS : 62.8%, 3.0, 97.8ms, 0 | 20260505 15:14:47.460 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:47.460 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:47.460 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 15:14:47.460 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:47.460 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:47.460 | INFO | Uploaded 36GiB to GPU in 2.17995s (16.5141GiB/s) | 20260505 15:14:47.460 | INFO | Downloaded 34GiB to GPU in 2.03746s (16.6874GiB/s) | 20260505 15:14:52.591 | INFO | Analyzing chunk frames = [4204032,4204544) | 20260505 15:14:52.591 | INFO | MemInfo: { | 20260505 15:14:52.591 | INFO | "gpu" : " 17 GB", | 20260505 15:14:52.591 | INFO | "huge" : "", | 20260505 15:14:52.591 | INFO | "huge_shared" : " 27 GB", | 20260505 15:14:52.591 | INFO | "malloc" : " 14 GB", | 20260505 15:14:52.591 | INFO | "pinned" : " 29 GB" | 20260505 15:14:52.591 | INFO | } | 20260505 15:14:52.591 | INFO | Perf report for the last 5.13197 seconds: | 20260505 15:14:52.591 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:14:52.591 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:52.591 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:52.592 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:52.592 | INFO | ANALYSIS : 62.7%, 3.0, 97.3ms, 0 | 20260505 15:14:52.592 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:52.592 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:52.592 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 15:14:52.592 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:52.592 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:52.592 | INFO | Uploaded 36GiB to GPU in 2.18573s (16.4704GiB/s) | 20260505 15:14:52.592 | INFO | Downloaded 34GiB to GPU in 2.13307s (15.9394GiB/s) | 20260505 15:14:57.729 | INFO | Analyzing chunk frames = [4204544,4205056) | 20260505 15:14:57.729 | INFO | MemInfo: { | 20260505 15:14:57.729 | INFO | "gpu" : " 17 GB", | 20260505 15:14:57.729 | INFO | "huge" : "", | 20260505 15:14:57.729 | INFO | "huge_shared" : " 27 GB", | 20260505 15:14:57.729 | INFO | "malloc" : " 14 GB", | 20260505 15:14:57.729 | INFO | "pinned" : " 29 GB" | 20260505 15:14:57.730 | INFO | } | 20260505 15:14:57.730 | INFO | Perf report for the last 5.13815 seconds: | 20260505 15:14:57.730 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:14:57.730 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:57.730 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:57.730 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:57.730 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 15:14:57.730 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:57.730 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:57.730 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 15:14:57.730 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:57.730 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:14:57.730 | INFO | Uploaded 36GiB to GPU in 2.16064s (16.6617GiB/s) | 20260505 15:14:57.730 | INFO | Downloaded 34GiB to GPU in 2.12594s (15.9929GiB/s) | 20260505 15:15:02.877 | INFO | Analyzing chunk frames = [4205056,4205568) | 20260505 15:15:02.877 | INFO | MemInfo: { | 20260505 15:15:02.877 | INFO | "gpu" : " 17 GB", | 20260505 15:15:02.877 | INFO | "huge" : "", | 20260505 15:15:02.877 | INFO | "huge_shared" : " 27 GB", | 20260505 15:15:02.877 | INFO | "malloc" : " 14 GB", | 20260505 15:15:02.877 | INFO | "pinned" : " 29 GB" | 20260505 15:15:02.877 | INFO | } | 20260505 15:15:02.877 | INFO | Perf report for the last 5.14771 seconds: | 20260505 15:15:02.877 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:15:02.877 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:02.877 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:02.877 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:02.877 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 15:15:02.877 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:02.877 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:02.877 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 15:15:02.877 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:02.877 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:02.878 | INFO | Uploaded 36GiB to GPU in 2.19208s (16.4228GiB/s) | 20260505 15:15:02.878 | INFO | Downloaded 34GiB to GPU in 2.11017s (16.1125GiB/s) | 20260505 15:15:08.014 | INFO | Analyzing chunk frames = [4205568,4206080) | 20260505 15:15:08.014 | INFO | MemInfo: { | 20260505 15:15:08.014 | INFO | "gpu" : " 17 GB", | 20260505 15:15:08.014 | INFO | "huge" : "", | 20260505 15:15:08.014 | INFO | "huge_shared" : " 27 GB", | 20260505 15:15:08.014 | INFO | "malloc" : " 14 GB", | 20260505 15:15:08.014 | INFO | "pinned" : " 29 GB" | 20260505 15:15:08.014 | INFO | } | 20260505 15:15:08.015 | INFO | Perf report for the last 5.13718 seconds: | 20260505 15:15:08.015 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:15:08.015 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:08.015 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:08.015 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:08.015 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 15:15:08.015 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:08.015 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:08.015 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 15:15:08.015 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:08.015 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:08.015 | INFO | Uploaded 36GiB to GPU in 2.14668s (16.7701GiB/s) | 20260505 15:15:08.015 | INFO | Downloaded 34GiB to GPU in 2.15081s (15.808GiB/s) | 20260505 15:15:13.153 | INFO | Analyzing chunk frames = [4206080,4206592) | 20260505 15:15:13.153 | INFO | MemInfo: { | 20260505 15:15:13.153 | INFO | "gpu" : " 17 GB", | 20260505 15:15:13.153 | INFO | "huge" : "", | 20260505 15:15:13.153 | INFO | "huge_shared" : " 27 GB", | 20260505 15:15:13.153 | INFO | "malloc" : " 14 GB", | 20260505 15:15:13.153 | INFO | "pinned" : " 29 GB" | 20260505 15:15:13.153 | INFO | } | 20260505 15:15:13.153 | INFO | Perf report for the last 5.13838 seconds: | 20260505 15:15:13.153 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:15:13.153 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:13.153 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:13.153 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:13.153 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 15:15:13.153 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:13.153 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:13.153 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 15:15:13.153 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:13.153 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:13.153 | INFO | Uploaded 36GiB to GPU in 2.23603s (16.0999GiB/s) | 20260505 15:15:13.153 | INFO | Downloaded 34GiB to GPU in 2.10371s (16.162GiB/s) | 20260505 15:15:13.287 | INFO | Starting a round of Baz IO | 20260505 15:15:16.509 | INFO | Populating RTMetrics report | 20260505 15:15:16.509 | INFO | RTMetrics report populated | 20260505 15:15:16.509 | INFO | Opening tmp RTMetrics json file | 20260505 15:15:16.535 | INFO | Serializing RTMetrics to json | 20260505 15:15:16.569 | INFO | Serialization done, swapping tmp file now... | 20260505 15:15:16.591 | INFO | RTMetrics done | 20260505 15:15:16.608 | INFO | Finished a round of Baz IO: Wrote 2.221 GiB in 3.32 seconds | 20260505 15:15:16.608 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.52% / 0.00% / 77.47% | 20260505 15:15:16.608 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.467344% / 7.94729e-06% / 7.94729e-05% | 20260505 15:15:16.608 | INFO | Baz IO overall: | 20260505 15:15:16.608 | INFO | BytesWritten : 2503526989957 | 20260505 15:15:16.608 | INFO | NumEvents : 1189529384556 | 20260505 15:15:18.286 | INFO | Analyzing chunk frames = [4206592,4207104) | 20260505 15:15:18.286 | INFO | MemInfo: { | 20260505 15:15:18.286 | INFO | "gpu" : " 17 GB", | 20260505 15:15:18.286 | INFO | "huge" : "", | 20260505 15:15:18.286 | INFO | "huge_shared" : " 27 GB", | 20260505 15:15:18.286 | INFO | "malloc" : " 14 GB", | 20260505 15:15:18.286 | INFO | "pinned" : " 29 GB" | 20260505 15:15:18.286 | INFO | } | 20260505 15:15:18.286 | INFO | Perf report for the last 5.13345 seconds: | 20260505 15:15:18.286 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:15:18.286 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:18.286 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:18.287 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:18.287 | INFO | ANALYSIS : 64.3%, 3.0, 99.8ms, 0 | 20260505 15:15:18.287 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:18.287 | INFO | RT_METRICS : 46.5%, 1.0, 24.4ms, 0 | 20260505 15:15:18.287 | INFO | PRE_HQ : 61.0%, 1.8, 57.2ms, 0 | 20260505 15:15:18.287 | INFO | BAZWRITER : 64.0%, 4.0, 130.8ms, 0 | 20260505 15:15:18.287 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 15:15:18.287 | INFO | Uploaded 36GiB to GPU in 2.2008s (16.3577GiB/s) | 20260505 15:15:18.287 | INFO | Downloaded 38GiB to GPU in 2.26643s (16.7665GiB/s) | 20260505 15:15:23.435 | INFO | Analyzing chunk frames = [4207104,4207616) | 20260505 15:15:23.435 | INFO | MemInfo: { | 20260505 15:15:23.435 | INFO | "gpu" : " 17 GB", | 20260505 15:15:23.435 | INFO | "huge" : "", | 20260505 15:15:23.435 | INFO | "huge_shared" : " 27 GB", | 20260505 15:15:23.435 | INFO | "malloc" : " 14 GB", | 20260505 15:15:23.435 | INFO | "pinned" : " 29 GB" | 20260505 15:15:23.436 | INFO | } | 20260505 15:15:23.436 | INFO | Perf report for the last 5.14918 seconds: | 20260505 15:15:23.436 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:15:23.436 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:23.436 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:23.436 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:23.436 | INFO | ANALYSIS : 63.3%, 3.0, 99.1ms, 0 | 20260505 15:15:23.436 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:23.436 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:23.436 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 15:15:23.436 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:23.436 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:23.436 | INFO | Uploaded 36GiB to GPU in 2.20592s (16.3197GiB/s) | 20260505 15:15:23.436 | INFO | Downloaded 34GiB to GPU in 2.04538s (16.6228GiB/s) | 20260505 15:15:28.576 | INFO | Analyzing chunk frames = [4207616,4208128) | 20260505 15:15:28.576 | INFO | MemInfo: { | 20260505 15:15:28.576 | INFO | "gpu" : " 17 GB", | 20260505 15:15:28.576 | INFO | "huge" : "", | 20260505 15:15:28.576 | INFO | "huge_shared" : " 27 GB", | 20260505 15:15:28.576 | INFO | "malloc" : " 14 GB", | 20260505 15:15:28.576 | INFO | "pinned" : " 29 GB" | 20260505 15:15:28.576 | INFO | } | 20260505 15:15:28.576 | INFO | Perf report for the last 5.14047 seconds: | 20260505 15:15:28.576 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:15:28.576 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:28.576 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:28.576 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:28.576 | INFO | ANALYSIS : 62.8%, 3.0, 97.8ms, 0 | 20260505 15:15:28.576 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:28.576 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:28.576 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 15:15:28.576 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:28.576 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:28.576 | INFO | Uploaded 36GiB to GPU in 2.18361s (16.4864GiB/s) | 20260505 15:15:28.576 | INFO | Downloaded 34GiB to GPU in 2.03165s (16.7352GiB/s) | 20260505 15:15:33.707 | INFO | Analyzing chunk frames = [4208128,4208640) | 20260505 15:15:33.707 | INFO | MemInfo: { | 20260505 15:15:33.707 | INFO | "gpu" : " 17 GB", | 20260505 15:15:33.707 | INFO | "huge" : "", | 20260505 15:15:33.707 | INFO | "huge_shared" : " 27 GB", | 20260505 15:15:33.707 | INFO | "malloc" : " 14 GB", | 20260505 15:15:33.707 | INFO | "pinned" : " 29 GB" | 20260505 15:15:33.707 | INFO | } | 20260505 15:15:33.707 | INFO | Perf report for the last 5.13069 seconds: | 20260505 15:15:33.707 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:15:33.707 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:33.707 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:33.707 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:33.707 | INFO | ANALYSIS : 62.7%, 3.0, 97.2ms, 0 | 20260505 15:15:33.707 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:33.707 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:33.707 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 15:15:33.707 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:33.707 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:33.707 | INFO | Uploaded 36GiB to GPU in 2.1804s (16.5107GiB/s) | 20260505 15:15:33.707 | INFO | Downloaded 34GiB to GPU in 2.13932s (15.8929GiB/s) | 20260505 15:15:38.842 | INFO | Analyzing chunk frames = [4208640,4209152) | 20260505 15:15:38.842 | INFO | MemInfo: { | 20260505 15:15:38.842 | INFO | "gpu" : " 17 GB", | 20260505 15:15:38.842 | INFO | "huge" : "", | 20260505 15:15:38.842 | INFO | "huge_shared" : " 27 GB", | 20260505 15:15:38.842 | INFO | "malloc" : " 14 GB", | 20260505 15:15:38.842 | INFO | "pinned" : " 29 GB" | 20260505 15:15:38.842 | INFO | } | 20260505 15:15:38.842 | INFO | Perf report for the last 5.1356 seconds: | 20260505 15:15:38.842 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:15:38.842 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:38.842 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:38.842 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:38.842 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 15:15:38.842 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:38.842 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:38.843 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 15:15:38.843 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:38.843 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:38.843 | INFO | Uploaded 36GiB to GPU in 2.17622s (16.5424GiB/s) | 20260505 15:15:38.843 | INFO | Downloaded 34GiB to GPU in 2.12808s (15.9769GiB/s) | 20260505 15:15:43.987 | INFO | Analyzing chunk frames = [4209152,4209664) | 20260505 15:15:43.987 | INFO | MemInfo: { | 20260505 15:15:43.987 | INFO | "gpu" : " 17 GB", | 20260505 15:15:43.987 | INFO | "huge" : "", | 20260505 15:15:43.987 | INFO | "huge_shared" : " 27 GB", | 20260505 15:15:43.987 | INFO | "malloc" : " 14 GB", | 20260505 15:15:43.987 | INFO | "pinned" : " 29 GB" | 20260505 15:15:43.987 | INFO | } | 20260505 15:15:43.987 | INFO | Perf report for the last 5.145 seconds: | 20260505 15:15:43.987 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:15:43.987 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:43.987 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:43.987 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:43.987 | INFO | ANALYSIS : 62.9%, 3.0, 98.4ms, 0 | 20260505 15:15:43.987 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:43.987 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:43.988 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 15:15:43.988 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:43.988 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:43.988 | INFO | Uploaded 36GiB to GPU in 2.164s (16.6359GiB/s) | 20260505 15:15:43.988 | INFO | Downloaded 34GiB to GPU in 2.109s (16.1214GiB/s) | 20260505 15:15:49.131 | INFO | Analyzing chunk frames = [4209664,4210176) | 20260505 15:15:49.131 | INFO | MemInfo: { | 20260505 15:15:49.131 | INFO | "gpu" : " 17 GB", | 20260505 15:15:49.131 | INFO | "huge" : "", | 20260505 15:15:49.131 | INFO | "huge_shared" : " 27 GB", | 20260505 15:15:49.131 | INFO | "malloc" : " 14 GB", | 20260505 15:15:49.131 | INFO | "pinned" : " 29 GB" | 20260505 15:15:49.131 | INFO | } | 20260505 15:15:49.131 | INFO | Perf report for the last 5.14345 seconds: | 20260505 15:15:49.131 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:15:49.131 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:49.131 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:49.131 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:49.131 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 15:15:49.131 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:49.131 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:49.131 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 15:15:49.131 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:49.131 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:49.131 | INFO | Uploaded 36GiB to GPU in 2.11169s (17.0479GiB/s) | 20260505 15:15:49.131 | INFO | Downloaded 34GiB to GPU in 2.17296s (15.6469GiB/s) | 20260505 15:15:54.277 | INFO | Analyzing chunk frames = [4210176,4210688) | 20260505 15:15:54.277 | INFO | MemInfo: { | 20260505 15:15:54.277 | INFO | "gpu" : " 17 GB", | 20260505 15:15:54.277 | INFO | "huge" : "", | 20260505 15:15:54.277 | INFO | "huge_shared" : " 27 GB", | 20260505 15:15:54.277 | INFO | "malloc" : " 14 GB", | 20260505 15:15:54.277 | INFO | "pinned" : " 29 GB" | 20260505 15:15:54.277 | INFO | } | 20260505 15:15:54.277 | INFO | Perf report for the last 5.14658 seconds: | 20260505 15:15:54.277 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:15:54.277 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:54.277 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:54.278 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:54.278 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 15:15:54.278 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:54.278 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:54.278 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 15:15:54.278 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:54.278 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:54.278 | INFO | Uploaded 36GiB to GPU in 2.16094s (16.6594GiB/s) | 20260505 15:15:54.278 | INFO | Downloaded 34GiB to GPU in 2.03363s (16.7188GiB/s) | 20260505 15:15:54.278 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 15:15:54.278 | INFO | Total mem usage: 15.3 GiB | 20260505 15:15:54.278 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 15:15:54.278 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.85%) | 20260505 15:15:54.278 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 15:15:54.278 | INFO | High Water Breakdown: | 20260505 15:15:54.278 | INFO | 61% : 8.67 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 15:15:54.278 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 15:15:54.278 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 15:15:54.278 | INFO | ------------------------------------------ | 20260505 15:15:54.278 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:15:54.278 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 15:15:54.278 | INFO | Total mem usage: 26.5 GiB | 20260505 15:15:54.278 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 15:15:54.278 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 15:15:54.278 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 15:15:54.278 | INFO | High Water Breakdown: | 20260505 15:15:54.278 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:15:54.278 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:15:54.278 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:15:54.278 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:15:54.278 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:15:54.278 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:15:54.278 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 15:15:54.278 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 15:15:54.278 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:15:54.279 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 15:15:54.279 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 15:15:54.279 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 15:15:54.279 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 15:15:54.279 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 15:15:54.279 | INFO | ------------------------------------------ | 20260505 15:15:54.279 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:15:54.279 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 15:15:54.279 | INFO | Total mem usage: 25.6 GiB | 20260505 15:15:54.279 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 15:15:54.279 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 15:15:54.279 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 15:15:54.279 | INFO | High Water Breakdown: | 20260505 15:15:54.279 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 15:15:54.279 | INFO | ------------------------------------------ | 20260505 15:15:54.279 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:15:54.279 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 15:15:54.279 | INFO | Total mem usage: 15.7 GiB | 20260505 15:15:54.279 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 15:15:54.279 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 15:15:54.279 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 15:15:54.279 | INFO | High Water Breakdown: | 20260505 15:15:54.279 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 15:15:54.279 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 15:15:54.279 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 15:15:54.279 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 15:15:54.279 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 15:15:54.279 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 15:15:54.279 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 15:15:54.279 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 15:15:54.279 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 15:15:54.279 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:15:54.279 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 15:15:54.279 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:15:54.279 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 15:15:54.279 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 15:15:54.279 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 15:15:54.279 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 15:15:54.279 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:15:54.279 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 15:15:54.279 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:15:54.279 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:15:54.279 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:15:54.279 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 15:15:54.279 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 15:15:54.279 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:15:54.279 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 15:15:54.279 | INFO | ------------------------------------------ | 20260505 15:15:54.279 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:15:54.279 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 15:15:54.279 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:15:54.279 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:15:54.279 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.58e+01, 1.36e+01 / 1.64e+00 | 20260505 15:15:54.279 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 1.01e-01, 1.05e-02 / 4.56e-03 | 20260505 15:15:54.279 | INFO | PerfReport: 11.78%, Baseline -- 3.74e+00 / 4.49e+00, 5.75e-01 / 1.53e-01 | 20260505 15:15:54.279 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.51e+00, 1.92e-01 / 1.15e-01 | 20260505 15:15:54.279 | INFO | PerfReport: 15.97%, DME -- 4.05e+01 / 4.54e+01, 4.53e+00 / 1.84e+00 | 20260505 15:15:54.279 | INFO | PerfReport: 56.50%, FrameLabeling -- 1.79e+01 / 1.94e+01, 1.35e+00 / 1.00e+00 | 20260505 15:15:54.279 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.36e+00 / 2.93e+00, 4.58e-01 / 1.45e-01 | 20260505 15:15:54.279 | INFO | PerfReport: 3.50%, Metrics -- 1.11e+00 / 1.44e+00, 3.11e-01 / 1.61e-01 | 20260505 15:15:54.279 | INFO | PerfReport: 1.22%, PulseSerializing -- 3.88e-01 / 4.48e-01, 6.90e-02 / 3.74e-02 | 20260505 15:15:54.279 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 15:15:54.279 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:15:54.279 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:15:54.279 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260505 15:15:54.279 | INFO | PerfReport: 22.59%, Upload -- 2.27e+01 / 0.00e+00, 8.02e+00 / 0.00e+00 | 20260505 15:15:54.279 | INFO | PerfReport: 21.96%, Download -- 2.21e+01 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260505 15:15:54.279 | INFO | PerfReport: 31.55%, Compute -- 3.17e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 15:15:54.279 | INFO | PerfReport: 2.32%, LockUpload -- 2.34e+00 / 0.00e+00, 8.07e+00 / 0.00e+00 | 20260505 15:15:54.279 | INFO | PerfReport: 1.48%, LockDownload -- 1.49e+00 / 0.00e+00, 3.02e+00 / 0.00e+00 | 20260505 15:15:54.279 | INFO | PerfReport: 20.09%, LockCompute -- 2.02e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 15:15:54.279 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.67e-02 / 0.00e+00 | 20260505 15:15:54.411 | INFO | Starting a round of Baz IO | 20260505 15:15:57.626 | INFO | Populating RTMetrics report | 20260505 15:15:57.626 | INFO | RTMetrics report populated | 20260505 15:15:57.626 | INFO | Opening tmp RTMetrics json file | 20260505 15:15:57.644 | INFO | Serializing RTMetrics to json | 20260505 15:15:57.667 | INFO | Serialization done, swapping tmp file now... | 20260505 15:15:57.690 | INFO | RTMetrics done | 20260505 15:15:57.723 | INFO | Finished a round of Baz IO: Wrote 2.222 GiB in 3.312 seconds | 20260505 15:15:57.723 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.52% / 0.00% / 77.48% | 20260505 15:15:57.723 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.467396% / 1.19209e-05% / 7.94729e-05% | 20260505 15:15:57.724 | INFO | Baz IO overall: | 20260505 15:15:57.724 | INFO | BytesWritten : 2505913315398 | 20260505 15:15:57.724 | INFO | NumEvents : 1190653077793 | 20260505 15:15:59.403 | INFO | Analyzing chunk frames = [4210688,4211200) | 20260505 15:15:59.403 | INFO | MemInfo: { | 20260505 15:15:59.403 | INFO | "gpu" : " 17 GB", | 20260505 15:15:59.403 | INFO | "huge" : "", | 20260505 15:15:59.403 | INFO | "huge_shared" : " 27 GB", | 20260505 15:15:59.403 | INFO | "malloc" : " 14 GB", | 20260505 15:15:59.403 | INFO | "pinned" : " 29 GB" | 20260505 15:15:59.403 | INFO | } | 20260505 15:15:59.403 | INFO | Perf report for the last 5.12573 seconds: | 20260505 15:15:59.403 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:15:59.403 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:59.403 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:59.403 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:59.403 | INFO | ANALYSIS : 64.2%, 3.0, 99.5ms, 0 | 20260505 15:15:59.403 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:15:59.403 | INFO | RT_METRICS : 46.1%, 1.0, 24.1ms, 0 | 20260505 15:15:59.403 | INFO | PRE_HQ : 61.3%, 1.8, 57.0ms, 0 | 20260505 15:15:59.403 | INFO | BAZWRITER : 64.0%, 4.0, 129.9ms, 0 | 20260505 15:15:59.403 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 15:15:59.404 | INFO | Uploaded 36GiB to GPU in 2.19886s (16.3721GiB/s) | 20260505 15:15:59.404 | INFO | Downloaded 38GiB to GPU in 2.26484s (16.7782GiB/s) | 20260505 15:16:04.539 | INFO | Analyzing chunk frames = [4211200,4211712) | 20260505 15:16:04.539 | INFO | MemInfo: { | 20260505 15:16:04.539 | INFO | "gpu" : " 17 GB", | 20260505 15:16:04.539 | INFO | "huge" : "", | 20260505 15:16:04.539 | INFO | "huge_shared" : " 27 GB", | 20260505 15:16:04.539 | INFO | "malloc" : " 14 GB", | 20260505 15:16:04.539 | INFO | "pinned" : " 29 GB" | 20260505 15:16:04.539 | INFO | } | 20260505 15:16:04.539 | INFO | Perf report for the last 5.13554 seconds: | 20260505 15:16:04.539 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:16:04.539 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:04.539 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:04.539 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:04.539 | INFO | ANALYSIS : 63.5%, 3.0, 99.0ms, 0 | 20260505 15:16:04.539 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:04.539 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:04.539 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:16:04.539 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:04.539 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:04.539 | INFO | Uploaded 36GiB to GPU in 2.20222s (16.3471GiB/s) | 20260505 15:16:04.539 | INFO | Downloaded 34GiB to GPU in 2.04372s (16.6363GiB/s) | 20260505 15:16:09.684 | INFO | Analyzing chunk frames = [4211712,4212224) | 20260505 15:16:09.684 | INFO | MemInfo: { | 20260505 15:16:09.684 | INFO | "gpu" : " 17 GB", | 20260505 15:16:09.684 | INFO | "huge" : "", | 20260505 15:16:09.684 | INFO | "huge_shared" : " 27 GB", | 20260505 15:16:09.684 | INFO | "malloc" : " 14 GB", | 20260505 15:16:09.684 | INFO | "pinned" : " 29 GB" | 20260505 15:16:09.684 | INFO | } | 20260505 15:16:09.684 | INFO | Perf report for the last 5.14522 seconds: | 20260505 15:16:09.684 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:16:09.684 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:09.684 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:09.684 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:09.684 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 15:16:09.684 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:09.684 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:09.684 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 15:16:09.684 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:09.684 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:09.684 | INFO | Uploaded 36GiB to GPU in 2.24015s (16.0704GiB/s) | 20260505 15:16:09.685 | INFO | Downloaded 34GiB to GPU in 2.09711s (16.2128GiB/s) | 20260505 15:16:14.823 | INFO | Analyzing chunk frames = [4212224,4212736) | 20260505 15:16:14.823 | INFO | MemInfo: { | 20260505 15:16:14.823 | INFO | "gpu" : " 17 GB", | 20260505 15:16:14.823 | INFO | "huge" : "", | 20260505 15:16:14.823 | INFO | "huge_shared" : " 27 GB", | 20260505 15:16:14.823 | INFO | "malloc" : " 14 GB", | 20260505 15:16:14.823 | INFO | "pinned" : " 29 GB" | 20260505 15:16:14.823 | INFO | } | 20260505 15:16:14.823 | INFO | Perf report for the last 5.13869 seconds: | 20260505 15:16:14.823 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:16:14.823 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:14.823 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:14.823 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:14.823 | INFO | ANALYSIS : 62.6%, 3.0, 97.2ms, 0 | 20260505 15:16:14.823 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:14.823 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:14.823 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 15:16:14.823 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:14.823 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:14.823 | INFO | Uploaded 36GiB to GPU in 2.19284s (16.417GiB/s) | 20260505 15:16:14.823 | INFO | Downloaded 34GiB to GPU in 2.1327s (15.9423GiB/s) | 20260505 15:16:19.964 | INFO | Analyzing chunk frames = [4212736,4213248) | 20260505 15:16:19.964 | INFO | MemInfo: { | 20260505 15:16:19.964 | INFO | "gpu" : " 17 GB", | 20260505 15:16:19.964 | INFO | "huge" : "", | 20260505 15:16:19.964 | INFO | "huge_shared" : " 27 GB", | 20260505 15:16:19.964 | INFO | "malloc" : " 14 GB", | 20260505 15:16:19.964 | INFO | "pinned" : " 29 GB" | 20260505 15:16:19.964 | INFO | } | 20260505 15:16:19.964 | INFO | Perf report for the last 5.14158 seconds: | 20260505 15:16:19.964 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:16:19.964 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:19.964 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:19.964 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:19.964 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 15:16:19.965 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:19.965 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:19.965 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 15:16:19.965 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:19.965 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:19.965 | INFO | Uploaded 36GiB to GPU in 2.18164s (16.5013GiB/s) | 20260505 15:16:19.965 | INFO | Downloaded 34GiB to GPU in 2.13109s (15.9543GiB/s) | 20260505 15:16:25.102 | INFO | Analyzing chunk frames = [4213248,4213760) | 20260505 15:16:25.102 | INFO | MemInfo: { | 20260505 15:16:25.102 | INFO | "gpu" : " 17 GB", | 20260505 15:16:25.102 | INFO | "huge" : "", | 20260505 15:16:25.102 | INFO | "huge_shared" : " 27 GB", | 20260505 15:16:25.102 | INFO | "malloc" : " 14 GB", | 20260505 15:16:25.102 | INFO | "pinned" : " 29 GB" | 20260505 15:16:25.102 | INFO | } | 20260505 15:16:25.102 | INFO | Perf report for the last 5.13795 seconds: | 20260505 15:16:25.102 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:16:25.102 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:25.102 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:25.103 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:25.103 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 15:16:25.103 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:25.103 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:25.103 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:16:25.103 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:25.103 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:25.103 | INFO | Uploaded 36GiB to GPU in 2.14148s (16.8108GiB/s) | 20260505 15:16:25.103 | INFO | Downloaded 34GiB to GPU in 2.02905s (16.7567GiB/s) | 20260505 15:16:30.242 | INFO | Analyzing chunk frames = [4213760,4214272) | 20260505 15:16:30.242 | INFO | MemInfo: { | 20260505 15:16:30.242 | INFO | "gpu" : " 17 GB", | 20260505 15:16:30.242 | INFO | "huge" : "", | 20260505 15:16:30.242 | INFO | "huge_shared" : " 27 GB", | 20260505 15:16:30.242 | INFO | "malloc" : " 14 GB", | 20260505 15:16:30.242 | INFO | "pinned" : " 29 GB" | 20260505 15:16:30.242 | INFO | } | 20260505 15:16:30.243 | INFO | Perf report for the last 5.14032 seconds: | 20260505 15:16:30.243 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:16:30.243 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:30.243 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:30.243 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:30.243 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 15:16:30.243 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:30.243 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:30.243 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 15:16:30.243 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:30.243 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:30.243 | INFO | Uploaded 36GiB to GPU in 2.11195s (17.0459GiB/s) | 20260505 15:16:30.243 | INFO | Downloaded 34GiB to GPU in 2.16446s (15.7083GiB/s) | 20260505 15:16:35.387 | INFO | Analyzing chunk frames = [4214272,4214784) | 20260505 15:16:35.387 | INFO | MemInfo: { | 20260505 15:16:35.387 | INFO | "gpu" : " 17 GB", | 20260505 15:16:35.387 | INFO | "huge" : "", | 20260505 15:16:35.387 | INFO | "huge_shared" : " 27 GB", | 20260505 15:16:35.387 | INFO | "malloc" : " 14 GB", | 20260505 15:16:35.387 | INFO | "pinned" : " 29 GB" | 20260505 15:16:35.387 | INFO | } | 20260505 15:16:35.387 | INFO | Perf report for the last 5.14439 seconds: | 20260505 15:16:35.387 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:16:35.387 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:35.387 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:35.387 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:35.387 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 15:16:35.387 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:35.387 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:35.387 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 15:16:35.387 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:35.387 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:35.387 | INFO | Uploaded 36GiB to GPU in 2.22548s (16.1763GiB/s) | 20260505 15:16:35.387 | INFO | Downloaded 34GiB to GPU in 2.09819s (16.2044GiB/s) | 20260505 15:16:35.518 | INFO | Starting a round of Baz IO | 20260505 15:16:38.734 | INFO | Populating RTMetrics report | 20260505 15:16:38.734 | INFO | RTMetrics report populated | 20260505 15:16:38.734 | INFO | Opening tmp RTMetrics json file | 20260505 15:16:38.757 | INFO | Serializing RTMetrics to json | 20260505 15:16:38.779 | INFO | Serialization done, swapping tmp file now... | 20260505 15:16:38.794 | INFO | RTMetrics done | 20260505 15:16:38.840 | INFO | Finished a round of Baz IO: Wrote 2.221 GiB in 3.322 seconds | 20260505 15:16:38.840 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.52% / 0.00% / 77.48% | 20260505 15:16:38.840 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.466776% / 1.19209e-05% / 8.34465e-05% | 20260505 15:16:38.840 | INFO | Baz IO overall: | 20260505 15:16:38.840 | INFO | BytesWritten : 2508298440686 | 20260505 15:16:38.840 | INFO | NumEvents : 1191775348794 | 20260505 15:16:40.522 | INFO | Analyzing chunk frames = [4214784,4215296) | 20260505 15:16:40.523 | INFO | MemInfo: { | 20260505 15:16:40.523 | INFO | "gpu" : " 17 GB", | 20260505 15:16:40.523 | INFO | "huge" : "", | 20260505 15:16:40.523 | INFO | "huge_shared" : " 27 GB", | 20260505 15:16:40.523 | INFO | "malloc" : " 14 GB", | 20260505 15:16:40.523 | INFO | "pinned" : " 29 GB" | 20260505 15:16:40.523 | INFO | } | 20260505 15:16:40.523 | INFO | Perf report for the last 5.13584 seconds: | 20260505 15:16:40.523 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:16:40.523 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:40.523 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:40.523 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:40.523 | INFO | ANALYSIS : 64.1%, 3.0, 99.3ms, 0 | 20260505 15:16:40.523 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:40.523 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 15:16:40.523 | INFO | PRE_HQ : 61.0%, 1.8, 57.1ms, 0 | 20260505 15:16:40.523 | INFO | BAZWRITER : 64.0%, 4.0, 130.8ms, 0 | 20260505 15:16:40.523 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 15:16:40.523 | INFO | Uploaded 36GiB to GPU in 2.19539s (16.398GiB/s) | 20260505 15:16:40.523 | INFO | Downloaded 38GiB to GPU in 2.26263s (16.7946GiB/s) | 20260505 15:16:45.656 | INFO | Analyzing chunk frames = [4215296,4215808) | 20260505 15:16:45.656 | INFO | MemInfo: { | 20260505 15:16:45.656 | INFO | "gpu" : " 17 GB", | 20260505 15:16:45.656 | INFO | "huge" : "", | 20260505 15:16:45.656 | INFO | "huge_shared" : " 27 GB", | 20260505 15:16:45.656 | INFO | "malloc" : " 14 GB", | 20260505 15:16:45.656 | INFO | "pinned" : " 29 GB" | 20260505 15:16:45.656 | INFO | } | 20260505 15:16:45.656 | INFO | Perf report for the last 5.13339 seconds: | 20260505 15:16:45.656 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:16:45.656 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:45.656 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:45.656 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:45.656 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 15:16:45.656 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:45.656 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:45.656 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:16:45.656 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:45.656 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:45.656 | INFO | Uploaded 36GiB to GPU in 2.20357s (16.3371GiB/s) | 20260505 15:16:45.657 | INFO | Downloaded 34GiB to GPU in 2.06678s (16.4507GiB/s) | 20260505 15:16:50.794 | INFO | Analyzing chunk frames = [4215808,4216320) | 20260505 15:16:50.794 | INFO | MemInfo: { | 20260505 15:16:50.794 | INFO | "gpu" : " 17 GB", | 20260505 15:16:50.794 | INFO | "huge" : "", | 20260505 15:16:50.794 | INFO | "huge_shared" : " 27 GB", | 20260505 15:16:50.794 | INFO | "malloc" : " 14 GB", | 20260505 15:16:50.794 | INFO | "pinned" : " 29 GB" | 20260505 15:16:50.794 | INFO | } | 20260505 15:16:50.794 | INFO | Perf report for the last 5.13808 seconds: | 20260505 15:16:50.794 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:16:50.794 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:50.794 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:50.794 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:50.794 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 15:16:50.794 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:50.794 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:50.794 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 15:16:50.794 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:50.795 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:50.795 | INFO | Uploaded 36GiB to GPU in 2.242s (16.0571GiB/s) | 20260505 15:16:50.795 | INFO | Downloaded 34GiB to GPU in 2.11452s (16.0793GiB/s) | 20260505 15:16:55.939 | INFO | Analyzing chunk frames = [4216320,4216832) | 20260505 15:16:55.939 | INFO | MemInfo: { | 20260505 15:16:55.939 | INFO | "gpu" : " 17 GB", | 20260505 15:16:55.939 | INFO | "huge" : "", | 20260505 15:16:55.939 | INFO | "huge_shared" : " 27 GB", | 20260505 15:16:55.939 | INFO | "malloc" : " 14 GB", | 20260505 15:16:55.939 | INFO | "pinned" : " 29 GB" | 20260505 15:16:55.939 | INFO | } | 20260505 15:16:55.939 | INFO | Perf report for the last 5.14509 seconds: | 20260505 15:16:55.939 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:16:55.939 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:55.939 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:55.939 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:55.940 | INFO | ANALYSIS : 62.9%, 3.0, 97.7ms, 0 | 20260505 15:16:55.940 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:55.940 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:55.940 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 15:16:55.940 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:55.940 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:16:55.940 | INFO | Uploaded 36GiB to GPU in 2.19914s (16.37GiB/s) | 20260505 15:16:55.940 | INFO | Downloaded 34GiB to GPU in 2.13568s (15.92GiB/s) | 20260505 15:17:01.083 | INFO | Analyzing chunk frames = [4216832,4217344) | 20260505 15:17:01.083 | INFO | MemInfo: { | 20260505 15:17:01.083 | INFO | "gpu" : " 17 GB", | 20260505 15:17:01.083 | INFO | "huge" : "", | 20260505 15:17:01.083 | INFO | "huge_shared" : " 27 GB", | 20260505 15:17:01.083 | INFO | "malloc" : " 14 GB", | 20260505 15:17:01.083 | INFO | "pinned" : " 29 GB" | 20260505 15:17:01.083 | INFO | } | 20260505 15:17:01.083 | INFO | Perf report for the last 5.14367 seconds: | 20260505 15:17:01.083 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:17:01.083 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:01.083 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:01.083 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:01.083 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260505 15:17:01.083 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:01.083 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:01.083 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 15:17:01.083 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:01.083 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:01.083 | INFO | Uploaded 36GiB to GPU in 2.15811s (16.6812GiB/s) | 20260505 15:17:01.083 | INFO | Downloaded 34GiB to GPU in 2.05834s (16.5181GiB/s) | 20260505 15:17:06.221 | INFO | Analyzing chunk frames = [4217344,4217856) | 20260505 15:17:06.221 | INFO | MemInfo: { | 20260505 15:17:06.221 | INFO | "gpu" : " 17 GB", | 20260505 15:17:06.221 | INFO | "huge" : "", | 20260505 15:17:06.221 | INFO | "huge_shared" : " 27 GB", | 20260505 15:17:06.221 | INFO | "malloc" : " 14 GB", | 20260505 15:17:06.221 | INFO | "pinned" : " 29 GB" | 20260505 15:17:06.221 | INFO | } | 20260505 15:17:06.221 | INFO | Perf report for the last 5.13751 seconds: | 20260505 15:17:06.221 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:17:06.221 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:06.221 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:06.221 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:06.221 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 15:17:06.221 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:06.221 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:06.221 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 15:17:06.221 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:06.221 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:06.221 | INFO | Uploaded 36GiB to GPU in 2.14095s (16.815GiB/s) | 20260505 15:17:06.221 | INFO | Downloaded 34GiB to GPU in 2.0301s (16.7479GiB/s) | 20260505 15:17:11.349 | INFO | Analyzing chunk frames = [4217856,4218368) | 20260505 15:17:11.349 | INFO | MemInfo: { | 20260505 15:17:11.349 | INFO | "gpu" : " 17 GB", | 20260505 15:17:11.349 | INFO | "huge" : "", | 20260505 15:17:11.349 | INFO | "huge_shared" : " 27 GB", | 20260505 15:17:11.349 | INFO | "malloc" : " 14 GB", | 20260505 15:17:11.349 | INFO | "pinned" : " 29 GB" | 20260505 15:17:11.349 | INFO | } | 20260505 15:17:11.350 | INFO | Perf report for the last 5.12891 seconds: | 20260505 15:17:11.350 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:17:11.350 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:11.350 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:11.350 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:11.350 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 15:17:11.350 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:11.350 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:11.350 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 15:17:11.350 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:11.350 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:11.350 | INFO | Uploaded 36GiB to GPU in 2.12164s (16.968GiB/s) | 20260505 15:17:11.350 | INFO | Downloaded 34GiB to GPU in 2.17045s (15.6649GiB/s) | 20260505 15:17:16.493 | INFO | Analyzing chunk frames = [4218368,4218880) | 20260505 15:17:16.493 | INFO | MemInfo: { | 20260505 15:17:16.493 | INFO | "gpu" : " 17 GB", | 20260505 15:17:16.493 | INFO | "huge" : "", | 20260505 15:17:16.493 | INFO | "huge_shared" : " 27 GB", | 20260505 15:17:16.493 | INFO | "malloc" : " 14 GB", | 20260505 15:17:16.493 | INFO | "pinned" : " 29 GB" | 20260505 15:17:16.493 | INFO | } | 20260505 15:17:16.493 | INFO | Perf report for the last 5.1431 seconds: | 20260505 15:17:16.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:17:16.493 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:16.493 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:16.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:16.493 | INFO | ANALYSIS : 63.5%, 3.0, 99.3ms, 0 | 20260505 15:17:16.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:16.493 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:16.493 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 15:17:16.493 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:16.493 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:16.493 | INFO | Uploaded 36GiB to GPU in 2.18939s (16.4429GiB/s) | 20260505 15:17:16.493 | INFO | Downloaded 34GiB to GPU in 2.10954s (16.1172GiB/s) | 20260505 15:17:16.493 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 15:17:16.493 | INFO | Total mem usage: 15.3 GiB | 20260505 15:17:16.493 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 15:17:16.493 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.84%) | 20260505 15:17:16.493 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 15:17:16.493 | INFO | High Water Breakdown: | 20260505 15:17:16.493 | INFO | 61% : 8.67 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 15:17:16.493 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 15:17:16.493 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 15:17:16.493 | INFO | ------------------------------------------ | 20260505 15:17:16.493 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:17:16.493 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 15:17:16.493 | INFO | Total mem usage: 26.5 GiB | 20260505 15:17:16.493 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 15:17:16.493 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 15:17:16.493 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 15:17:16.493 | INFO | High Water Breakdown: | 20260505 15:17:16.494 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:17:16.494 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:17:16.494 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:17:16.494 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:17:16.494 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:17:16.494 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:17:16.494 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 15:17:16.494 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 15:17:16.494 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:17:16.494 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 15:17:16.494 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 15:17:16.494 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 15:17:16.494 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 15:17:16.494 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 15:17:16.494 | INFO | ------------------------------------------ | 20260505 15:17:16.494 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:17:16.494 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 15:17:16.494 | INFO | Total mem usage: 25.6 GiB | 20260505 15:17:16.494 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 15:17:16.494 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 15:17:16.494 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 15:17:16.494 | INFO | High Water Breakdown: | 20260505 15:17:16.494 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 15:17:16.494 | INFO | ------------------------------------------ | 20260505 15:17:16.494 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:17:16.494 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 15:17:16.494 | INFO | Total mem usage: 15.7 GiB | 20260505 15:17:16.494 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 15:17:16.494 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 15:17:16.494 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260505 15:17:16.494 | INFO | High Water Breakdown: | 20260505 15:17:16.494 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 15:17:16.494 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 15:17:16.494 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 15:17:16.494 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 15:17:16.494 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 15:17:16.494 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 15:17:16.494 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 15:17:16.494 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 15:17:16.494 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 15:17:16.494 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:17:16.494 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 15:17:16.494 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:17:16.494 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 15:17:16.494 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 15:17:16.494 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 15:17:16.494 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 15:17:16.494 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:17:16.494 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 15:17:16.494 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:17:16.494 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:17:16.494 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:17:16.494 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 15:17:16.494 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 15:17:16.494 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:17:16.494 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 15:17:16.494 | INFO | ------------------------------------------ | 20260505 15:17:16.494 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:17:16.494 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 15:17:16.494 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:17:16.494 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:17:16.494 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.56e+01, 1.36e+01 / 2.07e+00 | 20260505 15:17:16.494 | INFO | PerfReport: 0.29%, AggregateStats -- 9.26e-02 / 1.03e-01, 1.11e-02 / 1.49e-02 | 20260505 15:17:16.494 | INFO | PerfReport: 11.77%, Baseline -- 3.73e+00 / 4.51e+00, 5.67e-01 / 1.82e-01 | 20260505 15:17:16.494 | INFO | PerfReport: 3.27%, Histogram -- 1.04e+00 / 1.49e+00, 1.92e-01 / 1.21e-01 | 20260505 15:17:16.494 | INFO | PerfReport: 15.96%, DME -- 4.05e+01 / 4.50e+01, 4.42e+00 / 2.25e+00 | 20260505 15:17:16.494 | INFO | PerfReport: 56.58%, FrameLabeling -- 1.79e+01 / 1.97e+01, 1.39e+00 / 1.65e+00 | 20260505 15:17:16.494 | INFO | PerfReport: 7.42%, PulseAccumulating -- 2.35e+00 / 2.92e+00, 4.45e-01 / 1.77e-01 | 20260505 15:17:16.494 | INFO | PerfReport: 3.48%, Metrics -- 1.10e+00 / 1.41e+00, 3.03e-01 / 1.72e-01 | 20260505 15:17:16.494 | INFO | PerfReport: 1.23%, PulseSerializing -- 3.88e-01 / 4.42e-01, 6.95e-02 / 3.86e-02 | 20260505 15:17:16.494 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 15:17:16.494 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:17:16.494 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:17:16.494 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.56e+01 / 0.00e+00 | 20260505 15:17:16.494 | INFO | PerfReport: 22.62%, Upload -- 2.28e+01 / 0.00e+00, 8.08e+00 / 0.00e+00 | 20260505 15:17:16.494 | INFO | PerfReport: 21.95%, Download -- 2.21e+01 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260505 15:17:16.494 | INFO | PerfReport: 31.51%, Compute -- 3.17e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 15:17:16.494 | INFO | PerfReport: 2.38%, LockUpload -- 2.40e+00 / 0.00e+00, 7.96e+00 / 0.00e+00 | 20260505 15:17:16.494 | INFO | PerfReport: 1.36%, LockDownload -- 1.36e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260505 15:17:16.494 | INFO | PerfReport: 20.17%, LockCompute -- 2.03e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 15:17:16.494 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.58e-02 / 0.00e+00 | 20260505 15:17:16.628 | INFO | Starting a round of Baz IO | 20260505 15:17:19.845 | INFO | Populating RTMetrics report | 20260505 15:17:19.845 | INFO | RTMetrics report populated | 20260505 15:17:19.845 | INFO | Opening tmp RTMetrics json file | 20260505 15:17:19.860 | INFO | Serializing RTMetrics to json | 20260505 15:17:19.862 | INFO | Serialization done, swapping tmp file now... | 20260505 15:17:19.863 | INFO | RTMetrics done | 20260505 15:17:19.953 | INFO | Finished a round of Baz IO: Wrote 2.219 GiB in 3.325 seconds | 20260505 15:17:19.953 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.51% / 0.00% / 77.48% | 20260505 15:17:19.953 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.463839% / 3.97364e-06% / 7.54992e-05% | 20260505 15:17:19.954 | INFO | Baz IO overall: | 20260505 15:17:19.954 | INFO | BytesWritten : 2510680815922 | 20260505 15:17:19.954 | INFO | NumEvents : 1192895737605 | 20260505 15:17:21.632 | INFO | Analyzing chunk frames = [4218880,4219392) | 20260505 15:17:21.632 | INFO | MemInfo: { | 20260505 15:17:21.632 | INFO | "gpu" : " 17 GB", | 20260505 15:17:21.632 | INFO | "huge" : "", | 20260505 15:17:21.632 | INFO | "huge_shared" : " 27 GB", | 20260505 15:17:21.632 | INFO | "malloc" : " 14 GB", | 20260505 15:17:21.633 | INFO | "pinned" : " 29 GB" | 20260505 15:17:21.633 | INFO | } | 20260505 15:17:21.633 | INFO | Perf report for the last 5.13989 seconds: | 20260505 15:17:21.633 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:17:21.633 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:21.633 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:21.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:21.633 | INFO | ANALYSIS : 64.1%, 3.0, 99.6ms, 0 | 20260505 15:17:21.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:21.633 | INFO | RT_METRICS : 45.3%, 1.0, 23.8ms, 0 | 20260505 15:17:21.633 | INFO | PRE_HQ : 61.1%, 1.8, 57.2ms, 0 | 20260505 15:17:21.633 | INFO | BAZWRITER : 64.1%, 4.0, 131.2ms, 0 | 20260505 15:17:21.633 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 15:17:21.633 | INFO | Uploaded 36GiB to GPU in 2.20898s (16.2971GiB/s) | 20260505 15:17:21.633 | INFO | Downloaded 38GiB to GPU in 2.26428s (16.7824GiB/s) | 20260505 15:17:26.780 | INFO | Analyzing chunk frames = [4219392,4219904) | 20260505 15:17:26.780 | INFO | MemInfo: { | 20260505 15:17:26.780 | INFO | "gpu" : " 17 GB", | 20260505 15:17:26.780 | INFO | "huge" : "", | 20260505 15:17:26.780 | INFO | "huge_shared" : " 27 GB", | 20260505 15:17:26.780 | INFO | "malloc" : " 14 GB", | 20260505 15:17:26.780 | INFO | "pinned" : " 29 GB" | 20260505 15:17:26.780 | INFO | } | 20260505 15:17:26.781 | INFO | Perf report for the last 5.14816 seconds: | 20260505 15:17:26.781 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:17:26.781 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:26.781 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:26.781 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:26.781 | INFO | ANALYSIS : 63.3%, 3.0, 99.0ms, 0 | 20260505 15:17:26.781 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:26.781 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:26.781 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 15:17:26.781 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:26.781 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:26.781 | INFO | Uploaded 36GiB to GPU in 2.19802s (16.3784GiB/s) | 20260505 15:17:26.781 | INFO | Downloaded 34GiB to GPU in 2.04016s (16.6654GiB/s) | 20260505 15:17:31.910 | INFO | Analyzing chunk frames = [4219904,4220416) | 20260505 15:17:31.910 | INFO | MemInfo: { | 20260505 15:17:31.910 | INFO | "gpu" : " 17 GB", | 20260505 15:17:31.910 | INFO | "huge" : "", | 20260505 15:17:31.910 | INFO | "huge_shared" : " 27 GB", | 20260505 15:17:31.910 | INFO | "malloc" : " 14 GB", | 20260505 15:17:31.910 | INFO | "pinned" : " 29 GB" | 20260505 15:17:31.911 | INFO | } | 20260505 15:17:31.911 | INFO | Perf report for the last 5.12984 seconds: | 20260505 15:17:31.911 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:17:31.911 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:31.911 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:31.911 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:31.911 | INFO | ANALYSIS : 62.9%, 3.0, 97.8ms, 0 | 20260505 15:17:31.911 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:31.911 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:31.911 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 15:17:31.911 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:31.911 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:31.911 | INFO | Uploaded 36GiB to GPU in 2.19905s (16.3707GiB/s) | 20260505 15:17:31.911 | INFO | Downloaded 34GiB to GPU in 2.02611s (16.7809GiB/s) | 20260505 15:17:37.049 | INFO | Analyzing chunk frames = [4220416,4220928) | 20260505 15:17:37.049 | INFO | MemInfo: { | 20260505 15:17:37.049 | INFO | "gpu" : " 17 GB", | 20260505 15:17:37.049 | INFO | "huge" : "", | 20260505 15:17:37.049 | INFO | "huge_shared" : " 27 GB", | 20260505 15:17:37.049 | INFO | "malloc" : " 14 GB", | 20260505 15:17:37.049 | INFO | "pinned" : " 29 GB" | 20260505 15:17:37.049 | INFO | } | 20260505 15:17:37.049 | INFO | Perf report for the last 5.13874 seconds: | 20260505 15:17:37.049 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:17:37.049 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:37.049 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:37.049 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:37.049 | INFO | ANALYSIS : 62.6%, 3.0, 97.3ms, 0 | 20260505 15:17:37.049 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:37.049 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:37.049 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:17:37.049 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:37.049 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:37.050 | INFO | Uploaded 36GiB to GPU in 2.17036s (16.5871GiB/s) | 20260505 15:17:37.050 | INFO | Downloaded 34GiB to GPU in 2.13112s (15.9541GiB/s) | 20260505 15:17:42.202 | INFO | Analyzing chunk frames = [4220928,4221440) | 20260505 15:17:42.202 | INFO | MemInfo: { | 20260505 15:17:42.202 | INFO | "gpu" : " 17 GB", | 20260505 15:17:42.202 | INFO | "huge" : "", | 20260505 15:17:42.202 | INFO | "huge_shared" : " 27 GB", | 20260505 15:17:42.202 | INFO | "malloc" : " 14 GB", | 20260505 15:17:42.202 | INFO | "pinned" : " 29 GB" | 20260505 15:17:42.202 | INFO | } | 20260505 15:17:42.202 | INFO | Perf report for the last 5.15298 seconds: | 20260505 15:17:42.202 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:17:42.202 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:42.202 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:42.202 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:42.202 | INFO | ANALYSIS : 62.5%, 3.0, 97.9ms, 0 | 20260505 15:17:42.202 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:42.202 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:42.202 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 15:17:42.202 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:42.202 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:42.203 | INFO | Uploaded 36GiB to GPU in 2.17209s (16.5739GiB/s) | 20260505 15:17:42.203 | INFO | Downloaded 34GiB to GPU in 2.13249s (15.9438GiB/s) | 20260505 15:17:47.334 | INFO | Analyzing chunk frames = [4221440,4221952) | 20260505 15:17:47.334 | INFO | MemInfo: { | 20260505 15:17:47.334 | INFO | "gpu" : " 17 GB", | 20260505 15:17:47.334 | INFO | "huge" : "", | 20260505 15:17:47.334 | INFO | "huge_shared" : " 27 GB", | 20260505 15:17:47.334 | INFO | "malloc" : " 14 GB", | 20260505 15:17:47.334 | INFO | "pinned" : " 29 GB" | 20260505 15:17:47.334 | INFO | } | 20260505 15:17:47.334 | INFO | Perf report for the last 5.13207 seconds: | 20260505 15:17:47.334 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:17:47.334 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:47.334 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:47.334 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:47.334 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 15:17:47.334 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:47.334 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:47.334 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 15:17:47.334 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:47.334 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:47.335 | INFO | Uploaded 36GiB to GPU in 2.14058s (16.8179GiB/s) | 20260505 15:17:47.335 | INFO | Downloaded 34GiB to GPU in 2.03225s (16.7303GiB/s) | 20260505 15:17:52.470 | INFO | Analyzing chunk frames = [4221952,4222464) | 20260505 15:17:52.470 | INFO | MemInfo: { | 20260505 15:17:52.470 | INFO | "gpu" : " 17 GB", | 20260505 15:17:52.470 | INFO | "huge" : "", | 20260505 15:17:52.470 | INFO | "huge_shared" : " 27 GB", | 20260505 15:17:52.470 | INFO | "malloc" : " 14 GB", | 20260505 15:17:52.470 | INFO | "pinned" : " 29 GB" | 20260505 15:17:52.470 | INFO | } | 20260505 15:17:52.470 | INFO | Perf report for the last 5.13544 seconds: | 20260505 15:17:52.470 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:17:52.470 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:52.470 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:52.470 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:52.470 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 15:17:52.470 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:52.470 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:52.470 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 15:17:52.470 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:52.470 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:52.470 | INFO | Uploaded 36GiB to GPU in 2.11672s (17.0075GiB/s) | 20260505 15:17:52.470 | INFO | Downloaded 34GiB to GPU in 2.17s (15.6682GiB/s) | 20260505 15:17:57.605 | INFO | Analyzing chunk frames = [4222464,4222976) | 20260505 15:17:57.605 | INFO | MemInfo: { | 20260505 15:17:57.605 | INFO | "gpu" : " 17 GB", | 20260505 15:17:57.605 | INFO | "huge" : "", | 20260505 15:17:57.605 | INFO | "huge_shared" : " 27 GB", | 20260505 15:17:57.605 | INFO | "malloc" : " 14 GB", | 20260505 15:17:57.605 | INFO | "pinned" : " 29 GB" | 20260505 15:17:57.606 | INFO | } | 20260505 15:17:57.606 | INFO | Perf report for the last 5.13573 seconds: | 20260505 15:17:57.606 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:17:57.606 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:57.606 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:57.606 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:57.606 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 15:17:57.606 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:57.606 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:57.606 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 15:17:57.606 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:57.606 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:17:57.606 | INFO | Uploaded 36GiB to GPU in 2.22781s (16.1594GiB/s) | 20260505 15:17:57.606 | INFO | Downloaded 34GiB to GPU in 2.09559s (16.2246GiB/s) | 20260505 15:17:57.742 | INFO | Starting a round of Baz IO | 20260505 15:18:00.960 | INFO | Populating RTMetrics report | 20260505 15:18:00.961 | INFO | RTMetrics report populated | 20260505 15:18:00.961 | INFO | Opening tmp RTMetrics json file | 20260505 15:18:00.973 | INFO | Serializing RTMetrics to json | 20260505 15:18:00.974 | INFO | Serialization done, swapping tmp file now... | 20260505 15:18:00.975 | INFO | RTMetrics done | 20260505 15:18:01.059 | INFO | Finished a round of Baz IO: Wrote 2.219 GiB in 3.316 seconds | 20260505 15:18:01.059 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.51% / 0.00% / 77.49% | 20260505 15:18:01.059 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.462914% / 3.97364e-06% / 7.94729e-05% | 20260505 15:18:01.059 | INFO | Baz IO overall: | 20260505 15:18:01.059 | INFO | BytesWritten : 2513063973624 | 20260505 15:18:01.059 | INFO | NumEvents : 1194017106023 | 20260505 15:18:02.751 | INFO | Analyzing chunk frames = [4222976,4223488) | 20260505 15:18:02.751 | INFO | MemInfo: { | 20260505 15:18:02.751 | INFO | "gpu" : " 17 GB", | 20260505 15:18:02.751 | INFO | "huge" : "", | 20260505 15:18:02.751 | INFO | "huge_shared" : " 27 GB", | 20260505 15:18:02.751 | INFO | "malloc" : " 14 GB", | 20260505 15:18:02.751 | INFO | "pinned" : " 29 GB" | 20260505 15:18:02.751 | INFO | } | 20260505 15:18:02.751 | INFO | Perf report for the last 5.14578 seconds: | 20260505 15:18:02.751 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:18:02.751 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:02.751 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:02.751 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:02.751 | INFO | ANALYSIS : 64.1%, 3.0, 99.7ms, 0 | 20260505 15:18:02.751 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:02.751 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260505 15:18:02.752 | INFO | PRE_HQ : 60.9%, 1.8, 57.2ms, 0 | 20260505 15:18:02.752 | INFO | BAZWRITER : 63.8%, 4.0, 130.3ms, 0 | 20260505 15:18:02.752 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 15:18:02.752 | INFO | Uploaded 36GiB to GPU in 2.20006s (16.3632GiB/s) | 20260505 15:18:02.752 | INFO | Downloaded 38GiB to GPU in 2.26249s (16.7957GiB/s) | 20260505 15:18:07.891 | INFO | Analyzing chunk frames = [4223488,4224000) | 20260505 15:18:07.891 | INFO | MemInfo: { | 20260505 15:18:07.891 | INFO | "gpu" : " 17 GB", | 20260505 15:18:07.891 | INFO | "huge" : "", | 20260505 15:18:07.891 | INFO | "huge_shared" : " 27 GB", | 20260505 15:18:07.891 | INFO | "malloc" : " 14 GB", | 20260505 15:18:07.891 | INFO | "pinned" : " 29 GB" | 20260505 15:18:07.891 | INFO | } | 20260505 15:18:07.891 | INFO | Perf report for the last 5.1394 seconds: | 20260505 15:18:07.891 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:18:07.891 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:07.891 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:07.891 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:07.891 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 15:18:07.891 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:07.891 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:07.891 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 15:18:07.891 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:07.891 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:07.891 | INFO | Uploaded 36GiB to GPU in 2.20444s (16.3307GiB/s) | 20260505 15:18:07.891 | INFO | Downloaded 34GiB to GPU in 2.06734s (16.4463GiB/s) | 20260505 15:18:13.031 | INFO | Analyzing chunk frames = [4224000,4224512) | 20260505 15:18:13.031 | INFO | MemInfo: { | 20260505 15:18:13.031 | INFO | "gpu" : " 17 GB", | 20260505 15:18:13.031 | INFO | "huge" : "", | 20260505 15:18:13.031 | INFO | "huge_shared" : " 27 GB", | 20260505 15:18:13.031 | INFO | "malloc" : " 14 GB", | 20260505 15:18:13.031 | INFO | "pinned" : " 29 GB" | 20260505 15:18:13.031 | INFO | } | 20260505 15:18:13.032 | INFO | Perf report for the last 5.14071 seconds: | 20260505 15:18:13.032 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:18:13.032 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:13.032 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:13.032 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:13.032 | INFO | ANALYSIS : 62.8%, 3.0, 97.8ms, 0 | 20260505 15:18:13.032 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:13.032 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:13.032 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 15:18:13.032 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:13.032 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:13.032 | INFO | Uploaded 36GiB to GPU in 2.1968s (16.3875GiB/s) | 20260505 15:18:13.032 | INFO | Downloaded 34GiB to GPU in 2.03429s (16.7135GiB/s) | 20260505 15:18:18.161 | INFO | Analyzing chunk frames = [4224512,4225024) | 20260505 15:18:18.161 | INFO | MemInfo: { | 20260505 15:18:18.161 | INFO | "gpu" : " 17 GB", | 20260505 15:18:18.161 | INFO | "huge" : "", | 20260505 15:18:18.161 | INFO | "huge_shared" : " 27 GB", | 20260505 15:18:18.161 | INFO | "malloc" : " 14 GB", | 20260505 15:18:18.161 | INFO | "pinned" : " 29 GB" | 20260505 15:18:18.161 | INFO | } | 20260505 15:18:18.161 | INFO | Perf report for the last 5.12963 seconds: | 20260505 15:18:18.161 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:18:18.161 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:18.161 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:18.161 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:18.161 | INFO | ANALYSIS : 62.7%, 3.0, 97.2ms, 0 | 20260505 15:18:18.161 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:18.162 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:18.162 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 15:18:18.162 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:18.162 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:18.162 | INFO | Uploaded 36GiB to GPU in 2.18797s (16.4536GiB/s) | 20260505 15:18:18.162 | INFO | Downloaded 34GiB to GPU in 2.13385s (15.9337GiB/s) | 20260505 15:18:23.300 | INFO | Analyzing chunk frames = [4225024,4225536) | 20260505 15:18:23.300 | INFO | MemInfo: { | 20260505 15:18:23.300 | INFO | "gpu" : " 17 GB", | 20260505 15:18:23.300 | INFO | "huge" : "", | 20260505 15:18:23.300 | INFO | "huge_shared" : " 27 GB", | 20260505 15:18:23.300 | INFO | "malloc" : " 14 GB", | 20260505 15:18:23.300 | INFO | "pinned" : " 29 GB" | 20260505 15:18:23.300 | INFO | } | 20260505 15:18:23.300 | INFO | Perf report for the last 5.13886 seconds: | 20260505 15:18:23.300 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:18:23.300 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:23.300 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:23.300 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:23.300 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 15:18:23.300 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:23.300 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:23.300 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:18:23.300 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:23.300 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:23.300 | INFO | Uploaded 36GiB to GPU in 2.17016s (16.5886GiB/s) | 20260505 15:18:23.300 | INFO | Downloaded 34GiB to GPU in 2.13135s (15.9523GiB/s) | 20260505 15:18:28.449 | INFO | Analyzing chunk frames = [4225536,4226048) | 20260505 15:18:28.449 | INFO | MemInfo: { | 20260505 15:18:28.449 | INFO | "gpu" : " 17 GB", | 20260505 15:18:28.449 | INFO | "huge" : "", | 20260505 15:18:28.449 | INFO | "huge_shared" : " 27 GB", | 20260505 15:18:28.449 | INFO | "malloc" : " 14 GB", | 20260505 15:18:28.449 | INFO | "pinned" : " 29 GB" | 20260505 15:18:28.449 | INFO | } | 20260505 15:18:28.449 | INFO | Perf report for the last 5.14885 seconds: | 20260505 15:18:28.449 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:18:28.449 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:28.449 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:28.449 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:28.449 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 15:18:28.449 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:28.449 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:28.449 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:18:28.449 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:28.449 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:28.449 | INFO | Uploaded 36GiB to GPU in 2.16571s (16.6227GiB/s) | 20260505 15:18:28.449 | INFO | Downloaded 34GiB to GPU in 2.11075s (16.108GiB/s) | 20260505 15:18:33.591 | INFO | Analyzing chunk frames = [4226048,4226560) | 20260505 15:18:33.591 | INFO | MemInfo: { | 20260505 15:18:33.591 | INFO | "gpu" : " 17 GB", | 20260505 15:18:33.591 | INFO | "huge" : "", | 20260505 15:18:33.591 | INFO | "huge_shared" : " 27 GB", | 20260505 15:18:33.591 | INFO | "malloc" : " 14 GB", | 20260505 15:18:33.591 | INFO | "pinned" : " 29 GB" | 20260505 15:18:33.591 | INFO | } | 20260505 15:18:33.591 | INFO | Perf report for the last 5.14194 seconds: | 20260505 15:18:33.591 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:18:33.591 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:33.591 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:33.591 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:33.591 | INFO | ANALYSIS : 63.2%, 3.0, 98.8ms, 0 | 20260505 15:18:33.591 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:33.591 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:33.591 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 15:18:33.591 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:33.591 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:33.591 | INFO | Uploaded 36GiB to GPU in 2.10242s (17.1232GiB/s) | 20260505 15:18:33.591 | INFO | Downloaded 34GiB to GPU in 2.17307s (15.6461GiB/s) | 20260505 15:18:38.727 | INFO | Analyzing chunk frames = [4226560,4227072) | 20260505 15:18:38.727 | INFO | MemInfo: { | 20260505 15:18:38.727 | INFO | "gpu" : " 17 GB", | 20260505 15:18:38.727 | INFO | "huge" : "", | 20260505 15:18:38.727 | INFO | "huge_shared" : " 27 GB", | 20260505 15:18:38.727 | INFO | "malloc" : " 14 GB", | 20260505 15:18:38.727 | INFO | "pinned" : " 29 GB" | 20260505 15:18:38.727 | INFO | } | 20260505 15:18:38.727 | INFO | Perf report for the last 5.13634 seconds: | 20260505 15:18:38.727 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:18:38.727 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:38.727 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:38.727 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:38.727 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 15:18:38.727 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:38.727 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:38.727 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 15:18:38.727 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:38.727 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:38.727 | INFO | Uploaded 36GiB to GPU in 2.15989s (16.6675GiB/s) | 20260505 15:18:38.728 | INFO | Downloaded 34GiB to GPU in 2.0312s (16.7389GiB/s) | 20260505 15:18:38.728 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 15:18:38.728 | INFO | Total mem usage: 15.3 GiB | 20260505 15:18:38.728 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 15:18:38.728 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.85%) | 20260505 15:18:38.728 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 15:18:38.728 | INFO | High Water Breakdown: | 20260505 15:18:38.728 | INFO | 61% : 8.67 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 15:18:38.728 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 15:18:38.728 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 15:18:38.728 | INFO | ------------------------------------------ | 20260505 15:18:38.728 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:18:38.728 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 15:18:38.728 | INFO | Total mem usage: 26.5 GiB | 20260505 15:18:38.728 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 15:18:38.728 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 15:18:38.728 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 15:18:38.728 | INFO | High Water Breakdown: | 20260505 15:18:38.728 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:18:38.728 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:18:38.728 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:18:38.728 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:18:38.728 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:18:38.728 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:18:38.728 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 15:18:38.728 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 15:18:38.728 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:18:38.728 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 15:18:38.728 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 15:18:38.728 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 15:18:38.728 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 15:18:38.728 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 15:18:38.728 | INFO | ------------------------------------------ | 20260505 15:18:38.728 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:18:38.728 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 15:18:38.728 | INFO | Total mem usage: 25.6 GiB | 20260505 15:18:38.728 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 15:18:38.728 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 15:18:38.728 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 15:18:38.728 | INFO | High Water Breakdown: | 20260505 15:18:38.728 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 15:18:38.728 | INFO | ------------------------------------------ | 20260505 15:18:38.728 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:18:38.729 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 15:18:38.729 | INFO | Total mem usage: 15.7 GiB | 20260505 15:18:38.729 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 15:18:38.729 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 15:18:38.729 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 15:18:38.729 | INFO | High Water Breakdown: | 20260505 15:18:38.729 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 15:18:38.729 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 15:18:38.729 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 15:18:38.729 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 15:18:38.729 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 15:18:38.729 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 15:18:38.729 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 15:18:38.729 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 15:18:38.729 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 15:18:38.729 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:18:38.729 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 15:18:38.729 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:18:38.729 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 15:18:38.729 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 15:18:38.729 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 15:18:38.729 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 15:18:38.729 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:18:38.729 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 15:18:38.729 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:18:38.729 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:18:38.729 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:18:38.729 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 15:18:38.729 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 15:18:38.729 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:18:38.729 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 15:18:38.729 | INFO | ------------------------------------------ | 20260505 15:18:38.729 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:18:38.729 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 15:18:38.729 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:18:38.729 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:18:38.729 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.55e+01, 1.37e+01 / 1.86e+00 | 20260505 15:18:38.729 | INFO | PerfReport: 0.29%, AggregateStats -- 9.30e-02 / 1.01e-01, 1.05e-02 / 4.14e-03 | 20260505 15:18:38.729 | INFO | PerfReport: 11.81%, Baseline -- 3.74e+00 / 4.52e+00, 5.76e-01 / 1.26e-01 | 20260505 15:18:38.729 | INFO | PerfReport: 3.30%, Histogram -- 1.05e+00 / 1.51e+00, 1.96e-01 / 9.22e-02 | 20260505 15:18:38.729 | INFO | PerfReport: 15.93%, DME -- 4.04e+01 / 4.49e+01, 4.31e+00 / 1.79e+00 | 20260505 15:18:38.729 | INFO | PerfReport: 56.53%, FrameLabeling -- 1.79e+01 / 1.98e+01, 1.38e+00 / 1.48e+00 | 20260505 15:18:38.729 | INFO | PerfReport: 7.43%, PulseAccumulating -- 2.36e+00 / 2.89e+00, 4.49e-01 / 2.02e-01 | 20260505 15:18:38.729 | INFO | PerfReport: 3.48%, Metrics -- 1.10e+00 / 1.42e+00, 3.02e-01 / 1.46e-01 | 20260505 15:18:38.729 | INFO | PerfReport: 1.22%, PulseSerializing -- 3.87e-01 / 4.37e-01, 6.84e-02 / 3.13e-02 | 20260505 15:18:38.729 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 15:18:38.729 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:18:38.729 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:18:38.729 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260505 15:18:38.729 | INFO | PerfReport: 22.56%, Upload -- 2.27e+01 / 0.00e+00, 7.94e+00 / 0.00e+00 | 20260505 15:18:38.729 | INFO | PerfReport: 21.92%, Download -- 2.20e+01 / 0.00e+00, 7.94e+00 / 0.00e+00 | 20260505 15:18:38.729 | INFO | PerfReport: 31.55%, Compute -- 3.17e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 15:18:38.729 | INFO | PerfReport: 2.30%, LockUpload -- 2.31e+00 / 0.00e+00, 8.04e+00 / 0.00e+00 | 20260505 15:18:38.729 | INFO | PerfReport: 1.50%, LockDownload -- 1.51e+00 / 0.00e+00, 3.09e+00 / 0.00e+00 | 20260505 15:18:38.729 | INFO | PerfReport: 20.15%, LockCompute -- 2.02e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 15:18:38.729 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.68e-02 / 0.00e+00 | 20260505 15:18:38.862 | INFO | Starting a round of Baz IO | 20260505 15:18:42.080 | INFO | Populating RTMetrics report | 20260505 15:18:42.081 | INFO | RTMetrics report populated | 20260505 15:18:42.081 | INFO | Opening tmp RTMetrics json file | 20260505 15:18:42.093 | INFO | Serializing RTMetrics to json | 20260505 15:18:42.123 | INFO | Serialization done, swapping tmp file now... | 20260505 15:18:42.145 | INFO | RTMetrics done | 20260505 15:18:42.179 | INFO | Finished a round of Baz IO: Wrote 2.22 GiB in 3.316 seconds | 20260505 15:18:42.179 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.51% / 0.00% / 77.49% | 20260505 15:18:42.179 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.461344% / 3.97364e-06% / 5.5631e-05% | 20260505 15:18:42.179 | INFO | Baz IO overall: | 20260505 15:18:42.180 | INFO | BytesWritten : 2515447287734 | 20260505 15:18:42.180 | INFO | NumEvents : 1195138349203 | 20260505 15:18:43.858 | INFO | Analyzing chunk frames = [4227072,4227584) | 20260505 15:18:43.858 | INFO | MemInfo: { | 20260505 15:18:43.858 | INFO | "gpu" : " 17 GB", | 20260505 15:18:43.858 | INFO | "huge" : "", | 20260505 15:18:43.858 | INFO | "huge_shared" : " 27 GB", | 20260505 15:18:43.858 | INFO | "malloc" : " 14 GB", | 20260505 15:18:43.858 | INFO | "pinned" : " 29 GB" | 20260505 15:18:43.858 | INFO | } | 20260505 15:18:43.858 | INFO | Perf report for the last 5.13074 seconds: | 20260505 15:18:43.858 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:18:43.858 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:43.858 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:43.858 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:43.858 | INFO | ANALYSIS : 64.2%, 3.0, 99.6ms, 0 | 20260505 15:18:43.858 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:43.858 | INFO | RT_METRICS : 46.2%, 1.0, 24.2ms, 0 | 20260505 15:18:43.858 | INFO | PRE_HQ : 61.0%, 1.8, 57.3ms, 0 | 20260505 15:18:43.858 | INFO | BAZWRITER : 63.8%, 4.0, 131.4ms, 0 | 20260505 15:18:43.858 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 15:18:43.858 | INFO | Uploaded 36GiB to GPU in 2.19858s (16.3742GiB/s) | 20260505 15:18:43.858 | INFO | Downloaded 38GiB to GPU in 2.26754s (16.7582GiB/s) | 20260505 15:18:49.001 | INFO | Analyzing chunk frames = [4227584,4228096) | 20260505 15:18:49.001 | INFO | MemInfo: { | 20260505 15:18:49.001 | INFO | "gpu" : " 17 GB", | 20260505 15:18:49.001 | INFO | "huge" : "", | 20260505 15:18:49.001 | INFO | "huge_shared" : " 27 GB", | 20260505 15:18:49.001 | INFO | "malloc" : " 14 GB", | 20260505 15:18:49.001 | INFO | "pinned" : " 29 GB" | 20260505 15:18:49.001 | INFO | } | 20260505 15:18:49.001 | INFO | Perf report for the last 5.14276 seconds: | 20260505 15:18:49.001 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:18:49.001 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:49.001 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:49.001 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:49.001 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 15:18:49.001 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:49.001 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:49.001 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:18:49.001 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:49.001 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:49.001 | INFO | Uploaded 36GiB to GPU in 2.18924s (16.444GiB/s) | 20260505 15:18:49.001 | INFO | Downloaded 34GiB to GPU in 2.02413s (16.7973GiB/s) | 20260505 15:18:54.143 | INFO | Analyzing chunk frames = [4228096,4228608) | 20260505 15:18:54.143 | INFO | MemInfo: { | 20260505 15:18:54.143 | INFO | "gpu" : " 17 GB", | 20260505 15:18:54.143 | INFO | "huge" : "", | 20260505 15:18:54.143 | INFO | "huge_shared" : " 27 GB", | 20260505 15:18:54.143 | INFO | "malloc" : " 14 GB", | 20260505 15:18:54.143 | INFO | "pinned" : " 29 GB" | 20260505 15:18:54.143 | INFO | } | 20260505 15:18:54.143 | INFO | Perf report for the last 5.14246 seconds: | 20260505 15:18:54.143 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:18:54.143 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:54.143 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:54.143 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:54.143 | INFO | ANALYSIS : 63.0%, 3.0, 98.1ms, 0 | 20260505 15:18:54.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:54.143 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:54.143 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 15:18:54.143 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:54.143 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:54.144 | INFO | Uploaded 36GiB to GPU in 2.17986s (16.5148GiB/s) | 20260505 15:18:54.144 | INFO | Downloaded 34GiB to GPU in 2.05531s (16.5425GiB/s) | 20260505 15:18:59.286 | INFO | Analyzing chunk frames = [4228608,4229120) | 20260505 15:18:59.286 | INFO | MemInfo: { | 20260505 15:18:59.286 | INFO | "gpu" : " 17 GB", | 20260505 15:18:59.286 | INFO | "huge" : "", | 20260505 15:18:59.286 | INFO | "huge_shared" : " 27 GB", | 20260505 15:18:59.286 | INFO | "malloc" : " 14 GB", | 20260505 15:18:59.286 | INFO | "pinned" : " 29 GB" | 20260505 15:18:59.286 | INFO | } | 20260505 15:18:59.286 | INFO | Perf report for the last 5.14306 seconds: | 20260505 15:18:59.286 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:18:59.286 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:59.286 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:59.286 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:59.286 | INFO | ANALYSIS : 62.6%, 3.0, 97.3ms, 0 | 20260505 15:18:59.286 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:59.286 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:59.286 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:18:59.286 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:59.286 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:18:59.287 | INFO | Uploaded 36GiB to GPU in 2.19144s (16.4275GiB/s) | 20260505 15:18:59.287 | INFO | Downloaded 34GiB to GPU in 2.13319s (15.9386GiB/s) | 20260505 15:19:04.420 | INFO | Analyzing chunk frames = [4229120,4229632) | 20260505 15:19:04.420 | INFO | MemInfo: { | 20260505 15:19:04.420 | INFO | "gpu" : " 17 GB", | 20260505 15:19:04.420 | INFO | "huge" : "", | 20260505 15:19:04.420 | INFO | "huge_shared" : " 27 GB", | 20260505 15:19:04.420 | INFO | "malloc" : " 14 GB", | 20260505 15:19:04.420 | INFO | "pinned" : " 29 GB" | 20260505 15:19:04.420 | INFO | } | 20260505 15:19:04.420 | INFO | Perf report for the last 5.13411 seconds: | 20260505 15:19:04.420 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:19:04.420 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:04.420 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:04.420 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:04.420 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260505 15:19:04.420 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:04.420 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:04.420 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:19:04.420 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:04.420 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:04.421 | INFO | Uploaded 36GiB to GPU in 2.17982s (16.5151GiB/s) | 20260505 15:19:04.421 | INFO | Downloaded 34GiB to GPU in 2.12719s (15.9835GiB/s) | 20260505 15:19:09.557 | INFO | Analyzing chunk frames = [4229632,4230144) | 20260505 15:19:09.557 | INFO | MemInfo: { | 20260505 15:19:09.557 | INFO | "gpu" : " 17 GB", | 20260505 15:19:09.557 | INFO | "huge" : "", | 20260505 15:19:09.557 | INFO | "huge_shared" : " 27 GB", | 20260505 15:19:09.557 | INFO | "malloc" : " 14 GB", | 20260505 15:19:09.557 | INFO | "pinned" : " 29 GB" | 20260505 15:19:09.557 | INFO | } | 20260505 15:19:09.558 | INFO | Perf report for the last 5.13714 seconds: | 20260505 15:19:09.558 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:19:09.558 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:09.558 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:09.558 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:09.558 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 15:19:09.558 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:09.558 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:09.558 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 15:19:09.558 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:09.558 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:09.558 | INFO | Uploaded 36GiB to GPU in 2.20409s (16.3333GiB/s) | 20260505 15:19:09.558 | INFO | Downloaded 34GiB to GPU in 2.10958s (16.117GiB/s) | 20260505 15:19:14.704 | INFO | Analyzing chunk frames = [4230144,4230656) | 20260505 15:19:14.704 | INFO | MemInfo: { | 20260505 15:19:14.704 | INFO | "gpu" : " 17 GB", | 20260505 15:19:14.704 | INFO | "huge" : "", | 20260505 15:19:14.704 | INFO | "huge_shared" : " 27 GB", | 20260505 15:19:14.704 | INFO | "malloc" : " 14 GB", | 20260505 15:19:14.704 | INFO | "pinned" : " 29 GB" | 20260505 15:19:14.704 | INFO | } | 20260505 15:19:14.704 | INFO | Perf report for the last 5.14676 seconds: | 20260505 15:19:14.704 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:19:14.704 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:14.704 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:14.704 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:14.704 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 15:19:14.704 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:14.704 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:14.704 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 15:19:14.704 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:14.704 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:14.705 | INFO | Uploaded 36GiB to GPU in 2.15413s (16.7121GiB/s) | 20260505 15:19:14.705 | INFO | Downloaded 34GiB to GPU in 2.19732s (15.4734GiB/s) | 20260505 15:19:19.849 | INFO | Analyzing chunk frames = [4230656,4231168) | 20260505 15:19:19.849 | INFO | MemInfo: { | 20260505 15:19:19.849 | INFO | "gpu" : " 17 GB", | 20260505 15:19:19.849 | INFO | "huge" : "", | 20260505 15:19:19.849 | INFO | "huge_shared" : " 27 GB", | 20260505 15:19:19.849 | INFO | "malloc" : " 14 GB", | 20260505 15:19:19.849 | INFO | "pinned" : " 29 GB" | 20260505 15:19:19.849 | INFO | } | 20260505 15:19:19.849 | INFO | Perf report for the last 5.14496 seconds: | 20260505 15:19:19.849 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:19:19.849 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:19.849 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:19.849 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:19.849 | INFO | ANALYSIS : 63.1%, 3.0, 98.7ms, 0 | 20260505 15:19:19.849 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:19.849 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:19.849 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 15:19:19.849 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:19.850 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:19.850 | INFO | Uploaded 36GiB to GPU in 2.1514s (16.7333GiB/s) | 20260505 15:19:19.850 | INFO | Downloaded 34GiB to GPU in 2.0348s (16.7093GiB/s) | 20260505 15:19:19.991 | INFO | Starting a round of Baz IO | 20260505 15:19:23.190 | INFO | Populating RTMetrics report | 20260505 15:19:23.191 | INFO | RTMetrics report populated | 20260505 15:19:23.191 | INFO | Opening tmp RTMetrics json file | 20260505 15:19:23.201 | INFO | Serializing RTMetrics to json | 20260505 15:19:23.204 | INFO | Serialization done, swapping tmp file now... | 20260505 15:19:23.208 | INFO | RTMetrics done | 20260505 15:19:23.296 | INFO | Finished a round of Baz IO: Wrote 2.223 GiB in 3.304 seconds | 20260505 15:19:23.296 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.50% / 0.00% / 77.49% | 20260505 15:19:23.296 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.464499% / 7.94729e-06% / 5.16574e-05% | 20260505 15:19:23.296 | INFO | Baz IO overall: | 20260505 15:19:23.296 | INFO | BytesWritten : 2517834412396 | 20260505 15:19:23.296 | INFO | NumEvents : 1196262683089 | 20260505 15:19:24.979 | INFO | Analyzing chunk frames = [4231168,4231680) | 20260505 15:19:24.979 | INFO | MemInfo: { | 20260505 15:19:24.979 | INFO | "gpu" : " 17 GB", | 20260505 15:19:24.979 | INFO | "huge" : "", | 20260505 15:19:24.979 | INFO | "huge_shared" : " 27 GB", | 20260505 15:19:24.979 | INFO | "malloc" : " 14 GB", | 20260505 15:19:24.979 | INFO | "pinned" : " 29 GB" | 20260505 15:19:24.979 | INFO | } | 20260505 15:19:24.979 | INFO | Perf report for the last 5.12966 seconds: | 20260505 15:19:24.979 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:19:24.979 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:24.979 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:24.979 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:24.979 | INFO | ANALYSIS : 64.0%, 3.0, 99.3ms, 0 | 20260505 15:19:24.979 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:24.979 | INFO | RT_METRICS : 45.3%, 1.0, 23.7ms, 0 | 20260505 15:19:24.979 | INFO | PRE_HQ : 60.7%, 1.8, 57.0ms, 0 | 20260505 15:19:24.979 | INFO | BAZWRITER : 63.9%, 4.0, 129.9ms, 0 | 20260505 15:19:24.979 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 15:19:24.979 | INFO | Uploaded 36GiB to GPU in 2.18654s (16.4644GiB/s) | 20260505 15:19:24.979 | INFO | Downloaded 38GiB to GPU in 2.26998s (16.7402GiB/s) | 20260505 15:19:30.112 | INFO | Analyzing chunk frames = [4231680,4232192) | 20260505 15:19:30.112 | INFO | MemInfo: { | 20260505 15:19:30.112 | INFO | "gpu" : " 17 GB", | 20260505 15:19:30.112 | INFO | "huge" : "", | 20260505 15:19:30.112 | INFO | "huge_shared" : " 27 GB", | 20260505 15:19:30.113 | INFO | "malloc" : " 14 GB", | 20260505 15:19:30.113 | INFO | "pinned" : " 29 GB" | 20260505 15:19:30.113 | INFO | } | 20260505 15:19:30.113 | INFO | Perf report for the last 5.13369 seconds: | 20260505 15:19:30.113 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:19:30.113 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:30.113 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:30.113 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:30.113 | INFO | ANALYSIS : 63.5%, 3.0, 99.0ms, 0 | 20260505 15:19:30.113 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:30.113 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:30.113 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 15:19:30.113 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:30.113 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:30.113 | INFO | Uploaded 36GiB to GPU in 2.20167s (16.3512GiB/s) | 20260505 15:19:30.113 | INFO | Downloaded 34GiB to GPU in 2.04237s (16.6473GiB/s) | 20260505 15:19:35.256 | INFO | Analyzing chunk frames = [4232192,4232704) | 20260505 15:19:35.256 | INFO | MemInfo: { | 20260505 15:19:35.256 | INFO | "gpu" : " 17 GB", | 20260505 15:19:35.256 | INFO | "huge" : "", | 20260505 15:19:35.256 | INFO | "huge_shared" : " 27 GB", | 20260505 15:19:35.256 | INFO | "malloc" : " 14 GB", | 20260505 15:19:35.256 | INFO | "pinned" : " 29 GB" | 20260505 15:19:35.256 | INFO | } | 20260505 15:19:35.256 | INFO | Perf report for the last 5.14326 seconds: | 20260505 15:19:35.256 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:19:35.256 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:35.256 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:35.256 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:35.256 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 15:19:35.256 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:35.256 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:35.256 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 15:19:35.256 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:35.256 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:35.256 | INFO | Uploaded 36GiB to GPU in 2.23751s (16.0893GiB/s) | 20260505 15:19:35.256 | INFO | Downloaded 34GiB to GPU in 2.09934s (16.1955GiB/s) | 20260505 15:19:40.398 | INFO | Analyzing chunk frames = [4232704,4233216) | 20260505 15:19:40.398 | INFO | MemInfo: { | 20260505 15:19:40.398 | INFO | "gpu" : " 17 GB", | 20260505 15:19:40.398 | INFO | "huge" : "", | 20260505 15:19:40.398 | INFO | "huge_shared" : " 27 GB", | 20260505 15:19:40.398 | INFO | "malloc" : " 14 GB", | 20260505 15:19:40.398 | INFO | "pinned" : " 29 GB" | 20260505 15:19:40.398 | INFO | } | 20260505 15:19:40.398 | INFO | Perf report for the last 5.14239 seconds: | 20260505 15:19:40.398 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:19:40.398 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:40.398 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:40.398 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:40.398 | INFO | ANALYSIS : 62.5%, 3.0, 97.1ms, 0 | 20260505 15:19:40.398 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:40.398 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:40.399 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:19:40.399 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:40.399 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:40.399 | INFO | Uploaded 36GiB to GPU in 2.18177s (16.5004GiB/s) | 20260505 15:19:40.399 | INFO | Downloaded 34GiB to GPU in 2.1304s (15.9594GiB/s) | 20260505 15:19:45.543 | INFO | Analyzing chunk frames = [4233216,4233728) | 20260505 15:19:45.543 | INFO | MemInfo: { | 20260505 15:19:45.543 | INFO | "gpu" : " 17 GB", | 20260505 15:19:45.543 | INFO | "huge" : "", | 20260505 15:19:45.543 | INFO | "huge_shared" : " 27 GB", | 20260505 15:19:45.544 | INFO | "malloc" : " 14 GB", | 20260505 15:19:45.544 | INFO | "pinned" : " 29 GB" | 20260505 15:19:45.544 | INFO | } | 20260505 15:19:45.544 | INFO | Perf report for the last 5.14521 seconds: | 20260505 15:19:45.544 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:19:45.544 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:45.544 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:45.544 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:45.544 | INFO | ANALYSIS : 62.6%, 3.0, 97.9ms, 0 | 20260505 15:19:45.544 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:45.544 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:45.544 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 15:19:45.544 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:45.544 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:45.544 | INFO | Uploaded 36GiB to GPU in 2.16724s (16.611GiB/s) | 20260505 15:19:45.544 | INFO | Downloaded 34GiB to GPU in 2.12867s (15.9724GiB/s) | 20260505 15:19:50.678 | INFO | Analyzing chunk frames = [4233728,4234240) | 20260505 15:19:50.678 | INFO | MemInfo: { | 20260505 15:19:50.678 | INFO | "gpu" : " 17 GB", | 20260505 15:19:50.678 | INFO | "huge" : "", | 20260505 15:19:50.678 | INFO | "huge_shared" : " 27 GB", | 20260505 15:19:50.678 | INFO | "malloc" : " 14 GB", | 20260505 15:19:50.678 | INFO | "pinned" : " 29 GB" | 20260505 15:19:50.678 | INFO | } | 20260505 15:19:50.678 | INFO | Perf report for the last 5.13423 seconds: | 20260505 15:19:50.678 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:19:50.678 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:50.678 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:50.678 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:50.678 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 15:19:50.678 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:50.678 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:50.678 | INFO | PRE_HQ : 27.9%, 1.0, 14.4ms, 0 | 20260505 15:19:50.678 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:50.678 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:50.678 | INFO | Uploaded 36GiB to GPU in 2.15564s (16.7004GiB/s) | 20260505 15:19:50.678 | INFO | Downloaded 34GiB to GPU in 2.0388s (16.6765GiB/s) | 20260505 15:19:55.808 | INFO | Analyzing chunk frames = [4234240,4234752) | 20260505 15:19:55.808 | INFO | MemInfo: { | 20260505 15:19:55.808 | INFO | "gpu" : " 17 GB", | 20260505 15:19:55.808 | INFO | "huge" : "", | 20260505 15:19:55.808 | INFO | "huge_shared" : " 27 GB", | 20260505 15:19:55.808 | INFO | "malloc" : " 14 GB", | 20260505 15:19:55.808 | INFO | "pinned" : " 29 GB" | 20260505 15:19:55.808 | INFO | } | 20260505 15:19:55.808 | INFO | Perf report for the last 5.13018 seconds: | 20260505 15:19:55.808 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:19:55.808 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:55.808 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:55.808 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:55.808 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 15:19:55.808 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:55.808 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:55.808 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 15:19:55.808 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:55.808 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:19:55.808 | INFO | Uploaded 36GiB to GPU in 2.14168s (16.8092GiB/s) | 20260505 15:19:55.808 | INFO | Downloaded 34GiB to GPU in 2.15969s (15.743GiB/s) | 20260505 15:20:00.956 | INFO | Analyzing chunk frames = [4234752,4235264) | 20260505 15:20:00.956 | INFO | MemInfo: { | 20260505 15:20:00.956 | INFO | "gpu" : " 17 GB", | 20260505 15:20:00.956 | INFO | "huge" : "", | 20260505 15:20:00.956 | INFO | "huge_shared" : " 27 GB", | 20260505 15:20:00.956 | INFO | "malloc" : " 14 GB", | 20260505 15:20:00.956 | INFO | "pinned" : " 29 GB" | 20260505 15:20:00.956 | INFO | } | 20260505 15:20:00.956 | INFO | Perf report for the last 5.14779 seconds: | 20260505 15:20:00.956 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:20:00.956 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:00.956 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:00.956 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:00.956 | INFO | ANALYSIS : 63.4%, 3.0, 99.2ms, 0 | 20260505 15:20:00.956 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:00.956 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:00.956 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 15:20:00.956 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:00.956 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:00.956 | INFO | Uploaded 36GiB to GPU in 2.2233s (16.1921GiB/s) | 20260505 15:20:00.956 | INFO | Downloaded 34GiB to GPU in 2.10566s (16.147GiB/s) | 20260505 15:20:00.956 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 15:20:00.956 | INFO | Total mem usage: 15.3 GiB | 20260505 15:20:00.956 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 15:20:00.956 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.85%) | 20260505 15:20:00.956 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 15:20:00.956 | INFO | High Water Breakdown: | 20260505 15:20:00.956 | INFO | 61% : 8.67 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 15:20:00.956 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 15:20:00.956 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 15:20:00.956 | INFO | ------------------------------------------ | 20260505 15:20:00.956 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:20:00.956 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 15:20:00.956 | INFO | Total mem usage: 26.5 GiB | 20260505 15:20:00.956 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 15:20:00.956 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 15:20:00.956 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 15:20:00.956 | INFO | High Water Breakdown: | 20260505 15:20:00.957 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:20:00.957 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:20:00.957 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:20:00.957 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:20:00.957 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:20:00.957 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:20:00.957 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 15:20:00.957 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 15:20:00.957 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:20:00.957 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 15:20:00.957 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 15:20:00.957 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 15:20:00.957 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 15:20:00.957 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 15:20:00.957 | INFO | ------------------------------------------ | 20260505 15:20:00.957 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:20:00.957 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 15:20:00.957 | INFO | Total mem usage: 25.6 GiB | 20260505 15:20:00.957 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 15:20:00.957 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 15:20:00.957 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 15:20:00.957 | INFO | High Water Breakdown: | 20260505 15:20:00.957 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 15:20:00.957 | INFO | ------------------------------------------ | 20260505 15:20:00.957 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:20:00.957 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 15:20:00.957 | INFO | Total mem usage: 15.7 GiB | 20260505 15:20:00.957 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 15:20:00.957 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 15:20:00.957 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 15:20:00.957 | INFO | High Water Breakdown: | 20260505 15:20:00.957 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 15:20:00.957 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 15:20:00.957 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 15:20:00.957 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 15:20:00.957 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 15:20:00.957 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 15:20:00.957 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 15:20:00.957 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 15:20:00.957 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 15:20:00.957 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:20:00.957 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 15:20:00.957 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:20:00.957 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 15:20:00.957 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 15:20:00.957 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 15:20:00.957 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 15:20:00.957 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:20:00.957 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 15:20:00.957 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:20:00.957 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:20:00.957 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:20:00.957 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 15:20:00.957 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 15:20:00.957 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:20:00.957 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 15:20:00.957 | INFO | ------------------------------------------ | 20260505 15:20:00.957 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:20:00.957 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 15:20:00.957 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:20:00.957 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:20:00.957 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.59e+01, 1.36e+01 / 1.90e+00 | 20260505 15:20:00.958 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 1.01e-01, 1.08e-02 / 6.15e-03 | 20260505 15:20:00.958 | INFO | PerfReport: 11.78%, Baseline -- 3.74e+00 / 4.48e+00, 5.81e-01 / 1.81e-01 | 20260505 15:20:00.958 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.48e+00, 1.94e-01 / 1.21e-01 | 20260505 15:20:00.958 | INFO | PerfReport: 15.94%, DME -- 4.05e+01 / 4.52e+01, 4.54e+00 / 1.96e+00 | 20260505 15:20:00.958 | INFO | PerfReport: 56.60%, FrameLabeling -- 1.80e+01 / 1.98e+01, 1.46e+00 / 1.87e+00 | 20260505 15:20:00.958 | INFO | PerfReport: 7.40%, PulseAccumulating -- 2.35e+00 / 2.90e+00, 4.51e-01 / 2.14e-01 | 20260505 15:20:00.958 | INFO | PerfReport: 3.48%, Metrics -- 1.10e+00 / 1.43e+00, 3.09e-01 / 1.55e-01 | 20260505 15:20:00.958 | INFO | PerfReport: 1.22%, PulseSerializing -- 3.88e-01 / 4.36e-01, 6.93e-02 / 3.38e-02 | 20260505 15:20:00.958 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 15:20:00.958 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:20:00.958 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:20:00.958 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260505 15:20:00.958 | INFO | PerfReport: 22.63%, Upload -- 2.28e+01 / 0.00e+00, 8.07e+00 / 0.00e+00 | 20260505 15:20:00.958 | INFO | PerfReport: 21.97%, Download -- 2.21e+01 / 0.00e+00, 8.04e+00 / 0.00e+00 | 20260505 15:20:00.958 | INFO | PerfReport: 31.59%, Compute -- 3.18e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 15:20:00.958 | INFO | PerfReport: 2.26%, LockUpload -- 2.27e+00 / 0.00e+00, 7.93e+00 / 0.00e+00 | 20260505 15:20:00.958 | INFO | PerfReport: 1.44%, LockDownload -- 1.45e+00 / 0.00e+00, 2.97e+00 / 0.00e+00 | 20260505 15:20:00.958 | INFO | PerfReport: 20.10%, LockCompute -- 2.02e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 15:20:00.958 | INFO | PerfReport: 0.01%, Misc -- 1.47e-02 / 0.00e+00, 3.18e-02 / 0.00e+00 | 20260505 15:20:01.084 | INFO | Starting a round of Baz IO | 20260505 15:20:04.303 | INFO | Populating RTMetrics report | 20260505 15:20:04.304 | INFO | RTMetrics report populated | 20260505 15:20:04.304 | INFO | Opening tmp RTMetrics json file | 20260505 15:20:04.323 | INFO | Serializing RTMetrics to json | 20260505 15:20:04.356 | INFO | Serialization done, swapping tmp file now... | 20260505 15:20:04.383 | INFO | RTMetrics done | 20260505 15:20:04.407 | INFO | Finished a round of Baz IO: Wrote 2.219 GiB in 3.322 seconds | 20260505 15:20:04.407 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.50% / 0.00% / 77.50% | 20260505 15:20:04.407 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.460343% / 1.19209e-05% / 8.34465e-05% | 20260505 15:20:04.407 | INFO | Baz IO overall: | 20260505 15:20:04.407 | INFO | BytesWritten : 2520217352949 | 20260505 15:20:04.407 | INFO | NumEvents : 1197383734012 | 20260505 15:20:06.096 | INFO | Analyzing chunk frames = [4235264,4235776) | 20260505 15:20:06.096 | INFO | MemInfo: { | 20260505 15:20:06.096 | INFO | "gpu" : " 17 GB", | 20260505 15:20:06.096 | INFO | "huge" : "", | 20260505 15:20:06.096 | INFO | "huge_shared" : " 27 GB", | 20260505 15:20:06.096 | INFO | "malloc" : " 14 GB", | 20260505 15:20:06.096 | INFO | "pinned" : " 29 GB" | 20260505 15:20:06.096 | INFO | } | 20260505 15:20:06.096 | INFO | Perf report for the last 5.14004 seconds: | 20260505 15:20:06.096 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:20:06.096 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:06.096 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:06.096 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:06.096 | INFO | ANALYSIS : 64.0%, 3.0, 99.6ms, 0 | 20260505 15:20:06.096 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:06.096 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260505 15:20:06.096 | INFO | PRE_HQ : 61.0%, 1.8, 56.8ms, 0 | 20260505 15:20:06.096 | INFO | BAZWRITER : 64.0%, 4.0, 130.5ms, 0 | 20260505 15:20:06.096 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 15:20:06.096 | INFO | Uploaded 36GiB to GPU in 2.19463s (16.4037GiB/s) | 20260505 15:20:06.096 | INFO | Downloaded 38GiB to GPU in 2.25513s (16.8505GiB/s) | 20260505 15:20:11.230 | INFO | Analyzing chunk frames = [4235776,4236288) | 20260505 15:20:11.230 | INFO | MemInfo: { | 20260505 15:20:11.230 | INFO | "gpu" : " 17 GB", | 20260505 15:20:11.230 | INFO | "huge" : "", | 20260505 15:20:11.230 | INFO | "huge_shared" : " 27 GB", | 20260505 15:20:11.230 | INFO | "malloc" : " 14 GB", | 20260505 15:20:11.231 | INFO | "pinned" : " 29 GB" | 20260505 15:20:11.231 | INFO | } | 20260505 15:20:11.231 | INFO | Perf report for the last 5.13471 seconds: | 20260505 15:20:11.231 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:20:11.231 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:11.231 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:11.231 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:11.231 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 15:20:11.231 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:11.231 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:11.231 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 15:20:11.231 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:11.231 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:11.231 | INFO | Uploaded 36GiB to GPU in 2.20504s (16.3262GiB/s) | 20260505 15:20:11.231 | INFO | Downloaded 34GiB to GPU in 2.06383s (16.4742GiB/s) | 20260505 15:20:16.367 | INFO | Analyzing chunk frames = [4236288,4236800) | 20260505 15:20:16.367 | INFO | MemInfo: { | 20260505 15:20:16.367 | INFO | "gpu" : " 17 GB", | 20260505 15:20:16.367 | INFO | "huge" : "", | 20260505 15:20:16.367 | INFO | "huge_shared" : " 27 GB", | 20260505 15:20:16.367 | INFO | "malloc" : " 14 GB", | 20260505 15:20:16.367 | INFO | "pinned" : " 29 GB" | 20260505 15:20:16.367 | INFO | } | 20260505 15:20:16.368 | INFO | Perf report for the last 5.13692 seconds: | 20260505 15:20:16.368 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:20:16.368 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:16.368 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:16.368 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:16.368 | INFO | ANALYSIS : 63.0%, 3.0, 98.0ms, 0 | 20260505 15:20:16.368 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:16.368 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:16.368 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:20:16.368 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:16.368 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:16.368 | INFO | Uploaded 36GiB to GPU in 2.1847s (16.4783GiB/s) | 20260505 15:20:16.368 | INFO | Downloaded 34GiB to GPU in 2.05089s (16.5782GiB/s) | 20260505 15:20:21.512 | INFO | Analyzing chunk frames = [4236800,4237312) | 20260505 15:20:21.512 | INFO | MemInfo: { | 20260505 15:20:21.512 | INFO | "gpu" : " 17 GB", | 20260505 15:20:21.512 | INFO | "huge" : "", | 20260505 15:20:21.512 | INFO | "huge_shared" : " 27 GB", | 20260505 15:20:21.512 | INFO | "malloc" : " 14 GB", | 20260505 15:20:21.512 | INFO | "pinned" : " 29 GB" | 20260505 15:20:21.512 | INFO | } | 20260505 15:20:21.512 | INFO | Perf report for the last 5.14467 seconds: | 20260505 15:20:21.512 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:20:21.512 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:21.512 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:21.512 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:21.512 | INFO | ANALYSIS : 62.9%, 3.0, 97.8ms, 0 | 20260505 15:20:21.512 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:21.512 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:21.512 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:20:21.512 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:21.512 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:21.513 | INFO | Uploaded 36GiB to GPU in 2.19962s (16.3665GiB/s) | 20260505 15:20:21.513 | INFO | Downloaded 34GiB to GPU in 2.13368s (15.9349GiB/s) | 20260505 15:20:26.657 | INFO | Analyzing chunk frames = [4237312,4237824) | 20260505 15:20:26.657 | INFO | MemInfo: { | 20260505 15:20:26.657 | INFO | "gpu" : " 17 GB", | 20260505 15:20:26.657 | INFO | "huge" : "", | 20260505 15:20:26.657 | INFO | "huge_shared" : " 27 GB", | 20260505 15:20:26.658 | INFO | "malloc" : " 14 GB", | 20260505 15:20:26.658 | INFO | "pinned" : " 29 GB" | 20260505 15:20:26.658 | INFO | } | 20260505 15:20:26.658 | INFO | Perf report for the last 5.14551 seconds: | 20260505 15:20:26.658 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:20:26.658 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:26.658 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:26.658 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:26.658 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260505 15:20:26.658 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:26.658 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:26.658 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:20:26.658 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:26.658 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:26.658 | INFO | Uploaded 36GiB to GPU in 2.16819s (16.6037GiB/s) | 20260505 15:20:26.658 | INFO | Downloaded 34GiB to GPU in 2.12919s (15.9685GiB/s) | 20260505 15:20:31.800 | INFO | Analyzing chunk frames = [4237824,4238336) | 20260505 15:20:31.800 | INFO | MemInfo: { | 20260505 15:20:31.800 | INFO | "gpu" : " 17 GB", | 20260505 15:20:31.800 | INFO | "huge" : "", | 20260505 15:20:31.800 | INFO | "huge_shared" : " 27 GB", | 20260505 15:20:31.800 | INFO | "malloc" : " 14 GB", | 20260505 15:20:31.800 | INFO | "pinned" : " 29 GB" | 20260505 15:20:31.800 | INFO | } | 20260505 15:20:31.800 | INFO | Perf report for the last 5.14241 seconds: | 20260505 15:20:31.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:20:31.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:31.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:31.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:31.800 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260505 15:20:31.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:31.800 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:31.800 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 15:20:31.800 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:31.800 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:31.800 | INFO | Uploaded 36GiB to GPU in 2.13599s (16.854GiB/s) | 20260505 15:20:31.801 | INFO | Downloaded 34GiB to GPU in 2.03711s (16.6903GiB/s) | 20260505 15:20:36.922 | INFO | Analyzing chunk frames = [4238336,4238848) | 20260505 15:20:36.922 | INFO | MemInfo: { | 20260505 15:20:36.922 | INFO | "gpu" : " 17 GB", | 20260505 15:20:36.922 | INFO | "huge" : "", | 20260505 15:20:36.922 | INFO | "huge_shared" : " 27 GB", | 20260505 15:20:36.923 | INFO | "malloc" : " 14 GB", | 20260505 15:20:36.923 | INFO | "pinned" : " 29 GB" | 20260505 15:20:36.923 | INFO | } | 20260505 15:20:36.923 | INFO | Perf report for the last 5.12247 seconds: | 20260505 15:20:36.923 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:20:36.923 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:36.923 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:36.923 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:36.923 | INFO | ANALYSIS : 63.9%, 3.0, 99.5ms, 0 | 20260505 15:20:36.923 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:36.923 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:36.923 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 15:20:36.923 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:36.923 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:36.923 | INFO | Uploaded 36GiB to GPU in 2.16632s (16.618GiB/s) | 20260505 15:20:36.923 | INFO | Downloaded 34GiB to GPU in 2.17293s (15.6471GiB/s) | 20260505 15:20:42.062 | INFO | Analyzing chunk frames = [4238848,4239360) | 20260505 15:20:42.062 | INFO | MemInfo: { | 20260505 15:20:42.062 | INFO | "gpu" : " 17 GB", | 20260505 15:20:42.062 | INFO | "huge" : "", | 20260505 15:20:42.062 | INFO | "huge_shared" : " 27 GB", | 20260505 15:20:42.062 | INFO | "malloc" : " 14 GB", | 20260505 15:20:42.062 | INFO | "pinned" : " 29 GB" | 20260505 15:20:42.062 | INFO | } | 20260505 15:20:42.062 | INFO | Perf report for the last 5.13928 seconds: | 20260505 15:20:42.062 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:20:42.062 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:42.062 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:42.062 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:42.062 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 15:20:42.062 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:42.062 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:42.062 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 15:20:42.062 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:42.062 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:42.062 | INFO | Uploaded 36GiB to GPU in 2.16446s (16.6323GiB/s) | 20260505 15:20:42.062 | INFO | Downloaded 34GiB to GPU in 2.11451s (16.0794GiB/s) | 20260505 15:20:42.200 | INFO | Starting a round of Baz IO | 20260505 15:20:45.414 | INFO | Populating RTMetrics report | 20260505 15:20:45.415 | INFO | RTMetrics report populated | 20260505 15:20:45.415 | INFO | Opening tmp RTMetrics json file | 20260505 15:20:45.457 | INFO | Serializing RTMetrics to json | 20260505 15:20:45.483 | INFO | Serialization done, swapping tmp file now... | 20260505 15:20:45.487 | INFO | RTMetrics done | 20260505 15:20:45.521 | INFO | Finished a round of Baz IO: Wrote 2.218 GiB in 3.321 seconds | 20260505 15:20:45.521 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.50% / 0.00% / 77.50% | 20260505 15:20:45.521 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.458475% / 3.97364e-06% / 6.35783e-05% | 20260505 15:20:45.521 | INFO | Baz IO overall: | 20260505 15:20:45.521 | INFO | BytesWritten : 2522599437464 | 20260505 15:20:45.521 | INFO | NumEvents : 1198504281225 | 20260505 15:20:47.216 | INFO | Analyzing chunk frames = [4239360,4239872) | 20260505 15:20:47.216 | INFO | MemInfo: { | 20260505 15:20:47.216 | INFO | "gpu" : " 17 GB", | 20260505 15:20:47.216 | INFO | "huge" : "", | 20260505 15:20:47.216 | INFO | "huge_shared" : " 27 GB", | 20260505 15:20:47.216 | INFO | "malloc" : " 14 GB", | 20260505 15:20:47.216 | INFO | "pinned" : " 29 GB" | 20260505 15:20:47.216 | INFO | } | 20260505 15:20:47.216 | INFO | Perf report for the last 5.15388 seconds: | 20260505 15:20:47.216 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:20:47.216 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:47.216 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:47.216 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:47.216 | INFO | ANALYSIS : 64.0%, 3.0, 99.7ms, 0 | 20260505 15:20:47.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:47.216 | INFO | RT_METRICS : 46.1%, 1.0, 24.3ms, 0 | 20260505 15:20:47.216 | INFO | PRE_HQ : 60.5%, 1.8, 56.9ms, 0 | 20260505 15:20:47.216 | INFO | BAZWRITER : 63.8%, 3.9, 129.3ms, 0 | 20260505 15:20:47.216 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 15:20:47.216 | INFO | Uploaded 36GiB to GPU in 2.19905s (16.3707GiB/s) | 20260505 15:20:47.216 | INFO | Downloaded 38GiB to GPU in 2.26298s (16.792GiB/s) | 20260505 15:20:52.355 | INFO | Analyzing chunk frames = [4239872,4240384) | 20260505 15:20:52.355 | INFO | MemInfo: { | 20260505 15:20:52.355 | INFO | "gpu" : " 17 GB", | 20260505 15:20:52.355 | INFO | "huge" : "", | 20260505 15:20:52.355 | INFO | "huge_shared" : " 27 GB", | 20260505 15:20:52.355 | INFO | "malloc" : " 14 GB", | 20260505 15:20:52.356 | INFO | "pinned" : " 29 GB" | 20260505 15:20:52.356 | INFO | } | 20260505 15:20:52.356 | INFO | Perf report for the last 5.13985 seconds: | 20260505 15:20:52.356 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:20:52.356 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:52.356 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:52.356 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:52.356 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 15:20:52.356 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:52.356 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:52.356 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:20:52.356 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:52.356 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:52.356 | INFO | Uploaded 36GiB to GPU in 2.1245s (16.9452GiB/s) | 20260505 15:20:52.356 | INFO | Downloaded 34GiB to GPU in 2.02897s (16.7573GiB/s) | 20260505 15:20:57.490 | INFO | Analyzing chunk frames = [4240384,4240896) | 20260505 15:20:57.490 | INFO | MemInfo: { | 20260505 15:20:57.490 | INFO | "gpu" : " 17 GB", | 20260505 15:20:57.490 | INFO | "huge" : "", | 20260505 15:20:57.490 | INFO | "huge_shared" : " 27 GB", | 20260505 15:20:57.490 | INFO | "malloc" : " 14 GB", | 20260505 15:20:57.490 | INFO | "pinned" : " 29 GB" | 20260505 15:20:57.490 | INFO | } | 20260505 15:20:57.490 | INFO | Perf report for the last 5.13411 seconds: | 20260505 15:20:57.490 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:20:57.490 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:57.490 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:57.490 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:57.490 | INFO | ANALYSIS : 62.9%, 3.0, 97.8ms, 0 | 20260505 15:20:57.490 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:57.490 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:57.490 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 15:20:57.490 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:57.490 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:20:57.490 | INFO | Uploaded 36GiB to GPU in 2.20313s (16.3404GiB/s) | 20260505 15:20:57.490 | INFO | Downloaded 34GiB to GPU in 2.0429s (16.643GiB/s) | 20260505 15:21:02.622 | INFO | Analyzing chunk frames = [4240896,4241408) | 20260505 15:21:02.622 | INFO | MemInfo: { | 20260505 15:21:02.622 | INFO | "gpu" : " 17 GB", | 20260505 15:21:02.622 | INFO | "huge" : "", | 20260505 15:21:02.622 | INFO | "huge_shared" : " 27 GB", | 20260505 15:21:02.622 | INFO | "malloc" : " 14 GB", | 20260505 15:21:02.622 | INFO | "pinned" : " 29 GB" | 20260505 15:21:02.622 | INFO | } | 20260505 15:21:02.622 | INFO | Perf report for the last 5.13193 seconds: | 20260505 15:21:02.622 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:21:02.622 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:02.622 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:02.622 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:02.622 | INFO | ANALYSIS : 62.8%, 3.0, 97.5ms, 0 | 20260505 15:21:02.622 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:02.622 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:02.622 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 15:21:02.622 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:02.622 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:02.622 | INFO | Uploaded 36GiB to GPU in 2.18596s (16.4687GiB/s) | 20260505 15:21:02.622 | INFO | Downloaded 34GiB to GPU in 2.13159s (15.9505GiB/s) | 20260505 15:21:07.766 | INFO | Analyzing chunk frames = [4241408,4241920) | 20260505 15:21:07.766 | INFO | MemInfo: { | 20260505 15:21:07.766 | INFO | "gpu" : " 17 GB", | 20260505 15:21:07.766 | INFO | "huge" : "", | 20260505 15:21:07.766 | INFO | "huge_shared" : " 27 GB", | 20260505 15:21:07.766 | INFO | "malloc" : " 14 GB", | 20260505 15:21:07.766 | INFO | "pinned" : " 29 GB" | 20260505 15:21:07.766 | INFO | } | 20260505 15:21:07.766 | INFO | Perf report for the last 5.14417 seconds: | 20260505 15:21:07.766 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:21:07.766 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:07.766 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:07.766 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:07.766 | INFO | ANALYSIS : 62.6%, 3.0, 97.9ms, 0 | 20260505 15:21:07.766 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:07.766 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:07.766 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 15:21:07.766 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:07.766 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:07.766 | INFO | Uploaded 36GiB to GPU in 2.16325s (16.6416GiB/s) | 20260505 15:21:07.766 | INFO | Downloaded 34GiB to GPU in 2.12803s (15.9772GiB/s) | 20260505 15:21:12.904 | INFO | Analyzing chunk frames = [4241920,4242432) | 20260505 15:21:12.904 | INFO | MemInfo: { | 20260505 15:21:12.904 | INFO | "gpu" : " 17 GB", | 20260505 15:21:12.904 | INFO | "huge" : "", | 20260505 15:21:12.905 | INFO | "huge_shared" : " 27 GB", | 20260505 15:21:12.905 | INFO | "malloc" : " 14 GB", | 20260505 15:21:12.905 | INFO | "pinned" : " 29 GB" | 20260505 15:21:12.905 | INFO | } | 20260505 15:21:12.905 | INFO | Perf report for the last 5.13865 seconds: | 20260505 15:21:12.905 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:21:12.905 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:12.905 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:12.905 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:12.905 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 15:21:12.905 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:12.905 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:12.905 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:21:12.905 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:12.905 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:12.905 | INFO | Uploaded 36GiB to GPU in 2.16681s (16.6143GiB/s) | 20260505 15:21:12.905 | INFO | Downloaded 34GiB to GPU in 2.03548s (16.7037GiB/s) | 20260505 15:21:18.044 | INFO | Analyzing chunk frames = [4242432,4242944) | 20260505 15:21:18.044 | INFO | MemInfo: { | 20260505 15:21:18.044 | INFO | "gpu" : " 17 GB", | 20260505 15:21:18.044 | INFO | "huge" : "", | 20260505 15:21:18.044 | INFO | "huge_shared" : " 27 GB", | 20260505 15:21:18.044 | INFO | "malloc" : " 14 GB", | 20260505 15:21:18.044 | INFO | "pinned" : " 29 GB" | 20260505 15:21:18.044 | INFO | } | 20260505 15:21:18.044 | INFO | Perf report for the last 5.13951 seconds: | 20260505 15:21:18.044 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:21:18.044 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:18.044 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:18.044 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:18.044 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 15:21:18.044 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:18.044 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:18.044 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 15:21:18.044 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:18.044 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:18.044 | INFO | Uploaded 36GiB to GPU in 2.12425s (16.9472GiB/s) | 20260505 15:21:18.044 | INFO | Downloaded 34GiB to GPU in 2.1718s (15.6552GiB/s) | 20260505 15:21:23.180 | INFO | Analyzing chunk frames = [4242944,4243456) | 20260505 15:21:23.180 | INFO | MemInfo: { | 20260505 15:21:23.180 | INFO | "gpu" : " 17 GB", | 20260505 15:21:23.180 | INFO | "huge" : "", | 20260505 15:21:23.180 | INFO | "huge_shared" : " 27 GB", | 20260505 15:21:23.180 | INFO | "malloc" : " 14 GB", | 20260505 15:21:23.180 | INFO | "pinned" : " 29 GB" | 20260505 15:21:23.180 | INFO | } | 20260505 15:21:23.180 | INFO | Perf report for the last 5.13564 seconds: | 20260505 15:21:23.180 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:21:23.180 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:23.180 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:23.180 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:23.180 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 15:21:23.180 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:23.180 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:23.180 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 15:21:23.180 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:23.180 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:23.180 | INFO | Uploaded 36GiB to GPU in 2.22579s (16.1741GiB/s) | 20260505 15:21:23.180 | INFO | Downloaded 34GiB to GPU in 2.10357s (16.163GiB/s) | 20260505 15:21:23.180 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 15:21:23.180 | INFO | Total mem usage: 15.3 GiB | 20260505 15:21:23.180 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 15:21:23.180 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -7.12%) | 20260505 15:21:23.180 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 15:21:23.180 | INFO | High Water Breakdown: | 20260505 15:21:23.180 | INFO | 61.1% : 8.75 GiB ( 91.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 15:21:23.180 | INFO | 38.7% : 5.54 GiB ( 83.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 15:21:23.180 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 15:21:23.180 | INFO | ------------------------------------------ | 20260505 15:21:23.180 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:21:23.180 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 15:21:23.180 | INFO | Total mem usage: 26.5 GiB | 20260505 15:21:23.180 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 15:21:23.180 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 15:21:23.180 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 15:21:23.180 | INFO | High Water Breakdown: | 20260505 15:21:23.181 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:21:23.181 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:21:23.181 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:21:23.181 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:21:23.181 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:21:23.181 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:21:23.181 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 15:21:23.181 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 15:21:23.181 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:21:23.181 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 15:21:23.181 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 15:21:23.181 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 15:21:23.181 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 15:21:23.181 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 15:21:23.181 | INFO | ------------------------------------------ | 20260505 15:21:23.181 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:21:23.181 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 15:21:23.181 | INFO | Total mem usage: 25.6 GiB | 20260505 15:21:23.181 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 15:21:23.181 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 15:21:23.181 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 15:21:23.181 | INFO | High Water Breakdown: | 20260505 15:21:23.181 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 15:21:23.181 | INFO | ------------------------------------------ | 20260505 15:21:23.181 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:21:23.181 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 15:21:23.181 | INFO | Total mem usage: 15.7 GiB | 20260505 15:21:23.181 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 15:21:23.181 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 15:21:23.181 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260505 15:21:23.181 | INFO | High Water Breakdown: | 20260505 15:21:23.181 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 15:21:23.181 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 15:21:23.181 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 15:21:23.181 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 15:21:23.181 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 15:21:23.181 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 15:21:23.181 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 15:21:23.181 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 15:21:23.181 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 15:21:23.181 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:21:23.181 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 15:21:23.181 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:21:23.181 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 15:21:23.181 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 15:21:23.181 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 15:21:23.181 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 15:21:23.181 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:21:23.181 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 15:21:23.181 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:21:23.181 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:21:23.181 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:21:23.181 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 15:21:23.181 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 15:21:23.181 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:21:23.181 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 15:21:23.181 | INFO | ------------------------------------------ | 20260505 15:21:23.181 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:21:23.181 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 15:21:23.181 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:21:23.181 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:21:23.181 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.60e+01, 1.37e+01 / 1.97e+00 | 20260505 15:21:23.181 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 1.00e-01, 1.06e-02 / 3.54e-03 | 20260505 15:21:23.181 | INFO | PerfReport: 11.77%, Baseline -- 3.74e+00 / 4.48e+00, 5.83e-01 / 1.80e-01 | 20260505 15:21:23.181 | INFO | PerfReport: 3.29%, Histogram -- 1.05e+00 / 1.48e+00, 1.96e-01 / 1.13e-01 | 20260505 15:21:23.181 | INFO | PerfReport: 15.94%, DME -- 4.05e+01 / 4.53e+01, 4.56e+00 / 1.96e+00 | 20260505 15:21:23.181 | INFO | PerfReport: 56.66%, FrameLabeling -- 1.80e+01 / 1.98e+01, 1.48e+00 / 1.59e+00 | 20260505 15:21:23.181 | INFO | PerfReport: 7.36%, PulseAccumulating -- 2.34e+00 / 2.95e+00, 4.45e-01 / 1.60e-01 | 20260505 15:21:23.181 | INFO | PerfReport: 3.46%, Metrics -- 1.10e+00 / 1.40e+00, 3.05e-01 / 1.46e-01 | 20260505 15:21:23.181 | INFO | PerfReport: 1.22%, PulseSerializing -- 3.87e-01 / 4.36e-01, 6.88e-02 / 2.85e-02 | 20260505 15:21:23.181 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 15:21:23.181 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:21:23.181 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:21:23.181 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.56e+01 / 0.00e+00 | 20260505 15:21:23.181 | INFO | PerfReport: 22.54%, Upload -- 2.27e+01 / 0.00e+00, 8.02e+00 / 0.00e+00 | 20260505 15:21:23.181 | INFO | PerfReport: 21.93%, Download -- 2.20e+01 / 0.00e+00, 8.02e+00 / 0.00e+00 | 20260505 15:21:23.181 | INFO | PerfReport: 31.61%, Compute -- 3.18e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 15:21:23.182 | INFO | PerfReport: 2.22%, LockUpload -- 2.23e+00 / 0.00e+00, 7.89e+00 / 0.00e+00 | 20260505 15:21:23.182 | INFO | PerfReport: 1.41%, LockDownload -- 1.42e+00 / 0.00e+00, 2.95e+00 / 0.00e+00 | 20260505 15:21:23.182 | INFO | PerfReport: 20.27%, LockCompute -- 2.04e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 15:21:23.182 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.09e-02 / 0.00e+00 | 20260505 15:21:23.316 | INFO | Starting a round of Baz IO | 20260505 15:21:26.532 | INFO | Populating RTMetrics report | 20260505 15:21:26.532 | INFO | RTMetrics report populated | 20260505 15:21:26.532 | INFO | Opening tmp RTMetrics json file | 20260505 15:21:26.546 | INFO | Serializing RTMetrics to json | 20260505 15:21:26.548 | INFO | Serialization done, swapping tmp file now... | 20260505 15:21:26.552 | INFO | RTMetrics done | 20260505 15:21:26.630 | INFO | Finished a round of Baz IO: Wrote 2.217 GiB in 3.313 seconds | 20260505 15:21:26.630 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.50% / 0.00% / 77.50% | 20260505 15:21:26.630 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.457132% / 7.94729e-06% / 7.54992e-05% | 20260505 15:21:26.630 | INFO | Baz IO overall: | 20260505 15:21:26.630 | INFO | BytesWritten : 2524980325989 | 20260505 15:21:26.630 | INFO | NumEvents : 1199623918347 | 20260505 15:21:28.319 | INFO | Analyzing chunk frames = [4243456,4243968) | 20260505 15:21:28.319 | INFO | MemInfo: { | 20260505 15:21:28.319 | INFO | "gpu" : " 17 GB", | 20260505 15:21:28.319 | INFO | "huge" : "", | 20260505 15:21:28.319 | INFO | "huge_shared" : " 27 GB", | 20260505 15:21:28.319 | INFO | "malloc" : " 14 GB", | 20260505 15:21:28.319 | INFO | "pinned" : " 29 GB" | 20260505 15:21:28.319 | INFO | } | 20260505 15:21:28.319 | INFO | Perf report for the last 5.13901 seconds: | 20260505 15:21:28.319 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:21:28.319 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:28.319 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:28.319 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:28.319 | INFO | ANALYSIS : 64.1%, 3.0, 99.6ms, 0 | 20260505 15:21:28.319 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:28.319 | INFO | RT_METRICS : 45.5%, 1.0, 23.8ms, 0 | 20260505 15:21:28.319 | INFO | PRE_HQ : 60.8%, 1.8, 57.5ms, 0 | 20260505 15:21:28.319 | INFO | BAZWRITER : 63.8%, 3.9, 129.0ms, 0 | 20260505 15:21:28.319 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 15:21:28.319 | INFO | Uploaded 36GiB to GPU in 2.19425s (16.4065GiB/s) | 20260505 15:21:28.319 | INFO | Downloaded 38GiB to GPU in 2.26758s (16.758GiB/s) | 20260505 15:21:33.467 | INFO | Analyzing chunk frames = [4243968,4244480) | 20260505 15:21:33.467 | INFO | MemInfo: { | 20260505 15:21:33.467 | INFO | "gpu" : " 17 GB", | 20260505 15:21:33.467 | INFO | "huge" : "", | 20260505 15:21:33.467 | INFO | "huge_shared" : " 27 GB", | 20260505 15:21:33.467 | INFO | "malloc" : " 14 GB", | 20260505 15:21:33.467 | INFO | "pinned" : " 29 GB" | 20260505 15:21:33.467 | INFO | } | 20260505 15:21:33.467 | INFO | Perf report for the last 5.14821 seconds: | 20260505 15:21:33.467 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:21:33.467 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:33.467 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:33.467 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:33.467 | INFO | ANALYSIS : 63.3%, 3.0, 99.0ms, 0 | 20260505 15:21:33.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:33.467 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:33.467 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 15:21:33.467 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:33.467 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:33.467 | INFO | Uploaded 36GiB to GPU in 2.19608s (16.3929GiB/s) | 20260505 15:21:33.467 | INFO | Downloaded 34GiB to GPU in 2.02852s (16.761GiB/s) | 20260505 15:21:38.606 | INFO | Analyzing chunk frames = [4244480,4244992) | 20260505 15:21:38.606 | INFO | MemInfo: { | 20260505 15:21:38.606 | INFO | "gpu" : " 17 GB", | 20260505 15:21:38.606 | INFO | "huge" : "", | 20260505 15:21:38.606 | INFO | "huge_shared" : " 27 GB", | 20260505 15:21:38.606 | INFO | "malloc" : " 14 GB", | 20260505 15:21:38.606 | INFO | "pinned" : " 29 GB" | 20260505 15:21:38.606 | INFO | } | 20260505 15:21:38.606 | INFO | Perf report for the last 5.13859 seconds: | 20260505 15:21:38.606 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:21:38.606 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:38.606 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:38.606 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:38.606 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260505 15:21:38.606 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:38.606 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:38.606 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 15:21:38.606 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:38.606 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:38.606 | INFO | Uploaded 36GiB to GPU in 2.19013s (16.4374GiB/s) | 20260505 15:21:38.606 | INFO | Downloaded 34GiB to GPU in 2.02396s (16.7987GiB/s) | 20260505 15:21:43.746 | INFO | Analyzing chunk frames = [4244992,4245504) | 20260505 15:21:43.746 | INFO | MemInfo: { | 20260505 15:21:43.746 | INFO | "gpu" : " 17 GB", | 20260505 15:21:43.746 | INFO | "huge" : "", | 20260505 15:21:43.746 | INFO | "huge_shared" : " 27 GB", | 20260505 15:21:43.746 | INFO | "malloc" : " 14 GB", | 20260505 15:21:43.746 | INFO | "pinned" : " 29 GB" | 20260505 15:21:43.746 | INFO | } | 20260505 15:21:43.746 | INFO | Perf report for the last 5.14054 seconds: | 20260505 15:21:43.746 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:21:43.746 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:43.746 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:43.746 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:43.746 | INFO | ANALYSIS : 62.5%, 3.0, 97.1ms, 0 | 20260505 15:21:43.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:43.746 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:43.746 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:21:43.746 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:43.746 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:43.747 | INFO | Uploaded 36GiB to GPU in 2.186s (16.4684GiB/s) | 20260505 15:21:43.747 | INFO | Downloaded 34GiB to GPU in 2.13095s (15.9554GiB/s) | 20260505 15:21:48.875 | INFO | Analyzing chunk frames = [4245504,4246016) | 20260505 15:21:48.875 | INFO | MemInfo: { | 20260505 15:21:48.875 | INFO | "gpu" : " 17 GB", | 20260505 15:21:48.875 | INFO | "huge" : "", | 20260505 15:21:48.875 | INFO | "huge_shared" : " 27 GB", | 20260505 15:21:48.875 | INFO | "malloc" : " 14 GB", | 20260505 15:21:48.875 | INFO | "pinned" : " 29 GB" | 20260505 15:21:48.875 | INFO | } | 20260505 15:21:48.875 | INFO | Perf report for the last 5.12847 seconds: | 20260505 15:21:48.875 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:21:48.875 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:48.875 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:48.875 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:48.875 | INFO | ANALYSIS : 62.8%, 3.0, 97.8ms, 0 | 20260505 15:21:48.875 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:48.875 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:48.875 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 15:21:48.875 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:48.875 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:48.875 | INFO | Uploaded 36GiB to GPU in 2.16714s (16.6117GiB/s) | 20260505 15:21:48.875 | INFO | Downloaded 34GiB to GPU in 2.13136s (15.9523GiB/s) | 20260505 15:21:54.020 | INFO | Analyzing chunk frames = [4246016,4246528) | 20260505 15:21:54.020 | INFO | MemInfo: { | 20260505 15:21:54.020 | INFO | "gpu" : " 17 GB", | 20260505 15:21:54.021 | INFO | "huge" : "", | 20260505 15:21:54.021 | INFO | "huge_shared" : " 27 GB", | 20260505 15:21:54.021 | INFO | "malloc" : " 14 GB", | 20260505 15:21:54.021 | INFO | "pinned" : " 29 GB" | 20260505 15:21:54.021 | INFO | } | 20260505 15:21:54.021 | INFO | Perf report for the last 5.14605 seconds: | 20260505 15:21:54.021 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:21:54.021 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:54.021 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:54.021 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:54.021 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 15:21:54.021 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:54.021 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:54.021 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:21:54.021 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:54.021 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:54.021 | INFO | Uploaded 36GiB to GPU in 2.17475s (16.5537GiB/s) | 20260505 15:21:54.021 | INFO | Downloaded 34GiB to GPU in 2.11099s (16.1062GiB/s) | 20260505 15:21:59.164 | INFO | Analyzing chunk frames = [4246528,4247040) | 20260505 15:21:59.164 | INFO | MemInfo: { | 20260505 15:21:59.164 | INFO | "gpu" : " 17 GB", | 20260505 15:21:59.164 | INFO | "huge" : "", | 20260505 15:21:59.164 | INFO | "huge_shared" : " 27 GB", | 20260505 15:21:59.164 | INFO | "malloc" : " 14 GB", | 20260505 15:21:59.164 | INFO | "pinned" : " 29 GB" | 20260505 15:21:59.164 | INFO | } | 20260505 15:21:59.164 | INFO | Perf report for the last 5.14365 seconds: | 20260505 15:21:59.164 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:21:59.164 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:59.164 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:59.164 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:59.164 | INFO | ANALYSIS : 63.2%, 3.0, 98.8ms, 0 | 20260505 15:21:59.164 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:59.164 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:59.164 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 15:21:59.164 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:59.164 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:21:59.165 | INFO | Uploaded 36GiB to GPU in 2.10599s (17.0941GiB/s) | 20260505 15:21:59.165 | INFO | Downloaded 34GiB to GPU in 2.1682s (15.6812GiB/s) | 20260505 15:22:04.305 | INFO | Analyzing chunk frames = [4247040,4247552) | 20260505 15:22:04.305 | INFO | MemInfo: { | 20260505 15:22:04.305 | INFO | "gpu" : " 17 GB", | 20260505 15:22:04.305 | INFO | "huge" : "", | 20260505 15:22:04.305 | INFO | "huge_shared" : " 27 GB", | 20260505 15:22:04.305 | INFO | "malloc" : " 14 GB", | 20260505 15:22:04.305 | INFO | "pinned" : " 29 GB" | 20260505 15:22:04.305 | INFO | } | 20260505 15:22:04.305 | INFO | Perf report for the last 5.14075 seconds: | 20260505 15:22:04.305 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:22:04.305 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:04.305 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:04.305 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:04.305 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 15:22:04.305 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:04.305 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:04.305 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 15:22:04.305 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:04.305 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:04.305 | INFO | Uploaded 36GiB to GPU in 2.15014s (16.7431GiB/s) | 20260505 15:22:04.306 | INFO | Downloaded 34GiB to GPU in 2.03383s (16.7173GiB/s) | 20260505 15:22:04.435 | INFO | Starting a round of Baz IO | 20260505 15:22:07.650 | INFO | Populating RTMetrics report | 20260505 15:22:07.651 | INFO | RTMetrics report populated | 20260505 15:22:07.651 | INFO | Opening tmp RTMetrics json file | 20260505 15:22:07.668 | INFO | Serializing RTMetrics to json | 20260505 15:22:07.675 | INFO | Serialization done, swapping tmp file now... | 20260505 15:22:07.692 | INFO | RTMetrics done | 20260505 15:22:07.762 | INFO | Finished a round of Baz IO: Wrote 2.218 GiB in 3.327 seconds | 20260505 15:22:07.762 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.49% / 0.00% / 77.50% | 20260505 15:22:07.762 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.457116% / 1.19209e-05% / 4.76837e-05% | 20260505 15:22:07.762 | INFO | Baz IO overall: | 20260505 15:22:07.762 | INFO | BytesWritten : 2527362320779 | 20260505 15:22:07.762 | INFO | NumEvents : 1200744652182 | 20260505 15:22:09.436 | INFO | Analyzing chunk frames = [4247552,4248064) | 20260505 15:22:09.436 | INFO | MemInfo: { | 20260505 15:22:09.436 | INFO | "gpu" : " 17 GB", | 20260505 15:22:09.436 | INFO | "huge" : "", | 20260505 15:22:09.436 | INFO | "huge_shared" : " 27 GB", | 20260505 15:22:09.436 | INFO | "malloc" : " 14 GB", | 20260505 15:22:09.436 | INFO | "pinned" : " 29 GB" | 20260505 15:22:09.436 | INFO | } | 20260505 15:22:09.436 | INFO | Perf report for the last 5.131 seconds: | 20260505 15:22:09.436 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:22:09.436 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:09.436 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:09.436 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:09.436 | INFO | ANALYSIS : 64.1%, 3.0, 99.7ms, 0 | 20260505 15:22:09.436 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:09.436 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260505 15:22:09.436 | INFO | PRE_HQ : 60.9%, 1.8, 56.9ms, 0 | 20260505 15:22:09.436 | INFO | BAZWRITER : 64.1%, 3.9, 128.3ms, 0 | 20260505 15:22:09.436 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 15:22:09.436 | INFO | Uploaded 36GiB to GPU in 2.18179s (16.5002GiB/s) | 20260505 15:22:09.437 | INFO | Downloaded 38GiB to GPU in 2.25579s (16.8456GiB/s) | 20260505 15:22:14.573 | INFO | Analyzing chunk frames = [4248064,4248576) | 20260505 15:22:14.573 | INFO | MemInfo: { | 20260505 15:22:14.573 | INFO | "gpu" : " 17 GB", | 20260505 15:22:14.573 | INFO | "huge" : "", | 20260505 15:22:14.573 | INFO | "huge_shared" : " 27 GB", | 20260505 15:22:14.573 | INFO | "malloc" : " 14 GB", | 20260505 15:22:14.573 | INFO | "pinned" : " 29 GB" | 20260505 15:22:14.573 | INFO | } | 20260505 15:22:14.574 | INFO | Perf report for the last 5.13741 seconds: | 20260505 15:22:14.574 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:22:14.574 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:14.574 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:14.574 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:14.574 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 15:22:14.574 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:14.574 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:14.574 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 15:22:14.574 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:14.574 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:14.574 | INFO | Uploaded 36GiB to GPU in 2.20435s (16.3314GiB/s) | 20260505 15:22:14.574 | INFO | Downloaded 34GiB to GPU in 2.05526s (16.5429GiB/s) | 20260505 15:22:19.722 | INFO | Analyzing chunk frames = [4248576,4249088) | 20260505 15:22:19.722 | INFO | MemInfo: { | 20260505 15:22:19.722 | INFO | "gpu" : " 17 GB", | 20260505 15:22:19.722 | INFO | "huge" : "", | 20260505 15:22:19.722 | INFO | "huge_shared" : " 27 GB", | 20260505 15:22:19.722 | INFO | "malloc" : " 14 GB", | 20260505 15:22:19.722 | INFO | "pinned" : " 29 GB" | 20260505 15:22:19.722 | INFO | } | 20260505 15:22:19.723 | INFO | Perf report for the last 5.14892 seconds: | 20260505 15:22:19.723 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:22:19.723 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:19.723 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:19.723 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:19.723 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260505 15:22:19.723 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:19.723 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:19.723 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 15:22:19.723 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:19.723 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:19.723 | INFO | Uploaded 36GiB to GPU in 2.19481s (16.4023GiB/s) | 20260505 15:22:19.723 | INFO | Downloaded 34GiB to GPU in 2.05983s (16.5062GiB/s) | 20260505 15:22:24.863 | INFO | Analyzing chunk frames = [4249088,4249600) | 20260505 15:22:24.863 | INFO | MemInfo: { | 20260505 15:22:24.863 | INFO | "gpu" : " 17 GB", | 20260505 15:22:24.863 | INFO | "huge" : "", | 20260505 15:22:24.863 | INFO | "huge_shared" : " 27 GB", | 20260505 15:22:24.863 | INFO | "malloc" : " 14 GB", | 20260505 15:22:24.863 | INFO | "pinned" : " 29 GB" | 20260505 15:22:24.863 | INFO | } | 20260505 15:22:24.863 | INFO | Perf report for the last 5.14054 seconds: | 20260505 15:22:24.863 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:22:24.863 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:24.863 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:24.863 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:24.863 | INFO | ANALYSIS : 62.5%, 3.0, 97.1ms, 0 | 20260505 15:22:24.863 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:24.863 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:24.863 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 15:22:24.863 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:24.863 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:24.863 | INFO | Uploaded 36GiB to GPU in 2.18314s (16.49GiB/s) | 20260505 15:22:24.863 | INFO | Downloaded 34GiB to GPU in 2.13254s (15.9434GiB/s) | 20260505 15:22:29.994 | INFO | Analyzing chunk frames = [4249600,4250112) | 20260505 15:22:29.994 | INFO | MemInfo: { | 20260505 15:22:29.994 | INFO | "gpu" : " 17 GB", | 20260505 15:22:29.994 | INFO | "huge" : "", | 20260505 15:22:29.994 | INFO | "huge_shared" : " 27 GB", | 20260505 15:22:29.994 | INFO | "malloc" : " 14 GB", | 20260505 15:22:29.994 | INFO | "pinned" : " 29 GB" | 20260505 15:22:29.994 | INFO | } | 20260505 15:22:29.995 | INFO | Perf report for the last 5.13161 seconds: | 20260505 15:22:29.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:22:29.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:29.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:29.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:29.995 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260505 15:22:29.995 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:29.995 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:29.995 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 15:22:29.995 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:29.995 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:29.995 | INFO | Uploaded 36GiB to GPU in 2.14484s (16.7845GiB/s) | 20260505 15:22:29.995 | INFO | Downloaded 34GiB to GPU in 2.05829s (16.5186GiB/s) | 20260505 15:22:35.129 | INFO | Analyzing chunk frames = [4250112,4250624) | 20260505 15:22:35.129 | INFO | MemInfo: { | 20260505 15:22:35.129 | INFO | "gpu" : " 17 GB", | 20260505 15:22:35.129 | INFO | "huge" : "", | 20260505 15:22:35.129 | INFO | "huge_shared" : " 27 GB", | 20260505 15:22:35.129 | INFO | "malloc" : " 14 GB", | 20260505 15:22:35.129 | INFO | "pinned" : " 29 GB" | 20260505 15:22:35.129 | INFO | } | 20260505 15:22:35.129 | INFO | Perf report for the last 5.13417 seconds: | 20260505 15:22:35.129 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:22:35.129 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:35.129 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:35.129 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:35.129 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 15:22:35.129 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:35.129 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:35.129 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 15:22:35.129 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:35.129 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:35.129 | INFO | Uploaded 36GiB to GPU in 2.2126s (16.2704GiB/s) | 20260505 15:22:35.129 | INFO | Downloaded 34GiB to GPU in 2.11452s (16.0793GiB/s) | 20260505 15:22:40.274 | INFO | Analyzing chunk frames = [4250624,4251136) | 20260505 15:22:40.274 | INFO | MemInfo: { | 20260505 15:22:40.274 | INFO | "gpu" : " 17 GB", | 20260505 15:22:40.274 | INFO | "huge" : "", | 20260505 15:22:40.274 | INFO | "huge_shared" : " 27 GB", | 20260505 15:22:40.274 | INFO | "malloc" : " 14 GB", | 20260505 15:22:40.274 | INFO | "pinned" : " 29 GB" | 20260505 15:22:40.274 | INFO | } | 20260505 15:22:40.274 | INFO | Perf report for the last 5.14498 seconds: | 20260505 15:22:40.274 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:22:40.274 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:40.274 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:40.274 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:40.274 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 15:22:40.274 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:40.274 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:40.274 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 15:22:40.274 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:40.274 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:40.274 | INFO | Uploaded 36GiB to GPU in 2.12384s (16.9504GiB/s) | 20260505 15:22:40.274 | INFO | Downloaded 34GiB to GPU in 2.17899s (15.6036GiB/s) | 20260505 15:22:45.416 | INFO | Analyzing chunk frames = [4251136,4251648) | 20260505 15:22:45.416 | INFO | MemInfo: { | 20260505 15:22:45.416 | INFO | "gpu" : " 17 GB", | 20260505 15:22:45.416 | INFO | "huge" : "", | 20260505 15:22:45.416 | INFO | "huge_shared" : " 27 GB", | 20260505 15:22:45.416 | INFO | "malloc" : " 14 GB", | 20260505 15:22:45.416 | INFO | "pinned" : " 29 GB" | 20260505 15:22:45.416 | INFO | } | 20260505 15:22:45.416 | INFO | Perf report for the last 5.14251 seconds: | 20260505 15:22:45.417 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:22:45.417 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:45.417 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:45.417 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:45.417 | INFO | ANALYSIS : 63.5%, 3.0, 99.3ms, 0 | 20260505 15:22:45.417 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:45.417 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:45.417 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 15:22:45.417 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:45.417 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:45.417 | INFO | Uploaded 36GiB to GPU in 2.22852s (16.1542GiB/s) | 20260505 15:22:45.417 | INFO | Downloaded 34GiB to GPU in 2.10603s (16.1441GiB/s) | 20260505 15:22:45.417 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 15:22:45.417 | INFO | Total mem usage: 15.3 GiB | 20260505 15:22:45.417 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 15:22:45.417 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.83%) | 20260505 15:22:45.417 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 15:22:45.417 | INFO | High Water Breakdown: | 20260505 15:22:45.417 | INFO | 61% : 8.68 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 15:22:45.417 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 15:22:45.417 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 15:22:45.417 | INFO | ------------------------------------------ | 20260505 15:22:45.417 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:22:45.417 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 15:22:45.417 | INFO | Total mem usage: 26.5 GiB | 20260505 15:22:45.417 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 15:22:45.417 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 15:22:45.417 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 15:22:45.417 | INFO | High Water Breakdown: | 20260505 15:22:45.417 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:22:45.417 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:22:45.417 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:22:45.417 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:22:45.417 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:22:45.417 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:22:45.417 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 15:22:45.417 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 15:22:45.417 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:22:45.417 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 15:22:45.417 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 15:22:45.417 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 15:22:45.417 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 15:22:45.417 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 15:22:45.417 | INFO | ------------------------------------------ | 20260505 15:22:45.417 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:22:45.417 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 15:22:45.417 | INFO | Total mem usage: 25.6 GiB | 20260505 15:22:45.417 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 15:22:45.417 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 15:22:45.417 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 15:22:45.417 | INFO | High Water Breakdown: | 20260505 15:22:45.417 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 15:22:45.417 | INFO | ------------------------------------------ | 20260505 15:22:45.417 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:22:45.417 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 15:22:45.417 | INFO | Total mem usage: 15.7 GiB | 20260505 15:22:45.417 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 15:22:45.417 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 15:22:45.417 | INFO | Unused Cache: 894 MiB (Waste: 5.57%) | 20260505 15:22:45.417 | INFO | High Water Breakdown: | 20260505 15:22:45.417 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 15:22:45.417 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 15:22:45.417 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 15:22:45.418 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 15:22:45.418 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 15:22:45.418 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 15:22:45.418 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 15:22:45.418 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 15:22:45.418 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 15:22:45.418 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:22:45.418 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 15:22:45.418 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:22:45.418 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 15:22:45.418 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 15:22:45.418 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 15:22:45.418 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 15:22:45.418 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:22:45.418 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 15:22:45.418 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:22:45.418 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:22:45.418 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:22:45.418 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 15:22:45.418 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 15:22:45.418 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:22:45.418 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 15:22:45.418 | INFO | ------------------------------------------ | 20260505 15:22:45.418 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:22:45.418 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 15:22:45.418 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:22:45.418 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:22:45.418 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.57e+01, 1.37e+01 / 1.96e+00 | 20260505 15:22:45.418 | INFO | PerfReport: 0.29%, AggregateStats -- 9.31e-02 / 1.03e-01, 1.18e-02 / 7.87e-03 | 20260505 15:22:45.418 | INFO | PerfReport: 11.79%, Baseline -- 3.74e+00 / 4.45e+00, 5.78e-01 / 2.15e-01 | 20260505 15:22:45.418 | INFO | PerfReport: 3.29%, Histogram -- 1.04e+00 / 1.49e+00, 1.95e-01 / 1.17e-01 | 20260505 15:22:45.418 | INFO | PerfReport: 15.94%, DME -- 4.05e+01 / 4.50e+01, 4.43e+00 / 2.00e+00 | 20260505 15:22:45.418 | INFO | PerfReport: 56.57%, FrameLabeling -- 1.80e+01 / 1.98e+01, 1.43e+00 / 1.89e+00 | 20260505 15:22:45.418 | INFO | PerfReport: 7.41%, PulseAccumulating -- 2.35e+00 / 2.90e+00, 4.52e-01 / 1.94e-01 | 20260505 15:22:45.418 | INFO | PerfReport: 3.48%, Metrics -- 1.10e+00 / 1.45e+00, 3.06e-01 / 1.60e-01 | 20260505 15:22:45.418 | INFO | PerfReport: 1.22%, PulseSerializing -- 3.87e-01 / 4.42e-01, 6.82e-02 / 3.04e-02 | 20260505 15:22:45.418 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 15:22:45.418 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:22:45.418 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:22:45.418 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260505 15:22:45.418 | INFO | PerfReport: 22.56%, Upload -- 2.27e+01 / 0.00e+00, 7.97e+00 / 0.00e+00 | 20260505 15:22:45.418 | INFO | PerfReport: 21.93%, Download -- 2.20e+01 / 0.00e+00, 7.98e+00 / 0.00e+00 | 20260505 15:22:45.418 | INFO | PerfReport: 31.57%, Compute -- 3.17e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 15:22:45.418 | INFO | PerfReport: 2.28%, LockUpload -- 2.29e+00 / 0.00e+00, 7.94e+00 / 0.00e+00 | 20260505 15:22:45.418 | INFO | PerfReport: 1.49%, LockDownload -- 1.50e+00 / 0.00e+00, 3.07e+00 / 0.00e+00 | 20260505 15:22:45.418 | INFO | PerfReport: 20.16%, LockCompute -- 2.03e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 15:22:45.418 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.71e-02 / 0.00e+00 | 20260505 15:22:45.551 | INFO | Starting a round of Baz IO | 20260505 15:22:48.755 | INFO | Populating RTMetrics report | 20260505 15:22:48.756 | INFO | RTMetrics report populated | 20260505 15:22:48.756 | INFO | Opening tmp RTMetrics json file | 20260505 15:22:48.772 | INFO | Serializing RTMetrics to json | 20260505 15:22:48.794 | INFO | Serialization done, swapping tmp file now... | 20260505 15:22:48.815 | INFO | RTMetrics done | 20260505 15:22:48.854 | INFO | Finished a round of Baz IO: Wrote 2.217 GiB in 3.302 seconds | 20260505 15:22:48.854 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.49% / 0.00% / 77.51% | 20260505 15:22:48.854 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.455038% / 3.97364e-06% / 5.5631e-05% | 20260505 15:22:48.854 | INFO | Baz IO overall: | 20260505 15:22:48.854 | INFO | BytesWritten : 2529742944172 | 20260505 15:22:48.854 | INFO | NumEvents : 1201864361393 | 20260505 15:22:50.553 | INFO | Analyzing chunk frames = [4251648,4252160) | 20260505 15:22:50.553 | INFO | MemInfo: { | 20260505 15:22:50.553 | INFO | "gpu" : " 17 GB", | 20260505 15:22:50.553 | INFO | "huge" : "", | 20260505 15:22:50.553 | INFO | "huge_shared" : " 27 GB", | 20260505 15:22:50.553 | INFO | "malloc" : " 14 GB", | 20260505 15:22:50.553 | INFO | "pinned" : " 29 GB" | 20260505 15:22:50.553 | INFO | } | 20260505 15:22:50.553 | INFO | Perf report for the last 5.13681 seconds: | 20260505 15:22:50.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:22:50.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:50.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:50.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:50.553 | INFO | ANALYSIS : 63.9%, 3.0, 99.4ms, 0 | 20260505 15:22:50.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:50.553 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260505 15:22:50.553 | INFO | PRE_HQ : 60.9%, 1.8, 56.9ms, 0 | 20260505 15:22:50.553 | INFO | BAZWRITER : 63.6%, 3.9, 128.0ms, 0 | 20260505 15:22:50.553 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 15:22:50.554 | INFO | Uploaded 36GiB to GPU in 2.16206s (16.6508GiB/s) | 20260505 15:22:50.554 | INFO | Downloaded 38GiB to GPU in 2.26102s (16.8065GiB/s) | 20260505 15:22:55.684 | INFO | Analyzing chunk frames = [4252160,4252672) | 20260505 15:22:55.684 | INFO | MemInfo: { | 20260505 15:22:55.684 | INFO | "gpu" : " 17 GB", | 20260505 15:22:55.684 | INFO | "huge" : "", | 20260505 15:22:55.684 | INFO | "huge_shared" : " 27 GB", | 20260505 15:22:55.684 | INFO | "malloc" : " 14 GB", | 20260505 15:22:55.684 | INFO | "pinned" : " 29 GB" | 20260505 15:22:55.684 | INFO | } | 20260505 15:22:55.684 | INFO | Perf report for the last 5.13058 seconds: | 20260505 15:22:55.684 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:22:55.684 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:55.684 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:55.684 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:55.684 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 15:22:55.684 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:55.684 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:55.684 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:22:55.684 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:55.684 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:22:55.684 | INFO | Uploaded 36GiB to GPU in 2.19804s (16.3782GiB/s) | 20260505 15:22:55.684 | INFO | Downloaded 34GiB to GPU in 2.03674s (16.6933GiB/s) | 20260505 15:23:00.825 | INFO | Analyzing chunk frames = [4252672,4253184) | 20260505 15:23:00.825 | INFO | MemInfo: { | 20260505 15:23:00.825 | INFO | "gpu" : " 17 GB", | 20260505 15:23:00.825 | INFO | "huge" : "", | 20260505 15:23:00.825 | INFO | "huge_shared" : " 27 GB", | 20260505 15:23:00.825 | INFO | "malloc" : " 14 GB", | 20260505 15:23:00.825 | INFO | "pinned" : " 29 GB" | 20260505 15:23:00.825 | INFO | } | 20260505 15:23:00.825 | INFO | Perf report for the last 5.14146 seconds: | 20260505 15:23:00.825 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:23:00.825 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:00.825 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:00.825 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:00.825 | INFO | ANALYSIS : 63.2%, 3.0, 98.4ms, 0 | 20260505 15:23:00.825 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:00.825 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:00.825 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 15:23:00.825 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:00.825 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:00.825 | INFO | Uploaded 36GiB to GPU in 2.20971s (16.2918GiB/s) | 20260505 15:23:00.825 | INFO | Downloaded 34GiB to GPU in 2.13692s (15.9107GiB/s) | 20260505 15:23:05.975 | INFO | Analyzing chunk frames = [4253184,4253696) | 20260505 15:23:05.975 | INFO | MemInfo: { | 20260505 15:23:05.975 | INFO | "gpu" : " 17 GB", | 20260505 15:23:05.975 | INFO | "huge" : "", | 20260505 15:23:05.975 | INFO | "huge_shared" : " 27 GB", | 20260505 15:23:05.975 | INFO | "malloc" : " 14 GB", | 20260505 15:23:05.975 | INFO | "pinned" : " 29 GB" | 20260505 15:23:05.975 | INFO | } | 20260505 15:23:05.975 | INFO | Perf report for the last 5.14952 seconds: | 20260505 15:23:05.975 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:23:05.975 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:05.975 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:05.975 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:05.975 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260505 15:23:05.975 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:05.975 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:05.975 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:23:05.975 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:05.975 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:05.975 | INFO | Uploaded 36GiB to GPU in 2.19373s (16.4104GiB/s) | 20260505 15:23:05.975 | INFO | Downloaded 34GiB to GPU in 2.13267s (15.9425GiB/s) | 20260505 15:23:11.118 | INFO | Analyzing chunk frames = [4253696,4254208) | 20260505 15:23:11.118 | INFO | MemInfo: { | 20260505 15:23:11.118 | INFO | "gpu" : " 17 GB", | 20260505 15:23:11.118 | INFO | "huge" : "", | 20260505 15:23:11.118 | INFO | "huge_shared" : " 27 GB", | 20260505 15:23:11.118 | INFO | "malloc" : " 14 GB", | 20260505 15:23:11.118 | INFO | "pinned" : " 29 GB" | 20260505 15:23:11.118 | INFO | } | 20260505 15:23:11.118 | INFO | Perf report for the last 5.14331 seconds: | 20260505 15:23:11.118 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:23:11.118 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:11.118 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:11.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:11.118 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260505 15:23:11.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:11.118 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:11.118 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 15:23:11.118 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:11.118 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:11.118 | INFO | Uploaded 36GiB to GPU in 2.14221s (16.8051GiB/s) | 20260505 15:23:11.118 | INFO | Downloaded 34GiB to GPU in 2.04961s (16.5885GiB/s) | 20260505 15:23:16.252 | INFO | Analyzing chunk frames = [4254208,4254720) | 20260505 15:23:16.252 | INFO | MemInfo: { | 20260505 15:23:16.252 | INFO | "gpu" : " 17 GB", | 20260505 15:23:16.252 | INFO | "huge" : "", | 20260505 15:23:16.252 | INFO | "huge_shared" : " 27 GB", | 20260505 15:23:16.252 | INFO | "malloc" : " 14 GB", | 20260505 15:23:16.252 | INFO | "pinned" : " 29 GB" | 20260505 15:23:16.252 | INFO | } | 20260505 15:23:16.252 | INFO | Perf report for the last 5.13412 seconds: | 20260505 15:23:16.252 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:23:16.252 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:16.252 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:16.252 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:16.252 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 15:23:16.252 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:16.252 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:16.252 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:23:16.252 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:16.252 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:16.252 | INFO | Uploaded 36GiB to GPU in 2.14366s (16.7937GiB/s) | 20260505 15:23:16.253 | INFO | Downloaded 34GiB to GPU in 2.0386s (16.6781GiB/s) | 20260505 15:23:21.381 | INFO | Analyzing chunk frames = [4254720,4255232) | 20260505 15:23:21.381 | INFO | MemInfo: { | 20260505 15:23:21.381 | INFO | "gpu" : " 17 GB", | 20260505 15:23:21.381 | INFO | "huge" : "", | 20260505 15:23:21.381 | INFO | "huge_shared" : " 27 GB", | 20260505 15:23:21.381 | INFO | "malloc" : " 14 GB", | 20260505 15:23:21.382 | INFO | "pinned" : " 29 GB" | 20260505 15:23:21.382 | INFO | } | 20260505 15:23:21.382 | INFO | Perf report for the last 5.12938 seconds: | 20260505 15:23:21.382 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:23:21.382 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:21.382 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:21.382 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:21.382 | INFO | ANALYSIS : 63.7%, 3.0, 99.2ms, 0 | 20260505 15:23:21.382 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:21.382 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:21.382 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 15:23:21.382 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:21.382 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:21.382 | INFO | Uploaded 36GiB to GPU in 2.14436s (16.7882GiB/s) | 20260505 15:23:21.382 | INFO | Downloaded 34GiB to GPU in 2.16797s (15.6829GiB/s) | 20260505 15:23:26.530 | INFO | Analyzing chunk frames = [4255232,4255744) | 20260505 15:23:26.530 | INFO | MemInfo: { | 20260505 15:23:26.530 | INFO | "gpu" : " 17 GB", | 20260505 15:23:26.530 | INFO | "huge" : "", | 20260505 15:23:26.530 | INFO | "huge_shared" : " 27 GB", | 20260505 15:23:26.530 | INFO | "malloc" : " 14 GB", | 20260505 15:23:26.531 | INFO | "pinned" : " 29 GB" | 20260505 15:23:26.531 | INFO | } | 20260505 15:23:26.531 | INFO | Perf report for the last 5.14896 seconds: | 20260505 15:23:26.531 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:23:26.531 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:26.531 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:26.531 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:26.531 | INFO | ANALYSIS : 63.3%, 3.0, 99.1ms, 0 | 20260505 15:23:26.531 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:26.531 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:26.531 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:23:26.531 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:26.531 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:26.531 | INFO | Uploaded 36GiB to GPU in 2.16823s (16.6034GiB/s) | 20260505 15:23:26.531 | INFO | Downloaded 34GiB to GPU in 2.10666s (16.1393GiB/s) | 20260505 15:23:26.664 | INFO | Starting a round of Baz IO | 20260505 15:23:29.878 | INFO | Populating RTMetrics report | 20260505 15:23:29.879 | INFO | RTMetrics report populated | 20260505 15:23:29.879 | INFO | Opening tmp RTMetrics json file | 20260505 15:23:29.888 | INFO | Serializing RTMetrics to json | 20260505 15:23:29.889 | INFO | Serialization done, swapping tmp file now... | 20260505 15:23:29.890 | INFO | RTMetrics done | 20260505 15:23:29.975 | INFO | Finished a round of Baz IO: Wrote 2.218 GiB in 3.311 seconds | 20260505 15:23:29.975 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.49% / 0.00% / 77.51% | 20260505 15:23:29.975 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.454656% / 3.97364e-06% / 7.15256e-05% | 20260505 15:23:29.975 | INFO | Baz IO overall: | 20260505 15:23:29.975 | INFO | BytesWritten : 2532124458366 | 20260505 15:23:29.975 | INFO | NumEvents : 1202984957696 | 20260505 15:23:31.666 | INFO | Analyzing chunk frames = [4255744,4256256) | 20260505 15:23:31.666 | INFO | MemInfo: { | 20260505 15:23:31.666 | INFO | "gpu" : " 17 GB", | 20260505 15:23:31.666 | INFO | "huge" : "", | 20260505 15:23:31.666 | INFO | "huge_shared" : " 27 GB", | 20260505 15:23:31.666 | INFO | "malloc" : " 14 GB", | 20260505 15:23:31.666 | INFO | "pinned" : " 29 GB" | 20260505 15:23:31.666 | INFO | } | 20260505 15:23:31.666 | INFO | Perf report for the last 5.13571 seconds: | 20260505 15:23:31.666 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:23:31.666 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:31.666 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:31.666 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:31.666 | INFO | ANALYSIS : 64.1%, 3.0, 99.5ms, 0 | 20260505 15:23:31.666 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:31.666 | INFO | RT_METRICS : 45.1%, 1.0, 23.7ms, 0 | 20260505 15:23:31.666 | INFO | PRE_HQ : 60.9%, 1.8, 56.9ms, 0 | 20260505 15:23:31.666 | INFO | BAZWRITER : 63.8%, 4.0, 129.5ms, 0 | 20260505 15:23:31.666 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 15:23:31.667 | INFO | Uploaded 36GiB to GPU in 2.19805s (16.3782GiB/s) | 20260505 15:23:31.667 | INFO | Downloaded 38GiB to GPU in 2.27202s (16.7252GiB/s) | 20260505 15:23:36.808 | INFO | Analyzing chunk frames = [4256256,4256768) | 20260505 15:23:36.808 | INFO | MemInfo: { | 20260505 15:23:36.808 | INFO | "gpu" : " 17 GB", | 20260505 15:23:36.808 | INFO | "huge" : "", | 20260505 15:23:36.808 | INFO | "huge_shared" : " 27 GB", | 20260505 15:23:36.808 | INFO | "malloc" : " 14 GB", | 20260505 15:23:36.808 | INFO | "pinned" : " 29 GB" | 20260505 15:23:36.808 | INFO | } | 20260505 15:23:36.808 | INFO | Perf report for the last 5.14151 seconds: | 20260505 15:23:36.808 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:23:36.808 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:36.808 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:36.808 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:36.808 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 15:23:36.808 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:36.808 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:36.808 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:23:36.808 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:36.808 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:36.808 | INFO | Uploaded 36GiB to GPU in 2.19791s (16.3792GiB/s) | 20260505 15:23:36.808 | INFO | Downloaded 34GiB to GPU in 2.04052s (16.6625GiB/s) | 20260505 15:23:41.946 | INFO | Analyzing chunk frames = [4256768,4257280) | 20260505 15:23:41.946 | INFO | MemInfo: { | 20260505 15:23:41.946 | INFO | "gpu" : " 17 GB", | 20260505 15:23:41.946 | INFO | "huge" : "", | 20260505 15:23:41.946 | INFO | "huge_shared" : " 27 GB", | 20260505 15:23:41.946 | INFO | "malloc" : " 14 GB", | 20260505 15:23:41.946 | INFO | "pinned" : " 29 GB" | 20260505 15:23:41.946 | INFO | } | 20260505 15:23:41.946 | INFO | Perf report for the last 5.13842 seconds: | 20260505 15:23:41.946 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:23:41.946 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:41.946 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:41.946 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:41.946 | INFO | ANALYSIS : 63.0%, 3.0, 98.0ms, 0 | 20260505 15:23:41.946 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:41.946 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:41.946 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 15:23:41.946 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:41.946 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:41.947 | INFO | Uploaded 36GiB to GPU in 2.19289s (16.4167GiB/s) | 20260505 15:23:41.947 | INFO | Downloaded 34GiB to GPU in 2.0513s (16.5748GiB/s) | 20260505 15:23:47.081 | INFO | Analyzing chunk frames = [4257280,4257792) | 20260505 15:23:47.081 | INFO | MemInfo: { | 20260505 15:23:47.081 | INFO | "gpu" : " 17 GB", | 20260505 15:23:47.081 | INFO | "huge" : "", | 20260505 15:23:47.081 | INFO | "huge_shared" : " 27 GB", | 20260505 15:23:47.081 | INFO | "malloc" : " 14 GB", | 20260505 15:23:47.081 | INFO | "pinned" : " 29 GB" | 20260505 15:23:47.081 | INFO | } | 20260505 15:23:47.081 | INFO | Perf report for the last 5.13479 seconds: | 20260505 15:23:47.081 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:23:47.081 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:47.081 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:47.081 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:47.081 | INFO | ANALYSIS : 62.7%, 3.0, 97.4ms, 0 | 20260505 15:23:47.081 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:47.081 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:47.081 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 15:23:47.081 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:47.081 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:47.081 | INFO | Uploaded 36GiB to GPU in 2.18277s (16.4928GiB/s) | 20260505 15:23:47.081 | INFO | Downloaded 34GiB to GPU in 2.13437s (15.9298GiB/s) | 20260505 15:23:52.226 | INFO | Analyzing chunk frames = [4257792,4258304) | 20260505 15:23:52.226 | INFO | MemInfo: { | 20260505 15:23:52.226 | INFO | "gpu" : " 17 GB", | 20260505 15:23:52.226 | INFO | "huge" : "", | 20260505 15:23:52.226 | INFO | "huge_shared" : " 27 GB", | 20260505 15:23:52.226 | INFO | "malloc" : " 14 GB", | 20260505 15:23:52.226 | INFO | "pinned" : " 29 GB" | 20260505 15:23:52.226 | INFO | } | 20260505 15:23:52.226 | INFO | Perf report for the last 5.14473 seconds: | 20260505 15:23:52.226 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:23:52.226 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:52.226 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:52.226 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:52.226 | INFO | ANALYSIS : 62.7%, 3.0, 98.0ms, 0 | 20260505 15:23:52.226 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:52.226 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:52.226 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 15:23:52.226 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:52.226 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:52.226 | INFO | Uploaded 36GiB to GPU in 2.19387s (16.4094GiB/s) | 20260505 15:23:52.226 | INFO | Downloaded 34GiB to GPU in 2.12994s (15.9629GiB/s) | 20260505 15:23:57.368 | INFO | Analyzing chunk frames = [4258304,4258816) | 20260505 15:23:57.368 | INFO | MemInfo: { | 20260505 15:23:57.368 | INFO | "gpu" : " 17 GB", | 20260505 15:23:57.368 | INFO | "huge" : "", | 20260505 15:23:57.368 | INFO | "huge_shared" : " 27 GB", | 20260505 15:23:57.368 | INFO | "malloc" : " 14 GB", | 20260505 15:23:57.368 | INFO | "pinned" : " 29 GB" | 20260505 15:23:57.368 | INFO | } | 20260505 15:23:57.368 | INFO | Perf report for the last 5.1421 seconds: | 20260505 15:23:57.368 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:23:57.368 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:57.368 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:57.368 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:57.368 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 15:23:57.368 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:57.368 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:57.368 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:23:57.368 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:57.368 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:23:57.368 | INFO | Uploaded 36GiB to GPU in 2.14039s (16.8193GiB/s) | 20260505 15:23:57.368 | INFO | Downloaded 34GiB to GPU in 2.03055s (16.7442GiB/s) | 20260505 15:24:02.504 | INFO | Analyzing chunk frames = [4258816,4259328) | 20260505 15:24:02.504 | INFO | MemInfo: { | 20260505 15:24:02.504 | INFO | "gpu" : " 17 GB", | 20260505 15:24:02.504 | INFO | "huge" : "", | 20260505 15:24:02.504 | INFO | "huge_shared" : " 27 GB", | 20260505 15:24:02.504 | INFO | "malloc" : " 14 GB", | 20260505 15:24:02.504 | INFO | "pinned" : " 29 GB" | 20260505 15:24:02.504 | INFO | } | 20260505 15:24:02.504 | INFO | Perf report for the last 5.13573 seconds: | 20260505 15:24:02.504 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:24:02.504 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:02.504 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:02.504 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:02.504 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 15:24:02.504 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:02.504 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:02.504 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 15:24:02.504 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:02.504 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:02.504 | INFO | Uploaded 36GiB to GPU in 2.12695s (16.9256GiB/s) | 20260505 15:24:02.504 | INFO | Downloaded 34GiB to GPU in 2.16539s (15.7015GiB/s) | 20260505 15:24:07.633 | INFO | Analyzing chunk frames = [4259328,4259840) | 20260505 15:24:07.634 | INFO | MemInfo: { | 20260505 15:24:07.634 | INFO | "gpu" : " 17 GB", | 20260505 15:24:07.634 | INFO | "huge" : "", | 20260505 15:24:07.634 | INFO | "huge_shared" : " 27 GB", | 20260505 15:24:07.634 | INFO | "malloc" : " 14 GB", | 20260505 15:24:07.634 | INFO | "pinned" : " 29 GB" | 20260505 15:24:07.634 | INFO | } | 20260505 15:24:07.634 | INFO | Perf report for the last 5.13002 seconds: | 20260505 15:24:07.634 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:24:07.634 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:07.634 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:07.634 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:07.634 | INFO | ANALYSIS : 63.7%, 3.0, 99.2ms, 0 | 20260505 15:24:07.634 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:07.634 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:07.634 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 15:24:07.634 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:07.634 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:07.634 | INFO | Uploaded 36GiB to GPU in 2.20987s (16.2905GiB/s) | 20260505 15:24:07.634 | INFO | Downloaded 34GiB to GPU in 2.1075s (16.1329GiB/s) | 20260505 15:24:07.634 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 15:24:07.634 | INFO | Total mem usage: 15.3 GiB | 20260505 15:24:07.634 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 15:24:07.634 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.82%) | 20260505 15:24:07.634 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 15:24:07.634 | INFO | High Water Breakdown: | 20260505 15:24:07.634 | INFO | 61% : 8.68 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 15:24:07.634 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 15:24:07.634 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 15:24:07.634 | INFO | ------------------------------------------ | 20260505 15:24:07.634 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:24:07.634 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 15:24:07.634 | INFO | Total mem usage: 26.5 GiB | 20260505 15:24:07.634 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 15:24:07.634 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 15:24:07.634 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 15:24:07.634 | INFO | High Water Breakdown: | 20260505 15:24:07.634 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:24:07.634 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:24:07.634 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:24:07.634 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:24:07.634 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:24:07.634 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:24:07.634 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 15:24:07.634 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 15:24:07.634 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:24:07.634 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 15:24:07.634 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 15:24:07.634 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 15:24:07.634 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 15:24:07.634 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 15:24:07.634 | INFO | ------------------------------------------ | 20260505 15:24:07.634 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:24:07.635 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 15:24:07.635 | INFO | Total mem usage: 25.6 GiB | 20260505 15:24:07.635 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 15:24:07.635 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 15:24:07.635 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 15:24:07.635 | INFO | High Water Breakdown: | 20260505 15:24:07.635 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 15:24:07.635 | INFO | ------------------------------------------ | 20260505 15:24:07.635 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:24:07.635 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 15:24:07.635 | INFO | Total mem usage: 15.7 GiB | 20260505 15:24:07.635 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 15:24:07.635 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 15:24:07.635 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260505 15:24:07.635 | INFO | High Water Breakdown: | 20260505 15:24:07.635 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 15:24:07.635 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 15:24:07.635 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 15:24:07.635 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 15:24:07.635 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 15:24:07.635 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 15:24:07.635 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 15:24:07.635 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 15:24:07.635 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 15:24:07.635 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:24:07.635 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 15:24:07.635 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:24:07.635 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 15:24:07.635 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 15:24:07.635 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 15:24:07.635 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 15:24:07.635 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:24:07.635 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 15:24:07.635 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:24:07.635 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:24:07.635 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:24:07.635 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 15:24:07.635 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 15:24:07.635 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:24:07.635 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 15:24:07.635 | INFO | ------------------------------------------ | 20260505 15:24:07.635 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:24:07.635 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 15:24:07.635 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:24:07.635 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:24:07.635 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.59e+01, 1.35e+01 / 2.15e+00 | 20260505 15:24:07.635 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 1.02e-01, 1.04e-02 / 5.27e-03 | 20260505 15:24:07.635 | INFO | PerfReport: 11.81%, Baseline -- 3.75e+00 / 4.49e+00, 5.88e-01 / 1.67e-01 | 20260505 15:24:07.635 | INFO | PerfReport: 3.29%, Histogram -- 1.05e+00 / 1.51e+00, 1.96e-01 / 9.36e-02 | 20260505 15:24:07.635 | INFO | PerfReport: 15.94%, DME -- 4.05e+01 / 4.53e+01, 4.30e+00 / 2.31e+00 | 20260505 15:24:07.635 | INFO | PerfReport: 56.60%, FrameLabeling -- 1.80e+01 / 1.98e+01, 1.45e+00 / 2.02e+00 | 20260505 15:24:07.635 | INFO | PerfReport: 7.37%, PulseAccumulating -- 2.34e+00 / 2.90e+00, 4.41e-01 / 1.84e-01 | 20260505 15:24:07.635 | INFO | PerfReport: 3.48%, Metrics -- 1.11e+00 / 1.42e+00, 3.09e-01 / 1.62e-01 | 20260505 15:24:07.635 | INFO | PerfReport: 1.22%, PulseSerializing -- 3.87e-01 / 4.34e-01, 6.89e-02 / 2.83e-02 | 20260505 15:24:07.635 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 15:24:07.635 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:24:07.635 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:24:07.635 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260505 15:24:07.635 | INFO | PerfReport: 22.55%, Upload -- 2.27e+01 / 0.00e+00, 8.01e+00 / 0.00e+00 | 20260505 15:24:07.635 | INFO | PerfReport: 21.94%, Download -- 2.20e+01 / 0.00e+00, 8.03e+00 / 0.00e+00 | 20260505 15:24:07.635 | INFO | PerfReport: 31.61%, Compute -- 3.18e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 15:24:07.635 | INFO | PerfReport: 2.20%, LockUpload -- 2.21e+00 / 0.00e+00, 7.75e+00 / 0.00e+00 | 20260505 15:24:07.635 | INFO | PerfReport: 1.43%, LockDownload -- 1.44e+00 / 0.00e+00, 3.00e+00 / 0.00e+00 | 20260505 15:24:07.635 | INFO | PerfReport: 20.25%, LockCompute -- 2.04e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260505 15:24:07.635 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.79e-02 / 0.00e+00 | 20260505 15:24:07.775 | INFO | Starting a round of Baz IO | 20260505 15:24:10.994 | INFO | Populating RTMetrics report | 20260505 15:24:10.994 | INFO | RTMetrics report populated | 20260505 15:24:10.994 | INFO | Opening tmp RTMetrics json file | 20260505 15:24:11.005 | INFO | Serializing RTMetrics to json | 20260505 15:24:11.007 | INFO | Serialization done, swapping tmp file now... | 20260505 15:24:11.007 | INFO | RTMetrics done | 20260505 15:24:11.102 | INFO | Finished a round of Baz IO: Wrote 2.216 GiB in 3.326 seconds | 20260505 15:24:11.102 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.48% / 0.00% / 77.51% | 20260505 15:24:11.102 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.453218% / 0% / 5.96046e-05% | 20260505 15:24:11.102 | INFO | Baz IO overall: | 20260505 15:24:11.102 | INFO | BytesWritten : 2534504054794 | 20260505 15:24:11.102 | INFO | NumEvents : 1204104134500 | 20260505 15:24:12.784 | INFO | Analyzing chunk frames = [4259840,4260352) | 20260505 15:24:12.784 | INFO | MemInfo: { | 20260505 15:24:12.784 | INFO | "gpu" : " 17 GB", | 20260505 15:24:12.784 | INFO | "huge" : "", | 20260505 15:24:12.784 | INFO | "huge_shared" : " 27 GB", | 20260505 15:24:12.784 | INFO | "malloc" : " 14 GB", | 20260505 15:24:12.784 | INFO | "pinned" : " 29 GB" | 20260505 15:24:12.784 | INFO | } | 20260505 15:24:12.784 | INFO | Perf report for the last 5.15001 seconds: | 20260505 15:24:12.784 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:24:12.784 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:12.784 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:12.784 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:12.784 | INFO | ANALYSIS : 64.2%, 3.0, 100.0ms, 0 | 20260505 15:24:12.784 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:12.784 | INFO | RT_METRICS : 44.9%, 1.0, 23.6ms, 0 | 20260505 15:24:12.784 | INFO | PRE_HQ : 60.5%, 1.8, 56.9ms, 0 | 20260505 15:24:12.784 | INFO | BAZWRITER : 64.0%, 3.9, 128.5ms, 0 | 20260505 15:24:12.784 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260505 15:24:12.784 | INFO | Uploaded 36GiB to GPU in 2.18471s (16.4782GiB/s) | 20260505 15:24:12.784 | INFO | Downloaded 38GiB to GPU in 2.27295s (16.7184GiB/s) | 20260505 15:24:17.928 | INFO | Analyzing chunk frames = [4260352,4260864) | 20260505 15:24:17.928 | INFO | MemInfo: { | 20260505 15:24:17.928 | INFO | "gpu" : " 17 GB", | 20260505 15:24:17.928 | INFO | "huge" : "", | 20260505 15:24:17.928 | INFO | "huge_shared" : " 27 GB", | 20260505 15:24:17.928 | INFO | "malloc" : " 14 GB", | 20260505 15:24:17.928 | INFO | "pinned" : " 29 GB" | 20260505 15:24:17.928 | INFO | } | 20260505 15:24:17.928 | INFO | Perf report for the last 5.14432 seconds: | 20260505 15:24:17.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:24:17.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:17.928 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:17.928 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:17.928 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 15:24:17.928 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:17.928 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:17.928 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 15:24:17.928 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:17.928 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:17.928 | INFO | Uploaded 36GiB to GPU in 2.13233s (16.883GiB/s) | 20260505 15:24:17.928 | INFO | Downloaded 34GiB to GPU in 2.04184s (16.6516GiB/s) | 20260505 15:24:23.068 | INFO | Analyzing chunk frames = [4260864,4261376) | 20260505 15:24:23.068 | INFO | MemInfo: { | 20260505 15:24:23.068 | INFO | "gpu" : " 17 GB", | 20260505 15:24:23.068 | INFO | "huge" : "", | 20260505 15:24:23.068 | INFO | "huge_shared" : " 27 GB", | 20260505 15:24:23.068 | INFO | "malloc" : " 14 GB", | 20260505 15:24:23.069 | INFO | "pinned" : " 29 GB" | 20260505 15:24:23.069 | INFO | } | 20260505 15:24:23.069 | INFO | Perf report for the last 5.14047 seconds: | 20260505 15:24:23.069 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:24:23.069 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:23.069 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:23.069 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:23.069 | INFO | ANALYSIS : 62.8%, 3.0, 97.8ms, 0 | 20260505 15:24:23.069 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:23.069 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:23.069 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 15:24:23.069 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:23.069 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:23.069 | INFO | Uploaded 36GiB to GPU in 2.20281s (16.3427GiB/s) | 20260505 15:24:23.069 | INFO | Downloaded 34GiB to GPU in 2.04498s (16.6261GiB/s) | 20260505 15:24:28.195 | INFO | Analyzing chunk frames = [4261376,4261888) | 20260505 15:24:28.195 | INFO | MemInfo: { | 20260505 15:24:28.195 | INFO | "gpu" : " 17 GB", | 20260505 15:24:28.195 | INFO | "huge" : "", | 20260505 15:24:28.195 | INFO | "huge_shared" : " 27 GB", | 20260505 15:24:28.195 | INFO | "malloc" : " 14 GB", | 20260505 15:24:28.195 | INFO | "pinned" : " 29 GB" | 20260505 15:24:28.195 | INFO | } | 20260505 15:24:28.195 | INFO | Perf report for the last 5.12611 seconds: | 20260505 15:24:28.195 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:24:28.195 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:28.195 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:28.195 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:28.195 | INFO | ANALYSIS : 62.8%, 3.0, 97.3ms, 0 | 20260505 15:24:28.195 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:28.195 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:28.195 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 15:24:28.195 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:28.195 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:28.195 | INFO | Uploaded 36GiB to GPU in 2.20477s (16.3282GiB/s) | 20260505 15:24:28.195 | INFO | Downloaded 34GiB to GPU in 2.13318s (15.9386GiB/s) | 20260505 15:24:33.335 | INFO | Analyzing chunk frames = [4261888,4262400) | 20260505 15:24:33.335 | INFO | MemInfo: { | 20260505 15:24:33.335 | INFO | "gpu" : " 17 GB", | 20260505 15:24:33.335 | INFO | "huge" : "", | 20260505 15:24:33.335 | INFO | "huge_shared" : " 27 GB", | 20260505 15:24:33.335 | INFO | "malloc" : " 14 GB", | 20260505 15:24:33.335 | INFO | "pinned" : " 29 GB" | 20260505 15:24:33.335 | INFO | } | 20260505 15:24:33.335 | INFO | Perf report for the last 5.14054 seconds: | 20260505 15:24:33.335 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:24:33.335 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:33.335 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:33.335 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:33.335 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 15:24:33.335 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:33.335 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:33.335 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 15:24:33.335 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:33.335 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:33.336 | INFO | Uploaded 36GiB to GPU in 2.16676s (16.6147GiB/s) | 20260505 15:24:33.336 | INFO | Downloaded 34GiB to GPU in 2.12527s (15.9979GiB/s) | 20260505 15:24:38.484 | INFO | Analyzing chunk frames = [4262400,4262912) | 20260505 15:24:38.484 | INFO | MemInfo: { | 20260505 15:24:38.484 | INFO | "gpu" : " 17 GB", | 20260505 15:24:38.484 | INFO | "huge" : "", | 20260505 15:24:38.484 | INFO | "huge_shared" : " 27 GB", | 20260505 15:24:38.484 | INFO | "malloc" : " 14 GB", | 20260505 15:24:38.484 | INFO | "pinned" : " 29 GB" | 20260505 15:24:38.484 | INFO | } | 20260505 15:24:38.484 | INFO | Perf report for the last 5.14906 seconds: | 20260505 15:24:38.484 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:24:38.484 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:38.484 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:38.484 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:38.484 | INFO | ANALYSIS : 62.9%, 3.0, 98.5ms, 0 | 20260505 15:24:38.484 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:38.484 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:38.484 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:24:38.484 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:38.484 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:38.485 | INFO | Uploaded 36GiB to GPU in 2.19001s (16.4383GiB/s) | 20260505 15:24:38.485 | INFO | Downloaded 34GiB to GPU in 2.11303s (16.0907GiB/s) | 20260505 15:24:43.626 | INFO | Analyzing chunk frames = [4262912,4263424) | 20260505 15:24:43.626 | INFO | MemInfo: { | 20260505 15:24:43.626 | INFO | "gpu" : " 17 GB", | 20260505 15:24:43.626 | INFO | "huge" : "", | 20260505 15:24:43.626 | INFO | "huge_shared" : " 27 GB", | 20260505 15:24:43.626 | INFO | "malloc" : " 14 GB", | 20260505 15:24:43.626 | INFO | "pinned" : " 29 GB" | 20260505 15:24:43.626 | INFO | } | 20260505 15:24:43.626 | INFO | Perf report for the last 5.14164 seconds: | 20260505 15:24:43.626 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:24:43.626 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:43.626 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:43.626 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:43.626 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 15:24:43.626 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:43.626 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:43.626 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 15:24:43.626 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:43.626 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:43.626 | INFO | Uploaded 36GiB to GPU in 2.13268s (16.8802GiB/s) | 20260505 15:24:43.626 | INFO | Downloaded 34GiB to GPU in 2.16532s (15.7021GiB/s) | 20260505 15:24:48.757 | INFO | Analyzing chunk frames = [4263424,4263936) | 20260505 15:24:48.757 | INFO | MemInfo: { | 20260505 15:24:48.757 | INFO | "gpu" : " 17 GB", | 20260505 15:24:48.757 | INFO | "huge" : "", | 20260505 15:24:48.757 | INFO | "huge_shared" : " 27 GB", | 20260505 15:24:48.757 | INFO | "malloc" : " 14 GB", | 20260505 15:24:48.757 | INFO | "pinned" : " 29 GB" | 20260505 15:24:48.757 | INFO | } | 20260505 15:24:48.757 | INFO | Perf report for the last 5.13126 seconds: | 20260505 15:24:48.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:24:48.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:48.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:48.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:48.757 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 15:24:48.757 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:48.757 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:48.757 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:24:48.757 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:48.757 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:48.758 | INFO | Uploaded 36GiB to GPU in 2.17014s (16.5888GiB/s) | 20260505 15:24:48.758 | INFO | Downloaded 34GiB to GPU in 2.0344s (16.7125GiB/s) | 20260505 15:24:48.893 | INFO | Starting a round of Baz IO | 20260505 15:24:52.108 | INFO | Populating RTMetrics report | 20260505 15:24:52.108 | INFO | RTMetrics report populated | 20260505 15:24:52.108 | INFO | Opening tmp RTMetrics json file | 20260505 15:24:52.124 | INFO | Serializing RTMetrics to json | 20260505 15:24:52.142 | INFO | Serialization done, swapping tmp file now... | 20260505 15:24:52.149 | INFO | RTMetrics done | 20260505 15:24:52.205 | INFO | Finished a round of Baz IO: Wrote 2.215 GiB in 3.312 seconds | 20260505 15:24:52.205 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.48% / 0.00% / 77.52% | 20260505 15:24:52.205 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.451303% / 3.97364e-06% / 5.16574e-05% | 20260505 15:24:52.205 | INFO | Baz IO overall: | 20260505 15:24:52.205 | INFO | BytesWritten : 2536882713388 | 20260505 15:24:52.205 | INFO | NumEvents : 1205222708610 | 20260505 15:24:53.892 | INFO | Analyzing chunk frames = [4263936,4264448) | 20260505 15:24:53.892 | INFO | MemInfo: { | 20260505 15:24:53.892 | INFO | "gpu" : " 17 GB", | 20260505 15:24:53.892 | INFO | "huge" : "", | 20260505 15:24:53.892 | INFO | "huge_shared" : " 27 GB", | 20260505 15:24:53.892 | INFO | "malloc" : " 14 GB", | 20260505 15:24:53.893 | INFO | "pinned" : " 29 GB" | 20260505 15:24:53.893 | INFO | } | 20260505 15:24:53.893 | INFO | Perf report for the last 5.13533 seconds: | 20260505 15:24:53.893 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:24:53.893 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:53.893 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:53.893 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:53.893 | INFO | ANALYSIS : 64.1%, 3.0, 99.7ms, 0 | 20260505 15:24:53.893 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:53.893 | INFO | RT_METRICS : 45.7%, 1.0, 23.9ms, 0 | 20260505 15:24:53.893 | INFO | PRE_HQ : 61.0%, 1.8, 57.0ms, 0 | 20260505 15:24:53.893 | INFO | BAZWRITER : 63.8%, 3.9, 128.7ms, 0 | 20260505 15:24:53.893 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 15:24:53.893 | INFO | Uploaded 36GiB to GPU in 2.19031s (16.436GiB/s) | 20260505 15:24:53.893 | INFO | Downloaded 38GiB to GPU in 2.27121s (16.7312GiB/s) | 20260505 15:24:59.039 | INFO | Analyzing chunk frames = [4264448,4264960) | 20260505 15:24:59.039 | INFO | MemInfo: { | 20260505 15:24:59.039 | INFO | "gpu" : " 17 GB", | 20260505 15:24:59.039 | INFO | "huge" : "", | 20260505 15:24:59.039 | INFO | "huge_shared" : " 27 GB", | 20260505 15:24:59.039 | INFO | "malloc" : " 14 GB", | 20260505 15:24:59.039 | INFO | "pinned" : " 29 GB" | 20260505 15:24:59.039 | INFO | } | 20260505 15:24:59.039 | INFO | Perf report for the last 5.14615 seconds: | 20260505 15:24:59.039 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:24:59.039 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:59.039 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:59.039 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:59.039 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 15:24:59.039 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:59.039 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:59.039 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 15:24:59.039 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:59.039 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:24:59.039 | INFO | Uploaded 36GiB to GPU in 2.19892s (16.3717GiB/s) | 20260505 15:24:59.039 | INFO | Downloaded 34GiB to GPU in 2.04057s (16.662GiB/s) | 20260505 15:25:04.175 | INFO | Analyzing chunk frames = [4264960,4265472) | 20260505 15:25:04.175 | INFO | MemInfo: { | 20260505 15:25:04.175 | INFO | "gpu" : " 17 GB", | 20260505 15:25:04.175 | INFO | "huge" : "", | 20260505 15:25:04.175 | INFO | "huge_shared" : " 27 GB", | 20260505 15:25:04.175 | INFO | "malloc" : " 14 GB", | 20260505 15:25:04.175 | INFO | "pinned" : " 29 GB" | 20260505 15:25:04.175 | INFO | } | 20260505 15:25:04.175 | INFO | Perf report for the last 5.13616 seconds: | 20260505 15:25:04.175 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:25:04.175 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:04.175 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:04.175 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:04.175 | INFO | ANALYSIS : 62.9%, 3.0, 97.8ms, 0 | 20260505 15:25:04.175 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:04.175 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:04.175 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:25:04.175 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:04.175 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:04.175 | INFO | Uploaded 36GiB to GPU in 2.18331s (16.4887GiB/s) | 20260505 15:25:04.175 | INFO | Downloaded 34GiB to GPU in 2.0379s (16.6839GiB/s) | 20260505 15:25:09.312 | INFO | Analyzing chunk frames = [4265472,4265984) | 20260505 15:25:09.312 | INFO | MemInfo: { | 20260505 15:25:09.313 | INFO | "gpu" : " 17 GB", | 20260505 15:25:09.313 | INFO | "huge" : "", | 20260505 15:25:09.313 | INFO | "huge_shared" : " 27 GB", | 20260505 15:25:09.313 | INFO | "malloc" : " 14 GB", | 20260505 15:25:09.313 | INFO | "pinned" : " 29 GB" | 20260505 15:25:09.313 | INFO | } | 20260505 15:25:09.313 | INFO | Perf report for the last 5.13764 seconds: | 20260505 15:25:09.313 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:25:09.313 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:09.313 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:09.313 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:09.313 | INFO | ANALYSIS : 62.6%, 3.0, 97.2ms, 0 | 20260505 15:25:09.313 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:09.313 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:09.313 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 15:25:09.313 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:09.313 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:09.313 | INFO | Uploaded 36GiB to GPU in 2.17959s (16.5169GiB/s) | 20260505 15:25:09.313 | INFO | Downloaded 34GiB to GPU in 2.12696s (15.9852GiB/s) | 20260505 15:25:14.452 | INFO | Analyzing chunk frames = [4265984,4266496) | 20260505 15:25:14.452 | INFO | MemInfo: { | 20260505 15:25:14.452 | INFO | "gpu" : " 17 GB", | 20260505 15:25:14.452 | INFO | "huge" : "", | 20260505 15:25:14.452 | INFO | "huge_shared" : " 27 GB", | 20260505 15:25:14.452 | INFO | "malloc" : " 14 GB", | 20260505 15:25:14.452 | INFO | "pinned" : " 29 GB" | 20260505 15:25:14.452 | INFO | } | 20260505 15:25:14.452 | INFO | Perf report for the last 5.13932 seconds: | 20260505 15:25:14.452 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:25:14.452 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:14.452 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:14.452 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:14.452 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 15:25:14.452 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:14.452 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:14.452 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:25:14.452 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:14.452 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:14.452 | INFO | Uploaded 36GiB to GPU in 2.13287s (16.8787GiB/s) | 20260505 15:25:14.452 | INFO | Downloaded 34GiB to GPU in 2.12841s (15.9743GiB/s) | 20260505 15:25:19.587 | INFO | Analyzing chunk frames = [4266496,4267008) | 20260505 15:25:19.587 | INFO | MemInfo: { | 20260505 15:25:19.587 | INFO | "gpu" : " 17 GB", | 20260505 15:25:19.587 | INFO | "huge" : "", | 20260505 15:25:19.587 | INFO | "huge_shared" : " 27 GB", | 20260505 15:25:19.587 | INFO | "malloc" : " 14 GB", | 20260505 15:25:19.587 | INFO | "pinned" : " 29 GB" | 20260505 15:25:19.587 | INFO | } | 20260505 15:25:19.587 | INFO | Perf report for the last 5.13496 seconds: | 20260505 15:25:19.587 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:25:19.587 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:19.587 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:19.587 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:19.587 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 15:25:19.587 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:19.587 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:19.587 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 15:25:19.587 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:19.587 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:19.587 | INFO | Uploaded 36GiB to GPU in 2.18661s (16.4638GiB/s) | 20260505 15:25:19.587 | INFO | Downloaded 34GiB to GPU in 2.11297s (16.0911GiB/s) | 20260505 15:25:24.737 | INFO | Analyzing chunk frames = [4267008,4267520) | 20260505 15:25:24.737 | INFO | MemInfo: { | 20260505 15:25:24.737 | INFO | "gpu" : " 17 GB", | 20260505 15:25:24.738 | INFO | "huge" : "", | 20260505 15:25:24.738 | INFO | "huge_shared" : " 27 GB", | 20260505 15:25:24.738 | INFO | "malloc" : " 14 GB", | 20260505 15:25:24.738 | INFO | "pinned" : " 29 GB" | 20260505 15:25:24.738 | INFO | } | 20260505 15:25:24.738 | INFO | Perf report for the last 5.15086 seconds: | 20260505 15:25:24.738 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:25:24.738 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:24.738 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:24.738 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:24.738 | INFO | ANALYSIS : 63.1%, 3.0, 98.7ms, 0 | 20260505 15:25:24.738 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:24.738 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:24.738 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 15:25:24.738 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:24.738 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:24.738 | INFO | Uploaded 36GiB to GPU in 2.17487s (16.5527GiB/s) | 20260505 15:25:24.738 | INFO | Downloaded 34GiB to GPU in 2.15527s (15.7753GiB/s) | 20260505 15:25:29.884 | INFO | Analyzing chunk frames = [4267520,4268032) | 20260505 15:25:29.884 | INFO | MemInfo: { | 20260505 15:25:29.884 | INFO | "gpu" : " 17 GB", | 20260505 15:25:29.884 | INFO | "huge" : "", | 20260505 15:25:29.884 | INFO | "huge_shared" : " 27 GB", | 20260505 15:25:29.884 | INFO | "malloc" : " 14 GB", | 20260505 15:25:29.884 | INFO | "pinned" : " 29 GB" | 20260505 15:25:29.884 | INFO | } | 20260505 15:25:29.884 | INFO | Perf report for the last 5.14659 seconds: | 20260505 15:25:29.884 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:25:29.884 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:29.884 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:29.884 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:29.884 | INFO | ANALYSIS : 63.2%, 3.0, 98.8ms, 0 | 20260505 15:25:29.884 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:29.884 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:29.884 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 15:25:29.885 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:29.885 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:29.885 | INFO | Uploaded 36GiB to GPU in 2.18251s (16.4948GiB/s) | 20260505 15:25:29.885 | INFO | Downloaded 34GiB to GPU in 2.03062s (16.7436GiB/s) | 20260505 15:25:29.885 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 15:25:29.885 | INFO | Total mem usage: 15.3 GiB | 20260505 15:25:29.885 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 15:25:29.885 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.86%) | 20260505 15:25:29.885 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 15:25:29.885 | INFO | High Water Breakdown: | 20260505 15:25:29.885 | INFO | 61% : 8.67 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 15:25:29.885 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 15:25:29.885 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 15:25:29.885 | INFO | ------------------------------------------ | 20260505 15:25:29.885 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:25:29.885 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 15:25:29.885 | INFO | Total mem usage: 26.5 GiB | 20260505 15:25:29.885 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 15:25:29.885 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 15:25:29.885 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 15:25:29.885 | INFO | High Water Breakdown: | 20260505 15:25:29.885 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:25:29.885 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:25:29.885 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:25:29.885 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:25:29.885 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:25:29.885 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:25:29.885 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 15:25:29.885 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 15:25:29.885 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:25:29.885 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 15:25:29.885 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 15:25:29.885 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 15:25:29.885 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 15:25:29.885 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 15:25:29.885 | INFO | ------------------------------------------ | 20260505 15:25:29.885 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:25:29.885 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 15:25:29.885 | INFO | Total mem usage: 25.6 GiB | 20260505 15:25:29.885 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 15:25:29.885 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 15:25:29.885 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 15:25:29.885 | INFO | High Water Breakdown: | 20260505 15:25:29.885 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 15:25:29.885 | INFO | ------------------------------------------ | 20260505 15:25:29.885 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:25:29.885 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 15:25:29.885 | INFO | Total mem usage: 15.7 GiB | 20260505 15:25:29.885 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 15:25:29.885 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 15:25:29.885 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260505 15:25:29.885 | INFO | High Water Breakdown: | 20260505 15:25:29.885 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 15:25:29.886 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 15:25:29.886 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 15:25:29.886 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 15:25:29.886 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 15:25:29.886 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 15:25:29.886 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 15:25:29.886 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 15:25:29.886 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 15:25:29.886 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:25:29.886 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 15:25:29.886 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:25:29.886 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 15:25:29.886 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 15:25:29.886 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 15:25:29.886 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 15:25:29.886 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:25:29.886 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 15:25:29.886 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:25:29.886 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:25:29.886 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:25:29.886 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 15:25:29.886 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 15:25:29.886 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:25:29.886 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 15:25:29.886 | INFO | ------------------------------------------ | 20260505 15:25:29.886 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:25:29.886 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 15:25:29.886 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:25:29.886 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:25:29.886 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.57e+01, 1.36e+01 / 1.59e+00 | 20260505 15:25:29.886 | INFO | PerfReport: 0.29%, AggregateStats -- 9.30e-02 / 1.01e-01, 1.08e-02 / 4.99e-03 | 20260505 15:25:29.886 | INFO | PerfReport: 11.78%, Baseline -- 3.74e+00 / 4.49e+00, 5.87e-01 / 2.14e-01 | 20260505 15:25:29.886 | INFO | PerfReport: 3.30%, Histogram -- 1.05e+00 / 1.47e+00, 1.98e-01 / 1.30e-01 | 20260505 15:25:29.886 | INFO | PerfReport: 15.88%, DME -- 4.04e+01 / 4.48e+01, 4.35e+00 / 1.59e+00 | 20260505 15:25:29.886 | INFO | PerfReport: 56.69%, FrameLabeling -- 1.80e+01 / 2.01e+01, 1.52e+00 / 1.65e+00 | 20260505 15:25:29.886 | INFO | PerfReport: 7.37%, PulseAccumulating -- 2.34e+00 / 2.89e+00, 4.50e-01 / 1.83e-01 | 20260505 15:25:29.886 | INFO | PerfReport: 3.47%, Metrics -- 1.10e+00 / 1.40e+00, 3.04e-01 / 1.14e-01 | 20260505 15:25:29.886 | INFO | PerfReport: 1.22%, PulseSerializing -- 3.87e-01 / 4.40e-01, 6.88e-02 / 3.26e-02 | 20260505 15:25:29.886 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 15:25:29.886 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:25:29.886 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:25:29.886 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260505 15:25:29.886 | INFO | PerfReport: 22.56%, Upload -- 2.27e+01 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260505 15:25:29.886 | INFO | PerfReport: 21.92%, Download -- 2.20e+01 / 0.00e+00, 8.05e+00 / 0.00e+00 | 20260505 15:25:29.886 | INFO | PerfReport: 31.63%, Compute -- 3.18e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 15:25:29.886 | INFO | PerfReport: 2.19%, LockUpload -- 2.20e+00 / 0.00e+00, 7.96e+00 / 0.00e+00 | 20260505 15:25:29.886 | INFO | PerfReport: 1.46%, LockDownload -- 1.47e+00 / 0.00e+00, 3.03e+00 / 0.00e+00 | 20260505 15:25:29.886 | INFO | PerfReport: 20.23%, LockCompute -- 2.03e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 15:25:29.886 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.94e-02 / 0.00e+00 | 20260505 15:25:30.019 | INFO | Starting a round of Baz IO | 20260505 15:25:33.226 | INFO | Populating RTMetrics report | 20260505 15:25:33.226 | INFO | RTMetrics report populated | 20260505 15:25:33.227 | INFO | Opening tmp RTMetrics json file | 20260505 15:25:33.241 | INFO | Serializing RTMetrics to json | 20260505 15:25:33.242 | INFO | Serialization done, swapping tmp file now... | 20260505 15:25:33.243 | INFO | RTMetrics done | 20260505 15:25:33.323 | INFO | Finished a round of Baz IO: Wrote 2.212 GiB in 3.303 seconds | 20260505 15:25:33.323 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.48% / 0.00% / 77.52% | 20260505 15:25:33.323 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.448056% / 3.97364e-06% / 7.15256e-05% | 20260505 15:25:33.323 | INFO | Baz IO overall: | 20260505 15:25:33.323 | INFO | BytesWritten : 2539257922204 | 20260505 15:25:33.323 | INFO | NumEvents : 1206339095571 | 20260505 15:25:35.015 | INFO | Analyzing chunk frames = [4268032,4268544) | 20260505 15:25:35.015 | INFO | MemInfo: { | 20260505 15:25:35.015 | INFO | "gpu" : " 17 GB", | 20260505 15:25:35.015 | INFO | "huge" : "", | 20260505 15:25:35.015 | INFO | "huge_shared" : " 27 GB", | 20260505 15:25:35.015 | INFO | "malloc" : " 14 GB", | 20260505 15:25:35.015 | INFO | "pinned" : " 29 GB" | 20260505 15:25:35.015 | INFO | } | 20260505 15:25:35.015 | INFO | Perf report for the last 5.13106 seconds: | 20260505 15:25:35.016 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:25:35.016 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:35.016 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:35.016 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:35.016 | INFO | ANALYSIS : 64.0%, 3.0, 99.5ms, 0 | 20260505 15:25:35.016 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:35.016 | INFO | RT_METRICS : 45.1%, 1.0, 23.6ms, 0 | 20260505 15:25:35.016 | INFO | PRE_HQ : 61.1%, 1.8, 56.7ms, 0 | 20260505 15:25:35.016 | INFO | BAZWRITER : 63.7%, 3.9, 127.5ms, 0 | 20260505 15:25:35.016 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 15:25:35.016 | INFO | Uploaded 36GiB to GPU in 2.19737s (16.3832GiB/s) | 20260505 15:25:35.016 | INFO | Downloaded 38GiB to GPU in 2.26156s (16.8025GiB/s) | 20260505 15:25:40.141 | INFO | Analyzing chunk frames = [4268544,4269056) | 20260505 15:25:40.141 | INFO | MemInfo: { | 20260505 15:25:40.141 | INFO | "gpu" : " 17 GB", | 20260505 15:25:40.141 | INFO | "huge" : "", | 20260505 15:25:40.141 | INFO | "huge_shared" : " 27 GB", | 20260505 15:25:40.141 | INFO | "malloc" : " 14 GB", | 20260505 15:25:40.141 | INFO | "pinned" : " 29 GB" | 20260505 15:25:40.141 | INFO | } | 20260505 15:25:40.141 | INFO | Perf report for the last 5.12555 seconds: | 20260505 15:25:40.141 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:25:40.141 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:40.141 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:40.141 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:40.141 | INFO | ANALYSIS : 63.6%, 3.0, 99.1ms, 0 | 20260505 15:25:40.141 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:40.141 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:40.141 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 15:25:40.141 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:40.141 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:40.141 | INFO | Uploaded 36GiB to GPU in 2.20241s (16.3457GiB/s) | 20260505 15:25:40.141 | INFO | Downloaded 34GiB to GPU in 2.05157s (16.5727GiB/s) | 20260505 15:25:45.291 | INFO | Analyzing chunk frames = [4269056,4269568) | 20260505 15:25:45.291 | INFO | MemInfo: { | 20260505 15:25:45.291 | INFO | "gpu" : " 17 GB", | 20260505 15:25:45.291 | INFO | "huge" : "", | 20260505 15:25:45.291 | INFO | "huge_shared" : " 27 GB", | 20260505 15:25:45.291 | INFO | "malloc" : " 14 GB", | 20260505 15:25:45.291 | INFO | "pinned" : " 29 GB" | 20260505 15:25:45.291 | INFO | } | 20260505 15:25:45.291 | INFO | Perf report for the last 5.15017 seconds: | 20260505 15:25:45.291 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:25:45.291 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:45.291 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:45.291 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:45.291 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 15:25:45.291 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:45.291 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:45.291 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 15:25:45.291 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:45.291 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:45.292 | INFO | Uploaded 36GiB to GPU in 2.2309s (16.137GiB/s) | 20260505 15:25:45.292 | INFO | Downloaded 34GiB to GPU in 2.0899s (16.2687GiB/s) | 20260505 15:25:50.441 | INFO | Analyzing chunk frames = [4269568,4270080) | 20260505 15:25:50.441 | INFO | MemInfo: { | 20260505 15:25:50.441 | INFO | "gpu" : " 17 GB", | 20260505 15:25:50.441 | INFO | "huge" : "", | 20260505 15:25:50.441 | INFO | "huge_shared" : " 27 GB", | 20260505 15:25:50.441 | INFO | "malloc" : " 14 GB", | 20260505 15:25:50.441 | INFO | "pinned" : " 29 GB" | 20260505 15:25:50.441 | INFO | } | 20260505 15:25:50.441 | INFO | Perf report for the last 5.1497 seconds: | 20260505 15:25:50.441 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:25:50.441 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:50.441 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:50.441 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:50.441 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260505 15:25:50.441 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:50.441 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:50.441 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 15:25:50.441 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:50.441 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:50.441 | INFO | Uploaded 36GiB to GPU in 2.19049s (16.4347GiB/s) | 20260505 15:25:50.441 | INFO | Downloaded 34GiB to GPU in 2.13193s (15.948GiB/s) | 20260505 15:25:55.568 | INFO | Analyzing chunk frames = [4270080,4270592) | 20260505 15:25:55.568 | INFO | MemInfo: { | 20260505 15:25:55.568 | INFO | "gpu" : " 17 GB", | 20260505 15:25:55.568 | INFO | "huge" : "", | 20260505 15:25:55.568 | INFO | "huge_shared" : " 27 GB", | 20260505 15:25:55.568 | INFO | "malloc" : " 14 GB", | 20260505 15:25:55.568 | INFO | "pinned" : " 29 GB" | 20260505 15:25:55.568 | INFO | } | 20260505 15:25:55.568 | INFO | Perf report for the last 5.12709 seconds: | 20260505 15:25:55.568 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:25:55.568 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:55.568 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:55.568 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:55.568 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260505 15:25:55.568 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:55.568 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:55.568 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 15:25:55.568 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:55.568 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:25:55.568 | INFO | Uploaded 36GiB to GPU in 2.15823s (16.6803GiB/s) | 20260505 15:25:55.568 | INFO | Downloaded 34GiB to GPU in 2.05794s (16.5214GiB/s) | 20260505 15:26:00.706 | INFO | Analyzing chunk frames = [4270592,4271104) | 20260505 15:26:00.706 | INFO | MemInfo: { | 20260505 15:26:00.706 | INFO | "gpu" : " 17 GB", | 20260505 15:26:00.706 | INFO | "huge" : "", | 20260505 15:26:00.706 | INFO | "huge_shared" : " 27 GB", | 20260505 15:26:00.706 | INFO | "malloc" : " 14 GB", | 20260505 15:26:00.706 | INFO | "pinned" : " 29 GB" | 20260505 15:26:00.706 | INFO | } | 20260505 15:26:00.706 | INFO | Perf report for the last 5.13798 seconds: | 20260505 15:26:00.706 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:26:00.706 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:00.706 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:00.706 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:00.706 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 15:26:00.706 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:00.706 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:00.706 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 15:26:00.706 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:00.706 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:00.706 | INFO | Uploaded 36GiB to GPU in 2.21172s (16.2769GiB/s) | 20260505 15:26:00.706 | INFO | Downloaded 34GiB to GPU in 2.11252s (16.0945GiB/s) | 20260505 15:26:05.847 | INFO | Analyzing chunk frames = [4271104,4271616) | 20260505 15:26:05.847 | INFO | MemInfo: { | 20260505 15:26:05.847 | INFO | "gpu" : " 17 GB", | 20260505 15:26:05.847 | INFO | "huge" : "", | 20260505 15:26:05.847 | INFO | "huge_shared" : " 27 GB", | 20260505 15:26:05.847 | INFO | "malloc" : " 14 GB", | 20260505 15:26:05.847 | INFO | "pinned" : " 29 GB" | 20260505 15:26:05.847 | INFO | } | 20260505 15:26:05.847 | INFO | Perf report for the last 5.14119 seconds: | 20260505 15:26:05.847 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:26:05.847 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:05.847 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:05.847 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:05.847 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 15:26:05.847 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:05.847 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:05.847 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 15:26:05.847 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:05.847 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:05.847 | INFO | Uploaded 36GiB to GPU in 2.17569s (16.5465GiB/s) | 20260505 15:26:05.847 | INFO | Downloaded 34GiB to GPU in 2.15475s (15.7791GiB/s) | 20260505 15:26:10.991 | INFO | Analyzing chunk frames = [4271616,4272128) | 20260505 15:26:10.991 | INFO | MemInfo: { | 20260505 15:26:10.991 | INFO | "gpu" : " 17 GB", | 20260505 15:26:10.991 | INFO | "huge" : "", | 20260505 15:26:10.991 | INFO | "huge_shared" : " 27 GB", | 20260505 15:26:10.991 | INFO | "malloc" : " 14 GB", | 20260505 15:26:10.991 | INFO | "pinned" : " 29 GB" | 20260505 15:26:10.991 | INFO | } | 20260505 15:26:10.991 | INFO | Perf report for the last 5.14429 seconds: | 20260505 15:26:10.992 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:26:10.992 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:10.992 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:10.992 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:10.992 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 15:26:10.992 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:10.992 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:10.992 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 15:26:10.992 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:10.992 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:10.992 | INFO | Uploaded 36GiB to GPU in 2.21219s (16.2734GiB/s) | 20260505 15:26:10.992 | INFO | Downloaded 34GiB to GPU in 2.09818s (16.2045GiB/s) | 20260505 15:26:11.123 | INFO | Starting a round of Baz IO | 20260505 15:26:14.339 | INFO | Populating RTMetrics report | 20260505 15:26:14.340 | INFO | RTMetrics report populated | 20260505 15:26:14.340 | INFO | Opening tmp RTMetrics json file | 20260505 15:26:14.354 | INFO | Serializing RTMetrics to json | 20260505 15:26:14.355 | INFO | Serialization done, swapping tmp file now... | 20260505 15:26:14.356 | INFO | RTMetrics done | 20260505 15:26:14.445 | INFO | Finished a round of Baz IO: Wrote 2.213 GiB in 3.321 seconds | 20260505 15:26:14.445 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.47% / 0.00% / 77.52% | 20260505 15:26:14.445 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.449212% / 3.97364e-06% / 7.94729e-05% | 20260505 15:26:14.445 | INFO | Baz IO overall: | 20260505 15:26:14.445 | INFO | BytesWritten : 2541633603366 | 20260505 15:26:14.445 | INFO | NumEvents : 1207456099323 | 20260505 15:26:16.127 | INFO | Analyzing chunk frames = [4272128,4272640) | 20260505 15:26:16.127 | INFO | MemInfo: { | 20260505 15:26:16.127 | INFO | "gpu" : " 17 GB", | 20260505 15:26:16.127 | INFO | "huge" : "", | 20260505 15:26:16.127 | INFO | "huge_shared" : " 27 GB", | 20260505 15:26:16.127 | INFO | "malloc" : " 14 GB", | 20260505 15:26:16.127 | INFO | "pinned" : " 29 GB" | 20260505 15:26:16.127 | INFO | } | 20260505 15:26:16.127 | INFO | Perf report for the last 5.1358 seconds: | 20260505 15:26:16.127 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:26:16.127 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:16.127 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:16.127 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:16.127 | INFO | ANALYSIS : 64.1%, 3.0, 99.6ms, 0 | 20260505 15:26:16.127 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:16.127 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260505 15:26:16.127 | INFO | PRE_HQ : 61.1%, 1.8, 57.0ms, 0 | 20260505 15:26:16.127 | INFO | BAZWRITER : 64.0%, 3.9, 128.0ms, 0 | 20260505 15:26:16.127 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 15:26:16.128 | INFO | Uploaded 36GiB to GPU in 2.20938s (16.2942GiB/s) | 20260505 15:26:16.128 | INFO | Downloaded 38GiB to GPU in 2.27136s (16.73GiB/s) | 20260505 15:26:21.264 | INFO | Analyzing chunk frames = [4272640,4273152) | 20260505 15:26:21.264 | INFO | MemInfo: { | 20260505 15:26:21.264 | INFO | "gpu" : " 17 GB", | 20260505 15:26:21.264 | INFO | "huge" : "", | 20260505 15:26:21.264 | INFO | "huge_shared" : " 27 GB", | 20260505 15:26:21.264 | INFO | "malloc" : " 14 GB", | 20260505 15:26:21.264 | INFO | "pinned" : " 29 GB" | 20260505 15:26:21.264 | INFO | } | 20260505 15:26:21.264 | INFO | Perf report for the last 5.13651 seconds: | 20260505 15:26:21.264 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:26:21.264 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:21.264 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:21.264 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:21.264 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 15:26:21.264 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:21.264 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:21.264 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:26:21.264 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:21.264 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:21.264 | INFO | Uploaded 36GiB to GPU in 2.20361s (16.3368GiB/s) | 20260505 15:26:21.264 | INFO | Downloaded 34GiB to GPU in 2.06926s (16.431GiB/s) | 20260505 15:26:26.400 | INFO | Analyzing chunk frames = [4273152,4273664) | 20260505 15:26:26.400 | INFO | MemInfo: { | 20260505 15:26:26.400 | INFO | "gpu" : " 17 GB", | 20260505 15:26:26.400 | INFO | "huge" : "", | 20260505 15:26:26.400 | INFO | "huge_shared" : " 27 GB", | 20260505 15:26:26.400 | INFO | "malloc" : " 14 GB", | 20260505 15:26:26.400 | INFO | "pinned" : " 29 GB" | 20260505 15:26:26.401 | INFO | } | 20260505 15:26:26.401 | INFO | Perf report for the last 5.13678 seconds: | 20260505 15:26:26.401 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:26:26.401 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:26.401 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:26.401 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:26.401 | INFO | ANALYSIS : 63.1%, 3.0, 98.2ms, 0 | 20260505 15:26:26.401 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:26.401 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:26.401 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:26:26.401 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:26.401 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:26.401 | INFO | Uploaded 36GiB to GPU in 2.16448s (16.6322GiB/s) | 20260505 15:26:26.401 | INFO | Downloaded 34GiB to GPU in 2.06592s (16.4576GiB/s) | 20260505 15:26:31.546 | INFO | Analyzing chunk frames = [4273664,4274176) | 20260505 15:26:31.546 | INFO | MemInfo: { | 20260505 15:26:31.546 | INFO | "gpu" : " 17 GB", | 20260505 15:26:31.546 | INFO | "huge" : "", | 20260505 15:26:31.546 | INFO | "huge_shared" : " 27 GB", | 20260505 15:26:31.546 | INFO | "malloc" : " 14 GB", | 20260505 15:26:31.547 | INFO | "pinned" : " 29 GB" | 20260505 15:26:31.547 | INFO | } | 20260505 15:26:31.547 | INFO | Perf report for the last 5.14601 seconds: | 20260505 15:26:31.547 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:26:31.547 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:31.547 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:31.547 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:31.547 | INFO | ANALYSIS : 62.6%, 3.0, 97.3ms, 0 | 20260505 15:26:31.547 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:31.547 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:31.547 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 15:26:31.547 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:31.547 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:31.547 | INFO | Uploaded 36GiB to GPU in 2.18755s (16.4567GiB/s) | 20260505 15:26:31.547 | INFO | Downloaded 34GiB to GPU in 2.13064s (15.9576GiB/s) | 20260505 15:26:36.693 | INFO | Analyzing chunk frames = [4274176,4274688) | 20260505 15:26:36.693 | INFO | MemInfo: { | 20260505 15:26:36.693 | INFO | "gpu" : " 17 GB", | 20260505 15:26:36.693 | INFO | "huge" : "", | 20260505 15:26:36.693 | INFO | "huge_shared" : " 27 GB", | 20260505 15:26:36.693 | INFO | "malloc" : " 14 GB", | 20260505 15:26:36.693 | INFO | "pinned" : " 29 GB" | 20260505 15:26:36.693 | INFO | } | 20260505 15:26:36.693 | INFO | Perf report for the last 5.14648 seconds: | 20260505 15:26:36.693 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:26:36.693 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:36.693 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:36.693 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:36.693 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260505 15:26:36.693 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:36.693 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:36.693 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 15:26:36.693 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:36.693 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:36.694 | INFO | Uploaded 36GiB to GPU in 2.16924s (16.5956GiB/s) | 20260505 15:26:36.694 | INFO | Downloaded 34GiB to GPU in 2.13128s (15.9528GiB/s) | 20260505 15:26:41.823 | INFO | Analyzing chunk frames = [4274688,4275200) | 20260505 15:26:41.823 | INFO | MemInfo: { | 20260505 15:26:41.823 | INFO | "gpu" : " 17 GB", | 20260505 15:26:41.823 | INFO | "huge" : "", | 20260505 15:26:41.823 | INFO | "huge_shared" : " 27 GB", | 20260505 15:26:41.823 | INFO | "malloc" : " 14 GB", | 20260505 15:26:41.824 | INFO | "pinned" : " 29 GB" | 20260505 15:26:41.824 | INFO | } | 20260505 15:26:41.824 | INFO | Perf report for the last 5.1304 seconds: | 20260505 15:26:41.824 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:26:41.824 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:41.824 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:41.824 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:41.824 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 15:26:41.824 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:41.824 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:41.824 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 15:26:41.824 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:41.824 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:41.824 | INFO | Uploaded 36GiB to GPU in 2.13668s (16.8486GiB/s) | 20260505 15:26:41.824 | INFO | Downloaded 34GiB to GPU in 2.03628s (16.6971GiB/s) | 20260505 15:26:46.957 | INFO | Analyzing chunk frames = [4275200,4275712) | 20260505 15:26:46.957 | INFO | MemInfo: { | 20260505 15:26:46.957 | INFO | "gpu" : " 17 GB", | 20260505 15:26:46.957 | INFO | "huge" : "", | 20260505 15:26:46.957 | INFO | "huge_shared" : " 27 GB", | 20260505 15:26:46.957 | INFO | "malloc" : " 14 GB", | 20260505 15:26:46.957 | INFO | "pinned" : " 29 GB" | 20260505 15:26:46.957 | INFO | } | 20260505 15:26:46.957 | INFO | Perf report for the last 5.13381 seconds: | 20260505 15:26:46.957 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:26:46.957 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:46.957 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:46.957 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:46.957 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 15:26:46.957 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:46.957 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:46.957 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 15:26:46.957 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:46.958 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:46.958 | INFO | Uploaded 36GiB to GPU in 2.12956s (16.9049GiB/s) | 20260505 15:26:46.958 | INFO | Downloaded 34GiB to GPU in 2.17963s (15.599GiB/s) | 20260505 15:26:52.099 | INFO | Analyzing chunk frames = [4275712,4276224) | 20260505 15:26:52.099 | INFO | MemInfo: { | 20260505 15:26:52.099 | INFO | "gpu" : " 17 GB", | 20260505 15:26:52.099 | INFO | "huge" : "", | 20260505 15:26:52.099 | INFO | "huge_shared" : " 27 GB", | 20260505 15:26:52.099 | INFO | "malloc" : " 14 GB", | 20260505 15:26:52.100 | INFO | "pinned" : " 29 GB" | 20260505 15:26:52.100 | INFO | } | 20260505 15:26:52.100 | INFO | Perf report for the last 5.14221 seconds: | 20260505 15:26:52.100 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:26:52.100 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:52.100 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:52.100 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:52.100 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 15:26:52.100 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:52.100 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:52.100 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 15:26:52.100 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:52.100 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:52.100 | INFO | Uploaded 36GiB to GPU in 2.21704s (16.2379GiB/s) | 20260505 15:26:52.100 | INFO | Downloaded 34GiB to GPU in 2.10755s (16.1325GiB/s) | 20260505 15:26:52.100 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 15:26:52.100 | INFO | Total mem usage: 15.3 GiB | 20260505 15:26:52.100 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 15:26:52.100 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.84%) | 20260505 15:26:52.100 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 15:26:52.100 | INFO | High Water Breakdown: | 20260505 15:26:52.100 | INFO | 61% : 8.67 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 15:26:52.100 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 15:26:52.100 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 15:26:52.100 | INFO | ------------------------------------------ | 20260505 15:26:52.100 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:26:52.100 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 15:26:52.100 | INFO | Total mem usage: 26.5 GiB | 20260505 15:26:52.100 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 15:26:52.100 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 15:26:52.100 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 15:26:52.100 | INFO | High Water Breakdown: | 20260505 15:26:52.100 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:26:52.100 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:26:52.100 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:26:52.100 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:26:52.100 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:26:52.100 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:26:52.100 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 15:26:52.100 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 15:26:52.100 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:26:52.100 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 15:26:52.100 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 15:26:52.100 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 15:26:52.100 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 15:26:52.100 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 15:26:52.100 | INFO | ------------------------------------------ | 20260505 15:26:52.100 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:26:52.100 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 15:26:52.100 | INFO | Total mem usage: 25.6 GiB | 20260505 15:26:52.100 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 15:26:52.100 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 15:26:52.100 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 15:26:52.100 | INFO | High Water Breakdown: | 20260505 15:26:52.100 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 15:26:52.100 | INFO | ------------------------------------------ | 20260505 15:26:52.100 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:26:52.100 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 15:26:52.101 | INFO | Total mem usage: 15.7 GiB | 20260505 15:26:52.101 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 15:26:52.101 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 15:26:52.101 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260505 15:26:52.101 | INFO | High Water Breakdown: | 20260505 15:26:52.101 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 15:26:52.101 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 15:26:52.101 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 15:26:52.101 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 15:26:52.101 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 15:26:52.101 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 15:26:52.101 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 15:26:52.101 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 15:26:52.101 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 15:26:52.101 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:26:52.101 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 15:26:52.101 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:26:52.101 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 15:26:52.101 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 15:26:52.101 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 15:26:52.101 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 15:26:52.101 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:26:52.101 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 15:26:52.101 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:26:52.101 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:26:52.101 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:26:52.101 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 15:26:52.101 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 15:26:52.101 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:26:52.101 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 15:26:52.101 | INFO | ------------------------------------------ | 20260505 15:26:52.101 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:26:52.101 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 15:26:52.101 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:26:52.101 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:26:52.101 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.57e+01, 1.35e+01 / 1.60e+00 | 20260505 15:26:52.101 | INFO | PerfReport: 0.29%, AggregateStats -- 9.26e-02 / 9.99e-02, 1.12e-02 / 5.06e-03 | 20260505 15:26:52.101 | INFO | PerfReport: 11.76%, Baseline -- 3.73e+00 / 4.49e+00, 5.69e-01 / 1.92e-01 | 20260505 15:26:52.101 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.50e+00, 1.92e-01 / 1.09e-01 | 20260505 15:26:52.101 | INFO | PerfReport: 15.96%, DME -- 4.04e+01 / 4.51e+01, 4.39e+00 / 1.93e+00 | 20260505 15:26:52.101 | INFO | PerfReport: 56.56%, FrameLabeling -- 1.79e+01 / 1.98e+01, 1.39e+00 / 1.95e+00 | 20260505 15:26:52.101 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.36e+00 / 2.91e+00, 4.49e-01 / 1.80e-01 | 20260505 15:26:52.101 | INFO | PerfReport: 3.49%, Metrics -- 1.11e+00 / 1.41e+00, 3.05e-01 / 1.37e-01 | 20260505 15:26:52.101 | INFO | PerfReport: 1.22%, PulseSerializing -- 3.87e-01 / 4.39e-01, 6.83e-02 / 3.11e-02 | 20260505 15:26:52.101 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 15:26:52.101 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:26:52.101 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:26:52.101 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260505 15:26:52.101 | INFO | PerfReport: 22.66%, Upload -- 2.28e+01 / 0.00e+00, 8.10e+00 / 0.00e+00 | 20260505 15:26:52.101 | INFO | PerfReport: 21.99%, Download -- 2.21e+01 / 0.00e+00, 8.03e+00 / 0.00e+00 | 20260505 15:26:52.101 | INFO | PerfReport: 31.52%, Compute -- 3.17e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 15:26:52.101 | INFO | PerfReport: 2.31%, LockUpload -- 2.32e+00 / 0.00e+00, 7.90e+00 / 0.00e+00 | 20260505 15:26:52.101 | INFO | PerfReport: 1.38%, LockDownload -- 1.39e+00 / 0.00e+00, 2.86e+00 / 0.00e+00 | 20260505 15:26:52.101 | INFO | PerfReport: 20.13%, LockCompute -- 2.02e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260505 15:26:52.101 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.84e-02 / 0.00e+00 | 20260505 15:26:52.235 | INFO | Starting a round of Baz IO | 20260505 15:26:55.451 | INFO | Populating RTMetrics report | 20260505 15:26:55.452 | INFO | RTMetrics report populated | 20260505 15:26:55.452 | INFO | Opening tmp RTMetrics json file | 20260505 15:26:55.464 | INFO | Serializing RTMetrics to json | 20260505 15:26:55.465 | INFO | Serialization done, swapping tmp file now... | 20260505 15:26:55.465 | INFO | RTMetrics done | 20260505 15:26:55.549 | INFO | Finished a round of Baz IO: Wrote 2.213 GiB in 3.314 seconds | 20260505 15:26:55.549 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.47% / 0.00% / 77.53% | 20260505 15:26:55.549 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.448386% / 3.97364e-06% / 6.75519e-05% | 20260505 15:26:55.549 | INFO | Baz IO overall: | 20260505 15:26:55.549 | INFO | BytesWritten : 2544010013796 | 20260505 15:26:55.549 | INFO | NumEvents : 1208573790777 | 20260505 15:26:57.244 | INFO | Analyzing chunk frames = [4276224,4276736) | 20260505 15:26:57.244 | INFO | MemInfo: { | 20260505 15:26:57.244 | INFO | "gpu" : " 17 GB", | 20260505 15:26:57.244 | INFO | "huge" : "", | 20260505 15:26:57.244 | INFO | "huge_shared" : " 27 GB", | 20260505 15:26:57.244 | INFO | "malloc" : " 14 GB", | 20260505 15:26:57.244 | INFO | "pinned" : " 29 GB" | 20260505 15:26:57.244 | INFO | } | 20260505 15:26:57.244 | INFO | Perf report for the last 5.14461 seconds: | 20260505 15:26:57.244 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:26:57.244 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:57.244 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:57.244 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:57.244 | INFO | ANALYSIS : 64.1%, 3.0, 99.9ms, 0 | 20260505 15:26:57.244 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:26:57.244 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260505 15:26:57.244 | INFO | PRE_HQ : 60.9%, 1.8, 56.8ms, 0 | 20260505 15:26:57.244 | INFO | BAZWRITER : 63.8%, 3.9, 127.2ms, 0 | 20260505 15:26:57.244 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 15:26:57.245 | INFO | Uploaded 36GiB to GPU in 2.18693s (16.4614GiB/s) | 20260505 15:26:57.245 | INFO | Downloaded 38GiB to GPU in 2.25123s (16.8797GiB/s) | 20260505 15:27:02.387 | INFO | Analyzing chunk frames = [4276736,4277248) | 20260505 15:27:02.387 | INFO | MemInfo: { | 20260505 15:27:02.387 | INFO | "gpu" : " 17 GB", | 20260505 15:27:02.387 | INFO | "huge" : "", | 20260505 15:27:02.387 | INFO | "huge_shared" : " 27 GB", | 20260505 15:27:02.387 | INFO | "malloc" : " 14 GB", | 20260505 15:27:02.387 | INFO | "pinned" : " 29 GB" | 20260505 15:27:02.387 | INFO | } | 20260505 15:27:02.388 | INFO | Perf report for the last 5.1433 seconds: | 20260505 15:27:02.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:27:02.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:02.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:02.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:02.388 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 15:27:02.388 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:02.388 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:02.388 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:27:02.388 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:02.388 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:02.388 | INFO | Uploaded 36GiB to GPU in 2.12689s (16.9261GiB/s) | 20260505 15:27:02.388 | INFO | Downloaded 34GiB to GPU in 2.03203s (16.7321GiB/s) | 20260505 15:27:07.517 | INFO | Analyzing chunk frames = [4277248,4277760) | 20260505 15:27:07.517 | INFO | MemInfo: { | 20260505 15:27:07.517 | INFO | "gpu" : " 17 GB", | 20260505 15:27:07.517 | INFO | "huge" : "", | 20260505 15:27:07.517 | INFO | "huge_shared" : " 27 GB", | 20260505 15:27:07.517 | INFO | "malloc" : " 14 GB", | 20260505 15:27:07.517 | INFO | "pinned" : " 29 GB" | 20260505 15:27:07.517 | INFO | } | 20260505 15:27:07.517 | INFO | Perf report for the last 5.1294 seconds: | 20260505 15:27:07.517 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:27:07.517 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:07.517 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:07.517 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:07.517 | INFO | ANALYSIS : 63.0%, 3.0, 97.8ms, 0 | 20260505 15:27:07.517 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:07.517 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:07.517 | INFO | PRE_HQ : 28.1%, 1.0, 14.5ms, 0 | 20260505 15:27:07.517 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:07.517 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:07.517 | INFO | Uploaded 36GiB to GPU in 2.17104s (16.5819GiB/s) | 20260505 15:27:07.517 | INFO | Downloaded 34GiB to GPU in 2.0324s (16.7289GiB/s) | 20260505 15:27:12.658 | INFO | Analyzing chunk frames = [4277760,4278272) | 20260505 15:27:12.658 | INFO | MemInfo: { | 20260505 15:27:12.658 | INFO | "gpu" : " 17 GB", | 20260505 15:27:12.658 | INFO | "huge" : "", | 20260505 15:27:12.658 | INFO | "huge_shared" : " 27 GB", | 20260505 15:27:12.658 | INFO | "malloc" : " 14 GB", | 20260505 15:27:12.658 | INFO | "pinned" : " 29 GB" | 20260505 15:27:12.658 | INFO | } | 20260505 15:27:12.658 | INFO | Perf report for the last 5.14133 seconds: | 20260505 15:27:12.658 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:27:12.658 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:12.658 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:12.658 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:12.658 | INFO | ANALYSIS : 62.6%, 3.0, 97.3ms, 0 | 20260505 15:27:12.658 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:12.658 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:12.658 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:27:12.658 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:12.658 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:12.659 | INFO | Uploaded 36GiB to GPU in 2.18466s (16.4785GiB/s) | 20260505 15:27:12.659 | INFO | Downloaded 34GiB to GPU in 2.1304s (15.9595GiB/s) | 20260505 15:27:17.806 | INFO | Analyzing chunk frames = [4278272,4278784) | 20260505 15:27:17.806 | INFO | MemInfo: { | 20260505 15:27:17.806 | INFO | "gpu" : " 17 GB", | 20260505 15:27:17.806 | INFO | "huge" : "", | 20260505 15:27:17.806 | INFO | "huge_shared" : " 27 GB", | 20260505 15:27:17.806 | INFO | "malloc" : " 14 GB", | 20260505 15:27:17.806 | INFO | "pinned" : " 29 GB" | 20260505 15:27:17.806 | INFO | } | 20260505 15:27:17.806 | INFO | Perf report for the last 5.1477 seconds: | 20260505 15:27:17.806 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:27:17.806 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:17.806 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:17.806 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:17.806 | INFO | ANALYSIS : 62.6%, 3.0, 97.9ms, 0 | 20260505 15:27:17.806 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:17.806 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:17.806 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:27:17.806 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:17.806 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:17.806 | INFO | Uploaded 36GiB to GPU in 2.17848s (16.5253GiB/s) | 20260505 15:27:17.806 | INFO | Downloaded 34GiB to GPU in 2.13266s (15.9425GiB/s) | 20260505 15:27:22.937 | INFO | Analyzing chunk frames = [4278784,4279296) | 20260505 15:27:22.937 | INFO | MemInfo: { | 20260505 15:27:22.937 | INFO | "gpu" : " 17 GB", | 20260505 15:27:22.937 | INFO | "huge" : "", | 20260505 15:27:22.937 | INFO | "huge_shared" : " 27 GB", | 20260505 15:27:22.937 | INFO | "malloc" : " 14 GB", | 20260505 15:27:22.937 | INFO | "pinned" : " 29 GB" | 20260505 15:27:22.937 | INFO | } | 20260505 15:27:22.937 | INFO | Perf report for the last 5.1309 seconds: | 20260505 15:27:22.937 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:27:22.937 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:22.937 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:22.937 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:22.937 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260505 15:27:22.937 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:22.937 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:22.937 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:27:22.937 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:22.937 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:22.937 | INFO | Uploaded 36GiB to GPU in 2.13646s (16.8503GiB/s) | 20260505 15:27:22.937 | INFO | Downloaded 34GiB to GPU in 2.03154s (16.736GiB/s) | 20260505 15:27:28.072 | INFO | Analyzing chunk frames = [4279296,4279808) | 20260505 15:27:28.072 | INFO | MemInfo: { | 20260505 15:27:28.072 | INFO | "gpu" : " 17 GB", | 20260505 15:27:28.072 | INFO | "huge" : "", | 20260505 15:27:28.072 | INFO | "huge_shared" : " 27 GB", | 20260505 15:27:28.072 | INFO | "malloc" : " 14 GB", | 20260505 15:27:28.072 | INFO | "pinned" : " 29 GB" | 20260505 15:27:28.072 | INFO | } | 20260505 15:27:28.072 | INFO | Perf report for the last 5.13528 seconds: | 20260505 15:27:28.072 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:27:28.072 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:28.072 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:28.072 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:28.072 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 15:27:28.072 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:28.072 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:28.072 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 15:27:28.072 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:28.073 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:28.073 | INFO | Uploaded 36GiB to GPU in 2.11212s (17.0444GiB/s) | 20260505 15:27:28.073 | INFO | Downloaded 34GiB to GPU in 2.17102s (15.6608GiB/s) | 20260505 15:27:33.207 | INFO | Analyzing chunk frames = [4279808,4280320) | 20260505 15:27:33.207 | INFO | MemInfo: { | 20260505 15:27:33.207 | INFO | "gpu" : " 17 GB", | 20260505 15:27:33.207 | INFO | "huge" : "", | 20260505 15:27:33.207 | INFO | "huge_shared" : " 27 GB", | 20260505 15:27:33.207 | INFO | "malloc" : " 14 GB", | 20260505 15:27:33.207 | INFO | "pinned" : " 29 GB" | 20260505 15:27:33.207 | INFO | } | 20260505 15:27:33.207 | INFO | Perf report for the last 5.13492 seconds: | 20260505 15:27:33.207 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:27:33.207 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:33.207 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:33.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:33.207 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 15:27:33.207 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:33.207 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:33.207 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 15:27:33.207 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:33.207 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:33.207 | INFO | Uploaded 36GiB to GPU in 2.22387s (16.188GiB/s) | 20260505 15:27:33.207 | INFO | Downloaded 34GiB to GPU in 2.10663s (16.1396GiB/s) | 20260505 15:27:33.344 | INFO | Starting a round of Baz IO | 20260505 15:27:36.560 | INFO | Populating RTMetrics report | 20260505 15:27:36.560 | INFO | RTMetrics report populated | 20260505 15:27:36.560 | INFO | Opening tmp RTMetrics json file | 20260505 15:27:36.573 | INFO | Serializing RTMetrics to json | 20260505 15:27:36.574 | INFO | Serialization done, swapping tmp file now... | 20260505 15:27:36.575 | INFO | RTMetrics done | 20260505 15:27:36.659 | INFO | Finished a round of Baz IO: Wrote 2.214 GiB in 3.314 seconds | 20260505 15:27:36.659 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.47% / 0.00% / 77.53% | 20260505 15:27:36.659 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.448803% / 3.97364e-06% / 7.15256e-05% | 20260505 15:27:36.659 | INFO | Baz IO overall: | 20260505 15:27:36.659 | INFO | BytesWritten : 2546387311854 | 20260505 15:27:36.659 | INFO | NumEvents : 1209692354491 | 20260505 15:27:38.352 | INFO | Analyzing chunk frames = [4280320,4280832) | 20260505 15:27:38.352 | INFO | MemInfo: { | 20260505 15:27:38.352 | INFO | "gpu" : " 17 GB", | 20260505 15:27:38.352 | INFO | "huge" : "", | 20260505 15:27:38.352 | INFO | "huge_shared" : " 27 GB", | 20260505 15:27:38.352 | INFO | "malloc" : " 14 GB", | 20260505 15:27:38.352 | INFO | "pinned" : " 29 GB" | 20260505 15:27:38.352 | INFO | } | 20260505 15:27:38.352 | INFO | Perf report for the last 5.14457 seconds: | 20260505 15:27:38.352 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:27:38.352 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:38.352 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:38.352 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:38.352 | INFO | ANALYSIS : 64.1%, 3.0, 99.8ms, 0 | 20260505 15:27:38.352 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:38.352 | INFO | RT_METRICS : 44.9%, 1.0, 23.6ms, 0 | 20260505 15:27:38.352 | INFO | PRE_HQ : 60.5%, 1.8, 56.6ms, 0 | 20260505 15:27:38.352 | INFO | BAZWRITER : 63.8%, 3.9, 127.8ms, 0 | 20260505 15:27:38.352 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 15:27:38.352 | INFO | Uploaded 36GiB to GPU in 2.1803s (16.5115GiB/s) | 20260505 15:27:38.352 | INFO | Downloaded 38GiB to GPU in 2.27836s (16.6786GiB/s) | 20260505 15:27:43.499 | INFO | Analyzing chunk frames = [4280832,4281344) | 20260505 15:27:43.499 | INFO | MemInfo: { | 20260505 15:27:43.499 | INFO | "gpu" : " 17 GB", | 20260505 15:27:43.499 | INFO | "huge" : "", | 20260505 15:27:43.499 | INFO | "huge_shared" : " 27 GB", | 20260505 15:27:43.499 | INFO | "malloc" : " 14 GB", | 20260505 15:27:43.499 | INFO | "pinned" : " 29 GB" | 20260505 15:27:43.499 | INFO | } | 20260505 15:27:43.500 | INFO | Perf report for the last 5.14783 seconds: | 20260505 15:27:43.500 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:27:43.500 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:43.500 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:43.500 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:43.500 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 15:27:43.500 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:43.500 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:43.500 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 15:27:43.500 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:43.500 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:43.500 | INFO | Uploaded 36GiB to GPU in 2.19635s (16.3908GiB/s) | 20260505 15:27:43.500 | INFO | Downloaded 34GiB to GPU in 2.05118s (16.5758GiB/s) | 20260505 15:27:48.645 | INFO | Analyzing chunk frames = [4281344,4281856) | 20260505 15:27:48.645 | INFO | MemInfo: { | 20260505 15:27:48.645 | INFO | "gpu" : " 17 GB", | 20260505 15:27:48.645 | INFO | "huge" : "", | 20260505 15:27:48.645 | INFO | "huge_shared" : " 27 GB", | 20260505 15:27:48.645 | INFO | "malloc" : " 14 GB", | 20260505 15:27:48.645 | INFO | "pinned" : " 29 GB" | 20260505 15:27:48.645 | INFO | } | 20260505 15:27:48.645 | INFO | Perf report for the last 5.1455 seconds: | 20260505 15:27:48.645 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:27:48.645 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:48.645 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:48.645 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:48.645 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 15:27:48.645 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:48.645 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:48.645 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 15:27:48.645 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:48.645 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:48.645 | INFO | Uploaded 36GiB to GPU in 2.19896s (16.3714GiB/s) | 20260505 15:27:48.646 | INFO | Downloaded 34GiB to GPU in 2.03697s (16.6915GiB/s) | 20260505 15:27:53.773 | INFO | Analyzing chunk frames = [4281856,4282368) | 20260505 15:27:53.773 | INFO | MemInfo: { | 20260505 15:27:53.773 | INFO | "gpu" : " 17 GB", | 20260505 15:27:53.773 | INFO | "huge" : "", | 20260505 15:27:53.773 | INFO | "huge_shared" : " 27 GB", | 20260505 15:27:53.773 | INFO | "malloc" : " 14 GB", | 20260505 15:27:53.773 | INFO | "pinned" : " 29 GB" | 20260505 15:27:53.773 | INFO | } | 20260505 15:27:53.773 | INFO | Perf report for the last 5.12798 seconds: | 20260505 15:27:53.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:27:53.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:53.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:53.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:53.773 | INFO | ANALYSIS : 62.8%, 3.0, 97.4ms, 0 | 20260505 15:27:53.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:53.773 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:53.773 | INFO | PRE_HQ : 28.3%, 1.0, 14.6ms, 0 | 20260505 15:27:53.773 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:53.773 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:53.774 | INFO | Uploaded 36GiB to GPU in 2.1717s (16.5769GiB/s) | 20260505 15:27:53.774 | INFO | Downloaded 34GiB to GPU in 2.14305s (15.8652GiB/s) | 20260505 15:27:58.903 | INFO | Analyzing chunk frames = [4282368,4282880) | 20260505 15:27:58.903 | INFO | MemInfo: { | 20260505 15:27:58.903 | INFO | "gpu" : " 17 GB", | 20260505 15:27:58.903 | INFO | "huge" : "", | 20260505 15:27:58.903 | INFO | "huge_shared" : " 27 GB", | 20260505 15:27:58.903 | INFO | "malloc" : " 14 GB", | 20260505 15:27:58.903 | INFO | "pinned" : " 29 GB" | 20260505 15:27:58.903 | INFO | } | 20260505 15:27:58.903 | INFO | Perf report for the last 5.1302 seconds: | 20260505 15:27:58.903 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:27:58.903 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:58.903 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:58.903 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:58.903 | INFO | ANALYSIS : 62.9%, 3.0, 98.0ms, 0 | 20260505 15:27:58.903 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:58.903 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:58.903 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 15:27:58.903 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:58.903 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:27:58.903 | INFO | Uploaded 36GiB to GPU in 2.18729s (16.4587GiB/s) | 20260505 15:27:58.903 | INFO | Downloaded 34GiB to GPU in 2.13069s (15.9573GiB/s) | 20260505 15:28:04.055 | INFO | Analyzing chunk frames = [4282880,4283392) | 20260505 15:28:04.055 | INFO | MemInfo: { | 20260505 15:28:04.055 | INFO | "gpu" : " 17 GB", | 20260505 15:28:04.055 | INFO | "huge" : "", | 20260505 15:28:04.055 | INFO | "huge_shared" : " 27 GB", | 20260505 15:28:04.055 | INFO | "malloc" : " 14 GB", | 20260505 15:28:04.055 | INFO | "pinned" : " 29 GB" | 20260505 15:28:04.055 | INFO | } | 20260505 15:28:04.055 | INFO | Perf report for the last 5.15216 seconds: | 20260505 15:28:04.056 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:28:04.056 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:04.056 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:04.056 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:04.056 | INFO | ANALYSIS : 62.8%, 3.0, 98.3ms, 0 | 20260505 15:28:04.056 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:04.056 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:04.056 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 15:28:04.056 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:04.056 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:04.056 | INFO | Uploaded 36GiB to GPU in 2.13668s (16.8485GiB/s) | 20260505 15:28:04.056 | INFO | Downloaded 34GiB to GPU in 2.10977s (16.1155GiB/s) | 20260505 15:28:09.193 | INFO | Analyzing chunk frames = [4283392,4283904) | 20260505 15:28:09.193 | INFO | MemInfo: { | 20260505 15:28:09.193 | INFO | "gpu" : " 17 GB", | 20260505 15:28:09.193 | INFO | "huge" : "", | 20260505 15:28:09.193 | INFO | "huge_shared" : " 27 GB", | 20260505 15:28:09.193 | INFO | "malloc" : " 14 GB", | 20260505 15:28:09.193 | INFO | "pinned" : " 29 GB" | 20260505 15:28:09.193 | INFO | } | 20260505 15:28:09.193 | INFO | Perf report for the last 5.1378 seconds: | 20260505 15:28:09.193 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:28:09.193 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:09.193 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:09.193 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:09.193 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 15:28:09.194 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:09.194 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:09.194 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 15:28:09.194 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:09.194 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:09.194 | INFO | Uploaded 36GiB to GPU in 2.10559s (17.0973GiB/s) | 20260505 15:28:09.194 | INFO | Downloaded 34GiB to GPU in 2.18277s (15.5765GiB/s) | 20260505 15:28:14.332 | INFO | Analyzing chunk frames = [4283904,4284416) | 20260505 15:28:14.332 | INFO | MemInfo: { | 20260505 15:28:14.332 | INFO | "gpu" : " 17 GB", | 20260505 15:28:14.332 | INFO | "huge" : "", | 20260505 15:28:14.332 | INFO | "huge_shared" : " 27 GB", | 20260505 15:28:14.332 | INFO | "malloc" : " 14 GB", | 20260505 15:28:14.332 | INFO | "pinned" : " 29 GB" | 20260505 15:28:14.332 | INFO | } | 20260505 15:28:14.332 | INFO | Perf report for the last 5.13883 seconds: | 20260505 15:28:14.332 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:28:14.332 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:14.332 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:14.332 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:14.332 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 15:28:14.332 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:14.332 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:14.332 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 15:28:14.332 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:14.332 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:14.332 | INFO | Uploaded 36GiB to GPU in 2.15808s (16.6815GiB/s) | 20260505 15:28:14.332 | INFO | Downloaded 34GiB to GPU in 2.03465s (16.7105GiB/s) | 20260505 15:28:14.332 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 15:28:14.332 | INFO | Total mem usage: 15.3 GiB | 20260505 15:28:14.333 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 15:28:14.333 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.86%) | 20260505 15:28:14.333 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 15:28:14.333 | INFO | High Water Breakdown: | 20260505 15:28:14.333 | INFO | 61% : 8.67 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 15:28:14.333 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 15:28:14.333 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 15:28:14.333 | INFO | ------------------------------------------ | 20260505 15:28:14.333 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:28:14.333 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 15:28:14.333 | INFO | Total mem usage: 26.5 GiB | 20260505 15:28:14.333 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 15:28:14.333 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 15:28:14.333 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 15:28:14.333 | INFO | High Water Breakdown: | 20260505 15:28:14.333 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:28:14.333 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:28:14.333 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:28:14.333 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:28:14.333 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:28:14.333 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:28:14.333 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 15:28:14.333 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 15:28:14.333 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:28:14.333 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 15:28:14.333 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 15:28:14.333 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 15:28:14.333 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 15:28:14.333 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 15:28:14.333 | INFO | ------------------------------------------ | 20260505 15:28:14.333 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:28:14.333 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 15:28:14.333 | INFO | Total mem usage: 25.6 GiB | 20260505 15:28:14.333 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 15:28:14.333 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 15:28:14.333 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 15:28:14.333 | INFO | High Water Breakdown: | 20260505 15:28:14.333 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 15:28:14.333 | INFO | ------------------------------------------ | 20260505 15:28:14.333 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:28:14.333 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 15:28:14.333 | INFO | Total mem usage: 15.7 GiB | 20260505 15:28:14.333 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 15:28:14.333 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 15:28:14.333 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260505 15:28:14.333 | INFO | High Water Breakdown: | 20260505 15:28:14.333 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 15:28:14.333 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 15:28:14.333 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 15:28:14.333 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 15:28:14.333 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 15:28:14.333 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 15:28:14.333 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 15:28:14.334 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 15:28:14.334 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 15:28:14.334 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:28:14.334 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 15:28:14.334 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:28:14.334 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 15:28:14.334 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 15:28:14.334 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 15:28:14.334 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 15:28:14.334 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:28:14.334 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 15:28:14.334 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:28:14.334 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:28:14.334 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:28:14.334 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 15:28:14.334 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 15:28:14.334 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:28:14.334 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 15:28:14.334 | INFO | ------------------------------------------ | 20260505 15:28:14.334 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:28:14.334 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 15:28:14.334 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:28:14.334 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:28:14.334 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.61e+01, 1.38e+01 / 2.41e+00 | 20260505 15:28:14.334 | INFO | PerfReport: 0.29%, AggregateStats -- 9.27e-02 / 1.02e-01, 1.09e-02 / 5.63e-03 | 20260505 15:28:14.334 | INFO | PerfReport: 11.77%, Baseline -- 3.74e+00 / 4.45e+00, 5.75e-01 / 2.02e-01 | 20260505 15:28:14.334 | INFO | PerfReport: 3.29%, Histogram -- 1.05e+00 / 1.49e+00, 1.96e-01 / 1.09e-01 | 20260505 15:28:14.334 | INFO | PerfReport: 16.02%, DME -- 4.07e+01 / 4.58e+01, 4.70e+00 / 2.50e+00 | 20260505 15:28:14.334 | INFO | PerfReport: 56.57%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.42e+00 / 1.44e+00 | 20260505 15:28:14.334 | INFO | PerfReport: 7.39%, PulseAccumulating -- 2.34e+00 / 2.94e+00, 4.52e-01 / 1.73e-01 | 20260505 15:28:14.334 | INFO | PerfReport: 3.45%, Metrics -- 1.09e+00 / 1.42e+00, 3.04e-01 / 1.26e-01 | 20260505 15:28:14.334 | INFO | PerfReport: 1.22%, PulseSerializing -- 3.86e-01 / 4.37e-01, 6.80e-02 / 2.85e-02 | 20260505 15:28:14.334 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 15:28:14.334 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:28:14.334 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:28:14.334 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.56e+01 / 0.00e+00 | 20260505 15:28:14.334 | INFO | PerfReport: 22.45%, Upload -- 2.26e+01 / 0.00e+00, 7.94e+00 / 0.00e+00 | 20260505 15:28:14.334 | INFO | PerfReport: 21.93%, Download -- 2.20e+01 / 0.00e+00, 8.01e+00 / 0.00e+00 | 20260505 15:28:14.334 | INFO | PerfReport: 31.58%, Compute -- 3.17e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 15:28:14.334 | INFO | PerfReport: 2.23%, LockUpload -- 2.24e+00 / 0.00e+00, 7.94e+00 / 0.00e+00 | 20260505 15:28:14.334 | INFO | PerfReport: 1.52%, LockDownload -- 1.53e+00 / 0.00e+00, 3.16e+00 / 0.00e+00 | 20260505 15:28:14.334 | INFO | PerfReport: 20.27%, LockCompute -- 2.04e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260505 15:28:14.334 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.86e-02 / 0.00e+00 | 20260505 15:28:14.462 | INFO | Starting a round of Baz IO | 20260505 15:28:17.678 | INFO | Populating RTMetrics report | 20260505 15:28:17.679 | INFO | RTMetrics report populated | 20260505 15:28:17.679 | INFO | Opening tmp RTMetrics json file | 20260505 15:28:17.692 | INFO | Serializing RTMetrics to json | 20260505 15:28:17.696 | INFO | Serialization done, swapping tmp file now... | 20260505 15:28:17.701 | INFO | RTMetrics done | 20260505 15:28:17.779 | INFO | Finished a round of Baz IO: Wrote 2.21 GiB in 3.316 seconds | 20260505 15:28:17.779 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.47% / 0.00% / 77.53% | 20260505 15:28:17.779 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.444436% / 0% / 7.54992e-05% | 20260505 15:28:17.779 | INFO | Baz IO overall: | 20260505 15:28:17.779 | INFO | BytesWritten : 2548760019444 | 20260505 15:28:17.779 | INFO | NumEvents : 1210807250640 | 20260505 15:28:19.467 | INFO | Analyzing chunk frames = [4284416,4284928) | 20260505 15:28:19.467 | INFO | MemInfo: { | 20260505 15:28:19.467 | INFO | "gpu" : " 17 GB", | 20260505 15:28:19.467 | INFO | "huge" : "", | 20260505 15:28:19.467 | INFO | "huge_shared" : " 27 GB", | 20260505 15:28:19.467 | INFO | "malloc" : " 14 GB", | 20260505 15:28:19.467 | INFO | "pinned" : " 29 GB" | 20260505 15:28:19.467 | INFO | } | 20260505 15:28:19.467 | INFO | Perf report for the last 5.13489 seconds: | 20260505 15:28:19.467 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:28:19.467 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:19.467 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:19.467 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:19.467 | INFO | ANALYSIS : 64.1%, 3.0, 99.7ms, 0 | 20260505 15:28:19.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:19.467 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260505 15:28:19.467 | INFO | PRE_HQ : 60.9%, 1.8, 57.1ms, 0 | 20260505 15:28:19.467 | INFO | BAZWRITER : 63.9%, 3.9, 127.5ms, 0 | 20260505 15:28:19.467 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 15:28:19.467 | INFO | Uploaded 36GiB to GPU in 2.19221s (16.4218GiB/s) | 20260505 15:28:19.467 | INFO | Downloaded 38GiB to GPU in 2.26154s (16.8027GiB/s) | 20260505 15:28:24.610 | INFO | Analyzing chunk frames = [4284928,4285440) | 20260505 15:28:24.610 | INFO | MemInfo: { | 20260505 15:28:24.610 | INFO | "gpu" : " 17 GB", | 20260505 15:28:24.610 | INFO | "huge" : "", | 20260505 15:28:24.610 | INFO | "huge_shared" : " 27 GB", | 20260505 15:28:24.610 | INFO | "malloc" : " 14 GB", | 20260505 15:28:24.610 | INFO | "pinned" : " 29 GB" | 20260505 15:28:24.610 | INFO | } | 20260505 15:28:24.610 | INFO | Perf report for the last 5.14306 seconds: | 20260505 15:28:24.610 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:28:24.610 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:24.610 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:24.610 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:24.610 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 15:28:24.610 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:24.610 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:24.610 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 15:28:24.610 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:24.610 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:24.610 | INFO | Uploaded 36GiB to GPU in 2.19332s (16.4135GiB/s) | 20260505 15:28:24.610 | INFO | Downloaded 34GiB to GPU in 2.03653s (16.6951GiB/s) | 20260505 15:28:29.753 | INFO | Analyzing chunk frames = [4285440,4285952) | 20260505 15:28:29.753 | INFO | MemInfo: { | 20260505 15:28:29.753 | INFO | "gpu" : " 17 GB", | 20260505 15:28:29.753 | INFO | "huge" : "", | 20260505 15:28:29.753 | INFO | "huge_shared" : " 27 GB", | 20260505 15:28:29.753 | INFO | "malloc" : " 14 GB", | 20260505 15:28:29.753 | INFO | "pinned" : " 29 GB" | 20260505 15:28:29.753 | INFO | } | 20260505 15:28:29.753 | INFO | Perf report for the last 5.14301 seconds: | 20260505 15:28:29.753 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:28:29.753 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:29.753 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:29.753 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:29.753 | INFO | ANALYSIS : 62.8%, 3.0, 97.8ms, 0 | 20260505 15:28:29.753 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:29.753 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:29.753 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:28:29.753 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:29.753 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:29.753 | INFO | Uploaded 36GiB to GPU in 2.18389s (16.4844GiB/s) | 20260505 15:28:29.753 | INFO | Downloaded 34GiB to GPU in 2.03664s (16.6942GiB/s) | 20260505 15:28:34.888 | INFO | Analyzing chunk frames = [4285952,4286464) | 20260505 15:28:34.888 | INFO | MemInfo: { | 20260505 15:28:34.888 | INFO | "gpu" : " 17 GB", | 20260505 15:28:34.888 | INFO | "huge" : "", | 20260505 15:28:34.888 | INFO | "huge_shared" : " 27 GB", | 20260505 15:28:34.888 | INFO | "malloc" : " 14 GB", | 20260505 15:28:34.888 | INFO | "pinned" : " 29 GB" | 20260505 15:28:34.888 | INFO | } | 20260505 15:28:34.888 | INFO | Perf report for the last 5.13489 seconds: | 20260505 15:28:34.888 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:28:34.888 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:34.888 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:34.888 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:34.888 | INFO | ANALYSIS : 62.6%, 3.0, 97.1ms, 0 | 20260505 15:28:34.888 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:34.888 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:34.888 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 15:28:34.888 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:34.888 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:34.888 | INFO | Uploaded 36GiB to GPU in 2.19301s (16.4158GiB/s) | 20260505 15:28:34.888 | INFO | Downloaded 34GiB to GPU in 2.13447s (15.929GiB/s) | 20260505 15:28:40.020 | INFO | Analyzing chunk frames = [4286464,4286976) | 20260505 15:28:40.020 | INFO | MemInfo: { | 20260505 15:28:40.020 | INFO | "gpu" : " 17 GB", | 20260505 15:28:40.020 | INFO | "huge" : "", | 20260505 15:28:40.020 | INFO | "huge_shared" : " 27 GB", | 20260505 15:28:40.020 | INFO | "malloc" : " 14 GB", | 20260505 15:28:40.020 | INFO | "pinned" : " 29 GB" | 20260505 15:28:40.020 | INFO | } | 20260505 15:28:40.021 | INFO | Perf report for the last 5.13251 seconds: | 20260505 15:28:40.021 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:28:40.021 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:40.021 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:40.021 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:40.021 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 15:28:40.021 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:40.021 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:40.021 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 15:28:40.021 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:40.021 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:40.021 | INFO | Uploaded 36GiB to GPU in 2.16092s (16.6595GiB/s) | 20260505 15:28:40.021 | INFO | Downloaded 34GiB to GPU in 2.13033s (15.9599GiB/s) | 20260505 15:28:45.162 | INFO | Analyzing chunk frames = [4286976,4287488) | 20260505 15:28:45.162 | INFO | MemInfo: { | 20260505 15:28:45.162 | INFO | "gpu" : " 17 GB", | 20260505 15:28:45.162 | INFO | "huge" : "", | 20260505 15:28:45.162 | INFO | "huge_shared" : " 27 GB", | 20260505 15:28:45.162 | INFO | "malloc" : " 14 GB", | 20260505 15:28:45.162 | INFO | "pinned" : " 29 GB" | 20260505 15:28:45.162 | INFO | } | 20260505 15:28:45.162 | INFO | Perf report for the last 5.14153 seconds: | 20260505 15:28:45.162 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:28:45.162 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:45.162 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:45.162 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:45.162 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 15:28:45.162 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:45.162 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:45.162 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 15:28:45.162 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:45.162 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:45.162 | INFO | Uploaded 36GiB to GPU in 2.19374s (16.4103GiB/s) | 20260505 15:28:45.162 | INFO | Downloaded 34GiB to GPU in 2.11561s (16.071GiB/s) | 20260505 15:28:50.308 | INFO | Analyzing chunk frames = [4287488,4288000) | 20260505 15:28:50.308 | INFO | MemInfo: { | 20260505 15:28:50.308 | INFO | "gpu" : " 17 GB", | 20260505 15:28:50.308 | INFO | "huge" : "", | 20260505 15:28:50.308 | INFO | "huge_shared" : " 27 GB", | 20260505 15:28:50.308 | INFO | "malloc" : " 14 GB", | 20260505 15:28:50.308 | INFO | "pinned" : " 29 GB" | 20260505 15:28:50.308 | INFO | } | 20260505 15:28:50.308 | INFO | Perf report for the last 5.14609 seconds: | 20260505 15:28:50.308 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:28:50.308 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:50.308 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:50.308 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:50.308 | INFO | ANALYSIS : 63.3%, 3.0, 99.0ms, 0 | 20260505 15:28:50.308 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:50.308 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:50.308 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 15:28:50.308 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:50.308 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:50.309 | INFO | Uploaded 36GiB to GPU in 2.16173s (16.6533GiB/s) | 20260505 15:28:50.309 | INFO | Downloaded 34GiB to GPU in 2.16698s (15.6901GiB/s) | 20260505 15:28:55.450 | INFO | Analyzing chunk frames = [4288000,4288512) | 20260505 15:28:55.450 | INFO | MemInfo: { | 20260505 15:28:55.450 | INFO | "gpu" : " 17 GB", | 20260505 15:28:55.450 | INFO | "huge" : "", | 20260505 15:28:55.450 | INFO | "huge_shared" : " 27 GB", | 20260505 15:28:55.450 | INFO | "malloc" : " 14 GB", | 20260505 15:28:55.450 | INFO | "pinned" : " 29 GB" | 20260505 15:28:55.450 | INFO | } | 20260505 15:28:55.450 | INFO | Perf report for the last 5.14195 seconds: | 20260505 15:28:55.450 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:28:55.450 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:55.450 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:55.450 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:55.450 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 15:28:55.450 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:55.450 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:55.450 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 15:28:55.450 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:55.450 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:28:55.451 | INFO | Uploaded 36GiB to GPU in 2.1704s (16.5868GiB/s) | 20260505 15:28:55.451 | INFO | Downloaded 34GiB to GPU in 2.0285s (16.7612GiB/s) | 20260505 15:28:55.584 | INFO | Starting a round of Baz IO | 20260505 15:28:58.790 | INFO | Populating RTMetrics report | 20260505 15:28:58.790 | INFO | RTMetrics report populated | 20260505 15:28:58.790 | INFO | Opening tmp RTMetrics json file | 20260505 15:28:58.803 | INFO | Serializing RTMetrics to json | 20260505 15:28:58.804 | INFO | Serialization done, swapping tmp file now... | 20260505 15:28:58.805 | INFO | RTMetrics done | 20260505 15:28:58.891 | INFO | Finished a round of Baz IO: Wrote 2.211 GiB in 3.306 seconds | 20260505 15:28:58.891 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.46% / 0.00% / 77.53% | 20260505 15:28:58.891 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.444412% / 3.97364e-06% / 5.96046e-05% | 20260505 15:28:58.891 | INFO | Baz IO overall: | 20260505 15:28:58.891 | INFO | BytesWritten : 2551133888336 | 20260505 15:28:58.891 | INFO | NumEvents : 1211923403262 | 20260505 15:29:00.588 | INFO | Analyzing chunk frames = [4288512,4289024) | 20260505 15:29:00.588 | INFO | MemInfo: { | 20260505 15:29:00.588 | INFO | "gpu" : " 17 GB", | 20260505 15:29:00.588 | INFO | "huge" : "", | 20260505 15:29:00.588 | INFO | "huge_shared" : " 27 GB", | 20260505 15:29:00.588 | INFO | "malloc" : " 14 GB", | 20260505 15:29:00.588 | INFO | "pinned" : " 29 GB" | 20260505 15:29:00.588 | INFO | } | 20260505 15:29:00.588 | INFO | Perf report for the last 5.13781 seconds: | 20260505 15:29:00.588 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:29:00.588 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:00.588 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:00.588 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:00.588 | INFO | ANALYSIS : 63.9%, 3.0, 99.5ms, 0 | 20260505 15:29:00.588 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:00.588 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260505 15:29:00.588 | INFO | PRE_HQ : 61.0%, 1.8, 57.2ms, 0 | 20260505 15:29:00.588 | INFO | BAZWRITER : 63.7%, 3.9, 127.8ms, 0 | 20260505 15:29:00.588 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 15:29:00.588 | INFO | Uploaded 36GiB to GPU in 2.18091s (16.5069GiB/s) | 20260505 15:29:00.588 | INFO | Downloaded 38GiB to GPU in 2.25948s (16.818GiB/s) | 20260505 15:29:05.717 | INFO | Analyzing chunk frames = [4289024,4289536) | 20260505 15:29:05.717 | INFO | MemInfo: { | 20260505 15:29:05.717 | INFO | "gpu" : " 17 GB", | 20260505 15:29:05.717 | INFO | "huge" : "", | 20260505 15:29:05.717 | INFO | "huge_shared" : " 27 GB", | 20260505 15:29:05.717 | INFO | "malloc" : " 14 GB", | 20260505 15:29:05.717 | INFO | "pinned" : " 29 GB" | 20260505 15:29:05.717 | INFO | } | 20260505 15:29:05.717 | INFO | Perf report for the last 5.12938 seconds: | 20260505 15:29:05.718 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:29:05.718 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:05.718 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:05.718 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:05.718 | INFO | ANALYSIS : 63.6%, 3.0, 99.1ms, 0 | 20260505 15:29:05.718 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:05.718 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:05.718 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 15:29:05.718 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:05.718 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:05.718 | INFO | Uploaded 36GiB to GPU in 2.20381s (16.3354GiB/s) | 20260505 15:29:05.718 | INFO | Downloaded 34GiB to GPU in 2.06206s (16.4883GiB/s) | 20260505 15:29:10.860 | INFO | Analyzing chunk frames = [4289536,4290048) | 20260505 15:29:10.860 | INFO | MemInfo: { | 20260505 15:29:10.860 | INFO | "gpu" : " 17 GB", | 20260505 15:29:10.860 | INFO | "huge" : "", | 20260505 15:29:10.860 | INFO | "huge_shared" : " 27 GB", | 20260505 15:29:10.860 | INFO | "malloc" : " 14 GB", | 20260505 15:29:10.860 | INFO | "pinned" : " 29 GB" | 20260505 15:29:10.860 | INFO | } | 20260505 15:29:10.860 | INFO | Perf report for the last 5.14312 seconds: | 20260505 15:29:10.861 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:29:10.861 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:10.861 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:10.861 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:10.861 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 15:29:10.861 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:10.861 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:10.861 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:29:10.861 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:10.861 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:10.861 | INFO | Uploaded 36GiB to GPU in 2.22956s (16.1467GiB/s) | 20260505 15:29:10.861 | INFO | Downloaded 34GiB to GPU in 2.09708s (16.213GiB/s) | 20260505 15:29:16.007 | INFO | Analyzing chunk frames = [4290048,4290560) | 20260505 15:29:16.007 | INFO | MemInfo: { | 20260505 15:29:16.007 | INFO | "gpu" : " 17 GB", | 20260505 15:29:16.007 | INFO | "huge" : "", | 20260505 15:29:16.007 | INFO | "huge_shared" : " 27 GB", | 20260505 15:29:16.007 | INFO | "malloc" : " 14 GB", | 20260505 15:29:16.007 | INFO | "pinned" : " 29 GB" | 20260505 15:29:16.007 | INFO | } | 20260505 15:29:16.007 | INFO | Perf report for the last 5.14687 seconds: | 20260505 15:29:16.007 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:29:16.007 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:16.007 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:16.007 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:16.007 | INFO | ANALYSIS : 62.6%, 3.0, 97.4ms, 0 | 20260505 15:29:16.007 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:16.007 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:16.007 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 15:29:16.007 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:16.007 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:16.008 | INFO | Uploaded 36GiB to GPU in 2.18867s (16.4484GiB/s) | 20260505 15:29:16.008 | INFO | Downloaded 34GiB to GPU in 2.13263s (15.9427GiB/s) | 20260505 15:29:21.153 | INFO | Analyzing chunk frames = [4290560,4291072) | 20260505 15:29:21.153 | INFO | MemInfo: { | 20260505 15:29:21.153 | INFO | "gpu" : " 17 GB", | 20260505 15:29:21.153 | INFO | "huge" : "", | 20260505 15:29:21.153 | INFO | "huge_shared" : " 27 GB", | 20260505 15:29:21.153 | INFO | "malloc" : " 14 GB", | 20260505 15:29:21.153 | INFO | "pinned" : " 29 GB" | 20260505 15:29:21.153 | INFO | } | 20260505 15:29:21.153 | INFO | Perf report for the last 5.14605 seconds: | 20260505 15:29:21.153 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:29:21.153 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:21.153 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:21.153 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:21.153 | INFO | ANALYSIS : 62.7%, 3.0, 98.0ms, 0 | 20260505 15:29:21.153 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:21.153 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:21.153 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 15:29:21.153 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:21.154 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:21.154 | INFO | Uploaded 36GiB to GPU in 2.19588s (16.3943GiB/s) | 20260505 15:29:21.154 | INFO | Downloaded 34GiB to GPU in 2.13313s (15.939GiB/s) | 20260505 15:29:26.278 | INFO | Analyzing chunk frames = [4291072,4291584) | 20260505 15:29:26.278 | INFO | MemInfo: { | 20260505 15:29:26.278 | INFO | "gpu" : " 17 GB", | 20260505 15:29:26.278 | INFO | "huge" : "", | 20260505 15:29:26.278 | INFO | "huge_shared" : " 27 GB", | 20260505 15:29:26.278 | INFO | "malloc" : " 14 GB", | 20260505 15:29:26.278 | INFO | "pinned" : " 29 GB" | 20260505 15:29:26.278 | INFO | } | 20260505 15:29:26.278 | INFO | Perf report for the last 5.12492 seconds: | 20260505 15:29:26.278 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:29:26.278 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:26.278 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:26.278 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:26.278 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 15:29:26.278 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:26.278 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:26.278 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 15:29:26.278 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:26.278 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:26.279 | INFO | Uploaded 36GiB to GPU in 2.14041s (16.8192GiB/s) | 20260505 15:29:26.279 | INFO | Downloaded 34GiB to GPU in 2.03621s (16.6976GiB/s) | 20260505 15:29:31.414 | INFO | Analyzing chunk frames = [4291584,4292096) | 20260505 15:29:31.414 | INFO | MemInfo: { | 20260505 15:29:31.414 | INFO | "gpu" : " 17 GB", | 20260505 15:29:31.414 | INFO | "huge" : "", | 20260505 15:29:31.414 | INFO | "huge_shared" : " 27 GB", | 20260505 15:29:31.414 | INFO | "malloc" : " 14 GB", | 20260505 15:29:31.414 | INFO | "pinned" : " 29 GB" | 20260505 15:29:31.414 | INFO | } | 20260505 15:29:31.414 | INFO | Perf report for the last 5.13615 seconds: | 20260505 15:29:31.414 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:29:31.414 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:31.414 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:31.415 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:31.415 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 15:29:31.415 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:31.415 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:31.415 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260505 15:29:31.415 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:31.415 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:31.415 | INFO | Uploaded 36GiB to GPU in 2.11811s (16.9963GiB/s) | 20260505 15:29:31.415 | INFO | Downloaded 34GiB to GPU in 2.1773s (15.6156GiB/s) | 20260505 15:29:36.564 | INFO | Analyzing chunk frames = [4292096,4292608) | 20260505 15:29:36.564 | INFO | MemInfo: { | 20260505 15:29:36.564 | INFO | "gpu" : " 17 GB", | 20260505 15:29:36.564 | INFO | "huge" : "", | 20260505 15:29:36.564 | INFO | "huge_shared" : " 27 GB", | 20260505 15:29:36.564 | INFO | "malloc" : " 14 GB", | 20260505 15:29:36.564 | INFO | "pinned" : " 29 GB" | 20260505 15:29:36.564 | INFO | } | 20260505 15:29:36.564 | INFO | Perf report for the last 5.1499 seconds: | 20260505 15:29:36.564 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:29:36.564 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:36.564 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:36.564 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:36.564 | INFO | ANALYSIS : 63.4%, 3.0, 99.3ms, 0 | 20260505 15:29:36.564 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:36.564 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:36.564 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 15:29:36.564 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:36.564 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:36.565 | INFO | Uploaded 36GiB to GPU in 2.22806s (16.1576GiB/s) | 20260505 15:29:36.565 | INFO | Downloaded 34GiB to GPU in 2.0965s (16.2175GiB/s) | 20260505 15:29:36.565 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 15:29:36.565 | INFO | Total mem usage: 15.3 GiB | 20260505 15:29:36.565 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 15:29:36.565 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.84%) | 20260505 15:29:36.565 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 15:29:36.565 | INFO | High Water Breakdown: | 20260505 15:29:36.565 | INFO | 61% : 8.67 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 15:29:36.565 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 15:29:36.565 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 15:29:36.565 | INFO | ------------------------------------------ | 20260505 15:29:36.565 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:29:36.565 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 15:29:36.565 | INFO | Total mem usage: 26.5 GiB | 20260505 15:29:36.565 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 15:29:36.565 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 15:29:36.565 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 15:29:36.565 | INFO | High Water Breakdown: | 20260505 15:29:36.565 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:29:36.565 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:29:36.565 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:29:36.565 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:29:36.565 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:29:36.565 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:29:36.565 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 15:29:36.565 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 15:29:36.565 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:29:36.565 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 15:29:36.565 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 15:29:36.565 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 15:29:36.565 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 15:29:36.565 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 15:29:36.565 | INFO | ------------------------------------------ | 20260505 15:29:36.565 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:29:36.565 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 15:29:36.565 | INFO | Total mem usage: 25.6 GiB | 20260505 15:29:36.565 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 15:29:36.565 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 15:29:36.565 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 15:29:36.565 | INFO | High Water Breakdown: | 20260505 15:29:36.565 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 15:29:36.566 | INFO | ------------------------------------------ | 20260505 15:29:36.566 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:29:36.566 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 15:29:36.566 | INFO | Total mem usage: 15.7 GiB | 20260505 15:29:36.566 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 15:29:36.566 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 15:29:36.566 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260505 15:29:36.566 | INFO | High Water Breakdown: | 20260505 15:29:36.566 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 15:29:36.566 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 15:29:36.566 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 15:29:36.566 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 15:29:36.566 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 15:29:36.566 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 15:29:36.566 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 15:29:36.566 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 15:29:36.566 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 15:29:36.566 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:29:36.566 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 15:29:36.566 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:29:36.566 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 15:29:36.566 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 15:29:36.566 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 15:29:36.566 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 15:29:36.566 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:29:36.566 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 15:29:36.566 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:29:36.566 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:29:36.566 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:29:36.566 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 15:29:36.566 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 15:29:36.566 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:29:36.566 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 15:29:36.566 | INFO | ------------------------------------------ | 20260505 15:29:36.566 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:29:36.566 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 15:29:36.566 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:29:36.566 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:29:36.566 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.57e+01, 1.31e+01 / 1.94e+00 | 20260505 15:29:36.566 | INFO | PerfReport: 0.29%, AggregateStats -- 9.30e-02 / 1.02e-01, 1.07e-02 / 4.33e-03 | 20260505 15:29:36.566 | INFO | PerfReport: 11.78%, Baseline -- 3.74e+00 / 4.46e+00, 5.71e-01 / 2.13e-01 | 20260505 15:29:36.566 | INFO | PerfReport: 3.29%, Histogram -- 1.04e+00 / 1.50e+00, 1.96e-01 / 1.05e-01 | 20260505 15:29:36.566 | INFO | PerfReport: 15.95%, DME -- 4.05e+01 / 4.51e+01, 4.49e+00 / 1.91e+00 | 20260505 15:29:36.566 | INFO | PerfReport: 56.57%, FrameLabeling -- 1.79e+01 / 1.98e+01, 1.42e+00 / 2.08e+00 | 20260505 15:29:36.566 | INFO | PerfReport: 7.42%, PulseAccumulating -- 2.35e+00 / 2.89e+00, 4.51e-01 / 2.33e-01 | 20260505 15:29:36.566 | INFO | PerfReport: 3.47%, Metrics -- 1.10e+00 / 1.39e+00, 3.07e-01 / 1.78e-01 | 20260505 15:29:36.566 | INFO | PerfReport: 1.22%, PulseSerializing -- 3.86e-01 / 4.40e-01, 6.81e-02 / 3.60e-02 | 20260505 15:29:36.566 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 15:29:36.566 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:29:36.566 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:29:36.566 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260505 15:29:36.566 | INFO | PerfReport: 22.62%, Upload -- 2.27e+01 / 0.00e+00, 8.07e+00 / 0.00e+00 | 20260505 15:29:36.566 | INFO | PerfReport: 21.95%, Download -- 2.21e+01 / 0.00e+00, 7.97e+00 / 0.00e+00 | 20260505 15:29:36.566 | INFO | PerfReport: 31.55%, Compute -- 3.17e+01 / 0.00e+00, 1.31e+01 / 0.00e+00 | 20260505 15:29:36.566 | INFO | PerfReport: 2.30%, LockUpload -- 2.31e+00 / 0.00e+00, 7.94e+00 / 0.00e+00 | 20260505 15:29:36.566 | INFO | PerfReport: 1.42%, LockDownload -- 1.43e+00 / 0.00e+00, 2.95e+00 / 0.00e+00 | 20260505 15:29:36.566 | INFO | PerfReport: 20.14%, LockCompute -- 2.03e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260505 15:29:36.566 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.60e-02 / 0.00e+00 | 20260505 15:29:36.696 | INFO | Starting a round of Baz IO | 20260505 15:29:39.905 | INFO | Populating RTMetrics report | 20260505 15:29:39.905 | INFO | RTMetrics report populated | 20260505 15:29:39.905 | INFO | Opening tmp RTMetrics json file | 20260505 15:29:39.922 | INFO | Serializing RTMetrics to json | 20260505 15:29:39.944 | INFO | Serialization done, swapping tmp file now... | 20260505 15:29:39.963 | INFO | RTMetrics done | 20260505 15:29:40.001 | INFO | Finished a round of Baz IO: Wrote 2.212 GiB in 3.304 seconds | 20260505 15:29:40.001 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.46% / 0.00% / 77.54% | 20260505 15:29:40.001 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.445167% / 0% / 5.96046e-05% | 20260505 15:29:40.001 | INFO | Baz IO overall: | 20260505 15:29:40.001 | INFO | BytesWritten : 2553508861501 | 20260505 15:29:40.001 | INFO | NumEvents : 1213040165467 | 20260505 15:29:41.703 | INFO | Analyzing chunk frames = [4292608,4293120) | 20260505 15:29:41.703 | INFO | MemInfo: { | 20260505 15:29:41.703 | INFO | "gpu" : " 17 GB", | 20260505 15:29:41.703 | INFO | "huge" : "", | 20260505 15:29:41.703 | INFO | "huge_shared" : " 27 GB", | 20260505 15:29:41.703 | INFO | "malloc" : " 14 GB", | 20260505 15:29:41.703 | INFO | "pinned" : " 29 GB" | 20260505 15:29:41.703 | INFO | } | 20260505 15:29:41.703 | INFO | Perf report for the last 5.13829 seconds: | 20260505 15:29:41.703 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:29:41.703 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:41.703 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:41.703 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:41.703 | INFO | ANALYSIS : 63.9%, 3.0, 99.3ms, 0 | 20260505 15:29:41.703 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:41.703 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260505 15:29:41.703 | INFO | PRE_HQ : 60.9%, 1.8, 57.0ms, 0 | 20260505 15:29:41.703 | INFO | BAZWRITER : 63.6%, 3.9, 128.5ms, 0 | 20260505 15:29:41.703 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 15:29:41.703 | INFO | Uploaded 36GiB to GPU in 2.16604s (16.6202GiB/s) | 20260505 15:29:41.703 | INFO | Downloaded 38GiB to GPU in 2.26115s (16.8056GiB/s) | 20260505 15:29:46.837 | INFO | Analyzing chunk frames = [4293120,4293632) | 20260505 15:29:46.837 | INFO | MemInfo: { | 20260505 15:29:46.837 | INFO | "gpu" : " 17 GB", | 20260505 15:29:46.837 | INFO | "huge" : "", | 20260505 15:29:46.837 | INFO | "huge_shared" : " 27 GB", | 20260505 15:29:46.837 | INFO | "malloc" : " 14 GB", | 20260505 15:29:46.837 | INFO | "pinned" : " 29 GB" | 20260505 15:29:46.837 | INFO | } | 20260505 15:29:46.837 | INFO | Perf report for the last 5.13408 seconds: | 20260505 15:29:46.837 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:29:46.837 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:46.837 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:46.837 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:46.837 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 15:29:46.837 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:46.837 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:46.837 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 15:29:46.837 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:46.837 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:46.837 | INFO | Uploaded 36GiB to GPU in 2.1339s (16.8705GiB/s) | 20260505 15:29:46.837 | INFO | Downloaded 34GiB to GPU in 2.03735s (16.6884GiB/s) | 20260505 15:29:51.977 | INFO | Analyzing chunk frames = [4293632,4294144) | 20260505 15:29:51.977 | INFO | MemInfo: { | 20260505 15:29:51.977 | INFO | "gpu" : " 17 GB", | 20260505 15:29:51.977 | INFO | "huge" : "", | 20260505 15:29:51.977 | INFO | "huge_shared" : " 27 GB", | 20260505 15:29:51.977 | INFO | "malloc" : " 14 GB", | 20260505 15:29:51.977 | INFO | "pinned" : " 29 GB" | 20260505 15:29:51.977 | INFO | } | 20260505 15:29:51.977 | INFO | Perf report for the last 5.14026 seconds: | 20260505 15:29:51.977 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:29:51.977 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:51.977 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:51.977 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:51.977 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 15:29:51.977 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:51.977 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:51.977 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 15:29:51.977 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:51.977 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:51.977 | INFO | Uploaded 36GiB to GPU in 2.24042s (16.0684GiB/s) | 20260505 15:29:51.977 | INFO | Downloaded 34GiB to GPU in 2.1158s (16.0696GiB/s) | 20260505 15:29:57.120 | INFO | Analyzing chunk frames = [4294144,4294656) | 20260505 15:29:57.120 | INFO | MemInfo: { | 20260505 15:29:57.120 | INFO | "gpu" : " 17 GB", | 20260505 15:29:57.120 | INFO | "huge" : "", | 20260505 15:29:57.120 | INFO | "huge_shared" : " 27 GB", | 20260505 15:29:57.120 | INFO | "malloc" : " 14 GB", | 20260505 15:29:57.120 | INFO | "pinned" : " 29 GB" | 20260505 15:29:57.120 | INFO | } | 20260505 15:29:57.120 | INFO | Perf report for the last 5.14276 seconds: | 20260505 15:29:57.120 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:29:57.120 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:57.120 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:57.120 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:57.120 | INFO | ANALYSIS : 62.6%, 3.0, 97.3ms, 0 | 20260505 15:29:57.120 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:57.120 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:57.120 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 15:29:57.120 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:57.120 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:29:57.120 | INFO | Uploaded 36GiB to GPU in 2.17893s (16.5219GiB/s) | 20260505 15:29:57.120 | INFO | Downloaded 34GiB to GPU in 2.13593s (15.9181GiB/s) | 20260505 15:30:02.259 | INFO | Analyzing chunk frames = [4294656,4295168) | 20260505 15:30:02.259 | INFO | MemInfo: { | 20260505 15:30:02.259 | INFO | "gpu" : " 17 GB", | 20260505 15:30:02.259 | INFO | "huge" : "", | 20260505 15:30:02.259 | INFO | "huge_shared" : " 27 GB", | 20260505 15:30:02.260 | INFO | "malloc" : " 14 GB", | 20260505 15:30:02.260 | INFO | "pinned" : " 29 GB" | 20260505 15:30:02.260 | INFO | } | 20260505 15:30:02.260 | INFO | Perf report for the last 5.13973 seconds: | 20260505 15:30:02.260 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:30:02.260 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:02.260 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:02.260 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:02.260 | INFO | ANALYSIS : 62.7%, 3.0, 98.0ms, 0 | 20260505 15:30:02.260 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:02.260 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:02.260 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 15:30:02.260 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:02.260 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:02.260 | INFO | Uploaded 36GiB to GPU in 2.19616s (16.3923GiB/s) | 20260505 15:30:02.260 | INFO | Downloaded 34GiB to GPU in 2.13117s (15.9537GiB/s) | 20260505 15:30:07.397 | INFO | Analyzing chunk frames = [4295168,4295680) | 20260505 15:30:07.397 | INFO | MemInfo: { | 20260505 15:30:07.397 | INFO | "gpu" : " 17 GB", | 20260505 15:30:07.397 | INFO | "huge" : "", | 20260505 15:30:07.397 | INFO | "huge_shared" : " 27 GB", | 20260505 15:30:07.397 | INFO | "malloc" : " 14 GB", | 20260505 15:30:07.397 | INFO | "pinned" : " 29 GB" | 20260505 15:30:07.397 | INFO | } | 20260505 15:30:07.397 | INFO | Perf report for the last 5.13762 seconds: | 20260505 15:30:07.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:30:07.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:07.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:07.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:07.397 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 15:30:07.397 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:07.397 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:07.397 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 15:30:07.397 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:07.397 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:07.397 | INFO | Uploaded 36GiB to GPU in 2.1413s (16.8122GiB/s) | 20260505 15:30:07.398 | INFO | Downloaded 34GiB to GPU in 2.03567s (16.7021GiB/s) | 20260505 15:30:12.530 | INFO | Analyzing chunk frames = [4295680,4296192) | 20260505 15:30:12.530 | INFO | MemInfo: { | 20260505 15:30:12.530 | INFO | "gpu" : " 17 GB", | 20260505 15:30:12.530 | INFO | "huge" : "", | 20260505 15:30:12.530 | INFO | "huge_shared" : " 27 GB", | 20260505 15:30:12.530 | INFO | "malloc" : " 14 GB", | 20260505 15:30:12.530 | INFO | "pinned" : " 29 GB" | 20260505 15:30:12.530 | INFO | } | 20260505 15:30:12.531 | INFO | Perf report for the last 5.13329 seconds: | 20260505 15:30:12.531 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:30:12.531 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:12.531 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:12.531 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:12.531 | INFO | ANALYSIS : 63.6%, 3.0, 99.1ms, 0 | 20260505 15:30:12.531 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:12.531 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:12.531 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 15:30:12.531 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:12.531 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:12.531 | INFO | Uploaded 36GiB to GPU in 2.12744s (16.9218GiB/s) | 20260505 15:30:12.531 | INFO | Downloaded 34GiB to GPU in 2.16327s (15.717GiB/s) | 20260505 15:30:17.670 | INFO | Analyzing chunk frames = [4296192,4296704) | 20260505 15:30:17.670 | INFO | MemInfo: { | 20260505 15:30:17.670 | INFO | "gpu" : " 17 GB", | 20260505 15:30:17.670 | INFO | "huge" : "", | 20260505 15:30:17.670 | INFO | "huge_shared" : " 27 GB", | 20260505 15:30:17.670 | INFO | "malloc" : " 14 GB", | 20260505 15:30:17.670 | INFO | "pinned" : " 29 GB" | 20260505 15:30:17.670 | INFO | } | 20260505 15:30:17.670 | INFO | Perf report for the last 5.13944 seconds: | 20260505 15:30:17.670 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:30:17.670 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:17.670 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:17.670 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:17.670 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 15:30:17.670 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:17.670 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:17.670 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 15:30:17.670 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:17.670 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:17.670 | INFO | Uploaded 36GiB to GPU in 2.22198s (16.2018GiB/s) | 20260505 15:30:17.670 | INFO | Downloaded 34GiB to GPU in 2.1065s (16.1405GiB/s) | 20260505 15:30:17.807 | INFO | Starting a round of Baz IO | 20260505 15:30:21.020 | INFO | Populating RTMetrics report | 20260505 15:30:21.020 | INFO | RTMetrics report populated | 20260505 15:30:21.020 | INFO | Opening tmp RTMetrics json file | 20260505 15:30:21.062 | INFO | Serializing RTMetrics to json | 20260505 15:30:21.103 | INFO | Serialization done, swapping tmp file now... | 20260505 15:30:21.104 | INFO | RTMetrics done | 20260505 15:30:21.116 | INFO | Finished a round of Baz IO: Wrote 2.212 GiB in 3.308 seconds | 20260505 15:30:21.116 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.46% / 0.00% / 77.54% | 20260505 15:30:21.116 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.443852% / 3.97364e-06% / 7.54992e-05% | 20260505 15:30:21.116 | INFO | Baz IO overall: | 20260505 15:30:21.116 | INFO | BytesWritten : 2555884132332 | 20260505 15:30:21.116 | INFO | NumEvents : 1214157459569 | 20260505 15:30:22.817 | INFO | Analyzing chunk frames = [4296704,4297216) | 20260505 15:30:22.817 | INFO | MemInfo: { | 20260505 15:30:22.817 | INFO | "gpu" : " 17 GB", | 20260505 15:30:22.817 | INFO | "huge" : "", | 20260505 15:30:22.817 | INFO | "huge_shared" : " 27 GB", | 20260505 15:30:22.817 | INFO | "malloc" : " 14 GB", | 20260505 15:30:22.817 | INFO | "pinned" : " 29 GB" | 20260505 15:30:22.817 | INFO | } | 20260505 15:30:22.818 | INFO | Perf report for the last 5.14754 seconds: | 20260505 15:30:22.818 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:30:22.818 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:22.818 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:22.818 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:22.818 | INFO | ANALYSIS : 64.0%, 3.0, 99.7ms, 0 | 20260505 15:30:22.818 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:22.818 | INFO | RT_METRICS : 46.5%, 1.0, 24.4ms, 0 | 20260505 15:30:22.818 | INFO | PRE_HQ : 60.9%, 1.8, 57.4ms, 0 | 20260505 15:30:22.818 | INFO | BAZWRITER : 63.7%, 3.9, 128.1ms, 0 | 20260505 15:30:22.818 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 15:30:22.818 | INFO | Uploaded 36GiB to GPU in 2.20251s (16.345GiB/s) | 20260505 15:30:22.818 | INFO | Downloaded 38GiB to GPU in 2.26327s (16.7899GiB/s) | 20260505 15:30:27.969 | INFO | Analyzing chunk frames = [4297216,4297728) | 20260505 15:30:27.969 | INFO | MemInfo: { | 20260505 15:30:27.969 | INFO | "gpu" : " 17 GB", | 20260505 15:30:27.969 | INFO | "huge" : "", | 20260505 15:30:27.970 | INFO | "huge_shared" : " 27 GB", | 20260505 15:30:27.970 | INFO | "malloc" : " 14 GB", | 20260505 15:30:27.970 | INFO | "pinned" : " 29 GB" | 20260505 15:30:27.970 | INFO | } | 20260505 15:30:27.970 | INFO | Perf report for the last 5.15207 seconds: | 20260505 15:30:27.970 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:30:27.970 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:27.970 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:27.970 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:27.970 | INFO | ANALYSIS : 62.9%, 3.0, 98.4ms, 0 | 20260505 15:30:27.970 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:27.970 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:27.970 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 15:30:27.970 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:27.970 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:27.970 | INFO | Uploaded 36GiB to GPU in 2.14375s (16.793GiB/s) | 20260505 15:30:27.970 | INFO | Downloaded 34GiB to GPU in 2.0419s (16.6511GiB/s) | 20260505 15:30:33.094 | INFO | Analyzing chunk frames = [4297728,4298240) | 20260505 15:30:33.094 | INFO | MemInfo: { | 20260505 15:30:33.094 | INFO | "gpu" : " 17 GB", | 20260505 15:30:33.094 | INFO | "huge" : "", | 20260505 15:30:33.094 | INFO | "huge_shared" : " 27 GB", | 20260505 15:30:33.095 | INFO | "malloc" : " 14 GB", | 20260505 15:30:33.095 | INFO | "pinned" : " 29 GB" | 20260505 15:30:33.095 | INFO | } | 20260505 15:30:33.095 | INFO | Perf report for the last 5.12496 seconds: | 20260505 15:30:33.095 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:30:33.095 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:33.095 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:33.095 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:33.095 | INFO | ANALYSIS : 63.0%, 3.0, 97.7ms, 0 | 20260505 15:30:33.095 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:33.095 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:33.095 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 15:30:33.095 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:33.095 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:33.095 | INFO | Uploaded 36GiB to GPU in 2.19794s (16.379GiB/s) | 20260505 15:30:33.095 | INFO | Downloaded 34GiB to GPU in 2.04309s (16.6415GiB/s) | 20260505 15:30:38.226 | INFO | Analyzing chunk frames = [4298240,4298752) | 20260505 15:30:38.226 | INFO | MemInfo: { | 20260505 15:30:38.226 | INFO | "gpu" : " 17 GB", | 20260505 15:30:38.226 | INFO | "huge" : "", | 20260505 15:30:38.226 | INFO | "huge_shared" : " 27 GB", | 20260505 15:30:38.226 | INFO | "malloc" : " 14 GB", | 20260505 15:30:38.226 | INFO | "pinned" : " 29 GB" | 20260505 15:30:38.226 | INFO | } | 20260505 15:30:38.226 | INFO | Perf report for the last 5.13153 seconds: | 20260505 15:30:38.226 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:30:38.226 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:38.226 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:38.226 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:38.226 | INFO | ANALYSIS : 62.8%, 3.0, 97.4ms, 0 | 20260505 15:30:38.226 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:38.226 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:38.226 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 15:30:38.226 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:38.226 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:38.226 | INFO | Uploaded 36GiB to GPU in 2.19652s (16.3895GiB/s) | 20260505 15:30:38.226 | INFO | Downloaded 34GiB to GPU in 2.13131s (15.9526GiB/s) | 20260505 15:30:43.374 | INFO | Analyzing chunk frames = [4298752,4299264) | 20260505 15:30:43.374 | INFO | MemInfo: { | 20260505 15:30:43.374 | INFO | "gpu" : " 17 GB", | 20260505 15:30:43.374 | INFO | "huge" : "", | 20260505 15:30:43.374 | INFO | "huge_shared" : " 27 GB", | 20260505 15:30:43.374 | INFO | "malloc" : " 14 GB", | 20260505 15:30:43.374 | INFO | "pinned" : " 29 GB" | 20260505 15:30:43.374 | INFO | } | 20260505 15:30:43.374 | INFO | Perf report for the last 5.14793 seconds: | 20260505 15:30:43.374 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:30:43.374 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:43.374 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:43.374 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:43.374 | INFO | ANALYSIS : 62.6%, 3.0, 97.9ms, 0 | 20260505 15:30:43.374 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:43.374 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:43.374 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:30:43.374 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:43.374 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:43.374 | INFO | Uploaded 36GiB to GPU in 2.1646s (16.6312GiB/s) | 20260505 15:30:43.374 | INFO | Downloaded 34GiB to GPU in 2.12528s (15.9979GiB/s) | 20260505 15:30:48.510 | INFO | Analyzing chunk frames = [4299264,4299776) | 20260505 15:30:48.510 | INFO | MemInfo: { | 20260505 15:30:48.510 | INFO | "gpu" : " 17 GB", | 20260505 15:30:48.510 | INFO | "huge" : "", | 20260505 15:30:48.510 | INFO | "huge_shared" : " 27 GB", | 20260505 15:30:48.510 | INFO | "malloc" : " 14 GB", | 20260505 15:30:48.510 | INFO | "pinned" : " 29 GB" | 20260505 15:30:48.510 | INFO | } | 20260505 15:30:48.510 | INFO | Perf report for the last 5.13586 seconds: | 20260505 15:30:48.510 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:30:48.510 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:48.510 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:48.510 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:48.510 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 15:30:48.510 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:48.510 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:48.510 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:30:48.510 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:48.510 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:48.510 | INFO | Uploaded 36GiB to GPU in 2.1426s (16.802GiB/s) | 20260505 15:30:48.510 | INFO | Downloaded 34GiB to GPU in 2.02906s (16.7566GiB/s) | 20260505 15:30:53.654 | INFO | Analyzing chunk frames = [4299776,4300288) | 20260505 15:30:53.654 | INFO | MemInfo: { | 20260505 15:30:53.654 | INFO | "gpu" : " 17 GB", | 20260505 15:30:53.654 | INFO | "huge" : "", | 20260505 15:30:53.654 | INFO | "huge_shared" : " 27 GB", | 20260505 15:30:53.654 | INFO | "malloc" : " 14 GB", | 20260505 15:30:53.654 | INFO | "pinned" : " 29 GB" | 20260505 15:30:53.654 | INFO | } | 20260505 15:30:53.654 | INFO | Perf report for the last 5.14383 seconds: | 20260505 15:30:53.654 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:30:53.654 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:53.654 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:53.654 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:53.654 | INFO | ANALYSIS : 63.2%, 3.0, 98.8ms, 0 | 20260505 15:30:53.654 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:53.654 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:53.654 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 15:30:53.654 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:53.654 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:53.654 | INFO | Uploaded 36GiB to GPU in 2.1156s (17.0165GiB/s) | 20260505 15:30:53.654 | INFO | Downloaded 34GiB to GPU in 2.17278s (15.6482GiB/s) | 20260505 15:30:58.792 | INFO | Analyzing chunk frames = [4300288,4300800) | 20260505 15:30:58.792 | INFO | MemInfo: { | 20260505 15:30:58.792 | INFO | "gpu" : " 17 GB", | 20260505 15:30:58.792 | INFO | "huge" : "", | 20260505 15:30:58.792 | INFO | "huge_shared" : " 27 GB", | 20260505 15:30:58.792 | INFO | "malloc" : " 14 GB", | 20260505 15:30:58.792 | INFO | "pinned" : " 29 GB" | 20260505 15:30:58.792 | INFO | } | 20260505 15:30:58.792 | INFO | Perf report for the last 5.13865 seconds: | 20260505 15:30:58.792 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:30:58.792 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:58.792 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:58.792 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:58.792 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 15:30:58.793 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:58.793 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:58.793 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 15:30:58.793 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:58.793 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:30:58.793 | INFO | Uploaded 36GiB to GPU in 2.17666s (16.5391GiB/s) | 20260505 15:30:58.793 | INFO | Downloaded 34GiB to GPU in 2.03015s (16.7475GiB/s) | 20260505 15:30:58.793 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 15:30:58.793 | INFO | Total mem usage: 15.3 GiB | 20260505 15:30:58.793 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 15:30:58.793 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.83%) | 20260505 15:30:58.793 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 15:30:58.793 | INFO | High Water Breakdown: | 20260505 15:30:58.793 | INFO | 61% : 8.68 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 15:30:58.793 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 15:30:58.793 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 15:30:58.793 | INFO | ------------------------------------------ | 20260505 15:30:58.793 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:30:58.793 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 15:30:58.793 | INFO | Total mem usage: 26.5 GiB | 20260505 15:30:58.793 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 15:30:58.793 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 15:30:58.793 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 15:30:58.793 | INFO | High Water Breakdown: | 20260505 15:30:58.793 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:30:58.793 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:30:58.793 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:30:58.793 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:30:58.793 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:30:58.793 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:30:58.793 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 15:30:58.793 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 15:30:58.793 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:30:58.793 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 15:30:58.793 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 15:30:58.793 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 15:30:58.793 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 15:30:58.793 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 15:30:58.793 | INFO | ------------------------------------------ | 20260505 15:30:58.793 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:30:58.793 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 15:30:58.793 | INFO | Total mem usage: 25.6 GiB | 20260505 15:30:58.793 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 15:30:58.793 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 15:30:58.793 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 15:30:58.793 | INFO | High Water Breakdown: | 20260505 15:30:58.793 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 15:30:58.793 | INFO | ------------------------------------------ | 20260505 15:30:58.793 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:30:58.793 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 15:30:58.793 | INFO | Total mem usage: 15.7 GiB | 20260505 15:30:58.793 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 15:30:58.793 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 15:30:58.793 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260505 15:30:58.793 | INFO | High Water Breakdown: | 20260505 15:30:58.793 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 15:30:58.793 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 15:30:58.793 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 15:30:58.793 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 15:30:58.793 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 15:30:58.793 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 15:30:58.793 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 15:30:58.793 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 15:30:58.793 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 15:30:58.793 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:30:58.793 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 15:30:58.793 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:30:58.793 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 15:30:58.793 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 15:30:58.793 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 15:30:58.793 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 15:30:58.793 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:30:58.793 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 15:30:58.793 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:30:58.793 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:30:58.793 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:30:58.793 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 15:30:58.793 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 15:30:58.793 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:30:58.793 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 15:30:58.793 | INFO | ------------------------------------------ | 20260505 15:30:58.793 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:30:58.794 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 15:30:58.794 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:30:58.794 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:30:58.794 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.56e+01, 1.36e+01 / 1.90e+00 | 20260505 15:30:58.794 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 1.01e-01, 1.08e-02 / 4.81e-03 | 20260505 15:30:58.794 | INFO | PerfReport: 11.77%, Baseline -- 3.74e+00 / 4.49e+00, 5.81e-01 / 1.49e-01 | 20260505 15:30:58.794 | INFO | PerfReport: 3.29%, Histogram -- 1.04e+00 / 1.50e+00, 1.95e-01 / 1.05e-01 | 20260505 15:30:58.794 | INFO | PerfReport: 15.91%, DME -- 4.04e+01 / 4.49e+01, 4.36e+00 / 1.95e+00 | 20260505 15:30:58.794 | INFO | PerfReport: 56.68%, FrameLabeling -- 1.80e+01 / 1.99e+01, 1.50e+00 / 2.09e+00 | 20260505 15:30:58.794 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.34e+00 / 2.89e+00, 4.47e-01 / 2.17e-01 | 20260505 15:30:58.794 | INFO | PerfReport: 3.47%, Metrics -- 1.10e+00 / 1.42e+00, 3.08e-01 / 1.81e-01 | 20260505 15:30:58.794 | INFO | PerfReport: 1.22%, PulseSerializing -- 3.86e-01 / 4.37e-01, 6.89e-02 / 3.03e-02 | 20260505 15:30:58.794 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 15:30:58.794 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:30:58.794 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:30:58.794 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260505 15:30:58.794 | INFO | PerfReport: 22.53%, Upload -- 2.26e+01 / 0.00e+00, 8.02e+00 / 0.00e+00 | 20260505 15:30:58.794 | INFO | PerfReport: 21.93%, Download -- 2.20e+01 / 0.00e+00, 8.04e+00 / 0.00e+00 | 20260505 15:30:58.794 | INFO | PerfReport: 31.62%, Compute -- 3.18e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 15:30:58.794 | INFO | PerfReport: 2.23%, LockUpload -- 2.24e+00 / 0.00e+00, 7.90e+00 / 0.00e+00 | 20260505 15:30:58.794 | INFO | PerfReport: 1.41%, LockDownload -- 1.41e+00 / 0.00e+00, 2.98e+00 / 0.00e+00 | 20260505 15:30:58.794 | INFO | PerfReport: 20.27%, LockCompute -- 2.04e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260505 15:30:58.794 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 2.33e-02 / 0.00e+00 | 20260505 15:30:58.929 | INFO | Starting a round of Baz IO | 20260505 15:31:02.142 | INFO | Populating RTMetrics report | 20260505 15:31:02.142 | INFO | RTMetrics report populated | 20260505 15:31:02.142 | INFO | Opening tmp RTMetrics json file | 20260505 15:31:02.160 | INFO | Serializing RTMetrics to json | 20260505 15:31:02.188 | INFO | Serialization done, swapping tmp file now... | 20260505 15:31:02.224 | INFO | RTMetrics done | 20260505 15:31:02.241 | INFO | Finished a round of Baz IO: Wrote 2.211 GiB in 3.312 seconds | 20260505 15:31:02.242 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.45% / 0.00% / 77.54% | 20260505 15:31:02.242 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.443649% / 7.94729e-06% / 8.34465e-05% | 20260505 15:31:02.242 | INFO | Baz IO overall: | 20260505 15:31:02.242 | INFO | BytesWritten : 2558258447680 | 20260505 15:31:02.242 | INFO | NumEvents : 1215273749220 | 20260505 15:31:03.927 | INFO | Analyzing chunk frames = [4300800,4301312) | 20260505 15:31:03.927 | INFO | MemInfo: { | 20260505 15:31:03.927 | INFO | "gpu" : " 17 GB", | 20260505 15:31:03.927 | INFO | "huge" : "", | 20260505 15:31:03.927 | INFO | "huge_shared" : " 27 GB", | 20260505 15:31:03.927 | INFO | "malloc" : " 14 GB", | 20260505 15:31:03.927 | INFO | "pinned" : " 29 GB" | 20260505 15:31:03.927 | INFO | } | 20260505 15:31:03.927 | INFO | Perf report for the last 5.13415 seconds: | 20260505 15:31:03.927 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:31:03.927 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:03.927 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:03.927 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:03.927 | INFO | ANALYSIS : 64.1%, 3.0, 99.7ms, 0 | 20260505 15:31:03.927 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:03.927 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260505 15:31:03.927 | INFO | PRE_HQ : 60.9%, 1.8, 56.8ms, 0 | 20260505 15:31:03.927 | INFO | BAZWRITER : 63.8%, 3.9, 127.3ms, 0 | 20260505 15:31:03.927 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 15:31:03.927 | INFO | Uploaded 36GiB to GPU in 2.21256s (16.2707GiB/s) | 20260505 15:31:03.927 | INFO | Downloaded 38GiB to GPU in 2.27596s (16.6962GiB/s) | 20260505 15:31:09.075 | INFO | Analyzing chunk frames = [4301312,4301824) | 20260505 15:31:09.075 | INFO | MemInfo: { | 20260505 15:31:09.075 | INFO | "gpu" : " 17 GB", | 20260505 15:31:09.075 | INFO | "huge" : "", | 20260505 15:31:09.075 | INFO | "huge_shared" : " 27 GB", | 20260505 15:31:09.075 | INFO | "malloc" : " 14 GB", | 20260505 15:31:09.075 | INFO | "pinned" : " 29 GB" | 20260505 15:31:09.075 | INFO | } | 20260505 15:31:09.075 | INFO | Perf report for the last 5.14837 seconds: | 20260505 15:31:09.075 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:31:09.075 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:09.075 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:09.075 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:09.075 | INFO | ANALYSIS : 63.3%, 3.0, 99.0ms, 0 | 20260505 15:31:09.075 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:09.075 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:09.075 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 15:31:09.075 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:09.075 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:09.075 | INFO | Uploaded 36GiB to GPU in 2.2036s (16.3369GiB/s) | 20260505 15:31:09.075 | INFO | Downloaded 34GiB to GPU in 2.04016s (16.6653GiB/s) | 20260505 15:31:14.211 | INFO | Analyzing chunk frames = [4301824,4302336) | 20260505 15:31:14.211 | INFO | MemInfo: { | 20260505 15:31:14.211 | INFO | "gpu" : " 17 GB", | 20260505 15:31:14.211 | INFO | "huge" : "", | 20260505 15:31:14.211 | INFO | "huge_shared" : " 27 GB", | 20260505 15:31:14.211 | INFO | "malloc" : " 14 GB", | 20260505 15:31:14.211 | INFO | "pinned" : " 29 GB" | 20260505 15:31:14.211 | INFO | } | 20260505 15:31:14.211 | INFO | Perf report for the last 5.13616 seconds: | 20260505 15:31:14.211 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:31:14.211 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:14.211 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:14.211 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:14.211 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260505 15:31:14.211 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:14.211 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:14.211 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 15:31:14.211 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:14.211 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:14.211 | INFO | Uploaded 36GiB to GPU in 2.20105s (16.3558GiB/s) | 20260505 15:31:14.212 | INFO | Downloaded 34GiB to GPU in 2.04195s (16.6508GiB/s) | 20260505 15:31:19.340 | INFO | Analyzing chunk frames = [4302336,4302848) | 20260505 15:31:19.340 | INFO | MemInfo: { | 20260505 15:31:19.340 | INFO | "gpu" : " 17 GB", | 20260505 15:31:19.340 | INFO | "huge" : "", | 20260505 15:31:19.340 | INFO | "huge_shared" : " 27 GB", | 20260505 15:31:19.340 | INFO | "malloc" : " 14 GB", | 20260505 15:31:19.340 | INFO | "pinned" : " 29 GB" | 20260505 15:31:19.340 | INFO | } | 20260505 15:31:19.340 | INFO | Perf report for the last 5.1291 seconds: | 20260505 15:31:19.340 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:31:19.340 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:19.340 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:19.340 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:19.340 | INFO | ANALYSIS : 62.7%, 3.0, 97.1ms, 0 | 20260505 15:31:19.340 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:19.340 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:19.340 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:31:19.340 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:19.340 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:19.341 | INFO | Uploaded 36GiB to GPU in 2.18188s (16.4996GiB/s) | 20260505 15:31:19.341 | INFO | Downloaded 34GiB to GPU in 2.13104s (15.9546GiB/s) | 20260505 15:31:24.481 | INFO | Analyzing chunk frames = [4302848,4303360) | 20260505 15:31:24.481 | INFO | MemInfo: { | 20260505 15:31:24.481 | INFO | "gpu" : " 17 GB", | 20260505 15:31:24.481 | INFO | "huge" : "", | 20260505 15:31:24.481 | INFO | "huge_shared" : " 27 GB", | 20260505 15:31:24.481 | INFO | "malloc" : " 14 GB", | 20260505 15:31:24.481 | INFO | "pinned" : " 29 GB" | 20260505 15:31:24.481 | INFO | } | 20260505 15:31:24.481 | INFO | Perf report for the last 5.14093 seconds: | 20260505 15:31:24.481 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:31:24.481 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:24.481 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:24.481 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:24.481 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 15:31:24.481 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:24.481 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:24.481 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 15:31:24.481 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:24.481 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:24.481 | INFO | Uploaded 36GiB to GPU in 2.17047s (16.5862GiB/s) | 20260505 15:31:24.481 | INFO | Downloaded 34GiB to GPU in 2.12711s (15.9841GiB/s) | 20260505 15:31:29.624 | INFO | Analyzing chunk frames = [4303360,4303872) | 20260505 15:31:29.624 | INFO | MemInfo: { | 20260505 15:31:29.624 | INFO | "gpu" : " 17 GB", | 20260505 15:31:29.624 | INFO | "huge" : "", | 20260505 15:31:29.624 | INFO | "huge_shared" : " 27 GB", | 20260505 15:31:29.624 | INFO | "malloc" : " 14 GB", | 20260505 15:31:29.624 | INFO | "pinned" : " 29 GB" | 20260505 15:31:29.624 | INFO | } | 20260505 15:31:29.624 | INFO | Perf report for the last 5.14298 seconds: | 20260505 15:31:29.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:31:29.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:29.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:29.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:29.624 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 15:31:29.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:29.624 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:29.624 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:31:29.624 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:29.624 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:29.625 | INFO | Uploaded 36GiB to GPU in 2.17634s (16.5415GiB/s) | 20260505 15:31:29.625 | INFO | Downloaded 34GiB to GPU in 2.11142s (16.1029GiB/s) | 20260505 15:31:34.769 | INFO | Analyzing chunk frames = [4303872,4304384) | 20260505 15:31:34.769 | INFO | MemInfo: { | 20260505 15:31:34.769 | INFO | "gpu" : " 17 GB", | 20260505 15:31:34.769 | INFO | "huge" : "", | 20260505 15:31:34.769 | INFO | "huge_shared" : " 27 GB", | 20260505 15:31:34.769 | INFO | "malloc" : " 14 GB", | 20260505 15:31:34.769 | INFO | "pinned" : " 29 GB" | 20260505 15:31:34.769 | INFO | } | 20260505 15:31:34.769 | INFO | Perf report for the last 5.14512 seconds: | 20260505 15:31:34.769 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:31:34.769 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:34.769 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:34.769 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:34.769 | INFO | ANALYSIS : 63.2%, 3.0, 98.8ms, 0 | 20260505 15:31:34.769 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:34.769 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:34.769 | INFO | PRE_HQ : 28.9%, 1.0, 15.1ms, 0 | 20260505 15:31:34.769 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:34.769 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:34.770 | INFO | Uploaded 36GiB to GPU in 2.1202s (16.9795GiB/s) | 20260505 15:31:34.770 | INFO | Downloaded 34GiB to GPU in 2.17426s (15.6375GiB/s) | 20260505 15:31:39.914 | INFO | Analyzing chunk frames = [4304384,4304896) | 20260505 15:31:39.914 | INFO | MemInfo: { | 20260505 15:31:39.914 | INFO | "gpu" : " 17 GB", | 20260505 15:31:39.914 | INFO | "huge" : "", | 20260505 15:31:39.914 | INFO | "huge_shared" : " 27 GB", | 20260505 15:31:39.914 | INFO | "malloc" : " 14 GB", | 20260505 15:31:39.914 | INFO | "pinned" : " 29 GB" | 20260505 15:31:39.914 | INFO | } | 20260505 15:31:39.914 | INFO | Perf report for the last 5.14439 seconds: | 20260505 15:31:39.914 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:31:39.914 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:39.914 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:39.914 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:39.914 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 15:31:39.914 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:39.914 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:39.914 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 15:31:39.914 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:39.914 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:39.914 | INFO | Uploaded 36GiB to GPU in 2.14683s (16.7689GiB/s) | 20260505 15:31:39.914 | INFO | Downloaded 34GiB to GPU in 2.02898s (16.7572GiB/s) | 20260505 15:31:40.046 | INFO | Starting a round of Baz IO | 20260505 15:31:43.256 | INFO | Populating RTMetrics report | 20260505 15:31:43.257 | INFO | RTMetrics report populated | 20260505 15:31:43.257 | INFO | Opening tmp RTMetrics json file | 20260505 15:31:43.269 | INFO | Serializing RTMetrics to json | 20260505 15:31:43.271 | INFO | Serialization done, swapping tmp file now... | 20260505 15:31:43.272 | INFO | RTMetrics done | 20260505 15:31:43.352 | INFO | Finished a round of Baz IO: Wrote 2.212 GiB in 3.306 seconds | 20260505 15:31:43.352 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.45% / 0.00% / 77.55% | 20260505 15:31:43.352 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.443784% / 1.58946e-05% / 7.54992e-05% | 20260505 15:31:43.353 | INFO | Baz IO overall: | 20260505 15:31:43.353 | INFO | BytesWritten : 2560633848602 | 20260505 15:31:43.353 | INFO | NumEvents : 1216391308814 | 20260505 15:31:45.041 | INFO | Analyzing chunk frames = [4304896,4305408) | 20260505 15:31:45.041 | INFO | MemInfo: { | 20260505 15:31:45.041 | INFO | "gpu" : " 17 GB", | 20260505 15:31:45.041 | INFO | "huge" : "", | 20260505 15:31:45.041 | INFO | "huge_shared" : " 27 GB", | 20260505 15:31:45.041 | INFO | "malloc" : " 14 GB", | 20260505 15:31:45.041 | INFO | "pinned" : " 29 GB" | 20260505 15:31:45.041 | INFO | } | 20260505 15:31:45.041 | INFO | Perf report for the last 5.12717 seconds: | 20260505 15:31:45.041 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:31:45.041 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:45.041 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:45.041 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:45.041 | INFO | ANALYSIS : 64.1%, 3.0, 99.6ms, 0 | 20260505 15:31:45.041 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:45.041 | INFO | RT_METRICS : 45.1%, 1.0, 23.6ms, 0 | 20260505 15:31:45.041 | INFO | PRE_HQ : 60.9%, 1.8, 56.7ms, 0 | 20260505 15:31:45.041 | INFO | BAZWRITER : 63.8%, 3.9, 127.7ms, 0 | 20260505 15:31:45.041 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 15:31:45.041 | INFO | Uploaded 36GiB to GPU in 2.20684s (16.3129GiB/s) | 20260505 15:31:45.041 | INFO | Downloaded 38GiB to GPU in 2.2714s (16.7298GiB/s) | 20260505 15:31:50.178 | INFO | Analyzing chunk frames = [4305408,4305920) | 20260505 15:31:50.178 | INFO | MemInfo: { | 20260505 15:31:50.178 | INFO | "gpu" : " 17 GB", | 20260505 15:31:50.178 | INFO | "huge" : "", | 20260505 15:31:50.178 | INFO | "huge_shared" : " 27 GB", | 20260505 15:31:50.178 | INFO | "malloc" : " 14 GB", | 20260505 15:31:50.178 | INFO | "pinned" : " 29 GB" | 20260505 15:31:50.178 | INFO | } | 20260505 15:31:50.178 | INFO | Perf report for the last 5.13682 seconds: | 20260505 15:31:50.178 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:31:50.178 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:50.178 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:50.178 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:50.178 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 15:31:50.178 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:50.178 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:50.178 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 15:31:50.178 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:50.178 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:50.178 | INFO | Uploaded 36GiB to GPU in 2.19276s (16.4177GiB/s) | 20260505 15:31:50.178 | INFO | Downloaded 34GiB to GPU in 2.03757s (16.6865GiB/s) | 20260505 15:31:55.326 | INFO | Analyzing chunk frames = [4305920,4306432) | 20260505 15:31:55.326 | INFO | MemInfo: { | 20260505 15:31:55.326 | INFO | "gpu" : " 17 GB", | 20260505 15:31:55.326 | INFO | "huge" : "", | 20260505 15:31:55.326 | INFO | "huge_shared" : " 27 GB", | 20260505 15:31:55.326 | INFO | "malloc" : " 14 GB", | 20260505 15:31:55.326 | INFO | "pinned" : " 29 GB" | 20260505 15:31:55.326 | INFO | } | 20260505 15:31:55.326 | INFO | Perf report for the last 5.1479 seconds: | 20260505 15:31:55.326 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:31:55.326 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:55.326 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:55.326 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:55.326 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 15:31:55.326 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:55.326 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:55.326 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 15:31:55.326 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:55.326 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:31:55.326 | INFO | Uploaded 36GiB to GPU in 2.19401s (16.4083GiB/s) | 20260505 15:31:55.326 | INFO | Downloaded 34GiB to GPU in 2.07119s (16.4157GiB/s) | 20260505 15:32:00.473 | INFO | Analyzing chunk frames = [4306432,4306944) | 20260505 15:32:00.473 | INFO | MemInfo: { | 20260505 15:32:00.473 | INFO | "gpu" : " 17 GB", | 20260505 15:32:00.473 | INFO | "huge" : "", | 20260505 15:32:00.473 | INFO | "huge_shared" : " 27 GB", | 20260505 15:32:00.473 | INFO | "malloc" : " 14 GB", | 20260505 15:32:00.473 | INFO | "pinned" : " 29 GB" | 20260505 15:32:00.473 | INFO | } | 20260505 15:32:00.473 | INFO | Perf report for the last 5.14745 seconds: | 20260505 15:32:00.473 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:32:00.473 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:00.473 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:00.473 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:00.473 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260505 15:32:00.473 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:00.473 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:00.473 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 15:32:00.473 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:00.473 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:00.473 | INFO | Uploaded 36GiB to GPU in 2.1922s (16.4219GiB/s) | 20260505 15:32:00.473 | INFO | Downloaded 34GiB to GPU in 2.13366s (15.9351GiB/s) | 20260505 15:32:05.603 | INFO | Analyzing chunk frames = [4306944,4307456) | 20260505 15:32:05.603 | INFO | MemInfo: { | 20260505 15:32:05.603 | INFO | "gpu" : " 17 GB", | 20260505 15:32:05.603 | INFO | "huge" : "", | 20260505 15:32:05.603 | INFO | "huge_shared" : " 27 GB", | 20260505 15:32:05.603 | INFO | "malloc" : " 14 GB", | 20260505 15:32:05.603 | INFO | "pinned" : " 29 GB" | 20260505 15:32:05.603 | INFO | } | 20260505 15:32:05.603 | INFO | Perf report for the last 5.13032 seconds: | 20260505 15:32:05.603 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:32:05.603 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:05.604 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:05.604 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:05.604 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260505 15:32:05.604 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:05.604 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:05.604 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:32:05.604 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:05.604 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:05.604 | INFO | Uploaded 36GiB to GPU in 2.14273s (16.801GiB/s) | 20260505 15:32:05.604 | INFO | Downloaded 34GiB to GPU in 2.05947s (16.5091GiB/s) | 20260505 15:32:10.739 | INFO | Analyzing chunk frames = [4307456,4307968) | 20260505 15:32:10.739 | INFO | MemInfo: { | 20260505 15:32:10.739 | INFO | "gpu" : " 17 GB", | 20260505 15:32:10.739 | INFO | "huge" : "", | 20260505 15:32:10.739 | INFO | "huge_shared" : " 27 GB", | 20260505 15:32:10.739 | INFO | "malloc" : " 14 GB", | 20260505 15:32:10.739 | INFO | "pinned" : " 29 GB" | 20260505 15:32:10.739 | INFO | } | 20260505 15:32:10.739 | INFO | Perf report for the last 5.1358 seconds: | 20260505 15:32:10.739 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:32:10.739 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:10.739 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:10.739 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:10.739 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 15:32:10.739 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:10.739 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:10.739 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 15:32:10.739 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:10.739 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:10.740 | INFO | Uploaded 36GiB to GPU in 2.14518s (16.7818GiB/s) | 20260505 15:32:10.740 | INFO | Downloaded 34GiB to GPU in 2.03146s (16.7367GiB/s) | 20260505 15:32:15.881 | INFO | Analyzing chunk frames = [4307968,4308480) | 20260505 15:32:15.881 | INFO | MemInfo: { | 20260505 15:32:15.881 | INFO | "gpu" : " 17 GB", | 20260505 15:32:15.881 | INFO | "huge" : "", | 20260505 15:32:15.881 | INFO | "huge_shared" : " 27 GB", | 20260505 15:32:15.881 | INFO | "malloc" : " 14 GB", | 20260505 15:32:15.881 | INFO | "pinned" : " 29 GB" | 20260505 15:32:15.881 | INFO | } | 20260505 15:32:15.881 | INFO | Perf report for the last 5.14187 seconds: | 20260505 15:32:15.881 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:32:15.881 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:15.881 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:15.881 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:15.881 | INFO | ANALYSIS : 63.2%, 3.0, 98.8ms, 0 | 20260505 15:32:15.881 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:15.881 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:15.881 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 15:32:15.881 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:15.881 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:15.882 | INFO | Uploaded 36GiB to GPU in 2.11214s (17.0443GiB/s) | 20260505 15:32:15.882 | INFO | Downloaded 34GiB to GPU in 2.1714s (15.6581GiB/s) | 20260505 15:32:21.019 | INFO | Analyzing chunk frames = [4308480,4308992) | 20260505 15:32:21.019 | INFO | MemInfo: { | 20260505 15:32:21.019 | INFO | "gpu" : " 17 GB", | 20260505 15:32:21.019 | INFO | "huge" : "", | 20260505 15:32:21.019 | INFO | "huge_shared" : " 27 GB", | 20260505 15:32:21.019 | INFO | "malloc" : " 14 GB", | 20260505 15:32:21.019 | INFO | "pinned" : " 29 GB" | 20260505 15:32:21.019 | INFO | } | 20260505 15:32:21.019 | INFO | Perf report for the last 5.13786 seconds: | 20260505 15:32:21.019 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:32:21.019 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:21.019 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:21.019 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:21.019 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 15:32:21.019 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:21.019 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:21.019 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 15:32:21.019 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:21.019 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:21.019 | INFO | Uploaded 36GiB to GPU in 2.22603s (16.1723GiB/s) | 20260505 15:32:21.019 | INFO | Downloaded 34GiB to GPU in 2.10529s (16.1498GiB/s) | 20260505 15:32:21.020 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 15:32:21.020 | INFO | Total mem usage: 15.3 GiB | 20260505 15:32:21.020 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 15:32:21.020 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.83%) | 20260505 15:32:21.020 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 15:32:21.020 | INFO | High Water Breakdown: | 20260505 15:32:21.020 | INFO | 61% : 8.68 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 15:32:21.020 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 15:32:21.020 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 15:32:21.020 | INFO | ------------------------------------------ | 20260505 15:32:21.020 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:32:21.020 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 15:32:21.020 | INFO | Total mem usage: 26.5 GiB | 20260505 15:32:21.020 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 15:32:21.020 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 15:32:21.020 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 15:32:21.020 | INFO | High Water Breakdown: | 20260505 15:32:21.020 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:32:21.020 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:32:21.020 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:32:21.020 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:32:21.020 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:32:21.020 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:32:21.020 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 15:32:21.020 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 15:32:21.020 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:32:21.020 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 15:32:21.020 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 15:32:21.020 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 15:32:21.020 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 15:32:21.020 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 15:32:21.020 | INFO | ------------------------------------------ | 20260505 15:32:21.020 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:32:21.020 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 15:32:21.020 | INFO | Total mem usage: 25.6 GiB | 20260505 15:32:21.020 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 15:32:21.020 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 15:32:21.020 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 15:32:21.020 | INFO | High Water Breakdown: | 20260505 15:32:21.020 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 15:32:21.020 | INFO | ------------------------------------------ | 20260505 15:32:21.020 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:32:21.020 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 15:32:21.020 | INFO | Total mem usage: 15.7 GiB | 20260505 15:32:21.020 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 15:32:21.020 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 15:32:21.020 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260505 15:32:21.020 | INFO | High Water Breakdown: | 20260505 15:32:21.020 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 15:32:21.020 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 15:32:21.020 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 15:32:21.020 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 15:32:21.020 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 15:32:21.020 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 15:32:21.020 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 15:32:21.020 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 15:32:21.020 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 15:32:21.020 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:32:21.020 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 15:32:21.020 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:32:21.020 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 15:32:21.020 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 15:32:21.020 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 15:32:21.020 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 15:32:21.020 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:32:21.020 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 15:32:21.020 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:32:21.020 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:32:21.020 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:32:21.020 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 15:32:21.020 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 15:32:21.020 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:32:21.020 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 15:32:21.020 | INFO | ------------------------------------------ | 20260505 15:32:21.020 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:32:21.020 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 15:32:21.020 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:32:21.020 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:32:21.020 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.54e+01, 1.36e+01 / 2.13e+00 | 20260505 15:32:21.020 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 1.02e-01, 1.09e-02 / 8.27e-03 | 20260505 15:32:21.020 | INFO | PerfReport: 11.78%, Baseline -- 3.73e+00 / 4.46e+00, 5.75e-01 / 1.98e-01 | 20260505 15:32:21.020 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.48e+00, 1.92e-01 / 1.15e-01 | 20260505 15:32:21.020 | INFO | PerfReport: 15.92%, DME -- 4.04e+01 / 4.47e+01, 4.17e+00 / 2.03e+00 | 20260505 15:32:21.020 | INFO | PerfReport: 56.60%, FrameLabeling -- 1.79e+01 / 1.98e+01, 1.40e+00 / 1.69e+00 | 20260505 15:32:21.020 | INFO | PerfReport: 7.43%, PulseAccumulating -- 2.35e+00 / 2.91e+00, 4.50e-01 / 1.58e-01 | 20260505 15:32:21.020 | INFO | PerfReport: 3.48%, Metrics -- 1.10e+00 / 1.42e+00, 3.08e-01 / 1.99e-01 | 20260505 15:32:21.020 | INFO | PerfReport: 1.22%, PulseSerializing -- 3.86e-01 / 4.42e-01, 6.85e-02 / 3.80e-02 | 20260505 15:32:21.020 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 15:32:21.020 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:32:21.020 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:32:21.020 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260505 15:32:21.020 | INFO | PerfReport: 22.57%, Upload -- 2.27e+01 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260505 15:32:21.020 | INFO | PerfReport: 21.92%, Download -- 2.20e+01 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260505 15:32:21.020 | INFO | PerfReport: 31.56%, Compute -- 3.17e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 15:32:21.020 | INFO | PerfReport: 2.25%, LockUpload -- 2.26e+00 / 0.00e+00, 8.03e+00 / 0.00e+00 | 20260505 15:32:21.020 | INFO | PerfReport: 1.48%, LockDownload -- 1.49e+00 / 0.00e+00, 3.04e+00 / 0.00e+00 | 20260505 15:32:21.020 | INFO | PerfReport: 20.20%, LockCompute -- 2.03e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 15:32:21.020 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.08e-02 / 0.00e+00 | 20260505 15:32:21.150 | INFO | Starting a round of Baz IO | 20260505 15:32:24.360 | INFO | Populating RTMetrics report | 20260505 15:32:24.360 | INFO | RTMetrics report populated | 20260505 15:32:24.360 | INFO | Opening tmp RTMetrics json file | 20260505 15:32:24.375 | INFO | Serializing RTMetrics to json | 20260505 15:32:24.379 | INFO | Serialization done, swapping tmp file now... | 20260505 15:32:24.386 | INFO | RTMetrics done | 20260505 15:32:24.460 | INFO | Finished a round of Baz IO: Wrote 2.212 GiB in 3.309 seconds | 20260505 15:32:24.460 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.45% / 0.00% / 77.55% | 20260505 15:32:24.460 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.441877% / 3.97364e-06% / 9.13938e-05% | 20260505 15:32:24.460 | INFO | Baz IO overall: | 20260505 15:32:24.460 | INFO | BytesWritten : 2563008927981 | 20260505 15:32:24.460 | INFO | NumEvents : 1217508293651 | 20260505 15:32:26.161 | INFO | Analyzing chunk frames = [4308992,4309504) | 20260505 15:32:26.161 | INFO | MemInfo: { | 20260505 15:32:26.161 | INFO | "gpu" : " 17 GB", | 20260505 15:32:26.161 | INFO | "huge" : "", | 20260505 15:32:26.161 | INFO | "huge_shared" : " 27 GB", | 20260505 15:32:26.161 | INFO | "malloc" : " 14 GB", | 20260505 15:32:26.161 | INFO | "pinned" : " 29 GB" | 20260505 15:32:26.161 | INFO | } | 20260505 15:32:26.161 | INFO | Perf report for the last 5.14154 seconds: | 20260505 15:32:26.161 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:32:26.161 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:26.161 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:26.161 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:26.161 | INFO | ANALYSIS : 63.9%, 3.0, 99.5ms, 0 | 20260505 15:32:26.161 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:26.161 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260505 15:32:26.161 | INFO | PRE_HQ : 60.7%, 1.8, 56.7ms, 0 | 20260505 15:32:26.161 | INFO | BAZWRITER : 63.7%, 3.9, 127.8ms, 0 | 20260505 15:32:26.161 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 15:32:26.161 | INFO | Uploaded 36GiB to GPU in 2.17097s (16.5825GiB/s) | 20260505 15:32:26.161 | INFO | Downloaded 38GiB to GPU in 2.25876s (16.8234GiB/s) | 20260505 15:32:31.293 | INFO | Analyzing chunk frames = [4309504,4310016) | 20260505 15:32:31.293 | INFO | MemInfo: { | 20260505 15:32:31.293 | INFO | "gpu" : " 17 GB", | 20260505 15:32:31.293 | INFO | "huge" : "", | 20260505 15:32:31.293 | INFO | "huge_shared" : " 27 GB", | 20260505 15:32:31.293 | INFO | "malloc" : " 14 GB", | 20260505 15:32:31.293 | INFO | "pinned" : " 29 GB" | 20260505 15:32:31.293 | INFO | } | 20260505 15:32:31.293 | INFO | Perf report for the last 5.13272 seconds: | 20260505 15:32:31.293 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:32:31.293 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:31.293 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:31.293 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:31.293 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 15:32:31.293 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:31.293 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:31.293 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:32:31.293 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:31.293 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:31.293 | INFO | Uploaded 36GiB to GPU in 2.20254s (16.3448GiB/s) | 20260505 15:32:31.294 | INFO | Downloaded 34GiB to GPU in 2.06493s (16.4654GiB/s) | 20260505 15:32:36.432 | INFO | Analyzing chunk frames = [4310016,4310528) | 20260505 15:32:36.432 | INFO | MemInfo: { | 20260505 15:32:36.432 | INFO | "gpu" : " 17 GB", | 20260505 15:32:36.432 | INFO | "huge" : "", | 20260505 15:32:36.432 | INFO | "huge_shared" : " 27 GB", | 20260505 15:32:36.432 | INFO | "malloc" : " 14 GB", | 20260505 15:32:36.432 | INFO | "pinned" : " 29 GB" | 20260505 15:32:36.432 | INFO | } | 20260505 15:32:36.432 | INFO | Perf report for the last 5.13832 seconds: | 20260505 15:32:36.432 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:32:36.432 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:36.432 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:36.432 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:36.432 | INFO | ANALYSIS : 63.1%, 3.0, 98.2ms, 0 | 20260505 15:32:36.432 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:36.432 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:36.432 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 15:32:36.432 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:36.432 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:36.432 | INFO | Uploaded 36GiB to GPU in 2.20104s (16.3559GiB/s) | 20260505 15:32:36.432 | INFO | Downloaded 34GiB to GPU in 2.11643s (16.0648GiB/s) | 20260505 15:32:41.583 | INFO | Analyzing chunk frames = [4310528,4311040) | 20260505 15:32:41.583 | INFO | MemInfo: { | 20260505 15:32:41.583 | INFO | "gpu" : " 17 GB", | 20260505 15:32:41.583 | INFO | "huge" : "", | 20260505 15:32:41.583 | INFO | "huge_shared" : " 27 GB", | 20260505 15:32:41.583 | INFO | "malloc" : " 14 GB", | 20260505 15:32:41.583 | INFO | "pinned" : " 29 GB" | 20260505 15:32:41.583 | INFO | } | 20260505 15:32:41.583 | INFO | Perf report for the last 5.15135 seconds: | 20260505 15:32:41.583 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:32:41.583 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:41.583 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:41.583 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:41.583 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 15:32:41.583 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:41.583 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:41.583 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:32:41.583 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:41.583 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:41.583 | INFO | Uploaded 36GiB to GPU in 2.19777s (16.3802GiB/s) | 20260505 15:32:41.583 | INFO | Downloaded 34GiB to GPU in 2.1389s (15.896GiB/s) | 20260505 15:32:46.720 | INFO | Analyzing chunk frames = [4311040,4311552) | 20260505 15:32:46.720 | INFO | MemInfo: { | 20260505 15:32:46.720 | INFO | "gpu" : " 17 GB", | 20260505 15:32:46.720 | INFO | "huge" : "", | 20260505 15:32:46.720 | INFO | "huge_shared" : " 27 GB", | 20260505 15:32:46.720 | INFO | "malloc" : " 14 GB", | 20260505 15:32:46.720 | INFO | "pinned" : " 29 GB" | 20260505 15:32:46.720 | INFO | } | 20260505 15:32:46.721 | INFO | Perf report for the last 5.1375 seconds: | 20260505 15:32:46.721 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:32:46.721 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:46.721 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:46.721 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:46.721 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 15:32:46.721 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:46.721 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:46.721 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 15:32:46.721 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:46.721 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:46.721 | INFO | Uploaded 36GiB to GPU in 2.15508s (16.7047GiB/s) | 20260505 15:32:46.721 | INFO | Downloaded 34GiB to GPU in 2.05779s (16.5226GiB/s) | 20260505 15:32:51.852 | INFO | Analyzing chunk frames = [4311552,4312064) | 20260505 15:32:51.852 | INFO | MemInfo: { | 20260505 15:32:51.852 | INFO | "gpu" : " 17 GB", | 20260505 15:32:51.852 | INFO | "huge" : "", | 20260505 15:32:51.852 | INFO | "huge_shared" : " 27 GB", | 20260505 15:32:51.852 | INFO | "malloc" : " 14 GB", | 20260505 15:32:51.852 | INFO | "pinned" : " 29 GB" | 20260505 15:32:51.852 | INFO | } | 20260505 15:32:51.852 | INFO | Perf report for the last 5.13148 seconds: | 20260505 15:32:51.852 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:32:51.852 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:51.852 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:51.852 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:51.852 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 15:32:51.852 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:51.852 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:51.852 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 15:32:51.852 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:51.852 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:51.852 | INFO | Uploaded 36GiB to GPU in 2.14189s (16.8076GiB/s) | 20260505 15:32:51.852 | INFO | Downloaded 34GiB to GPU in 2.03458s (16.7111GiB/s) | 20260505 15:32:56.985 | INFO | Analyzing chunk frames = [4312064,4312576) | 20260505 15:32:56.985 | INFO | MemInfo: { | 20260505 15:32:56.985 | INFO | "gpu" : " 17 GB", | 20260505 15:32:56.985 | INFO | "huge" : "", | 20260505 15:32:56.985 | INFO | "huge_shared" : " 27 GB", | 20260505 15:32:56.985 | INFO | "malloc" : " 14 GB", | 20260505 15:32:56.985 | INFO | "pinned" : " 29 GB" | 20260505 15:32:56.985 | INFO | } | 20260505 15:32:56.985 | INFO | Perf report for the last 5.13314 seconds: | 20260505 15:32:56.985 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:32:56.985 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:56.985 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:56.985 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:56.985 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 15:32:56.985 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:56.985 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:56.985 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 15:32:56.985 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:56.985 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:32:56.985 | INFO | Uploaded 36GiB to GPU in 2.15928s (16.6722GiB/s) | 20260505 15:32:56.985 | INFO | Downloaded 34GiB to GPU in 2.17193s (15.6543GiB/s) | 20260505 15:33:02.135 | INFO | Analyzing chunk frames = [4312576,4313088) | 20260505 15:33:02.135 | INFO | MemInfo: { | 20260505 15:33:02.135 | INFO | "gpu" : " 17 GB", | 20260505 15:33:02.135 | INFO | "huge" : "", | 20260505 15:33:02.135 | INFO | "huge_shared" : " 27 GB", | 20260505 15:33:02.135 | INFO | "malloc" : " 14 GB", | 20260505 15:33:02.135 | INFO | "pinned" : " 29 GB" | 20260505 15:33:02.135 | INFO | } | 20260505 15:33:02.135 | INFO | Perf report for the last 5.15008 seconds: | 20260505 15:33:02.135 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:33:02.135 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:02.135 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:02.135 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:02.135 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 15:33:02.135 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:02.135 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:02.135 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 15:33:02.135 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:02.135 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:02.136 | INFO | Uploaded 36GiB to GPU in 2.20787s (16.3053GiB/s) | 20260505 15:33:02.136 | INFO | Downloaded 34GiB to GPU in 2.11014s (16.1127GiB/s) | 20260505 15:33:02.265 | INFO | Starting a round of Baz IO | 20260505 15:33:05.482 | INFO | Populating RTMetrics report | 20260505 15:33:05.482 | INFO | RTMetrics report populated | 20260505 15:33:05.482 | INFO | Opening tmp RTMetrics json file | 20260505 15:33:05.493 | INFO | Serializing RTMetrics to json | 20260505 15:33:05.494 | INFO | Serialization done, swapping tmp file now... | 20260505 15:33:05.495 | INFO | RTMetrics done | 20260505 15:33:05.582 | INFO | Finished a round of Baz IO: Wrote 2.211 GiB in 3.317 seconds | 20260505 15:33:05.582 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.45% / 0.00% / 77.55% | 20260505 15:33:05.582 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.439862% / 3.97364e-06% / 6.35783e-05% | 20260505 15:33:05.582 | INFO | Baz IO overall: | 20260505 15:33:05.582 | INFO | BytesWritten : 2565382436217 | 20260505 15:33:05.582 | INFO | NumEvents : 1218624092407 | 20260505 15:33:07.279 | INFO | Analyzing chunk frames = [4313088,4313600) | 20260505 15:33:07.279 | INFO | MemInfo: { | 20260505 15:33:07.279 | INFO | "gpu" : " 17 GB", | 20260505 15:33:07.279 | INFO | "huge" : "", | 20260505 15:33:07.279 | INFO | "huge_shared" : " 27 GB", | 20260505 15:33:07.279 | INFO | "malloc" : " 14 GB", | 20260505 15:33:07.279 | INFO | "pinned" : " 29 GB" | 20260505 15:33:07.279 | INFO | } | 20260505 15:33:07.279 | INFO | Perf report for the last 5.14353 seconds: | 20260505 15:33:07.279 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:33:07.279 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:07.279 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:07.279 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:07.279 | INFO | ANALYSIS : 64.0%, 3.0, 99.6ms, 0 | 20260505 15:33:07.279 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:07.279 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260505 15:33:07.279 | INFO | PRE_HQ : 61.1%, 1.8, 57.0ms, 0 | 20260505 15:33:07.279 | INFO | BAZWRITER : 63.8%, 3.9, 129.2ms, 0 | 20260505 15:33:07.279 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 15:33:07.279 | INFO | Uploaded 36GiB to GPU in 2.19978s (16.3653GiB/s) | 20260505 15:33:07.279 | INFO | Downloaded 38GiB to GPU in 2.25403s (16.8587GiB/s) | 20260505 15:33:12.417 | INFO | Analyzing chunk frames = [4313600,4314112) | 20260505 15:33:12.417 | INFO | MemInfo: { | 20260505 15:33:12.417 | INFO | "gpu" : " 17 GB", | 20260505 15:33:12.417 | INFO | "huge" : "", | 20260505 15:33:12.417 | INFO | "huge_shared" : " 27 GB", | 20260505 15:33:12.417 | INFO | "malloc" : " 14 GB", | 20260505 15:33:12.417 | INFO | "pinned" : " 29 GB" | 20260505 15:33:12.417 | INFO | } | 20260505 15:33:12.417 | INFO | Perf report for the last 5.13853 seconds: | 20260505 15:33:12.417 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:33:12.417 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:12.417 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:12.417 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:12.417 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 15:33:12.417 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:12.417 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:12.417 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 15:33:12.417 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:12.417 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:12.418 | INFO | Uploaded 36GiB to GPU in 2.12708s (16.9246GiB/s) | 20260505 15:33:12.418 | INFO | Downloaded 34GiB to GPU in 2.04064s (16.6615GiB/s) | 20260505 15:33:17.552 | INFO | Analyzing chunk frames = [4314112,4314624) | 20260505 15:33:17.552 | INFO | MemInfo: { | 20260505 15:33:17.552 | INFO | "gpu" : " 17 GB", | 20260505 15:33:17.552 | INFO | "huge" : "", | 20260505 15:33:17.553 | INFO | "huge_shared" : " 27 GB", | 20260505 15:33:17.553 | INFO | "malloc" : " 14 GB", | 20260505 15:33:17.553 | INFO | "pinned" : " 29 GB" | 20260505 15:33:17.553 | INFO | } | 20260505 15:33:17.553 | INFO | Perf report for the last 5.13523 seconds: | 20260505 15:33:17.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:33:17.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:17.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:17.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:17.553 | INFO | ANALYSIS : 62.9%, 3.0, 97.8ms, 0 | 20260505 15:33:17.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:17.553 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:17.553 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 15:33:17.553 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:17.553 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:17.553 | INFO | Uploaded 36GiB to GPU in 2.17742s (16.5333GiB/s) | 20260505 15:33:17.553 | INFO | Downloaded 34GiB to GPU in 2.03664s (16.6942GiB/s) | 20260505 15:33:22.689 | INFO | Analyzing chunk frames = [4314624,4315136) | 20260505 15:33:22.689 | INFO | MemInfo: { | 20260505 15:33:22.689 | INFO | "gpu" : " 17 GB", | 20260505 15:33:22.689 | INFO | "huge" : "", | 20260505 15:33:22.689 | INFO | "huge_shared" : " 27 GB", | 20260505 15:33:22.690 | INFO | "malloc" : " 14 GB", | 20260505 15:33:22.690 | INFO | "pinned" : " 29 GB" | 20260505 15:33:22.690 | INFO | } | 20260505 15:33:22.690 | INFO | Perf report for the last 5.13705 seconds: | 20260505 15:33:22.690 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:33:22.690 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:22.690 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:22.690 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:22.690 | INFO | ANALYSIS : 62.6%, 3.0, 97.1ms, 0 | 20260505 15:33:22.690 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:22.690 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:22.690 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 15:33:22.690 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:22.690 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:22.690 | INFO | Uploaded 36GiB to GPU in 2.18679s (16.4625GiB/s) | 20260505 15:33:22.690 | INFO | Downloaded 34GiB to GPU in 2.13647s (15.9141GiB/s) | 20260505 15:33:27.837 | INFO | Analyzing chunk frames = [4315136,4315648) | 20260505 15:33:27.837 | INFO | MemInfo: { | 20260505 15:33:27.837 | INFO | "gpu" : " 17 GB", | 20260505 15:33:27.837 | INFO | "huge" : "", | 20260505 15:33:27.837 | INFO | "huge_shared" : " 27 GB", | 20260505 15:33:27.837 | INFO | "malloc" : " 14 GB", | 20260505 15:33:27.837 | INFO | "pinned" : " 29 GB" | 20260505 15:33:27.837 | INFO | } | 20260505 15:33:27.837 | INFO | Perf report for the last 5.14753 seconds: | 20260505 15:33:27.837 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:33:27.837 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:27.837 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:27.837 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:27.837 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260505 15:33:27.837 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:27.837 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:27.837 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 15:33:27.837 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:27.837 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:27.837 | INFO | Uploaded 36GiB to GPU in 2.17839s (16.5259GiB/s) | 20260505 15:33:27.838 | INFO | Downloaded 34GiB to GPU in 2.13308s (15.9394GiB/s) | 20260505 15:33:32.978 | INFO | Analyzing chunk frames = [4315648,4316160) | 20260505 15:33:32.978 | INFO | MemInfo: { | 20260505 15:33:32.978 | INFO | "gpu" : " 17 GB", | 20260505 15:33:32.978 | INFO | "huge" : "", | 20260505 15:33:32.978 | INFO | "huge_shared" : " 27 GB", | 20260505 15:33:32.978 | INFO | "malloc" : " 14 GB", | 20260505 15:33:32.978 | INFO | "pinned" : " 29 GB" | 20260505 15:33:32.978 | INFO | } | 20260505 15:33:32.978 | INFO | Perf report for the last 5.14062 seconds: | 20260505 15:33:32.978 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:33:32.978 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:32.978 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:32.978 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:32.978 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260505 15:33:32.978 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:32.978 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:32.978 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:33:32.978 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:32.978 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:32.978 | INFO | Uploaded 36GiB to GPU in 2.13988s (16.8233GiB/s) | 20260505 15:33:32.978 | INFO | Downloaded 34GiB to GPU in 2.04171s (16.6527GiB/s) | 20260505 15:33:38.111 | INFO | Analyzing chunk frames = [4316160,4316672) | 20260505 15:33:38.111 | INFO | MemInfo: { | 20260505 15:33:38.111 | INFO | "gpu" : " 17 GB", | 20260505 15:33:38.111 | INFO | "huge" : "", | 20260505 15:33:38.111 | INFO | "huge_shared" : " 27 GB", | 20260505 15:33:38.111 | INFO | "malloc" : " 14 GB", | 20260505 15:33:38.111 | INFO | "pinned" : " 29 GB" | 20260505 15:33:38.111 | INFO | } | 20260505 15:33:38.111 | INFO | Perf report for the last 5.13342 seconds: | 20260505 15:33:38.111 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:33:38.111 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:38.111 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:38.111 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:38.111 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 15:33:38.111 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:38.111 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:38.111 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 15:33:38.111 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:38.111 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:38.111 | INFO | Uploaded 36GiB to GPU in 2.16454s (16.6317GiB/s) | 20260505 15:33:38.112 | INFO | Downloaded 34GiB to GPU in 2.19698s (15.4758GiB/s) | 20260505 15:33:43.244 | INFO | Analyzing chunk frames = [4316672,4317184) | 20260505 15:33:43.244 | INFO | MemInfo: { | 20260505 15:33:43.244 | INFO | "gpu" : " 17 GB", | 20260505 15:33:43.244 | INFO | "huge" : "", | 20260505 15:33:43.244 | INFO | "huge_shared" : " 27 GB", | 20260505 15:33:43.244 | INFO | "malloc" : " 14 GB", | 20260505 15:33:43.244 | INFO | "pinned" : " 29 GB" | 20260505 15:33:43.244 | INFO | } | 20260505 15:33:43.244 | INFO | Perf report for the last 5.13293 seconds: | 20260505 15:33:43.244 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:33:43.244 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:43.244 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:43.244 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:43.244 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 15:33:43.244 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:43.244 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:43.244 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 15:33:43.244 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:43.244 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:43.244 | INFO | Uploaded 36GiB to GPU in 2.23362s (16.1174GiB/s) | 20260505 15:33:43.245 | INFO | Downloaded 34GiB to GPU in 2.10515s (16.1509GiB/s) | 20260505 15:33:43.245 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 15:33:43.245 | INFO | Total mem usage: 15.3 GiB | 20260505 15:33:43.245 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 15:33:43.245 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.72%) | 20260505 15:33:43.245 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 15:33:43.245 | INFO | High Water Breakdown: | 20260505 15:33:43.245 | INFO | 61% : 8.68 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 15:33:43.245 | INFO | 38.9% : 5.53 GiB ( 83.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 15:33:43.245 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 15:33:43.245 | INFO | ------------------------------------------ | 20260505 15:33:43.245 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:33:43.245 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 15:33:43.245 | INFO | Total mem usage: 26.5 GiB | 20260505 15:33:43.245 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 15:33:43.245 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 15:33:43.245 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 15:33:43.245 | INFO | High Water Breakdown: | 20260505 15:33:43.245 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:33:43.245 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:33:43.245 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:33:43.245 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:33:43.245 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:33:43.245 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:33:43.245 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 15:33:43.245 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 15:33:43.245 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:33:43.245 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 15:33:43.245 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 15:33:43.245 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 15:33:43.245 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 15:33:43.245 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 15:33:43.245 | INFO | ------------------------------------------ | 20260505 15:33:43.245 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:33:43.245 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 15:33:43.245 | INFO | Total mem usage: 25.6 GiB | 20260505 15:33:43.245 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 15:33:43.245 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 15:33:43.245 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 15:33:43.245 | INFO | High Water Breakdown: | 20260505 15:33:43.245 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 15:33:43.245 | INFO | ------------------------------------------ | 20260505 15:33:43.245 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:33:43.245 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 15:33:43.245 | INFO | Total mem usage: 15.7 GiB | 20260505 15:33:43.245 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 15:33:43.245 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.81%) | 20260505 15:33:43.245 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260505 15:33:43.245 | INFO | High Water Breakdown: | 20260505 15:33:43.245 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 15:33:43.245 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 15:33:43.245 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 15:33:43.245 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 15:33:43.245 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 15:33:43.245 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 15:33:43.245 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 15:33:43.245 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 15:33:43.245 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 15:33:43.245 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:33:43.245 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 15:33:43.245 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:33:43.245 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 15:33:43.245 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 15:33:43.245 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 15:33:43.245 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 15:33:43.245 | INFO | 0.635% : 107 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:33:43.245 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 15:33:43.245 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:33:43.245 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:33:43.245 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:33:43.245 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 15:33:43.245 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 15:33:43.245 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:33:43.245 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 15:33:43.245 | INFO | ------------------------------------------ | 20260505 15:33:43.245 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:33:43.245 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 15:33:43.245 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:33:43.245 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:33:43.245 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.59e+01, 1.37e+01 / 1.94e+00 | 20260505 15:33:43.245 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 1.02e-01, 1.06e-02 / 4.13e-03 | 20260505 15:33:43.245 | INFO | PerfReport: 11.79%, Baseline -- 3.74e+00 / 4.48e+00, 5.84e-01 / 2.09e-01 | 20260505 15:33:43.245 | INFO | PerfReport: 3.30%, Histogram -- 1.05e+00 / 1.51e+00, 1.97e-01 / 1.03e-01 | 20260505 15:33:43.245 | INFO | PerfReport: 15.94%, DME -- 4.04e+01 / 4.52e+01, 4.34e+00 / 1.85e+00 | 20260505 15:33:43.245 | INFO | PerfReport: 56.61%, FrameLabeling -- 1.80e+01 / 1.98e+01, 1.43e+00 / 1.75e+00 | 20260505 15:33:43.245 | INFO | PerfReport: 7.39%, PulseAccumulating -- 2.34e+00 / 2.90e+00, 4.48e-01 / 1.89e-01 | 20260505 15:33:43.245 | INFO | PerfReport: 3.46%, Metrics -- 1.10e+00 / 1.42e+00, 3.06e-01 / 1.60e-01 | 20260505 15:33:43.245 | INFO | PerfReport: 1.21%, PulseSerializing -- 3.84e-01 / 4.41e-01, 6.86e-02 / 3.67e-02 | 20260505 15:33:43.245 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 15:33:43.245 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:33:43.245 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:33:43.245 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.56e+01 / 0.00e+00 | 20260505 15:33:43.245 | INFO | PerfReport: 22.59%, Upload -- 2.27e+01 / 0.00e+00, 8.05e+00 / 0.00e+00 | 20260505 15:33:43.245 | INFO | PerfReport: 21.97%, Download -- 2.21e+01 / 0.00e+00, 8.01e+00 / 0.00e+00 | 20260505 15:33:43.245 | INFO | PerfReport: 31.58%, Compute -- 3.17e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 15:33:43.245 | INFO | PerfReport: 2.22%, LockUpload -- 2.23e+00 / 0.00e+00, 7.77e+00 / 0.00e+00 | 20260505 15:33:43.245 | INFO | PerfReport: 1.41%, LockDownload -- 1.42e+00 / 0.00e+00, 2.95e+00 / 0.00e+00 | 20260505 15:33:43.245 | INFO | PerfReport: 20.21%, LockCompute -- 2.03e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 15:33:43.245 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.58e-02 / 0.00e+00 | 20260505 15:33:43.381 | INFO | Starting a round of Baz IO | 20260505 15:33:46.602 | INFO | Populating RTMetrics report | 20260505 15:33:46.602 | INFO | RTMetrics report populated | 20260505 15:33:46.602 | INFO | Opening tmp RTMetrics json file | 20260505 15:33:46.614 | INFO | Serializing RTMetrics to json | 20260505 15:33:46.615 | INFO | Serialization done, swapping tmp file now... | 20260505 15:33:46.617 | INFO | RTMetrics done | 20260505 15:33:46.704 | INFO | Finished a round of Baz IO: Wrote 2.207 GiB in 3.322 seconds | 20260505 15:33:46.704 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.44% / 0.00% / 77.55% | 20260505 15:33:46.704 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.437629% / 0% / 8.34465e-05% | 20260505 15:33:46.704 | INFO | Baz IO overall: | 20260505 15:33:46.704 | INFO | BytesWritten : 2567752100283 | 20260505 15:33:46.704 | INFO | NumEvents : 1219737214765 | 20260505 15:33:48.387 | INFO | Analyzing chunk frames = [4317184,4317696) | 20260505 15:33:48.387 | INFO | MemInfo: { | 20260505 15:33:48.387 | INFO | "gpu" : " 17 GB", | 20260505 15:33:48.387 | INFO | "huge" : "", | 20260505 15:33:48.387 | INFO | "huge_shared" : " 27 GB", | 20260505 15:33:48.387 | INFO | "malloc" : " 14 GB", | 20260505 15:33:48.387 | INFO | "pinned" : " 29 GB" | 20260505 15:33:48.387 | INFO | } | 20260505 15:33:48.387 | INFO | Perf report for the last 5.14322 seconds: | 20260505 15:33:48.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:33:48.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:48.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:48.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:48.388 | INFO | ANALYSIS : 64.2%, 3.0, 99.9ms, 0 | 20260505 15:33:48.388 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:48.388 | INFO | RT_METRICS : 45.3%, 1.0, 23.8ms, 0 | 20260505 15:33:48.388 | INFO | PRE_HQ : 61.0%, 1.8, 57.3ms, 0 | 20260505 15:33:48.388 | INFO | BAZWRITER : 63.9%, 3.9, 128.9ms, 0 | 20260505 15:33:48.388 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260505 15:33:48.388 | INFO | Uploaded 36GiB to GPU in 2.19453s (16.4044GiB/s) | 20260505 15:33:48.388 | INFO | Downloaded 38GiB to GPU in 2.26194s (16.7998GiB/s) | 20260505 15:33:53.529 | INFO | Analyzing chunk frames = [4317696,4318208) | 20260505 15:33:53.529 | INFO | MemInfo: { | 20260505 15:33:53.529 | INFO | "gpu" : " 17 GB", | 20260505 15:33:53.529 | INFO | "huge" : "", | 20260505 15:33:53.529 | INFO | "huge_shared" : " 27 GB", | 20260505 15:33:53.529 | INFO | "malloc" : " 14 GB", | 20260505 15:33:53.529 | INFO | "pinned" : " 29 GB" | 20260505 15:33:53.529 | INFO | } | 20260505 15:33:53.529 | INFO | Perf report for the last 5.14193 seconds: | 20260505 15:33:53.529 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:33:53.529 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:53.529 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:53.529 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:53.529 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 15:33:53.529 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:53.529 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:53.529 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:33:53.529 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:53.530 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:53.530 | INFO | Uploaded 36GiB to GPU in 2.20521s (16.325GiB/s) | 20260505 15:33:53.530 | INFO | Downloaded 34GiB to GPU in 2.06866s (16.4357GiB/s) | 20260505 15:33:58.670 | INFO | Analyzing chunk frames = [4318208,4318720) | 20260505 15:33:58.670 | INFO | MemInfo: { | 20260505 15:33:58.670 | INFO | "gpu" : " 17 GB", | 20260505 15:33:58.670 | INFO | "huge" : "", | 20260505 15:33:58.670 | INFO | "huge_shared" : " 27 GB", | 20260505 15:33:58.670 | INFO | "malloc" : " 14 GB", | 20260505 15:33:58.670 | INFO | "pinned" : " 29 GB" | 20260505 15:33:58.670 | INFO | } | 20260505 15:33:58.670 | INFO | Perf report for the last 5.14044 seconds: | 20260505 15:33:58.670 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:33:58.670 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:58.670 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:58.670 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:58.670 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260505 15:33:58.670 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:58.670 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:58.670 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 15:33:58.670 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:58.670 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:33:58.670 | INFO | Uploaded 36GiB to GPU in 2.20086s (16.3572GiB/s) | 20260505 15:33:58.670 | INFO | Downloaded 34GiB to GPU in 2.03805s (16.6826GiB/s) | 20260505 15:34:03.802 | INFO | Analyzing chunk frames = [4318720,4319232) | 20260505 15:34:03.802 | INFO | MemInfo: { | 20260505 15:34:03.802 | INFO | "gpu" : " 17 GB", | 20260505 15:34:03.802 | INFO | "huge" : "", | 20260505 15:34:03.802 | INFO | "huge_shared" : " 27 GB", | 20260505 15:34:03.802 | INFO | "malloc" : " 14 GB", | 20260505 15:34:03.802 | INFO | "pinned" : " 29 GB" | 20260505 15:34:03.802 | INFO | } | 20260505 15:34:03.802 | INFO | Perf report for the last 5.132 seconds: | 20260505 15:34:03.802 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:34:03.802 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:03.802 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:03.802 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:03.802 | INFO | ANALYSIS : 62.5%, 3.0, 97.0ms, 0 | 20260505 15:34:03.802 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:03.802 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:03.802 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:34:03.802 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:03.802 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:03.802 | INFO | Uploaded 36GiB to GPU in 2.18162s (16.5015GiB/s) | 20260505 15:34:03.802 | INFO | Downloaded 34GiB to GPU in 2.13197s (15.9477GiB/s) | 20260505 15:34:08.938 | INFO | Analyzing chunk frames = [4319232,4319744) | 20260505 15:34:08.938 | INFO | MemInfo: { | 20260505 15:34:08.938 | INFO | "gpu" : " 17 GB", | 20260505 15:34:08.938 | INFO | "huge" : "", | 20260505 15:34:08.938 | INFO | "huge_shared" : " 27 GB", | 20260505 15:34:08.938 | INFO | "malloc" : " 14 GB", | 20260505 15:34:08.938 | INFO | "pinned" : " 29 GB" | 20260505 15:34:08.938 | INFO | } | 20260505 15:34:08.938 | INFO | Perf report for the last 5.13609 seconds: | 20260505 15:34:08.938 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:34:08.938 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:08.938 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:08.938 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:08.938 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 15:34:08.938 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:08.938 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:08.938 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 15:34:08.938 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:08.938 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:08.938 | INFO | Uploaded 36GiB to GPU in 2.18312s (16.4902GiB/s) | 20260505 15:34:08.938 | INFO | Downloaded 34GiB to GPU in 2.12822s (15.9758GiB/s) | 20260505 15:34:14.086 | INFO | Analyzing chunk frames = [4319744,4320256) | 20260505 15:34:14.086 | INFO | MemInfo: { | 20260505 15:34:14.086 | INFO | "gpu" : " 17 GB", | 20260505 15:34:14.086 | INFO | "huge" : "", | 20260505 15:34:14.086 | INFO | "huge_shared" : " 27 GB", | 20260505 15:34:14.086 | INFO | "malloc" : " 14 GB", | 20260505 15:34:14.086 | INFO | "pinned" : " 29 GB" | 20260505 15:34:14.086 | INFO | } | 20260505 15:34:14.086 | INFO | Perf report for the last 5.14848 seconds: | 20260505 15:34:14.086 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:34:14.087 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:14.087 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:14.087 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:14.087 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 15:34:14.087 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:14.087 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:14.087 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 15:34:14.087 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:14.087 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:14.087 | INFO | Uploaded 36GiB to GPU in 2.16883s (16.5988GiB/s) | 20260505 15:34:14.087 | INFO | Downloaded 34GiB to GPU in 2.04959s (16.5887GiB/s) | 20260505 15:34:19.239 | INFO | Analyzing chunk frames = [4320256,4320768) | 20260505 15:34:19.239 | INFO | MemInfo: { | 20260505 15:34:19.239 | INFO | "gpu" : " 17 GB", | 20260505 15:34:19.239 | INFO | "huge" : "", | 20260505 15:34:19.239 | INFO | "huge_shared" : " 27 GB", | 20260505 15:34:19.239 | INFO | "malloc" : " 14 GB", | 20260505 15:34:19.239 | INFO | "pinned" : " 29 GB" | 20260505 15:34:19.239 | INFO | } | 20260505 15:34:19.239 | INFO | Perf report for the last 5.1522 seconds: | 20260505 15:34:19.239 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:34:19.239 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:19.239 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:19.239 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:19.239 | INFO | ANALYSIS : 63.5%, 3.0, 99.3ms, 0 | 20260505 15:34:19.239 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:19.239 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:19.239 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 15:34:19.239 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:19.239 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:19.239 | INFO | Uploaded 36GiB to GPU in 2.1127s (17.0398GiB/s) | 20260505 15:34:19.239 | INFO | Downloaded 34GiB to GPU in 2.17326s (15.6447GiB/s) | 20260505 15:34:24.366 | INFO | Analyzing chunk frames = [4320768,4321280) | 20260505 15:34:24.366 | INFO | MemInfo: { | 20260505 15:34:24.366 | INFO | "gpu" : " 17 GB", | 20260505 15:34:24.366 | INFO | "huge" : "", | 20260505 15:34:24.366 | INFO | "huge_shared" : " 27 GB", | 20260505 15:34:24.366 | INFO | "malloc" : " 14 GB", | 20260505 15:34:24.366 | INFO | "pinned" : " 29 GB" | 20260505 15:34:24.366 | INFO | } | 20260505 15:34:24.366 | INFO | Perf report for the last 5.12711 seconds: | 20260505 15:34:24.366 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:34:24.366 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:24.366 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:24.366 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:24.366 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 15:34:24.366 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:24.366 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:24.366 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 15:34:24.366 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:24.366 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:24.366 | INFO | Uploaded 36GiB to GPU in 2.16118s (16.6576GiB/s) | 20260505 15:34:24.366 | INFO | Downloaded 34GiB to GPU in 2.01771s (16.8508GiB/s) | 20260505 15:34:24.499 | INFO | Starting a round of Baz IO | 20260505 15:34:27.705 | INFO | Populating RTMetrics report | 20260505 15:34:27.705 | INFO | RTMetrics report populated | 20260505 15:34:27.705 | INFO | Opening tmp RTMetrics json file | 20260505 15:34:27.713 | INFO | Serializing RTMetrics to json | 20260505 15:34:27.714 | INFO | Serialization done, swapping tmp file now... | 20260505 15:34:27.715 | INFO | RTMetrics done | 20260505 15:34:27.805 | INFO | Finished a round of Baz IO: Wrote 2.208 GiB in 3.306 seconds | 20260505 15:34:27.805 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.44% / 0.00% / 77.56% | 20260505 15:34:27.805 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.436131% / 7.94729e-06% / 6.75519e-05% | 20260505 15:34:27.805 | INFO | Baz IO overall: | 20260505 15:34:27.805 | INFO | BytesWritten : 2570122633563 | 20260505 15:34:27.805 | INFO | NumEvents : 1220850988157 | 20260505 15:34:29.499 | INFO | Analyzing chunk frames = [4321280,4321792) | 20260505 15:34:29.499 | INFO | MemInfo: { | 20260505 15:34:29.499 | INFO | "gpu" : " 17 GB", | 20260505 15:34:29.499 | INFO | "huge" : "", | 20260505 15:34:29.499 | INFO | "huge_shared" : " 27 GB", | 20260505 15:34:29.499 | INFO | "malloc" : " 14 GB", | 20260505 15:34:29.499 | INFO | "pinned" : " 29 GB" | 20260505 15:34:29.499 | INFO | } | 20260505 15:34:29.499 | INFO | Perf report for the last 5.13329 seconds: | 20260505 15:34:29.499 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:34:29.499 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:29.499 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:29.499 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:29.499 | INFO | ANALYSIS : 63.9%, 3.0, 99.3ms, 0 | 20260505 15:34:29.499 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:29.499 | INFO | RT_METRICS : 44.9%, 1.0, 23.5ms, 0 | 20260505 15:34:29.499 | INFO | PRE_HQ : 60.9%, 1.8, 57.0ms, 0 | 20260505 15:34:29.499 | INFO | BAZWRITER : 63.6%, 3.9, 128.5ms, 0 | 20260505 15:34:29.499 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 15:34:29.499 | INFO | Uploaded 36GiB to GPU in 2.16568s (16.623GiB/s) | 20260505 15:34:29.499 | INFO | Downloaded 38GiB to GPU in 2.26305s (16.7915GiB/s) | 20260505 15:34:34.647 | INFO | Analyzing chunk frames = [4321792,4322304) | 20260505 15:34:34.647 | INFO | MemInfo: { | 20260505 15:34:34.647 | INFO | "gpu" : " 17 GB", | 20260505 15:34:34.647 | INFO | "huge" : "", | 20260505 15:34:34.647 | INFO | "huge_shared" : " 27 GB", | 20260505 15:34:34.647 | INFO | "malloc" : " 14 GB", | 20260505 15:34:34.647 | INFO | "pinned" : " 29 GB" | 20260505 15:34:34.647 | INFO | } | 20260505 15:34:34.647 | INFO | Perf report for the last 5.14793 seconds: | 20260505 15:34:34.647 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:34:34.647 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:34.647 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:34.647 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:34.647 | INFO | ANALYSIS : 63.3%, 3.0, 99.0ms, 0 | 20260505 15:34:34.647 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:34.647 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:34.647 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 15:34:34.647 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:34.647 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:34.647 | INFO | Uploaded 36GiB to GPU in 2.19954s (16.3671GiB/s) | 20260505 15:34:34.647 | INFO | Downloaded 34GiB to GPU in 2.05321s (16.5594GiB/s) | 20260505 15:34:39.784 | INFO | Analyzing chunk frames = [4322304,4322816) | 20260505 15:34:39.784 | INFO | MemInfo: { | 20260505 15:34:39.784 | INFO | "gpu" : " 17 GB", | 20260505 15:34:39.784 | INFO | "huge" : "", | 20260505 15:34:39.784 | INFO | "huge_shared" : " 27 GB", | 20260505 15:34:39.784 | INFO | "malloc" : " 14 GB", | 20260505 15:34:39.784 | INFO | "pinned" : " 29 GB" | 20260505 15:34:39.784 | INFO | } | 20260505 15:34:39.784 | INFO | Perf report for the last 5.13669 seconds: | 20260505 15:34:39.784 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:34:39.784 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:39.784 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:39.784 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:39.784 | INFO | ANALYSIS : 62.8%, 3.0, 97.6ms, 0 | 20260505 15:34:39.784 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:39.784 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:39.784 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:34:39.784 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:39.784 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:39.784 | INFO | Uploaded 36GiB to GPU in 2.18067s (16.5087GiB/s) | 20260505 15:34:39.784 | INFO | Downloaded 34GiB to GPU in 2.02669s (16.7762GiB/s) | 20260505 15:34:44.924 | INFO | Analyzing chunk frames = [4322816,4323328) | 20260505 15:34:44.924 | INFO | MemInfo: { | 20260505 15:34:44.924 | INFO | "gpu" : " 17 GB", | 20260505 15:34:44.924 | INFO | "huge" : "", | 20260505 15:34:44.924 | INFO | "huge_shared" : " 27 GB", | 20260505 15:34:44.924 | INFO | "malloc" : " 14 GB", | 20260505 15:34:44.924 | INFO | "pinned" : " 29 GB" | 20260505 15:34:44.924 | INFO | } | 20260505 15:34:44.924 | INFO | Perf report for the last 5.14008 seconds: | 20260505 15:34:44.924 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:34:44.924 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:44.924 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:44.924 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:44.924 | INFO | ANALYSIS : 62.4%, 3.0, 97.0ms, 0 | 20260505 15:34:44.924 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:44.924 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:44.924 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 15:34:44.924 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:44.924 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:44.924 | INFO | Uploaded 36GiB to GPU in 2.16668s (16.6153GiB/s) | 20260505 15:34:44.924 | INFO | Downloaded 34GiB to GPU in 2.13216s (15.9463GiB/s) | 20260505 15:34:50.064 | INFO | Analyzing chunk frames = [4323328,4323840) | 20260505 15:34:50.064 | INFO | MemInfo: { | 20260505 15:34:50.064 | INFO | "gpu" : " 17 GB", | 20260505 15:34:50.064 | INFO | "huge" : "", | 20260505 15:34:50.064 | INFO | "huge_shared" : " 27 GB", | 20260505 15:34:50.064 | INFO | "malloc" : " 14 GB", | 20260505 15:34:50.064 | INFO | "pinned" : " 29 GB" | 20260505 15:34:50.064 | INFO | } | 20260505 15:34:50.064 | INFO | Perf report for the last 5.14064 seconds: | 20260505 15:34:50.064 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:34:50.064 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:50.064 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:50.064 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:50.064 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 15:34:50.065 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:50.065 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:50.065 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 15:34:50.065 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:50.065 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:50.065 | INFO | Uploaded 36GiB to GPU in 2.16893s (16.598GiB/s) | 20260505 15:34:50.065 | INFO | Downloaded 34GiB to GPU in 2.13124s (15.9532GiB/s) | 20260505 15:34:55.194 | INFO | Analyzing chunk frames = [4323840,4324352) | 20260505 15:34:55.194 | INFO | MemInfo: { | 20260505 15:34:55.194 | INFO | "gpu" : " 17 GB", | 20260505 15:34:55.194 | INFO | "huge" : "", | 20260505 15:34:55.194 | INFO | "huge_shared" : " 27 GB", | 20260505 15:34:55.194 | INFO | "malloc" : " 14 GB", | 20260505 15:34:55.194 | INFO | "pinned" : " 29 GB" | 20260505 15:34:55.194 | INFO | } | 20260505 15:34:55.194 | INFO | Perf report for the last 5.12939 seconds: | 20260505 15:34:55.194 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:34:55.194 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:55.194 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:55.194 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:55.194 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 15:34:55.194 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:55.194 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:55.194 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:34:55.194 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:55.194 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:34:55.194 | INFO | Uploaded 36GiB to GPU in 2.2096s (16.2926GiB/s) | 20260505 15:34:55.194 | INFO | Downloaded 34GiB to GPU in 2.11508s (16.075GiB/s) | 20260505 15:35:00.341 | INFO | Analyzing chunk frames = [4324352,4324864) | 20260505 15:35:00.341 | INFO | MemInfo: { | 20260505 15:35:00.341 | INFO | "gpu" : " 17 GB", | 20260505 15:35:00.341 | INFO | "huge" : "", | 20260505 15:35:00.341 | INFO | "huge_shared" : " 27 GB", | 20260505 15:35:00.341 | INFO | "malloc" : " 14 GB", | 20260505 15:35:00.341 | INFO | "pinned" : " 29 GB" | 20260505 15:35:00.341 | INFO | } | 20260505 15:35:00.341 | INFO | Perf report for the last 5.14686 seconds: | 20260505 15:35:00.341 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:35:00.341 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:00.341 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:00.341 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:00.341 | INFO | ANALYSIS : 63.1%, 3.0, 98.7ms, 0 | 20260505 15:35:00.341 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:00.341 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:00.341 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 15:35:00.341 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:00.341 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:00.341 | INFO | Uploaded 36GiB to GPU in 2.18777s (16.4551GiB/s) | 20260505 15:35:00.341 | INFO | Downloaded 34GiB to GPU in 2.14269s (15.8679GiB/s) | 20260505 15:35:05.480 | INFO | Analyzing chunk frames = [4324864,4325376) | 20260505 15:35:05.480 | INFO | MemInfo: { | 20260505 15:35:05.480 | INFO | "gpu" : " 17 GB", | 20260505 15:35:05.480 | INFO | "huge" : "", | 20260505 15:35:05.480 | INFO | "huge_shared" : " 27 GB", | 20260505 15:35:05.480 | INFO | "malloc" : " 14 GB", | 20260505 15:35:05.480 | INFO | "pinned" : " 29 GB" | 20260505 15:35:05.480 | INFO | } | 20260505 15:35:05.480 | INFO | Perf report for the last 5.13924 seconds: | 20260505 15:35:05.480 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:35:05.480 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:05.480 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:05.480 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:05.480 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 15:35:05.480 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:05.480 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:05.480 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 15:35:05.480 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:05.480 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:05.480 | INFO | Uploaded 36GiB to GPU in 2.16251s (16.6473GiB/s) | 20260505 15:35:05.480 | INFO | Downloaded 34GiB to GPU in 2.03075s (16.7426GiB/s) | 20260505 15:35:05.480 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 15:35:05.480 | INFO | Total mem usage: 15.3 GiB | 20260505 15:35:05.480 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 15:35:05.480 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.83%) | 20260505 15:35:05.480 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 15:35:05.480 | INFO | High Water Breakdown: | 20260505 15:35:05.480 | INFO | 61% : 8.68 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 15:35:05.480 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 15:35:05.480 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 15:35:05.480 | INFO | ------------------------------------------ | 20260505 15:35:05.480 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:35:05.480 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 15:35:05.480 | INFO | Total mem usage: 26.5 GiB | 20260505 15:35:05.480 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 15:35:05.480 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 15:35:05.480 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 15:35:05.480 | INFO | High Water Breakdown: | 20260505 15:35:05.480 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:35:05.480 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:35:05.480 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:35:05.480 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:35:05.480 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:35:05.480 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:35:05.480 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 15:35:05.480 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 15:35:05.481 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:35:05.481 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 15:35:05.481 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 15:35:05.481 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 15:35:05.481 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 15:35:05.481 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 15:35:05.481 | INFO | ------------------------------------------ | 20260505 15:35:05.481 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:35:05.481 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 15:35:05.481 | INFO | Total mem usage: 25.6 GiB | 20260505 15:35:05.481 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 15:35:05.481 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 15:35:05.481 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 15:35:05.481 | INFO | High Water Breakdown: | 20260505 15:35:05.481 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 15:35:05.481 | INFO | ------------------------------------------ | 20260505 15:35:05.481 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:35:05.481 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 15:35:05.481 | INFO | Total mem usage: 15.7 GiB | 20260505 15:35:05.481 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 15:35:05.481 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 15:35:05.481 | INFO | Unused Cache: 871 MiB (Waste: 5.43%) | 20260505 15:35:05.481 | INFO | High Water Breakdown: | 20260505 15:35:05.481 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 15:35:05.481 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 15:35:05.481 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 15:35:05.481 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 15:35:05.481 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 15:35:05.481 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 15:35:05.481 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 15:35:05.481 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 15:35:05.481 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 15:35:05.481 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:35:05.481 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 15:35:05.481 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:35:05.481 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 15:35:05.481 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 15:35:05.481 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 15:35:05.481 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 15:35:05.481 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:35:05.481 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 15:35:05.481 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:35:05.481 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:35:05.481 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:35:05.481 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 15:35:05.481 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 15:35:05.481 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:35:05.481 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 15:35:05.481 | INFO | ------------------------------------------ | 20260505 15:35:05.481 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:35:05.481 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 15:35:05.481 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:35:05.481 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:35:05.481 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.53e+01, 1.36e+01 / 2.08e+00 | 20260505 15:35:05.481 | INFO | PerfReport: 0.29%, AggregateStats -- 9.31e-02 / 1.02e-01, 1.20e-02 / 4.77e-03 | 20260505 15:35:05.481 | INFO | PerfReport: 11.77%, Baseline -- 3.73e+00 / 4.46e+00, 5.73e-01 / 2.18e-01 | 20260505 15:35:05.481 | INFO | PerfReport: 3.27%, Histogram -- 1.04e+00 / 1.49e+00, 1.92e-01 / 1.06e-01 | 20260505 15:35:05.481 | INFO | PerfReport: 15.88%, DME -- 4.03e+01 / 4.46e+01, 4.38e+00 / 2.31e+00 | 20260505 15:35:05.481 | INFO | PerfReport: 56.70%, FrameLabeling -- 1.80e+01 / 1.99e+01, 1.49e+00 / 1.69e+00 | 20260505 15:35:05.481 | INFO | PerfReport: 7.42%, PulseAccumulating -- 2.35e+00 / 2.90e+00, 4.53e-01 / 1.89e-01 | 20260505 15:35:05.481 | INFO | PerfReport: 3.46%, Metrics -- 1.10e+00 / 1.43e+00, 3.03e-01 / 1.62e-01 | 20260505 15:35:05.481 | INFO | PerfReport: 1.21%, PulseSerializing -- 3.83e-01 / 4.34e-01, 6.76e-02 / 3.70e-02 | 20260505 15:35:05.481 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 15:35:05.481 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:35:05.481 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:35:05.481 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260505 15:35:05.481 | INFO | PerfReport: 22.59%, Upload -- 2.27e+01 / 0.00e+00, 8.01e+00 / 0.00e+00 | 20260505 15:35:05.481 | INFO | PerfReport: 21.89%, Download -- 2.20e+01 / 0.00e+00, 8.04e+00 / 0.00e+00 | 20260505 15:35:05.481 | INFO | PerfReport: 31.58%, Compute -- 3.17e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 15:35:05.481 | INFO | PerfReport: 2.26%, LockUpload -- 2.27e+00 / 0.00e+00, 7.94e+00 / 0.00e+00 | 20260505 15:35:05.481 | INFO | PerfReport: 1.45%, LockDownload -- 1.46e+00 / 0.00e+00, 3.02e+00 / 0.00e+00 | 20260505 15:35:05.481 | INFO | PerfReport: 20.21%, LockCompute -- 2.03e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 15:35:05.481 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.66e-02 / 0.00e+00 | 20260505 15:35:05.614 | INFO | Starting a round of Baz IO | 20260505 15:35:08.824 | INFO | Populating RTMetrics report | 20260505 15:35:08.824 | INFO | RTMetrics report populated | 20260505 15:35:08.824 | INFO | Opening tmp RTMetrics json file | 20260505 15:35:08.832 | INFO | Serializing RTMetrics to json | 20260505 15:35:08.836 | INFO | Serialization done, swapping tmp file now... | 20260505 15:35:08.839 | INFO | RTMetrics done | 20260505 15:35:08.917 | INFO | Finished a round of Baz IO: Wrote 2.209 GiB in 3.303 seconds | 20260505 15:35:08.917 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.44% / 0.00% / 77.56% | 20260505 15:35:08.917 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.436795% / 0% / 5.96046e-05% | 20260505 15:35:08.917 | INFO | Baz IO overall: | 20260505 15:35:08.917 | INFO | BytesWritten : 2572494127396 | 20260505 15:35:08.917 | INFO | NumEvents : 1221965678122 | 20260505 15:35:10.614 | INFO | Analyzing chunk frames = [4325376,4325888) | 20260505 15:35:10.614 | INFO | MemInfo: { | 20260505 15:35:10.614 | INFO | "gpu" : " 17 GB", | 20260505 15:35:10.614 | INFO | "huge" : "", | 20260505 15:35:10.614 | INFO | "huge_shared" : " 27 GB", | 20260505 15:35:10.614 | INFO | "malloc" : " 14 GB", | 20260505 15:35:10.614 | INFO | "pinned" : " 29 GB" | 20260505 15:35:10.614 | INFO | } | 20260505 15:35:10.614 | INFO | Perf report for the last 5.13395 seconds: | 20260505 15:35:10.614 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:35:10.614 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:10.614 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:10.614 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:10.614 | INFO | ANALYSIS : 64.0%, 3.0, 99.5ms, 0 | 20260505 15:35:10.614 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:10.614 | INFO | RT_METRICS : 45.1%, 1.0, 23.6ms, 0 | 20260505 15:35:10.614 | INFO | PRE_HQ : 61.0%, 1.8, 57.2ms, 0 | 20260505 15:35:10.614 | INFO | BAZWRITER : 63.7%, 3.9, 128.7ms, 0 | 20260505 15:35:10.614 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 15:35:10.614 | INFO | Uploaded 36GiB to GPU in 2.18823s (16.4516GiB/s) | 20260505 15:35:10.614 | INFO | Downloaded 38GiB to GPU in 2.27134s (16.7302GiB/s) | 20260505 15:35:15.747 | INFO | Analyzing chunk frames = [4325888,4326400) | 20260505 15:35:15.747 | INFO | MemInfo: { | 20260505 15:35:15.747 | INFO | "gpu" : " 17 GB", | 20260505 15:35:15.747 | INFO | "huge" : "", | 20260505 15:35:15.747 | INFO | "huge_shared" : " 27 GB", | 20260505 15:35:15.747 | INFO | "malloc" : " 14 GB", | 20260505 15:35:15.747 | INFO | "pinned" : " 29 GB" | 20260505 15:35:15.747 | INFO | } | 20260505 15:35:15.747 | INFO | Perf report for the last 5.13294 seconds: | 20260505 15:35:15.747 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:35:15.747 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:15.747 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:15.747 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:15.747 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 15:35:15.747 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:15.747 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:15.747 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:35:15.747 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:15.747 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:15.747 | INFO | Uploaded 36GiB to GPU in 2.19172s (16.4255GiB/s) | 20260505 15:35:15.747 | INFO | Downloaded 34GiB to GPU in 2.03011s (16.7478GiB/s) | 20260505 15:35:20.898 | INFO | Analyzing chunk frames = [4326400,4326912) | 20260505 15:35:20.898 | INFO | MemInfo: { | 20260505 15:35:20.898 | INFO | "gpu" : " 17 GB", | 20260505 15:35:20.898 | INFO | "huge" : "", | 20260505 15:35:20.898 | INFO | "huge_shared" : " 27 GB", | 20260505 15:35:20.898 | INFO | "malloc" : " 14 GB", | 20260505 15:35:20.898 | INFO | "pinned" : " 29 GB" | 20260505 15:35:20.898 | INFO | } | 20260505 15:35:20.898 | INFO | Perf report for the last 5.15103 seconds: | 20260505 15:35:20.898 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:35:20.898 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:20.898 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:20.898 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:20.898 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 15:35:20.898 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:20.898 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:20.898 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:35:20.898 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:20.898 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:20.898 | INFO | Uploaded 36GiB to GPU in 2.22595s (16.1729GiB/s) | 20260505 15:35:20.898 | INFO | Downloaded 34GiB to GPU in 2.1149s (16.0764GiB/s) | 20260505 15:35:26.035 | INFO | Analyzing chunk frames = [4326912,4327424) | 20260505 15:35:26.035 | INFO | MemInfo: { | 20260505 15:35:26.035 | INFO | "gpu" : " 17 GB", | 20260505 15:35:26.035 | INFO | "huge" : "", | 20260505 15:35:26.035 | INFO | "huge_shared" : " 27 GB", | 20260505 15:35:26.036 | INFO | "malloc" : " 14 GB", | 20260505 15:35:26.036 | INFO | "pinned" : " 29 GB" | 20260505 15:35:26.036 | INFO | } | 20260505 15:35:26.036 | INFO | Perf report for the last 5.13761 seconds: | 20260505 15:35:26.036 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:35:26.036 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:26.036 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:26.036 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:26.036 | INFO | ANALYSIS : 62.6%, 3.0, 97.2ms, 0 | 20260505 15:35:26.036 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:26.036 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:26.036 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 15:35:26.036 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:26.036 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:26.036 | INFO | Uploaded 36GiB to GPU in 2.18408s (16.4829GiB/s) | 20260505 15:35:26.036 | INFO | Downloaded 34GiB to GPU in 2.13609s (15.917GiB/s) | 20260505 15:35:31.179 | INFO | Analyzing chunk frames = [4327424,4327936) | 20260505 15:35:31.179 | INFO | MemInfo: { | 20260505 15:35:31.179 | INFO | "gpu" : " 17 GB", | 20260505 15:35:31.179 | INFO | "huge" : "", | 20260505 15:35:31.179 | INFO | "huge_shared" : " 27 GB", | 20260505 15:35:31.179 | INFO | "malloc" : " 14 GB", | 20260505 15:35:31.179 | INFO | "pinned" : " 29 GB" | 20260505 15:35:31.179 | INFO | } | 20260505 15:35:31.179 | INFO | Perf report for the last 5.14324 seconds: | 20260505 15:35:31.179 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:35:31.179 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:31.179 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:31.179 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:31.179 | INFO | ANALYSIS : 62.6%, 3.0, 97.9ms, 0 | 20260505 15:35:31.179 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:31.179 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:31.179 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 15:35:31.179 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:31.179 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:31.179 | INFO | Uploaded 36GiB to GPU in 2.18423s (16.4818GiB/s) | 20260505 15:35:31.179 | INFO | Downloaded 34GiB to GPU in 2.12752s (15.9811GiB/s) | 20260505 15:35:36.316 | INFO | Analyzing chunk frames = [4327936,4328448) | 20260505 15:35:36.316 | INFO | MemInfo: { | 20260505 15:35:36.316 | INFO | "gpu" : " 17 GB", | 20260505 15:35:36.316 | INFO | "huge" : "", | 20260505 15:35:36.316 | INFO | "huge_shared" : " 27 GB", | 20260505 15:35:36.316 | INFO | "malloc" : " 14 GB", | 20260505 15:35:36.316 | INFO | "pinned" : " 29 GB" | 20260505 15:35:36.316 | INFO | } | 20260505 15:35:36.316 | INFO | Perf report for the last 5.13713 seconds: | 20260505 15:35:36.316 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:35:36.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:36.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:36.316 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:36.316 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 15:35:36.316 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:36.316 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:36.316 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 15:35:36.316 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:36.316 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:36.316 | INFO | Uploaded 36GiB to GPU in 2.14799s (16.7599GiB/s) | 20260505 15:35:36.316 | INFO | Downloaded 34GiB to GPU in 2.03565s (16.7023GiB/s) | 20260505 15:35:41.452 | INFO | Analyzing chunk frames = [4328448,4328960) | 20260505 15:35:41.452 | INFO | MemInfo: { | 20260505 15:35:41.452 | INFO | "gpu" : " 17 GB", | 20260505 15:35:41.452 | INFO | "huge" : "", | 20260505 15:35:41.452 | INFO | "huge_shared" : " 27 GB", | 20260505 15:35:41.452 | INFO | "malloc" : " 14 GB", | 20260505 15:35:41.452 | INFO | "pinned" : " 29 GB" | 20260505 15:35:41.452 | INFO | } | 20260505 15:35:41.452 | INFO | Perf report for the last 5.13639 seconds: | 20260505 15:35:41.452 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:35:41.452 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:41.452 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:41.452 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:41.452 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 15:35:41.452 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:41.452 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:41.452 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 15:35:41.452 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:41.453 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:41.453 | INFO | Uploaded 36GiB to GPU in 2.1452s (16.7817GiB/s) | 20260505 15:35:41.453 | INFO | Downloaded 34GiB to GPU in 2.15698s (15.7628GiB/s) | 20260505 15:35:46.596 | INFO | Analyzing chunk frames = [4328960,4329472) | 20260505 15:35:46.596 | INFO | MemInfo: { | 20260505 15:35:46.596 | INFO | "gpu" : " 17 GB", | 20260505 15:35:46.596 | INFO | "huge" : "", | 20260505 15:35:46.596 | INFO | "huge_shared" : " 27 GB", | 20260505 15:35:46.596 | INFO | "malloc" : " 14 GB", | 20260505 15:35:46.596 | INFO | "pinned" : " 29 GB" | 20260505 15:35:46.596 | INFO | } | 20260505 15:35:46.596 | INFO | Perf report for the last 5.1433 seconds: | 20260505 15:35:46.596 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:35:46.596 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:46.596 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:46.596 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:46.596 | INFO | ANALYSIS : 63.5%, 3.0, 98.2ms, 0 | 20260505 15:35:46.596 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:46.596 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:46.596 | INFO | PRE_HQ : 28.4%, 1.0, 14.6ms, 0 | 20260505 15:35:46.596 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:46.596 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:46.596 | INFO | Uploaded 36GiB to GPU in 2.22534s (16.1773GiB/s) | 20260505 15:35:46.596 | INFO | Downloaded 34GiB to GPU in 2.10197s (16.1753GiB/s) | 20260505 15:35:46.727 | INFO | Starting a round of Baz IO | 20260505 15:35:49.934 | INFO | Populating RTMetrics report | 20260505 15:35:49.934 | INFO | RTMetrics report populated | 20260505 15:35:49.934 | INFO | Opening tmp RTMetrics json file | 20260505 15:35:49.942 | INFO | Serializing RTMetrics to json | 20260505 15:35:49.945 | INFO | Serialization done, swapping tmp file now... | 20260505 15:35:49.945 | INFO | RTMetrics done | 20260505 15:35:50.032 | INFO | Finished a round of Baz IO: Wrote 2.208 GiB in 3.305 seconds | 20260505 15:35:50.032 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.43% / 0.00% / 77.56% | 20260505 15:35:50.032 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.435309% / 0% / 8.34465e-05% | 20260505 15:35:50.032 | INFO | Baz IO overall: | 20260505 15:35:50.032 | INFO | BytesWritten : 2574864599010 | 20260505 15:35:50.032 | INFO | NumEvents : 1223079610886 | 20260505 15:35:51.738 | INFO | Analyzing chunk frames = [4329472,4329984) | 20260505 15:35:51.738 | INFO | MemInfo: { | 20260505 15:35:51.738 | INFO | "gpu" : " 17 GB", | 20260505 15:35:51.738 | INFO | "huge" : "", | 20260505 15:35:51.738 | INFO | "huge_shared" : " 27 GB", | 20260505 15:35:51.738 | INFO | "malloc" : " 14 GB", | 20260505 15:35:51.738 | INFO | "pinned" : " 29 GB" | 20260505 15:35:51.738 | INFO | } | 20260505 15:35:51.738 | INFO | Perf report for the last 5.14221 seconds: | 20260505 15:35:51.738 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:35:51.738 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:51.738 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:51.738 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:51.738 | INFO | ANALYSIS : 63.8%, 3.0, 99.3ms, 0 | 20260505 15:35:51.738 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:51.738 | INFO | RT_METRICS : 44.9%, 1.0, 23.6ms, 0 | 20260505 15:35:51.738 | INFO | PRE_HQ : 60.8%, 1.8, 56.9ms, 0 | 20260505 15:35:51.738 | INFO | BAZWRITER : 63.6%, 3.9, 128.7ms, 0 | 20260505 15:35:51.738 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 15:35:51.738 | INFO | Uploaded 36GiB to GPU in 2.15628s (16.6955GiB/s) | 20260505 15:35:51.738 | INFO | Downloaded 38GiB to GPU in 2.25592s (16.8445GiB/s) | 20260505 15:35:56.870 | INFO | Analyzing chunk frames = [4329984,4330496) | 20260505 15:35:56.870 | INFO | MemInfo: { | 20260505 15:35:56.870 | INFO | "gpu" : " 17 GB", | 20260505 15:35:56.870 | INFO | "huge" : "", | 20260505 15:35:56.870 | INFO | "huge_shared" : " 27 GB", | 20260505 15:35:56.870 | INFO | "malloc" : " 14 GB", | 20260505 15:35:56.870 | INFO | "pinned" : " 29 GB" | 20260505 15:35:56.870 | INFO | } | 20260505 15:35:56.870 | INFO | Perf report for the last 5.13208 seconds: | 20260505 15:35:56.870 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:35:56.870 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:56.870 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:56.870 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:56.870 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 15:35:56.870 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:56.870 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:56.870 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:35:56.870 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:56.870 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:35:56.870 | INFO | Uploaded 36GiB to GPU in 2.13359s (16.873GiB/s) | 20260505 15:35:56.870 | INFO | Downloaded 34GiB to GPU in 2.03942s (16.6714GiB/s) | 20260505 15:36:02.007 | INFO | Analyzing chunk frames = [4330496,4331008) | 20260505 15:36:02.007 | INFO | MemInfo: { | 20260505 15:36:02.007 | INFO | "gpu" : " 17 GB", | 20260505 15:36:02.007 | INFO | "huge" : "", | 20260505 15:36:02.007 | INFO | "huge_shared" : " 27 GB", | 20260505 15:36:02.007 | INFO | "malloc" : " 14 GB", | 20260505 15:36:02.007 | INFO | "pinned" : " 29 GB" | 20260505 15:36:02.007 | INFO | } | 20260505 15:36:02.007 | INFO | Perf report for the last 5.13698 seconds: | 20260505 15:36:02.007 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:36:02.007 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:02.007 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:02.007 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:02.007 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 15:36:02.007 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:02.007 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:02.007 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:36:02.007 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:02.007 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:02.007 | INFO | Uploaded 36GiB to GPU in 2.2375s (16.0894GiB/s) | 20260505 15:36:02.007 | INFO | Downloaded 34GiB to GPU in 2.11708s (16.0598GiB/s) | 20260505 15:36:07.152 | INFO | Analyzing chunk frames = [4331008,4331520) | 20260505 15:36:07.152 | INFO | MemInfo: { | 20260505 15:36:07.152 | INFO | "gpu" : " 17 GB", | 20260505 15:36:07.152 | INFO | "huge" : "", | 20260505 15:36:07.152 | INFO | "huge_shared" : " 27 GB", | 20260505 15:36:07.152 | INFO | "malloc" : " 14 GB", | 20260505 15:36:07.152 | INFO | "pinned" : " 29 GB" | 20260505 15:36:07.152 | INFO | } | 20260505 15:36:07.153 | INFO | Perf report for the last 5.14552 seconds: | 20260505 15:36:07.153 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:36:07.153 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:07.153 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:07.153 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:07.153 | INFO | ANALYSIS : 62.7%, 3.0, 97.5ms, 0 | 20260505 15:36:07.153 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:07.153 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:07.153 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:36:07.153 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:07.153 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:07.153 | INFO | Uploaded 36GiB to GPU in 2.17242s (16.5714GiB/s) | 20260505 15:36:07.153 | INFO | Downloaded 34GiB to GPU in 2.13698s (15.9103GiB/s) | 20260505 15:36:12.295 | INFO | Analyzing chunk frames = [4331520,4332032) | 20260505 15:36:12.295 | INFO | MemInfo: { | 20260505 15:36:12.295 | INFO | "gpu" : " 17 GB", | 20260505 15:36:12.295 | INFO | "huge" : "", | 20260505 15:36:12.295 | INFO | "huge_shared" : " 27 GB", | 20260505 15:36:12.295 | INFO | "malloc" : " 14 GB", | 20260505 15:36:12.295 | INFO | "pinned" : " 29 GB" | 20260505 15:36:12.295 | INFO | } | 20260505 15:36:12.296 | INFO | Perf report for the last 5.14295 seconds: | 20260505 15:36:12.296 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:36:12.296 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:12.296 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:12.296 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:12.296 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260505 15:36:12.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:12.296 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:12.296 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 15:36:12.296 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:12.296 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:12.296 | INFO | Uploaded 36GiB to GPU in 2.15334s (16.7182GiB/s) | 20260505 15:36:12.296 | INFO | Downloaded 34GiB to GPU in 2.05855s (16.5164GiB/s) | 20260505 15:36:17.433 | INFO | Analyzing chunk frames = [4332032,4332544) | 20260505 15:36:17.433 | INFO | MemInfo: { | 20260505 15:36:17.433 | INFO | "gpu" : " 17 GB", | 20260505 15:36:17.433 | INFO | "huge" : "", | 20260505 15:36:17.433 | INFO | "huge_shared" : " 27 GB", | 20260505 15:36:17.433 | INFO | "malloc" : " 14 GB", | 20260505 15:36:17.433 | INFO | "pinned" : " 29 GB" | 20260505 15:36:17.433 | INFO | } | 20260505 15:36:17.433 | INFO | Perf report for the last 5.13769 seconds: | 20260505 15:36:17.433 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:36:17.433 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:17.433 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:17.433 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:17.433 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260505 15:36:17.433 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:17.433 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:17.433 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:36:17.433 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:17.433 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:17.433 | INFO | Uploaded 36GiB to GPU in 2.13419s (16.8682GiB/s) | 20260505 15:36:17.433 | INFO | Downloaded 34GiB to GPU in 2.03487s (16.7087GiB/s) | 20260505 15:36:22.562 | INFO | Analyzing chunk frames = [4332544,4333056) | 20260505 15:36:22.562 | INFO | MemInfo: { | 20260505 15:36:22.562 | INFO | "gpu" : " 17 GB", | 20260505 15:36:22.562 | INFO | "huge" : "", | 20260505 15:36:22.562 | INFO | "huge_shared" : " 27 GB", | 20260505 15:36:22.562 | INFO | "malloc" : " 14 GB", | 20260505 15:36:22.562 | INFO | "pinned" : " 29 GB" | 20260505 15:36:22.562 | INFO | } | 20260505 15:36:22.562 | INFO | Perf report for the last 5.12907 seconds: | 20260505 15:36:22.562 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:36:22.562 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:22.562 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:22.562 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:22.562 | INFO | ANALYSIS : 63.5%, 3.0, 99.0ms, 0 | 20260505 15:36:22.562 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:22.562 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:22.562 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 15:36:22.562 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:22.562 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:22.562 | INFO | Uploaded 36GiB to GPU in 2.12145s (16.9696GiB/s) | 20260505 15:36:22.563 | INFO | Downloaded 34GiB to GPU in 2.16733s (15.6875GiB/s) | 20260505 15:36:27.703 | INFO | Analyzing chunk frames = [4333056,4333568) | 20260505 15:36:27.703 | INFO | MemInfo: { | 20260505 15:36:27.703 | INFO | "gpu" : " 17 GB", | 20260505 15:36:27.703 | INFO | "huge" : "", | 20260505 15:36:27.703 | INFO | "huge_shared" : " 27 GB", | 20260505 15:36:27.703 | INFO | "malloc" : " 14 GB", | 20260505 15:36:27.703 | INFO | "pinned" : " 29 GB" | 20260505 15:36:27.703 | INFO | } | 20260505 15:36:27.703 | INFO | Perf report for the last 5.14049 seconds: | 20260505 15:36:27.703 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:36:27.703 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:27.703 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:27.703 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:27.703 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 15:36:27.703 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:27.703 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:27.703 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 15:36:27.703 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:27.703 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:27.703 | INFO | Uploaded 36GiB to GPU in 2.20878s (16.2986GiB/s) | 20260505 15:36:27.703 | INFO | Downloaded 34GiB to GPU in 2.09425s (16.2349GiB/s) | 20260505 15:36:27.703 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 15:36:27.703 | INFO | Total mem usage: 15.3 GiB | 20260505 15:36:27.703 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 15:36:27.703 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.86%) | 20260505 15:36:27.703 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 15:36:27.703 | INFO | High Water Breakdown: | 20260505 15:36:27.703 | INFO | 61% : 8.67 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 15:36:27.703 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 15:36:27.703 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 15:36:27.703 | INFO | ------------------------------------------ | 20260505 15:36:27.703 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:36:27.703 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 15:36:27.703 | INFO | Total mem usage: 26.5 GiB | 20260505 15:36:27.703 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 15:36:27.703 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 15:36:27.703 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 15:36:27.703 | INFO | High Water Breakdown: | 20260505 15:36:27.703 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:36:27.703 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:36:27.703 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:36:27.703 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:36:27.703 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:36:27.703 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:36:27.703 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 15:36:27.703 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 15:36:27.703 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:36:27.703 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 15:36:27.703 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 15:36:27.703 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 15:36:27.703 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 15:36:27.703 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 15:36:27.703 | INFO | ------------------------------------------ | 20260505 15:36:27.703 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:36:27.703 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 15:36:27.703 | INFO | Total mem usage: 25.6 GiB | 20260505 15:36:27.703 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 15:36:27.703 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 15:36:27.703 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 15:36:27.703 | INFO | High Water Breakdown: | 20260505 15:36:27.703 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 15:36:27.703 | INFO | ------------------------------------------ | 20260505 15:36:27.703 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:36:27.703 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 15:36:27.703 | INFO | Total mem usage: 15.7 GiB | 20260505 15:36:27.703 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 15:36:27.703 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 15:36:27.703 | INFO | Unused Cache: 871 MiB (Waste: 5.43%) | 20260505 15:36:27.704 | INFO | High Water Breakdown: | 20260505 15:36:27.704 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 15:36:27.704 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 15:36:27.704 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 15:36:27.704 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 15:36:27.704 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 15:36:27.704 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 15:36:27.704 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 15:36:27.704 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 15:36:27.704 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 15:36:27.704 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:36:27.704 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 15:36:27.704 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:36:27.704 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 15:36:27.704 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 15:36:27.704 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 15:36:27.704 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 15:36:27.704 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:36:27.704 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 15:36:27.704 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:36:27.704 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:36:27.704 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:36:27.704 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 15:36:27.704 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 15:36:27.704 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:36:27.704 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 15:36:27.704 | INFO | ------------------------------------------ | 20260505 15:36:27.704 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:36:27.704 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 15:36:27.704 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:36:27.704 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:36:27.704 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.56e+01, 1.37e+01 / 1.81e+00 | 20260505 15:36:27.704 | INFO | PerfReport: 0.29%, AggregateStats -- 9.24e-02 / 1.00e-01, 1.05e-02 / 4.05e-03 | 20260505 15:36:27.704 | INFO | PerfReport: 11.79%, Baseline -- 3.74e+00 / 4.47e+00, 5.85e-01 / 2.47e-01 | 20260505 15:36:27.704 | INFO | PerfReport: 3.27%, Histogram -- 1.04e+00 / 1.49e+00, 1.93e-01 / 1.25e-01 | 20260505 15:36:27.704 | INFO | PerfReport: 15.91%, DME -- 4.04e+01 / 4.48e+01, 4.31e+00 / 1.79e+00 | 20260505 15:36:27.704 | INFO | PerfReport: 56.66%, FrameLabeling -- 1.80e+01 / 2.00e+01, 1.47e+00 / 1.69e+00 | 20260505 15:36:27.704 | INFO | PerfReport: 7.39%, PulseAccumulating -- 2.34e+00 / 2.87e+00, 4.45e-01 / 2.05e-01 | 20260505 15:36:27.704 | INFO | PerfReport: 3.47%, Metrics -- 1.10e+00 / 1.42e+00, 3.04e-01 / 1.35e-01 | 20260505 15:36:27.704 | INFO | PerfReport: 1.21%, PulseSerializing -- 3.83e-01 / 4.36e-01, 6.83e-02 / 3.13e-02 | 20260505 15:36:27.704 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 15:36:27.704 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:36:27.704 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:36:27.704 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260505 15:36:27.704 | INFO | PerfReport: 22.57%, Upload -- 2.27e+01 / 0.00e+00, 8.05e+00 / 0.00e+00 | 20260505 15:36:27.704 | INFO | PerfReport: 21.97%, Download -- 2.21e+01 / 0.00e+00, 8.06e+00 / 0.00e+00 | 20260505 15:36:27.704 | INFO | PerfReport: 31.59%, Compute -- 3.17e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 15:36:27.704 | INFO | PerfReport: 2.28%, LockUpload -- 2.29e+00 / 0.00e+00, 7.83e+00 / 0.00e+00 | 20260505 15:36:27.704 | INFO | PerfReport: 1.39%, LockDownload -- 1.40e+00 / 0.00e+00, 2.96e+00 / 0.00e+00 | 20260505 15:36:27.704 | INFO | PerfReport: 20.19%, LockCompute -- 2.03e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 15:36:27.704 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.83e-02 / 0.00e+00 | 20260505 15:36:27.838 | INFO | Starting a round of Baz IO | 20260505 15:36:31.049 | INFO | Populating RTMetrics report | 20260505 15:36:31.049 | INFO | RTMetrics report populated | 20260505 15:36:31.049 | INFO | Opening tmp RTMetrics json file | 20260505 15:36:31.058 | INFO | Serializing RTMetrics to json | 20260505 15:36:31.060 | INFO | Serialization done, swapping tmp file now... | 20260505 15:36:31.062 | INFO | RTMetrics done | 20260505 15:36:31.144 | INFO | Finished a round of Baz IO: Wrote 2.207 GiB in 3.306 seconds | 20260505 15:36:31.144 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.43% / 0.00% / 77.57% | 20260505 15:36:31.144 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.433926% / 7.94729e-06% / 6.35783e-05% | 20260505 15:36:31.144 | INFO | Baz IO overall: | 20260505 15:36:31.144 | INFO | BytesWritten : 2577234598630 | 20260505 15:36:31.144 | INFO | NumEvents : 1224193676222 | 20260505 15:36:32.849 | INFO | Analyzing chunk frames = [4333568,4334080) | 20260505 15:36:32.849 | INFO | MemInfo: { | 20260505 15:36:32.849 | INFO | "gpu" : " 17 GB", | 20260505 15:36:32.849 | INFO | "huge" : "", | 20260505 15:36:32.849 | INFO | "huge_shared" : " 27 GB", | 20260505 15:36:32.849 | INFO | "malloc" : " 14 GB", | 20260505 15:36:32.849 | INFO | "pinned" : " 29 GB" | 20260505 15:36:32.850 | INFO | } | 20260505 15:36:32.850 | INFO | Perf report for the last 5.14676 seconds: | 20260505 15:36:32.850 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:36:32.850 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:32.850 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:32.850 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:32.850 | INFO | ANALYSIS : 63.9%, 3.0, 99.6ms, 0 | 20260505 15:36:32.850 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:32.850 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260505 15:36:32.850 | INFO | PRE_HQ : 60.8%, 1.8, 57.4ms, 0 | 20260505 15:36:32.850 | INFO | BAZWRITER : 63.6%, 3.9, 128.7ms, 0 | 20260505 15:36:32.850 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 15:36:32.850 | INFO | Uploaded 36GiB to GPU in 2.20776s (16.3061GiB/s) | 20260505 15:36:32.850 | INFO | Downloaded 38GiB to GPU in 2.26832s (16.7525GiB/s) | 20260505 15:36:37.992 | INFO | Analyzing chunk frames = [4334080,4334592) | 20260505 15:36:37.992 | INFO | MemInfo: { | 20260505 15:36:37.992 | INFO | "gpu" : " 17 GB", | 20260505 15:36:37.992 | INFO | "huge" : "", | 20260505 15:36:37.992 | INFO | "huge_shared" : " 27 GB", | 20260505 15:36:37.992 | INFO | "malloc" : " 14 GB", | 20260505 15:36:37.992 | INFO | "pinned" : " 29 GB" | 20260505 15:36:37.992 | INFO | } | 20260505 15:36:37.992 | INFO | Perf report for the last 5.14236 seconds: | 20260505 15:36:37.992 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:36:37.992 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:37.992 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:37.992 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:37.992 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 15:36:37.992 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:37.992 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:37.992 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:36:37.992 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:37.992 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:37.992 | INFO | Uploaded 36GiB to GPU in 2.13164s (16.8884GiB/s) | 20260505 15:36:37.992 | INFO | Downloaded 34GiB to GPU in 2.03334s (16.7213GiB/s) | 20260505 15:36:43.129 | INFO | Analyzing chunk frames = [4334592,4335104) | 20260505 15:36:43.129 | INFO | MemInfo: { | 20260505 15:36:43.129 | INFO | "gpu" : " 17 GB", | 20260505 15:36:43.129 | INFO | "huge" : "", | 20260505 15:36:43.129 | INFO | "huge_shared" : " 27 GB", | 20260505 15:36:43.129 | INFO | "malloc" : " 14 GB", | 20260505 15:36:43.129 | INFO | "pinned" : " 29 GB" | 20260505 15:36:43.129 | INFO | } | 20260505 15:36:43.129 | INFO | Perf report for the last 5.13706 seconds: | 20260505 15:36:43.129 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:36:43.129 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:43.129 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:43.129 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:43.129 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260505 15:36:43.129 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:43.129 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:43.129 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:36:43.129 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:43.129 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:43.129 | INFO | Uploaded 36GiB to GPU in 2.16185s (16.6524GiB/s) | 20260505 15:36:43.129 | INFO | Downloaded 34GiB to GPU in 2.03104s (16.7402GiB/s) | 20260505 15:36:48.261 | INFO | Analyzing chunk frames = [4335104,4335616) | 20260505 15:36:48.261 | INFO | MemInfo: { | 20260505 15:36:48.261 | INFO | "gpu" : " 17 GB", | 20260505 15:36:48.261 | INFO | "huge" : "", | 20260505 15:36:48.261 | INFO | "huge_shared" : " 27 GB", | 20260505 15:36:48.261 | INFO | "malloc" : " 14 GB", | 20260505 15:36:48.261 | INFO | "pinned" : " 29 GB" | 20260505 15:36:48.261 | INFO | } | 20260505 15:36:48.261 | INFO | Perf report for the last 5.13214 seconds: | 20260505 15:36:48.261 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:36:48.261 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:48.261 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:48.261 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:48.261 | INFO | ANALYSIS : 62.6%, 3.0, 97.1ms, 0 | 20260505 15:36:48.261 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:48.261 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:48.261 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 15:36:48.261 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:48.261 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:48.261 | INFO | Uploaded 36GiB to GPU in 2.17458s (16.5549GiB/s) | 20260505 15:36:48.261 | INFO | Downloaded 34GiB to GPU in 2.13085s (15.9561GiB/s) | 20260505 15:36:53.411 | INFO | Analyzing chunk frames = [4335616,4336128) | 20260505 15:36:53.411 | INFO | MemInfo: { | 20260505 15:36:53.411 | INFO | "gpu" : " 17 GB", | 20260505 15:36:53.411 | INFO | "huge" : "", | 20260505 15:36:53.411 | INFO | "huge_shared" : " 27 GB", | 20260505 15:36:53.411 | INFO | "malloc" : " 14 GB", | 20260505 15:36:53.411 | INFO | "pinned" : " 29 GB" | 20260505 15:36:53.411 | INFO | } | 20260505 15:36:53.411 | INFO | Perf report for the last 5.14968 seconds: | 20260505 15:36:53.411 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:36:53.411 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:53.411 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:53.411 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:53.411 | INFO | ANALYSIS : 62.6%, 3.0, 97.9ms, 0 | 20260505 15:36:53.411 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:53.411 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:53.411 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 15:36:53.411 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:53.411 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:53.411 | INFO | Uploaded 36GiB to GPU in 2.15637s (16.6947GiB/s) | 20260505 15:36:53.411 | INFO | Downloaded 34GiB to GPU in 2.12576s (15.9943GiB/s) | 20260505 15:36:58.546 | INFO | Analyzing chunk frames = [4336128,4336640) | 20260505 15:36:58.546 | INFO | MemInfo: { | 20260505 15:36:58.546 | INFO | "gpu" : " 17 GB", | 20260505 15:36:58.546 | INFO | "huge" : "", | 20260505 15:36:58.546 | INFO | "huge_shared" : " 27 GB", | 20260505 15:36:58.546 | INFO | "malloc" : " 14 GB", | 20260505 15:36:58.546 | INFO | "pinned" : " 29 GB" | 20260505 15:36:58.546 | INFO | } | 20260505 15:36:58.546 | INFO | Perf report for the last 5.13481 seconds: | 20260505 15:36:58.546 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:36:58.546 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:58.546 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:58.546 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:58.546 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 15:36:58.546 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:58.546 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:58.546 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 15:36:58.546 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:58.546 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:36:58.546 | INFO | Uploaded 36GiB to GPU in 2.13695s (16.8464GiB/s) | 20260505 15:36:58.546 | INFO | Downloaded 34GiB to GPU in 2.04117s (16.6571GiB/s) | 20260505 15:37:03.683 | INFO | Analyzing chunk frames = [4336640,4337152) | 20260505 15:37:03.683 | INFO | MemInfo: { | 20260505 15:37:03.683 | INFO | "gpu" : " 17 GB", | 20260505 15:37:03.683 | INFO | "huge" : "", | 20260505 15:37:03.683 | INFO | "huge_shared" : " 27 GB", | 20260505 15:37:03.683 | INFO | "malloc" : " 14 GB", | 20260505 15:37:03.683 | INFO | "pinned" : " 29 GB" | 20260505 15:37:03.683 | INFO | } | 20260505 15:37:03.683 | INFO | Perf report for the last 5.13701 seconds: | 20260505 15:37:03.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:37:03.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:03.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:03.683 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:03.683 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 15:37:03.683 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:03.683 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:03.683 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 15:37:03.683 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:03.683 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:03.683 | INFO | Uploaded 36GiB to GPU in 2.12127s (16.971GiB/s) | 20260505 15:37:03.683 | INFO | Downloaded 34GiB to GPU in 2.18357s (15.5708GiB/s) | 20260505 15:37:08.821 | INFO | Analyzing chunk frames = [4337152,4337664) | 20260505 15:37:08.821 | INFO | MemInfo: { | 20260505 15:37:08.821 | INFO | "gpu" : " 17 GB", | 20260505 15:37:08.821 | INFO | "huge" : "", | 20260505 15:37:08.821 | INFO | "huge_shared" : " 27 GB", | 20260505 15:37:08.821 | INFO | "malloc" : " 14 GB", | 20260505 15:37:08.821 | INFO | "pinned" : " 29 GB" | 20260505 15:37:08.821 | INFO | } | 20260505 15:37:08.821 | INFO | Perf report for the last 5.13828 seconds: | 20260505 15:37:08.821 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:37:08.821 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:08.821 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:08.821 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:08.821 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 15:37:08.821 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:08.821 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:08.821 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 15:37:08.821 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:08.821 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:08.821 | INFO | Uploaded 36GiB to GPU in 2.22419s (16.1857GiB/s) | 20260505 15:37:08.821 | INFO | Downloaded 34GiB to GPU in 2.10432s (16.1573GiB/s) | 20260505 15:37:08.954 | INFO | Starting a round of Baz IO | 20260505 15:37:12.169 | INFO | Populating RTMetrics report | 20260505 15:37:12.169 | INFO | RTMetrics report populated | 20260505 15:37:12.169 | INFO | Opening tmp RTMetrics json file | 20260505 15:37:12.207 | INFO | Serializing RTMetrics to json | 20260505 15:37:12.239 | INFO | Serialization done, swapping tmp file now... | 20260505 15:37:12.255 | INFO | RTMetrics done | 20260505 15:37:12.274 | INFO | Finished a round of Baz IO: Wrote 2.206 GiB in 3.319 seconds | 20260505 15:37:12.274 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.43% / 0.00% / 77.57% | 20260505 15:37:12.274 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.432805% / 0% / 5.5631e-05% | 20260505 15:37:12.274 | INFO | Baz IO overall: | 20260505 15:37:12.274 | INFO | BytesWritten : 2579603598529 | 20260505 15:37:12.274 | INFO | NumEvents : 1225307338120 | 20260505 15:37:13.961 | INFO | Analyzing chunk frames = [4337664,4338176) | 20260505 15:37:13.961 | INFO | MemInfo: { | 20260505 15:37:13.961 | INFO | "gpu" : " 17 GB", | 20260505 15:37:13.961 | INFO | "huge" : "", | 20260505 15:37:13.961 | INFO | "huge_shared" : " 27 GB", | 20260505 15:37:13.961 | INFO | "malloc" : " 14 GB", | 20260505 15:37:13.961 | INFO | "pinned" : " 29 GB" | 20260505 15:37:13.961 | INFO | } | 20260505 15:37:13.961 | INFO | Perf report for the last 5.14005 seconds: | 20260505 15:37:13.961 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:37:13.961 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:13.961 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:13.961 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:13.961 | INFO | ANALYSIS : 64.1%, 3.0, 99.7ms, 0 | 20260505 15:37:13.961 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:13.961 | INFO | RT_METRICS : 46.6%, 1.0, 24.4ms, 0 | 20260505 15:37:13.961 | INFO | PRE_HQ : 60.9%, 1.8, 57.3ms, 0 | 20260505 15:37:13.961 | INFO | BAZWRITER : 63.9%, 3.9, 129.4ms, 0 | 20260505 15:37:13.961 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260505 15:37:13.961 | INFO | Uploaded 36GiB to GPU in 2.19493s (16.4014GiB/s) | 20260505 15:37:13.961 | INFO | Downloaded 38GiB to GPU in 2.25118s (16.8801GiB/s) | 20260505 15:37:19.106 | INFO | Analyzing chunk frames = [4338176,4338688) | 20260505 15:37:19.106 | INFO | MemInfo: { | 20260505 15:37:19.106 | INFO | "gpu" : " 17 GB", | 20260505 15:37:19.106 | INFO | "huge" : "", | 20260505 15:37:19.106 | INFO | "huge_shared" : " 27 GB", | 20260505 15:37:19.106 | INFO | "malloc" : " 14 GB", | 20260505 15:37:19.107 | INFO | "pinned" : " 29 GB" | 20260505 15:37:19.107 | INFO | } | 20260505 15:37:19.107 | INFO | Perf report for the last 5.14551 seconds: | 20260505 15:37:19.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:37:19.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:19.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:19.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:19.107 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 15:37:19.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:19.107 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:19.107 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 15:37:19.107 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:19.107 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:19.107 | INFO | Uploaded 36GiB to GPU in 2.20215s (16.3477GiB/s) | 20260505 15:37:19.107 | INFO | Downloaded 34GiB to GPU in 2.06833s (16.4384GiB/s) | 20260505 15:37:24.247 | INFO | Analyzing chunk frames = [4338688,4339200) | 20260505 15:37:24.247 | INFO | MemInfo: { | 20260505 15:37:24.247 | INFO | "gpu" : " 17 GB", | 20260505 15:37:24.247 | INFO | "huge" : "", | 20260505 15:37:24.248 | INFO | "huge_shared" : " 27 GB", | 20260505 15:37:24.248 | INFO | "malloc" : " 14 GB", | 20260505 15:37:24.248 | INFO | "pinned" : " 29 GB" | 20260505 15:37:24.248 | INFO | } | 20260505 15:37:24.248 | INFO | Perf report for the last 5.14103 seconds: | 20260505 15:37:24.248 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:37:24.248 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:24.248 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:24.248 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:24.248 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260505 15:37:24.248 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:24.248 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:24.248 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 15:37:24.248 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:24.248 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:24.248 | INFO | Uploaded 36GiB to GPU in 2.19826s (16.3766GiB/s) | 20260505 15:37:24.248 | INFO | Downloaded 34GiB to GPU in 2.04508s (16.6253GiB/s) | 20260505 15:37:29.379 | INFO | Analyzing chunk frames = [4339200,4339712) | 20260505 15:37:29.379 | INFO | MemInfo: { | 20260505 15:37:29.379 | INFO | "gpu" : " 17 GB", | 20260505 15:37:29.379 | INFO | "huge" : "", | 20260505 15:37:29.379 | INFO | "huge_shared" : " 27 GB", | 20260505 15:37:29.379 | INFO | "malloc" : " 14 GB", | 20260505 15:37:29.379 | INFO | "pinned" : " 29 GB" | 20260505 15:37:29.380 | INFO | } | 20260505 15:37:29.380 | INFO | Perf report for the last 5.13193 seconds: | 20260505 15:37:29.380 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:37:29.380 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:29.380 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:29.380 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:29.380 | INFO | ANALYSIS : 62.6%, 3.0, 97.1ms, 0 | 20260505 15:37:29.380 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:29.380 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:29.380 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:37:29.380 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:29.380 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:29.380 | INFO | Uploaded 36GiB to GPU in 2.13674s (16.8481GiB/s) | 20260505 15:37:29.380 | INFO | Downloaded 34GiB to GPU in 2.14156s (15.8762GiB/s) | 20260505 15:37:34.509 | INFO | Analyzing chunk frames = [4339712,4340224) | 20260505 15:37:34.509 | INFO | MemInfo: { | 20260505 15:37:34.509 | INFO | "gpu" : " 17 GB", | 20260505 15:37:34.509 | INFO | "huge" : "", | 20260505 15:37:34.509 | INFO | "huge_shared" : " 27 GB", | 20260505 15:37:34.509 | INFO | "malloc" : " 14 GB", | 20260505 15:37:34.509 | INFO | "pinned" : " 29 GB" | 20260505 15:37:34.509 | INFO | } | 20260505 15:37:34.509 | INFO | Perf report for the last 5.12931 seconds: | 20260505 15:37:34.509 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:37:34.509 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:34.509 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:34.509 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:34.509 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 15:37:34.509 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:34.509 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:34.509 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 15:37:34.509 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:34.509 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:34.509 | INFO | Uploaded 36GiB to GPU in 2.14716s (16.7663GiB/s) | 20260505 15:37:34.509 | INFO | Downloaded 34GiB to GPU in 2.12735s (15.9824GiB/s) | 20260505 15:37:39.662 | INFO | Analyzing chunk frames = [4340224,4340736) | 20260505 15:37:39.662 | INFO | MemInfo: { | 20260505 15:37:39.662 | INFO | "gpu" : " 17 GB", | 20260505 15:37:39.662 | INFO | "huge" : "", | 20260505 15:37:39.662 | INFO | "huge_shared" : " 27 GB", | 20260505 15:37:39.662 | INFO | "malloc" : " 14 GB", | 20260505 15:37:39.662 | INFO | "pinned" : " 29 GB" | 20260505 15:37:39.662 | INFO | } | 20260505 15:37:39.662 | INFO | Perf report for the last 5.15316 seconds: | 20260505 15:37:39.662 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:37:39.662 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:39.662 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:39.662 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:39.662 | INFO | ANALYSIS : 62.8%, 3.0, 98.3ms, 0 | 20260505 15:37:39.662 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:39.662 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:39.662 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 15:37:39.662 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:39.662 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:39.662 | INFO | Uploaded 36GiB to GPU in 2.18616s (16.4672GiB/s) | 20260505 15:37:39.662 | INFO | Downloaded 34GiB to GPU in 2.1161s (16.0673GiB/s) | 20260505 15:37:44.802 | INFO | Analyzing chunk frames = [4340736,4341248) | 20260505 15:37:44.802 | INFO | MemInfo: { | 20260505 15:37:44.802 | INFO | "gpu" : " 17 GB", | 20260505 15:37:44.802 | INFO | "huge" : "", | 20260505 15:37:44.802 | INFO | "huge_shared" : " 27 GB", | 20260505 15:37:44.802 | INFO | "malloc" : " 14 GB", | 20260505 15:37:44.802 | INFO | "pinned" : " 29 GB" | 20260505 15:37:44.802 | INFO | } | 20260505 15:37:44.802 | INFO | Perf report for the last 5.14018 seconds: | 20260505 15:37:44.802 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:37:44.802 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:44.802 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:44.802 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:44.802 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 15:37:44.802 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:44.802 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:44.802 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 15:37:44.802 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:44.802 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:44.802 | INFO | Uploaded 36GiB to GPU in 2.14739s (16.7646GiB/s) | 20260505 15:37:44.803 | INFO | Downloaded 34GiB to GPU in 2.17531s (15.63GiB/s) | 20260505 15:37:49.943 | INFO | Analyzing chunk frames = [4341248,4341760) | 20260505 15:37:49.943 | INFO | MemInfo: { | 20260505 15:37:49.943 | INFO | "gpu" : " 17 GB", | 20260505 15:37:49.943 | INFO | "huge" : "", | 20260505 15:37:49.943 | INFO | "huge_shared" : " 27 GB", | 20260505 15:37:49.943 | INFO | "malloc" : " 14 GB", | 20260505 15:37:49.943 | INFO | "pinned" : " 29 GB" | 20260505 15:37:49.943 | INFO | } | 20260505 15:37:49.943 | INFO | Perf report for the last 5.14063 seconds: | 20260505 15:37:49.943 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:37:49.943 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:49.943 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:49.943 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:49.943 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 15:37:49.943 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:49.943 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:49.943 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 15:37:49.943 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:49.943 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:49.943 | INFO | Uploaded 36GiB to GPU in 2.11106s (17.053GiB/s) | 20260505 15:37:49.943 | INFO | Downloaded 34GiB to GPU in 2.01589s (16.866GiB/s) | 20260505 15:37:49.943 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 15:37:49.943 | INFO | Total mem usage: 15.3 GiB | 20260505 15:37:49.943 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 15:37:49.943 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -7.18%) | 20260505 15:37:49.943 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 15:37:49.943 | INFO | High Water Breakdown: | 20260505 15:37:49.943 | INFO | 61.1% : 8.74 GiB ( 91.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 15:37:49.943 | INFO | 38.8% : 5.54 GiB ( 83.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 15:37:49.943 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 15:37:49.943 | INFO | ------------------------------------------ | 20260505 15:37:49.943 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:37:49.943 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 15:37:49.943 | INFO | Total mem usage: 26.5 GiB | 20260505 15:37:49.943 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 15:37:49.943 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 15:37:49.943 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 15:37:49.943 | INFO | High Water Breakdown: | 20260505 15:37:49.943 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:37:49.943 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:37:49.943 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:37:49.943 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:37:49.943 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:37:49.943 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:37:49.943 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 15:37:49.943 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 15:37:49.943 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:37:49.943 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 15:37:49.943 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 15:37:49.943 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 15:37:49.944 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 15:37:49.944 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 15:37:49.944 | INFO | ------------------------------------------ | 20260505 15:37:49.944 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:37:49.944 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 15:37:49.944 | INFO | Total mem usage: 25.6 GiB | 20260505 15:37:49.944 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 15:37:49.944 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 15:37:49.944 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 15:37:49.944 | INFO | High Water Breakdown: | 20260505 15:37:49.944 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 15:37:49.944 | INFO | ------------------------------------------ | 20260505 15:37:49.944 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:37:49.944 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 15:37:49.944 | INFO | Total mem usage: 15.7 GiB | 20260505 15:37:49.944 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 15:37:49.944 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 15:37:49.944 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 15:37:49.944 | INFO | High Water Breakdown: | 20260505 15:37:49.944 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 15:37:49.944 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 15:37:49.944 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 15:37:49.944 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 15:37:49.944 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 15:37:49.944 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 15:37:49.944 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 15:37:49.944 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 15:37:49.944 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 15:37:49.944 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:37:49.944 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 15:37:49.944 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:37:49.944 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 15:37:49.944 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 15:37:49.944 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 15:37:49.944 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 15:37:49.944 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:37:49.944 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 15:37:49.944 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:37:49.944 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:37:49.944 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:37:49.944 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 15:37:49.944 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 15:37:49.944 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:37:49.944 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 15:37:49.944 | INFO | ------------------------------------------ | 20260505 15:37:49.944 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:37:49.944 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 15:37:49.944 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:37:49.944 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:37:49.944 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.57e+01, 1.36e+01 / 2.17e+00 | 20260505 15:37:49.944 | INFO | PerfReport: 0.29%, AggregateStats -- 9.27e-02 / 1.01e-01, 1.03e-02 / 3.65e-03 | 20260505 15:37:49.944 | INFO | PerfReport: 11.78%, Baseline -- 3.73e+00 / 4.47e+00, 5.72e-01 / 1.92e-01 | 20260505 15:37:49.944 | INFO | PerfReport: 3.29%, Histogram -- 1.04e+00 / 1.50e+00, 1.95e-01 / 1.09e-01 | 20260505 15:37:49.944 | INFO | PerfReport: 15.99%, DME -- 4.06e+01 / 4.54e+01, 4.52e+00 / 2.41e+00 | 20260505 15:37:49.944 | INFO | PerfReport: 56.57%, FrameLabeling -- 1.79e+01 / 1.94e+01, 1.39e+00 / 1.65e+00 | 20260505 15:37:49.944 | INFO | PerfReport: 7.42%, PulseAccumulating -- 2.35e+00 / 2.94e+00, 4.52e-01 / 1.53e-01 | 20260505 15:37:49.944 | INFO | PerfReport: 3.46%, Metrics -- 1.10e+00 / 1.44e+00, 3.01e-01 / 1.26e-01 | 20260505 15:37:49.944 | INFO | PerfReport: 1.21%, PulseSerializing -- 3.82e-01 / 4.41e-01, 6.79e-02 / 2.45e-02 | 20260505 15:37:49.944 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 15:37:49.944 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:37:49.944 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:37:49.944 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260505 15:37:49.944 | INFO | PerfReport: 22.46%, Upload -- 2.26e+01 / 0.00e+00, 7.87e+00 / 0.00e+00 | 20260505 15:37:49.944 | INFO | PerfReport: 21.95%, Download -- 2.20e+01 / 0.00e+00, 7.92e+00 / 0.00e+00 | 20260505 15:37:49.944 | INFO | PerfReport: 31.58%, Compute -- 3.17e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 15:37:49.944 | INFO | PerfReport: 2.22%, LockUpload -- 2.23e+00 / 0.00e+00, 7.97e+00 / 0.00e+00 | 20260505 15:37:49.944 | INFO | PerfReport: 1.54%, LockDownload -- 1.54e+00 / 0.00e+00, 3.09e+00 / 0.00e+00 | 20260505 15:37:49.944 | INFO | PerfReport: 20.23%, LockCompute -- 2.03e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 15:37:49.944 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.85e-02 / 0.00e+00 | 20260505 15:37:50.079 | INFO | Starting a round of Baz IO | 20260505 15:37:53.286 | INFO | Populating RTMetrics report | 20260505 15:37:53.286 | INFO | RTMetrics report populated | 20260505 15:37:53.286 | INFO | Opening tmp RTMetrics json file | 20260505 15:37:53.297 | INFO | Serializing RTMetrics to json | 20260505 15:37:53.299 | INFO | Serialization done, swapping tmp file now... | 20260505 15:37:53.299 | INFO | RTMetrics done | 20260505 15:37:53.387 | INFO | Finished a round of Baz IO: Wrote 2.205 GiB in 3.307 seconds | 20260505 15:37:53.387 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.43% / 0.00% / 77.57% | 20260505 15:37:53.387 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.43238% / 3.97364e-06% / 5.96046e-05% | 20260505 15:37:53.387 | INFO | Baz IO overall: | 20260505 15:37:53.387 | INFO | BytesWritten : 2581971602717 | 20260505 15:37:53.387 | INFO | NumEvents : 1226420459127 | 20260505 15:37:55.075 | INFO | Analyzing chunk frames = [4341760,4342272) | 20260505 15:37:55.075 | INFO | MemInfo: { | 20260505 15:37:55.075 | INFO | "gpu" : " 17 GB", | 20260505 15:37:55.075 | INFO | "huge" : "", | 20260505 15:37:55.075 | INFO | "huge_shared" : " 27 GB", | 20260505 15:37:55.075 | INFO | "malloc" : " 14 GB", | 20260505 15:37:55.075 | INFO | "pinned" : " 29 GB" | 20260505 15:37:55.075 | INFO | } | 20260505 15:37:55.075 | INFO | Perf report for the last 5.13196 seconds: | 20260505 15:37:55.075 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:37:55.075 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:55.075 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:55.075 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:55.075 | INFO | ANALYSIS : 64.0%, 3.0, 99.6ms, 0 | 20260505 15:37:55.075 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:37:55.075 | INFO | RT_METRICS : 45.1%, 1.0, 23.6ms, 0 | 20260505 15:37:55.075 | INFO | PRE_HQ : 60.8%, 1.8, 56.6ms, 0 | 20260505 15:37:55.075 | INFO | BAZWRITER : 63.7%, 3.9, 127.5ms, 0 | 20260505 15:37:55.075 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 15:37:55.075 | INFO | Uploaded 36GiB to GPU in 2.17514s (16.5507GiB/s) | 20260505 15:37:55.075 | INFO | Downloaded 38GiB to GPU in 2.26727s (16.7602GiB/s) | 20260505 15:38:00.217 | INFO | Analyzing chunk frames = [4342272,4342784) | 20260505 15:38:00.217 | INFO | MemInfo: { | 20260505 15:38:00.217 | INFO | "gpu" : " 17 GB", | 20260505 15:38:00.217 | INFO | "huge" : "", | 20260505 15:38:00.217 | INFO | "huge_shared" : " 27 GB", | 20260505 15:38:00.217 | INFO | "malloc" : " 14 GB", | 20260505 15:38:00.217 | INFO | "pinned" : " 29 GB" | 20260505 15:38:00.217 | INFO | } | 20260505 15:38:00.217 | INFO | Perf report for the last 5.1423 seconds: | 20260505 15:38:00.217 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:38:00.217 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:00.217 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:00.217 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:00.217 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 15:38:00.217 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:00.217 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:00.217 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 15:38:00.217 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:00.217 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:00.217 | INFO | Uploaded 36GiB to GPU in 2.20019s (16.3622GiB/s) | 20260505 15:38:00.217 | INFO | Downloaded 34GiB to GPU in 2.04337s (16.6391GiB/s) | 20260505 15:38:05.360 | INFO | Analyzing chunk frames = [4342784,4343296) | 20260505 15:38:05.360 | INFO | MemInfo: { | 20260505 15:38:05.360 | INFO | "gpu" : " 17 GB", | 20260505 15:38:05.360 | INFO | "huge" : "", | 20260505 15:38:05.360 | INFO | "huge_shared" : " 27 GB", | 20260505 15:38:05.360 | INFO | "malloc" : " 14 GB", | 20260505 15:38:05.360 | INFO | "pinned" : " 29 GB" | 20260505 15:38:05.360 | INFO | } | 20260505 15:38:05.361 | INFO | Perf report for the last 5.14333 seconds: | 20260505 15:38:05.361 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:38:05.361 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:05.361 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:05.361 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:05.361 | INFO | ANALYSIS : 62.9%, 3.0, 97.9ms, 0 | 20260505 15:38:05.361 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:05.361 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:05.361 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 15:38:05.361 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:05.361 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:05.361 | INFO | Uploaded 36GiB to GPU in 2.1745s (16.5555GiB/s) | 20260505 15:38:05.361 | INFO | Downloaded 34GiB to GPU in 2.05805s (16.5205GiB/s) | 20260505 15:38:10.496 | INFO | Analyzing chunk frames = [4343296,4343808) | 20260505 15:38:10.496 | INFO | MemInfo: { | 20260505 15:38:10.496 | INFO | "gpu" : " 17 GB", | 20260505 15:38:10.496 | INFO | "huge" : "", | 20260505 15:38:10.496 | INFO | "huge_shared" : " 27 GB", | 20260505 15:38:10.496 | INFO | "malloc" : " 14 GB", | 20260505 15:38:10.496 | INFO | "pinned" : " 29 GB" | 20260505 15:38:10.496 | INFO | } | 20260505 15:38:10.496 | INFO | Perf report for the last 5.13521 seconds: | 20260505 15:38:10.496 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:38:10.496 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:10.496 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:10.496 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:10.496 | INFO | ANALYSIS : 62.5%, 3.0, 97.1ms, 0 | 20260505 15:38:10.496 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:10.496 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:10.496 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 15:38:10.496 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:10.496 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:10.496 | INFO | Uploaded 36GiB to GPU in 2.12469s (16.9436GiB/s) | 20260505 15:38:10.496 | INFO | Downloaded 34GiB to GPU in 2.13916s (15.8941GiB/s) | 20260505 15:38:15.629 | INFO | Analyzing chunk frames = [4343808,4344320) | 20260505 15:38:15.629 | INFO | MemInfo: { | 20260505 15:38:15.629 | INFO | "gpu" : " 17 GB", | 20260505 15:38:15.629 | INFO | "huge" : "", | 20260505 15:38:15.629 | INFO | "huge_shared" : " 27 GB", | 20260505 15:38:15.629 | INFO | "malloc" : " 14 GB", | 20260505 15:38:15.629 | INFO | "pinned" : " 29 GB" | 20260505 15:38:15.629 | INFO | } | 20260505 15:38:15.629 | INFO | Perf report for the last 5.1337 seconds: | 20260505 15:38:15.629 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:38:15.629 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:15.629 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:15.629 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:15.629 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260505 15:38:15.629 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:15.629 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:15.630 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 15:38:15.630 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:15.630 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:15.630 | INFO | Uploaded 36GiB to GPU in 2.18645s (16.465GiB/s) | 20260505 15:38:15.630 | INFO | Downloaded 34GiB to GPU in 2.1328s (15.9415GiB/s) | 20260505 15:38:20.765 | INFO | Analyzing chunk frames = [4344320,4344832) | 20260505 15:38:20.765 | INFO | MemInfo: { | 20260505 15:38:20.765 | INFO | "gpu" : " 17 GB", | 20260505 15:38:20.765 | INFO | "huge" : "", | 20260505 15:38:20.765 | INFO | "huge_shared" : " 27 GB", | 20260505 15:38:20.765 | INFO | "malloc" : " 14 GB", | 20260505 15:38:20.765 | INFO | "pinned" : " 29 GB" | 20260505 15:38:20.765 | INFO | } | 20260505 15:38:20.765 | INFO | Perf report for the last 5.13545 seconds: | 20260505 15:38:20.765 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:38:20.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:20.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:20.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:20.765 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 15:38:20.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:20.765 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:20.765 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:38:20.765 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:20.765 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:20.765 | INFO | Uploaded 36GiB to GPU in 2.15931s (16.672GiB/s) | 20260505 15:38:20.765 | INFO | Downloaded 34GiB to GPU in 2.03913s (16.6738GiB/s) | 20260505 15:38:25.914 | INFO | Analyzing chunk frames = [4344832,4345344) | 20260505 15:38:25.914 | INFO | MemInfo: { | 20260505 15:38:25.914 | INFO | "gpu" : " 17 GB", | 20260505 15:38:25.914 | INFO | "huge" : "", | 20260505 15:38:25.914 | INFO | "huge_shared" : " 27 GB", | 20260505 15:38:25.914 | INFO | "malloc" : " 14 GB", | 20260505 15:38:25.914 | INFO | "pinned" : " 29 GB" | 20260505 15:38:25.914 | INFO | } | 20260505 15:38:25.914 | INFO | Perf report for the last 5.14911 seconds: | 20260505 15:38:25.914 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:38:25.914 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:25.914 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:25.914 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:25.914 | INFO | ANALYSIS : 63.2%, 3.0, 98.9ms, 0 | 20260505 15:38:25.914 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:25.914 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:25.914 | INFO | PRE_HQ : 28.7%, 1.0, 15.0ms, 0 | 20260505 15:38:25.914 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:25.914 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:25.914 | INFO | Uploaded 36GiB to GPU in 2.12529s (16.9389GiB/s) | 20260505 15:38:25.914 | INFO | Downloaded 34GiB to GPU in 2.17602s (15.6249GiB/s) | 20260505 15:38:31.058 | INFO | Analyzing chunk frames = [4345344,4345856) | 20260505 15:38:31.058 | INFO | MemInfo: { | 20260505 15:38:31.058 | INFO | "gpu" : " 17 GB", | 20260505 15:38:31.059 | INFO | "huge" : "", | 20260505 15:38:31.059 | INFO | "huge_shared" : " 27 GB", | 20260505 15:38:31.059 | INFO | "malloc" : " 14 GB", | 20260505 15:38:31.059 | INFO | "pinned" : " 29 GB" | 20260505 15:38:31.059 | INFO | } | 20260505 15:38:31.059 | INFO | Perf report for the last 5.14457 seconds: | 20260505 15:38:31.059 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:38:31.059 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:31.059 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:31.059 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:31.059 | INFO | ANALYSIS : 63.1%, 3.0, 98.7ms, 0 | 20260505 15:38:31.059 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:31.059 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:31.059 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 15:38:31.059 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:31.059 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:31.059 | INFO | Uploaded 36GiB to GPU in 2.16044s (16.6633GiB/s) | 20260505 15:38:31.059 | INFO | Downloaded 34GiB to GPU in 2.03605s (16.699GiB/s) | 20260505 15:38:31.192 | INFO | Starting a round of Baz IO | 20260505 15:38:34.391 | INFO | Populating RTMetrics report | 20260505 15:38:34.391 | INFO | RTMetrics report populated | 20260505 15:38:34.391 | INFO | Opening tmp RTMetrics json file | 20260505 15:38:34.404 | INFO | Serializing RTMetrics to json | 20260505 15:38:34.406 | INFO | Serialization done, swapping tmp file now... | 20260505 15:38:34.407 | INFO | RTMetrics done | 20260505 15:38:34.489 | INFO | Finished a round of Baz IO: Wrote 2.205 GiB in 3.296 seconds | 20260505 15:38:34.489 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.42% / 0.00% / 77.57% | 20260505 15:38:34.489 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.431108% / 3.97364e-06% / 7.15256e-05% | 20260505 15:38:34.489 | INFO | Baz IO overall: | 20260505 15:38:34.489 | INFO | BytesWritten : 2584338693207 | 20260505 15:38:34.489 | INFO | NumEvents : 1227533273481 | 20260505 15:38:36.192 | INFO | Analyzing chunk frames = [4345856,4346368) | 20260505 15:38:36.192 | INFO | MemInfo: { | 20260505 15:38:36.192 | INFO | "gpu" : " 17 GB", | 20260505 15:38:36.192 | INFO | "huge" : "", | 20260505 15:38:36.192 | INFO | "huge_shared" : " 27 GB", | 20260505 15:38:36.192 | INFO | "malloc" : " 14 GB", | 20260505 15:38:36.192 | INFO | "pinned" : " 29 GB" | 20260505 15:38:36.192 | INFO | } | 20260505 15:38:36.192 | INFO | Perf report for the last 5.13366 seconds: | 20260505 15:38:36.192 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:38:36.192 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:36.192 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:36.192 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:36.192 | INFO | ANALYSIS : 63.8%, 3.0, 99.3ms, 0 | 20260505 15:38:36.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:36.192 | INFO | RT_METRICS : 45.1%, 1.0, 23.6ms, 0 | 20260505 15:38:36.192 | INFO | PRE_HQ : 60.6%, 1.8, 56.6ms, 0 | 20260505 15:38:36.192 | INFO | BAZWRITER : 63.4%, 3.9, 127.1ms, 0 | 20260505 15:38:36.192 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 15:38:36.193 | INFO | Uploaded 36GiB to GPU in 2.15026s (16.7422GiB/s) | 20260505 15:38:36.193 | INFO | Downloaded 38GiB to GPU in 2.25252s (16.87GiB/s) | 20260505 15:38:41.323 | INFO | Analyzing chunk frames = [4346368,4346880) | 20260505 15:38:41.323 | INFO | MemInfo: { | 20260505 15:38:41.323 | INFO | "gpu" : " 17 GB", | 20260505 15:38:41.323 | INFO | "huge" : "", | 20260505 15:38:41.323 | INFO | "huge_shared" : " 27 GB", | 20260505 15:38:41.323 | INFO | "malloc" : " 14 GB", | 20260505 15:38:41.323 | INFO | "pinned" : " 29 GB" | 20260505 15:38:41.323 | INFO | } | 20260505 15:38:41.323 | INFO | Perf report for the last 5.1307 seconds: | 20260505 15:38:41.323 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:38:41.323 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:41.323 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:41.323 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:41.323 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 15:38:41.323 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:41.323 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:41.323 | INFO | PRE_HQ : 27.9%, 1.0, 14.4ms, 0 | 20260505 15:38:41.323 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:41.323 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:41.323 | INFO | Uploaded 36GiB to GPU in 2.19366s (16.4109GiB/s) | 20260505 15:38:41.323 | INFO | Downloaded 34GiB to GPU in 2.0293s (16.7545GiB/s) | 20260505 15:38:46.468 | INFO | Analyzing chunk frames = [4346880,4347392) | 20260505 15:38:46.468 | INFO | MemInfo: { | 20260505 15:38:46.468 | INFO | "gpu" : " 17 GB", | 20260505 15:38:46.468 | INFO | "huge" : "", | 20260505 15:38:46.468 | INFO | "huge_shared" : " 27 GB", | 20260505 15:38:46.468 | INFO | "malloc" : " 14 GB", | 20260505 15:38:46.468 | INFO | "pinned" : " 29 GB" | 20260505 15:38:46.468 | INFO | } | 20260505 15:38:46.468 | INFO | Perf report for the last 5.14501 seconds: | 20260505 15:38:46.468 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:38:46.468 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:46.468 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:46.468 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:46.468 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 15:38:46.468 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:46.468 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:46.468 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:38:46.468 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:46.468 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:46.468 | INFO | Uploaded 36GiB to GPU in 2.23721s (16.0914GiB/s) | 20260505 15:38:46.468 | INFO | Downloaded 34GiB to GPU in 2.09972s (16.1926GiB/s) | 20260505 15:38:51.609 | INFO | Analyzing chunk frames = [4347392,4347904) | 20260505 15:38:51.609 | INFO | MemInfo: { | 20260505 15:38:51.609 | INFO | "gpu" : " 17 GB", | 20260505 15:38:51.609 | INFO | "huge" : "", | 20260505 15:38:51.609 | INFO | "huge_shared" : " 27 GB", | 20260505 15:38:51.609 | INFO | "malloc" : " 14 GB", | 20260505 15:38:51.609 | INFO | "pinned" : " 29 GB" | 20260505 15:38:51.609 | INFO | } | 20260505 15:38:51.609 | INFO | Perf report for the last 5.14121 seconds: | 20260505 15:38:51.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:38:51.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:51.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:51.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:51.609 | INFO | ANALYSIS : 62.6%, 3.0, 97.3ms, 0 | 20260505 15:38:51.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:51.609 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:51.609 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:38:51.609 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:51.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:51.610 | INFO | Uploaded 36GiB to GPU in 2.19191s (16.4241GiB/s) | 20260505 15:38:51.610 | INFO | Downloaded 34GiB to GPU in 2.13655s (15.9135GiB/s) | 20260505 15:38:56.754 | INFO | Analyzing chunk frames = [4347904,4348416) | 20260505 15:38:56.754 | INFO | MemInfo: { | 20260505 15:38:56.754 | INFO | "gpu" : " 17 GB", | 20260505 15:38:56.754 | INFO | "huge" : "", | 20260505 15:38:56.754 | INFO | "huge_shared" : " 27 GB", | 20260505 15:38:56.754 | INFO | "malloc" : " 14 GB", | 20260505 15:38:56.754 | INFO | "pinned" : " 29 GB" | 20260505 15:38:56.754 | INFO | } | 20260505 15:38:56.754 | INFO | Perf report for the last 5.14465 seconds: | 20260505 15:38:56.754 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:38:56.754 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:56.754 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:56.754 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:56.754 | INFO | ANALYSIS : 62.3%, 3.0, 97.4ms, 0 | 20260505 15:38:56.754 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:56.754 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:56.754 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 15:38:56.754 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:56.754 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:38:56.754 | INFO | Uploaded 36GiB to GPU in 2.15796s (16.6825GiB/s) | 20260505 15:38:56.754 | INFO | Downloaded 34GiB to GPU in 2.06245s (16.4852GiB/s) | 20260505 15:39:01.889 | INFO | Analyzing chunk frames = [4348416,4348928) | 20260505 15:39:01.889 | INFO | MemInfo: { | 20260505 15:39:01.889 | INFO | "gpu" : " 17 GB", | 20260505 15:39:01.889 | INFO | "huge" : "", | 20260505 15:39:01.889 | INFO | "huge_shared" : " 27 GB", | 20260505 15:39:01.889 | INFO | "malloc" : " 14 GB", | 20260505 15:39:01.889 | INFO | "pinned" : " 29 GB" | 20260505 15:39:01.889 | INFO | } | 20260505 15:39:01.889 | INFO | Perf report for the last 5.13519 seconds: | 20260505 15:39:01.889 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:39:01.889 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:01.889 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:01.889 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:01.889 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 15:39:01.889 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:01.889 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:01.889 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 15:39:01.889 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:01.889 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:01.889 | INFO | Uploaded 36GiB to GPU in 2.13661s (16.8491GiB/s) | 20260505 15:39:01.889 | INFO | Downloaded 34GiB to GPU in 2.03291s (16.7248GiB/s) | 20260505 15:39:07.024 | INFO | Analyzing chunk frames = [4348928,4349440) | 20260505 15:39:07.024 | INFO | MemInfo: { | 20260505 15:39:07.024 | INFO | "gpu" : " 17 GB", | 20260505 15:39:07.024 | INFO | "huge" : "", | 20260505 15:39:07.024 | INFO | "huge_shared" : " 27 GB", | 20260505 15:39:07.024 | INFO | "malloc" : " 14 GB", | 20260505 15:39:07.024 | INFO | "pinned" : " 29 GB" | 20260505 15:39:07.024 | INFO | } | 20260505 15:39:07.024 | INFO | Perf report for the last 5.13499 seconds: | 20260505 15:39:07.024 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:39:07.024 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:07.024 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:07.024 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:07.024 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 15:39:07.024 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:07.024 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:07.024 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 15:39:07.024 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:07.024 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:07.024 | INFO | Uploaded 36GiB to GPU in 2.09578s (17.1773GiB/s) | 20260505 15:39:07.024 | INFO | Downloaded 34GiB to GPU in 2.17055s (15.6642GiB/s) | 20260505 15:39:12.175 | INFO | Analyzing chunk frames = [4349440,4349952) | 20260505 15:39:12.175 | INFO | MemInfo: { | 20260505 15:39:12.175 | INFO | "gpu" : " 17 GB", | 20260505 15:39:12.175 | INFO | "huge" : "", | 20260505 15:39:12.175 | INFO | "huge_shared" : " 27 GB", | 20260505 15:39:12.175 | INFO | "malloc" : " 14 GB", | 20260505 15:39:12.175 | INFO | "pinned" : " 29 GB" | 20260505 15:39:12.175 | INFO | } | 20260505 15:39:12.175 | INFO | Perf report for the last 5.15113 seconds: | 20260505 15:39:12.175 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:39:12.175 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:12.175 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:12.175 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:12.175 | INFO | ANALYSIS : 63.4%, 3.0, 99.2ms, 0 | 20260505 15:39:12.175 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:12.175 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:12.175 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:39:12.175 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:12.175 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:12.176 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 15:39:12.176 | INFO | Total mem usage: 15.3 GiB | 20260505 15:39:12.176 | INFO | Uploaded 36GiB to GPU in 2.2189s (16.2243GiB/s) | 20260505 15:39:12.176 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 15:39:12.176 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.82%) | 20260505 15:39:12.176 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 15:39:12.176 | INFO | High Water Breakdown: | 20260505 15:39:12.176 | INFO | 61% : 8.68 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 15:39:12.176 | INFO | Downloaded 34GiB to GPU in 2.0989s (16.199GiB/s) | 20260505 15:39:12.176 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 15:39:12.176 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 15:39:12.176 | INFO | ------------------------------------------ | 20260505 15:39:12.176 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:39:12.176 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 15:39:12.176 | INFO | Total mem usage: 26.5 GiB | 20260505 15:39:12.176 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 15:39:12.176 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 15:39:12.176 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 15:39:12.176 | INFO | High Water Breakdown: | 20260505 15:39:12.176 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:39:12.176 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:39:12.176 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:39:12.176 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:39:12.176 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:39:12.176 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:39:12.176 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 15:39:12.176 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 15:39:12.176 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:39:12.176 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 15:39:12.176 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 15:39:12.176 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 15:39:12.176 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 15:39:12.176 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 15:39:12.176 | INFO | ------------------------------------------ | 20260505 15:39:12.176 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:39:12.176 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 15:39:12.176 | INFO | Total mem usage: 25.6 GiB | 20260505 15:39:12.176 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 15:39:12.176 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 15:39:12.176 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 15:39:12.176 | INFO | High Water Breakdown: | 20260505 15:39:12.176 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 15:39:12.176 | INFO | ------------------------------------------ | 20260505 15:39:12.176 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:39:12.176 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 15:39:12.176 | INFO | Total mem usage: 15.7 GiB | 20260505 15:39:12.176 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 15:39:12.176 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 15:39:12.176 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 15:39:12.176 | INFO | High Water Breakdown: | 20260505 15:39:12.176 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 15:39:12.176 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 15:39:12.176 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 15:39:12.176 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 15:39:12.176 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 15:39:12.176 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 15:39:12.176 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 15:39:12.176 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 15:39:12.176 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 15:39:12.176 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:39:12.176 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 15:39:12.176 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:39:12.176 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 15:39:12.176 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 15:39:12.176 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 15:39:12.176 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 15:39:12.176 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:39:12.176 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 15:39:12.176 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:39:12.176 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:39:12.176 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:39:12.176 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 15:39:12.176 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 15:39:12.176 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:39:12.176 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 15:39:12.176 | INFO | ------------------------------------------ | 20260505 15:39:12.176 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:39:12.176 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 15:39:12.176 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:39:12.176 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:39:12.176 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.58e+01, 1.38e+01 / 1.87e+00 | 20260505 15:39:12.176 | INFO | PerfReport: 0.29%, AggregateStats -- 9.26e-02 / 1.02e-01, 1.04e-02 / 4.46e-03 | 20260505 15:39:12.176 | INFO | PerfReport: 11.78%, Baseline -- 3.74e+00 / 4.49e+00, 5.73e-01 / 1.81e-01 | 20260505 15:39:12.176 | INFO | PerfReport: 3.27%, Histogram -- 1.04e+00 / 1.52e+00, 1.94e-01 / 9.03e-02 | 20260505 15:39:12.176 | INFO | PerfReport: 15.97%, DME -- 4.05e+01 / 4.54e+01, 4.47e+00 / 1.97e+00 | 20260505 15:39:12.176 | INFO | PerfReport: 56.63%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.46e+00 / 1.38e+00 | 20260505 15:39:12.176 | INFO | PerfReport: 7.39%, PulseAccumulating -- 2.35e+00 / 2.91e+00, 4.48e-01 / 1.75e-01 | 20260505 15:39:12.176 | INFO | PerfReport: 3.45%, Metrics -- 1.09e+00 / 1.43e+00, 3.02e-01 / 1.50e-01 | 20260505 15:39:12.176 | INFO | PerfReport: 1.21%, PulseSerializing -- 3.83e-01 / 4.38e-01, 6.91e-02 / 3.75e-02 | 20260505 15:39:12.177 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 15:39:12.177 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:39:12.177 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:39:12.177 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260505 15:39:12.177 | INFO | PerfReport: 22.49%, Upload -- 2.26e+01 / 0.00e+00, 7.91e+00 / 0.00e+00 | 20260505 15:39:12.177 | INFO | PerfReport: 21.90%, Download -- 2.20e+01 / 0.00e+00, 7.90e+00 / 0.00e+00 | 20260505 15:39:12.177 | INFO | PerfReport: 31.58%, Compute -- 3.17e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 15:39:12.177 | INFO | PerfReport: 2.30%, LockUpload -- 2.31e+00 / 0.00e+00, 7.92e+00 / 0.00e+00 | 20260505 15:39:12.177 | INFO | PerfReport: 1.46%, LockDownload -- 1.47e+00 / 0.00e+00, 3.08e+00 / 0.00e+00 | 20260505 15:39:12.177 | INFO | PerfReport: 20.26%, LockCompute -- 2.03e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 15:39:12.177 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.69e-02 / 0.00e+00 | 20260505 15:39:12.312 | INFO | Starting a round of Baz IO | 20260505 15:39:15.510 | INFO | Populating RTMetrics report | 20260505 15:39:15.511 | INFO | RTMetrics report populated | 20260505 15:39:15.511 | INFO | Opening tmp RTMetrics json file | 20260505 15:39:15.516 | INFO | Serializing RTMetrics to json | 20260505 15:39:15.518 | INFO | Serialization done, swapping tmp file now... | 20260505 15:39:15.519 | INFO | RTMetrics done | 20260505 15:39:15.607 | INFO | Finished a round of Baz IO: Wrote 2.205 GiB in 3.294 seconds | 20260505 15:39:15.607 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.42% / 0.00% / 77.58% | 20260505 15:39:15.607 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.430628% / 7.94729e-06% / 8.34465e-05% | 20260505 15:39:15.607 | INFO | Baz IO overall: | 20260505 15:39:15.607 | INFO | BytesWritten : 2586705999214 | 20260505 15:39:15.607 | INFO | NumEvents : 1228646042492 | 20260505 15:39:17.308 | INFO | Analyzing chunk frames = [4349952,4350464) | 20260505 15:39:17.309 | INFO | MemInfo: { | 20260505 15:39:17.309 | INFO | "gpu" : " 17 GB", | 20260505 15:39:17.309 | INFO | "huge" : "", | 20260505 15:39:17.309 | INFO | "huge_shared" : " 27 GB", | 20260505 15:39:17.309 | INFO | "malloc" : " 14 GB", | 20260505 15:39:17.309 | INFO | "pinned" : " 29 GB" | 20260505 15:39:17.309 | INFO | } | 20260505 15:39:17.309 | INFO | Perf report for the last 5.13332 seconds: | 20260505 15:39:17.309 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:39:17.309 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:17.309 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:17.309 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:17.309 | INFO | ANALYSIS : 63.9%, 3.0, 99.3ms, 0 | 20260505 15:39:17.309 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:17.309 | INFO | RT_METRICS : 45.1%, 1.0, 23.6ms, 0 | 20260505 15:39:17.309 | INFO | PRE_HQ : 60.7%, 1.8, 57.1ms, 0 | 20260505 15:39:17.309 | INFO | BAZWRITER : 63.6%, 3.9, 127.2ms, 0 | 20260505 15:39:17.309 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 15:39:17.309 | INFO | Uploaded 36GiB to GPU in 2.18285s (16.4922GiB/s) | 20260505 15:39:17.309 | INFO | Downloaded 38GiB to GPU in 2.26253s (16.7953GiB/s) | 20260505 15:39:22.444 | INFO | Analyzing chunk frames = [4350464,4350976) | 20260505 15:39:22.444 | INFO | MemInfo: { | 20260505 15:39:22.444 | INFO | "gpu" : " 17 GB", | 20260505 15:39:22.444 | INFO | "huge" : "", | 20260505 15:39:22.444 | INFO | "huge_shared" : " 27 GB", | 20260505 15:39:22.444 | INFO | "malloc" : " 14 GB", | 20260505 15:39:22.444 | INFO | "pinned" : " 29 GB" | 20260505 15:39:22.444 | INFO | } | 20260505 15:39:22.444 | INFO | Perf report for the last 5.13576 seconds: | 20260505 15:39:22.444 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:39:22.444 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:22.444 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:22.444 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:22.444 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 15:39:22.444 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:22.444 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:22.444 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 15:39:22.444 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:22.444 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:22.445 | INFO | Uploaded 36GiB to GPU in 2.1372s (16.8444GiB/s) | 20260505 15:39:22.445 | INFO | Downloaded 34GiB to GPU in 2.04401s (16.634GiB/s) | 20260505 15:39:27.582 | INFO | Analyzing chunk frames = [4350976,4351488) | 20260505 15:39:27.582 | INFO | MemInfo: { | 20260505 15:39:27.582 | INFO | "gpu" : " 17 GB", | 20260505 15:39:27.582 | INFO | "huge" : "", | 20260505 15:39:27.583 | INFO | "huge_shared" : " 27 GB", | 20260505 15:39:27.583 | INFO | "malloc" : " 14 GB", | 20260505 15:39:27.583 | INFO | "pinned" : " 29 GB" | 20260505 15:39:27.583 | INFO | } | 20260505 15:39:27.583 | INFO | Perf report for the last 5.13822 seconds: | 20260505 15:39:27.583 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:39:27.583 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:27.583 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:27.583 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:27.583 | INFO | ANALYSIS : 63.0%, 3.0, 98.1ms, 0 | 20260505 15:39:27.583 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:27.583 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:27.583 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 15:39:27.583 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:27.583 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:27.583 | INFO | Uploaded 36GiB to GPU in 2.18285s (16.4922GiB/s) | 20260505 15:39:27.583 | INFO | Downloaded 34GiB to GPU in 2.05984s (16.5061GiB/s) | 20260505 15:39:32.721 | INFO | Analyzing chunk frames = [4351488,4352000) | 20260505 15:39:32.721 | INFO | MemInfo: { | 20260505 15:39:32.721 | INFO | "gpu" : " 17 GB", | 20260505 15:39:32.721 | INFO | "huge" : "", | 20260505 15:39:32.721 | INFO | "huge_shared" : " 27 GB", | 20260505 15:39:32.721 | INFO | "malloc" : " 14 GB", | 20260505 15:39:32.721 | INFO | "pinned" : " 29 GB" | 20260505 15:39:32.721 | INFO | } | 20260505 15:39:32.721 | INFO | Perf report for the last 5.13888 seconds: | 20260505 15:39:32.721 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:39:32.721 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:32.721 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:32.722 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:32.722 | INFO | ANALYSIS : 62.6%, 3.0, 97.3ms, 0 | 20260505 15:39:32.722 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:32.722 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:32.722 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:39:32.722 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:32.722 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:32.722 | INFO | Uploaded 36GiB to GPU in 2.18073s (16.5082GiB/s) | 20260505 15:39:32.722 | INFO | Downloaded 34GiB to GPU in 2.13376s (15.9343GiB/s) | 20260505 15:39:37.865 | INFO | Analyzing chunk frames = [4352000,4352512) | 20260505 15:39:37.865 | INFO | MemInfo: { | 20260505 15:39:37.865 | INFO | "gpu" : " 17 GB", | 20260505 15:39:37.865 | INFO | "huge" : "", | 20260505 15:39:37.865 | INFO | "huge_shared" : " 27 GB", | 20260505 15:39:37.865 | INFO | "malloc" : " 14 GB", | 20260505 15:39:37.865 | INFO | "pinned" : " 29 GB" | 20260505 15:39:37.865 | INFO | } | 20260505 15:39:37.865 | INFO | Perf report for the last 5.14346 seconds: | 20260505 15:39:37.865 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:39:37.865 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:37.865 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:37.865 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:37.865 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260505 15:39:37.865 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:37.865 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:37.865 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:39:37.865 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:37.865 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:37.865 | INFO | Uploaded 36GiB to GPU in 2.18112s (16.5052GiB/s) | 20260505 15:39:37.865 | INFO | Downloaded 34GiB to GPU in 2.13221s (15.9459GiB/s) | 20260505 15:39:43.012 | INFO | Analyzing chunk frames = [4352512,4353024) | 20260505 15:39:43.012 | INFO | MemInfo: { | 20260505 15:39:43.012 | INFO | "gpu" : " 17 GB", | 20260505 15:39:43.012 | INFO | "huge" : "", | 20260505 15:39:43.012 | INFO | "huge_shared" : " 27 GB", | 20260505 15:39:43.012 | INFO | "malloc" : " 14 GB", | 20260505 15:39:43.012 | INFO | "pinned" : " 29 GB" | 20260505 15:39:43.012 | INFO | } | 20260505 15:39:43.012 | INFO | Perf report for the last 5.14689 seconds: | 20260505 15:39:43.012 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:39:43.012 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:43.012 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:43.012 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:43.012 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260505 15:39:43.012 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:43.012 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:43.012 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 15:39:43.012 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:43.012 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:43.012 | INFO | Uploaded 36GiB to GPU in 2.13562s (16.8569GiB/s) | 20260505 15:39:43.012 | INFO | Downloaded 34GiB to GPU in 2.03504s (16.7073GiB/s) | 20260505 15:39:48.139 | INFO | Analyzing chunk frames = [4353024,4353536) | 20260505 15:39:48.139 | INFO | MemInfo: { | 20260505 15:39:48.139 | INFO | "gpu" : " 17 GB", | 20260505 15:39:48.139 | INFO | "huge" : "", | 20260505 15:39:48.139 | INFO | "huge_shared" : " 27 GB", | 20260505 15:39:48.139 | INFO | "malloc" : " 14 GB", | 20260505 15:39:48.139 | INFO | "pinned" : " 29 GB" | 20260505 15:39:48.139 | INFO | } | 20260505 15:39:48.139 | INFO | Perf report for the last 5.12685 seconds: | 20260505 15:39:48.139 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:39:48.139 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:48.139 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:48.139 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:48.139 | INFO | ANALYSIS : 63.8%, 3.0, 99.3ms, 0 | 20260505 15:39:48.139 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:48.139 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:48.139 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 15:39:48.139 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:48.139 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:48.139 | INFO | Uploaded 36GiB to GPU in 2.10596s (17.0943GiB/s) | 20260505 15:39:48.139 | INFO | Downloaded 34GiB to GPU in 2.17809s (15.61GiB/s) | 20260505 15:39:53.272 | INFO | Analyzing chunk frames = [4353536,4354048) | 20260505 15:39:53.272 | INFO | MemInfo: { | 20260505 15:39:53.272 | INFO | "gpu" : " 17 GB", | 20260505 15:39:53.272 | INFO | "huge" : "", | 20260505 15:39:53.272 | INFO | "huge_shared" : " 27 GB", | 20260505 15:39:53.272 | INFO | "malloc" : " 14 GB", | 20260505 15:39:53.272 | INFO | "pinned" : " 29 GB" | 20260505 15:39:53.272 | INFO | } | 20260505 15:39:53.272 | INFO | Perf report for the last 5.13357 seconds: | 20260505 15:39:53.272 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:39:53.272 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:53.272 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:53.272 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:53.272 | INFO | ANALYSIS : 63.5%, 3.0, 99.0ms, 0 | 20260505 15:39:53.272 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:53.272 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:53.272 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 15:39:53.272 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:53.272 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:53.273 | INFO | Uploaded 36GiB to GPU in 2.20911s (16.2962GiB/s) | 20260505 15:39:53.273 | INFO | Downloaded 34GiB to GPU in 2.10149s (16.179GiB/s) | 20260505 15:39:53.409 | INFO | Starting a round of Baz IO | 20260505 15:39:56.621 | INFO | Populating RTMetrics report | 20260505 15:39:56.621 | INFO | RTMetrics report populated | 20260505 15:39:56.621 | INFO | Opening tmp RTMetrics json file | 20260505 15:39:56.665 | INFO | Serializing RTMetrics to json | 20260505 15:39:56.706 | INFO | Serialization done, swapping tmp file now... | 20260505 15:39:56.708 | INFO | RTMetrics done | 20260505 15:39:56.724 | INFO | Finished a round of Baz IO: Wrote 2.202 GiB in 3.314 seconds | 20260505 15:39:56.724 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.42% / 0.00% / 77.58% | 20260505 15:39:56.724 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.427103% / 3.97364e-06% / 8.34465e-05% | 20260505 15:39:56.724 | INFO | Baz IO overall: | 20260505 15:39:56.724 | INFO | BytesWritten : 2589070715314 | 20260505 15:39:56.724 | INFO | NumEvents : 1229757065212 | 20260505 15:39:58.424 | INFO | Analyzing chunk frames = [4354048,4354560) | 20260505 15:39:58.424 | INFO | MemInfo: { | 20260505 15:39:58.424 | INFO | "gpu" : " 17 GB", | 20260505 15:39:58.424 | INFO | "huge" : "", | 20260505 15:39:58.424 | INFO | "huge_shared" : " 27 GB", | 20260505 15:39:58.424 | INFO | "malloc" : " 14 GB", | 20260505 15:39:58.424 | INFO | "pinned" : " 29 GB" | 20260505 15:39:58.424 | INFO | } | 20260505 15:39:58.424 | INFO | Perf report for the last 5.15174 seconds: | 20260505 15:39:58.424 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:39:58.424 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:58.424 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:58.424 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:58.424 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 15:39:58.424 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:39:58.424 | INFO | RT_METRICS : 46.4%, 1.0, 24.4ms, 0 | 20260505 15:39:58.424 | INFO | PRE_HQ : 60.5%, 1.8, 57.0ms, 0 | 20260505 15:39:58.424 | INFO | BAZWRITER : 63.7%, 3.9, 127.1ms, 0 | 20260505 15:39:58.424 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 15:39:58.424 | INFO | Uploaded 36GiB to GPU in 2.19958s (16.3668GiB/s) | 20260505 15:39:58.424 | INFO | Downloaded 38GiB to GPU in 2.26936s (16.7448GiB/s) | 20260505 15:40:03.566 | INFO | Analyzing chunk frames = [4354560,4355072) | 20260505 15:40:03.566 | INFO | MemInfo: { | 20260505 15:40:03.566 | INFO | "gpu" : " 17 GB", | 20260505 15:40:03.567 | INFO | "huge" : "", | 20260505 15:40:03.567 | INFO | "huge_shared" : " 27 GB", | 20260505 15:40:03.567 | INFO | "malloc" : " 14 GB", | 20260505 15:40:03.567 | INFO | "pinned" : " 29 GB" | 20260505 15:40:03.567 | INFO | } | 20260505 15:40:03.567 | INFO | Perf report for the last 5.14263 seconds: | 20260505 15:40:03.567 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:40:03.567 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:03.567 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:03.567 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:03.567 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 15:40:03.567 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:03.567 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:03.567 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 15:40:03.567 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:03.567 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:03.567 | INFO | Uploaded 36GiB to GPU in 2.20588s (16.32GiB/s) | 20260505 15:40:03.567 | INFO | Downloaded 34GiB to GPU in 2.06515s (16.4637GiB/s) | 20260505 15:40:08.701 | INFO | Analyzing chunk frames = [4355072,4355584) | 20260505 15:40:08.701 | INFO | MemInfo: { | 20260505 15:40:08.701 | INFO | "gpu" : " 17 GB", | 20260505 15:40:08.701 | INFO | "huge" : "", | 20260505 15:40:08.701 | INFO | "huge_shared" : " 27 GB", | 20260505 15:40:08.701 | INFO | "malloc" : " 14 GB", | 20260505 15:40:08.701 | INFO | "pinned" : " 29 GB" | 20260505 15:40:08.701 | INFO | } | 20260505 15:40:08.701 | INFO | Perf report for the last 5.13419 seconds: | 20260505 15:40:08.701 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:40:08.701 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:08.701 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:08.701 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:08.701 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260505 15:40:08.701 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:08.701 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:08.701 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 15:40:08.701 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:08.701 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:08.701 | INFO | Uploaded 36GiB to GPU in 2.19782s (16.3799GiB/s) | 20260505 15:40:08.701 | INFO | Downloaded 34GiB to GPU in 2.02753s (16.7692GiB/s) | 20260505 15:40:13.836 | INFO | Analyzing chunk frames = [4355584,4356096) | 20260505 15:40:13.836 | INFO | MemInfo: { | 20260505 15:40:13.836 | INFO | "gpu" : " 17 GB", | 20260505 15:40:13.836 | INFO | "huge" : "", | 20260505 15:40:13.836 | INFO | "huge_shared" : " 27 GB", | 20260505 15:40:13.836 | INFO | "malloc" : " 14 GB", | 20260505 15:40:13.836 | INFO | "pinned" : " 29 GB" | 20260505 15:40:13.836 | INFO | } | 20260505 15:40:13.836 | INFO | Perf report for the last 5.13485 seconds: | 20260505 15:40:13.836 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:40:13.836 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:13.836 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:13.836 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:13.836 | INFO | ANALYSIS : 62.6%, 3.0, 97.1ms, 0 | 20260505 15:40:13.836 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:13.836 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:13.836 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:40:13.836 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:13.836 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:13.836 | INFO | Uploaded 36GiB to GPU in 2.18896s (16.4462GiB/s) | 20260505 15:40:13.836 | INFO | Downloaded 34GiB to GPU in 2.13314s (15.9389GiB/s) | 20260505 15:40:18.978 | INFO | Analyzing chunk frames = [4356096,4356608) | 20260505 15:40:18.978 | INFO | MemInfo: { | 20260505 15:40:18.978 | INFO | "gpu" : " 17 GB", | 20260505 15:40:18.978 | INFO | "huge" : "", | 20260505 15:40:18.978 | INFO | "huge_shared" : " 27 GB", | 20260505 15:40:18.978 | INFO | "malloc" : " 14 GB", | 20260505 15:40:18.978 | INFO | "pinned" : " 29 GB" | 20260505 15:40:18.978 | INFO | } | 20260505 15:40:18.978 | INFO | Perf report for the last 5.14225 seconds: | 20260505 15:40:18.978 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:40:18.978 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:18.978 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:18.978 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:18.978 | INFO | ANALYSIS : 62.6%, 3.0, 97.9ms, 0 | 20260505 15:40:18.978 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:18.978 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:18.978 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 15:40:18.978 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:18.978 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:18.978 | INFO | Uploaded 36GiB to GPU in 2.17313s (16.566GiB/s) | 20260505 15:40:18.978 | INFO | Downloaded 34GiB to GPU in 2.13121s (15.9534GiB/s) | 20260505 15:40:24.119 | INFO | Analyzing chunk frames = [4356608,4357120) | 20260505 15:40:24.119 | INFO | MemInfo: { | 20260505 15:40:24.119 | INFO | "gpu" : " 17 GB", | 20260505 15:40:24.119 | INFO | "huge" : "", | 20260505 15:40:24.119 | INFO | "huge_shared" : " 27 GB", | 20260505 15:40:24.119 | INFO | "malloc" : " 14 GB", | 20260505 15:40:24.119 | INFO | "pinned" : " 29 GB" | 20260505 15:40:24.119 | INFO | } | 20260505 15:40:24.119 | INFO | Perf report for the last 5.14127 seconds: | 20260505 15:40:24.119 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:40:24.119 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:24.119 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:24.119 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:24.119 | INFO | ANALYSIS : 62.5%, 3.0, 97.7ms, 0 | 20260505 15:40:24.119 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:24.119 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:24.119 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 15:40:24.119 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:24.119 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:24.120 | INFO | Uploaded 36GiB to GPU in 2.14451s (16.787GiB/s) | 20260505 15:40:24.120 | INFO | Downloaded 34GiB to GPU in 2.02958s (16.7522GiB/s) | 20260505 15:40:29.259 | INFO | Analyzing chunk frames = [4357120,4357632) | 20260505 15:40:29.259 | INFO | MemInfo: { | 20260505 15:40:29.259 | INFO | "gpu" : " 17 GB", | 20260505 15:40:29.259 | INFO | "huge" : "", | 20260505 15:40:29.259 | INFO | "huge_shared" : " 27 GB", | 20260505 15:40:29.259 | INFO | "malloc" : " 14 GB", | 20260505 15:40:29.259 | INFO | "pinned" : " 29 GB" | 20260505 15:40:29.259 | INFO | } | 20260505 15:40:29.259 | INFO | Perf report for the last 5.14013 seconds: | 20260505 15:40:29.259 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:40:29.259 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:29.259 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:29.259 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:29.260 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 15:40:29.260 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:29.260 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:29.260 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 15:40:29.260 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:29.260 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:29.260 | INFO | Uploaded 36GiB to GPU in 2.14165s (16.8094GiB/s) | 20260505 15:40:29.260 | INFO | Downloaded 34GiB to GPU in 2.16831s (15.6804GiB/s) | 20260505 15:40:34.391 | INFO | Analyzing chunk frames = [4357632,4358144) | 20260505 15:40:34.391 | INFO | MemInfo: { | 20260505 15:40:34.391 | INFO | "gpu" : " 17 GB", | 20260505 15:40:34.391 | INFO | "huge" : "", | 20260505 15:40:34.391 | INFO | "huge_shared" : " 27 GB", | 20260505 15:40:34.391 | INFO | "malloc" : " 14 GB", | 20260505 15:40:34.391 | INFO | "pinned" : " 29 GB" | 20260505 15:40:34.391 | INFO | } | 20260505 15:40:34.392 | INFO | Perf report for the last 5.13206 seconds: | 20260505 15:40:34.392 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:40:34.392 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:34.392 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:34.392 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:34.392 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 15:40:34.392 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:34.392 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:34.392 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 15:40:34.392 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:34.392 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:34.392 | INFO | Uploaded 36GiB to GPU in 2.17331s (16.5646GiB/s) | 20260505 15:40:34.392 | INFO | Downloaded 34GiB to GPU in 2.0333s (16.7216GiB/s) | 20260505 15:40:34.392 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 15:40:34.392 | INFO | Total mem usage: 15.3 GiB | 20260505 15:40:34.392 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 15:40:34.392 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.85%) | 20260505 15:40:34.392 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 15:40:34.392 | INFO | High Water Breakdown: | 20260505 15:40:34.392 | INFO | 61% : 8.67 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 15:40:34.392 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 15:40:34.392 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 15:40:34.392 | INFO | ------------------------------------------ | 20260505 15:40:34.392 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:40:34.392 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 15:40:34.392 | INFO | Total mem usage: 26.5 GiB | 20260505 15:40:34.392 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 15:40:34.392 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 15:40:34.392 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 15:40:34.392 | INFO | High Water Breakdown: | 20260505 15:40:34.392 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:40:34.392 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:40:34.392 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:40:34.392 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:40:34.392 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:40:34.392 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:40:34.392 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 15:40:34.392 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 15:40:34.392 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:40:34.392 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 15:40:34.392 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 15:40:34.392 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 15:40:34.392 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 15:40:34.392 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 15:40:34.392 | INFO | ------------------------------------------ | 20260505 15:40:34.393 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:40:34.393 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 15:40:34.393 | INFO | Total mem usage: 25.6 GiB | 20260505 15:40:34.393 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 15:40:34.393 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 15:40:34.393 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 15:40:34.393 | INFO | High Water Breakdown: | 20260505 15:40:34.393 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 15:40:34.393 | INFO | ------------------------------------------ | 20260505 15:40:34.393 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:40:34.393 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 15:40:34.393 | INFO | Total mem usage: 15.7 GiB | 20260505 15:40:34.393 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 15:40:34.393 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 15:40:34.393 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 15:40:34.393 | INFO | High Water Breakdown: | 20260505 15:40:34.393 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 15:40:34.393 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 15:40:34.393 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 15:40:34.393 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 15:40:34.393 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 15:40:34.393 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 15:40:34.393 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 15:40:34.393 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 15:40:34.393 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 15:40:34.393 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:40:34.393 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 15:40:34.393 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:40:34.393 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 15:40:34.393 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 15:40:34.393 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 15:40:34.393 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 15:40:34.393 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:40:34.393 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 15:40:34.393 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:40:34.393 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:40:34.393 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:40:34.393 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 15:40:34.393 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 15:40:34.393 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:40:34.393 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 15:40:34.393 | INFO | ------------------------------------------ | 20260505 15:40:34.393 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:40:34.393 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 15:40:34.393 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:40:34.393 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:40:34.393 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.57e+01, 1.36e+01 / 2.15e+00 | 20260505 15:40:34.393 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 1.01e-01, 1.12e-02 / 5.31e-03 | 20260505 15:40:34.393 | INFO | PerfReport: 11.78%, Baseline -- 3.74e+00 / 4.45e+00, 5.80e-01 / 2.40e-01 | 20260505 15:40:34.393 | INFO | PerfReport: 3.29%, Histogram -- 1.04e+00 / 1.48e+00, 1.93e-01 / 1.23e-01 | 20260505 15:40:34.393 | INFO | PerfReport: 15.90%, DME -- 4.04e+01 / 4.49e+01, 4.46e+00 / 2.31e+00 | 20260505 15:40:34.393 | INFO | PerfReport: 56.73%, FrameLabeling -- 1.80e+01 / 2.00e+01, 1.51e+00 / 1.91e+00 | 20260505 15:40:34.393 | INFO | PerfReport: 7.35%, PulseAccumulating -- 2.34e+00 / 2.90e+00, 4.43e-01 / 1.83e-01 | 20260505 15:40:34.393 | INFO | PerfReport: 3.45%, Metrics -- 1.10e+00 / 1.43e+00, 3.06e-01 / 1.64e-01 | 20260505 15:40:34.393 | INFO | PerfReport: 1.20%, PulseSerializing -- 3.82e-01 / 4.37e-01, 6.89e-02 / 3.16e-02 | 20260505 15:40:34.393 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 15:40:34.393 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:40:34.393 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:40:34.393 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260505 15:40:34.393 | INFO | PerfReport: 22.53%, Upload -- 2.26e+01 / 0.00e+00, 7.84e+00 / 0.00e+00 | 20260505 15:40:34.393 | INFO | PerfReport: 21.92%, Download -- 2.20e+01 / 0.00e+00, 7.94e+00 / 0.00e+00 | 20260505 15:40:34.393 | INFO | PerfReport: 31.64%, Compute -- 3.18e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 15:40:34.393 | INFO | PerfReport: 2.19%, LockUpload -- 2.20e+00 / 0.00e+00, 7.81e+00 / 0.00e+00 | 20260505 15:40:34.393 | INFO | PerfReport: 1.46%, LockDownload -- 1.46e+00 / 0.00e+00, 3.03e+00 / 0.00e+00 | 20260505 15:40:34.393 | INFO | PerfReport: 20.25%, LockCompute -- 2.03e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 15:40:34.393 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 2.09e-02 / 0.00e+00 | 20260505 15:40:34.528 | INFO | Starting a round of Baz IO | 20260505 15:40:37.745 | INFO | Populating RTMetrics report | 20260505 15:40:37.746 | INFO | RTMetrics report populated | 20260505 15:40:37.746 | INFO | Opening tmp RTMetrics json file | 20260505 15:40:37.760 | INFO | Serializing RTMetrics to json | 20260505 15:40:37.761 | INFO | Serialization done, swapping tmp file now... | 20260505 15:40:37.762 | INFO | RTMetrics done | 20260505 15:40:37.851 | INFO | Finished a round of Baz IO: Wrote 2.203 GiB in 3.322 seconds | 20260505 15:40:37.851 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.42% / 0.00% / 77.58% | 20260505 15:40:37.851 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.427063% / 1.19209e-05% / 7.15256e-05% | 20260505 15:40:37.851 | INFO | Baz IO overall: | 20260505 15:40:37.851 | INFO | BytesWritten : 2591436208636 | 20260505 15:40:37.851 | INFO | NumEvents : 1230868665330 | 20260505 15:40:39.535 | INFO | Analyzing chunk frames = [4358144,4358656) | 20260505 15:40:39.535 | INFO | MemInfo: { | 20260505 15:40:39.535 | INFO | "gpu" : " 17 GB", | 20260505 15:40:39.535 | INFO | "huge" : "", | 20260505 15:40:39.535 | INFO | "huge_shared" : " 27 GB", | 20260505 15:40:39.535 | INFO | "malloc" : " 14 GB", | 20260505 15:40:39.535 | INFO | "pinned" : " 29 GB" | 20260505 15:40:39.535 | INFO | } | 20260505 15:40:39.535 | INFO | Perf report for the last 5.14323 seconds: | 20260505 15:40:39.535 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:40:39.535 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:39.535 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:39.535 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:39.535 | INFO | ANALYSIS : 64.1%, 3.0, 100.0ms, 0 | 20260505 15:40:39.535 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:39.535 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260505 15:40:39.535 | INFO | PRE_HQ : 60.7%, 1.8, 57.0ms, 0 | 20260505 15:40:39.535 | INFO | BAZWRITER : 63.9%, 3.9, 128.2ms, 0 | 20260505 15:40:39.535 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 15:40:39.535 | INFO | Uploaded 36GiB to GPU in 2.19388s (16.4093GiB/s) | 20260505 15:40:39.535 | INFO | Downloaded 38GiB to GPU in 2.26402s (16.7843GiB/s) | 20260505 15:40:44.686 | INFO | Analyzing chunk frames = [4358656,4359168) | 20260505 15:40:44.686 | INFO | MemInfo: { | 20260505 15:40:44.686 | INFO | "gpu" : " 17 GB", | 20260505 15:40:44.686 | INFO | "huge" : "", | 20260505 15:40:44.686 | INFO | "huge_shared" : " 27 GB", | 20260505 15:40:44.686 | INFO | "malloc" : " 14 GB", | 20260505 15:40:44.686 | INFO | "pinned" : " 29 GB" | 20260505 15:40:44.686 | INFO | } | 20260505 15:40:44.687 | INFO | Perf report for the last 5.15176 seconds: | 20260505 15:40:44.687 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:40:44.687 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:44.687 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:44.687 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:44.687 | INFO | ANALYSIS : 63.3%, 3.0, 99.0ms, 0 | 20260505 15:40:44.687 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:44.687 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:44.687 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 15:40:44.687 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:44.687 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:44.687 | INFO | Uploaded 36GiB to GPU in 2.20625s (16.3173GiB/s) | 20260505 15:40:44.687 | INFO | Downloaded 34GiB to GPU in 2.06651s (16.4528GiB/s) | 20260505 15:40:49.818 | INFO | Analyzing chunk frames = [4359168,4359680) | 20260505 15:40:49.818 | INFO | MemInfo: { | 20260505 15:40:49.818 | INFO | "gpu" : " 17 GB", | 20260505 15:40:49.818 | INFO | "huge" : "", | 20260505 15:40:49.818 | INFO | "huge_shared" : " 27 GB", | 20260505 15:40:49.819 | INFO | "malloc" : " 14 GB", | 20260505 15:40:49.819 | INFO | "pinned" : " 29 GB" | 20260505 15:40:49.819 | INFO | } | 20260505 15:40:49.819 | INFO | Perf report for the last 5.13204 seconds: | 20260505 15:40:49.819 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:40:49.819 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:49.819 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:49.819 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:49.819 | INFO | ANALYSIS : 62.9%, 3.0, 97.7ms, 0 | 20260505 15:40:49.819 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:49.819 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:49.819 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 15:40:49.819 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:49.819 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:49.819 | INFO | Uploaded 36GiB to GPU in 2.20832s (16.302GiB/s) | 20260505 15:40:49.819 | INFO | Downloaded 34GiB to GPU in 2.06383s (16.4742GiB/s) | 20260505 15:40:54.951 | INFO | Analyzing chunk frames = [4359680,4360192) | 20260505 15:40:54.951 | INFO | MemInfo: { | 20260505 15:40:54.951 | INFO | "gpu" : " 17 GB", | 20260505 15:40:54.951 | INFO | "huge" : "", | 20260505 15:40:54.951 | INFO | "huge_shared" : " 27 GB", | 20260505 15:40:54.951 | INFO | "malloc" : " 14 GB", | 20260505 15:40:54.951 | INFO | "pinned" : " 29 GB" | 20260505 15:40:54.951 | INFO | } | 20260505 15:40:54.951 | INFO | Perf report for the last 5.13246 seconds: | 20260505 15:40:54.951 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:40:54.951 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:54.951 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:54.951 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:54.951 | INFO | ANALYSIS : 62.7%, 3.0, 97.3ms, 0 | 20260505 15:40:54.951 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:54.951 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:54.951 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:40:54.951 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:54.951 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:40:54.951 | INFO | Uploaded 36GiB to GPU in 2.17262s (16.5698GiB/s) | 20260505 15:40:54.951 | INFO | Downloaded 34GiB to GPU in 2.14009s (15.8872GiB/s) | 20260505 15:41:00.091 | INFO | Analyzing chunk frames = [4360192,4360704) | 20260505 15:41:00.091 | INFO | MemInfo: { | 20260505 15:41:00.091 | INFO | "gpu" : " 17 GB", | 20260505 15:41:00.091 | INFO | "huge" : "", | 20260505 15:41:00.091 | INFO | "huge_shared" : " 27 GB", | 20260505 15:41:00.092 | INFO | "malloc" : " 14 GB", | 20260505 15:41:00.092 | INFO | "pinned" : " 29 GB" | 20260505 15:41:00.092 | INFO | } | 20260505 15:41:00.092 | INFO | Perf report for the last 5.14053 seconds: | 20260505 15:41:00.092 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:41:00.092 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:00.092 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:00.092 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:00.092 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260505 15:41:00.092 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:00.092 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:00.092 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:41:00.092 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:00.092 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:00.092 | INFO | Uploaded 36GiB to GPU in 2.1373s (16.8437GiB/s) | 20260505 15:41:00.092 | INFO | Downloaded 34GiB to GPU in 2.12679s (15.9865GiB/s) | 20260505 15:41:05.231 | INFO | Analyzing chunk frames = [4360704,4361216) | 20260505 15:41:05.231 | INFO | MemInfo: { | 20260505 15:41:05.231 | INFO | "gpu" : " 17 GB", | 20260505 15:41:05.231 | INFO | "huge" : "", | 20260505 15:41:05.231 | INFO | "huge_shared" : " 27 GB", | 20260505 15:41:05.231 | INFO | "malloc" : " 14 GB", | 20260505 15:41:05.232 | INFO | "pinned" : " 29 GB" | 20260505 15:41:05.232 | INFO | } | 20260505 15:41:05.232 | INFO | Perf report for the last 5.1401 seconds: | 20260505 15:41:05.232 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:41:05.232 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:05.232 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:05.232 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:05.232 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 15:41:05.232 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:05.232 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:05.232 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:41:05.232 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:05.232 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:05.232 | INFO | Uploaded 36GiB to GPU in 2.21164s (16.2775GiB/s) | 20260505 15:41:05.232 | INFO | Downloaded 34GiB to GPU in 2.11185s (16.0996GiB/s) | 20260505 15:41:10.374 | INFO | Analyzing chunk frames = [4361216,4361728) | 20260505 15:41:10.374 | INFO | MemInfo: { | 20260505 15:41:10.374 | INFO | "gpu" : " 17 GB", | 20260505 15:41:10.375 | INFO | "huge" : "", | 20260505 15:41:10.375 | INFO | "huge_shared" : " 27 GB", | 20260505 15:41:10.375 | INFO | "malloc" : " 14 GB", | 20260505 15:41:10.375 | INFO | "pinned" : " 29 GB" | 20260505 15:41:10.375 | INFO | } | 20260505 15:41:10.375 | INFO | Perf report for the last 5.14303 seconds: | 20260505 15:41:10.375 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:41:10.375 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:10.375 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:10.375 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:10.375 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 15:41:10.375 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:10.375 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:10.375 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 15:41:10.375 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:10.375 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:10.375 | INFO | Uploaded 36GiB to GPU in 2.11032s (17.0591GiB/s) | 20260505 15:41:10.375 | INFO | Downloaded 34GiB to GPU in 2.17164s (15.6564GiB/s) | 20260505 15:41:15.515 | INFO | Analyzing chunk frames = [4361728,4362240) | 20260505 15:41:15.515 | INFO | MemInfo: { | 20260505 15:41:15.515 | INFO | "gpu" : " 17 GB", | 20260505 15:41:15.515 | INFO | "huge" : "", | 20260505 15:41:15.515 | INFO | "huge_shared" : " 27 GB", | 20260505 15:41:15.515 | INFO | "malloc" : " 14 GB", | 20260505 15:41:15.515 | INFO | "pinned" : " 29 GB" | 20260505 15:41:15.515 | INFO | } | 20260505 15:41:15.515 | INFO | Perf report for the last 5.14071 seconds: | 20260505 15:41:15.515 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:41:15.515 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:15.516 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:15.516 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:15.516 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 15:41:15.516 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:15.516 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:15.516 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 15:41:15.516 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:15.516 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:15.516 | INFO | Uploaded 36GiB to GPU in 2.23545s (16.1042GiB/s) | 20260505 15:41:15.516 | INFO | Downloaded 34GiB to GPU in 2.10549s (16.1482GiB/s) | 20260505 15:41:15.649 | INFO | Starting a round of Baz IO | 20260505 15:41:18.849 | INFO | Populating RTMetrics report | 20260505 15:41:18.849 | INFO | RTMetrics report populated | 20260505 15:41:18.849 | INFO | Opening tmp RTMetrics json file | 20260505 15:41:18.892 | INFO | Serializing RTMetrics to json | 20260505 15:41:18.933 | INFO | Serialization done, swapping tmp file now... | 20260505 15:41:18.937 | INFO | RTMetrics done | 20260505 15:41:18.952 | INFO | Finished a round of Baz IO: Wrote 2.202 GiB in 3.303 seconds | 20260505 15:41:18.952 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.41% / 0.00% / 77.58% | 20260505 15:41:18.952 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.425017% / 3.97364e-06% / 5.96046e-05% | 20260505 15:41:18.952 | INFO | Baz IO overall: | 20260505 15:41:18.953 | INFO | BytesWritten : 2593800860679 | 20260505 15:41:18.953 | INFO | NumEvents : 1231979537938 | 20260505 15:41:20.649 | INFO | Analyzing chunk frames = [4362240,4362752) | 20260505 15:41:20.649 | INFO | MemInfo: { | 20260505 15:41:20.649 | INFO | "gpu" : " 17 GB", | 20260505 15:41:20.649 | INFO | "huge" : "", | 20260505 15:41:20.650 | INFO | "huge_shared" : " 27 GB", | 20260505 15:41:20.650 | INFO | "malloc" : " 14 GB", | 20260505 15:41:20.650 | INFO | "pinned" : " 29 GB" | 20260505 15:41:20.650 | INFO | } | 20260505 15:41:20.650 | INFO | Perf report for the last 5.13423 seconds: | 20260505 15:41:20.650 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:41:20.650 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:20.650 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:20.650 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:20.650 | INFO | ANALYSIS : 63.8%, 3.0, 99.2ms, 0 | 20260505 15:41:20.650 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:20.650 | INFO | RT_METRICS : 46.5%, 1.0, 24.4ms, 0 | 20260505 15:41:20.650 | INFO | PRE_HQ : 61.0%, 1.8, 56.9ms, 0 | 20260505 15:41:20.650 | INFO | BAZWRITER : 63.7%, 3.9, 127.3ms, 0 | 20260505 15:41:20.650 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 15:41:20.650 | INFO | Uploaded 36GiB to GPU in 2.1472s (16.766GiB/s) | 20260505 15:41:20.650 | INFO | Downloaded 38GiB to GPU in 2.24262s (16.9445GiB/s) | 20260505 15:41:25.786 | INFO | Analyzing chunk frames = [4362752,4363264) | 20260505 15:41:25.786 | INFO | MemInfo: { | 20260505 15:41:25.786 | INFO | "gpu" : " 17 GB", | 20260505 15:41:25.786 | INFO | "huge" : "", | 20260505 15:41:25.786 | INFO | "huge_shared" : " 27 GB", | 20260505 15:41:25.786 | INFO | "malloc" : " 14 GB", | 20260505 15:41:25.786 | INFO | "pinned" : " 29 GB" | 20260505 15:41:25.786 | INFO | } | 20260505 15:41:25.786 | INFO | Perf report for the last 5.13648 seconds: | 20260505 15:41:25.786 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:41:25.786 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:25.786 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:25.786 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:25.786 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 15:41:25.786 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:25.786 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:25.786 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:41:25.786 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:25.786 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:25.786 | INFO | Uploaded 36GiB to GPU in 2.2062s (16.3177GiB/s) | 20260505 15:41:25.787 | INFO | Downloaded 34GiB to GPU in 2.054s (16.5531GiB/s) | 20260505 15:41:30.928 | INFO | Analyzing chunk frames = [4363264,4363776) | 20260505 15:41:30.928 | INFO | MemInfo: { | 20260505 15:41:30.928 | INFO | "gpu" : " 17 GB", | 20260505 15:41:30.928 | INFO | "huge" : "", | 20260505 15:41:30.928 | INFO | "huge_shared" : " 27 GB", | 20260505 15:41:30.928 | INFO | "malloc" : " 14 GB", | 20260505 15:41:30.928 | INFO | "pinned" : " 29 GB" | 20260505 15:41:30.928 | INFO | } | 20260505 15:41:30.928 | INFO | Perf report for the last 5.14224 seconds: | 20260505 15:41:30.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:41:30.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:30.928 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:30.929 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:30.929 | INFO | ANALYSIS : 63.0%, 3.0, 98.0ms, 0 | 20260505 15:41:30.929 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:30.929 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:30.929 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:41:30.929 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:30.929 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:30.929 | INFO | Uploaded 36GiB to GPU in 2.19519s (16.3995GiB/s) | 20260505 15:41:30.929 | INFO | Downloaded 34GiB to GPU in 2.04926s (16.5913GiB/s) | 20260505 15:41:36.069 | INFO | Analyzing chunk frames = [4363776,4364288) | 20260505 15:41:36.069 | INFO | MemInfo: { | 20260505 15:41:36.069 | INFO | "gpu" : " 17 GB", | 20260505 15:41:36.069 | INFO | "huge" : "", | 20260505 15:41:36.069 | INFO | "huge_shared" : " 27 GB", | 20260505 15:41:36.069 | INFO | "malloc" : " 14 GB", | 20260505 15:41:36.069 | INFO | "pinned" : " 29 GB" | 20260505 15:41:36.069 | INFO | } | 20260505 15:41:36.070 | INFO | Perf report for the last 5.14108 seconds: | 20260505 15:41:36.070 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:41:36.070 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:36.070 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:36.070 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:36.070 | INFO | ANALYSIS : 62.5%, 3.0, 97.2ms, 0 | 20260505 15:41:36.070 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:36.070 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:36.070 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:41:36.070 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:36.070 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:36.070 | INFO | Uploaded 36GiB to GPU in 2.1899s (16.4391GiB/s) | 20260505 15:41:36.070 | INFO | Downloaded 34GiB to GPU in 2.13437s (15.9298GiB/s) | 20260505 15:41:41.214 | INFO | Analyzing chunk frames = [4364288,4364800) | 20260505 15:41:41.214 | INFO | MemInfo: { | 20260505 15:41:41.214 | INFO | "gpu" : " 17 GB", | 20260505 15:41:41.214 | INFO | "huge" : "", | 20260505 15:41:41.214 | INFO | "huge_shared" : " 27 GB", | 20260505 15:41:41.214 | INFO | "malloc" : " 14 GB", | 20260505 15:41:41.214 | INFO | "pinned" : " 29 GB" | 20260505 15:41:41.214 | INFO | } | 20260505 15:41:41.214 | INFO | Perf report for the last 5.1448 seconds: | 20260505 15:41:41.214 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:41:41.214 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:41.214 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:41.214 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:41.214 | INFO | ANALYSIS : 62.6%, 3.0, 97.9ms, 0 | 20260505 15:41:41.214 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:41.214 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:41.214 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 15:41:41.214 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:41.214 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:41.215 | INFO | Uploaded 36GiB to GPU in 2.18562s (16.4713GiB/s) | 20260505 15:41:41.215 | INFO | Downloaded 34GiB to GPU in 2.12959s (15.9655GiB/s) | 20260505 15:41:46.344 | INFO | Analyzing chunk frames = [4364800,4365312) | 20260505 15:41:46.344 | INFO | MemInfo: { | 20260505 15:41:46.344 | INFO | "gpu" : " 17 GB", | 20260505 15:41:46.344 | INFO | "huge" : "", | 20260505 15:41:46.344 | INFO | "huge_shared" : " 27 GB", | 20260505 15:41:46.344 | INFO | "malloc" : " 14 GB", | 20260505 15:41:46.344 | INFO | "pinned" : " 29 GB" | 20260505 15:41:46.344 | INFO | } | 20260505 15:41:46.344 | INFO | Perf report for the last 5.12953 seconds: | 20260505 15:41:46.344 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:41:46.344 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:46.344 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:46.344 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:46.344 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 15:41:46.344 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:46.344 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:46.344 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 15:41:46.344 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:46.344 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:46.344 | INFO | Uploaded 36GiB to GPU in 2.21338s (16.2647GiB/s) | 20260505 15:41:46.344 | INFO | Downloaded 34GiB to GPU in 2.11219s (16.0971GiB/s) | 20260505 15:41:51.486 | INFO | Analyzing chunk frames = [4365312,4365824) | 20260505 15:41:51.486 | INFO | MemInfo: { | 20260505 15:41:51.486 | INFO | "gpu" : " 17 GB", | 20260505 15:41:51.486 | INFO | "huge" : "", | 20260505 15:41:51.486 | INFO | "huge_shared" : " 27 GB", | 20260505 15:41:51.486 | INFO | "malloc" : " 14 GB", | 20260505 15:41:51.486 | INFO | "pinned" : " 29 GB" | 20260505 15:41:51.486 | INFO | } | 20260505 15:41:51.486 | INFO | Perf report for the last 5.14216 seconds: | 20260505 15:41:51.486 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:41:51.486 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:51.486 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:51.486 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:51.486 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 15:41:51.486 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:51.486 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:51.486 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 15:41:51.486 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:51.486 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:51.486 | INFO | Uploaded 36GiB to GPU in 2.1624s (16.6482GiB/s) | 20260505 15:41:51.486 | INFO | Downloaded 34GiB to GPU in 2.15159s (15.8023GiB/s) | 20260505 15:41:56.632 | INFO | Analyzing chunk frames = [4365824,4366336) | 20260505 15:41:56.632 | INFO | MemInfo: { | 20260505 15:41:56.632 | INFO | "gpu" : " 17 GB", | 20260505 15:41:56.632 | INFO | "huge" : "", | 20260505 15:41:56.632 | INFO | "huge_shared" : " 27 GB", | 20260505 15:41:56.632 | INFO | "malloc" : " 14 GB", | 20260505 15:41:56.632 | INFO | "pinned" : " 29 GB" | 20260505 15:41:56.632 | INFO | } | 20260505 15:41:56.632 | INFO | Perf report for the last 5.14552 seconds: | 20260505 15:41:56.632 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:41:56.632 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:56.632 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:56.632 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:56.632 | INFO | ANALYSIS : 63.2%, 3.0, 98.8ms, 0 | 20260505 15:41:56.632 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:56.632 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:56.632 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:41:56.632 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:56.632 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:41:56.632 | INFO | Uploaded 36GiB to GPU in 2.17612s (16.5432GiB/s) | 20260505 15:41:56.632 | INFO | Downloaded 34GiB to GPU in 2.03193s (16.7329GiB/s) | 20260505 15:41:56.632 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 15:41:56.632 | INFO | Total mem usage: 15.3 GiB | 20260505 15:41:56.632 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 15:41:56.632 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.83%) | 20260505 15:41:56.632 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 15:41:56.632 | INFO | High Water Breakdown: | 20260505 15:41:56.632 | INFO | 61% : 8.68 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 15:41:56.632 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 15:41:56.632 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 15:41:56.632 | INFO | ------------------------------------------ | 20260505 15:41:56.632 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:41:56.632 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 15:41:56.632 | INFO | Total mem usage: 26.5 GiB | 20260505 15:41:56.632 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 15:41:56.632 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 15:41:56.632 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 15:41:56.632 | INFO | High Water Breakdown: | 20260505 15:41:56.632 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:41:56.632 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:41:56.632 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:41:56.632 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:41:56.632 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:41:56.632 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:41:56.633 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 15:41:56.633 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 15:41:56.633 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:41:56.633 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 15:41:56.633 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 15:41:56.633 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 15:41:56.633 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 15:41:56.633 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 15:41:56.633 | INFO | ------------------------------------------ | 20260505 15:41:56.633 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:41:56.633 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 15:41:56.633 | INFO | Total mem usage: 25.6 GiB | 20260505 15:41:56.633 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 15:41:56.633 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 15:41:56.633 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 15:41:56.633 | INFO | High Water Breakdown: | 20260505 15:41:56.633 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 15:41:56.633 | INFO | ------------------------------------------ | 20260505 15:41:56.633 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:41:56.633 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 15:41:56.633 | INFO | Total mem usage: 15.7 GiB | 20260505 15:41:56.633 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 15:41:56.633 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 15:41:56.633 | INFO | Unused Cache: 896 MiB (Waste: 5.58%) | 20260505 15:41:56.633 | INFO | High Water Breakdown: | 20260505 15:41:56.633 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 15:41:56.633 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 15:41:56.633 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 15:41:56.633 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 15:41:56.633 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 15:41:56.633 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 15:41:56.633 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 15:41:56.633 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 15:41:56.633 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 15:41:56.633 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:41:56.633 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 15:41:56.633 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:41:56.633 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 15:41:56.633 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 15:41:56.633 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 15:41:56.633 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 15:41:56.633 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:41:56.633 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 15:41:56.633 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:41:56.633 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:41:56.633 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:41:56.633 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 15:41:56.633 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 15:41:56.633 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:41:56.633 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 15:41:56.633 | INFO | ------------------------------------------ | 20260505 15:41:56.633 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:41:56.633 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 15:41:56.633 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:41:56.633 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:41:56.633 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.58e+01, 1.35e+01 / 1.77e+00 | 20260505 15:41:56.633 | INFO | PerfReport: 0.29%, AggregateStats -- 9.26e-02 / 9.97e-02, 1.11e-02 / 6.30e-03 | 20260505 15:41:56.633 | INFO | PerfReport: 11.76%, Baseline -- 3.72e+00 / 4.49e+00, 5.66e-01 / 1.78e-01 | 20260505 15:41:56.633 | INFO | PerfReport: 3.26%, Histogram -- 1.03e+00 / 1.50e+00, 1.89e-01 / 1.11e-01 | 20260505 15:41:56.633 | INFO | PerfReport: 15.98%, DME -- 4.05e+01 / 4.53e+01, 4.60e+00 / 1.95e+00 | 20260505 15:41:56.633 | INFO | PerfReport: 56.59%, FrameLabeling -- 1.79e+01 / 1.96e+01, 1.37e+00 / 1.52e+00 | 20260505 15:41:56.633 | INFO | PerfReport: 7.43%, PulseAccumulating -- 2.35e+00 / 2.92e+00, 4.56e-01 / 1.79e-01 | 20260505 15:41:56.633 | INFO | PerfReport: 3.46%, Metrics -- 1.10e+00 / 1.43e+00, 3.08e-01 / 1.77e-01 | 20260505 15:41:56.633 | INFO | PerfReport: 1.21%, PulseSerializing -- 3.83e-01 / 4.34e-01, 6.89e-02 / 3.45e-02 | 20260505 15:41:56.633 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 15:41:56.633 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:41:56.633 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:41:56.633 | INFO | PerfReport: Full Execution -- 1.01e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260505 15:41:56.633 | INFO | PerfReport: 22.63%, Upload -- 2.28e+01 / 0.00e+00, 8.11e+00 / 0.00e+00 | 20260505 15:41:56.633 | INFO | PerfReport: 21.99%, Download -- 2.21e+01 / 0.00e+00, 8.04e+00 / 0.00e+00 | 20260505 15:41:56.633 | INFO | PerfReport: 31.49%, Compute -- 3.17e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 15:41:56.633 | INFO | PerfReport: 2.41%, LockUpload -- 2.43e+00 / 0.00e+00, 8.03e+00 / 0.00e+00 | 20260505 15:41:56.633 | INFO | PerfReport: 1.38%, LockDownload -- 1.38e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260505 15:41:56.633 | INFO | PerfReport: 20.08%, LockCompute -- 2.02e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 15:41:56.633 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.72e-02 / 0.00e+00 | 20260505 15:41:56.769 | INFO | Starting a round of Baz IO | 20260505 15:41:59.959 | INFO | Populating RTMetrics report | 20260505 15:41:59.959 | INFO | RTMetrics report populated | 20260505 15:41:59.959 | INFO | Opening tmp RTMetrics json file | 20260505 15:41:59.971 | INFO | Serializing RTMetrics to json | 20260505 15:41:59.972 | INFO | Serialization done, swapping tmp file now... | 20260505 15:41:59.974 | INFO | RTMetrics done | 20260505 15:42:00.056 | INFO | Finished a round of Baz IO: Wrote 2.201 GiB in 3.286 seconds | 20260505 15:42:00.056 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.41% / 0.00% / 77.59% | 20260505 15:42:00.056 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.42363% / 7.94729e-06% / 6.75519e-05% | 20260505 15:42:00.056 | INFO | Baz IO overall: | 20260505 15:42:00.056 | INFO | BytesWritten : 2596164183086 | 20260505 15:42:00.056 | INFO | NumEvents : 1233090030476 | 20260505 15:42:01.768 | INFO | Analyzing chunk frames = [4366336,4366848) | 20260505 15:42:01.768 | INFO | MemInfo: { | 20260505 15:42:01.768 | INFO | "gpu" : " 17 GB", | 20260505 15:42:01.768 | INFO | "huge" : "", | 20260505 15:42:01.768 | INFO | "huge_shared" : " 27 GB", | 20260505 15:42:01.768 | INFO | "malloc" : " 14 GB", | 20260505 15:42:01.768 | INFO | "pinned" : " 29 GB" | 20260505 15:42:01.768 | INFO | } | 20260505 15:42:01.768 | INFO | Perf report for the last 5.13619 seconds: | 20260505 15:42:01.768 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:42:01.768 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:01.768 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:01.768 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:01.768 | INFO | ANALYSIS : 63.7%, 3.0, 99.0ms, 0 | 20260505 15:42:01.768 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:01.768 | INFO | RT_METRICS : 45.1%, 1.0, 23.7ms, 0 | 20260505 15:42:01.768 | INFO | PRE_HQ : 60.7%, 1.8, 56.9ms, 0 | 20260505 15:42:01.768 | INFO | BAZWRITER : 63.4%, 4.0, 128.8ms, 0 | 20260505 15:42:01.768 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 15:42:01.768 | INFO | Uploaded 36GiB to GPU in 2.16705s (16.6124GiB/s) | 20260505 15:42:01.768 | INFO | Downloaded 38GiB to GPU in 2.25894s (16.822GiB/s) | 20260505 15:42:06.896 | INFO | Analyzing chunk frames = [4366848,4367360) | 20260505 15:42:06.896 | INFO | MemInfo: { | 20260505 15:42:06.896 | INFO | "gpu" : " 17 GB", | 20260505 15:42:06.896 | INFO | "huge" : "", | 20260505 15:42:06.896 | INFO | "huge_shared" : " 27 GB", | 20260505 15:42:06.896 | INFO | "malloc" : " 14 GB", | 20260505 15:42:06.896 | INFO | "pinned" : " 29 GB" | 20260505 15:42:06.896 | INFO | } | 20260505 15:42:06.896 | INFO | Perf report for the last 5.12852 seconds: | 20260505 15:42:06.896 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:42:06.896 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:06.896 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:06.896 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:06.896 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 15:42:06.896 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:06.896 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:06.896 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 15:42:06.896 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:06.896 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:06.897 | INFO | Uploaded 36GiB to GPU in 2.20363s (16.3366GiB/s) | 20260505 15:42:06.897 | INFO | Downloaded 34GiB to GPU in 2.06687s (16.45GiB/s) | 20260505 15:42:12.038 | INFO | Analyzing chunk frames = [4367360,4367872) | 20260505 15:42:12.038 | INFO | MemInfo: { | 20260505 15:42:12.038 | INFO | "gpu" : " 17 GB", | 20260505 15:42:12.038 | INFO | "huge" : "", | 20260505 15:42:12.038 | INFO | "huge_shared" : " 27 GB", | 20260505 15:42:12.038 | INFO | "malloc" : " 14 GB", | 20260505 15:42:12.038 | INFO | "pinned" : " 29 GB" | 20260505 15:42:12.039 | INFO | } | 20260505 15:42:12.039 | INFO | Perf report for the last 5.14227 seconds: | 20260505 15:42:12.039 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:42:12.039 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:12.039 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:12.039 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:12.039 | INFO | ANALYSIS : 63.2%, 3.0, 98.4ms, 0 | 20260505 15:42:12.039 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:12.039 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:12.039 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:42:12.039 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:12.039 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:12.039 | INFO | Uploaded 36GiB to GPU in 2.22843s (16.1549GiB/s) | 20260505 15:42:12.039 | INFO | Downloaded 34GiB to GPU in 2.10884s (16.1226GiB/s) | 20260505 15:42:17.186 | INFO | Analyzing chunk frames = [4367872,4368384) | 20260505 15:42:17.186 | INFO | MemInfo: { | 20260505 15:42:17.186 | INFO | "gpu" : " 17 GB", | 20260505 15:42:17.186 | INFO | "huge" : "", | 20260505 15:42:17.186 | INFO | "huge_shared" : " 27 GB", | 20260505 15:42:17.187 | INFO | "malloc" : " 14 GB", | 20260505 15:42:17.187 | INFO | "pinned" : " 29 GB" | 20260505 15:42:17.187 | INFO | } | 20260505 15:42:17.187 | INFO | Perf report for the last 5.14795 seconds: | 20260505 15:42:17.187 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:42:17.187 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:17.187 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:17.187 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:17.187 | INFO | ANALYSIS : 62.7%, 3.0, 97.5ms, 0 | 20260505 15:42:17.187 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:17.187 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:17.187 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 15:42:17.187 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:17.187 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:17.187 | INFO | Uploaded 36GiB to GPU in 2.18771s (16.4556GiB/s) | 20260505 15:42:17.187 | INFO | Downloaded 34GiB to GPU in 2.13526s (15.9232GiB/s) | 20260505 15:42:22.329 | INFO | Analyzing chunk frames = [4368384,4368896) | 20260505 15:42:22.329 | INFO | MemInfo: { | 20260505 15:42:22.329 | INFO | "gpu" : " 17 GB", | 20260505 15:42:22.329 | INFO | "huge" : "", | 20260505 15:42:22.329 | INFO | "huge_shared" : " 27 GB", | 20260505 15:42:22.329 | INFO | "malloc" : " 14 GB", | 20260505 15:42:22.329 | INFO | "pinned" : " 29 GB" | 20260505 15:42:22.329 | INFO | } | 20260505 15:42:22.329 | INFO | Perf report for the last 5.14221 seconds: | 20260505 15:42:22.329 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:42:22.329 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:22.329 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:22.329 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:22.329 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 15:42:22.329 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:22.329 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:22.329 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:42:22.329 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:22.329 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:22.329 | INFO | Uploaded 36GiB to GPU in 2.15058s (16.7397GiB/s) | 20260505 15:42:22.329 | INFO | Downloaded 34GiB to GPU in 2.0559s (16.5377GiB/s) | 20260505 15:42:27.461 | INFO | Analyzing chunk frames = [4368896,4369408) | 20260505 15:42:27.461 | INFO | MemInfo: { | 20260505 15:42:27.461 | INFO | "gpu" : " 17 GB", | 20260505 15:42:27.461 | INFO | "huge" : "", | 20260505 15:42:27.461 | INFO | "huge_shared" : " 27 GB", | 20260505 15:42:27.461 | INFO | "malloc" : " 14 GB", | 20260505 15:42:27.462 | INFO | "pinned" : " 29 GB" | 20260505 15:42:27.462 | INFO | } | 20260505 15:42:27.462 | INFO | Perf report for the last 5.13281 seconds: | 20260505 15:42:27.462 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:42:27.462 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:27.462 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:27.462 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:27.462 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260505 15:42:27.462 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:27.462 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:27.462 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 15:42:27.462 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:27.462 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:27.462 | INFO | Uploaded 36GiB to GPU in 2.14164s (16.8095GiB/s) | 20260505 15:42:27.462 | INFO | Downloaded 34GiB to GPU in 2.03851s (16.6788GiB/s) | 20260505 15:42:32.596 | INFO | Analyzing chunk frames = [4369408,4369920) | 20260505 15:42:32.596 | INFO | MemInfo: { | 20260505 15:42:32.596 | INFO | "gpu" : " 17 GB", | 20260505 15:42:32.596 | INFO | "huge" : "", | 20260505 15:42:32.596 | INFO | "huge_shared" : " 27 GB", | 20260505 15:42:32.596 | INFO | "malloc" : " 14 GB", | 20260505 15:42:32.596 | INFO | "pinned" : " 29 GB" | 20260505 15:42:32.596 | INFO | } | 20260505 15:42:32.596 | INFO | Perf report for the last 5.13456 seconds: | 20260505 15:42:32.596 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:42:32.596 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:32.596 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:32.596 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:32.596 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 15:42:32.596 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:32.596 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:32.596 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 15:42:32.596 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:32.596 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:32.596 | INFO | Uploaded 36GiB to GPU in 2.12438s (16.9462GiB/s) | 20260505 15:42:32.597 | INFO | Downloaded 34GiB to GPU in 2.1726s (15.6495GiB/s) | 20260505 15:42:37.740 | INFO | Analyzing chunk frames = [4369920,4370432) | 20260505 15:42:37.740 | INFO | MemInfo: { | 20260505 15:42:37.740 | INFO | "gpu" : " 17 GB", | 20260505 15:42:37.740 | INFO | "huge" : "", | 20260505 15:42:37.740 | INFO | "huge_shared" : " 27 GB", | 20260505 15:42:37.740 | INFO | "malloc" : " 14 GB", | 20260505 15:42:37.740 | INFO | "pinned" : " 29 GB" | 20260505 15:42:37.740 | INFO | } | 20260505 15:42:37.740 | INFO | Perf report for the last 5.14386 seconds: | 20260505 15:42:37.740 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:42:37.740 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:37.740 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:37.740 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:37.740 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 15:42:37.740 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:37.740 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:37.740 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 15:42:37.740 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:37.740 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:37.740 | INFO | Uploaded 36GiB to GPU in 2.21182s (16.2762GiB/s) | 20260505 15:42:37.740 | INFO | Downloaded 34GiB to GPU in 2.09837s (16.2031GiB/s) | 20260505 15:42:37.871 | INFO | Starting a round of Baz IO | 20260505 15:42:41.079 | INFO | Populating RTMetrics report | 20260505 15:42:41.079 | INFO | RTMetrics report populated | 20260505 15:42:41.079 | INFO | Opening tmp RTMetrics json file | 20260505 15:42:41.093 | INFO | Serializing RTMetrics to json | 20260505 15:42:41.094 | INFO | Serialization done, swapping tmp file now... | 20260505 15:42:41.094 | INFO | RTMetrics done | 20260505 15:42:41.178 | INFO | Finished a round of Baz IO: Wrote 2.202 GiB in 3.306 seconds | 20260505 15:42:41.178 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.41% / 0.00% / 77.59% | 20260505 15:42:41.178 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.421822% / 0% / 4.76837e-05% | 20260505 15:42:41.178 | INFO | Baz IO overall: | 20260505 15:42:41.178 | INFO | BytesWritten : 2598529045376 | 20260505 15:42:41.178 | INFO | NumEvents : 1234199927912 | 20260505 15:42:42.879 | INFO | Analyzing chunk frames = [4370432,4370944) | 20260505 15:42:42.879 | INFO | MemInfo: { | 20260505 15:42:42.879 | INFO | "gpu" : " 17 GB", | 20260505 15:42:42.879 | INFO | "huge" : "", | 20260505 15:42:42.879 | INFO | "huge_shared" : " 27 GB", | 20260505 15:42:42.879 | INFO | "malloc" : " 14 GB", | 20260505 15:42:42.879 | INFO | "pinned" : " 29 GB" | 20260505 15:42:42.879 | INFO | } | 20260505 15:42:42.879 | INFO | Perf report for the last 5.13868 seconds: | 20260505 15:42:42.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:42:42.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:42.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:42.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:42.879 | INFO | ANALYSIS : 63.9%, 3.0, 99.5ms, 0 | 20260505 15:42:42.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:42.879 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260505 15:42:42.879 | INFO | PRE_HQ : 60.7%, 1.8, 56.6ms, 0 | 20260505 15:42:42.879 | INFO | BAZWRITER : 63.7%, 3.9, 128.0ms, 0 | 20260505 15:42:42.879 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 15:42:42.879 | INFO | Uploaded 36GiB to GPU in 2.16154s (16.6548GiB/s) | 20260505 15:42:42.879 | INFO | Downloaded 38GiB to GPU in 2.25853s (16.8251GiB/s) | 20260505 15:42:48.021 | INFO | Analyzing chunk frames = [4370944,4371456) | 20260505 15:42:48.021 | INFO | MemInfo: { | 20260505 15:42:48.021 | INFO | "gpu" : " 17 GB", | 20260505 15:42:48.021 | INFO | "huge" : "", | 20260505 15:42:48.021 | INFO | "huge_shared" : " 27 GB", | 20260505 15:42:48.021 | INFO | "malloc" : " 14 GB", | 20260505 15:42:48.021 | INFO | "pinned" : " 29 GB" | 20260505 15:42:48.021 | INFO | } | 20260505 15:42:48.021 | INFO | Perf report for the last 5.14248 seconds: | 20260505 15:42:48.021 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:42:48.021 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:48.021 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:48.021 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:48.021 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 15:42:48.021 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:48.021 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:48.021 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 15:42:48.021 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:48.021 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:48.022 | INFO | Uploaded 36GiB to GPU in 2.204s (16.3339GiB/s) | 20260505 15:42:48.022 | INFO | Downloaded 34GiB to GPU in 2.06206s (16.4883GiB/s) | 20260505 15:42:53.158 | INFO | Analyzing chunk frames = [4371456,4371968) | 20260505 15:42:53.158 | INFO | MemInfo: { | 20260505 15:42:53.158 | INFO | "gpu" : " 17 GB", | 20260505 15:42:53.158 | INFO | "huge" : "", | 20260505 15:42:53.158 | INFO | "huge_shared" : " 27 GB", | 20260505 15:42:53.158 | INFO | "malloc" : " 14 GB", | 20260505 15:42:53.158 | INFO | "pinned" : " 29 GB" | 20260505 15:42:53.158 | INFO | } | 20260505 15:42:53.158 | INFO | Perf report for the last 5.13719 seconds: | 20260505 15:42:53.158 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:42:53.158 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:53.158 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:53.159 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:53.159 | INFO | ANALYSIS : 62.9%, 3.0, 97.9ms, 0 | 20260505 15:42:53.159 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:53.159 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:53.159 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 15:42:53.159 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:53.159 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:53.159 | INFO | Uploaded 36GiB to GPU in 2.16735s (16.6101GiB/s) | 20260505 15:42:53.159 | INFO | Downloaded 34GiB to GPU in 2.05092s (16.5779GiB/s) | 20260505 15:42:58.294 | INFO | Analyzing chunk frames = [4371968,4372480) | 20260505 15:42:58.294 | INFO | MemInfo: { | 20260505 15:42:58.294 | INFO | "gpu" : " 17 GB", | 20260505 15:42:58.294 | INFO | "huge" : "", | 20260505 15:42:58.294 | INFO | "huge_shared" : " 27 GB", | 20260505 15:42:58.294 | INFO | "malloc" : " 14 GB", | 20260505 15:42:58.294 | INFO | "pinned" : " 29 GB" | 20260505 15:42:58.294 | INFO | } | 20260505 15:42:58.295 | INFO | Perf report for the last 5.13606 seconds: | 20260505 15:42:58.295 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:42:58.295 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:58.295 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:58.295 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:58.295 | INFO | ANALYSIS : 62.8%, 3.0, 97.4ms, 0 | 20260505 15:42:58.295 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:58.295 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:58.295 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:42:58.295 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:58.295 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:42:58.295 | INFO | Uploaded 36GiB to GPU in 2.18207s (16.4981GiB/s) | 20260505 15:42:58.295 | INFO | Downloaded 34GiB to GPU in 2.12898s (15.9701GiB/s) | 20260505 15:43:03.442 | INFO | Analyzing chunk frames = [4372480,4372992) | 20260505 15:43:03.442 | INFO | MemInfo: { | 20260505 15:43:03.442 | INFO | "gpu" : " 17 GB", | 20260505 15:43:03.442 | INFO | "huge" : "", | 20260505 15:43:03.442 | INFO | "huge_shared" : " 27 GB", | 20260505 15:43:03.442 | INFO | "malloc" : " 14 GB", | 20260505 15:43:03.442 | INFO | "pinned" : " 29 GB" | 20260505 15:43:03.442 | INFO | } | 20260505 15:43:03.442 | INFO | Perf report for the last 5.14764 seconds: | 20260505 15:43:03.442 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:43:03.442 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:03.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:03.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:03.442 | INFO | ANALYSIS : 62.6%, 3.0, 97.9ms, 0 | 20260505 15:43:03.442 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:03.442 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:03.442 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:43:03.442 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:03.442 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:03.443 | INFO | Uploaded 36GiB to GPU in 2.17652s (16.5401GiB/s) | 20260505 15:43:03.443 | INFO | Downloaded 34GiB to GPU in 2.13332s (15.9376GiB/s) | 20260505 15:43:08.587 | INFO | Analyzing chunk frames = [4372992,4373504) | 20260505 15:43:08.587 | INFO | MemInfo: { | 20260505 15:43:08.587 | INFO | "gpu" : " 17 GB", | 20260505 15:43:08.587 | INFO | "huge" : "", | 20260505 15:43:08.587 | INFO | "huge_shared" : " 27 GB", | 20260505 15:43:08.587 | INFO | "malloc" : " 14 GB", | 20260505 15:43:08.587 | INFO | "pinned" : " 29 GB" | 20260505 15:43:08.587 | INFO | } | 20260505 15:43:08.588 | INFO | Perf report for the last 5.14531 seconds: | 20260505 15:43:08.588 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:43:08.588 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:08.588 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:08.588 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:08.588 | INFO | ANALYSIS : 62.4%, 3.0, 97.6ms, 0 | 20260505 15:43:08.588 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:08.588 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:08.588 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 15:43:08.588 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:08.588 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:08.588 | INFO | Uploaded 36GiB to GPU in 2.1366s (16.8492GiB/s) | 20260505 15:43:08.588 | INFO | Downloaded 34GiB to GPU in 2.04075s (16.6605GiB/s) | 20260505 15:43:13.714 | INFO | Analyzing chunk frames = [4373504,4374016) | 20260505 15:43:13.714 | INFO | MemInfo: { | 20260505 15:43:13.714 | INFO | "gpu" : " 17 GB", | 20260505 15:43:13.714 | INFO | "huge" : "", | 20260505 15:43:13.714 | INFO | "huge_shared" : " 27 GB", | 20260505 15:43:13.714 | INFO | "malloc" : " 14 GB", | 20260505 15:43:13.714 | INFO | "pinned" : " 29 GB" | 20260505 15:43:13.714 | INFO | } | 20260505 15:43:13.714 | INFO | Perf report for the last 5.12649 seconds: | 20260505 15:43:13.714 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:43:13.714 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:13.714 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:13.714 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:13.714 | INFO | ANALYSIS : 63.7%, 3.0, 99.2ms, 0 | 20260505 15:43:13.714 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:13.714 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:13.714 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 15:43:13.714 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:13.714 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:13.714 | INFO | Uploaded 36GiB to GPU in 2.14521s (16.7816GiB/s) | 20260505 15:43:13.714 | INFO | Downloaded 34GiB to GPU in 2.14313s (15.8647GiB/s) | 20260505 15:43:18.847 | INFO | Analyzing chunk frames = [4374016,4374528) | 20260505 15:43:18.847 | INFO | MemInfo: { | 20260505 15:43:18.847 | INFO | "gpu" : " 17 GB", | 20260505 15:43:18.847 | INFO | "huge" : "", | 20260505 15:43:18.847 | INFO | "huge_shared" : " 27 GB", | 20260505 15:43:18.847 | INFO | "malloc" : " 14 GB", | 20260505 15:43:18.847 | INFO | "pinned" : " 29 GB" | 20260505 15:43:18.847 | INFO | } | 20260505 15:43:18.847 | INFO | Perf report for the last 5.13278 seconds: | 20260505 15:43:18.847 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:43:18.847 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:18.847 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:18.847 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:18.847 | INFO | ANALYSIS : 63.8%, 3.0, 99.4ms, 0 | 20260505 15:43:18.847 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:18.847 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:18.847 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 15:43:18.847 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:18.847 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:18.847 | INFO | Uploaded 36GiB to GPU in 2.23345s (16.1185GiB/s) | 20260505 15:43:18.847 | INFO | Downloaded 34GiB to GPU in 2.10364s (16.1624GiB/s) | 20260505 15:43:18.847 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 15:43:18.847 | INFO | Total mem usage: 15.3 GiB | 20260505 15:43:18.847 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 15:43:18.847 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -7.19%) | 20260505 15:43:18.847 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 15:43:18.847 | INFO | High Water Breakdown: | 20260505 15:43:18.847 | INFO | 61.1% : 8.74 GiB ( 91.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 15:43:18.847 | INFO | 38.8% : 5.54 GiB ( 83.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 15:43:18.847 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 15:43:18.847 | INFO | ------------------------------------------ | 20260505 15:43:18.847 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:43:18.847 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 15:43:18.847 | INFO | Total mem usage: 26.5 GiB | 20260505 15:43:18.847 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 15:43:18.847 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 15:43:18.847 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 15:43:18.847 | INFO | High Water Breakdown: | 20260505 15:43:18.847 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:43:18.847 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:43:18.848 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:43:18.848 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:43:18.848 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:43:18.848 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:43:18.848 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 15:43:18.848 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 15:43:18.848 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:43:18.848 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 15:43:18.848 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 15:43:18.848 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 15:43:18.848 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 15:43:18.848 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 15:43:18.848 | INFO | ------------------------------------------ | 20260505 15:43:18.848 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:43:18.848 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 15:43:18.848 | INFO | Total mem usage: 25.6 GiB | 20260505 15:43:18.848 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 15:43:18.848 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 15:43:18.848 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 15:43:18.848 | INFO | High Water Breakdown: | 20260505 15:43:18.848 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 15:43:18.848 | INFO | ------------------------------------------ | 20260505 15:43:18.848 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:43:18.848 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 15:43:18.848 | INFO | Total mem usage: 15.7 GiB | 20260505 15:43:18.848 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 15:43:18.848 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 15:43:18.848 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 15:43:18.848 | INFO | High Water Breakdown: | 20260505 15:43:18.848 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 15:43:18.848 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 15:43:18.848 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 15:43:18.848 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 15:43:18.848 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 15:43:18.848 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 15:43:18.848 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 15:43:18.848 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 15:43:18.848 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 15:43:18.848 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:43:18.848 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 15:43:18.848 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:43:18.848 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 15:43:18.848 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 15:43:18.848 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 15:43:18.848 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 15:43:18.848 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:43:18.848 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 15:43:18.848 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:43:18.848 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:43:18.848 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:43:18.848 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 15:43:18.848 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 15:43:18.848 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:43:18.848 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 15:43:18.848 | INFO | ------------------------------------------ | 20260505 15:43:18.848 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:43:18.848 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 15:43:18.848 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:43:18.848 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:43:18.848 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.58e+01, 1.35e+01 / 1.99e+00 | 20260505 15:43:18.848 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 1.01e-01, 1.09e-02 / 4.82e-03 | 20260505 15:43:18.848 | INFO | PerfReport: 11.77%, Baseline -- 3.74e+00 / 4.49e+00, 5.76e-01 / 1.74e-01 | 20260505 15:43:18.848 | INFO | PerfReport: 3.29%, Histogram -- 1.04e+00 / 1.51e+00, 1.96e-01 / 8.91e-02 | 20260505 15:43:18.848 | INFO | PerfReport: 15.97%, DME -- 4.05e+01 / 4.51e+01, 4.40e+00 / 2.07e+00 | 20260505 15:43:18.848 | INFO | PerfReport: 56.64%, FrameLabeling -- 1.80e+01 / 1.98e+01, 1.46e+00 / 1.93e+00 | 20260505 15:43:18.848 | INFO | PerfReport: 7.37%, PulseAccumulating -- 2.34e+00 / 2.90e+00, 4.46e-01 / 1.85e-01 | 20260505 15:43:18.848 | INFO | PerfReport: 3.46%, Metrics -- 1.10e+00 / 1.41e+00, 3.05e-01 / 1.72e-01 | 20260505 15:43:18.848 | INFO | PerfReport: 1.21%, PulseSerializing -- 3.82e-01 / 4.40e-01, 6.90e-02 / 3.77e-02 | 20260505 15:43:18.848 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 15:43:18.848 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:43:18.848 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:43:18.848 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.56e+01 / 0.00e+00 | 20260505 15:43:18.848 | INFO | PerfReport: 22.56%, Upload -- 2.27e+01 / 0.00e+00, 8.06e+00 / 0.00e+00 | 20260505 15:43:18.848 | INFO | PerfReport: 21.94%, Download -- 2.20e+01 / 0.00e+00, 8.00e+00 / 0.00e+00 | 20260505 15:43:18.848 | INFO | PerfReport: 31.58%, Compute -- 3.17e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 15:43:18.848 | INFO | PerfReport: 2.30%, LockUpload -- 2.31e+00 / 0.00e+00, 7.87e+00 / 0.00e+00 | 20260505 15:43:18.848 | INFO | PerfReport: 1.38%, LockDownload -- 1.39e+00 / 0.00e+00, 2.90e+00 / 0.00e+00 | 20260505 15:43:18.848 | INFO | PerfReport: 20.23%, LockCompute -- 2.03e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 15:43:18.848 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.57e-02 / 0.00e+00 | 20260505 15:43:18.983 | INFO | Starting a round of Baz IO | 20260505 15:43:22.204 | INFO | Populating RTMetrics report | 20260505 15:43:22.204 | INFO | RTMetrics report populated | 20260505 15:43:22.204 | INFO | Opening tmp RTMetrics json file | 20260505 15:43:22.209 | INFO | Serializing RTMetrics to json | 20260505 15:43:22.231 | INFO | Serialization done, swapping tmp file now... | 20260505 15:43:22.251 | INFO | RTMetrics done | 20260505 15:43:22.308 | INFO | Finished a round of Baz IO: Wrote 2.205 GiB in 3.325 seconds | 20260505 15:43:22.308 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.40% / 0.00% / 77.59% | 20260505 15:43:22.308 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.421774% / 0% / 7.54992e-05% | 20260505 15:43:22.308 | INFO | Baz IO overall: | 20260505 15:43:22.308 | INFO | BytesWritten : 2600897110179 | 20260505 15:43:22.308 | INFO | NumEvents : 1235311133860 | 20260505 15:43:23.997 | INFO | Analyzing chunk frames = [4374528,4375040) | 20260505 15:43:23.997 | INFO | MemInfo: { | 20260505 15:43:23.997 | INFO | "gpu" : " 17 GB", | 20260505 15:43:23.997 | INFO | "huge" : "", | 20260505 15:43:23.997 | INFO | "huge_shared" : " 27 GB", | 20260505 15:43:23.997 | INFO | "malloc" : " 14 GB", | 20260505 15:43:23.997 | INFO | "pinned" : " 29 GB" | 20260505 15:43:23.997 | INFO | } | 20260505 15:43:23.997 | INFO | Perf report for the last 5.14995 seconds: | 20260505 15:43:23.997 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:43:23.997 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:23.997 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:23.997 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:23.997 | INFO | ANALYSIS : 64.1%, 3.0, 99.9ms, 0 | 20260505 15:43:23.997 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:23.997 | INFO | RT_METRICS : 45.5%, 1.0, 23.9ms, 0 | 20260505 15:43:23.997 | INFO | PRE_HQ : 60.6%, 1.8, 57.0ms, 0 | 20260505 15:43:23.997 | INFO | BAZWRITER : 63.8%, 3.9, 128.6ms, 0 | 20260505 15:43:23.997 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 15:43:23.997 | INFO | Uploaded 36GiB to GPU in 2.17214s (16.5736GiB/s) | 20260505 15:43:23.997 | INFO | Downloaded 38GiB to GPU in 2.27323s (16.7163GiB/s) | 20260505 15:43:29.138 | INFO | Analyzing chunk frames = [4375040,4375552) | 20260505 15:43:29.138 | INFO | MemInfo: { | 20260505 15:43:29.138 | INFO | "gpu" : " 17 GB", | 20260505 15:43:29.138 | INFO | "huge" : "", | 20260505 15:43:29.138 | INFO | "huge_shared" : " 27 GB", | 20260505 15:43:29.138 | INFO | "malloc" : " 14 GB", | 20260505 15:43:29.138 | INFO | "pinned" : " 29 GB" | 20260505 15:43:29.138 | INFO | } | 20260505 15:43:29.138 | INFO | Perf report for the last 5.1413 seconds: | 20260505 15:43:29.138 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:43:29.138 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:29.138 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:29.138 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:29.138 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 15:43:29.138 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:29.138 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:29.138 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 15:43:29.138 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:29.138 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:29.138 | INFO | Uploaded 36GiB to GPU in 2.12704s (16.9249GiB/s) | 20260505 15:43:29.138 | INFO | Downloaded 34GiB to GPU in 2.03733s (16.6885GiB/s) | 20260505 15:43:34.277 | INFO | Analyzing chunk frames = [4375552,4376064) | 20260505 15:43:34.277 | INFO | MemInfo: { | 20260505 15:43:34.277 | INFO | "gpu" : " 17 GB", | 20260505 15:43:34.277 | INFO | "huge" : "", | 20260505 15:43:34.277 | INFO | "huge_shared" : " 27 GB", | 20260505 15:43:34.277 | INFO | "malloc" : " 14 GB", | 20260505 15:43:34.277 | INFO | "pinned" : " 29 GB" | 20260505 15:43:34.277 | INFO | } | 20260505 15:43:34.277 | INFO | Perf report for the last 5.13892 seconds: | 20260505 15:43:34.277 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:43:34.277 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:34.277 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:34.277 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:34.277 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260505 15:43:34.277 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:34.277 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:34.277 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:43:34.277 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:34.277 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:34.277 | INFO | Uploaded 36GiB to GPU in 2.19624s (16.3917GiB/s) | 20260505 15:43:34.277 | INFO | Downloaded 34GiB to GPU in 2.04368s (16.6367GiB/s) | 20260505 15:43:39.415 | INFO | Analyzing chunk frames = [4376064,4376576) | 20260505 15:43:39.415 | INFO | MemInfo: { | 20260505 15:43:39.415 | INFO | "gpu" : " 17 GB", | 20260505 15:43:39.415 | INFO | "huge" : "", | 20260505 15:43:39.415 | INFO | "huge_shared" : " 27 GB", | 20260505 15:43:39.415 | INFO | "malloc" : " 14 GB", | 20260505 15:43:39.415 | INFO | "pinned" : " 29 GB" | 20260505 15:43:39.415 | INFO | } | 20260505 15:43:39.415 | INFO | Perf report for the last 5.13802 seconds: | 20260505 15:43:39.415 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:43:39.415 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:39.415 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:39.415 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:39.415 | INFO | ANALYSIS : 62.5%, 3.0, 97.1ms, 0 | 20260505 15:43:39.415 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:39.415 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:39.415 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 15:43:39.415 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:39.415 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:39.415 | INFO | Uploaded 36GiB to GPU in 2.18707s (16.4604GiB/s) | 20260505 15:43:39.415 | INFO | Downloaded 34GiB to GPU in 2.13587s (15.9186GiB/s) | 20260505 15:43:44.546 | INFO | Analyzing chunk frames = [4376576,4377088) | 20260505 15:43:44.546 | INFO | MemInfo: { | 20260505 15:43:44.546 | INFO | "gpu" : " 17 GB", | 20260505 15:43:44.546 | INFO | "huge" : "", | 20260505 15:43:44.546 | INFO | "huge_shared" : " 27 GB", | 20260505 15:43:44.546 | INFO | "malloc" : " 14 GB", | 20260505 15:43:44.546 | INFO | "pinned" : " 29 GB" | 20260505 15:43:44.546 | INFO | } | 20260505 15:43:44.546 | INFO | Perf report for the last 5.13121 seconds: | 20260505 15:43:44.546 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:43:44.546 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:44.546 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:44.546 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:44.546 | INFO | ANALYSIS : 62.9%, 3.0, 98.0ms, 0 | 20260505 15:43:44.546 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:44.546 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:44.546 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:43:44.546 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:44.546 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:44.547 | INFO | Uploaded 36GiB to GPU in 2.18392s (16.4841GiB/s) | 20260505 15:43:44.547 | INFO | Downloaded 34GiB to GPU in 2.13099s (15.955GiB/s) | 20260505 15:43:49.692 | INFO | Analyzing chunk frames = [4377088,4377600) | 20260505 15:43:49.692 | INFO | MemInfo: { | 20260505 15:43:49.692 | INFO | "gpu" : " 17 GB", | 20260505 15:43:49.692 | INFO | "huge" : "", | 20260505 15:43:49.692 | INFO | "huge_shared" : " 27 GB", | 20260505 15:43:49.692 | INFO | "malloc" : " 14 GB", | 20260505 15:43:49.692 | INFO | "pinned" : " 29 GB" | 20260505 15:43:49.692 | INFO | } | 20260505 15:43:49.692 | INFO | Perf report for the last 5.14551 seconds: | 20260505 15:43:49.692 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:43:49.692 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:49.692 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:49.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:49.692 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 15:43:49.692 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:49.692 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:49.692 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 15:43:49.692 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:49.692 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:49.692 | INFO | Uploaded 36GiB to GPU in 2.2064s (16.3162GiB/s) | 20260505 15:43:49.692 | INFO | Downloaded 34GiB to GPU in 2.11315s (16.0897GiB/s) | 20260505 15:43:54.836 | INFO | Analyzing chunk frames = [4377600,4378112) | 20260505 15:43:54.836 | INFO | MemInfo: { | 20260505 15:43:54.836 | INFO | "gpu" : " 17 GB", | 20260505 15:43:54.836 | INFO | "huge" : "", | 20260505 15:43:54.836 | INFO | "huge_shared" : " 27 GB", | 20260505 15:43:54.836 | INFO | "malloc" : " 14 GB", | 20260505 15:43:54.836 | INFO | "pinned" : " 29 GB" | 20260505 15:43:54.836 | INFO | } | 20260505 15:43:54.836 | INFO | Perf report for the last 5.1441 seconds: | 20260505 15:43:54.836 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:43:54.836 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:54.836 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:54.836 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:54.836 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 15:43:54.836 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:54.836 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:54.836 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260505 15:43:54.836 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:54.836 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:54.836 | INFO | Uploaded 36GiB to GPU in 2.14251s (16.8027GiB/s) | 20260505 15:43:54.836 | INFO | Downloaded 34GiB to GPU in 2.17917s (15.6023GiB/s) | 20260505 15:43:59.969 | INFO | Analyzing chunk frames = [4378112,4378624) | 20260505 15:43:59.969 | INFO | MemInfo: { | 20260505 15:43:59.969 | INFO | "gpu" : " 17 GB", | 20260505 15:43:59.969 | INFO | "huge" : "", | 20260505 15:43:59.969 | INFO | "huge_shared" : " 27 GB", | 20260505 15:43:59.969 | INFO | "malloc" : " 14 GB", | 20260505 15:43:59.969 | INFO | "pinned" : " 29 GB" | 20260505 15:43:59.969 | INFO | } | 20260505 15:43:59.969 | INFO | Perf report for the last 5.13311 seconds: | 20260505 15:43:59.969 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:43:59.969 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:59.969 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:59.969 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:59.969 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 15:43:59.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:59.969 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:59.969 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 15:43:59.969 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:59.969 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:43:59.969 | INFO | Uploaded 36GiB to GPU in 2.14862s (16.7549GiB/s) | 20260505 15:43:59.969 | INFO | Downloaded 34GiB to GPU in 2.01816s (16.847GiB/s) | 20260505 15:44:00.102 | INFO | Starting a round of Baz IO | 20260505 15:44:03.320 | INFO | Populating RTMetrics report | 20260505 15:44:03.320 | INFO | RTMetrics report populated | 20260505 15:44:03.320 | INFO | Opening tmp RTMetrics json file | 20260505 15:44:03.363 | INFO | Serializing RTMetrics to json | 20260505 15:44:03.389 | INFO | Serialization done, swapping tmp file now... | 20260505 15:44:03.389 | INFO | RTMetrics done | 20260505 15:44:03.422 | INFO | Finished a round of Baz IO: Wrote 2.202 GiB in 3.32 seconds | 20260505 15:44:03.422 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.40% / 0.00% / 77.60% | 20260505 15:44:03.422 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.420133% / 0% / 4.76837e-05% | 20260505 15:44:03.422 | INFO | Baz IO overall: | 20260505 15:44:03.422 | INFO | BytesWritten : 2603262025466 | 20260505 15:44:03.422 | INFO | NumEvents : 1236420926685 | 20260505 15:44:05.104 | INFO | Analyzing chunk frames = [4378624,4379136) | 20260505 15:44:05.104 | INFO | MemInfo: { | 20260505 15:44:05.104 | INFO | "gpu" : " 17 GB", | 20260505 15:44:05.104 | INFO | "huge" : "", | 20260505 15:44:05.104 | INFO | "huge_shared" : " 27 GB", | 20260505 15:44:05.104 | INFO | "malloc" : " 14 GB", | 20260505 15:44:05.104 | INFO | "pinned" : " 29 GB" | 20260505 15:44:05.104 | INFO | } | 20260505 15:44:05.104 | INFO | Perf report for the last 5.13516 seconds: | 20260505 15:44:05.104 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:44:05.104 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:05.104 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:05.104 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:05.104 | INFO | ANALYSIS : 64.2%, 3.0, 99.9ms, 0 | 20260505 15:44:05.104 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:05.104 | INFO | RT_METRICS : 46.2%, 1.0, 24.2ms, 0 | 20260505 15:44:05.104 | INFO | PRE_HQ : 61.1%, 1.8, 57.0ms, 0 | 20260505 15:44:05.104 | INFO | BAZWRITER : 64.0%, 3.9, 127.8ms, 0 | 20260505 15:44:05.104 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 15:44:05.104 | INFO | Uploaded 36GiB to GPU in 2.21279s (16.2691GiB/s) | 20260505 15:44:05.105 | INFO | Downloaded 38GiB to GPU in 2.26714s (16.7612GiB/s) | 20260505 15:44:10.251 | INFO | Analyzing chunk frames = [4379136,4379648) | 20260505 15:44:10.251 | INFO | MemInfo: { | 20260505 15:44:10.251 | INFO | "gpu" : " 17 GB", | 20260505 15:44:10.251 | INFO | "huge" : "", | 20260505 15:44:10.251 | INFO | "huge_shared" : " 27 GB", | 20260505 15:44:10.251 | INFO | "malloc" : " 14 GB", | 20260505 15:44:10.251 | INFO | "pinned" : " 29 GB" | 20260505 15:44:10.251 | INFO | } | 20260505 15:44:10.251 | INFO | Perf report for the last 5.1467 seconds: | 20260505 15:44:10.251 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:44:10.251 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:10.251 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:10.251 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:10.251 | INFO | ANALYSIS : 63.3%, 3.0, 99.0ms, 0 | 20260505 15:44:10.251 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:10.251 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:10.251 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 15:44:10.251 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:10.251 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:10.251 | INFO | Uploaded 36GiB to GPU in 2.19865s (16.3737GiB/s) | 20260505 15:44:10.251 | INFO | Downloaded 34GiB to GPU in 2.03833s (16.6803GiB/s) | 20260505 15:44:15.393 | INFO | Analyzing chunk frames = [4379648,4380160) | 20260505 15:44:15.393 | INFO | MemInfo: { | 20260505 15:44:15.393 | INFO | "gpu" : " 17 GB", | 20260505 15:44:15.393 | INFO | "huge" : "", | 20260505 15:44:15.393 | INFO | "huge_shared" : " 27 GB", | 20260505 15:44:15.393 | INFO | "malloc" : " 14 GB", | 20260505 15:44:15.393 | INFO | "pinned" : " 29 GB" | 20260505 15:44:15.393 | INFO | } | 20260505 15:44:15.393 | INFO | Perf report for the last 5.14234 seconds: | 20260505 15:44:15.393 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:44:15.393 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:15.393 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:15.393 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:15.393 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260505 15:44:15.393 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:15.393 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:15.393 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 15:44:15.393 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:15.393 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:15.394 | INFO | Uploaded 36GiB to GPU in 2.19688s (16.3869GiB/s) | 20260505 15:44:15.394 | INFO | Downloaded 34GiB to GPU in 2.03051s (16.7446GiB/s) | 20260505 15:44:20.532 | INFO | Analyzing chunk frames = [4380160,4380672) | 20260505 15:44:20.532 | INFO | MemInfo: { | 20260505 15:44:20.532 | INFO | "gpu" : " 17 GB", | 20260505 15:44:20.532 | INFO | "huge" : "", | 20260505 15:44:20.532 | INFO | "huge_shared" : " 27 GB", | 20260505 15:44:20.532 | INFO | "malloc" : " 14 GB", | 20260505 15:44:20.532 | INFO | "pinned" : " 29 GB" | 20260505 15:44:20.532 | INFO | } | 20260505 15:44:20.532 | INFO | Perf report for the last 5.13912 seconds: | 20260505 15:44:20.532 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:44:20.532 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:20.532 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:20.532 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:20.532 | INFO | ANALYSIS : 62.7%, 3.0, 97.3ms, 0 | 20260505 15:44:20.532 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:20.532 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:20.533 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 15:44:20.533 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:20.533 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:20.533 | INFO | Uploaded 36GiB to GPU in 2.17221s (16.573GiB/s) | 20260505 15:44:20.533 | INFO | Downloaded 34GiB to GPU in 2.14251s (15.8692GiB/s) | 20260505 15:44:25.662 | INFO | Analyzing chunk frames = [4380672,4381184) | 20260505 15:44:25.662 | INFO | MemInfo: { | 20260505 15:44:25.662 | INFO | "gpu" : " 17 GB", | 20260505 15:44:25.662 | INFO | "huge" : "", | 20260505 15:44:25.662 | INFO | "huge_shared" : " 27 GB", | 20260505 15:44:25.662 | INFO | "malloc" : " 14 GB", | 20260505 15:44:25.662 | INFO | "pinned" : " 29 GB" | 20260505 15:44:25.662 | INFO | } | 20260505 15:44:25.662 | INFO | Perf report for the last 5.12961 seconds: | 20260505 15:44:25.662 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:44:25.662 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:25.662 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:25.662 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:25.662 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260505 15:44:25.662 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:25.662 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:25.662 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:44:25.662 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:25.662 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:25.662 | INFO | Uploaded 36GiB to GPU in 2.12594s (16.9337GiB/s) | 20260505 15:44:25.662 | INFO | Downloaded 34GiB to GPU in 2.05423s (16.5512GiB/s) | 20260505 15:44:30.798 | INFO | Analyzing chunk frames = [4381184,4381696) | 20260505 15:44:30.798 | INFO | MemInfo: { | 20260505 15:44:30.798 | INFO | "gpu" : " 17 GB", | 20260505 15:44:30.798 | INFO | "huge" : "", | 20260505 15:44:30.798 | INFO | "huge_shared" : " 27 GB", | 20260505 15:44:30.799 | INFO | "malloc" : " 14 GB", | 20260505 15:44:30.799 | INFO | "pinned" : " 29 GB" | 20260505 15:44:30.799 | INFO | } | 20260505 15:44:30.799 | INFO | Perf report for the last 5.13668 seconds: | 20260505 15:44:30.799 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:44:30.799 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:30.799 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:30.799 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:30.799 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 15:44:30.799 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:30.799 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:30.799 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 15:44:30.799 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:30.799 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:30.799 | INFO | Uploaded 36GiB to GPU in 2.16724s (16.611GiB/s) | 20260505 15:44:30.799 | INFO | Downloaded 34GiB to GPU in 2.04047s (16.6628GiB/s) | 20260505 15:44:35.951 | INFO | Analyzing chunk frames = [4381696,4382208) | 20260505 15:44:35.951 | INFO | MemInfo: { | 20260505 15:44:35.951 | INFO | "gpu" : " 17 GB", | 20260505 15:44:35.951 | INFO | "huge" : "", | 20260505 15:44:35.951 | INFO | "huge_shared" : " 27 GB", | 20260505 15:44:35.951 | INFO | "malloc" : " 14 GB", | 20260505 15:44:35.951 | INFO | "pinned" : " 29 GB" | 20260505 15:44:35.951 | INFO | } | 20260505 15:44:35.951 | INFO | Perf report for the last 5.15214 seconds: | 20260505 15:44:35.951 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:44:35.951 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:35.951 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:35.951 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:35.951 | INFO | ANALYSIS : 63.1%, 3.0, 98.7ms, 0 | 20260505 15:44:35.951 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:35.951 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:35.951 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 15:44:35.951 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:35.951 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:35.951 | INFO | Uploaded 36GiB to GPU in 2.18232s (16.4962GiB/s) | 20260505 15:44:35.951 | INFO | Downloaded 34GiB to GPU in 2.14769s (15.8309GiB/s) | 20260505 15:44:41.093 | INFO | Analyzing chunk frames = [4382208,4382720) | 20260505 15:44:41.093 | INFO | MemInfo: { | 20260505 15:44:41.093 | INFO | "gpu" : " 17 GB", | 20260505 15:44:41.093 | INFO | "huge" : "", | 20260505 15:44:41.093 | INFO | "huge_shared" : " 27 GB", | 20260505 15:44:41.093 | INFO | "malloc" : " 14 GB", | 20260505 15:44:41.093 | INFO | "pinned" : " 29 GB" | 20260505 15:44:41.093 | INFO | } | 20260505 15:44:41.093 | INFO | Perf report for the last 5.14234 seconds: | 20260505 15:44:41.093 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:44:41.093 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:41.093 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:41.093 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:41.093 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 15:44:41.093 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:41.093 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:41.093 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:44:41.093 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:41.093 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:41.093 | INFO | Uploaded 36GiB to GPU in 2.15187s (16.7297GiB/s) | 20260505 15:44:41.094 | INFO | Downloaded 34GiB to GPU in 2.0202s (16.83GiB/s) | 20260505 15:44:41.094 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 15:44:41.094 | INFO | Total mem usage: 15.3 GiB | 20260505 15:44:41.094 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 15:44:41.094 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.72%) | 20260505 15:44:41.094 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 15:44:41.094 | INFO | High Water Breakdown: | 20260505 15:44:41.094 | INFO | 61% : 8.68 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 15:44:41.094 | INFO | 38.9% : 5.53 GiB ( 83.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 15:44:41.094 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 15:44:41.094 | INFO | ------------------------------------------ | 20260505 15:44:41.094 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:44:41.094 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 15:44:41.094 | INFO | Total mem usage: 26.5 GiB | 20260505 15:44:41.094 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 15:44:41.094 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 15:44:41.094 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 15:44:41.094 | INFO | High Water Breakdown: | 20260505 15:44:41.094 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:44:41.094 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:44:41.094 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:44:41.094 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:44:41.094 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:44:41.094 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:44:41.094 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 15:44:41.094 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 15:44:41.094 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:44:41.094 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 15:44:41.094 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 15:44:41.094 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 15:44:41.094 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 15:44:41.094 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 15:44:41.094 | INFO | ------------------------------------------ | 20260505 15:44:41.094 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:44:41.094 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 15:44:41.094 | INFO | Total mem usage: 25.6 GiB | 20260505 15:44:41.094 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 15:44:41.094 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 15:44:41.094 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 15:44:41.094 | INFO | High Water Breakdown: | 20260505 15:44:41.094 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 15:44:41.094 | INFO | ------------------------------------------ | 20260505 15:44:41.094 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:44:41.094 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 15:44:41.094 | INFO | Total mem usage: 15.7 GiB | 20260505 15:44:41.094 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 15:44:41.094 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 15:44:41.094 | INFO | Unused Cache: 871 MiB (Waste: 5.43%) | 20260505 15:44:41.094 | INFO | High Water Breakdown: | 20260505 15:44:41.094 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 15:44:41.094 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 15:44:41.094 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 15:44:41.094 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 15:44:41.094 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 15:44:41.094 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 15:44:41.094 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 15:44:41.095 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 15:44:41.095 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 15:44:41.095 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:44:41.095 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 15:44:41.095 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:44:41.095 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 15:44:41.095 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 15:44:41.095 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 15:44:41.095 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 15:44:41.095 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:44:41.095 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 15:44:41.095 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:44:41.095 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:44:41.095 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:44:41.095 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 15:44:41.095 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 15:44:41.095 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:44:41.095 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 15:44:41.095 | INFO | ------------------------------------------ | 20260505 15:44:41.095 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:44:41.095 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 15:44:41.095 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:44:41.095 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:44:41.095 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.55e+01, 1.35e+01 / 2.20e+00 | 20260505 15:44:41.095 | INFO | PerfReport: 0.29%, AggregateStats -- 9.27e-02 / 1.01e-01, 1.09e-02 / 4.92e-03 | 20260505 15:44:41.095 | INFO | PerfReport: 11.78%, Baseline -- 3.74e+00 / 4.48e+00, 5.79e-01 / 2.07e-01 | 20260505 15:44:41.095 | INFO | PerfReport: 3.27%, Histogram -- 1.04e+00 / 1.50e+00, 1.91e-01 / 1.07e-01 | 20260505 15:44:41.095 | INFO | PerfReport: 15.95%, DME -- 4.05e+01 / 4.52e+01, 4.42e+00 / 2.47e+00 | 20260505 15:44:41.095 | INFO | PerfReport: 56.68%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.52e+00 / 1.63e+00 | 20260505 15:44:41.095 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.34e+00 / 2.91e+00, 4.49e-01 / 2.01e-01 | 20260505 15:44:41.095 | INFO | PerfReport: 3.44%, Metrics -- 1.09e+00 / 1.44e+00, 3.04e-01 / 1.83e-01 | 20260505 15:44:41.095 | INFO | PerfReport: 1.20%, PulseSerializing -- 3.81e-01 / 4.36e-01, 6.74e-02 / 3.69e-02 | 20260505 15:44:41.095 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 15:44:41.095 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:44:41.095 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:44:41.095 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.56e+01 / 0.00e+00 | 20260505 15:44:41.095 | INFO | PerfReport: 22.55%, Upload -- 2.26e+01 / 0.00e+00, 7.95e+00 / 0.00e+00 | 20260505 15:44:41.095 | INFO | PerfReport: 21.84%, Download -- 2.19e+01 / 0.00e+00, 7.91e+00 / 0.00e+00 | 20260505 15:44:41.095 | INFO | PerfReport: 31.62%, Compute -- 3.17e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 15:44:41.095 | INFO | PerfReport: 2.23%, LockUpload -- 2.23e+00 / 0.00e+00, 7.91e+00 / 0.00e+00 | 20260505 15:44:41.095 | INFO | PerfReport: 1.45%, LockDownload -- 1.46e+00 / 0.00e+00, 2.97e+00 / 0.00e+00 | 20260505 15:44:41.095 | INFO | PerfReport: 20.30%, LockCompute -- 2.04e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 15:44:41.095 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.81e-02 / 0.00e+00 | 20260505 15:44:41.230 | INFO | Starting a round of Baz IO | 20260505 15:44:44.429 | INFO | Populating RTMetrics report | 20260505 15:44:44.429 | INFO | RTMetrics report populated | 20260505 15:44:44.429 | INFO | Opening tmp RTMetrics json file | 20260505 15:44:44.432 | INFO | Serializing RTMetrics to json | 20260505 15:44:44.454 | INFO | Serialization done, swapping tmp file now... | 20260505 15:44:44.457 | INFO | RTMetrics done | 20260505 15:44:44.537 | INFO | Finished a round of Baz IO: Wrote 2.202 GiB in 3.306 seconds | 20260505 15:44:44.537 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.40% / 0.00% / 77.60% | 20260505 15:44:44.537 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.420237% / 7.94729e-06% / 8.74201e-05% | 20260505 15:44:44.537 | INFO | Baz IO overall: | 20260505 15:44:44.537 | INFO | BytesWritten : 2605626275931 | 20260505 15:44:44.537 | INFO | NumEvents : 1237531388999 | 20260505 15:44:46.222 | INFO | Analyzing chunk frames = [4382720,4383232) | 20260505 15:44:46.222 | INFO | MemInfo: { | 20260505 15:44:46.222 | INFO | "gpu" : " 17 GB", | 20260505 15:44:46.222 | INFO | "huge" : "", | 20260505 15:44:46.222 | INFO | "huge_shared" : " 27 GB", | 20260505 15:44:46.222 | INFO | "malloc" : " 14 GB", | 20260505 15:44:46.222 | INFO | "pinned" : " 29 GB" | 20260505 15:44:46.222 | INFO | } | 20260505 15:44:46.222 | INFO | Perf report for the last 5.12899 seconds: | 20260505 15:44:46.222 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:44:46.222 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:46.222 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:46.222 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:46.222 | INFO | ANALYSIS : 63.9%, 3.0, 99.1ms, 0 | 20260505 15:44:46.222 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:46.222 | INFO | RT_METRICS : 45.5%, 1.0, 23.8ms, 0 | 20260505 15:44:46.222 | INFO | PRE_HQ : 61.0%, 1.8, 57.4ms, 0 | 20260505 15:44:46.222 | INFO | BAZWRITER : 63.9%, 3.9, 128.9ms, 0 | 20260505 15:44:46.222 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 15:44:46.223 | INFO | Uploaded 36GiB to GPU in 2.18962s (16.4412GiB/s) | 20260505 15:44:46.223 | INFO | Downloaded 38GiB to GPU in 2.26814s (16.7538GiB/s) | 20260505 15:44:51.363 | INFO | Analyzing chunk frames = [4383232,4383744) | 20260505 15:44:51.363 | INFO | MemInfo: { | 20260505 15:44:51.363 | INFO | "gpu" : " 17 GB", | 20260505 15:44:51.363 | INFO | "huge" : "", | 20260505 15:44:51.363 | INFO | "huge_shared" : " 27 GB", | 20260505 15:44:51.363 | INFO | "malloc" : " 14 GB", | 20260505 15:44:51.363 | INFO | "pinned" : " 29 GB" | 20260505 15:44:51.363 | INFO | } | 20260505 15:44:51.363 | INFO | Perf report for the last 5.14052 seconds: | 20260505 15:44:51.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:44:51.363 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:51.363 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:51.363 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:51.363 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 15:44:51.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:51.363 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:51.363 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 15:44:51.363 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:51.363 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:51.363 | INFO | Uploaded 36GiB to GPU in 2.2008s (16.3577GiB/s) | 20260505 15:44:51.363 | INFO | Downloaded 34GiB to GPU in 2.04047s (16.6628GiB/s) | 20260505 15:44:56.504 | INFO | Analyzing chunk frames = [4383744,4384256) | 20260505 15:44:56.504 | INFO | MemInfo: { | 20260505 15:44:56.504 | INFO | "gpu" : " 17 GB", | 20260505 15:44:56.504 | INFO | "huge" : "", | 20260505 15:44:56.504 | INFO | "huge_shared" : " 27 GB", | 20260505 15:44:56.504 | INFO | "malloc" : " 14 GB", | 20260505 15:44:56.505 | INFO | "pinned" : " 29 GB" | 20260505 15:44:56.505 | INFO | } | 20260505 15:44:56.505 | INFO | Perf report for the last 5.14182 seconds: | 20260505 15:44:56.505 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:44:56.505 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:56.505 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:56.505 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:56.505 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 15:44:56.505 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:56.505 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:56.505 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 15:44:56.505 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:56.505 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:44:56.505 | INFO | Uploaded 36GiB to GPU in 2.24345s (16.0467GiB/s) | 20260505 15:44:56.505 | INFO | Downloaded 34GiB to GPU in 2.11951s (16.0414GiB/s) | 20260505 15:45:01.640 | INFO | Analyzing chunk frames = [4384256,4384768) | 20260505 15:45:01.640 | INFO | MemInfo: { | 20260505 15:45:01.640 | INFO | "gpu" : " 17 GB", | 20260505 15:45:01.640 | INFO | "huge" : "", | 20260505 15:45:01.640 | INFO | "huge_shared" : " 27 GB", | 20260505 15:45:01.640 | INFO | "malloc" : " 14 GB", | 20260505 15:45:01.640 | INFO | "pinned" : " 29 GB" | 20260505 15:45:01.641 | INFO | } | 20260505 15:45:01.641 | INFO | Perf report for the last 5.13607 seconds: | 20260505 15:45:01.641 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:45:01.641 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:01.641 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:01.641 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:01.641 | INFO | ANALYSIS : 62.7%, 3.0, 97.3ms, 0 | 20260505 15:45:01.641 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:01.641 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:01.641 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 15:45:01.641 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:01.641 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:01.641 | INFO | Uploaded 36GiB to GPU in 2.19759s (16.3816GiB/s) | 20260505 15:45:01.641 | INFO | Downloaded 34GiB to GPU in 2.1361s (15.9168GiB/s) | 20260505 15:45:06.784 | INFO | Analyzing chunk frames = [4384768,4385280) | 20260505 15:45:06.784 | INFO | MemInfo: { | 20260505 15:45:06.784 | INFO | "gpu" : " 17 GB", | 20260505 15:45:06.784 | INFO | "huge" : "", | 20260505 15:45:06.784 | INFO | "huge_shared" : " 27 GB", | 20260505 15:45:06.784 | INFO | "malloc" : " 14 GB", | 20260505 15:45:06.784 | INFO | "pinned" : " 29 GB" | 20260505 15:45:06.784 | INFO | } | 20260505 15:45:06.784 | INFO | Perf report for the last 5.14337 seconds: | 20260505 15:45:06.784 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:45:06.784 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:06.784 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:06.784 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:06.784 | INFO | ANALYSIS : 62.5%, 3.0, 97.7ms, 0 | 20260505 15:45:06.784 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:06.784 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:06.784 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:45:06.784 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:06.784 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:06.784 | INFO | Uploaded 36GiB to GPU in 2.18468s (16.4784GiB/s) | 20260505 15:45:06.784 | INFO | Downloaded 34GiB to GPU in 2.13045s (15.959GiB/s) | 20260505 15:45:11.920 | INFO | Analyzing chunk frames = [4385280,4385792) | 20260505 15:45:11.920 | INFO | MemInfo: { | 20260505 15:45:11.920 | INFO | "gpu" : " 17 GB", | 20260505 15:45:11.920 | INFO | "huge" : "", | 20260505 15:45:11.920 | INFO | "huge_shared" : " 27 GB", | 20260505 15:45:11.920 | INFO | "malloc" : " 14 GB", | 20260505 15:45:11.920 | INFO | "pinned" : " 29 GB" | 20260505 15:45:11.920 | INFO | } | 20260505 15:45:11.921 | INFO | Perf report for the last 5.13647 seconds: | 20260505 15:45:11.921 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:45:11.921 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:11.921 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:11.921 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:11.921 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 15:45:11.921 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:11.921 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:11.921 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 15:45:11.921 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:11.921 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:11.921 | INFO | Uploaded 36GiB to GPU in 2.13486s (16.8629GiB/s) | 20260505 15:45:11.921 | INFO | Downloaded 34GiB to GPU in 2.03295s (16.7244GiB/s) | 20260505 15:45:17.055 | INFO | Analyzing chunk frames = [4385792,4386304) | 20260505 15:45:17.055 | INFO | MemInfo: { | 20260505 15:45:17.055 | INFO | "gpu" : " 17 GB", | 20260505 15:45:17.055 | INFO | "huge" : "", | 20260505 15:45:17.055 | INFO | "huge_shared" : " 27 GB", | 20260505 15:45:17.055 | INFO | "malloc" : " 14 GB", | 20260505 15:45:17.055 | INFO | "pinned" : " 29 GB" | 20260505 15:45:17.055 | INFO | } | 20260505 15:45:17.055 | INFO | Perf report for the last 5.13428 seconds: | 20260505 15:45:17.055 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:45:17.055 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:17.055 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:17.055 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:17.055 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 15:45:17.055 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:17.055 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:17.055 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 15:45:17.055 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:17.055 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:17.055 | INFO | Uploaded 36GiB to GPU in 2.13589s (16.8548GiB/s) | 20260505 15:45:17.055 | INFO | Downloaded 34GiB to GPU in 2.16421s (15.7101GiB/s) | 20260505 15:45:22.204 | INFO | Analyzing chunk frames = [4386304,4386816) | 20260505 15:45:22.204 | INFO | MemInfo: { | 20260505 15:45:22.204 | INFO | "gpu" : " 17 GB", | 20260505 15:45:22.204 | INFO | "huge" : "", | 20260505 15:45:22.204 | INFO | "huge_shared" : " 27 GB", | 20260505 15:45:22.204 | INFO | "malloc" : " 14 GB", | 20260505 15:45:22.204 | INFO | "pinned" : " 29 GB" | 20260505 15:45:22.204 | INFO | } | 20260505 15:45:22.204 | INFO | Perf report for the last 5.14965 seconds: | 20260505 15:45:22.204 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:45:22.204 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:22.204 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:22.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:22.205 | INFO | ANALYSIS : 63.3%, 3.0, 99.1ms, 0 | 20260505 15:45:22.205 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:22.205 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:22.205 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 15:45:22.205 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:22.205 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:22.205 | INFO | Uploaded 36GiB to GPU in 2.21792s (16.2314GiB/s) | 20260505 15:45:22.205 | INFO | Downloaded 34GiB to GPU in 2.10043s (16.1872GiB/s) | 20260505 15:45:22.344 | INFO | Starting a round of Baz IO | 20260505 15:45:25.536 | INFO | Populating RTMetrics report | 20260505 15:45:25.537 | INFO | RTMetrics report populated | 20260505 15:45:25.537 | INFO | Opening tmp RTMetrics json file | 20260505 15:45:25.550 | INFO | Serializing RTMetrics to json | 20260505 15:45:25.558 | INFO | Serialization done, swapping tmp file now... | 20260505 15:45:25.573 | INFO | RTMetrics done | 20260505 15:45:25.635 | INFO | Finished a round of Baz IO: Wrote 2.2 GiB in 3.29 seconds | 20260505 15:45:25.635 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.40% / 0.00% / 77.60% | 20260505 15:45:25.635 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.419394% / 3.97364e-06% / 9.53674e-05% | 20260505 15:45:25.635 | INFO | Baz IO overall: | 20260505 15:45:25.635 | INFO | BytesWritten : 2607988785226 | 20260505 15:45:25.635 | INFO | NumEvents : 1238640771136 | 20260505 15:45:27.346 | INFO | Analyzing chunk frames = [4386816,4387328) | 20260505 15:45:27.346 | INFO | MemInfo: { | 20260505 15:45:27.346 | INFO | "gpu" : " 17 GB", | 20260505 15:45:27.346 | INFO | "huge" : "", | 20260505 15:45:27.346 | INFO | "huge_shared" : " 27 GB", | 20260505 15:45:27.346 | INFO | "malloc" : " 14 GB", | 20260505 15:45:27.346 | INFO | "pinned" : " 29 GB" | 20260505 15:45:27.346 | INFO | } | 20260505 15:45:27.346 | INFO | Perf report for the last 5.14176 seconds: | 20260505 15:45:27.346 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:45:27.346 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:27.346 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:27.346 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:27.346 | INFO | ANALYSIS : 63.7%, 3.0, 99.1ms, 0 | 20260505 15:45:27.346 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:27.346 | INFO | RT_METRICS : 45.3%, 1.0, 23.8ms, 0 | 20260505 15:45:27.346 | INFO | PRE_HQ : 60.5%, 1.8, 56.7ms, 0 | 20260505 15:45:27.346 | INFO | BAZWRITER : 63.4%, 3.9, 128.3ms, 0 | 20260505 15:45:27.346 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 15:45:27.347 | INFO | Uploaded 36GiB to GPU in 2.15984s (16.6679GiB/s) | 20260505 15:45:27.347 | INFO | Downloaded 38GiB to GPU in 2.25416s (16.8577GiB/s) | 20260505 15:45:32.472 | INFO | Analyzing chunk frames = [4387328,4387840) | 20260505 15:45:32.472 | INFO | MemInfo: { | 20260505 15:45:32.472 | INFO | "gpu" : " 17 GB", | 20260505 15:45:32.472 | INFO | "huge" : "", | 20260505 15:45:32.472 | INFO | "huge_shared" : " 27 GB", | 20260505 15:45:32.472 | INFO | "malloc" : " 14 GB", | 20260505 15:45:32.472 | INFO | "pinned" : " 29 GB" | 20260505 15:45:32.472 | INFO | } | 20260505 15:45:32.472 | INFO | Perf report for the last 5.12592 seconds: | 20260505 15:45:32.472 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:45:32.472 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:32.472 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:32.472 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:32.472 | INFO | ANALYSIS : 63.2%, 3.0, 98.4ms, 0 | 20260505 15:45:32.472 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:32.472 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:32.472 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:45:32.472 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:32.472 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:32.473 | INFO | Uploaded 36GiB to GPU in 2.13178s (16.8873GiB/s) | 20260505 15:45:32.473 | INFO | Downloaded 34GiB to GPU in 2.0303s (16.7463GiB/s) | 20260505 15:45:37.610 | INFO | Analyzing chunk frames = [4387840,4388352) | 20260505 15:45:37.610 | INFO | MemInfo: { | 20260505 15:45:37.610 | INFO | "gpu" : " 17 GB", | 20260505 15:45:37.610 | INFO | "huge" : "", | 20260505 15:45:37.610 | INFO | "huge_shared" : " 27 GB", | 20260505 15:45:37.610 | INFO | "malloc" : " 14 GB", | 20260505 15:45:37.610 | INFO | "pinned" : " 29 GB" | 20260505 15:45:37.610 | INFO | } | 20260505 15:45:37.610 | INFO | Perf report for the last 5.1382 seconds: | 20260505 15:45:37.610 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:45:37.610 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:37.610 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:37.610 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:37.610 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 15:45:37.610 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:37.610 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:37.610 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:45:37.611 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:37.611 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:37.611 | INFO | Uploaded 36GiB to GPU in 2.24062s (16.067GiB/s) | 20260505 15:45:37.611 | INFO | Downloaded 34GiB to GPU in 2.1185s (16.0491GiB/s) | 20260505 15:45:42.757 | INFO | Analyzing chunk frames = [4388352,4388864) | 20260505 15:45:42.757 | INFO | MemInfo: { | 20260505 15:45:42.757 | INFO | "gpu" : " 17 GB", | 20260505 15:45:42.757 | INFO | "huge" : "", | 20260505 15:45:42.757 | INFO | "huge_shared" : " 27 GB", | 20260505 15:45:42.757 | INFO | "malloc" : " 14 GB", | 20260505 15:45:42.758 | INFO | "pinned" : " 29 GB" | 20260505 15:45:42.758 | INFO | } | 20260505 15:45:42.758 | INFO | Perf report for the last 5.14719 seconds: | 20260505 15:45:42.758 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:45:42.758 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:42.758 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:42.758 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:42.758 | INFO | ANALYSIS : 62.6%, 3.0, 97.4ms, 0 | 20260505 15:45:42.758 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:42.758 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:42.758 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:45:42.758 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:42.758 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:42.758 | INFO | Uploaded 36GiB to GPU in 2.17976s (16.5156GiB/s) | 20260505 15:45:42.758 | INFO | Downloaded 34GiB to GPU in 2.13098s (15.9551GiB/s) | 20260505 15:45:47.907 | INFO | Analyzing chunk frames = [4388864,4389376) | 20260505 15:45:47.907 | INFO | MemInfo: { | 20260505 15:45:47.907 | INFO | "gpu" : " 17 GB", | 20260505 15:45:47.907 | INFO | "huge" : "", | 20260505 15:45:47.907 | INFO | "huge_shared" : " 27 GB", | 20260505 15:45:47.907 | INFO | "malloc" : " 14 GB", | 20260505 15:45:47.907 | INFO | "pinned" : " 29 GB" | 20260505 15:45:47.907 | INFO | } | 20260505 15:45:47.907 | INFO | Perf report for the last 5.14924 seconds: | 20260505 15:45:47.907 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:45:47.907 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:47.907 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:47.907 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:47.907 | INFO | ANALYSIS : 62.2%, 3.0, 97.4ms, 0 | 20260505 15:45:47.907 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:47.907 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:47.907 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 15:45:47.907 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:47.907 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:47.907 | INFO | Uploaded 36GiB to GPU in 2.14431s (16.7886GiB/s) | 20260505 15:45:47.907 | INFO | Downloaded 34GiB to GPU in 2.05614s (16.5359GiB/s) | 20260505 15:45:53.040 | INFO | Analyzing chunk frames = [4389376,4389888) | 20260505 15:45:53.040 | INFO | MemInfo: { | 20260505 15:45:53.040 | INFO | "gpu" : " 17 GB", | 20260505 15:45:53.040 | INFO | "huge" : "", | 20260505 15:45:53.040 | INFO | "huge_shared" : " 27 GB", | 20260505 15:45:53.040 | INFO | "malloc" : " 14 GB", | 20260505 15:45:53.040 | INFO | "pinned" : " 29 GB" | 20260505 15:45:53.040 | INFO | } | 20260505 15:45:53.040 | INFO | Perf report for the last 5.13322 seconds: | 20260505 15:45:53.040 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:45:53.040 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:53.040 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:53.040 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:53.040 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260505 15:45:53.040 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:53.040 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:53.040 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 15:45:53.040 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:53.040 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:53.040 | INFO | Uploaded 36GiB to GPU in 2.13472s (16.864GiB/s) | 20260505 15:45:53.040 | INFO | Downloaded 34GiB to GPU in 2.04081s (16.66GiB/s) | 20260505 15:45:58.172 | INFO | Analyzing chunk frames = [4389888,4390400) | 20260505 15:45:58.172 | INFO | MemInfo: { | 20260505 15:45:58.172 | INFO | "gpu" : " 17 GB", | 20260505 15:45:58.172 | INFO | "huge" : "", | 20260505 15:45:58.172 | INFO | "huge_shared" : " 27 GB", | 20260505 15:45:58.172 | INFO | "malloc" : " 14 GB", | 20260505 15:45:58.172 | INFO | "pinned" : " 29 GB" | 20260505 15:45:58.172 | INFO | } | 20260505 15:45:58.172 | INFO | Perf report for the last 5.13194 seconds: | 20260505 15:45:58.172 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:45:58.172 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:58.172 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:58.172 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:58.172 | INFO | ANALYSIS : 63.6%, 3.0, 99.1ms, 0 | 20260505 15:45:58.172 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:58.172 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:58.172 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 15:45:58.172 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:58.172 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:45:58.172 | INFO | Uploaded 36GiB to GPU in 2.13372s (16.872GiB/s) | 20260505 15:45:58.172 | INFO | Downloaded 34GiB to GPU in 2.16057s (15.7366GiB/s) | 20260505 15:46:03.312 | INFO | Analyzing chunk frames = [4390400,4390912) | 20260505 15:46:03.312 | INFO | MemInfo: { | 20260505 15:46:03.312 | INFO | "gpu" : " 17 GB", | 20260505 15:46:03.312 | INFO | "huge" : "", | 20260505 15:46:03.312 | INFO | "huge_shared" : " 27 GB", | 20260505 15:46:03.312 | INFO | "malloc" : " 14 GB", | 20260505 15:46:03.312 | INFO | "pinned" : " 29 GB" | 20260505 15:46:03.312 | INFO | } | 20260505 15:46:03.312 | INFO | Perf report for the last 5.14019 seconds: | 20260505 15:46:03.312 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:46:03.312 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:03.312 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:03.312 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:03.312 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 15:46:03.312 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:03.312 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:03.312 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 15:46:03.312 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:03.312 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:03.312 | INFO | Uploaded 36GiB to GPU in 2.22066s (16.2114GiB/s) | 20260505 15:46:03.312 | INFO | Downloaded 34GiB to GPU in 2.10556s (16.1477GiB/s) | 20260505 15:46:03.313 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 15:46:03.313 | INFO | Total mem usage: 15.3 GiB | 20260505 15:46:03.313 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 15:46:03.313 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.86%) | 20260505 15:46:03.313 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 15:46:03.313 | INFO | High Water Breakdown: | 20260505 15:46:03.313 | INFO | 61% : 8.67 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 15:46:03.313 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 15:46:03.313 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 15:46:03.313 | INFO | ------------------------------------------ | 20260505 15:46:03.313 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:46:03.313 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 15:46:03.313 | INFO | Total mem usage: 26.5 GiB | 20260505 15:46:03.313 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 15:46:03.313 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 15:46:03.313 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 15:46:03.313 | INFO | High Water Breakdown: | 20260505 15:46:03.313 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:46:03.313 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:46:03.313 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:46:03.313 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:46:03.313 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:46:03.313 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:46:03.313 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 15:46:03.313 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 15:46:03.313 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:46:03.313 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 15:46:03.313 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 15:46:03.313 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 15:46:03.313 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 15:46:03.313 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 15:46:03.313 | INFO | ------------------------------------------ | 20260505 15:46:03.313 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:46:03.313 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 15:46:03.313 | INFO | Total mem usage: 25.6 GiB | 20260505 15:46:03.313 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 15:46:03.313 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 15:46:03.313 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 15:46:03.313 | INFO | High Water Breakdown: | 20260505 15:46:03.313 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 15:46:03.313 | INFO | ------------------------------------------ | 20260505 15:46:03.313 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:46:03.313 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 15:46:03.313 | INFO | Total mem usage: 15.7 GiB | 20260505 15:46:03.313 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 15:46:03.313 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 15:46:03.313 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 15:46:03.313 | INFO | High Water Breakdown: | 20260505 15:46:03.313 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 15:46:03.313 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 15:46:03.313 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 15:46:03.313 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 15:46:03.313 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 15:46:03.313 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 15:46:03.313 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 15:46:03.313 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 15:46:03.313 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 15:46:03.313 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:46:03.313 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 15:46:03.313 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:46:03.313 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 15:46:03.313 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 15:46:03.313 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 15:46:03.313 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 15:46:03.313 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:46:03.313 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 15:46:03.313 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:46:03.314 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:46:03.314 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:46:03.314 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 15:46:03.314 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 15:46:03.314 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:46:03.314 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 15:46:03.314 | INFO | ------------------------------------------ | 20260505 15:46:03.314 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:46:03.314 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 15:46:03.314 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:46:03.314 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:46:03.314 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.55e+01, 1.38e+01 / 2.15e+00 | 20260505 15:46:03.314 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 1.02e-01, 1.05e-02 / 4.99e-03 | 20260505 15:46:03.314 | INFO | PerfReport: 11.76%, Baseline -- 3.73e+00 / 4.48e+00, 5.67e-01 / 2.22e-01 | 20260505 15:46:03.314 | INFO | PerfReport: 3.29%, Histogram -- 1.04e+00 / 1.49e+00, 1.94e-01 / 1.08e-01 | 20260505 15:46:03.314 | INFO | PerfReport: 15.95%, DME -- 4.05e+01 / 4.49e+01, 4.35e+00 / 2.12e+00 | 20260505 15:46:03.314 | INFO | PerfReport: 56.65%, FrameLabeling -- 1.80e+01 / 1.98e+01, 1.43e+00 / 1.65e+00 | 20260505 15:46:03.314 | INFO | PerfReport: 7.39%, PulseAccumulating -- 2.34e+00 / 2.90e+00, 4.47e-01 / 1.84e-01 | 20260505 15:46:03.314 | INFO | PerfReport: 3.47%, Metrics -- 1.10e+00 / 1.38e+00, 3.20e-01 / 1.49e-01 | 20260505 15:46:03.314 | INFO | PerfReport: 1.21%, PulseSerializing -- 3.82e-01 / 4.34e-01, 6.84e-02 / 3.22e-02 | 20260505 15:46:03.314 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 15:46:03.314 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:46:03.314 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:46:03.314 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260505 15:46:03.314 | INFO | PerfReport: 22.59%, Upload -- 2.27e+01 / 0.00e+00, 8.07e+00 / 0.00e+00 | 20260505 15:46:03.314 | INFO | PerfReport: 21.96%, Download -- 2.21e+01 / 0.00e+00, 8.00e+00 / 0.00e+00 | 20260505 15:46:03.314 | INFO | PerfReport: 31.57%, Compute -- 3.17e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 15:46:03.314 | INFO | PerfReport: 2.33%, LockUpload -- 2.34e+00 / 0.00e+00, 7.82e+00 / 0.00e+00 | 20260505 15:46:03.314 | INFO | PerfReport: 1.34%, LockDownload -- 1.35e+00 / 0.00e+00, 2.84e+00 / 0.00e+00 | 20260505 15:46:03.314 | INFO | PerfReport: 20.20%, LockCompute -- 2.03e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 15:46:03.314 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.65e-02 / 0.00e+00 | 20260505 15:46:03.462 | INFO | Starting a round of Baz IO | 20260505 15:46:06.655 | INFO | Populating RTMetrics report | 20260505 15:46:06.655 | INFO | RTMetrics report populated | 20260505 15:46:06.655 | INFO | Opening tmp RTMetrics json file | 20260505 15:46:06.662 | INFO | Serializing RTMetrics to json | 20260505 15:46:06.664 | INFO | Serialization done, swapping tmp file now... | 20260505 15:46:06.665 | INFO | RTMetrics done | 20260505 15:46:06.755 | INFO | Finished a round of Baz IO: Wrote 2.2 GiB in 3.292 seconds | 20260505 15:46:06.755 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.39% / 0.00% / 77.60% | 20260505 15:46:06.755 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.418599% / 3.97364e-06% / 7.54992e-05% | 20260505 15:46:06.755 | INFO | Baz IO overall: | 20260505 15:46:06.755 | INFO | BytesWritten : 2610351253355 | 20260505 15:46:06.755 | INFO | NumEvents : 1239750078613 | 20260505 15:46:08.457 | INFO | Analyzing chunk frames = [4390912,4391424) | 20260505 15:46:08.457 | INFO | MemInfo: { | 20260505 15:46:08.457 | INFO | "gpu" : " 17 GB", | 20260505 15:46:08.457 | INFO | "huge" : "", | 20260505 15:46:08.457 | INFO | "huge_shared" : " 27 GB", | 20260505 15:46:08.457 | INFO | "malloc" : " 14 GB", | 20260505 15:46:08.457 | INFO | "pinned" : " 29 GB" | 20260505 15:46:08.457 | INFO | } | 20260505 15:46:08.457 | INFO | Perf report for the last 5.1448 seconds: | 20260505 15:46:08.457 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:46:08.457 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:08.457 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:08.457 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:08.457 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 15:46:08.457 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:08.457 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260505 15:46:08.457 | INFO | PRE_HQ : 60.6%, 1.8, 56.8ms, 0 | 20260505 15:46:08.457 | INFO | BAZWRITER : 64.0%, 3.9, 127.1ms, 0 | 20260505 15:46:08.457 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 15:46:08.457 | INFO | Uploaded 36GiB to GPU in 2.17653s (16.5401GiB/s) | 20260505 15:46:08.457 | INFO | Downloaded 38GiB to GPU in 2.25667s (16.839GiB/s) | 20260505 15:46:13.594 | INFO | Analyzing chunk frames = [4391424,4391936) | 20260505 15:46:13.594 | INFO | MemInfo: { | 20260505 15:46:13.594 | INFO | "gpu" : " 17 GB", | 20260505 15:46:13.594 | INFO | "huge" : "", | 20260505 15:46:13.594 | INFO | "huge_shared" : " 27 GB", | 20260505 15:46:13.594 | INFO | "malloc" : " 14 GB", | 20260505 15:46:13.594 | INFO | "pinned" : " 29 GB" | 20260505 15:46:13.594 | INFO | } | 20260505 15:46:13.594 | INFO | Perf report for the last 5.13732 seconds: | 20260505 15:46:13.594 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:46:13.594 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:13.594 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:13.594 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:13.594 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 15:46:13.594 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:13.594 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:13.594 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 15:46:13.594 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:13.594 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:13.595 | INFO | Uploaded 36GiB to GPU in 2.13247s (16.8819GiB/s) | 20260505 15:46:13.595 | INFO | Downloaded 34GiB to GPU in 2.03272s (16.7263GiB/s) | 20260505 15:46:18.731 | INFO | Analyzing chunk frames = [4391936,4392448) | 20260505 15:46:18.731 | INFO | MemInfo: { | 20260505 15:46:18.731 | INFO | "gpu" : " 17 GB", | 20260505 15:46:18.731 | INFO | "huge" : "", | 20260505 15:46:18.731 | INFO | "huge_shared" : " 27 GB", | 20260505 15:46:18.731 | INFO | "malloc" : " 14 GB", | 20260505 15:46:18.731 | INFO | "pinned" : " 29 GB" | 20260505 15:46:18.731 | INFO | } | 20260505 15:46:18.731 | INFO | Perf report for the last 5.13703 seconds: | 20260505 15:46:18.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:46:18.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:18.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:18.731 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:18.731 | INFO | ANALYSIS : 62.8%, 3.0, 97.8ms, 0 | 20260505 15:46:18.731 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:18.731 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:18.732 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 15:46:18.732 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:18.732 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:18.732 | INFO | Uploaded 36GiB to GPU in 2.17716s (16.5353GiB/s) | 20260505 15:46:18.732 | INFO | Downloaded 34GiB to GPU in 2.03333s (16.7213GiB/s) | 20260505 15:46:23.866 | INFO | Analyzing chunk frames = [4392448,4392960) | 20260505 15:46:23.866 | INFO | MemInfo: { | 20260505 15:46:23.866 | INFO | "gpu" : " 17 GB", | 20260505 15:46:23.866 | INFO | "huge" : "", | 20260505 15:46:23.866 | INFO | "huge_shared" : " 27 GB", | 20260505 15:46:23.866 | INFO | "malloc" : " 14 GB", | 20260505 15:46:23.866 | INFO | "pinned" : " 29 GB" | 20260505 15:46:23.867 | INFO | } | 20260505 15:46:23.867 | INFO | Perf report for the last 5.13516 seconds: | 20260505 15:46:23.867 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:46:23.867 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:23.867 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:23.867 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:23.867 | INFO | ANALYSIS : 62.6%, 3.0, 97.2ms, 0 | 20260505 15:46:23.867 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:23.867 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:23.867 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:46:23.867 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:23.867 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:23.867 | INFO | Uploaded 36GiB to GPU in 2.17096s (16.5825GiB/s) | 20260505 15:46:23.867 | INFO | Downloaded 34GiB to GPU in 2.13642s (15.9145GiB/s) | 20260505 15:46:29.008 | INFO | Analyzing chunk frames = [4392960,4393472) | 20260505 15:46:29.008 | INFO | MemInfo: { | 20260505 15:46:29.008 | INFO | "gpu" : " 17 GB", | 20260505 15:46:29.008 | INFO | "huge" : "", | 20260505 15:46:29.008 | INFO | "huge_shared" : " 27 GB", | 20260505 15:46:29.008 | INFO | "malloc" : " 14 GB", | 20260505 15:46:29.008 | INFO | "pinned" : " 29 GB" | 20260505 15:46:29.008 | INFO | } | 20260505 15:46:29.009 | INFO | Perf report for the last 5.14192 seconds: | 20260505 15:46:29.009 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:46:29.009 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:29.009 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:29.009 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:29.009 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260505 15:46:29.009 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:29.009 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:29.009 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 15:46:29.009 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:29.009 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:29.009 | INFO | Uploaded 36GiB to GPU in 2.179s (16.5213GiB/s) | 20260505 15:46:29.009 | INFO | Downloaded 34GiB to GPU in 2.13212s (15.9466GiB/s) | 20260505 15:46:34.152 | INFO | Analyzing chunk frames = [4393472,4393984) | 20260505 15:46:34.152 | INFO | MemInfo: { | 20260505 15:46:34.152 | INFO | "gpu" : " 17 GB", | 20260505 15:46:34.152 | INFO | "huge" : "", | 20260505 15:46:34.153 | INFO | "huge_shared" : " 27 GB", | 20260505 15:46:34.153 | INFO | "malloc" : " 14 GB", | 20260505 15:46:34.153 | INFO | "pinned" : " 29 GB" | 20260505 15:46:34.153 | INFO | } | 20260505 15:46:34.153 | INFO | Perf report for the last 5.14416 seconds: | 20260505 15:46:34.153 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:46:34.153 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:34.153 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:34.153 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:34.153 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 15:46:34.153 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:34.153 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:34.153 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 15:46:34.153 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:34.153 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:34.153 | INFO | Uploaded 36GiB to GPU in 2.1724s (16.5715GiB/s) | 20260505 15:46:34.153 | INFO | Downloaded 34GiB to GPU in 2.04253s (16.646GiB/s) | 20260505 15:46:39.290 | INFO | Analyzing chunk frames = [4393984,4394496) | 20260505 15:46:39.290 | INFO | MemInfo: { | 20260505 15:46:39.290 | INFO | "gpu" : " 17 GB", | 20260505 15:46:39.290 | INFO | "huge" : "", | 20260505 15:46:39.290 | INFO | "huge_shared" : " 27 GB", | 20260505 15:46:39.290 | INFO | "malloc" : " 14 GB", | 20260505 15:46:39.290 | INFO | "pinned" : " 29 GB" | 20260505 15:46:39.290 | INFO | } | 20260505 15:46:39.290 | INFO | Perf report for the last 5.13743 seconds: | 20260505 15:46:39.290 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:46:39.290 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:39.290 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:39.290 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:39.290 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 15:46:39.290 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:39.290 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:39.290 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 15:46:39.290 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:39.290 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:39.290 | INFO | Uploaded 36GiB to GPU in 2.11565s (17.016GiB/s) | 20260505 15:46:39.290 | INFO | Downloaded 34GiB to GPU in 2.16136s (15.7308GiB/s) | 20260505 15:46:44.421 | INFO | Analyzing chunk frames = [4394496,4395008) | 20260505 15:46:44.421 | INFO | MemInfo: { | 20260505 15:46:44.421 | INFO | "gpu" : " 17 GB", | 20260505 15:46:44.421 | INFO | "huge" : "", | 20260505 15:46:44.421 | INFO | "huge_shared" : " 27 GB", | 20260505 15:46:44.421 | INFO | "malloc" : " 14 GB", | 20260505 15:46:44.421 | INFO | "pinned" : " 29 GB" | 20260505 15:46:44.421 | INFO | } | 20260505 15:46:44.421 | INFO | Perf report for the last 5.13084 seconds: | 20260505 15:46:44.421 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:46:44.421 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:44.421 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:44.421 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:44.421 | INFO | ANALYSIS : 63.8%, 3.0, 99.4ms, 0 | 20260505 15:46:44.421 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:44.421 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:44.421 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:46:44.421 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:44.421 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:44.421 | INFO | Uploaded 36GiB to GPU in 2.23379s (16.1161GiB/s) | 20260505 15:46:44.421 | INFO | Downloaded 34GiB to GPU in 2.10662s (16.1396GiB/s) | 20260505 15:46:44.557 | INFO | Starting a round of Baz IO | 20260505 15:46:47.769 | INFO | Populating RTMetrics report | 20260505 15:46:47.770 | INFO | RTMetrics report populated | 20260505 15:46:47.770 | INFO | Opening tmp RTMetrics json file | 20260505 15:46:47.783 | INFO | Serializing RTMetrics to json | 20260505 15:46:47.785 | INFO | Serialization done, swapping tmp file now... | 20260505 15:46:47.786 | INFO | RTMetrics done | 20260505 15:46:47.869 | INFO | Finished a round of Baz IO: Wrote 2.201 GiB in 3.312 seconds | 20260505 15:46:47.869 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.39% / 0.00% / 77.61% | 20260505 15:46:47.869 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.418508% / 7.94729e-06% / 6.35783e-05% | 20260505 15:46:47.869 | INFO | Baz IO overall: | 20260505 15:46:47.869 | INFO | BytesWritten : 2612714983657 | 20260505 15:46:47.869 | INFO | NumEvents : 1240860637541 | 20260505 15:46:49.564 | INFO | Analyzing chunk frames = [4395008,4395520) | 20260505 15:46:49.564 | INFO | MemInfo: { | 20260505 15:46:49.564 | INFO | "gpu" : " 17 GB", | 20260505 15:46:49.564 | INFO | "huge" : "", | 20260505 15:46:49.565 | INFO | "huge_shared" : " 27 GB", | 20260505 15:46:49.565 | INFO | "malloc" : " 14 GB", | 20260505 15:46:49.565 | INFO | "pinned" : " 29 GB" | 20260505 15:46:49.565 | INFO | } | 20260505 15:46:49.565 | INFO | Perf report for the last 5.14368 seconds: | 20260505 15:46:49.565 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:46:49.565 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:49.565 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:49.565 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:49.565 | INFO | ANALYSIS : 64.0%, 3.0, 99.6ms, 0 | 20260505 15:46:49.565 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:49.565 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260505 15:46:49.565 | INFO | PRE_HQ : 60.5%, 1.8, 56.9ms, 0 | 20260505 15:46:49.565 | INFO | BAZWRITER : 63.7%, 3.9, 127.3ms, 0 | 20260505 15:46:49.565 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 15:46:49.565 | INFO | Uploaded 36GiB to GPU in 2.17977s (16.5155GiB/s) | 20260505 15:46:49.565 | INFO | Downloaded 38GiB to GPU in 2.2662s (16.7682GiB/s) | 20260505 15:46:54.712 | INFO | Analyzing chunk frames = [4395520,4396032) | 20260505 15:46:54.712 | INFO | MemInfo: { | 20260505 15:46:54.712 | INFO | "gpu" : " 17 GB", | 20260505 15:46:54.712 | INFO | "huge" : "", | 20260505 15:46:54.712 | INFO | "huge_shared" : " 27 GB", | 20260505 15:46:54.713 | INFO | "malloc" : " 14 GB", | 20260505 15:46:54.713 | INFO | "pinned" : " 29 GB" | 20260505 15:46:54.713 | INFO | } | 20260505 15:46:54.713 | INFO | Perf report for the last 5.14794 seconds: | 20260505 15:46:54.713 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:46:54.713 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:54.713 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:54.713 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:54.713 | INFO | ANALYSIS : 63.2%, 3.0, 98.9ms, 0 | 20260505 15:46:54.713 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:54.713 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:54.713 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 15:46:54.713 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:54.713 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:54.713 | INFO | Uploaded 36GiB to GPU in 2.20412s (16.3331GiB/s) | 20260505 15:46:54.713 | INFO | Downloaded 34GiB to GPU in 2.06874s (16.4351GiB/s) | 20260505 15:46:59.854 | INFO | Analyzing chunk frames = [4396032,4396544) | 20260505 15:46:59.854 | INFO | MemInfo: { | 20260505 15:46:59.854 | INFO | "gpu" : " 17 GB", | 20260505 15:46:59.854 | INFO | "huge" : "", | 20260505 15:46:59.854 | INFO | "huge_shared" : " 27 GB", | 20260505 15:46:59.854 | INFO | "malloc" : " 14 GB", | 20260505 15:46:59.854 | INFO | "pinned" : " 29 GB" | 20260505 15:46:59.854 | INFO | } | 20260505 15:46:59.854 | INFO | Perf report for the last 5.14163 seconds: | 20260505 15:46:59.854 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:46:59.854 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:59.854 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:59.854 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:59.855 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260505 15:46:59.855 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:59.855 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:59.855 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 15:46:59.855 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:59.855 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:46:59.855 | INFO | Uploaded 36GiB to GPU in 2.17461s (16.5547GiB/s) | 20260505 15:46:59.855 | INFO | Downloaded 34GiB to GPU in 2.02866s (16.7599GiB/s) | 20260505 15:47:04.987 | INFO | Analyzing chunk frames = [4396544,4397056) | 20260505 15:47:04.987 | INFO | MemInfo: { | 20260505 15:47:04.987 | INFO | "gpu" : " 17 GB", | 20260505 15:47:04.987 | INFO | "huge" : "", | 20260505 15:47:04.987 | INFO | "huge_shared" : " 27 GB", | 20260505 15:47:04.987 | INFO | "malloc" : " 14 GB", | 20260505 15:47:04.987 | INFO | "pinned" : " 29 GB" | 20260505 15:47:04.987 | INFO | } | 20260505 15:47:04.987 | INFO | Perf report for the last 5.13277 seconds: | 20260505 15:47:04.987 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:47:04.987 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:04.987 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:04.987 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:04.987 | INFO | ANALYSIS : 62.8%, 3.0, 97.4ms, 0 | 20260505 15:47:04.987 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:04.987 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:04.987 | INFO | PRE_HQ : 28.1%, 1.0, 14.5ms, 0 | 20260505 15:47:04.987 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:04.987 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:04.987 | INFO | Uploaded 36GiB to GPU in 2.2132s (16.2661GiB/s) | 20260505 15:47:04.987 | INFO | Downloaded 34GiB to GPU in 2.13276s (15.9418GiB/s) | 20260505 15:47:10.116 | INFO | Analyzing chunk frames = [4397056,4397568) | 20260505 15:47:10.116 | INFO | MemInfo: { | 20260505 15:47:10.116 | INFO | "gpu" : " 17 GB", | 20260505 15:47:10.116 | INFO | "huge" : "", | 20260505 15:47:10.116 | INFO | "huge_shared" : " 27 GB", | 20260505 15:47:10.116 | INFO | "malloc" : " 14 GB", | 20260505 15:47:10.116 | INFO | "pinned" : " 29 GB" | 20260505 15:47:10.116 | INFO | } | 20260505 15:47:10.116 | INFO | Perf report for the last 5.12934 seconds: | 20260505 15:47:10.116 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:47:10.116 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:10.116 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:10.116 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:10.116 | INFO | ANALYSIS : 62.9%, 3.0, 97.9ms, 0 | 20260505 15:47:10.116 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:10.116 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:10.117 | INFO | PRE_HQ : 28.1%, 1.0, 14.5ms, 0 | 20260505 15:47:10.117 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:10.117 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:10.117 | INFO | Uploaded 36GiB to GPU in 2.19404s (16.4081GiB/s) | 20260505 15:47:10.117 | INFO | Downloaded 34GiB to GPU in 2.13105s (15.9545GiB/s) | 20260505 15:47:15.266 | INFO | Analyzing chunk frames = [4397568,4398080) | 20260505 15:47:15.266 | INFO | MemInfo: { | 20260505 15:47:15.266 | INFO | "gpu" : " 17 GB", | 20260505 15:47:15.266 | INFO | "huge" : "", | 20260505 15:47:15.267 | INFO | "huge_shared" : " 27 GB", | 20260505 15:47:15.267 | INFO | "malloc" : " 14 GB", | 20260505 15:47:15.267 | INFO | "pinned" : " 29 GB" | 20260505 15:47:15.267 | INFO | } | 20260505 15:47:15.267 | INFO | Perf report for the last 5.15026 seconds: | 20260505 15:47:15.267 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:47:15.267 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:15.267 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:15.267 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:15.267 | INFO | ANALYSIS : 62.9%, 3.0, 98.4ms, 0 | 20260505 15:47:15.267 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:15.267 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:15.267 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 15:47:15.267 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:15.267 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:15.267 | INFO | Uploaded 36GiB to GPU in 2.17822s (16.5273GiB/s) | 20260505 15:47:15.267 | INFO | Downloaded 34GiB to GPU in 2.11149s (16.1024GiB/s) | 20260505 15:47:20.411 | INFO | Analyzing chunk frames = [4398080,4398592) | 20260505 15:47:20.411 | INFO | MemInfo: { | 20260505 15:47:20.411 | INFO | "gpu" : " 17 GB", | 20260505 15:47:20.411 | INFO | "huge" : "", | 20260505 15:47:20.412 | INFO | "huge_shared" : " 27 GB", | 20260505 15:47:20.412 | INFO | "malloc" : " 14 GB", | 20260505 15:47:20.412 | INFO | "pinned" : " 29 GB" | 20260505 15:47:20.412 | INFO | } | 20260505 15:47:20.412 | INFO | Perf report for the last 5.14492 seconds: | 20260505 15:47:20.412 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:47:20.412 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:20.412 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:20.412 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:20.412 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 15:47:20.412 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:20.412 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:20.412 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 15:47:20.412 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:20.412 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:20.412 | INFO | Uploaded 36GiB to GPU in 2.12106s (16.9726GiB/s) | 20260505 15:47:20.412 | INFO | Downloaded 34GiB to GPU in 2.15572s (15.772GiB/s) | 20260505 15:47:25.547 | INFO | Analyzing chunk frames = [4398592,4399104) | 20260505 15:47:25.547 | INFO | MemInfo: { | 20260505 15:47:25.547 | INFO | "gpu" : " 17 GB", | 20260505 15:47:25.547 | INFO | "huge" : "", | 20260505 15:47:25.547 | INFO | "huge_shared" : " 27 GB", | 20260505 15:47:25.547 | INFO | "malloc" : " 14 GB", | 20260505 15:47:25.547 | INFO | "pinned" : " 29 GB" | 20260505 15:47:25.547 | INFO | } | 20260505 15:47:25.548 | INFO | Perf report for the last 5.13591 seconds: | 20260505 15:47:25.548 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:47:25.548 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:25.548 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:25.548 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:25.548 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 15:47:25.548 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:25.548 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:25.548 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 15:47:25.548 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:25.548 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:25.548 | INFO | Uploaded 36GiB to GPU in 2.15658s (16.6931GiB/s) | 20260505 15:47:25.548 | INFO | Downloaded 34GiB to GPU in 2.02093s (16.8239GiB/s) | 20260505 15:47:25.548 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 15:47:25.548 | INFO | Total mem usage: 15.3 GiB | 20260505 15:47:25.548 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 15:47:25.548 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.71%) | 20260505 15:47:25.548 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 15:47:25.548 | INFO | High Water Breakdown: | 20260505 15:47:25.548 | INFO | 61% : 8.68 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 15:47:25.548 | INFO | 38.9% : 5.53 GiB ( 83.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 15:47:25.548 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 15:47:25.548 | INFO | ------------------------------------------ | 20260505 15:47:25.548 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:47:25.548 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 15:47:25.548 | INFO | Total mem usage: 26.5 GiB | 20260505 15:47:25.548 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 15:47:25.548 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 15:47:25.548 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 15:47:25.548 | INFO | High Water Breakdown: | 20260505 15:47:25.548 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:47:25.548 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:47:25.548 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:47:25.548 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:47:25.548 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:47:25.548 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:47:25.548 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 15:47:25.548 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 15:47:25.548 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:47:25.548 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 15:47:25.548 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 15:47:25.548 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 15:47:25.548 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 15:47:25.548 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 15:47:25.548 | INFO | ------------------------------------------ | 20260505 15:47:25.548 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:47:25.548 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 15:47:25.548 | INFO | Total mem usage: 25.6 GiB | 20260505 15:47:25.548 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 15:47:25.548 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 15:47:25.548 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 15:47:25.548 | INFO | High Water Breakdown: | 20260505 15:47:25.548 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 15:47:25.548 | INFO | ------------------------------------------ | 20260505 15:47:25.548 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:47:25.549 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 15:47:25.549 | INFO | Total mem usage: 15.7 GiB | 20260505 15:47:25.549 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 15:47:25.549 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 15:47:25.549 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 15:47:25.549 | INFO | High Water Breakdown: | 20260505 15:47:25.549 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 15:47:25.549 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 15:47:25.549 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 15:47:25.549 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 15:47:25.549 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 15:47:25.549 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 15:47:25.549 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 15:47:25.549 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 15:47:25.549 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 15:47:25.549 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:47:25.549 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 15:47:25.549 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:47:25.549 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 15:47:25.549 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 15:47:25.549 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 15:47:25.549 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 15:47:25.549 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:47:25.549 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 15:47:25.549 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:47:25.549 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:47:25.549 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:47:25.549 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 15:47:25.549 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 15:47:25.549 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:47:25.549 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 15:47:25.549 | INFO | ------------------------------------------ | 20260505 15:47:25.549 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:47:25.549 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 15:47:25.549 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:47:25.549 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:47:25.549 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.55e+01, 1.36e+01 / 1.83e+00 | 20260505 15:47:25.549 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 1.01e-01, 1.08e-02 / 4.24e-03 | 20260505 15:47:25.549 | INFO | PerfReport: 11.78%, Baseline -- 3.73e+00 / 4.49e+00, 5.72e-01 / 1.72e-01 | 20260505 15:47:25.549 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.50e+00, 1.90e-01 / 9.27e-02 | 20260505 15:47:25.549 | INFO | PerfReport: 15.97%, DME -- 4.05e+01 / 4.51e+01, 4.23e+00 / 1.55e+00 | 20260505 15:47:25.549 | INFO | PerfReport: 56.58%, FrameLabeling -- 1.79e+01 / 1.95e+01, 1.40e+00 / 1.44e+00 | 20260505 15:47:25.549 | INFO | PerfReport: 7.43%, PulseAccumulating -- 2.35e+00 / 2.92e+00, 4.54e-01 / 1.51e-01 | 20260505 15:47:25.549 | INFO | PerfReport: 3.46%, Metrics -- 1.10e+00 / 1.44e+00, 3.04e-01 / 1.56e-01 | 20260505 15:47:25.549 | INFO | PerfReport: 1.21%, PulseSerializing -- 3.82e-01 / 4.45e-01, 6.88e-02 / 3.11e-02 | 20260505 15:47:25.549 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 15:47:25.549 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:47:25.549 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:47:25.549 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260505 15:47:25.549 | INFO | PerfReport: 22.55%, Upload -- 2.26e+01 / 0.00e+00, 7.94e+00 / 0.00e+00 | 20260505 15:47:25.549 | INFO | PerfReport: 21.93%, Download -- 2.20e+01 / 0.00e+00, 7.98e+00 / 0.00e+00 | 20260505 15:47:25.549 | INFO | PerfReport: 31.56%, Compute -- 3.17e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 15:47:25.549 | INFO | PerfReport: 2.34%, LockUpload -- 2.35e+00 / 0.00e+00, 8.06e+00 / 0.00e+00 | 20260505 15:47:25.549 | INFO | PerfReport: 1.45%, LockDownload -- 1.45e+00 / 0.00e+00, 3.02e+00 / 0.00e+00 | 20260505 15:47:25.549 | INFO | PerfReport: 20.15%, LockCompute -- 2.02e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 15:47:25.549 | INFO | PerfReport: 0.01%, Misc -- 1.23e-02 / 0.00e+00, 1.07e-02 / 0.00e+00 | 20260505 15:47:25.676 | INFO | Starting a round of Baz IO | 20260505 15:47:28.895 | INFO | Populating RTMetrics report | 20260505 15:47:28.895 | INFO | RTMetrics report populated | 20260505 15:47:28.895 | INFO | Opening tmp RTMetrics json file | 20260505 15:47:28.909 | INFO | Serializing RTMetrics to json | 20260505 15:47:28.910 | INFO | Serialization done, swapping tmp file now... | 20260505 15:47:28.910 | INFO | RTMetrics done | 20260505 15:47:28.999 | INFO | Finished a round of Baz IO: Wrote 2.2 GiB in 3.322 seconds | 20260505 15:47:28.999 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.39% / 0.00% / 77.61% | 20260505 15:47:28.999 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.416172% / 1.19209e-05% / 5.5631e-05% | 20260505 15:47:28.999 | INFO | Baz IO overall: | 20260505 15:47:28.999 | INFO | BytesWritten : 2615077005975 | 20260505 15:47:28.999 | INFO | NumEvents : 1241969882458 | 20260505 15:47:30.683 | INFO | Analyzing chunk frames = [4399104,4399616) | 20260505 15:47:30.683 | INFO | MemInfo: { | 20260505 15:47:30.683 | INFO | "gpu" : " 17 GB", | 20260505 15:47:30.683 | INFO | "huge" : "", | 20260505 15:47:30.683 | INFO | "huge_shared" : " 27 GB", | 20260505 15:47:30.683 | INFO | "malloc" : " 14 GB", | 20260505 15:47:30.683 | INFO | "pinned" : " 29 GB" | 20260505 15:47:30.683 | INFO | } | 20260505 15:47:30.683 | INFO | Perf report for the last 5.13589 seconds: | 20260505 15:47:30.684 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:47:30.684 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:30.684 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:30.684 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:30.684 | INFO | ANALYSIS : 64.0%, 3.0, 99.6ms, 0 | 20260505 15:47:30.684 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:30.684 | INFO | RT_METRICS : 45.1%, 1.0, 23.6ms, 0 | 20260505 15:47:30.684 | INFO | PRE_HQ : 61.0%, 1.8, 57.0ms, 0 | 20260505 15:47:30.684 | INFO | BAZWRITER : 64.1%, 3.9, 128.4ms, 0 | 20260505 15:47:30.684 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 15:47:30.684 | INFO | Uploaded 36GiB to GPU in 2.18138s (16.5033GiB/s) | 20260505 15:47:30.684 | INFO | Downloaded 38GiB to GPU in 2.26715s (16.7612GiB/s) | 20260505 15:47:35.817 | INFO | Analyzing chunk frames = [4399616,4400128) | 20260505 15:47:35.817 | INFO | MemInfo: { | 20260505 15:47:35.817 | INFO | "gpu" : " 17 GB", | 20260505 15:47:35.817 | INFO | "huge" : "", | 20260505 15:47:35.817 | INFO | "huge_shared" : " 27 GB", | 20260505 15:47:35.817 | INFO | "malloc" : " 14 GB", | 20260505 15:47:35.817 | INFO | "pinned" : " 29 GB" | 20260505 15:47:35.817 | INFO | } | 20260505 15:47:35.817 | INFO | Perf report for the last 5.13392 seconds: | 20260505 15:47:35.817 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:47:35.817 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:35.817 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:35.817 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:35.817 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 15:47:35.817 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:35.817 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:35.817 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 15:47:35.817 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:35.817 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:35.818 | INFO | Uploaded 36GiB to GPU in 2.20313s (16.3404GiB/s) | 20260505 15:47:35.818 | INFO | Downloaded 34GiB to GPU in 2.05097s (16.5775GiB/s) | 20260505 15:47:40.965 | INFO | Analyzing chunk frames = [4400128,4400640) | 20260505 15:47:40.965 | INFO | MemInfo: { | 20260505 15:47:40.965 | INFO | "gpu" : " 17 GB", | 20260505 15:47:40.965 | INFO | "huge" : "", | 20260505 15:47:40.965 | INFO | "huge_shared" : " 27 GB", | 20260505 15:47:40.965 | INFO | "malloc" : " 14 GB", | 20260505 15:47:40.965 | INFO | "pinned" : " 29 GB" | 20260505 15:47:40.965 | INFO | } | 20260505 15:47:40.965 | INFO | Perf report for the last 5.148 seconds: | 20260505 15:47:40.965 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:47:40.965 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:40.965 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:40.965 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:40.965 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260505 15:47:40.965 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:40.965 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:40.965 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 15:47:40.966 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:40.966 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:40.966 | INFO | Uploaded 36GiB to GPU in 2.1676s (16.6083GiB/s) | 20260505 15:47:40.966 | INFO | Downloaded 34GiB to GPU in 2.05671s (16.5312GiB/s) | 20260505 15:47:46.103 | INFO | Analyzing chunk frames = [4400640,4401152) | 20260505 15:47:46.103 | INFO | MemInfo: { | 20260505 15:47:46.103 | INFO | "gpu" : " 17 GB", | 20260505 15:47:46.103 | INFO | "huge" : "", | 20260505 15:47:46.104 | INFO | "huge_shared" : " 27 GB", | 20260505 15:47:46.104 | INFO | "malloc" : " 14 GB", | 20260505 15:47:46.104 | INFO | "pinned" : " 29 GB" | 20260505 15:47:46.104 | INFO | } | 20260505 15:47:46.104 | INFO | Perf report for the last 5.13858 seconds: | 20260505 15:47:46.104 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:47:46.104 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:46.104 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:46.104 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:46.104 | INFO | ANALYSIS : 62.7%, 3.0, 97.3ms, 0 | 20260505 15:47:46.104 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:46.104 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:46.104 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 15:47:46.104 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:46.104 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:46.104 | INFO | Uploaded 36GiB to GPU in 2.17536s (16.549GiB/s) | 20260505 15:47:46.104 | INFO | Downloaded 34GiB to GPU in 2.14583s (15.8447GiB/s) | 20260505 15:47:51.237 | INFO | Analyzing chunk frames = [4401152,4401664) | 20260505 15:47:51.237 | INFO | MemInfo: { | 20260505 15:47:51.237 | INFO | "gpu" : " 17 GB", | 20260505 15:47:51.237 | INFO | "huge" : "", | 20260505 15:47:51.237 | INFO | "huge_shared" : " 27 GB", | 20260505 15:47:51.237 | INFO | "malloc" : " 14 GB", | 20260505 15:47:51.237 | INFO | "pinned" : " 29 GB" | 20260505 15:47:51.237 | INFO | } | 20260505 15:47:51.237 | INFO | Perf report for the last 5.13335 seconds: | 20260505 15:47:51.237 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:47:51.237 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:51.237 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:51.237 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:51.237 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 15:47:51.237 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:51.237 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:51.237 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:47:51.237 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:51.237 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:51.238 | INFO | Uploaded 36GiB to GPU in 2.17956s (16.5171GiB/s) | 20260505 15:47:51.238 | INFO | Downloaded 34GiB to GPU in 2.13292s (15.9406GiB/s) | 20260505 15:47:56.373 | INFO | Analyzing chunk frames = [4401664,4402176) | 20260505 15:47:56.373 | INFO | MemInfo: { | 20260505 15:47:56.373 | INFO | "gpu" : " 17 GB", | 20260505 15:47:56.373 | INFO | "huge" : "", | 20260505 15:47:56.373 | INFO | "huge_shared" : " 27 GB", | 20260505 15:47:56.373 | INFO | "malloc" : " 14 GB", | 20260505 15:47:56.373 | INFO | "pinned" : " 29 GB" | 20260505 15:47:56.373 | INFO | } | 20260505 15:47:56.373 | INFO | Perf report for the last 5.13588 seconds: | 20260505 15:47:56.373 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:47:56.373 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:56.373 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:56.373 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:56.373 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 15:47:56.373 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:56.373 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:56.373 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 15:47:56.373 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:56.373 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:47:56.373 | INFO | Uploaded 36GiB to GPU in 2.19525s (16.3991GiB/s) | 20260505 15:47:56.374 | INFO | Downloaded 34GiB to GPU in 2.11306s (16.0904GiB/s) | 20260505 15:48:01.523 | INFO | Analyzing chunk frames = [4402176,4402688) | 20260505 15:48:01.523 | INFO | MemInfo: { | 20260505 15:48:01.523 | INFO | "gpu" : " 17 GB", | 20260505 15:48:01.523 | INFO | "huge" : "", | 20260505 15:48:01.523 | INFO | "huge_shared" : " 27 GB", | 20260505 15:48:01.523 | INFO | "malloc" : " 14 GB", | 20260505 15:48:01.523 | INFO | "pinned" : " 29 GB" | 20260505 15:48:01.523 | INFO | } | 20260505 15:48:01.523 | INFO | Perf report for the last 5.14996 seconds: | 20260505 15:48:01.523 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:48:01.523 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:01.523 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:01.523 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:01.523 | INFO | ANALYSIS : 63.1%, 3.0, 98.7ms, 0 | 20260505 15:48:01.523 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:01.523 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:01.523 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 15:48:01.523 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:01.523 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:01.523 | INFO | Uploaded 36GiB to GPU in 2.12624s (16.9313GiB/s) | 20260505 15:48:01.524 | INFO | Downloaded 34GiB to GPU in 2.16398s (15.7118GiB/s) | 20260505 15:48:06.663 | INFO | Analyzing chunk frames = [4402688,4403200) | 20260505 15:48:06.663 | INFO | MemInfo: { | 20260505 15:48:06.663 | INFO | "gpu" : " 17 GB", | 20260505 15:48:06.663 | INFO | "huge" : "", | 20260505 15:48:06.663 | INFO | "huge_shared" : " 27 GB", | 20260505 15:48:06.663 | INFO | "malloc" : " 14 GB", | 20260505 15:48:06.663 | INFO | "pinned" : " 29 GB" | 20260505 15:48:06.663 | INFO | } | 20260505 15:48:06.663 | INFO | Perf report for the last 5.1398 seconds: | 20260505 15:48:06.663 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:48:06.663 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:06.663 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:06.663 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:06.663 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 15:48:06.663 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:06.663 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:06.663 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:48:06.663 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:06.663 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:06.663 | INFO | Uploaded 36GiB to GPU in 2.1539s (16.7139GiB/s) | 20260505 15:48:06.663 | INFO | Downloaded 34GiB to GPU in 2.0355s (16.7035GiB/s) | 20260505 15:48:06.797 | INFO | Starting a round of Baz IO | 20260505 15:48:10.000 | INFO | Populating RTMetrics report | 20260505 15:48:10.001 | INFO | RTMetrics report populated | 20260505 15:48:10.001 | INFO | Opening tmp RTMetrics json file | 20260505 15:48:10.008 | INFO | Serializing RTMetrics to json | 20260505 15:48:10.009 | INFO | Serialization done, swapping tmp file now... | 20260505 15:48:10.009 | INFO | RTMetrics done | 20260505 15:48:10.096 | INFO | Finished a round of Baz IO: Wrote 2.198 GiB in 3.299 seconds | 20260505 15:48:10.096 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.39% / 0.00% / 77.61% | 20260505 15:48:10.096 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.416688% / 7.94729e-06% / 7.15256e-05% | 20260505 15:48:10.096 | INFO | Baz IO overall: | 20260505 15:48:10.096 | INFO | BytesWritten : 2617437436385 | 20260505 15:48:10.096 | INFO | NumEvents : 1243078516472 | 20260505 15:48:11.800 | INFO | Analyzing chunk frames = [4403200,4403712) | 20260505 15:48:11.800 | INFO | MemInfo: { | 20260505 15:48:11.800 | INFO | "gpu" : " 17 GB", | 20260505 15:48:11.800 | INFO | "huge" : "", | 20260505 15:48:11.800 | INFO | "huge_shared" : " 27 GB", | 20260505 15:48:11.800 | INFO | "malloc" : " 14 GB", | 20260505 15:48:11.800 | INFO | "pinned" : " 29 GB" | 20260505 15:48:11.800 | INFO | } | 20260505 15:48:11.800 | INFO | Perf report for the last 5.13671 seconds: | 20260505 15:48:11.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:48:11.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:11.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:11.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:11.800 | INFO | ANALYSIS : 63.9%, 3.0, 99.4ms, 0 | 20260505 15:48:11.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:11.800 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260505 15:48:11.800 | INFO | PRE_HQ : 60.7%, 1.8, 56.5ms, 0 | 20260505 15:48:11.800 | INFO | BAZWRITER : 63.6%, 3.9, 127.5ms, 0 | 20260505 15:48:11.800 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 15:48:11.800 | INFO | Uploaded 36GiB to GPU in 2.17105s (16.5819GiB/s) | 20260505 15:48:11.800 | INFO | Downloaded 38GiB to GPU in 2.26241s (16.7962GiB/s) | 20260505 15:48:16.933 | INFO | Analyzing chunk frames = [4403712,4404224) | 20260505 15:48:16.933 | INFO | MemInfo: { | 20260505 15:48:16.933 | INFO | "gpu" : " 17 GB", | 20260505 15:48:16.933 | INFO | "huge" : "", | 20260505 15:48:16.933 | INFO | "huge_shared" : " 27 GB", | 20260505 15:48:16.933 | INFO | "malloc" : " 14 GB", | 20260505 15:48:16.933 | INFO | "pinned" : " 29 GB" | 20260505 15:48:16.933 | INFO | } | 20260505 15:48:16.933 | INFO | Perf report for the last 5.1335 seconds: | 20260505 15:48:16.933 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:48:16.933 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:16.933 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:16.933 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:16.933 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 15:48:16.933 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:16.933 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:16.933 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:48:16.933 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:16.933 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:16.934 | INFO | Uploaded 36GiB to GPU in 2.12693s (16.9258GiB/s) | 20260505 15:48:16.934 | INFO | Downloaded 34GiB to GPU in 2.03444s (16.7122GiB/s) | 20260505 15:48:22.070 | INFO | Analyzing chunk frames = [4404224,4404736) | 20260505 15:48:22.070 | INFO | MemInfo: { | 20260505 15:48:22.070 | INFO | "gpu" : " 17 GB", | 20260505 15:48:22.070 | INFO | "huge" : "", | 20260505 15:48:22.070 | INFO | "huge_shared" : " 27 GB", | 20260505 15:48:22.071 | INFO | "malloc" : " 14 GB", | 20260505 15:48:22.071 | INFO | "pinned" : " 29 GB" | 20260505 15:48:22.071 | INFO | } | 20260505 15:48:22.071 | INFO | Perf report for the last 5.13775 seconds: | 20260505 15:48:22.071 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:48:22.071 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:22.071 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:22.071 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:22.071 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260505 15:48:22.071 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:22.071 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:22.071 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:48:22.071 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:22.071 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:22.071 | INFO | Uploaded 36GiB to GPU in 2.24043s (16.0684GiB/s) | 20260505 15:48:22.071 | INFO | Downloaded 34GiB to GPU in 2.12283s (16.0164GiB/s) | 20260505 15:48:27.218 | INFO | Analyzing chunk frames = [4404736,4405248) | 20260505 15:48:27.218 | INFO | MemInfo: { | 20260505 15:48:27.218 | INFO | "gpu" : " 17 GB", | 20260505 15:48:27.218 | INFO | "huge" : "", | 20260505 15:48:27.218 | INFO | "huge_shared" : " 27 GB", | 20260505 15:48:27.218 | INFO | "malloc" : " 14 GB", | 20260505 15:48:27.218 | INFO | "pinned" : " 29 GB" | 20260505 15:48:27.218 | INFO | } | 20260505 15:48:27.218 | INFO | Perf report for the last 5.14734 seconds: | 20260505 15:48:27.218 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:48:27.218 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:27.218 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:27.218 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:27.218 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260505 15:48:27.218 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:27.218 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:27.218 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 15:48:27.218 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:27.219 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:27.219 | INFO | Uploaded 36GiB to GPU in 2.1773s (16.5342GiB/s) | 20260505 15:48:27.219 | INFO | Downloaded 34GiB to GPU in 2.12967s (15.9649GiB/s) | 20260505 15:48:32.369 | INFO | Analyzing chunk frames = [4405248,4405760) | 20260505 15:48:32.369 | INFO | MemInfo: { | 20260505 15:48:32.369 | INFO | "gpu" : " 17 GB", | 20260505 15:48:32.369 | INFO | "huge" : "", | 20260505 15:48:32.369 | INFO | "huge_shared" : " 27 GB", | 20260505 15:48:32.369 | INFO | "malloc" : " 14 GB", | 20260505 15:48:32.369 | INFO | "pinned" : " 29 GB" | 20260505 15:48:32.369 | INFO | } | 20260505 15:48:32.369 | INFO | Perf report for the last 5.15061 seconds: | 20260505 15:48:32.369 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:48:32.369 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:32.369 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:32.369 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:32.369 | INFO | ANALYSIS : 62.1%, 3.0, 97.2ms, 0 | 20260505 15:48:32.369 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:32.369 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:32.369 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 15:48:32.369 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:32.369 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:32.369 | INFO | Uploaded 36GiB to GPU in 2.1485s (16.7559GiB/s) | 20260505 15:48:32.369 | INFO | Downloaded 34GiB to GPU in 2.05936s (16.51GiB/s) | 20260505 15:48:37.498 | INFO | Analyzing chunk frames = [4405760,4406272) | 20260505 15:48:37.498 | INFO | MemInfo: { | 20260505 15:48:37.498 | INFO | "gpu" : " 17 GB", | 20260505 15:48:37.498 | INFO | "huge" : "", | 20260505 15:48:37.498 | INFO | "huge_shared" : " 27 GB", | 20260505 15:48:37.498 | INFO | "malloc" : " 14 GB", | 20260505 15:48:37.498 | INFO | "pinned" : " 29 GB" | 20260505 15:48:37.498 | INFO | } | 20260505 15:48:37.498 | INFO | Perf report for the last 5.12886 seconds: | 20260505 15:48:37.498 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:48:37.498 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:37.498 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:37.498 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:37.498 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260505 15:48:37.498 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:37.498 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:37.498 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 15:48:37.498 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:37.498 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:37.498 | INFO | Uploaded 36GiB to GPU in 2.1381s (16.8374GiB/s) | 20260505 15:48:37.498 | INFO | Downloaded 34GiB to GPU in 2.04187s (16.6514GiB/s) | 20260505 15:48:42.627 | INFO | Analyzing chunk frames = [4406272,4406784) | 20260505 15:48:42.627 | INFO | MemInfo: { | 20260505 15:48:42.627 | INFO | "gpu" : " 17 GB", | 20260505 15:48:42.627 | INFO | "huge" : "", | 20260505 15:48:42.627 | INFO | "huge_shared" : " 27 GB", | 20260505 15:48:42.627 | INFO | "malloc" : " 14 GB", | 20260505 15:48:42.627 | INFO | "pinned" : " 29 GB" | 20260505 15:48:42.627 | INFO | } | 20260505 15:48:42.627 | INFO | Perf report for the last 5.12913 seconds: | 20260505 15:48:42.627 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:48:42.627 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:42.627 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:42.627 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:42.627 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 15:48:42.627 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:42.627 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:42.627 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 15:48:42.627 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:42.627 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:42.627 | INFO | Uploaded 36GiB to GPU in 2.1598s (16.6682GiB/s) | 20260505 15:48:42.627 | INFO | Downloaded 34GiB to GPU in 2.15022s (15.8123GiB/s) | 20260505 15:48:47.777 | INFO | Analyzing chunk frames = [4406784,4407296) | 20260505 15:48:47.777 | INFO | MemInfo: { | 20260505 15:48:47.777 | INFO | "gpu" : " 17 GB", | 20260505 15:48:47.777 | INFO | "huge" : "", | 20260505 15:48:47.777 | INFO | "huge_shared" : " 27 GB", | 20260505 15:48:47.777 | INFO | "malloc" : " 14 GB", | 20260505 15:48:47.777 | INFO | "pinned" : " 29 GB" | 20260505 15:48:47.777 | INFO | } | 20260505 15:48:47.777 | INFO | Perf report for the last 5.14984 seconds: | 20260505 15:48:47.777 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:48:47.777 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:47.777 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:47.777 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:47.777 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 15:48:47.777 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:47.777 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:47.777 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:48:47.777 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:47.777 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:47.777 | INFO | Uploaded 36GiB to GPU in 2.20797s (16.3046GiB/s) | 20260505 15:48:47.777 | INFO | Downloaded 34GiB to GPU in 2.09648s (16.2176GiB/s) | 20260505 15:48:47.777 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 15:48:47.777 | INFO | Total mem usage: 15.3 GiB | 20260505 15:48:47.777 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 15:48:47.777 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.72%) | 20260505 15:48:47.777 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 15:48:47.777 | INFO | High Water Breakdown: | 20260505 15:48:47.777 | INFO | 61% : 8.68 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 15:48:47.777 | INFO | 38.9% : 5.53 GiB ( 83.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 15:48:47.777 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 15:48:47.777 | INFO | ------------------------------------------ | 20260505 15:48:47.777 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:48:47.777 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 15:48:47.777 | INFO | Total mem usage: 26.5 GiB | 20260505 15:48:47.777 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 15:48:47.778 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 15:48:47.778 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 15:48:47.778 | INFO | High Water Breakdown: | 20260505 15:48:47.778 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:48:47.778 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:48:47.778 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:48:47.778 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:48:47.778 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:48:47.778 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:48:47.778 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 15:48:47.778 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 15:48:47.778 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:48:47.778 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 15:48:47.778 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 15:48:47.778 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 15:48:47.778 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 15:48:47.778 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 15:48:47.778 | INFO | ------------------------------------------ | 20260505 15:48:47.778 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:48:47.778 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 15:48:47.778 | INFO | Total mem usage: 25.6 GiB | 20260505 15:48:47.778 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 15:48:47.778 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 15:48:47.778 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 15:48:47.778 | INFO | High Water Breakdown: | 20260505 15:48:47.778 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 15:48:47.778 | INFO | ------------------------------------------ | 20260505 15:48:47.778 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:48:47.778 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 15:48:47.778 | INFO | Total mem usage: 15.7 GiB | 20260505 15:48:47.778 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 15:48:47.778 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 15:48:47.778 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 15:48:47.778 | INFO | High Water Breakdown: | 20260505 15:48:47.778 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 15:48:47.778 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 15:48:47.778 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 15:48:47.778 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 15:48:47.778 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 15:48:47.778 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 15:48:47.778 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 15:48:47.778 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 15:48:47.778 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 15:48:47.778 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:48:47.778 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 15:48:47.778 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:48:47.778 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 15:48:47.778 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 15:48:47.778 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 15:48:47.778 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 15:48:47.778 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:48:47.778 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 15:48:47.778 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:48:47.778 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:48:47.778 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:48:47.778 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 15:48:47.778 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 15:48:47.778 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:48:47.778 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 15:48:47.778 | INFO | ------------------------------------------ | 20260505 15:48:47.778 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:48:47.778 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 15:48:47.778 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:48:47.778 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:48:47.778 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.54e+01, 1.36e+01 / 1.49e+00 | 20260505 15:48:47.778 | INFO | PerfReport: 0.29%, AggregateStats -- 9.25e-02 / 1.02e-01, 1.05e-02 / 6.14e-03 | 20260505 15:48:47.778 | INFO | PerfReport: 11.77%, Baseline -- 3.73e+00 / 4.49e+00, 5.72e-01 / 2.21e-01 | 20260505 15:48:47.778 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.50e+00, 1.93e-01 / 9.94e-02 | 20260505 15:48:47.778 | INFO | PerfReport: 15.92%, DME -- 4.04e+01 / 4.50e+01, 4.40e+00 / 1.64e+00 | 20260505 15:48:47.778 | INFO | PerfReport: 56.70%, FrameLabeling -- 1.80e+01 / 1.96e+01, 1.50e+00 / 1.47e+00 | 20260505 15:48:47.778 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.34e+00 / 2.88e+00, 4.48e-01 / 1.88e-01 | 20260505 15:48:47.778 | INFO | PerfReport: 3.45%, Metrics -- 1.09e+00 / 1.44e+00, 3.04e-01 / 1.82e-01 | 20260505 15:48:47.778 | INFO | PerfReport: 1.20%, PulseSerializing -- 3.82e-01 / 4.37e-01, 6.83e-02 / 3.35e-02 | 20260505 15:48:47.778 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 15:48:47.779 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:48:47.779 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:48:47.779 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260505 15:48:47.779 | INFO | PerfReport: 22.55%, Upload -- 2.26e+01 / 0.00e+00, 7.89e+00 / 0.00e+00 | 20260505 15:48:47.779 | INFO | PerfReport: 21.97%, Download -- 2.20e+01 / 0.00e+00, 7.96e+00 / 0.00e+00 | 20260505 15:48:47.779 | INFO | PerfReport: 31.63%, Compute -- 3.17e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 15:48:47.779 | INFO | PerfReport: 2.25%, LockUpload -- 2.26e+00 / 0.00e+00, 7.82e+00 / 0.00e+00 | 20260505 15:48:47.779 | INFO | PerfReport: 1.40%, LockDownload -- 1.41e+00 / 0.00e+00, 2.94e+00 / 0.00e+00 | 20260505 15:48:47.779 | INFO | PerfReport: 20.19%, LockCompute -- 2.03e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 15:48:47.779 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.78e-02 / 0.00e+00 | 20260505 15:48:47.911 | INFO | Starting a round of Baz IO | 20260505 15:48:51.117 | INFO | Populating RTMetrics report | 20260505 15:48:51.118 | INFO | RTMetrics report populated | 20260505 15:48:51.118 | INFO | Opening tmp RTMetrics json file | 20260505 15:48:51.134 | INFO | Serializing RTMetrics to json | 20260505 15:48:51.193 | INFO | Serialization done, swapping tmp file now... | 20260505 15:48:51.215 | INFO | RTMetrics done | 20260505 15:48:51.231 | INFO | Finished a round of Baz IO: Wrote 2.198 GiB in 3.32 seconds | 20260505 15:48:51.231 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.38% / 0.00% / 77.61% | 20260505 15:48:51.231 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.415917% / 3.97364e-06% / 6.35783e-05% | 20260505 15:48:51.231 | INFO | Baz IO overall: | 20260505 15:48:51.231 | INFO | BytesWritten : 2619797804119 | 20260505 15:48:51.231 | INFO | NumEvents : 1244187130315 | 20260505 15:48:52.913 | INFO | Analyzing chunk frames = [4407296,4407808) | 20260505 15:48:52.913 | INFO | MemInfo: { | 20260505 15:48:52.913 | INFO | "gpu" : " 17 GB", | 20260505 15:48:52.913 | INFO | "huge" : "", | 20260505 15:48:52.913 | INFO | "huge_shared" : " 27 GB", | 20260505 15:48:52.913 | INFO | "malloc" : " 14 GB", | 20260505 15:48:52.913 | INFO | "pinned" : " 29 GB" | 20260505 15:48:52.913 | INFO | } | 20260505 15:48:52.913 | INFO | Perf report for the last 5.136 seconds: | 20260505 15:48:52.913 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:48:52.913 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:52.913 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:52.913 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:52.913 | INFO | ANALYSIS : 63.9%, 3.0, 99.4ms, 0 | 20260505 15:48:52.913 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:52.913 | INFO | RT_METRICS : 46.8%, 1.0, 24.5ms, 0 | 20260505 15:48:52.913 | INFO | PRE_HQ : 60.9%, 1.8, 57.1ms, 0 | 20260505 15:48:52.913 | INFO | BAZWRITER : 64.0%, 3.9, 128.7ms, 0 | 20260505 15:48:52.913 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 15:48:52.913 | INFO | Uploaded 36GiB to GPU in 2.18855s (16.4492GiB/s) | 20260505 15:48:52.913 | INFO | Downloaded 38GiB to GPU in 2.26393s (16.7849GiB/s) | 20260505 15:48:58.048 | INFO | Analyzing chunk frames = [4407808,4408320) | 20260505 15:48:58.048 | INFO | MemInfo: { | 20260505 15:48:58.048 | INFO | "gpu" : " 17 GB", | 20260505 15:48:58.048 | INFO | "huge" : "", | 20260505 15:48:58.048 | INFO | "huge_shared" : " 27 GB", | 20260505 15:48:58.048 | INFO | "malloc" : " 14 GB", | 20260505 15:48:58.048 | INFO | "pinned" : " 29 GB" | 20260505 15:48:58.048 | INFO | } | 20260505 15:48:58.048 | INFO | Perf report for the last 5.13541 seconds: | 20260505 15:48:58.048 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:48:58.048 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:58.048 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:58.048 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:58.048 | INFO | ANALYSIS : 63.5%, 3.0, 99.0ms, 0 | 20260505 15:48:58.048 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:58.048 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:58.048 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 15:48:58.048 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:58.048 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:48:58.049 | INFO | Uploaded 36GiB to GPU in 2.20645s (16.3158GiB/s) | 20260505 15:48:58.049 | INFO | Downloaded 34GiB to GPU in 2.06628s (16.4547GiB/s) | 20260505 15:49:03.185 | INFO | Analyzing chunk frames = [4408320,4408832) | 20260505 15:49:03.185 | INFO | MemInfo: { | 20260505 15:49:03.185 | INFO | "gpu" : " 17 GB", | 20260505 15:49:03.185 | INFO | "huge" : "", | 20260505 15:49:03.185 | INFO | "huge_shared" : " 27 GB", | 20260505 15:49:03.185 | INFO | "malloc" : " 14 GB", | 20260505 15:49:03.185 | INFO | "pinned" : " 29 GB" | 20260505 15:49:03.185 | INFO | } | 20260505 15:49:03.185 | INFO | Perf report for the last 5.13649 seconds: | 20260505 15:49:03.185 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:49:03.185 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:03.185 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:03.185 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:03.185 | INFO | ANALYSIS : 63.0%, 3.0, 98.0ms, 0 | 20260505 15:49:03.185 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:03.185 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:03.185 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:49:03.185 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:03.185 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:03.185 | INFO | Uploaded 36GiB to GPU in 2.18571s (16.4706GiB/s) | 20260505 15:49:03.185 | INFO | Downloaded 34GiB to GPU in 2.05225s (16.5671GiB/s) | 20260505 15:49:08.326 | INFO | Analyzing chunk frames = [4408832,4409344) | 20260505 15:49:08.326 | INFO | MemInfo: { | 20260505 15:49:08.326 | INFO | "gpu" : " 17 GB", | 20260505 15:49:08.327 | INFO | "huge" : "", | 20260505 15:49:08.327 | INFO | "huge_shared" : " 27 GB", | 20260505 15:49:08.327 | INFO | "malloc" : " 14 GB", | 20260505 15:49:08.327 | INFO | "pinned" : " 29 GB" | 20260505 15:49:08.327 | INFO | } | 20260505 15:49:08.327 | INFO | Perf report for the last 5.14189 seconds: | 20260505 15:49:08.327 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:49:08.327 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:08.327 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:08.327 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:08.327 | INFO | ANALYSIS : 62.6%, 3.0, 97.3ms, 0 | 20260505 15:49:08.327 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:08.327 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:08.327 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 15:49:08.327 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:08.327 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:08.327 | INFO | Uploaded 36GiB to GPU in 2.18326s (16.4891GiB/s) | 20260505 15:49:08.327 | INFO | Downloaded 34GiB to GPU in 2.13221s (15.9459GiB/s) | 20260505 15:49:13.475 | INFO | Analyzing chunk frames = [4409344,4409856) | 20260505 15:49:13.475 | INFO | MemInfo: { | 20260505 15:49:13.475 | INFO | "gpu" : " 17 GB", | 20260505 15:49:13.475 | INFO | "huge" : "", | 20260505 15:49:13.475 | INFO | "huge_shared" : " 27 GB", | 20260505 15:49:13.475 | INFO | "malloc" : " 14 GB", | 20260505 15:49:13.475 | INFO | "pinned" : " 29 GB" | 20260505 15:49:13.475 | INFO | } | 20260505 15:49:13.475 | INFO | Perf report for the last 5.14837 seconds: | 20260505 15:49:13.475 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:49:13.475 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:13.475 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:13.475 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:13.475 | INFO | ANALYSIS : 62.2%, 3.0, 97.3ms, 0 | 20260505 15:49:13.475 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:13.475 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:13.475 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 15:49:13.475 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:13.475 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:13.475 | INFO | Uploaded 36GiB to GPU in 2.1366s (16.8492GiB/s) | 20260505 15:49:13.475 | INFO | Downloaded 34GiB to GPU in 2.05239s (16.566GiB/s) | 20260505 15:49:18.614 | INFO | Analyzing chunk frames = [4409856,4410368) | 20260505 15:49:18.614 | INFO | MemInfo: { | 20260505 15:49:18.614 | INFO | "gpu" : " 17 GB", | 20260505 15:49:18.614 | INFO | "huge" : "", | 20260505 15:49:18.614 | INFO | "huge_shared" : " 27 GB", | 20260505 15:49:18.614 | INFO | "malloc" : " 14 GB", | 20260505 15:49:18.614 | INFO | "pinned" : " 29 GB" | 20260505 15:49:18.614 | INFO | } | 20260505 15:49:18.614 | INFO | Perf report for the last 5.13888 seconds: | 20260505 15:49:18.614 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:49:18.614 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:18.614 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:18.614 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:18.614 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260505 15:49:18.614 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:18.614 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:18.614 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 15:49:18.614 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:18.614 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:18.614 | INFO | Uploaded 36GiB to GPU in 2.13836s (16.8353GiB/s) | 20260505 15:49:18.614 | INFO | Downloaded 34GiB to GPU in 2.03584s (16.7007GiB/s) | 20260505 15:49:23.744 | INFO | Analyzing chunk frames = [4410368,4410880) | 20260505 15:49:23.744 | INFO | MemInfo: { | 20260505 15:49:23.744 | INFO | "gpu" : " 17 GB", | 20260505 15:49:23.744 | INFO | "huge" : "", | 20260505 15:49:23.744 | INFO | "huge_shared" : " 27 GB", | 20260505 15:49:23.744 | INFO | "malloc" : " 14 GB", | 20260505 15:49:23.744 | INFO | "pinned" : " 29 GB" | 20260505 15:49:23.744 | INFO | } | 20260505 15:49:23.744 | INFO | Perf report for the last 5.13037 seconds: | 20260505 15:49:23.745 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:49:23.745 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:23.745 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:23.745 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:23.745 | INFO | ANALYSIS : 63.6%, 3.0, 99.1ms, 0 | 20260505 15:49:23.745 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:23.745 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:23.745 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 15:49:23.745 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:23.745 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:23.745 | INFO | Uploaded 36GiB to GPU in 2.13763s (16.841GiB/s) | 20260505 15:49:23.745 | INFO | Downloaded 34GiB to GPU in 2.1634s (15.716GiB/s) | 20260505 15:49:28.880 | INFO | Analyzing chunk frames = [4410880,4411392) | 20260505 15:49:28.880 | INFO | MemInfo: { | 20260505 15:49:28.880 | INFO | "gpu" : " 17 GB", | 20260505 15:49:28.880 | INFO | "huge" : "", | 20260505 15:49:28.880 | INFO | "huge_shared" : " 27 GB", | 20260505 15:49:28.880 | INFO | "malloc" : " 14 GB", | 20260505 15:49:28.880 | INFO | "pinned" : " 29 GB" | 20260505 15:49:28.880 | INFO | } | 20260505 15:49:28.880 | INFO | Perf report for the last 5.1359 seconds: | 20260505 15:49:28.880 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:49:28.880 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:28.880 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:28.880 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:28.880 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 15:49:28.880 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:28.881 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:28.881 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:49:28.881 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:28.881 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:28.881 | INFO | Uploaded 36GiB to GPU in 2.20797s (16.3046GiB/s) | 20260505 15:49:28.881 | INFO | Downloaded 34GiB to GPU in 2.10841s (16.1259GiB/s) | 20260505 15:49:29.016 | INFO | Starting a round of Baz IO | 20260505 15:49:32.232 | INFO | Populating RTMetrics report | 20260505 15:49:32.233 | INFO | RTMetrics report populated | 20260505 15:49:32.233 | INFO | Opening tmp RTMetrics json file | 20260505 15:49:32.246 | INFO | Serializing RTMetrics to json | 20260505 15:49:32.250 | INFO | Serialization done, swapping tmp file now... | 20260505 15:49:32.252 | INFO | RTMetrics done | 20260505 15:49:32.338 | INFO | Finished a round of Baz IO: Wrote 2.198 GiB in 3.321 seconds | 20260505 15:49:32.338 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.38% / 0.00% / 77.62% | 20260505 15:49:32.338 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.415226% / 3.97364e-06% / 6.75519e-05% | 20260505 15:49:32.338 | INFO | Baz IO overall: | 20260505 15:49:32.338 | INFO | BytesWritten : 2622157472786 | 20260505 15:49:32.338 | INFO | NumEvents : 1245295641234 | 20260505 15:49:34.028 | INFO | Analyzing chunk frames = [4411392,4411904) | 20260505 15:49:34.028 | INFO | MemInfo: { | 20260505 15:49:34.028 | INFO | "gpu" : " 17 GB", | 20260505 15:49:34.028 | INFO | "huge" : "", | 20260505 15:49:34.028 | INFO | "huge_shared" : " 27 GB", | 20260505 15:49:34.028 | INFO | "malloc" : " 14 GB", | 20260505 15:49:34.028 | INFO | "pinned" : " 29 GB" | 20260505 15:49:34.028 | INFO | } | 20260505 15:49:34.028 | INFO | Perf report for the last 5.14776 seconds: | 20260505 15:49:34.028 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:49:34.028 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:34.028 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:34.028 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:34.028 | INFO | ANALYSIS : 64.0%, 3.0, 99.7ms, 0 | 20260505 15:49:34.028 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:34.028 | INFO | RT_METRICS : 45.1%, 1.0, 23.7ms, 0 | 20260505 15:49:34.028 | INFO | PRE_HQ : 60.7%, 1.8, 57.0ms, 0 | 20260505 15:49:34.028 | INFO | BAZWRITER : 63.7%, 3.9, 128.0ms, 0 | 20260505 15:49:34.028 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 15:49:34.028 | INFO | Uploaded 36GiB to GPU in 2.1535s (16.717GiB/s) | 20260505 15:49:34.028 | INFO | Downloaded 38GiB to GPU in 2.26411s (16.7837GiB/s) | 20260505 15:49:39.173 | INFO | Analyzing chunk frames = [4411904,4412416) | 20260505 15:49:39.173 | INFO | MemInfo: { | 20260505 15:49:39.173 | INFO | "gpu" : " 17 GB", | 20260505 15:49:39.173 | INFO | "huge" : "", | 20260505 15:49:39.173 | INFO | "huge_shared" : " 27 GB", | 20260505 15:49:39.173 | INFO | "malloc" : " 14 GB", | 20260505 15:49:39.173 | INFO | "pinned" : " 29 GB" | 20260505 15:49:39.173 | INFO | } | 20260505 15:49:39.173 | INFO | Perf report for the last 5.14494 seconds: | 20260505 15:49:39.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:49:39.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:39.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:39.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:39.173 | INFO | ANALYSIS : 62.9%, 3.0, 98.4ms, 0 | 20260505 15:49:39.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:39.173 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:39.173 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 15:49:39.173 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:39.173 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:39.173 | INFO | Uploaded 36GiB to GPU in 2.12894s (16.9098GiB/s) | 20260505 15:49:39.173 | INFO | Downloaded 34GiB to GPU in 2.03102s (16.7403GiB/s) | 20260505 15:49:44.315 | INFO | Analyzing chunk frames = [4412416,4412928) | 20260505 15:49:44.315 | INFO | MemInfo: { | 20260505 15:49:44.315 | INFO | "gpu" : " 17 GB", | 20260505 15:49:44.315 | INFO | "huge" : "", | 20260505 15:49:44.315 | INFO | "huge_shared" : " 27 GB", | 20260505 15:49:44.315 | INFO | "malloc" : " 14 GB", | 20260505 15:49:44.315 | INFO | "pinned" : " 29 GB" | 20260505 15:49:44.315 | INFO | } | 20260505 15:49:44.315 | INFO | Perf report for the last 5.14213 seconds: | 20260505 15:49:44.315 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:49:44.315 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:44.315 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:44.315 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:44.315 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 15:49:44.315 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:44.315 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:44.315 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 15:49:44.315 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:44.315 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:44.316 | INFO | Uploaded 36GiB to GPU in 2.19743s (16.3827GiB/s) | 20260505 15:49:44.316 | INFO | Downloaded 34GiB to GPU in 2.0422s (16.6487GiB/s) | 20260505 15:49:49.442 | INFO | Analyzing chunk frames = [4412928,4413440) | 20260505 15:49:49.442 | INFO | MemInfo: { | 20260505 15:49:49.442 | INFO | "gpu" : " 17 GB", | 20260505 15:49:49.442 | INFO | "huge" : "", | 20260505 15:49:49.442 | INFO | "huge_shared" : " 27 GB", | 20260505 15:49:49.442 | INFO | "malloc" : " 14 GB", | 20260505 15:49:49.442 | INFO | "pinned" : " 29 GB" | 20260505 15:49:49.442 | INFO | } | 20260505 15:49:49.442 | INFO | Perf report for the last 5.12729 seconds: | 20260505 15:49:49.442 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:49:49.442 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:49.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:49.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:49.442 | INFO | ANALYSIS : 62.7%, 3.0, 97.2ms, 0 | 20260505 15:49:49.443 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:49.443 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:49.443 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 15:49:49.443 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:49.443 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:49.443 | INFO | Uploaded 36GiB to GPU in 2.16415s (16.6347GiB/s) | 20260505 15:49:49.443 | INFO | Downloaded 34GiB to GPU in 2.14199s (15.8731GiB/s) | 20260505 15:49:54.583 | INFO | Analyzing chunk frames = [4413440,4413952) | 20260505 15:49:54.583 | INFO | MemInfo: { | 20260505 15:49:54.583 | INFO | "gpu" : " 17 GB", | 20260505 15:49:54.583 | INFO | "huge" : "", | 20260505 15:49:54.583 | INFO | "huge_shared" : " 27 GB", | 20260505 15:49:54.583 | INFO | "malloc" : " 14 GB", | 20260505 15:49:54.583 | INFO | "pinned" : " 29 GB" | 20260505 15:49:54.583 | INFO | } | 20260505 15:49:54.583 | INFO | Perf report for the last 5.14096 seconds: | 20260505 15:49:54.583 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:49:54.583 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:54.583 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:54.583 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:54.583 | INFO | ANALYSIS : 62.5%, 3.0, 97.7ms, 0 | 20260505 15:49:54.583 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:54.584 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:54.584 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 15:49:54.584 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:54.584 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:54.584 | INFO | Uploaded 36GiB to GPU in 2.1528s (16.7224GiB/s) | 20260505 15:49:54.584 | INFO | Downloaded 34GiB to GPU in 2.12782s (15.9788GiB/s) | 20260505 15:49:59.727 | INFO | Analyzing chunk frames = [4413952,4414464) | 20260505 15:49:59.727 | INFO | MemInfo: { | 20260505 15:49:59.727 | INFO | "gpu" : " 17 GB", | 20260505 15:49:59.727 | INFO | "huge" : "", | 20260505 15:49:59.727 | INFO | "huge_shared" : " 27 GB", | 20260505 15:49:59.727 | INFO | "malloc" : " 14 GB", | 20260505 15:49:59.727 | INFO | "pinned" : " 29 GB" | 20260505 15:49:59.727 | INFO | } | 20260505 15:49:59.727 | INFO | Perf report for the last 5.14401 seconds: | 20260505 15:49:59.727 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:49:59.727 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:59.727 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:59.727 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:59.727 | INFO | ANALYSIS : 62.5%, 3.0, 97.7ms, 0 | 20260505 15:49:59.727 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:59.728 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:59.728 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260505 15:49:59.728 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:59.728 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:49:59.728 | INFO | Uploaded 36GiB to GPU in 2.14596s (16.7757GiB/s) | 20260505 15:49:59.728 | INFO | Downloaded 34GiB to GPU in 2.0362s (16.6978GiB/s) | 20260505 15:50:04.868 | INFO | Analyzing chunk frames = [4414464,4414976) | 20260505 15:50:04.868 | INFO | MemInfo: { | 20260505 15:50:04.868 | INFO | "gpu" : " 17 GB", | 20260505 15:50:04.868 | INFO | "huge" : "", | 20260505 15:50:04.868 | INFO | "huge_shared" : " 27 GB", | 20260505 15:50:04.868 | INFO | "malloc" : " 14 GB", | 20260505 15:50:04.868 | INFO | "pinned" : " 29 GB" | 20260505 15:50:04.868 | INFO | } | 20260505 15:50:04.868 | INFO | Perf report for the last 5.14046 seconds: | 20260505 15:50:04.868 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:50:04.868 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:04.868 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:04.868 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:04.868 | INFO | ANALYSIS : 63.7%, 3.0, 99.6ms, 0 | 20260505 15:50:04.868 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:04.868 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:04.868 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 15:50:04.868 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:04.868 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:04.868 | INFO | Uploaded 36GiB to GPU in 2.14856s (16.7554GiB/s) | 20260505 15:50:04.868 | INFO | Downloaded 34GiB to GPU in 2.17951s (15.5999GiB/s) | 20260505 15:50:09.999 | INFO | Analyzing chunk frames = [4414976,4415488) | 20260505 15:50:09.999 | INFO | MemInfo: { | 20260505 15:50:09.999 | INFO | "gpu" : " 17 GB", | 20260505 15:50:09.999 | INFO | "huge" : "", | 20260505 15:50:09.999 | INFO | "huge_shared" : " 27 GB", | 20260505 15:50:09.999 | INFO | "malloc" : " 14 GB", | 20260505 15:50:09.999 | INFO | "pinned" : " 29 GB" | 20260505 15:50:09.999 | INFO | } | 20260505 15:50:09.999 | INFO | Perf report for the last 5.13101 seconds: | 20260505 15:50:09.999 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:50:09.999 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:09.999 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:09.999 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:09.999 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 15:50:09.999 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:09.999 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:09.999 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 15:50:09.999 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:09.999 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:09.999 | INFO | Uploaded 36GiB to GPU in 2.16499s (16.6282GiB/s) | 20260505 15:50:09.999 | INFO | Downloaded 34GiB to GPU in 2.0352s (16.706GiB/s) | 20260505 15:50:09.999 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 15:50:09.999 | INFO | Total mem usage: 15.3 GiB | 20260505 15:50:09.999 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 15:50:09.999 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.82%) | 20260505 15:50:09.999 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 15:50:09.999 | INFO | High Water Breakdown: | 20260505 15:50:09.999 | INFO | 61% : 8.68 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 15:50:09.999 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 15:50:09.999 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 15:50:09.999 | INFO | ------------------------------------------ | 20260505 15:50:09.999 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:50:09.999 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 15:50:09.999 | INFO | Total mem usage: 26.5 GiB | 20260505 15:50:09.999 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 15:50:09.999 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 15:50:09.999 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 15:50:09.999 | INFO | High Water Breakdown: | 20260505 15:50:09.999 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:50:09.999 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:50:10.000 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:50:10.000 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:50:10.000 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:50:10.000 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:50:10.000 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 15:50:10.000 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 15:50:10.000 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:50:10.000 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 15:50:10.000 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 15:50:10.000 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 15:50:10.000 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 15:50:10.000 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 15:50:10.000 | INFO | ------------------------------------------ | 20260505 15:50:10.000 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:50:10.000 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 15:50:10.000 | INFO | Total mem usage: 25.6 GiB | 20260505 15:50:10.000 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 15:50:10.000 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 15:50:10.000 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 15:50:10.000 | INFO | High Water Breakdown: | 20260505 15:50:10.000 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 15:50:10.000 | INFO | ------------------------------------------ | 20260505 15:50:10.000 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:50:10.000 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 15:50:10.000 | INFO | Total mem usage: 15.7 GiB | 20260505 15:50:10.000 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 15:50:10.000 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 15:50:10.000 | INFO | Unused Cache: 871 MiB (Waste: 5.43%) | 20260505 15:50:10.000 | INFO | High Water Breakdown: | 20260505 15:50:10.000 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 15:50:10.000 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 15:50:10.000 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 15:50:10.000 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 15:50:10.000 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 15:50:10.000 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 15:50:10.000 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 15:50:10.000 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 15:50:10.000 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 15:50:10.000 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:50:10.000 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 15:50:10.000 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:50:10.000 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 15:50:10.000 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 15:50:10.000 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 15:50:10.000 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 15:50:10.000 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:50:10.000 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 15:50:10.000 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:50:10.000 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:50:10.000 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:50:10.000 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 15:50:10.000 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 15:50:10.000 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:50:10.000 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 15:50:10.000 | INFO | ------------------------------------------ | 20260505 15:50:10.000 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:50:10.000 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 15:50:10.000 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:50:10.000 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:50:10.000 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.58e+01, 1.37e+01 / 2.10e+00 | 20260505 15:50:10.000 | INFO | PerfReport: 0.29%, AggregateStats -- 9.27e-02 / 1.02e-01, 1.05e-02 / 5.73e-03 | 20260505 15:50:10.000 | INFO | PerfReport: 11.76%, Baseline -- 3.74e+00 / 4.48e+00, 5.86e-01 / 1.82e-01 | 20260505 15:50:10.000 | INFO | PerfReport: 3.29%, Histogram -- 1.05e+00 / 1.50e+00, 1.98e-01 / 9.04e-02 | 20260505 15:50:10.000 | INFO | PerfReport: 15.96%, DME -- 4.06e+01 / 4.53e+01, 4.44e+00 / 2.48e+00 | 20260505 15:50:10.000 | INFO | PerfReport: 56.74%, FrameLabeling -- 1.80e+01 / 1.96e+01, 1.52e+00 / 1.49e+00 | 20260505 15:50:10.000 | INFO | PerfReport: 7.32%, PulseAccumulating -- 2.33e+00 / 2.91e+00, 4.45e-01 / 2.04e-01 | 20260505 15:50:10.000 | INFO | PerfReport: 3.43%, Metrics -- 1.09e+00 / 1.44e+00, 3.09e-01 / 1.54e-01 | 20260505 15:50:10.000 | INFO | PerfReport: 1.20%, PulseSerializing -- 3.81e-01 / 4.38e-01, 6.83e-02 / 3.20e-02 | 20260505 15:50:10.000 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 15:50:10.000 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:50:10.000 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:50:10.000 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.58e+01 / 0.00e+00 | 20260505 15:50:10.000 | INFO | PerfReport: 22.47%, Upload -- 2.26e+01 / 0.00e+00, 7.89e+00 / 0.00e+00 | 20260505 15:50:10.000 | INFO | PerfReport: 21.88%, Download -- 2.20e+01 / 0.00e+00, 7.95e+00 / 0.00e+00 | 20260505 15:50:10.000 | INFO | PerfReport: 31.66%, Compute -- 3.18e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 15:50:10.000 | INFO | PerfReport: 2.20%, LockUpload -- 2.21e+00 / 0.00e+00, 7.86e+00 / 0.00e+00 | 20260505 15:50:10.000 | INFO | PerfReport: 1.44%, LockDownload -- 1.45e+00 / 0.00e+00, 3.04e+00 / 0.00e+00 | 20260505 15:50:10.000 | INFO | PerfReport: 20.33%, LockCompute -- 2.04e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 15:50:10.000 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.22e-02 / 0.00e+00 | 20260505 15:50:10.133 | INFO | Starting a round of Baz IO | 20260505 15:50:13.350 | INFO | Populating RTMetrics report | 20260505 15:50:13.351 | INFO | RTMetrics report populated | 20260505 15:50:13.351 | INFO | Opening tmp RTMetrics json file | 20260505 15:50:13.362 | INFO | Serializing RTMetrics to json | 20260505 15:50:13.363 | INFO | Serialization done, swapping tmp file now... | 20260505 15:50:13.364 | INFO | RTMetrics done | 20260505 15:50:13.449 | INFO | Finished a round of Baz IO: Wrote 2.2 GiB in 3.316 seconds | 20260505 15:50:13.449 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.38% / 0.00% / 77.62% | 20260505 15:50:13.449 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.415822% / 0% / 6.75519e-05% | 20260505 15:50:13.449 | INFO | Baz IO overall: | 20260505 15:50:13.449 | INFO | BytesWritten : 2624519598632 | 20260505 15:50:13.449 | INFO | NumEvents : 1246406122376 | 20260505 15:50:15.136 | INFO | Analyzing chunk frames = [4415488,4416000) | 20260505 15:50:15.136 | INFO | MemInfo: { | 20260505 15:50:15.136 | INFO | "gpu" : " 17 GB", | 20260505 15:50:15.136 | INFO | "huge" : "", | 20260505 15:50:15.136 | INFO | "huge_shared" : " 27 GB", | 20260505 15:50:15.136 | INFO | "malloc" : " 14 GB", | 20260505 15:50:15.136 | INFO | "pinned" : " 29 GB" | 20260505 15:50:15.136 | INFO | } | 20260505 15:50:15.136 | INFO | Perf report for the last 5.13703 seconds: | 20260505 15:50:15.136 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:50:15.136 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:15.136 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:15.136 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:15.136 | INFO | ANALYSIS : 64.1%, 3.0, 99.8ms, 0 | 20260505 15:50:15.136 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:15.136 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260505 15:50:15.136 | INFO | PRE_HQ : 60.7%, 1.8, 56.8ms, 0 | 20260505 15:50:15.136 | INFO | BAZWRITER : 63.9%, 3.9, 126.8ms, 0 | 20260505 15:50:15.136 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 15:50:15.136 | INFO | Uploaded 36GiB to GPU in 2.2008s (16.3577GiB/s) | 20260505 15:50:15.136 | INFO | Downloaded 38GiB to GPU in 2.26043s (16.811GiB/s) | 20260505 15:50:20.286 | INFO | Analyzing chunk frames = [4416000,4416512) | 20260505 15:50:20.286 | INFO | MemInfo: { | 20260505 15:50:20.286 | INFO | "gpu" : " 17 GB", | 20260505 15:50:20.286 | INFO | "huge" : "", | 20260505 15:50:20.286 | INFO | "huge_shared" : " 27 GB", | 20260505 15:50:20.286 | INFO | "malloc" : " 14 GB", | 20260505 15:50:20.286 | INFO | "pinned" : " 29 GB" | 20260505 15:50:20.286 | INFO | } | 20260505 15:50:20.286 | INFO | Perf report for the last 5.15009 seconds: | 20260505 15:50:20.286 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:50:20.286 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:20.286 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:20.286 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:20.286 | INFO | ANALYSIS : 63.3%, 3.0, 99.0ms, 0 | 20260505 15:50:20.286 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:20.286 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:20.286 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 15:50:20.286 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:20.286 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:20.286 | INFO | Uploaded 36GiB to GPU in 2.20322s (16.3397GiB/s) | 20260505 15:50:20.286 | INFO | Downloaded 34GiB to GPU in 2.06787s (16.4421GiB/s) | 20260505 15:50:25.426 | INFO | Analyzing chunk frames = [4416512,4417024) | 20260505 15:50:25.426 | INFO | MemInfo: { | 20260505 15:50:25.426 | INFO | "gpu" : " 17 GB", | 20260505 15:50:25.426 | INFO | "huge" : "", | 20260505 15:50:25.426 | INFO | "huge_shared" : " 27 GB", | 20260505 15:50:25.426 | INFO | "malloc" : " 14 GB", | 20260505 15:50:25.426 | INFO | "pinned" : " 29 GB" | 20260505 15:50:25.426 | INFO | } | 20260505 15:50:25.426 | INFO | Perf report for the last 5.14023 seconds: | 20260505 15:50:25.426 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:50:25.426 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:25.426 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:25.426 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:25.426 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 15:50:25.426 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:25.426 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:25.426 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 15:50:25.426 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:25.426 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:25.427 | INFO | Uploaded 36GiB to GPU in 2.19808s (16.378GiB/s) | 20260505 15:50:25.427 | INFO | Downloaded 34GiB to GPU in 2.03121s (16.7388GiB/s) | 20260505 15:50:30.557 | INFO | Analyzing chunk frames = [4417024,4417536) | 20260505 15:50:30.557 | INFO | MemInfo: { | 20260505 15:50:30.557 | INFO | "gpu" : " 17 GB", | 20260505 15:50:30.557 | INFO | "huge" : "", | 20260505 15:50:30.557 | INFO | "huge_shared" : " 27 GB", | 20260505 15:50:30.557 | INFO | "malloc" : " 14 GB", | 20260505 15:50:30.557 | INFO | "pinned" : " 29 GB" | 20260505 15:50:30.557 | INFO | } | 20260505 15:50:30.557 | INFO | Perf report for the last 5.13099 seconds: | 20260505 15:50:30.557 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:50:30.557 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:30.557 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:30.557 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:30.557 | INFO | ANALYSIS : 62.7%, 3.0, 97.2ms, 0 | 20260505 15:50:30.557 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:30.557 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:30.557 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 15:50:30.557 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:30.557 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:30.558 | INFO | Uploaded 36GiB to GPU in 2.18959s (16.4414GiB/s) | 20260505 15:50:30.558 | INFO | Downloaded 34GiB to GPU in 2.14223s (15.8713GiB/s) | 20260505 15:50:35.692 | INFO | Analyzing chunk frames = [4417536,4418048) | 20260505 15:50:35.692 | INFO | MemInfo: { | 20260505 15:50:35.692 | INFO | "gpu" : " 17 GB", | 20260505 15:50:35.692 | INFO | "huge" : "", | 20260505 15:50:35.692 | INFO | "huge_shared" : " 27 GB", | 20260505 15:50:35.692 | INFO | "malloc" : " 14 GB", | 20260505 15:50:35.692 | INFO | "pinned" : " 29 GB" | 20260505 15:50:35.692 | INFO | } | 20260505 15:50:35.692 | INFO | Perf report for the last 5.13467 seconds: | 20260505 15:50:35.692 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:50:35.692 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:35.692 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:35.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:35.692 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260505 15:50:35.692 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:35.692 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:35.692 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 15:50:35.692 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:35.692 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:35.692 | INFO | Uploaded 36GiB to GPU in 2.18012s (16.5128GiB/s) | 20260505 15:50:35.692 | INFO | Downloaded 34GiB to GPU in 2.13078s (15.9566GiB/s) | 20260505 15:50:40.838 | INFO | Analyzing chunk frames = [4418048,4418560) | 20260505 15:50:40.838 | INFO | MemInfo: { | 20260505 15:50:40.838 | INFO | "gpu" : " 17 GB", | 20260505 15:50:40.838 | INFO | "huge" : "", | 20260505 15:50:40.838 | INFO | "huge_shared" : " 27 GB", | 20260505 15:50:40.838 | INFO | "malloc" : " 14 GB", | 20260505 15:50:40.838 | INFO | "pinned" : " 29 GB" | 20260505 15:50:40.838 | INFO | } | 20260505 15:50:40.839 | INFO | Perf report for the last 5.1465 seconds: | 20260505 15:50:40.839 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:50:40.839 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:40.839 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:40.839 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:40.839 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 15:50:40.839 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:40.839 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:40.839 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 15:50:40.839 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:40.839 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:40.839 | INFO | Uploaded 36GiB to GPU in 2.19805s (16.3781GiB/s) | 20260505 15:50:40.839 | INFO | Downloaded 34GiB to GPU in 2.11439s (16.0803GiB/s) | 20260505 15:50:45.979 | INFO | Analyzing chunk frames = [4418560,4419072) | 20260505 15:50:45.979 | INFO | MemInfo: { | 20260505 15:50:45.979 | INFO | "gpu" : " 17 GB", | 20260505 15:50:45.979 | INFO | "huge" : "", | 20260505 15:50:45.979 | INFO | "huge_shared" : " 27 GB", | 20260505 15:50:45.979 | INFO | "malloc" : " 14 GB", | 20260505 15:50:45.979 | INFO | "pinned" : " 29 GB" | 20260505 15:50:45.979 | INFO | } | 20260505 15:50:45.979 | INFO | Perf report for the last 5.1409 seconds: | 20260505 15:50:45.979 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:50:45.979 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:45.979 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:45.979 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:45.979 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 15:50:45.979 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:45.980 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:45.980 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 15:50:45.980 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:45.980 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:45.980 | INFO | Uploaded 36GiB to GPU in 2.13187s (16.8866GiB/s) | 20260505 15:50:45.980 | INFO | Downloaded 34GiB to GPU in 2.16243s (15.7231GiB/s) | 20260505 15:50:51.124 | INFO | Analyzing chunk frames = [4419072,4419584) | 20260505 15:50:51.124 | INFO | MemInfo: { | 20260505 15:50:51.124 | INFO | "gpu" : " 17 GB", | 20260505 15:50:51.124 | INFO | "huge" : "", | 20260505 15:50:51.124 | INFO | "huge_shared" : " 27 GB", | 20260505 15:50:51.124 | INFO | "malloc" : " 14 GB", | 20260505 15:50:51.124 | INFO | "pinned" : " 29 GB" | 20260505 15:50:51.124 | INFO | } | 20260505 15:50:51.124 | INFO | Perf report for the last 5.14456 seconds: | 20260505 15:50:51.124 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:50:51.124 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:51.124 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:51.124 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:51.124 | INFO | ANALYSIS : 62.9%, 3.0, 98.4ms, 0 | 20260505 15:50:51.124 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:51.124 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:51.124 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 15:50:51.124 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:51.124 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:51.124 | INFO | Uploaded 36GiB to GPU in 2.15236s (16.7258GiB/s) | 20260505 15:50:51.125 | INFO | Downloaded 34GiB to GPU in 2.02401s (16.7984GiB/s) | 20260505 15:50:51.259 | INFO | Starting a round of Baz IO | 20260505 15:50:54.461 | INFO | Populating RTMetrics report | 20260505 15:50:54.461 | INFO | RTMetrics report populated | 20260505 15:50:54.461 | INFO | Opening tmp RTMetrics json file | 20260505 15:50:54.496 | INFO | Serializing RTMetrics to json | 20260505 15:50:54.528 | INFO | Serialization done, swapping tmp file now... | 20260505 15:50:54.543 | INFO | RTMetrics done | 20260505 15:50:54.561 | INFO | Finished a round of Baz IO: Wrote 2.198 GiB in 3.301 seconds | 20260505 15:50:54.561 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.38% / 0.00% / 77.62% | 20260505 15:50:54.561 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.413521% / 7.94729e-06% / 8.74201e-05% | 20260505 15:50:54.561 | INFO | Baz IO overall: | 20260505 15:50:54.561 | INFO | BytesWritten : 2626879360026 | 20260505 15:50:54.561 | INFO | NumEvents : 1247514850638 | 20260505 15:50:56.259 | INFO | Analyzing chunk frames = [4419584,4420096) | 20260505 15:50:56.259 | INFO | MemInfo: { | 20260505 15:50:56.259 | INFO | "gpu" : " 17 GB", | 20260505 15:50:56.259 | INFO | "huge" : "", | 20260505 15:50:56.259 | INFO | "huge_shared" : " 27 GB", | 20260505 15:50:56.259 | INFO | "malloc" : " 14 GB", | 20260505 15:50:56.259 | INFO | "pinned" : " 29 GB" | 20260505 15:50:56.259 | INFO | } | 20260505 15:50:56.260 | INFO | Perf report for the last 5.13554 seconds: | 20260505 15:50:56.260 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:50:56.260 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:56.260 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:56.260 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:56.260 | INFO | ANALYSIS : 63.9%, 3.0, 99.3ms, 0 | 20260505 15:50:56.260 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:50:56.260 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260505 15:50:56.260 | INFO | PRE_HQ : 60.7%, 1.8, 56.9ms, 0 | 20260505 15:50:56.260 | INFO | BAZWRITER : 63.5%, 3.9, 127.5ms, 0 | 20260505 15:50:56.260 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 15:50:56.260 | INFO | Uploaded 36GiB to GPU in 2.18389s (16.4843GiB/s) | 20260505 15:50:56.260 | INFO | Downloaded 38GiB to GPU in 2.2635s (16.7881GiB/s) | 20260505 15:51:01.387 | INFO | Analyzing chunk frames = [4420096,4420608) | 20260505 15:51:01.387 | INFO | MemInfo: { | 20260505 15:51:01.387 | INFO | "gpu" : " 17 GB", | 20260505 15:51:01.387 | INFO | "huge" : "", | 20260505 15:51:01.387 | INFO | "huge_shared" : " 27 GB", | 20260505 15:51:01.387 | INFO | "malloc" : " 14 GB", | 20260505 15:51:01.387 | INFO | "pinned" : " 29 GB" | 20260505 15:51:01.387 | INFO | } | 20260505 15:51:01.387 | INFO | Perf report for the last 5.12729 seconds: | 20260505 15:51:01.387 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:51:01.387 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:01.387 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:01.387 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:01.387 | INFO | ANALYSIS : 63.2%, 3.0, 98.4ms, 0 | 20260505 15:51:01.387 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:01.387 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:01.387 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 15:51:01.387 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:01.387 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:01.387 | INFO | Uploaded 36GiB to GPU in 2.13288s (16.8786GiB/s) | 20260505 15:51:01.387 | INFO | Downloaded 34GiB to GPU in 2.03986s (16.6678GiB/s) | 20260505 15:51:06.542 | INFO | Analyzing chunk frames = [4420608,4421120) | 20260505 15:51:06.542 | INFO | MemInfo: { | 20260505 15:51:06.542 | INFO | "gpu" : " 17 GB", | 20260505 15:51:06.542 | INFO | "huge" : "", | 20260505 15:51:06.542 | INFO | "huge_shared" : " 27 GB", | 20260505 15:51:06.542 | INFO | "malloc" : " 14 GB", | 20260505 15:51:06.542 | INFO | "pinned" : " 29 GB" | 20260505 15:51:06.542 | INFO | } | 20260505 15:51:06.542 | INFO | Perf report for the last 5.15526 seconds: | 20260505 15:51:06.542 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:51:06.542 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:06.542 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:06.542 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:06.542 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 15:51:06.542 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:06.542 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:06.542 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 15:51:06.542 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:06.542 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:06.542 | INFO | Uploaded 36GiB to GPU in 2.24071s (16.0664GiB/s) | 20260505 15:51:06.542 | INFO | Downloaded 34GiB to GPU in 2.09882s (16.1996GiB/s) | 20260505 15:51:11.685 | INFO | Analyzing chunk frames = [4421120,4421632) | 20260505 15:51:11.685 | INFO | MemInfo: { | 20260505 15:51:11.685 | INFO | "gpu" : " 17 GB", | 20260505 15:51:11.685 | INFO | "huge" : "", | 20260505 15:51:11.685 | INFO | "huge_shared" : " 27 GB", | 20260505 15:51:11.685 | INFO | "malloc" : " 14 GB", | 20260505 15:51:11.685 | INFO | "pinned" : " 29 GB" | 20260505 15:51:11.685 | INFO | } | 20260505 15:51:11.685 | INFO | Perf report for the last 5.14332 seconds: | 20260505 15:51:11.685 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:51:11.685 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:11.685 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:11.685 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:11.686 | INFO | ANALYSIS : 62.6%, 3.0, 97.4ms, 0 | 20260505 15:51:11.686 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:11.686 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:11.686 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:51:11.686 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:11.686 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:11.686 | INFO | Uploaded 36GiB to GPU in 2.20019s (16.3623GiB/s) | 20260505 15:51:11.686 | INFO | Downloaded 34GiB to GPU in 2.13753s (15.9062GiB/s) | 20260505 15:51:16.814 | INFO | Analyzing chunk frames = [4421632,4422144) | 20260505 15:51:16.814 | INFO | MemInfo: { | 20260505 15:51:16.814 | INFO | "gpu" : " 17 GB", | 20260505 15:51:16.814 | INFO | "huge" : "", | 20260505 15:51:16.814 | INFO | "huge_shared" : " 27 GB", | 20260505 15:51:16.814 | INFO | "malloc" : " 14 GB", | 20260505 15:51:16.814 | INFO | "pinned" : " 29 GB" | 20260505 15:51:16.815 | INFO | } | 20260505 15:51:16.815 | INFO | Perf report for the last 5.12909 seconds: | 20260505 15:51:16.815 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:51:16.815 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:16.815 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:16.815 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:16.815 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260505 15:51:16.815 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:16.815 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:16.815 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 15:51:16.815 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:16.815 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:16.815 | INFO | Uploaded 36GiB to GPU in 2.16252s (16.6472GiB/s) | 20260505 15:51:16.815 | INFO | Downloaded 34GiB to GPU in 2.05506s (16.5445GiB/s) | 20260505 15:51:21.951 | INFO | Analyzing chunk frames = [4422144,4422656) | 20260505 15:51:21.951 | INFO | MemInfo: { | 20260505 15:51:21.951 | INFO | "gpu" : " 17 GB", | 20260505 15:51:21.951 | INFO | "huge" : "", | 20260505 15:51:21.951 | INFO | "huge_shared" : " 27 GB", | 20260505 15:51:21.951 | INFO | "malloc" : " 14 GB", | 20260505 15:51:21.951 | INFO | "pinned" : " 29 GB" | 20260505 15:51:21.951 | INFO | } | 20260505 15:51:21.951 | INFO | Perf report for the last 5.1363 seconds: | 20260505 15:51:21.951 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:51:21.951 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:21.951 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:21.951 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:21.951 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 15:51:21.951 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:21.951 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:21.951 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:51:21.951 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:21.951 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:21.951 | INFO | Uploaded 36GiB to GPU in 2.17048s (16.5862GiB/s) | 20260505 15:51:21.951 | INFO | Downloaded 34GiB to GPU in 2.0392s (16.6732GiB/s) | 20260505 15:51:27.091 | INFO | Analyzing chunk frames = [4422656,4423168) | 20260505 15:51:27.091 | INFO | MemInfo: { | 20260505 15:51:27.091 | INFO | "gpu" : " 17 GB", | 20260505 15:51:27.091 | INFO | "huge" : "", | 20260505 15:51:27.091 | INFO | "huge_shared" : " 27 GB", | 20260505 15:51:27.091 | INFO | "malloc" : " 14 GB", | 20260505 15:51:27.091 | INFO | "pinned" : " 29 GB" | 20260505 15:51:27.091 | INFO | } | 20260505 15:51:27.091 | INFO | Perf report for the last 5.14024 seconds: | 20260505 15:51:27.091 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:51:27.091 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:27.091 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:27.091 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:27.091 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 15:51:27.091 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:27.091 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:27.091 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 15:51:27.091 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:27.091 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:27.091 | INFO | Uploaded 36GiB to GPU in 2.15612s (16.6967GiB/s) | 20260505 15:51:27.092 | INFO | Downloaded 34GiB to GPU in 2.14139s (15.8775GiB/s) | 20260505 15:51:32.237 | INFO | Analyzing chunk frames = [4423168,4423680) | 20260505 15:51:32.237 | INFO | MemInfo: { | 20260505 15:51:32.237 | INFO | "gpu" : " 17 GB", | 20260505 15:51:32.237 | INFO | "huge" : "", | 20260505 15:51:32.237 | INFO | "huge_shared" : " 27 GB", | 20260505 15:51:32.237 | INFO | "malloc" : " 14 GB", | 20260505 15:51:32.237 | INFO | "pinned" : " 29 GB" | 20260505 15:51:32.237 | INFO | } | 20260505 15:51:32.237 | INFO | Perf report for the last 5.14569 seconds: | 20260505 15:51:32.237 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:51:32.237 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:32.237 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:32.237 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:32.237 | INFO | ANALYSIS : 63.2%, 3.0, 98.8ms, 0 | 20260505 15:51:32.237 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:32.237 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:32.237 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 15:51:32.237 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:32.237 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:32.237 | INFO | Uploaded 36GiB to GPU in 2.16881s (16.5989GiB/s) | 20260505 15:51:32.237 | INFO | Downloaded 34GiB to GPU in 2.03406s (16.7154GiB/s) | 20260505 15:51:32.237 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 15:51:32.237 | INFO | Total mem usage: 15.3 GiB | 20260505 15:51:32.237 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 15:51:32.237 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.86%) | 20260505 15:51:32.237 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 15:51:32.237 | INFO | High Water Breakdown: | 20260505 15:51:32.237 | INFO | 61% : 8.67 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 15:51:32.237 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 15:51:32.237 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 15:51:32.237 | INFO | ------------------------------------------ | 20260505 15:51:32.237 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:51:32.237 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 15:51:32.237 | INFO | Total mem usage: 26.5 GiB | 20260505 15:51:32.237 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 15:51:32.237 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 15:51:32.238 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 15:51:32.238 | INFO | High Water Breakdown: | 20260505 15:51:32.238 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:51:32.238 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:51:32.238 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:51:32.238 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:51:32.238 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:51:32.238 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:51:32.238 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 15:51:32.238 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 15:51:32.238 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:51:32.238 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 15:51:32.238 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 15:51:32.238 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 15:51:32.238 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 15:51:32.238 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 15:51:32.238 | INFO | ------------------------------------------ | 20260505 15:51:32.238 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:51:32.238 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 15:51:32.238 | INFO | Total mem usage: 25.6 GiB | 20260505 15:51:32.238 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 15:51:32.238 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 15:51:32.238 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 15:51:32.238 | INFO | High Water Breakdown: | 20260505 15:51:32.238 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 15:51:32.238 | INFO | ------------------------------------------ | 20260505 15:51:32.238 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:51:32.238 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 15:51:32.238 | INFO | Total mem usage: 15.7 GiB | 20260505 15:51:32.238 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 15:51:32.238 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 15:51:32.238 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 15:51:32.238 | INFO | High Water Breakdown: | 20260505 15:51:32.238 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 15:51:32.238 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 15:51:32.238 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 15:51:32.238 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 15:51:32.238 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 15:51:32.238 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 15:51:32.238 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 15:51:32.238 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 15:51:32.238 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 15:51:32.238 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:51:32.238 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 15:51:32.238 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:51:32.238 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 15:51:32.238 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 15:51:32.238 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 15:51:32.238 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 15:51:32.238 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:51:32.238 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 15:51:32.238 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:51:32.238 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:51:32.238 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:51:32.238 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 15:51:32.238 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 15:51:32.238 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:51:32.238 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 15:51:32.238 | INFO | ------------------------------------------ | 20260505 15:51:32.238 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:51:32.238 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 15:51:32.238 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:51:32.238 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:51:32.238 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.55e+01, 1.36e+01 / 1.82e+00 | 20260505 15:51:32.238 | INFO | PerfReport: 0.29%, AggregateStats -- 9.27e-02 / 1.00e-01, 1.13e-02 / 4.99e-03 | 20260505 15:51:32.238 | INFO | PerfReport: 11.79%, Baseline -- 3.74e+00 / 4.48e+00, 5.77e-01 / 1.82e-01 | 20260505 15:51:32.238 | INFO | PerfReport: 3.27%, Histogram -- 1.04e+00 / 1.50e+00, 1.91e-01 / 1.11e-01 | 20260505 15:51:32.238 | INFO | PerfReport: 15.98%, DME -- 4.05e+01 / 4.53e+01, 4.40e+00 / 1.97e+00 | 20260505 15:51:32.238 | INFO | PerfReport: 56.59%, FrameLabeling -- 1.79e+01 / 1.93e+01, 1.44e+00 / 1.23e+00 | 20260505 15:51:32.238 | INFO | PerfReport: 7.42%, PulseAccumulating -- 2.35e+00 / 2.93e+00, 4.55e-01 / 1.69e-01 | 20260505 15:51:32.238 | INFO | PerfReport: 3.45%, Metrics -- 1.09e+00 / 1.45e+00, 3.04e-01 / 1.81e-01 | 20260505 15:51:32.238 | INFO | PerfReport: 1.20%, PulseSerializing -- 3.81e-01 / 4.39e-01, 6.83e-02 / 3.33e-02 | 20260505 15:51:32.238 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 15:51:32.238 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:51:32.238 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:51:32.238 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.57e+01 / 0.00e+00 | 20260505 15:51:32.238 | INFO | PerfReport: 22.62%, Upload -- 2.27e+01 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260505 15:51:32.238 | INFO | PerfReport: 21.89%, Download -- 2.20e+01 / 0.00e+00, 7.94e+00 / 0.00e+00 | 20260505 15:51:32.238 | INFO | PerfReport: 31.58%, Compute -- 3.17e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 15:51:32.238 | INFO | PerfReport: 2.35%, LockUpload -- 2.36e+00 / 0.00e+00, 8.03e+00 / 0.00e+00 | 20260505 15:51:32.238 | INFO | PerfReport: 1.37%, LockDownload -- 1.37e+00 / 0.00e+00, 2.85e+00 / 0.00e+00 | 20260505 15:51:32.238 | INFO | PerfReport: 20.18%, LockCompute -- 2.03e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 15:51:32.238 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.83e-02 / 0.00e+00 | 20260505 15:51:32.377 | INFO | Starting a round of Baz IO | 20260505 15:51:35.570 | INFO | Populating RTMetrics report | 20260505 15:51:35.571 | INFO | RTMetrics report populated | 20260505 15:51:35.571 | INFO | Opening tmp RTMetrics json file | 20260505 15:51:35.589 | INFO | Serializing RTMetrics to json | 20260505 15:51:35.624 | INFO | Serialization done, swapping tmp file now... | 20260505 15:51:35.634 | INFO | RTMetrics done | 20260505 15:51:35.670 | INFO | Finished a round of Baz IO: Wrote 2.196 GiB in 3.293 seconds | 20260505 15:51:35.670 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.37% / 0.00% / 77.62% | 20260505 15:51:35.670 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.412365% / 3.97364e-06% / 5.5631e-05% | 20260505 15:51:35.670 | INFO | Baz IO overall: | 20260505 15:51:35.670 | INFO | BytesWritten : 2629237428220 | 20260505 15:51:35.670 | INFO | NumEvents : 1248622444887 | 20260505 15:51:37.372 | INFO | Analyzing chunk frames = [4423680,4424192) | 20260505 15:51:37.372 | INFO | MemInfo: { | 20260505 15:51:37.372 | INFO | "gpu" : " 17 GB", | 20260505 15:51:37.372 | INFO | "huge" : "", | 20260505 15:51:37.372 | INFO | "huge_shared" : " 27 GB", | 20260505 15:51:37.372 | INFO | "malloc" : " 14 GB", | 20260505 15:51:37.372 | INFO | "pinned" : " 29 GB" | 20260505 15:51:37.372 | INFO | } | 20260505 15:51:37.372 | INFO | Perf report for the last 5.13544 seconds: | 20260505 15:51:37.372 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:51:37.372 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:37.372 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:37.372 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:37.372 | INFO | ANALYSIS : 63.8%, 3.0, 99.2ms, 0 | 20260505 15:51:37.372 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:37.372 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 15:51:37.372 | INFO | PRE_HQ : 60.6%, 1.8, 57.0ms, 0 | 20260505 15:51:37.372 | INFO | BAZWRITER : 63.4%, 4.0, 128.9ms, 0 | 20260505 15:51:37.372 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 15:51:37.373 | INFO | Uploaded 36GiB to GPU in 2.18445s (16.4801GiB/s) | 20260505 15:51:37.373 | INFO | Downloaded 38GiB to GPU in 2.26952s (16.7436GiB/s) | 20260505 15:51:42.507 | INFO | Analyzing chunk frames = [4424192,4424704) | 20260505 15:51:42.508 | INFO | MemInfo: { | 20260505 15:51:42.508 | INFO | "gpu" : " 17 GB", | 20260505 15:51:42.508 | INFO | "huge" : "", | 20260505 15:51:42.508 | INFO | "huge_shared" : " 27 GB", | 20260505 15:51:42.508 | INFO | "malloc" : " 14 GB", | 20260505 15:51:42.508 | INFO | "pinned" : " 29 GB" | 20260505 15:51:42.508 | INFO | } | 20260505 15:51:42.508 | INFO | Perf report for the last 5.1355 seconds: | 20260505 15:51:42.508 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:51:42.508 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:42.508 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:42.508 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:42.508 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 15:51:42.508 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:42.508 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:42.508 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 15:51:42.508 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:42.508 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:42.508 | INFO | Uploaded 36GiB to GPU in 2.2014s (16.3532GiB/s) | 20260505 15:51:42.508 | INFO | Downloaded 34GiB to GPU in 2.06467s (16.4675GiB/s) | 20260505 15:51:47.644 | INFO | Analyzing chunk frames = [4424704,4425216) | 20260505 15:51:47.644 | INFO | MemInfo: { | 20260505 15:51:47.644 | INFO | "gpu" : " 17 GB", | 20260505 15:51:47.644 | INFO | "huge" : "", | 20260505 15:51:47.644 | INFO | "huge_shared" : " 27 GB", | 20260505 15:51:47.645 | INFO | "malloc" : " 14 GB", | 20260505 15:51:47.645 | INFO | "pinned" : " 29 GB" | 20260505 15:51:47.645 | INFO | } | 20260505 15:51:47.645 | INFO | Perf report for the last 5.13679 seconds: | 20260505 15:51:47.645 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:51:47.645 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:47.645 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:47.645 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:47.645 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 15:51:47.645 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:47.645 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:47.645 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:51:47.645 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:47.645 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:47.645 | INFO | Uploaded 36GiB to GPU in 2.22819s (16.1566GiB/s) | 20260505 15:51:47.645 | INFO | Downloaded 34GiB to GPU in 2.10465s (16.1547GiB/s) | 20260505 15:51:52.795 | INFO | Analyzing chunk frames = [4425216,4425728) | 20260505 15:51:52.795 | INFO | MemInfo: { | 20260505 15:51:52.795 | INFO | "gpu" : " 17 GB", | 20260505 15:51:52.795 | INFO | "huge" : "", | 20260505 15:51:52.795 | INFO | "huge_shared" : " 27 GB", | 20260505 15:51:52.795 | INFO | "malloc" : " 14 GB", | 20260505 15:51:52.795 | INFO | "pinned" : " 29 GB" | 20260505 15:51:52.795 | INFO | } | 20260505 15:51:52.795 | INFO | Perf report for the last 5.15027 seconds: | 20260505 15:51:52.795 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:51:52.795 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:52.795 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:52.795 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:52.795 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260505 15:51:52.795 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:52.795 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:52.795 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:51:52.795 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:52.795 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:52.795 | INFO | Uploaded 36GiB to GPU in 2.19168s (16.4257GiB/s) | 20260505 15:51:52.795 | INFO | Downloaded 34GiB to GPU in 2.13121s (15.9533GiB/s) | 20260505 15:51:57.930 | INFO | Analyzing chunk frames = [4425728,4426240) | 20260505 15:51:57.931 | INFO | MemInfo: { | 20260505 15:51:57.931 | INFO | "gpu" : " 17 GB", | 20260505 15:51:57.931 | INFO | "huge" : "", | 20260505 15:51:57.931 | INFO | "huge_shared" : " 27 GB", | 20260505 15:51:57.931 | INFO | "malloc" : " 14 GB", | 20260505 15:51:57.931 | INFO | "pinned" : " 29 GB" | 20260505 15:51:57.931 | INFO | } | 20260505 15:51:57.931 | INFO | Perf report for the last 5.13588 seconds: | 20260505 15:51:57.931 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:51:57.931 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:57.931 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:57.931 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:57.931 | INFO | ANALYSIS : 62.2%, 3.0, 96.0ms, 0 | 20260505 15:51:57.931 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:57.931 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:57.931 | INFO | PRE_HQ : 27.8%, 1.0, 14.3ms, 0 | 20260505 15:51:57.931 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:57.931 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:51:57.931 | INFO | Uploaded 36GiB to GPU in 2.14742s (16.7643GiB/s) | 20260505 15:51:57.931 | INFO | Downloaded 34GiB to GPU in 2.05594s (16.5375GiB/s) | 20260505 15:52:03.070 | INFO | Analyzing chunk frames = [4426240,4426752) | 20260505 15:52:03.070 | INFO | MemInfo: { | 20260505 15:52:03.070 | INFO | "gpu" : " 17 GB", | 20260505 15:52:03.070 | INFO | "huge" : "", | 20260505 15:52:03.070 | INFO | "huge_shared" : " 27 GB", | 20260505 15:52:03.070 | INFO | "malloc" : " 14 GB", | 20260505 15:52:03.070 | INFO | "pinned" : " 29 GB" | 20260505 15:52:03.070 | INFO | } | 20260505 15:52:03.070 | INFO | Perf report for the last 5.13944 seconds: | 20260505 15:52:03.070 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:52:03.070 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:03.070 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:03.070 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:03.070 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260505 15:52:03.070 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:03.070 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:03.070 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 15:52:03.070 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:03.070 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:03.071 | INFO | Uploaded 36GiB to GPU in 2.14091s (16.8152GiB/s) | 20260505 15:52:03.071 | INFO | Downloaded 34GiB to GPU in 2.03321s (16.7223GiB/s) | 20260505 15:52:08.207 | INFO | Analyzing chunk frames = [4426752,4427264) | 20260505 15:52:08.207 | INFO | MemInfo: { | 20260505 15:52:08.207 | INFO | "gpu" : " 17 GB", | 20260505 15:52:08.207 | INFO | "huge" : "", | 20260505 15:52:08.207 | INFO | "huge_shared" : " 27 GB", | 20260505 15:52:08.207 | INFO | "malloc" : " 14 GB", | 20260505 15:52:08.207 | INFO | "pinned" : " 29 GB" | 20260505 15:52:08.207 | INFO | } | 20260505 15:52:08.207 | INFO | Perf report for the last 5.13724 seconds: | 20260505 15:52:08.208 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:52:08.208 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:08.208 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:08.208 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:08.208 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 15:52:08.208 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:08.208 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:08.208 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 15:52:08.208 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:08.208 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:08.208 | INFO | Uploaded 36GiB to GPU in 2.10101s (17.1346GiB/s) | 20260505 15:52:08.208 | INFO | Downloaded 34GiB to GPU in 2.17179s (15.6553GiB/s) | 20260505 15:52:13.343 | INFO | Analyzing chunk frames = [4427264,4427776) | 20260505 15:52:13.343 | INFO | MemInfo: { | 20260505 15:52:13.343 | INFO | "gpu" : " 17 GB", | 20260505 15:52:13.343 | INFO | "huge" : "", | 20260505 15:52:13.343 | INFO | "huge_shared" : " 27 GB", | 20260505 15:52:13.343 | INFO | "malloc" : " 14 GB", | 20260505 15:52:13.343 | INFO | "pinned" : " 29 GB" | 20260505 15:52:13.343 | INFO | } | 20260505 15:52:13.343 | INFO | Perf report for the last 5.13591 seconds: | 20260505 15:52:13.344 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:52:13.344 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:13.344 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:13.344 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:13.344 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 15:52:13.344 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:13.344 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:13.344 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:52:13.344 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:13.344 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:13.344 | INFO | Uploaded 36GiB to GPU in 2.21763s (16.2335GiB/s) | 20260505 15:52:13.344 | INFO | Downloaded 34GiB to GPU in 2.10089s (16.1836GiB/s) | 20260505 15:52:13.474 | INFO | Starting a round of Baz IO | 20260505 15:52:16.680 | INFO | Populating RTMetrics report | 20260505 15:52:16.681 | INFO | RTMetrics report populated | 20260505 15:52:16.681 | INFO | Opening tmp RTMetrics json file | 20260505 15:52:16.690 | INFO | Serializing RTMetrics to json | 20260505 15:52:16.692 | INFO | Serialization done, swapping tmp file now... | 20260505 15:52:16.694 | INFO | RTMetrics done | 20260505 15:52:16.777 | INFO | Finished a round of Baz IO: Wrote 2.195 GiB in 3.303 seconds | 20260505 15:52:16.777 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.37% / 0.00% / 77.63% | 20260505 15:52:16.777 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.409913% / 1.19209e-05% / 6.35783e-05% | 20260505 15:52:16.777 | INFO | Baz IO overall: | 20260505 15:52:16.778 | INFO | BytesWritten : 2631594301829 | 20260505 15:52:16.778 | INFO | NumEvents : 1249729469975 | 20260505 15:52:18.493 | INFO | Analyzing chunk frames = [4427776,4428288) | 20260505 15:52:18.493 | INFO | MemInfo: { | 20260505 15:52:18.493 | INFO | "gpu" : " 17 GB", | 20260505 15:52:18.493 | INFO | "huge" : "", | 20260505 15:52:18.493 | INFO | "huge_shared" : " 27 GB", | 20260505 15:52:18.493 | INFO | "malloc" : " 14 GB", | 20260505 15:52:18.493 | INFO | "pinned" : " 29 GB" | 20260505 15:52:18.493 | INFO | } | 20260505 15:52:18.493 | INFO | Perf report for the last 5.14939 seconds: | 20260505 15:52:18.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:52:18.493 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:18.493 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:18.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:18.493 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 15:52:18.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:18.493 | INFO | RT_METRICS : 44.9%, 1.0, 23.6ms, 0 | 20260505 15:52:18.493 | INFO | PRE_HQ : 60.6%, 1.8, 56.8ms, 0 | 20260505 15:52:18.493 | INFO | BAZWRITER : 63.5%, 3.9, 127.7ms, 0 | 20260505 15:52:18.493 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 15:52:18.493 | INFO | Uploaded 36GiB to GPU in 2.15701s (16.6897GiB/s) | 20260505 15:52:18.493 | INFO | Downloaded 38GiB to GPU in 2.25084s (16.8826GiB/s) | 20260505 15:52:23.631 | INFO | Analyzing chunk frames = [4428288,4428800) | 20260505 15:52:23.631 | INFO | MemInfo: { | 20260505 15:52:23.631 | INFO | "gpu" : " 17 GB", | 20260505 15:52:23.631 | INFO | "huge" : "", | 20260505 15:52:23.631 | INFO | "huge_shared" : " 27 GB", | 20260505 15:52:23.631 | INFO | "malloc" : " 14 GB", | 20260505 15:52:23.631 | INFO | "pinned" : " 29 GB" | 20260505 15:52:23.631 | INFO | } | 20260505 15:52:23.631 | INFO | Perf report for the last 5.13847 seconds: | 20260505 15:52:23.631 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:52:23.631 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:23.631 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:23.631 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:23.631 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 15:52:23.631 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:23.631 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:23.631 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 15:52:23.631 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:23.631 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:23.631 | INFO | Uploaded 36GiB to GPU in 2.13231s (16.8831GiB/s) | 20260505 15:52:23.632 | INFO | Downloaded 34GiB to GPU in 2.03871s (16.6772GiB/s) | 20260505 15:52:28.762 | INFO | Analyzing chunk frames = [4428800,4429312) | 20260505 15:52:28.762 | INFO | MemInfo: { | 20260505 15:52:28.762 | INFO | "gpu" : " 17 GB", | 20260505 15:52:28.762 | INFO | "huge" : "", | 20260505 15:52:28.762 | INFO | "huge_shared" : " 27 GB", | 20260505 15:52:28.762 | INFO | "malloc" : " 14 GB", | 20260505 15:52:28.762 | INFO | "pinned" : " 29 GB" | 20260505 15:52:28.762 | INFO | } | 20260505 15:52:28.762 | INFO | Perf report for the last 5.13074 seconds: | 20260505 15:52:28.762 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:52:28.762 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:28.762 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:28.762 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:28.762 | INFO | ANALYSIS : 62.9%, 3.0, 97.8ms, 0 | 20260505 15:52:28.762 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:28.762 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:28.762 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 15:52:28.762 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:28.762 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:28.762 | INFO | Uploaded 36GiB to GPU in 2.19322s (16.4142GiB/s) | 20260505 15:52:28.762 | INFO | Downloaded 34GiB to GPU in 2.0343s (16.7133GiB/s) | 20260505 15:52:33.898 | INFO | Analyzing chunk frames = [4429312,4429824) | 20260505 15:52:33.898 | INFO | MemInfo: { | 20260505 15:52:33.898 | INFO | "gpu" : " 17 GB", | 20260505 15:52:33.898 | INFO | "huge" : "", | 20260505 15:52:33.898 | INFO | "huge_shared" : " 27 GB", | 20260505 15:52:33.898 | INFO | "malloc" : " 14 GB", | 20260505 15:52:33.898 | INFO | "pinned" : " 29 GB" | 20260505 15:52:33.898 | INFO | } | 20260505 15:52:33.898 | INFO | Perf report for the last 5.13568 seconds: | 20260505 15:52:33.898 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:52:33.898 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:33.898 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:33.898 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:33.898 | INFO | ANALYSIS : 62.7%, 3.0, 97.3ms, 0 | 20260505 15:52:33.898 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:33.898 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:33.898 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 15:52:33.898 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:33.898 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:33.898 | INFO | Uploaded 36GiB to GPU in 2.18877s (16.4476GiB/s) | 20260505 15:52:33.898 | INFO | Downloaded 34GiB to GPU in 2.13271s (15.9422GiB/s) | 20260505 15:52:39.047 | INFO | Analyzing chunk frames = [4429824,4430336) | 20260505 15:52:39.047 | INFO | MemInfo: { | 20260505 15:52:39.047 | INFO | "gpu" : " 17 GB", | 20260505 15:52:39.047 | INFO | "huge" : "", | 20260505 15:52:39.047 | INFO | "huge_shared" : " 27 GB", | 20260505 15:52:39.047 | INFO | "malloc" : " 14 GB", | 20260505 15:52:39.047 | INFO | "pinned" : " 29 GB" | 20260505 15:52:39.047 | INFO | } | 20260505 15:52:39.047 | INFO | Perf report for the last 5.1495 seconds: | 20260505 15:52:39.047 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:52:39.047 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:39.047 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:39.047 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:39.047 | INFO | ANALYSIS : 62.5%, 3.0, 97.8ms, 0 | 20260505 15:52:39.047 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:39.047 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:39.047 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 15:52:39.047 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:39.047 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:39.048 | INFO | Uploaded 36GiB to GPU in 2.17177s (16.5764GiB/s) | 20260505 15:52:39.048 | INFO | Downloaded 34GiB to GPU in 2.13233s (15.945GiB/s) | 20260505 15:52:44.186 | INFO | Analyzing chunk frames = [4430336,4430848) | 20260505 15:52:44.186 | INFO | MemInfo: { | 20260505 15:52:44.186 | INFO | "gpu" : " 17 GB", | 20260505 15:52:44.186 | INFO | "huge" : "", | 20260505 15:52:44.186 | INFO | "huge_shared" : " 27 GB", | 20260505 15:52:44.186 | INFO | "malloc" : " 14 GB", | 20260505 15:52:44.186 | INFO | "pinned" : " 29 GB" | 20260505 15:52:44.186 | INFO | } | 20260505 15:52:44.186 | INFO | Perf report for the last 5.13905 seconds: | 20260505 15:52:44.186 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:52:44.186 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:44.186 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:44.186 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:44.186 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260505 15:52:44.186 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:44.186 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:44.186 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 15:52:44.186 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:44.186 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:44.187 | INFO | Uploaded 36GiB to GPU in 2.13857s (16.8337GiB/s) | 20260505 15:52:44.187 | INFO | Downloaded 34GiB to GPU in 2.03181s (16.7338GiB/s) | 20260505 15:52:49.322 | INFO | Analyzing chunk frames = [4430848,4431360) | 20260505 15:52:49.322 | INFO | MemInfo: { | 20260505 15:52:49.322 | INFO | "gpu" : " 17 GB", | 20260505 15:52:49.322 | INFO | "huge" : "", | 20260505 15:52:49.322 | INFO | "huge_shared" : " 27 GB", | 20260505 15:52:49.322 | INFO | "malloc" : " 14 GB", | 20260505 15:52:49.322 | INFO | "pinned" : " 29 GB" | 20260505 15:52:49.322 | INFO | } | 20260505 15:52:49.322 | INFO | Perf report for the last 5.13581 seconds: | 20260505 15:52:49.322 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:52:49.322 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:49.322 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:49.322 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:49.322 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 15:52:49.322 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:49.322 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:49.322 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 15:52:49.322 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:49.322 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:49.322 | INFO | Uploaded 36GiB to GPU in 2.1224s (16.9619GiB/s) | 20260505 15:52:49.323 | INFO | Downloaded 34GiB to GPU in 2.18365s (15.5703GiB/s) | 20260505 15:52:54.452 | INFO | Analyzing chunk frames = [4431360,4431872) | 20260505 15:52:54.452 | INFO | MemInfo: { | 20260505 15:52:54.452 | INFO | "gpu" : " 17 GB", | 20260505 15:52:54.452 | INFO | "huge" : "", | 20260505 15:52:54.452 | INFO | "huge_shared" : " 27 GB", | 20260505 15:52:54.453 | INFO | "malloc" : " 14 GB", | 20260505 15:52:54.453 | INFO | "pinned" : " 29 GB" | 20260505 15:52:54.453 | INFO | } | 20260505 15:52:54.453 | INFO | Perf report for the last 5.13054 seconds: | 20260505 15:52:54.453 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:52:54.453 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:54.453 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:54.453 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:54.453 | INFO | ANALYSIS : 63.7%, 3.0, 99.2ms, 0 | 20260505 15:52:54.453 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:54.453 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:54.453 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 15:52:54.453 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:54.453 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:54.453 | INFO | Uploaded 36GiB to GPU in 2.21865s (16.2261GiB/s) | 20260505 15:52:54.453 | INFO | Downloaded 34GiB to GPU in 2.09895s (16.1986GiB/s) | 20260505 15:52:54.453 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 15:52:54.453 | INFO | Total mem usage: 15.3 GiB | 20260505 15:52:54.453 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 15:52:54.453 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.84%) | 20260505 15:52:54.453 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 15:52:54.453 | INFO | High Water Breakdown: | 20260505 15:52:54.453 | INFO | 61% : 8.68 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 15:52:54.453 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 15:52:54.453 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 15:52:54.453 | INFO | ------------------------------------------ | 20260505 15:52:54.453 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:52:54.453 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 15:52:54.453 | INFO | Total mem usage: 26.5 GiB | 20260505 15:52:54.453 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 15:52:54.453 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 15:52:54.453 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 15:52:54.453 | INFO | High Water Breakdown: | 20260505 15:52:54.453 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:52:54.453 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:52:54.453 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:52:54.453 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:52:54.453 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:52:54.453 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:52:54.453 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 15:52:54.453 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 15:52:54.453 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:52:54.453 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 15:52:54.453 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 15:52:54.454 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 15:52:54.454 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 15:52:54.454 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 15:52:54.454 | INFO | ------------------------------------------ | 20260505 15:52:54.454 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:52:54.454 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 15:52:54.454 | INFO | Total mem usage: 25.6 GiB | 20260505 15:52:54.454 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 15:52:54.454 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 15:52:54.454 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 15:52:54.454 | INFO | High Water Breakdown: | 20260505 15:52:54.454 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 15:52:54.454 | INFO | ------------------------------------------ | 20260505 15:52:54.454 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:52:54.454 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 15:52:54.454 | INFO | Total mem usage: 15.7 GiB | 20260505 15:52:54.454 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 15:52:54.454 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 15:52:54.454 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 15:52:54.454 | INFO | High Water Breakdown: | 20260505 15:52:54.454 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 15:52:54.454 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 15:52:54.454 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 15:52:54.454 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 15:52:54.454 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 15:52:54.454 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 15:52:54.454 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 15:52:54.454 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 15:52:54.454 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 15:52:54.454 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:52:54.454 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 15:52:54.454 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:52:54.454 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 15:52:54.454 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 15:52:54.454 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 15:52:54.454 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 15:52:54.454 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:52:54.454 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 15:52:54.454 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:52:54.454 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:52:54.454 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:52:54.454 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 15:52:54.454 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 15:52:54.454 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:52:54.454 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 15:52:54.454 | INFO | ------------------------------------------ | 20260505 15:52:54.454 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:52:54.454 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 15:52:54.454 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:52:54.454 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:52:54.454 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.59e+01, 1.38e+01 / 2.05e+00 | 20260505 15:52:54.454 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 1.01e-01, 1.07e-02 / 3.83e-03 | 20260505 15:52:54.454 | INFO | PerfReport: 11.79%, Baseline -- 3.74e+00 / 4.50e+00, 5.81e-01 / 1.84e-01 | 20260505 15:52:54.454 | INFO | PerfReport: 3.29%, Histogram -- 1.04e+00 / 1.52e+00, 1.97e-01 / 9.30e-02 | 20260505 15:52:54.454 | INFO | PerfReport: 15.96%, DME -- 4.05e+01 / 4.51e+01, 4.30e+00 / 2.29e+00 | 20260505 15:52:54.454 | INFO | PerfReport: 56.64%, FrameLabeling -- 1.80e+01 / 1.99e+01, 1.45e+00 / 1.59e+00 | 20260505 15:52:54.454 | INFO | PerfReport: 7.36%, PulseAccumulating -- 2.33e+00 / 2.90e+00, 4.41e-01 / 1.65e-01 | 20260505 15:52:54.454 | INFO | PerfReport: 3.45%, Metrics -- 1.09e+00 / 1.42e+00, 3.04e-01 / 1.44e-01 | 20260505 15:52:54.454 | INFO | PerfReport: 1.20%, PulseSerializing -- 3.81e-01 / 4.33e-01, 6.86e-02 / 2.87e-02 | 20260505 15:52:54.454 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 15:52:54.454 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:52:54.454 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:52:54.454 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260505 15:52:54.454 | INFO | PerfReport: 22.53%, Upload -- 2.26e+01 / 0.00e+00, 7.97e+00 / 0.00e+00 | 20260505 15:52:54.454 | INFO | PerfReport: 21.94%, Download -- 2.20e+01 / 0.00e+00, 7.94e+00 / 0.00e+00 | 20260505 15:52:54.454 | INFO | PerfReport: 31.60%, Compute -- 3.17e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260505 15:52:54.454 | INFO | PerfReport: 2.25%, LockUpload -- 2.25e+00 / 0.00e+00, 7.80e+00 / 0.00e+00 | 20260505 15:52:54.454 | INFO | PerfReport: 1.44%, LockDownload -- 1.45e+00 / 0.00e+00, 3.01e+00 / 0.00e+00 | 20260505 15:52:54.454 | INFO | PerfReport: 20.22%, LockCompute -- 2.03e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 15:52:54.454 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 2.05e-02 / 0.00e+00 | 20260505 15:52:54.593 | INFO | Starting a round of Baz IO | 20260505 15:52:57.808 | INFO | Populating RTMetrics report | 20260505 15:52:57.808 | INFO | RTMetrics report populated | 20260505 15:52:57.808 | INFO | Opening tmp RTMetrics json file | 20260505 15:52:57.817 | INFO | Serializing RTMetrics to json | 20260505 15:52:57.819 | INFO | Serialization done, swapping tmp file now... | 20260505 15:52:57.821 | INFO | RTMetrics done | 20260505 15:52:57.909 | INFO | Finished a round of Baz IO: Wrote 2.194 GiB in 3.315 seconds | 20260505 15:52:57.909 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.37% / 0.00% / 77.63% | 20260505 15:52:57.909 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.409881% / 3.97364e-06% / 7.54992e-05% | 20260505 15:52:57.909 | INFO | Baz IO overall: | 20260505 15:52:57.909 | INFO | BytesWritten : 2633950589628 | 20260505 15:52:57.909 | INFO | NumEvents : 1250835948894 | 20260505 15:52:59.602 | INFO | Analyzing chunk frames = [4431872,4432384) | 20260505 15:52:59.602 | INFO | MemInfo: { | 20260505 15:52:59.602 | INFO | "gpu" : " 17 GB", | 20260505 15:52:59.602 | INFO | "huge" : "", | 20260505 15:52:59.602 | INFO | "huge_shared" : " 27 GB", | 20260505 15:52:59.602 | INFO | "malloc" : " 14 GB", | 20260505 15:52:59.602 | INFO | "pinned" : " 29 GB" | 20260505 15:52:59.602 | INFO | } | 20260505 15:52:59.602 | INFO | Perf report for the last 5.14941 seconds: | 20260505 15:52:59.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:52:59.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:59.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:59.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:59.602 | INFO | ANALYSIS : 64.1%, 3.0, 100.0ms, 0 | 20260505 15:52:59.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:52:59.602 | INFO | RT_METRICS : 44.9%, 1.0, 23.6ms, 0 | 20260505 15:52:59.602 | INFO | PRE_HQ : 60.4%, 1.8, 56.5ms, 0 | 20260505 15:52:59.602 | INFO | BAZWRITER : 63.7%, 3.9, 127.3ms, 0 | 20260505 15:52:59.602 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 15:52:59.603 | INFO | Uploaded 36GiB to GPU in 2.16591s (16.6212GiB/s) | 20260505 15:52:59.603 | INFO | Downloaded 38GiB to GPU in 2.26768s (16.7572GiB/s) | 20260505 15:53:04.742 | INFO | Analyzing chunk frames = [4432384,4432896) | 20260505 15:53:04.742 | INFO | MemInfo: { | 20260505 15:53:04.742 | INFO | "gpu" : " 17 GB", | 20260505 15:53:04.742 | INFO | "huge" : "", | 20260505 15:53:04.742 | INFO | "huge_shared" : " 27 GB", | 20260505 15:53:04.742 | INFO | "malloc" : " 14 GB", | 20260505 15:53:04.742 | INFO | "pinned" : " 29 GB" | 20260505 15:53:04.742 | INFO | } | 20260505 15:53:04.742 | INFO | Perf report for the last 5.14008 seconds: | 20260505 15:53:04.742 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:53:04.742 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:04.742 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:04.742 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:04.742 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 15:53:04.742 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:04.742 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:04.742 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 15:53:04.742 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:04.742 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:04.742 | INFO | Uploaded 36GiB to GPU in 2.13723s (16.8443GiB/s) | 20260505 15:53:04.742 | INFO | Downloaded 34GiB to GPU in 2.04237s (16.6474GiB/s) | 20260505 15:53:09.891 | INFO | Analyzing chunk frames = [4432896,4433408) | 20260505 15:53:09.891 | INFO | MemInfo: { | 20260505 15:53:09.891 | INFO | "gpu" : " 17 GB", | 20260505 15:53:09.891 | INFO | "huge" : "", | 20260505 15:53:09.891 | INFO | "huge_shared" : " 27 GB", | 20260505 15:53:09.891 | INFO | "malloc" : " 14 GB", | 20260505 15:53:09.891 | INFO | "pinned" : " 29 GB" | 20260505 15:53:09.891 | INFO | } | 20260505 15:53:09.891 | INFO | Perf report for the last 5.14896 seconds: | 20260505 15:53:09.891 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:53:09.891 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:09.891 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:09.891 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:09.891 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 15:53:09.891 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:09.891 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:09.891 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 15:53:09.891 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:09.891 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:09.891 | INFO | Uploaded 36GiB to GPU in 2.19678s (16.3876GiB/s) | 20260505 15:53:09.891 | INFO | Downloaded 34GiB to GPU in 2.03862s (16.6779GiB/s) | 20260505 15:53:15.021 | INFO | Analyzing chunk frames = [4433408,4433920) | 20260505 15:53:15.021 | INFO | MemInfo: { | 20260505 15:53:15.021 | INFO | "gpu" : " 17 GB", | 20260505 15:53:15.021 | INFO | "huge" : "", | 20260505 15:53:15.021 | INFO | "huge_shared" : " 27 GB", | 20260505 15:53:15.021 | INFO | "malloc" : " 14 GB", | 20260505 15:53:15.021 | INFO | "pinned" : " 29 GB" | 20260505 15:53:15.021 | INFO | } | 20260505 15:53:15.021 | INFO | Perf report for the last 5.13013 seconds: | 20260505 15:53:15.021 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:53:15.021 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:15.021 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:15.021 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:15.021 | INFO | ANALYSIS : 62.8%, 3.0, 97.4ms, 0 | 20260505 15:53:15.021 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:15.021 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:15.021 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:53:15.021 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:15.021 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:15.022 | INFO | Uploaded 36GiB to GPU in 2.2146s (16.2557GiB/s) | 20260505 15:53:15.022 | INFO | Downloaded 34GiB to GPU in 2.12639s (15.9896GiB/s) | 20260505 15:53:20.153 | INFO | Analyzing chunk frames = [4433920,4434432) | 20260505 15:53:20.153 | INFO | MemInfo: { | 20260505 15:53:20.153 | INFO | "gpu" : " 17 GB", | 20260505 15:53:20.153 | INFO | "huge" : "", | 20260505 15:53:20.153 | INFO | "huge_shared" : " 27 GB", | 20260505 15:53:20.153 | INFO | "malloc" : " 14 GB", | 20260505 15:53:20.153 | INFO | "pinned" : " 29 GB" | 20260505 15:53:20.153 | INFO | } | 20260505 15:53:20.153 | INFO | Perf report for the last 5.13169 seconds: | 20260505 15:53:20.153 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:53:20.153 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:20.153 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:20.153 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:20.153 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260505 15:53:20.153 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:20.153 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:20.153 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:53:20.153 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:20.153 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:20.153 | INFO | Uploaded 36GiB to GPU in 2.17548s (16.5481GiB/s) | 20260505 15:53:20.153 | INFO | Downloaded 34GiB to GPU in 2.13123s (15.9532GiB/s) | 20260505 15:53:25.294 | INFO | Analyzing chunk frames = [4434432,4434944) | 20260505 15:53:25.294 | INFO | MemInfo: { | 20260505 15:53:25.294 | INFO | "gpu" : " 17 GB", | 20260505 15:53:25.294 | INFO | "huge" : "", | 20260505 15:53:25.294 | INFO | "huge_shared" : " 27 GB", | 20260505 15:53:25.294 | INFO | "malloc" : " 14 GB", | 20260505 15:53:25.294 | INFO | "pinned" : " 29 GB" | 20260505 15:53:25.294 | INFO | } | 20260505 15:53:25.294 | INFO | Perf report for the last 5.14144 seconds: | 20260505 15:53:25.294 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:53:25.294 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:25.294 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:25.294 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:25.294 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260505 15:53:25.295 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:25.295 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:25.295 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 15:53:25.295 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:25.295 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:25.295 | INFO | Uploaded 36GiB to GPU in 2.14406s (16.7905GiB/s) | 20260505 15:53:25.295 | INFO | Downloaded 34GiB to GPU in 2.02875s (16.7591GiB/s) | 20260505 15:53:30.442 | INFO | Analyzing chunk frames = [4434944,4435456) | 20260505 15:53:30.442 | INFO | MemInfo: { | 20260505 15:53:30.442 | INFO | "gpu" : " 17 GB", | 20260505 15:53:30.442 | INFO | "huge" : "", | 20260505 15:53:30.442 | INFO | "huge_shared" : " 27 GB", | 20260505 15:53:30.442 | INFO | "malloc" : " 14 GB", | 20260505 15:53:30.442 | INFO | "pinned" : " 29 GB" | 20260505 15:53:30.442 | INFO | } | 20260505 15:53:30.442 | INFO | Perf report for the last 5.14717 seconds: | 20260505 15:53:30.442 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:53:30.442 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:30.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:30.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:30.442 | INFO | ANALYSIS : 63.2%, 3.0, 98.9ms, 0 | 20260505 15:53:30.442 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:30.442 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:30.442 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 15:53:30.442 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:30.442 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:30.442 | INFO | Uploaded 36GiB to GPU in 2.1982s (16.377GiB/s) | 20260505 15:53:30.442 | INFO | Downloaded 34GiB to GPU in 2.15201s (15.7992GiB/s) | 20260505 15:53:35.576 | INFO | Analyzing chunk frames = [4435456,4435968) | 20260505 15:53:35.576 | INFO | MemInfo: { | 20260505 15:53:35.576 | INFO | "gpu" : " 17 GB", | 20260505 15:53:35.576 | INFO | "huge" : "", | 20260505 15:53:35.576 | INFO | "huge_shared" : " 27 GB", | 20260505 15:53:35.576 | INFO | "malloc" : " 14 GB", | 20260505 15:53:35.576 | INFO | "pinned" : " 29 GB" | 20260505 15:53:35.576 | INFO | } | 20260505 15:53:35.576 | INFO | Perf report for the last 5.13404 seconds: | 20260505 15:53:35.576 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:53:35.576 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:35.576 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:35.576 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:35.576 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 15:53:35.576 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:35.576 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:35.576 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 15:53:35.576 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:35.576 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:35.576 | INFO | Uploaded 36GiB to GPU in 2.17313s (16.566GiB/s) | 20260505 15:53:35.576 | INFO | Downloaded 34GiB to GPU in 2.03357s (16.7194GiB/s) | 20260505 15:53:35.707 | INFO | Starting a round of Baz IO | 20260505 15:53:38.923 | INFO | Populating RTMetrics report | 20260505 15:53:38.923 | INFO | RTMetrics report populated | 20260505 15:53:38.923 | INFO | Opening tmp RTMetrics json file | 20260505 15:53:38.966 | INFO | Serializing RTMetrics to json | 20260505 15:53:39.000 | INFO | Serialization done, swapping tmp file now... | 20260505 15:53:39.007 | INFO | RTMetrics done | 20260505 15:53:39.024 | INFO | Finished a round of Baz IO: Wrote 2.195 GiB in 3.316 seconds | 20260505 15:53:39.024 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.37% / 0.00% / 77.63% | 20260505 15:53:39.024 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.408292% / 7.94729e-06% / 7.15256e-05% | 20260505 15:53:39.024 | INFO | Baz IO overall: | 20260505 15:53:39.024 | INFO | BytesWritten : 2636306919754 | 20260505 15:53:39.024 | INFO | NumEvents : 1251942234495 | 20260505 15:53:40.710 | INFO | Analyzing chunk frames = [4435968,4436480) | 20260505 15:53:40.710 | INFO | MemInfo: { | 20260505 15:53:40.710 | INFO | "gpu" : " 17 GB", | 20260505 15:53:40.710 | INFO | "huge" : "", | 20260505 15:53:40.710 | INFO | "huge_shared" : " 27 GB", | 20260505 15:53:40.710 | INFO | "malloc" : " 14 GB", | 20260505 15:53:40.710 | INFO | "pinned" : " 29 GB" | 20260505 15:53:40.710 | INFO | } | 20260505 15:53:40.710 | INFO | Perf report for the last 5.13433 seconds: | 20260505 15:53:40.710 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:53:40.710 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:40.710 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:40.710 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:40.710 | INFO | ANALYSIS : 64.1%, 3.0, 99.7ms, 0 | 20260505 15:53:40.710 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:40.710 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260505 15:53:40.710 | INFO | PRE_HQ : 61.2%, 1.8, 56.9ms, 0 | 20260505 15:53:40.710 | INFO | BAZWRITER : 63.9%, 3.9, 128.4ms, 0 | 20260505 15:53:40.710 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 15:53:40.710 | INFO | Uploaded 36GiB to GPU in 2.20407s (16.3334GiB/s) | 20260505 15:53:40.710 | INFO | Downloaded 38GiB to GPU in 2.26853s (16.7509GiB/s) | 20260505 15:53:45.856 | INFO | Analyzing chunk frames = [4436480,4436992) | 20260505 15:53:45.856 | INFO | MemInfo: { | 20260505 15:53:45.856 | INFO | "gpu" : " 17 GB", | 20260505 15:53:45.856 | INFO | "huge" : "", | 20260505 15:53:45.856 | INFO | "huge_shared" : " 27 GB", | 20260505 15:53:45.856 | INFO | "malloc" : " 14 GB", | 20260505 15:53:45.856 | INFO | "pinned" : " 29 GB" | 20260505 15:53:45.856 | INFO | } | 20260505 15:53:45.856 | INFO | Perf report for the last 5.14592 seconds: | 20260505 15:53:45.856 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:53:45.856 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:45.856 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:45.856 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:45.856 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 15:53:45.856 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:45.856 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:45.856 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:53:45.856 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:45.856 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:45.856 | INFO | Uploaded 36GiB to GPU in 2.19957s (16.3669GiB/s) | 20260505 15:53:45.856 | INFO | Downloaded 34GiB to GPU in 2.04264s (16.6451GiB/s) | 20260505 15:53:50.998 | INFO | Analyzing chunk frames = [4436992,4437504) | 20260505 15:53:50.998 | INFO | MemInfo: { | 20260505 15:53:50.998 | INFO | "gpu" : " 17 GB", | 20260505 15:53:50.998 | INFO | "huge" : "", | 20260505 15:53:50.998 | INFO | "huge_shared" : " 27 GB", | 20260505 15:53:50.998 | INFO | "malloc" : " 14 GB", | 20260505 15:53:50.998 | INFO | "pinned" : " 29 GB" | 20260505 15:53:50.998 | INFO | } | 20260505 15:53:50.998 | INFO | Perf report for the last 5.1422 seconds: | 20260505 15:53:50.998 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:53:50.998 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:50.998 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:50.998 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:50.998 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 15:53:50.998 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:50.998 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:50.998 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 15:53:50.998 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:50.998 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:50.999 | INFO | Uploaded 36GiB to GPU in 2.17983s (16.5151GiB/s) | 20260505 15:53:50.999 | INFO | Downloaded 34GiB to GPU in 2.02854s (16.7608GiB/s) | 20260505 15:53:56.137 | INFO | Analyzing chunk frames = [4437504,4438016) | 20260505 15:53:56.137 | INFO | MemInfo: { | 20260505 15:53:56.137 | INFO | "gpu" : " 17 GB", | 20260505 15:53:56.137 | INFO | "huge" : "", | 20260505 15:53:56.137 | INFO | "huge_shared" : " 27 GB", | 20260505 15:53:56.137 | INFO | "malloc" : " 14 GB", | 20260505 15:53:56.137 | INFO | "pinned" : " 29 GB" | 20260505 15:53:56.137 | INFO | } | 20260505 15:53:56.137 | INFO | Perf report for the last 5.13883 seconds: | 20260505 15:53:56.137 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:53:56.137 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:56.137 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:56.137 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:56.137 | INFO | ANALYSIS : 62.6%, 3.0, 97.2ms, 0 | 20260505 15:53:56.137 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:56.137 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:56.137 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 15:53:56.137 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:56.137 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:53:56.137 | INFO | Uploaded 36GiB to GPU in 2.17091s (16.5829GiB/s) | 20260505 15:53:56.137 | INFO | Downloaded 34GiB to GPU in 2.14125s (15.8786GiB/s) | 20260505 15:54:01.265 | INFO | Analyzing chunk frames = [4438016,4438528) | 20260505 15:54:01.265 | INFO | MemInfo: { | 20260505 15:54:01.265 | INFO | "gpu" : " 17 GB", | 20260505 15:54:01.265 | INFO | "huge" : "", | 20260505 15:54:01.265 | INFO | "huge_shared" : " 27 GB", | 20260505 15:54:01.265 | INFO | "malloc" : " 14 GB", | 20260505 15:54:01.265 | INFO | "pinned" : " 29 GB" | 20260505 15:54:01.265 | INFO | } | 20260505 15:54:01.265 | INFO | Perf report for the last 5.12824 seconds: | 20260505 15:54:01.265 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:54:01.265 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:01.265 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:01.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:01.265 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260505 15:54:01.265 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:01.265 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:01.265 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 15:54:01.265 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:01.265 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:01.266 | INFO | Uploaded 36GiB to GPU in 2.15265s (16.7236GiB/s) | 20260505 15:54:01.266 | INFO | Downloaded 34GiB to GPU in 2.05627s (16.5348GiB/s) | 20260505 15:54:06.403 | INFO | Analyzing chunk frames = [4438528,4439040) | 20260505 15:54:06.403 | INFO | MemInfo: { | 20260505 15:54:06.403 | INFO | "gpu" : " 17 GB", | 20260505 15:54:06.403 | INFO | "huge" : "", | 20260505 15:54:06.403 | INFO | "huge_shared" : " 27 GB", | 20260505 15:54:06.403 | INFO | "malloc" : " 14 GB", | 20260505 15:54:06.404 | INFO | "pinned" : " 29 GB" | 20260505 15:54:06.404 | INFO | } | 20260505 15:54:06.404 | INFO | Perf report for the last 5.13839 seconds: | 20260505 15:54:06.404 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:54:06.404 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:06.404 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:06.404 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:06.404 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 15:54:06.404 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:06.404 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:06.404 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:54:06.404 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:06.404 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:06.404 | INFO | Uploaded 36GiB to GPU in 2.20894s (16.2974GiB/s) | 20260505 15:54:06.404 | INFO | Downloaded 34GiB to GPU in 2.11213s (16.0975GiB/s) | 20260505 15:54:11.555 | INFO | Analyzing chunk frames = [4439040,4439552) | 20260505 15:54:11.555 | INFO | MemInfo: { | 20260505 15:54:11.555 | INFO | "gpu" : " 17 GB", | 20260505 15:54:11.555 | INFO | "huge" : "", | 20260505 15:54:11.555 | INFO | "huge_shared" : " 27 GB", | 20260505 15:54:11.555 | INFO | "malloc" : " 14 GB", | 20260505 15:54:11.555 | INFO | "pinned" : " 29 GB" | 20260505 15:54:11.555 | INFO | } | 20260505 15:54:11.555 | INFO | Perf report for the last 5.15155 seconds: | 20260505 15:54:11.555 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:54:11.555 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:11.555 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:11.555 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:11.555 | INFO | ANALYSIS : 62.9%, 3.0, 98.4ms, 0 | 20260505 15:54:11.555 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:11.555 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:11.555 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 15:54:11.555 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:11.555 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:11.556 | INFO | Uploaded 36GiB to GPU in 2.15394s (16.7136GiB/s) | 20260505 15:54:11.556 | INFO | Downloaded 34GiB to GPU in 2.15475s (15.7791GiB/s) | 20260505 15:54:16.699 | INFO | Analyzing chunk frames = [4439552,4440064) | 20260505 15:54:16.699 | INFO | MemInfo: { | 20260505 15:54:16.699 | INFO | "gpu" : " 17 GB", | 20260505 15:54:16.699 | INFO | "huge" : "", | 20260505 15:54:16.699 | INFO | "huge_shared" : " 27 GB", | 20260505 15:54:16.699 | INFO | "malloc" : " 14 GB", | 20260505 15:54:16.699 | INFO | "pinned" : " 29 GB" | 20260505 15:54:16.699 | INFO | } | 20260505 15:54:16.699 | INFO | Perf report for the last 5.14396 seconds: | 20260505 15:54:16.699 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:54:16.699 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:16.699 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:16.699 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:16.699 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 15:54:16.699 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:16.699 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:16.699 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:54:16.699 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:16.699 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:16.700 | INFO | Uploaded 36GiB to GPU in 2.14402s (16.7909GiB/s) | 20260505 15:54:16.700 | INFO | Downloaded 34GiB to GPU in 2.02043s (16.8281GiB/s) | 20260505 15:54:16.700 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 15:54:16.700 | INFO | Total mem usage: 15.3 GiB | 20260505 15:54:16.700 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 15:54:16.700 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.86%) | 20260505 15:54:16.700 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 15:54:16.700 | INFO | High Water Breakdown: | 20260505 15:54:16.700 | INFO | 61% : 8.67 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 15:54:16.700 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 15:54:16.700 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 15:54:16.700 | INFO | ------------------------------------------ | 20260505 15:54:16.700 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:54:16.700 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 15:54:16.700 | INFO | Total mem usage: 26.5 GiB | 20260505 15:54:16.700 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 15:54:16.700 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 15:54:16.700 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 15:54:16.700 | INFO | High Water Breakdown: | 20260505 15:54:16.700 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:54:16.700 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:54:16.700 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:54:16.700 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:54:16.700 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:54:16.700 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:54:16.700 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 15:54:16.700 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 15:54:16.700 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:54:16.700 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 15:54:16.700 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 15:54:16.700 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 15:54:16.700 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 15:54:16.700 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 15:54:16.700 | INFO | ------------------------------------------ | 20260505 15:54:16.700 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:54:16.700 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 15:54:16.700 | INFO | Total mem usage: 25.6 GiB | 20260505 15:54:16.700 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 15:54:16.700 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 15:54:16.700 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 15:54:16.700 | INFO | High Water Breakdown: | 20260505 15:54:16.700 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 15:54:16.700 | INFO | ------------------------------------------ | 20260505 15:54:16.700 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:54:16.700 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 15:54:16.700 | INFO | Total mem usage: 15.7 GiB | 20260505 15:54:16.700 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 15:54:16.700 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 15:54:16.700 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 15:54:16.700 | INFO | High Water Breakdown: | 20260505 15:54:16.700 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 15:54:16.700 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 15:54:16.700 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 15:54:16.700 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 15:54:16.700 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 15:54:16.700 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 15:54:16.700 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 15:54:16.700 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 15:54:16.700 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 15:54:16.700 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:54:16.700 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 15:54:16.700 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:54:16.700 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 15:54:16.700 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 15:54:16.700 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 15:54:16.700 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 15:54:16.700 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:54:16.700 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 15:54:16.700 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:54:16.700 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:54:16.700 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:54:16.700 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 15:54:16.700 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 15:54:16.700 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:54:16.700 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 15:54:16.700 | INFO | ------------------------------------------ | 20260505 15:54:16.700 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:54:16.700 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 15:54:16.700 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:54:16.700 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:54:16.700 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.50e+01, 1.34e+01 / 1.41e+00 | 20260505 15:54:16.700 | INFO | PerfReport: 0.29%, AggregateStats -- 9.30e-02 / 1.01e-01, 1.12e-02 / 4.59e-03 | 20260505 15:54:16.701 | INFO | PerfReport: 11.78%, Baseline -- 3.74e+00 / 4.47e+00, 5.80e-01 / 2.23e-01 | 20260505 15:54:16.701 | INFO | PerfReport: 3.27%, Histogram -- 1.04e+00 / 1.50e+00, 1.93e-01 / 1.19e-01 | 20260505 15:54:16.701 | INFO | PerfReport: 15.90%, DME -- 4.04e+01 / 4.47e+01, 4.23e+00 / 1.63e+00 | 20260505 15:54:16.701 | INFO | PerfReport: 56.72%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.53e+00 / 1.58e+00 | 20260505 15:54:16.701 | INFO | PerfReport: 7.39%, PulseAccumulating -- 2.34e+00 / 2.91e+00, 4.56e-01 / 2.14e-01 | 20260505 15:54:16.701 | INFO | PerfReport: 3.44%, Metrics -- 1.09e+00 / 1.43e+00, 3.03e-01 / 1.65e-01 | 20260505 15:54:16.701 | INFO | PerfReport: 1.20%, PulseSerializing -- 3.81e-01 / 4.37e-01, 6.86e-02 / 3.60e-02 | 20260505 15:54:16.701 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 15:54:16.701 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:54:16.701 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:54:16.701 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.56e+01 / 0.00e+00 | 20260505 15:54:16.701 | INFO | PerfReport: 22.59%, Upload -- 2.27e+01 / 0.00e+00, 7.95e+00 / 0.00e+00 | 20260505 15:54:16.701 | INFO | PerfReport: 21.83%, Download -- 2.19e+01 / 0.00e+00, 7.93e+00 / 0.00e+00 | 20260505 15:54:16.701 | INFO | PerfReport: 31.63%, Compute -- 3.17e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 15:54:16.701 | INFO | PerfReport: 2.28%, LockUpload -- 2.29e+00 / 0.00e+00, 7.92e+00 / 0.00e+00 | 20260505 15:54:16.701 | INFO | PerfReport: 1.41%, LockDownload -- 1.42e+00 / 0.00e+00, 2.94e+00 / 0.00e+00 | 20260505 15:54:16.701 | INFO | PerfReport: 20.24%, LockCompute -- 2.03e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 15:54:16.701 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.53e-02 / 0.00e+00 | 20260505 15:54:16.833 | INFO | Starting a round of Baz IO | 20260505 15:54:20.040 | INFO | Populating RTMetrics report | 20260505 15:54:20.041 | INFO | RTMetrics report populated | 20260505 15:54:20.041 | INFO | Opening tmp RTMetrics json file | 20260505 15:54:20.050 | INFO | Serializing RTMetrics to json | 20260505 15:54:20.053 | INFO | Serialization done, swapping tmp file now... | 20260505 15:54:20.054 | INFO | RTMetrics done | 20260505 15:54:20.138 | INFO | Finished a round of Baz IO: Wrote 2.193 GiB in 3.304 seconds | 20260505 15:54:20.138 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.36% / 0.00% / 77.63% | 20260505 15:54:20.138 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.407672% / 0% / 6.35783e-05% | 20260505 15:54:20.138 | INFO | Baz IO overall: | 20260505 15:54:20.138 | INFO | BytesWritten : 2638662097316 | 20260505 15:54:20.138 | INFO | NumEvents : 1253047968972 | 20260505 15:54:21.838 | INFO | Analyzing chunk frames = [4440064,4440576) | 20260505 15:54:21.838 | INFO | MemInfo: { | 20260505 15:54:21.838 | INFO | "gpu" : " 17 GB", | 20260505 15:54:21.838 | INFO | "huge" : "", | 20260505 15:54:21.838 | INFO | "huge_shared" : " 27 GB", | 20260505 15:54:21.838 | INFO | "malloc" : " 14 GB", | 20260505 15:54:21.838 | INFO | "pinned" : " 29 GB" | 20260505 15:54:21.838 | INFO | } | 20260505 15:54:21.838 | INFO | Perf report for the last 5.13891 seconds: | 20260505 15:54:21.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:54:21.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:21.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:21.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:21.838 | INFO | ANALYSIS : 63.9%, 3.0, 99.4ms, 0 | 20260505 15:54:21.838 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:21.838 | INFO | RT_METRICS : 44.9%, 1.0, 23.5ms, 0 | 20260505 15:54:21.838 | INFO | PRE_HQ : 60.8%, 1.8, 56.7ms, 0 | 20260505 15:54:21.838 | INFO | BAZWRITER : 63.6%, 3.9, 128.5ms, 0 | 20260505 15:54:21.838 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 15:54:21.839 | INFO | Uploaded 36GiB to GPU in 2.1877s (16.4556GiB/s) | 20260505 15:54:21.839 | INFO | Downloaded 38GiB to GPU in 2.26854s (16.7508GiB/s) | 20260505 15:54:26.967 | INFO | Analyzing chunk frames = [4440576,4441088) | 20260505 15:54:26.967 | INFO | MemInfo: { | 20260505 15:54:26.967 | INFO | "gpu" : " 17 GB", | 20260505 15:54:26.967 | INFO | "huge" : "", | 20260505 15:54:26.967 | INFO | "huge_shared" : " 27 GB", | 20260505 15:54:26.967 | INFO | "malloc" : " 14 GB", | 20260505 15:54:26.967 | INFO | "pinned" : " 29 GB" | 20260505 15:54:26.967 | INFO | } | 20260505 15:54:26.968 | INFO | Perf report for the last 5.12951 seconds: | 20260505 15:54:26.968 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:54:26.968 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:26.968 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:26.968 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:26.968 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 15:54:26.968 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:26.968 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:26.968 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 15:54:26.968 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:26.968 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:26.968 | INFO | Uploaded 36GiB to GPU in 2.14127s (16.8125GiB/s) | 20260505 15:54:26.968 | INFO | Downloaded 34GiB to GPU in 2.04021s (16.665GiB/s) | 20260505 15:54:32.110 | INFO | Analyzing chunk frames = [4441088,4441600) | 20260505 15:54:32.110 | INFO | MemInfo: { | 20260505 15:54:32.110 | INFO | "gpu" : " 17 GB", | 20260505 15:54:32.110 | INFO | "huge" : "", | 20260505 15:54:32.110 | INFO | "huge_shared" : " 27 GB", | 20260505 15:54:32.110 | INFO | "malloc" : " 14 GB", | 20260505 15:54:32.110 | INFO | "pinned" : " 29 GB" | 20260505 15:54:32.110 | INFO | } | 20260505 15:54:32.110 | INFO | Perf report for the last 5.14264 seconds: | 20260505 15:54:32.110 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:54:32.110 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:32.110 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:32.110 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:32.110 | INFO | ANALYSIS : 63.0%, 3.0, 98.1ms, 0 | 20260505 15:54:32.110 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:32.110 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:32.110 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 15:54:32.110 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:32.110 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:32.111 | INFO | Uploaded 36GiB to GPU in 2.1835s (16.4873GiB/s) | 20260505 15:54:32.111 | INFO | Downloaded 34GiB to GPU in 2.05053s (16.5811GiB/s) | 20260505 15:54:37.249 | INFO | Analyzing chunk frames = [4441600,4442112) | 20260505 15:54:37.249 | INFO | MemInfo: { | 20260505 15:54:37.249 | INFO | "gpu" : " 17 GB", | 20260505 15:54:37.249 | INFO | "huge" : "", | 20260505 15:54:37.249 | INFO | "huge_shared" : " 27 GB", | 20260505 15:54:37.249 | INFO | "malloc" : " 14 GB", | 20260505 15:54:37.249 | INFO | "pinned" : " 29 GB" | 20260505 15:54:37.249 | INFO | } | 20260505 15:54:37.249 | INFO | Perf report for the last 5.13902 seconds: | 20260505 15:54:37.249 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:54:37.249 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:37.249 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:37.249 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:37.249 | INFO | ANALYSIS : 62.6%, 3.0, 97.3ms, 0 | 20260505 15:54:37.249 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:37.249 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:37.249 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:54:37.249 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:37.250 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:37.250 | INFO | Uploaded 36GiB to GPU in 2.17726s (16.5345GiB/s) | 20260505 15:54:37.250 | INFO | Downloaded 34GiB to GPU in 2.14259s (15.8686GiB/s) | 20260505 15:54:42.390 | INFO | Analyzing chunk frames = [4442112,4442624) | 20260505 15:54:42.390 | INFO | MemInfo: { | 20260505 15:54:42.390 | INFO | "gpu" : " 17 GB", | 20260505 15:54:42.390 | INFO | "huge" : "", | 20260505 15:54:42.390 | INFO | "huge_shared" : " 27 GB", | 20260505 15:54:42.390 | INFO | "malloc" : " 14 GB", | 20260505 15:54:42.390 | INFO | "pinned" : " 29 GB" | 20260505 15:54:42.390 | INFO | } | 20260505 15:54:42.390 | INFO | Perf report for the last 5.14058 seconds: | 20260505 15:54:42.390 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:54:42.390 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:42.390 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:42.390 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:42.390 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260505 15:54:42.390 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:42.390 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:42.390 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 15:54:42.390 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:42.390 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:42.390 | INFO | Uploaded 36GiB to GPU in 2.14734s (16.7649GiB/s) | 20260505 15:54:42.390 | INFO | Downloaded 34GiB to GPU in 2.05755s (16.5245GiB/s) | 20260505 15:54:47.527 | INFO | Analyzing chunk frames = [4442624,4443136) | 20260505 15:54:47.527 | INFO | MemInfo: { | 20260505 15:54:47.527 | INFO | "gpu" : " 17 GB", | 20260505 15:54:47.527 | INFO | "huge" : "", | 20260505 15:54:47.527 | INFO | "huge_shared" : " 27 GB", | 20260505 15:54:47.527 | INFO | "malloc" : " 14 GB", | 20260505 15:54:47.527 | INFO | "pinned" : " 29 GB" | 20260505 15:54:47.527 | INFO | } | 20260505 15:54:47.527 | INFO | Perf report for the last 5.13684 seconds: | 20260505 15:54:47.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:54:47.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:47.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:47.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:47.527 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 15:54:47.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:47.527 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:47.527 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 15:54:47.527 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:47.527 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:47.527 | INFO | Uploaded 36GiB to GPU in 2.13341s (16.8744GiB/s) | 20260505 15:54:47.527 | INFO | Downloaded 34GiB to GPU in 2.03644s (16.6958GiB/s) | 20260505 15:54:52.666 | INFO | Analyzing chunk frames = [4443136,4443648) | 20260505 15:54:52.666 | INFO | MemInfo: { | 20260505 15:54:52.666 | INFO | "gpu" : " 17 GB", | 20260505 15:54:52.666 | INFO | "huge" : "", | 20260505 15:54:52.666 | INFO | "huge_shared" : " 27 GB", | 20260505 15:54:52.666 | INFO | "malloc" : " 14 GB", | 20260505 15:54:52.667 | INFO | "pinned" : " 29 GB" | 20260505 15:54:52.667 | INFO | } | 20260505 15:54:52.667 | INFO | Perf report for the last 5.1399 seconds: | 20260505 15:54:52.667 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:54:52.667 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:52.667 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:52.667 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:52.667 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 15:54:52.667 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:52.667 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:52.667 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 15:54:52.667 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:52.667 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:52.667 | INFO | Uploaded 36GiB to GPU in 2.11556s (17.0168GiB/s) | 20260505 15:54:52.667 | INFO | Downloaded 34GiB to GPU in 2.16904s (15.6752GiB/s) | 20260505 15:54:57.810 | INFO | Analyzing chunk frames = [4443648,4444160) | 20260505 15:54:57.810 | INFO | MemInfo: { | 20260505 15:54:57.810 | INFO | "gpu" : " 17 GB", | 20260505 15:54:57.810 | INFO | "huge" : "", | 20260505 15:54:57.810 | INFO | "huge_shared" : " 27 GB", | 20260505 15:54:57.810 | INFO | "malloc" : " 14 GB", | 20260505 15:54:57.810 | INFO | "pinned" : " 29 GB" | 20260505 15:54:57.810 | INFO | } | 20260505 15:54:57.810 | INFO | Perf report for the last 5.14367 seconds: | 20260505 15:54:57.810 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:54:57.810 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:57.810 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:57.810 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:57.810 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 15:54:57.810 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:57.810 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:57.810 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:54:57.811 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:57.811 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:54:57.811 | INFO | Uploaded 36GiB to GPU in 2.2345s (16.111GiB/s) | 20260505 15:54:57.811 | INFO | Downloaded 34GiB to GPU in 2.10254s (16.1709GiB/s) | 20260505 15:54:57.944 | INFO | Starting a round of Baz IO | 20260505 15:55:01.147 | INFO | Populating RTMetrics report | 20260505 15:55:01.147 | INFO | RTMetrics report populated | 20260505 15:55:01.147 | INFO | Opening tmp RTMetrics json file | 20260505 15:55:01.159 | INFO | Serializing RTMetrics to json | 20260505 15:55:01.161 | INFO | Serialization done, swapping tmp file now... | 20260505 15:55:01.162 | INFO | RTMetrics done | 20260505 15:55:01.245 | INFO | Finished a round of Baz IO: Wrote 2.192 GiB in 3.301 seconds | 20260505 15:55:01.245 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.36% / 0.00% / 77.64% | 20260505 15:55:01.245 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.40673% / 1.19209e-05% / 6.75519e-05% | 20260505 15:55:01.245 | INFO | Baz IO overall: | 20260505 15:55:01.245 | INFO | BytesWritten : 2641015773348 | 20260505 15:55:01.245 | INFO | NumEvents : 1254152566698 | 20260505 15:55:02.950 | INFO | Analyzing chunk frames = [4444160,4444672) | 20260505 15:55:02.950 | INFO | MemInfo: { | 20260505 15:55:02.950 | INFO | "gpu" : " 17 GB", | 20260505 15:55:02.951 | INFO | "huge" : "", | 20260505 15:55:02.951 | INFO | "huge_shared" : " 27 GB", | 20260505 15:55:02.951 | INFO | "malloc" : " 14 GB", | 20260505 15:55:02.951 | INFO | "pinned" : " 29 GB" | 20260505 15:55:02.951 | INFO | } | 20260505 15:55:02.951 | INFO | Perf report for the last 5.14032 seconds: | 20260505 15:55:02.951 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:55:02.951 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:02.951 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:02.951 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:02.951 | INFO | ANALYSIS : 63.8%, 3.0, 99.4ms, 0 | 20260505 15:55:02.951 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:02.951 | INFO | RT_METRICS : 45.1%, 1.0, 23.6ms, 0 | 20260505 15:55:02.951 | INFO | PRE_HQ : 60.7%, 1.8, 56.7ms, 0 | 20260505 15:55:02.951 | INFO | BAZWRITER : 63.6%, 3.9, 127.7ms, 0 | 20260505 15:55:02.951 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 15:55:02.951 | INFO | Uploaded 36GiB to GPU in 2.17711s (16.5357GiB/s) | 20260505 15:55:02.951 | INFO | Downloaded 38GiB to GPU in 2.26184s (16.8005GiB/s) | 20260505 15:55:08.083 | INFO | Analyzing chunk frames = [4444672,4445184) | 20260505 15:55:08.083 | INFO | MemInfo: { | 20260505 15:55:08.083 | INFO | "gpu" : " 17 GB", | 20260505 15:55:08.083 | INFO | "huge" : "", | 20260505 15:55:08.083 | INFO | "huge_shared" : " 27 GB", | 20260505 15:55:08.083 | INFO | "malloc" : " 14 GB", | 20260505 15:55:08.083 | INFO | "pinned" : " 29 GB" | 20260505 15:55:08.083 | INFO | } | 20260505 15:55:08.084 | INFO | Perf report for the last 5.1328 seconds: | 20260505 15:55:08.084 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:55:08.084 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:08.084 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:08.084 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:08.084 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260505 15:55:08.084 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:08.084 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:08.084 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 15:55:08.084 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:08.084 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:08.084 | INFO | Uploaded 36GiB to GPU in 2.11877s (16.991GiB/s) | 20260505 15:55:08.084 | INFO | Downloaded 34GiB to GPU in 2.0291s (16.7562GiB/s) | 20260505 15:55:13.214 | INFO | Analyzing chunk frames = [4445184,4445696) | 20260505 15:55:13.214 | INFO | MemInfo: { | 20260505 15:55:13.214 | INFO | "gpu" : " 17 GB", | 20260505 15:55:13.214 | INFO | "huge" : "", | 20260505 15:55:13.214 | INFO | "huge_shared" : " 27 GB", | 20260505 15:55:13.214 | INFO | "malloc" : " 14 GB", | 20260505 15:55:13.215 | INFO | "pinned" : " 29 GB" | 20260505 15:55:13.215 | INFO | } | 20260505 15:55:13.215 | INFO | Perf report for the last 5.13107 seconds: | 20260505 15:55:13.215 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:55:13.215 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:13.215 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:13.215 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:13.215 | INFO | ANALYSIS : 63.0%, 3.0, 97.8ms, 0 | 20260505 15:55:13.215 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:13.215 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:13.215 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:55:13.215 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:13.215 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:13.215 | INFO | Uploaded 36GiB to GPU in 2.17274s (16.5689GiB/s) | 20260505 15:55:13.215 | INFO | Downloaded 34GiB to GPU in 2.03639s (16.6962GiB/s) | 20260505 15:55:18.363 | INFO | Analyzing chunk frames = [4445696,4446208) | 20260505 15:55:18.363 | INFO | MemInfo: { | 20260505 15:55:18.363 | INFO | "gpu" : " 17 GB", | 20260505 15:55:18.363 | INFO | "huge" : "", | 20260505 15:55:18.363 | INFO | "huge_shared" : " 27 GB", | 20260505 15:55:18.363 | INFO | "malloc" : " 14 GB", | 20260505 15:55:18.363 | INFO | "pinned" : " 29 GB" | 20260505 15:55:18.363 | INFO | } | 20260505 15:55:18.363 | INFO | Perf report for the last 5.14826 seconds: | 20260505 15:55:18.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:55:18.363 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:18.363 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:18.363 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:18.363 | INFO | ANALYSIS : 62.7%, 3.0, 97.5ms, 0 | 20260505 15:55:18.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:18.363 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:18.363 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 15:55:18.363 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:18.363 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:18.363 | INFO | Uploaded 36GiB to GPU in 2.18715s (16.4598GiB/s) | 20260505 15:55:18.363 | INFO | Downloaded 34GiB to GPU in 2.13423s (15.9308GiB/s) | 20260505 15:55:23.510 | INFO | Analyzing chunk frames = [4446208,4446720) | 20260505 15:55:23.510 | INFO | MemInfo: { | 20260505 15:55:23.510 | INFO | "gpu" : " 17 GB", | 20260505 15:55:23.510 | INFO | "huge" : "", | 20260505 15:55:23.510 | INFO | "huge_shared" : " 27 GB", | 20260505 15:55:23.510 | INFO | "malloc" : " 14 GB", | 20260505 15:55:23.510 | INFO | "pinned" : " 29 GB" | 20260505 15:55:23.510 | INFO | } | 20260505 15:55:23.510 | INFO | Perf report for the last 5.14732 seconds: | 20260505 15:55:23.510 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:55:23.510 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:23.510 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:23.510 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:23.510 | INFO | ANALYSIS : 62.2%, 3.0, 97.3ms, 0 | 20260505 15:55:23.510 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:23.510 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:23.510 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 15:55:23.510 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:23.510 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:23.510 | INFO | Uploaded 36GiB to GPU in 2.16325s (16.6416GiB/s) | 20260505 15:55:23.510 | INFO | Downloaded 34GiB to GPU in 2.05418s (16.5516GiB/s) | 20260505 15:55:28.648 | INFO | Analyzing chunk frames = [4446720,4447232) | 20260505 15:55:28.648 | INFO | MemInfo: { | 20260505 15:55:28.648 | INFO | "gpu" : " 17 GB", | 20260505 15:55:28.648 | INFO | "huge" : "", | 20260505 15:55:28.648 | INFO | "huge_shared" : " 27 GB", | 20260505 15:55:28.648 | INFO | "malloc" : " 14 GB", | 20260505 15:55:28.648 | INFO | "pinned" : " 29 GB" | 20260505 15:55:28.648 | INFO | } | 20260505 15:55:28.648 | INFO | Perf report for the last 5.13792 seconds: | 20260505 15:55:28.648 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:55:28.648 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:28.648 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:28.648 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:28.648 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260505 15:55:28.648 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:28.648 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:28.648 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 15:55:28.648 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:28.648 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:28.648 | INFO | Uploaded 36GiB to GPU in 2.13915s (16.8291GiB/s) | 20260505 15:55:28.648 | INFO | Downloaded 34GiB to GPU in 2.03991s (16.6674GiB/s) | 20260505 15:55:33.776 | INFO | Analyzing chunk frames = [4447232,4447744) | 20260505 15:55:33.776 | INFO | MemInfo: { | 20260505 15:55:33.776 | INFO | "gpu" : " 17 GB", | 20260505 15:55:33.776 | INFO | "huge" : "", | 20260505 15:55:33.776 | INFO | "huge_shared" : " 27 GB", | 20260505 15:55:33.776 | INFO | "malloc" : " 14 GB", | 20260505 15:55:33.776 | INFO | "pinned" : " 29 GB" | 20260505 15:55:33.776 | INFO | } | 20260505 15:55:33.776 | INFO | Perf report for the last 5.1279 seconds: | 20260505 15:55:33.776 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:55:33.776 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:33.776 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:33.776 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:33.776 | INFO | ANALYSIS : 63.7%, 3.0, 99.2ms, 0 | 20260505 15:55:33.776 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:33.776 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:33.776 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 15:55:33.776 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:33.776 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:33.776 | INFO | Uploaded 36GiB to GPU in 2.1252s (16.9396GiB/s) | 20260505 15:55:33.776 | INFO | Downloaded 34GiB to GPU in 2.18359s (15.5707GiB/s) | 20260505 15:55:38.916 | INFO | Analyzing chunk frames = [4447744,4448256) | 20260505 15:55:38.916 | INFO | MemInfo: { | 20260505 15:55:38.916 | INFO | "gpu" : " 17 GB", | 20260505 15:55:38.916 | INFO | "huge" : "", | 20260505 15:55:38.916 | INFO | "huge_shared" : " 27 GB", | 20260505 15:55:38.916 | INFO | "malloc" : " 14 GB", | 20260505 15:55:38.916 | INFO | "pinned" : " 29 GB" | 20260505 15:55:38.916 | INFO | } | 20260505 15:55:38.916 | INFO | Perf report for the last 5.1402 seconds: | 20260505 15:55:38.916 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:55:38.916 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:38.916 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:38.916 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:38.916 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 15:55:38.916 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:38.916 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:38.916 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 15:55:38.916 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:38.916 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:38.917 | INFO | Uploaded 36GiB to GPU in 2.21597s (16.2457GiB/s) | 20260505 15:55:38.917 | INFO | Downloaded 34GiB to GPU in 2.10135s (16.1801GiB/s) | 20260505 15:55:38.917 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 15:55:38.917 | INFO | Total mem usage: 15.3 GiB | 20260505 15:55:38.917 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 15:55:38.917 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.87%) | 20260505 15:55:38.917 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 15:55:38.917 | INFO | High Water Breakdown: | 20260505 15:55:38.917 | INFO | 61% : 8.67 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 15:55:38.917 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 15:55:38.917 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 15:55:38.917 | INFO | ------------------------------------------ | 20260505 15:55:38.917 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:55:38.917 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 15:55:38.917 | INFO | Total mem usage: 26.5 GiB | 20260505 15:55:38.917 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 15:55:38.917 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 15:55:38.917 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 15:55:38.917 | INFO | High Water Breakdown: | 20260505 15:55:38.917 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:55:38.917 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:55:38.917 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:55:38.917 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:55:38.917 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:55:38.917 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:55:38.917 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 15:55:38.917 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 15:55:38.917 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:55:38.917 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 15:55:38.917 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 15:55:38.917 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 15:55:38.917 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 15:55:38.917 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 15:55:38.917 | INFO | ------------------------------------------ | 20260505 15:55:38.917 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:55:38.917 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 15:55:38.917 | INFO | Total mem usage: 25.6 GiB | 20260505 15:55:38.917 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 15:55:38.917 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 15:55:38.917 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 15:55:38.917 | INFO | High Water Breakdown: | 20260505 15:55:38.917 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 15:55:38.917 | INFO | ------------------------------------------ | 20260505 15:55:38.917 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:55:38.917 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 15:55:38.918 | INFO | Total mem usage: 15.7 GiB | 20260505 15:55:38.918 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 15:55:38.918 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 15:55:38.918 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 15:55:38.918 | INFO | High Water Breakdown: | 20260505 15:55:38.918 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 15:55:38.918 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 15:55:38.918 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 15:55:38.918 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 15:55:38.918 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 15:55:38.918 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 15:55:38.918 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 15:55:38.918 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 15:55:38.918 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 15:55:38.918 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:55:38.918 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 15:55:38.918 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:55:38.918 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 15:55:38.918 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 15:55:38.918 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 15:55:38.918 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 15:55:38.918 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:55:38.918 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 15:55:38.918 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:55:38.918 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:55:38.918 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:55:38.918 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 15:55:38.918 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 15:55:38.918 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:55:38.918 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 15:55:38.918 | INFO | ------------------------------------------ | 20260505 15:55:38.918 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:55:38.918 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 15:55:38.918 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:55:38.918 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:55:38.918 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.58e+01, 1.36e+01 / 1.89e+00 | 20260505 15:55:38.918 | INFO | PerfReport: 0.29%, AggregateStats -- 9.31e-02 / 1.01e-01, 1.09e-02 / 4.49e-03 | 20260505 15:55:38.918 | INFO | PerfReport: 11.77%, Baseline -- 3.74e+00 / 4.46e+00, 5.84e-01 / 2.04e-01 | 20260505 15:55:38.918 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.50e+00, 1.96e-01 / 1.02e-01 | 20260505 15:55:38.918 | INFO | PerfReport: 15.96%, DME -- 4.06e+01 / 4.54e+01, 4.43e+00 / 2.11e+00 | 20260505 15:55:38.918 | INFO | PerfReport: 56.73%, FrameLabeling -- 1.80e+01 / 1.96e+01, 1.53e+00 / 1.26e+00 | 20260505 15:55:38.918 | INFO | PerfReport: 7.34%, PulseAccumulating -- 2.33e+00 / 2.91e+00, 4.49e-01 / 1.65e-01 | 20260505 15:55:38.918 | INFO | PerfReport: 3.42%, Metrics -- 1.09e+00 / 1.40e+00, 3.04e-01 / 1.48e-01 | 20260505 15:55:38.918 | INFO | PerfReport: 1.20%, PulseSerializing -- 3.80e-01 / 4.38e-01, 6.91e-02 / 3.43e-02 | 20260505 15:55:38.918 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 15:55:38.918 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:55:38.918 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:55:38.918 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.57e+01 / 0.00e+00 | 20260505 15:55:38.918 | INFO | PerfReport: 22.46%, Upload -- 2.25e+01 / 0.00e+00, 7.90e+00 / 0.00e+00 | 20260505 15:55:38.918 | INFO | PerfReport: 21.87%, Download -- 2.19e+01 / 0.00e+00, 7.95e+00 / 0.00e+00 | 20260505 15:55:38.918 | INFO | PerfReport: 31.68%, Compute -- 3.18e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 15:55:38.918 | INFO | PerfReport: 2.22%, LockUpload -- 2.23e+00 / 0.00e+00, 7.83e+00 / 0.00e+00 | 20260505 15:55:38.918 | INFO | PerfReport: 1.41%, LockDownload -- 1.41e+00 / 0.00e+00, 2.95e+00 / 0.00e+00 | 20260505 15:55:38.918 | INFO | PerfReport: 20.35%, LockCompute -- 2.04e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 15:55:38.918 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.05e-02 / 0.00e+00 | 20260505 15:55:39.052 | INFO | Starting a round of Baz IO | 20260505 15:55:42.249 | INFO | Populating RTMetrics report | 20260505 15:55:42.249 | INFO | RTMetrics report populated | 20260505 15:55:42.249 | INFO | Opening tmp RTMetrics json file | 20260505 15:55:42.262 | INFO | Serializing RTMetrics to json | 20260505 15:55:42.272 | INFO | Serialization done, swapping tmp file now... | 20260505 15:55:42.288 | INFO | RTMetrics done | 20260505 15:55:42.349 | INFO | Finished a round of Baz IO: Wrote 2.193 GiB in 3.297 seconds | 20260505 15:55:42.349 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.36% / 0.00% / 77.64% | 20260505 15:55:42.349 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.404517% / 0% / 8.74201e-05% | 20260505 15:55:42.349 | INFO | Baz IO overall: | 20260505 15:55:42.349 | INFO | BytesWritten : 2643370193524 | 20260505 15:55:42.349 | INFO | NumEvents : 1255257741228 | 20260505 15:55:44.063 | INFO | Analyzing chunk frames = [4448256,4448768) | 20260505 15:55:44.063 | INFO | MemInfo: { | 20260505 15:55:44.063 | INFO | "gpu" : " 17 GB", | 20260505 15:55:44.063 | INFO | "huge" : "", | 20260505 15:55:44.063 | INFO | "huge_shared" : " 27 GB", | 20260505 15:55:44.063 | INFO | "malloc" : " 14 GB", | 20260505 15:55:44.063 | INFO | "pinned" : " 29 GB" | 20260505 15:55:44.063 | INFO | } | 20260505 15:55:44.063 | INFO | Perf report for the last 5.14663 seconds: | 20260505 15:55:44.063 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:55:44.063 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:44.063 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:44.063 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:44.063 | INFO | ANALYSIS : 63.7%, 3.0, 99.1ms, 0 | 20260505 15:55:44.063 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:44.063 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260505 15:55:44.063 | INFO | PRE_HQ : 60.5%, 1.8, 56.9ms, 0 | 20260505 15:55:44.063 | INFO | BAZWRITER : 63.5%, 3.9, 128.8ms, 0 | 20260505 15:55:44.063 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 15:55:44.063 | INFO | Uploaded 36GiB to GPU in 2.15731s (16.6874GiB/s) | 20260505 15:55:44.063 | INFO | Downloaded 38GiB to GPU in 2.25139s (16.8785GiB/s) | 20260505 15:55:49.198 | INFO | Analyzing chunk frames = [4448768,4449280) | 20260505 15:55:49.198 | INFO | MemInfo: { | 20260505 15:55:49.198 | INFO | "gpu" : " 17 GB", | 20260505 15:55:49.198 | INFO | "huge" : "", | 20260505 15:55:49.198 | INFO | "huge_shared" : " 27 GB", | 20260505 15:55:49.198 | INFO | "malloc" : " 14 GB", | 20260505 15:55:49.198 | INFO | "pinned" : " 29 GB" | 20260505 15:55:49.198 | INFO | } | 20260505 15:55:49.198 | INFO | Perf report for the last 5.13555 seconds: | 20260505 15:55:49.199 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:55:49.199 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:49.199 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:49.199 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:49.199 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 15:55:49.199 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:49.199 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:49.199 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 15:55:49.199 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:49.199 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:49.199 | INFO | Uploaded 36GiB to GPU in 2.12709s (16.9245GiB/s) | 20260505 15:55:49.199 | INFO | Downloaded 34GiB to GPU in 2.03228s (16.7299GiB/s) | 20260505 15:55:54.336 | INFO | Analyzing chunk frames = [4449280,4449792) | 20260505 15:55:54.336 | INFO | MemInfo: { | 20260505 15:55:54.336 | INFO | "gpu" : " 17 GB", | 20260505 15:55:54.336 | INFO | "huge" : "", | 20260505 15:55:54.336 | INFO | "huge_shared" : " 27 GB", | 20260505 15:55:54.336 | INFO | "malloc" : " 14 GB", | 20260505 15:55:54.336 | INFO | "pinned" : " 29 GB" | 20260505 15:55:54.336 | INFO | } | 20260505 15:55:54.336 | INFO | Perf report for the last 5.13783 seconds: | 20260505 15:55:54.336 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:55:54.336 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:54.336 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:54.336 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:54.336 | INFO | ANALYSIS : 62.8%, 3.0, 97.6ms, 0 | 20260505 15:55:54.336 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:54.336 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:54.336 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:55:54.336 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:54.336 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:54.337 | INFO | Uploaded 36GiB to GPU in 2.18047s (16.5102GiB/s) | 20260505 15:55:54.337 | INFO | Downloaded 34GiB to GPU in 2.03057s (16.7441GiB/s) | 20260505 15:55:59.478 | INFO | Analyzing chunk frames = [4449792,4450304) | 20260505 15:55:59.478 | INFO | MemInfo: { | 20260505 15:55:59.478 | INFO | "gpu" : " 17 GB", | 20260505 15:55:59.478 | INFO | "huge" : "", | 20260505 15:55:59.478 | INFO | "huge_shared" : " 27 GB", | 20260505 15:55:59.478 | INFO | "malloc" : " 14 GB", | 20260505 15:55:59.479 | INFO | "pinned" : " 29 GB" | 20260505 15:55:59.479 | INFO | } | 20260505 15:55:59.479 | INFO | Perf report for the last 5.14223 seconds: | 20260505 15:55:59.479 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:55:59.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:59.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:59.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:59.479 | INFO | ANALYSIS : 62.5%, 3.0, 97.2ms, 0 | 20260505 15:55:59.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:59.479 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:59.479 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:55:59.479 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:59.479 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:55:59.479 | INFO | Uploaded 36GiB to GPU in 2.17946s (16.5178GiB/s) | 20260505 15:55:59.479 | INFO | Downloaded 34GiB to GPU in 2.13039s (15.9596GiB/s) | 20260505 15:56:04.623 | INFO | Analyzing chunk frames = [4450304,4450816) | 20260505 15:56:04.623 | INFO | MemInfo: { | 20260505 15:56:04.623 | INFO | "gpu" : " 17 GB", | 20260505 15:56:04.623 | INFO | "huge" : "", | 20260505 15:56:04.623 | INFO | "huge_shared" : " 27 GB", | 20260505 15:56:04.624 | INFO | "malloc" : " 14 GB", | 20260505 15:56:04.624 | INFO | "pinned" : " 29 GB" | 20260505 15:56:04.624 | INFO | } | 20260505 15:56:04.624 | INFO | Perf report for the last 5.14509 seconds: | 20260505 15:56:04.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:56:04.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:04.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:04.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:04.624 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260505 15:56:04.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:04.624 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:04.624 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 15:56:04.624 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:04.624 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:04.624 | INFO | Uploaded 36GiB to GPU in 2.12788s (16.9182GiB/s) | 20260505 15:56:04.624 | INFO | Downloaded 34GiB to GPU in 2.05216s (16.5679GiB/s) | 20260505 15:56:09.758 | INFO | Analyzing chunk frames = [4450816,4451328) | 20260505 15:56:09.758 | INFO | MemInfo: { | 20260505 15:56:09.758 | INFO | "gpu" : " 17 GB", | 20260505 15:56:09.758 | INFO | "huge" : "", | 20260505 15:56:09.758 | INFO | "huge_shared" : " 27 GB", | 20260505 15:56:09.758 | INFO | "malloc" : " 14 GB", | 20260505 15:56:09.758 | INFO | "pinned" : " 29 GB" | 20260505 15:56:09.758 | INFO | } | 20260505 15:56:09.758 | INFO | Perf report for the last 5.13476 seconds: | 20260505 15:56:09.758 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:56:09.758 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:09.759 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:09.759 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:09.759 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260505 15:56:09.759 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:09.759 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:09.759 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 15:56:09.759 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:09.759 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:09.759 | INFO | Uploaded 36GiB to GPU in 2.13781s (16.8397GiB/s) | 20260505 15:56:09.759 | INFO | Downloaded 34GiB to GPU in 2.03328s (16.7217GiB/s) | 20260505 15:56:14.898 | INFO | Analyzing chunk frames = [4451328,4451840) | 20260505 15:56:14.898 | INFO | MemInfo: { | 20260505 15:56:14.898 | INFO | "gpu" : " 17 GB", | 20260505 15:56:14.898 | INFO | "huge" : "", | 20260505 15:56:14.898 | INFO | "huge_shared" : " 27 GB", | 20260505 15:56:14.898 | INFO | "malloc" : " 14 GB", | 20260505 15:56:14.898 | INFO | "pinned" : " 29 GB" | 20260505 15:56:14.898 | INFO | } | 20260505 15:56:14.898 | INFO | Perf report for the last 5.13962 seconds: | 20260505 15:56:14.898 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:56:14.898 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:14.898 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:14.898 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:14.898 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 15:56:14.898 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:14.898 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:14.898 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 15:56:14.898 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:14.898 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:14.898 | INFO | Uploaded 36GiB to GPU in 2.11593s (17.0138GiB/s) | 20260505 15:56:14.898 | INFO | Downloaded 34GiB to GPU in 2.16378s (15.7133GiB/s) | 20260505 15:56:20.032 | INFO | Analyzing chunk frames = [4451840,4452352) | 20260505 15:56:20.032 | INFO | MemInfo: { | 20260505 15:56:20.032 | INFO | "gpu" : " 17 GB", | 20260505 15:56:20.032 | INFO | "huge" : "", | 20260505 15:56:20.032 | INFO | "huge_shared" : " 27 GB", | 20260505 15:56:20.033 | INFO | "malloc" : " 14 GB", | 20260505 15:56:20.033 | INFO | "pinned" : " 29 GB" | 20260505 15:56:20.033 | INFO | } | 20260505 15:56:20.033 | INFO | Perf report for the last 5.13459 seconds: | 20260505 15:56:20.033 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:56:20.033 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:20.033 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:20.033 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:20.033 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 15:56:20.033 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:20.033 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:20.033 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 15:56:20.033 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:20.033 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:20.033 | INFO | Uploaded 36GiB to GPU in 2.22919s (16.1494GiB/s) | 20260505 15:56:20.033 | INFO | Downloaded 34GiB to GPU in 2.10547s (16.1484GiB/s) | 20260505 15:56:20.164 | INFO | Starting a round of Baz IO | 20260505 15:56:23.366 | INFO | Populating RTMetrics report | 20260505 15:56:23.366 | INFO | RTMetrics report populated | 20260505 15:56:23.366 | INFO | Opening tmp RTMetrics json file | 20260505 15:56:23.377 | INFO | Serializing RTMetrics to json | 20260505 15:56:23.378 | INFO | Serialization done, swapping tmp file now... | 20260505 15:56:23.378 | INFO | RTMetrics done | 20260505 15:56:23.463 | INFO | Finished a round of Baz IO: Wrote 2.193 GiB in 3.299 seconds | 20260505 15:56:23.463 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.35% / 0.00% / 77.64% | 20260505 15:56:23.463 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.404569% / 3.97364e-06% / 6.75519e-05% | 20260505 15:56:23.463 | INFO | Baz IO overall: | 20260505 15:56:23.463 | INFO | BytesWritten : 2645725050634 | 20260505 15:56:23.463 | INFO | NumEvents : 1256363549372 | 20260505 15:56:25.169 | INFO | Analyzing chunk frames = [4452352,4452864) | 20260505 15:56:25.169 | INFO | MemInfo: { | 20260505 15:56:25.169 | INFO | "gpu" : " 17 GB", | 20260505 15:56:25.169 | INFO | "huge" : "", | 20260505 15:56:25.169 | INFO | "huge_shared" : " 27 GB", | 20260505 15:56:25.169 | INFO | "malloc" : " 14 GB", | 20260505 15:56:25.169 | INFO | "pinned" : " 29 GB" | 20260505 15:56:25.169 | INFO | } | 20260505 15:56:25.170 | INFO | Perf report for the last 5.1368 seconds: | 20260505 15:56:25.170 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:56:25.170 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:25.170 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:25.170 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:25.170 | INFO | ANALYSIS : 63.8%, 3.0, 99.3ms, 0 | 20260505 15:56:25.170 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:25.170 | INFO | RT_METRICS : 44.9%, 1.0, 23.6ms, 0 | 20260505 15:56:25.170 | INFO | PRE_HQ : 60.6%, 1.8, 56.4ms, 0 | 20260505 15:56:25.170 | INFO | BAZWRITER : 63.6%, 3.9, 128.2ms, 0 | 20260505 15:56:25.170 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 15:56:25.170 | INFO | Uploaded 36GiB to GPU in 2.16368s (16.6383GiB/s) | 20260505 15:56:25.170 | INFO | Downloaded 38GiB to GPU in 2.25777s (16.8308GiB/s) | 20260505 15:56:30.317 | INFO | Analyzing chunk frames = [4452864,4453376) | 20260505 15:56:30.317 | INFO | MemInfo: { | 20260505 15:56:30.317 | INFO | "gpu" : " 17 GB", | 20260505 15:56:30.317 | INFO | "huge" : "", | 20260505 15:56:30.317 | INFO | "huge_shared" : " 27 GB", | 20260505 15:56:30.318 | INFO | "malloc" : " 14 GB", | 20260505 15:56:30.318 | INFO | "pinned" : " 29 GB" | 20260505 15:56:30.318 | INFO | } | 20260505 15:56:30.318 | INFO | Perf report for the last 5.14814 seconds: | 20260505 15:56:30.318 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:56:30.318 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:30.318 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:30.318 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:30.318 | INFO | ANALYSIS : 63.3%, 3.0, 99.0ms, 0 | 20260505 15:56:30.318 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:30.318 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:30.318 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 15:56:30.318 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:30.318 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:30.318 | INFO | Uploaded 36GiB to GPU in 2.20384s (16.3351GiB/s) | 20260505 15:56:30.318 | INFO | Downloaded 34GiB to GPU in 2.06301s (16.4807GiB/s) | 20260505 15:56:35.462 | INFO | Analyzing chunk frames = [4453376,4453888) | 20260505 15:56:35.462 | INFO | MemInfo: { | 20260505 15:56:35.462 | INFO | "gpu" : " 17 GB", | 20260505 15:56:35.462 | INFO | "huge" : "", | 20260505 15:56:35.462 | INFO | "huge_shared" : " 27 GB", | 20260505 15:56:35.463 | INFO | "malloc" : " 14 GB", | 20260505 15:56:35.463 | INFO | "pinned" : " 29 GB" | 20260505 15:56:35.463 | INFO | } | 20260505 15:56:35.463 | INFO | Perf report for the last 5.14503 seconds: | 20260505 15:56:35.463 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:56:35.463 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:35.463 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:35.463 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:35.463 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260505 15:56:35.463 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:35.463 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:35.463 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:56:35.463 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:35.463 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:35.463 | INFO | Uploaded 36GiB to GPU in 2.2007s (16.3584GiB/s) | 20260505 15:56:35.463 | INFO | Downloaded 34GiB to GPU in 2.04371s (16.6364GiB/s) | 20260505 15:56:40.592 | INFO | Analyzing chunk frames = [4453888,4454400) | 20260505 15:56:40.592 | INFO | MemInfo: { | 20260505 15:56:40.592 | INFO | "gpu" : " 17 GB", | 20260505 15:56:40.592 | INFO | "huge" : "", | 20260505 15:56:40.592 | INFO | "huge_shared" : " 27 GB", | 20260505 15:56:40.592 | INFO | "malloc" : " 14 GB", | 20260505 15:56:40.592 | INFO | "pinned" : " 29 GB" | 20260505 15:56:40.593 | INFO | } | 20260505 15:56:40.593 | INFO | Perf report for the last 5.12997 seconds: | 20260505 15:56:40.593 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:56:40.593 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:40.593 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:40.593 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:40.593 | INFO | ANALYSIS : 62.7%, 3.0, 97.2ms, 0 | 20260505 15:56:40.593 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:40.593 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:40.593 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:56:40.593 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:40.593 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:40.593 | INFO | Uploaded 36GiB to GPU in 2.17734s (16.5339GiB/s) | 20260505 15:56:40.593 | INFO | Downloaded 34GiB to GPU in 2.14152s (15.8766GiB/s) | 20260505 15:56:45.723 | INFO | Analyzing chunk frames = [4454400,4454912) | 20260505 15:56:45.723 | INFO | MemInfo: { | 20260505 15:56:45.723 | INFO | "gpu" : " 17 GB", | 20260505 15:56:45.723 | INFO | "huge" : "", | 20260505 15:56:45.723 | INFO | "huge_shared" : " 27 GB", | 20260505 15:56:45.723 | INFO | "malloc" : " 14 GB", | 20260505 15:56:45.724 | INFO | "pinned" : " 29 GB" | 20260505 15:56:45.724 | INFO | } | 20260505 15:56:45.724 | INFO | Perf report for the last 5.13103 seconds: | 20260505 15:56:45.724 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:56:45.724 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:45.724 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:45.724 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:45.724 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260505 15:56:45.724 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:45.724 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:45.724 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:56:45.724 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:45.724 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:45.724 | INFO | Uploaded 36GiB to GPU in 2.17841s (16.5258GiB/s) | 20260505 15:56:45.724 | INFO | Downloaded 34GiB to GPU in 2.13414s (15.9314GiB/s) | 20260505 15:56:50.875 | INFO | Analyzing chunk frames = [4454912,4455424) | 20260505 15:56:50.875 | INFO | MemInfo: { | 20260505 15:56:50.875 | INFO | "gpu" : " 17 GB", | 20260505 15:56:50.875 | INFO | "huge" : "", | 20260505 15:56:50.875 | INFO | "huge_shared" : " 27 GB", | 20260505 15:56:50.875 | INFO | "malloc" : " 14 GB", | 20260505 15:56:50.875 | INFO | "pinned" : " 29 GB" | 20260505 15:56:50.875 | INFO | } | 20260505 15:56:50.875 | INFO | Perf report for the last 5.15168 seconds: | 20260505 15:56:50.875 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:56:50.875 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:50.875 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:50.875 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:50.876 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 15:56:50.876 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:50.876 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:50.876 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 15:56:50.876 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:50.876 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:50.876 | INFO | Uploaded 36GiB to GPU in 2.20558s (16.3223GiB/s) | 20260505 15:56:50.876 | INFO | Downloaded 34GiB to GPU in 2.11485s (16.0768GiB/s) | 20260505 15:56:56.014 | INFO | Analyzing chunk frames = [4455424,4455936) | 20260505 15:56:56.014 | INFO | MemInfo: { | 20260505 15:56:56.014 | INFO | "gpu" : " 17 GB", | 20260505 15:56:56.014 | INFO | "huge" : "", | 20260505 15:56:56.014 | INFO | "huge_shared" : " 27 GB", | 20260505 15:56:56.014 | INFO | "malloc" : " 14 GB", | 20260505 15:56:56.014 | INFO | "pinned" : " 29 GB" | 20260505 15:56:56.014 | INFO | } | 20260505 15:56:56.014 | INFO | Perf report for the last 5.13857 seconds: | 20260505 15:56:56.014 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:56:56.014 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:56.014 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:56.014 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:56.014 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 15:56:56.014 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:56.014 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:56.014 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 15:56:56.014 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:56.014 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:56:56.014 | INFO | Uploaded 36GiB to GPU in 2.16553s (16.6241GiB/s) | 20260505 15:56:56.014 | INFO | Downloaded 34GiB to GPU in 2.17872s (15.6055GiB/s) | 20260505 15:57:01.148 | INFO | Analyzing chunk frames = [4455936,4456448) | 20260505 15:57:01.148 | INFO | MemInfo: { | 20260505 15:57:01.148 | INFO | "gpu" : " 17 GB", | 20260505 15:57:01.148 | INFO | "huge" : "", | 20260505 15:57:01.148 | INFO | "huge_shared" : " 27 GB", | 20260505 15:57:01.148 | INFO | "malloc" : " 14 GB", | 20260505 15:57:01.148 | INFO | "pinned" : " 29 GB" | 20260505 15:57:01.148 | INFO | } | 20260505 15:57:01.148 | INFO | Perf report for the last 5.13432 seconds: | 20260505 15:57:01.148 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:57:01.148 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:01.148 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:01.148 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:01.148 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 15:57:01.148 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:01.148 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:01.148 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 15:57:01.148 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:01.149 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:01.149 | INFO | Uploaded 36GiB to GPU in 2.15056s (16.7398GiB/s) | 20260505 15:57:01.149 | INFO | Downloaded 34GiB to GPU in 2.01945s (16.8363GiB/s) | 20260505 15:57:01.149 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 15:57:01.149 | INFO | Total mem usage: 15.3 GiB | 20260505 15:57:01.149 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 15:57:01.149 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.84%) | 20260505 15:57:01.149 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 15:57:01.149 | INFO | High Water Breakdown: | 20260505 15:57:01.149 | INFO | 61% : 8.68 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 15:57:01.149 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 15:57:01.149 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 15:57:01.149 | INFO | ------------------------------------------ | 20260505 15:57:01.149 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:57:01.149 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 15:57:01.149 | INFO | Total mem usage: 26.5 GiB | 20260505 15:57:01.149 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 15:57:01.149 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 15:57:01.149 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 15:57:01.149 | INFO | High Water Breakdown: | 20260505 15:57:01.149 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:57:01.149 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:57:01.149 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:57:01.149 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:57:01.149 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:57:01.149 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:57:01.149 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 15:57:01.149 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 15:57:01.149 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:57:01.149 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 15:57:01.149 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 15:57:01.149 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 15:57:01.149 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 15:57:01.149 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 15:57:01.149 | INFO | ------------------------------------------ | 20260505 15:57:01.149 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:57:01.149 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 15:57:01.149 | INFO | Total mem usage: 25.6 GiB | 20260505 15:57:01.149 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 15:57:01.149 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 15:57:01.149 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 15:57:01.149 | INFO | High Water Breakdown: | 20260505 15:57:01.149 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 15:57:01.149 | INFO | ------------------------------------------ | 20260505 15:57:01.149 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:57:01.149 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 15:57:01.149 | INFO | Total mem usage: 15.7 GiB | 20260505 15:57:01.149 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 15:57:01.149 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 15:57:01.149 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 15:57:01.149 | INFO | High Water Breakdown: | 20260505 15:57:01.149 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 15:57:01.149 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 15:57:01.149 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 15:57:01.149 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 15:57:01.149 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 15:57:01.149 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 15:57:01.149 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 15:57:01.149 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 15:57:01.149 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 15:57:01.149 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:57:01.149 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 15:57:01.149 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:57:01.149 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 15:57:01.149 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 15:57:01.149 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 15:57:01.149 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 15:57:01.149 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:57:01.149 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 15:57:01.150 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:57:01.150 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:57:01.150 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:57:01.150 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 15:57:01.150 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 15:57:01.150 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:57:01.150 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 15:57:01.150 | INFO | ------------------------------------------ | 20260505 15:57:01.150 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:57:01.150 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 15:57:01.150 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:57:01.150 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:57:01.150 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.54e+01, 1.35e+01 / 1.76e+00 | 20260505 15:57:01.150 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 1.01e-01, 1.07e-02 / 4.24e-03 | 20260505 15:57:01.150 | INFO | PerfReport: 11.77%, Baseline -- 3.73e+00 / 4.53e+00, 5.72e-01 / 1.67e-01 | 20260505 15:57:01.150 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.53e+00, 1.94e-01 / 8.05e-02 | 20260505 15:57:01.150 | INFO | PerfReport: 15.95%, DME -- 4.04e+01 / 4.51e+01, 4.46e+00 / 1.89e+00 | 20260505 15:57:01.150 | INFO | PerfReport: 56.64%, FrameLabeling -- 1.79e+01 / 1.94e+01, 1.41e+00 / 1.45e+00 | 20260505 15:57:01.150 | INFO | PerfReport: 7.41%, PulseAccumulating -- 2.35e+00 / 2.90e+00, 4.53e-01 / 2.06e-01 | 20260505 15:57:01.150 | INFO | PerfReport: 3.45%, Metrics -- 1.09e+00 / 1.43e+00, 3.03e-01 / 1.65e-01 | 20260505 15:57:01.150 | INFO | PerfReport: 1.20%, PulseSerializing -- 3.81e-01 / 4.46e-01, 6.90e-02 / 4.10e-02 | 20260505 15:57:01.150 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 15:57:01.150 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:57:01.150 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:57:01.150 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260505 15:57:01.150 | INFO | PerfReport: 22.54%, Upload -- 2.26e+01 / 0.00e+00, 7.94e+00 / 0.00e+00 | 20260505 15:57:01.150 | INFO | PerfReport: 21.92%, Download -- 2.20e+01 / 0.00e+00, 7.93e+00 / 0.00e+00 | 20260505 15:57:01.150 | INFO | PerfReport: 31.59%, Compute -- 3.17e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 15:57:01.150 | INFO | PerfReport: 2.29%, LockUpload -- 2.30e+00 / 0.00e+00, 7.91e+00 / 0.00e+00 | 20260505 15:57:01.150 | INFO | PerfReport: 1.42%, LockDownload -- 1.42e+00 / 0.00e+00, 2.96e+00 / 0.00e+00 | 20260505 15:57:01.150 | INFO | PerfReport: 20.22%, LockCompute -- 2.03e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 15:57:01.150 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.93e-02 / 0.00e+00 | 20260505 15:57:01.278 | INFO | Starting a round of Baz IO | 20260505 15:57:04.484 | INFO | Populating RTMetrics report | 20260505 15:57:04.485 | INFO | RTMetrics report populated | 20260505 15:57:04.485 | INFO | Opening tmp RTMetrics json file | 20260505 15:57:04.492 | INFO | Serializing RTMetrics to json | 20260505 15:57:04.493 | INFO | Serialization done, swapping tmp file now... | 20260505 15:57:04.494 | INFO | RTMetrics done | 20260505 15:57:04.584 | INFO | Finished a round of Baz IO: Wrote 2.191 GiB in 3.306 seconds | 20260505 15:57:04.584 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.35% / 0.00% / 77.65% | 20260505 15:57:04.584 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.403015% / 7.94729e-06% / 5.96046e-05% | 20260505 15:57:04.584 | INFO | Baz IO overall: | 20260505 15:57:04.584 | INFO | BytesWritten : 2648077247177 | 20260505 15:57:04.584 | INFO | NumEvents : 1257467555286 | 20260505 15:57:06.284 | INFO | Analyzing chunk frames = [4456448,4456960) | 20260505 15:57:06.284 | INFO | MemInfo: { | 20260505 15:57:06.284 | INFO | "gpu" : " 17 GB", | 20260505 15:57:06.284 | INFO | "huge" : "", | 20260505 15:57:06.284 | INFO | "huge_shared" : " 27 GB", | 20260505 15:57:06.284 | INFO | "malloc" : " 14 GB", | 20260505 15:57:06.284 | INFO | "pinned" : " 29 GB" | 20260505 15:57:06.284 | INFO | } | 20260505 15:57:06.284 | INFO | Perf report for the last 5.13603 seconds: | 20260505 15:57:06.285 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:57:06.285 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:06.285 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:06.285 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:06.285 | INFO | ANALYSIS : 63.9%, 3.0, 99.3ms, 0 | 20260505 15:57:06.285 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:06.285 | INFO | RT_METRICS : 44.9%, 1.0, 23.5ms, 0 | 20260505 15:57:06.285 | INFO | PRE_HQ : 60.6%, 1.8, 56.6ms, 0 | 20260505 15:57:06.285 | INFO | BAZWRITER : 63.7%, 3.9, 128.1ms, 0 | 20260505 15:57:06.285 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 15:57:06.285 | INFO | Uploaded 36GiB to GPU in 2.1557s (16.6999GiB/s) | 20260505 15:57:06.285 | INFO | Downloaded 38GiB to GPU in 2.2504s (16.8859GiB/s) | 20260505 15:57:11.425 | INFO | Analyzing chunk frames = [4456960,4457472) | 20260505 15:57:11.425 | INFO | MemInfo: { | 20260505 15:57:11.425 | INFO | "gpu" : " 17 GB", | 20260505 15:57:11.425 | INFO | "huge" : "", | 20260505 15:57:11.425 | INFO | "huge_shared" : " 27 GB", | 20260505 15:57:11.425 | INFO | "malloc" : " 14 GB", | 20260505 15:57:11.425 | INFO | "pinned" : " 29 GB" | 20260505 15:57:11.425 | INFO | } | 20260505 15:57:11.425 | INFO | Perf report for the last 5.14101 seconds: | 20260505 15:57:11.425 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:57:11.425 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:11.425 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:11.425 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:11.425 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 15:57:11.425 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:11.426 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:11.426 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 15:57:11.426 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:11.426 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:11.426 | INFO | Uploaded 36GiB to GPU in 2.19937s (16.3683GiB/s) | 20260505 15:57:11.426 | INFO | Downloaded 34GiB to GPU in 2.04244s (16.6468GiB/s) | 20260505 15:57:16.572 | INFO | Analyzing chunk frames = [4457472,4457984) | 20260505 15:57:16.572 | INFO | MemInfo: { | 20260505 15:57:16.572 | INFO | "gpu" : " 17 GB", | 20260505 15:57:16.572 | INFO | "huge" : "", | 20260505 15:57:16.573 | INFO | "huge_shared" : " 27 GB", | 20260505 15:57:16.573 | INFO | "malloc" : " 14 GB", | 20260505 15:57:16.573 | INFO | "pinned" : " 29 GB" | 20260505 15:57:16.573 | INFO | } | 20260505 15:57:16.573 | INFO | Perf report for the last 5.14729 seconds: | 20260505 15:57:16.573 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:57:16.573 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:16.573 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:16.573 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:16.573 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260505 15:57:16.573 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:16.573 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:16.573 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:57:16.573 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:16.573 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:16.573 | INFO | Uploaded 36GiB to GPU in 2.19886s (16.3721GiB/s) | 20260505 15:57:16.573 | INFO | Downloaded 34GiB to GPU in 2.04549s (16.622GiB/s) | 20260505 15:57:21.713 | INFO | Analyzing chunk frames = [4457984,4458496) | 20260505 15:57:21.713 | INFO | MemInfo: { | 20260505 15:57:21.713 | INFO | "gpu" : " 17 GB", | 20260505 15:57:21.713 | INFO | "huge" : "", | 20260505 15:57:21.713 | INFO | "huge_shared" : " 27 GB", | 20260505 15:57:21.713 | INFO | "malloc" : " 14 GB", | 20260505 15:57:21.713 | INFO | "pinned" : " 29 GB" | 20260505 15:57:21.713 | INFO | } | 20260505 15:57:21.713 | INFO | Perf report for the last 5.14004 seconds: | 20260505 15:57:21.713 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:57:21.713 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:21.713 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:21.713 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:21.713 | INFO | ANALYSIS : 62.5%, 3.0, 97.1ms, 0 | 20260505 15:57:21.713 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:21.713 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:21.713 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 15:57:21.713 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:21.713 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:21.713 | INFO | Uploaded 36GiB to GPU in 2.1721s (16.5738GiB/s) | 20260505 15:57:21.713 | INFO | Downloaded 34GiB to GPU in 2.14254s (15.869GiB/s) | 20260505 15:57:26.843 | INFO | Analyzing chunk frames = [4458496,4459008) | 20260505 15:57:26.843 | INFO | MemInfo: { | 20260505 15:57:26.843 | INFO | "gpu" : " 17 GB", | 20260505 15:57:26.843 | INFO | "huge" : "", | 20260505 15:57:26.843 | INFO | "huge_shared" : " 27 GB", | 20260505 15:57:26.843 | INFO | "malloc" : " 14 GB", | 20260505 15:57:26.843 | INFO | "pinned" : " 29 GB" | 20260505 15:57:26.843 | INFO | } | 20260505 15:57:26.843 | INFO | Perf report for the last 5.13071 seconds: | 20260505 15:57:26.843 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:57:26.843 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:26.844 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:26.844 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:26.844 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260505 15:57:26.844 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:26.844 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:26.844 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:57:26.844 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:26.844 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:26.844 | INFO | Uploaded 36GiB to GPU in 2.14068s (16.8171GiB/s) | 20260505 15:57:26.844 | INFO | Downloaded 34GiB to GPU in 2.0545s (16.549GiB/s) | 20260505 15:57:31.978 | INFO | Analyzing chunk frames = [4459008,4459520) | 20260505 15:57:31.978 | INFO | MemInfo: { | 20260505 15:57:31.978 | INFO | "gpu" : " 17 GB", | 20260505 15:57:31.978 | INFO | "huge" : "", | 20260505 15:57:31.978 | INFO | "huge_shared" : " 27 GB", | 20260505 15:57:31.978 | INFO | "malloc" : " 14 GB", | 20260505 15:57:31.978 | INFO | "pinned" : " 29 GB" | 20260505 15:57:31.978 | INFO | } | 20260505 15:57:31.978 | INFO | Perf report for the last 5.13468 seconds: | 20260505 15:57:31.978 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:57:31.978 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:31.978 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:31.978 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:31.978 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 15:57:31.978 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:31.978 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:31.978 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 15:57:31.978 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:31.978 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:31.978 | INFO | Uploaded 36GiB to GPU in 2.19224s (16.4216GiB/s) | 20260505 15:57:31.978 | INFO | Downloaded 34GiB to GPU in 2.11152s (16.1021GiB/s) | 20260505 15:57:37.125 | INFO | Analyzing chunk frames = [4459520,4460032) | 20260505 15:57:37.125 | INFO | MemInfo: { | 20260505 15:57:37.125 | INFO | "gpu" : " 17 GB", | 20260505 15:57:37.125 | INFO | "huge" : "", | 20260505 15:57:37.125 | INFO | "huge_shared" : " 27 GB", | 20260505 15:57:37.125 | INFO | "malloc" : " 14 GB", | 20260505 15:57:37.125 | INFO | "pinned" : " 29 GB" | 20260505 15:57:37.125 | INFO | } | 20260505 15:57:37.125 | INFO | Perf report for the last 5.14676 seconds: | 20260505 15:57:37.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:57:37.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:37.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:37.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:37.125 | INFO | ANALYSIS : 63.2%, 3.0, 98.9ms, 0 | 20260505 15:57:37.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:37.125 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:37.125 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 15:57:37.125 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:37.125 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:37.125 | INFO | Uploaded 36GiB to GPU in 2.19585s (16.3946GiB/s) | 20260505 15:57:37.125 | INFO | Downloaded 34GiB to GPU in 2.16441s (15.7086GiB/s) | 20260505 15:57:42.267 | INFO | Analyzing chunk frames = [4460032,4460544) | 20260505 15:57:42.267 | INFO | MemInfo: { | 20260505 15:57:42.267 | INFO | "gpu" : " 17 GB", | 20260505 15:57:42.267 | INFO | "huge" : "", | 20260505 15:57:42.267 | INFO | "huge_shared" : " 27 GB", | 20260505 15:57:42.268 | INFO | "malloc" : " 14 GB", | 20260505 15:57:42.268 | INFO | "pinned" : " 29 GB" | 20260505 15:57:42.268 | INFO | } | 20260505 15:57:42.268 | INFO | Perf report for the last 5.14276 seconds: | 20260505 15:57:42.268 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:57:42.268 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:42.268 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:42.268 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:42.268 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 15:57:42.268 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:42.268 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:42.268 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 15:57:42.268 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:42.268 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:42.268 | INFO | Uploaded 36GiB to GPU in 2.15931s (16.672GiB/s) | 20260505 15:57:42.268 | INFO | Downloaded 34GiB to GPU in 2.0232s (16.8051GiB/s) | 20260505 15:57:42.407 | INFO | Starting a round of Baz IO | 20260505 15:57:45.593 | INFO | Populating RTMetrics report | 20260505 15:57:45.594 | INFO | RTMetrics report populated | 20260505 15:57:45.594 | INFO | Opening tmp RTMetrics json file | 20260505 15:57:45.607 | INFO | Serializing RTMetrics to json | 20260505 15:57:45.614 | INFO | Serialization done, swapping tmp file now... | 20260505 15:57:45.630 | INFO | RTMetrics done | 20260505 15:57:45.689 | INFO | Finished a round of Baz IO: Wrote 2.191 GiB in 3.281 seconds | 20260505 15:57:45.689 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.35% / 0.00% / 77.65% | 20260505 15:57:45.689 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.402693% / 3.97364e-06% / 5.96046e-05% | 20260505 15:57:45.689 | INFO | Baz IO overall: | 20260505 15:57:45.689 | INFO | BytesWritten : 2650430227154 | 20260505 15:57:45.689 | INFO | NumEvents : 1258572317024 | 20260505 15:57:47.406 | INFO | Analyzing chunk frames = [4460544,4461056) | 20260505 15:57:47.406 | INFO | MemInfo: { | 20260505 15:57:47.406 | INFO | "gpu" : " 17 GB", | 20260505 15:57:47.406 | INFO | "huge" : "", | 20260505 15:57:47.406 | INFO | "huge_shared" : " 27 GB", | 20260505 15:57:47.406 | INFO | "malloc" : " 14 GB", | 20260505 15:57:47.406 | INFO | "pinned" : " 29 GB" | 20260505 15:57:47.406 | INFO | } | 20260505 15:57:47.406 | INFO | Perf report for the last 5.13845 seconds: | 20260505 15:57:47.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:57:47.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:47.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:47.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:47.406 | INFO | ANALYSIS : 63.6%, 3.0, 98.8ms, 0 | 20260505 15:57:47.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:47.406 | INFO | RT_METRICS : 45.5%, 1.0, 23.9ms, 0 | 20260505 15:57:47.406 | INFO | PRE_HQ : 60.5%, 1.8, 56.9ms, 0 | 20260505 15:57:47.406 | INFO | BAZWRITER : 63.1%, 3.9, 127.4ms, 0 | 20260505 15:57:47.406 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 15:57:47.406 | INFO | Uploaded 36GiB to GPU in 2.15817s (16.6808GiB/s) | 20260505 15:57:47.406 | INFO | Downloaded 38GiB to GPU in 2.25742s (16.8333GiB/s) | 20260505 15:57:52.542 | INFO | Analyzing chunk frames = [4461056,4461568) | 20260505 15:57:52.542 | INFO | MemInfo: { | 20260505 15:57:52.542 | INFO | "gpu" : " 17 GB", | 20260505 15:57:52.542 | INFO | "huge" : "", | 20260505 15:57:52.542 | INFO | "huge_shared" : " 27 GB", | 20260505 15:57:52.542 | INFO | "malloc" : " 14 GB", | 20260505 15:57:52.542 | INFO | "pinned" : " 29 GB" | 20260505 15:57:52.542 | INFO | } | 20260505 15:57:52.542 | INFO | Perf report for the last 5.13619 seconds: | 20260505 15:57:52.542 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:57:52.542 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:52.542 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:52.542 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:52.542 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 15:57:52.542 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:52.542 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:52.542 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:57:52.542 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:52.542 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:52.543 | INFO | Uploaded 36GiB to GPU in 2.13142s (16.8902GiB/s) | 20260505 15:57:52.543 | INFO | Downloaded 34GiB to GPU in 2.03793s (16.6836GiB/s) | 20260505 15:57:57.679 | INFO | Analyzing chunk frames = [4461568,4462080) | 20260505 15:57:57.679 | INFO | MemInfo: { | 20260505 15:57:57.679 | INFO | "gpu" : " 17 GB", | 20260505 15:57:57.679 | INFO | "huge" : "", | 20260505 15:57:57.679 | INFO | "huge_shared" : " 27 GB", | 20260505 15:57:57.679 | INFO | "malloc" : " 14 GB", | 20260505 15:57:57.679 | INFO | "pinned" : " 29 GB" | 20260505 15:57:57.679 | INFO | } | 20260505 15:57:57.679 | INFO | Perf report for the last 5.1363 seconds: | 20260505 15:57:57.679 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:57:57.679 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:57.679 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:57.679 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:57.679 | INFO | ANALYSIS : 63.0%, 3.0, 98.0ms, 0 | 20260505 15:57:57.679 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:57.679 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:57.679 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 15:57:57.679 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:57.679 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:57:57.679 | INFO | Uploaded 36GiB to GPU in 2.20338s (16.3385GiB/s) | 20260505 15:57:57.679 | INFO | Downloaded 34GiB to GPU in 2.06462s (16.4679GiB/s) | 20260505 15:58:02.827 | INFO | Analyzing chunk frames = [4462080,4462592) | 20260505 15:58:02.827 | INFO | MemInfo: { | 20260505 15:58:02.827 | INFO | "gpu" : " 17 GB", | 20260505 15:58:02.827 | INFO | "huge" : "", | 20260505 15:58:02.827 | INFO | "huge_shared" : " 27 GB", | 20260505 15:58:02.827 | INFO | "malloc" : " 14 GB", | 20260505 15:58:02.827 | INFO | "pinned" : " 29 GB" | 20260505 15:58:02.827 | INFO | } | 20260505 15:58:02.827 | INFO | Perf report for the last 5.14853 seconds: | 20260505 15:58:02.827 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:58:02.827 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:02.827 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:02.827 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:02.827 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260505 15:58:02.827 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:02.827 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:02.827 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 15:58:02.827 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:02.827 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:02.827 | INFO | Uploaded 36GiB to GPU in 2.17962s (16.5166GiB/s) | 20260505 15:58:02.828 | INFO | Downloaded 34GiB to GPU in 2.13295s (15.9404GiB/s) | 20260505 15:58:07.970 | INFO | Analyzing chunk frames = [4462592,4463104) | 20260505 15:58:07.970 | INFO | MemInfo: { | 20260505 15:58:07.970 | INFO | "gpu" : " 17 GB", | 20260505 15:58:07.970 | INFO | "huge" : "", | 20260505 15:58:07.970 | INFO | "huge_shared" : " 27 GB", | 20260505 15:58:07.970 | INFO | "malloc" : " 14 GB", | 20260505 15:58:07.970 | INFO | "pinned" : " 29 GB" | 20260505 15:58:07.970 | INFO | } | 20260505 15:58:07.971 | INFO | Perf report for the last 5.1433 seconds: | 20260505 15:58:07.971 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:58:07.971 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:07.971 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:07.971 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:07.971 | INFO | ANALYSIS : 62.3%, 3.0, 97.4ms, 0 | 20260505 15:58:07.971 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:07.971 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:07.971 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 15:58:07.971 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:07.971 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:07.971 | INFO | Uploaded 36GiB to GPU in 2.15911s (16.6736GiB/s) | 20260505 15:58:07.971 | INFO | Downloaded 34GiB to GPU in 2.06171s (16.4911GiB/s) | 20260505 15:58:13.098 | INFO | Analyzing chunk frames = [4463104,4463616) | 20260505 15:58:13.098 | INFO | MemInfo: { | 20260505 15:58:13.098 | INFO | "gpu" : " 17 GB", | 20260505 15:58:13.098 | INFO | "huge" : "", | 20260505 15:58:13.098 | INFO | "huge_shared" : " 27 GB", | 20260505 15:58:13.098 | INFO | "malloc" : " 14 GB", | 20260505 15:58:13.098 | INFO | "pinned" : " 29 GB" | 20260505 15:58:13.098 | INFO | } | 20260505 15:58:13.098 | INFO | Perf report for the last 5.12797 seconds: | 20260505 15:58:13.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:58:13.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:13.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:13.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:13.099 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260505 15:58:13.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:13.099 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:13.099 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 15:58:13.099 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:13.099 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:13.099 | INFO | Uploaded 36GiB to GPU in 2.14042s (16.8191GiB/s) | 20260505 15:58:13.099 | INFO | Downloaded 34GiB to GPU in 2.04288s (16.6432GiB/s) | 20260505 15:58:18.240 | INFO | Analyzing chunk frames = [4463616,4464128) | 20260505 15:58:18.240 | INFO | MemInfo: { | 20260505 15:58:18.240 | INFO | "gpu" : " 17 GB", | 20260505 15:58:18.240 | INFO | "huge" : "", | 20260505 15:58:18.240 | INFO | "huge_shared" : " 27 GB", | 20260505 15:58:18.240 | INFO | "malloc" : " 14 GB", | 20260505 15:58:18.240 | INFO | "pinned" : " 29 GB" | 20260505 15:58:18.240 | INFO | } | 20260505 15:58:18.240 | INFO | Perf report for the last 5.14154 seconds: | 20260505 15:58:18.240 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:58:18.240 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:18.240 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:18.240 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:18.240 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 15:58:18.240 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:18.240 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:18.240 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 15:58:18.240 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:18.240 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:18.240 | INFO | Uploaded 36GiB to GPU in 2.11911s (16.9883GiB/s) | 20260505 15:58:18.240 | INFO | Downloaded 34GiB to GPU in 2.17322s (15.645GiB/s) | 20260505 15:58:23.383 | INFO | Analyzing chunk frames = [4464128,4464640) | 20260505 15:58:23.383 | INFO | MemInfo: { | 20260505 15:58:23.383 | INFO | "gpu" : " 17 GB", | 20260505 15:58:23.383 | INFO | "huge" : "", | 20260505 15:58:23.383 | INFO | "huge_shared" : " 27 GB", | 20260505 15:58:23.383 | INFO | "malloc" : " 14 GB", | 20260505 15:58:23.383 | INFO | "pinned" : " 29 GB" | 20260505 15:58:23.383 | INFO | } | 20260505 15:58:23.383 | INFO | Perf report for the last 5.14322 seconds: | 20260505 15:58:23.383 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:58:23.383 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:23.383 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:23.383 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:23.383 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 15:58:23.383 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:23.383 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:23.383 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 15:58:23.383 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:23.383 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:23.384 | INFO | Uploaded 36GiB to GPU in 2.15811s (16.6812GiB/s) | 20260505 15:58:23.384 | INFO | Downloaded 34GiB to GPU in 2.03415s (16.7146GiB/s) | 20260505 15:58:23.384 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 15:58:23.384 | INFO | Total mem usage: 15.3 GiB | 20260505 15:58:23.384 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 15:58:23.384 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -7.13%) | 20260505 15:58:23.384 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 15:58:23.384 | INFO | High Water Breakdown: | 20260505 15:58:23.384 | INFO | 61.1% : 8.75 GiB ( 91.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 15:58:23.384 | INFO | 38.7% : 5.54 GiB ( 83.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 15:58:23.384 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 15:58:23.384 | INFO | ------------------------------------------ | 20260505 15:58:23.384 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:58:23.384 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 15:58:23.384 | INFO | Total mem usage: 26.5 GiB | 20260505 15:58:23.384 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 15:58:23.384 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 15:58:23.384 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 15:58:23.384 | INFO | High Water Breakdown: | 20260505 15:58:23.384 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:58:23.384 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:58:23.384 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:58:23.384 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:58:23.384 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:58:23.384 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:58:23.384 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 15:58:23.384 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 15:58:23.384 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:58:23.384 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 15:58:23.384 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 15:58:23.384 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 15:58:23.384 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 15:58:23.384 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 15:58:23.384 | INFO | ------------------------------------------ | 20260505 15:58:23.384 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:58:23.384 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 15:58:23.384 | INFO | Total mem usage: 25.6 GiB | 20260505 15:58:23.384 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 15:58:23.384 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 15:58:23.384 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 15:58:23.384 | INFO | High Water Breakdown: | 20260505 15:58:23.384 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 15:58:23.384 | INFO | ------------------------------------------ | 20260505 15:58:23.384 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:58:23.384 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 15:58:23.384 | INFO | Total mem usage: 15.7 GiB | 20260505 15:58:23.384 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 15:58:23.384 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 15:58:23.384 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 15:58:23.384 | INFO | High Water Breakdown: | 20260505 15:58:23.384 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 15:58:23.384 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 15:58:23.384 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 15:58:23.384 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 15:58:23.384 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 15:58:23.384 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 15:58:23.384 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 15:58:23.384 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 15:58:23.384 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 15:58:23.384 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:58:23.384 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 15:58:23.384 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:58:23.384 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 15:58:23.384 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 15:58:23.384 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 15:58:23.384 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 15:58:23.384 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:58:23.384 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 15:58:23.384 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:58:23.384 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:58:23.384 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:58:23.384 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 15:58:23.384 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 15:58:23.384 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:58:23.384 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 15:58:23.384 | INFO | ------------------------------------------ | 20260505 15:58:23.384 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:58:23.384 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 15:58:23.384 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:58:23.384 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:58:23.384 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.53e+01, 1.36e+01 / 1.49e+00 | 20260505 15:58:23.384 | INFO | PerfReport: 0.29%, AggregateStats -- 9.27e-02 / 1.02e-01, 1.03e-02 / 5.15e-03 | 20260505 15:58:23.384 | INFO | PerfReport: 11.78%, Baseline -- 3.74e+00 / 4.48e+00, 5.78e-01 / 1.70e-01 | 20260505 15:58:23.384 | INFO | PerfReport: 3.29%, Histogram -- 1.04e+00 / 1.50e+00, 1.94e-01 / 9.51e-02 | 20260505 15:58:23.384 | INFO | PerfReport: 15.87%, DME -- 4.03e+01 / 4.48e+01, 4.17e+00 / 1.66e+00 | 20260505 15:58:23.384 | INFO | PerfReport: 56.82%, FrameLabeling -- 1.81e+01 / 1.97e+01, 1.57e+00 / 1.48e+00 | 20260505 15:58:23.384 | INFO | PerfReport: 7.33%, PulseAccumulating -- 2.33e+00 / 2.87e+00, 4.44e-01 / 2.06e-01 | 20260505 15:58:23.384 | INFO | PerfReport: 3.43%, Metrics -- 1.09e+00 / 1.43e+00, 3.04e-01 / 1.55e-01 | 20260505 15:58:23.384 | INFO | PerfReport: 1.20%, PulseSerializing -- 3.80e-01 / 4.32e-01, 6.88e-02 / 3.00e-02 | 20260505 15:58:23.385 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 15:58:23.385 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:58:23.385 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:58:23.385 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260505 15:58:23.385 | INFO | PerfReport: 22.52%, Upload -- 2.26e+01 / 0.00e+00, 7.82e+00 / 0.00e+00 | 20260505 15:58:23.385 | INFO | PerfReport: 21.86%, Download -- 2.19e+01 / 0.00e+00, 7.85e+00 / 0.00e+00 | 20260505 15:58:23.385 | INFO | PerfReport: 31.72%, Compute -- 3.18e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 15:58:23.385 | INFO | PerfReport: 2.18%, LockUpload -- 2.19e+00 / 0.00e+00, 7.79e+00 / 0.00e+00 | 20260505 15:58:23.385 | INFO | PerfReport: 1.42%, LockDownload -- 1.42e+00 / 0.00e+00, 2.98e+00 / 0.00e+00 | 20260505 15:58:23.385 | INFO | PerfReport: 20.29%, LockCompute -- 2.03e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 15:58:23.385 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.64e-02 / 0.00e+00 | 20260505 15:58:23.518 | INFO | Starting a round of Baz IO | 20260505 15:58:26.719 | INFO | Populating RTMetrics report | 20260505 15:58:26.720 | INFO | RTMetrics report populated | 20260505 15:58:26.720 | INFO | Opening tmp RTMetrics json file | 20260505 15:58:26.733 | INFO | Serializing RTMetrics to json | 20260505 15:58:26.734 | INFO | Serialization done, swapping tmp file now... | 20260505 15:58:26.735 | INFO | RTMetrics done | 20260505 15:58:26.821 | INFO | Finished a round of Baz IO: Wrote 2.191 GiB in 3.303 seconds | 20260505 15:58:26.821 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.35% / 0.00% / 77.65% | 20260505 15:58:26.821 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.400809% / 1.19209e-05% / 7.94729e-05% | 20260505 15:58:26.821 | INFO | Baz IO overall: | 20260505 15:58:26.821 | INFO | BytesWritten : 2652782975483 | 20260505 15:58:26.821 | INFO | NumEvents : 1259676797346 | 20260505 15:58:28.523 | INFO | Analyzing chunk frames = [4464640,4465152) | 20260505 15:58:28.523 | INFO | MemInfo: { | 20260505 15:58:28.523 | INFO | "gpu" : " 17 GB", | 20260505 15:58:28.523 | INFO | "huge" : "", | 20260505 15:58:28.523 | INFO | "huge_shared" : " 27 GB", | 20260505 15:58:28.523 | INFO | "malloc" : " 14 GB", | 20260505 15:58:28.523 | INFO | "pinned" : " 29 GB" | 20260505 15:58:28.523 | INFO | } | 20260505 15:58:28.523 | INFO | Perf report for the last 5.13965 seconds: | 20260505 15:58:28.523 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:58:28.523 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:28.523 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:28.523 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:28.523 | INFO | ANALYSIS : 63.8%, 3.0, 99.3ms, 0 | 20260505 15:58:28.523 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:28.523 | INFO | RT_METRICS : 44.9%, 1.0, 23.6ms, 0 | 20260505 15:58:28.523 | INFO | PRE_HQ : 60.7%, 1.8, 56.4ms, 0 | 20260505 15:58:28.523 | INFO | BAZWRITER : 63.6%, 4.0, 129.3ms, 0 | 20260505 15:58:28.523 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 15:58:28.523 | INFO | Uploaded 36GiB to GPU in 2.16813s (16.6041GiB/s) | 20260505 15:58:28.523 | INFO | Downloaded 38GiB to GPU in 2.25086s (16.8824GiB/s) | 20260505 15:58:33.660 | INFO | Analyzing chunk frames = [4465152,4465664) | 20260505 15:58:33.660 | INFO | MemInfo: { | 20260505 15:58:33.660 | INFO | "gpu" : " 17 GB", | 20260505 15:58:33.660 | INFO | "huge" : "", | 20260505 15:58:33.660 | INFO | "huge_shared" : " 27 GB", | 20260505 15:58:33.660 | INFO | "malloc" : " 14 GB", | 20260505 15:58:33.661 | INFO | "pinned" : " 29 GB" | 20260505 15:58:33.661 | INFO | } | 20260505 15:58:33.661 | INFO | Perf report for the last 5.13762 seconds: | 20260505 15:58:33.661 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:58:33.661 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:33.661 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:33.661 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:33.661 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 15:58:33.661 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:33.661 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:33.661 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:58:33.661 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:33.661 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:33.661 | INFO | Uploaded 36GiB to GPU in 2.13416s (16.8685GiB/s) | 20260505 15:58:33.661 | INFO | Downloaded 34GiB to GPU in 2.03595s (16.6999GiB/s) | 20260505 15:58:38.789 | INFO | Analyzing chunk frames = [4465664,4466176) | 20260505 15:58:38.789 | INFO | MemInfo: { | 20260505 15:58:38.789 | INFO | "gpu" : " 17 GB", | 20260505 15:58:38.789 | INFO | "huge" : "", | 20260505 15:58:38.789 | INFO | "huge_shared" : " 27 GB", | 20260505 15:58:38.789 | INFO | "malloc" : " 14 GB", | 20260505 15:58:38.789 | INFO | "pinned" : " 29 GB" | 20260505 15:58:38.789 | INFO | } | 20260505 15:58:38.789 | INFO | Perf report for the last 5.12874 seconds: | 20260505 15:58:38.789 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:58:38.789 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:38.789 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:38.789 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:38.789 | INFO | ANALYSIS : 63.1%, 3.0, 97.9ms, 0 | 20260505 15:58:38.789 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:38.789 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:38.789 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:58:38.789 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:38.789 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:38.790 | INFO | Uploaded 36GiB to GPU in 2.18297s (16.4913GiB/s) | 20260505 15:58:38.790 | INFO | Downloaded 34GiB to GPU in 2.04746s (16.6059GiB/s) | 20260505 15:58:43.930 | INFO | Analyzing chunk frames = [4466176,4466688) | 20260505 15:58:43.930 | INFO | MemInfo: { | 20260505 15:58:43.930 | INFO | "gpu" : " 17 GB", | 20260505 15:58:43.930 | INFO | "huge" : "", | 20260505 15:58:43.930 | INFO | "huge_shared" : " 27 GB", | 20260505 15:58:43.930 | INFO | "malloc" : " 14 GB", | 20260505 15:58:43.930 | INFO | "pinned" : " 29 GB" | 20260505 15:58:43.930 | INFO | } | 20260505 15:58:43.930 | INFO | Perf report for the last 5.14064 seconds: | 20260505 15:58:43.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:58:43.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:43.930 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:43.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:43.930 | INFO | ANALYSIS : 62.7%, 3.0, 97.5ms, 0 | 20260505 15:58:43.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:43.930 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:43.930 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:58:43.930 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:43.930 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:43.930 | INFO | Uploaded 36GiB to GPU in 2.18977s (16.4401GiB/s) | 20260505 15:58:43.930 | INFO | Downloaded 34GiB to GPU in 2.13493s (15.9256GiB/s) | 20260505 15:58:49.076 | INFO | Analyzing chunk frames = [4466688,4467200) | 20260505 15:58:49.076 | INFO | MemInfo: { | 20260505 15:58:49.076 | INFO | "gpu" : " 17 GB", | 20260505 15:58:49.076 | INFO | "huge" : "", | 20260505 15:58:49.076 | INFO | "huge_shared" : " 27 GB", | 20260505 15:58:49.076 | INFO | "malloc" : " 14 GB", | 20260505 15:58:49.076 | INFO | "pinned" : " 29 GB" | 20260505 15:58:49.076 | INFO | } | 20260505 15:58:49.076 | INFO | Perf report for the last 5.14616 seconds: | 20260505 15:58:49.076 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:58:49.076 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:49.076 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:49.076 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:49.076 | INFO | ANALYSIS : 62.6%, 3.0, 97.9ms, 0 | 20260505 15:58:49.076 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:49.076 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:49.076 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 15:58:49.076 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:49.076 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:49.076 | INFO | Uploaded 36GiB to GPU in 2.16069s (16.6613GiB/s) | 20260505 15:58:49.076 | INFO | Downloaded 34GiB to GPU in 2.1259s (15.9932GiB/s) | 20260505 15:58:54.223 | INFO | Analyzing chunk frames = [4467200,4467712) | 20260505 15:58:54.223 | INFO | MemInfo: { | 20260505 15:58:54.223 | INFO | "gpu" : " 17 GB", | 20260505 15:58:54.223 | INFO | "huge" : "", | 20260505 15:58:54.223 | INFO | "huge_shared" : " 27 GB", | 20260505 15:58:54.223 | INFO | "malloc" : " 14 GB", | 20260505 15:58:54.223 | INFO | "pinned" : " 29 GB" | 20260505 15:58:54.223 | INFO | } | 20260505 15:58:54.223 | INFO | Perf report for the last 5.14725 seconds: | 20260505 15:58:54.223 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:58:54.223 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:54.223 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:54.223 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:54.223 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260505 15:58:54.224 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:54.224 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:54.224 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:58:54.224 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:54.224 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:54.224 | INFO | Uploaded 36GiB to GPU in 2.13534s (16.8592GiB/s) | 20260505 15:58:54.224 | INFO | Downloaded 34GiB to GPU in 2.03465s (16.7105GiB/s) | 20260505 15:58:59.356 | INFO | Analyzing chunk frames = [4467712,4468224) | 20260505 15:58:59.356 | INFO | MemInfo: { | 20260505 15:58:59.356 | INFO | "gpu" : " 17 GB", | 20260505 15:58:59.356 | INFO | "huge" : "", | 20260505 15:58:59.356 | INFO | "huge_shared" : " 27 GB", | 20260505 15:58:59.356 | INFO | "malloc" : " 14 GB", | 20260505 15:58:59.356 | INFO | "pinned" : " 29 GB" | 20260505 15:58:59.356 | INFO | } | 20260505 15:58:59.356 | INFO | Perf report for the last 5.13306 seconds: | 20260505 15:58:59.356 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:58:59.356 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:59.356 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:59.357 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:59.357 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 15:58:59.357 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:59.357 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:59.357 | INFO | PRE_HQ : 29.8%, 1.0, 15.4ms, 0 | 20260505 15:58:59.357 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:59.357 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:58:59.357 | INFO | Uploaded 36GiB to GPU in 2.15007s (16.7436GiB/s) | 20260505 15:58:59.357 | INFO | Downloaded 34GiB to GPU in 2.17771s (15.6128GiB/s) | 20260505 15:59:04.484 | INFO | Analyzing chunk frames = [4468224,4468736) | 20260505 15:59:04.484 | INFO | MemInfo: { | 20260505 15:59:04.484 | INFO | "gpu" : " 17 GB", | 20260505 15:59:04.484 | INFO | "huge" : "", | 20260505 15:59:04.484 | INFO | "huge_shared" : " 27 GB", | 20260505 15:59:04.484 | INFO | "malloc" : " 14 GB", | 20260505 15:59:04.484 | INFO | "pinned" : " 29 GB" | 20260505 15:59:04.484 | INFO | } | 20260505 15:59:04.484 | INFO | Perf report for the last 5.12759 seconds: | 20260505 15:59:04.484 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:59:04.484 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:04.484 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:04.484 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:04.484 | INFO | ANALYSIS : 63.7%, 3.0, 99.2ms, 0 | 20260505 15:59:04.484 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:04.484 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:04.484 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 15:59:04.484 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:04.484 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:04.484 | INFO | Uploaded 36GiB to GPU in 2.21602s (16.2454GiB/s) | 20260505 15:59:04.484 | INFO | Downloaded 34GiB to GPU in 2.10322s (16.1657GiB/s) | 20260505 15:59:04.619 | INFO | Starting a round of Baz IO | 20260505 15:59:07.830 | INFO | Populating RTMetrics report | 20260505 15:59:07.830 | INFO | RTMetrics report populated | 20260505 15:59:07.830 | INFO | Opening tmp RTMetrics json file | 20260505 15:59:07.842 | INFO | Serializing RTMetrics to json | 20260505 15:59:07.844 | INFO | Serialization done, swapping tmp file now... | 20260505 15:59:07.844 | INFO | RTMetrics done | 20260505 15:59:07.928 | INFO | Finished a round of Baz IO: Wrote 2.19 GiB in 3.308 seconds | 20260505 15:59:07.928 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.34% / 0.00% / 77.65% | 20260505 15:59:07.928 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.400241% / 3.97364e-06% / 7.54992e-05% | 20260505 15:59:07.928 | INFO | Baz IO overall: | 20260505 15:59:07.928 | INFO | BytesWritten : 2655134876713 | 20260505 15:59:07.928 | INFO | NumEvents : 1260780959929 | 20260505 15:59:09.638 | INFO | Analyzing chunk frames = [4468736,4469248) | 20260505 15:59:09.638 | INFO | MemInfo: { | 20260505 15:59:09.638 | INFO | "gpu" : " 17 GB", | 20260505 15:59:09.638 | INFO | "huge" : "", | 20260505 15:59:09.638 | INFO | "huge_shared" : " 27 GB", | 20260505 15:59:09.638 | INFO | "malloc" : " 14 GB", | 20260505 15:59:09.638 | INFO | "pinned" : " 29 GB" | 20260505 15:59:09.638 | INFO | } | 20260505 15:59:09.638 | INFO | Perf report for the last 5.15365 seconds: | 20260505 15:59:09.638 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:59:09.638 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:09.638 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:09.638 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:09.638 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 15:59:09.638 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:09.638 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260505 15:59:09.638 | INFO | PRE_HQ : 60.4%, 1.8, 56.5ms, 0 | 20260505 15:59:09.638 | INFO | BAZWRITER : 63.6%, 3.9, 126.4ms, 0 | 20260505 15:59:09.638 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 15:59:09.638 | INFO | Uploaded 36GiB to GPU in 2.20847s (16.3009GiB/s) | 20260505 15:59:09.638 | INFO | Downloaded 38GiB to GPU in 2.26421s (16.7829GiB/s) | 20260505 15:59:14.776 | INFO | Analyzing chunk frames = [4469248,4469760) | 20260505 15:59:14.776 | INFO | MemInfo: { | 20260505 15:59:14.776 | INFO | "gpu" : " 17 GB", | 20260505 15:59:14.776 | INFO | "huge" : "", | 20260505 15:59:14.776 | INFO | "huge_shared" : " 27 GB", | 20260505 15:59:14.776 | INFO | "malloc" : " 14 GB", | 20260505 15:59:14.776 | INFO | "pinned" : " 29 GB" | 20260505 15:59:14.776 | INFO | } | 20260505 15:59:14.776 | INFO | Perf report for the last 5.13811 seconds: | 20260505 15:59:14.776 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:59:14.776 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:14.776 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:14.776 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:14.776 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 15:59:14.776 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:14.776 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:14.776 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 15:59:14.776 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:14.776 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:14.776 | INFO | Uploaded 36GiB to GPU in 2.13253s (16.8813GiB/s) | 20260505 15:59:14.776 | INFO | Downloaded 34GiB to GPU in 2.03472s (16.7099GiB/s) | 20260505 15:59:19.915 | INFO | Analyzing chunk frames = [4469760,4470272) | 20260505 15:59:19.915 | INFO | MemInfo: { | 20260505 15:59:19.915 | INFO | "gpu" : " 17 GB", | 20260505 15:59:19.915 | INFO | "huge" : "", | 20260505 15:59:19.916 | INFO | "huge_shared" : " 27 GB", | 20260505 15:59:19.916 | INFO | "malloc" : " 14 GB", | 20260505 15:59:19.916 | INFO | "pinned" : " 29 GB" | 20260505 15:59:19.916 | INFO | } | 20260505 15:59:19.916 | INFO | Perf report for the last 5.13984 seconds: | 20260505 15:59:19.916 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:59:19.916 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:19.916 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:19.916 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:19.916 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260505 15:59:19.916 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:19.916 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:19.916 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 15:59:19.916 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:19.916 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:19.916 | INFO | Uploaded 36GiB to GPU in 2.18509s (16.4753GiB/s) | 20260505 15:59:19.916 | INFO | Downloaded 34GiB to GPU in 2.03245s (16.7286GiB/s) | 20260505 15:59:25.049 | INFO | Analyzing chunk frames = [4470272,4470784) | 20260505 15:59:25.049 | INFO | MemInfo: { | 20260505 15:59:25.049 | INFO | "gpu" : " 17 GB", | 20260505 15:59:25.049 | INFO | "huge" : "", | 20260505 15:59:25.049 | INFO | "huge_shared" : " 27 GB", | 20260505 15:59:25.049 | INFO | "malloc" : " 14 GB", | 20260505 15:59:25.049 | INFO | "pinned" : " 29 GB" | 20260505 15:59:25.049 | INFO | } | 20260505 15:59:25.049 | INFO | Perf report for the last 5.13329 seconds: | 20260505 15:59:25.049 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:59:25.049 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:25.049 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:25.049 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:25.049 | INFO | ANALYSIS : 62.5%, 3.0, 97.0ms, 0 | 20260505 15:59:25.049 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:25.049 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:25.049 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 15:59:25.049 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:25.049 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:25.049 | INFO | Uploaded 36GiB to GPU in 2.17808s (16.5283GiB/s) | 20260505 15:59:25.049 | INFO | Downloaded 34GiB to GPU in 2.13545s (15.9217GiB/s) | 20260505 15:59:30.193 | INFO | Analyzing chunk frames = [4470784,4471296) | 20260505 15:59:30.193 | INFO | MemInfo: { | 20260505 15:59:30.193 | INFO | "gpu" : " 17 GB", | 20260505 15:59:30.193 | INFO | "huge" : "", | 20260505 15:59:30.193 | INFO | "huge_shared" : " 27 GB", | 20260505 15:59:30.193 | INFO | "malloc" : " 14 GB", | 20260505 15:59:30.193 | INFO | "pinned" : " 29 GB" | 20260505 15:59:30.193 | INFO | } | 20260505 15:59:30.193 | INFO | Perf report for the last 5.14424 seconds: | 20260505 15:59:30.193 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:59:30.193 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:30.193 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:30.193 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:30.193 | INFO | ANALYSIS : 62.3%, 3.0, 97.4ms, 0 | 20260505 15:59:30.193 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:30.193 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:30.193 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 15:59:30.193 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:30.193 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:30.194 | INFO | Uploaded 36GiB to GPU in 2.14785s (16.7609GiB/s) | 20260505 15:59:30.194 | INFO | Downloaded 34GiB to GPU in 2.06292s (16.4815GiB/s) | 20260505 15:59:35.338 | INFO | Analyzing chunk frames = [4471296,4471808) | 20260505 15:59:35.338 | INFO | MemInfo: { | 20260505 15:59:35.338 | INFO | "gpu" : " 17 GB", | 20260505 15:59:35.338 | INFO | "huge" : "", | 20260505 15:59:35.338 | INFO | "huge_shared" : " 27 GB", | 20260505 15:59:35.338 | INFO | "malloc" : " 14 GB", | 20260505 15:59:35.338 | INFO | "pinned" : " 29 GB" | 20260505 15:59:35.338 | INFO | } | 20260505 15:59:35.338 | INFO | Perf report for the last 5.14472 seconds: | 20260505 15:59:35.338 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:59:35.338 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:35.338 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:35.338 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:35.338 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260505 15:59:35.338 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:35.338 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:35.338 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 15:59:35.338 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:35.338 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:35.338 | INFO | Uploaded 36GiB to GPU in 2.14032s (16.8199GiB/s) | 20260505 15:59:35.338 | INFO | Downloaded 34GiB to GPU in 2.04007s (16.6661GiB/s) | 20260505 15:59:40.473 | INFO | Analyzing chunk frames = [4471808,4472320) | 20260505 15:59:40.473 | INFO | MemInfo: { | 20260505 15:59:40.473 | INFO | "gpu" : " 17 GB", | 20260505 15:59:40.473 | INFO | "huge" : "", | 20260505 15:59:40.473 | INFO | "huge_shared" : " 27 GB", | 20260505 15:59:40.473 | INFO | "malloc" : " 14 GB", | 20260505 15:59:40.473 | INFO | "pinned" : " 29 GB" | 20260505 15:59:40.473 | INFO | } | 20260505 15:59:40.473 | INFO | Perf report for the last 5.13542 seconds: | 20260505 15:59:40.473 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:59:40.473 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:40.473 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:40.473 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:40.473 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 15:59:40.474 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:40.474 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:40.474 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 15:59:40.474 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:40.474 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:40.474 | INFO | Uploaded 36GiB to GPU in 2.13625s (16.852GiB/s) | 20260505 15:59:40.474 | INFO | Downloaded 34GiB to GPU in 2.13929s (15.8931GiB/s) | 20260505 15:59:45.607 | INFO | Analyzing chunk frames = [4472320,4472832) | 20260505 15:59:45.607 | INFO | MemInfo: { | 20260505 15:59:45.607 | INFO | "gpu" : " 17 GB", | 20260505 15:59:45.607 | INFO | "huge" : "", | 20260505 15:59:45.607 | INFO | "huge_shared" : " 27 GB", | 20260505 15:59:45.607 | INFO | "malloc" : " 14 GB", | 20260505 15:59:45.607 | INFO | "pinned" : " 29 GB" | 20260505 15:59:45.607 | INFO | } | 20260505 15:59:45.607 | INFO | Perf report for the last 5.13402 seconds: | 20260505 15:59:45.607 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:59:45.607 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:45.607 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:45.607 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:45.608 | INFO | ANALYSIS : 63.5%, 3.0, 99.0ms, 0 | 20260505 15:59:45.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:45.608 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:45.608 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 15:59:45.608 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:45.608 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:45.608 | INFO | Uploaded 36GiB to GPU in 2.1777s (16.5312GiB/s) | 20260505 15:59:45.608 | INFO | Downloaded 34GiB to GPU in 2.04056s (16.6621GiB/s) | 20260505 15:59:45.608 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 15:59:45.608 | INFO | Total mem usage: 15.3 GiB | 20260505 15:59:45.608 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 15:59:45.608 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.67%) | 20260505 15:59:45.608 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 15:59:45.608 | INFO | High Water Breakdown: | 20260505 15:59:45.608 | INFO | 61.1% : 8.69 GiB ( 91.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 15:59:45.608 | INFO | 38.8% : 5.52 GiB ( 83.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 15:59:45.608 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 15:59:45.608 | INFO | ------------------------------------------ | 20260505 15:59:45.608 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:59:45.608 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 15:59:45.608 | INFO | Total mem usage: 26.5 GiB | 20260505 15:59:45.608 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 15:59:45.608 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 15:59:45.608 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 15:59:45.608 | INFO | High Water Breakdown: | 20260505 15:59:45.608 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:59:45.608 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:59:45.608 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:59:45.608 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:59:45.608 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:59:45.608 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:59:45.608 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 15:59:45.608 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 15:59:45.608 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:59:45.608 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 15:59:45.608 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 15:59:45.608 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 15:59:45.608 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 15:59:45.608 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 15:59:45.608 | INFO | ------------------------------------------ | 20260505 15:59:45.608 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:59:45.608 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 15:59:45.608 | INFO | Total mem usage: 25.6 GiB | 20260505 15:59:45.608 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 15:59:45.608 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 15:59:45.608 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 15:59:45.609 | INFO | High Water Breakdown: | 20260505 15:59:45.609 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 15:59:45.609 | INFO | ------------------------------------------ | 20260505 15:59:45.609 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:59:45.609 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 15:59:45.609 | INFO | Total mem usage: 15.7 GiB | 20260505 15:59:45.609 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 15:59:45.609 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 15:59:45.609 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 15:59:45.609 | INFO | High Water Breakdown: | 20260505 15:59:45.609 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 15:59:45.609 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 15:59:45.609 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 15:59:45.609 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 15:59:45.609 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 15:59:45.609 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 15:59:45.609 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 15:59:45.609 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 15:59:45.609 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 15:59:45.609 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 15:59:45.609 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 15:59:45.609 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 15:59:45.609 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 15:59:45.609 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 15:59:45.609 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 15:59:45.609 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 15:59:45.609 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 15:59:45.609 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 15:59:45.609 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 15:59:45.609 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 15:59:45.609 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 15:59:45.609 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 15:59:45.609 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 15:59:45.609 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 15:59:45.609 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 15:59:45.609 | INFO | ------------------------------------------ | 20260505 15:59:45.609 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 15:59:45.609 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 15:59:45.609 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:59:45.609 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:59:45.609 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.58e+01, 1.36e+01 / 2.35e+00 | 20260505 15:59:45.609 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 1.02e-01, 1.04e-02 / 7.08e-03 | 20260505 15:59:45.609 | INFO | PerfReport: 11.78%, Baseline -- 3.74e+00 / 4.45e+00, 5.84e-01 / 2.19e-01 | 20260505 15:59:45.609 | INFO | PerfReport: 3.30%, Histogram -- 1.05e+00 / 1.49e+00, 1.96e-01 / 1.12e-01 | 20260505 15:59:45.609 | INFO | PerfReport: 15.95%, DME -- 4.05e+01 / 4.51e+01, 4.40e+00 / 2.54e+00 | 20260505 15:59:45.609 | INFO | PerfReport: 56.78%, FrameLabeling -- 1.80e+01 / 1.99e+01, 1.53e+00 / 1.65e+00 | 20260505 15:59:45.609 | INFO | PerfReport: 7.29%, PulseAccumulating -- 2.32e+00 / 2.87e+00, 4.38e-01 / 2.16e-01 | 20260505 15:59:45.609 | INFO | PerfReport: 3.42%, Metrics -- 1.09e+00 / 1.43e+00, 3.03e-01 / 1.87e-01 | 20260505 15:59:45.609 | INFO | PerfReport: 1.20%, PulseSerializing -- 3.80e-01 / 4.38e-01, 6.87e-02 / 3.40e-02 | 20260505 15:59:45.609 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 15:59:45.609 | INFO | PerfReport: Gathered 1536 samples. | 20260505 15:59:45.609 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 15:59:45.609 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.58e+01 / 0.00e+00 | 20260505 15:59:45.609 | INFO | PerfReport: 22.48%, Upload -- 2.26e+01 / 0.00e+00, 7.88e+00 / 0.00e+00 | 20260505 15:59:45.609 | INFO | PerfReport: 21.84%, Download -- 2.19e+01 / 0.00e+00, 7.94e+00 / 0.00e+00 | 20260505 15:59:45.609 | INFO | PerfReport: 31.67%, Compute -- 3.18e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 15:59:45.609 | INFO | PerfReport: 2.18%, LockUpload -- 2.19e+00 / 0.00e+00, 7.81e+00 / 0.00e+00 | 20260505 15:59:45.609 | INFO | PerfReport: 1.42%, LockDownload -- 1.43e+00 / 0.00e+00, 2.94e+00 / 0.00e+00 | 20260505 15:59:45.609 | INFO | PerfReport: 20.40%, LockCompute -- 2.05e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 15:59:45.609 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.69e-02 / 0.00e+00 | 20260505 15:59:45.744 | INFO | Starting a round of Baz IO | 20260505 15:59:48.955 | INFO | Populating RTMetrics report | 20260505 15:59:48.955 | INFO | RTMetrics report populated | 20260505 15:59:48.955 | INFO | Opening tmp RTMetrics json file | 20260505 15:59:48.963 | INFO | Serializing RTMetrics to json | 20260505 15:59:48.966 | INFO | Serialization done, swapping tmp file now... | 20260505 15:59:48.966 | INFO | RTMetrics done | 20260505 15:59:49.053 | INFO | Finished a round of Baz IO: Wrote 2.187 GiB in 3.308 seconds | 20260505 15:59:49.053 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.34% / 0.00% / 77.66% | 20260505 15:59:49.053 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.399196% / 0% / 8.34465e-05% | 20260505 15:59:49.053 | INFO | Baz IO overall: | 20260505 15:59:49.053 | INFO | BytesWritten : 2657483630222 | 20260505 15:59:49.053 | INFO | NumEvents : 1261883106335 | 20260505 15:59:50.742 | INFO | Analyzing chunk frames = [4472832,4473344) | 20260505 15:59:50.742 | INFO | MemInfo: { | 20260505 15:59:50.742 | INFO | "gpu" : " 17 GB", | 20260505 15:59:50.742 | INFO | "huge" : "", | 20260505 15:59:50.742 | INFO | "huge_shared" : " 27 GB", | 20260505 15:59:50.742 | INFO | "malloc" : " 14 GB", | 20260505 15:59:50.742 | INFO | "pinned" : " 29 GB" | 20260505 15:59:50.742 | INFO | } | 20260505 15:59:50.742 | INFO | Perf report for the last 5.13497 seconds: | 20260505 15:59:50.743 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:59:50.743 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:50.743 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:50.743 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:50.743 | INFO | ANALYSIS : 64.1%, 3.0, 99.7ms, 0 | 20260505 15:59:50.743 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:50.743 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260505 15:59:50.743 | INFO | PRE_HQ : 60.8%, 1.8, 56.7ms, 0 | 20260505 15:59:50.743 | INFO | BAZWRITER : 63.8%, 3.9, 127.3ms, 0 | 20260505 15:59:50.743 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 15:59:50.743 | INFO | Uploaded 36GiB to GPU in 2.18721s (16.4593GiB/s) | 20260505 15:59:50.743 | INFO | Downloaded 38GiB to GPU in 2.26285s (16.793GiB/s) | 20260505 15:59:55.889 | INFO | Analyzing chunk frames = [4473344,4473856) | 20260505 15:59:55.889 | INFO | MemInfo: { | 20260505 15:59:55.889 | INFO | "gpu" : " 17 GB", | 20260505 15:59:55.889 | INFO | "huge" : "", | 20260505 15:59:55.889 | INFO | "huge_shared" : " 27 GB", | 20260505 15:59:55.889 | INFO | "malloc" : " 14 GB", | 20260505 15:59:55.889 | INFO | "pinned" : " 29 GB" | 20260505 15:59:55.889 | INFO | } | 20260505 15:59:55.889 | INFO | Perf report for the last 5.14636 seconds: | 20260505 15:59:55.889 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 15:59:55.889 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:55.889 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:55.889 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:55.889 | INFO | ANALYSIS : 63.2%, 3.0, 98.8ms, 0 | 20260505 15:59:55.889 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:55.889 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:55.889 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 15:59:55.889 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:55.889 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 15:59:55.889 | INFO | Uploaded 36GiB to GPU in 2.19349s (16.4122GiB/s) | 20260505 15:59:55.889 | INFO | Downloaded 34GiB to GPU in 2.03716s (16.6899GiB/s) | 20260505 16:00:01.032 | INFO | Analyzing chunk frames = [4473856,4474368) | 20260505 16:00:01.032 | INFO | MemInfo: { | 20260505 16:00:01.032 | INFO | "gpu" : " 17 GB", | 20260505 16:00:01.032 | INFO | "huge" : "", | 20260505 16:00:01.032 | INFO | "huge_shared" : " 27 GB", | 20260505 16:00:01.032 | INFO | "malloc" : " 14 GB", | 20260505 16:00:01.032 | INFO | "pinned" : " 29 GB" | 20260505 16:00:01.032 | INFO | } | 20260505 16:00:01.032 | INFO | Perf report for the last 5.14352 seconds: | 20260505 16:00:01.032 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:00:01.032 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:01.032 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:01.032 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:01.032 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 16:00:01.032 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:01.032 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:01.032 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 16:00:01.032 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:01.032 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:01.033 | INFO | Uploaded 36GiB to GPU in 2.16631s (16.6181GiB/s) | 20260505 16:00:01.033 | INFO | Downloaded 34GiB to GPU in 2.03038s (16.7456GiB/s) | 20260505 16:00:06.169 | INFO | Analyzing chunk frames = [4474368,4474880) | 20260505 16:00:06.169 | INFO | MemInfo: { | 20260505 16:00:06.169 | INFO | "gpu" : " 17 GB", | 20260505 16:00:06.169 | INFO | "huge" : "", | 20260505 16:00:06.169 | INFO | "huge_shared" : " 27 GB", | 20260505 16:00:06.169 | INFO | "malloc" : " 14 GB", | 20260505 16:00:06.169 | INFO | "pinned" : " 29 GB" | 20260505 16:00:06.169 | INFO | } | 20260505 16:00:06.169 | INFO | Perf report for the last 5.13652 seconds: | 20260505 16:00:06.169 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:00:06.169 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:06.169 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:06.169 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:06.169 | INFO | ANALYSIS : 62.6%, 3.0, 97.2ms, 0 | 20260505 16:00:06.169 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:06.169 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:06.169 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:00:06.169 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:06.169 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:06.169 | INFO | Uploaded 36GiB to GPU in 2.16966s (16.5925GiB/s) | 20260505 16:00:06.169 | INFO | Downloaded 34GiB to GPU in 2.13977s (15.8896GiB/s) | 20260505 16:00:11.300 | INFO | Analyzing chunk frames = [4474880,4475392) | 20260505 16:00:11.300 | INFO | MemInfo: { | 20260505 16:00:11.300 | INFO | "gpu" : " 17 GB", | 20260505 16:00:11.300 | INFO | "huge" : "", | 20260505 16:00:11.300 | INFO | "huge_shared" : " 27 GB", | 20260505 16:00:11.301 | INFO | "malloc" : " 14 GB", | 20260505 16:00:11.301 | INFO | "pinned" : " 29 GB" | 20260505 16:00:11.301 | INFO | } | 20260505 16:00:11.301 | INFO | Perf report for the last 5.13172 seconds: | 20260505 16:00:11.301 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:00:11.301 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:11.301 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:11.301 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:11.301 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260505 16:00:11.301 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:11.301 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:11.301 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 16:00:11.301 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:11.301 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:11.301 | INFO | Uploaded 36GiB to GPU in 2.13832s (16.8356GiB/s) | 20260505 16:00:11.301 | INFO | Downloaded 34GiB to GPU in 2.0572s (16.5273GiB/s) | 20260505 16:00:16.441 | INFO | Analyzing chunk frames = [4475392,4475904) | 20260505 16:00:16.441 | INFO | MemInfo: { | 20260505 16:00:16.441 | INFO | "gpu" : " 17 GB", | 20260505 16:00:16.441 | INFO | "huge" : "", | 20260505 16:00:16.441 | INFO | "huge_shared" : " 27 GB", | 20260505 16:00:16.441 | INFO | "malloc" : " 14 GB", | 20260505 16:00:16.441 | INFO | "pinned" : " 29 GB" | 20260505 16:00:16.442 | INFO | } | 20260505 16:00:16.442 | INFO | Perf report for the last 5.14098 seconds: | 20260505 16:00:16.442 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:00:16.442 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:16.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:16.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:16.442 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 16:00:16.442 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:16.442 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:16.442 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:00:16.442 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:16.442 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:16.442 | INFO | Uploaded 36GiB to GPU in 2.20809s (16.3037GiB/s) | 20260505 16:00:16.442 | INFO | Downloaded 34GiB to GPU in 2.11425s (16.0814GiB/s) | 20260505 16:00:21.585 | INFO | Analyzing chunk frames = [4475904,4476416) | 20260505 16:00:21.585 | INFO | MemInfo: { | 20260505 16:00:21.585 | INFO | "gpu" : " 17 GB", | 20260505 16:00:21.585 | INFO | "huge" : "", | 20260505 16:00:21.585 | INFO | "huge_shared" : " 27 GB", | 20260505 16:00:21.585 | INFO | "malloc" : " 14 GB", | 20260505 16:00:21.585 | INFO | "pinned" : " 29 GB" | 20260505 16:00:21.585 | INFO | } | 20260505 16:00:21.585 | INFO | Perf report for the last 5.14384 seconds: | 20260505 16:00:21.585 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:00:21.585 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:21.585 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:21.585 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:21.586 | INFO | ANALYSIS : 63.2%, 3.0, 98.8ms, 0 | 20260505 16:00:21.586 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:21.586 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:21.586 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 16:00:21.586 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:21.586 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:21.586 | INFO | Uploaded 36GiB to GPU in 2.11942s (16.9858GiB/s) | 20260505 16:00:21.586 | INFO | Downloaded 34GiB to GPU in 2.17461s (15.635GiB/s) | 20260505 16:00:26.730 | INFO | Analyzing chunk frames = [4476416,4476928) | 20260505 16:00:26.730 | INFO | MemInfo: { | 20260505 16:00:26.730 | INFO | "gpu" : " 17 GB", | 20260505 16:00:26.730 | INFO | "huge" : "", | 20260505 16:00:26.730 | INFO | "huge_shared" : " 27 GB", | 20260505 16:00:26.730 | INFO | "malloc" : " 14 GB", | 20260505 16:00:26.730 | INFO | "pinned" : " 29 GB" | 20260505 16:00:26.730 | INFO | } | 20260505 16:00:26.730 | INFO | Perf report for the last 5.14456 seconds: | 20260505 16:00:26.730 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:00:26.730 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:26.730 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:26.730 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:26.730 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 16:00:26.730 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:26.730 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:26.730 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:00:26.730 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:26.730 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:26.730 | INFO | Uploaded 36GiB to GPU in 2.15506s (16.7049GiB/s) | 20260505 16:00:26.730 | INFO | Downloaded 34GiB to GPU in 2.02972s (16.751GiB/s) | 20260505 16:00:26.864 | INFO | Starting a round of Baz IO | 20260505 16:00:30.066 | INFO | Populating RTMetrics report | 20260505 16:00:30.066 | INFO | RTMetrics report populated | 20260505 16:00:30.066 | INFO | Opening tmp RTMetrics json file | 20260505 16:00:30.076 | INFO | Serializing RTMetrics to json | 20260505 16:00:30.077 | INFO | Serialization done, swapping tmp file now... | 20260505 16:00:30.078 | INFO | RTMetrics done | 20260505 16:00:30.171 | INFO | Finished a round of Baz IO: Wrote 2.188 GiB in 3.307 seconds | 20260505 16:00:30.171 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.34% / 0.00% / 77.66% | 20260505 16:00:30.171 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.398429% / 3.97364e-06% / 8.34465e-05% | 20260505 16:00:30.171 | INFO | Baz IO overall: | 20260505 16:00:30.171 | INFO | BytesWritten : 2659833321095 | 20260505 16:00:30.171 | INFO | NumEvents : 1262986208178 | 20260505 16:00:31.865 | INFO | Analyzing chunk frames = [4476928,4477440) | 20260505 16:00:31.865 | INFO | MemInfo: { | 20260505 16:00:31.865 | INFO | "gpu" : " 17 GB", | 20260505 16:00:31.865 | INFO | "huge" : "", | 20260505 16:00:31.865 | INFO | "huge_shared" : " 27 GB", | 20260505 16:00:31.865 | INFO | "malloc" : " 14 GB", | 20260505 16:00:31.865 | INFO | "pinned" : " 29 GB" | 20260505 16:00:31.865 | INFO | } | 20260505 16:00:31.865 | INFO | Perf report for the last 5.13516 seconds: | 20260505 16:00:31.865 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:00:31.865 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:31.865 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:31.865 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:31.865 | INFO | ANALYSIS : 63.9%, 3.0, 99.4ms, 0 | 20260505 16:00:31.865 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:31.865 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260505 16:00:31.865 | INFO | PRE_HQ : 60.8%, 1.8, 56.6ms, 0 | 20260505 16:00:31.865 | INFO | BAZWRITER : 63.7%, 3.9, 127.2ms, 0 | 20260505 16:00:31.865 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260505 16:00:31.866 | INFO | Uploaded 36GiB to GPU in 2.16652s (16.6165GiB/s) | 20260505 16:00:31.866 | INFO | Downloaded 38GiB to GPU in 2.26289s (16.7927GiB/s) | 20260505 16:00:36.999 | INFO | Analyzing chunk frames = [4477440,4477952) | 20260505 16:00:36.999 | INFO | MemInfo: { | 20260505 16:00:36.999 | INFO | "gpu" : " 17 GB", | 20260505 16:00:36.999 | INFO | "huge" : "", | 20260505 16:00:36.999 | INFO | "huge_shared" : " 27 GB", | 20260505 16:00:36.999 | INFO | "malloc" : " 14 GB", | 20260505 16:00:36.999 | INFO | "pinned" : " 29 GB" | 20260505 16:00:36.999 | INFO | } | 20260505 16:00:36.999 | INFO | Perf report for the last 5.13365 seconds: | 20260505 16:00:36.999 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:00:36.999 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:36.999 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:36.999 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:36.999 | INFO | ANALYSIS : 63.5%, 3.0, 99.0ms, 0 | 20260505 16:00:36.999 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:36.999 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:36.999 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:00:36.999 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:36.999 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:36.999 | INFO | Uploaded 36GiB to GPU in 2.20598s (16.3192GiB/s) | 20260505 16:00:36.999 | INFO | Downloaded 34GiB to GPU in 2.06648s (16.4531GiB/s) | 20260505 16:00:42.148 | INFO | Analyzing chunk frames = [4477952,4478464) | 20260505 16:00:42.148 | INFO | MemInfo: { | 20260505 16:00:42.148 | INFO | "gpu" : " 17 GB", | 20260505 16:00:42.148 | INFO | "huge" : "", | 20260505 16:00:42.148 | INFO | "huge_shared" : " 27 GB", | 20260505 16:00:42.148 | INFO | "malloc" : " 14 GB", | 20260505 16:00:42.148 | INFO | "pinned" : " 29 GB" | 20260505 16:00:42.148 | INFO | } | 20260505 16:00:42.148 | INFO | Perf report for the last 5.14934 seconds: | 20260505 16:00:42.148 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:00:42.148 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:42.148 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:42.148 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:42.148 | INFO | ANALYSIS : 62.9%, 3.0, 98.0ms, 0 | 20260505 16:00:42.148 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:42.148 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:42.148 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 16:00:42.148 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:42.148 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:42.148 | INFO | Uploaded 36GiB to GPU in 2.20748s (16.3082GiB/s) | 20260505 16:00:42.149 | INFO | Downloaded 34GiB to GPU in 2.05926s (16.5108GiB/s) | 20260505 16:00:47.285 | INFO | Analyzing chunk frames = [4478464,4478976) | 20260505 16:00:47.285 | INFO | MemInfo: { | 20260505 16:00:47.285 | INFO | "gpu" : " 17 GB", | 20260505 16:00:47.285 | INFO | "huge" : "", | 20260505 16:00:47.285 | INFO | "huge_shared" : " 27 GB", | 20260505 16:00:47.285 | INFO | "malloc" : " 14 GB", | 20260505 16:00:47.285 | INFO | "pinned" : " 29 GB" | 20260505 16:00:47.285 | INFO | } | 20260505 16:00:47.285 | INFO | Perf report for the last 5.13677 seconds: | 20260505 16:00:47.285 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:00:47.285 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:47.285 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:47.285 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:47.285 | INFO | ANALYSIS : 62.7%, 3.0, 97.4ms, 0 | 20260505 16:00:47.285 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:47.285 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:47.285 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:00:47.285 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:47.285 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:47.285 | INFO | Uploaded 36GiB to GPU in 2.21348s (16.264GiB/s) | 20260505 16:00:47.285 | INFO | Downloaded 34GiB to GPU in 2.14127s (15.8784GiB/s) | 20260505 16:00:52.417 | INFO | Analyzing chunk frames = [4478976,4479488) | 20260505 16:00:52.417 | INFO | MemInfo: { | 20260505 16:00:52.417 | INFO | "gpu" : " 17 GB", | 20260505 16:00:52.417 | INFO | "huge" : "", | 20260505 16:00:52.417 | INFO | "huge_shared" : " 27 GB", | 20260505 16:00:52.417 | INFO | "malloc" : " 14 GB", | 20260505 16:00:52.417 | INFO | "pinned" : " 29 GB" | 20260505 16:00:52.417 | INFO | } | 20260505 16:00:52.417 | INFO | Perf report for the last 5.13181 seconds: | 20260505 16:00:52.417 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:00:52.417 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:52.417 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:52.417 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:52.417 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260505 16:00:52.417 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:52.417 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:52.417 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:00:52.417 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:52.417 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:52.417 | INFO | Uploaded 36GiB to GPU in 2.16021s (16.665GiB/s) | 20260505 16:00:52.417 | INFO | Downloaded 34GiB to GPU in 2.05887s (16.5139GiB/s) | 20260505 16:00:57.556 | INFO | Analyzing chunk frames = [4479488,4480000) | 20260505 16:00:57.556 | INFO | MemInfo: { | 20260505 16:00:57.556 | INFO | "gpu" : " 17 GB", | 20260505 16:00:57.556 | INFO | "huge" : "", | 20260505 16:00:57.556 | INFO | "huge_shared" : " 27 GB", | 20260505 16:00:57.556 | INFO | "malloc" : " 14 GB", | 20260505 16:00:57.556 | INFO | "pinned" : " 29 GB" | 20260505 16:00:57.557 | INFO | } | 20260505 16:00:57.557 | INFO | Perf report for the last 5.13982 seconds: | 20260505 16:00:57.557 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:00:57.557 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:57.557 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:57.557 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:57.557 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 16:00:57.557 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:57.557 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:57.557 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 16:00:57.557 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:57.557 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:00:57.557 | INFO | Uploaded 36GiB to GPU in 2.16589s (16.6213GiB/s) | 20260505 16:00:57.557 | INFO | Downloaded 34GiB to GPU in 2.04012s (16.6657GiB/s) | 20260505 16:01:02.695 | INFO | Analyzing chunk frames = [4480000,4480512) | 20260505 16:01:02.695 | INFO | MemInfo: { | 20260505 16:01:02.695 | INFO | "gpu" : " 17 GB", | 20260505 16:01:02.695 | INFO | "huge" : "", | 20260505 16:01:02.695 | INFO | "huge_shared" : " 27 GB", | 20260505 16:01:02.695 | INFO | "malloc" : " 14 GB", | 20260505 16:01:02.695 | INFO | "pinned" : " 29 GB" | 20260505 16:01:02.695 | INFO | } | 20260505 16:01:02.695 | INFO | Perf report for the last 5.13847 seconds: | 20260505 16:01:02.695 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:01:02.695 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:02.695 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:02.695 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:02.695 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 16:01:02.695 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:02.695 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:02.695 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 16:01:02.695 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:02.695 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:02.695 | INFO | Uploaded 36GiB to GPU in 2.21237s (16.2721GiB/s) | 20260505 16:01:02.695 | INFO | Downloaded 34GiB to GPU in 2.15515s (15.7762GiB/s) | 20260505 16:01:07.838 | INFO | Analyzing chunk frames = [4480512,4481024) | 20260505 16:01:07.838 | INFO | MemInfo: { | 20260505 16:01:07.838 | INFO | "gpu" : " 17 GB", | 20260505 16:01:07.838 | INFO | "huge" : "", | 20260505 16:01:07.838 | INFO | "huge_shared" : " 27 GB", | 20260505 16:01:07.838 | INFO | "malloc" : " 14 GB", | 20260505 16:01:07.839 | INFO | "pinned" : " 29 GB" | 20260505 16:01:07.839 | INFO | } | 20260505 16:01:07.839 | INFO | Perf report for the last 5.14348 seconds: | 20260505 16:01:07.839 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:01:07.839 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:07.839 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:07.839 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:07.839 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 16:01:07.839 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:07.839 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:07.839 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 16:01:07.839 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:07.839 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:07.839 | INFO | Uploaded 36GiB to GPU in 2.15915s (16.6732GiB/s) | 20260505 16:01:07.839 | INFO | Downloaded 34GiB to GPU in 2.03366s (16.7187GiB/s) | 20260505 16:01:07.839 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 16:01:07.839 | INFO | Total mem usage: 15.3 GiB | 20260505 16:01:07.839 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 16:01:07.839 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.84%) | 20260505 16:01:07.839 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 16:01:07.839 | INFO | High Water Breakdown: | 20260505 16:01:07.839 | INFO | 61% : 8.68 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 16:01:07.839 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 16:01:07.839 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 16:01:07.839 | INFO | ------------------------------------------ | 20260505 16:01:07.839 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:01:07.839 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 16:01:07.839 | INFO | Total mem usage: 26.5 GiB | 20260505 16:01:07.839 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 16:01:07.839 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 16:01:07.839 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 16:01:07.839 | INFO | High Water Breakdown: | 20260505 16:01:07.839 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:01:07.839 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:01:07.839 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:01:07.839 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:01:07.839 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:01:07.839 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:01:07.839 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 16:01:07.839 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 16:01:07.839 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:01:07.839 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 16:01:07.839 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 16:01:07.839 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 16:01:07.839 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 16:01:07.839 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 16:01:07.839 | INFO | ------------------------------------------ | 20260505 16:01:07.839 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:01:07.839 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 16:01:07.839 | INFO | Total mem usage: 25.6 GiB | 20260505 16:01:07.839 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 16:01:07.839 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 16:01:07.839 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 16:01:07.840 | INFO | High Water Breakdown: | 20260505 16:01:07.840 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 16:01:07.840 | INFO | ------------------------------------------ | 20260505 16:01:07.840 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:01:07.840 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 16:01:07.840 | INFO | Total mem usage: 15.7 GiB | 20260505 16:01:07.840 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 16:01:07.840 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 16:01:07.840 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 16:01:07.840 | INFO | High Water Breakdown: | 20260505 16:01:07.840 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 16:01:07.840 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 16:01:07.840 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 16:01:07.840 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 16:01:07.840 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 16:01:07.840 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 16:01:07.840 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 16:01:07.840 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 16:01:07.840 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 16:01:07.840 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:01:07.840 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 16:01:07.840 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:01:07.840 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 16:01:07.840 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 16:01:07.840 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 16:01:07.840 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 16:01:07.840 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:01:07.840 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 16:01:07.840 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:01:07.840 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:01:07.840 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:01:07.840 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 16:01:07.840 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 16:01:07.840 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:01:07.840 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 16:01:07.840 | INFO | ------------------------------------------ | 20260505 16:01:07.840 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:01:07.840 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 16:01:07.840 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:01:07.840 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:01:07.840 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.51e+01, 1.36e+01 / 1.81e+00 | 20260505 16:01:07.840 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.00e-01, 1.03e-02 / 3.96e-03 | 20260505 16:01:07.840 | INFO | PerfReport: 11.77%, Baseline -- 3.73e+00 / 4.46e+00, 5.71e-01 / 2.36e-01 | 20260505 16:01:07.840 | INFO | PerfReport: 3.27%, Histogram -- 1.04e+00 / 1.51e+00, 1.92e-01 / 1.01e-01 | 20260505 16:01:07.840 | INFO | PerfReport: 15.93%, DME -- 4.04e+01 / 4.48e+01, 4.22e+00 / 1.65e+00 | 20260505 16:01:07.840 | INFO | PerfReport: 56.73%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.50e+00 / 1.53e+00 | 20260505 16:01:07.840 | INFO | PerfReport: 7.37%, PulseAccumulating -- 2.34e+00 / 2.90e+00, 4.51e-01 / 1.81e-01 | 20260505 16:01:07.840 | INFO | PerfReport: 3.44%, Metrics -- 1.09e+00 / 1.41e+00, 3.00e-01 / 1.44e-01 | 20260505 16:01:07.840 | INFO | PerfReport: 1.20%, PulseSerializing -- 3.80e-01 / 4.38e-01, 6.76e-02 / 3.86e-02 | 20260505 16:01:07.840 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 16:01:07.840 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:01:07.840 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:01:07.840 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.57e+01 / 0.00e+00 | 20260505 16:01:07.840 | INFO | PerfReport: 22.59%, Upload -- 2.27e+01 / 0.00e+00, 7.95e+00 / 0.00e+00 | 20260505 16:01:07.840 | INFO | PerfReport: 21.84%, Download -- 2.19e+01 / 0.00e+00, 7.91e+00 / 0.00e+00 | 20260505 16:01:07.840 | INFO | PerfReport: 31.59%, Compute -- 3.17e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 16:01:07.840 | INFO | PerfReport: 2.33%, LockUpload -- 2.34e+00 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260505 16:01:07.840 | INFO | PerfReport: 1.43%, LockDownload -- 1.43e+00 / 0.00e+00, 2.95e+00 / 0.00e+00 | 20260505 16:01:07.840 | INFO | PerfReport: 20.21%, LockCompute -- 2.03e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 16:01:07.840 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.15e-02 / 0.00e+00 | 20260505 16:01:07.971 | INFO | Starting a round of Baz IO | 20260505 16:01:11.180 | INFO | Populating RTMetrics report | 20260505 16:01:11.181 | INFO | RTMetrics report populated | 20260505 16:01:11.181 | INFO | Opening tmp RTMetrics json file | 20260505 16:01:11.194 | INFO | Serializing RTMetrics to json | 20260505 16:01:11.195 | INFO | Serialization done, swapping tmp file now... | 20260505 16:01:11.196 | INFO | RTMetrics done | 20260505 16:01:11.283 | INFO | Finished a round of Baz IO: Wrote 2.191 GiB in 3.312 seconds | 20260505 16:01:11.283 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.34% / 0.00% / 77.66% | 20260505 16:01:11.283 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.398894% / 0% / 6.35783e-05% | 20260505 16:01:11.284 | INFO | Baz IO overall: | 20260505 16:01:11.284 | INFO | BytesWritten : 2662185626530 | 20260505 16:01:11.284 | INFO | NumEvents : 1264091014903 | 20260505 16:01:12.982 | INFO | Analyzing chunk frames = [4481024,4481536) | 20260505 16:01:12.982 | INFO | MemInfo: { | 20260505 16:01:12.982 | INFO | "gpu" : " 17 GB", | 20260505 16:01:12.982 | INFO | "huge" : "", | 20260505 16:01:12.982 | INFO | "huge_shared" : " 27 GB", | 20260505 16:01:12.983 | INFO | "malloc" : " 14 GB", | 20260505 16:01:12.983 | INFO | "pinned" : " 29 GB" | 20260505 16:01:12.983 | INFO | } | 20260505 16:01:12.983 | INFO | Perf report for the last 5.14393 seconds: | 20260505 16:01:12.983 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:01:12.983 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:12.983 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:12.983 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:12.983 | INFO | ANALYSIS : 63.9%, 3.0, 99.5ms, 0 | 20260505 16:01:12.983 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:12.983 | INFO | RT_METRICS : 44.9%, 1.0, 23.6ms, 0 | 20260505 16:01:12.983 | INFO | PRE_HQ : 60.7%, 1.8, 56.6ms, 0 | 20260505 16:01:12.983 | INFO | BAZWRITER : 63.7%, 3.9, 128.7ms, 0 | 20260505 16:01:12.983 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260505 16:01:12.983 | INFO | Uploaded 36GiB to GPU in 2.19948s (16.3675GiB/s) | 20260505 16:01:12.983 | INFO | Downloaded 38GiB to GPU in 2.2658s (16.7711GiB/s) | 20260505 16:01:18.117 | INFO | Analyzing chunk frames = [4481536,4482048) | 20260505 16:01:18.117 | INFO | MemInfo: { | 20260505 16:01:18.117 | INFO | "gpu" : " 17 GB", | 20260505 16:01:18.117 | INFO | "huge" : "", | 20260505 16:01:18.117 | INFO | "huge_shared" : " 27 GB", | 20260505 16:01:18.117 | INFO | "malloc" : " 14 GB", | 20260505 16:01:18.117 | INFO | "pinned" : " 29 GB" | 20260505 16:01:18.117 | INFO | } | 20260505 16:01:18.117 | INFO | Perf report for the last 5.13402 seconds: | 20260505 16:01:18.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:01:18.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:18.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:18.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:18.117 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260505 16:01:18.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:18.117 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:18.117 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:01:18.117 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:18.117 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:18.117 | INFO | Uploaded 36GiB to GPU in 2.11877s (16.991GiB/s) | 20260505 16:01:18.117 | INFO | Downloaded 34GiB to GPU in 2.03683s (16.6926GiB/s) | 20260505 16:01:23.248 | INFO | Analyzing chunk frames = [4482048,4482560) | 20260505 16:01:23.248 | INFO | MemInfo: { | 20260505 16:01:23.248 | INFO | "gpu" : " 17 GB", | 20260505 16:01:23.248 | INFO | "huge" : "", | 20260505 16:01:23.248 | INFO | "huge_shared" : " 27 GB", | 20260505 16:01:23.248 | INFO | "malloc" : " 14 GB", | 20260505 16:01:23.248 | INFO | "pinned" : " 29 GB" | 20260505 16:01:23.248 | INFO | } | 20260505 16:01:23.248 | INFO | Perf report for the last 5.13143 seconds: | 20260505 16:01:23.248 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:01:23.248 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:23.248 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:23.248 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:23.248 | INFO | ANALYSIS : 63.1%, 3.0, 98.1ms, 0 | 20260505 16:01:23.248 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:23.248 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:23.248 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 16:01:23.248 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:23.248 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:23.248 | INFO | Uploaded 36GiB to GPU in 2.19123s (16.4291GiB/s) | 20260505 16:01:23.248 | INFO | Downloaded 34GiB to GPU in 2.06245s (16.4852GiB/s) | 20260505 16:01:28.398 | INFO | Analyzing chunk frames = [4482560,4483072) | 20260505 16:01:28.398 | INFO | MemInfo: { | 20260505 16:01:28.398 | INFO | "gpu" : " 17 GB", | 20260505 16:01:28.398 | INFO | "huge" : "", | 20260505 16:01:28.398 | INFO | "huge_shared" : " 27 GB", | 20260505 16:01:28.398 | INFO | "malloc" : " 14 GB", | 20260505 16:01:28.398 | INFO | "pinned" : " 29 GB" | 20260505 16:01:28.399 | INFO | } | 20260505 16:01:28.399 | INFO | Perf report for the last 5.15064 seconds: | 20260505 16:01:28.399 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:01:28.399 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:28.399 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:28.399 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:28.399 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260505 16:01:28.399 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:28.399 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:28.399 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 16:01:28.399 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:28.399 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:28.399 | INFO | Uploaded 36GiB to GPU in 2.16838s (16.6022GiB/s) | 20260505 16:01:28.399 | INFO | Downloaded 34GiB to GPU in 2.13539s (15.9221GiB/s) | 20260505 16:01:33.539 | INFO | Analyzing chunk frames = [4483072,4483584) | 20260505 16:01:33.539 | INFO | MemInfo: { | 20260505 16:01:33.539 | INFO | "gpu" : " 17 GB", | 20260505 16:01:33.539 | INFO | "huge" : "", | 20260505 16:01:33.539 | INFO | "huge_shared" : " 27 GB", | 20260505 16:01:33.539 | INFO | "malloc" : " 14 GB", | 20260505 16:01:33.540 | INFO | "pinned" : " 29 GB" | 20260505 16:01:33.540 | INFO | } | 20260505 16:01:33.540 | INFO | Perf report for the last 5.14089 seconds: | 20260505 16:01:33.540 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:01:33.540 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:33.540 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:33.540 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:33.540 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260505 16:01:33.540 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:33.540 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:33.540 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 16:01:33.540 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:33.540 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:33.540 | INFO | Uploaded 36GiB to GPU in 2.1562s (16.696GiB/s) | 20260505 16:01:33.540 | INFO | Downloaded 34GiB to GPU in 2.06266s (16.4836GiB/s) | 20260505 16:01:38.675 | INFO | Analyzing chunk frames = [4483584,4484096) | 20260505 16:01:38.675 | INFO | MemInfo: { | 20260505 16:01:38.675 | INFO | "gpu" : " 17 GB", | 20260505 16:01:38.675 | INFO | "huge" : "", | 20260505 16:01:38.675 | INFO | "huge_shared" : " 27 GB", | 20260505 16:01:38.675 | INFO | "malloc" : " 14 GB", | 20260505 16:01:38.675 | INFO | "pinned" : " 29 GB" | 20260505 16:01:38.675 | INFO | } | 20260505 16:01:38.675 | INFO | Perf report for the last 5.13577 seconds: | 20260505 16:01:38.675 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:01:38.675 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:38.675 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:38.675 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:38.675 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260505 16:01:38.675 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:38.675 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:38.675 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:01:38.676 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:38.676 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:38.676 | INFO | Uploaded 36GiB to GPU in 2.13567s (16.8565GiB/s) | 20260505 16:01:38.676 | INFO | Downloaded 34GiB to GPU in 2.04147s (16.6547GiB/s) | 20260505 16:01:43.816 | INFO | Analyzing chunk frames = [4484096,4484608) | 20260505 16:01:43.816 | INFO | MemInfo: { | 20260505 16:01:43.816 | INFO | "gpu" : " 17 GB", | 20260505 16:01:43.816 | INFO | "huge" : "", | 20260505 16:01:43.816 | INFO | "huge_shared" : " 27 GB", | 20260505 16:01:43.816 | INFO | "malloc" : " 14 GB", | 20260505 16:01:43.816 | INFO | "pinned" : " 29 GB" | 20260505 16:01:43.816 | INFO | } | 20260505 16:01:43.816 | INFO | Perf report for the last 5.14042 seconds: | 20260505 16:01:43.816 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:01:43.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:43.816 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:43.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:43.816 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 16:01:43.816 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:43.816 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:43.816 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 16:01:43.816 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:43.816 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:43.816 | INFO | Uploaded 36GiB to GPU in 2.1198s (16.9827GiB/s) | 20260505 16:01:43.816 | INFO | Downloaded 34GiB to GPU in 2.17276s (15.6483GiB/s) | 20260505 16:01:48.953 | INFO | Analyzing chunk frames = [4484608,4485120) | 20260505 16:01:48.953 | INFO | MemInfo: { | 20260505 16:01:48.953 | INFO | "gpu" : " 17 GB", | 20260505 16:01:48.953 | INFO | "huge" : "", | 20260505 16:01:48.953 | INFO | "huge_shared" : " 27 GB", | 20260505 16:01:48.953 | INFO | "malloc" : " 14 GB", | 20260505 16:01:48.953 | INFO | "pinned" : " 29 GB" | 20260505 16:01:48.953 | INFO | } | 20260505 16:01:48.953 | INFO | Perf report for the last 5.13757 seconds: | 20260505 16:01:48.953 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:01:48.953 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:48.953 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:48.953 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:48.953 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 16:01:48.953 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:48.954 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:48.954 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 16:01:48.954 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:48.954 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:48.954 | INFO | Uploaded 36GiB to GPU in 2.22606s (16.1721GiB/s) | 20260505 16:01:48.954 | INFO | Downloaded 34GiB to GPU in 2.10683s (16.138GiB/s) | 20260505 16:01:49.083 | INFO | Starting a round of Baz IO | 20260505 16:01:52.295 | INFO | Populating RTMetrics report | 20260505 16:01:52.296 | INFO | RTMetrics report populated | 20260505 16:01:52.296 | INFO | Opening tmp RTMetrics json file | 20260505 16:01:52.310 | INFO | Serializing RTMetrics to json | 20260505 16:01:52.311 | INFO | Serialization done, swapping tmp file now... | 20260505 16:01:52.312 | INFO | RTMetrics done | 20260505 16:01:52.395 | INFO | Finished a round of Baz IO: Wrote 2.19 GiB in 3.312 seconds | 20260505 16:01:52.395 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.33% / 0.00% / 77.66% | 20260505 16:01:52.395 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.398521% / 0% / 6.75519e-05% | 20260505 16:01:52.395 | INFO | Baz IO overall: | 20260505 16:01:52.396 | INFO | BytesWritten : 2664536906238 | 20260505 16:01:52.396 | INFO | NumEvents : 1265195443127 | 20260505 16:01:54.095 | INFO | Analyzing chunk frames = [4485120,4485632) | 20260505 16:01:54.095 | INFO | MemInfo: { | 20260505 16:01:54.095 | INFO | "gpu" : " 17 GB", | 20260505 16:01:54.095 | INFO | "huge" : "", | 20260505 16:01:54.095 | INFO | "huge_shared" : " 27 GB", | 20260505 16:01:54.095 | INFO | "malloc" : " 14 GB", | 20260505 16:01:54.096 | INFO | "pinned" : " 29 GB" | 20260505 16:01:54.096 | INFO | } | 20260505 16:01:54.096 | INFO | Perf report for the last 5.14214 seconds: | 20260505 16:01:54.096 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:01:54.096 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:54.096 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:54.096 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:54.096 | INFO | ANALYSIS : 63.9%, 3.0, 99.5ms, 0 | 20260505 16:01:54.096 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:54.096 | INFO | RT_METRICS : 44.9%, 1.0, 23.6ms, 0 | 20260505 16:01:54.096 | INFO | PRE_HQ : 60.8%, 1.8, 56.7ms, 0 | 20260505 16:01:54.096 | INFO | BAZWRITER : 63.7%, 3.9, 128.2ms, 0 | 20260505 16:01:54.096 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 16:01:54.096 | INFO | Uploaded 36GiB to GPU in 2.19731s (16.3836GiB/s) | 20260505 16:01:54.096 | INFO | Downloaded 38GiB to GPU in 2.2601s (16.8134GiB/s) | 20260505 16:01:59.238 | INFO | Analyzing chunk frames = [4485632,4486144) | 20260505 16:01:59.238 | INFO | MemInfo: { | 20260505 16:01:59.238 | INFO | "gpu" : " 17 GB", | 20260505 16:01:59.238 | INFO | "huge" : "", | 20260505 16:01:59.238 | INFO | "huge_shared" : " 27 GB", | 20260505 16:01:59.238 | INFO | "malloc" : " 14 GB", | 20260505 16:01:59.238 | INFO | "pinned" : " 29 GB" | 20260505 16:01:59.238 | INFO | } | 20260505 16:01:59.238 | INFO | Perf report for the last 5.14275 seconds: | 20260505 16:01:59.238 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:01:59.238 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:59.238 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:59.238 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:59.238 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 16:01:59.238 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:59.238 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:59.238 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:01:59.238 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:59.238 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:01:59.238 | INFO | Uploaded 36GiB to GPU in 2.11711s (17.0043GiB/s) | 20260505 16:01:59.239 | INFO | Downloaded 34GiB to GPU in 2.02334s (16.8039GiB/s) | 20260505 16:02:04.367 | INFO | Analyzing chunk frames = [4486144,4486656) | 20260505 16:02:04.367 | INFO | MemInfo: { | 20260505 16:02:04.367 | INFO | "gpu" : " 17 GB", | 20260505 16:02:04.367 | INFO | "huge" : "", | 20260505 16:02:04.367 | INFO | "huge_shared" : " 27 GB", | 20260505 16:02:04.367 | INFO | "malloc" : " 14 GB", | 20260505 16:02:04.367 | INFO | "pinned" : " 29 GB" | 20260505 16:02:04.367 | INFO | } | 20260505 16:02:04.367 | INFO | Perf report for the last 5.12891 seconds: | 20260505 16:02:04.367 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:02:04.367 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:04.367 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:04.367 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:04.367 | INFO | ANALYSIS : 62.8%, 3.0, 97.6ms, 0 | 20260505 16:02:04.367 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:04.367 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:04.367 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 16:02:04.367 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:04.367 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:04.368 | INFO | Uploaded 36GiB to GPU in 2.19101s (16.4308GiB/s) | 20260505 16:02:04.368 | INFO | Downloaded 34GiB to GPU in 2.04225s (16.6483GiB/s) | 20260505 16:02:09.503 | INFO | Analyzing chunk frames = [4486656,4487168) | 20260505 16:02:09.503 | INFO | MemInfo: { | 20260505 16:02:09.503 | INFO | "gpu" : " 17 GB", | 20260505 16:02:09.503 | INFO | "huge" : "", | 20260505 16:02:09.503 | INFO | "huge_shared" : " 27 GB", | 20260505 16:02:09.503 | INFO | "malloc" : " 14 GB", | 20260505 16:02:09.503 | INFO | "pinned" : " 29 GB" | 20260505 16:02:09.503 | INFO | } | 20260505 16:02:09.503 | INFO | Perf report for the last 5.13591 seconds: | 20260505 16:02:09.503 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:02:09.503 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:09.503 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:09.503 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:09.503 | INFO | ANALYSIS : 62.7%, 3.0, 97.3ms, 0 | 20260505 16:02:09.503 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:09.503 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:09.503 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:02:09.503 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:09.503 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:09.503 | INFO | Uploaded 36GiB to GPU in 2.18653s (16.4644GiB/s) | 20260505 16:02:09.503 | INFO | Downloaded 34GiB to GPU in 2.13364s (15.9352GiB/s) | 20260505 16:02:14.653 | INFO | Analyzing chunk frames = [4487168,4487680) | 20260505 16:02:14.653 | INFO | MemInfo: { | 20260505 16:02:14.653 | INFO | "gpu" : " 17 GB", | 20260505 16:02:14.653 | INFO | "huge" : "", | 20260505 16:02:14.653 | INFO | "huge_shared" : " 27 GB", | 20260505 16:02:14.653 | INFO | "malloc" : " 14 GB", | 20260505 16:02:14.653 | INFO | "pinned" : " 29 GB" | 20260505 16:02:14.653 | INFO | } | 20260505 16:02:14.653 | INFO | Perf report for the last 5.15027 seconds: | 20260505 16:02:14.653 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:02:14.653 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:14.653 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:14.653 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:14.654 | INFO | ANALYSIS : 62.6%, 3.0, 97.9ms, 0 | 20260505 16:02:14.654 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:14.654 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:14.654 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 16:02:14.654 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:14.654 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:14.654 | INFO | Uploaded 36GiB to GPU in 2.18089s (16.507GiB/s) | 20260505 16:02:14.654 | INFO | Downloaded 34GiB to GPU in 2.13116s (15.9537GiB/s) | 20260505 16:02:19.799 | INFO | Analyzing chunk frames = [4487680,4488192) | 20260505 16:02:19.799 | INFO | MemInfo: { | 20260505 16:02:19.799 | INFO | "gpu" : " 17 GB", | 20260505 16:02:19.799 | INFO | "huge" : "", | 20260505 16:02:19.799 | INFO | "huge_shared" : " 27 GB", | 20260505 16:02:19.799 | INFO | "malloc" : " 14 GB", | 20260505 16:02:19.799 | INFO | "pinned" : " 29 GB" | 20260505 16:02:19.799 | INFO | } | 20260505 16:02:19.799 | INFO | Perf report for the last 5.14555 seconds: | 20260505 16:02:19.799 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:02:19.799 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:19.799 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:19.799 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:19.799 | INFO | ANALYSIS : 62.5%, 3.0, 97.7ms, 0 | 20260505 16:02:19.799 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:19.799 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:19.799 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:02:19.799 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:19.799 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:19.799 | INFO | Uploaded 36GiB to GPU in 2.147s (16.7676GiB/s) | 20260505 16:02:19.800 | INFO | Downloaded 34GiB to GPU in 2.05244s (16.5657GiB/s) | 20260505 16:02:24.930 | INFO | Analyzing chunk frames = [4488192,4488704) | 20260505 16:02:24.930 | INFO | MemInfo: { | 20260505 16:02:24.930 | INFO | "gpu" : " 17 GB", | 20260505 16:02:24.930 | INFO | "huge" : "", | 20260505 16:02:24.930 | INFO | "huge_shared" : " 27 GB", | 20260505 16:02:24.930 | INFO | "malloc" : " 14 GB", | 20260505 16:02:24.930 | INFO | "pinned" : " 29 GB" | 20260505 16:02:24.930 | INFO | } | 20260505 16:02:24.931 | INFO | Perf report for the last 5.1315 seconds: | 20260505 16:02:24.931 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:02:24.931 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:24.931 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:24.931 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:24.931 | INFO | ANALYSIS : 63.5%, 3.0, 99.0ms, 0 | 20260505 16:02:24.931 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:24.931 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:24.931 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 16:02:24.931 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:24.931 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:24.931 | INFO | Uploaded 36GiB to GPU in 2.1366s (16.8492GiB/s) | 20260505 16:02:24.931 | INFO | Downloaded 34GiB to GPU in 2.13907s (15.8947GiB/s) | 20260505 16:02:30.062 | INFO | Analyzing chunk frames = [4488704,4489216) | 20260505 16:02:30.062 | INFO | MemInfo: { | 20260505 16:02:30.062 | INFO | "gpu" : " 17 GB", | 20260505 16:02:30.062 | INFO | "huge" : "", | 20260505 16:02:30.062 | INFO | "huge_shared" : " 27 GB", | 20260505 16:02:30.062 | INFO | "malloc" : " 14 GB", | 20260505 16:02:30.062 | INFO | "pinned" : " 29 GB" | 20260505 16:02:30.062 | INFO | } | 20260505 16:02:30.062 | INFO | Perf report for the last 5.13117 seconds: | 20260505 16:02:30.062 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:02:30.062 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:30.062 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:30.062 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:30.062 | INFO | ANALYSIS : 63.8%, 3.0, 99.5ms, 0 | 20260505 16:02:30.062 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:30.062 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:30.062 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:02:30.062 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:30.062 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:30.062 | INFO | Uploaded 36GiB to GPU in 2.23428s (16.1126GiB/s) | 20260505 16:02:30.062 | INFO | Downloaded 34GiB to GPU in 2.10633s (16.1418GiB/s) | 20260505 16:02:30.062 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 16:02:30.062 | INFO | Total mem usage: 15.3 GiB | 20260505 16:02:30.062 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 16:02:30.062 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.87%) | 20260505 16:02:30.062 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 16:02:30.062 | INFO | High Water Breakdown: | 20260505 16:02:30.062 | INFO | 61% : 8.67 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 16:02:30.062 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 16:02:30.062 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 16:02:30.062 | INFO | ------------------------------------------ | 20260505 16:02:30.062 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:02:30.062 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 16:02:30.062 | INFO | Total mem usage: 26.5 GiB | 20260505 16:02:30.062 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 16:02:30.062 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 16:02:30.062 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 16:02:30.062 | INFO | High Water Breakdown: | 20260505 16:02:30.063 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:02:30.063 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:02:30.063 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:02:30.063 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:02:30.063 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:02:30.063 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:02:30.063 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 16:02:30.063 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 16:02:30.063 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:02:30.063 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 16:02:30.063 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 16:02:30.063 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 16:02:30.063 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 16:02:30.063 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 16:02:30.063 | INFO | ------------------------------------------ | 20260505 16:02:30.063 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:02:30.063 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 16:02:30.063 | INFO | Total mem usage: 25.6 GiB | 20260505 16:02:30.063 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 16:02:30.063 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 16:02:30.063 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 16:02:30.063 | INFO | High Water Breakdown: | 20260505 16:02:30.063 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 16:02:30.063 | INFO | ------------------------------------------ | 20260505 16:02:30.063 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:02:30.063 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 16:02:30.063 | INFO | Total mem usage: 15.7 GiB | 20260505 16:02:30.063 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 16:02:30.063 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 16:02:30.063 | INFO | Unused Cache: 896 MiB (Waste: 5.58%) | 20260505 16:02:30.063 | INFO | High Water Breakdown: | 20260505 16:02:30.063 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 16:02:30.063 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 16:02:30.063 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 16:02:30.063 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 16:02:30.063 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 16:02:30.063 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 16:02:30.063 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 16:02:30.063 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 16:02:30.063 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 16:02:30.063 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:02:30.063 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 16:02:30.063 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:02:30.063 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 16:02:30.063 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 16:02:30.063 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 16:02:30.063 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 16:02:30.063 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:02:30.063 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 16:02:30.063 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:02:30.063 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:02:30.063 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:02:30.063 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 16:02:30.063 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 16:02:30.063 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:02:30.063 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 16:02:30.063 | INFO | ------------------------------------------ | 20260505 16:02:30.063 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:02:30.063 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 16:02:30.063 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:02:30.063 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:02:30.063 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.53e+01, 1.37e+01 / 2.09e+00 | 20260505 16:02:30.063 | INFO | PerfReport: 0.29%, AggregateStats -- 9.25e-02 / 1.00e-01, 1.07e-02 / 4.03e-03 | 20260505 16:02:30.063 | INFO | PerfReport: 11.76%, Baseline -- 3.73e+00 / 4.51e+00, 5.72e-01 / 1.51e-01 | 20260505 16:02:30.063 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.50e+00, 1.95e-01 / 1.04e-01 | 20260505 16:02:30.063 | INFO | PerfReport: 15.92%, DME -- 4.04e+01 / 4.47e+01, 4.17e+00 / 2.11e+00 | 20260505 16:02:30.063 | INFO | PerfReport: 56.74%, FrameLabeling -- 1.80e+01 / 1.97e+01, 1.47e+00 / 1.48e+00 | 20260505 16:02:30.063 | INFO | PerfReport: 7.37%, PulseAccumulating -- 2.33e+00 / 2.90e+00, 4.45e-01 / 1.93e-01 | 20260505 16:02:30.063 | INFO | PerfReport: 3.44%, Metrics -- 1.09e+00 / 1.41e+00, 3.01e-01 / 1.89e-01 | 20260505 16:02:30.063 | INFO | PerfReport: 1.20%, PulseSerializing -- 3.81e-01 / 4.46e-01, 6.82e-02 / 5.16e-02 | 20260505 16:02:30.063 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 16:02:30.063 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:02:30.063 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:02:30.063 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260505 16:02:30.063 | INFO | PerfReport: 22.52%, Upload -- 2.26e+01 / 0.00e+00, 8.02e+00 / 0.00e+00 | 20260505 16:02:30.063 | INFO | PerfReport: 21.92%, Download -- 2.20e+01 / 0.00e+00, 7.98e+00 / 0.00e+00 | 20260505 16:02:30.063 | INFO | PerfReport: 31.60%, Compute -- 3.17e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 16:02:30.063 | INFO | PerfReport: 2.27%, LockUpload -- 2.28e+00 / 0.00e+00, 7.89e+00 / 0.00e+00 | 20260505 16:02:30.063 | INFO | PerfReport: 1.37%, LockDownload -- 1.37e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260505 16:02:30.063 | INFO | PerfReport: 20.31%, LockCompute -- 2.04e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 16:02:30.063 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.85e-02 / 0.00e+00 | 20260505 16:02:30.196 | INFO | Starting a round of Baz IO | 20260505 16:02:33.410 | INFO | Populating RTMetrics report | 20260505 16:02:33.411 | INFO | RTMetrics report populated | 20260505 16:02:33.411 | INFO | Opening tmp RTMetrics json file | 20260505 16:02:33.423 | INFO | Serializing RTMetrics to json | 20260505 16:02:33.424 | INFO | Serialization done, swapping tmp file now... | 20260505 16:02:33.426 | INFO | RTMetrics done | 20260505 16:02:33.508 | INFO | Finished a round of Baz IO: Wrote 2.188 GiB in 3.312 seconds | 20260505 16:02:33.508 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.33% / 0.00% / 77.67% | 20260505 16:02:33.508 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.395803% / 7.94729e-06% / 5.96046e-05% | 20260505 16:02:33.508 | INFO | Baz IO overall: | 20260505 16:02:33.508 | INFO | BytesWritten : 2666886106034 | 20260505 16:02:33.508 | INFO | NumEvents : 1266298531814 | 20260505 16:02:35.204 | INFO | Analyzing chunk frames = [4489216,4489728) | 20260505 16:02:35.204 | INFO | MemInfo: { | 20260505 16:02:35.204 | INFO | "gpu" : " 17 GB", | 20260505 16:02:35.204 | INFO | "huge" : "", | 20260505 16:02:35.204 | INFO | "huge_shared" : " 27 GB", | 20260505 16:02:35.204 | INFO | "malloc" : " 14 GB", | 20260505 16:02:35.204 | INFO | "pinned" : " 29 GB" | 20260505 16:02:35.204 | INFO | } | 20260505 16:02:35.204 | INFO | Perf report for the last 5.14237 seconds: | 20260505 16:02:35.204 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:02:35.204 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:35.204 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:35.204 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:35.204 | INFO | ANALYSIS : 64.0%, 3.0, 99.8ms, 0 | 20260505 16:02:35.204 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:35.204 | INFO | RT_METRICS : 44.9%, 1.0, 23.6ms, 0 | 20260505 16:02:35.204 | INFO | PRE_HQ : 60.5%, 1.8, 56.4ms, 0 | 20260505 16:02:35.204 | INFO | BAZWRITER : 63.7%, 3.9, 127.7ms, 0 | 20260505 16:02:35.204 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 16:02:35.205 | INFO | Uploaded 36GiB to GPU in 2.21003s (16.2894GiB/s) | 20260505 16:02:35.205 | INFO | Downloaded 38GiB to GPU in 2.25798s (16.8292GiB/s) | 20260505 16:02:40.345 | INFO | Analyzing chunk frames = [4489728,4490240) | 20260505 16:02:40.345 | INFO | MemInfo: { | 20260505 16:02:40.345 | INFO | "gpu" : " 17 GB", | 20260505 16:02:40.345 | INFO | "huge" : "", | 20260505 16:02:40.345 | INFO | "huge_shared" : " 27 GB", | 20260505 16:02:40.345 | INFO | "malloc" : " 14 GB", | 20260505 16:02:40.345 | INFO | "pinned" : " 29 GB" | 20260505 16:02:40.345 | INFO | } | 20260505 16:02:40.345 | INFO | Perf report for the last 5.14075 seconds: | 20260505 16:02:40.345 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:02:40.345 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:40.345 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:40.345 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:40.345 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 16:02:40.345 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:40.345 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:40.345 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:02:40.345 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:40.345 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:40.345 | INFO | Uploaded 36GiB to GPU in 2.20167s (16.3513GiB/s) | 20260505 16:02:40.345 | INFO | Downloaded 34GiB to GPU in 2.05687s (16.5299GiB/s) | 20260505 16:02:45.486 | INFO | Analyzing chunk frames = [4490240,4490752) | 20260505 16:02:45.486 | INFO | MemInfo: { | 20260505 16:02:45.486 | INFO | "gpu" : " 17 GB", | 20260505 16:02:45.486 | INFO | "huge" : "", | 20260505 16:02:45.486 | INFO | "huge_shared" : " 27 GB", | 20260505 16:02:45.486 | INFO | "malloc" : " 14 GB", | 20260505 16:02:45.486 | INFO | "pinned" : " 29 GB" | 20260505 16:02:45.486 | INFO | } | 20260505 16:02:45.486 | INFO | Perf report for the last 5.14133 seconds: | 20260505 16:02:45.486 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:02:45.486 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:45.486 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:45.486 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:45.486 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 16:02:45.486 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:45.486 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:45.486 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 16:02:45.486 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:45.486 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:45.487 | INFO | Uploaded 36GiB to GPU in 2.19272s (16.418GiB/s) | 20260505 16:02:45.487 | INFO | Downloaded 34GiB to GPU in 2.0354s (16.7043GiB/s) | 20260505 16:02:50.621 | INFO | Analyzing chunk frames = [4490752,4491264) | 20260505 16:02:50.621 | INFO | MemInfo: { | 20260505 16:02:50.621 | INFO | "gpu" : " 17 GB", | 20260505 16:02:50.621 | INFO | "huge" : "", | 20260505 16:02:50.621 | INFO | "huge_shared" : " 27 GB", | 20260505 16:02:50.621 | INFO | "malloc" : " 14 GB", | 20260505 16:02:50.621 | INFO | "pinned" : " 29 GB" | 20260505 16:02:50.621 | INFO | } | 20260505 16:02:50.621 | INFO | Perf report for the last 5.1351 seconds: | 20260505 16:02:50.621 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:02:50.621 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:50.621 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:50.621 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:50.621 | INFO | ANALYSIS : 62.7%, 3.0, 97.3ms, 0 | 20260505 16:02:50.621 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:50.621 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:50.621 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 16:02:50.621 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:50.621 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:50.622 | INFO | Uploaded 36GiB to GPU in 2.16582s (16.6219GiB/s) | 20260505 16:02:50.622 | INFO | Downloaded 34GiB to GPU in 2.14209s (15.8724GiB/s) | 20260505 16:02:55.761 | INFO | Analyzing chunk frames = [4491264,4491776) | 20260505 16:02:55.761 | INFO | MemInfo: { | 20260505 16:02:55.761 | INFO | "gpu" : " 17 GB", | 20260505 16:02:55.761 | INFO | "huge" : "", | 20260505 16:02:55.761 | INFO | "huge_shared" : " 27 GB", | 20260505 16:02:55.761 | INFO | "malloc" : " 14 GB", | 20260505 16:02:55.761 | INFO | "pinned" : " 29 GB" | 20260505 16:02:55.762 | INFO | } | 20260505 16:02:55.762 | INFO | Perf report for the last 5.14033 seconds: | 20260505 16:02:55.762 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:02:55.762 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:55.762 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:55.762 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:55.762 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260505 16:02:55.762 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:55.762 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:55.762 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:02:55.762 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:55.762 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:02:55.762 | INFO | Uploaded 36GiB to GPU in 2.17571s (16.5463GiB/s) | 20260505 16:02:55.762 | INFO | Downloaded 34GiB to GPU in 2.13139s (15.9521GiB/s) | 20260505 16:03:00.913 | INFO | Analyzing chunk frames = [4491776,4492288) | 20260505 16:03:00.913 | INFO | MemInfo: { | 20260505 16:03:00.913 | INFO | "gpu" : " 17 GB", | 20260505 16:03:00.913 | INFO | "huge" : "", | 20260505 16:03:00.913 | INFO | "huge_shared" : " 27 GB", | 20260505 16:03:00.913 | INFO | "malloc" : " 14 GB", | 20260505 16:03:00.913 | INFO | "pinned" : " 29 GB" | 20260505 16:03:00.913 | INFO | } | 20260505 16:03:00.913 | INFO | Perf report for the last 5.15158 seconds: | 20260505 16:03:00.913 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:03:00.913 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:00.913 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:00.913 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:00.913 | INFO | ANALYSIS : 63.0%, 3.0, 98.6ms, 0 | 20260505 16:03:00.913 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:00.913 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:00.913 | INFO | PRE_HQ : 28.1%, 1.0, 14.7ms, 0 | 20260505 16:03:00.913 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:00.913 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:00.914 | INFO | Uploaded 36GiB to GPU in 2.20572s (16.3212GiB/s) | 20260505 16:03:00.914 | INFO | Downloaded 34GiB to GPU in 2.11463s (16.0784GiB/s) | 20260505 16:03:06.047 | INFO | Analyzing chunk frames = [4492288,4492800) | 20260505 16:03:06.047 | INFO | MemInfo: { | 20260505 16:03:06.047 | INFO | "gpu" : " 17 GB", | 20260505 16:03:06.047 | INFO | "huge" : "", | 20260505 16:03:06.047 | INFO | "huge_shared" : " 27 GB", | 20260505 16:03:06.047 | INFO | "malloc" : " 14 GB", | 20260505 16:03:06.047 | INFO | "pinned" : " 29 GB" | 20260505 16:03:06.047 | INFO | } | 20260505 16:03:06.047 | INFO | Perf report for the last 5.13393 seconds: | 20260505 16:03:06.047 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:03:06.047 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:06.047 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:06.047 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:06.047 | INFO | ANALYSIS : 63.6%, 3.0, 99.1ms, 0 | 20260505 16:03:06.047 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:06.047 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:06.047 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 16:03:06.047 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:06.047 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:06.047 | INFO | Uploaded 36GiB to GPU in 2.15278s (16.7225GiB/s) | 20260505 16:03:06.047 | INFO | Downloaded 34GiB to GPU in 2.17618s (15.6237GiB/s) | 20260505 16:03:11.181 | INFO | Analyzing chunk frames = [4492800,4493312) | 20260505 16:03:11.181 | INFO | MemInfo: { | 20260505 16:03:11.181 | INFO | "gpu" : " 17 GB", | 20260505 16:03:11.181 | INFO | "huge" : "", | 20260505 16:03:11.181 | INFO | "huge_shared" : " 27 GB", | 20260505 16:03:11.181 | INFO | "malloc" : " 14 GB", | 20260505 16:03:11.182 | INFO | "pinned" : " 29 GB" | 20260505 16:03:11.182 | INFO | } | 20260505 16:03:11.182 | INFO | Perf report for the last 5.13443 seconds: | 20260505 16:03:11.182 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:03:11.182 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:11.182 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:11.182 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:11.182 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 16:03:11.182 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:11.182 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:11.182 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 16:03:11.182 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:11.182 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:11.182 | INFO | Uploaded 36GiB to GPU in 2.15017s (16.7429GiB/s) | 20260505 16:03:11.182 | INFO | Downloaded 34GiB to GPU in 2.02577s (16.7837GiB/s) | 20260505 16:03:11.312 | INFO | Starting a round of Baz IO | 20260505 16:03:14.527 | INFO | Populating RTMetrics report | 20260505 16:03:14.527 | INFO | RTMetrics report populated | 20260505 16:03:14.527 | INFO | Opening tmp RTMetrics json file | 20260505 16:03:14.538 | INFO | Serializing RTMetrics to json | 20260505 16:03:14.541 | INFO | Serialization done, swapping tmp file now... | 20260505 16:03:14.543 | INFO | RTMetrics done | 20260505 16:03:14.627 | INFO | Finished a round of Baz IO: Wrote 2.188 GiB in 3.314 seconds | 20260505 16:03:14.627 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.33% / 0.00% / 77.67% | 20260505 16:03:14.627 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.394313% / 3.97364e-06% / 5.5631e-05% | 20260505 16:03:14.627 | INFO | Baz IO overall: | 20260505 16:03:14.627 | INFO | BytesWritten : 2669235514221 | 20260505 16:03:14.627 | INFO | NumEvents : 1267401820635 | 20260505 16:03:16.316 | INFO | Analyzing chunk frames = [4493312,4493824) | 20260505 16:03:16.316 | INFO | MemInfo: { | 20260505 16:03:16.316 | INFO | "gpu" : " 17 GB", | 20260505 16:03:16.316 | INFO | "huge" : "", | 20260505 16:03:16.316 | INFO | "huge_shared" : " 27 GB", | 20260505 16:03:16.316 | INFO | "malloc" : " 14 GB", | 20260505 16:03:16.316 | INFO | "pinned" : " 29 GB" | 20260505 16:03:16.316 | INFO | } | 20260505 16:03:16.316 | INFO | Perf report for the last 5.13448 seconds: | 20260505 16:03:16.316 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:03:16.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:16.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:16.316 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:16.316 | INFO | ANALYSIS : 64.1%, 3.0, 99.5ms, 0 | 20260505 16:03:16.316 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:16.316 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260505 16:03:16.316 | INFO | PRE_HQ : 61.0%, 1.8, 56.6ms, 0 | 20260505 16:03:16.316 | INFO | BAZWRITER : 63.9%, 3.9, 128.6ms, 0 | 20260505 16:03:16.316 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 16:03:16.316 | INFO | Uploaded 36GiB to GPU in 2.21s (16.2896GiB/s) | 20260505 16:03:16.317 | INFO | Downloaded 38GiB to GPU in 2.26435s (16.7819GiB/s) | 20260505 16:03:21.463 | INFO | Analyzing chunk frames = [4493824,4494336) | 20260505 16:03:21.463 | INFO | MemInfo: { | 20260505 16:03:21.463 | INFO | "gpu" : " 17 GB", | 20260505 16:03:21.463 | INFO | "huge" : "", | 20260505 16:03:21.463 | INFO | "huge_shared" : " 27 GB", | 20260505 16:03:21.463 | INFO | "malloc" : " 14 GB", | 20260505 16:03:21.463 | INFO | "pinned" : " 29 GB" | 20260505 16:03:21.463 | INFO | } | 20260505 16:03:21.463 | INFO | Perf report for the last 5.14699 seconds: | 20260505 16:03:21.463 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:03:21.463 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:21.463 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:21.463 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:21.463 | INFO | ANALYSIS : 63.2%, 3.0, 98.8ms, 0 | 20260505 16:03:21.463 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:21.463 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:21.463 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:03:21.463 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:21.463 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:21.463 | INFO | Uploaded 36GiB to GPU in 2.19535s (16.3983GiB/s) | 20260505 16:03:21.463 | INFO | Downloaded 34GiB to GPU in 2.03613s (16.6984GiB/s) | 20260505 16:03:26.603 | INFO | Analyzing chunk frames = [4494336,4494848) | 20260505 16:03:26.603 | INFO | MemInfo: { | 20260505 16:03:26.603 | INFO | "gpu" : " 17 GB", | 20260505 16:03:26.603 | INFO | "huge" : "", | 20260505 16:03:26.603 | INFO | "huge_shared" : " 27 GB", | 20260505 16:03:26.603 | INFO | "malloc" : " 14 GB", | 20260505 16:03:26.603 | INFO | "pinned" : " 29 GB" | 20260505 16:03:26.603 | INFO | } | 20260505 16:03:26.603 | INFO | Perf report for the last 5.13973 seconds: | 20260505 16:03:26.603 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:03:26.603 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:26.603 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:26.603 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:26.603 | INFO | ANALYSIS : 62.7%, 3.0, 97.5ms, 0 | 20260505 16:03:26.603 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:26.603 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:26.603 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:03:26.603 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:26.603 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:26.603 | INFO | Uploaded 36GiB to GPU in 2.19964s (16.3663GiB/s) | 20260505 16:03:26.603 | INFO | Downloaded 34GiB to GPU in 2.03831s (16.6805GiB/s) | 20260505 16:03:31.747 | INFO | Analyzing chunk frames = [4494848,4495360) | 20260505 16:03:31.747 | INFO | MemInfo: { | 20260505 16:03:31.747 | INFO | "gpu" : " 17 GB", | 20260505 16:03:31.747 | INFO | "huge" : "", | 20260505 16:03:31.747 | INFO | "huge_shared" : " 27 GB", | 20260505 16:03:31.747 | INFO | "malloc" : " 14 GB", | 20260505 16:03:31.747 | INFO | "pinned" : " 29 GB" | 20260505 16:03:31.747 | INFO | } | 20260505 16:03:31.747 | INFO | Perf report for the last 5.14437 seconds: | 20260505 16:03:31.747 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:03:31.747 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:31.747 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:31.747 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:31.747 | INFO | ANALYSIS : 62.5%, 3.0, 97.2ms, 0 | 20260505 16:03:31.747 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:31.747 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:31.747 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 16:03:31.747 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:31.747 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:31.748 | INFO | Uploaded 36GiB to GPU in 2.14957s (16.7475GiB/s) | 20260505 16:03:31.748 | INFO | Downloaded 34GiB to GPU in 2.14244s (15.8697GiB/s) | 20260505 16:03:36.875 | INFO | Analyzing chunk frames = [4495360,4495872) | 20260505 16:03:36.875 | INFO | MemInfo: { | 20260505 16:03:36.875 | INFO | "gpu" : " 17 GB", | 20260505 16:03:36.875 | INFO | "huge" : "", | 20260505 16:03:36.875 | INFO | "huge_shared" : " 27 GB", | 20260505 16:03:36.875 | INFO | "malloc" : " 14 GB", | 20260505 16:03:36.875 | INFO | "pinned" : " 29 GB" | 20260505 16:03:36.875 | INFO | } | 20260505 16:03:36.875 | INFO | Perf report for the last 5.12812 seconds: | 20260505 16:03:36.875 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:03:36.875 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:36.875 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:36.875 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:36.875 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260505 16:03:36.875 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:36.875 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:36.875 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 16:03:36.875 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:36.875 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:36.876 | INFO | Uploaded 36GiB to GPU in 2.1508s (16.738GiB/s) | 20260505 16:03:36.876 | INFO | Downloaded 34GiB to GPU in 2.05976s (16.5068GiB/s) | 20260505 16:03:42.011 | INFO | Analyzing chunk frames = [4495872,4496384) | 20260505 16:03:42.011 | INFO | MemInfo: { | 20260505 16:03:42.011 | INFO | "gpu" : " 17 GB", | 20260505 16:03:42.011 | INFO | "huge" : "", | 20260505 16:03:42.011 | INFO | "huge_shared" : " 27 GB", | 20260505 16:03:42.011 | INFO | "malloc" : " 14 GB", | 20260505 16:03:42.011 | INFO | "pinned" : " 29 GB" | 20260505 16:03:42.011 | INFO | } | 20260505 16:03:42.011 | INFO | Perf report for the last 5.13562 seconds: | 20260505 16:03:42.011 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:03:42.011 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:42.011 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:42.011 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:42.011 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260505 16:03:42.011 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:42.011 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:42.011 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 16:03:42.011 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:42.011 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:42.011 | INFO | Uploaded 36GiB to GPU in 2.17302s (16.5668GiB/s) | 20260505 16:03:42.011 | INFO | Downloaded 34GiB to GPU in 2.04033s (16.6639GiB/s) | 20260505 16:03:47.159 | INFO | Analyzing chunk frames = [4496384,4496896) | 20260505 16:03:47.159 | INFO | MemInfo: { | 20260505 16:03:47.159 | INFO | "gpu" : " 17 GB", | 20260505 16:03:47.159 | INFO | "huge" : "", | 20260505 16:03:47.159 | INFO | "huge_shared" : " 27 GB", | 20260505 16:03:47.159 | INFO | "malloc" : " 14 GB", | 20260505 16:03:47.159 | INFO | "pinned" : " 29 GB" | 20260505 16:03:47.159 | INFO | } | 20260505 16:03:47.159 | INFO | Perf report for the last 5.14784 seconds: | 20260505 16:03:47.159 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:03:47.159 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:47.159 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:47.159 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:47.159 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 16:03:47.159 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:47.159 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:47.159 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 16:03:47.159 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:47.159 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:47.159 | INFO | Uploaded 36GiB to GPU in 2.12363s (16.9521GiB/s) | 20260505 16:03:47.159 | INFO | Downloaded 34GiB to GPU in 2.1668s (15.6913GiB/s) | 20260505 16:03:52.300 | INFO | Analyzing chunk frames = [4496896,4497408) | 20260505 16:03:52.300 | INFO | MemInfo: { | 20260505 16:03:52.300 | INFO | "gpu" : " 17 GB", | 20260505 16:03:52.300 | INFO | "huge" : "", | 20260505 16:03:52.300 | INFO | "huge_shared" : " 27 GB", | 20260505 16:03:52.300 | INFO | "malloc" : " 14 GB", | 20260505 16:03:52.300 | INFO | "pinned" : " 29 GB" | 20260505 16:03:52.300 | INFO | } | 20260505 16:03:52.300 | INFO | Perf report for the last 5.14136 seconds: | 20260505 16:03:52.300 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:03:52.300 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:52.300 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:52.300 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:52.300 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 16:03:52.300 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:52.300 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:52.300 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 16:03:52.300 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:52.301 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:52.301 | INFO | Uploaded 36GiB to GPU in 2.18196s (16.4989GiB/s) | 20260505 16:03:52.301 | INFO | Downloaded 34GiB to GPU in 2.02843s (16.7617GiB/s) | 20260505 16:03:52.301 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 16:03:52.301 | INFO | Total mem usage: 15.3 GiB | 20260505 16:03:52.301 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 16:03:52.301 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.95%) | 20260505 16:03:52.301 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 16:03:52.301 | INFO | High Water Breakdown: | 20260505 16:03:52.301 | INFO | 61% : 8.66 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 16:03:52.301 | INFO | 38.9% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 16:03:52.301 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 16:03:52.301 | INFO | ------------------------------------------ | 20260505 16:03:52.301 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:03:52.301 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 16:03:52.301 | INFO | Total mem usage: 26.5 GiB | 20260505 16:03:52.301 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 16:03:52.301 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 16:03:52.301 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 16:03:52.301 | INFO | High Water Breakdown: | 20260505 16:03:52.301 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:03:52.301 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:03:52.301 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:03:52.301 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:03:52.301 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:03:52.301 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:03:52.301 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 16:03:52.301 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 16:03:52.301 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:03:52.301 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 16:03:52.301 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 16:03:52.301 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 16:03:52.301 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 16:03:52.301 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 16:03:52.301 | INFO | ------------------------------------------ | 20260505 16:03:52.301 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:03:52.301 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 16:03:52.301 | INFO | Total mem usage: 25.6 GiB | 20260505 16:03:52.301 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 16:03:52.301 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 16:03:52.301 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 16:03:52.301 | INFO | High Water Breakdown: | 20260505 16:03:52.301 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 16:03:52.301 | INFO | ------------------------------------------ | 20260505 16:03:52.301 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:03:52.301 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 16:03:52.301 | INFO | Total mem usage: 15.7 GiB | 20260505 16:03:52.301 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 16:03:52.301 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 16:03:52.301 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 16:03:52.301 | INFO | High Water Breakdown: | 20260505 16:03:52.301 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 16:03:52.301 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 16:03:52.302 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 16:03:52.302 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 16:03:52.302 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 16:03:52.302 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 16:03:52.302 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 16:03:52.302 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 16:03:52.302 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 16:03:52.302 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:03:52.302 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 16:03:52.302 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:03:52.302 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 16:03:52.302 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 16:03:52.302 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 16:03:52.302 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 16:03:52.302 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:03:52.302 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 16:03:52.302 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:03:52.302 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:03:52.302 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:03:52.302 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 16:03:52.302 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 16:03:52.302 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:03:52.302 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 16:03:52.302 | INFO | ------------------------------------------ | 20260505 16:03:52.302 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:03:52.302 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 16:03:52.302 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:03:52.302 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:03:52.302 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.54e+01, 1.35e+01 / 2.51e+00 | 20260505 16:03:52.302 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 1.00e-01, 1.11e-02 / 3.52e-03 | 20260505 16:03:52.302 | INFO | PerfReport: 11.78%, Baseline -- 3.73e+00 / 4.50e+00, 5.72e-01 / 1.64e-01 | 20260505 16:03:52.302 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.50e+00, 1.91e-01 / 1.06e-01 | 20260505 16:03:52.302 | INFO | PerfReport: 16.00%, DME -- 4.05e+01 / 4.55e+01, 4.34e+00 / 2.48e+00 | 20260505 16:03:52.302 | INFO | PerfReport: 56.60%, FrameLabeling -- 1.79e+01 / 1.89e+01, 1.39e+00 / 1.04e+00 | 20260505 16:03:52.302 | INFO | PerfReport: 7.42%, PulseAccumulating -- 2.35e+00 / 2.98e+00, 4.57e-01 / 1.28e-01 | 20260505 16:03:52.302 | INFO | PerfReport: 3.44%, Metrics -- 1.09e+00 / 1.43e+00, 3.00e-01 / 1.75e-01 | 20260505 16:03:52.302 | INFO | PerfReport: 1.20%, PulseSerializing -- 3.79e-01 / 4.36e-01, 6.76e-02 / 3.13e-02 | 20260505 16:03:52.302 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 16:03:52.302 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:03:52.302 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:03:52.302 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.57e+01 / 0.00e+00 | 20260505 16:03:52.302 | INFO | PerfReport: 22.60%, Upload -- 2.27e+01 / 0.00e+00, 7.96e+00 / 0.00e+00 | 20260505 16:03:52.302 | INFO | PerfReport: 21.87%, Download -- 2.20e+01 / 0.00e+00, 7.92e+00 / 0.00e+00 | 20260505 16:03:52.302 | INFO | PerfReport: 31.54%, Compute -- 3.17e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 16:03:52.302 | INFO | PerfReport: 2.31%, LockUpload -- 2.32e+00 / 0.00e+00, 7.95e+00 / 0.00e+00 | 20260505 16:03:52.302 | INFO | PerfReport: 1.47%, LockDownload -- 1.47e+00 / 0.00e+00, 2.96e+00 / 0.00e+00 | 20260505 16:03:52.302 | INFO | PerfReport: 20.20%, LockCompute -- 2.03e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260505 16:03:52.302 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.11e-02 / 0.00e+00 | 20260505 16:03:52.431 | INFO | Starting a round of Baz IO | 20260505 16:03:55.644 | INFO | Populating RTMetrics report | 20260505 16:03:55.644 | INFO | RTMetrics report populated | 20260505 16:03:55.644 | INFO | Opening tmp RTMetrics json file | 20260505 16:03:55.645 | INFO | Serializing RTMetrics to json | 20260505 16:03:55.646 | INFO | Serialization done, swapping tmp file now... | 20260505 16:03:55.647 | INFO | RTMetrics done | 20260505 16:03:55.783 | INFO | Finished a round of Baz IO: Wrote 2.186 GiB in 3.351 seconds | 20260505 16:03:55.784 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.33% / 0.00% / 77.67% | 20260505 16:03:55.784 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.393053% / 0% / 5.96046e-05% | 20260505 16:03:55.784 | INFO | Baz IO overall: | 20260505 16:03:55.784 | INFO | BytesWritten : 2671583160849 | 20260505 16:03:55.784 | INFO | NumEvents : 1268503498653 | 20260505 16:03:57.437 | INFO | Analyzing chunk frames = [4497408,4497920) | 20260505 16:03:57.437 | INFO | MemInfo: { | 20260505 16:03:57.437 | INFO | "gpu" : " 17 GB", | 20260505 16:03:57.437 | INFO | "huge" : "", | 20260505 16:03:57.437 | INFO | "huge_shared" : " 27 GB", | 20260505 16:03:57.437 | INFO | "malloc" : " 14 GB", | 20260505 16:03:57.437 | INFO | "pinned" : " 29 GB" | 20260505 16:03:57.437 | INFO | } | 20260505 16:03:57.437 | INFO | Perf report for the last 5.13707 seconds: | 20260505 16:03:57.437 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:03:57.437 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:57.437 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:57.437 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:57.437 | INFO | ANALYSIS : 64.0%, 3.0, 99.4ms, 0 | 20260505 16:03:57.437 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:03:57.437 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260505 16:03:57.437 | INFO | PRE_HQ : 61.0%, 1.8, 56.9ms, 0 | 20260505 16:03:57.437 | INFO | BAZWRITER : 64.6%, 3.9, 130.6ms, 0 | 20260505 16:03:57.437 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 16:03:57.438 | INFO | Uploaded 36GiB to GPU in 2.2051s (16.3258GiB/s) | 20260505 16:03:57.438 | INFO | Downloaded 38GiB to GPU in 2.26112s (16.8059GiB/s) | 20260505 16:04:02.574 | INFO | Analyzing chunk frames = [4497920,4498432) | 20260505 16:04:02.574 | INFO | MemInfo: { | 20260505 16:04:02.574 | INFO | "gpu" : " 17 GB", | 20260505 16:04:02.574 | INFO | "huge" : "", | 20260505 16:04:02.574 | INFO | "huge_shared" : " 27 GB", | 20260505 16:04:02.574 | INFO | "malloc" : " 14 GB", | 20260505 16:04:02.574 | INFO | "pinned" : " 29 GB" | 20260505 16:04:02.574 | INFO | } | 20260505 16:04:02.574 | INFO | Perf report for the last 5.13703 seconds: | 20260505 16:04:02.574 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:04:02.574 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:02.574 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:02.574 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:02.574 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 16:04:02.574 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:02.574 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:02.574 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 16:04:02.575 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:02.575 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:02.575 | INFO | Uploaded 36GiB to GPU in 2.11464s (17.0242GiB/s) | 20260505 16:04:02.575 | INFO | Downloaded 34GiB to GPU in 2.0205s (16.8275GiB/s) | 20260505 16:04:07.717 | INFO | Analyzing chunk frames = [4498432,4498944) | 20260505 16:04:07.717 | INFO | MemInfo: { | 20260505 16:04:07.717 | INFO | "gpu" : " 17 GB", | 20260505 16:04:07.717 | INFO | "huge" : "", | 20260505 16:04:07.717 | INFO | "huge_shared" : " 27 GB", | 20260505 16:04:07.717 | INFO | "malloc" : " 14 GB", | 20260505 16:04:07.717 | INFO | "pinned" : " 29 GB" | 20260505 16:04:07.717 | INFO | } | 20260505 16:04:07.717 | INFO | Perf report for the last 5.14269 seconds: | 20260505 16:04:07.717 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:04:07.717 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:07.717 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:07.717 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:07.717 | INFO | ANALYSIS : 62.9%, 3.0, 98.0ms, 0 | 20260505 16:04:07.717 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:07.717 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:07.717 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 16:04:07.717 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:07.717 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:07.717 | INFO | Uploaded 36GiB to GPU in 2.19861s (16.374GiB/s) | 20260505 16:04:07.717 | INFO | Downloaded 34GiB to GPU in 2.05293s (16.5617GiB/s) | 20260505 16:04:12.857 | INFO | Analyzing chunk frames = [4498944,4499456) | 20260505 16:04:12.857 | INFO | MemInfo: { | 20260505 16:04:12.857 | INFO | "gpu" : " 17 GB", | 20260505 16:04:12.857 | INFO | "huge" : "", | 20260505 16:04:12.857 | INFO | "huge_shared" : " 27 GB", | 20260505 16:04:12.857 | INFO | "malloc" : " 14 GB", | 20260505 16:04:12.857 | INFO | "pinned" : " 29 GB" | 20260505 16:04:12.857 | INFO | } | 20260505 16:04:12.857 | INFO | Perf report for the last 5.14024 seconds: | 20260505 16:04:12.857 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:04:12.857 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:12.857 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:12.858 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:12.858 | INFO | ANALYSIS : 62.6%, 3.0, 97.3ms, 0 | 20260505 16:04:12.858 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:12.858 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:12.858 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 16:04:12.858 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:12.858 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:12.858 | INFO | Uploaded 36GiB to GPU in 2.17179s (16.5762GiB/s) | 20260505 16:04:12.858 | INFO | Downloaded 34GiB to GPU in 2.14386s (15.8593GiB/s) | 20260505 16:04:17.994 | INFO | Analyzing chunk frames = [4499456,4499968) | 20260505 16:04:17.994 | INFO | MemInfo: { | 20260505 16:04:17.994 | INFO | "gpu" : " 17 GB", | 20260505 16:04:17.994 | INFO | "huge" : "", | 20260505 16:04:17.994 | INFO | "huge_shared" : " 27 GB", | 20260505 16:04:17.994 | INFO | "malloc" : " 14 GB", | 20260505 16:04:17.994 | INFO | "pinned" : " 29 GB" | 20260505 16:04:17.994 | INFO | } | 20260505 16:04:17.994 | INFO | Perf report for the last 5.13645 seconds: | 20260505 16:04:17.994 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:04:17.994 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:17.994 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:17.994 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:17.994 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 16:04:17.994 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:17.994 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:17.994 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 16:04:17.994 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:17.994 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:17.994 | INFO | Uploaded 36GiB to GPU in 2.15254s (16.7244GiB/s) | 20260505 16:04:17.994 | INFO | Downloaded 34GiB to GPU in 2.05883s (16.5142GiB/s) | 20260505 16:04:23.131 | INFO | Analyzing chunk frames = [4499968,4500480) | 20260505 16:04:23.132 | INFO | MemInfo: { | 20260505 16:04:23.132 | INFO | "gpu" : " 17 GB", | 20260505 16:04:23.132 | INFO | "huge" : "", | 20260505 16:04:23.132 | INFO | "huge_shared" : " 27 GB", | 20260505 16:04:23.132 | INFO | "malloc" : " 14 GB", | 20260505 16:04:23.132 | INFO | "pinned" : " 29 GB" | 20260505 16:04:23.132 | INFO | } | 20260505 16:04:23.132 | INFO | Perf report for the last 5.13778 seconds: | 20260505 16:04:23.132 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:04:23.132 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:23.132 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:23.132 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:23.132 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 16:04:23.132 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:23.132 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:23.132 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:04:23.132 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:23.132 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:23.132 | INFO | Uploaded 36GiB to GPU in 2.13975s (16.8244GiB/s) | 20260505 16:04:23.132 | INFO | Downloaded 34GiB to GPU in 2.03509s (16.7069GiB/s) | 20260505 16:04:28.269 | INFO | Analyzing chunk frames = [4500480,4500992) | 20260505 16:04:28.269 | INFO | MemInfo: { | 20260505 16:04:28.269 | INFO | "gpu" : " 17 GB", | 20260505 16:04:28.269 | INFO | "huge" : "", | 20260505 16:04:28.269 | INFO | "huge_shared" : " 27 GB", | 20260505 16:04:28.269 | INFO | "malloc" : " 14 GB", | 20260505 16:04:28.269 | INFO | "pinned" : " 29 GB" | 20260505 16:04:28.269 | INFO | } | 20260505 16:04:28.269 | INFO | Perf report for the last 5.1372 seconds: | 20260505 16:04:28.269 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:04:28.269 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:28.269 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:28.269 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:28.269 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 16:04:28.269 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:28.269 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:28.269 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 16:04:28.269 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:28.269 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:28.269 | INFO | Uploaded 36GiB to GPU in 2.11384s (17.0306GiB/s) | 20260505 16:04:28.269 | INFO | Downloaded 34GiB to GPU in 2.17071s (15.6631GiB/s) | 20260505 16:04:33.416 | INFO | Analyzing chunk frames = [4500992,4501504) | 20260505 16:04:33.416 | INFO | MemInfo: { | 20260505 16:04:33.416 | INFO | "gpu" : " 17 GB", | 20260505 16:04:33.416 | INFO | "huge" : "", | 20260505 16:04:33.416 | INFO | "huge_shared" : " 27 GB", | 20260505 16:04:33.416 | INFO | "malloc" : " 14 GB", | 20260505 16:04:33.416 | INFO | "pinned" : " 29 GB" | 20260505 16:04:33.416 | INFO | } | 20260505 16:04:33.416 | INFO | Perf report for the last 5.14723 seconds: | 20260505 16:04:33.416 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:04:33.416 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:33.416 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:33.416 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:33.416 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 16:04:33.416 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:33.416 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:33.416 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 16:04:33.416 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:33.416 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:33.417 | INFO | Uploaded 36GiB to GPU in 2.23602s (16.1GiB/s) | 20260505 16:04:33.417 | INFO | Downloaded 34GiB to GPU in 2.10609s (16.1437GiB/s) | 20260505 16:04:33.554 | INFO | Starting a round of Baz IO | 20260505 16:04:36.742 | INFO | Populating RTMetrics report | 20260505 16:04:36.742 | INFO | RTMetrics report populated | 20260505 16:04:36.742 | INFO | Opening tmp RTMetrics json file | 20260505 16:04:36.758 | INFO | Serializing RTMetrics to json | 20260505 16:04:36.780 | INFO | Serialization done, swapping tmp file now... | 20260505 16:04:36.801 | INFO | RTMetrics done | 20260505 16:04:36.840 | INFO | Finished a round of Baz IO: Wrote 2.188 GiB in 3.286 seconds | 20260505 16:04:36.840 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.32% / 0.00% / 77.67% | 20260505 16:04:36.840 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.393796% / 0% / 5.96046e-05% | 20260505 16:04:36.840 | INFO | Baz IO overall: | 20260505 16:04:36.840 | INFO | BytesWritten : 2673932270917 | 20260505 16:04:36.840 | INFO | NumEvents : 1269606483745 | 20260505 16:04:38.561 | INFO | Analyzing chunk frames = [4501504,4502016) | 20260505 16:04:38.561 | INFO | MemInfo: { | 20260505 16:04:38.561 | INFO | "gpu" : " 17 GB", | 20260505 16:04:38.561 | INFO | "huge" : "", | 20260505 16:04:38.561 | INFO | "huge_shared" : " 27 GB", | 20260505 16:04:38.561 | INFO | "malloc" : " 14 GB", | 20260505 16:04:38.561 | INFO | "pinned" : " 29 GB" | 20260505 16:04:38.561 | INFO | } | 20260505 16:04:38.561 | INFO | Perf report for the last 5.14535 seconds: | 20260505 16:04:38.561 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:04:38.561 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:38.562 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:38.562 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:38.562 | INFO | ANALYSIS : 63.6%, 3.0, 99.0ms, 0 | 20260505 16:04:38.562 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:38.562 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260505 16:04:38.562 | INFO | PRE_HQ : 60.5%, 1.8, 56.7ms, 0 | 20260505 16:04:38.562 | INFO | BAZWRITER : 63.3%, 4.0, 129.6ms, 0 | 20260505 16:04:38.562 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 16:04:38.562 | INFO | Uploaded 36GiB to GPU in 2.15997s (16.6669GiB/s) | 20260505 16:04:38.562 | INFO | Downloaded 38GiB to GPU in 2.26076s (16.8085GiB/s) | 20260505 16:04:43.691 | INFO | Analyzing chunk frames = [4502016,4502528) | 20260505 16:04:43.691 | INFO | MemInfo: { | 20260505 16:04:43.691 | INFO | "gpu" : " 17 GB", | 20260505 16:04:43.691 | INFO | "huge" : "", | 20260505 16:04:43.691 | INFO | "huge_shared" : " 27 GB", | 20260505 16:04:43.691 | INFO | "malloc" : " 14 GB", | 20260505 16:04:43.691 | INFO | "pinned" : " 29 GB" | 20260505 16:04:43.691 | INFO | } | 20260505 16:04:43.691 | INFO | Perf report for the last 5.12944 seconds: | 20260505 16:04:43.691 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:04:43.691 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:43.691 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:43.691 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:43.691 | INFO | ANALYSIS : 63.0%, 3.0, 98.1ms, 0 | 20260505 16:04:43.691 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:43.691 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:43.691 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:04:43.691 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:43.691 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:43.691 | INFO | Uploaded 36GiB to GPU in 2.13062s (16.8965GiB/s) | 20260505 16:04:43.691 | INFO | Downloaded 34GiB to GPU in 2.0267s (16.776GiB/s) | 20260505 16:04:48.824 | INFO | Analyzing chunk frames = [4502528,4503040) | 20260505 16:04:48.824 | INFO | MemInfo: { | 20260505 16:04:48.824 | INFO | "gpu" : " 17 GB", | 20260505 16:04:48.824 | INFO | "huge" : "", | 20260505 16:04:48.824 | INFO | "huge_shared" : " 27 GB", | 20260505 16:04:48.824 | INFO | "malloc" : " 14 GB", | 20260505 16:04:48.824 | INFO | "pinned" : " 29 GB" | 20260505 16:04:48.824 | INFO | } | 20260505 16:04:48.824 | INFO | Perf report for the last 5.13288 seconds: | 20260505 16:04:48.824 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:04:48.824 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:48.824 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:48.824 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:48.824 | INFO | ANALYSIS : 63.0%, 3.0, 98.0ms, 0 | 20260505 16:04:48.824 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:48.824 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:48.824 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:04:48.824 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:48.824 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:48.824 | INFO | Uploaded 36GiB to GPU in 2.17891s (16.522GiB/s) | 20260505 16:04:48.824 | INFO | Downloaded 34GiB to GPU in 2.06117s (16.4955GiB/s) | 20260505 16:04:53.965 | INFO | Analyzing chunk frames = [4503040,4503552) | 20260505 16:04:53.965 | INFO | MemInfo: { | 20260505 16:04:53.965 | INFO | "gpu" : " 17 GB", | 20260505 16:04:53.965 | INFO | "huge" : "", | 20260505 16:04:53.965 | INFO | "huge_shared" : " 27 GB", | 20260505 16:04:53.965 | INFO | "malloc" : " 14 GB", | 20260505 16:04:53.965 | INFO | "pinned" : " 29 GB" | 20260505 16:04:53.965 | INFO | } | 20260505 16:04:53.965 | INFO | Perf report for the last 5.14173 seconds: | 20260505 16:04:53.966 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:04:53.966 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:53.966 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:53.966 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:53.966 | INFO | ANALYSIS : 62.5%, 3.0, 97.1ms, 0 | 20260505 16:04:53.966 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:53.966 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:53.966 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:04:53.966 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:53.966 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:53.966 | INFO | Uploaded 36GiB to GPU in 2.17274s (16.569GiB/s) | 20260505 16:04:53.966 | INFO | Downloaded 34GiB to GPU in 2.13076s (15.9567GiB/s) | 20260505 16:04:59.109 | INFO | Analyzing chunk frames = [4503552,4504064) | 20260505 16:04:59.109 | INFO | MemInfo: { | 20260505 16:04:59.109 | INFO | "gpu" : " 17 GB", | 20260505 16:04:59.109 | INFO | "huge" : "", | 20260505 16:04:59.109 | INFO | "huge_shared" : " 27 GB", | 20260505 16:04:59.109 | INFO | "malloc" : " 14 GB", | 20260505 16:04:59.109 | INFO | "pinned" : " 29 GB" | 20260505 16:04:59.109 | INFO | } | 20260505 16:04:59.109 | INFO | Perf report for the last 5.14383 seconds: | 20260505 16:04:59.109 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:04:59.109 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:59.109 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:59.109 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:59.109 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260505 16:04:59.109 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:59.109 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:59.109 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:04:59.109 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:59.110 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:04:59.110 | INFO | Uploaded 36GiB to GPU in 2.15722s (16.6881GiB/s) | 20260505 16:04:59.110 | INFO | Downloaded 34GiB to GPU in 2.13302s (15.9399GiB/s) | 20260505 16:05:04.250 | INFO | Analyzing chunk frames = [4504064,4504576) | 20260505 16:05:04.250 | INFO | MemInfo: { | 20260505 16:05:04.250 | INFO | "gpu" : " 17 GB", | 20260505 16:05:04.250 | INFO | "huge" : "", | 20260505 16:05:04.250 | INFO | "huge_shared" : " 27 GB", | 20260505 16:05:04.250 | INFO | "malloc" : " 14 GB", | 20260505 16:05:04.250 | INFO | "pinned" : " 29 GB" | 20260505 16:05:04.250 | INFO | } | 20260505 16:05:04.250 | INFO | Perf report for the last 5.14075 seconds: | 20260505 16:05:04.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:05:04.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:04.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:04.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:04.250 | INFO | ANALYSIS : 62.5%, 3.0, 97.7ms, 0 | 20260505 16:05:04.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:04.250 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:04.250 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 16:05:04.250 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:04.250 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:04.250 | INFO | Uploaded 36GiB to GPU in 2.14017s (16.8211GiB/s) | 20260505 16:05:04.251 | INFO | Downloaded 34GiB to GPU in 2.03526s (16.7055GiB/s) | 20260505 16:05:09.387 | INFO | Analyzing chunk frames = [4504576,4505088) | 20260505 16:05:09.387 | INFO | MemInfo: { | 20260505 16:05:09.387 | INFO | "gpu" : " 17 GB", | 20260505 16:05:09.387 | INFO | "huge" : "", | 20260505 16:05:09.387 | INFO | "huge_shared" : " 27 GB", | 20260505 16:05:09.387 | INFO | "malloc" : " 14 GB", | 20260505 16:05:09.387 | INFO | "pinned" : " 29 GB" | 20260505 16:05:09.387 | INFO | } | 20260505 16:05:09.388 | INFO | Perf report for the last 5.13745 seconds: | 20260505 16:05:09.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:05:09.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:09.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:09.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:09.388 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 16:05:09.388 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:09.388 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:09.388 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 16:05:09.388 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:09.388 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:09.388 | INFO | Uploaded 36GiB to GPU in 2.10925s (17.0677GiB/s) | 20260505 16:05:09.388 | INFO | Downloaded 34GiB to GPU in 2.18378s (15.5693GiB/s) | 20260505 16:05:14.521 | INFO | Analyzing chunk frames = [4505088,4505600) | 20260505 16:05:14.521 | INFO | MemInfo: { | 20260505 16:05:14.521 | INFO | "gpu" : " 17 GB", | 20260505 16:05:14.521 | INFO | "huge" : "", | 20260505 16:05:14.521 | INFO | "huge_shared" : " 27 GB", | 20260505 16:05:14.521 | INFO | "malloc" : " 14 GB", | 20260505 16:05:14.521 | INFO | "pinned" : " 29 GB" | 20260505 16:05:14.521 | INFO | } | 20260505 16:05:14.521 | INFO | Perf report for the last 5.13304 seconds: | 20260505 16:05:14.521 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:05:14.521 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:14.521 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:14.521 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:14.521 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 16:05:14.521 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:14.521 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:14.521 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 16:05:14.521 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:14.521 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:14.521 | INFO | Uploaded 36GiB to GPU in 2.17416s (16.5581GiB/s) | 20260505 16:05:14.521 | INFO | Downloaded 34GiB to GPU in 2.03034s (16.746GiB/s) | 20260505 16:05:14.521 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 16:05:14.521 | INFO | Total mem usage: 15.3 GiB | 20260505 16:05:14.521 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 16:05:14.521 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.85%) | 20260505 16:05:14.521 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 16:05:14.521 | INFO | High Water Breakdown: | 20260505 16:05:14.521 | INFO | 61% : 8.67 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 16:05:14.521 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 16:05:14.521 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 16:05:14.521 | INFO | ------------------------------------------ | 20260505 16:05:14.521 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:05:14.521 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 16:05:14.521 | INFO | Total mem usage: 26.5 GiB | 20260505 16:05:14.521 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 16:05:14.521 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 16:05:14.521 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 16:05:14.521 | INFO | High Water Breakdown: | 20260505 16:05:14.521 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:05:14.521 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:05:14.521 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:05:14.521 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:05:14.521 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:05:14.521 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:05:14.521 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 16:05:14.521 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 16:05:14.521 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:05:14.521 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 16:05:14.522 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 16:05:14.522 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 16:05:14.522 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 16:05:14.522 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 16:05:14.522 | INFO | ------------------------------------------ | 20260505 16:05:14.522 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:05:14.522 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 16:05:14.522 | INFO | Total mem usage: 25.6 GiB | 20260505 16:05:14.522 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 16:05:14.522 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 16:05:14.522 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 16:05:14.522 | INFO | High Water Breakdown: | 20260505 16:05:14.522 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 16:05:14.522 | INFO | ------------------------------------------ | 20260505 16:05:14.522 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:05:14.522 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 16:05:14.522 | INFO | Total mem usage: 15.7 GiB | 20260505 16:05:14.522 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 16:05:14.522 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 16:05:14.522 | INFO | Unused Cache: 914 MiB (Waste: 5.7%) | 20260505 16:05:14.522 | INFO | High Water Breakdown: | 20260505 16:05:14.522 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 16:05:14.522 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 16:05:14.522 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 16:05:14.522 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 16:05:14.522 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 16:05:14.522 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 16:05:14.522 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 16:05:14.522 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 16:05:14.522 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 16:05:14.522 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:05:14.522 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 16:05:14.522 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:05:14.522 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 16:05:14.522 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 16:05:14.522 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 16:05:14.522 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 16:05:14.522 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:05:14.522 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 16:05:14.522 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:05:14.522 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:05:14.522 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:05:14.522 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 16:05:14.522 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 16:05:14.522 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:05:14.522 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 16:05:14.522 | INFO | ------------------------------------------ | 20260505 16:05:14.522 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:05:14.522 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 16:05:14.522 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:05:14.522 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:05:14.522 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.56e+01, 1.37e+01 / 2.46e+00 | 20260505 16:05:14.522 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 1.03e-01, 1.08e-02 / 9.03e-03 | 20260505 16:05:14.522 | INFO | PerfReport: 11.77%, Baseline -- 3.74e+00 / 4.47e+00, 5.77e-01 / 1.85e-01 | 20260505 16:05:14.522 | INFO | PerfReport: 3.30%, Histogram -- 1.05e+00 / 1.51e+00, 1.98e-01 / 1.14e-01 | 20260505 16:05:14.522 | INFO | PerfReport: 15.92%, DME -- 4.04e+01 / 4.51e+01, 4.34e+00 / 2.45e+00 | 20260505 16:05:14.522 | INFO | PerfReport: 56.74%, FrameLabeling -- 1.80e+01 / 1.97e+01, 1.51e+00 / 1.82e+00 | 20260505 16:05:14.522 | INFO | PerfReport: 7.35%, PulseAccumulating -- 2.33e+00 / 2.90e+00, 4.50e-01 / 1.99e-01 | 20260505 16:05:14.522 | INFO | PerfReport: 3.42%, Metrics -- 1.09e+00 / 1.41e+00, 3.03e-01 / 1.78e-01 | 20260505 16:05:14.522 | INFO | PerfReport: 1.20%, PulseSerializing -- 3.80e-01 / 4.35e-01, 6.87e-02 / 4.45e-02 | 20260505 16:05:14.522 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 16:05:14.522 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:05:14.522 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:05:14.522 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260505 16:05:14.522 | INFO | PerfReport: 22.44%, Upload -- 2.25e+01 / 0.00e+00, 7.87e+00 / 0.00e+00 | 20260505 16:05:14.522 | INFO | PerfReport: 21.90%, Download -- 2.19e+01 / 0.00e+00, 7.92e+00 / 0.00e+00 | 20260505 16:05:14.522 | INFO | PerfReport: 31.65%, Compute -- 3.17e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 16:05:14.522 | INFO | PerfReport: 2.20%, LockUpload -- 2.20e+00 / 0.00e+00, 7.85e+00 / 0.00e+00 | 20260505 16:05:14.522 | INFO | PerfReport: 1.50%, LockDownload -- 1.50e+00 / 0.00e+00, 3.11e+00 / 0.00e+00 | 20260505 16:05:14.522 | INFO | PerfReport: 20.29%, LockCompute -- 2.03e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 16:05:14.522 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.98e-02 / 0.00e+00 | 20260505 16:05:14.654 | INFO | Starting a round of Baz IO | 20260505 16:05:17.857 | INFO | Populating RTMetrics report | 20260505 16:05:17.857 | INFO | RTMetrics report populated | 20260505 16:05:17.857 | INFO | Opening tmp RTMetrics json file | 20260505 16:05:17.858 | INFO | Serializing RTMetrics to json | 20260505 16:05:17.859 | INFO | Serialization done, swapping tmp file now... | 20260505 16:05:17.860 | INFO | RTMetrics done | 20260505 16:05:17.995 | INFO | Finished a round of Baz IO: Wrote 2.186 GiB in 3.34 seconds | 20260505 16:05:17.995 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.32% / 0.00% / 77.68% | 20260505 16:05:17.995 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.392699% / 0% / 7.15256e-05% | 20260505 16:05:17.995 | INFO | Baz IO overall: | 20260505 16:05:17.995 | INFO | BytesWritten : 2676279138989 | 20260505 16:05:17.995 | INFO | NumEvents : 1270707669112 | 20260505 16:05:19.670 | INFO | Analyzing chunk frames = [4505600,4506112) | 20260505 16:05:19.670 | INFO | MemInfo: { | 20260505 16:05:19.670 | INFO | "gpu" : " 17 GB", | 20260505 16:05:19.670 | INFO | "huge" : "", | 20260505 16:05:19.670 | INFO | "huge_shared" : " 27 GB", | 20260505 16:05:19.670 | INFO | "malloc" : " 14 GB", | 20260505 16:05:19.670 | INFO | "pinned" : " 29 GB" | 20260505 16:05:19.670 | INFO | } | 20260505 16:05:19.670 | INFO | Perf report for the last 5.14902 seconds: | 20260505 16:05:19.670 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:05:19.670 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:19.670 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:19.670 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:19.670 | INFO | ANALYSIS : 63.7%, 3.0, 99.1ms, 0 | 20260505 16:05:19.670 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:19.670 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260505 16:05:19.670 | INFO | PRE_HQ : 60.5%, 1.8, 56.4ms, 0 | 20260505 16:05:19.670 | INFO | BAZWRITER : 64.2%, 4.0, 131.3ms, 0 | 20260505 16:05:19.670 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 16:05:19.670 | INFO | Uploaded 36GiB to GPU in 2.17253s (16.5706GiB/s) | 20260505 16:05:19.670 | INFO | Downloaded 38GiB to GPU in 2.2686s (16.7504GiB/s) | 20260505 16:05:24.807 | INFO | Analyzing chunk frames = [4506112,4506624) | 20260505 16:05:24.807 | INFO | MemInfo: { | 20260505 16:05:24.807 | INFO | "gpu" : " 17 GB", | 20260505 16:05:24.807 | INFO | "huge" : "", | 20260505 16:05:24.807 | INFO | "huge_shared" : " 27 GB", | 20260505 16:05:24.807 | INFO | "malloc" : " 14 GB", | 20260505 16:05:24.807 | INFO | "pinned" : " 29 GB" | 20260505 16:05:24.807 | INFO | } | 20260505 16:05:24.807 | INFO | Perf report for the last 5.13753 seconds: | 20260505 16:05:24.807 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:05:24.807 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:24.807 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:24.807 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:24.807 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 16:05:24.807 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:24.807 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:24.807 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:05:24.807 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:24.807 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:24.808 | INFO | Uploaded 36GiB to GPU in 2.11287s (17.0385GiB/s) | 20260505 16:05:24.808 | INFO | Downloaded 34GiB to GPU in 2.02232s (16.8124GiB/s) | 20260505 16:05:29.941 | INFO | Analyzing chunk frames = [4506624,4507136) | 20260505 16:05:29.941 | INFO | MemInfo: { | 20260505 16:05:29.941 | INFO | "gpu" : " 17 GB", | 20260505 16:05:29.941 | INFO | "huge" : "", | 20260505 16:05:29.941 | INFO | "huge_shared" : " 27 GB", | 20260505 16:05:29.941 | INFO | "malloc" : " 14 GB", | 20260505 16:05:29.941 | INFO | "pinned" : " 29 GB" | 20260505 16:05:29.941 | INFO | } | 20260505 16:05:29.941 | INFO | Perf report for the last 5.13404 seconds: | 20260505 16:05:29.941 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:05:29.941 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:29.941 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:29.941 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:29.941 | INFO | ANALYSIS : 62.8%, 3.0, 97.6ms, 0 | 20260505 16:05:29.941 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:29.941 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:29.941 | INFO | PRE_HQ : 27.9%, 1.0, 14.4ms, 0 | 20260505 16:05:29.942 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:29.942 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:29.942 | INFO | Uploaded 36GiB to GPU in 2.18095s (16.5066GiB/s) | 20260505 16:05:29.942 | INFO | Downloaded 34GiB to GPU in 2.03027s (16.7466GiB/s) | 20260505 16:05:35.080 | INFO | Analyzing chunk frames = [4507136,4507648) | 20260505 16:05:35.080 | INFO | MemInfo: { | 20260505 16:05:35.080 | INFO | "gpu" : " 17 GB", | 20260505 16:05:35.080 | INFO | "huge" : "", | 20260505 16:05:35.080 | INFO | "huge_shared" : " 27 GB", | 20260505 16:05:35.080 | INFO | "malloc" : " 14 GB", | 20260505 16:05:35.080 | INFO | "pinned" : " 29 GB" | 20260505 16:05:35.080 | INFO | } | 20260505 16:05:35.080 | INFO | Perf report for the last 5.13861 seconds: | 20260505 16:05:35.080 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:05:35.080 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:35.080 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:35.080 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:35.080 | INFO | ANALYSIS : 62.6%, 3.0, 97.2ms, 0 | 20260505 16:05:35.080 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:35.080 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:35.080 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 16:05:35.080 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:35.080 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:35.080 | INFO | Uploaded 36GiB to GPU in 2.18798s (16.4535GiB/s) | 20260505 16:05:35.080 | INFO | Downloaded 34GiB to GPU in 2.13251s (15.9437GiB/s) | 20260505 16:05:40.222 | INFO | Analyzing chunk frames = [4507648,4508160) | 20260505 16:05:40.222 | INFO | MemInfo: { | 20260505 16:05:40.222 | INFO | "gpu" : " 17 GB", | 20260505 16:05:40.222 | INFO | "huge" : "", | 20260505 16:05:40.222 | INFO | "huge_shared" : " 27 GB", | 20260505 16:05:40.222 | INFO | "malloc" : " 14 GB", | 20260505 16:05:40.222 | INFO | "pinned" : " 29 GB" | 20260505 16:05:40.222 | INFO | } | 20260505 16:05:40.222 | INFO | Perf report for the last 5.14249 seconds: | 20260505 16:05:40.223 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:05:40.223 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:40.223 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:40.223 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:40.223 | INFO | ANALYSIS : 62.6%, 3.0, 97.9ms, 0 | 20260505 16:05:40.223 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:40.223 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:40.223 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:05:40.223 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:40.223 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:40.223 | INFO | Uploaded 36GiB to GPU in 2.17297s (16.5672GiB/s) | 20260505 16:05:40.223 | INFO | Downloaded 34GiB to GPU in 2.1307s (15.9572GiB/s) | 20260505 16:05:45.368 | INFO | Analyzing chunk frames = [4508160,4508672) | 20260505 16:05:45.368 | INFO | MemInfo: { | 20260505 16:05:45.368 | INFO | "gpu" : " 17 GB", | 20260505 16:05:45.368 | INFO | "huge" : "", | 20260505 16:05:45.368 | INFO | "huge_shared" : " 27 GB", | 20260505 16:05:45.368 | INFO | "malloc" : " 14 GB", | 20260505 16:05:45.368 | INFO | "pinned" : " 29 GB" | 20260505 16:05:45.368 | INFO | } | 20260505 16:05:45.368 | INFO | Perf report for the last 5.14591 seconds: | 20260505 16:05:45.368 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:05:45.369 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:45.369 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:45.369 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:45.369 | INFO | ANALYSIS : 62.5%, 3.0, 97.7ms, 0 | 20260505 16:05:45.369 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:45.369 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:45.369 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260505 16:05:45.369 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:45.369 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:45.369 | INFO | Uploaded 36GiB to GPU in 2.13375s (16.8717GiB/s) | 20260505 16:05:45.369 | INFO | Downloaded 34GiB to GPU in 2.0361s (16.6986GiB/s) | 20260505 16:05:50.512 | INFO | Analyzing chunk frames = [4508672,4509184) | 20260505 16:05:50.512 | INFO | MemInfo: { | 20260505 16:05:50.512 | INFO | "gpu" : " 17 GB", | 20260505 16:05:50.512 | INFO | "huge" : "", | 20260505 16:05:50.512 | INFO | "huge_shared" : " 27 GB", | 20260505 16:05:50.512 | INFO | "malloc" : " 14 GB", | 20260505 16:05:50.512 | INFO | "pinned" : " 29 GB" | 20260505 16:05:50.512 | INFO | } | 20260505 16:05:50.512 | INFO | Perf report for the last 5.14349 seconds: | 20260505 16:05:50.512 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:05:50.512 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:50.512 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:50.512 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:50.512 | INFO | ANALYSIS : 63.3%, 3.0, 99.0ms, 0 | 20260505 16:05:50.512 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:50.512 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:50.512 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 16:05:50.512 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:50.512 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:50.512 | INFO | Uploaded 36GiB to GPU in 2.14121s (16.8129GiB/s) | 20260505 16:05:50.512 | INFO | Downloaded 34GiB to GPU in 2.14439s (15.8553GiB/s) | 20260505 16:05:55.635 | INFO | Analyzing chunk frames = [4509184,4509696) | 20260505 16:05:55.635 | INFO | MemInfo: { | 20260505 16:05:55.635 | INFO | "gpu" : " 17 GB", | 20260505 16:05:55.635 | INFO | "huge" : "", | 20260505 16:05:55.635 | INFO | "huge_shared" : " 27 GB", | 20260505 16:05:55.636 | INFO | "malloc" : " 14 GB", | 20260505 16:05:55.636 | INFO | "pinned" : " 29 GB" | 20260505 16:05:55.636 | INFO | } | 20260505 16:05:55.636 | INFO | Perf report for the last 5.12384 seconds: | 20260505 16:05:55.636 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:05:55.636 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:55.636 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:55.636 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:55.636 | INFO | ANALYSIS : 63.2%, 3.0, 98.3ms, 0 | 20260505 16:05:55.636 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:55.636 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:55.636 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 16:05:55.636 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:55.636 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:05:55.636 | INFO | Uploaded 36GiB to GPU in 2.16769s (16.6075GiB/s) | 20260505 16:05:55.636 | INFO | Downloaded 34GiB to GPU in 2.02085s (16.8246GiB/s) | 20260505 16:05:55.771 | INFO | Starting a round of Baz IO | 20260505 16:05:58.985 | INFO | Populating RTMetrics report | 20260505 16:05:58.986 | INFO | RTMetrics report populated | 20260505 16:05:58.986 | INFO | Opening tmp RTMetrics json file | 20260505 16:05:59.000 | INFO | Serializing RTMetrics to json | 20260505 16:05:59.002 | INFO | Serialization done, swapping tmp file now... | 20260505 16:05:59.002 | INFO | RTMetrics done | 20260505 16:05:59.089 | INFO | Finished a round of Baz IO: Wrote 2.186 GiB in 3.317 seconds | 20260505 16:05:59.089 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.32% / 0.00% / 77.68% | 20260505 16:05:59.089 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.391571% / 0% / 6.35783e-05% | 20260505 16:05:59.089 | INFO | Baz IO overall: | 20260505 16:05:59.089 | INFO | BytesWritten : 2678625987343 | 20260505 16:05:59.089 | INFO | NumEvents : 1271809131062 | 20260505 16:06:00.774 | INFO | Analyzing chunk frames = [4509696,4510208) | 20260505 16:06:00.774 | INFO | MemInfo: { | 20260505 16:06:00.774 | INFO | "gpu" : " 17 GB", | 20260505 16:06:00.774 | INFO | "huge" : "", | 20260505 16:06:00.774 | INFO | "huge_shared" : " 27 GB", | 20260505 16:06:00.774 | INFO | "malloc" : " 14 GB", | 20260505 16:06:00.774 | INFO | "pinned" : " 29 GB" | 20260505 16:06:00.774 | INFO | } | 20260505 16:06:00.774 | INFO | Perf report for the last 5.13807 seconds: | 20260505 16:06:00.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:06:00.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:00.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:00.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:00.774 | INFO | ANALYSIS : 64.0%, 3.0, 99.7ms, 0 | 20260505 16:06:00.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:00.774 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260505 16:06:00.774 | INFO | PRE_HQ : 60.7%, 1.8, 56.7ms, 0 | 20260505 16:06:00.774 | INFO | BAZWRITER : 63.9%, 3.9, 129.1ms, 0 | 20260505 16:06:00.774 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 16:06:00.774 | INFO | Uploaded 36GiB to GPU in 2.21161s (16.2778GiB/s) | 20260505 16:06:00.774 | INFO | Downloaded 38GiB to GPU in 2.26939s (16.7446GiB/s) | 20260505 16:06:05.923 | INFO | Analyzing chunk frames = [4510208,4510720) | 20260505 16:06:05.923 | INFO | MemInfo: { | 20260505 16:06:05.923 | INFO | "gpu" : " 17 GB", | 20260505 16:06:05.923 | INFO | "huge" : "", | 20260505 16:06:05.923 | INFO | "huge_shared" : " 27 GB", | 20260505 16:06:05.923 | INFO | "malloc" : " 14 GB", | 20260505 16:06:05.923 | INFO | "pinned" : " 29 GB" | 20260505 16:06:05.923 | INFO | } | 20260505 16:06:05.923 | INFO | Perf report for the last 5.14897 seconds: | 20260505 16:06:05.923 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:06:05.923 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:05.923 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:05.923 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:05.923 | INFO | ANALYSIS : 63.3%, 3.0, 99.0ms, 0 | 20260505 16:06:05.923 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:05.923 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:05.923 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:06:05.923 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:05.923 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:05.923 | INFO | Uploaded 36GiB to GPU in 2.20228s (16.3467GiB/s) | 20260505 16:06:05.923 | INFO | Downloaded 34GiB to GPU in 2.05163s (16.5722GiB/s) | 20260505 16:06:11.062 | INFO | Analyzing chunk frames = [4510720,4511232) | 20260505 16:06:11.063 | INFO | MemInfo: { | 20260505 16:06:11.063 | INFO | "gpu" : " 17 GB", | 20260505 16:06:11.063 | INFO | "huge" : "", | 20260505 16:06:11.063 | INFO | "huge_shared" : " 27 GB", | 20260505 16:06:11.063 | INFO | "malloc" : " 14 GB", | 20260505 16:06:11.063 | INFO | "pinned" : " 29 GB" | 20260505 16:06:11.063 | INFO | } | 20260505 16:06:11.063 | INFO | Perf report for the last 5.14004 seconds: | 20260505 16:06:11.063 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:06:11.063 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:11.063 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:11.063 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:11.063 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260505 16:06:11.063 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:11.063 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:11.063 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:06:11.063 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:11.063 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:11.063 | INFO | Uploaded 36GiB to GPU in 2.20184s (16.35GiB/s) | 20260505 16:06:11.063 | INFO | Downloaded 34GiB to GPU in 2.03025s (16.7467GiB/s) | 20260505 16:06:16.195 | INFO | Analyzing chunk frames = [4511232,4511744) | 20260505 16:06:16.195 | INFO | MemInfo: { | 20260505 16:06:16.195 | INFO | "gpu" : " 17 GB", | 20260505 16:06:16.195 | INFO | "huge" : "", | 20260505 16:06:16.195 | INFO | "huge_shared" : " 27 GB", | 20260505 16:06:16.195 | INFO | "malloc" : " 14 GB", | 20260505 16:06:16.195 | INFO | "pinned" : " 29 GB" | 20260505 16:06:16.195 | INFO | } | 20260505 16:06:16.195 | INFO | Perf report for the last 5.13238 seconds: | 20260505 16:06:16.195 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:06:16.195 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:16.195 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:16.195 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:16.195 | INFO | ANALYSIS : 62.5%, 3.0, 97.0ms, 0 | 20260505 16:06:16.195 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:16.195 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:16.195 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 16:06:16.195 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:16.195 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:16.196 | INFO | Uploaded 36GiB to GPU in 2.17955s (16.5172GiB/s) | 20260505 16:06:16.196 | INFO | Downloaded 34GiB to GPU in 2.13519s (15.9236GiB/s) | 20260505 16:06:21.333 | INFO | Analyzing chunk frames = [4511744,4512256) | 20260505 16:06:21.333 | INFO | MemInfo: { | 20260505 16:06:21.333 | INFO | "gpu" : " 17 GB", | 20260505 16:06:21.333 | INFO | "huge" : "", | 20260505 16:06:21.333 | INFO | "huge_shared" : " 27 GB", | 20260505 16:06:21.333 | INFO | "malloc" : " 14 GB", | 20260505 16:06:21.333 | INFO | "pinned" : " 29 GB" | 20260505 16:06:21.333 | INFO | } | 20260505 16:06:21.333 | INFO | Perf report for the last 5.13747 seconds: | 20260505 16:06:21.333 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:06:21.333 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:21.333 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:21.333 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:21.333 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 16:06:21.333 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:21.333 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:21.333 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 16:06:21.333 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:21.333 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:21.333 | INFO | Uploaded 36GiB to GPU in 2.18765s (16.456GiB/s) | 20260505 16:06:21.333 | INFO | Downloaded 34GiB to GPU in 2.12971s (15.9646GiB/s) | 20260505 16:06:26.477 | INFO | Analyzing chunk frames = [4512256,4512768) | 20260505 16:06:26.477 | INFO | MemInfo: { | 20260505 16:06:26.477 | INFO | "gpu" : " 17 GB", | 20260505 16:06:26.477 | INFO | "huge" : "", | 20260505 16:06:26.477 | INFO | "huge_shared" : " 27 GB", | 20260505 16:06:26.477 | INFO | "malloc" : " 14 GB", | 20260505 16:06:26.477 | INFO | "pinned" : " 29 GB" | 20260505 16:06:26.477 | INFO | } | 20260505 16:06:26.477 | INFO | Perf report for the last 5.14433 seconds: | 20260505 16:06:26.477 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:06:26.477 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:26.477 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:26.477 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:26.477 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 16:06:26.477 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:26.477 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:26.477 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 16:06:26.477 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:26.477 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:26.477 | INFO | Uploaded 36GiB to GPU in 2.20144s (16.3529GiB/s) | 20260505 16:06:26.477 | INFO | Downloaded 34GiB to GPU in 2.10897s (16.1216GiB/s) | 20260505 16:06:31.618 | INFO | Analyzing chunk frames = [4512768,4513280) | 20260505 16:06:31.618 | INFO | MemInfo: { | 20260505 16:06:31.618 | INFO | "gpu" : " 17 GB", | 20260505 16:06:31.618 | INFO | "huge" : "", | 20260505 16:06:31.618 | INFO | "huge_shared" : " 27 GB", | 20260505 16:06:31.618 | INFO | "malloc" : " 14 GB", | 20260505 16:06:31.618 | INFO | "pinned" : " 29 GB" | 20260505 16:06:31.618 | INFO | } | 20260505 16:06:31.618 | INFO | Perf report for the last 5.14151 seconds: | 20260505 16:06:31.619 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:06:31.619 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:31.619 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:31.619 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:31.619 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 16:06:31.619 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:31.619 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:31.619 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 16:06:31.619 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:31.619 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:31.619 | INFO | Uploaded 36GiB to GPU in 2.12135s (16.9703GiB/s) | 20260505 16:06:31.619 | INFO | Downloaded 34GiB to GPU in 2.17435s (15.6368GiB/s) | 20260505 16:06:36.755 | INFO | Analyzing chunk frames = [4513280,4513792) | 20260505 16:06:36.755 | INFO | MemInfo: { | 20260505 16:06:36.755 | INFO | "gpu" : " 17 GB", | 20260505 16:06:36.755 | INFO | "huge" : "", | 20260505 16:06:36.755 | INFO | "huge_shared" : " 27 GB", | 20260505 16:06:36.755 | INFO | "malloc" : " 14 GB", | 20260505 16:06:36.755 | INFO | "pinned" : " 29 GB" | 20260505 16:06:36.755 | INFO | } | 20260505 16:06:36.755 | INFO | Perf report for the last 5.1366 seconds: | 20260505 16:06:36.755 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:06:36.755 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:36.755 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:36.755 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:36.755 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 16:06:36.755 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:36.755 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:36.755 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 16:06:36.755 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:36.755 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:36.755 | INFO | Uploaded 36GiB to GPU in 2.17685s (16.5377GiB/s) | 20260505 16:06:36.755 | INFO | Downloaded 34GiB to GPU in 2.03087s (16.7416GiB/s) | 20260505 16:06:36.755 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 16:06:36.755 | INFO | Total mem usage: 15.3 GiB | 20260505 16:06:36.755 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 16:06:36.755 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -7.52%) | 20260505 16:06:36.755 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 16:06:36.755 | INFO | High Water Breakdown: | 20260505 16:06:36.755 | INFO | 61.1% : 8.71 GiB ( 91.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 16:06:36.755 | INFO | 38.7% : 5.52 GiB ( 83.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 16:06:36.755 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 16:06:36.755 | INFO | ------------------------------------------ | 20260505 16:06:36.755 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:06:36.756 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 16:06:36.756 | INFO | Total mem usage: 26.5 GiB | 20260505 16:06:36.756 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 16:06:36.756 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 16:06:36.756 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 16:06:36.756 | INFO | High Water Breakdown: | 20260505 16:06:36.756 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:06:36.756 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:06:36.756 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:06:36.756 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:06:36.756 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:06:36.756 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:06:36.756 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 16:06:36.756 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 16:06:36.756 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:06:36.756 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 16:06:36.756 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 16:06:36.756 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 16:06:36.756 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 16:06:36.756 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 16:06:36.756 | INFO | ------------------------------------------ | 20260505 16:06:36.756 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:06:36.756 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 16:06:36.756 | INFO | Total mem usage: 25.6 GiB | 20260505 16:06:36.756 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 16:06:36.756 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 16:06:36.756 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 16:06:36.756 | INFO | High Water Breakdown: | 20260505 16:06:36.756 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 16:06:36.756 | INFO | ------------------------------------------ | 20260505 16:06:36.756 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:06:36.756 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 16:06:36.756 | INFO | Total mem usage: 15.7 GiB | 20260505 16:06:36.756 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 16:06:36.756 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 16:06:36.756 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 16:06:36.756 | INFO | High Water Breakdown: | 20260505 16:06:36.756 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 16:06:36.756 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 16:06:36.756 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 16:06:36.756 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 16:06:36.756 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 16:06:36.756 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 16:06:36.756 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 16:06:36.756 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 16:06:36.756 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 16:06:36.756 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:06:36.756 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 16:06:36.756 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:06:36.756 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 16:06:36.756 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 16:06:36.756 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 16:06:36.756 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 16:06:36.756 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:06:36.756 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 16:06:36.756 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:06:36.756 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:06:36.756 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:06:36.756 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 16:06:36.756 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 16:06:36.756 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:06:36.756 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 16:06:36.756 | INFO | ------------------------------------------ | 20260505 16:06:36.757 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:06:36.757 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 16:06:36.757 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:06:36.757 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:06:36.757 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.55e+01, 1.36e+01 / 1.57e+00 | 20260505 16:06:36.757 | INFO | PerfReport: 0.29%, AggregateStats -- 9.26e-02 / 1.01e-01, 1.05e-02 / 3.39e-03 | 20260505 16:06:36.757 | INFO | PerfReport: 11.81%, Baseline -- 3.74e+00 / 4.49e+00, 5.82e-01 / 1.56e-01 | 20260505 16:06:36.757 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.52e+00, 1.94e-01 / 8.14e-02 | 20260505 16:06:36.757 | INFO | PerfReport: 15.92%, DME -- 4.03e+01 / 4.49e+01, 4.29e+00 / 1.97e+00 | 20260505 16:06:36.757 | INFO | PerfReport: 56.67%, FrameLabeling -- 1.79e+01 / 1.97e+01, 1.43e+00 / 1.46e+00 | 20260505 16:06:36.757 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.34e+00 / 2.90e+00, 4.46e-01 / 1.56e-01 | 20260505 16:06:36.757 | INFO | PerfReport: 3.45%, Metrics -- 1.09e+00 / 1.42e+00, 3.03e-01 / 1.44e-01 | 20260505 16:06:36.757 | INFO | PerfReport: 1.20%, PulseSerializing -- 3.79e-01 / 4.37e-01, 6.72e-02 / 3.55e-02 | 20260505 16:06:36.757 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 16:06:36.757 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:06:36.757 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:06:36.757 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260505 16:06:36.757 | INFO | PerfReport: 22.56%, Upload -- 2.26e+01 / 0.00e+00, 7.94e+00 / 0.00e+00 | 20260505 16:06:36.757 | INFO | PerfReport: 21.89%, Download -- 2.20e+01 / 0.00e+00, 7.95e+00 / 0.00e+00 | 20260505 16:06:36.757 | INFO | PerfReport: 31.58%, Compute -- 3.17e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 16:06:36.757 | INFO | PerfReport: 2.25%, LockUpload -- 2.26e+00 / 0.00e+00, 7.94e+00 / 0.00e+00 | 20260505 16:06:36.757 | INFO | PerfReport: 1.49%, LockDownload -- 1.50e+00 / 0.00e+00, 3.08e+00 / 0.00e+00 | 20260505 16:06:36.757 | INFO | PerfReport: 20.20%, LockCompute -- 2.03e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 16:06:36.757 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.74e-02 / 0.00e+00 | 20260505 16:06:36.888 | INFO | Starting a round of Baz IO | 20260505 16:06:40.099 | INFO | Populating RTMetrics report | 20260505 16:06:40.099 | INFO | RTMetrics report populated | 20260505 16:06:40.099 | INFO | Opening tmp RTMetrics json file | 20260505 16:06:40.139 | INFO | Serializing RTMetrics to json | 20260505 16:06:40.173 | INFO | Serialization done, swapping tmp file now... | 20260505 16:06:40.185 | INFO | RTMetrics done | 20260505 16:06:40.201 | INFO | Finished a round of Baz IO: Wrote 2.188 GiB in 3.312 seconds | 20260505 16:06:40.201 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.32% / 0.00% / 77.68% | 20260505 16:06:40.201 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.391575% / 0% / 4.37101e-05% | 20260505 16:06:40.201 | INFO | Baz IO overall: | 20260505 16:06:40.201 | INFO | BytesWritten : 2680974881136 | 20260505 16:06:40.201 | INFO | NumEvents : 1272912035528 | 20260505 16:06:41.896 | INFO | Analyzing chunk frames = [4513792,4514304) | 20260505 16:06:41.896 | INFO | MemInfo: { | 20260505 16:06:41.896 | INFO | "gpu" : " 17 GB", | 20260505 16:06:41.896 | INFO | "huge" : "", | 20260505 16:06:41.896 | INFO | "huge_shared" : " 27 GB", | 20260505 16:06:41.896 | INFO | "malloc" : " 14 GB", | 20260505 16:06:41.896 | INFO | "pinned" : " 29 GB" | 20260505 16:06:41.896 | INFO | } | 20260505 16:06:41.896 | INFO | Perf report for the last 5.14116 seconds: | 20260505 16:06:41.896 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:06:41.897 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:41.897 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:41.897 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:41.897 | INFO | ANALYSIS : 64.0%, 3.0, 99.5ms, 0 | 20260505 16:06:41.897 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:41.897 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260505 16:06:41.897 | INFO | PRE_HQ : 60.9%, 1.8, 56.8ms, 0 | 20260505 16:06:41.897 | INFO | BAZWRITER : 63.8%, 3.9, 128.9ms, 0 | 20260505 16:06:41.897 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 16:06:41.897 | INFO | Uploaded 36GiB to GPU in 2.19439s (16.4055GiB/s) | 20260505 16:06:41.897 | INFO | Downloaded 38GiB to GPU in 2.25675s (16.8384GiB/s) | 20260505 16:06:47.032 | INFO | Analyzing chunk frames = [4514304,4514816) | 20260505 16:06:47.032 | INFO | MemInfo: { | 20260505 16:06:47.032 | INFO | "gpu" : " 17 GB", | 20260505 16:06:47.032 | INFO | "huge" : "", | 20260505 16:06:47.032 | INFO | "huge_shared" : " 27 GB", | 20260505 16:06:47.032 | INFO | "malloc" : " 14 GB", | 20260505 16:06:47.032 | INFO | "pinned" : " 29 GB" | 20260505 16:06:47.032 | INFO | } | 20260505 16:06:47.032 | INFO | Perf report for the last 5.13534 seconds: | 20260505 16:06:47.032 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:06:47.032 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:47.032 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:47.032 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:47.032 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 16:06:47.032 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:47.032 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:47.032 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 16:06:47.032 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:47.032 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:47.032 | INFO | Uploaded 36GiB to GPU in 2.20647s (16.3156GiB/s) | 20260505 16:06:47.032 | INFO | Downloaded 34GiB to GPU in 2.06616s (16.4557GiB/s) | 20260505 16:06:52.176 | INFO | Analyzing chunk frames = [4514816,4515328) | 20260505 16:06:52.176 | INFO | MemInfo: { | 20260505 16:06:52.176 | INFO | "gpu" : " 17 GB", | 20260505 16:06:52.176 | INFO | "huge" : "", | 20260505 16:06:52.176 | INFO | "huge_shared" : " 27 GB", | 20260505 16:06:52.176 | INFO | "malloc" : " 14 GB", | 20260505 16:06:52.176 | INFO | "pinned" : " 29 GB" | 20260505 16:06:52.176 | INFO | } | 20260505 16:06:52.176 | INFO | Perf report for the last 5.14434 seconds: | 20260505 16:06:52.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:06:52.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:52.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:52.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:52.176 | INFO | ANALYSIS : 62.9%, 3.0, 97.9ms, 0 | 20260505 16:06:52.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:52.176 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:52.176 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 16:06:52.176 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:52.176 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:52.176 | INFO | Uploaded 36GiB to GPU in 2.18073s (16.5083GiB/s) | 20260505 16:06:52.176 | INFO | Downloaded 34GiB to GPU in 2.05734s (16.5262GiB/s) | 20260505 16:06:57.321 | INFO | Analyzing chunk frames = [4515328,4515840) | 20260505 16:06:57.321 | INFO | MemInfo: { | 20260505 16:06:57.321 | INFO | "gpu" : " 17 GB", | 20260505 16:06:57.321 | INFO | "huge" : "", | 20260505 16:06:57.321 | INFO | "huge_shared" : " 27 GB", | 20260505 16:06:57.322 | INFO | "malloc" : " 14 GB", | 20260505 16:06:57.322 | INFO | "pinned" : " 29 GB" | 20260505 16:06:57.322 | INFO | } | 20260505 16:06:57.322 | INFO | Perf report for the last 5.14568 seconds: | 20260505 16:06:57.322 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:06:57.322 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:57.322 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:57.322 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:57.322 | INFO | ANALYSIS : 62.6%, 3.0, 97.3ms, 0 | 20260505 16:06:57.322 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:57.322 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:57.322 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:06:57.322 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:57.322 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:06:57.322 | INFO | Uploaded 36GiB to GPU in 2.17171s (16.5768GiB/s) | 20260505 16:06:57.322 | INFO | Downloaded 34GiB to GPU in 2.14083s (15.8817GiB/s) | 20260505 16:07:02.455 | INFO | Analyzing chunk frames = [4515840,4516352) | 20260505 16:07:02.455 | INFO | MemInfo: { | 20260505 16:07:02.455 | INFO | "gpu" : " 17 GB", | 20260505 16:07:02.455 | INFO | "huge" : "", | 20260505 16:07:02.455 | INFO | "huge_shared" : " 27 GB", | 20260505 16:07:02.455 | INFO | "malloc" : " 14 GB", | 20260505 16:07:02.455 | INFO | "pinned" : " 29 GB" | 20260505 16:07:02.455 | INFO | } | 20260505 16:07:02.455 | INFO | Perf report for the last 5.13362 seconds: | 20260505 16:07:02.455 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:07:02.455 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:02.455 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:02.455 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:02.455 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 16:07:02.455 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:02.455 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:02.455 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:07:02.455 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:02.455 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:02.456 | INFO | Uploaded 36GiB to GPU in 2.14858s (16.7552GiB/s) | 20260505 16:07:02.456 | INFO | Downloaded 34GiB to GPU in 2.0543s (16.5507GiB/s) | 20260505 16:07:07.584 | INFO | Analyzing chunk frames = [4516352,4516864) | 20260505 16:07:07.584 | INFO | MemInfo: { | 20260505 16:07:07.584 | INFO | "gpu" : " 17 GB", | 20260505 16:07:07.584 | INFO | "huge" : "", | 20260505 16:07:07.584 | INFO | "huge_shared" : " 27 GB", | 20260505 16:07:07.584 | INFO | "malloc" : " 14 GB", | 20260505 16:07:07.584 | INFO | "pinned" : " 29 GB" | 20260505 16:07:07.584 | INFO | } | 20260505 16:07:07.584 | INFO | Perf report for the last 5.12855 seconds: | 20260505 16:07:07.584 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:07:07.584 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:07.584 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:07.584 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:07.584 | INFO | ANALYSIS : 62.8%, 3.0, 97.8ms, 0 | 20260505 16:07:07.584 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:07.584 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:07.584 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 16:07:07.584 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:07.584 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:07.584 | INFO | Uploaded 36GiB to GPU in 2.16266s (16.6462GiB/s) | 20260505 16:07:07.584 | INFO | Downloaded 34GiB to GPU in 2.04058s (16.6619GiB/s) | 20260505 16:07:12.732 | INFO | Analyzing chunk frames = [4516864,4517376) | 20260505 16:07:12.732 | INFO | MemInfo: { | 20260505 16:07:12.732 | INFO | "gpu" : " 17 GB", | 20260505 16:07:12.732 | INFO | "huge" : "", | 20260505 16:07:12.732 | INFO | "huge_shared" : " 27 GB", | 20260505 16:07:12.732 | INFO | "malloc" : " 14 GB", | 20260505 16:07:12.732 | INFO | "pinned" : " 29 GB" | 20260505 16:07:12.732 | INFO | } | 20260505 16:07:12.732 | INFO | Perf report for the last 5.14838 seconds: | 20260505 16:07:12.732 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:07:12.732 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:12.732 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:12.732 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:12.732 | INFO | ANALYSIS : 63.0%, 3.0, 98.6ms, 0 | 20260505 16:07:12.732 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:12.732 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:12.732 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 16:07:12.732 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:12.732 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:12.733 | INFO | Uploaded 36GiB to GPU in 2.16876s (16.5994GiB/s) | 20260505 16:07:12.733 | INFO | Downloaded 34GiB to GPU in 2.15244s (15.7961GiB/s) | 20260505 16:07:17.881 | INFO | Analyzing chunk frames = [4517376,4517888) | 20260505 16:07:17.881 | INFO | MemInfo: { | 20260505 16:07:17.881 | INFO | "gpu" : " 17 GB", | 20260505 16:07:17.881 | INFO | "huge" : "", | 20260505 16:07:17.881 | INFO | "huge_shared" : " 27 GB", | 20260505 16:07:17.881 | INFO | "malloc" : " 14 GB", | 20260505 16:07:17.881 | INFO | "pinned" : " 29 GB" | 20260505 16:07:17.882 | INFO | } | 20260505 16:07:17.882 | INFO | Perf report for the last 5.14946 seconds: | 20260505 16:07:17.882 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:07:17.882 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:17.882 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:17.882 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:17.882 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 16:07:17.882 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:17.882 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:17.882 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 16:07:17.882 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:17.882 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:17.882 | INFO | Uploaded 36GiB to GPU in 2.15993s (16.6672GiB/s) | 20260505 16:07:17.882 | INFO | Downloaded 34GiB to GPU in 2.03105s (16.7401GiB/s) | 20260505 16:07:18.019 | INFO | Starting a round of Baz IO | 20260505 16:07:21.211 | INFO | Populating RTMetrics report | 20260505 16:07:21.211 | INFO | RTMetrics report populated | 20260505 16:07:21.211 | INFO | Opening tmp RTMetrics json file | 20260505 16:07:21.220 | INFO | Serializing RTMetrics to json | 20260505 16:07:21.224 | INFO | Serialization done, swapping tmp file now... | 20260505 16:07:21.226 | INFO | RTMetrics done | 20260505 16:07:21.322 | INFO | Finished a round of Baz IO: Wrote 2.184 GiB in 3.302 seconds | 20260505 16:07:21.322 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.31% / 0.00% / 77.68% | 20260505 16:07:21.322 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.387784% / 0% / 9.53674e-05% | 20260505 16:07:21.322 | INFO | Baz IO overall: | 20260505 16:07:21.322 | INFO | BytesWritten : 2683320095819 | 20260505 16:07:21.322 | INFO | NumEvents : 1274012361265 | 20260505 16:07:23.012 | INFO | Analyzing chunk frames = [4517888,4518400) | 20260505 16:07:23.012 | INFO | MemInfo: { | 20260505 16:07:23.012 | INFO | "gpu" : " 17 GB", | 20260505 16:07:23.012 | INFO | "huge" : "", | 20260505 16:07:23.012 | INFO | "huge_shared" : " 27 GB", | 20260505 16:07:23.012 | INFO | "malloc" : " 14 GB", | 20260505 16:07:23.012 | INFO | "pinned" : " 29 GB" | 20260505 16:07:23.012 | INFO | } | 20260505 16:07:23.013 | INFO | Perf report for the last 5.13088 seconds: | 20260505 16:07:23.013 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:07:23.013 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:23.013 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:23.013 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:23.013 | INFO | ANALYSIS : 63.8%, 3.0, 99.1ms, 0 | 20260505 16:07:23.013 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:23.013 | INFO | RT_METRICS : 45.1%, 1.0, 23.6ms, 0 | 20260505 16:07:23.013 | INFO | PRE_HQ : 60.5%, 1.8, 56.5ms, 0 | 20260505 16:07:23.013 | INFO | BAZWRITER : 63.9%, 3.9, 129.2ms, 0 | 20260505 16:07:23.013 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 16:07:23.013 | INFO | Uploaded 36GiB to GPU in 2.1649s (16.6289GiB/s) | 20260505 16:07:23.013 | INFO | Downloaded 38GiB to GPU in 2.24954s (16.8924GiB/s) | 20260505 16:07:28.146 | INFO | Analyzing chunk frames = [4518400,4518912) | 20260505 16:07:28.146 | INFO | MemInfo: { | 20260505 16:07:28.146 | INFO | "gpu" : " 17 GB", | 20260505 16:07:28.146 | INFO | "huge" : "", | 20260505 16:07:28.146 | INFO | "huge_shared" : " 27 GB", | 20260505 16:07:28.146 | INFO | "malloc" : " 14 GB", | 20260505 16:07:28.146 | INFO | "pinned" : " 29 GB" | 20260505 16:07:28.146 | INFO | } | 20260505 16:07:28.146 | INFO | Perf report for the last 5.1336 seconds: | 20260505 16:07:28.146 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:07:28.146 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:28.146 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:28.146 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:28.146 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 16:07:28.146 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:28.146 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:28.146 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:07:28.146 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:28.146 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:28.146 | INFO | Uploaded 36GiB to GPU in 2.13648s (16.8502GiB/s) | 20260505 16:07:28.147 | INFO | Downloaded 34GiB to GPU in 2.03254s (16.7279GiB/s) | 20260505 16:07:33.284 | INFO | Analyzing chunk frames = [4518912,4519424) | 20260505 16:07:33.284 | INFO | MemInfo: { | 20260505 16:07:33.284 | INFO | "gpu" : " 17 GB", | 20260505 16:07:33.284 | INFO | "huge" : "", | 20260505 16:07:33.284 | INFO | "huge_shared" : " 27 GB", | 20260505 16:07:33.284 | INFO | "malloc" : " 14 GB", | 20260505 16:07:33.284 | INFO | "pinned" : " 29 GB" | 20260505 16:07:33.284 | INFO | } | 20260505 16:07:33.284 | INFO | Perf report for the last 5.13811 seconds: | 20260505 16:07:33.284 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:07:33.284 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:33.284 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:33.284 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:33.284 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260505 16:07:33.285 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:33.285 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:33.285 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:07:33.285 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:33.285 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:33.285 | INFO | Uploaded 36GiB to GPU in 2.24175s (16.0589GiB/s) | 20260505 16:07:33.285 | INFO | Downloaded 34GiB to GPU in 2.11587s (16.069GiB/s) | 20260505 16:07:38.432 | INFO | Analyzing chunk frames = [4519424,4519936) | 20260505 16:07:38.432 | INFO | MemInfo: { | 20260505 16:07:38.432 | INFO | "gpu" : " 17 GB", | 20260505 16:07:38.432 | INFO | "huge" : "", | 20260505 16:07:38.432 | INFO | "huge_shared" : " 27 GB", | 20260505 16:07:38.432 | INFO | "malloc" : " 14 GB", | 20260505 16:07:38.432 | INFO | "pinned" : " 29 GB" | 20260505 16:07:38.432 | INFO | } | 20260505 16:07:38.432 | INFO | Perf report for the last 5.14795 seconds: | 20260505 16:07:38.432 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:07:38.432 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:38.432 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:38.432 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:38.432 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260505 16:07:38.432 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:38.432 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:38.433 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 16:07:38.433 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:38.433 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:38.433 | INFO | Uploaded 36GiB to GPU in 2.18725s (16.459GiB/s) | 20260505 16:07:38.433 | INFO | Downloaded 34GiB to GPU in 2.13101s (15.9549GiB/s) | 20260505 16:07:43.566 | INFO | Analyzing chunk frames = [4519936,4520448) | 20260505 16:07:43.566 | INFO | MemInfo: { | 20260505 16:07:43.566 | INFO | "gpu" : " 17 GB", | 20260505 16:07:43.566 | INFO | "huge" : "", | 20260505 16:07:43.566 | INFO | "huge_shared" : " 27 GB", | 20260505 16:07:43.566 | INFO | "malloc" : " 14 GB", | 20260505 16:07:43.566 | INFO | "pinned" : " 29 GB" | 20260505 16:07:43.566 | INFO | } | 20260505 16:07:43.566 | INFO | Perf report for the last 5.13331 seconds: | 20260505 16:07:43.566 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:07:43.566 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:43.566 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:43.566 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:43.566 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 16:07:43.566 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:43.566 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:43.566 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:07:43.566 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:43.566 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:43.566 | INFO | Uploaded 36GiB to GPU in 2.12573s (16.9354GiB/s) | 20260505 16:07:43.566 | INFO | Downloaded 34GiB to GPU in 2.0457s (16.6202GiB/s) | 20260505 16:07:48.701 | INFO | Analyzing chunk frames = [4520448,4520960) | 20260505 16:07:48.701 | INFO | MemInfo: { | 20260505 16:07:48.701 | INFO | "gpu" : " 17 GB", | 20260505 16:07:48.701 | INFO | "huge" : "", | 20260505 16:07:48.701 | INFO | "huge_shared" : " 27 GB", | 20260505 16:07:48.701 | INFO | "malloc" : " 14 GB", | 20260505 16:07:48.701 | INFO | "pinned" : " 29 GB" | 20260505 16:07:48.701 | INFO | } | 20260505 16:07:48.702 | INFO | Perf report for the last 5.1359 seconds: | 20260505 16:07:48.702 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:07:48.702 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:48.702 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:48.702 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:48.702 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 16:07:48.702 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:48.702 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:48.702 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:07:48.702 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:48.702 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:48.702 | INFO | Uploaded 36GiB to GPU in 2.16524s (16.6263GiB/s) | 20260505 16:07:48.702 | INFO | Downloaded 34GiB to GPU in 2.03935s (16.672GiB/s) | 20260505 16:07:53.840 | INFO | Analyzing chunk frames = [4520960,4521472) | 20260505 16:07:53.840 | INFO | MemInfo: { | 20260505 16:07:53.840 | INFO | "gpu" : " 17 GB", | 20260505 16:07:53.840 | INFO | "huge" : "", | 20260505 16:07:53.840 | INFO | "huge_shared" : " 27 GB", | 20260505 16:07:53.840 | INFO | "malloc" : " 14 GB", | 20260505 16:07:53.840 | INFO | "pinned" : " 29 GB" | 20260505 16:07:53.840 | INFO | } | 20260505 16:07:53.840 | INFO | Perf report for the last 5.13845 seconds: | 20260505 16:07:53.840 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:07:53.840 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:53.840 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:53.840 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:53.840 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 16:07:53.840 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:53.840 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:53.840 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 16:07:53.840 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:53.840 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:53.840 | INFO | Uploaded 36GiB to GPU in 2.13169s (16.888GiB/s) | 20260505 16:07:53.840 | INFO | Downloaded 34GiB to GPU in 2.18711s (15.5456GiB/s) | 20260505 16:07:58.987 | INFO | Analyzing chunk frames = [4521472,4521984) | 20260505 16:07:58.987 | INFO | MemInfo: { | 20260505 16:07:58.987 | INFO | "gpu" : " 17 GB", | 20260505 16:07:58.987 | INFO | "huge" : "", | 20260505 16:07:58.987 | INFO | "huge_shared" : " 27 GB", | 20260505 16:07:58.987 | INFO | "malloc" : " 14 GB", | 20260505 16:07:58.987 | INFO | "pinned" : " 29 GB" | 20260505 16:07:58.987 | INFO | } | 20260505 16:07:58.987 | INFO | Perf report for the last 5.14705 seconds: | 20260505 16:07:58.987 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:07:58.987 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:58.987 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:58.987 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:58.987 | INFO | ANALYSIS : 63.3%, 3.0, 99.0ms, 0 | 20260505 16:07:58.987 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:58.987 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:58.987 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 16:07:58.987 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:58.987 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:07:58.987 | INFO | Uploaded 36GiB to GPU in 2.2115s (16.2785GiB/s) | 20260505 16:07:58.987 | INFO | Downloaded 34GiB to GPU in 2.09999s (16.1905GiB/s) | 20260505 16:07:58.988 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 16:07:58.988 | INFO | Total mem usage: 15.3 GiB | 20260505 16:07:58.988 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 16:07:58.988 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.85%) | 20260505 16:07:58.988 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 16:07:58.988 | INFO | High Water Breakdown: | 20260505 16:07:58.988 | INFO | 61% : 8.67 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 16:07:58.988 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 16:07:58.988 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 16:07:58.988 | INFO | ------------------------------------------ | 20260505 16:07:58.988 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:07:58.988 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 16:07:58.988 | INFO | Total mem usage: 26.5 GiB | 20260505 16:07:58.988 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 16:07:58.988 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 16:07:58.988 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 16:07:58.988 | INFO | High Water Breakdown: | 20260505 16:07:58.988 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:07:58.988 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:07:58.988 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:07:58.988 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:07:58.988 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:07:58.988 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:07:58.988 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 16:07:58.988 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 16:07:58.988 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:07:58.988 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 16:07:58.988 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 16:07:58.988 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 16:07:58.988 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 16:07:58.988 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 16:07:58.988 | INFO | ------------------------------------------ | 20260505 16:07:58.988 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:07:58.988 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 16:07:58.988 | INFO | Total mem usage: 25.6 GiB | 20260505 16:07:58.988 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 16:07:58.988 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 16:07:58.988 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 16:07:58.988 | INFO | High Water Breakdown: | 20260505 16:07:58.988 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 16:07:58.988 | INFO | ------------------------------------------ | 20260505 16:07:58.988 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:07:58.988 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 16:07:58.988 | INFO | Total mem usage: 15.7 GiB | 20260505 16:07:58.988 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 16:07:58.988 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 16:07:58.989 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 16:07:58.989 | INFO | High Water Breakdown: | 20260505 16:07:58.989 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 16:07:58.989 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 16:07:58.989 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 16:07:58.989 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 16:07:58.989 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 16:07:58.989 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 16:07:58.989 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 16:07:58.989 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 16:07:58.989 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 16:07:58.989 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:07:58.989 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 16:07:58.989 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:07:58.989 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 16:07:58.989 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 16:07:58.989 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 16:07:58.989 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 16:07:58.989 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:07:58.989 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 16:07:58.989 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:07:58.989 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:07:58.989 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:07:58.989 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 16:07:58.989 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 16:07:58.989 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:07:58.989 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 16:07:58.989 | INFO | ------------------------------------------ | 20260505 16:07:58.989 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:07:58.989 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 16:07:58.989 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:07:58.989 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:07:58.989 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.54e+01, 1.35e+01 / 1.96e+00 | 20260505 16:07:58.989 | INFO | PerfReport: 0.29%, AggregateStats -- 9.25e-02 / 1.01e-01, 1.05e-02 / 6.16e-03 | 20260505 16:07:58.989 | INFO | PerfReport: 11.77%, Baseline -- 3.74e+00 / 4.48e+00, 5.79e-01 / 1.62e-01 | 20260505 16:07:58.989 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.49e+00, 1.95e-01 / 1.20e-01 | 20260505 16:07:58.989 | INFO | PerfReport: 15.92%, DME -- 4.04e+01 / 4.48e+01, 4.28e+00 / 2.22e+00 | 20260505 16:07:58.989 | INFO | PerfReport: 56.77%, FrameLabeling -- 1.80e+01 / 1.98e+01, 1.54e+00 / 1.69e+00 | 20260505 16:07:58.989 | INFO | PerfReport: 7.35%, PulseAccumulating -- 2.33e+00 / 2.90e+00, 4.45e-01 / 1.86e-01 | 20260505 16:07:58.989 | INFO | PerfReport: 3.42%, Metrics -- 1.09e+00 / 1.40e+00, 3.02e-01 / 1.65e-01 | 20260505 16:07:58.989 | INFO | PerfReport: 1.19%, PulseSerializing -- 3.78e-01 / 4.28e-01, 6.78e-02 / 3.58e-02 | 20260505 16:07:58.989 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 16:07:58.989 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:07:58.989 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:07:58.989 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260505 16:07:58.989 | INFO | PerfReport: 22.56%, Upload -- 2.26e+01 / 0.00e+00, 7.88e+00 / 0.00e+00 | 20260505 16:07:58.989 | INFO | PerfReport: 21.88%, Download -- 2.19e+01 / 0.00e+00, 7.90e+00 / 0.00e+00 | 20260505 16:07:58.989 | INFO | PerfReport: 31.66%, Compute -- 3.18e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 16:07:58.989 | INFO | PerfReport: 2.21%, LockUpload -- 2.21e+00 / 0.00e+00, 7.76e+00 / 0.00e+00 | 20260505 16:07:58.989 | INFO | PerfReport: 1.40%, LockDownload -- 1.41e+00 / 0.00e+00, 2.93e+00 / 0.00e+00 | 20260505 16:07:58.989 | INFO | PerfReport: 20.28%, LockCompute -- 2.03e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260505 16:07:58.989 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.87e-02 / 0.00e+00 | 20260505 16:07:59.122 | INFO | Starting a round of Baz IO | 20260505 16:08:02.321 | INFO | Populating RTMetrics report | 20260505 16:08:02.321 | INFO | RTMetrics report populated | 20260505 16:08:02.321 | INFO | Opening tmp RTMetrics json file | 20260505 16:08:02.322 | INFO | Serializing RTMetrics to json | 20260505 16:08:02.323 | INFO | Serialization done, swapping tmp file now... | 20260505 16:08:02.324 | INFO | RTMetrics done | 20260505 16:08:02.435 | INFO | Finished a round of Baz IO: Wrote 2.186 GiB in 3.313 seconds | 20260505 16:08:02.435 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.31% / 0.00% / 77.69% | 20260505 16:08:02.435 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.389258% / 7.94729e-06% / 0.000103315% | 20260505 16:08:02.435 | INFO | Baz IO overall: | 20260505 16:08:02.435 | INFO | BytesWritten : 2685667679536 | 20260505 16:08:02.435 | INFO | NumEvents : 1275114499841 | 20260505 16:08:04.131 | INFO | Analyzing chunk frames = [4521984,4522496) | 20260505 16:08:04.131 | INFO | MemInfo: { | 20260505 16:08:04.131 | INFO | "gpu" : " 17 GB", | 20260505 16:08:04.131 | INFO | "huge" : "", | 20260505 16:08:04.131 | INFO | "huge_shared" : " 27 GB", | 20260505 16:08:04.131 | INFO | "malloc" : " 14 GB", | 20260505 16:08:04.131 | INFO | "pinned" : " 29 GB" | 20260505 16:08:04.131 | INFO | } | 20260505 16:08:04.132 | INFO | Perf report for the last 5.14448 seconds: | 20260505 16:08:04.132 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:08:04.132 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:04.132 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:04.132 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:04.132 | INFO | ANALYSIS : 63.7%, 3.0, 99.2ms, 0 | 20260505 16:08:04.132 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:04.132 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260505 16:08:04.132 | INFO | PRE_HQ : 60.7%, 1.8, 57.2ms, 0 | 20260505 16:08:04.132 | INFO | BAZWRITER : 63.7%, 4.0, 129.9ms, 0 | 20260505 16:08:04.132 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 16:08:04.132 | INFO | Uploaded 36GiB to GPU in 2.15589s (16.6985GiB/s) | 20260505 16:08:04.132 | INFO | Downloaded 38GiB to GPU in 2.25741s (16.8335GiB/s) | 20260505 16:08:09.268 | INFO | Analyzing chunk frames = [4522496,4523008) | 20260505 16:08:09.268 | INFO | MemInfo: { | 20260505 16:08:09.268 | INFO | "gpu" : " 17 GB", | 20260505 16:08:09.268 | INFO | "huge" : "", | 20260505 16:08:09.268 | INFO | "huge_shared" : " 27 GB", | 20260505 16:08:09.268 | INFO | "malloc" : " 14 GB", | 20260505 16:08:09.268 | INFO | "pinned" : " 29 GB" | 20260505 16:08:09.268 | INFO | } | 20260505 16:08:09.268 | INFO | Perf report for the last 5.1363 seconds: | 20260505 16:08:09.268 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:08:09.268 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:09.268 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:09.268 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:09.268 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 16:08:09.268 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:09.268 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:09.268 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 16:08:09.268 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:09.268 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:09.268 | INFO | Uploaded 36GiB to GPU in 2.14363s (16.7939GiB/s) | 20260505 16:08:09.268 | INFO | Downloaded 34GiB to GPU in 2.0286s (16.7603GiB/s) | 20260505 16:08:14.399 | INFO | Analyzing chunk frames = [4523008,4523520) | 20260505 16:08:14.399 | INFO | MemInfo: { | 20260505 16:08:14.399 | INFO | "gpu" : " 17 GB", | 20260505 16:08:14.399 | INFO | "huge" : "", | 20260505 16:08:14.399 | INFO | "huge_shared" : " 27 GB", | 20260505 16:08:14.399 | INFO | "malloc" : " 14 GB", | 20260505 16:08:14.399 | INFO | "pinned" : " 29 GB" | 20260505 16:08:14.399 | INFO | } | 20260505 16:08:14.399 | INFO | Perf report for the last 5.13112 seconds: | 20260505 16:08:14.399 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:08:14.399 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:14.399 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:14.399 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:14.399 | INFO | ANALYSIS : 62.9%, 3.0, 97.8ms, 0 | 20260505 16:08:14.399 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:14.399 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:14.399 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 16:08:14.399 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:14.399 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:14.399 | INFO | Uploaded 36GiB to GPU in 2.20336s (16.3387GiB/s) | 20260505 16:08:14.399 | INFO | Downloaded 34GiB to GPU in 2.02523s (16.7882GiB/s) | 20260505 16:08:19.535 | INFO | Analyzing chunk frames = [4523520,4524032) | 20260505 16:08:19.535 | INFO | MemInfo: { | 20260505 16:08:19.535 | INFO | "gpu" : " 17 GB", | 20260505 16:08:19.535 | INFO | "huge" : "", | 20260505 16:08:19.535 | INFO | "huge_shared" : " 27 GB", | 20260505 16:08:19.535 | INFO | "malloc" : " 14 GB", | 20260505 16:08:19.535 | INFO | "pinned" : " 29 GB" | 20260505 16:08:19.535 | INFO | } | 20260505 16:08:19.535 | INFO | Perf report for the last 5.13634 seconds: | 20260505 16:08:19.535 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:08:19.536 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:19.536 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:19.536 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:19.536 | INFO | ANALYSIS : 62.5%, 3.0, 97.1ms, 0 | 20260505 16:08:19.536 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:19.536 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:19.536 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 16:08:19.536 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:19.536 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:19.536 | INFO | Uploaded 36GiB to GPU in 2.18865s (16.4485GiB/s) | 20260505 16:08:19.536 | INFO | Downloaded 34GiB to GPU in 2.12849s (15.9737GiB/s) | 20260505 16:08:24.684 | INFO | Analyzing chunk frames = [4524032,4524544) | 20260505 16:08:24.684 | INFO | MemInfo: { | 20260505 16:08:24.684 | INFO | "gpu" : " 17 GB", | 20260505 16:08:24.684 | INFO | "huge" : "", | 20260505 16:08:24.684 | INFO | "huge_shared" : " 27 GB", | 20260505 16:08:24.684 | INFO | "malloc" : " 14 GB", | 20260505 16:08:24.684 | INFO | "pinned" : " 29 GB" | 20260505 16:08:24.684 | INFO | } | 20260505 16:08:24.684 | INFO | Perf report for the last 5.14879 seconds: | 20260505 16:08:24.684 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:08:24.684 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:24.684 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:24.684 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:24.684 | INFO | ANALYSIS : 62.5%, 3.0, 97.8ms, 0 | 20260505 16:08:24.684 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:24.684 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:24.684 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:08:24.684 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:24.684 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:24.685 | INFO | Uploaded 36GiB to GPU in 2.17774s (16.5309GiB/s) | 20260505 16:08:24.685 | INFO | Downloaded 34GiB to GPU in 2.13053s (15.9584GiB/s) | 20260505 16:08:29.832 | INFO | Analyzing chunk frames = [4524544,4525056) | 20260505 16:08:29.832 | INFO | MemInfo: { | 20260505 16:08:29.832 | INFO | "gpu" : " 17 GB", | 20260505 16:08:29.832 | INFO | "huge" : "", | 20260505 16:08:29.832 | INFO | "huge_shared" : " 27 GB", | 20260505 16:08:29.832 | INFO | "malloc" : " 14 GB", | 20260505 16:08:29.832 | INFO | "pinned" : " 29 GB" | 20260505 16:08:29.832 | INFO | } | 20260505 16:08:29.832 | INFO | Perf report for the last 5.14764 seconds: | 20260505 16:08:29.832 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:08:29.832 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:29.832 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:29.832 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:29.832 | INFO | ANALYSIS : 62.3%, 3.0, 97.5ms, 0 | 20260505 16:08:29.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:29.832 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:29.832 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:08:29.832 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:29.832 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:29.832 | INFO | Uploaded 36GiB to GPU in 2.13735s (16.8433GiB/s) | 20260505 16:08:29.832 | INFO | Downloaded 34GiB to GPU in 2.03582s (16.7008GiB/s) | 20260505 16:08:34.963 | INFO | Analyzing chunk frames = [4525056,4525568) | 20260505 16:08:34.963 | INFO | MemInfo: { | 20260505 16:08:34.963 | INFO | "gpu" : " 17 GB", | 20260505 16:08:34.963 | INFO | "huge" : "", | 20260505 16:08:34.963 | INFO | "huge_shared" : " 27 GB", | 20260505 16:08:34.963 | INFO | "malloc" : " 14 GB", | 20260505 16:08:34.963 | INFO | "pinned" : " 29 GB" | 20260505 16:08:34.963 | INFO | } | 20260505 16:08:34.963 | INFO | Perf report for the last 5.1312 seconds: | 20260505 16:08:34.963 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:08:34.963 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:34.963 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:34.963 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:34.963 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 16:08:34.963 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:34.963 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:34.963 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260505 16:08:34.963 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:34.963 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:34.963 | INFO | Uploaded 36GiB to GPU in 2.14413s (16.79GiB/s) | 20260505 16:08:34.963 | INFO | Downloaded 34GiB to GPU in 2.16728s (15.6879GiB/s) | 20260505 16:08:40.097 | INFO | Analyzing chunk frames = [4525568,4526080) | 20260505 16:08:40.097 | INFO | MemInfo: { | 20260505 16:08:40.097 | INFO | "gpu" : " 17 GB", | 20260505 16:08:40.097 | INFO | "huge" : "", | 20260505 16:08:40.097 | INFO | "huge_shared" : " 27 GB", | 20260505 16:08:40.097 | INFO | "malloc" : " 14 GB", | 20260505 16:08:40.097 | INFO | "pinned" : " 29 GB" | 20260505 16:08:40.097 | INFO | } | 20260505 16:08:40.097 | INFO | Perf report for the last 5.13421 seconds: | 20260505 16:08:40.097 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:08:40.097 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:40.097 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:40.097 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:40.097 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 16:08:40.097 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:40.097 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:40.097 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 16:08:40.097 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:40.097 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:40.098 | INFO | Uploaded 36GiB to GPU in 2.22586s (16.1735GiB/s) | 20260505 16:08:40.098 | INFO | Downloaded 34GiB to GPU in 2.10648s (16.1406GiB/s) | 20260505 16:08:40.231 | INFO | Starting a round of Baz IO | 20260505 16:08:43.439 | INFO | Populating RTMetrics report | 20260505 16:08:43.440 | INFO | RTMetrics report populated | 20260505 16:08:43.440 | INFO | Opening tmp RTMetrics json file | 20260505 16:08:43.450 | INFO | Serializing RTMetrics to json | 20260505 16:08:43.452 | INFO | Serialization done, swapping tmp file now... | 20260505 16:08:43.454 | INFO | RTMetrics done | 20260505 16:08:43.538 | INFO | Finished a round of Baz IO: Wrote 2.184 GiB in 3.307 seconds | 20260505 16:08:43.538 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.31% / 0.00% / 77.69% | 20260505 16:08:43.538 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.386933% / 3.97364e-06% / 7.54992e-05% | 20260505 16:08:43.538 | INFO | Baz IO overall: | 20260505 16:08:43.538 | INFO | BytesWritten : 2688012896770 | 20260505 16:08:43.538 | INFO | NumEvents : 1276215119885 | 20260505 16:08:45.246 | INFO | Analyzing chunk frames = [4526080,4526592) | 20260505 16:08:45.246 | INFO | MemInfo: { | 20260505 16:08:45.246 | INFO | "gpu" : " 17 GB", | 20260505 16:08:45.246 | INFO | "huge" : "", | 20260505 16:08:45.246 | INFO | "huge_shared" : " 27 GB", | 20260505 16:08:45.246 | INFO | "malloc" : " 14 GB", | 20260505 16:08:45.246 | INFO | "pinned" : " 29 GB" | 20260505 16:08:45.246 | INFO | } | 20260505 16:08:45.247 | INFO | Perf report for the last 5.14927 seconds: | 20260505 16:08:45.247 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:08:45.247 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:45.247 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:45.247 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:45.247 | INFO | ANALYSIS : 63.8%, 3.0, 99.4ms, 0 | 20260505 16:08:45.247 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:45.247 | INFO | RT_METRICS : 44.8%, 1.0, 23.6ms, 0 | 20260505 16:08:45.247 | INFO | PRE_HQ : 60.5%, 1.8, 56.7ms, 0 | 20260505 16:08:45.247 | INFO | BAZWRITER : 63.5%, 3.9, 129.0ms, 0 | 20260505 16:08:45.247 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 16:08:45.247 | INFO | Uploaded 36GiB to GPU in 2.1977s (16.3808GiB/s) | 20260505 16:08:45.247 | INFO | Downloaded 38GiB to GPU in 2.26556s (16.7729GiB/s) | 20260505 16:08:50.380 | INFO | Analyzing chunk frames = [4526592,4527104) | 20260505 16:08:50.380 | INFO | MemInfo: { | 20260505 16:08:50.380 | INFO | "gpu" : " 17 GB", | 20260505 16:08:50.380 | INFO | "huge" : "", | 20260505 16:08:50.380 | INFO | "huge_shared" : " 27 GB", | 20260505 16:08:50.380 | INFO | "malloc" : " 14 GB", | 20260505 16:08:50.380 | INFO | "pinned" : " 29 GB" | 20260505 16:08:50.380 | INFO | } | 20260505 16:08:50.380 | INFO | Perf report for the last 5.13333 seconds: | 20260505 16:08:50.380 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:08:50.380 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:50.380 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:50.380 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:50.380 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 16:08:50.380 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:50.380 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:50.380 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:08:50.380 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:50.380 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:50.380 | INFO | Uploaded 36GiB to GPU in 2.11748s (17.0013GiB/s) | 20260505 16:08:50.380 | INFO | Downloaded 34GiB to GPU in 2.02543s (16.7865GiB/s) | 20260505 16:08:55.517 | INFO | Analyzing chunk frames = [4527104,4527616) | 20260505 16:08:55.517 | INFO | MemInfo: { | 20260505 16:08:55.517 | INFO | "gpu" : " 17 GB", | 20260505 16:08:55.517 | INFO | "huge" : "", | 20260505 16:08:55.517 | INFO | "huge_shared" : " 27 GB", | 20260505 16:08:55.517 | INFO | "malloc" : " 14 GB", | 20260505 16:08:55.517 | INFO | "pinned" : " 29 GB" | 20260505 16:08:55.517 | INFO | } | 20260505 16:08:55.517 | INFO | Perf report for the last 5.13745 seconds: | 20260505 16:08:55.517 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:08:55.517 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:55.517 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:55.517 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:55.517 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260505 16:08:55.517 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:55.517 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:55.517 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 16:08:55.517 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:55.517 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:08:55.518 | INFO | Uploaded 36GiB to GPU in 2.19811s (16.3777GiB/s) | 20260505 16:08:55.518 | INFO | Downloaded 34GiB to GPU in 2.02967s (16.7515GiB/s) | 20260505 16:09:00.655 | INFO | Analyzing chunk frames = [4527616,4528128) | 20260505 16:09:00.655 | INFO | MemInfo: { | 20260505 16:09:00.655 | INFO | "gpu" : " 17 GB", | 20260505 16:09:00.655 | INFO | "huge" : "", | 20260505 16:09:00.655 | INFO | "huge_shared" : " 27 GB", | 20260505 16:09:00.655 | INFO | "malloc" : " 14 GB", | 20260505 16:09:00.655 | INFO | "pinned" : " 29 GB" | 20260505 16:09:00.655 | INFO | } | 20260505 16:09:00.655 | INFO | Perf report for the last 5.13802 seconds: | 20260505 16:09:00.655 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:09:00.655 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:00.655 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:00.655 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:00.655 | INFO | ANALYSIS : 62.4%, 3.0, 96.9ms, 0 | 20260505 16:09:00.656 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:00.656 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:00.656 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:09:00.656 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:00.656 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:00.656 | INFO | Uploaded 36GiB to GPU in 2.18406s (16.483GiB/s) | 20260505 16:09:00.656 | INFO | Downloaded 34GiB to GPU in 2.13151s (15.9511GiB/s) | 20260505 16:09:05.791 | INFO | Analyzing chunk frames = [4528128,4528640) | 20260505 16:09:05.791 | INFO | MemInfo: { | 20260505 16:09:05.791 | INFO | "gpu" : " 17 GB", | 20260505 16:09:05.791 | INFO | "huge" : "", | 20260505 16:09:05.791 | INFO | "huge_shared" : " 27 GB", | 20260505 16:09:05.791 | INFO | "malloc" : " 14 GB", | 20260505 16:09:05.791 | INFO | "pinned" : " 29 GB" | 20260505 16:09:05.791 | INFO | } | 20260505 16:09:05.791 | INFO | Perf report for the last 5.13558 seconds: | 20260505 16:09:05.791 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:09:05.791 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:05.791 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:05.791 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:05.791 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 16:09:05.791 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:05.791 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:05.791 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:09:05.791 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:05.791 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:05.791 | INFO | Uploaded 36GiB to GPU in 2.17505s (16.5514GiB/s) | 20260505 16:09:05.791 | INFO | Downloaded 34GiB to GPU in 2.13153s (15.951GiB/s) | 20260505 16:09:10.942 | INFO | Analyzing chunk frames = [4528640,4529152) | 20260505 16:09:10.942 | INFO | MemInfo: { | 20260505 16:09:10.942 | INFO | "gpu" : " 17 GB", | 20260505 16:09:10.942 | INFO | "huge" : "", | 20260505 16:09:10.942 | INFO | "huge_shared" : " 27 GB", | 20260505 16:09:10.942 | INFO | "malloc" : " 14 GB", | 20260505 16:09:10.942 | INFO | "pinned" : " 29 GB" | 20260505 16:09:10.942 | INFO | } | 20260505 16:09:10.942 | INFO | Perf report for the last 5.15083 seconds: | 20260505 16:09:10.942 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:09:10.942 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:10.942 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:10.942 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:10.942 | INFO | ANALYSIS : 62.4%, 3.0, 97.7ms, 0 | 20260505 16:09:10.942 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:10.942 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:10.942 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:09:10.942 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:10.942 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:10.942 | INFO | Uploaded 36GiB to GPU in 2.14056s (16.818GiB/s) | 20260505 16:09:10.942 | INFO | Downloaded 34GiB to GPU in 2.03557s (16.7029GiB/s) | 20260505 16:09:16.082 | INFO | Analyzing chunk frames = [4529152,4529664) | 20260505 16:09:16.082 | INFO | MemInfo: { | 20260505 16:09:16.082 | INFO | "gpu" : " 17 GB", | 20260505 16:09:16.082 | INFO | "huge" : "", | 20260505 16:09:16.082 | INFO | "huge_shared" : " 27 GB", | 20260505 16:09:16.082 | INFO | "malloc" : " 14 GB", | 20260505 16:09:16.082 | INFO | "pinned" : " 29 GB" | 20260505 16:09:16.082 | INFO | } | 20260505 16:09:16.082 | INFO | Perf report for the last 5.13992 seconds: | 20260505 16:09:16.082 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:09:16.082 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:16.082 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:16.082 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:16.082 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 16:09:16.082 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:16.082 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:16.082 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 16:09:16.082 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:16.082 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:16.082 | INFO | Uploaded 36GiB to GPU in 2.13725s (16.844GiB/s) | 20260505 16:09:16.082 | INFO | Downloaded 34GiB to GPU in 2.17407s (15.6389GiB/s) | 20260505 16:09:21.211 | INFO | Analyzing chunk frames = [4529664,4530176) | 20260505 16:09:21.211 | INFO | MemInfo: { | 20260505 16:09:21.211 | INFO | "gpu" : " 17 GB", | 20260505 16:09:21.211 | INFO | "huge" : "", | 20260505 16:09:21.211 | INFO | "huge_shared" : " 27 GB", | 20260505 16:09:21.211 | INFO | "malloc" : " 14 GB", | 20260505 16:09:21.211 | INFO | "pinned" : " 29 GB" | 20260505 16:09:21.211 | INFO | } | 20260505 16:09:21.211 | INFO | Perf report for the last 5.12947 seconds: | 20260505 16:09:21.211 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:09:21.211 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:21.211 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:21.211 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:21.211 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 16:09:21.211 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:21.211 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:21.211 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 16:09:21.211 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:21.211 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:21.212 | INFO | Uploaded 36GiB to GPU in 2.16704s (16.6125GiB/s) | 20260505 16:09:21.212 | INFO | Downloaded 34GiB to GPU in 2.03625s (16.6973GiB/s) | 20260505 16:09:21.212 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 16:09:21.212 | INFO | Total mem usage: 15.3 GiB | 20260505 16:09:21.212 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 16:09:21.212 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.85%) | 20260505 16:09:21.212 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 16:09:21.212 | INFO | High Water Breakdown: | 20260505 16:09:21.212 | INFO | 61% : 8.67 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 16:09:21.212 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 16:09:21.212 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 16:09:21.212 | INFO | ------------------------------------------ | 20260505 16:09:21.212 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:09:21.212 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 16:09:21.212 | INFO | Total mem usage: 26.5 GiB | 20260505 16:09:21.212 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 16:09:21.212 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 16:09:21.212 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 16:09:21.212 | INFO | High Water Breakdown: | 20260505 16:09:21.212 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:09:21.212 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:09:21.212 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:09:21.212 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:09:21.212 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:09:21.212 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:09:21.212 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 16:09:21.212 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 16:09:21.212 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:09:21.212 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 16:09:21.212 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 16:09:21.212 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 16:09:21.212 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 16:09:21.212 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 16:09:21.212 | INFO | ------------------------------------------ | 20260505 16:09:21.212 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:09:21.212 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 16:09:21.212 | INFO | Total mem usage: 25.6 GiB | 20260505 16:09:21.212 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 16:09:21.212 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 16:09:21.212 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 16:09:21.212 | INFO | High Water Breakdown: | 20260505 16:09:21.212 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 16:09:21.212 | INFO | ------------------------------------------ | 20260505 16:09:21.212 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:09:21.212 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 16:09:21.212 | INFO | Total mem usage: 15.7 GiB | 20260505 16:09:21.212 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 16:09:21.212 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 16:09:21.212 | INFO | Unused Cache: 896 MiB (Waste: 5.58%) | 20260505 16:09:21.212 | INFO | High Water Breakdown: | 20260505 16:09:21.212 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 16:09:21.212 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 16:09:21.212 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 16:09:21.212 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 16:09:21.212 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 16:09:21.212 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 16:09:21.212 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 16:09:21.212 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 16:09:21.212 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 16:09:21.212 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:09:21.212 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 16:09:21.212 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:09:21.212 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 16:09:21.212 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 16:09:21.212 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 16:09:21.212 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 16:09:21.212 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:09:21.212 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 16:09:21.212 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:09:21.212 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:09:21.212 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:09:21.212 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 16:09:21.212 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 16:09:21.212 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:09:21.212 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 16:09:21.212 | INFO | ------------------------------------------ | 20260505 16:09:21.212 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:09:21.213 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 16:09:21.213 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:09:21.213 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:09:21.213 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.54e+01, 1.35e+01 / 2.06e+00 | 20260505 16:09:21.213 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 1.04e-01, 1.16e-02 / 1.85e-02 | 20260505 16:09:21.213 | INFO | PerfReport: 11.81%, Baseline -- 3.74e+00 / 4.52e+00, 5.83e-01 / 1.78e-01 | 20260505 16:09:21.213 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.51e+00, 1.93e-01 / 1.05e-01 | 20260505 16:09:21.213 | INFO | PerfReport: 15.89%, DME -- 4.03e+01 / 4.45e+01, 4.26e+00 / 2.17e+00 | 20260505 16:09:21.213 | INFO | PerfReport: 56.73%, FrameLabeling -- 1.80e+01 / 1.99e+01, 1.47e+00 / 1.88e+00 | 20260505 16:09:21.213 | INFO | PerfReport: 7.37%, PulseAccumulating -- 2.33e+00 / 2.89e+00, 4.44e-01 / 2.02e-01 | 20260505 16:09:21.213 | INFO | PerfReport: 3.43%, Metrics -- 1.09e+00 / 1.42e+00, 3.03e-01 / 1.57e-01 | 20260505 16:09:21.213 | INFO | PerfReport: 1.19%, PulseSerializing -- 3.79e-01 / 4.31e-01, 6.84e-02 / 3.59e-02 | 20260505 16:09:21.213 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 16:09:21.213 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:09:21.213 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:09:21.213 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260505 16:09:21.213 | INFO | PerfReport: 22.53%, Upload -- 2.26e+01 / 0.00e+00, 7.95e+00 / 0.00e+00 | 20260505 16:09:21.213 | INFO | PerfReport: 21.89%, Download -- 2.19e+01 / 0.00e+00, 7.94e+00 / 0.00e+00 | 20260505 16:09:21.213 | INFO | PerfReport: 31.61%, Compute -- 3.17e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 16:09:21.213 | INFO | PerfReport: 2.23%, LockUpload -- 2.23e+00 / 0.00e+00, 7.91e+00 / 0.00e+00 | 20260505 16:09:21.213 | INFO | PerfReport: 1.44%, LockDownload -- 1.45e+00 / 0.00e+00, 2.99e+00 / 0.00e+00 | 20260505 16:09:21.213 | INFO | PerfReport: 20.28%, LockCompute -- 2.03e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 16:09:21.213 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 2.16e-02 / 0.00e+00 | 20260505 16:09:21.341 | INFO | Starting a round of Baz IO | 20260505 16:09:24.553 | INFO | Populating RTMetrics report | 20260505 16:09:24.553 | INFO | RTMetrics report populated | 20260505 16:09:24.553 | INFO | Opening tmp RTMetrics json file | 20260505 16:09:24.571 | INFO | Serializing RTMetrics to json | 20260505 16:09:24.577 | INFO | Serialization done, swapping tmp file now... | 20260505 16:09:24.595 | INFO | RTMetrics done | 20260505 16:09:24.654 | INFO | Finished a round of Baz IO: Wrote 2.183 GiB in 3.313 seconds | 20260505 16:09:24.654 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.31% / 0.00% / 77.69% | 20260505 16:09:24.654 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.385876% / 1.19209e-05% / 0.000111262% | 20260505 16:09:24.654 | INFO | Baz IO overall: | 20260505 16:09:24.654 | INFO | BytesWritten : 2690357331867 | 20260505 16:09:24.654 | INFO | NumEvents : 1277315050500 | 20260505 16:09:26.346 | INFO | Analyzing chunk frames = [4530176,4530688) | 20260505 16:09:26.346 | INFO | MemInfo: { | 20260505 16:09:26.346 | INFO | "gpu" : " 17 GB", | 20260505 16:09:26.346 | INFO | "huge" : "", | 20260505 16:09:26.346 | INFO | "huge_shared" : " 27 GB", | 20260505 16:09:26.346 | INFO | "malloc" : " 14 GB", | 20260505 16:09:26.346 | INFO | "pinned" : " 29 GB" | 20260505 16:09:26.346 | INFO | } | 20260505 16:09:26.346 | INFO | Perf report for the last 5.13451 seconds: | 20260505 16:09:26.346 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:09:26.346 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:26.346 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:26.346 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:26.346 | INFO | ANALYSIS : 64.0%, 3.0, 99.3ms, 0 | 20260505 16:09:26.346 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:26.346 | INFO | RT_METRICS : 45.5%, 1.0, 23.9ms, 0 | 20260505 16:09:26.346 | INFO | PRE_HQ : 60.9%, 1.8, 56.7ms, 0 | 20260505 16:09:26.346 | INFO | BAZWRITER : 63.8%, 4.0, 129.8ms, 0 | 20260505 16:09:26.346 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 16:09:26.346 | INFO | Uploaded 36GiB to GPU in 2.18329s (16.4888GiB/s) | 20260505 16:09:26.346 | INFO | Downloaded 38GiB to GPU in 2.27006s (16.7397GiB/s) | 20260505 16:09:31.496 | INFO | Analyzing chunk frames = [4530688,4531200) | 20260505 16:09:31.496 | INFO | MemInfo: { | 20260505 16:09:31.496 | INFO | "gpu" : " 17 GB", | 20260505 16:09:31.496 | INFO | "huge" : "", | 20260505 16:09:31.496 | INFO | "huge_shared" : " 27 GB", | 20260505 16:09:31.496 | INFO | "malloc" : " 14 GB", | 20260505 16:09:31.496 | INFO | "pinned" : " 29 GB" | 20260505 16:09:31.496 | INFO | } | 20260505 16:09:31.496 | INFO | Perf report for the last 5.1505 seconds: | 20260505 16:09:31.496 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:09:31.496 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:31.496 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:31.496 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:31.496 | INFO | ANALYSIS : 63.2%, 3.0, 98.9ms, 0 | 20260505 16:09:31.496 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:31.496 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:31.496 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:09:31.496 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:31.496 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:31.496 | INFO | Uploaded 36GiB to GPU in 2.19958s (16.3668GiB/s) | 20260505 16:09:31.497 | INFO | Downloaded 34GiB to GPU in 2.04557s (16.6213GiB/s) | 20260505 16:09:36.639 | INFO | Analyzing chunk frames = [4531200,4531712) | 20260505 16:09:36.639 | INFO | MemInfo: { | 20260505 16:09:36.639 | INFO | "gpu" : " 17 GB", | 20260505 16:09:36.639 | INFO | "huge" : "", | 20260505 16:09:36.639 | INFO | "huge_shared" : " 27 GB", | 20260505 16:09:36.639 | INFO | "malloc" : " 14 GB", | 20260505 16:09:36.639 | INFO | "pinned" : " 29 GB" | 20260505 16:09:36.639 | INFO | } | 20260505 16:09:36.640 | INFO | Perf report for the last 5.14344 seconds: | 20260505 16:09:36.640 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:09:36.640 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:36.640 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:36.640 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:36.640 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 16:09:36.640 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:36.640 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:36.640 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:09:36.640 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:36.640 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:36.640 | INFO | Uploaded 36GiB to GPU in 2.20046s (16.3602GiB/s) | 20260505 16:09:36.640 | INFO | Downloaded 34GiB to GPU in 2.03772s (16.6853GiB/s) | 20260505 16:09:41.778 | INFO | Analyzing chunk frames = [4531712,4532224) | 20260505 16:09:41.778 | INFO | MemInfo: { | 20260505 16:09:41.778 | INFO | "gpu" : " 17 GB", | 20260505 16:09:41.778 | INFO | "huge" : "", | 20260505 16:09:41.778 | INFO | "huge_shared" : " 27 GB", | 20260505 16:09:41.778 | INFO | "malloc" : " 14 GB", | 20260505 16:09:41.778 | INFO | "pinned" : " 29 GB" | 20260505 16:09:41.778 | INFO | } | 20260505 16:09:41.778 | INFO | Perf report for the last 5.13849 seconds: | 20260505 16:09:41.778 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:09:41.778 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:41.778 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:41.778 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:41.778 | INFO | ANALYSIS : 62.6%, 3.0, 97.2ms, 0 | 20260505 16:09:41.778 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:41.778 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:41.778 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:09:41.778 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:41.778 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:41.778 | INFO | Uploaded 36GiB to GPU in 2.2069s (16.3124GiB/s) | 20260505 16:09:41.779 | INFO | Downloaded 34GiB to GPU in 2.11915s (16.0441GiB/s) | 20260505 16:09:46.913 | INFO | Analyzing chunk frames = [4532224,4532736) | 20260505 16:09:46.913 | INFO | MemInfo: { | 20260505 16:09:46.913 | INFO | "gpu" : " 17 GB", | 20260505 16:09:46.913 | INFO | "huge" : "", | 20260505 16:09:46.913 | INFO | "huge_shared" : " 27 GB", | 20260505 16:09:46.913 | INFO | "malloc" : " 14 GB", | 20260505 16:09:46.913 | INFO | "pinned" : " 29 GB" | 20260505 16:09:46.913 | INFO | } | 20260505 16:09:46.913 | INFO | Perf report for the last 5.13486 seconds: | 20260505 16:09:46.913 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:09:46.913 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:46.913 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:46.913 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:46.913 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 16:09:46.913 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:46.913 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:46.913 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 16:09:46.913 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:46.913 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:46.913 | INFO | Uploaded 36GiB to GPU in 2.15222s (16.7269GiB/s) | 20260505 16:09:46.913 | INFO | Downloaded 34GiB to GPU in 2.05856s (16.5164GiB/s) | 20260505 16:09:52.047 | INFO | Analyzing chunk frames = [4532736,4533248) | 20260505 16:09:52.047 | INFO | MemInfo: { | 20260505 16:09:52.047 | INFO | "gpu" : " 17 GB", | 20260505 16:09:52.047 | INFO | "huge" : "", | 20260505 16:09:52.047 | INFO | "huge_shared" : " 27 GB", | 20260505 16:09:52.047 | INFO | "malloc" : " 14 GB", | 20260505 16:09:52.047 | INFO | "pinned" : " 29 GB" | 20260505 16:09:52.047 | INFO | } | 20260505 16:09:52.047 | INFO | Perf report for the last 5.13373 seconds: | 20260505 16:09:52.047 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:09:52.047 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:52.047 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:52.047 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:52.047 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 16:09:52.047 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:52.047 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:52.047 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 16:09:52.047 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:52.047 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:52.047 | INFO | Uploaded 36GiB to GPU in 2.17177s (16.5763GiB/s) | 20260505 16:09:52.047 | INFO | Downloaded 34GiB to GPU in 2.04231s (16.6478GiB/s) | 20260505 16:09:57.188 | INFO | Analyzing chunk frames = [4533248,4533760) | 20260505 16:09:57.188 | INFO | MemInfo: { | 20260505 16:09:57.188 | INFO | "gpu" : " 17 GB", | 20260505 16:09:57.188 | INFO | "huge" : "", | 20260505 16:09:57.188 | INFO | "huge_shared" : " 27 GB", | 20260505 16:09:57.188 | INFO | "malloc" : " 14 GB", | 20260505 16:09:57.188 | INFO | "pinned" : " 29 GB" | 20260505 16:09:57.188 | INFO | } | 20260505 16:09:57.188 | INFO | Perf report for the last 5.14161 seconds: | 20260505 16:09:57.188 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:09:57.188 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:57.188 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:57.188 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:57.188 | INFO | ANALYSIS : 63.2%, 3.0, 98.8ms, 0 | 20260505 16:09:57.188 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:57.188 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:57.188 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 16:09:57.188 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:57.188 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:09:57.189 | INFO | Uploaded 36GiB to GPU in 2.10686s (17.0871GiB/s) | 20260505 16:09:57.189 | INFO | Downloaded 34GiB to GPU in 2.17584s (15.6262GiB/s) | 20260505 16:10:02.331 | INFO | Analyzing chunk frames = [4533760,4534272) | 20260505 16:10:02.331 | INFO | MemInfo: { | 20260505 16:10:02.331 | INFO | "gpu" : " 17 GB", | 20260505 16:10:02.331 | INFO | "huge" : "", | 20260505 16:10:02.331 | INFO | "huge_shared" : " 27 GB", | 20260505 16:10:02.331 | INFO | "malloc" : " 14 GB", | 20260505 16:10:02.331 | INFO | "pinned" : " 29 GB" | 20260505 16:10:02.331 | INFO | } | 20260505 16:10:02.331 | INFO | Perf report for the last 5.14295 seconds: | 20260505 16:10:02.331 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:10:02.331 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:02.331 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:02.332 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:02.332 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 16:10:02.332 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:02.332 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:02.332 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 16:10:02.332 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:02.332 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:02.332 | INFO | Uploaded 36GiB to GPU in 2.1714s (16.5792GiB/s) | 20260505 16:10:02.332 | INFO | Downloaded 34GiB to GPU in 2.03599s (16.6995GiB/s) | 20260505 16:10:02.465 | INFO | Starting a round of Baz IO | 20260505 16:10:05.669 | INFO | Populating RTMetrics report | 20260505 16:10:05.670 | INFO | RTMetrics report populated | 20260505 16:10:05.670 | INFO | Opening tmp RTMetrics json file | 20260505 16:10:05.684 | INFO | Serializing RTMetrics to json | 20260505 16:10:05.685 | INFO | Serialization done, swapping tmp file now... | 20260505 16:10:05.685 | INFO | RTMetrics done | 20260505 16:10:05.780 | INFO | Finished a round of Baz IO: Wrote 2.183 GiB in 3.314 seconds | 20260505 16:10:05.780 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.30% / 0.00% / 77.69% | 20260505 16:10:05.780 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.385074% / 7.94729e-06% / 7.54992e-05% | 20260505 16:10:05.780 | INFO | Baz IO overall: | 20260505 16:10:05.780 | INFO | BytesWritten : 2692701133391 | 20260505 16:10:05.780 | INFO | NumEvents : 1278414802898 | 20260505 16:10:07.466 | INFO | Analyzing chunk frames = [4534272,4534784) | 20260505 16:10:07.466 | INFO | MemInfo: { | 20260505 16:10:07.466 | INFO | "gpu" : " 17 GB", | 20260505 16:10:07.466 | INFO | "huge" : "", | 20260505 16:10:07.466 | INFO | "huge_shared" : " 27 GB", | 20260505 16:10:07.466 | INFO | "malloc" : " 14 GB", | 20260505 16:10:07.466 | INFO | "pinned" : " 29 GB" | 20260505 16:10:07.466 | INFO | } | 20260505 16:10:07.466 | INFO | Perf report for the last 5.13453 seconds: | 20260505 16:10:07.466 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:10:07.466 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:07.466 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:07.466 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:07.466 | INFO | ANALYSIS : 63.9%, 3.0, 99.2ms, 0 | 20260505 16:10:07.466 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:07.466 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260505 16:10:07.466 | INFO | PRE_HQ : 60.8%, 1.8, 56.7ms, 0 | 20260505 16:10:07.466 | INFO | BAZWRITER : 63.9%, 4.0, 129.7ms, 0 | 20260505 16:10:07.466 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 16:10:07.466 | INFO | Uploaded 36GiB to GPU in 2.18138s (16.5033GiB/s) | 20260505 16:10:07.466 | INFO | Downloaded 38GiB to GPU in 2.26836s (16.7522GiB/s) | 20260505 16:10:12.601 | INFO | Analyzing chunk frames = [4534784,4535296) | 20260505 16:10:12.601 | INFO | MemInfo: { | 20260505 16:10:12.601 | INFO | "gpu" : " 17 GB", | 20260505 16:10:12.601 | INFO | "huge" : "", | 20260505 16:10:12.601 | INFO | "huge_shared" : " 27 GB", | 20260505 16:10:12.601 | INFO | "malloc" : " 14 GB", | 20260505 16:10:12.601 | INFO | "pinned" : " 29 GB" | 20260505 16:10:12.601 | INFO | } | 20260505 16:10:12.601 | INFO | Perf report for the last 5.13515 seconds: | 20260505 16:10:12.601 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:10:12.601 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:12.601 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:12.601 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:12.601 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 16:10:12.601 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:12.601 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:12.601 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:10:12.601 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:12.601 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:12.602 | INFO | Uploaded 36GiB to GPU in 2.20346s (16.338GiB/s) | 20260505 16:10:12.602 | INFO | Downloaded 34GiB to GPU in 2.06559s (16.4602GiB/s) | 20260505 16:10:17.753 | INFO | Analyzing chunk frames = [4535296,4535808) | 20260505 16:10:17.753 | INFO | MemInfo: { | 20260505 16:10:17.753 | INFO | "gpu" : " 17 GB", | 20260505 16:10:17.753 | INFO | "huge" : "", | 20260505 16:10:17.753 | INFO | "huge_shared" : " 27 GB", | 20260505 16:10:17.753 | INFO | "malloc" : " 14 GB", | 20260505 16:10:17.753 | INFO | "pinned" : " 29 GB" | 20260505 16:10:17.753 | INFO | } | 20260505 16:10:17.753 | INFO | Perf report for the last 5.15208 seconds: | 20260505 16:10:17.753 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:10:17.753 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:17.753 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:17.753 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:17.753 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 16:10:17.753 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:17.753 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:17.753 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:10:17.753 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:17.753 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:17.754 | INFO | Uploaded 36GiB to GPU in 2.20083s (16.3575GiB/s) | 20260505 16:10:17.754 | INFO | Downloaded 34GiB to GPU in 2.0505s (16.5813GiB/s) | 20260505 16:10:22.897 | INFO | Analyzing chunk frames = [4535808,4536320) | 20260505 16:10:22.897 | INFO | MemInfo: { | 20260505 16:10:22.897 | INFO | "gpu" : " 17 GB", | 20260505 16:10:22.897 | INFO | "huge" : "", | 20260505 16:10:22.897 | INFO | "huge_shared" : " 27 GB", | 20260505 16:10:22.897 | INFO | "malloc" : " 14 GB", | 20260505 16:10:22.897 | INFO | "pinned" : " 29 GB" | 20260505 16:10:22.897 | INFO | } | 20260505 16:10:22.897 | INFO | Perf report for the last 5.14423 seconds: | 20260505 16:10:22.898 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:10:22.898 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:22.898 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:22.898 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:22.898 | INFO | ANALYSIS : 62.8%, 3.0, 97.6ms, 0 | 20260505 16:10:22.898 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:22.898 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:22.898 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 16:10:22.898 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:22.898 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:22.898 | INFO | Uploaded 36GiB to GPU in 2.22222s (16.2GiB/s) | 20260505 16:10:22.898 | INFO | Downloaded 34GiB to GPU in 2.12419s (16.0061GiB/s) | 20260505 16:10:28.027 | INFO | Analyzing chunk frames = [4536320,4536832) | 20260505 16:10:28.027 | INFO | MemInfo: { | 20260505 16:10:28.027 | INFO | "gpu" : " 17 GB", | 20260505 16:10:28.027 | INFO | "huge" : "", | 20260505 16:10:28.027 | INFO | "huge_shared" : " 27 GB", | 20260505 16:10:28.027 | INFO | "malloc" : " 14 GB", | 20260505 16:10:28.027 | INFO | "pinned" : " 29 GB" | 20260505 16:10:28.027 | INFO | } | 20260505 16:10:28.027 | INFO | Perf report for the last 5.1297 seconds: | 20260505 16:10:28.027 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:10:28.027 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:28.027 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:28.027 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:28.027 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 16:10:28.027 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:28.027 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:28.027 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 16:10:28.027 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:28.027 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:28.028 | INFO | Uploaded 36GiB to GPU in 2.14736s (16.7648GiB/s) | 20260505 16:10:28.028 | INFO | Downloaded 34GiB to GPU in 2.0594s (16.5097GiB/s) | 20260505 16:10:33.160 | INFO | Analyzing chunk frames = [4536832,4537344) | 20260505 16:10:33.160 | INFO | MemInfo: { | 20260505 16:10:33.160 | INFO | "gpu" : " 17 GB", | 20260505 16:10:33.160 | INFO | "huge" : "", | 20260505 16:10:33.160 | INFO | "huge_shared" : " 27 GB", | 20260505 16:10:33.160 | INFO | "malloc" : " 14 GB", | 20260505 16:10:33.160 | INFO | "pinned" : " 29 GB" | 20260505 16:10:33.160 | INFO | } | 20260505 16:10:33.160 | INFO | Perf report for the last 5.13293 seconds: | 20260505 16:10:33.160 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:10:33.160 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:33.160 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:33.160 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:33.160 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260505 16:10:33.160 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:33.160 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:33.160 | INFO | PRE_HQ : 27.7%, 1.0, 14.3ms, 0 | 20260505 16:10:33.160 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:33.160 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:33.160 | INFO | Uploaded 36GiB to GPU in 2.15066s (16.739GiB/s) | 20260505 16:10:33.161 | INFO | Downloaded 34GiB to GPU in 2.03549s (16.7036GiB/s) | 20260505 16:10:38.302 | INFO | Analyzing chunk frames = [4537344,4537856) | 20260505 16:10:38.302 | INFO | MemInfo: { | 20260505 16:10:38.302 | INFO | "gpu" : " 17 GB", | 20260505 16:10:38.302 | INFO | "huge" : "", | 20260505 16:10:38.302 | INFO | "huge_shared" : " 27 GB", | 20260505 16:10:38.302 | INFO | "malloc" : " 14 GB", | 20260505 16:10:38.302 | INFO | "pinned" : " 29 GB" | 20260505 16:10:38.302 | INFO | } | 20260505 16:10:38.302 | INFO | Perf report for the last 5.14203 seconds: | 20260505 16:10:38.302 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:10:38.302 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:38.302 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:38.302 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:38.302 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 16:10:38.302 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:38.302 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:38.302 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 16:10:38.302 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:38.302 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:38.302 | INFO | Uploaded 36GiB to GPU in 2.12581s (16.9347GiB/s) | 20260505 16:10:38.303 | INFO | Downloaded 34GiB to GPU in 2.1706s (15.6638GiB/s) | 20260505 16:10:43.445 | INFO | Analyzing chunk frames = [4537856,4538368) | 20260505 16:10:43.445 | INFO | MemInfo: { | 20260505 16:10:43.445 | INFO | "gpu" : " 17 GB", | 20260505 16:10:43.445 | INFO | "huge" : "", | 20260505 16:10:43.445 | INFO | "huge_shared" : " 27 GB", | 20260505 16:10:43.445 | INFO | "malloc" : " 14 GB", | 20260505 16:10:43.445 | INFO | "pinned" : " 29 GB" | 20260505 16:10:43.445 | INFO | } | 20260505 16:10:43.445 | INFO | Perf report for the last 5.14302 seconds: | 20260505 16:10:43.445 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:10:43.445 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:43.445 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:43.445 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:43.445 | INFO | ANALYSIS : 63.7%, 3.0, 99.6ms, 0 | 20260505 16:10:43.445 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:43.445 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:43.445 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 16:10:43.445 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:43.445 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:43.445 | INFO | Uploaded 36GiB to GPU in 2.237s (16.093GiB/s) | 20260505 16:10:43.446 | INFO | Downloaded 34GiB to GPU in 2.10528s (16.1499GiB/s) | 20260505 16:10:43.446 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 16:10:43.446 | INFO | Total mem usage: 15.3 GiB | 20260505 16:10:43.446 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 16:10:43.446 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.88%) | 20260505 16:10:43.446 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 16:10:43.446 | INFO | High Water Breakdown: | 20260505 16:10:43.446 | INFO | 61% : 8.67 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 16:10:43.446 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 16:10:43.446 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 16:10:43.446 | INFO | ------------------------------------------ | 20260505 16:10:43.446 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:10:43.446 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 16:10:43.446 | INFO | Total mem usage: 26.5 GiB | 20260505 16:10:43.446 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 16:10:43.446 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 16:10:43.446 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 16:10:43.446 | INFO | High Water Breakdown: | 20260505 16:10:43.446 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:10:43.446 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:10:43.446 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:10:43.446 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:10:43.446 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:10:43.446 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:10:43.446 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 16:10:43.446 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 16:10:43.446 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:10:43.446 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 16:10:43.446 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 16:10:43.446 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 16:10:43.446 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 16:10:43.446 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 16:10:43.446 | INFO | ------------------------------------------ | 20260505 16:10:43.446 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:10:43.446 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 16:10:43.446 | INFO | Total mem usage: 25.6 GiB | 20260505 16:10:43.446 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 16:10:43.446 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 16:10:43.446 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 16:10:43.446 | INFO | High Water Breakdown: | 20260505 16:10:43.446 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 16:10:43.446 | INFO | ------------------------------------------ | 20260505 16:10:43.446 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:10:43.446 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 16:10:43.446 | INFO | Total mem usage: 15.7 GiB | 20260505 16:10:43.446 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 16:10:43.446 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 16:10:43.446 | INFO | Unused Cache: 896 MiB (Waste: 5.58%) | 20260505 16:10:43.446 | INFO | High Water Breakdown: | 20260505 16:10:43.446 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 16:10:43.446 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 16:10:43.446 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 16:10:43.446 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 16:10:43.446 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 16:10:43.446 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 16:10:43.446 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 16:10:43.446 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 16:10:43.446 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 16:10:43.446 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:10:43.446 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 16:10:43.446 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:10:43.446 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 16:10:43.446 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 16:10:43.446 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 16:10:43.446 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 16:10:43.446 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:10:43.446 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 16:10:43.446 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:10:43.446 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:10:43.446 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:10:43.446 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 16:10:43.446 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 16:10:43.446 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:10:43.446 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 16:10:43.446 | INFO | ------------------------------------------ | 20260505 16:10:43.446 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:10:43.446 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 16:10:43.446 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:10:43.446 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:10:43.446 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.51e+01, 1.34e+01 / 1.85e+00 | 20260505 16:10:43.446 | INFO | PerfReport: 0.29%, AggregateStats -- 9.24e-02 / 1.02e-01, 1.03e-02 / 4.92e-03 | 20260505 16:10:43.446 | INFO | PerfReport: 11.77%, Baseline -- 3.72e+00 / 4.51e+00, 5.66e-01 / 1.63e-01 | 20260505 16:10:43.446 | INFO | PerfReport: 3.26%, Histogram -- 1.03e+00 / 1.51e+00, 1.89e-01 / 1.09e-01 | 20260505 16:10:43.446 | INFO | PerfReport: 15.96%, DME -- 4.04e+01 / 4.49e+01, 4.20e+00 / 1.87e+00 | 20260505 16:10:43.447 | INFO | PerfReport: 56.66%, FrameLabeling -- 1.79e+01 / 1.93e+01, 1.40e+00 / 1.35e+00 | 20260505 16:10:43.447 | INFO | PerfReport: 7.43%, PulseAccumulating -- 2.35e+00 / 2.91e+00, 4.55e-01 / 1.92e-01 | 20260505 16:10:43.447 | INFO | PerfReport: 3.44%, Metrics -- 1.09e+00 / 1.41e+00, 3.01e-01 / 1.70e-01 | 20260505 16:10:43.447 | INFO | PerfReport: 1.19%, PulseSerializing -- 3.78e-01 / 4.33e-01, 6.74e-02 / 4.06e-02 | 20260505 16:10:43.447 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 16:10:43.447 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:10:43.447 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:10:43.447 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.58e+01 / 0.00e+00 | 20260505 16:10:43.447 | INFO | PerfReport: 22.62%, Upload -- 2.27e+01 / 0.00e+00, 8.01e+00 / 0.00e+00 | 20260505 16:10:43.447 | INFO | PerfReport: 21.84%, Download -- 2.19e+01 / 0.00e+00, 7.90e+00 / 0.00e+00 | 20260505 16:10:43.447 | INFO | PerfReport: 31.53%, Compute -- 3.16e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 16:10:43.447 | INFO | PerfReport: 2.46%, LockUpload -- 2.47e+00 / 0.00e+00, 8.10e+00 / 0.00e+00 | 20260505 16:10:43.447 | INFO | PerfReport: 1.37%, LockDownload -- 1.37e+00 / 0.00e+00, 2.93e+00 / 0.00e+00 | 20260505 16:10:43.447 | INFO | PerfReport: 20.16%, LockCompute -- 2.02e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260505 16:10:43.447 | INFO | PerfReport: 0.02%, Misc -- 1.58e-02 / 0.00e+00, 3.18e-02 / 0.00e+00 | 20260505 16:10:43.580 | INFO | Starting a round of Baz IO | 20260505 16:10:46.788 | INFO | Populating RTMetrics report | 20260505 16:10:46.788 | INFO | RTMetrics report populated | 20260505 16:10:46.788 | INFO | Opening tmp RTMetrics json file | 20260505 16:10:46.831 | INFO | Serializing RTMetrics to json | 20260505 16:10:46.871 | INFO | Serialization done, swapping tmp file now... | 20260505 16:10:46.875 | INFO | RTMetrics done | 20260505 16:10:46.893 | INFO | Finished a round of Baz IO: Wrote 2.18 GiB in 3.312 seconds | 20260505 16:10:46.893 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.30% / 0.00% / 77.70% | 20260505 16:10:46.893 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.383194% / 0% / 6.35783e-05% | 20260505 16:10:46.893 | INFO | Baz IO overall: | 20260505 16:10:46.893 | INFO | BytesWritten : 2695041864750 | 20260505 16:10:46.893 | INFO | NumEvents : 1279512582275 | 20260505 16:10:48.588 | INFO | Analyzing chunk frames = [4538368,4538880) | 20260505 16:10:48.588 | INFO | MemInfo: { | 20260505 16:10:48.588 | INFO | "gpu" : " 17 GB", | 20260505 16:10:48.588 | INFO | "huge" : "", | 20260505 16:10:48.588 | INFO | "huge_shared" : " 27 GB", | 20260505 16:10:48.588 | INFO | "malloc" : " 14 GB", | 20260505 16:10:48.588 | INFO | "pinned" : " 29 GB" | 20260505 16:10:48.588 | INFO | } | 20260505 16:10:48.588 | INFO | Perf report for the last 5.1425 seconds: | 20260505 16:10:48.588 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:10:48.588 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:48.588 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:48.588 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:48.588 | INFO | ANALYSIS : 63.9%, 3.0, 99.4ms, 0 | 20260505 16:10:48.588 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:48.588 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260505 16:10:48.588 | INFO | PRE_HQ : 60.9%, 1.8, 56.6ms, 0 | 20260505 16:10:48.588 | INFO | BAZWRITER : 63.8%, 3.9, 128.9ms, 0 | 20260505 16:10:48.588 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 16:10:48.588 | INFO | Uploaded 36GiB to GPU in 2.19196s (16.4236GiB/s) | 20260505 16:10:48.588 | INFO | Downloaded 38GiB to GPU in 2.27213s (16.7244GiB/s) | 20260505 16:10:53.723 | INFO | Analyzing chunk frames = [4538880,4539392) | 20260505 16:10:53.723 | INFO | MemInfo: { | 20260505 16:10:53.723 | INFO | "gpu" : " 17 GB", | 20260505 16:10:53.723 | INFO | "huge" : "", | 20260505 16:10:53.723 | INFO | "huge_shared" : " 27 GB", | 20260505 16:10:53.723 | INFO | "malloc" : " 14 GB", | 20260505 16:10:53.723 | INFO | "pinned" : " 29 GB" | 20260505 16:10:53.723 | INFO | } | 20260505 16:10:53.723 | INFO | Perf report for the last 5.13569 seconds: | 20260505 16:10:53.723 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:10:53.723 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:53.723 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:53.723 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:53.723 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 16:10:53.723 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:53.723 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:53.723 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:10:53.723 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:53.723 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:53.724 | INFO | Uploaded 36GiB to GPU in 2.13286s (16.8788GiB/s) | 20260505 16:10:53.724 | INFO | Downloaded 34GiB to GPU in 2.02482s (16.7916GiB/s) | 20260505 16:10:58.859 | INFO | Analyzing chunk frames = [4539392,4539904) | 20260505 16:10:58.859 | INFO | MemInfo: { | 20260505 16:10:58.859 | INFO | "gpu" : " 17 GB", | 20260505 16:10:58.859 | INFO | "huge" : "", | 20260505 16:10:58.859 | INFO | "huge_shared" : " 27 GB", | 20260505 16:10:58.859 | INFO | "malloc" : " 14 GB", | 20260505 16:10:58.859 | INFO | "pinned" : " 29 GB" | 20260505 16:10:58.859 | INFO | } | 20260505 16:10:58.859 | INFO | Perf report for the last 5.1354 seconds: | 20260505 16:10:58.859 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:10:58.859 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:58.859 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:58.859 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:58.859 | INFO | ANALYSIS : 62.9%, 3.0, 97.1ms, 0 | 20260505 16:10:58.859 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:58.859 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:58.859 | INFO | PRE_HQ : 28.0%, 1.0, 14.4ms, 0 | 20260505 16:10:58.859 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:58.859 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:10:58.859 | INFO | Uploaded 36GiB to GPU in 2.17468s (16.5542GiB/s) | 20260505 16:10:58.859 | INFO | Downloaded 34GiB to GPU in 2.04012s (16.6657GiB/s) | 20260505 16:11:04.007 | INFO | Analyzing chunk frames = [4539904,4540416) | 20260505 16:11:04.007 | INFO | MemInfo: { | 20260505 16:11:04.007 | INFO | "gpu" : " 17 GB", | 20260505 16:11:04.007 | INFO | "huge" : "", | 20260505 16:11:04.007 | INFO | "huge_shared" : " 27 GB", | 20260505 16:11:04.007 | INFO | "malloc" : " 14 GB", | 20260505 16:11:04.007 | INFO | "pinned" : " 29 GB" | 20260505 16:11:04.007 | INFO | } | 20260505 16:11:04.007 | INFO | Perf report for the last 5.14834 seconds: | 20260505 16:11:04.007 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:11:04.007 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:04.007 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:04.007 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:04.007 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260505 16:11:04.007 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:04.007 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:04.007 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 16:11:04.007 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:04.007 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:04.007 | INFO | Uploaded 36GiB to GPU in 2.18547s (16.4724GiB/s) | 20260505 16:11:04.007 | INFO | Downloaded 34GiB to GPU in 2.13139s (15.952GiB/s) | 20260505 16:11:09.142 | INFO | Analyzing chunk frames = [4540416,4540928) | 20260505 16:11:09.142 | INFO | MemInfo: { | 20260505 16:11:09.142 | INFO | "gpu" : " 17 GB", | 20260505 16:11:09.142 | INFO | "huge" : "", | 20260505 16:11:09.142 | INFO | "huge_shared" : " 27 GB", | 20260505 16:11:09.142 | INFO | "malloc" : " 14 GB", | 20260505 16:11:09.142 | INFO | "pinned" : " 29 GB" | 20260505 16:11:09.142 | INFO | } | 20260505 16:11:09.142 | INFO | Perf report for the last 5.13527 seconds: | 20260505 16:11:09.142 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:11:09.142 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:09.142 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:09.142 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:09.142 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 16:11:09.142 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:09.142 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:09.142 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:11:09.142 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:09.142 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:09.143 | INFO | Uploaded 36GiB to GPU in 2.15608s (16.697GiB/s) | 20260505 16:11:09.143 | INFO | Downloaded 34GiB to GPU in 2.0602s (16.5033GiB/s) | 20260505 16:11:14.278 | INFO | Analyzing chunk frames = [4540928,4541440) | 20260505 16:11:14.278 | INFO | MemInfo: { | 20260505 16:11:14.278 | INFO | "gpu" : " 17 GB", | 20260505 16:11:14.278 | INFO | "huge" : "", | 20260505 16:11:14.278 | INFO | "huge_shared" : " 27 GB", | 20260505 16:11:14.278 | INFO | "malloc" : " 14 GB", | 20260505 16:11:14.278 | INFO | "pinned" : " 29 GB" | 20260505 16:11:14.278 | INFO | } | 20260505 16:11:14.278 | INFO | Perf report for the last 5.13576 seconds: | 20260505 16:11:14.278 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:11:14.278 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:14.278 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:14.278 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:14.278 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260505 16:11:14.278 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:14.278 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:14.278 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:11:14.278 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:14.278 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:14.278 | INFO | Uploaded 36GiB to GPU in 2.1424s (16.8036GiB/s) | 20260505 16:11:14.279 | INFO | Downloaded 34GiB to GPU in 2.03529s (16.7053GiB/s) | 20260505 16:11:19.414 | INFO | Analyzing chunk frames = [4541440,4541952) | 20260505 16:11:19.414 | INFO | MemInfo: { | 20260505 16:11:19.414 | INFO | "gpu" : " 17 GB", | 20260505 16:11:19.414 | INFO | "huge" : "", | 20260505 16:11:19.414 | INFO | "huge_shared" : " 27 GB", | 20260505 16:11:19.414 | INFO | "malloc" : " 14 GB", | 20260505 16:11:19.414 | INFO | "pinned" : " 29 GB" | 20260505 16:11:19.414 | INFO | } | 20260505 16:11:19.414 | INFO | Perf report for the last 5.13625 seconds: | 20260505 16:11:19.414 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:11:19.414 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:19.414 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:19.414 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:19.414 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 16:11:19.414 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:19.415 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:19.415 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 16:11:19.415 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:19.415 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:19.415 | INFO | Uploaded 36GiB to GPU in 2.11425s (17.0273GiB/s) | 20260505 16:11:19.415 | INFO | Downloaded 34GiB to GPU in 2.17649s (15.6215GiB/s) | 20260505 16:11:24.554 | INFO | Analyzing chunk frames = [4541952,4542464) | 20260505 16:11:24.554 | INFO | MemInfo: { | 20260505 16:11:24.554 | INFO | "gpu" : " 17 GB", | 20260505 16:11:24.554 | INFO | "huge" : "", | 20260505 16:11:24.554 | INFO | "huge_shared" : " 27 GB", | 20260505 16:11:24.554 | INFO | "malloc" : " 14 GB", | 20260505 16:11:24.554 | INFO | "pinned" : " 29 GB" | 20260505 16:11:24.554 | INFO | } | 20260505 16:11:24.554 | INFO | Perf report for the last 5.14003 seconds: | 20260505 16:11:24.554 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:11:24.554 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:24.554 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:24.554 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:24.555 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 16:11:24.555 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:24.555 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:24.555 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:11:24.555 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:24.555 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:24.555 | INFO | Uploaded 36GiB to GPU in 2.17981s (16.5152GiB/s) | 20260505 16:11:24.555 | INFO | Downloaded 34GiB to GPU in 2.03414s (16.7146GiB/s) | 20260505 16:11:24.684 | INFO | Starting a round of Baz IO | 20260505 16:11:27.900 | INFO | Populating RTMetrics report | 20260505 16:11:27.900 | INFO | RTMetrics report populated | 20260505 16:11:27.900 | INFO | Opening tmp RTMetrics json file | 20260505 16:11:27.910 | INFO | Serializing RTMetrics to json | 20260505 16:11:27.913 | INFO | Serialization done, swapping tmp file now... | 20260505 16:11:27.916 | INFO | RTMetrics done | 20260505 16:11:28.003 | INFO | Finished a round of Baz IO: Wrote 2.181 GiB in 3.319 seconds | 20260505 16:11:28.003 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.30% / 0.00% / 77.70% | 20260505 16:11:28.003 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.383862% / 3.97364e-06% / 9.13938e-05% | 20260505 16:11:28.003 | INFO | Baz IO overall: | 20260505 16:11:28.003 | INFO | BytesWritten : 2697384024131 | 20260505 16:11:28.003 | INFO | NumEvents : 1280611328965 | 20260505 16:11:29.700 | INFO | Analyzing chunk frames = [4542464,4542976) | 20260505 16:11:29.700 | INFO | MemInfo: { | 20260505 16:11:29.700 | INFO | "gpu" : " 17 GB", | 20260505 16:11:29.700 | INFO | "huge" : "", | 20260505 16:11:29.700 | INFO | "huge_shared" : " 27 GB", | 20260505 16:11:29.700 | INFO | "malloc" : " 14 GB", | 20260505 16:11:29.700 | INFO | "pinned" : " 29 GB" | 20260505 16:11:29.700 | INFO | } | 20260505 16:11:29.701 | INFO | Perf report for the last 5.14617 seconds: | 20260505 16:11:29.701 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:11:29.701 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:29.701 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:29.701 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:29.701 | INFO | ANALYSIS : 63.9%, 3.0, 99.6ms, 0 | 20260505 16:11:29.701 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:29.701 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260505 16:11:29.701 | INFO | PRE_HQ : 60.8%, 1.8, 56.4ms, 0 | 20260505 16:11:29.701 | INFO | BAZWRITER : 63.8%, 3.9, 127.7ms, 0 | 20260505 16:11:29.701 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 16:11:29.701 | INFO | Uploaded 36GiB to GPU in 2.19979s (16.3652GiB/s) | 20260505 16:11:29.701 | INFO | Downloaded 38GiB to GPU in 2.25777s (16.8307GiB/s) | 20260505 16:11:34.844 | INFO | Analyzing chunk frames = [4542976,4543488) | 20260505 16:11:34.844 | INFO | MemInfo: { | 20260505 16:11:34.844 | INFO | "gpu" : " 17 GB", | 20260505 16:11:34.844 | INFO | "huge" : "", | 20260505 16:11:34.844 | INFO | "huge_shared" : " 27 GB", | 20260505 16:11:34.844 | INFO | "malloc" : " 14 GB", | 20260505 16:11:34.844 | INFO | "pinned" : " 29 GB" | 20260505 16:11:34.844 | INFO | } | 20260505 16:11:34.844 | INFO | Perf report for the last 5.14323 seconds: | 20260505 16:11:34.844 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:11:34.844 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:34.844 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:34.844 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:34.844 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 16:11:34.844 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:34.844 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:34.844 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:11:34.844 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:34.844 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:34.844 | INFO | Uploaded 36GiB to GPU in 2.14003s (16.8222GiB/s) | 20260505 16:11:34.844 | INFO | Downloaded 34GiB to GPU in 2.02896s (16.7573GiB/s) | 20260505 16:11:39.977 | INFO | Analyzing chunk frames = [4543488,4544000) | 20260505 16:11:39.977 | INFO | MemInfo: { | 20260505 16:11:39.977 | INFO | "gpu" : " 17 GB", | 20260505 16:11:39.977 | INFO | "huge" : "", | 20260505 16:11:39.977 | INFO | "huge_shared" : " 27 GB", | 20260505 16:11:39.977 | INFO | "malloc" : " 14 GB", | 20260505 16:11:39.977 | INFO | "pinned" : " 29 GB" | 20260505 16:11:39.977 | INFO | } | 20260505 16:11:39.977 | INFO | Perf report for the last 5.13312 seconds: | 20260505 16:11:39.977 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:11:39.977 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:39.977 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:39.977 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:39.977 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260505 16:11:39.977 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:39.977 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:39.977 | INFO | PRE_HQ : 27.9%, 1.0, 14.4ms, 0 | 20260505 16:11:39.977 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:39.977 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:39.977 | INFO | Uploaded 36GiB to GPU in 2.20148s (16.3526GiB/s) | 20260505 16:11:39.977 | INFO | Downloaded 34GiB to GPU in 2.03465s (16.7105GiB/s) | 20260505 16:11:45.109 | INFO | Analyzing chunk frames = [4544000,4544512) | 20260505 16:11:45.109 | INFO | MemInfo: { | 20260505 16:11:45.109 | INFO | "gpu" : " 17 GB", | 20260505 16:11:45.109 | INFO | "huge" : "", | 20260505 16:11:45.109 | INFO | "huge_shared" : " 27 GB", | 20260505 16:11:45.109 | INFO | "malloc" : " 14 GB", | 20260505 16:11:45.109 | INFO | "pinned" : " 29 GB" | 20260505 16:11:45.109 | INFO | } | 20260505 16:11:45.109 | INFO | Perf report for the last 5.13239 seconds: | 20260505 16:11:45.110 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:11:45.110 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:45.110 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:45.110 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:45.110 | INFO | ANALYSIS : 62.6%, 3.0, 97.1ms, 0 | 20260505 16:11:45.110 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:45.110 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:45.110 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 16:11:45.110 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:45.110 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:45.110 | INFO | Uploaded 36GiB to GPU in 2.18595s (16.4688GiB/s) | 20260505 16:11:45.110 | INFO | Downloaded 34GiB to GPU in 2.12666s (15.9875GiB/s) | 20260505 16:11:50.259 | INFO | Analyzing chunk frames = [4544512,4545024) | 20260505 16:11:50.259 | INFO | MemInfo: { | 20260505 16:11:50.259 | INFO | "gpu" : " 17 GB", | 20260505 16:11:50.259 | INFO | "huge" : "", | 20260505 16:11:50.259 | INFO | "huge_shared" : " 27 GB", | 20260505 16:11:50.259 | INFO | "malloc" : " 14 GB", | 20260505 16:11:50.259 | INFO | "pinned" : " 29 GB" | 20260505 16:11:50.259 | INFO | } | 20260505 16:11:50.259 | INFO | Perf report for the last 5.14937 seconds: | 20260505 16:11:50.259 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:11:50.259 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:50.259 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:50.259 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:50.259 | INFO | ANALYSIS : 62.5%, 3.0, 97.7ms, 0 | 20260505 16:11:50.259 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:50.259 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:50.259 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:11:50.259 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:50.259 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:50.259 | INFO | Uploaded 36GiB to GPU in 2.1593s (16.6721GiB/s) | 20260505 16:11:50.259 | INFO | Downloaded 34GiB to GPU in 2.12924s (15.9682GiB/s) | 20260505 16:11:55.403 | INFO | Analyzing chunk frames = [4545024,4545536) | 20260505 16:11:55.403 | INFO | MemInfo: { | 20260505 16:11:55.403 | INFO | "gpu" : " 17 GB", | 20260505 16:11:55.403 | INFO | "huge" : "", | 20260505 16:11:55.403 | INFO | "huge_shared" : " 27 GB", | 20260505 16:11:55.403 | INFO | "malloc" : " 14 GB", | 20260505 16:11:55.403 | INFO | "pinned" : " 29 GB" | 20260505 16:11:55.403 | INFO | } | 20260505 16:11:55.403 | INFO | Perf report for the last 5.1442 seconds: | 20260505 16:11:55.403 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:11:55.403 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:55.403 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:55.403 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:55.403 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260505 16:11:55.403 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:55.403 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:55.403 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 16:11:55.403 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:55.403 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:11:55.403 | INFO | Uploaded 36GiB to GPU in 2.13952s (16.8262GiB/s) | 20260505 16:11:55.403 | INFO | Downloaded 34GiB to GPU in 2.03502s (16.7075GiB/s) | 20260505 16:12:00.535 | INFO | Analyzing chunk frames = [4545536,4546048) | 20260505 16:12:00.535 | INFO | MemInfo: { | 20260505 16:12:00.535 | INFO | "gpu" : " 17 GB", | 20260505 16:12:00.535 | INFO | "huge" : "", | 20260505 16:12:00.535 | INFO | "huge_shared" : " 27 GB", | 20260505 16:12:00.535 | INFO | "malloc" : " 14 GB", | 20260505 16:12:00.535 | INFO | "pinned" : " 29 GB" | 20260505 16:12:00.535 | INFO | } | 20260505 16:12:00.535 | INFO | Perf report for the last 5.13215 seconds: | 20260505 16:12:00.535 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:12:00.535 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:00.535 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:00.535 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:00.535 | INFO | ANALYSIS : 63.6%, 3.0, 99.1ms, 0 | 20260505 16:12:00.535 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:00.535 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:00.535 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 16:12:00.535 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:00.535 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:00.535 | INFO | Uploaded 36GiB to GPU in 2.16303s (16.6433GiB/s) | 20260505 16:12:00.535 | INFO | Downloaded 34GiB to GPU in 2.17194s (15.6542GiB/s) | 20260505 16:12:05.671 | INFO | Analyzing chunk frames = [4546048,4546560) | 20260505 16:12:05.671 | INFO | MemInfo: { | 20260505 16:12:05.671 | INFO | "gpu" : " 17 GB", | 20260505 16:12:05.671 | INFO | "huge" : "", | 20260505 16:12:05.671 | INFO | "huge_shared" : " 27 GB", | 20260505 16:12:05.671 | INFO | "malloc" : " 14 GB", | 20260505 16:12:05.671 | INFO | "pinned" : " 29 GB" | 20260505 16:12:05.671 | INFO | } | 20260505 16:12:05.671 | INFO | Perf report for the last 5.13627 seconds: | 20260505 16:12:05.671 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:12:05.671 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:05.671 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:05.671 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:05.671 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 16:12:05.671 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:05.672 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:05.672 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 16:12:05.672 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:05.672 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:05.672 | INFO | Uploaded 36GiB to GPU in 2.15775s (16.6841GiB/s) | 20260505 16:12:05.672 | INFO | Downloaded 34GiB to GPU in 2.03366s (16.7187GiB/s) | 20260505 16:12:05.672 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 16:12:05.672 | INFO | Total mem usage: 15.3 GiB | 20260505 16:12:05.672 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 16:12:05.672 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.85%) | 20260505 16:12:05.672 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 16:12:05.672 | INFO | High Water Breakdown: | 20260505 16:12:05.672 | INFO | 61% : 8.67 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 16:12:05.672 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 16:12:05.672 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 16:12:05.672 | INFO | ------------------------------------------ | 20260505 16:12:05.672 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:12:05.672 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 16:12:05.672 | INFO | Total mem usage: 26.5 GiB | 20260505 16:12:05.672 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 16:12:05.672 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 16:12:05.672 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 16:12:05.672 | INFO | High Water Breakdown: | 20260505 16:12:05.672 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:12:05.672 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:12:05.672 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:12:05.672 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:12:05.672 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:12:05.672 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:12:05.672 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 16:12:05.672 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 16:12:05.672 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:12:05.672 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 16:12:05.672 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 16:12:05.672 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 16:12:05.672 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 16:12:05.672 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 16:12:05.672 | INFO | ------------------------------------------ | 20260505 16:12:05.672 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:12:05.672 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 16:12:05.672 | INFO | Total mem usage: 25.6 GiB | 20260505 16:12:05.672 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 16:12:05.672 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 16:12:05.672 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 16:12:05.672 | INFO | High Water Breakdown: | 20260505 16:12:05.672 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 16:12:05.672 | INFO | ------------------------------------------ | 20260505 16:12:05.672 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:12:05.672 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 16:12:05.672 | INFO | Total mem usage: 15.7 GiB | 20260505 16:12:05.672 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 16:12:05.672 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 16:12:05.672 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 16:12:05.672 | INFO | High Water Breakdown: | 20260505 16:12:05.672 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 16:12:05.672 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 16:12:05.672 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 16:12:05.672 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 16:12:05.672 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 16:12:05.672 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 16:12:05.672 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 16:12:05.672 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 16:12:05.672 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 16:12:05.672 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:12:05.672 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 16:12:05.672 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:12:05.672 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 16:12:05.672 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 16:12:05.672 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 16:12:05.672 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 16:12:05.672 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:12:05.672 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 16:12:05.672 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:12:05.672 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:12:05.672 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:12:05.673 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 16:12:05.673 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 16:12:05.673 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:12:05.673 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 16:12:05.673 | INFO | ------------------------------------------ | 20260505 16:12:05.673 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:12:05.673 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 16:12:05.673 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:12:05.673 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:12:05.673 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.52e+01, 1.36e+01 / 1.90e+00 | 20260505 16:12:05.673 | INFO | PerfReport: 0.29%, AggregateStats -- 9.26e-02 / 1.02e-01, 1.06e-02 / 3.60e-03 | 20260505 16:12:05.673 | INFO | PerfReport: 11.79%, Baseline -- 3.74e+00 / 4.52e+00, 5.80e-01 / 1.71e-01 | 20260505 16:12:05.673 | INFO | PerfReport: 3.30%, Histogram -- 1.05e+00 / 1.52e+00, 1.97e-01 / 7.39e-02 | 20260505 16:12:05.673 | INFO | PerfReport: 15.85%, DME -- 4.02e+01 / 4.44e+01, 4.06e+00 / 2.05e+00 | 20260505 16:12:05.673 | INFO | PerfReport: 56.80%, FrameLabeling -- 1.80e+01 / 1.99e+01, 1.53e+00 / 1.38e+00 | 20260505 16:12:05.673 | INFO | PerfReport: 7.34%, PulseAccumulating -- 2.33e+00 / 2.89e+00, 4.40e-01 / 1.37e-01 | 20260505 16:12:05.673 | INFO | PerfReport: 3.43%, Metrics -- 1.09e+00 / 1.41e+00, 3.03e-01 / 1.51e-01 | 20260505 16:12:05.673 | INFO | PerfReport: 1.19%, PulseSerializing -- 3.79e-01 / 4.38e-01, 6.82e-02 / 4.25e-02 | 20260505 16:12:05.673 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 16:12:05.673 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:12:05.673 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:12:05.673 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260505 16:12:05.673 | INFO | PerfReport: 22.50%, Upload -- 2.25e+01 / 0.00e+00, 7.87e+00 / 0.00e+00 | 20260505 16:12:05.673 | INFO | PerfReport: 21.83%, Download -- 2.19e+01 / 0.00e+00, 7.85e+00 / 0.00e+00 | 20260505 16:12:05.673 | INFO | PerfReport: 31.66%, Compute -- 3.17e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 16:12:05.673 | INFO | PerfReport: 2.17%, LockUpload -- 2.18e+00 / 0.00e+00, 7.80e+00 / 0.00e+00 | 20260505 16:12:05.673 | INFO | PerfReport: 1.43%, LockDownload -- 1.43e+00 / 0.00e+00, 2.97e+00 / 0.00e+00 | 20260505 16:12:05.673 | INFO | PerfReport: 20.40%, LockCompute -- 2.04e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 16:12:05.673 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.82e-02 / 0.00e+00 | 20260505 16:12:05.806 | INFO | Starting a round of Baz IO | 20260505 16:12:09.021 | INFO | Populating RTMetrics report | 20260505 16:12:09.022 | INFO | RTMetrics report populated | 20260505 16:12:09.022 | INFO | Opening tmp RTMetrics json file | 20260505 16:12:09.031 | INFO | Serializing RTMetrics to json | 20260505 16:12:09.033 | INFO | Serialization done, swapping tmp file now... | 20260505 16:12:09.033 | INFO | RTMetrics done | 20260505 16:12:09.116 | INFO | Finished a round of Baz IO: Wrote 2.18 GiB in 3.309 seconds | 20260505 16:12:09.116 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.30% / 0.00% / 77.70% | 20260505 16:12:09.116 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.381621% / 0% / 7.15256e-05% | 20260505 16:12:09.116 | INFO | Baz IO overall: | 20260505 16:12:09.116 | INFO | BytesWritten : 2699725031837 | 20260505 16:12:09.116 | INFO | NumEvents : 1281709281896 | 20260505 16:12:10.813 | INFO | Analyzing chunk frames = [4546560,4547072) | 20260505 16:12:10.813 | INFO | MemInfo: { | 20260505 16:12:10.813 | INFO | "gpu" : " 17 GB", | 20260505 16:12:10.813 | INFO | "huge" : "", | 20260505 16:12:10.813 | INFO | "huge_shared" : " 27 GB", | 20260505 16:12:10.813 | INFO | "malloc" : " 14 GB", | 20260505 16:12:10.813 | INFO | "pinned" : " 29 GB" | 20260505 16:12:10.813 | INFO | } | 20260505 16:12:10.813 | INFO | Perf report for the last 5.14117 seconds: | 20260505 16:12:10.813 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:12:10.813 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:10.813 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:10.813 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:10.813 | INFO | ANALYSIS : 64.1%, 3.0, 99.7ms, 0 | 20260505 16:12:10.813 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:10.813 | INFO | RT_METRICS : 44.9%, 1.0, 23.5ms, 0 | 20260505 16:12:10.813 | INFO | PRE_HQ : 60.8%, 1.8, 56.6ms, 0 | 20260505 16:12:10.813 | INFO | BAZWRITER : 63.6%, 3.9, 127.8ms, 0 | 20260505 16:12:10.813 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 16:12:10.813 | INFO | Uploaded 36GiB to GPU in 2.20863s (16.2997GiB/s) | 20260505 16:12:10.813 | INFO | Downloaded 38GiB to GPU in 2.26003s (16.8139GiB/s) | 20260505 16:12:15.954 | INFO | Analyzing chunk frames = [4547072,4547584) | 20260505 16:12:15.954 | INFO | MemInfo: { | 20260505 16:12:15.954 | INFO | "gpu" : " 17 GB", | 20260505 16:12:15.954 | INFO | "huge" : "", | 20260505 16:12:15.954 | INFO | "huge_shared" : " 27 GB", | 20260505 16:12:15.954 | INFO | "malloc" : " 14 GB", | 20260505 16:12:15.954 | INFO | "pinned" : " 29 GB" | 20260505 16:12:15.954 | INFO | } | 20260505 16:12:15.954 | INFO | Perf report for the last 5.14176 seconds: | 20260505 16:12:15.954 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:12:15.954 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:15.954 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:15.954 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:15.954 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 16:12:15.954 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:15.954 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:15.954 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:12:15.955 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:15.955 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:15.955 | INFO | Uploaded 36GiB to GPU in 2.13721s (16.8444GiB/s) | 20260505 16:12:15.955 | INFO | Downloaded 34GiB to GPU in 2.04185s (16.6516GiB/s) | 20260505 16:12:21.096 | INFO | Analyzing chunk frames = [4547584,4548096) | 20260505 16:12:21.096 | INFO | MemInfo: { | 20260505 16:12:21.096 | INFO | "gpu" : " 17 GB", | 20260505 16:12:21.096 | INFO | "huge" : "", | 20260505 16:12:21.096 | INFO | "huge_shared" : " 27 GB", | 20260505 16:12:21.096 | INFO | "malloc" : " 14 GB", | 20260505 16:12:21.096 | INFO | "pinned" : " 29 GB" | 20260505 16:12:21.096 | INFO | } | 20260505 16:12:21.096 | INFO | Perf report for the last 5.14194 seconds: | 20260505 16:12:21.096 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:12:21.096 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:21.096 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:21.096 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:21.096 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260505 16:12:21.096 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:21.096 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:21.096 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 16:12:21.097 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:21.097 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:21.097 | INFO | Uploaded 36GiB to GPU in 2.20315s (16.3402GiB/s) | 20260505 16:12:21.097 | INFO | Downloaded 34GiB to GPU in 2.03848s (16.6791GiB/s) | 20260505 16:12:26.228 | INFO | Analyzing chunk frames = [4548096,4548608) | 20260505 16:12:26.228 | INFO | MemInfo: { | 20260505 16:12:26.228 | INFO | "gpu" : " 17 GB", | 20260505 16:12:26.228 | INFO | "huge" : "", | 20260505 16:12:26.228 | INFO | "huge_shared" : " 27 GB", | 20260505 16:12:26.228 | INFO | "malloc" : " 14 GB", | 20260505 16:12:26.228 | INFO | "pinned" : " 29 GB" | 20260505 16:12:26.229 | INFO | } | 20260505 16:12:26.229 | INFO | Perf report for the last 5.13224 seconds: | 20260505 16:12:26.229 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:12:26.229 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:26.229 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:26.229 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:26.229 | INFO | ANALYSIS : 62.8%, 3.0, 97.4ms, 0 | 20260505 16:12:26.229 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:26.229 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:26.229 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:12:26.229 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:26.229 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:26.229 | INFO | Uploaded 36GiB to GPU in 2.17088s (16.5831GiB/s) | 20260505 16:12:26.229 | INFO | Downloaded 34GiB to GPU in 2.14506s (15.8503GiB/s) | 20260505 16:12:31.364 | INFO | Analyzing chunk frames = [4548608,4549120) | 20260505 16:12:31.364 | INFO | MemInfo: { | 20260505 16:12:31.364 | INFO | "gpu" : " 17 GB", | 20260505 16:12:31.364 | INFO | "huge" : "", | 20260505 16:12:31.364 | INFO | "huge_shared" : " 27 GB", | 20260505 16:12:31.364 | INFO | "malloc" : " 14 GB", | 20260505 16:12:31.364 | INFO | "pinned" : " 29 GB" | 20260505 16:12:31.365 | INFO | } | 20260505 16:12:31.365 | INFO | Perf report for the last 5.13599 seconds: | 20260505 16:12:31.365 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:12:31.365 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:31.365 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:31.365 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:31.365 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 16:12:31.365 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:31.365 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:31.365 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:12:31.365 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:31.365 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:31.365 | INFO | Uploaded 36GiB to GPU in 2.17829s (16.5268GiB/s) | 20260505 16:12:31.365 | INFO | Downloaded 34GiB to GPU in 2.12676s (15.9868GiB/s) | 20260505 16:12:36.514 | INFO | Analyzing chunk frames = [4549120,4549632) | 20260505 16:12:36.515 | INFO | MemInfo: { | 20260505 16:12:36.515 | INFO | "gpu" : " 17 GB", | 20260505 16:12:36.515 | INFO | "huge" : "", | 20260505 16:12:36.515 | INFO | "huge_shared" : " 27 GB", | 20260505 16:12:36.515 | INFO | "malloc" : " 14 GB", | 20260505 16:12:36.515 | INFO | "pinned" : " 29 GB" | 20260505 16:12:36.515 | INFO | } | 20260505 16:12:36.515 | INFO | Perf report for the last 5.15011 seconds: | 20260505 16:12:36.515 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:12:36.515 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:36.515 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:36.515 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:36.515 | INFO | ANALYSIS : 62.5%, 3.0, 97.9ms, 0 | 20260505 16:12:36.515 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:36.515 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:36.515 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 16:12:36.515 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:36.515 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:36.515 | INFO | Uploaded 36GiB to GPU in 2.1681s (16.6044GiB/s) | 20260505 16:12:36.515 | INFO | Downloaded 34GiB to GPU in 2.04187s (16.6514GiB/s) | 20260505 16:12:41.656 | INFO | Analyzing chunk frames = [4549632,4550144) | 20260505 16:12:41.656 | INFO | MemInfo: { | 20260505 16:12:41.656 | INFO | "gpu" : " 17 GB", | 20260505 16:12:41.656 | INFO | "huge" : "", | 20260505 16:12:41.656 | INFO | "huge_shared" : " 27 GB", | 20260505 16:12:41.656 | INFO | "malloc" : " 14 GB", | 20260505 16:12:41.657 | INFO | "pinned" : " 29 GB" | 20260505 16:12:41.657 | INFO | } | 20260505 16:12:41.657 | INFO | Perf report for the last 5.14196 seconds: | 20260505 16:12:41.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:12:41.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:41.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:41.657 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:41.657 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 16:12:41.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:41.657 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:41.657 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 16:12:41.657 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:41.657 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:41.657 | INFO | Uploaded 36GiB to GPU in 2.14638s (16.7725GiB/s) | 20260505 16:12:41.657 | INFO | Downloaded 34GiB to GPU in 2.13688s (15.911GiB/s) | 20260505 16:12:46.790 | INFO | Analyzing chunk frames = [4550144,4550656) | 20260505 16:12:46.790 | INFO | MemInfo: { | 20260505 16:12:46.790 | INFO | "gpu" : " 17 GB", | 20260505 16:12:46.790 | INFO | "huge" : "", | 20260505 16:12:46.790 | INFO | "huge_shared" : " 27 GB", | 20260505 16:12:46.790 | INFO | "malloc" : " 14 GB", | 20260505 16:12:46.790 | INFO | "pinned" : " 29 GB" | 20260505 16:12:46.790 | INFO | } | 20260505 16:12:46.790 | INFO | Perf report for the last 5.13317 seconds: | 20260505 16:12:46.790 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:12:46.790 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:46.790 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:46.790 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:46.790 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 16:12:46.790 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:46.790 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:46.790 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 16:12:46.790 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:46.790 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:46.790 | INFO | Uploaded 36GiB to GPU in 2.19848s (16.375GiB/s) | 20260505 16:12:46.790 | INFO | Downloaded 34GiB to GPU in 2.03047s (16.7449GiB/s) | 20260505 16:12:46.923 | INFO | Starting a round of Baz IO | 20260505 16:12:50.121 | INFO | Populating RTMetrics report | 20260505 16:12:50.121 | INFO | RTMetrics report populated | 20260505 16:12:50.121 | INFO | Opening tmp RTMetrics json file | 20260505 16:12:50.140 | INFO | Serializing RTMetrics to json | 20260505 16:12:50.162 | INFO | Serialization done, swapping tmp file now... | 20260505 16:12:50.163 | INFO | RTMetrics done | 20260505 16:12:50.228 | INFO | Finished a round of Baz IO: Wrote 2.179 GiB in 3.304 seconds | 20260505 16:12:50.228 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.29% / 0.00% / 77.70% | 20260505 16:12:50.228 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.380937% / 0% / 9.93411e-05% | 20260505 16:12:50.228 | INFO | Baz IO overall: | 20260505 16:12:50.228 | INFO | BytesWritten : 2702065182824 | 20260505 16:12:50.228 | INFO | NumEvents : 1282806870794 | 20260505 16:12:51.919 | INFO | Analyzing chunk frames = [4550656,4551168) | 20260505 16:12:51.919 | INFO | MemInfo: { | 20260505 16:12:51.919 | INFO | "gpu" : " 17 GB", | 20260505 16:12:51.919 | INFO | "huge" : "", | 20260505 16:12:51.919 | INFO | "huge_shared" : " 27 GB", | 20260505 16:12:51.919 | INFO | "malloc" : " 14 GB", | 20260505 16:12:51.919 | INFO | "pinned" : " 29 GB" | 20260505 16:12:51.919 | INFO | } | 20260505 16:12:51.919 | INFO | Perf report for the last 5.12946 seconds: | 20260505 16:12:51.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:12:51.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:51.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:51.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:51.919 | INFO | ANALYSIS : 63.9%, 3.0, 99.1ms, 0 | 20260505 16:12:51.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:51.919 | INFO | RT_METRICS : 45.5%, 1.0, 23.8ms, 0 | 20260505 16:12:51.919 | INFO | PRE_HQ : 60.9%, 1.8, 56.5ms, 0 | 20260505 16:12:51.919 | INFO | BAZWRITER : 63.7%, 4.0, 129.7ms, 0 | 20260505 16:12:51.919 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 16:12:51.920 | INFO | Uploaded 36GiB to GPU in 2.15761s (16.6851GiB/s) | 20260505 16:12:51.920 | INFO | Downloaded 38GiB to GPU in 2.25024s (16.8871GiB/s) | 20260505 16:12:57.065 | INFO | Analyzing chunk frames = [4551168,4551680) | 20260505 16:12:57.065 | INFO | MemInfo: { | 20260505 16:12:57.065 | INFO | "gpu" : " 17 GB", | 20260505 16:12:57.065 | INFO | "huge" : "", | 20260505 16:12:57.065 | INFO | "huge_shared" : " 27 GB", | 20260505 16:12:57.065 | INFO | "malloc" : " 14 GB", | 20260505 16:12:57.065 | INFO | "pinned" : " 29 GB" | 20260505 16:12:57.065 | INFO | } | 20260505 16:12:57.065 | INFO | Perf report for the last 5.14581 seconds: | 20260505 16:12:57.065 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:12:57.065 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:57.065 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:57.065 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:57.065 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 16:12:57.065 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:57.065 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:57.065 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 16:12:57.065 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:57.065 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:12:57.065 | INFO | Uploaded 36GiB to GPU in 2.20071s (16.3584GiB/s) | 20260505 16:12:57.066 | INFO | Downloaded 34GiB to GPU in 2.034s (16.7158GiB/s) | 20260505 16:13:02.211 | INFO | Analyzing chunk frames = [4551680,4552192) | 20260505 16:13:02.211 | INFO | MemInfo: { | 20260505 16:13:02.211 | INFO | "gpu" : " 17 GB", | 20260505 16:13:02.211 | INFO | "huge" : "", | 20260505 16:13:02.211 | INFO | "huge_shared" : " 27 GB", | 20260505 16:13:02.211 | INFO | "malloc" : " 14 GB", | 20260505 16:13:02.211 | INFO | "pinned" : " 29 GB" | 20260505 16:13:02.211 | INFO | } | 20260505 16:13:02.211 | INFO | Perf report for the last 5.14615 seconds: | 20260505 16:13:02.211 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:13:02.211 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:02.211 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:02.212 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:02.212 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260505 16:13:02.212 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:02.212 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:02.212 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 16:13:02.212 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:02.212 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:02.212 | INFO | Uploaded 36GiB to GPU in 2.18815s (16.4522GiB/s) | 20260505 16:13:02.212 | INFO | Downloaded 34GiB to GPU in 2.02168s (16.8177GiB/s) | 20260505 16:13:07.353 | INFO | Analyzing chunk frames = [4552192,4552704) | 20260505 16:13:07.353 | INFO | MemInfo: { | 20260505 16:13:07.353 | INFO | "gpu" : " 17 GB", | 20260505 16:13:07.353 | INFO | "huge" : "", | 20260505 16:13:07.353 | INFO | "huge_shared" : " 27 GB", | 20260505 16:13:07.353 | INFO | "malloc" : " 14 GB", | 20260505 16:13:07.353 | INFO | "pinned" : " 29 GB" | 20260505 16:13:07.353 | INFO | } | 20260505 16:13:07.353 | INFO | Perf report for the last 5.14214 seconds: | 20260505 16:13:07.353 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:13:07.353 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:07.353 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:07.353 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:07.354 | INFO | ANALYSIS : 62.6%, 3.0, 97.3ms, 0 | 20260505 16:13:07.354 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:07.354 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:07.354 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 16:13:07.354 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:07.354 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:07.354 | INFO | Uploaded 36GiB to GPU in 2.18728s (16.4588GiB/s) | 20260505 16:13:07.354 | INFO | Downloaded 34GiB to GPU in 2.14221s (15.8715GiB/s) | 20260505 16:13:12.486 | INFO | Analyzing chunk frames = [4552704,4553216) | 20260505 16:13:12.486 | INFO | MemInfo: { | 20260505 16:13:12.486 | INFO | "gpu" : " 17 GB", | 20260505 16:13:12.486 | INFO | "huge" : "", | 20260505 16:13:12.486 | INFO | "huge_shared" : " 27 GB", | 20260505 16:13:12.486 | INFO | "malloc" : " 14 GB", | 20260505 16:13:12.486 | INFO | "pinned" : " 29 GB" | 20260505 16:13:12.486 | INFO | } | 20260505 16:13:12.486 | INFO | Perf report for the last 5.13289 seconds: | 20260505 16:13:12.486 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:13:12.486 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:12.486 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:12.486 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:12.486 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 16:13:12.486 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:12.486 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:12.486 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 16:13:12.487 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:12.487 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:12.487 | INFO | Uploaded 36GiB to GPU in 2.15772s (16.6843GiB/s) | 20260505 16:13:12.487 | INFO | Downloaded 34GiB to GPU in 2.05825s (16.5189GiB/s) | 20260505 16:13:17.619 | INFO | Analyzing chunk frames = [4553216,4553728) | 20260505 16:13:17.619 | INFO | MemInfo: { | 20260505 16:13:17.619 | INFO | "gpu" : " 17 GB", | 20260505 16:13:17.619 | INFO | "huge" : "", | 20260505 16:13:17.619 | INFO | "huge_shared" : " 27 GB", | 20260505 16:13:17.619 | INFO | "malloc" : " 14 GB", | 20260505 16:13:17.619 | INFO | "pinned" : " 29 GB" | 20260505 16:13:17.619 | INFO | } | 20260505 16:13:17.619 | INFO | Perf report for the last 5.13259 seconds: | 20260505 16:13:17.619 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:13:17.619 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:17.619 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:17.619 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:17.619 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 16:13:17.619 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:17.619 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:17.619 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 16:13:17.619 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:17.619 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:17.619 | INFO | Uploaded 36GiB to GPU in 2.14246s (16.8031GiB/s) | 20260505 16:13:17.619 | INFO | Downloaded 34GiB to GPU in 2.03759s (16.6864GiB/s) | 20260505 16:13:22.769 | INFO | Analyzing chunk frames = [4553728,4554240) | 20260505 16:13:22.769 | INFO | MemInfo: { | 20260505 16:13:22.769 | INFO | "gpu" : " 17 GB", | 20260505 16:13:22.769 | INFO | "huge" : "", | 20260505 16:13:22.769 | INFO | "huge_shared" : " 27 GB", | 20260505 16:13:22.769 | INFO | "malloc" : " 14 GB", | 20260505 16:13:22.769 | INFO | "pinned" : " 29 GB" | 20260505 16:13:22.769 | INFO | } | 20260505 16:13:22.769 | INFO | Perf report for the last 5.15033 seconds: | 20260505 16:13:22.769 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:13:22.769 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:22.769 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:22.769 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:22.769 | INFO | ANALYSIS : 63.2%, 3.0, 98.9ms, 0 | 20260505 16:13:22.769 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:22.769 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:22.769 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 16:13:22.769 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:22.769 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:22.770 | INFO | Uploaded 36GiB to GPU in 2.11373s (17.0315GiB/s) | 20260505 16:13:22.770 | INFO | Downloaded 34GiB to GPU in 2.16557s (15.7003GiB/s) | 20260505 16:13:27.912 | INFO | Analyzing chunk frames = [4554240,4554752) | 20260505 16:13:27.912 | INFO | MemInfo: { | 20260505 16:13:27.912 | INFO | "gpu" : " 17 GB", | 20260505 16:13:27.912 | INFO | "huge" : "", | 20260505 16:13:27.912 | INFO | "huge_shared" : " 27 GB", | 20260505 16:13:27.912 | INFO | "malloc" : " 14 GB", | 20260505 16:13:27.912 | INFO | "pinned" : " 29 GB" | 20260505 16:13:27.912 | INFO | } | 20260505 16:13:27.912 | INFO | Perf report for the last 5.14298 seconds: | 20260505 16:13:27.912 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:13:27.912 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:27.912 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:27.912 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:27.912 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 16:13:27.912 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:27.912 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:27.912 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 16:13:27.912 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:27.912 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:27.913 | INFO | Uploaded 36GiB to GPU in 2.18627s (16.4664GiB/s) | 20260505 16:13:27.913 | INFO | Downloaded 34GiB to GPU in 2.02667s (16.7763GiB/s) | 20260505 16:13:27.913 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 16:13:27.913 | INFO | Total mem usage: 15.3 GiB | 20260505 16:13:27.913 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 16:13:27.913 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.88%) | 20260505 16:13:27.913 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 16:13:27.913 | INFO | High Water Breakdown: | 20260505 16:13:27.913 | INFO | 61% : 8.67 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 16:13:27.913 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 16:13:27.913 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 16:13:27.913 | INFO | ------------------------------------------ | 20260505 16:13:27.913 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:13:27.913 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 16:13:27.913 | INFO | Total mem usage: 26.5 GiB | 20260505 16:13:27.913 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 16:13:27.913 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 16:13:27.913 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 16:13:27.913 | INFO | High Water Breakdown: | 20260505 16:13:27.913 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:13:27.913 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:13:27.913 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:13:27.913 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:13:27.913 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:13:27.913 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:13:27.913 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 16:13:27.913 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 16:13:27.913 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:13:27.913 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 16:13:27.913 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 16:13:27.913 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 16:13:27.913 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 16:13:27.913 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 16:13:27.913 | INFO | ------------------------------------------ | 20260505 16:13:27.913 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:13:27.913 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 16:13:27.913 | INFO | Total mem usage: 25.6 GiB | 20260505 16:13:27.913 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 16:13:27.913 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 16:13:27.913 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 16:13:27.913 | INFO | High Water Breakdown: | 20260505 16:13:27.913 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 16:13:27.913 | INFO | ------------------------------------------ | 20260505 16:13:27.913 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:13:27.913 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 16:13:27.913 | INFO | Total mem usage: 15.7 GiB | 20260505 16:13:27.913 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 16:13:27.913 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 16:13:27.913 | INFO | Unused Cache: 871 MiB (Waste: 5.43%) | 20260505 16:13:27.913 | INFO | High Water Breakdown: | 20260505 16:13:27.913 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 16:13:27.913 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 16:13:27.913 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 16:13:27.913 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 16:13:27.913 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 16:13:27.913 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 16:13:27.913 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 16:13:27.913 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 16:13:27.913 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 16:13:27.913 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:13:27.913 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 16:13:27.913 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:13:27.913 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 16:13:27.913 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 16:13:27.913 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 16:13:27.913 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 16:13:27.913 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:13:27.913 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 16:13:27.913 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:13:27.913 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:13:27.913 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:13:27.913 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 16:13:27.913 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 16:13:27.913 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:13:27.913 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 16:13:27.913 | INFO | ------------------------------------------ | 20260505 16:13:27.913 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:13:27.913 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 16:13:27.913 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:13:27.913 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:13:27.913 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.53e+01, 1.34e+01 / 2.25e+00 | 20260505 16:13:27.913 | INFO | PerfReport: 0.29%, AggregateStats -- 9.30e-02 / 1.01e-01, 1.12e-02 / 2.90e-03 | 20260505 16:13:27.913 | INFO | PerfReport: 11.79%, Baseline -- 3.74e+00 / 4.52e+00, 5.76e-01 / 1.43e-01 | 20260505 16:13:27.913 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.52e+00, 1.94e-01 / 9.37e-02 | 20260505 16:13:27.914 | INFO | PerfReport: 15.99%, DME -- 4.05e+01 / 4.53e+01, 4.40e+00 / 2.23e+00 | 20260505 16:13:27.914 | INFO | PerfReport: 56.63%, FrameLabeling -- 1.79e+01 / 1.90e+01, 1.45e+00 / 1.03e+00 | 20260505 16:13:27.914 | INFO | PerfReport: 7.40%, PulseAccumulating -- 2.34e+00 / 2.96e+00, 4.52e-01 / 1.26e-01 | 20260505 16:13:27.914 | INFO | PerfReport: 3.42%, Metrics -- 1.08e+00 / 1.43e+00, 2.98e-01 / 1.55e-01 | 20260505 16:13:27.914 | INFO | PerfReport: 1.20%, PulseSerializing -- 3.79e-01 / 4.37e-01, 6.71e-02 / 3.58e-02 | 20260505 16:13:27.914 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 16:13:27.914 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:13:27.914 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:13:27.914 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.56e+01 / 0.00e+00 | 20260505 16:13:27.914 | INFO | PerfReport: 22.56%, Upload -- 2.26e+01 / 0.00e+00, 7.92e+00 / 0.00e+00 | 20260505 16:13:27.914 | INFO | PerfReport: 21.79%, Download -- 2.18e+01 / 0.00e+00, 7.81e+00 / 0.00e+00 | 20260505 16:13:27.914 | INFO | PerfReport: 31.61%, Compute -- 3.17e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 16:13:27.914 | INFO | PerfReport: 2.30%, LockUpload -- 2.30e+00 / 0.00e+00, 8.04e+00 / 0.00e+00 | 20260505 16:13:27.914 | INFO | PerfReport: 1.42%, LockDownload -- 1.42e+00 / 0.00e+00, 2.89e+00 / 0.00e+00 | 20260505 16:13:27.914 | INFO | PerfReport: 20.31%, LockCompute -- 2.04e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260505 16:13:27.914 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.18e-02 / 0.00e+00 | 20260505 16:13:28.051 | INFO | Starting a round of Baz IO | 20260505 16:13:31.231 | INFO | Populating RTMetrics report | 20260505 16:13:31.231 | INFO | RTMetrics report populated | 20260505 16:13:31.231 | INFO | Opening tmp RTMetrics json file | 20260505 16:13:31.245 | INFO | Serializing RTMetrics to json | 20260505 16:13:31.248 | INFO | Serialization done, swapping tmp file now... | 20260505 16:13:31.249 | INFO | RTMetrics done | 20260505 16:13:31.336 | INFO | Finished a round of Baz IO: Wrote 2.179 GiB in 3.284 seconds | 20260505 16:13:31.336 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.29% / 0.00% / 77.71% | 20260505 16:13:31.336 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.381045% / 0% / 8.74201e-05% | 20260505 16:13:31.336 | INFO | Baz IO overall: | 20260505 16:13:31.336 | INFO | BytesWritten : 2704405304644 | 20260505 16:13:31.336 | INFO | NumEvents : 1283904411199 | 20260505 16:13:33.045 | INFO | Analyzing chunk frames = [4554752,4555264) | 20260505 16:13:33.045 | INFO | MemInfo: { | 20260505 16:13:33.045 | INFO | "gpu" : " 17 GB", | 20260505 16:13:33.045 | INFO | "huge" : "", | 20260505 16:13:33.045 | INFO | "huge_shared" : " 27 GB", | 20260505 16:13:33.045 | INFO | "malloc" : " 14 GB", | 20260505 16:13:33.045 | INFO | "pinned" : " 29 GB" | 20260505 16:13:33.045 | INFO | } | 20260505 16:13:33.045 | INFO | Perf report for the last 5.13252 seconds: | 20260505 16:13:33.045 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:13:33.045 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:33.045 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:33.045 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:33.045 | INFO | ANALYSIS : 63.6%, 3.0, 98.7ms, 0 | 20260505 16:13:33.045 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:33.045 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260505 16:13:33.045 | INFO | PRE_HQ : 61.0%, 1.8, 57.3ms, 0 | 20260505 16:13:33.045 | INFO | BAZWRITER : 63.5%, 3.9, 128.4ms, 0 | 20260505 16:13:33.045 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 16:13:33.045 | INFO | Uploaded 36GiB to GPU in 2.15276s (16.7227GiB/s) | 20260505 16:13:33.045 | INFO | Downloaded 38GiB to GPU in 2.24429s (16.9318GiB/s) | 20260505 16:13:38.182 | INFO | Analyzing chunk frames = [4555264,4555776) | 20260505 16:13:38.182 | INFO | MemInfo: { | 20260505 16:13:38.182 | INFO | "gpu" : " 17 GB", | 20260505 16:13:38.182 | INFO | "huge" : "", | 20260505 16:13:38.182 | INFO | "huge_shared" : " 27 GB", | 20260505 16:13:38.182 | INFO | "malloc" : " 14 GB", | 20260505 16:13:38.182 | INFO | "pinned" : " 29 GB" | 20260505 16:13:38.182 | INFO | } | 20260505 16:13:38.182 | INFO | Perf report for the last 5.137 seconds: | 20260505 16:13:38.182 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:13:38.182 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:38.182 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:38.182 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:38.182 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 16:13:38.182 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:38.182 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:38.182 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 16:13:38.182 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:38.182 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:38.182 | INFO | Uploaded 36GiB to GPU in 2.13644s (16.8505GiB/s) | 20260505 16:13:38.182 | INFO | Downloaded 34GiB to GPU in 2.03955s (16.6704GiB/s) | 20260505 16:13:43.318 | INFO | Analyzing chunk frames = [4555776,4556288) | 20260505 16:13:43.318 | INFO | MemInfo: { | 20260505 16:13:43.318 | INFO | "gpu" : " 17 GB", | 20260505 16:13:43.318 | INFO | "huge" : "", | 20260505 16:13:43.318 | INFO | "huge_shared" : " 27 GB", | 20260505 16:13:43.318 | INFO | "malloc" : " 14 GB", | 20260505 16:13:43.318 | INFO | "pinned" : " 29 GB" | 20260505 16:13:43.318 | INFO | } | 20260505 16:13:43.318 | INFO | Perf report for the last 5.13652 seconds: | 20260505 16:13:43.318 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:13:43.318 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:43.318 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:43.318 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:43.318 | INFO | ANALYSIS : 63.0%, 3.0, 97.9ms, 0 | 20260505 16:13:43.318 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:43.318 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:43.319 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 16:13:43.319 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:43.319 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:43.319 | INFO | Uploaded 36GiB to GPU in 2.19485s (16.402GiB/s) | 20260505 16:13:43.319 | INFO | Downloaded 34GiB to GPU in 2.05127s (16.5751GiB/s) | 20260505 16:13:48.464 | INFO | Analyzing chunk frames = [4556288,4556800) | 20260505 16:13:48.464 | INFO | MemInfo: { | 20260505 16:13:48.464 | INFO | "gpu" : " 17 GB", | 20260505 16:13:48.464 | INFO | "huge" : "", | 20260505 16:13:48.464 | INFO | "huge_shared" : " 27 GB", | 20260505 16:13:48.464 | INFO | "malloc" : " 14 GB", | 20260505 16:13:48.464 | INFO | "pinned" : " 29 GB" | 20260505 16:13:48.464 | INFO | } | 20260505 16:13:48.465 | INFO | Perf report for the last 5.1461 seconds: | 20260505 16:13:48.465 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:13:48.465 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:48.465 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:48.465 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:48.465 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260505 16:13:48.465 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:48.465 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:48.465 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 16:13:48.465 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:48.465 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:48.465 | INFO | Uploaded 36GiB to GPU in 2.17656s (16.5399GiB/s) | 20260505 16:13:48.465 | INFO | Downloaded 34GiB to GPU in 2.13549s (15.9214GiB/s) | 20260505 16:13:53.607 | INFO | Analyzing chunk frames = [4556800,4557312) | 20260505 16:13:53.607 | INFO | MemInfo: { | 20260505 16:13:53.607 | INFO | "gpu" : " 17 GB", | 20260505 16:13:53.607 | INFO | "huge" : "", | 20260505 16:13:53.607 | INFO | "huge_shared" : " 27 GB", | 20260505 16:13:53.607 | INFO | "malloc" : " 14 GB", | 20260505 16:13:53.607 | INFO | "pinned" : " 29 GB" | 20260505 16:13:53.607 | INFO | } | 20260505 16:13:53.607 | INFO | Perf report for the last 5.14296 seconds: | 20260505 16:13:53.607 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:13:53.607 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:53.608 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:53.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:53.608 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260505 16:13:53.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:53.608 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:53.608 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 16:13:53.608 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:53.608 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:53.608 | INFO | Uploaded 36GiB to GPU in 2.1485s (16.7559GiB/s) | 20260505 16:13:53.608 | INFO | Downloaded 34GiB to GPU in 2.05751s (16.5248GiB/s) | 20260505 16:13:58.731 | INFO | Analyzing chunk frames = [4557312,4557824) | 20260505 16:13:58.731 | INFO | MemInfo: { | 20260505 16:13:58.731 | INFO | "gpu" : " 17 GB", | 20260505 16:13:58.731 | INFO | "huge" : "", | 20260505 16:13:58.731 | INFO | "huge_shared" : " 27 GB", | 20260505 16:13:58.731 | INFO | "malloc" : " 14 GB", | 20260505 16:13:58.731 | INFO | "pinned" : " 29 GB" | 20260505 16:13:58.731 | INFO | } | 20260505 16:13:58.731 | INFO | Perf report for the last 5.12389 seconds: | 20260505 16:13:58.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:13:58.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:58.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:58.731 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:58.731 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260505 16:13:58.731 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:58.731 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:58.732 | INFO | PRE_HQ : 28.1%, 1.0, 14.5ms, 0 | 20260505 16:13:58.732 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:58.732 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:13:58.732 | INFO | Uploaded 36GiB to GPU in 2.13793s (16.8387GiB/s) | 20260505 16:13:58.732 | INFO | Downloaded 34GiB to GPU in 2.0369s (16.692GiB/s) | 20260505 16:14:03.872 | INFO | Analyzing chunk frames = [4557824,4558336) | 20260505 16:14:03.872 | INFO | MemInfo: { | 20260505 16:14:03.872 | INFO | "gpu" : " 17 GB", | 20260505 16:14:03.872 | INFO | "huge" : "", | 20260505 16:14:03.872 | INFO | "huge_shared" : " 27 GB", | 20260505 16:14:03.872 | INFO | "malloc" : " 14 GB", | 20260505 16:14:03.872 | INFO | "pinned" : " 29 GB" | 20260505 16:14:03.872 | INFO | } | 20260505 16:14:03.872 | INFO | Perf report for the last 5.14106 seconds: | 20260505 16:14:03.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:14:03.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:03.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:03.873 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:03.873 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 16:14:03.873 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:03.873 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:03.873 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 16:14:03.873 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:03.873 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:03.873 | INFO | Uploaded 36GiB to GPU in 2.11423s (17.0275GiB/s) | 20260505 16:14:03.873 | INFO | Downloaded 34GiB to GPU in 2.17013s (15.6672GiB/s) | 20260505 16:14:09.020 | INFO | Analyzing chunk frames = [4558336,4558848) | 20260505 16:14:09.020 | INFO | MemInfo: { | 20260505 16:14:09.020 | INFO | "gpu" : " 17 GB", | 20260505 16:14:09.020 | INFO | "huge" : "", | 20260505 16:14:09.020 | INFO | "huge_shared" : " 27 GB", | 20260505 16:14:09.020 | INFO | "malloc" : " 14 GB", | 20260505 16:14:09.020 | INFO | "pinned" : " 29 GB" | 20260505 16:14:09.020 | INFO | } | 20260505 16:14:09.020 | INFO | Perf report for the last 5.14784 seconds: | 20260505 16:14:09.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:14:09.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:09.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:09.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:09.020 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 16:14:09.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:09.020 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:09.020 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 16:14:09.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:09.021 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:09.021 | INFO | Uploaded 36GiB to GPU in 2.23177s (16.1307GiB/s) | 20260505 16:14:09.021 | INFO | Downloaded 34GiB to GPU in 2.10472s (16.1541GiB/s) | 20260505 16:14:09.156 | INFO | Starting a round of Baz IO | 20260505 16:14:12.363 | INFO | Populating RTMetrics report | 20260505 16:14:12.364 | INFO | RTMetrics report populated | 20260505 16:14:12.364 | INFO | Opening tmp RTMetrics json file | 20260505 16:14:12.375 | INFO | Serializing RTMetrics to json | 20260505 16:14:12.376 | INFO | Serialization done, swapping tmp file now... | 20260505 16:14:12.377 | INFO | RTMetrics done | 20260505 16:14:12.466 | INFO | Finished a round of Baz IO: Wrote 2.18 GiB in 3.31 seconds | 20260505 16:14:12.466 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.29% / 0.00% / 77.71% | 20260505 16:14:12.466 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.380115% / 0% / 5.5631e-05% | 20260505 16:14:12.466 | INFO | Baz IO overall: | 20260505 16:14:12.466 | INFO | BytesWritten : 2706745595766 | 20260505 16:14:12.466 | INFO | NumEvents : 1285002080120 | 20260505 16:14:14.166 | INFO | Analyzing chunk frames = [4558848,4559360) | 20260505 16:14:14.166 | INFO | MemInfo: { | 20260505 16:14:14.166 | INFO | "gpu" : " 17 GB", | 20260505 16:14:14.166 | INFO | "huge" : "", | 20260505 16:14:14.166 | INFO | "huge_shared" : " 27 GB", | 20260505 16:14:14.166 | INFO | "malloc" : " 14 GB", | 20260505 16:14:14.166 | INFO | "pinned" : " 29 GB" | 20260505 16:14:14.166 | INFO | } | 20260505 16:14:14.166 | INFO | Perf report for the last 5.14598 seconds: | 20260505 16:14:14.166 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:14:14.166 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:14.166 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:14.166 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:14.166 | INFO | ANALYSIS : 63.9%, 3.0, 99.5ms, 0 | 20260505 16:14:14.166 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:14.166 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260505 16:14:14.166 | INFO | PRE_HQ : 60.7%, 1.8, 56.9ms, 0 | 20260505 16:14:14.166 | INFO | BAZWRITER : 63.7%, 3.9, 127.9ms, 0 | 20260505 16:14:14.166 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 16:14:14.167 | INFO | Uploaded 36GiB to GPU in 2.20105s (16.3558GiB/s) | 20260505 16:14:14.167 | INFO | Downloaded 38GiB to GPU in 2.26805s (16.7545GiB/s) | 20260505 16:14:19.300 | INFO | Analyzing chunk frames = [4559360,4559872) | 20260505 16:14:19.300 | INFO | MemInfo: { | 20260505 16:14:19.300 | INFO | "gpu" : " 17 GB", | 20260505 16:14:19.300 | INFO | "huge" : "", | 20260505 16:14:19.300 | INFO | "huge_shared" : " 27 GB", | 20260505 16:14:19.300 | INFO | "malloc" : " 14 GB", | 20260505 16:14:19.300 | INFO | "pinned" : " 29 GB" | 20260505 16:14:19.300 | INFO | } | 20260505 16:14:19.300 | INFO | Perf report for the last 5.1338 seconds: | 20260505 16:14:19.300 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:14:19.300 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:19.300 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:19.300 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:19.300 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 16:14:19.300 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:19.300 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:19.300 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:14:19.300 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:19.300 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:19.300 | INFO | Uploaded 36GiB to GPU in 2.13117s (16.8922GiB/s) | 20260505 16:14:19.300 | INFO | Downloaded 34GiB to GPU in 2.02288s (16.8077GiB/s) | 20260505 16:14:24.434 | INFO | Analyzing chunk frames = [4559872,4560384) | 20260505 16:14:24.434 | INFO | MemInfo: { | 20260505 16:14:24.434 | INFO | "gpu" : " 17 GB", | 20260505 16:14:24.434 | INFO | "huge" : "", | 20260505 16:14:24.434 | INFO | "huge_shared" : " 27 GB", | 20260505 16:14:24.434 | INFO | "malloc" : " 14 GB", | 20260505 16:14:24.434 | INFO | "pinned" : " 29 GB" | 20260505 16:14:24.434 | INFO | } | 20260505 16:14:24.434 | INFO | Perf report for the last 5.13424 seconds: | 20260505 16:14:24.434 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:14:24.434 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:24.434 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:24.434 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:24.434 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260505 16:14:24.434 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:24.434 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:24.434 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:14:24.435 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:24.435 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:24.435 | INFO | Uploaded 36GiB to GPU in 2.17542s (16.5485GiB/s) | 20260505 16:14:24.435 | INFO | Downloaded 34GiB to GPU in 2.03476s (16.7096GiB/s) | 20260505 16:14:29.574 | INFO | Analyzing chunk frames = [4560384,4560896) | 20260505 16:14:29.574 | INFO | MemInfo: { | 20260505 16:14:29.574 | INFO | "gpu" : " 17 GB", | 20260505 16:14:29.574 | INFO | "huge" : "", | 20260505 16:14:29.574 | INFO | "huge_shared" : " 27 GB", | 20260505 16:14:29.574 | INFO | "malloc" : " 14 GB", | 20260505 16:14:29.574 | INFO | "pinned" : " 29 GB" | 20260505 16:14:29.574 | INFO | } | 20260505 16:14:29.575 | INFO | Perf report for the last 5.14015 seconds: | 20260505 16:14:29.575 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:14:29.575 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:29.575 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:29.575 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:29.575 | INFO | ANALYSIS : 62.5%, 3.0, 97.1ms, 0 | 20260505 16:14:29.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:29.575 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:29.575 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 16:14:29.575 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:29.575 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:29.575 | INFO | Uploaded 36GiB to GPU in 2.18825s (16.4515GiB/s) | 20260505 16:14:29.575 | INFO | Downloaded 34GiB to GPU in 2.13022s (15.9608GiB/s) | 20260505 16:14:34.717 | INFO | Analyzing chunk frames = [4560896,4561408) | 20260505 16:14:34.717 | INFO | MemInfo: { | 20260505 16:14:34.717 | INFO | "gpu" : " 17 GB", | 20260505 16:14:34.717 | INFO | "huge" : "", | 20260505 16:14:34.717 | INFO | "huge_shared" : " 27 GB", | 20260505 16:14:34.717 | INFO | "malloc" : " 14 GB", | 20260505 16:14:34.717 | INFO | "pinned" : " 29 GB" | 20260505 16:14:34.718 | INFO | } | 20260505 16:14:34.718 | INFO | Perf report for the last 5.14299 seconds: | 20260505 16:14:34.718 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:14:34.718 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:34.718 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:34.718 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:34.718 | INFO | ANALYSIS : 62.3%, 3.0, 97.4ms, 0 | 20260505 16:14:34.718 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:34.718 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:34.718 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:14:34.718 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:34.718 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:34.718 | INFO | Uploaded 36GiB to GPU in 2.15947s (16.6708GiB/s) | 20260505 16:14:34.718 | INFO | Downloaded 34GiB to GPU in 2.05903s (16.5126GiB/s) | 20260505 16:14:39.859 | INFO | Analyzing chunk frames = [4561408,4561920) | 20260505 16:14:39.859 | INFO | MemInfo: { | 20260505 16:14:39.859 | INFO | "gpu" : " 17 GB", | 20260505 16:14:39.859 | INFO | "huge" : "", | 20260505 16:14:39.859 | INFO | "huge_shared" : " 27 GB", | 20260505 16:14:39.859 | INFO | "malloc" : " 14 GB", | 20260505 16:14:39.859 | INFO | "pinned" : " 29 GB" | 20260505 16:14:39.859 | INFO | } | 20260505 16:14:39.859 | INFO | Perf report for the last 5.14178 seconds: | 20260505 16:14:39.859 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:14:39.859 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:39.859 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:39.859 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:39.859 | INFO | ANALYSIS : 62.3%, 3.0, 97.4ms, 0 | 20260505 16:14:39.859 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:39.859 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:39.859 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:14:39.859 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:39.860 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:39.860 | INFO | Uploaded 36GiB to GPU in 2.13368s (16.8723GiB/s) | 20260505 16:14:39.860 | INFO | Downloaded 34GiB to GPU in 2.04507s (16.6254GiB/s) | 20260505 16:14:44.995 | INFO | Analyzing chunk frames = [4561920,4562432) | 20260505 16:14:44.995 | INFO | MemInfo: { | 20260505 16:14:44.995 | INFO | "gpu" : " 17 GB", | 20260505 16:14:44.995 | INFO | "huge" : "", | 20260505 16:14:44.995 | INFO | "huge_shared" : " 27 GB", | 20260505 16:14:44.995 | INFO | "malloc" : " 14 GB", | 20260505 16:14:44.995 | INFO | "pinned" : " 29 GB" | 20260505 16:14:44.995 | INFO | } | 20260505 16:14:44.995 | INFO | Perf report for the last 5.13607 seconds: | 20260505 16:14:44.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:14:44.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:44.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:44.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:44.995 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 16:14:44.996 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:44.996 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:44.996 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 16:14:44.996 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:44.996 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:44.996 | INFO | Uploaded 36GiB to GPU in 2.13542s (16.8585GiB/s) | 20260505 16:14:44.996 | INFO | Downloaded 34GiB to GPU in 2.1738s (15.6408GiB/s) | 20260505 16:14:50.131 | INFO | Analyzing chunk frames = [4562432,4562944) | 20260505 16:14:50.131 | INFO | MemInfo: { | 20260505 16:14:50.131 | INFO | "gpu" : " 17 GB", | 20260505 16:14:50.131 | INFO | "huge" : "", | 20260505 16:14:50.131 | INFO | "huge_shared" : " 27 GB", | 20260505 16:14:50.131 | INFO | "malloc" : " 14 GB", | 20260505 16:14:50.131 | INFO | "pinned" : " 29 GB" | 20260505 16:14:50.131 | INFO | } | 20260505 16:14:50.131 | INFO | Perf report for the last 5.136 seconds: | 20260505 16:14:50.131 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:14:50.131 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:50.131 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:50.131 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:50.132 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 16:14:50.132 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:50.132 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:50.132 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 16:14:50.132 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:50.132 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:50.132 | INFO | Uploaded 36GiB to GPU in 2.22931s (16.1485GiB/s) | 20260505 16:14:50.132 | INFO | Downloaded 34GiB to GPU in 2.10175s (16.177GiB/s) | 20260505 16:14:50.132 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 16:14:50.132 | INFO | Total mem usage: 15.3 GiB | 20260505 16:14:50.132 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 16:14:50.132 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.89%) | 20260505 16:14:50.132 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 16:14:50.132 | INFO | High Water Breakdown: | 20260505 16:14:50.132 | INFO | 61% : 8.67 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 16:14:50.132 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 16:14:50.132 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 16:14:50.132 | INFO | ------------------------------------------ | 20260505 16:14:50.132 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:14:50.132 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 16:14:50.132 | INFO | Total mem usage: 26.5 GiB | 20260505 16:14:50.132 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 16:14:50.132 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 16:14:50.132 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 16:14:50.132 | INFO | High Water Breakdown: | 20260505 16:14:50.132 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:14:50.132 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:14:50.132 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:14:50.132 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:14:50.132 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:14:50.132 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:14:50.132 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 16:14:50.132 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 16:14:50.132 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:14:50.132 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 16:14:50.132 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 16:14:50.132 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 16:14:50.132 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 16:14:50.132 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 16:14:50.132 | INFO | ------------------------------------------ | 20260505 16:14:50.132 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:14:50.132 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 16:14:50.132 | INFO | Total mem usage: 25.6 GiB | 20260505 16:14:50.132 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 16:14:50.132 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 16:14:50.132 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 16:14:50.132 | INFO | High Water Breakdown: | 20260505 16:14:50.132 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 16:14:50.132 | INFO | ------------------------------------------ | 20260505 16:14:50.132 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:14:50.132 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 16:14:50.132 | INFO | Total mem usage: 15.7 GiB | 20260505 16:14:50.132 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 16:14:50.132 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 16:14:50.132 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 16:14:50.132 | INFO | High Water Breakdown: | 20260505 16:14:50.132 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 16:14:50.132 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 16:14:50.132 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 16:14:50.132 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 16:14:50.132 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 16:14:50.132 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 16:14:50.132 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 16:14:50.132 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 16:14:50.132 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 16:14:50.132 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:14:50.132 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 16:14:50.132 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:14:50.132 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 16:14:50.132 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 16:14:50.132 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 16:14:50.133 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 16:14:50.133 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:14:50.133 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 16:14:50.133 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:14:50.133 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:14:50.133 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:14:50.133 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 16:14:50.133 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 16:14:50.133 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:14:50.133 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 16:14:50.133 | INFO | ------------------------------------------ | 20260505 16:14:50.133 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:14:50.133 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 16:14:50.133 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:14:50.133 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:14:50.133 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.55e+01, 1.33e+01 / 1.79e+00 | 20260505 16:14:50.133 | INFO | PerfReport: 0.29%, AggregateStats -- 9.32e-02 / 1.03e-01, 1.44e-02 / 6.13e-03 | 20260505 16:14:50.133 | INFO | PerfReport: 11.79%, Baseline -- 3.74e+00 / 4.47e+00, 5.86e-01 / 1.67e-01 | 20260505 16:14:50.133 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.49e+00, 1.95e-01 / 1.20e-01 | 20260505 16:14:50.133 | INFO | PerfReport: 15.90%, DME -- 4.04e+01 / 4.47e+01, 4.21e+00 / 1.92e+00 | 20260505 16:14:50.133 | INFO | PerfReport: 56.79%, FrameLabeling -- 1.80e+01 / 1.99e+01, 1.53e+00 / 1.67e+00 | 20260505 16:14:50.133 | INFO | PerfReport: 7.34%, PulseAccumulating -- 2.33e+00 / 2.91e+00, 4.47e-01 / 1.64e-01 | 20260505 16:14:50.133 | INFO | PerfReport: 3.42%, Metrics -- 1.09e+00 / 1.39e+00, 3.01e-01 / 1.59e-01 | 20260505 16:14:50.133 | INFO | PerfReport: 1.19%, PulseSerializing -- 3.78e-01 / 4.32e-01, 6.72e-02 / 3.17e-02 | 20260505 16:14:50.133 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 16:14:50.133 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:14:50.133 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:14:50.133 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260505 16:14:50.133 | INFO | PerfReport: 22.51%, Upload -- 2.26e+01 / 0.00e+00, 7.97e+00 / 0.00e+00 | 20260505 16:14:50.133 | INFO | PerfReport: 21.88%, Download -- 2.19e+01 / 0.00e+00, 7.96e+00 / 0.00e+00 | 20260505 16:14:50.133 | INFO | PerfReport: 31.67%, Compute -- 3.17e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260505 16:14:50.133 | INFO | PerfReport: 2.19%, LockUpload -- 2.20e+00 / 0.00e+00, 7.83e+00 / 0.00e+00 | 20260505 16:14:50.133 | INFO | PerfReport: 1.40%, LockDownload -- 1.40e+00 / 0.00e+00, 2.97e+00 / 0.00e+00 | 20260505 16:14:50.133 | INFO | PerfReport: 20.33%, LockCompute -- 2.04e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260505 16:14:50.133 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.65e-02 / 0.00e+00 | 20260505 16:14:50.261 | INFO | Starting a round of Baz IO | 20260505 16:14:53.469 | INFO | Populating RTMetrics report | 20260505 16:14:53.469 | INFO | RTMetrics report populated | 20260505 16:14:53.469 | INFO | Opening tmp RTMetrics json file | 20260505 16:14:53.483 | INFO | Serializing RTMetrics to json | 20260505 16:14:53.486 | INFO | Serialization done, swapping tmp file now... | 20260505 16:14:53.491 | INFO | RTMetrics done | 20260505 16:14:53.567 | INFO | Finished a round of Baz IO: Wrote 2.18 GiB in 3.305 seconds | 20260505 16:14:53.567 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.29% / 0.00% / 77.71% | 20260505 16:14:53.567 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.379439% / 7.94729e-06% / 8.74201e-05% | 20260505 16:14:53.567 | INFO | Baz IO overall: | 20260505 16:14:53.567 | INFO | BytesWritten : 2709085976635 | 20260505 16:14:53.567 | INFO | NumEvents : 1286099483539 | 20260505 16:14:55.275 | INFO | Analyzing chunk frames = [4562944,4563456) | 20260505 16:14:55.275 | INFO | MemInfo: { | 20260505 16:14:55.275 | INFO | "gpu" : " 17 GB", | 20260505 16:14:55.275 | INFO | "huge" : "", | 20260505 16:14:55.275 | INFO | "huge_shared" : " 27 GB", | 20260505 16:14:55.275 | INFO | "malloc" : " 14 GB", | 20260505 16:14:55.275 | INFO | "pinned" : " 29 GB" | 20260505 16:14:55.275 | INFO | } | 20260505 16:14:55.276 | INFO | Perf report for the last 5.1441 seconds: | 20260505 16:14:55.276 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:14:55.276 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:55.276 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:55.276 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:55.276 | INFO | ANALYSIS : 63.8%, 3.0, 99.1ms, 0 | 20260505 16:14:55.276 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:14:55.276 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260505 16:14:55.276 | INFO | PRE_HQ : 60.6%, 1.8, 57.0ms, 0 | 20260505 16:14:55.276 | INFO | BAZWRITER : 63.5%, 3.9, 128.9ms, 0 | 20260505 16:14:55.276 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 16:14:55.276 | INFO | Uploaded 36GiB to GPU in 2.17065s (16.5849GiB/s) | 20260505 16:14:55.276 | INFO | Downloaded 38GiB to GPU in 2.25632s (16.8416GiB/s) | 20260505 16:15:00.417 | INFO | Analyzing chunk frames = [4563456,4563968) | 20260505 16:15:00.417 | INFO | MemInfo: { | 20260505 16:15:00.417 | INFO | "gpu" : " 17 GB", | 20260505 16:15:00.417 | INFO | "huge" : "", | 20260505 16:15:00.417 | INFO | "huge_shared" : " 27 GB", | 20260505 16:15:00.417 | INFO | "malloc" : " 14 GB", | 20260505 16:15:00.417 | INFO | "pinned" : " 29 GB" | 20260505 16:15:00.417 | INFO | } | 20260505 16:15:00.417 | INFO | Perf report for the last 5.14186 seconds: | 20260505 16:15:00.417 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:15:00.417 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:00.417 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:00.417 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:00.417 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 16:15:00.417 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:00.418 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:00.418 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:15:00.418 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:00.418 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:00.418 | INFO | Uploaded 36GiB to GPU in 2.13397s (16.87GiB/s) | 20260505 16:15:00.418 | INFO | Downloaded 34GiB to GPU in 2.02545s (16.7864GiB/s) | 20260505 16:15:05.552 | INFO | Analyzing chunk frames = [4563968,4564480) | 20260505 16:15:05.552 | INFO | MemInfo: { | 20260505 16:15:05.552 | INFO | "gpu" : " 17 GB", | 20260505 16:15:05.552 | INFO | "huge" : "", | 20260505 16:15:05.552 | INFO | "huge_shared" : " 27 GB", | 20260505 16:15:05.552 | INFO | "malloc" : " 14 GB", | 20260505 16:15:05.552 | INFO | "pinned" : " 29 GB" | 20260505 16:15:05.552 | INFO | } | 20260505 16:15:05.553 | INFO | Perf report for the last 5.13509 seconds: | 20260505 16:15:05.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:15:05.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:05.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:05.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:05.553 | INFO | ANALYSIS : 62.8%, 3.0, 97.6ms, 0 | 20260505 16:15:05.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:05.553 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:05.553 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 16:15:05.553 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:05.553 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:05.553 | INFO | Uploaded 36GiB to GPU in 2.20252s (16.3449GiB/s) | 20260505 16:15:05.553 | INFO | Downloaded 34GiB to GPU in 2.03729s (16.6888GiB/s) | 20260505 16:15:10.680 | INFO | Analyzing chunk frames = [4564480,4564992) | 20260505 16:15:10.680 | INFO | MemInfo: { | 20260505 16:15:10.680 | INFO | "gpu" : " 17 GB", | 20260505 16:15:10.680 | INFO | "huge" : "", | 20260505 16:15:10.680 | INFO | "huge_shared" : " 27 GB", | 20260505 16:15:10.680 | INFO | "malloc" : " 14 GB", | 20260505 16:15:10.680 | INFO | "pinned" : " 29 GB" | 20260505 16:15:10.680 | INFO | } | 20260505 16:15:10.681 | INFO | Perf report for the last 5.12801 seconds: | 20260505 16:15:10.681 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:15:10.681 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:10.681 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:10.681 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:10.681 | INFO | ANALYSIS : 62.6%, 3.0, 97.0ms, 0 | 20260505 16:15:10.681 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:10.681 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:10.681 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 16:15:10.681 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:10.681 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:10.681 | INFO | Uploaded 36GiB to GPU in 2.18452s (16.4796GiB/s) | 20260505 16:15:10.681 | INFO | Downloaded 34GiB to GPU in 2.13151s (15.9511GiB/s) | 20260505 16:15:15.829 | INFO | Analyzing chunk frames = [4564992,4565504) | 20260505 16:15:15.829 | INFO | MemInfo: { | 20260505 16:15:15.829 | INFO | "gpu" : " 17 GB", | 20260505 16:15:15.829 | INFO | "huge" : "", | 20260505 16:15:15.829 | INFO | "huge_shared" : " 27 GB", | 20260505 16:15:15.830 | INFO | "malloc" : " 14 GB", | 20260505 16:15:15.830 | INFO | "pinned" : " 29 GB" | 20260505 16:15:15.830 | INFO | } | 20260505 16:15:15.830 | INFO | Perf report for the last 5.14918 seconds: | 20260505 16:15:15.830 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:15:15.830 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:15.830 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:15.830 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:15.830 | INFO | ANALYSIS : 62.6%, 3.0, 98.0ms, 0 | 20260505 16:15:15.830 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:15.830 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:15.830 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 16:15:15.830 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:15.830 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:15.830 | INFO | Uploaded 36GiB to GPU in 2.17867s (16.5239GiB/s) | 20260505 16:15:15.830 | INFO | Downloaded 34GiB to GPU in 2.13016s (15.9612GiB/s) | 20260505 16:15:20.976 | INFO | Analyzing chunk frames = [4565504,4566016) | 20260505 16:15:20.976 | INFO | MemInfo: { | 20260505 16:15:20.976 | INFO | "gpu" : " 17 GB", | 20260505 16:15:20.976 | INFO | "huge" : "", | 20260505 16:15:20.976 | INFO | "huge_shared" : " 27 GB", | 20260505 16:15:20.976 | INFO | "malloc" : " 14 GB", | 20260505 16:15:20.976 | INFO | "pinned" : " 29 GB" | 20260505 16:15:20.976 | INFO | } | 20260505 16:15:20.976 | INFO | Perf report for the last 5.14625 seconds: | 20260505 16:15:20.976 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:15:20.976 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:20.976 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:20.976 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:20.976 | INFO | ANALYSIS : 62.4%, 3.0, 97.6ms, 0 | 20260505 16:15:20.976 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:20.976 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:20.976 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:15:20.976 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:20.976 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:20.976 | INFO | Uploaded 36GiB to GPU in 2.14645s (16.7719GiB/s) | 20260505 16:15:20.976 | INFO | Downloaded 34GiB to GPU in 2.04695s (16.6101GiB/s) | 20260505 16:15:26.116 | INFO | Analyzing chunk frames = [4566016,4566528) | 20260505 16:15:26.116 | INFO | MemInfo: { | 20260505 16:15:26.116 | INFO | "gpu" : " 17 GB", | 20260505 16:15:26.116 | INFO | "huge" : "", | 20260505 16:15:26.116 | INFO | "huge_shared" : " 27 GB", | 20260505 16:15:26.116 | INFO | "malloc" : " 14 GB", | 20260505 16:15:26.116 | INFO | "pinned" : " 29 GB" | 20260505 16:15:26.116 | INFO | } | 20260505 16:15:26.116 | INFO | Perf report for the last 5.14042 seconds: | 20260505 16:15:26.116 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:15:26.116 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:26.116 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:26.116 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:26.116 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 16:15:26.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:26.117 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:26.117 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 16:15:26.117 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:26.117 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:26.117 | INFO | Uploaded 36GiB to GPU in 2.17433s (16.5569GiB/s) | 20260505 16:15:26.117 | INFO | Downloaded 34GiB to GPU in 2.18565s (15.556GiB/s) | 20260505 16:15:31.244 | INFO | Analyzing chunk frames = [4566528,4567040) | 20260505 16:15:31.244 | INFO | MemInfo: { | 20260505 16:15:31.244 | INFO | "gpu" : " 17 GB", | 20260505 16:15:31.244 | INFO | "huge" : "", | 20260505 16:15:31.244 | INFO | "huge_shared" : " 27 GB", | 20260505 16:15:31.244 | INFO | "malloc" : " 14 GB", | 20260505 16:15:31.244 | INFO | "pinned" : " 29 GB" | 20260505 16:15:31.244 | INFO | } | 20260505 16:15:31.244 | INFO | Perf report for the last 5.12789 seconds: | 20260505 16:15:31.244 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:15:31.244 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:31.244 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:31.244 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:31.244 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 16:15:31.244 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:31.244 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:31.244 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 16:15:31.244 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:31.244 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:31.245 | INFO | Uploaded 36GiB to GPU in 2.19144s (16.4276GiB/s) | 20260505 16:15:31.245 | INFO | Downloaded 34GiB to GPU in 2.02592s (16.7825GiB/s) | 20260505 16:15:31.379 | INFO | Starting a round of Baz IO | 20260505 16:15:34.592 | INFO | Populating RTMetrics report | 20260505 16:15:34.593 | INFO | RTMetrics report populated | 20260505 16:15:34.593 | INFO | Opening tmp RTMetrics json file | 20260505 16:15:34.603 | INFO | Serializing RTMetrics to json | 20260505 16:15:34.605 | INFO | Serialization done, swapping tmp file now... | 20260505 16:15:34.606 | INFO | RTMetrics done | 20260505 16:15:34.689 | INFO | Finished a round of Baz IO: Wrote 2.179 GiB in 3.309 seconds | 20260505 16:15:34.689 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.28% / 0.00% / 77.71% | 20260505 16:15:34.689 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.378088% / 0% / 8.34465e-05% | 20260505 16:15:34.689 | INFO | Baz IO overall: | 20260505 16:15:34.689 | INFO | BytesWritten : 2711425937082 | 20260505 16:15:34.689 | INFO | NumEvents : 1287196747030 | 20260505 16:15:36.383 | INFO | Analyzing chunk frames = [4567040,4567552) | 20260505 16:15:36.383 | INFO | MemInfo: { | 20260505 16:15:36.383 | INFO | "gpu" : " 17 GB", | 20260505 16:15:36.383 | INFO | "huge" : "", | 20260505 16:15:36.383 | INFO | "huge_shared" : " 27 GB", | 20260505 16:15:36.383 | INFO | "malloc" : " 14 GB", | 20260505 16:15:36.383 | INFO | "pinned" : " 29 GB" | 20260505 16:15:36.383 | INFO | } | 20260505 16:15:36.383 | INFO | Perf report for the last 5.13894 seconds: | 20260505 16:15:36.383 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:15:36.383 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:36.383 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:36.383 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:36.383 | INFO | ANALYSIS : 64.1%, 3.0, 99.6ms, 0 | 20260505 16:15:36.383 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:36.383 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260505 16:15:36.383 | INFO | PRE_HQ : 60.9%, 1.8, 56.9ms, 0 | 20260505 16:15:36.383 | INFO | BAZWRITER : 63.6%, 3.9, 128.9ms, 0 | 20260505 16:15:36.383 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 16:15:36.384 | INFO | Uploaded 36GiB to GPU in 2.18266s (16.4936GiB/s) | 20260505 16:15:36.384 | INFO | Downloaded 38GiB to GPU in 2.25507s (16.8509GiB/s) | 20260505 16:15:41.531 | INFO | Analyzing chunk frames = [4567552,4568064) | 20260505 16:15:41.531 | INFO | MemInfo: { | 20260505 16:15:41.531 | INFO | "gpu" : " 17 GB", | 20260505 16:15:41.531 | INFO | "huge" : "", | 20260505 16:15:41.531 | INFO | "huge_shared" : " 27 GB", | 20260505 16:15:41.532 | INFO | "malloc" : " 14 GB", | 20260505 16:15:41.532 | INFO | "pinned" : " 29 GB" | 20260505 16:15:41.532 | INFO | } | 20260505 16:15:41.532 | INFO | Perf report for the last 5.14831 seconds: | 20260505 16:15:41.532 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:15:41.532 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:41.532 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:41.532 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:41.532 | INFO | ANALYSIS : 63.2%, 3.0, 98.9ms, 0 | 20260505 16:15:41.532 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:41.532 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:41.532 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:15:41.532 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:41.532 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:41.532 | INFO | Uploaded 36GiB to GPU in 2.20617s (16.3179GiB/s) | 20260505 16:15:41.532 | INFO | Downloaded 34GiB to GPU in 2.06627s (16.4547GiB/s) | 20260505 16:15:46.667 | INFO | Analyzing chunk frames = [4568064,4568576) | 20260505 16:15:46.667 | INFO | MemInfo: { | 20260505 16:15:46.667 | INFO | "gpu" : " 17 GB", | 20260505 16:15:46.667 | INFO | "huge" : "", | 20260505 16:15:46.667 | INFO | "huge_shared" : " 27 GB", | 20260505 16:15:46.667 | INFO | "malloc" : " 14 GB", | 20260505 16:15:46.667 | INFO | "pinned" : " 29 GB" | 20260505 16:15:46.667 | INFO | } | 20260505 16:15:46.667 | INFO | Perf report for the last 5.13502 seconds: | 20260505 16:15:46.667 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:15:46.667 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:46.667 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:46.667 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:46.667 | INFO | ANALYSIS : 62.9%, 3.0, 97.8ms, 0 | 20260505 16:15:46.667 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:46.667 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:46.667 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 16:15:46.667 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:46.667 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:46.667 | INFO | Uploaded 36GiB to GPU in 2.20213s (16.3478GiB/s) | 20260505 16:15:46.667 | INFO | Downloaded 34GiB to GPU in 2.0403s (16.6642GiB/s) | 20260505 16:15:51.805 | INFO | Analyzing chunk frames = [4568576,4569088) | 20260505 16:15:51.805 | INFO | MemInfo: { | 20260505 16:15:51.805 | INFO | "gpu" : " 17 GB", | 20260505 16:15:51.805 | INFO | "huge" : "", | 20260505 16:15:51.805 | INFO | "huge_shared" : " 27 GB", | 20260505 16:15:51.805 | INFO | "malloc" : " 14 GB", | 20260505 16:15:51.805 | INFO | "pinned" : " 29 GB" | 20260505 16:15:51.805 | INFO | } | 20260505 16:15:51.805 | INFO | Perf report for the last 5.13825 seconds: | 20260505 16:15:51.805 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:15:51.805 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:51.805 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:51.805 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:51.805 | INFO | ANALYSIS : 62.5%, 3.0, 97.1ms, 0 | 20260505 16:15:51.805 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:51.805 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:51.805 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:15:51.805 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:51.805 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:51.805 | INFO | Uploaded 36GiB to GPU in 2.17485s (16.5529GiB/s) | 20260505 16:15:51.805 | INFO | Downloaded 34GiB to GPU in 2.14177s (15.8747GiB/s) | 20260505 16:15:56.943 | INFO | Analyzing chunk frames = [4569088,4569600) | 20260505 16:15:56.943 | INFO | MemInfo: { | 20260505 16:15:56.943 | INFO | "gpu" : " 17 GB", | 20260505 16:15:56.943 | INFO | "huge" : "", | 20260505 16:15:56.943 | INFO | "huge_shared" : " 27 GB", | 20260505 16:15:56.943 | INFO | "malloc" : " 14 GB", | 20260505 16:15:56.943 | INFO | "pinned" : " 29 GB" | 20260505 16:15:56.943 | INFO | } | 20260505 16:15:56.943 | INFO | Perf report for the last 5.13795 seconds: | 20260505 16:15:56.943 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:15:56.943 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:56.943 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:56.943 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:56.943 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260505 16:15:56.943 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:56.943 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:56.943 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:15:56.943 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:56.943 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:15:56.943 | INFO | Uploaded 36GiB to GPU in 2.14791s (16.7605GiB/s) | 20260505 16:15:56.943 | INFO | Downloaded 34GiB to GPU in 2.0564s (16.5337GiB/s) | 20260505 16:16:02.084 | INFO | Analyzing chunk frames = [4569600,4570112) | 20260505 16:16:02.084 | INFO | MemInfo: { | 20260505 16:16:02.084 | INFO | "gpu" : " 17 GB", | 20260505 16:16:02.084 | INFO | "huge" : "", | 20260505 16:16:02.084 | INFO | "huge_shared" : " 27 GB", | 20260505 16:16:02.084 | INFO | "malloc" : " 14 GB", | 20260505 16:16:02.084 | INFO | "pinned" : " 29 GB" | 20260505 16:16:02.084 | INFO | } | 20260505 16:16:02.084 | INFO | Perf report for the last 5.14161 seconds: | 20260505 16:16:02.084 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:16:02.084 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:02.084 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:02.084 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:02.084 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260505 16:16:02.085 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:02.085 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:02.085 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 16:16:02.085 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:02.085 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:02.085 | INFO | Uploaded 36GiB to GPU in 2.14022s (16.8207GiB/s) | 20260505 16:16:02.085 | INFO | Downloaded 34GiB to GPU in 2.03458s (16.711GiB/s) | 20260505 16:16:07.228 | INFO | Analyzing chunk frames = [4570112,4570624) | 20260505 16:16:07.228 | INFO | MemInfo: { | 20260505 16:16:07.228 | INFO | "gpu" : " 17 GB", | 20260505 16:16:07.228 | INFO | "huge" : "", | 20260505 16:16:07.228 | INFO | "huge_shared" : " 27 GB", | 20260505 16:16:07.228 | INFO | "malloc" : " 14 GB", | 20260505 16:16:07.228 | INFO | "pinned" : " 29 GB" | 20260505 16:16:07.228 | INFO | } | 20260505 16:16:07.228 | INFO | Perf report for the last 5.14355 seconds: | 20260505 16:16:07.228 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:16:07.228 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:07.228 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:07.228 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:07.228 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 16:16:07.228 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:07.228 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:07.228 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 16:16:07.228 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:07.228 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:07.228 | INFO | Uploaded 36GiB to GPU in 2.15248s (16.7249GiB/s) | 20260505 16:16:07.228 | INFO | Downloaded 34GiB to GPU in 2.18202s (15.5819GiB/s) | 20260505 16:16:12.362 | INFO | Analyzing chunk frames = [4570624,4571136) | 20260505 16:16:12.362 | INFO | MemInfo: { | 20260505 16:16:12.362 | INFO | "gpu" : " 17 GB", | 20260505 16:16:12.362 | INFO | "huge" : "", | 20260505 16:16:12.362 | INFO | "huge_shared" : " 27 GB", | 20260505 16:16:12.362 | INFO | "malloc" : " 14 GB", | 20260505 16:16:12.362 | INFO | "pinned" : " 29 GB" | 20260505 16:16:12.362 | INFO | } | 20260505 16:16:12.363 | INFO | Perf report for the last 5.13452 seconds: | 20260505 16:16:12.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:16:12.363 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:12.363 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:12.363 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:12.363 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 16:16:12.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:12.363 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:12.363 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 16:16:12.363 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:12.363 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:12.363 | INFO | Uploaded 36GiB to GPU in 2.15377s (16.7149GiB/s) | 20260505 16:16:12.363 | INFO | Downloaded 34GiB to GPU in 2.01886s (16.8412GiB/s) | 20260505 16:16:12.363 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 16:16:12.363 | INFO | Total mem usage: 15.3 GiB | 20260505 16:16:12.363 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 16:16:12.363 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.75%) | 20260505 16:16:12.363 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 16:16:12.363 | INFO | High Water Breakdown: | 20260505 16:16:12.363 | INFO | 61% : 8.67 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 16:16:12.363 | INFO | 38.9% : 5.53 GiB ( 83.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 16:16:12.363 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 16:16:12.363 | INFO | ------------------------------------------ | 20260505 16:16:12.363 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:16:12.363 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 16:16:12.363 | INFO | Total mem usage: 26.5 GiB | 20260505 16:16:12.363 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 16:16:12.363 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 16:16:12.363 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 16:16:12.363 | INFO | High Water Breakdown: | 20260505 16:16:12.363 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:16:12.363 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:16:12.363 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:16:12.363 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:16:12.363 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:16:12.363 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:16:12.363 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 16:16:12.363 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 16:16:12.363 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:16:12.363 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 16:16:12.363 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 16:16:12.363 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 16:16:12.363 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 16:16:12.363 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 16:16:12.363 | INFO | ------------------------------------------ | 20260505 16:16:12.363 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:16:12.363 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 16:16:12.363 | INFO | Total mem usage: 25.6 GiB | 20260505 16:16:12.363 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 16:16:12.363 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 16:16:12.363 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 16:16:12.363 | INFO | High Water Breakdown: | 20260505 16:16:12.363 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 16:16:12.363 | INFO | ------------------------------------------ | 20260505 16:16:12.363 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:16:12.363 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 16:16:12.363 | INFO | Total mem usage: 15.7 GiB | 20260505 16:16:12.364 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 16:16:12.364 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 16:16:12.364 | INFO | Unused Cache: 896 MiB (Waste: 5.58%) | 20260505 16:16:12.364 | INFO | High Water Breakdown: | 20260505 16:16:12.364 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 16:16:12.364 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 16:16:12.364 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 16:16:12.364 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 16:16:12.364 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 16:16:12.364 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 16:16:12.364 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 16:16:12.364 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 16:16:12.364 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 16:16:12.364 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:16:12.364 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 16:16:12.364 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:16:12.364 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 16:16:12.364 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 16:16:12.364 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 16:16:12.364 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 16:16:12.364 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:16:12.364 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 16:16:12.364 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:16:12.364 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:16:12.364 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:16:12.364 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 16:16:12.364 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 16:16:12.364 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:16:12.364 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 16:16:12.364 | INFO | ------------------------------------------ | 20260505 16:16:12.364 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:16:12.364 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 16:16:12.364 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:16:12.364 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:16:12.364 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.53e+01, 1.36e+01 / 2.47e+00 | 20260505 16:16:12.364 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 1.02e-01, 1.14e-02 / 4.99e-03 | 20260505 16:16:12.364 | INFO | PerfReport: 11.78%, Baseline -- 3.73e+00 / 4.48e+00, 5.71e-01 / 1.74e-01 | 20260505 16:16:12.364 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.52e+00, 1.94e-01 / 8.73e-02 | 20260505 16:16:12.364 | INFO | PerfReport: 15.97%, DME -- 4.04e+01 / 4.51e+01, 4.46e+00 / 2.63e+00 | 20260505 16:16:12.364 | INFO | PerfReport: 56.64%, FrameLabeling -- 1.79e+01 / 1.92e+01, 1.40e+00 / 1.37e+00 | 20260505 16:16:12.364 | INFO | PerfReport: 7.40%, PulseAccumulating -- 2.34e+00 / 2.96e+00, 4.51e-01 / 1.24e-01 | 20260505 16:16:12.364 | INFO | PerfReport: 3.43%, Metrics -- 1.09e+00 / 1.43e+00, 3.00e-01 / 1.38e-01 | 20260505 16:16:12.364 | INFO | PerfReport: 1.20%, PulseSerializing -- 3.79e-01 / 4.37e-01, 6.67e-02 / 3.01e-02 | 20260505 16:16:12.364 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 16:16:12.364 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:16:12.364 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:16:12.364 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.58e+01 / 0.00e+00 | 20260505 16:16:12.364 | INFO | PerfReport: 22.57%, Upload -- 2.26e+01 / 0.00e+00, 7.88e+00 / 0.00e+00 | 20260505 16:16:12.364 | INFO | PerfReport: 21.85%, Download -- 2.19e+01 / 0.00e+00, 7.86e+00 / 0.00e+00 | 20260505 16:16:12.364 | INFO | PerfReport: 31.59%, Compute -- 3.17e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 16:16:12.364 | INFO | PerfReport: 2.24%, LockUpload -- 2.24e+00 / 0.00e+00, 7.98e+00 / 0.00e+00 | 20260505 16:16:12.364 | INFO | PerfReport: 1.47%, LockDownload -- 1.47e+00 / 0.00e+00, 2.98e+00 / 0.00e+00 | 20260505 16:16:12.364 | INFO | PerfReport: 20.27%, LockCompute -- 2.03e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 16:16:12.364 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.05e-02 / 0.00e+00 | 20260505 16:16:12.493 | INFO | Starting a round of Baz IO | 20260505 16:16:15.700 | INFO | Populating RTMetrics report | 20260505 16:16:15.700 | INFO | RTMetrics report populated | 20260505 16:16:15.700 | INFO | Opening tmp RTMetrics json file | 20260505 16:16:15.734 | INFO | Serializing RTMetrics to json | 20260505 16:16:15.762 | INFO | Serialization done, swapping tmp file now... | 20260505 16:16:15.780 | INFO | RTMetrics done | 20260505 16:16:15.795 | INFO | Finished a round of Baz IO: Wrote 2.181 GiB in 3.302 seconds | 20260505 16:16:15.795 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.28% / 0.00% / 77.72% | 20260505 16:16:15.795 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.379431% / 1.19209e-05% / 4.76837e-05% | 20260505 16:16:15.795 | INFO | Baz IO overall: | 20260505 16:16:15.795 | INFO | BytesWritten : 2713767629552 | 20260505 16:16:15.795 | INFO | NumEvents : 1288295699202 | 20260505 16:16:17.497 | INFO | Analyzing chunk frames = [4571136,4571648) | 20260505 16:16:17.497 | INFO | MemInfo: { | 20260505 16:16:17.497 | INFO | "gpu" : " 17 GB", | 20260505 16:16:17.497 | INFO | "huge" : "", | 20260505 16:16:17.497 | INFO | "huge_shared" : " 27 GB", | 20260505 16:16:17.497 | INFO | "malloc" : " 14 GB", | 20260505 16:16:17.497 | INFO | "pinned" : " 29 GB" | 20260505 16:16:17.497 | INFO | } | 20260505 16:16:17.498 | INFO | Perf report for the last 5.13509 seconds: | 20260505 16:16:17.498 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:16:17.498 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:17.498 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:17.498 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:17.498 | INFO | ANALYSIS : 63.9%, 3.0, 99.2ms, 0 | 20260505 16:16:17.498 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:17.498 | INFO | RT_METRICS : 46.5%, 1.0, 24.3ms, 0 | 20260505 16:16:17.498 | INFO | PRE_HQ : 60.7%, 1.8, 56.4ms, 0 | 20260505 16:16:17.498 | INFO | BAZWRITER : 63.7%, 3.9, 128.1ms, 0 | 20260505 16:16:17.498 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 16:16:17.498 | INFO | Uploaded 36GiB to GPU in 2.20122s (16.3546GiB/s) | 20260505 16:16:17.498 | INFO | Downloaded 38GiB to GPU in 2.2668s (16.7637GiB/s) | 20260505 16:16:22.630 | INFO | Analyzing chunk frames = [4571648,4572160) | 20260505 16:16:22.631 | INFO | MemInfo: { | 20260505 16:16:22.631 | INFO | "gpu" : " 17 GB", | 20260505 16:16:22.631 | INFO | "huge" : "", | 20260505 16:16:22.631 | INFO | "huge_shared" : " 27 GB", | 20260505 16:16:22.631 | INFO | "malloc" : " 14 GB", | 20260505 16:16:22.631 | INFO | "pinned" : " 29 GB" | 20260505 16:16:22.631 | INFO | } | 20260505 16:16:22.631 | INFO | Perf report for the last 5.13308 seconds: | 20260505 16:16:22.631 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:16:22.631 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:22.631 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:22.631 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:22.631 | INFO | ANALYSIS : 63.5%, 3.0, 99.0ms, 0 | 20260505 16:16:22.631 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:22.631 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:22.631 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:16:22.631 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:22.631 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:22.631 | INFO | Uploaded 36GiB to GPU in 2.20382s (16.3353GiB/s) | 20260505 16:16:22.631 | INFO | Downloaded 34GiB to GPU in 2.05532s (16.5424GiB/s) | 20260505 16:16:27.780 | INFO | Analyzing chunk frames = [4572160,4572672) | 20260505 16:16:27.780 | INFO | MemInfo: { | 20260505 16:16:27.780 | INFO | "gpu" : " 17 GB", | 20260505 16:16:27.780 | INFO | "huge" : "", | 20260505 16:16:27.780 | INFO | "huge_shared" : " 27 GB", | 20260505 16:16:27.781 | INFO | "malloc" : " 14 GB", | 20260505 16:16:27.781 | INFO | "pinned" : " 29 GB" | 20260505 16:16:27.781 | INFO | } | 20260505 16:16:27.781 | INFO | Perf report for the last 5.14989 seconds: | 20260505 16:16:27.781 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:16:27.781 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:27.781 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:27.781 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:27.781 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 16:16:27.781 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:27.781 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:27.781 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:16:27.781 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:27.781 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:27.781 | INFO | Uploaded 36GiB to GPU in 2.17685s (16.5377GiB/s) | 20260505 16:16:27.781 | INFO | Downloaded 34GiB to GPU in 2.03351s (16.7199GiB/s) | 20260505 16:16:32.929 | INFO | Analyzing chunk frames = [4572672,4573184) | 20260505 16:16:32.929 | INFO | MemInfo: { | 20260505 16:16:32.929 | INFO | "gpu" : " 17 GB", | 20260505 16:16:32.929 | INFO | "huge" : "", | 20260505 16:16:32.929 | INFO | "huge_shared" : " 27 GB", | 20260505 16:16:32.929 | INFO | "malloc" : " 14 GB", | 20260505 16:16:32.929 | INFO | "pinned" : " 29 GB" | 20260505 16:16:32.929 | INFO | } | 20260505 16:16:32.930 | INFO | Perf report for the last 5.14886 seconds: | 20260505 16:16:32.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:16:32.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:32.930 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:32.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:32.930 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260505 16:16:32.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:32.930 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:32.930 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:16:32.930 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:32.930 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:32.930 | INFO | Uploaded 36GiB to GPU in 2.19883s (16.3723GiB/s) | 20260505 16:16:32.930 | INFO | Downloaded 34GiB to GPU in 2.13769s (15.905GiB/s) | 20260505 16:16:38.059 | INFO | Analyzing chunk frames = [4573184,4573696) | 20260505 16:16:38.059 | INFO | MemInfo: { | 20260505 16:16:38.059 | INFO | "gpu" : " 17 GB", | 20260505 16:16:38.059 | INFO | "huge" : "", | 20260505 16:16:38.059 | INFO | "huge_shared" : " 27 GB", | 20260505 16:16:38.059 | INFO | "malloc" : " 14 GB", | 20260505 16:16:38.059 | INFO | "pinned" : " 29 GB" | 20260505 16:16:38.059 | INFO | } | 20260505 16:16:38.059 | INFO | Perf report for the last 5.12981 seconds: | 20260505 16:16:38.059 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:16:38.059 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:38.059 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:38.059 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:38.059 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260505 16:16:38.059 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:38.059 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:38.059 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:16:38.059 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:38.059 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:38.060 | INFO | Uploaded 36GiB to GPU in 2.1378s (16.8397GiB/s) | 20260505 16:16:38.060 | INFO | Downloaded 34GiB to GPU in 2.05548s (16.5412GiB/s) | 20260505 16:16:43.192 | INFO | Analyzing chunk frames = [4573696,4574208) | 20260505 16:16:43.192 | INFO | MemInfo: { | 20260505 16:16:43.192 | INFO | "gpu" : " 17 GB", | 20260505 16:16:43.192 | INFO | "huge" : "", | 20260505 16:16:43.192 | INFO | "huge_shared" : " 27 GB", | 20260505 16:16:43.192 | INFO | "malloc" : " 14 GB", | 20260505 16:16:43.192 | INFO | "pinned" : " 29 GB" | 20260505 16:16:43.192 | INFO | } | 20260505 16:16:43.193 | INFO | Perf report for the last 5.13322 seconds: | 20260505 16:16:43.193 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:16:43.193 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:43.193 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:43.193 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:43.193 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 16:16:43.193 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:43.193 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:43.193 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 16:16:43.193 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:43.193 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:43.193 | INFO | Uploaded 36GiB to GPU in 2.14798s (16.7599GiB/s) | 20260505 16:16:43.193 | INFO | Downloaded 34GiB to GPU in 2.04021s (16.665GiB/s) | 20260505 16:16:48.337 | INFO | Analyzing chunk frames = [4574208,4574720) | 20260505 16:16:48.337 | INFO | MemInfo: { | 20260505 16:16:48.337 | INFO | "gpu" : " 17 GB", | 20260505 16:16:48.337 | INFO | "huge" : "", | 20260505 16:16:48.337 | INFO | "huge_shared" : " 27 GB", | 20260505 16:16:48.337 | INFO | "malloc" : " 14 GB", | 20260505 16:16:48.337 | INFO | "pinned" : " 29 GB" | 20260505 16:16:48.337 | INFO | } | 20260505 16:16:48.338 | INFO | Perf report for the last 5.14494 seconds: | 20260505 16:16:48.338 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:16:48.338 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:48.338 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:48.338 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:48.338 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 16:16:48.338 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:48.338 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:48.338 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 16:16:48.338 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:48.338 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:48.338 | INFO | Uploaded 36GiB to GPU in 2.12307s (16.9566GiB/s) | 20260505 16:16:48.338 | INFO | Downloaded 34GiB to GPU in 2.17121s (15.6595GiB/s) | 20260505 16:16:53.481 | INFO | Analyzing chunk frames = [4574720,4575232) | 20260505 16:16:53.481 | INFO | MemInfo: { | 20260505 16:16:53.481 | INFO | "gpu" : " 17 GB", | 20260505 16:16:53.481 | INFO | "huge" : "", | 20260505 16:16:53.481 | INFO | "huge_shared" : " 27 GB", | 20260505 16:16:53.481 | INFO | "malloc" : " 14 GB", | 20260505 16:16:53.481 | INFO | "pinned" : " 29 GB" | 20260505 16:16:53.481 | INFO | } | 20260505 16:16:53.481 | INFO | Perf report for the last 5.14328 seconds: | 20260505 16:16:53.481 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:16:53.481 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:53.481 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:53.481 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:53.481 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 16:16:53.481 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:53.481 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:53.481 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:16:53.481 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:53.481 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:53.481 | INFO | Uploaded 36GiB to GPU in 2.16037s (16.6638GiB/s) | 20260505 16:16:53.481 | INFO | Downloaded 34GiB to GPU in 2.03256s (16.7277GiB/s) | 20260505 16:16:53.620 | INFO | Starting a round of Baz IO | 20260505 16:16:56.800 | INFO | Populating RTMetrics report | 20260505 16:16:56.801 | INFO | RTMetrics report populated | 20260505 16:16:56.801 | INFO | Opening tmp RTMetrics json file | 20260505 16:16:56.810 | INFO | Serializing RTMetrics to json | 20260505 16:16:56.812 | INFO | Serialization done, swapping tmp file now... | 20260505 16:16:56.813 | INFO | RTMetrics done | 20260505 16:16:56.896 | INFO | Finished a round of Baz IO: Wrote 2.178 GiB in 3.276 seconds | 20260505 16:16:56.896 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.28% / 0.00% / 77.72% | 20260505 16:16:56.896 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.378001% / 3.97364e-06% / 6.75519e-05% | 20260505 16:16:56.896 | INFO | Baz IO overall: | 20260505 16:16:56.896 | INFO | BytesWritten : 2716106642098 | 20260505 16:16:56.896 | INFO | NumEvents : 1289392789036 | 20260505 16:16:58.620 | INFO | Analyzing chunk frames = [4575232,4575744) | 20260505 16:16:58.620 | INFO | MemInfo: { | 20260505 16:16:58.620 | INFO | "gpu" : " 17 GB", | 20260505 16:16:58.620 | INFO | "huge" : "", | 20260505 16:16:58.620 | INFO | "huge_shared" : " 27 GB", | 20260505 16:16:58.620 | INFO | "malloc" : " 14 GB", | 20260505 16:16:58.620 | INFO | "pinned" : " 29 GB" | 20260505 16:16:58.620 | INFO | } | 20260505 16:16:58.620 | INFO | Perf report for the last 5.13919 seconds: | 20260505 16:16:58.620 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:16:58.620 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:58.620 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:58.620 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:58.620 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260505 16:16:58.620 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:16:58.620 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260505 16:16:58.620 | INFO | PRE_HQ : 60.5%, 1.8, 56.5ms, 0 | 20260505 16:16:58.620 | INFO | BAZWRITER : 63.2%, 3.9, 127.8ms, 0 | 20260505 16:16:58.620 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260505 16:16:58.620 | INFO | Uploaded 36GiB to GPU in 2.15379s (16.7147GiB/s) | 20260505 16:16:58.620 | INFO | Downloaded 38GiB to GPU in 2.25415s (16.8578GiB/s) | 20260505 16:17:03.757 | INFO | Analyzing chunk frames = [4575744,4576256) | 20260505 16:17:03.757 | INFO | MemInfo: { | 20260505 16:17:03.757 | INFO | "gpu" : " 17 GB", | 20260505 16:17:03.757 | INFO | "huge" : "", | 20260505 16:17:03.757 | INFO | "huge_shared" : " 27 GB", | 20260505 16:17:03.757 | INFO | "malloc" : " 14 GB", | 20260505 16:17:03.757 | INFO | "pinned" : " 29 GB" | 20260505 16:17:03.757 | INFO | } | 20260505 16:17:03.757 | INFO | Perf report for the last 5.13708 seconds: | 20260505 16:17:03.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:17:03.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:03.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:03.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:03.757 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 16:17:03.757 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:03.757 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:03.757 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:17:03.757 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:03.757 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:03.758 | INFO | Uploaded 36GiB to GPU in 2.12146s (16.9694GiB/s) | 20260505 16:17:03.758 | INFO | Downloaded 34GiB to GPU in 2.03283s (16.7254GiB/s) | 20260505 16:17:08.888 | INFO | Analyzing chunk frames = [4576256,4576768) | 20260505 16:17:08.888 | INFO | MemInfo: { | 20260505 16:17:08.888 | INFO | "gpu" : " 17 GB", | 20260505 16:17:08.888 | INFO | "huge" : "", | 20260505 16:17:08.888 | INFO | "huge_shared" : " 27 GB", | 20260505 16:17:08.888 | INFO | "malloc" : " 14 GB", | 20260505 16:17:08.888 | INFO | "pinned" : " 29 GB" | 20260505 16:17:08.888 | INFO | } | 20260505 16:17:08.889 | INFO | Perf report for the last 5.13135 seconds: | 20260505 16:17:08.889 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:17:08.889 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:08.889 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:08.889 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:08.889 | INFO | ANALYSIS : 62.9%, 3.0, 97.7ms, 0 | 20260505 16:17:08.889 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:08.889 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:08.889 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 16:17:08.889 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:08.889 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:08.889 | INFO | Uploaded 36GiB to GPU in 2.20207s (16.3483GiB/s) | 20260505 16:17:08.889 | INFO | Downloaded 34GiB to GPU in 2.03872s (16.6771GiB/s) | 20260505 16:17:14.040 | INFO | Analyzing chunk frames = [4576768,4577280) | 20260505 16:17:14.040 | INFO | MemInfo: { | 20260505 16:17:14.040 | INFO | "gpu" : " 17 GB", | 20260505 16:17:14.040 | INFO | "huge" : "", | 20260505 16:17:14.040 | INFO | "huge_shared" : " 27 GB", | 20260505 16:17:14.040 | INFO | "malloc" : " 14 GB", | 20260505 16:17:14.040 | INFO | "pinned" : " 29 GB" | 20260505 16:17:14.040 | INFO | } | 20260505 16:17:14.040 | INFO | Perf report for the last 5.15133 seconds: | 20260505 16:17:14.040 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:17:14.040 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:14.040 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:14.040 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:14.040 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260505 16:17:14.040 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:14.040 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:14.040 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 16:17:14.040 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:14.040 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:14.040 | INFO | Uploaded 36GiB to GPU in 2.17394s (16.5598GiB/s) | 20260505 16:17:14.040 | INFO | Downloaded 34GiB to GPU in 2.1356s (15.9206GiB/s) | 20260505 16:17:19.183 | INFO | Analyzing chunk frames = [4577280,4577792) | 20260505 16:17:19.183 | INFO | MemInfo: { | 20260505 16:17:19.183 | INFO | "gpu" : " 17 GB", | 20260505 16:17:19.183 | INFO | "huge" : "", | 20260505 16:17:19.183 | INFO | "huge_shared" : " 27 GB", | 20260505 16:17:19.183 | INFO | "malloc" : " 14 GB", | 20260505 16:17:19.183 | INFO | "pinned" : " 29 GB" | 20260505 16:17:19.183 | INFO | } | 20260505 16:17:19.183 | INFO | Perf report for the last 5.14327 seconds: | 20260505 16:17:19.183 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:17:19.183 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:19.183 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:19.183 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:19.183 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260505 16:17:19.183 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:19.183 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:19.183 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 16:17:19.183 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:19.183 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:19.184 | INFO | Uploaded 36GiB to GPU in 2.14724s (16.7657GiB/s) | 20260505 16:17:19.184 | INFO | Downloaded 34GiB to GPU in 2.05752s (16.5247GiB/s) | 20260505 16:17:24.306 | INFO | Analyzing chunk frames = [4577792,4578304) | 20260505 16:17:24.306 | INFO | MemInfo: { | 20260505 16:17:24.306 | INFO | "gpu" : " 17 GB", | 20260505 16:17:24.306 | INFO | "huge" : "", | 20260505 16:17:24.306 | INFO | "huge_shared" : " 27 GB", | 20260505 16:17:24.306 | INFO | "malloc" : " 14 GB", | 20260505 16:17:24.306 | INFO | "pinned" : " 29 GB" | 20260505 16:17:24.306 | INFO | } | 20260505 16:17:24.306 | INFO | Perf report for the last 5.1233 seconds: | 20260505 16:17:24.307 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:17:24.307 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:24.307 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:24.307 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:24.307 | INFO | ANALYSIS : 62.7%, 3.0, 97.5ms, 0 | 20260505 16:17:24.307 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:24.307 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:24.307 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:17:24.307 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:24.307 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:24.307 | INFO | Uploaded 36GiB to GPU in 2.1418s (16.8083GiB/s) | 20260505 16:17:24.307 | INFO | Downloaded 34GiB to GPU in 2.04181s (16.6519GiB/s) | 20260505 16:17:29.446 | INFO | Analyzing chunk frames = [4578304,4578816) | 20260505 16:17:29.446 | INFO | MemInfo: { | 20260505 16:17:29.446 | INFO | "gpu" : " 17 GB", | 20260505 16:17:29.446 | INFO | "huge" : "", | 20260505 16:17:29.446 | INFO | "huge_shared" : " 27 GB", | 20260505 16:17:29.446 | INFO | "malloc" : " 14 GB", | 20260505 16:17:29.446 | INFO | "pinned" : " 29 GB" | 20260505 16:17:29.446 | INFO | } | 20260505 16:17:29.446 | INFO | Perf report for the last 5.13916 seconds: | 20260505 16:17:29.446 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:17:29.446 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:29.446 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:29.446 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:29.446 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 16:17:29.446 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:29.446 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:29.446 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 16:17:29.446 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:29.446 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:29.446 | INFO | Uploaded 36GiB to GPU in 2.10633s (17.0913GiB/s) | 20260505 16:17:29.446 | INFO | Downloaded 34GiB to GPU in 2.17636s (15.6224GiB/s) | 20260505 16:17:34.590 | INFO | Analyzing chunk frames = [4578816,4579328) | 20260505 16:17:34.590 | INFO | MemInfo: { | 20260505 16:17:34.590 | INFO | "gpu" : " 17 GB", | 20260505 16:17:34.590 | INFO | "huge" : "", | 20260505 16:17:34.590 | INFO | "huge_shared" : " 27 GB", | 20260505 16:17:34.590 | INFO | "malloc" : " 14 GB", | 20260505 16:17:34.590 | INFO | "pinned" : " 29 GB" | 20260505 16:17:34.590 | INFO | } | 20260505 16:17:34.590 | INFO | Perf report for the last 5.14461 seconds: | 20260505 16:17:34.590 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:17:34.590 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:34.590 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:34.590 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:34.590 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 16:17:34.590 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:34.590 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:34.591 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:17:34.591 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:34.591 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:34.591 | INFO | Uploaded 36GiB to GPU in 2.21821s (16.2293GiB/s) | 20260505 16:17:34.591 | INFO | Downloaded 34GiB to GPU in 2.09726s (16.2116GiB/s) | 20260505 16:17:34.591 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 16:17:34.591 | INFO | Total mem usage: 15.3 GiB | 20260505 16:17:34.591 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 16:17:34.591 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.87%) | 20260505 16:17:34.591 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 16:17:34.591 | INFO | High Water Breakdown: | 20260505 16:17:34.591 | INFO | 61% : 8.67 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 16:17:34.591 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 16:17:34.591 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 16:17:34.591 | INFO | ------------------------------------------ | 20260505 16:17:34.591 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:17:34.591 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 16:17:34.591 | INFO | Total mem usage: 26.5 GiB | 20260505 16:17:34.591 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 16:17:34.591 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 16:17:34.591 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 16:17:34.591 | INFO | High Water Breakdown: | 20260505 16:17:34.591 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:17:34.591 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:17:34.591 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:17:34.591 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:17:34.591 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:17:34.591 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:17:34.591 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 16:17:34.591 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 16:17:34.591 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:17:34.591 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 16:17:34.591 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 16:17:34.591 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 16:17:34.591 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 16:17:34.591 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 16:17:34.591 | INFO | ------------------------------------------ | 20260505 16:17:34.591 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:17:34.591 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 16:17:34.591 | INFO | Total mem usage: 25.6 GiB | 20260505 16:17:34.591 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 16:17:34.591 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 16:17:34.591 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 16:17:34.591 | INFO | High Water Breakdown: | 20260505 16:17:34.591 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 16:17:34.591 | INFO | ------------------------------------------ | 20260505 16:17:34.591 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:17:34.591 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 16:17:34.591 | INFO | Total mem usage: 15.7 GiB | 20260505 16:17:34.591 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 16:17:34.592 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 16:17:34.592 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 16:17:34.592 | INFO | High Water Breakdown: | 20260505 16:17:34.592 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 16:17:34.592 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 16:17:34.592 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 16:17:34.592 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 16:17:34.592 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 16:17:34.592 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 16:17:34.592 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 16:17:34.592 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 16:17:34.592 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 16:17:34.592 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:17:34.592 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 16:17:34.592 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:17:34.592 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 16:17:34.592 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 16:17:34.592 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 16:17:34.592 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 16:17:34.592 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:17:34.592 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 16:17:34.592 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:17:34.592 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:17:34.592 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:17:34.592 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 16:17:34.592 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 16:17:34.592 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:17:34.592 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 16:17:34.592 | INFO | ------------------------------------------ | 20260505 16:17:34.592 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:17:34.592 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 16:17:34.592 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:17:34.592 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:17:34.592 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.52e+01, 1.37e+01 / 2.08e+00 | 20260505 16:17:34.592 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 1.02e-01, 1.10e-02 / 4.65e-03 | 20260505 16:17:34.592 | INFO | PerfReport: 11.78%, Baseline -- 3.73e+00 / 4.44e+00, 5.73e-01 / 2.51e-01 | 20260505 16:17:34.592 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.51e+00, 1.94e-01 / 1.10e-01 | 20260505 16:17:34.592 | INFO | PerfReport: 15.92%, DME -- 4.03e+01 / 4.46e+01, 4.28e+00 / 2.16e+00 | 20260505 16:17:34.592 | INFO | PerfReport: 56.72%, FrameLabeling -- 1.80e+01 / 1.98e+01, 1.47e+00 / 1.58e+00 | 20260505 16:17:34.592 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.34e+00 / 2.91e+00, 4.51e-01 / 1.56e-01 | 20260505 16:17:34.592 | INFO | PerfReport: 3.43%, Metrics -- 1.09e+00 / 1.41e+00, 3.02e-01 / 1.63e-01 | 20260505 16:17:34.592 | INFO | PerfReport: 1.20%, PulseSerializing -- 3.79e-01 / 4.33e-01, 6.79e-02 / 3.59e-02 | 20260505 16:17:34.592 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 16:17:34.592 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:17:34.592 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:17:34.592 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260505 16:17:34.592 | INFO | PerfReport: 22.50%, Upload -- 2.25e+01 / 0.00e+00, 7.86e+00 / 0.00e+00 | 20260505 16:17:34.592 | INFO | PerfReport: 21.86%, Download -- 2.19e+01 / 0.00e+00, 7.87e+00 / 0.00e+00 | 20260505 16:17:34.592 | INFO | PerfReport: 31.63%, Compute -- 3.17e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 16:17:34.592 | INFO | PerfReport: 2.23%, LockUpload -- 2.23e+00 / 0.00e+00, 7.88e+00 / 0.00e+00 | 20260505 16:17:34.592 | INFO | PerfReport: 1.49%, LockDownload -- 1.49e+00 / 0.00e+00, 3.12e+00 / 0.00e+00 | 20260505 16:17:34.592 | INFO | PerfReport: 20.28%, LockCompute -- 2.03e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 16:17:34.592 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.84e-02 / 0.00e+00 | 20260505 16:17:34.723 | INFO | Starting a round of Baz IO | 20260505 16:17:37.929 | INFO | Populating RTMetrics report | 20260505 16:17:37.929 | INFO | RTMetrics report populated | 20260505 16:17:37.929 | INFO | Opening tmp RTMetrics json file | 20260505 16:17:37.940 | INFO | Serializing RTMetrics to json | 20260505 16:17:37.943 | INFO | Serialization done, swapping tmp file now... | 20260505 16:17:37.944 | INFO | RTMetrics done | 20260505 16:17:38.032 | INFO | Finished a round of Baz IO: Wrote 2.179 GiB in 3.309 seconds | 20260505 16:17:38.032 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.28% / 0.00% / 77.72% | 20260505 16:17:38.032 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.377607% / 3.97364e-06% / 7.94729e-05% | 20260505 16:17:38.032 | INFO | Baz IO overall: | 20260505 16:17:38.032 | INFO | BytesWritten : 2718446784733 | 20260505 16:17:38.032 | INFO | NumEvents : 1290490734056 | 20260505 16:17:39.734 | INFO | Analyzing chunk frames = [4579328,4579840) | 20260505 16:17:39.734 | INFO | MemInfo: { | 20260505 16:17:39.734 | INFO | "gpu" : " 17 GB", | 20260505 16:17:39.734 | INFO | "huge" : "", | 20260505 16:17:39.734 | INFO | "huge_shared" : " 27 GB", | 20260505 16:17:39.734 | INFO | "malloc" : " 14 GB", | 20260505 16:17:39.734 | INFO | "pinned" : " 29 GB" | 20260505 16:17:39.734 | INFO | } | 20260505 16:17:39.734 | INFO | Perf report for the last 5.14387 seconds: | 20260505 16:17:39.734 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:17:39.735 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:39.735 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:39.735 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:39.735 | INFO | ANALYSIS : 63.8%, 3.0, 99.3ms, 0 | 20260505 16:17:39.735 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:39.735 | INFO | RT_METRICS : 44.9%, 1.0, 23.6ms, 0 | 20260505 16:17:39.735 | INFO | PRE_HQ : 60.7%, 1.8, 56.4ms, 0 | 20260505 16:17:39.735 | INFO | BAZWRITER : 63.7%, 3.9, 127.8ms, 0 | 20260505 16:17:39.735 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 16:17:39.735 | INFO | Uploaded 36GiB to GPU in 2.1804s (16.5107GiB/s) | 20260505 16:17:39.735 | INFO | Downloaded 38GiB to GPU in 2.26751s (16.7585GiB/s) | 20260505 16:17:44.870 | INFO | Analyzing chunk frames = [4579840,4580352) | 20260505 16:17:44.870 | INFO | MemInfo: { | 20260505 16:17:44.870 | INFO | "gpu" : " 17 GB", | 20260505 16:17:44.870 | INFO | "huge" : "", | 20260505 16:17:44.870 | INFO | "huge_shared" : " 27 GB", | 20260505 16:17:44.870 | INFO | "malloc" : " 14 GB", | 20260505 16:17:44.870 | INFO | "pinned" : " 29 GB" | 20260505 16:17:44.870 | INFO | } | 20260505 16:17:44.870 | INFO | Perf report for the last 5.13621 seconds: | 20260505 16:17:44.870 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:17:44.870 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:44.871 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:44.871 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:44.871 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 16:17:44.871 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:44.871 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:44.871 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:17:44.871 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:44.871 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:44.871 | INFO | Uploaded 36GiB to GPU in 2.13787s (16.8392GiB/s) | 20260505 16:17:44.871 | INFO | Downloaded 34GiB to GPU in 2.02459s (16.7936GiB/s) | 20260505 16:17:50.007 | INFO | Analyzing chunk frames = [4580352,4580864) | 20260505 16:17:50.007 | INFO | MemInfo: { | 20260505 16:17:50.007 | INFO | "gpu" : " 17 GB", | 20260505 16:17:50.007 | INFO | "huge" : "", | 20260505 16:17:50.007 | INFO | "huge_shared" : " 27 GB", | 20260505 16:17:50.007 | INFO | "malloc" : " 14 GB", | 20260505 16:17:50.007 | INFO | "pinned" : " 29 GB" | 20260505 16:17:50.007 | INFO | } | 20260505 16:17:50.007 | INFO | Perf report for the last 5.13671 seconds: | 20260505 16:17:50.007 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:17:50.007 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:50.007 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:50.007 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:50.007 | INFO | ANALYSIS : 62.9%, 3.0, 97.8ms, 0 | 20260505 16:17:50.007 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:50.007 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:50.007 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:17:50.007 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:50.007 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:50.007 | INFO | Uploaded 36GiB to GPU in 2.20574s (16.321GiB/s) | 20260505 16:17:50.007 | INFO | Downloaded 34GiB to GPU in 2.0326s (16.7273GiB/s) | 20260505 16:17:55.145 | INFO | Analyzing chunk frames = [4580864,4581376) | 20260505 16:17:55.145 | INFO | MemInfo: { | 20260505 16:17:55.145 | INFO | "gpu" : " 17 GB", | 20260505 16:17:55.145 | INFO | "huge" : "", | 20260505 16:17:55.145 | INFO | "huge_shared" : " 27 GB", | 20260505 16:17:55.145 | INFO | "malloc" : " 14 GB", | 20260505 16:17:55.145 | INFO | "pinned" : " 29 GB" | 20260505 16:17:55.145 | INFO | } | 20260505 16:17:55.145 | INFO | Perf report for the last 5.13791 seconds: | 20260505 16:17:55.145 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:17:55.145 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:55.145 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:55.145 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:55.145 | INFO | ANALYSIS : 62.6%, 3.0, 97.2ms, 0 | 20260505 16:17:55.145 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:55.145 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:55.145 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:17:55.145 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:55.145 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:17:55.145 | INFO | Uploaded 36GiB to GPU in 2.18839s (16.4504GiB/s) | 20260505 16:17:55.145 | INFO | Downloaded 34GiB to GPU in 2.13201s (15.9474GiB/s) | 20260505 16:18:00.289 | INFO | Analyzing chunk frames = [4581376,4581888) | 20260505 16:18:00.289 | INFO | MemInfo: { | 20260505 16:18:00.289 | INFO | "gpu" : " 17 GB", | 20260505 16:18:00.289 | INFO | "huge" : "", | 20260505 16:18:00.289 | INFO | "huge_shared" : " 27 GB", | 20260505 16:18:00.289 | INFO | "malloc" : " 14 GB", | 20260505 16:18:00.289 | INFO | "pinned" : " 29 GB" | 20260505 16:18:00.289 | INFO | } | 20260505 16:18:00.289 | INFO | Perf report for the last 5.14426 seconds: | 20260505 16:18:00.289 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:18:00.289 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:00.289 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:00.289 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:00.289 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 16:18:00.289 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:00.289 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:00.289 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:18:00.289 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:00.289 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:00.290 | INFO | Uploaded 36GiB to GPU in 2.18615s (16.4673GiB/s) | 20260505 16:18:00.290 | INFO | Downloaded 34GiB to GPU in 2.1298s (15.9639GiB/s) | 20260505 16:18:05.435 | INFO | Analyzing chunk frames = [4581888,4582400) | 20260505 16:18:05.435 | INFO | MemInfo: { | 20260505 16:18:05.435 | INFO | "gpu" : " 17 GB", | 20260505 16:18:05.435 | INFO | "huge" : "", | 20260505 16:18:05.435 | INFO | "huge_shared" : " 27 GB", | 20260505 16:18:05.435 | INFO | "malloc" : " 14 GB", | 20260505 16:18:05.435 | INFO | "pinned" : " 29 GB" | 20260505 16:18:05.435 | INFO | } | 20260505 16:18:05.435 | INFO | Perf report for the last 5.14564 seconds: | 20260505 16:18:05.435 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:18:05.435 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:05.435 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:05.435 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:05.435 | INFO | ANALYSIS : 62.3%, 3.0, 97.4ms, 0 | 20260505 16:18:05.435 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:05.435 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:05.435 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:18:05.435 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:05.435 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:05.435 | INFO | Uploaded 36GiB to GPU in 2.14228s (16.8045GiB/s) | 20260505 16:18:05.435 | INFO | Downloaded 34GiB to GPU in 2.03819s (16.6815GiB/s) | 20260505 16:18:10.570 | INFO | Analyzing chunk frames = [4582400,4582912) | 20260505 16:18:10.570 | INFO | MemInfo: { | 20260505 16:18:10.570 | INFO | "gpu" : " 17 GB", | 20260505 16:18:10.570 | INFO | "huge" : "", | 20260505 16:18:10.570 | INFO | "huge_shared" : " 27 GB", | 20260505 16:18:10.570 | INFO | "malloc" : " 14 GB", | 20260505 16:18:10.570 | INFO | "pinned" : " 29 GB" | 20260505 16:18:10.570 | INFO | } | 20260505 16:18:10.570 | INFO | Perf report for the last 5.13552 seconds: | 20260505 16:18:10.570 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:18:10.570 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:10.570 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:10.570 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:10.571 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 16:18:10.571 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:10.571 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:10.571 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 16:18:10.571 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:10.571 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:10.571 | INFO | Uploaded 36GiB to GPU in 2.16531s (16.6258GiB/s) | 20260505 16:18:10.571 | INFO | Downloaded 34GiB to GPU in 2.18709s (15.5458GiB/s) | 20260505 16:18:15.704 | INFO | Analyzing chunk frames = [4582912,4583424) | 20260505 16:18:15.704 | INFO | MemInfo: { | 20260505 16:18:15.704 | INFO | "gpu" : " 17 GB", | 20260505 16:18:15.704 | INFO | "huge" : "", | 20260505 16:18:15.704 | INFO | "huge_shared" : " 27 GB", | 20260505 16:18:15.704 | INFO | "malloc" : " 14 GB", | 20260505 16:18:15.704 | INFO | "pinned" : " 29 GB" | 20260505 16:18:15.704 | INFO | } | 20260505 16:18:15.704 | INFO | Perf report for the last 5.1339 seconds: | 20260505 16:18:15.704 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:18:15.704 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:15.704 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:15.704 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:15.704 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 16:18:15.704 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:15.704 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:15.704 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 16:18:15.704 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:15.704 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:15.705 | INFO | Uploaded 36GiB to GPU in 2.16199s (16.6513GiB/s) | 20260505 16:18:15.705 | INFO | Downloaded 34GiB to GPU in 2.02887s (16.7581GiB/s) | 20260505 16:18:15.834 | INFO | Starting a round of Baz IO | 20260505 16:18:19.040 | INFO | Populating RTMetrics report | 20260505 16:18:19.041 | INFO | RTMetrics report populated | 20260505 16:18:19.041 | INFO | Opening tmp RTMetrics json file | 20260505 16:18:19.051 | INFO | Serializing RTMetrics to json | 20260505 16:18:19.052 | INFO | Serialization done, swapping tmp file now... | 20260505 16:18:19.053 | INFO | RTMetrics done | 20260505 16:18:19.141 | INFO | Finished a round of Baz IO: Wrote 2.178 GiB in 3.306 seconds | 20260505 16:18:19.141 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.27% / 0.00% / 77.72% | 20260505 16:18:19.141 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.374838% / 7.94729e-06% / 8.74201e-05% | 20260505 16:18:19.141 | INFO | Baz IO overall: | 20260505 16:18:19.141 | INFO | BytesWritten : 2720784896137 | 20260505 16:18:19.141 | INFO | NumEvents : 1291587175505 | 20260505 16:18:20.847 | INFO | Analyzing chunk frames = [4583424,4583936) | 20260505 16:18:20.847 | INFO | MemInfo: { | 20260505 16:18:20.847 | INFO | "gpu" : " 17 GB", | 20260505 16:18:20.847 | INFO | "huge" : "", | 20260505 16:18:20.847 | INFO | "huge_shared" : " 27 GB", | 20260505 16:18:20.847 | INFO | "malloc" : " 14 GB", | 20260505 16:18:20.847 | INFO | "pinned" : " 29 GB" | 20260505 16:18:20.847 | INFO | } | 20260505 16:18:20.847 | INFO | Perf report for the last 5.14286 seconds: | 20260505 16:18:20.847 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:18:20.847 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:20.847 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:20.847 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:20.847 | INFO | ANALYSIS : 63.8%, 3.0, 99.5ms, 0 | 20260505 16:18:20.847 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:20.847 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260505 16:18:20.847 | INFO | PRE_HQ : 60.7%, 1.8, 56.3ms, 0 | 20260505 16:18:20.847 | INFO | BAZWRITER : 63.6%, 3.9, 127.5ms, 0 | 20260505 16:18:20.847 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 16:18:20.848 | INFO | Uploaded 36GiB to GPU in 2.18918s (16.4445GiB/s) | 20260505 16:18:20.848 | INFO | Downloaded 38GiB to GPU in 2.26045s (16.8108GiB/s) | 20260505 16:18:25.984 | INFO | Analyzing chunk frames = [4583936,4584448) | 20260505 16:18:25.984 | INFO | MemInfo: { | 20260505 16:18:25.984 | INFO | "gpu" : " 17 GB", | 20260505 16:18:25.984 | INFO | "huge" : "", | 20260505 16:18:25.984 | INFO | "huge_shared" : " 27 GB", | 20260505 16:18:25.984 | INFO | "malloc" : " 14 GB", | 20260505 16:18:25.984 | INFO | "pinned" : " 29 GB" | 20260505 16:18:25.984 | INFO | } | 20260505 16:18:25.984 | INFO | Perf report for the last 5.13683 seconds: | 20260505 16:18:25.984 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:18:25.984 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:25.984 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:25.984 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:25.984 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 16:18:25.984 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:25.984 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:25.984 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:18:25.984 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:25.984 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:25.984 | INFO | Uploaded 36GiB to GPU in 2.1178s (16.9988GiB/s) | 20260505 16:18:25.984 | INFO | Downloaded 34GiB to GPU in 2.02977s (16.7507GiB/s) | 20260505 16:18:31.127 | INFO | Analyzing chunk frames = [4584448,4584960) | 20260505 16:18:31.127 | INFO | MemInfo: { | 20260505 16:18:31.127 | INFO | "gpu" : " 17 GB", | 20260505 16:18:31.127 | INFO | "huge" : "", | 20260505 16:18:31.127 | INFO | "huge_shared" : " 27 GB", | 20260505 16:18:31.127 | INFO | "malloc" : " 14 GB", | 20260505 16:18:31.127 | INFO | "pinned" : " 29 GB" | 20260505 16:18:31.128 | INFO | } | 20260505 16:18:31.128 | INFO | Perf report for the last 5.1436 seconds: | 20260505 16:18:31.128 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:18:31.128 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:31.128 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:31.128 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:31.128 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 16:18:31.128 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:31.128 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:31.128 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 16:18:31.128 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:31.128 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:31.128 | INFO | Uploaded 36GiB to GPU in 2.17421s (16.5578GiB/s) | 20260505 16:18:31.128 | INFO | Downloaded 34GiB to GPU in 2.02613s (16.7807GiB/s) | 20260505 16:18:36.261 | INFO | Analyzing chunk frames = [4584960,4585472) | 20260505 16:18:36.261 | INFO | MemInfo: { | 20260505 16:18:36.261 | INFO | "gpu" : " 17 GB", | 20260505 16:18:36.261 | INFO | "huge" : "", | 20260505 16:18:36.261 | INFO | "huge_shared" : " 27 GB", | 20260505 16:18:36.261 | INFO | "malloc" : " 14 GB", | 20260505 16:18:36.261 | INFO | "pinned" : " 29 GB" | 20260505 16:18:36.261 | INFO | } | 20260505 16:18:36.261 | INFO | Perf report for the last 5.1333 seconds: | 20260505 16:18:36.261 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:18:36.261 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:36.261 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:36.261 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:36.261 | INFO | ANALYSIS : 62.7%, 3.0, 97.3ms, 0 | 20260505 16:18:36.261 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:36.261 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:36.261 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:18:36.261 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:36.261 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:36.261 | INFO | Uploaded 36GiB to GPU in 2.17402s (16.5592GiB/s) | 20260505 16:18:36.261 | INFO | Downloaded 34GiB to GPU in 2.14245s (15.8697GiB/s) | 20260505 16:18:41.397 | INFO | Analyzing chunk frames = [4585472,4585984) | 20260505 16:18:41.397 | INFO | MemInfo: { | 20260505 16:18:41.397 | INFO | "gpu" : " 17 GB", | 20260505 16:18:41.397 | INFO | "huge" : "", | 20260505 16:18:41.397 | INFO | "huge_shared" : " 27 GB", | 20260505 16:18:41.397 | INFO | "malloc" : " 14 GB", | 20260505 16:18:41.397 | INFO | "pinned" : " 29 GB" | 20260505 16:18:41.397 | INFO | } | 20260505 16:18:41.397 | INFO | Perf report for the last 5.13623 seconds: | 20260505 16:18:41.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:18:41.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:41.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:41.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:41.397 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 16:18:41.397 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:41.397 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:41.397 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:18:41.397 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:41.397 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:41.397 | INFO | Uploaded 36GiB to GPU in 2.18775s (16.4553GiB/s) | 20260505 16:18:41.398 | INFO | Downloaded 34GiB to GPU in 2.13213s (15.9465GiB/s) | 20260505 16:18:46.547 | INFO | Analyzing chunk frames = [4585984,4586496) | 20260505 16:18:46.547 | INFO | MemInfo: { | 20260505 16:18:46.547 | INFO | "gpu" : " 17 GB", | 20260505 16:18:46.547 | INFO | "huge" : "", | 20260505 16:18:46.547 | INFO | "huge_shared" : " 27 GB", | 20260505 16:18:46.547 | INFO | "malloc" : " 14 GB", | 20260505 16:18:46.547 | INFO | "pinned" : " 29 GB" | 20260505 16:18:46.547 | INFO | } | 20260505 16:18:46.547 | INFO | Perf report for the last 5.15026 seconds: | 20260505 16:18:46.548 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:18:46.548 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:46.548 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:46.548 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:46.548 | INFO | ANALYSIS : 62.5%, 3.0, 97.8ms, 0 | 20260505 16:18:46.548 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:46.548 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:46.548 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:18:46.548 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:46.548 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:46.548 | INFO | Uploaded 36GiB to GPU in 2.16769s (16.6075GiB/s) | 20260505 16:18:46.548 | INFO | Downloaded 34GiB to GPU in 2.03964s (16.6696GiB/s) | 20260505 16:18:51.686 | INFO | Analyzing chunk frames = [4586496,4587008) | 20260505 16:18:51.686 | INFO | MemInfo: { | 20260505 16:18:51.686 | INFO | "gpu" : " 17 GB", | 20260505 16:18:51.686 | INFO | "huge" : "", | 20260505 16:18:51.686 | INFO | "huge_shared" : " 27 GB", | 20260505 16:18:51.686 | INFO | "malloc" : " 14 GB", | 20260505 16:18:51.686 | INFO | "pinned" : " 29 GB" | 20260505 16:18:51.686 | INFO | } | 20260505 16:18:51.687 | INFO | Perf report for the last 5.13902 seconds: | 20260505 16:18:51.687 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:18:51.687 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:51.687 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:51.687 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:51.687 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 16:18:51.687 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:51.687 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:51.687 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260505 16:18:51.687 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:51.687 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:51.687 | INFO | Uploaded 36GiB to GPU in 2.15799s (16.6822GiB/s) | 20260505 16:18:51.687 | INFO | Downloaded 34GiB to GPU in 2.1741s (15.6386GiB/s) | 20260505 16:18:56.820 | INFO | Analyzing chunk frames = [4587008,4587520) | 20260505 16:18:56.820 | INFO | MemInfo: { | 20260505 16:18:56.820 | INFO | "gpu" : " 17 GB", | 20260505 16:18:56.820 | INFO | "huge" : "", | 20260505 16:18:56.820 | INFO | "huge_shared" : " 27 GB", | 20260505 16:18:56.820 | INFO | "malloc" : " 14 GB", | 20260505 16:18:56.820 | INFO | "pinned" : " 29 GB" | 20260505 16:18:56.820 | INFO | } | 20260505 16:18:56.820 | INFO | Perf report for the last 5.13385 seconds: | 20260505 16:18:56.820 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:18:56.820 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:56.820 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:56.820 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:56.820 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 16:18:56.820 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:56.820 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:56.821 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 16:18:56.821 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:56.821 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:18:56.821 | INFO | Uploaded 36GiB to GPU in 2.16445s (16.6324GiB/s) | 20260505 16:18:56.821 | INFO | Downloaded 34GiB to GPU in 2.02243s (16.8115GiB/s) | 20260505 16:18:56.821 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 16:18:56.821 | INFO | Total mem usage: 15.3 GiB | 20260505 16:18:56.821 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 16:18:56.821 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.87%) | 20260505 16:18:56.821 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 16:18:56.821 | INFO | High Water Breakdown: | 20260505 16:18:56.821 | INFO | 61% : 8.67 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 16:18:56.821 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 16:18:56.821 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 16:18:56.821 | INFO | ------------------------------------------ | 20260505 16:18:56.821 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:18:56.821 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 16:18:56.821 | INFO | Total mem usage: 26.5 GiB | 20260505 16:18:56.821 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 16:18:56.821 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 16:18:56.821 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 16:18:56.821 | INFO | High Water Breakdown: | 20260505 16:18:56.821 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:18:56.821 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:18:56.821 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:18:56.821 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:18:56.821 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:18:56.821 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:18:56.821 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 16:18:56.821 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 16:18:56.821 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:18:56.821 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 16:18:56.821 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 16:18:56.821 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 16:18:56.821 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 16:18:56.821 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 16:18:56.821 | INFO | ------------------------------------------ | 20260505 16:18:56.821 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:18:56.821 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 16:18:56.821 | INFO | Total mem usage: 25.6 GiB | 20260505 16:18:56.821 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 16:18:56.821 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 16:18:56.821 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 16:18:56.821 | INFO | High Water Breakdown: | 20260505 16:18:56.821 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 16:18:56.821 | INFO | ------------------------------------------ | 20260505 16:18:56.821 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:18:56.821 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 16:18:56.821 | INFO | Total mem usage: 15.7 GiB | 20260505 16:18:56.821 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 16:18:56.821 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 16:18:56.821 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 16:18:56.821 | INFO | High Water Breakdown: | 20260505 16:18:56.821 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 16:18:56.821 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 16:18:56.821 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 16:18:56.821 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 16:18:56.821 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 16:18:56.821 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 16:18:56.821 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 16:18:56.822 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 16:18:56.822 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 16:18:56.822 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:18:56.822 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 16:18:56.822 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:18:56.822 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 16:18:56.822 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 16:18:56.822 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 16:18:56.822 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 16:18:56.822 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:18:56.822 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 16:18:56.822 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:18:56.822 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:18:56.822 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:18:56.822 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 16:18:56.822 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 16:18:56.822 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:18:56.822 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 16:18:56.822 | INFO | ------------------------------------------ | 20260505 16:18:56.822 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:18:56.822 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 16:18:56.822 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:18:56.822 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:18:56.822 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.55e+01, 1.35e+01 / 2.27e+00 | 20260505 16:18:56.822 | INFO | PerfReport: 0.29%, AggregateStats -- 9.27e-02 / 1.03e-01, 1.05e-02 / 6.79e-03 | 20260505 16:18:56.822 | INFO | PerfReport: 11.79%, Baseline -- 3.74e+00 / 4.51e+00, 5.78e-01 / 1.59e-01 | 20260505 16:18:56.822 | INFO | PerfReport: 3.29%, Histogram -- 1.04e+00 / 1.51e+00, 1.93e-01 / 9.28e-02 | 20260505 16:18:56.822 | INFO | PerfReport: 15.98%, DME -- 4.05e+01 / 4.53e+01, 4.59e+00 / 2.24e+00 | 20260505 16:18:56.822 | INFO | PerfReport: 56.67%, FrameLabeling -- 1.80e+01 / 1.92e+01, 1.43e+00 / 1.07e+00 | 20260505 16:18:56.822 | INFO | PerfReport: 7.36%, PulseAccumulating -- 2.33e+00 / 2.92e+00, 4.50e-01 / 1.68e-01 | 20260505 16:18:56.822 | INFO | PerfReport: 3.42%, Metrics -- 1.08e+00 / 1.40e+00, 3.00e-01 / 1.29e-01 | 20260505 16:18:56.822 | INFO | PerfReport: 1.19%, PulseSerializing -- 3.77e-01 / 4.31e-01, 6.75e-02 / 2.88e-02 | 20260505 16:18:56.822 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 16:18:56.822 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:18:56.822 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:18:56.822 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.57e+01 / 0.00e+00 | 20260505 16:18:56.822 | INFO | PerfReport: 22.54%, Upload -- 2.26e+01 / 0.00e+00, 7.89e+00 / 0.00e+00 | 20260505 16:18:56.822 | INFO | PerfReport: 21.87%, Download -- 2.19e+01 / 0.00e+00, 7.91e+00 / 0.00e+00 | 20260505 16:18:56.822 | INFO | PerfReport: 31.61%, Compute -- 3.17e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 16:18:56.822 | INFO | PerfReport: 2.25%, LockUpload -- 2.25e+00 / 0.00e+00, 7.92e+00 / 0.00e+00 | 20260505 16:18:56.822 | INFO | PerfReport: 1.45%, LockDownload -- 1.46e+00 / 0.00e+00, 3.01e+00 / 0.00e+00 | 20260505 16:18:56.822 | INFO | PerfReport: 20.26%, LockCompute -- 2.03e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 16:18:56.822 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 2.10e-02 / 0.00e+00 | 20260505 16:18:56.954 | INFO | Starting a round of Baz IO | 20260505 16:19:00.166 | INFO | Populating RTMetrics report | 20260505 16:19:00.167 | INFO | RTMetrics report populated | 20260505 16:19:00.167 | INFO | Opening tmp RTMetrics json file | 20260505 16:19:00.177 | INFO | Serializing RTMetrics to json | 20260505 16:19:00.179 | INFO | Serialization done, swapping tmp file now... | 20260505 16:19:00.182 | INFO | RTMetrics done | 20260505 16:19:00.265 | INFO | Finished a round of Baz IO: Wrote 2.178 GiB in 3.311 seconds | 20260505 16:19:00.266 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.27% / 0.00% / 77.73% | 20260505 16:19:00.266 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.374913% / 0% / 9.13938e-05% | 20260505 16:19:00.266 | INFO | Baz IO overall: | 20260505 16:19:00.266 | INFO | BytesWritten : 2723123666044 | 20260505 16:19:00.266 | INFO | NumEvents : 1292683844048 | 20260505 16:19:01.951 | INFO | Analyzing chunk frames = [4587520,4588032) | 20260505 16:19:01.951 | INFO | MemInfo: { | 20260505 16:19:01.951 | INFO | "gpu" : " 17 GB", | 20260505 16:19:01.951 | INFO | "huge" : "", | 20260505 16:19:01.951 | INFO | "huge_shared" : " 27 GB", | 20260505 16:19:01.951 | INFO | "malloc" : " 14 GB", | 20260505 16:19:01.951 | INFO | "pinned" : " 29 GB" | 20260505 16:19:01.951 | INFO | } | 20260505 16:19:01.951 | INFO | Perf report for the last 5.13038 seconds: | 20260505 16:19:01.951 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:19:01.951 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:01.951 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:01.951 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:01.951 | INFO | ANALYSIS : 64.1%, 3.0, 99.7ms, 0 | 20260505 16:19:01.951 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:01.951 | INFO | RT_METRICS : 45.0%, 1.0, 23.5ms, 0 | 20260505 16:19:01.951 | INFO | PRE_HQ : 60.8%, 1.8, 56.7ms, 0 | 20260505 16:19:01.951 | INFO | BAZWRITER : 63.9%, 3.9, 129.0ms, 0 | 20260505 16:19:01.951 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 16:19:01.951 | INFO | Uploaded 36GiB to GPU in 2.20734s (16.3092GiB/s) | 20260505 16:19:01.951 | INFO | Downloaded 38GiB to GPU in 2.26256s (16.7951GiB/s) | 20260505 16:19:07.102 | INFO | Analyzing chunk frames = [4588032,4588544) | 20260505 16:19:07.102 | INFO | MemInfo: { | 20260505 16:19:07.102 | INFO | "gpu" : " 17 GB", | 20260505 16:19:07.102 | INFO | "huge" : "", | 20260505 16:19:07.102 | INFO | "huge_shared" : " 27 GB", | 20260505 16:19:07.102 | INFO | "malloc" : " 14 GB", | 20260505 16:19:07.102 | INFO | "pinned" : " 29 GB" | 20260505 16:19:07.102 | INFO | } | 20260505 16:19:07.102 | INFO | Perf report for the last 5.15109 seconds: | 20260505 16:19:07.102 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:19:07.102 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:07.102 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:07.102 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:07.102 | INFO | ANALYSIS : 63.1%, 3.0, 98.7ms, 0 | 20260505 16:19:07.102 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:07.102 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:07.102 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:19:07.102 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:07.102 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:07.102 | INFO | Uploaded 36GiB to GPU in 2.19039s (16.4354GiB/s) | 20260505 16:19:07.102 | INFO | Downloaded 34GiB to GPU in 2.03414s (16.7147GiB/s) | 20260505 16:19:12.240 | INFO | Analyzing chunk frames = [4588544,4589056) | 20260505 16:19:12.240 | INFO | MemInfo: { | 20260505 16:19:12.240 | INFO | "gpu" : " 17 GB", | 20260505 16:19:12.240 | INFO | "huge" : "", | 20260505 16:19:12.240 | INFO | "huge_shared" : " 27 GB", | 20260505 16:19:12.240 | INFO | "malloc" : " 14 GB", | 20260505 16:19:12.240 | INFO | "pinned" : " 29 GB" | 20260505 16:19:12.240 | INFO | } | 20260505 16:19:12.240 | INFO | Perf report for the last 5.13804 seconds: | 20260505 16:19:12.240 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:19:12.240 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:12.240 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:12.240 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:12.240 | INFO | ANALYSIS : 62.8%, 3.0, 97.8ms, 0 | 20260505 16:19:12.240 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:12.240 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:12.240 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:19:12.240 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:12.240 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:12.240 | INFO | Uploaded 36GiB to GPU in 2.20507s (16.326GiB/s) | 20260505 16:19:12.240 | INFO | Downloaded 34GiB to GPU in 2.04565s (16.6206GiB/s) | 20260505 16:19:17.384 | INFO | Analyzing chunk frames = [4589056,4589568) | 20260505 16:19:17.384 | INFO | MemInfo: { | 20260505 16:19:17.384 | INFO | "gpu" : " 17 GB", | 20260505 16:19:17.384 | INFO | "huge" : "", | 20260505 16:19:17.384 | INFO | "huge_shared" : " 27 GB", | 20260505 16:19:17.384 | INFO | "malloc" : " 14 GB", | 20260505 16:19:17.384 | INFO | "pinned" : " 29 GB" | 20260505 16:19:17.384 | INFO | } | 20260505 16:19:17.384 | INFO | Perf report for the last 5.14437 seconds: | 20260505 16:19:17.384 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:19:17.384 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:17.384 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:17.384 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:17.384 | INFO | ANALYSIS : 62.5%, 3.0, 97.2ms, 0 | 20260505 16:19:17.385 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:17.385 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:17.385 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:19:17.385 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:17.385 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:17.385 | INFO | Uploaded 36GiB to GPU in 2.18009s (16.5131GiB/s) | 20260505 16:19:17.385 | INFO | Downloaded 34GiB to GPU in 2.14117s (15.8792GiB/s) | 20260505 16:19:22.518 | INFO | Analyzing chunk frames = [4589568,4590080) | 20260505 16:19:22.518 | INFO | MemInfo: { | 20260505 16:19:22.518 | INFO | "gpu" : " 17 GB", | 20260505 16:19:22.518 | INFO | "huge" : "", | 20260505 16:19:22.518 | INFO | "huge_shared" : " 27 GB", | 20260505 16:19:22.518 | INFO | "malloc" : " 14 GB", | 20260505 16:19:22.518 | INFO | "pinned" : " 29 GB" | 20260505 16:19:22.518 | INFO | } | 20260505 16:19:22.518 | INFO | Perf report for the last 5.13404 seconds: | 20260505 16:19:22.519 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:19:22.519 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:22.519 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:22.519 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:22.519 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 16:19:22.519 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:22.519 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:22.519 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:19:22.519 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:22.519 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:22.519 | INFO | Uploaded 36GiB to GPU in 2.15827s (16.68GiB/s) | 20260505 16:19:22.519 | INFO | Downloaded 34GiB to GPU in 2.05728s (16.5267GiB/s) | 20260505 16:19:27.658 | INFO | Analyzing chunk frames = [4590080,4590592) | 20260505 16:19:27.658 | INFO | MemInfo: { | 20260505 16:19:27.658 | INFO | "gpu" : " 17 GB", | 20260505 16:19:27.658 | INFO | "huge" : "", | 20260505 16:19:27.658 | INFO | "huge_shared" : " 27 GB", | 20260505 16:19:27.658 | INFO | "malloc" : " 14 GB", | 20260505 16:19:27.658 | INFO | "pinned" : " 29 GB" | 20260505 16:19:27.658 | INFO | } | 20260505 16:19:27.658 | INFO | Perf report for the last 5.13923 seconds: | 20260505 16:19:27.658 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:19:27.658 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:27.658 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:27.658 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:27.658 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260505 16:19:27.658 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:27.658 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:27.658 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 16:19:27.658 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:27.658 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:27.658 | INFO | Uploaded 36GiB to GPU in 2.14123s (16.8128GiB/s) | 20260505 16:19:27.658 | INFO | Downloaded 34GiB to GPU in 2.0397s (16.6691GiB/s) | 20260505 16:19:32.802 | INFO | Analyzing chunk frames = [4590592,4591104) | 20260505 16:19:32.802 | INFO | MemInfo: { | 20260505 16:19:32.802 | INFO | "gpu" : " 17 GB", | 20260505 16:19:32.802 | INFO | "huge" : "", | 20260505 16:19:32.802 | INFO | "huge_shared" : " 27 GB", | 20260505 16:19:32.802 | INFO | "malloc" : " 14 GB", | 20260505 16:19:32.802 | INFO | "pinned" : " 29 GB" | 20260505 16:19:32.802 | INFO | } | 20260505 16:19:32.802 | INFO | Perf report for the last 5.144 seconds: | 20260505 16:19:32.802 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:19:32.802 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:32.802 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:32.802 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:32.802 | INFO | ANALYSIS : 63.3%, 3.0, 99.0ms, 0 | 20260505 16:19:32.802 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:32.802 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:32.802 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 16:19:32.802 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:32.802 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:32.802 | INFO | Uploaded 36GiB to GPU in 2.1445s (16.7872GiB/s) | 20260505 16:19:32.802 | INFO | Downloaded 34GiB to GPU in 2.16989s (15.669GiB/s) | 20260505 16:19:37.937 | INFO | Analyzing chunk frames = [4591104,4591616) | 20260505 16:19:37.937 | INFO | MemInfo: { | 20260505 16:19:37.937 | INFO | "gpu" : " 17 GB", | 20260505 16:19:37.937 | INFO | "huge" : "", | 20260505 16:19:37.937 | INFO | "huge_shared" : " 27 GB", | 20260505 16:19:37.937 | INFO | "malloc" : " 14 GB", | 20260505 16:19:37.937 | INFO | "pinned" : " 29 GB" | 20260505 16:19:37.937 | INFO | } | 20260505 16:19:37.937 | INFO | Perf report for the last 5.13514 seconds: | 20260505 16:19:37.937 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:19:37.937 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:37.937 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:37.937 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:37.937 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 16:19:37.937 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:37.937 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:37.937 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 16:19:37.937 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:37.937 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:37.937 | INFO | Uploaded 36GiB to GPU in 2.17079s (16.5838GiB/s) | 20260505 16:19:37.937 | INFO | Downloaded 34GiB to GPU in 2.02142s (16.8199GiB/s) | 20260505 16:19:38.070 | INFO | Starting a round of Baz IO | 20260505 16:19:41.276 | INFO | Populating RTMetrics report | 20260505 16:19:41.277 | INFO | RTMetrics report populated | 20260505 16:19:41.277 | INFO | Opening tmp RTMetrics json file | 20260505 16:19:41.287 | INFO | Serializing RTMetrics to json | 20260505 16:19:41.289 | INFO | Serialization done, swapping tmp file now... | 20260505 16:19:41.290 | INFO | RTMetrics done | 20260505 16:19:41.373 | INFO | Finished a round of Baz IO: Wrote 2.178 GiB in 3.303 seconds | 20260505 16:19:41.374 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.27% / 0.00% / 77.73% | 20260505 16:19:41.374 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.374031% / 0% / 7.54992e-05% | 20260505 16:19:41.374 | INFO | Baz IO overall: | 20260505 16:19:41.374 | INFO | BytesWritten : 2725462270752 | 20260505 16:19:41.374 | INFO | NumEvents : 1293780522295 | 20260505 16:19:43.076 | INFO | Analyzing chunk frames = [4591616,4592128) | 20260505 16:19:43.076 | INFO | MemInfo: { | 20260505 16:19:43.076 | INFO | "gpu" : " 17 GB", | 20260505 16:19:43.076 | INFO | "huge" : "", | 20260505 16:19:43.076 | INFO | "huge_shared" : " 27 GB", | 20260505 16:19:43.076 | INFO | "malloc" : " 14 GB", | 20260505 16:19:43.076 | INFO | "pinned" : " 29 GB" | 20260505 16:19:43.076 | INFO | } | 20260505 16:19:43.076 | INFO | Perf report for the last 5.13947 seconds: | 20260505 16:19:43.076 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:19:43.076 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:43.076 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:43.076 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:43.076 | INFO | ANALYSIS : 63.9%, 3.0, 99.5ms, 0 | 20260505 16:19:43.076 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:43.076 | INFO | RT_METRICS : 44.9%, 1.0, 23.5ms, 0 | 20260505 16:19:43.076 | INFO | PRE_HQ : 60.6%, 1.8, 56.5ms, 0 | 20260505 16:19:43.076 | INFO | BAZWRITER : 63.5%, 3.9, 127.8ms, 0 | 20260505 16:19:43.076 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 16:19:43.077 | INFO | Uploaded 36GiB to GPU in 2.18723s (16.4592GiB/s) | 20260505 16:19:43.077 | INFO | Downloaded 38GiB to GPU in 2.26977s (16.7418GiB/s) | 20260505 16:19:48.210 | INFO | Analyzing chunk frames = [4592128,4592640) | 20260505 16:19:48.210 | INFO | MemInfo: { | 20260505 16:19:48.210 | INFO | "gpu" : " 17 GB", | 20260505 16:19:48.210 | INFO | "huge" : "", | 20260505 16:19:48.210 | INFO | "huge_shared" : " 27 GB", | 20260505 16:19:48.210 | INFO | "malloc" : " 14 GB", | 20260505 16:19:48.210 | INFO | "pinned" : " 29 GB" | 20260505 16:19:48.210 | INFO | } | 20260505 16:19:48.210 | INFO | Perf report for the last 5.13344 seconds: | 20260505 16:19:48.210 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:19:48.210 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:48.210 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:48.210 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:48.210 | INFO | ANALYSIS : 63.4%, 3.0, 98.8ms, 0 | 20260505 16:19:48.210 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:48.210 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:48.210 | INFO | PRE_HQ : 28.1%, 1.0, 14.5ms, 0 | 20260505 16:19:48.210 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:48.210 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:48.210 | INFO | Uploaded 36GiB to GPU in 2.14526s (16.7812GiB/s) | 20260505 16:19:48.210 | INFO | Downloaded 34GiB to GPU in 2.03765s (16.6859GiB/s) | 20260505 16:19:53.355 | INFO | Analyzing chunk frames = [4592640,4593152) | 20260505 16:19:53.355 | INFO | MemInfo: { | 20260505 16:19:53.355 | INFO | "gpu" : " 17 GB", | 20260505 16:19:53.355 | INFO | "huge" : "", | 20260505 16:19:53.355 | INFO | "huge_shared" : " 27 GB", | 20260505 16:19:53.355 | INFO | "malloc" : " 14 GB", | 20260505 16:19:53.355 | INFO | "pinned" : " 29 GB" | 20260505 16:19:53.355 | INFO | } | 20260505 16:19:53.356 | INFO | Perf report for the last 5.14577 seconds: | 20260505 16:19:53.356 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:19:53.356 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:53.356 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:53.356 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:53.356 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260505 16:19:53.356 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:53.356 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:53.356 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:19:53.356 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:53.356 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:53.356 | INFO | Uploaded 36GiB to GPU in 2.19117s (16.4296GiB/s) | 20260505 16:19:53.356 | INFO | Downloaded 34GiB to GPU in 2.05923s (16.511GiB/s) | 20260505 16:19:58.499 | INFO | Analyzing chunk frames = [4593152,4593664) | 20260505 16:19:58.499 | INFO | MemInfo: { | 20260505 16:19:58.499 | INFO | "gpu" : " 17 GB", | 20260505 16:19:58.499 | INFO | "huge" : "", | 20260505 16:19:58.500 | INFO | "huge_shared" : " 27 GB", | 20260505 16:19:58.500 | INFO | "malloc" : " 14 GB", | 20260505 16:19:58.500 | INFO | "pinned" : " 29 GB" | 20260505 16:19:58.500 | INFO | } | 20260505 16:19:58.500 | INFO | Perf report for the last 5.14406 seconds: | 20260505 16:19:58.500 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:19:58.500 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:58.500 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:58.500 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:58.500 | INFO | ANALYSIS : 62.5%, 3.0, 97.1ms, 0 | 20260505 16:19:58.500 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:58.500 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:58.500 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 16:19:58.500 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:58.500 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:19:58.500 | INFO | Uploaded 36GiB to GPU in 2.17534s (16.5491GiB/s) | 20260505 16:19:58.500 | INFO | Downloaded 34GiB to GPU in 2.14139s (15.8775GiB/s) | 20260505 16:20:03.635 | INFO | Analyzing chunk frames = [4593664,4594176) | 20260505 16:20:03.635 | INFO | MemInfo: { | 20260505 16:20:03.635 | INFO | "gpu" : " 17 GB", | 20260505 16:20:03.635 | INFO | "huge" : "", | 20260505 16:20:03.635 | INFO | "huge_shared" : " 27 GB", | 20260505 16:20:03.635 | INFO | "malloc" : " 14 GB", | 20260505 16:20:03.635 | INFO | "pinned" : " 29 GB" | 20260505 16:20:03.635 | INFO | } | 20260505 16:20:03.635 | INFO | Perf report for the last 5.13508 seconds: | 20260505 16:20:03.635 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:20:03.635 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:03.635 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:03.635 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:03.635 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 16:20:03.635 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:03.635 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:03.635 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:20:03.635 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:03.635 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:03.635 | INFO | Uploaded 36GiB to GPU in 2.14878s (16.7537GiB/s) | 20260505 16:20:03.635 | INFO | Downloaded 34GiB to GPU in 2.06102s (16.4967GiB/s) | 20260505 16:20:08.764 | INFO | Analyzing chunk frames = [4594176,4594688) | 20260505 16:20:08.764 | INFO | MemInfo: { | 20260505 16:20:08.764 | INFO | "gpu" : " 17 GB", | 20260505 16:20:08.764 | INFO | "huge" : "", | 20260505 16:20:08.764 | INFO | "huge_shared" : " 27 GB", | 20260505 16:20:08.764 | INFO | "malloc" : " 14 GB", | 20260505 16:20:08.764 | INFO | "pinned" : " 29 GB" | 20260505 16:20:08.764 | INFO | } | 20260505 16:20:08.764 | INFO | Perf report for the last 5.12944 seconds: | 20260505 16:20:08.764 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:20:08.764 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:08.764 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:08.764 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:08.764 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 16:20:08.764 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:08.764 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:08.764 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 16:20:08.764 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:08.764 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:08.765 | INFO | Uploaded 36GiB to GPU in 2.13347s (16.874GiB/s) | 20260505 16:20:08.765 | INFO | Downloaded 34GiB to GPU in 2.03827s (16.6808GiB/s) | 20260505 16:20:13.905 | INFO | Analyzing chunk frames = [4594688,4595200) | 20260505 16:20:13.905 | INFO | MemInfo: { | 20260505 16:20:13.905 | INFO | "gpu" : " 17 GB", | 20260505 16:20:13.905 | INFO | "huge" : "", | 20260505 16:20:13.905 | INFO | "huge_shared" : " 27 GB", | 20260505 16:20:13.905 | INFO | "malloc" : " 14 GB", | 20260505 16:20:13.905 | INFO | "pinned" : " 29 GB" | 20260505 16:20:13.905 | INFO | } | 20260505 16:20:13.905 | INFO | Perf report for the last 5.14114 seconds: | 20260505 16:20:13.905 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:20:13.905 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:13.905 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:13.905 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:13.905 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 16:20:13.905 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:13.905 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:13.905 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 16:20:13.905 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:13.905 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:13.905 | INFO | Uploaded 36GiB to GPU in 2.11853s (16.9929GiB/s) | 20260505 16:20:13.905 | INFO | Downloaded 34GiB to GPU in 2.18135s (15.5866GiB/s) | 20260505 16:20:19.053 | INFO | Analyzing chunk frames = [4595200,4595712) | 20260505 16:20:19.053 | INFO | MemInfo: { | 20260505 16:20:19.053 | INFO | "gpu" : " 17 GB", | 20260505 16:20:19.053 | INFO | "huge" : "", | 20260505 16:20:19.053 | INFO | "huge_shared" : " 27 GB", | 20260505 16:20:19.053 | INFO | "malloc" : " 14 GB", | 20260505 16:20:19.053 | INFO | "pinned" : " 29 GB" | 20260505 16:20:19.053 | INFO | } | 20260505 16:20:19.053 | INFO | Perf report for the last 5.14757 seconds: | 20260505 16:20:19.053 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:20:19.053 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:19.053 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:19.053 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:19.053 | INFO | ANALYSIS : 63.5%, 3.0, 99.3ms, 0 | 20260505 16:20:19.053 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:19.053 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:19.053 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:20:19.053 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:19.053 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:19.053 | INFO | Uploaded 36GiB to GPU in 2.22441s (16.1841GiB/s) | 20260505 16:20:19.053 | INFO | Downloaded 34GiB to GPU in 2.099s (16.1982GiB/s) | 20260505 16:20:19.053 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 16:20:19.053 | INFO | Total mem usage: 15.3 GiB | 20260505 16:20:19.053 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 16:20:19.053 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.88%) | 20260505 16:20:19.053 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 16:20:19.053 | INFO | High Water Breakdown: | 20260505 16:20:19.053 | INFO | 61% : 8.67 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 16:20:19.053 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 16:20:19.053 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 16:20:19.053 | INFO | ------------------------------------------ | 20260505 16:20:19.054 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:20:19.054 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 16:20:19.054 | INFO | Total mem usage: 26.5 GiB | 20260505 16:20:19.054 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 16:20:19.054 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 16:20:19.054 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 16:20:19.054 | INFO | High Water Breakdown: | 20260505 16:20:19.054 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:20:19.054 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:20:19.054 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:20:19.054 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:20:19.054 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:20:19.054 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:20:19.054 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 16:20:19.054 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 16:20:19.054 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:20:19.054 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 16:20:19.054 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 16:20:19.054 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 16:20:19.054 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 16:20:19.054 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 16:20:19.054 | INFO | ------------------------------------------ | 20260505 16:20:19.054 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:20:19.054 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 16:20:19.054 | INFO | Total mem usage: 25.6 GiB | 20260505 16:20:19.054 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 16:20:19.054 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 16:20:19.054 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 16:20:19.054 | INFO | High Water Breakdown: | 20260505 16:20:19.054 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 16:20:19.054 | INFO | ------------------------------------------ | 20260505 16:20:19.054 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:20:19.054 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 16:20:19.054 | INFO | Total mem usage: 15.7 GiB | 20260505 16:20:19.054 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 16:20:19.054 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 16:20:19.054 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 16:20:19.054 | INFO | High Water Breakdown: | 20260505 16:20:19.054 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 16:20:19.054 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 16:20:19.054 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 16:20:19.054 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 16:20:19.054 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 16:20:19.054 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 16:20:19.054 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 16:20:19.054 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 16:20:19.054 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 16:20:19.054 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:20:19.054 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 16:20:19.054 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:20:19.054 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 16:20:19.054 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 16:20:19.054 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 16:20:19.054 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 16:20:19.054 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:20:19.054 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 16:20:19.054 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:20:19.054 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:20:19.054 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:20:19.054 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 16:20:19.054 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 16:20:19.054 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:20:19.054 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 16:20:19.054 | INFO | ------------------------------------------ | 20260505 16:20:19.054 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:20:19.054 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 16:20:19.054 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:20:19.054 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:20:19.054 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.52e+01, 1.37e+01 / 2.14e+00 | 20260505 16:20:19.054 | INFO | PerfReport: 0.29%, AggregateStats -- 9.30e-02 / 1.01e-01, 1.14e-02 / 3.53e-03 | 20260505 16:20:19.054 | INFO | PerfReport: 11.78%, Baseline -- 3.73e+00 / 4.50e+00, 5.69e-01 / 1.55e-01 | 20260505 16:20:19.054 | INFO | PerfReport: 3.29%, Histogram -- 1.04e+00 / 1.52e+00, 1.95e-01 / 9.08e-02 | 20260505 16:20:19.054 | INFO | PerfReport: 15.96%, DME -- 4.05e+01 / 4.51e+01, 4.34e+00 / 2.12e+00 | 20260505 16:20:19.054 | INFO | PerfReport: 56.67%, FrameLabeling -- 1.80e+01 / 1.92e+01, 1.43e+00 / 1.24e+00 | 20260505 16:20:19.054 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.34e+00 / 2.92e+00, 4.50e-01 / 1.68e-01 | 20260505 16:20:19.054 | INFO | PerfReport: 3.42%, Metrics -- 1.08e+00 / 1.42e+00, 2.99e-01 / 1.31e-01 | 20260505 16:20:19.054 | INFO | PerfReport: 1.19%, PulseSerializing -- 3.78e-01 / 4.36e-01, 6.78e-02 / 3.30e-02 | 20260505 16:20:19.054 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 16:20:19.054 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:20:19.054 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:20:19.054 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260505 16:20:19.054 | INFO | PerfReport: 22.54%, Upload -- 2.26e+01 / 0.00e+00, 7.94e+00 / 0.00e+00 | 20260505 16:20:19.054 | INFO | PerfReport: 21.85%, Download -- 2.19e+01 / 0.00e+00, 7.89e+00 / 0.00e+00 | 20260505 16:20:19.054 | INFO | PerfReport: 31.59%, Compute -- 3.17e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 16:20:19.054 | INFO | PerfReport: 2.26%, LockUpload -- 2.27e+00 / 0.00e+00, 7.91e+00 / 0.00e+00 | 20260505 16:20:19.054 | INFO | PerfReport: 1.45%, LockDownload -- 1.45e+00 / 0.00e+00, 2.95e+00 / 0.00e+00 | 20260505 16:20:19.054 | INFO | PerfReport: 20.31%, LockCompute -- 2.04e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260505 16:20:19.054 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.79e-02 / 0.00e+00 | 20260505 16:20:19.192 | INFO | Starting a round of Baz IO | 20260505 16:20:22.376 | INFO | Populating RTMetrics report | 20260505 16:20:22.377 | INFO | RTMetrics report populated | 20260505 16:20:22.377 | INFO | Opening tmp RTMetrics json file | 20260505 16:20:22.387 | INFO | Serializing RTMetrics to json | 20260505 16:20:22.390 | INFO | Serialization done, swapping tmp file now... | 20260505 16:20:22.393 | INFO | RTMetrics done | 20260505 16:20:22.470 | INFO | Finished a round of Baz IO: Wrote 2.177 GiB in 3.277 seconds | 20260505 16:20:22.470 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.27% / 0.00% / 77.73% | 20260505 16:20:22.470 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.374305% / 3.97364e-06% / 7.54992e-05% | 20260505 16:20:22.470 | INFO | Baz IO overall: | 20260505 16:20:22.470 | INFO | BytesWritten : 2727799800414 | 20260505 16:20:22.470 | INFO | NumEvents : 1294877204030 | 20260505 16:20:24.197 | INFO | Analyzing chunk frames = [4595712,4596224) | 20260505 16:20:24.197 | INFO | MemInfo: { | 20260505 16:20:24.197 | INFO | "gpu" : " 17 GB", | 20260505 16:20:24.197 | INFO | "huge" : "", | 20260505 16:20:24.197 | INFO | "huge_shared" : " 27 GB", | 20260505 16:20:24.197 | INFO | "malloc" : " 14 GB", | 20260505 16:20:24.197 | INFO | "pinned" : " 29 GB" | 20260505 16:20:24.197 | INFO | } | 20260505 16:20:24.198 | INFO | Perf report for the last 5.1447 seconds: | 20260505 16:20:24.198 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:20:24.198 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:24.198 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:24.198 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:24.198 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 16:20:24.198 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:24.198 | INFO | RT_METRICS : 44.9%, 1.0, 23.6ms, 0 | 20260505 16:20:24.198 | INFO | PRE_HQ : 60.2%, 1.8, 56.5ms, 0 | 20260505 16:20:24.198 | INFO | BAZWRITER : 63.2%, 4.0, 128.5ms, 0 | 20260505 16:20:24.198 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 16:20:24.198 | INFO | Uploaded 36GiB to GPU in 2.14291s (16.7996GiB/s) | 20260505 16:20:24.198 | INFO | Downloaded 38GiB to GPU in 2.23718s (16.9856GiB/s) | 20260505 16:20:29.337 | INFO | Analyzing chunk frames = [4596224,4596736) | 20260505 16:20:29.337 | INFO | MemInfo: { | 20260505 16:20:29.337 | INFO | "gpu" : " 17 GB", | 20260505 16:20:29.337 | INFO | "huge" : "", | 20260505 16:20:29.337 | INFO | "huge_shared" : " 27 GB", | 20260505 16:20:29.337 | INFO | "malloc" : " 14 GB", | 20260505 16:20:29.337 | INFO | "pinned" : " 29 GB" | 20260505 16:20:29.337 | INFO | } | 20260505 16:20:29.337 | INFO | Perf report for the last 5.13984 seconds: | 20260505 16:20:29.337 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:20:29.337 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:29.337 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:29.337 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:29.337 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 16:20:29.337 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:29.338 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:29.338 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:20:29.338 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:29.338 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:29.338 | INFO | Uploaded 36GiB to GPU in 2.12721s (16.9236GiB/s) | 20260505 16:20:29.338 | INFO | Downloaded 34GiB to GPU in 2.02141s (16.8199GiB/s) | 20260505 16:20:34.466 | INFO | Analyzing chunk frames = [4596736,4597248) | 20260505 16:20:34.466 | INFO | MemInfo: { | 20260505 16:20:34.466 | INFO | "gpu" : " 17 GB", | 20260505 16:20:34.466 | INFO | "huge" : "", | 20260505 16:20:34.466 | INFO | "huge_shared" : " 27 GB", | 20260505 16:20:34.466 | INFO | "malloc" : " 14 GB", | 20260505 16:20:34.466 | INFO | "pinned" : " 29 GB" | 20260505 16:20:34.466 | INFO | } | 20260505 16:20:34.466 | INFO | Perf report for the last 5.12875 seconds: | 20260505 16:20:34.466 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:20:34.466 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:34.466 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:34.466 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:34.466 | INFO | ANALYSIS : 63.0%, 3.0, 97.8ms, 0 | 20260505 16:20:34.466 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:34.466 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:34.466 | INFO | PRE_HQ : 28.1%, 1.0, 14.5ms, 0 | 20260505 16:20:34.466 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:34.466 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:34.466 | INFO | Uploaded 36GiB to GPU in 2.20204s (16.3485GiB/s) | 20260505 16:20:34.467 | INFO | Downloaded 34GiB to GPU in 2.0454s (16.6226GiB/s) | 20260505 16:20:39.615 | INFO | Analyzing chunk frames = [4597248,4597760) | 20260505 16:20:39.615 | INFO | MemInfo: { | 20260505 16:20:39.615 | INFO | "gpu" : " 17 GB", | 20260505 16:20:39.615 | INFO | "huge" : "", | 20260505 16:20:39.615 | INFO | "huge_shared" : " 27 GB", | 20260505 16:20:39.615 | INFO | "malloc" : " 14 GB", | 20260505 16:20:39.615 | INFO | "pinned" : " 29 GB" | 20260505 16:20:39.615 | INFO | } | 20260505 16:20:39.615 | INFO | Perf report for the last 5.14894 seconds: | 20260505 16:20:39.615 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:20:39.615 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:39.615 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:39.615 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:39.615 | INFO | ANALYSIS : 62.5%, 3.0, 97.2ms, 0 | 20260505 16:20:39.615 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:39.615 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:39.615 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 16:20:39.615 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:39.615 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:39.616 | INFO | Uploaded 36GiB to GPU in 2.1845s (16.4798GiB/s) | 20260505 16:20:39.616 | INFO | Downloaded 34GiB to GPU in 2.13301s (15.9399GiB/s) | 20260505 16:20:44.749 | INFO | Analyzing chunk frames = [4597760,4598272) | 20260505 16:20:44.750 | INFO | MemInfo: { | 20260505 16:20:44.750 | INFO | "gpu" : " 17 GB", | 20260505 16:20:44.750 | INFO | "huge" : "", | 20260505 16:20:44.750 | INFO | "huge_shared" : " 27 GB", | 20260505 16:20:44.750 | INFO | "malloc" : " 14 GB", | 20260505 16:20:44.750 | INFO | "pinned" : " 29 GB" | 20260505 16:20:44.750 | INFO | } | 20260505 16:20:44.750 | INFO | Perf report for the last 5.13444 seconds: | 20260505 16:20:44.750 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:20:44.750 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:44.750 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:44.750 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:44.750 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 16:20:44.750 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:44.750 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:44.750 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:20:44.750 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:44.750 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:44.750 | INFO | Uploaded 36GiB to GPU in 2.14312s (16.798GiB/s) | 20260505 16:20:44.750 | INFO | Downloaded 34GiB to GPU in 2.06005s (16.5044GiB/s) | 20260505 16:20:49.883 | INFO | Analyzing chunk frames = [4598272,4598784) | 20260505 16:20:49.883 | INFO | MemInfo: { | 20260505 16:20:49.883 | INFO | "gpu" : " 17 GB", | 20260505 16:20:49.883 | INFO | "huge" : "", | 20260505 16:20:49.883 | INFO | "huge_shared" : " 27 GB", | 20260505 16:20:49.883 | INFO | "malloc" : " 14 GB", | 20260505 16:20:49.883 | INFO | "pinned" : " 29 GB" | 20260505 16:20:49.883 | INFO | } | 20260505 16:20:49.884 | INFO | Perf report for the last 5.13402 seconds: | 20260505 16:20:49.884 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:20:49.884 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:49.884 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:49.884 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:49.884 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 16:20:49.884 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:49.884 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:49.884 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 16:20:49.884 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:49.884 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:49.884 | INFO | Uploaded 36GiB to GPU in 2.14704s (16.7673GiB/s) | 20260505 16:20:49.884 | INFO | Downloaded 34GiB to GPU in 2.04331s (16.6397GiB/s) | 20260505 16:20:55.023 | INFO | Analyzing chunk frames = [4598784,4599296) | 20260505 16:20:55.023 | INFO | MemInfo: { | 20260505 16:20:55.023 | INFO | "gpu" : " 17 GB", | 20260505 16:20:55.023 | INFO | "huge" : "", | 20260505 16:20:55.023 | INFO | "huge_shared" : " 27 GB", | 20260505 16:20:55.023 | INFO | "malloc" : " 14 GB", | 20260505 16:20:55.023 | INFO | "pinned" : " 29 GB" | 20260505 16:20:55.023 | INFO | } | 20260505 16:20:55.023 | INFO | Perf report for the last 5.13952 seconds: | 20260505 16:20:55.023 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:20:55.023 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:55.023 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:55.023 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:55.023 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 16:20:55.023 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:55.023 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:55.023 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 16:20:55.023 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:55.023 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:20:55.023 | INFO | Uploaded 36GiB to GPU in 2.11161s (17.0486GiB/s) | 20260505 16:20:55.023 | INFO | Downloaded 34GiB to GPU in 2.16628s (15.6951GiB/s) | 20260505 16:21:00.162 | INFO | Analyzing chunk frames = [4599296,4599808) | 20260505 16:21:00.162 | INFO | MemInfo: { | 20260505 16:21:00.162 | INFO | "gpu" : " 17 GB", | 20260505 16:21:00.162 | INFO | "huge" : "", | 20260505 16:21:00.162 | INFO | "huge_shared" : " 27 GB", | 20260505 16:21:00.162 | INFO | "malloc" : " 14 GB", | 20260505 16:21:00.162 | INFO | "pinned" : " 29 GB" | 20260505 16:21:00.162 | INFO | } | 20260505 16:21:00.162 | INFO | Perf report for the last 5.13903 seconds: | 20260505 16:21:00.162 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:21:00.162 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:00.162 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:00.162 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:00.162 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 16:21:00.162 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:00.162 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:00.162 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 16:21:00.162 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:00.162 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:00.162 | INFO | Uploaded 36GiB to GPU in 2.22584s (16.1737GiB/s) | 20260505 16:21:00.163 | INFO | Downloaded 34GiB to GPU in 2.10596s (16.1447GiB/s) | 20260505 16:21:00.298 | INFO | Starting a round of Baz IO | 20260505 16:21:03.495 | INFO | Populating RTMetrics report | 20260505 16:21:03.495 | INFO | RTMetrics report populated | 20260505 16:21:03.495 | INFO | Opening tmp RTMetrics json file | 20260505 16:21:03.508 | INFO | Serializing RTMetrics to json | 20260505 16:21:03.511 | INFO | Serialization done, swapping tmp file now... | 20260505 16:21:03.513 | INFO | RTMetrics done | 20260505 16:21:03.595 | INFO | Finished a round of Baz IO: Wrote 2.174 GiB in 3.297 seconds | 20260505 16:21:03.595 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.26% / 0.00% / 77.73% | 20260505 16:21:03.595 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.371186% / 0% / 0.000103315% | 20260505 16:21:03.595 | INFO | Baz IO overall: | 20260505 16:21:03.595 | INFO | BytesWritten : 2730134447881 | 20260505 16:21:03.595 | INFO | NumEvents : 1295971519013 | 20260505 16:21:05.308 | INFO | Analyzing chunk frames = [4599808,4600320) | 20260505 16:21:05.308 | INFO | MemInfo: { | 20260505 16:21:05.308 | INFO | "gpu" : " 17 GB", | 20260505 16:21:05.308 | INFO | "huge" : "", | 20260505 16:21:05.308 | INFO | "huge_shared" : " 27 GB", | 20260505 16:21:05.308 | INFO | "malloc" : " 14 GB", | 20260505 16:21:05.308 | INFO | "pinned" : " 29 GB" | 20260505 16:21:05.308 | INFO | } | 20260505 16:21:05.308 | INFO | Perf report for the last 5.14601 seconds: | 20260505 16:21:05.308 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:21:05.308 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:05.308 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:05.308 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:05.308 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 16:21:05.308 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:05.308 | INFO | RT_METRICS : 44.9%, 1.0, 23.6ms, 0 | 20260505 16:21:05.308 | INFO | PRE_HQ : 60.5%, 1.8, 56.4ms, 0 | 20260505 16:21:05.308 | INFO | BAZWRITER : 63.4%, 3.9, 128.9ms, 0 | 20260505 16:21:05.308 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 16:21:05.308 | INFO | Uploaded 36GiB to GPU in 2.16877s (16.5993GiB/s) | 20260505 16:21:05.309 | INFO | Downloaded 38GiB to GPU in 2.26068s (16.8091GiB/s) | 20260505 16:21:10.451 | INFO | Analyzing chunk frames = [4600320,4600832) | 20260505 16:21:10.451 | INFO | MemInfo: { | 20260505 16:21:10.451 | INFO | "gpu" : " 17 GB", | 20260505 16:21:10.451 | INFO | "huge" : "", | 20260505 16:21:10.451 | INFO | "huge_shared" : " 27 GB", | 20260505 16:21:10.451 | INFO | "malloc" : " 14 GB", | 20260505 16:21:10.451 | INFO | "pinned" : " 29 GB" | 20260505 16:21:10.451 | INFO | } | 20260505 16:21:10.451 | INFO | Perf report for the last 5.14305 seconds: | 20260505 16:21:10.451 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:21:10.451 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:10.451 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:10.451 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:10.451 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 16:21:10.451 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:10.451 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:10.451 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260505 16:21:10.451 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:10.451 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:10.452 | INFO | Uploaded 36GiB to GPU in 2.133s (16.8777GiB/s) | 20260505 16:21:10.452 | INFO | Downloaded 34GiB to GPU in 2.0209s (16.8242GiB/s) | 20260505 16:21:15.577 | INFO | Analyzing chunk frames = [4600832,4601344) | 20260505 16:21:15.577 | INFO | MemInfo: { | 20260505 16:21:15.577 | INFO | "gpu" : " 17 GB", | 20260505 16:21:15.577 | INFO | "huge" : "", | 20260505 16:21:15.577 | INFO | "huge_shared" : " 27 GB", | 20260505 16:21:15.577 | INFO | "malloc" : " 14 GB", | 20260505 16:21:15.577 | INFO | "pinned" : " 29 GB" | 20260505 16:21:15.577 | INFO | } | 20260505 16:21:15.577 | INFO | Perf report for the last 5.12586 seconds: | 20260505 16:21:15.577 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:21:15.577 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:15.577 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:15.577 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:15.577 | INFO | ANALYSIS : 62.8%, 3.0, 97.5ms, 0 | 20260505 16:21:15.577 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:15.577 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:15.577 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:21:15.577 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:15.577 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:15.577 | INFO | Uploaded 36GiB to GPU in 2.195s (16.4009GiB/s) | 20260505 16:21:15.577 | INFO | Downloaded 34GiB to GPU in 2.02748s (16.7696GiB/s) | 20260505 16:21:20.712 | INFO | Analyzing chunk frames = [4601344,4601856) | 20260505 16:21:20.712 | INFO | MemInfo: { | 20260505 16:21:20.712 | INFO | "gpu" : " 17 GB", | 20260505 16:21:20.712 | INFO | "huge" : "", | 20260505 16:21:20.712 | INFO | "huge_shared" : " 27 GB", | 20260505 16:21:20.712 | INFO | "malloc" : " 14 GB", | 20260505 16:21:20.712 | INFO | "pinned" : " 29 GB" | 20260505 16:21:20.712 | INFO | } | 20260505 16:21:20.712 | INFO | Perf report for the last 5.13531 seconds: | 20260505 16:21:20.712 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:21:20.712 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:20.712 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:20.713 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:20.713 | INFO | ANALYSIS : 62.6%, 3.0, 97.2ms, 0 | 20260505 16:21:20.713 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:20.713 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:20.713 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:21:20.713 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:20.713 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:20.713 | INFO | Uploaded 36GiB to GPU in 2.18542s (16.4728GiB/s) | 20260505 16:21:20.713 | INFO | Downloaded 34GiB to GPU in 2.13563s (15.9204GiB/s) | 20260505 16:21:25.863 | INFO | Analyzing chunk frames = [4601856,4602368) | 20260505 16:21:25.863 | INFO | MemInfo: { | 20260505 16:21:25.863 | INFO | "gpu" : " 17 GB", | 20260505 16:21:25.863 | INFO | "huge" : "", | 20260505 16:21:25.863 | INFO | "huge_shared" : " 27 GB", | 20260505 16:21:25.863 | INFO | "malloc" : " 14 GB", | 20260505 16:21:25.863 | INFO | "pinned" : " 29 GB" | 20260505 16:21:25.863 | INFO | } | 20260505 16:21:25.863 | INFO | Perf report for the last 5.1504 seconds: | 20260505 16:21:25.863 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:21:25.863 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:25.863 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:25.863 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:25.863 | INFO | ANALYSIS : 62.6%, 3.0, 98.0ms, 0 | 20260505 16:21:25.863 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:25.863 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:25.863 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:21:25.863 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:25.863 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:25.863 | INFO | Uploaded 36GiB to GPU in 2.17366s (16.5619GiB/s) | 20260505 16:21:25.863 | INFO | Downloaded 34GiB to GPU in 2.12706s (15.9845GiB/s) | 20260505 16:21:31.000 | INFO | Analyzing chunk frames = [4602368,4602880) | 20260505 16:21:31.000 | INFO | MemInfo: { | 20260505 16:21:31.000 | INFO | "gpu" : " 17 GB", | 20260505 16:21:31.000 | INFO | "huge" : "", | 20260505 16:21:31.000 | INFO | "huge_shared" : " 27 GB", | 20260505 16:21:31.000 | INFO | "malloc" : " 14 GB", | 20260505 16:21:31.000 | INFO | "pinned" : " 29 GB" | 20260505 16:21:31.000 | INFO | } | 20260505 16:21:31.000 | INFO | Perf report for the last 5.13724 seconds: | 20260505 16:21:31.000 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:21:31.000 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:31.000 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:31.000 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:31.000 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260505 16:21:31.000 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:31.000 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:31.000 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:21:31.000 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:31.000 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:31.000 | INFO | Uploaded 36GiB to GPU in 2.13844s (16.8347GiB/s) | 20260505 16:21:31.001 | INFO | Downloaded 34GiB to GPU in 2.03395s (16.7163GiB/s) | 20260505 16:21:36.145 | INFO | Analyzing chunk frames = [4602880,4603392) | 20260505 16:21:36.145 | INFO | MemInfo: { | 20260505 16:21:36.145 | INFO | "gpu" : " 17 GB", | 20260505 16:21:36.145 | INFO | "huge" : "", | 20260505 16:21:36.145 | INFO | "huge_shared" : " 27 GB", | 20260505 16:21:36.145 | INFO | "malloc" : " 14 GB", | 20260505 16:21:36.145 | INFO | "pinned" : " 29 GB" | 20260505 16:21:36.145 | INFO | } | 20260505 16:21:36.145 | INFO | Perf report for the last 5.1451 seconds: | 20260505 16:21:36.145 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:21:36.145 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:36.145 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:36.145 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:36.145 | INFO | ANALYSIS : 63.4%, 3.0, 99.2ms, 0 | 20260505 16:21:36.145 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:36.145 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:36.145 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 16:21:36.145 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:36.145 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:36.146 | INFO | Uploaded 36GiB to GPU in 2.12688s (16.9262GiB/s) | 20260505 16:21:36.146 | INFO | Downloaded 34GiB to GPU in 2.18497s (15.5609GiB/s) | 20260505 16:21:41.285 | INFO | Analyzing chunk frames = [4603392,4603904) | 20260505 16:21:41.285 | INFO | MemInfo: { | 20260505 16:21:41.285 | INFO | "gpu" : " 17 GB", | 20260505 16:21:41.285 | INFO | "huge" : "", | 20260505 16:21:41.285 | INFO | "huge_shared" : " 27 GB", | 20260505 16:21:41.285 | INFO | "malloc" : " 14 GB", | 20260505 16:21:41.285 | INFO | "pinned" : " 29 GB" | 20260505 16:21:41.285 | INFO | } | 20260505 16:21:41.285 | INFO | Perf report for the last 5.1398 seconds: | 20260505 16:21:41.285 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:21:41.285 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:41.285 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:41.285 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:41.285 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 16:21:41.285 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:41.285 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:41.285 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 16:21:41.285 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:41.285 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:41.285 | INFO | Uploaded 36GiB to GPU in 2.15352s (16.7168GiB/s) | 20260505 16:21:41.285 | INFO | Downloaded 34GiB to GPU in 2.02202s (16.8148GiB/s) | 20260505 16:21:41.286 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 16:21:41.286 | INFO | Total mem usage: 15.3 GiB | 20260505 16:21:41.286 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 16:21:41.286 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.88%) | 20260505 16:21:41.286 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 16:21:41.286 | INFO | High Water Breakdown: | 20260505 16:21:41.286 | INFO | 61% : 8.67 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 16:21:41.286 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 16:21:41.286 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 16:21:41.286 | INFO | ------------------------------------------ | 20260505 16:21:41.286 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:21:41.286 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 16:21:41.286 | INFO | Total mem usage: 26.5 GiB | 20260505 16:21:41.286 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 16:21:41.286 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 16:21:41.286 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 16:21:41.286 | INFO | High Water Breakdown: | 20260505 16:21:41.286 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:21:41.286 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:21:41.286 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:21:41.286 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:21:41.286 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:21:41.286 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:21:41.286 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 16:21:41.286 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 16:21:41.286 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:21:41.286 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 16:21:41.286 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 16:21:41.286 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 16:21:41.286 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 16:21:41.286 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 16:21:41.286 | INFO | ------------------------------------------ | 20260505 16:21:41.286 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:21:41.286 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 16:21:41.286 | INFO | Total mem usage: 25.6 GiB | 20260505 16:21:41.286 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 16:21:41.286 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 16:21:41.286 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 16:21:41.286 | INFO | High Water Breakdown: | 20260505 16:21:41.286 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 16:21:41.286 | INFO | ------------------------------------------ | 20260505 16:21:41.286 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:21:41.286 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 16:21:41.286 | INFO | Total mem usage: 15.7 GiB | 20260505 16:21:41.286 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 16:21:41.286 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 16:21:41.286 | INFO | Unused Cache: 896 MiB (Waste: 5.58%) | 20260505 16:21:41.286 | INFO | High Water Breakdown: | 20260505 16:21:41.286 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 16:21:41.286 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 16:21:41.286 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 16:21:41.286 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 16:21:41.286 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 16:21:41.286 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 16:21:41.286 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 16:21:41.286 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 16:21:41.286 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 16:21:41.286 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:21:41.286 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 16:21:41.286 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:21:41.286 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 16:21:41.287 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 16:21:41.287 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 16:21:41.287 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 16:21:41.287 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:21:41.287 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 16:21:41.287 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:21:41.287 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:21:41.287 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:21:41.287 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 16:21:41.287 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 16:21:41.287 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:21:41.287 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 16:21:41.287 | INFO | ------------------------------------------ | 20260505 16:21:41.287 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:21:41.287 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 16:21:41.287 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:21:41.287 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:21:41.287 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.54e+01, 1.36e+01 / 2.22e+00 | 20260505 16:21:41.287 | INFO | PerfReport: 0.29%, AggregateStats -- 9.25e-02 / 1.03e-01, 1.05e-02 / 3.80e-03 | 20260505 16:21:41.287 | INFO | PerfReport: 11.80%, Baseline -- 3.74e+00 / 4.48e+00, 5.82e-01 / 1.81e-01 | 20260505 16:21:41.287 | INFO | PerfReport: 3.29%, Histogram -- 1.04e+00 / 1.52e+00, 1.95e-01 / 1.02e-01 | 20260505 16:21:41.287 | INFO | PerfReport: 15.90%, DME -- 4.03e+01 / 4.47e+01, 4.16e+00 / 2.21e+00 | 20260505 16:21:41.287 | INFO | PerfReport: 56.76%, FrameLabeling -- 1.80e+01 / 1.99e+01, 1.50e+00 / 1.79e+00 | 20260505 16:21:41.287 | INFO | PerfReport: 7.34%, PulseAccumulating -- 2.33e+00 / 2.90e+00, 4.38e-01 / 1.63e-01 | 20260505 16:21:41.287 | INFO | PerfReport: 3.42%, Metrics -- 1.08e+00 / 1.40e+00, 3.01e-01 / 2.10e-01 | 20260505 16:21:41.287 | INFO | PerfReport: 1.19%, PulseSerializing -- 3.78e-01 / 4.31e-01, 6.81e-02 / 3.55e-02 | 20260505 16:21:41.287 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 16:21:41.287 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:21:41.287 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:21:41.287 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260505 16:21:41.287 | INFO | PerfReport: 22.47%, Upload -- 2.25e+01 / 0.00e+00, 7.82e+00 / 0.00e+00 | 20260505 16:21:41.287 | INFO | PerfReport: 21.86%, Download -- 2.19e+01 / 0.00e+00, 7.88e+00 / 0.00e+00 | 20260505 16:21:41.287 | INFO | PerfReport: 31.66%, Compute -- 3.17e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 16:21:41.287 | INFO | PerfReport: 2.18%, LockUpload -- 2.18e+00 / 0.00e+00, 7.81e+00 / 0.00e+00 | 20260505 16:21:41.287 | INFO | PerfReport: 1.44%, LockDownload -- 1.44e+00 / 0.00e+00, 3.04e+00 / 0.00e+00 | 20260505 16:21:41.287 | INFO | PerfReport: 20.38%, LockCompute -- 2.04e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 16:21:41.287 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.84e-02 / 0.00e+00 | 20260505 16:21:41.418 | INFO | Starting a round of Baz IO | 20260505 16:21:44.627 | INFO | Populating RTMetrics report | 20260505 16:21:44.627 | INFO | RTMetrics report populated | 20260505 16:21:44.627 | INFO | Opening tmp RTMetrics json file | 20260505 16:21:44.640 | INFO | Serializing RTMetrics to json | 20260505 16:21:44.642 | INFO | Serialization done, swapping tmp file now... | 20260505 16:21:44.643 | INFO | RTMetrics done | 20260505 16:21:44.733 | INFO | Finished a round of Baz IO: Wrote 2.175 GiB in 3.314 seconds | 20260505 16:21:44.733 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.26% / 0.00% / 77.74% | 20260505 16:21:44.733 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.3721% / 0% / 8.34465e-05% | 20260505 16:21:44.733 | INFO | Baz IO overall: | 20260505 16:21:44.733 | INFO | BytesWritten : 2732469810753 | 20260505 16:21:44.733 | INFO | NumEvents : 1297066872018 | 20260505 16:21:46.420 | INFO | Analyzing chunk frames = [4603904,4604416) | 20260505 16:21:46.420 | INFO | MemInfo: { | 20260505 16:21:46.420 | INFO | "gpu" : " 17 GB", | 20260505 16:21:46.420 | INFO | "huge" : "", | 20260505 16:21:46.420 | INFO | "huge_shared" : " 27 GB", | 20260505 16:21:46.420 | INFO | "malloc" : " 14 GB", | 20260505 16:21:46.420 | INFO | "pinned" : " 29 GB" | 20260505 16:21:46.420 | INFO | } | 20260505 16:21:46.420 | INFO | Perf report for the last 5.13499 seconds: | 20260505 16:21:46.420 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:21:46.420 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:46.420 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:46.420 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:46.420 | INFO | ANALYSIS : 64.0%, 3.0, 99.3ms, 0 | 20260505 16:21:46.420 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:46.420 | INFO | RT_METRICS : 45.1%, 1.0, 23.6ms, 0 | 20260505 16:21:46.420 | INFO | PRE_HQ : 60.6%, 1.8, 56.8ms, 0 | 20260505 16:21:46.420 | INFO | BAZWRITER : 63.9%, 3.9, 129.0ms, 0 | 20260505 16:21:46.420 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 16:21:46.420 | INFO | Uploaded 36GiB to GPU in 2.1809s (16.5069GiB/s) | 20260505 16:21:46.421 | INFO | Downloaded 38GiB to GPU in 2.2687s (16.7497GiB/s) | 20260505 16:21:51.563 | INFO | Analyzing chunk frames = [4604416,4604928) | 20260505 16:21:51.563 | INFO | MemInfo: { | 20260505 16:21:51.563 | INFO | "gpu" : " 17 GB", | 20260505 16:21:51.563 | INFO | "huge" : "", | 20260505 16:21:51.563 | INFO | "huge_shared" : " 27 GB", | 20260505 16:21:51.563 | INFO | "malloc" : " 14 GB", | 20260505 16:21:51.563 | INFO | "pinned" : " 29 GB" | 20260505 16:21:51.563 | INFO | } | 20260505 16:21:51.563 | INFO | Perf report for the last 5.14339 seconds: | 20260505 16:21:51.564 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:21:51.564 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:51.564 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:51.564 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:51.564 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 16:21:51.564 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:51.564 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:51.564 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 16:21:51.564 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:51.564 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:51.564 | INFO | Uploaded 36GiB to GPU in 2.12875s (16.9114GiB/s) | 20260505 16:21:51.564 | INFO | Downloaded 34GiB to GPU in 2.03482s (16.7091GiB/s) | 20260505 16:21:56.701 | INFO | Analyzing chunk frames = [4604928,4605440) | 20260505 16:21:56.701 | INFO | MemInfo: { | 20260505 16:21:56.701 | INFO | "gpu" : " 17 GB", | 20260505 16:21:56.701 | INFO | "huge" : "", | 20260505 16:21:56.701 | INFO | "huge_shared" : " 27 GB", | 20260505 16:21:56.701 | INFO | "malloc" : " 14 GB", | 20260505 16:21:56.701 | INFO | "pinned" : " 29 GB" | 20260505 16:21:56.701 | INFO | } | 20260505 16:21:56.701 | INFO | Perf report for the last 5.13786 seconds: | 20260505 16:21:56.701 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:21:56.701 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:56.701 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:56.701 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:56.701 | INFO | ANALYSIS : 62.7%, 3.0, 97.5ms, 0 | 20260505 16:21:56.701 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:56.701 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:56.701 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:21:56.701 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:56.701 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:21:56.702 | INFO | Uploaded 36GiB to GPU in 2.19791s (16.3792GiB/s) | 20260505 16:21:56.702 | INFO | Downloaded 34GiB to GPU in 2.03315s (16.7228GiB/s) | 20260505 16:22:01.828 | INFO | Analyzing chunk frames = [4605440,4605952) | 20260505 16:22:01.828 | INFO | MemInfo: { | 20260505 16:22:01.828 | INFO | "gpu" : " 17 GB", | 20260505 16:22:01.828 | INFO | "huge" : "", | 20260505 16:22:01.828 | INFO | "huge_shared" : " 27 GB", | 20260505 16:22:01.828 | INFO | "malloc" : " 14 GB", | 20260505 16:22:01.828 | INFO | "pinned" : " 29 GB" | 20260505 16:22:01.828 | INFO | } | 20260505 16:22:01.828 | INFO | Perf report for the last 5.12712 seconds: | 20260505 16:22:01.828 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:22:01.828 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:01.828 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:01.829 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:01.829 | INFO | ANALYSIS : 62.7%, 3.0, 97.1ms, 0 | 20260505 16:22:01.829 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:01.829 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:01.829 | INFO | PRE_HQ : 28.3%, 1.0, 14.6ms, 0 | 20260505 16:22:01.829 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:01.829 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:01.829 | INFO | Uploaded 36GiB to GPU in 2.17987s (16.5148GiB/s) | 20260505 16:22:01.829 | INFO | Downloaded 34GiB to GPU in 2.14012s (15.887GiB/s) | 20260505 16:22:06.967 | INFO | Analyzing chunk frames = [4605952,4606464) | 20260505 16:22:06.967 | INFO | MemInfo: { | 20260505 16:22:06.967 | INFO | "gpu" : " 17 GB", | 20260505 16:22:06.967 | INFO | "huge" : "", | 20260505 16:22:06.967 | INFO | "huge_shared" : " 27 GB", | 20260505 16:22:06.967 | INFO | "malloc" : " 14 GB", | 20260505 16:22:06.967 | INFO | "pinned" : " 29 GB" | 20260505 16:22:06.967 | INFO | } | 20260505 16:22:06.968 | INFO | Perf report for the last 5.13946 seconds: | 20260505 16:22:06.968 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:22:06.968 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:06.968 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:06.968 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:06.968 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 16:22:06.968 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:06.968 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:06.968 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 16:22:06.968 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:06.968 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:06.968 | INFO | Uploaded 36GiB to GPU in 2.19105s (16.4305GiB/s) | 20260505 16:22:06.968 | INFO | Downloaded 34GiB to GPU in 2.13283s (15.9413GiB/s) | 20260505 16:22:12.115 | INFO | Analyzing chunk frames = [4606464,4606976) | 20260505 16:22:12.115 | INFO | MemInfo: { | 20260505 16:22:12.115 | INFO | "gpu" : " 17 GB", | 20260505 16:22:12.115 | INFO | "huge" : "", | 20260505 16:22:12.115 | INFO | "huge_shared" : " 27 GB", | 20260505 16:22:12.115 | INFO | "malloc" : " 14 GB", | 20260505 16:22:12.115 | INFO | "pinned" : " 29 GB" | 20260505 16:22:12.115 | INFO | } | 20260505 16:22:12.115 | INFO | Perf report for the last 5.14743 seconds: | 20260505 16:22:12.115 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:22:12.115 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:12.115 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:12.115 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:12.115 | INFO | ANALYSIS : 63.0%, 3.0, 98.6ms, 0 | 20260505 16:22:12.115 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:12.115 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:12.115 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:22:12.116 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:12.116 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:12.116 | INFO | Uploaded 36GiB to GPU in 2.2116s (16.2778GiB/s) | 20260505 16:22:12.116 | INFO | Downloaded 34GiB to GPU in 2.11476s (16.0774GiB/s) | 20260505 16:22:17.265 | INFO | Analyzing chunk frames = [4606976,4607488) | 20260505 16:22:17.265 | INFO | MemInfo: { | 20260505 16:22:17.265 | INFO | "gpu" : " 17 GB", | 20260505 16:22:17.265 | INFO | "huge" : "", | 20260505 16:22:17.265 | INFO | "huge_shared" : " 27 GB", | 20260505 16:22:17.265 | INFO | "malloc" : " 14 GB", | 20260505 16:22:17.265 | INFO | "pinned" : " 29 GB" | 20260505 16:22:17.265 | INFO | } | 20260505 16:22:17.265 | INFO | Perf report for the last 5.14995 seconds: | 20260505 16:22:17.265 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:22:17.265 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:17.265 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:17.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:17.265 | INFO | ANALYSIS : 63.4%, 3.0, 99.2ms, 0 | 20260505 16:22:17.265 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:17.265 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:17.265 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 16:22:17.265 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:17.265 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:17.266 | INFO | Uploaded 36GiB to GPU in 2.09825s (17.1571GiB/s) | 20260505 16:22:17.266 | INFO | Downloaded 34GiB to GPU in 2.17371s (15.6415GiB/s) | 20260505 16:22:22.399 | INFO | Analyzing chunk frames = [4607488,4608000) | 20260505 16:22:22.399 | INFO | MemInfo: { | 20260505 16:22:22.399 | INFO | "gpu" : " 17 GB", | 20260505 16:22:22.399 | INFO | "huge" : "", | 20260505 16:22:22.399 | INFO | "huge_shared" : " 27 GB", | 20260505 16:22:22.399 | INFO | "malloc" : " 14 GB", | 20260505 16:22:22.399 | INFO | "pinned" : " 29 GB" | 20260505 16:22:22.399 | INFO | } | 20260505 16:22:22.399 | INFO | Perf report for the last 5.13398 seconds: | 20260505 16:22:22.399 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:22:22.399 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:22.399 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:22.399 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:22.399 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 16:22:22.399 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:22.399 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:22.399 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 16:22:22.399 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:22.399 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:22.400 | INFO | Uploaded 36GiB to GPU in 2.19477s (16.4026GiB/s) | 20260505 16:22:22.400 | INFO | Downloaded 34GiB to GPU in 2.02223s (16.8131GiB/s) | 20260505 16:22:22.533 | INFO | Starting a round of Baz IO | 20260505 16:22:25.737 | INFO | Populating RTMetrics report | 20260505 16:22:25.737 | INFO | RTMetrics report populated | 20260505 16:22:25.737 | INFO | Opening tmp RTMetrics json file | 20260505 16:22:25.746 | INFO | Serializing RTMetrics to json | 20260505 16:22:25.748 | INFO | Serialization done, swapping tmp file now... | 20260505 16:22:25.748 | INFO | RTMetrics done | 20260505 16:22:25.835 | INFO | Finished a round of Baz IO: Wrote 2.173 GiB in 3.302 seconds | 20260505 16:22:25.835 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.26% / 0.00% / 77.74% | 20260505 16:22:25.835 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.370236% / 3.97364e-06% / 8.74201e-05% | 20260505 16:22:25.835 | INFO | Baz IO overall: | 20260505 16:22:25.835 | INFO | BytesWritten : 2734802659841 | 20260505 16:22:25.835 | INFO | NumEvents : 1298160533599 | 20260505 16:22:27.531 | INFO | Analyzing chunk frames = [4608000,4608512) | 20260505 16:22:27.531 | INFO | MemInfo: { | 20260505 16:22:27.531 | INFO | "gpu" : " 17 GB", | 20260505 16:22:27.531 | INFO | "huge" : "", | 20260505 16:22:27.531 | INFO | "huge_shared" : " 27 GB", | 20260505 16:22:27.531 | INFO | "malloc" : " 14 GB", | 20260505 16:22:27.531 | INFO | "pinned" : " 29 GB" | 20260505 16:22:27.531 | INFO | } | 20260505 16:22:27.531 | INFO | Perf report for the last 5.13213 seconds: | 20260505 16:22:27.531 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:22:27.531 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:27.532 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:27.532 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:27.532 | INFO | ANALYSIS : 63.9%, 3.0, 99.4ms, 0 | 20260505 16:22:27.532 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:27.532 | INFO | RT_METRICS : 44.8%, 1.0, 23.4ms, 0 | 20260505 16:22:27.532 | INFO | PRE_HQ : 60.8%, 1.8, 56.3ms, 0 | 20260505 16:22:27.532 | INFO | BAZWRITER : 63.7%, 3.9, 128.2ms, 0 | 20260505 16:22:27.532 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260505 16:22:27.532 | INFO | Uploaded 36GiB to GPU in 2.18698s (16.4611GiB/s) | 20260505 16:22:27.532 | INFO | Downloaded 38GiB to GPU in 2.27059s (16.7357GiB/s) | 20260505 16:22:32.672 | INFO | Analyzing chunk frames = [4608512,4609024) | 20260505 16:22:32.672 | INFO | MemInfo: { | 20260505 16:22:32.672 | INFO | "gpu" : " 17 GB", | 20260505 16:22:32.672 | INFO | "huge" : "", | 20260505 16:22:32.672 | INFO | "huge_shared" : " 27 GB", | 20260505 16:22:32.672 | INFO | "malloc" : " 14 GB", | 20260505 16:22:32.672 | INFO | "pinned" : " 29 GB" | 20260505 16:22:32.672 | INFO | } | 20260505 16:22:32.672 | INFO | Perf report for the last 5.1404 seconds: | 20260505 16:22:32.672 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:22:32.672 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:32.672 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:32.672 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:32.672 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 16:22:32.672 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:32.672 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:32.672 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:22:32.672 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:32.672 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:32.672 | INFO | Uploaded 36GiB to GPU in 2.13495s (16.8622GiB/s) | 20260505 16:22:32.672 | INFO | Downloaded 34GiB to GPU in 2.04148s (16.6546GiB/s) | 20260505 16:22:37.810 | INFO | Analyzing chunk frames = [4609024,4609536) | 20260505 16:22:37.810 | INFO | MemInfo: { | 20260505 16:22:37.810 | INFO | "gpu" : " 17 GB", | 20260505 16:22:37.810 | INFO | "huge" : "", | 20260505 16:22:37.810 | INFO | "huge_shared" : " 27 GB", | 20260505 16:22:37.810 | INFO | "malloc" : " 14 GB", | 20260505 16:22:37.810 | INFO | "pinned" : " 29 GB" | 20260505 16:22:37.810 | INFO | } | 20260505 16:22:37.810 | INFO | Perf report for the last 5.13827 seconds: | 20260505 16:22:37.810 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:22:37.810 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:37.810 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:37.810 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:37.810 | INFO | ANALYSIS : 62.8%, 3.0, 97.6ms, 0 | 20260505 16:22:37.810 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:37.810 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:37.810 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 16:22:37.810 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:37.810 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:37.810 | INFO | Uploaded 36GiB to GPU in 2.17952s (16.5174GiB/s) | 20260505 16:22:37.810 | INFO | Downloaded 34GiB to GPU in 2.03177s (16.7342GiB/s) | 20260505 16:22:42.954 | INFO | Analyzing chunk frames = [4609536,4610048) | 20260505 16:22:42.954 | INFO | MemInfo: { | 20260505 16:22:42.954 | INFO | "gpu" : " 17 GB", | 20260505 16:22:42.954 | INFO | "huge" : "", | 20260505 16:22:42.954 | INFO | "huge_shared" : " 27 GB", | 20260505 16:22:42.954 | INFO | "malloc" : " 14 GB", | 20260505 16:22:42.954 | INFO | "pinned" : " 29 GB" | 20260505 16:22:42.954 | INFO | } | 20260505 16:22:42.954 | INFO | Perf report for the last 5.14362 seconds: | 20260505 16:22:42.954 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:22:42.954 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:42.954 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:42.954 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:42.954 | INFO | ANALYSIS : 62.4%, 3.0, 96.9ms, 0 | 20260505 16:22:42.954 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:42.954 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:42.954 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 16:22:42.954 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:42.954 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:42.954 | INFO | Uploaded 36GiB to GPU in 2.18826s (16.4514GiB/s) | 20260505 16:22:42.954 | INFO | Downloaded 34GiB to GPU in 2.13383s (15.9338GiB/s) | 20260505 16:22:48.089 | INFO | Analyzing chunk frames = [4610048,4610560) | 20260505 16:22:48.089 | INFO | MemInfo: { | 20260505 16:22:48.089 | INFO | "gpu" : " 17 GB", | 20260505 16:22:48.089 | INFO | "huge" : "", | 20260505 16:22:48.089 | INFO | "huge_shared" : " 27 GB", | 20260505 16:22:48.089 | INFO | "malloc" : " 14 GB", | 20260505 16:22:48.089 | INFO | "pinned" : " 29 GB" | 20260505 16:22:48.090 | INFO | } | 20260505 16:22:48.090 | INFO | Perf report for the last 5.13593 seconds: | 20260505 16:22:48.090 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:22:48.090 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:48.090 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:48.090 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:48.090 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260505 16:22:48.090 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:48.090 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:48.090 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 16:22:48.090 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:48.090 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:48.090 | INFO | Uploaded 36GiB to GPU in 2.15544s (16.7019GiB/s) | 20260505 16:22:48.090 | INFO | Downloaded 34GiB to GPU in 2.0593s (16.5105GiB/s) | 20260505 16:22:53.228 | INFO | Analyzing chunk frames = [4610560,4611072) | 20260505 16:22:53.228 | INFO | MemInfo: { | 20260505 16:22:53.228 | INFO | "gpu" : " 17 GB", | 20260505 16:22:53.228 | INFO | "huge" : "", | 20260505 16:22:53.228 | INFO | "huge_shared" : " 27 GB", | 20260505 16:22:53.228 | INFO | "malloc" : " 14 GB", | 20260505 16:22:53.228 | INFO | "pinned" : " 29 GB" | 20260505 16:22:53.228 | INFO | } | 20260505 16:22:53.229 | INFO | Perf report for the last 5.13894 seconds: | 20260505 16:22:53.229 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:22:53.229 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:53.229 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:53.229 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:53.229 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260505 16:22:53.229 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:53.229 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:53.229 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:22:53.229 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:53.229 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:53.229 | INFO | Uploaded 36GiB to GPU in 2.14098s (16.8147GiB/s) | 20260505 16:22:53.229 | INFO | Downloaded 34GiB to GPU in 2.03932s (16.6722GiB/s) | 20260505 16:22:58.375 | INFO | Analyzing chunk frames = [4611072,4611584) | 20260505 16:22:58.375 | INFO | MemInfo: { | 20260505 16:22:58.375 | INFO | "gpu" : " 17 GB", | 20260505 16:22:58.375 | INFO | "huge" : "", | 20260505 16:22:58.375 | INFO | "huge_shared" : " 27 GB", | 20260505 16:22:58.375 | INFO | "malloc" : " 14 GB", | 20260505 16:22:58.375 | INFO | "pinned" : " 29 GB" | 20260505 16:22:58.375 | INFO | } | 20260505 16:22:58.375 | INFO | Perf report for the last 5.14664 seconds: | 20260505 16:22:58.375 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:22:58.375 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:58.375 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:58.375 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:58.375 | INFO | ANALYSIS : 63.3%, 3.0, 99.0ms, 0 | 20260505 16:22:58.375 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:58.375 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:58.375 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 16:22:58.375 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:58.375 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:22:58.376 | INFO | Uploaded 36GiB to GPU in 2.12128s (16.9709GiB/s) | 20260505 16:22:58.376 | INFO | Downloaded 34GiB to GPU in 2.17242s (15.6507GiB/s) | 20260505 16:23:03.514 | INFO | Analyzing chunk frames = [4611584,4612096) | 20260505 16:23:03.514 | INFO | MemInfo: { | 20260505 16:23:03.514 | INFO | "gpu" : " 17 GB", | 20260505 16:23:03.514 | INFO | "huge" : "", | 20260505 16:23:03.514 | INFO | "huge_shared" : " 27 GB", | 20260505 16:23:03.514 | INFO | "malloc" : " 14 GB", | 20260505 16:23:03.514 | INFO | "pinned" : " 29 GB" | 20260505 16:23:03.514 | INFO | } | 20260505 16:23:03.515 | INFO | Perf report for the last 5.13915 seconds: | 20260505 16:23:03.515 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:23:03.515 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:03.515 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:03.515 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:03.515 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 16:23:03.515 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:03.515 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:03.515 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 16:23:03.515 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:03.515 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:03.515 | INFO | Uploaded 36GiB to GPU in 2.19159s (16.4264GiB/s) | 20260505 16:23:03.515 | INFO | Downloaded 34GiB to GPU in 2.02164s (16.818GiB/s) | 20260505 16:23:03.515 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 16:23:03.515 | INFO | Total mem usage: 15.3 GiB | 20260505 16:23:03.515 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 16:23:03.515 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.88%) | 20260505 16:23:03.515 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 16:23:03.515 | INFO | High Water Breakdown: | 20260505 16:23:03.515 | INFO | 61% : 8.67 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 16:23:03.515 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 16:23:03.515 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 16:23:03.515 | INFO | ------------------------------------------ | 20260505 16:23:03.515 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:23:03.515 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 16:23:03.515 | INFO | Total mem usage: 26.5 GiB | 20260505 16:23:03.515 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 16:23:03.515 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 16:23:03.515 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 16:23:03.515 | INFO | High Water Breakdown: | 20260505 16:23:03.515 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:23:03.515 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:23:03.515 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:23:03.515 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:23:03.515 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:23:03.515 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:23:03.515 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 16:23:03.515 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 16:23:03.515 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:23:03.515 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 16:23:03.515 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 16:23:03.515 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 16:23:03.515 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 16:23:03.515 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 16:23:03.515 | INFO | ------------------------------------------ | 20260505 16:23:03.515 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:23:03.515 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 16:23:03.515 | INFO | Total mem usage: 25.6 GiB | 20260505 16:23:03.515 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 16:23:03.515 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 16:23:03.515 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 16:23:03.515 | INFO | High Water Breakdown: | 20260505 16:23:03.515 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 16:23:03.515 | INFO | ------------------------------------------ | 20260505 16:23:03.516 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:23:03.516 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 16:23:03.516 | INFO | Total mem usage: 15.7 GiB | 20260505 16:23:03.516 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 16:23:03.516 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 16:23:03.516 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 16:23:03.516 | INFO | High Water Breakdown: | 20260505 16:23:03.516 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 16:23:03.516 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 16:23:03.516 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 16:23:03.516 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 16:23:03.516 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 16:23:03.516 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 16:23:03.516 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 16:23:03.516 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 16:23:03.516 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 16:23:03.516 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:23:03.516 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 16:23:03.516 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:23:03.516 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 16:23:03.516 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 16:23:03.516 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 16:23:03.516 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 16:23:03.516 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:23:03.516 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 16:23:03.516 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:23:03.516 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:23:03.516 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:23:03.516 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 16:23:03.516 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 16:23:03.516 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:23:03.516 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 16:23:03.516 | INFO | ------------------------------------------ | 20260505 16:23:03.516 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:23:03.516 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 16:23:03.516 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:23:03.516 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:23:03.516 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.55e+01, 1.37e+01 / 2.25e+00 | 20260505 16:23:03.516 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 1.01e-01, 1.11e-02 / 3.91e-03 | 20260505 16:23:03.516 | INFO | PerfReport: 11.80%, Baseline -- 3.74e+00 / 4.47e+00, 5.75e-01 / 1.91e-01 | 20260505 16:23:03.516 | INFO | PerfReport: 3.29%, Histogram -- 1.04e+00 / 1.50e+00, 1.94e-01 / 1.05e-01 | 20260505 16:23:03.516 | INFO | PerfReport: 15.93%, DME -- 4.04e+01 / 4.49e+01, 4.22e+00 / 2.35e+00 | 20260505 16:23:03.516 | INFO | PerfReport: 56.70%, FrameLabeling -- 1.80e+01 / 1.97e+01, 1.45e+00 / 1.29e+00 | 20260505 16:23:03.516 | INFO | PerfReport: 7.37%, PulseAccumulating -- 2.33e+00 / 2.92e+00, 4.45e-01 / 1.31e-01 | 20260505 16:23:03.516 | INFO | PerfReport: 3.43%, Metrics -- 1.09e+00 / 1.45e+00, 3.01e-01 / 1.75e-01 | 20260505 16:23:03.516 | INFO | PerfReport: 1.19%, PulseSerializing -- 3.78e-01 / 4.41e-01, 6.75e-02 / 4.21e-02 | 20260505 16:23:03.516 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 16:23:03.516 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:23:03.516 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:23:03.516 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260505 16:23:03.516 | INFO | PerfReport: 22.53%, Upload -- 2.26e+01 / 0.00e+00, 7.82e+00 / 0.00e+00 | 20260505 16:23:03.516 | INFO | PerfReport: 21.89%, Download -- 2.19e+01 / 0.00e+00, 7.87e+00 / 0.00e+00 | 20260505 16:23:03.516 | INFO | PerfReport: 31.62%, Compute -- 3.17e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 16:23:03.516 | INFO | PerfReport: 2.25%, LockUpload -- 2.26e+00 / 0.00e+00, 7.94e+00 / 0.00e+00 | 20260505 16:23:03.516 | INFO | PerfReport: 1.47%, LockDownload -- 1.47e+00 / 0.00e+00, 3.06e+00 / 0.00e+00 | 20260505 16:23:03.516 | INFO | PerfReport: 20.22%, LockCompute -- 2.03e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260505 16:23:03.516 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.49e-02 / 0.00e+00 | 20260505 16:23:03.651 | INFO | Starting a round of Baz IO | 20260505 16:23:06.835 | INFO | Populating RTMetrics report | 20260505 16:23:06.836 | INFO | RTMetrics report populated | 20260505 16:23:06.836 | INFO | Opening tmp RTMetrics json file | 20260505 16:23:06.875 | INFO | Serializing RTMetrics to json | 20260505 16:23:06.915 | INFO | Serialization done, swapping tmp file now... | 20260505 16:23:06.916 | INFO | RTMetrics done | 20260505 16:23:06.933 | INFO | Finished a round of Baz IO: Wrote 2.171 GiB in 3.282 seconds | 20260505 16:23:06.933 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.26% / 0.00% / 77.74% | 20260505 16:23:06.933 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.367971% / 0% / 7.94729e-05% | 20260505 16:23:06.933 | INFO | Baz IO overall: | 20260505 16:23:06.933 | INFO | BytesWritten : 2737134103288 | 20260505 16:23:06.933 | INFO | NumEvents : 1299253404380 | 20260505 16:23:08.649 | INFO | Analyzing chunk frames = [4612096,4612608) | 20260505 16:23:08.649 | INFO | MemInfo: { | 20260505 16:23:08.649 | INFO | "gpu" : " 17 GB", | 20260505 16:23:08.649 | INFO | "huge" : "", | 20260505 16:23:08.649 | INFO | "huge_shared" : " 27 GB", | 20260505 16:23:08.649 | INFO | "malloc" : " 14 GB", | 20260505 16:23:08.649 | INFO | "pinned" : " 29 GB" | 20260505 16:23:08.649 | INFO | } | 20260505 16:23:08.649 | INFO | Perf report for the last 5.1348 seconds: | 20260505 16:23:08.649 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:23:08.649 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:08.649 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:08.649 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:08.649 | INFO | ANALYSIS : 63.6%, 3.0, 98.8ms, 0 | 20260505 16:23:08.649 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:08.649 | INFO | RT_METRICS : 46.3%, 1.0, 24.2ms, 0 | 20260505 16:23:08.650 | INFO | PRE_HQ : 60.6%, 1.8, 56.3ms, 0 | 20260505 16:23:08.650 | INFO | BAZWRITER : 63.4%, 3.9, 128.3ms, 0 | 20260505 16:23:08.650 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 16:23:08.650 | INFO | Uploaded 36GiB to GPU in 2.14476s (16.7851GiB/s) | 20260505 16:23:08.650 | INFO | Downloaded 38GiB to GPU in 2.2391s (16.9711GiB/s) | 20260505 16:23:13.780 | INFO | Analyzing chunk frames = [4612608,4613120) | 20260505 16:23:13.780 | INFO | MemInfo: { | 20260505 16:23:13.780 | INFO | "gpu" : " 17 GB", | 20260505 16:23:13.780 | INFO | "huge" : "", | 20260505 16:23:13.780 | INFO | "huge_shared" : " 27 GB", | 20260505 16:23:13.780 | INFO | "malloc" : " 14 GB", | 20260505 16:23:13.780 | INFO | "pinned" : " 29 GB" | 20260505 16:23:13.780 | INFO | } | 20260505 16:23:13.780 | INFO | Perf report for the last 5.13079 seconds: | 20260505 16:23:13.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:23:13.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:13.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:13.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:13.780 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 16:23:13.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:13.780 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:13.780 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:23:13.780 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:13.780 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:13.781 | INFO | Uploaded 36GiB to GPU in 2.13268s (16.8802GiB/s) | 20260505 16:23:13.781 | INFO | Downloaded 34GiB to GPU in 2.03242s (16.7288GiB/s) | 20260505 16:23:18.923 | INFO | Analyzing chunk frames = [4613120,4613632) | 20260505 16:23:18.923 | INFO | MemInfo: { | 20260505 16:23:18.923 | INFO | "gpu" : " 17 GB", | 20260505 16:23:18.923 | INFO | "huge" : "", | 20260505 16:23:18.923 | INFO | "huge_shared" : " 27 GB", | 20260505 16:23:18.923 | INFO | "malloc" : " 14 GB", | 20260505 16:23:18.923 | INFO | "pinned" : " 29 GB" | 20260505 16:23:18.923 | INFO | } | 20260505 16:23:18.923 | INFO | Perf report for the last 5.14302 seconds: | 20260505 16:23:18.923 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:23:18.923 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:18.923 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:18.923 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:18.923 | INFO | ANALYSIS : 63.0%, 3.0, 98.1ms, 0 | 20260505 16:23:18.923 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:18.923 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:18.923 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:23:18.923 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:18.923 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:18.924 | INFO | Uploaded 36GiB to GPU in 2.19462s (16.4038GiB/s) | 20260505 16:23:18.924 | INFO | Downloaded 34GiB to GPU in 2.05976s (16.5068GiB/s) | 20260505 16:23:24.070 | INFO | Analyzing chunk frames = [4613632,4614144) | 20260505 16:23:24.070 | INFO | MemInfo: { | 20260505 16:23:24.070 | INFO | "gpu" : " 17 GB", | 20260505 16:23:24.070 | INFO | "huge" : "", | 20260505 16:23:24.070 | INFO | "huge_shared" : " 27 GB", | 20260505 16:23:24.070 | INFO | "malloc" : " 14 GB", | 20260505 16:23:24.070 | INFO | "pinned" : " 29 GB" | 20260505 16:23:24.070 | INFO | } | 20260505 16:23:24.070 | INFO | Perf report for the last 5.14643 seconds: | 20260505 16:23:24.070 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:23:24.070 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:24.070 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:24.070 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:24.070 | INFO | ANALYSIS : 62.5%, 3.0, 97.2ms, 0 | 20260505 16:23:24.070 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:24.070 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:24.070 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:23:24.070 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:24.070 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:24.070 | INFO | Uploaded 36GiB to GPU in 2.19237s (16.4206GiB/s) | 20260505 16:23:24.070 | INFO | Downloaded 34GiB to GPU in 2.13244s (15.9442GiB/s) | 20260505 16:23:29.208 | INFO | Analyzing chunk frames = [4614144,4614656) | 20260505 16:23:29.208 | INFO | MemInfo: { | 20260505 16:23:29.208 | INFO | "gpu" : " 17 GB", | 20260505 16:23:29.208 | INFO | "huge" : "", | 20260505 16:23:29.208 | INFO | "huge_shared" : " 27 GB", | 20260505 16:23:29.208 | INFO | "malloc" : " 14 GB", | 20260505 16:23:29.208 | INFO | "pinned" : " 29 GB" | 20260505 16:23:29.208 | INFO | } | 20260505 16:23:29.209 | INFO | Perf report for the last 5.13891 seconds: | 20260505 16:23:29.209 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:23:29.209 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:29.209 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:29.209 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:29.209 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 16:23:29.209 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:29.209 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:29.209 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:23:29.209 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:29.209 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:29.209 | INFO | Uploaded 36GiB to GPU in 2.14016s (16.8212GiB/s) | 20260505 16:23:29.209 | INFO | Downloaded 34GiB to GPU in 2.05627s (16.5348GiB/s) | 20260505 16:23:34.344 | INFO | Analyzing chunk frames = [4614656,4615168) | 20260505 16:23:34.344 | INFO | MemInfo: { | 20260505 16:23:34.344 | INFO | "gpu" : " 17 GB", | 20260505 16:23:34.344 | INFO | "huge" : "", | 20260505 16:23:34.344 | INFO | "huge_shared" : " 27 GB", | 20260505 16:23:34.344 | INFO | "malloc" : " 14 GB", | 20260505 16:23:34.344 | INFO | "pinned" : " 29 GB" | 20260505 16:23:34.344 | INFO | } | 20260505 16:23:34.344 | INFO | Perf report for the last 5.13564 seconds: | 20260505 16:23:34.344 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:23:34.344 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:34.344 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:34.344 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:34.344 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260505 16:23:34.344 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:34.344 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:34.344 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 16:23:34.344 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:34.344 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:34.345 | INFO | Uploaded 36GiB to GPU in 2.14695s (16.768GiB/s) | 20260505 16:23:34.345 | INFO | Downloaded 34GiB to GPU in 2.03776s (16.685GiB/s) | 20260505 16:23:39.480 | INFO | Analyzing chunk frames = [4615168,4615680) | 20260505 16:23:39.480 | INFO | MemInfo: { | 20260505 16:23:39.480 | INFO | "gpu" : " 17 GB", | 20260505 16:23:39.480 | INFO | "huge" : "", | 20260505 16:23:39.480 | INFO | "huge_shared" : " 27 GB", | 20260505 16:23:39.480 | INFO | "malloc" : " 14 GB", | 20260505 16:23:39.480 | INFO | "pinned" : " 29 GB" | 20260505 16:23:39.480 | INFO | } | 20260505 16:23:39.480 | INFO | Perf report for the last 5.13619 seconds: | 20260505 16:23:39.481 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:23:39.481 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:39.481 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:39.481 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:39.481 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 16:23:39.481 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:39.481 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:39.481 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 16:23:39.481 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:39.481 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:39.481 | INFO | Uploaded 36GiB to GPU in 2.12299s (16.9572GiB/s) | 20260505 16:23:39.481 | INFO | Downloaded 34GiB to GPU in 2.17224s (15.652GiB/s) | 20260505 16:23:44.632 | INFO | Analyzing chunk frames = [4615680,4616192) | 20260505 16:23:44.632 | INFO | MemInfo: { | 20260505 16:23:44.632 | INFO | "gpu" : " 17 GB", | 20260505 16:23:44.632 | INFO | "huge" : "", | 20260505 16:23:44.632 | INFO | "huge_shared" : " 27 GB", | 20260505 16:23:44.632 | INFO | "malloc" : " 14 GB", | 20260505 16:23:44.632 | INFO | "pinned" : " 29 GB" | 20260505 16:23:44.632 | INFO | } | 20260505 16:23:44.632 | INFO | Perf report for the last 5.15144 seconds: | 20260505 16:23:44.632 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:23:44.632 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:44.632 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:44.632 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:44.632 | INFO | ANALYSIS : 63.5%, 3.0, 99.3ms, 0 | 20260505 16:23:44.632 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:44.632 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:44.632 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 16:23:44.632 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:44.632 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:44.632 | INFO | Uploaded 36GiB to GPU in 2.2295s (16.1471GiB/s) | 20260505 16:23:44.632 | INFO | Downloaded 34GiB to GPU in 2.10356s (16.1631GiB/s) | 20260505 16:23:44.767 | INFO | Starting a round of Baz IO | 20260505 16:23:47.948 | INFO | Populating RTMetrics report | 20260505 16:23:47.949 | INFO | RTMetrics report populated | 20260505 16:23:47.949 | INFO | Opening tmp RTMetrics json file | 20260505 16:23:47.963 | INFO | Serializing RTMetrics to json | 20260505 16:23:47.984 | INFO | Serialization done, swapping tmp file now... | 20260505 16:23:48.000 | INFO | RTMetrics done | 20260505 16:23:48.045 | INFO | Finished a round of Baz IO: Wrote 2.171 GiB in 3.277 seconds | 20260505 16:23:48.045 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.26% / 0.00% / 77.74% | 20260505 16:23:48.045 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.366577% / 0% / 7.15256e-05% | 20260505 16:23:48.045 | INFO | Baz IO overall: | 20260505 16:23:48.045 | INFO | BytesWritten : 2739465102629 | 20260505 16:23:48.045 | INFO | NumEvents : 1300346357600 | 20260505 16:23:49.764 | INFO | Analyzing chunk frames = [4616192,4616704) | 20260505 16:23:49.764 | INFO | MemInfo: { | 20260505 16:23:49.764 | INFO | "gpu" : " 17 GB", | 20260505 16:23:49.764 | INFO | "huge" : "", | 20260505 16:23:49.764 | INFO | "huge_shared" : " 27 GB", | 20260505 16:23:49.764 | INFO | "malloc" : " 14 GB", | 20260505 16:23:49.764 | INFO | "pinned" : " 29 GB" | 20260505 16:23:49.764 | INFO | } | 20260505 16:23:49.764 | INFO | Perf report for the last 5.13257 seconds: | 20260505 16:23:49.764 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:23:49.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:49.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:49.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:49.765 | INFO | ANALYSIS : 63.5%, 3.0, 98.6ms, 0 | 20260505 16:23:49.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:49.765 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260505 16:23:49.765 | INFO | PRE_HQ : 60.8%, 1.8, 56.4ms, 0 | 20260505 16:23:49.765 | INFO | BAZWRITER : 63.4%, 3.9, 127.2ms, 0 | 20260505 16:23:49.765 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 16:23:49.765 | INFO | Uploaded 36GiB to GPU in 2.17653s (16.5401GiB/s) | 20260505 16:23:49.765 | INFO | Downloaded 38GiB to GPU in 2.25103s (16.8811GiB/s) | 20260505 16:23:54.901 | INFO | Analyzing chunk frames = [4616704,4617216) | 20260505 16:23:54.901 | INFO | MemInfo: { | 20260505 16:23:54.901 | INFO | "gpu" : " 17 GB", | 20260505 16:23:54.901 | INFO | "huge" : "", | 20260505 16:23:54.901 | INFO | "huge_shared" : " 27 GB", | 20260505 16:23:54.901 | INFO | "malloc" : " 14 GB", | 20260505 16:23:54.901 | INFO | "pinned" : " 29 GB" | 20260505 16:23:54.901 | INFO | } | 20260505 16:23:54.901 | INFO | Perf report for the last 5.13634 seconds: | 20260505 16:23:54.901 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:23:54.901 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:54.901 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:54.901 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:54.901 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 16:23:54.901 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:54.901 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:54.901 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:23:54.901 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:54.901 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:23:54.901 | INFO | Uploaded 36GiB to GPU in 2.1341s (16.8689GiB/s) | 20260505 16:23:54.901 | INFO | Downloaded 34GiB to GPU in 2.03886s (16.676GiB/s) | 20260505 16:24:00.039 | INFO | Analyzing chunk frames = [4617216,4617728) | 20260505 16:24:00.039 | INFO | MemInfo: { | 20260505 16:24:00.039 | INFO | "gpu" : " 17 GB", | 20260505 16:24:00.039 | INFO | "huge" : "", | 20260505 16:24:00.039 | INFO | "huge_shared" : " 27 GB", | 20260505 16:24:00.039 | INFO | "malloc" : " 14 GB", | 20260505 16:24:00.039 | INFO | "pinned" : " 29 GB" | 20260505 16:24:00.039 | INFO | } | 20260505 16:24:00.039 | INFO | Perf report for the last 5.13788 seconds: | 20260505 16:24:00.039 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:24:00.039 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:00.039 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:00.039 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:00.039 | INFO | ANALYSIS : 62.8%, 3.0, 97.6ms, 0 | 20260505 16:24:00.039 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:00.039 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:00.039 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 16:24:00.039 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:00.039 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:00.039 | INFO | Uploaded 36GiB to GPU in 2.18485s (16.4771GiB/s) | 20260505 16:24:00.039 | INFO | Downloaded 34GiB to GPU in 2.02539s (16.7869GiB/s) | 20260505 16:24:05.176 | INFO | Analyzing chunk frames = [4617728,4618240) | 20260505 16:24:05.176 | INFO | MemInfo: { | 20260505 16:24:05.176 | INFO | "gpu" : " 17 GB", | 20260505 16:24:05.176 | INFO | "huge" : "", | 20260505 16:24:05.176 | INFO | "huge_shared" : " 27 GB", | 20260505 16:24:05.176 | INFO | "malloc" : " 14 GB", | 20260505 16:24:05.176 | INFO | "pinned" : " 29 GB" | 20260505 16:24:05.176 | INFO | } | 20260505 16:24:05.176 | INFO | Perf report for the last 5.13715 seconds: | 20260505 16:24:05.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:24:05.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:05.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:05.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:05.176 | INFO | ANALYSIS : 62.6%, 3.0, 97.3ms, 0 | 20260505 16:24:05.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:05.176 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:05.176 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:24:05.176 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:05.176 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:05.176 | INFO | Uploaded 36GiB to GPU in 2.18793s (16.4539GiB/s) | 20260505 16:24:05.176 | INFO | Downloaded 34GiB to GPU in 2.13149s (15.9513GiB/s) | 20260505 16:24:10.326 | INFO | Analyzing chunk frames = [4618240,4618752) | 20260505 16:24:10.326 | INFO | MemInfo: { | 20260505 16:24:10.326 | INFO | "gpu" : " 17 GB", | 20260505 16:24:10.326 | INFO | "huge" : "", | 20260505 16:24:10.326 | INFO | "huge_shared" : " 27 GB", | 20260505 16:24:10.326 | INFO | "malloc" : " 14 GB", | 20260505 16:24:10.326 | INFO | "pinned" : " 29 GB" | 20260505 16:24:10.326 | INFO | } | 20260505 16:24:10.326 | INFO | Perf report for the last 5.14977 seconds: | 20260505 16:24:10.326 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:24:10.326 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:10.326 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:10.326 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:10.326 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260505 16:24:10.326 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:10.326 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:10.326 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:24:10.326 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:10.326 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:10.326 | INFO | Uploaded 36GiB to GPU in 2.13597s (16.8542GiB/s) | 20260505 16:24:10.326 | INFO | Downloaded 34GiB to GPU in 2.05267s (16.5638GiB/s) | 20260505 16:24:15.467 | INFO | Analyzing chunk frames = [4618752,4619264) | 20260505 16:24:15.467 | INFO | MemInfo: { | 20260505 16:24:15.467 | INFO | "gpu" : " 17 GB", | 20260505 16:24:15.467 | INFO | "huge" : "", | 20260505 16:24:15.467 | INFO | "huge_shared" : " 27 GB", | 20260505 16:24:15.467 | INFO | "malloc" : " 14 GB", | 20260505 16:24:15.468 | INFO | "pinned" : " 29 GB" | 20260505 16:24:15.468 | INFO | } | 20260505 16:24:15.468 | INFO | Perf report for the last 5.14192 seconds: | 20260505 16:24:15.468 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:24:15.468 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:15.468 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:15.468 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:15.468 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260505 16:24:15.468 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:15.468 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:15.468 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260505 16:24:15.468 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:15.468 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:15.468 | INFO | Uploaded 36GiB to GPU in 2.14268s (16.8014GiB/s) | 20260505 16:24:15.468 | INFO | Downloaded 34GiB to GPU in 2.0406s (16.6618GiB/s) | 20260505 16:24:20.601 | INFO | Analyzing chunk frames = [4619264,4619776) | 20260505 16:24:20.601 | INFO | MemInfo: { | 20260505 16:24:20.601 | INFO | "gpu" : " 17 GB", | 20260505 16:24:20.601 | INFO | "huge" : "", | 20260505 16:24:20.601 | INFO | "huge_shared" : " 27 GB", | 20260505 16:24:20.601 | INFO | "malloc" : " 14 GB", | 20260505 16:24:20.601 | INFO | "pinned" : " 29 GB" | 20260505 16:24:20.601 | INFO | } | 20260505 16:24:20.601 | INFO | Perf report for the last 5.13354 seconds: | 20260505 16:24:20.601 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:24:20.601 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:20.601 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:20.601 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:20.601 | INFO | ANALYSIS : 63.5%, 3.0, 99.0ms, 0 | 20260505 16:24:20.601 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:20.601 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:20.601 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 16:24:20.601 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:20.601 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:20.601 | INFO | Uploaded 36GiB to GPU in 2.14306s (16.7984GiB/s) | 20260505 16:24:20.601 | INFO | Downloaded 34GiB to GPU in 2.14256s (15.8688GiB/s) | 20260505 16:24:25.734 | INFO | Analyzing chunk frames = [4619776,4620288) | 20260505 16:24:25.734 | INFO | MemInfo: { | 20260505 16:24:25.734 | INFO | "gpu" : " 17 GB", | 20260505 16:24:25.734 | INFO | "huge" : "", | 20260505 16:24:25.734 | INFO | "huge_shared" : " 27 GB", | 20260505 16:24:25.734 | INFO | "malloc" : " 14 GB", | 20260505 16:24:25.734 | INFO | "pinned" : " 29 GB" | 20260505 16:24:25.734 | INFO | } | 20260505 16:24:25.734 | INFO | Perf report for the last 5.13314 seconds: | 20260505 16:24:25.734 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:24:25.734 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:25.734 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:25.734 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:25.734 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 16:24:25.734 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:25.734 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:25.734 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 16:24:25.734 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:25.735 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:25.735 | INFO | Uploaded 36GiB to GPU in 2.22859s (16.1537GiB/s) | 20260505 16:24:25.735 | INFO | Downloaded 34GiB to GPU in 2.10659s (16.1398GiB/s) | 20260505 16:24:25.735 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 16:24:25.735 | INFO | Total mem usage: 15.3 GiB | 20260505 16:24:25.735 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 16:24:25.735 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.88%) | 20260505 16:24:25.735 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 16:24:25.735 | INFO | High Water Breakdown: | 20260505 16:24:25.735 | INFO | 61% : 8.67 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 16:24:25.735 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 16:24:25.735 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 16:24:25.735 | INFO | ------------------------------------------ | 20260505 16:24:25.735 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:24:25.735 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 16:24:25.735 | INFO | Total mem usage: 26.5 GiB | 20260505 16:24:25.735 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 16:24:25.735 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 16:24:25.735 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 16:24:25.735 | INFO | High Water Breakdown: | 20260505 16:24:25.735 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:24:25.735 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:24:25.735 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:24:25.735 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:24:25.735 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:24:25.735 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:24:25.735 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 16:24:25.735 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 16:24:25.735 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:24:25.735 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 16:24:25.735 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 16:24:25.735 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 16:24:25.735 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 16:24:25.735 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 16:24:25.735 | INFO | ------------------------------------------ | 20260505 16:24:25.735 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:24:25.735 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 16:24:25.735 | INFO | Total mem usage: 25.6 GiB | 20260505 16:24:25.735 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 16:24:25.735 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 16:24:25.735 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 16:24:25.735 | INFO | High Water Breakdown: | 20260505 16:24:25.735 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 16:24:25.735 | INFO | ------------------------------------------ | 20260505 16:24:25.735 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:24:25.735 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 16:24:25.735 | INFO | Total mem usage: 15.7 GiB | 20260505 16:24:25.735 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 16:24:25.735 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 16:24:25.735 | INFO | Unused Cache: 871 MiB (Waste: 5.43%) | 20260505 16:24:25.735 | INFO | High Water Breakdown: | 20260505 16:24:25.735 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 16:24:25.735 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 16:24:25.735 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 16:24:25.735 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 16:24:25.735 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 16:24:25.735 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 16:24:25.735 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 16:24:25.735 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 16:24:25.735 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 16:24:25.736 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:24:25.736 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 16:24:25.736 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:24:25.736 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 16:24:25.736 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 16:24:25.736 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 16:24:25.736 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 16:24:25.736 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:24:25.736 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 16:24:25.736 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:24:25.736 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:24:25.736 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:24:25.736 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 16:24:25.736 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 16:24:25.736 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:24:25.736 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 16:24:25.736 | INFO | ------------------------------------------ | 20260505 16:24:25.736 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:24:25.736 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 16:24:25.736 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:24:25.736 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:24:25.736 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.57e+01, 1.37e+01 / 1.88e+00 | 20260505 16:24:25.736 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 1.03e-01, 1.07e-02 / 4.92e-03 | 20260505 16:24:25.736 | INFO | PerfReport: 11.79%, Baseline -- 3.74e+00 / 4.51e+00, 5.83e-01 / 1.51e-01 | 20260505 16:24:25.736 | INFO | PerfReport: 3.29%, Histogram -- 1.05e+00 / 1.51e+00, 1.96e-01 / 8.33e-02 | 20260505 16:24:25.736 | INFO | PerfReport: 15.90%, DME -- 4.04e+01 / 4.50e+01, 4.34e+00 / 1.85e+00 | 20260505 16:24:25.736 | INFO | PerfReport: 56.86%, FrameLabeling -- 1.81e+01 / 2.00e+01, 1.57e+00 / 1.78e+00 | 20260505 16:24:25.736 | INFO | PerfReport: 7.28%, PulseAccumulating -- 2.31e+00 / 2.85e+00, 4.37e-01 / 2.18e-01 | 20260505 16:24:25.736 | INFO | PerfReport: 3.40%, Metrics -- 1.08e+00 / 1.41e+00, 3.00e-01 / 1.88e-01 | 20260505 16:24:25.736 | INFO | PerfReport: 1.19%, PulseSerializing -- 3.77e-01 / 4.24e-01, 6.78e-02 / 3.32e-02 | 20260505 16:24:25.736 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 16:24:25.736 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:24:25.736 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:24:25.736 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260505 16:24:25.736 | INFO | PerfReport: 22.51%, Upload -- 2.26e+01 / 0.00e+00, 7.90e+00 / 0.00e+00 | 20260505 16:24:25.736 | INFO | PerfReport: 21.85%, Download -- 2.19e+01 / 0.00e+00, 7.84e+00 / 0.00e+00 | 20260505 16:24:25.736 | INFO | PerfReport: 31.71%, Compute -- 3.18e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 16:24:25.736 | INFO | PerfReport: 2.21%, LockUpload -- 2.22e+00 / 0.00e+00, 7.79e+00 / 0.00e+00 | 20260505 16:24:25.736 | INFO | PerfReport: 1.35%, LockDownload -- 1.35e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260505 16:24:25.736 | INFO | PerfReport: 20.35%, LockCompute -- 2.04e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 16:24:25.736 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.05e-02 / 0.00e+00 | 20260505 16:24:25.867 | INFO | Starting a round of Baz IO | 20260505 16:24:29.077 | INFO | Populating RTMetrics report | 20260505 16:24:29.077 | INFO | RTMetrics report populated | 20260505 16:24:29.077 | INFO | Opening tmp RTMetrics json file | 20260505 16:24:29.119 | INFO | Serializing RTMetrics to json | 20260505 16:24:29.124 | INFO | Serialization done, swapping tmp file now... | 20260505 16:24:29.125 | INFO | RTMetrics done | 20260505 16:24:29.180 | INFO | Finished a round of Baz IO: Wrote 2.171 GiB in 3.312 seconds | 20260505 16:24:29.180 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.25% / 0.00% / 77.74% | 20260505 16:24:29.180 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.367661% / 0% / 7.94729e-05% | 20260505 16:24:29.180 | INFO | Baz IO overall: | 20260505 16:24:29.180 | INFO | BytesWritten : 2741796594936 | 20260505 16:24:29.180 | INFO | NumEvents : 1301439303546 | 20260505 16:24:30.883 | INFO | Analyzing chunk frames = [4620288,4620800) | 20260505 16:24:30.883 | INFO | MemInfo: { | 20260505 16:24:30.883 | INFO | "gpu" : " 17 GB", | 20260505 16:24:30.883 | INFO | "huge" : "", | 20260505 16:24:30.883 | INFO | "huge_shared" : " 27 GB", | 20260505 16:24:30.884 | INFO | "malloc" : " 14 GB", | 20260505 16:24:30.884 | INFO | "pinned" : " 29 GB" | 20260505 16:24:30.884 | INFO | } | 20260505 16:24:30.884 | INFO | Perf report for the last 5.14929 seconds: | 20260505 16:24:30.884 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:24:30.884 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:30.884 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:30.884 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:30.884 | INFO | ANALYSIS : 63.8%, 3.0, 99.5ms, 0 | 20260505 16:24:30.884 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:30.884 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260505 16:24:30.884 | INFO | PRE_HQ : 60.5%, 1.8, 56.9ms, 0 | 20260505 16:24:30.884 | INFO | BAZWRITER : 63.7%, 3.9, 127.9ms, 0 | 20260505 16:24:30.884 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 16:24:30.884 | INFO | Uploaded 36GiB to GPU in 2.19085s (16.432GiB/s) | 20260505 16:24:30.884 | INFO | Downloaded 38GiB to GPU in 2.26676s (16.764GiB/s) | 20260505 16:24:36.029 | INFO | Analyzing chunk frames = [4620800,4621312) | 20260505 16:24:36.029 | INFO | MemInfo: { | 20260505 16:24:36.029 | INFO | "gpu" : " 17 GB", | 20260505 16:24:36.029 | INFO | "huge" : "", | 20260505 16:24:36.029 | INFO | "huge_shared" : " 27 GB", | 20260505 16:24:36.029 | INFO | "malloc" : " 14 GB", | 20260505 16:24:36.029 | INFO | "pinned" : " 29 GB" | 20260505 16:24:36.029 | INFO | } | 20260505 16:24:36.029 | INFO | Perf report for the last 5.1456 seconds: | 20260505 16:24:36.029 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:24:36.029 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:36.029 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:36.029 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:36.029 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 16:24:36.029 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:36.029 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:36.029 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 16:24:36.029 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:36.029 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:36.029 | INFO | Uploaded 36GiB to GPU in 2.12856s (16.9128GiB/s) | 20260505 16:24:36.030 | INFO | Downloaded 34GiB to GPU in 2.02272s (16.809GiB/s) | 20260505 16:24:41.158 | INFO | Analyzing chunk frames = [4621312,4621824) | 20260505 16:24:41.158 | INFO | MemInfo: { | 20260505 16:24:41.158 | INFO | "gpu" : " 17 GB", | 20260505 16:24:41.158 | INFO | "huge" : "", | 20260505 16:24:41.158 | INFO | "huge_shared" : " 27 GB", | 20260505 16:24:41.158 | INFO | "malloc" : " 14 GB", | 20260505 16:24:41.158 | INFO | "pinned" : " 29 GB" | 20260505 16:24:41.158 | INFO | } | 20260505 16:24:41.158 | INFO | Perf report for the last 5.1291 seconds: | 20260505 16:24:41.158 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:24:41.158 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:41.158 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:41.158 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:41.158 | INFO | ANALYSIS : 62.9%, 3.0, 97.7ms, 0 | 20260505 16:24:41.158 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:41.158 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:41.158 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 16:24:41.158 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:41.158 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:41.159 | INFO | Uploaded 36GiB to GPU in 2.2099s (16.2904GiB/s) | 20260505 16:24:41.159 | INFO | Downloaded 34GiB to GPU in 2.06764s (16.4438GiB/s) | 20260505 16:24:46.293 | INFO | Analyzing chunk frames = [4621824,4622336) | 20260505 16:24:46.293 | INFO | MemInfo: { | 20260505 16:24:46.293 | INFO | "gpu" : " 17 GB", | 20260505 16:24:46.293 | INFO | "huge" : "", | 20260505 16:24:46.293 | INFO | "huge_shared" : " 27 GB", | 20260505 16:24:46.293 | INFO | "malloc" : " 14 GB", | 20260505 16:24:46.293 | INFO | "pinned" : " 29 GB" | 20260505 16:24:46.293 | INFO | } | 20260505 16:24:46.293 | INFO | Perf report for the last 5.13476 seconds: | 20260505 16:24:46.293 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:24:46.293 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:46.293 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:46.293 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:46.293 | INFO | ANALYSIS : 62.6%, 3.0, 97.2ms, 0 | 20260505 16:24:46.293 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:46.293 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:46.293 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 16:24:46.293 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:46.293 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:46.293 | INFO | Uploaded 36GiB to GPU in 2.18923s (16.4441GiB/s) | 20260505 16:24:46.293 | INFO | Downloaded 34GiB to GPU in 2.14279s (15.8671GiB/s) | 20260505 16:24:51.434 | INFO | Analyzing chunk frames = [4622336,4622848) | 20260505 16:24:51.434 | INFO | MemInfo: { | 20260505 16:24:51.434 | INFO | "gpu" : " 17 GB", | 20260505 16:24:51.434 | INFO | "huge" : "", | 20260505 16:24:51.434 | INFO | "huge_shared" : " 27 GB", | 20260505 16:24:51.434 | INFO | "malloc" : " 14 GB", | 20260505 16:24:51.434 | INFO | "pinned" : " 29 GB" | 20260505 16:24:51.434 | INFO | } | 20260505 16:24:51.434 | INFO | Perf report for the last 5.14088 seconds: | 20260505 16:24:51.434 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:24:51.434 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:51.434 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:51.434 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:51.434 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 16:24:51.434 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:51.434 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:51.434 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 16:24:51.434 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:51.434 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:51.434 | INFO | Uploaded 36GiB to GPU in 2.19085s (16.432GiB/s) | 20260505 16:24:51.434 | INFO | Downloaded 34GiB to GPU in 2.13172s (15.9496GiB/s) | 20260505 16:24:56.578 | INFO | Analyzing chunk frames = [4622848,4623360) | 20260505 16:24:56.578 | INFO | MemInfo: { | 20260505 16:24:56.578 | INFO | "gpu" : " 17 GB", | 20260505 16:24:56.578 | INFO | "huge" : "", | 20260505 16:24:56.578 | INFO | "huge_shared" : " 27 GB", | 20260505 16:24:56.578 | INFO | "malloc" : " 14 GB", | 20260505 16:24:56.579 | INFO | "pinned" : " 29 GB" | 20260505 16:24:56.579 | INFO | } | 20260505 16:24:56.579 | INFO | Perf report for the last 5.14472 seconds: | 20260505 16:24:56.579 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:24:56.579 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:56.579 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:56.579 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:56.579 | INFO | ANALYSIS : 62.4%, 3.0, 97.6ms, 0 | 20260505 16:24:56.579 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:56.579 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:56.579 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:24:56.579 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:56.579 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:24:56.579 | INFO | Uploaded 36GiB to GPU in 2.1398s (16.824GiB/s) | 20260505 16:24:56.579 | INFO | Downloaded 34GiB to GPU in 2.03292s (16.7247GiB/s) | 20260505 16:25:01.716 | INFO | Analyzing chunk frames = [4623360,4623872) | 20260505 16:25:01.716 | INFO | MemInfo: { | 20260505 16:25:01.716 | INFO | "gpu" : " 17 GB", | 20260505 16:25:01.716 | INFO | "huge" : "", | 20260505 16:25:01.716 | INFO | "huge_shared" : " 27 GB", | 20260505 16:25:01.716 | INFO | "malloc" : " 14 GB", | 20260505 16:25:01.716 | INFO | "pinned" : " 29 GB" | 20260505 16:25:01.716 | INFO | } | 20260505 16:25:01.716 | INFO | Perf report for the last 5.13765 seconds: | 20260505 16:25:01.716 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:25:01.716 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:01.716 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:01.716 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:01.716 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 16:25:01.716 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:01.716 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:01.716 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 16:25:01.716 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:01.716 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:01.717 | INFO | Uploaded 36GiB to GPU in 2.13941s (16.8271GiB/s) | 20260505 16:25:01.717 | INFO | Downloaded 34GiB to GPU in 2.14372s (15.8603GiB/s) | 20260505 16:25:06.853 | INFO | Analyzing chunk frames = [4623872,4624384) | 20260505 16:25:06.853 | INFO | MemInfo: { | 20260505 16:25:06.853 | INFO | "gpu" : " 17 GB", | 20260505 16:25:06.853 | INFO | "huge" : "", | 20260505 16:25:06.853 | INFO | "huge_shared" : " 27 GB", | 20260505 16:25:06.853 | INFO | "malloc" : " 14 GB", | 20260505 16:25:06.853 | INFO | "pinned" : " 29 GB" | 20260505 16:25:06.853 | INFO | } | 20260505 16:25:06.853 | INFO | Perf report for the last 5.13669 seconds: | 20260505 16:25:06.853 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:25:06.853 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:06.853 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:06.853 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:06.853 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 16:25:06.853 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:06.853 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:06.853 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 16:25:06.853 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:06.853 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:06.853 | INFO | Uploaded 36GiB to GPU in 2.15299s (16.721GiB/s) | 20260505 16:25:06.853 | INFO | Downloaded 34GiB to GPU in 2.01817s (16.847GiB/s) | 20260505 16:25:06.984 | INFO | Starting a round of Baz IO | 20260505 16:25:10.201 | INFO | Populating RTMetrics report | 20260505 16:25:10.201 | INFO | RTMetrics report populated | 20260505 16:25:10.201 | INFO | Opening tmp RTMetrics json file | 20260505 16:25:10.219 | INFO | Serializing RTMetrics to json | 20260505 16:25:10.240 | INFO | Serialization done, swapping tmp file now... | 20260505 16:25:10.262 | INFO | RTMetrics done | 20260505 16:25:10.300 | INFO | Finished a round of Baz IO: Wrote 2.172 GiB in 3.316 seconds | 20260505 16:25:10.300 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.25% / 0.00% / 77.75% | 20260505 16:25:10.300 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.365973% / 3.97364e-06% / 7.94729e-05% | 20260505 16:25:10.300 | INFO | Baz IO overall: | 20260505 16:25:10.300 | INFO | BytesWritten : 2744128567501 | 20260505 16:25:10.300 | INFO | NumEvents : 1302532828427 | 20260505 16:25:11.989 | INFO | Analyzing chunk frames = [4624384,4624896) | 20260505 16:25:11.989 | INFO | MemInfo: { | 20260505 16:25:11.989 | INFO | "gpu" : " 17 GB", | 20260505 16:25:11.989 | INFO | "huge" : "", | 20260505 16:25:11.989 | INFO | "huge_shared" : " 27 GB", | 20260505 16:25:11.989 | INFO | "malloc" : " 14 GB", | 20260505 16:25:11.989 | INFO | "pinned" : " 29 GB" | 20260505 16:25:11.989 | INFO | } | 20260505 16:25:11.989 | INFO | Perf report for the last 5.13572 seconds: | 20260505 16:25:11.989 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:25:11.989 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:11.989 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:11.989 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:11.989 | INFO | ANALYSIS : 64.1%, 3.0, 99.7ms, 0 | 20260505 16:25:11.989 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:11.989 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260505 16:25:11.989 | INFO | PRE_HQ : 60.8%, 1.8, 56.5ms, 0 | 20260505 16:25:11.989 | INFO | BAZWRITER : 63.9%, 3.9, 126.9ms, 0 | 20260505 16:25:11.989 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 16:25:11.989 | INFO | Uploaded 36GiB to GPU in 2.20411s (16.3331GiB/s) | 20260505 16:25:11.989 | INFO | Downloaded 38GiB to GPU in 2.25891s (16.8222GiB/s) | 20260505 16:25:17.136 | INFO | Analyzing chunk frames = [4624896,4625408) | 20260505 16:25:17.136 | INFO | MemInfo: { | 20260505 16:25:17.136 | INFO | "gpu" : " 17 GB", | 20260505 16:25:17.136 | INFO | "huge" : "", | 20260505 16:25:17.136 | INFO | "huge_shared" : " 27 GB", | 20260505 16:25:17.136 | INFO | "malloc" : " 14 GB", | 20260505 16:25:17.136 | INFO | "pinned" : " 29 GB" | 20260505 16:25:17.136 | INFO | } | 20260505 16:25:17.136 | INFO | Perf report for the last 5.14738 seconds: | 20260505 16:25:17.136 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:25:17.136 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:17.136 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:17.136 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:17.136 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 16:25:17.136 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:17.136 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:17.136 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:25:17.136 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:17.136 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:17.136 | INFO | Uploaded 36GiB to GPU in 2.20806s (16.3039GiB/s) | 20260505 16:25:17.136 | INFO | Downloaded 34GiB to GPU in 2.07393s (16.394GiB/s) | 20260505 16:25:22.280 | INFO | Analyzing chunk frames = [4625408,4625920) | 20260505 16:25:22.280 | INFO | MemInfo: { | 20260505 16:25:22.280 | INFO | "gpu" : " 17 GB", | 20260505 16:25:22.280 | INFO | "huge" : "", | 20260505 16:25:22.280 | INFO | "huge_shared" : " 27 GB", | 20260505 16:25:22.280 | INFO | "malloc" : " 14 GB", | 20260505 16:25:22.280 | INFO | "pinned" : " 29 GB" | 20260505 16:25:22.280 | INFO | } | 20260505 16:25:22.280 | INFO | Perf report for the last 5.14387 seconds: | 20260505 16:25:22.280 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:25:22.280 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:22.280 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:22.280 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:22.280 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 16:25:22.280 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:22.280 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:22.280 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:25:22.280 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:22.280 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:22.280 | INFO | Uploaded 36GiB to GPU in 2.19997s (16.3639GiB/s) | 20260505 16:25:22.280 | INFO | Downloaded 34GiB to GPU in 2.03575s (16.7014GiB/s) | 20260505 16:25:27.413 | INFO | Analyzing chunk frames = [4625920,4626432) | 20260505 16:25:27.413 | INFO | MemInfo: { | 20260505 16:25:27.413 | INFO | "gpu" : " 17 GB", | 20260505 16:25:27.413 | INFO | "huge" : "", | 20260505 16:25:27.413 | INFO | "huge_shared" : " 27 GB", | 20260505 16:25:27.413 | INFO | "malloc" : " 14 GB", | 20260505 16:25:27.413 | INFO | "pinned" : " 29 GB" | 20260505 16:25:27.413 | INFO | } | 20260505 16:25:27.413 | INFO | Perf report for the last 5.13301 seconds: | 20260505 16:25:27.413 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:25:27.413 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:27.413 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:27.413 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:27.413 | INFO | ANALYSIS : 62.6%, 3.0, 97.1ms, 0 | 20260505 16:25:27.413 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:27.413 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:27.413 | INFO | PRE_HQ : 27.9%, 1.0, 14.4ms, 0 | 20260505 16:25:27.413 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:27.413 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:27.413 | INFO | Uploaded 36GiB to GPU in 2.18522s (16.4743GiB/s) | 20260505 16:25:27.414 | INFO | Downloaded 34GiB to GPU in 2.1379s (15.9035GiB/s) | 20260505 16:25:32.545 | INFO | Analyzing chunk frames = [4626432,4626944) | 20260505 16:25:32.545 | INFO | MemInfo: { | 20260505 16:25:32.545 | INFO | "gpu" : " 17 GB", | 20260505 16:25:32.545 | INFO | "huge" : "", | 20260505 16:25:32.545 | INFO | "huge_shared" : " 27 GB", | 20260505 16:25:32.545 | INFO | "malloc" : " 14 GB", | 20260505 16:25:32.546 | INFO | "pinned" : " 29 GB" | 20260505 16:25:32.546 | INFO | } | 20260505 16:25:32.546 | INFO | Perf report for the last 5.1326 seconds: | 20260505 16:25:32.546 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:25:32.546 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:32.546 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:32.546 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:32.546 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260505 16:25:32.546 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:32.546 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:32.546 | INFO | PRE_HQ : 28.1%, 1.0, 14.5ms, 0 | 20260505 16:25:32.546 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:32.546 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:32.546 | INFO | Uploaded 36GiB to GPU in 2.16297s (16.6438GiB/s) | 20260505 16:25:32.546 | INFO | Downloaded 34GiB to GPU in 2.06044s (16.5013GiB/s) | 20260505 16:25:37.691 | INFO | Analyzing chunk frames = [4626944,4627456) | 20260505 16:25:37.691 | INFO | MemInfo: { | 20260505 16:25:37.691 | INFO | "gpu" : " 17 GB", | 20260505 16:25:37.691 | INFO | "huge" : "", | 20260505 16:25:37.691 | INFO | "huge_shared" : " 27 GB", | 20260505 16:25:37.692 | INFO | "malloc" : " 14 GB", | 20260505 16:25:37.692 | INFO | "pinned" : " 29 GB" | 20260505 16:25:37.692 | INFO | } | 20260505 16:25:37.692 | INFO | Perf report for the last 5.14604 seconds: | 20260505 16:25:37.692 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:25:37.692 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:37.692 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:37.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:37.692 | INFO | ANALYSIS : 62.4%, 3.0, 97.6ms, 0 | 20260505 16:25:37.692 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:37.692 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:37.692 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260505 16:25:37.692 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:37.692 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:37.692 | INFO | Uploaded 36GiB to GPU in 2.15244s (16.7252GiB/s) | 20260505 16:25:37.692 | INFO | Downloaded 34GiB to GPU in 2.0354s (16.7044GiB/s) | 20260505 16:25:42.830 | INFO | Analyzing chunk frames = [4627456,4627968) | 20260505 16:25:42.830 | INFO | MemInfo: { | 20260505 16:25:42.830 | INFO | "gpu" : " 17 GB", | 20260505 16:25:42.830 | INFO | "huge" : "", | 20260505 16:25:42.830 | INFO | "huge_shared" : " 27 GB", | 20260505 16:25:42.830 | INFO | "malloc" : " 14 GB", | 20260505 16:25:42.830 | INFO | "pinned" : " 29 GB" | 20260505 16:25:42.830 | INFO | } | 20260505 16:25:42.831 | INFO | Perf report for the last 5.13879 seconds: | 20260505 16:25:42.831 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:25:42.831 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:42.831 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:42.831 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:42.831 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 16:25:42.831 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:42.831 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:42.831 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 16:25:42.831 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:42.831 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:42.831 | INFO | Uploaded 36GiB to GPU in 2.15549s (16.7015GiB/s) | 20260505 16:25:42.831 | INFO | Downloaded 34GiB to GPU in 2.17326s (15.6447GiB/s) | 20260505 16:25:47.976 | INFO | Analyzing chunk frames = [4627968,4628480) | 20260505 16:25:47.976 | INFO | MemInfo: { | 20260505 16:25:47.976 | INFO | "gpu" : " 17 GB", | 20260505 16:25:47.976 | INFO | "huge" : "", | 20260505 16:25:47.976 | INFO | "huge_shared" : " 27 GB", | 20260505 16:25:47.976 | INFO | "malloc" : " 14 GB", | 20260505 16:25:47.976 | INFO | "pinned" : " 29 GB" | 20260505 16:25:47.976 | INFO | } | 20260505 16:25:47.976 | INFO | Perf report for the last 5.14583 seconds: | 20260505 16:25:47.976 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:25:47.976 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:47.976 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:47.976 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:47.976 | INFO | ANALYSIS : 62.9%, 3.0, 98.4ms, 0 | 20260505 16:25:47.976 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:47.977 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:47.977 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 16:25:47.977 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:47.977 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:47.977 | INFO | Uploaded 36GiB to GPU in 2.19181s (16.4247GiB/s) | 20260505 16:25:47.977 | INFO | Downloaded 34GiB to GPU in 2.02913s (16.7559GiB/s) | 20260505 16:25:47.977 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 16:25:47.977 | INFO | Total mem usage: 15.3 GiB | 20260505 16:25:47.977 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 16:25:47.977 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.88%) | 20260505 16:25:47.977 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 16:25:47.977 | INFO | High Water Breakdown: | 20260505 16:25:47.977 | INFO | 61% : 8.67 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 16:25:47.977 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 16:25:47.977 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 16:25:47.977 | INFO | ------------------------------------------ | 20260505 16:25:47.977 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:25:47.977 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 16:25:47.977 | INFO | Total mem usage: 26.5 GiB | 20260505 16:25:47.977 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 16:25:47.977 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 16:25:47.977 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 16:25:47.977 | INFO | High Water Breakdown: | 20260505 16:25:47.977 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:25:47.977 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:25:47.977 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:25:47.977 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:25:47.977 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:25:47.977 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:25:47.977 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 16:25:47.977 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 16:25:47.977 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:25:47.977 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 16:25:47.977 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 16:25:47.977 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 16:25:47.977 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 16:25:47.977 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 16:25:47.977 | INFO | ------------------------------------------ | 20260505 16:25:47.977 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:25:47.977 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 16:25:47.977 | INFO | Total mem usage: 25.6 GiB | 20260505 16:25:47.977 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 16:25:47.977 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 16:25:47.977 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 16:25:47.978 | INFO | High Water Breakdown: | 20260505 16:25:47.978 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 16:25:47.978 | INFO | ------------------------------------------ | 20260505 16:25:47.978 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:25:47.978 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 16:25:47.978 | INFO | Total mem usage: 15.7 GiB | 20260505 16:25:47.978 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 16:25:47.978 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 16:25:47.978 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 16:25:47.978 | INFO | High Water Breakdown: | 20260505 16:25:47.978 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 16:25:47.978 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 16:25:47.978 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 16:25:47.978 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 16:25:47.978 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 16:25:47.978 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 16:25:47.978 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 16:25:47.978 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 16:25:47.978 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 16:25:47.978 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:25:47.978 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 16:25:47.978 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:25:47.978 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 16:25:47.978 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 16:25:47.978 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 16:25:47.978 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 16:25:47.978 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:25:47.978 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 16:25:47.978 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:25:47.978 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:25:47.978 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:25:47.978 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 16:25:47.978 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 16:25:47.978 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:25:47.978 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 16:25:47.978 | INFO | ------------------------------------------ | 20260505 16:25:47.978 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:25:47.978 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 16:25:47.978 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:25:47.978 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:25:47.978 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.54e+01, 1.36e+01 / 1.78e+00 | 20260505 16:25:47.978 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 1.01e-01, 1.15e-02 / 3.96e-03 | 20260505 16:25:47.978 | INFO | PerfReport: 11.78%, Baseline -- 3.73e+00 / 4.51e+00, 5.77e-01 / 1.61e-01 | 20260505 16:25:47.978 | INFO | PerfReport: 3.27%, Histogram -- 1.04e+00 / 1.51e+00, 1.91e-01 / 9.94e-02 | 20260505 16:25:47.978 | INFO | PerfReport: 16.02%, DME -- 4.05e+01 / 4.55e+01, 4.31e+00 / 1.95e+00 | 20260505 16:25:47.978 | INFO | PerfReport: 56.64%, FrameLabeling -- 1.79e+01 / 1.90e+01, 1.41e+00 / 9.89e-01 | 20260505 16:25:47.978 | INFO | PerfReport: 7.39%, PulseAccumulating -- 2.34e+00 / 2.95e+00, 4.51e-01 / 1.35e-01 | 20260505 16:25:47.978 | INFO | PerfReport: 3.41%, Metrics -- 1.08e+00 / 1.44e+00, 2.99e-01 / 1.47e-01 | 20260505 16:25:47.978 | INFO | PerfReport: 1.19%, PulseSerializing -- 3.77e-01 / 4.42e-01, 6.80e-02 / 4.12e-02 | 20260505 16:25:47.978 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 16:25:47.978 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:25:47.978 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:25:47.978 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.59e+01 / 0.00e+00 | 20260505 16:25:47.978 | INFO | PerfReport: 22.60%, Upload -- 2.27e+01 / 0.00e+00, 7.91e+00 / 0.00e+00 | 20260505 16:25:47.978 | INFO | PerfReport: 21.84%, Download -- 2.19e+01 / 0.00e+00, 7.90e+00 / 0.00e+00 | 20260505 16:25:47.978 | INFO | PerfReport: 31.57%, Compute -- 3.16e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 16:25:47.978 | INFO | PerfReport: 2.30%, LockUpload -- 2.31e+00 / 0.00e+00, 8.02e+00 / 0.00e+00 | 20260505 16:25:47.978 | INFO | PerfReport: 1.43%, LockDownload -- 1.43e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260505 16:25:47.978 | INFO | PerfReport: 20.24%, LockCompute -- 2.03e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260505 16:25:47.978 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.57e-02 / 0.00e+00 | 20260505 16:25:48.114 | INFO | Starting a round of Baz IO | 20260505 16:25:51.300 | INFO | Populating RTMetrics report | 20260505 16:25:51.300 | INFO | RTMetrics report populated | 20260505 16:25:51.300 | INFO | Opening tmp RTMetrics json file | 20260505 16:25:51.312 | INFO | Serializing RTMetrics to json | 20260505 16:25:51.315 | INFO | Serialization done, swapping tmp file now... | 20260505 16:25:51.315 | INFO | RTMetrics done | 20260505 16:25:51.424 | INFO | Finished a round of Baz IO: Wrote 2.168 GiB in 3.31 seconds | 20260505 16:25:51.424 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.25% / 0.00% / 77.75% | 20260505 16:25:51.424 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.36459% / 0% / 5.96046e-05% | 20260505 16:25:51.424 | INFO | Baz IO overall: | 20260505 16:25:51.424 | INFO | BytesWritten : 2746456873567 | 20260505 16:25:51.424 | INFO | NumEvents : 1303623786379 | 20260505 16:25:53.111 | INFO | Analyzing chunk frames = [4628480,4628992) | 20260505 16:25:53.111 | INFO | MemInfo: { | 20260505 16:25:53.111 | INFO | "gpu" : " 17 GB", | 20260505 16:25:53.111 | INFO | "huge" : "", | 20260505 16:25:53.111 | INFO | "huge_shared" : " 27 GB", | 20260505 16:25:53.111 | INFO | "malloc" : " 14 GB", | 20260505 16:25:53.111 | INFO | "pinned" : " 29 GB" | 20260505 16:25:53.111 | INFO | } | 20260505 16:25:53.111 | INFO | Perf report for the last 5.13499 seconds: | 20260505 16:25:53.111 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:25:53.111 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:53.111 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:53.111 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:53.111 | INFO | ANALYSIS : 63.6%, 3.0, 99.0ms, 0 | 20260505 16:25:53.111 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:53.112 | INFO | RT_METRICS : 44.9%, 1.0, 23.5ms, 0 | 20260505 16:25:53.112 | INFO | PRE_HQ : 60.3%, 1.8, 56.4ms, 0 | 20260505 16:25:53.112 | INFO | BAZWRITER : 63.9%, 3.9, 127.8ms, 0 | 20260505 16:25:53.112 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 16:25:53.112 | INFO | Uploaded 36GiB to GPU in 2.14465s (16.786GiB/s) | 20260505 16:25:53.112 | INFO | Downloaded 38GiB to GPU in 2.25143s (16.8782GiB/s) | 20260505 16:25:58.243 | INFO | Analyzing chunk frames = [4628992,4629504) | 20260505 16:25:58.243 | INFO | MemInfo: { | 20260505 16:25:58.243 | INFO | "gpu" : " 17 GB", | 20260505 16:25:58.243 | INFO | "huge" : "", | 20260505 16:25:58.243 | INFO | "huge_shared" : " 27 GB", | 20260505 16:25:58.244 | INFO | "malloc" : " 14 GB", | 20260505 16:25:58.244 | INFO | "pinned" : " 29 GB" | 20260505 16:25:58.244 | INFO | } | 20260505 16:25:58.244 | INFO | Perf report for the last 5.13234 seconds: | 20260505 16:25:58.244 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:25:58.244 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:58.244 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:58.244 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:58.244 | INFO | ANALYSIS : 63.1%, 3.0, 98.3ms, 0 | 20260505 16:25:58.244 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:58.244 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:58.244 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:25:58.244 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:58.244 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:25:58.244 | INFO | Uploaded 36GiB to GPU in 2.13924s (16.8284GiB/s) | 20260505 16:25:58.244 | INFO | Downloaded 34GiB to GPU in 2.04217s (16.6489GiB/s) | 20260505 16:26:03.389 | INFO | Analyzing chunk frames = [4629504,4630016) | 20260505 16:26:03.389 | INFO | MemInfo: { | 20260505 16:26:03.389 | INFO | "gpu" : " 17 GB", | 20260505 16:26:03.389 | INFO | "huge" : "", | 20260505 16:26:03.389 | INFO | "huge_shared" : " 27 GB", | 20260505 16:26:03.389 | INFO | "malloc" : " 14 GB", | 20260505 16:26:03.389 | INFO | "pinned" : " 29 GB" | 20260505 16:26:03.389 | INFO | } | 20260505 16:26:03.389 | INFO | Perf report for the last 5.14527 seconds: | 20260505 16:26:03.389 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:26:03.389 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:03.389 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:03.389 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:03.389 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260505 16:26:03.389 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:03.389 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:03.389 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:26:03.389 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:03.389 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:03.389 | INFO | Uploaded 36GiB to GPU in 2.19753s (16.3821GiB/s) | 20260505 16:26:03.389 | INFO | Downloaded 34GiB to GPU in 2.05354s (16.5568GiB/s) | 20260505 16:26:08.528 | INFO | Analyzing chunk frames = [4630016,4630528) | 20260505 16:26:08.529 | INFO | MemInfo: { | 20260505 16:26:08.529 | INFO | "gpu" : " 17 GB", | 20260505 16:26:08.529 | INFO | "huge" : "", | 20260505 16:26:08.529 | INFO | "huge_shared" : " 27 GB", | 20260505 16:26:08.529 | INFO | "malloc" : " 14 GB", | 20260505 16:26:08.529 | INFO | "pinned" : " 29 GB" | 20260505 16:26:08.529 | INFO | } | 20260505 16:26:08.529 | INFO | Perf report for the last 5.13974 seconds: | 20260505 16:26:08.529 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:26:08.529 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:08.529 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:08.529 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:08.529 | INFO | ANALYSIS : 62.6%, 3.0, 97.3ms, 0 | 20260505 16:26:08.529 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:08.529 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:08.529 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 16:26:08.529 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:08.529 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:08.529 | INFO | Uploaded 36GiB to GPU in 2.21099s (16.2823GiB/s) | 20260505 16:26:08.529 | INFO | Downloaded 34GiB to GPU in 2.13792s (15.9033GiB/s) | 20260505 16:26:13.662 | INFO | Analyzing chunk frames = [4630528,4631040) | 20260505 16:26:13.662 | INFO | MemInfo: { | 20260505 16:26:13.662 | INFO | "gpu" : " 17 GB", | 20260505 16:26:13.662 | INFO | "huge" : "", | 20260505 16:26:13.662 | INFO | "huge_shared" : " 27 GB", | 20260505 16:26:13.662 | INFO | "malloc" : " 14 GB", | 20260505 16:26:13.663 | INFO | "pinned" : " 29 GB" | 20260505 16:26:13.663 | INFO | } | 20260505 16:26:13.663 | INFO | Perf report for the last 5.13381 seconds: | 20260505 16:26:13.663 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:26:13.663 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:13.663 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:13.663 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:13.663 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 16:26:13.663 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:13.663 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:13.663 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:26:13.663 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:13.663 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:13.663 | INFO | Uploaded 36GiB to GPU in 2.12863s (16.9123GiB/s) | 20260505 16:26:13.663 | INFO | Downloaded 34GiB to GPU in 2.05198s (16.5694GiB/s) | 20260505 16:26:18.798 | INFO | Analyzing chunk frames = [4631040,4631552) | 20260505 16:26:18.798 | INFO | MemInfo: { | 20260505 16:26:18.798 | INFO | "gpu" : " 17 GB", | 20260505 16:26:18.798 | INFO | "huge" : "", | 20260505 16:26:18.798 | INFO | "huge_shared" : " 27 GB", | 20260505 16:26:18.798 | INFO | "malloc" : " 14 GB", | 20260505 16:26:18.798 | INFO | "pinned" : " 29 GB" | 20260505 16:26:18.798 | INFO | } | 20260505 16:26:18.799 | INFO | Perf report for the last 5.13593 seconds: | 20260505 16:26:18.799 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:26:18.799 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:18.799 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:18.799 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:18.799 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 16:26:18.799 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:18.799 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:18.799 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:26:18.799 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:18.799 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:18.799 | INFO | Uploaded 36GiB to GPU in 2.16141s (16.6558GiB/s) | 20260505 16:26:18.799 | INFO | Downloaded 34GiB to GPU in 2.03967s (16.6693GiB/s) | 20260505 16:26:23.944 | INFO | Analyzing chunk frames = [4631552,4632064) | 20260505 16:26:23.944 | INFO | MemInfo: { | 20260505 16:26:23.944 | INFO | "gpu" : " 17 GB", | 20260505 16:26:23.944 | INFO | "huge" : "", | 20260505 16:26:23.944 | INFO | "huge_shared" : " 27 GB", | 20260505 16:26:23.944 | INFO | "malloc" : " 14 GB", | 20260505 16:26:23.944 | INFO | "pinned" : " 29 GB" | 20260505 16:26:23.944 | INFO | } | 20260505 16:26:23.944 | INFO | Perf report for the last 5.1458 seconds: | 20260505 16:26:23.944 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:26:23.944 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:23.944 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:23.944 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:23.944 | INFO | ANALYSIS : 63.3%, 3.0, 99.0ms, 0 | 20260505 16:26:23.944 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:23.944 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:23.944 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 16:26:23.944 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:23.944 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:23.945 | INFO | Uploaded 36GiB to GPU in 2.18456s (16.4793GiB/s) | 20260505 16:26:23.945 | INFO | Downloaded 34GiB to GPU in 2.14094s (15.8809GiB/s) | 20260505 16:26:29.087 | INFO | Analyzing chunk frames = [4632064,4632576) | 20260505 16:26:29.087 | INFO | MemInfo: { | 20260505 16:26:29.087 | INFO | "gpu" : " 17 GB", | 20260505 16:26:29.087 | INFO | "huge" : "", | 20260505 16:26:29.087 | INFO | "huge_shared" : " 27 GB", | 20260505 16:26:29.087 | INFO | "malloc" : " 14 GB", | 20260505 16:26:29.088 | INFO | "pinned" : " 29 GB" | 20260505 16:26:29.088 | INFO | } | 20260505 16:26:29.088 | INFO | Perf report for the last 5.14337 seconds: | 20260505 16:26:29.088 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:26:29.088 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:29.088 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:29.088 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:29.088 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 16:26:29.088 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:29.088 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:29.088 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 16:26:29.088 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:29.088 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:29.088 | INFO | Uploaded 36GiB to GPU in 2.1539s (16.7138GiB/s) | 20260505 16:26:29.088 | INFO | Downloaded 34GiB to GPU in 2.02056s (16.827GiB/s) | 20260505 16:26:29.225 | INFO | Starting a round of Baz IO | 20260505 16:26:32.416 | INFO | Populating RTMetrics report | 20260505 16:26:32.416 | INFO | RTMetrics report populated | 20260505 16:26:32.416 | INFO | Opening tmp RTMetrics json file | 20260505 16:26:32.425 | INFO | Serializing RTMetrics to json | 20260505 16:26:32.429 | INFO | Serialization done, swapping tmp file now... | 20260505 16:26:32.432 | INFO | RTMetrics done | 20260505 16:26:32.516 | INFO | Finished a round of Baz IO: Wrote 2.17 GiB in 3.29 seconds | 20260505 16:26:32.516 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.25% / 0.00% / 77.75% | 20260505 16:26:32.516 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.364904% / 7.94729e-06% / 7.94729e-05% | 20260505 16:26:32.516 | INFO | Baz IO overall: | 20260505 16:26:32.516 | INFO | BytesWritten : 2748786816056 | 20260505 16:26:32.516 | INFO | NumEvents : 1304715968536 | 20260505 16:26:34.227 | INFO | Analyzing chunk frames = [4632576,4633088) | 20260505 16:26:34.227 | INFO | MemInfo: { | 20260505 16:26:34.227 | INFO | "gpu" : " 17 GB", | 20260505 16:26:34.227 | INFO | "huge" : "", | 20260505 16:26:34.227 | INFO | "huge_shared" : " 27 GB", | 20260505 16:26:34.228 | INFO | "malloc" : " 14 GB", | 20260505 16:26:34.228 | INFO | "pinned" : " 29 GB" | 20260505 16:26:34.228 | INFO | } | 20260505 16:26:34.228 | INFO | Perf report for the last 5.13987 seconds: | 20260505 16:26:34.228 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:26:34.228 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:34.228 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:34.228 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:34.228 | INFO | ANALYSIS : 63.7%, 3.0, 99.2ms, 0 | 20260505 16:26:34.228 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:34.228 | INFO | RT_METRICS : 44.9%, 1.0, 23.5ms, 0 | 20260505 16:26:34.228 | INFO | PRE_HQ : 60.5%, 1.8, 56.3ms, 0 | 20260505 16:26:34.228 | INFO | BAZWRITER : 63.5%, 3.9, 127.1ms, 0 | 20260505 16:26:34.228 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 16:26:34.228 | INFO | Uploaded 36GiB to GPU in 2.15565s (16.7003GiB/s) | 20260505 16:26:34.228 | INFO | Downloaded 38GiB to GPU in 2.25615s (16.8429GiB/s) | 20260505 16:26:39.353 | INFO | Analyzing chunk frames = [4633088,4633600) | 20260505 16:26:39.353 | INFO | MemInfo: { | 20260505 16:26:39.353 | INFO | "gpu" : " 17 GB", | 20260505 16:26:39.353 | INFO | "huge" : "", | 20260505 16:26:39.353 | INFO | "huge_shared" : " 27 GB", | 20260505 16:26:39.353 | INFO | "malloc" : " 14 GB", | 20260505 16:26:39.353 | INFO | "pinned" : " 29 GB" | 20260505 16:26:39.353 | INFO | } | 20260505 16:26:39.353 | INFO | Perf report for the last 5.12571 seconds: | 20260505 16:26:39.353 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:26:39.353 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:39.353 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:39.353 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:39.353 | INFO | ANALYSIS : 63.1%, 3.0, 98.2ms, 0 | 20260505 16:26:39.353 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:39.353 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:39.353 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:26:39.353 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:39.354 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:39.354 | INFO | Uploaded 36GiB to GPU in 2.12378s (16.9509GiB/s) | 20260505 16:26:39.354 | INFO | Downloaded 34GiB to GPU in 2.02959s (16.7521GiB/s) | 20260505 16:26:44.493 | INFO | Analyzing chunk frames = [4633600,4634112) | 20260505 16:26:44.493 | INFO | MemInfo: { | 20260505 16:26:44.493 | INFO | "gpu" : " 17 GB", | 20260505 16:26:44.493 | INFO | "huge" : "", | 20260505 16:26:44.493 | INFO | "huge_shared" : " 27 GB", | 20260505 16:26:44.493 | INFO | "malloc" : " 14 GB", | 20260505 16:26:44.493 | INFO | "pinned" : " 29 GB" | 20260505 16:26:44.493 | INFO | } | 20260505 16:26:44.493 | INFO | Perf report for the last 5.14002 seconds: | 20260505 16:26:44.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:26:44.493 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:44.493 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:44.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:44.493 | INFO | ANALYSIS : 63.0%, 3.0, 98.1ms, 0 | 20260505 16:26:44.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:44.493 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:44.493 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 16:26:44.493 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:44.494 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:44.494 | INFO | Uploaded 36GiB to GPU in 2.19351s (16.412GiB/s) | 20260505 16:26:44.494 | INFO | Downloaded 34GiB to GPU in 2.06878s (16.4348GiB/s) | 20260505 16:26:49.643 | INFO | Analyzing chunk frames = [4634112,4634624) | 20260505 16:26:49.643 | INFO | MemInfo: { | 20260505 16:26:49.643 | INFO | "gpu" : " 17 GB", | 20260505 16:26:49.643 | INFO | "huge" : "", | 20260505 16:26:49.643 | INFO | "huge_shared" : " 27 GB", | 20260505 16:26:49.643 | INFO | "malloc" : " 14 GB", | 20260505 16:26:49.643 | INFO | "pinned" : " 29 GB" | 20260505 16:26:49.643 | INFO | } | 20260505 16:26:49.643 | INFO | Perf report for the last 5.14975 seconds: | 20260505 16:26:49.643 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:26:49.643 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:49.643 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:49.643 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:49.643 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260505 16:26:49.643 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:49.643 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:49.643 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:26:49.643 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:49.643 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:49.643 | INFO | Uploaded 36GiB to GPU in 2.17602s (16.5439GiB/s) | 20260505 16:26:49.644 | INFO | Downloaded 34GiB to GPU in 2.13427s (15.9305GiB/s) | 20260505 16:26:54.785 | INFO | Analyzing chunk frames = [4634624,4635136) | 20260505 16:26:54.785 | INFO | MemInfo: { | 20260505 16:26:54.785 | INFO | "gpu" : " 17 GB", | 20260505 16:26:54.786 | INFO | "huge" : "", | 20260505 16:26:54.786 | INFO | "huge_shared" : " 27 GB", | 20260505 16:26:54.786 | INFO | "malloc" : " 14 GB", | 20260505 16:26:54.786 | INFO | "pinned" : " 29 GB" | 20260505 16:26:54.786 | INFO | } | 20260505 16:26:54.786 | INFO | Perf report for the last 5.14258 seconds: | 20260505 16:26:54.786 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:26:54.786 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:54.786 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:54.786 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:54.786 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260505 16:26:54.786 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:54.786 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:54.786 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 16:26:54.786 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:54.786 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:54.786 | INFO | Uploaded 36GiB to GPU in 2.1593s (16.6721GiB/s) | 20260505 16:26:54.786 | INFO | Downloaded 34GiB to GPU in 2.05927s (16.5107GiB/s) | 20260505 16:26:59.921 | INFO | Analyzing chunk frames = [4635136,4635648) | 20260505 16:26:59.921 | INFO | MemInfo: { | 20260505 16:26:59.921 | INFO | "gpu" : " 17 GB", | 20260505 16:26:59.921 | INFO | "huge" : "", | 20260505 16:26:59.921 | INFO | "huge_shared" : " 27 GB", | 20260505 16:26:59.921 | INFO | "malloc" : " 14 GB", | 20260505 16:26:59.921 | INFO | "pinned" : " 29 GB" | 20260505 16:26:59.921 | INFO | } | 20260505 16:26:59.921 | INFO | Perf report for the last 5.13523 seconds: | 20260505 16:26:59.921 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:26:59.921 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:59.921 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:59.921 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:59.921 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260505 16:26:59.921 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:59.921 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:59.921 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:26:59.921 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:59.921 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:26:59.921 | INFO | Uploaded 36GiB to GPU in 2.1409s (16.8153GiB/s) | 20260505 16:26:59.921 | INFO | Downloaded 34GiB to GPU in 2.04401s (16.634GiB/s) | 20260505 16:27:05.059 | INFO | Analyzing chunk frames = [4635648,4636160) | 20260505 16:27:05.059 | INFO | MemInfo: { | 20260505 16:27:05.059 | INFO | "gpu" : " 17 GB", | 20260505 16:27:05.059 | INFO | "huge" : "", | 20260505 16:27:05.059 | INFO | "huge_shared" : " 27 GB", | 20260505 16:27:05.059 | INFO | "malloc" : " 14 GB", | 20260505 16:27:05.059 | INFO | "pinned" : " 29 GB" | 20260505 16:27:05.059 | INFO | } | 20260505 16:27:05.060 | INFO | Perf report for the last 5.13853 seconds: | 20260505 16:27:05.060 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:27:05.060 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:05.060 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:05.060 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:05.060 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 16:27:05.060 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:05.060 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:05.060 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 16:27:05.060 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:05.060 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:05.060 | INFO | Uploaded 36GiB to GPU in 2.1289s (16.9101GiB/s) | 20260505 16:27:05.060 | INFO | Downloaded 34GiB to GPU in 2.17863s (15.6062GiB/s) | 20260505 16:27:10.195 | INFO | Analyzing chunk frames = [4636160,4636672) | 20260505 16:27:10.195 | INFO | MemInfo: { | 20260505 16:27:10.195 | INFO | "gpu" : " 17 GB", | 20260505 16:27:10.195 | INFO | "huge" : "", | 20260505 16:27:10.195 | INFO | "huge_shared" : " 27 GB", | 20260505 16:27:10.195 | INFO | "malloc" : " 14 GB", | 20260505 16:27:10.195 | INFO | "pinned" : " 29 GB" | 20260505 16:27:10.195 | INFO | } | 20260505 16:27:10.195 | INFO | Perf report for the last 5.13578 seconds: | 20260505 16:27:10.195 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:27:10.195 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:10.195 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:10.195 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:10.195 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 16:27:10.195 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:10.195 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:10.195 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:27:10.195 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:10.196 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:10.196 | INFO | Uploaded 36GiB to GPU in 2.23164s (16.1317GiB/s) | 20260505 16:27:10.196 | INFO | Downloaded 34GiB to GPU in 2.1091s (16.1206GiB/s) | 20260505 16:27:10.196 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 16:27:10.196 | INFO | Total mem usage: 15.3 GiB | 20260505 16:27:10.196 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 16:27:10.196 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.88%) | 20260505 16:27:10.196 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 16:27:10.196 | INFO | High Water Breakdown: | 20260505 16:27:10.196 | INFO | 61% : 8.67 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 16:27:10.196 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 16:27:10.196 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 16:27:10.196 | INFO | ------------------------------------------ | 20260505 16:27:10.196 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:27:10.196 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 16:27:10.196 | INFO | Total mem usage: 26.5 GiB | 20260505 16:27:10.196 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 16:27:10.196 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 16:27:10.196 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 16:27:10.196 | INFO | High Water Breakdown: | 20260505 16:27:10.196 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:27:10.196 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:27:10.196 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:27:10.196 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:27:10.196 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:27:10.196 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:27:10.196 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 16:27:10.196 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 16:27:10.196 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:27:10.196 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 16:27:10.196 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 16:27:10.196 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 16:27:10.196 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 16:27:10.196 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 16:27:10.196 | INFO | ------------------------------------------ | 20260505 16:27:10.196 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:27:10.196 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 16:27:10.196 | INFO | Total mem usage: 25.6 GiB | 20260505 16:27:10.196 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 16:27:10.196 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 16:27:10.196 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 16:27:10.196 | INFO | High Water Breakdown: | 20260505 16:27:10.196 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 16:27:10.196 | INFO | ------------------------------------------ | 20260505 16:27:10.196 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:27:10.196 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 16:27:10.196 | INFO | Total mem usage: 15.7 GiB | 20260505 16:27:10.196 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 16:27:10.196 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 16:27:10.196 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 16:27:10.196 | INFO | High Water Breakdown: | 20260505 16:27:10.196 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 16:27:10.196 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 16:27:10.196 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 16:27:10.196 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 16:27:10.196 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 16:27:10.196 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 16:27:10.196 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 16:27:10.196 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 16:27:10.197 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 16:27:10.197 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:27:10.197 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 16:27:10.197 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:27:10.197 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 16:27:10.197 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 16:27:10.197 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 16:27:10.197 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 16:27:10.197 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:27:10.197 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 16:27:10.197 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:27:10.197 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:27:10.197 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:27:10.197 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 16:27:10.197 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 16:27:10.197 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:27:10.197 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 16:27:10.197 | INFO | ------------------------------------------ | 20260505 16:27:10.197 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:27:10.197 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 16:27:10.197 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:27:10.197 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:27:10.197 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.52e+01, 1.36e+01 / 1.81e+00 | 20260505 16:27:10.197 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 1.03e-01, 1.10e-02 / 4.57e-03 | 20260505 16:27:10.197 | INFO | PerfReport: 11.78%, Baseline -- 3.74e+00 / 4.46e+00, 5.83e-01 / 2.13e-01 | 20260505 16:27:10.197 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.48e+00, 1.94e-01 / 9.68e-02 | 20260505 16:27:10.197 | INFO | PerfReport: 15.88%, DME -- 4.03e+01 / 4.47e+01, 4.22e+00 / 1.88e+00 | 20260505 16:27:10.197 | INFO | PerfReport: 56.84%, FrameLabeling -- 1.80e+01 / 1.97e+01, 1.58e+00 / 1.83e+00 | 20260505 16:27:10.197 | INFO | PerfReport: 7.32%, PulseAccumulating -- 2.32e+00 / 2.90e+00, 4.42e-01 / 2.09e-01 | 20260505 16:27:10.197 | INFO | PerfReport: 3.41%, Metrics -- 1.08e+00 / 1.44e+00, 3.05e-01 / 1.79e-01 | 20260505 16:27:10.197 | INFO | PerfReport: 1.19%, PulseSerializing -- 3.77e-01 / 4.37e-01, 6.79e-02 / 3.36e-02 | 20260505 16:27:10.197 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 16:27:10.197 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:27:10.197 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:27:10.197 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260505 16:27:10.197 | INFO | PerfReport: 22.51%, Upload -- 2.25e+01 / 0.00e+00, 7.87e+00 / 0.00e+00 | 20260505 16:27:10.197 | INFO | PerfReport: 21.85%, Download -- 2.19e+01 / 0.00e+00, 7.90e+00 / 0.00e+00 | 20260505 16:27:10.197 | INFO | PerfReport: 31.69%, Compute -- 3.17e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 16:27:10.197 | INFO | PerfReport: 2.19%, LockUpload -- 2.20e+00 / 0.00e+00, 7.80e+00 / 0.00e+00 | 20260505 16:27:10.197 | INFO | PerfReport: 1.39%, LockDownload -- 1.39e+00 / 0.00e+00, 2.96e+00 / 0.00e+00 | 20260505 16:27:10.197 | INFO | PerfReport: 20.36%, LockCompute -- 2.04e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 16:27:10.197 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.28e-02 / 0.00e+00 | 20260505 16:27:10.329 | INFO | Starting a round of Baz IO | 20260505 16:27:13.531 | INFO | Populating RTMetrics report | 20260505 16:27:13.531 | INFO | RTMetrics report populated | 20260505 16:27:13.531 | INFO | Opening tmp RTMetrics json file | 20260505 16:27:13.546 | INFO | Serializing RTMetrics to json | 20260505 16:27:13.568 | INFO | Serialization done, swapping tmp file now... | 20260505 16:27:13.583 | INFO | RTMetrics done | 20260505 16:27:13.627 | INFO | Finished a round of Baz IO: Wrote 2.172 GiB in 3.297 seconds | 20260505 16:27:13.627 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.24% / 0.00% / 77.75% | 20260505 16:27:13.627 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.365448% / 1.19209e-05% / 7.15256e-05% | 20260505 16:27:13.627 | INFO | Baz IO overall: | 20260505 16:27:13.627 | INFO | BytesWritten : 2751118629709 | 20260505 16:27:13.627 | INFO | NumEvents : 1305809300539 | 20260505 16:27:15.335 | INFO | Analyzing chunk frames = [4636672,4637184) | 20260505 16:27:15.335 | INFO | MemInfo: { | 20260505 16:27:15.335 | INFO | "gpu" : " 17 GB", | 20260505 16:27:15.335 | INFO | "huge" : "", | 20260505 16:27:15.335 | INFO | "huge_shared" : " 27 GB", | 20260505 16:27:15.335 | INFO | "malloc" : " 14 GB", | 20260505 16:27:15.335 | INFO | "pinned" : " 29 GB" | 20260505 16:27:15.335 | INFO | } | 20260505 16:27:15.335 | INFO | Perf report for the last 5.13989 seconds: | 20260505 16:27:15.335 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:27:15.335 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:15.335 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:15.335 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:15.335 | INFO | ANALYSIS : 63.8%, 3.0, 99.2ms, 0 | 20260505 16:27:15.335 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:15.335 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260505 16:27:15.335 | INFO | PRE_HQ : 60.7%, 1.8, 56.4ms, 0 | 20260505 16:27:15.335 | INFO | BAZWRITER : 63.5%, 3.9, 128.1ms, 0 | 20260505 16:27:15.335 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 16:27:15.336 | INFO | Uploaded 36GiB to GPU in 2.18346s (16.4876GiB/s) | 20260505 16:27:15.336 | INFO | Downloaded 38GiB to GPU in 2.26927s (16.7455GiB/s) | 20260505 16:27:20.475 | INFO | Analyzing chunk frames = [4637184,4637696) | 20260505 16:27:20.475 | INFO | MemInfo: { | 20260505 16:27:20.475 | INFO | "gpu" : " 17 GB", | 20260505 16:27:20.475 | INFO | "huge" : "", | 20260505 16:27:20.475 | INFO | "huge_shared" : " 27 GB", | 20260505 16:27:20.475 | INFO | "malloc" : " 14 GB", | 20260505 16:27:20.475 | INFO | "pinned" : " 29 GB" | 20260505 16:27:20.475 | INFO | } | 20260505 16:27:20.475 | INFO | Perf report for the last 5.14018 seconds: | 20260505 16:27:20.476 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:27:20.476 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:20.476 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:20.476 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:20.476 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 16:27:20.476 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:20.476 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:20.476 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:27:20.476 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:20.476 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:20.476 | INFO | Uploaded 36GiB to GPU in 2.13894s (16.8308GiB/s) | 20260505 16:27:20.476 | INFO | Downloaded 34GiB to GPU in 2.04388s (16.635GiB/s) | 20260505 16:27:25.613 | INFO | Analyzing chunk frames = [4637696,4638208) | 20260505 16:27:25.613 | INFO | MemInfo: { | 20260505 16:27:25.613 | INFO | "gpu" : " 17 GB", | 20260505 16:27:25.613 | INFO | "huge" : "", | 20260505 16:27:25.613 | INFO | "huge_shared" : " 27 GB", | 20260505 16:27:25.613 | INFO | "malloc" : " 14 GB", | 20260505 16:27:25.613 | INFO | "pinned" : " 29 GB" | 20260505 16:27:25.613 | INFO | } | 20260505 16:27:25.613 | INFO | Perf report for the last 5.13796 seconds: | 20260505 16:27:25.613 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:27:25.614 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:25.614 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:25.614 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:25.614 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260505 16:27:25.614 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:25.614 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:25.614 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 16:27:25.614 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:25.614 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:25.614 | INFO | Uploaded 36GiB to GPU in 2.18693s (16.4614GiB/s) | 20260505 16:27:25.614 | INFO | Downloaded 34GiB to GPU in 2.03337s (16.721GiB/s) | 20260505 16:27:30.748 | INFO | Analyzing chunk frames = [4638208,4638720) | 20260505 16:27:30.748 | INFO | MemInfo: { | 20260505 16:27:30.748 | INFO | "gpu" : " 17 GB", | 20260505 16:27:30.748 | INFO | "huge" : "", | 20260505 16:27:30.748 | INFO | "huge_shared" : " 27 GB", | 20260505 16:27:30.748 | INFO | "malloc" : " 14 GB", | 20260505 16:27:30.748 | INFO | "pinned" : " 29 GB" | 20260505 16:27:30.748 | INFO | } | 20260505 16:27:30.748 | INFO | Perf report for the last 5.13479 seconds: | 20260505 16:27:30.748 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:27:30.748 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:30.748 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:30.748 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:30.748 | INFO | ANALYSIS : 62.5%, 3.0, 97.0ms, 0 | 20260505 16:27:30.748 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:30.748 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:30.748 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:27:30.748 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:30.748 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:30.749 | INFO | Uploaded 36GiB to GPU in 2.17658s (16.5397GiB/s) | 20260505 16:27:30.749 | INFO | Downloaded 34GiB to GPU in 2.13198s (15.9476GiB/s) | 20260505 16:27:35.899 | INFO | Analyzing chunk frames = [4638720,4639232) | 20260505 16:27:35.899 | INFO | MemInfo: { | 20260505 16:27:35.899 | INFO | "gpu" : " 17 GB", | 20260505 16:27:35.899 | INFO | "huge" : "", | 20260505 16:27:35.899 | INFO | "huge_shared" : " 27 GB", | 20260505 16:27:35.899 | INFO | "malloc" : " 14 GB", | 20260505 16:27:35.899 | INFO | "pinned" : " 29 GB" | 20260505 16:27:35.899 | INFO | } | 20260505 16:27:35.899 | INFO | Perf report for the last 5.15048 seconds: | 20260505 16:27:35.899 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:27:35.899 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:35.899 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:35.899 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:35.899 | INFO | ANALYSIS : 62.5%, 3.0, 97.8ms, 0 | 20260505 16:27:35.899 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:35.899 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:35.899 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 16:27:35.899 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:35.899 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:35.899 | INFO | Uploaded 36GiB to GPU in 2.18622s (16.4668GiB/s) | 20260505 16:27:35.899 | INFO | Downloaded 34GiB to GPU in 2.13239s (15.9446GiB/s) | 20260505 16:27:41.043 | INFO | Analyzing chunk frames = [4639232,4639744) | 20260505 16:27:41.043 | INFO | MemInfo: { | 20260505 16:27:41.043 | INFO | "gpu" : " 17 GB", | 20260505 16:27:41.043 | INFO | "huge" : "", | 20260505 16:27:41.043 | INFO | "huge_shared" : " 27 GB", | 20260505 16:27:41.043 | INFO | "malloc" : " 14 GB", | 20260505 16:27:41.044 | INFO | "pinned" : " 29 GB" | 20260505 16:27:41.044 | INFO | } | 20260505 16:27:41.044 | INFO | Perf report for the last 5.14502 seconds: | 20260505 16:27:41.044 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:27:41.044 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:41.044 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:41.044 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:41.044 | INFO | ANALYSIS : 62.3%, 3.0, 97.4ms, 0 | 20260505 16:27:41.044 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:41.044 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:41.044 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 16:27:41.044 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:41.044 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:41.044 | INFO | Uploaded 36GiB to GPU in 2.14206s (16.8062GiB/s) | 20260505 16:27:41.044 | INFO | Downloaded 34GiB to GPU in 2.03996s (16.667GiB/s) | 20260505 16:27:46.172 | INFO | Analyzing chunk frames = [4639744,4640256) | 20260505 16:27:46.172 | INFO | MemInfo: { | 20260505 16:27:46.172 | INFO | "gpu" : " 17 GB", | 20260505 16:27:46.172 | INFO | "huge" : "", | 20260505 16:27:46.172 | INFO | "huge_shared" : " 27 GB", | 20260505 16:27:46.172 | INFO | "malloc" : " 14 GB", | 20260505 16:27:46.172 | INFO | "pinned" : " 29 GB" | 20260505 16:27:46.172 | INFO | } | 20260505 16:27:46.172 | INFO | Perf report for the last 5.12852 seconds: | 20260505 16:27:46.172 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:27:46.172 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:46.172 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:46.172 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:46.172 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 16:27:46.172 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:46.172 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:46.172 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 16:27:46.172 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:46.172 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:46.172 | INFO | Uploaded 36GiB to GPU in 2.13142s (16.8901GiB/s) | 20260505 16:27:46.173 | INFO | Downloaded 34GiB to GPU in 2.14825s (15.8269GiB/s) | 20260505 16:27:51.306 | INFO | Analyzing chunk frames = [4640256,4640768) | 20260505 16:27:51.306 | INFO | MemInfo: { | 20260505 16:27:51.306 | INFO | "gpu" : " 17 GB", | 20260505 16:27:51.306 | INFO | "huge" : "", | 20260505 16:27:51.306 | INFO | "huge_shared" : " 27 GB", | 20260505 16:27:51.306 | INFO | "malloc" : " 14 GB", | 20260505 16:27:51.306 | INFO | "pinned" : " 29 GB" | 20260505 16:27:51.306 | INFO | } | 20260505 16:27:51.306 | INFO | Perf report for the last 5.13353 seconds: | 20260505 16:27:51.306 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:27:51.306 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:51.306 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:51.306 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:51.306 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 16:27:51.306 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:51.306 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:51.306 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:27:51.306 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:51.306 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:51.306 | INFO | Uploaded 36GiB to GPU in 2.17523s (16.55GiB/s) | 20260505 16:27:51.306 | INFO | Downloaded 34GiB to GPU in 2.0353s (16.7051GiB/s) | 20260505 16:27:51.438 | INFO | Starting a round of Baz IO | 20260505 16:27:54.651 | INFO | Populating RTMetrics report | 20260505 16:27:54.651 | INFO | RTMetrics report populated | 20260505 16:27:54.651 | INFO | Opening tmp RTMetrics json file | 20260505 16:27:54.653 | INFO | Serializing RTMetrics to json | 20260505 16:27:54.656 | INFO | Serialization done, swapping tmp file now... | 20260505 16:27:54.656 | INFO | RTMetrics done | 20260505 16:27:54.765 | INFO | Finished a round of Baz IO: Wrote 2.17 GiB in 3.326 seconds | 20260505 16:27:54.765 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.24% / 0.00% / 77.76% | 20260505 16:27:54.765 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.364196% / 1.19209e-05% / 8.34465e-05% | 20260505 16:27:54.765 | INFO | Baz IO overall: | 20260505 16:27:54.765 | INFO | BytesWritten : 2753448162381 | 20260505 16:27:54.765 | INFO | NumEvents : 1306901044743 | 20260505 16:27:56.448 | INFO | Analyzing chunk frames = [4640768,4641280) | 20260505 16:27:56.448 | INFO | MemInfo: { | 20260505 16:27:56.448 | INFO | "gpu" : " 17 GB", | 20260505 16:27:56.448 | INFO | "huge" : "", | 20260505 16:27:56.448 | INFO | "huge_shared" : " 27 GB", | 20260505 16:27:56.448 | INFO | "malloc" : " 14 GB", | 20260505 16:27:56.448 | INFO | "pinned" : " 29 GB" | 20260505 16:27:56.449 | INFO | } | 20260505 16:27:56.449 | INFO | Perf report for the last 5.14285 seconds: | 20260505 16:27:56.449 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:27:56.449 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:56.449 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:56.449 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:56.449 | INFO | ANALYSIS : 64.0%, 3.0, 99.5ms, 0 | 20260505 16:27:56.449 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:27:56.449 | INFO | RT_METRICS : 44.7%, 1.0, 23.5ms, 0 | 20260505 16:27:56.449 | INFO | PRE_HQ : 60.7%, 1.8, 56.7ms, 0 | 20260505 16:27:56.449 | INFO | BAZWRITER : 64.0%, 3.9, 128.6ms, 0 | 20260505 16:27:56.449 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 16:27:56.449 | INFO | Uploaded 36GiB to GPU in 2.19381s (16.4098GiB/s) | 20260505 16:27:56.449 | INFO | Downloaded 38GiB to GPU in 2.26374s (16.7864GiB/s) | 20260505 16:28:01.601 | INFO | Analyzing chunk frames = [4641280,4641792) | 20260505 16:28:01.601 | INFO | MemInfo: { | 20260505 16:28:01.601 | INFO | "gpu" : " 17 GB", | 20260505 16:28:01.601 | INFO | "huge" : "", | 20260505 16:28:01.601 | INFO | "huge_shared" : " 27 GB", | 20260505 16:28:01.601 | INFO | "malloc" : " 14 GB", | 20260505 16:28:01.601 | INFO | "pinned" : " 29 GB" | 20260505 16:28:01.601 | INFO | } | 20260505 16:28:01.601 | INFO | Perf report for the last 5.15231 seconds: | 20260505 16:28:01.601 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:28:01.601 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:01.601 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:01.601 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:01.601 | INFO | ANALYSIS : 62.8%, 3.0, 98.3ms, 0 | 20260505 16:28:01.601 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:01.601 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:01.601 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 16:28:01.601 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:01.601 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:01.601 | INFO | Uploaded 36GiB to GPU in 2.13785s (16.8393GiB/s) | 20260505 16:28:01.601 | INFO | Downloaded 34GiB to GPU in 2.03756s (16.6866GiB/s) | 20260505 16:28:06.737 | INFO | Analyzing chunk frames = [4641792,4642304) | 20260505 16:28:06.737 | INFO | MemInfo: { | 20260505 16:28:06.737 | INFO | "gpu" : " 17 GB", | 20260505 16:28:06.737 | INFO | "huge" : "", | 20260505 16:28:06.737 | INFO | "huge_shared" : " 27 GB", | 20260505 16:28:06.737 | INFO | "malloc" : " 14 GB", | 20260505 16:28:06.737 | INFO | "pinned" : " 29 GB" | 20260505 16:28:06.737 | INFO | } | 20260505 16:28:06.737 | INFO | Perf report for the last 5.13587 seconds: | 20260505 16:28:06.737 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:28:06.737 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:06.737 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:06.737 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:06.737 | INFO | ANALYSIS : 62.8%, 3.0, 97.6ms, 0 | 20260505 16:28:06.737 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:06.737 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:06.737 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 16:28:06.737 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:06.737 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:06.737 | INFO | Uploaded 36GiB to GPU in 2.20275s (16.3432GiB/s) | 20260505 16:28:06.737 | INFO | Downloaded 34GiB to GPU in 2.05072s (16.5795GiB/s) | 20260505 16:28:11.868 | INFO | Analyzing chunk frames = [4642304,4642816) | 20260505 16:28:11.868 | INFO | MemInfo: { | 20260505 16:28:11.868 | INFO | "gpu" : " 17 GB", | 20260505 16:28:11.868 | INFO | "huge" : "", | 20260505 16:28:11.868 | INFO | "huge_shared" : " 27 GB", | 20260505 16:28:11.868 | INFO | "malloc" : " 14 GB", | 20260505 16:28:11.868 | INFO | "pinned" : " 29 GB" | 20260505 16:28:11.868 | INFO | } | 20260505 16:28:11.868 | INFO | Perf report for the last 5.13129 seconds: | 20260505 16:28:11.868 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:28:11.868 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:11.868 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:11.868 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:11.868 | INFO | ANALYSIS : 62.7%, 3.0, 97.3ms, 0 | 20260505 16:28:11.868 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:11.868 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:11.868 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 16:28:11.868 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:11.868 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:11.868 | INFO | Uploaded 36GiB to GPU in 2.20346s (16.3379GiB/s) | 20260505 16:28:11.868 | INFO | Downloaded 34GiB to GPU in 2.13943s (15.8921GiB/s) | 20260505 16:28:17.007 | INFO | Analyzing chunk frames = [4642816,4643328) | 20260505 16:28:17.007 | INFO | MemInfo: { | 20260505 16:28:17.007 | INFO | "gpu" : " 17 GB", | 20260505 16:28:17.007 | INFO | "huge" : "", | 20260505 16:28:17.007 | INFO | "huge_shared" : " 27 GB", | 20260505 16:28:17.007 | INFO | "malloc" : " 14 GB", | 20260505 16:28:17.007 | INFO | "pinned" : " 29 GB" | 20260505 16:28:17.007 | INFO | } | 20260505 16:28:17.007 | INFO | Perf report for the last 5.13878 seconds: | 20260505 16:28:17.007 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:28:17.007 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:17.007 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:17.007 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:17.007 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 16:28:17.007 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:17.007 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:17.007 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 16:28:17.007 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:17.007 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:17.007 | INFO | Uploaded 36GiB to GPU in 2.17499s (16.5518GiB/s) | 20260505 16:28:17.007 | INFO | Downloaded 34GiB to GPU in 2.13414s (15.9315GiB/s) | 20260505 16:28:22.150 | INFO | Analyzing chunk frames = [4643328,4643840) | 20260505 16:28:22.150 | INFO | MemInfo: { | 20260505 16:28:22.150 | INFO | "gpu" : " 17 GB", | 20260505 16:28:22.150 | INFO | "huge" : "", | 20260505 16:28:22.150 | INFO | "huge_shared" : " 27 GB", | 20260505 16:28:22.150 | INFO | "malloc" : " 14 GB", | 20260505 16:28:22.150 | INFO | "pinned" : " 29 GB" | 20260505 16:28:22.150 | INFO | } | 20260505 16:28:22.150 | INFO | Perf report for the last 5.14314 seconds: | 20260505 16:28:22.150 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:28:22.150 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:22.150 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:22.150 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:22.150 | INFO | ANALYSIS : 62.6%, 3.0, 97.9ms, 0 | 20260505 16:28:22.150 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:22.150 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:22.150 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:28:22.150 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:22.150 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:22.150 | INFO | Uploaded 36GiB to GPU in 2.16108s (16.6584GiB/s) | 20260505 16:28:22.151 | INFO | Downloaded 34GiB to GPU in 2.04158s (16.6538GiB/s) | 20260505 16:28:27.294 | INFO | Analyzing chunk frames = [4643840,4644352) | 20260505 16:28:27.294 | INFO | MemInfo: { | 20260505 16:28:27.294 | INFO | "gpu" : " 17 GB", | 20260505 16:28:27.294 | INFO | "huge" : "", | 20260505 16:28:27.294 | INFO | "huge_shared" : " 27 GB", | 20260505 16:28:27.294 | INFO | "malloc" : " 14 GB", | 20260505 16:28:27.294 | INFO | "pinned" : " 29 GB" | 20260505 16:28:27.294 | INFO | } | 20260505 16:28:27.295 | INFO | Perf report for the last 5.14438 seconds: | 20260505 16:28:27.295 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:28:27.295 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:27.295 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:27.295 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:27.295 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 16:28:27.295 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:27.295 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:27.295 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 16:28:27.295 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:27.295 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:27.295 | INFO | Uploaded 36GiB to GPU in 2.16086s (16.6601GiB/s) | 20260505 16:28:27.295 | INFO | Downloaded 34GiB to GPU in 2.17854s (15.6068GiB/s) | 20260505 16:28:32.430 | INFO | Analyzing chunk frames = [4644352,4644864) | 20260505 16:28:32.430 | INFO | MemInfo: { | 20260505 16:28:32.430 | INFO | "gpu" : " 17 GB", | 20260505 16:28:32.430 | INFO | "huge" : "", | 20260505 16:28:32.430 | INFO | "huge_shared" : " 27 GB", | 20260505 16:28:32.430 | INFO | "malloc" : " 14 GB", | 20260505 16:28:32.430 | INFO | "pinned" : " 29 GB" | 20260505 16:28:32.430 | INFO | } | 20260505 16:28:32.430 | INFO | Perf report for the last 5.13567 seconds: | 20260505 16:28:32.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:28:32.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:32.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:32.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:32.430 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 16:28:32.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:32.430 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:32.430 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 16:28:32.430 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:32.430 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:32.431 | INFO | Uploaded 36GiB to GPU in 2.19229s (16.4212GiB/s) | 20260505 16:28:32.431 | INFO | Downloaded 34GiB to GPU in 2.02804s (16.7649GiB/s) | 20260505 16:28:32.431 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 16:28:32.431 | INFO | Total mem usage: 15.3 GiB | 20260505 16:28:32.431 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 16:28:32.431 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.87%) | 20260505 16:28:32.431 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 16:28:32.431 | INFO | High Water Breakdown: | 20260505 16:28:32.431 | INFO | 61% : 8.67 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 16:28:32.431 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 16:28:32.431 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 16:28:32.431 | INFO | ------------------------------------------ | 20260505 16:28:32.431 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:28:32.431 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 16:28:32.431 | INFO | Total mem usage: 26.5 GiB | 20260505 16:28:32.431 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 16:28:32.431 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 16:28:32.431 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 16:28:32.431 | INFO | High Water Breakdown: | 20260505 16:28:32.431 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:28:32.431 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:28:32.431 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:28:32.431 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:28:32.431 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:28:32.431 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:28:32.431 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 16:28:32.431 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 16:28:32.431 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:28:32.431 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 16:28:32.431 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 16:28:32.431 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 16:28:32.431 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 16:28:32.431 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 16:28:32.431 | INFO | ------------------------------------------ | 20260505 16:28:32.431 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:28:32.431 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 16:28:32.431 | INFO | Total mem usage: 25.6 GiB | 20260505 16:28:32.431 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 16:28:32.431 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 16:28:32.431 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 16:28:32.431 | INFO | High Water Breakdown: | 20260505 16:28:32.431 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 16:28:32.431 | INFO | ------------------------------------------ | 20260505 16:28:32.431 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:28:32.431 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 16:28:32.431 | INFO | Total mem usage: 15.7 GiB | 20260505 16:28:32.431 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 16:28:32.431 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 16:28:32.432 | INFO | Unused Cache: 871 MiB (Waste: 5.43%) | 20260505 16:28:32.432 | INFO | High Water Breakdown: | 20260505 16:28:32.432 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 16:28:32.432 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 16:28:32.432 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 16:28:32.432 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 16:28:32.432 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 16:28:32.432 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 16:28:32.432 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 16:28:32.432 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 16:28:32.432 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 16:28:32.432 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:28:32.432 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 16:28:32.432 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:28:32.432 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 16:28:32.432 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 16:28:32.432 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 16:28:32.432 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 16:28:32.432 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:28:32.432 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 16:28:32.432 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:28:32.432 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:28:32.432 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:28:32.432 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 16:28:32.432 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 16:28:32.432 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:28:32.432 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 16:28:32.432 | INFO | ------------------------------------------ | 20260505 16:28:32.432 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:28:32.432 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 16:28:32.432 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:28:32.432 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:28:32.432 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.52e+01, 1.33e+01 / 1.86e+00 | 20260505 16:28:32.432 | INFO | PerfReport: 0.29%, AggregateStats -- 9.27e-02 / 1.01e-01, 1.02e-02 / 3.92e-03 | 20260505 16:28:32.432 | INFO | PerfReport: 11.78%, Baseline -- 3.73e+00 / 4.49e+00, 5.73e-01 / 1.87e-01 | 20260505 16:28:32.432 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.51e+00, 1.93e-01 / 9.08e-02 | 20260505 16:28:32.432 | INFO | PerfReport: 15.91%, DME -- 4.03e+01 / 4.47e+01, 4.31e+00 / 2.19e+00 | 20260505 16:28:32.432 | INFO | PerfReport: 56.77%, FrameLabeling -- 1.80e+01 / 1.96e+01, 1.49e+00 / 1.47e+00 | 20260505 16:28:32.432 | INFO | PerfReport: 7.36%, PulseAccumulating -- 2.33e+00 / 2.89e+00, 4.45e-01 / 1.87e-01 | 20260505 16:28:32.432 | INFO | PerfReport: 3.42%, Metrics -- 1.08e+00 / 1.41e+00, 2.98e-01 / 1.20e-01 | 20260505 16:28:32.432 | INFO | PerfReport: 1.19%, PulseSerializing -- 3.78e-01 / 4.44e-01, 6.80e-02 / 3.18e-02 | 20260505 16:28:32.432 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 16:28:32.432 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:28:32.432 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:28:32.432 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.57e+01 / 0.00e+00 | 20260505 16:28:32.432 | INFO | PerfReport: 22.58%, Upload -- 2.26e+01 / 0.00e+00, 7.91e+00 / 0.00e+00 | 20260505 16:28:32.432 | INFO | PerfReport: 21.90%, Download -- 2.19e+01 / 0.00e+00, 7.93e+00 / 0.00e+00 | 20260505 16:28:32.432 | INFO | PerfReport: 31.60%, Compute -- 3.17e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260505 16:28:32.432 | INFO | PerfReport: 2.26%, LockUpload -- 2.26e+00 / 0.00e+00, 7.93e+00 / 0.00e+00 | 20260505 16:28:32.432 | INFO | PerfReport: 1.45%, LockDownload -- 1.45e+00 / 0.00e+00, 2.98e+00 / 0.00e+00 | 20260505 16:28:32.432 | INFO | PerfReport: 20.20%, LockCompute -- 2.02e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 16:28:32.432 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 2.20e-02 / 0.00e+00 | 20260505 16:28:32.561 | INFO | Starting a round of Baz IO | 20260505 16:28:35.772 | INFO | Populating RTMetrics report | 20260505 16:28:35.773 | INFO | RTMetrics report populated | 20260505 16:28:35.773 | INFO | Opening tmp RTMetrics json file | 20260505 16:28:35.785 | INFO | Serializing RTMetrics to json | 20260505 16:28:35.787 | INFO | Serialization done, swapping tmp file now... | 20260505 16:28:35.790 | INFO | RTMetrics done | 20260505 16:28:35.871 | INFO | Finished a round of Baz IO: Wrote 2.173 GiB in 3.31 seconds | 20260505 16:28:35.871 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.24% / 0.00% / 77.76% | 20260505 16:28:35.871 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.364093% / 0% / 5.96046e-05% | 20260505 16:28:35.871 | INFO | Baz IO overall: | 20260505 16:28:35.871 | INFO | BytesWritten : 2755781210338 | 20260505 16:28:35.871 | INFO | NumEvents : 1307995883275 | 20260505 16:28:37.561 | INFO | Analyzing chunk frames = [4644864,4645376) | 20260505 16:28:37.561 | INFO | MemInfo: { | 20260505 16:28:37.561 | INFO | "gpu" : " 17 GB", | 20260505 16:28:37.561 | INFO | "huge" : "", | 20260505 16:28:37.561 | INFO | "huge_shared" : " 27 GB", | 20260505 16:28:37.561 | INFO | "malloc" : " 14 GB", | 20260505 16:28:37.561 | INFO | "pinned" : " 29 GB" | 20260505 16:28:37.561 | INFO | } | 20260505 16:28:37.561 | INFO | Perf report for the last 5.13129 seconds: | 20260505 16:28:37.562 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:28:37.562 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:37.562 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:37.562 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:37.562 | INFO | ANALYSIS : 64.0%, 3.0, 99.5ms, 0 | 20260505 16:28:37.562 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:37.562 | INFO | RT_METRICS : 45.1%, 1.0, 23.6ms, 0 | 20260505 16:28:37.562 | INFO | PRE_HQ : 61.0%, 1.8, 56.7ms, 0 | 20260505 16:28:37.562 | INFO | BAZWRITER : 63.9%, 3.9, 128.3ms, 0 | 20260505 16:28:37.562 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 16:28:37.562 | INFO | Uploaded 36GiB to GPU in 2.20076s (16.358GiB/s) | 20260505 16:28:37.562 | INFO | Downloaded 38GiB to GPU in 2.26874s (16.7494GiB/s) | 20260505 16:28:42.704 | INFO | Analyzing chunk frames = [4645376,4645888) | 20260505 16:28:42.704 | INFO | MemInfo: { | 20260505 16:28:42.704 | INFO | "gpu" : " 17 GB", | 20260505 16:28:42.704 | INFO | "huge" : "", | 20260505 16:28:42.704 | INFO | "huge_shared" : " 27 GB", | 20260505 16:28:42.704 | INFO | "malloc" : " 14 GB", | 20260505 16:28:42.704 | INFO | "pinned" : " 29 GB" | 20260505 16:28:42.704 | INFO | } | 20260505 16:28:42.704 | INFO | Perf report for the last 5.14241 seconds: | 20260505 16:28:42.704 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:28:42.704 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:42.704 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:42.704 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:42.704 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 16:28:42.704 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:42.704 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:42.704 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:28:42.704 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:42.704 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:42.704 | INFO | Uploaded 36GiB to GPU in 2.2033s (16.3391GiB/s) | 20260505 16:28:42.704 | INFO | Downloaded 34GiB to GPU in 2.0676s (16.4442GiB/s) | 20260505 16:28:47.850 | INFO | Analyzing chunk frames = [4645888,4646400) | 20260505 16:28:47.850 | INFO | MemInfo: { | 20260505 16:28:47.850 | INFO | "gpu" : " 17 GB", | 20260505 16:28:47.850 | INFO | "huge" : "", | 20260505 16:28:47.850 | INFO | "huge_shared" : " 27 GB", | 20260505 16:28:47.850 | INFO | "malloc" : " 14 GB", | 20260505 16:28:47.850 | INFO | "pinned" : " 29 GB" | 20260505 16:28:47.850 | INFO | } | 20260505 16:28:47.850 | INFO | Perf report for the last 5.14651 seconds: | 20260505 16:28:47.850 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:28:47.850 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:47.850 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:47.850 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:47.850 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260505 16:28:47.850 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:47.850 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:47.850 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:28:47.850 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:47.850 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:47.851 | INFO | Uploaded 36GiB to GPU in 2.19821s (16.377GiB/s) | 20260505 16:28:47.851 | INFO | Downloaded 34GiB to GPU in 2.03406s (16.7154GiB/s) | 20260505 16:28:52.986 | INFO | Analyzing chunk frames = [4646400,4646912) | 20260505 16:28:52.986 | INFO | MemInfo: { | 20260505 16:28:52.986 | INFO | "gpu" : " 17 GB", | 20260505 16:28:52.986 | INFO | "huge" : "", | 20260505 16:28:52.986 | INFO | "huge_shared" : " 27 GB", | 20260505 16:28:52.986 | INFO | "malloc" : " 14 GB", | 20260505 16:28:52.986 | INFO | "pinned" : " 29 GB" | 20260505 16:28:52.986 | INFO | } | 20260505 16:28:52.987 | INFO | Perf report for the last 5.13607 seconds: | 20260505 16:28:52.987 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:28:52.987 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:52.987 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:52.987 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:52.987 | INFO | ANALYSIS : 62.9%, 3.0, 97.7ms, 0 | 20260505 16:28:52.987 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:52.987 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:52.987 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 16:28:52.987 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:52.987 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:52.987 | INFO | Uploaded 36GiB to GPU in 2.22382s (16.1884GiB/s) | 20260505 16:28:52.987 | INFO | Downloaded 34GiB to GPU in 2.12522s (15.9984GiB/s) | 20260505 16:28:58.115 | INFO | Analyzing chunk frames = [4646912,4647424) | 20260505 16:28:58.115 | INFO | MemInfo: { | 20260505 16:28:58.115 | INFO | "gpu" : " 17 GB", | 20260505 16:28:58.115 | INFO | "huge" : "", | 20260505 16:28:58.115 | INFO | "huge_shared" : " 27 GB", | 20260505 16:28:58.115 | INFO | "malloc" : " 14 GB", | 20260505 16:28:58.115 | INFO | "pinned" : " 29 GB" | 20260505 16:28:58.115 | INFO | } | 20260505 16:28:58.115 | INFO | Perf report for the last 5.12864 seconds: | 20260505 16:28:58.115 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:28:58.115 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:58.115 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:58.115 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:58.115 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260505 16:28:58.115 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:58.115 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:58.115 | INFO | PRE_HQ : 27.9%, 1.0, 14.4ms, 0 | 20260505 16:28:58.115 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:58.115 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:28:58.115 | INFO | Uploaded 36GiB to GPU in 2.14234s (16.8041GiB/s) | 20260505 16:28:58.115 | INFO | Downloaded 34GiB to GPU in 2.05845s (16.5173GiB/s) | 20260505 16:29:03.260 | INFO | Analyzing chunk frames = [4647424,4647936) | 20260505 16:29:03.260 | INFO | MemInfo: { | 20260505 16:29:03.260 | INFO | "gpu" : " 17 GB", | 20260505 16:29:03.261 | INFO | "huge" : "", | 20260505 16:29:03.261 | INFO | "huge_shared" : " 27 GB", | 20260505 16:29:03.261 | INFO | "malloc" : " 14 GB", | 20260505 16:29:03.261 | INFO | "pinned" : " 29 GB" | 20260505 16:29:03.261 | INFO | } | 20260505 16:29:03.261 | INFO | Perf report for the last 5.14551 seconds: | 20260505 16:29:03.261 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:29:03.261 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:03.261 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:03.261 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:03.261 | INFO | ANALYSIS : 62.7%, 3.0, 98.1ms, 0 | 20260505 16:29:03.261 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:03.261 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:03.261 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:29:03.261 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:03.261 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:03.261 | INFO | Uploaded 36GiB to GPU in 2.17979s (16.5154GiB/s) | 20260505 16:29:03.261 | INFO | Downloaded 34GiB to GPU in 2.11153s (16.1021GiB/s) | 20260505 16:29:08.405 | INFO | Analyzing chunk frames = [4647936,4648448) | 20260505 16:29:08.405 | INFO | MemInfo: { | 20260505 16:29:08.405 | INFO | "gpu" : " 17 GB", | 20260505 16:29:08.405 | INFO | "huge" : "", | 20260505 16:29:08.405 | INFO | "huge_shared" : " 27 GB", | 20260505 16:29:08.405 | INFO | "malloc" : " 14 GB", | 20260505 16:29:08.405 | INFO | "pinned" : " 29 GB" | 20260505 16:29:08.405 | INFO | } | 20260505 16:29:08.405 | INFO | Perf report for the last 5.14399 seconds: | 20260505 16:29:08.405 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:29:08.405 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:08.405 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:08.405 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:08.405 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 16:29:08.405 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:08.405 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:08.405 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 16:29:08.405 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:08.405 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:08.405 | INFO | Uploaded 36GiB to GPU in 2.13385s (16.8709GiB/s) | 20260505 16:29:08.405 | INFO | Downloaded 34GiB to GPU in 2.17166s (15.6562GiB/s) | 20260505 16:29:13.547 | INFO | Analyzing chunk frames = [4648448,4648960) | 20260505 16:29:13.547 | INFO | MemInfo: { | 20260505 16:29:13.547 | INFO | "gpu" : " 17 GB", | 20260505 16:29:13.547 | INFO | "huge" : "", | 20260505 16:29:13.547 | INFO | "huge_shared" : " 27 GB", | 20260505 16:29:13.547 | INFO | "malloc" : " 14 GB", | 20260505 16:29:13.548 | INFO | "pinned" : " 29 GB" | 20260505 16:29:13.548 | INFO | } | 20260505 16:29:13.548 | INFO | Perf report for the last 5.1429 seconds: | 20260505 16:29:13.548 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:29:13.548 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:13.548 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:13.548 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:13.548 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 16:29:13.548 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:13.548 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:13.548 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 16:29:13.548 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:13.548 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:13.548 | INFO | Uploaded 36GiB to GPU in 2.15678s (16.6916GiB/s) | 20260505 16:29:13.548 | INFO | Downloaded 34GiB to GPU in 2.01918s (16.8385GiB/s) | 20260505 16:29:13.685 | INFO | Starting a round of Baz IO | 20260505 16:29:16.882 | INFO | Populating RTMetrics report | 20260505 16:29:16.883 | INFO | RTMetrics report populated | 20260505 16:29:16.883 | INFO | Opening tmp RTMetrics json file | 20260505 16:29:16.895 | INFO | Serializing RTMetrics to json | 20260505 16:29:16.900 | INFO | Serialization done, swapping tmp file now... | 20260505 16:29:16.907 | INFO | RTMetrics done | 20260505 16:29:16.982 | INFO | Finished a round of Baz IO: Wrote 2.17 GiB in 3.296 seconds | 20260505 16:29:16.982 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.24% / 0.00% / 77.76% | 20260505 16:29:16.982 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.361367% / 7.94729e-06% / 5.96046e-05% | 20260505 16:29:16.982 | INFO | Baz IO overall: | 20260505 16:29:16.982 | INFO | BytesWritten : 2758110922768 | 20260505 16:29:16.982 | INFO | NumEvents : 1309087926452 | 20260505 16:29:18.685 | INFO | Analyzing chunk frames = [4648960,4649472) | 20260505 16:29:18.685 | INFO | MemInfo: { | 20260505 16:29:18.685 | INFO | "gpu" : " 17 GB", | 20260505 16:29:18.685 | INFO | "huge" : "", | 20260505 16:29:18.685 | INFO | "huge_shared" : " 27 GB", | 20260505 16:29:18.685 | INFO | "malloc" : " 14 GB", | 20260505 16:29:18.685 | INFO | "pinned" : " 29 GB" | 20260505 16:29:18.685 | INFO | } | 20260505 16:29:18.685 | INFO | Perf report for the last 5.13712 seconds: | 20260505 16:29:18.685 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:29:18.685 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:18.685 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:18.685 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:18.685 | INFO | ANALYSIS : 63.8%, 3.0, 99.2ms, 0 | 20260505 16:29:18.685 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:18.685 | INFO | RT_METRICS : 45.1%, 1.0, 23.7ms, 0 | 20260505 16:29:18.685 | INFO | PRE_HQ : 60.8%, 1.8, 56.5ms, 0 | 20260505 16:29:18.685 | INFO | BAZWRITER : 63.6%, 3.9, 128.8ms, 0 | 20260505 16:29:18.685 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 16:29:18.685 | INFO | Uploaded 36GiB to GPU in 2.19513s (16.4GiB/s) | 20260505 16:29:18.685 | INFO | Downloaded 38GiB to GPU in 2.26055s (16.8101GiB/s) | 20260505 16:29:23.816 | INFO | Analyzing chunk frames = [4649472,4649984) | 20260505 16:29:23.816 | INFO | MemInfo: { | 20260505 16:29:23.816 | INFO | "gpu" : " 17 GB", | 20260505 16:29:23.816 | INFO | "huge" : "", | 20260505 16:29:23.816 | INFO | "huge_shared" : " 27 GB", | 20260505 16:29:23.816 | INFO | "malloc" : " 14 GB", | 20260505 16:29:23.816 | INFO | "pinned" : " 29 GB" | 20260505 16:29:23.816 | INFO | } | 20260505 16:29:23.816 | INFO | Perf report for the last 5.13155 seconds: | 20260505 16:29:23.816 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:29:23.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:23.816 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:23.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:23.816 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 16:29:23.816 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:23.816 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:23.816 | INFO | PRE_HQ : 27.7%, 1.0, 14.3ms, 0 | 20260505 16:29:23.816 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:23.816 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:23.817 | INFO | Uploaded 36GiB to GPU in 2.12567s (16.9359GiB/s) | 20260505 16:29:23.817 | INFO | Downloaded 34GiB to GPU in 2.02823s (16.7634GiB/s) | 20260505 16:29:28.959 | INFO | Analyzing chunk frames = [4649984,4650496) | 20260505 16:29:28.959 | INFO | MemInfo: { | 20260505 16:29:28.959 | INFO | "gpu" : " 17 GB", | 20260505 16:29:28.959 | INFO | "huge" : "", | 20260505 16:29:28.959 | INFO | "huge_shared" : " 27 GB", | 20260505 16:29:28.959 | INFO | "malloc" : " 14 GB", | 20260505 16:29:28.959 | INFO | "pinned" : " 29 GB" | 20260505 16:29:28.959 | INFO | } | 20260505 16:29:28.959 | INFO | Perf report for the last 5.14263 seconds: | 20260505 16:29:28.959 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:29:28.959 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:28.959 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:28.959 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:28.959 | INFO | ANALYSIS : 63.0%, 3.0, 98.0ms, 0 | 20260505 16:29:28.959 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:28.959 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:28.959 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:29:28.959 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:28.959 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:28.959 | INFO | Uploaded 36GiB to GPU in 2.19547s (16.3974GiB/s) | 20260505 16:29:28.959 | INFO | Downloaded 34GiB to GPU in 2.06043s (16.5014GiB/s) | 20260505 16:29:34.101 | INFO | Analyzing chunk frames = [4650496,4651008) | 20260505 16:29:34.101 | INFO | MemInfo: { | 20260505 16:29:34.101 | INFO | "gpu" : " 17 GB", | 20260505 16:29:34.101 | INFO | "huge" : "", | 20260505 16:29:34.101 | INFO | "huge_shared" : " 27 GB", | 20260505 16:29:34.101 | INFO | "malloc" : " 14 GB", | 20260505 16:29:34.101 | INFO | "pinned" : " 29 GB" | 20260505 16:29:34.101 | INFO | } | 20260505 16:29:34.101 | INFO | Perf report for the last 5.14221 seconds: | 20260505 16:29:34.101 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:29:34.101 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:34.101 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:34.101 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:34.101 | INFO | ANALYSIS : 62.7%, 3.0, 97.4ms, 0 | 20260505 16:29:34.101 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:34.101 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:34.101 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 16:29:34.101 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:34.101 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:34.101 | INFO | Uploaded 36GiB to GPU in 2.18105s (16.5058GiB/s) | 20260505 16:29:34.102 | INFO | Downloaded 34GiB to GPU in 2.14342s (15.8625GiB/s) | 20260505 16:29:39.247 | INFO | Analyzing chunk frames = [4651008,4651520) | 20260505 16:29:39.247 | INFO | MemInfo: { | 20260505 16:29:39.247 | INFO | "gpu" : " 17 GB", | 20260505 16:29:39.247 | INFO | "huge" : "", | 20260505 16:29:39.247 | INFO | "huge_shared" : " 27 GB", | 20260505 16:29:39.247 | INFO | "malloc" : " 14 GB", | 20260505 16:29:39.247 | INFO | "pinned" : " 29 GB" | 20260505 16:29:39.247 | INFO | } | 20260505 16:29:39.247 | INFO | Perf report for the last 5.14619 seconds: | 20260505 16:29:39.247 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:29:39.247 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:39.247 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:39.247 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:39.247 | INFO | ANALYSIS : 62.3%, 3.0, 97.4ms, 0 | 20260505 16:29:39.247 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:39.248 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:39.248 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:29:39.248 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:39.248 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:39.248 | INFO | Uploaded 36GiB to GPU in 2.14709s (16.7669GiB/s) | 20260505 16:29:39.248 | INFO | Downloaded 34GiB to GPU in 2.05768s (16.5235GiB/s) | 20260505 16:29:44.380 | INFO | Analyzing chunk frames = [4651520,4652032) | 20260505 16:29:44.380 | INFO | MemInfo: { | 20260505 16:29:44.380 | INFO | "gpu" : " 17 GB", | 20260505 16:29:44.380 | INFO | "huge" : "", | 20260505 16:29:44.380 | INFO | "huge_shared" : " 27 GB", | 20260505 16:29:44.380 | INFO | "malloc" : " 14 GB", | 20260505 16:29:44.381 | INFO | "pinned" : " 29 GB" | 20260505 16:29:44.381 | INFO | } | 20260505 16:29:44.381 | INFO | Perf report for the last 5.13323 seconds: | 20260505 16:29:44.381 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:29:44.381 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:44.381 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:44.381 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:44.381 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260505 16:29:44.381 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:44.381 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:44.381 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 16:29:44.381 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:44.381 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:44.381 | INFO | Uploaded 36GiB to GPU in 2.14265s (16.8016GiB/s) | 20260505 16:29:44.381 | INFO | Downloaded 34GiB to GPU in 2.04148s (16.6546GiB/s) | 20260505 16:29:49.512 | INFO | Analyzing chunk frames = [4652032,4652544) | 20260505 16:29:49.512 | INFO | MemInfo: { | 20260505 16:29:49.512 | INFO | "gpu" : " 17 GB", | 20260505 16:29:49.512 | INFO | "huge" : "", | 20260505 16:29:49.512 | INFO | "huge_shared" : " 27 GB", | 20260505 16:29:49.513 | INFO | "malloc" : " 14 GB", | 20260505 16:29:49.513 | INFO | "pinned" : " 29 GB" | 20260505 16:29:49.513 | INFO | } | 20260505 16:29:49.513 | INFO | Perf report for the last 5.13201 seconds: | 20260505 16:29:49.513 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:29:49.513 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:49.513 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:49.513 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:49.513 | INFO | ANALYSIS : 63.4%, 3.0, 98.8ms, 0 | 20260505 16:29:49.513 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:49.513 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:49.513 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 16:29:49.513 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:49.513 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:49.513 | INFO | Uploaded 36GiB to GPU in 2.12943s (16.9059GiB/s) | 20260505 16:29:49.513 | INFO | Downloaded 34GiB to GPU in 2.16903s (15.6752GiB/s) | 20260505 16:29:54.660 | INFO | Analyzing chunk frames = [4652544,4653056) | 20260505 16:29:54.660 | INFO | MemInfo: { | 20260505 16:29:54.660 | INFO | "gpu" : " 17 GB", | 20260505 16:29:54.660 | INFO | "huge" : "", | 20260505 16:29:54.660 | INFO | "huge_shared" : " 27 GB", | 20260505 16:29:54.660 | INFO | "malloc" : " 14 GB", | 20260505 16:29:54.660 | INFO | "pinned" : " 29 GB" | 20260505 16:29:54.660 | INFO | } | 20260505 16:29:54.660 | INFO | Perf report for the last 5.14741 seconds: | 20260505 16:29:54.660 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:29:54.660 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:54.660 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:54.660 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:54.660 | INFO | ANALYSIS : 63.5%, 3.0, 99.3ms, 0 | 20260505 16:29:54.660 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:54.660 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:54.660 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:29:54.660 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:54.660 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:54.660 | INFO | Uploaded 36GiB to GPU in 2.23462s (16.1102GiB/s) | 20260505 16:29:54.660 | INFO | Downloaded 34GiB to GPU in 2.10788s (16.1299GiB/s) | 20260505 16:29:54.660 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 16:29:54.660 | INFO | Total mem usage: 15.3 GiB | 20260505 16:29:54.660 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 16:29:54.660 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.86%) | 20260505 16:29:54.660 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 16:29:54.660 | INFO | High Water Breakdown: | 20260505 16:29:54.660 | INFO | 61% : 8.67 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 16:29:54.660 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 16:29:54.660 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 16:29:54.660 | INFO | ------------------------------------------ | 20260505 16:29:54.661 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:29:54.661 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 16:29:54.661 | INFO | Total mem usage: 26.5 GiB | 20260505 16:29:54.661 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 16:29:54.661 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 16:29:54.661 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 16:29:54.661 | INFO | High Water Breakdown: | 20260505 16:29:54.661 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:29:54.661 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:29:54.661 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:29:54.661 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:29:54.661 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:29:54.661 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:29:54.661 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 16:29:54.661 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 16:29:54.661 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:29:54.661 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 16:29:54.661 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 16:29:54.661 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 16:29:54.661 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 16:29:54.661 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 16:29:54.661 | INFO | ------------------------------------------ | 20260505 16:29:54.661 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:29:54.661 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 16:29:54.661 | INFO | Total mem usage: 25.6 GiB | 20260505 16:29:54.661 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 16:29:54.661 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 16:29:54.661 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 16:29:54.661 | INFO | High Water Breakdown: | 20260505 16:29:54.661 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 16:29:54.661 | INFO | ------------------------------------------ | 20260505 16:29:54.661 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:29:54.661 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 16:29:54.661 | INFO | Total mem usage: 15.7 GiB | 20260505 16:29:54.661 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 16:29:54.661 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 16:29:54.661 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 16:29:54.661 | INFO | High Water Breakdown: | 20260505 16:29:54.661 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 16:29:54.661 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 16:29:54.661 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 16:29:54.661 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 16:29:54.661 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 16:29:54.661 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 16:29:54.661 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 16:29:54.661 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 16:29:54.661 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 16:29:54.661 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:29:54.661 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 16:29:54.661 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:29:54.661 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 16:29:54.661 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 16:29:54.661 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 16:29:54.661 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 16:29:54.661 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:29:54.661 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 16:29:54.661 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:29:54.661 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:29:54.661 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:29:54.661 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 16:29:54.661 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 16:29:54.661 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:29:54.661 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 16:29:54.661 | INFO | ------------------------------------------ | 20260505 16:29:54.661 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:29:54.661 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 16:29:54.661 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:29:54.661 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:29:54.661 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.52e+01, 1.35e+01 / 2.13e+00 | 20260505 16:29:54.661 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.01e-01, 1.05e-02 / 7.25e-03 | 20260505 16:29:54.661 | INFO | PerfReport: 11.73%, Baseline -- 3.71e+00 / 4.49e+00, 5.50e-01 / 1.64e-01 | 20260505 16:29:54.661 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.49e+00, 1.90e-01 / 1.27e-01 | 20260505 16:29:54.661 | INFO | PerfReport: 16.00%, DME -- 4.05e+01 / 4.52e+01, 4.24e+00 / 2.00e+00 | 20260505 16:29:54.661 | INFO | PerfReport: 56.69%, FrameLabeling -- 1.79e+01 / 1.91e+01, 1.39e+00 / 1.12e+00 | 20260505 16:29:54.661 | INFO | PerfReport: 7.41%, PulseAccumulating -- 2.34e+00 / 2.96e+00, 4.54e-01 / 9.48e-02 | 20260505 16:29:54.661 | INFO | PerfReport: 3.42%, Metrics -- 1.08e+00 / 1.41e+00, 2.98e-01 / 1.38e-01 | 20260505 16:29:54.661 | INFO | PerfReport: 1.19%, PulseSerializing -- 3.76e-01 / 4.33e-01, 6.68e-02 / 3.62e-02 | 20260505 16:29:54.661 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 16:29:54.661 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:29:54.661 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:29:54.661 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.57e+01 / 0.00e+00 | 20260505 16:29:54.661 | INFO | PerfReport: 22.58%, Upload -- 2.27e+01 / 0.00e+00, 7.91e+00 / 0.00e+00 | 20260505 16:29:54.661 | INFO | PerfReport: 21.89%, Download -- 2.20e+01 / 0.00e+00, 7.88e+00 / 0.00e+00 | 20260505 16:29:54.661 | INFO | PerfReport: 31.52%, Compute -- 3.16e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 16:29:54.661 | INFO | PerfReport: 2.39%, LockUpload -- 2.40e+00 / 0.00e+00, 8.04e+00 / 0.00e+00 | 20260505 16:29:54.661 | INFO | PerfReport: 1.42%, LockDownload -- 1.43e+00 / 0.00e+00, 2.97e+00 / 0.00e+00 | 20260505 16:29:54.661 | INFO | PerfReport: 20.18%, LockCompute -- 2.02e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 16:29:54.661 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.69e-02 / 0.00e+00 | 20260505 16:29:54.796 | INFO | Starting a round of Baz IO | 20260505 16:29:57.976 | INFO | Populating RTMetrics report | 20260505 16:29:57.976 | INFO | RTMetrics report populated | 20260505 16:29:57.976 | INFO | Opening tmp RTMetrics json file | 20260505 16:29:58.010 | INFO | Serializing RTMetrics to json | 20260505 16:29:58.035 | INFO | Serialization done, swapping tmp file now... | 20260505 16:29:58.041 | INFO | RTMetrics done | 20260505 16:29:58.076 | INFO | Finished a round of Baz IO: Wrote 2.169 GiB in 3.28 seconds | 20260505 16:29:58.076 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.23% / 0.00% / 77.76% | 20260505 16:29:58.076 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.360036% / 7.94729e-06% / 6.75519e-05% | 20260505 16:29:58.076 | INFO | Baz IO overall: | 20260505 16:29:58.076 | INFO | BytesWritten : 2760440354760 | 20260505 16:29:58.077 | INFO | NumEvents : 1310179713721 | 20260505 16:29:59.802 | INFO | Analyzing chunk frames = [4653056,4653568) | 20260505 16:29:59.802 | INFO | MemInfo: { | 20260505 16:29:59.802 | INFO | "gpu" : " 17 GB", | 20260505 16:29:59.802 | INFO | "huge" : "", | 20260505 16:29:59.802 | INFO | "huge_shared" : " 27 GB", | 20260505 16:29:59.802 | INFO | "malloc" : " 14 GB", | 20260505 16:29:59.802 | INFO | "pinned" : " 29 GB" | 20260505 16:29:59.802 | INFO | } | 20260505 16:29:59.802 | INFO | Perf report for the last 5.14211 seconds: | 20260505 16:29:59.802 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:29:59.802 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:59.802 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:59.802 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:59.802 | INFO | ANALYSIS : 63.4%, 3.0, 98.8ms, 0 | 20260505 16:29:59.802 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:29:59.802 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260505 16:29:59.802 | INFO | PRE_HQ : 60.6%, 1.8, 56.4ms, 0 | 20260505 16:29:59.802 | INFO | BAZWRITER : 63.3%, 3.9, 127.4ms, 0 | 20260505 16:29:59.802 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 16:29:59.803 | INFO | Uploaded 36GiB to GPU in 2.14838s (16.7568GiB/s) | 20260505 16:29:59.803 | INFO | Downloaded 38GiB to GPU in 2.23808s (16.9788GiB/s) | 20260505 16:30:04.933 | INFO | Analyzing chunk frames = [4653568,4654080) | 20260505 16:30:04.933 | INFO | MemInfo: { | 20260505 16:30:04.933 | INFO | "gpu" : " 17 GB", | 20260505 16:30:04.933 | INFO | "huge" : "", | 20260505 16:30:04.933 | INFO | "huge_shared" : " 27 GB", | 20260505 16:30:04.933 | INFO | "malloc" : " 14 GB", | 20260505 16:30:04.933 | INFO | "pinned" : " 29 GB" | 20260505 16:30:04.933 | INFO | } | 20260505 16:30:04.933 | INFO | Perf report for the last 5.13108 seconds: | 20260505 16:30:04.933 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:30:04.933 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:04.933 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:04.933 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:04.933 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 16:30:04.933 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:04.933 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:04.933 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:30:04.933 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:04.933 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:04.934 | INFO | Uploaded 36GiB to GPU in 2.12108s (16.9725GiB/s) | 20260505 16:30:04.934 | INFO | Downloaded 34GiB to GPU in 2.02551s (16.7859GiB/s) | 20260505 16:30:10.068 | INFO | Analyzing chunk frames = [4654080,4654592) | 20260505 16:30:10.068 | INFO | MemInfo: { | 20260505 16:30:10.068 | INFO | "gpu" : " 17 GB", | 20260505 16:30:10.068 | INFO | "huge" : "", | 20260505 16:30:10.068 | INFO | "huge_shared" : " 27 GB", | 20260505 16:30:10.068 | INFO | "malloc" : " 14 GB", | 20260505 16:30:10.068 | INFO | "pinned" : " 29 GB" | 20260505 16:30:10.068 | INFO | } | 20260505 16:30:10.068 | INFO | Perf report for the last 5.13507 seconds: | 20260505 16:30:10.068 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:30:10.068 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:10.068 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:10.068 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:10.068 | INFO | ANALYSIS : 63.0%, 3.0, 97.9ms, 0 | 20260505 16:30:10.068 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:10.068 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:10.068 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:30:10.068 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:10.068 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:10.069 | INFO | Uploaded 36GiB to GPU in 2.19034s (16.4358GiB/s) | 20260505 16:30:10.069 | INFO | Downloaded 34GiB to GPU in 2.04262s (16.6453GiB/s) | 20260505 16:30:15.218 | INFO | Analyzing chunk frames = [4654592,4655104) | 20260505 16:30:15.218 | INFO | MemInfo: { | 20260505 16:30:15.218 | INFO | "gpu" : " 17 GB", | 20260505 16:30:15.218 | INFO | "huge" : "", | 20260505 16:30:15.218 | INFO | "huge_shared" : " 27 GB", | 20260505 16:30:15.218 | INFO | "malloc" : " 14 GB", | 20260505 16:30:15.218 | INFO | "pinned" : " 29 GB" | 20260505 16:30:15.218 | INFO | } | 20260505 16:30:15.218 | INFO | Perf report for the last 5.14949 seconds: | 20260505 16:30:15.218 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:30:15.218 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:15.218 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:15.218 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:15.218 | INFO | ANALYSIS : 62.5%, 3.0, 97.2ms, 0 | 20260505 16:30:15.218 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:15.218 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:15.218 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:30:15.218 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:15.218 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:15.218 | INFO | Uploaded 36GiB to GPU in 2.18239s (16.4957GiB/s) | 20260505 16:30:15.218 | INFO | Downloaded 34GiB to GPU in 2.13146s (15.9515GiB/s) | 20260505 16:30:20.362 | INFO | Analyzing chunk frames = [4655104,4655616) | 20260505 16:30:20.362 | INFO | MemInfo: { | 20260505 16:30:20.362 | INFO | "gpu" : " 17 GB", | 20260505 16:30:20.362 | INFO | "huge" : "", | 20260505 16:30:20.362 | INFO | "huge_shared" : " 27 GB", | 20260505 16:30:20.362 | INFO | "malloc" : " 14 GB", | 20260505 16:30:20.362 | INFO | "pinned" : " 29 GB" | 20260505 16:30:20.362 | INFO | } | 20260505 16:30:20.362 | INFO | Perf report for the last 5.14402 seconds: | 20260505 16:30:20.362 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:30:20.362 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:20.362 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:20.362 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:20.362 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260505 16:30:20.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:20.362 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:20.362 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 16:30:20.362 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:20.362 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:20.362 | INFO | Uploaded 36GiB to GPU in 2.15694s (16.6903GiB/s) | 20260505 16:30:20.362 | INFO | Downloaded 34GiB to GPU in 2.05129s (16.575GiB/s) | 20260505 16:30:25.494 | INFO | Analyzing chunk frames = [4655616,4656128) | 20260505 16:30:25.494 | INFO | MemInfo: { | 20260505 16:30:25.494 | INFO | "gpu" : " 17 GB", | 20260505 16:30:25.494 | INFO | "huge" : "", | 20260505 16:30:25.494 | INFO | "huge_shared" : " 27 GB", | 20260505 16:30:25.494 | INFO | "malloc" : " 14 GB", | 20260505 16:30:25.494 | INFO | "pinned" : " 29 GB" | 20260505 16:30:25.494 | INFO | } | 20260505 16:30:25.494 | INFO | Perf report for the last 5.13226 seconds: | 20260505 16:30:25.494 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:30:25.494 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:25.494 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:25.494 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:25.494 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260505 16:30:25.494 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:25.494 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:25.494 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:30:25.494 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:25.494 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:25.494 | INFO | Uploaded 36GiB to GPU in 2.14171s (16.809GiB/s) | 20260505 16:30:25.495 | INFO | Downloaded 34GiB to GPU in 2.05184s (16.5705GiB/s) | 20260505 16:30:30.627 | INFO | Analyzing chunk frames = [4656128,4656640) | 20260505 16:30:30.627 | INFO | MemInfo: { | 20260505 16:30:30.627 | INFO | "gpu" : " 17 GB", | 20260505 16:30:30.627 | INFO | "huge" : "", | 20260505 16:30:30.627 | INFO | "huge_shared" : " 27 GB", | 20260505 16:30:30.627 | INFO | "malloc" : " 14 GB", | 20260505 16:30:30.628 | INFO | "pinned" : " 29 GB" | 20260505 16:30:30.628 | INFO | } | 20260505 16:30:30.628 | INFO | Perf report for the last 5.13344 seconds: | 20260505 16:30:30.628 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:30:30.628 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:30.628 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:30.628 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:30.628 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 16:30:30.628 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:30.628 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:30.628 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 16:30:30.628 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:30.628 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:30.628 | INFO | Uploaded 36GiB to GPU in 2.12266s (16.9598GiB/s) | 20260505 16:30:30.628 | INFO | Downloaded 34GiB to GPU in 2.18449s (15.5643GiB/s) | 20260505 16:30:35.767 | INFO | Analyzing chunk frames = [4656640,4657152) | 20260505 16:30:35.767 | INFO | MemInfo: { | 20260505 16:30:35.767 | INFO | "gpu" : " 17 GB", | 20260505 16:30:35.767 | INFO | "huge" : "", | 20260505 16:30:35.767 | INFO | "huge_shared" : " 27 GB", | 20260505 16:30:35.767 | INFO | "malloc" : " 14 GB", | 20260505 16:30:35.767 | INFO | "pinned" : " 29 GB" | 20260505 16:30:35.767 | INFO | } | 20260505 16:30:35.767 | INFO | Perf report for the last 5.13921 seconds: | 20260505 16:30:35.767 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:30:35.767 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:35.767 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:35.767 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:35.767 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 16:30:35.767 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:35.767 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:35.767 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 16:30:35.767 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:35.767 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:35.767 | INFO | Uploaded 36GiB to GPU in 2.23733s (16.0906GiB/s) | 20260505 16:30:35.767 | INFO | Downloaded 34GiB to GPU in 2.10127s (16.1807GiB/s) | 20260505 16:30:35.895 | INFO | Starting a round of Baz IO | 20260505 16:30:39.107 | INFO | Populating RTMetrics report | 20260505 16:30:39.107 | INFO | RTMetrics report populated | 20260505 16:30:39.107 | INFO | Opening tmp RTMetrics json file | 20260505 16:30:39.117 | INFO | Serializing RTMetrics to json | 20260505 16:30:39.127 | INFO | Serialization done, swapping tmp file now... | 20260505 16:30:39.143 | INFO | RTMetrics done | 20260505 16:30:39.204 | INFO | Finished a round of Baz IO: Wrote 2.17 GiB in 3.309 seconds | 20260505 16:30:39.205 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.23% / 0.00% / 77.77% | 20260505 16:30:39.205 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.360509% / 0% / 6.35783e-05% | 20260505 16:30:39.205 | INFO | Baz IO overall: | 20260505 16:30:39.205 | INFO | BytesWritten : 2762770873775 | 20260505 16:30:39.205 | INFO | NumEvents : 1311272456803 | 20260505 16:30:40.911 | INFO | Analyzing chunk frames = [4657152,4657664) | 20260505 16:30:40.911 | INFO | MemInfo: { | 20260505 16:30:40.911 | INFO | "gpu" : " 17 GB", | 20260505 16:30:40.911 | INFO | "huge" : "", | 20260505 16:30:40.911 | INFO | "huge_shared" : " 27 GB", | 20260505 16:30:40.911 | INFO | "malloc" : " 14 GB", | 20260505 16:30:40.911 | INFO | "pinned" : " 29 GB" | 20260505 16:30:40.911 | INFO | } | 20260505 16:30:40.911 | INFO | Perf report for the last 5.14416 seconds: | 20260505 16:30:40.911 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:30:40.911 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:40.911 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:40.911 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:40.911 | INFO | ANALYSIS : 63.8%, 3.0, 99.3ms, 0 | 20260505 16:30:40.911 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:40.911 | INFO | RT_METRICS : 45.3%, 1.0, 23.8ms, 0 | 20260505 16:30:40.911 | INFO | PRE_HQ : 60.8%, 1.8, 56.7ms, 0 | 20260505 16:30:40.911 | INFO | BAZWRITER : 63.6%, 3.9, 128.5ms, 0 | 20260505 16:30:40.911 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260505 16:30:40.911 | INFO | Uploaded 36GiB to GPU in 2.17602s (16.5439GiB/s) | 20260505 16:30:40.911 | INFO | Downloaded 38GiB to GPU in 2.26306s (16.7914GiB/s) | 20260505 16:30:46.053 | INFO | Analyzing chunk frames = [4657664,4658176) | 20260505 16:30:46.053 | INFO | MemInfo: { | 20260505 16:30:46.053 | INFO | "gpu" : " 17 GB", | 20260505 16:30:46.053 | INFO | "huge" : "", | 20260505 16:30:46.053 | INFO | "huge_shared" : " 27 GB", | 20260505 16:30:46.053 | INFO | "malloc" : " 14 GB", | 20260505 16:30:46.053 | INFO | "pinned" : " 29 GB" | 20260505 16:30:46.053 | INFO | } | 20260505 16:30:46.053 | INFO | Perf report for the last 5.14236 seconds: | 20260505 16:30:46.053 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:30:46.053 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:46.053 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:46.053 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:46.053 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 16:30:46.053 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:46.053 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:46.053 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:30:46.053 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:46.054 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:46.054 | INFO | Uploaded 36GiB to GPU in 2.12316s (16.9558GiB/s) | 20260505 16:30:46.054 | INFO | Downloaded 34GiB to GPU in 2.03287s (16.7251GiB/s) | 20260505 16:30:51.193 | INFO | Analyzing chunk frames = [4658176,4658688) | 20260505 16:30:51.193 | INFO | MemInfo: { | 20260505 16:30:51.193 | INFO | "gpu" : " 17 GB", | 20260505 16:30:51.193 | INFO | "huge" : "", | 20260505 16:30:51.193 | INFO | "huge_shared" : " 27 GB", | 20260505 16:30:51.193 | INFO | "malloc" : " 14 GB", | 20260505 16:30:51.194 | INFO | "pinned" : " 29 GB" | 20260505 16:30:51.194 | INFO | } | 20260505 16:30:51.194 | INFO | Perf report for the last 5.1403 seconds: | 20260505 16:30:51.194 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:30:51.194 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:51.194 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:51.194 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:51.194 | INFO | ANALYSIS : 62.7%, 3.0, 97.5ms, 0 | 20260505 16:30:51.194 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:51.194 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:51.194 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 16:30:51.194 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:51.194 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:51.194 | INFO | Uploaded 36GiB to GPU in 2.19918s (16.3698GiB/s) | 20260505 16:30:51.194 | INFO | Downloaded 34GiB to GPU in 2.03496s (16.7079GiB/s) | 20260505 16:30:56.327 | INFO | Analyzing chunk frames = [4658688,4659200) | 20260505 16:30:56.327 | INFO | MemInfo: { | 20260505 16:30:56.327 | INFO | "gpu" : " 17 GB", | 20260505 16:30:56.327 | INFO | "huge" : "", | 20260505 16:30:56.327 | INFO | "huge_shared" : " 27 GB", | 20260505 16:30:56.327 | INFO | "malloc" : " 14 GB", | 20260505 16:30:56.327 | INFO | "pinned" : " 29 GB" | 20260505 16:30:56.328 | INFO | } | 20260505 16:30:56.328 | INFO | Perf report for the last 5.13391 seconds: | 20260505 16:30:56.328 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:30:56.328 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:56.328 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:56.328 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:56.328 | INFO | ANALYSIS : 62.6%, 3.0, 97.2ms, 0 | 20260505 16:30:56.328 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:56.328 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:56.328 | INFO | PRE_HQ : 27.9%, 1.0, 14.4ms, 0 | 20260505 16:30:56.328 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:56.328 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:30:56.328 | INFO | Uploaded 36GiB to GPU in 2.16713s (16.6118GiB/s) | 20260505 16:30:56.328 | INFO | Downloaded 34GiB to GPU in 2.14375s (15.86GiB/s) | 20260505 16:31:01.470 | INFO | Analyzing chunk frames = [4659200,4659712) | 20260505 16:31:01.470 | INFO | MemInfo: { | 20260505 16:31:01.470 | INFO | "gpu" : " 17 GB", | 20260505 16:31:01.470 | INFO | "huge" : "", | 20260505 16:31:01.470 | INFO | "huge_shared" : " 27 GB", | 20260505 16:31:01.470 | INFO | "malloc" : " 14 GB", | 20260505 16:31:01.470 | INFO | "pinned" : " 29 GB" | 20260505 16:31:01.470 | INFO | } | 20260505 16:31:01.471 | INFO | Perf report for the last 5.14299 seconds: | 20260505 16:31:01.471 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:31:01.471 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:01.471 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:01.471 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:01.471 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260505 16:31:01.471 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:01.471 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:01.471 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:31:01.471 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:01.471 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:01.471 | INFO | Uploaded 36GiB to GPU in 2.13791s (16.8389GiB/s) | 20260505 16:31:01.471 | INFO | Downloaded 34GiB to GPU in 2.05742s (16.5255GiB/s) | 20260505 16:31:06.610 | INFO | Analyzing chunk frames = [4659712,4660224) | 20260505 16:31:06.610 | INFO | MemInfo: { | 20260505 16:31:06.610 | INFO | "gpu" : " 17 GB", | 20260505 16:31:06.610 | INFO | "huge" : "", | 20260505 16:31:06.610 | INFO | "huge_shared" : " 27 GB", | 20260505 16:31:06.610 | INFO | "malloc" : " 14 GB", | 20260505 16:31:06.610 | INFO | "pinned" : " 29 GB" | 20260505 16:31:06.610 | INFO | } | 20260505 16:31:06.610 | INFO | Perf report for the last 5.13937 seconds: | 20260505 16:31:06.610 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:31:06.610 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:06.610 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:06.610 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:06.610 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260505 16:31:06.610 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:06.610 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:06.610 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:31:06.610 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:06.610 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:06.610 | INFO | Uploaded 36GiB to GPU in 2.13609s (16.8532GiB/s) | 20260505 16:31:06.610 | INFO | Downloaded 34GiB to GPU in 2.03599s (16.6995GiB/s) | 20260505 16:31:11.745 | INFO | Analyzing chunk frames = [4660224,4660736) | 20260505 16:31:11.745 | INFO | MemInfo: { | 20260505 16:31:11.745 | INFO | "gpu" : " 17 GB", | 20260505 16:31:11.745 | INFO | "huge" : "", | 20260505 16:31:11.745 | INFO | "huge_shared" : " 27 GB", | 20260505 16:31:11.745 | INFO | "malloc" : " 14 GB", | 20260505 16:31:11.745 | INFO | "pinned" : " 29 GB" | 20260505 16:31:11.745 | INFO | } | 20260505 16:31:11.745 | INFO | Perf report for the last 5.13545 seconds: | 20260505 16:31:11.745 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:31:11.745 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:11.745 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:11.746 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:11.746 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 16:31:11.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:11.746 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:11.746 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 16:31:11.746 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:11.746 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:11.746 | INFO | Uploaded 36GiB to GPU in 2.15911s (16.6735GiB/s) | 20260505 16:31:11.746 | INFO | Downloaded 34GiB to GPU in 2.17244s (15.6506GiB/s) | 20260505 16:31:16.881 | INFO | Analyzing chunk frames = [4660736,4661248) | 20260505 16:31:16.881 | INFO | MemInfo: { | 20260505 16:31:16.881 | INFO | "gpu" : " 17 GB", | 20260505 16:31:16.881 | INFO | "huge" : "", | 20260505 16:31:16.881 | INFO | "huge_shared" : " 27 GB", | 20260505 16:31:16.882 | INFO | "malloc" : " 14 GB", | 20260505 16:31:16.882 | INFO | "pinned" : " 29 GB" | 20260505 16:31:16.882 | INFO | } | 20260505 16:31:16.882 | INFO | Perf report for the last 5.13619 seconds: | 20260505 16:31:16.882 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:31:16.882 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:16.882 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:16.882 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:16.882 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 16:31:16.882 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:16.882 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:16.882 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 16:31:16.882 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:16.882 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:16.882 | INFO | Uploaded 36GiB to GPU in 2.23689s (16.0938GiB/s) | 20260505 16:31:16.882 | INFO | Downloaded 34GiB to GPU in 2.10317s (16.1661GiB/s) | 20260505 16:31:16.882 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 16:31:16.882 | INFO | Total mem usage: 15.3 GiB | 20260505 16:31:16.882 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 16:31:16.882 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.89%) | 20260505 16:31:16.882 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 16:31:16.882 | INFO | High Water Breakdown: | 20260505 16:31:16.882 | INFO | 61% : 8.67 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 16:31:16.882 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 16:31:16.882 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 16:31:16.882 | INFO | ------------------------------------------ | 20260505 16:31:16.882 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:31:16.882 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 16:31:16.882 | INFO | Total mem usage: 26.5 GiB | 20260505 16:31:16.882 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 16:31:16.882 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 16:31:16.882 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 16:31:16.882 | INFO | High Water Breakdown: | 20260505 16:31:16.882 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:31:16.882 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:31:16.882 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:31:16.882 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:31:16.882 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:31:16.882 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:31:16.882 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 16:31:16.882 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 16:31:16.882 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:31:16.882 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 16:31:16.882 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 16:31:16.883 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 16:31:16.883 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 16:31:16.883 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 16:31:16.883 | INFO | ------------------------------------------ | 20260505 16:31:16.883 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:31:16.883 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 16:31:16.883 | INFO | Total mem usage: 25.6 GiB | 20260505 16:31:16.883 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 16:31:16.883 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 16:31:16.883 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 16:31:16.883 | INFO | High Water Breakdown: | 20260505 16:31:16.883 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 16:31:16.883 | INFO | ------------------------------------------ | 20260505 16:31:16.883 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:31:16.883 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 16:31:16.883 | INFO | Total mem usage: 15.7 GiB | 20260505 16:31:16.883 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 16:31:16.883 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 16:31:16.883 | INFO | Unused Cache: 871 MiB (Waste: 5.43%) | 20260505 16:31:16.883 | INFO | High Water Breakdown: | 20260505 16:31:16.883 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 16:31:16.883 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 16:31:16.883 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 16:31:16.883 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 16:31:16.883 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 16:31:16.883 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 16:31:16.883 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 16:31:16.883 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 16:31:16.883 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 16:31:16.883 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:31:16.883 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 16:31:16.883 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:31:16.883 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 16:31:16.883 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 16:31:16.883 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 16:31:16.883 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 16:31:16.883 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:31:16.883 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 16:31:16.883 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:31:16.883 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:31:16.883 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:31:16.883 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 16:31:16.883 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 16:31:16.883 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:31:16.883 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 16:31:16.883 | INFO | ------------------------------------------ | 20260505 16:31:16.883 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:31:16.883 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 16:31:16.883 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:31:16.883 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:31:16.883 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.57e+01, 1.36e+01 / 2.52e+00 | 20260505 16:31:16.883 | INFO | PerfReport: 0.29%, AggregateStats -- 9.26e-02 / 1.02e-01, 1.07e-02 / 5.36e-03 | 20260505 16:31:16.883 | INFO | PerfReport: 11.76%, Baseline -- 3.73e+00 / 4.48e+00, 5.71e-01 / 1.30e-01 | 20260505 16:31:16.883 | INFO | PerfReport: 3.29%, Histogram -- 1.04e+00 / 1.52e+00, 1.95e-01 / 8.99e-02 | 20260505 16:31:16.883 | INFO | PerfReport: 15.99%, DME -- 4.05e+01 / 4.53e+01, 4.45e+00 / 2.68e+00 | 20260505 16:31:16.883 | INFO | PerfReport: 56.74%, FrameLabeling -- 1.80e+01 / 1.96e+01, 1.48e+00 / 1.41e+00 | 20260505 16:31:16.883 | INFO | PerfReport: 7.34%, PulseAccumulating -- 2.33e+00 / 2.88e+00, 4.45e-01 / 1.93e-01 | 20260505 16:31:16.883 | INFO | PerfReport: 3.40%, Metrics -- 1.08e+00 / 1.41e+00, 2.96e-01 / 1.50e-01 | 20260505 16:31:16.883 | INFO | PerfReport: 1.19%, PulseSerializing -- 3.77e-01 / 4.36e-01, 6.71e-02 / 2.72e-02 | 20260505 16:31:16.883 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 16:31:16.883 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:31:16.883 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:31:16.883 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.58e+01 / 0.00e+00 | 20260505 16:31:16.883 | INFO | PerfReport: 22.50%, Upload -- 2.26e+01 / 0.00e+00, 7.86e+00 / 0.00e+00 | 20260505 16:31:16.883 | INFO | PerfReport: 21.87%, Download -- 2.19e+01 / 0.00e+00, 7.90e+00 / 0.00e+00 | 20260505 16:31:16.883 | INFO | PerfReport: 31.63%, Compute -- 3.17e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 16:31:16.883 | INFO | PerfReport: 2.30%, LockUpload -- 2.30e+00 / 0.00e+00, 7.89e+00 / 0.00e+00 | 20260505 16:31:16.883 | INFO | PerfReport: 1.41%, LockDownload -- 1.41e+00 / 0.00e+00, 3.00e+00 / 0.00e+00 | 20260505 16:31:16.883 | INFO | PerfReport: 20.27%, LockCompute -- 2.03e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 16:31:16.883 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.56e-02 / 0.00e+00 | 20260505 16:31:17.012 | INFO | Starting a round of Baz IO | 20260505 16:31:20.215 | INFO | Populating RTMetrics report | 20260505 16:31:20.215 | INFO | RTMetrics report populated | 20260505 16:31:20.215 | INFO | Opening tmp RTMetrics json file | 20260505 16:31:20.224 | INFO | Serializing RTMetrics to json | 20260505 16:31:20.227 | INFO | Serialization done, swapping tmp file now... | 20260505 16:31:20.230 | INFO | RTMetrics done | 20260505 16:31:20.311 | INFO | Finished a round of Baz IO: Wrote 2.17 GiB in 3.298 seconds | 20260505 16:31:20.311 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.23% / 0.00% / 77.77% | 20260505 16:31:20.311 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.360795% / 0% / 6.75519e-05% | 20260505 16:31:20.311 | INFO | Baz IO overall: | 20260505 16:31:20.311 | INFO | BytesWritten : 2765100942863 | 20260505 16:31:20.311 | INFO | NumEvents : 1312365350865 | 20260505 16:31:22.020 | INFO | Analyzing chunk frames = [4661248,4661760) | 20260505 16:31:22.020 | INFO | MemInfo: { | 20260505 16:31:22.020 | INFO | "gpu" : " 17 GB", | 20260505 16:31:22.020 | INFO | "huge" : "", | 20260505 16:31:22.020 | INFO | "huge_shared" : " 27 GB", | 20260505 16:31:22.020 | INFO | "malloc" : " 14 GB", | 20260505 16:31:22.020 | INFO | "pinned" : " 29 GB" | 20260505 16:31:22.020 | INFO | } | 20260505 16:31:22.020 | INFO | Perf report for the last 5.13794 seconds: | 20260505 16:31:22.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:31:22.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:22.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:22.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:22.020 | INFO | ANALYSIS : 63.8%, 3.0, 99.3ms, 0 | 20260505 16:31:22.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:22.020 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260505 16:31:22.020 | INFO | PRE_HQ : 60.8%, 1.8, 56.6ms, 0 | 20260505 16:31:22.020 | INFO | BAZWRITER : 63.7%, 3.9, 127.6ms, 0 | 20260505 16:31:22.020 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 16:31:22.020 | INFO | Uploaded 36GiB to GPU in 2.18779s (16.4549GiB/s) | 20260505 16:31:22.020 | INFO | Downloaded 38GiB to GPU in 2.26001s (16.8141GiB/s) | 20260505 16:31:27.170 | INFO | Analyzing chunk frames = [4661760,4662272) | 20260505 16:31:27.170 | INFO | MemInfo: { | 20260505 16:31:27.170 | INFO | "gpu" : " 17 GB", | 20260505 16:31:27.170 | INFO | "huge" : "", | 20260505 16:31:27.170 | INFO | "huge_shared" : " 27 GB", | 20260505 16:31:27.170 | INFO | "malloc" : " 14 GB", | 20260505 16:31:27.170 | INFO | "pinned" : " 29 GB" | 20260505 16:31:27.170 | INFO | } | 20260505 16:31:27.170 | INFO | Perf report for the last 5.1501 seconds: | 20260505 16:31:27.170 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:31:27.170 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:27.170 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:27.170 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:27.170 | INFO | ANALYSIS : 62.9%, 3.0, 98.4ms, 0 | 20260505 16:31:27.170 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:27.170 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:27.170 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:31:27.170 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:27.170 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:27.170 | INFO | Uploaded 36GiB to GPU in 2.14097s (16.8148GiB/s) | 20260505 16:31:27.170 | INFO | Downloaded 34GiB to GPU in 2.03928s (16.6725GiB/s) | 20260505 16:31:32.312 | INFO | Analyzing chunk frames = [4662272,4662784) | 20260505 16:31:32.312 | INFO | MemInfo: { | 20260505 16:31:32.312 | INFO | "gpu" : " 17 GB", | 20260505 16:31:32.312 | INFO | "huge" : "", | 20260505 16:31:32.312 | INFO | "huge_shared" : " 27 GB", | 20260505 16:31:32.312 | INFO | "malloc" : " 14 GB", | 20260505 16:31:32.312 | INFO | "pinned" : " 29 GB" | 20260505 16:31:32.312 | INFO | } | 20260505 16:31:32.312 | INFO | Perf report for the last 5.14253 seconds: | 20260505 16:31:32.312 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:31:32.312 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:32.312 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:32.312 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:32.312 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260505 16:31:32.312 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:32.312 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:32.312 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 16:31:32.312 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:32.312 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:32.312 | INFO | Uploaded 36GiB to GPU in 2.20604s (16.3189GiB/s) | 20260505 16:31:32.312 | INFO | Downloaded 34GiB to GPU in 2.04548s (16.622GiB/s) | 20260505 16:31:37.438 | INFO | Analyzing chunk frames = [4662784,4663296) | 20260505 16:31:37.438 | INFO | MemInfo: { | 20260505 16:31:37.438 | INFO | "gpu" : " 17 GB", | 20260505 16:31:37.438 | INFO | "huge" : "", | 20260505 16:31:37.438 | INFO | "huge_shared" : " 27 GB", | 20260505 16:31:37.438 | INFO | "malloc" : " 14 GB", | 20260505 16:31:37.438 | INFO | "pinned" : " 29 GB" | 20260505 16:31:37.438 | INFO | } | 20260505 16:31:37.439 | INFO | Perf report for the last 5.12629 seconds: | 20260505 16:31:37.439 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:31:37.439 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:37.439 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:37.439 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:37.439 | INFO | ANALYSIS : 63.1%, 3.0, 97.7ms, 0 | 20260505 16:31:37.439 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:37.439 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:37.439 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 16:31:37.439 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:37.439 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:37.439 | INFO | Uploaded 36GiB to GPU in 2.22848s (16.1545GiB/s) | 20260505 16:31:37.439 | INFO | Downloaded 34GiB to GPU in 2.11978s (16.0394GiB/s) | 20260505 16:31:42.575 | INFO | Analyzing chunk frames = [4663296,4663808) | 20260505 16:31:42.575 | INFO | MemInfo: { | 20260505 16:31:42.575 | INFO | "gpu" : " 17 GB", | 20260505 16:31:42.575 | INFO | "huge" : "", | 20260505 16:31:42.575 | INFO | "huge_shared" : " 27 GB", | 20260505 16:31:42.575 | INFO | "malloc" : " 14 GB", | 20260505 16:31:42.575 | INFO | "pinned" : " 29 GB" | 20260505 16:31:42.575 | INFO | } | 20260505 16:31:42.575 | INFO | Perf report for the last 5.13675 seconds: | 20260505 16:31:42.575 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:31:42.575 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:42.575 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:42.575 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:42.575 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 16:31:42.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:42.575 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:42.575 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:31:42.575 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:42.575 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:42.576 | INFO | Uploaded 36GiB to GPU in 2.17181s (16.576GiB/s) | 20260505 16:31:42.576 | INFO | Downloaded 34GiB to GPU in 2.13157s (15.9507GiB/s) | 20260505 16:31:47.724 | INFO | Analyzing chunk frames = [4663808,4664320) | 20260505 16:31:47.725 | INFO | MemInfo: { | 20260505 16:31:47.725 | INFO | "gpu" : " 17 GB", | 20260505 16:31:47.725 | INFO | "huge" : "", | 20260505 16:31:47.725 | INFO | "huge_shared" : " 27 GB", | 20260505 16:31:47.725 | INFO | "malloc" : " 14 GB", | 20260505 16:31:47.725 | INFO | "pinned" : " 29 GB" | 20260505 16:31:47.725 | INFO | } | 20260505 16:31:47.725 | INFO | Perf report for the last 5.14943 seconds: | 20260505 16:31:47.725 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:31:47.725 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:47.725 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:47.725 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:47.725 | INFO | ANALYSIS : 62.9%, 3.0, 98.4ms, 0 | 20260505 16:31:47.725 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:47.725 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:47.725 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 16:31:47.725 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:47.725 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:47.725 | INFO | Uploaded 36GiB to GPU in 2.20748s (16.3082GiB/s) | 20260505 16:31:47.725 | INFO | Downloaded 34GiB to GPU in 2.11573s (16.0701GiB/s) | 20260505 16:31:52.868 | INFO | Analyzing chunk frames = [4664320,4664832) | 20260505 16:31:52.868 | INFO | MemInfo: { | 20260505 16:31:52.868 | INFO | "gpu" : " 17 GB", | 20260505 16:31:52.868 | INFO | "huge" : "", | 20260505 16:31:52.868 | INFO | "huge_shared" : " 27 GB", | 20260505 16:31:52.868 | INFO | "malloc" : " 14 GB", | 20260505 16:31:52.868 | INFO | "pinned" : " 29 GB" | 20260505 16:31:52.868 | INFO | } | 20260505 16:31:52.868 | INFO | Perf report for the last 5.14316 seconds: | 20260505 16:31:52.868 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:31:52.868 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:52.868 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:52.868 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:52.868 | INFO | ANALYSIS : 63.2%, 3.0, 98.8ms, 0 | 20260505 16:31:52.868 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:52.868 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:52.868 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 16:31:52.868 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:52.868 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:52.868 | INFO | Uploaded 36GiB to GPU in 2.1316s (16.8887GiB/s) | 20260505 16:31:52.868 | INFO | Downloaded 34GiB to GPU in 2.17495s (15.6325GiB/s) | 20260505 16:31:58.007 | INFO | Analyzing chunk frames = [4664832,4665344) | 20260505 16:31:58.007 | INFO | MemInfo: { | 20260505 16:31:58.007 | INFO | "gpu" : " 17 GB", | 20260505 16:31:58.007 | INFO | "huge" : "", | 20260505 16:31:58.007 | INFO | "huge_shared" : " 27 GB", | 20260505 16:31:58.007 | INFO | "malloc" : " 14 GB", | 20260505 16:31:58.007 | INFO | "pinned" : " 29 GB" | 20260505 16:31:58.007 | INFO | } | 20260505 16:31:58.007 | INFO | Perf report for the last 5.13901 seconds: | 20260505 16:31:58.007 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:31:58.007 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:58.007 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:58.007 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:58.007 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 16:31:58.007 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:58.007 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:58.007 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 16:31:58.007 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:58.007 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:31:58.007 | INFO | Uploaded 36GiB to GPU in 2.20084s (16.3574GiB/s) | 20260505 16:31:58.007 | INFO | Downloaded 34GiB to GPU in 2.01897s (16.8403GiB/s) | 20260505 16:31:58.144 | INFO | Starting a round of Baz IO | 20260505 16:32:01.332 | INFO | Populating RTMetrics report | 20260505 16:32:01.332 | INFO | RTMetrics report populated | 20260505 16:32:01.332 | INFO | Opening tmp RTMetrics json file | 20260505 16:32:01.342 | INFO | Serializing RTMetrics to json | 20260505 16:32:01.344 | INFO | Serialization done, swapping tmp file now... | 20260505 16:32:01.349 | INFO | RTMetrics done | 20260505 16:32:01.430 | INFO | Finished a round of Baz IO: Wrote 2.168 GiB in 3.286 seconds | 20260505 16:32:01.430 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.23% / 0.00% / 77.77% | 20260505 16:32:01.430 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.358101% / 0% / 9.53674e-05% | 20260505 16:32:01.431 | INFO | Baz IO overall: | 20260505 16:32:01.431 | INFO | BytesWritten : 2767428301642 | 20260505 16:32:01.431 | INFO | NumEvents : 1313456339473 | 20260505 16:32:03.136 | INFO | Analyzing chunk frames = [4665344,4665856) | 20260505 16:32:03.136 | INFO | MemInfo: { | 20260505 16:32:03.136 | INFO | "gpu" : " 17 GB", | 20260505 16:32:03.136 | INFO | "huge" : "", | 20260505 16:32:03.136 | INFO | "huge_shared" : " 27 GB", | 20260505 16:32:03.136 | INFO | "malloc" : " 14 GB", | 20260505 16:32:03.136 | INFO | "pinned" : " 29 GB" | 20260505 16:32:03.136 | INFO | } | 20260505 16:32:03.136 | INFO | Perf report for the last 5.12931 seconds: | 20260505 16:32:03.136 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:32:03.136 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:03.136 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:03.136 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:03.136 | INFO | ANALYSIS : 63.7%, 3.0, 98.8ms, 0 | 20260505 16:32:03.136 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:03.136 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260505 16:32:03.136 | INFO | PRE_HQ : 60.6%, 1.8, 56.4ms, 0 | 20260505 16:32:03.136 | INFO | BAZWRITER : 63.4%, 3.9, 128.1ms, 0 | 20260505 16:32:03.136 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 16:32:03.137 | INFO | Uploaded 36GiB to GPU in 2.15516s (16.7041GiB/s) | 20260505 16:32:03.137 | INFO | Downloaded 38GiB to GPU in 2.25281s (16.8678GiB/s) | 20260505 16:32:08.276 | INFO | Analyzing chunk frames = [4665856,4666368) | 20260505 16:32:08.276 | INFO | MemInfo: { | 20260505 16:32:08.276 | INFO | "gpu" : " 17 GB", | 20260505 16:32:08.276 | INFO | "huge" : "", | 20260505 16:32:08.276 | INFO | "huge_shared" : " 27 GB", | 20260505 16:32:08.276 | INFO | "malloc" : " 14 GB", | 20260505 16:32:08.276 | INFO | "pinned" : " 29 GB" | 20260505 16:32:08.276 | INFO | } | 20260505 16:32:08.276 | INFO | Perf report for the last 5.13954 seconds: | 20260505 16:32:08.276 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:32:08.276 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:08.276 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:08.276 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:08.276 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 16:32:08.276 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:08.276 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:08.276 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:32:08.276 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:08.276 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:08.276 | INFO | Uploaded 36GiB to GPU in 2.19497s (16.4012GiB/s) | 20260505 16:32:08.276 | INFO | Downloaded 34GiB to GPU in 2.06096s (16.4971GiB/s) | 20260505 16:32:13.425 | INFO | Analyzing chunk frames = [4666368,4666880) | 20260505 16:32:13.425 | INFO | MemInfo: { | 20260505 16:32:13.425 | INFO | "gpu" : " 17 GB", | 20260505 16:32:13.425 | INFO | "huge" : "", | 20260505 16:32:13.425 | INFO | "huge_shared" : " 27 GB", | 20260505 16:32:13.425 | INFO | "malloc" : " 14 GB", | 20260505 16:32:13.425 | INFO | "pinned" : " 29 GB" | 20260505 16:32:13.425 | INFO | } | 20260505 16:32:13.426 | INFO | Perf report for the last 5.1497 seconds: | 20260505 16:32:13.426 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:32:13.426 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:13.426 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:13.426 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:13.426 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 16:32:13.426 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:13.426 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:13.426 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:32:13.426 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:13.426 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:13.426 | INFO | Uploaded 36GiB to GPU in 2.19556s (16.3968GiB/s) | 20260505 16:32:13.426 | INFO | Downloaded 34GiB to GPU in 2.02815s (16.764GiB/s) | 20260505 16:32:18.561 | INFO | Analyzing chunk frames = [4666880,4667392) | 20260505 16:32:18.561 | INFO | MemInfo: { | 20260505 16:32:18.561 | INFO | "gpu" : " 17 GB", | 20260505 16:32:18.561 | INFO | "huge" : "", | 20260505 16:32:18.561 | INFO | "huge_shared" : " 27 GB", | 20260505 16:32:18.561 | INFO | "malloc" : " 14 GB", | 20260505 16:32:18.561 | INFO | "pinned" : " 29 GB" | 20260505 16:32:18.561 | INFO | } | 20260505 16:32:18.561 | INFO | Perf report for the last 5.13584 seconds: | 20260505 16:32:18.561 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:32:18.561 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:18.561 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:18.561 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:18.561 | INFO | ANALYSIS : 62.8%, 3.0, 97.4ms, 0 | 20260505 16:32:18.561 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:18.561 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:18.561 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 16:32:18.562 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:18.562 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:18.562 | INFO | Uploaded 36GiB to GPU in 2.22013s (16.2153GiB/s) | 20260505 16:32:18.562 | INFO | Downloaded 34GiB to GPU in 2.13379s (15.9341GiB/s) | 20260505 16:32:23.691 | INFO | Analyzing chunk frames = [4667392,4667904) | 20260505 16:32:23.691 | INFO | MemInfo: { | 20260505 16:32:23.691 | INFO | "gpu" : " 17 GB", | 20260505 16:32:23.691 | INFO | "huge" : "", | 20260505 16:32:23.691 | INFO | "huge_shared" : " 27 GB", | 20260505 16:32:23.691 | INFO | "malloc" : " 14 GB", | 20260505 16:32:23.691 | INFO | "pinned" : " 29 GB" | 20260505 16:32:23.691 | INFO | } | 20260505 16:32:23.691 | INFO | Perf report for the last 5.1297 seconds: | 20260505 16:32:23.691 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:32:23.691 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:23.691 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:23.691 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:23.691 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260505 16:32:23.691 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:23.691 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:23.691 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:32:23.691 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:23.691 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:23.691 | INFO | Uploaded 36GiB to GPU in 2.15018s (16.7428GiB/s) | 20260505 16:32:23.691 | INFO | Downloaded 34GiB to GPU in 2.05952s (16.5087GiB/s) | 20260505 16:32:28.832 | INFO | Analyzing chunk frames = [4667904,4668416) | 20260505 16:32:28.832 | INFO | MemInfo: { | 20260505 16:32:28.832 | INFO | "gpu" : " 17 GB", | 20260505 16:32:28.832 | INFO | "huge" : "", | 20260505 16:32:28.832 | INFO | "huge_shared" : " 27 GB", | 20260505 16:32:28.832 | INFO | "malloc" : " 14 GB", | 20260505 16:32:28.832 | INFO | "pinned" : " 29 GB" | 20260505 16:32:28.832 | INFO | } | 20260505 16:32:28.832 | INFO | Perf report for the last 5.14103 seconds: | 20260505 16:32:28.832 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:32:28.832 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:28.832 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:28.832 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:28.832 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 16:32:28.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:28.832 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:28.832 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:32:28.832 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:28.832 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:28.832 | INFO | Uploaded 36GiB to GPU in 2.16794s (16.6056GiB/s) | 20260505 16:32:28.832 | INFO | Downloaded 34GiB to GPU in 2.0422s (16.6487GiB/s) | 20260505 16:32:33.981 | INFO | Analyzing chunk frames = [4668416,4668928) | 20260505 16:32:33.981 | INFO | MemInfo: { | 20260505 16:32:33.981 | INFO | "gpu" : " 17 GB", | 20260505 16:32:33.981 | INFO | "huge" : "", | 20260505 16:32:33.981 | INFO | "huge_shared" : " 27 GB", | 20260505 16:32:33.981 | INFO | "malloc" : " 14 GB", | 20260505 16:32:33.981 | INFO | "pinned" : " 29 GB" | 20260505 16:32:33.981 | INFO | } | 20260505 16:32:33.981 | INFO | Perf report for the last 5.14877 seconds: | 20260505 16:32:33.981 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:32:33.981 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:33.981 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:33.981 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:33.981 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 16:32:33.981 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:33.981 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:33.981 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 16:32:33.981 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:33.981 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:33.981 | INFO | Uploaded 36GiB to GPU in 2.18263s (16.4939GiB/s) | 20260505 16:32:33.981 | INFO | Downloaded 34GiB to GPU in 2.14753s (15.8322GiB/s) | 20260505 16:32:39.120 | INFO | Analyzing chunk frames = [4668928,4669440) | 20260505 16:32:39.120 | INFO | MemInfo: { | 20260505 16:32:39.120 | INFO | "gpu" : " 17 GB", | 20260505 16:32:39.120 | INFO | "huge" : "", | 20260505 16:32:39.120 | INFO | "huge_shared" : " 27 GB", | 20260505 16:32:39.120 | INFO | "malloc" : " 14 GB", | 20260505 16:32:39.120 | INFO | "pinned" : " 29 GB" | 20260505 16:32:39.120 | INFO | } | 20260505 16:32:39.120 | INFO | Perf report for the last 5.13897 seconds: | 20260505 16:32:39.120 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:32:39.120 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:39.120 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:39.120 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:39.120 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 16:32:39.120 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:39.120 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:39.120 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 16:32:39.120 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:39.120 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:39.120 | INFO | Uploaded 36GiB to GPU in 2.19627s (16.3914GiB/s) | 20260505 16:32:39.120 | INFO | Downloaded 34GiB to GPU in 2.02297s (16.807GiB/s) | 20260505 16:32:39.120 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 16:32:39.121 | INFO | Total mem usage: 15.3 GiB | 20260505 16:32:39.121 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 16:32:39.121 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.91%) | 20260505 16:32:39.121 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 16:32:39.121 | INFO | High Water Breakdown: | 20260505 16:32:39.121 | INFO | 61% : 8.67 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 16:32:39.121 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 16:32:39.121 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 16:32:39.121 | INFO | ------------------------------------------ | 20260505 16:32:39.121 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:32:39.121 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 16:32:39.121 | INFO | Total mem usage: 26.5 GiB | 20260505 16:32:39.121 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 16:32:39.121 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 16:32:39.121 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 16:32:39.121 | INFO | High Water Breakdown: | 20260505 16:32:39.121 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:32:39.121 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:32:39.121 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:32:39.121 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:32:39.121 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:32:39.121 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:32:39.121 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 16:32:39.121 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 16:32:39.121 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:32:39.121 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 16:32:39.121 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 16:32:39.121 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 16:32:39.121 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 16:32:39.121 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 16:32:39.121 | INFO | ------------------------------------------ | 20260505 16:32:39.121 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:32:39.121 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 16:32:39.121 | INFO | Total mem usage: 25.6 GiB | 20260505 16:32:39.121 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 16:32:39.121 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 16:32:39.121 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 16:32:39.121 | INFO | High Water Breakdown: | 20260505 16:32:39.121 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 16:32:39.121 | INFO | ------------------------------------------ | 20260505 16:32:39.121 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:32:39.121 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 16:32:39.121 | INFO | Total mem usage: 15.7 GiB | 20260505 16:32:39.121 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 16:32:39.121 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 16:32:39.121 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 16:32:39.121 | INFO | High Water Breakdown: | 20260505 16:32:39.121 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 16:32:39.121 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 16:32:39.121 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 16:32:39.121 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 16:32:39.121 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 16:32:39.121 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 16:32:39.121 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 16:32:39.121 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 16:32:39.121 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 16:32:39.121 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:32:39.121 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 16:32:39.121 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:32:39.121 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 16:32:39.121 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 16:32:39.121 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 16:32:39.121 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 16:32:39.121 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:32:39.121 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 16:32:39.121 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:32:39.121 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:32:39.121 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:32:39.121 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 16:32:39.121 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 16:32:39.121 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:32:39.121 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 16:32:39.121 | INFO | ------------------------------------------ | 20260505 16:32:39.121 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:32:39.121 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 16:32:39.121 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:32:39.121 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:32:39.122 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.49e+01, 1.35e+01 / 1.24e+00 | 20260505 16:32:39.122 | INFO | PerfReport: 0.29%, AggregateStats -- 9.26e-02 / 1.02e-01, 1.04e-02 / 4.35e-03 | 20260505 16:32:39.122 | INFO | PerfReport: 11.76%, Baseline -- 3.71e+00 / 4.48e+00, 5.55e-01 / 1.55e-01 | 20260505 16:32:39.122 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.51e+00, 1.88e-01 / 1.05e-01 | 20260505 16:32:39.122 | INFO | PerfReport: 15.94%, DME -- 4.03e+01 / 4.48e+01, 4.20e+00 / 1.38e+00 | 20260505 16:32:39.122 | INFO | PerfReport: 56.71%, FrameLabeling -- 1.79e+01 / 1.93e+01, 1.42e+00 / 1.22e+00 | 20260505 16:32:39.122 | INFO | PerfReport: 7.42%, PulseAccumulating -- 2.34e+00 / 2.92e+00, 4.53e-01 / 1.77e-01 | 20260505 16:32:39.122 | INFO | PerfReport: 3.42%, Metrics -- 1.08e+00 / 1.41e+00, 2.95e-01 / 1.36e-01 | 20260505 16:32:39.122 | INFO | PerfReport: 1.19%, PulseSerializing -- 3.77e-01 / 4.41e-01, 6.78e-02 / 4.07e-02 | 20260505 16:32:39.122 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 16:32:39.122 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:32:39.122 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:32:39.122 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260505 16:32:39.122 | INFO | PerfReport: 22.69%, Upload -- 2.27e+01 / 0.00e+00, 8.00e+00 / 0.00e+00 | 20260505 16:32:39.122 | INFO | PerfReport: 21.86%, Download -- 2.19e+01 / 0.00e+00, 7.82e+00 / 0.00e+00 | 20260505 16:32:39.122 | INFO | PerfReport: 31.52%, Compute -- 3.16e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 16:32:39.122 | INFO | PerfReport: 2.47%, LockUpload -- 2.48e+00 / 0.00e+00, 8.14e+00 / 0.00e+00 | 20260505 16:32:39.122 | INFO | PerfReport: 1.36%, LockDownload -- 1.36e+00 / 0.00e+00, 2.84e+00 / 0.00e+00 | 20260505 16:32:39.122 | INFO | PerfReport: 20.09%, LockCompute -- 2.01e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 16:32:39.122 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.47e-02 / 0.00e+00 | 20260505 16:32:39.257 | INFO | Starting a round of Baz IO | 20260505 16:32:42.454 | INFO | Populating RTMetrics report | 20260505 16:32:42.454 | INFO | RTMetrics report populated | 20260505 16:32:42.454 | INFO | Opening tmp RTMetrics json file | 20260505 16:32:42.466 | INFO | Serializing RTMetrics to json | 20260505 16:32:42.468 | INFO | Serialization done, swapping tmp file now... | 20260505 16:32:42.469 | INFO | RTMetrics done | 20260505 16:32:42.561 | INFO | Finished a round of Baz IO: Wrote 2.168 GiB in 3.304 seconds | 20260505 16:32:42.561 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.23% / 0.00% / 77.77% | 20260505 16:32:42.561 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.357862% / 0% / 9.93411e-05% | 20260505 16:32:42.561 | INFO | Baz IO overall: | 20260505 16:32:42.561 | INFO | BytesWritten : 2769756595852 | 20260505 16:32:42.561 | INFO | NumEvents : 1314548016803 | 20260505 16:32:44.254 | INFO | Analyzing chunk frames = [4669440,4669952) | 20260505 16:32:44.254 | INFO | MemInfo: { | 20260505 16:32:44.254 | INFO | "gpu" : " 17 GB", | 20260505 16:32:44.254 | INFO | "huge" : "", | 20260505 16:32:44.254 | INFO | "huge_shared" : " 27 GB", | 20260505 16:32:44.254 | INFO | "malloc" : " 14 GB", | 20260505 16:32:44.254 | INFO | "pinned" : " 29 GB" | 20260505 16:32:44.254 | INFO | } | 20260505 16:32:44.254 | INFO | Perf report for the last 5.13451 seconds: | 20260505 16:32:44.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:32:44.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:44.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:44.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:44.255 | INFO | ANALYSIS : 63.8%, 3.0, 99.2ms, 0 | 20260505 16:32:44.255 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:44.255 | INFO | RT_METRICS : 44.9%, 1.0, 23.5ms, 0 | 20260505 16:32:44.255 | INFO | PRE_HQ : 60.7%, 1.8, 56.6ms, 0 | 20260505 16:32:44.255 | INFO | BAZWRITER : 63.8%, 4.0, 129.5ms, 0 | 20260505 16:32:44.255 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 16:32:44.255 | INFO | Uploaded 36GiB to GPU in 2.18199s (16.4987GiB/s) | 20260505 16:32:44.255 | INFO | Downloaded 38GiB to GPU in 2.26127s (16.8047GiB/s) | 20260505 16:32:49.388 | INFO | Analyzing chunk frames = [4669952,4670464) | 20260505 16:32:49.388 | INFO | MemInfo: { | 20260505 16:32:49.388 | INFO | "gpu" : " 17 GB", | 20260505 16:32:49.388 | INFO | "huge" : "", | 20260505 16:32:49.388 | INFO | "huge_shared" : " 27 GB", | 20260505 16:32:49.388 | INFO | "malloc" : " 14 GB", | 20260505 16:32:49.388 | INFO | "pinned" : " 29 GB" | 20260505 16:32:49.388 | INFO | } | 20260505 16:32:49.388 | INFO | Perf report for the last 5.13327 seconds: | 20260505 16:32:49.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:32:49.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:49.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:49.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:49.388 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260505 16:32:49.388 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:49.388 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:49.388 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 16:32:49.388 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:49.388 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:49.388 | INFO | Uploaded 36GiB to GPU in 2.12764s (16.9201GiB/s) | 20260505 16:32:49.388 | INFO | Downloaded 34GiB to GPU in 2.03374s (16.7179GiB/s) | 20260505 16:32:54.529 | INFO | Analyzing chunk frames = [4670464,4670976) | 20260505 16:32:54.529 | INFO | MemInfo: { | 20260505 16:32:54.529 | INFO | "gpu" : " 17 GB", | 20260505 16:32:54.529 | INFO | "huge" : "", | 20260505 16:32:54.529 | INFO | "huge_shared" : " 27 GB", | 20260505 16:32:54.529 | INFO | "malloc" : " 14 GB", | 20260505 16:32:54.530 | INFO | "pinned" : " 29 GB" | 20260505 16:32:54.530 | INFO | } | 20260505 16:32:54.530 | INFO | Perf report for the last 5.14197 seconds: | 20260505 16:32:54.530 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:32:54.530 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:54.530 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:54.530 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:54.530 | INFO | ANALYSIS : 63.4%, 3.0, 98.8ms, 0 | 20260505 16:32:54.530 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:54.530 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:54.530 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 16:32:54.530 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:54.530 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:54.530 | INFO | Uploaded 36GiB to GPU in 2.24048s (16.068GiB/s) | 20260505 16:32:54.530 | INFO | Downloaded 34GiB to GPU in 2.11967s (16.0402GiB/s) | 20260505 16:32:59.673 | INFO | Analyzing chunk frames = [4670976,4671488) | 20260505 16:32:59.673 | INFO | MemInfo: { | 20260505 16:32:59.673 | INFO | "gpu" : " 17 GB", | 20260505 16:32:59.673 | INFO | "huge" : "", | 20260505 16:32:59.673 | INFO | "huge_shared" : " 27 GB", | 20260505 16:32:59.673 | INFO | "malloc" : " 14 GB", | 20260505 16:32:59.673 | INFO | "pinned" : " 29 GB" | 20260505 16:32:59.674 | INFO | } | 20260505 16:32:59.674 | INFO | Perf report for the last 5.14389 seconds: | 20260505 16:32:59.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:32:59.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:59.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:59.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:59.674 | INFO | ANALYSIS : 62.4%, 3.0, 97.0ms, 0 | 20260505 16:32:59.674 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:59.674 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:59.674 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:32:59.674 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:59.674 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:32:59.674 | INFO | Uploaded 36GiB to GPU in 2.17678s (16.5382GiB/s) | 20260505 16:32:59.674 | INFO | Downloaded 34GiB to GPU in 2.13492s (15.9257GiB/s) | 20260505 16:33:04.821 | INFO | Analyzing chunk frames = [4671488,4672000) | 20260505 16:33:04.821 | INFO | MemInfo: { | 20260505 16:33:04.821 | INFO | "gpu" : " 17 GB", | 20260505 16:33:04.821 | INFO | "huge" : "", | 20260505 16:33:04.821 | INFO | "huge_shared" : " 27 GB", | 20260505 16:33:04.821 | INFO | "malloc" : " 14 GB", | 20260505 16:33:04.822 | INFO | "pinned" : " 29 GB" | 20260505 16:33:04.822 | INFO | } | 20260505 16:33:04.822 | INFO | Perf report for the last 5.14809 seconds: | 20260505 16:33:04.822 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:33:04.822 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:04.822 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:04.822 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:04.822 | INFO | ANALYSIS : 62.1%, 3.0, 97.1ms, 0 | 20260505 16:33:04.822 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:04.822 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:04.822 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:33:04.822 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:04.822 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:04.822 | INFO | Uploaded 36GiB to GPU in 2.14229s (16.8044GiB/s) | 20260505 16:33:04.822 | INFO | Downloaded 34GiB to GPU in 2.0532s (16.5595GiB/s) | 20260505 16:33:09.955 | INFO | Analyzing chunk frames = [4672000,4672512) | 20260505 16:33:09.955 | INFO | MemInfo: { | 20260505 16:33:09.955 | INFO | "gpu" : " 17 GB", | 20260505 16:33:09.955 | INFO | "huge" : "", | 20260505 16:33:09.955 | INFO | "huge_shared" : " 27 GB", | 20260505 16:33:09.955 | INFO | "malloc" : " 14 GB", | 20260505 16:33:09.955 | INFO | "pinned" : " 29 GB" | 20260505 16:33:09.955 | INFO | } | 20260505 16:33:09.955 | INFO | Perf report for the last 5.13314 seconds: | 20260505 16:33:09.955 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:33:09.955 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:09.955 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:09.955 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:09.955 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260505 16:33:09.955 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:09.955 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:09.955 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:33:09.955 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:09.955 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:09.955 | INFO | Uploaded 36GiB to GPU in 2.12928s (16.9071GiB/s) | 20260505 16:33:09.955 | INFO | Downloaded 34GiB to GPU in 2.04655s (16.6133GiB/s) | 20260505 16:33:15.085 | INFO | Analyzing chunk frames = [4672512,4673024) | 20260505 16:33:15.085 | INFO | MemInfo: { | 20260505 16:33:15.085 | INFO | "gpu" : " 17 GB", | 20260505 16:33:15.085 | INFO | "huge" : "", | 20260505 16:33:15.085 | INFO | "huge_shared" : " 27 GB", | 20260505 16:33:15.085 | INFO | "malloc" : " 14 GB", | 20260505 16:33:15.085 | INFO | "pinned" : " 29 GB" | 20260505 16:33:15.085 | INFO | } | 20260505 16:33:15.085 | INFO | Perf report for the last 5.13054 seconds: | 20260505 16:33:15.085 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:33:15.085 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:15.085 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:15.085 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:15.085 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 16:33:15.085 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:15.085 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:15.086 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 16:33:15.086 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:15.086 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:15.086 | INFO | Uploaded 36GiB to GPU in 2.09533s (17.181GiB/s) | 20260505 16:33:15.086 | INFO | Downloaded 34GiB to GPU in 2.16976s (15.6699GiB/s) | 20260505 16:33:20.234 | INFO | Analyzing chunk frames = [4673024,4673536) | 20260505 16:33:20.234 | INFO | MemInfo: { | 20260505 16:33:20.234 | INFO | "gpu" : " 17 GB", | 20260505 16:33:20.234 | INFO | "huge" : "", | 20260505 16:33:20.234 | INFO | "huge_shared" : " 27 GB", | 20260505 16:33:20.234 | INFO | "malloc" : " 14 GB", | 20260505 16:33:20.234 | INFO | "pinned" : " 29 GB" | 20260505 16:33:20.234 | INFO | } | 20260505 16:33:20.234 | INFO | Perf report for the last 5.14818 seconds: | 20260505 16:33:20.234 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:33:20.234 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:20.234 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:20.234 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:20.234 | INFO | ANALYSIS : 63.4%, 3.0, 99.2ms, 0 | 20260505 16:33:20.234 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:20.234 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:20.234 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 16:33:20.234 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:20.234 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:20.234 | INFO | Uploaded 36GiB to GPU in 2.22128s (16.2069GiB/s) | 20260505 16:33:20.234 | INFO | Downloaded 34GiB to GPU in 2.09927s (16.1961GiB/s) | 20260505 16:33:20.372 | INFO | Starting a round of Baz IO | 20260505 16:33:23.560 | INFO | Populating RTMetrics report | 20260505 16:33:23.560 | INFO | RTMetrics report populated | 20260505 16:33:23.560 | INFO | Opening tmp RTMetrics json file | 20260505 16:33:23.572 | INFO | Serializing RTMetrics to json | 20260505 16:33:23.580 | INFO | Serialization done, swapping tmp file now... | 20260505 16:33:23.598 | INFO | RTMetrics done | 20260505 16:33:23.657 | INFO | Finished a round of Baz IO: Wrote 2.166 GiB in 3.285 seconds | 20260505 16:33:23.657 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.22% / 0.00% / 77.77% | 20260505 16:33:23.657 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.356372% / 0% / 9.53674e-05% | 20260505 16:33:23.657 | INFO | Baz IO overall: | 20260505 16:33:23.657 | INFO | BytesWritten : 2772082613934 | 20260505 16:33:23.657 | INFO | NumEvents : 1315638325435 | 20260505 16:33:25.383 | INFO | Analyzing chunk frames = [4673536,4674048) | 20260505 16:33:25.383 | INFO | MemInfo: { | 20260505 16:33:25.383 | INFO | "gpu" : " 17 GB", | 20260505 16:33:25.383 | INFO | "huge" : "", | 20260505 16:33:25.383 | INFO | "huge_shared" : " 27 GB", | 20260505 16:33:25.383 | INFO | "malloc" : " 14 GB", | 20260505 16:33:25.383 | INFO | "pinned" : " 29 GB" | 20260505 16:33:25.383 | INFO | } | 20260505 16:33:25.383 | INFO | Perf report for the last 5.14923 seconds: | 20260505 16:33:25.383 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:33:25.383 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:25.383 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:25.383 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:25.383 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 16:33:25.383 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:25.383 | INFO | RT_METRICS : 45.2%, 1.0, 23.8ms, 0 | 20260505 16:33:25.383 | INFO | PRE_HQ : 60.3%, 1.8, 56.3ms, 0 | 20260505 16:33:25.383 | INFO | BAZWRITER : 63.1%, 3.9, 127.7ms, 0 | 20260505 16:33:25.383 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 16:33:25.383 | INFO | Uploaded 36GiB to GPU in 2.1627s (16.6458GiB/s) | 20260505 16:33:25.383 | INFO | Downloaded 38GiB to GPU in 2.2572s (16.835GiB/s) | 20260505 16:33:30.511 | INFO | Analyzing chunk frames = [4674048,4674560) | 20260505 16:33:30.511 | INFO | MemInfo: { | 20260505 16:33:30.511 | INFO | "gpu" : " 17 GB", | 20260505 16:33:30.511 | INFO | "huge" : "", | 20260505 16:33:30.511 | INFO | "huge_shared" : " 27 GB", | 20260505 16:33:30.512 | INFO | "malloc" : " 14 GB", | 20260505 16:33:30.512 | INFO | "pinned" : " 29 GB" | 20260505 16:33:30.512 | INFO | } | 20260505 16:33:30.512 | INFO | Perf report for the last 5.12878 seconds: | 20260505 16:33:30.512 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:33:30.512 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:30.512 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:30.512 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:30.512 | INFO | ANALYSIS : 63.0%, 3.0, 98.1ms, 0 | 20260505 16:33:30.512 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:30.512 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:30.512 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:33:30.512 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:30.512 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:30.512 | INFO | Uploaded 36GiB to GPU in 2.13837s (16.8353GiB/s) | 20260505 16:33:30.512 | INFO | Downloaded 34GiB to GPU in 2.02827s (16.763GiB/s) | 20260505 16:33:35.642 | INFO | Analyzing chunk frames = [4674560,4675072) | 20260505 16:33:35.642 | INFO | MemInfo: { | 20260505 16:33:35.642 | INFO | "gpu" : " 17 GB", | 20260505 16:33:35.642 | INFO | "huge" : "", | 20260505 16:33:35.642 | INFO | "huge_shared" : " 27 GB", | 20260505 16:33:35.642 | INFO | "malloc" : " 14 GB", | 20260505 16:33:35.643 | INFO | "pinned" : " 29 GB" | 20260505 16:33:35.643 | INFO | } | 20260505 16:33:35.643 | INFO | Perf report for the last 5.13097 seconds: | 20260505 16:33:35.643 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:33:35.643 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:35.643 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:35.643 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:35.643 | INFO | ANALYSIS : 62.9%, 3.0, 97.6ms, 0 | 20260505 16:33:35.643 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:35.643 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:35.643 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 16:33:35.643 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:35.643 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:35.643 | INFO | Uploaded 36GiB to GPU in 2.17449s (16.5556GiB/s) | 20260505 16:33:35.643 | INFO | Downloaded 34GiB to GPU in 2.02932s (16.7544GiB/s) | 20260505 16:33:40.785 | INFO | Analyzing chunk frames = [4675072,4675584) | 20260505 16:33:40.785 | INFO | MemInfo: { | 20260505 16:33:40.785 | INFO | "gpu" : " 17 GB", | 20260505 16:33:40.785 | INFO | "huge" : "", | 20260505 16:33:40.785 | INFO | "huge_shared" : " 27 GB", | 20260505 16:33:40.786 | INFO | "malloc" : " 14 GB", | 20260505 16:33:40.786 | INFO | "pinned" : " 29 GB" | 20260505 16:33:40.786 | INFO | } | 20260505 16:33:40.786 | INFO | Perf report for the last 5.14308 seconds: | 20260505 16:33:40.786 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:33:40.786 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:40.786 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:40.786 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:40.786 | INFO | ANALYSIS : 62.6%, 3.0, 97.3ms, 0 | 20260505 16:33:40.786 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:40.786 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:40.786 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 16:33:40.786 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:40.786 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:40.786 | INFO | Uploaded 36GiB to GPU in 2.19369s (16.4107GiB/s) | 20260505 16:33:40.786 | INFO | Downloaded 34GiB to GPU in 2.13149s (15.9513GiB/s) | 20260505 16:33:45.930 | INFO | Analyzing chunk frames = [4675584,4676096) | 20260505 16:33:45.930 | INFO | MemInfo: { | 20260505 16:33:45.930 | INFO | "gpu" : " 17 GB", | 20260505 16:33:45.930 | INFO | "huge" : "", | 20260505 16:33:45.930 | INFO | "huge_shared" : " 27 GB", | 20260505 16:33:45.930 | INFO | "malloc" : " 14 GB", | 20260505 16:33:45.930 | INFO | "pinned" : " 29 GB" | 20260505 16:33:45.930 | INFO | } | 20260505 16:33:45.930 | INFO | Perf report for the last 5.14417 seconds: | 20260505 16:33:45.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:33:45.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:45.930 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:45.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:45.930 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260505 16:33:45.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:45.930 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:45.930 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:33:45.930 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:45.930 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:45.930 | INFO | Uploaded 36GiB to GPU in 2.14798s (16.7599GiB/s) | 20260505 16:33:45.930 | INFO | Downloaded 34GiB to GPU in 2.05921s (16.5112GiB/s) | 20260505 16:33:51.070 | INFO | Analyzing chunk frames = [4676096,4676608) | 20260505 16:33:51.070 | INFO | MemInfo: { | 20260505 16:33:51.070 | INFO | "gpu" : " 17 GB", | 20260505 16:33:51.070 | INFO | "huge" : "", | 20260505 16:33:51.070 | INFO | "huge_shared" : " 27 GB", | 20260505 16:33:51.070 | INFO | "malloc" : " 14 GB", | 20260505 16:33:51.070 | INFO | "pinned" : " 29 GB" | 20260505 16:33:51.070 | INFO | } | 20260505 16:33:51.070 | INFO | Perf report for the last 5.14041 seconds: | 20260505 16:33:51.070 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:33:51.070 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:51.070 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:51.070 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:51.070 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260505 16:33:51.070 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:51.070 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:51.071 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260505 16:33:51.071 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:51.071 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:51.071 | INFO | Uploaded 36GiB to GPU in 2.13949s (16.8265GiB/s) | 20260505 16:33:51.071 | INFO | Downloaded 34GiB to GPU in 2.04398s (16.6342GiB/s) | 20260505 16:33:56.201 | INFO | Analyzing chunk frames = [4676608,4677120) | 20260505 16:33:56.201 | INFO | MemInfo: { | 20260505 16:33:56.201 | INFO | "gpu" : " 17 GB", | 20260505 16:33:56.201 | INFO | "huge" : "", | 20260505 16:33:56.201 | INFO | "huge_shared" : " 27 GB", | 20260505 16:33:56.201 | INFO | "malloc" : " 14 GB", | 20260505 16:33:56.201 | INFO | "pinned" : " 29 GB" | 20260505 16:33:56.201 | INFO | } | 20260505 16:33:56.201 | INFO | Perf report for the last 5.13117 seconds: | 20260505 16:33:56.201 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:33:56.201 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:56.201 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:56.201 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:56.201 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 16:33:56.201 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:56.202 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:56.202 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 16:33:56.202 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:56.202 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:33:56.202 | INFO | Uploaded 36GiB to GPU in 2.12555s (16.9368GiB/s) | 20260505 16:33:56.202 | INFO | Downloaded 34GiB to GPU in 2.17575s (15.6268GiB/s) | 20260505 16:34:01.337 | INFO | Analyzing chunk frames = [4677120,4677632) | 20260505 16:34:01.337 | INFO | MemInfo: { | 20260505 16:34:01.337 | INFO | "gpu" : " 17 GB", | 20260505 16:34:01.337 | INFO | "huge" : "", | 20260505 16:34:01.337 | INFO | "huge_shared" : " 27 GB", | 20260505 16:34:01.337 | INFO | "malloc" : " 14 GB", | 20260505 16:34:01.337 | INFO | "pinned" : " 29 GB" | 20260505 16:34:01.337 | INFO | } | 20260505 16:34:01.337 | INFO | Perf report for the last 5.13574 seconds: | 20260505 16:34:01.337 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:34:01.337 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:01.337 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:01.337 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:01.337 | INFO | ANALYSIS : 63.5%, 3.0, 99.0ms, 0 | 20260505 16:34:01.337 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:01.337 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:01.337 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:34:01.337 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:01.337 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:01.338 | INFO | Uploaded 36GiB to GPU in 2.20848s (16.3008GiB/s) | 20260505 16:34:01.338 | INFO | Downloaded 34GiB to GPU in 2.10251s (16.1711GiB/s) | 20260505 16:34:01.338 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 16:34:01.338 | INFO | Total mem usage: 15.3 GiB | 20260505 16:34:01.338 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 16:34:01.338 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.91%) | 20260505 16:34:01.338 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 16:34:01.338 | INFO | High Water Breakdown: | 20260505 16:34:01.338 | INFO | 61% : 8.67 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 16:34:01.338 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 16:34:01.338 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 16:34:01.338 | INFO | ------------------------------------------ | 20260505 16:34:01.338 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:34:01.338 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 16:34:01.338 | INFO | Total mem usage: 26.5 GiB | 20260505 16:34:01.338 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 16:34:01.338 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 16:34:01.338 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 16:34:01.338 | INFO | High Water Breakdown: | 20260505 16:34:01.338 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:34:01.338 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:34:01.338 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:34:01.338 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:34:01.338 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:34:01.338 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:34:01.338 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 16:34:01.338 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 16:34:01.338 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:34:01.338 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 16:34:01.338 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 16:34:01.338 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 16:34:01.338 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 16:34:01.338 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 16:34:01.338 | INFO | ------------------------------------------ | 20260505 16:34:01.338 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:34:01.338 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 16:34:01.338 | INFO | Total mem usage: 25.6 GiB | 20260505 16:34:01.338 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 16:34:01.338 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 16:34:01.338 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 16:34:01.338 | INFO | High Water Breakdown: | 20260505 16:34:01.338 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 16:34:01.338 | INFO | ------------------------------------------ | 20260505 16:34:01.338 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:34:01.338 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 16:34:01.338 | INFO | Total mem usage: 15.7 GiB | 20260505 16:34:01.338 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 16:34:01.338 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 16:34:01.338 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 16:34:01.338 | INFO | High Water Breakdown: | 20260505 16:34:01.338 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 16:34:01.338 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 16:34:01.338 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 16:34:01.338 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 16:34:01.338 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 16:34:01.338 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 16:34:01.338 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 16:34:01.338 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 16:34:01.338 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 16:34:01.338 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:34:01.338 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 16:34:01.338 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:34:01.338 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 16:34:01.338 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 16:34:01.338 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 16:34:01.338 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 16:34:01.338 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:34:01.338 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 16:34:01.338 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:34:01.338 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:34:01.338 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:34:01.339 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 16:34:01.339 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 16:34:01.339 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:34:01.339 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 16:34:01.339 | INFO | ------------------------------------------ | 20260505 16:34:01.339 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:34:01.339 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 16:34:01.339 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:34:01.339 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:34:01.339 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.56e+01, 1.36e+01 / 1.92e+00 | 20260505 16:34:01.339 | INFO | PerfReport: 0.29%, AggregateStats -- 9.26e-02 / 1.02e-01, 1.06e-02 / 4.70e-03 | 20260505 16:34:01.339 | INFO | PerfReport: 11.80%, Baseline -- 3.74e+00 / 4.49e+00, 5.79e-01 / 1.91e-01 | 20260505 16:34:01.339 | INFO | PerfReport: 3.29%, Histogram -- 1.04e+00 / 1.49e+00, 1.95e-01 / 9.47e-02 | 20260505 16:34:01.339 | INFO | PerfReport: 15.91%, DME -- 4.03e+01 / 4.48e+01, 4.28e+00 / 2.06e+00 | 20260505 16:34:01.339 | INFO | PerfReport: 56.76%, FrameLabeling -- 1.80e+01 / 2.00e+01, 1.46e+00 / 1.71e+00 | 20260505 16:34:01.339 | INFO | PerfReport: 7.33%, PulseAccumulating -- 2.32e+00 / 2.88e+00, 4.41e-01 / 2.18e-01 | 20260505 16:34:01.339 | INFO | PerfReport: 3.42%, Metrics -- 1.08e+00 / 1.40e+00, 3.02e-01 / 1.58e-01 | 20260505 16:34:01.339 | INFO | PerfReport: 1.19%, PulseSerializing -- 3.77e-01 / 4.36e-01, 6.77e-02 / 4.09e-02 | 20260505 16:34:01.339 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 16:34:01.339 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:34:01.339 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:34:01.339 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260505 16:34:01.339 | INFO | PerfReport: 22.49%, Upload -- 2.25e+01 / 0.00e+00, 7.90e+00 / 0.00e+00 | 20260505 16:34:01.339 | INFO | PerfReport: 21.93%, Download -- 2.20e+01 / 0.00e+00, 7.95e+00 / 0.00e+00 | 20260505 16:34:01.339 | INFO | PerfReport: 31.64%, Compute -- 3.17e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 16:34:01.339 | INFO | PerfReport: 2.21%, LockUpload -- 2.21e+00 / 0.00e+00, 7.85e+00 / 0.00e+00 | 20260505 16:34:01.339 | INFO | PerfReport: 1.44%, LockDownload -- 1.45e+00 / 0.00e+00, 3.05e+00 / 0.00e+00 | 20260505 16:34:01.339 | INFO | PerfReport: 20.28%, LockCompute -- 2.03e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260505 16:34:01.339 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.97e-02 / 0.00e+00 | 20260505 16:34:01.473 | INFO | Starting a round of Baz IO | 20260505 16:34:04.685 | INFO | Populating RTMetrics report | 20260505 16:34:04.685 | INFO | RTMetrics report populated | 20260505 16:34:04.685 | INFO | Opening tmp RTMetrics json file | 20260505 16:34:04.701 | INFO | Serializing RTMetrics to json | 20260505 16:34:04.707 | INFO | Serialization done, swapping tmp file now... | 20260505 16:34:04.722 | INFO | RTMetrics done | 20260505 16:34:04.784 | INFO | Finished a round of Baz IO: Wrote 2.166 GiB in 3.311 seconds | 20260505 16:34:04.785 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.22% / 0.00% / 77.78% | 20260505 16:34:04.785 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.354576% / 3.97364e-06% / 8.74201e-05% | 20260505 16:34:04.785 | INFO | Baz IO overall: | 20260505 16:34:04.785 | INFO | BytesWritten : 2774408432393 | 20260505 16:34:04.785 | INFO | NumEvents : 1316728467125 | 20260505 16:34:06.485 | INFO | Analyzing chunk frames = [4677632,4678144) | 20260505 16:34:06.485 | INFO | MemInfo: { | 20260505 16:34:06.485 | INFO | "gpu" : " 17 GB", | 20260505 16:34:06.485 | INFO | "huge" : "", | 20260505 16:34:06.485 | INFO | "huge_shared" : " 27 GB", | 20260505 16:34:06.485 | INFO | "malloc" : " 14 GB", | 20260505 16:34:06.485 | INFO | "pinned" : " 29 GB" | 20260505 16:34:06.485 | INFO | } | 20260505 16:34:06.485 | INFO | Perf report for the last 5.14758 seconds: | 20260505 16:34:06.485 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:34:06.485 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:06.485 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:06.485 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:06.485 | INFO | ANALYSIS : 63.9%, 3.0, 99.7ms, 0 | 20260505 16:34:06.485 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:06.485 | INFO | RT_METRICS : 45.1%, 1.0, 23.7ms, 0 | 20260505 16:34:06.485 | INFO | PRE_HQ : 60.3%, 1.8, 56.2ms, 0 | 20260505 16:34:06.485 | INFO | BAZWRITER : 63.7%, 3.9, 128.4ms, 0 | 20260505 16:34:06.485 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 16:34:06.485 | INFO | Uploaded 36GiB to GPU in 2.20423s (16.3323GiB/s) | 20260505 16:34:06.485 | INFO | Downloaded 38GiB to GPU in 2.26458s (16.7802GiB/s) | 20260505 16:34:11.627 | INFO | Analyzing chunk frames = [4678144,4678656) | 20260505 16:34:11.627 | INFO | MemInfo: { | 20260505 16:34:11.627 | INFO | "gpu" : " 17 GB", | 20260505 16:34:11.627 | INFO | "huge" : "", | 20260505 16:34:11.627 | INFO | "huge_shared" : " 27 GB", | 20260505 16:34:11.627 | INFO | "malloc" : " 14 GB", | 20260505 16:34:11.627 | INFO | "pinned" : " 29 GB" | 20260505 16:34:11.627 | INFO | } | 20260505 16:34:11.627 | INFO | Perf report for the last 5.14265 seconds: | 20260505 16:34:11.627 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:34:11.627 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:11.627 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:11.627 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:11.627 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 16:34:11.628 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:11.628 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:11.628 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:34:11.628 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:11.628 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:11.628 | INFO | Uploaded 36GiB to GPU in 2.1269s (16.9261GiB/s) | 20260505 16:34:11.628 | INFO | Downloaded 34GiB to GPU in 2.03138s (16.7374GiB/s) | 20260505 16:34:16.764 | INFO | Analyzing chunk frames = [4678656,4679168) | 20260505 16:34:16.764 | INFO | MemInfo: { | 20260505 16:34:16.764 | INFO | "gpu" : " 17 GB", | 20260505 16:34:16.764 | INFO | "huge" : "", | 20260505 16:34:16.764 | INFO | "huge_shared" : " 27 GB", | 20260505 16:34:16.764 | INFO | "malloc" : " 14 GB", | 20260505 16:34:16.764 | INFO | "pinned" : " 29 GB" | 20260505 16:34:16.764 | INFO | } | 20260505 16:34:16.764 | INFO | Perf report for the last 5.13637 seconds: | 20260505 16:34:16.764 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:34:16.764 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:16.764 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:16.764 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:16.764 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260505 16:34:16.764 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:16.764 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:16.764 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:34:16.764 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:16.764 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:16.764 | INFO | Uploaded 36GiB to GPU in 2.20077s (16.3579GiB/s) | 20260505 16:34:16.764 | INFO | Downloaded 34GiB to GPU in 2.03927s (16.6727GiB/s) | 20260505 16:34:21.902 | INFO | Analyzing chunk frames = [4679168,4679680) | 20260505 16:34:21.902 | INFO | MemInfo: { | 20260505 16:34:21.902 | INFO | "gpu" : " 17 GB", | 20260505 16:34:21.902 | INFO | "huge" : "", | 20260505 16:34:21.902 | INFO | "huge_shared" : " 27 GB", | 20260505 16:34:21.902 | INFO | "malloc" : " 14 GB", | 20260505 16:34:21.902 | INFO | "pinned" : " 29 GB" | 20260505 16:34:21.902 | INFO | } | 20260505 16:34:21.903 | INFO | Perf report for the last 5.13863 seconds: | 20260505 16:34:21.903 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:34:21.903 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:21.903 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:21.903 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:21.903 | INFO | ANALYSIS : 62.6%, 3.0, 97.2ms, 0 | 20260505 16:34:21.903 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:21.903 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:21.903 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:34:21.903 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:21.903 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:21.903 | INFO | Uploaded 36GiB to GPU in 2.18026s (16.5118GiB/s) | 20260505 16:34:21.903 | INFO | Downloaded 34GiB to GPU in 2.14078s (15.882GiB/s) | 20260505 16:34:27.040 | INFO | Analyzing chunk frames = [4679680,4680192) | 20260505 16:34:27.040 | INFO | MemInfo: { | 20260505 16:34:27.040 | INFO | "gpu" : " 17 GB", | 20260505 16:34:27.040 | INFO | "huge" : "", | 20260505 16:34:27.040 | INFO | "huge_shared" : " 27 GB", | 20260505 16:34:27.040 | INFO | "malloc" : " 14 GB", | 20260505 16:34:27.040 | INFO | "pinned" : " 29 GB" | 20260505 16:34:27.040 | INFO | } | 20260505 16:34:27.040 | INFO | Perf report for the last 5.13749 seconds: | 20260505 16:34:27.040 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:34:27.040 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:27.040 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:27.040 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:27.040 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 16:34:27.040 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:27.040 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:27.041 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:34:27.041 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:27.041 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:27.041 | INFO | Uploaded 36GiB to GPU in 2.1437s (16.7934GiB/s) | 20260505 16:34:27.041 | INFO | Downloaded 34GiB to GPU in 2.05187s (16.5703GiB/s) | 20260505 16:34:32.185 | INFO | Analyzing chunk frames = [4680192,4680704) | 20260505 16:34:32.185 | INFO | MemInfo: { | 20260505 16:34:32.185 | INFO | "gpu" : " 17 GB", | 20260505 16:34:32.185 | INFO | "huge" : "", | 20260505 16:34:32.185 | INFO | "huge_shared" : " 27 GB", | 20260505 16:34:32.185 | INFO | "malloc" : " 14 GB", | 20260505 16:34:32.185 | INFO | "pinned" : " 29 GB" | 20260505 16:34:32.185 | INFO | } | 20260505 16:34:32.185 | INFO | Perf report for the last 5.14505 seconds: | 20260505 16:34:32.185 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:34:32.185 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:32.185 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:32.185 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:32.185 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260505 16:34:32.185 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:32.185 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:32.185 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:34:32.185 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:32.185 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:32.185 | INFO | Uploaded 36GiB to GPU in 2.13328s (16.8754GiB/s) | 20260505 16:34:32.185 | INFO | Downloaded 34GiB to GPU in 2.03777s (16.6849GiB/s) | 20260505 16:34:37.333 | INFO | Analyzing chunk frames = [4680704,4681216) | 20260505 16:34:37.333 | INFO | MemInfo: { | 20260505 16:34:37.333 | INFO | "gpu" : " 17 GB", | 20260505 16:34:37.333 | INFO | "huge" : "", | 20260505 16:34:37.333 | INFO | "huge_shared" : " 27 GB", | 20260505 16:34:37.333 | INFO | "malloc" : " 14 GB", | 20260505 16:34:37.333 | INFO | "pinned" : " 29 GB" | 20260505 16:34:37.333 | INFO | } | 20260505 16:34:37.333 | INFO | Perf report for the last 5.14813 seconds: | 20260505 16:34:37.333 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:34:37.333 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:37.333 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:37.333 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:37.333 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 16:34:37.333 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:37.333 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:37.333 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 16:34:37.333 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:37.333 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:37.334 | INFO | Uploaded 36GiB to GPU in 2.16068s (16.6615GiB/s) | 20260505 16:34:37.334 | INFO | Downloaded 34GiB to GPU in 2.18309s (15.5743GiB/s) | 20260505 16:34:42.461 | INFO | Analyzing chunk frames = [4681216,4681728) | 20260505 16:34:42.461 | INFO | MemInfo: { | 20260505 16:34:42.461 | INFO | "gpu" : " 17 GB", | 20260505 16:34:42.461 | INFO | "huge" : "", | 20260505 16:34:42.461 | INFO | "huge_shared" : " 27 GB", | 20260505 16:34:42.461 | INFO | "malloc" : " 14 GB", | 20260505 16:34:42.461 | INFO | "pinned" : " 29 GB" | 20260505 16:34:42.461 | INFO | } | 20260505 16:34:42.461 | INFO | Perf report for the last 5.12758 seconds: | 20260505 16:34:42.461 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:34:42.461 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:42.461 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:42.461 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:42.461 | INFO | ANALYSIS : 63.1%, 3.0, 98.3ms, 0 | 20260505 16:34:42.461 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:42.461 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:42.461 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 16:34:42.461 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:42.461 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:42.461 | INFO | Uploaded 36GiB to GPU in 2.18588s (16.4693GiB/s) | 20260505 16:34:42.461 | INFO | Downloaded 34GiB to GPU in 2.02263s (16.8098GiB/s) | 20260505 16:34:42.597 | INFO | Starting a round of Baz IO | 20260505 16:34:45.800 | INFO | Populating RTMetrics report | 20260505 16:34:45.800 | INFO | RTMetrics report populated | 20260505 16:34:45.800 | INFO | Opening tmp RTMetrics json file | 20260505 16:34:45.809 | INFO | Serializing RTMetrics to json | 20260505 16:34:45.811 | INFO | Serialization done, swapping tmp file now... | 20260505 16:34:45.813 | INFO | RTMetrics done | 20260505 16:34:45.899 | INFO | Finished a round of Baz IO: Wrote 2.166 GiB in 3.302 seconds | 20260505 16:34:45.899 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.22% / 0.00% / 77.78% | 20260505 16:34:45.899 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.355303% / 3.97364e-06% / 8.34465e-05% | 20260505 16:34:45.899 | INFO | Baz IO overall: | 20260505 16:34:45.899 | INFO | BytesWritten : 2776734538075 | 20260505 16:34:45.899 | INFO | NumEvents : 1317818896867 | 20260505 16:34:47.593 | INFO | Analyzing chunk frames = [4681728,4682240) | 20260505 16:34:47.593 | INFO | MemInfo: { | 20260505 16:34:47.593 | INFO | "gpu" : " 17 GB", | 20260505 16:34:47.593 | INFO | "huge" : "", | 20260505 16:34:47.593 | INFO | "huge_shared" : " 27 GB", | 20260505 16:34:47.593 | INFO | "malloc" : " 14 GB", | 20260505 16:34:47.593 | INFO | "pinned" : " 29 GB" | 20260505 16:34:47.593 | INFO | } | 20260505 16:34:47.593 | INFO | Perf report for the last 5.13206 seconds: | 20260505 16:34:47.593 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:34:47.593 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:47.593 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:47.593 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:47.593 | INFO | ANALYSIS : 64.0%, 3.0, 99.5ms, 0 | 20260505 16:34:47.593 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:47.593 | INFO | RT_METRICS : 45.0%, 1.0, 23.5ms, 0 | 20260505 16:34:47.593 | INFO | PRE_HQ : 60.9%, 1.8, 56.6ms, 0 | 20260505 16:34:47.593 | INFO | BAZWRITER : 63.7%, 3.9, 128.2ms, 0 | 20260505 16:34:47.593 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 16:34:47.593 | INFO | Uploaded 36GiB to GPU in 2.1937s (16.4106GiB/s) | 20260505 16:34:47.593 | INFO | Downloaded 38GiB to GPU in 2.27215s (16.7242GiB/s) | 20260505 16:34:52.743 | INFO | Analyzing chunk frames = [4682240,4682752) | 20260505 16:34:52.743 | INFO | MemInfo: { | 20260505 16:34:52.743 | INFO | "gpu" : " 17 GB", | 20260505 16:34:52.743 | INFO | "huge" : "", | 20260505 16:34:52.743 | INFO | "huge_shared" : " 27 GB", | 20260505 16:34:52.743 | INFO | "malloc" : " 14 GB", | 20260505 16:34:52.744 | INFO | "pinned" : " 29 GB" | 20260505 16:34:52.744 | INFO | } | 20260505 16:34:52.744 | INFO | Perf report for the last 5.15071 seconds: | 20260505 16:34:52.744 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:34:52.744 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:52.744 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:52.744 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:52.744 | INFO | ANALYSIS : 63.2%, 3.0, 98.9ms, 0 | 20260505 16:34:52.744 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:52.744 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:52.744 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:34:52.744 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:52.744 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:52.744 | INFO | Uploaded 36GiB to GPU in 2.20135s (16.3536GiB/s) | 20260505 16:34:52.744 | INFO | Downloaded 34GiB to GPU in 2.04585s (16.619GiB/s) | 20260505 16:34:57.883 | INFO | Analyzing chunk frames = [4682752,4683264) | 20260505 16:34:57.883 | INFO | MemInfo: { | 20260505 16:34:57.883 | INFO | "gpu" : " 17 GB", | 20260505 16:34:57.883 | INFO | "huge" : "", | 20260505 16:34:57.883 | INFO | "huge_shared" : " 27 GB", | 20260505 16:34:57.883 | INFO | "malloc" : " 14 GB", | 20260505 16:34:57.883 | INFO | "pinned" : " 29 GB" | 20260505 16:34:57.884 | INFO | } | 20260505 16:34:57.884 | INFO | Perf report for the last 5.13998 seconds: | 20260505 16:34:57.884 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:34:57.884 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:57.884 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:57.884 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:57.884 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 16:34:57.884 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:57.884 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:57.884 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:34:57.884 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:57.884 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:34:57.884 | INFO | Uploaded 36GiB to GPU in 2.20631s (16.3168GiB/s) | 20260505 16:34:57.884 | INFO | Downloaded 34GiB to GPU in 2.06953s (16.4289GiB/s) | 20260505 16:35:03.019 | INFO | Analyzing chunk frames = [4683264,4683776) | 20260505 16:35:03.019 | INFO | MemInfo: { | 20260505 16:35:03.019 | INFO | "gpu" : " 17 GB", | 20260505 16:35:03.019 | INFO | "huge" : "", | 20260505 16:35:03.019 | INFO | "huge_shared" : " 27 GB", | 20260505 16:35:03.019 | INFO | "malloc" : " 14 GB", | 20260505 16:35:03.019 | INFO | "pinned" : " 29 GB" | 20260505 16:35:03.020 | INFO | } | 20260505 16:35:03.020 | INFO | Perf report for the last 5.13597 seconds: | 20260505 16:35:03.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:35:03.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:03.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:03.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:03.020 | INFO | ANALYSIS : 62.7%, 3.0, 97.3ms, 0 | 20260505 16:35:03.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:03.020 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:03.020 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:35:03.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:03.020 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:03.020 | INFO | Uploaded 36GiB to GPU in 2.20532s (16.3242GiB/s) | 20260505 16:35:03.020 | INFO | Downloaded 34GiB to GPU in 2.14077s (15.8822GiB/s) | 20260505 16:35:08.151 | INFO | Analyzing chunk frames = [4683776,4684288) | 20260505 16:35:08.151 | INFO | MemInfo: { | 20260505 16:35:08.151 | INFO | "gpu" : " 17 GB", | 20260505 16:35:08.151 | INFO | "huge" : "", | 20260505 16:35:08.151 | INFO | "huge_shared" : " 27 GB", | 20260505 16:35:08.151 | INFO | "malloc" : " 14 GB", | 20260505 16:35:08.151 | INFO | "pinned" : " 29 GB" | 20260505 16:35:08.151 | INFO | } | 20260505 16:35:08.151 | INFO | Perf report for the last 5.13133 seconds: | 20260505 16:35:08.151 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:35:08.151 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:08.151 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:08.151 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:08.151 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260505 16:35:08.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:08.151 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:08.151 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:35:08.151 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:08.151 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:08.151 | INFO | Uploaded 36GiB to GPU in 2.15997s (16.6669GiB/s) | 20260505 16:35:08.151 | INFO | Downloaded 34GiB to GPU in 2.0589s (16.5137GiB/s) | 20260505 16:35:13.295 | INFO | Analyzing chunk frames = [4684288,4684800) | 20260505 16:35:13.295 | INFO | MemInfo: { | 20260505 16:35:13.295 | INFO | "gpu" : " 17 GB", | 20260505 16:35:13.295 | INFO | "huge" : "", | 20260505 16:35:13.295 | INFO | "huge_shared" : " 27 GB", | 20260505 16:35:13.295 | INFO | "malloc" : " 14 GB", | 20260505 16:35:13.295 | INFO | "pinned" : " 29 GB" | 20260505 16:35:13.295 | INFO | } | 20260505 16:35:13.295 | INFO | Perf report for the last 5.14402 seconds: | 20260505 16:35:13.295 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:35:13.295 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:13.295 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:13.295 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:13.295 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 16:35:13.295 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:13.295 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:13.295 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 16:35:13.295 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:13.295 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:13.295 | INFO | Uploaded 36GiB to GPU in 2.20436s (16.3312GiB/s) | 20260505 16:35:13.295 | INFO | Downloaded 34GiB to GPU in 2.11508s (16.075GiB/s) | 20260505 16:35:18.435 | INFO | Analyzing chunk frames = [4684800,4685312) | 20260505 16:35:18.435 | INFO | MemInfo: { | 20260505 16:35:18.435 | INFO | "gpu" : " 17 GB", | 20260505 16:35:18.435 | INFO | "huge" : "", | 20260505 16:35:18.435 | INFO | "huge_shared" : " 27 GB", | 20260505 16:35:18.435 | INFO | "malloc" : " 14 GB", | 20260505 16:35:18.435 | INFO | "pinned" : " 29 GB" | 20260505 16:35:18.435 | INFO | } | 20260505 16:35:18.435 | INFO | Perf report for the last 5.14008 seconds: | 20260505 16:35:18.435 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:35:18.435 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:18.435 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:18.435 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:18.435 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 16:35:18.435 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:18.435 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:18.435 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 16:35:18.435 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:18.435 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:18.435 | INFO | Uploaded 36GiB to GPU in 2.11282s (17.0389GiB/s) | 20260505 16:35:18.435 | INFO | Downloaded 34GiB to GPU in 2.17754s (15.614GiB/s) | 20260505 16:35:23.586 | INFO | Analyzing chunk frames = [4685312,4685824) | 20260505 16:35:23.586 | INFO | MemInfo: { | 20260505 16:35:23.586 | INFO | "gpu" : " 17 GB", | 20260505 16:35:23.586 | INFO | "huge" : "", | 20260505 16:35:23.586 | INFO | "huge_shared" : " 27 GB", | 20260505 16:35:23.586 | INFO | "malloc" : " 14 GB", | 20260505 16:35:23.586 | INFO | "pinned" : " 29 GB" | 20260505 16:35:23.586 | INFO | } | 20260505 16:35:23.586 | INFO | Perf report for the last 5.1513 seconds: | 20260505 16:35:23.587 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:35:23.587 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:23.587 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:23.587 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:23.587 | INFO | ANALYSIS : 63.0%, 3.0, 98.6ms, 0 | 20260505 16:35:23.587 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:23.587 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:23.587 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 16:35:23.587 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:23.587 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:23.587 | INFO | Uploaded 36GiB to GPU in 2.16611s (16.6196GiB/s) | 20260505 16:35:23.587 | INFO | Downloaded 34GiB to GPU in 2.03087s (16.7416GiB/s) | 20260505 16:35:23.587 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 16:35:23.587 | INFO | Total mem usage: 15.3 GiB | 20260505 16:35:23.587 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 16:35:23.587 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.89%) | 20260505 16:35:23.587 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 16:35:23.587 | INFO | High Water Breakdown: | 20260505 16:35:23.587 | INFO | 61% : 8.67 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 16:35:23.587 | INFO | 38.8% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 16:35:23.587 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 16:35:23.587 | INFO | ------------------------------------------ | 20260505 16:35:23.587 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:35:23.587 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 16:35:23.587 | INFO | Total mem usage: 26.5 GiB | 20260505 16:35:23.587 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 16:35:23.587 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 16:35:23.587 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 16:35:23.587 | INFO | High Water Breakdown: | 20260505 16:35:23.587 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:35:23.587 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:35:23.587 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:35:23.587 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:35:23.587 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:35:23.587 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:35:23.587 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 16:35:23.587 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 16:35:23.587 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:35:23.587 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 16:35:23.587 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 16:35:23.587 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 16:35:23.587 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 16:35:23.587 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 16:35:23.587 | INFO | ------------------------------------------ | 20260505 16:35:23.587 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:35:23.587 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 16:35:23.587 | INFO | Total mem usage: 25.6 GiB | 20260505 16:35:23.587 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 16:35:23.587 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 16:35:23.588 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 16:35:23.588 | INFO | High Water Breakdown: | 20260505 16:35:23.588 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 16:35:23.588 | INFO | ------------------------------------------ | 20260505 16:35:23.588 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:35:23.588 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 16:35:23.588 | INFO | Total mem usage: 15.7 GiB | 20260505 16:35:23.588 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 16:35:23.588 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 16:35:23.588 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 16:35:23.588 | INFO | High Water Breakdown: | 20260505 16:35:23.588 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 16:35:23.588 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 16:35:23.588 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 16:35:23.588 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 16:35:23.588 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 16:35:23.588 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 16:35:23.588 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 16:35:23.588 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 16:35:23.588 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 16:35:23.588 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:35:23.588 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 16:35:23.588 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:35:23.588 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 16:35:23.588 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 16:35:23.588 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 16:35:23.588 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 16:35:23.588 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:35:23.588 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 16:35:23.588 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:35:23.588 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:35:23.588 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:35:23.588 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 16:35:23.588 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 16:35:23.588 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:35:23.588 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 16:35:23.588 | INFO | ------------------------------------------ | 20260505 16:35:23.588 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:35:23.588 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 16:35:23.588 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:35:23.588 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:35:23.588 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.50e+01, 1.37e+01 / 2.02e+00 | 20260505 16:35:23.588 | INFO | PerfReport: 0.29%, AggregateStats -- 9.26e-02 / 1.02e-01, 1.08e-02 / 3.08e-03 | 20260505 16:35:23.588 | INFO | PerfReport: 11.77%, Baseline -- 3.72e+00 / 4.48e+00, 5.66e-01 / 2.01e-01 | 20260505 16:35:23.588 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.51e+00, 1.92e-01 / 9.72e-02 | 20260505 16:35:23.588 | INFO | PerfReport: 15.98%, DME -- 4.04e+01 / 4.50e+01, 4.39e+00 / 2.04e+00 | 20260505 16:35:23.588 | INFO | PerfReport: 56.70%, FrameLabeling -- 1.79e+01 / 1.90e+01, 1.42e+00 / 1.10e+00 | 20260505 16:35:23.588 | INFO | PerfReport: 7.39%, PulseAccumulating -- 2.34e+00 / 2.94e+00, 4.54e-01 / 1.81e-01 | 20260505 16:35:23.588 | INFO | PerfReport: 3.41%, Metrics -- 1.08e+00 / 1.42e+00, 2.92e-01 / 1.09e-01 | 20260505 16:35:23.588 | INFO | PerfReport: 1.19%, PulseSerializing -- 3.76e-01 / 4.42e-01, 6.72e-02 / 3.82e-02 | 20260505 16:35:23.588 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 16:35:23.588 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:35:23.588 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:35:23.588 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.58e+01 / 0.00e+00 | 20260505 16:35:23.588 | INFO | PerfReport: 22.59%, Upload -- 2.26e+01 / 0.00e+00, 7.94e+00 / 0.00e+00 | 20260505 16:35:23.588 | INFO | PerfReport: 21.88%, Download -- 2.19e+01 / 0.00e+00, 7.88e+00 / 0.00e+00 | 20260505 16:35:23.588 | INFO | PerfReport: 31.56%, Compute -- 3.16e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 16:35:23.588 | INFO | PerfReport: 2.34%, LockUpload -- 2.35e+00 / 0.00e+00, 8.00e+00 / 0.00e+00 | 20260505 16:35:23.588 | INFO | PerfReport: 1.43%, LockDownload -- 1.43e+00 / 0.00e+00, 2.95e+00 / 0.00e+00 | 20260505 16:35:23.588 | INFO | PerfReport: 20.18%, LockCompute -- 2.02e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260505 16:35:23.588 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.02e-02 / 0.00e+00 | 20260505 16:35:23.723 | INFO | Starting a round of Baz IO | 20260505 16:35:26.909 | INFO | Populating RTMetrics report | 20260505 16:35:26.909 | INFO | RTMetrics report populated | 20260505 16:35:26.909 | INFO | Opening tmp RTMetrics json file | 20260505 16:35:26.915 | INFO | Serializing RTMetrics to json | 20260505 16:35:26.917 | INFO | Serialization done, swapping tmp file now... | 20260505 16:35:26.917 | INFO | RTMetrics done | 20260505 16:35:27.008 | INFO | Finished a round of Baz IO: Wrote 2.165 GiB in 3.284 seconds | 20260505 16:35:27.008 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.22% / 0.00% / 77.78% | 20260505 16:35:27.008 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.353797% / 0% / 9.13938e-05% | 20260505 16:35:27.008 | INFO | Baz IO overall: | 20260505 16:35:27.008 | INFO | BytesWritten : 2779059592492 | 20260505 16:35:27.008 | INFO | NumEvents : 1318908693884 | 20260505 16:35:28.712 | INFO | Analyzing chunk frames = [4685824,4686336) | 20260505 16:35:28.712 | INFO | MemInfo: { | 20260505 16:35:28.712 | INFO | "gpu" : " 17 GB", | 20260505 16:35:28.712 | INFO | "huge" : "", | 20260505 16:35:28.712 | INFO | "huge_shared" : " 27 GB", | 20260505 16:35:28.712 | INFO | "malloc" : " 14 GB", | 20260505 16:35:28.712 | INFO | "pinned" : " 29 GB" | 20260505 16:35:28.712 | INFO | } | 20260505 16:35:28.712 | INFO | Perf report for the last 5.12558 seconds: | 20260505 16:35:28.712 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:35:28.712 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:28.712 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:28.712 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:28.712 | INFO | ANALYSIS : 63.7%, 3.0, 99.0ms, 0 | 20260505 16:35:28.712 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:28.712 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260505 16:35:28.712 | INFO | PRE_HQ : 60.7%, 1.8, 56.3ms, 0 | 20260505 16:35:28.712 | INFO | BAZWRITER : 63.5%, 3.9, 126.4ms, 0 | 20260505 16:35:28.712 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 16:35:28.712 | INFO | Uploaded 36GiB to GPU in 2.14166s (16.8094GiB/s) | 20260505 16:35:28.712 | INFO | Downloaded 38GiB to GPU in 2.25349s (16.8628GiB/s) | 20260505 16:35:33.849 | INFO | Analyzing chunk frames = [4686336,4686848) | 20260505 16:35:33.849 | INFO | MemInfo: { | 20260505 16:35:33.849 | INFO | "gpu" : " 17 GB", | 20260505 16:35:33.849 | INFO | "huge" : "", | 20260505 16:35:33.849 | INFO | "huge_shared" : " 27 GB", | 20260505 16:35:33.849 | INFO | "malloc" : " 14 GB", | 20260505 16:35:33.849 | INFO | "pinned" : " 29 GB" | 20260505 16:35:33.849 | INFO | } | 20260505 16:35:33.850 | INFO | Perf report for the last 5.13763 seconds: | 20260505 16:35:33.850 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:35:33.850 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:33.850 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:33.850 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:33.850 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 16:35:33.850 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:33.850 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:33.850 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 16:35:33.850 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:33.850 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:33.850 | INFO | Uploaded 36GiB to GPU in 2.2042s (16.3325GiB/s) | 20260505 16:35:33.850 | INFO | Downloaded 34GiB to GPU in 2.0657s (16.4593GiB/s) | 20260505 16:35:38.998 | INFO | Analyzing chunk frames = [4686848,4687360) | 20260505 16:35:38.998 | INFO | MemInfo: { | 20260505 16:35:38.998 | INFO | "gpu" : " 17 GB", | 20260505 16:35:38.998 | INFO | "huge" : "", | 20260505 16:35:38.998 | INFO | "huge_shared" : " 27 GB", | 20260505 16:35:38.998 | INFO | "malloc" : " 14 GB", | 20260505 16:35:38.998 | INFO | "pinned" : " 29 GB" | 20260505 16:35:38.998 | INFO | } | 20260505 16:35:38.998 | INFO | Perf report for the last 5.14805 seconds: | 20260505 16:35:38.998 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:35:38.998 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:38.998 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:38.998 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:38.998 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 16:35:38.998 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:38.998 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:38.998 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 16:35:38.998 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:38.998 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:38.998 | INFO | Uploaded 36GiB to GPU in 2.17929s (16.5191GiB/s) | 20260505 16:35:38.998 | INFO | Downloaded 34GiB to GPU in 2.03041s (16.7454GiB/s) | 20260505 16:35:44.132 | INFO | Analyzing chunk frames = [4687360,4687872) | 20260505 16:35:44.132 | INFO | MemInfo: { | 20260505 16:35:44.132 | INFO | "gpu" : " 17 GB", | 20260505 16:35:44.132 | INFO | "huge" : "", | 20260505 16:35:44.132 | INFO | "huge_shared" : " 27 GB", | 20260505 16:35:44.132 | INFO | "malloc" : " 14 GB", | 20260505 16:35:44.132 | INFO | "pinned" : " 29 GB" | 20260505 16:35:44.132 | INFO | } | 20260505 16:35:44.132 | INFO | Perf report for the last 5.13435 seconds: | 20260505 16:35:44.132 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:35:44.132 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:44.132 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:44.132 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:44.132 | INFO | ANALYSIS : 62.9%, 3.0, 97.6ms, 0 | 20260505 16:35:44.132 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:44.132 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:44.132 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:35:44.132 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:44.132 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:44.132 | INFO | Uploaded 36GiB to GPU in 2.2265s (16.1689GiB/s) | 20260505 16:35:44.132 | INFO | Downloaded 34GiB to GPU in 2.11888s (16.0462GiB/s) | 20260505 16:35:49.271 | INFO | Analyzing chunk frames = [4687872,4688384) | 20260505 16:35:49.271 | INFO | MemInfo: { | 20260505 16:35:49.271 | INFO | "gpu" : " 17 GB", | 20260505 16:35:49.271 | INFO | "huge" : "", | 20260505 16:35:49.271 | INFO | "huge_shared" : " 27 GB", | 20260505 16:35:49.271 | INFO | "malloc" : " 14 GB", | 20260505 16:35:49.271 | INFO | "pinned" : " 29 GB" | 20260505 16:35:49.271 | INFO | } | 20260505 16:35:49.271 | INFO | Perf report for the last 5.13927 seconds: | 20260505 16:35:49.271 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:35:49.271 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:49.271 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:49.271 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:49.271 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 16:35:49.271 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:49.271 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:49.271 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:35:49.271 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:49.271 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:49.272 | INFO | Uploaded 36GiB to GPU in 2.13945s (16.8267GiB/s) | 20260505 16:35:49.272 | INFO | Downloaded 34GiB to GPU in 2.05281s (16.5627GiB/s) | 20260505 16:35:54.406 | INFO | Analyzing chunk frames = [4688384,4688896) | 20260505 16:35:54.406 | INFO | MemInfo: { | 20260505 16:35:54.406 | INFO | "gpu" : " 17 GB", | 20260505 16:35:54.406 | INFO | "huge" : "", | 20260505 16:35:54.407 | INFO | "huge_shared" : " 27 GB", | 20260505 16:35:54.407 | INFO | "malloc" : " 14 GB", | 20260505 16:35:54.407 | INFO | "pinned" : " 29 GB" | 20260505 16:35:54.407 | INFO | } | 20260505 16:35:54.407 | INFO | Perf report for the last 5.13541 seconds: | 20260505 16:35:54.407 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:35:54.407 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:54.407 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:54.407 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:54.407 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260505 16:35:54.407 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:54.407 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:54.407 | INFO | PRE_HQ : 27.7%, 1.0, 14.3ms, 0 | 20260505 16:35:54.407 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:54.407 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:54.407 | INFO | Uploaded 36GiB to GPU in 2.13785s (16.8393GiB/s) | 20260505 16:35:54.407 | INFO | Downloaded 34GiB to GPU in 2.03952s (16.6706GiB/s) | 20260505 16:35:59.551 | INFO | Analyzing chunk frames = [4688896,4689408) | 20260505 16:35:59.551 | INFO | MemInfo: { | 20260505 16:35:59.551 | INFO | "gpu" : " 17 GB", | 20260505 16:35:59.551 | INFO | "huge" : "", | 20260505 16:35:59.551 | INFO | "huge_shared" : " 27 GB", | 20260505 16:35:59.551 | INFO | "malloc" : " 14 GB", | 20260505 16:35:59.551 | INFO | "pinned" : " 29 GB" | 20260505 16:35:59.551 | INFO | } | 20260505 16:35:59.551 | INFO | Perf report for the last 5.14452 seconds: | 20260505 16:35:59.551 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:35:59.551 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:59.551 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:59.551 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:59.551 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 16:35:59.551 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:59.551 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:59.551 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 16:35:59.551 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:59.551 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:35:59.551 | INFO | Uploaded 36GiB to GPU in 2.11134s (17.0508GiB/s) | 20260505 16:35:59.552 | INFO | Downloaded 34GiB to GPU in 2.17899s (15.6036GiB/s) | 20260505 16:36:04.697 | INFO | Analyzing chunk frames = [4689408,4689920) | 20260505 16:36:04.697 | INFO | MemInfo: { | 20260505 16:36:04.697 | INFO | "gpu" : " 17 GB", | 20260505 16:36:04.697 | INFO | "huge" : "", | 20260505 16:36:04.697 | INFO | "huge_shared" : " 27 GB", | 20260505 16:36:04.697 | INFO | "malloc" : " 14 GB", | 20260505 16:36:04.697 | INFO | "pinned" : " 29 GB" | 20260505 16:36:04.697 | INFO | } | 20260505 16:36:04.697 | INFO | Perf report for the last 5.14621 seconds: | 20260505 16:36:04.697 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:36:04.697 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:04.697 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:04.697 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:04.697 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 16:36:04.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:04.698 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:04.698 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 16:36:04.698 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:04.698 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:04.698 | INFO | Uploaded 36GiB to GPU in 2.16421s (16.6343GiB/s) | 20260505 16:36:04.698 | INFO | Downloaded 34GiB to GPU in 2.02682s (16.775GiB/s) | 20260505 16:36:04.835 | INFO | Starting a round of Baz IO | 20260505 16:36:08.012 | INFO | Populating RTMetrics report | 20260505 16:36:08.012 | INFO | RTMetrics report populated | 20260505 16:36:08.012 | INFO | Opening tmp RTMetrics json file | 20260505 16:36:08.020 | INFO | Serializing RTMetrics to json | 20260505 16:36:08.023 | INFO | Serialization done, swapping tmp file now... | 20260505 16:36:08.026 | INFO | RTMetrics done | 20260505 16:36:08.108 | INFO | Finished a round of Baz IO: Wrote 2.167 GiB in 3.272 seconds | 20260505 16:36:08.108 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.21% / 0.00% / 77.78% | 20260505 16:36:08.108 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.353913% / 7.94729e-06% / 9.13938e-05% | 20260505 16:36:08.108 | INFO | Baz IO overall: | 20260505 16:36:08.108 | INFO | BytesWritten : 2781385919983 | 20260505 16:36:08.108 | INFO | NumEvents : 1319999534553 | 20260505 16:36:09.828 | INFO | Analyzing chunk frames = [4689920,4690432) | 20260505 16:36:09.829 | INFO | MemInfo: { | 20260505 16:36:09.829 | INFO | "gpu" : " 17 GB", | 20260505 16:36:09.829 | INFO | "huge" : "", | 20260505 16:36:09.829 | INFO | "huge_shared" : " 27 GB", | 20260505 16:36:09.829 | INFO | "malloc" : " 14 GB", | 20260505 16:36:09.829 | INFO | "pinned" : " 29 GB" | 20260505 16:36:09.829 | INFO | } | 20260505 16:36:09.829 | INFO | Perf report for the last 5.1312 seconds: | 20260505 16:36:09.829 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:36:09.829 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:09.829 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:09.829 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:09.829 | INFO | ANALYSIS : 63.5%, 3.0, 98.6ms, 0 | 20260505 16:36:09.829 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:09.829 | INFO | RT_METRICS : 44.9%, 1.0, 23.5ms, 0 | 20260505 16:36:09.829 | INFO | PRE_HQ : 60.7%, 1.8, 56.3ms, 0 | 20260505 16:36:09.829 | INFO | BAZWRITER : 63.3%, 3.9, 127.8ms, 0 | 20260505 16:36:09.829 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 16:36:09.829 | INFO | Uploaded 36GiB to GPU in 2.1614s (16.6559GiB/s) | 20260505 16:36:09.829 | INFO | Downloaded 38GiB to GPU in 2.24998s (16.889GiB/s) | 20260505 16:36:14.960 | INFO | Analyzing chunk frames = [4690432,4690944) | 20260505 16:36:14.960 | INFO | MemInfo: { | 20260505 16:36:14.960 | INFO | "gpu" : " 17 GB", | 20260505 16:36:14.960 | INFO | "huge" : "", | 20260505 16:36:14.960 | INFO | "huge_shared" : " 27 GB", | 20260505 16:36:14.960 | INFO | "malloc" : " 14 GB", | 20260505 16:36:14.960 | INFO | "pinned" : " 29 GB" | 20260505 16:36:14.960 | INFO | } | 20260505 16:36:14.960 | INFO | Perf report for the last 5.1316 seconds: | 20260505 16:36:14.960 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:36:14.960 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:14.960 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:14.960 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:14.960 | INFO | ANALYSIS : 63.1%, 3.0, 98.3ms, 0 | 20260505 16:36:14.960 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:14.960 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:14.960 | INFO | PRE_HQ : 27.7%, 1.0, 14.3ms, 0 | 20260505 16:36:14.960 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:14.960 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:14.961 | INFO | Uploaded 36GiB to GPU in 2.13312s (16.8767GiB/s) | 20260505 16:36:14.961 | INFO | Downloaded 34GiB to GPU in 2.03969s (16.6692GiB/s) | 20260505 16:36:20.099 | INFO | Analyzing chunk frames = [4690944,4691456) | 20260505 16:36:20.099 | INFO | MemInfo: { | 20260505 16:36:20.099 | INFO | "gpu" : " 17 GB", | 20260505 16:36:20.099 | INFO | "huge" : "", | 20260505 16:36:20.099 | INFO | "huge_shared" : " 27 GB", | 20260505 16:36:20.099 | INFO | "malloc" : " 14 GB", | 20260505 16:36:20.099 | INFO | "pinned" : " 29 GB" | 20260505 16:36:20.099 | INFO | } | 20260505 16:36:20.099 | INFO | Perf report for the last 5.13895 seconds: | 20260505 16:36:20.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:36:20.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:20.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:20.100 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:20.100 | INFO | ANALYSIS : 62.9%, 3.0, 97.9ms, 0 | 20260505 16:36:20.100 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:20.100 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:20.100 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:36:20.100 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:20.100 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:20.100 | INFO | Uploaded 36GiB to GPU in 2.18599s (16.4685GiB/s) | 20260505 16:36:20.100 | INFO | Downloaded 34GiB to GPU in 2.04711s (16.6088GiB/s) | 20260505 16:36:25.250 | INFO | Analyzing chunk frames = [4691456,4691968) | 20260505 16:36:25.250 | INFO | MemInfo: { | 20260505 16:36:25.250 | INFO | "gpu" : " 17 GB", | 20260505 16:36:25.250 | INFO | "huge" : "", | 20260505 16:36:25.250 | INFO | "huge_shared" : " 27 GB", | 20260505 16:36:25.250 | INFO | "malloc" : " 14 GB", | 20260505 16:36:25.251 | INFO | "pinned" : " 29 GB" | 20260505 16:36:25.251 | INFO | } | 20260505 16:36:25.251 | INFO | Perf report for the last 5.15136 seconds: | 20260505 16:36:25.251 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:36:25.251 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:25.251 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:25.251 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:25.251 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260505 16:36:25.251 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:25.251 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:25.251 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:36:25.251 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:25.251 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:25.251 | INFO | Uploaded 36GiB to GPU in 2.18565s (16.4711GiB/s) | 20260505 16:36:25.251 | INFO | Downloaded 34GiB to GPU in 2.13696s (15.9105GiB/s) | 20260505 16:36:30.387 | INFO | Analyzing chunk frames = [4691968,4692480) | 20260505 16:36:30.387 | INFO | MemInfo: { | 20260505 16:36:30.387 | INFO | "gpu" : " 17 GB", | 20260505 16:36:30.387 | INFO | "huge" : "", | 20260505 16:36:30.387 | INFO | "huge_shared" : " 27 GB", | 20260505 16:36:30.387 | INFO | "malloc" : " 14 GB", | 20260505 16:36:30.387 | INFO | "pinned" : " 29 GB" | 20260505 16:36:30.387 | INFO | } | 20260505 16:36:30.387 | INFO | Perf report for the last 5.13616 seconds: | 20260505 16:36:30.387 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:36:30.387 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:30.387 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:30.387 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:30.387 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 16:36:30.387 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:30.387 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:30.387 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:36:30.387 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:30.387 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:30.387 | INFO | Uploaded 36GiB to GPU in 2.15108s (16.7358GiB/s) | 20260505 16:36:30.387 | INFO | Downloaded 34GiB to GPU in 2.06073s (16.499GiB/s) | 20260505 16:36:35.531 | INFO | Analyzing chunk frames = [4692480,4692992) | 20260505 16:36:35.531 | INFO | MemInfo: { | 20260505 16:36:35.531 | INFO | "gpu" : " 17 GB", | 20260505 16:36:35.531 | INFO | "huge" : "", | 20260505 16:36:35.531 | INFO | "huge_shared" : " 27 GB", | 20260505 16:36:35.531 | INFO | "malloc" : " 14 GB", | 20260505 16:36:35.531 | INFO | "pinned" : " 29 GB" | 20260505 16:36:35.531 | INFO | } | 20260505 16:36:35.531 | INFO | Perf report for the last 5.14434 seconds: | 20260505 16:36:35.531 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:36:35.531 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:35.531 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:35.531 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:35.531 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260505 16:36:35.531 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:35.531 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:35.531 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:36:35.531 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:35.531 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:35.531 | INFO | Uploaded 36GiB to GPU in 2.13865s (16.833GiB/s) | 20260505 16:36:35.532 | INFO | Downloaded 34GiB to GPU in 2.04066s (16.6613GiB/s) | 20260505 16:36:40.660 | INFO | Analyzing chunk frames = [4692992,4693504) | 20260505 16:36:40.660 | INFO | MemInfo: { | 20260505 16:36:40.660 | INFO | "gpu" : " 17 GB", | 20260505 16:36:40.660 | INFO | "huge" : "", | 20260505 16:36:40.660 | INFO | "huge_shared" : " 27 GB", | 20260505 16:36:40.660 | INFO | "malloc" : " 14 GB", | 20260505 16:36:40.660 | INFO | "pinned" : " 29 GB" | 20260505 16:36:40.660 | INFO | } | 20260505 16:36:40.660 | INFO | Perf report for the last 5.1293 seconds: | 20260505 16:36:40.660 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:36:40.661 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:40.661 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:40.661 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:40.661 | INFO | ANALYSIS : 63.8%, 3.0, 99.4ms, 0 | 20260505 16:36:40.661 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:40.661 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:40.661 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 16:36:40.661 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:40.661 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:40.661 | INFO | Uploaded 36GiB to GPU in 2.12029s (16.9788GiB/s) | 20260505 16:36:40.661 | INFO | Downloaded 34GiB to GPU in 2.17586s (15.626GiB/s) | 20260505 16:36:45.803 | INFO | Analyzing chunk frames = [4693504,4694016) | 20260505 16:36:45.804 | INFO | MemInfo: { | 20260505 16:36:45.804 | INFO | "gpu" : " 17 GB", | 20260505 16:36:45.804 | INFO | "huge" : "", | 20260505 16:36:45.804 | INFO | "huge_shared" : " 27 GB", | 20260505 16:36:45.804 | INFO | "malloc" : " 14 GB", | 20260505 16:36:45.804 | INFO | "pinned" : " 29 GB" | 20260505 16:36:45.804 | INFO | } | 20260505 16:36:45.804 | INFO | Perf report for the last 5.14327 seconds: | 20260505 16:36:45.804 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:36:45.804 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:45.804 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:45.804 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:45.804 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 16:36:45.804 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:45.804 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:45.804 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 16:36:45.804 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:45.804 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:45.804 | INFO | Uploaded 36GiB to GPU in 2.21977s (16.2179GiB/s) | 20260505 16:36:45.804 | INFO | Downloaded 34GiB to GPU in 2.10066s (16.1854GiB/s) | 20260505 16:36:45.804 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 16:36:45.804 | INFO | Total mem usage: 15.3 GiB | 20260505 16:36:45.804 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 16:36:45.804 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.89%) | 20260505 16:36:45.804 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 16:36:45.804 | INFO | High Water Breakdown: | 20260505 16:36:45.804 | INFO | 61% : 8.67 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 16:36:45.804 | INFO | 38.8% : 5.51 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 16:36:45.804 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 16:36:45.804 | INFO | ------------------------------------------ | 20260505 16:36:45.805 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:36:45.805 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 16:36:45.805 | INFO | Total mem usage: 26.5 GiB | 20260505 16:36:45.805 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 16:36:45.805 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 16:36:45.805 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 16:36:45.805 | INFO | High Water Breakdown: | 20260505 16:36:45.805 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:36:45.805 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:36:45.805 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:36:45.805 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:36:45.805 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:36:45.805 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:36:45.805 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 16:36:45.805 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 16:36:45.805 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:36:45.805 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 16:36:45.805 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 16:36:45.805 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 16:36:45.805 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 16:36:45.805 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 16:36:45.805 | INFO | ------------------------------------------ | 20260505 16:36:45.805 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:36:45.805 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 16:36:45.805 | INFO | Total mem usage: 25.6 GiB | 20260505 16:36:45.805 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 16:36:45.805 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 16:36:45.805 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 16:36:45.805 | INFO | High Water Breakdown: | 20260505 16:36:45.805 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 16:36:45.805 | INFO | ------------------------------------------ | 20260505 16:36:45.805 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:36:45.805 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 16:36:45.805 | INFO | Total mem usage: 15.7 GiB | 20260505 16:36:45.805 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 16:36:45.805 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 16:36:45.805 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 16:36:45.805 | INFO | High Water Breakdown: | 20260505 16:36:45.805 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 16:36:45.805 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 16:36:45.805 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 16:36:45.805 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 16:36:45.805 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 16:36:45.805 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 16:36:45.805 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 16:36:45.805 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 16:36:45.805 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 16:36:45.805 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:36:45.805 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 16:36:45.805 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:36:45.805 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 16:36:45.805 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 16:36:45.805 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 16:36:45.805 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 16:36:45.805 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:36:45.805 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 16:36:45.805 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:36:45.805 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:36:45.805 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:36:45.805 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 16:36:45.805 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 16:36:45.805 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:36:45.805 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 16:36:45.805 | INFO | ------------------------------------------ | 20260505 16:36:45.805 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:36:45.805 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 16:36:45.805 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:36:45.805 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:36:45.805 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.53e+01, 1.36e+01 / 2.00e+00 | 20260505 16:36:45.805 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 1.05e-01, 1.26e-02 / 1.57e-02 | 20260505 16:36:45.805 | INFO | PerfReport: 11.79%, Baseline -- 3.74e+00 / 4.50e+00, 5.81e-01 / 1.57e-01 | 20260505 16:36:45.805 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.52e+00, 1.95e-01 / 9.48e-02 | 20260505 16:36:45.805 | INFO | PerfReport: 15.92%, DME -- 4.04e+01 / 4.49e+01, 4.35e+00 / 1.90e+00 | 20260505 16:36:45.805 | INFO | PerfReport: 56.81%, FrameLabeling -- 1.80e+01 / 1.96e+01, 1.52e+00 / 1.41e+00 | 20260505 16:36:45.805 | INFO | PerfReport: 7.32%, PulseAccumulating -- 2.32e+00 / 2.91e+00, 4.44e-01 / 1.80e-01 | 20260505 16:36:45.805 | INFO | PerfReport: 3.40%, Metrics -- 1.08e+00 / 1.39e+00, 2.99e-01 / 1.52e-01 | 20260505 16:36:45.805 | INFO | PerfReport: 1.19%, PulseSerializing -- 3.77e-01 / 4.35e-01, 6.82e-02 / 3.95e-02 | 20260505 16:36:45.805 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 16:36:45.805 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:36:45.805 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:36:45.805 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260505 16:36:45.805 | INFO | PerfReport: 22.49%, Upload -- 2.25e+01 / 0.00e+00, 7.84e+00 / 0.00e+00 | 20260505 16:36:45.805 | INFO | PerfReport: 21.85%, Download -- 2.19e+01 / 0.00e+00, 7.88e+00 / 0.00e+00 | 20260505 16:36:45.805 | INFO | PerfReport: 31.67%, Compute -- 3.17e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 16:36:45.805 | INFO | PerfReport: 2.27%, LockUpload -- 2.27e+00 / 0.00e+00, 7.84e+00 / 0.00e+00 | 20260505 16:36:45.805 | INFO | PerfReport: 1.41%, LockDownload -- 1.41e+00 / 0.00e+00, 3.02e+00 / 0.00e+00 | 20260505 16:36:45.805 | INFO | PerfReport: 20.30%, LockCompute -- 2.03e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 16:36:45.805 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 2.01e-02 / 0.00e+00 | 20260505 16:36:45.937 | INFO | Starting a round of Baz IO | 20260505 16:36:49.141 | INFO | Populating RTMetrics report | 20260505 16:36:49.141 | INFO | RTMetrics report populated | 20260505 16:36:49.141 | INFO | Opening tmp RTMetrics json file | 20260505 16:36:49.156 | INFO | Serializing RTMetrics to json | 20260505 16:36:49.157 | INFO | Serialization done, swapping tmp file now... | 20260505 16:36:49.160 | INFO | RTMetrics done | 20260505 16:36:49.240 | INFO | Finished a round of Baz IO: Wrote 2.166 GiB in 3.302 seconds | 20260505 16:36:49.240 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.21% / 0.00% / 77.79% | 20260505 16:36:49.240 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.353161% / 3.97364e-06% / 0.000103315% | 20260505 16:36:49.240 | INFO | Baz IO overall: | 20260505 16:36:49.240 | INFO | BytesWritten : 2783712093994 | 20260505 16:36:49.240 | INFO | NumEvents : 1321090316862 | 20260505 16:36:50.946 | INFO | Analyzing chunk frames = [4694016,4694528) | 20260505 16:36:50.946 | INFO | MemInfo: { | 20260505 16:36:50.946 | INFO | "gpu" : " 17 GB", | 20260505 16:36:50.946 | INFO | "huge" : "", | 20260505 16:36:50.946 | INFO | "huge_shared" : " 27 GB", | 20260505 16:36:50.946 | INFO | "malloc" : " 14 GB", | 20260505 16:36:50.946 | INFO | "pinned" : " 29 GB" | 20260505 16:36:50.946 | INFO | } | 20260505 16:36:50.946 | INFO | Perf report for the last 5.14257 seconds: | 20260505 16:36:50.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:36:50.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:50.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:50.947 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:50.947 | INFO | ANALYSIS : 63.8%, 3.0, 99.3ms, 0 | 20260505 16:36:50.947 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:50.947 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260505 16:36:50.947 | INFO | PRE_HQ : 60.6%, 1.8, 56.3ms, 0 | 20260505 16:36:50.947 | INFO | BAZWRITER : 63.6%, 3.9, 127.4ms, 0 | 20260505 16:36:50.947 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 16:36:50.947 | INFO | Uploaded 36GiB to GPU in 2.1669s (16.6136GiB/s) | 20260505 16:36:50.947 | INFO | Downloaded 38GiB to GPU in 2.26059s (16.8098GiB/s) | 20260505 16:36:56.082 | INFO | Analyzing chunk frames = [4694528,4695040) | 20260505 16:36:56.082 | INFO | MemInfo: { | 20260505 16:36:56.082 | INFO | "gpu" : " 17 GB", | 20260505 16:36:56.082 | INFO | "huge" : "", | 20260505 16:36:56.082 | INFO | "huge_shared" : " 27 GB", | 20260505 16:36:56.082 | INFO | "malloc" : " 14 GB", | 20260505 16:36:56.082 | INFO | "pinned" : " 29 GB" | 20260505 16:36:56.082 | INFO | } | 20260505 16:36:56.082 | INFO | Perf report for the last 5.13611 seconds: | 20260505 16:36:56.082 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:36:56.082 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:56.082 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:56.082 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:56.083 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260505 16:36:56.083 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:56.083 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:56.083 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:36:56.083 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:56.083 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:36:56.083 | INFO | Uploaded 36GiB to GPU in 2.12569s (16.9356GiB/s) | 20260505 16:36:56.083 | INFO | Downloaded 34GiB to GPU in 2.02386s (16.7995GiB/s) | 20260505 16:37:01.219 | INFO | Analyzing chunk frames = [4695040,4695552) | 20260505 16:37:01.219 | INFO | MemInfo: { | 20260505 16:37:01.219 | INFO | "gpu" : " 17 GB", | 20260505 16:37:01.219 | INFO | "huge" : "", | 20260505 16:37:01.219 | INFO | "huge_shared" : " 27 GB", | 20260505 16:37:01.219 | INFO | "malloc" : " 14 GB", | 20260505 16:37:01.219 | INFO | "pinned" : " 29 GB" | 20260505 16:37:01.219 | INFO | } | 20260505 16:37:01.219 | INFO | Perf report for the last 5.13669 seconds: | 20260505 16:37:01.219 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:37:01.219 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:01.219 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:01.219 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:01.219 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260505 16:37:01.219 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:01.219 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:01.219 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 16:37:01.219 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:01.219 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:01.219 | INFO | Uploaded 36GiB to GPU in 2.20278s (16.343GiB/s) | 20260505 16:37:01.219 | INFO | Downloaded 34GiB to GPU in 2.04402s (16.6339GiB/s) | 20260505 16:37:06.354 | INFO | Analyzing chunk frames = [4695552,4696064) | 20260505 16:37:06.354 | INFO | MemInfo: { | 20260505 16:37:06.354 | INFO | "gpu" : " 17 GB", | 20260505 16:37:06.354 | INFO | "huge" : "", | 20260505 16:37:06.354 | INFO | "huge_shared" : " 27 GB", | 20260505 16:37:06.354 | INFO | "malloc" : " 14 GB", | 20260505 16:37:06.354 | INFO | "pinned" : " 29 GB" | 20260505 16:37:06.354 | INFO | } | 20260505 16:37:06.354 | INFO | Perf report for the last 5.13487 seconds: | 20260505 16:37:06.354 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:37:06.354 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:06.354 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:06.354 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:06.354 | INFO | ANALYSIS : 62.7%, 3.0, 97.3ms, 0 | 20260505 16:37:06.354 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:06.354 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:06.354 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 16:37:06.354 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:06.354 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:06.354 | INFO | Uploaded 36GiB to GPU in 2.17373s (16.5614GiB/s) | 20260505 16:37:06.354 | INFO | Downloaded 34GiB to GPU in 2.14624s (15.8417GiB/s) | 20260505 16:37:11.504 | INFO | Analyzing chunk frames = [4696064,4696576) | 20260505 16:37:11.504 | INFO | MemInfo: { | 20260505 16:37:11.504 | INFO | "gpu" : " 17 GB", | 20260505 16:37:11.504 | INFO | "huge" : "", | 20260505 16:37:11.504 | INFO | "huge_shared" : " 27 GB", | 20260505 16:37:11.504 | INFO | "malloc" : " 14 GB", | 20260505 16:37:11.504 | INFO | "pinned" : " 29 GB" | 20260505 16:37:11.504 | INFO | } | 20260505 16:37:11.504 | INFO | Perf report for the last 5.15006 seconds: | 20260505 16:37:11.504 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:37:11.504 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:11.504 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:11.504 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:11.504 | INFO | ANALYSIS : 62.5%, 3.0, 97.8ms, 0 | 20260505 16:37:11.504 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:11.504 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:11.504 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 16:37:11.504 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:11.504 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:11.505 | INFO | Uploaded 36GiB to GPU in 2.18048s (16.5101GiB/s) | 20260505 16:37:11.505 | INFO | Downloaded 34GiB to GPU in 2.13362s (15.9353GiB/s) | 20260505 16:37:16.651 | INFO | Analyzing chunk frames = [4696576,4697088) | 20260505 16:37:16.651 | INFO | MemInfo: { | 20260505 16:37:16.651 | INFO | "gpu" : " 17 GB", | 20260505 16:37:16.651 | INFO | "huge" : "", | 20260505 16:37:16.651 | INFO | "huge_shared" : " 27 GB", | 20260505 16:37:16.651 | INFO | "malloc" : " 14 GB", | 20260505 16:37:16.651 | INFO | "pinned" : " 29 GB" | 20260505 16:37:16.651 | INFO | } | 20260505 16:37:16.652 | INFO | Perf report for the last 5.14735 seconds: | 20260505 16:37:16.652 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:37:16.652 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:16.652 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:16.652 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:16.652 | INFO | ANALYSIS : 62.4%, 3.0, 97.6ms, 0 | 20260505 16:37:16.652 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:16.652 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:16.652 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 16:37:16.652 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:16.652 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:16.652 | INFO | Uploaded 36GiB to GPU in 2.14256s (16.8023GiB/s) | 20260505 16:37:16.652 | INFO | Downloaded 34GiB to GPU in 2.04504s (16.6256GiB/s) | 20260505 16:37:21.777 | INFO | Analyzing chunk frames = [4697088,4697600) | 20260505 16:37:21.777 | INFO | MemInfo: { | 20260505 16:37:21.777 | INFO | "gpu" : " 17 GB", | 20260505 16:37:21.777 | INFO | "huge" : "", | 20260505 16:37:21.777 | INFO | "huge_shared" : " 27 GB", | 20260505 16:37:21.777 | INFO | "malloc" : " 14 GB", | 20260505 16:37:21.777 | INFO | "pinned" : " 29 GB" | 20260505 16:37:21.777 | INFO | } | 20260505 16:37:21.777 | INFO | Perf report for the last 5.12529 seconds: | 20260505 16:37:21.777 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:37:21.777 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:21.777 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:21.777 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:21.777 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 16:37:21.777 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:21.777 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:21.777 | INFO | PRE_HQ : 28.7%, 1.0, 14.8ms, 0 | 20260505 16:37:21.777 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:21.777 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:21.777 | INFO | Uploaded 36GiB to GPU in 2.12038s (16.9781GiB/s) | 20260505 16:37:21.778 | INFO | Downloaded 34GiB to GPU in 2.13815s (15.9016GiB/s) | 20260505 16:37:26.908 | INFO | Analyzing chunk frames = [4697600,4698112) | 20260505 16:37:26.908 | INFO | MemInfo: { | 20260505 16:37:26.908 | INFO | "gpu" : " 17 GB", | 20260505 16:37:26.909 | INFO | "huge" : "", | 20260505 16:37:26.909 | INFO | "huge_shared" : " 27 GB", | 20260505 16:37:26.909 | INFO | "malloc" : " 14 GB", | 20260505 16:37:26.909 | INFO | "pinned" : " 29 GB" | 20260505 16:37:26.909 | INFO | } | 20260505 16:37:26.909 | INFO | Perf report for the last 5.13193 seconds: | 20260505 16:37:26.909 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:37:26.909 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:26.909 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:26.909 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:26.909 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 16:37:26.909 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:26.909 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:26.909 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:37:26.909 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:26.909 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:26.909 | INFO | Uploaded 36GiB to GPU in 2.15102s (16.7362GiB/s) | 20260505 16:37:26.909 | INFO | Downloaded 34GiB to GPU in 2.028s (16.7653GiB/s) | 20260505 16:37:27.043 | INFO | Starting a round of Baz IO | 20260505 16:37:30.253 | INFO | Populating RTMetrics report | 20260505 16:37:30.253 | INFO | RTMetrics report populated | 20260505 16:37:30.253 | INFO | Opening tmp RTMetrics json file | 20260505 16:37:30.257 | INFO | Serializing RTMetrics to json | 20260505 16:37:30.258 | INFO | Serialization done, swapping tmp file now... | 20260505 16:37:30.259 | INFO | RTMetrics done | 20260505 16:37:30.361 | INFO | Finished a round of Baz IO: Wrote 2.166 GiB in 3.318 seconds | 20260505 16:37:30.361 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.21% / 0.00% / 77.79% | 20260505 16:37:30.361 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.352816% / 0% / 9.53674e-05% | 20260505 16:37:30.362 | INFO | Baz IO overall: | 20260505 16:37:30.362 | INFO | BytesWritten : 2786038233101 | 20260505 16:37:30.362 | INFO | NumEvents : 1322181352886 | 20260505 16:37:32.054 | INFO | Analyzing chunk frames = [4698112,4698624) | 20260505 16:37:32.054 | INFO | MemInfo: { | 20260505 16:37:32.054 | INFO | "gpu" : " 17 GB", | 20260505 16:37:32.054 | INFO | "huge" : "", | 20260505 16:37:32.054 | INFO | "huge_shared" : " 27 GB", | 20260505 16:37:32.054 | INFO | "malloc" : " 14 GB", | 20260505 16:37:32.054 | INFO | "pinned" : " 29 GB" | 20260505 16:37:32.054 | INFO | } | 20260505 16:37:32.054 | INFO | Perf report for the last 5.1455 seconds: | 20260505 16:37:32.054 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:37:32.054 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:32.054 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:32.054 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:32.054 | INFO | ANALYSIS : 63.9%, 3.0, 99.5ms, 0 | 20260505 16:37:32.054 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:32.054 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260505 16:37:32.054 | INFO | PRE_HQ : 60.5%, 1.8, 56.7ms, 0 | 20260505 16:37:32.054 | INFO | BAZWRITER : 63.9%, 3.9, 127.7ms, 0 | 20260505 16:37:32.054 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 16:37:32.055 | INFO | Uploaded 36GiB to GPU in 2.19171s (16.4255GiB/s) | 20260505 16:37:32.055 | INFO | Downloaded 38GiB to GPU in 2.26417s (16.7832GiB/s) | 20260505 16:37:37.200 | INFO | Analyzing chunk frames = [4698624,4699136) | 20260505 16:37:37.200 | INFO | MemInfo: { | 20260505 16:37:37.200 | INFO | "gpu" : " 17 GB", | 20260505 16:37:37.200 | INFO | "huge" : "", | 20260505 16:37:37.200 | INFO | "huge_shared" : " 27 GB", | 20260505 16:37:37.200 | INFO | "malloc" : " 14 GB", | 20260505 16:37:37.201 | INFO | "pinned" : " 29 GB" | 20260505 16:37:37.201 | INFO | } | 20260505 16:37:37.201 | INFO | Perf report for the last 5.14648 seconds: | 20260505 16:37:37.201 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:37:37.201 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:37.201 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:37.201 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:37.201 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 16:37:37.201 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:37.201 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:37.201 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:37:37.201 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:37.201 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:37.201 | INFO | Uploaded 36GiB to GPU in 2.13373s (16.8718GiB/s) | 20260505 16:37:37.201 | INFO | Downloaded 34GiB to GPU in 2.04515s (16.6247GiB/s) | 20260505 16:37:42.342 | INFO | Analyzing chunk frames = [4699136,4699648) | 20260505 16:37:42.342 | INFO | MemInfo: { | 20260505 16:37:42.342 | INFO | "gpu" : " 17 GB", | 20260505 16:37:42.342 | INFO | "huge" : "", | 20260505 16:37:42.342 | INFO | "huge_shared" : " 27 GB", | 20260505 16:37:42.342 | INFO | "malloc" : " 14 GB", | 20260505 16:37:42.342 | INFO | "pinned" : " 29 GB" | 20260505 16:37:42.342 | INFO | } | 20260505 16:37:42.342 | INFO | Perf report for the last 5.14167 seconds: | 20260505 16:37:42.342 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:37:42.342 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:42.342 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:42.342 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:42.342 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 16:37:42.342 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:42.343 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:42.343 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 16:37:42.343 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:42.343 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:42.343 | INFO | Uploaded 36GiB to GPU in 2.20559s (16.3222GiB/s) | 20260505 16:37:42.343 | INFO | Downloaded 34GiB to GPU in 2.0608s (16.4985GiB/s) | 20260505 16:37:47.479 | INFO | Analyzing chunk frames = [4699648,4700160) | 20260505 16:37:47.479 | INFO | MemInfo: { | 20260505 16:37:47.479 | INFO | "gpu" : " 17 GB", | 20260505 16:37:47.479 | INFO | "huge" : "", | 20260505 16:37:47.479 | INFO | "huge_shared" : " 27 GB", | 20260505 16:37:47.479 | INFO | "malloc" : " 14 GB", | 20260505 16:37:47.479 | INFO | "pinned" : " 29 GB" | 20260505 16:37:47.479 | INFO | } | 20260505 16:37:47.479 | INFO | Perf report for the last 5.13664 seconds: | 20260505 16:37:47.479 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:37:47.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:47.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:47.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:47.479 | INFO | ANALYSIS : 62.6%, 3.0, 97.2ms, 0 | 20260505 16:37:47.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:47.479 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:47.479 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:37:47.479 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:47.479 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:47.479 | INFO | Uploaded 36GiB to GPU in 2.18685s (16.462GiB/s) | 20260505 16:37:47.479 | INFO | Downloaded 34GiB to GPU in 2.13909s (15.8946GiB/s) | 20260505 16:37:52.609 | INFO | Analyzing chunk frames = [4700160,4700672) | 20260505 16:37:52.609 | INFO | MemInfo: { | 20260505 16:37:52.609 | INFO | "gpu" : " 17 GB", | 20260505 16:37:52.609 | INFO | "huge" : "", | 20260505 16:37:52.609 | INFO | "huge_shared" : " 27 GB", | 20260505 16:37:52.609 | INFO | "malloc" : " 14 GB", | 20260505 16:37:52.609 | INFO | "pinned" : " 29 GB" | 20260505 16:37:52.609 | INFO | } | 20260505 16:37:52.609 | INFO | Perf report for the last 5.13026 seconds: | 20260505 16:37:52.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:37:52.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:52.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:52.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:52.609 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260505 16:37:52.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:52.609 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:52.609 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:37:52.609 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:52.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:52.610 | INFO | Uploaded 36GiB to GPU in 2.14318s (16.7975GiB/s) | 20260505 16:37:52.610 | INFO | Downloaded 34GiB to GPU in 2.05448s (16.5492GiB/s) | 20260505 16:37:57.754 | INFO | Analyzing chunk frames = [4700672,4701184) | 20260505 16:37:57.754 | INFO | MemInfo: { | 20260505 16:37:57.754 | INFO | "gpu" : " 17 GB", | 20260505 16:37:57.754 | INFO | "huge" : "", | 20260505 16:37:57.754 | INFO | "huge_shared" : " 27 GB", | 20260505 16:37:57.754 | INFO | "malloc" : " 14 GB", | 20260505 16:37:57.754 | INFO | "pinned" : " 29 GB" | 20260505 16:37:57.754 | INFO | } | 20260505 16:37:57.754 | INFO | Perf report for the last 5.14463 seconds: | 20260505 16:37:57.754 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:37:57.754 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:57.754 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:57.754 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:57.754 | INFO | ANALYSIS : 62.5%, 3.0, 97.7ms, 0 | 20260505 16:37:57.754 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:57.754 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:57.754 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 16:37:57.754 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:57.754 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:37:57.754 | INFO | Uploaded 36GiB to GPU in 2.14437s (16.7881GiB/s) | 20260505 16:37:57.754 | INFO | Downloaded 34GiB to GPU in 2.03664s (16.6942GiB/s) | 20260505 16:38:02.899 | INFO | Analyzing chunk frames = [4701184,4701696) | 20260505 16:38:02.899 | INFO | MemInfo: { | 20260505 16:38:02.899 | INFO | "gpu" : " 17 GB", | 20260505 16:38:02.899 | INFO | "huge" : "", | 20260505 16:38:02.899 | INFO | "huge_shared" : " 27 GB", | 20260505 16:38:02.899 | INFO | "malloc" : " 14 GB", | 20260505 16:38:02.899 | INFO | "pinned" : " 29 GB" | 20260505 16:38:02.899 | INFO | } | 20260505 16:38:02.899 | INFO | Perf report for the last 5.14526 seconds: | 20260505 16:38:02.899 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:38:02.899 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:02.899 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:02.899 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:02.899 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 16:38:02.899 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:02.899 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:02.899 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 16:38:02.899 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:02.899 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:02.900 | INFO | Uploaded 36GiB to GPU in 2.16376s (16.6377GiB/s) | 20260505 16:38:02.900 | INFO | Downloaded 34GiB to GPU in 2.17637s (15.6224GiB/s) | 20260505 16:38:08.034 | INFO | Analyzing chunk frames = [4701696,4702208) | 20260505 16:38:08.034 | INFO | MemInfo: { | 20260505 16:38:08.034 | INFO | "gpu" : " 17 GB", | 20260505 16:38:08.034 | INFO | "huge" : "", | 20260505 16:38:08.034 | INFO | "huge_shared" : " 27 GB", | 20260505 16:38:08.034 | INFO | "malloc" : " 14 GB", | 20260505 16:38:08.034 | INFO | "pinned" : " 29 GB" | 20260505 16:38:08.035 | INFO | } | 20260505 16:38:08.035 | INFO | Perf report for the last 5.1353 seconds: | 20260505 16:38:08.035 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:38:08.035 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:08.035 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:08.035 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:08.035 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 16:38:08.035 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:08.035 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:08.035 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:38:08.035 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:08.035 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:08.035 | INFO | Uploaded 36GiB to GPU in 2.12196s (16.9655GiB/s) | 20260505 16:38:08.035 | INFO | Downloaded 34GiB to GPU in 2.00713s (16.9396GiB/s) | 20260505 16:38:08.035 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 16:38:08.035 | INFO | Total mem usage: 15.3 GiB | 20260505 16:38:08.035 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 16:38:08.035 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.9%) | 20260505 16:38:08.035 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 16:38:08.035 | INFO | High Water Breakdown: | 20260505 16:38:08.035 | INFO | 61% : 8.67 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 16:38:08.035 | INFO | 38.8% : 5.51 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 16:38:08.035 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 16:38:08.035 | INFO | ------------------------------------------ | 20260505 16:38:08.035 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:38:08.035 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 16:38:08.035 | INFO | Total mem usage: 26.5 GiB | 20260505 16:38:08.035 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 16:38:08.035 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 16:38:08.035 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260505 16:38:08.035 | INFO | High Water Breakdown: | 20260505 16:38:08.035 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:38:08.035 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:38:08.035 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:38:08.035 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:38:08.035 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:38:08.035 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:38:08.035 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 16:38:08.035 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 16:38:08.035 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:38:08.035 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 16:38:08.035 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 16:38:08.035 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 16:38:08.035 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 16:38:08.036 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 16:38:08.036 | INFO | ------------------------------------------ | 20260505 16:38:08.036 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:38:08.036 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 16:38:08.036 | INFO | Total mem usage: 25.6 GiB | 20260505 16:38:08.036 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 16:38:08.036 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 16:38:08.036 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 16:38:08.036 | INFO | High Water Breakdown: | 20260505 16:38:08.036 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 16:38:08.036 | INFO | ------------------------------------------ | 20260505 16:38:08.036 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:38:08.036 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 16:38:08.036 | INFO | Total mem usage: 15.7 GiB | 20260505 16:38:08.036 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 16:38:08.036 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 16:38:08.036 | INFO | Unused Cache: 871 MiB (Waste: 5.43%) | 20260505 16:38:08.036 | INFO | High Water Breakdown: | 20260505 16:38:08.036 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 16:38:08.036 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 16:38:08.036 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 16:38:08.036 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 16:38:08.036 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 16:38:08.036 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 16:38:08.036 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 16:38:08.036 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 16:38:08.036 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 16:38:08.036 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:38:08.036 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 16:38:08.036 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:38:08.036 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 16:38:08.036 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 16:38:08.036 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 16:38:08.036 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 16:38:08.036 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:38:08.036 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 16:38:08.036 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:38:08.036 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:38:08.036 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:38:08.036 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 16:38:08.036 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 16:38:08.036 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:38:08.036 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 16:38:08.036 | INFO | ------------------------------------------ | 20260505 16:38:08.036 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:38:08.036 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 16:38:08.036 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:38:08.036 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:38:08.036 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.54e+01, 1.37e+01 / 2.41e+00 | 20260505 16:38:08.036 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 1.02e-01, 1.09e-02 / 9.31e-03 | 20260505 16:38:08.036 | INFO | PerfReport: 11.77%, Baseline -- 3.73e+00 / 4.54e+00, 5.80e-01 / 1.32e-01 | 20260505 16:38:08.036 | INFO | PerfReport: 3.27%, Histogram -- 1.04e+00 / 1.50e+00, 1.91e-01 / 8.81e-02 | 20260505 16:38:08.036 | INFO | PerfReport: 16.01%, DME -- 4.06e+01 / 4.54e+01, 4.49e+00 / 2.51e+00 | 20260505 16:38:08.036 | INFO | PerfReport: 56.72%, FrameLabeling -- 1.80e+01 / 1.90e+01, 1.49e+00 / 1.11e+00 | 20260505 16:38:08.036 | INFO | PerfReport: 7.35%, PulseAccumulating -- 2.33e+00 / 2.94e+00, 4.50e-01 / 1.49e-01 | 20260505 16:38:08.036 | INFO | PerfReport: 3.40%, Metrics -- 1.08e+00 / 1.43e+00, 2.98e-01 / 1.51e-01 | 20260505 16:38:08.036 | INFO | PerfReport: 1.19%, PulseSerializing -- 3.76e-01 / 4.35e-01, 6.62e-02 / 3.81e-02 | 20260505 16:38:08.036 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 16:38:08.036 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:38:08.036 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:38:08.036 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.59e+01 / 0.00e+00 | 20260505 16:38:08.036 | INFO | PerfReport: 22.46%, Upload -- 2.25e+01 / 0.00e+00, 7.85e+00 / 0.00e+00 | 20260505 16:38:08.036 | INFO | PerfReport: 21.84%, Download -- 2.19e+01 / 0.00e+00, 7.89e+00 / 0.00e+00 | 20260505 16:38:08.036 | INFO | PerfReport: 31.65%, Compute -- 3.17e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 16:38:08.036 | INFO | PerfReport: 2.25%, LockUpload -- 2.25e+00 / 0.00e+00, 7.94e+00 / 0.00e+00 | 20260505 16:38:08.036 | INFO | PerfReport: 1.44%, LockDownload -- 1.45e+00 / 0.00e+00, 3.00e+00 / 0.00e+00 | 20260505 16:38:08.036 | INFO | PerfReport: 20.34%, LockCompute -- 2.04e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 16:38:08.036 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.28e-02 / 0.00e+00 | 20260505 16:38:08.167 | INFO | Starting a round of Baz IO | 20260505 16:38:11.363 | INFO | Populating RTMetrics report | 20260505 16:38:11.364 | INFO | RTMetrics report populated | 20260505 16:38:11.364 | INFO | Opening tmp RTMetrics json file | 20260505 16:38:11.412 | INFO | Serializing RTMetrics to json | 20260505 16:38:11.413 | INFO | Serialization done, swapping tmp file now... | 20260505 16:38:11.413 | INFO | RTMetrics done | 20260505 16:38:11.753 | INFO | Finished a round of Baz IO: Wrote 2.163 GiB in 3.585 seconds | 20260505 16:38:11.753 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.21% / 0.00% / 77.79% | 20260505 16:38:11.753 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.351795% / 0% / 9.13938e-05% | 20260505 16:38:11.753 | INFO | Baz IO overall: | 20260505 16:38:11.753 | INFO | BytesWritten : 2788360566337 | 20260505 16:38:11.753 | INFO | NumEvents : 1323269421299 | 20260505 16:38:13.170 | INFO | Analyzing chunk frames = [4702208,4702720) | 20260505 16:38:13.170 | INFO | MemInfo: { | 20260505 16:38:13.170 | INFO | "gpu" : " 17 GB", | 20260505 16:38:13.170 | INFO | "huge" : "", | 20260505 16:38:13.170 | INFO | "huge_shared" : " 27 GB", | 20260505 16:38:13.170 | INFO | "malloc" : " 14 GB", | 20260505 16:38:13.170 | INFO | "pinned" : " 29 GB" | 20260505 16:38:13.170 | INFO | } | 20260505 16:38:13.170 | INFO | Perf report for the last 5.13567 seconds: | 20260505 16:38:13.170 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:38:13.170 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:13.170 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:13.170 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:13.170 | INFO | ANALYSIS : 63.7%, 2.9, 97.9ms, 0 | 20260505 16:38:13.170 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:13.170 | INFO | RT_METRICS : 45.5%, 1.0, 23.8ms, 0 | 20260505 16:38:13.170 | INFO | PRE_HQ : 60.5%, 1.8, 55.8ms, 0 | 20260505 16:38:13.170 | INFO | BAZWRITER : 69.1%, 4.2, 150.6ms, 0 | 20260505 16:38:13.170 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260505 16:38:13.171 | INFO | Uploaded 36GiB to GPU in 2.18125s (16.5043GiB/s) | 20260505 16:38:13.171 | INFO | Downloaded 38GiB to GPU in 2.2766s (16.6916GiB/s) | 20260505 16:38:18.315 | INFO | Analyzing chunk frames = [4702720,4703232) | 20260505 16:38:18.315 | INFO | MemInfo: { | 20260505 16:38:18.315 | INFO | "gpu" : " 17 GB", | 20260505 16:38:18.315 | INFO | "huge" : "", | 20260505 16:38:18.315 | INFO | "huge_shared" : " 27 GB", | 20260505 16:38:18.315 | INFO | "malloc" : " 14 GB", | 20260505 16:38:18.315 | INFO | "pinned" : " 29 GB" | 20260505 16:38:18.315 | INFO | } | 20260505 16:38:18.315 | INFO | Perf report for the last 5.14489 seconds: | 20260505 16:38:18.315 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:38:18.315 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:18.315 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:18.315 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:18.315 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 16:38:18.315 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:18.315 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:18.315 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:38:18.315 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:18.315 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:18.315 | INFO | Uploaded 36GiB to GPU in 2.13167s (16.8882GiB/s) | 20260505 16:38:18.315 | INFO | Downloaded 34GiB to GPU in 2.03346s (16.7203GiB/s) | 20260505 16:38:23.454 | INFO | Analyzing chunk frames = [4703232,4703744) | 20260505 16:38:23.454 | INFO | MemInfo: { | 20260505 16:38:23.454 | INFO | "gpu" : " 17 GB", | 20260505 16:38:23.454 | INFO | "huge" : "", | 20260505 16:38:23.454 | INFO | "huge_shared" : " 27 GB", | 20260505 16:38:23.454 | INFO | "malloc" : " 14 GB", | 20260505 16:38:23.454 | INFO | "pinned" : " 29 GB" | 20260505 16:38:23.454 | INFO | } | 20260505 16:38:23.454 | INFO | Perf report for the last 5.1388 seconds: | 20260505 16:38:23.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:38:23.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:23.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:23.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:23.454 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260505 16:38:23.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:23.454 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:23.454 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:38:23.454 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:23.454 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:23.454 | INFO | Uploaded 36GiB to GPU in 2.19798s (16.3787GiB/s) | 20260505 16:38:23.454 | INFO | Downloaded 34GiB to GPU in 2.03831s (16.6805GiB/s) | 20260505 16:38:28.593 | INFO | Analyzing chunk frames = [4703744,4704256) | 20260505 16:38:28.593 | INFO | MemInfo: { | 20260505 16:38:28.593 | INFO | "gpu" : " 17 GB", | 20260505 16:38:28.593 | INFO | "huge" : "", | 20260505 16:38:28.593 | INFO | "huge_shared" : " 27 GB", | 20260505 16:38:28.593 | INFO | "malloc" : " 14 GB", | 20260505 16:38:28.593 | INFO | "pinned" : " 29 GB" | 20260505 16:38:28.593 | INFO | } | 20260505 16:38:28.593 | INFO | Perf report for the last 5.13864 seconds: | 20260505 16:38:28.593 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:38:28.593 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:28.593 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:28.593 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:28.593 | INFO | ANALYSIS : 62.6%, 3.0, 97.2ms, 0 | 20260505 16:38:28.593 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:28.593 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:28.593 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:38:28.593 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:28.593 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:28.593 | INFO | Uploaded 36GiB to GPU in 2.18815s (16.4522GiB/s) | 20260505 16:38:28.593 | INFO | Downloaded 34GiB to GPU in 2.13781s (15.9041GiB/s) | 20260505 16:38:33.729 | INFO | Analyzing chunk frames = [4704256,4704768) | 20260505 16:38:33.729 | INFO | MemInfo: { | 20260505 16:38:33.729 | INFO | "gpu" : " 17 GB", | 20260505 16:38:33.729 | INFO | "huge" : "", | 20260505 16:38:33.729 | INFO | "huge_shared" : " 27 GB", | 20260505 16:38:33.729 | INFO | "malloc" : " 14 GB", | 20260505 16:38:33.729 | INFO | "pinned" : " 29 GB" | 20260505 16:38:33.729 | INFO | } | 20260505 16:38:33.730 | INFO | Perf report for the last 5.13686 seconds: | 20260505 16:38:33.730 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:38:33.730 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:33.730 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:33.730 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:33.730 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260505 16:38:33.730 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:33.730 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:33.730 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:38:33.730 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:33.730 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:33.730 | INFO | Uploaded 36GiB to GPU in 2.16452s (16.6318GiB/s) | 20260505 16:38:33.730 | INFO | Downloaded 34GiB to GPU in 2.06242s (16.4855GiB/s) | 20260505 16:38:38.862 | INFO | Analyzing chunk frames = [4704768,4705280) | 20260505 16:38:38.862 | INFO | MemInfo: { | 20260505 16:38:38.862 | INFO | "gpu" : " 17 GB", | 20260505 16:38:38.862 | INFO | "huge" : "", | 20260505 16:38:38.862 | INFO | "huge_shared" : " 27 GB", | 20260505 16:38:38.862 | INFO | "malloc" : " 14 GB", | 20260505 16:38:38.863 | INFO | "pinned" : " 29 GB" | 20260505 16:38:38.863 | INFO | } | 20260505 16:38:38.863 | INFO | Perf report for the last 5.13339 seconds: | 20260505 16:38:38.863 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:38:38.863 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:38.863 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:38.863 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:38.863 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 16:38:38.863 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:38.863 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:38.863 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 16:38:38.863 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:38.863 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:38.863 | INFO | Uploaded 36GiB to GPU in 2.1382s (16.8366GiB/s) | 20260505 16:38:38.863 | INFO | Downloaded 34GiB to GPU in 2.03851s (16.6789GiB/s) | 20260505 16:38:44.010 | INFO | Analyzing chunk frames = [4705280,4705792) | 20260505 16:38:44.010 | INFO | MemInfo: { | 20260505 16:38:44.010 | INFO | "gpu" : " 17 GB", | 20260505 16:38:44.010 | INFO | "huge" : "", | 20260505 16:38:44.010 | INFO | "huge_shared" : " 27 GB", | 20260505 16:38:44.011 | INFO | "malloc" : " 14 GB", | 20260505 16:38:44.011 | INFO | "pinned" : " 29 GB" | 20260505 16:38:44.011 | INFO | } | 20260505 16:38:44.011 | INFO | Perf report for the last 5.1477 seconds: | 20260505 16:38:44.011 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:38:44.011 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:44.011 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:44.011 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:44.011 | INFO | ANALYSIS : 63.3%, 3.0, 99.0ms, 0 | 20260505 16:38:44.011 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:44.011 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:44.011 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 16:38:44.011 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:44.011 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:44.011 | INFO | Uploaded 36GiB to GPU in 2.12108s (16.9725GiB/s) | 20260505 16:38:44.011 | INFO | Downloaded 34GiB to GPU in 2.16555s (15.7004GiB/s) | 20260505 16:38:49.161 | INFO | Analyzing chunk frames = [4705792,4706304) | 20260505 16:38:49.161 | INFO | MemInfo: { | 20260505 16:38:49.161 | INFO | "gpu" : " 17 GB", | 20260505 16:38:49.161 | INFO | "huge" : "", | 20260505 16:38:49.161 | INFO | "huge_shared" : " 27 GB", | 20260505 16:38:49.161 | INFO | "malloc" : " 14 GB", | 20260505 16:38:49.161 | INFO | "pinned" : " 29 GB" | 20260505 16:38:49.161 | INFO | } | 20260505 16:38:49.161 | INFO | Perf report for the last 5.15084 seconds: | 20260505 16:38:49.161 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:38:49.161 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:49.161 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:49.161 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:49.162 | INFO | ANALYSIS : 62.9%, 3.0, 98.4ms, 0 | 20260505 16:38:49.162 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:49.162 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:49.162 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 16:38:49.162 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:49.162 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:49.162 | INFO | Uploaded 36GiB to GPU in 2.19346s (16.4124GiB/s) | 20260505 16:38:49.162 | INFO | Downloaded 34GiB to GPU in 2.03509s (16.7068GiB/s) | 20260505 16:38:49.293 | INFO | Starting a round of Baz IO | 20260505 16:38:52.482 | INFO | Populating RTMetrics report | 20260505 16:38:52.482 | INFO | RTMetrics report populated | 20260505 16:38:52.482 | INFO | Opening tmp RTMetrics json file | 20260505 16:38:52.493 | INFO | Serializing RTMetrics to json | 20260505 16:38:52.494 | INFO | Serialization done, swapping tmp file now... | 20260505 16:38:52.495 | INFO | RTMetrics done | 20260505 16:38:52.579 | INFO | Finished a round of Baz IO: Wrote 2.164 GiB in 3.286 seconds | 20260505 16:38:52.579 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.20% / 0.00% / 77.79% | 20260505 16:38:52.579 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.350543% / 3.97364e-06% / 0.000107288% | 20260505 16:38:52.579 | INFO | Baz IO overall: | 20260505 16:38:52.579 | INFO | BytesWritten : 2790683690301 | 20260505 16:38:52.579 | INFO | NumEvents : 1324358331236 | 20260505 16:38:54.291 | INFO | Analyzing chunk frames = [4706304,4706816) | 20260505 16:38:54.291 | INFO | MemInfo: { | 20260505 16:38:54.291 | INFO | "gpu" : " 17 GB", | 20260505 16:38:54.291 | INFO | "huge" : "", | 20260505 16:38:54.291 | INFO | "huge_shared" : " 27 GB", | 20260505 16:38:54.291 | INFO | "malloc" : " 14 GB", | 20260505 16:38:54.291 | INFO | "pinned" : " 29 GB" | 20260505 16:38:54.291 | INFO | } | 20260505 16:38:54.291 | INFO | Perf report for the last 5.12982 seconds: | 20260505 16:38:54.291 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:38:54.291 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:54.291 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:54.291 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:54.291 | INFO | ANALYSIS : 63.6%, 3.0, 98.9ms, 0 | 20260505 16:38:54.291 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:54.291 | INFO | RT_METRICS : 45.0%, 1.0, 23.5ms, 0 | 20260505 16:38:54.291 | INFO | PRE_HQ : 60.7%, 1.8, 56.2ms, 0 | 20260505 16:38:54.291 | INFO | BAZWRITER : 63.7%, 3.9, 127.1ms, 0 | 20260505 16:38:54.291 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 16:38:54.292 | INFO | Uploaded 36GiB to GPU in 2.18139s (16.5032GiB/s) | 20260505 16:38:54.292 | INFO | Downloaded 38GiB to GPU in 2.2593s (16.8193GiB/s) | 20260505 16:38:59.425 | INFO | Analyzing chunk frames = [4706816,4707328) | 20260505 16:38:59.425 | INFO | MemInfo: { | 20260505 16:38:59.425 | INFO | "gpu" : " 17 GB", | 20260505 16:38:59.425 | INFO | "huge" : "", | 20260505 16:38:59.425 | INFO | "huge_shared" : " 27 GB", | 20260505 16:38:59.425 | INFO | "malloc" : " 14 GB", | 20260505 16:38:59.425 | INFO | "pinned" : " 29 GB" | 20260505 16:38:59.425 | INFO | } | 20260505 16:38:59.425 | INFO | Perf report for the last 5.13394 seconds: | 20260505 16:38:59.425 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:38:59.425 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:59.425 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:59.425 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:59.425 | INFO | ANALYSIS : 62.9%, 3.0, 98.0ms, 0 | 20260505 16:38:59.425 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:59.425 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:59.425 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:38:59.425 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:59.425 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:38:59.426 | INFO | Uploaded 36GiB to GPU in 2.1143s (17.0269GiB/s) | 20260505 16:38:59.426 | INFO | Downloaded 34GiB to GPU in 2.02942s (16.7535GiB/s) | 20260505 16:39:04.570 | INFO | Analyzing chunk frames = [4707328,4707840) | 20260505 16:39:04.570 | INFO | MemInfo: { | 20260505 16:39:04.570 | INFO | "gpu" : " 17 GB", | 20260505 16:39:04.570 | INFO | "huge" : "", | 20260505 16:39:04.570 | INFO | "huge_shared" : " 27 GB", | 20260505 16:39:04.570 | INFO | "malloc" : " 14 GB", | 20260505 16:39:04.570 | INFO | "pinned" : " 29 GB" | 20260505 16:39:04.570 | INFO | } | 20260505 16:39:04.570 | INFO | Perf report for the last 5.14476 seconds: | 20260505 16:39:04.570 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:39:04.570 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:04.570 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:04.570 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:04.570 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 16:39:04.570 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:04.570 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:04.570 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:39:04.570 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:04.570 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:04.570 | INFO | Uploaded 36GiB to GPU in 2.17982s (16.5151GiB/s) | 20260505 16:39:04.570 | INFO | Downloaded 34GiB to GPU in 2.02927s (16.7548GiB/s) | 20260505 16:39:09.706 | INFO | Analyzing chunk frames = [4707840,4708352) | 20260505 16:39:09.706 | INFO | MemInfo: { | 20260505 16:39:09.706 | INFO | "gpu" : " 17 GB", | 20260505 16:39:09.706 | INFO | "huge" : "", | 20260505 16:39:09.706 | INFO | "huge_shared" : " 27 GB", | 20260505 16:39:09.706 | INFO | "malloc" : " 14 GB", | 20260505 16:39:09.706 | INFO | "pinned" : " 29 GB" | 20260505 16:39:09.706 | INFO | } | 20260505 16:39:09.706 | INFO | Perf report for the last 5.13611 seconds: | 20260505 16:39:09.706 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:39:09.706 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:09.706 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:09.706 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:09.706 | INFO | ANALYSIS : 62.9%, 3.0, 97.6ms, 0 | 20260505 16:39:09.706 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:09.706 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:09.706 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:39:09.706 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:09.706 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:09.706 | INFO | Uploaded 36GiB to GPU in 2.22461s (16.1826GiB/s) | 20260505 16:39:09.706 | INFO | Downloaded 34GiB to GPU in 2.12523s (15.9982GiB/s) | 20260505 16:39:14.849 | INFO | Analyzing chunk frames = [4708352,4708864) | 20260505 16:39:14.849 | INFO | MemInfo: { | 20260505 16:39:14.849 | INFO | "gpu" : " 17 GB", | 20260505 16:39:14.849 | INFO | "huge" : "", | 20260505 16:39:14.849 | INFO | "huge_shared" : " 27 GB", | 20260505 16:39:14.849 | INFO | "malloc" : " 14 GB", | 20260505 16:39:14.849 | INFO | "pinned" : " 29 GB" | 20260505 16:39:14.849 | INFO | } | 20260505 16:39:14.849 | INFO | Perf report for the last 5.1433 seconds: | 20260505 16:39:14.849 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:39:14.849 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:14.849 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:14.850 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:14.850 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260505 16:39:14.850 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:14.850 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:14.850 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 16:39:14.850 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:14.850 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:14.850 | INFO | Uploaded 36GiB to GPU in 2.15089s (16.7373GiB/s) | 20260505 16:39:14.850 | INFO | Downloaded 34GiB to GPU in 2.06296s (16.4812GiB/s) | 20260505 16:39:19.980 | INFO | Analyzing chunk frames = [4708864,4709376) | 20260505 16:39:19.980 | INFO | MemInfo: { | 20260505 16:39:19.980 | INFO | "gpu" : " 17 GB", | 20260505 16:39:19.980 | INFO | "huge" : "", | 20260505 16:39:19.980 | INFO | "huge_shared" : " 27 GB", | 20260505 16:39:19.980 | INFO | "malloc" : " 14 GB", | 20260505 16:39:19.980 | INFO | "pinned" : " 29 GB" | 20260505 16:39:19.980 | INFO | } | 20260505 16:39:19.980 | INFO | Perf report for the last 5.13059 seconds: | 20260505 16:39:19.980 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:39:19.980 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:19.980 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:19.980 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:19.980 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260505 16:39:19.980 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:19.980 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:19.980 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:39:19.980 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:19.980 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:19.980 | INFO | Uploaded 36GiB to GPU in 2.13555s (16.8575GiB/s) | 20260505 16:39:19.980 | INFO | Downloaded 34GiB to GPU in 2.03697s (16.6915GiB/s) | 20260505 16:39:25.120 | INFO | Analyzing chunk frames = [4709376,4709888) | 20260505 16:39:25.120 | INFO | MemInfo: { | 20260505 16:39:25.120 | INFO | "gpu" : " 17 GB", | 20260505 16:39:25.120 | INFO | "huge" : "", | 20260505 16:39:25.120 | INFO | "huge_shared" : " 27 GB", | 20260505 16:39:25.120 | INFO | "malloc" : " 14 GB", | 20260505 16:39:25.120 | INFO | "pinned" : " 29 GB" | 20260505 16:39:25.120 | INFO | } | 20260505 16:39:25.120 | INFO | Perf report for the last 5.14018 seconds: | 20260505 16:39:25.120 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:39:25.120 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:25.120 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:25.120 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:25.120 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 16:39:25.120 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:25.120 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:25.120 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 16:39:25.120 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:25.120 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:25.121 | INFO | Uploaded 36GiB to GPU in 2.11191s (17.0462GiB/s) | 20260505 16:39:25.121 | INFO | Downloaded 34GiB to GPU in 2.17261s (15.6494GiB/s) | 20260505 16:39:30.265 | INFO | Analyzing chunk frames = [4709888,4710400) | 20260505 16:39:30.265 | INFO | MemInfo: { | 20260505 16:39:30.265 | INFO | "gpu" : " 17 GB", | 20260505 16:39:30.265 | INFO | "huge" : "", | 20260505 16:39:30.265 | INFO | "huge_shared" : " 27 GB", | 20260505 16:39:30.265 | INFO | "malloc" : " 14 GB", | 20260505 16:39:30.265 | INFO | "pinned" : " 29 GB" | 20260505 16:39:30.265 | INFO | } | 20260505 16:39:30.265 | INFO | Perf report for the last 5.14505 seconds: | 20260505 16:39:30.265 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:39:30.265 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:30.265 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:30.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:30.265 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 16:39:30.265 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:30.265 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:30.265 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 16:39:30.265 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:30.265 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:30.266 | INFO | Uploaded 36GiB to GPU in 2.16013s (16.6657GiB/s) | 20260505 16:39:30.266 | INFO | Downloaded 34GiB to GPU in 2.01953s (16.8356GiB/s) | 20260505 16:39:30.266 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 16:39:30.266 | INFO | Total mem usage: 15.3 GiB | 20260505 16:39:30.266 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 16:39:30.266 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -0.361%) | 20260505 16:39:30.266 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 16:39:30.266 | INFO | High Water Breakdown: | 20260505 16:39:30.266 | INFO | 62.1% : 9.48 GiB ( 99.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 16:39:30.266 | INFO | 37.8% : 5.77 GiB ( 86.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 16:39:30.266 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 16:39:30.266 | INFO | ------------------------------------------ | 20260505 16:39:30.266 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:39:30.266 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 16:39:30.266 | INFO | Total mem usage: 26.5 GiB | 20260505 16:39:30.266 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 16:39:30.266 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 16:39:30.266 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260505 16:39:30.266 | INFO | High Water Breakdown: | 20260505 16:39:30.266 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:39:30.266 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:39:30.266 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:39:30.266 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:39:30.266 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:39:30.266 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:39:30.266 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 16:39:30.266 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 16:39:30.266 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:39:30.266 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 16:39:30.266 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 16:39:30.266 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 16:39:30.266 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 16:39:30.266 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 16:39:30.266 | INFO | ------------------------------------------ | 20260505 16:39:30.266 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:39:30.266 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 16:39:30.266 | INFO | Total mem usage: 25.6 GiB | 20260505 16:39:30.266 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 16:39:30.266 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 16:39:30.266 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 16:39:30.266 | INFO | High Water Breakdown: | 20260505 16:39:30.266 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 16:39:30.266 | INFO | ------------------------------------------ | 20260505 16:39:30.266 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:39:30.266 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 16:39:30.267 | INFO | Total mem usage: 15.7 GiB | 20260505 16:39:30.267 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 16:39:30.267 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 16:39:30.267 | INFO | Unused Cache: 871 MiB (Waste: 5.43%) | 20260505 16:39:30.267 | INFO | High Water Breakdown: | 20260505 16:39:30.267 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 16:39:30.267 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 16:39:30.267 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 16:39:30.267 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 16:39:30.267 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 16:39:30.267 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 16:39:30.267 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 16:39:30.267 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 16:39:30.267 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 16:39:30.267 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:39:30.267 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 16:39:30.267 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:39:30.267 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 16:39:30.267 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 16:39:30.267 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 16:39:30.267 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 16:39:30.267 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:39:30.267 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 16:39:30.267 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:39:30.267 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:39:30.267 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:39:30.267 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 16:39:30.267 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 16:39:30.267 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:39:30.267 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 16:39:30.267 | INFO | ------------------------------------------ | 20260505 16:39:30.267 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:39:30.267 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 16:39:30.267 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:39:30.267 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:39:30.267 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.50e+01, 1.35e+01 / 2.10e+00 | 20260505 16:39:30.267 | INFO | PerfReport: 0.29%, AggregateStats -- 9.27e-02 / 1.01e-01, 1.03e-02 / 4.49e-03 | 20260505 16:39:30.267 | INFO | PerfReport: 11.80%, Baseline -- 3.73e+00 / 4.52e+00, 5.74e-01 / 1.60e-01 | 20260505 16:39:30.267 | INFO | PerfReport: 3.27%, Histogram -- 1.04e+00 / 1.50e+00, 1.91e-01 / 1.08e-01 | 20260505 16:39:30.267 | INFO | PerfReport: 15.95%, DME -- 4.04e+01 / 4.50e+01, 4.15e+00 / 1.97e+00 | 20260505 16:39:30.267 | INFO | PerfReport: 56.68%, FrameLabeling -- 1.79e+01 / 1.92e+01, 1.44e+00 / 1.32e+00 | 20260505 16:39:30.267 | INFO | PerfReport: 7.40%, PulseAccumulating -- 2.34e+00 / 2.94e+00, 4.52e-01 / 1.74e-01 | 20260505 16:39:30.267 | INFO | PerfReport: 3.42%, Metrics -- 1.08e+00 / 1.41e+00, 2.94e-01 / 1.55e-01 | 20260505 16:39:30.267 | INFO | PerfReport: 1.19%, PulseSerializing -- 3.77e-01 / 4.37e-01, 6.64e-02 / 4.00e-02 | 20260505 16:39:30.267 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 16:39:30.267 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:39:30.267 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:39:30.267 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260505 16:39:30.267 | INFO | PerfReport: 22.52%, Upload -- 2.25e+01 / 0.00e+00, 7.76e+00 / 0.00e+00 | 20260505 16:39:30.267 | INFO | PerfReport: 21.83%, Download -- 2.18e+01 / 0.00e+00, 7.77e+00 / 0.00e+00 | 20260505 16:39:30.267 | INFO | PerfReport: 31.65%, Compute -- 3.16e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 16:39:30.267 | INFO | PerfReport: 2.28%, LockUpload -- 2.28e+00 / 0.00e+00, 8.02e+00 / 0.00e+00 | 20260505 16:39:30.267 | INFO | PerfReport: 1.46%, LockDownload -- 1.46e+00 / 0.00e+00, 3.05e+00 / 0.00e+00 | 20260505 16:39:30.267 | INFO | PerfReport: 20.24%, LockCompute -- 2.02e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 16:39:30.267 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.60e-02 / 0.00e+00 | 20260505 16:39:30.403 | INFO | Starting a round of Baz IO | 20260505 16:39:33.589 | INFO | Populating RTMetrics report | 20260505 16:39:33.589 | INFO | RTMetrics report populated | 20260505 16:39:33.589 | INFO | Opening tmp RTMetrics json file | 20260505 16:39:33.603 | INFO | Serializing RTMetrics to json | 20260505 16:39:33.609 | INFO | Serialization done, swapping tmp file now... | 20260505 16:39:33.624 | INFO | RTMetrics done | 20260505 16:39:33.687 | INFO | Finished a round of Baz IO: Wrote 2.165 GiB in 3.284 seconds | 20260505 16:39:33.687 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.20% / 0.00% / 77.80% | 20260505 16:39:33.687 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.350237% / 7.94729e-06% / 7.15256e-05% | 20260505 16:39:33.687 | INFO | Baz IO overall: | 20260505 16:39:33.687 | INFO | BytesWritten : 2793007836009 | 20260505 16:39:33.687 | INFO | NumEvents : 1325447907430 | 20260505 16:39:35.407 | INFO | Analyzing chunk frames = [4710400,4710912) | 20260505 16:39:35.407 | INFO | MemInfo: { | 20260505 16:39:35.407 | INFO | "gpu" : " 17 GB", | 20260505 16:39:35.407 | INFO | "huge" : "", | 20260505 16:39:35.407 | INFO | "huge_shared" : " 27 GB", | 20260505 16:39:35.407 | INFO | "malloc" : " 14 GB", | 20260505 16:39:35.407 | INFO | "pinned" : " 29 GB" | 20260505 16:39:35.407 | INFO | } | 20260505 16:39:35.407 | INFO | Perf report for the last 5.14144 seconds: | 20260505 16:39:35.407 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:39:35.407 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:35.407 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:35.407 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:35.407 | INFO | ANALYSIS : 63.5%, 3.0, 99.0ms, 0 | 20260505 16:39:35.407 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:35.407 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260505 16:39:35.407 | INFO | PRE_HQ : 60.4%, 1.8, 56.4ms, 0 | 20260505 16:39:35.407 | INFO | BAZWRITER : 63.3%, 3.9, 127.2ms, 0 | 20260505 16:39:35.407 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 16:39:35.407 | INFO | Uploaded 36GiB to GPU in 2.15188s (16.7296GiB/s) | 20260505 16:39:35.407 | INFO | Downloaded 38GiB to GPU in 2.26016s (16.813GiB/s) | 20260505 16:39:40.538 | INFO | Analyzing chunk frames = [4710912,4711424) | 20260505 16:39:40.538 | INFO | MemInfo: { | 20260505 16:39:40.538 | INFO | "gpu" : " 17 GB", | 20260505 16:39:40.538 | INFO | "huge" : "", | 20260505 16:39:40.538 | INFO | "huge_shared" : " 27 GB", | 20260505 16:39:40.538 | INFO | "malloc" : " 14 GB", | 20260505 16:39:40.538 | INFO | "pinned" : " 29 GB" | 20260505 16:39:40.538 | INFO | } | 20260505 16:39:40.538 | INFO | Perf report for the last 5.1314 seconds: | 20260505 16:39:40.538 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:39:40.538 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:40.538 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:40.538 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:40.538 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260505 16:39:40.538 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:40.538 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:40.538 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:39:40.538 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:40.538 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:40.538 | INFO | Uploaded 36GiB to GPU in 2.13796s (16.8384GiB/s) | 20260505 16:39:40.539 | INFO | Downloaded 34GiB to GPU in 2.02847s (16.7614GiB/s) | 20260505 16:39:45.673 | INFO | Analyzing chunk frames = [4711424,4711936) | 20260505 16:39:45.673 | INFO | MemInfo: { | 20260505 16:39:45.673 | INFO | "gpu" : " 17 GB", | 20260505 16:39:45.673 | INFO | "huge" : "", | 20260505 16:39:45.673 | INFO | "huge_shared" : " 27 GB", | 20260505 16:39:45.673 | INFO | "malloc" : " 14 GB", | 20260505 16:39:45.673 | INFO | "pinned" : " 29 GB" | 20260505 16:39:45.673 | INFO | } | 20260505 16:39:45.674 | INFO | Perf report for the last 5.13535 seconds: | 20260505 16:39:45.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:39:45.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:45.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:45.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:45.674 | INFO | ANALYSIS : 62.8%, 3.0, 97.6ms, 0 | 20260505 16:39:45.674 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:45.674 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:45.674 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:39:45.674 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:45.674 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:45.674 | INFO | Uploaded 36GiB to GPU in 2.16918s (16.5962GiB/s) | 20260505 16:39:45.674 | INFO | Downloaded 34GiB to GPU in 2.03287s (16.7251GiB/s) | 20260505 16:39:50.818 | INFO | Analyzing chunk frames = [4711936,4712448) | 20260505 16:39:50.818 | INFO | MemInfo: { | 20260505 16:39:50.818 | INFO | "gpu" : " 17 GB", | 20260505 16:39:50.818 | INFO | "huge" : "", | 20260505 16:39:50.818 | INFO | "huge_shared" : " 27 GB", | 20260505 16:39:50.818 | INFO | "malloc" : " 14 GB", | 20260505 16:39:50.818 | INFO | "pinned" : " 29 GB" | 20260505 16:39:50.818 | INFO | } | 20260505 16:39:50.818 | INFO | Perf report for the last 5.14472 seconds: | 20260505 16:39:50.818 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:39:50.818 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:50.818 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:50.818 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:50.818 | INFO | ANALYSIS : 62.5%, 3.0, 97.2ms, 0 | 20260505 16:39:50.818 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:50.818 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:50.818 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:39:50.818 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:50.818 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:50.819 | INFO | Uploaded 36GiB to GPU in 2.16804s (16.6048GiB/s) | 20260505 16:39:50.819 | INFO | Downloaded 34GiB to GPU in 2.13297s (15.9402GiB/s) | 20260505 16:39:55.960 | INFO | Analyzing chunk frames = [4712448,4712960) | 20260505 16:39:55.960 | INFO | MemInfo: { | 20260505 16:39:55.960 | INFO | "gpu" : " 17 GB", | 20260505 16:39:55.960 | INFO | "huge" : "", | 20260505 16:39:55.960 | INFO | "huge_shared" : " 27 GB", | 20260505 16:39:55.960 | INFO | "malloc" : " 14 GB", | 20260505 16:39:55.960 | INFO | "pinned" : " 29 GB" | 20260505 16:39:55.960 | INFO | } | 20260505 16:39:55.960 | INFO | Perf report for the last 5.14193 seconds: | 20260505 16:39:55.960 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:39:55.960 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:55.960 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:55.960 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:55.960 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 16:39:55.960 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:55.960 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:55.960 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:39:55.960 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:55.960 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:39:55.960 | INFO | Uploaded 36GiB to GPU in 2.1482s (16.7582GiB/s) | 20260505 16:39:55.961 | INFO | Downloaded 34GiB to GPU in 2.05675s (16.5309GiB/s) | 20260505 16:40:01.102 | INFO | Analyzing chunk frames = [4712960,4713472) | 20260505 16:40:01.102 | INFO | MemInfo: { | 20260505 16:40:01.102 | INFO | "gpu" : " 17 GB", | 20260505 16:40:01.102 | INFO | "huge" : "", | 20260505 16:40:01.102 | INFO | "huge_shared" : " 27 GB", | 20260505 16:40:01.102 | INFO | "malloc" : " 14 GB", | 20260505 16:40:01.102 | INFO | "pinned" : " 29 GB" | 20260505 16:40:01.102 | INFO | } | 20260505 16:40:01.102 | INFO | Perf report for the last 5.14218 seconds: | 20260505 16:40:01.102 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:40:01.102 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:01.102 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:01.102 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:01.102 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260505 16:40:01.102 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:01.103 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:01.103 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:40:01.103 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:01.103 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:01.103 | INFO | Uploaded 36GiB to GPU in 2.14138s (16.8116GiB/s) | 20260505 16:40:01.103 | INFO | Downloaded 34GiB to GPU in 2.04392s (16.6347GiB/s) | 20260505 16:40:06.237 | INFO | Analyzing chunk frames = [4713472,4713984) | 20260505 16:40:06.237 | INFO | MemInfo: { | 20260505 16:40:06.237 | INFO | "gpu" : " 17 GB", | 20260505 16:40:06.237 | INFO | "huge" : "", | 20260505 16:40:06.237 | INFO | "huge_shared" : " 27 GB", | 20260505 16:40:06.237 | INFO | "malloc" : " 14 GB", | 20260505 16:40:06.237 | INFO | "pinned" : " 29 GB" | 20260505 16:40:06.237 | INFO | } | 20260505 16:40:06.237 | INFO | Perf report for the last 5.13446 seconds: | 20260505 16:40:06.237 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:40:06.237 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:06.237 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:06.237 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:06.237 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 16:40:06.237 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:06.237 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:06.237 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 16:40:06.237 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:06.237 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:06.237 | INFO | Uploaded 36GiB to GPU in 2.16134s (16.6563GiB/s) | 20260505 16:40:06.237 | INFO | Downloaded 34GiB to GPU in 2.17811s (15.6099GiB/s) | 20260505 16:40:11.373 | INFO | Analyzing chunk frames = [4713984,4714496) | 20260505 16:40:11.373 | INFO | MemInfo: { | 20260505 16:40:11.373 | INFO | "gpu" : " 17 GB", | 20260505 16:40:11.373 | INFO | "huge" : "", | 20260505 16:40:11.373 | INFO | "huge_shared" : " 27 GB", | 20260505 16:40:11.373 | INFO | "malloc" : " 14 GB", | 20260505 16:40:11.373 | INFO | "pinned" : " 29 GB" | 20260505 16:40:11.373 | INFO | } | 20260505 16:40:11.373 | INFO | Perf report for the last 5.13649 seconds: | 20260505 16:40:11.373 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:40:11.373 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:11.373 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:11.373 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:11.373 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 16:40:11.373 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:11.373 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:11.373 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 16:40:11.373 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:11.374 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:11.374 | INFO | Uploaded 36GiB to GPU in 2.16829s (16.6029GiB/s) | 20260505 16:40:11.374 | INFO | Downloaded 34GiB to GPU in 2.03488s (16.7086GiB/s) | 20260505 16:40:11.503 | INFO | Starting a round of Baz IO | 20260505 16:40:14.711 | INFO | Populating RTMetrics report | 20260505 16:40:14.712 | INFO | RTMetrics report populated | 20260505 16:40:14.712 | INFO | Opening tmp RTMetrics json file | 20260505 16:40:14.732 | INFO | Serializing RTMetrics to json | 20260505 16:40:14.773 | INFO | Serialization done, swapping tmp file now... | 20260505 16:40:14.794 | INFO | RTMetrics done | 20260505 16:40:14.812 | INFO | Finished a round of Baz IO: Wrote 2.163 GiB in 3.309 seconds | 20260505 16:40:14.812 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.20% / 0.00% / 77.80% | 20260505 16:40:14.812 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.348524% / 0% / 9.53674e-05% | 20260505 16:40:14.812 | INFO | Baz IO overall: | 20260505 16:40:14.812 | INFO | BytesWritten : 2795330371235 | 20260505 16:40:14.812 | INFO | NumEvents : 1326536203584 | 20260505 16:40:16.525 | INFO | Analyzing chunk frames = [4714496,4715008) | 20260505 16:40:16.525 | INFO | MemInfo: { | 20260505 16:40:16.525 | INFO | "gpu" : " 17 GB", | 20260505 16:40:16.525 | INFO | "huge" : "", | 20260505 16:40:16.526 | INFO | "huge_shared" : " 27 GB", | 20260505 16:40:16.526 | INFO | "malloc" : " 14 GB", | 20260505 16:40:16.526 | INFO | "pinned" : " 29 GB" | 20260505 16:40:16.526 | INFO | } | 20260505 16:40:16.526 | INFO | Perf report for the last 5.15233 seconds: | 20260505 16:40:16.526 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:40:16.526 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:16.526 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:16.526 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:16.526 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 16:40:16.526 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:16.526 | INFO | RT_METRICS : 46.2%, 1.0, 24.3ms, 0 | 20260505 16:40:16.526 | INFO | PRE_HQ : 60.8%, 1.8, 56.6ms, 0 | 20260505 16:40:16.526 | INFO | BAZWRITER : 63.6%, 3.9, 127.8ms, 0 | 20260505 16:40:16.526 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260505 16:40:16.526 | INFO | Uploaded 36GiB to GPU in 2.19431s (16.406GiB/s) | 20260505 16:40:16.526 | INFO | Downloaded 38GiB to GPU in 2.26427s (16.7825GiB/s) | 20260505 16:40:21.660 | INFO | Analyzing chunk frames = [4715008,4715520) | 20260505 16:40:21.660 | INFO | MemInfo: { | 20260505 16:40:21.660 | INFO | "gpu" : " 17 GB", | 20260505 16:40:21.660 | INFO | "huge" : "", | 20260505 16:40:21.660 | INFO | "huge_shared" : " 27 GB", | 20260505 16:40:21.660 | INFO | "malloc" : " 14 GB", | 20260505 16:40:21.660 | INFO | "pinned" : " 29 GB" | 20260505 16:40:21.660 | INFO | } | 20260505 16:40:21.660 | INFO | Perf report for the last 5.13433 seconds: | 20260505 16:40:21.660 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:40:21.660 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:21.660 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:21.660 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:21.660 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 16:40:21.660 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:21.660 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:21.660 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:40:21.660 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:21.660 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:21.660 | INFO | Uploaded 36GiB to GPU in 2.14137s (16.8116GiB/s) | 20260505 16:40:21.660 | INFO | Downloaded 34GiB to GPU in 2.02862s (16.7602GiB/s) | 20260505 16:40:26.795 | INFO | Analyzing chunk frames = [4715520,4716032) | 20260505 16:40:26.795 | INFO | MemInfo: { | 20260505 16:40:26.795 | INFO | "gpu" : " 17 GB", | 20260505 16:40:26.795 | INFO | "huge" : "", | 20260505 16:40:26.795 | INFO | "huge_shared" : " 27 GB", | 20260505 16:40:26.795 | INFO | "malloc" : " 14 GB", | 20260505 16:40:26.795 | INFO | "pinned" : " 29 GB" | 20260505 16:40:26.795 | INFO | } | 20260505 16:40:26.795 | INFO | Perf report for the last 5.13473 seconds: | 20260505 16:40:26.795 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:40:26.795 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:26.795 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:26.795 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:26.795 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260505 16:40:26.795 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:26.795 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:26.795 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:40:26.795 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:26.795 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:26.795 | INFO | Uploaded 36GiB to GPU in 2.17978s (16.5154GiB/s) | 20260505 16:40:26.795 | INFO | Downloaded 34GiB to GPU in 2.03122s (16.7387GiB/s) | 20260505 16:40:31.934 | INFO | Analyzing chunk frames = [4716032,4716544) | 20260505 16:40:31.934 | INFO | MemInfo: { | 20260505 16:40:31.934 | INFO | "gpu" : " 17 GB", | 20260505 16:40:31.934 | INFO | "huge" : "", | 20260505 16:40:31.934 | INFO | "huge_shared" : " 27 GB", | 20260505 16:40:31.934 | INFO | "malloc" : " 14 GB", | 20260505 16:40:31.934 | INFO | "pinned" : " 29 GB" | 20260505 16:40:31.934 | INFO | } | 20260505 16:40:31.934 | INFO | Perf report for the last 5.13944 seconds: | 20260505 16:40:31.934 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:40:31.934 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:31.934 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:31.934 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:31.934 | INFO | ANALYSIS : 62.4%, 3.0, 96.9ms, 0 | 20260505 16:40:31.934 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:31.934 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:31.934 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:40:31.934 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:31.934 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:31.935 | INFO | Uploaded 36GiB to GPU in 2.18043s (16.5105GiB/s) | 20260505 16:40:31.935 | INFO | Downloaded 34GiB to GPU in 2.13291s (15.9407GiB/s) | 20260505 16:40:37.076 | INFO | Analyzing chunk frames = [4716544,4717056) | 20260505 16:40:37.076 | INFO | MemInfo: { | 20260505 16:40:37.076 | INFO | "gpu" : " 17 GB", | 20260505 16:40:37.076 | INFO | "huge" : "", | 20260505 16:40:37.076 | INFO | "huge_shared" : " 27 GB", | 20260505 16:40:37.076 | INFO | "malloc" : " 14 GB", | 20260505 16:40:37.076 | INFO | "pinned" : " 29 GB" | 20260505 16:40:37.076 | INFO | } | 20260505 16:40:37.076 | INFO | Perf report for the last 5.14193 seconds: | 20260505 16:40:37.076 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:40:37.076 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:37.076 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:37.076 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:37.076 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 16:40:37.076 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:37.076 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:37.076 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:40:37.076 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:37.076 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:37.077 | INFO | Uploaded 36GiB to GPU in 2.15897s (16.6746GiB/s) | 20260505 16:40:37.077 | INFO | Downloaded 34GiB to GPU in 2.062s (16.4888GiB/s) | 20260505 16:40:42.217 | INFO | Analyzing chunk frames = [4717056,4717568) | 20260505 16:40:42.217 | INFO | MemInfo: { | 20260505 16:40:42.217 | INFO | "gpu" : " 17 GB", | 20260505 16:40:42.217 | INFO | "huge" : "", | 20260505 16:40:42.217 | INFO | "huge_shared" : " 27 GB", | 20260505 16:40:42.217 | INFO | "malloc" : " 14 GB", | 20260505 16:40:42.217 | INFO | "pinned" : " 29 GB" | 20260505 16:40:42.217 | INFO | } | 20260505 16:40:42.217 | INFO | Perf report for the last 5.1412 seconds: | 20260505 16:40:42.217 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:40:42.217 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:42.217 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:42.217 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:42.217 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 16:40:42.218 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:42.218 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:42.218 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 16:40:42.218 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:42.218 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:42.218 | INFO | Uploaded 36GiB to GPU in 2.13098s (16.8936GiB/s) | 20260505 16:40:42.218 | INFO | Downloaded 34GiB to GPU in 2.0438s (16.6357GiB/s) | 20260505 16:40:47.358 | INFO | Analyzing chunk frames = [4717568,4718080) | 20260505 16:40:47.358 | INFO | MemInfo: { | 20260505 16:40:47.358 | INFO | "gpu" : " 17 GB", | 20260505 16:40:47.358 | INFO | "huge" : "", | 20260505 16:40:47.358 | INFO | "huge_shared" : " 27 GB", | 20260505 16:40:47.358 | INFO | "malloc" : " 14 GB", | 20260505 16:40:47.358 | INFO | "pinned" : " 29 GB" | 20260505 16:40:47.358 | INFO | } | 20260505 16:40:47.358 | INFO | Perf report for the last 5.14078 seconds: | 20260505 16:40:47.358 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:40:47.358 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:47.358 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:47.358 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:47.358 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 16:40:47.358 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:47.358 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:47.358 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 16:40:47.358 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:47.358 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:47.358 | INFO | Uploaded 36GiB to GPU in 2.14481s (16.7847GiB/s) | 20260505 16:40:47.358 | INFO | Downloaded 34GiB to GPU in 2.1466s (15.839GiB/s) | 20260505 16:40:52.493 | INFO | Analyzing chunk frames = [4718080,4718592) | 20260505 16:40:52.493 | INFO | MemInfo: { | 20260505 16:40:52.493 | INFO | "gpu" : " 17 GB", | 20260505 16:40:52.493 | INFO | "huge" : "", | 20260505 16:40:52.493 | INFO | "huge_shared" : " 27 GB", | 20260505 16:40:52.493 | INFO | "malloc" : " 14 GB", | 20260505 16:40:52.493 | INFO | "pinned" : " 29 GB" | 20260505 16:40:52.493 | INFO | } | 20260505 16:40:52.493 | INFO | Perf report for the last 5.13491 seconds: | 20260505 16:40:52.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:40:52.493 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:52.493 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:52.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:52.493 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 16:40:52.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:52.493 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:52.493 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:40:52.493 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:52.493 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:52.493 | INFO | Uploaded 36GiB to GPU in 2.15758s (16.6854GiB/s) | 20260505 16:40:52.494 | INFO | Downloaded 34GiB to GPU in 2.02261s (16.81GiB/s) | 20260505 16:40:52.494 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 16:40:52.494 | INFO | Total mem usage: 15.3 GiB | 20260505 16:40:52.494 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 16:40:52.494 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.9%) | 20260505 16:40:52.494 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 16:40:52.494 | INFO | High Water Breakdown: | 20260505 16:40:52.494 | INFO | 61% : 8.67 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 16:40:52.494 | INFO | 38.8% : 5.51 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 16:40:52.494 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 16:40:52.494 | INFO | ------------------------------------------ | 20260505 16:40:52.494 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:40:52.494 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 16:40:52.494 | INFO | Total mem usage: 26.5 GiB | 20260505 16:40:52.494 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 16:40:52.494 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 16:40:52.494 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 16:40:52.494 | INFO | High Water Breakdown: | 20260505 16:40:52.494 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:40:52.494 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:40:52.494 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:40:52.494 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:40:52.494 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:40:52.494 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:40:52.494 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 16:40:52.494 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 16:40:52.494 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:40:52.494 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 16:40:52.494 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 16:40:52.494 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 16:40:52.494 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 16:40:52.494 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 16:40:52.494 | INFO | ------------------------------------------ | 20260505 16:40:52.494 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:40:52.494 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 16:40:52.494 | INFO | Total mem usage: 25.6 GiB | 20260505 16:40:52.494 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 16:40:52.494 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 16:40:52.494 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 16:40:52.494 | INFO | High Water Breakdown: | 20260505 16:40:52.494 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 16:40:52.494 | INFO | ------------------------------------------ | 20260505 16:40:52.494 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:40:52.494 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 16:40:52.494 | INFO | Total mem usage: 15.7 GiB | 20260505 16:40:52.494 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 16:40:52.494 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 16:40:52.494 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 16:40:52.494 | INFO | High Water Breakdown: | 20260505 16:40:52.494 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 16:40:52.494 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 16:40:52.494 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 16:40:52.494 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 16:40:52.494 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 16:40:52.494 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 16:40:52.494 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 16:40:52.495 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 16:40:52.495 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 16:40:52.495 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:40:52.495 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 16:40:52.495 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:40:52.495 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 16:40:52.495 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 16:40:52.495 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 16:40:52.495 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 16:40:52.495 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:40:52.495 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 16:40:52.495 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:40:52.495 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:40:52.495 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:40:52.495 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 16:40:52.495 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 16:40:52.495 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:40:52.495 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 16:40:52.495 | INFO | ------------------------------------------ | 20260505 16:40:52.495 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:40:52.495 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 16:40:52.495 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:40:52.495 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:40:52.495 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.50e+01, 1.36e+01 / 2.09e+00 | 20260505 16:40:52.495 | INFO | PerfReport: 0.29%, AggregateStats -- 9.30e-02 / 1.03e-01, 1.07e-02 / 5.18e-03 | 20260505 16:40:52.495 | INFO | PerfReport: 11.80%, Baseline -- 3.74e+00 / 4.51e+00, 5.79e-01 / 1.67e-01 | 20260505 16:40:52.495 | INFO | PerfReport: 3.29%, Histogram -- 1.04e+00 / 1.50e+00, 1.96e-01 / 1.02e-01 | 20260505 16:40:52.495 | INFO | PerfReport: 15.84%, DME -- 4.02e+01 / 4.43e+01, 4.08e+00 / 2.24e+00 | 20260505 16:40:52.495 | INFO | PerfReport: 56.87%, FrameLabeling -- 1.80e+01 / 1.99e+01, 1.55e+00 / 1.53e+00 | 20260505 16:40:52.495 | INFO | PerfReport: 7.32%, PulseAccumulating -- 2.32e+00 / 2.86e+00, 4.41e-01 / 2.24e-01 | 20260505 16:40:52.495 | INFO | PerfReport: 3.40%, Metrics -- 1.08e+00 / 1.38e+00, 2.98e-01 / 1.32e-01 | 20260505 16:40:52.495 | INFO | PerfReport: 1.18%, PulseSerializing -- 3.75e-01 / 4.31e-01, 6.59e-02 / 4.12e-02 | 20260505 16:40:52.495 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 16:40:52.495 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:40:52.495 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:40:52.495 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260505 16:40:52.495 | INFO | PerfReport: 22.47%, Upload -- 2.25e+01 / 0.00e+00, 7.77e+00 / 0.00e+00 | 20260505 16:40:52.495 | INFO | PerfReport: 21.80%, Download -- 2.18e+01 / 0.00e+00, 7.83e+00 / 0.00e+00 | 20260505 16:40:52.495 | INFO | PerfReport: 31.71%, Compute -- 3.17e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 16:40:52.495 | INFO | PerfReport: 2.13%, LockUpload -- 2.13e+00 / 0.00e+00, 7.80e+00 / 0.00e+00 | 20260505 16:40:52.495 | INFO | PerfReport: 1.45%, LockDownload -- 1.45e+00 / 0.00e+00, 3.06e+00 / 0.00e+00 | 20260505 16:40:52.495 | INFO | PerfReport: 20.42%, LockCompute -- 2.04e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 16:40:52.495 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.31e-02 / 0.00e+00 | 20260505 16:40:52.625 | INFO | Starting a round of Baz IO | 20260505 16:40:55.832 | INFO | Populating RTMetrics report | 20260505 16:40:55.832 | INFO | RTMetrics report populated | 20260505 16:40:55.832 | INFO | Opening tmp RTMetrics json file | 20260505 16:40:55.837 | INFO | Serializing RTMetrics to json | 20260505 16:40:55.840 | INFO | Serialization done, swapping tmp file now... | 20260505 16:40:55.841 | INFO | RTMetrics done | 20260505 16:40:55.933 | INFO | Finished a round of Baz IO: Wrote 2.164 GiB in 3.308 seconds | 20260505 16:40:55.933 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.20% / 0.00% / 77.80% | 20260505 16:40:55.933 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.348183% / 0% / 8.74201e-05% | 20260505 16:40:55.933 | INFO | Baz IO overall: | 20260505 16:40:55.933 | INFO | BytesWritten : 2797653518743 | 20260505 16:40:55.933 | INFO | NumEvents : 1327625177699 | 20260505 16:40:57.625 | INFO | Analyzing chunk frames = [4718592,4719104) | 20260505 16:40:57.625 | INFO | MemInfo: { | 20260505 16:40:57.625 | INFO | "gpu" : " 17 GB", | 20260505 16:40:57.625 | INFO | "huge" : "", | 20260505 16:40:57.625 | INFO | "huge_shared" : " 27 GB", | 20260505 16:40:57.625 | INFO | "malloc" : " 14 GB", | 20260505 16:40:57.625 | INFO | "pinned" : " 29 GB" | 20260505 16:40:57.625 | INFO | } | 20260505 16:40:57.626 | INFO | Perf report for the last 5.1324 seconds: | 20260505 16:40:57.626 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:40:57.626 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:57.626 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:57.626 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:57.626 | INFO | ANALYSIS : 64.0%, 3.0, 99.4ms, 0 | 20260505 16:40:57.626 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:40:57.626 | INFO | RT_METRICS : 44.7%, 1.0, 23.4ms, 0 | 20260505 16:40:57.626 | INFO | PRE_HQ : 60.6%, 1.8, 56.1ms, 0 | 20260505 16:40:57.626 | INFO | BAZWRITER : 63.8%, 3.9, 127.7ms, 0 | 20260505 16:40:57.626 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 16:40:57.626 | INFO | Uploaded 36GiB to GPU in 2.18103s (16.506GiB/s) | 20260505 16:40:57.626 | INFO | Downloaded 38GiB to GPU in 2.27214s (16.7244GiB/s) | 20260505 16:41:02.768 | INFO | Analyzing chunk frames = [4719104,4719616) | 20260505 16:41:02.768 | INFO | MemInfo: { | 20260505 16:41:02.768 | INFO | "gpu" : " 17 GB", | 20260505 16:41:02.768 | INFO | "huge" : "", | 20260505 16:41:02.768 | INFO | "huge_shared" : " 27 GB", | 20260505 16:41:02.768 | INFO | "malloc" : " 14 GB", | 20260505 16:41:02.768 | INFO | "pinned" : " 29 GB" | 20260505 16:41:02.768 | INFO | } | 20260505 16:41:02.768 | INFO | Perf report for the last 5.14228 seconds: | 20260505 16:41:02.768 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:41:02.768 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:02.768 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:02.768 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:02.768 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 16:41:02.768 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:02.768 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:02.768 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 16:41:02.768 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:02.768 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:02.768 | INFO | Uploaded 36GiB to GPU in 2.19301s (16.4158GiB/s) | 20260505 16:41:02.768 | INFO | Downloaded 34GiB to GPU in 2.04559s (16.6211GiB/s) | 20260505 16:41:07.915 | INFO | Analyzing chunk frames = [4719616,4720128) | 20260505 16:41:07.915 | INFO | MemInfo: { | 20260505 16:41:07.915 | INFO | "gpu" : " 17 GB", | 20260505 16:41:07.915 | INFO | "huge" : "", | 20260505 16:41:07.915 | INFO | "huge_shared" : " 27 GB", | 20260505 16:41:07.915 | INFO | "malloc" : " 14 GB", | 20260505 16:41:07.915 | INFO | "pinned" : " 29 GB" | 20260505 16:41:07.915 | INFO | } | 20260505 16:41:07.915 | INFO | Perf report for the last 5.1476 seconds: | 20260505 16:41:07.915 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:41:07.915 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:07.915 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:07.915 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:07.915 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260505 16:41:07.915 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:07.916 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:07.916 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:41:07.916 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:07.916 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:07.916 | INFO | Uploaded 36GiB to GPU in 2.19136s (16.4281GiB/s) | 20260505 16:41:07.916 | INFO | Downloaded 34GiB to GPU in 2.02838s (16.7621GiB/s) | 20260505 16:41:13.047 | INFO | Analyzing chunk frames = [4720128,4720640) | 20260505 16:41:13.047 | INFO | MemInfo: { | 20260505 16:41:13.047 | INFO | "gpu" : " 17 GB", | 20260505 16:41:13.047 | INFO | "huge" : "", | 20260505 16:41:13.047 | INFO | "huge_shared" : " 27 GB", | 20260505 16:41:13.047 | INFO | "malloc" : " 14 GB", | 20260505 16:41:13.047 | INFO | "pinned" : " 29 GB" | 20260505 16:41:13.048 | INFO | } | 20260505 16:41:13.048 | INFO | Perf report for the last 5.13211 seconds: | 20260505 16:41:13.048 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:41:13.048 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:13.048 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:13.048 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:13.048 | INFO | ANALYSIS : 62.6%, 3.0, 97.2ms, 0 | 20260505 16:41:13.048 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:13.048 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:13.048 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 16:41:13.048 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:13.048 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:13.048 | INFO | Uploaded 36GiB to GPU in 2.17548s (16.548GiB/s) | 20260505 16:41:13.048 | INFO | Downloaded 34GiB to GPU in 2.14147s (15.8769GiB/s) | 20260505 16:41:18.183 | INFO | Analyzing chunk frames = [4720640,4721152) | 20260505 16:41:18.183 | INFO | MemInfo: { | 20260505 16:41:18.183 | INFO | "gpu" : " 17 GB", | 20260505 16:41:18.183 | INFO | "huge" : "", | 20260505 16:41:18.184 | INFO | "huge_shared" : " 27 GB", | 20260505 16:41:18.184 | INFO | "malloc" : " 14 GB", | 20260505 16:41:18.184 | INFO | "pinned" : " 29 GB" | 20260505 16:41:18.184 | INFO | } | 20260505 16:41:18.184 | INFO | Perf report for the last 5.13604 seconds: | 20260505 16:41:18.184 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:41:18.184 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:18.184 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:18.184 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:18.184 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260505 16:41:18.184 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:18.184 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:18.184 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 16:41:18.184 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:18.184 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:18.184 | INFO | Uploaded 36GiB to GPU in 2.13726s (16.844GiB/s) | 20260505 16:41:18.184 | INFO | Downloaded 34GiB to GPU in 2.05935s (16.5101GiB/s) | 20260505 16:41:23.329 | INFO | Analyzing chunk frames = [4721152,4721664) | 20260505 16:41:23.329 | INFO | MemInfo: { | 20260505 16:41:23.329 | INFO | "gpu" : " 17 GB", | 20260505 16:41:23.329 | INFO | "huge" : "", | 20260505 16:41:23.329 | INFO | "huge_shared" : " 27 GB", | 20260505 16:41:23.329 | INFO | "malloc" : " 14 GB", | 20260505 16:41:23.329 | INFO | "pinned" : " 29 GB" | 20260505 16:41:23.329 | INFO | } | 20260505 16:41:23.329 | INFO | Perf report for the last 5.14528 seconds: | 20260505 16:41:23.329 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:41:23.329 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:23.329 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:23.329 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:23.329 | INFO | ANALYSIS : 62.4%, 3.0, 97.6ms, 0 | 20260505 16:41:23.329 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:23.329 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:23.329 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 16:41:23.329 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:23.329 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:23.329 | INFO | Uploaded 36GiB to GPU in 2.13583s (16.8553GiB/s) | 20260505 16:41:23.329 | INFO | Downloaded 34GiB to GPU in 2.04206s (16.6498GiB/s) | 20260505 16:41:28.472 | INFO | Analyzing chunk frames = [4721664,4722176) | 20260505 16:41:28.472 | INFO | MemInfo: { | 20260505 16:41:28.472 | INFO | "gpu" : " 17 GB", | 20260505 16:41:28.472 | INFO | "huge" : "", | 20260505 16:41:28.472 | INFO | "huge_shared" : " 27 GB", | 20260505 16:41:28.472 | INFO | "malloc" : " 14 GB", | 20260505 16:41:28.472 | INFO | "pinned" : " 29 GB" | 20260505 16:41:28.472 | INFO | } | 20260505 16:41:28.472 | INFO | Perf report for the last 5.14314 seconds: | 20260505 16:41:28.472 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:41:28.472 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:28.472 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:28.472 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:28.472 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 16:41:28.472 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:28.472 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:28.472 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 16:41:28.472 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:28.472 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:28.472 | INFO | Uploaded 36GiB to GPU in 2.14941s (16.7488GiB/s) | 20260505 16:41:28.472 | INFO | Downloaded 34GiB to GPU in 2.18098s (15.5893GiB/s) | 20260505 16:41:33.606 | INFO | Analyzing chunk frames = [4722176,4722688) | 20260505 16:41:33.606 | INFO | MemInfo: { | 20260505 16:41:33.606 | INFO | "gpu" : " 17 GB", | 20260505 16:41:33.606 | INFO | "huge" : "", | 20260505 16:41:33.606 | INFO | "huge_shared" : " 27 GB", | 20260505 16:41:33.607 | INFO | "malloc" : " 14 GB", | 20260505 16:41:33.607 | INFO | "pinned" : " 29 GB" | 20260505 16:41:33.607 | INFO | } | 20260505 16:41:33.607 | INFO | Perf report for the last 5.13456 seconds: | 20260505 16:41:33.607 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:41:33.607 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:33.607 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:33.607 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:33.607 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 16:41:33.607 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:33.607 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:33.607 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 16:41:33.607 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:33.607 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:33.607 | INFO | Uploaded 36GiB to GPU in 2.14886s (16.7531GiB/s) | 20260505 16:41:33.607 | INFO | Downloaded 34GiB to GPU in 2.02436s (16.7954GiB/s) | 20260505 16:41:33.740 | INFO | Starting a round of Baz IO | 20260505 16:41:36.943 | INFO | Populating RTMetrics report | 20260505 16:41:36.944 | INFO | RTMetrics report populated | 20260505 16:41:36.944 | INFO | Opening tmp RTMetrics json file | 20260505 16:41:36.954 | INFO | Serializing RTMetrics to json | 20260505 16:41:36.955 | INFO | Serialization done, swapping tmp file now... | 20260505 16:41:36.955 | INFO | RTMetrics done | 20260505 16:41:37.042 | INFO | Finished a round of Baz IO: Wrote 2.16 GiB in 3.301 seconds | 20260505 16:41:37.042 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.20% / 0.00% / 77.80% | 20260505 16:41:37.042 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.345377% / 1.19209e-05% / 9.13938e-05% | 20260505 16:41:37.042 | INFO | Baz IO overall: | 20260505 16:41:37.042 | INFO | BytesWritten : 2799972592840 | 20260505 16:41:37.042 | INFO | NumEvents : 1328711167448 | 20260505 16:41:38.740 | INFO | Analyzing chunk frames = [4722688,4723200) | 20260505 16:41:38.740 | INFO | MemInfo: { | 20260505 16:41:38.740 | INFO | "gpu" : " 17 GB", | 20260505 16:41:38.740 | INFO | "huge" : "", | 20260505 16:41:38.740 | INFO | "huge_shared" : " 27 GB", | 20260505 16:41:38.740 | INFO | "malloc" : " 14 GB", | 20260505 16:41:38.740 | INFO | "pinned" : " 29 GB" | 20260505 16:41:38.740 | INFO | } | 20260505 16:41:38.740 | INFO | Perf report for the last 5.13306 seconds: | 20260505 16:41:38.740 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:41:38.740 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:38.740 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:38.740 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:38.740 | INFO | ANALYSIS : 63.9%, 3.0, 99.4ms, 0 | 20260505 16:41:38.740 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:38.740 | INFO | RT_METRICS : 44.9%, 1.0, 23.5ms, 0 | 20260505 16:41:38.740 | INFO | PRE_HQ : 60.6%, 1.8, 56.0ms, 0 | 20260505 16:41:38.740 | INFO | BAZWRITER : 63.6%, 3.9, 128.3ms, 0 | 20260505 16:41:38.740 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 16:41:38.740 | INFO | Uploaded 36GiB to GPU in 2.16484s (16.6294GiB/s) | 20260505 16:41:38.740 | INFO | Downloaded 38GiB to GPU in 2.26167s (16.8017GiB/s) | 20260505 16:41:43.882 | INFO | Analyzing chunk frames = [4723200,4723712) | 20260505 16:41:43.882 | INFO | MemInfo: { | 20260505 16:41:43.882 | INFO | "gpu" : " 17 GB", | 20260505 16:41:43.882 | INFO | "huge" : "", | 20260505 16:41:43.882 | INFO | "huge_shared" : " 27 GB", | 20260505 16:41:43.882 | INFO | "malloc" : " 14 GB", | 20260505 16:41:43.882 | INFO | "pinned" : " 29 GB" | 20260505 16:41:43.882 | INFO | } | 20260505 16:41:43.882 | INFO | Perf report for the last 5.14209 seconds: | 20260505 16:41:43.882 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:41:43.882 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:43.882 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:43.882 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:43.882 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 16:41:43.882 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:43.882 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:43.882 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 16:41:43.882 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:43.882 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:43.882 | INFO | Uploaded 36GiB to GPU in 2.20607s (16.3186GiB/s) | 20260505 16:41:43.882 | INFO | Downloaded 34GiB to GPU in 2.0669s (16.4497GiB/s) | 20260505 16:41:49.028 | INFO | Analyzing chunk frames = [4723712,4724224) | 20260505 16:41:49.028 | INFO | MemInfo: { | 20260505 16:41:49.028 | INFO | "gpu" : " 17 GB", | 20260505 16:41:49.028 | INFO | "huge" : "", | 20260505 16:41:49.028 | INFO | "huge_shared" : " 27 GB", | 20260505 16:41:49.029 | INFO | "malloc" : " 14 GB", | 20260505 16:41:49.029 | INFO | "pinned" : " 29 GB" | 20260505 16:41:49.029 | INFO | } | 20260505 16:41:49.029 | INFO | Perf report for the last 5.14674 seconds: | 20260505 16:41:49.029 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:41:49.029 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:49.029 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:49.029 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:49.029 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 16:41:49.029 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:49.029 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:49.029 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:41:49.029 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:49.029 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:49.029 | INFO | Uploaded 36GiB to GPU in 2.20282s (16.3427GiB/s) | 20260505 16:41:49.029 | INFO | Downloaded 34GiB to GPU in 2.03363s (16.7189GiB/s) | 20260505 16:41:54.170 | INFO | Analyzing chunk frames = [4724224,4724736) | 20260505 16:41:54.170 | INFO | MemInfo: { | 20260505 16:41:54.170 | INFO | "gpu" : " 17 GB", | 20260505 16:41:54.170 | INFO | "huge" : "", | 20260505 16:41:54.170 | INFO | "huge_shared" : " 27 GB", | 20260505 16:41:54.170 | INFO | "malloc" : " 14 GB", | 20260505 16:41:54.170 | INFO | "pinned" : " 29 GB" | 20260505 16:41:54.170 | INFO | } | 20260505 16:41:54.170 | INFO | Perf report for the last 5.14162 seconds: | 20260505 16:41:54.170 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:41:54.170 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:54.170 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:54.170 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:54.170 | INFO | ANALYSIS : 62.8%, 3.0, 97.6ms, 0 | 20260505 16:41:54.170 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:54.170 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:54.170 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:41:54.170 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:54.170 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:54.171 | INFO | Uploaded 36GiB to GPU in 2.22682s (16.1666GiB/s) | 20260505 16:41:54.171 | INFO | Downloaded 34GiB to GPU in 2.12257s (16.0183GiB/s) | 20260505 16:41:59.301 | INFO | Analyzing chunk frames = [4724736,4725248) | 20260505 16:41:59.301 | INFO | MemInfo: { | 20260505 16:41:59.301 | INFO | "gpu" : " 17 GB", | 20260505 16:41:59.301 | INFO | "huge" : "", | 20260505 16:41:59.301 | INFO | "huge_shared" : " 27 GB", | 20260505 16:41:59.301 | INFO | "malloc" : " 14 GB", | 20260505 16:41:59.301 | INFO | "pinned" : " 29 GB" | 20260505 16:41:59.301 | INFO | } | 20260505 16:41:59.301 | INFO | Perf report for the last 5.13081 seconds: | 20260505 16:41:59.301 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:41:59.301 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:59.301 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:59.301 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:59.301 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260505 16:41:59.301 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:59.301 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:59.301 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:41:59.301 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:59.301 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:41:59.301 | INFO | Uploaded 36GiB to GPU in 2.15179s (16.7303GiB/s) | 20260505 16:41:59.301 | INFO | Downloaded 34GiB to GPU in 2.05131s (16.5748GiB/s) | 20260505 16:42:04.438 | INFO | Analyzing chunk frames = [4725248,4725760) | 20260505 16:42:04.438 | INFO | MemInfo: { | 20260505 16:42:04.438 | INFO | "gpu" : " 17 GB", | 20260505 16:42:04.438 | INFO | "huge" : "", | 20260505 16:42:04.438 | INFO | "huge_shared" : " 27 GB", | 20260505 16:42:04.438 | INFO | "malloc" : " 14 GB", | 20260505 16:42:04.439 | INFO | "pinned" : " 29 GB" | 20260505 16:42:04.439 | INFO | } | 20260505 16:42:04.439 | INFO | Perf report for the last 5.1378 seconds: | 20260505 16:42:04.439 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:42:04.439 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:04.439 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:04.439 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:04.439 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 16:42:04.439 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:04.439 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:04.439 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 16:42:04.439 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:04.439 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:04.439 | INFO | Uploaded 36GiB to GPU in 2.2125s (16.2712GiB/s) | 20260505 16:42:04.439 | INFO | Downloaded 34GiB to GPU in 2.11326s (16.0889GiB/s) | 20260505 16:42:09.584 | INFO | Analyzing chunk frames = [4725760,4726272) | 20260505 16:42:09.584 | INFO | MemInfo: { | 20260505 16:42:09.584 | INFO | "gpu" : " 17 GB", | 20260505 16:42:09.584 | INFO | "huge" : "", | 20260505 16:42:09.584 | INFO | "huge_shared" : " 27 GB", | 20260505 16:42:09.584 | INFO | "malloc" : " 14 GB", | 20260505 16:42:09.584 | INFO | "pinned" : " 29 GB" | 20260505 16:42:09.584 | INFO | } | 20260505 16:42:09.584 | INFO | Perf report for the last 5.145 seconds: | 20260505 16:42:09.584 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:42:09.584 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:09.584 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:09.584 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:09.584 | INFO | ANALYSIS : 63.5%, 3.0, 99.3ms, 0 | 20260505 16:42:09.584 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:09.584 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:09.584 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 16:42:09.584 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:09.584 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:09.584 | INFO | Uploaded 36GiB to GPU in 2.10789s (17.0787GiB/s) | 20260505 16:42:09.584 | INFO | Downloaded 34GiB to GPU in 2.16867s (15.6778GiB/s) | 20260505 16:42:14.723 | INFO | Analyzing chunk frames = [4726272,4726784) | 20260505 16:42:14.723 | INFO | MemInfo: { | 20260505 16:42:14.723 | INFO | "gpu" : " 17 GB", | 20260505 16:42:14.723 | INFO | "huge" : "", | 20260505 16:42:14.723 | INFO | "huge_shared" : " 27 GB", | 20260505 16:42:14.723 | INFO | "malloc" : " 14 GB", | 20260505 16:42:14.723 | INFO | "pinned" : " 29 GB" | 20260505 16:42:14.723 | INFO | } | 20260505 16:42:14.723 | INFO | Perf report for the last 5.139 seconds: | 20260505 16:42:14.723 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:42:14.723 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:14.723 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:14.723 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:14.723 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 16:42:14.723 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:14.723 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:14.723 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:42:14.723 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:14.723 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:14.723 | INFO | Uploaded 36GiB to GPU in 2.17415s (16.5582GiB/s) | 20260505 16:42:14.723 | INFO | Downloaded 34GiB to GPU in 2.01915s (16.8387GiB/s) | 20260505 16:42:14.723 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 16:42:14.723 | INFO | Total mem usage: 15.3 GiB | 20260505 16:42:14.723 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 16:42:14.723 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.9%) | 20260505 16:42:14.723 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 16:42:14.723 | INFO | High Water Breakdown: | 20260505 16:42:14.723 | INFO | 61% : 8.67 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 16:42:14.723 | INFO | 38.8% : 5.51 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 16:42:14.723 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 16:42:14.723 | INFO | ------------------------------------------ | 20260505 16:42:14.724 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:42:14.724 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 16:42:14.724 | INFO | Total mem usage: 26.5 GiB | 20260505 16:42:14.724 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 16:42:14.724 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 16:42:14.724 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 16:42:14.724 | INFO | High Water Breakdown: | 20260505 16:42:14.724 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:42:14.724 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:42:14.724 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:42:14.724 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:42:14.724 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:42:14.724 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:42:14.724 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 16:42:14.724 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 16:42:14.724 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:42:14.724 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 16:42:14.724 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 16:42:14.724 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 16:42:14.724 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 16:42:14.724 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 16:42:14.724 | INFO | ------------------------------------------ | 20260505 16:42:14.724 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:42:14.724 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 16:42:14.724 | INFO | Total mem usage: 25.6 GiB | 20260505 16:42:14.724 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 16:42:14.724 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 16:42:14.724 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 16:42:14.724 | INFO | High Water Breakdown: | 20260505 16:42:14.724 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 16:42:14.724 | INFO | ------------------------------------------ | 20260505 16:42:14.724 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:42:14.724 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 16:42:14.724 | INFO | Total mem usage: 15.7 GiB | 20260505 16:42:14.724 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 16:42:14.724 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 16:42:14.724 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 16:42:14.724 | INFO | High Water Breakdown: | 20260505 16:42:14.724 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 16:42:14.724 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 16:42:14.724 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 16:42:14.724 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 16:42:14.724 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 16:42:14.724 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 16:42:14.724 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 16:42:14.724 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 16:42:14.724 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 16:42:14.724 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:42:14.724 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 16:42:14.724 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:42:14.724 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 16:42:14.724 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 16:42:14.724 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 16:42:14.724 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 16:42:14.724 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:42:14.724 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 16:42:14.724 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:42:14.724 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:42:14.724 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:42:14.724 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 16:42:14.724 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 16:42:14.724 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:42:14.724 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 16:42:14.724 | INFO | ------------------------------------------ | 20260505 16:42:14.724 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:42:14.724 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 16:42:14.724 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:42:14.724 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:42:14.724 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.52e+01, 1.36e+01 / 2.17e+00 | 20260505 16:42:14.724 | INFO | PerfReport: 0.29%, AggregateStats -- 9.25e-02 / 1.01e-01, 1.04e-02 / 4.05e-03 | 20260505 16:42:14.724 | INFO | PerfReport: 11.77%, Baseline -- 3.72e+00 / 4.48e+00, 5.64e-01 / 1.69e-01 | 20260505 16:42:14.724 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.53e+00, 1.92e-01 / 9.50e-02 | 20260505 16:42:14.724 | INFO | PerfReport: 16.00%, DME -- 4.05e+01 / 4.52e+01, 4.29e+00 / 2.26e+00 | 20260505 16:42:14.724 | INFO | PerfReport: 56.69%, FrameLabeling -- 1.79e+01 / 1.91e+01, 1.42e+00 / 9.21e-01 | 20260505 16:42:14.724 | INFO | PerfReport: 7.39%, PulseAccumulating -- 2.34e+00 / 2.94e+00, 4.48e-01 / 1.39e-01 | 20260505 16:42:14.724 | INFO | PerfReport: 3.40%, Metrics -- 1.08e+00 / 1.42e+00, 2.96e-01 / 1.36e-01 | 20260505 16:42:14.725 | INFO | PerfReport: 1.19%, PulseSerializing -- 3.75e-01 / 4.32e-01, 6.55e-02 / 3.28e-02 | 20260505 16:42:14.725 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 16:42:14.725 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:42:14.725 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:42:14.725 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.60e+01 / 0.00e+00 | 20260505 16:42:14.725 | INFO | PerfReport: 22.56%, Upload -- 2.26e+01 / 0.00e+00, 7.88e+00 / 0.00e+00 | 20260505 16:42:14.725 | INFO | PerfReport: 21.83%, Download -- 2.19e+01 / 0.00e+00, 7.88e+00 / 0.00e+00 | 20260505 16:42:14.725 | INFO | PerfReport: 31.53%, Compute -- 3.16e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 16:42:14.725 | INFO | PerfReport: 2.41%, LockUpload -- 2.42e+00 / 0.00e+00, 8.08e+00 / 0.00e+00 | 20260505 16:42:14.725 | INFO | PerfReport: 1.44%, LockDownload -- 1.45e+00 / 0.00e+00, 3.03e+00 / 0.00e+00 | 20260505 16:42:14.725 | INFO | PerfReport: 20.21%, LockCompute -- 2.03e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260505 16:42:14.725 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.79e-02 / 0.00e+00 | 20260505 16:42:14.857 | INFO | Starting a round of Baz IO | 20260505 16:42:18.059 | INFO | Populating RTMetrics report | 20260505 16:42:18.059 | INFO | RTMetrics report populated | 20260505 16:42:18.059 | INFO | Opening tmp RTMetrics json file | 20260505 16:42:18.099 | INFO | Serializing RTMetrics to json | 20260505 16:42:18.139 | INFO | Serialization done, swapping tmp file now... | 20260505 16:42:18.141 | INFO | RTMetrics done | 20260505 16:42:18.157 | INFO | Finished a round of Baz IO: Wrote 2.16 GiB in 3.3 seconds | 20260505 16:42:18.157 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.19% / 0.00% / 77.80% | 20260505 16:42:18.157 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.345413% / 3.97364e-06% / 8.34465e-05% | 20260505 16:42:18.157 | INFO | Baz IO overall: | 20260505 16:42:18.157 | INFO | BytesWritten : 2802291897426 | 20260505 16:42:18.157 | INFO | NumEvents : 1329797442931 | 20260505 16:42:19.863 | INFO | Analyzing chunk frames = [4726784,4727296) | 20260505 16:42:19.863 | INFO | MemInfo: { | 20260505 16:42:19.863 | INFO | "gpu" : " 17 GB", | 20260505 16:42:19.863 | INFO | "huge" : "", | 20260505 16:42:19.863 | INFO | "huge_shared" : " 27 GB", | 20260505 16:42:19.863 | INFO | "malloc" : " 14 GB", | 20260505 16:42:19.863 | INFO | "pinned" : " 29 GB" | 20260505 16:42:19.863 | INFO | } | 20260505 16:42:19.863 | INFO | Perf report for the last 5.13971 seconds: | 20260505 16:42:19.863 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:42:19.863 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:19.863 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:19.863 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:19.863 | INFO | ANALYSIS : 63.8%, 3.0, 99.3ms, 0 | 20260505 16:42:19.863 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:19.863 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260505 16:42:19.863 | INFO | PRE_HQ : 60.6%, 1.8, 56.3ms, 0 | 20260505 16:42:19.863 | INFO | BAZWRITER : 63.5%, 3.9, 127.1ms, 0 | 20260505 16:42:19.863 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 16:42:19.863 | INFO | Uploaded 36GiB to GPU in 2.17782s (16.5303GiB/s) | 20260505 16:42:19.863 | INFO | Downloaded 38GiB to GPU in 2.26097s (16.807GiB/s) | 20260505 16:42:25.000 | INFO | Analyzing chunk frames = [4727296,4727808) | 20260505 16:42:25.000 | INFO | MemInfo: { | 20260505 16:42:25.000 | INFO | "gpu" : " 17 GB", | 20260505 16:42:25.000 | INFO | "huge" : "", | 20260505 16:42:25.000 | INFO | "huge_shared" : " 27 GB", | 20260505 16:42:25.000 | INFO | "malloc" : " 14 GB", | 20260505 16:42:25.000 | INFO | "pinned" : " 29 GB" | 20260505 16:42:25.000 | INFO | } | 20260505 16:42:25.000 | INFO | Perf report for the last 5.1378 seconds: | 20260505 16:42:25.000 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:42:25.000 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:25.000 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:25.000 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:25.000 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 16:42:25.001 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:25.001 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:25.001 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:42:25.001 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:25.001 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:25.001 | INFO | Uploaded 36GiB to GPU in 2.12217s (16.9638GiB/s) | 20260505 16:42:25.001 | INFO | Downloaded 34GiB to GPU in 2.02852s (16.761GiB/s) | 20260505 16:42:30.136 | INFO | Analyzing chunk frames = [4727808,4728320) | 20260505 16:42:30.136 | INFO | MemInfo: { | 20260505 16:42:30.136 | INFO | "gpu" : " 17 GB", | 20260505 16:42:30.136 | INFO | "huge" : "", | 20260505 16:42:30.136 | INFO | "huge_shared" : " 27 GB", | 20260505 16:42:30.136 | INFO | "malloc" : " 14 GB", | 20260505 16:42:30.136 | INFO | "pinned" : " 29 GB" | 20260505 16:42:30.136 | INFO | } | 20260505 16:42:30.136 | INFO | Perf report for the last 5.13583 seconds: | 20260505 16:42:30.136 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:42:30.136 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:30.136 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:30.136 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:30.136 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260505 16:42:30.136 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:30.136 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:30.136 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:42:30.136 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:30.136 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:30.137 | INFO | Uploaded 36GiB to GPU in 2.19298s (16.416GiB/s) | 20260505 16:42:30.137 | INFO | Downloaded 34GiB to GPU in 2.0294s (16.7537GiB/s) | 20260505 16:42:35.282 | INFO | Analyzing chunk frames = [4728320,4728832) | 20260505 16:42:35.282 | INFO | MemInfo: { | 20260505 16:42:35.282 | INFO | "gpu" : " 17 GB", | 20260505 16:42:35.282 | INFO | "huge" : "", | 20260505 16:42:35.282 | INFO | "huge_shared" : " 27 GB", | 20260505 16:42:35.282 | INFO | "malloc" : " 14 GB", | 20260505 16:42:35.282 | INFO | "pinned" : " 29 GB" | 20260505 16:42:35.282 | INFO | } | 20260505 16:42:35.282 | INFO | Perf report for the last 5.14592 seconds: | 20260505 16:42:35.282 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:42:35.282 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:35.282 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:35.282 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:35.282 | INFO | ANALYSIS : 62.5%, 3.0, 97.2ms, 0 | 20260505 16:42:35.282 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:35.282 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:35.282 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:42:35.282 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:35.282 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:35.283 | INFO | Uploaded 36GiB to GPU in 2.16564s (16.6232GiB/s) | 20260505 16:42:35.283 | INFO | Downloaded 34GiB to GPU in 2.14072s (15.8825GiB/s) | 20260505 16:42:40.425 | INFO | Analyzing chunk frames = [4728832,4729344) | 20260505 16:42:40.425 | INFO | MemInfo: { | 20260505 16:42:40.425 | INFO | "gpu" : " 17 GB", | 20260505 16:42:40.425 | INFO | "huge" : "", | 20260505 16:42:40.425 | INFO | "huge_shared" : " 27 GB", | 20260505 16:42:40.425 | INFO | "malloc" : " 14 GB", | 20260505 16:42:40.425 | INFO | "pinned" : " 29 GB" | 20260505 16:42:40.426 | INFO | } | 20260505 16:42:40.426 | INFO | Perf report for the last 5.14343 seconds: | 20260505 16:42:40.426 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:42:40.426 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:40.426 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:40.426 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:40.426 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 16:42:40.426 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:40.426 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:40.426 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:42:40.426 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:40.426 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:40.426 | INFO | Uploaded 36GiB to GPU in 2.14507s (16.7826GiB/s) | 20260505 16:42:40.426 | INFO | Downloaded 34GiB to GPU in 2.05121s (16.5756GiB/s) | 20260505 16:42:45.552 | INFO | Analyzing chunk frames = [4729344,4729856) | 20260505 16:42:45.552 | INFO | MemInfo: { | 20260505 16:42:45.552 | INFO | "gpu" : " 17 GB", | 20260505 16:42:45.552 | INFO | "huge" : "", | 20260505 16:42:45.552 | INFO | "huge_shared" : " 27 GB", | 20260505 16:42:45.552 | INFO | "malloc" : " 14 GB", | 20260505 16:42:45.552 | INFO | "pinned" : " 29 GB" | 20260505 16:42:45.552 | INFO | } | 20260505 16:42:45.553 | INFO | Perf report for the last 5.12687 seconds: | 20260505 16:42:45.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:42:45.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:45.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:45.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:45.553 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260505 16:42:45.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:45.553 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:45.553 | INFO | PRE_HQ : 27.7%, 1.0, 14.3ms, 0 | 20260505 16:42:45.553 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:45.553 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:45.553 | INFO | Uploaded 36GiB to GPU in 2.14425s (16.7891GiB/s) | 20260505 16:42:45.553 | INFO | Downloaded 34GiB to GPU in 2.0452s (16.6243GiB/s) | 20260505 16:42:50.688 | INFO | Analyzing chunk frames = [4729856,4730368) | 20260505 16:42:50.688 | INFO | MemInfo: { | 20260505 16:42:50.688 | INFO | "gpu" : " 17 GB", | 20260505 16:42:50.688 | INFO | "huge" : "", | 20260505 16:42:50.688 | INFO | "huge_shared" : " 27 GB", | 20260505 16:42:50.689 | INFO | "malloc" : " 14 GB", | 20260505 16:42:50.689 | INFO | "pinned" : " 29 GB" | 20260505 16:42:50.689 | INFO | } | 20260505 16:42:50.689 | INFO | Perf report for the last 5.13612 seconds: | 20260505 16:42:50.689 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:42:50.689 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:50.689 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:50.689 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:50.689 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 16:42:50.689 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:50.689 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:50.689 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 16:42:50.689 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:50.689 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:50.689 | INFO | Uploaded 36GiB to GPU in 2.13443s (16.8663GiB/s) | 20260505 16:42:50.689 | INFO | Downloaded 34GiB to GPU in 2.15789s (15.7561GiB/s) | 20260505 16:42:55.838 | INFO | Analyzing chunk frames = [4730368,4730880) | 20260505 16:42:55.839 | INFO | MemInfo: { | 20260505 16:42:55.839 | INFO | "gpu" : " 17 GB", | 20260505 16:42:55.839 | INFO | "huge" : "", | 20260505 16:42:55.839 | INFO | "huge_shared" : " 27 GB", | 20260505 16:42:55.839 | INFO | "malloc" : " 14 GB", | 20260505 16:42:55.839 | INFO | "pinned" : " 29 GB" | 20260505 16:42:55.839 | INFO | } | 20260505 16:42:55.839 | INFO | Perf report for the last 5.15009 seconds: | 20260505 16:42:55.839 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:42:55.839 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:55.839 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:55.839 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:55.839 | INFO | ANALYSIS : 63.2%, 3.0, 99.0ms, 0 | 20260505 16:42:55.839 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:55.839 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:55.839 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 16:42:55.839 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:55.839 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:42:55.839 | INFO | Uploaded 36GiB to GPU in 2.17165s (16.5772GiB/s) | 20260505 16:42:55.839 | INFO | Downloaded 34GiB to GPU in 2.03613s (16.6984GiB/s) | 20260505 16:42:55.976 | INFO | Starting a round of Baz IO | 20260505 16:42:59.176 | INFO | Populating RTMetrics report | 20260505 16:42:59.176 | INFO | RTMetrics report populated | 20260505 16:42:59.176 | INFO | Opening tmp RTMetrics json file | 20260505 16:42:59.196 | INFO | Serializing RTMetrics to json | 20260505 16:42:59.218 | INFO | Serialization done, swapping tmp file now... | 20260505 16:42:59.238 | INFO | RTMetrics done | 20260505 16:42:59.281 | INFO | Finished a round of Baz IO: Wrote 2.171 GiB in 3.305 seconds | 20260505 16:42:59.281 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.19% / 0.00% / 77.81% | 20260505 16:42:59.281 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.349589% / 3.97364e-06% / 9.53674e-05% | 20260505 16:42:59.281 | INFO | Baz IO overall: | 20260505 16:42:59.281 | INFO | BytesWritten : 2804623474729 | 20260505 16:42:59.281 | INFO | NumEvents : 1330888633097 | 20260505 16:43:00.983 | INFO | Analyzing chunk frames = [4730880,4731392) | 20260505 16:43:00.983 | INFO | MemInfo: { | 20260505 16:43:00.983 | INFO | "gpu" : " 17 GB", | 20260505 16:43:00.983 | INFO | "huge" : "", | 20260505 16:43:00.983 | INFO | "huge_shared" : " 27 GB", | 20260505 16:43:00.984 | INFO | "malloc" : " 14 GB", | 20260505 16:43:00.984 | INFO | "pinned" : " 29 GB" | 20260505 16:43:00.984 | INFO | } | 20260505 16:43:00.984 | INFO | Perf report for the last 5.14489 seconds: | 20260505 16:43:00.984 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:43:00.984 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:00.984 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:00.984 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:00.984 | INFO | ANALYSIS : 63.8%, 3.0, 99.3ms, 0 | 20260505 16:43:00.984 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:00.984 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260505 16:43:00.984 | INFO | PRE_HQ : 60.6%, 1.8, 56.8ms, 0 | 20260505 16:43:00.984 | INFO | BAZWRITER : 63.6%, 3.9, 128.3ms, 0 | 20260505 16:43:00.984 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260505 16:43:00.984 | INFO | Uploaded 36GiB to GPU in 2.18686s (16.462GiB/s) | 20260505 16:43:00.984 | INFO | Downloaded 38GiB to GPU in 2.26924s (16.7457GiB/s) | 20260505 16:43:06.118 | INFO | Analyzing chunk frames = [4731392,4731904) | 20260505 16:43:06.118 | INFO | MemInfo: { | 20260505 16:43:06.118 | INFO | "gpu" : " 17 GB", | 20260505 16:43:06.118 | INFO | "huge" : "", | 20260505 16:43:06.118 | INFO | "huge_shared" : " 27 GB", | 20260505 16:43:06.118 | INFO | "malloc" : " 14 GB", | 20260505 16:43:06.118 | INFO | "pinned" : " 29 GB" | 20260505 16:43:06.118 | INFO | } | 20260505 16:43:06.118 | INFO | Perf report for the last 5.13474 seconds: | 20260505 16:43:06.118 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:43:06.118 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:06.118 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:06.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:06.118 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 16:43:06.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:06.119 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:06.119 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 16:43:06.119 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:06.119 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:06.119 | INFO | Uploaded 36GiB to GPU in 2.11798s (16.9974GiB/s) | 20260505 16:43:06.119 | INFO | Downloaded 34GiB to GPU in 2.02101s (16.8233GiB/s) | 20260505 16:43:11.252 | INFO | Analyzing chunk frames = [4731904,4732416) | 20260505 16:43:11.252 | INFO | MemInfo: { | 20260505 16:43:11.252 | INFO | "gpu" : " 17 GB", | 20260505 16:43:11.252 | INFO | "huge" : "", | 20260505 16:43:11.252 | INFO | "huge_shared" : " 27 GB", | 20260505 16:43:11.252 | INFO | "malloc" : " 14 GB", | 20260505 16:43:11.252 | INFO | "pinned" : " 29 GB" | 20260505 16:43:11.252 | INFO | } | 20260505 16:43:11.252 | INFO | Perf report for the last 5.1334 seconds: | 20260505 16:43:11.252 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:43:11.252 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:11.252 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:11.252 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:11.252 | INFO | ANALYSIS : 62.9%, 3.0, 97.7ms, 0 | 20260505 16:43:11.252 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:11.252 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:11.252 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 16:43:11.252 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:11.252 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:11.252 | INFO | Uploaded 36GiB to GPU in 2.19929s (16.3689GiB/s) | 20260505 16:43:11.252 | INFO | Downloaded 34GiB to GPU in 2.02898s (16.7572GiB/s) | 20260505 16:43:16.388 | INFO | Analyzing chunk frames = [4732416,4732928) | 20260505 16:43:16.388 | INFO | MemInfo: { | 20260505 16:43:16.388 | INFO | "gpu" : " 17 GB", | 20260505 16:43:16.388 | INFO | "huge" : "", | 20260505 16:43:16.389 | INFO | "huge_shared" : " 27 GB", | 20260505 16:43:16.389 | INFO | "malloc" : " 14 GB", | 20260505 16:43:16.389 | INFO | "pinned" : " 29 GB" | 20260505 16:43:16.389 | INFO | } | 20260505 16:43:16.389 | INFO | Perf report for the last 5.1369 seconds: | 20260505 16:43:16.389 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:43:16.389 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:16.389 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:16.389 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:16.389 | INFO | ANALYSIS : 62.5%, 3.0, 97.0ms, 0 | 20260505 16:43:16.389 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:16.389 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:16.389 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 16:43:16.389 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:16.389 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:16.389 | INFO | Uploaded 36GiB to GPU in 2.18198s (16.4988GiB/s) | 20260505 16:43:16.389 | INFO | Downloaded 34GiB to GPU in 2.13591s (15.9183GiB/s) | 20260505 16:43:21.533 | INFO | Analyzing chunk frames = [4732928,4733440) | 20260505 16:43:21.533 | INFO | MemInfo: { | 20260505 16:43:21.533 | INFO | "gpu" : " 17 GB", | 20260505 16:43:21.533 | INFO | "huge" : "", | 20260505 16:43:21.533 | INFO | "huge_shared" : " 27 GB", | 20260505 16:43:21.533 | INFO | "malloc" : " 14 GB", | 20260505 16:43:21.533 | INFO | "pinned" : " 29 GB" | 20260505 16:43:21.533 | INFO | } | 20260505 16:43:21.533 | INFO | Perf report for the last 5.14451 seconds: | 20260505 16:43:21.533 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:43:21.533 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:21.533 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:21.533 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:21.533 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 16:43:21.533 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:21.533 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:21.533 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:43:21.533 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:21.533 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:21.534 | INFO | Uploaded 36GiB to GPU in 2.13354s (16.8734GiB/s) | 20260505 16:43:21.534 | INFO | Downloaded 34GiB to GPU in 2.05214s (16.5681GiB/s) | 20260505 16:43:26.678 | INFO | Analyzing chunk frames = [4733440,4733952) | 20260505 16:43:26.678 | INFO | MemInfo: { | 20260505 16:43:26.678 | INFO | "gpu" : " 17 GB", | 20260505 16:43:26.678 | INFO | "huge" : "", | 20260505 16:43:26.678 | INFO | "huge_shared" : " 27 GB", | 20260505 16:43:26.678 | INFO | "malloc" : " 14 GB", | 20260505 16:43:26.678 | INFO | "pinned" : " 29 GB" | 20260505 16:43:26.678 | INFO | } | 20260505 16:43:26.678 | INFO | Perf report for the last 5.1445 seconds: | 20260505 16:43:26.678 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:43:26.678 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:26.678 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:26.678 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:26.678 | INFO | ANALYSIS : 62.2%, 3.0, 97.3ms, 0 | 20260505 16:43:26.678 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:26.678 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:26.678 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:43:26.678 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:26.678 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:26.678 | INFO | Uploaded 36GiB to GPU in 2.12722s (16.9235GiB/s) | 20260505 16:43:26.678 | INFO | Downloaded 34GiB to GPU in 2.03758s (16.6865GiB/s) | 20260505 16:43:31.806 | INFO | Analyzing chunk frames = [4733952,4734464) | 20260505 16:43:31.806 | INFO | MemInfo: { | 20260505 16:43:31.806 | INFO | "gpu" : " 17 GB", | 20260505 16:43:31.806 | INFO | "huge" : "", | 20260505 16:43:31.806 | INFO | "huge_shared" : " 27 GB", | 20260505 16:43:31.806 | INFO | "malloc" : " 14 GB", | 20260505 16:43:31.806 | INFO | "pinned" : " 29 GB" | 20260505 16:43:31.806 | INFO | } | 20260505 16:43:31.806 | INFO | Perf report for the last 5.12834 seconds: | 20260505 16:43:31.806 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:43:31.806 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:31.806 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:31.806 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:31.806 | INFO | ANALYSIS : 63.6%, 3.0, 99.1ms, 0 | 20260505 16:43:31.806 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:31.806 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:31.806 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 16:43:31.806 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:31.806 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:31.807 | INFO | Uploaded 36GiB to GPU in 2.10768s (17.0804GiB/s) | 20260505 16:43:31.807 | INFO | Downloaded 34GiB to GPU in 2.17731s (15.6156GiB/s) | 20260505 16:43:36.946 | INFO | Analyzing chunk frames = [4734464,4734976) | 20260505 16:43:36.946 | INFO | MemInfo: { | 20260505 16:43:36.946 | INFO | "gpu" : " 17 GB", | 20260505 16:43:36.946 | INFO | "huge" : "", | 20260505 16:43:36.946 | INFO | "huge_shared" : " 27 GB", | 20260505 16:43:36.946 | INFO | "malloc" : " 14 GB", | 20260505 16:43:36.946 | INFO | "pinned" : " 29 GB" | 20260505 16:43:36.946 | INFO | } | 20260505 16:43:36.946 | INFO | Perf report for the last 5.13959 seconds: | 20260505 16:43:36.946 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:43:36.946 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:36.946 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:36.946 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:36.946 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 16:43:36.946 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:36.946 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:36.946 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 16:43:36.946 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:36.946 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:36.946 | INFO | Uploaded 36GiB to GPU in 2.21691s (16.2388GiB/s) | 20260505 16:43:36.946 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 16:43:36.946 | INFO | Total mem usage: 15.3 GiB | 20260505 16:43:36.946 | INFO | Downloaded 34GiB to GPU in 2.10008s (16.1898GiB/s) | 20260505 16:43:36.946 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 16:43:36.946 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.89%) | 20260505 16:43:36.946 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 16:43:36.946 | INFO | High Water Breakdown: | 20260505 16:43:36.946 | INFO | 61% : 8.67 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 16:43:36.946 | INFO | 38.8% : 5.51 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 16:43:36.946 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 16:43:36.946 | INFO | ------------------------------------------ | 20260505 16:43:36.946 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:43:36.946 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 16:43:36.946 | INFO | Total mem usage: 26.5 GiB | 20260505 16:43:36.946 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 16:43:36.946 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 16:43:36.946 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 16:43:36.946 | INFO | High Water Breakdown: | 20260505 16:43:36.946 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:43:36.946 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:43:36.946 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:43:36.946 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:43:36.946 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:43:36.946 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:43:36.946 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 16:43:36.946 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 16:43:36.946 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:43:36.946 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 16:43:36.946 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 16:43:36.946 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 16:43:36.946 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 16:43:36.946 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 16:43:36.947 | INFO | ------------------------------------------ | 20260505 16:43:36.947 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:43:36.947 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 16:43:36.947 | INFO | Total mem usage: 25.6 GiB | 20260505 16:43:36.947 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 16:43:36.947 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 16:43:36.947 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 16:43:36.947 | INFO | High Water Breakdown: | 20260505 16:43:36.947 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 16:43:36.947 | INFO | ------------------------------------------ | 20260505 16:43:36.947 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:43:36.947 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 16:43:36.947 | INFO | Total mem usage: 15.7 GiB | 20260505 16:43:36.947 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 16:43:36.947 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 16:43:36.947 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 16:43:36.947 | INFO | High Water Breakdown: | 20260505 16:43:36.947 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 16:43:36.947 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 16:43:36.947 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 16:43:36.947 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 16:43:36.947 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 16:43:36.947 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 16:43:36.947 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 16:43:36.947 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 16:43:36.947 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 16:43:36.947 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:43:36.947 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 16:43:36.947 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:43:36.947 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 16:43:36.947 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 16:43:36.947 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 16:43:36.947 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 16:43:36.947 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:43:36.947 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 16:43:36.947 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:43:36.947 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:43:36.947 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:43:36.947 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 16:43:36.947 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 16:43:36.947 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:43:36.947 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 16:43:36.947 | INFO | ------------------------------------------ | 20260505 16:43:36.947 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:43:36.947 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 16:43:36.947 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:43:36.947 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:43:36.947 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.52e+01, 1.37e+01 / 2.41e+00 | 20260505 16:43:36.947 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 1.02e-01, 1.13e-02 / 4.28e-03 | 20260505 16:43:36.947 | INFO | PerfReport: 11.78%, Baseline -- 3.74e+00 / 4.47e+00, 5.78e-01 / 2.05e-01 | 20260505 16:43:36.947 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.51e+00, 1.95e-01 / 1.11e-01 | 20260505 16:43:36.947 | INFO | PerfReport: 15.93%, DME -- 4.04e+01 / 4.49e+01, 4.18e+00 / 2.43e+00 | 20260505 16:43:36.947 | INFO | PerfReport: 56.78%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.51e+00 / 1.55e+00 | 20260505 16:43:36.947 | INFO | PerfReport: 7.33%, PulseAccumulating -- 2.32e+00 / 2.91e+00, 4.45e-01 / 2.06e-01 | 20260505 16:43:36.947 | INFO | PerfReport: 3.40%, Metrics -- 1.08e+00 / 1.42e+00, 2.99e-01 / 1.36e-01 | 20260505 16:43:36.947 | INFO | PerfReport: 1.19%, PulseSerializing -- 3.78e-01 / 4.36e-01, 6.73e-02 / 3.64e-02 | 20260505 16:43:36.947 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 16:43:36.947 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:43:36.947 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:43:36.947 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260505 16:43:36.947 | INFO | PerfReport: 22.45%, Upload -- 2.25e+01 / 0.00e+00, 7.81e+00 / 0.00e+00 | 20260505 16:43:36.947 | INFO | PerfReport: 21.83%, Download -- 2.19e+01 / 0.00e+00, 7.86e+00 / 0.00e+00 | 20260505 16:43:36.947 | INFO | PerfReport: 31.67%, Compute -- 3.17e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 16:43:36.947 | INFO | PerfReport: 2.21%, LockUpload -- 2.21e+00 / 0.00e+00, 7.88e+00 / 0.00e+00 | 20260505 16:43:36.947 | INFO | PerfReport: 1.45%, LockDownload -- 1.45e+00 / 0.00e+00, 3.00e+00 / 0.00e+00 | 20260505 16:43:36.947 | INFO | PerfReport: 20.38%, LockCompute -- 2.04e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 16:43:36.947 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.71e-02 / 0.00e+00 | 20260505 16:43:37.077 | INFO | Starting a round of Baz IO | 20260505 16:43:40.282 | INFO | Populating RTMetrics report | 20260505 16:43:40.283 | INFO | RTMetrics report populated | 20260505 16:43:40.283 | INFO | Opening tmp RTMetrics json file | 20260505 16:43:40.294 | INFO | Serializing RTMetrics to json | 20260505 16:43:40.297 | INFO | Serialization done, swapping tmp file now... | 20260505 16:43:40.300 | INFO | RTMetrics done | 20260505 16:43:40.387 | INFO | Finished a round of Baz IO: Wrote 2.17 GiB in 3.31 seconds | 20260505 16:43:40.387 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.19% / 0.00% / 77.81% | 20260505 16:43:40.388 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.349251% / 0% / 0.000107288% | 20260505 16:43:40.388 | INFO | Baz IO overall: | 20260505 16:43:40.388 | INFO | BytesWritten : 2806953788903 | 20260505 16:43:40.388 | INFO | NumEvents : 1331979287186 | 20260505 16:43:42.093 | INFO | Analyzing chunk frames = [4734976,4735488) | 20260505 16:43:42.093 | INFO | MemInfo: { | 20260505 16:43:42.093 | INFO | "gpu" : " 17 GB", | 20260505 16:43:42.093 | INFO | "huge" : "", | 20260505 16:43:42.093 | INFO | "huge_shared" : " 27 GB", | 20260505 16:43:42.093 | INFO | "malloc" : " 14 GB", | 20260505 16:43:42.093 | INFO | "pinned" : " 29 GB" | 20260505 16:43:42.093 | INFO | } | 20260505 16:43:42.093 | INFO | Perf report for the last 5.14756 seconds: | 20260505 16:43:42.093 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:43:42.093 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:42.093 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:42.093 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:42.093 | INFO | ANALYSIS : 63.7%, 3.0, 99.2ms, 0 | 20260505 16:43:42.093 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:42.093 | INFO | RT_METRICS : 44.7%, 1.0, 23.5ms, 0 | 20260505 16:43:42.093 | INFO | PRE_HQ : 60.5%, 1.8, 56.3ms, 0 | 20260505 16:43:42.093 | INFO | BAZWRITER : 63.6%, 3.9, 127.9ms, 0 | 20260505 16:43:42.093 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 16:43:42.094 | INFO | Uploaded 36GiB to GPU in 2.15675s (16.6918GiB/s) | 20260505 16:43:42.094 | INFO | Downloaded 38GiB to GPU in 2.25475s (16.8533GiB/s) | 20260505 16:43:47.230 | INFO | Analyzing chunk frames = [4735488,4736000) | 20260505 16:43:47.230 | INFO | MemInfo: { | 20260505 16:43:47.230 | INFO | "gpu" : " 17 GB", | 20260505 16:43:47.230 | INFO | "huge" : "", | 20260505 16:43:47.230 | INFO | "huge_shared" : " 27 GB", | 20260505 16:43:47.230 | INFO | "malloc" : " 14 GB", | 20260505 16:43:47.230 | INFO | "pinned" : " 29 GB" | 20260505 16:43:47.230 | INFO | } | 20260505 16:43:47.230 | INFO | Perf report for the last 5.13702 seconds: | 20260505 16:43:47.230 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:43:47.230 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:47.230 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:47.230 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:47.230 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260505 16:43:47.230 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:47.230 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:47.230 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:43:47.230 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:47.230 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:47.231 | INFO | Uploaded 36GiB to GPU in 2.13723s (16.8442GiB/s) | 20260505 16:43:47.231 | INFO | Downloaded 34GiB to GPU in 2.029s (16.757GiB/s) | 20260505 16:43:52.364 | INFO | Analyzing chunk frames = [4736000,4736512) | 20260505 16:43:52.364 | INFO | MemInfo: { | 20260505 16:43:52.364 | INFO | "gpu" : " 17 GB", | 20260505 16:43:52.364 | INFO | "huge" : "", | 20260505 16:43:52.364 | INFO | "huge_shared" : " 27 GB", | 20260505 16:43:52.364 | INFO | "malloc" : " 14 GB", | 20260505 16:43:52.364 | INFO | "pinned" : " 29 GB" | 20260505 16:43:52.365 | INFO | } | 20260505 16:43:52.365 | INFO | Perf report for the last 5.13458 seconds: | 20260505 16:43:52.365 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:43:52.365 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:52.365 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:52.365 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:52.365 | INFO | ANALYSIS : 62.9%, 3.0, 97.7ms, 0 | 20260505 16:43:52.365 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:52.365 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:52.365 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:43:52.365 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:52.365 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:52.365 | INFO | Uploaded 36GiB to GPU in 2.19986s (16.3647GiB/s) | 20260505 16:43:52.365 | INFO | Downloaded 34GiB to GPU in 2.02809s (16.7645GiB/s) | 20260505 16:43:57.507 | INFO | Analyzing chunk frames = [4736512,4737024) | 20260505 16:43:57.507 | INFO | MemInfo: { | 20260505 16:43:57.507 | INFO | "gpu" : " 17 GB", | 20260505 16:43:57.507 | INFO | "huge" : "", | 20260505 16:43:57.507 | INFO | "huge_shared" : " 27 GB", | 20260505 16:43:57.507 | INFO | "malloc" : " 14 GB", | 20260505 16:43:57.507 | INFO | "pinned" : " 29 GB" | 20260505 16:43:57.507 | INFO | } | 20260505 16:43:57.507 | INFO | Perf report for the last 5.14233 seconds: | 20260505 16:43:57.507 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:43:57.507 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:57.507 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:57.507 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:57.507 | INFO | ANALYSIS : 62.5%, 3.0, 97.1ms, 0 | 20260505 16:43:57.507 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:57.507 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:57.507 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260505 16:43:57.507 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:57.507 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:43:57.508 | INFO | Uploaded 36GiB to GPU in 2.18067s (16.5087GiB/s) | 20260505 16:43:57.508 | INFO | Downloaded 34GiB to GPU in 2.13428s (15.9304GiB/s) | 20260505 16:44:02.646 | INFO | Analyzing chunk frames = [4737024,4737536) | 20260505 16:44:02.646 | INFO | MemInfo: { | 20260505 16:44:02.646 | INFO | "gpu" : " 17 GB", | 20260505 16:44:02.646 | INFO | "huge" : "", | 20260505 16:44:02.646 | INFO | "huge_shared" : " 27 GB", | 20260505 16:44:02.646 | INFO | "malloc" : " 14 GB", | 20260505 16:44:02.646 | INFO | "pinned" : " 29 GB" | 20260505 16:44:02.646 | INFO | } | 20260505 16:44:02.646 | INFO | Perf report for the last 5.13878 seconds: | 20260505 16:44:02.646 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:44:02.646 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:02.646 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:02.646 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:02.646 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 16:44:02.646 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:02.646 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:02.646 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 16:44:02.646 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:02.646 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:02.646 | INFO | Uploaded 36GiB to GPU in 2.16831s (16.6028GiB/s) | 20260505 16:44:02.646 | INFO | Downloaded 34GiB to GPU in 2.13232s (15.9451GiB/s) | 20260505 16:44:07.794 | INFO | Analyzing chunk frames = [4737536,4738048) | 20260505 16:44:07.794 | INFO | MemInfo: { | 20260505 16:44:07.794 | INFO | "gpu" : " 17 GB", | 20260505 16:44:07.794 | INFO | "huge" : "", | 20260505 16:44:07.794 | INFO | "huge_shared" : " 27 GB", | 20260505 16:44:07.794 | INFO | "malloc" : " 14 GB", | 20260505 16:44:07.794 | INFO | "pinned" : " 29 GB" | 20260505 16:44:07.794 | INFO | } | 20260505 16:44:07.794 | INFO | Perf report for the last 5.14818 seconds: | 20260505 16:44:07.794 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:44:07.794 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:07.794 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:07.794 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:07.794 | INFO | ANALYSIS : 62.3%, 3.0, 97.5ms, 0 | 20260505 16:44:07.794 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:07.794 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:07.794 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 16:44:07.794 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:07.794 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:07.795 | INFO | Uploaded 36GiB to GPU in 2.13471s (16.8641GiB/s) | 20260505 16:44:07.795 | INFO | Downloaded 34GiB to GPU in 2.03714s (16.69GiB/s) | 20260505 16:44:12.932 | INFO | Analyzing chunk frames = [4738048,4738560) | 20260505 16:44:12.932 | INFO | MemInfo: { | 20260505 16:44:12.932 | INFO | "gpu" : " 17 GB", | 20260505 16:44:12.932 | INFO | "huge" : "", | 20260505 16:44:12.932 | INFO | "huge_shared" : " 27 GB", | 20260505 16:44:12.932 | INFO | "malloc" : " 14 GB", | 20260505 16:44:12.932 | INFO | "pinned" : " 29 GB" | 20260505 16:44:12.932 | INFO | } | 20260505 16:44:12.932 | INFO | Perf report for the last 5.13771 seconds: | 20260505 16:44:12.932 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:44:12.932 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:12.932 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:12.932 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:12.932 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 16:44:12.932 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:12.932 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:12.932 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 16:44:12.932 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:12.932 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:12.932 | INFO | Uploaded 36GiB to GPU in 2.12815s (16.9161GiB/s) | 20260505 16:44:12.932 | INFO | Downloaded 34GiB to GPU in 2.14844s (15.8254GiB/s) | 20260505 16:44:18.060 | INFO | Analyzing chunk frames = [4738560,4739072) | 20260505 16:44:18.060 | INFO | MemInfo: { | 20260505 16:44:18.060 | INFO | "gpu" : " 17 GB", | 20260505 16:44:18.060 | INFO | "huge" : "", | 20260505 16:44:18.060 | INFO | "huge_shared" : " 27 GB", | 20260505 16:44:18.060 | INFO | "malloc" : " 14 GB", | 20260505 16:44:18.060 | INFO | "pinned" : " 29 GB" | 20260505 16:44:18.060 | INFO | } | 20260505 16:44:18.060 | INFO | Perf report for the last 5.12813 seconds: | 20260505 16:44:18.060 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:44:18.060 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:18.060 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:18.060 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:18.060 | INFO | ANALYSIS : 63.2%, 3.0, 98.4ms, 0 | 20260505 16:44:18.060 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:18.060 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:18.060 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 16:44:18.060 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:18.060 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:18.060 | INFO | Uploaded 36GiB to GPU in 2.19657s (16.3892GiB/s) | 20260505 16:44:18.060 | INFO | Downloaded 34GiB to GPU in 2.02186s (16.8162GiB/s) | 20260505 16:44:18.190 | INFO | Starting a round of Baz IO | 20260505 16:44:21.399 | INFO | Populating RTMetrics report | 20260505 16:44:21.400 | INFO | RTMetrics report populated | 20260505 16:44:21.400 | INFO | Opening tmp RTMetrics json file | 20260505 16:44:21.415 | INFO | Serializing RTMetrics to json | 20260505 16:44:21.416 | INFO | Serialization done, swapping tmp file now... | 20260505 16:44:21.417 | INFO | RTMetrics done | 20260505 16:44:21.500 | INFO | Finished a round of Baz IO: Wrote 2.166 GiB in 3.309 seconds | 20260505 16:44:21.500 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.19% / 0.00% / 77.81% | 20260505 16:44:21.500 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.346851% / 3.97364e-06% / 6.75519e-05% | 20260505 16:44:21.500 | INFO | Baz IO overall: | 20260505 16:44:21.500 | INFO | BytesWritten : 2809279673972 | 20260505 16:44:21.500 | INFO | NumEvents : 1333068387331 | 20260505 16:44:23.199 | INFO | Analyzing chunk frames = [4739072,4739584) | 20260505 16:44:23.199 | INFO | MemInfo: { | 20260505 16:44:23.199 | INFO | "gpu" : " 17 GB", | 20260505 16:44:23.199 | INFO | "huge" : "", | 20260505 16:44:23.199 | INFO | "huge_shared" : " 27 GB", | 20260505 16:44:23.200 | INFO | "malloc" : " 14 GB", | 20260505 16:44:23.200 | INFO | "pinned" : " 29 GB" | 20260505 16:44:23.200 | INFO | } | 20260505 16:44:23.200 | INFO | Perf report for the last 5.13942 seconds: | 20260505 16:44:23.200 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:44:23.200 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:23.200 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:23.200 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:23.200 | INFO | ANALYSIS : 63.9%, 3.0, 99.2ms, 0 | 20260505 16:44:23.200 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:23.200 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260505 16:44:23.200 | INFO | PRE_HQ : 60.9%, 1.8, 56.7ms, 0 | 20260505 16:44:23.200 | INFO | BAZWRITER : 63.7%, 4.0, 129.4ms, 0 | 20260505 16:44:23.200 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 16:44:23.200 | INFO | Uploaded 36GiB to GPU in 2.19126s (16.4289GiB/s) | 20260505 16:44:23.200 | INFO | Downloaded 38GiB to GPU in 2.25538s (16.8486GiB/s) | 20260505 16:44:28.349 | INFO | Analyzing chunk frames = [4739584,4740096) | 20260505 16:44:28.349 | INFO | MemInfo: { | 20260505 16:44:28.349 | INFO | "gpu" : " 17 GB", | 20260505 16:44:28.349 | INFO | "huge" : "", | 20260505 16:44:28.349 | INFO | "huge_shared" : " 27 GB", | 20260505 16:44:28.349 | INFO | "malloc" : " 14 GB", | 20260505 16:44:28.349 | INFO | "pinned" : " 29 GB" | 20260505 16:44:28.349 | INFO | } | 20260505 16:44:28.349 | INFO | Perf report for the last 5.14961 seconds: | 20260505 16:44:28.349 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:44:28.349 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:28.349 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:28.349 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:28.349 | INFO | ANALYSIS : 63.3%, 3.0, 99.0ms, 0 | 20260505 16:44:28.349 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:28.349 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:28.349 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 16:44:28.349 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:28.349 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:28.349 | INFO | Uploaded 36GiB to GPU in 2.20452s (16.3301GiB/s) | 20260505 16:44:28.350 | INFO | Downloaded 34GiB to GPU in 2.05096s (16.5776GiB/s) | 20260505 16:44:33.491 | INFO | Analyzing chunk frames = [4740096,4740608) | 20260505 16:44:33.491 | INFO | MemInfo: { | 20260505 16:44:33.491 | INFO | "gpu" : " 17 GB", | 20260505 16:44:33.491 | INFO | "huge" : "", | 20260505 16:44:33.491 | INFO | "huge_shared" : " 27 GB", | 20260505 16:44:33.491 | INFO | "malloc" : " 14 GB", | 20260505 16:44:33.491 | INFO | "pinned" : " 29 GB" | 20260505 16:44:33.491 | INFO | } | 20260505 16:44:33.491 | INFO | Perf report for the last 5.14161 seconds: | 20260505 16:44:33.491 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:44:33.491 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:33.491 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:33.491 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:33.491 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 16:44:33.491 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:33.491 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:33.491 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:44:33.491 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:33.491 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:33.491 | INFO | Uploaded 36GiB to GPU in 2.20445s (16.3306GiB/s) | 20260505 16:44:33.491 | INFO | Downloaded 34GiB to GPU in 2.05159s (16.5725GiB/s) | 20260505 16:44:38.628 | INFO | Analyzing chunk frames = [4740608,4741120) | 20260505 16:44:38.628 | INFO | MemInfo: { | 20260505 16:44:38.628 | INFO | "gpu" : " 17 GB", | 20260505 16:44:38.628 | INFO | "huge" : "", | 20260505 16:44:38.628 | INFO | "huge_shared" : " 27 GB", | 20260505 16:44:38.628 | INFO | "malloc" : " 14 GB", | 20260505 16:44:38.628 | INFO | "pinned" : " 29 GB" | 20260505 16:44:38.628 | INFO | } | 20260505 16:44:38.628 | INFO | Perf report for the last 5.13727 seconds: | 20260505 16:44:38.628 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:44:38.628 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:38.628 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:38.628 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:38.628 | INFO | ANALYSIS : 62.7%, 3.0, 97.4ms, 0 | 20260505 16:44:38.628 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:38.628 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:38.628 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 16:44:38.628 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:38.628 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:38.628 | INFO | Uploaded 36GiB to GPU in 2.21919s (16.2222GiB/s) | 20260505 16:44:38.628 | INFO | Downloaded 34GiB to GPU in 2.12613s (15.9915GiB/s) | 20260505 16:44:43.759 | INFO | Analyzing chunk frames = [4741120,4741632) | 20260505 16:44:43.759 | INFO | MemInfo: { | 20260505 16:44:43.759 | INFO | "gpu" : " 17 GB", | 20260505 16:44:43.759 | INFO | "huge" : "", | 20260505 16:44:43.759 | INFO | "huge_shared" : " 27 GB", | 20260505 16:44:43.759 | INFO | "malloc" : " 14 GB", | 20260505 16:44:43.759 | INFO | "pinned" : " 29 GB" | 20260505 16:44:43.759 | INFO | } | 20260505 16:44:43.760 | INFO | Perf report for the last 5.1314 seconds: | 20260505 16:44:43.760 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:44:43.760 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:43.760 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:43.760 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:43.760 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260505 16:44:43.760 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:43.760 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:43.760 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 16:44:43.760 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:43.760 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:43.760 | INFO | Uploaded 36GiB to GPU in 2.14746s (16.764GiB/s) | 20260505 16:44:43.760 | INFO | Downloaded 34GiB to GPU in 2.05425s (16.551GiB/s) | 20260505 16:44:48.899 | INFO | Analyzing chunk frames = [4741632,4742144) | 20260505 16:44:48.899 | INFO | MemInfo: { | 20260505 16:44:48.899 | INFO | "gpu" : " 17 GB", | 20260505 16:44:48.899 | INFO | "huge" : "", | 20260505 16:44:48.899 | INFO | "huge_shared" : " 27 GB", | 20260505 16:44:48.899 | INFO | "malloc" : " 14 GB", | 20260505 16:44:48.899 | INFO | "pinned" : " 29 GB" | 20260505 16:44:48.899 | INFO | } | 20260505 16:44:48.899 | INFO | Perf report for the last 5.13973 seconds: | 20260505 16:44:48.899 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:44:48.899 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:48.899 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:48.899 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:48.899 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260505 16:44:48.899 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:48.899 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:48.899 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 16:44:48.899 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:48.899 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:48.900 | INFO | Uploaded 36GiB to GPU in 2.14083s (16.8159GiB/s) | 20260505 16:44:48.900 | INFO | Downloaded 34GiB to GPU in 2.03901s (16.6747GiB/s) | 20260505 16:44:54.040 | INFO | Analyzing chunk frames = [4742144,4742656) | 20260505 16:44:54.040 | INFO | MemInfo: { | 20260505 16:44:54.040 | INFO | "gpu" : " 17 GB", | 20260505 16:44:54.040 | INFO | "huge" : "", | 20260505 16:44:54.040 | INFO | "huge_shared" : " 27 GB", | 20260505 16:44:54.040 | INFO | "malloc" : " 14 GB", | 20260505 16:44:54.040 | INFO | "pinned" : " 29 GB" | 20260505 16:44:54.040 | INFO | } | 20260505 16:44:54.040 | INFO | Perf report for the last 5.14093 seconds: | 20260505 16:44:54.040 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:44:54.040 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:54.040 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:54.040 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:54.040 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 16:44:54.040 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:54.040 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:54.040 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 16:44:54.040 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:54.040 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:54.041 | INFO | Uploaded 36GiB to GPU in 2.1306s (16.8967GiB/s) | 20260505 16:44:54.041 | INFO | Downloaded 34GiB to GPU in 2.1726s (15.6494GiB/s) | 20260505 16:44:59.182 | INFO | Analyzing chunk frames = [4742656,4743168) | 20260505 16:44:59.182 | INFO | MemInfo: { | 20260505 16:44:59.182 | INFO | "gpu" : " 17 GB", | 20260505 16:44:59.182 | INFO | "huge" : "", | 20260505 16:44:59.182 | INFO | "huge_shared" : " 27 GB", | 20260505 16:44:59.182 | INFO | "malloc" : " 14 GB", | 20260505 16:44:59.183 | INFO | "pinned" : " 29 GB" | 20260505 16:44:59.183 | INFO | } | 20260505 16:44:59.183 | INFO | Perf report for the last 5.14249 seconds: | 20260505 16:44:59.183 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:44:59.183 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:59.183 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:59.183 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:59.183 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 16:44:59.183 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:59.183 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:59.183 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 16:44:59.183 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:59.183 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:44:59.183 | INFO | Uploaded 36GiB to GPU in 2.15884s (16.6756GiB/s) | 20260505 16:44:59.183 | INFO | Downloaded 34GiB to GPU in 2.02438s (16.7953GiB/s) | 20260505 16:44:59.183 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 16:44:59.183 | INFO | Total mem usage: 15.3 GiB | 20260505 16:44:59.183 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 16:44:59.183 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.92%) | 20260505 16:44:59.183 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 16:44:59.183 | INFO | High Water Breakdown: | 20260505 16:44:59.183 | INFO | 61% : 8.67 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 16:44:59.183 | INFO | 38.8% : 5.51 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 16:44:59.183 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 16:44:59.183 | INFO | ------------------------------------------ | 20260505 16:44:59.183 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:44:59.183 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 16:44:59.183 | INFO | Total mem usage: 26.5 GiB | 20260505 16:44:59.183 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 16:44:59.183 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 16:44:59.183 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 16:44:59.183 | INFO | High Water Breakdown: | 20260505 16:44:59.183 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:44:59.183 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:44:59.183 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:44:59.183 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:44:59.183 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:44:59.183 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:44:59.183 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 16:44:59.183 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 16:44:59.183 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:44:59.183 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 16:44:59.183 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 16:44:59.183 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 16:44:59.183 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 16:44:59.183 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 16:44:59.183 | INFO | ------------------------------------------ | 20260505 16:44:59.183 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:44:59.183 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 16:44:59.183 | INFO | Total mem usage: 25.6 GiB | 20260505 16:44:59.183 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 16:44:59.183 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 16:44:59.183 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 16:44:59.183 | INFO | High Water Breakdown: | 20260505 16:44:59.184 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 16:44:59.184 | INFO | ------------------------------------------ | 20260505 16:44:59.184 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:44:59.184 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 16:44:59.184 | INFO | Total mem usage: 15.7 GiB | 20260505 16:44:59.184 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 16:44:59.184 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 16:44:59.184 | INFO | Unused Cache: 914 MiB (Waste: 5.7%) | 20260505 16:44:59.184 | INFO | High Water Breakdown: | 20260505 16:44:59.184 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 16:44:59.184 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 16:44:59.184 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 16:44:59.184 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 16:44:59.184 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 16:44:59.184 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 16:44:59.184 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 16:44:59.184 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 16:44:59.184 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 16:44:59.184 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:44:59.184 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 16:44:59.184 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:44:59.184 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 16:44:59.184 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 16:44:59.184 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 16:44:59.184 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 16:44:59.184 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:44:59.184 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 16:44:59.184 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:44:59.184 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:44:59.184 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:44:59.184 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 16:44:59.184 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 16:44:59.184 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:44:59.184 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 16:44:59.184 | INFO | ------------------------------------------ | 20260505 16:44:59.184 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:44:59.184 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 16:44:59.184 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:44:59.184 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:44:59.184 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.48e+01, 1.35e+01 / 2.07e+00 | 20260505 16:44:59.184 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 1.03e-01, 1.13e-02 / 5.63e-03 | 20260505 16:44:59.184 | INFO | PerfReport: 11.80%, Baseline -- 3.73e+00 / 4.54e+00, 5.75e-01 / 1.26e-01 | 20260505 16:44:59.184 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.54e+00, 1.94e-01 / 8.17e-02 | 20260505 16:44:59.184 | INFO | PerfReport: 15.90%, DME -- 4.02e+01 / 4.45e+01, 4.07e+00 / 2.12e+00 | 20260505 16:44:59.184 | INFO | PerfReport: 56.72%, FrameLabeling -- 1.79e+01 / 1.93e+01, 1.45e+00 / 1.20e+00 | 20260505 16:44:59.184 | INFO | PerfReport: 7.40%, PulseAccumulating -- 2.34e+00 / 2.93e+00, 4.47e-01 / 1.17e-01 | 20260505 16:44:59.184 | INFO | PerfReport: 3.42%, Metrics -- 1.08e+00 / 1.43e+00, 2.98e-01 / 1.82e-01 | 20260505 16:44:59.184 | INFO | PerfReport: 1.19%, PulseSerializing -- 3.76e-01 / 4.37e-01, 6.63e-02 / 3.28e-02 | 20260505 16:44:59.184 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 16:44:59.184 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:44:59.184 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:44:59.184 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.56e+01 / 0.00e+00 | 20260505 16:44:59.184 | INFO | PerfReport: 22.55%, Upload -- 2.26e+01 / 0.00e+00, 7.89e+00 / 0.00e+00 | 20260505 16:44:59.184 | INFO | PerfReport: 21.82%, Download -- 2.19e+01 / 0.00e+00, 7.88e+00 / 0.00e+00 | 20260505 16:44:59.184 | INFO | PerfReport: 31.59%, Compute -- 3.16e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 16:44:59.184 | INFO | PerfReport: 2.31%, LockUpload -- 2.31e+00 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260505 16:44:59.184 | INFO | PerfReport: 1.44%, LockDownload -- 1.44e+00 / 0.00e+00, 3.00e+00 / 0.00e+00 | 20260505 16:44:59.184 | INFO | PerfReport: 20.28%, LockCompute -- 2.03e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 16:44:59.184 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.89e-02 / 0.00e+00 | 20260505 16:44:59.322 | INFO | Starting a round of Baz IO | 20260505 16:45:02.513 | INFO | Populating RTMetrics report | 20260505 16:45:02.514 | INFO | RTMetrics report populated | 20260505 16:45:02.514 | INFO | Opening tmp RTMetrics json file | 20260505 16:45:02.530 | INFO | Serializing RTMetrics to json | 20260505 16:45:02.540 | INFO | Serialization done, swapping tmp file now... | 20260505 16:45:02.541 | INFO | RTMetrics done | 20260505 16:45:02.605 | INFO | Finished a round of Baz IO: Wrote 2.163 GiB in 3.282 seconds | 20260505 16:45:02.605 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.19% / 0.00% / 77.81% | 20260505 16:45:02.605 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.344984% / 0% / 6.75519e-05% | 20260505 16:45:02.605 | INFO | Baz IO overall: | 20260505 16:45:02.605 | INFO | BytesWritten : 2811601644887 | 20260505 16:45:02.605 | INFO | NumEvents : 1334155459253 | 20260505 16:45:04.316 | INFO | Analyzing chunk frames = [4743168,4743680) | 20260505 16:45:04.316 | INFO | MemInfo: { | 20260505 16:45:04.316 | INFO | "gpu" : " 17 GB", | 20260505 16:45:04.316 | INFO | "huge" : "", | 20260505 16:45:04.316 | INFO | "huge_shared" : " 27 GB", | 20260505 16:45:04.316 | INFO | "malloc" : " 14 GB", | 20260505 16:45:04.316 | INFO | "pinned" : " 29 GB" | 20260505 16:45:04.316 | INFO | } | 20260505 16:45:04.316 | INFO | Perf report for the last 5.13377 seconds: | 20260505 16:45:04.317 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:45:04.317 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:04.317 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:04.317 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:04.317 | INFO | ANALYSIS : 63.8%, 3.0, 99.2ms, 0 | 20260505 16:45:04.317 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:04.317 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260505 16:45:04.317 | INFO | PRE_HQ : 60.4%, 1.8, 56.7ms, 0 | 20260505 16:45:04.317 | INFO | BAZWRITER : 63.4%, 3.9, 127.0ms, 0 | 20260505 16:45:04.317 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 16:45:04.317 | INFO | Uploaded 36GiB to GPU in 2.15432s (16.7106GiB/s) | 20260505 16:45:04.317 | INFO | Downloaded 38GiB to GPU in 2.25346s (16.863GiB/s) | 20260505 16:45:09.456 | INFO | Analyzing chunk frames = [4743680,4744192) | 20260505 16:45:09.456 | INFO | MemInfo: { | 20260505 16:45:09.456 | INFO | "gpu" : " 17 GB", | 20260505 16:45:09.456 | INFO | "huge" : "", | 20260505 16:45:09.456 | INFO | "huge_shared" : " 27 GB", | 20260505 16:45:09.456 | INFO | "malloc" : " 14 GB", | 20260505 16:45:09.456 | INFO | "pinned" : " 29 GB" | 20260505 16:45:09.456 | INFO | } | 20260505 16:45:09.456 | INFO | Perf report for the last 5.13979 seconds: | 20260505 16:45:09.456 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:45:09.456 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:09.456 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:09.456 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:09.456 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 16:45:09.456 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:09.456 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:09.456 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:45:09.456 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:09.456 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:09.457 | INFO | Uploaded 36GiB to GPU in 2.20261s (16.3443GiB/s) | 20260505 16:45:09.457 | INFO | Downloaded 34GiB to GPU in 2.05561s (16.5401GiB/s) | 20260505 16:45:14.604 | INFO | Analyzing chunk frames = [4744192,4744704) | 20260505 16:45:14.604 | INFO | MemInfo: { | 20260505 16:45:14.604 | INFO | "gpu" : " 17 GB", | 20260505 16:45:14.604 | INFO | "huge" : "", | 20260505 16:45:14.604 | INFO | "huge_shared" : " 27 GB", | 20260505 16:45:14.604 | INFO | "malloc" : " 14 GB", | 20260505 16:45:14.604 | INFO | "pinned" : " 29 GB" | 20260505 16:45:14.604 | INFO | } | 20260505 16:45:14.605 | INFO | Perf report for the last 5.14823 seconds: | 20260505 16:45:14.605 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:45:14.605 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:14.605 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:14.605 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:14.605 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260505 16:45:14.605 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:14.605 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:14.605 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:45:14.605 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:14.605 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:14.605 | INFO | Uploaded 36GiB to GPU in 2.17966s (16.5164GiB/s) | 20260505 16:45:14.605 | INFO | Downloaded 34GiB to GPU in 2.0288s (16.7587GiB/s) | 20260505 16:45:19.746 | INFO | Analyzing chunk frames = [4744704,4745216) | 20260505 16:45:19.746 | INFO | MemInfo: { | 20260505 16:45:19.746 | INFO | "gpu" : " 17 GB", | 20260505 16:45:19.746 | INFO | "huge" : "", | 20260505 16:45:19.746 | INFO | "huge_shared" : " 27 GB", | 20260505 16:45:19.746 | INFO | "malloc" : " 14 GB", | 20260505 16:45:19.746 | INFO | "pinned" : " 29 GB" | 20260505 16:45:19.747 | INFO | } | 20260505 16:45:19.747 | INFO | Perf report for the last 5.14213 seconds: | 20260505 16:45:19.747 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:45:19.747 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:19.747 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:19.747 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:19.747 | INFO | ANALYSIS : 62.7%, 3.0, 97.4ms, 0 | 20260505 16:45:19.747 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:19.747 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:19.747 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 16:45:19.747 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:19.747 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:19.747 | INFO | Uploaded 36GiB to GPU in 2.21731s (16.2359GiB/s) | 20260505 16:45:19.747 | INFO | Downloaded 34GiB to GPU in 2.14036s (15.8852GiB/s) | 20260505 16:45:24.880 | INFO | Analyzing chunk frames = [4745216,4745728) | 20260505 16:45:24.880 | INFO | MemInfo: { | 20260505 16:45:24.880 | INFO | "gpu" : " 17 GB", | 20260505 16:45:24.880 | INFO | "huge" : "", | 20260505 16:45:24.880 | INFO | "huge_shared" : " 27 GB", | 20260505 16:45:24.880 | INFO | "malloc" : " 14 GB", | 20260505 16:45:24.880 | INFO | "pinned" : " 29 GB" | 20260505 16:45:24.880 | INFO | } | 20260505 16:45:24.880 | INFO | Perf report for the last 5.13315 seconds: | 20260505 16:45:24.880 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:45:24.880 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:24.880 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:24.880 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:24.880 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 16:45:24.880 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:24.880 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:24.880 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:45:24.880 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:24.880 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:24.880 | INFO | Uploaded 36GiB to GPU in 2.14664s (16.7704GiB/s) | 20260505 16:45:24.880 | INFO | Downloaded 34GiB to GPU in 2.05533s (16.5424GiB/s) | 20260505 16:45:30.009 | INFO | Analyzing chunk frames = [4745728,4746240) | 20260505 16:45:30.009 | INFO | MemInfo: { | 20260505 16:45:30.009 | INFO | "gpu" : " 17 GB", | 20260505 16:45:30.009 | INFO | "huge" : "", | 20260505 16:45:30.009 | INFO | "huge_shared" : " 27 GB", | 20260505 16:45:30.010 | INFO | "malloc" : " 14 GB", | 20260505 16:45:30.010 | INFO | "pinned" : " 29 GB" | 20260505 16:45:30.010 | INFO | } | 20260505 16:45:30.010 | INFO | Perf report for the last 5.12989 seconds: | 20260505 16:45:30.010 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:45:30.010 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:30.010 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:30.010 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:30.010 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260505 16:45:30.010 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:30.010 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:30.010 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:45:30.010 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:30.010 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:30.010 | INFO | Uploaded 36GiB to GPU in 2.14063s (16.8175GiB/s) | 20260505 16:45:30.010 | INFO | Downloaded 34GiB to GPU in 2.03881s (16.6764GiB/s) | 20260505 16:45:35.153 | INFO | Analyzing chunk frames = [4746240,4746752) | 20260505 16:45:35.153 | INFO | MemInfo: { | 20260505 16:45:35.153 | INFO | "gpu" : " 17 GB", | 20260505 16:45:35.153 | INFO | "huge" : "", | 20260505 16:45:35.153 | INFO | "huge_shared" : " 27 GB", | 20260505 16:45:35.153 | INFO | "malloc" : " 14 GB", | 20260505 16:45:35.153 | INFO | "pinned" : " 29 GB" | 20260505 16:45:35.153 | INFO | } | 20260505 16:45:35.153 | INFO | Perf report for the last 5.14371 seconds: | 20260505 16:45:35.153 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:45:35.153 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:35.153 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:35.153 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:35.154 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 16:45:35.154 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:35.154 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:35.154 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 16:45:35.154 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:35.154 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:35.154 | INFO | Uploaded 36GiB to GPU in 2.12163s (16.9681GiB/s) | 20260505 16:45:35.154 | INFO | Downloaded 34GiB to GPU in 2.17917s (15.6022GiB/s) | 20260505 16:45:40.296 | INFO | Analyzing chunk frames = [4746752,4747264) | 20260505 16:45:40.296 | INFO | MemInfo: { | 20260505 16:45:40.296 | INFO | "gpu" : " 17 GB", | 20260505 16:45:40.296 | INFO | "huge" : "", | 20260505 16:45:40.296 | INFO | "huge_shared" : " 27 GB", | 20260505 16:45:40.296 | INFO | "malloc" : " 14 GB", | 20260505 16:45:40.296 | INFO | "pinned" : " 29 GB" | 20260505 16:45:40.296 | INFO | } | 20260505 16:45:40.296 | INFO | Perf report for the last 5.14249 seconds: | 20260505 16:45:40.296 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:45:40.296 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:40.296 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:40.296 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:40.296 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 16:45:40.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:40.296 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:40.296 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 16:45:40.296 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:40.296 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:40.296 | INFO | Uploaded 36GiB to GPU in 2.17259s (16.5701GiB/s) | 20260505 16:45:40.296 | INFO | Downloaded 34GiB to GPU in 2.0356s (16.7027GiB/s) | 20260505 16:45:40.435 | INFO | Starting a round of Baz IO | 20260505 16:45:43.622 | INFO | Populating RTMetrics report | 20260505 16:45:43.622 | INFO | RTMetrics report populated | 20260505 16:45:43.622 | INFO | Opening tmp RTMetrics json file | 20260505 16:45:43.631 | INFO | Serializing RTMetrics to json | 20260505 16:45:43.634 | INFO | Serialization done, swapping tmp file now... | 20260505 16:45:43.634 | INFO | RTMetrics done | 20260505 16:45:43.720 | INFO | Finished a round of Baz IO: Wrote 2.163 GiB in 3.285 seconds | 20260505 16:45:43.720 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.18% / 0.00% / 77.81% | 20260505 16:45:43.721 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.345548% / 3.97364e-06% / 6.75519e-05% | 20260505 16:45:43.721 | INFO | Baz IO overall: | 20260505 16:45:43.721 | INFO | BytesWritten : 2813923649491 | 20260505 16:45:43.721 | INFO | NumEvents : 1335243181025 | 20260505 16:45:45.444 | INFO | Analyzing chunk frames = [4747264,4747776) | 20260505 16:45:45.444 | INFO | MemInfo: { | 20260505 16:45:45.444 | INFO | "gpu" : " 17 GB", | 20260505 16:45:45.444 | INFO | "huge" : "", | 20260505 16:45:45.444 | INFO | "huge_shared" : " 27 GB", | 20260505 16:45:45.444 | INFO | "malloc" : " 14 GB", | 20260505 16:45:45.444 | INFO | "pinned" : " 29 GB" | 20260505 16:45:45.444 | INFO | } | 20260505 16:45:45.444 | INFO | Perf report for the last 5.14798 seconds: | 20260505 16:45:45.444 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:45:45.444 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:45.444 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:45.444 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:45.444 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 16:45:45.444 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:45.444 | INFO | RT_METRICS : 44.7%, 1.0, 23.5ms, 0 | 20260505 16:45:45.444 | INFO | PRE_HQ : 60.3%, 1.8, 56.3ms, 0 | 20260505 16:45:45.444 | INFO | BAZWRITER : 63.3%, 3.9, 127.3ms, 0 | 20260505 16:45:45.444 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 16:45:45.444 | INFO | Uploaded 36GiB to GPU in 2.1657s (16.6228GiB/s) | 20260505 16:45:45.444 | INFO | Downloaded 38GiB to GPU in 2.25835s (16.8265GiB/s) | 20260505 16:45:50.577 | INFO | Analyzing chunk frames = [4747776,4748288) | 20260505 16:45:50.577 | INFO | MemInfo: { | 20260505 16:45:50.577 | INFO | "gpu" : " 17 GB", | 20260505 16:45:50.577 | INFO | "huge" : "", | 20260505 16:45:50.577 | INFO | "huge_shared" : " 27 GB", | 20260505 16:45:50.577 | INFO | "malloc" : " 14 GB", | 20260505 16:45:50.578 | INFO | "pinned" : " 29 GB" | 20260505 16:45:50.578 | INFO | } | 20260505 16:45:50.578 | INFO | Perf report for the last 5.13364 seconds: | 20260505 16:45:50.578 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:45:50.578 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:50.578 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:50.578 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:50.578 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260505 16:45:50.578 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:50.578 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:50.578 | INFO | PRE_HQ : 27.7%, 1.0, 14.3ms, 0 | 20260505 16:45:50.578 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:50.578 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:50.578 | INFO | Uploaded 36GiB to GPU in 2.14232s (16.8042GiB/s) | 20260505 16:45:50.578 | INFO | Downloaded 34GiB to GPU in 2.03419s (16.7143GiB/s) | 20260505 16:45:55.707 | INFO | Analyzing chunk frames = [4748288,4748800) | 20260505 16:45:55.707 | INFO | MemInfo: { | 20260505 16:45:55.707 | INFO | "gpu" : " 17 GB", | 20260505 16:45:55.707 | INFO | "huge" : "", | 20260505 16:45:55.707 | INFO | "huge_shared" : " 27 GB", | 20260505 16:45:55.707 | INFO | "malloc" : " 14 GB", | 20260505 16:45:55.707 | INFO | "pinned" : " 29 GB" | 20260505 16:45:55.707 | INFO | } | 20260505 16:45:55.707 | INFO | Perf report for the last 5.12983 seconds: | 20260505 16:45:55.707 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:45:55.707 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:55.707 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:55.708 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:55.708 | INFO | ANALYSIS : 62.9%, 3.0, 97.6ms, 0 | 20260505 16:45:55.708 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:55.708 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:55.708 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:45:55.708 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:55.708 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:45:55.708 | INFO | Uploaded 36GiB to GPU in 2.19031s (16.436GiB/s) | 20260505 16:45:55.708 | INFO | Downloaded 34GiB to GPU in 2.02779s (16.767GiB/s) | 20260505 16:46:00.858 | INFO | Analyzing chunk frames = [4748800,4749312) | 20260505 16:46:00.858 | INFO | MemInfo: { | 20260505 16:46:00.858 | INFO | "gpu" : " 17 GB", | 20260505 16:46:00.858 | INFO | "huge" : "", | 20260505 16:46:00.858 | INFO | "huge_shared" : " 27 GB", | 20260505 16:46:00.858 | INFO | "malloc" : " 14 GB", | 20260505 16:46:00.858 | INFO | "pinned" : " 29 GB" | 20260505 16:46:00.858 | INFO | } | 20260505 16:46:00.858 | INFO | Perf report for the last 5.15086 seconds: | 20260505 16:46:00.858 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:46:00.858 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:00.858 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:00.858 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:00.858 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260505 16:46:00.858 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:00.858 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:00.858 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 16:46:00.858 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:00.858 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:00.858 | INFO | Uploaded 36GiB to GPU in 2.18742s (16.4577GiB/s) | 20260505 16:46:00.858 | INFO | Downloaded 34GiB to GPU in 2.13469s (15.9274GiB/s) | 20260505 16:46:05.992 | INFO | Analyzing chunk frames = [4749312,4749824) | 20260505 16:46:05.992 | INFO | MemInfo: { | 20260505 16:46:05.992 | INFO | "gpu" : " 17 GB", | 20260505 16:46:05.992 | INFO | "huge" : "", | 20260505 16:46:05.992 | INFO | "huge_shared" : " 27 GB", | 20260505 16:46:05.992 | INFO | "malloc" : " 14 GB", | 20260505 16:46:05.992 | INFO | "pinned" : " 29 GB" | 20260505 16:46:05.992 | INFO | } | 20260505 16:46:05.992 | INFO | Perf report for the last 5.13356 seconds: | 20260505 16:46:05.992 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:46:05.992 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:05.992 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:05.992 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:05.992 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 16:46:05.992 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:05.992 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:05.992 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 16:46:05.992 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:05.992 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:05.992 | INFO | Uploaded 36GiB to GPU in 2.15267s (16.7234GiB/s) | 20260505 16:46:05.992 | INFO | Downloaded 34GiB to GPU in 2.05868s (16.5154GiB/s) | 20260505 16:46:11.128 | INFO | Analyzing chunk frames = [4749824,4750336) | 20260505 16:46:11.128 | INFO | MemInfo: { | 20260505 16:46:11.128 | INFO | "gpu" : " 17 GB", | 20260505 16:46:11.128 | INFO | "huge" : "", | 20260505 16:46:11.128 | INFO | "huge_shared" : " 27 GB", | 20260505 16:46:11.128 | INFO | "malloc" : " 14 GB", | 20260505 16:46:11.128 | INFO | "pinned" : " 29 GB" | 20260505 16:46:11.128 | INFO | } | 20260505 16:46:11.128 | INFO | Perf report for the last 5.13655 seconds: | 20260505 16:46:11.128 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:46:11.128 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:11.129 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:11.129 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:11.129 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260505 16:46:11.129 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:11.129 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:11.129 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:46:11.129 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:11.129 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:11.129 | INFO | Uploaded 36GiB to GPU in 2.1426s (16.802GiB/s) | 20260505 16:46:11.129 | INFO | Downloaded 34GiB to GPU in 2.0414s (16.6552GiB/s) | 20260505 16:46:16.266 | INFO | Analyzing chunk frames = [4750336,4750848) | 20260505 16:46:16.266 | INFO | MemInfo: { | 20260505 16:46:16.266 | INFO | "gpu" : " 17 GB", | 20260505 16:46:16.266 | INFO | "huge" : "", | 20260505 16:46:16.266 | INFO | "huge_shared" : " 27 GB", | 20260505 16:46:16.266 | INFO | "malloc" : " 14 GB", | 20260505 16:46:16.266 | INFO | "pinned" : " 29 GB" | 20260505 16:46:16.266 | INFO | } | 20260505 16:46:16.266 | INFO | Perf report for the last 5.13732 seconds: | 20260505 16:46:16.266 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:46:16.266 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:16.266 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:16.266 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:16.266 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 16:46:16.266 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:16.266 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:16.266 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 16:46:16.266 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:16.266 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:16.266 | INFO | Uploaded 36GiB to GPU in 2.11949s (16.9853GiB/s) | 20260505 16:46:16.266 | INFO | Downloaded 34GiB to GPU in 2.17163s (15.6565GiB/s) | 20260505 16:46:21.409 | INFO | Analyzing chunk frames = [4750848,4751360) | 20260505 16:46:21.409 | INFO | MemInfo: { | 20260505 16:46:21.409 | INFO | "gpu" : " 17 GB", | 20260505 16:46:21.409 | INFO | "huge" : "", | 20260505 16:46:21.409 | INFO | "huge_shared" : " 27 GB", | 20260505 16:46:21.410 | INFO | "malloc" : " 14 GB", | 20260505 16:46:21.410 | INFO | "pinned" : " 29 GB" | 20260505 16:46:21.410 | INFO | } | 20260505 16:46:21.410 | INFO | Perf report for the last 5.14385 seconds: | 20260505 16:46:21.410 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:46:21.410 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:21.410 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:21.410 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:21.410 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 16:46:21.410 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:21.410 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:21.410 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 16:46:21.410 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:21.410 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:21.410 | INFO | Uploaded 36GiB to GPU in 2.232s (16.129GiB/s) | 20260505 16:46:21.410 | INFO | Downloaded 34GiB to GPU in 2.10744s (16.1333GiB/s) | 20260505 16:46:21.410 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 16:46:21.410 | INFO | Total mem usage: 15.3 GiB | 20260505 16:46:21.410 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 16:46:21.410 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.89%) | 20260505 16:46:21.410 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 16:46:21.410 | INFO | High Water Breakdown: | 20260505 16:46:21.410 | INFO | 61% : 8.67 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 16:46:21.410 | INFO | 38.8% : 5.51 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 16:46:21.410 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 16:46:21.410 | INFO | ------------------------------------------ | 20260505 16:46:21.410 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:46:21.410 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 16:46:21.410 | INFO | Total mem usage: 26.5 GiB | 20260505 16:46:21.410 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 16:46:21.410 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 16:46:21.410 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 16:46:21.410 | INFO | High Water Breakdown: | 20260505 16:46:21.410 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:46:21.410 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:46:21.410 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:46:21.410 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:46:21.410 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:46:21.410 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:46:21.410 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 16:46:21.410 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 16:46:21.410 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:46:21.410 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 16:46:21.410 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 16:46:21.410 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 16:46:21.411 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 16:46:21.411 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 16:46:21.411 | INFO | ------------------------------------------ | 20260505 16:46:21.411 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:46:21.411 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 16:46:21.411 | INFO | Total mem usage: 25.6 GiB | 20260505 16:46:21.411 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 16:46:21.411 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 16:46:21.411 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 16:46:21.411 | INFO | High Water Breakdown: | 20260505 16:46:21.411 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 16:46:21.411 | INFO | ------------------------------------------ | 20260505 16:46:21.411 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:46:21.411 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 16:46:21.411 | INFO | Total mem usage: 15.7 GiB | 20260505 16:46:21.411 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 16:46:21.411 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 16:46:21.411 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 16:46:21.411 | INFO | High Water Breakdown: | 20260505 16:46:21.411 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 16:46:21.411 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 16:46:21.411 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 16:46:21.411 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 16:46:21.411 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 16:46:21.411 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 16:46:21.411 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 16:46:21.411 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 16:46:21.411 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 16:46:21.411 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:46:21.411 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 16:46:21.411 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:46:21.411 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 16:46:21.411 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 16:46:21.411 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 16:46:21.411 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 16:46:21.411 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:46:21.411 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 16:46:21.411 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:46:21.411 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:46:21.411 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:46:21.411 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 16:46:21.411 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 16:46:21.411 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:46:21.411 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 16:46:21.411 | INFO | ------------------------------------------ | 20260505 16:46:21.411 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:46:21.411 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 16:46:21.411 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:46:21.411 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:46:21.411 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.51e+01, 1.32e+01 / 2.18e+00 | 20260505 16:46:21.411 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 1.02e-01, 1.06e-02 / 4.19e-03 | 20260505 16:46:21.411 | INFO | PerfReport: 11.78%, Baseline -- 3.73e+00 / 4.46e+00, 5.74e-01 / 1.95e-01 | 20260505 16:46:21.411 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.49e+00, 1.93e-01 / 1.18e-01 | 20260505 16:46:21.411 | INFO | PerfReport: 15.92%, DME -- 4.03e+01 / 4.47e+01, 4.32e+00 / 2.14e+00 | 20260505 16:46:21.411 | INFO | PerfReport: 56.79%, FrameLabeling -- 1.80e+01 / 1.96e+01, 1.50e+00 / 1.60e+00 | 20260505 16:46:21.411 | INFO | PerfReport: 7.34%, PulseAccumulating -- 2.33e+00 / 2.90e+00, 4.45e-01 / 1.97e-01 | 20260505 16:46:21.411 | INFO | PerfReport: 3.41%, Metrics -- 1.08e+00 / 1.38e+00, 2.99e-01 / 1.32e-01 | 20260505 16:46:21.411 | INFO | PerfReport: 1.19%, PulseSerializing -- 3.76e-01 / 4.35e-01, 6.69e-02 / 4.33e-02 | 20260505 16:46:21.411 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 16:46:21.411 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:46:21.411 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:46:21.411 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260505 16:46:21.411 | INFO | PerfReport: 22.53%, Upload -- 2.26e+01 / 0.00e+00, 7.94e+00 / 0.00e+00 | 20260505 16:46:21.412 | INFO | PerfReport: 21.85%, Download -- 2.19e+01 / 0.00e+00, 7.93e+00 / 0.00e+00 | 20260505 16:46:21.412 | INFO | PerfReport: 31.62%, Compute -- 3.17e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260505 16:46:21.412 | INFO | PerfReport: 2.28%, LockUpload -- 2.29e+00 / 0.00e+00, 7.89e+00 / 0.00e+00 | 20260505 16:46:21.412 | INFO | PerfReport: 1.40%, LockDownload -- 1.40e+00 / 0.00e+00, 2.97e+00 / 0.00e+00 | 20260505 16:46:21.412 | INFO | PerfReport: 20.29%, LockCompute -- 2.03e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 16:46:21.412 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.79e-02 / 0.00e+00 | 20260505 16:46:21.543 | INFO | Starting a round of Baz IO | 20260505 16:46:24.744 | INFO | Populating RTMetrics report | 20260505 16:46:24.744 | INFO | RTMetrics report populated | 20260505 16:46:24.744 | INFO | Opening tmp RTMetrics json file | 20260505 16:46:24.755 | INFO | Serializing RTMetrics to json | 20260505 16:46:24.756 | INFO | Serialization done, swapping tmp file now... | 20260505 16:46:24.757 | INFO | RTMetrics done | 20260505 16:46:24.845 | INFO | Finished a round of Baz IO: Wrote 2.161 GiB in 3.301 seconds | 20260505 16:46:24.845 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.18% / 0.00% / 77.82% | 20260505 16:46:24.845 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.344419% / 0% / 6.75519e-05% | 20260505 16:46:24.845 | INFO | Baz IO overall: | 20260505 16:46:24.845 | INFO | BytesWritten : 2816243891331 | 20260505 16:46:24.845 | INFO | NumEvents : 1336330048241 | 20260505 16:46:26.555 | INFO | Analyzing chunk frames = [4751360,4751872) | 20260505 16:46:26.555 | INFO | MemInfo: { | 20260505 16:46:26.555 | INFO | "gpu" : " 17 GB", | 20260505 16:46:26.555 | INFO | "huge" : "", | 20260505 16:46:26.555 | INFO | "huge_shared" : " 27 GB", | 20260505 16:46:26.555 | INFO | "malloc" : " 14 GB", | 20260505 16:46:26.555 | INFO | "pinned" : " 29 GB" | 20260505 16:46:26.555 | INFO | } | 20260505 16:46:26.555 | INFO | Perf report for the last 5.14538 seconds: | 20260505 16:46:26.555 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:46:26.555 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:26.555 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:26.555 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:26.555 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 16:46:26.555 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:26.555 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260505 16:46:26.555 | INFO | PRE_HQ : 60.3%, 1.8, 56.3ms, 0 | 20260505 16:46:26.555 | INFO | BAZWRITER : 63.5%, 3.9, 126.9ms, 0 | 20260505 16:46:26.555 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 16:46:26.555 | INFO | Uploaded 36GiB to GPU in 2.1727s (16.5692GiB/s) | 20260505 16:46:26.556 | INFO | Downloaded 38GiB to GPU in 2.26163s (16.802GiB/s) | 20260505 16:46:31.696 | INFO | Analyzing chunk frames = [4751872,4752384) | 20260505 16:46:31.696 | INFO | MemInfo: { | 20260505 16:46:31.696 | INFO | "gpu" : " 17 GB", | 20260505 16:46:31.696 | INFO | "huge" : "", | 20260505 16:46:31.696 | INFO | "huge_shared" : " 27 GB", | 20260505 16:46:31.696 | INFO | "malloc" : " 14 GB", | 20260505 16:46:31.696 | INFO | "pinned" : " 29 GB" | 20260505 16:46:31.696 | INFO | } | 20260505 16:46:31.696 | INFO | Perf report for the last 5.14141 seconds: | 20260505 16:46:31.696 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:46:31.696 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:31.696 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:31.696 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:31.697 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 16:46:31.697 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:31.697 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:31.697 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:46:31.697 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:31.697 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:31.697 | INFO | Uploaded 36GiB to GPU in 2.12477s (16.943GiB/s) | 20260505 16:46:31.697 | INFO | Downloaded 34GiB to GPU in 2.02422s (16.7966GiB/s) | 20260505 16:46:36.826 | INFO | Analyzing chunk frames = [4752384,4752896) | 20260505 16:46:36.826 | INFO | MemInfo: { | 20260505 16:46:36.826 | INFO | "gpu" : " 17 GB", | 20260505 16:46:36.826 | INFO | "huge" : "", | 20260505 16:46:36.826 | INFO | "huge_shared" : " 27 GB", | 20260505 16:46:36.826 | INFO | "malloc" : " 14 GB", | 20260505 16:46:36.827 | INFO | "pinned" : " 29 GB" | 20260505 16:46:36.827 | INFO | } | 20260505 16:46:36.827 | INFO | Perf report for the last 5.13024 seconds: | 20260505 16:46:36.827 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:46:36.827 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:36.827 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:36.827 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:36.827 | INFO | ANALYSIS : 62.8%, 3.0, 97.6ms, 0 | 20260505 16:46:36.827 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:36.827 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:36.827 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:46:36.827 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:36.827 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:36.827 | INFO | Uploaded 36GiB to GPU in 2.20135s (16.3536GiB/s) | 20260505 16:46:36.827 | INFO | Downloaded 34GiB to GPU in 2.05335s (16.5583GiB/s) | 20260505 16:46:41.958 | INFO | Analyzing chunk frames = [4752896,4753408) | 20260505 16:46:41.958 | INFO | MemInfo: { | 20260505 16:46:41.958 | INFO | "gpu" : " 17 GB", | 20260505 16:46:41.958 | INFO | "huge" : "", | 20260505 16:46:41.958 | INFO | "huge_shared" : " 27 GB", | 20260505 16:46:41.958 | INFO | "malloc" : " 14 GB", | 20260505 16:46:41.958 | INFO | "pinned" : " 29 GB" | 20260505 16:46:41.958 | INFO | } | 20260505 16:46:41.958 | INFO | Perf report for the last 5.13162 seconds: | 20260505 16:46:41.958 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:46:41.958 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:41.959 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:41.959 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:41.959 | INFO | ANALYSIS : 62.5%, 3.0, 96.9ms, 0 | 20260505 16:46:41.959 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:41.959 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:41.959 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:46:41.959 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:41.959 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:41.959 | INFO | Uploaded 36GiB to GPU in 2.17183s (16.5759GiB/s) | 20260505 16:46:41.959 | INFO | Downloaded 34GiB to GPU in 2.13392s (15.9331GiB/s) | 20260505 16:46:47.109 | INFO | Analyzing chunk frames = [4753408,4753920) | 20260505 16:46:47.109 | INFO | MemInfo: { | 20260505 16:46:47.109 | INFO | "gpu" : " 17 GB", | 20260505 16:46:47.109 | INFO | "huge" : "", | 20260505 16:46:47.109 | INFO | "huge_shared" : " 27 GB", | 20260505 16:46:47.109 | INFO | "malloc" : " 14 GB", | 20260505 16:46:47.109 | INFO | "pinned" : " 29 GB" | 20260505 16:46:47.109 | INFO | } | 20260505 16:46:47.110 | INFO | Perf report for the last 5.15117 seconds: | 20260505 16:46:47.110 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:46:47.110 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:47.110 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:47.110 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:47.110 | INFO | ANALYSIS : 62.6%, 3.0, 97.9ms, 0 | 20260505 16:46:47.110 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:47.110 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:47.110 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 16:46:47.110 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:47.110 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:47.110 | INFO | Uploaded 36GiB to GPU in 2.17279s (16.5685GiB/s) | 20260505 16:46:47.110 | INFO | Downloaded 34GiB to GPU in 2.13255s (15.9433GiB/s) | 20260505 16:46:52.256 | INFO | Analyzing chunk frames = [4753920,4754432) | 20260505 16:46:52.256 | INFO | MemInfo: { | 20260505 16:46:52.256 | INFO | "gpu" : " 17 GB", | 20260505 16:46:52.256 | INFO | "huge" : "", | 20260505 16:46:52.256 | INFO | "huge_shared" : " 27 GB", | 20260505 16:46:52.256 | INFO | "malloc" : " 14 GB", | 20260505 16:46:52.256 | INFO | "pinned" : " 29 GB" | 20260505 16:46:52.256 | INFO | } | 20260505 16:46:52.256 | INFO | Perf report for the last 5.1467 seconds: | 20260505 16:46:52.256 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:46:52.256 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:52.256 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:52.256 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:52.256 | INFO | ANALYSIS : 62.1%, 3.0, 97.1ms, 0 | 20260505 16:46:52.256 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:52.256 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:52.256 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:46:52.256 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:52.256 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:52.257 | INFO | Uploaded 36GiB to GPU in 2.14542s (16.7799GiB/s) | 20260505 16:46:52.257 | INFO | Downloaded 34GiB to GPU in 2.05273s (16.5633GiB/s) | 20260505 16:46:57.390 | INFO | Analyzing chunk frames = [4754432,4754944) | 20260505 16:46:57.390 | INFO | MemInfo: { | 20260505 16:46:57.390 | INFO | "gpu" : " 17 GB", | 20260505 16:46:57.390 | INFO | "huge" : "", | 20260505 16:46:57.390 | INFO | "huge_shared" : " 27 GB", | 20260505 16:46:57.390 | INFO | "malloc" : " 14 GB", | 20260505 16:46:57.390 | INFO | "pinned" : " 29 GB" | 20260505 16:46:57.390 | INFO | } | 20260505 16:46:57.391 | INFO | Perf report for the last 5.13429 seconds: | 20260505 16:46:57.391 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:46:57.391 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:57.391 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:57.391 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:57.391 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 16:46:57.391 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:57.391 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:57.391 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 16:46:57.391 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:57.391 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:46:57.391 | INFO | Uploaded 36GiB to GPU in 2.12568s (16.9357GiB/s) | 20260505 16:46:57.391 | INFO | Downloaded 34GiB to GPU in 2.13525s (15.9232GiB/s) | 20260505 16:47:02.521 | INFO | Analyzing chunk frames = [4754944,4755456) | 20260505 16:47:02.521 | INFO | MemInfo: { | 20260505 16:47:02.521 | INFO | "gpu" : " 17 GB", | 20260505 16:47:02.521 | INFO | "huge" : "", | 20260505 16:47:02.521 | INFO | "huge_shared" : " 27 GB", | 20260505 16:47:02.522 | INFO | "malloc" : " 14 GB", | 20260505 16:47:02.522 | INFO | "pinned" : " 29 GB" | 20260505 16:47:02.522 | INFO | } | 20260505 16:47:02.522 | INFO | Perf report for the last 5.13147 seconds: | 20260505 16:47:02.522 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:47:02.522 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:02.522 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:02.522 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:02.522 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 16:47:02.522 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:02.522 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:02.522 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 16:47:02.522 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:02.522 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:02.522 | INFO | Uploaded 36GiB to GPU in 2.17289s (16.5678GiB/s) | 20260505 16:47:02.522 | INFO | Downloaded 34GiB to GPU in 2.02503s (16.7899GiB/s) | 20260505 16:47:02.655 | INFO | Starting a round of Baz IO | 20260505 16:47:05.865 | INFO | Populating RTMetrics report | 20260505 16:47:05.865 | INFO | RTMetrics report populated | 20260505 16:47:05.865 | INFO | Opening tmp RTMetrics json file | 20260505 16:47:05.869 | INFO | Serializing RTMetrics to json | 20260505 16:47:05.871 | INFO | Serialization done, swapping tmp file now... | 20260505 16:47:05.874 | INFO | RTMetrics done | 20260505 16:47:05.966 | INFO | Finished a round of Baz IO: Wrote 2.161 GiB in 3.311 seconds | 20260505 16:47:05.966 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.18% / 0.00% / 77.82% | 20260505 16:47:05.966 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.343088% / 3.97364e-06% / 0.000107288% | 20260505 16:47:05.966 | INFO | Baz IO overall: | 20260505 16:47:05.966 | INFO | BytesWritten : 2818564053647 | 20260505 16:47:05.966 | INFO | NumEvents : 1337416490162 | 20260505 16:47:07.663 | INFO | Analyzing chunk frames = [4755456,4755968) | 20260505 16:47:07.663 | INFO | MemInfo: { | 20260505 16:47:07.663 | INFO | "gpu" : " 17 GB", | 20260505 16:47:07.663 | INFO | "huge" : "", | 20260505 16:47:07.663 | INFO | "huge_shared" : " 27 GB", | 20260505 16:47:07.663 | INFO | "malloc" : " 14 GB", | 20260505 16:47:07.663 | INFO | "pinned" : " 29 GB" | 20260505 16:47:07.663 | INFO | } | 20260505 16:47:07.663 | INFO | Perf report for the last 5.14138 seconds: | 20260505 16:47:07.663 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:47:07.663 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:07.663 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:07.663 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:07.663 | INFO | ANALYSIS : 63.9%, 3.0, 99.6ms, 0 | 20260505 16:47:07.664 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:07.664 | INFO | RT_METRICS : 44.7%, 1.0, 23.4ms, 0 | 20260505 16:47:07.664 | INFO | PRE_HQ : 60.6%, 1.8, 56.2ms, 0 | 20260505 16:47:07.664 | INFO | BAZWRITER : 63.7%, 3.9, 127.0ms, 0 | 20260505 16:47:07.664 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 16:47:07.664 | INFO | Uploaded 36GiB to GPU in 2.19057s (16.4341GiB/s) | 20260505 16:47:07.664 | INFO | Downloaded 38GiB to GPU in 2.26349s (16.7882GiB/s) | 20260505 16:47:12.807 | INFO | Analyzing chunk frames = [4755968,4756480) | 20260505 16:47:12.807 | INFO | MemInfo: { | 20260505 16:47:12.807 | INFO | "gpu" : " 17 GB", | 20260505 16:47:12.807 | INFO | "huge" : "", | 20260505 16:47:12.807 | INFO | "huge_shared" : " 27 GB", | 20260505 16:47:12.807 | INFO | "malloc" : " 14 GB", | 20260505 16:47:12.807 | INFO | "pinned" : " 29 GB" | 20260505 16:47:12.807 | INFO | } | 20260505 16:47:12.807 | INFO | Perf report for the last 5.14404 seconds: | 20260505 16:47:12.807 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:47:12.807 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:12.807 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:12.808 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:12.808 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 16:47:12.808 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:12.808 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:12.808 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260505 16:47:12.808 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:12.808 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:12.808 | INFO | Uploaded 36GiB to GPU in 2.12432s (16.9466GiB/s) | 20260505 16:47:12.808 | INFO | Downloaded 34GiB to GPU in 2.02747s (16.7697GiB/s) | 20260505 16:47:17.944 | INFO | Analyzing chunk frames = [4756480,4756992) | 20260505 16:47:17.944 | INFO | MemInfo: { | 20260505 16:47:17.944 | INFO | "gpu" : " 17 GB", | 20260505 16:47:17.944 | INFO | "huge" : "", | 20260505 16:47:17.944 | INFO | "huge_shared" : " 27 GB", | 20260505 16:47:17.944 | INFO | "malloc" : " 14 GB", | 20260505 16:47:17.945 | INFO | "pinned" : " 29 GB" | 20260505 16:47:17.945 | INFO | } | 20260505 16:47:17.945 | INFO | Perf report for the last 5.13712 seconds: | 20260505 16:47:17.945 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:47:17.945 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:17.945 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:17.945 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:17.945 | INFO | ANALYSIS : 62.9%, 3.0, 97.8ms, 0 | 20260505 16:47:17.945 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:17.945 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:17.945 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:47:17.945 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:17.945 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:17.945 | INFO | Uploaded 36GiB to GPU in 2.20704s (16.3114GiB/s) | 20260505 16:47:17.945 | INFO | Downloaded 34GiB to GPU in 2.04337s (16.6391GiB/s) | 20260505 16:47:23.082 | INFO | Analyzing chunk frames = [4756992,4757504) | 20260505 16:47:23.082 | INFO | MemInfo: { | 20260505 16:47:23.082 | INFO | "gpu" : " 17 GB", | 20260505 16:47:23.082 | INFO | "huge" : "", | 20260505 16:47:23.082 | INFO | "huge_shared" : " 27 GB", | 20260505 16:47:23.082 | INFO | "malloc" : " 14 GB", | 20260505 16:47:23.082 | INFO | "pinned" : " 29 GB" | 20260505 16:47:23.082 | INFO | } | 20260505 16:47:23.082 | INFO | Perf report for the last 5.1377 seconds: | 20260505 16:47:23.082 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:47:23.082 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:23.082 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:23.082 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:23.082 | INFO | ANALYSIS : 62.6%, 3.0, 97.3ms, 0 | 20260505 16:47:23.082 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:23.082 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:23.082 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:47:23.082 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:23.082 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:23.083 | INFO | Uploaded 36GiB to GPU in 2.18049s (16.5101GiB/s) | 20260505 16:47:23.083 | INFO | Downloaded 34GiB to GPU in 2.14471s (15.853GiB/s) | 20260505 16:47:28.218 | INFO | Analyzing chunk frames = [4757504,4758016) | 20260505 16:47:28.218 | INFO | MemInfo: { | 20260505 16:47:28.218 | INFO | "gpu" : " 17 GB", | 20260505 16:47:28.218 | INFO | "huge" : "", | 20260505 16:47:28.218 | INFO | "huge_shared" : " 27 GB", | 20260505 16:47:28.218 | INFO | "malloc" : " 14 GB", | 20260505 16:47:28.218 | INFO | "pinned" : " 29 GB" | 20260505 16:47:28.218 | INFO | } | 20260505 16:47:28.218 | INFO | Perf report for the last 5.13569 seconds: | 20260505 16:47:28.218 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:47:28.218 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:28.218 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:28.218 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:28.218 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 16:47:28.218 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:28.218 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:28.218 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 16:47:28.218 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:28.218 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:28.218 | INFO | Uploaded 36GiB to GPU in 2.18513s (16.475GiB/s) | 20260505 16:47:28.218 | INFO | Downloaded 34GiB to GPU in 2.1304s (15.9594GiB/s) | 20260505 16:47:33.362 | INFO | Analyzing chunk frames = [4758016,4758528) | 20260505 16:47:33.362 | INFO | MemInfo: { | 20260505 16:47:33.362 | INFO | "gpu" : " 17 GB", | 20260505 16:47:33.362 | INFO | "huge" : "", | 20260505 16:47:33.362 | INFO | "huge_shared" : " 27 GB", | 20260505 16:47:33.362 | INFO | "malloc" : " 14 GB", | 20260505 16:47:33.362 | INFO | "pinned" : " 29 GB" | 20260505 16:47:33.362 | INFO | } | 20260505 16:47:33.362 | INFO | Perf report for the last 5.14383 seconds: | 20260505 16:47:33.362 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:47:33.362 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:33.362 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:33.362 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:33.362 | INFO | ANALYSIS : 62.5%, 3.0, 97.7ms, 0 | 20260505 16:47:33.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:33.362 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:33.362 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:47:33.362 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:33.362 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:33.362 | INFO | Uploaded 36GiB to GPU in 2.14693s (16.7681GiB/s) | 20260505 16:47:33.362 | INFO | Downloaded 34GiB to GPU in 2.04245s (16.6467GiB/s) | 20260505 16:47:38.508 | INFO | Analyzing chunk frames = [4758528,4759040) | 20260505 16:47:38.508 | INFO | MemInfo: { | 20260505 16:47:38.508 | INFO | "gpu" : " 17 GB", | 20260505 16:47:38.508 | INFO | "huge" : "", | 20260505 16:47:38.508 | INFO | "huge_shared" : " 27 GB", | 20260505 16:47:38.508 | INFO | "malloc" : " 14 GB", | 20260505 16:47:38.508 | INFO | "pinned" : " 29 GB" | 20260505 16:47:38.508 | INFO | } | 20260505 16:47:38.508 | INFO | Perf report for the last 5.14615 seconds: | 20260505 16:47:38.508 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:47:38.508 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:38.508 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:38.508 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:38.508 | INFO | ANALYSIS : 63.5%, 3.0, 99.3ms, 0 | 20260505 16:47:38.508 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:38.508 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:38.508 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 16:47:38.508 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:38.508 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:38.508 | INFO | Uploaded 36GiB to GPU in 2.15885s (16.6756GiB/s) | 20260505 16:47:38.508 | INFO | Downloaded 34GiB to GPU in 2.16742s (15.6868GiB/s) | 20260505 16:47:43.637 | INFO | Analyzing chunk frames = [4759040,4759552) | 20260505 16:47:43.637 | INFO | MemInfo: { | 20260505 16:47:43.637 | INFO | "gpu" : " 17 GB", | 20260505 16:47:43.637 | INFO | "huge" : "", | 20260505 16:47:43.637 | INFO | "huge_shared" : " 27 GB", | 20260505 16:47:43.637 | INFO | "malloc" : " 14 GB", | 20260505 16:47:43.637 | INFO | "pinned" : " 29 GB" | 20260505 16:47:43.637 | INFO | } | 20260505 16:47:43.637 | INFO | Perf report for the last 5.12882 seconds: | 20260505 16:47:43.637 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:47:43.637 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:43.637 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:43.637 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:43.637 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 16:47:43.637 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:43.637 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:43.637 | INFO | PRE_HQ : 27.9%, 1.0, 14.4ms, 0 | 20260505 16:47:43.637 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:43.637 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:43.637 | INFO | Uploaded 36GiB to GPU in 2.19565s (16.396GiB/s) | 20260505 16:47:43.637 | INFO | Downloaded 34GiB to GPU in 2.02206s (16.8146GiB/s) | 20260505 16:47:43.637 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 16:47:43.637 | INFO | Total mem usage: 15.3 GiB | 20260505 16:47:43.637 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 16:47:43.637 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.03%) | 20260505 16:47:43.637 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 16:47:43.637 | INFO | High Water Breakdown: | 20260505 16:47:43.637 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 16:47:43.637 | INFO | 38.9% : 5.51 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 16:47:43.637 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 16:47:43.637 | INFO | ------------------------------------------ | 20260505 16:47:43.637 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:47:43.637 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 16:47:43.637 | INFO | Total mem usage: 26.5 GiB | 20260505 16:47:43.637 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 16:47:43.637 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 16:47:43.637 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 16:47:43.637 | INFO | High Water Breakdown: | 20260505 16:47:43.637 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:47:43.637 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:47:43.637 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:47:43.637 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:47:43.637 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:47:43.637 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:47:43.637 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 16:47:43.637 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 16:47:43.637 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:47:43.637 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 16:47:43.638 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 16:47:43.638 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 16:47:43.638 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 16:47:43.638 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 16:47:43.638 | INFO | ------------------------------------------ | 20260505 16:47:43.638 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:47:43.638 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 16:47:43.638 | INFO | Total mem usage: 25.6 GiB | 20260505 16:47:43.638 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 16:47:43.638 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 16:47:43.638 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 16:47:43.638 | INFO | High Water Breakdown: | 20260505 16:47:43.638 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 16:47:43.638 | INFO | ------------------------------------------ | 20260505 16:47:43.638 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:47:43.638 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 16:47:43.638 | INFO | Total mem usage: 15.7 GiB | 20260505 16:47:43.638 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 16:47:43.638 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 16:47:43.638 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 16:47:43.638 | INFO | High Water Breakdown: | 20260505 16:47:43.638 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 16:47:43.638 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 16:47:43.638 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 16:47:43.638 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 16:47:43.638 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 16:47:43.638 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 16:47:43.638 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 16:47:43.638 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 16:47:43.638 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 16:47:43.638 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:47:43.638 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 16:47:43.638 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:47:43.638 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 16:47:43.638 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 16:47:43.638 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 16:47:43.638 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 16:47:43.638 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:47:43.638 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 16:47:43.638 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:47:43.638 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:47:43.638 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:47:43.638 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 16:47:43.638 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 16:47:43.638 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:47:43.638 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 16:47:43.638 | INFO | ------------------------------------------ | 20260505 16:47:43.638 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:47:43.638 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 16:47:43.638 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:47:43.638 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:47:43.638 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.54e+01, 1.37e+01 / 2.44e+00 | 20260505 16:47:43.638 | INFO | PerfReport: 0.29%, AggregateStats -- 9.27e-02 / 1.01e-01, 1.13e-02 / 4.19e-03 | 20260505 16:47:43.638 | INFO | PerfReport: 11.78%, Baseline -- 3.73e+00 / 4.52e+00, 5.67e-01 / 1.17e-01 | 20260505 16:47:43.638 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.52e+00, 1.92e-01 / 8.77e-02 | 20260505 16:47:43.638 | INFO | PerfReport: 15.97%, DME -- 4.04e+01 / 4.50e+01, 4.57e+00 / 2.70e+00 | 20260505 16:47:43.638 | INFO | PerfReport: 56.69%, FrameLabeling -- 1.79e+01 / 1.95e+01, 1.41e+00 / 1.42e+00 | 20260505 16:47:43.638 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.33e+00 / 2.91e+00, 4.48e-01 / 1.87e-01 | 20260505 16:47:43.638 | INFO | PerfReport: 3.41%, Metrics -- 1.08e+00 / 1.43e+00, 3.00e-01 / 1.61e-01 | 20260505 16:47:43.638 | INFO | PerfReport: 1.19%, PulseSerializing -- 3.75e-01 / 4.36e-01, 6.63e-02 / 3.57e-02 | 20260505 16:47:43.638 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 16:47:43.638 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:47:43.638 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:47:43.638 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260505 16:47:43.638 | INFO | PerfReport: 22.54%, Upload -- 2.26e+01 / 0.00e+00, 7.92e+00 / 0.00e+00 | 20260505 16:47:43.638 | INFO | PerfReport: 21.88%, Download -- 2.19e+01 / 0.00e+00, 7.87e+00 / 0.00e+00 | 20260505 16:47:43.638 | INFO | PerfReport: 31.59%, Compute -- 3.16e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 16:47:43.638 | INFO | PerfReport: 2.25%, LockUpload -- 2.26e+00 / 0.00e+00, 7.93e+00 / 0.00e+00 | 20260505 16:47:43.639 | INFO | PerfReport: 1.40%, LockDownload -- 1.40e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260505 16:47:43.639 | INFO | PerfReport: 20.32%, LockCompute -- 2.03e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 16:47:43.639 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.73e-02 / 0.00e+00 | 20260505 16:47:43.767 | INFO | Starting a round of Baz IO | 20260505 16:47:46.966 | INFO | Populating RTMetrics report | 20260505 16:47:46.966 | INFO | RTMetrics report populated | 20260505 16:47:46.966 | INFO | Opening tmp RTMetrics json file | 20260505 16:47:46.988 | INFO | Serializing RTMetrics to json | 20260505 16:47:47.001 | INFO | Serialization done, swapping tmp file now... | 20260505 16:47:47.005 | INFO | RTMetrics done | 20260505 16:47:47.084 | INFO | Finished a round of Baz IO: Wrote 2.16 GiB in 3.316 seconds | 20260505 16:47:47.084 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.18% / 0.00% / 77.82% | 20260505 16:47:47.084 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.342572% / 3.97364e-06% / 9.13938e-05% | 20260505 16:47:47.084 | INFO | Baz IO overall: | 20260505 16:47:47.084 | INFO | BytesWritten : 2820883615930 | 20260505 16:47:47.084 | INFO | NumEvents : 1338502140566 | 20260505 16:47:48.775 | INFO | Analyzing chunk frames = [4759552,4760064) | 20260505 16:47:48.775 | INFO | MemInfo: { | 20260505 16:47:48.775 | INFO | "gpu" : " 17 GB", | 20260505 16:47:48.775 | INFO | "huge" : "", | 20260505 16:47:48.775 | INFO | "huge_shared" : " 27 GB", | 20260505 16:47:48.775 | INFO | "malloc" : " 14 GB", | 20260505 16:47:48.775 | INFO | "pinned" : " 29 GB" | 20260505 16:47:48.775 | INFO | } | 20260505 16:47:48.775 | INFO | Perf report for the last 5.13834 seconds: | 20260505 16:47:48.775 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:47:48.775 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:48.775 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:48.775 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:48.775 | INFO | ANALYSIS : 63.7%, 3.0, 99.0ms, 0 | 20260505 16:47:48.775 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:48.775 | INFO | RT_METRICS : 45.3%, 1.0, 23.8ms, 0 | 20260505 16:47:48.775 | INFO | PRE_HQ : 60.7%, 1.8, 56.4ms, 0 | 20260505 16:47:48.775 | INFO | BAZWRITER : 63.7%, 3.9, 128.5ms, 0 | 20260505 16:47:48.775 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260505 16:47:48.776 | INFO | Uploaded 36GiB to GPU in 2.15812s (16.6812GiB/s) | 20260505 16:47:48.776 | INFO | Downloaded 38GiB to GPU in 2.25611s (16.8431GiB/s) | 20260505 16:47:53.918 | INFO | Analyzing chunk frames = [4760064,4760576) | 20260505 16:47:53.918 | INFO | MemInfo: { | 20260505 16:47:53.918 | INFO | "gpu" : " 17 GB", | 20260505 16:47:53.918 | INFO | "huge" : "", | 20260505 16:47:53.918 | INFO | "huge_shared" : " 27 GB", | 20260505 16:47:53.918 | INFO | "malloc" : " 14 GB", | 20260505 16:47:53.918 | INFO | "pinned" : " 29 GB" | 20260505 16:47:53.918 | INFO | } | 20260505 16:47:53.918 | INFO | Perf report for the last 5.14323 seconds: | 20260505 16:47:53.918 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:47:53.918 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:53.918 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:53.918 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:53.919 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 16:47:53.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:53.919 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:53.919 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260505 16:47:53.919 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:53.919 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:53.919 | INFO | Uploaded 36GiB to GPU in 2.20051s (16.3599GiB/s) | 20260505 16:47:53.919 | INFO | Downloaded 34GiB to GPU in 2.04675s (16.6117GiB/s) | 20260505 16:47:59.062 | INFO | Analyzing chunk frames = [4760576,4761088) | 20260505 16:47:59.062 | INFO | MemInfo: { | 20260505 16:47:59.062 | INFO | "gpu" : " 17 GB", | 20260505 16:47:59.062 | INFO | "huge" : "", | 20260505 16:47:59.062 | INFO | "huge_shared" : " 27 GB", | 20260505 16:47:59.062 | INFO | "malloc" : " 14 GB", | 20260505 16:47:59.062 | INFO | "pinned" : " 29 GB" | 20260505 16:47:59.062 | INFO | } | 20260505 16:47:59.062 | INFO | Perf report for the last 5.14369 seconds: | 20260505 16:47:59.062 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:47:59.062 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:59.062 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:59.062 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:59.062 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260505 16:47:59.062 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:59.062 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:59.062 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:47:59.062 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:59.062 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:47:59.062 | INFO | Uploaded 36GiB to GPU in 2.19173s (16.4254GiB/s) | 20260505 16:47:59.062 | INFO | Downloaded 34GiB to GPU in 2.0297s (16.7513GiB/s) | 20260505 16:48:04.204 | INFO | Analyzing chunk frames = [4761088,4761600) | 20260505 16:48:04.204 | INFO | MemInfo: { | 20260505 16:48:04.204 | INFO | "gpu" : " 17 GB", | 20260505 16:48:04.204 | INFO | "huge" : "", | 20260505 16:48:04.204 | INFO | "huge_shared" : " 27 GB", | 20260505 16:48:04.204 | INFO | "malloc" : " 14 GB", | 20260505 16:48:04.204 | INFO | "pinned" : " 29 GB" | 20260505 16:48:04.204 | INFO | } | 20260505 16:48:04.204 | INFO | Perf report for the last 5.14166 seconds: | 20260505 16:48:04.204 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:48:04.204 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:04.204 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:04.204 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:04.204 | INFO | ANALYSIS : 62.9%, 3.0, 97.7ms, 0 | 20260505 16:48:04.204 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:04.204 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:04.204 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:48:04.204 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:04.204 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:04.204 | INFO | Uploaded 36GiB to GPU in 2.22553s (16.1759GiB/s) | 20260505 16:48:04.205 | INFO | Downloaded 34GiB to GPU in 2.12069s (16.0325GiB/s) | 20260505 16:48:09.335 | INFO | Analyzing chunk frames = [4761600,4762112) | 20260505 16:48:09.335 | INFO | MemInfo: { | 20260505 16:48:09.335 | INFO | "gpu" : " 17 GB", | 20260505 16:48:09.335 | INFO | "huge" : "", | 20260505 16:48:09.335 | INFO | "huge_shared" : " 27 GB", | 20260505 16:48:09.335 | INFO | "malloc" : " 14 GB", | 20260505 16:48:09.335 | INFO | "pinned" : " 29 GB" | 20260505 16:48:09.335 | INFO | } | 20260505 16:48:09.335 | INFO | Perf report for the last 5.13093 seconds: | 20260505 16:48:09.335 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:48:09.335 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:09.335 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:09.335 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:09.335 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 16:48:09.335 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:09.335 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:09.335 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 16:48:09.335 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:09.335 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:09.335 | INFO | Uploaded 36GiB to GPU in 2.1484s (16.7566GiB/s) | 20260505 16:48:09.335 | INFO | Downloaded 34GiB to GPU in 2.06005s (16.5044GiB/s) | 20260505 16:48:14.469 | INFO | Analyzing chunk frames = [4762112,4762624) | 20260505 16:48:14.469 | INFO | MemInfo: { | 20260505 16:48:14.469 | INFO | "gpu" : " 17 GB", | 20260505 16:48:14.469 | INFO | "huge" : "", | 20260505 16:48:14.469 | INFO | "huge_shared" : " 27 GB", | 20260505 16:48:14.469 | INFO | "malloc" : " 14 GB", | 20260505 16:48:14.469 | INFO | "pinned" : " 29 GB" | 20260505 16:48:14.469 | INFO | } | 20260505 16:48:14.469 | INFO | Perf report for the last 5.13447 seconds: | 20260505 16:48:14.469 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:48:14.469 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:14.469 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:14.469 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:14.469 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 16:48:14.469 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:14.469 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:14.469 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:48:14.469 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:14.469 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:14.470 | INFO | Uploaded 36GiB to GPU in 2.16342s (16.6403GiB/s) | 20260505 16:48:14.470 | INFO | Downloaded 34GiB to GPU in 2.03893s (16.6754GiB/s) | 20260505 16:48:19.619 | INFO | Analyzing chunk frames = [4762624,4763136) | 20260505 16:48:19.619 | INFO | MemInfo: { | 20260505 16:48:19.619 | INFO | "gpu" : " 17 GB", | 20260505 16:48:19.619 | INFO | "huge" : "", | 20260505 16:48:19.619 | INFO | "huge_shared" : " 27 GB", | 20260505 16:48:19.619 | INFO | "malloc" : " 14 GB", | 20260505 16:48:19.619 | INFO | "pinned" : " 29 GB" | 20260505 16:48:19.619 | INFO | } | 20260505 16:48:19.619 | INFO | Perf report for the last 5.15011 seconds: | 20260505 16:48:19.619 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:48:19.619 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:19.619 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:19.619 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:19.619 | INFO | ANALYSIS : 63.2%, 3.0, 98.8ms, 0 | 20260505 16:48:19.619 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:19.619 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:19.620 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 16:48:19.620 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:19.620 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:19.620 | INFO | Uploaded 36GiB to GPU in 2.11516s (17.02GiB/s) | 20260505 16:48:19.620 | INFO | Downloaded 34GiB to GPU in 2.16278s (15.7205GiB/s) | 20260505 16:48:24.759 | INFO | Analyzing chunk frames = [4763136,4763648) | 20260505 16:48:24.759 | INFO | MemInfo: { | 20260505 16:48:24.759 | INFO | "gpu" : " 17 GB", | 20260505 16:48:24.759 | INFO | "huge" : "", | 20260505 16:48:24.759 | INFO | "huge_shared" : " 27 GB", | 20260505 16:48:24.759 | INFO | "malloc" : " 14 GB", | 20260505 16:48:24.759 | INFO | "pinned" : " 29 GB" | 20260505 16:48:24.759 | INFO | } | 20260505 16:48:24.759 | INFO | Perf report for the last 5.13945 seconds: | 20260505 16:48:24.759 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:48:24.759 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:24.759 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:24.759 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:24.759 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 16:48:24.759 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:24.759 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:24.759 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 16:48:24.759 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:24.759 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:24.759 | INFO | Uploaded 36GiB to GPU in 2.18702s (16.4608GiB/s) | 20260505 16:48:24.759 | INFO | Downloaded 34GiB to GPU in 2.02218s (16.8135GiB/s) | 20260505 16:48:24.900 | INFO | Starting a round of Baz IO | 20260505 16:48:28.090 | INFO | Populating RTMetrics report | 20260505 16:48:28.091 | INFO | RTMetrics report populated | 20260505 16:48:28.091 | INFO | Opening tmp RTMetrics json file | 20260505 16:48:28.101 | INFO | Serializing RTMetrics to json | 20260505 16:48:28.102 | INFO | Serialization done, swapping tmp file now... | 20260505 16:48:28.103 | INFO | RTMetrics done | 20260505 16:48:28.186 | INFO | Finished a round of Baz IO: Wrote 2.158 GiB in 3.286 seconds | 20260505 16:48:28.186 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.17% / 0.00% / 77.82% | 20260505 16:48:28.186 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.34097% / 0% / 7.54992e-05% | 20260505 16:48:28.186 | INFO | Baz IO overall: | 20260505 16:48:28.186 | INFO | BytesWritten : 2823200809074 | 20260505 16:48:28.187 | INFO | NumEvents : 1339586805944 | 20260505 16:48:29.894 | INFO | Analyzing chunk frames = [4763648,4764160) | 20260505 16:48:29.894 | INFO | MemInfo: { | 20260505 16:48:29.894 | INFO | "gpu" : " 17 GB", | 20260505 16:48:29.894 | INFO | "huge" : "", | 20260505 16:48:29.894 | INFO | "huge_shared" : " 27 GB", | 20260505 16:48:29.894 | INFO | "malloc" : " 14 GB", | 20260505 16:48:29.894 | INFO | "pinned" : " 29 GB" | 20260505 16:48:29.894 | INFO | } | 20260505 16:48:29.894 | INFO | Perf report for the last 5.13519 seconds: | 20260505 16:48:29.894 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:48:29.894 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:29.894 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:29.894 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:29.894 | INFO | ANALYSIS : 63.8%, 3.0, 99.2ms, 0 | 20260505 16:48:29.894 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:29.894 | INFO | RT_METRICS : 44.9%, 1.0, 23.5ms, 0 | 20260505 16:48:29.894 | INFO | PRE_HQ : 60.3%, 1.8, 56.4ms, 0 | 20260505 16:48:29.894 | INFO | BAZWRITER : 63.5%, 3.9, 126.8ms, 0 | 20260505 16:48:29.894 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260505 16:48:29.894 | INFO | Uploaded 36GiB to GPU in 2.17669s (16.5389GiB/s) | 20260505 16:48:29.894 | INFO | Downloaded 38GiB to GPU in 2.27209s (16.7247GiB/s) | 20260505 16:48:35.027 | INFO | Analyzing chunk frames = [4764160,4764672) | 20260505 16:48:35.027 | INFO | MemInfo: { | 20260505 16:48:35.027 | INFO | "gpu" : " 17 GB", | 20260505 16:48:35.027 | INFO | "huge" : "", | 20260505 16:48:35.027 | INFO | "huge_shared" : " 27 GB", | 20260505 16:48:35.027 | INFO | "malloc" : " 14 GB", | 20260505 16:48:35.027 | INFO | "pinned" : " 29 GB" | 20260505 16:48:35.027 | INFO | } | 20260505 16:48:35.027 | INFO | Perf report for the last 5.13274 seconds: | 20260505 16:48:35.027 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:48:35.027 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:35.027 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:35.027 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:35.027 | INFO | ANALYSIS : 63.1%, 3.0, 98.3ms, 0 | 20260505 16:48:35.027 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:35.027 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:35.027 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:48:35.027 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:35.027 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:35.027 | INFO | Uploaded 36GiB to GPU in 2.13998s (16.8226GiB/s) | 20260505 16:48:35.027 | INFO | Downloaded 34GiB to GPU in 2.0473s (16.6072GiB/s) | 20260505 16:48:40.169 | INFO | Analyzing chunk frames = [4764672,4765184) | 20260505 16:48:40.169 | INFO | MemInfo: { | 20260505 16:48:40.169 | INFO | "gpu" : " 17 GB", | 20260505 16:48:40.170 | INFO | "huge" : "", | 20260505 16:48:40.170 | INFO | "huge_shared" : " 27 GB", | 20260505 16:48:40.170 | INFO | "malloc" : " 14 GB", | 20260505 16:48:40.170 | INFO | "pinned" : " 29 GB" | 20260505 16:48:40.170 | INFO | } | 20260505 16:48:40.170 | INFO | Perf report for the last 5.14295 seconds: | 20260505 16:48:40.170 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:48:40.170 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:40.170 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:40.170 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:40.170 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260505 16:48:40.170 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:40.170 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:40.170 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:48:40.170 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:40.170 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:40.170 | INFO | Uploaded 36GiB to GPU in 2.17375s (16.5613GiB/s) | 20260505 16:48:40.170 | INFO | Downloaded 34GiB to GPU in 2.04364s (16.637GiB/s) | 20260505 16:48:45.313 | INFO | Analyzing chunk frames = [4765184,4765696) | 20260505 16:48:45.313 | INFO | MemInfo: { | 20260505 16:48:45.313 | INFO | "gpu" : " 17 GB", | 20260505 16:48:45.313 | INFO | "huge" : "", | 20260505 16:48:45.313 | INFO | "huge_shared" : " 27 GB", | 20260505 16:48:45.313 | INFO | "malloc" : " 14 GB", | 20260505 16:48:45.313 | INFO | "pinned" : " 29 GB" | 20260505 16:48:45.313 | INFO | } | 20260505 16:48:45.313 | INFO | Perf report for the last 5.14356 seconds: | 20260505 16:48:45.313 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:48:45.313 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:45.313 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:45.313 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:45.313 | INFO | ANALYSIS : 62.5%, 3.0, 97.2ms, 0 | 20260505 16:48:45.313 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:45.313 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:45.313 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:48:45.313 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:45.313 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:45.314 | INFO | Uploaded 36GiB to GPU in 2.16976s (16.5917GiB/s) | 20260505 16:48:45.314 | INFO | Downloaded 34GiB to GPU in 2.14178s (15.8747GiB/s) | 20260505 16:48:50.451 | INFO | Analyzing chunk frames = [4765696,4766208) | 20260505 16:48:50.451 | INFO | MemInfo: { | 20260505 16:48:50.451 | INFO | "gpu" : " 17 GB", | 20260505 16:48:50.451 | INFO | "huge" : "", | 20260505 16:48:50.451 | INFO | "huge_shared" : " 27 GB", | 20260505 16:48:50.451 | INFO | "malloc" : " 14 GB", | 20260505 16:48:50.452 | INFO | "pinned" : " 29 GB" | 20260505 16:48:50.452 | INFO | } | 20260505 16:48:50.452 | INFO | Perf report for the last 5.13823 seconds: | 20260505 16:48:50.452 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:48:50.452 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:50.452 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:50.452 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:50.452 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 16:48:50.452 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:50.452 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:50.452 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:48:50.452 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:50.452 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:50.452 | INFO | Uploaded 36GiB to GPU in 2.14533s (16.7806GiB/s) | 20260505 16:48:50.452 | INFO | Downloaded 34GiB to GPU in 2.05599s (16.537GiB/s) | 20260505 16:48:55.585 | INFO | Analyzing chunk frames = [4766208,4766720) | 20260505 16:48:55.585 | INFO | MemInfo: { | 20260505 16:48:55.585 | INFO | "gpu" : " 17 GB", | 20260505 16:48:55.585 | INFO | "huge" : "", | 20260505 16:48:55.585 | INFO | "huge_shared" : " 27 GB", | 20260505 16:48:55.585 | INFO | "malloc" : " 14 GB", | 20260505 16:48:55.585 | INFO | "pinned" : " 29 GB" | 20260505 16:48:55.585 | INFO | } | 20260505 16:48:55.585 | INFO | Perf report for the last 5.13364 seconds: | 20260505 16:48:55.585 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:48:55.585 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:55.585 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:55.585 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:55.585 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260505 16:48:55.585 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:55.585 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:55.585 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:48:55.585 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:55.585 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:48:55.586 | INFO | Uploaded 36GiB to GPU in 2.1377s (16.8405GiB/s) | 20260505 16:48:55.586 | INFO | Downloaded 34GiB to GPU in 2.03761s (16.6862GiB/s) | 20260505 16:49:00.722 | INFO | Analyzing chunk frames = [4766720,4767232) | 20260505 16:49:00.722 | INFO | MemInfo: { | 20260505 16:49:00.722 | INFO | "gpu" : " 17 GB", | 20260505 16:49:00.722 | INFO | "huge" : "", | 20260505 16:49:00.722 | INFO | "huge_shared" : " 27 GB", | 20260505 16:49:00.722 | INFO | "malloc" : " 14 GB", | 20260505 16:49:00.722 | INFO | "pinned" : " 29 GB" | 20260505 16:49:00.722 | INFO | } | 20260505 16:49:00.722 | INFO | Perf report for the last 5.13677 seconds: | 20260505 16:49:00.722 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:49:00.722 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:00.722 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:00.722 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:00.722 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 16:49:00.722 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:00.722 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:00.722 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 16:49:00.722 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:00.722 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:00.722 | INFO | Uploaded 36GiB to GPU in 2.17287s (16.5679GiB/s) | 20260505 16:49:00.722 | INFO | Downloaded 34GiB to GPU in 2.1491s (15.8206GiB/s) | 20260505 16:49:05.872 | INFO | Analyzing chunk frames = [4767232,4767744) | 20260505 16:49:05.872 | INFO | MemInfo: { | 20260505 16:49:05.872 | INFO | "gpu" : " 17 GB", | 20260505 16:49:05.872 | INFO | "huge" : "", | 20260505 16:49:05.872 | INFO | "huge_shared" : " 27 GB", | 20260505 16:49:05.872 | INFO | "malloc" : " 14 GB", | 20260505 16:49:05.872 | INFO | "pinned" : " 29 GB" | 20260505 16:49:05.872 | INFO | } | 20260505 16:49:05.872 | INFO | Perf report for the last 5.15046 seconds: | 20260505 16:49:05.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:49:05.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:05.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:05.872 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:05.873 | INFO | ANALYSIS : 63.5%, 3.0, 99.3ms, 0 | 20260505 16:49:05.873 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:05.873 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:05.873 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 16:49:05.873 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:05.873 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:05.873 | INFO | Uploaded 36GiB to GPU in 2.23286s (16.1228GiB/s) | 20260505 16:49:05.873 | INFO | Downloaded 34GiB to GPU in 2.10547s (16.1484GiB/s) | 20260505 16:49:05.873 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 16:49:05.873 | INFO | Total mem usage: 15.3 GiB | 20260505 16:49:05.873 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 16:49:05.873 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.9%) | 20260505 16:49:05.873 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 16:49:05.873 | INFO | High Water Breakdown: | 20260505 16:49:05.873 | INFO | 61% : 8.67 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 16:49:05.873 | INFO | 38.8% : 5.51 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 16:49:05.873 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 16:49:05.873 | INFO | ------------------------------------------ | 20260505 16:49:05.873 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:49:05.873 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 16:49:05.873 | INFO | Total mem usage: 26.5 GiB | 20260505 16:49:05.873 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 16:49:05.873 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 16:49:05.873 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 16:49:05.873 | INFO | High Water Breakdown: | 20260505 16:49:05.873 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:49:05.873 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:49:05.873 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:49:05.873 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:49:05.873 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:49:05.873 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:49:05.873 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 16:49:05.873 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 16:49:05.873 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:49:05.873 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 16:49:05.873 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 16:49:05.873 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 16:49:05.873 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 16:49:05.873 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 16:49:05.873 | INFO | ------------------------------------------ | 20260505 16:49:05.873 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:49:05.873 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 16:49:05.873 | INFO | Total mem usage: 25.6 GiB | 20260505 16:49:05.873 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 16:49:05.873 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 16:49:05.873 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 16:49:05.873 | INFO | High Water Breakdown: | 20260505 16:49:05.873 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 16:49:05.873 | INFO | ------------------------------------------ | 20260505 16:49:05.873 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:49:05.873 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 16:49:05.873 | INFO | Total mem usage: 15.7 GiB | 20260505 16:49:05.873 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 16:49:05.874 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 16:49:05.874 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 16:49:05.874 | INFO | High Water Breakdown: | 20260505 16:49:05.874 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 16:49:05.874 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 16:49:05.874 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 16:49:05.874 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 16:49:05.874 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 16:49:05.874 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 16:49:05.874 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 16:49:05.874 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 16:49:05.874 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 16:49:05.874 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:49:05.874 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 16:49:05.874 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:49:05.874 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 16:49:05.874 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 16:49:05.874 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 16:49:05.874 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 16:49:05.874 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:49:05.874 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 16:49:05.874 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:49:05.874 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:49:05.874 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:49:05.874 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 16:49:05.874 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 16:49:05.874 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:49:05.874 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 16:49:05.874 | INFO | ------------------------------------------ | 20260505 16:49:05.874 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:49:05.874 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 16:49:05.874 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:49:05.874 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:49:05.874 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.47e+01, 1.35e+01 / 1.92e+00 | 20260505 16:49:05.874 | INFO | PerfReport: 0.29%, AggregateStats -- 9.25e-02 / 1.02e-01, 1.06e-02 / 5.06e-03 | 20260505 16:49:05.874 | INFO | PerfReport: 11.76%, Baseline -- 3.72e+00 / 4.49e+00, 5.62e-01 / 2.30e-01 | 20260505 16:49:05.874 | INFO | PerfReport: 3.26%, Histogram -- 1.03e+00 / 1.51e+00, 1.89e-01 / 1.13e-01 | 20260505 16:49:05.874 | INFO | PerfReport: 15.94%, DME -- 4.03e+01 / 4.46e+01, 4.12e+00 / 1.83e+00 | 20260505 16:49:05.874 | INFO | PerfReport: 56.76%, FrameLabeling -- 1.79e+01 / 1.93e+01, 1.45e+00 / 1.29e+00 | 20260505 16:49:05.874 | INFO | PerfReport: 7.39%, PulseAccumulating -- 2.33e+00 / 2.90e+00, 4.50e-01 / 2.13e-01 | 20260505 16:49:05.874 | INFO | PerfReport: 3.41%, Metrics -- 1.08e+00 / 1.41e+00, 2.97e-01 / 1.46e-01 | 20260505 16:49:05.874 | INFO | PerfReport: 1.19%, PulseSerializing -- 3.75e-01 / 4.38e-01, 6.72e-02 / 3.82e-02 | 20260505 16:49:05.874 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 16:49:05.874 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:49:05.874 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:49:05.874 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.58e+01 / 0.00e+00 | 20260505 16:49:05.874 | INFO | PerfReport: 22.58%, Upload -- 2.26e+01 / 0.00e+00, 7.88e+00 / 0.00e+00 | 20260505 16:49:05.874 | INFO | PerfReport: 21.84%, Download -- 2.19e+01 / 0.00e+00, 7.87e+00 / 0.00e+00 | 20260505 16:49:05.874 | INFO | PerfReport: 31.57%, Compute -- 3.16e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 16:49:05.874 | INFO | PerfReport: 2.36%, LockUpload -- 2.37e+00 / 0.00e+00, 8.00e+00 / 0.00e+00 | 20260505 16:49:05.874 | INFO | PerfReport: 1.41%, LockDownload -- 1.41e+00 / 0.00e+00, 2.97e+00 / 0.00e+00 | 20260505 16:49:05.874 | INFO | PerfReport: 20.22%, LockCompute -- 2.03e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 16:49:05.874 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 2.78e-02 / 0.00e+00 | 20260505 16:49:06.010 | INFO | Starting a round of Baz IO | 20260505 16:49:09.188 | INFO | Populating RTMetrics report | 20260505 16:49:09.188 | INFO | RTMetrics report populated | 20260505 16:49:09.188 | INFO | Opening tmp RTMetrics json file | 20260505 16:49:09.200 | INFO | Serializing RTMetrics to json | 20260505 16:49:09.203 | INFO | Serialization done, swapping tmp file now... | 20260505 16:49:09.205 | INFO | RTMetrics done | 20260505 16:49:09.282 | INFO | Finished a round of Baz IO: Wrote 2.158 GiB in 3.271 seconds | 20260505 16:49:09.282 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.17% / 0.00% / 77.83% | 20260505 16:49:09.282 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.341153% / 0% / 7.94729e-05% | 20260505 16:49:09.282 | INFO | Baz IO overall: | 20260505 16:49:09.282 | INFO | BytesWritten : 2825517990350 | 20260505 16:49:09.282 | INFO | NumEvents : 1340671887899 | 20260505 16:49:11.016 | INFO | Analyzing chunk frames = [4767744,4768256) | 20260505 16:49:11.016 | INFO | MemInfo: { | 20260505 16:49:11.016 | INFO | "gpu" : " 17 GB", | 20260505 16:49:11.016 | INFO | "huge" : "", | 20260505 16:49:11.016 | INFO | "huge_shared" : " 27 GB", | 20260505 16:49:11.016 | INFO | "malloc" : " 14 GB", | 20260505 16:49:11.016 | INFO | "pinned" : " 29 GB" | 20260505 16:49:11.016 | INFO | } | 20260505 16:49:11.016 | INFO | Perf report for the last 5.14348 seconds: | 20260505 16:49:11.016 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:49:11.016 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:11.016 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:11.016 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:11.016 | INFO | ANALYSIS : 63.4%, 3.0, 98.8ms, 0 | 20260505 16:49:11.016 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:11.016 | INFO | RT_METRICS : 44.9%, 1.0, 23.5ms, 0 | 20260505 16:49:11.016 | INFO | PRE_HQ : 60.6%, 1.8, 56.4ms, 0 | 20260505 16:49:11.016 | INFO | BAZWRITER : 63.1%, 3.8, 124.7ms, 0 | 20260505 16:49:11.016 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 16:49:11.016 | INFO | Uploaded 36GiB to GPU in 2.14729s (16.7653GiB/s) | 20260505 16:49:11.016 | INFO | Downloaded 38GiB to GPU in 2.24597s (16.9192GiB/s) | 20260505 16:49:16.154 | INFO | Analyzing chunk frames = [4768256,4768768) | 20260505 16:49:16.154 | INFO | MemInfo: { | 20260505 16:49:16.154 | INFO | "gpu" : " 17 GB", | 20260505 16:49:16.154 | INFO | "huge" : "", | 20260505 16:49:16.154 | INFO | "huge_shared" : " 27 GB", | 20260505 16:49:16.154 | INFO | "malloc" : " 14 GB", | 20260505 16:49:16.154 | INFO | "pinned" : " 29 GB" | 20260505 16:49:16.154 | INFO | } | 20260505 16:49:16.154 | INFO | Perf report for the last 5.13819 seconds: | 20260505 16:49:16.154 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:49:16.154 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:16.154 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:16.154 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:16.154 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 16:49:16.154 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:16.154 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:16.154 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:49:16.154 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:16.154 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:16.154 | INFO | Uploaded 36GiB to GPU in 2.13813s (16.8372GiB/s) | 20260505 16:49:16.154 | INFO | Downloaded 34GiB to GPU in 2.03069s (16.7431GiB/s) | 20260505 16:49:21.282 | INFO | Analyzing chunk frames = [4768768,4769280) | 20260505 16:49:21.282 | INFO | MemInfo: { | 20260505 16:49:21.282 | INFO | "gpu" : " 17 GB", | 20260505 16:49:21.282 | INFO | "huge" : "", | 20260505 16:49:21.282 | INFO | "huge_shared" : " 27 GB", | 20260505 16:49:21.283 | INFO | "malloc" : " 14 GB", | 20260505 16:49:21.283 | INFO | "pinned" : " 29 GB" | 20260505 16:49:21.283 | INFO | } | 20260505 16:49:21.283 | INFO | Perf report for the last 5.12851 seconds: | 20260505 16:49:21.283 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:49:21.283 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:21.283 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:21.283 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:21.283 | INFO | ANALYSIS : 62.9%, 3.0, 97.6ms, 0 | 20260505 16:49:21.283 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:21.283 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:21.283 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:49:21.283 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:21.283 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:21.283 | INFO | Uploaded 36GiB to GPU in 2.20066s (16.3587GiB/s) | 20260505 16:49:21.283 | INFO | Downloaded 34GiB to GPU in 2.04432s (16.6315GiB/s) | 20260505 16:49:26.426 | INFO | Analyzing chunk frames = [4769280,4769792) | 20260505 16:49:26.426 | INFO | MemInfo: { | 20260505 16:49:26.426 | INFO | "gpu" : " 17 GB", | 20260505 16:49:26.427 | INFO | "huge" : "", | 20260505 16:49:26.427 | INFO | "huge_shared" : " 27 GB", | 20260505 16:49:26.427 | INFO | "malloc" : " 14 GB", | 20260505 16:49:26.427 | INFO | "pinned" : " 29 GB" | 20260505 16:49:26.427 | INFO | } | 20260505 16:49:26.427 | INFO | Perf report for the last 5.14396 seconds: | 20260505 16:49:26.427 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:49:26.427 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:26.427 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:26.427 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:26.427 | INFO | ANALYSIS : 62.4%, 3.0, 97.0ms, 0 | 20260505 16:49:26.427 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:26.427 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:26.427 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:49:26.427 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:26.427 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:26.427 | INFO | Uploaded 36GiB to GPU in 2.19319s (16.4144GiB/s) | 20260505 16:49:26.427 | INFO | Downloaded 34GiB to GPU in 2.13613s (15.9166GiB/s) | 20260505 16:49:31.568 | INFO | Analyzing chunk frames = [4769792,4770304) | 20260505 16:49:31.568 | INFO | MemInfo: { | 20260505 16:49:31.568 | INFO | "gpu" : " 17 GB", | 20260505 16:49:31.568 | INFO | "huge" : "", | 20260505 16:49:31.568 | INFO | "huge_shared" : " 27 GB", | 20260505 16:49:31.569 | INFO | "malloc" : " 14 GB", | 20260505 16:49:31.569 | INFO | "pinned" : " 29 GB" | 20260505 16:49:31.569 | INFO | } | 20260505 16:49:31.569 | INFO | Perf report for the last 5.14194 seconds: | 20260505 16:49:31.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:49:31.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:31.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:31.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:31.569 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 16:49:31.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:31.569 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:31.569 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 16:49:31.569 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:31.569 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:31.569 | INFO | Uploaded 36GiB to GPU in 2.15894s (16.6748GiB/s) | 20260505 16:49:31.569 | INFO | Downloaded 34GiB to GPU in 2.06128s (16.4946GiB/s) | 20260505 16:49:36.708 | INFO | Analyzing chunk frames = [4770304,4770816) | 20260505 16:49:36.708 | INFO | MemInfo: { | 20260505 16:49:36.708 | INFO | "gpu" : " 17 GB", | 20260505 16:49:36.708 | INFO | "huge" : "", | 20260505 16:49:36.709 | INFO | "huge_shared" : " 27 GB", | 20260505 16:49:36.709 | INFO | "malloc" : " 14 GB", | 20260505 16:49:36.709 | INFO | "pinned" : " 29 GB" | 20260505 16:49:36.709 | INFO | } | 20260505 16:49:36.709 | INFO | Perf report for the last 5.14008 seconds: | 20260505 16:49:36.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:49:36.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:36.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:36.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:36.709 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260505 16:49:36.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:36.709 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:36.709 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 16:49:36.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:36.709 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:36.709 | INFO | Uploaded 36GiB to GPU in 2.14287s (16.7999GiB/s) | 20260505 16:49:36.709 | INFO | Downloaded 34GiB to GPU in 2.04738s (16.6066GiB/s) | 20260505 16:49:41.842 | INFO | Analyzing chunk frames = [4770816,4771328) | 20260505 16:49:41.842 | INFO | MemInfo: { | 20260505 16:49:41.842 | INFO | "gpu" : " 17 GB", | 20260505 16:49:41.842 | INFO | "huge" : "", | 20260505 16:49:41.842 | INFO | "huge_shared" : " 27 GB", | 20260505 16:49:41.842 | INFO | "malloc" : " 14 GB", | 20260505 16:49:41.842 | INFO | "pinned" : " 29 GB" | 20260505 16:49:41.842 | INFO | } | 20260505 16:49:41.842 | INFO | Perf report for the last 5.13332 seconds: | 20260505 16:49:41.842 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:49:41.842 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:41.842 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:41.842 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:41.842 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 16:49:41.842 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:41.842 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:41.842 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 16:49:41.842 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:41.842 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:41.842 | INFO | Uploaded 36GiB to GPU in 2.11549s (17.0174GiB/s) | 20260505 16:49:41.842 | INFO | Downloaded 34GiB to GPU in 2.17846s (15.6074GiB/s) | 20260505 16:49:46.977 | INFO | Analyzing chunk frames = [4771328,4771840) | 20260505 16:49:46.977 | INFO | MemInfo: { | 20260505 16:49:46.977 | INFO | "gpu" : " 17 GB", | 20260505 16:49:46.977 | INFO | "huge" : "", | 20260505 16:49:46.977 | INFO | "huge_shared" : " 27 GB", | 20260505 16:49:46.977 | INFO | "malloc" : " 14 GB", | 20260505 16:49:46.977 | INFO | "pinned" : " 29 GB" | 20260505 16:49:46.977 | INFO | } | 20260505 16:49:46.977 | INFO | Perf report for the last 5.13491 seconds: | 20260505 16:49:46.977 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:49:46.977 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:46.977 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:46.977 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:46.977 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 16:49:46.977 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:46.977 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:46.977 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 16:49:46.977 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:46.977 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:46.977 | INFO | Uploaded 36GiB to GPU in 2.17997s (16.514GiB/s) | 20260505 16:49:46.977 | INFO | Downloaded 34GiB to GPU in 2.0329s (16.7248GiB/s) | 20260505 16:49:47.108 | INFO | Starting a round of Baz IO | 20260505 16:49:50.322 | INFO | Populating RTMetrics report | 20260505 16:49:50.322 | INFO | RTMetrics report populated | 20260505 16:49:50.322 | INFO | Opening tmp RTMetrics json file | 20260505 16:49:50.325 | INFO | Serializing RTMetrics to json | 20260505 16:49:50.328 | INFO | Serialization done, swapping tmp file now... | 20260505 16:49:50.329 | INFO | RTMetrics done | 20260505 16:49:50.420 | INFO | Finished a round of Baz IO: Wrote 2.158 GiB in 3.311 seconds | 20260505 16:49:50.420 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.17% / 0.00% / 77.83% | 20260505 16:49:50.420 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.339838% / 0% / 0.000127157% | 20260505 16:49:50.420 | INFO | Baz IO overall: | 20260505 16:49:50.420 | INFO | BytesWritten : 2827835106458 | 20260505 16:49:50.420 | INFO | NumEvents : 1341757008242 | 20260505 16:49:52.129 | INFO | Analyzing chunk frames = [4771840,4772352) | 20260505 16:49:52.129 | INFO | MemInfo: { | 20260505 16:49:52.129 | INFO | "gpu" : " 17 GB", | 20260505 16:49:52.129 | INFO | "huge" : "", | 20260505 16:49:52.129 | INFO | "huge_shared" : " 27 GB", | 20260505 16:49:52.129 | INFO | "malloc" : " 14 GB", | 20260505 16:49:52.129 | INFO | "pinned" : " 29 GB" | 20260505 16:49:52.129 | INFO | } | 20260505 16:49:52.129 | INFO | Perf report for the last 5.15245 seconds: | 20260505 16:49:52.129 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:49:52.129 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:52.129 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:52.130 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:52.130 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 16:49:52.130 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:52.130 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260505 16:49:52.130 | INFO | PRE_HQ : 60.6%, 1.8, 56.3ms, 0 | 20260505 16:49:52.130 | INFO | BAZWRITER : 63.6%, 3.8, 125.5ms, 0 | 20260505 16:49:52.130 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 16:49:52.130 | INFO | Uploaded 36GiB to GPU in 2.19439s (16.4054GiB/s) | 20260505 16:49:52.130 | INFO | Downloaded 38GiB to GPU in 2.25822s (16.8274GiB/s) | 20260505 16:49:57.267 | INFO | Analyzing chunk frames = [4772352,4772864) | 20260505 16:49:57.267 | INFO | MemInfo: { | 20260505 16:49:57.267 | INFO | "gpu" : " 17 GB", | 20260505 16:49:57.267 | INFO | "huge" : "", | 20260505 16:49:57.268 | INFO | "huge_shared" : " 27 GB", | 20260505 16:49:57.268 | INFO | "malloc" : " 14 GB", | 20260505 16:49:57.268 | INFO | "pinned" : " 29 GB" | 20260505 16:49:57.268 | INFO | } | 20260505 16:49:57.268 | INFO | Perf report for the last 5.13827 seconds: | 20260505 16:49:57.268 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:49:57.268 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:57.268 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:57.268 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:57.268 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 16:49:57.268 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:57.268 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:57.268 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:49:57.268 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:57.268 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:49:57.268 | INFO | Uploaded 36GiB to GPU in 2.12317s (16.9558GiB/s) | 20260505 16:49:57.268 | INFO | Downloaded 34GiB to GPU in 2.02226s (16.8128GiB/s) | 20260505 16:50:02.400 | INFO | Analyzing chunk frames = [4772864,4773376) | 20260505 16:50:02.400 | INFO | MemInfo: { | 20260505 16:50:02.400 | INFO | "gpu" : " 17 GB", | 20260505 16:50:02.400 | INFO | "huge" : "", | 20260505 16:50:02.400 | INFO | "huge_shared" : " 27 GB", | 20260505 16:50:02.400 | INFO | "malloc" : " 14 GB", | 20260505 16:50:02.400 | INFO | "pinned" : " 29 GB" | 20260505 16:50:02.400 | INFO | } | 20260505 16:50:02.400 | INFO | Perf report for the last 5.1327 seconds: | 20260505 16:50:02.400 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:50:02.400 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:02.400 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:02.401 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:02.401 | INFO | ANALYSIS : 62.9%, 3.0, 97.7ms, 0 | 20260505 16:50:02.401 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:02.401 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:02.401 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 16:50:02.401 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:02.401 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:02.401 | INFO | Uploaded 36GiB to GPU in 2.20449s (16.3303GiB/s) | 20260505 16:50:02.401 | INFO | Downloaded 34GiB to GPU in 2.04389s (16.6349GiB/s) | 20260505 16:50:07.532 | INFO | Analyzing chunk frames = [4773376,4773888) | 20260505 16:50:07.532 | INFO | MemInfo: { | 20260505 16:50:07.532 | INFO | "gpu" : " 17 GB", | 20260505 16:50:07.532 | INFO | "huge" : "", | 20260505 16:50:07.533 | INFO | "huge_shared" : " 27 GB", | 20260505 16:50:07.533 | INFO | "malloc" : " 14 GB", | 20260505 16:50:07.533 | INFO | "pinned" : " 29 GB" | 20260505 16:50:07.533 | INFO | } | 20260505 16:50:07.533 | INFO | Perf report for the last 5.13226 seconds: | 20260505 16:50:07.533 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:50:07.533 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:07.533 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:07.533 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:07.533 | INFO | ANALYSIS : 62.5%, 3.0, 96.9ms, 0 | 20260505 16:50:07.533 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:07.533 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:07.533 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:50:07.533 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:07.533 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:07.533 | INFO | Uploaded 36GiB to GPU in 2.18045s (16.5103GiB/s) | 20260505 16:50:07.533 | INFO | Downloaded 34GiB to GPU in 2.13119s (15.9535GiB/s) | 20260505 16:50:12.679 | INFO | Analyzing chunk frames = [4773888,4774400) | 20260505 16:50:12.679 | INFO | MemInfo: { | 20260505 16:50:12.679 | INFO | "gpu" : " 17 GB", | 20260505 16:50:12.679 | INFO | "huge" : "", | 20260505 16:50:12.679 | INFO | "huge_shared" : " 27 GB", | 20260505 16:50:12.679 | INFO | "malloc" : " 14 GB", | 20260505 16:50:12.679 | INFO | "pinned" : " 29 GB" | 20260505 16:50:12.679 | INFO | } | 20260505 16:50:12.679 | INFO | Perf report for the last 5.14616 seconds: | 20260505 16:50:12.679 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:50:12.679 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:12.679 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:12.679 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:12.679 | INFO | ANALYSIS : 62.6%, 3.0, 97.9ms, 0 | 20260505 16:50:12.679 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:12.679 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:12.679 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260505 16:50:12.679 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:12.679 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:12.679 | INFO | Uploaded 36GiB to GPU in 2.17478s (16.5534GiB/s) | 20260505 16:50:12.679 | INFO | Downloaded 34GiB to GPU in 2.12707s (15.9844GiB/s) | 20260505 16:50:17.824 | INFO | Analyzing chunk frames = [4774400,4774912) | 20260505 16:50:17.824 | INFO | MemInfo: { | 20260505 16:50:17.824 | INFO | "gpu" : " 17 GB", | 20260505 16:50:17.824 | INFO | "huge" : "", | 20260505 16:50:17.825 | INFO | "huge_shared" : " 27 GB", | 20260505 16:50:17.825 | INFO | "malloc" : " 14 GB", | 20260505 16:50:17.825 | INFO | "pinned" : " 29 GB" | 20260505 16:50:17.825 | INFO | } | 20260505 16:50:17.825 | INFO | Perf report for the last 5.14583 seconds: | 20260505 16:50:17.825 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:50:17.825 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:17.825 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:17.825 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:17.825 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260505 16:50:17.825 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:17.825 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:17.825 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 16:50:17.825 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:17.825 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:17.825 | INFO | Uploaded 36GiB to GPU in 2.14728s (16.7654GiB/s) | 20260505 16:50:17.825 | INFO | Downloaded 34GiB to GPU in 2.0399s (16.6675GiB/s) | 20260505 16:50:22.965 | INFO | Analyzing chunk frames = [4774912,4775424) | 20260505 16:50:22.965 | INFO | MemInfo: { | 20260505 16:50:22.965 | INFO | "gpu" : " 17 GB", | 20260505 16:50:22.965 | INFO | "huge" : "", | 20260505 16:50:22.965 | INFO | "huge_shared" : " 27 GB", | 20260505 16:50:22.965 | INFO | "malloc" : " 14 GB", | 20260505 16:50:22.965 | INFO | "pinned" : " 29 GB" | 20260505 16:50:22.966 | INFO | } | 20260505 16:50:22.966 | INFO | Perf report for the last 5.14091 seconds: | 20260505 16:50:22.966 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:50:22.966 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:22.966 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:22.966 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:22.966 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 16:50:22.966 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:22.966 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:22.966 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 16:50:22.966 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:22.966 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:22.966 | INFO | Uploaded 36GiB to GPU in 2.13455s (16.8654GiB/s) | 20260505 16:50:22.966 | INFO | Downloaded 34GiB to GPU in 2.14556s (15.8467GiB/s) | 20260505 16:50:28.100 | INFO | Analyzing chunk frames = [4775424,4775936) | 20260505 16:50:28.100 | INFO | MemInfo: { | 20260505 16:50:28.100 | INFO | "gpu" : " 17 GB", | 20260505 16:50:28.100 | INFO | "huge" : "", | 20260505 16:50:28.100 | INFO | "huge_shared" : " 27 GB", | 20260505 16:50:28.100 | INFO | "malloc" : " 14 GB", | 20260505 16:50:28.100 | INFO | "pinned" : " 29 GB" | 20260505 16:50:28.100 | INFO | } | 20260505 16:50:28.100 | INFO | Perf report for the last 5.13429 seconds: | 20260505 16:50:28.100 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:50:28.100 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:28.100 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:28.100 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:28.100 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 16:50:28.100 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:28.100 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:28.100 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:50:28.100 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:28.100 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:28.100 | INFO | Uploaded 36GiB to GPU in 2.19829s (16.3763GiB/s) | 20260505 16:50:28.100 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 16:50:28.100 | INFO | Total mem usage: 15.3 GiB | 20260505 16:50:28.100 | INFO | Downloaded 34GiB to GPU in 2.02572s (16.7841GiB/s) | 20260505 16:50:28.100 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 16:50:28.101 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.89%) | 20260505 16:50:28.101 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 16:50:28.101 | INFO | High Water Breakdown: | 20260505 16:50:28.101 | INFO | 61% : 8.67 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 16:50:28.101 | INFO | 38.8% : 5.51 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 16:50:28.101 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 16:50:28.101 | INFO | ------------------------------------------ | 20260505 16:50:28.101 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:50:28.101 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 16:50:28.101 | INFO | Total mem usage: 26.5 GiB | 20260505 16:50:28.101 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 16:50:28.101 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 16:50:28.101 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 16:50:28.101 | INFO | High Water Breakdown: | 20260505 16:50:28.101 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:50:28.101 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:50:28.101 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:50:28.101 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:50:28.101 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:50:28.101 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:50:28.101 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 16:50:28.101 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 16:50:28.101 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:50:28.101 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 16:50:28.101 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 16:50:28.101 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 16:50:28.101 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 16:50:28.101 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 16:50:28.101 | INFO | ------------------------------------------ | 20260505 16:50:28.101 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:50:28.101 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 16:50:28.101 | INFO | Total mem usage: 25.6 GiB | 20260505 16:50:28.101 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 16:50:28.101 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 16:50:28.101 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 16:50:28.101 | INFO | High Water Breakdown: | 20260505 16:50:28.101 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 16:50:28.101 | INFO | ------------------------------------------ | 20260505 16:50:28.101 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:50:28.101 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 16:50:28.101 | INFO | Total mem usage: 15.7 GiB | 20260505 16:50:28.101 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 16:50:28.101 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 16:50:28.101 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 16:50:28.101 | INFO | High Water Breakdown: | 20260505 16:50:28.101 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 16:50:28.101 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 16:50:28.101 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 16:50:28.101 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 16:50:28.101 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 16:50:28.101 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 16:50:28.101 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 16:50:28.101 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 16:50:28.101 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 16:50:28.101 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:50:28.101 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 16:50:28.101 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:50:28.101 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 16:50:28.101 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 16:50:28.101 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 16:50:28.101 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 16:50:28.101 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:50:28.101 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 16:50:28.101 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:50:28.101 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:50:28.101 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:50:28.101 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 16:50:28.101 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 16:50:28.101 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:50:28.101 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 16:50:28.101 | INFO | ------------------------------------------ | 20260505 16:50:28.101 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:50:28.101 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 16:50:28.101 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:50:28.101 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:50:28.101 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.49e+01, 1.33e+01 / 2.08e+00 | 20260505 16:50:28.101 | INFO | PerfReport: 0.29%, AggregateStats -- 9.26e-02 / 1.02e-01, 1.10e-02 / 4.31e-03 | 20260505 16:50:28.101 | INFO | PerfReport: 11.79%, Baseline -- 3.73e+00 / 4.48e+00, 5.68e-01 / 1.79e-01 | 20260505 16:50:28.101 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.52e+00, 2.02e-01 / 1.09e-01 | 20260505 16:50:28.101 | INFO | PerfReport: 15.85%, DME -- 4.01e+01 / 4.41e+01, 4.06e+00 / 2.05e+00 | 20260505 16:50:28.101 | INFO | PerfReport: 56.81%, FrameLabeling -- 1.80e+01 / 1.99e+01, 1.46e+00 / 1.56e+00 | 20260505 16:50:28.101 | INFO | PerfReport: 7.37%, PulseAccumulating -- 2.33e+00 / 2.92e+00, 4.41e-01 / 1.58e-01 | 20260505 16:50:28.101 | INFO | PerfReport: 3.41%, Metrics -- 1.08e+00 / 1.41e+00, 2.99e-01 / 1.64e-01 | 20260505 16:50:28.101 | INFO | PerfReport: 1.19%, PulseSerializing -- 3.75e-01 / 4.43e-01, 6.71e-02 / 3.82e-02 | 20260505 16:50:28.101 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 16:50:28.101 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:50:28.101 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:50:28.101 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260505 16:50:28.101 | INFO | PerfReport: 22.53%, Upload -- 2.25e+01 / 0.00e+00, 7.87e+00 / 0.00e+00 | 20260505 16:50:28.101 | INFO | PerfReport: 21.84%, Download -- 2.19e+01 / 0.00e+00, 7.88e+00 / 0.00e+00 | 20260505 16:50:28.101 | INFO | PerfReport: 31.61%, Compute -- 3.16e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260505 16:50:28.102 | INFO | PerfReport: 2.19%, LockUpload -- 2.20e+00 / 0.00e+00, 7.86e+00 / 0.00e+00 | 20260505 16:50:28.102 | INFO | PerfReport: 1.44%, LockDownload -- 1.44e+00 / 0.00e+00, 2.97e+00 / 0.00e+00 | 20260505 16:50:28.102 | INFO | PerfReport: 20.36%, LockCompute -- 2.04e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 16:50:28.102 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.63e-02 / 0.00e+00 | 20260505 16:50:28.230 | INFO | Starting a round of Baz IO | 20260505 16:50:31.451 | INFO | Populating RTMetrics report | 20260505 16:50:31.451 | INFO | RTMetrics report populated | 20260505 16:50:31.451 | INFO | Opening tmp RTMetrics json file | 20260505 16:50:31.460 | INFO | Serializing RTMetrics to json | 20260505 16:50:31.461 | INFO | Serialization done, swapping tmp file now... | 20260505 16:50:31.462 | INFO | RTMetrics done | 20260505 16:50:31.549 | INFO | Finished a round of Baz IO: Wrote 2.159 GiB in 3.318 seconds | 20260505 16:50:31.549 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.17% / 0.00% / 77.83% | 20260505 16:50:31.549 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.339067% / 3.97364e-06% / 8.34465e-05% | 20260505 16:50:31.549 | INFO | Baz IO overall: | 20260505 16:50:31.549 | INFO | BytesWritten : 2830153182669 | 20260505 16:50:31.549 | INFO | NumEvents : 1342842906286 | 20260505 16:50:33.230 | INFO | Analyzing chunk frames = [4775936,4776448) | 20260505 16:50:33.230 | INFO | MemInfo: { | 20260505 16:50:33.230 | INFO | "gpu" : " 17 GB", | 20260505 16:50:33.230 | INFO | "huge" : "", | 20260505 16:50:33.230 | INFO | "huge_shared" : " 27 GB", | 20260505 16:50:33.230 | INFO | "malloc" : " 14 GB", | 20260505 16:50:33.230 | INFO | "pinned" : " 29 GB" | 20260505 16:50:33.230 | INFO | } | 20260505 16:50:33.230 | INFO | Perf report for the last 5.13017 seconds: | 20260505 16:50:33.230 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:50:33.230 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:33.230 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:33.230 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:33.230 | INFO | ANALYSIS : 64.2%, 3.0, 99.9ms, 0 | 20260505 16:50:33.230 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:33.230 | INFO | RT_METRICS : 44.9%, 1.0, 23.5ms, 0 | 20260505 16:50:33.230 | INFO | PRE_HQ : 61.0%, 1.8, 56.4ms, 0 | 20260505 16:50:33.230 | INFO | BAZWRITER : 64.0%, 3.8, 125.1ms, 0 | 20260505 16:50:33.230 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 16:50:33.231 | INFO | Uploaded 36GiB to GPU in 2.21372s (16.2622GiB/s) | 20260505 16:50:33.231 | INFO | Downloaded 38GiB to GPU in 2.26395s (16.7848GiB/s) | 20260505 16:50:38.380 | INFO | Analyzing chunk frames = [4776448,4776960) | 20260505 16:50:38.380 | INFO | MemInfo: { | 20260505 16:50:38.380 | INFO | "gpu" : " 17 GB", | 20260505 16:50:38.380 | INFO | "huge" : "", | 20260505 16:50:38.380 | INFO | "huge_shared" : " 27 GB", | 20260505 16:50:38.380 | INFO | "malloc" : " 14 GB", | 20260505 16:50:38.380 | INFO | "pinned" : " 29 GB" | 20260505 16:50:38.380 | INFO | } | 20260505 16:50:38.380 | INFO | Perf report for the last 5.14994 seconds: | 20260505 16:50:38.380 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:50:38.380 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:38.380 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:38.380 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:38.380 | INFO | ANALYSIS : 62.9%, 3.0, 98.4ms, 0 | 20260505 16:50:38.380 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:38.380 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:38.380 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:50:38.380 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:38.380 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:38.380 | INFO | Uploaded 36GiB to GPU in 2.13162s (16.8886GiB/s) | 20260505 16:50:38.380 | INFO | Downloaded 34GiB to GPU in 2.04916s (16.5922GiB/s) | 20260505 16:50:43.519 | INFO | Analyzing chunk frames = [4776960,4777472) | 20260505 16:50:43.520 | INFO | MemInfo: { | 20260505 16:50:43.520 | INFO | "gpu" : " 17 GB", | 20260505 16:50:43.520 | INFO | "huge" : "", | 20260505 16:50:43.520 | INFO | "huge_shared" : " 27 GB", | 20260505 16:50:43.520 | INFO | "malloc" : " 14 GB", | 20260505 16:50:43.520 | INFO | "pinned" : " 29 GB" | 20260505 16:50:43.520 | INFO | } | 20260505 16:50:43.520 | INFO | Perf report for the last 5.13968 seconds: | 20260505 16:50:43.520 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:50:43.520 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:43.520 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:43.520 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:43.520 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260505 16:50:43.520 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:43.520 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:43.520 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:50:43.520 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:43.520 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:43.520 | INFO | Uploaded 36GiB to GPU in 2.19886s (16.3721GiB/s) | 20260505 16:50:43.520 | INFO | Downloaded 34GiB to GPU in 2.04515s (16.6247GiB/s) | 20260505 16:50:48.654 | INFO | Analyzing chunk frames = [4777472,4777984) | 20260505 16:50:48.654 | INFO | MemInfo: { | 20260505 16:50:48.654 | INFO | "gpu" : " 17 GB", | 20260505 16:50:48.654 | INFO | "huge" : "", | 20260505 16:50:48.654 | INFO | "huge_shared" : " 27 GB", | 20260505 16:50:48.654 | INFO | "malloc" : " 14 GB", | 20260505 16:50:48.654 | INFO | "pinned" : " 29 GB" | 20260505 16:50:48.654 | INFO | } | 20260505 16:50:48.654 | INFO | Perf report for the last 5.13464 seconds: | 20260505 16:50:48.654 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:50:48.654 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:48.654 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:48.654 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:48.655 | INFO | ANALYSIS : 62.7%, 3.0, 97.2ms, 0 | 20260505 16:50:48.655 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:48.655 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:48.655 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 16:50:48.655 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:48.655 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:48.655 | INFO | Uploaded 36GiB to GPU in 2.16927s (16.5954GiB/s) | 20260505 16:50:48.655 | INFO | Downloaded 34GiB to GPU in 2.14465s (15.8534GiB/s) | 20260505 16:50:53.789 | INFO | Analyzing chunk frames = [4777984,4778496) | 20260505 16:50:53.789 | INFO | MemInfo: { | 20260505 16:50:53.789 | INFO | "gpu" : " 17 GB", | 20260505 16:50:53.789 | INFO | "huge" : "", | 20260505 16:50:53.789 | INFO | "huge_shared" : " 27 GB", | 20260505 16:50:53.789 | INFO | "malloc" : " 14 GB", | 20260505 16:50:53.789 | INFO | "pinned" : " 29 GB" | 20260505 16:50:53.789 | INFO | } | 20260505 16:50:53.789 | INFO | Perf report for the last 5.13436 seconds: | 20260505 16:50:53.789 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:50:53.789 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:53.789 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:53.789 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:53.789 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 16:50:53.789 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:53.789 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:53.789 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 16:50:53.789 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:53.789 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:53.789 | INFO | Uploaded 36GiB to GPU in 2.18597s (16.4687GiB/s) | 20260505 16:50:53.789 | INFO | Downloaded 34GiB to GPU in 2.13123s (15.9532GiB/s) | 20260505 16:50:58.937 | INFO | Analyzing chunk frames = [4778496,4779008) | 20260505 16:50:58.937 | INFO | MemInfo: { | 20260505 16:50:58.937 | INFO | "gpu" : " 17 GB", | 20260505 16:50:58.937 | INFO | "huge" : "", | 20260505 16:50:58.937 | INFO | "huge_shared" : " 27 GB", | 20260505 16:50:58.937 | INFO | "malloc" : " 14 GB", | 20260505 16:50:58.937 | INFO | "pinned" : " 29 GB" | 20260505 16:50:58.937 | INFO | } | 20260505 16:50:58.937 | INFO | Perf report for the last 5.14846 seconds: | 20260505 16:50:58.937 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:50:58.937 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:58.937 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:58.937 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:58.937 | INFO | ANALYSIS : 62.5%, 3.0, 97.7ms, 0 | 20260505 16:50:58.937 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:58.937 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:58.937 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 16:50:58.937 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:58.937 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:50:58.937 | INFO | Uploaded 36GiB to GPU in 2.16075s (16.6609GiB/s) | 20260505 16:50:58.938 | INFO | Downloaded 34GiB to GPU in 2.03784s (16.6844GiB/s) | 20260505 16:51:04.079 | INFO | Analyzing chunk frames = [4779008,4779520) | 20260505 16:51:04.079 | INFO | MemInfo: { | 20260505 16:51:04.079 | INFO | "gpu" : " 17 GB", | 20260505 16:51:04.079 | INFO | "huge" : "", | 20260505 16:51:04.079 | INFO | "huge_shared" : " 27 GB", | 20260505 16:51:04.079 | INFO | "malloc" : " 14 GB", | 20260505 16:51:04.079 | INFO | "pinned" : " 29 GB" | 20260505 16:51:04.079 | INFO | } | 20260505 16:51:04.080 | INFO | Perf report for the last 5.14224 seconds: | 20260505 16:51:04.080 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:51:04.080 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:04.080 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:04.080 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:04.080 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 16:51:04.080 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:04.080 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:04.080 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 16:51:04.080 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:04.080 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:04.080 | INFO | Uploaded 36GiB to GPU in 2.16813s (16.6041GiB/s) | 20260505 16:51:04.080 | INFO | Downloaded 34GiB to GPU in 2.17602s (15.6248GiB/s) | 20260505 16:51:09.215 | INFO | Analyzing chunk frames = [4779520,4780032) | 20260505 16:51:09.215 | INFO | MemInfo: { | 20260505 16:51:09.215 | INFO | "gpu" : " 17 GB", | 20260505 16:51:09.215 | INFO | "huge" : "", | 20260505 16:51:09.215 | INFO | "huge_shared" : " 27 GB", | 20260505 16:51:09.215 | INFO | "malloc" : " 14 GB", | 20260505 16:51:09.215 | INFO | "pinned" : " 29 GB" | 20260505 16:51:09.215 | INFO | } | 20260505 16:51:09.215 | INFO | Perf report for the last 5.13554 seconds: | 20260505 16:51:09.215 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:51:09.215 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:09.215 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:09.215 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:09.215 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 16:51:09.215 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:09.215 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:09.215 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:51:09.215 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:09.215 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:09.215 | INFO | Uploaded 36GiB to GPU in 2.19745s (16.3826GiB/s) | 20260505 16:51:09.215 | INFO | Downloaded 34GiB to GPU in 2.02263s (16.8098GiB/s) | 20260505 16:51:09.355 | INFO | Starting a round of Baz IO | 20260505 16:51:12.543 | INFO | Populating RTMetrics report | 20260505 16:51:12.544 | INFO | RTMetrics report populated | 20260505 16:51:12.544 | INFO | Opening tmp RTMetrics json file | 20260505 16:51:12.550 | INFO | Serializing RTMetrics to json | 20260505 16:51:12.551 | INFO | Serialization done, swapping tmp file now... | 20260505 16:51:12.552 | INFO | RTMetrics done | 20260505 16:51:12.640 | INFO | Finished a round of Baz IO: Wrote 2.158 GiB in 3.284 seconds | 20260505 16:51:12.640 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.17% / 0.00% / 77.83% | 20260505 16:51:12.640 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.338415% / 0% / 0.000111262% | 20260505 16:51:12.640 | INFO | Baz IO overall: | 20260505 16:51:12.640 | INFO | BytesWritten : 2832470671631 | 20260505 16:51:12.640 | INFO | NumEvents : 1343928271689 | 20260505 16:51:14.348 | INFO | Analyzing chunk frames = [4780032,4780544) | 20260505 16:51:14.348 | INFO | MemInfo: { | 20260505 16:51:14.348 | INFO | "gpu" : " 17 GB", | 20260505 16:51:14.348 | INFO | "huge" : "", | 20260505 16:51:14.348 | INFO | "huge_shared" : " 27 GB", | 20260505 16:51:14.348 | INFO | "malloc" : " 14 GB", | 20260505 16:51:14.348 | INFO | "pinned" : " 29 GB" | 20260505 16:51:14.348 | INFO | } | 20260505 16:51:14.348 | INFO | Perf report for the last 5.13329 seconds: | 20260505 16:51:14.348 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:51:14.348 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:14.348 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:14.348 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:14.348 | INFO | ANALYSIS : 63.7%, 3.0, 99.2ms, 0 | 20260505 16:51:14.348 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:14.348 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260505 16:51:14.348 | INFO | PRE_HQ : 60.4%, 1.8, 56.6ms, 0 | 20260505 16:51:14.348 | INFO | BAZWRITER : 63.4%, 3.8, 125.2ms, 0 | 20260505 16:51:14.348 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 16:51:14.349 | INFO | Uploaded 36GiB to GPU in 2.16351s (16.6396GiB/s) | 20260505 16:51:14.349 | INFO | Downloaded 38GiB to GPU in 2.26198s (16.7994GiB/s) | 20260505 16:51:19.487 | INFO | Analyzing chunk frames = [4780544,4781056) | 20260505 16:51:19.487 | INFO | MemInfo: { | 20260505 16:51:19.487 | INFO | "gpu" : " 17 GB", | 20260505 16:51:19.487 | INFO | "huge" : "", | 20260505 16:51:19.487 | INFO | "huge_shared" : " 27 GB", | 20260505 16:51:19.487 | INFO | "malloc" : " 14 GB", | 20260505 16:51:19.487 | INFO | "pinned" : " 29 GB" | 20260505 16:51:19.487 | INFO | } | 20260505 16:51:19.487 | INFO | Perf report for the last 5.13876 seconds: | 20260505 16:51:19.487 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:51:19.487 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:19.487 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:19.487 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:19.487 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 16:51:19.487 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:19.487 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:19.487 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 16:51:19.487 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:19.487 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:19.487 | INFO | Uploaded 36GiB to GPU in 2.19956s (16.3669GiB/s) | 20260505 16:51:19.488 | INFO | Downloaded 34GiB to GPU in 2.03724s (16.6893GiB/s) | 20260505 16:51:24.633 | INFO | Analyzing chunk frames = [4781056,4781568) | 20260505 16:51:24.633 | INFO | MemInfo: { | 20260505 16:51:24.633 | INFO | "gpu" : " 17 GB", | 20260505 16:51:24.633 | INFO | "huge" : "", | 20260505 16:51:24.634 | INFO | "huge_shared" : " 27 GB", | 20260505 16:51:24.634 | INFO | "malloc" : " 14 GB", | 20260505 16:51:24.634 | INFO | "pinned" : " 29 GB" | 20260505 16:51:24.634 | INFO | } | 20260505 16:51:24.634 | INFO | Perf report for the last 5.14656 seconds: | 20260505 16:51:24.634 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:51:24.634 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:24.634 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:24.634 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:24.634 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 16:51:24.634 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:24.634 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:24.634 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 16:51:24.634 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:24.634 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:24.634 | INFO | Uploaded 36GiB to GPU in 2.17612s (16.5432GiB/s) | 20260505 16:51:24.635 | INFO | Downloaded 34GiB to GPU in 2.03319s (16.7225GiB/s) | 20260505 16:51:29.779 | INFO | Analyzing chunk frames = [4781568,4782080) | 20260505 16:51:29.779 | INFO | MemInfo: { | 20260505 16:51:29.779 | INFO | "gpu" : " 17 GB", | 20260505 16:51:29.779 | INFO | "huge" : "", | 20260505 16:51:29.779 | INFO | "huge_shared" : " 27 GB", | 20260505 16:51:29.779 | INFO | "malloc" : " 14 GB", | 20260505 16:51:29.779 | INFO | "pinned" : " 29 GB" | 20260505 16:51:29.779 | INFO | } | 20260505 16:51:29.779 | INFO | Perf report for the last 5.14572 seconds: | 20260505 16:51:29.779 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:51:29.779 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:29.779 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:29.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:29.780 | INFO | ANALYSIS : 62.6%, 3.0, 97.3ms, 0 | 20260505 16:51:29.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:29.780 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:29.780 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 16:51:29.780 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:29.780 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:29.780 | INFO | Uploaded 36GiB to GPU in 2.20661s (16.3146GiB/s) | 20260505 16:51:29.780 | INFO | Downloaded 34GiB to GPU in 2.13669s (15.9124GiB/s) | 20260505 16:51:34.915 | INFO | Analyzing chunk frames = [4782080,4782592) | 20260505 16:51:34.915 | INFO | MemInfo: { | 20260505 16:51:34.915 | INFO | "gpu" : " 17 GB", | 20260505 16:51:34.915 | INFO | "huge" : "", | 20260505 16:51:34.915 | INFO | "huge_shared" : " 27 GB", | 20260505 16:51:34.915 | INFO | "malloc" : " 14 GB", | 20260505 16:51:34.915 | INFO | "pinned" : " 29 GB" | 20260505 16:51:34.915 | INFO | } | 20260505 16:51:34.915 | INFO | Perf report for the last 5.13571 seconds: | 20260505 16:51:34.915 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:51:34.915 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:34.915 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:34.915 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:34.915 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 16:51:34.915 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:34.915 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:34.915 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 16:51:34.915 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:34.915 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:34.916 | INFO | Uploaded 36GiB to GPU in 2.15247s (16.725GiB/s) | 20260505 16:51:34.916 | INFO | Downloaded 34GiB to GPU in 2.06127s (16.4947GiB/s) | 20260505 16:51:40.047 | INFO | Analyzing chunk frames = [4782592,4783104) | 20260505 16:51:40.047 | INFO | MemInfo: { | 20260505 16:51:40.047 | INFO | "gpu" : " 17 GB", | 20260505 16:51:40.047 | INFO | "huge" : "", | 20260505 16:51:40.047 | INFO | "huge_shared" : " 27 GB", | 20260505 16:51:40.047 | INFO | "malloc" : " 14 GB", | 20260505 16:51:40.047 | INFO | "pinned" : " 29 GB" | 20260505 16:51:40.047 | INFO | } | 20260505 16:51:40.047 | INFO | Perf report for the last 5.13192 seconds: | 20260505 16:51:40.047 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:51:40.047 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:40.047 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:40.047 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:40.047 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 16:51:40.047 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:40.047 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:40.047 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 16:51:40.047 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:40.047 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:40.047 | INFO | Uploaded 36GiB to GPU in 2.14212s (16.8058GiB/s) | 20260505 16:51:40.048 | INFO | Downloaded 34GiB to GPU in 2.04753s (16.6054GiB/s) | 20260505 16:51:45.189 | INFO | Analyzing chunk frames = [4783104,4783616) | 20260505 16:51:45.189 | INFO | MemInfo: { | 20260505 16:51:45.189 | INFO | "gpu" : " 17 GB", | 20260505 16:51:45.189 | INFO | "huge" : "", | 20260505 16:51:45.189 | INFO | "huge_shared" : " 27 GB", | 20260505 16:51:45.189 | INFO | "malloc" : " 14 GB", | 20260505 16:51:45.189 | INFO | "pinned" : " 29 GB" | 20260505 16:51:45.189 | INFO | } | 20260505 16:51:45.189 | INFO | Perf report for the last 5.14205 seconds: | 20260505 16:51:45.189 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:51:45.189 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:45.189 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:45.189 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:45.189 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 16:51:45.189 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:45.189 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:45.189 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 16:51:45.189 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:45.189 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:45.189 | INFO | Uploaded 36GiB to GPU in 2.13442s (16.8664GiB/s) | 20260505 16:51:45.190 | INFO | Downloaded 34GiB to GPU in 2.16828s (15.6807GiB/s) | 20260505 16:51:50.326 | INFO | Analyzing chunk frames = [4783616,4784128) | 20260505 16:51:50.326 | INFO | MemInfo: { | 20260505 16:51:50.326 | INFO | "gpu" : " 17 GB", | 20260505 16:51:50.327 | INFO | "huge" : "", | 20260505 16:51:50.327 | INFO | "huge_shared" : " 27 GB", | 20260505 16:51:50.327 | INFO | "malloc" : " 14 GB", | 20260505 16:51:50.327 | INFO | "pinned" : " 29 GB" | 20260505 16:51:50.327 | INFO | } | 20260505 16:51:50.327 | INFO | Perf report for the last 5.13758 seconds: | 20260505 16:51:50.327 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:51:50.327 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:50.327 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:50.327 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:50.327 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 16:51:50.327 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:50.327 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:50.327 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 16:51:50.327 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:50.327 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:50.327 | INFO | Uploaded 36GiB to GPU in 2.15747s (16.6862GiB/s) | 20260505 16:51:50.327 | INFO | Downloaded 34GiB to GPU in 2.01609s (16.8644GiB/s) | 20260505 16:51:50.327 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 16:51:50.327 | INFO | Total mem usage: 15.3 GiB | 20260505 16:51:50.327 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 16:51:50.327 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.9%) | 20260505 16:51:50.327 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 16:51:50.327 | INFO | High Water Breakdown: | 20260505 16:51:50.327 | INFO | 61% : 8.67 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 16:51:50.327 | INFO | 38.8% : 5.51 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 16:51:50.328 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 16:51:50.328 | INFO | ------------------------------------------ | 20260505 16:51:50.328 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:51:50.328 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 16:51:50.328 | INFO | Total mem usage: 26.5 GiB | 20260505 16:51:50.328 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 16:51:50.328 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 16:51:50.328 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 16:51:50.328 | INFO | High Water Breakdown: | 20260505 16:51:50.328 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:51:50.328 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:51:50.328 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:51:50.328 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:51:50.328 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:51:50.328 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:51:50.328 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 16:51:50.328 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 16:51:50.328 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:51:50.328 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 16:51:50.328 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 16:51:50.328 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 16:51:50.328 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 16:51:50.328 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 16:51:50.328 | INFO | ------------------------------------------ | 20260505 16:51:50.328 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:51:50.328 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 16:51:50.328 | INFO | Total mem usage: 25.6 GiB | 20260505 16:51:50.328 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 16:51:50.328 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 16:51:50.328 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 16:51:50.328 | INFO | High Water Breakdown: | 20260505 16:51:50.328 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 16:51:50.328 | INFO | ------------------------------------------ | 20260505 16:51:50.328 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:51:50.328 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 16:51:50.328 | INFO | Total mem usage: 15.7 GiB | 20260505 16:51:50.328 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 16:51:50.328 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 16:51:50.328 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 16:51:50.328 | INFO | High Water Breakdown: | 20260505 16:51:50.328 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 16:51:50.328 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 16:51:50.328 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 16:51:50.328 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 16:51:50.328 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 16:51:50.328 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 16:51:50.328 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 16:51:50.328 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 16:51:50.328 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 16:51:50.328 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:51:50.328 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 16:51:50.328 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:51:50.328 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 16:51:50.328 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 16:51:50.328 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 16:51:50.328 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 16:51:50.328 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:51:50.328 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 16:51:50.328 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:51:50.328 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:51:50.328 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:51:50.328 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 16:51:50.328 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 16:51:50.328 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:51:50.328 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 16:51:50.328 | INFO | ------------------------------------------ | 20260505 16:51:50.328 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:51:50.328 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 16:51:50.328 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:51:50.328 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:51:50.328 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.49e+01, 1.35e+01 / 2.04e+00 | 20260505 16:51:50.328 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 1.02e-01, 1.08e-02 / 3.73e-03 | 20260505 16:51:50.328 | INFO | PerfReport: 11.79%, Baseline -- 3.73e+00 / 4.52e+00, 5.75e-01 / 1.33e-01 | 20260505 16:51:50.328 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.52e+00, 1.92e-01 / 7.18e-02 | 20260505 16:51:50.328 | INFO | PerfReport: 15.99%, DME -- 4.05e+01 / 4.50e+01, 4.23e+00 / 2.07e+00 | 20260505 16:51:50.328 | INFO | PerfReport: 56.67%, FrameLabeling -- 1.79e+01 / 1.90e+01, 1.42e+00 / 9.78e-01 | 20260505 16:51:50.328 | INFO | PerfReport: 7.39%, PulseAccumulating -- 2.34e+00 / 2.95e+00, 4.55e-01 / 1.41e-01 | 20260505 16:51:50.328 | INFO | PerfReport: 3.40%, Metrics -- 1.07e+00 / 1.40e+00, 2.95e-01 / 1.22e-01 | 20260505 16:51:50.328 | INFO | PerfReport: 1.19%, PulseSerializing -- 3.75e-01 / 4.36e-01, 6.69e-02 / 3.53e-02 | 20260505 16:51:50.328 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 16:51:50.328 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:51:50.328 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:51:50.328 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.56e+01 / 0.00e+00 | 20260505 16:51:50.328 | INFO | PerfReport: 22.58%, Upload -- 2.26e+01 / 0.00e+00, 7.93e+00 / 0.00e+00 | 20260505 16:51:50.329 | INFO | PerfReport: 21.85%, Download -- 2.19e+01 / 0.00e+00, 7.90e+00 / 0.00e+00 | 20260505 16:51:50.329 | INFO | PerfReport: 31.58%, Compute -- 3.16e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 16:51:50.329 | INFO | PerfReport: 2.29%, LockUpload -- 2.30e+00 / 0.00e+00, 7.90e+00 / 0.00e+00 | 20260505 16:51:50.329 | INFO | PerfReport: 1.45%, LockDownload -- 1.45e+00 / 0.00e+00, 2.96e+00 / 0.00e+00 | 20260505 16:51:50.329 | INFO | PerfReport: 20.25%, LockCompute -- 2.03e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 16:51:50.329 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.11e-02 / 0.00e+00 | 20260505 16:51:50.468 | INFO | Starting a round of Baz IO | 20260505 16:51:53.653 | INFO | Populating RTMetrics report | 20260505 16:51:53.653 | INFO | RTMetrics report populated | 20260505 16:51:53.653 | INFO | Opening tmp RTMetrics json file | 20260505 16:51:53.658 | INFO | Serializing RTMetrics to json | 20260505 16:51:53.660 | INFO | Serialization done, swapping tmp file now... | 20260505 16:51:53.661 | INFO | RTMetrics done | 20260505 16:51:53.746 | INFO | Finished a round of Baz IO: Wrote 2.157 GiB in 3.277 seconds | 20260505 16:51:53.746 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.16% / 0.00% / 77.83% | 20260505 16:51:53.746 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.337672% / 0% / 0.000115236% | 20260505 16:51:53.746 | INFO | Baz IO overall: | 20260505 16:51:53.746 | INFO | BytesWritten : 2834787227792 | 20260505 16:51:53.746 | INFO | NumEvents : 1345013189474 | 20260505 16:51:55.471 | INFO | Analyzing chunk frames = [4784128,4784640) | 20260505 16:51:55.471 | INFO | MemInfo: { | 20260505 16:51:55.471 | INFO | "gpu" : " 17 GB", | 20260505 16:51:55.471 | INFO | "huge" : "", | 20260505 16:51:55.471 | INFO | "huge_shared" : " 27 GB", | 20260505 16:51:55.471 | INFO | "malloc" : " 14 GB", | 20260505 16:51:55.471 | INFO | "pinned" : " 29 GB" | 20260505 16:51:55.471 | INFO | } | 20260505 16:51:55.471 | INFO | Perf report for the last 5.14404 seconds: | 20260505 16:51:55.471 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:51:55.471 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:55.471 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:55.471 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:55.471 | INFO | ANALYSIS : 63.6%, 3.0, 99.1ms, 0 | 20260505 16:51:55.471 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:51:55.471 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260505 16:51:55.471 | INFO | PRE_HQ : 60.2%, 1.8, 56.4ms, 0 | 20260505 16:51:55.471 | INFO | BAZWRITER : 63.1%, 3.9, 125.5ms, 0 | 20260505 16:51:55.471 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 16:51:55.471 | INFO | Uploaded 36GiB to GPU in 2.16379s (16.6375GiB/s) | 20260505 16:51:55.471 | INFO | Downloaded 38GiB to GPU in 2.26386s (16.7855GiB/s) | 20260505 16:52:00.607 | INFO | Analyzing chunk frames = [4784640,4785152) | 20260505 16:52:00.607 | INFO | MemInfo: { | 20260505 16:52:00.607 | INFO | "gpu" : " 17 GB", | 20260505 16:52:00.607 | INFO | "huge" : "", | 20260505 16:52:00.607 | INFO | "huge_shared" : " 27 GB", | 20260505 16:52:00.607 | INFO | "malloc" : " 14 GB", | 20260505 16:52:00.607 | INFO | "pinned" : " 29 GB" | 20260505 16:52:00.607 | INFO | } | 20260505 16:52:00.608 | INFO | Perf report for the last 5.13666 seconds: | 20260505 16:52:00.608 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:52:00.608 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:00.608 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:00.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:00.608 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260505 16:52:00.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:00.608 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:00.608 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 16:52:00.608 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:00.608 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:00.608 | INFO | Uploaded 36GiB to GPU in 2.12517s (16.9398GiB/s) | 20260505 16:52:00.608 | INFO | Downloaded 34GiB to GPU in 2.02963s (16.7518GiB/s) | 20260505 16:52:05.741 | INFO | Analyzing chunk frames = [4785152,4785664) | 20260505 16:52:05.741 | INFO | MemInfo: { | 20260505 16:52:05.741 | INFO | "gpu" : " 17 GB", | 20260505 16:52:05.741 | INFO | "huge" : "", | 20260505 16:52:05.742 | INFO | "huge_shared" : " 27 GB", | 20260505 16:52:05.742 | INFO | "malloc" : " 14 GB", | 20260505 16:52:05.742 | INFO | "pinned" : " 29 GB" | 20260505 16:52:05.742 | INFO | } | 20260505 16:52:05.742 | INFO | Perf report for the last 5.13421 seconds: | 20260505 16:52:05.742 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:52:05.742 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:05.742 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:05.742 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:05.742 | INFO | ANALYSIS : 62.8%, 3.0, 97.6ms, 0 | 20260505 16:52:05.742 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:05.742 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:05.742 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:52:05.742 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:05.742 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:05.742 | INFO | Uploaded 36GiB to GPU in 2.18677s (16.4626GiB/s) | 20260505 16:52:05.742 | INFO | Downloaded 34GiB to GPU in 2.0259s (16.7827GiB/s) | 20260505 16:52:10.888 | INFO | Analyzing chunk frames = [4785664,4786176) | 20260505 16:52:10.888 | INFO | MemInfo: { | 20260505 16:52:10.888 | INFO | "gpu" : " 17 GB", | 20260505 16:52:10.888 | INFO | "huge" : "", | 20260505 16:52:10.888 | INFO | "huge_shared" : " 27 GB", | 20260505 16:52:10.888 | INFO | "malloc" : " 14 GB", | 20260505 16:52:10.888 | INFO | "pinned" : " 29 GB" | 20260505 16:52:10.888 | INFO | } | 20260505 16:52:10.888 | INFO | Perf report for the last 5.14654 seconds: | 20260505 16:52:10.888 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:52:10.888 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:10.888 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:10.888 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:10.888 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260505 16:52:10.888 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:10.888 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:10.888 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 16:52:10.888 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:10.888 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:10.888 | INFO | Uploaded 36GiB to GPU in 2.18636s (16.4657GiB/s) | 20260505 16:52:10.889 | INFO | Downloaded 34GiB to GPU in 2.13003s (15.9622GiB/s) | 20260505 16:52:16.029 | INFO | Analyzing chunk frames = [4786176,4786688) | 20260505 16:52:16.029 | INFO | MemInfo: { | 20260505 16:52:16.029 | INFO | "gpu" : " 17 GB", | 20260505 16:52:16.029 | INFO | "huge" : "", | 20260505 16:52:16.029 | INFO | "huge_shared" : " 27 GB", | 20260505 16:52:16.029 | INFO | "malloc" : " 14 GB", | 20260505 16:52:16.029 | INFO | "pinned" : " 29 GB" | 20260505 16:52:16.029 | INFO | } | 20260505 16:52:16.030 | INFO | Perf report for the last 5.14121 seconds: | 20260505 16:52:16.030 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:52:16.030 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:16.030 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:16.030 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:16.030 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 16:52:16.030 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:16.030 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:16.030 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:52:16.030 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:16.030 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:16.030 | INFO | Uploaded 36GiB to GPU in 2.15374s (16.7151GiB/s) | 20260505 16:52:16.030 | INFO | Downloaded 34GiB to GPU in 2.05674s (16.531GiB/s) | 20260505 16:52:21.162 | INFO | Analyzing chunk frames = [4786688,4787200) | 20260505 16:52:21.162 | INFO | MemInfo: { | 20260505 16:52:21.162 | INFO | "gpu" : " 17 GB", | 20260505 16:52:21.162 | INFO | "huge" : "", | 20260505 16:52:21.163 | INFO | "huge_shared" : " 27 GB", | 20260505 16:52:21.163 | INFO | "malloc" : " 14 GB", | 20260505 16:52:21.163 | INFO | "pinned" : " 29 GB" | 20260505 16:52:21.163 | INFO | } | 20260505 16:52:21.163 | INFO | Perf report for the last 5.13315 seconds: | 20260505 16:52:21.163 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:52:21.163 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:21.163 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:21.163 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:21.163 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260505 16:52:21.163 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:21.163 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:21.163 | INFO | PRE_HQ : 27.7%, 1.0, 14.3ms, 0 | 20260505 16:52:21.163 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:21.163 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:21.163 | INFO | Uploaded 36GiB to GPU in 2.13684s (16.8473GiB/s) | 20260505 16:52:21.163 | INFO | Downloaded 34GiB to GPU in 2.04895s (16.5939GiB/s) | 20260505 16:52:26.296 | INFO | Analyzing chunk frames = [4787200,4787712) | 20260505 16:52:26.296 | INFO | MemInfo: { | 20260505 16:52:26.296 | INFO | "gpu" : " 17 GB", | 20260505 16:52:26.296 | INFO | "huge" : "", | 20260505 16:52:26.297 | INFO | "huge_shared" : " 27 GB", | 20260505 16:52:26.297 | INFO | "malloc" : " 14 GB", | 20260505 16:52:26.297 | INFO | "pinned" : " 29 GB" | 20260505 16:52:26.297 | INFO | } | 20260505 16:52:26.297 | INFO | Perf report for the last 5.13407 seconds: | 20260505 16:52:26.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:52:26.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:26.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:26.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:26.297 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 16:52:26.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:26.297 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:26.297 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 16:52:26.297 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:26.297 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:26.297 | INFO | Uploaded 36GiB to GPU in 2.15246s (16.725GiB/s) | 20260505 16:52:26.297 | INFO | Downloaded 34GiB to GPU in 2.15309s (15.7912GiB/s) | 20260505 16:52:31.442 | INFO | Analyzing chunk frames = [4787712,4788224) | 20260505 16:52:31.442 | INFO | MemInfo: { | 20260505 16:52:31.442 | INFO | "gpu" : " 17 GB", | 20260505 16:52:31.442 | INFO | "huge" : "", | 20260505 16:52:31.442 | INFO | "huge_shared" : " 27 GB", | 20260505 16:52:31.442 | INFO | "malloc" : " 14 GB", | 20260505 16:52:31.442 | INFO | "pinned" : " 29 GB" | 20260505 16:52:31.442 | INFO | } | 20260505 16:52:31.442 | INFO | Perf report for the last 5.14561 seconds: | 20260505 16:52:31.442 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:52:31.442 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:31.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:31.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:31.442 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 16:52:31.442 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:31.443 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:31.443 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 16:52:31.443 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:31.443 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:31.443 | INFO | Uploaded 36GiB to GPU in 2.21354s (16.2635GiB/s) | 20260505 16:52:31.443 | INFO | Downloaded 34GiB to GPU in 2.09904s (16.1979GiB/s) | 20260505 16:52:31.584 | INFO | Starting a round of Baz IO | 20260505 16:52:34.777 | INFO | Populating RTMetrics report | 20260505 16:52:34.777 | INFO | RTMetrics report populated | 20260505 16:52:34.777 | INFO | Opening tmp RTMetrics json file | 20260505 16:52:34.789 | INFO | Serializing RTMetrics to json | 20260505 16:52:34.790 | INFO | Serialization done, swapping tmp file now... | 20260505 16:52:34.791 | INFO | RTMetrics done | 20260505 16:52:34.881 | INFO | Finished a round of Baz IO: Wrote 2.157 GiB in 3.296 seconds | 20260505 16:52:34.881 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.16% / 0.00% / 77.84% | 20260505 16:52:34.881 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.337791% / 0% / 7.54992e-05% | 20260505 16:52:34.881 | INFO | Baz IO overall: | 20260505 16:52:34.881 | INFO | BytesWritten : 2837103721715 | 20260505 16:52:34.881 | INFO | NumEvents : 1346098205106 | 20260505 16:52:36.586 | INFO | Analyzing chunk frames = [4788224,4788736) | 20260505 16:52:36.586 | INFO | MemInfo: { | 20260505 16:52:36.586 | INFO | "gpu" : " 17 GB", | 20260505 16:52:36.586 | INFO | "huge" : "", | 20260505 16:52:36.586 | INFO | "huge_shared" : " 27 GB", | 20260505 16:52:36.586 | INFO | "malloc" : " 14 GB", | 20260505 16:52:36.586 | INFO | "pinned" : " 29 GB" | 20260505 16:52:36.586 | INFO | } | 20260505 16:52:36.586 | INFO | Perf report for the last 5.14374 seconds: | 20260505 16:52:36.586 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:52:36.586 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:36.586 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:36.586 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:36.586 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 16:52:36.586 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:36.586 | INFO | RT_METRICS : 44.7%, 1.0, 23.5ms, 0 | 20260505 16:52:36.586 | INFO | PRE_HQ : 60.4%, 1.8, 56.3ms, 0 | 20260505 16:52:36.586 | INFO | BAZWRITER : 63.5%, 3.9, 126.7ms, 0 | 20260505 16:52:36.586 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 16:52:36.587 | INFO | Uploaded 36GiB to GPU in 2.18313s (16.4901GiB/s) | 20260505 16:52:36.587 | INFO | Downloaded 38GiB to GPU in 2.26942s (16.7444GiB/s) | 20260505 16:52:41.728 | INFO | Analyzing chunk frames = [4788736,4789248) | 20260505 16:52:41.728 | INFO | MemInfo: { | 20260505 16:52:41.728 | INFO | "gpu" : " 17 GB", | 20260505 16:52:41.728 | INFO | "huge" : "", | 20260505 16:52:41.728 | INFO | "huge_shared" : " 27 GB", | 20260505 16:52:41.728 | INFO | "malloc" : " 14 GB", | 20260505 16:52:41.728 | INFO | "pinned" : " 29 GB" | 20260505 16:52:41.728 | INFO | } | 20260505 16:52:41.728 | INFO | Perf report for the last 5.14228 seconds: | 20260505 16:52:41.728 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:52:41.728 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:41.728 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:41.728 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:41.728 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 16:52:41.728 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:41.728 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:41.728 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260505 16:52:41.729 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:41.729 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:41.729 | INFO | Uploaded 36GiB to GPU in 2.12679s (16.9269GiB/s) | 20260505 16:52:41.729 | INFO | Downloaded 34GiB to GPU in 2.0305s (16.7446GiB/s) | 20260505 16:52:46.859 | INFO | Analyzing chunk frames = [4789248,4789760) | 20260505 16:52:46.859 | INFO | MemInfo: { | 20260505 16:52:46.859 | INFO | "gpu" : " 17 GB", | 20260505 16:52:46.859 | INFO | "huge" : "", | 20260505 16:52:46.859 | INFO | "huge_shared" : " 27 GB", | 20260505 16:52:46.859 | INFO | "malloc" : " 14 GB", | 20260505 16:52:46.859 | INFO | "pinned" : " 29 GB" | 20260505 16:52:46.859 | INFO | } | 20260505 16:52:46.859 | INFO | Perf report for the last 5.13096 seconds: | 20260505 16:52:46.859 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:52:46.859 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:46.859 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:46.859 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:46.859 | INFO | ANALYSIS : 62.9%, 3.0, 97.7ms, 0 | 20260505 16:52:46.859 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:46.860 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:46.860 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:52:46.860 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:46.860 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:46.860 | INFO | Uploaded 36GiB to GPU in 2.18528s (16.4739GiB/s) | 20260505 16:52:46.860 | INFO | Downloaded 34GiB to GPU in 2.03328s (16.7218GiB/s) | 20260505 16:52:51.997 | INFO | Analyzing chunk frames = [4789760,4790272) | 20260505 16:52:51.997 | INFO | MemInfo: { | 20260505 16:52:51.997 | INFO | "gpu" : " 17 GB", | 20260505 16:52:51.997 | INFO | "huge" : "", | 20260505 16:52:51.997 | INFO | "huge_shared" : " 27 GB", | 20260505 16:52:51.997 | INFO | "malloc" : " 14 GB", | 20260505 16:52:51.997 | INFO | "pinned" : " 29 GB" | 20260505 16:52:51.997 | INFO | } | 20260505 16:52:51.997 | INFO | Perf report for the last 5.13793 seconds: | 20260505 16:52:51.997 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:52:51.997 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:51.997 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:51.997 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:51.997 | INFO | ANALYSIS : 62.5%, 3.0, 97.1ms, 0 | 20260505 16:52:51.997 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:51.997 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:51.997 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 16:52:51.998 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:51.998 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:51.998 | INFO | Uploaded 36GiB to GPU in 2.19945s (16.3677GiB/s) | 20260505 16:52:51.998 | INFO | Downloaded 34GiB to GPU in 2.14242s (15.8699GiB/s) | 20260505 16:52:57.143 | INFO | Analyzing chunk frames = [4790272,4790784) | 20260505 16:52:57.143 | INFO | MemInfo: { | 20260505 16:52:57.143 | INFO | "gpu" : " 17 GB", | 20260505 16:52:57.144 | INFO | "huge" : "", | 20260505 16:52:57.144 | INFO | "huge_shared" : " 27 GB", | 20260505 16:52:57.144 | INFO | "malloc" : " 14 GB", | 20260505 16:52:57.144 | INFO | "pinned" : " 29 GB" | 20260505 16:52:57.144 | INFO | } | 20260505 16:52:57.144 | INFO | Perf report for the last 5.14632 seconds: | 20260505 16:52:57.144 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:52:57.144 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:57.144 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:57.144 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:57.144 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 16:52:57.144 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:57.144 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:57.144 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 16:52:57.144 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:57.144 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:52:57.144 | INFO | Uploaded 36GiB to GPU in 2.15886s (16.6755GiB/s) | 20260505 16:52:57.144 | INFO | Downloaded 34GiB to GPU in 2.05596s (16.5373GiB/s) | 20260505 16:53:02.280 | INFO | Analyzing chunk frames = [4790784,4791296) | 20260505 16:53:02.280 | INFO | MemInfo: { | 20260505 16:53:02.280 | INFO | "gpu" : " 17 GB", | 20260505 16:53:02.280 | INFO | "huge" : "", | 20260505 16:53:02.280 | INFO | "huge_shared" : " 27 GB", | 20260505 16:53:02.280 | INFO | "malloc" : " 14 GB", | 20260505 16:53:02.280 | INFO | "pinned" : " 29 GB" | 20260505 16:53:02.280 | INFO | } | 20260505 16:53:02.280 | INFO | Perf report for the last 5.1363 seconds: | 20260505 16:53:02.280 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:53:02.280 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:02.280 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:02.280 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:02.280 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 16:53:02.280 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:02.280 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:02.280 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:53:02.280 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:02.280 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:02.280 | INFO | Uploaded 36GiB to GPU in 2.13491s (16.8626GiB/s) | 20260505 16:53:02.280 | INFO | Downloaded 34GiB to GPU in 2.04309s (16.6414GiB/s) | 20260505 16:53:07.419 | INFO | Analyzing chunk frames = [4791296,4791808) | 20260505 16:53:07.419 | INFO | MemInfo: { | 20260505 16:53:07.419 | INFO | "gpu" : " 17 GB", | 20260505 16:53:07.419 | INFO | "huge" : "", | 20260505 16:53:07.419 | INFO | "huge_shared" : " 27 GB", | 20260505 16:53:07.419 | INFO | "malloc" : " 14 GB", | 20260505 16:53:07.419 | INFO | "pinned" : " 29 GB" | 20260505 16:53:07.419 | INFO | } | 20260505 16:53:07.419 | INFO | Perf report for the last 5.13885 seconds: | 20260505 16:53:07.419 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:53:07.419 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:07.419 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:07.419 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:07.419 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 16:53:07.419 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:07.419 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:07.419 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 16:53:07.419 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:07.419 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:07.419 | INFO | Uploaded 36GiB to GPU in 2.14135s (16.8119GiB/s) | 20260505 16:53:07.419 | INFO | Downloaded 34GiB to GPU in 2.16783s (15.6839GiB/s) | 20260505 16:53:12.559 | INFO | Analyzing chunk frames = [4791808,4792320) | 20260505 16:53:12.559 | INFO | MemInfo: { | 20260505 16:53:12.559 | INFO | "gpu" : " 17 GB", | 20260505 16:53:12.560 | INFO | "huge" : "", | 20260505 16:53:12.560 | INFO | "huge_shared" : " 27 GB", | 20260505 16:53:12.560 | INFO | "malloc" : " 14 GB", | 20260505 16:53:12.560 | INFO | "pinned" : " 29 GB" | 20260505 16:53:12.560 | INFO | } | 20260505 16:53:12.560 | INFO | Perf report for the last 5.1409 seconds: | 20260505 16:53:12.560 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:53:12.560 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:12.560 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:12.560 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:12.560 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 16:53:12.560 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:12.560 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:12.560 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:53:12.560 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:12.560 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:12.560 | INFO | Uploaded 36GiB to GPU in 2.17767s (16.5314GiB/s) | 20260505 16:53:12.560 | INFO | Downloaded 34GiB to GPU in 2.03334s (16.7212GiB/s) | 20260505 16:53:12.560 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 16:53:12.560 | INFO | Total mem usage: 15.3 GiB | 20260505 16:53:12.560 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 16:53:12.560 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.93%) | 20260505 16:53:12.560 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 16:53:12.560 | INFO | High Water Breakdown: | 20260505 16:53:12.560 | INFO | 61% : 8.67 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 16:53:12.560 | INFO | 38.8% : 5.51 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 16:53:12.560 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 16:53:12.560 | INFO | ------------------------------------------ | 20260505 16:53:12.560 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:53:12.560 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 16:53:12.560 | INFO | Total mem usage: 26.5 GiB | 20260505 16:53:12.560 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 16:53:12.561 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 16:53:12.561 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 16:53:12.561 | INFO | High Water Breakdown: | 20260505 16:53:12.561 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:53:12.561 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:53:12.561 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:53:12.561 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:53:12.561 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:53:12.561 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:53:12.561 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 16:53:12.561 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 16:53:12.561 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:53:12.561 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 16:53:12.561 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 16:53:12.561 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 16:53:12.561 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 16:53:12.561 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 16:53:12.561 | INFO | ------------------------------------------ | 20260505 16:53:12.561 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:53:12.561 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 16:53:12.561 | INFO | Total mem usage: 25.6 GiB | 20260505 16:53:12.561 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 16:53:12.561 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 16:53:12.561 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 16:53:12.561 | INFO | High Water Breakdown: | 20260505 16:53:12.561 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 16:53:12.561 | INFO | ------------------------------------------ | 20260505 16:53:12.561 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:53:12.561 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 16:53:12.561 | INFO | Total mem usage: 15.7 GiB | 20260505 16:53:12.561 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 16:53:12.561 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 16:53:12.561 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 16:53:12.561 | INFO | High Water Breakdown: | 20260505 16:53:12.561 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 16:53:12.561 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 16:53:12.561 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 16:53:12.561 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 16:53:12.561 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 16:53:12.561 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 16:53:12.561 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 16:53:12.561 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 16:53:12.561 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 16:53:12.561 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:53:12.561 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 16:53:12.561 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:53:12.561 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 16:53:12.561 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 16:53:12.561 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 16:53:12.561 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 16:53:12.561 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:53:12.561 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 16:53:12.561 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:53:12.561 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:53:12.561 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:53:12.561 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 16:53:12.561 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 16:53:12.561 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:53:12.561 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 16:53:12.561 | INFO | ------------------------------------------ | 20260505 16:53:12.561 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:53:12.561 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 16:53:12.561 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:53:12.561 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:53:12.561 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.50e+01, 1.35e+01 / 1.78e+00 | 20260505 16:53:12.561 | INFO | PerfReport: 0.29%, AggregateStats -- 9.27e-02 / 1.02e-01, 1.06e-02 / 5.56e-03 | 20260505 16:53:12.561 | INFO | PerfReport: 11.80%, Baseline -- 3.74e+00 / 4.50e+00, 5.77e-01 / 1.83e-01 | 20260505 16:53:12.561 | INFO | PerfReport: 3.29%, Histogram -- 1.04e+00 / 1.51e+00, 1.95e-01 / 1.07e-01 | 20260505 16:53:12.561 | INFO | PerfReport: 15.89%, DME -- 4.03e+01 / 4.46e+01, 4.15e+00 / 1.80e+00 | 20260505 16:53:12.561 | INFO | PerfReport: 56.79%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.53e+00 / 1.49e+00 | 20260505 16:53:12.561 | INFO | PerfReport: 7.34%, PulseAccumulating -- 2.33e+00 / 2.89e+00, 4.44e-01 / 1.97e-01 | 20260505 16:53:12.561 | INFO | PerfReport: 3.40%, Metrics -- 1.08e+00 / 1.41e+00, 2.97e-01 / 1.39e-01 | 20260505 16:53:12.561 | INFO | PerfReport: 1.18%, PulseSerializing -- 3.75e-01 / 4.39e-01, 6.68e-02 / 3.93e-02 | 20260505 16:53:12.561 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 16:53:12.561 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:53:12.561 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:53:12.561 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260505 16:53:12.561 | INFO | PerfReport: 22.52%, Upload -- 2.25e+01 / 0.00e+00, 7.81e+00 / 0.00e+00 | 20260505 16:53:12.561 | INFO | PerfReport: 21.85%, Download -- 2.19e+01 / 0.00e+00, 7.87e+00 / 0.00e+00 | 20260505 16:53:12.561 | INFO | PerfReport: 31.67%, Compute -- 3.17e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 16:53:12.561 | INFO | PerfReport: 2.17%, LockUpload -- 2.17e+00 / 0.00e+00, 7.83e+00 / 0.00e+00 | 20260505 16:53:12.561 | INFO | PerfReport: 1.46%, LockDownload -- 1.46e+00 / 0.00e+00, 3.03e+00 / 0.00e+00 | 20260505 16:53:12.561 | INFO | PerfReport: 20.32%, LockCompute -- 2.03e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 16:53:12.561 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.53e-02 / 0.00e+00 | 20260505 16:53:12.691 | INFO | Starting a round of Baz IO | 20260505 16:53:15.905 | INFO | Populating RTMetrics report | 20260505 16:53:15.905 | INFO | RTMetrics report populated | 20260505 16:53:15.905 | INFO | Opening tmp RTMetrics json file | 20260505 16:53:15.913 | INFO | Serializing RTMetrics to json | 20260505 16:53:15.915 | INFO | Serialization done, swapping tmp file now... | 20260505 16:53:15.918 | INFO | RTMetrics done | 20260505 16:53:16.002 | INFO | Finished a round of Baz IO: Wrote 2.157 GiB in 3.31 seconds | 20260505 16:53:16.002 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.16% / 0.00% / 77.84% | 20260505 16:53:16.002 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.336576% / 0% / 0.000115236% | 20260505 16:53:16.002 | INFO | Baz IO overall: | 20260505 16:53:16.002 | INFO | BytesWritten : 2839419496342 | 20260505 16:53:16.002 | INFO | NumEvents : 1347182963656 | 20260505 16:53:17.698 | INFO | Analyzing chunk frames = [4792320,4792832) | 20260505 16:53:17.698 | INFO | MemInfo: { | 20260505 16:53:17.698 | INFO | "gpu" : " 17 GB", | 20260505 16:53:17.698 | INFO | "huge" : "", | 20260505 16:53:17.698 | INFO | "huge_shared" : " 27 GB", | 20260505 16:53:17.698 | INFO | "malloc" : " 14 GB", | 20260505 16:53:17.698 | INFO | "pinned" : " 29 GB" | 20260505 16:53:17.698 | INFO | } | 20260505 16:53:17.698 | INFO | Perf report for the last 5.13871 seconds: | 20260505 16:53:17.699 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:53:17.699 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:17.699 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:17.699 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:17.699 | INFO | ANALYSIS : 64.0%, 3.0, 99.8ms, 0 | 20260505 16:53:17.699 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:17.699 | INFO | RT_METRICS : 44.9%, 1.0, 23.5ms, 0 | 20260505 16:53:17.699 | INFO | PRE_HQ : 60.7%, 1.8, 56.4ms, 0 | 20260505 16:53:17.699 | INFO | BAZWRITER : 63.7%, 3.8, 125.0ms, 0 | 20260505 16:53:17.699 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 16:53:17.699 | INFO | Uploaded 36GiB to GPU in 2.20467s (16.329GiB/s) | 20260505 16:53:17.699 | INFO | Downloaded 38GiB to GPU in 2.26425s (16.7826GiB/s) | 20260505 16:53:22.840 | INFO | Analyzing chunk frames = [4792832,4793344) | 20260505 16:53:22.840 | INFO | MemInfo: { | 20260505 16:53:22.840 | INFO | "gpu" : " 17 GB", | 20260505 16:53:22.840 | INFO | "huge" : "", | 20260505 16:53:22.840 | INFO | "huge_shared" : " 27 GB", | 20260505 16:53:22.840 | INFO | "malloc" : " 14 GB", | 20260505 16:53:22.840 | INFO | "pinned" : " 29 GB" | 20260505 16:53:22.840 | INFO | } | 20260505 16:53:22.840 | INFO | Perf report for the last 5.14186 seconds: | 20260505 16:53:22.840 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:53:22.840 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:22.840 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:22.840 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:22.840 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 16:53:22.840 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:22.840 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:22.840 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 16:53:22.840 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:22.840 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:22.841 | INFO | Uploaded 36GiB to GPU in 2.13s (16.9014GiB/s) | 20260505 16:53:22.841 | INFO | Downloaded 34GiB to GPU in 2.0336s (16.7191GiB/s) | 20260505 16:53:27.975 | INFO | Analyzing chunk frames = [4793344,4793856) | 20260505 16:53:27.975 | INFO | MemInfo: { | 20260505 16:53:27.975 | INFO | "gpu" : " 17 GB", | 20260505 16:53:27.975 | INFO | "huge" : "", | 20260505 16:53:27.975 | INFO | "huge_shared" : " 27 GB", | 20260505 16:53:27.975 | INFO | "malloc" : " 14 GB", | 20260505 16:53:27.975 | INFO | "pinned" : " 29 GB" | 20260505 16:53:27.976 | INFO | } | 20260505 16:53:27.976 | INFO | Perf report for the last 5.13525 seconds: | 20260505 16:53:27.976 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:53:27.976 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:27.976 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:27.976 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:27.976 | INFO | ANALYSIS : 62.7%, 3.0, 97.5ms, 0 | 20260505 16:53:27.976 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:27.976 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:27.976 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:53:27.976 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:27.976 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:27.976 | INFO | Uploaded 36GiB to GPU in 2.19566s (16.396GiB/s) | 20260505 16:53:27.976 | INFO | Downloaded 34GiB to GPU in 2.03003s (16.7485GiB/s) | 20260505 16:53:33.110 | INFO | Analyzing chunk frames = [4793856,4794368) | 20260505 16:53:33.110 | INFO | MemInfo: { | 20260505 16:53:33.110 | INFO | "gpu" : " 17 GB", | 20260505 16:53:33.110 | INFO | "huge" : "", | 20260505 16:53:33.111 | INFO | "huge_shared" : " 27 GB", | 20260505 16:53:33.111 | INFO | "malloc" : " 14 GB", | 20260505 16:53:33.111 | INFO | "pinned" : " 29 GB" | 20260505 16:53:33.111 | INFO | } | 20260505 16:53:33.111 | INFO | Perf report for the last 5.13504 seconds: | 20260505 16:53:33.111 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:53:33.111 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:33.111 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:33.111 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:33.111 | INFO | ANALYSIS : 62.6%, 3.0, 97.2ms, 0 | 20260505 16:53:33.111 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:33.111 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:33.111 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 16:53:33.111 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:33.111 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:33.111 | INFO | Uploaded 36GiB to GPU in 2.17438s (16.5564GiB/s) | 20260505 16:53:33.111 | INFO | Downloaded 34GiB to GPU in 2.14309s (15.8649GiB/s) | 20260505 16:53:38.249 | INFO | Analyzing chunk frames = [4794368,4794880) | 20260505 16:53:38.249 | INFO | MemInfo: { | 20260505 16:53:38.249 | INFO | "gpu" : " 17 GB", | 20260505 16:53:38.249 | INFO | "huge" : "", | 20260505 16:53:38.249 | INFO | "huge_shared" : " 27 GB", | 20260505 16:53:38.249 | INFO | "malloc" : " 14 GB", | 20260505 16:53:38.249 | INFO | "pinned" : " 29 GB" | 20260505 16:53:38.250 | INFO | } | 20260505 16:53:38.250 | INFO | Perf report for the last 5.13903 seconds: | 20260505 16:53:38.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:53:38.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:38.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:38.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:38.250 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 16:53:38.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:38.250 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:38.250 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:53:38.250 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:38.250 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:38.250 | INFO | Uploaded 36GiB to GPU in 2.13232s (16.883GiB/s) | 20260505 16:53:38.250 | INFO | Downloaded 34GiB to GPU in 2.05777s (16.5228GiB/s) | 20260505 16:53:43.396 | INFO | Analyzing chunk frames = [4794880,4795392) | 20260505 16:53:43.396 | INFO | MemInfo: { | 20260505 16:53:43.396 | INFO | "gpu" : " 17 GB", | 20260505 16:53:43.396 | INFO | "huge" : "", | 20260505 16:53:43.396 | INFO | "huge_shared" : " 27 GB", | 20260505 16:53:43.396 | INFO | "malloc" : " 14 GB", | 20260505 16:53:43.396 | INFO | "pinned" : " 29 GB" | 20260505 16:53:43.396 | INFO | } | 20260505 16:53:43.397 | INFO | Perf report for the last 5.14683 seconds: | 20260505 16:53:43.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:53:43.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:43.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:43.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:43.397 | INFO | ANALYSIS : 62.3%, 3.0, 97.4ms, 0 | 20260505 16:53:43.397 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:43.397 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:43.397 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 16:53:43.397 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:43.397 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:43.397 | INFO | Uploaded 36GiB to GPU in 2.13944s (16.8268GiB/s) | 20260505 16:53:43.397 | INFO | Downloaded 34GiB to GPU in 2.03594s (16.6999GiB/s) | 20260505 16:53:48.540 | INFO | Analyzing chunk frames = [4795392,4795904) | 20260505 16:53:48.540 | INFO | MemInfo: { | 20260505 16:53:48.540 | INFO | "gpu" : " 17 GB", | 20260505 16:53:48.540 | INFO | "huge" : "", | 20260505 16:53:48.540 | INFO | "huge_shared" : " 27 GB", | 20260505 16:53:48.540 | INFO | "malloc" : " 14 GB", | 20260505 16:53:48.540 | INFO | "pinned" : " 29 GB" | 20260505 16:53:48.540 | INFO | } | 20260505 16:53:48.540 | INFO | Perf report for the last 5.14392 seconds: | 20260505 16:53:48.540 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:53:48.540 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:48.540 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:48.541 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:48.541 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 16:53:48.541 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:48.541 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:48.541 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 16:53:48.541 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:48.541 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:48.541 | INFO | Uploaded 36GiB to GPU in 2.15679s (16.6915GiB/s) | 20260505 16:53:48.541 | INFO | Downloaded 34GiB to GPU in 2.14909s (15.8206GiB/s) | 20260505 16:53:53.675 | INFO | Analyzing chunk frames = [4795904,4796416) | 20260505 16:53:53.675 | INFO | MemInfo: { | 20260505 16:53:53.675 | INFO | "gpu" : " 17 GB", | 20260505 16:53:53.675 | INFO | "huge" : "", | 20260505 16:53:53.675 | INFO | "huge_shared" : " 27 GB", | 20260505 16:53:53.675 | INFO | "malloc" : " 14 GB", | 20260505 16:53:53.675 | INFO | "pinned" : " 29 GB" | 20260505 16:53:53.675 | INFO | } | 20260505 16:53:53.675 | INFO | Perf report for the last 5.13458 seconds: | 20260505 16:53:53.675 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:53:53.675 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:53.675 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:53.675 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:53.675 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 16:53:53.675 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:53.675 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:53.675 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:53:53.675 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:53.675 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:53.675 | INFO | Uploaded 36GiB to GPU in 2.17646s (16.5406GiB/s) | 20260505 16:53:53.675 | INFO | Downloaded 34GiB to GPU in 2.0249s (16.791GiB/s) | 20260505 16:53:53.810 | INFO | Starting a round of Baz IO | 20260505 16:53:57.016 | INFO | Populating RTMetrics report | 20260505 16:53:57.016 | INFO | RTMetrics report populated | 20260505 16:53:57.016 | INFO | Opening tmp RTMetrics json file | 20260505 16:53:57.025 | INFO | Serializing RTMetrics to json | 20260505 16:53:57.027 | INFO | Serialization done, swapping tmp file now... | 20260505 16:53:57.028 | INFO | RTMetrics done | 20260505 16:53:57.110 | INFO | Finished a round of Baz IO: Wrote 2.156 GiB in 3.299 seconds | 20260505 16:53:57.110 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.16% / 0.00% / 77.84% | 20260505 16:53:57.110 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.335745% / 3.97364e-06% / 0.000111262% | 20260505 16:53:57.110 | INFO | Baz IO overall: | 20260505 16:53:57.110 | INFO | BytesWritten : 2841734378380 | 20260505 16:53:57.110 | INFO | NumEvents : 1348267299662 | 20260505 16:53:58.808 | INFO | Analyzing chunk frames = [4796416,4796928) | 20260505 16:53:58.808 | INFO | MemInfo: { | 20260505 16:53:58.808 | INFO | "gpu" : " 17 GB", | 20260505 16:53:58.808 | INFO | "huge" : "", | 20260505 16:53:58.808 | INFO | "huge_shared" : " 27 GB", | 20260505 16:53:58.808 | INFO | "malloc" : " 14 GB", | 20260505 16:53:58.808 | INFO | "pinned" : " 29 GB" | 20260505 16:53:58.808 | INFO | } | 20260505 16:53:58.809 | INFO | Perf report for the last 5.13348 seconds: | 20260505 16:53:58.809 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:53:58.809 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:58.809 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:58.809 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:58.809 | INFO | ANALYSIS : 64.0%, 3.0, 99.6ms, 0 | 20260505 16:53:58.809 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:53:58.809 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260505 16:53:58.809 | INFO | PRE_HQ : 60.9%, 1.8, 56.6ms, 0 | 20260505 16:53:58.809 | INFO | BAZWRITER : 63.6%, 3.8, 124.4ms, 0 | 20260505 16:53:58.809 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 16:53:58.809 | INFO | Uploaded 36GiB to GPU in 2.20013s (16.3627GiB/s) | 20260505 16:53:58.809 | INFO | Downloaded 38GiB to GPU in 2.26202s (16.7991GiB/s) | 20260505 16:54:03.953 | INFO | Analyzing chunk frames = [4796928,4797440) | 20260505 16:54:03.953 | INFO | MemInfo: { | 20260505 16:54:03.953 | INFO | "gpu" : " 17 GB", | 20260505 16:54:03.953 | INFO | "huge" : "", | 20260505 16:54:03.953 | INFO | "huge_shared" : " 27 GB", | 20260505 16:54:03.953 | INFO | "malloc" : " 14 GB", | 20260505 16:54:03.953 | INFO | "pinned" : " 29 GB" | 20260505 16:54:03.953 | INFO | } | 20260505 16:54:03.953 | INFO | Perf report for the last 5.1444 seconds: | 20260505 16:54:03.953 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:54:03.953 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:03.953 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:03.953 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:03.953 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 16:54:03.953 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:03.953 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:03.953 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 16:54:03.953 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:03.953 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:03.953 | INFO | Uploaded 36GiB to GPU in 2.14147s (16.8109GiB/s) | 20260505 16:54:03.953 | INFO | Downloaded 34GiB to GPU in 2.04026s (16.6646GiB/s) | 20260505 16:54:09.095 | INFO | Analyzing chunk frames = [4797440,4797952) | 20260505 16:54:09.095 | INFO | MemInfo: { | 20260505 16:54:09.095 | INFO | "gpu" : " 17 GB", | 20260505 16:54:09.095 | INFO | "huge" : "", | 20260505 16:54:09.095 | INFO | "huge_shared" : " 27 GB", | 20260505 16:54:09.095 | INFO | "malloc" : " 14 GB", | 20260505 16:54:09.095 | INFO | "pinned" : " 29 GB" | 20260505 16:54:09.095 | INFO | } | 20260505 16:54:09.095 | INFO | Perf report for the last 5.14241 seconds: | 20260505 16:54:09.095 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:54:09.095 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:09.095 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:09.095 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:09.095 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260505 16:54:09.095 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:09.095 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:09.095 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:54:09.095 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:09.095 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:09.096 | INFO | Uploaded 36GiB to GPU in 2.19791s (16.3792GiB/s) | 20260505 16:54:09.096 | INFO | Downloaded 34GiB to GPU in 2.04559s (16.6211GiB/s) | 20260505 16:54:14.230 | INFO | Analyzing chunk frames = [4797952,4798464) | 20260505 16:54:14.230 | INFO | MemInfo: { | 20260505 16:54:14.230 | INFO | "gpu" : " 17 GB", | 20260505 16:54:14.230 | INFO | "huge" : "", | 20260505 16:54:14.230 | INFO | "huge_shared" : " 27 GB", | 20260505 16:54:14.230 | INFO | "malloc" : " 14 GB", | 20260505 16:54:14.230 | INFO | "pinned" : " 29 GB" | 20260505 16:54:14.230 | INFO | } | 20260505 16:54:14.230 | INFO | Perf report for the last 5.13504 seconds: | 20260505 16:54:14.230 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:54:14.230 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:14.230 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:14.230 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:14.230 | INFO | ANALYSIS : 62.9%, 3.0, 97.6ms, 0 | 20260505 16:54:14.230 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:14.230 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:14.230 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:54:14.230 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:14.230 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:14.231 | INFO | Uploaded 36GiB to GPU in 2.22242s (16.1986GiB/s) | 20260505 16:54:14.231 | INFO | Downloaded 34GiB to GPU in 2.12539s (15.9971GiB/s) | 20260505 16:54:19.365 | INFO | Analyzing chunk frames = [4798464,4798976) | 20260505 16:54:19.365 | INFO | MemInfo: { | 20260505 16:54:19.365 | INFO | "gpu" : " 17 GB", | 20260505 16:54:19.365 | INFO | "huge" : "", | 20260505 16:54:19.365 | INFO | "huge_shared" : " 27 GB", | 20260505 16:54:19.365 | INFO | "malloc" : " 14 GB", | 20260505 16:54:19.365 | INFO | "pinned" : " 29 GB" | 20260505 16:54:19.365 | INFO | } | 20260505 16:54:19.365 | INFO | Perf report for the last 5.13473 seconds: | 20260505 16:54:19.365 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:54:19.365 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:19.365 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:19.365 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:19.365 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 16:54:19.365 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:19.365 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:19.365 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 16:54:19.365 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:19.365 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:19.365 | INFO | Uploaded 36GiB to GPU in 2.14935s (16.7492GiB/s) | 20260505 16:54:19.366 | INFO | Downloaded 34GiB to GPU in 2.06163s (16.4918GiB/s) | 20260505 16:54:24.506 | INFO | Analyzing chunk frames = [4798976,4799488) | 20260505 16:54:24.506 | INFO | MemInfo: { | 20260505 16:54:24.506 | INFO | "gpu" : " 17 GB", | 20260505 16:54:24.506 | INFO | "huge" : "", | 20260505 16:54:24.506 | INFO | "huge_shared" : " 27 GB", | 20260505 16:54:24.506 | INFO | "malloc" : " 14 GB", | 20260505 16:54:24.506 | INFO | "pinned" : " 29 GB" | 20260505 16:54:24.506 | INFO | } | 20260505 16:54:24.506 | INFO | Perf report for the last 5.14068 seconds: | 20260505 16:54:24.506 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:54:24.506 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:24.506 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:24.506 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:24.506 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 16:54:24.506 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:24.506 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:24.506 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 16:54:24.506 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:24.506 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:24.506 | INFO | Uploaded 36GiB to GPU in 2.21144s (16.279GiB/s) | 20260505 16:54:24.506 | INFO | Downloaded 34GiB to GPU in 2.11603s (16.0678GiB/s) | 20260505 16:54:29.648 | INFO | Analyzing chunk frames = [4799488,4800000) | 20260505 16:54:29.648 | INFO | MemInfo: { | 20260505 16:54:29.648 | INFO | "gpu" : " 17 GB", | 20260505 16:54:29.648 | INFO | "huge" : "", | 20260505 16:54:29.648 | INFO | "huge_shared" : " 27 GB", | 20260505 16:54:29.648 | INFO | "malloc" : " 14 GB", | 20260505 16:54:29.648 | INFO | "pinned" : " 29 GB" | 20260505 16:54:29.649 | INFO | } | 20260505 16:54:29.649 | INFO | Perf report for the last 5.14273 seconds: | 20260505 16:54:29.649 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:54:29.649 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:29.649 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:29.649 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:29.649 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 16:54:29.649 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:29.649 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:29.649 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 16:54:29.649 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:29.649 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:29.649 | INFO | Uploaded 36GiB to GPU in 2.12112s (16.9722GiB/s) | 20260505 16:54:29.649 | INFO | Downloaded 34GiB to GPU in 2.15698s (15.7628GiB/s) | 20260505 16:54:34.796 | INFO | Analyzing chunk frames = [4800000,4800512) | 20260505 16:54:34.796 | INFO | MemInfo: { | 20260505 16:54:34.796 | INFO | "gpu" : " 17 GB", | 20260505 16:54:34.796 | INFO | "huge" : "", | 20260505 16:54:34.796 | INFO | "huge_shared" : " 27 GB", | 20260505 16:54:34.796 | INFO | "malloc" : " 14 GB", | 20260505 16:54:34.796 | INFO | "pinned" : " 29 GB" | 20260505 16:54:34.797 | INFO | } | 20260505 16:54:34.797 | INFO | Perf report for the last 5.14804 seconds: | 20260505 16:54:34.797 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:54:34.797 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:34.797 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:34.797 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:34.797 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 16:54:34.797 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:34.797 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:34.797 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 16:54:34.797 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:34.797 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:34.797 | INFO | Uploaded 36GiB to GPU in 2.18842s (16.4502GiB/s) | 20260505 16:54:34.797 | INFO | Downloaded 34GiB to GPU in 2.01957s (16.8353GiB/s) | 20260505 16:54:34.797 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 16:54:34.797 | INFO | Total mem usage: 15.3 GiB | 20260505 16:54:34.797 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 16:54:34.797 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.9%) | 20260505 16:54:34.797 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 16:54:34.797 | INFO | High Water Breakdown: | 20260505 16:54:34.797 | INFO | 61% : 8.67 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 16:54:34.797 | INFO | 38.8% : 5.51 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 16:54:34.797 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 16:54:34.797 | INFO | ------------------------------------------ | 20260505 16:54:34.797 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:54:34.797 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 16:54:34.797 | INFO | Total mem usage: 26.5 GiB | 20260505 16:54:34.797 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 16:54:34.797 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 16:54:34.797 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 16:54:34.797 | INFO | High Water Breakdown: | 20260505 16:54:34.798 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:54:34.798 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:54:34.798 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:54:34.798 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:54:34.798 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:54:34.798 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:54:34.798 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 16:54:34.798 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 16:54:34.798 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:54:34.798 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 16:54:34.798 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 16:54:34.798 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 16:54:34.798 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 16:54:34.798 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 16:54:34.798 | INFO | ------------------------------------------ | 20260505 16:54:34.798 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:54:34.798 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 16:54:34.798 | INFO | Total mem usage: 25.6 GiB | 20260505 16:54:34.798 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 16:54:34.798 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 16:54:34.798 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 16:54:34.798 | INFO | High Water Breakdown: | 20260505 16:54:34.798 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 16:54:34.798 | INFO | ------------------------------------------ | 20260505 16:54:34.798 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:54:34.798 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 16:54:34.798 | INFO | Total mem usage: 15.7 GiB | 20260505 16:54:34.798 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 16:54:34.798 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 16:54:34.798 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 16:54:34.798 | INFO | High Water Breakdown: | 20260505 16:54:34.798 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 16:54:34.798 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 16:54:34.798 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 16:54:34.798 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 16:54:34.798 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 16:54:34.798 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 16:54:34.798 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 16:54:34.798 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 16:54:34.798 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 16:54:34.798 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:54:34.798 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 16:54:34.798 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:54:34.798 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 16:54:34.798 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 16:54:34.798 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 16:54:34.798 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 16:54:34.798 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:54:34.798 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 16:54:34.798 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:54:34.798 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:54:34.798 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:54:34.798 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 16:54:34.798 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 16:54:34.798 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:54:34.798 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 16:54:34.798 | INFO | ------------------------------------------ | 20260505 16:54:34.798 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:54:34.798 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 16:54:34.798 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:54:34.798 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:54:34.798 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.49e+01, 1.36e+01 / 1.77e+00 | 20260505 16:54:34.798 | INFO | PerfReport: 0.29%, AggregateStats -- 9.26e-02 / 1.01e-01, 1.06e-02 / 4.83e-03 | 20260505 16:54:34.798 | INFO | PerfReport: 11.75%, Baseline -- 3.72e+00 / 4.49e+00, 5.64e-01 / 1.93e-01 | 20260505 16:54:34.798 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.51e+00, 1.91e-01 / 1.07e-01 | 20260505 16:54:34.798 | INFO | PerfReport: 15.95%, DME -- 4.04e+01 / 4.48e+01, 4.20e+00 / 1.71e+00 | 20260505 16:54:34.798 | INFO | PerfReport: 56.78%, FrameLabeling -- 1.80e+01 / 1.92e+01, 1.46e+00 / 1.09e+00 | 20260505 16:54:34.798 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.33e+00 / 2.95e+00, 4.56e-01 / 1.41e-01 | 20260505 16:54:34.798 | INFO | PerfReport: 3.39%, Metrics -- 1.07e+00 / 1.40e+00, 2.96e-01 / 1.86e-01 | 20260505 16:54:34.798 | INFO | PerfReport: 1.19%, PulseSerializing -- 3.75e-01 / 4.34e-01, 6.70e-02 / 3.49e-02 | 20260505 16:54:34.798 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 16:54:34.798 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:54:34.798 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:54:34.798 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.58e+01 / 0.00e+00 | 20260505 16:54:34.798 | INFO | PerfReport: 22.58%, Upload -- 2.26e+01 / 0.00e+00, 7.98e+00 / 0.00e+00 | 20260505 16:54:34.798 | INFO | PerfReport: 21.82%, Download -- 2.19e+01 / 0.00e+00, 7.88e+00 / 0.00e+00 | 20260505 16:54:34.798 | INFO | PerfReport: 31.58%, Compute -- 3.16e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 16:54:34.798 | INFO | PerfReport: 2.36%, LockUpload -- 2.37e+00 / 0.00e+00, 7.98e+00 / 0.00e+00 | 20260505 16:54:34.798 | INFO | PerfReport: 1.35%, LockDownload -- 1.35e+00 / 0.00e+00, 2.82e+00 / 0.00e+00 | 20260505 16:54:34.798 | INFO | PerfReport: 20.30%, LockCompute -- 2.03e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260505 16:54:34.798 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.93e-02 / 0.00e+00 | 20260505 16:54:34.934 | INFO | Starting a round of Baz IO | 20260505 16:54:38.111 | INFO | Populating RTMetrics report | 20260505 16:54:38.111 | INFO | RTMetrics report populated | 20260505 16:54:38.111 | INFO | Opening tmp RTMetrics json file | 20260505 16:54:38.115 | INFO | Serializing RTMetrics to json | 20260505 16:54:38.116 | INFO | Serialization done, swapping tmp file now... | 20260505 16:54:38.116 | INFO | RTMetrics done | 20260505 16:54:38.206 | INFO | Finished a round of Baz IO: Wrote 2.155 GiB in 3.271 seconds | 20260505 16:54:38.206 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.16% / 0.00% / 77.84% | 20260505 16:54:38.206 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.335224% / 0% / 8.74201e-05% | 20260505 16:54:38.206 | INFO | Baz IO overall: | 20260505 16:54:38.206 | INFO | BytesWritten : 2844048672041 | 20260505 16:54:38.206 | INFO | NumEvents : 1349351318438 | 20260505 16:54:39.924 | INFO | Analyzing chunk frames = [4800512,4801024) | 20260505 16:54:39.924 | INFO | MemInfo: { | 20260505 16:54:39.924 | INFO | "gpu" : " 17 GB", | 20260505 16:54:39.924 | INFO | "huge" : "", | 20260505 16:54:39.924 | INFO | "huge_shared" : " 27 GB", | 20260505 16:54:39.924 | INFO | "malloc" : " 14 GB", | 20260505 16:54:39.925 | INFO | "pinned" : " 29 GB" | 20260505 16:54:39.925 | INFO | } | 20260505 16:54:39.925 | INFO | Perf report for the last 5.12795 seconds: | 20260505 16:54:39.925 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:54:39.925 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:39.925 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:39.925 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:39.925 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260505 16:54:39.925 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:39.925 | INFO | RT_METRICS : 44.7%, 1.0, 23.4ms, 0 | 20260505 16:54:39.925 | INFO | PRE_HQ : 60.8%, 1.8, 56.7ms, 0 | 20260505 16:54:39.925 | INFO | BAZWRITER : 63.3%, 3.8, 124.7ms, 0 | 20260505 16:54:39.925 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 16:54:39.925 | INFO | Uploaded 36GiB to GPU in 2.16371s (16.6381GiB/s) | 20260505 16:54:39.925 | INFO | Downloaded 38GiB to GPU in 2.25731s (16.8342GiB/s) | 20260505 16:54:45.060 | INFO | Analyzing chunk frames = [4801024,4801536) | 20260505 16:54:45.060 | INFO | MemInfo: { | 20260505 16:54:45.060 | INFO | "gpu" : " 17 GB", | 20260505 16:54:45.060 | INFO | "huge" : "", | 20260505 16:54:45.060 | INFO | "huge_shared" : " 27 GB", | 20260505 16:54:45.060 | INFO | "malloc" : " 14 GB", | 20260505 16:54:45.060 | INFO | "pinned" : " 29 GB" | 20260505 16:54:45.060 | INFO | } | 20260505 16:54:45.060 | INFO | Perf report for the last 5.13567 seconds: | 20260505 16:54:45.060 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:54:45.060 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:45.060 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:45.060 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:45.060 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 16:54:45.060 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:45.060 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:45.060 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:54:45.060 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:45.060 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:45.061 | INFO | Uploaded 36GiB to GPU in 2.12539s (16.9381GiB/s) | 20260505 16:54:45.061 | INFO | Downloaded 34GiB to GPU in 2.03658s (16.6947GiB/s) | 20260505 16:54:50.206 | INFO | Analyzing chunk frames = [4801536,4802048) | 20260505 16:54:50.206 | INFO | MemInfo: { | 20260505 16:54:50.206 | INFO | "gpu" : " 17 GB", | 20260505 16:54:50.206 | INFO | "huge" : "", | 20260505 16:54:50.206 | INFO | "huge_shared" : " 27 GB", | 20260505 16:54:50.206 | INFO | "malloc" : " 14 GB", | 20260505 16:54:50.206 | INFO | "pinned" : " 29 GB" | 20260505 16:54:50.206 | INFO | } | 20260505 16:54:50.206 | INFO | Perf report for the last 5.14609 seconds: | 20260505 16:54:50.206 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:54:50.206 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:50.206 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:50.206 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:50.206 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260505 16:54:50.206 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:50.206 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:50.206 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 16:54:50.206 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:50.207 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:50.207 | INFO | Uploaded 36GiB to GPU in 2.1741s (16.5585GiB/s) | 20260505 16:54:50.207 | INFO | Downloaded 34GiB to GPU in 2.0342s (16.7142GiB/s) | 20260505 16:54:55.348 | INFO | Analyzing chunk frames = [4802048,4802560) | 20260505 16:54:55.348 | INFO | MemInfo: { | 20260505 16:54:55.348 | INFO | "gpu" : " 17 GB", | 20260505 16:54:55.348 | INFO | "huge" : "", | 20260505 16:54:55.348 | INFO | "huge_shared" : " 27 GB", | 20260505 16:54:55.348 | INFO | "malloc" : " 14 GB", | 20260505 16:54:55.348 | INFO | "pinned" : " 29 GB" | 20260505 16:54:55.348 | INFO | } | 20260505 16:54:55.348 | INFO | Perf report for the last 5.14205 seconds: | 20260505 16:54:55.348 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:54:55.348 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:55.348 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:55.349 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:55.349 | INFO | ANALYSIS : 62.7%, 3.0, 97.5ms, 0 | 20260505 16:54:55.349 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:55.349 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:55.349 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 16:54:55.349 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:55.349 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:54:55.349 | INFO | Uploaded 36GiB to GPU in 2.22265s (16.1969GiB/s) | 20260505 16:54:55.349 | INFO | Downloaded 34GiB to GPU in 2.12912s (15.969GiB/s) | 20260505 16:55:00.481 | INFO | Analyzing chunk frames = [4802560,4803072) | 20260505 16:55:00.481 | INFO | MemInfo: { | 20260505 16:55:00.481 | INFO | "gpu" : " 17 GB", | 20260505 16:55:00.481 | INFO | "huge" : "", | 20260505 16:55:00.481 | INFO | "huge_shared" : " 27 GB", | 20260505 16:55:00.481 | INFO | "malloc" : " 14 GB", | 20260505 16:55:00.481 | INFO | "pinned" : " 29 GB" | 20260505 16:55:00.481 | INFO | } | 20260505 16:55:00.481 | INFO | Perf report for the last 5.13275 seconds: | 20260505 16:55:00.481 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:55:00.481 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:00.481 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:00.481 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:00.481 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260505 16:55:00.481 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:00.481 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:00.481 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:55:00.481 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:00.481 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:00.481 | INFO | Uploaded 36GiB to GPU in 2.13264s (16.8805GiB/s) | 20260505 16:55:00.481 | INFO | Downloaded 34GiB to GPU in 2.05516s (16.5437GiB/s) | 20260505 16:55:05.620 | INFO | Analyzing chunk frames = [4803072,4803584) | 20260505 16:55:05.620 | INFO | MemInfo: { | 20260505 16:55:05.620 | INFO | "gpu" : " 17 GB", | 20260505 16:55:05.620 | INFO | "huge" : "", | 20260505 16:55:05.620 | INFO | "huge_shared" : " 27 GB", | 20260505 16:55:05.620 | INFO | "malloc" : " 14 GB", | 20260505 16:55:05.620 | INFO | "pinned" : " 29 GB" | 20260505 16:55:05.620 | INFO | } | 20260505 16:55:05.620 | INFO | Perf report for the last 5.13871 seconds: | 20260505 16:55:05.620 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:55:05.620 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:05.620 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:05.620 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:05.620 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260505 16:55:05.620 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:05.620 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:05.620 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 16:55:05.620 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:05.620 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:05.620 | INFO | Uploaded 36GiB to GPU in 2.13611s (16.8531GiB/s) | 20260505 16:55:05.620 | INFO | Downloaded 34GiB to GPU in 2.03847s (16.6792GiB/s) | 20260505 16:55:10.762 | INFO | Analyzing chunk frames = [4803584,4804096) | 20260505 16:55:10.762 | INFO | MemInfo: { | 20260505 16:55:10.762 | INFO | "gpu" : " 17 GB", | 20260505 16:55:10.762 | INFO | "huge" : "", | 20260505 16:55:10.762 | INFO | "huge_shared" : " 27 GB", | 20260505 16:55:10.762 | INFO | "malloc" : " 14 GB", | 20260505 16:55:10.762 | INFO | "pinned" : " 29 GB" | 20260505 16:55:10.762 | INFO | } | 20260505 16:55:10.762 | INFO | Perf report for the last 5.14238 seconds: | 20260505 16:55:10.762 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:55:10.762 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:10.762 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:10.762 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:10.762 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 16:55:10.762 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:10.762 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:10.762 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 16:55:10.762 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:10.762 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:10.763 | INFO | Uploaded 36GiB to GPU in 2.11304s (17.0371GiB/s) | 20260505 16:55:10.763 | INFO | Downloaded 34GiB to GPU in 2.17157s (15.6569GiB/s) | 20260505 16:55:15.908 | INFO | Analyzing chunk frames = [4804096,4804608) | 20260505 16:55:15.908 | INFO | MemInfo: { | 20260505 16:55:15.908 | INFO | "gpu" : " 17 GB", | 20260505 16:55:15.908 | INFO | "huge" : "", | 20260505 16:55:15.908 | INFO | "huge_shared" : " 27 GB", | 20260505 16:55:15.908 | INFO | "malloc" : " 14 GB", | 20260505 16:55:15.908 | INFO | "pinned" : " 29 GB" | 20260505 16:55:15.908 | INFO | } | 20260505 16:55:15.908 | INFO | Perf report for the last 5.14562 seconds: | 20260505 16:55:15.908 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:55:15.908 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:15.908 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:15.908 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:15.908 | INFO | ANALYSIS : 63.1%, 3.0, 98.7ms, 0 | 20260505 16:55:15.908 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:15.908 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:15.908 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:55:15.908 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:15.908 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:15.908 | INFO | Uploaded 36GiB to GPU in 2.15178s (16.7304GiB/s) | 20260505 16:55:15.908 | INFO | Downloaded 34GiB to GPU in 2.03153s (16.7361GiB/s) | 20260505 16:55:16.041 | INFO | Starting a round of Baz IO | 20260505 16:55:19.224 | INFO | Populating RTMetrics report | 20260505 16:55:19.224 | INFO | RTMetrics report populated | 20260505 16:55:19.224 | INFO | Opening tmp RTMetrics json file | 20260505 16:55:19.228 | INFO | Serializing RTMetrics to json | 20260505 16:55:19.229 | INFO | Serialization done, swapping tmp file now... | 20260505 16:55:19.229 | INFO | RTMetrics done | 20260505 16:55:19.320 | INFO | Finished a round of Baz IO: Wrote 2.154 GiB in 3.279 seconds | 20260505 16:55:19.320 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.15% / 0.00% / 77.84% | 20260505 16:55:19.320 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.333691% / 7.94729e-06% / 8.74201e-05% | 20260505 16:55:19.320 | INFO | Baz IO overall: | 20260505 16:55:19.320 | INFO | BytesWritten : 2846361520912 | 20260505 16:55:19.320 | INFO | NumEvents : 1350433990877 | 20260505 16:55:21.047 | INFO | Analyzing chunk frames = [4804608,4805120) | 20260505 16:55:21.047 | INFO | MemInfo: { | 20260505 16:55:21.047 | INFO | "gpu" : " 17 GB", | 20260505 16:55:21.047 | INFO | "huge" : "", | 20260505 16:55:21.047 | INFO | "huge_shared" : " 27 GB", | 20260505 16:55:21.047 | INFO | "malloc" : " 14 GB", | 20260505 16:55:21.047 | INFO | "pinned" : " 29 GB" | 20260505 16:55:21.047 | INFO | } | 20260505 16:55:21.047 | INFO | Perf report for the last 5.13914 seconds: | 20260505 16:55:21.047 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:55:21.047 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:21.047 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:21.047 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:21.047 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 16:55:21.047 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:21.047 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260505 16:55:21.047 | INFO | PRE_HQ : 60.5%, 1.8, 56.3ms, 0 | 20260505 16:55:21.047 | INFO | BAZWRITER : 63.2%, 3.8, 124.3ms, 0 | 20260505 16:55:21.047 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 16:55:21.047 | INFO | Uploaded 36GiB to GPU in 2.16229s (16.649GiB/s) | 20260505 16:55:21.048 | INFO | Downloaded 38GiB to GPU in 2.24468s (16.9289GiB/s) | 20260505 16:55:26.173 | INFO | Analyzing chunk frames = [4805120,4805632) | 20260505 16:55:26.173 | INFO | MemInfo: { | 20260505 16:55:26.173 | INFO | "gpu" : " 17 GB", | 20260505 16:55:26.173 | INFO | "huge" : "", | 20260505 16:55:26.173 | INFO | "huge_shared" : " 27 GB", | 20260505 16:55:26.173 | INFO | "malloc" : " 14 GB", | 20260505 16:55:26.173 | INFO | "pinned" : " 29 GB" | 20260505 16:55:26.173 | INFO | } | 20260505 16:55:26.173 | INFO | Perf report for the last 5.12617 seconds: | 20260505 16:55:26.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:55:26.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:26.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:26.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:26.173 | INFO | ANALYSIS : 63.0%, 3.0, 98.0ms, 0 | 20260505 16:55:26.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:26.173 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:26.173 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 16:55:26.173 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:26.173 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:26.174 | INFO | Uploaded 36GiB to GPU in 2.12653s (16.929GiB/s) | 20260505 16:55:26.174 | INFO | Downloaded 34GiB to GPU in 2.02262s (16.8099GiB/s) | 20260505 16:55:31.312 | INFO | Analyzing chunk frames = [4805632,4806144) | 20260505 16:55:31.312 | INFO | MemInfo: { | 20260505 16:55:31.312 | INFO | "gpu" : " 17 GB", | 20260505 16:55:31.312 | INFO | "huge" : "", | 20260505 16:55:31.312 | INFO | "huge_shared" : " 27 GB", | 20260505 16:55:31.312 | INFO | "malloc" : " 14 GB", | 20260505 16:55:31.312 | INFO | "pinned" : " 29 GB" | 20260505 16:55:31.312 | INFO | } | 20260505 16:55:31.312 | INFO | Perf report for the last 5.13918 seconds: | 20260505 16:55:31.312 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:55:31.312 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:31.312 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:31.312 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:31.313 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260505 16:55:31.313 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:31.313 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:31.313 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:55:31.313 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:31.313 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:31.313 | INFO | Uploaded 36GiB to GPU in 2.18481s (16.4774GiB/s) | 20260505 16:55:31.313 | INFO | Downloaded 34GiB to GPU in 2.03725s (16.6892GiB/s) | 20260505 16:55:36.459 | INFO | Analyzing chunk frames = [4806144,4806656) | 20260505 16:55:36.459 | INFO | MemInfo: { | 20260505 16:55:36.459 | INFO | "gpu" : " 17 GB", | 20260505 16:55:36.459 | INFO | "huge" : "", | 20260505 16:55:36.460 | INFO | "huge_shared" : " 27 GB", | 20260505 16:55:36.460 | INFO | "malloc" : " 14 GB", | 20260505 16:55:36.460 | INFO | "pinned" : " 29 GB" | 20260505 16:55:36.460 | INFO | } | 20260505 16:55:36.460 | INFO | Perf report for the last 5.14725 seconds: | 20260505 16:55:36.460 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:55:36.460 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:36.460 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:36.460 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:36.460 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260505 16:55:36.460 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:36.460 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:36.460 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 16:55:36.460 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:36.460 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:36.460 | INFO | Uploaded 36GiB to GPU in 2.18473s (16.478GiB/s) | 20260505 16:55:36.460 | INFO | Downloaded 34GiB to GPU in 2.13144s (15.9516GiB/s) | 20260505 16:55:41.603 | INFO | Analyzing chunk frames = [4806656,4807168) | 20260505 16:55:41.603 | INFO | MemInfo: { | 20260505 16:55:41.603 | INFO | "gpu" : " 17 GB", | 20260505 16:55:41.603 | INFO | "huge" : "", | 20260505 16:55:41.603 | INFO | "huge_shared" : " 27 GB", | 20260505 16:55:41.603 | INFO | "malloc" : " 14 GB", | 20260505 16:55:41.603 | INFO | "pinned" : " 29 GB" | 20260505 16:55:41.603 | INFO | } | 20260505 16:55:41.603 | INFO | Perf report for the last 5.14332 seconds: | 20260505 16:55:41.603 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:55:41.603 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:41.603 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:41.603 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:41.603 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 16:55:41.603 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:41.603 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:41.603 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:55:41.603 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:41.603 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:41.603 | INFO | Uploaded 36GiB to GPU in 2.13944s (16.8268GiB/s) | 20260505 16:55:41.603 | INFO | Downloaded 34GiB to GPU in 2.05976s (16.5068GiB/s) | 20260505 16:55:46.741 | INFO | Analyzing chunk frames = [4807168,4807680) | 20260505 16:55:46.741 | INFO | MemInfo: { | 20260505 16:55:46.741 | INFO | "gpu" : " 17 GB", | 20260505 16:55:46.741 | INFO | "huge" : "", | 20260505 16:55:46.741 | INFO | "huge_shared" : " 27 GB", | 20260505 16:55:46.741 | INFO | "malloc" : " 14 GB", | 20260505 16:55:46.741 | INFO | "pinned" : " 29 GB" | 20260505 16:55:46.741 | INFO | } | 20260505 16:55:46.741 | INFO | Perf report for the last 5.13774 seconds: | 20260505 16:55:46.741 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:55:46.741 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:46.741 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:46.741 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:46.741 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 16:55:46.741 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:46.741 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:46.741 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:55:46.741 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:46.741 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:46.741 | INFO | Uploaded 36GiB to GPU in 2.13946s (16.8267GiB/s) | 20260505 16:55:46.741 | INFO | Downloaded 34GiB to GPU in 2.04413s (16.633GiB/s) | 20260505 16:55:51.872 | INFO | Analyzing chunk frames = [4807680,4808192) | 20260505 16:55:51.872 | INFO | MemInfo: { | 20260505 16:55:51.872 | INFO | "gpu" : " 17 GB", | 20260505 16:55:51.872 | INFO | "huge" : "", | 20260505 16:55:51.872 | INFO | "huge_shared" : " 27 GB", | 20260505 16:55:51.872 | INFO | "malloc" : " 14 GB", | 20260505 16:55:51.872 | INFO | "pinned" : " 29 GB" | 20260505 16:55:51.872 | INFO | } | 20260505 16:55:51.872 | INFO | Perf report for the last 5.13139 seconds: | 20260505 16:55:51.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:55:51.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:51.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:51.872 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:51.872 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 16:55:51.872 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:51.872 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:51.872 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 16:55:51.872 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:51.872 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:51.872 | INFO | Uploaded 36GiB to GPU in 2.16625s (16.6186GiB/s) | 20260505 16:55:51.873 | INFO | Downloaded 34GiB to GPU in 2.18251s (15.5784GiB/s) | 20260505 16:55:57.012 | INFO | Analyzing chunk frames = [4808192,4808704) | 20260505 16:55:57.012 | INFO | MemInfo: { | 20260505 16:55:57.012 | INFO | "gpu" : " 17 GB", | 20260505 16:55:57.012 | INFO | "huge" : "", | 20260505 16:55:57.012 | INFO | "huge_shared" : " 27 GB", | 20260505 16:55:57.012 | INFO | "malloc" : " 14 GB", | 20260505 16:55:57.012 | INFO | "pinned" : " 29 GB" | 20260505 16:55:57.012 | INFO | } | 20260505 16:55:57.012 | INFO | Perf report for the last 5.13982 seconds: | 20260505 16:55:57.012 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:55:57.012 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:57.012 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:57.012 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:57.012 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 16:55:57.012 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:57.012 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:57.012 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 16:55:57.012 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:57.012 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:55:57.012 | INFO | Uploaded 36GiB to GPU in 2.22834s (16.1556GiB/s) | 20260505 16:55:57.012 | INFO | Downloaded 34GiB to GPU in 2.10573s (16.1464GiB/s) | 20260505 16:55:57.012 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 16:55:57.012 | INFO | Total mem usage: 15.3 GiB | 20260505 16:55:57.012 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 16:55:57.013 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.93%) | 20260505 16:55:57.013 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 16:55:57.013 | INFO | High Water Breakdown: | 20260505 16:55:57.013 | INFO | 61% : 8.67 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 16:55:57.013 | INFO | 38.8% : 5.51 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 16:55:57.013 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 16:55:57.013 | INFO | ------------------------------------------ | 20260505 16:55:57.013 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:55:57.013 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 16:55:57.013 | INFO | Total mem usage: 26.5 GiB | 20260505 16:55:57.013 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 16:55:57.013 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 16:55:57.013 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 16:55:57.013 | INFO | High Water Breakdown: | 20260505 16:55:57.013 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:55:57.013 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:55:57.013 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:55:57.013 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:55:57.013 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:55:57.013 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:55:57.013 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 16:55:57.013 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 16:55:57.013 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:55:57.013 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 16:55:57.013 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 16:55:57.013 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 16:55:57.013 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 16:55:57.013 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 16:55:57.013 | INFO | ------------------------------------------ | 20260505 16:55:57.013 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:55:57.013 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 16:55:57.013 | INFO | Total mem usage: 25.6 GiB | 20260505 16:55:57.013 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 16:55:57.013 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 16:55:57.013 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 16:55:57.013 | INFO | High Water Breakdown: | 20260505 16:55:57.013 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 16:55:57.013 | INFO | ------------------------------------------ | 20260505 16:55:57.013 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:55:57.013 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 16:55:57.013 | INFO | Total mem usage: 15.7 GiB | 20260505 16:55:57.013 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 16:55:57.013 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 16:55:57.013 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 16:55:57.013 | INFO | High Water Breakdown: | 20260505 16:55:57.013 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 16:55:57.013 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 16:55:57.013 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 16:55:57.013 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 16:55:57.013 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 16:55:57.013 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 16:55:57.013 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 16:55:57.013 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 16:55:57.013 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 16:55:57.013 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:55:57.013 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 16:55:57.013 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:55:57.013 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 16:55:57.013 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 16:55:57.013 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 16:55:57.013 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 16:55:57.013 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:55:57.013 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 16:55:57.013 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:55:57.013 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:55:57.013 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:55:57.013 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 16:55:57.013 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 16:55:57.013 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:55:57.013 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 16:55:57.013 | INFO | ------------------------------------------ | 20260505 16:55:57.013 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:55:57.013 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 16:55:57.013 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:55:57.013 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:55:57.013 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.51e+01, 1.36e+01 / 2.14e+00 | 20260505 16:55:57.013 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 1.03e-01, 1.07e-02 / 5.18e-03 | 20260505 16:55:57.013 | INFO | PerfReport: 11.80%, Baseline -- 3.74e+00 / 4.52e+00, 5.84e-01 / 1.52e-01 | 20260505 16:55:57.013 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.52e+00, 1.94e-01 / 1.00e-01 | 20260505 16:55:57.013 | INFO | PerfReport: 15.94%, DME -- 4.04e+01 / 4.49e+01, 4.21e+00 / 2.12e+00 | 20260505 16:55:57.013 | INFO | PerfReport: 56.75%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.50e+00 / 1.48e+00 | 20260505 16:55:57.013 | INFO | PerfReport: 7.35%, PulseAccumulating -- 2.33e+00 / 2.90e+00, 4.51e-01 / 1.72e-01 | 20260505 16:55:57.013 | INFO | PerfReport: 3.40%, Metrics -- 1.08e+00 / 1.42e+00, 3.02e-01 / 1.19e-01 | 20260505 16:55:57.013 | INFO | PerfReport: 1.19%, PulseSerializing -- 3.76e-01 / 4.33e-01, 6.76e-02 / 4.18e-02 | 20260505 16:55:57.013 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 16:55:57.013 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:55:57.013 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:55:57.013 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260505 16:55:57.013 | INFO | PerfReport: 22.48%, Upload -- 2.25e+01 / 0.00e+00, 7.87e+00 / 0.00e+00 | 20260505 16:55:57.013 | INFO | PerfReport: 21.85%, Download -- 2.19e+01 / 0.00e+00, 7.88e+00 / 0.00e+00 | 20260505 16:55:57.013 | INFO | PerfReport: 31.67%, Compute -- 3.17e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 16:55:57.013 | INFO | PerfReport: 2.26%, LockUpload -- 2.26e+00 / 0.00e+00, 7.85e+00 / 0.00e+00 | 20260505 16:55:57.013 | INFO | PerfReport: 1.39%, LockDownload -- 1.39e+00 / 0.00e+00, 2.93e+00 / 0.00e+00 | 20260505 16:55:57.013 | INFO | PerfReport: 20.32%, LockCompute -- 2.03e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 16:55:57.013 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.52e-02 / 0.00e+00 | 20260505 16:55:57.145 | INFO | Starting a round of Baz IO | 20260505 16:56:00.343 | INFO | Populating RTMetrics report | 20260505 16:56:00.343 | INFO | RTMetrics report populated | 20260505 16:56:00.343 | INFO | Opening tmp RTMetrics json file | 20260505 16:56:00.349 | INFO | Serializing RTMetrics to json | 20260505 16:56:00.350 | INFO | Serialization done, swapping tmp file now... | 20260505 16:56:00.351 | INFO | RTMetrics done | 20260505 16:56:00.438 | INFO | Finished a round of Baz IO: Wrote 2.157 GiB in 3.293 seconds | 20260505 16:56:00.438 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.15% / 0.00% / 77.85% | 20260505 16:56:00.438 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.335121% / 3.97364e-06% / 9.13938e-05% | 20260505 16:56:00.438 | INFO | Baz IO overall: | 20260505 16:56:00.438 | INFO | BytesWritten : 2848678113154 | 20260505 16:56:00.438 | INFO | NumEvents : 1351518763912 | 20260505 16:56:02.158 | INFO | Analyzing chunk frames = [4808704,4809216) | 20260505 16:56:02.158 | INFO | MemInfo: { | 20260505 16:56:02.158 | INFO | "gpu" : " 17 GB", | 20260505 16:56:02.158 | INFO | "huge" : "", | 20260505 16:56:02.158 | INFO | "huge_shared" : " 27 GB", | 20260505 16:56:02.158 | INFO | "malloc" : " 14 GB", | 20260505 16:56:02.158 | INFO | "pinned" : " 29 GB" | 20260505 16:56:02.158 | INFO | } | 20260505 16:56:02.158 | INFO | Perf report for the last 5.14563 seconds: | 20260505 16:56:02.158 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:56:02.158 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:02.158 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:02.158 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:02.158 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 16:56:02.158 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:02.158 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260505 16:56:02.158 | INFO | PRE_HQ : 60.6%, 1.8, 56.3ms, 0 | 20260505 16:56:02.158 | INFO | BAZWRITER : 63.3%, 3.8, 125.2ms, 0 | 20260505 16:56:02.158 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 16:56:02.158 | INFO | Uploaded 36GiB to GPU in 2.16796s (16.6055GiB/s) | 20260505 16:56:02.158 | INFO | Downloaded 38GiB to GPU in 2.25443s (16.8557GiB/s) | 20260505 16:56:07.294 | INFO | Analyzing chunk frames = [4809216,4809728) | 20260505 16:56:07.294 | INFO | MemInfo: { | 20260505 16:56:07.294 | INFO | "gpu" : " 17 GB", | 20260505 16:56:07.294 | INFO | "huge" : "", | 20260505 16:56:07.294 | INFO | "huge_shared" : " 27 GB", | 20260505 16:56:07.294 | INFO | "malloc" : " 14 GB", | 20260505 16:56:07.294 | INFO | "pinned" : " 29 GB" | 20260505 16:56:07.294 | INFO | } | 20260505 16:56:07.294 | INFO | Perf report for the last 5.13671 seconds: | 20260505 16:56:07.294 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:56:07.294 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:07.294 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:07.294 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:07.294 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 16:56:07.294 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:07.295 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:07.295 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 16:56:07.295 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:07.295 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:07.295 | INFO | Uploaded 36GiB to GPU in 2.13025s (16.8995GiB/s) | 20260505 16:56:07.295 | INFO | Downloaded 34GiB to GPU in 2.0193s (16.8375GiB/s) | 20260505 16:56:12.433 | INFO | Analyzing chunk frames = [4809728,4810240) | 20260505 16:56:12.433 | INFO | MemInfo: { | 20260505 16:56:12.433 | INFO | "gpu" : " 17 GB", | 20260505 16:56:12.433 | INFO | "huge" : "", | 20260505 16:56:12.433 | INFO | "huge_shared" : " 27 GB", | 20260505 16:56:12.433 | INFO | "malloc" : " 14 GB", | 20260505 16:56:12.433 | INFO | "pinned" : " 29 GB" | 20260505 16:56:12.433 | INFO | } | 20260505 16:56:12.433 | INFO | Perf report for the last 5.13841 seconds: | 20260505 16:56:12.433 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:56:12.433 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:12.433 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:12.433 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:12.433 | INFO | ANALYSIS : 62.7%, 3.0, 97.5ms, 0 | 20260505 16:56:12.433 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:12.433 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:12.433 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:56:12.433 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:12.433 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:12.433 | INFO | Uploaded 36GiB to GPU in 2.19885s (16.3722GiB/s) | 20260505 16:56:12.433 | INFO | Downloaded 34GiB to GPU in 2.02724s (16.7716GiB/s) | 20260505 16:56:17.573 | INFO | Analyzing chunk frames = [4810240,4810752) | 20260505 16:56:17.573 | INFO | MemInfo: { | 20260505 16:56:17.573 | INFO | "gpu" : " 17 GB", | 20260505 16:56:17.573 | INFO | "huge" : "", | 20260505 16:56:17.573 | INFO | "huge_shared" : " 27 GB", | 20260505 16:56:17.573 | INFO | "malloc" : " 14 GB", | 20260505 16:56:17.573 | INFO | "pinned" : " 29 GB" | 20260505 16:56:17.573 | INFO | } | 20260505 16:56:17.573 | INFO | Perf report for the last 5.14054 seconds: | 20260505 16:56:17.573 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:56:17.573 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:17.573 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:17.573 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:17.573 | INFO | ANALYSIS : 62.7%, 3.0, 97.4ms, 0 | 20260505 16:56:17.573 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:17.573 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:17.574 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 16:56:17.574 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:17.574 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:17.574 | INFO | Uploaded 36GiB to GPU in 2.18858s (16.449GiB/s) | 20260505 16:56:17.574 | INFO | Downloaded 34GiB to GPU in 2.14242s (15.8699GiB/s) | 20260505 16:56:22.720 | INFO | Analyzing chunk frames = [4810752,4811264) | 20260505 16:56:22.720 | INFO | MemInfo: { | 20260505 16:56:22.720 | INFO | "gpu" : " 17 GB", | 20260505 16:56:22.720 | INFO | "huge" : "", | 20260505 16:56:22.720 | INFO | "huge_shared" : " 27 GB", | 20260505 16:56:22.720 | INFO | "malloc" : " 14 GB", | 20260505 16:56:22.720 | INFO | "pinned" : " 29 GB" | 20260505 16:56:22.720 | INFO | } | 20260505 16:56:22.720 | INFO | Perf report for the last 5.14683 seconds: | 20260505 16:56:22.720 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:56:22.720 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:22.720 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:22.720 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:22.720 | INFO | ANALYSIS : 62.5%, 3.0, 97.8ms, 0 | 20260505 16:56:22.720 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:22.720 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:22.720 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260505 16:56:22.720 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:22.720 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:22.721 | INFO | Uploaded 36GiB to GPU in 2.18819s (16.4519GiB/s) | 20260505 16:56:22.721 | INFO | Downloaded 34GiB to GPU in 2.13104s (15.9547GiB/s) | 20260505 16:56:27.861 | INFO | Analyzing chunk frames = [4811264,4811776) | 20260505 16:56:27.861 | INFO | MemInfo: { | 20260505 16:56:27.861 | INFO | "gpu" : " 17 GB", | 20260505 16:56:27.861 | INFO | "huge" : "", | 20260505 16:56:27.861 | INFO | "huge_shared" : " 27 GB", | 20260505 16:56:27.861 | INFO | "malloc" : " 14 GB", | 20260505 16:56:27.861 | INFO | "pinned" : " 29 GB" | 20260505 16:56:27.861 | INFO | } | 20260505 16:56:27.861 | INFO | Perf report for the last 5.14089 seconds: | 20260505 16:56:27.861 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:56:27.861 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:27.861 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:27.861 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:27.861 | INFO | ANALYSIS : 62.3%, 3.0, 97.4ms, 0 | 20260505 16:56:27.861 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:27.861 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:27.861 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 16:56:27.861 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:27.861 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:27.861 | INFO | Uploaded 36GiB to GPU in 2.13906s (16.8298GiB/s) | 20260505 16:56:27.861 | INFO | Downloaded 34GiB to GPU in 2.04206s (16.6498GiB/s) | 20260505 16:56:32.991 | INFO | Analyzing chunk frames = [4811776,4812288) | 20260505 16:56:32.991 | INFO | MemInfo: { | 20260505 16:56:32.991 | INFO | "gpu" : " 17 GB", | 20260505 16:56:32.991 | INFO | "huge" : "", | 20260505 16:56:32.991 | INFO | "huge_shared" : " 27 GB", | 20260505 16:56:32.991 | INFO | "malloc" : " 14 GB", | 20260505 16:56:32.991 | INFO | "pinned" : " 29 GB" | 20260505 16:56:32.991 | INFO | } | 20260505 16:56:32.991 | INFO | Perf report for the last 5.12994 seconds: | 20260505 16:56:32.991 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:56:32.991 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:32.991 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:32.991 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:32.991 | INFO | ANALYSIS : 63.6%, 3.0, 99.1ms, 0 | 20260505 16:56:32.991 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:32.991 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:32.991 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 16:56:32.991 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:32.991 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:32.991 | INFO | Uploaded 36GiB to GPU in 2.15452s (16.709GiB/s) | 20260505 16:56:32.991 | INFO | Downloaded 34GiB to GPU in 2.18637s (15.5509GiB/s) | 20260505 16:56:38.126 | INFO | Analyzing chunk frames = [4812288,4812800) | 20260505 16:56:38.126 | INFO | MemInfo: { | 20260505 16:56:38.126 | INFO | "gpu" : " 17 GB", | 20260505 16:56:38.126 | INFO | "huge" : "", | 20260505 16:56:38.126 | INFO | "huge_shared" : " 27 GB", | 20260505 16:56:38.126 | INFO | "malloc" : " 14 GB", | 20260505 16:56:38.126 | INFO | "pinned" : " 29 GB" | 20260505 16:56:38.126 | INFO | } | 20260505 16:56:38.126 | INFO | Perf report for the last 5.13511 seconds: | 20260505 16:56:38.126 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:56:38.126 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:38.126 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:38.126 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:38.126 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 16:56:38.126 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:38.126 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:38.126 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 16:56:38.126 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:38.126 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:38.127 | INFO | Uploaded 36GiB to GPU in 2.16953s (16.5935GiB/s) | 20260505 16:56:38.127 | INFO | Downloaded 34GiB to GPU in 2.03745s (16.6875GiB/s) | 20260505 16:56:38.258 | INFO | Starting a round of Baz IO | 20260505 16:56:41.467 | INFO | Populating RTMetrics report | 20260505 16:56:41.468 | INFO | RTMetrics report populated | 20260505 16:56:41.468 | INFO | Opening tmp RTMetrics json file | 20260505 16:56:41.478 | INFO | Serializing RTMetrics to json | 20260505 16:56:41.480 | INFO | Serialization done, swapping tmp file now... | 20260505 16:56:41.482 | INFO | RTMetrics done | 20260505 16:56:41.558 | INFO | Finished a round of Baz IO: Wrote 2.154 GiB in 3.3 seconds | 20260505 16:56:41.559 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.15% / 0.00% / 77.85% | 20260505 16:56:41.559 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.33232% / 0% / 0.000115236% | 20260505 16:56:41.559 | INFO | Baz IO overall: | 20260505 16:56:41.559 | INFO | BytesWritten : 2850991132621 | 20260505 16:56:41.559 | INFO | NumEvents : 1352600815988 | 20260505 16:56:43.267 | INFO | Analyzing chunk frames = [4812800,4813312) | 20260505 16:56:43.267 | INFO | MemInfo: { | 20260505 16:56:43.267 | INFO | "gpu" : " 17 GB", | 20260505 16:56:43.267 | INFO | "huge" : "", | 20260505 16:56:43.267 | INFO | "huge_shared" : " 27 GB", | 20260505 16:56:43.267 | INFO | "malloc" : " 14 GB", | 20260505 16:56:43.267 | INFO | "pinned" : " 29 GB" | 20260505 16:56:43.267 | INFO | } | 20260505 16:56:43.267 | INFO | Perf report for the last 5.1411 seconds: | 20260505 16:56:43.267 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:56:43.267 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:43.267 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:43.267 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:43.267 | INFO | ANALYSIS : 63.9%, 3.0, 99.5ms, 0 | 20260505 16:56:43.267 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:43.267 | INFO | RT_METRICS : 44.7%, 1.0, 23.5ms, 0 | 20260505 16:56:43.267 | INFO | PRE_HQ : 60.6%, 1.8, 56.2ms, 0 | 20260505 16:56:43.267 | INFO | BAZWRITER : 63.5%, 3.8, 125.4ms, 0 | 20260505 16:56:43.267 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 16:56:43.268 | INFO | Uploaded 36GiB to GPU in 2.20907s (16.2964GiB/s) | 20260505 16:56:43.268 | INFO | Downloaded 38GiB to GPU in 2.2678s (16.7564GiB/s) | 20260505 16:56:48.413 | INFO | Analyzing chunk frames = [4813312,4813824) | 20260505 16:56:48.413 | INFO | MemInfo: { | 20260505 16:56:48.413 | INFO | "gpu" : " 17 GB", | 20260505 16:56:48.413 | INFO | "huge" : "", | 20260505 16:56:48.413 | INFO | "huge_shared" : " 27 GB", | 20260505 16:56:48.413 | INFO | "malloc" : " 14 GB", | 20260505 16:56:48.413 | INFO | "pinned" : " 29 GB" | 20260505 16:56:48.413 | INFO | } | 20260505 16:56:48.413 | INFO | Perf report for the last 5.14561 seconds: | 20260505 16:56:48.413 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:56:48.413 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:48.413 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:48.413 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:48.413 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 16:56:48.413 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:48.413 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:48.413 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260505 16:56:48.413 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:48.413 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:48.413 | INFO | Uploaded 36GiB to GPU in 2.12429s (16.9468GiB/s) | 20260505 16:56:48.413 | INFO | Downloaded 34GiB to GPU in 2.03014s (16.7476GiB/s) | 20260505 16:56:53.556 | INFO | Analyzing chunk frames = [4813824,4814336) | 20260505 16:56:53.556 | INFO | MemInfo: { | 20260505 16:56:53.556 | INFO | "gpu" : " 17 GB", | 20260505 16:56:53.556 | INFO | "huge" : "", | 20260505 16:56:53.556 | INFO | "huge_shared" : " 27 GB", | 20260505 16:56:53.556 | INFO | "malloc" : " 14 GB", | 20260505 16:56:53.556 | INFO | "pinned" : " 29 GB" | 20260505 16:56:53.556 | INFO | } | 20260505 16:56:53.556 | INFO | Perf report for the last 5.14343 seconds: | 20260505 16:56:53.556 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:56:53.556 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:53.556 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:53.556 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:53.556 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 16:56:53.556 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:53.556 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:53.556 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 16:56:53.556 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:53.556 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:53.557 | INFO | Uploaded 36GiB to GPU in 2.202s (16.3488GiB/s) | 20260505 16:56:53.557 | INFO | Downloaded 34GiB to GPU in 2.04478s (16.6277GiB/s) | 20260505 16:56:58.688 | INFO | Analyzing chunk frames = [4814336,4814848) | 20260505 16:56:58.688 | INFO | MemInfo: { | 20260505 16:56:58.688 | INFO | "gpu" : " 17 GB", | 20260505 16:56:58.688 | INFO | "huge" : "", | 20260505 16:56:58.688 | INFO | "huge_shared" : " 27 GB", | 20260505 16:56:58.688 | INFO | "malloc" : " 14 GB", | 20260505 16:56:58.688 | INFO | "pinned" : " 29 GB" | 20260505 16:56:58.688 | INFO | } | 20260505 16:56:58.688 | INFO | Perf report for the last 5.13186 seconds: | 20260505 16:56:58.688 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:56:58.688 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:58.688 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:58.688 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:58.688 | INFO | ANALYSIS : 62.9%, 3.0, 97.5ms, 0 | 20260505 16:56:58.688 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:58.688 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:58.688 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 16:56:58.688 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:58.688 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:56:58.689 | INFO | Uploaded 36GiB to GPU in 2.2216s (16.2046GiB/s) | 20260505 16:56:58.689 | INFO | Downloaded 34GiB to GPU in 2.12212s (16.0217GiB/s) | 20260505 16:57:03.821 | INFO | Analyzing chunk frames = [4814848,4815360) | 20260505 16:57:03.821 | INFO | MemInfo: { | 20260505 16:57:03.821 | INFO | "gpu" : " 17 GB", | 20260505 16:57:03.821 | INFO | "huge" : "", | 20260505 16:57:03.821 | INFO | "huge_shared" : " 27 GB", | 20260505 16:57:03.821 | INFO | "malloc" : " 14 GB", | 20260505 16:57:03.821 | INFO | "pinned" : " 29 GB" | 20260505 16:57:03.821 | INFO | } | 20260505 16:57:03.821 | INFO | Perf report for the last 5.13272 seconds: | 20260505 16:57:03.821 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:57:03.821 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:03.821 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:03.821 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:03.821 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 16:57:03.821 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:03.821 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:03.821 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:57:03.821 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:03.821 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:03.821 | INFO | Uploaded 36GiB to GPU in 2.15335s (16.7181GiB/s) | 20260505 16:57:03.821 | INFO | Downloaded 34GiB to GPU in 2.05737s (16.526GiB/s) | 20260505 16:57:08.969 | INFO | Analyzing chunk frames = [4815360,4815872) | 20260505 16:57:08.969 | INFO | MemInfo: { | 20260505 16:57:08.969 | INFO | "gpu" : " 17 GB", | 20260505 16:57:08.969 | INFO | "huge" : "", | 20260505 16:57:08.969 | INFO | "huge_shared" : " 27 GB", | 20260505 16:57:08.969 | INFO | "malloc" : " 14 GB", | 20260505 16:57:08.969 | INFO | "pinned" : " 29 GB" | 20260505 16:57:08.969 | INFO | } | 20260505 16:57:08.969 | INFO | Perf report for the last 5.14809 seconds: | 20260505 16:57:08.969 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:57:08.969 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:08.969 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:08.969 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:08.969 | INFO | ANALYSIS : 62.4%, 3.0, 97.6ms, 0 | 20260505 16:57:08.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:08.969 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:08.969 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 16:57:08.969 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:08.969 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:08.969 | INFO | Uploaded 36GiB to GPU in 2.16699s (16.6129GiB/s) | 20260505 16:57:08.969 | INFO | Downloaded 34GiB to GPU in 2.03841s (16.6797GiB/s) | 20260505 16:57:14.110 | INFO | Analyzing chunk frames = [4815872,4816384) | 20260505 16:57:14.110 | INFO | MemInfo: { | 20260505 16:57:14.110 | INFO | "gpu" : " 17 GB", | 20260505 16:57:14.110 | INFO | "huge" : "", | 20260505 16:57:14.110 | INFO | "huge_shared" : " 27 GB", | 20260505 16:57:14.110 | INFO | "malloc" : " 14 GB", | 20260505 16:57:14.110 | INFO | "pinned" : " 29 GB" | 20260505 16:57:14.110 | INFO | } | 20260505 16:57:14.110 | INFO | Perf report for the last 5.14101 seconds: | 20260505 16:57:14.110 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:57:14.110 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:14.110 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:14.110 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:14.110 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 16:57:14.110 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:14.110 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:14.110 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 16:57:14.110 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:14.110 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:14.110 | INFO | Uploaded 36GiB to GPU in 2.16471s (16.6304GiB/s) | 20260505 16:57:14.110 | INFO | Downloaded 34GiB to GPU in 2.17575s (15.6268GiB/s) | 20260505 16:57:19.244 | INFO | Analyzing chunk frames = [4816384,4816896) | 20260505 16:57:19.244 | INFO | MemInfo: { | 20260505 16:57:19.244 | INFO | "gpu" : " 17 GB", | 20260505 16:57:19.244 | INFO | "huge" : "", | 20260505 16:57:19.244 | INFO | "huge_shared" : " 27 GB", | 20260505 16:57:19.244 | INFO | "malloc" : " 14 GB", | 20260505 16:57:19.244 | INFO | "pinned" : " 29 GB" | 20260505 16:57:19.244 | INFO | } | 20260505 16:57:19.244 | INFO | Perf report for the last 5.13376 seconds: | 20260505 16:57:19.244 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:57:19.244 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:19.244 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:19.244 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:19.244 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 16:57:19.244 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:19.244 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:19.244 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 16:57:19.244 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:19.244 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:19.244 | INFO | Uploaded 36GiB to GPU in 2.19809s (16.3778GiB/s) | 20260505 16:57:19.244 | INFO | Downloaded 34GiB to GPU in 2.02668s (16.7762GiB/s) | 20260505 16:57:19.244 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 16:57:19.244 | INFO | Total mem usage: 15.3 GiB | 20260505 16:57:19.244 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 16:57:19.244 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.91%) | 20260505 16:57:19.244 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 16:57:19.244 | INFO | High Water Breakdown: | 20260505 16:57:19.244 | INFO | 61% : 8.67 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 16:57:19.245 | INFO | 38.8% : 5.51 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 16:57:19.245 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 16:57:19.245 | INFO | ------------------------------------------ | 20260505 16:57:19.245 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:57:19.245 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 16:57:19.245 | INFO | Total mem usage: 26.5 GiB | 20260505 16:57:19.245 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 16:57:19.245 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 16:57:19.245 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 16:57:19.245 | INFO | High Water Breakdown: | 20260505 16:57:19.245 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:57:19.245 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:57:19.245 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:57:19.245 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:57:19.245 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:57:19.245 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:57:19.245 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 16:57:19.245 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 16:57:19.245 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:57:19.245 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 16:57:19.245 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 16:57:19.245 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 16:57:19.245 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 16:57:19.245 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 16:57:19.245 | INFO | ------------------------------------------ | 20260505 16:57:19.245 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:57:19.245 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 16:57:19.245 | INFO | Total mem usage: 25.6 GiB | 20260505 16:57:19.245 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 16:57:19.245 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 16:57:19.245 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 16:57:19.245 | INFO | High Water Breakdown: | 20260505 16:57:19.245 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 16:57:19.245 | INFO | ------------------------------------------ | 20260505 16:57:19.245 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:57:19.245 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 16:57:19.245 | INFO | Total mem usage: 15.7 GiB | 20260505 16:57:19.245 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 16:57:19.245 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 16:57:19.245 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 16:57:19.245 | INFO | High Water Breakdown: | 20260505 16:57:19.245 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 16:57:19.245 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 16:57:19.245 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 16:57:19.245 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 16:57:19.245 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 16:57:19.245 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 16:57:19.245 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 16:57:19.245 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 16:57:19.245 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 16:57:19.245 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:57:19.245 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 16:57:19.245 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:57:19.245 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 16:57:19.245 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 16:57:19.245 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 16:57:19.245 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 16:57:19.245 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:57:19.245 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 16:57:19.245 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:57:19.245 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:57:19.245 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:57:19.245 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 16:57:19.245 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 16:57:19.245 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:57:19.245 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 16:57:19.245 | INFO | ------------------------------------------ | 20260505 16:57:19.245 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:57:19.245 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 16:57:19.245 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:57:19.245 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:57:19.245 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.55e+01, 1.36e+01 / 2.35e+00 | 20260505 16:57:19.245 | INFO | PerfReport: 0.29%, AggregateStats -- 9.24e-02 / 1.01e-01, 1.03e-02 / 3.75e-03 | 20260505 16:57:19.246 | INFO | PerfReport: 11.78%, Baseline -- 3.73e+00 / 4.52e+00, 5.71e-01 / 1.46e-01 | 20260505 16:57:19.246 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.53e+00, 1.91e-01 / 9.27e-02 | 20260505 16:57:19.246 | INFO | PerfReport: 16.04%, DME -- 4.06e+01 / 4.56e+01, 4.60e+00 / 2.55e+00 | 20260505 16:57:19.246 | INFO | PerfReport: 56.64%, FrameLabeling -- 1.79e+01 / 1.90e+01, 1.36e+00 / 1.03e+00 | 20260505 16:57:19.246 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.33e+00 / 2.96e+00, 4.55e-01 / 1.28e-01 | 20260505 16:57:19.246 | INFO | PerfReport: 3.40%, Metrics -- 1.07e+00 / 1.43e+00, 2.98e-01 / 1.79e-01 | 20260505 16:57:19.246 | INFO | PerfReport: 1.18%, PulseSerializing -- 3.74e-01 / 4.34e-01, 6.62e-02 / 4.00e-02 | 20260505 16:57:19.246 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 16:57:19.246 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:57:19.246 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:57:19.246 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.58e+01 / 0.00e+00 | 20260505 16:57:19.246 | INFO | PerfReport: 22.60%, Upload -- 2.26e+01 / 0.00e+00, 7.93e+00 / 0.00e+00 | 20260505 16:57:19.246 | INFO | PerfReport: 21.84%, Download -- 2.19e+01 / 0.00e+00, 7.86e+00 / 0.00e+00 | 20260505 16:57:19.246 | INFO | PerfReport: 31.56%, Compute -- 3.16e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 16:57:19.246 | INFO | PerfReport: 2.35%, LockUpload -- 2.35e+00 / 0.00e+00, 8.08e+00 / 0.00e+00 | 20260505 16:57:19.246 | INFO | PerfReport: 1.40%, LockDownload -- 1.40e+00 / 0.00e+00, 2.86e+00 / 0.00e+00 | 20260505 16:57:19.246 | INFO | PerfReport: 20.25%, LockCompute -- 2.03e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 16:57:19.246 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 2.01e-02 / 0.00e+00 | 20260505 16:57:19.377 | INFO | Starting a round of Baz IO | 20260505 16:57:22.583 | INFO | Populating RTMetrics report | 20260505 16:57:22.583 | INFO | RTMetrics report populated | 20260505 16:57:22.583 | INFO | Opening tmp RTMetrics json file | 20260505 16:57:22.590 | INFO | Serializing RTMetrics to json | 20260505 16:57:22.593 | INFO | Serialization done, swapping tmp file now... | 20260505 16:57:22.595 | INFO | RTMetrics done | 20260505 16:57:22.679 | INFO | Finished a round of Baz IO: Wrote 2.151 GiB in 3.301 seconds | 20260505 16:57:22.679 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.15% / 0.00% / 77.85% | 20260505 16:57:22.679 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.330448% / 3.97364e-06% / 0.000107288% | 20260505 16:57:22.679 | INFO | Baz IO overall: | 20260505 16:57:22.679 | INFO | BytesWritten : 2853301194447 | 20260505 16:57:22.679 | INFO | NumEvents : 1353681284099 | 20260505 16:57:24.383 | INFO | Analyzing chunk frames = [4816896,4817408) | 20260505 16:57:24.383 | INFO | MemInfo: { | 20260505 16:57:24.383 | INFO | "gpu" : " 17 GB", | 20260505 16:57:24.383 | INFO | "huge" : "", | 20260505 16:57:24.383 | INFO | "huge_shared" : " 27 GB", | 20260505 16:57:24.383 | INFO | "malloc" : " 14 GB", | 20260505 16:57:24.383 | INFO | "pinned" : " 29 GB" | 20260505 16:57:24.383 | INFO | } | 20260505 16:57:24.383 | INFO | Perf report for the last 5.13878 seconds: | 20260505 16:57:24.383 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:57:24.383 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:24.383 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:24.383 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:24.383 | INFO | ANALYSIS : 63.9%, 3.0, 99.5ms, 0 | 20260505 16:57:24.383 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:24.383 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260505 16:57:24.383 | INFO | PRE_HQ : 60.6%, 1.8, 56.3ms, 0 | 20260505 16:57:24.383 | INFO | BAZWRITER : 63.5%, 3.9, 125.6ms, 0 | 20260505 16:57:24.383 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260505 16:57:24.383 | INFO | Uploaded 36GiB to GPU in 2.18782s (16.4547GiB/s) | 20260505 16:57:24.383 | INFO | Downloaded 38GiB to GPU in 2.26528s (16.775GiB/s) | 20260505 16:57:29.524 | INFO | Analyzing chunk frames = [4817408,4817920) | 20260505 16:57:29.524 | INFO | MemInfo: { | 20260505 16:57:29.524 | INFO | "gpu" : " 17 GB", | 20260505 16:57:29.524 | INFO | "huge" : "", | 20260505 16:57:29.524 | INFO | "huge_shared" : " 27 GB", | 20260505 16:57:29.524 | INFO | "malloc" : " 14 GB", | 20260505 16:57:29.524 | INFO | "pinned" : " 29 GB" | 20260505 16:57:29.524 | INFO | } | 20260505 16:57:29.524 | INFO | Perf report for the last 5.14144 seconds: | 20260505 16:57:29.524 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:57:29.524 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:29.524 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:29.524 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:29.524 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 16:57:29.524 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:29.524 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:29.524 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:57:29.524 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:29.524 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:29.525 | INFO | Uploaded 36GiB to GPU in 2.14197s (16.807GiB/s) | 20260505 16:57:29.525 | INFO | Downloaded 34GiB to GPU in 2.038s (16.683GiB/s) | 20260505 16:57:34.668 | INFO | Analyzing chunk frames = [4817920,4818432) | 20260505 16:57:34.668 | INFO | MemInfo: { | 20260505 16:57:34.668 | INFO | "gpu" : " 17 GB", | 20260505 16:57:34.668 | INFO | "huge" : "", | 20260505 16:57:34.668 | INFO | "huge_shared" : " 27 GB", | 20260505 16:57:34.668 | INFO | "malloc" : " 14 GB", | 20260505 16:57:34.668 | INFO | "pinned" : " 29 GB" | 20260505 16:57:34.668 | INFO | } | 20260505 16:57:34.669 | INFO | Perf report for the last 5.14439 seconds: | 20260505 16:57:34.669 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:57:34.669 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:34.669 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:34.669 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:34.669 | INFO | ANALYSIS : 62.8%, 3.0, 97.8ms, 0 | 20260505 16:57:34.669 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:34.669 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:34.669 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 16:57:34.669 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:34.669 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:34.669 | INFO | Uploaded 36GiB to GPU in 2.20523s (16.3248GiB/s) | 20260505 16:57:34.669 | INFO | Downloaded 34GiB to GPU in 2.04144s (16.6549GiB/s) | 20260505 16:57:39.804 | INFO | Analyzing chunk frames = [4818432,4818944) | 20260505 16:57:39.804 | INFO | MemInfo: { | 20260505 16:57:39.804 | INFO | "gpu" : " 17 GB", | 20260505 16:57:39.804 | INFO | "huge" : "", | 20260505 16:57:39.804 | INFO | "huge_shared" : " 27 GB", | 20260505 16:57:39.804 | INFO | "malloc" : " 14 GB", | 20260505 16:57:39.804 | INFO | "pinned" : " 29 GB" | 20260505 16:57:39.804 | INFO | } | 20260505 16:57:39.804 | INFO | Perf report for the last 5.13591 seconds: | 20260505 16:57:39.804 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:57:39.804 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:39.804 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:39.805 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:39.805 | INFO | ANALYSIS : 62.9%, 3.0, 97.7ms, 0 | 20260505 16:57:39.805 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:39.805 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:39.805 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 16:57:39.805 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:39.805 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:39.805 | INFO | Uploaded 36GiB to GPU in 2.2271s (16.1645GiB/s) | 20260505 16:57:39.805 | INFO | Downloaded 34GiB to GPU in 2.12155s (16.026GiB/s) | 20260505 16:57:44.934 | INFO | Analyzing chunk frames = [4818944,4819456) | 20260505 16:57:44.934 | INFO | MemInfo: { | 20260505 16:57:44.934 | INFO | "gpu" : " 17 GB", | 20260505 16:57:44.935 | INFO | "huge" : "", | 20260505 16:57:44.935 | INFO | "huge_shared" : " 27 GB", | 20260505 16:57:44.935 | INFO | "malloc" : " 14 GB", | 20260505 16:57:44.935 | INFO | "pinned" : " 29 GB" | 20260505 16:57:44.935 | INFO | } | 20260505 16:57:44.935 | INFO | Perf report for the last 5.13025 seconds: | 20260505 16:57:44.935 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:57:44.935 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:44.935 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:44.935 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:44.935 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260505 16:57:44.935 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:44.935 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:44.935 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:57:44.935 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:44.935 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:44.935 | INFO | Uploaded 36GiB to GPU in 2.1438s (16.7926GiB/s) | 20260505 16:57:44.935 | INFO | Downloaded 34GiB to GPU in 2.05824s (16.519GiB/s) | 20260505 16:57:50.080 | INFO | Analyzing chunk frames = [4819456,4819968) | 20260505 16:57:50.080 | INFO | MemInfo: { | 20260505 16:57:50.080 | INFO | "gpu" : " 17 GB", | 20260505 16:57:50.080 | INFO | "huge" : "", | 20260505 16:57:50.080 | INFO | "huge_shared" : " 27 GB", | 20260505 16:57:50.080 | INFO | "malloc" : " 14 GB", | 20260505 16:57:50.080 | INFO | "pinned" : " 29 GB" | 20260505 16:57:50.080 | INFO | } | 20260505 16:57:50.080 | INFO | Perf report for the last 5.14551 seconds: | 20260505 16:57:50.080 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:57:50.080 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:50.080 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:50.080 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:50.080 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 16:57:50.080 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:50.080 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:50.080 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 16:57:50.080 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:50.080 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:50.080 | INFO | Uploaded 36GiB to GPU in 2.20156s (16.352GiB/s) | 20260505 16:57:50.080 | INFO | Downloaded 34GiB to GPU in 2.11473s (16.0777GiB/s) | 20260505 16:57:55.223 | INFO | Analyzing chunk frames = [4819968,4820480) | 20260505 16:57:55.223 | INFO | MemInfo: { | 20260505 16:57:55.223 | INFO | "gpu" : " 17 GB", | 20260505 16:57:55.223 | INFO | "huge" : "", | 20260505 16:57:55.223 | INFO | "huge_shared" : " 27 GB", | 20260505 16:57:55.223 | INFO | "malloc" : " 14 GB", | 20260505 16:57:55.223 | INFO | "pinned" : " 29 GB" | 20260505 16:57:55.223 | INFO | } | 20260505 16:57:55.223 | INFO | Perf report for the last 5.14276 seconds: | 20260505 16:57:55.223 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:57:55.223 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:55.223 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:55.223 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:55.223 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 16:57:55.223 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:55.223 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:55.223 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 16:57:55.223 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:55.223 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:57:55.223 | INFO | Uploaded 36GiB to GPU in 2.15128s (16.7342GiB/s) | 20260505 16:57:55.223 | INFO | Downloaded 34GiB to GPU in 2.17686s (15.6188GiB/s) | 20260505 16:58:00.369 | INFO | Analyzing chunk frames = [4820480,4820992) | 20260505 16:58:00.369 | INFO | MemInfo: { | 20260505 16:58:00.369 | INFO | "gpu" : " 17 GB", | 20260505 16:58:00.370 | INFO | "huge" : "", | 20260505 16:58:00.370 | INFO | "huge_shared" : " 27 GB", | 20260505 16:58:00.370 | INFO | "malloc" : " 14 GB", | 20260505 16:58:00.370 | INFO | "pinned" : " 29 GB" | 20260505 16:58:00.370 | INFO | } | 20260505 16:58:00.370 | INFO | Perf report for the last 5.14667 seconds: | 20260505 16:58:00.370 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:58:00.370 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:00.370 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:00.370 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:00.370 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 16:58:00.370 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:00.370 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:00.370 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 16:58:00.370 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:00.370 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:00.370 | INFO | Uploaded 36GiB to GPU in 2.18408s (16.4829GiB/s) | 20260505 16:58:00.370 | INFO | Downloaded 34GiB to GPU in 2.02256s (16.8104GiB/s) | 20260505 16:58:00.504 | INFO | Starting a round of Baz IO | 20260505 16:58:03.683 | INFO | Populating RTMetrics report | 20260505 16:58:03.683 | INFO | RTMetrics report populated | 20260505 16:58:03.683 | INFO | Opening tmp RTMetrics json file | 20260505 16:58:03.693 | INFO | Serializing RTMetrics to json | 20260505 16:58:03.695 | INFO | Serialization done, swapping tmp file now... | 20260505 16:58:03.696 | INFO | RTMetrics done | 20260505 16:58:03.777 | INFO | Finished a round of Baz IO: Wrote 2.153 GiB in 3.273 seconds | 20260505 16:58:03.777 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.14% / 0.00% / 77.85% | 20260505 16:58:03.777 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.330961% / 0% / 0.000115236% | 20260505 16:58:03.777 | INFO | Baz IO overall: | 20260505 16:58:03.777 | INFO | BytesWritten : 2855613409428 | 20260505 16:58:03.777 | INFO | NumEvents : 1354764101391 | 20260505 16:58:05.504 | INFO | Analyzing chunk frames = [4820992,4821504) | 20260505 16:58:05.504 | INFO | MemInfo: { | 20260505 16:58:05.504 | INFO | "gpu" : " 17 GB", | 20260505 16:58:05.504 | INFO | "huge" : "", | 20260505 16:58:05.504 | INFO | "huge_shared" : " 27 GB", | 20260505 16:58:05.504 | INFO | "malloc" : " 14 GB", | 20260505 16:58:05.504 | INFO | "pinned" : " 29 GB" | 20260505 16:58:05.504 | INFO | } | 20260505 16:58:05.505 | INFO | Perf report for the last 5.13483 seconds: | 20260505 16:58:05.505 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:58:05.505 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:05.505 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:05.505 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:05.505 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 16:58:05.505 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:05.505 | INFO | RT_METRICS : 45.1%, 1.0, 23.6ms, 0 | 20260505 16:58:05.505 | INFO | PRE_HQ : 60.7%, 1.8, 56.8ms, 0 | 20260505 16:58:05.505 | INFO | BAZWRITER : 63.2%, 3.9, 125.9ms, 0 | 20260505 16:58:05.505 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 16:58:05.505 | INFO | Uploaded 36GiB to GPU in 2.15923s (16.6726GiB/s) | 20260505 16:58:05.505 | INFO | Downloaded 38GiB to GPU in 2.23424s (17.008GiB/s) | 20260505 16:58:10.634 | INFO | Analyzing chunk frames = [4821504,4822016) | 20260505 16:58:10.635 | INFO | MemInfo: { | 20260505 16:58:10.635 | INFO | "gpu" : " 17 GB", | 20260505 16:58:10.635 | INFO | "huge" : "", | 20260505 16:58:10.635 | INFO | "huge_shared" : " 27 GB", | 20260505 16:58:10.635 | INFO | "malloc" : " 14 GB", | 20260505 16:58:10.635 | INFO | "pinned" : " 29 GB" | 20260505 16:58:10.635 | INFO | } | 20260505 16:58:10.635 | INFO | Perf report for the last 5.13025 seconds: | 20260505 16:58:10.635 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:58:10.635 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:10.635 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:10.635 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:10.635 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 16:58:10.635 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:10.635 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:10.635 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 16:58:10.635 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:10.635 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:10.635 | INFO | Uploaded 36GiB to GPU in 2.12409s (16.9484GiB/s) | 20260505 16:58:10.635 | INFO | Downloaded 34GiB to GPU in 2.03276s (16.726GiB/s) | 20260505 16:58:15.777 | INFO | Analyzing chunk frames = [4822016,4822528) | 20260505 16:58:15.777 | INFO | MemInfo: { | 20260505 16:58:15.777 | INFO | "gpu" : " 17 GB", | 20260505 16:58:15.777 | INFO | "huge" : "", | 20260505 16:58:15.777 | INFO | "huge_shared" : " 27 GB", | 20260505 16:58:15.777 | INFO | "malloc" : " 14 GB", | 20260505 16:58:15.777 | INFO | "pinned" : " 29 GB" | 20260505 16:58:15.777 | INFO | } | 20260505 16:58:15.777 | INFO | Perf report for the last 5.14202 seconds: | 20260505 16:58:15.777 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:58:15.777 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:15.777 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:15.777 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:15.777 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260505 16:58:15.777 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:15.777 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:15.777 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 16:58:15.777 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:15.777 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:15.777 | INFO | Uploaded 36GiB to GPU in 2.17817s (16.5276GiB/s) | 20260505 16:58:15.777 | INFO | Downloaded 34GiB to GPU in 2.04037s (16.6637GiB/s) | 20260505 16:58:20.919 | INFO | Analyzing chunk frames = [4822528,4823040) | 20260505 16:58:20.919 | INFO | MemInfo: { | 20260505 16:58:20.919 | INFO | "gpu" : " 17 GB", | 20260505 16:58:20.919 | INFO | "huge" : "", | 20260505 16:58:20.919 | INFO | "huge_shared" : " 27 GB", | 20260505 16:58:20.919 | INFO | "malloc" : " 14 GB", | 20260505 16:58:20.919 | INFO | "pinned" : " 29 GB" | 20260505 16:58:20.919 | INFO | } | 20260505 16:58:20.919 | INFO | Perf report for the last 5.14236 seconds: | 20260505 16:58:20.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:58:20.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:20.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:20.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:20.919 | INFO | ANALYSIS : 62.6%, 3.0, 97.3ms, 0 | 20260505 16:58:20.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:20.919 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:20.919 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:58:20.919 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:20.919 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:20.919 | INFO | Uploaded 36GiB to GPU in 2.16621s (16.6189GiB/s) | 20260505 16:58:20.920 | INFO | Downloaded 34GiB to GPU in 2.14197s (15.8732GiB/s) | 20260505 16:58:26.060 | INFO | Analyzing chunk frames = [4823040,4823552) | 20260505 16:58:26.060 | INFO | MemInfo: { | 20260505 16:58:26.060 | INFO | "gpu" : " 17 GB", | 20260505 16:58:26.060 | INFO | "huge" : "", | 20260505 16:58:26.060 | INFO | "huge_shared" : " 27 GB", | 20260505 16:58:26.061 | INFO | "malloc" : " 14 GB", | 20260505 16:58:26.061 | INFO | "pinned" : " 29 GB" | 20260505 16:58:26.061 | INFO | } | 20260505 16:58:26.061 | INFO | Perf report for the last 5.14148 seconds: | 20260505 16:58:26.061 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:58:26.061 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:26.061 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:26.061 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:26.061 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 16:58:26.061 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:26.061 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:26.061 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 16:58:26.061 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:26.061 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:26.061 | INFO | Uploaded 36GiB to GPU in 2.15419s (16.7116GiB/s) | 20260505 16:58:26.061 | INFO | Downloaded 34GiB to GPU in 2.06124s (16.4949GiB/s) | 20260505 16:58:31.195 | INFO | Analyzing chunk frames = [4823552,4824064) | 20260505 16:58:31.195 | INFO | MemInfo: { | 20260505 16:58:31.195 | INFO | "gpu" : " 17 GB", | 20260505 16:58:31.195 | INFO | "huge" : "", | 20260505 16:58:31.195 | INFO | "huge_shared" : " 27 GB", | 20260505 16:58:31.195 | INFO | "malloc" : " 14 GB", | 20260505 16:58:31.195 | INFO | "pinned" : " 29 GB" | 20260505 16:58:31.195 | INFO | } | 20260505 16:58:31.195 | INFO | Perf report for the last 5.13431 seconds: | 20260505 16:58:31.195 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:58:31.195 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:31.195 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:31.195 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:31.195 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260505 16:58:31.195 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:31.195 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:31.195 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 16:58:31.195 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:31.195 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:31.195 | INFO | Uploaded 36GiB to GPU in 2.13881s (16.8318GiB/s) | 20260505 16:58:31.195 | INFO | Downloaded 34GiB to GPU in 2.03778s (16.6848GiB/s) | 20260505 16:58:36.331 | INFO | Analyzing chunk frames = [4824064,4824576) | 20260505 16:58:36.331 | INFO | MemInfo: { | 20260505 16:58:36.331 | INFO | "gpu" : " 17 GB", | 20260505 16:58:36.331 | INFO | "huge" : "", | 20260505 16:58:36.331 | INFO | "huge_shared" : " 27 GB", | 20260505 16:58:36.331 | INFO | "malloc" : " 14 GB", | 20260505 16:58:36.331 | INFO | "pinned" : " 29 GB" | 20260505 16:58:36.331 | INFO | } | 20260505 16:58:36.331 | INFO | Perf report for the last 5.13635 seconds: | 20260505 16:58:36.331 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:58:36.331 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:36.331 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:36.331 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:36.331 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 16:58:36.331 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:36.331 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:36.331 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 16:58:36.331 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:36.332 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:36.332 | INFO | Uploaded 36GiB to GPU in 2.13394s (16.8702GiB/s) | 20260505 16:58:36.332 | INFO | Downloaded 34GiB to GPU in 2.16856s (15.6786GiB/s) | 20260505 16:58:41.482 | INFO | Analyzing chunk frames = [4824576,4825088) | 20260505 16:58:41.482 | INFO | MemInfo: { | 20260505 16:58:41.482 | INFO | "gpu" : " 17 GB", | 20260505 16:58:41.482 | INFO | "huge" : "", | 20260505 16:58:41.482 | INFO | "huge_shared" : " 27 GB", | 20260505 16:58:41.482 | INFO | "malloc" : " 14 GB", | 20260505 16:58:41.482 | INFO | "pinned" : " 29 GB" | 20260505 16:58:41.482 | INFO | } | 20260505 16:58:41.482 | INFO | Perf report for the last 5.15067 seconds: | 20260505 16:58:41.482 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:58:41.482 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:41.482 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:41.482 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:41.482 | INFO | ANALYSIS : 63.0%, 3.0, 98.6ms, 0 | 20260505 16:58:41.482 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:41.482 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:41.482 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 16:58:41.482 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:41.482 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:41.482 | INFO | Uploaded 36GiB to GPU in 2.15995s (16.6671GiB/s) | 20260505 16:58:41.482 | INFO | Downloaded 34GiB to GPU in 2.03399s (16.7159GiB/s) | 20260505 16:58:41.483 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 16:58:41.483 | INFO | Total mem usage: 15.3 GiB | 20260505 16:58:41.483 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 16:58:41.483 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.91%) | 20260505 16:58:41.483 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 16:58:41.483 | INFO | High Water Breakdown: | 20260505 16:58:41.483 | INFO | 61% : 8.67 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 16:58:41.483 | INFO | 38.8% : 5.51 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 16:58:41.483 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 16:58:41.483 | INFO | ------------------------------------------ | 20260505 16:58:41.483 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:58:41.483 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 16:58:41.483 | INFO | Total mem usage: 26.5 GiB | 20260505 16:58:41.483 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 16:58:41.483 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 16:58:41.483 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 16:58:41.483 | INFO | High Water Breakdown: | 20260505 16:58:41.483 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:58:41.483 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:58:41.483 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:58:41.483 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:58:41.483 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:58:41.483 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:58:41.483 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 16:58:41.483 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 16:58:41.483 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:58:41.483 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 16:58:41.483 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 16:58:41.483 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 16:58:41.483 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 16:58:41.483 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 16:58:41.483 | INFO | ------------------------------------------ | 20260505 16:58:41.483 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:58:41.483 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 16:58:41.483 | INFO | Total mem usage: 25.6 GiB | 20260505 16:58:41.483 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 16:58:41.483 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 16:58:41.483 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 16:58:41.483 | INFO | High Water Breakdown: | 20260505 16:58:41.483 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 16:58:41.483 | INFO | ------------------------------------------ | 20260505 16:58:41.483 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:58:41.483 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 16:58:41.483 | INFO | Total mem usage: 15.7 GiB | 20260505 16:58:41.483 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 16:58:41.483 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 16:58:41.483 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 16:58:41.483 | INFO | High Water Breakdown: | 20260505 16:58:41.483 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 16:58:41.483 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 16:58:41.483 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 16:58:41.483 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 16:58:41.483 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 16:58:41.483 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 16:58:41.483 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 16:58:41.483 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 16:58:41.483 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 16:58:41.483 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 16:58:41.483 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 16:58:41.483 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 16:58:41.483 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 16:58:41.483 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 16:58:41.483 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 16:58:41.483 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 16:58:41.483 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 16:58:41.483 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 16:58:41.483 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 16:58:41.483 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 16:58:41.483 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 16:58:41.483 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 16:58:41.483 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 16:58:41.483 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 16:58:41.483 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 16:58:41.483 | INFO | ------------------------------------------ | 20260505 16:58:41.483 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 16:58:41.484 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 16:58:41.484 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:58:41.484 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:58:41.484 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.51e+01, 1.35e+01 / 2.14e+00 | 20260505 16:58:41.484 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 1.02e-01, 1.04e-02 / 5.14e-03 | 20260505 16:58:41.484 | INFO | PerfReport: 11.77%, Baseline -- 3.73e+00 / 4.48e+00, 5.67e-01 / 2.15e-01 | 20260505 16:58:41.484 | INFO | PerfReport: 3.27%, Histogram -- 1.04e+00 / 1.51e+00, 1.93e-01 / 1.05e-01 | 20260505 16:58:41.484 | INFO | PerfReport: 15.95%, DME -- 4.04e+01 / 4.49e+01, 4.27e+00 / 2.28e+00 | 20260505 16:58:41.484 | INFO | PerfReport: 56.82%, FrameLabeling -- 1.80e+01 / 1.93e+01, 1.50e+00 / 1.41e+00 | 20260505 16:58:41.484 | INFO | PerfReport: 7.34%, PulseAccumulating -- 2.32e+00 / 2.92e+00, 4.49e-01 / 2.09e-01 | 20260505 16:58:41.484 | INFO | PerfReport: 3.37%, Metrics -- 1.07e+00 / 1.40e+00, 2.91e-01 / 1.56e-01 | 20260505 16:58:41.484 | INFO | PerfReport: 1.18%, PulseSerializing -- 3.75e-01 / 4.32e-01, 6.72e-02 / 3.82e-02 | 20260505 16:58:41.484 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 16:58:41.484 | INFO | PerfReport: Gathered 1536 samples. | 20260505 16:58:41.484 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 16:58:41.484 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.56e+01 / 0.00e+00 | 20260505 16:58:41.484 | INFO | PerfReport: 22.52%, Upload -- 2.26e+01 / 0.00e+00, 7.88e+00 / 0.00e+00 | 20260505 16:58:41.484 | INFO | PerfReport: 21.83%, Download -- 2.19e+01 / 0.00e+00, 7.81e+00 / 0.00e+00 | 20260505 16:58:41.484 | INFO | PerfReport: 31.61%, Compute -- 3.17e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 16:58:41.484 | INFO | PerfReport: 2.38%, LockUpload -- 2.39e+00 / 0.00e+00, 8.03e+00 / 0.00e+00 | 20260505 16:58:41.484 | INFO | PerfReport: 1.38%, LockDownload -- 1.38e+00 / 0.00e+00, 2.92e+00 / 0.00e+00 | 20260505 16:58:41.484 | INFO | PerfReport: 20.26%, LockCompute -- 2.03e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260505 16:58:41.484 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.56e-02 / 0.00e+00 | 20260505 16:58:41.614 | INFO | Starting a round of Baz IO | 20260505 16:58:44.794 | INFO | Populating RTMetrics report | 20260505 16:58:44.794 | INFO | RTMetrics report populated | 20260505 16:58:44.794 | INFO | Opening tmp RTMetrics json file | 20260505 16:58:44.810 | INFO | Serializing RTMetrics to json | 20260505 16:58:44.832 | INFO | Serialization done, swapping tmp file now... | 20260505 16:58:44.864 | INFO | RTMetrics done | 20260505 16:58:44.885 | INFO | Finished a round of Baz IO: Wrote 2.153 GiB in 3.271 seconds | 20260505 16:58:44.885 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.14% / 0.00% / 77.86% | 20260505 16:58:44.885 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.330671% / 3.97364e-06% / 0.000115236% | 20260505 16:58:44.885 | INFO | Baz IO overall: | 20260505 16:58:44.885 | INFO | BytesWritten : 2857924779021 | 20260505 16:58:44.885 | INFO | NumEvents : 1355845757679 | 20260505 16:58:46.616 | INFO | Analyzing chunk frames = [4825088,4825600) | 20260505 16:58:46.616 | INFO | MemInfo: { | 20260505 16:58:46.616 | INFO | "gpu" : " 17 GB", | 20260505 16:58:46.616 | INFO | "huge" : "", | 20260505 16:58:46.616 | INFO | "huge_shared" : " 27 GB", | 20260505 16:58:46.616 | INFO | "malloc" : " 14 GB", | 20260505 16:58:46.616 | INFO | "pinned" : " 29 GB" | 20260505 16:58:46.616 | INFO | } | 20260505 16:58:46.616 | INFO | Perf report for the last 5.1339 seconds: | 20260505 16:58:46.616 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:58:46.616 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:46.616 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:46.616 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:46.616 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 16:58:46.616 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:46.616 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 16:58:46.616 | INFO | PRE_HQ : 60.5%, 1.8, 56.8ms, 0 | 20260505 16:58:46.616 | INFO | BAZWRITER : 63.0%, 3.9, 124.7ms, 0 | 20260505 16:58:46.616 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 16:58:46.616 | INFO | Uploaded 36GiB to GPU in 2.16513s (16.6272GiB/s) | 20260505 16:58:46.616 | INFO | Downloaded 38GiB to GPU in 2.23845s (16.976GiB/s) | 20260505 16:58:51.748 | INFO | Analyzing chunk frames = [4825600,4826112) | 20260505 16:58:51.748 | INFO | MemInfo: { | 20260505 16:58:51.748 | INFO | "gpu" : " 17 GB", | 20260505 16:58:51.748 | INFO | "huge" : "", | 20260505 16:58:51.748 | INFO | "huge_shared" : " 27 GB", | 20260505 16:58:51.748 | INFO | "malloc" : " 14 GB", | 20260505 16:58:51.748 | INFO | "pinned" : " 29 GB" | 20260505 16:58:51.748 | INFO | } | 20260505 16:58:51.748 | INFO | Perf report for the last 5.13234 seconds: | 20260505 16:58:51.748 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:58:51.748 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:51.748 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:51.748 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:51.748 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 16:58:51.748 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:51.748 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:51.748 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 16:58:51.748 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:51.748 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:51.749 | INFO | Uploaded 36GiB to GPU in 2.13722s (16.8443GiB/s) | 20260505 16:58:51.749 | INFO | Downloaded 34GiB to GPU in 2.02874s (16.7592GiB/s) | 20260505 16:58:56.887 | INFO | Analyzing chunk frames = [4826112,4826624) | 20260505 16:58:56.887 | INFO | MemInfo: { | 20260505 16:58:56.887 | INFO | "gpu" : " 17 GB", | 20260505 16:58:56.887 | INFO | "huge" : "", | 20260505 16:58:56.887 | INFO | "huge_shared" : " 27 GB", | 20260505 16:58:56.887 | INFO | "malloc" : " 14 GB", | 20260505 16:58:56.887 | INFO | "pinned" : " 29 GB" | 20260505 16:58:56.887 | INFO | } | 20260505 16:58:56.887 | INFO | Perf report for the last 5.1391 seconds: | 20260505 16:58:56.887 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:58:56.887 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:56.887 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:56.887 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:56.887 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260505 16:58:56.888 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:56.888 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:56.888 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 16:58:56.888 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:56.888 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:58:56.888 | INFO | Uploaded 36GiB to GPU in 2.19219s (16.4219GiB/s) | 20260505 16:58:56.888 | INFO | Downloaded 34GiB to GPU in 2.03299s (16.7241GiB/s) | 20260505 16:59:02.032 | INFO | Analyzing chunk frames = [4826624,4827136) | 20260505 16:59:02.032 | INFO | MemInfo: { | 20260505 16:59:02.032 | INFO | "gpu" : " 17 GB", | 20260505 16:59:02.032 | INFO | "huge" : "", | 20260505 16:59:02.032 | INFO | "huge_shared" : " 27 GB", | 20260505 16:59:02.032 | INFO | "malloc" : " 14 GB", | 20260505 16:59:02.032 | INFO | "pinned" : " 29 GB" | 20260505 16:59:02.032 | INFO | } | 20260505 16:59:02.032 | INFO | Perf report for the last 5.14453 seconds: | 20260505 16:59:02.032 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:59:02.032 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:02.032 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:02.032 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:02.032 | INFO | ANALYSIS : 62.6%, 3.0, 97.4ms, 0 | 20260505 16:59:02.032 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:02.032 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:02.032 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 16:59:02.032 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:02.032 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:02.032 | INFO | Uploaded 36GiB to GPU in 2.19219s (16.4219GiB/s) | 20260505 16:59:02.032 | INFO | Downloaded 34GiB to GPU in 2.13101s (15.9549GiB/s) | 20260505 16:59:07.175 | INFO | Analyzing chunk frames = [4827136,4827648) | 20260505 16:59:07.175 | INFO | MemInfo: { | 20260505 16:59:07.175 | INFO | "gpu" : " 17 GB", | 20260505 16:59:07.175 | INFO | "huge" : "", | 20260505 16:59:07.175 | INFO | "huge_shared" : " 27 GB", | 20260505 16:59:07.175 | INFO | "malloc" : " 14 GB", | 20260505 16:59:07.175 | INFO | "pinned" : " 29 GB" | 20260505 16:59:07.175 | INFO | } | 20260505 16:59:07.175 | INFO | Perf report for the last 5.14335 seconds: | 20260505 16:59:07.175 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:59:07.175 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:07.175 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:07.175 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:07.175 | INFO | ANALYSIS : 62.1%, 3.0, 97.1ms, 0 | 20260505 16:59:07.175 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:07.175 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:07.176 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:59:07.176 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:07.176 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:07.176 | INFO | Uploaded 36GiB to GPU in 2.15487s (16.7064GiB/s) | 20260505 16:59:07.176 | INFO | Downloaded 34GiB to GPU in 2.05787s (16.5219GiB/s) | 20260505 16:59:12.315 | INFO | Analyzing chunk frames = [4827648,4828160) | 20260505 16:59:12.315 | INFO | MemInfo: { | 20260505 16:59:12.315 | INFO | "gpu" : " 17 GB", | 20260505 16:59:12.315 | INFO | "huge" : "", | 20260505 16:59:12.315 | INFO | "huge_shared" : " 27 GB", | 20260505 16:59:12.315 | INFO | "malloc" : " 14 GB", | 20260505 16:59:12.315 | INFO | "pinned" : " 29 GB" | 20260505 16:59:12.316 | INFO | } | 20260505 16:59:12.316 | INFO | Perf report for the last 5.14022 seconds: | 20260505 16:59:12.316 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:59:12.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:12.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:12.316 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:12.316 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260505 16:59:12.316 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:12.316 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:12.316 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 16:59:12.316 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:12.316 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:12.316 | INFO | Uploaded 36GiB to GPU in 2.13822s (16.8364GiB/s) | 20260505 16:59:12.316 | INFO | Downloaded 34GiB to GPU in 2.04341s (16.6388GiB/s) | 20260505 16:59:17.449 | INFO | Analyzing chunk frames = [4828160,4828672) | 20260505 16:59:17.449 | INFO | MemInfo: { | 20260505 16:59:17.449 | INFO | "gpu" : " 17 GB", | 20260505 16:59:17.449 | INFO | "huge" : "", | 20260505 16:59:17.449 | INFO | "huge_shared" : " 27 GB", | 20260505 16:59:17.449 | INFO | "malloc" : " 14 GB", | 20260505 16:59:17.449 | INFO | "pinned" : " 29 GB" | 20260505 16:59:17.449 | INFO | } | 20260505 16:59:17.449 | INFO | Perf report for the last 5.13339 seconds: | 20260505 16:59:17.449 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:59:17.449 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:17.449 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:17.449 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:17.449 | INFO | ANALYSIS : 63.5%, 3.0, 99.0ms, 0 | 20260505 16:59:17.449 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:17.449 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:17.449 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 16:59:17.449 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:17.449 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:17.449 | INFO | Uploaded 36GiB to GPU in 2.12885s (16.9105GiB/s) | 20260505 16:59:17.449 | INFO | Downloaded 34GiB to GPU in 2.14188s (15.8739GiB/s) | 20260505 16:59:22.588 | INFO | Analyzing chunk frames = [4828672,4829184) | 20260505 16:59:22.588 | INFO | MemInfo: { | 20260505 16:59:22.588 | INFO | "gpu" : " 17 GB", | 20260505 16:59:22.588 | INFO | "huge" : "", | 20260505 16:59:22.588 | INFO | "huge_shared" : " 27 GB", | 20260505 16:59:22.588 | INFO | "malloc" : " 14 GB", | 20260505 16:59:22.588 | INFO | "pinned" : " 29 GB" | 20260505 16:59:22.588 | INFO | } | 20260505 16:59:22.588 | INFO | Perf report for the last 5.13888 seconds: | 20260505 16:59:22.588 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:59:22.588 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:22.588 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:22.588 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:22.588 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 16:59:22.588 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:22.588 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:22.588 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 16:59:22.588 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:22.588 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:22.588 | INFO | Uploaded 36GiB to GPU in 2.22581s (16.1739GiB/s) | 20260505 16:59:22.588 | INFO | Downloaded 34GiB to GPU in 2.10648s (16.1407GiB/s) | 20260505 16:59:22.734 | INFO | Starting a round of Baz IO | 20260505 16:59:25.935 | INFO | Populating RTMetrics report | 20260505 16:59:25.935 | INFO | RTMetrics report populated | 20260505 16:59:25.935 | INFO | Opening tmp RTMetrics json file | 20260505 16:59:25.938 | INFO | Serializing RTMetrics to json | 20260505 16:59:25.940 | INFO | Serialization done, swapping tmp file now... | 20260505 16:59:25.941 | INFO | RTMetrics done | 20260505 16:59:26.027 | INFO | Finished a round of Baz IO: Wrote 2.152 GiB in 3.292 seconds | 20260505 16:59:26.027 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.14% / 0.00% / 77.86% | 20260505 16:59:26.027 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.329705% / 0% / 0.000119209% | 20260505 16:59:26.027 | INFO | Baz IO overall: | 20260505 16:59:26.027 | INFO | BytesWritten : 2860235670745 | 20260505 16:59:26.027 | INFO | NumEvents : 1356927078661 | 20260505 16:59:27.736 | INFO | Analyzing chunk frames = [4829184,4829696) | 20260505 16:59:27.736 | INFO | MemInfo: { | 20260505 16:59:27.736 | INFO | "gpu" : " 17 GB", | 20260505 16:59:27.736 | INFO | "huge" : "", | 20260505 16:59:27.736 | INFO | "huge_shared" : " 27 GB", | 20260505 16:59:27.736 | INFO | "malloc" : " 14 GB", | 20260505 16:59:27.736 | INFO | "pinned" : " 29 GB" | 20260505 16:59:27.736 | INFO | } | 20260505 16:59:27.736 | INFO | Perf report for the last 5.1481 seconds: | 20260505 16:59:27.736 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:59:27.736 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:27.736 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:27.736 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:27.736 | INFO | ANALYSIS : 63.8%, 3.0, 99.7ms, 0 | 20260505 16:59:27.736 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:27.736 | INFO | RT_METRICS : 44.7%, 1.0, 23.5ms, 0 | 20260505 16:59:27.736 | INFO | PRE_HQ : 60.6%, 1.8, 56.7ms, 0 | 20260505 16:59:27.736 | INFO | BAZWRITER : 63.8%, 3.8, 125.5ms, 0 | 20260505 16:59:27.736 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 16:59:27.736 | INFO | Uploaded 36GiB to GPU in 2.2015s (16.3525GiB/s) | 20260505 16:59:27.736 | INFO | Downloaded 38GiB to GPU in 2.2615s (16.803GiB/s) | 20260505 16:59:32.877 | INFO | Analyzing chunk frames = [4829696,4830208) | 20260505 16:59:32.877 | INFO | MemInfo: { | 20260505 16:59:32.877 | INFO | "gpu" : " 17 GB", | 20260505 16:59:32.877 | INFO | "huge" : "", | 20260505 16:59:32.877 | INFO | "huge_shared" : " 27 GB", | 20260505 16:59:32.877 | INFO | "malloc" : " 14 GB", | 20260505 16:59:32.877 | INFO | "pinned" : " 29 GB" | 20260505 16:59:32.877 | INFO | } | 20260505 16:59:32.878 | INFO | Perf report for the last 5.14156 seconds: | 20260505 16:59:32.878 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:59:32.878 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:32.878 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:32.878 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:32.878 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 16:59:32.878 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:32.878 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:32.878 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 16:59:32.878 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:32.878 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:32.878 | INFO | Uploaded 36GiB to GPU in 2.13802s (16.838GiB/s) | 20260505 16:59:32.878 | INFO | Downloaded 34GiB to GPU in 2.02891s (16.7578GiB/s) | 20260505 16:59:38.009 | INFO | Analyzing chunk frames = [4830208,4830720) | 20260505 16:59:38.010 | INFO | MemInfo: { | 20260505 16:59:38.010 | INFO | "gpu" : " 17 GB", | 20260505 16:59:38.010 | INFO | "huge" : "", | 20260505 16:59:38.010 | INFO | "huge_shared" : " 27 GB", | 20260505 16:59:38.010 | INFO | "malloc" : " 14 GB", | 20260505 16:59:38.010 | INFO | "pinned" : " 29 GB" | 20260505 16:59:38.010 | INFO | } | 20260505 16:59:38.010 | INFO | Perf report for the last 5.13221 seconds: | 20260505 16:59:38.010 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:59:38.010 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:38.010 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:38.010 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:38.010 | INFO | ANALYSIS : 62.8%, 3.0, 97.6ms, 0 | 20260505 16:59:38.010 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:38.010 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:38.010 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 16:59:38.010 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:38.010 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:38.010 | INFO | Uploaded 36GiB to GPU in 2.1998s (16.3651GiB/s) | 20260505 16:59:38.010 | INFO | Downloaded 34GiB to GPU in 2.03428s (16.7135GiB/s) | 20260505 16:59:43.145 | INFO | Analyzing chunk frames = [4830720,4831232) | 20260505 16:59:43.145 | INFO | MemInfo: { | 20260505 16:59:43.145 | INFO | "gpu" : " 17 GB", | 20260505 16:59:43.145 | INFO | "huge" : "", | 20260505 16:59:43.145 | INFO | "huge_shared" : " 27 GB", | 20260505 16:59:43.145 | INFO | "malloc" : " 14 GB", | 20260505 16:59:43.145 | INFO | "pinned" : " 29 GB" | 20260505 16:59:43.145 | INFO | } | 20260505 16:59:43.145 | INFO | Perf report for the last 5.13533 seconds: | 20260505 16:59:43.145 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:59:43.145 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:43.145 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:43.145 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:43.145 | INFO | ANALYSIS : 62.6%, 3.0, 97.2ms, 0 | 20260505 16:59:43.145 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:43.145 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:43.145 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 16:59:43.145 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:43.145 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:43.145 | INFO | Uploaded 36GiB to GPU in 2.19903s (16.3708GiB/s) | 20260505 16:59:43.145 | INFO | Downloaded 34GiB to GPU in 2.13706s (15.9097GiB/s) | 20260505 16:59:48.285 | INFO | Analyzing chunk frames = [4831232,4831744) | 20260505 16:59:48.285 | INFO | MemInfo: { | 20260505 16:59:48.285 | INFO | "gpu" : " 17 GB", | 20260505 16:59:48.285 | INFO | "huge" : "", | 20260505 16:59:48.285 | INFO | "huge_shared" : " 27 GB", | 20260505 16:59:48.285 | INFO | "malloc" : " 14 GB", | 20260505 16:59:48.285 | INFO | "pinned" : " 29 GB" | 20260505 16:59:48.285 | INFO | } | 20260505 16:59:48.286 | INFO | Perf report for the last 5.14043 seconds: | 20260505 16:59:48.286 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:59:48.286 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:48.286 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:48.286 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:48.286 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 16:59:48.286 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:48.286 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:48.286 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 16:59:48.286 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:48.286 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:48.286 | INFO | Uploaded 36GiB to GPU in 2.16228s (16.6491GiB/s) | 20260505 16:59:48.286 | INFO | Downloaded 34GiB to GPU in 2.06087s (16.4979GiB/s) | 20260505 16:59:53.429 | INFO | Analyzing chunk frames = [4831744,4832256) | 20260505 16:59:53.429 | INFO | MemInfo: { | 20260505 16:59:53.429 | INFO | "gpu" : " 17 GB", | 20260505 16:59:53.429 | INFO | "huge" : "", | 20260505 16:59:53.429 | INFO | "huge_shared" : " 27 GB", | 20260505 16:59:53.429 | INFO | "malloc" : " 14 GB", | 20260505 16:59:53.429 | INFO | "pinned" : " 29 GB" | 20260505 16:59:53.429 | INFO | } | 20260505 16:59:53.429 | INFO | Perf report for the last 5.14334 seconds: | 20260505 16:59:53.429 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:59:53.429 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:53.429 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:53.429 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:53.429 | INFO | ANALYSIS : 62.4%, 3.0, 96.6ms, 0 | 20260505 16:59:53.429 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:53.429 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:53.429 | INFO | PRE_HQ : 28.1%, 1.0, 14.5ms, 0 | 20260505 16:59:53.429 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:53.429 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:53.429 | INFO | Uploaded 36GiB to GPU in 2.14208s (16.8061GiB/s) | 20260505 16:59:53.429 | INFO | Downloaded 34GiB to GPU in 2.0419s (16.6512GiB/s) | 20260505 16:59:58.568 | INFO | Analyzing chunk frames = [4832256,4832768) | 20260505 16:59:58.568 | INFO | MemInfo: { | 20260505 16:59:58.568 | INFO | "gpu" : " 17 GB", | 20260505 16:59:58.568 | INFO | "huge" : "", | 20260505 16:59:58.569 | INFO | "huge_shared" : " 27 GB", | 20260505 16:59:58.569 | INFO | "malloc" : " 14 GB", | 20260505 16:59:58.569 | INFO | "pinned" : " 29 GB" | 20260505 16:59:58.569 | INFO | } | 20260505 16:59:58.569 | INFO | Perf report for the last 5.13975 seconds: | 20260505 16:59:58.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 16:59:58.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:58.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:58.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:58.569 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 16:59:58.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:58.569 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:58.569 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 16:59:58.569 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:58.569 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 16:59:58.569 | INFO | Uploaded 36GiB to GPU in 2.1264s (16.93GiB/s) | 20260505 16:59:58.569 | INFO | Downloaded 34GiB to GPU in 2.13294s (15.9404GiB/s) | 20260505 17:00:03.703 | INFO | Analyzing chunk frames = [4832768,4833280) | 20260505 17:00:03.703 | INFO | MemInfo: { | 20260505 17:00:03.703 | INFO | "gpu" : " 17 GB", | 20260505 17:00:03.703 | INFO | "huge" : "", | 20260505 17:00:03.703 | INFO | "huge_shared" : " 27 GB", | 20260505 17:00:03.703 | INFO | "malloc" : " 14 GB", | 20260505 17:00:03.703 | INFO | "pinned" : " 29 GB" | 20260505 17:00:03.703 | INFO | } | 20260505 17:00:03.703 | INFO | Perf report for the last 5.13457 seconds: | 20260505 17:00:03.703 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:00:03.703 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:03.703 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:03.703 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:03.703 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 17:00:03.703 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:03.703 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:03.703 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 17:00:03.703 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:03.703 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:03.703 | INFO | Uploaded 36GiB to GPU in 2.14977s (16.746GiB/s) | 20260505 17:00:03.703 | INFO | Downloaded 34GiB to GPU in 2.02651s (16.7776GiB/s) | 20260505 17:00:03.704 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 17:00:03.704 | INFO | Total mem usage: 15.3 GiB | 20260505 17:00:03.704 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 17:00:03.704 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.94%) | 20260505 17:00:03.704 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 17:00:03.704 | INFO | High Water Breakdown: | 20260505 17:00:03.704 | INFO | 61% : 8.67 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 17:00:03.704 | INFO | 38.8% : 5.51 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 17:00:03.704 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 17:00:03.704 | INFO | ------------------------------------------ | 20260505 17:00:03.704 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:00:03.704 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 17:00:03.704 | INFO | Total mem usage: 26.5 GiB | 20260505 17:00:03.704 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 17:00:03.704 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 17:00:03.704 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 17:00:03.704 | INFO | High Water Breakdown: | 20260505 17:00:03.704 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:00:03.704 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:00:03.704 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:00:03.704 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:00:03.704 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:00:03.704 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:00:03.704 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 17:00:03.704 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 17:00:03.704 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:00:03.704 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 17:00:03.704 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 17:00:03.704 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 17:00:03.704 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 17:00:03.704 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 17:00:03.704 | INFO | ------------------------------------------ | 20260505 17:00:03.704 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:00:03.704 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 17:00:03.704 | INFO | Total mem usage: 25.6 GiB | 20260505 17:00:03.704 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 17:00:03.704 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 17:00:03.704 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 17:00:03.704 | INFO | High Water Breakdown: | 20260505 17:00:03.704 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 17:00:03.704 | INFO | ------------------------------------------ | 20260505 17:00:03.704 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:00:03.704 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 17:00:03.704 | INFO | Total mem usage: 15.7 GiB | 20260505 17:00:03.704 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 17:00:03.704 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 17:00:03.704 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 17:00:03.704 | INFO | High Water Breakdown: | 20260505 17:00:03.704 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 17:00:03.704 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 17:00:03.704 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 17:00:03.704 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 17:00:03.704 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 17:00:03.704 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 17:00:03.704 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 17:00:03.704 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 17:00:03.704 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 17:00:03.704 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:00:03.704 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 17:00:03.704 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:00:03.704 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 17:00:03.704 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 17:00:03.704 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 17:00:03.704 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 17:00:03.704 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:00:03.704 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 17:00:03.704 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:00:03.704 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:00:03.704 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:00:03.704 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 17:00:03.704 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 17:00:03.704 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:00:03.704 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 17:00:03.704 | INFO | ------------------------------------------ | 20260505 17:00:03.704 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:00:03.704 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 17:00:03.704 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:00:03.704 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:00:03.704 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.52e+01, 1.35e+01 / 1.82e+00 | 20260505 17:00:03.704 | INFO | PerfReport: 0.29%, AggregateStats -- 9.30e-02 / 1.02e-01, 1.04e-02 / 3.81e-03 | 20260505 17:00:03.704 | INFO | PerfReport: 11.78%, Baseline -- 3.73e+00 / 4.54e+00, 5.79e-01 / 1.18e-01 | 20260505 17:00:03.704 | INFO | PerfReport: 3.29%, Histogram -- 1.04e+00 / 1.53e+00, 1.95e-01 / 9.33e-02 | 20260505 17:00:03.704 | INFO | PerfReport: 15.98%, DME -- 4.05e+01 / 4.49e+01, 4.38e+00 / 1.88e+00 | 20260505 17:00:03.704 | INFO | PerfReport: 56.71%, FrameLabeling -- 1.79e+01 / 1.93e+01, 1.44e+00 / 1.27e+00 | 20260505 17:00:03.704 | INFO | PerfReport: 7.36%, PulseAccumulating -- 2.33e+00 / 2.92e+00, 4.48e-01 / 1.51e-01 | 20260505 17:00:03.704 | INFO | PerfReport: 3.40%, Metrics -- 1.08e+00 / 1.39e+00, 2.99e-01 / 1.65e-01 | 20260505 17:00:03.704 | INFO | PerfReport: 1.18%, PulseSerializing -- 3.75e-01 / 4.36e-01, 6.60e-02 / 4.09e-02 | 20260505 17:00:03.704 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 17:00:03.704 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:00:03.704 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:00:03.704 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.56e+01 / 0.00e+00 | 20260505 17:00:03.704 | INFO | PerfReport: 22.53%, Upload -- 2.26e+01 / 0.00e+00, 7.96e+00 / 0.00e+00 | 20260505 17:00:03.704 | INFO | PerfReport: 21.79%, Download -- 2.18e+01 / 0.00e+00, 7.88e+00 / 0.00e+00 | 20260505 17:00:03.704 | INFO | PerfReport: 31.61%, Compute -- 3.17e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 17:00:03.704 | INFO | PerfReport: 2.28%, LockUpload -- 2.29e+00 / 0.00e+00, 7.91e+00 / 0.00e+00 | 20260505 17:00:03.704 | INFO | PerfReport: 1.38%, LockDownload -- 1.39e+00 / 0.00e+00, 2.85e+00 / 0.00e+00 | 20260505 17:00:03.704 | INFO | PerfReport: 20.39%, LockCompute -- 2.04e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 17:00:03.704 | INFO | PerfReport: 0.01%, Misc -- 1.27e-02 / 0.00e+00, 1.22e-02 / 0.00e+00 | 20260505 17:00:03.838 | INFO | Starting a round of Baz IO | 20260505 17:00:07.053 | INFO | Populating RTMetrics report | 20260505 17:00:07.053 | INFO | RTMetrics report populated | 20260505 17:00:07.053 | INFO | Opening tmp RTMetrics json file | 20260505 17:00:07.060 | INFO | Serializing RTMetrics to json | 20260505 17:00:07.061 | INFO | Serialization done, swapping tmp file now... | 20260505 17:00:07.062 | INFO | RTMetrics done | 20260505 17:00:07.151 | INFO | Finished a round of Baz IO: Wrote 2.151 GiB in 3.313 seconds | 20260505 17:00:07.151 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.14% / 0.00% / 77.86% | 20260505 17:00:07.151 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.328302% / 0% / 0.000103315% | 20260505 17:00:07.151 | INFO | Baz IO overall: | 20260505 17:00:07.151 | INFO | BytesWritten : 2862545563117 | 20260505 17:00:07.151 | INFO | NumEvents : 1358007882471 | 20260505 17:00:08.838 | INFO | Analyzing chunk frames = [4833280,4833792) | 20260505 17:00:08.838 | INFO | MemInfo: { | 20260505 17:00:08.838 | INFO | "gpu" : " 17 GB", | 20260505 17:00:08.838 | INFO | "huge" : "", | 20260505 17:00:08.838 | INFO | "huge_shared" : " 27 GB", | 20260505 17:00:08.838 | INFO | "malloc" : " 14 GB", | 20260505 17:00:08.838 | INFO | "pinned" : " 29 GB" | 20260505 17:00:08.838 | INFO | } | 20260505 17:00:08.838 | INFO | Perf report for the last 5.13471 seconds: | 20260505 17:00:08.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:00:08.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:08.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:08.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:08.838 | INFO | ANALYSIS : 64.2%, 3.0, 99.9ms, 0 | 20260505 17:00:08.838 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:08.838 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260505 17:00:08.838 | INFO | PRE_HQ : 60.6%, 1.8, 56.6ms, 0 | 20260505 17:00:08.838 | INFO | BAZWRITER : 63.9%, 3.8, 125.8ms, 0 | 20260505 17:00:08.838 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 17:00:08.838 | INFO | Uploaded 36GiB to GPU in 2.18403s (16.4833GiB/s) | 20260505 17:00:08.838 | INFO | Downloaded 38GiB to GPU in 2.2595s (16.8179GiB/s) | 20260505 17:00:13.989 | INFO | Analyzing chunk frames = [4833792,4834304) | 20260505 17:00:13.989 | INFO | MemInfo: { | 20260505 17:00:13.989 | INFO | "gpu" : " 17 GB", | 20260505 17:00:13.989 | INFO | "huge" : "", | 20260505 17:00:13.989 | INFO | "huge_shared" : " 27 GB", | 20260505 17:00:13.989 | INFO | "malloc" : " 14 GB", | 20260505 17:00:13.989 | INFO | "pinned" : " 29 GB" | 20260505 17:00:13.989 | INFO | } | 20260505 17:00:13.989 | INFO | Perf report for the last 5.15103 seconds: | 20260505 17:00:13.989 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:00:13.989 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:13.989 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:13.989 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:13.989 | INFO | ANALYSIS : 62.9%, 3.0, 98.4ms, 0 | 20260505 17:00:13.989 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:13.989 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:13.989 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260505 17:00:13.989 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:13.989 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:13.989 | INFO | Uploaded 36GiB to GPU in 2.14044s (16.819GiB/s) | 20260505 17:00:13.989 | INFO | Downloaded 34GiB to GPU in 2.05108s (16.5766GiB/s) | 20260505 17:00:19.133 | INFO | Analyzing chunk frames = [4834304,4834816) | 20260505 17:00:19.133 | INFO | MemInfo: { | 20260505 17:00:19.133 | INFO | "gpu" : " 17 GB", | 20260505 17:00:19.133 | INFO | "huge" : "", | 20260505 17:00:19.133 | INFO | "huge_shared" : " 27 GB", | 20260505 17:00:19.133 | INFO | "malloc" : " 14 GB", | 20260505 17:00:19.133 | INFO | "pinned" : " 29 GB" | 20260505 17:00:19.133 | INFO | } | 20260505 17:00:19.133 | INFO | Perf report for the last 5.14407 seconds: | 20260505 17:00:19.133 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:00:19.133 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:19.133 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:19.133 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:19.133 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260505 17:00:19.133 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:19.133 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:19.133 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:00:19.133 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:19.133 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:19.133 | INFO | Uploaded 36GiB to GPU in 2.20038s (16.3608GiB/s) | 20260505 17:00:19.133 | INFO | Downloaded 34GiB to GPU in 2.05569s (16.5395GiB/s) | 20260505 17:00:24.259 | INFO | Analyzing chunk frames = [4834816,4835328) | 20260505 17:00:24.259 | INFO | MemInfo: { | 20260505 17:00:24.259 | INFO | "gpu" : " 17 GB", | 20260505 17:00:24.259 | INFO | "huge" : "", | 20260505 17:00:24.259 | INFO | "huge_shared" : " 27 GB", | 20260505 17:00:24.259 | INFO | "malloc" : " 14 GB", | 20260505 17:00:24.259 | INFO | "pinned" : " 29 GB" | 20260505 17:00:24.259 | INFO | } | 20260505 17:00:24.259 | INFO | Perf report for the last 5.12615 seconds: | 20260505 17:00:24.259 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:00:24.259 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:24.259 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:24.259 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:24.259 | INFO | ANALYSIS : 62.5%, 3.0, 96.9ms, 0 | 20260505 17:00:24.259 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:24.259 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:24.259 | INFO | PRE_HQ : 27.9%, 1.0, 14.4ms, 0 | 20260505 17:00:24.259 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:24.259 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:24.260 | INFO | Uploaded 36GiB to GPU in 2.17871s (16.5235GiB/s) | 20260505 17:00:24.260 | INFO | Downloaded 34GiB to GPU in 2.06095s (16.4972GiB/s) | 20260505 17:00:29.392 | INFO | Analyzing chunk frames = [4835328,4835840) | 20260505 17:00:29.392 | INFO | MemInfo: { | 20260505 17:00:29.392 | INFO | "gpu" : " 17 GB", | 20260505 17:00:29.392 | INFO | "huge" : "", | 20260505 17:00:29.392 | INFO | "huge_shared" : " 27 GB", | 20260505 17:00:29.392 | INFO | "malloc" : " 14 GB", | 20260505 17:00:29.393 | INFO | "pinned" : " 29 GB" | 20260505 17:00:29.393 | INFO | } | 20260505 17:00:29.393 | INFO | Perf report for the last 5.13335 seconds: | 20260505 17:00:29.393 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:00:29.393 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:29.393 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:29.393 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:29.393 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 17:00:29.393 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:29.393 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:29.393 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:00:29.393 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:29.393 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:29.393 | INFO | Uploaded 36GiB to GPU in 2.18244s (16.4953GiB/s) | 20260505 17:00:29.393 | INFO | Downloaded 34GiB to GPU in 2.1313s (15.9527GiB/s) | 20260505 17:00:34.541 | INFO | Analyzing chunk frames = [4835840,4836352) | 20260505 17:00:34.541 | INFO | MemInfo: { | 20260505 17:00:34.541 | INFO | "gpu" : " 17 GB", | 20260505 17:00:34.541 | INFO | "huge" : "", | 20260505 17:00:34.541 | INFO | "huge_shared" : " 27 GB", | 20260505 17:00:34.541 | INFO | "malloc" : " 14 GB", | 20260505 17:00:34.541 | INFO | "pinned" : " 29 GB" | 20260505 17:00:34.541 | INFO | } | 20260505 17:00:34.541 | INFO | Perf report for the last 5.14864 seconds: | 20260505 17:00:34.541 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:00:34.541 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:34.541 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:34.541 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:34.541 | INFO | ANALYSIS : 62.5%, 3.0, 97.8ms, 0 | 20260505 17:00:34.541 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:34.541 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:34.541 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:00:34.541 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:34.542 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:34.542 | INFO | Uploaded 36GiB to GPU in 2.16917s (16.5962GiB/s) | 20260505 17:00:34.542 | INFO | Downloaded 34GiB to GPU in 2.04564s (16.6207GiB/s) | 20260505 17:00:39.685 | INFO | Analyzing chunk frames = [4836352,4836864) | 20260505 17:00:39.685 | INFO | MemInfo: { | 20260505 17:00:39.685 | INFO | "gpu" : " 17 GB", | 20260505 17:00:39.685 | INFO | "huge" : "", | 20260505 17:00:39.685 | INFO | "huge_shared" : " 27 GB", | 20260505 17:00:39.685 | INFO | "malloc" : " 14 GB", | 20260505 17:00:39.685 | INFO | "pinned" : " 29 GB" | 20260505 17:00:39.685 | INFO | } | 20260505 17:00:39.685 | INFO | Perf report for the last 5.144 seconds: | 20260505 17:00:39.685 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:00:39.685 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:39.685 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:39.685 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:39.685 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 17:00:39.685 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:39.685 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:39.685 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 17:00:39.685 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:39.685 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:39.686 | INFO | Uploaded 36GiB to GPU in 2.13321s (16.876GiB/s) | 20260505 17:00:39.686 | INFO | Downloaded 34GiB to GPU in 2.18705s (15.546GiB/s) | 20260505 17:00:44.827 | INFO | Analyzing chunk frames = [4836864,4837376) | 20260505 17:00:44.827 | INFO | MemInfo: { | 20260505 17:00:44.827 | INFO | "gpu" : " 17 GB", | 20260505 17:00:44.827 | INFO | "huge" : "", | 20260505 17:00:44.827 | INFO | "huge_shared" : " 27 GB", | 20260505 17:00:44.827 | INFO | "malloc" : " 14 GB", | 20260505 17:00:44.827 | INFO | "pinned" : " 29 GB" | 20260505 17:00:44.827 | INFO | } | 20260505 17:00:44.827 | INFO | Perf report for the last 5.14179 seconds: | 20260505 17:00:44.827 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:00:44.827 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:44.827 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:44.827 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:44.827 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 17:00:44.827 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:44.827 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:44.827 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 17:00:44.827 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:44.827 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:44.827 | INFO | Uploaded 36GiB to GPU in 2.20052s (16.3597GiB/s) | 20260505 17:00:44.827 | INFO | Downloaded 34GiB to GPU in 2.02809s (16.7646GiB/s) | 20260505 17:00:44.963 | INFO | Starting a round of Baz IO | 20260505 17:00:48.148 | INFO | Populating RTMetrics report | 20260505 17:00:48.148 | INFO | RTMetrics report populated | 20260505 17:00:48.148 | INFO | Opening tmp RTMetrics json file | 20260505 17:00:48.157 | INFO | Serializing RTMetrics to json | 20260505 17:00:48.158 | INFO | Serialization done, swapping tmp file now... | 20260505 17:00:48.158 | INFO | RTMetrics done | 20260505 17:00:48.245 | INFO | Finished a round of Baz IO: Wrote 2.15 GiB in 3.281 seconds | 20260505 17:00:48.245 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.14% / 0.00% / 77.86% | 20260505 17:00:48.245 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.327679% / 3.97364e-06% / 0.000103315% | 20260505 17:00:48.245 | INFO | Baz IO overall: | 20260505 17:00:48.245 | INFO | BytesWritten : 2864854013364 | 20260505 17:00:48.245 | INFO | NumEvents : 1359088173299 | 20260505 17:00:49.957 | INFO | Analyzing chunk frames = [4837376,4837888) | 20260505 17:00:49.957 | INFO | MemInfo: { | 20260505 17:00:49.957 | INFO | "gpu" : " 17 GB", | 20260505 17:00:49.957 | INFO | "huge" : "", | 20260505 17:00:49.957 | INFO | "huge_shared" : " 27 GB", | 20260505 17:00:49.957 | INFO | "malloc" : " 14 GB", | 20260505 17:00:49.957 | INFO | "pinned" : " 29 GB" | 20260505 17:00:49.957 | INFO | } | 20260505 17:00:49.957 | INFO | Perf report for the last 5.13021 seconds: | 20260505 17:00:49.957 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:00:49.957 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:49.957 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:49.957 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:49.957 | INFO | ANALYSIS : 63.6%, 3.0, 98.7ms, 0 | 20260505 17:00:49.957 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:49.957 | INFO | RT_METRICS : 44.9%, 1.0, 23.5ms, 0 | 20260505 17:00:49.957 | INFO | PRE_HQ : 60.6%, 1.8, 56.9ms, 0 | 20260505 17:00:49.957 | INFO | BAZWRITER : 63.4%, 3.9, 126.9ms, 0 | 20260505 17:00:49.957 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260505 17:00:49.958 | INFO | Uploaded 36GiB to GPU in 2.13798s (16.8383GiB/s) | 20260505 17:00:49.958 | INFO | Downloaded 38GiB to GPU in 2.23891s (16.9726GiB/s) | 20260505 17:00:55.095 | INFO | Analyzing chunk frames = [4837888,4838400) | 20260505 17:00:55.095 | INFO | MemInfo: { | 20260505 17:00:55.095 | INFO | "gpu" : " 17 GB", | 20260505 17:00:55.095 | INFO | "huge" : "", | 20260505 17:00:55.095 | INFO | "huge_shared" : " 27 GB", | 20260505 17:00:55.095 | INFO | "malloc" : " 14 GB", | 20260505 17:00:55.095 | INFO | "pinned" : " 29 GB" | 20260505 17:00:55.095 | INFO | } | 20260505 17:00:55.095 | INFO | Perf report for the last 5.13763 seconds: | 20260505 17:00:55.095 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:00:55.095 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:55.095 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:55.095 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:55.095 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 17:00:55.095 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:55.095 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:55.095 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:00:55.095 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:55.095 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:00:55.095 | INFO | Uploaded 36GiB to GPU in 2.13407s (16.8692GiB/s) | 20260505 17:00:55.095 | INFO | Downloaded 34GiB to GPU in 2.04132s (16.6558GiB/s) | 20260505 17:01:00.243 | INFO | Analyzing chunk frames = [4838400,4838912) | 20260505 17:01:00.243 | INFO | MemInfo: { | 20260505 17:01:00.243 | INFO | "gpu" : " 17 GB", | 20260505 17:01:00.243 | INFO | "huge" : "", | 20260505 17:01:00.244 | INFO | "huge_shared" : " 27 GB", | 20260505 17:01:00.244 | INFO | "malloc" : " 14 GB", | 20260505 17:01:00.244 | INFO | "pinned" : " 29 GB" | 20260505 17:01:00.244 | INFO | } | 20260505 17:01:00.244 | INFO | Perf report for the last 5.14873 seconds: | 20260505 17:01:00.244 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:01:00.244 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:00.244 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:00.244 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:00.244 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 17:01:00.244 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:00.244 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:00.244 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 17:01:00.244 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:00.244 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:00.244 | INFO | Uploaded 36GiB to GPU in 2.1986s (16.374GiB/s) | 20260505 17:01:00.244 | INFO | Downloaded 34GiB to GPU in 2.0308s (16.7421GiB/s) | 20260505 17:01:05.379 | INFO | Analyzing chunk frames = [4838912,4839424) | 20260505 17:01:05.379 | INFO | MemInfo: { | 20260505 17:01:05.379 | INFO | "gpu" : " 17 GB", | 20260505 17:01:05.379 | INFO | "huge" : "", | 20260505 17:01:05.379 | INFO | "huge_shared" : " 27 GB", | 20260505 17:01:05.379 | INFO | "malloc" : " 14 GB", | 20260505 17:01:05.379 | INFO | "pinned" : " 29 GB" | 20260505 17:01:05.379 | INFO | } | 20260505 17:01:05.379 | INFO | Perf report for the last 5.13527 seconds: | 20260505 17:01:05.379 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:01:05.379 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:05.379 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:05.379 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:05.379 | INFO | ANALYSIS : 62.9%, 3.0, 97.7ms, 0 | 20260505 17:01:05.379 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:05.379 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:05.379 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 17:01:05.379 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:05.379 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:05.379 | INFO | Uploaded 36GiB to GPU in 2.22859s (16.1537GiB/s) | 20260505 17:01:05.379 | INFO | Downloaded 34GiB to GPU in 2.11488s (16.0766GiB/s) | 20260505 17:01:10.516 | INFO | Analyzing chunk frames = [4839424,4839936) | 20260505 17:01:10.516 | INFO | MemInfo: { | 20260505 17:01:10.516 | INFO | "gpu" : " 17 GB", | 20260505 17:01:10.516 | INFO | "huge" : "", | 20260505 17:01:10.516 | INFO | "huge_shared" : " 27 GB", | 20260505 17:01:10.516 | INFO | "malloc" : " 14 GB", | 20260505 17:01:10.516 | INFO | "pinned" : " 29 GB" | 20260505 17:01:10.516 | INFO | } | 20260505 17:01:10.516 | INFO | Perf report for the last 5.13684 seconds: | 20260505 17:01:10.516 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:01:10.516 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:10.516 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:10.516 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:10.516 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 17:01:10.516 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:10.516 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:10.516 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 17:01:10.516 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:10.516 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:10.516 | INFO | Uploaded 36GiB to GPU in 2.15288s (16.7218GiB/s) | 20260505 17:01:10.516 | INFO | Downloaded 34GiB to GPU in 2.05507s (16.5445GiB/s) | 20260505 17:01:15.654 | INFO | Analyzing chunk frames = [4839936,4840448) | 20260505 17:01:15.654 | INFO | MemInfo: { | 20260505 17:01:15.654 | INFO | "gpu" : " 17 GB", | 20260505 17:01:15.654 | INFO | "huge" : "", | 20260505 17:01:15.654 | INFO | "huge_shared" : " 27 GB", | 20260505 17:01:15.654 | INFO | "malloc" : " 14 GB", | 20260505 17:01:15.654 | INFO | "pinned" : " 29 GB" | 20260505 17:01:15.654 | INFO | } | 20260505 17:01:15.654 | INFO | Perf report for the last 5.13836 seconds: | 20260505 17:01:15.654 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:01:15.654 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:15.654 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:15.654 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:15.654 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260505 17:01:15.654 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:15.654 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:15.654 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 17:01:15.654 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:15.654 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:15.655 | INFO | Uploaded 36GiB to GPU in 2.14504s (16.7829GiB/s) | 20260505 17:01:15.655 | INFO | Downloaded 34GiB to GPU in 2.03425s (16.7138GiB/s) | 20260505 17:01:20.797 | INFO | Analyzing chunk frames = [4840448,4840960) | 20260505 17:01:20.797 | INFO | MemInfo: { | 20260505 17:01:20.797 | INFO | "gpu" : " 17 GB", | 20260505 17:01:20.797 | INFO | "huge" : "", | 20260505 17:01:20.797 | INFO | "huge_shared" : " 27 GB", | 20260505 17:01:20.797 | INFO | "malloc" : " 14 GB", | 20260505 17:01:20.797 | INFO | "pinned" : " 29 GB" | 20260505 17:01:20.797 | INFO | } | 20260505 17:01:20.797 | INFO | Perf report for the last 5.14321 seconds: | 20260505 17:01:20.797 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:01:20.797 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:20.797 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:20.797 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:20.798 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 17:01:20.798 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:20.798 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:20.798 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 17:01:20.798 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:20.798 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:20.798 | INFO | Uploaded 36GiB to GPU in 2.13138s (16.8905GiB/s) | 20260505 17:01:20.798 | INFO | Downloaded 34GiB to GPU in 2.1654s (15.7015GiB/s) | 20260505 17:01:25.936 | INFO | Analyzing chunk frames = [4840960,4841472) | 20260505 17:01:25.937 | INFO | MemInfo: { | 20260505 17:01:25.937 | INFO | "gpu" : " 17 GB", | 20260505 17:01:25.937 | INFO | "huge" : "", | 20260505 17:01:25.937 | INFO | "huge_shared" : " 27 GB", | 20260505 17:01:25.937 | INFO | "malloc" : " 14 GB", | 20260505 17:01:25.937 | INFO | "pinned" : " 29 GB" | 20260505 17:01:25.937 | INFO | } | 20260505 17:01:25.937 | INFO | Perf report for the last 5.13928 seconds: | 20260505 17:01:25.937 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:01:25.937 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:25.937 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:25.937 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:25.937 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 17:01:25.937 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:25.937 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:25.937 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 17:01:25.937 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:25.937 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:25.937 | INFO | Uploaded 36GiB to GPU in 2.20164s (16.3515GiB/s) | 20260505 17:01:25.937 | INFO | Downloaded 34GiB to GPU in 2.02564s (16.7848GiB/s) | 20260505 17:01:25.937 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 17:01:25.937 | INFO | Total mem usage: 15.3 GiB | 20260505 17:01:25.937 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 17:01:25.937 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.91%) | 20260505 17:01:25.937 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 17:01:25.937 | INFO | High Water Breakdown: | 20260505 17:01:25.937 | INFO | 61% : 8.67 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 17:01:25.937 | INFO | 38.8% : 5.51 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 17:01:25.937 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 17:01:25.937 | INFO | ------------------------------------------ | 20260505 17:01:25.937 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:01:25.937 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 17:01:25.937 | INFO | Total mem usage: 26.5 GiB | 20260505 17:01:25.937 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 17:01:25.937 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 17:01:25.937 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 17:01:25.937 | INFO | High Water Breakdown: | 20260505 17:01:25.937 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:01:25.937 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:01:25.937 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:01:25.938 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:01:25.938 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:01:25.938 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:01:25.938 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 17:01:25.938 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 17:01:25.938 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:01:25.938 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 17:01:25.938 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 17:01:25.938 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 17:01:25.938 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 17:01:25.938 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 17:01:25.938 | INFO | ------------------------------------------ | 20260505 17:01:25.938 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:01:25.938 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 17:01:25.938 | INFO | Total mem usage: 25.6 GiB | 20260505 17:01:25.938 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 17:01:25.938 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 17:01:25.938 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 17:01:25.938 | INFO | High Water Breakdown: | 20260505 17:01:25.938 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 17:01:25.938 | INFO | ------------------------------------------ | 20260505 17:01:25.938 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:01:25.938 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 17:01:25.938 | INFO | Total mem usage: 15.7 GiB | 20260505 17:01:25.938 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 17:01:25.938 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 17:01:25.938 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 17:01:25.938 | INFO | High Water Breakdown: | 20260505 17:01:25.938 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 17:01:25.938 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 17:01:25.938 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 17:01:25.938 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 17:01:25.938 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 17:01:25.938 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 17:01:25.938 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 17:01:25.938 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 17:01:25.938 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 17:01:25.938 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:01:25.938 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 17:01:25.938 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:01:25.938 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 17:01:25.938 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 17:01:25.938 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 17:01:25.938 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 17:01:25.938 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:01:25.938 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 17:01:25.938 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:01:25.938 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:01:25.938 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:01:25.938 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 17:01:25.938 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 17:01:25.938 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:01:25.938 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 17:01:25.938 | INFO | ------------------------------------------ | 20260505 17:01:25.938 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:01:25.938 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 17:01:25.938 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:01:25.938 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:01:25.938 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.51e+01, 1.36e+01 / 2.38e+00 | 20260505 17:01:25.938 | INFO | PerfReport: 0.29%, AggregateStats -- 9.26e-02 / 1.02e-01, 1.05e-02 / 3.71e-03 | 20260505 17:01:25.938 | INFO | PerfReport: 11.80%, Baseline -- 3.73e+00 / 4.50e+00, 5.77e-01 / 1.82e-01 | 20260505 17:01:25.938 | INFO | PerfReport: 3.27%, Histogram -- 1.04e+00 / 1.52e+00, 1.91e-01 / 8.78e-02 | 20260505 17:01:25.938 | INFO | PerfReport: 15.94%, DME -- 4.03e+01 / 4.48e+01, 4.32e+00 / 2.21e+00 | 20260505 17:01:25.938 | INFO | PerfReport: 56.77%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.47e+00 / 1.57e+00 | 20260505 17:01:25.938 | INFO | PerfReport: 7.36%, PulseAccumulating -- 2.33e+00 / 2.92e+00, 4.47e-01 / 1.78e-01 | 20260505 17:01:25.938 | INFO | PerfReport: 3.39%, Metrics -- 1.07e+00 / 1.43e+00, 2.98e-01 / 1.80e-01 | 20260505 17:01:25.938 | INFO | PerfReport: 1.18%, PulseSerializing -- 3.74e-01 / 4.40e-01, 6.59e-02 / 4.14e-02 | 20260505 17:01:25.938 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 17:01:25.938 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:01:25.938 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:01:25.938 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.59e+01 / 0.00e+00 | 20260505 17:01:25.938 | INFO | PerfReport: 22.57%, Upload -- 2.26e+01 / 0.00e+00, 7.88e+00 / 0.00e+00 | 20260505 17:01:25.938 | INFO | PerfReport: 21.79%, Download -- 2.18e+01 / 0.00e+00, 7.81e+00 / 0.00e+00 | 20260505 17:01:25.938 | INFO | PerfReport: 31.58%, Compute -- 3.16e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 17:01:25.938 | INFO | PerfReport: 2.39%, LockUpload -- 2.40e+00 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260505 17:01:25.938 | INFO | PerfReport: 1.40%, LockDownload -- 1.40e+00 / 0.00e+00, 2.96e+00 / 0.00e+00 | 20260505 17:01:25.938 | INFO | PerfReport: 20.25%, LockCompute -- 2.03e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260505 17:01:25.938 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.31e-02 / 0.00e+00 | 20260505 17:01:26.073 | INFO | Starting a round of Baz IO | 20260505 17:01:29.259 | INFO | Populating RTMetrics report | 20260505 17:01:29.260 | INFO | RTMetrics report populated | 20260505 17:01:29.260 | INFO | Opening tmp RTMetrics json file | 20260505 17:01:29.275 | INFO | Serializing RTMetrics to json | 20260505 17:01:29.281 | INFO | Serialization done, swapping tmp file now... | 20260505 17:01:29.294 | INFO | RTMetrics done | 20260505 17:01:29.353 | INFO | Finished a round of Baz IO: Wrote 2.148 GiB in 3.279 seconds | 20260505 17:01:29.353 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.13% / 0.00% / 77.86% | 20260505 17:01:29.353 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.32678% / 7.94729e-06% / 0.000107288% | 20260505 17:01:29.353 | INFO | Baz IO overall: | 20260505 17:01:29.353 | INFO | BytesWritten : 2867160294690 | 20260505 17:01:29.353 | INFO | NumEvents : 1360166899762 | 20260505 17:01:31.075 | INFO | Analyzing chunk frames = [4841472,4841984) | 20260505 17:01:31.075 | INFO | MemInfo: { | 20260505 17:01:31.075 | INFO | "gpu" : " 17 GB", | 20260505 17:01:31.075 | INFO | "huge" : "", | 20260505 17:01:31.075 | INFO | "huge_shared" : " 27 GB", | 20260505 17:01:31.075 | INFO | "malloc" : " 14 GB", | 20260505 17:01:31.075 | INFO | "pinned" : " 29 GB" | 20260505 17:01:31.075 | INFO | } | 20260505 17:01:31.075 | INFO | Perf report for the last 5.13836 seconds: | 20260505 17:01:31.075 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:01:31.075 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:31.075 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:31.075 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:31.075 | INFO | ANALYSIS : 63.6%, 3.0, 99.0ms, 0 | 20260505 17:01:31.075 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:31.075 | INFO | RT_METRICS : 45.3%, 1.0, 23.7ms, 0 | 20260505 17:01:31.075 | INFO | PRE_HQ : 60.3%, 1.8, 56.6ms, 0 | 20260505 17:01:31.075 | INFO | BAZWRITER : 63.3%, 3.9, 126.1ms, 0 | 20260505 17:01:31.075 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 17:01:31.075 | INFO | Uploaded 36GiB to GPU in 2.14319s (16.7974GiB/s) | 20260505 17:01:31.075 | INFO | Downloaded 38GiB to GPU in 2.24464s (16.9292GiB/s) | 20260505 17:01:36.208 | INFO | Analyzing chunk frames = [4841984,4842496) | 20260505 17:01:36.208 | INFO | MemInfo: { | 20260505 17:01:36.208 | INFO | "gpu" : " 17 GB", | 20260505 17:01:36.208 | INFO | "huge" : "", | 20260505 17:01:36.208 | INFO | "huge_shared" : " 27 GB", | 20260505 17:01:36.208 | INFO | "malloc" : " 14 GB", | 20260505 17:01:36.208 | INFO | "pinned" : " 29 GB" | 20260505 17:01:36.208 | INFO | } | 20260505 17:01:36.208 | INFO | Perf report for the last 5.1331 seconds: | 20260505 17:01:36.208 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:01:36.208 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:36.208 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:36.208 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:36.208 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 17:01:36.208 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:36.208 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:36.208 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:01:36.208 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:36.208 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:36.209 | INFO | Uploaded 36GiB to GPU in 2.12683s (16.9266GiB/s) | 20260505 17:01:36.209 | INFO | Downloaded 34GiB to GPU in 2.02655s (16.7772GiB/s) | 20260505 17:01:41.347 | INFO | Analyzing chunk frames = [4842496,4843008) | 20260505 17:01:41.347 | INFO | MemInfo: { | 20260505 17:01:41.347 | INFO | "gpu" : " 17 GB", | 20260505 17:01:41.347 | INFO | "huge" : "", | 20260505 17:01:41.347 | INFO | "huge_shared" : " 27 GB", | 20260505 17:01:41.347 | INFO | "malloc" : " 14 GB", | 20260505 17:01:41.347 | INFO | "pinned" : " 29 GB" | 20260505 17:01:41.347 | INFO | } | 20260505 17:01:41.347 | INFO | Perf report for the last 5.1391 seconds: | 20260505 17:01:41.347 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:01:41.347 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:41.347 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:41.347 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:41.347 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260505 17:01:41.347 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:41.347 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:41.347 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:01:41.347 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:41.347 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:41.348 | INFO | Uploaded 36GiB to GPU in 2.18137s (16.5034GiB/s) | 20260505 17:01:41.348 | INFO | Downloaded 34GiB to GPU in 2.03571s (16.7018GiB/s) | 20260505 17:01:46.497 | INFO | Analyzing chunk frames = [4843008,4843520) | 20260505 17:01:46.497 | INFO | MemInfo: { | 20260505 17:01:46.497 | INFO | "gpu" : " 17 GB", | 20260505 17:01:46.497 | INFO | "huge" : "", | 20260505 17:01:46.497 | INFO | "huge_shared" : " 27 GB", | 20260505 17:01:46.497 | INFO | "malloc" : " 14 GB", | 20260505 17:01:46.497 | INFO | "pinned" : " 29 GB" | 20260505 17:01:46.498 | INFO | } | 20260505 17:01:46.498 | INFO | Perf report for the last 5.15028 seconds: | 20260505 17:01:46.498 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:01:46.498 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:46.498 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:46.498 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:46.498 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260505 17:01:46.498 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:46.498 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:46.498 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 17:01:46.498 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:46.498 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:46.498 | INFO | Uploaded 36GiB to GPU in 2.18445s (16.4802GiB/s) | 20260505 17:01:46.498 | INFO | Downloaded 34GiB to GPU in 2.13477s (15.9268GiB/s) | 20260505 17:01:51.640 | INFO | Analyzing chunk frames = [4843520,4844032) | 20260505 17:01:51.640 | INFO | MemInfo: { | 20260505 17:01:51.640 | INFO | "gpu" : " 17 GB", | 20260505 17:01:51.640 | INFO | "huge" : "", | 20260505 17:01:51.640 | INFO | "huge_shared" : " 27 GB", | 20260505 17:01:51.640 | INFO | "malloc" : " 14 GB", | 20260505 17:01:51.640 | INFO | "pinned" : " 29 GB" | 20260505 17:01:51.640 | INFO | } | 20260505 17:01:51.640 | INFO | Perf report for the last 5.14257 seconds: | 20260505 17:01:51.640 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:01:51.640 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:51.640 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:51.640 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:51.640 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260505 17:01:51.640 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:51.640 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:51.640 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 17:01:51.640 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:51.640 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:51.641 | INFO | Uploaded 36GiB to GPU in 2.12809s (16.9165GiB/s) | 20260505 17:01:51.641 | INFO | Downloaded 34GiB to GPU in 2.0531s (16.5604GiB/s) | 20260505 17:01:56.773 | INFO | Analyzing chunk frames = [4844032,4844544) | 20260505 17:01:56.773 | INFO | MemInfo: { | 20260505 17:01:56.773 | INFO | "gpu" : " 17 GB", | 20260505 17:01:56.773 | INFO | "huge" : "", | 20260505 17:01:56.773 | INFO | "huge_shared" : " 27 GB", | 20260505 17:01:56.773 | INFO | "malloc" : " 14 GB", | 20260505 17:01:56.773 | INFO | "pinned" : " 29 GB" | 20260505 17:01:56.773 | INFO | } | 20260505 17:01:56.773 | INFO | Perf report for the last 5.13301 seconds: | 20260505 17:01:56.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:01:56.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:56.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:56.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:56.773 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260505 17:01:56.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:56.773 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:56.773 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:01:56.773 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:56.773 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:01:56.773 | INFO | Uploaded 36GiB to GPU in 2.13946s (16.8267GiB/s) | 20260505 17:01:56.774 | INFO | Downloaded 34GiB to GPU in 2.04934s (16.5907GiB/s) | 20260505 17:02:01.907 | INFO | Analyzing chunk frames = [4844544,4845056) | 20260505 17:02:01.907 | INFO | MemInfo: { | 20260505 17:02:01.907 | INFO | "gpu" : " 17 GB", | 20260505 17:02:01.907 | INFO | "huge" : "", | 20260505 17:02:01.907 | INFO | "huge_shared" : " 27 GB", | 20260505 17:02:01.907 | INFO | "malloc" : " 14 GB", | 20260505 17:02:01.907 | INFO | "pinned" : " 29 GB" | 20260505 17:02:01.907 | INFO | } | 20260505 17:02:01.907 | INFO | Perf report for the last 5.13371 seconds: | 20260505 17:02:01.907 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:02:01.907 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:01.907 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:01.907 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:01.907 | INFO | ANALYSIS : 63.4%, 3.0, 98.8ms, 0 | 20260505 17:02:01.907 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:01.907 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:01.907 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260505 17:02:01.907 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:01.907 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:01.907 | INFO | Uploaded 36GiB to GPU in 2.15s (16.7442GiB/s) | 20260505 17:02:01.907 | INFO | Downloaded 34GiB to GPU in 2.15645s (15.7667GiB/s) | 20260505 17:02:07.052 | INFO | Analyzing chunk frames = [4845056,4845568) | 20260505 17:02:07.052 | INFO | MemInfo: { | 20260505 17:02:07.052 | INFO | "gpu" : " 17 GB", | 20260505 17:02:07.052 | INFO | "huge" : "", | 20260505 17:02:07.052 | INFO | "huge_shared" : " 27 GB", | 20260505 17:02:07.052 | INFO | "malloc" : " 14 GB", | 20260505 17:02:07.052 | INFO | "pinned" : " 29 GB" | 20260505 17:02:07.052 | INFO | } | 20260505 17:02:07.052 | INFO | Perf report for the last 5.14495 seconds: | 20260505 17:02:07.052 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:02:07.052 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:07.052 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:07.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:07.052 | INFO | ANALYSIS : 63.1%, 3.0, 98.7ms, 0 | 20260505 17:02:07.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:07.052 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:07.052 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 17:02:07.052 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:07.052 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:07.052 | INFO | Uploaded 36GiB to GPU in 2.18521s (16.4744GiB/s) | 20260505 17:02:07.052 | INFO | Downloaded 34GiB to GPU in 2.03444s (16.7122GiB/s) | 20260505 17:02:07.189 | INFO | Starting a round of Baz IO | 20260505 17:02:10.365 | INFO | Populating RTMetrics report | 20260505 17:02:10.365 | INFO | RTMetrics report populated | 20260505 17:02:10.365 | INFO | Opening tmp RTMetrics json file | 20260505 17:02:10.377 | INFO | Serializing RTMetrics to json | 20260505 17:02:10.379 | INFO | Serialization done, swapping tmp file now... | 20260505 17:02:10.379 | INFO | RTMetrics done | 20260505 17:02:10.460 | INFO | Finished a round of Baz IO: Wrote 2.15 GiB in 3.271 seconds | 20260505 17:02:10.460 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.13% / 0.00% / 77.87% | 20260505 17:02:10.460 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.326435% / 3.97364e-06% / 0.000111262% | 20260505 17:02:10.460 | INFO | Baz IO overall: | 20260505 17:02:10.460 | INFO | BytesWritten : 2869468431205 | 20260505 17:02:10.460 | INFO | NumEvents : 1361246593997 | 20260505 17:02:12.187 | INFO | Analyzing chunk frames = [4845568,4846080) | 20260505 17:02:12.187 | INFO | MemInfo: { | 20260505 17:02:12.187 | INFO | "gpu" : " 17 GB", | 20260505 17:02:12.187 | INFO | "huge" : "", | 20260505 17:02:12.187 | INFO | "huge_shared" : " 27 GB", | 20260505 17:02:12.187 | INFO | "malloc" : " 14 GB", | 20260505 17:02:12.187 | INFO | "pinned" : " 29 GB" | 20260505 17:02:12.187 | INFO | } | 20260505 17:02:12.187 | INFO | Perf report for the last 5.13555 seconds: | 20260505 17:02:12.187 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:02:12.187 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:12.187 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:12.188 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:12.188 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 17:02:12.188 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:12.188 | INFO | RT_METRICS : 44.9%, 1.0, 23.5ms, 0 | 20260505 17:02:12.188 | INFO | PRE_HQ : 60.5%, 1.8, 56.3ms, 0 | 20260505 17:02:12.188 | INFO | BAZWRITER : 63.2%, 3.8, 124.6ms, 0 | 20260505 17:02:12.188 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 17:02:12.188 | INFO | Uploaded 36GiB to GPU in 2.16277s (16.6453GiB/s) | 20260505 17:02:12.188 | INFO | Downloaded 38GiB to GPU in 2.248s (16.9039GiB/s) | 20260505 17:02:17.327 | INFO | Analyzing chunk frames = [4846080,4846592) | 20260505 17:02:17.327 | INFO | MemInfo: { | 20260505 17:02:17.327 | INFO | "gpu" : " 17 GB", | 20260505 17:02:17.327 | INFO | "huge" : "", | 20260505 17:02:17.327 | INFO | "huge_shared" : " 27 GB", | 20260505 17:02:17.327 | INFO | "malloc" : " 14 GB", | 20260505 17:02:17.327 | INFO | "pinned" : " 29 GB" | 20260505 17:02:17.327 | INFO | } | 20260505 17:02:17.327 | INFO | Perf report for the last 5.13954 seconds: | 20260505 17:02:17.327 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:02:17.327 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:17.327 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:17.327 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:17.327 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 17:02:17.327 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:17.327 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:17.327 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:02:17.327 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:17.327 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:17.327 | INFO | Uploaded 36GiB to GPU in 2.12104s (16.9728GiB/s) | 20260505 17:02:17.327 | INFO | Downloaded 34GiB to GPU in 2.02913s (16.756GiB/s) | 20260505 17:02:22.464 | INFO | Analyzing chunk frames = [4846592,4847104) | 20260505 17:02:22.464 | INFO | MemInfo: { | 20260505 17:02:22.464 | INFO | "gpu" : " 17 GB", | 20260505 17:02:22.464 | INFO | "huge" : "", | 20260505 17:02:22.465 | INFO | "huge_shared" : " 27 GB", | 20260505 17:02:22.465 | INFO | "malloc" : " 14 GB", | 20260505 17:02:22.465 | INFO | "pinned" : " 29 GB" | 20260505 17:02:22.465 | INFO | } | 20260505 17:02:22.465 | INFO | Perf report for the last 5.13766 seconds: | 20260505 17:02:22.465 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:02:22.465 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:22.465 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:22.465 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:22.465 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260505 17:02:22.465 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:22.465 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:22.465 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:02:22.465 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:22.465 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:22.465 | INFO | Uploaded 36GiB to GPU in 2.2005s (16.3599GiB/s) | 20260505 17:02:22.465 | INFO | Downloaded 34GiB to GPU in 2.04457s (16.6294GiB/s) | 20260505 17:02:27.604 | INFO | Analyzing chunk frames = [4847104,4847616) | 20260505 17:02:27.604 | INFO | MemInfo: { | 20260505 17:02:27.604 | INFO | "gpu" : " 17 GB", | 20260505 17:02:27.604 | INFO | "huge" : "", | 20260505 17:02:27.604 | INFO | "huge_shared" : " 27 GB", | 20260505 17:02:27.604 | INFO | "malloc" : " 14 GB", | 20260505 17:02:27.605 | INFO | "pinned" : " 29 GB" | 20260505 17:02:27.605 | INFO | } | 20260505 17:02:27.605 | INFO | Perf report for the last 5.13998 seconds: | 20260505 17:02:27.605 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:02:27.605 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:27.605 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:27.605 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:27.605 | INFO | ANALYSIS : 62.4%, 3.0, 96.9ms, 0 | 20260505 17:02:27.605 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:27.605 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:27.605 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 17:02:27.605 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:27.605 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:27.605 | INFO | Uploaded 36GiB to GPU in 2.18508s (16.4754GiB/s) | 20260505 17:02:27.605 | INFO | Downloaded 34GiB to GPU in 2.13613s (15.9166GiB/s) | 20260505 17:02:32.752 | INFO | Analyzing chunk frames = [4847616,4848128) | 20260505 17:02:32.752 | INFO | MemInfo: { | 20260505 17:02:32.752 | INFO | "gpu" : " 17 GB", | 20260505 17:02:32.752 | INFO | "huge" : "", | 20260505 17:02:32.752 | INFO | "huge_shared" : " 27 GB", | 20260505 17:02:32.752 | INFO | "malloc" : " 14 GB", | 20260505 17:02:32.752 | INFO | "pinned" : " 29 GB" | 20260505 17:02:32.752 | INFO | } | 20260505 17:02:32.752 | INFO | Perf report for the last 5.14729 seconds: | 20260505 17:02:32.752 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:02:32.752 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:32.752 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:32.752 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:32.752 | INFO | ANALYSIS : 62.1%, 3.0, 97.1ms, 0 | 20260505 17:02:32.752 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:32.752 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:32.752 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 17:02:32.752 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:32.752 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:32.752 | INFO | Uploaded 36GiB to GPU in 2.1452s (16.7817GiB/s) | 20260505 17:02:32.752 | INFO | Downloaded 34GiB to GPU in 2.06097s (16.4971GiB/s) | 20260505 17:02:37.889 | INFO | Analyzing chunk frames = [4848128,4848640) | 20260505 17:02:37.889 | INFO | MemInfo: { | 20260505 17:02:37.889 | INFO | "gpu" : " 17 GB", | 20260505 17:02:37.889 | INFO | "huge" : "", | 20260505 17:02:37.889 | INFO | "huge_shared" : " 27 GB", | 20260505 17:02:37.889 | INFO | "malloc" : " 14 GB", | 20260505 17:02:37.889 | INFO | "pinned" : " 29 GB" | 20260505 17:02:37.889 | INFO | } | 20260505 17:02:37.889 | INFO | Perf report for the last 5.13689 seconds: | 20260505 17:02:37.889 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:02:37.889 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:37.889 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:37.889 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:37.889 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260505 17:02:37.889 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:37.889 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:37.889 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:02:37.889 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:37.889 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:37.889 | INFO | Uploaded 36GiB to GPU in 2.13978s (16.8242GiB/s) | 20260505 17:02:37.889 | INFO | Downloaded 34GiB to GPU in 2.04735s (16.6068GiB/s) | 20260505 17:02:43.020 | INFO | Analyzing chunk frames = [4848640,4849152) | 20260505 17:02:43.020 | INFO | MemInfo: { | 20260505 17:02:43.020 | INFO | "gpu" : " 17 GB", | 20260505 17:02:43.020 | INFO | "huge" : "", | 20260505 17:02:43.020 | INFO | "huge_shared" : " 27 GB", | 20260505 17:02:43.020 | INFO | "malloc" : " 14 GB", | 20260505 17:02:43.020 | INFO | "pinned" : " 29 GB" | 20260505 17:02:43.020 | INFO | } | 20260505 17:02:43.020 | INFO | Perf report for the last 5.13113 seconds: | 20260505 17:02:43.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:02:43.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:43.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:43.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:43.020 | INFO | ANALYSIS : 63.6%, 3.0, 99.1ms, 0 | 20260505 17:02:43.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:43.020 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:43.020 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 17:02:43.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:43.020 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:43.020 | INFO | Uploaded 36GiB to GPU in 2.15755s (16.6856GiB/s) | 20260505 17:02:43.020 | INFO | Downloaded 34GiB to GPU in 2.17102s (15.6608GiB/s) | 20260505 17:02:48.157 | INFO | Analyzing chunk frames = [4849152,4849664) | 20260505 17:02:48.157 | INFO | MemInfo: { | 20260505 17:02:48.157 | INFO | "gpu" : " 17 GB", | 20260505 17:02:48.157 | INFO | "huge" : "", | 20260505 17:02:48.157 | INFO | "huge_shared" : " 27 GB", | 20260505 17:02:48.157 | INFO | "malloc" : " 14 GB", | 20260505 17:02:48.157 | INFO | "pinned" : " 29 GB" | 20260505 17:02:48.157 | INFO | } | 20260505 17:02:48.157 | INFO | Perf report for the last 5.13723 seconds: | 20260505 17:02:48.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:02:48.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:48.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:48.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:48.157 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 17:02:48.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:48.157 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:48.157 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 17:02:48.157 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:48.157 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:48.158 | INFO | Uploaded 36GiB to GPU in 2.23744s (16.0898GiB/s) | 20260505 17:02:48.158 | INFO | Downloaded 34GiB to GPU in 2.10275s (16.1693GiB/s) | 20260505 17:02:48.158 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 17:02:48.158 | INFO | Total mem usage: 15.3 GiB | 20260505 17:02:48.158 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 17:02:48.158 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.05%) | 20260505 17:02:48.158 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 17:02:48.158 | INFO | High Water Breakdown: | 20260505 17:02:48.158 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 17:02:48.158 | INFO | 38.9% : 5.51 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 17:02:48.158 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 17:02:48.158 | INFO | ------------------------------------------ | 20260505 17:02:48.158 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:02:48.158 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 17:02:48.158 | INFO | Total mem usage: 26.5 GiB | 20260505 17:02:48.158 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 17:02:48.158 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 17:02:48.158 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 17:02:48.158 | INFO | High Water Breakdown: | 20260505 17:02:48.158 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:02:48.158 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:02:48.158 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:02:48.158 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:02:48.158 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:02:48.158 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:02:48.158 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 17:02:48.158 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 17:02:48.158 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:02:48.158 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 17:02:48.158 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 17:02:48.158 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 17:02:48.158 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 17:02:48.158 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 17:02:48.158 | INFO | ------------------------------------------ | 20260505 17:02:48.158 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:02:48.158 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 17:02:48.158 | INFO | Total mem usage: 25.6 GiB | 20260505 17:02:48.158 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 17:02:48.158 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 17:02:48.158 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 17:02:48.158 | INFO | High Water Breakdown: | 20260505 17:02:48.158 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 17:02:48.158 | INFO | ------------------------------------------ | 20260505 17:02:48.158 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:02:48.158 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 17:02:48.158 | INFO | Total mem usage: 15.7 GiB | 20260505 17:02:48.158 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 17:02:48.158 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 17:02:48.158 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 17:02:48.158 | INFO | High Water Breakdown: | 20260505 17:02:48.158 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 17:02:48.158 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 17:02:48.158 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 17:02:48.158 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 17:02:48.158 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 17:02:48.158 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 17:02:48.158 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 17:02:48.158 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 17:02:48.158 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 17:02:48.158 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:02:48.159 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 17:02:48.159 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:02:48.159 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 17:02:48.159 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 17:02:48.159 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 17:02:48.159 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 17:02:48.159 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:02:48.159 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 17:02:48.159 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:02:48.159 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:02:48.159 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:02:48.159 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 17:02:48.159 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 17:02:48.159 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:02:48.159 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 17:02:48.159 | INFO | ------------------------------------------ | 20260505 17:02:48.159 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:02:48.159 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 17:02:48.159 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:02:48.159 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:02:48.159 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.49e+01, 1.37e+01 / 1.72e+00 | 20260505 17:02:48.159 | INFO | PerfReport: 0.29%, AggregateStats -- 9.27e-02 / 1.02e-01, 1.04e-02 / 6.07e-03 | 20260505 17:02:48.159 | INFO | PerfReport: 11.79%, Baseline -- 3.73e+00 / 4.49e+00, 5.76e-01 / 2.04e-01 | 20260505 17:02:48.159 | INFO | PerfReport: 3.29%, Histogram -- 1.04e+00 / 1.52e+00, 1.97e-01 / 8.81e-02 | 20260505 17:02:48.159 | INFO | PerfReport: 15.86%, DME -- 4.02e+01 / 4.41e+01, 4.03e+00 / 1.70e+00 | 20260505 17:02:48.159 | INFO | PerfReport: 56.88%, FrameLabeling -- 1.80e+01 / 2.00e+01, 1.53e+00 / 1.53e+00 | 20260505 17:02:48.159 | INFO | PerfReport: 7.32%, PulseAccumulating -- 2.32e+00 / 2.87e+00, 4.39e-01 / 1.87e-01 | 20260505 17:02:48.159 | INFO | PerfReport: 3.39%, Metrics -- 1.07e+00 / 1.38e+00, 3.00e-01 / 1.77e-01 | 20260505 17:02:48.159 | INFO | PerfReport: 1.18%, PulseSerializing -- 3.74e-01 / 4.32e-01, 6.62e-02 / 4.69e-02 | 20260505 17:02:48.159 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 17:02:48.159 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:02:48.159 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:02:48.159 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260505 17:02:48.159 | INFO | PerfReport: 22.51%, Upload -- 2.25e+01 / 0.00e+00, 7.87e+00 / 0.00e+00 | 20260505 17:02:48.159 | INFO | PerfReport: 21.85%, Download -- 2.19e+01 / 0.00e+00, 7.90e+00 / 0.00e+00 | 20260505 17:02:48.159 | INFO | PerfReport: 31.66%, Compute -- 3.17e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 17:02:48.159 | INFO | PerfReport: 2.19%, LockUpload -- 2.19e+00 / 0.00e+00, 7.73e+00 / 0.00e+00 | 20260505 17:02:48.159 | INFO | PerfReport: 1.41%, LockDownload -- 1.41e+00 / 0.00e+00, 2.95e+00 / 0.00e+00 | 20260505 17:02:48.159 | INFO | PerfReport: 20.37%, LockCompute -- 2.04e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 17:02:48.159 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.66e-02 / 0.00e+00 | 20260505 17:02:48.291 | INFO | Starting a round of Baz IO | 20260505 17:02:51.489 | INFO | Populating RTMetrics report | 20260505 17:02:51.489 | INFO | RTMetrics report populated | 20260505 17:02:51.489 | INFO | Opening tmp RTMetrics json file | 20260505 17:02:51.494 | INFO | Serializing RTMetrics to json | 20260505 17:02:51.496 | INFO | Serialization done, swapping tmp file now... | 20260505 17:02:51.498 | INFO | RTMetrics done | 20260505 17:02:51.583 | INFO | Finished a round of Baz IO: Wrote 2.149 GiB in 3.292 seconds | 20260505 17:02:51.583 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.13% / 0.00% / 77.87% | 20260505 17:02:51.583 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.325159% / 7.94729e-06% / 0.000127157% | 20260505 17:02:51.583 | INFO | Baz IO overall: | 20260505 17:02:51.583 | INFO | BytesWritten : 2871775977561 | 20260505 17:02:51.583 | INFO | NumEvents : 1362326179560 | 20260505 17:02:53.302 | INFO | Analyzing chunk frames = [4849664,4850176) | 20260505 17:02:53.302 | INFO | MemInfo: { | 20260505 17:02:53.302 | INFO | "gpu" : " 17 GB", | 20260505 17:02:53.302 | INFO | "huge" : "", | 20260505 17:02:53.302 | INFO | "huge_shared" : " 27 GB", | 20260505 17:02:53.302 | INFO | "malloc" : " 14 GB", | 20260505 17:02:53.302 | INFO | "pinned" : " 29 GB" | 20260505 17:02:53.302 | INFO | } | 20260505 17:02:53.302 | INFO | Perf report for the last 5.14509 seconds: | 20260505 17:02:53.302 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:02:53.302 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:53.302 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:53.302 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:53.303 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 17:02:53.303 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:53.303 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260505 17:02:53.303 | INFO | PRE_HQ : 60.7%, 1.8, 56.4ms, 0 | 20260505 17:02:53.303 | INFO | BAZWRITER : 63.4%, 3.8, 124.6ms, 0 | 20260505 17:02:53.303 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 17:02:53.303 | INFO | Uploaded 36GiB to GPU in 2.15738s (16.6869GiB/s) | 20260505 17:02:53.303 | INFO | Downloaded 38GiB to GPU in 2.25417s (16.8576GiB/s) | 20260505 17:02:58.447 | INFO | Analyzing chunk frames = [4850176,4850688) | 20260505 17:02:58.447 | INFO | MemInfo: { | 20260505 17:02:58.447 | INFO | "gpu" : " 17 GB", | 20260505 17:02:58.447 | INFO | "huge" : "", | 20260505 17:02:58.447 | INFO | "huge_shared" : " 27 GB", | 20260505 17:02:58.447 | INFO | "malloc" : " 14 GB", | 20260505 17:02:58.447 | INFO | "pinned" : " 29 GB" | 20260505 17:02:58.447 | INFO | } | 20260505 17:02:58.447 | INFO | Perf report for the last 5.14475 seconds: | 20260505 17:02:58.447 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:02:58.447 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:58.447 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:58.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:58.447 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 17:02:58.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:58.447 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:58.447 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:02:58.447 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:58.447 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:02:58.447 | INFO | Uploaded 36GiB to GPU in 2.12854s (16.913GiB/s) | 20260505 17:02:58.447 | INFO | Downloaded 34GiB to GPU in 2.02641s (16.7785GiB/s) | 20260505 17:03:03.585 | INFO | Analyzing chunk frames = [4850688,4851200) | 20260505 17:03:03.585 | INFO | MemInfo: { | 20260505 17:03:03.585 | INFO | "gpu" : " 17 GB", | 20260505 17:03:03.585 | INFO | "huge" : "", | 20260505 17:03:03.585 | INFO | "huge_shared" : " 27 GB", | 20260505 17:03:03.585 | INFO | "malloc" : " 14 GB", | 20260505 17:03:03.585 | INFO | "pinned" : " 29 GB" | 20260505 17:03:03.585 | INFO | } | 20260505 17:03:03.585 | INFO | Perf report for the last 5.13788 seconds: | 20260505 17:03:03.585 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:03:03.585 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:03.585 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:03.585 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:03.585 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260505 17:03:03.585 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:03.585 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:03.585 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 17:03:03.585 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:03.586 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:03.586 | INFO | Uploaded 36GiB to GPU in 2.2083s (16.3021GiB/s) | 20260505 17:03:03.586 | INFO | Downloaded 34GiB to GPU in 2.07218s (16.4079GiB/s) | 20260505 17:03:08.718 | INFO | Analyzing chunk frames = [4851200,4851712) | 20260505 17:03:08.718 | INFO | MemInfo: { | 20260505 17:03:08.718 | INFO | "gpu" : " 17 GB", | 20260505 17:03:08.718 | INFO | "huge" : "", | 20260505 17:03:08.718 | INFO | "huge_shared" : " 27 GB", | 20260505 17:03:08.718 | INFO | "malloc" : " 14 GB", | 20260505 17:03:08.718 | INFO | "pinned" : " 29 GB" | 20260505 17:03:08.718 | INFO | } | 20260505 17:03:08.718 | INFO | Perf report for the last 5.13277 seconds: | 20260505 17:03:08.718 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:03:08.718 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:08.718 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:08.718 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:08.718 | INFO | ANALYSIS : 63.0%, 3.0, 97.7ms, 0 | 20260505 17:03:08.718 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:08.718 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:08.718 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 17:03:08.718 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:08.718 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:08.718 | INFO | Uploaded 36GiB to GPU in 2.22498s (16.1799GiB/s) | 20260505 17:03:08.718 | INFO | Downloaded 34GiB to GPU in 2.12269s (16.0174GiB/s) | 20260505 17:03:13.860 | INFO | Analyzing chunk frames = [4851712,4852224) | 20260505 17:03:13.860 | INFO | MemInfo: { | 20260505 17:03:13.860 | INFO | "gpu" : " 17 GB", | 20260505 17:03:13.860 | INFO | "huge" : "", | 20260505 17:03:13.860 | INFO | "huge_shared" : " 27 GB", | 20260505 17:03:13.860 | INFO | "malloc" : " 14 GB", | 20260505 17:03:13.860 | INFO | "pinned" : " 29 GB" | 20260505 17:03:13.860 | INFO | } | 20260505 17:03:13.860 | INFO | Perf report for the last 5.14238 seconds: | 20260505 17:03:13.860 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:03:13.860 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:13.860 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:13.860 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:13.860 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 17:03:13.860 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:13.860 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:13.860 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 17:03:13.860 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:13.860 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:13.860 | INFO | Uploaded 36GiB to GPU in 2.14707s (16.767GiB/s) | 20260505 17:03:13.860 | INFO | Downloaded 34GiB to GPU in 2.06172s (16.491GiB/s) | 20260505 17:03:19.002 | INFO | Analyzing chunk frames = [4852224,4852736) | 20260505 17:03:19.002 | INFO | MemInfo: { | 20260505 17:03:19.002 | INFO | "gpu" : " 17 GB", | 20260505 17:03:19.002 | INFO | "huge" : "", | 20260505 17:03:19.002 | INFO | "huge_shared" : " 27 GB", | 20260505 17:03:19.002 | INFO | "malloc" : " 14 GB", | 20260505 17:03:19.002 | INFO | "pinned" : " 29 GB" | 20260505 17:03:19.002 | INFO | } | 20260505 17:03:19.002 | INFO | Perf report for the last 5.14222 seconds: | 20260505 17:03:19.002 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:03:19.002 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:19.002 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:19.002 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:19.003 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260505 17:03:19.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:19.003 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:19.003 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:03:19.003 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:19.003 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:19.003 | INFO | Uploaded 36GiB to GPU in 2.14373s (16.7932GiB/s) | 20260505 17:03:19.003 | INFO | Downloaded 34GiB to GPU in 2.04299s (16.6423GiB/s) | 20260505 17:03:24.140 | INFO | Analyzing chunk frames = [4852736,4853248) | 20260505 17:03:24.140 | INFO | MemInfo: { | 20260505 17:03:24.140 | INFO | "gpu" : " 17 GB", | 20260505 17:03:24.140 | INFO | "huge" : "", | 20260505 17:03:24.140 | INFO | "huge_shared" : " 27 GB", | 20260505 17:03:24.140 | INFO | "malloc" : " 14 GB", | 20260505 17:03:24.140 | INFO | "pinned" : " 29 GB" | 20260505 17:03:24.140 | INFO | } | 20260505 17:03:24.140 | INFO | Perf report for the last 5.1376 seconds: | 20260505 17:03:24.140 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:03:24.140 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:24.140 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:24.140 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:24.140 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 17:03:24.140 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:24.140 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:24.140 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 17:03:24.140 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:24.140 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:24.140 | INFO | Uploaded 36GiB to GPU in 2.17494s (16.5522GiB/s) | 20260505 17:03:24.140 | INFO | Downloaded 34GiB to GPU in 2.19526s (15.4879GiB/s) | 20260505 17:03:29.280 | INFO | Analyzing chunk frames = [4853248,4853760) | 20260505 17:03:29.280 | INFO | MemInfo: { | 20260505 17:03:29.280 | INFO | "gpu" : " 17 GB", | 20260505 17:03:29.280 | INFO | "huge" : "", | 20260505 17:03:29.280 | INFO | "huge_shared" : " 27 GB", | 20260505 17:03:29.280 | INFO | "malloc" : " 14 GB", | 20260505 17:03:29.280 | INFO | "pinned" : " 29 GB" | 20260505 17:03:29.280 | INFO | } | 20260505 17:03:29.280 | INFO | Perf report for the last 5.13958 seconds: | 20260505 17:03:29.280 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:03:29.280 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:29.280 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:29.280 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:29.280 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 17:03:29.280 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:29.280 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:29.280 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 17:03:29.280 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:29.280 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:29.280 | INFO | Uploaded 36GiB to GPU in 2.1704s (16.5868GiB/s) | 20260505 17:03:29.280 | INFO | Downloaded 34GiB to GPU in 2.02082s (16.8248GiB/s) | 20260505 17:03:29.419 | INFO | Starting a round of Baz IO | 20260505 17:03:32.606 | INFO | Populating RTMetrics report | 20260505 17:03:32.606 | INFO | RTMetrics report populated | 20260505 17:03:32.606 | INFO | Opening tmp RTMetrics json file | 20260505 17:03:32.608 | INFO | Serializing RTMetrics to json | 20260505 17:03:32.610 | INFO | Serialization done, swapping tmp file now... | 20260505 17:03:32.610 | INFO | RTMetrics done | 20260505 17:03:32.703 | INFO | Finished a round of Baz IO: Wrote 2.149 GiB in 3.284 seconds | 20260505 17:03:32.703 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.13% / 0.00% / 77.87% | 20260505 17:03:32.703 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.324686% / 7.94729e-06% / 0.000107288% | 20260505 17:03:32.703 | INFO | Baz IO overall: | 20260505 17:03:32.704 | INFO | BytesWritten : 2874083007810 | 20260505 17:03:32.704 | INFO | NumEvents : 1363405533011 | 20260505 17:03:34.413 | INFO | Analyzing chunk frames = [4853760,4854272) | 20260505 17:03:34.413 | INFO | MemInfo: { | 20260505 17:03:34.413 | INFO | "gpu" : " 17 GB", | 20260505 17:03:34.413 | INFO | "huge" : "", | 20260505 17:03:34.413 | INFO | "huge_shared" : " 27 GB", | 20260505 17:03:34.413 | INFO | "malloc" : " 14 GB", | 20260505 17:03:34.413 | INFO | "pinned" : " 29 GB" | 20260505 17:03:34.413 | INFO | } | 20260505 17:03:34.413 | INFO | Perf report for the last 5.13361 seconds: | 20260505 17:03:34.413 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:03:34.413 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:34.413 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:34.413 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:34.413 | INFO | ANALYSIS : 63.7%, 3.0, 99.0ms, 0 | 20260505 17:03:34.413 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:34.413 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260505 17:03:34.413 | INFO | PRE_HQ : 60.3%, 1.8, 56.5ms, 0 | 20260505 17:03:34.413 | INFO | BAZWRITER : 63.4%, 3.8, 124.6ms, 0 | 20260505 17:03:34.413 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 17:03:34.414 | INFO | Uploaded 36GiB to GPU in 2.14927s (16.7499GiB/s) | 20260505 17:03:34.414 | INFO | Downloaded 38GiB to GPU in 2.25038s (16.886GiB/s) | 20260505 17:03:39.559 | INFO | Analyzing chunk frames = [4854272,4854784) | 20260505 17:03:39.559 | INFO | MemInfo: { | 20260505 17:03:39.559 | INFO | "gpu" : " 17 GB", | 20260505 17:03:39.559 | INFO | "huge" : "", | 20260505 17:03:39.559 | INFO | "huge_shared" : " 27 GB", | 20260505 17:03:39.559 | INFO | "malloc" : " 14 GB", | 20260505 17:03:39.560 | INFO | "pinned" : " 29 GB" | 20260505 17:03:39.560 | INFO | } | 20260505 17:03:39.560 | INFO | Perf report for the last 5.14643 seconds: | 20260505 17:03:39.560 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:03:39.560 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:39.560 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:39.560 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:39.560 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 17:03:39.560 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:39.560 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:39.560 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 17:03:39.560 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:39.560 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:39.560 | INFO | Uploaded 36GiB to GPU in 2.13409s (16.869GiB/s) | 20260505 17:03:39.560 | INFO | Downloaded 34GiB to GPU in 2.03818s (16.6816GiB/s) | 20260505 17:03:44.700 | INFO | Analyzing chunk frames = [4854784,4855296) | 20260505 17:03:44.700 | INFO | MemInfo: { | 20260505 17:03:44.700 | INFO | "gpu" : " 17 GB", | 20260505 17:03:44.700 | INFO | "huge" : "", | 20260505 17:03:44.700 | INFO | "huge_shared" : " 27 GB", | 20260505 17:03:44.700 | INFO | "malloc" : " 14 GB", | 20260505 17:03:44.700 | INFO | "pinned" : " 29 GB" | 20260505 17:03:44.700 | INFO | } | 20260505 17:03:44.701 | INFO | Perf report for the last 5.14087 seconds: | 20260505 17:03:44.701 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:03:44.701 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:44.701 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:44.701 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:44.701 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 17:03:44.702 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:44.702 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:44.702 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:03:44.702 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:44.702 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:44.702 | INFO | Uploaded 36GiB to GPU in 2.20355s (16.3373GiB/s) | 20260505 17:03:44.702 | INFO | Downloaded 34GiB to GPU in 2.04234s (16.6475GiB/s) | 20260505 17:03:49.836 | INFO | Analyzing chunk frames = [4855296,4855808) | 20260505 17:03:49.836 | INFO | MemInfo: { | 20260505 17:03:49.836 | INFO | "gpu" : " 17 GB", | 20260505 17:03:49.836 | INFO | "huge" : "", | 20260505 17:03:49.836 | INFO | "huge_shared" : " 27 GB", | 20260505 17:03:49.836 | INFO | "malloc" : " 14 GB", | 20260505 17:03:49.836 | INFO | "pinned" : " 29 GB" | 20260505 17:03:49.836 | INFO | } | 20260505 17:03:49.836 | INFO | Perf report for the last 5.13568 seconds: | 20260505 17:03:49.836 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:03:49.836 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:49.836 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:49.836 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:49.836 | INFO | ANALYSIS : 62.9%, 3.0, 97.7ms, 0 | 20260505 17:03:49.836 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:49.836 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:49.836 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:03:49.836 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:49.836 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:49.836 | INFO | Uploaded 36GiB to GPU in 2.22549s (16.1762GiB/s) | 20260505 17:03:49.837 | INFO | Downloaded 34GiB to GPU in 2.11458s (16.0788GiB/s) | 20260505 17:03:54.966 | INFO | Analyzing chunk frames = [4855808,4856320) | 20260505 17:03:54.966 | INFO | MemInfo: { | 20260505 17:03:54.966 | INFO | "gpu" : " 17 GB", | 20260505 17:03:54.966 | INFO | "huge" : "", | 20260505 17:03:54.966 | INFO | "huge_shared" : " 27 GB", | 20260505 17:03:54.966 | INFO | "malloc" : " 14 GB", | 20260505 17:03:54.966 | INFO | "pinned" : " 29 GB" | 20260505 17:03:54.966 | INFO | } | 20260505 17:03:54.966 | INFO | Perf report for the last 5.12996 seconds: | 20260505 17:03:54.966 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:03:54.966 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:54.966 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:54.966 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:54.966 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260505 17:03:54.966 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:54.966 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:54.966 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:03:54.966 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:54.966 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:03:54.967 | INFO | Uploaded 36GiB to GPU in 2.14565s (16.7781GiB/s) | 20260505 17:03:54.967 | INFO | Downloaded 34GiB to GPU in 2.05802s (16.5207GiB/s) | 20260505 17:04:00.110 | INFO | Analyzing chunk frames = [4856320,4856832) | 20260505 17:04:00.110 | INFO | MemInfo: { | 20260505 17:04:00.110 | INFO | "gpu" : " 17 GB", | 20260505 17:04:00.110 | INFO | "huge" : "", | 20260505 17:04:00.110 | INFO | "huge_shared" : " 27 GB", | 20260505 17:04:00.110 | INFO | "malloc" : " 14 GB", | 20260505 17:04:00.110 | INFO | "pinned" : " 29 GB" | 20260505 17:04:00.110 | INFO | } | 20260505 17:04:00.110 | INFO | Perf report for the last 5.14423 seconds: | 20260505 17:04:00.110 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:04:00.110 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:00.110 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:00.111 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:00.111 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260505 17:04:00.111 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:00.111 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:00.111 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 17:04:00.111 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:00.111 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:00.111 | INFO | Uploaded 36GiB to GPU in 2.16189s (16.6521GiB/s) | 20260505 17:04:00.111 | INFO | Downloaded 34GiB to GPU in 2.04486s (16.6271GiB/s) | 20260505 17:04:05.257 | INFO | Analyzing chunk frames = [4856832,4857344) | 20260505 17:04:05.257 | INFO | MemInfo: { | 20260505 17:04:05.257 | INFO | "gpu" : " 17 GB", | 20260505 17:04:05.257 | INFO | "huge" : "", | 20260505 17:04:05.258 | INFO | "huge_shared" : " 27 GB", | 20260505 17:04:05.258 | INFO | "malloc" : " 14 GB", | 20260505 17:04:05.258 | INFO | "pinned" : " 29 GB" | 20260505 17:04:05.258 | INFO | } | 20260505 17:04:05.258 | INFO | Perf report for the last 5.14718 seconds: | 20260505 17:04:05.258 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:04:05.258 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:05.258 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:05.258 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:05.258 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 17:04:05.258 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:05.258 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:05.258 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 17:04:05.258 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:05.258 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:05.258 | INFO | Uploaded 36GiB to GPU in 2.12671s (16.9275GiB/s) | 20260505 17:04:05.258 | INFO | Downloaded 34GiB to GPU in 2.17683s (15.6191GiB/s) | 20260505 17:04:10.401 | INFO | Analyzing chunk frames = [4857344,4857856) | 20260505 17:04:10.401 | INFO | MemInfo: { | 20260505 17:04:10.401 | INFO | "gpu" : " 17 GB", | 20260505 17:04:10.401 | INFO | "huge" : "", | 20260505 17:04:10.401 | INFO | "huge_shared" : " 27 GB", | 20260505 17:04:10.401 | INFO | "malloc" : " 14 GB", | 20260505 17:04:10.401 | INFO | "pinned" : " 29 GB" | 20260505 17:04:10.401 | INFO | } | 20260505 17:04:10.401 | INFO | Perf report for the last 5.14354 seconds: | 20260505 17:04:10.401 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:04:10.401 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:10.401 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:10.401 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:10.401 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 17:04:10.401 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:10.401 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:10.401 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 17:04:10.401 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:10.401 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:10.401 | INFO | Uploaded 36GiB to GPU in 2.19492s (16.4015GiB/s) | 20260505 17:04:10.402 | INFO | Downloaded 34GiB to GPU in 2.02222s (16.8132GiB/s) | 20260505 17:04:10.402 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 17:04:10.402 | INFO | Total mem usage: 15.3 GiB | 20260505 17:04:10.402 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 17:04:10.402 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.97%) | 20260505 17:04:10.402 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 17:04:10.402 | INFO | High Water Breakdown: | 20260505 17:04:10.402 | INFO | 61% : 8.66 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 17:04:10.402 | INFO | 38.8% : 5.51 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 17:04:10.402 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 17:04:10.402 | INFO | ------------------------------------------ | 20260505 17:04:10.402 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:04:10.402 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 17:04:10.402 | INFO | Total mem usage: 26.5 GiB | 20260505 17:04:10.402 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 17:04:10.402 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 17:04:10.402 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 17:04:10.402 | INFO | High Water Breakdown: | 20260505 17:04:10.402 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:04:10.402 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:04:10.402 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:04:10.402 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:04:10.402 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:04:10.402 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:04:10.402 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 17:04:10.402 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 17:04:10.402 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:04:10.402 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 17:04:10.402 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 17:04:10.402 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 17:04:10.402 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 17:04:10.402 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 17:04:10.402 | INFO | ------------------------------------------ | 20260505 17:04:10.402 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:04:10.402 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 17:04:10.402 | INFO | Total mem usage: 25.6 GiB | 20260505 17:04:10.402 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 17:04:10.402 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 17:04:10.402 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 17:04:10.402 | INFO | High Water Breakdown: | 20260505 17:04:10.402 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 17:04:10.402 | INFO | ------------------------------------------ | 20260505 17:04:10.402 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:04:10.402 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 17:04:10.402 | INFO | Total mem usage: 15.7 GiB | 20260505 17:04:10.402 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 17:04:10.402 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 17:04:10.402 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 17:04:10.402 | INFO | High Water Breakdown: | 20260505 17:04:10.402 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 17:04:10.402 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 17:04:10.402 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 17:04:10.402 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 17:04:10.402 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 17:04:10.402 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 17:04:10.402 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 17:04:10.402 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 17:04:10.402 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 17:04:10.402 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:04:10.402 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 17:04:10.402 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:04:10.402 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 17:04:10.402 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 17:04:10.402 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 17:04:10.403 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 17:04:10.403 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:04:10.403 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 17:04:10.403 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:04:10.403 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:04:10.403 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:04:10.403 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 17:04:10.403 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 17:04:10.403 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:04:10.403 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 17:04:10.403 | INFO | ------------------------------------------ | 20260505 17:04:10.403 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:04:10.403 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 17:04:10.403 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:04:10.403 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:04:10.403 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.50e+01, 1.34e+01 / 2.38e+00 | 20260505 17:04:10.403 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.02e-01, 1.02e-02 / 5.21e-03 | 20260505 17:04:10.403 | INFO | PerfReport: 11.72%, Baseline -- 3.71e+00 / 4.48e+00, 5.56e-01 / 1.71e-01 | 20260505 17:04:10.403 | INFO | PerfReport: 3.26%, Histogram -- 1.03e+00 / 1.52e+00, 1.91e-01 / 8.08e-02 | 20260505 17:04:10.403 | INFO | PerfReport: 15.98%, DME -- 4.04e+01 / 4.48e+01, 4.36e+00 / 2.40e+00 | 20260505 17:04:10.403 | INFO | PerfReport: 56.82%, FrameLabeling -- 1.80e+01 / 1.93e+01, 1.46e+00 / 9.49e-01 | 20260505 17:04:10.403 | INFO | PerfReport: 7.35%, PulseAccumulating -- 2.32e+00 / 2.94e+00, 4.48e-01 / 1.25e-01 | 20260505 17:04:10.403 | INFO | PerfReport: 3.39%, Metrics -- 1.07e+00 / 1.42e+00, 2.98e-01 / 1.75e-01 | 20260505 17:04:10.403 | INFO | PerfReport: 1.18%, PulseSerializing -- 3.74e-01 / 4.41e-01, 6.58e-02 / 3.68e-02 | 20260505 17:04:10.403 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 17:04:10.403 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:04:10.403 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:04:10.403 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.60e+01 / 0.00e+00 | 20260505 17:04:10.403 | INFO | PerfReport: 22.55%, Upload -- 2.26e+01 / 0.00e+00, 7.90e+00 / 0.00e+00 | 20260505 17:04:10.403 | INFO | PerfReport: 21.80%, Download -- 2.18e+01 / 0.00e+00, 7.82e+00 / 0.00e+00 | 20260505 17:04:10.403 | INFO | PerfReport: 31.56%, Compute -- 3.16e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 17:04:10.403 | INFO | PerfReport: 2.42%, LockUpload -- 2.43e+00 / 0.00e+00, 8.10e+00 / 0.00e+00 | 20260505 17:04:10.403 | INFO | PerfReport: 1.33%, LockDownload -- 1.33e+00 / 0.00e+00, 2.84e+00 / 0.00e+00 | 20260505 17:04:10.403 | INFO | PerfReport: 20.32%, LockCompute -- 2.04e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260505 17:04:10.403 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.95e-02 / 0.00e+00 | 20260505 17:04:10.535 | INFO | Starting a round of Baz IO | 20260505 17:04:13.719 | INFO | Populating RTMetrics report | 20260505 17:04:13.719 | INFO | RTMetrics report populated | 20260505 17:04:13.719 | INFO | Opening tmp RTMetrics json file | 20260505 17:04:13.729 | INFO | Serializing RTMetrics to json | 20260505 17:04:13.731 | INFO | Serialization done, swapping tmp file now... | 20260505 17:04:13.733 | INFO | RTMetrics done | 20260505 17:04:13.819 | INFO | Finished a round of Baz IO: Wrote 2.151 GiB in 3.283 seconds | 20260505 17:04:13.819 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.13% / 0.00% / 77.87% | 20260505 17:04:13.819 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.324849% / 0% / 8.74201e-05% | 20260505 17:04:13.819 | INFO | Baz IO overall: | 20260505 17:04:13.819 | INFO | BytesWritten : 2876392208905 | 20260505 17:04:13.819 | INFO | NumEvents : 1364486953285 | 20260505 17:04:15.530 | INFO | Analyzing chunk frames = [4857856,4858368) | 20260505 17:04:15.530 | INFO | MemInfo: { | 20260505 17:04:15.530 | INFO | "gpu" : " 17 GB", | 20260505 17:04:15.530 | INFO | "huge" : "", | 20260505 17:04:15.530 | INFO | "huge_shared" : " 27 GB", | 20260505 17:04:15.530 | INFO | "malloc" : " 14 GB", | 20260505 17:04:15.530 | INFO | "pinned" : " 29 GB" | 20260505 17:04:15.530 | INFO | } | 20260505 17:04:15.530 | INFO | Perf report for the last 5.12865 seconds: | 20260505 17:04:15.530 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:04:15.530 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:15.530 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:15.530 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:15.530 | INFO | ANALYSIS : 63.6%, 3.0, 98.9ms, 0 | 20260505 17:04:15.530 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:15.530 | INFO | RT_METRICS : 44.9%, 1.0, 23.5ms, 0 | 20260505 17:04:15.530 | INFO | PRE_HQ : 60.7%, 1.8, 56.3ms, 0 | 20260505 17:04:15.530 | INFO | BAZWRITER : 63.6%, 3.9, 125.6ms, 0 | 20260505 17:04:15.530 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 17:04:15.530 | INFO | Uploaded 36GiB to GPU in 2.16118s (16.6576GiB/s) | 20260505 17:04:15.530 | INFO | Downloaded 38GiB to GPU in 2.24784s (16.9051GiB/s) | 20260505 17:04:20.665 | INFO | Analyzing chunk frames = [4858368,4858880) | 20260505 17:04:20.665 | INFO | MemInfo: { | 20260505 17:04:20.665 | INFO | "gpu" : " 17 GB", | 20260505 17:04:20.665 | INFO | "huge" : "", | 20260505 17:04:20.665 | INFO | "huge_shared" : " 27 GB", | 20260505 17:04:20.665 | INFO | "malloc" : " 14 GB", | 20260505 17:04:20.665 | INFO | "pinned" : " 29 GB" | 20260505 17:04:20.665 | INFO | } | 20260505 17:04:20.665 | INFO | Perf report for the last 5.13556 seconds: | 20260505 17:04:20.665 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:04:20.665 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:20.665 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:20.666 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:20.666 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 17:04:20.666 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:20.666 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:20.666 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:04:20.666 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:20.666 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:20.666 | INFO | Uploaded 36GiB to GPU in 2.13403s (16.8695GiB/s) | 20260505 17:04:20.666 | INFO | Downloaded 34GiB to GPU in 2.03227s (16.73GiB/s) | 20260505 17:04:25.817 | INFO | Analyzing chunk frames = [4858880,4859392) | 20260505 17:04:25.817 | INFO | MemInfo: { | 20260505 17:04:25.817 | INFO | "gpu" : " 17 GB", | 20260505 17:04:25.817 | INFO | "huge" : "", | 20260505 17:04:25.817 | INFO | "huge_shared" : " 27 GB", | 20260505 17:04:25.817 | INFO | "malloc" : " 14 GB", | 20260505 17:04:25.817 | INFO | "pinned" : " 29 GB" | 20260505 17:04:25.817 | INFO | } | 20260505 17:04:25.817 | INFO | Perf report for the last 5.15201 seconds: | 20260505 17:04:25.817 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:04:25.817 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:25.817 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:25.818 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:25.818 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260505 17:04:25.818 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:25.818 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:25.818 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:04:25.818 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:25.818 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:25.818 | INFO | Uploaded 36GiB to GPU in 2.18988s (16.4392GiB/s) | 20260505 17:04:25.818 | INFO | Downloaded 34GiB to GPU in 2.05337s (16.5582GiB/s) | 20260505 17:04:30.950 | INFO | Analyzing chunk frames = [4859392,4859904) | 20260505 17:04:30.950 | INFO | MemInfo: { | 20260505 17:04:30.950 | INFO | "gpu" : " 17 GB", | 20260505 17:04:30.950 | INFO | "huge" : "", | 20260505 17:04:30.950 | INFO | "huge_shared" : " 27 GB", | 20260505 17:04:30.950 | INFO | "malloc" : " 14 GB", | 20260505 17:04:30.950 | INFO | "pinned" : " 29 GB" | 20260505 17:04:30.950 | INFO | } | 20260505 17:04:30.950 | INFO | Perf report for the last 5.13272 seconds: | 20260505 17:04:30.950 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:04:30.950 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:30.950 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:30.950 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:30.950 | INFO | ANALYSIS : 62.9%, 3.0, 97.6ms, 0 | 20260505 17:04:30.950 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:30.950 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:30.950 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 17:04:30.950 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:30.950 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:30.950 | INFO | Uploaded 36GiB to GPU in 2.22266s (16.1968GiB/s) | 20260505 17:04:30.951 | INFO | Downloaded 34GiB to GPU in 2.12546s (15.9965GiB/s) | 20260505 17:04:36.088 | INFO | Analyzing chunk frames = [4859904,4860416) | 20260505 17:04:36.088 | INFO | MemInfo: { | 20260505 17:04:36.088 | INFO | "gpu" : " 17 GB", | 20260505 17:04:36.088 | INFO | "huge" : "", | 20260505 17:04:36.088 | INFO | "huge_shared" : " 27 GB", | 20260505 17:04:36.088 | INFO | "malloc" : " 14 GB", | 20260505 17:04:36.088 | INFO | "pinned" : " 29 GB" | 20260505 17:04:36.088 | INFO | } | 20260505 17:04:36.089 | INFO | Perf report for the last 5.1383 seconds: | 20260505 17:04:36.089 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:04:36.089 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:36.089 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:36.089 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:36.089 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 17:04:36.089 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:36.089 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:36.089 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:04:36.089 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:36.089 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:36.089 | INFO | Uploaded 36GiB to GPU in 2.14864s (16.7548GiB/s) | 20260505 17:04:36.089 | INFO | Downloaded 34GiB to GPU in 2.06062s (16.4999GiB/s) | 20260505 17:04:41.228 | INFO | Analyzing chunk frames = [4860416,4860928) | 20260505 17:04:41.228 | INFO | MemInfo: { | 20260505 17:04:41.228 | INFO | "gpu" : " 17 GB", | 20260505 17:04:41.228 | INFO | "huge" : "", | 20260505 17:04:41.228 | INFO | "huge_shared" : " 27 GB", | 20260505 17:04:41.228 | INFO | "malloc" : " 14 GB", | 20260505 17:04:41.228 | INFO | "pinned" : " 29 GB" | 20260505 17:04:41.228 | INFO | } | 20260505 17:04:41.228 | INFO | Perf report for the last 5.13962 seconds: | 20260505 17:04:41.228 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:04:41.228 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:41.228 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:41.228 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:41.228 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260505 17:04:41.228 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:41.228 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:41.228 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:04:41.228 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:41.228 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:41.228 | INFO | Uploaded 36GiB to GPU in 2.13634s (16.8513GiB/s) | 20260505 17:04:41.228 | INFO | Downloaded 34GiB to GPU in 2.03476s (16.7096GiB/s) | 20260505 17:04:46.366 | INFO | Analyzing chunk frames = [4860928,4861440) | 20260505 17:04:46.366 | INFO | MemInfo: { | 20260505 17:04:46.366 | INFO | "gpu" : " 17 GB", | 20260505 17:04:46.366 | INFO | "huge" : "", | 20260505 17:04:46.366 | INFO | "huge_shared" : " 27 GB", | 20260505 17:04:46.366 | INFO | "malloc" : " 14 GB", | 20260505 17:04:46.366 | INFO | "pinned" : " 29 GB" | 20260505 17:04:46.366 | INFO | } | 20260505 17:04:46.366 | INFO | Perf report for the last 5.1377 seconds: | 20260505 17:04:46.366 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:04:46.366 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:46.366 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:46.366 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:46.366 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 17:04:46.366 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:46.366 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:46.366 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 17:04:46.366 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:46.366 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:46.366 | INFO | Uploaded 36GiB to GPU in 2.14415s (16.7899GiB/s) | 20260505 17:04:46.366 | INFO | Downloaded 34GiB to GPU in 2.15496s (15.7776GiB/s) | 20260505 17:04:51.512 | INFO | Analyzing chunk frames = [4861440,4861952) | 20260505 17:04:51.512 | INFO | MemInfo: { | 20260505 17:04:51.512 | INFO | "gpu" : " 17 GB", | 20260505 17:04:51.512 | INFO | "huge" : "", | 20260505 17:04:51.512 | INFO | "huge_shared" : " 27 GB", | 20260505 17:04:51.512 | INFO | "malloc" : " 14 GB", | 20260505 17:04:51.512 | INFO | "pinned" : " 29 GB" | 20260505 17:04:51.512 | INFO | } | 20260505 17:04:51.512 | INFO | Perf report for the last 5.14595 seconds: | 20260505 17:04:51.512 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:04:51.512 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:51.512 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:51.512 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:51.512 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 17:04:51.512 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:51.512 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:51.512 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 17:04:51.512 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:51.512 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:51.512 | INFO | Uploaded 36GiB to GPU in 2.15822s (16.6804GiB/s) | 20260505 17:04:51.512 | INFO | Downloaded 34GiB to GPU in 2.0237s (16.8009GiB/s) | 20260505 17:04:51.646 | INFO | Starting a round of Baz IO | 20260505 17:04:54.829 | INFO | Populating RTMetrics report | 20260505 17:04:54.829 | INFO | RTMetrics report populated | 20260505 17:04:54.829 | INFO | Opening tmp RTMetrics json file | 20260505 17:04:54.837 | INFO | Serializing RTMetrics to json | 20260505 17:04:54.840 | INFO | Serialization done, swapping tmp file now... | 20260505 17:04:54.842 | INFO | RTMetrics done | 20260505 17:04:54.922 | INFO | Finished a round of Baz IO: Wrote 2.148 GiB in 3.275 seconds | 20260505 17:04:54.922 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.12% / 0.00% / 77.87% | 20260505 17:04:54.922 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.323709% / 0% / 0.000107288% | 20260505 17:04:54.922 | INFO | Baz IO overall: | 20260505 17:04:54.922 | INFO | BytesWritten : 2878698884166 | 20260505 17:04:54.922 | INFO | NumEvents : 1365566983026 | 20260505 17:04:56.652 | INFO | Analyzing chunk frames = [4861952,4862464) | 20260505 17:04:56.652 | INFO | MemInfo: { | 20260505 17:04:56.652 | INFO | "gpu" : " 17 GB", | 20260505 17:04:56.652 | INFO | "huge" : "", | 20260505 17:04:56.652 | INFO | "huge_shared" : " 27 GB", | 20260505 17:04:56.652 | INFO | "malloc" : " 14 GB", | 20260505 17:04:56.652 | INFO | "pinned" : " 29 GB" | 20260505 17:04:56.652 | INFO | } | 20260505 17:04:56.652 | INFO | Perf report for the last 5.14017 seconds: | 20260505 17:04:56.652 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:04:56.652 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:56.652 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:56.652 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:56.652 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 17:04:56.652 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:04:56.652 | INFO | RT_METRICS : 44.7%, 1.0, 23.4ms, 0 | 20260505 17:04:56.652 | INFO | PRE_HQ : 60.3%, 1.8, 56.1ms, 0 | 20260505 17:04:56.652 | INFO | BAZWRITER : 63.3%, 3.9, 125.9ms, 0 | 20260505 17:04:56.652 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 17:04:56.652 | INFO | Uploaded 36GiB to GPU in 2.14977s (16.746GiB/s) | 20260505 17:04:56.652 | INFO | Downloaded 38GiB to GPU in 2.24578s (16.9206GiB/s) | 20260505 17:05:01.778 | INFO | Analyzing chunk frames = [4862464,4862976) | 20260505 17:05:01.778 | INFO | MemInfo: { | 20260505 17:05:01.778 | INFO | "gpu" : " 17 GB", | 20260505 17:05:01.778 | INFO | "huge" : "", | 20260505 17:05:01.778 | INFO | "huge_shared" : " 27 GB", | 20260505 17:05:01.778 | INFO | "malloc" : " 14 GB", | 20260505 17:05:01.778 | INFO | "pinned" : " 29 GB" | 20260505 17:05:01.778 | INFO | } | 20260505 17:05:01.778 | INFO | Perf report for the last 5.12627 seconds: | 20260505 17:05:01.778 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:05:01.778 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:01.778 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:01.778 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:01.778 | INFO | ANALYSIS : 63.0%, 3.0, 98.1ms, 0 | 20260505 17:05:01.778 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:01.778 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:01.778 | INFO | PRE_HQ : 27.7%, 1.0, 14.3ms, 0 | 20260505 17:05:01.778 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:01.778 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:01.779 | INFO | Uploaded 36GiB to GPU in 2.13415s (16.8685GiB/s) | 20260505 17:05:01.779 | INFO | Downloaded 34GiB to GPU in 2.02764s (16.7682GiB/s) | 20260505 17:05:06.916 | INFO | Analyzing chunk frames = [4862976,4863488) | 20260505 17:05:06.916 | INFO | MemInfo: { | 20260505 17:05:06.916 | INFO | "gpu" : " 17 GB", | 20260505 17:05:06.916 | INFO | "huge" : "", | 20260505 17:05:06.916 | INFO | "huge_shared" : " 27 GB", | 20260505 17:05:06.916 | INFO | "malloc" : " 14 GB", | 20260505 17:05:06.916 | INFO | "pinned" : " 29 GB" | 20260505 17:05:06.916 | INFO | } | 20260505 17:05:06.916 | INFO | Perf report for the last 5.13807 seconds: | 20260505 17:05:06.916 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:05:06.916 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:06.916 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:06.916 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:06.916 | INFO | ANALYSIS : 63.0%, 3.0, 98.0ms, 0 | 20260505 17:05:06.916 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:06.916 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:06.916 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:05:06.916 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:06.916 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:06.917 | INFO | Uploaded 36GiB to GPU in 2.19564s (16.3961GiB/s) | 20260505 17:05:06.917 | INFO | Downloaded 34GiB to GPU in 2.06229s (16.4865GiB/s) | 20260505 17:05:12.069 | INFO | Analyzing chunk frames = [4863488,4864000) | 20260505 17:05:12.069 | INFO | MemInfo: { | 20260505 17:05:12.069 | INFO | "gpu" : " 17 GB", | 20260505 17:05:12.069 | INFO | "huge" : "", | 20260505 17:05:12.069 | INFO | "huge_shared" : " 27 GB", | 20260505 17:05:12.069 | INFO | "malloc" : " 14 GB", | 20260505 17:05:12.069 | INFO | "pinned" : " 29 GB" | 20260505 17:05:12.069 | INFO | } | 20260505 17:05:12.069 | INFO | Perf report for the last 5.15298 seconds: | 20260505 17:05:12.069 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:05:12.069 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:12.069 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:12.069 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:12.069 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260505 17:05:12.069 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:12.069 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:12.069 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 17:05:12.069 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:12.069 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:12.070 | INFO | Uploaded 36GiB to GPU in 2.18989s (16.4392GiB/s) | 20260505 17:05:12.070 | INFO | Downloaded 34GiB to GPU in 2.12904s (15.9697GiB/s) | 20260505 17:05:17.213 | INFO | Analyzing chunk frames = [4864000,4864512) | 20260505 17:05:17.213 | INFO | MemInfo: { | 20260505 17:05:17.213 | INFO | "gpu" : " 17 GB", | 20260505 17:05:17.213 | INFO | "huge" : "", | 20260505 17:05:17.213 | INFO | "huge_shared" : " 27 GB", | 20260505 17:05:17.213 | INFO | "malloc" : " 14 GB", | 20260505 17:05:17.213 | INFO | "pinned" : " 29 GB" | 20260505 17:05:17.213 | INFO | } | 20260505 17:05:17.213 | INFO | Perf report for the last 5.14389 seconds: | 20260505 17:05:17.213 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:05:17.213 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:17.213 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:17.213 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:17.213 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 17:05:17.213 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:17.213 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:17.213 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 17:05:17.213 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:17.213 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:17.214 | INFO | Uploaded 36GiB to GPU in 2.13936s (16.8275GiB/s) | 20260505 17:05:17.214 | INFO | Downloaded 34GiB to GPU in 2.05885s (16.5141GiB/s) | 20260505 17:05:22.351 | INFO | Analyzing chunk frames = [4864512,4865024) | 20260505 17:05:22.351 | INFO | MemInfo: { | 20260505 17:05:22.351 | INFO | "gpu" : " 17 GB", | 20260505 17:05:22.351 | INFO | "huge" : "", | 20260505 17:05:22.351 | INFO | "huge_shared" : " 27 GB", | 20260505 17:05:22.351 | INFO | "malloc" : " 14 GB", | 20260505 17:05:22.351 | INFO | "pinned" : " 29 GB" | 20260505 17:05:22.351 | INFO | } | 20260505 17:05:22.351 | INFO | Perf report for the last 5.13774 seconds: | 20260505 17:05:22.351 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:05:22.351 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:22.351 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:22.351 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:22.351 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260505 17:05:22.351 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:22.351 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:22.351 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 17:05:22.351 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:22.351 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:22.351 | INFO | Uploaded 36GiB to GPU in 2.14221s (16.8051GiB/s) | 20260505 17:05:22.351 | INFO | Downloaded 34GiB to GPU in 2.04964s (16.5883GiB/s) | 20260505 17:05:27.478 | INFO | Analyzing chunk frames = [4865024,4865536) | 20260505 17:05:27.478 | INFO | MemInfo: { | 20260505 17:05:27.478 | INFO | "gpu" : " 17 GB", | 20260505 17:05:27.478 | INFO | "huge" : "", | 20260505 17:05:27.478 | INFO | "huge_shared" : " 27 GB", | 20260505 17:05:27.478 | INFO | "malloc" : " 14 GB", | 20260505 17:05:27.478 | INFO | "pinned" : " 29 GB" | 20260505 17:05:27.478 | INFO | } | 20260505 17:05:27.479 | INFO | Perf report for the last 5.1275 seconds: | 20260505 17:05:27.479 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:05:27.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:27.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:27.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:27.479 | INFO | ANALYSIS : 63.4%, 3.0, 98.8ms, 0 | 20260505 17:05:27.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:27.479 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:27.479 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 17:05:27.479 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:27.479 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:27.479 | INFO | Uploaded 36GiB to GPU in 2.13618s (16.8525GiB/s) | 20260505 17:05:27.479 | INFO | Downloaded 34GiB to GPU in 2.13767s (15.9052GiB/s) | 20260505 17:05:32.619 | INFO | Analyzing chunk frames = [4865536,4866048) | 20260505 17:05:32.619 | INFO | MemInfo: { | 20260505 17:05:32.619 | INFO | "gpu" : " 17 GB", | 20260505 17:05:32.619 | INFO | "huge" : "", | 20260505 17:05:32.619 | INFO | "huge_shared" : " 27 GB", | 20260505 17:05:32.619 | INFO | "malloc" : " 14 GB", | 20260505 17:05:32.619 | INFO | "pinned" : " 29 GB" | 20260505 17:05:32.619 | INFO | } | 20260505 17:05:32.619 | INFO | Perf report for the last 5.1403 seconds: | 20260505 17:05:32.619 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:05:32.619 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:32.619 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:32.619 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:32.619 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 17:05:32.619 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:32.619 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:32.619 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 17:05:32.619 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:32.619 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:32.619 | INFO | Uploaded 36GiB to GPU in 2.23575s (16.102GiB/s) | 20260505 17:05:32.619 | INFO | Downloaded 34GiB to GPU in 2.10547s (16.1484GiB/s) | 20260505 17:05:32.619 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 17:05:32.619 | INFO | Total mem usage: 15.3 GiB | 20260505 17:05:32.619 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 17:05:32.619 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.93%) | 20260505 17:05:32.619 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 17:05:32.619 | INFO | High Water Breakdown: | 20260505 17:05:32.619 | INFO | 61% : 8.67 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 17:05:32.619 | INFO | 38.8% : 5.51 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 17:05:32.619 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 17:05:32.619 | INFO | ------------------------------------------ | 20260505 17:05:32.619 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:05:32.619 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 17:05:32.619 | INFO | Total mem usage: 26.5 GiB | 20260505 17:05:32.619 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 17:05:32.619 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 17:05:32.619 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 17:05:32.619 | INFO | High Water Breakdown: | 20260505 17:05:32.619 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:05:32.619 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:05:32.619 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:05:32.619 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:05:32.619 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:05:32.619 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:05:32.620 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 17:05:32.620 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 17:05:32.620 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:05:32.620 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 17:05:32.620 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 17:05:32.620 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 17:05:32.620 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 17:05:32.620 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 17:05:32.620 | INFO | ------------------------------------------ | 20260505 17:05:32.620 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:05:32.620 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 17:05:32.620 | INFO | Total mem usage: 25.6 GiB | 20260505 17:05:32.620 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 17:05:32.620 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 17:05:32.620 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 17:05:32.620 | INFO | High Water Breakdown: | 20260505 17:05:32.620 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 17:05:32.620 | INFO | ------------------------------------------ | 20260505 17:05:32.620 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:05:32.620 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 17:05:32.620 | INFO | Total mem usage: 15.7 GiB | 20260505 17:05:32.620 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 17:05:32.620 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 17:05:32.620 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 17:05:32.620 | INFO | High Water Breakdown: | 20260505 17:05:32.620 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 17:05:32.620 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 17:05:32.620 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 17:05:32.620 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 17:05:32.620 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 17:05:32.620 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 17:05:32.620 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 17:05:32.620 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 17:05:32.620 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 17:05:32.620 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:05:32.620 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 17:05:32.620 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:05:32.620 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 17:05:32.620 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 17:05:32.620 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 17:05:32.620 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 17:05:32.620 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:05:32.620 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 17:05:32.620 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:05:32.620 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:05:32.620 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:05:32.620 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 17:05:32.620 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 17:05:32.620 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:05:32.620 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 17:05:32.620 | INFO | ------------------------------------------ | 20260505 17:05:32.620 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:05:32.620 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 17:05:32.620 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:05:32.620 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:05:32.620 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.51e+01, 1.37e+01 / 1.64e+00 | 20260505 17:05:32.620 | INFO | PerfReport: 0.29%, AggregateStats -- 9.27e-02 / 1.01e-01, 1.06e-02 / 5.48e-03 | 20260505 17:05:32.620 | INFO | PerfReport: 11.78%, Baseline -- 3.73e+00 / 4.53e+00, 5.73e-01 / 1.35e-01 | 20260505 17:05:32.620 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.52e+00, 1.95e-01 / 8.52e-02 | 20260505 17:05:32.620 | INFO | PerfReport: 15.90%, DME -- 4.03e+01 / 4.46e+01, 4.18e+00 / 1.76e+00 | 20260505 17:05:32.620 | INFO | PerfReport: 56.87%, FrameLabeling -- 1.80e+01 / 1.97e+01, 1.53e+00 / 1.27e+00 | 20260505 17:05:32.620 | INFO | PerfReport: 7.31%, PulseAccumulating -- 2.31e+00 / 2.87e+00, 4.43e-01 / 1.75e-01 | 20260505 17:05:32.620 | INFO | PerfReport: 3.39%, Metrics -- 1.07e+00 / 1.39e+00, 2.98e-01 / 1.42e-01 | 20260505 17:05:32.620 | INFO | PerfReport: 1.18%, PulseSerializing -- 3.74e-01 / 4.34e-01, 6.73e-02 / 3.35e-02 | 20260505 17:05:32.621 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 17:05:32.621 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:05:32.621 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:05:32.621 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.56e+01 / 0.00e+00 | 20260505 17:05:32.621 | INFO | PerfReport: 22.52%, Upload -- 2.25e+01 / 0.00e+00, 7.91e+00 / 0.00e+00 | 20260505 17:05:32.621 | INFO | PerfReport: 21.83%, Download -- 2.18e+01 / 0.00e+00, 7.86e+00 / 0.00e+00 | 20260505 17:05:32.621 | INFO | PerfReport: 31.65%, Compute -- 3.17e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 17:05:32.621 | INFO | PerfReport: 2.30%, LockUpload -- 2.30e+00 / 0.00e+00, 7.86e+00 / 0.00e+00 | 20260505 17:05:32.621 | INFO | PerfReport: 1.35%, LockDownload -- 1.35e+00 / 0.00e+00, 2.87e+00 / 0.00e+00 | 20260505 17:05:32.621 | INFO | PerfReport: 20.34%, LockCompute -- 2.04e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 17:05:32.621 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.69e-02 / 0.00e+00 | 20260505 17:05:32.753 | INFO | Starting a round of Baz IO | 20260505 17:05:35.944 | INFO | Populating RTMetrics report | 20260505 17:05:35.944 | INFO | RTMetrics report populated | 20260505 17:05:35.944 | INFO | Opening tmp RTMetrics json file | 20260505 17:05:35.954 | INFO | Serializing RTMetrics to json | 20260505 17:05:35.957 | INFO | Serialization done, swapping tmp file now... | 20260505 17:05:35.959 | INFO | RTMetrics done | 20260505 17:05:36.042 | INFO | Finished a round of Baz IO: Wrote 2.146 GiB in 3.289 seconds | 20260505 17:05:36.042 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.12% / 0.00% / 77.88% | 20260505 17:05:36.042 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.320371% / 3.97364e-06% / 9.53674e-05% | 20260505 17:05:36.042 | INFO | Baz IO overall: | 20260505 17:05:36.042 | INFO | BytesWritten : 2881002799257 | 20260505 17:05:36.043 | INFO | NumEvents : 1366644557496 | 20260505 17:05:37.767 | INFO | Analyzing chunk frames = [4866048,4866560) | 20260505 17:05:37.767 | INFO | MemInfo: { | 20260505 17:05:37.767 | INFO | "gpu" : " 17 GB", | 20260505 17:05:37.767 | INFO | "huge" : "", | 20260505 17:05:37.767 | INFO | "huge_shared" : " 27 GB", | 20260505 17:05:37.767 | INFO | "malloc" : " 14 GB", | 20260505 17:05:37.767 | INFO | "pinned" : " 29 GB" | 20260505 17:05:37.767 | INFO | } | 20260505 17:05:37.767 | INFO | Perf report for the last 5.14852 seconds: | 20260505 17:05:37.767 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:05:37.767 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:37.768 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:37.768 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:37.768 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 17:05:37.768 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:37.768 | INFO | RT_METRICS : 44.7%, 1.0, 23.5ms, 0 | 20260505 17:05:37.768 | INFO | PRE_HQ : 60.3%, 1.8, 56.1ms, 0 | 20260505 17:05:37.768 | INFO | BAZWRITER : 63.2%, 3.9, 125.9ms, 0 | 20260505 17:05:37.768 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 17:05:37.768 | INFO | Uploaded 36GiB to GPU in 2.16533s (16.6256GiB/s) | 20260505 17:05:37.768 | INFO | Downloaded 38GiB to GPU in 2.24641s (16.9159GiB/s) | 20260505 17:05:42.908 | INFO | Analyzing chunk frames = [4866560,4867072) | 20260505 17:05:42.908 | INFO | MemInfo: { | 20260505 17:05:42.908 | INFO | "gpu" : " 17 GB", | 20260505 17:05:42.908 | INFO | "huge" : "", | 20260505 17:05:42.908 | INFO | "huge_shared" : " 27 GB", | 20260505 17:05:42.908 | INFO | "malloc" : " 14 GB", | 20260505 17:05:42.908 | INFO | "pinned" : " 29 GB" | 20260505 17:05:42.908 | INFO | } | 20260505 17:05:42.908 | INFO | Perf report for the last 5.1405 seconds: | 20260505 17:05:42.908 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:05:42.908 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:42.908 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:42.908 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:42.908 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 17:05:42.908 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:42.908 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:42.908 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 17:05:42.908 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:42.908 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:42.908 | INFO | Uploaded 36GiB to GPU in 2.1324s (16.8824GiB/s) | 20260505 17:05:42.908 | INFO | Downloaded 34GiB to GPU in 2.02633s (16.7791GiB/s) | 20260505 17:05:48.043 | INFO | Analyzing chunk frames = [4867072,4867584) | 20260505 17:05:48.043 | INFO | MemInfo: { | 20260505 17:05:48.043 | INFO | "gpu" : " 17 GB", | 20260505 17:05:48.043 | INFO | "huge" : "", | 20260505 17:05:48.043 | INFO | "huge_shared" : " 27 GB", | 20260505 17:05:48.044 | INFO | "malloc" : " 14 GB", | 20260505 17:05:48.044 | INFO | "pinned" : " 29 GB" | 20260505 17:05:48.044 | INFO | } | 20260505 17:05:48.044 | INFO | Perf report for the last 5.13572 seconds: | 20260505 17:05:48.044 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:05:48.044 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:48.044 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:48.044 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:48.044 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260505 17:05:48.044 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:48.044 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:48.044 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 17:05:48.044 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:48.044 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:48.044 | INFO | Uploaded 36GiB to GPU in 2.20871s (16.2991GiB/s) | 20260505 17:05:48.044 | INFO | Downloaded 34GiB to GPU in 2.07352s (16.3973GiB/s) | 20260505 17:05:53.177 | INFO | Analyzing chunk frames = [4867584,4868096) | 20260505 17:05:53.177 | INFO | MemInfo: { | 20260505 17:05:53.177 | INFO | "gpu" : " 17 GB", | 20260505 17:05:53.177 | INFO | "huge" : "", | 20260505 17:05:53.177 | INFO | "huge_shared" : " 27 GB", | 20260505 17:05:53.177 | INFO | "malloc" : " 14 GB", | 20260505 17:05:53.177 | INFO | "pinned" : " 29 GB" | 20260505 17:05:53.177 | INFO | } | 20260505 17:05:53.177 | INFO | Perf report for the last 5.13387 seconds: | 20260505 17:05:53.177 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:05:53.177 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:53.178 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:53.178 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:53.178 | INFO | ANALYSIS : 62.7%, 3.0, 97.3ms, 0 | 20260505 17:05:53.178 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:53.178 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:53.178 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:05:53.178 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:53.178 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:53.178 | INFO | Uploaded 36GiB to GPU in 2.17923s (16.5196GiB/s) | 20260505 17:05:53.178 | INFO | Downloaded 34GiB to GPU in 2.14194s (15.8735GiB/s) | 20260505 17:05:58.320 | INFO | Analyzing chunk frames = [4868096,4868608) | 20260505 17:05:58.320 | INFO | MemInfo: { | 20260505 17:05:58.320 | INFO | "gpu" : " 17 GB", | 20260505 17:05:58.320 | INFO | "huge" : "", | 20260505 17:05:58.320 | INFO | "huge_shared" : " 27 GB", | 20260505 17:05:58.320 | INFO | "malloc" : " 14 GB", | 20260505 17:05:58.320 | INFO | "pinned" : " 29 GB" | 20260505 17:05:58.321 | INFO | } | 20260505 17:05:58.321 | INFO | Perf report for the last 5.1432 seconds: | 20260505 17:05:58.321 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:05:58.321 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:58.321 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:58.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:58.321 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260505 17:05:58.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:58.321 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:58.321 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 17:05:58.321 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:58.321 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:05:58.321 | INFO | Uploaded 36GiB to GPU in 2.18776s (16.4552GiB/s) | 20260505 17:05:58.321 | INFO | Downloaded 34GiB to GPU in 2.13241s (15.9444GiB/s) | 20260505 17:06:03.460 | INFO | Analyzing chunk frames = [4868608,4869120) | 20260505 17:06:03.460 | INFO | MemInfo: { | 20260505 17:06:03.460 | INFO | "gpu" : " 17 GB", | 20260505 17:06:03.460 | INFO | "huge" : "", | 20260505 17:06:03.460 | INFO | "huge_shared" : " 27 GB", | 20260505 17:06:03.460 | INFO | "malloc" : " 14 GB", | 20260505 17:06:03.460 | INFO | "pinned" : " 29 GB" | 20260505 17:06:03.460 | INFO | } | 20260505 17:06:03.460 | INFO | Perf report for the last 5.13936 seconds: | 20260505 17:06:03.460 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:06:03.460 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:03.460 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:03.460 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:03.460 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260505 17:06:03.460 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:03.460 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:03.460 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:06:03.460 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:03.460 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:03.460 | INFO | Uploaded 36GiB to GPU in 2.14519s (16.7817GiB/s) | 20260505 17:06:03.460 | INFO | Downloaded 34GiB to GPU in 2.04385s (16.6353GiB/s) | 20260505 17:06:08.596 | INFO | Analyzing chunk frames = [4869120,4869632) | 20260505 17:06:08.596 | INFO | MemInfo: { | 20260505 17:06:08.596 | INFO | "gpu" : " 17 GB", | 20260505 17:06:08.596 | INFO | "huge" : "", | 20260505 17:06:08.596 | INFO | "huge_shared" : " 27 GB", | 20260505 17:06:08.596 | INFO | "malloc" : " 14 GB", | 20260505 17:06:08.596 | INFO | "pinned" : " 29 GB" | 20260505 17:06:08.596 | INFO | } | 20260505 17:06:08.596 | INFO | Perf report for the last 5.13593 seconds: | 20260505 17:06:08.596 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:06:08.596 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:08.596 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:08.596 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:08.596 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 17:06:08.596 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:08.596 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:08.596 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 17:06:08.596 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:08.596 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:08.596 | INFO | Uploaded 36GiB to GPU in 2.12767s (16.9199GiB/s) | 20260505 17:06:08.596 | INFO | Downloaded 34GiB to GPU in 2.14039s (15.8849GiB/s) | 20260505 17:06:13.732 | INFO | Analyzing chunk frames = [4869632,4870144) | 20260505 17:06:13.732 | INFO | MemInfo: { | 20260505 17:06:13.732 | INFO | "gpu" : " 17 GB", | 20260505 17:06:13.732 | INFO | "huge" : "", | 20260505 17:06:13.732 | INFO | "huge_shared" : " 27 GB", | 20260505 17:06:13.732 | INFO | "malloc" : " 14 GB", | 20260505 17:06:13.732 | INFO | "pinned" : " 29 GB" | 20260505 17:06:13.732 | INFO | } | 20260505 17:06:13.733 | INFO | Perf report for the last 5.13651 seconds: | 20260505 17:06:13.733 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:06:13.733 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:13.733 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:13.733 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:13.733 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 17:06:13.733 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:13.733 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:13.733 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 17:06:13.733 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:13.733 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:13.733 | INFO | Uploaded 36GiB to GPU in 2.16352s (16.6396GiB/s) | 20260505 17:06:13.733 | INFO | Downloaded 34GiB to GPU in 2.03498s (16.7077GiB/s) | 20260505 17:06:13.866 | INFO | Starting a round of Baz IO | 20260505 17:06:17.076 | INFO | Populating RTMetrics report | 20260505 17:06:17.077 | INFO | RTMetrics report populated | 20260505 17:06:17.077 | INFO | Opening tmp RTMetrics json file | 20260505 17:06:17.088 | INFO | Serializing RTMetrics to json | 20260505 17:06:17.090 | INFO | Serialization done, swapping tmp file now... | 20260505 17:06:17.090 | INFO | RTMetrics done | 20260505 17:06:17.174 | INFO | Finished a round of Baz IO: Wrote 2.147 GiB in 3.308 seconds | 20260505 17:06:17.174 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.12% / 0.00% / 77.88% | 20260505 17:06:17.174 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.321448% / 0% / 0.000103315% | 20260505 17:06:17.174 | INFO | Baz IO overall: | 20260505 17:06:17.174 | INFO | BytesWritten : 2883307994638 | 20260505 17:06:17.174 | INFO | NumEvents : 1367723602250 | 20260505 17:06:18.872 | INFO | Analyzing chunk frames = [4870144,4870656) | 20260505 17:06:18.872 | INFO | MemInfo: { | 20260505 17:06:18.872 | INFO | "gpu" : " 17 GB", | 20260505 17:06:18.872 | INFO | "huge" : "", | 20260505 17:06:18.872 | INFO | "huge_shared" : " 27 GB", | 20260505 17:06:18.872 | INFO | "malloc" : " 14 GB", | 20260505 17:06:18.872 | INFO | "pinned" : " 29 GB" | 20260505 17:06:18.872 | INFO | } | 20260505 17:06:18.872 | INFO | Perf report for the last 5.13934 seconds: | 20260505 17:06:18.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:06:18.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:18.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:18.872 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:18.872 | INFO | ANALYSIS : 64.0%, 3.0, 99.6ms, 0 | 20260505 17:06:18.872 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:18.872 | INFO | RT_METRICS : 44.9%, 1.0, 23.6ms, 0 | 20260505 17:06:18.872 | INFO | PRE_HQ : 60.8%, 1.8, 56.4ms, 0 | 20260505 17:06:18.872 | INFO | BAZWRITER : 63.8%, 3.8, 125.9ms, 0 | 20260505 17:06:18.872 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260505 17:06:18.872 | INFO | Uploaded 36GiB to GPU in 2.20669s (16.314GiB/s) | 20260505 17:06:18.872 | INFO | Downloaded 38GiB to GPU in 2.2525s (16.8702GiB/s) | 20260505 17:06:24.024 | INFO | Analyzing chunk frames = [4870656,4871168) | 20260505 17:06:24.024 | INFO | MemInfo: { | 20260505 17:06:24.024 | INFO | "gpu" : " 17 GB", | 20260505 17:06:24.024 | INFO | "huge" : "", | 20260505 17:06:24.024 | INFO | "huge_shared" : " 27 GB", | 20260505 17:06:24.024 | INFO | "malloc" : " 14 GB", | 20260505 17:06:24.024 | INFO | "pinned" : " 29 GB" | 20260505 17:06:24.024 | INFO | } | 20260505 17:06:24.024 | INFO | Perf report for the last 5.15236 seconds: | 20260505 17:06:24.024 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:06:24.024 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:24.024 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:24.024 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:24.024 | INFO | ANALYSIS : 63.2%, 3.0, 98.9ms, 0 | 20260505 17:06:24.024 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:24.024 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:24.024 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:06:24.024 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:24.024 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:24.025 | INFO | Uploaded 36GiB to GPU in 2.207s (16.3117GiB/s) | 20260505 17:06:24.025 | INFO | Downloaded 34GiB to GPU in 2.068s (16.441GiB/s) | 20260505 17:06:29.163 | INFO | Analyzing chunk frames = [4871168,4871680) | 20260505 17:06:29.163 | INFO | MemInfo: { | 20260505 17:06:29.163 | INFO | "gpu" : " 17 GB", | 20260505 17:06:29.163 | INFO | "huge" : "", | 20260505 17:06:29.163 | INFO | "huge_shared" : " 27 GB", | 20260505 17:06:29.164 | INFO | "malloc" : " 14 GB", | 20260505 17:06:29.164 | INFO | "pinned" : " 29 GB" | 20260505 17:06:29.164 | INFO | } | 20260505 17:06:29.164 | INFO | Perf report for the last 5.13945 seconds: | 20260505 17:06:29.164 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:06:29.164 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:29.164 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:29.164 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:29.164 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 17:06:29.164 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:29.164 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:29.164 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 17:06:29.164 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:29.164 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:29.164 | INFO | Uploaded 36GiB to GPU in 2.2063s (16.3169GiB/s) | 20260505 17:06:29.164 | INFO | Downloaded 34GiB to GPU in 2.04499s (16.626GiB/s) | 20260505 17:06:34.293 | INFO | Analyzing chunk frames = [4871680,4872192) | 20260505 17:06:34.293 | INFO | MemInfo: { | 20260505 17:06:34.293 | INFO | "gpu" : " 17 GB", | 20260505 17:06:34.293 | INFO | "huge" : "", | 20260505 17:06:34.293 | INFO | "huge_shared" : " 27 GB", | 20260505 17:06:34.293 | INFO | "malloc" : " 14 GB", | 20260505 17:06:34.293 | INFO | "pinned" : " 29 GB" | 20260505 17:06:34.293 | INFO | } | 20260505 17:06:34.293 | INFO | Perf report for the last 5.12959 seconds: | 20260505 17:06:34.293 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:06:34.293 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:34.293 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:34.293 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:34.293 | INFO | ANALYSIS : 63.0%, 3.0, 97.7ms, 0 | 20260505 17:06:34.293 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:34.293 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:34.293 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:06:34.293 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:34.293 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:34.293 | INFO | Uploaded 36GiB to GPU in 2.22743s (16.1621GiB/s) | 20260505 17:06:34.293 | INFO | Downloaded 34GiB to GPU in 2.11868s (16.0477GiB/s) | 20260505 17:06:39.426 | INFO | Analyzing chunk frames = [4872192,4872704) | 20260505 17:06:39.426 | INFO | MemInfo: { | 20260505 17:06:39.426 | INFO | "gpu" : " 17 GB", | 20260505 17:06:39.426 | INFO | "huge" : "", | 20260505 17:06:39.426 | INFO | "huge_shared" : " 27 GB", | 20260505 17:06:39.426 | INFO | "malloc" : " 14 GB", | 20260505 17:06:39.426 | INFO | "pinned" : " 29 GB" | 20260505 17:06:39.426 | INFO | } | 20260505 17:06:39.426 | INFO | Perf report for the last 5.1325 seconds: | 20260505 17:06:39.426 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:06:39.426 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:39.426 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:39.426 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:39.426 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 17:06:39.426 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:39.426 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:39.426 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 17:06:39.426 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:39.426 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:39.426 | INFO | Uploaded 36GiB to GPU in 2.15828s (16.68GiB/s) | 20260505 17:06:39.426 | INFO | Downloaded 34GiB to GPU in 2.05973s (16.507GiB/s) | 20260505 17:06:44.571 | INFO | Analyzing chunk frames = [4872704,4873216) | 20260505 17:06:44.571 | INFO | MemInfo: { | 20260505 17:06:44.571 | INFO | "gpu" : " 17 GB", | 20260505 17:06:44.571 | INFO | "huge" : "", | 20260505 17:06:44.571 | INFO | "huge_shared" : " 27 GB", | 20260505 17:06:44.571 | INFO | "malloc" : " 14 GB", | 20260505 17:06:44.571 | INFO | "pinned" : " 29 GB" | 20260505 17:06:44.571 | INFO | } | 20260505 17:06:44.571 | INFO | Perf report for the last 5.14539 seconds: | 20260505 17:06:44.571 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:06:44.571 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:44.571 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:44.571 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:44.571 | INFO | ANALYSIS : 62.5%, 3.0, 97.7ms, 0 | 20260505 17:06:44.571 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:44.571 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:44.571 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 17:06:44.571 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:44.571 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:44.572 | INFO | Uploaded 36GiB to GPU in 2.14914s (16.7509GiB/s) | 20260505 17:06:44.572 | INFO | Downloaded 34GiB to GPU in 2.03627s (16.6972GiB/s) | 20260505 17:06:49.715 | INFO | Analyzing chunk frames = [4873216,4873728) | 20260505 17:06:49.715 | INFO | MemInfo: { | 20260505 17:06:49.715 | INFO | "gpu" : " 17 GB", | 20260505 17:06:49.715 | INFO | "huge" : "", | 20260505 17:06:49.715 | INFO | "huge_shared" : " 27 GB", | 20260505 17:06:49.715 | INFO | "malloc" : " 14 GB", | 20260505 17:06:49.715 | INFO | "pinned" : " 29 GB" | 20260505 17:06:49.715 | INFO | } | 20260505 17:06:49.715 | INFO | Perf report for the last 5.14394 seconds: | 20260505 17:06:49.715 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:06:49.715 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:49.715 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:49.715 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:49.715 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 17:06:49.715 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:49.715 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:49.715 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 17:06:49.715 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:49.715 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:49.715 | INFO | Uploaded 36GiB to GPU in 2.1425s (16.8028GiB/s) | 20260505 17:06:49.715 | INFO | Downloaded 34GiB to GPU in 2.17738s (15.6151GiB/s) | 20260505 17:06:54.855 | INFO | Analyzing chunk frames = [4873728,4874240) | 20260505 17:06:54.855 | INFO | MemInfo: { | 20260505 17:06:54.855 | INFO | "gpu" : " 17 GB", | 20260505 17:06:54.855 | INFO | "huge" : "", | 20260505 17:06:54.855 | INFO | "huge_shared" : " 27 GB", | 20260505 17:06:54.855 | INFO | "malloc" : " 14 GB", | 20260505 17:06:54.855 | INFO | "pinned" : " 29 GB" | 20260505 17:06:54.855 | INFO | } | 20260505 17:06:54.855 | INFO | Perf report for the last 5.14024 seconds: | 20260505 17:06:54.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:06:54.855 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:54.855 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:54.855 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:54.856 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 17:06:54.856 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:54.856 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:54.856 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 17:06:54.856 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:54.856 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:54.856 | INFO | Uploaded 36GiB to GPU in 2.20309s (16.3407GiB/s) | 20260505 17:06:54.856 | INFO | Downloaded 34GiB to GPU in 2.02348s (16.8028GiB/s) | 20260505 17:06:54.856 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 17:06:54.856 | INFO | Total mem usage: 15.3 GiB | 20260505 17:06:54.856 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 17:06:54.856 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.97%) | 20260505 17:06:54.856 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 17:06:54.856 | INFO | High Water Breakdown: | 20260505 17:06:54.856 | INFO | 61% : 8.66 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 17:06:54.856 | INFO | 38.8% : 5.51 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 17:06:54.856 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 17:06:54.856 | INFO | ------------------------------------------ | 20260505 17:06:54.856 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:06:54.856 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 17:06:54.856 | INFO | Total mem usage: 26.5 GiB | 20260505 17:06:54.856 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 17:06:54.856 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 17:06:54.856 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260505 17:06:54.856 | INFO | High Water Breakdown: | 20260505 17:06:54.856 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:06:54.856 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:06:54.856 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:06:54.856 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:06:54.856 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:06:54.856 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:06:54.856 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 17:06:54.856 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 17:06:54.856 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:06:54.856 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 17:06:54.856 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 17:06:54.856 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 17:06:54.856 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 17:06:54.856 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 17:06:54.856 | INFO | ------------------------------------------ | 20260505 17:06:54.856 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:06:54.856 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 17:06:54.856 | INFO | Total mem usage: 25.6 GiB | 20260505 17:06:54.856 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 17:06:54.856 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 17:06:54.856 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 17:06:54.856 | INFO | High Water Breakdown: | 20260505 17:06:54.856 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 17:06:54.856 | INFO | ------------------------------------------ | 20260505 17:06:54.856 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:06:54.856 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 17:06:54.856 | INFO | Total mem usage: 15.7 GiB | 20260505 17:06:54.856 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 17:06:54.856 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 17:06:54.856 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 17:06:54.856 | INFO | High Water Breakdown: | 20260505 17:06:54.856 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 17:06:54.856 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 17:06:54.856 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 17:06:54.856 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 17:06:54.856 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 17:06:54.856 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 17:06:54.856 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 17:06:54.856 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 17:06:54.856 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 17:06:54.856 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:06:54.856 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 17:06:54.856 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:06:54.856 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 17:06:54.856 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 17:06:54.856 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 17:06:54.856 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 17:06:54.856 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:06:54.857 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 17:06:54.857 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:06:54.857 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:06:54.857 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:06:54.857 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 17:06:54.857 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 17:06:54.857 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:06:54.857 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 17:06:54.857 | INFO | ------------------------------------------ | 20260505 17:06:54.857 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:06:54.857 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 17:06:54.857 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:06:54.857 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:06:54.857 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.51e+01, 1.37e+01 / 2.21e+00 | 20260505 17:06:54.857 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.01e-01, 1.06e-02 / 4.21e-03 | 20260505 17:06:54.857 | INFO | PerfReport: 11.76%, Baseline -- 3.71e+00 / 4.50e+00, 5.58e-01 / 1.44e-01 | 20260505 17:06:54.857 | INFO | PerfReport: 3.25%, Histogram -- 1.03e+00 / 1.53e+00, 1.88e-01 / 7.00e-02 | 20260505 17:06:54.857 | INFO | PerfReport: 15.99%, DME -- 4.04e+01 / 4.50e+01, 4.24e+00 / 2.26e+00 | 20260505 17:06:54.857 | INFO | PerfReport: 56.75%, FrameLabeling -- 1.79e+01 / 1.92e+01, 1.38e+00 / 1.06e+00 | 20260505 17:06:54.857 | INFO | PerfReport: 7.39%, PulseAccumulating -- 2.33e+00 / 2.94e+00, 4.52e-01 / 1.44e-01 | 20260505 17:06:54.857 | INFO | PerfReport: 3.39%, Metrics -- 1.07e+00 / 1.41e+00, 2.94e-01 / 1.45e-01 | 20260505 17:06:54.857 | INFO | PerfReport: 1.18%, PulseSerializing -- 3.73e-01 / 4.41e-01, 6.67e-02 / 3.83e-02 | 20260505 17:06:54.857 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 17:06:54.857 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:06:54.857 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:06:54.857 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.57e+01 / 0.00e+00 | 20260505 17:06:54.857 | INFO | PerfReport: 22.61%, Upload -- 2.27e+01 / 0.00e+00, 8.00e+00 / 0.00e+00 | 20260505 17:06:54.857 | INFO | PerfReport: 21.84%, Download -- 2.19e+01 / 0.00e+00, 7.85e+00 / 0.00e+00 | 20260505 17:06:54.857 | INFO | PerfReport: 31.52%, Compute -- 3.16e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 17:06:54.857 | INFO | PerfReport: 2.40%, LockUpload -- 2.41e+00 / 0.00e+00, 8.06e+00 / 0.00e+00 | 20260505 17:06:54.857 | INFO | PerfReport: 1.36%, LockDownload -- 1.36e+00 / 0.00e+00, 2.82e+00 / 0.00e+00 | 20260505 17:06:54.857 | INFO | PerfReport: 20.25%, LockCompute -- 2.03e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 17:06:54.857 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.53e-02 / 0.00e+00 | 20260505 17:06:54.997 | INFO | Starting a round of Baz IO | 20260505 17:06:58.180 | INFO | Populating RTMetrics report | 20260505 17:06:58.181 | INFO | RTMetrics report populated | 20260505 17:06:58.181 | INFO | Opening tmp RTMetrics json file | 20260505 17:06:58.191 | INFO | Serializing RTMetrics to json | 20260505 17:06:58.193 | INFO | Serialization done, swapping tmp file now... | 20260505 17:06:58.194 | INFO | RTMetrics done | 20260505 17:06:58.277 | INFO | Finished a round of Baz IO: Wrote 2.143 GiB in 3.279 seconds | 20260505 17:06:58.277 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.12% / 0.00% / 77.88% | 20260505 17:06:58.277 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.320387% / 3.97364e-06% / 9.13938e-05% | 20260505 17:06:58.277 | INFO | Baz IO overall: | 20260505 17:06:58.277 | INFO | BytesWritten : 2885608982137 | 20260505 17:06:58.277 | INFO | NumEvents : 1368800132590 | 20260505 17:06:59.994 | INFO | Analyzing chunk frames = [4874240,4874752) | 20260505 17:06:59.994 | INFO | MemInfo: { | 20260505 17:06:59.994 | INFO | "gpu" : " 17 GB", | 20260505 17:06:59.994 | INFO | "huge" : "", | 20260505 17:06:59.994 | INFO | "huge_shared" : " 27 GB", | 20260505 17:06:59.994 | INFO | "malloc" : " 14 GB", | 20260505 17:06:59.994 | INFO | "pinned" : " 29 GB" | 20260505 17:06:59.994 | INFO | } | 20260505 17:06:59.994 | INFO | Perf report for the last 5.1389 seconds: | 20260505 17:06:59.994 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:06:59.994 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:59.994 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:59.994 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:59.994 | INFO | ANALYSIS : 63.6%, 3.0, 99.1ms, 0 | 20260505 17:06:59.994 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:06:59.994 | INFO | RT_METRICS : 44.7%, 1.0, 23.4ms, 0 | 20260505 17:06:59.994 | INFO | PRE_HQ : 60.4%, 1.8, 56.1ms, 0 | 20260505 17:06:59.994 | INFO | BAZWRITER : 63.3%, 3.8, 124.8ms, 0 | 20260505 17:06:59.994 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 17:06:59.995 | INFO | Uploaded 36GiB to GPU in 2.16535s (16.6255GiB/s) | 20260505 17:06:59.995 | INFO | Downloaded 38GiB to GPU in 2.25658s (16.8397GiB/s) | 20260505 17:07:05.130 | INFO | Analyzing chunk frames = [4874752,4875264) | 20260505 17:07:05.130 | INFO | MemInfo: { | 20260505 17:07:05.130 | INFO | "gpu" : " 17 GB", | 20260505 17:07:05.130 | INFO | "huge" : "", | 20260505 17:07:05.130 | INFO | "huge_shared" : " 27 GB", | 20260505 17:07:05.130 | INFO | "malloc" : " 14 GB", | 20260505 17:07:05.131 | INFO | "pinned" : " 29 GB" | 20260505 17:07:05.131 | INFO | } | 20260505 17:07:05.131 | INFO | Perf report for the last 5.13633 seconds: | 20260505 17:07:05.131 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:07:05.131 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:05.131 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:05.131 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:05.131 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260505 17:07:05.131 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:05.131 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:05.131 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:07:05.131 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:05.131 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:05.131 | INFO | Uploaded 36GiB to GPU in 2.11503s (17.0211GiB/s) | 20260505 17:07:05.131 | INFO | Downloaded 34GiB to GPU in 2.03016s (16.7474GiB/s) | 20260505 17:07:10.272 | INFO | Analyzing chunk frames = [4875264,4875776) | 20260505 17:07:10.272 | INFO | MemInfo: { | 20260505 17:07:10.272 | INFO | "gpu" : " 17 GB", | 20260505 17:07:10.272 | INFO | "huge" : "", | 20260505 17:07:10.272 | INFO | "huge_shared" : " 27 GB", | 20260505 17:07:10.272 | INFO | "malloc" : " 14 GB", | 20260505 17:07:10.272 | INFO | "pinned" : " 29 GB" | 20260505 17:07:10.272 | INFO | } | 20260505 17:07:10.272 | INFO | Perf report for the last 5.14156 seconds: | 20260505 17:07:10.272 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:07:10.272 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:10.272 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:10.272 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:10.272 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260505 17:07:10.272 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:10.272 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:10.272 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:07:10.272 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:10.272 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:10.273 | INFO | Uploaded 36GiB to GPU in 2.20131s (16.3539GiB/s) | 20260505 17:07:10.273 | INFO | Downloaded 34GiB to GPU in 2.04477s (16.6278GiB/s) | 20260505 17:07:15.413 | INFO | Analyzing chunk frames = [4875776,4876288) | 20260505 17:07:15.413 | INFO | MemInfo: { | 20260505 17:07:15.413 | INFO | "gpu" : " 17 GB", | 20260505 17:07:15.413 | INFO | "huge" : "", | 20260505 17:07:15.413 | INFO | "huge_shared" : " 27 GB", | 20260505 17:07:15.414 | INFO | "malloc" : " 14 GB", | 20260505 17:07:15.414 | INFO | "pinned" : " 29 GB" | 20260505 17:07:15.414 | INFO | } | 20260505 17:07:15.414 | INFO | Perf report for the last 5.14178 seconds: | 20260505 17:07:15.414 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:07:15.414 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:15.414 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:15.414 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:15.414 | INFO | ANALYSIS : 62.8%, 3.0, 97.6ms, 0 | 20260505 17:07:15.414 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:15.414 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:15.414 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:07:15.414 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:15.414 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:15.414 | INFO | Uploaded 36GiB to GPU in 2.22351s (16.1906GiB/s) | 20260505 17:07:15.414 | INFO | Downloaded 34GiB to GPU in 2.1178s (16.0544GiB/s) | 20260505 17:07:20.542 | INFO | Analyzing chunk frames = [4876288,4876800) | 20260505 17:07:20.542 | INFO | MemInfo: { | 20260505 17:07:20.542 | INFO | "gpu" : " 17 GB", | 20260505 17:07:20.542 | INFO | "huge" : "", | 20260505 17:07:20.542 | INFO | "huge_shared" : " 27 GB", | 20260505 17:07:20.542 | INFO | "malloc" : " 14 GB", | 20260505 17:07:20.542 | INFO | "pinned" : " 29 GB" | 20260505 17:07:20.542 | INFO | } | 20260505 17:07:20.543 | INFO | Perf report for the last 5.12845 seconds: | 20260505 17:07:20.543 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:07:20.543 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:20.543 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:20.543 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:20.543 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260505 17:07:20.543 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:20.543 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:20.543 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:07:20.543 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:20.543 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:20.543 | INFO | Uploaded 36GiB to GPU in 2.14503s (16.783GiB/s) | 20260505 17:07:20.543 | INFO | Downloaded 34GiB to GPU in 2.06069s (16.4993GiB/s) | 20260505 17:07:25.677 | INFO | Analyzing chunk frames = [4876800,4877312) | 20260505 17:07:25.677 | INFO | MemInfo: { | 20260505 17:07:25.677 | INFO | "gpu" : " 17 GB", | 20260505 17:07:25.677 | INFO | "huge" : "", | 20260505 17:07:25.677 | INFO | "huge_shared" : " 27 GB", | 20260505 17:07:25.678 | INFO | "malloc" : " 14 GB", | 20260505 17:07:25.678 | INFO | "pinned" : " 29 GB" | 20260505 17:07:25.678 | INFO | } | 20260505 17:07:25.678 | INFO | Perf report for the last 5.13506 seconds: | 20260505 17:07:25.678 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:07:25.678 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:25.678 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:25.678 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:25.678 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 17:07:25.678 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:25.678 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:25.678 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 17:07:25.678 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:25.678 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:25.678 | INFO | Uploaded 36GiB to GPU in 2.16364s (16.6387GiB/s) | 20260505 17:07:25.678 | INFO | Downloaded 34GiB to GPU in 2.04606s (16.6173GiB/s) | 20260505 17:07:30.831 | INFO | Analyzing chunk frames = [4877312,4877824) | 20260505 17:07:30.831 | INFO | MemInfo: { | 20260505 17:07:30.831 | INFO | "gpu" : " 17 GB", | 20260505 17:07:30.831 | INFO | "huge" : "", | 20260505 17:07:30.831 | INFO | "huge_shared" : " 27 GB", | 20260505 17:07:30.831 | INFO | "malloc" : " 14 GB", | 20260505 17:07:30.831 | INFO | "pinned" : " 29 GB" | 20260505 17:07:30.831 | INFO | } | 20260505 17:07:30.831 | INFO | Perf report for the last 5.15375 seconds: | 20260505 17:07:30.831 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:07:30.831 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:30.831 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:30.831 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:30.831 | INFO | ANALYSIS : 63.1%, 3.0, 98.7ms, 0 | 20260505 17:07:30.831 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:30.831 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:30.832 | INFO | PRE_HQ : 28.7%, 1.0, 15.0ms, 0 | 20260505 17:07:30.832 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:30.832 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:30.832 | INFO | Uploaded 36GiB to GPU in 2.1165s (17.0092GiB/s) | 20260505 17:07:30.832 | INFO | Downloaded 34GiB to GPU in 2.17758s (15.6137GiB/s) | 20260505 17:07:35.974 | INFO | Analyzing chunk frames = [4877824,4878336) | 20260505 17:07:35.974 | INFO | MemInfo: { | 20260505 17:07:35.974 | INFO | "gpu" : " 17 GB", | 20260505 17:07:35.974 | INFO | "huge" : "", | 20260505 17:07:35.974 | INFO | "huge_shared" : " 27 GB", | 20260505 17:07:35.975 | INFO | "malloc" : " 14 GB", | 20260505 17:07:35.975 | INFO | "pinned" : " 29 GB" | 20260505 17:07:35.975 | INFO | } | 20260505 17:07:35.975 | INFO | Perf report for the last 5.14333 seconds: | 20260505 17:07:35.975 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:07:35.975 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:35.975 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:35.975 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:35.975 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 17:07:35.975 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:35.975 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:35.975 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 17:07:35.975 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:35.975 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:35.975 | INFO | Uploaded 36GiB to GPU in 2.19676s (16.3878GiB/s) | 20260505 17:07:35.975 | INFO | Downloaded 34GiB to GPU in 2.02492s (16.7908GiB/s) | 20260505 17:07:36.111 | INFO | Starting a round of Baz IO | 20260505 17:07:39.295 | INFO | Populating RTMetrics report | 20260505 17:07:39.295 | INFO | RTMetrics report populated | 20260505 17:07:39.295 | INFO | Opening tmp RTMetrics json file | 20260505 17:07:39.300 | INFO | Serializing RTMetrics to json | 20260505 17:07:39.302 | INFO | Serialization done, swapping tmp file now... | 20260505 17:07:39.302 | INFO | RTMetrics done | 20260505 17:07:39.390 | INFO | Finished a round of Baz IO: Wrote 2.144 GiB in 3.278 seconds | 20260505 17:07:39.390 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.12% / 0.00% / 77.88% | 20260505 17:07:39.390 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.320613% / 3.97364e-06% / 0.000103315% | 20260505 17:07:39.390 | INFO | Baz IO overall: | 20260505 17:07:39.390 | INFO | BytesWritten : 2887911225038 | 20260505 17:07:39.390 | INFO | NumEvents : 1369877110115 | 20260505 17:07:41.108 | INFO | Analyzing chunk frames = [4878336,4878848) | 20260505 17:07:41.108 | INFO | MemInfo: { | 20260505 17:07:41.108 | INFO | "gpu" : " 17 GB", | 20260505 17:07:41.108 | INFO | "huge" : "", | 20260505 17:07:41.108 | INFO | "huge_shared" : " 27 GB", | 20260505 17:07:41.108 | INFO | "malloc" : " 14 GB", | 20260505 17:07:41.108 | INFO | "pinned" : " 29 GB" | 20260505 17:07:41.108 | INFO | } | 20260505 17:07:41.108 | INFO | Perf report for the last 5.13374 seconds: | 20260505 17:07:41.108 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:07:41.108 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:41.108 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:41.109 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:41.109 | INFO | ANALYSIS : 63.6%, 3.0, 99.0ms, 0 | 20260505 17:07:41.109 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:41.109 | INFO | RT_METRICS : 44.7%, 1.0, 23.4ms, 0 | 20260505 17:07:41.109 | INFO | PRE_HQ : 60.6%, 1.8, 56.7ms, 0 | 20260505 17:07:41.109 | INFO | BAZWRITER : 63.3%, 3.9, 125.3ms, 0 | 20260505 17:07:41.109 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 17:07:41.109 | INFO | Uploaded 36GiB to GPU in 2.1395s (16.8263GiB/s) | 20260505 17:07:41.109 | INFO | Downloaded 38GiB to GPU in 2.23554s (16.9981GiB/s) | 20260505 17:07:46.242 | INFO | Analyzing chunk frames = [4878848,4879360) | 20260505 17:07:46.242 | INFO | MemInfo: { | 20260505 17:07:46.242 | INFO | "gpu" : " 17 GB", | 20260505 17:07:46.242 | INFO | "huge" : "", | 20260505 17:07:46.242 | INFO | "huge_shared" : " 27 GB", | 20260505 17:07:46.242 | INFO | "malloc" : " 14 GB", | 20260505 17:07:46.242 | INFO | "pinned" : " 29 GB" | 20260505 17:07:46.242 | INFO | } | 20260505 17:07:46.242 | INFO | Perf report for the last 5.13356 seconds: | 20260505 17:07:46.242 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:07:46.242 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:46.242 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:46.242 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:46.242 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 17:07:46.242 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:46.242 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:46.242 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 17:07:46.242 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:46.242 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:46.242 | INFO | Uploaded 36GiB to GPU in 2.13201s (16.8855GiB/s) | 20260505 17:07:46.242 | INFO | Downloaded 34GiB to GPU in 2.02535s (16.7872GiB/s) | 20260505 17:07:51.384 | INFO | Analyzing chunk frames = [4879360,4879872) | 20260505 17:07:51.384 | INFO | MemInfo: { | 20260505 17:07:51.384 | INFO | "gpu" : " 17 GB", | 20260505 17:07:51.384 | INFO | "huge" : "", | 20260505 17:07:51.384 | INFO | "huge_shared" : " 27 GB", | 20260505 17:07:51.384 | INFO | "malloc" : " 14 GB", | 20260505 17:07:51.384 | INFO | "pinned" : " 29 GB" | 20260505 17:07:51.384 | INFO | } | 20260505 17:07:51.384 | INFO | Perf report for the last 5.14176 seconds: | 20260505 17:07:51.384 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:07:51.384 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:51.384 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:51.384 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:51.384 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260505 17:07:51.384 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:51.384 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:51.384 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 17:07:51.384 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:51.384 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:51.384 | INFO | Uploaded 36GiB to GPU in 2.19549s (16.3973GiB/s) | 20260505 17:07:51.384 | INFO | Downloaded 34GiB to GPU in 2.02602s (16.7817GiB/s) | 20260505 17:07:56.522 | INFO | Analyzing chunk frames = [4879872,4880384) | 20260505 17:07:56.522 | INFO | MemInfo: { | 20260505 17:07:56.522 | INFO | "gpu" : " 17 GB", | 20260505 17:07:56.522 | INFO | "huge" : "", | 20260505 17:07:56.522 | INFO | "huge_shared" : " 27 GB", | 20260505 17:07:56.522 | INFO | "malloc" : " 14 GB", | 20260505 17:07:56.522 | INFO | "pinned" : " 29 GB" | 20260505 17:07:56.522 | INFO | } | 20260505 17:07:56.522 | INFO | Perf report for the last 5.13816 seconds: | 20260505 17:07:56.522 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:07:56.522 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:56.522 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:56.522 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:56.522 | INFO | ANALYSIS : 62.6%, 3.0, 97.3ms, 0 | 20260505 17:07:56.522 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:56.522 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:56.522 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 17:07:56.522 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:56.522 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:07:56.522 | INFO | Uploaded 36GiB to GPU in 2.2133s (16.2653GiB/s) | 20260505 17:07:56.522 | INFO | Downloaded 34GiB to GPU in 2.14024s (15.886GiB/s) | 20260505 17:08:01.669 | INFO | Analyzing chunk frames = [4880384,4880896) | 20260505 17:08:01.669 | INFO | MemInfo: { | 20260505 17:08:01.669 | INFO | "gpu" : " 17 GB", | 20260505 17:08:01.669 | INFO | "huge" : "", | 20260505 17:08:01.669 | INFO | "huge_shared" : " 27 GB", | 20260505 17:08:01.669 | INFO | "malloc" : " 14 GB", | 20260505 17:08:01.669 | INFO | "pinned" : " 29 GB" | 20260505 17:08:01.669 | INFO | } | 20260505 17:08:01.669 | INFO | Perf report for the last 5.14701 seconds: | 20260505 17:08:01.669 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:08:01.669 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:01.669 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:01.669 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:01.669 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260505 17:08:01.669 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:01.669 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:01.669 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 17:08:01.669 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:01.669 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:01.669 | INFO | Uploaded 36GiB to GPU in 2.15737s (16.687GiB/s) | 20260505 17:08:01.669 | INFO | Downloaded 34GiB to GPU in 2.06124s (16.4949GiB/s) | 20260505 17:08:06.806 | INFO | Analyzing chunk frames = [4880896,4881408) | 20260505 17:08:06.806 | INFO | MemInfo: { | 20260505 17:08:06.806 | INFO | "gpu" : " 17 GB", | 20260505 17:08:06.806 | INFO | "huge" : "", | 20260505 17:08:06.806 | INFO | "huge_shared" : " 27 GB", | 20260505 17:08:06.806 | INFO | "malloc" : " 14 GB", | 20260505 17:08:06.806 | INFO | "pinned" : " 29 GB" | 20260505 17:08:06.806 | INFO | } | 20260505 17:08:06.806 | INFO | Perf report for the last 5.13721 seconds: | 20260505 17:08:06.806 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:08:06.806 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:06.806 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:06.806 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:06.806 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 17:08:06.806 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:06.806 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:06.806 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 17:08:06.806 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:06.806 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:06.806 | INFO | Uploaded 36GiB to GPU in 2.13669s (16.8485GiB/s) | 20260505 17:08:06.806 | INFO | Downloaded 34GiB to GPU in 2.04328s (16.6399GiB/s) | 20260505 17:08:11.938 | INFO | Analyzing chunk frames = [4881408,4881920) | 20260505 17:08:11.938 | INFO | MemInfo: { | 20260505 17:08:11.938 | INFO | "gpu" : " 17 GB", | 20260505 17:08:11.938 | INFO | "huge" : "", | 20260505 17:08:11.938 | INFO | "huge_shared" : " 27 GB", | 20260505 17:08:11.938 | INFO | "malloc" : " 14 GB", | 20260505 17:08:11.938 | INFO | "pinned" : " 29 GB" | 20260505 17:08:11.938 | INFO | } | 20260505 17:08:11.938 | INFO | Perf report for the last 5.13173 seconds: | 20260505 17:08:11.938 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:08:11.938 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:11.938 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:11.938 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:11.938 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 17:08:11.938 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:11.938 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:11.938 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 17:08:11.938 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:11.938 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:11.938 | INFO | Uploaded 36GiB to GPU in 2.16916s (16.5963GiB/s) | 20260505 17:08:11.938 | INFO | Downloaded 34GiB to GPU in 2.19421s (15.4953GiB/s) | 20260505 17:08:17.085 | INFO | Analyzing chunk frames = [4881920,4882432) | 20260505 17:08:17.085 | INFO | MemInfo: { | 20260505 17:08:17.085 | INFO | "gpu" : " 17 GB", | 20260505 17:08:17.085 | INFO | "huge" : "", | 20260505 17:08:17.085 | INFO | "huge_shared" : " 27 GB", | 20260505 17:08:17.086 | INFO | "malloc" : " 14 GB", | 20260505 17:08:17.086 | INFO | "pinned" : " 29 GB" | 20260505 17:08:17.086 | INFO | } | 20260505 17:08:17.086 | INFO | Perf report for the last 5.14763 seconds: | 20260505 17:08:17.086 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:08:17.086 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:17.086 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:17.086 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:17.086 | INFO | ANALYSIS : 62.9%, 3.0, 98.4ms, 0 | 20260505 17:08:17.086 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:17.086 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:17.086 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 17:08:17.086 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:17.086 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:17.086 | INFO | Uploaded 36GiB to GPU in 2.16994s (16.5903GiB/s) | 20260505 17:08:17.086 | INFO | Downloaded 34GiB to GPU in 2.03195s (16.7327GiB/s) | 20260505 17:08:17.086 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 17:08:17.086 | INFO | Total mem usage: 15.3 GiB | 20260505 17:08:17.086 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 17:08:17.086 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.09%) | 20260505 17:08:17.086 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 17:08:17.086 | INFO | High Water Breakdown: | 20260505 17:08:17.086 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 17:08:17.086 | INFO | 38.9% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 17:08:17.086 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 17:08:17.086 | INFO | ------------------------------------------ | 20260505 17:08:17.086 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:08:17.086 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 17:08:17.086 | INFO | Total mem usage: 26.5 GiB | 20260505 17:08:17.086 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 17:08:17.086 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 17:08:17.086 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 17:08:17.086 | INFO | High Water Breakdown: | 20260505 17:08:17.086 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:08:17.086 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:08:17.086 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:08:17.086 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:08:17.086 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:08:17.086 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:08:17.086 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 17:08:17.086 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 17:08:17.086 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:08:17.086 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 17:08:17.087 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 17:08:17.087 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 17:08:17.087 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 17:08:17.087 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 17:08:17.087 | INFO | ------------------------------------------ | 20260505 17:08:17.087 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:08:17.087 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 17:08:17.087 | INFO | Total mem usage: 25.6 GiB | 20260505 17:08:17.087 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 17:08:17.087 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 17:08:17.087 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 17:08:17.087 | INFO | High Water Breakdown: | 20260505 17:08:17.087 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 17:08:17.087 | INFO | ------------------------------------------ | 20260505 17:08:17.087 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:08:17.087 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 17:08:17.087 | INFO | Total mem usage: 15.7 GiB | 20260505 17:08:17.087 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 17:08:17.087 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 17:08:17.087 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 17:08:17.087 | INFO | High Water Breakdown: | 20260505 17:08:17.087 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 17:08:17.087 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 17:08:17.087 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 17:08:17.087 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 17:08:17.087 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 17:08:17.087 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 17:08:17.087 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 17:08:17.087 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 17:08:17.087 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 17:08:17.087 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:08:17.087 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 17:08:17.087 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:08:17.087 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 17:08:17.087 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 17:08:17.087 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 17:08:17.087 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 17:08:17.087 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:08:17.087 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 17:08:17.087 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:08:17.087 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:08:17.087 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:08:17.087 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 17:08:17.087 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 17:08:17.087 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:08:17.087 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 17:08:17.087 | INFO | ------------------------------------------ | 20260505 17:08:17.087 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:08:17.087 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 17:08:17.087 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:08:17.087 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:08:17.087 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.50e+01, 1.34e+01 / 2.35e+00 | 20260505 17:08:17.087 | INFO | PerfReport: 0.29%, AggregateStats -- 9.26e-02 / 1.02e-01, 1.06e-02 / 4.08e-03 | 20260505 17:08:17.087 | INFO | PerfReport: 11.79%, Baseline -- 3.73e+00 / 4.52e+00, 5.76e-01 / 1.33e-01 | 20260505 17:08:17.087 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.51e+00, 1.92e-01 / 1.00e-01 | 20260505 17:08:17.087 | INFO | PerfReport: 15.99%, DME -- 4.05e+01 / 4.50e+01, 4.22e+00 / 2.34e+00 | 20260505 17:08:17.087 | INFO | PerfReport: 56.71%, FrameLabeling -- 1.79e+01 / 1.91e+01, 1.43e+00 / 1.05e+00 | 20260505 17:08:17.087 | INFO | PerfReport: 7.36%, PulseAccumulating -- 2.33e+00 / 2.92e+00, 4.54e-01 / 2.03e-01 | 20260505 17:08:17.087 | INFO | PerfReport: 3.39%, Metrics -- 1.07e+00 / 1.41e+00, 2.97e-01 / 1.70e-01 | 20260505 17:08:17.087 | INFO | PerfReport: 1.18%, PulseSerializing -- 3.73e-01 / 4.30e-01, 6.64e-02 / 4.18e-02 | 20260505 17:08:17.087 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 17:08:17.087 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:08:17.087 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:08:17.087 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.57e+01 / 0.00e+00 | 20260505 17:08:17.087 | INFO | PerfReport: 22.54%, Upload -- 2.26e+01 / 0.00e+00, 7.91e+00 / 0.00e+00 | 20260505 17:08:17.087 | INFO | PerfReport: 21.81%, Download -- 2.18e+01 / 0.00e+00, 7.85e+00 / 0.00e+00 | 20260505 17:08:17.087 | INFO | PerfReport: 31.61%, Compute -- 3.16e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 17:08:17.087 | INFO | PerfReport: 2.33%, LockUpload -- 2.33e+00 / 0.00e+00, 7.98e+00 / 0.00e+00 | 20260505 17:08:17.087 | INFO | PerfReport: 1.40%, LockDownload -- 1.40e+00 / 0.00e+00, 2.93e+00 / 0.00e+00 | 20260505 17:08:17.087 | INFO | PerfReport: 20.29%, LockCompute -- 2.03e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 17:08:17.087 | INFO | PerfReport: 0.01%, Misc -- 1.26e-02 / 0.00e+00, 1.35e-02 / 0.00e+00 | 20260505 17:08:17.222 | INFO | Starting a round of Baz IO | 20260505 17:08:20.402 | INFO | Populating RTMetrics report | 20260505 17:08:20.403 | INFO | RTMetrics report populated | 20260505 17:08:20.403 | INFO | Opening tmp RTMetrics json file | 20260505 17:08:20.412 | INFO | Serializing RTMetrics to json | 20260505 17:08:20.413 | INFO | Serialization done, swapping tmp file now... | 20260505 17:08:20.414 | INFO | RTMetrics done | 20260505 17:08:20.498 | INFO | Finished a round of Baz IO: Wrote 2.144 GiB in 3.276 seconds | 20260505 17:08:20.498 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.11% / 0.00% / 77.88% | 20260505 17:08:20.498 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.320121% / 1.19209e-05% / 7.15256e-05% | 20260505 17:08:20.498 | INFO | Baz IO overall: | 20260505 17:08:20.498 | INFO | BytesWritten : 2890213398471 | 20260505 17:08:20.498 | INFO | NumEvents : 1370954142604 | 20260505 17:08:22.223 | INFO | Analyzing chunk frames = [4882432,4882944) | 20260505 17:08:22.223 | INFO | MemInfo: { | 20260505 17:08:22.223 | INFO | "gpu" : " 17 GB", | 20260505 17:08:22.223 | INFO | "huge" : "", | 20260505 17:08:22.223 | INFO | "huge_shared" : " 27 GB", | 20260505 17:08:22.223 | INFO | "malloc" : " 14 GB", | 20260505 17:08:22.223 | INFO | "pinned" : " 29 GB" | 20260505 17:08:22.223 | INFO | } | 20260505 17:08:22.223 | INFO | Perf report for the last 5.13776 seconds: | 20260505 17:08:22.223 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:08:22.223 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:22.223 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:22.223 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:22.224 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260505 17:08:22.224 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:22.224 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260505 17:08:22.224 | INFO | PRE_HQ : 60.7%, 1.8, 56.4ms, 0 | 20260505 17:08:22.224 | INFO | BAZWRITER : 63.3%, 3.9, 126.4ms, 0 | 20260505 17:08:22.224 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260505 17:08:22.224 | INFO | Uploaded 36GiB to GPU in 2.16612s (16.6196GiB/s) | 20260505 17:08:22.224 | INFO | Downloaded 38GiB to GPU in 2.2422s (16.9476GiB/s) | 20260505 17:08:27.357 | INFO | Analyzing chunk frames = [4882944,4883456) | 20260505 17:08:27.357 | INFO | MemInfo: { | 20260505 17:08:27.357 | INFO | "gpu" : " 17 GB", | 20260505 17:08:27.357 | INFO | "huge" : "", | 20260505 17:08:27.357 | INFO | "huge_shared" : " 27 GB", | 20260505 17:08:27.357 | INFO | "malloc" : " 14 GB", | 20260505 17:08:27.357 | INFO | "pinned" : " 29 GB" | 20260505 17:08:27.357 | INFO | } | 20260505 17:08:27.357 | INFO | Perf report for the last 5.13375 seconds: | 20260505 17:08:27.357 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:08:27.357 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:27.357 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:27.357 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:27.357 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260505 17:08:27.357 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:27.357 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:27.357 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:08:27.357 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:27.357 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:27.357 | INFO | Uploaded 36GiB to GPU in 2.12951s (16.9053GiB/s) | 20260505 17:08:27.358 | INFO | Downloaded 34GiB to GPU in 2.0257s (16.7843GiB/s) | 20260505 17:08:32.491 | INFO | Analyzing chunk frames = [4883456,4883968) | 20260505 17:08:32.491 | INFO | MemInfo: { | 20260505 17:08:32.491 | INFO | "gpu" : " 17 GB", | 20260505 17:08:32.491 | INFO | "huge" : "", | 20260505 17:08:32.491 | INFO | "huge_shared" : " 27 GB", | 20260505 17:08:32.491 | INFO | "malloc" : " 14 GB", | 20260505 17:08:32.491 | INFO | "pinned" : " 29 GB" | 20260505 17:08:32.491 | INFO | } | 20260505 17:08:32.491 | INFO | Perf report for the last 5.134 seconds: | 20260505 17:08:32.491 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:08:32.491 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:32.491 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:32.491 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:32.491 | INFO | ANALYSIS : 62.9%, 3.0, 97.7ms, 0 | 20260505 17:08:32.491 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:32.491 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:32.491 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 17:08:32.491 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:32.491 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:32.491 | INFO | Uploaded 36GiB to GPU in 2.20178s (16.3504GiB/s) | 20260505 17:08:32.491 | INFO | Downloaded 34GiB to GPU in 2.0366s (16.6945GiB/s) | 20260505 17:08:37.635 | INFO | Analyzing chunk frames = [4883968,4884480) | 20260505 17:08:37.635 | INFO | MemInfo: { | 20260505 17:08:37.635 | INFO | "gpu" : " 17 GB", | 20260505 17:08:37.635 | INFO | "huge" : "", | 20260505 17:08:37.635 | INFO | "huge_shared" : " 27 GB", | 20260505 17:08:37.635 | INFO | "malloc" : " 14 GB", | 20260505 17:08:37.635 | INFO | "pinned" : " 29 GB" | 20260505 17:08:37.635 | INFO | } | 20260505 17:08:37.635 | INFO | Perf report for the last 5.14401 seconds: | 20260505 17:08:37.635 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:08:37.635 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:37.635 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:37.635 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:37.635 | INFO | ANALYSIS : 62.4%, 3.0, 97.0ms, 0 | 20260505 17:08:37.635 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:37.635 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:37.635 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:08:37.635 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:37.635 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:37.635 | INFO | Uploaded 36GiB to GPU in 2.15007s (16.7437GiB/s) | 20260505 17:08:37.635 | INFO | Downloaded 34GiB to GPU in 2.13228s (15.9454GiB/s) | 20260505 17:08:42.778 | INFO | Analyzing chunk frames = [4884480,4884992) | 20260505 17:08:42.778 | INFO | MemInfo: { | 20260505 17:08:42.778 | INFO | "gpu" : " 17 GB", | 20260505 17:08:42.778 | INFO | "huge" : "", | 20260505 17:08:42.778 | INFO | "huge_shared" : " 27 GB", | 20260505 17:08:42.778 | INFO | "malloc" : " 14 GB", | 20260505 17:08:42.778 | INFO | "pinned" : " 29 GB" | 20260505 17:08:42.778 | INFO | } | 20260505 17:08:42.778 | INFO | Perf report for the last 5.1432 seconds: | 20260505 17:08:42.778 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:08:42.778 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:42.778 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:42.778 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:42.778 | INFO | ANALYSIS : 62.1%, 3.0, 97.1ms, 0 | 20260505 17:08:42.778 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:42.778 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:42.779 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:08:42.779 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:42.779 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:42.779 | INFO | Uploaded 36GiB to GPU in 2.14677s (16.7694GiB/s) | 20260505 17:08:42.779 | INFO | Downloaded 34GiB to GPU in 2.06026s (16.5028GiB/s) | 20260505 17:08:47.922 | INFO | Analyzing chunk frames = [4884992,4885504) | 20260505 17:08:47.922 | INFO | MemInfo: { | 20260505 17:08:47.922 | INFO | "gpu" : " 17 GB", | 20260505 17:08:47.922 | INFO | "huge" : "", | 20260505 17:08:47.922 | INFO | "huge_shared" : " 27 GB", | 20260505 17:08:47.922 | INFO | "malloc" : " 14 GB", | 20260505 17:08:47.922 | INFO | "pinned" : " 29 GB" | 20260505 17:08:47.922 | INFO | } | 20260505 17:08:47.923 | INFO | Perf report for the last 5.14412 seconds: | 20260505 17:08:47.923 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:08:47.923 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:47.923 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:47.923 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:47.923 | INFO | ANALYSIS : 62.3%, 3.0, 97.4ms, 0 | 20260505 17:08:47.923 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:47.923 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:47.923 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:08:47.923 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:47.923 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:47.923 | INFO | Uploaded 36GiB to GPU in 2.13754s (16.8418GiB/s) | 20260505 17:08:47.923 | INFO | Downloaded 34GiB to GPU in 2.04223s (16.6484GiB/s) | 20260505 17:08:53.058 | INFO | Analyzing chunk frames = [4885504,4886016) | 20260505 17:08:53.058 | INFO | MemInfo: { | 20260505 17:08:53.058 | INFO | "gpu" : " 17 GB", | 20260505 17:08:53.058 | INFO | "huge" : "", | 20260505 17:08:53.058 | INFO | "huge_shared" : " 27 GB", | 20260505 17:08:53.058 | INFO | "malloc" : " 14 GB", | 20260505 17:08:53.058 | INFO | "pinned" : " 29 GB" | 20260505 17:08:53.058 | INFO | } | 20260505 17:08:53.059 | INFO | Perf report for the last 5.136 seconds: | 20260505 17:08:53.059 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:08:53.059 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:53.059 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:53.059 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:53.059 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 17:08:53.059 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:53.059 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:53.059 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 17:08:53.059 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:53.059 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:53.059 | INFO | Uploaded 36GiB to GPU in 2.12466s (16.9439GiB/s) | 20260505 17:08:53.059 | INFO | Downloaded 34GiB to GPU in 2.14013s (15.8869GiB/s) | 20260505 17:08:58.194 | INFO | Analyzing chunk frames = [4886016,4886528) | 20260505 17:08:58.194 | INFO | MemInfo: { | 20260505 17:08:58.194 | INFO | "gpu" : " 17 GB", | 20260505 17:08:58.194 | INFO | "huge" : "", | 20260505 17:08:58.194 | INFO | "huge_shared" : " 27 GB", | 20260505 17:08:58.194 | INFO | "malloc" : " 14 GB", | 20260505 17:08:58.194 | INFO | "pinned" : " 29 GB" | 20260505 17:08:58.194 | INFO | } | 20260505 17:08:58.194 | INFO | Perf report for the last 5.13559 seconds: | 20260505 17:08:58.194 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:08:58.194 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:58.194 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:58.194 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:58.194 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 17:08:58.194 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:58.194 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:58.194 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 17:08:58.194 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:58.194 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:08:58.194 | INFO | Uploaded 36GiB to GPU in 2.16794s (16.6056GiB/s) | 20260505 17:08:58.194 | INFO | Downloaded 34GiB to GPU in 2.03524s (16.7057GiB/s) | 20260505 17:08:58.327 | INFO | Starting a round of Baz IO | 20260505 17:09:01.530 | INFO | Populating RTMetrics report | 20260505 17:09:01.530 | INFO | RTMetrics report populated | 20260505 17:09:01.530 | INFO | Opening tmp RTMetrics json file | 20260505 17:09:01.539 | INFO | Serializing RTMetrics to json | 20260505 17:09:01.540 | INFO | Serialization done, swapping tmp file now... | 20260505 17:09:01.541 | INFO | RTMetrics done | 20260505 17:09:01.628 | INFO | Finished a round of Baz IO: Wrote 2.144 GiB in 3.3 seconds | 20260505 17:09:01.628 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.11% / 0.00% / 77.89% | 20260505 17:09:01.628 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.319258% / 3.97364e-06% / 0.000119209% | 20260505 17:09:01.628 | INFO | Baz IO overall: | 20260505 17:09:01.628 | INFO | BytesWritten : 2892515593756 | 20260505 17:09:01.628 | INFO | NumEvents : 1372031335452 | 20260505 17:09:03.343 | INFO | Analyzing chunk frames = [4886528,4887040) | 20260505 17:09:03.343 | INFO | MemInfo: { | 20260505 17:09:03.343 | INFO | "gpu" : " 17 GB", | 20260505 17:09:03.343 | INFO | "huge" : "", | 20260505 17:09:03.343 | INFO | "huge_shared" : " 27 GB", | 20260505 17:09:03.343 | INFO | "malloc" : " 14 GB", | 20260505 17:09:03.343 | INFO | "pinned" : " 29 GB" | 20260505 17:09:03.343 | INFO | } | 20260505 17:09:03.343 | INFO | Perf report for the last 5.14869 seconds: | 20260505 17:09:03.343 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:09:03.343 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:03.343 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:03.343 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:03.343 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 17:09:03.343 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:03.343 | INFO | RT_METRICS : 44.6%, 1.0, 23.5ms, 0 | 20260505 17:09:03.343 | INFO | PRE_HQ : 60.5%, 1.8, 56.6ms, 0 | 20260505 17:09:03.343 | INFO | BAZWRITER : 63.4%, 3.8, 124.9ms, 0 | 20260505 17:09:03.343 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 17:09:03.343 | INFO | Uploaded 36GiB to GPU in 2.1763s (16.5419GiB/s) | 20260505 17:09:03.343 | INFO | Downloaded 38GiB to GPU in 2.26417s (16.7832GiB/s) | 20260505 17:09:08.476 | INFO | Analyzing chunk frames = [4887040,4887552) | 20260505 17:09:08.476 | INFO | MemInfo: { | 20260505 17:09:08.476 | INFO | "gpu" : " 17 GB", | 20260505 17:09:08.476 | INFO | "huge" : "", | 20260505 17:09:08.476 | INFO | "huge_shared" : " 27 GB", | 20260505 17:09:08.476 | INFO | "malloc" : " 14 GB", | 20260505 17:09:08.476 | INFO | "pinned" : " 29 GB" | 20260505 17:09:08.476 | INFO | } | 20260505 17:09:08.476 | INFO | Perf report for the last 5.13349 seconds: | 20260505 17:09:08.476 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:09:08.476 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:08.476 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:08.476 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:08.476 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260505 17:09:08.476 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:08.476 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:08.476 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 17:09:08.476 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:08.476 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:08.477 | INFO | Uploaded 36GiB to GPU in 2.13073s (16.8956GiB/s) | 20260505 17:09:08.477 | INFO | Downloaded 34GiB to GPU in 2.02202s (16.8149GiB/s) | 20260505 17:09:13.615 | INFO | Analyzing chunk frames = [4887552,4888064) | 20260505 17:09:13.615 | INFO | MemInfo: { | 20260505 17:09:13.615 | INFO | "gpu" : " 17 GB", | 20260505 17:09:13.615 | INFO | "huge" : "", | 20260505 17:09:13.615 | INFO | "huge_shared" : " 27 GB", | 20260505 17:09:13.615 | INFO | "malloc" : " 14 GB", | 20260505 17:09:13.615 | INFO | "pinned" : " 29 GB" | 20260505 17:09:13.615 | INFO | } | 20260505 17:09:13.616 | INFO | Perf report for the last 5.13918 seconds: | 20260505 17:09:13.616 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:09:13.616 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:13.616 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:13.616 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:13.616 | INFO | ANALYSIS : 62.6%, 3.0, 97.4ms, 0 | 20260505 17:09:13.616 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:13.616 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:13.616 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 17:09:13.616 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:13.616 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:13.616 | INFO | Uploaded 36GiB to GPU in 2.19688s (16.3869GiB/s) | 20260505 17:09:13.616 | INFO | Downloaded 34GiB to GPU in 2.03622s (16.6976GiB/s) | 20260505 17:09:18.753 | INFO | Analyzing chunk frames = [4888064,4888576) | 20260505 17:09:18.753 | INFO | MemInfo: { | 20260505 17:09:18.753 | INFO | "gpu" : " 17 GB", | 20260505 17:09:18.753 | INFO | "huge" : "", | 20260505 17:09:18.753 | INFO | "huge_shared" : " 27 GB", | 20260505 17:09:18.753 | INFO | "malloc" : " 14 GB", | 20260505 17:09:18.754 | INFO | "pinned" : " 29 GB" | 20260505 17:09:18.754 | INFO | } | 20260505 17:09:18.754 | INFO | Perf report for the last 5.13812 seconds: | 20260505 17:09:18.754 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:09:18.754 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:18.754 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:18.754 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:18.754 | INFO | ANALYSIS : 62.6%, 3.0, 97.2ms, 0 | 20260505 17:09:18.754 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:18.754 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:18.754 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:09:18.754 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:18.754 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:18.754 | INFO | Uploaded 36GiB to GPU in 2.16784s (16.6064GiB/s) | 20260505 17:09:18.754 | INFO | Downloaded 34GiB to GPU in 2.14209s (15.8724GiB/s) | 20260505 17:09:23.892 | INFO | Analyzing chunk frames = [4888576,4889088) | 20260505 17:09:23.892 | INFO | MemInfo: { | 20260505 17:09:23.892 | INFO | "gpu" : " 17 GB", | 20260505 17:09:23.892 | INFO | "huge" : "", | 20260505 17:09:23.892 | INFO | "huge_shared" : " 27 GB", | 20260505 17:09:23.892 | INFO | "malloc" : " 14 GB", | 20260505 17:09:23.892 | INFO | "pinned" : " 29 GB" | 20260505 17:09:23.892 | INFO | } | 20260505 17:09:23.892 | INFO | Perf report for the last 5.13845 seconds: | 20260505 17:09:23.892 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:09:23.892 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:23.892 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:23.892 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:23.892 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 17:09:23.892 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:23.892 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:23.892 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 17:09:23.892 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:23.892 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:23.892 | INFO | Uploaded 36GiB to GPU in 2.1565s (16.6937GiB/s) | 20260505 17:09:23.892 | INFO | Downloaded 34GiB to GPU in 2.05954s (16.5085GiB/s) | 20260505 17:09:29.033 | INFO | Analyzing chunk frames = [4889088,4889600) | 20260505 17:09:29.033 | INFO | MemInfo: { | 20260505 17:09:29.033 | INFO | "gpu" : " 17 GB", | 20260505 17:09:29.033 | INFO | "huge" : "", | 20260505 17:09:29.033 | INFO | "huge_shared" : " 27 GB", | 20260505 17:09:29.033 | INFO | "malloc" : " 14 GB", | 20260505 17:09:29.033 | INFO | "pinned" : " 29 GB" | 20260505 17:09:29.033 | INFO | } | 20260505 17:09:29.034 | INFO | Perf report for the last 5.14139 seconds: | 20260505 17:09:29.034 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:09:29.034 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:29.034 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:29.034 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:29.034 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260505 17:09:29.034 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:29.034 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:29.034 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:09:29.034 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:29.034 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:29.034 | INFO | Uploaded 36GiB to GPU in 2.14191s (16.8074GiB/s) | 20260505 17:09:29.034 | INFO | Downloaded 34GiB to GPU in 2.03616s (16.6981GiB/s) | 20260505 17:09:34.176 | INFO | Analyzing chunk frames = [4889600,4890112) | 20260505 17:09:34.176 | INFO | MemInfo: { | 20260505 17:09:34.176 | INFO | "gpu" : " 17 GB", | 20260505 17:09:34.176 | INFO | "huge" : "", | 20260505 17:09:34.176 | INFO | "huge_shared" : " 27 GB", | 20260505 17:09:34.176 | INFO | "malloc" : " 14 GB", | 20260505 17:09:34.176 | INFO | "pinned" : " 29 GB" | 20260505 17:09:34.176 | INFO | } | 20260505 17:09:34.176 | INFO | Perf report for the last 5.14294 seconds: | 20260505 17:09:34.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:09:34.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:34.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:34.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:34.177 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 17:09:34.177 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:34.177 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:34.177 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 17:09:34.177 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:34.177 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:34.177 | INFO | Uploaded 36GiB to GPU in 2.12136s (16.9703GiB/s) | 20260505 17:09:34.177 | INFO | Downloaded 34GiB to GPU in 2.14102s (15.8802GiB/s) | 20260505 17:09:39.304 | INFO | Analyzing chunk frames = [4890112,4890624) | 20260505 17:09:39.304 | INFO | MemInfo: { | 20260505 17:09:39.304 | INFO | "gpu" : " 17 GB", | 20260505 17:09:39.304 | INFO | "huge" : "", | 20260505 17:09:39.304 | INFO | "huge_shared" : " 27 GB", | 20260505 17:09:39.305 | INFO | "malloc" : " 14 GB", | 20260505 17:09:39.305 | INFO | "pinned" : " 29 GB" | 20260505 17:09:39.305 | INFO | } | 20260505 17:09:39.305 | INFO | Perf report for the last 5.12815 seconds: | 20260505 17:09:39.305 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:09:39.305 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:39.305 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:39.305 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:39.305 | INFO | ANALYSIS : 63.1%, 3.0, 98.3ms, 0 | 20260505 17:09:39.305 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:39.305 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:39.305 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 17:09:39.305 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:39.305 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:39.305 | INFO | Uploaded 36GiB to GPU in 2.18487s (16.4769GiB/s) | 20260505 17:09:39.305 | INFO | Downloaded 34GiB to GPU in 2.02346s (16.8029GiB/s) | 20260505 17:09:39.305 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 17:09:39.305 | INFO | Total mem usage: 15.3 GiB | 20260505 17:09:39.305 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 17:09:39.305 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.94%) | 20260505 17:09:39.305 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 17:09:39.305 | INFO | High Water Breakdown: | 20260505 17:09:39.305 | INFO | 61% : 8.67 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 17:09:39.305 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 17:09:39.305 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 17:09:39.305 | INFO | ------------------------------------------ | 20260505 17:09:39.305 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:09:39.305 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 17:09:39.305 | INFO | Total mem usage: 26.5 GiB | 20260505 17:09:39.305 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 17:09:39.305 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 17:09:39.305 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 17:09:39.305 | INFO | High Water Breakdown: | 20260505 17:09:39.305 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:09:39.305 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:09:39.305 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:09:39.305 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:09:39.306 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:09:39.306 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:09:39.306 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 17:09:39.306 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 17:09:39.306 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:09:39.306 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 17:09:39.306 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 17:09:39.306 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 17:09:39.306 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 17:09:39.306 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 17:09:39.306 | INFO | ------------------------------------------ | 20260505 17:09:39.306 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:09:39.306 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 17:09:39.306 | INFO | Total mem usage: 25.6 GiB | 20260505 17:09:39.306 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 17:09:39.306 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 17:09:39.306 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 17:09:39.306 | INFO | High Water Breakdown: | 20260505 17:09:39.306 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 17:09:39.306 | INFO | ------------------------------------------ | 20260505 17:09:39.306 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:09:39.306 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 17:09:39.306 | INFO | Total mem usage: 15.7 GiB | 20260505 17:09:39.306 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 17:09:39.306 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 17:09:39.306 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 17:09:39.306 | INFO | High Water Breakdown: | 20260505 17:09:39.306 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 17:09:39.306 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 17:09:39.306 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 17:09:39.306 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 17:09:39.306 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 17:09:39.306 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 17:09:39.306 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 17:09:39.306 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 17:09:39.306 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 17:09:39.306 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:09:39.306 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 17:09:39.306 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:09:39.306 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 17:09:39.306 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 17:09:39.306 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 17:09:39.306 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 17:09:39.306 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:09:39.306 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 17:09:39.306 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:09:39.306 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:09:39.306 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:09:39.306 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 17:09:39.306 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 17:09:39.306 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:09:39.306 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 17:09:39.306 | INFO | ------------------------------------------ | 20260505 17:09:39.306 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:09:39.306 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 17:09:39.306 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:09:39.306 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:09:39.306 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.50e+01, 1.37e+01 / 2.19e+00 | 20260505 17:09:39.306 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 1.02e-01, 1.04e-02 / 4.18e-03 | 20260505 17:09:39.306 | INFO | PerfReport: 11.81%, Baseline -- 3.74e+00 / 4.53e+00, 5.83e-01 / 1.56e-01 | 20260505 17:09:39.306 | INFO | PerfReport: 3.30%, Histogram -- 1.04e+00 / 1.52e+00, 1.97e-01 / 8.91e-02 | 20260505 17:09:39.306 | INFO | PerfReport: 15.94%, DME -- 4.04e+01 / 4.47e+01, 4.31e+00 / 2.17e+00 | 20260505 17:09:39.306 | INFO | PerfReport: 56.76%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.47e+00 / 1.12e+00 | 20260505 17:09:39.306 | INFO | PerfReport: 7.33%, PulseAccumulating -- 2.32e+00 / 2.90e+00, 4.46e-01 / 1.69e-01 | 20260505 17:09:39.306 | INFO | PerfReport: 3.38%, Metrics -- 1.07e+00 / 1.40e+00, 2.97e-01 / 1.60e-01 | 20260505 17:09:39.306 | INFO | PerfReport: 1.18%, PulseSerializing -- 3.73e-01 / 4.30e-01, 6.69e-02 / 3.91e-02 | 20260505 17:09:39.306 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 17:09:39.306 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:09:39.306 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:09:39.306 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.56e+01 / 0.00e+00 | 20260505 17:09:39.306 | INFO | PerfReport: 22.46%, Upload -- 2.25e+01 / 0.00e+00, 7.79e+00 / 0.00e+00 | 20260505 17:09:39.306 | INFO | PerfReport: 21.77%, Download -- 2.18e+01 / 0.00e+00, 7.80e+00 / 0.00e+00 | 20260505 17:09:39.306 | INFO | PerfReport: 31.66%, Compute -- 3.17e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 17:09:39.306 | INFO | PerfReport: 2.20%, LockUpload -- 2.20e+00 / 0.00e+00, 7.88e+00 / 0.00e+00 | 20260505 17:09:39.306 | INFO | PerfReport: 1.46%, LockDownload -- 1.46e+00 / 0.00e+00, 3.00e+00 / 0.00e+00 | 20260505 17:09:39.306 | INFO | PerfReport: 20.42%, LockCompute -- 2.04e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260505 17:09:39.306 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.69e-02 / 0.00e+00 | 20260505 17:09:39.433 | INFO | Starting a round of Baz IO | 20260505 17:09:42.646 | INFO | Populating RTMetrics report | 20260505 17:09:42.646 | INFO | RTMetrics report populated | 20260505 17:09:42.646 | INFO | Opening tmp RTMetrics json file | 20260505 17:09:42.655 | INFO | Serializing RTMetrics to json | 20260505 17:09:42.656 | INFO | Serialization done, swapping tmp file now... | 20260505 17:09:42.656 | INFO | RTMetrics done | 20260505 17:09:42.742 | INFO | Finished a round of Baz IO: Wrote 2.145 GiB in 3.308 seconds | 20260505 17:09:42.742 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.11% / 0.00% / 77.89% | 20260505 17:09:42.742 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.319008% / 0% / 8.74201e-05% | 20260505 17:09:42.742 | INFO | Baz IO overall: | 20260505 17:09:42.742 | INFO | BytesWritten : 2894818339299 | 20260505 17:09:42.742 | INFO | NumEvents : 1373108984212 | 20260505 17:09:44.440 | INFO | Analyzing chunk frames = [4890624,4891136) | 20260505 17:09:44.440 | INFO | MemInfo: { | 20260505 17:09:44.440 | INFO | "gpu" : " 17 GB", | 20260505 17:09:44.440 | INFO | "huge" : "", | 20260505 17:09:44.440 | INFO | "huge_shared" : " 27 GB", | 20260505 17:09:44.440 | INFO | "malloc" : " 14 GB", | 20260505 17:09:44.440 | INFO | "pinned" : " 29 GB" | 20260505 17:09:44.440 | INFO | } | 20260505 17:09:44.440 | INFO | Perf report for the last 5.13511 seconds: | 20260505 17:09:44.440 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:09:44.440 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:44.440 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:44.440 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:44.440 | INFO | ANALYSIS : 64.0%, 3.0, 99.6ms, 0 | 20260505 17:09:44.440 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:44.440 | INFO | RT_METRICS : 44.9%, 1.0, 23.5ms, 0 | 20260505 17:09:44.440 | INFO | PRE_HQ : 61.0%, 1.8, 56.4ms, 0 | 20260505 17:09:44.440 | INFO | BAZWRITER : 63.7%, 3.8, 123.6ms, 0 | 20260505 17:09:44.440 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 17:09:44.440 | INFO | Uploaded 36GiB to GPU in 2.19391s (16.4091GiB/s) | 20260505 17:09:44.440 | INFO | Downloaded 38GiB to GPU in 2.25838s (16.8263GiB/s) | 20260505 17:09:49.590 | INFO | Analyzing chunk frames = [4891136,4891648) | 20260505 17:09:49.590 | INFO | MemInfo: { | 20260505 17:09:49.590 | INFO | "gpu" : " 17 GB", | 20260505 17:09:49.590 | INFO | "huge" : "", | 20260505 17:09:49.590 | INFO | "huge_shared" : " 27 GB", | 20260505 17:09:49.590 | INFO | "malloc" : " 14 GB", | 20260505 17:09:49.590 | INFO | "pinned" : " 29 GB" | 20260505 17:09:49.590 | INFO | } | 20260505 17:09:49.590 | INFO | Perf report for the last 5.15067 seconds: | 20260505 17:09:49.590 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:09:49.590 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:49.590 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:49.590 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:49.590 | INFO | ANALYSIS : 63.2%, 3.0, 98.9ms, 0 | 20260505 17:09:49.590 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:49.591 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:49.591 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 17:09:49.591 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:49.591 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:49.591 | INFO | Uploaded 36GiB to GPU in 2.20422s (16.3323GiB/s) | 20260505 17:09:49.591 | INFO | Downloaded 34GiB to GPU in 2.05216s (16.5679GiB/s) | 20260505 17:09:54.736 | INFO | Analyzing chunk frames = [4891648,4892160) | 20260505 17:09:54.736 | INFO | MemInfo: { | 20260505 17:09:54.736 | INFO | "gpu" : " 17 GB", | 20260505 17:09:54.736 | INFO | "huge" : "", | 20260505 17:09:54.736 | INFO | "huge_shared" : " 27 GB", | 20260505 17:09:54.736 | INFO | "malloc" : " 14 GB", | 20260505 17:09:54.736 | INFO | "pinned" : " 29 GB" | 20260505 17:09:54.736 | INFO | } | 20260505 17:09:54.737 | INFO | Perf report for the last 5.14612 seconds: | 20260505 17:09:54.737 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:09:54.737 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:54.737 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:54.737 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:54.737 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260505 17:09:54.737 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:54.737 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:54.737 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260505 17:09:54.737 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:54.737 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:54.737 | INFO | Uploaded 36GiB to GPU in 2.20171s (16.3509GiB/s) | 20260505 17:09:54.737 | INFO | Downloaded 34GiB to GPU in 2.03636s (16.6965GiB/s) | 20260505 17:09:59.870 | INFO | Analyzing chunk frames = [4892160,4892672) | 20260505 17:09:59.870 | INFO | MemInfo: { | 20260505 17:09:59.870 | INFO | "gpu" : " 17 GB", | 20260505 17:09:59.870 | INFO | "huge" : "", | 20260505 17:09:59.870 | INFO | "huge_shared" : " 27 GB", | 20260505 17:09:59.870 | INFO | "malloc" : " 14 GB", | 20260505 17:09:59.870 | INFO | "pinned" : " 29 GB" | 20260505 17:09:59.870 | INFO | } | 20260505 17:09:59.870 | INFO | Perf report for the last 5.1334 seconds: | 20260505 17:09:59.870 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:09:59.870 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:59.870 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:59.870 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:59.870 | INFO | ANALYSIS : 62.5%, 3.0, 97.0ms, 0 | 20260505 17:09:59.870 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:59.870 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:59.870 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:09:59.870 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:59.870 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:09:59.870 | INFO | Uploaded 36GiB to GPU in 2.19724s (16.3842GiB/s) | 20260505 17:09:59.870 | INFO | Downloaded 34GiB to GPU in 2.04449s (16.63GiB/s) | 20260505 17:10:05.003 | INFO | Analyzing chunk frames = [4892672,4893184) | 20260505 17:10:05.003 | INFO | MemInfo: { | 20260505 17:10:05.003 | INFO | "gpu" : " 17 GB", | 20260505 17:10:05.003 | INFO | "huge" : "", | 20260505 17:10:05.003 | INFO | "huge_shared" : " 27 GB", | 20260505 17:10:05.003 | INFO | "malloc" : " 14 GB", | 20260505 17:10:05.003 | INFO | "pinned" : " 29 GB" | 20260505 17:10:05.003 | INFO | } | 20260505 17:10:05.003 | INFO | Perf report for the last 5.13348 seconds: | 20260505 17:10:05.004 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:10:05.004 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:05.004 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:05.004 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:05.004 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 17:10:05.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:05.004 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:05.004 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:10:05.004 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:05.004 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:05.004 | INFO | Uploaded 36GiB to GPU in 2.14885s (16.7531GiB/s) | 20260505 17:10:05.004 | INFO | Downloaded 34GiB to GPU in 2.05794s (16.5214GiB/s) | 20260505 17:10:10.147 | INFO | Analyzing chunk frames = [4893184,4893696) | 20260505 17:10:10.147 | INFO | MemInfo: { | 20260505 17:10:10.147 | INFO | "gpu" : " 17 GB", | 20260505 17:10:10.147 | INFO | "huge" : "", | 20260505 17:10:10.147 | INFO | "huge_shared" : " 27 GB", | 20260505 17:10:10.147 | INFO | "malloc" : " 14 GB", | 20260505 17:10:10.147 | INFO | "pinned" : " 29 GB" | 20260505 17:10:10.147 | INFO | } | 20260505 17:10:10.147 | INFO | Perf report for the last 5.14375 seconds: | 20260505 17:10:10.147 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:10:10.147 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:10.147 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:10.147 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:10.147 | INFO | ANALYSIS : 62.4%, 3.0, 97.6ms, 0 | 20260505 17:10:10.147 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:10.147 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:10.147 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 17:10:10.147 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:10.147 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:10.147 | INFO | Uploaded 36GiB to GPU in 2.13773s (16.8403GiB/s) | 20260505 17:10:10.148 | INFO | Downloaded 34GiB to GPU in 2.03861s (16.678GiB/s) | 20260505 17:10:15.289 | INFO | Analyzing chunk frames = [4893696,4894208) | 20260505 17:10:15.289 | INFO | MemInfo: { | 20260505 17:10:15.289 | INFO | "gpu" : " 17 GB", | 20260505 17:10:15.289 | INFO | "huge" : "", | 20260505 17:10:15.289 | INFO | "huge_shared" : " 27 GB", | 20260505 17:10:15.289 | INFO | "malloc" : " 14 GB", | 20260505 17:10:15.289 | INFO | "pinned" : " 29 GB" | 20260505 17:10:15.289 | INFO | } | 20260505 17:10:15.290 | INFO | Perf report for the last 5.14243 seconds: | 20260505 17:10:15.290 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:10:15.290 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:15.290 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:15.290 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:15.290 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 17:10:15.290 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:15.290 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:15.290 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 17:10:15.290 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:15.290 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:15.290 | INFO | Uploaded 36GiB to GPU in 2.15003s (16.744GiB/s) | 20260505 17:10:15.290 | INFO | Downloaded 34GiB to GPU in 2.16741s (15.6869GiB/s) | 20260505 17:10:20.430 | INFO | Analyzing chunk frames = [4894208,4894720) | 20260505 17:10:20.430 | INFO | MemInfo: { | 20260505 17:10:20.430 | INFO | "gpu" : " 17 GB", | 20260505 17:10:20.430 | INFO | "huge" : "", | 20260505 17:10:20.430 | INFO | "huge_shared" : " 27 GB", | 20260505 17:10:20.430 | INFO | "malloc" : " 14 GB", | 20260505 17:10:20.430 | INFO | "pinned" : " 29 GB" | 20260505 17:10:20.430 | INFO | } | 20260505 17:10:20.430 | INFO | Perf report for the last 5.14028 seconds: | 20260505 17:10:20.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:10:20.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:20.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:20.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:20.430 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 17:10:20.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:20.430 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:20.430 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 17:10:20.430 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:20.430 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:20.430 | INFO | Uploaded 36GiB to GPU in 2.19753s (16.382GiB/s) | 20260505 17:10:20.430 | INFO | Downloaded 34GiB to GPU in 2.0239s (16.7992GiB/s) | 20260505 17:10:20.568 | INFO | Starting a round of Baz IO | 20260505 17:10:23.761 | INFO | Populating RTMetrics report | 20260505 17:10:23.761 | INFO | RTMetrics report populated | 20260505 17:10:23.761 | INFO | Opening tmp RTMetrics json file | 20260505 17:10:23.771 | INFO | Serializing RTMetrics to json | 20260505 17:10:23.773 | INFO | Serialization done, swapping tmp file now... | 20260505 17:10:23.774 | INFO | RTMetrics done | 20260505 17:10:23.855 | INFO | Finished a round of Baz IO: Wrote 2.144 GiB in 3.287 seconds | 20260505 17:10:23.855 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.11% / 0.00% / 77.89% | 20260505 17:10:23.855 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.317931% / 0% / 9.53674e-05% | 20260505 17:10:23.855 | INFO | Baz IO overall: | 20260505 17:10:23.855 | INFO | BytesWritten : 2897120683871 | 20260505 17:10:23.855 | INFO | NumEvents : 1374186156588 | 20260505 17:10:25.560 | INFO | Analyzing chunk frames = [4894720,4895232) | 20260505 17:10:25.560 | INFO | MemInfo: { | 20260505 17:10:25.560 | INFO | "gpu" : " 17 GB", | 20260505 17:10:25.560 | INFO | "huge" : "", | 20260505 17:10:25.560 | INFO | "huge_shared" : " 27 GB", | 20260505 17:10:25.560 | INFO | "malloc" : " 14 GB", | 20260505 17:10:25.560 | INFO | "pinned" : " 29 GB" | 20260505 17:10:25.560 | INFO | } | 20260505 17:10:25.560 | INFO | Perf report for the last 5.13013 seconds: | 20260505 17:10:25.560 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:10:25.560 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:25.560 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:25.560 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:25.560 | INFO | ANALYSIS : 63.8%, 3.0, 99.2ms, 0 | 20260505 17:10:25.560 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:25.560 | INFO | RT_METRICS : 44.9%, 1.0, 23.5ms, 0 | 20260505 17:10:25.560 | INFO | PRE_HQ : 60.4%, 1.8, 56.3ms, 0 | 20260505 17:10:25.560 | INFO | BAZWRITER : 63.5%, 3.8, 125.1ms, 0 | 20260505 17:10:25.560 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 17:10:25.560 | INFO | Uploaded 36GiB to GPU in 2.18108s (16.5056GiB/s) | 20260505 17:10:25.560 | INFO | Downloaded 38GiB to GPU in 2.2655s (16.7734GiB/s) | 20260505 17:10:30.699 | INFO | Analyzing chunk frames = [4895232,4895744) | 20260505 17:10:30.699 | INFO | MemInfo: { | 20260505 17:10:30.699 | INFO | "gpu" : " 17 GB", | 20260505 17:10:30.699 | INFO | "huge" : "", | 20260505 17:10:30.699 | INFO | "huge_shared" : " 27 GB", | 20260505 17:10:30.699 | INFO | "malloc" : " 14 GB", | 20260505 17:10:30.699 | INFO | "pinned" : " 29 GB" | 20260505 17:10:30.699 | INFO | } | 20260505 17:10:30.699 | INFO | Perf report for the last 5.1391 seconds: | 20260505 17:10:30.699 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:10:30.699 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:30.699 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:30.699 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:30.699 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 17:10:30.699 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:30.699 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:30.699 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:10:30.699 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:30.699 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:30.699 | INFO | Uploaded 36GiB to GPU in 2.13627s (16.8518GiB/s) | 20260505 17:10:30.700 | INFO | Downloaded 34GiB to GPU in 2.03793s (16.6836GiB/s) | 20260505 17:10:35.853 | INFO | Analyzing chunk frames = [4895744,4896256) | 20260505 17:10:35.853 | INFO | MemInfo: { | 20260505 17:10:35.853 | INFO | "gpu" : " 17 GB", | 20260505 17:10:35.853 | INFO | "huge" : "", | 20260505 17:10:35.853 | INFO | "huge_shared" : " 27 GB", | 20260505 17:10:35.853 | INFO | "malloc" : " 14 GB", | 20260505 17:10:35.853 | INFO | "pinned" : " 29 GB" | 20260505 17:10:35.853 | INFO | } | 20260505 17:10:35.854 | INFO | Perf report for the last 5.15433 seconds: | 20260505 17:10:35.854 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:10:35.854 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:35.854 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:35.854 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:35.854 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 17:10:35.854 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:35.854 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:35.854 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 17:10:35.854 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:35.854 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:35.854 | INFO | Uploaded 36GiB to GPU in 2.18954s (16.4418GiB/s) | 20260505 17:10:35.854 | INFO | Downloaded 34GiB to GPU in 2.02933s (16.7543GiB/s) | 20260505 17:10:40.992 | INFO | Analyzing chunk frames = [4896256,4896768) | 20260505 17:10:40.992 | INFO | MemInfo: { | 20260505 17:10:40.992 | INFO | "gpu" : " 17 GB", | 20260505 17:10:40.993 | INFO | "huge" : "", | 20260505 17:10:40.993 | INFO | "huge_shared" : " 27 GB", | 20260505 17:10:40.993 | INFO | "malloc" : " 14 GB", | 20260505 17:10:40.993 | INFO | "pinned" : " 29 GB" | 20260505 17:10:40.993 | INFO | } | 20260505 17:10:40.993 | INFO | Perf report for the last 5.13919 seconds: | 20260505 17:10:40.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:10:40.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:40.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:40.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:40.993 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260505 17:10:40.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:40.993 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:40.993 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:10:40.993 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:40.993 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:40.993 | INFO | Uploaded 36GiB to GPU in 2.16813s (16.6042GiB/s) | 20260505 17:10:40.993 | INFO | Downloaded 34GiB to GPU in 2.04774s (16.6036GiB/s) | 20260505 17:10:46.118 | INFO | Analyzing chunk frames = [4896768,4897280) | 20260505 17:10:46.118 | INFO | MemInfo: { | 20260505 17:10:46.118 | INFO | "gpu" : " 17 GB", | 20260505 17:10:46.118 | INFO | "huge" : "", | 20260505 17:10:46.118 | INFO | "huge_shared" : " 27 GB", | 20260505 17:10:46.118 | INFO | "malloc" : " 14 GB", | 20260505 17:10:46.118 | INFO | "pinned" : " 29 GB" | 20260505 17:10:46.118 | INFO | } | 20260505 17:10:46.118 | INFO | Perf report for the last 5.12543 seconds: | 20260505 17:10:46.118 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:10:46.118 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:46.118 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:46.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:46.118 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260505 17:10:46.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:46.118 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:46.118 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 17:10:46.118 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:46.118 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:46.118 | INFO | Uploaded 36GiB to GPU in 2.13905s (16.8299GiB/s) | 20260505 17:10:46.119 | INFO | Downloaded 34GiB to GPU in 2.0591s (16.512GiB/s) | 20260505 17:10:51.253 | INFO | Analyzing chunk frames = [4897280,4897792) | 20260505 17:10:51.253 | INFO | MemInfo: { | 20260505 17:10:51.253 | INFO | "gpu" : " 17 GB", | 20260505 17:10:51.253 | INFO | "huge" : "", | 20260505 17:10:51.253 | INFO | "huge_shared" : " 27 GB", | 20260505 17:10:51.253 | INFO | "malloc" : " 14 GB", | 20260505 17:10:51.253 | INFO | "pinned" : " 29 GB" | 20260505 17:10:51.253 | INFO | } | 20260505 17:10:51.253 | INFO | Perf report for the last 5.13498 seconds: | 20260505 17:10:51.253 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:10:51.253 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:51.253 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:51.253 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:51.253 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260505 17:10:51.253 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:51.253 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:51.253 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:10:51.253 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:51.253 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:51.253 | INFO | Uploaded 36GiB to GPU in 2.14368s (16.7936GiB/s) | 20260505 17:10:51.254 | INFO | Downloaded 34GiB to GPU in 2.03647s (16.6955GiB/s) | 20260505 17:10:56.400 | INFO | Analyzing chunk frames = [4897792,4898304) | 20260505 17:10:56.400 | INFO | MemInfo: { | 20260505 17:10:56.400 | INFO | "gpu" : " 17 GB", | 20260505 17:10:56.400 | INFO | "huge" : "", | 20260505 17:10:56.400 | INFO | "huge_shared" : " 27 GB", | 20260505 17:10:56.400 | INFO | "malloc" : " 14 GB", | 20260505 17:10:56.400 | INFO | "pinned" : " 29 GB" | 20260505 17:10:56.400 | INFO | } | 20260505 17:10:56.400 | INFO | Perf report for the last 5.14714 seconds: | 20260505 17:10:56.400 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:10:56.400 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:56.400 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:56.400 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:56.400 | INFO | ANALYSIS : 63.4%, 3.0, 99.2ms, 0 | 20260505 17:10:56.400 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:56.400 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:56.400 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 17:10:56.400 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:56.400 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:10:56.401 | INFO | Uploaded 36GiB to GPU in 2.11886s (16.9902GiB/s) | 20260505 17:10:56.401 | INFO | Downloaded 34GiB to GPU in 2.18579s (15.555GiB/s) | 20260505 17:11:01.539 | INFO | Analyzing chunk frames = [4898304,4898816) | 20260505 17:11:01.539 | INFO | MemInfo: { | 20260505 17:11:01.539 | INFO | "gpu" : " 17 GB", | 20260505 17:11:01.539 | INFO | "huge" : "", | 20260505 17:11:01.539 | INFO | "huge_shared" : " 27 GB", | 20260505 17:11:01.539 | INFO | "malloc" : " 14 GB", | 20260505 17:11:01.539 | INFO | "pinned" : " 29 GB" | 20260505 17:11:01.539 | INFO | } | 20260505 17:11:01.539 | INFO | Perf report for the last 5.13913 seconds: | 20260505 17:11:01.539 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:11:01.539 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:01.539 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:01.540 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:01.540 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 17:11:01.540 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:01.540 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:01.540 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 17:11:01.540 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:01.540 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:01.540 | INFO | Uploaded 36GiB to GPU in 2.19106s (16.4304GiB/s) | 20260505 17:11:01.540 | INFO | Downloaded 34GiB to GPU in 2.02482s (16.7916GiB/s) | 20260505 17:11:01.540 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 17:11:01.540 | INFO | Total mem usage: 15.3 GiB | 20260505 17:11:01.540 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 17:11:01.540 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.93%) | 20260505 17:11:01.540 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 17:11:01.540 | INFO | High Water Breakdown: | 20260505 17:11:01.540 | INFO | 61% : 8.67 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 17:11:01.540 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 17:11:01.540 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 17:11:01.540 | INFO | ------------------------------------------ | 20260505 17:11:01.540 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:11:01.540 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 17:11:01.540 | INFO | Total mem usage: 26.5 GiB | 20260505 17:11:01.540 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 17:11:01.540 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 17:11:01.540 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 17:11:01.540 | INFO | High Water Breakdown: | 20260505 17:11:01.540 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:11:01.540 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:11:01.540 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:11:01.540 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:11:01.540 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:11:01.540 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:11:01.540 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 17:11:01.540 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 17:11:01.540 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:11:01.540 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 17:11:01.540 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 17:11:01.540 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 17:11:01.540 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 17:11:01.540 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 17:11:01.540 | INFO | ------------------------------------------ | 20260505 17:11:01.540 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:11:01.540 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 17:11:01.540 | INFO | Total mem usage: 25.6 GiB | 20260505 17:11:01.540 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 17:11:01.540 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 17:11:01.540 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 17:11:01.540 | INFO | High Water Breakdown: | 20260505 17:11:01.540 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 17:11:01.540 | INFO | ------------------------------------------ | 20260505 17:11:01.541 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:11:01.541 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 17:11:01.541 | INFO | Total mem usage: 15.7 GiB | 20260505 17:11:01.541 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 17:11:01.541 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 17:11:01.541 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 17:11:01.541 | INFO | High Water Breakdown: | 20260505 17:11:01.541 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 17:11:01.541 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 17:11:01.541 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 17:11:01.541 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 17:11:01.541 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 17:11:01.541 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 17:11:01.541 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 17:11:01.541 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 17:11:01.541 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 17:11:01.541 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:11:01.541 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 17:11:01.541 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:11:01.541 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 17:11:01.541 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 17:11:01.541 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 17:11:01.541 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 17:11:01.541 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:11:01.541 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 17:11:01.541 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:11:01.541 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:11:01.541 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:11:01.541 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 17:11:01.541 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 17:11:01.541 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:11:01.541 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 17:11:01.541 | INFO | ------------------------------------------ | 20260505 17:11:01.541 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:11:01.541 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 17:11:01.541 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:11:01.541 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:11:01.541 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.45e+01, 1.36e+01 / 2.23e+00 | 20260505 17:11:01.541 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 1.02e-01, 1.05e-02 / 3.82e-03 | 20260505 17:11:01.541 | INFO | PerfReport: 11.79%, Baseline -- 3.73e+00 / 4.48e+00, 5.74e-01 / 1.65e-01 | 20260505 17:11:01.541 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.50e+00, 1.92e-01 / 1.25e-01 | 20260505 17:11:01.541 | INFO | PerfReport: 15.90%, DME -- 4.03e+01 / 4.44e+01, 4.23e+00 / 2.36e+00 | 20260505 17:11:01.541 | INFO | PerfReport: 56.84%, FrameLabeling -- 1.80e+01 / 1.93e+01, 1.52e+00 / 1.07e+00 | 20260505 17:11:01.541 | INFO | PerfReport: 7.34%, PulseAccumulating -- 2.32e+00 / 2.94e+00, 4.48e-01 / 1.74e-01 | 20260505 17:11:01.541 | INFO | PerfReport: 3.37%, Metrics -- 1.07e+00 / 1.41e+00, 2.93e-01 / 1.34e-01 | 20260505 17:11:01.541 | INFO | PerfReport: 1.18%, PulseSerializing -- 3.73e-01 / 4.39e-01, 6.63e-02 / 4.00e-02 | 20260505 17:11:01.541 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 17:11:01.541 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:11:01.541 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:11:01.541 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.59e+01 / 0.00e+00 | 20260505 17:11:01.541 | INFO | PerfReport: 22.57%, Upload -- 2.26e+01 / 0.00e+00, 7.80e+00 / 0.00e+00 | 20260505 17:11:01.541 | INFO | PerfReport: 21.70%, Download -- 2.17e+01 / 0.00e+00, 7.75e+00 / 0.00e+00 | 20260505 17:11:01.541 | INFO | PerfReport: 31.62%, Compute -- 3.17e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 17:11:01.541 | INFO | PerfReport: 2.34%, LockUpload -- 2.34e+00 / 0.00e+00, 8.11e+00 / 0.00e+00 | 20260505 17:11:01.541 | INFO | PerfReport: 1.44%, LockDownload -- 1.44e+00 / 0.00e+00, 3.01e+00 / 0.00e+00 | 20260505 17:11:01.541 | INFO | PerfReport: 20.32%, LockCompute -- 2.03e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260505 17:11:01.541 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.88e-02 / 0.00e+00 | 20260505 17:11:01.676 | INFO | Starting a round of Baz IO | 20260505 17:11:04.861 | INFO | Populating RTMetrics report | 20260505 17:11:04.861 | INFO | RTMetrics report populated | 20260505 17:11:04.861 | INFO | Opening tmp RTMetrics json file | 20260505 17:11:04.862 | INFO | Serializing RTMetrics to json | 20260505 17:11:04.864 | INFO | Serialization done, swapping tmp file now... | 20260505 17:11:04.864 | INFO | RTMetrics done | 20260505 17:11:04.957 | INFO | Finished a round of Baz IO: Wrote 2.143 GiB in 3.28 seconds | 20260505 17:11:04.957 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.11% / 0.00% / 77.89% | 20260505 17:11:04.957 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.317442% / 7.94729e-06% / 9.53674e-05% | 20260505 17:11:04.957 | INFO | Baz IO overall: | 20260505 17:11:04.957 | INFO | BytesWritten : 2899421866930 | 20260505 17:11:04.957 | INFO | NumEvents : 1375262305095 | 20260505 17:11:06.685 | INFO | Analyzing chunk frames = [4898816,4899328) | 20260505 17:11:06.685 | INFO | MemInfo: { | 20260505 17:11:06.685 | INFO | "gpu" : " 17 GB", | 20260505 17:11:06.685 | INFO | "huge" : "", | 20260505 17:11:06.685 | INFO | "huge_shared" : " 27 GB", | 20260505 17:11:06.685 | INFO | "malloc" : " 14 GB", | 20260505 17:11:06.685 | INFO | "pinned" : " 29 GB" | 20260505 17:11:06.685 | INFO | } | 20260505 17:11:06.685 | INFO | Perf report for the last 5.14554 seconds: | 20260505 17:11:06.685 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:11:06.685 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:06.685 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:06.685 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:06.685 | INFO | ANALYSIS : 63.5%, 3.0, 99.0ms, 0 | 20260505 17:11:06.685 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:06.685 | INFO | RT_METRICS : 44.5%, 1.0, 23.4ms, 0 | 20260505 17:11:06.685 | INFO | PRE_HQ : 60.5%, 1.8, 56.3ms, 0 | 20260505 17:11:06.685 | INFO | BAZWRITER : 63.2%, 3.8, 124.1ms, 0 | 20260505 17:11:06.685 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 17:11:06.685 | INFO | Uploaded 36GiB to GPU in 2.15296s (16.7212GiB/s) | 20260505 17:11:06.685 | INFO | Downloaded 38GiB to GPU in 2.23627s (16.9926GiB/s) | 20260505 17:11:11.821 | INFO | Analyzing chunk frames = [4899328,4899840) | 20260505 17:11:11.821 | INFO | MemInfo: { | 20260505 17:11:11.821 | INFO | "gpu" : " 17 GB", | 20260505 17:11:11.821 | INFO | "huge" : "", | 20260505 17:11:11.821 | INFO | "huge_shared" : " 27 GB", | 20260505 17:11:11.821 | INFO | "malloc" : " 14 GB", | 20260505 17:11:11.821 | INFO | "pinned" : " 29 GB" | 20260505 17:11:11.821 | INFO | } | 20260505 17:11:11.821 | INFO | Perf report for the last 5.13579 seconds: | 20260505 17:11:11.821 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:11:11.821 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:11.821 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:11.821 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:11.821 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260505 17:11:11.821 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:11.821 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:11.821 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:11:11.821 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:11.821 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:11.821 | INFO | Uploaded 36GiB to GPU in 2.14023s (16.8206GiB/s) | 20260505 17:11:11.821 | INFO | Downloaded 34GiB to GPU in 2.03151s (16.7364GiB/s) | 20260505 17:11:16.955 | INFO | Analyzing chunk frames = [4899840,4900352) | 20260505 17:11:16.955 | INFO | MemInfo: { | 20260505 17:11:16.955 | INFO | "gpu" : " 17 GB", | 20260505 17:11:16.955 | INFO | "huge" : "", | 20260505 17:11:16.955 | INFO | "huge_shared" : " 27 GB", | 20260505 17:11:16.955 | INFO | "malloc" : " 14 GB", | 20260505 17:11:16.955 | INFO | "pinned" : " 29 GB" | 20260505 17:11:16.955 | INFO | } | 20260505 17:11:16.955 | INFO | Perf report for the last 5.13455 seconds: | 20260505 17:11:16.955 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:11:16.955 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:16.955 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:16.955 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:16.955 | INFO | ANALYSIS : 62.9%, 3.0, 97.8ms, 0 | 20260505 17:11:16.955 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:16.955 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:16.956 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 17:11:16.956 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:16.956 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:16.956 | INFO | Uploaded 36GiB to GPU in 2.20687s (16.3127GiB/s) | 20260505 17:11:16.956 | INFO | Downloaded 34GiB to GPU in 2.03157s (16.7358GiB/s) | 20260505 17:11:22.097 | INFO | Analyzing chunk frames = [4900352,4900864) | 20260505 17:11:22.097 | INFO | MemInfo: { | 20260505 17:11:22.097 | INFO | "gpu" : " 17 GB", | 20260505 17:11:22.097 | INFO | "huge" : "", | 20260505 17:11:22.097 | INFO | "huge_shared" : " 27 GB", | 20260505 17:11:22.098 | INFO | "malloc" : " 14 GB", | 20260505 17:11:22.098 | INFO | "pinned" : " 29 GB" | 20260505 17:11:22.098 | INFO | } | 20260505 17:11:22.098 | INFO | Perf report for the last 5.14218 seconds: | 20260505 17:11:22.098 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:11:22.098 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:22.098 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:22.098 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:22.098 | INFO | ANALYSIS : 62.5%, 3.0, 97.1ms, 0 | 20260505 17:11:22.098 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:22.098 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:22.098 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:11:22.098 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:22.098 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:22.098 | INFO | Uploaded 36GiB to GPU in 2.16573s (16.6225GiB/s) | 20260505 17:11:22.098 | INFO | Downloaded 34GiB to GPU in 2.1436s (15.8612GiB/s) | 20260505 17:11:27.236 | INFO | Analyzing chunk frames = [4900864,4901376) | 20260505 17:11:27.236 | INFO | MemInfo: { | 20260505 17:11:27.236 | INFO | "gpu" : " 17 GB", | 20260505 17:11:27.236 | INFO | "huge" : "", | 20260505 17:11:27.236 | INFO | "huge_shared" : " 27 GB", | 20260505 17:11:27.236 | INFO | "malloc" : " 14 GB", | 20260505 17:11:27.236 | INFO | "pinned" : " 29 GB" | 20260505 17:11:27.236 | INFO | } | 20260505 17:11:27.236 | INFO | Perf report for the last 5.1386 seconds: | 20260505 17:11:27.236 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:11:27.236 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:27.236 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:27.236 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:27.236 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 17:11:27.236 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:27.236 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:27.236 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:11:27.236 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:27.236 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:27.236 | INFO | Uploaded 36GiB to GPU in 2.15012s (16.7433GiB/s) | 20260505 17:11:27.236 | INFO | Downloaded 34GiB to GPU in 2.05781s (16.5224GiB/s) | 20260505 17:11:32.377 | INFO | Analyzing chunk frames = [4901376,4901888) | 20260505 17:11:32.377 | INFO | MemInfo: { | 20260505 17:11:32.377 | INFO | "gpu" : " 17 GB", | 20260505 17:11:32.377 | INFO | "huge" : "", | 20260505 17:11:32.377 | INFO | "huge_shared" : " 27 GB", | 20260505 17:11:32.377 | INFO | "malloc" : " 14 GB", | 20260505 17:11:32.377 | INFO | "pinned" : " 29 GB" | 20260505 17:11:32.377 | INFO | } | 20260505 17:11:32.377 | INFO | Perf report for the last 5.14079 seconds: | 20260505 17:11:32.377 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:11:32.377 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:32.377 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:32.377 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:32.377 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260505 17:11:32.377 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:32.377 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:32.377 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 17:11:32.377 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:32.377 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:32.377 | INFO | Uploaded 36GiB to GPU in 2.14447s (16.7873GiB/s) | 20260505 17:11:32.377 | INFO | Downloaded 34GiB to GPU in 2.03542s (16.7041GiB/s) | 20260505 17:11:37.509 | INFO | Analyzing chunk frames = [4901888,4902400) | 20260505 17:11:37.509 | INFO | MemInfo: { | 20260505 17:11:37.509 | INFO | "gpu" : " 17 GB", | 20260505 17:11:37.509 | INFO | "huge" : "", | 20260505 17:11:37.509 | INFO | "huge_shared" : " 27 GB", | 20260505 17:11:37.509 | INFO | "malloc" : " 14 GB", | 20260505 17:11:37.509 | INFO | "pinned" : " 29 GB" | 20260505 17:11:37.509 | INFO | } | 20260505 17:11:37.509 | INFO | Perf report for the last 5.13236 seconds: | 20260505 17:11:37.509 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:11:37.509 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:37.509 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:37.509 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:37.509 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 17:11:37.509 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:37.509 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:37.510 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 17:11:37.510 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:37.510 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:37.510 | INFO | Uploaded 36GiB to GPU in 2.15099s (16.7365GiB/s) | 20260505 17:11:37.510 | INFO | Downloaded 34GiB to GPU in 2.16052s (15.737GiB/s) | 20260505 17:11:42.655 | INFO | Analyzing chunk frames = [4902400,4902912) | 20260505 17:11:42.655 | INFO | MemInfo: { | 20260505 17:11:42.655 | INFO | "gpu" : " 17 GB", | 20260505 17:11:42.655 | INFO | "huge" : "", | 20260505 17:11:42.655 | INFO | "huge_shared" : " 27 GB", | 20260505 17:11:42.655 | INFO | "malloc" : " 14 GB", | 20260505 17:11:42.655 | INFO | "pinned" : " 29 GB" | 20260505 17:11:42.655 | INFO | } | 20260505 17:11:42.655 | INFO | Perf report for the last 5.14581 seconds: | 20260505 17:11:42.655 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:11:42.655 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:42.655 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:42.655 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:42.655 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 17:11:42.655 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:42.655 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:42.655 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 17:11:42.655 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:42.655 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:42.655 | INFO | Uploaded 36GiB to GPU in 2.17537s (16.5489GiB/s) | 20260505 17:11:42.655 | INFO | Downloaded 34GiB to GPU in 2.04057s (16.662GiB/s) | 20260505 17:11:42.797 | INFO | Starting a round of Baz IO | 20260505 17:11:45.980 | INFO | Populating RTMetrics report | 20260505 17:11:45.980 | INFO | RTMetrics report populated | 20260505 17:11:45.980 | INFO | Opening tmp RTMetrics json file | 20260505 17:11:45.982 | INFO | Serializing RTMetrics to json | 20260505 17:11:45.983 | INFO | Serialization done, swapping tmp file now... | 20260505 17:11:45.983 | INFO | RTMetrics done | 20260505 17:11:46.075 | INFO | Finished a round of Baz IO: Wrote 2.143 GiB in 3.277 seconds | 20260505 17:11:46.075 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.10% / 0.00% / 77.89% | 20260505 17:11:46.075 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.317911% / 3.97364e-06% / 0.000115236% | 20260505 17:11:46.075 | INFO | Baz IO overall: | 20260505 17:11:46.075 | INFO | BytesWritten : 2901723340203 | 20260505 17:11:46.075 | INFO | NumEvents : 1376338901726 | 20260505 17:11:47.800 | INFO | Analyzing chunk frames = [4902912,4903424) | 20260505 17:11:47.800 | INFO | MemInfo: { | 20260505 17:11:47.800 | INFO | "gpu" : " 17 GB", | 20260505 17:11:47.800 | INFO | "huge" : "", | 20260505 17:11:47.800 | INFO | "huge_shared" : " 27 GB", | 20260505 17:11:47.800 | INFO | "malloc" : " 14 GB", | 20260505 17:11:47.800 | INFO | "pinned" : " 29 GB" | 20260505 17:11:47.800 | INFO | } | 20260505 17:11:47.800 | INFO | Perf report for the last 5.14479 seconds: | 20260505 17:11:47.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:11:47.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:47.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:47.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:47.800 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 17:11:47.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:47.800 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260505 17:11:47.800 | INFO | PRE_HQ : 60.6%, 1.8, 56.9ms, 0 | 20260505 17:11:47.800 | INFO | BAZWRITER : 63.2%, 3.8, 125.0ms, 0 | 20260505 17:11:47.800 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 17:11:47.800 | INFO | Uploaded 36GiB to GPU in 2.1649s (16.629GiB/s) | 20260505 17:11:47.800 | INFO | Downloaded 38GiB to GPU in 2.26181s (16.8007GiB/s) | 20260505 17:11:52.936 | INFO | Analyzing chunk frames = [4903424,4903936) | 20260505 17:11:52.936 | INFO | MemInfo: { | 20260505 17:11:52.936 | INFO | "gpu" : " 17 GB", | 20260505 17:11:52.936 | INFO | "huge" : "", | 20260505 17:11:52.936 | INFO | "huge_shared" : " 27 GB", | 20260505 17:11:52.936 | INFO | "malloc" : " 14 GB", | 20260505 17:11:52.936 | INFO | "pinned" : " 29 GB" | 20260505 17:11:52.936 | INFO | } | 20260505 17:11:52.936 | INFO | Perf report for the last 5.13605 seconds: | 20260505 17:11:52.936 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:11:52.936 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:52.936 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:52.936 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:52.936 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 17:11:52.936 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:52.936 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:52.936 | INFO | PRE_HQ : 27.7%, 1.0, 14.3ms, 0 | 20260505 17:11:52.936 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:52.936 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:52.936 | INFO | Uploaded 36GiB to GPU in 2.12057s (16.9766GiB/s) | 20260505 17:11:52.936 | INFO | Downloaded 34GiB to GPU in 2.02155s (16.8188GiB/s) | 20260505 17:11:58.064 | INFO | Analyzing chunk frames = [4903936,4904448) | 20260505 17:11:58.064 | INFO | MemInfo: { | 20260505 17:11:58.064 | INFO | "gpu" : " 17 GB", | 20260505 17:11:58.064 | INFO | "huge" : "", | 20260505 17:11:58.064 | INFO | "huge_shared" : " 27 GB", | 20260505 17:11:58.064 | INFO | "malloc" : " 14 GB", | 20260505 17:11:58.064 | INFO | "pinned" : " 29 GB" | 20260505 17:11:58.064 | INFO | } | 20260505 17:11:58.064 | INFO | Perf report for the last 5.12839 seconds: | 20260505 17:11:58.064 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:11:58.064 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:58.064 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:58.064 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:58.065 | INFO | ANALYSIS : 62.8%, 3.0, 97.5ms, 0 | 20260505 17:11:58.065 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:58.065 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:58.065 | INFO | PRE_HQ : 28.1%, 1.0, 14.5ms, 0 | 20260505 17:11:58.065 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:58.065 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:11:58.065 | INFO | Uploaded 36GiB to GPU in 2.19541s (16.3979GiB/s) | 20260505 17:11:58.065 | INFO | Downloaded 34GiB to GPU in 2.03018s (16.7473GiB/s) | 20260505 17:12:03.204 | INFO | Analyzing chunk frames = [4904448,4904960) | 20260505 17:12:03.204 | INFO | MemInfo: { | 20260505 17:12:03.204 | INFO | "gpu" : " 17 GB", | 20260505 17:12:03.204 | INFO | "huge" : "", | 20260505 17:12:03.204 | INFO | "huge_shared" : " 27 GB", | 20260505 17:12:03.204 | INFO | "malloc" : " 14 GB", | 20260505 17:12:03.204 | INFO | "pinned" : " 29 GB" | 20260505 17:12:03.204 | INFO | } | 20260505 17:12:03.204 | INFO | Perf report for the last 5.13977 seconds: | 20260505 17:12:03.204 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:12:03.204 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:03.204 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:03.204 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:03.204 | INFO | ANALYSIS : 62.5%, 3.0, 97.0ms, 0 | 20260505 17:12:03.204 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:03.204 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:03.204 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 17:12:03.204 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:03.204 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:03.205 | INFO | Uploaded 36GiB to GPU in 2.18148s (16.5025GiB/s) | 20260505 17:12:03.205 | INFO | Downloaded 34GiB to GPU in 2.13236s (15.9448GiB/s) | 20260505 17:12:08.352 | INFO | Analyzing chunk frames = [4904960,4905472) | 20260505 17:12:08.352 | INFO | MemInfo: { | 20260505 17:12:08.352 | INFO | "gpu" : " 17 GB", | 20260505 17:12:08.353 | INFO | "huge" : "", | 20260505 17:12:08.353 | INFO | "huge_shared" : " 27 GB", | 20260505 17:12:08.353 | INFO | "malloc" : " 14 GB", | 20260505 17:12:08.353 | INFO | "pinned" : " 29 GB" | 20260505 17:12:08.353 | INFO | } | 20260505 17:12:08.353 | INFO | Perf report for the last 5.14844 seconds: | 20260505 17:12:08.353 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:12:08.353 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:08.353 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:08.353 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:08.353 | INFO | ANALYSIS : 62.4%, 3.0, 97.6ms, 0 | 20260505 17:12:08.353 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:08.353 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:08.353 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:12:08.353 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:08.353 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:08.353 | INFO | Uploaded 36GiB to GPU in 2.17812s (16.528GiB/s) | 20260505 17:12:08.353 | INFO | Downloaded 34GiB to GPU in 2.13429s (15.9303GiB/s) | 20260505 17:12:13.495 | INFO | Analyzing chunk frames = [4905472,4905984) | 20260505 17:12:13.495 | INFO | MemInfo: { | 20260505 17:12:13.495 | INFO | "gpu" : " 17 GB", | 20260505 17:12:13.496 | INFO | "huge" : "", | 20260505 17:12:13.496 | INFO | "huge_shared" : " 27 GB", | 20260505 17:12:13.496 | INFO | "malloc" : " 14 GB", | 20260505 17:12:13.496 | INFO | "pinned" : " 29 GB" | 20260505 17:12:13.496 | INFO | } | 20260505 17:12:13.496 | INFO | Perf report for the last 5.1429 seconds: | 20260505 17:12:13.496 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:12:13.496 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:13.496 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:13.496 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:13.496 | INFO | ANALYSIS : 62.3%, 3.0, 97.4ms, 0 | 20260505 17:12:13.496 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:13.496 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:13.496 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:12:13.496 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:13.496 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:13.496 | INFO | Uploaded 36GiB to GPU in 2.13924s (16.8284GiB/s) | 20260505 17:12:13.496 | INFO | Downloaded 34GiB to GPU in 2.04253s (16.646GiB/s) | 20260505 17:12:18.636 | INFO | Analyzing chunk frames = [4905984,4906496) | 20260505 17:12:18.636 | INFO | MemInfo: { | 20260505 17:12:18.636 | INFO | "gpu" : " 17 GB", | 20260505 17:12:18.636 | INFO | "huge" : "", | 20260505 17:12:18.636 | INFO | "huge_shared" : " 27 GB", | 20260505 17:12:18.636 | INFO | "malloc" : " 14 GB", | 20260505 17:12:18.636 | INFO | "pinned" : " 29 GB" | 20260505 17:12:18.636 | INFO | } | 20260505 17:12:18.636 | INFO | Perf report for the last 5.1403 seconds: | 20260505 17:12:18.636 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:12:18.636 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:18.636 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:18.636 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:18.636 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 17:12:18.636 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:18.636 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:18.636 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 17:12:18.636 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:18.636 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:18.636 | INFO | Uploaded 36GiB to GPU in 2.17659s (16.5396GiB/s) | 20260505 17:12:18.636 | INFO | Downloaded 34GiB to GPU in 2.16724s (15.6882GiB/s) | 20260505 17:12:23.767 | INFO | Analyzing chunk frames = [4906496,4907008) | 20260505 17:12:23.767 | INFO | MemInfo: { | 20260505 17:12:23.767 | INFO | "gpu" : " 17 GB", | 20260505 17:12:23.767 | INFO | "huge" : "", | 20260505 17:12:23.767 | INFO | "huge_shared" : " 27 GB", | 20260505 17:12:23.767 | INFO | "malloc" : " 14 GB", | 20260505 17:12:23.767 | INFO | "pinned" : " 29 GB" | 20260505 17:12:23.767 | INFO | } | 20260505 17:12:23.767 | INFO | Perf report for the last 5.13119 seconds: | 20260505 17:12:23.767 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:12:23.767 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:23.767 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:23.767 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:23.767 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 17:12:23.767 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:23.767 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:23.767 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 17:12:23.767 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:23.767 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:23.767 | INFO | Uploaded 36GiB to GPU in 2.19442s (16.4052GiB/s) | 20260505 17:12:23.767 | INFO | Downloaded 34GiB to GPU in 2.02585s (16.7831GiB/s) | 20260505 17:12:23.768 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 17:12:23.768 | INFO | Total mem usage: 15.3 GiB | 20260505 17:12:23.768 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 17:12:23.768 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.93%) | 20260505 17:12:23.768 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 17:12:23.768 | INFO | High Water Breakdown: | 20260505 17:12:23.768 | INFO | 61% : 8.67 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 17:12:23.768 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 17:12:23.768 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 17:12:23.768 | INFO | ------------------------------------------ | 20260505 17:12:23.768 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:12:23.768 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 17:12:23.768 | INFO | Total mem usage: 26.5 GiB | 20260505 17:12:23.768 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 17:12:23.768 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 17:12:23.768 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 17:12:23.768 | INFO | High Water Breakdown: | 20260505 17:12:23.768 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:12:23.768 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:12:23.768 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:12:23.768 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:12:23.768 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:12:23.768 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:12:23.768 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 17:12:23.768 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 17:12:23.768 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:12:23.768 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 17:12:23.768 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 17:12:23.768 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 17:12:23.768 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 17:12:23.768 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 17:12:23.768 | INFO | ------------------------------------------ | 20260505 17:12:23.768 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:12:23.768 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 17:12:23.768 | INFO | Total mem usage: 25.6 GiB | 20260505 17:12:23.768 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 17:12:23.768 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 17:12:23.768 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 17:12:23.768 | INFO | High Water Breakdown: | 20260505 17:12:23.768 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 17:12:23.768 | INFO | ------------------------------------------ | 20260505 17:12:23.768 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:12:23.768 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 17:12:23.768 | INFO | Total mem usage: 15.7 GiB | 20260505 17:12:23.768 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 17:12:23.768 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 17:12:23.768 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 17:12:23.768 | INFO | High Water Breakdown: | 20260505 17:12:23.768 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 17:12:23.768 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 17:12:23.768 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 17:12:23.768 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 17:12:23.768 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 17:12:23.768 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 17:12:23.768 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 17:12:23.768 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 17:12:23.768 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 17:12:23.768 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:12:23.768 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 17:12:23.768 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:12:23.768 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 17:12:23.768 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 17:12:23.768 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 17:12:23.768 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 17:12:23.768 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:12:23.768 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 17:12:23.768 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:12:23.768 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:12:23.768 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:12:23.768 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 17:12:23.768 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 17:12:23.768 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:12:23.768 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 17:12:23.768 | INFO | ------------------------------------------ | 20260505 17:12:23.768 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:12:23.768 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 17:12:23.768 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:12:23.768 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:12:23.768 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.52e+01, 1.34e+01 / 2.03e+00 | 20260505 17:12:23.768 | INFO | PerfReport: 0.29%, AggregateStats -- 9.27e-02 / 1.02e-01, 1.04e-02 / 4.20e-03 | 20260505 17:12:23.769 | INFO | PerfReport: 11.79%, Baseline -- 3.73e+00 / 4.50e+00, 5.76e-01 / 1.66e-01 | 20260505 17:12:23.769 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.52e+00, 1.93e-01 / 9.02e-02 | 20260505 17:12:23.769 | INFO | PerfReport: 15.92%, DME -- 4.03e+01 / 4.47e+01, 4.31e+00 / 2.20e+00 | 20260505 17:12:23.769 | INFO | PerfReport: 56.81%, FrameLabeling -- 1.80e+01 / 1.96e+01, 1.49e+00 / 1.35e+00 | 20260505 17:12:23.769 | INFO | PerfReport: 7.34%, PulseAccumulating -- 2.32e+00 / 2.86e+00, 4.49e-01 / 1.61e-01 | 20260505 17:12:23.769 | INFO | PerfReport: 3.38%, Metrics -- 1.07e+00 / 1.41e+00, 2.98e-01 / 1.45e-01 | 20260505 17:12:23.769 | INFO | PerfReport: 1.18%, PulseSerializing -- 3.73e-01 / 4.36e-01, 6.74e-02 / 4.12e-02 | 20260505 17:12:23.769 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 17:12:23.769 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:12:23.769 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:12:23.769 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260505 17:12:23.769 | INFO | PerfReport: 22.55%, Upload -- 2.26e+01 / 0.00e+00, 7.93e+00 / 0.00e+00 | 20260505 17:12:23.769 | INFO | PerfReport: 21.84%, Download -- 2.18e+01 / 0.00e+00, 7.89e+00 / 0.00e+00 | 20260505 17:12:23.769 | INFO | PerfReport: 31.64%, Compute -- 3.16e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 17:12:23.769 | INFO | PerfReport: 2.18%, LockUpload -- 2.18e+00 / 0.00e+00, 7.85e+00 / 0.00e+00 | 20260505 17:12:23.769 | INFO | PerfReport: 1.44%, LockDownload -- 1.44e+00 / 0.00e+00, 2.97e+00 / 0.00e+00 | 20260505 17:12:23.769 | INFO | PerfReport: 20.33%, LockCompute -- 2.03e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 17:12:23.769 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.74e-02 / 0.00e+00 | 20260505 17:12:23.901 | INFO | Starting a round of Baz IO | 20260505 17:12:27.100 | INFO | Populating RTMetrics report | 20260505 17:12:27.100 | INFO | RTMetrics report populated | 20260505 17:12:27.100 | INFO | Opening tmp RTMetrics json file | 20260505 17:12:27.103 | INFO | Serializing RTMetrics to json | 20260505 17:12:27.105 | INFO | Serialization done, swapping tmp file now... | 20260505 17:12:27.105 | INFO | RTMetrics done | 20260505 17:12:27.191 | INFO | Finished a round of Baz IO: Wrote 2.142 GiB in 3.29 seconds | 20260505 17:12:27.191 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.10% / 0.00% / 77.90% | 20260505 17:12:27.191 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.315424% / 0% / 0.000127157% | 20260505 17:12:27.191 | INFO | Baz IO overall: | 20260505 17:12:27.191 | INFO | BytesWritten : 2904022832417 | 20260505 17:12:27.191 | INFO | NumEvents : 1377414125690 | 20260505 17:12:28.908 | INFO | Analyzing chunk frames = [4907008,4907520) | 20260505 17:12:28.908 | INFO | MemInfo: { | 20260505 17:12:28.908 | INFO | "gpu" : " 17 GB", | 20260505 17:12:28.908 | INFO | "huge" : "", | 20260505 17:12:28.908 | INFO | "huge_shared" : " 27 GB", | 20260505 17:12:28.908 | INFO | "malloc" : " 14 GB", | 20260505 17:12:28.908 | INFO | "pinned" : " 29 GB" | 20260505 17:12:28.908 | INFO | } | 20260505 17:12:28.908 | INFO | Perf report for the last 5.14089 seconds: | 20260505 17:12:28.908 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:12:28.908 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:28.908 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:28.908 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:28.908 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 17:12:28.908 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:28.908 | INFO | RT_METRICS : 44.7%, 1.0, 23.4ms, 0 | 20260505 17:12:28.908 | INFO | PRE_HQ : 60.4%, 1.8, 56.1ms, 0 | 20260505 17:12:28.908 | INFO | BAZWRITER : 63.3%, 3.8, 123.7ms, 0 | 20260505 17:12:28.908 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 17:12:28.908 | INFO | Uploaded 36GiB to GPU in 2.18436s (16.4808GiB/s) | 20260505 17:12:28.908 | INFO | Downloaded 38GiB to GPU in 2.2685s (16.7512GiB/s) | 20260505 17:12:34.050 | INFO | Analyzing chunk frames = [4907520,4908032) | 20260505 17:12:34.050 | INFO | MemInfo: { | 20260505 17:12:34.050 | INFO | "gpu" : " 17 GB", | 20260505 17:12:34.050 | INFO | "huge" : "", | 20260505 17:12:34.050 | INFO | "huge_shared" : " 27 GB", | 20260505 17:12:34.051 | INFO | "malloc" : " 14 GB", | 20260505 17:12:34.051 | INFO | "pinned" : " 29 GB" | 20260505 17:12:34.051 | INFO | } | 20260505 17:12:34.051 | INFO | Perf report for the last 5.14258 seconds: | 20260505 17:12:34.051 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:12:34.051 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:34.051 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:34.051 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:34.051 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 17:12:34.051 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:34.051 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:34.051 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:12:34.051 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:34.051 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:34.051 | INFO | Uploaded 36GiB to GPU in 2.12807s (16.9167GiB/s) | 20260505 17:12:34.051 | INFO | Downloaded 34GiB to GPU in 2.03136s (16.7376GiB/s) | 20260505 17:12:39.190 | INFO | Analyzing chunk frames = [4908032,4908544) | 20260505 17:12:39.190 | INFO | MemInfo: { | 20260505 17:12:39.190 | INFO | "gpu" : " 17 GB", | 20260505 17:12:39.190 | INFO | "huge" : "", | 20260505 17:12:39.190 | INFO | "huge_shared" : " 27 GB", | 20260505 17:12:39.190 | INFO | "malloc" : " 14 GB", | 20260505 17:12:39.190 | INFO | "pinned" : " 29 GB" | 20260505 17:12:39.190 | INFO | } | 20260505 17:12:39.190 | INFO | Perf report for the last 5.13964 seconds: | 20260505 17:12:39.190 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:12:39.190 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:39.190 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:39.190 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:39.190 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 17:12:39.190 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:39.190 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:39.190 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:12:39.190 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:39.190 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:39.191 | INFO | Uploaded 36GiB to GPU in 2.20497s (16.3267GiB/s) | 20260505 17:12:39.191 | INFO | Downloaded 34GiB to GPU in 2.0573s (16.5265GiB/s) | 20260505 17:12:44.325 | INFO | Analyzing chunk frames = [4908544,4909056) | 20260505 17:12:44.325 | INFO | MemInfo: { | 20260505 17:12:44.325 | INFO | "gpu" : " 17 GB", | 20260505 17:12:44.325 | INFO | "huge" : "", | 20260505 17:12:44.325 | INFO | "huge_shared" : " 27 GB", | 20260505 17:12:44.325 | INFO | "malloc" : " 14 GB", | 20260505 17:12:44.325 | INFO | "pinned" : " 29 GB" | 20260505 17:12:44.325 | INFO | } | 20260505 17:12:44.325 | INFO | Perf report for the last 5.1348 seconds: | 20260505 17:12:44.325 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:12:44.325 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:44.325 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:44.325 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:44.325 | INFO | ANALYSIS : 62.6%, 3.0, 97.2ms, 0 | 20260505 17:12:44.325 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:44.325 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:44.325 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:12:44.325 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:44.325 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:44.325 | INFO | Uploaded 36GiB to GPU in 2.18827s (16.4514GiB/s) | 20260505 17:12:44.325 | INFO | Downloaded 34GiB to GPU in 2.14154s (15.8764GiB/s) | 20260505 17:12:49.459 | INFO | Analyzing chunk frames = [4909056,4909568) | 20260505 17:12:49.459 | INFO | MemInfo: { | 20260505 17:12:49.459 | INFO | "gpu" : " 17 GB", | 20260505 17:12:49.459 | INFO | "huge" : "", | 20260505 17:12:49.459 | INFO | "huge_shared" : " 27 GB", | 20260505 17:12:49.459 | INFO | "malloc" : " 14 GB", | 20260505 17:12:49.459 | INFO | "pinned" : " 29 GB" | 20260505 17:12:49.459 | INFO | } | 20260505 17:12:49.459 | INFO | Perf report for the last 5.13439 seconds: | 20260505 17:12:49.459 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:12:49.459 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:49.459 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:49.460 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:49.460 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260505 17:12:49.460 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:49.460 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:49.460 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 17:12:49.460 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:49.460 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:49.460 | INFO | Uploaded 36GiB to GPU in 2.15349s (16.717GiB/s) | 20260505 17:12:49.460 | INFO | Downloaded 34GiB to GPU in 2.0567s (16.5314GiB/s) | 20260505 17:12:54.612 | INFO | Analyzing chunk frames = [4909568,4910080) | 20260505 17:12:54.612 | INFO | MemInfo: { | 20260505 17:12:54.612 | INFO | "gpu" : " 17 GB", | 20260505 17:12:54.612 | INFO | "huge" : "", | 20260505 17:12:54.612 | INFO | "huge_shared" : " 27 GB", | 20260505 17:12:54.612 | INFO | "malloc" : " 14 GB", | 20260505 17:12:54.612 | INFO | "pinned" : " 29 GB" | 20260505 17:12:54.612 | INFO | } | 20260505 17:12:54.612 | INFO | Perf report for the last 5.15298 seconds: | 20260505 17:12:54.612 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:12:54.612 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:54.613 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:54.613 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:54.613 | INFO | ANALYSIS : 62.3%, 3.0, 97.6ms, 0 | 20260505 17:12:54.613 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:54.613 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:54.613 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 17:12:54.613 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:54.613 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:54.613 | INFO | Uploaded 36GiB to GPU in 2.13997s (16.8226GiB/s) | 20260505 17:12:54.613 | INFO | Downloaded 34GiB to GPU in 2.04068s (16.6611GiB/s) | 20260505 17:12:59.752 | INFO | Analyzing chunk frames = [4910080,4910592) | 20260505 17:12:59.752 | INFO | MemInfo: { | 20260505 17:12:59.752 | INFO | "gpu" : " 17 GB", | 20260505 17:12:59.752 | INFO | "huge" : "", | 20260505 17:12:59.752 | INFO | "huge_shared" : " 27 GB", | 20260505 17:12:59.752 | INFO | "malloc" : " 14 GB", | 20260505 17:12:59.752 | INFO | "pinned" : " 29 GB" | 20260505 17:12:59.752 | INFO | } | 20260505 17:12:59.752 | INFO | Perf report for the last 5.13975 seconds: | 20260505 17:12:59.752 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:12:59.752 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:59.752 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:59.752 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:59.752 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 17:12:59.752 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:59.752 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:59.752 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 17:12:59.752 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:59.752 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:12:59.752 | INFO | Uploaded 36GiB to GPU in 2.13512s (16.8609GiB/s) | 20260505 17:12:59.753 | INFO | Downloaded 34GiB to GPU in 2.13349s (15.9364GiB/s) | 20260505 17:13:04.880 | INFO | Analyzing chunk frames = [4910592,4911104) | 20260505 17:13:04.880 | INFO | MemInfo: { | 20260505 17:13:04.880 | INFO | "gpu" : " 17 GB", | 20260505 17:13:04.880 | INFO | "huge" : "", | 20260505 17:13:04.880 | INFO | "huge_shared" : " 27 GB", | 20260505 17:13:04.880 | INFO | "malloc" : " 14 GB", | 20260505 17:13:04.880 | INFO | "pinned" : " 29 GB" | 20260505 17:13:04.880 | INFO | } | 20260505 17:13:04.880 | INFO | Perf report for the last 5.12788 seconds: | 20260505 17:13:04.880 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:13:04.880 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:04.880 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:04.880 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:04.880 | INFO | ANALYSIS : 63.1%, 3.0, 98.3ms, 0 | 20260505 17:13:04.880 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:04.880 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:04.880 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 17:13:04.880 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:04.880 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:04.880 | INFO | Uploaded 36GiB to GPU in 2.19947s (16.3676GiB/s) | 20260505 17:13:04.880 | INFO | Downloaded 34GiB to GPU in 2.02405s (16.798GiB/s) | 20260505 17:13:05.008 | INFO | Starting a round of Baz IO | 20260505 17:13:08.221 | INFO | Populating RTMetrics report | 20260505 17:13:08.221 | INFO | RTMetrics report populated | 20260505 17:13:08.221 | INFO | Opening tmp RTMetrics json file | 20260505 17:13:08.229 | INFO | Serializing RTMetrics to json | 20260505 17:13:08.230 | INFO | Serialization done, swapping tmp file now... | 20260505 17:13:08.231 | INFO | RTMetrics done | 20260505 17:13:08.315 | INFO | Finished a round of Baz IO: Wrote 2.141 GiB in 3.307 seconds | 20260505 17:13:08.315 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.10% / 0.00% / 77.90% | 20260505 17:13:08.315 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.314009% / 3.97364e-06% / 0.000111262% | 20260505 17:13:08.316 | INFO | Baz IO overall: | 20260505 17:13:08.316 | INFO | BytesWritten : 2906321962167 | 20260505 17:13:08.316 | INFO | NumEvents : 1378489273217 | 20260505 17:13:10.016 | INFO | Analyzing chunk frames = [4911104,4911616) | 20260505 17:13:10.016 | INFO | MemInfo: { | 20260505 17:13:10.016 | INFO | "gpu" : " 17 GB", | 20260505 17:13:10.016 | INFO | "huge" : "", | 20260505 17:13:10.016 | INFO | "huge_shared" : " 27 GB", | 20260505 17:13:10.016 | INFO | "malloc" : " 14 GB", | 20260505 17:13:10.016 | INFO | "pinned" : " 29 GB" | 20260505 17:13:10.016 | INFO | } | 20260505 17:13:10.016 | INFO | Perf report for the last 5.13598 seconds: | 20260505 17:13:10.016 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:13:10.016 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:10.016 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:10.016 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:10.016 | INFO | ANALYSIS : 64.0%, 3.0, 99.6ms, 0 | 20260505 17:13:10.016 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:10.016 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260505 17:13:10.016 | INFO | PRE_HQ : 60.7%, 1.8, 56.3ms, 0 | 20260505 17:13:10.016 | INFO | BAZWRITER : 63.7%, 3.8, 125.4ms, 0 | 20260505 17:13:10.016 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 17:13:10.016 | INFO | Uploaded 36GiB to GPU in 2.1823s (16.4963GiB/s) | 20260505 17:13:10.016 | INFO | Downloaded 38GiB to GPU in 2.26348s (16.7883GiB/s) | 20260505 17:13:15.164 | INFO | Analyzing chunk frames = [4911616,4912128) | 20260505 17:13:15.164 | INFO | MemInfo: { | 20260505 17:13:15.164 | INFO | "gpu" : " 17 GB", | 20260505 17:13:15.164 | INFO | "huge" : "", | 20260505 17:13:15.164 | INFO | "huge_shared" : " 27 GB", | 20260505 17:13:15.164 | INFO | "malloc" : " 14 GB", | 20260505 17:13:15.164 | INFO | "pinned" : " 29 GB" | 20260505 17:13:15.164 | INFO | } | 20260505 17:13:15.164 | INFO | Perf report for the last 5.14813 seconds: | 20260505 17:13:15.164 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:13:15.164 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:15.164 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:15.164 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:15.164 | INFO | ANALYSIS : 63.2%, 3.0, 98.9ms, 0 | 20260505 17:13:15.164 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:15.164 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:15.164 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:13:15.164 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:15.164 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:15.165 | INFO | Uploaded 36GiB to GPU in 2.19864s (16.3738GiB/s) | 20260505 17:13:15.165 | INFO | Downloaded 34GiB to GPU in 2.04891s (16.5942GiB/s) | 20260505 17:13:20.301 | INFO | Analyzing chunk frames = [4912128,4912640) | 20260505 17:13:20.301 | INFO | MemInfo: { | 20260505 17:13:20.301 | INFO | "gpu" : " 17 GB", | 20260505 17:13:20.301 | INFO | "huge" : "", | 20260505 17:13:20.301 | INFO | "huge_shared" : " 27 GB", | 20260505 17:13:20.301 | INFO | "malloc" : " 14 GB", | 20260505 17:13:20.301 | INFO | "pinned" : " 29 GB" | 20260505 17:13:20.302 | INFO | } | 20260505 17:13:20.302 | INFO | Perf report for the last 5.13729 seconds: | 20260505 17:13:20.302 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:13:20.302 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:20.302 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:20.302 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:20.302 | INFO | ANALYSIS : 62.7%, 3.0, 97.5ms, 0 | 20260505 17:13:20.302 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:20.302 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:20.302 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:13:20.302 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:20.302 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:20.302 | INFO | Uploaded 36GiB to GPU in 2.20069s (16.3585GiB/s) | 20260505 17:13:20.302 | INFO | Downloaded 34GiB to GPU in 2.04402s (16.6339GiB/s) | 20260505 17:13:25.445 | INFO | Analyzing chunk frames = [4912640,4913152) | 20260505 17:13:25.445 | INFO | MemInfo: { | 20260505 17:13:25.445 | INFO | "gpu" : " 17 GB", | 20260505 17:13:25.445 | INFO | "huge" : "", | 20260505 17:13:25.445 | INFO | "huge_shared" : " 27 GB", | 20260505 17:13:25.445 | INFO | "malloc" : " 14 GB", | 20260505 17:13:25.445 | INFO | "pinned" : " 29 GB" | 20260505 17:13:25.445 | INFO | } | 20260505 17:13:25.445 | INFO | Perf report for the last 5.14326 seconds: | 20260505 17:13:25.445 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:13:25.445 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:25.445 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:25.445 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:25.445 | INFO | ANALYSIS : 62.7%, 3.0, 97.5ms, 0 | 20260505 17:13:25.445 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:25.445 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:25.445 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:13:25.445 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:25.445 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:25.445 | INFO | Uploaded 36GiB to GPU in 2.21872s (16.2256GiB/s) | 20260505 17:13:25.445 | INFO | Downloaded 34GiB to GPU in 2.12406s (16.0071GiB/s) | 20260505 17:13:30.578 | INFO | Analyzing chunk frames = [4913152,4913664) | 20260505 17:13:30.578 | INFO | MemInfo: { | 20260505 17:13:30.578 | INFO | "gpu" : " 17 GB", | 20260505 17:13:30.578 | INFO | "huge" : "", | 20260505 17:13:30.578 | INFO | "huge_shared" : " 27 GB", | 20260505 17:13:30.578 | INFO | "malloc" : " 14 GB", | 20260505 17:13:30.578 | INFO | "pinned" : " 29 GB" | 20260505 17:13:30.578 | INFO | } | 20260505 17:13:30.579 | INFO | Perf report for the last 5.13367 seconds: | 20260505 17:13:30.579 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:13:30.579 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:30.579 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:30.579 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:30.579 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 17:13:30.579 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:30.579 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:30.579 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 17:13:30.579 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:30.579 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:30.579 | INFO | Uploaded 36GiB to GPU in 2.14661s (16.7707GiB/s) | 20260505 17:13:30.579 | INFO | Downloaded 34GiB to GPU in 2.06158s (16.4922GiB/s) | 20260505 17:13:35.718 | INFO | Analyzing chunk frames = [4913664,4914176) | 20260505 17:13:35.718 | INFO | MemInfo: { | 20260505 17:13:35.718 | INFO | "gpu" : " 17 GB", | 20260505 17:13:35.718 | INFO | "huge" : "", | 20260505 17:13:35.718 | INFO | "huge_shared" : " 27 GB", | 20260505 17:13:35.718 | INFO | "malloc" : " 14 GB", | 20260505 17:13:35.718 | INFO | "pinned" : " 29 GB" | 20260505 17:13:35.718 | INFO | } | 20260505 17:13:35.718 | INFO | Perf report for the last 5.13996 seconds: | 20260505 17:13:35.718 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:13:35.718 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:35.718 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:35.718 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:35.719 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260505 17:13:35.719 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:35.719 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:35.719 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:13:35.719 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:35.719 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:35.719 | INFO | Uploaded 36GiB to GPU in 2.12713s (16.9242GiB/s) | 20260505 17:13:35.719 | INFO | Downloaded 34GiB to GPU in 2.03459s (16.711GiB/s) | 20260505 17:13:40.862 | INFO | Analyzing chunk frames = [4914176,4914688) | 20260505 17:13:40.862 | INFO | MemInfo: { | 20260505 17:13:40.862 | INFO | "gpu" : " 17 GB", | 20260505 17:13:40.862 | INFO | "huge" : "", | 20260505 17:13:40.862 | INFO | "huge_shared" : " 27 GB", | 20260505 17:13:40.862 | INFO | "malloc" : " 14 GB", | 20260505 17:13:40.862 | INFO | "pinned" : " 29 GB" | 20260505 17:13:40.862 | INFO | } | 20260505 17:13:40.862 | INFO | Perf report for the last 5.14398 seconds: | 20260505 17:13:40.862 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:13:40.862 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:40.862 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:40.862 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:40.862 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 17:13:40.863 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:40.863 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:40.863 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 17:13:40.863 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:40.863 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:40.863 | INFO | Uploaded 36GiB to GPU in 2.15389s (16.714GiB/s) | 20260505 17:13:40.863 | INFO | Downloaded 34GiB to GPU in 2.18122s (15.5876GiB/s) | 20260505 17:13:46.003 | INFO | Analyzing chunk frames = [4914688,4915200) | 20260505 17:13:46.003 | INFO | MemInfo: { | 20260505 17:13:46.003 | INFO | "gpu" : " 17 GB", | 20260505 17:13:46.003 | INFO | "huge" : "", | 20260505 17:13:46.003 | INFO | "huge_shared" : " 27 GB", | 20260505 17:13:46.003 | INFO | "malloc" : " 14 GB", | 20260505 17:13:46.003 | INFO | "pinned" : " 29 GB" | 20260505 17:13:46.003 | INFO | } | 20260505 17:13:46.004 | INFO | Perf report for the last 5.14103 seconds: | 20260505 17:13:46.004 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:13:46.004 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:46.004 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:46.004 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:46.004 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 17:13:46.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:46.004 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:46.004 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 17:13:46.004 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:46.004 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:46.004 | INFO | Uploaded 36GiB to GPU in 2.19883s (16.3724GiB/s) | 20260505 17:13:46.004 | INFO | Downloaded 34GiB to GPU in 2.02601s (16.7818GiB/s) | 20260505 17:13:46.004 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 17:13:46.004 | INFO | Total mem usage: 15.3 GiB | 20260505 17:13:46.004 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 17:13:46.004 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.01%) | 20260505 17:13:46.004 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 17:13:46.004 | INFO | High Water Breakdown: | 20260505 17:13:46.004 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 17:13:46.004 | INFO | 38.9% : 5.52 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 17:13:46.004 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 17:13:46.004 | INFO | ------------------------------------------ | 20260505 17:13:46.004 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:13:46.004 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 17:13:46.004 | INFO | Total mem usage: 26.5 GiB | 20260505 17:13:46.004 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 17:13:46.004 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 17:13:46.004 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260505 17:13:46.004 | INFO | High Water Breakdown: | 20260505 17:13:46.004 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:13:46.004 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:13:46.004 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:13:46.004 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:13:46.004 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:13:46.004 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:13:46.004 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 17:13:46.004 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 17:13:46.004 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:13:46.004 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 17:13:46.004 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 17:13:46.004 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 17:13:46.004 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 17:13:46.004 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 17:13:46.004 | INFO | ------------------------------------------ | 20260505 17:13:46.004 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:13:46.005 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 17:13:46.005 | INFO | Total mem usage: 25.6 GiB | 20260505 17:13:46.005 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 17:13:46.005 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 17:13:46.005 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 17:13:46.005 | INFO | High Water Breakdown: | 20260505 17:13:46.005 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 17:13:46.005 | INFO | ------------------------------------------ | 20260505 17:13:46.005 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:13:46.005 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 17:13:46.005 | INFO | Total mem usage: 15.7 GiB | 20260505 17:13:46.005 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 17:13:46.005 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 17:13:46.005 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 17:13:46.005 | INFO | High Water Breakdown: | 20260505 17:13:46.005 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 17:13:46.005 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 17:13:46.005 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 17:13:46.005 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 17:13:46.005 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 17:13:46.005 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 17:13:46.005 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 17:13:46.005 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 17:13:46.005 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 17:13:46.005 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:13:46.005 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 17:13:46.005 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:13:46.005 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 17:13:46.005 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 17:13:46.005 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 17:13:46.005 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 17:13:46.005 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:13:46.005 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 17:13:46.005 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:13:46.005 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:13:46.005 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:13:46.005 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 17:13:46.005 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 17:13:46.005 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:13:46.005 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 17:13:46.005 | INFO | ------------------------------------------ | 20260505 17:13:46.005 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:13:46.005 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 17:13:46.005 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:13:46.005 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:13:46.005 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.48e+01, 1.37e+01 / 1.89e+00 | 20260505 17:13:46.005 | INFO | PerfReport: 0.29%, AggregateStats -- 9.25e-02 / 1.01e-01, 1.07e-02 / 2.90e-03 | 20260505 17:13:46.005 | INFO | PerfReport: 11.77%, Baseline -- 3.72e+00 / 4.53e+00, 5.64e-01 / 1.29e-01 | 20260505 17:13:46.005 | INFO | PerfReport: 3.26%, Histogram -- 1.03e+00 / 1.52e+00, 1.88e-01 / 1.02e-01 | 20260505 17:13:46.005 | INFO | PerfReport: 15.96%, DME -- 4.03e+01 / 4.47e+01, 4.21e+00 / 2.03e+00 | 20260505 17:13:46.005 | INFO | PerfReport: 56.75%, FrameLabeling -- 1.79e+01 / 1.92e+01, 1.43e+00 / 1.15e+00 | 20260505 17:13:46.005 | INFO | PerfReport: 7.39%, PulseAccumulating -- 2.33e+00 / 2.94e+00, 4.52e-01 / 1.35e-01 | 20260505 17:13:46.005 | INFO | PerfReport: 3.39%, Metrics -- 1.07e+00 / 1.41e+00, 2.93e-01 / 1.31e-01 | 20260505 17:13:46.005 | INFO | PerfReport: 1.18%, PulseSerializing -- 3.73e-01 / 4.35e-01, 6.59e-02 / 3.59e-02 | 20260505 17:13:46.005 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 17:13:46.005 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:13:46.005 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:13:46.005 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.58e+01 / 0.00e+00 | 20260505 17:13:46.005 | INFO | PerfReport: 22.60%, Upload -- 2.26e+01 / 0.00e+00, 7.96e+00 / 0.00e+00 | 20260505 17:13:46.005 | INFO | PerfReport: 21.81%, Download -- 2.18e+01 / 0.00e+00, 7.84e+00 / 0.00e+00 | 20260505 17:13:46.005 | INFO | PerfReport: 31.54%, Compute -- 3.16e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 17:13:46.005 | INFO | PerfReport: 2.36%, LockUpload -- 2.36e+00 / 0.00e+00, 8.02e+00 / 0.00e+00 | 20260505 17:13:46.005 | INFO | PerfReport: 1.39%, LockDownload -- 1.39e+00 / 0.00e+00, 2.89e+00 / 0.00e+00 | 20260505 17:13:46.005 | INFO | PerfReport: 20.28%, LockCompute -- 2.03e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260505 17:13:46.005 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.87e-02 / 0.00e+00 | 20260505 17:13:46.142 | INFO | Starting a round of Baz IO | 20260505 17:13:49.324 | INFO | Populating RTMetrics report | 20260505 17:13:49.324 | INFO | RTMetrics report populated | 20260505 17:13:49.324 | INFO | Opening tmp RTMetrics json file | 20260505 17:13:49.329 | INFO | Serializing RTMetrics to json | 20260505 17:13:49.331 | INFO | Serialization done, swapping tmp file now... | 20260505 17:13:49.333 | INFO | RTMetrics done | 20260505 17:13:49.419 | INFO | Finished a round of Baz IO: Wrote 2.139 GiB in 3.277 seconds | 20260505 17:13:49.419 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.10% / 0.00% / 77.90% | 20260505 17:13:49.419 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.313532% / 7.94729e-06% / 0.000135104% | 20260505 17:13:49.419 | INFO | Baz IO overall: | 20260505 17:13:49.419 | INFO | BytesWritten : 2908618894766 | 20260505 17:13:49.419 | INFO | NumEvents : 1379563023526 | 20260505 17:13:51.137 | INFO | Analyzing chunk frames = [4915200,4915712) | 20260505 17:13:51.137 | INFO | MemInfo: { | 20260505 17:13:51.137 | INFO | "gpu" : " 17 GB", | 20260505 17:13:51.137 | INFO | "huge" : "", | 20260505 17:13:51.137 | INFO | "huge_shared" : " 27 GB", | 20260505 17:13:51.137 | INFO | "malloc" : " 14 GB", | 20260505 17:13:51.137 | INFO | "pinned" : " 29 GB" | 20260505 17:13:51.137 | INFO | } | 20260505 17:13:51.137 | INFO | Perf report for the last 5.13369 seconds: | 20260505 17:13:51.137 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:13:51.137 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:51.137 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:51.137 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:51.137 | INFO | ANALYSIS : 63.6%, 3.0, 99.0ms, 0 | 20260505 17:13:51.137 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:51.137 | INFO | RT_METRICS : 44.7%, 1.0, 23.4ms, 0 | 20260505 17:13:51.137 | INFO | PRE_HQ : 60.3%, 1.8, 56.2ms, 0 | 20260505 17:13:51.137 | INFO | BAZWRITER : 63.3%, 3.8, 124.2ms, 0 | 20260505 17:13:51.137 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 17:13:51.137 | INFO | Uploaded 36GiB to GPU in 2.13806s (16.8377GiB/s) | 20260505 17:13:51.138 | INFO | Downloaded 38GiB to GPU in 2.25171s (16.8761GiB/s) | 20260505 17:13:56.271 | INFO | Analyzing chunk frames = [4915712,4916224) | 20260505 17:13:56.271 | INFO | MemInfo: { | 20260505 17:13:56.271 | INFO | "gpu" : " 17 GB", | 20260505 17:13:56.271 | INFO | "huge" : "", | 20260505 17:13:56.272 | INFO | "huge_shared" : " 27 GB", | 20260505 17:13:56.272 | INFO | "malloc" : " 14 GB", | 20260505 17:13:56.272 | INFO | "pinned" : " 29 GB" | 20260505 17:13:56.272 | INFO | } | 20260505 17:13:56.272 | INFO | Perf report for the last 5.13442 seconds: | 20260505 17:13:56.272 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:13:56.272 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:56.272 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:56.272 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:56.272 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260505 17:13:56.272 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:56.272 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:56.272 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 17:13:56.272 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:56.272 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:13:56.272 | INFO | Uploaded 36GiB to GPU in 2.11115s (17.0523GiB/s) | 20260505 17:13:56.272 | INFO | Downloaded 34GiB to GPU in 2.02792s (16.766GiB/s) | 20260505 17:14:01.416 | INFO | Analyzing chunk frames = [4916224,4916736) | 20260505 17:14:01.416 | INFO | MemInfo: { | 20260505 17:14:01.416 | INFO | "gpu" : " 17 GB", | 20260505 17:14:01.416 | INFO | "huge" : "", | 20260505 17:14:01.417 | INFO | "huge_shared" : " 27 GB", | 20260505 17:14:01.417 | INFO | "malloc" : " 14 GB", | 20260505 17:14:01.417 | INFO | "pinned" : " 29 GB" | 20260505 17:14:01.417 | INFO | } | 20260505 17:14:01.417 | INFO | Perf report for the last 5.14501 seconds: | 20260505 17:14:01.417 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:14:01.417 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:01.417 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:01.417 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:01.417 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 17:14:01.417 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:01.417 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:01.417 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:14:01.417 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:01.417 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:01.417 | INFO | Uploaded 36GiB to GPU in 2.1826s (16.4941GiB/s) | 20260505 17:14:01.417 | INFO | Downloaded 34GiB to GPU in 2.03095s (16.741GiB/s) | 20260505 17:14:06.561 | INFO | Analyzing chunk frames = [4916736,4917248) | 20260505 17:14:06.561 | INFO | MemInfo: { | 20260505 17:14:06.561 | INFO | "gpu" : " 17 GB", | 20260505 17:14:06.561 | INFO | "huge" : "", | 20260505 17:14:06.561 | INFO | "huge_shared" : " 27 GB", | 20260505 17:14:06.561 | INFO | "malloc" : " 14 GB", | 20260505 17:14:06.561 | INFO | "pinned" : " 29 GB" | 20260505 17:14:06.561 | INFO | } | 20260505 17:14:06.561 | INFO | Perf report for the last 5.14434 seconds: | 20260505 17:14:06.561 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:14:06.561 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:06.561 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:06.561 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:06.561 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 17:14:06.561 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:06.561 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:06.561 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260505 17:14:06.561 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:06.561 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:06.561 | INFO | Uploaded 36GiB to GPU in 2.22253s (16.1978GiB/s) | 20260505 17:14:06.561 | INFO | Downloaded 34GiB to GPU in 2.12265s (16.0177GiB/s) | 20260505 17:14:11.697 | INFO | Analyzing chunk frames = [4917248,4917760) | 20260505 17:14:11.697 | INFO | MemInfo: { | 20260505 17:14:11.697 | INFO | "gpu" : " 17 GB", | 20260505 17:14:11.697 | INFO | "huge" : "", | 20260505 17:14:11.697 | INFO | "huge_shared" : " 27 GB", | 20260505 17:14:11.697 | INFO | "malloc" : " 14 GB", | 20260505 17:14:11.697 | INFO | "pinned" : " 29 GB" | 20260505 17:14:11.697 | INFO | } | 20260505 17:14:11.697 | INFO | Perf report for the last 5.13618 seconds: | 20260505 17:14:11.697 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:14:11.697 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:11.697 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:11.697 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:11.697 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 17:14:11.697 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:11.697 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:11.697 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:14:11.697 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:11.697 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:11.697 | INFO | Uploaded 36GiB to GPU in 2.14461s (16.7862GiB/s) | 20260505 17:14:11.697 | INFO | Downloaded 34GiB to GPU in 2.06262s (16.4839GiB/s) | 20260505 17:14:16.831 | INFO | Analyzing chunk frames = [4917760,4918272) | 20260505 17:14:16.831 | INFO | MemInfo: { | 20260505 17:14:16.831 | INFO | "gpu" : " 17 GB", | 20260505 17:14:16.831 | INFO | "huge" : "", | 20260505 17:14:16.831 | INFO | "huge_shared" : " 27 GB", | 20260505 17:14:16.831 | INFO | "malloc" : " 14 GB", | 20260505 17:14:16.831 | INFO | "pinned" : " 29 GB" | 20260505 17:14:16.831 | INFO | } | 20260505 17:14:16.831 | INFO | Perf report for the last 5.13422 seconds: | 20260505 17:14:16.831 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:14:16.831 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:16.831 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:16.831 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:16.831 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260505 17:14:16.831 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:16.831 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:16.832 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:14:16.832 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:16.832 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:16.832 | INFO | Uploaded 36GiB to GPU in 2.13697s (16.8463GiB/s) | 20260505 17:14:16.832 | INFO | Downloaded 34GiB to GPU in 2.04691s (16.6104GiB/s) | 20260505 17:14:21.968 | INFO | Analyzing chunk frames = [4918272,4918784) | 20260505 17:14:21.968 | INFO | MemInfo: { | 20260505 17:14:21.968 | INFO | "gpu" : " 17 GB", | 20260505 17:14:21.968 | INFO | "huge" : "", | 20260505 17:14:21.968 | INFO | "huge_shared" : " 27 GB", | 20260505 17:14:21.968 | INFO | "malloc" : " 14 GB", | 20260505 17:14:21.968 | INFO | "pinned" : " 29 GB" | 20260505 17:14:21.968 | INFO | } | 20260505 17:14:21.968 | INFO | Perf report for the last 5.13669 seconds: | 20260505 17:14:21.968 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:14:21.968 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:21.968 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:21.968 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:21.968 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 17:14:21.968 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:21.968 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:21.968 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 17:14:21.968 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:21.968 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:21.969 | INFO | Uploaded 36GiB to GPU in 2.1239s (16.95GiB/s) | 20260505 17:14:21.969 | INFO | Downloaded 34GiB to GPU in 2.17824s (15.609GiB/s) | 20260505 17:14:27.116 | INFO | Analyzing chunk frames = [4918784,4919296) | 20260505 17:14:27.116 | INFO | MemInfo: { | 20260505 17:14:27.116 | INFO | "gpu" : " 17 GB", | 20260505 17:14:27.116 | INFO | "huge" : "", | 20260505 17:14:27.116 | INFO | "huge_shared" : " 27 GB", | 20260505 17:14:27.116 | INFO | "malloc" : " 14 GB", | 20260505 17:14:27.116 | INFO | "pinned" : " 29 GB" | 20260505 17:14:27.116 | INFO | } | 20260505 17:14:27.116 | INFO | Perf report for the last 5.14803 seconds: | 20260505 17:14:27.116 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:14:27.116 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:27.116 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:27.116 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:27.116 | INFO | ANALYSIS : 63.5%, 3.0, 99.3ms, 0 | 20260505 17:14:27.116 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:27.116 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:27.116 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 17:14:27.116 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:27.116 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:27.116 | INFO | Uploaded 36GiB to GPU in 2.23034s (16.1411GiB/s) | 20260505 17:14:27.116 | INFO | Downloaded 34GiB to GPU in 2.1084s (16.126GiB/s) | 20260505 17:14:27.253 | INFO | Starting a round of Baz IO | 20260505 17:14:30.438 | INFO | Populating RTMetrics report | 20260505 17:14:30.439 | INFO | RTMetrics report populated | 20260505 17:14:30.439 | INFO | Opening tmp RTMetrics json file | 20260505 17:14:30.444 | INFO | Serializing RTMetrics to json | 20260505 17:14:30.446 | INFO | Serialization done, swapping tmp file now... | 20260505 17:14:30.449 | INFO | RTMetrics done | 20260505 17:14:30.540 | INFO | Finished a round of Baz IO: Wrote 2.142 GiB in 3.287 seconds | 20260505 17:14:30.540 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.10% / 0.00% / 77.90% | 20260505 17:14:30.540 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.31358% / 0% / 9.93411e-05% | 20260505 17:14:30.540 | INFO | Baz IO overall: | 20260505 17:14:30.540 | INFO | BytesWritten : 2910918423506 | 20260505 17:14:30.540 | INFO | NumEvents : 1380638802098 | 20260505 17:14:32.257 | INFO | Analyzing chunk frames = [4919296,4919808) | 20260505 17:14:32.257 | INFO | MemInfo: { | 20260505 17:14:32.257 | INFO | "gpu" : " 17 GB", | 20260505 17:14:32.257 | INFO | "huge" : "", | 20260505 17:14:32.258 | INFO | "huge_shared" : " 27 GB", | 20260505 17:14:32.258 | INFO | "malloc" : " 14 GB", | 20260505 17:14:32.258 | INFO | "pinned" : " 29 GB" | 20260505 17:14:32.258 | INFO | } | 20260505 17:14:32.258 | INFO | Perf report for the last 5.14141 seconds: | 20260505 17:14:32.258 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:14:32.258 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:32.258 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:32.258 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:32.258 | INFO | ANALYSIS : 63.5%, 3.0, 99.0ms, 0 | 20260505 17:14:32.258 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:32.258 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260505 17:14:32.258 | INFO | PRE_HQ : 60.6%, 1.8, 56.7ms, 0 | 20260505 17:14:32.258 | INFO | BAZWRITER : 63.3%, 3.9, 125.7ms, 0 | 20260505 17:14:32.258 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 17:14:32.258 | INFO | Uploaded 36GiB to GPU in 2.16674s (16.6148GiB/s) | 20260505 17:14:32.258 | INFO | Downloaded 38GiB to GPU in 2.25631s (16.8417GiB/s) | 20260505 17:14:37.393 | INFO | Analyzing chunk frames = [4919808,4920320) | 20260505 17:14:37.393 | INFO | MemInfo: { | 20260505 17:14:37.393 | INFO | "gpu" : " 17 GB", | 20260505 17:14:37.393 | INFO | "huge" : "", | 20260505 17:14:37.393 | INFO | "huge_shared" : " 27 GB", | 20260505 17:14:37.393 | INFO | "malloc" : " 14 GB", | 20260505 17:14:37.393 | INFO | "pinned" : " 29 GB" | 20260505 17:14:37.393 | INFO | } | 20260505 17:14:37.393 | INFO | Perf report for the last 5.13571 seconds: | 20260505 17:14:37.393 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:14:37.393 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:37.393 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:37.393 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:37.393 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 17:14:37.393 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:37.393 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:37.393 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:14:37.393 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:37.393 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:37.394 | INFO | Uploaded 36GiB to GPU in 2.13484s (16.8631GiB/s) | 20260505 17:14:37.394 | INFO | Downloaded 34GiB to GPU in 2.02542s (16.7866GiB/s) | 20260505 17:14:42.526 | INFO | Analyzing chunk frames = [4920320,4920832) | 20260505 17:14:42.526 | INFO | MemInfo: { | 20260505 17:14:42.526 | INFO | "gpu" : " 17 GB", | 20260505 17:14:42.526 | INFO | "huge" : "", | 20260505 17:14:42.526 | INFO | "huge_shared" : " 27 GB", | 20260505 17:14:42.526 | INFO | "malloc" : " 14 GB", | 20260505 17:14:42.526 | INFO | "pinned" : " 29 GB" | 20260505 17:14:42.526 | INFO | } | 20260505 17:14:42.526 | INFO | Perf report for the last 5.13282 seconds: | 20260505 17:14:42.526 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:14:42.526 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:42.526 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:42.526 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:42.526 | INFO | ANALYSIS : 62.8%, 3.0, 97.6ms, 0 | 20260505 17:14:42.526 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:42.526 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:42.526 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:14:42.526 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:42.526 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:42.526 | INFO | Uploaded 36GiB to GPU in 2.20222s (16.3471GiB/s) | 20260505 17:14:42.527 | INFO | Downloaded 34GiB to GPU in 2.04433s (16.6314GiB/s) | 20260505 17:14:47.674 | INFO | Analyzing chunk frames = [4920832,4921344) | 20260505 17:14:47.674 | INFO | MemInfo: { | 20260505 17:14:47.674 | INFO | "gpu" : " 17 GB", | 20260505 17:14:47.674 | INFO | "huge" : "", | 20260505 17:14:47.674 | INFO | "huge_shared" : " 27 GB", | 20260505 17:14:47.674 | INFO | "malloc" : " 14 GB", | 20260505 17:14:47.674 | INFO | "pinned" : " 29 GB" | 20260505 17:14:47.674 | INFO | } | 20260505 17:14:47.674 | INFO | Perf report for the last 5.14759 seconds: | 20260505 17:14:47.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:14:47.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:47.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:47.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:47.674 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260505 17:14:47.674 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:47.674 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:47.674 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 17:14:47.674 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:47.674 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:47.674 | INFO | Uploaded 36GiB to GPU in 2.19181s (16.4248GiB/s) | 20260505 17:14:47.674 | INFO | Downloaded 34GiB to GPU in 2.13456s (15.9283GiB/s) | 20260505 17:14:52.810 | INFO | Analyzing chunk frames = [4921344,4921856) | 20260505 17:14:52.810 | INFO | MemInfo: { | 20260505 17:14:52.810 | INFO | "gpu" : " 17 GB", | 20260505 17:14:52.810 | INFO | "huge" : "", | 20260505 17:14:52.810 | INFO | "huge_shared" : " 27 GB", | 20260505 17:14:52.810 | INFO | "malloc" : " 14 GB", | 20260505 17:14:52.810 | INFO | "pinned" : " 29 GB" | 20260505 17:14:52.810 | INFO | } | 20260505 17:14:52.810 | INFO | Perf report for the last 5.13631 seconds: | 20260505 17:14:52.810 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:14:52.810 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:52.810 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:52.810 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:52.810 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 17:14:52.810 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:52.810 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:52.810 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 17:14:52.810 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:52.810 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:52.810 | INFO | Uploaded 36GiB to GPU in 2.14938s (16.749GiB/s) | 20260505 17:14:52.810 | INFO | Downloaded 34GiB to GPU in 2.05103s (16.577GiB/s) | 20260505 17:14:57.952 | INFO | Analyzing chunk frames = [4921856,4922368) | 20260505 17:14:57.952 | INFO | MemInfo: { | 20260505 17:14:57.952 | INFO | "gpu" : " 17 GB", | 20260505 17:14:57.952 | INFO | "huge" : "", | 20260505 17:14:57.952 | INFO | "huge_shared" : " 27 GB", | 20260505 17:14:57.952 | INFO | "malloc" : " 14 GB", | 20260505 17:14:57.952 | INFO | "pinned" : " 29 GB" | 20260505 17:14:57.952 | INFO | } | 20260505 17:14:57.952 | INFO | Perf report for the last 5.14207 seconds: | 20260505 17:14:57.952 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:14:57.952 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:57.952 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:57.952 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:57.952 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260505 17:14:57.952 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:57.952 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:57.952 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 17:14:57.952 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:57.952 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:14:57.952 | INFO | Uploaded 36GiB to GPU in 2.13841s (16.8349GiB/s) | 20260505 17:14:57.952 | INFO | Downloaded 34GiB to GPU in 2.03267s (16.7268GiB/s) | 20260505 17:15:03.090 | INFO | Analyzing chunk frames = [4922368,4922880) | 20260505 17:15:03.091 | INFO | MemInfo: { | 20260505 17:15:03.091 | INFO | "gpu" : " 17 GB", | 20260505 17:15:03.091 | INFO | "huge" : "", | 20260505 17:15:03.091 | INFO | "huge_shared" : " 27 GB", | 20260505 17:15:03.091 | INFO | "malloc" : " 14 GB", | 20260505 17:15:03.091 | INFO | "pinned" : " 29 GB" | 20260505 17:15:03.091 | INFO | } | 20260505 17:15:03.091 | INFO | Perf report for the last 5.13855 seconds: | 20260505 17:15:03.091 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:15:03.091 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:03.091 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:03.091 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:03.091 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 17:15:03.091 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:03.091 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:03.091 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 17:15:03.091 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:03.091 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:03.091 | INFO | Uploaded 36GiB to GPU in 2.14106s (16.8141GiB/s) | 20260505 17:15:03.091 | INFO | Downloaded 34GiB to GPU in 2.17617s (15.6238GiB/s) | 20260505 17:15:08.224 | INFO | Analyzing chunk frames = [4922880,4923392) | 20260505 17:15:08.224 | INFO | MemInfo: { | 20260505 17:15:08.224 | INFO | "gpu" : " 17 GB", | 20260505 17:15:08.224 | INFO | "huge" : "", | 20260505 17:15:08.224 | INFO | "huge_shared" : " 27 GB", | 20260505 17:15:08.224 | INFO | "malloc" : " 14 GB", | 20260505 17:15:08.224 | INFO | "pinned" : " 29 GB" | 20260505 17:15:08.224 | INFO | } | 20260505 17:15:08.224 | INFO | Perf report for the last 5.13376 seconds: | 20260505 17:15:08.224 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:15:08.224 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:08.224 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:08.225 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:08.225 | INFO | ANALYSIS : 63.1%, 3.0, 97.4ms, 0 | 20260505 17:15:08.225 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:08.225 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:08.225 | INFO | PRE_HQ : 28.1%, 1.0, 14.4ms, 0 | 20260505 17:15:08.225 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:08.225 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:08.225 | INFO | Uploaded 36GiB to GPU in 2.15831s (16.6797GiB/s) | 20260505 17:15:08.225 | INFO | Downloaded 34GiB to GPU in 2.03529s (16.7052GiB/s) | 20260505 17:15:08.225 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 17:15:08.225 | INFO | Total mem usage: 15.3 GiB | 20260505 17:15:08.225 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 17:15:08.225 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.96%) | 20260505 17:15:08.225 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 17:15:08.225 | INFO | High Water Breakdown: | 20260505 17:15:08.225 | INFO | 61% : 8.66 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 17:15:08.225 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 17:15:08.225 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 17:15:08.225 | INFO | ------------------------------------------ | 20260505 17:15:08.225 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:15:08.225 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 17:15:08.225 | INFO | Total mem usage: 26.5 GiB | 20260505 17:15:08.225 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 17:15:08.225 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 17:15:08.225 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 17:15:08.225 | INFO | High Water Breakdown: | 20260505 17:15:08.225 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:15:08.225 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:15:08.225 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:15:08.225 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:15:08.225 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:15:08.225 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:15:08.225 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 17:15:08.225 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 17:15:08.225 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:15:08.225 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 17:15:08.225 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 17:15:08.225 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 17:15:08.225 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 17:15:08.225 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 17:15:08.225 | INFO | ------------------------------------------ | 20260505 17:15:08.225 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:15:08.225 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 17:15:08.225 | INFO | Total mem usage: 25.6 GiB | 20260505 17:15:08.225 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 17:15:08.225 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 17:15:08.225 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 17:15:08.225 | INFO | High Water Breakdown: | 20260505 17:15:08.225 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 17:15:08.225 | INFO | ------------------------------------------ | 20260505 17:15:08.225 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:15:08.225 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 17:15:08.225 | INFO | Total mem usage: 15.7 GiB | 20260505 17:15:08.225 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 17:15:08.225 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 17:15:08.225 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 17:15:08.225 | INFO | High Water Breakdown: | 20260505 17:15:08.225 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 17:15:08.225 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 17:15:08.225 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 17:15:08.225 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 17:15:08.225 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 17:15:08.225 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 17:15:08.225 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 17:15:08.225 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 17:15:08.225 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 17:15:08.225 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:15:08.225 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 17:15:08.225 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:15:08.225 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 17:15:08.225 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 17:15:08.225 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 17:15:08.225 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 17:15:08.225 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:15:08.225 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 17:15:08.225 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:15:08.225 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:15:08.225 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:15:08.225 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 17:15:08.225 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 17:15:08.225 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:15:08.225 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 17:15:08.225 | INFO | ------------------------------------------ | 20260505 17:15:08.225 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:15:08.225 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 17:15:08.225 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:15:08.225 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:15:08.225 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.51e+01, 1.35e+01 / 2.33e+00 | 20260505 17:15:08.225 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 1.02e-01, 1.08e-02 / 5.38e-03 | 20260505 17:15:08.225 | INFO | PerfReport: 11.78%, Baseline -- 3.73e+00 / 4.47e+00, 5.68e-01 / 2.12e-01 | 20260505 17:15:08.225 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.50e+00, 1.90e-01 / 1.15e-01 | 20260505 17:15:08.225 | INFO | PerfReport: 15.93%, DME -- 4.03e+01 / 4.46e+01, 4.13e+00 / 2.31e+00 | 20260505 17:15:08.225 | INFO | PerfReport: 56.78%, FrameLabeling -- 1.80e+01 / 1.98e+01, 1.44e+00 / 1.66e+00 | 20260505 17:15:08.225 | INFO | PerfReport: 7.36%, PulseAccumulating -- 2.33e+00 / 2.88e+00, 4.45e-01 / 2.02e-01 | 20260505 17:15:08.225 | INFO | PerfReport: 3.39%, Metrics -- 1.07e+00 / 1.40e+00, 2.97e-01 / 1.57e-01 | 20260505 17:15:08.225 | INFO | PerfReport: 1.18%, PulseSerializing -- 3.73e-01 / 4.30e-01, 6.59e-02 / 3.32e-02 | 20260505 17:15:08.225 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 17:15:08.225 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:15:08.225 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:15:08.225 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260505 17:15:08.225 | INFO | PerfReport: 22.50%, Upload -- 2.25e+01 / 0.00e+00, 7.87e+00 / 0.00e+00 | 20260505 17:15:08.225 | INFO | PerfReport: 21.85%, Download -- 2.19e+01 / 0.00e+00, 7.86e+00 / 0.00e+00 | 20260505 17:15:08.225 | INFO | PerfReport: 31.60%, Compute -- 3.16e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 17:15:08.225 | INFO | PerfReport: 2.29%, LockUpload -- 2.29e+00 / 0.00e+00, 7.92e+00 / 0.00e+00 | 20260505 17:15:08.225 | INFO | PerfReport: 1.43%, LockDownload -- 1.44e+00 / 0.00e+00, 3.05e+00 / 0.00e+00 | 20260505 17:15:08.225 | INFO | PerfReport: 20.31%, LockCompute -- 2.03e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 17:15:08.225 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.86e-02 / 0.00e+00 | 20260505 17:15:08.359 | INFO | Starting a round of Baz IO | 20260505 17:15:11.559 | INFO | Populating RTMetrics report | 20260505 17:15:11.559 | INFO | RTMetrics report populated | 20260505 17:15:11.559 | INFO | Opening tmp RTMetrics json file | 20260505 17:15:11.569 | INFO | Serializing RTMetrics to json | 20260505 17:15:11.571 | INFO | Serialization done, swapping tmp file now... | 20260505 17:15:11.572 | INFO | RTMetrics done | 20260505 17:15:11.661 | INFO | Finished a round of Baz IO: Wrote 2.14 GiB in 3.301 seconds | 20260505 17:15:11.661 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.09% / 0.00% / 77.90% | 20260505 17:15:11.661 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.31275% / 0% / 0.000103315% | 20260505 17:15:11.661 | INFO | Baz IO overall: | 20260505 17:15:11.661 | INFO | BytesWritten : 2913216753397 | 20260505 17:15:11.661 | INFO | NumEvents : 1381714098329 | 20260505 17:15:13.377 | INFO | Analyzing chunk frames = [4923392,4923904) | 20260505 17:15:13.377 | INFO | MemInfo: { | 20260505 17:15:13.377 | INFO | "gpu" : " 17 GB", | 20260505 17:15:13.377 | INFO | "huge" : "", | 20260505 17:15:13.377 | INFO | "huge_shared" : " 27 GB", | 20260505 17:15:13.377 | INFO | "malloc" : " 14 GB", | 20260505 17:15:13.377 | INFO | "pinned" : " 29 GB" | 20260505 17:15:13.377 | INFO | } | 20260505 17:15:13.377 | INFO | Perf report for the last 5.1528 seconds: | 20260505 17:15:13.377 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:15:13.377 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:13.377 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:13.377 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:13.377 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 17:15:13.377 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:13.377 | INFO | RT_METRICS : 44.6%, 1.0, 23.5ms, 0 | 20260505 17:15:13.377 | INFO | PRE_HQ : 60.5%, 1.8, 56.6ms, 0 | 20260505 17:15:13.377 | INFO | BAZWRITER : 63.4%, 3.8, 125.5ms, 0 | 20260505 17:15:13.377 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260505 17:15:13.378 | INFO | Uploaded 36GiB to GPU in 2.17813s (16.5279GiB/s) | 20260505 17:15:13.378 | INFO | Downloaded 38GiB to GPU in 2.26792s (16.7555GiB/s) | 20260505 17:15:18.512 | INFO | Analyzing chunk frames = [4923904,4924416) | 20260505 17:15:18.512 | INFO | MemInfo: { | 20260505 17:15:18.512 | INFO | "gpu" : " 17 GB", | 20260505 17:15:18.512 | INFO | "huge" : "", | 20260505 17:15:18.512 | INFO | "huge_shared" : " 27 GB", | 20260505 17:15:18.512 | INFO | "malloc" : " 14 GB", | 20260505 17:15:18.512 | INFO | "pinned" : " 29 GB" | 20260505 17:15:18.512 | INFO | } | 20260505 17:15:18.512 | INFO | Perf report for the last 5.13487 seconds: | 20260505 17:15:18.512 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:15:18.512 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:18.512 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:18.512 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:18.512 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 17:15:18.512 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:18.512 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:18.512 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 17:15:18.512 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:18.512 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:18.512 | INFO | Uploaded 36GiB to GPU in 2.14637s (16.7725GiB/s) | 20260505 17:15:18.512 | INFO | Downloaded 34GiB to GPU in 2.03839s (16.6798GiB/s) | 20260505 17:15:23.643 | INFO | Analyzing chunk frames = [4924416,4924928) | 20260505 17:15:23.643 | INFO | MemInfo: { | 20260505 17:15:23.643 | INFO | "gpu" : " 17 GB", | 20260505 17:15:23.644 | INFO | "huge" : "", | 20260505 17:15:23.644 | INFO | "huge_shared" : " 27 GB", | 20260505 17:15:23.644 | INFO | "malloc" : " 14 GB", | 20260505 17:15:23.644 | INFO | "pinned" : " 29 GB" | 20260505 17:15:23.644 | INFO | } | 20260505 17:15:23.644 | INFO | Perf report for the last 5.13141 seconds: | 20260505 17:15:23.644 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:15:23.644 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:23.644 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:23.644 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:23.644 | INFO | ANALYSIS : 62.9%, 3.0, 97.7ms, 0 | 20260505 17:15:23.644 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:23.644 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:23.644 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 17:15:23.644 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:23.644 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:23.644 | INFO | Uploaded 36GiB to GPU in 2.20451s (16.3302GiB/s) | 20260505 17:15:23.644 | INFO | Downloaded 34GiB to GPU in 2.04817s (16.6002GiB/s) | 20260505 17:15:28.781 | INFO | Analyzing chunk frames = [4924928,4925440) | 20260505 17:15:28.781 | INFO | MemInfo: { | 20260505 17:15:28.781 | INFO | "gpu" : " 17 GB", | 20260505 17:15:28.781 | INFO | "huge" : "", | 20260505 17:15:28.781 | INFO | "huge_shared" : " 27 GB", | 20260505 17:15:28.781 | INFO | "malloc" : " 14 GB", | 20260505 17:15:28.781 | INFO | "pinned" : " 29 GB" | 20260505 17:15:28.781 | INFO | } | 20260505 17:15:28.781 | INFO | Perf report for the last 5.13753 seconds: | 20260505 17:15:28.781 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:15:28.781 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:28.781 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:28.781 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:28.781 | INFO | ANALYSIS : 62.5%, 3.0, 97.1ms, 0 | 20260505 17:15:28.781 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:28.781 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:28.781 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 17:15:28.781 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:28.781 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:28.781 | INFO | Uploaded 36GiB to GPU in 2.18003s (16.5135GiB/s) | 20260505 17:15:28.781 | INFO | Downloaded 34GiB to GPU in 2.13197s (15.9477GiB/s) | 20260505 17:15:33.928 | INFO | Analyzing chunk frames = [4925440,4925952) | 20260505 17:15:33.928 | INFO | MemInfo: { | 20260505 17:15:33.928 | INFO | "gpu" : " 17 GB", | 20260505 17:15:33.928 | INFO | "huge" : "", | 20260505 17:15:33.928 | INFO | "huge_shared" : " 27 GB", | 20260505 17:15:33.928 | INFO | "malloc" : " 14 GB", | 20260505 17:15:33.928 | INFO | "pinned" : " 29 GB" | 20260505 17:15:33.928 | INFO | } | 20260505 17:15:33.928 | INFO | Perf report for the last 5.14716 seconds: | 20260505 17:15:33.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:15:33.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:33.928 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:33.928 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:33.928 | INFO | ANALYSIS : 62.2%, 3.0, 97.3ms, 0 | 20260505 17:15:33.928 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:33.928 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:33.928 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 17:15:33.928 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:33.928 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:33.929 | INFO | Uploaded 36GiB to GPU in 2.1388s (16.8319GiB/s) | 20260505 17:15:33.929 | INFO | Downloaded 34GiB to GPU in 2.05622s (16.5352GiB/s) | 20260505 17:15:39.072 | INFO | Analyzing chunk frames = [4925952,4926464) | 20260505 17:15:39.072 | INFO | MemInfo: { | 20260505 17:15:39.072 | INFO | "gpu" : " 17 GB", | 20260505 17:15:39.072 | INFO | "huge" : "", | 20260505 17:15:39.072 | INFO | "huge_shared" : " 27 GB", | 20260505 17:15:39.072 | INFO | "malloc" : " 14 GB", | 20260505 17:15:39.072 | INFO | "pinned" : " 29 GB" | 20260505 17:15:39.072 | INFO | } | 20260505 17:15:39.072 | INFO | Perf report for the last 5.14345 seconds: | 20260505 17:15:39.072 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:15:39.072 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:39.072 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:39.072 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:39.072 | INFO | ANALYSIS : 62.3%, 3.0, 97.4ms, 0 | 20260505 17:15:39.072 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:39.072 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:39.072 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 17:15:39.072 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:39.072 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:39.072 | INFO | Uploaded 36GiB to GPU in 2.14302s (16.7987GiB/s) | 20260505 17:15:39.072 | INFO | Downloaded 34GiB to GPU in 2.04736s (16.6067GiB/s) | 20260505 17:15:44.210 | INFO | Analyzing chunk frames = [4926464,4926976) | 20260505 17:15:44.210 | INFO | MemInfo: { | 20260505 17:15:44.210 | INFO | "gpu" : " 17 GB", | 20260505 17:15:44.210 | INFO | "huge" : "", | 20260505 17:15:44.210 | INFO | "huge_shared" : " 27 GB", | 20260505 17:15:44.210 | INFO | "malloc" : " 14 GB", | 20260505 17:15:44.210 | INFO | "pinned" : " 29 GB" | 20260505 17:15:44.210 | INFO | } | 20260505 17:15:44.210 | INFO | Perf report for the last 5.13846 seconds: | 20260505 17:15:44.210 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:15:44.210 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:44.210 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:44.210 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:44.210 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 17:15:44.210 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:44.210 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:44.210 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 17:15:44.210 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:44.210 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:44.210 | INFO | Uploaded 36GiB to GPU in 2.14876s (16.7538GiB/s) | 20260505 17:15:44.211 | INFO | Downloaded 34GiB to GPU in 2.14487s (15.8518GiB/s) | 20260505 17:15:49.338 | INFO | Analyzing chunk frames = [4926976,4927488) | 20260505 17:15:49.338 | INFO | MemInfo: { | 20260505 17:15:49.338 | INFO | "gpu" : " 17 GB", | 20260505 17:15:49.338 | INFO | "huge" : "", | 20260505 17:15:49.338 | INFO | "huge_shared" : " 27 GB", | 20260505 17:15:49.338 | INFO | "malloc" : " 14 GB", | 20260505 17:15:49.338 | INFO | "pinned" : " 29 GB" | 20260505 17:15:49.338 | INFO | } | 20260505 17:15:49.339 | INFO | Perf report for the last 5.12828 seconds: | 20260505 17:15:49.339 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:15:49.339 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:49.339 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:49.339 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:49.339 | INFO | ANALYSIS : 63.1%, 3.0, 98.3ms, 0 | 20260505 17:15:49.339 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:49.339 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:49.339 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 17:15:49.339 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:49.339 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:49.339 | INFO | Uploaded 36GiB to GPU in 2.20398s (16.3341GiB/s) | 20260505 17:15:49.339 | INFO | Downloaded 34GiB to GPU in 2.01976s (16.8337GiB/s) | 20260505 17:15:49.469 | INFO | Starting a round of Baz IO | 20260505 17:15:52.670 | INFO | Populating RTMetrics report | 20260505 17:15:52.670 | INFO | RTMetrics report populated | 20260505 17:15:52.670 | INFO | Opening tmp RTMetrics json file | 20260505 17:15:52.677 | INFO | Serializing RTMetrics to json | 20260505 17:15:52.679 | INFO | Serialization done, swapping tmp file now... | 20260505 17:15:52.679 | INFO | RTMetrics done | 20260505 17:15:52.764 | INFO | Finished a round of Baz IO: Wrote 2.139 GiB in 3.295 seconds | 20260505 17:15:52.764 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.09% / 0.00% / 77.91% | 20260505 17:15:52.764 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.311553% / 7.94729e-06% / 0.000103315% | 20260505 17:15:52.764 | INFO | Baz IO overall: | 20260505 17:15:52.764 | INFO | BytesWritten : 2915513113278 | 20260505 17:15:52.764 | INFO | NumEvents : 1382787917160 | 20260505 17:15:54.477 | INFO | Analyzing chunk frames = [4927488,4928000) | 20260505 17:15:54.477 | INFO | MemInfo: { | 20260505 17:15:54.477 | INFO | "gpu" : " 17 GB", | 20260505 17:15:54.477 | INFO | "huge" : "", | 20260505 17:15:54.477 | INFO | "huge_shared" : " 27 GB", | 20260505 17:15:54.477 | INFO | "malloc" : " 14 GB", | 20260505 17:15:54.477 | INFO | "pinned" : " 29 GB" | 20260505 17:15:54.477 | INFO | } | 20260505 17:15:54.477 | INFO | Perf report for the last 5.13864 seconds: | 20260505 17:15:54.477 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:15:54.477 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:54.477 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:54.477 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:54.477 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 17:15:54.477 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:54.477 | INFO | RT_METRICS : 44.7%, 1.0, 23.5ms, 0 | 20260505 17:15:54.477 | INFO | PRE_HQ : 60.7%, 1.8, 56.6ms, 0 | 20260505 17:15:54.477 | INFO | BAZWRITER : 63.4%, 3.8, 124.1ms, 0 | 20260505 17:15:54.477 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 17:15:54.477 | INFO | Uploaded 36GiB to GPU in 2.16854s (16.6011GiB/s) | 20260505 17:15:54.477 | INFO | Downloaded 38GiB to GPU in 2.2549s (16.8522GiB/s) | 20260505 17:15:59.624 | INFO | Analyzing chunk frames = [4928000,4928512) | 20260505 17:15:59.624 | INFO | MemInfo: { | 20260505 17:15:59.624 | INFO | "gpu" : " 17 GB", | 20260505 17:15:59.624 | INFO | "huge" : "", | 20260505 17:15:59.624 | INFO | "huge_shared" : " 27 GB", | 20260505 17:15:59.624 | INFO | "malloc" : " 14 GB", | 20260505 17:15:59.624 | INFO | "pinned" : " 29 GB" | 20260505 17:15:59.624 | INFO | } | 20260505 17:15:59.624 | INFO | Perf report for the last 5.14706 seconds: | 20260505 17:15:59.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:15:59.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:59.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:59.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:59.624 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 17:15:59.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:59.624 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:59.624 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260505 17:15:59.624 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:59.624 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:15:59.624 | INFO | Uploaded 36GiB to GPU in 2.13481s (16.8634GiB/s) | 20260505 17:15:59.625 | INFO | Downloaded 34GiB to GPU in 2.03173s (16.7345GiB/s) | 20260505 17:16:04.766 | INFO | Analyzing chunk frames = [4928512,4929024) | 20260505 17:16:04.766 | INFO | MemInfo: { | 20260505 17:16:04.766 | INFO | "gpu" : " 17 GB", | 20260505 17:16:04.766 | INFO | "huge" : "", | 20260505 17:16:04.766 | INFO | "huge_shared" : " 27 GB", | 20260505 17:16:04.766 | INFO | "malloc" : " 14 GB", | 20260505 17:16:04.766 | INFO | "pinned" : " 29 GB" | 20260505 17:16:04.766 | INFO | } | 20260505 17:16:04.766 | INFO | Perf report for the last 5.14181 seconds: | 20260505 17:16:04.766 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:16:04.766 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:04.766 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:04.766 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:04.766 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 17:16:04.766 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:04.766 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:04.766 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 17:16:04.766 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:04.766 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:04.766 | INFO | Uploaded 36GiB to GPU in 2.20571s (16.3213GiB/s) | 20260505 17:16:04.766 | INFO | Downloaded 34GiB to GPU in 2.07574s (16.3797GiB/s) | 20260505 17:16:09.904 | INFO | Analyzing chunk frames = [4929024,4929536) | 20260505 17:16:09.904 | INFO | MemInfo: { | 20260505 17:16:09.904 | INFO | "gpu" : " 17 GB", | 20260505 17:16:09.904 | INFO | "huge" : "", | 20260505 17:16:09.904 | INFO | "huge_shared" : " 27 GB", | 20260505 17:16:09.904 | INFO | "malloc" : " 14 GB", | 20260505 17:16:09.904 | INFO | "pinned" : " 29 GB" | 20260505 17:16:09.904 | INFO | } | 20260505 17:16:09.904 | INFO | Perf report for the last 5.13784 seconds: | 20260505 17:16:09.904 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:16:09.904 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:09.904 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:09.904 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:09.904 | INFO | ANALYSIS : 62.9%, 3.0, 97.7ms, 0 | 20260505 17:16:09.904 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:09.904 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:09.904 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:16:09.904 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:09.904 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:09.904 | INFO | Uploaded 36GiB to GPU in 2.22433s (16.1846GiB/s) | 20260505 17:16:09.904 | INFO | Downloaded 34GiB to GPU in 2.12542s (15.9969GiB/s) | 20260505 17:16:15.040 | INFO | Analyzing chunk frames = [4929536,4930048) | 20260505 17:16:15.040 | INFO | MemInfo: { | 20260505 17:16:15.040 | INFO | "gpu" : " 17 GB", | 20260505 17:16:15.040 | INFO | "huge" : "", | 20260505 17:16:15.040 | INFO | "huge_shared" : " 27 GB", | 20260505 17:16:15.040 | INFO | "malloc" : " 14 GB", | 20260505 17:16:15.040 | INFO | "pinned" : " 29 GB" | 20260505 17:16:15.040 | INFO | } | 20260505 17:16:15.040 | INFO | Perf report for the last 5.13618 seconds: | 20260505 17:16:15.040 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:16:15.040 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:15.040 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:15.040 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:15.040 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 17:16:15.040 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:15.040 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:15.040 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:16:15.040 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:15.040 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:15.040 | INFO | Uploaded 36GiB to GPU in 2.1504s (16.741GiB/s) | 20260505 17:16:15.040 | INFO | Downloaded 34GiB to GPU in 2.05986s (16.506GiB/s) | 20260505 17:16:20.182 | INFO | Analyzing chunk frames = [4930048,4930560) | 20260505 17:16:20.182 | INFO | MemInfo: { | 20260505 17:16:20.182 | INFO | "gpu" : " 17 GB", | 20260505 17:16:20.182 | INFO | "huge" : "", | 20260505 17:16:20.182 | INFO | "huge_shared" : " 27 GB", | 20260505 17:16:20.182 | INFO | "malloc" : " 14 GB", | 20260505 17:16:20.182 | INFO | "pinned" : " 29 GB" | 20260505 17:16:20.182 | INFO | } | 20260505 17:16:20.182 | INFO | Perf report for the last 5.14196 seconds: | 20260505 17:16:20.182 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:16:20.182 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:20.182 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:20.182 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:20.182 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260505 17:16:20.182 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:20.182 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:20.182 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:16:20.182 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:20.182 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:20.183 | INFO | Uploaded 36GiB to GPU in 2.14201s (16.8066GiB/s) | 20260505 17:16:20.183 | INFO | Downloaded 34GiB to GPU in 2.03849s (16.679GiB/s) | 20260505 17:16:25.323 | INFO | Analyzing chunk frames = [4930560,4931072) | 20260505 17:16:25.323 | INFO | MemInfo: { | 20260505 17:16:25.323 | INFO | "gpu" : " 17 GB", | 20260505 17:16:25.323 | INFO | "huge" : "", | 20260505 17:16:25.323 | INFO | "huge_shared" : " 27 GB", | 20260505 17:16:25.323 | INFO | "malloc" : " 14 GB", | 20260505 17:16:25.323 | INFO | "pinned" : " 29 GB" | 20260505 17:16:25.323 | INFO | } | 20260505 17:16:25.323 | INFO | Perf report for the last 5.14069 seconds: | 20260505 17:16:25.323 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:16:25.323 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:25.323 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:25.323 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:25.323 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 17:16:25.323 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:25.323 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:25.323 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 17:16:25.323 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:25.323 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:25.323 | INFO | Uploaded 36GiB to GPU in 2.12399s (16.9493GiB/s) | 20260505 17:16:25.323 | INFO | Downloaded 34GiB to GPU in 2.14139s (15.8776GiB/s) | 20260505 17:16:30.454 | INFO | Analyzing chunk frames = [4931072,4931584) | 20260505 17:16:30.454 | INFO | MemInfo: { | 20260505 17:16:30.454 | INFO | "gpu" : " 17 GB", | 20260505 17:16:30.454 | INFO | "huge" : "", | 20260505 17:16:30.454 | INFO | "huge_shared" : " 27 GB", | 20260505 17:16:30.454 | INFO | "malloc" : " 14 GB", | 20260505 17:16:30.454 | INFO | "pinned" : " 29 GB" | 20260505 17:16:30.454 | INFO | } | 20260505 17:16:30.454 | INFO | Perf report for the last 5.13098 seconds: | 20260505 17:16:30.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:16:30.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:30.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:30.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:30.454 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 17:16:30.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:30.454 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:30.454 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 17:16:30.454 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:30.454 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:30.454 | INFO | Uploaded 36GiB to GPU in 2.19843s (16.3753GiB/s) | 20260505 17:16:30.454 | INFO | Downloaded 34GiB to GPU in 2.03067s (16.7432GiB/s) | 20260505 17:16:30.454 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 17:16:30.454 | INFO | Total mem usage: 15.3 GiB | 20260505 17:16:30.454 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 17:16:30.454 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.93%) | 20260505 17:16:30.454 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 17:16:30.454 | INFO | High Water Breakdown: | 20260505 17:16:30.454 | INFO | 61% : 8.67 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 17:16:30.454 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 17:16:30.454 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 17:16:30.454 | INFO | ------------------------------------------ | 20260505 17:16:30.454 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:16:30.454 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 17:16:30.454 | INFO | Total mem usage: 26.5 GiB | 20260505 17:16:30.454 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 17:16:30.454 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 17:16:30.454 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 17:16:30.454 | INFO | High Water Breakdown: | 20260505 17:16:30.454 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:16:30.454 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:16:30.454 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:16:30.454 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:16:30.454 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:16:30.454 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:16:30.454 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 17:16:30.454 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 17:16:30.454 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:16:30.454 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 17:16:30.454 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 17:16:30.454 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 17:16:30.454 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 17:16:30.454 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 17:16:30.454 | INFO | ------------------------------------------ | 20260505 17:16:30.454 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:16:30.454 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 17:16:30.454 | INFO | Total mem usage: 25.6 GiB | 20260505 17:16:30.455 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 17:16:30.455 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 17:16:30.455 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 17:16:30.455 | INFO | High Water Breakdown: | 20260505 17:16:30.455 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 17:16:30.455 | INFO | ------------------------------------------ | 20260505 17:16:30.455 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:16:30.455 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 17:16:30.455 | INFO | Total mem usage: 15.7 GiB | 20260505 17:16:30.455 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 17:16:30.455 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 17:16:30.455 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 17:16:30.455 | INFO | High Water Breakdown: | 20260505 17:16:30.455 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 17:16:30.455 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 17:16:30.455 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 17:16:30.455 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 17:16:30.455 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 17:16:30.455 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 17:16:30.455 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 17:16:30.455 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 17:16:30.455 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 17:16:30.455 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:16:30.455 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 17:16:30.455 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:16:30.455 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 17:16:30.455 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 17:16:30.455 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 17:16:30.455 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 17:16:30.455 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:16:30.455 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 17:16:30.455 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:16:30.455 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:16:30.455 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:16:30.455 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 17:16:30.455 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 17:16:30.455 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:16:30.455 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 17:16:30.455 | INFO | ------------------------------------------ | 20260505 17:16:30.455 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:16:30.455 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 17:16:30.455 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:16:30.455 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:16:30.455 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.50e+01, 1.34e+01 / 2.20e+00 | 20260505 17:16:30.455 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.02e-01, 1.05e-02 / 4.52e-03 | 20260505 17:16:30.455 | INFO | PerfReport: 11.76%, Baseline -- 3.72e+00 / 4.50e+00, 5.61e-01 / 1.79e-01 | 20260505 17:16:30.455 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.51e+00, 1.91e-01 / 1.02e-01 | 20260505 17:16:30.455 | INFO | PerfReport: 15.94%, DME -- 4.03e+01 / 4.45e+01, 4.14e+00 / 2.38e+00 | 20260505 17:16:30.455 | INFO | PerfReport: 56.82%, FrameLabeling -- 1.80e+01 / 1.96e+01, 1.46e+00 / 1.36e+00 | 20260505 17:16:30.455 | INFO | PerfReport: 7.35%, PulseAccumulating -- 2.32e+00 / 2.91e+00, 4.43e-01 / 1.36e-01 | 20260505 17:16:30.455 | INFO | PerfReport: 3.38%, Metrics -- 1.07e+00 / 1.39e+00, 2.95e-01 / 1.48e-01 | 20260505 17:16:30.455 | INFO | PerfReport: 1.18%, PulseSerializing -- 3.73e-01 / 4.35e-01, 6.66e-02 / 4.08e-02 | 20260505 17:16:30.455 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 17:16:30.455 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:16:30.455 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:16:30.455 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.57e+01 / 0.00e+00 | 20260505 17:16:30.455 | INFO | PerfReport: 22.56%, Upload -- 2.26e+01 / 0.00e+00, 7.91e+00 / 0.00e+00 | 20260505 17:16:30.455 | INFO | PerfReport: 21.80%, Download -- 2.18e+01 / 0.00e+00, 7.86e+00 / 0.00e+00 | 20260505 17:16:30.455 | INFO | PerfReport: 31.58%, Compute -- 3.16e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 17:16:30.455 | INFO | PerfReport: 2.33%, LockUpload -- 2.34e+00 / 0.00e+00, 7.96e+00 / 0.00e+00 | 20260505 17:16:30.455 | INFO | PerfReport: 1.35%, LockDownload -- 1.35e+00 / 0.00e+00, 2.84e+00 / 0.00e+00 | 20260505 17:16:30.455 | INFO | PerfReport: 20.37%, LockCompute -- 2.04e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 17:16:30.455 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.90e-02 / 0.00e+00 | 20260505 17:16:30.588 | INFO | Starting a round of Baz IO | 20260505 17:16:33.783 | INFO | Populating RTMetrics report | 20260505 17:16:33.783 | INFO | RTMetrics report populated | 20260505 17:16:33.783 | INFO | Opening tmp RTMetrics json file | 20260505 17:16:33.789 | INFO | Serializing RTMetrics to json | 20260505 17:16:33.792 | INFO | Serialization done, swapping tmp file now... | 20260505 17:16:33.795 | INFO | RTMetrics done | 20260505 17:16:33.872 | INFO | Finished a round of Baz IO: Wrote 2.142 GiB in 3.283 seconds | 20260505 17:16:33.872 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.09% / 0.00% / 77.91% | 20260505 17:16:33.872 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.312944% / 3.97364e-06% / 0.000103315% | 20260505 17:16:33.872 | INFO | Baz IO overall: | 20260505 17:16:33.872 | INFO | BytesWritten : 2917812625737 | 20260505 17:16:33.872 | INFO | NumEvents : 1383863856251 | 20260505 17:16:35.591 | INFO | Analyzing chunk frames = [4931584,4932096) | 20260505 17:16:35.591 | INFO | MemInfo: { | 20260505 17:16:35.591 | INFO | "gpu" : " 17 GB", | 20260505 17:16:35.591 | INFO | "huge" : "", | 20260505 17:16:35.591 | INFO | "huge_shared" : " 27 GB", | 20260505 17:16:35.591 | INFO | "malloc" : " 14 GB", | 20260505 17:16:35.591 | INFO | "pinned" : " 29 GB" | 20260505 17:16:35.591 | INFO | } | 20260505 17:16:35.591 | INFO | Perf report for the last 5.13722 seconds: | 20260505 17:16:35.591 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:16:35.591 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:35.591 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:35.591 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:35.591 | INFO | ANALYSIS : 63.7%, 3.0, 99.2ms, 0 | 20260505 17:16:35.591 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:35.591 | INFO | RT_METRICS : 44.7%, 1.0, 23.4ms, 0 | 20260505 17:16:35.591 | INFO | PRE_HQ : 60.6%, 1.8, 56.1ms, 0 | 20260505 17:16:35.591 | INFO | BAZWRITER : 63.3%, 3.8, 124.5ms, 0 | 20260505 17:16:35.591 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 17:16:35.591 | INFO | Uploaded 36GiB to GPU in 2.16104s (16.6587GiB/s) | 20260505 17:16:35.591 | INFO | Downloaded 38GiB to GPU in 2.25051s (16.8851GiB/s) | 20260505 17:16:40.732 | INFO | Analyzing chunk frames = [4932096,4932608) | 20260505 17:16:40.732 | INFO | MemInfo: { | 20260505 17:16:40.732 | INFO | "gpu" : " 17 GB", | 20260505 17:16:40.732 | INFO | "huge" : "", | 20260505 17:16:40.732 | INFO | "huge_shared" : " 27 GB", | 20260505 17:16:40.732 | INFO | "malloc" : " 14 GB", | 20260505 17:16:40.732 | INFO | "pinned" : " 29 GB" | 20260505 17:16:40.732 | INFO | } | 20260505 17:16:40.732 | INFO | Perf report for the last 5.14115 seconds: | 20260505 17:16:40.732 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:16:40.732 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:40.732 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:40.732 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:40.732 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 17:16:40.732 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:40.732 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:40.732 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:16:40.732 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:40.732 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:40.733 | INFO | Uploaded 36GiB to GPU in 2.20078s (16.3579GiB/s) | 20260505 17:16:40.733 | INFO | Downloaded 34GiB to GPU in 2.07s (16.4251GiB/s) | 20260505 17:16:45.878 | INFO | Analyzing chunk frames = [4932608,4933120) | 20260505 17:16:45.878 | INFO | MemInfo: { | 20260505 17:16:45.878 | INFO | "gpu" : " 17 GB", | 20260505 17:16:45.878 | INFO | "huge" : "", | 20260505 17:16:45.878 | INFO | "huge_shared" : " 27 GB", | 20260505 17:16:45.878 | INFO | "malloc" : " 14 GB", | 20260505 17:16:45.878 | INFO | "pinned" : " 29 GB" | 20260505 17:16:45.878 | INFO | } | 20260505 17:16:45.878 | INFO | Perf report for the last 5.14544 seconds: | 20260505 17:16:45.878 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:16:45.878 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:45.878 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:45.878 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:45.878 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260505 17:16:45.878 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:45.878 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:45.878 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:16:45.878 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:45.878 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:45.878 | INFO | Uploaded 36GiB to GPU in 2.20579s (16.3207GiB/s) | 20260505 17:16:45.878 | INFO | Downloaded 34GiB to GPU in 2.0471s (16.6089GiB/s) | 20260505 17:16:51.023 | INFO | Analyzing chunk frames = [4933120,4933632) | 20260505 17:16:51.023 | INFO | MemInfo: { | 20260505 17:16:51.023 | INFO | "gpu" : " 17 GB", | 20260505 17:16:51.023 | INFO | "huge" : "", | 20260505 17:16:51.023 | INFO | "huge_shared" : " 27 GB", | 20260505 17:16:51.023 | INFO | "malloc" : " 14 GB", | 20260505 17:16:51.024 | INFO | "pinned" : " 29 GB" | 20260505 17:16:51.024 | INFO | } | 20260505 17:16:51.024 | INFO | Perf report for the last 5.14596 seconds: | 20260505 17:16:51.024 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:16:51.024 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:51.024 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:51.024 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:51.024 | INFO | ANALYSIS : 62.8%, 3.0, 97.6ms, 0 | 20260505 17:16:51.024 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:51.024 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:51.024 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:16:51.024 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:51.024 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:51.024 | INFO | Uploaded 36GiB to GPU in 2.22388s (16.1879GiB/s) | 20260505 17:16:51.024 | INFO | Downloaded 34GiB to GPU in 2.12438s (16.0047GiB/s) | 20260505 17:16:56.155 | INFO | Analyzing chunk frames = [4933632,4934144) | 20260505 17:16:56.155 | INFO | MemInfo: { | 20260505 17:16:56.155 | INFO | "gpu" : " 17 GB", | 20260505 17:16:56.155 | INFO | "huge" : "", | 20260505 17:16:56.156 | INFO | "huge_shared" : " 27 GB", | 20260505 17:16:56.156 | INFO | "malloc" : " 14 GB", | 20260505 17:16:56.156 | INFO | "pinned" : " 29 GB" | 20260505 17:16:56.156 | INFO | } | 20260505 17:16:56.156 | INFO | Perf report for the last 5.13198 seconds: | 20260505 17:16:56.156 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:16:56.156 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:56.156 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:56.156 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:56.156 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 17:16:56.156 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:56.156 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:56.156 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 17:16:56.156 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:56.156 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:16:56.156 | INFO | Uploaded 36GiB to GPU in 2.14468s (16.7857GiB/s) | 20260505 17:16:56.156 | INFO | Downloaded 34GiB to GPU in 2.06531s (16.4624GiB/s) | 20260505 17:17:01.285 | INFO | Analyzing chunk frames = [4934144,4934656) | 20260505 17:17:01.285 | INFO | MemInfo: { | 20260505 17:17:01.285 | INFO | "gpu" : " 17 GB", | 20260505 17:17:01.285 | INFO | "huge" : "", | 20260505 17:17:01.285 | INFO | "huge_shared" : " 27 GB", | 20260505 17:17:01.285 | INFO | "malloc" : " 14 GB", | 20260505 17:17:01.285 | INFO | "pinned" : " 29 GB" | 20260505 17:17:01.285 | INFO | } | 20260505 17:17:01.285 | INFO | Perf report for the last 5.12978 seconds: | 20260505 17:17:01.285 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:17:01.285 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:01.285 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:01.285 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:01.285 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 17:17:01.285 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:01.285 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:01.285 | INFO | PRE_HQ : 27.7%, 1.0, 14.3ms, 0 | 20260505 17:17:01.285 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:01.285 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:01.286 | INFO | Uploaded 36GiB to GPU in 2.1395s (16.8264GiB/s) | 20260505 17:17:01.286 | INFO | Downloaded 34GiB to GPU in 2.04338s (16.6391GiB/s) | 20260505 17:17:06.431 | INFO | Analyzing chunk frames = [4934656,4935168) | 20260505 17:17:06.431 | INFO | MemInfo: { | 20260505 17:17:06.431 | INFO | "gpu" : " 17 GB", | 20260505 17:17:06.431 | INFO | "huge" : "", | 20260505 17:17:06.431 | INFO | "huge_shared" : " 27 GB", | 20260505 17:17:06.431 | INFO | "malloc" : " 14 GB", | 20260505 17:17:06.431 | INFO | "pinned" : " 29 GB" | 20260505 17:17:06.431 | INFO | } | 20260505 17:17:06.431 | INFO | Perf report for the last 5.14537 seconds: | 20260505 17:17:06.431 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:17:06.431 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:06.431 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:06.431 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:06.431 | INFO | ANALYSIS : 63.2%, 3.0, 98.8ms, 0 | 20260505 17:17:06.431 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:06.431 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:06.431 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 17:17:06.431 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:06.431 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:06.431 | INFO | Uploaded 36GiB to GPU in 2.12479s (16.9429GiB/s) | 20260505 17:17:06.431 | INFO | Downloaded 34GiB to GPU in 2.16981s (15.6696GiB/s) | 20260505 17:17:11.570 | INFO | Analyzing chunk frames = [4935168,4935680) | 20260505 17:17:11.570 | INFO | MemInfo: { | 20260505 17:17:11.570 | INFO | "gpu" : " 17 GB", | 20260505 17:17:11.571 | INFO | "huge" : "", | 20260505 17:17:11.571 | INFO | "huge_shared" : " 27 GB", | 20260505 17:17:11.571 | INFO | "malloc" : " 14 GB", | 20260505 17:17:11.571 | INFO | "pinned" : " 29 GB" | 20260505 17:17:11.571 | INFO | } | 20260505 17:17:11.571 | INFO | Perf report for the last 5.13979 seconds: | 20260505 17:17:11.571 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:17:11.571 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:11.571 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:11.571 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:11.571 | INFO | ANALYSIS : 62.9%, 3.0, 97.2ms, 0 | 20260505 17:17:11.571 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:11.571 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:11.571 | INFO | PRE_HQ : 27.9%, 1.0, 14.3ms, 0 | 20260505 17:17:11.571 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:11.571 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:11.571 | INFO | Uploaded 36GiB to GPU in 2.1567s (16.6921GiB/s) | 20260505 17:17:11.571 | INFO | Downloaded 34GiB to GPU in 2.0272s (16.7719GiB/s) | 20260505 17:17:11.705 | INFO | Starting a round of Baz IO | 20260505 17:17:14.906 | INFO | Populating RTMetrics report | 20260505 17:17:14.906 | INFO | RTMetrics report populated | 20260505 17:17:14.906 | INFO | Opening tmp RTMetrics json file | 20260505 17:17:14.913 | INFO | Serializing RTMetrics to json | 20260505 17:17:14.917 | INFO | Serialization done, swapping tmp file now... | 20260505 17:17:14.917 | INFO | RTMetrics done | 20260505 17:17:14.997 | INFO | Finished a round of Baz IO: Wrote 2.136 GiB in 3.291 seconds | 20260505 17:17:14.997 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.09% / 0.00% / 77.91% | 20260505 17:17:14.997 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.309698% / 7.94729e-06% / 7.54992e-05% | 20260505 17:17:14.997 | INFO | Baz IO overall: | 20260505 17:17:14.997 | INFO | BytesWritten : 2920106283504 | 20260505 17:17:14.997 | INFO | NumEvents : 1384935914523 | 20260505 17:17:16.715 | INFO | Analyzing chunk frames = [4935680,4936192) | 20260505 17:17:16.715 | INFO | MemInfo: { | 20260505 17:17:16.715 | INFO | "gpu" : " 17 GB", | 20260505 17:17:16.715 | INFO | "huge" : "", | 20260505 17:17:16.715 | INFO | "huge_shared" : " 27 GB", | 20260505 17:17:16.715 | INFO | "malloc" : " 14 GB", | 20260505 17:17:16.715 | INFO | "pinned" : " 29 GB" | 20260505 17:17:16.715 | INFO | } | 20260505 17:17:16.715 | INFO | Perf report for the last 5.14412 seconds: | 20260505 17:17:16.715 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:17:16.715 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:16.715 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:16.715 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:16.715 | INFO | ANALYSIS : 63.8%, 3.0, 99.5ms, 0 | 20260505 17:17:16.715 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:16.715 | INFO | RT_METRICS : 44.7%, 1.0, 23.5ms, 0 | 20260505 17:17:16.715 | INFO | PRE_HQ : 60.5%, 1.8, 56.3ms, 0 | 20260505 17:17:16.715 | INFO | BAZWRITER : 63.3%, 3.8, 124.0ms, 0 | 20260505 17:17:16.715 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 17:17:16.715 | INFO | Uploaded 36GiB to GPU in 2.18766s (16.456GiB/s) | 20260505 17:17:16.715 | INFO | Downloaded 38GiB to GPU in 2.27516s (16.7021GiB/s) | 20260505 17:17:21.849 | INFO | Analyzing chunk frames = [4936192,4936704) | 20260505 17:17:21.849 | INFO | MemInfo: { | 20260505 17:17:21.849 | INFO | "gpu" : " 17 GB", | 20260505 17:17:21.849 | INFO | "huge" : "", | 20260505 17:17:21.849 | INFO | "huge_shared" : " 27 GB", | 20260505 17:17:21.849 | INFO | "malloc" : " 14 GB", | 20260505 17:17:21.849 | INFO | "pinned" : " 29 GB" | 20260505 17:17:21.849 | INFO | } | 20260505 17:17:21.849 | INFO | Perf report for the last 5.13473 seconds: | 20260505 17:17:21.849 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:17:21.849 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:21.849 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:21.850 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:21.850 | INFO | ANALYSIS : 62.9%, 3.0, 98.0ms, 0 | 20260505 17:17:21.850 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:21.850 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:21.850 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:17:21.850 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:21.850 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:21.850 | INFO | Uploaded 36GiB to GPU in 2.14617s (16.774GiB/s) | 20260505 17:17:21.850 | INFO | Downloaded 34GiB to GPU in 2.03536s (16.7047GiB/s) | 20260505 17:17:26.995 | INFO | Analyzing chunk frames = [4936704,4937216) | 20260505 17:17:26.995 | INFO | MemInfo: { | 20260505 17:17:26.995 | INFO | "gpu" : " 17 GB", | 20260505 17:17:26.995 | INFO | "huge" : "", | 20260505 17:17:26.995 | INFO | "huge_shared" : " 27 GB", | 20260505 17:17:26.995 | INFO | "malloc" : " 14 GB", | 20260505 17:17:26.995 | INFO | "pinned" : " 29 GB" | 20260505 17:17:26.995 | INFO | } | 20260505 17:17:26.995 | INFO | Perf report for the last 5.14527 seconds: | 20260505 17:17:26.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:17:26.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:26.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:26.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:26.995 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260505 17:17:26.995 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:26.995 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:26.995 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:17:26.995 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:26.995 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:26.995 | INFO | Uploaded 36GiB to GPU in 2.19903s (16.3709GiB/s) | 20260505 17:17:26.995 | INFO | Downloaded 34GiB to GPU in 2.03893s (16.6754GiB/s) | 20260505 17:17:32.133 | INFO | Analyzing chunk frames = [4937216,4937728) | 20260505 17:17:32.133 | INFO | MemInfo: { | 20260505 17:17:32.133 | INFO | "gpu" : " 17 GB", | 20260505 17:17:32.133 | INFO | "huge" : "", | 20260505 17:17:32.133 | INFO | "huge_shared" : " 27 GB", | 20260505 17:17:32.133 | INFO | "malloc" : " 14 GB", | 20260505 17:17:32.133 | INFO | "pinned" : " 29 GB" | 20260505 17:17:32.133 | INFO | } | 20260505 17:17:32.133 | INFO | Perf report for the last 5.13803 seconds: | 20260505 17:17:32.133 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:17:32.133 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:32.133 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:32.133 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:32.133 | INFO | ANALYSIS : 62.8%, 3.0, 97.6ms, 0 | 20260505 17:17:32.133 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:32.133 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:32.133 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 17:17:32.133 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:32.133 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:32.133 | INFO | Uploaded 36GiB to GPU in 2.21931s (16.2213GiB/s) | 20260505 17:17:32.133 | INFO | Downloaded 34GiB to GPU in 2.12577s (15.9942GiB/s) | 20260505 17:17:37.271 | INFO | Analyzing chunk frames = [4937728,4938240) | 20260505 17:17:37.271 | INFO | MemInfo: { | 20260505 17:17:37.271 | INFO | "gpu" : " 17 GB", | 20260505 17:17:37.271 | INFO | "huge" : "", | 20260505 17:17:37.271 | INFO | "huge_shared" : " 27 GB", | 20260505 17:17:37.271 | INFO | "malloc" : " 14 GB", | 20260505 17:17:37.271 | INFO | "pinned" : " 29 GB" | 20260505 17:17:37.271 | INFO | } | 20260505 17:17:37.271 | INFO | Perf report for the last 5.13836 seconds: | 20260505 17:17:37.271 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:17:37.271 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:37.271 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:37.271 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:37.271 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 17:17:37.271 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:37.271 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:37.271 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 17:17:37.271 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:37.271 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:37.271 | INFO | Uploaded 36GiB to GPU in 2.14999s (16.7443GiB/s) | 20260505 17:17:37.271 | INFO | Downloaded 34GiB to GPU in 2.06303s (16.4806GiB/s) | 20260505 17:17:42.409 | INFO | Analyzing chunk frames = [4938240,4938752) | 20260505 17:17:42.409 | INFO | MemInfo: { | 20260505 17:17:42.409 | INFO | "gpu" : " 17 GB", | 20260505 17:17:42.409 | INFO | "huge" : "", | 20260505 17:17:42.409 | INFO | "huge_shared" : " 27 GB", | 20260505 17:17:42.409 | INFO | "malloc" : " 14 GB", | 20260505 17:17:42.409 | INFO | "pinned" : " 29 GB" | 20260505 17:17:42.409 | INFO | } | 20260505 17:17:42.409 | INFO | Perf report for the last 5.13816 seconds: | 20260505 17:17:42.409 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:17:42.409 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:42.409 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:42.409 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:42.409 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260505 17:17:42.409 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:42.409 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:42.409 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 17:17:42.409 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:42.409 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:42.410 | INFO | Uploaded 36GiB to GPU in 2.13996s (16.8228GiB/s) | 20260505 17:17:42.410 | INFO | Downloaded 34GiB to GPU in 2.05108s (16.5767GiB/s) | 20260505 17:17:47.541 | INFO | Analyzing chunk frames = [4938752,4939264) | 20260505 17:17:47.541 | INFO | MemInfo: { | 20260505 17:17:47.541 | INFO | "gpu" : " 17 GB", | 20260505 17:17:47.541 | INFO | "huge" : "", | 20260505 17:17:47.541 | INFO | "huge_shared" : " 27 GB", | 20260505 17:17:47.541 | INFO | "malloc" : " 14 GB", | 20260505 17:17:47.541 | INFO | "pinned" : " 29 GB" | 20260505 17:17:47.541 | INFO | } | 20260505 17:17:47.541 | INFO | Perf report for the last 5.13202 seconds: | 20260505 17:17:47.541 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:17:47.541 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:47.541 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:47.541 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:47.541 | INFO | ANALYSIS : 63.6%, 3.0, 99.1ms, 0 | 20260505 17:17:47.541 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:47.541 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:47.541 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 17:17:47.541 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:47.541 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:47.542 | INFO | Uploaded 36GiB to GPU in 2.15435s (16.7103GiB/s) | 20260505 17:17:47.542 | INFO | Downloaded 34GiB to GPU in 2.18502s (15.5605GiB/s) | 20260505 17:17:52.687 | INFO | Analyzing chunk frames = [4939264,4939776) | 20260505 17:17:52.687 | INFO | MemInfo: { | 20260505 17:17:52.687 | INFO | "gpu" : " 17 GB", | 20260505 17:17:52.687 | INFO | "huge" : "", | 20260505 17:17:52.687 | INFO | "huge_shared" : " 27 GB", | 20260505 17:17:52.687 | INFO | "malloc" : " 14 GB", | 20260505 17:17:52.687 | INFO | "pinned" : " 29 GB" | 20260505 17:17:52.687 | INFO | } | 20260505 17:17:52.687 | INFO | Perf report for the last 5.14612 seconds: | 20260505 17:17:52.687 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:17:52.687 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:52.687 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:52.688 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:52.688 | INFO | ANALYSIS : 63.1%, 3.0, 98.7ms, 0 | 20260505 17:17:52.688 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:52.688 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:52.688 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 17:17:52.688 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:52.688 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:52.688 | INFO | Uploaded 36GiB to GPU in 2.1776s (16.5319GiB/s) | 20260505 17:17:52.688 | INFO | Downloaded 34GiB to GPU in 2.0383s (16.6806GiB/s) | 20260505 17:17:52.688 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 17:17:52.688 | INFO | Total mem usage: 15.3 GiB | 20260505 17:17:52.688 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 17:17:52.688 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.92%) | 20260505 17:17:52.688 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 17:17:52.688 | INFO | High Water Breakdown: | 20260505 17:17:52.688 | INFO | 61% : 8.67 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 17:17:52.688 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 17:17:52.688 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 17:17:52.688 | INFO | ------------------------------------------ | 20260505 17:17:52.688 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:17:52.688 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 17:17:52.688 | INFO | Total mem usage: 26.5 GiB | 20260505 17:17:52.688 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 17:17:52.688 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 17:17:52.688 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 17:17:52.688 | INFO | High Water Breakdown: | 20260505 17:17:52.688 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:17:52.688 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:17:52.688 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:17:52.688 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:17:52.688 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:17:52.688 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:17:52.688 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 17:17:52.688 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 17:17:52.688 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:17:52.688 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 17:17:52.688 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 17:17:52.688 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 17:17:52.688 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 17:17:52.688 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 17:17:52.688 | INFO | ------------------------------------------ | 20260505 17:17:52.688 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:17:52.688 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 17:17:52.688 | INFO | Total mem usage: 25.6 GiB | 20260505 17:17:52.688 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 17:17:52.688 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 17:17:52.688 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 17:17:52.688 | INFO | High Water Breakdown: | 20260505 17:17:52.688 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 17:17:52.688 | INFO | ------------------------------------------ | 20260505 17:17:52.688 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:17:52.688 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 17:17:52.688 | INFO | Total mem usage: 15.7 GiB | 20260505 17:17:52.688 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 17:17:52.688 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 17:17:52.688 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 17:17:52.688 | INFO | High Water Breakdown: | 20260505 17:17:52.688 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 17:17:52.688 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 17:17:52.688 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 17:17:52.688 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 17:17:52.688 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 17:17:52.688 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 17:17:52.688 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 17:17:52.688 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 17:17:52.688 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 17:17:52.688 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:17:52.688 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 17:17:52.688 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:17:52.688 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 17:17:52.688 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 17:17:52.688 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 17:17:52.688 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 17:17:52.688 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:17:52.688 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 17:17:52.688 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:17:52.688 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:17:52.688 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:17:52.688 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 17:17:52.688 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 17:17:52.688 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:17:52.688 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 17:17:52.688 | INFO | ------------------------------------------ | 20260505 17:17:52.688 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:17:52.689 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 17:17:52.689 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:17:52.689 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:17:52.689 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.49e+01, 1.36e+01 / 2.02e+00 | 20260505 17:17:52.689 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.02e-01, 1.05e-02 / 4.00e-03 | 20260505 17:17:52.689 | INFO | PerfReport: 11.75%, Baseline -- 3.71e+00 / 4.50e+00, 5.58e-01 / 1.75e-01 | 20260505 17:17:52.689 | INFO | PerfReport: 3.25%, Histogram -- 1.03e+00 / 1.52e+00, 1.87e-01 / 9.41e-02 | 20260505 17:17:52.689 | INFO | PerfReport: 15.96%, DME -- 4.03e+01 / 4.47e+01, 4.22e+00 / 2.09e+00 | 20260505 17:17:52.689 | INFO | PerfReport: 56.78%, FrameLabeling -- 1.79e+01 / 1.93e+01, 1.40e+00 / 1.05e+00 | 20260505 17:17:52.689 | INFO | PerfReport: 7.39%, PulseAccumulating -- 2.33e+00 / 2.94e+00, 4.51e-01 / 1.37e-01 | 20260505 17:17:52.689 | INFO | PerfReport: 3.39%, Metrics -- 1.07e+00 / 1.42e+00, 2.95e-01 / 1.45e-01 | 20260505 17:17:52.689 | INFO | PerfReport: 1.18%, PulseSerializing -- 3.72e-01 / 4.39e-01, 6.59e-02 / 3.92e-02 | 20260505 17:17:52.689 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 17:17:52.689 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:17:52.689 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:17:52.689 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.59e+01 / 0.00e+00 | 20260505 17:17:52.689 | INFO | PerfReport: 22.58%, Upload -- 2.26e+01 / 0.00e+00, 7.95e+00 / 0.00e+00 | 20260505 17:17:52.689 | INFO | PerfReport: 21.85%, Download -- 2.19e+01 / 0.00e+00, 7.90e+00 / 0.00e+00 | 20260505 17:17:52.689 | INFO | PerfReport: 31.51%, Compute -- 3.16e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 17:17:52.689 | INFO | PerfReport: 2.44%, LockUpload -- 2.44e+00 / 0.00e+00, 8.09e+00 / 0.00e+00 | 20260505 17:17:52.689 | INFO | PerfReport: 1.39%, LockDownload -- 1.40e+00 / 0.00e+00, 2.96e+00 / 0.00e+00 | 20260505 17:17:52.689 | INFO | PerfReport: 20.22%, LockCompute -- 2.03e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260505 17:17:52.689 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.89e-02 / 0.00e+00 | 20260505 17:17:52.824 | INFO | Starting a round of Baz IO | 20260505 17:17:56.003 | INFO | Populating RTMetrics report | 20260505 17:17:56.003 | INFO | RTMetrics report populated | 20260505 17:17:56.003 | INFO | Opening tmp RTMetrics json file | 20260505 17:17:56.014 | INFO | Serializing RTMetrics to json | 20260505 17:17:56.015 | INFO | Serialization done, swapping tmp file now... | 20260505 17:17:56.016 | INFO | RTMetrics done | 20260505 17:17:56.101 | INFO | Finished a round of Baz IO: Wrote 2.136 GiB in 3.276 seconds | 20260505 17:17:56.101 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.09% / 0.00% / 77.91% | 20260505 17:17:56.101 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.307473% / 0% / 9.53674e-05% | 20260505 17:17:56.101 | INFO | Baz IO overall: | 20260505 17:17:56.101 | INFO | BytesWritten : 2922399748457 | 20260505 17:17:56.101 | INFO | NumEvents : 1386007886412 | 20260505 17:17:57.830 | INFO | Analyzing chunk frames = [4939776,4940288) | 20260505 17:17:57.830 | INFO | MemInfo: { | 20260505 17:17:57.830 | INFO | "gpu" : " 17 GB", | 20260505 17:17:57.830 | INFO | "huge" : "", | 20260505 17:17:57.830 | INFO | "huge_shared" : " 27 GB", | 20260505 17:17:57.830 | INFO | "malloc" : " 14 GB", | 20260505 17:17:57.830 | INFO | "pinned" : " 29 GB" | 20260505 17:17:57.830 | INFO | } | 20260505 17:17:57.830 | INFO | Perf report for the last 5.14292 seconds: | 20260505 17:17:57.830 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:17:57.830 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:57.830 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:57.830 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:57.830 | INFO | ANALYSIS : 63.4%, 3.0, 98.8ms, 0 | 20260505 17:17:57.830 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:17:57.830 | INFO | RT_METRICS : 44.7%, 1.0, 23.4ms, 0 | 20260505 17:17:57.830 | INFO | PRE_HQ : 60.3%, 1.8, 56.2ms, 0 | 20260505 17:17:57.831 | INFO | BAZWRITER : 63.2%, 3.8, 124.6ms, 0 | 20260505 17:17:57.831 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260505 17:17:57.831 | INFO | Uploaded 36GiB to GPU in 2.15843s (16.6788GiB/s) | 20260505 17:17:57.831 | INFO | Downloaded 38GiB to GPU in 2.25233s (16.8714GiB/s) | 20260505 17:18:02.964 | INFO | Analyzing chunk frames = [4940288,4940800) | 20260505 17:18:02.964 | INFO | MemInfo: { | 20260505 17:18:02.964 | INFO | "gpu" : " 17 GB", | 20260505 17:18:02.964 | INFO | "huge" : "", | 20260505 17:18:02.964 | INFO | "huge_shared" : " 27 GB", | 20260505 17:18:02.964 | INFO | "malloc" : " 14 GB", | 20260505 17:18:02.964 | INFO | "pinned" : " 29 GB" | 20260505 17:18:02.964 | INFO | } | 20260505 17:18:02.964 | INFO | Perf report for the last 5.13378 seconds: | 20260505 17:18:02.964 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:18:02.964 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:02.964 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:02.964 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:02.964 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 17:18:02.964 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:02.964 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:02.964 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:18:02.964 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:02.964 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:02.964 | INFO | Uploaded 36GiB to GPU in 2.14346s (16.7953GiB/s) | 20260505 17:18:02.964 | INFO | Downloaded 34GiB to GPU in 2.03291s (16.7248GiB/s) | 20260505 17:18:08.102 | INFO | Analyzing chunk frames = [4940800,4941312) | 20260505 17:18:08.102 | INFO | MemInfo: { | 20260505 17:18:08.102 | INFO | "gpu" : " 17 GB", | 20260505 17:18:08.102 | INFO | "huge" : "", | 20260505 17:18:08.102 | INFO | "huge_shared" : " 27 GB", | 20260505 17:18:08.102 | INFO | "malloc" : " 14 GB", | 20260505 17:18:08.102 | INFO | "pinned" : " 29 GB" | 20260505 17:18:08.102 | INFO | } | 20260505 17:18:08.102 | INFO | Perf report for the last 5.13759 seconds: | 20260505 17:18:08.102 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:18:08.102 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:08.102 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:08.102 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:08.102 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260505 17:18:08.102 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:08.102 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:08.102 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:18:08.102 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:08.102 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:08.102 | INFO | Uploaded 36GiB to GPU in 2.19166s (16.4259GiB/s) | 20260505 17:18:08.102 | INFO | Downloaded 34GiB to GPU in 2.02962s (16.7519GiB/s) | 20260505 17:18:13.236 | INFO | Analyzing chunk frames = [4941312,4941824) | 20260505 17:18:13.236 | INFO | MemInfo: { | 20260505 17:18:13.236 | INFO | "gpu" : " 17 GB", | 20260505 17:18:13.236 | INFO | "huge" : "", | 20260505 17:18:13.236 | INFO | "huge_shared" : " 27 GB", | 20260505 17:18:13.236 | INFO | "malloc" : " 14 GB", | 20260505 17:18:13.236 | INFO | "pinned" : " 29 GB" | 20260505 17:18:13.237 | INFO | } | 20260505 17:18:13.237 | INFO | Perf report for the last 5.13469 seconds: | 20260505 17:18:13.237 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:18:13.237 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:13.237 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:13.237 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:13.237 | INFO | ANALYSIS : 62.5%, 3.0, 97.0ms, 0 | 20260505 17:18:13.237 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:13.237 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:13.237 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:18:13.237 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:13.237 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:13.237 | INFO | Uploaded 36GiB to GPU in 2.17437s (16.5565GiB/s) | 20260505 17:18:13.237 | INFO | Downloaded 34GiB to GPU in 2.13675s (15.912GiB/s) | 20260505 17:18:18.385 | INFO | Analyzing chunk frames = [4941824,4942336) | 20260505 17:18:18.385 | INFO | MemInfo: { | 20260505 17:18:18.385 | INFO | "gpu" : " 17 GB", | 20260505 17:18:18.385 | INFO | "huge" : "", | 20260505 17:18:18.385 | INFO | "huge_shared" : " 27 GB", | 20260505 17:18:18.385 | INFO | "malloc" : " 14 GB", | 20260505 17:18:18.385 | INFO | "pinned" : " 29 GB" | 20260505 17:18:18.385 | INFO | } | 20260505 17:18:18.385 | INFO | Perf report for the last 5.14843 seconds: | 20260505 17:18:18.385 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:18:18.385 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:18.385 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:18.385 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:18.385 | INFO | ANALYSIS : 62.3%, 3.0, 97.4ms, 0 | 20260505 17:18:18.385 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:18.385 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:18.385 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 17:18:18.385 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:18.385 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:18.385 | INFO | Uploaded 36GiB to GPU in 2.13724s (16.8442GiB/s) | 20260505 17:18:18.385 | INFO | Downloaded 34GiB to GPU in 2.06073s (16.499GiB/s) | 20260505 17:18:23.527 | INFO | Analyzing chunk frames = [4942336,4942848) | 20260505 17:18:23.527 | INFO | MemInfo: { | 20260505 17:18:23.527 | INFO | "gpu" : " 17 GB", | 20260505 17:18:23.527 | INFO | "huge" : "", | 20260505 17:18:23.527 | INFO | "huge_shared" : " 27 GB", | 20260505 17:18:23.527 | INFO | "malloc" : " 14 GB", | 20260505 17:18:23.527 | INFO | "pinned" : " 29 GB" | 20260505 17:18:23.527 | INFO | } | 20260505 17:18:23.527 | INFO | Perf report for the last 5.14225 seconds: | 20260505 17:18:23.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:18:23.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:23.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:23.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:23.527 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260505 17:18:23.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:23.527 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:23.527 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 17:18:23.527 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:23.527 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:23.527 | INFO | Uploaded 36GiB to GPU in 2.13814s (16.8371GiB/s) | 20260505 17:18:23.528 | INFO | Downloaded 34GiB to GPU in 2.04347s (16.6384GiB/s) | 20260505 17:18:28.662 | INFO | Analyzing chunk frames = [4942848,4943360) | 20260505 17:18:28.662 | INFO | MemInfo: { | 20260505 17:18:28.662 | INFO | "gpu" : " 17 GB", | 20260505 17:18:28.662 | INFO | "huge" : "", | 20260505 17:18:28.662 | INFO | "huge_shared" : " 27 GB", | 20260505 17:18:28.662 | INFO | "malloc" : " 14 GB", | 20260505 17:18:28.662 | INFO | "pinned" : " 29 GB" | 20260505 17:18:28.662 | INFO | } | 20260505 17:18:28.662 | INFO | Perf report for the last 5.13456 seconds: | 20260505 17:18:28.662 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:18:28.662 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:28.662 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:28.662 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:28.662 | INFO | ANALYSIS : 63.5%, 3.0, 98.0ms, 0 | 20260505 17:18:28.662 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:28.662 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:28.662 | INFO | PRE_HQ : 28.6%, 1.0, 14.7ms, 0 | 20260505 17:18:28.662 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:28.662 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:28.662 | INFO | Uploaded 36GiB to GPU in 2.16021s (16.665GiB/s) | 20260505 17:18:28.662 | INFO | Downloaded 34GiB to GPU in 2.17958s (15.5993GiB/s) | 20260505 17:18:33.794 | INFO | Analyzing chunk frames = [4943360,4943872) | 20260505 17:18:33.794 | INFO | MemInfo: { | 20260505 17:18:33.794 | INFO | "gpu" : " 17 GB", | 20260505 17:18:33.794 | INFO | "huge" : "", | 20260505 17:18:33.794 | INFO | "huge_shared" : " 27 GB", | 20260505 17:18:33.794 | INFO | "malloc" : " 14 GB", | 20260505 17:18:33.794 | INFO | "pinned" : " 29 GB" | 20260505 17:18:33.794 | INFO | } | 20260505 17:18:33.794 | INFO | Perf report for the last 5.13265 seconds: | 20260505 17:18:33.794 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:18:33.794 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:33.794 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:33.794 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:33.794 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 17:18:33.794 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:33.795 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:33.795 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 17:18:33.795 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:33.795 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:33.795 | INFO | Uploaded 36GiB to GPU in 2.16233s (16.6487GiB/s) | 20260505 17:18:33.795 | INFO | Downloaded 34GiB to GPU in 2.03446s (16.7121GiB/s) | 20260505 17:18:33.924 | INFO | Starting a round of Baz IO | 20260505 17:18:37.132 | INFO | Populating RTMetrics report | 20260505 17:18:37.133 | INFO | RTMetrics report populated | 20260505 17:18:37.133 | INFO | Opening tmp RTMetrics json file | 20260505 17:18:37.139 | INFO | Serializing RTMetrics to json | 20260505 17:18:37.140 | INFO | Serialization done, swapping tmp file now... | 20260505 17:18:37.141 | INFO | RTMetrics done | 20260505 17:18:37.227 | INFO | Finished a round of Baz IO: Wrote 2.135 GiB in 3.303 seconds | 20260505 17:18:37.227 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.08% / 0.00% / 77.91% | 20260505 17:18:37.227 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.307461% / 0% / 0.00013113% | 20260505 17:18:37.227 | INFO | Baz IO overall: | 20260505 17:18:37.227 | INFO | BytesWritten : 2924692537136 | 20260505 17:18:37.227 | INFO | NumEvents : 1387079663767 | 20260505 17:18:38.947 | INFO | Analyzing chunk frames = [4943872,4944384) | 20260505 17:18:38.947 | INFO | MemInfo: { | 20260505 17:18:38.947 | INFO | "gpu" : " 17 GB", | 20260505 17:18:38.947 | INFO | "huge" : "", | 20260505 17:18:38.947 | INFO | "huge_shared" : " 27 GB", | 20260505 17:18:38.947 | INFO | "malloc" : " 14 GB", | 20260505 17:18:38.947 | INFO | "pinned" : " 29 GB" | 20260505 17:18:38.947 | INFO | } | 20260505 17:18:38.947 | INFO | Perf report for the last 5.15263 seconds: | 20260505 17:18:38.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:18:38.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:38.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:38.947 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:38.947 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 17:18:38.947 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:38.947 | INFO | RT_METRICS : 44.5%, 1.0, 23.4ms, 0 | 20260505 17:18:38.947 | INFO | PRE_HQ : 60.4%, 1.8, 56.2ms, 0 | 20260505 17:18:38.947 | INFO | BAZWRITER : 63.4%, 3.8, 123.5ms, 0 | 20260505 17:18:38.947 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 17:18:38.947 | INFO | Uploaded 36GiB to GPU in 2.16777s (16.6069GiB/s) | 20260505 17:18:38.947 | INFO | Downloaded 38GiB to GPU in 2.2566s (16.8395GiB/s) | 20260505 17:18:44.080 | INFO | Analyzing chunk frames = [4944384,4944896) | 20260505 17:18:44.080 | INFO | MemInfo: { | 20260505 17:18:44.080 | INFO | "gpu" : " 17 GB", | 20260505 17:18:44.080 | INFO | "huge" : "", | 20260505 17:18:44.080 | INFO | "huge_shared" : " 27 GB", | 20260505 17:18:44.080 | INFO | "malloc" : " 14 GB", | 20260505 17:18:44.080 | INFO | "pinned" : " 29 GB" | 20260505 17:18:44.080 | INFO | } | 20260505 17:18:44.080 | INFO | Perf report for the last 5.13305 seconds: | 20260505 17:18:44.080 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:18:44.080 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:44.080 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:44.080 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:44.080 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 17:18:44.080 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:44.080 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:44.080 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:18:44.080 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:44.080 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:44.080 | INFO | Uploaded 36GiB to GPU in 2.12896s (16.9096GiB/s) | 20260505 17:18:44.081 | INFO | Downloaded 34GiB to GPU in 2.02463s (16.7932GiB/s) | 20260505 17:18:49.217 | INFO | Analyzing chunk frames = [4944896,4945408) | 20260505 17:18:49.217 | INFO | MemInfo: { | 20260505 17:18:49.217 | INFO | "gpu" : " 17 GB", | 20260505 17:18:49.217 | INFO | "huge" : "", | 20260505 17:18:49.217 | INFO | "huge_shared" : " 27 GB", | 20260505 17:18:49.218 | INFO | "malloc" : " 14 GB", | 20260505 17:18:49.218 | INFO | "pinned" : " 29 GB" | 20260505 17:18:49.218 | INFO | } | 20260505 17:18:49.218 | INFO | Perf report for the last 5.13741 seconds: | 20260505 17:18:49.218 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:18:49.218 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:49.218 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:49.218 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:49.218 | INFO | ANALYSIS : 62.8%, 3.0, 97.6ms, 0 | 20260505 17:18:49.218 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:49.218 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:49.218 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:18:49.218 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:49.218 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:49.218 | INFO | Uploaded 36GiB to GPU in 2.20134s (16.3537GiB/s) | 20260505 17:18:49.218 | INFO | Downloaded 34GiB to GPU in 2.04018s (16.6652GiB/s) | 20260505 17:18:54.364 | INFO | Analyzing chunk frames = [4945408,4945920) | 20260505 17:18:54.364 | INFO | MemInfo: { | 20260505 17:18:54.364 | INFO | "gpu" : " 17 GB", | 20260505 17:18:54.364 | INFO | "huge" : "", | 20260505 17:18:54.364 | INFO | "huge_shared" : " 27 GB", | 20260505 17:18:54.364 | INFO | "malloc" : " 14 GB", | 20260505 17:18:54.364 | INFO | "pinned" : " 29 GB" | 20260505 17:18:54.364 | INFO | } | 20260505 17:18:54.364 | INFO | Perf report for the last 5.14616 seconds: | 20260505 17:18:54.364 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:18:54.364 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:54.364 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:54.364 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:54.364 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260505 17:18:54.364 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:54.364 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:54.364 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 17:18:54.364 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:54.364 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:54.364 | INFO | Uploaded 36GiB to GPU in 2.1813s (16.5039GiB/s) | 20260505 17:18:54.364 | INFO | Downloaded 34GiB to GPU in 2.13403s (15.9323GiB/s) | 20260505 17:18:59.496 | INFO | Analyzing chunk frames = [4945920,4946432) | 20260505 17:18:59.496 | INFO | MemInfo: { | 20260505 17:18:59.496 | INFO | "gpu" : " 17 GB", | 20260505 17:18:59.496 | INFO | "huge" : "", | 20260505 17:18:59.496 | INFO | "huge_shared" : " 27 GB", | 20260505 17:18:59.496 | INFO | "malloc" : " 14 GB", | 20260505 17:18:59.496 | INFO | "pinned" : " 29 GB" | 20260505 17:18:59.496 | INFO | } | 20260505 17:18:59.496 | INFO | Perf report for the last 5.13232 seconds: | 20260505 17:18:59.496 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:18:59.496 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:59.496 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:59.496 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:59.496 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 17:18:59.496 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:59.496 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:59.496 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:18:59.496 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:59.496 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:18:59.496 | INFO | Uploaded 36GiB to GPU in 2.15142s (16.7331GiB/s) | 20260505 17:18:59.496 | INFO | Downloaded 34GiB to GPU in 2.05374s (16.5552GiB/s) | 20260505 17:19:04.642 | INFO | Analyzing chunk frames = [4946432,4946944) | 20260505 17:19:04.642 | INFO | MemInfo: { | 20260505 17:19:04.642 | INFO | "gpu" : " 17 GB", | 20260505 17:19:04.642 | INFO | "huge" : "", | 20260505 17:19:04.642 | INFO | "huge_shared" : " 27 GB", | 20260505 17:19:04.642 | INFO | "malloc" : " 14 GB", | 20260505 17:19:04.642 | INFO | "pinned" : " 29 GB" | 20260505 17:19:04.642 | INFO | } | 20260505 17:19:04.642 | INFO | Perf report for the last 5.14608 seconds: | 20260505 17:19:04.642 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:19:04.642 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:04.642 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:04.642 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:04.642 | INFO | ANALYSIS : 62.3%, 3.0, 97.5ms, 0 | 20260505 17:19:04.642 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:04.642 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:04.642 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 17:19:04.642 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:04.642 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:04.642 | INFO | Uploaded 36GiB to GPU in 2.14317s (16.7975GiB/s) | 20260505 17:19:04.642 | INFO | Downloaded 34GiB to GPU in 2.03735s (16.6883GiB/s) | 20260505 17:19:09.783 | INFO | Analyzing chunk frames = [4946944,4947456) | 20260505 17:19:09.783 | INFO | MemInfo: { | 20260505 17:19:09.783 | INFO | "gpu" : " 17 GB", | 20260505 17:19:09.783 | INFO | "huge" : "", | 20260505 17:19:09.783 | INFO | "huge_shared" : " 27 GB", | 20260505 17:19:09.783 | INFO | "malloc" : " 14 GB", | 20260505 17:19:09.783 | INFO | "pinned" : " 29 GB" | 20260505 17:19:09.783 | INFO | } | 20260505 17:19:09.783 | INFO | Perf report for the last 5.14111 seconds: | 20260505 17:19:09.783 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:19:09.783 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:09.783 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:09.783 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:09.783 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 17:19:09.783 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:09.783 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:09.783 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 17:19:09.783 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:09.783 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:09.784 | INFO | Uploaded 36GiB to GPU in 2.13969s (16.8249GiB/s) | 20260505 17:19:09.784 | INFO | Downloaded 34GiB to GPU in 2.13677s (15.9118GiB/s) | 20260505 17:19:14.911 | INFO | Analyzing chunk frames = [4947456,4947968) | 20260505 17:19:14.911 | INFO | MemInfo: { | 20260505 17:19:14.911 | INFO | "gpu" : " 17 GB", | 20260505 17:19:14.911 | INFO | "huge" : "", | 20260505 17:19:14.911 | INFO | "huge_shared" : " 27 GB", | 20260505 17:19:14.911 | INFO | "malloc" : " 14 GB", | 20260505 17:19:14.911 | INFO | "pinned" : " 29 GB" | 20260505 17:19:14.911 | INFO | } | 20260505 17:19:14.911 | INFO | Perf report for the last 5.12737 seconds: | 20260505 17:19:14.911 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:19:14.911 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:14.911 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:14.911 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:14.911 | INFO | ANALYSIS : 63.1%, 3.0, 98.3ms, 0 | 20260505 17:19:14.911 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:14.911 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:14.911 | INFO | PRE_HQ : 28.5%, 1.0, 14.7ms, 0 | 20260505 17:19:14.911 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:14.911 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:14.911 | INFO | Uploaded 36GiB to GPU in 2.19719s (16.3845GiB/s) | 20260505 17:19:14.911 | INFO | Downloaded 34GiB to GPU in 2.03363s (16.7189GiB/s) | 20260505 17:19:14.911 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 17:19:14.911 | INFO | Total mem usage: 15.3 GiB | 20260505 17:19:14.911 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 17:19:14.911 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.98%) | 20260505 17:19:14.911 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 17:19:14.911 | INFO | High Water Breakdown: | 20260505 17:19:14.911 | INFO | 61% : 8.66 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 17:19:14.911 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 17:19:14.911 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 17:19:14.911 | INFO | ------------------------------------------ | 20260505 17:19:14.911 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:19:14.911 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 17:19:14.911 | INFO | Total mem usage: 26.5 GiB | 20260505 17:19:14.911 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 17:19:14.911 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 17:19:14.911 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 17:19:14.911 | INFO | High Water Breakdown: | 20260505 17:19:14.911 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:19:14.911 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:19:14.911 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:19:14.911 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:19:14.911 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:19:14.911 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:19:14.911 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 17:19:14.911 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 17:19:14.911 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:19:14.911 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 17:19:14.911 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 17:19:14.911 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 17:19:14.911 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 17:19:14.911 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 17:19:14.911 | INFO | ------------------------------------------ | 20260505 17:19:14.911 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:19:14.911 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 17:19:14.911 | INFO | Total mem usage: 25.6 GiB | 20260505 17:19:14.911 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 17:19:14.911 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 17:19:14.911 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 17:19:14.911 | INFO | High Water Breakdown: | 20260505 17:19:14.911 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 17:19:14.911 | INFO | ------------------------------------------ | 20260505 17:19:14.911 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:19:14.911 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 17:19:14.911 | INFO | Total mem usage: 15.7 GiB | 20260505 17:19:14.911 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 17:19:14.911 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 17:19:14.911 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 17:19:14.911 | INFO | High Water Breakdown: | 20260505 17:19:14.911 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 17:19:14.911 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 17:19:14.911 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 17:19:14.911 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 17:19:14.912 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 17:19:14.912 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 17:19:14.912 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 17:19:14.912 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 17:19:14.912 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 17:19:14.912 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:19:14.912 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 17:19:14.912 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:19:14.912 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 17:19:14.912 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 17:19:14.912 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 17:19:14.912 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 17:19:14.912 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:19:14.912 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 17:19:14.912 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:19:14.912 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:19:14.912 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:19:14.912 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 17:19:14.912 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 17:19:14.912 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:19:14.912 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 17:19:14.912 | INFO | ------------------------------------------ | 20260505 17:19:14.912 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:19:14.912 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 17:19:14.912 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:19:14.912 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:19:14.912 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.48e+01, 1.34e+01 / 2.13e+00 | 20260505 17:19:14.912 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 1.03e-01, 1.08e-02 / 3.95e-03 | 20260505 17:19:14.912 | INFO | PerfReport: 11.81%, Baseline -- 3.74e+00 / 4.51e+00, 5.82e-01 / 1.64e-01 | 20260505 17:19:14.912 | INFO | PerfReport: 3.29%, Histogram -- 1.04e+00 / 1.52e+00, 1.96e-01 / 1.02e-01 | 20260505 17:19:14.912 | INFO | PerfReport: 15.87%, DME -- 4.02e+01 / 4.42e+01, 4.08e+00 / 2.24e+00 | 20260505 17:19:14.912 | INFO | PerfReport: 56.89%, FrameLabeling -- 1.80e+01 / 1.99e+01, 1.53e+00 / 1.66e+00 | 20260505 17:19:14.912 | INFO | PerfReport: 7.29%, PulseAccumulating -- 2.31e+00 / 2.87e+00, 4.39e-01 / 1.90e-01 | 20260505 17:19:14.912 | INFO | PerfReport: 3.38%, Metrics -- 1.07e+00 / 1.40e+00, 3.01e-01 / 1.78e-01 | 20260505 17:19:14.912 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.71e-01 / 4.24e-01, 6.58e-02 / 3.35e-02 | 20260505 17:19:14.912 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 17:19:14.912 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:19:14.912 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:19:14.912 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.56e+01 / 0.00e+00 | 20260505 17:19:14.912 | INFO | PerfReport: 22.51%, Upload -- 2.25e+01 / 0.00e+00, 7.89e+00 / 0.00e+00 | 20260505 17:19:14.912 | INFO | PerfReport: 21.80%, Download -- 2.18e+01 / 0.00e+00, 7.88e+00 / 0.00e+00 | 20260505 17:19:14.912 | INFO | PerfReport: 31.67%, Compute -- 3.17e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 17:19:14.912 | INFO | PerfReport: 2.16%, LockUpload -- 2.16e+00 / 0.00e+00, 7.75e+00 / 0.00e+00 | 20260505 17:19:14.912 | INFO | PerfReport: 1.41%, LockDownload -- 1.41e+00 / 0.00e+00, 2.95e+00 / 0.00e+00 | 20260505 17:19:14.912 | INFO | PerfReport: 20.43%, LockCompute -- 2.04e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 17:19:14.912 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.93e-02 / 0.00e+00 | 20260505 17:19:15.040 | INFO | Starting a round of Baz IO | 20260505 17:19:18.238 | INFO | Populating RTMetrics report | 20260505 17:19:18.238 | INFO | RTMetrics report populated | 20260505 17:19:18.238 | INFO | Opening tmp RTMetrics json file | 20260505 17:19:18.239 | INFO | Serializing RTMetrics to json | 20260505 17:19:18.240 | INFO | Serialization done, swapping tmp file now... | 20260505 17:19:18.241 | INFO | RTMetrics done | 20260505 17:19:18.331 | INFO | Finished a round of Baz IO: Wrote 2.136 GiB in 3.29 seconds | 20260505 17:19:18.331 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.08% / 0.00% / 77.92% | 20260505 17:19:18.331 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.30694% / 0% / 7.54992e-05% | 20260505 17:19:18.331 | INFO | Baz IO overall: | 20260505 17:19:18.331 | INFO | BytesWritten : 2926985513891 | 20260505 17:19:18.331 | INFO | NumEvents : 1388151803303 | 20260505 17:19:20.048 | INFO | Analyzing chunk frames = [4947968,4948480) | 20260505 17:19:20.048 | INFO | MemInfo: { | 20260505 17:19:20.048 | INFO | "gpu" : " 17 GB", | 20260505 17:19:20.048 | INFO | "huge" : "", | 20260505 17:19:20.048 | INFO | "huge_shared" : " 27 GB", | 20260505 17:19:20.048 | INFO | "malloc" : " 14 GB", | 20260505 17:19:20.048 | INFO | "pinned" : " 29 GB" | 20260505 17:19:20.048 | INFO | } | 20260505 17:19:20.048 | INFO | Perf report for the last 5.13769 seconds: | 20260505 17:19:20.048 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:19:20.048 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:20.048 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:20.048 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:20.048 | INFO | ANALYSIS : 63.7%, 3.0, 99.2ms, 0 | 20260505 17:19:20.048 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:20.048 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260505 17:19:20.048 | INFO | PRE_HQ : 60.9%, 1.8, 56.8ms, 0 | 20260505 17:19:20.048 | INFO | BAZWRITER : 63.4%, 3.9, 126.1ms, 0 | 20260505 17:19:20.048 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 17:19:20.049 | INFO | Uploaded 36GiB to GPU in 2.15619s (16.6961GiB/s) | 20260505 17:19:20.049 | INFO | Downloaded 38GiB to GPU in 2.25099s (16.8815GiB/s) | 20260505 17:19:25.193 | INFO | Analyzing chunk frames = [4948480,4948992) | 20260505 17:19:25.193 | INFO | MemInfo: { | 20260505 17:19:25.193 | INFO | "gpu" : " 17 GB", | 20260505 17:19:25.193 | INFO | "huge" : "", | 20260505 17:19:25.193 | INFO | "huge_shared" : " 27 GB", | 20260505 17:19:25.193 | INFO | "malloc" : " 14 GB", | 20260505 17:19:25.193 | INFO | "pinned" : " 29 GB" | 20260505 17:19:25.193 | INFO | } | 20260505 17:19:25.193 | INFO | Perf report for the last 5.14424 seconds: | 20260505 17:19:25.193 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:19:25.193 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:25.193 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:25.193 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:25.193 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 17:19:25.193 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:25.193 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:25.193 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 17:19:25.193 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:25.193 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:25.193 | INFO | Uploaded 36GiB to GPU in 2.20595s (16.3195GiB/s) | 20260505 17:19:25.193 | INFO | Downloaded 34GiB to GPU in 2.07069s (16.4197GiB/s) | 20260505 17:19:30.343 | INFO | Analyzing chunk frames = [4948992,4949504) | 20260505 17:19:30.343 | INFO | MemInfo: { | 20260505 17:19:30.343 | INFO | "gpu" : " 17 GB", | 20260505 17:19:30.343 | INFO | "huge" : "", | 20260505 17:19:30.343 | INFO | "huge_shared" : " 27 GB", | 20260505 17:19:30.343 | INFO | "malloc" : " 14 GB", | 20260505 17:19:30.343 | INFO | "pinned" : " 29 GB" | 20260505 17:19:30.343 | INFO | } | 20260505 17:19:30.343 | INFO | Perf report for the last 5.15015 seconds: | 20260505 17:19:30.343 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:19:30.343 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:30.343 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:30.343 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:30.343 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 17:19:30.343 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:30.343 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:30.343 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 17:19:30.343 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:30.343 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:30.343 | INFO | Uploaded 36GiB to GPU in 2.202s (16.3488GiB/s) | 20260505 17:19:30.343 | INFO | Downloaded 34GiB to GPU in 2.04644s (16.6142GiB/s) | 20260505 17:19:35.475 | INFO | Analyzing chunk frames = [4949504,4950016) | 20260505 17:19:35.475 | INFO | MemInfo: { | 20260505 17:19:35.475 | INFO | "gpu" : " 17 GB", | 20260505 17:19:35.475 | INFO | "huge" : "", | 20260505 17:19:35.475 | INFO | "huge_shared" : " 27 GB", | 20260505 17:19:35.475 | INFO | "malloc" : " 14 GB", | 20260505 17:19:35.475 | INFO | "pinned" : " 29 GB" | 20260505 17:19:35.475 | INFO | } | 20260505 17:19:35.476 | INFO | Perf report for the last 5.13267 seconds: | 20260505 17:19:35.476 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:19:35.476 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:35.476 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:35.476 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:35.476 | INFO | ANALYSIS : 62.9%, 3.0, 97.6ms, 0 | 20260505 17:19:35.476 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:35.476 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:35.476 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 17:19:35.476 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:35.476 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:35.476 | INFO | Uploaded 36GiB to GPU in 2.22749s (16.1617GiB/s) | 20260505 17:19:35.476 | INFO | Downloaded 34GiB to GPU in 2.12128s (16.0281GiB/s) | 20260505 17:19:40.604 | INFO | Analyzing chunk frames = [4950016,4950528) | 20260505 17:19:40.604 | INFO | MemInfo: { | 20260505 17:19:40.604 | INFO | "gpu" : " 17 GB", | 20260505 17:19:40.604 | INFO | "huge" : "", | 20260505 17:19:40.605 | INFO | "huge_shared" : " 27 GB", | 20260505 17:19:40.605 | INFO | "malloc" : " 14 GB", | 20260505 17:19:40.605 | INFO | "pinned" : " 29 GB" | 20260505 17:19:40.605 | INFO | } | 20260505 17:19:40.605 | INFO | Perf report for the last 5.12907 seconds: | 20260505 17:19:40.605 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:19:40.605 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:40.605 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:40.605 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:40.605 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260505 17:19:40.605 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:40.605 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:40.605 | INFO | PRE_HQ : 27.7%, 1.0, 14.3ms, 0 | 20260505 17:19:40.605 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:40.605 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:40.605 | INFO | Uploaded 36GiB to GPU in 2.16199s (16.6513GiB/s) | 20260505 17:19:40.605 | INFO | Downloaded 34GiB to GPU in 2.06225s (16.4868GiB/s) | 20260505 17:19:45.749 | INFO | Analyzing chunk frames = [4950528,4951040) | 20260505 17:19:45.749 | INFO | MemInfo: { | 20260505 17:19:45.749 | INFO | "gpu" : " 17 GB", | 20260505 17:19:45.749 | INFO | "huge" : "", | 20260505 17:19:45.749 | INFO | "huge_shared" : " 27 GB", | 20260505 17:19:45.749 | INFO | "malloc" : " 14 GB", | 20260505 17:19:45.749 | INFO | "pinned" : " 29 GB" | 20260505 17:19:45.749 | INFO | } | 20260505 17:19:45.749 | INFO | Perf report for the last 5.1444 seconds: | 20260505 17:19:45.749 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:19:45.749 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:45.749 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:45.749 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:45.749 | INFO | ANALYSIS : 62.4%, 3.0, 97.6ms, 0 | 20260505 17:19:45.749 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:45.749 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:45.749 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:19:45.749 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:45.749 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:45.749 | INFO | Uploaded 36GiB to GPU in 2.14044s (16.819GiB/s) | 20260505 17:19:45.749 | INFO | Downloaded 34GiB to GPU in 2.0374s (16.6879GiB/s) | 20260505 17:19:50.893 | INFO | Analyzing chunk frames = [4951040,4951552) | 20260505 17:19:50.893 | INFO | MemInfo: { | 20260505 17:19:50.893 | INFO | "gpu" : " 17 GB", | 20260505 17:19:50.893 | INFO | "huge" : "", | 20260505 17:19:50.893 | INFO | "huge_shared" : " 27 GB", | 20260505 17:19:50.893 | INFO | "malloc" : " 14 GB", | 20260505 17:19:50.893 | INFO | "pinned" : " 29 GB" | 20260505 17:19:50.893 | INFO | } | 20260505 17:19:50.893 | INFO | Perf report for the last 5.14365 seconds: | 20260505 17:19:50.893 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:19:50.893 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:50.893 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:50.893 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:50.893 | INFO | ANALYSIS : 63.3%, 3.0, 99.0ms, 0 | 20260505 17:19:50.893 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:50.893 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:50.893 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 17:19:50.893 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:50.893 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:50.893 | INFO | Uploaded 36GiB to GPU in 2.15622s (16.6959GiB/s) | 20260505 17:19:50.893 | INFO | Downloaded 34GiB to GPU in 2.17485s (15.6333GiB/s) | 20260505 17:19:56.032 | INFO | Analyzing chunk frames = [4951552,4952064) | 20260505 17:19:56.032 | INFO | MemInfo: { | 20260505 17:19:56.032 | INFO | "gpu" : " 17 GB", | 20260505 17:19:56.032 | INFO | "huge" : "", | 20260505 17:19:56.032 | INFO | "huge_shared" : " 27 GB", | 20260505 17:19:56.032 | INFO | "malloc" : " 14 GB", | 20260505 17:19:56.032 | INFO | "pinned" : " 29 GB" | 20260505 17:19:56.032 | INFO | } | 20260505 17:19:56.032 | INFO | Perf report for the last 5.13949 seconds: | 20260505 17:19:56.032 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:19:56.032 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:56.032 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:56.032 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:56.032 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 17:19:56.032 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:56.032 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:56.032 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 17:19:56.032 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:56.032 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:19:56.032 | INFO | Uploaded 36GiB to GPU in 2.18768s (16.4558GiB/s) | 20260505 17:19:56.032 | INFO | Downloaded 34GiB to GPU in 2.02451s (16.7942GiB/s) | 20260505 17:19:56.171 | INFO | Starting a round of Baz IO | 20260505 17:19:59.351 | INFO | Populating RTMetrics report | 20260505 17:19:59.351 | INFO | RTMetrics report populated | 20260505 17:19:59.351 | INFO | Opening tmp RTMetrics json file | 20260505 17:19:59.356 | INFO | Serializing RTMetrics to json | 20260505 17:19:59.357 | INFO | Serialization done, swapping tmp file now... | 20260505 17:19:59.357 | INFO | RTMetrics done | 20260505 17:19:59.451 | INFO | Finished a round of Baz IO: Wrote 2.135 GiB in 3.28 seconds | 20260505 17:19:59.451 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.08% / 0.00% / 77.92% | 20260505 17:19:59.451 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.306424% / 1.19209e-05% / 0.000107288% | 20260505 17:19:59.451 | INFO | Baz IO overall: | 20260505 17:19:59.451 | INFO | BytesWritten : 2929278146118 | 20260505 17:19:59.451 | INFO | NumEvents : 1389223488182 | 20260505 17:20:01.170 | INFO | Analyzing chunk frames = [4952064,4952576) | 20260505 17:20:01.170 | INFO | MemInfo: { | 20260505 17:20:01.170 | INFO | "gpu" : " 17 GB", | 20260505 17:20:01.170 | INFO | "huge" : "", | 20260505 17:20:01.170 | INFO | "huge_shared" : " 27 GB", | 20260505 17:20:01.170 | INFO | "malloc" : " 14 GB", | 20260505 17:20:01.170 | INFO | "pinned" : " 29 GB" | 20260505 17:20:01.170 | INFO | } | 20260505 17:20:01.170 | INFO | Perf report for the last 5.13754 seconds: | 20260505 17:20:01.170 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:20:01.170 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:01.170 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:01.170 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:01.170 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 17:20:01.170 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:01.170 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260505 17:20:01.170 | INFO | PRE_HQ : 60.4%, 1.8, 56.4ms, 0 | 20260505 17:20:01.170 | INFO | BAZWRITER : 63.3%, 3.8, 125.2ms, 0 | 20260505 17:20:01.170 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 17:20:01.170 | INFO | Uploaded 36GiB to GPU in 2.16768s (16.6076GiB/s) | 20260505 17:20:01.170 | INFO | Downloaded 38GiB to GPU in 2.25404s (16.8586GiB/s) | 20260505 17:20:06.311 | INFO | Analyzing chunk frames = [4952576,4953088) | 20260505 17:20:06.311 | INFO | MemInfo: { | 20260505 17:20:06.311 | INFO | "gpu" : " 17 GB", | 20260505 17:20:06.311 | INFO | "huge" : "", | 20260505 17:20:06.311 | INFO | "huge_shared" : " 27 GB", | 20260505 17:20:06.311 | INFO | "malloc" : " 14 GB", | 20260505 17:20:06.311 | INFO | "pinned" : " 29 GB" | 20260505 17:20:06.311 | INFO | } | 20260505 17:20:06.311 | INFO | Perf report for the last 5.14132 seconds: | 20260505 17:20:06.311 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:20:06.311 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:06.311 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:06.311 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:06.311 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 17:20:06.311 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:06.311 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:06.311 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:20:06.311 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:06.311 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:06.312 | INFO | Uploaded 36GiB to GPU in 2.13999s (16.8225GiB/s) | 20260505 17:20:06.312 | INFO | Downloaded 34GiB to GPU in 2.04105s (16.6581GiB/s) | 20260505 17:20:11.457 | INFO | Analyzing chunk frames = [4953088,4953600) | 20260505 17:20:11.457 | INFO | MemInfo: { | 20260505 17:20:11.457 | INFO | "gpu" : " 17 GB", | 20260505 17:20:11.457 | INFO | "huge" : "", | 20260505 17:20:11.457 | INFO | "huge_shared" : " 27 GB", | 20260505 17:20:11.457 | INFO | "malloc" : " 14 GB", | 20260505 17:20:11.457 | INFO | "pinned" : " 29 GB" | 20260505 17:20:11.457 | INFO | } | 20260505 17:20:11.457 | INFO | Perf report for the last 5.1461 seconds: | 20260505 17:20:11.457 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:20:11.457 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:11.457 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:11.457 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:11.457 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 17:20:11.457 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:11.457 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:11.457 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:20:11.457 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:11.457 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:11.457 | INFO | Uploaded 36GiB to GPU in 2.20094s (16.3566GiB/s) | 20260505 17:20:11.458 | INFO | Downloaded 34GiB to GPU in 2.03335s (16.7211GiB/s) | 20260505 17:20:16.595 | INFO | Analyzing chunk frames = [4953600,4954112) | 20260505 17:20:16.595 | INFO | MemInfo: { | 20260505 17:20:16.595 | INFO | "gpu" : " 17 GB", | 20260505 17:20:16.595 | INFO | "huge" : "", | 20260505 17:20:16.595 | INFO | "huge_shared" : " 27 GB", | 20260505 17:20:16.595 | INFO | "malloc" : " 14 GB", | 20260505 17:20:16.595 | INFO | "pinned" : " 29 GB" | 20260505 17:20:16.595 | INFO | } | 20260505 17:20:16.595 | INFO | Perf report for the last 5.13809 seconds: | 20260505 17:20:16.595 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:20:16.595 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:16.595 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:16.595 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:16.595 | INFO | ANALYSIS : 62.5%, 3.0, 97.1ms, 0 | 20260505 17:20:16.595 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:16.595 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:16.595 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 17:20:16.595 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:16.595 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:16.596 | INFO | Uploaded 36GiB to GPU in 2.17605s (16.5438GiB/s) | 20260505 17:20:16.596 | INFO | Downloaded 34GiB to GPU in 2.06101s (16.4968GiB/s) | 20260505 17:20:21.725 | INFO | Analyzing chunk frames = [4954112,4954624) | 20260505 17:20:21.725 | INFO | MemInfo: { | 20260505 17:20:21.725 | INFO | "gpu" : " 17 GB", | 20260505 17:20:21.725 | INFO | "huge" : "", | 20260505 17:20:21.725 | INFO | "huge_shared" : " 27 GB", | 20260505 17:20:21.725 | INFO | "malloc" : " 14 GB", | 20260505 17:20:21.725 | INFO | "pinned" : " 29 GB" | 20260505 17:20:21.725 | INFO | } | 20260505 17:20:21.726 | INFO | Perf report for the last 5.13024 seconds: | 20260505 17:20:21.726 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:20:21.726 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:21.726 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:21.726 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:21.726 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260505 17:20:21.726 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:21.726 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:21.726 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 17:20:21.726 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:21.726 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:21.726 | INFO | Uploaded 36GiB to GPU in 2.15049s (16.7404GiB/s) | 20260505 17:20:21.726 | INFO | Downloaded 34GiB to GPU in 2.05453s (16.5488GiB/s) | 20260505 17:20:26.859 | INFO | Analyzing chunk frames = [4954624,4955136) | 20260505 17:20:26.859 | INFO | MemInfo: { | 20260505 17:20:26.859 | INFO | "gpu" : " 17 GB", | 20260505 17:20:26.859 | INFO | "huge" : "", | 20260505 17:20:26.859 | INFO | "huge_shared" : " 27 GB", | 20260505 17:20:26.859 | INFO | "malloc" : " 14 GB", | 20260505 17:20:26.859 | INFO | "pinned" : " 29 GB" | 20260505 17:20:26.859 | INFO | } | 20260505 17:20:26.859 | INFO | Perf report for the last 5.13374 seconds: | 20260505 17:20:26.859 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:20:26.859 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:26.859 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:26.859 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:26.859 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260505 17:20:26.859 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:26.859 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:26.859 | INFO | PRE_HQ : 27.5%, 1.0, 14.2ms, 0 | 20260505 17:20:26.859 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:26.859 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:26.859 | INFO | Uploaded 36GiB to GPU in 2.1449s (16.784GiB/s) | 20260505 17:20:26.859 | INFO | Downloaded 34GiB to GPU in 2.03804s (16.6827GiB/s) | 20260505 17:20:32.005 | INFO | Analyzing chunk frames = [4955136,4955648) | 20260505 17:20:32.005 | INFO | MemInfo: { | 20260505 17:20:32.005 | INFO | "gpu" : " 17 GB", | 20260505 17:20:32.005 | INFO | "huge" : "", | 20260505 17:20:32.006 | INFO | "huge_shared" : " 27 GB", | 20260505 17:20:32.006 | INFO | "malloc" : " 14 GB", | 20260505 17:20:32.006 | INFO | "pinned" : " 29 GB" | 20260505 17:20:32.006 | INFO | } | 20260505 17:20:32.006 | INFO | Perf report for the last 5.14629 seconds: | 20260505 17:20:32.006 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:20:32.006 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:32.006 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:32.006 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:32.006 | INFO | ANALYSIS : 63.4%, 3.0, 99.2ms, 0 | 20260505 17:20:32.006 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:32.006 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:32.006 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 17:20:32.006 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:32.006 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:32.006 | INFO | Uploaded 36GiB to GPU in 2.11863s (16.9921GiB/s) | 20260505 17:20:32.006 | INFO | Downloaded 34GiB to GPU in 2.17598s (15.6251GiB/s) | 20260505 17:20:37.149 | INFO | Analyzing chunk frames = [4955648,4956160) | 20260505 17:20:37.149 | INFO | MemInfo: { | 20260505 17:20:37.149 | INFO | "gpu" : " 17 GB", | 20260505 17:20:37.149 | INFO | "huge" : "", | 20260505 17:20:37.149 | INFO | "huge_shared" : " 27 GB", | 20260505 17:20:37.149 | INFO | "malloc" : " 14 GB", | 20260505 17:20:37.149 | INFO | "pinned" : " 29 GB" | 20260505 17:20:37.149 | INFO | } | 20260505 17:20:37.149 | INFO | Perf report for the last 5.14338 seconds: | 20260505 17:20:37.149 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:20:37.149 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:37.149 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:37.149 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:37.149 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 17:20:37.149 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:37.149 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:37.149 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:20:37.149 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:37.149 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:37.149 | INFO | Uploaded 36GiB to GPU in 2.20223s (16.3471GiB/s) | 20260505 17:20:37.149 | INFO | Downloaded 34GiB to GPU in 2.02451s (16.7942GiB/s) | 20260505 17:20:37.149 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 17:20:37.149 | INFO | Total mem usage: 15.3 GiB | 20260505 17:20:37.149 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 17:20:37.149 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -7.29%) | 20260505 17:20:37.149 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 17:20:37.149 | INFO | High Water Breakdown: | 20260505 17:20:37.149 | INFO | 61.1% : 8.73 GiB ( 91.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 17:20:37.149 | INFO | 38.8% : 5.54 GiB ( 83.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 17:20:37.149 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 17:20:37.149 | INFO | ------------------------------------------ | 20260505 17:20:37.150 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:20:37.150 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 17:20:37.150 | INFO | Total mem usage: 26.5 GiB | 20260505 17:20:37.150 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 17:20:37.150 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 17:20:37.150 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 17:20:37.150 | INFO | High Water Breakdown: | 20260505 17:20:37.150 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:20:37.150 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:20:37.150 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:20:37.150 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:20:37.150 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:20:37.150 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:20:37.150 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 17:20:37.150 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 17:20:37.150 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:20:37.150 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 17:20:37.150 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 17:20:37.150 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 17:20:37.150 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 17:20:37.150 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 17:20:37.150 | INFO | ------------------------------------------ | 20260505 17:20:37.150 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:20:37.150 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 17:20:37.150 | INFO | Total mem usage: 25.6 GiB | 20260505 17:20:37.150 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 17:20:37.150 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 17:20:37.150 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 17:20:37.150 | INFO | High Water Breakdown: | 20260505 17:20:37.150 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 17:20:37.150 | INFO | ------------------------------------------ | 20260505 17:20:37.150 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:20:37.150 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 17:20:37.150 | INFO | Total mem usage: 15.7 GiB | 20260505 17:20:37.150 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 17:20:37.150 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 17:20:37.150 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 17:20:37.150 | INFO | High Water Breakdown: | 20260505 17:20:37.150 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 17:20:37.150 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 17:20:37.150 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 17:20:37.150 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 17:20:37.150 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 17:20:37.150 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 17:20:37.150 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 17:20:37.150 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 17:20:37.150 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 17:20:37.150 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:20:37.150 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 17:20:37.150 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:20:37.150 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 17:20:37.150 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 17:20:37.150 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 17:20:37.150 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 17:20:37.150 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:20:37.150 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 17:20:37.150 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:20:37.150 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:20:37.150 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:20:37.150 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 17:20:37.150 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 17:20:37.150 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:20:37.150 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 17:20:37.150 | INFO | ------------------------------------------ | 20260505 17:20:37.150 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:20:37.151 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 17:20:37.151 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:20:37.151 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:20:37.151 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.48e+01, 1.35e+01 / 1.95e+00 | 20260505 17:20:37.151 | INFO | PerfReport: 0.29%, AggregateStats -- 9.25e-02 / 1.03e-01, 1.12e-02 / 4.60e-03 | 20260505 17:20:37.151 | INFO | PerfReport: 11.77%, Baseline -- 3.72e+00 / 4.50e+00, 5.70e-01 / 1.55e-01 | 20260505 17:20:37.151 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.52e+00, 1.90e-01 / 7.01e-02 | 20260505 17:20:37.151 | INFO | PerfReport: 15.95%, DME -- 4.03e+01 / 4.47e+01, 4.19e+00 / 2.15e+00 | 20260505 17:20:37.151 | INFO | PerfReport: 56.83%, FrameLabeling -- 1.80e+01 / 1.92e+01, 1.48e+00 / 1.04e+00 | 20260505 17:20:37.151 | INFO | PerfReport: 7.35%, PulseAccumulating -- 2.32e+00 / 2.92e+00, 4.48e-01 / 1.63e-01 | 20260505 17:20:37.151 | INFO | PerfReport: 3.37%, Metrics -- 1.07e+00 / 1.40e+00, 2.96e-01 / 1.51e-01 | 20260505 17:20:37.151 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.71e-01 / 4.33e-01, 6.57e-02 / 3.93e-02 | 20260505 17:20:37.151 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 17:20:37.151 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:20:37.151 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:20:37.151 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.58e+01 / 0.00e+00 | 20260505 17:20:37.151 | INFO | PerfReport: 22.59%, Upload -- 2.26e+01 / 0.00e+00, 7.93e+00 / 0.00e+00 | 20260505 17:20:37.151 | INFO | PerfReport: 21.76%, Download -- 2.18e+01 / 0.00e+00, 7.82e+00 / 0.00e+00 | 20260505 17:20:37.151 | INFO | PerfReport: 31.58%, Compute -- 3.16e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 17:20:37.151 | INFO | PerfReport: 2.38%, LockUpload -- 2.38e+00 / 0.00e+00, 8.04e+00 / 0.00e+00 | 20260505 17:20:37.151 | INFO | PerfReport: 1.40%, LockDownload -- 1.41e+00 / 0.00e+00, 2.99e+00 / 0.00e+00 | 20260505 17:20:37.151 | INFO | PerfReport: 20.28%, LockCompute -- 2.03e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260505 17:20:37.151 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 2.16e-02 / 0.00e+00 | 20260505 17:20:37.285 | INFO | Starting a round of Baz IO | 20260505 17:20:40.474 | INFO | Populating RTMetrics report | 20260505 17:20:40.474 | INFO | RTMetrics report populated | 20260505 17:20:40.474 | INFO | Opening tmp RTMetrics json file | 20260505 17:20:40.478 | INFO | Serializing RTMetrics to json | 20260505 17:20:40.479 | INFO | Serialization done, swapping tmp file now... | 20260505 17:20:40.480 | INFO | RTMetrics done | 20260505 17:20:40.576 | INFO | Finished a round of Baz IO: Wrote 2.135 GiB in 3.29 seconds | 20260505 17:20:40.576 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.08% / 0.00% / 77.92% | 20260505 17:20:40.576 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.305418% / 3.97364e-06% / 0.000103315% | 20260505 17:20:40.576 | INFO | Baz IO overall: | 20260505 17:20:40.576 | INFO | BytesWritten : 2931570864407 | 20260505 17:20:40.576 | INFO | NumEvents : 1390295376677 | 20260505 17:20:42.293 | INFO | Analyzing chunk frames = [4956160,4956672) | 20260505 17:20:42.293 | INFO | MemInfo: { | 20260505 17:20:42.293 | INFO | "gpu" : " 17 GB", | 20260505 17:20:42.293 | INFO | "huge" : "", | 20260505 17:20:42.293 | INFO | "huge_shared" : " 27 GB", | 20260505 17:20:42.293 | INFO | "malloc" : " 14 GB", | 20260505 17:20:42.294 | INFO | "pinned" : " 29 GB" | 20260505 17:20:42.294 | INFO | } | 20260505 17:20:42.294 | INFO | Perf report for the last 5.14456 seconds: | 20260505 17:20:42.294 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:20:42.294 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:42.294 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:42.294 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:42.294 | INFO | ANALYSIS : 63.6%, 3.0, 99.1ms, 0 | 20260505 17:20:42.294 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:42.294 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260505 17:20:42.294 | INFO | PRE_HQ : 60.5%, 1.8, 55.8ms, 0 | 20260505 17:20:42.294 | INFO | BAZWRITER : 63.5%, 3.8, 124.6ms, 0 | 20260505 17:20:42.294 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 17:20:42.294 | INFO | Uploaded 36GiB to GPU in 2.19473s (16.403GiB/s) | 20260505 17:20:42.294 | INFO | Downloaded 38GiB to GPU in 2.25983s (16.8154GiB/s) | 20260505 17:20:47.422 | INFO | Analyzing chunk frames = [4956672,4957184) | 20260505 17:20:47.422 | INFO | MemInfo: { | 20260505 17:20:47.422 | INFO | "gpu" : " 17 GB", | 20260505 17:20:47.422 | INFO | "huge" : "", | 20260505 17:20:47.422 | INFO | "huge_shared" : " 27 GB", | 20260505 17:20:47.422 | INFO | "malloc" : " 14 GB", | 20260505 17:20:47.422 | INFO | "pinned" : " 29 GB" | 20260505 17:20:47.422 | INFO | } | 20260505 17:20:47.422 | INFO | Perf report for the last 5.12886 seconds: | 20260505 17:20:47.422 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:20:47.422 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:47.422 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:47.422 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:47.422 | INFO | ANALYSIS : 62.9%, 3.0, 98.0ms, 0 | 20260505 17:20:47.423 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:47.423 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:47.423 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:20:47.423 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:47.423 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:47.423 | INFO | Uploaded 36GiB to GPU in 2.14345s (16.7954GiB/s) | 20260505 17:20:47.423 | INFO | Downloaded 34GiB to GPU in 2.03523s (16.7057GiB/s) | 20260505 17:20:52.557 | INFO | Analyzing chunk frames = [4957184,4957696) | 20260505 17:20:52.557 | INFO | MemInfo: { | 20260505 17:20:52.557 | INFO | "gpu" : " 17 GB", | 20260505 17:20:52.557 | INFO | "huge" : "", | 20260505 17:20:52.557 | INFO | "huge_shared" : " 27 GB", | 20260505 17:20:52.557 | INFO | "malloc" : " 14 GB", | 20260505 17:20:52.557 | INFO | "pinned" : " 29 GB" | 20260505 17:20:52.557 | INFO | } | 20260505 17:20:52.557 | INFO | Perf report for the last 5.13495 seconds: | 20260505 17:20:52.557 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:20:52.557 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:52.557 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:52.557 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:52.557 | INFO | ANALYSIS : 62.9%, 3.0, 97.7ms, 0 | 20260505 17:20:52.557 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:52.558 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:52.558 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:20:52.558 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:52.558 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:52.558 | INFO | Uploaded 36GiB to GPU in 2.18612s (16.4675GiB/s) | 20260505 17:20:52.558 | INFO | Downloaded 34GiB to GPU in 2.03361s (16.7191GiB/s) | 20260505 17:20:57.708 | INFO | Analyzing chunk frames = [4957696,4958208) | 20260505 17:20:57.708 | INFO | MemInfo: { | 20260505 17:20:57.708 | INFO | "gpu" : " 17 GB", | 20260505 17:20:57.708 | INFO | "huge" : "", | 20260505 17:20:57.708 | INFO | "huge_shared" : " 27 GB", | 20260505 17:20:57.708 | INFO | "malloc" : " 14 GB", | 20260505 17:20:57.708 | INFO | "pinned" : " 29 GB" | 20260505 17:20:57.708 | INFO | } | 20260505 17:20:57.708 | INFO | Perf report for the last 5.15058 seconds: | 20260505 17:20:57.708 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:20:57.708 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:57.708 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:57.708 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:57.708 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260505 17:20:57.708 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:57.708 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:57.708 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 17:20:57.708 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:57.708 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:20:57.708 | INFO | Uploaded 36GiB to GPU in 2.17291s (16.5677GiB/s) | 20260505 17:20:57.708 | INFO | Downloaded 34GiB to GPU in 2.14481s (15.8522GiB/s) | 20260505 17:21:02.843 | INFO | Analyzing chunk frames = [4958208,4958720) | 20260505 17:21:02.843 | INFO | MemInfo: { | 20260505 17:21:02.843 | INFO | "gpu" : " 17 GB", | 20260505 17:21:02.843 | INFO | "huge" : "", | 20260505 17:21:02.843 | INFO | "huge_shared" : " 27 GB", | 20260505 17:21:02.843 | INFO | "malloc" : " 14 GB", | 20260505 17:21:02.843 | INFO | "pinned" : " 29 GB" | 20260505 17:21:02.843 | INFO | } | 20260505 17:21:02.843 | INFO | Perf report for the last 5.13488 seconds: | 20260505 17:21:02.843 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:21:02.843 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:02.843 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:02.843 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:02.843 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 17:21:02.843 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:02.843 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:02.843 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:21:02.843 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:02.843 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:02.843 | INFO | Uploaded 36GiB to GPU in 2.13854s (16.8339GiB/s) | 20260505 17:21:02.843 | INFO | Downloaded 34GiB to GPU in 2.0582s (16.5193GiB/s) | 20260505 17:21:07.980 | INFO | Analyzing chunk frames = [4958720,4959232) | 20260505 17:21:07.980 | INFO | MemInfo: { | 20260505 17:21:07.980 | INFO | "gpu" : " 17 GB", | 20260505 17:21:07.980 | INFO | "huge" : "", | 20260505 17:21:07.980 | INFO | "huge_shared" : " 27 GB", | 20260505 17:21:07.980 | INFO | "malloc" : " 14 GB", | 20260505 17:21:07.980 | INFO | "pinned" : " 29 GB" | 20260505 17:21:07.980 | INFO | } | 20260505 17:21:07.980 | INFO | Perf report for the last 5.13723 seconds: | 20260505 17:21:07.980 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:21:07.980 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:07.980 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:07.980 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:07.980 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260505 17:21:07.980 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:07.980 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:07.980 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 17:21:07.980 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:07.980 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:07.980 | INFO | Uploaded 36GiB to GPU in 2.1408s (16.8162GiB/s) | 20260505 17:21:07.980 | INFO | Downloaded 34GiB to GPU in 2.04615s (16.6166GiB/s) | 20260505 17:21:13.117 | INFO | Analyzing chunk frames = [4959232,4959744) | 20260505 17:21:13.117 | INFO | MemInfo: { | 20260505 17:21:13.117 | INFO | "gpu" : " 17 GB", | 20260505 17:21:13.117 | INFO | "huge" : "", | 20260505 17:21:13.117 | INFO | "huge_shared" : " 27 GB", | 20260505 17:21:13.117 | INFO | "malloc" : " 14 GB", | 20260505 17:21:13.117 | INFO | "pinned" : " 29 GB" | 20260505 17:21:13.117 | INFO | } | 20260505 17:21:13.117 | INFO | Perf report for the last 5.13732 seconds: | 20260505 17:21:13.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:21:13.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:13.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:13.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:13.117 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 17:21:13.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:13.117 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:13.117 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 17:21:13.118 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:13.118 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:13.118 | INFO | Uploaded 36GiB to GPU in 2.17236s (16.5719GiB/s) | 20260505 17:21:13.118 | INFO | Downloaded 34GiB to GPU in 2.17983s (15.5976GiB/s) | 20260505 17:21:18.262 | INFO | Analyzing chunk frames = [4959744,4960256) | 20260505 17:21:18.262 | INFO | MemInfo: { | 20260505 17:21:18.262 | INFO | "gpu" : " 17 GB", | 20260505 17:21:18.262 | INFO | "huge" : "", | 20260505 17:21:18.262 | INFO | "huge_shared" : " 27 GB", | 20260505 17:21:18.262 | INFO | "malloc" : " 14 GB", | 20260505 17:21:18.262 | INFO | "pinned" : " 29 GB" | 20260505 17:21:18.262 | INFO | } | 20260505 17:21:18.262 | INFO | Perf report for the last 5.14483 seconds: | 20260505 17:21:18.262 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:21:18.262 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:18.262 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:18.262 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:18.262 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 17:21:18.262 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:18.262 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:18.262 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260505 17:21:18.262 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:18.262 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:18.263 | INFO | Uploaded 36GiB to GPU in 2.1742s (16.5578GiB/s) | 20260505 17:21:18.263 | INFO | Downloaded 34GiB to GPU in 2.03671s (16.6936GiB/s) | 20260505 17:21:18.396 | INFO | Starting a round of Baz IO | 20260505 17:21:21.595 | INFO | Populating RTMetrics report | 20260505 17:21:21.596 | INFO | RTMetrics report populated | 20260505 17:21:21.596 | INFO | Opening tmp RTMetrics json file | 20260505 17:21:21.601 | INFO | Serializing RTMetrics to json | 20260505 17:21:21.604 | INFO | Serialization done, swapping tmp file now... | 20260505 17:21:21.606 | INFO | RTMetrics done | 20260505 17:21:21.690 | INFO | Finished a round of Baz IO: Wrote 2.132 GiB in 3.293 seconds | 20260505 17:21:21.690 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.08% / 0.00% / 77.92% | 20260505 17:21:21.690 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.304333% / 3.97364e-06% / 8.74201e-05% | 20260505 17:21:21.690 | INFO | Baz IO overall: | 20260505 17:21:21.690 | INFO | BytesWritten : 2933860561819 | 20260505 17:21:21.690 | INFO | NumEvents : 1391364880744 | 20260505 17:21:23.401 | INFO | Analyzing chunk frames = [4960256,4960768) | 20260505 17:21:23.401 | INFO | MemInfo: { | 20260505 17:21:23.401 | INFO | "gpu" : " 17 GB", | 20260505 17:21:23.401 | INFO | "huge" : "", | 20260505 17:21:23.401 | INFO | "huge_shared" : " 27 GB", | 20260505 17:21:23.401 | INFO | "malloc" : " 14 GB", | 20260505 17:21:23.401 | INFO | "pinned" : " 29 GB" | 20260505 17:21:23.401 | INFO | } | 20260505 17:21:23.402 | INFO | Perf report for the last 5.13928 seconds: | 20260505 17:21:23.402 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:21:23.402 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:23.402 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:23.402 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:23.402 | INFO | ANALYSIS : 63.8%, 3.0, 99.4ms, 0 | 20260505 17:21:23.402 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:23.402 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260505 17:21:23.402 | INFO | PRE_HQ : 60.5%, 1.8, 56.0ms, 0 | 20260505 17:21:23.402 | INFO | BAZWRITER : 63.4%, 3.8, 124.8ms, 0 | 20260505 17:21:23.402 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 17:21:23.402 | INFO | Uploaded 36GiB to GPU in 2.17943s (16.5181GiB/s) | 20260505 17:21:23.402 | INFO | Downloaded 38GiB to GPU in 2.26913s (16.7465GiB/s) | 20260505 17:21:28.544 | INFO | Analyzing chunk frames = [4960768,4961280) | 20260505 17:21:28.544 | INFO | MemInfo: { | 20260505 17:21:28.544 | INFO | "gpu" : " 17 GB", | 20260505 17:21:28.544 | INFO | "huge" : "", | 20260505 17:21:28.544 | INFO | "huge_shared" : " 27 GB", | 20260505 17:21:28.544 | INFO | "malloc" : " 14 GB", | 20260505 17:21:28.544 | INFO | "pinned" : " 29 GB" | 20260505 17:21:28.544 | INFO | } | 20260505 17:21:28.544 | INFO | Perf report for the last 5.14264 seconds: | 20260505 17:21:28.544 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:21:28.544 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:28.544 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:28.544 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:28.544 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 17:21:28.544 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:28.544 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:28.544 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 17:21:28.544 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:28.544 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:28.545 | INFO | Uploaded 36GiB to GPU in 2.1381s (16.8374GiB/s) | 20260505 17:21:28.545 | INFO | Downloaded 34GiB to GPU in 2.0297s (16.7513GiB/s) | 20260505 17:21:33.676 | INFO | Analyzing chunk frames = [4961280,4961792) | 20260505 17:21:33.676 | INFO | MemInfo: { | 20260505 17:21:33.676 | INFO | "gpu" : " 17 GB", | 20260505 17:21:33.676 | INFO | "huge" : "", | 20260505 17:21:33.676 | INFO | "huge_shared" : " 27 GB", | 20260505 17:21:33.676 | INFO | "malloc" : " 14 GB", | 20260505 17:21:33.676 | INFO | "pinned" : " 29 GB" | 20260505 17:21:33.676 | INFO | } | 20260505 17:21:33.676 | INFO | Perf report for the last 5.13225 seconds: | 20260505 17:21:33.676 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:21:33.676 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:33.677 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:33.677 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:33.677 | INFO | ANALYSIS : 62.8%, 3.0, 97.6ms, 0 | 20260505 17:21:33.677 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:33.677 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:33.677 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:21:33.677 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:33.677 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:33.677 | INFO | Uploaded 36GiB to GPU in 2.20164s (16.3514GiB/s) | 20260505 17:21:33.677 | INFO | Downloaded 34GiB to GPU in 2.04566s (16.6206GiB/s) | 20260505 17:21:38.809 | INFO | Analyzing chunk frames = [4961792,4962304) | 20260505 17:21:38.809 | INFO | MemInfo: { | 20260505 17:21:38.809 | INFO | "gpu" : " 17 GB", | 20260505 17:21:38.809 | INFO | "huge" : "", | 20260505 17:21:38.809 | INFO | "huge_shared" : " 27 GB", | 20260505 17:21:38.809 | INFO | "malloc" : " 14 GB", | 20260505 17:21:38.809 | INFO | "pinned" : " 29 GB" | 20260505 17:21:38.809 | INFO | } | 20260505 17:21:38.809 | INFO | Perf report for the last 5.13259 seconds: | 20260505 17:21:38.809 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:21:38.809 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:38.809 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:38.809 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:38.809 | INFO | ANALYSIS : 62.6%, 3.0, 97.1ms, 0 | 20260505 17:21:38.809 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:38.809 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:38.809 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 17:21:38.809 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:38.809 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:38.809 | INFO | Uploaded 36GiB to GPU in 2.16463s (16.631GiB/s) | 20260505 17:21:38.809 | INFO | Downloaded 34GiB to GPU in 2.14475s (15.8526GiB/s) | 20260505 17:21:43.961 | INFO | Analyzing chunk frames = [4962304,4962816) | 20260505 17:21:43.961 | INFO | MemInfo: { | 20260505 17:21:43.961 | INFO | "gpu" : " 17 GB", | 20260505 17:21:43.961 | INFO | "huge" : "", | 20260505 17:21:43.961 | INFO | "huge_shared" : " 27 GB", | 20260505 17:21:43.961 | INFO | "malloc" : " 14 GB", | 20260505 17:21:43.961 | INFO | "pinned" : " 29 GB" | 20260505 17:21:43.961 | INFO | } | 20260505 17:21:43.961 | INFO | Perf report for the last 5.15158 seconds: | 20260505 17:21:43.961 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:21:43.961 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:43.961 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:43.961 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:43.961 | INFO | ANALYSIS : 62.5%, 3.0, 97.8ms, 0 | 20260505 17:21:43.961 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:43.961 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:43.961 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260505 17:21:43.961 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:43.961 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:43.961 | INFO | Uploaded 36GiB to GPU in 2.18019s (16.5123GiB/s) | 20260505 17:21:43.961 | INFO | Downloaded 34GiB to GPU in 2.13251s (15.9437GiB/s) | 20260505 17:21:49.104 | INFO | Analyzing chunk frames = [4962816,4963328) | 20260505 17:21:49.104 | INFO | MemInfo: { | 20260505 17:21:49.104 | INFO | "gpu" : " 17 GB", | 20260505 17:21:49.104 | INFO | "huge" : "", | 20260505 17:21:49.104 | INFO | "huge_shared" : " 27 GB", | 20260505 17:21:49.104 | INFO | "malloc" : " 14 GB", | 20260505 17:21:49.104 | INFO | "pinned" : " 29 GB" | 20260505 17:21:49.104 | INFO | } | 20260505 17:21:49.104 | INFO | Perf report for the last 5.14322 seconds: | 20260505 17:21:49.104 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:21:49.104 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:49.104 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:49.104 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:49.104 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 17:21:49.104 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:49.104 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:49.104 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260505 17:21:49.104 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:49.104 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:49.104 | INFO | Uploaded 36GiB to GPU in 2.13899s (16.8303GiB/s) | 20260505 17:21:49.104 | INFO | Downloaded 34GiB to GPU in 2.05073s (16.5795GiB/s) | 20260505 17:21:54.238 | INFO | Analyzing chunk frames = [4963328,4963840) | 20260505 17:21:54.238 | INFO | MemInfo: { | 20260505 17:21:54.238 | INFO | "gpu" : " 17 GB", | 20260505 17:21:54.238 | INFO | "huge" : "", | 20260505 17:21:54.238 | INFO | "huge_shared" : " 27 GB", | 20260505 17:21:54.238 | INFO | "malloc" : " 14 GB", | 20260505 17:21:54.238 | INFO | "pinned" : " 29 GB" | 20260505 17:21:54.238 | INFO | } | 20260505 17:21:54.238 | INFO | Perf report for the last 5.13395 seconds: | 20260505 17:21:54.238 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:21:54.238 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:54.238 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:54.238 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:54.238 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 17:21:54.238 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:54.238 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:54.238 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 17:21:54.238 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:54.238 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:54.238 | INFO | Uploaded 36GiB to GPU in 2.12624s (16.9313GiB/s) | 20260505 17:21:54.238 | INFO | Downloaded 34GiB to GPU in 2.13546s (15.9217GiB/s) | 20260505 17:21:59.371 | INFO | Analyzing chunk frames = [4963840,4964352) | 20260505 17:21:59.371 | INFO | MemInfo: { | 20260505 17:21:59.371 | INFO | "gpu" : " 17 GB", | 20260505 17:21:59.371 | INFO | "huge" : "", | 20260505 17:21:59.371 | INFO | "huge_shared" : " 27 GB", | 20260505 17:21:59.371 | INFO | "malloc" : " 14 GB", | 20260505 17:21:59.371 | INFO | "pinned" : " 29 GB" | 20260505 17:21:59.371 | INFO | } | 20260505 17:21:59.371 | INFO | Perf report for the last 5.13295 seconds: | 20260505 17:21:59.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:21:59.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:59.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:59.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:59.371 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 17:21:59.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:59.371 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:59.371 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 17:21:59.371 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:59.371 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:21:59.371 | INFO | Uploaded 36GiB to GPU in 2.15712s (16.6889GiB/s) | 20260505 17:21:59.371 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 17:21:59.371 | INFO | Total mem usage: 15.3 GiB | 20260505 17:21:59.371 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 17:21:59.371 | INFO | Downloaded 34GiB to GPU in 2.02606s (16.7813GiB/s) | 20260505 17:21:59.371 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.83%) | 20260505 17:21:59.371 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 17:21:59.371 | INFO | High Water Breakdown: | 20260505 17:21:59.371 | INFO | 61% : 8.67 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 17:21:59.371 | INFO | 38.9% : 5.53 GiB ( 83.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 17:21:59.371 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 17:21:59.371 | INFO | ------------------------------------------ | 20260505 17:21:59.371 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:21:59.371 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 17:21:59.371 | INFO | Total mem usage: 26.5 GiB | 20260505 17:21:59.371 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 17:21:59.371 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 17:21:59.371 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 17:21:59.371 | INFO | High Water Breakdown: | 20260505 17:21:59.371 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:21:59.371 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:21:59.371 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:21:59.371 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:21:59.372 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:21:59.372 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:21:59.372 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 17:21:59.372 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 17:21:59.372 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:21:59.372 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 17:21:59.372 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 17:21:59.372 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 17:21:59.372 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 17:21:59.372 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 17:21:59.372 | INFO | ------------------------------------------ | 20260505 17:21:59.372 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:21:59.372 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 17:21:59.372 | INFO | Total mem usage: 25.6 GiB | 20260505 17:21:59.372 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 17:21:59.372 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 17:21:59.372 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 17:21:59.372 | INFO | High Water Breakdown: | 20260505 17:21:59.372 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 17:21:59.372 | INFO | ------------------------------------------ | 20260505 17:21:59.372 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:21:59.372 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 17:21:59.372 | INFO | Total mem usage: 15.7 GiB | 20260505 17:21:59.372 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 17:21:59.372 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 17:21:59.372 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 17:21:59.372 | INFO | High Water Breakdown: | 20260505 17:21:59.372 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 17:21:59.372 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 17:21:59.372 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 17:21:59.372 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 17:21:59.372 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 17:21:59.372 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 17:21:59.372 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 17:21:59.372 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 17:21:59.372 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 17:21:59.372 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:21:59.372 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 17:21:59.372 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:21:59.372 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 17:21:59.372 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 17:21:59.372 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 17:21:59.372 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 17:21:59.372 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:21:59.372 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 17:21:59.372 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:21:59.372 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:21:59.372 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:21:59.372 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 17:21:59.372 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 17:21:59.372 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:21:59.372 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 17:21:59.372 | INFO | ------------------------------------------ | 20260505 17:21:59.372 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:21:59.372 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 17:21:59.372 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:21:59.372 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:21:59.372 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.52e+01, 1.36e+01 / 2.32e+00 | 20260505 17:21:59.372 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 1.02e-01, 1.10e-02 / 8.32e-03 | 20260505 17:21:59.372 | INFO | PerfReport: 11.78%, Baseline -- 3.72e+00 / 4.53e+00, 5.69e-01 / 1.19e-01 | 20260505 17:21:59.372 | INFO | PerfReport: 3.27%, Histogram -- 1.04e+00 / 1.51e+00, 1.91e-01 / 9.33e-02 | 20260505 17:21:59.372 | INFO | PerfReport: 16.03%, DME -- 4.05e+01 / 4.53e+01, 4.61e+00 / 2.56e+00 | 20260505 17:21:59.372 | INFO | PerfReport: 56.75%, FrameLabeling -- 1.79e+01 / 1.90e+01, 1.42e+00 / 9.85e-01 | 20260505 17:21:59.372 | INFO | PerfReport: 7.33%, PulseAccumulating -- 2.32e+00 / 2.93e+00, 4.52e-01 / 1.72e-01 | 20260505 17:21:59.372 | INFO | PerfReport: 3.37%, Metrics -- 1.06e+00 / 1.40e+00, 2.95e-01 / 1.24e-01 | 20260505 17:21:59.372 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.71e-01 / 4.30e-01, 6.59e-02 / 3.51e-02 | 20260505 17:21:59.372 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 17:21:59.372 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:21:59.372 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:21:59.372 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.56e+01 / 0.00e+00 | 20260505 17:21:59.372 | INFO | PerfReport: 22.52%, Upload -- 2.25e+01 / 0.00e+00, 7.88e+00 / 0.00e+00 | 20260505 17:21:59.372 | INFO | PerfReport: 21.88%, Download -- 2.19e+01 / 0.00e+00, 7.88e+00 / 0.00e+00 | 20260505 17:21:59.372 | INFO | PerfReport: 31.60%, Compute -- 3.16e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 17:21:59.372 | INFO | PerfReport: 2.23%, LockUpload -- 2.23e+00 / 0.00e+00, 7.87e+00 / 0.00e+00 | 20260505 17:21:59.372 | INFO | PerfReport: 1.48%, LockDownload -- 1.48e+00 / 0.00e+00, 2.98e+00 / 0.00e+00 | 20260505 17:21:59.372 | INFO | PerfReport: 20.28%, LockCompute -- 2.03e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 17:21:59.372 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.77e-02 / 0.00e+00 | 20260505 17:21:59.498 | INFO | Starting a round of Baz IO | 20260505 17:22:02.715 | INFO | Populating RTMetrics report | 20260505 17:22:02.715 | INFO | RTMetrics report populated | 20260505 17:22:02.715 | INFO | Opening tmp RTMetrics json file | 20260505 17:22:02.722 | INFO | Serializing RTMetrics to json | 20260505 17:22:02.723 | INFO | Serialization done, swapping tmp file now... | 20260505 17:22:02.724 | INFO | RTMetrics done | 20260505 17:22:02.809 | INFO | Finished a round of Baz IO: Wrote 2.133 GiB in 3.31 seconds | 20260505 17:22:02.809 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.07% / 0.00% / 77.92% | 20260505 17:22:02.809 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.304063% / 0% / 0.000107288% | 20260505 17:22:02.809 | INFO | Baz IO overall: | 20260505 17:22:02.809 | INFO | BytesWritten : 2936150503342 | 20260505 17:22:02.809 | INFO | NumEvents : 1392434986182 | 20260505 17:22:04.512 | INFO | Analyzing chunk frames = [4964352,4964864) | 20260505 17:22:04.512 | INFO | MemInfo: { | 20260505 17:22:04.512 | INFO | "gpu" : " 17 GB", | 20260505 17:22:04.512 | INFO | "huge" : "", | 20260505 17:22:04.512 | INFO | "huge_shared" : " 27 GB", | 20260505 17:22:04.512 | INFO | "malloc" : " 14 GB", | 20260505 17:22:04.512 | INFO | "pinned" : " 29 GB" | 20260505 17:22:04.512 | INFO | } | 20260505 17:22:04.512 | INFO | Perf report for the last 5.1416 seconds: | 20260505 17:22:04.512 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:22:04.512 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:04.513 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:04.513 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:04.513 | INFO | ANALYSIS : 64.0%, 3.0, 99.7ms, 0 | 20260505 17:22:04.513 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:04.513 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260505 17:22:04.513 | INFO | PRE_HQ : 60.8%, 1.8, 56.3ms, 0 | 20260505 17:22:04.513 | INFO | BAZWRITER : 63.5%, 3.8, 124.6ms, 0 | 20260505 17:22:04.513 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 17:22:04.513 | INFO | Uploaded 36GiB to GPU in 2.20652s (16.3153GiB/s) | 20260505 17:22:04.513 | INFO | Downloaded 38GiB to GPU in 2.25751s (16.8327GiB/s) | 20260505 17:22:09.655 | INFO | Analyzing chunk frames = [4964864,4965376) | 20260505 17:22:09.655 | INFO | MemInfo: { | 20260505 17:22:09.655 | INFO | "gpu" : " 17 GB", | 20260505 17:22:09.655 | INFO | "huge" : "", | 20260505 17:22:09.655 | INFO | "huge_shared" : " 27 GB", | 20260505 17:22:09.655 | INFO | "malloc" : " 14 GB", | 20260505 17:22:09.655 | INFO | "pinned" : " 29 GB" | 20260505 17:22:09.655 | INFO | } | 20260505 17:22:09.655 | INFO | Perf report for the last 5.14284 seconds: | 20260505 17:22:09.655 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:22:09.655 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:09.655 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:09.655 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:09.655 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 17:22:09.655 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:09.655 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:09.655 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 17:22:09.655 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:09.655 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:09.656 | INFO | Uploaded 36GiB to GPU in 2.1189s (16.9899GiB/s) | 20260505 17:22:09.656 | INFO | Downloaded 34GiB to GPU in 2.03032s (16.7461GiB/s) | 20260505 17:22:14.795 | INFO | Analyzing chunk frames = [4965376,4965888) | 20260505 17:22:14.795 | INFO | MemInfo: { | 20260505 17:22:14.795 | INFO | "gpu" : " 17 GB", | 20260505 17:22:14.795 | INFO | "huge" : "", | 20260505 17:22:14.795 | INFO | "huge_shared" : " 27 GB", | 20260505 17:22:14.795 | INFO | "malloc" : " 14 GB", | 20260505 17:22:14.795 | INFO | "pinned" : " 29 GB" | 20260505 17:22:14.795 | INFO | } | 20260505 17:22:14.796 | INFO | Perf report for the last 5.14019 seconds: | 20260505 17:22:14.796 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:22:14.796 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:14.796 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:14.796 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:14.796 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 17:22:14.796 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:14.796 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:14.796 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 17:22:14.796 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:14.796 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:14.796 | INFO | Uploaded 36GiB to GPU in 2.19859s (16.3741GiB/s) | 20260505 17:22:14.796 | INFO | Downloaded 34GiB to GPU in 2.04254s (16.6459GiB/s) | 20260505 17:22:19.932 | INFO | Analyzing chunk frames = [4965888,4966400) | 20260505 17:22:19.932 | INFO | MemInfo: { | 20260505 17:22:19.932 | INFO | "gpu" : " 17 GB", | 20260505 17:22:19.932 | INFO | "huge" : "", | 20260505 17:22:19.932 | INFO | "huge_shared" : " 27 GB", | 20260505 17:22:19.932 | INFO | "malloc" : " 14 GB", | 20260505 17:22:19.932 | INFO | "pinned" : " 29 GB" | 20260505 17:22:19.932 | INFO | } | 20260505 17:22:19.932 | INFO | Perf report for the last 5.13621 seconds: | 20260505 17:22:19.932 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:22:19.932 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:19.932 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:19.932 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:19.932 | INFO | ANALYSIS : 62.6%, 3.0, 97.1ms, 0 | 20260505 17:22:19.932 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:19.932 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:19.932 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:22:19.932 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:19.932 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:19.932 | INFO | Uploaded 36GiB to GPU in 2.18285s (16.4922GiB/s) | 20260505 17:22:19.932 | INFO | Downloaded 34GiB to GPU in 2.14492s (15.8514GiB/s) | 20260505 17:22:25.067 | INFO | Analyzing chunk frames = [4966400,4966912) | 20260505 17:22:25.067 | INFO | MemInfo: { | 20260505 17:22:25.067 | INFO | "gpu" : " 17 GB", | 20260505 17:22:25.067 | INFO | "huge" : "", | 20260505 17:22:25.067 | INFO | "huge_shared" : " 27 GB", | 20260505 17:22:25.067 | INFO | "malloc" : " 14 GB", | 20260505 17:22:25.067 | INFO | "pinned" : " 29 GB" | 20260505 17:22:25.068 | INFO | } | 20260505 17:22:25.068 | INFO | Perf report for the last 5.1358 seconds: | 20260505 17:22:25.068 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:22:25.068 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:25.068 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:25.068 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:25.068 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 17:22:25.068 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:25.068 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:25.068 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 17:22:25.068 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:25.068 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:25.068 | INFO | Uploaded 36GiB to GPU in 2.13006s (16.9009GiB/s) | 20260505 17:22:25.068 | INFO | Downloaded 34GiB to GPU in 2.05329s (16.5588GiB/s) | 20260505 17:22:30.215 | INFO | Analyzing chunk frames = [4966912,4967424) | 20260505 17:22:30.215 | INFO | MemInfo: { | 20260505 17:22:30.215 | INFO | "gpu" : " 17 GB", | 20260505 17:22:30.215 | INFO | "huge" : "", | 20260505 17:22:30.215 | INFO | "huge_shared" : " 27 GB", | 20260505 17:22:30.215 | INFO | "malloc" : " 14 GB", | 20260505 17:22:30.215 | INFO | "pinned" : " 29 GB" | 20260505 17:22:30.215 | INFO | } | 20260505 17:22:30.215 | INFO | Perf report for the last 5.1478 seconds: | 20260505 17:22:30.215 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:22:30.215 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:30.215 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:30.215 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:30.215 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260505 17:22:30.215 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:30.215 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:30.215 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 17:22:30.215 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:30.215 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:30.216 | INFO | Uploaded 36GiB to GPU in 2.14091s (16.8153GiB/s) | 20260505 17:22:30.216 | INFO | Downloaded 34GiB to GPU in 2.04487s (16.627GiB/s) | 20260505 17:22:35.355 | INFO | Analyzing chunk frames = [4967424,4967936) | 20260505 17:22:35.355 | INFO | MemInfo: { | 20260505 17:22:35.355 | INFO | "gpu" : " 17 GB", | 20260505 17:22:35.355 | INFO | "huge" : "", | 20260505 17:22:35.355 | INFO | "huge_shared" : " 27 GB", | 20260505 17:22:35.355 | INFO | "malloc" : " 14 GB", | 20260505 17:22:35.355 | INFO | "pinned" : " 29 GB" | 20260505 17:22:35.355 | INFO | } | 20260505 17:22:35.355 | INFO | Perf report for the last 5.13951 seconds: | 20260505 17:22:35.355 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:22:35.355 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:35.355 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:35.355 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:35.355 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 17:22:35.355 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:35.355 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:35.355 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260505 17:22:35.355 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:35.355 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:35.355 | INFO | Uploaded 36GiB to GPU in 2.15193s (16.7292GiB/s) | 20260505 17:22:35.355 | INFO | Downloaded 34GiB to GPU in 2.14767s (15.8311GiB/s) | 20260505 17:22:40.487 | INFO | Analyzing chunk frames = [4967936,4968448) | 20260505 17:22:40.487 | INFO | MemInfo: { | 20260505 17:22:40.487 | INFO | "gpu" : " 17 GB", | 20260505 17:22:40.487 | INFO | "huge" : "", | 20260505 17:22:40.487 | INFO | "huge_shared" : " 27 GB", | 20260505 17:22:40.487 | INFO | "malloc" : " 14 GB", | 20260505 17:22:40.487 | INFO | "pinned" : " 29 GB" | 20260505 17:22:40.487 | INFO | } | 20260505 17:22:40.487 | INFO | Perf report for the last 5.13235 seconds: | 20260505 17:22:40.487 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:22:40.487 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:40.487 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:40.487 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:40.487 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 17:22:40.487 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:40.487 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:40.487 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 17:22:40.487 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:40.487 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:40.487 | INFO | Uploaded 36GiB to GPU in 2.19253s (16.4194GiB/s) | 20260505 17:22:40.488 | INFO | Downloaded 34GiB to GPU in 2.02521s (16.7884GiB/s) | 20260505 17:22:40.621 | INFO | Starting a round of Baz IO | 20260505 17:22:43.820 | INFO | Populating RTMetrics report | 20260505 17:22:43.821 | INFO | RTMetrics report populated | 20260505 17:22:43.821 | INFO | Opening tmp RTMetrics json file | 20260505 17:22:43.827 | INFO | Serializing RTMetrics to json | 20260505 17:22:43.829 | INFO | Serialization done, swapping tmp file now... | 20260505 17:22:43.829 | INFO | RTMetrics done | 20260505 17:22:43.917 | INFO | Finished a round of Baz IO: Wrote 2.135 GiB in 3.296 seconds | 20260505 17:22:43.917 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.07% / 0.00% / 77.93% | 20260505 17:22:43.917 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.304619% / 0% / 0.000107288% | 20260505 17:22:43.917 | INFO | Baz IO overall: | 20260505 17:22:43.917 | INFO | BytesWritten : 2938442872028 | 20260505 17:22:43.917 | INFO | NumEvents : 1393506622430 | 20260505 17:22:45.625 | INFO | Analyzing chunk frames = [4968448,4968960) | 20260505 17:22:45.625 | INFO | MemInfo: { | 20260505 17:22:45.625 | INFO | "gpu" : " 17 GB", | 20260505 17:22:45.625 | INFO | "huge" : "", | 20260505 17:22:45.625 | INFO | "huge_shared" : " 27 GB", | 20260505 17:22:45.625 | INFO | "malloc" : " 14 GB", | 20260505 17:22:45.625 | INFO | "pinned" : " 29 GB" | 20260505 17:22:45.625 | INFO | } | 20260505 17:22:45.626 | INFO | Perf report for the last 5.13826 seconds: | 20260505 17:22:45.626 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:22:45.626 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:45.626 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:45.626 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:45.626 | INFO | ANALYSIS : 63.8%, 3.0, 99.4ms, 0 | 20260505 17:22:45.626 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:45.626 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260505 17:22:45.626 | INFO | PRE_HQ : 60.5%, 1.8, 56.4ms, 0 | 20260505 17:22:45.626 | INFO | BAZWRITER : 63.5%, 3.8, 124.6ms, 0 | 20260505 17:22:45.626 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 17:22:45.626 | INFO | Uploaded 36GiB to GPU in 2.1631s (16.6428GiB/s) | 20260505 17:22:45.626 | INFO | Downloaded 38GiB to GPU in 2.25916s (16.8204GiB/s) | 20260505 17:22:50.768 | INFO | Analyzing chunk frames = [4968960,4969472) | 20260505 17:22:50.768 | INFO | MemInfo: { | 20260505 17:22:50.768 | INFO | "gpu" : " 17 GB", | 20260505 17:22:50.768 | INFO | "huge" : "", | 20260505 17:22:50.768 | INFO | "huge_shared" : " 27 GB", | 20260505 17:22:50.768 | INFO | "malloc" : " 14 GB", | 20260505 17:22:50.768 | INFO | "pinned" : " 29 GB" | 20260505 17:22:50.768 | INFO | } | 20260505 17:22:50.768 | INFO | Perf report for the last 5.14245 seconds: | 20260505 17:22:50.768 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:22:50.768 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:50.768 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:50.768 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:50.768 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 17:22:50.768 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:50.768 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:50.768 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 17:22:50.768 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:50.768 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:50.768 | INFO | Uploaded 36GiB to GPU in 2.14015s (16.8212GiB/s) | 20260505 17:22:50.768 | INFO | Downloaded 34GiB to GPU in 2.03859s (16.6782GiB/s) | 20260505 17:22:55.911 | INFO | Analyzing chunk frames = [4969472,4969984) | 20260505 17:22:55.911 | INFO | MemInfo: { | 20260505 17:22:55.911 | INFO | "gpu" : " 17 GB", | 20260505 17:22:55.911 | INFO | "huge" : "", | 20260505 17:22:55.911 | INFO | "huge_shared" : " 27 GB", | 20260505 17:22:55.911 | INFO | "malloc" : " 14 GB", | 20260505 17:22:55.911 | INFO | "pinned" : " 29 GB" | 20260505 17:22:55.911 | INFO | } | 20260505 17:22:55.911 | INFO | Perf report for the last 5.14331 seconds: | 20260505 17:22:55.911 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:22:55.911 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:55.911 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:55.911 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:55.911 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 17:22:55.911 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:55.911 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:55.911 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 17:22:55.911 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:55.911 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:22:55.912 | INFO | Uploaded 36GiB to GPU in 2.20248s (16.3452GiB/s) | 20260505 17:22:55.912 | INFO | Downloaded 34GiB to GPU in 2.03843s (16.6795GiB/s) | 20260505 17:23:01.059 | INFO | Analyzing chunk frames = [4969984,4970496) | 20260505 17:23:01.059 | INFO | MemInfo: { | 20260505 17:23:01.059 | INFO | "gpu" : " 17 GB", | 20260505 17:23:01.059 | INFO | "huge" : "", | 20260505 17:23:01.059 | INFO | "huge_shared" : " 27 GB", | 20260505 17:23:01.059 | INFO | "malloc" : " 14 GB", | 20260505 17:23:01.059 | INFO | "pinned" : " 29 GB" | 20260505 17:23:01.059 | INFO | } | 20260505 17:23:01.059 | INFO | Perf report for the last 5.14798 seconds: | 20260505 17:23:01.059 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:23:01.059 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:01.059 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:01.059 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:01.059 | INFO | ANALYSIS : 62.8%, 3.0, 97.8ms, 0 | 20260505 17:23:01.059 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:01.059 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:01.059 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 17:23:01.059 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:01.059 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:01.059 | INFO | Uploaded 36GiB to GPU in 2.22869s (16.153GiB/s) | 20260505 17:23:01.060 | INFO | Downloaded 34GiB to GPU in 2.11127s (16.1041GiB/s) | 20260505 17:23:06.183 | INFO | Analyzing chunk frames = [4970496,4971008) | 20260505 17:23:06.183 | INFO | MemInfo: { | 20260505 17:23:06.183 | INFO | "gpu" : " 17 GB", | 20260505 17:23:06.183 | INFO | "huge" : "", | 20260505 17:23:06.183 | INFO | "huge_shared" : " 27 GB", | 20260505 17:23:06.183 | INFO | "malloc" : " 14 GB", | 20260505 17:23:06.183 | INFO | "pinned" : " 29 GB" | 20260505 17:23:06.183 | INFO | } | 20260505 17:23:06.183 | INFO | Perf report for the last 5.12347 seconds: | 20260505 17:23:06.183 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:23:06.183 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:06.183 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:06.183 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:06.183 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260505 17:23:06.183 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:06.183 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:06.183 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 17:23:06.183 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:06.183 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:06.183 | INFO | Uploaded 36GiB to GPU in 2.14377s (16.7929GiB/s) | 20260505 17:23:06.183 | INFO | Downloaded 34GiB to GPU in 2.05637s (16.534GiB/s) | 20260505 17:23:11.319 | INFO | Analyzing chunk frames = [4971008,4971520) | 20260505 17:23:11.319 | INFO | MemInfo: { | 20260505 17:23:11.319 | INFO | "gpu" : " 17 GB", | 20260505 17:23:11.319 | INFO | "huge" : "", | 20260505 17:23:11.319 | INFO | "huge_shared" : " 27 GB", | 20260505 17:23:11.319 | INFO | "malloc" : " 14 GB", | 20260505 17:23:11.319 | INFO | "pinned" : " 29 GB" | 20260505 17:23:11.319 | INFO | } | 20260505 17:23:11.319 | INFO | Perf report for the last 5.13627 seconds: | 20260505 17:23:11.319 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:23:11.319 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:11.319 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:11.319 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:11.319 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260505 17:23:11.319 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:11.319 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:11.319 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:23:11.319 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:11.319 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:11.319 | INFO | Uploaded 36GiB to GPU in 2.14295s (16.7993GiB/s) | 20260505 17:23:11.319 | INFO | Downloaded 34GiB to GPU in 2.03896s (16.6752GiB/s) | 20260505 17:23:16.466 | INFO | Analyzing chunk frames = [4971520,4972032) | 20260505 17:23:16.466 | INFO | MemInfo: { | 20260505 17:23:16.466 | INFO | "gpu" : " 17 GB", | 20260505 17:23:16.466 | INFO | "huge" : "", | 20260505 17:23:16.466 | INFO | "huge_shared" : " 27 GB", | 20260505 17:23:16.466 | INFO | "malloc" : " 14 GB", | 20260505 17:23:16.466 | INFO | "pinned" : " 29 GB" | 20260505 17:23:16.466 | INFO | } | 20260505 17:23:16.467 | INFO | Perf report for the last 5.14748 seconds: | 20260505 17:23:16.467 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:23:16.467 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:16.467 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:16.467 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:16.467 | INFO | ANALYSIS : 63.1%, 3.0, 98.7ms, 0 | 20260505 17:23:16.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:16.467 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:16.467 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 17:23:16.467 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:16.467 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:16.467 | INFO | Uploaded 36GiB to GPU in 2.12784s (16.9186GiB/s) | 20260505 17:23:16.467 | INFO | Downloaded 34GiB to GPU in 2.1661s (15.6964GiB/s) | 20260505 17:23:21.606 | INFO | Analyzing chunk frames = [4972032,4972544) | 20260505 17:23:21.606 | INFO | MemInfo: { | 20260505 17:23:21.606 | INFO | "gpu" : " 17 GB", | 20260505 17:23:21.606 | INFO | "huge" : "", | 20260505 17:23:21.606 | INFO | "huge_shared" : " 27 GB", | 20260505 17:23:21.606 | INFO | "malloc" : " 14 GB", | 20260505 17:23:21.606 | INFO | "pinned" : " 29 GB" | 20260505 17:23:21.606 | INFO | } | 20260505 17:23:21.606 | INFO | Perf report for the last 5.13928 seconds: | 20260505 17:23:21.606 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:23:21.606 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:21.606 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:21.606 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:21.606 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 17:23:21.606 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:21.606 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:21.606 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 17:23:21.606 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:21.606 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:21.606 | INFO | Uploaded 36GiB to GPU in 2.17069s (16.5846GiB/s) | 20260505 17:23:21.606 | INFO | Downloaded 34GiB to GPU in 2.02232s (16.8124GiB/s) | 20260505 17:23:21.606 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 17:23:21.606 | INFO | Total mem usage: 15.3 GiB | 20260505 17:23:21.606 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 17:23:21.606 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.98%) | 20260505 17:23:21.606 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 17:23:21.606 | INFO | High Water Breakdown: | 20260505 17:23:21.606 | INFO | 61% : 8.66 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 17:23:21.606 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 17:23:21.606 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 17:23:21.606 | INFO | ------------------------------------------ | 20260505 17:23:21.606 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:23:21.606 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 17:23:21.606 | INFO | Total mem usage: 26.5 GiB | 20260505 17:23:21.606 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 17:23:21.606 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 17:23:21.606 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 17:23:21.606 | INFO | High Water Breakdown: | 20260505 17:23:21.606 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:23:21.606 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:23:21.606 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:23:21.606 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:23:21.606 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:23:21.606 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:23:21.607 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 17:23:21.607 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 17:23:21.607 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:23:21.607 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 17:23:21.607 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 17:23:21.607 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 17:23:21.607 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 17:23:21.607 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 17:23:21.607 | INFO | ------------------------------------------ | 20260505 17:23:21.607 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:23:21.607 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 17:23:21.607 | INFO | Total mem usage: 25.6 GiB | 20260505 17:23:21.607 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 17:23:21.607 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 17:23:21.607 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 17:23:21.607 | INFO | High Water Breakdown: | 20260505 17:23:21.607 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 17:23:21.607 | INFO | ------------------------------------------ | 20260505 17:23:21.607 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:23:21.607 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 17:23:21.607 | INFO | Total mem usage: 15.7 GiB | 20260505 17:23:21.607 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 17:23:21.607 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 17:23:21.607 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 17:23:21.607 | INFO | High Water Breakdown: | 20260505 17:23:21.607 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 17:23:21.607 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 17:23:21.607 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 17:23:21.607 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 17:23:21.607 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 17:23:21.607 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 17:23:21.607 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 17:23:21.607 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 17:23:21.607 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 17:23:21.607 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:23:21.607 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 17:23:21.607 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:23:21.607 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 17:23:21.607 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 17:23:21.607 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 17:23:21.607 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 17:23:21.607 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:23:21.607 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 17:23:21.607 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:23:21.607 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:23:21.607 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:23:21.607 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 17:23:21.607 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 17:23:21.607 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:23:21.607 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 17:23:21.607 | INFO | ------------------------------------------ | 20260505 17:23:21.607 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:23:21.607 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 17:23:21.607 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:23:21.607 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:23:21.607 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.47e+01, 1.35e+01 / 1.81e+00 | 20260505 17:23:21.607 | INFO | PerfReport: 0.29%, AggregateStats -- 9.32e-02 / 1.03e-01, 1.24e-02 / 6.44e-03 | 20260505 17:23:21.607 | INFO | PerfReport: 11.77%, Baseline -- 3.72e+00 / 4.53e+00, 5.64e-01 / 1.40e-01 | 20260505 17:23:21.607 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.53e+00, 1.93e-01 / 1.02e-01 | 20260505 17:23:21.607 | INFO | PerfReport: 15.94%, DME -- 4.03e+01 / 4.46e+01, 4.11e+00 / 1.88e+00 | 20260505 17:23:21.607 | INFO | PerfReport: 56.77%, FrameLabeling -- 1.79e+01 / 1.92e+01, 1.44e+00 / 9.43e-01 | 20260505 17:23:21.607 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.33e+00 / 2.94e+00, 4.53e-01 / 1.23e-01 | 20260505 17:23:21.607 | INFO | PerfReport: 3.38%, Metrics -- 1.07e+00 / 1.40e+00, 2.93e-01 / 1.74e-01 | 20260505 17:23:21.607 | INFO | PerfReport: 1.18%, PulseSerializing -- 3.72e-01 / 4.33e-01, 6.62e-02 / 4.06e-02 | 20260505 17:23:21.607 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 17:23:21.607 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:23:21.607 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:23:21.607 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.57e+01 / 0.00e+00 | 20260505 17:23:21.607 | INFO | PerfReport: 22.54%, Upload -- 2.26e+01 / 0.00e+00, 7.87e+00 / 0.00e+00 | 20260505 17:23:21.607 | INFO | PerfReport: 21.78%, Download -- 2.18e+01 / 0.00e+00, 7.78e+00 / 0.00e+00 | 20260505 17:23:21.607 | INFO | PerfReport: 31.60%, Compute -- 3.16e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 17:23:21.607 | INFO | PerfReport: 2.33%, LockUpload -- 2.33e+00 / 0.00e+00, 7.95e+00 / 0.00e+00 | 20260505 17:23:21.607 | INFO | PerfReport: 1.41%, LockDownload -- 1.41e+00 / 0.00e+00, 2.95e+00 / 0.00e+00 | 20260505 17:23:21.607 | INFO | PerfReport: 20.32%, LockCompute -- 2.03e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260505 17:23:21.607 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.90e-02 / 0.00e+00 | 20260505 17:23:21.743 | INFO | Starting a round of Baz IO | 20260505 17:23:24.923 | INFO | Populating RTMetrics report | 20260505 17:23:24.923 | INFO | RTMetrics report populated | 20260505 17:23:24.923 | INFO | Opening tmp RTMetrics json file | 20260505 17:23:24.932 | INFO | Serializing RTMetrics to json | 20260505 17:23:24.933 | INFO | Serialization done, swapping tmp file now... | 20260505 17:23:24.933 | INFO | RTMetrics done | 20260505 17:23:25.017 | INFO | Finished a round of Baz IO: Wrote 2.136 GiB in 3.273 seconds | 20260505 17:23:25.017 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.07% / 0.00% / 77.93% | 20260505 17:23:25.017 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.304902% / 3.97364e-06% / 0.000123183% | 20260505 17:23:25.017 | INFO | Baz IO overall: | 20260505 17:23:25.017 | INFO | BytesWritten : 2940736750978 | 20260505 17:23:25.017 | INFO | NumEvents : 1394579397952 | 20260505 17:23:26.740 | INFO | Analyzing chunk frames = [4972544,4973056) | 20260505 17:23:26.740 | INFO | MemInfo: { | 20260505 17:23:26.740 | INFO | "gpu" : " 17 GB", | 20260505 17:23:26.740 | INFO | "huge" : "", | 20260505 17:23:26.740 | INFO | "huge_shared" : " 27 GB", | 20260505 17:23:26.740 | INFO | "malloc" : " 14 GB", | 20260505 17:23:26.740 | INFO | "pinned" : " 29 GB" | 20260505 17:23:26.740 | INFO | } | 20260505 17:23:26.740 | INFO | Perf report for the last 5.13395 seconds: | 20260505 17:23:26.740 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:23:26.740 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:26.740 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:26.740 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:26.740 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260505 17:23:26.740 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:26.740 | INFO | RT_METRICS : 44.7%, 1.0, 23.4ms, 0 | 20260505 17:23:26.740 | INFO | PRE_HQ : 60.4%, 1.8, 56.1ms, 0 | 20260505 17:23:26.740 | INFO | BAZWRITER : 63.1%, 3.9, 125.1ms, 0 | 20260505 17:23:26.740 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 17:23:26.740 | INFO | Uploaded 36GiB to GPU in 2.1686s (16.6006GiB/s) | 20260505 17:23:26.740 | INFO | Downloaded 38GiB to GPU in 2.25438s (16.8561GiB/s) | 20260505 17:23:31.879 | INFO | Analyzing chunk frames = [4973056,4973568) | 20260505 17:23:31.879 | INFO | MemInfo: { | 20260505 17:23:31.879 | INFO | "gpu" : " 17 GB", | 20260505 17:23:31.879 | INFO | "huge" : "", | 20260505 17:23:31.879 | INFO | "huge_shared" : " 27 GB", | 20260505 17:23:31.879 | INFO | "malloc" : " 14 GB", | 20260505 17:23:31.879 | INFO | "pinned" : " 29 GB" | 20260505 17:23:31.879 | INFO | } | 20260505 17:23:31.879 | INFO | Perf report for the last 5.13925 seconds: | 20260505 17:23:31.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:23:31.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:31.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:31.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:31.879 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 17:23:31.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:31.879 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:31.879 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 17:23:31.879 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:31.879 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:31.879 | INFO | Uploaded 36GiB to GPU in 2.15169s (16.731GiB/s) | 20260505 17:23:31.879 | INFO | Downloaded 34GiB to GPU in 2.03897s (16.6751GiB/s) | 20260505 17:23:37.024 | INFO | Analyzing chunk frames = [4973568,4974080) | 20260505 17:23:37.024 | INFO | MemInfo: { | 20260505 17:23:37.024 | INFO | "gpu" : " 17 GB", | 20260505 17:23:37.024 | INFO | "huge" : "", | 20260505 17:23:37.024 | INFO | "huge_shared" : " 27 GB", | 20260505 17:23:37.025 | INFO | "malloc" : " 14 GB", | 20260505 17:23:37.025 | INFO | "pinned" : " 29 GB" | 20260505 17:23:37.025 | INFO | } | 20260505 17:23:37.025 | INFO | Perf report for the last 5.14552 seconds: | 20260505 17:23:37.025 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:23:37.025 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:37.025 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:37.025 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:37.025 | INFO | ANALYSIS : 62.8%, 3.0, 97.8ms, 0 | 20260505 17:23:37.025 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:37.025 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:37.025 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 17:23:37.025 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:37.025 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:37.025 | INFO | Uploaded 36GiB to GPU in 2.20917s (16.2957GiB/s) | 20260505 17:23:37.025 | INFO | Downloaded 34GiB to GPU in 2.04253s (16.646GiB/s) | 20260505 17:23:42.166 | INFO | Analyzing chunk frames = [4974080,4974592) | 20260505 17:23:42.166 | INFO | MemInfo: { | 20260505 17:23:42.166 | INFO | "gpu" : " 17 GB", | 20260505 17:23:42.166 | INFO | "huge" : "", | 20260505 17:23:42.166 | INFO | "huge_shared" : " 27 GB", | 20260505 17:23:42.166 | INFO | "malloc" : " 14 GB", | 20260505 17:23:42.166 | INFO | "pinned" : " 29 GB" | 20260505 17:23:42.166 | INFO | } | 20260505 17:23:42.166 | INFO | Perf report for the last 5.14131 seconds: | 20260505 17:23:42.166 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:23:42.166 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:42.166 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:42.166 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:42.166 | INFO | ANALYSIS : 62.8%, 3.0, 97.5ms, 0 | 20260505 17:23:42.166 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:42.166 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:42.166 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 17:23:42.166 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:42.166 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:42.166 | INFO | Uploaded 36GiB to GPU in 2.22711s (16.1645GiB/s) | 20260505 17:23:42.166 | INFO | Downloaded 34GiB to GPU in 2.12385s (16.0087GiB/s) | 20260505 17:23:47.301 | INFO | Analyzing chunk frames = [4974592,4975104) | 20260505 17:23:47.301 | INFO | MemInfo: { | 20260505 17:23:47.301 | INFO | "gpu" : " 17 GB", | 20260505 17:23:47.301 | INFO | "huge" : "", | 20260505 17:23:47.301 | INFO | "huge_shared" : " 27 GB", | 20260505 17:23:47.301 | INFO | "malloc" : " 14 GB", | 20260505 17:23:47.301 | INFO | "pinned" : " 29 GB" | 20260505 17:23:47.301 | INFO | } | 20260505 17:23:47.301 | INFO | Perf report for the last 5.13488 seconds: | 20260505 17:23:47.301 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:23:47.301 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:47.301 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:47.301 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:47.301 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260505 17:23:47.301 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:47.301 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:47.301 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 17:23:47.301 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:47.301 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:47.301 | INFO | Uploaded 36GiB to GPU in 2.14797s (16.76GiB/s) | 20260505 17:23:47.301 | INFO | Downloaded 34GiB to GPU in 2.05365s (16.5559GiB/s) | 20260505 17:23:52.432 | INFO | Analyzing chunk frames = [4975104,4975616) | 20260505 17:23:52.432 | INFO | MemInfo: { | 20260505 17:23:52.432 | INFO | "gpu" : " 17 GB", | 20260505 17:23:52.432 | INFO | "huge" : "", | 20260505 17:23:52.432 | INFO | "huge_shared" : " 27 GB", | 20260505 17:23:52.432 | INFO | "malloc" : " 14 GB", | 20260505 17:23:52.433 | INFO | "pinned" : " 29 GB" | 20260505 17:23:52.433 | INFO | } | 20260505 17:23:52.433 | INFO | Perf report for the last 5.13173 seconds: | 20260505 17:23:52.433 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:23:52.433 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:52.433 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:52.433 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:52.433 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260505 17:23:52.433 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:52.433 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:52.433 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:23:52.433 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:52.433 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:52.433 | INFO | Uploaded 36GiB to GPU in 2.13882s (16.8317GiB/s) | 20260505 17:23:52.433 | INFO | Downloaded 34GiB to GPU in 2.03485s (16.7088GiB/s) | 20260505 17:23:57.573 | INFO | Analyzing chunk frames = [4975616,4976128) | 20260505 17:23:57.573 | INFO | MemInfo: { | 20260505 17:23:57.573 | INFO | "gpu" : " 17 GB", | 20260505 17:23:57.573 | INFO | "huge" : "", | 20260505 17:23:57.573 | INFO | "huge_shared" : " 27 GB", | 20260505 17:23:57.573 | INFO | "malloc" : " 14 GB", | 20260505 17:23:57.573 | INFO | "pinned" : " 29 GB" | 20260505 17:23:57.573 | INFO | } | 20260505 17:23:57.573 | INFO | Perf report for the last 5.1409 seconds: | 20260505 17:23:57.573 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:23:57.573 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:57.573 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:57.573 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:57.574 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 17:23:57.574 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:57.574 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:57.574 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 17:23:57.574 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:57.574 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:23:57.574 | INFO | Uploaded 36GiB to GPU in 2.12858s (16.9127GiB/s) | 20260505 17:23:57.574 | INFO | Downloaded 34GiB to GPU in 2.17978s (15.5979GiB/s) | 20260505 17:24:02.721 | INFO | Analyzing chunk frames = [4976128,4976640) | 20260505 17:24:02.721 | INFO | MemInfo: { | 20260505 17:24:02.721 | INFO | "gpu" : " 17 GB", | 20260505 17:24:02.721 | INFO | "huge" : "", | 20260505 17:24:02.721 | INFO | "huge_shared" : " 27 GB", | 20260505 17:24:02.721 | INFO | "malloc" : " 14 GB", | 20260505 17:24:02.721 | INFO | "pinned" : " 29 GB" | 20260505 17:24:02.721 | INFO | } | 20260505 17:24:02.721 | INFO | Perf report for the last 5.14762 seconds: | 20260505 17:24:02.721 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:24:02.721 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:02.721 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:02.721 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:02.721 | INFO | ANALYSIS : 63.1%, 3.0, 98.7ms, 0 | 20260505 17:24:02.721 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:02.721 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:02.721 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 17:24:02.721 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:02.721 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:02.721 | INFO | Uploaded 36GiB to GPU in 2.18901s (16.4458GiB/s) | 20260505 17:24:02.721 | INFO | Downloaded 34GiB to GPU in 2.02979s (16.7505GiB/s) | 20260505 17:24:02.856 | INFO | Starting a round of Baz IO | 20260505 17:24:06.040 | INFO | Populating RTMetrics report | 20260505 17:24:06.040 | INFO | RTMetrics report populated | 20260505 17:24:06.040 | INFO | Opening tmp RTMetrics json file | 20260505 17:24:06.046 | INFO | Serializing RTMetrics to json | 20260505 17:24:06.048 | INFO | Serialization done, swapping tmp file now... | 20260505 17:24:06.051 | INFO | RTMetrics done | 20260505 17:24:06.133 | INFO | Finished a round of Baz IO: Wrote 2.137 GiB in 3.276 seconds | 20260505 17:24:06.133 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.07% / 0.00% / 77.93% | 20260505 17:24:06.133 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.303769% / 7.94729e-06% / 9.93411e-05% | 20260505 17:24:06.133 | INFO | Baz IO overall: | 20260505 17:24:06.133 | INFO | BytesWritten : 2943030958659 | 20260505 17:24:06.133 | INFO | NumEvents : 1395651922942 | 20260505 17:24:07.868 | INFO | Analyzing chunk frames = [4976640,4977152) | 20260505 17:24:07.868 | INFO | MemInfo: { | 20260505 17:24:07.868 | INFO | "gpu" : " 17 GB", | 20260505 17:24:07.868 | INFO | "huge" : "", | 20260505 17:24:07.868 | INFO | "huge_shared" : " 27 GB", | 20260505 17:24:07.868 | INFO | "malloc" : " 14 GB", | 20260505 17:24:07.868 | INFO | "pinned" : " 29 GB" | 20260505 17:24:07.868 | INFO | } | 20260505 17:24:07.868 | INFO | Perf report for the last 5.14663 seconds: | 20260505 17:24:07.868 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:24:07.868 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:07.868 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:07.868 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:07.868 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 17:24:07.868 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:07.868 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260505 17:24:07.868 | INFO | PRE_HQ : 60.3%, 1.8, 56.2ms, 0 | 20260505 17:24:07.868 | INFO | BAZWRITER : 63.0%, 3.9, 125.1ms, 0 | 20260505 17:24:07.868 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 17:24:07.868 | INFO | Uploaded 36GiB to GPU in 2.16953s (16.5935GiB/s) | 20260505 17:24:07.868 | INFO | Downloaded 38GiB to GPU in 2.24684s (16.9126GiB/s) | 20260505 17:24:12.999 | INFO | Analyzing chunk frames = [4977152,4977664) | 20260505 17:24:12.999 | INFO | MemInfo: { | 20260505 17:24:12.999 | INFO | "gpu" : " 17 GB", | 20260505 17:24:12.999 | INFO | "huge" : "", | 20260505 17:24:12.999 | INFO | "huge_shared" : " 27 GB", | 20260505 17:24:12.999 | INFO | "malloc" : " 14 GB", | 20260505 17:24:12.999 | INFO | "pinned" : " 29 GB" | 20260505 17:24:12.999 | INFO | } | 20260505 17:24:12.999 | INFO | Perf report for the last 5.13159 seconds: | 20260505 17:24:12.999 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:24:12.999 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:12.999 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:12.999 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:12.999 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260505 17:24:12.999 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:12.999 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:12.999 | INFO | PRE_HQ : 27.7%, 1.0, 14.3ms, 0 | 20260505 17:24:12.999 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:13.000 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:13.000 | INFO | Uploaded 36GiB to GPU in 2.15379s (16.7147GiB/s) | 20260505 17:24:13.000 | INFO | Downloaded 34GiB to GPU in 2.04747s (16.6058GiB/s) | 20260505 17:24:18.133 | INFO | Analyzing chunk frames = [4977664,4978176) | 20260505 17:24:18.133 | INFO | MemInfo: { | 20260505 17:24:18.133 | INFO | "gpu" : " 17 GB", | 20260505 17:24:18.133 | INFO | "huge" : "", | 20260505 17:24:18.133 | INFO | "huge_shared" : " 27 GB", | 20260505 17:24:18.133 | INFO | "malloc" : " 14 GB", | 20260505 17:24:18.133 | INFO | "pinned" : " 29 GB" | 20260505 17:24:18.133 | INFO | } | 20260505 17:24:18.133 | INFO | Perf report for the last 5.13358 seconds: | 20260505 17:24:18.133 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:24:18.133 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:18.133 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:18.133 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:18.133 | INFO | ANALYSIS : 62.9%, 3.0, 97.7ms, 0 | 20260505 17:24:18.133 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:18.133 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:18.133 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:24:18.133 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:18.133 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:18.133 | INFO | Uploaded 36GiB to GPU in 2.20603s (16.3189GiB/s) | 20260505 17:24:18.133 | INFO | Downloaded 34GiB to GPU in 2.04096s (16.6588GiB/s) | 20260505 17:24:23.279 | INFO | Analyzing chunk frames = [4978176,4978688) | 20260505 17:24:23.279 | INFO | MemInfo: { | 20260505 17:24:23.279 | INFO | "gpu" : " 17 GB", | 20260505 17:24:23.279 | INFO | "huge" : "", | 20260505 17:24:23.279 | INFO | "huge_shared" : " 27 GB", | 20260505 17:24:23.279 | INFO | "malloc" : " 14 GB", | 20260505 17:24:23.279 | INFO | "pinned" : " 29 GB" | 20260505 17:24:23.279 | INFO | } | 20260505 17:24:23.279 | INFO | Perf report for the last 5.14583 seconds: | 20260505 17:24:23.279 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:24:23.279 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:23.279 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:23.279 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:23.279 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260505 17:24:23.279 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:23.279 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:23.279 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:24:23.279 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:23.279 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:23.279 | INFO | Uploaded 36GiB to GPU in 2.17901s (16.5213GiB/s) | 20260505 17:24:23.279 | INFO | Downloaded 34GiB to GPU in 2.13391s (15.9332GiB/s) | 20260505 17:24:28.415 | INFO | Analyzing chunk frames = [4978688,4979200) | 20260505 17:24:28.415 | INFO | MemInfo: { | 20260505 17:24:28.415 | INFO | "gpu" : " 17 GB", | 20260505 17:24:28.415 | INFO | "huge" : "", | 20260505 17:24:28.415 | INFO | "huge_shared" : " 27 GB", | 20260505 17:24:28.415 | INFO | "malloc" : " 14 GB", | 20260505 17:24:28.415 | INFO | "pinned" : " 29 GB" | 20260505 17:24:28.415 | INFO | } | 20260505 17:24:28.415 | INFO | Perf report for the last 5.13625 seconds: | 20260505 17:24:28.415 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:24:28.415 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:28.415 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:28.415 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:28.415 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 17:24:28.415 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:28.415 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:28.415 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 17:24:28.415 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:28.415 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:28.415 | INFO | Uploaded 36GiB to GPU in 2.14953s (16.7478GiB/s) | 20260505 17:24:28.416 | INFO | Downloaded 34GiB to GPU in 2.05724s (16.527GiB/s) | 20260505 17:24:33.556 | INFO | Analyzing chunk frames = [4979200,4979712) | 20260505 17:24:33.556 | INFO | MemInfo: { | 20260505 17:24:33.556 | INFO | "gpu" : " 17 GB", | 20260505 17:24:33.556 | INFO | "huge" : "", | 20260505 17:24:33.556 | INFO | "huge_shared" : " 27 GB", | 20260505 17:24:33.556 | INFO | "malloc" : " 14 GB", | 20260505 17:24:33.556 | INFO | "pinned" : " 29 GB" | 20260505 17:24:33.556 | INFO | } | 20260505 17:24:33.556 | INFO | Perf report for the last 5.14109 seconds: | 20260505 17:24:33.556 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:24:33.556 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:33.556 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:33.556 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:33.556 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260505 17:24:33.556 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:33.556 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:33.556 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:24:33.556 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:33.556 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:33.556 | INFO | Uploaded 36GiB to GPU in 2.14259s (16.8021GiB/s) | 20260505 17:24:33.556 | INFO | Downloaded 34GiB to GPU in 2.03688s (16.6922GiB/s) | 20260505 17:24:38.693 | INFO | Analyzing chunk frames = [4979712,4980224) | 20260505 17:24:38.693 | INFO | MemInfo: { | 20260505 17:24:38.693 | INFO | "gpu" : " 17 GB", | 20260505 17:24:38.693 | INFO | "huge" : "", | 20260505 17:24:38.693 | INFO | "huge_shared" : " 27 GB", | 20260505 17:24:38.693 | INFO | "malloc" : " 14 GB", | 20260505 17:24:38.693 | INFO | "pinned" : " 29 GB" | 20260505 17:24:38.693 | INFO | } | 20260505 17:24:38.693 | INFO | Perf report for the last 5.13659 seconds: | 20260505 17:24:38.693 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:24:38.693 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:38.693 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:38.693 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:38.693 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 17:24:38.693 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:38.693 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:38.693 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 17:24:38.693 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:38.693 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:38.693 | INFO | Uploaded 36GiB to GPU in 2.16776s (16.607GiB/s) | 20260505 17:24:38.693 | INFO | Downloaded 34GiB to GPU in 2.17804s (15.6104GiB/s) | 20260505 17:24:43.830 | INFO | Analyzing chunk frames = [4980224,4980736) | 20260505 17:24:43.830 | INFO | MemInfo: { | 20260505 17:24:43.830 | INFO | "gpu" : " 17 GB", | 20260505 17:24:43.830 | INFO | "huge" : "", | 20260505 17:24:43.830 | INFO | "huge_shared" : " 27 GB", | 20260505 17:24:43.830 | INFO | "malloc" : " 14 GB", | 20260505 17:24:43.830 | INFO | "pinned" : " 29 GB" | 20260505 17:24:43.830 | INFO | } | 20260505 17:24:43.830 | INFO | Perf report for the last 5.13733 seconds: | 20260505 17:24:43.830 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:24:43.830 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:43.830 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:43.830 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:43.830 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 17:24:43.830 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:43.830 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:43.830 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 17:24:43.830 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:43.830 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:43.831 | INFO | Uploaded 36GiB to GPU in 2.1734s (16.5639GiB/s) | 20260505 17:24:43.831 | INFO | Downloaded 34GiB to GPU in 2.03487s (16.7087GiB/s) | 20260505 17:24:43.831 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 17:24:43.831 | INFO | Total mem usage: 15.3 GiB | 20260505 17:24:43.831 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 17:24:43.831 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.96%) | 20260505 17:24:43.831 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 17:24:43.831 | INFO | High Water Breakdown: | 20260505 17:24:43.831 | INFO | 61% : 8.66 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 17:24:43.831 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 17:24:43.831 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 17:24:43.831 | INFO | ------------------------------------------ | 20260505 17:24:43.831 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:24:43.831 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 17:24:43.831 | INFO | Total mem usage: 26.5 GiB | 20260505 17:24:43.831 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 17:24:43.831 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 17:24:43.831 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 17:24:43.831 | INFO | High Water Breakdown: | 20260505 17:24:43.831 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:24:43.831 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:24:43.831 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:24:43.831 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:24:43.831 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:24:43.831 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:24:43.831 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 17:24:43.831 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 17:24:43.831 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:24:43.831 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 17:24:43.831 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 17:24:43.831 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 17:24:43.831 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 17:24:43.831 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 17:24:43.831 | INFO | ------------------------------------------ | 20260505 17:24:43.831 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:24:43.831 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 17:24:43.831 | INFO | Total mem usage: 25.6 GiB | 20260505 17:24:43.831 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 17:24:43.831 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 17:24:43.831 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 17:24:43.831 | INFO | High Water Breakdown: | 20260505 17:24:43.831 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 17:24:43.831 | INFO | ------------------------------------------ | 20260505 17:24:43.831 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:24:43.831 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 17:24:43.831 | INFO | Total mem usage: 15.7 GiB | 20260505 17:24:43.831 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 17:24:43.831 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 17:24:43.831 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 17:24:43.831 | INFO | High Water Breakdown: | 20260505 17:24:43.831 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 17:24:43.831 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 17:24:43.831 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 17:24:43.831 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 17:24:43.831 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 17:24:43.831 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 17:24:43.831 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 17:24:43.831 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 17:24:43.831 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 17:24:43.831 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:24:43.831 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 17:24:43.831 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:24:43.831 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 17:24:43.831 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 17:24:43.831 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 17:24:43.831 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 17:24:43.831 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:24:43.831 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 17:24:43.831 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:24:43.831 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:24:43.831 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:24:43.831 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 17:24:43.831 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 17:24:43.831 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:24:43.831 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 17:24:43.831 | INFO | ------------------------------------------ | 20260505 17:24:43.831 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:24:43.831 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 17:24:43.831 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:24:43.831 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:24:43.831 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.50e+01, 1.36e+01 / 2.20e+00 | 20260505 17:24:43.831 | INFO | PerfReport: 0.29%, AggregateStats -- 9.24e-02 / 1.03e-01, 1.03e-02 / 6.88e-03 | 20260505 17:24:43.831 | INFO | PerfReport: 11.78%, Baseline -- 3.73e+00 / 4.46e+00, 5.75e-01 / 2.39e-01 | 20260505 17:24:43.831 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.51e+00, 1.92e-01 / 1.13e-01 | 20260505 17:24:43.831 | INFO | PerfReport: 15.91%, DME -- 4.03e+01 / 4.44e+01, 4.07e+00 / 2.26e+00 | 20260505 17:24:43.831 | INFO | PerfReport: 56.87%, FrameLabeling -- 1.80e+01 / 1.98e+01, 1.52e+00 / 1.65e+00 | 20260505 17:24:43.831 | INFO | PerfReport: 7.33%, PulseAccumulating -- 2.32e+00 / 2.86e+00, 4.46e-01 / 2.38e-01 | 20260505 17:24:43.831 | INFO | PerfReport: 3.38%, Metrics -- 1.07e+00 / 1.39e+00, 2.93e-01 / 1.27e-01 | 20260505 17:24:43.831 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.72e-01 / 4.26e-01, 6.67e-02 / 3.70e-02 | 20260505 17:24:43.831 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 17:24:43.831 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:24:43.831 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:24:43.831 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.56e+01 / 0.00e+00 | 20260505 17:24:43.831 | INFO | PerfReport: 22.58%, Upload -- 2.26e+01 / 0.00e+00, 7.92e+00 / 0.00e+00 | 20260505 17:24:43.831 | INFO | PerfReport: 21.82%, Download -- 2.18e+01 / 0.00e+00, 7.88e+00 / 0.00e+00 | 20260505 17:24:43.831 | INFO | PerfReport: 31.63%, Compute -- 3.16e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 17:24:43.831 | INFO | PerfReport: 2.29%, LockUpload -- 2.29e+00 / 0.00e+00, 7.89e+00 / 0.00e+00 | 20260505 17:24:43.831 | INFO | PerfReport: 1.37%, LockDownload -- 1.37e+00 / 0.00e+00, 2.87e+00 / 0.00e+00 | 20260505 17:24:43.831 | INFO | PerfReport: 20.30%, LockCompute -- 2.03e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 17:24:43.831 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.14e-02 / 0.00e+00 | 20260505 17:24:43.966 | INFO | Starting a round of Baz IO | 20260505 17:24:47.174 | INFO | Populating RTMetrics report | 20260505 17:24:47.174 | INFO | RTMetrics report populated | 20260505 17:24:47.174 | INFO | Opening tmp RTMetrics json file | 20260505 17:24:47.180 | INFO | Serializing RTMetrics to json | 20260505 17:24:47.181 | INFO | Serialization done, swapping tmp file now... | 20260505 17:24:47.182 | INFO | RTMetrics done | 20260505 17:24:47.274 | INFO | Finished a round of Baz IO: Wrote 2.134 GiB in 3.307 seconds | 20260505 17:24:47.274 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.07% / 0.00% / 77.93% | 20260505 17:24:47.274 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.302784% / 3.97364e-06% / 0.000107288% | 20260505 17:24:47.274 | INFO | Baz IO overall: | 20260505 17:24:47.274 | INFO | BytesWritten : 2945322821312 | 20260505 17:24:47.274 | INFO | NumEvents : 1396722579301 | 20260505 17:24:48.976 | INFO | Analyzing chunk frames = [4980736,4981248) | 20260505 17:24:48.976 | INFO | MemInfo: { | 20260505 17:24:48.976 | INFO | "gpu" : " 17 GB", | 20260505 17:24:48.976 | INFO | "huge" : "", | 20260505 17:24:48.976 | INFO | "huge_shared" : " 27 GB", | 20260505 17:24:48.976 | INFO | "malloc" : " 14 GB", | 20260505 17:24:48.976 | INFO | "pinned" : " 29 GB" | 20260505 17:24:48.976 | INFO | } | 20260505 17:24:48.976 | INFO | Perf report for the last 5.1458 seconds: | 20260505 17:24:48.976 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:24:48.976 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:48.976 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:48.976 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:48.976 | INFO | ANALYSIS : 63.9%, 3.0, 99.6ms, 0 | 20260505 17:24:48.976 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:48.976 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260505 17:24:48.976 | INFO | PRE_HQ : 59.7%, 1.8, 56.5ms, 0 | 20260505 17:24:48.976 | INFO | BAZWRITER : 63.3%, 3.9, 125.5ms, 0 | 20260505 17:24:48.976 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 17:24:48.976 | INFO | Uploaded 36GiB to GPU in 2.16547s (16.6245GiB/s) | 20260505 17:24:48.976 | INFO | Downloaded 38GiB to GPU in 2.26375s (16.7863GiB/s) | 20260505 17:24:54.119 | INFO | Analyzing chunk frames = [4981248,4981760) | 20260505 17:24:54.119 | INFO | MemInfo: { | 20260505 17:24:54.119 | INFO | "gpu" : " 17 GB", | 20260505 17:24:54.119 | INFO | "huge" : "", | 20260505 17:24:54.119 | INFO | "huge_shared" : " 27 GB", | 20260505 17:24:54.119 | INFO | "malloc" : " 14 GB", | 20260505 17:24:54.119 | INFO | "pinned" : " 29 GB" | 20260505 17:24:54.119 | INFO | } | 20260505 17:24:54.119 | INFO | Perf report for the last 5.1436 seconds: | 20260505 17:24:54.120 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:24:54.120 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:54.120 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:54.120 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:54.120 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 17:24:54.120 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:54.120 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:54.120 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 17:24:54.120 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:54.120 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:54.120 | INFO | Uploaded 36GiB to GPU in 2.13395s (16.8702GiB/s) | 20260505 17:24:54.120 | INFO | Downloaded 34GiB to GPU in 2.02285s (16.808GiB/s) | 20260505 17:24:59.247 | INFO | Analyzing chunk frames = [4981760,4982272) | 20260505 17:24:59.247 | INFO | MemInfo: { | 20260505 17:24:59.247 | INFO | "gpu" : " 17 GB", | 20260505 17:24:59.247 | INFO | "huge" : "", | 20260505 17:24:59.247 | INFO | "huge_shared" : " 27 GB", | 20260505 17:24:59.247 | INFO | "malloc" : " 14 GB", | 20260505 17:24:59.247 | INFO | "pinned" : " 29 GB" | 20260505 17:24:59.247 | INFO | } | 20260505 17:24:59.247 | INFO | Perf report for the last 5.12795 seconds: | 20260505 17:24:59.247 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:24:59.247 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:59.247 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:59.248 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:59.248 | INFO | ANALYSIS : 62.8%, 3.0, 97.6ms, 0 | 20260505 17:24:59.248 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:59.248 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:59.248 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:24:59.248 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:59.248 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:24:59.248 | INFO | Uploaded 36GiB to GPU in 2.2034s (16.3384GiB/s) | 20260505 17:24:59.248 | INFO | Downloaded 34GiB to GPU in 2.05227s (16.567GiB/s) | 20260505 17:25:04.382 | INFO | Analyzing chunk frames = [4982272,4982784) | 20260505 17:25:04.382 | INFO | MemInfo: { | 20260505 17:25:04.382 | INFO | "gpu" : " 17 GB", | 20260505 17:25:04.382 | INFO | "huge" : "", | 20260505 17:25:04.382 | INFO | "huge_shared" : " 27 GB", | 20260505 17:25:04.382 | INFO | "malloc" : " 14 GB", | 20260505 17:25:04.382 | INFO | "pinned" : " 29 GB" | 20260505 17:25:04.382 | INFO | } | 20260505 17:25:04.382 | INFO | Perf report for the last 5.1346 seconds: | 20260505 17:25:04.382 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:25:04.382 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:04.382 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:04.382 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:04.382 | INFO | ANALYSIS : 62.6%, 3.0, 97.2ms, 0 | 20260505 17:25:04.382 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:04.382 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:04.382 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:25:04.382 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:04.382 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:04.382 | INFO | Uploaded 36GiB to GPU in 2.19482s (16.4023GiB/s) | 20260505 17:25:04.382 | INFO | Downloaded 34GiB to GPU in 2.13418s (15.9312GiB/s) | 20260505 17:25:09.531 | INFO | Analyzing chunk frames = [4982784,4983296) | 20260505 17:25:09.531 | INFO | MemInfo: { | 20260505 17:25:09.531 | INFO | "gpu" : " 17 GB", | 20260505 17:25:09.531 | INFO | "huge" : "", | 20260505 17:25:09.531 | INFO | "huge_shared" : " 27 GB", | 20260505 17:25:09.531 | INFO | "malloc" : " 14 GB", | 20260505 17:25:09.531 | INFO | "pinned" : " 29 GB" | 20260505 17:25:09.531 | INFO | } | 20260505 17:25:09.531 | INFO | Perf report for the last 5.14904 seconds: | 20260505 17:25:09.531 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:25:09.531 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:09.531 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:09.531 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:09.531 | INFO | ANALYSIS : 62.6%, 3.0, 97.9ms, 0 | 20260505 17:25:09.531 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:09.531 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:09.531 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 17:25:09.531 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:09.531 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:09.531 | INFO | Uploaded 36GiB to GPU in 2.18513s (16.475GiB/s) | 20260505 17:25:09.531 | INFO | Downloaded 34GiB to GPU in 2.13121s (15.9534GiB/s) | 20260505 17:25:14.682 | INFO | Analyzing chunk frames = [4983296,4983808) | 20260505 17:25:14.682 | INFO | MemInfo: { | 20260505 17:25:14.682 | INFO | "gpu" : " 17 GB", | 20260505 17:25:14.682 | INFO | "huge" : "", | 20260505 17:25:14.682 | INFO | "huge_shared" : " 27 GB", | 20260505 17:25:14.682 | INFO | "malloc" : " 14 GB", | 20260505 17:25:14.682 | INFO | "pinned" : " 29 GB" | 20260505 17:25:14.682 | INFO | } | 20260505 17:25:14.682 | INFO | Perf report for the last 5.15064 seconds: | 20260505 17:25:14.682 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:25:14.682 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:14.682 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:14.682 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:14.682 | INFO | ANALYSIS : 62.2%, 3.0, 97.3ms, 0 | 20260505 17:25:14.682 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:14.682 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:14.682 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260505 17:25:14.682 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:14.682 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:14.682 | INFO | Uploaded 36GiB to GPU in 2.13796s (16.8385GiB/s) | 20260505 17:25:14.682 | INFO | Downloaded 34GiB to GPU in 2.04439s (16.6309GiB/s) | 20260505 17:25:19.816 | INFO | Analyzing chunk frames = [4983808,4984320) | 20260505 17:25:19.816 | INFO | MemInfo: { | 20260505 17:25:19.816 | INFO | "gpu" : " 17 GB", | 20260505 17:25:19.816 | INFO | "huge" : "", | 20260505 17:25:19.816 | INFO | "huge_shared" : " 27 GB", | 20260505 17:25:19.816 | INFO | "malloc" : " 14 GB", | 20260505 17:25:19.816 | INFO | "pinned" : " 29 GB" | 20260505 17:25:19.816 | INFO | } | 20260505 17:25:19.816 | INFO | Perf report for the last 5.13442 seconds: | 20260505 17:25:19.816 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:25:19.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:19.816 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:19.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:19.816 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 17:25:19.816 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:19.816 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:19.816 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 17:25:19.816 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:19.816 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:19.817 | INFO | Uploaded 36GiB to GPU in 2.1278s (16.9189GiB/s) | 20260505 17:25:19.817 | INFO | Downloaded 34GiB to GPU in 2.13366s (15.935GiB/s) | 20260505 17:25:24.945 | INFO | Analyzing chunk frames = [4984320,4984832) | 20260505 17:25:24.945 | INFO | MemInfo: { | 20260505 17:25:24.945 | INFO | "gpu" : " 17 GB", | 20260505 17:25:24.945 | INFO | "huge" : "", | 20260505 17:25:24.945 | INFO | "huge_shared" : " 27 GB", | 20260505 17:25:24.946 | INFO | "malloc" : " 14 GB", | 20260505 17:25:24.946 | INFO | "pinned" : " 29 GB" | 20260505 17:25:24.946 | INFO | } | 20260505 17:25:24.946 | INFO | Perf report for the last 5.12939 seconds: | 20260505 17:25:24.946 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:25:24.946 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:24.946 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:24.946 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:24.946 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 17:25:24.946 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:24.946 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:24.946 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 17:25:24.946 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:24.946 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:24.946 | INFO | Uploaded 36GiB to GPU in 2.20262s (16.3442GiB/s) | 20260505 17:25:24.946 | INFO | Downloaded 34GiB to GPU in 2.01954s (16.8355GiB/s) | 20260505 17:25:25.087 | INFO | Starting a round of Baz IO | 20260505 17:25:28.274 | INFO | Populating RTMetrics report | 20260505 17:25:28.275 | INFO | RTMetrics report populated | 20260505 17:25:28.275 | INFO | Opening tmp RTMetrics json file | 20260505 17:25:28.281 | INFO | Serializing RTMetrics to json | 20260505 17:25:28.283 | INFO | Serialization done, swapping tmp file now... | 20260505 17:25:28.284 | INFO | RTMetrics done | 20260505 17:25:28.371 | INFO | Finished a round of Baz IO: Wrote 2.134 GiB in 3.283 seconds | 20260505 17:25:28.371 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.06% / 0.00% / 77.93% | 20260505 17:25:28.371 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.301858% / 7.94729e-06% / 0.000107288% | 20260505 17:25:28.371 | INFO | Baz IO overall: | 20260505 17:25:28.371 | INFO | BytesWritten : 2947614026578 | 20260505 17:25:28.371 | INFO | NumEvents : 1397793071213 | 20260505 17:25:30.085 | INFO | Analyzing chunk frames = [4984832,4985344) | 20260505 17:25:30.085 | INFO | MemInfo: { | 20260505 17:25:30.085 | INFO | "gpu" : " 17 GB", | 20260505 17:25:30.085 | INFO | "huge" : "", | 20260505 17:25:30.085 | INFO | "huge_shared" : " 27 GB", | 20260505 17:25:30.085 | INFO | "malloc" : " 14 GB", | 20260505 17:25:30.085 | INFO | "pinned" : " 29 GB" | 20260505 17:25:30.085 | INFO | } | 20260505 17:25:30.085 | INFO | Perf report for the last 5.13986 seconds: | 20260505 17:25:30.085 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:25:30.085 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:30.086 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:30.086 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:30.086 | INFO | ANALYSIS : 63.7%, 3.0, 99.2ms, 0 | 20260505 17:25:30.086 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:30.086 | INFO | RT_METRICS : 44.7%, 1.0, 23.4ms, 0 | 20260505 17:25:30.086 | INFO | PRE_HQ : 60.4%, 1.8, 56.5ms, 0 | 20260505 17:25:30.086 | INFO | BAZWRITER : 63.3%, 3.9, 125.6ms, 0 | 20260505 17:25:30.086 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 17:25:30.086 | INFO | Uploaded 36GiB to GPU in 2.16196s (16.6515GiB/s) | 20260505 17:25:30.086 | INFO | Downloaded 38GiB to GPU in 2.25231s (16.8716GiB/s) | 20260505 17:25:35.233 | INFO | Analyzing chunk frames = [4985344,4985856) | 20260505 17:25:35.233 | INFO | MemInfo: { | 20260505 17:25:35.233 | INFO | "gpu" : " 17 GB", | 20260505 17:25:35.233 | INFO | "huge" : "", | 20260505 17:25:35.233 | INFO | "huge_shared" : " 27 GB", | 20260505 17:25:35.233 | INFO | "malloc" : " 14 GB", | 20260505 17:25:35.233 | INFO | "pinned" : " 29 GB" | 20260505 17:25:35.233 | INFO | } | 20260505 17:25:35.234 | INFO | Perf report for the last 5.14803 seconds: | 20260505 17:25:35.234 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:25:35.234 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:35.234 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:35.234 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:35.234 | INFO | ANALYSIS : 62.8%, 3.0, 98.3ms, 0 | 20260505 17:25:35.234 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:35.234 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:35.234 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:25:35.234 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:35.234 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:35.234 | INFO | Uploaded 36GiB to GPU in 2.13557s (16.8573GiB/s) | 20260505 17:25:35.234 | INFO | Downloaded 34GiB to GPU in 2.03789s (16.684GiB/s) | 20260505 17:25:40.372 | INFO | Analyzing chunk frames = [4985856,4986368) | 20260505 17:25:40.372 | INFO | MemInfo: { | 20260505 17:25:40.372 | INFO | "gpu" : " 17 GB", | 20260505 17:25:40.372 | INFO | "huge" : "", | 20260505 17:25:40.372 | INFO | "huge_shared" : " 27 GB", | 20260505 17:25:40.372 | INFO | "malloc" : " 14 GB", | 20260505 17:25:40.372 | INFO | "pinned" : " 29 GB" | 20260505 17:25:40.372 | INFO | } | 20260505 17:25:40.373 | INFO | Perf report for the last 5.13911 seconds: | 20260505 17:25:40.373 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:25:40.373 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:40.373 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:40.373 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:40.373 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 17:25:40.373 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:40.373 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:40.373 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 17:25:40.373 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:40.373 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:40.373 | INFO | Uploaded 36GiB to GPU in 2.20258s (16.3444GiB/s) | 20260505 17:25:40.373 | INFO | Downloaded 34GiB to GPU in 2.04486s (16.6271GiB/s) | 20260505 17:25:45.500 | INFO | Analyzing chunk frames = [4986368,4986880) | 20260505 17:25:45.500 | INFO | MemInfo: { | 20260505 17:25:45.500 | INFO | "gpu" : " 17 GB", | 20260505 17:25:45.500 | INFO | "huge" : "", | 20260505 17:25:45.500 | INFO | "huge_shared" : " 27 GB", | 20260505 17:25:45.500 | INFO | "malloc" : " 14 GB", | 20260505 17:25:45.500 | INFO | "pinned" : " 29 GB" | 20260505 17:25:45.500 | INFO | } | 20260505 17:25:45.500 | INFO | Perf report for the last 5.12749 seconds: | 20260505 17:25:45.500 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:25:45.500 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:45.500 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:45.500 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:45.500 | INFO | ANALYSIS : 63.0%, 3.0, 97.7ms, 0 | 20260505 17:25:45.500 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:45.500 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:45.500 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:25:45.500 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:45.500 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:45.500 | INFO | Uploaded 36GiB to GPU in 2.22885s (16.1518GiB/s) | 20260505 17:25:45.500 | INFO | Downloaded 34GiB to GPU in 2.11918s (16.0439GiB/s) | 20260505 17:25:50.641 | INFO | Analyzing chunk frames = [4986880,4987392) | 20260505 17:25:50.641 | INFO | MemInfo: { | 20260505 17:25:50.641 | INFO | "gpu" : " 17 GB", | 20260505 17:25:50.641 | INFO | "huge" : "", | 20260505 17:25:50.641 | INFO | "huge_shared" : " 27 GB", | 20260505 17:25:50.641 | INFO | "malloc" : " 14 GB", | 20260505 17:25:50.641 | INFO | "pinned" : " 29 GB" | 20260505 17:25:50.641 | INFO | } | 20260505 17:25:50.641 | INFO | Perf report for the last 5.14109 seconds: | 20260505 17:25:50.641 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:25:50.641 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:50.641 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:50.641 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:50.641 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 17:25:50.641 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:50.641 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:50.641 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 17:25:50.641 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:50.641 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:50.642 | INFO | Uploaded 36GiB to GPU in 2.13454s (16.8654GiB/s) | 20260505 17:25:50.642 | INFO | Downloaded 34GiB to GPU in 2.05851s (16.5168GiB/s) | 20260505 17:25:55.788 | INFO | Analyzing chunk frames = [4987392,4987904) | 20260505 17:25:55.788 | INFO | MemInfo: { | 20260505 17:25:55.788 | INFO | "gpu" : " 17 GB", | 20260505 17:25:55.788 | INFO | "huge" : "", | 20260505 17:25:55.788 | INFO | "huge_shared" : " 27 GB", | 20260505 17:25:55.788 | INFO | "malloc" : " 14 GB", | 20260505 17:25:55.788 | INFO | "pinned" : " 29 GB" | 20260505 17:25:55.788 | INFO | } | 20260505 17:25:55.788 | INFO | Perf report for the last 5.1468 seconds: | 20260505 17:25:55.788 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:25:55.788 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:55.788 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:55.788 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:55.788 | INFO | ANALYSIS : 62.4%, 3.0, 97.6ms, 0 | 20260505 17:25:55.788 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:55.788 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:55.788 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:25:55.788 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:55.788 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:25:55.788 | INFO | Uploaded 36GiB to GPU in 2.14359s (16.7942GiB/s) | 20260505 17:25:55.788 | INFO | Downloaded 34GiB to GPU in 2.04013s (16.6656GiB/s) | 20260505 17:26:00.927 | INFO | Analyzing chunk frames = [4987904,4988416) | 20260505 17:26:00.927 | INFO | MemInfo: { | 20260505 17:26:00.927 | INFO | "gpu" : " 17 GB", | 20260505 17:26:00.927 | INFO | "huge" : "", | 20260505 17:26:00.927 | INFO | "huge_shared" : " 27 GB", | 20260505 17:26:00.927 | INFO | "malloc" : " 14 GB", | 20260505 17:26:00.927 | INFO | "pinned" : " 29 GB" | 20260505 17:26:00.927 | INFO | } | 20260505 17:26:00.927 | INFO | Perf report for the last 5.13881 seconds: | 20260505 17:26:00.927 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:26:00.927 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:00.927 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:00.927 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:00.927 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 17:26:00.927 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:00.927 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:00.927 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 17:26:00.927 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:00.927 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:00.927 | INFO | Uploaded 36GiB to GPU in 2.14436s (16.7882GiB/s) | 20260505 17:26:00.927 | INFO | Downloaded 34GiB to GPU in 2.14905s (15.8209GiB/s) | 20260505 17:26:06.066 | INFO | Analyzing chunk frames = [4988416,4988928) | 20260505 17:26:06.066 | INFO | MemInfo: { | 20260505 17:26:06.066 | INFO | "gpu" : " 17 GB", | 20260505 17:26:06.066 | INFO | "huge" : "", | 20260505 17:26:06.066 | INFO | "huge_shared" : " 27 GB", | 20260505 17:26:06.066 | INFO | "malloc" : " 14 GB", | 20260505 17:26:06.066 | INFO | "pinned" : " 29 GB" | 20260505 17:26:06.066 | INFO | } | 20260505 17:26:06.066 | INFO | Perf report for the last 5.13894 seconds: | 20260505 17:26:06.066 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:26:06.066 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:06.066 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:06.066 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:06.066 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 17:26:06.066 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:06.066 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:06.066 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 17:26:06.066 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:06.066 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:06.066 | INFO | Uploaded 36GiB to GPU in 2.19371s (16.4106GiB/s) | 20260505 17:26:06.066 | INFO | Downloaded 34GiB to GPU in 2.02764s (16.7683GiB/s) | 20260505 17:26:06.066 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 17:26:06.066 | INFO | Total mem usage: 15.3 GiB | 20260505 17:26:06.066 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 17:26:06.066 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -7.33%) | 20260505 17:26:06.066 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 17:26:06.066 | INFO | High Water Breakdown: | 20260505 17:26:06.066 | INFO | 61.2% : 8.73 GiB ( 91.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 17:26:06.066 | INFO | 38.7% : 5.53 GiB ( 83.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 17:26:06.066 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 17:26:06.067 | INFO | ------------------------------------------ | 20260505 17:26:06.067 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:26:06.067 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 17:26:06.067 | INFO | Total mem usage: 26.5 GiB | 20260505 17:26:06.067 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 17:26:06.067 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 17:26:06.067 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 17:26:06.067 | INFO | High Water Breakdown: | 20260505 17:26:06.067 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:26:06.067 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:26:06.067 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:26:06.067 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:26:06.067 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:26:06.067 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:26:06.067 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 17:26:06.067 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 17:26:06.067 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:26:06.067 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 17:26:06.067 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 17:26:06.067 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 17:26:06.067 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 17:26:06.067 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 17:26:06.067 | INFO | ------------------------------------------ | 20260505 17:26:06.067 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:26:06.067 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 17:26:06.067 | INFO | Total mem usage: 25.6 GiB | 20260505 17:26:06.067 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 17:26:06.067 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 17:26:06.067 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 17:26:06.067 | INFO | High Water Breakdown: | 20260505 17:26:06.067 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 17:26:06.067 | INFO | ------------------------------------------ | 20260505 17:26:06.067 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:26:06.067 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 17:26:06.067 | INFO | Total mem usage: 15.7 GiB | 20260505 17:26:06.067 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 17:26:06.067 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 17:26:06.067 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 17:26:06.067 | INFO | High Water Breakdown: | 20260505 17:26:06.067 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 17:26:06.067 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 17:26:06.067 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 17:26:06.068 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 17:26:06.068 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 17:26:06.068 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 17:26:06.068 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 17:26:06.068 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 17:26:06.068 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 17:26:06.068 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:26:06.068 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 17:26:06.068 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:26:06.068 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 17:26:06.068 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 17:26:06.068 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 17:26:06.068 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 17:26:06.068 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:26:06.068 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 17:26:06.068 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:26:06.068 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:26:06.068 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:26:06.068 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 17:26:06.068 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 17:26:06.068 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:26:06.068 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 17:26:06.068 | INFO | ------------------------------------------ | 20260505 17:26:06.068 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:26:06.068 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 17:26:06.068 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:26:06.068 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:26:06.068 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.52e+01, 1.35e+01 / 2.13e+00 | 20260505 17:26:06.068 | INFO | PerfReport: 0.29%, AggregateStats -- 9.25e-02 / 1.03e-01, 1.09e-02 / 1.05e-02 | 20260505 17:26:06.068 | INFO | PerfReport: 11.75%, Baseline -- 3.71e+00 / 4.53e+00, 5.54e-01 / 1.43e-01 | 20260505 17:26:06.068 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.52e+00, 1.91e-01 / 8.16e-02 | 20260505 17:26:06.068 | INFO | PerfReport: 15.93%, DME -- 4.03e+01 / 4.47e+01, 4.16e+00 / 1.99e+00 | 20260505 17:26:06.068 | INFO | PerfReport: 56.88%, FrameLabeling -- 1.80e+01 / 1.96e+01, 1.48e+00 / 1.37e+00 | 20260505 17:26:06.068 | INFO | PerfReport: 7.33%, PulseAccumulating -- 2.32e+00 / 2.88e+00, 4.38e-01 / 1.87e-01 | 20260505 17:26:06.068 | INFO | PerfReport: 3.37%, Metrics -- 1.07e+00 / 1.40e+00, 2.94e-01 / 1.86e-01 | 20260505 17:26:06.068 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.71e-01 / 4.33e-01, 6.64e-02 / 3.78e-02 | 20260505 17:26:06.068 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 17:26:06.068 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:26:06.068 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:26:06.068 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260505 17:26:06.068 | INFO | PerfReport: 22.57%, Upload -- 2.26e+01 / 0.00e+00, 7.90e+00 / 0.00e+00 | 20260505 17:26:06.068 | INFO | PerfReport: 21.81%, Download -- 2.18e+01 / 0.00e+00, 7.75e+00 / 0.00e+00 | 20260505 17:26:06.068 | INFO | PerfReport: 31.57%, Compute -- 3.16e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 17:26:06.068 | INFO | PerfReport: 2.37%, LockUpload -- 2.37e+00 / 0.00e+00, 8.00e+00 / 0.00e+00 | 20260505 17:26:06.068 | INFO | PerfReport: 1.35%, LockDownload -- 1.35e+00 / 0.00e+00, 2.86e+00 / 0.00e+00 | 20260505 17:26:06.068 | INFO | PerfReport: 20.31%, LockCompute -- 2.03e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 17:26:06.068 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.61e-02 / 0.00e+00 | 20260505 17:26:06.206 | INFO | Starting a round of Baz IO | 20260505 17:26:09.401 | INFO | Populating RTMetrics report | 20260505 17:26:09.402 | INFO | RTMetrics report populated | 20260505 17:26:09.402 | INFO | Opening tmp RTMetrics json file | 20260505 17:26:09.412 | INFO | Serializing RTMetrics to json | 20260505 17:26:09.414 | INFO | Serialization done, swapping tmp file now... | 20260505 17:26:09.414 | INFO | RTMetrics done | 20260505 17:26:09.497 | INFO | Finished a round of Baz IO: Wrote 2.131 GiB in 3.29 seconds | 20260505 17:26:09.497 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.06% / 0.00% / 77.94% | 20260505 17:26:09.497 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.300213% / 0% / 0.000103315% | 20260505 17:26:09.497 | INFO | Baz IO overall: | 20260505 17:26:09.497 | INFO | BytesWritten : 2949902522849 | 20260505 17:26:09.497 | INFO | NumEvents : 1398862093819 | 20260505 17:26:11.198 | INFO | Analyzing chunk frames = [4988928,4989440) | 20260505 17:26:11.198 | INFO | MemInfo: { | 20260505 17:26:11.198 | INFO | "gpu" : " 17 GB", | 20260505 17:26:11.198 | INFO | "huge" : "", | 20260505 17:26:11.198 | INFO | "huge_shared" : " 27 GB", | 20260505 17:26:11.198 | INFO | "malloc" : " 14 GB", | 20260505 17:26:11.198 | INFO | "pinned" : " 29 GB" | 20260505 17:26:11.198 | INFO | } | 20260505 17:26:11.198 | INFO | Perf report for the last 5.1319 seconds: | 20260505 17:26:11.198 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:26:11.198 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:11.198 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:11.198 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:11.198 | INFO | ANALYSIS : 63.9%, 3.0, 99.4ms, 0 | 20260505 17:26:11.198 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:11.198 | INFO | RT_METRICS : 44.7%, 1.0, 23.4ms, 0 | 20260505 17:26:11.198 | INFO | PRE_HQ : 60.7%, 1.8, 56.3ms, 0 | 20260505 17:26:11.198 | INFO | BAZWRITER : 63.6%, 3.8, 124.4ms, 0 | 20260505 17:26:11.198 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260505 17:26:11.198 | INFO | Uploaded 36GiB to GPU in 2.19472s (16.403GiB/s) | 20260505 17:26:11.198 | INFO | Downloaded 38GiB to GPU in 2.26122s (16.8051GiB/s) | 20260505 17:26:16.336 | INFO | Analyzing chunk frames = [4989440,4989952) | 20260505 17:26:16.336 | INFO | MemInfo: { | 20260505 17:26:16.336 | INFO | "gpu" : " 17 GB", | 20260505 17:26:16.336 | INFO | "huge" : "", | 20260505 17:26:16.336 | INFO | "huge_shared" : " 27 GB", | 20260505 17:26:16.336 | INFO | "malloc" : " 14 GB", | 20260505 17:26:16.336 | INFO | "pinned" : " 29 GB" | 20260505 17:26:16.336 | INFO | } | 20260505 17:26:16.336 | INFO | Perf report for the last 5.13841 seconds: | 20260505 17:26:16.336 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:26:16.336 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:16.336 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:16.336 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:16.336 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 17:26:16.336 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:16.336 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:16.336 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 17:26:16.336 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:16.336 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:16.336 | INFO | Uploaded 36GiB to GPU in 2.20468s (16.3289GiB/s) | 20260505 17:26:16.337 | INFO | Downloaded 34GiB to GPU in 2.06205s (16.4884GiB/s) | 20260505 17:26:21.485 | INFO | Analyzing chunk frames = [4989952,4990464) | 20260505 17:26:21.485 | INFO | MemInfo: { | 20260505 17:26:21.485 | INFO | "gpu" : " 17 GB", | 20260505 17:26:21.485 | INFO | "huge" : "", | 20260505 17:26:21.485 | INFO | "huge_shared" : " 27 GB", | 20260505 17:26:21.485 | INFO | "malloc" : " 14 GB", | 20260505 17:26:21.485 | INFO | "pinned" : " 29 GB" | 20260505 17:26:21.485 | INFO | } | 20260505 17:26:21.485 | INFO | Perf report for the last 5.14913 seconds: | 20260505 17:26:21.485 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:26:21.485 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:21.485 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:21.485 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:21.485 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 17:26:21.485 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:21.485 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:21.485 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:26:21.485 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:21.485 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:21.486 | INFO | Uploaded 36GiB to GPU in 2.20252s (16.3449GiB/s) | 20260505 17:26:21.486 | INFO | Downloaded 34GiB to GPU in 2.03529s (16.7052GiB/s) | 20260505 17:26:26.636 | INFO | Analyzing chunk frames = [4990464,4990976) | 20260505 17:26:26.636 | INFO | MemInfo: { | 20260505 17:26:26.636 | INFO | "gpu" : " 17 GB", | 20260505 17:26:26.636 | INFO | "huge" : "", | 20260505 17:26:26.637 | INFO | "huge_shared" : " 27 GB", | 20260505 17:26:26.637 | INFO | "malloc" : " 14 GB", | 20260505 17:26:26.637 | INFO | "pinned" : " 29 GB" | 20260505 17:26:26.637 | INFO | } | 20260505 17:26:26.637 | INFO | Perf report for the last 5.15141 seconds: | 20260505 17:26:26.637 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:26:26.637 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:26.637 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:26.637 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:26.637 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260505 17:26:26.637 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:26.637 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:26.637 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 17:26:26.637 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:26.637 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:26.637 | INFO | Uploaded 36GiB to GPU in 2.22616s (16.1713GiB/s) | 20260505 17:26:26.637 | INFO | Downloaded 34GiB to GPU in 2.12055s (16.0336GiB/s) | 20260505 17:26:31.758 | INFO | Analyzing chunk frames = [4990976,4991488) | 20260505 17:26:31.758 | INFO | MemInfo: { | 20260505 17:26:31.758 | INFO | "gpu" : " 17 GB", | 20260505 17:26:31.758 | INFO | "huge" : "", | 20260505 17:26:31.758 | INFO | "huge_shared" : " 27 GB", | 20260505 17:26:31.759 | INFO | "malloc" : " 14 GB", | 20260505 17:26:31.759 | INFO | "pinned" : " 29 GB" | 20260505 17:26:31.759 | INFO | } | 20260505 17:26:31.759 | INFO | Perf report for the last 5.12197 seconds: | 20260505 17:26:31.759 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:26:31.759 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:31.759 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:31.759 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:31.759 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260505 17:26:31.759 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:31.759 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:31.759 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:26:31.759 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:31.759 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:31.759 | INFO | Uploaded 36GiB to GPU in 2.13896s (16.8306GiB/s) | 20260505 17:26:31.759 | INFO | Downloaded 34GiB to GPU in 2.05578s (16.5388GiB/s) | 20260505 17:26:36.894 | INFO | Analyzing chunk frames = [4991488,4992000) | 20260505 17:26:36.894 | INFO | MemInfo: { | 20260505 17:26:36.894 | INFO | "gpu" : " 17 GB", | 20260505 17:26:36.894 | INFO | "huge" : "", | 20260505 17:26:36.894 | INFO | "huge_shared" : " 27 GB", | 20260505 17:26:36.894 | INFO | "malloc" : " 14 GB", | 20260505 17:26:36.894 | INFO | "pinned" : " 29 GB" | 20260505 17:26:36.895 | INFO | } | 20260505 17:26:36.895 | INFO | Perf report for the last 5.13585 seconds: | 20260505 17:26:36.895 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:26:36.895 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:36.895 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:36.895 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:36.895 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260505 17:26:36.895 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:36.895 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:36.895 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 17:26:36.895 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:36.895 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:36.895 | INFO | Uploaded 36GiB to GPU in 2.14059s (16.8178GiB/s) | 20260505 17:26:36.895 | INFO | Downloaded 34GiB to GPU in 2.04423s (16.6322GiB/s) | 20260505 17:26:42.045 | INFO | Analyzing chunk frames = [4992000,4992512) | 20260505 17:26:42.045 | INFO | MemInfo: { | 20260505 17:26:42.045 | INFO | "gpu" : " 17 GB", | 20260505 17:26:42.045 | INFO | "huge" : "", | 20260505 17:26:42.045 | INFO | "huge_shared" : " 27 GB", | 20260505 17:26:42.045 | INFO | "malloc" : " 14 GB", | 20260505 17:26:42.045 | INFO | "pinned" : " 29 GB" | 20260505 17:26:42.045 | INFO | } | 20260505 17:26:42.045 | INFO | Perf report for the last 5.15062 seconds: | 20260505 17:26:42.045 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:26:42.045 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:42.045 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:42.045 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:42.045 | INFO | ANALYSIS : 63.2%, 3.0, 98.9ms, 0 | 20260505 17:26:42.045 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:42.045 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:42.045 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 17:26:42.045 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:42.045 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:42.046 | INFO | Uploaded 36GiB to GPU in 2.13217s (16.8842GiB/s) | 20260505 17:26:42.046 | INFO | Downloaded 34GiB to GPU in 2.16561s (15.6999GiB/s) | 20260505 17:26:47.182 | INFO | Analyzing chunk frames = [4992512,4993024) | 20260505 17:26:47.182 | INFO | MemInfo: { | 20260505 17:26:47.182 | INFO | "gpu" : " 17 GB", | 20260505 17:26:47.182 | INFO | "huge" : "", | 20260505 17:26:47.182 | INFO | "huge_shared" : " 27 GB", | 20260505 17:26:47.182 | INFO | "malloc" : " 14 GB", | 20260505 17:26:47.182 | INFO | "pinned" : " 29 GB" | 20260505 17:26:47.182 | INFO | } | 20260505 17:26:47.182 | INFO | Perf report for the last 5.13651 seconds: | 20260505 17:26:47.182 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:26:47.182 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:47.182 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:47.182 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:47.182 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260505 17:26:47.182 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:47.182 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:47.182 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 17:26:47.182 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:47.182 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:47.182 | INFO | Uploaded 36GiB to GPU in 2.1998s (16.3652GiB/s) | 20260505 17:26:47.182 | INFO | Downloaded 34GiB to GPU in 2.02287s (16.8078GiB/s) | 20260505 17:26:47.317 | INFO | Starting a round of Baz IO | 20260505 17:26:50.495 | INFO | Populating RTMetrics report | 20260505 17:26:50.496 | INFO | RTMetrics report populated | 20260505 17:26:50.496 | INFO | Opening tmp RTMetrics json file | 20260505 17:26:50.506 | INFO | Serializing RTMetrics to json | 20260505 17:26:50.509 | INFO | Serialization done, swapping tmp file now... | 20260505 17:26:50.511 | INFO | RTMetrics done | 20260505 17:26:50.597 | INFO | Finished a round of Baz IO: Wrote 2.13 GiB in 3.28 seconds | 20260505 17:26:50.597 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.06% / 0.00% / 77.94% | 20260505 17:26:50.597 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.298878% / 0% / 9.93411e-05% | 20260505 17:26:50.597 | INFO | Baz IO overall: | 20260505 17:26:50.597 | INFO | BytesWritten : 2952189768313 | 20260505 17:26:50.597 | INFO | NumEvents : 1399930397930 | 20260505 17:26:52.324 | INFO | Analyzing chunk frames = [4993024,4993536) | 20260505 17:26:52.324 | INFO | MemInfo: { | 20260505 17:26:52.324 | INFO | "gpu" : " 17 GB", | 20260505 17:26:52.324 | INFO | "huge" : "", | 20260505 17:26:52.324 | INFO | "huge_shared" : " 27 GB", | 20260505 17:26:52.324 | INFO | "malloc" : " 14 GB", | 20260505 17:26:52.324 | INFO | "pinned" : " 29 GB" | 20260505 17:26:52.324 | INFO | } | 20260505 17:26:52.325 | INFO | Perf report for the last 5.14276 seconds: | 20260505 17:26:52.325 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:26:52.325 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:52.325 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:52.325 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:52.325 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 17:26:52.325 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:52.325 | INFO | RT_METRICS : 44.9%, 1.0, 23.5ms, 0 | 20260505 17:26:52.325 | INFO | PRE_HQ : 60.6%, 1.8, 56.8ms, 0 | 20260505 17:26:52.325 | INFO | BAZWRITER : 63.2%, 3.8, 124.4ms, 0 | 20260505 17:26:52.325 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 17:26:52.325 | INFO | Uploaded 36GiB to GPU in 2.16937s (16.5947GiB/s) | 20260505 17:26:52.325 | INFO | Downloaded 38GiB to GPU in 2.24491s (16.9272GiB/s) | 20260505 17:26:57.458 | INFO | Analyzing chunk frames = [4993536,4994048) | 20260505 17:26:57.458 | INFO | MemInfo: { | 20260505 17:26:57.458 | INFO | "gpu" : " 17 GB", | 20260505 17:26:57.458 | INFO | "huge" : "", | 20260505 17:26:57.458 | INFO | "huge_shared" : " 27 GB", | 20260505 17:26:57.458 | INFO | "malloc" : " 14 GB", | 20260505 17:26:57.458 | INFO | "pinned" : " 29 GB" | 20260505 17:26:57.458 | INFO | } | 20260505 17:26:57.458 | INFO | Perf report for the last 5.13386 seconds: | 20260505 17:26:57.458 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:26:57.458 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:57.458 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:57.458 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:57.458 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 17:26:57.458 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:57.459 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:57.459 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 17:26:57.459 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:57.459 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:26:57.459 | INFO | Uploaded 36GiB to GPU in 2.14051s (16.8184GiB/s) | 20260505 17:26:57.459 | INFO | Downloaded 34GiB to GPU in 2.02936s (16.754GiB/s) | 20260505 17:27:02.593 | INFO | Analyzing chunk frames = [4994048,4994560) | 20260505 17:27:02.593 | INFO | MemInfo: { | 20260505 17:27:02.593 | INFO | "gpu" : " 17 GB", | 20260505 17:27:02.593 | INFO | "huge" : "", | 20260505 17:27:02.593 | INFO | "huge_shared" : " 27 GB", | 20260505 17:27:02.593 | INFO | "malloc" : " 14 GB", | 20260505 17:27:02.593 | INFO | "pinned" : " 29 GB" | 20260505 17:27:02.593 | INFO | } | 20260505 17:27:02.593 | INFO | Perf report for the last 5.13438 seconds: | 20260505 17:27:02.593 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:27:02.593 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:02.593 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:02.593 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:02.593 | INFO | ANALYSIS : 62.7%, 3.0, 97.5ms, 0 | 20260505 17:27:02.593 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:02.593 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:02.593 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 17:27:02.593 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:02.593 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:02.593 | INFO | Uploaded 36GiB to GPU in 2.2015s (16.3525GiB/s) | 20260505 17:27:02.593 | INFO | Downloaded 34GiB to GPU in 2.03968s (16.6693GiB/s) | 20260505 17:27:07.737 | INFO | Analyzing chunk frames = [4994560,4995072) | 20260505 17:27:07.737 | INFO | MemInfo: { | 20260505 17:27:07.737 | INFO | "gpu" : " 17 GB", | 20260505 17:27:07.737 | INFO | "huge" : "", | 20260505 17:27:07.737 | INFO | "huge_shared" : " 27 GB", | 20260505 17:27:07.737 | INFO | "malloc" : " 14 GB", | 20260505 17:27:07.737 | INFO | "pinned" : " 29 GB" | 20260505 17:27:07.737 | INFO | } | 20260505 17:27:07.737 | INFO | Perf report for the last 5.1443 seconds: | 20260505 17:27:07.737 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:27:07.737 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:07.737 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:07.737 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:07.737 | INFO | ANALYSIS : 62.5%, 3.0, 97.2ms, 0 | 20260505 17:27:07.737 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:07.737 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:07.737 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 17:27:07.737 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:07.737 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:07.737 | INFO | Uploaded 36GiB to GPU in 2.17756s (16.5323GiB/s) | 20260505 17:27:07.737 | INFO | Downloaded 34GiB to GPU in 2.14079s (15.882GiB/s) | 20260505 17:27:12.878 | INFO | Analyzing chunk frames = [4995072,4995584) | 20260505 17:27:12.878 | INFO | MemInfo: { | 20260505 17:27:12.878 | INFO | "gpu" : " 17 GB", | 20260505 17:27:12.878 | INFO | "huge" : "", | 20260505 17:27:12.878 | INFO | "huge_shared" : " 27 GB", | 20260505 17:27:12.878 | INFO | "malloc" : " 14 GB", | 20260505 17:27:12.878 | INFO | "pinned" : " 29 GB" | 20260505 17:27:12.878 | INFO | } | 20260505 17:27:12.879 | INFO | Perf report for the last 5.14144 seconds: | 20260505 17:27:12.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:27:12.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:12.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:12.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:12.879 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 17:27:12.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:12.879 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:12.879 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:27:12.879 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:12.879 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:12.879 | INFO | Uploaded 36GiB to GPU in 2.13825s (16.8362GiB/s) | 20260505 17:27:12.879 | INFO | Downloaded 34GiB to GPU in 2.05812s (16.5199GiB/s) | 20260505 17:27:18.008 | INFO | Analyzing chunk frames = [4995584,4996096) | 20260505 17:27:18.008 | INFO | MemInfo: { | 20260505 17:27:18.008 | INFO | "gpu" : " 17 GB", | 20260505 17:27:18.008 | INFO | "huge" : "", | 20260505 17:27:18.008 | INFO | "huge_shared" : " 27 GB", | 20260505 17:27:18.008 | INFO | "malloc" : " 14 GB", | 20260505 17:27:18.008 | INFO | "pinned" : " 29 GB" | 20260505 17:27:18.008 | INFO | } | 20260505 17:27:18.008 | INFO | Perf report for the last 5.1292 seconds: | 20260505 17:27:18.008 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:27:18.008 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:18.008 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:18.008 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:18.008 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260505 17:27:18.008 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:18.008 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:18.008 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 17:27:18.008 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:18.008 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:18.008 | INFO | Uploaded 36GiB to GPU in 2.14071s (16.8169GiB/s) | 20260505 17:27:18.008 | INFO | Downloaded 34GiB to GPU in 2.05399s (16.5531GiB/s) | 20260505 17:27:23.145 | INFO | Analyzing chunk frames = [4996096,4996608) | 20260505 17:27:23.145 | INFO | MemInfo: { | 20260505 17:27:23.145 | INFO | "gpu" : " 17 GB", | 20260505 17:27:23.145 | INFO | "huge" : "", | 20260505 17:27:23.145 | INFO | "huge_shared" : " 27 GB", | 20260505 17:27:23.145 | INFO | "malloc" : " 14 GB", | 20260505 17:27:23.145 | INFO | "pinned" : " 29 GB" | 20260505 17:27:23.145 | INFO | } | 20260505 17:27:23.145 | INFO | Perf report for the last 5.1376 seconds: | 20260505 17:27:23.145 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:27:23.145 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:23.145 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:23.145 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:23.145 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 17:27:23.145 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:23.145 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:23.145 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 17:27:23.146 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:23.146 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:23.146 | INFO | Uploaded 36GiB to GPU in 2.10175s (17.1286GiB/s) | 20260505 17:27:23.146 | INFO | Downloaded 34GiB to GPU in 2.17394s (15.6398GiB/s) | 20260505 17:27:28.294 | INFO | Analyzing chunk frames = [4996608,4997120) | 20260505 17:27:28.294 | INFO | MemInfo: { | 20260505 17:27:28.294 | INFO | "gpu" : " 17 GB", | 20260505 17:27:28.294 | INFO | "huge" : "", | 20260505 17:27:28.294 | INFO | "huge_shared" : " 27 GB", | 20260505 17:27:28.294 | INFO | "malloc" : " 14 GB", | 20260505 17:27:28.294 | INFO | "pinned" : " 29 GB" | 20260505 17:27:28.294 | INFO | } | 20260505 17:27:28.294 | INFO | Perf report for the last 5.14894 seconds: | 20260505 17:27:28.294 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:27:28.294 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:28.294 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:28.294 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:28.294 | INFO | ANALYSIS : 63.5%, 3.0, 99.4ms, 0 | 20260505 17:27:28.294 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:28.294 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:28.294 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 17:27:28.294 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:28.294 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:28.295 | INFO | Uploaded 36GiB to GPU in 2.23649s (16.0967GiB/s) | 20260505 17:27:28.295 | INFO | Downloaded 34GiB to GPU in 2.10625s (16.1424GiB/s) | 20260505 17:27:28.295 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 17:27:28.295 | INFO | Total mem usage: 15.3 GiB | 20260505 17:27:28.295 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 17:27:28.295 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.96%) | 20260505 17:27:28.295 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 17:27:28.295 | INFO | High Water Breakdown: | 20260505 17:27:28.295 | INFO | 61% : 8.67 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 17:27:28.295 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 17:27:28.295 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 17:27:28.295 | INFO | ------------------------------------------ | 20260505 17:27:28.295 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:27:28.295 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 17:27:28.295 | INFO | Total mem usage: 26.5 GiB | 20260505 17:27:28.295 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 17:27:28.295 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 17:27:28.295 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 17:27:28.295 | INFO | High Water Breakdown: | 20260505 17:27:28.295 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:27:28.295 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:27:28.295 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:27:28.295 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:27:28.295 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:27:28.295 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:27:28.295 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 17:27:28.295 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 17:27:28.295 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:27:28.295 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 17:27:28.295 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 17:27:28.295 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 17:27:28.295 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 17:27:28.295 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 17:27:28.295 | INFO | ------------------------------------------ | 20260505 17:27:28.295 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:27:28.295 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 17:27:28.295 | INFO | Total mem usage: 25.6 GiB | 20260505 17:27:28.295 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 17:27:28.295 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 17:27:28.295 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 17:27:28.295 | INFO | High Water Breakdown: | 20260505 17:27:28.295 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 17:27:28.295 | INFO | ------------------------------------------ | 20260505 17:27:28.295 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:27:28.295 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 17:27:28.295 | INFO | Total mem usage: 15.7 GiB | 20260505 17:27:28.295 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 17:27:28.295 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 17:27:28.295 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 17:27:28.295 | INFO | High Water Breakdown: | 20260505 17:27:28.295 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 17:27:28.295 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 17:27:28.295 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 17:27:28.295 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 17:27:28.295 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 17:27:28.295 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 17:27:28.295 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 17:27:28.295 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 17:27:28.295 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 17:27:28.295 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:27:28.295 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 17:27:28.295 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:27:28.295 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 17:27:28.295 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 17:27:28.295 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 17:27:28.295 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 17:27:28.295 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:27:28.295 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 17:27:28.295 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:27:28.295 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:27:28.295 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:27:28.295 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 17:27:28.295 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 17:27:28.295 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:27:28.295 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 17:27:28.295 | INFO | ------------------------------------------ | 20260505 17:27:28.296 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:27:28.296 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 17:27:28.296 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:27:28.296 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:27:28.296 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.49e+01, 1.35e+01 / 2.09e+00 | 20260505 17:27:28.296 | INFO | PerfReport: 0.29%, AggregateStats -- 9.25e-02 / 1.00e-01, 1.08e-02 / 4.19e-03 | 20260505 17:27:28.296 | INFO | PerfReport: 11.78%, Baseline -- 3.72e+00 / 4.50e+00, 5.62e-01 / 2.02e-01 | 20260505 17:27:28.296 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.51e+00, 1.89e-01 / 1.28e-01 | 20260505 17:27:28.296 | INFO | PerfReport: 16.00%, DME -- 4.04e+01 / 4.48e+01, 4.23e+00 / 1.89e+00 | 20260505 17:27:28.296 | INFO | PerfReport: 56.69%, FrameLabeling -- 1.79e+01 / 1.92e+01, 1.35e+00 / 1.28e+00 | 20260505 17:27:28.296 | INFO | PerfReport: 7.41%, PulseAccumulating -- 2.34e+00 / 2.94e+00, 4.52e-01 / 1.29e-01 | 20260505 17:27:28.296 | INFO | PerfReport: 3.38%, Metrics -- 1.07e+00 / 1.39e+00, 2.92e-01 / 1.40e-01 | 20260505 17:27:28.296 | INFO | PerfReport: 1.18%, PulseSerializing -- 3.71e-01 / 4.32e-01, 6.59e-02 / 3.75e-02 | 20260505 17:27:28.296 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 17:27:28.296 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:27:28.296 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:27:28.296 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260505 17:27:28.296 | INFO | PerfReport: 22.61%, Upload -- 2.26e+01 / 0.00e+00, 7.89e+00 / 0.00e+00 | 20260505 17:27:28.296 | INFO | PerfReport: 21.87%, Download -- 2.19e+01 / 0.00e+00, 7.86e+00 / 0.00e+00 | 20260505 17:27:28.296 | INFO | PerfReport: 31.53%, Compute -- 3.15e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 17:27:28.296 | INFO | PerfReport: 2.37%, LockUpload -- 2.37e+00 / 0.00e+00, 7.93e+00 / 0.00e+00 | 20260505 17:27:28.296 | INFO | PerfReport: 1.43%, LockDownload -- 1.43e+00 / 0.00e+00, 2.98e+00 / 0.00e+00 | 20260505 17:27:28.296 | INFO | PerfReport: 20.18%, LockCompute -- 2.02e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 17:27:28.296 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.49e-02 / 0.00e+00 | 20260505 17:27:28.431 | INFO | Starting a round of Baz IO | 20260505 17:27:31.609 | INFO | Populating RTMetrics report | 20260505 17:27:31.609 | INFO | RTMetrics report populated | 20260505 17:27:31.609 | INFO | Opening tmp RTMetrics json file | 20260505 17:27:31.615 | INFO | Serializing RTMetrics to json | 20260505 17:27:31.616 | INFO | Serialization done, swapping tmp file now... | 20260505 17:27:31.617 | INFO | RTMetrics done | 20260505 17:27:31.703 | INFO | Finished a round of Baz IO: Wrote 2.131 GiB in 3.271 seconds | 20260505 17:27:31.703 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.06% / 0.00% / 77.94% | 20260505 17:27:31.703 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.298917% / 0% / 0.000107288% | 20260505 17:27:31.703 | INFO | Baz IO overall: | 20260505 17:27:31.703 | INFO | BytesWritten : 2954477532537 | 20260505 17:27:31.703 | INFO | NumEvents : 1400999729863 | 20260505 17:27:33.438 | INFO | Analyzing chunk frames = [4997120,4997632) | 20260505 17:27:33.438 | INFO | MemInfo: { | 20260505 17:27:33.438 | INFO | "gpu" : " 17 GB", | 20260505 17:27:33.438 | INFO | "huge" : "", | 20260505 17:27:33.438 | INFO | "huge_shared" : " 27 GB", | 20260505 17:27:33.438 | INFO | "malloc" : " 14 GB", | 20260505 17:27:33.438 | INFO | "pinned" : " 29 GB" | 20260505 17:27:33.438 | INFO | } | 20260505 17:27:33.439 | INFO | Perf report for the last 5.14419 seconds: | 20260505 17:27:33.439 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:27:33.439 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:33.439 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:33.439 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:33.439 | INFO | ANALYSIS : 63.4%, 3.0, 98.8ms, 0 | 20260505 17:27:33.439 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:33.439 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260505 17:27:33.439 | INFO | PRE_HQ : 60.3%, 1.8, 56.4ms, 0 | 20260505 17:27:33.439 | INFO | BAZWRITER : 63.1%, 3.8, 124.9ms, 0 | 20260505 17:27:33.439 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 17:27:33.439 | INFO | Uploaded 36GiB to GPU in 2.14772s (16.762GiB/s) | 20260505 17:27:33.439 | INFO | Downloaded 38GiB to GPU in 2.25061s (16.8843GiB/s) | 20260505 17:27:38.576 | INFO | Analyzing chunk frames = [4997632,4998144) | 20260505 17:27:38.576 | INFO | MemInfo: { | 20260505 17:27:38.576 | INFO | "gpu" : " 17 GB", | 20260505 17:27:38.576 | INFO | "huge" : "", | 20260505 17:27:38.576 | INFO | "huge_shared" : " 27 GB", | 20260505 17:27:38.576 | INFO | "malloc" : " 14 GB", | 20260505 17:27:38.576 | INFO | "pinned" : " 29 GB" | 20260505 17:27:38.576 | INFO | } | 20260505 17:27:38.576 | INFO | Perf report for the last 5.13746 seconds: | 20260505 17:27:38.576 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:27:38.576 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:38.576 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:38.576 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:38.576 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 17:27:38.576 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:38.576 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:38.576 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:27:38.576 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:38.576 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:38.576 | INFO | Uploaded 36GiB to GPU in 2.13605s (16.8535GiB/s) | 20260505 17:27:38.576 | INFO | Downloaded 34GiB to GPU in 2.03204s (16.7319GiB/s) | 20260505 17:27:43.707 | INFO | Analyzing chunk frames = [4998144,4998656) | 20260505 17:27:43.707 | INFO | MemInfo: { | 20260505 17:27:43.707 | INFO | "gpu" : " 17 GB", | 20260505 17:27:43.707 | INFO | "huge" : "", | 20260505 17:27:43.707 | INFO | "huge_shared" : " 27 GB", | 20260505 17:27:43.707 | INFO | "malloc" : " 14 GB", | 20260505 17:27:43.707 | INFO | "pinned" : " 29 GB" | 20260505 17:27:43.707 | INFO | } | 20260505 17:27:43.707 | INFO | Perf report for the last 5.13129 seconds: | 20260505 17:27:43.707 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:27:43.707 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:43.707 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:43.707 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:43.707 | INFO | ANALYSIS : 62.8%, 3.0, 97.6ms, 0 | 20260505 17:27:43.707 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:43.707 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:43.707 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:27:43.707 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:43.707 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:43.708 | INFO | Uploaded 36GiB to GPU in 2.20197s (16.349GiB/s) | 20260505 17:27:43.708 | INFO | Downloaded 34GiB to GPU in 2.04056s (16.6621GiB/s) | 20260505 17:27:48.848 | INFO | Analyzing chunk frames = [4998656,4999168) | 20260505 17:27:48.848 | INFO | MemInfo: { | 20260505 17:27:48.848 | INFO | "gpu" : " 17 GB", | 20260505 17:27:48.848 | INFO | "huge" : "", | 20260505 17:27:48.848 | INFO | "huge_shared" : " 27 GB", | 20260505 17:27:48.848 | INFO | "malloc" : " 14 GB", | 20260505 17:27:48.848 | INFO | "pinned" : " 29 GB" | 20260505 17:27:48.848 | INFO | } | 20260505 17:27:48.848 | INFO | Perf report for the last 5.14063 seconds: | 20260505 17:27:48.848 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:27:48.848 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:48.848 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:48.848 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:48.848 | INFO | ANALYSIS : 62.5%, 3.0, 97.1ms, 0 | 20260505 17:27:48.848 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:48.848 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:48.848 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:27:48.848 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:48.848 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:48.848 | INFO | Uploaded 36GiB to GPU in 2.16531s (16.6258GiB/s) | 20260505 17:27:48.848 | INFO | Downloaded 34GiB to GPU in 2.14533s (15.8484GiB/s) | 20260505 17:27:53.996 | INFO | Analyzing chunk frames = [4999168,4999680) | 20260505 17:27:53.996 | INFO | MemInfo: { | 20260505 17:27:53.996 | INFO | "gpu" : " 17 GB", | 20260505 17:27:53.996 | INFO | "huge" : "", | 20260505 17:27:53.996 | INFO | "huge_shared" : " 27 GB", | 20260505 17:27:53.996 | INFO | "malloc" : " 14 GB", | 20260505 17:27:53.996 | INFO | "pinned" : " 29 GB" | 20260505 17:27:53.997 | INFO | } | 20260505 17:27:53.997 | INFO | Perf report for the last 5.14875 seconds: | 20260505 17:27:53.997 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:27:53.997 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:53.997 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:53.997 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:53.997 | INFO | ANALYSIS : 62.1%, 3.0, 97.1ms, 0 | 20260505 17:27:53.997 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:53.997 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:53.997 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 17:27:53.997 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:53.997 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:53.997 | INFO | Uploaded 36GiB to GPU in 2.14788s (16.7607GiB/s) | 20260505 17:27:53.997 | INFO | Downloaded 34GiB to GPU in 2.05605s (16.5365GiB/s) | 20260505 17:27:59.131 | INFO | Analyzing chunk frames = [4999680,5000192) | 20260505 17:27:59.131 | INFO | MemInfo: { | 20260505 17:27:59.131 | INFO | "gpu" : " 17 GB", | 20260505 17:27:59.131 | INFO | "huge" : "", | 20260505 17:27:59.131 | INFO | "huge_shared" : " 27 GB", | 20260505 17:27:59.131 | INFO | "malloc" : " 14 GB", | 20260505 17:27:59.131 | INFO | "pinned" : " 29 GB" | 20260505 17:27:59.131 | INFO | } | 20260505 17:27:59.131 | INFO | Perf report for the last 5.13436 seconds: | 20260505 17:27:59.131 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:27:59.131 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:59.131 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:59.131 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:59.131 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260505 17:27:59.131 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:59.131 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:59.131 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 17:27:59.131 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:59.131 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:27:59.131 | INFO | Uploaded 36GiB to GPU in 2.13387s (16.8708GiB/s) | 20260505 17:27:59.131 | INFO | Downloaded 34GiB to GPU in 2.04846s (16.5978GiB/s) | 20260505 17:28:04.271 | INFO | Analyzing chunk frames = [5000192,5000704) | 20260505 17:28:04.271 | INFO | MemInfo: { | 20260505 17:28:04.271 | INFO | "gpu" : " 17 GB", | 20260505 17:28:04.271 | INFO | "huge" : "", | 20260505 17:28:04.271 | INFO | "huge_shared" : " 27 GB", | 20260505 17:28:04.271 | INFO | "malloc" : " 14 GB", | 20260505 17:28:04.271 | INFO | "pinned" : " 29 GB" | 20260505 17:28:04.271 | INFO | } | 20260505 17:28:04.271 | INFO | Perf report for the last 5.14032 seconds: | 20260505 17:28:04.271 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:28:04.271 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:04.271 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:04.271 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:04.271 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 17:28:04.271 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:04.271 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:04.271 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 17:28:04.271 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:04.271 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:04.272 | INFO | Uploaded 36GiB to GPU in 2.1312s (16.8919GiB/s) | 20260505 17:28:04.272 | INFO | Downloaded 34GiB to GPU in 2.1646s (15.7073GiB/s) | 20260505 17:28:09.408 | INFO | Analyzing chunk frames = [5000704,5001216) | 20260505 17:28:09.408 | INFO | MemInfo: { | 20260505 17:28:09.408 | INFO | "gpu" : " 17 GB", | 20260505 17:28:09.408 | INFO | "huge" : "", | 20260505 17:28:09.408 | INFO | "huge_shared" : " 27 GB", | 20260505 17:28:09.408 | INFO | "malloc" : " 14 GB", | 20260505 17:28:09.408 | INFO | "pinned" : " 29 GB" | 20260505 17:28:09.408 | INFO | } | 20260505 17:28:09.408 | INFO | Perf report for the last 5.13647 seconds: | 20260505 17:28:09.408 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:28:09.408 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:09.408 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:09.408 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:09.408 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 17:28:09.408 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:09.408 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:09.408 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:28:09.408 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:09.408 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:09.408 | INFO | Uploaded 36GiB to GPU in 2.18031s (16.5114GiB/s) | 20260505 17:28:09.408 | INFO | Downloaded 34GiB to GPU in 2.02504s (16.7898GiB/s) | 20260505 17:28:09.542 | INFO | Starting a round of Baz IO | 20260505 17:28:12.738 | INFO | Populating RTMetrics report | 20260505 17:28:12.738 | INFO | RTMetrics report populated | 20260505 17:28:12.738 | INFO | Opening tmp RTMetrics json file | 20260505 17:28:12.745 | INFO | Serializing RTMetrics to json | 20260505 17:28:12.746 | INFO | Serialization done, swapping tmp file now... | 20260505 17:28:12.747 | INFO | RTMetrics done | 20260505 17:28:12.835 | INFO | Finished a round of Baz IO: Wrote 2.129 GiB in 3.292 seconds | 20260505 17:28:12.835 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.06% / 0.00% / 77.94% | 20260505 17:28:12.835 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.298119% / 0% / 0.000103315% | 20260505 17:28:12.835 | INFO | Baz IO overall: | 20260505 17:28:12.835 | INFO | BytesWritten : 2956763014366 | 20260505 17:28:12.835 | INFO | NumEvents : 1402067809828 | 20260505 17:28:14.552 | INFO | Analyzing chunk frames = [5001216,5001728) | 20260505 17:28:14.552 | INFO | MemInfo: { | 20260505 17:28:14.552 | INFO | "gpu" : " 17 GB", | 20260505 17:28:14.552 | INFO | "huge" : "", | 20260505 17:28:14.552 | INFO | "huge_shared" : " 27 GB", | 20260505 17:28:14.552 | INFO | "malloc" : " 14 GB", | 20260505 17:28:14.552 | INFO | "pinned" : " 29 GB" | 20260505 17:28:14.552 | INFO | } | 20260505 17:28:14.552 | INFO | Perf report for the last 5.14396 seconds: | 20260505 17:28:14.552 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:28:14.552 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:14.552 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:14.552 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:14.552 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 17:28:14.552 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:14.552 | INFO | RT_METRICS : 44.5%, 1.0, 23.4ms, 0 | 20260505 17:28:14.552 | INFO | PRE_HQ : 60.5%, 1.8, 56.2ms, 0 | 20260505 17:28:14.552 | INFO | BAZWRITER : 63.4%, 3.8, 124.5ms, 0 | 20260505 17:28:14.552 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 17:28:14.552 | INFO | Uploaded 36GiB to GPU in 2.1588s (16.676GiB/s) | 20260505 17:28:14.552 | INFO | Downloaded 38GiB to GPU in 2.25915s (16.8205GiB/s) | 20260505 17:28:19.688 | INFO | Analyzing chunk frames = [5001728,5002240) | 20260505 17:28:19.688 | INFO | MemInfo: { | 20260505 17:28:19.688 | INFO | "gpu" : " 17 GB", | 20260505 17:28:19.688 | INFO | "huge" : "", | 20260505 17:28:19.688 | INFO | "huge_shared" : " 27 GB", | 20260505 17:28:19.688 | INFO | "malloc" : " 14 GB", | 20260505 17:28:19.688 | INFO | "pinned" : " 29 GB" | 20260505 17:28:19.688 | INFO | } | 20260505 17:28:19.688 | INFO | Perf report for the last 5.13613 seconds: | 20260505 17:28:19.688 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:28:19.688 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:19.688 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:19.688 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:19.688 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 17:28:19.688 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:19.688 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:19.688 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 17:28:19.688 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:19.688 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:19.688 | INFO | Uploaded 36GiB to GPU in 2.12654s (16.9289GiB/s) | 20260505 17:28:19.688 | INFO | Downloaded 34GiB to GPU in 2.0241s (16.7976GiB/s) | 20260505 17:28:24.823 | INFO | Analyzing chunk frames = [5002240,5002752) | 20260505 17:28:24.823 | INFO | MemInfo: { | 20260505 17:28:24.823 | INFO | "gpu" : " 17 GB", | 20260505 17:28:24.823 | INFO | "huge" : "", | 20260505 17:28:24.823 | INFO | "huge_shared" : " 27 GB", | 20260505 17:28:24.823 | INFO | "malloc" : " 14 GB", | 20260505 17:28:24.823 | INFO | "pinned" : " 29 GB" | 20260505 17:28:24.824 | INFO | } | 20260505 17:28:24.824 | INFO | Perf report for the last 5.13557 seconds: | 20260505 17:28:24.824 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:28:24.824 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:24.824 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:24.824 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:24.824 | INFO | ANALYSIS : 62.7%, 3.0, 97.5ms, 0 | 20260505 17:28:24.824 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:24.824 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:24.824 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 17:28:24.824 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:24.824 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:24.824 | INFO | Uploaded 36GiB to GPU in 2.19563s (16.3962GiB/s) | 20260505 17:28:24.824 | INFO | Downloaded 34GiB to GPU in 2.02944s (16.7534GiB/s) | 20260505 17:28:29.961 | INFO | Analyzing chunk frames = [5002752,5003264) | 20260505 17:28:29.961 | INFO | MemInfo: { | 20260505 17:28:29.961 | INFO | "gpu" : " 17 GB", | 20260505 17:28:29.961 | INFO | "huge" : "", | 20260505 17:28:29.961 | INFO | "huge_shared" : " 27 GB", | 20260505 17:28:29.961 | INFO | "malloc" : " 14 GB", | 20260505 17:28:29.961 | INFO | "pinned" : " 29 GB" | 20260505 17:28:29.961 | INFO | } | 20260505 17:28:29.961 | INFO | Perf report for the last 5.13787 seconds: | 20260505 17:28:29.961 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:28:29.961 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:29.961 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:29.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:29.962 | INFO | ANALYSIS : 62.5%, 3.0, 97.1ms, 0 | 20260505 17:28:29.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:29.962 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:29.962 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:28:29.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:29.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:29.962 | INFO | Uploaded 36GiB to GPU in 2.19012s (16.4374GiB/s) | 20260505 17:28:29.962 | INFO | Downloaded 34GiB to GPU in 2.1438s (15.8597GiB/s) | 20260505 17:28:35.099 | INFO | Analyzing chunk frames = [5003264,5003776) | 20260505 17:28:35.099 | INFO | MemInfo: { | 20260505 17:28:35.099 | INFO | "gpu" : " 17 GB", | 20260505 17:28:35.099 | INFO | "huge" : "", | 20260505 17:28:35.099 | INFO | "huge_shared" : " 27 GB", | 20260505 17:28:35.099 | INFO | "malloc" : " 14 GB", | 20260505 17:28:35.099 | INFO | "pinned" : " 29 GB" | 20260505 17:28:35.099 | INFO | } | 20260505 17:28:35.099 | INFO | Perf report for the last 5.13753 seconds: | 20260505 17:28:35.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:28:35.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:35.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:35.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:35.099 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 17:28:35.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:35.099 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:35.099 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:28:35.099 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:35.099 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:35.099 | INFO | Uploaded 36GiB to GPU in 2.16127s (16.6568GiB/s) | 20260505 17:28:35.099 | INFO | Downloaded 34GiB to GPU in 2.05857s (16.5163GiB/s) | 20260505 17:28:40.248 | INFO | Analyzing chunk frames = [5003776,5004288) | 20260505 17:28:40.248 | INFO | MemInfo: { | 20260505 17:28:40.248 | INFO | "gpu" : " 17 GB", | 20260505 17:28:40.248 | INFO | "huge" : "", | 20260505 17:28:40.248 | INFO | "huge_shared" : " 27 GB", | 20260505 17:28:40.248 | INFO | "malloc" : " 14 GB", | 20260505 17:28:40.248 | INFO | "pinned" : " 29 GB" | 20260505 17:28:40.248 | INFO | } | 20260505 17:28:40.248 | INFO | Perf report for the last 5.14935 seconds: | 20260505 17:28:40.249 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:28:40.249 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:40.249 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:40.249 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:40.249 | INFO | ANALYSIS : 62.2%, 3.0, 97.4ms, 0 | 20260505 17:28:40.249 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:40.249 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:40.249 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 17:28:40.249 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:40.249 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:40.249 | INFO | Uploaded 36GiB to GPU in 2.13701s (16.8459GiB/s) | 20260505 17:28:40.249 | INFO | Downloaded 34GiB to GPU in 2.04655s (16.6133GiB/s) | 20260505 17:28:45.391 | INFO | Analyzing chunk frames = [5004288,5004800) | 20260505 17:28:45.391 | INFO | MemInfo: { | 20260505 17:28:45.391 | INFO | "gpu" : " 17 GB", | 20260505 17:28:45.391 | INFO | "huge" : "", | 20260505 17:28:45.391 | INFO | "huge_shared" : " 27 GB", | 20260505 17:28:45.391 | INFO | "malloc" : " 14 GB", | 20260505 17:28:45.391 | INFO | "pinned" : " 29 GB" | 20260505 17:28:45.391 | INFO | } | 20260505 17:28:45.392 | INFO | Perf report for the last 5.14318 seconds: | 20260505 17:28:45.392 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:28:45.392 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:45.392 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:45.392 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:45.392 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 17:28:45.392 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:45.392 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:45.392 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 17:28:45.392 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:45.392 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:45.392 | INFO | Uploaded 36GiB to GPU in 2.1556s (16.7007GiB/s) | 20260505 17:28:45.392 | INFO | Downloaded 34GiB to GPU in 2.15943s (15.7449GiB/s) | 20260505 17:28:50.518 | INFO | Analyzing chunk frames = [5004800,5005312) | 20260505 17:28:50.518 | INFO | MemInfo: { | 20260505 17:28:50.518 | INFO | "gpu" : " 17 GB", | 20260505 17:28:50.518 | INFO | "huge" : "", | 20260505 17:28:50.518 | INFO | "huge_shared" : " 27 GB", | 20260505 17:28:50.518 | INFO | "malloc" : " 14 GB", | 20260505 17:28:50.518 | INFO | "pinned" : " 29 GB" | 20260505 17:28:50.518 | INFO | } | 20260505 17:28:50.518 | INFO | Perf report for the last 5.12648 seconds: | 20260505 17:28:50.518 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:28:50.518 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:50.518 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:50.518 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:50.518 | INFO | ANALYSIS : 63.0%, 3.0, 98.1ms, 0 | 20260505 17:28:50.518 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:50.518 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:50.518 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 17:28:50.518 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:50.518 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:50.518 | INFO | Uploaded 36GiB to GPU in 2.19473s (16.4029GiB/s) | 20260505 17:28:50.518 | INFO | Downloaded 34GiB to GPU in 2.01886s (16.8412GiB/s) | 20260505 17:28:50.518 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 17:28:50.518 | INFO | Total mem usage: 15.3 GiB | 20260505 17:28:50.518 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 17:28:50.518 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.99%) | 20260505 17:28:50.518 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 17:28:50.518 | INFO | High Water Breakdown: | 20260505 17:28:50.518 | INFO | 61% : 8.66 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 17:28:50.518 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 17:28:50.519 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 17:28:50.519 | INFO | ------------------------------------------ | 20260505 17:28:50.519 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:28:50.519 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 17:28:50.519 | INFO | Total mem usage: 26.5 GiB | 20260505 17:28:50.519 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 17:28:50.519 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 17:28:50.519 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 17:28:50.519 | INFO | High Water Breakdown: | 20260505 17:28:50.519 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:28:50.519 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:28:50.519 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:28:50.519 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:28:50.519 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:28:50.519 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:28:50.519 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 17:28:50.519 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 17:28:50.519 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:28:50.519 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 17:28:50.519 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 17:28:50.519 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 17:28:50.519 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 17:28:50.519 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 17:28:50.519 | INFO | ------------------------------------------ | 20260505 17:28:50.519 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:28:50.519 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 17:28:50.519 | INFO | Total mem usage: 25.6 GiB | 20260505 17:28:50.519 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 17:28:50.519 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 17:28:50.519 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 17:28:50.519 | INFO | High Water Breakdown: | 20260505 17:28:50.519 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 17:28:50.519 | INFO | ------------------------------------------ | 20260505 17:28:50.519 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:28:50.519 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 17:28:50.519 | INFO | Total mem usage: 15.7 GiB | 20260505 17:28:50.519 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 17:28:50.519 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 17:28:50.519 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 17:28:50.519 | INFO | High Water Breakdown: | 20260505 17:28:50.519 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 17:28:50.519 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 17:28:50.519 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 17:28:50.519 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 17:28:50.519 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 17:28:50.519 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 17:28:50.519 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 17:28:50.519 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 17:28:50.519 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 17:28:50.519 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:28:50.519 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 17:28:50.519 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:28:50.519 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 17:28:50.519 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 17:28:50.519 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 17:28:50.519 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 17:28:50.519 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:28:50.519 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 17:28:50.519 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:28:50.519 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:28:50.519 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:28:50.519 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 17:28:50.519 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 17:28:50.519 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:28:50.519 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 17:28:50.519 | INFO | ------------------------------------------ | 20260505 17:28:50.519 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:28:50.519 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 17:28:50.519 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:28:50.519 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:28:50.519 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.49e+01, 1.36e+01 / 2.19e+00 | 20260505 17:28:50.519 | INFO | PerfReport: 0.29%, AggregateStats -- 9.30e-02 / 1.03e-01, 1.16e-02 / 5.25e-03 | 20260505 17:28:50.519 | INFO | PerfReport: 11.81%, Baseline -- 3.73e+00 / 4.52e+00, 5.77e-01 / 1.42e-01 | 20260505 17:28:50.519 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.54e+00, 1.94e-01 / 7.86e-02 | 20260505 17:28:50.519 | INFO | PerfReport: 15.96%, DME -- 4.03e+01 / 4.48e+01, 4.23e+00 / 2.16e+00 | 20260505 17:28:50.519 | INFO | PerfReport: 56.73%, FrameLabeling -- 1.79e+01 / 1.92e+01, 1.43e+00 / 1.16e+00 | 20260505 17:28:50.520 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.33e+00 / 2.92e+00, 4.50e-01 / 1.60e-01 | 20260505 17:28:50.520 | INFO | PerfReport: 3.37%, Metrics -- 1.07e+00 / 1.41e+00, 2.93e-01 / 1.39e-01 | 20260505 17:28:50.520 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.70e-01 / 4.29e-01, 6.57e-02 / 3.34e-02 | 20260505 17:28:50.520 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 17:28:50.520 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:28:50.520 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:28:50.520 | INFO | PerfReport: Full Execution -- 9.99e+01 / -nan, 2.56e+01 / 0.00e+00 | 20260505 17:28:50.520 | INFO | PerfReport: 22.53%, Upload -- 2.25e+01 / 0.00e+00, 7.81e+00 / 0.00e+00 | 20260505 17:28:50.520 | INFO | PerfReport: 21.84%, Download -- 2.18e+01 / 0.00e+00, 7.85e+00 / 0.00e+00 | 20260505 17:28:50.520 | INFO | PerfReport: 31.63%, Compute -- 3.16e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 17:28:50.520 | INFO | PerfReport: 2.21%, LockUpload -- 2.20e+00 / 0.00e+00, 7.89e+00 / 0.00e+00 | 20260505 17:28:50.520 | INFO | PerfReport: 1.48%, LockDownload -- 1.48e+00 / 0.00e+00, 3.03e+00 / 0.00e+00 | 20260505 17:28:50.520 | INFO | PerfReport: 20.31%, LockCompute -- 2.03e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 17:28:50.520 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 2.19e-02 / 0.00e+00 | 20260505 17:28:50.650 | INFO | Starting a round of Baz IO | 20260505 17:28:53.863 | INFO | Populating RTMetrics report | 20260505 17:28:53.864 | INFO | RTMetrics report populated | 20260505 17:28:53.864 | INFO | Opening tmp RTMetrics json file | 20260505 17:28:53.867 | INFO | Serializing RTMetrics to json | 20260505 17:28:53.868 | INFO | Serialization done, swapping tmp file now... | 20260505 17:28:53.868 | INFO | RTMetrics done | 20260505 17:28:53.957 | INFO | Finished a round of Baz IO: Wrote 2.125 GiB in 3.307 seconds | 20260505 17:28:53.957 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.06% / 0.00% / 77.94% | 20260505 17:28:53.957 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.297181% / 0% / 9.53674e-05% | 20260505 17:28:53.957 | INFO | Baz IO overall: | 20260505 17:28:53.958 | INFO | BytesWritten : 2959044895812 | 20260505 17:28:53.958 | INFO | NumEvents : 1403133628313 | 20260505 17:28:55.654 | INFO | Analyzing chunk frames = [5005312,5005824) | 20260505 17:28:55.654 | INFO | MemInfo: { | 20260505 17:28:55.654 | INFO | "gpu" : " 17 GB", | 20260505 17:28:55.654 | INFO | "huge" : "", | 20260505 17:28:55.654 | INFO | "huge_shared" : " 27 GB", | 20260505 17:28:55.654 | INFO | "malloc" : " 14 GB", | 20260505 17:28:55.654 | INFO | "pinned" : " 29 GB" | 20260505 17:28:55.654 | INFO | } | 20260505 17:28:55.654 | INFO | Perf report for the last 5.13617 seconds: | 20260505 17:28:55.654 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:28:55.654 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:55.654 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:55.654 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:55.654 | INFO | ANALYSIS : 64.1%, 3.0, 99.8ms, 0 | 20260505 17:28:55.654 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:28:55.654 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260505 17:28:55.654 | INFO | PRE_HQ : 60.6%, 1.8, 55.9ms, 0 | 20260505 17:28:55.654 | INFO | BAZWRITER : 63.7%, 3.8, 124.3ms, 0 | 20260505 17:28:55.654 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 17:28:55.655 | INFO | Uploaded 36GiB to GPU in 2.19555s (16.3968GiB/s) | 20260505 17:28:55.655 | INFO | Downloaded 38GiB to GPU in 2.25657s (16.8397GiB/s) | 20260505 17:29:00.803 | INFO | Analyzing chunk frames = [5005824,5006336) | 20260505 17:29:00.803 | INFO | MemInfo: { | 20260505 17:29:00.803 | INFO | "gpu" : " 17 GB", | 20260505 17:29:00.803 | INFO | "huge" : "", | 20260505 17:29:00.803 | INFO | "huge_shared" : " 27 GB", | 20260505 17:29:00.803 | INFO | "malloc" : " 14 GB", | 20260505 17:29:00.803 | INFO | "pinned" : " 29 GB" | 20260505 17:29:00.803 | INFO | } | 20260505 17:29:00.803 | INFO | Perf report for the last 5.14852 seconds: | 20260505 17:29:00.803 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:29:00.803 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:00.803 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:00.803 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:00.803 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 17:29:00.803 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:00.803 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:00.803 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 17:29:00.803 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:00.803 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:00.803 | INFO | Uploaded 36GiB to GPU in 2.14143s (16.8112GiB/s) | 20260505 17:29:00.803 | INFO | Downloaded 34GiB to GPU in 2.04004s (16.6663GiB/s) | 20260505 17:29:05.949 | INFO | Analyzing chunk frames = [5006336,5006848) | 20260505 17:29:05.949 | INFO | MemInfo: { | 20260505 17:29:05.949 | INFO | "gpu" : " 17 GB", | 20260505 17:29:05.949 | INFO | "huge" : "", | 20260505 17:29:05.949 | INFO | "huge_shared" : " 27 GB", | 20260505 17:29:05.949 | INFO | "malloc" : " 14 GB", | 20260505 17:29:05.949 | INFO | "pinned" : " 29 GB" | 20260505 17:29:05.949 | INFO | } | 20260505 17:29:05.949 | INFO | Perf report for the last 5.14647 seconds: | 20260505 17:29:05.949 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:29:05.949 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:05.949 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:05.949 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:05.949 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 17:29:05.949 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:05.949 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:05.950 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:29:05.950 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:05.950 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:05.950 | INFO | Uploaded 36GiB to GPU in 2.20494s (16.327GiB/s) | 20260505 17:29:05.950 | INFO | Downloaded 34GiB to GPU in 2.04008s (16.666GiB/s) | 20260505 17:29:11.080 | INFO | Analyzing chunk frames = [5006848,5007360) | 20260505 17:29:11.080 | INFO | MemInfo: { | 20260505 17:29:11.080 | INFO | "gpu" : " 17 GB", | 20260505 17:29:11.080 | INFO | "huge" : "", | 20260505 17:29:11.080 | INFO | "huge_shared" : " 27 GB", | 20260505 17:29:11.080 | INFO | "malloc" : " 14 GB", | 20260505 17:29:11.080 | INFO | "pinned" : " 29 GB" | 20260505 17:29:11.080 | INFO | } | 20260505 17:29:11.080 | INFO | Perf report for the last 5.1308 seconds: | 20260505 17:29:11.080 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:29:11.080 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:11.080 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:11.080 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:11.080 | INFO | ANALYSIS : 62.6%, 3.0, 97.0ms, 0 | 20260505 17:29:11.080 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:11.080 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:11.080 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:29:11.080 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:11.080 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:11.080 | INFO | Uploaded 36GiB to GPU in 2.17776s (16.5307GiB/s) | 20260505 17:29:11.080 | INFO | Downloaded 34GiB to GPU in 2.06277s (16.4827GiB/s) | 20260505 17:29:16.215 | INFO | Analyzing chunk frames = [5007360,5007872) | 20260505 17:29:16.215 | INFO | MemInfo: { | 20260505 17:29:16.215 | INFO | "gpu" : " 17 GB", | 20260505 17:29:16.215 | INFO | "huge" : "", | 20260505 17:29:16.215 | INFO | "huge_shared" : " 27 GB", | 20260505 17:29:16.215 | INFO | "malloc" : " 14 GB", | 20260505 17:29:16.215 | INFO | "pinned" : " 29 GB" | 20260505 17:29:16.215 | INFO | } | 20260505 17:29:16.215 | INFO | Perf report for the last 5.13529 seconds: | 20260505 17:29:16.215 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:29:16.215 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:16.215 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:16.215 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:16.215 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 17:29:16.215 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:16.215 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:16.215 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:29:16.215 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:16.215 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:16.216 | INFO | Uploaded 36GiB to GPU in 2.14522s (16.7815GiB/s) | 20260505 17:29:16.216 | INFO | Downloaded 34GiB to GPU in 2.05803s (16.5207GiB/s) | 20260505 17:29:21.361 | INFO | Analyzing chunk frames = [5007872,5008384) | 20260505 17:29:21.361 | INFO | MemInfo: { | 20260505 17:29:21.361 | INFO | "gpu" : " 17 GB", | 20260505 17:29:21.361 | INFO | "huge" : "", | 20260505 17:29:21.361 | INFO | "huge_shared" : " 27 GB", | 20260505 17:29:21.361 | INFO | "malloc" : " 14 GB", | 20260505 17:29:21.361 | INFO | "pinned" : " 29 GB" | 20260505 17:29:21.361 | INFO | } | 20260505 17:29:21.361 | INFO | Perf report for the last 5.14594 seconds: | 20260505 17:29:21.361 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:29:21.361 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:21.361 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:21.361 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:21.361 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260505 17:29:21.361 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:21.361 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:21.361 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 17:29:21.361 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:21.361 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:21.362 | INFO | Uploaded 36GiB to GPU in 2.14232s (16.8042GiB/s) | 20260505 17:29:21.362 | INFO | Downloaded 34GiB to GPU in 2.03523s (16.7058GiB/s) | 20260505 17:29:26.500 | INFO | Analyzing chunk frames = [5008384,5008896) | 20260505 17:29:26.500 | INFO | MemInfo: { | 20260505 17:29:26.500 | INFO | "gpu" : " 17 GB", | 20260505 17:29:26.500 | INFO | "huge" : "", | 20260505 17:29:26.500 | INFO | "huge_shared" : " 27 GB", | 20260505 17:29:26.500 | INFO | "malloc" : " 14 GB", | 20260505 17:29:26.500 | INFO | "pinned" : " 29 GB" | 20260505 17:29:26.500 | INFO | } | 20260505 17:29:26.500 | INFO | Perf report for the last 5.13901 seconds: | 20260505 17:29:26.500 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:29:26.500 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:26.500 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:26.500 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:26.500 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 17:29:26.500 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:26.500 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:26.500 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 17:29:26.500 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:26.500 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:26.501 | INFO | Uploaded 36GiB to GPU in 2.14689s (16.7685GiB/s) | 20260505 17:29:26.501 | INFO | Downloaded 34GiB to GPU in 2.17275s (15.6484GiB/s) | 20260505 17:29:31.638 | INFO | Analyzing chunk frames = [5008896,5009408) | 20260505 17:29:31.638 | INFO | MemInfo: { | 20260505 17:29:31.638 | INFO | "gpu" : " 17 GB", | 20260505 17:29:31.638 | INFO | "huge" : "", | 20260505 17:29:31.638 | INFO | "huge_shared" : " 27 GB", | 20260505 17:29:31.638 | INFO | "malloc" : " 14 GB", | 20260505 17:29:31.638 | INFO | "pinned" : " 29 GB" | 20260505 17:29:31.638 | INFO | } | 20260505 17:29:31.638 | INFO | Perf report for the last 5.13725 seconds: | 20260505 17:29:31.638 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:29:31.638 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:31.638 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:31.638 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:31.638 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 17:29:31.638 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:31.638 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:31.638 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 17:29:31.638 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:31.638 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:31.638 | INFO | Uploaded 36GiB to GPU in 2.18027s (16.5118GiB/s) | 20260505 17:29:31.638 | INFO | Downloaded 34GiB to GPU in 2.02156s (16.8187GiB/s) | 20260505 17:29:31.771 | INFO | Starting a round of Baz IO | 20260505 17:29:34.970 | INFO | Populating RTMetrics report | 20260505 17:29:34.970 | INFO | RTMetrics report populated | 20260505 17:29:34.970 | INFO | Opening tmp RTMetrics json file | 20260505 17:29:34.977 | INFO | Serializing RTMetrics to json | 20260505 17:29:34.978 | INFO | Serialization done, swapping tmp file now... | 20260505 17:29:34.978 | INFO | RTMetrics done | 20260505 17:29:35.069 | INFO | Finished a round of Baz IO: Wrote 2.128 GiB in 3.297 seconds | 20260505 17:29:35.069 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.05% / 0.00% / 77.94% | 20260505 17:29:35.069 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.297912% / 3.97364e-06% / 0.000103315% | 20260505 17:29:35.069 | INFO | Baz IO overall: | 20260505 17:29:35.069 | INFO | BytesWritten : 2961329910312 | 20260505 17:29:35.069 | INFO | NumEvents : 1404201715336 | 20260505 17:29:36.775 | INFO | Analyzing chunk frames = [5009408,5009920) | 20260505 17:29:36.775 | INFO | MemInfo: { | 20260505 17:29:36.775 | INFO | "gpu" : " 17 GB", | 20260505 17:29:36.775 | INFO | "huge" : "", | 20260505 17:29:36.775 | INFO | "huge_shared" : " 27 GB", | 20260505 17:29:36.775 | INFO | "malloc" : " 14 GB", | 20260505 17:29:36.775 | INFO | "pinned" : " 29 GB" | 20260505 17:29:36.775 | INFO | } | 20260505 17:29:36.775 | INFO | Perf report for the last 5.13785 seconds: | 20260505 17:29:36.775 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:29:36.775 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:36.776 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:36.776 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:36.776 | INFO | ANALYSIS : 63.8%, 3.0, 99.3ms, 0 | 20260505 17:29:36.776 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:36.776 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260505 17:29:36.776 | INFO | PRE_HQ : 60.5%, 1.8, 56.3ms, 0 | 20260505 17:29:36.776 | INFO | BAZWRITER : 63.5%, 3.8, 124.7ms, 0 | 20260505 17:29:36.776 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 17:29:36.776 | INFO | Uploaded 36GiB to GPU in 2.1735s (16.5632GiB/s) | 20260505 17:29:36.776 | INFO | Downloaded 38GiB to GPU in 2.25739s (16.8336GiB/s) | 20260505 17:29:41.915 | INFO | Analyzing chunk frames = [5009920,5010432) | 20260505 17:29:41.915 | INFO | MemInfo: { | 20260505 17:29:41.915 | INFO | "gpu" : " 17 GB", | 20260505 17:29:41.915 | INFO | "huge" : "", | 20260505 17:29:41.915 | INFO | "huge_shared" : " 27 GB", | 20260505 17:29:41.915 | INFO | "malloc" : " 14 GB", | 20260505 17:29:41.915 | INFO | "pinned" : " 29 GB" | 20260505 17:29:41.915 | INFO | } | 20260505 17:29:41.915 | INFO | Perf report for the last 5.13919 seconds: | 20260505 17:29:41.915 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:29:41.915 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:41.915 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:41.915 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:41.915 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 17:29:41.915 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:41.915 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:41.915 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:29:41.915 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:41.915 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:41.915 | INFO | Uploaded 36GiB to GPU in 2.13416s (16.8684GiB/s) | 20260505 17:29:41.915 | INFO | Downloaded 34GiB to GPU in 2.03316s (16.7227GiB/s) | 20260505 17:29:47.056 | INFO | Analyzing chunk frames = [5010432,5010944) | 20260505 17:29:47.056 | INFO | MemInfo: { | 20260505 17:29:47.056 | INFO | "gpu" : " 17 GB", | 20260505 17:29:47.056 | INFO | "huge" : "", | 20260505 17:29:47.056 | INFO | "huge_shared" : " 27 GB", | 20260505 17:29:47.056 | INFO | "malloc" : " 14 GB", | 20260505 17:29:47.056 | INFO | "pinned" : " 29 GB" | 20260505 17:29:47.056 | INFO | } | 20260505 17:29:47.056 | INFO | Perf report for the last 5.14177 seconds: | 20260505 17:29:47.056 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:29:47.056 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:47.056 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:47.056 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:47.056 | INFO | ANALYSIS : 62.6%, 3.0, 97.4ms, 0 | 20260505 17:29:47.057 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:47.057 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:47.057 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:29:47.057 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:47.057 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:47.057 | INFO | Uploaded 36GiB to GPU in 2.19598s (16.3936GiB/s) | 20260505 17:29:47.057 | INFO | Downloaded 34GiB to GPU in 2.03519s (16.706GiB/s) | 20260505 17:29:52.198 | INFO | Analyzing chunk frames = [5010944,5011456) | 20260505 17:29:52.198 | INFO | MemInfo: { | 20260505 17:29:52.198 | INFO | "gpu" : " 17 GB", | 20260505 17:29:52.198 | INFO | "huge" : "", | 20260505 17:29:52.198 | INFO | "huge_shared" : " 27 GB", | 20260505 17:29:52.198 | INFO | "malloc" : " 14 GB", | 20260505 17:29:52.198 | INFO | "pinned" : " 29 GB" | 20260505 17:29:52.198 | INFO | } | 20260505 17:29:52.198 | INFO | Perf report for the last 5.14163 seconds: | 20260505 17:29:52.198 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:29:52.198 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:52.198 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:52.198 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:52.198 | INFO | ANALYSIS : 62.8%, 3.0, 97.6ms, 0 | 20260505 17:29:52.198 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:52.198 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:52.198 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:29:52.198 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:52.198 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:52.198 | INFO | Uploaded 36GiB to GPU in 2.2278s (16.1594GiB/s) | 20260505 17:29:52.198 | INFO | Downloaded 34GiB to GPU in 2.11993s (16.0383GiB/s) | 20260505 17:29:57.336 | INFO | Analyzing chunk frames = [5011456,5011968) | 20260505 17:29:57.336 | INFO | MemInfo: { | 20260505 17:29:57.336 | INFO | "gpu" : " 17 GB", | 20260505 17:29:57.336 | INFO | "huge" : "", | 20260505 17:29:57.336 | INFO | "huge_shared" : " 27 GB", | 20260505 17:29:57.336 | INFO | "malloc" : " 14 GB", | 20260505 17:29:57.336 | INFO | "pinned" : " 29 GB" | 20260505 17:29:57.336 | INFO | } | 20260505 17:29:57.336 | INFO | Perf report for the last 5.13764 seconds: | 20260505 17:29:57.336 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:29:57.336 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:57.336 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:57.336 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:57.336 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260505 17:29:57.336 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:57.336 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:57.336 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:29:57.336 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:57.336 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:29:57.336 | INFO | Uploaded 36GiB to GPU in 2.14861s (16.755GiB/s) | 20260505 17:29:57.336 | INFO | Downloaded 34GiB to GPU in 2.05606s (16.5365GiB/s) | 20260505 17:30:02.465 | INFO | Analyzing chunk frames = [5011968,5012480) | 20260505 17:30:02.465 | INFO | MemInfo: { | 20260505 17:30:02.465 | INFO | "gpu" : " 17 GB", | 20260505 17:30:02.465 | INFO | "huge" : "", | 20260505 17:30:02.466 | INFO | "huge_shared" : " 27 GB", | 20260505 17:30:02.466 | INFO | "malloc" : " 14 GB", | 20260505 17:30:02.466 | INFO | "pinned" : " 29 GB" | 20260505 17:30:02.466 | INFO | } | 20260505 17:30:02.466 | INFO | Perf report for the last 5.13 seconds: | 20260505 17:30:02.466 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:30:02.466 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:02.466 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:02.466 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:02.466 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260505 17:30:02.466 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:02.466 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:02.466 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 17:30:02.466 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:02.466 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:02.466 | INFO | Uploaded 36GiB to GPU in 2.14207s (16.8062GiB/s) | 20260505 17:30:02.466 | INFO | Downloaded 34GiB to GPU in 2.03862s (16.678GiB/s) | 20260505 17:30:07.613 | INFO | Analyzing chunk frames = [5012480,5012992) | 20260505 17:30:07.613 | INFO | MemInfo: { | 20260505 17:30:07.613 | INFO | "gpu" : " 17 GB", | 20260505 17:30:07.613 | INFO | "huge" : "", | 20260505 17:30:07.613 | INFO | "huge_shared" : " 27 GB", | 20260505 17:30:07.613 | INFO | "malloc" : " 14 GB", | 20260505 17:30:07.613 | INFO | "pinned" : " 29 GB" | 20260505 17:30:07.613 | INFO | } | 20260505 17:30:07.613 | INFO | Perf report for the last 5.14748 seconds: | 20260505 17:30:07.613 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:30:07.613 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:07.613 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:07.613 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:07.613 | INFO | ANALYSIS : 63.3%, 3.0, 99.1ms, 0 | 20260505 17:30:07.613 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:07.613 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:07.613 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 17:30:07.613 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:07.613 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:07.614 | INFO | Uploaded 36GiB to GPU in 2.11315s (17.0362GiB/s) | 20260505 17:30:07.614 | INFO | Downloaded 34GiB to GPU in 2.1803s (15.5942GiB/s) | 20260505 17:30:12.755 | INFO | Analyzing chunk frames = [5012992,5013504) | 20260505 17:30:12.755 | INFO | MemInfo: { | 20260505 17:30:12.755 | INFO | "gpu" : " 17 GB", | 20260505 17:30:12.755 | INFO | "huge" : "", | 20260505 17:30:12.755 | INFO | "huge_shared" : " 27 GB", | 20260505 17:30:12.755 | INFO | "malloc" : " 14 GB", | 20260505 17:30:12.755 | INFO | "pinned" : " 29 GB" | 20260505 17:30:12.755 | INFO | } | 20260505 17:30:12.755 | INFO | Perf report for the last 5.14218 seconds: | 20260505 17:30:12.755 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:30:12.755 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:12.755 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:12.755 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:12.755 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 17:30:12.755 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:12.755 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:12.756 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 17:30:12.756 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:12.756 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:12.756 | INFO | Uploaded 36GiB to GPU in 2.20316s (16.3401GiB/s) | 20260505 17:30:12.756 | INFO | Downloaded 34GiB to GPU in 2.02329s (16.8043GiB/s) | 20260505 17:30:12.756 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 17:30:12.756 | INFO | Total mem usage: 15.3 GiB | 20260505 17:30:12.756 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 17:30:12.756 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.98%) | 20260505 17:30:12.756 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 17:30:12.756 | INFO | High Water Breakdown: | 20260505 17:30:12.756 | INFO | 61% : 8.66 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 17:30:12.756 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 17:30:12.756 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 17:30:12.756 | INFO | ------------------------------------------ | 20260505 17:30:12.756 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:30:12.756 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 17:30:12.756 | INFO | Total mem usage: 26.5 GiB | 20260505 17:30:12.756 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 17:30:12.756 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 17:30:12.756 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 17:30:12.756 | INFO | High Water Breakdown: | 20260505 17:30:12.756 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:30:12.756 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:30:12.756 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:30:12.756 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:30:12.756 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:30:12.756 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:30:12.756 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 17:30:12.756 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 17:30:12.756 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:30:12.756 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 17:30:12.756 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 17:30:12.756 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 17:30:12.756 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 17:30:12.756 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 17:30:12.756 | INFO | ------------------------------------------ | 20260505 17:30:12.756 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:30:12.756 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 17:30:12.756 | INFO | Total mem usage: 25.6 GiB | 20260505 17:30:12.756 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 17:30:12.756 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 17:30:12.756 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 17:30:12.756 | INFO | High Water Breakdown: | 20260505 17:30:12.756 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 17:30:12.756 | INFO | ------------------------------------------ | 20260505 17:30:12.756 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:30:12.756 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 17:30:12.756 | INFO | Total mem usage: 15.7 GiB | 20260505 17:30:12.756 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 17:30:12.756 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 17:30:12.756 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 17:30:12.756 | INFO | High Water Breakdown: | 20260505 17:30:12.756 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 17:30:12.756 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 17:30:12.756 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 17:30:12.756 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 17:30:12.756 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 17:30:12.756 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 17:30:12.756 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 17:30:12.756 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 17:30:12.756 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 17:30:12.756 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:30:12.756 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 17:30:12.756 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:30:12.756 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 17:30:12.756 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 17:30:12.756 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 17:30:12.756 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 17:30:12.756 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:30:12.757 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 17:30:12.757 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:30:12.757 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:30:12.757 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:30:12.757 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 17:30:12.757 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 17:30:12.757 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:30:12.757 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 17:30:12.757 | INFO | ------------------------------------------ | 20260505 17:30:12.757 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:30:12.757 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 17:30:12.757 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:30:12.757 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:30:12.757 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.49e+01, 1.35e+01 / 2.30e+00 | 20260505 17:30:12.757 | INFO | PerfReport: 0.29%, AggregateStats -- 9.30e-02 / 1.04e-01, 1.10e-02 / 4.48e-03 | 20260505 17:30:12.757 | INFO | PerfReport: 11.79%, Baseline -- 3.73e+00 / 4.48e+00, 5.72e-01 / 1.87e-01 | 20260505 17:30:12.757 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.51e+00, 1.91e-01 / 9.37e-02 | 20260505 17:30:12.757 | INFO | PerfReport: 15.94%, DME -- 4.03e+01 / 4.46e+01, 4.19e+00 / 2.17e+00 | 20260505 17:30:12.757 | INFO | PerfReport: 56.84%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.50e+00 / 1.52e+00 | 20260505 17:30:12.757 | INFO | PerfReport: 7.33%, PulseAccumulating -- 2.32e+00 / 2.91e+00, 4.45e-01 / 1.95e-01 | 20260505 17:30:12.757 | INFO | PerfReport: 3.36%, Metrics -- 1.06e+00 / 1.42e+00, 2.92e-01 / 1.76e-01 | 20260505 17:30:12.757 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.71e-01 / 4.31e-01, 6.60e-02 / 3.58e-02 | 20260505 17:30:12.757 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 17:30:12.757 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:30:12.757 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:30:12.757 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.58e+01 / 0.00e+00 | 20260505 17:30:12.757 | INFO | PerfReport: 22.55%, Upload -- 2.26e+01 / 0.00e+00, 7.90e+00 / 0.00e+00 | 20260505 17:30:12.757 | INFO | PerfReport: 21.75%, Download -- 2.18e+01 / 0.00e+00, 7.77e+00 / 0.00e+00 | 20260505 17:30:12.757 | INFO | PerfReport: 31.59%, Compute -- 3.16e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 17:30:12.757 | INFO | PerfReport: 2.37%, LockUpload -- 2.37e+00 / 0.00e+00, 8.04e+00 / 0.00e+00 | 20260505 17:30:12.757 | INFO | PerfReport: 1.39%, LockDownload -- 1.39e+00 / 0.00e+00, 2.94e+00 / 0.00e+00 | 20260505 17:30:12.757 | INFO | PerfReport: 20.34%, LockCompute -- 2.04e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260505 17:30:12.757 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.77e-02 / 0.00e+00 | 20260505 17:30:12.890 | INFO | Starting a round of Baz IO | 20260505 17:30:16.068 | INFO | Populating RTMetrics report | 20260505 17:30:16.069 | INFO | RTMetrics report populated | 20260505 17:30:16.069 | INFO | Opening tmp RTMetrics json file | 20260505 17:30:16.076 | INFO | Serializing RTMetrics to json | 20260505 17:30:16.078 | INFO | Serialization done, swapping tmp file now... | 20260505 17:30:16.078 | INFO | RTMetrics done | 20260505 17:30:16.163 | INFO | Finished a round of Baz IO: Wrote 2.127 GiB in 3.272 seconds | 20260505 17:30:16.163 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.05% / 0.00% / 77.95% | 20260505 17:30:16.163 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.296215% / 7.94729e-06% / 0.000107288% | 20260505 17:30:16.163 | INFO | Baz IO overall: | 20260505 17:30:16.163 | INFO | BytesWritten : 2963613762571 | 20260505 17:30:16.163 | INFO | NumEvents : 1405268892361 | 20260505 17:30:17.901 | INFO | Analyzing chunk frames = [5013504,5014016) | 20260505 17:30:17.901 | INFO | MemInfo: { | 20260505 17:30:17.901 | INFO | "gpu" : " 17 GB", | 20260505 17:30:17.901 | INFO | "huge" : "", | 20260505 17:30:17.901 | INFO | "huge_shared" : " 27 GB", | 20260505 17:30:17.901 | INFO | "malloc" : " 14 GB", | 20260505 17:30:17.901 | INFO | "pinned" : " 29 GB" | 20260505 17:30:17.901 | INFO | } | 20260505 17:30:17.901 | INFO | Perf report for the last 5.14525 seconds: | 20260505 17:30:17.901 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:30:17.901 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:17.901 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:17.901 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:17.901 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 17:30:17.901 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:17.901 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260505 17:30:17.901 | INFO | PRE_HQ : 60.5%, 1.8, 56.2ms, 0 | 20260505 17:30:17.901 | INFO | BAZWRITER : 63.1%, 3.9, 125.4ms, 0 | 20260505 17:30:17.901 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260505 17:30:17.901 | INFO | Uploaded 36GiB to GPU in 2.17082s (16.5836GiB/s) | 20260505 17:30:17.901 | INFO | Downloaded 38GiB to GPU in 2.25425s (16.857GiB/s) | 20260505 17:30:23.034 | INFO | Analyzing chunk frames = [5014016,5014528) | 20260505 17:30:23.034 | INFO | MemInfo: { | 20260505 17:30:23.034 | INFO | "gpu" : " 17 GB", | 20260505 17:30:23.034 | INFO | "huge" : "", | 20260505 17:30:23.034 | INFO | "huge_shared" : " 27 GB", | 20260505 17:30:23.034 | INFO | "malloc" : " 14 GB", | 20260505 17:30:23.034 | INFO | "pinned" : " 29 GB" | 20260505 17:30:23.034 | INFO | } | 20260505 17:30:23.034 | INFO | Perf report for the last 5.13337 seconds: | 20260505 17:30:23.034 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:30:23.034 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:23.034 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:23.034 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:23.034 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260505 17:30:23.034 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:23.034 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:23.034 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:30:23.034 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:23.034 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:23.034 | INFO | Uploaded 36GiB to GPU in 2.1316s (16.8887GiB/s) | 20260505 17:30:23.034 | INFO | Downloaded 34GiB to GPU in 2.02054s (16.8272GiB/s) | 20260505 17:30:28.164 | INFO | Analyzing chunk frames = [5014528,5015040) | 20260505 17:30:28.164 | INFO | MemInfo: { | 20260505 17:30:28.164 | INFO | "gpu" : " 17 GB", | 20260505 17:30:28.164 | INFO | "huge" : "", | 20260505 17:30:28.164 | INFO | "huge_shared" : " 27 GB", | 20260505 17:30:28.164 | INFO | "malloc" : " 14 GB", | 20260505 17:30:28.164 | INFO | "pinned" : " 29 GB" | 20260505 17:30:28.164 | INFO | } | 20260505 17:30:28.164 | INFO | Perf report for the last 5.12969 seconds: | 20260505 17:30:28.164 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:30:28.164 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:28.164 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:28.164 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:28.164 | INFO | ANALYSIS : 62.7%, 3.0, 97.4ms, 0 | 20260505 17:30:28.164 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:28.164 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:28.164 | INFO | PRE_HQ : 27.7%, 1.0, 14.3ms, 0 | 20260505 17:30:28.164 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:28.164 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:28.164 | INFO | Uploaded 36GiB to GPU in 2.19905s (16.3707GiB/s) | 20260505 17:30:28.164 | INFO | Downloaded 34GiB to GPU in 2.03112s (16.7395GiB/s) | 20260505 17:30:33.308 | INFO | Analyzing chunk frames = [5015040,5015552) | 20260505 17:30:33.308 | INFO | MemInfo: { | 20260505 17:30:33.308 | INFO | "gpu" : " 17 GB", | 20260505 17:30:33.308 | INFO | "huge" : "", | 20260505 17:30:33.308 | INFO | "huge_shared" : " 27 GB", | 20260505 17:30:33.308 | INFO | "malloc" : " 14 GB", | 20260505 17:30:33.308 | INFO | "pinned" : " 29 GB" | 20260505 17:30:33.308 | INFO | } | 20260505 17:30:33.308 | INFO | Perf report for the last 5.14392 seconds: | 20260505 17:30:33.308 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:30:33.308 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:33.308 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:33.308 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:33.308 | INFO | ANALYSIS : 62.5%, 3.0, 97.2ms, 0 | 20260505 17:30:33.308 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:33.308 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:33.308 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260505 17:30:33.308 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:33.308 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:33.308 | INFO | Uploaded 36GiB to GPU in 2.19023s (16.4366GiB/s) | 20260505 17:30:33.308 | INFO | Downloaded 34GiB to GPU in 2.13682s (15.9115GiB/s) | 20260505 17:30:38.458 | INFO | Analyzing chunk frames = [5015552,5016064) | 20260505 17:30:38.458 | INFO | MemInfo: { | 20260505 17:30:38.458 | INFO | "gpu" : " 17 GB", | 20260505 17:30:38.458 | INFO | "huge" : "", | 20260505 17:30:38.458 | INFO | "huge_shared" : " 27 GB", | 20260505 17:30:38.458 | INFO | "malloc" : " 14 GB", | 20260505 17:30:38.458 | INFO | "pinned" : " 29 GB" | 20260505 17:30:38.458 | INFO | } | 20260505 17:30:38.458 | INFO | Perf report for the last 5.15052 seconds: | 20260505 17:30:38.458 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:30:38.458 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:38.458 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:38.458 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:38.458 | INFO | ANALYSIS : 62.0%, 3.0, 97.1ms, 0 | 20260505 17:30:38.458 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:38.458 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:38.458 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:30:38.458 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:38.458 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:38.459 | INFO | Uploaded 36GiB to GPU in 2.15331s (16.7185GiB/s) | 20260505 17:30:38.459 | INFO | Downloaded 34GiB to GPU in 2.06131s (16.4943GiB/s) | 20260505 17:30:43.587 | INFO | Analyzing chunk frames = [5016064,5016576) | 20260505 17:30:43.587 | INFO | MemInfo: { | 20260505 17:30:43.587 | INFO | "gpu" : " 17 GB", | 20260505 17:30:43.587 | INFO | "huge" : "", | 20260505 17:30:43.587 | INFO | "huge_shared" : " 27 GB", | 20260505 17:30:43.587 | INFO | "malloc" : " 14 GB", | 20260505 17:30:43.587 | INFO | "pinned" : " 29 GB" | 20260505 17:30:43.587 | INFO | } | 20260505 17:30:43.587 | INFO | Perf report for the last 5.12888 seconds: | 20260505 17:30:43.587 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:30:43.587 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:43.587 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:43.587 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:43.587 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260505 17:30:43.587 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:43.587 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:43.587 | INFO | PRE_HQ : 27.5%, 1.0, 14.2ms, 0 | 20260505 17:30:43.587 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:43.587 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:43.587 | INFO | Uploaded 36GiB to GPU in 2.13751s (16.842GiB/s) | 20260505 17:30:43.588 | INFO | Downloaded 34GiB to GPU in 2.0465s (16.6137GiB/s) | 20260505 17:30:48.723 | INFO | Analyzing chunk frames = [5016576,5017088) | 20260505 17:30:48.723 | INFO | MemInfo: { | 20260505 17:30:48.723 | INFO | "gpu" : " 17 GB", | 20260505 17:30:48.723 | INFO | "huge" : "", | 20260505 17:30:48.723 | INFO | "huge_shared" : " 27 GB", | 20260505 17:30:48.723 | INFO | "malloc" : " 14 GB", | 20260505 17:30:48.723 | INFO | "pinned" : " 29 GB" | 20260505 17:30:48.723 | INFO | } | 20260505 17:30:48.723 | INFO | Perf report for the last 5.13542 seconds: | 20260505 17:30:48.723 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:30:48.723 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:48.723 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:48.723 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:48.723 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 17:30:48.723 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:48.723 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:48.723 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 17:30:48.723 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:48.723 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:48.723 | INFO | Uploaded 36GiB to GPU in 2.16621s (16.6189GiB/s) | 20260505 17:30:48.723 | INFO | Downloaded 34GiB to GPU in 2.15615s (15.7689GiB/s) | 20260505 17:30:53.862 | INFO | Analyzing chunk frames = [5017088,5017600) | 20260505 17:30:53.862 | INFO | MemInfo: { | 20260505 17:30:53.862 | INFO | "gpu" : " 17 GB", | 20260505 17:30:53.862 | INFO | "huge" : "", | 20260505 17:30:53.862 | INFO | "huge_shared" : " 27 GB", | 20260505 17:30:53.862 | INFO | "malloc" : " 14 GB", | 20260505 17:30:53.862 | INFO | "pinned" : " 29 GB" | 20260505 17:30:53.862 | INFO | } | 20260505 17:30:53.862 | INFO | Perf report for the last 5.13953 seconds: | 20260505 17:30:53.862 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:30:53.862 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:53.862 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:53.862 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:53.862 | INFO | ANALYSIS : 63.6%, 3.0, 99.4ms, 0 | 20260505 17:30:53.862 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:53.862 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:53.862 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:30:53.862 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:53.862 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:53.863 | INFO | Uploaded 36GiB to GPU in 2.23685s (16.094GiB/s) | 20260505 17:30:53.863 | INFO | Downloaded 34GiB to GPU in 2.10747s (16.1331GiB/s) | 20260505 17:30:54.002 | INFO | Starting a round of Baz IO | 20260505 17:30:57.187 | INFO | Populating RTMetrics report | 20260505 17:30:57.188 | INFO | RTMetrics report populated | 20260505 17:30:57.188 | INFO | Opening tmp RTMetrics json file | 20260505 17:30:57.194 | INFO | Serializing RTMetrics to json | 20260505 17:30:57.195 | INFO | Serialization done, swapping tmp file now... | 20260505 17:30:57.195 | INFO | RTMetrics done | 20260505 17:30:57.284 | INFO | Finished a round of Baz IO: Wrote 2.129 GiB in 3.281 seconds | 20260505 17:30:57.284 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.05% / 0.00% / 77.95% | 20260505 17:30:57.284 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.296152% / 3.97364e-06% / 0.000107288% | 20260505 17:30:57.284 | INFO | Baz IO overall: | 20260505 17:30:57.284 | INFO | BytesWritten : 2965899756756 | 20260505 17:30:57.284 | INFO | NumEvents : 1406337763786 | 20260505 17:30:59.009 | INFO | Analyzing chunk frames = [5017600,5018112) | 20260505 17:30:59.009 | INFO | MemInfo: { | 20260505 17:30:59.009 | INFO | "gpu" : " 17 GB", | 20260505 17:30:59.009 | INFO | "huge" : "", | 20260505 17:30:59.009 | INFO | "huge_shared" : " 27 GB", | 20260505 17:30:59.009 | INFO | "malloc" : " 14 GB", | 20260505 17:30:59.009 | INFO | "pinned" : " 29 GB" | 20260505 17:30:59.009 | INFO | } | 20260505 17:30:59.009 | INFO | Perf report for the last 5.14674 seconds: | 20260505 17:30:59.009 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:30:59.009 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:59.009 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:59.009 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:59.009 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 17:30:59.009 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:30:59.009 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260505 17:30:59.009 | INFO | PRE_HQ : 60.2%, 1.8, 56.5ms, 0 | 20260505 17:30:59.009 | INFO | BAZWRITER : 63.2%, 3.9, 125.4ms, 0 | 20260505 17:30:59.009 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 17:30:59.009 | INFO | Uploaded 36GiB to GPU in 2.16115s (16.6578GiB/s) | 20260505 17:30:59.009 | INFO | Downloaded 38GiB to GPU in 2.25423s (16.8572GiB/s) | 20260505 17:31:04.153 | INFO | Analyzing chunk frames = [5018112,5018624) | 20260505 17:31:04.153 | INFO | MemInfo: { | 20260505 17:31:04.153 | INFO | "gpu" : " 17 GB", | 20260505 17:31:04.153 | INFO | "huge" : "", | 20260505 17:31:04.153 | INFO | "huge_shared" : " 27 GB", | 20260505 17:31:04.153 | INFO | "malloc" : " 14 GB", | 20260505 17:31:04.153 | INFO | "pinned" : " 29 GB" | 20260505 17:31:04.153 | INFO | } | 20260505 17:31:04.153 | INFO | Perf report for the last 5.1439 seconds: | 20260505 17:31:04.153 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:31:04.153 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:04.153 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:04.153 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:04.153 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 17:31:04.153 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:04.153 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:04.153 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 17:31:04.153 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:04.153 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:04.153 | INFO | Uploaded 36GiB to GPU in 2.137s (16.8461GiB/s) | 20260505 17:31:04.153 | INFO | Downloaded 34GiB to GPU in 2.02556s (16.7854GiB/s) | 20260505 17:31:09.282 | INFO | Analyzing chunk frames = [5018624,5019136) | 20260505 17:31:09.282 | INFO | MemInfo: { | 20260505 17:31:09.282 | INFO | "gpu" : " 17 GB", | 20260505 17:31:09.282 | INFO | "huge" : "", | 20260505 17:31:09.282 | INFO | "huge_shared" : " 27 GB", | 20260505 17:31:09.282 | INFO | "malloc" : " 14 GB", | 20260505 17:31:09.282 | INFO | "pinned" : " 29 GB" | 20260505 17:31:09.282 | INFO | } | 20260505 17:31:09.282 | INFO | Perf report for the last 5.12959 seconds: | 20260505 17:31:09.282 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:31:09.283 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:09.283 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:09.283 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:09.283 | INFO | ANALYSIS : 62.8%, 3.0, 97.6ms, 0 | 20260505 17:31:09.283 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:09.283 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:09.283 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 17:31:09.283 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:09.283 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:09.283 | INFO | Uploaded 36GiB to GPU in 2.20478s (16.3281GiB/s) | 20260505 17:31:09.283 | INFO | Downloaded 34GiB to GPU in 2.04203s (16.6501GiB/s) | 20260505 17:31:14.416 | INFO | Analyzing chunk frames = [5019136,5019648) | 20260505 17:31:14.416 | INFO | MemInfo: { | 20260505 17:31:14.416 | INFO | "gpu" : " 17 GB", | 20260505 17:31:14.416 | INFO | "huge" : "", | 20260505 17:31:14.416 | INFO | "huge_shared" : " 27 GB", | 20260505 17:31:14.416 | INFO | "malloc" : " 14 GB", | 20260505 17:31:14.416 | INFO | "pinned" : " 29 GB" | 20260505 17:31:14.416 | INFO | } | 20260505 17:31:14.416 | INFO | Perf report for the last 5.134 seconds: | 20260505 17:31:14.417 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:31:14.417 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:14.417 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:14.417 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:14.417 | INFO | ANALYSIS : 62.5%, 3.0, 97.0ms, 0 | 20260505 17:31:14.417 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:14.417 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:14.417 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 17:31:14.417 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:14.417 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:14.417 | INFO | Uploaded 36GiB to GPU in 2.16557s (16.6238GiB/s) | 20260505 17:31:14.417 | INFO | Downloaded 34GiB to GPU in 2.14171s (15.8752GiB/s) | 20260505 17:31:19.567 | INFO | Analyzing chunk frames = [5019648,5020160) | 20260505 17:31:19.567 | INFO | MemInfo: { | 20260505 17:31:19.567 | INFO | "gpu" : " 17 GB", | 20260505 17:31:19.567 | INFO | "huge" : "", | 20260505 17:31:19.567 | INFO | "huge_shared" : " 27 GB", | 20260505 17:31:19.567 | INFO | "malloc" : " 14 GB", | 20260505 17:31:19.567 | INFO | "pinned" : " 29 GB" | 20260505 17:31:19.567 | INFO | } | 20260505 17:31:19.567 | INFO | Perf report for the last 5.15038 seconds: | 20260505 17:31:19.567 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:31:19.567 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:19.567 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:19.567 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:19.567 | INFO | ANALYSIS : 62.6%, 3.0, 97.9ms, 0 | 20260505 17:31:19.567 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:19.567 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:19.567 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 17:31:19.567 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:19.567 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:19.567 | INFO | Uploaded 36GiB to GPU in 2.19092s (16.4315GiB/s) | 20260505 17:31:19.567 | INFO | Downloaded 34GiB to GPU in 2.13285s (15.9411GiB/s) | 20260505 17:31:24.709 | INFO | Analyzing chunk frames = [5020160,5020672) | 20260505 17:31:24.709 | INFO | MemInfo: { | 20260505 17:31:24.709 | INFO | "gpu" : " 17 GB", | 20260505 17:31:24.709 | INFO | "huge" : "", | 20260505 17:31:24.709 | INFO | "huge_shared" : " 27 GB", | 20260505 17:31:24.709 | INFO | "malloc" : " 14 GB", | 20260505 17:31:24.709 | INFO | "pinned" : " 29 GB" | 20260505 17:31:24.709 | INFO | } | 20260505 17:31:24.709 | INFO | Perf report for the last 5.1427 seconds: | 20260505 17:31:24.710 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:31:24.710 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:24.710 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:24.710 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:24.710 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 17:31:24.710 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:24.710 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:24.710 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 17:31:24.710 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:24.710 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:24.710 | INFO | Uploaded 36GiB to GPU in 2.13129s (16.8912GiB/s) | 20260505 17:31:24.710 | INFO | Downloaded 34GiB to GPU in 2.04397s (16.6343GiB/s) | 20260505 17:31:29.844 | INFO | Analyzing chunk frames = [5020672,5021184) | 20260505 17:31:29.844 | INFO | MemInfo: { | 20260505 17:31:29.844 | INFO | "gpu" : " 17 GB", | 20260505 17:31:29.844 | INFO | "huge" : "", | 20260505 17:31:29.844 | INFO | "huge_shared" : " 27 GB", | 20260505 17:31:29.844 | INFO | "malloc" : " 14 GB", | 20260505 17:31:29.844 | INFO | "pinned" : " 29 GB" | 20260505 17:31:29.844 | INFO | } | 20260505 17:31:29.844 | INFO | Perf report for the last 5.13467 seconds: | 20260505 17:31:29.849 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:31:29.849 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:29.849 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:29.849 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:29.849 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 17:31:29.849 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:29.849 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:29.849 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 17:31:29.849 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:29.849 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:29.849 | INFO | Uploaded 36GiB to GPU in 2.13552s (16.8577GiB/s) | 20260505 17:31:29.849 | INFO | Downloaded 34GiB to GPU in 2.14311s (15.8648GiB/s) | 20260505 17:31:34.981 | INFO | Analyzing chunk frames = [5021184,5021696) | 20260505 17:31:34.981 | INFO | MemInfo: { | 20260505 17:31:34.981 | INFO | "gpu" : " 17 GB", | 20260505 17:31:34.981 | INFO | "huge" : "", | 20260505 17:31:34.981 | INFO | "huge_shared" : " 27 GB", | 20260505 17:31:34.981 | INFO | "malloc" : " 14 GB", | 20260505 17:31:34.981 | INFO | "pinned" : " 29 GB" | 20260505 17:31:34.981 | INFO | } | 20260505 17:31:34.981 | INFO | Perf report for the last 5.13649 seconds: | 20260505 17:31:34.981 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:31:34.981 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:34.981 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:34.981 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:34.981 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 17:31:34.981 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:34.981 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:34.981 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 17:31:34.981 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:34.981 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:34.981 | INFO | Uploaded 36GiB to GPU in 2.19997s (16.3638GiB/s) | 20260505 17:31:34.981 | INFO | Downloaded 34GiB to GPU in 2.03278s (16.7258GiB/s) | 20260505 17:31:34.981 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 17:31:34.981 | INFO | Total mem usage: 15.3 GiB | 20260505 17:31:34.981 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 17:31:34.981 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.99%) | 20260505 17:31:34.981 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 17:31:34.981 | INFO | High Water Breakdown: | 20260505 17:31:34.981 | INFO | 61% : 8.66 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 17:31:34.981 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 17:31:34.981 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 17:31:34.981 | INFO | ------------------------------------------ | 20260505 17:31:34.981 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:31:34.981 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 17:31:34.981 | INFO | Total mem usage: 26.5 GiB | 20260505 17:31:34.981 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 17:31:34.981 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 17:31:34.981 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 17:31:34.981 | INFO | High Water Breakdown: | 20260505 17:31:34.981 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:31:34.981 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:31:34.981 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:31:34.981 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:31:34.982 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:31:34.982 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:31:34.982 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 17:31:34.982 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 17:31:34.982 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:31:34.982 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 17:31:34.982 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 17:31:34.982 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 17:31:34.982 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 17:31:34.982 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 17:31:34.982 | INFO | ------------------------------------------ | 20260505 17:31:34.982 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:31:34.982 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 17:31:34.982 | INFO | Total mem usage: 25.6 GiB | 20260505 17:31:34.982 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 17:31:34.982 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 17:31:34.982 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 17:31:34.982 | INFO | High Water Breakdown: | 20260505 17:31:34.982 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 17:31:34.982 | INFO | ------------------------------------------ | 20260505 17:31:34.982 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:31:34.982 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 17:31:34.982 | INFO | Total mem usage: 15.7 GiB | 20260505 17:31:34.982 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 17:31:34.982 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 17:31:34.982 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 17:31:34.982 | INFO | High Water Breakdown: | 20260505 17:31:34.982 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 17:31:34.982 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 17:31:34.982 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 17:31:34.982 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 17:31:34.982 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 17:31:34.982 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 17:31:34.982 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 17:31:34.982 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 17:31:34.982 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 17:31:34.982 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:31:34.982 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 17:31:34.982 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:31:34.982 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 17:31:34.982 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 17:31:34.982 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 17:31:34.982 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 17:31:34.982 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:31:34.982 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 17:31:34.982 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:31:34.982 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:31:34.982 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:31:34.982 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 17:31:34.982 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 17:31:34.982 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:31:34.982 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 17:31:34.982 | INFO | ------------------------------------------ | 20260505 17:31:34.982 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:31:34.982 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 17:31:34.982 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:31:34.982 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:31:34.982 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.49e+01, 1.36e+01 / 1.80e+00 | 20260505 17:31:34.982 | INFO | PerfReport: 0.29%, AggregateStats -- 9.27e-02 / 1.02e-01, 1.13e-02 / 5.43e-03 | 20260505 17:31:34.982 | INFO | PerfReport: 11.81%, Baseline -- 3.73e+00 / 4.53e+00, 5.73e-01 / 1.57e-01 | 20260505 17:31:34.982 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.52e+00, 1.89e-01 / 8.89e-02 | 20260505 17:31:34.982 | INFO | PerfReport: 15.93%, DME -- 4.02e+01 / 4.45e+01, 4.15e+00 / 1.80e+00 | 20260505 17:31:34.982 | INFO | PerfReport: 56.77%, FrameLabeling -- 1.79e+01 / 1.95e+01, 1.41e+00 / 1.30e+00 | 20260505 17:31:34.982 | INFO | PerfReport: 7.36%, PulseAccumulating -- 2.32e+00 / 2.89e+00, 4.44e-01 / 1.83e-01 | 20260505 17:31:34.982 | INFO | PerfReport: 3.38%, Metrics -- 1.07e+00 / 1.39e+00, 2.96e-01 / 1.65e-01 | 20260505 17:31:34.982 | INFO | PerfReport: 1.18%, PulseSerializing -- 3.71e-01 / 4.31e-01, 6.64e-02 / 4.17e-02 | 20260505 17:31:34.982 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 17:31:34.982 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:31:34.982 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:31:34.982 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260505 17:31:34.982 | INFO | PerfReport: 22.61%, Upload -- 2.26e+01 / 0.00e+00, 7.96e+00 / 0.00e+00 | 20260505 17:31:34.982 | INFO | PerfReport: 21.91%, Download -- 2.19e+01 / 0.00e+00, 7.89e+00 / 0.00e+00 | 20260505 17:31:34.982 | INFO | PerfReport: 31.58%, Compute -- 3.16e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 17:31:34.982 | INFO | PerfReport: 2.27%, LockUpload -- 2.27e+00 / 0.00e+00, 7.89e+00 / 0.00e+00 | 20260505 17:31:34.982 | INFO | PerfReport: 1.38%, LockDownload -- 1.38e+00 / 0.00e+00, 2.84e+00 / 0.00e+00 | 20260505 17:31:34.982 | INFO | PerfReport: 20.25%, LockCompute -- 2.02e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 17:31:34.982 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.46e-02 / 0.00e+00 | 20260505 17:31:35.121 | INFO | Starting a round of Baz IO | 20260505 17:31:38.309 | INFO | Populating RTMetrics report | 20260505 17:31:38.309 | INFO | RTMetrics report populated | 20260505 17:31:38.309 | INFO | Opening tmp RTMetrics json file | 20260505 17:31:38.315 | INFO | Serializing RTMetrics to json | 20260505 17:31:38.317 | INFO | Serialization done, swapping tmp file now... | 20260505 17:31:38.317 | INFO | RTMetrics done | 20260505 17:31:38.401 | INFO | Finished a round of Baz IO: Wrote 2.127 GiB in 3.279 seconds | 20260505 17:31:38.401 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.05% / 0.00% / 77.95% | 20260505 17:31:38.401 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.295571% / 0% / 0.000115236% | 20260505 17:31:38.401 | INFO | Baz IO overall: | 20260505 17:31:38.401 | INFO | BytesWritten : 2968183978096 | 20260505 17:31:38.401 | INFO | NumEvents : 1407405050939 | 20260505 17:31:40.121 | INFO | Analyzing chunk frames = [5021696,5022208) | 20260505 17:31:40.121 | INFO | MemInfo: { | 20260505 17:31:40.121 | INFO | "gpu" : " 17 GB", | 20260505 17:31:40.121 | INFO | "huge" : "", | 20260505 17:31:40.121 | INFO | "huge_shared" : " 27 GB", | 20260505 17:31:40.121 | INFO | "malloc" : " 14 GB", | 20260505 17:31:40.121 | INFO | "pinned" : " 29 GB" | 20260505 17:31:40.121 | INFO | } | 20260505 17:31:40.121 | INFO | Perf report for the last 5.14021 seconds: | 20260505 17:31:40.121 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:31:40.121 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:40.121 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:40.121 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:40.121 | INFO | ANALYSIS : 63.7%, 3.0, 99.2ms, 0 | 20260505 17:31:40.121 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:40.121 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260505 17:31:40.121 | INFO | PRE_HQ : 60.2%, 1.8, 56.0ms, 0 | 20260505 17:31:40.121 | INFO | BAZWRITER : 63.3%, 3.8, 123.8ms, 0 | 20260505 17:31:40.121 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 17:31:40.121 | INFO | Uploaded 36GiB to GPU in 2.16911s (16.5966GiB/s) | 20260505 17:31:40.121 | INFO | Downloaded 38GiB to GPU in 2.26387s (16.7854GiB/s) | 20260505 17:31:45.263 | INFO | Analyzing chunk frames = [5022208,5022720) | 20260505 17:31:45.263 | INFO | MemInfo: { | 20260505 17:31:45.263 | INFO | "gpu" : " 17 GB", | 20260505 17:31:45.263 | INFO | "huge" : "", | 20260505 17:31:45.263 | INFO | "huge_shared" : " 27 GB", | 20260505 17:31:45.263 | INFO | "malloc" : " 14 GB", | 20260505 17:31:45.263 | INFO | "pinned" : " 29 GB" | 20260505 17:31:45.263 | INFO | } | 20260505 17:31:45.263 | INFO | Perf report for the last 5.14253 seconds: | 20260505 17:31:45.264 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:31:45.264 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:45.264 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:45.264 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:45.264 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 17:31:45.264 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:45.264 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:45.264 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260505 17:31:45.264 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:45.264 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:45.264 | INFO | Uploaded 36GiB to GPU in 2.13307s (16.8771GiB/s) | 20260505 17:31:45.264 | INFO | Downloaded 34GiB to GPU in 2.02102s (16.8232GiB/s) | 20260505 17:31:50.405 | INFO | Analyzing chunk frames = [5022720,5023232) | 20260505 17:31:50.405 | INFO | MemInfo: { | 20260505 17:31:50.405 | INFO | "gpu" : " 17 GB", | 20260505 17:31:50.405 | INFO | "huge" : "", | 20260505 17:31:50.405 | INFO | "huge_shared" : " 27 GB", | 20260505 17:31:50.405 | INFO | "malloc" : " 14 GB", | 20260505 17:31:50.405 | INFO | "pinned" : " 29 GB" | 20260505 17:31:50.405 | INFO | } | 20260505 17:31:50.405 | INFO | Perf report for the last 5.14181 seconds: | 20260505 17:31:50.405 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:31:50.405 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:50.405 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:50.405 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:50.405 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260505 17:31:50.405 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:50.405 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:50.405 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 17:31:50.405 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:50.405 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:50.406 | INFO | Uploaded 36GiB to GPU in 2.2006s (16.3591GiB/s) | 20260505 17:31:50.406 | INFO | Downloaded 34GiB to GPU in 2.04935s (16.5906GiB/s) | 20260505 17:31:55.535 | INFO | Analyzing chunk frames = [5023232,5023744) | 20260505 17:31:55.535 | INFO | MemInfo: { | 20260505 17:31:55.535 | INFO | "gpu" : " 17 GB", | 20260505 17:31:55.535 | INFO | "huge" : "", | 20260505 17:31:55.535 | INFO | "huge_shared" : " 27 GB", | 20260505 17:31:55.535 | INFO | "malloc" : " 14 GB", | 20260505 17:31:55.535 | INFO | "pinned" : " 29 GB" | 20260505 17:31:55.535 | INFO | } | 20260505 17:31:55.536 | INFO | Perf report for the last 5.13026 seconds: | 20260505 17:31:55.536 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:31:55.536 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:55.536 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:55.536 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:55.536 | INFO | ANALYSIS : 62.6%, 3.0, 97.0ms, 0 | 20260505 17:31:55.536 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:55.536 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:55.536 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:31:55.536 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:55.536 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:31:55.536 | INFO | Uploaded 36GiB to GPU in 2.18913s (16.4449GiB/s) | 20260505 17:31:55.536 | INFO | Downloaded 34GiB to GPU in 2.13934s (15.8928GiB/s) | 20260505 17:32:00.672 | INFO | Analyzing chunk frames = [5023744,5024256) | 20260505 17:32:00.672 | INFO | MemInfo: { | 20260505 17:32:00.672 | INFO | "gpu" : " 17 GB", | 20260505 17:32:00.672 | INFO | "huge" : "", | 20260505 17:32:00.672 | INFO | "huge_shared" : " 27 GB", | 20260505 17:32:00.672 | INFO | "malloc" : " 14 GB", | 20260505 17:32:00.672 | INFO | "pinned" : " 29 GB" | 20260505 17:32:00.672 | INFO | } | 20260505 17:32:00.672 | INFO | Perf report for the last 5.13631 seconds: | 20260505 17:32:00.672 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:32:00.672 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:00.672 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:00.672 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:00.672 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 17:32:00.672 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:00.672 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:00.672 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 17:32:00.672 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:00.672 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:00.672 | INFO | Uploaded 36GiB to GPU in 2.18493s (16.4765GiB/s) | 20260505 17:32:00.672 | INFO | Downloaded 34GiB to GPU in 2.13486s (15.9261GiB/s) | 20260505 17:32:05.821 | INFO | Analyzing chunk frames = [5024256,5024768) | 20260505 17:32:05.821 | INFO | MemInfo: { | 20260505 17:32:05.821 | INFO | "gpu" : " 17 GB", | 20260505 17:32:05.822 | INFO | "huge" : "", | 20260505 17:32:05.822 | INFO | "huge_shared" : " 27 GB", | 20260505 17:32:05.822 | INFO | "malloc" : " 14 GB", | 20260505 17:32:05.822 | INFO | "pinned" : " 29 GB" | 20260505 17:32:05.822 | INFO | } | 20260505 17:32:05.822 | INFO | Perf report for the last 5.14972 seconds: | 20260505 17:32:05.822 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:32:05.822 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:05.822 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:05.822 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:05.822 | INFO | ANALYSIS : 62.3%, 3.0, 97.5ms, 0 | 20260505 17:32:05.822 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:05.822 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:05.822 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 17:32:05.822 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:05.822 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:05.822 | INFO | Uploaded 36GiB to GPU in 2.14088s (16.8156GiB/s) | 20260505 17:32:05.822 | INFO | Downloaded 34GiB to GPU in 2.03467s (16.7104GiB/s) | 20260505 17:32:10.965 | INFO | Analyzing chunk frames = [5024768,5025280) | 20260505 17:32:10.965 | INFO | MemInfo: { | 20260505 17:32:10.965 | INFO | "gpu" : " 17 GB", | 20260505 17:32:10.965 | INFO | "huge" : "", | 20260505 17:32:10.965 | INFO | "huge_shared" : " 27 GB", | 20260505 17:32:10.965 | INFO | "malloc" : " 14 GB", | 20260505 17:32:10.965 | INFO | "pinned" : " 29 GB" | 20260505 17:32:10.965 | INFO | } | 20260505 17:32:10.965 | INFO | Perf report for the last 5.14306 seconds: | 20260505 17:32:10.965 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:32:10.965 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:10.965 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:10.965 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:10.965 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 17:32:10.965 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:10.965 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:10.965 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 17:32:10.965 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:10.965 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:10.965 | INFO | Uploaded 36GiB to GPU in 2.12896s (16.9097GiB/s) | 20260505 17:32:10.965 | INFO | Downloaded 34GiB to GPU in 2.1628s (15.7204GiB/s) | 20260505 17:32:16.097 | INFO | Analyzing chunk frames = [5025280,5025792) | 20260505 17:32:16.097 | INFO | MemInfo: { | 20260505 17:32:16.097 | INFO | "gpu" : " 17 GB", | 20260505 17:32:16.097 | INFO | "huge" : "", | 20260505 17:32:16.097 | INFO | "huge_shared" : " 27 GB", | 20260505 17:32:16.097 | INFO | "malloc" : " 14 GB", | 20260505 17:32:16.097 | INFO | "pinned" : " 29 GB" | 20260505 17:32:16.097 | INFO | } | 20260505 17:32:16.097 | INFO | Perf report for the last 5.13226 seconds: | 20260505 17:32:16.097 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:32:16.097 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:16.097 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:16.097 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:16.097 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 17:32:16.097 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:16.097 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:16.097 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 17:32:16.097 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:16.097 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:16.097 | INFO | Uploaded 36GiB to GPU in 2.19837s (16.3758GiB/s) | 20260505 17:32:16.097 | INFO | Downloaded 34GiB to GPU in 2.04131s (16.656GiB/s) | 20260505 17:32:16.235 | INFO | Starting a round of Baz IO | 20260505 17:32:19.429 | INFO | Populating RTMetrics report | 20260505 17:32:19.429 | INFO | RTMetrics report populated | 20260505 17:32:19.429 | INFO | Opening tmp RTMetrics json file | 20260505 17:32:19.435 | INFO | Serializing RTMetrics to json | 20260505 17:32:19.437 | INFO | Serialization done, swapping tmp file now... | 20260505 17:32:19.437 | INFO | RTMetrics done | 20260505 17:32:19.526 | INFO | Finished a round of Baz IO: Wrote 2.126 GiB in 3.291 seconds | 20260505 17:32:19.526 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.05% / 0.00% / 77.95% | 20260505 17:32:19.526 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.294864% / 0% / 0.000103315% | 20260505 17:32:19.526 | INFO | Baz IO overall: | 20260505 17:32:19.526 | INFO | BytesWritten : 2970466993424 | 20260505 17:32:19.526 | INFO | NumEvents : 1408471586026 | 20260505 17:32:21.233 | INFO | Analyzing chunk frames = [5025792,5026304) | 20260505 17:32:21.233 | INFO | MemInfo: { | 20260505 17:32:21.233 | INFO | "gpu" : " 17 GB", | 20260505 17:32:21.233 | INFO | "huge" : "", | 20260505 17:32:21.233 | INFO | "huge_shared" : " 27 GB", | 20260505 17:32:21.233 | INFO | "malloc" : " 14 GB", | 20260505 17:32:21.233 | INFO | "pinned" : " 29 GB" | 20260505 17:32:21.233 | INFO | } | 20260505 17:32:21.233 | INFO | Perf report for the last 5.13593 seconds: | 20260505 17:32:21.233 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:32:21.233 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:21.233 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:21.233 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:21.233 | INFO | ANALYSIS : 63.8%, 3.0, 99.3ms, 0 | 20260505 17:32:21.233 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:21.233 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260505 17:32:21.233 | INFO | PRE_HQ : 60.6%, 1.8, 56.1ms, 0 | 20260505 17:32:21.233 | INFO | BAZWRITER : 63.5%, 3.8, 124.7ms, 0 | 20260505 17:32:21.233 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 17:32:21.233 | INFO | Uploaded 36GiB to GPU in 2.19538s (16.3981GiB/s) | 20260505 17:32:21.233 | INFO | Downloaded 38GiB to GPU in 2.26219s (16.7979GiB/s) | 20260505 17:32:26.375 | INFO | Analyzing chunk frames = [5026304,5026816) | 20260505 17:32:26.375 | INFO | MemInfo: { | 20260505 17:32:26.375 | INFO | "gpu" : " 17 GB", | 20260505 17:32:26.375 | INFO | "huge" : "", | 20260505 17:32:26.375 | INFO | "huge_shared" : " 27 GB", | 20260505 17:32:26.375 | INFO | "malloc" : " 14 GB", | 20260505 17:32:26.375 | INFO | "pinned" : " 29 GB" | 20260505 17:32:26.375 | INFO | } | 20260505 17:32:26.375 | INFO | Perf report for the last 5.14211 seconds: | 20260505 17:32:26.375 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:32:26.375 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:26.375 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:26.375 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:26.375 | INFO | ANALYSIS : 62.7%, 3.0, 98.0ms, 0 | 20260505 17:32:26.375 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:26.375 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:26.375 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 17:32:26.375 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:26.375 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:26.375 | INFO | Uploaded 36GiB to GPU in 2.11761s (17.0003GiB/s) | 20260505 17:32:26.375 | INFO | Downloaded 34GiB to GPU in 2.02486s (16.7913GiB/s) | 20260505 17:32:31.514 | INFO | Analyzing chunk frames = [5026816,5027328) | 20260505 17:32:31.514 | INFO | MemInfo: { | 20260505 17:32:31.514 | INFO | "gpu" : " 17 GB", | 20260505 17:32:31.514 | INFO | "huge" : "", | 20260505 17:32:31.514 | INFO | "huge_shared" : " 27 GB", | 20260505 17:32:31.514 | INFO | "malloc" : " 14 GB", | 20260505 17:32:31.514 | INFO | "pinned" : " 29 GB" | 20260505 17:32:31.514 | INFO | } | 20260505 17:32:31.514 | INFO | Perf report for the last 5.13914 seconds: | 20260505 17:32:31.514 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:32:31.514 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:31.514 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:31.514 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:31.514 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 17:32:31.514 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:31.514 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:31.514 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:32:31.514 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:31.515 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:31.515 | INFO | Uploaded 36GiB to GPU in 2.204s (16.3339GiB/s) | 20260505 17:32:31.515 | INFO | Downloaded 34GiB to GPU in 2.04448s (16.6301GiB/s) | 20260505 17:32:36.662 | INFO | Analyzing chunk frames = [5027328,5027840) | 20260505 17:32:36.662 | INFO | MemInfo: { | 20260505 17:32:36.662 | INFO | "gpu" : " 17 GB", | 20260505 17:32:36.662 | INFO | "huge" : "", | 20260505 17:32:36.662 | INFO | "huge_shared" : " 27 GB", | 20260505 17:32:36.662 | INFO | "malloc" : " 14 GB", | 20260505 17:32:36.662 | INFO | "pinned" : " 29 GB" | 20260505 17:32:36.662 | INFO | } | 20260505 17:32:36.662 | INFO | Perf report for the last 5.14827 seconds: | 20260505 17:32:36.662 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:32:36.662 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:36.662 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:36.662 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:36.662 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260505 17:32:36.663 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:36.663 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:36.663 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 17:32:36.663 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:36.663 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:36.663 | INFO | Uploaded 36GiB to GPU in 2.2184s (16.2279GiB/s) | 20260505 17:32:36.663 | INFO | Downloaded 34GiB to GPU in 2.12943s (15.9667GiB/s) | 20260505 17:32:41.794 | INFO | Analyzing chunk frames = [5027840,5028352) | 20260505 17:32:41.795 | INFO | MemInfo: { | 20260505 17:32:41.795 | INFO | "gpu" : " 17 GB", | 20260505 17:32:41.795 | INFO | "huge" : "", | 20260505 17:32:41.795 | INFO | "huge_shared" : " 27 GB", | 20260505 17:32:41.795 | INFO | "malloc" : " 14 GB", | 20260505 17:32:41.795 | INFO | "pinned" : " 29 GB" | 20260505 17:32:41.795 | INFO | } | 20260505 17:32:41.795 | INFO | Perf report for the last 5.13214 seconds: | 20260505 17:32:41.795 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:32:41.795 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:41.795 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:41.795 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:41.795 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260505 17:32:41.795 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:41.795 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:41.795 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:32:41.795 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:41.795 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:41.795 | INFO | Uploaded 36GiB to GPU in 2.14152s (16.8105GiB/s) | 20260505 17:32:41.795 | INFO | Downloaded 34GiB to GPU in 2.05971s (16.5072GiB/s) | 20260505 17:32:46.927 | INFO | Analyzing chunk frames = [5028352,5028864) | 20260505 17:32:46.927 | INFO | MemInfo: { | 20260505 17:32:46.927 | INFO | "gpu" : " 17 GB", | 20260505 17:32:46.927 | INFO | "huge" : "", | 20260505 17:32:46.927 | INFO | "huge_shared" : " 27 GB", | 20260505 17:32:46.927 | INFO | "malloc" : " 14 GB", | 20260505 17:32:46.927 | INFO | "pinned" : " 29 GB" | 20260505 17:32:46.927 | INFO | } | 20260505 17:32:46.927 | INFO | Perf report for the last 5.13218 seconds: | 20260505 17:32:46.927 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:32:46.927 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:46.927 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:46.927 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:46.927 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260505 17:32:46.927 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:46.927 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:46.927 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 17:32:46.927 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:46.927 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:46.927 | INFO | Uploaded 36GiB to GPU in 2.14241s (16.8035GiB/s) | 20260505 17:32:46.927 | INFO | Downloaded 34GiB to GPU in 2.0469s (16.6105GiB/s) | 20260505 17:32:52.073 | INFO | Analyzing chunk frames = [5028864,5029376) | 20260505 17:32:52.073 | INFO | MemInfo: { | 20260505 17:32:52.073 | INFO | "gpu" : " 17 GB", | 20260505 17:32:52.073 | INFO | "huge" : "", | 20260505 17:32:52.073 | INFO | "huge_shared" : " 27 GB", | 20260505 17:32:52.073 | INFO | "malloc" : " 14 GB", | 20260505 17:32:52.073 | INFO | "pinned" : " 29 GB" | 20260505 17:32:52.073 | INFO | } | 20260505 17:32:52.073 | INFO | Perf report for the last 5.146 seconds: | 20260505 17:32:52.073 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:32:52.073 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:52.073 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:52.073 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:52.073 | INFO | ANALYSIS : 63.3%, 3.0, 99.0ms, 0 | 20260505 17:32:52.073 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:52.073 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:52.073 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 17:32:52.073 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:52.073 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:52.073 | INFO | Uploaded 36GiB to GPU in 2.16202s (16.6511GiB/s) | 20260505 17:32:52.073 | INFO | Downloaded 34GiB to GPU in 2.17945s (15.6002GiB/s) | 20260505 17:32:57.212 | INFO | Analyzing chunk frames = [5029376,5029888) | 20260505 17:32:57.212 | INFO | MemInfo: { | 20260505 17:32:57.212 | INFO | "gpu" : " 17 GB", | 20260505 17:32:57.212 | INFO | "huge" : "", | 20260505 17:32:57.212 | INFO | "huge_shared" : " 27 GB", | 20260505 17:32:57.212 | INFO | "malloc" : " 14 GB", | 20260505 17:32:57.212 | INFO | "pinned" : " 29 GB" | 20260505 17:32:57.212 | INFO | } | 20260505 17:32:57.212 | INFO | Perf report for the last 5.1393 seconds: | 20260505 17:32:57.212 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:32:57.212 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:57.212 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:57.212 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:57.212 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 17:32:57.212 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:57.212 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:57.212 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 17:32:57.212 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:57.212 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:32:57.213 | INFO | Uploaded 36GiB to GPU in 2.19395s (16.4088GiB/s) | 20260505 17:32:57.213 | INFO | Downloaded 34GiB to GPU in 2.01966s (16.8345GiB/s) | 20260505 17:32:57.213 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 17:32:57.213 | INFO | Total mem usage: 15.3 GiB | 20260505 17:32:57.213 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 17:32:57.213 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8%) | 20260505 17:32:57.213 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 17:32:57.213 | INFO | High Water Breakdown: | 20260505 17:32:57.213 | INFO | 61% : 8.66 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 17:32:57.213 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 17:32:57.213 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 17:32:57.213 | INFO | ------------------------------------------ | 20260505 17:32:57.213 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:32:57.213 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 17:32:57.213 | INFO | Total mem usage: 26.5 GiB | 20260505 17:32:57.213 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 17:32:57.213 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 17:32:57.213 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 17:32:57.213 | INFO | High Water Breakdown: | 20260505 17:32:57.213 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:32:57.213 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:32:57.213 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:32:57.213 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:32:57.213 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:32:57.213 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:32:57.213 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 17:32:57.213 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 17:32:57.213 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:32:57.213 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 17:32:57.213 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 17:32:57.213 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 17:32:57.213 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 17:32:57.213 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 17:32:57.213 | INFO | ------------------------------------------ | 20260505 17:32:57.213 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:32:57.213 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 17:32:57.213 | INFO | Total mem usage: 25.6 GiB | 20260505 17:32:57.213 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 17:32:57.213 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 17:32:57.213 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 17:32:57.213 | INFO | High Water Breakdown: | 20260505 17:32:57.213 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 17:32:57.213 | INFO | ------------------------------------------ | 20260505 17:32:57.213 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:32:57.213 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 17:32:57.213 | INFO | Total mem usage: 15.7 GiB | 20260505 17:32:57.213 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 17:32:57.213 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 17:32:57.213 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 17:32:57.213 | INFO | High Water Breakdown: | 20260505 17:32:57.213 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 17:32:57.213 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 17:32:57.213 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 17:32:57.213 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 17:32:57.213 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 17:32:57.213 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 17:32:57.213 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 17:32:57.213 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 17:32:57.213 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 17:32:57.213 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:32:57.213 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 17:32:57.213 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:32:57.213 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 17:32:57.213 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 17:32:57.213 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 17:32:57.213 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 17:32:57.213 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:32:57.213 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 17:32:57.213 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:32:57.213 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:32:57.213 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:32:57.213 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 17:32:57.213 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 17:32:57.213 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:32:57.213 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 17:32:57.213 | INFO | ------------------------------------------ | 20260505 17:32:57.213 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:32:57.214 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 17:32:57.214 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:32:57.214 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:32:57.214 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.48e+01, 1.36e+01 / 1.92e+00 | 20260505 17:32:57.214 | INFO | PerfReport: 0.29%, AggregateStats -- 9.25e-02 / 1.01e-01, 1.11e-02 / 3.64e-03 | 20260505 17:32:57.214 | INFO | PerfReport: 11.81%, Baseline -- 3.72e+00 / 4.55e+00, 5.70e-01 / 9.86e-02 | 20260505 17:32:57.214 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.51e+00, 1.89e-01 / 9.98e-02 | 20260505 17:32:57.214 | INFO | PerfReport: 15.99%, DME -- 4.03e+01 / 4.50e+01, 4.30e+00 / 2.01e+00 | 20260505 17:32:57.214 | INFO | PerfReport: 56.71%, FrameLabeling -- 1.79e+01 / 1.89e+01, 1.37e+00 / 7.97e-01 | 20260505 17:32:57.214 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.33e+00 / 2.93e+00, 4.52e-01 / 1.39e-01 | 20260505 17:32:57.214 | INFO | PerfReport: 3.37%, Metrics -- 1.06e+00 / 1.40e+00, 2.92e-01 / 1.37e-01 | 20260505 17:32:57.214 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.70e-01 / 4.28e-01, 6.53e-02 / 3.95e-02 | 20260505 17:32:57.214 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 17:32:57.214 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:32:57.214 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:32:57.214 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.57e+01 / 0.00e+00 | 20260505 17:32:57.214 | INFO | PerfReport: 22.61%, Upload -- 2.26e+01 / 0.00e+00, 7.95e+00 / 0.00e+00 | 20260505 17:32:57.214 | INFO | PerfReport: 21.89%, Download -- 2.19e+01 / 0.00e+00, 7.85e+00 / 0.00e+00 | 20260505 17:32:57.214 | INFO | PerfReport: 31.54%, Compute -- 3.15e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 17:32:57.214 | INFO | PerfReport: 2.32%, LockUpload -- 2.32e+00 / 0.00e+00, 7.97e+00 / 0.00e+00 | 20260505 17:32:57.214 | INFO | PerfReport: 1.44%, LockDownload -- 1.44e+00 / 0.00e+00, 2.93e+00 / 0.00e+00 | 20260505 17:32:57.214 | INFO | PerfReport: 20.19%, LockCompute -- 2.02e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260505 17:32:57.214 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 2.25e-02 / 0.00e+00 | 20260505 17:32:57.347 | INFO | Starting a round of Baz IO | 20260505 17:33:00.530 | INFO | Populating RTMetrics report | 20260505 17:33:00.531 | INFO | RTMetrics report populated | 20260505 17:33:00.531 | INFO | Opening tmp RTMetrics json file | 20260505 17:33:00.537 | INFO | Serializing RTMetrics to json | 20260505 17:33:00.539 | INFO | Serialization done, swapping tmp file now... | 20260505 17:33:00.540 | INFO | RTMetrics done | 20260505 17:33:00.627 | INFO | Finished a round of Baz IO: Wrote 2.127 GiB in 3.279 seconds | 20260505 17:33:00.627 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.04% / 0.00% / 77.95% | 20260505 17:33:00.627 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.294328% / 7.94729e-06% / 0.000115236% | 20260505 17:33:00.627 | INFO | Baz IO overall: | 20260505 17:33:00.627 | INFO | BytesWritten : 2972751293606 | 20260505 17:33:00.627 | INFO | NumEvents : 1409539353898 | 20260505 17:33:02.352 | INFO | Analyzing chunk frames = [5029888,5030400) | 20260505 17:33:02.352 | INFO | MemInfo: { | 20260505 17:33:02.352 | INFO | "gpu" : " 17 GB", | 20260505 17:33:02.352 | INFO | "huge" : "", | 20260505 17:33:02.352 | INFO | "huge_shared" : " 27 GB", | 20260505 17:33:02.352 | INFO | "malloc" : " 14 GB", | 20260505 17:33:02.352 | INFO | "pinned" : " 29 GB" | 20260505 17:33:02.352 | INFO | } | 20260505 17:33:02.352 | INFO | Perf report for the last 5.1402 seconds: | 20260505 17:33:02.352 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:33:02.352 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:02.352 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:02.352 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:02.353 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 17:33:02.353 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:02.353 | INFO | RT_METRICS : 44.7%, 1.0, 23.4ms, 0 | 20260505 17:33:02.353 | INFO | PRE_HQ : 60.5%, 1.8, 56.2ms, 0 | 20260505 17:33:02.353 | INFO | BAZWRITER : 63.2%, 3.9, 125.3ms, 0 | 20260505 17:33:02.353 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 17:33:02.353 | INFO | Uploaded 36GiB to GPU in 2.16064s (16.6617GiB/s) | 20260505 17:33:02.353 | INFO | Downloaded 38GiB to GPU in 2.25019s (16.8874GiB/s) | 20260505 17:33:07.481 | INFO | Analyzing chunk frames = [5030400,5030912) | 20260505 17:33:07.481 | INFO | MemInfo: { | 20260505 17:33:07.481 | INFO | "gpu" : " 17 GB", | 20260505 17:33:07.481 | INFO | "huge" : "", | 20260505 17:33:07.481 | INFO | "huge_shared" : " 27 GB", | 20260505 17:33:07.481 | INFO | "malloc" : " 14 GB", | 20260505 17:33:07.481 | INFO | "pinned" : " 29 GB" | 20260505 17:33:07.481 | INFO | } | 20260505 17:33:07.481 | INFO | Perf report for the last 5.1283 seconds: | 20260505 17:33:07.481 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:33:07.481 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:07.481 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:07.481 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:07.481 | INFO | ANALYSIS : 63.1%, 3.0, 98.3ms, 0 | 20260505 17:33:07.481 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:07.481 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:07.481 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 17:33:07.481 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:07.481 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:07.481 | INFO | Uploaded 36GiB to GPU in 2.13921s (16.8287GiB/s) | 20260505 17:33:07.481 | INFO | Downloaded 34GiB to GPU in 2.03817s (16.6816GiB/s) | 20260505 17:33:12.629 | INFO | Analyzing chunk frames = [5030912,5031424) | 20260505 17:33:12.629 | INFO | MemInfo: { | 20260505 17:33:12.629 | INFO | "gpu" : " 17 GB", | 20260505 17:33:12.629 | INFO | "huge" : "", | 20260505 17:33:12.629 | INFO | "huge_shared" : " 27 GB", | 20260505 17:33:12.629 | INFO | "malloc" : " 14 GB", | 20260505 17:33:12.629 | INFO | "pinned" : " 29 GB" | 20260505 17:33:12.629 | INFO | } | 20260505 17:33:12.629 | INFO | Perf report for the last 5.14805 seconds: | 20260505 17:33:12.629 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:33:12.629 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:12.629 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:12.629 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:12.629 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260505 17:33:12.629 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:12.629 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:12.629 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260505 17:33:12.629 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:12.629 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:12.629 | INFO | Uploaded 36GiB to GPU in 2.18192s (16.4992GiB/s) | 20260505 17:33:12.629 | INFO | Downloaded 34GiB to GPU in 2.05877s (16.5147GiB/s) | 20260505 17:33:17.774 | INFO | Analyzing chunk frames = [5031424,5031936) | 20260505 17:33:17.774 | INFO | MemInfo: { | 20260505 17:33:17.774 | INFO | "gpu" : " 17 GB", | 20260505 17:33:17.774 | INFO | "huge" : "", | 20260505 17:33:17.774 | INFO | "huge_shared" : " 27 GB", | 20260505 17:33:17.774 | INFO | "malloc" : " 14 GB", | 20260505 17:33:17.774 | INFO | "pinned" : " 29 GB" | 20260505 17:33:17.774 | INFO | } | 20260505 17:33:17.774 | INFO | Perf report for the last 5.14556 seconds: | 20260505 17:33:17.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:33:17.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:17.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:17.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:17.774 | INFO | ANALYSIS : 62.5%, 3.0, 97.2ms, 0 | 20260505 17:33:17.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:17.774 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:17.774 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:33:17.774 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:17.774 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:17.775 | INFO | Uploaded 36GiB to GPU in 2.19449s (16.4047GiB/s) | 20260505 17:33:17.775 | INFO | Downloaded 34GiB to GPU in 2.14027s (15.8858GiB/s) | 20260505 17:33:22.912 | INFO | Analyzing chunk frames = [5031936,5032448) | 20260505 17:33:22.912 | INFO | MemInfo: { | 20260505 17:33:22.912 | INFO | "gpu" : " 17 GB", | 20260505 17:33:22.912 | INFO | "huge" : "", | 20260505 17:33:22.912 | INFO | "huge_shared" : " 27 GB", | 20260505 17:33:22.912 | INFO | "malloc" : " 14 GB", | 20260505 17:33:22.912 | INFO | "pinned" : " 29 GB" | 20260505 17:33:22.913 | INFO | } | 20260505 17:33:22.913 | INFO | Perf report for the last 5.1384 seconds: | 20260505 17:33:22.913 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:33:22.913 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:22.913 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:22.913 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:22.913 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260505 17:33:22.913 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:22.913 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:22.913 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:33:22.913 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:22.913 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:22.913 | INFO | Uploaded 36GiB to GPU in 2.1387s (16.8326GiB/s) | 20260505 17:33:22.913 | INFO | Downloaded 34GiB to GPU in 2.0604s (16.5016GiB/s) | 20260505 17:33:28.047 | INFO | Analyzing chunk frames = [5032448,5032960) | 20260505 17:33:28.047 | INFO | MemInfo: { | 20260505 17:33:28.047 | INFO | "gpu" : " 17 GB", | 20260505 17:33:28.047 | INFO | "huge" : "", | 20260505 17:33:28.047 | INFO | "huge_shared" : " 27 GB", | 20260505 17:33:28.047 | INFO | "malloc" : " 14 GB", | 20260505 17:33:28.047 | INFO | "pinned" : " 29 GB" | 20260505 17:33:28.047 | INFO | } | 20260505 17:33:28.047 | INFO | Perf report for the last 5.13402 seconds: | 20260505 17:33:28.047 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:33:28.047 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:28.047 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:28.047 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:28.047 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260505 17:33:28.047 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:28.047 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:28.047 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 17:33:28.047 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:28.047 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:28.047 | INFO | Uploaded 36GiB to GPU in 2.1356s (16.8571GiB/s) | 20260505 17:33:28.047 | INFO | Downloaded 34GiB to GPU in 2.04716s (16.6084GiB/s) | 20260505 17:33:33.182 | INFO | Analyzing chunk frames = [5032960,5033472) | 20260505 17:33:33.182 | INFO | MemInfo: { | 20260505 17:33:33.182 | INFO | "gpu" : " 17 GB", | 20260505 17:33:33.182 | INFO | "huge" : "", | 20260505 17:33:33.182 | INFO | "huge_shared" : " 27 GB", | 20260505 17:33:33.182 | INFO | "malloc" : " 14 GB", | 20260505 17:33:33.182 | INFO | "pinned" : " 29 GB" | 20260505 17:33:33.183 | INFO | } | 20260505 17:33:33.183 | INFO | Perf report for the last 5.13582 seconds: | 20260505 17:33:33.183 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:33:33.183 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:33.183 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:33.183 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:33.183 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 17:33:33.183 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:33.183 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:33.183 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 17:33:33.183 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:33.183 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:33.183 | INFO | Uploaded 36GiB to GPU in 2.12073s (16.9753GiB/s) | 20260505 17:33:33.183 | INFO | Downloaded 34GiB to GPU in 2.18537s (15.558GiB/s) | 20260505 17:33:38.332 | INFO | Analyzing chunk frames = [5033472,5033984) | 20260505 17:33:38.332 | INFO | MemInfo: { | 20260505 17:33:38.332 | INFO | "gpu" : " 17 GB", | 20260505 17:33:38.332 | INFO | "huge" : "", | 20260505 17:33:38.332 | INFO | "huge_shared" : " 27 GB", | 20260505 17:33:38.332 | INFO | "malloc" : " 14 GB", | 20260505 17:33:38.332 | INFO | "pinned" : " 29 GB" | 20260505 17:33:38.332 | INFO | } | 20260505 17:33:38.332 | INFO | Perf report for the last 5.14956 seconds: | 20260505 17:33:38.332 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:33:38.332 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:38.332 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:38.332 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:38.332 | INFO | ANALYSIS : 62.9%, 3.0, 98.5ms, 0 | 20260505 17:33:38.332 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:38.332 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:38.332 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:33:38.332 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:38.332 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:38.332 | INFO | Uploaded 36GiB to GPU in 2.15151s (16.7324GiB/s) | 20260505 17:33:38.333 | INFO | Downloaded 34GiB to GPU in 2.03356s (16.7195GiB/s) | 20260505 17:33:38.466 | INFO | Starting a round of Baz IO | 20260505 17:33:41.646 | INFO | Populating RTMetrics report | 20260505 17:33:41.646 | INFO | RTMetrics report populated | 20260505 17:33:41.646 | INFO | Opening tmp RTMetrics json file | 20260505 17:33:41.652 | INFO | Serializing RTMetrics to json | 20260505 17:33:41.655 | INFO | Serialization done, swapping tmp file now... | 20260505 17:33:41.658 | INFO | RTMetrics done | 20260505 17:33:41.742 | INFO | Finished a round of Baz IO: Wrote 2.128 GiB in 3.275 seconds | 20260505 17:33:41.742 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.04% / 0.00% / 77.96% | 20260505 17:33:41.742 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.294149% / 7.94729e-06% / 9.13938e-05% | 20260505 17:33:41.742 | INFO | Baz IO overall: | 20260505 17:33:41.742 | INFO | BytesWritten : 2975036549180 | 20260505 17:33:41.742 | INFO | NumEvents : 1410607571667 | 20260505 17:33:43.470 | INFO | Analyzing chunk frames = [5033984,5034496) | 20260505 17:33:43.470 | INFO | MemInfo: { | 20260505 17:33:43.470 | INFO | "gpu" : " 17 GB", | 20260505 17:33:43.470 | INFO | "huge" : "", | 20260505 17:33:43.470 | INFO | "huge_shared" : " 27 GB", | 20260505 17:33:43.470 | INFO | "malloc" : " 14 GB", | 20260505 17:33:43.470 | INFO | "pinned" : " 29 GB" | 20260505 17:33:43.470 | INFO | } | 20260505 17:33:43.470 | INFO | Perf report for the last 5.13749 seconds: | 20260505 17:33:43.470 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:33:43.470 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:43.470 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:43.470 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:43.470 | INFO | ANALYSIS : 63.4%, 3.0, 98.8ms, 0 | 20260505 17:33:43.470 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:43.470 | INFO | RT_METRICS : 44.7%, 1.0, 23.4ms, 0 | 20260505 17:33:43.470 | INFO | PRE_HQ : 60.3%, 1.8, 56.0ms, 0 | 20260505 17:33:43.470 | INFO | BAZWRITER : 63.3%, 3.8, 124.4ms, 0 | 20260505 17:33:43.470 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 17:33:43.470 | INFO | Uploaded 36GiB to GPU in 2.14649s (16.7716GiB/s) | 20260505 17:33:43.470 | INFO | Downloaded 38GiB to GPU in 2.23703s (16.9868GiB/s) | 20260505 17:33:48.613 | INFO | Analyzing chunk frames = [5034496,5035008) | 20260505 17:33:48.613 | INFO | MemInfo: { | 20260505 17:33:48.613 | INFO | "gpu" : " 17 GB", | 20260505 17:33:48.613 | INFO | "huge" : "", | 20260505 17:33:48.613 | INFO | "huge_shared" : " 27 GB", | 20260505 17:33:48.613 | INFO | "malloc" : " 14 GB", | 20260505 17:33:48.613 | INFO | "pinned" : " 29 GB" | 20260505 17:33:48.613 | INFO | } | 20260505 17:33:48.613 | INFO | Perf report for the last 5.14341 seconds: | 20260505 17:33:48.613 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:33:48.613 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:48.613 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:48.613 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:48.613 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260505 17:33:48.613 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:48.613 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:48.613 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:33:48.613 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:48.613 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:48.613 | INFO | Uploaded 36GiB to GPU in 2.12191s (16.9659GiB/s) | 20260505 17:33:48.613 | INFO | Downloaded 34GiB to GPU in 2.0181s (16.8476GiB/s) | 20260505 17:33:53.744 | INFO | Analyzing chunk frames = [5035008,5035520) | 20260505 17:33:53.744 | INFO | MemInfo: { | 20260505 17:33:53.744 | INFO | "gpu" : " 17 GB", | 20260505 17:33:53.744 | INFO | "huge" : "", | 20260505 17:33:53.744 | INFO | "huge_shared" : " 27 GB", | 20260505 17:33:53.744 | INFO | "malloc" : " 14 GB", | 20260505 17:33:53.744 | INFO | "pinned" : " 29 GB" | 20260505 17:33:53.744 | INFO | } | 20260505 17:33:53.744 | INFO | Perf report for the last 5.13124 seconds: | 20260505 17:33:53.744 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:33:53.744 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:53.744 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:53.744 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:53.744 | INFO | ANALYSIS : 62.7%, 3.0, 97.5ms, 0 | 20260505 17:33:53.744 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:53.744 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:53.744 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:33:53.744 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:53.744 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:53.745 | INFO | Uploaded 36GiB to GPU in 2.20755s (16.3077GiB/s) | 20260505 17:33:53.745 | INFO | Downloaded 34GiB to GPU in 2.06109s (16.4962GiB/s) | 20260505 17:33:58.886 | INFO | Analyzing chunk frames = [5035520,5036032) | 20260505 17:33:58.886 | INFO | MemInfo: { | 20260505 17:33:58.886 | INFO | "gpu" : " 17 GB", | 20260505 17:33:58.886 | INFO | "huge" : "", | 20260505 17:33:58.886 | INFO | "huge_shared" : " 27 GB", | 20260505 17:33:58.886 | INFO | "malloc" : " 14 GB", | 20260505 17:33:58.886 | INFO | "pinned" : " 29 GB" | 20260505 17:33:58.886 | INFO | } | 20260505 17:33:58.886 | INFO | Perf report for the last 5.14138 seconds: | 20260505 17:33:58.886 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:33:58.886 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:58.886 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:58.886 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:58.886 | INFO | ANALYSIS : 62.5%, 3.0, 97.1ms, 0 | 20260505 17:33:58.886 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:58.886 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:58.886 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:33:58.886 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:58.886 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:33:58.886 | INFO | Uploaded 36GiB to GPU in 2.18168s (16.501GiB/s) | 20260505 17:33:58.886 | INFO | Downloaded 34GiB to GPU in 2.14256s (15.8689GiB/s) | 20260505 17:34:04.025 | INFO | Analyzing chunk frames = [5036032,5036544) | 20260505 17:34:04.025 | INFO | MemInfo: { | 20260505 17:34:04.026 | INFO | "gpu" : " 17 GB", | 20260505 17:34:04.026 | INFO | "huge" : "", | 20260505 17:34:04.026 | INFO | "huge_shared" : " 27 GB", | 20260505 17:34:04.026 | INFO | "malloc" : " 14 GB", | 20260505 17:34:04.026 | INFO | "pinned" : " 29 GB" | 20260505 17:34:04.026 | INFO | } | 20260505 17:34:04.026 | INFO | Perf report for the last 5.13988 seconds: | 20260505 17:34:04.026 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:34:04.026 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:04.026 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:04.026 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:04.026 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 17:34:04.026 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:04.026 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:04.026 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:34:04.026 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:04.026 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:04.026 | INFO | Uploaded 36GiB to GPU in 2.13475s (16.8638GiB/s) | 20260505 17:34:04.026 | INFO | Downloaded 34GiB to GPU in 2.06038s (16.5018GiB/s) | 20260505 17:34:09.161 | INFO | Analyzing chunk frames = [5036544,5037056) | 20260505 17:34:09.161 | INFO | MemInfo: { | 20260505 17:34:09.161 | INFO | "gpu" : " 17 GB", | 20260505 17:34:09.161 | INFO | "huge" : "", | 20260505 17:34:09.161 | INFO | "huge_shared" : " 27 GB", | 20260505 17:34:09.161 | INFO | "malloc" : " 14 GB", | 20260505 17:34:09.161 | INFO | "pinned" : " 29 GB" | 20260505 17:34:09.161 | INFO | } | 20260505 17:34:09.161 | INFO | Perf report for the last 5.13552 seconds: | 20260505 17:34:09.161 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:34:09.161 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:09.161 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:09.161 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:09.161 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260505 17:34:09.161 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:09.161 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:09.161 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 17:34:09.161 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:09.161 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:09.161 | INFO | Uploaded 36GiB to GPU in 2.13743s (16.8426GiB/s) | 20260505 17:34:09.161 | INFO | Downloaded 34GiB to GPU in 2.04816s (16.6002GiB/s) | 20260505 17:34:14.296 | INFO | Analyzing chunk frames = [5037056,5037568) | 20260505 17:34:14.296 | INFO | MemInfo: { | 20260505 17:34:14.296 | INFO | "gpu" : " 17 GB", | 20260505 17:34:14.296 | INFO | "huge" : "", | 20260505 17:34:14.296 | INFO | "huge_shared" : " 27 GB", | 20260505 17:34:14.296 | INFO | "malloc" : " 14 GB", | 20260505 17:34:14.296 | INFO | "pinned" : " 29 GB" | 20260505 17:34:14.296 | INFO | } | 20260505 17:34:14.296 | INFO | Perf report for the last 5.13498 seconds: | 20260505 17:34:14.296 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:34:14.296 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:14.296 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:14.296 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:14.296 | INFO | ANALYSIS : 63.5%, 3.0, 99.0ms, 0 | 20260505 17:34:14.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:14.296 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:14.296 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 17:34:14.296 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:14.296 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:14.297 | INFO | Uploaded 36GiB to GPU in 2.17273s (16.569GiB/s) | 20260505 17:34:14.297 | INFO | Downloaded 34GiB to GPU in 2.17759s (15.6136GiB/s) | 20260505 17:34:19.434 | INFO | Analyzing chunk frames = [5037568,5038080) | 20260505 17:34:19.434 | INFO | MemInfo: { | 20260505 17:34:19.434 | INFO | "gpu" : " 17 GB", | 20260505 17:34:19.434 | INFO | "huge" : "", | 20260505 17:34:19.434 | INFO | "huge_shared" : " 27 GB", | 20260505 17:34:19.434 | INFO | "malloc" : " 14 GB", | 20260505 17:34:19.434 | INFO | "pinned" : " 29 GB" | 20260505 17:34:19.435 | INFO | } | 20260505 17:34:19.435 | INFO | Perf report for the last 5.13845 seconds: | 20260505 17:34:19.435 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:34:19.435 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:19.435 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:19.435 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:19.435 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 17:34:19.435 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:19.435 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:19.435 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:34:19.435 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:19.435 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:19.435 | INFO | Uploaded 36GiB to GPU in 2.15466s (16.7079GiB/s) | 20260505 17:34:19.435 | INFO | Downloaded 34GiB to GPU in 2.03467s (16.7103GiB/s) | 20260505 17:34:19.435 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 17:34:19.435 | INFO | Total mem usage: 15.3 GiB | 20260505 17:34:19.435 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 17:34:19.435 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.97%) | 20260505 17:34:19.435 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 17:34:19.435 | INFO | High Water Breakdown: | 20260505 17:34:19.435 | INFO | 61% : 8.66 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 17:34:19.435 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 17:34:19.435 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 17:34:19.435 | INFO | ------------------------------------------ | 20260505 17:34:19.435 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:34:19.435 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 17:34:19.435 | INFO | Total mem usage: 26.5 GiB | 20260505 17:34:19.435 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 17:34:19.435 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 17:34:19.435 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 17:34:19.435 | INFO | High Water Breakdown: | 20260505 17:34:19.435 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:34:19.435 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:34:19.436 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:34:19.436 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:34:19.436 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:34:19.436 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:34:19.436 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 17:34:19.436 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 17:34:19.436 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:34:19.436 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 17:34:19.436 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 17:34:19.436 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 17:34:19.436 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 17:34:19.436 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 17:34:19.436 | INFO | ------------------------------------------ | 20260505 17:34:19.436 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:34:19.436 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 17:34:19.436 | INFO | Total mem usage: 25.6 GiB | 20260505 17:34:19.436 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 17:34:19.436 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 17:34:19.436 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 17:34:19.436 | INFO | High Water Breakdown: | 20260505 17:34:19.436 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 17:34:19.436 | INFO | ------------------------------------------ | 20260505 17:34:19.436 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:34:19.436 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 17:34:19.436 | INFO | Total mem usage: 15.7 GiB | 20260505 17:34:19.436 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 17:34:19.436 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 17:34:19.436 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 17:34:19.436 | INFO | High Water Breakdown: | 20260505 17:34:19.436 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 17:34:19.436 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 17:34:19.436 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 17:34:19.436 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 17:34:19.436 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 17:34:19.436 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 17:34:19.436 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 17:34:19.436 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 17:34:19.436 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 17:34:19.436 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:34:19.436 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 17:34:19.436 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:34:19.436 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 17:34:19.436 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 17:34:19.436 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 17:34:19.436 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 17:34:19.436 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:34:19.436 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 17:34:19.436 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:34:19.436 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:34:19.436 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:34:19.436 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 17:34:19.436 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 17:34:19.436 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:34:19.436 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 17:34:19.436 | INFO | ------------------------------------------ | 20260505 17:34:19.436 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:34:19.436 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 17:34:19.436 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:34:19.436 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:34:19.436 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.54e+01, 1.36e+01 / 2.30e+00 | 20260505 17:34:19.436 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 1.02e-01, 1.10e-02 / 3.87e-03 | 20260505 17:34:19.436 | INFO | PerfReport: 11.78%, Baseline -- 3.73e+00 / 4.50e+00, 5.73e-01 / 1.76e-01 | 20260505 17:34:19.436 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.52e+00, 1.94e-01 / 8.16e-02 | 20260505 17:34:19.436 | INFO | PerfReport: 15.97%, DME -- 4.04e+01 / 4.51e+01, 4.46e+00 / 2.41e+00 | 20260505 17:34:19.436 | INFO | PerfReport: 56.85%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.50e+00 / 1.43e+00 | 20260505 17:34:19.436 | INFO | PerfReport: 7.29%, PulseAccumulating -- 2.31e+00 / 2.91e+00, 4.43e-01 / 1.84e-01 | 20260505 17:34:19.436 | INFO | PerfReport: 3.36%, Metrics -- 1.06e+00 / 1.41e+00, 2.98e-01 / 1.49e-01 | 20260505 17:34:19.436 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.71e-01 / 4.24e-01, 6.62e-02 / 3.18e-02 | 20260505 17:34:19.436 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 17:34:19.436 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:34:19.436 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:34:19.436 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260505 17:34:19.436 | INFO | PerfReport: 22.46%, Upload -- 2.24e+01 / 0.00e+00, 7.70e+00 / 0.00e+00 | 20260505 17:34:19.436 | INFO | PerfReport: 21.88%, Download -- 2.19e+01 / 0.00e+00, 7.82e+00 / 0.00e+00 | 20260505 17:34:19.436 | INFO | PerfReport: 31.68%, Compute -- 3.17e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 17:34:19.436 | INFO | PerfReport: 2.17%, LockUpload -- 2.17e+00 / 0.00e+00, 7.71e+00 / 0.00e+00 | 20260505 17:34:19.436 | INFO | PerfReport: 1.47%, LockDownload -- 1.47e+00 / 0.00e+00, 3.05e+00 / 0.00e+00 | 20260505 17:34:19.436 | INFO | PerfReport: 20.33%, LockCompute -- 2.03e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 17:34:19.436 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 1.98e-02 / 0.00e+00 | 20260505 17:34:19.568 | INFO | Starting a round of Baz IO | 20260505 17:34:22.765 | INFO | Populating RTMetrics report | 20260505 17:34:22.766 | INFO | RTMetrics report populated | 20260505 17:34:22.766 | INFO | Opening tmp RTMetrics json file | 20260505 17:34:22.772 | INFO | Serializing RTMetrics to json | 20260505 17:34:22.773 | INFO | Serialization done, swapping tmp file now... | 20260505 17:34:22.774 | INFO | RTMetrics done | 20260505 17:34:22.858 | INFO | Finished a round of Baz IO: Wrote 2.126 GiB in 3.289 seconds | 20260505 17:34:22.858 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.04% / 0.00% / 77.96% | 20260505 17:34:22.858 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.292997% / 0% / 9.13938e-05% | 20260505 17:34:22.858 | INFO | Baz IO overall: | 20260505 17:34:22.858 | INFO | BytesWritten : 2977319265319 | 20260505 17:34:22.858 | INFO | NumEvents : 1411674101368 | 20260505 17:34:24.585 | INFO | Analyzing chunk frames = [5038080,5038592) | 20260505 17:34:24.585 | INFO | MemInfo: { | 20260505 17:34:24.585 | INFO | "gpu" : " 17 GB", | 20260505 17:34:24.585 | INFO | "huge" : "", | 20260505 17:34:24.585 | INFO | "huge_shared" : " 27 GB", | 20260505 17:34:24.585 | INFO | "malloc" : " 14 GB", | 20260505 17:34:24.585 | INFO | "pinned" : " 29 GB" | 20260505 17:34:24.585 | INFO | } | 20260505 17:34:24.585 | INFO | Perf report for the last 5.1501 seconds: | 20260505 17:34:24.585 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:34:24.585 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:24.585 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:24.585 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:24.585 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 17:34:24.585 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:24.585 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260505 17:34:24.585 | INFO | PRE_HQ : 60.5%, 1.8, 55.9ms, 0 | 20260505 17:34:24.585 | INFO | BAZWRITER : 63.2%, 3.8, 124.4ms, 0 | 20260505 17:34:24.585 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 17:34:24.585 | INFO | Uploaded 36GiB to GPU in 2.17661s (16.5395GiB/s) | 20260505 17:34:24.585 | INFO | Downloaded 38GiB to GPU in 2.26205s (16.7989GiB/s) | 20260505 17:34:29.727 | INFO | Analyzing chunk frames = [5038592,5039104) | 20260505 17:34:29.727 | INFO | MemInfo: { | 20260505 17:34:29.727 | INFO | "gpu" : " 17 GB", | 20260505 17:34:29.727 | INFO | "huge" : "", | 20260505 17:34:29.727 | INFO | "huge_shared" : " 27 GB", | 20260505 17:34:29.727 | INFO | "malloc" : " 14 GB", | 20260505 17:34:29.727 | INFO | "pinned" : " 29 GB" | 20260505 17:34:29.727 | INFO | } | 20260505 17:34:29.727 | INFO | Perf report for the last 5.14215 seconds: | 20260505 17:34:29.727 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:34:29.727 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:29.727 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:29.727 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:29.727 | INFO | ANALYSIS : 62.7%, 3.0, 98.0ms, 0 | 20260505 17:34:29.727 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:29.727 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:29.727 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 17:34:29.727 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:29.727 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:29.727 | INFO | Uploaded 36GiB to GPU in 2.137s (16.8461GiB/s) | 20260505 17:34:29.727 | INFO | Downloaded 34GiB to GPU in 2.02824s (16.7633GiB/s) | 20260505 17:34:34.859 | INFO | Analyzing chunk frames = [5039104,5039616) | 20260505 17:34:34.859 | INFO | MemInfo: { | 20260505 17:34:34.859 | INFO | "gpu" : " 17 GB", | 20260505 17:34:34.859 | INFO | "huge" : "", | 20260505 17:34:34.859 | INFO | "huge_shared" : " 27 GB", | 20260505 17:34:34.859 | INFO | "malloc" : " 14 GB", | 20260505 17:34:34.859 | INFO | "pinned" : " 29 GB" | 20260505 17:34:34.859 | INFO | } | 20260505 17:34:34.859 | INFO | Perf report for the last 5.13252 seconds: | 20260505 17:34:34.860 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:34:34.860 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:34.860 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:34.860 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:34.860 | INFO | ANALYSIS : 62.7%, 3.0, 97.4ms, 0 | 20260505 17:34:34.860 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:34.860 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:34.860 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:34:34.860 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:34.860 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:34.860 | INFO | Uploaded 36GiB to GPU in 2.20081s (16.3576GiB/s) | 20260505 17:34:34.860 | INFO | Downloaded 34GiB to GPU in 2.03337s (16.721GiB/s) | 20260505 17:34:39.992 | INFO | Analyzing chunk frames = [5039616,5040128) | 20260505 17:34:39.992 | INFO | MemInfo: { | 20260505 17:34:39.992 | INFO | "gpu" : " 17 GB", | 20260505 17:34:39.992 | INFO | "huge" : "", | 20260505 17:34:39.992 | INFO | "huge_shared" : " 27 GB", | 20260505 17:34:39.992 | INFO | "malloc" : " 14 GB", | 20260505 17:34:39.992 | INFO | "pinned" : " 29 GB" | 20260505 17:34:39.992 | INFO | } | 20260505 17:34:39.992 | INFO | Perf report for the last 5.13279 seconds: | 20260505 17:34:39.992 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:34:39.992 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:39.992 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:39.992 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:39.992 | INFO | ANALYSIS : 62.7%, 3.0, 97.3ms, 0 | 20260505 17:34:39.992 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:39.992 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:39.992 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 17:34:39.992 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:39.992 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:39.993 | INFO | Uploaded 36GiB to GPU in 2.21923s (16.2218GiB/s) | 20260505 17:34:39.993 | INFO | Downloaded 34GiB to GPU in 2.13184s (15.9486GiB/s) | 20260505 17:34:45.139 | INFO | Analyzing chunk frames = [5040128,5040640) | 20260505 17:34:45.139 | INFO | MemInfo: { | 20260505 17:34:45.139 | INFO | "gpu" : " 17 GB", | 20260505 17:34:45.139 | INFO | "huge" : "", | 20260505 17:34:45.139 | INFO | "huge_shared" : " 27 GB", | 20260505 17:34:45.139 | INFO | "malloc" : " 14 GB", | 20260505 17:34:45.139 | INFO | "pinned" : " 29 GB" | 20260505 17:34:45.139 | INFO | } | 20260505 17:34:45.139 | INFO | Perf report for the last 5.14673 seconds: | 20260505 17:34:45.139 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:34:45.139 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:45.139 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:45.139 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:45.139 | INFO | ANALYSIS : 62.2%, 3.0, 97.3ms, 0 | 20260505 17:34:45.139 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:45.139 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:45.139 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 17:34:45.139 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:45.139 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:45.139 | INFO | Uploaded 36GiB to GPU in 2.1637s (16.6382GiB/s) | 20260505 17:34:45.139 | INFO | Downloaded 34GiB to GPU in 2.05726s (16.5268GiB/s) | 20260505 17:34:50.276 | INFO | Analyzing chunk frames = [5040640,5041152) | 20260505 17:34:50.276 | INFO | MemInfo: { | 20260505 17:34:50.276 | INFO | "gpu" : " 17 GB", | 20260505 17:34:50.276 | INFO | "huge" : "", | 20260505 17:34:50.276 | INFO | "huge_shared" : " 27 GB", | 20260505 17:34:50.276 | INFO | "malloc" : " 14 GB", | 20260505 17:34:50.276 | INFO | "pinned" : " 29 GB" | 20260505 17:34:50.276 | INFO | } | 20260505 17:34:50.276 | INFO | Perf report for the last 5.1371 seconds: | 20260505 17:34:50.276 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:34:50.276 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:50.276 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:50.276 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:50.276 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 17:34:50.276 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:50.276 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:50.276 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 17:34:50.276 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:50.276 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:50.276 | INFO | Uploaded 36GiB to GPU in 2.14051s (16.8184GiB/s) | 20260505 17:34:50.276 | INFO | Downloaded 34GiB to GPU in 2.04524s (16.624GiB/s) | 20260505 17:34:55.419 | INFO | Analyzing chunk frames = [5041152,5041664) | 20260505 17:34:55.419 | INFO | MemInfo: { | 20260505 17:34:55.419 | INFO | "gpu" : " 17 GB", | 20260505 17:34:55.419 | INFO | "huge" : "", | 20260505 17:34:55.419 | INFO | "huge_shared" : " 27 GB", | 20260505 17:34:55.419 | INFO | "malloc" : " 14 GB", | 20260505 17:34:55.419 | INFO | "pinned" : " 29 GB" | 20260505 17:34:55.419 | INFO | } | 20260505 17:34:55.419 | INFO | Perf report for the last 5.14315 seconds: | 20260505 17:34:55.419 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:34:55.419 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:55.419 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:55.419 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:55.419 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 17:34:55.419 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:55.419 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:55.419 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 17:34:55.420 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:55.420 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:34:55.420 | INFO | Uploaded 36GiB to GPU in 2.14811s (16.7589GiB/s) | 20260505 17:34:55.420 | INFO | Downloaded 34GiB to GPU in 2.13933s (15.8929GiB/s) | 20260505 17:35:00.555 | INFO | Analyzing chunk frames = [5041664,5042176) | 20260505 17:35:00.555 | INFO | MemInfo: { | 20260505 17:35:00.555 | INFO | "gpu" : " 17 GB", | 20260505 17:35:00.555 | INFO | "huge" : "", | 20260505 17:35:00.555 | INFO | "huge_shared" : " 27 GB", | 20260505 17:35:00.555 | INFO | "malloc" : " 14 GB", | 20260505 17:35:00.555 | INFO | "pinned" : " 29 GB" | 20260505 17:35:00.555 | INFO | } | 20260505 17:35:00.555 | INFO | Perf report for the last 5.13552 seconds: | 20260505 17:35:00.555 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:35:00.555 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:00.555 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:00.555 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:00.555 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 17:35:00.555 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:00.555 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:00.555 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:35:00.555 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:00.555 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:00.555 | INFO | Uploaded 36GiB to GPU in 2.19998s (16.3638GiB/s) | 20260505 17:35:00.555 | INFO | Downloaded 34GiB to GPU in 2.03441s (16.7124GiB/s) | 20260505 17:35:00.690 | INFO | Starting a round of Baz IO | 20260505 17:35:03.889 | INFO | Populating RTMetrics report | 20260505 17:35:03.889 | INFO | RTMetrics report populated | 20260505 17:35:03.889 | INFO | Opening tmp RTMetrics json file | 20260505 17:35:03.894 | INFO | Serializing RTMetrics to json | 20260505 17:35:03.896 | INFO | Serialization done, swapping tmp file now... | 20260505 17:35:03.898 | INFO | RTMetrics done | 20260505 17:35:03.982 | INFO | Finished a round of Baz IO: Wrote 2.125 GiB in 3.291 seconds | 20260505 17:35:03.982 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.04% / 0.00% / 77.96% | 20260505 17:35:03.982 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.292118% / 3.97364e-06% / 7.54992e-05% | 20260505 17:35:03.982 | INFO | Baz IO overall: | 20260505 17:35:03.982 | INFO | BytesWritten : 2979601076190 | 20260505 17:35:03.982 | INFO | NumEvents : 1412740247428 | 20260505 17:35:05.692 | INFO | Analyzing chunk frames = [5042176,5042688) | 20260505 17:35:05.692 | INFO | MemInfo: { | 20260505 17:35:05.692 | INFO | "gpu" : " 17 GB", | 20260505 17:35:05.692 | INFO | "huge" : "", | 20260505 17:35:05.692 | INFO | "huge_shared" : " 27 GB", | 20260505 17:35:05.692 | INFO | "malloc" : " 14 GB", | 20260505 17:35:05.692 | INFO | "pinned" : " 29 GB" | 20260505 17:35:05.692 | INFO | } | 20260505 17:35:05.692 | INFO | Perf report for the last 5.13761 seconds: | 20260505 17:35:05.692 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:35:05.692 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:05.692 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:05.693 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:05.693 | INFO | ANALYSIS : 63.8%, 3.0, 99.4ms, 0 | 20260505 17:35:05.693 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:05.693 | INFO | RT_METRICS : 44.9%, 1.0, 23.6ms, 0 | 20260505 17:35:05.693 | INFO | PRE_HQ : 61.0%, 1.8, 56.3ms, 0 | 20260505 17:35:05.693 | INFO | BAZWRITER : 63.4%, 3.8, 125.2ms, 0 | 20260505 17:35:05.693 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 17:35:05.693 | INFO | Uploaded 36GiB to GPU in 2.20409s (16.3333GiB/s) | 20260505 17:35:05.693 | INFO | Downloaded 38GiB to GPU in 2.27078s (16.7343GiB/s) | 20260505 17:35:10.840 | INFO | Analyzing chunk frames = [5042688,5043200) | 20260505 17:35:10.840 | INFO | MemInfo: { | 20260505 17:35:10.840 | INFO | "gpu" : " 17 GB", | 20260505 17:35:10.840 | INFO | "huge" : "", | 20260505 17:35:10.840 | INFO | "huge_shared" : " 27 GB", | 20260505 17:35:10.840 | INFO | "malloc" : " 14 GB", | 20260505 17:35:10.840 | INFO | "pinned" : " 29 GB" | 20260505 17:35:10.840 | INFO | } | 20260505 17:35:10.840 | INFO | Perf report for the last 5.14805 seconds: | 20260505 17:35:10.841 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:35:10.841 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:10.841 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:10.841 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:10.841 | INFO | ANALYSIS : 62.6%, 3.0, 97.9ms, 0 | 20260505 17:35:10.841 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:10.841 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:10.841 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 17:35:10.841 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:10.841 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:10.841 | INFO | Uploaded 36GiB to GPU in 2.12096s (16.9735GiB/s) | 20260505 17:35:10.841 | INFO | Downloaded 34GiB to GPU in 2.0285s (16.7612GiB/s) | 20260505 17:35:15.983 | INFO | Analyzing chunk frames = [5043200,5043712) | 20260505 17:35:15.983 | INFO | MemInfo: { | 20260505 17:35:15.983 | INFO | "gpu" : " 17 GB", | 20260505 17:35:15.983 | INFO | "huge" : "", | 20260505 17:35:15.983 | INFO | "huge_shared" : " 27 GB", | 20260505 17:35:15.983 | INFO | "malloc" : " 14 GB", | 20260505 17:35:15.983 | INFO | "pinned" : " 29 GB" | 20260505 17:35:15.983 | INFO | } | 20260505 17:35:15.983 | INFO | Perf report for the last 5.14273 seconds: | 20260505 17:35:15.983 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:35:15.983 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:15.983 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:15.983 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:15.983 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260505 17:35:15.983 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:15.983 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:15.983 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 17:35:15.983 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:15.983 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:15.984 | INFO | Uploaded 36GiB to GPU in 2.12777s (16.9191GiB/s) | 20260505 17:35:15.984 | INFO | Downloaded 34GiB to GPU in 2.02949s (16.753GiB/s) | 20260505 17:35:21.108 | INFO | Analyzing chunk frames = [5043712,5044224) | 20260505 17:35:21.108 | INFO | MemInfo: { | 20260505 17:35:21.108 | INFO | "gpu" : " 17 GB", | 20260505 17:35:21.108 | INFO | "huge" : "", | 20260505 17:35:21.108 | INFO | "huge_shared" : " 27 GB", | 20260505 17:35:21.108 | INFO | "malloc" : " 14 GB", | 20260505 17:35:21.108 | INFO | "pinned" : " 29 GB" | 20260505 17:35:21.108 | INFO | } | 20260505 17:35:21.108 | INFO | Perf report for the last 5.12532 seconds: | 20260505 17:35:21.109 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:35:21.109 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:21.109 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:21.109 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:21.109 | INFO | ANALYSIS : 62.5%, 3.0, 96.8ms, 0 | 20260505 17:35:21.109 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:21.109 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:21.109 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 17:35:21.109 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:21.109 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:21.109 | INFO | Uploaded 36GiB to GPU in 2.17037s (16.5871GiB/s) | 20260505 17:35:21.109 | INFO | Downloaded 34GiB to GPU in 2.05453s (16.5488GiB/s) | 20260505 17:35:26.245 | INFO | Analyzing chunk frames = [5044224,5044736) | 20260505 17:35:26.245 | INFO | MemInfo: { | 20260505 17:35:26.245 | INFO | "gpu" : " 17 GB", | 20260505 17:35:26.245 | INFO | "huge" : "", | 20260505 17:35:26.245 | INFO | "huge_shared" : " 27 GB", | 20260505 17:35:26.245 | INFO | "malloc" : " 14 GB", | 20260505 17:35:26.245 | INFO | "pinned" : " 29 GB" | 20260505 17:35:26.245 | INFO | } | 20260505 17:35:26.245 | INFO | Perf report for the last 5.13657 seconds: | 20260505 17:35:26.245 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:35:26.245 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:26.245 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:26.245 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:26.245 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 17:35:26.245 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:26.245 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:26.245 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 17:35:26.245 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:26.245 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:26.245 | INFO | Uploaded 36GiB to GPU in 2.15299s (16.7209GiB/s) | 20260505 17:35:26.245 | INFO | Downloaded 34GiB to GPU in 2.0595s (16.5089GiB/s) | 20260505 17:35:31.390 | INFO | Analyzing chunk frames = [5044736,5045248) | 20260505 17:35:31.390 | INFO | MemInfo: { | 20260505 17:35:31.390 | INFO | "gpu" : " 17 GB", | 20260505 17:35:31.390 | INFO | "huge" : "", | 20260505 17:35:31.390 | INFO | "huge_shared" : " 27 GB", | 20260505 17:35:31.390 | INFO | "malloc" : " 14 GB", | 20260505 17:35:31.390 | INFO | "pinned" : " 29 GB" | 20260505 17:35:31.390 | INFO | } | 20260505 17:35:31.390 | INFO | Perf report for the last 5.14498 seconds: | 20260505 17:35:31.390 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:35:31.390 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:31.390 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:31.390 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:31.390 | INFO | ANALYSIS : 62.4%, 3.0, 97.6ms, 0 | 20260505 17:35:31.390 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:31.390 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:31.390 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 17:35:31.390 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:31.390 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:31.390 | INFO | Uploaded 36GiB to GPU in 2.14089s (16.8155GiB/s) | 20260505 17:35:31.390 | INFO | Downloaded 34GiB to GPU in 2.03704s (16.6908GiB/s) | 20260505 17:35:36.534 | INFO | Analyzing chunk frames = [5045248,5045760) | 20260505 17:35:36.534 | INFO | MemInfo: { | 20260505 17:35:36.534 | INFO | "gpu" : " 17 GB", | 20260505 17:35:36.534 | INFO | "huge" : "", | 20260505 17:35:36.534 | INFO | "huge_shared" : " 27 GB", | 20260505 17:35:36.534 | INFO | "malloc" : " 14 GB", | 20260505 17:35:36.534 | INFO | "pinned" : " 29 GB" | 20260505 17:35:36.534 | INFO | } | 20260505 17:35:36.535 | INFO | Perf report for the last 5.1445 seconds: | 20260505 17:35:36.535 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:35:36.535 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:36.535 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:36.535 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:36.535 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 17:35:36.535 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:36.535 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:36.535 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 17:35:36.535 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:36.535 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:36.535 | INFO | Uploaded 36GiB to GPU in 2.11829s (16.9949GiB/s) | 20260505 17:35:36.535 | INFO | Downloaded 34GiB to GPU in 2.1797s (15.5985GiB/s) | 20260505 17:35:41.672 | INFO | Analyzing chunk frames = [5045760,5046272) | 20260505 17:35:41.672 | INFO | MemInfo: { | 20260505 17:35:41.672 | INFO | "gpu" : " 17 GB", | 20260505 17:35:41.672 | INFO | "huge" : "", | 20260505 17:35:41.672 | INFO | "huge_shared" : " 27 GB", | 20260505 17:35:41.672 | INFO | "malloc" : " 14 GB", | 20260505 17:35:41.672 | INFO | "pinned" : " 29 GB" | 20260505 17:35:41.672 | INFO | } | 20260505 17:35:41.672 | INFO | Perf report for the last 5.13766 seconds: | 20260505 17:35:41.672 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:35:41.672 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:41.672 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:41.672 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:41.672 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 17:35:41.672 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:41.672 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:41.672 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:35:41.673 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:41.673 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:41.673 | INFO | Uploaded 36GiB to GPU in 2.19652s (16.3895GiB/s) | 20260505 17:35:41.673 | INFO | Downloaded 34GiB to GPU in 2.03891s (16.6756GiB/s) | 20260505 17:35:41.673 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 17:35:41.673 | INFO | Total mem usage: 15.3 GiB | 20260505 17:35:41.673 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 17:35:41.673 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.97%) | 20260505 17:35:41.673 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 17:35:41.673 | INFO | High Water Breakdown: | 20260505 17:35:41.673 | INFO | 61% : 8.66 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 17:35:41.673 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 17:35:41.673 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 17:35:41.673 | INFO | ------------------------------------------ | 20260505 17:35:41.673 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:35:41.673 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 17:35:41.673 | INFO | Total mem usage: 26.5 GiB | 20260505 17:35:41.673 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 17:35:41.673 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 17:35:41.673 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 17:35:41.673 | INFO | High Water Breakdown: | 20260505 17:35:41.673 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:35:41.673 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:35:41.673 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:35:41.673 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:35:41.673 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:35:41.673 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:35:41.673 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 17:35:41.673 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 17:35:41.673 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:35:41.673 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 17:35:41.673 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 17:35:41.673 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 17:35:41.673 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 17:35:41.673 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 17:35:41.673 | INFO | ------------------------------------------ | 20260505 17:35:41.673 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:35:41.673 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 17:35:41.673 | INFO | Total mem usage: 25.6 GiB | 20260505 17:35:41.673 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 17:35:41.673 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 17:35:41.673 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 17:35:41.673 | INFO | High Water Breakdown: | 20260505 17:35:41.673 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 17:35:41.673 | INFO | ------------------------------------------ | 20260505 17:35:41.673 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:35:41.673 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 17:35:41.673 | INFO | Total mem usage: 15.7 GiB | 20260505 17:35:41.673 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 17:35:41.673 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 17:35:41.673 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 17:35:41.673 | INFO | High Water Breakdown: | 20260505 17:35:41.673 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 17:35:41.673 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 17:35:41.673 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 17:35:41.674 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 17:35:41.674 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 17:35:41.674 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 17:35:41.674 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 17:35:41.674 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 17:35:41.674 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 17:35:41.674 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:35:41.674 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 17:35:41.674 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:35:41.674 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 17:35:41.674 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 17:35:41.674 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 17:35:41.674 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 17:35:41.674 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:35:41.674 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 17:35:41.674 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:35:41.674 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:35:41.674 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:35:41.674 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 17:35:41.674 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 17:35:41.674 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:35:41.674 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 17:35:41.674 | INFO | ------------------------------------------ | 20260505 17:35:41.674 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:35:41.674 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 17:35:41.674 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:35:41.674 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:35:41.674 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.48e+01, 1.34e+01 / 2.02e+00 | 20260505 17:35:41.674 | INFO | PerfReport: 0.29%, AggregateStats -- 9.30e-02 / 1.02e-01, 1.07e-02 / 5.10e-03 | 20260505 17:35:41.674 | INFO | PerfReport: 11.81%, Baseline -- 3.73e+00 / 4.53e+00, 5.75e-01 / 1.28e-01 | 20260505 17:35:41.674 | INFO | PerfReport: 3.29%, Histogram -- 1.04e+00 / 1.52e+00, 1.90e-01 / 9.19e-02 | 20260505 17:35:41.674 | INFO | PerfReport: 15.94%, DME -- 4.03e+01 / 4.47e+01, 4.04e+00 / 1.99e+00 | 20260505 17:35:41.674 | INFO | PerfReport: 56.76%, FrameLabeling -- 1.79e+01 / 1.91e+01, 1.43e+00 / 1.13e+00 | 20260505 17:35:41.674 | INFO | PerfReport: 7.36%, PulseAccumulating -- 2.32e+00 / 2.95e+00, 4.52e-01 / 1.45e-01 | 20260505 17:35:41.674 | INFO | PerfReport: 3.38%, Metrics -- 1.07e+00 / 1.44e+00, 2.99e-01 / 1.44e-01 | 20260505 17:35:41.674 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.70e-01 / 4.33e-01, 6.57e-02 / 3.86e-02 | 20260505 17:35:41.674 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 17:35:41.674 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:35:41.674 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:35:41.674 | INFO | PerfReport: Full Execution -- 9.99e+01 / -nan, 2.56e+01 / 0.00e+00 | 20260505 17:35:41.674 | INFO | PerfReport: 22.55%, Upload -- 2.25e+01 / 0.00e+00, 7.86e+00 / 0.00e+00 | 20260505 17:35:41.674 | INFO | PerfReport: 21.78%, Download -- 2.18e+01 / 0.00e+00, 7.81e+00 / 0.00e+00 | 20260505 17:35:41.674 | INFO | PerfReport: 31.61%, Compute -- 3.16e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 17:35:41.674 | INFO | PerfReport: 2.32%, LockUpload -- 2.32e+00 / 0.00e+00, 8.07e+00 / 0.00e+00 | 20260505 17:35:41.674 | INFO | PerfReport: 1.38%, LockDownload -- 1.38e+00 / 0.00e+00, 2.89e+00 / 0.00e+00 | 20260505 17:35:41.674 | INFO | PerfReport: 20.35%, LockCompute -- 2.03e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260505 17:35:41.674 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.72e-02 / 0.00e+00 | 20260505 17:35:41.809 | INFO | Starting a round of Baz IO | 20260505 17:35:44.990 | INFO | Populating RTMetrics report | 20260505 17:35:44.990 | INFO | RTMetrics report populated | 20260505 17:35:44.990 | INFO | Opening tmp RTMetrics json file | 20260505 17:35:44.996 | INFO | Serializing RTMetrics to json | 20260505 17:35:44.998 | INFO | Serialization done, swapping tmp file now... | 20260505 17:35:44.998 | INFO | RTMetrics done | 20260505 17:35:45.085 | INFO | Finished a round of Baz IO: Wrote 2.124 GiB in 3.275 seconds | 20260505 17:35:45.085 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.04% / 0.00% / 77.96% | 20260505 17:35:45.085 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.290974% / 7.94729e-06% / 7.94729e-05% | 20260505 17:35:45.085 | INFO | Baz IO overall: | 20260505 17:35:45.085 | INFO | BytesWritten : 2981881777564 | 20260505 17:35:45.085 | INFO | NumEvents : 1413805320178 | 20260505 17:35:46.804 | INFO | Analyzing chunk frames = [5046272,5046784) | 20260505 17:35:46.804 | INFO | MemInfo: { | 20260505 17:35:46.804 | INFO | "gpu" : " 17 GB", | 20260505 17:35:46.804 | INFO | "huge" : "", | 20260505 17:35:46.804 | INFO | "huge_shared" : " 27 GB", | 20260505 17:35:46.804 | INFO | "malloc" : " 14 GB", | 20260505 17:35:46.804 | INFO | "pinned" : " 29 GB" | 20260505 17:35:46.804 | INFO | } | 20260505 17:35:46.804 | INFO | Perf report for the last 5.13151 seconds: | 20260505 17:35:46.804 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:35:46.804 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:46.804 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:46.804 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:46.804 | INFO | ANALYSIS : 63.6%, 3.0, 98.9ms, 0 | 20260505 17:35:46.804 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:46.804 | INFO | RT_METRICS : 44.7%, 1.0, 23.4ms, 0 | 20260505 17:35:46.804 | INFO | PRE_HQ : 60.6%, 1.8, 56.2ms, 0 | 20260505 17:35:46.804 | INFO | BAZWRITER : 63.1%, 3.8, 124.6ms, 0 | 20260505 17:35:46.804 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 17:35:46.804 | INFO | Uploaded 36GiB to GPU in 2.1551s (16.7046GiB/s) | 20260505 17:35:46.804 | INFO | Downloaded 38GiB to GPU in 2.25774s (16.831GiB/s) | 20260505 17:35:51.943 | INFO | Analyzing chunk frames = [5046784,5047296) | 20260505 17:35:51.943 | INFO | MemInfo: { | 20260505 17:35:51.943 | INFO | "gpu" : " 17 GB", | 20260505 17:35:51.943 | INFO | "huge" : "", | 20260505 17:35:51.943 | INFO | "huge_shared" : " 27 GB", | 20260505 17:35:51.943 | INFO | "malloc" : " 14 GB", | 20260505 17:35:51.943 | INFO | "pinned" : " 29 GB" | 20260505 17:35:51.943 | INFO | } | 20260505 17:35:51.943 | INFO | Perf report for the last 5.13933 seconds: | 20260505 17:35:51.943 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:35:51.943 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:51.943 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:51.943 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:51.943 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 17:35:51.943 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:51.943 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:51.943 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 17:35:51.943 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:51.943 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:51.944 | INFO | Uploaded 36GiB to GPU in 2.13203s (16.8853GiB/s) | 20260505 17:35:51.944 | INFO | Downloaded 34GiB to GPU in 2.03248s (16.7284GiB/s) | 20260505 17:35:57.090 | INFO | Analyzing chunk frames = [5047296,5047808) | 20260505 17:35:57.090 | INFO | MemInfo: { | 20260505 17:35:57.090 | INFO | "gpu" : " 17 GB", | 20260505 17:35:57.090 | INFO | "huge" : "", | 20260505 17:35:57.090 | INFO | "huge_shared" : " 27 GB", | 20260505 17:35:57.090 | INFO | "malloc" : " 14 GB", | 20260505 17:35:57.090 | INFO | "pinned" : " 29 GB" | 20260505 17:35:57.090 | INFO | } | 20260505 17:35:57.090 | INFO | Perf report for the last 5.14706 seconds: | 20260505 17:35:57.090 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:35:57.090 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:57.090 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:57.090 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:57.090 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 17:35:57.090 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:57.090 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:57.090 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 17:35:57.090 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:57.090 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:35:57.091 | INFO | Uploaded 36GiB to GPU in 2.19782s (16.3799GiB/s) | 20260505 17:35:57.091 | INFO | Downloaded 34GiB to GPU in 2.02533s (16.7874GiB/s) | 20260505 17:36:02.230 | INFO | Analyzing chunk frames = [5047808,5048320) | 20260505 17:36:02.230 | INFO | MemInfo: { | 20260505 17:36:02.230 | INFO | "gpu" : " 17 GB", | 20260505 17:36:02.230 | INFO | "huge" : "", | 20260505 17:36:02.230 | INFO | "huge_shared" : " 27 GB", | 20260505 17:36:02.230 | INFO | "malloc" : " 14 GB", | 20260505 17:36:02.230 | INFO | "pinned" : " 29 GB" | 20260505 17:36:02.230 | INFO | } | 20260505 17:36:02.230 | INFO | Perf report for the last 5.14007 seconds: | 20260505 17:36:02.230 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:36:02.230 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:02.230 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:02.230 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:02.230 | INFO | ANALYSIS : 62.8%, 3.0, 97.6ms, 0 | 20260505 17:36:02.231 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:02.231 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:02.231 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:36:02.231 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:02.231 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:02.231 | INFO | Uploaded 36GiB to GPU in 2.22877s (16.1524GiB/s) | 20260505 17:36:02.231 | INFO | Downloaded 34GiB to GPU in 2.11473s (16.0777GiB/s) | 20260505 17:36:07.368 | INFO | Analyzing chunk frames = [5048320,5048832) | 20260505 17:36:07.368 | INFO | MemInfo: { | 20260505 17:36:07.368 | INFO | "gpu" : " 17 GB", | 20260505 17:36:07.368 | INFO | "huge" : "", | 20260505 17:36:07.368 | INFO | "huge_shared" : " 27 GB", | 20260505 17:36:07.368 | INFO | "malloc" : " 14 GB", | 20260505 17:36:07.368 | INFO | "pinned" : " 29 GB" | 20260505 17:36:07.368 | INFO | } | 20260505 17:36:07.368 | INFO | Perf report for the last 5.13819 seconds: | 20260505 17:36:07.369 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:36:07.369 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:07.369 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:07.369 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:07.369 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 17:36:07.369 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:07.369 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:07.369 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 17:36:07.369 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:07.369 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:07.369 | INFO | Uploaded 36GiB to GPU in 2.15011s (16.7434GiB/s) | 20260505 17:36:07.369 | INFO | Downloaded 34GiB to GPU in 2.0585s (16.5168GiB/s) | 20260505 17:36:12.505 | INFO | Analyzing chunk frames = [5048832,5049344) | 20260505 17:36:12.505 | INFO | MemInfo: { | 20260505 17:36:12.505 | INFO | "gpu" : " 17 GB", | 20260505 17:36:12.505 | INFO | "huge" : "", | 20260505 17:36:12.505 | INFO | "huge_shared" : " 27 GB", | 20260505 17:36:12.505 | INFO | "malloc" : " 14 GB", | 20260505 17:36:12.505 | INFO | "pinned" : " 29 GB" | 20260505 17:36:12.505 | INFO | } | 20260505 17:36:12.505 | INFO | Perf report for the last 5.13696 seconds: | 20260505 17:36:12.506 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:36:12.506 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:12.506 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:12.506 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:12.506 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260505 17:36:12.506 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:12.506 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:12.506 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 17:36:12.506 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:12.506 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:12.506 | INFO | Uploaded 36GiB to GPU in 2.14683s (16.7689GiB/s) | 20260505 17:36:12.506 | INFO | Downloaded 34GiB to GPU in 2.03767s (16.6857GiB/s) | 20260505 17:36:17.649 | INFO | Analyzing chunk frames = [5049344,5049856) | 20260505 17:36:17.649 | INFO | MemInfo: { | 20260505 17:36:17.649 | INFO | "gpu" : " 17 GB", | 20260505 17:36:17.649 | INFO | "huge" : "", | 20260505 17:36:17.649 | INFO | "huge_shared" : " 27 GB", | 20260505 17:36:17.649 | INFO | "malloc" : " 14 GB", | 20260505 17:36:17.649 | INFO | "pinned" : " 29 GB" | 20260505 17:36:17.649 | INFO | } | 20260505 17:36:17.649 | INFO | Perf report for the last 5.14341 seconds: | 20260505 17:36:17.649 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:36:17.649 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:17.649 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:17.649 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:17.649 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 17:36:17.649 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:17.649 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:17.649 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 17:36:17.649 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:17.649 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:17.649 | INFO | Uploaded 36GiB to GPU in 2.17344s (16.5636GiB/s) | 20260505 17:36:17.649 | INFO | Downloaded 34GiB to GPU in 2.18725s (15.5446GiB/s) | 20260505 17:36:22.787 | INFO | Analyzing chunk frames = [5049856,5050368) | 20260505 17:36:22.787 | INFO | MemInfo: { | 20260505 17:36:22.787 | INFO | "gpu" : " 17 GB", | 20260505 17:36:22.787 | INFO | "huge" : "", | 20260505 17:36:22.787 | INFO | "huge_shared" : " 27 GB", | 20260505 17:36:22.787 | INFO | "malloc" : " 14 GB", | 20260505 17:36:22.787 | INFO | "pinned" : " 29 GB" | 20260505 17:36:22.787 | INFO | } | 20260505 17:36:22.787 | INFO | Perf report for the last 5.13845 seconds: | 20260505 17:36:22.787 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:36:22.787 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:22.787 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:22.787 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:22.787 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 17:36:22.788 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:22.788 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:22.788 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:36:22.788 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:22.788 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:22.788 | INFO | Uploaded 36GiB to GPU in 2.19959s (16.3667GiB/s) | 20260505 17:36:22.788 | INFO | Downloaded 34GiB to GPU in 2.03634s (16.6966GiB/s) | 20260505 17:36:22.921 | INFO | Starting a round of Baz IO | 20260505 17:36:26.099 | INFO | Populating RTMetrics report | 20260505 17:36:26.099 | INFO | RTMetrics report populated | 20260505 17:36:26.099 | INFO | Opening tmp RTMetrics json file | 20260505 17:36:26.106 | INFO | Serializing RTMetrics to json | 20260505 17:36:26.108 | INFO | Serialization done, swapping tmp file now... | 20260505 17:36:26.111 | INFO | RTMetrics done | 20260505 17:36:26.196 | INFO | Finished a round of Baz IO: Wrote 2.125 GiB in 3.275 seconds | 20260505 17:36:26.196 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.03% / 0.00% / 77.96% | 20260505 17:36:26.196 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.290159% / 3.97364e-06% / 0.000103315% | 20260505 17:36:26.196 | INFO | Baz IO overall: | 20260505 17:36:26.196 | INFO | BytesWritten : 2984163471800 | 20260505 17:36:26.196 | INFO | NumEvents : 1414870510304 | 20260505 17:36:27.928 | INFO | Analyzing chunk frames = [5050368,5050880) | 20260505 17:36:27.928 | INFO | MemInfo: { | 20260505 17:36:27.928 | INFO | "gpu" : " 17 GB", | 20260505 17:36:27.928 | INFO | "huge" : "", | 20260505 17:36:27.928 | INFO | "huge_shared" : " 27 GB", | 20260505 17:36:27.928 | INFO | "malloc" : " 14 GB", | 20260505 17:36:27.928 | INFO | "pinned" : " 29 GB" | 20260505 17:36:27.928 | INFO | } | 20260505 17:36:27.928 | INFO | Perf report for the last 5.14049 seconds: | 20260505 17:36:27.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:36:27.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:27.928 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:27.928 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:27.928 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 17:36:27.928 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:27.928 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260505 17:36:27.928 | INFO | PRE_HQ : 60.4%, 1.8, 56.0ms, 0 | 20260505 17:36:27.928 | INFO | BAZWRITER : 63.3%, 3.8, 124.0ms, 0 | 20260505 17:36:27.928 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 17:36:27.928 | INFO | Uploaded 36GiB to GPU in 2.16751s (16.6089GiB/s) | 20260505 17:36:27.928 | INFO | Downloaded 38GiB to GPU in 2.24666s (16.914GiB/s) | 20260505 17:36:33.057 | INFO | Analyzing chunk frames = [5050880,5051392) | 20260505 17:36:33.057 | INFO | MemInfo: { | 20260505 17:36:33.057 | INFO | "gpu" : " 17 GB", | 20260505 17:36:33.057 | INFO | "huge" : "", | 20260505 17:36:33.057 | INFO | "huge_shared" : " 27 GB", | 20260505 17:36:33.057 | INFO | "malloc" : " 14 GB", | 20260505 17:36:33.057 | INFO | "pinned" : " 29 GB" | 20260505 17:36:33.057 | INFO | } | 20260505 17:36:33.057 | INFO | Perf report for the last 5.12904 seconds: | 20260505 17:36:33.057 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:36:33.057 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:33.057 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:33.057 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:33.057 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 17:36:33.057 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:33.057 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:33.057 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:36:33.057 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:33.057 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:33.057 | INFO | Uploaded 36GiB to GPU in 2.1414s (16.8114GiB/s) | 20260505 17:36:33.057 | INFO | Downloaded 34GiB to GPU in 2.03156s (16.7359GiB/s) | 20260505 17:36:38.196 | INFO | Analyzing chunk frames = [5051392,5051904) | 20260505 17:36:38.196 | INFO | MemInfo: { | 20260505 17:36:38.196 | INFO | "gpu" : " 17 GB", | 20260505 17:36:38.196 | INFO | "huge" : "", | 20260505 17:36:38.196 | INFO | "huge_shared" : " 27 GB", | 20260505 17:36:38.196 | INFO | "malloc" : " 14 GB", | 20260505 17:36:38.196 | INFO | "pinned" : " 29 GB" | 20260505 17:36:38.196 | INFO | } | 20260505 17:36:38.197 | INFO | Perf report for the last 5.1397 seconds: | 20260505 17:36:38.197 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:36:38.197 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:38.197 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:38.197 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:38.197 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260505 17:36:38.197 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:38.197 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:38.197 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 17:36:38.197 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:38.197 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:38.197 | INFO | Uploaded 36GiB to GPU in 2.19189s (16.4242GiB/s) | 20260505 17:36:38.197 | INFO | Downloaded 34GiB to GPU in 2.03513s (16.7065GiB/s) | 20260505 17:36:43.346 | INFO | Analyzing chunk frames = [5051904,5052416) | 20260505 17:36:43.346 | INFO | MemInfo: { | 20260505 17:36:43.346 | INFO | "gpu" : " 17 GB", | 20260505 17:36:43.346 | INFO | "huge" : "", | 20260505 17:36:43.346 | INFO | "huge_shared" : " 27 GB", | 20260505 17:36:43.346 | INFO | "malloc" : " 14 GB", | 20260505 17:36:43.346 | INFO | "pinned" : " 29 GB" | 20260505 17:36:43.346 | INFO | } | 20260505 17:36:43.346 | INFO | Perf report for the last 5.14924 seconds: | 20260505 17:36:43.346 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:36:43.346 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:43.346 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:43.346 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:43.346 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260505 17:36:43.346 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:43.346 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:43.346 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:36:43.346 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:43.346 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:43.346 | INFO | Uploaded 36GiB to GPU in 2.19267s (16.4184GiB/s) | 20260505 17:36:43.346 | INFO | Downloaded 34GiB to GPU in 2.13409s (15.9318GiB/s) | 20260505 17:36:48.489 | INFO | Analyzing chunk frames = [5052416,5052928) | 20260505 17:36:48.489 | INFO | MemInfo: { | 20260505 17:36:48.489 | INFO | "gpu" : " 17 GB", | 20260505 17:36:48.489 | INFO | "huge" : "", | 20260505 17:36:48.489 | INFO | "huge_shared" : " 27 GB", | 20260505 17:36:48.489 | INFO | "malloc" : " 14 GB", | 20260505 17:36:48.489 | INFO | "pinned" : " 29 GB" | 20260505 17:36:48.489 | INFO | } | 20260505 17:36:48.489 | INFO | Perf report for the last 5.14345 seconds: | 20260505 17:36:48.489 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:36:48.489 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:48.489 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:48.489 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:48.489 | INFO | ANALYSIS : 61.9%, 3.0, 96.8ms, 0 | 20260505 17:36:48.489 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:48.489 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:48.489 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:36:48.489 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:48.489 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:48.490 | INFO | Uploaded 36GiB to GPU in 2.13406s (16.8693GiB/s) | 20260505 17:36:48.490 | INFO | Downloaded 34GiB to GPU in 2.05724s (16.527GiB/s) | 20260505 17:36:53.623 | INFO | Analyzing chunk frames = [5052928,5053440) | 20260505 17:36:53.623 | INFO | MemInfo: { | 20260505 17:36:53.623 | INFO | "gpu" : " 17 GB", | 20260505 17:36:53.623 | INFO | "huge" : "", | 20260505 17:36:53.623 | INFO | "huge_shared" : " 27 GB", | 20260505 17:36:53.623 | INFO | "malloc" : " 14 GB", | 20260505 17:36:53.623 | INFO | "pinned" : " 29 GB" | 20260505 17:36:53.623 | INFO | } | 20260505 17:36:53.623 | INFO | Perf report for the last 5.13411 seconds: | 20260505 17:36:53.623 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:36:53.623 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:53.623 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:53.623 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:53.623 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260505 17:36:53.623 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:53.623 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:53.623 | INFO | PRE_HQ : 27.5%, 1.0, 14.2ms, 0 | 20260505 17:36:53.623 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:53.624 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:53.624 | INFO | Uploaded 36GiB to GPU in 2.14862s (16.755GiB/s) | 20260505 17:36:53.624 | INFO | Downloaded 34GiB to GPU in 2.05163s (16.5722GiB/s) | 20260505 17:36:58.754 | INFO | Analyzing chunk frames = [5053440,5053952) | 20260505 17:36:58.754 | INFO | MemInfo: { | 20260505 17:36:58.754 | INFO | "gpu" : " 17 GB", | 20260505 17:36:58.754 | INFO | "huge" : "", | 20260505 17:36:58.754 | INFO | "huge_shared" : " 27 GB", | 20260505 17:36:58.754 | INFO | "malloc" : " 14 GB", | 20260505 17:36:58.754 | INFO | "pinned" : " 29 GB" | 20260505 17:36:58.754 | INFO | } | 20260505 17:36:58.754 | INFO | Perf report for the last 5.13037 seconds: | 20260505 17:36:58.754 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:36:58.754 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:58.754 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:58.754 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:58.754 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 17:36:58.754 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:58.754 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:58.754 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 17:36:58.754 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:58.754 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:36:58.754 | INFO | Uploaded 36GiB to GPU in 2.12612s (16.9323GiB/s) | 20260505 17:36:58.754 | INFO | Downloaded 34GiB to GPU in 2.17675s (15.6196GiB/s) | 20260505 17:37:03.897 | INFO | Analyzing chunk frames = [5053952,5054464) | 20260505 17:37:03.897 | INFO | MemInfo: { | 20260505 17:37:03.897 | INFO | "gpu" : " 17 GB", | 20260505 17:37:03.897 | INFO | "huge" : "", | 20260505 17:37:03.897 | INFO | "huge_shared" : " 27 GB", | 20260505 17:37:03.898 | INFO | "malloc" : " 14 GB", | 20260505 17:37:03.898 | INFO | "pinned" : " 29 GB" | 20260505 17:37:03.898 | INFO | } | 20260505 17:37:03.898 | INFO | Perf report for the last 5.14384 seconds: | 20260505 17:37:03.898 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:37:03.898 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:03.898 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:03.898 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:03.898 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 17:37:03.898 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:03.898 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:03.898 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 17:37:03.898 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:03.898 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:03.898 | INFO | Uploaded 36GiB to GPU in 2.15556s (16.701GiB/s) | 20260505 17:37:03.898 | INFO | Downloaded 34GiB to GPU in 2.03317s (16.7227GiB/s) | 20260505 17:37:03.898 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 17:37:03.898 | INFO | Total mem usage: 15.3 GiB | 20260505 17:37:03.898 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 17:37:03.898 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.98%) | 20260505 17:37:03.898 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 17:37:03.898 | INFO | High Water Breakdown: | 20260505 17:37:03.898 | INFO | 61% : 8.66 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 17:37:03.898 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 17:37:03.898 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 17:37:03.898 | INFO | ------------------------------------------ | 20260505 17:37:03.898 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:37:03.898 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 17:37:03.898 | INFO | Total mem usage: 26.5 GiB | 20260505 17:37:03.898 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 17:37:03.898 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 17:37:03.898 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 17:37:03.898 | INFO | High Water Breakdown: | 20260505 17:37:03.898 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:37:03.898 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:37:03.899 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:37:03.899 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:37:03.899 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:37:03.899 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:37:03.899 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 17:37:03.899 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 17:37:03.899 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:37:03.899 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 17:37:03.899 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 17:37:03.899 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 17:37:03.899 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 17:37:03.899 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 17:37:03.899 | INFO | ------------------------------------------ | 20260505 17:37:03.899 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:37:03.899 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 17:37:03.899 | INFO | Total mem usage: 25.6 GiB | 20260505 17:37:03.899 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 17:37:03.899 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 17:37:03.899 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 17:37:03.899 | INFO | High Water Breakdown: | 20260505 17:37:03.899 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 17:37:03.899 | INFO | ------------------------------------------ | 20260505 17:37:03.899 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:37:03.899 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 17:37:03.899 | INFO | Total mem usage: 15.7 GiB | 20260505 17:37:03.899 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 17:37:03.899 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 17:37:03.899 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 17:37:03.899 | INFO | High Water Breakdown: | 20260505 17:37:03.899 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 17:37:03.899 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 17:37:03.899 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 17:37:03.899 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 17:37:03.899 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 17:37:03.899 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 17:37:03.899 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 17:37:03.899 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 17:37:03.899 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 17:37:03.899 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:37:03.899 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 17:37:03.899 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:37:03.899 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 17:37:03.899 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 17:37:03.899 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 17:37:03.899 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 17:37:03.899 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:37:03.899 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 17:37:03.899 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:37:03.899 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:37:03.899 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:37:03.899 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 17:37:03.899 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 17:37:03.899 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:37:03.899 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 17:37:03.899 | INFO | ------------------------------------------ | 20260505 17:37:03.899 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:37:03.899 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 17:37:03.899 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:37:03.899 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:37:03.899 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.49e+01, 1.36e+01 / 2.03e+00 | 20260505 17:37:03.899 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 1.02e-01, 1.08e-02 / 3.34e-03 | 20260505 17:37:03.899 | INFO | PerfReport: 11.79%, Baseline -- 3.73e+00 / 4.50e+00, 5.78e-01 / 1.80e-01 | 20260505 17:37:03.899 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.52e+00, 1.93e-01 / 9.48e-02 | 20260505 17:37:03.899 | INFO | PerfReport: 15.90%, DME -- 4.02e+01 / 4.43e+01, 4.11e+00 / 2.08e+00 | 20260505 17:37:03.899 | INFO | PerfReport: 56.88%, FrameLabeling -- 1.80e+01 / 1.98e+01, 1.51e+00 / 1.41e+00 | 20260505 17:37:03.899 | INFO | PerfReport: 7.31%, PulseAccumulating -- 2.31e+00 / 2.87e+00, 4.47e-01 / 1.76e-01 | 20260505 17:37:03.899 | INFO | PerfReport: 3.37%, Metrics -- 1.07e+00 / 1.38e+00, 2.98e-01 / 1.60e-01 | 20260505 17:37:03.899 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.71e-01 / 4.32e-01, 6.75e-02 / 3.74e-02 | 20260505 17:37:03.899 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 17:37:03.899 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:37:03.899 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:37:03.899 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260505 17:37:03.899 | INFO | PerfReport: 22.55%, Upload -- 2.26e+01 / 0.00e+00, 7.90e+00 / 0.00e+00 | 20260505 17:37:03.899 | INFO | PerfReport: 21.82%, Download -- 2.18e+01 / 0.00e+00, 7.85e+00 / 0.00e+00 | 20260505 17:37:03.899 | INFO | PerfReport: 31.63%, Compute -- 3.16e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 17:37:03.899 | INFO | PerfReport: 2.29%, LockUpload -- 2.29e+00 / 0.00e+00, 7.90e+00 / 0.00e+00 | 20260505 17:37:03.900 | INFO | PerfReport: 1.39%, LockDownload -- 1.39e+00 / 0.00e+00, 2.94e+00 / 0.00e+00 | 20260505 17:37:03.900 | INFO | PerfReport: 20.29%, LockCompute -- 2.03e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 17:37:03.900 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 1.90e-02 / 0.00e+00 | 20260505 17:37:04.033 | INFO | Starting a round of Baz IO | 20260505 17:37:07.211 | INFO | Populating RTMetrics report | 20260505 17:37:07.211 | INFO | RTMetrics report populated | 20260505 17:37:07.211 | INFO | Opening tmp RTMetrics json file | 20260505 17:37:07.223 | INFO | Serializing RTMetrics to json | 20260505 17:37:07.226 | INFO | Serialization done, swapping tmp file now... | 20260505 17:37:07.227 | INFO | RTMetrics done | 20260505 17:37:07.307 | INFO | Finished a round of Baz IO: Wrote 2.126 GiB in 3.274 seconds | 20260505 17:37:07.307 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.03% / 0.00% / 77.97% | 20260505 17:37:07.307 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.292091% / 0% / 9.93411e-05% | 20260505 17:37:07.307 | INFO | Baz IO overall: | 20260505 17:37:07.308 | INFO | BytesWritten : 2986445862273 | 20260505 17:37:07.308 | INFO | NumEvents : 1415936458372 | 20260505 17:37:09.039 | INFO | Analyzing chunk frames = [5054464,5054976) | 20260505 17:37:09.039 | INFO | MemInfo: { | 20260505 17:37:09.039 | INFO | "gpu" : " 17 GB", | 20260505 17:37:09.039 | INFO | "huge" : "", | 20260505 17:37:09.039 | INFO | "huge_shared" : " 27 GB", | 20260505 17:37:09.039 | INFO | "malloc" : " 14 GB", | 20260505 17:37:09.039 | INFO | "pinned" : " 29 GB" | 20260505 17:37:09.039 | INFO | } | 20260505 17:37:09.039 | INFO | Perf report for the last 5.14122 seconds: | 20260505 17:37:09.039 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:37:09.039 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:09.039 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:09.039 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:09.039 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 17:37:09.039 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:09.039 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260505 17:37:09.039 | INFO | PRE_HQ : 60.5%, 1.8, 56.3ms, 0 | 20260505 17:37:09.039 | INFO | BAZWRITER : 63.2%, 3.9, 125.2ms, 0 | 20260505 17:37:09.039 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 17:37:09.039 | INFO | Uploaded 36GiB to GPU in 2.1567s (16.6921GiB/s) | 20260505 17:37:09.039 | INFO | Downloaded 38GiB to GPU in 2.24626s (16.917GiB/s) | 20260505 17:37:14.177 | INFO | Analyzing chunk frames = [5054976,5055488) | 20260505 17:37:14.177 | INFO | MemInfo: { | 20260505 17:37:14.177 | INFO | "gpu" : " 17 GB", | 20260505 17:37:14.177 | INFO | "huge" : "", | 20260505 17:37:14.177 | INFO | "huge_shared" : " 27 GB", | 20260505 17:37:14.177 | INFO | "malloc" : " 14 GB", | 20260505 17:37:14.177 | INFO | "pinned" : " 29 GB" | 20260505 17:37:14.177 | INFO | } | 20260505 17:37:14.177 | INFO | Perf report for the last 5.13822 seconds: | 20260505 17:37:14.177 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:37:14.177 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:14.177 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:14.177 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:14.177 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 17:37:14.177 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:14.177 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:14.177 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 17:37:14.177 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:14.177 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:14.177 | INFO | Uploaded 36GiB to GPU in 2.13436s (16.8669GiB/s) | 20260505 17:37:14.178 | INFO | Downloaded 34GiB to GPU in 2.02237s (16.8119GiB/s) | 20260505 17:37:19.319 | INFO | Analyzing chunk frames = [5055488,5056000) | 20260505 17:37:19.319 | INFO | MemInfo: { | 20260505 17:37:19.319 | INFO | "gpu" : " 17 GB", | 20260505 17:37:19.319 | INFO | "huge" : "", | 20260505 17:37:19.319 | INFO | "huge_shared" : " 27 GB", | 20260505 17:37:19.319 | INFO | "malloc" : " 14 GB", | 20260505 17:37:19.319 | INFO | "pinned" : " 29 GB" | 20260505 17:37:19.319 | INFO | } | 20260505 17:37:19.319 | INFO | Perf report for the last 5.14158 seconds: | 20260505 17:37:19.319 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:37:19.319 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:19.319 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:19.319 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:19.319 | INFO | ANALYSIS : 62.6%, 3.0, 97.4ms, 0 | 20260505 17:37:19.319 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:19.319 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:19.319 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 17:37:19.319 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:19.319 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:19.319 | INFO | Uploaded 36GiB to GPU in 2.20065s (16.3588GiB/s) | 20260505 17:37:19.319 | INFO | Downloaded 34GiB to GPU in 2.04028s (16.6644GiB/s) | 20260505 17:37:24.455 | INFO | Analyzing chunk frames = [5056000,5056512) | 20260505 17:37:24.455 | INFO | MemInfo: { | 20260505 17:37:24.455 | INFO | "gpu" : " 17 GB", | 20260505 17:37:24.455 | INFO | "huge" : "", | 20260505 17:37:24.455 | INFO | "huge_shared" : " 27 GB", | 20260505 17:37:24.455 | INFO | "malloc" : " 14 GB", | 20260505 17:37:24.455 | INFO | "pinned" : " 29 GB" | 20260505 17:37:24.455 | INFO | } | 20260505 17:37:24.456 | INFO | Perf report for the last 5.13683 seconds: | 20260505 17:37:24.456 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:37:24.456 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:24.456 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:24.456 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:24.456 | INFO | ANALYSIS : 62.5%, 3.0, 97.1ms, 0 | 20260505 17:37:24.456 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:24.456 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:24.456 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:37:24.456 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:24.456 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:24.456 | INFO | Uploaded 36GiB to GPU in 2.18407s (16.483GiB/s) | 20260505 17:37:24.456 | INFO | Downloaded 34GiB to GPU in 2.14134s (15.8779GiB/s) | 20260505 17:37:29.605 | INFO | Analyzing chunk frames = [5056512,5057024) | 20260505 17:37:29.605 | INFO | MemInfo: { | 20260505 17:37:29.605 | INFO | "gpu" : " 17 GB", | 20260505 17:37:29.605 | INFO | "huge" : "", | 20260505 17:37:29.605 | INFO | "huge_shared" : " 27 GB", | 20260505 17:37:29.605 | INFO | "malloc" : " 14 GB", | 20260505 17:37:29.605 | INFO | "pinned" : " 29 GB" | 20260505 17:37:29.605 | INFO | } | 20260505 17:37:29.605 | INFO | Perf report for the last 5.14981 seconds: | 20260505 17:37:29.605 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:37:29.605 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:29.605 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:29.605 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:29.605 | INFO | ANALYSIS : 62.1%, 3.0, 97.1ms, 0 | 20260505 17:37:29.605 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:29.605 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:29.605 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260505 17:37:29.605 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:29.606 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:29.606 | INFO | Uploaded 36GiB to GPU in 2.1575s (16.686GiB/s) | 20260505 17:37:29.606 | INFO | Downloaded 34GiB to GPU in 2.05798s (16.5211GiB/s) | 20260505 17:37:34.740 | INFO | Analyzing chunk frames = [5057024,5057536) | 20260505 17:37:34.740 | INFO | MemInfo: { | 20260505 17:37:34.740 | INFO | "gpu" : " 17 GB", | 20260505 17:37:34.740 | INFO | "huge" : "", | 20260505 17:37:34.740 | INFO | "huge_shared" : " 27 GB", | 20260505 17:37:34.740 | INFO | "malloc" : " 14 GB", | 20260505 17:37:34.740 | INFO | "pinned" : " 29 GB" | 20260505 17:37:34.740 | INFO | } | 20260505 17:37:34.740 | INFO | Perf report for the last 5.13447 seconds: | 20260505 17:37:34.740 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:37:34.740 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:34.740 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:34.740 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:34.740 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260505 17:37:34.740 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:34.740 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:34.740 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 17:37:34.740 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:34.740 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:34.740 | INFO | Uploaded 36GiB to GPU in 2.14733s (16.765GiB/s) | 20260505 17:37:34.740 | INFO | Downloaded 34GiB to GPU in 2.05348s (16.5573GiB/s) | 20260505 17:37:39.869 | INFO | Analyzing chunk frames = [5057536,5058048) | 20260505 17:37:39.869 | INFO | MemInfo: { | 20260505 17:37:39.869 | INFO | "gpu" : " 17 GB", | 20260505 17:37:39.869 | INFO | "huge" : "", | 20260505 17:37:39.869 | INFO | "huge_shared" : " 27 GB", | 20260505 17:37:39.870 | INFO | "malloc" : " 14 GB", | 20260505 17:37:39.870 | INFO | "pinned" : " 29 GB" | 20260505 17:37:39.870 | INFO | } | 20260505 17:37:39.870 | INFO | Perf report for the last 5.12986 seconds: | 20260505 17:37:39.870 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:37:39.870 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:39.870 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:39.870 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:39.870 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 17:37:39.870 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:39.870 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:39.870 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 17:37:39.870 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:39.870 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:39.870 | INFO | Uploaded 36GiB to GPU in 2.14003s (16.8222GiB/s) | 20260505 17:37:39.870 | INFO | Downloaded 34GiB to GPU in 2.1429s (15.8664GiB/s) | 20260505 17:37:45.008 | INFO | Analyzing chunk frames = [5058048,5058560) | 20260505 17:37:45.008 | INFO | MemInfo: { | 20260505 17:37:45.008 | INFO | "gpu" : " 17 GB", | 20260505 17:37:45.008 | INFO | "huge" : "", | 20260505 17:37:45.008 | INFO | "huge_shared" : " 27 GB", | 20260505 17:37:45.008 | INFO | "malloc" : " 14 GB", | 20260505 17:37:45.008 | INFO | "pinned" : " 29 GB" | 20260505 17:37:45.008 | INFO | } | 20260505 17:37:45.008 | INFO | Perf report for the last 5.13829 seconds: | 20260505 17:37:45.008 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:37:45.008 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:45.008 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:45.008 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:45.008 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 17:37:45.008 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:45.008 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:45.008 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:37:45.008 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:45.008 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:45.008 | INFO | Uploaded 36GiB to GPU in 2.18393s (16.484GiB/s) | 20260505 17:37:45.008 | INFO | Downloaded 34GiB to GPU in 2.03248s (16.7283GiB/s) | 20260505 17:37:45.141 | INFO | Starting a round of Baz IO | 20260505 17:37:48.338 | INFO | Populating RTMetrics report | 20260505 17:37:48.339 | INFO | RTMetrics report populated | 20260505 17:37:48.339 | INFO | Opening tmp RTMetrics json file | 20260505 17:37:48.341 | INFO | Serializing RTMetrics to json | 20260505 17:37:48.342 | INFO | Serialization done, swapping tmp file now... | 20260505 17:37:48.343 | INFO | RTMetrics done | 20260505 17:37:48.434 | INFO | Finished a round of Baz IO: Wrote 2.123 GiB in 3.293 seconds | 20260505 17:37:48.434 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.03% / 0.00% / 77.97% | 20260505 17:37:48.434 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.291022% / 3.97364e-06% / 9.93411e-05% | 20260505 17:37:48.434 | INFO | Baz IO overall: | 20260505 17:37:48.434 | INFO | BytesWritten : 2988725824885 | 20260505 17:37:48.434 | INFO | NumEvents : 1417001190972 | 20260505 17:37:50.152 | INFO | Analyzing chunk frames = [5058560,5059072) | 20260505 17:37:50.152 | INFO | MemInfo: { | 20260505 17:37:50.152 | INFO | "gpu" : " 17 GB", | 20260505 17:37:50.152 | INFO | "huge" : "", | 20260505 17:37:50.152 | INFO | "huge_shared" : " 27 GB", | 20260505 17:37:50.152 | INFO | "malloc" : " 14 GB", | 20260505 17:37:50.152 | INFO | "pinned" : " 29 GB" | 20260505 17:37:50.152 | INFO | } | 20260505 17:37:50.152 | INFO | Perf report for the last 5.14441 seconds: | 20260505 17:37:50.152 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:37:50.152 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:50.152 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:50.152 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:50.152 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 17:37:50.153 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:50.153 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260505 17:37:50.153 | INFO | PRE_HQ : 60.3%, 1.8, 55.9ms, 0 | 20260505 17:37:50.153 | INFO | BAZWRITER : 63.2%, 3.8, 124.8ms, 0 | 20260505 17:37:50.153 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 17:37:50.153 | INFO | Uploaded 36GiB to GPU in 2.17267s (16.5695GiB/s) | 20260505 17:37:50.153 | INFO | Downloaded 38GiB to GPU in 2.26202s (16.7992GiB/s) | 20260505 17:37:55.296 | INFO | Analyzing chunk frames = [5059072,5059584) | 20260505 17:37:55.296 | INFO | MemInfo: { | 20260505 17:37:55.296 | INFO | "gpu" : " 17 GB", | 20260505 17:37:55.296 | INFO | "huge" : "", | 20260505 17:37:55.296 | INFO | "huge_shared" : " 27 GB", | 20260505 17:37:55.296 | INFO | "malloc" : " 14 GB", | 20260505 17:37:55.296 | INFO | "pinned" : " 29 GB" | 20260505 17:37:55.296 | INFO | } | 20260505 17:37:55.297 | INFO | Perf report for the last 5.14405 seconds: | 20260505 17:37:55.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:37:55.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:55.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:55.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:55.297 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 17:37:55.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:55.297 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:55.297 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:37:55.297 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:55.297 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:37:55.297 | INFO | Uploaded 36GiB to GPU in 2.13731s (16.8436GiB/s) | 20260505 17:37:55.297 | INFO | Downloaded 34GiB to GPU in 2.02838s (16.7622GiB/s) | 20260505 17:38:00.433 | INFO | Analyzing chunk frames = [5059584,5060096) | 20260505 17:38:00.433 | INFO | MemInfo: { | 20260505 17:38:00.433 | INFO | "gpu" : " 17 GB", | 20260505 17:38:00.433 | INFO | "huge" : "", | 20260505 17:38:00.433 | INFO | "huge_shared" : " 27 GB", | 20260505 17:38:00.433 | INFO | "malloc" : " 14 GB", | 20260505 17:38:00.433 | INFO | "pinned" : " 29 GB" | 20260505 17:38:00.433 | INFO | } | 20260505 17:38:00.433 | INFO | Perf report for the last 5.13647 seconds: | 20260505 17:38:00.433 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:38:00.433 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:00.433 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:00.433 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:00.433 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260505 17:38:00.433 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:00.433 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:00.433 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:38:00.433 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:00.433 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:00.433 | INFO | Uploaded 36GiB to GPU in 2.13739s (16.843GiB/s) | 20260505 17:38:00.433 | INFO | Downloaded 34GiB to GPU in 2.0403s (16.6642GiB/s) | 20260505 17:38:05.572 | INFO | Analyzing chunk frames = [5060096,5060608) | 20260505 17:38:05.572 | INFO | MemInfo: { | 20260505 17:38:05.572 | INFO | "gpu" : " 17 GB", | 20260505 17:38:05.572 | INFO | "huge" : "", | 20260505 17:38:05.572 | INFO | "huge_shared" : " 27 GB", | 20260505 17:38:05.572 | INFO | "malloc" : " 14 GB", | 20260505 17:38:05.572 | INFO | "pinned" : " 29 GB" | 20260505 17:38:05.572 | INFO | } | 20260505 17:38:05.572 | INFO | Perf report for the last 5.13888 seconds: | 20260505 17:38:05.572 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:38:05.572 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:05.572 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:05.572 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:05.572 | INFO | ANALYSIS : 62.9%, 3.0, 97.7ms, 0 | 20260505 17:38:05.572 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:05.572 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:05.572 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:38:05.572 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:05.572 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:05.572 | INFO | Uploaded 36GiB to GPU in 2.22753s (16.1614GiB/s) | 20260505 17:38:05.572 | INFO | Downloaded 34GiB to GPU in 2.11805s (16.0525GiB/s) | 20260505 17:38:10.707 | INFO | Analyzing chunk frames = [5060608,5061120) | 20260505 17:38:10.707 | INFO | MemInfo: { | 20260505 17:38:10.707 | INFO | "gpu" : " 17 GB", | 20260505 17:38:10.707 | INFO | "huge" : "", | 20260505 17:38:10.707 | INFO | "huge_shared" : " 27 GB", | 20260505 17:38:10.707 | INFO | "malloc" : " 14 GB", | 20260505 17:38:10.707 | INFO | "pinned" : " 29 GB" | 20260505 17:38:10.707 | INFO | } | 20260505 17:38:10.707 | INFO | Perf report for the last 5.13515 seconds: | 20260505 17:38:10.707 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:38:10.707 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:10.707 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:10.707 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:10.707 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 17:38:10.707 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:10.707 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:10.707 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:38:10.707 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:10.707 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:10.707 | INFO | Uploaded 36GiB to GPU in 2.15452s (16.709GiB/s) | 20260505 17:38:10.708 | INFO | Downloaded 34GiB to GPU in 2.05598s (16.5371GiB/s) | 20260505 17:38:15.853 | INFO | Analyzing chunk frames = [5061120,5061632) | 20260505 17:38:15.853 | INFO | MemInfo: { | 20260505 17:38:15.853 | INFO | "gpu" : " 17 GB", | 20260505 17:38:15.853 | INFO | "huge" : "", | 20260505 17:38:15.853 | INFO | "huge_shared" : " 27 GB", | 20260505 17:38:15.853 | INFO | "malloc" : " 14 GB", | 20260505 17:38:15.853 | INFO | "pinned" : " 29 GB" | 20260505 17:38:15.853 | INFO | } | 20260505 17:38:15.854 | INFO | Perf report for the last 5.14659 seconds: | 20260505 17:38:15.854 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:38:15.854 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:15.854 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:15.854 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:15.854 | INFO | ANALYSIS : 62.3%, 3.0, 96.4ms, 0 | 20260505 17:38:15.854 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:15.854 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:15.854 | INFO | PRE_HQ : 27.8%, 1.0, 14.3ms, 0 | 20260505 17:38:15.854 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:15.854 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:15.854 | INFO | Uploaded 36GiB to GPU in 2.14982s (16.7456GiB/s) | 20260505 17:38:15.854 | INFO | Downloaded 34GiB to GPU in 2.04413s (16.633GiB/s) | 20260505 17:38:20.994 | INFO | Analyzing chunk frames = [5061632,5062144) | 20260505 17:38:20.994 | INFO | MemInfo: { | 20260505 17:38:20.994 | INFO | "gpu" : " 17 GB", | 20260505 17:38:20.994 | INFO | "huge" : "", | 20260505 17:38:20.994 | INFO | "huge_shared" : " 27 GB", | 20260505 17:38:20.994 | INFO | "malloc" : " 14 GB", | 20260505 17:38:20.994 | INFO | "pinned" : " 29 GB" | 20260505 17:38:20.994 | INFO | } | 20260505 17:38:20.994 | INFO | Perf report for the last 5.14052 seconds: | 20260505 17:38:20.994 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:38:20.994 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:20.994 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:20.994 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:20.994 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 17:38:20.994 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:20.994 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:20.994 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 17:38:20.994 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:20.994 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:20.994 | INFO | Uploaded 36GiB to GPU in 2.13348s (16.8738GiB/s) | 20260505 17:38:20.994 | INFO | Downloaded 34GiB to GPU in 2.1368s (15.9116GiB/s) | 20260505 17:38:26.127 | INFO | Analyzing chunk frames = [5062144,5062656) | 20260505 17:38:26.127 | INFO | MemInfo: { | 20260505 17:38:26.127 | INFO | "gpu" : " 17 GB", | 20260505 17:38:26.127 | INFO | "huge" : "", | 20260505 17:38:26.127 | INFO | "huge_shared" : " 27 GB", | 20260505 17:38:26.127 | INFO | "malloc" : " 14 GB", | 20260505 17:38:26.127 | INFO | "pinned" : " 29 GB" | 20260505 17:38:26.127 | INFO | } | 20260505 17:38:26.127 | INFO | Perf report for the last 5.13301 seconds: | 20260505 17:38:26.127 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:38:26.127 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:26.127 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:26.127 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:26.127 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 17:38:26.127 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:26.127 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:26.127 | INFO | PRE_HQ : 27.7%, 1.0, 14.3ms, 0 | 20260505 17:38:26.127 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:26.127 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:26.127 | INFO | Uploaded 36GiB to GPU in 2.19275s (16.4177GiB/s) | 20260505 17:38:26.127 | INFO | Downloaded 34GiB to GPU in 2.03064s (16.7435GiB/s) | 20260505 17:38:26.128 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 17:38:26.128 | INFO | Total mem usage: 15.3 GiB | 20260505 17:38:26.128 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 17:38:26.128 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.11%) | 20260505 17:38:26.128 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 17:38:26.128 | INFO | High Water Breakdown: | 20260505 17:38:26.128 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 17:38:26.128 | INFO | 38.9% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 17:38:26.128 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 17:38:26.128 | INFO | ------------------------------------------ | 20260505 17:38:26.128 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:38:26.128 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 17:38:26.128 | INFO | Total mem usage: 26.5 GiB | 20260505 17:38:26.128 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 17:38:26.128 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 17:38:26.128 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 17:38:26.128 | INFO | High Water Breakdown: | 20260505 17:38:26.128 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:38:26.128 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:38:26.128 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:38:26.128 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:38:26.128 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:38:26.128 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:38:26.128 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 17:38:26.128 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 17:38:26.128 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:38:26.128 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 17:38:26.128 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 17:38:26.128 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 17:38:26.128 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 17:38:26.128 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 17:38:26.128 | INFO | ------------------------------------------ | 20260505 17:38:26.128 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:38:26.128 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 17:38:26.128 | INFO | Total mem usage: 25.6 GiB | 20260505 17:38:26.128 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 17:38:26.128 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 17:38:26.128 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 17:38:26.128 | INFO | High Water Breakdown: | 20260505 17:38:26.128 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 17:38:26.128 | INFO | ------------------------------------------ | 20260505 17:38:26.128 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:38:26.128 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 17:38:26.128 | INFO | Total mem usage: 15.7 GiB | 20260505 17:38:26.128 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 17:38:26.128 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 17:38:26.128 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 17:38:26.128 | INFO | High Water Breakdown: | 20260505 17:38:26.128 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 17:38:26.128 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 17:38:26.128 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 17:38:26.128 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 17:38:26.128 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 17:38:26.128 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 17:38:26.128 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 17:38:26.128 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 17:38:26.128 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 17:38:26.128 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:38:26.128 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 17:38:26.128 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:38:26.128 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 17:38:26.128 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 17:38:26.128 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 17:38:26.128 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 17:38:26.128 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:38:26.128 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 17:38:26.128 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:38:26.128 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:38:26.128 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:38:26.128 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 17:38:26.128 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 17:38:26.128 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:38:26.128 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 17:38:26.129 | INFO | ------------------------------------------ | 20260505 17:38:26.129 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:38:26.129 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 17:38:26.129 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:38:26.129 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:38:26.129 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.48e+01, 1.36e+01 / 1.94e+00 | 20260505 17:38:26.129 | INFO | PerfReport: 0.29%, AggregateStats -- 9.25e-02 / 1.02e-01, 1.05e-02 / 3.39e-03 | 20260505 17:38:26.129 | INFO | PerfReport: 11.79%, Baseline -- 3.72e+00 / 4.54e+00, 5.74e-01 / 1.33e-01 | 20260505 17:38:26.129 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.53e+00, 1.91e-01 / 7.79e-02 | 20260505 17:38:26.129 | INFO | PerfReport: 15.95%, DME -- 4.03e+01 / 4.48e+01, 4.28e+00 / 2.09e+00 | 20260505 17:38:26.129 | INFO | PerfReport: 56.81%, FrameLabeling -- 1.79e+01 / 1.91e+01, 1.45e+00 / 1.00e+00 | 20260505 17:38:26.129 | INFO | PerfReport: 7.34%, PulseAccumulating -- 2.32e+00 / 2.92e+00, 4.48e-01 / 1.71e-01 | 20260505 17:38:26.129 | INFO | PerfReport: 3.37%, Metrics -- 1.06e+00 / 1.41e+00, 2.92e-01 / 1.42e-01 | 20260505 17:38:26.129 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.70e-01 / 4.36e-01, 6.60e-02 / 3.97e-02 | 20260505 17:38:26.129 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 17:38:26.129 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:38:26.129 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:38:26.129 | INFO | PerfReport: Full Execution -- 9.99e+01 / -nan, 2.57e+01 / 0.00e+00 | 20260505 17:38:26.129 | INFO | PerfReport: 22.55%, Upload -- 2.25e+01 / 0.00e+00, 7.90e+00 / 0.00e+00 | 20260505 17:38:26.129 | INFO | PerfReport: 21.79%, Download -- 2.18e+01 / 0.00e+00, 7.83e+00 / 0.00e+00 | 20260505 17:38:26.129 | INFO | PerfReport: 31.60%, Compute -- 3.16e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 17:38:26.129 | INFO | PerfReport: 2.34%, LockUpload -- 2.34e+00 / 0.00e+00, 8.05e+00 / 0.00e+00 | 20260505 17:38:26.129 | INFO | PerfReport: 1.38%, LockDownload -- 1.38e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260505 17:38:26.129 | INFO | PerfReport: 20.32%, LockCompute -- 2.03e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260505 17:38:26.129 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.27e-02 / 0.00e+00 | 20260505 17:38:26.264 | INFO | Starting a round of Baz IO | 20260505 17:38:29.458 | INFO | Populating RTMetrics report | 20260505 17:38:29.458 | INFO | RTMetrics report populated | 20260505 17:38:29.458 | INFO | Opening tmp RTMetrics json file | 20260505 17:38:29.463 | INFO | Serializing RTMetrics to json | 20260505 17:38:29.466 | INFO | Serialization done, swapping tmp file now... | 20260505 17:38:29.468 | INFO | RTMetrics done | 20260505 17:38:29.554 | INFO | Finished a round of Baz IO: Wrote 2.123 GiB in 3.29 seconds | 20260505 17:38:29.554 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.03% / 0.00% / 77.97% | 20260505 17:38:29.554 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.289353% / 0% / 9.93411e-05% | 20260505 17:38:29.554 | INFO | Baz IO overall: | 20260505 17:38:29.554 | INFO | BytesWritten : 2991005115340 | 20260505 17:38:29.554 | INFO | NumEvents : 1418065203833 | 20260505 17:38:31.264 | INFO | Analyzing chunk frames = [5062656,5063168) | 20260505 17:38:31.264 | INFO | MemInfo: { | 20260505 17:38:31.264 | INFO | "gpu" : " 17 GB", | 20260505 17:38:31.264 | INFO | "huge" : "", | 20260505 17:38:31.264 | INFO | "huge_shared" : " 27 GB", | 20260505 17:38:31.264 | INFO | "malloc" : " 14 GB", | 20260505 17:38:31.264 | INFO | "pinned" : " 29 GB" | 20260505 17:38:31.264 | INFO | } | 20260505 17:38:31.264 | INFO | Perf report for the last 5.13724 seconds: | 20260505 17:38:31.264 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:38:31.264 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:31.265 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:31.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:31.265 | INFO | ANALYSIS : 63.8%, 3.0, 99.2ms, 0 | 20260505 17:38:31.265 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:31.265 | INFO | RT_METRICS : 44.7%, 1.0, 23.4ms, 0 | 20260505 17:38:31.265 | INFO | PRE_HQ : 60.6%, 1.8, 55.9ms, 0 | 20260505 17:38:31.265 | INFO | BAZWRITER : 63.5%, 3.8, 123.9ms, 0 | 20260505 17:38:31.265 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 17:38:31.265 | INFO | Uploaded 36GiB to GPU in 2.19289s (16.4167GiB/s) | 20260505 17:38:31.265 | INFO | Downloaded 38GiB to GPU in 2.26389s (16.7852GiB/s) | 20260505 17:38:36.410 | INFO | Analyzing chunk frames = [5063168,5063680) | 20260505 17:38:36.410 | INFO | MemInfo: { | 20260505 17:38:36.410 | INFO | "gpu" : " 17 GB", | 20260505 17:38:36.410 | INFO | "huge" : "", | 20260505 17:38:36.410 | INFO | "huge_shared" : " 27 GB", | 20260505 17:38:36.410 | INFO | "malloc" : " 14 GB", | 20260505 17:38:36.410 | INFO | "pinned" : " 29 GB" | 20260505 17:38:36.410 | INFO | } | 20260505 17:38:36.410 | INFO | Perf report for the last 5.14573 seconds: | 20260505 17:38:36.410 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:38:36.410 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:36.410 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:36.410 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:36.410 | INFO | ANALYSIS : 62.7%, 3.0, 98.0ms, 0 | 20260505 17:38:36.410 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:36.410 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:36.410 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 17:38:36.410 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:36.410 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:36.411 | INFO | Uploaded 36GiB to GPU in 2.12494s (16.9416GiB/s) | 20260505 17:38:36.411 | INFO | Downloaded 34GiB to GPU in 2.02465s (16.793GiB/s) | 20260505 17:38:41.555 | INFO | Analyzing chunk frames = [5063680,5064192) | 20260505 17:38:41.555 | INFO | MemInfo: { | 20260505 17:38:41.555 | INFO | "gpu" : " 17 GB", | 20260505 17:38:41.555 | INFO | "huge" : "", | 20260505 17:38:41.555 | INFO | "huge_shared" : " 27 GB", | 20260505 17:38:41.555 | INFO | "malloc" : " 14 GB", | 20260505 17:38:41.555 | INFO | "pinned" : " 29 GB" | 20260505 17:38:41.555 | INFO | } | 20260505 17:38:41.555 | INFO | Perf report for the last 5.14452 seconds: | 20260505 17:38:41.555 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:38:41.555 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:41.555 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:41.555 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:41.555 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 17:38:41.555 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:41.555 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:41.555 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:38:41.555 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:41.555 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:41.555 | INFO | Uploaded 36GiB to GPU in 2.20946s (16.2936GiB/s) | 20260505 17:38:41.555 | INFO | Downloaded 34GiB to GPU in 2.07133s (16.4146GiB/s) | 20260505 17:38:46.687 | INFO | Analyzing chunk frames = [5064192,5064704) | 20260505 17:38:46.687 | INFO | MemInfo: { | 20260505 17:38:46.687 | INFO | "gpu" : " 17 GB", | 20260505 17:38:46.687 | INFO | "huge" : "", | 20260505 17:38:46.687 | INFO | "huge_shared" : " 27 GB", | 20260505 17:38:46.687 | INFO | "malloc" : " 14 GB", | 20260505 17:38:46.687 | INFO | "pinned" : " 29 GB" | 20260505 17:38:46.687 | INFO | } | 20260505 17:38:46.687 | INFO | Perf report for the last 5.13213 seconds: | 20260505 17:38:46.687 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:38:46.687 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:46.687 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:46.687 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:46.687 | INFO | ANALYSIS : 62.5%, 3.0, 96.9ms, 0 | 20260505 17:38:46.687 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:46.687 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:46.687 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:38:46.687 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:46.687 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:46.687 | INFO | Uploaded 36GiB to GPU in 2.17966s (16.5163GiB/s) | 20260505 17:38:46.687 | INFO | Downloaded 34GiB to GPU in 2.05684s (16.5302GiB/s) | 20260505 17:38:51.821 | INFO | Analyzing chunk frames = [5064704,5065216) | 20260505 17:38:51.821 | INFO | MemInfo: { | 20260505 17:38:51.821 | INFO | "gpu" : " 17 GB", | 20260505 17:38:51.821 | INFO | "huge" : "", | 20260505 17:38:51.821 | INFO | "huge_shared" : " 27 GB", | 20260505 17:38:51.821 | INFO | "malloc" : " 14 GB", | 20260505 17:38:51.821 | INFO | "pinned" : " 29 GB" | 20260505 17:38:51.821 | INFO | } | 20260505 17:38:51.821 | INFO | Perf report for the last 5.13388 seconds: | 20260505 17:38:51.821 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:38:51.821 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:51.821 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:51.821 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:51.821 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 17:38:51.821 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:51.821 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:51.821 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:38:51.821 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:51.821 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:51.821 | INFO | Uploaded 36GiB to GPU in 2.14429s (16.7888GiB/s) | 20260505 17:38:51.821 | INFO | Downloaded 34GiB to GPU in 2.05415s (16.5519GiB/s) | 20260505 17:38:56.961 | INFO | Analyzing chunk frames = [5065216,5065728) | 20260505 17:38:56.961 | INFO | MemInfo: { | 20260505 17:38:56.961 | INFO | "gpu" : " 17 GB", | 20260505 17:38:56.961 | INFO | "huge" : "", | 20260505 17:38:56.961 | INFO | "huge_shared" : " 27 GB", | 20260505 17:38:56.961 | INFO | "malloc" : " 14 GB", | 20260505 17:38:56.961 | INFO | "pinned" : " 29 GB" | 20260505 17:38:56.961 | INFO | } | 20260505 17:38:56.961 | INFO | Perf report for the last 5.14027 seconds: | 20260505 17:38:56.961 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:38:56.961 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:56.961 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:56.961 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:56.961 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260505 17:38:56.961 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:56.961 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:56.961 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 17:38:56.961 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:56.961 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:38:56.961 | INFO | Uploaded 36GiB to GPU in 2.13645s (16.8504GiB/s) | 20260505 17:38:56.961 | INFO | Downloaded 34GiB to GPU in 2.04242s (16.6469GiB/s) | 20260505 17:39:02.106 | INFO | Analyzing chunk frames = [5065728,5066240) | 20260505 17:39:02.106 | INFO | MemInfo: { | 20260505 17:39:02.106 | INFO | "gpu" : " 17 GB", | 20260505 17:39:02.106 | INFO | "huge" : "", | 20260505 17:39:02.106 | INFO | "huge_shared" : " 27 GB", | 20260505 17:39:02.106 | INFO | "malloc" : " 14 GB", | 20260505 17:39:02.106 | INFO | "pinned" : " 29 GB" | 20260505 17:39:02.106 | INFO | } | 20260505 17:39:02.106 | INFO | Perf report for the last 5.14488 seconds: | 20260505 17:39:02.106 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:39:02.106 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:02.106 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:02.106 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:02.106 | INFO | ANALYSIS : 63.5%, 3.0, 99.3ms, 0 | 20260505 17:39:02.106 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:02.106 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:02.106 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 17:39:02.106 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:02.106 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:02.106 | INFO | Uploaded 36GiB to GPU in 2.17816s (16.5277GiB/s) | 20260505 17:39:02.106 | INFO | Downloaded 34GiB to GPU in 2.16713s (15.689GiB/s) | 20260505 17:39:07.251 | INFO | Analyzing chunk frames = [5066240,5066752) | 20260505 17:39:07.251 | INFO | MemInfo: { | 20260505 17:39:07.251 | INFO | "gpu" : " 17 GB", | 20260505 17:39:07.251 | INFO | "huge" : "", | 20260505 17:39:07.251 | INFO | "huge_shared" : " 27 GB", | 20260505 17:39:07.251 | INFO | "malloc" : " 14 GB", | 20260505 17:39:07.251 | INFO | "pinned" : " 29 GB" | 20260505 17:39:07.251 | INFO | } | 20260505 17:39:07.251 | INFO | Perf report for the last 5.14524 seconds: | 20260505 17:39:07.251 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:39:07.251 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:07.251 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:07.251 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:07.251 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 17:39:07.251 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:07.251 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:07.251 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260505 17:39:07.251 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:07.251 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:07.252 | INFO | Uploaded 36GiB to GPU in 2.19812s (16.3776GiB/s) | 20260505 17:39:07.252 | INFO | Downloaded 34GiB to GPU in 2.0202s (16.83GiB/s) | 20260505 17:39:07.382 | INFO | Starting a round of Baz IO | 20260505 17:39:10.560 | INFO | Populating RTMetrics report | 20260505 17:39:10.561 | INFO | RTMetrics report populated | 20260505 17:39:10.561 | INFO | Opening tmp RTMetrics json file | 20260505 17:39:10.566 | INFO | Serializing RTMetrics to json | 20260505 17:39:10.567 | INFO | Serialization done, swapping tmp file now... | 20260505 17:39:10.567 | INFO | RTMetrics done | 20260505 17:39:10.659 | INFO | Finished a round of Baz IO: Wrote 2.123 GiB in 3.277 seconds | 20260505 17:39:10.660 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.03% / 0.00% / 77.97% | 20260505 17:39:10.660 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.288824% / 3.97364e-06% / 8.34465e-05% | 20260505 17:39:10.660 | INFO | Baz IO overall: | 20260505 17:39:10.660 | INFO | BytesWritten : 2993285181056 | 20260505 17:39:10.660 | INFO | NumEvents : 1419129938884 | 20260505 17:39:12.378 | INFO | Analyzing chunk frames = [5066752,5067264) | 20260505 17:39:12.378 | INFO | MemInfo: { | 20260505 17:39:12.378 | INFO | "gpu" : " 17 GB", | 20260505 17:39:12.378 | INFO | "huge" : "", | 20260505 17:39:12.378 | INFO | "huge_shared" : " 27 GB", | 20260505 17:39:12.378 | INFO | "malloc" : " 14 GB", | 20260505 17:39:12.378 | INFO | "pinned" : " 29 GB" | 20260505 17:39:12.378 | INFO | } | 20260505 17:39:12.378 | INFO | Perf report for the last 5.12718 seconds: | 20260505 17:39:12.378 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:39:12.378 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:12.378 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:12.378 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:12.379 | INFO | ANALYSIS : 63.4%, 3.0, 98.5ms, 0 | 20260505 17:39:12.379 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:12.379 | INFO | RT_METRICS : 44.8%, 1.0, 23.4ms, 0 | 20260505 17:39:12.379 | INFO | PRE_HQ : 60.4%, 1.8, 55.9ms, 0 | 20260505 17:39:12.379 | INFO | BAZWRITER : 63.4%, 3.9, 125.3ms, 0 | 20260505 17:39:12.379 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 17:39:12.379 | INFO | Uploaded 36GiB to GPU in 2.16339s (16.6406GiB/s) | 20260505 17:39:12.379 | INFO | Downloaded 38GiB to GPU in 2.25196s (16.8742GiB/s) | 20260505 17:39:17.515 | INFO | Analyzing chunk frames = [5067264,5067776) | 20260505 17:39:17.515 | INFO | MemInfo: { | 20260505 17:39:17.515 | INFO | "gpu" : " 17 GB", | 20260505 17:39:17.515 | INFO | "huge" : "", | 20260505 17:39:17.515 | INFO | "huge_shared" : " 27 GB", | 20260505 17:39:17.515 | INFO | "malloc" : " 14 GB", | 20260505 17:39:17.515 | INFO | "pinned" : " 29 GB" | 20260505 17:39:17.515 | INFO | } | 20260505 17:39:17.515 | INFO | Perf report for the last 5.13673 seconds: | 20260505 17:39:17.515 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:39:17.515 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:17.515 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:17.515 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:17.515 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 17:39:17.515 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:17.515 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:17.515 | INFO | PRE_HQ : 27.7%, 1.0, 14.3ms, 0 | 20260505 17:39:17.515 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:17.515 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:17.516 | INFO | Uploaded 36GiB to GPU in 2.13545s (16.8583GiB/s) | 20260505 17:39:17.516 | INFO | Downloaded 34GiB to GPU in 2.03493s (16.7082GiB/s) | 20260505 17:39:22.660 | INFO | Analyzing chunk frames = [5067776,5068288) | 20260505 17:39:22.660 | INFO | MemInfo: { | 20260505 17:39:22.660 | INFO | "gpu" : " 17 GB", | 20260505 17:39:22.660 | INFO | "huge" : "", | 20260505 17:39:22.660 | INFO | "huge_shared" : " 27 GB", | 20260505 17:39:22.660 | INFO | "malloc" : " 14 GB", | 20260505 17:39:22.660 | INFO | "pinned" : " 29 GB" | 20260505 17:39:22.660 | INFO | } | 20260505 17:39:22.660 | INFO | Perf report for the last 5.14474 seconds: | 20260505 17:39:22.660 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:39:22.660 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:22.660 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:22.660 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:22.660 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260505 17:39:22.660 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:22.660 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:22.660 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:39:22.660 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:22.660 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:22.661 | INFO | Uploaded 36GiB to GPU in 2.17331s (16.5646GiB/s) | 20260505 17:39:22.661 | INFO | Downloaded 34GiB to GPU in 2.03761s (16.6862GiB/s) | 20260505 17:39:27.800 | INFO | Analyzing chunk frames = [5068288,5068800) | 20260505 17:39:27.800 | INFO | MemInfo: { | 20260505 17:39:27.800 | INFO | "gpu" : " 17 GB", | 20260505 17:39:27.800 | INFO | "huge" : "", | 20260505 17:39:27.800 | INFO | "huge_shared" : " 27 GB", | 20260505 17:39:27.800 | INFO | "malloc" : " 14 GB", | 20260505 17:39:27.800 | INFO | "pinned" : " 29 GB" | 20260505 17:39:27.800 | INFO | } | 20260505 17:39:27.800 | INFO | Perf report for the last 5.14027 seconds: | 20260505 17:39:27.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:39:27.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:27.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:27.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:27.800 | INFO | ANALYSIS : 62.8%, 3.0, 97.5ms, 0 | 20260505 17:39:27.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:27.800 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:27.800 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:39:27.800 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:27.800 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:27.800 | INFO | Uploaded 36GiB to GPU in 2.22417s (16.1858GiB/s) | 20260505 17:39:27.800 | INFO | Downloaded 34GiB to GPU in 2.12028s (16.0356GiB/s) | 20260505 17:39:32.940 | INFO | Analyzing chunk frames = [5068800,5069312) | 20260505 17:39:32.940 | INFO | MemInfo: { | 20260505 17:39:32.940 | INFO | "gpu" : " 17 GB", | 20260505 17:39:32.940 | INFO | "huge" : "", | 20260505 17:39:32.940 | INFO | "huge_shared" : " 27 GB", | 20260505 17:39:32.940 | INFO | "malloc" : " 14 GB", | 20260505 17:39:32.940 | INFO | "pinned" : " 29 GB" | 20260505 17:39:32.940 | INFO | } | 20260505 17:39:32.940 | INFO | Perf report for the last 5.13971 seconds: | 20260505 17:39:32.940 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:39:32.940 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:32.940 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:32.940 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:32.940 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 17:39:32.940 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:32.940 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:32.940 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260505 17:39:32.940 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:32.940 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:32.940 | INFO | Uploaded 36GiB to GPU in 2.14512s (16.7823GiB/s) | 20260505 17:39:32.940 | INFO | Downloaded 34GiB to GPU in 2.06198s (16.489GiB/s) | 20260505 17:39:38.074 | INFO | Analyzing chunk frames = [5069312,5069824) | 20260505 17:39:38.074 | INFO | MemInfo: { | 20260505 17:39:38.074 | INFO | "gpu" : " 17 GB", | 20260505 17:39:38.074 | INFO | "huge" : "", | 20260505 17:39:38.074 | INFO | "huge_shared" : " 27 GB", | 20260505 17:39:38.074 | INFO | "malloc" : " 14 GB", | 20260505 17:39:38.074 | INFO | "pinned" : " 29 GB" | 20260505 17:39:38.074 | INFO | } | 20260505 17:39:38.074 | INFO | Perf report for the last 5.13421 seconds: | 20260505 17:39:38.074 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:39:38.074 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:38.074 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:38.074 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:38.074 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260505 17:39:38.074 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:38.074 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:38.074 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 17:39:38.074 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:38.074 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:38.074 | INFO | Uploaded 36GiB to GPU in 2.14062s (16.8176GiB/s) | 20260505 17:39:38.074 | INFO | Downloaded 34GiB to GPU in 2.05293s (16.5617GiB/s) | 20260505 17:39:43.216 | INFO | Analyzing chunk frames = [5069824,5070336) | 20260505 17:39:43.216 | INFO | MemInfo: { | 20260505 17:39:43.216 | INFO | "gpu" : " 17 GB", | 20260505 17:39:43.216 | INFO | "huge" : "", | 20260505 17:39:43.216 | INFO | "huge_shared" : " 27 GB", | 20260505 17:39:43.216 | INFO | "malloc" : " 14 GB", | 20260505 17:39:43.216 | INFO | "pinned" : " 29 GB" | 20260505 17:39:43.216 | INFO | } | 20260505 17:39:43.216 | INFO | Perf report for the last 5.14162 seconds: | 20260505 17:39:43.216 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:39:43.216 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:43.216 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:43.216 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:43.216 | INFO | ANALYSIS : 62.7%, 3.0, 98.0ms, 0 | 20260505 17:39:43.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:43.216 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:43.216 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 17:39:43.216 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:43.216 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:43.216 | INFO | Uploaded 36GiB to GPU in 2.1103s (17.0592GiB/s) | 20260505 17:39:43.216 | INFO | Downloaded 34GiB to GPU in 2.15698s (15.7628GiB/s) | 20260505 17:39:48.362 | INFO | Analyzing chunk frames = [5070336,5070848) | 20260505 17:39:48.362 | INFO | MemInfo: { | 20260505 17:39:48.362 | INFO | "gpu" : " 17 GB", | 20260505 17:39:48.362 | INFO | "huge" : "", | 20260505 17:39:48.362 | INFO | "huge_shared" : " 27 GB", | 20260505 17:39:48.362 | INFO | "malloc" : " 14 GB", | 20260505 17:39:48.362 | INFO | "pinned" : " 29 GB" | 20260505 17:39:48.362 | INFO | } | 20260505 17:39:48.362 | INFO | Perf report for the last 5.14633 seconds: | 20260505 17:39:48.362 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:39:48.362 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:48.362 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:48.362 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:48.362 | INFO | ANALYSIS : 62.9%, 3.0, 98.4ms, 0 | 20260505 17:39:48.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:48.362 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:48.362 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:39:48.362 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:48.362 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:48.362 | INFO | Uploaded 36GiB to GPU in 2.18818s (16.452GiB/s) | 20260505 17:39:48.363 | INFO | Downloaded 34GiB to GPU in 2.03406s (16.7153GiB/s) | 20260505 17:39:48.363 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 17:39:48.363 | INFO | Total mem usage: 15.3 GiB | 20260505 17:39:48.363 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 17:39:48.363 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.97%) | 20260505 17:39:48.363 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 17:39:48.363 | INFO | High Water Breakdown: | 20260505 17:39:48.363 | INFO | 61% : 8.66 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 17:39:48.363 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 17:39:48.363 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 17:39:48.363 | INFO | ------------------------------------------ | 20260505 17:39:48.363 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:39:48.363 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 17:39:48.363 | INFO | Total mem usage: 26.5 GiB | 20260505 17:39:48.363 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 17:39:48.363 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 17:39:48.363 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 17:39:48.363 | INFO | High Water Breakdown: | 20260505 17:39:48.363 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:39:48.363 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:39:48.363 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:39:48.363 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:39:48.363 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:39:48.363 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:39:48.363 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 17:39:48.363 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 17:39:48.363 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:39:48.363 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 17:39:48.363 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 17:39:48.363 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 17:39:48.363 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 17:39:48.363 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 17:39:48.363 | INFO | ------------------------------------------ | 20260505 17:39:48.363 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:39:48.363 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 17:39:48.363 | INFO | Total mem usage: 25.6 GiB | 20260505 17:39:48.363 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 17:39:48.363 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 17:39:48.363 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 17:39:48.363 | INFO | High Water Breakdown: | 20260505 17:39:48.363 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 17:39:48.363 | INFO | ------------------------------------------ | 20260505 17:39:48.363 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:39:48.363 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 17:39:48.363 | INFO | Total mem usage: 15.7 GiB | 20260505 17:39:48.363 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 17:39:48.364 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 17:39:48.364 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 17:39:48.364 | INFO | High Water Breakdown: | 20260505 17:39:48.364 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 17:39:48.364 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 17:39:48.364 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 17:39:48.364 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 17:39:48.364 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 17:39:48.364 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 17:39:48.364 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 17:39:48.364 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 17:39:48.364 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 17:39:48.364 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:39:48.364 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 17:39:48.364 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:39:48.364 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 17:39:48.364 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 17:39:48.364 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 17:39:48.364 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 17:39:48.364 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:39:48.364 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 17:39:48.364 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:39:48.364 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:39:48.364 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:39:48.364 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 17:39:48.364 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 17:39:48.364 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:39:48.364 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 17:39:48.364 | INFO | ------------------------------------------ | 20260505 17:39:48.364 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:39:48.364 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 17:39:48.364 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:39:48.364 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:39:48.364 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.48e+01, 1.35e+01 / 2.22e+00 | 20260505 17:39:48.364 | INFO | PerfReport: 0.29%, AggregateStats -- 9.25e-02 / 1.02e-01, 1.06e-02 / 4.81e-03 | 20260505 17:39:48.364 | INFO | PerfReport: 11.75%, Baseline -- 3.71e+00 / 4.52e+00, 5.60e-01 / 1.35e-01 | 20260505 17:39:48.364 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.51e+00, 1.90e-01 / 1.16e-01 | 20260505 17:39:48.364 | INFO | PerfReport: 15.93%, DME -- 4.03e+01 / 4.47e+01, 4.19e+00 / 2.15e+00 | 20260505 17:39:48.364 | INFO | PerfReport: 56.88%, FrameLabeling -- 1.80e+01 / 1.92e+01, 1.49e+00 / 1.09e+00 | 20260505 17:39:48.364 | INFO | PerfReport: 7.34%, PulseAccumulating -- 2.32e+00 / 2.91e+00, 4.48e-01 / 1.55e-01 | 20260505 17:39:48.364 | INFO | PerfReport: 3.36%, Metrics -- 1.06e+00 / 1.40e+00, 2.91e-01 / 1.24e-01 | 20260505 17:39:48.364 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.70e-01 / 4.32e-01, 6.56e-02 / 3.53e-02 | 20260505 17:39:48.364 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 17:39:48.364 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:39:48.364 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:39:48.364 | INFO | PerfReport: Full Execution -- 9.99e+01 / -nan, 2.56e+01 / 0.00e+00 | 20260505 17:39:48.364 | INFO | PerfReport: 22.57%, Upload -- 2.26e+01 / 0.00e+00, 7.82e+00 / 0.00e+00 | 20260505 17:39:48.364 | INFO | PerfReport: 21.79%, Download -- 2.18e+01 / 0.00e+00, 7.78e+00 / 0.00e+00 | 20260505 17:39:48.364 | INFO | PerfReport: 31.62%, Compute -- 3.16e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 17:39:48.364 | INFO | PerfReport: 2.33%, LockUpload -- 2.33e+00 / 0.00e+00, 7.98e+00 / 0.00e+00 | 20260505 17:39:48.364 | INFO | PerfReport: 1.38%, LockDownload -- 1.37e+00 / 0.00e+00, 2.90e+00 / 0.00e+00 | 20260505 17:39:48.364 | INFO | PerfReport: 20.29%, LockCompute -- 2.03e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 17:39:48.364 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.61e-02 / 0.00e+00 | 20260505 17:39:48.496 | INFO | Starting a round of Baz IO | 20260505 17:39:51.677 | INFO | Populating RTMetrics report | 20260505 17:39:51.677 | INFO | RTMetrics report populated | 20260505 17:39:51.677 | INFO | Opening tmp RTMetrics json file | 20260505 17:39:51.683 | INFO | Serializing RTMetrics to json | 20260505 17:39:51.685 | INFO | Serialization done, swapping tmp file now... | 20260505 17:39:51.688 | INFO | RTMetrics done | 20260505 17:39:51.769 | INFO | Finished a round of Baz IO: Wrote 2.124 GiB in 3.273 seconds | 20260505 17:39:51.769 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.03% / 0.00% / 77.97% | 20260505 17:39:51.769 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.289285% / 0% / 7.54992e-05% | 20260505 17:39:51.769 | INFO | Baz IO overall: | 20260505 17:39:51.769 | INFO | BytesWritten : 2995566343962 | 20260505 17:39:51.769 | INFO | NumEvents : 1420195750025 | 20260505 17:39:53.500 | INFO | Analyzing chunk frames = [5070848,5071360) | 20260505 17:39:53.500 | INFO | MemInfo: { | 20260505 17:39:53.500 | INFO | "gpu" : " 17 GB", | 20260505 17:39:53.500 | INFO | "huge" : "", | 20260505 17:39:53.500 | INFO | "huge_shared" : " 27 GB", | 20260505 17:39:53.500 | INFO | "malloc" : " 14 GB", | 20260505 17:39:53.500 | INFO | "pinned" : " 29 GB" | 20260505 17:39:53.500 | INFO | } | 20260505 17:39:53.500 | INFO | Perf report for the last 5.13804 seconds: | 20260505 17:39:53.500 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:39:53.500 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:53.500 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:53.500 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:53.500 | INFO | ANALYSIS : 63.4%, 3.0, 98.8ms, 0 | 20260505 17:39:53.500 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:53.500 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260505 17:39:53.500 | INFO | PRE_HQ : 60.5%, 1.8, 55.8ms, 0 | 20260505 17:39:53.500 | INFO | BAZWRITER : 63.2%, 3.8, 124.5ms, 0 | 20260505 17:39:53.500 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 17:39:53.501 | INFO | Uploaded 36GiB to GPU in 2.16897s (16.5977GiB/s) | 20260505 17:39:53.501 | INFO | Downloaded 38GiB to GPU in 2.25559s (16.8471GiB/s) | 20260505 17:39:58.630 | INFO | Analyzing chunk frames = [5071360,5071872) | 20260505 17:39:58.630 | INFO | MemInfo: { | 20260505 17:39:58.630 | INFO | "gpu" : " 17 GB", | 20260505 17:39:58.630 | INFO | "huge" : "", | 20260505 17:39:58.630 | INFO | "huge_shared" : " 27 GB", | 20260505 17:39:58.630 | INFO | "malloc" : " 14 GB", | 20260505 17:39:58.630 | INFO | "pinned" : " 29 GB" | 20260505 17:39:58.630 | INFO | } | 20260505 17:39:58.630 | INFO | Perf report for the last 5.12974 seconds: | 20260505 17:39:58.630 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:39:58.630 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:58.630 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:58.630 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:58.630 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260505 17:39:58.630 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:58.630 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:58.630 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:39:58.630 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:58.630 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:39:58.630 | INFO | Uploaded 36GiB to GPU in 2.12939s (16.9063GiB/s) | 20260505 17:39:58.630 | INFO | Downloaded 34GiB to GPU in 2.03167s (16.735GiB/s) | 20260505 17:40:03.771 | INFO | Analyzing chunk frames = [5071872,5072384) | 20260505 17:40:03.771 | INFO | MemInfo: { | 20260505 17:40:03.771 | INFO | "gpu" : " 17 GB", | 20260505 17:40:03.771 | INFO | "huge" : "", | 20260505 17:40:03.771 | INFO | "huge_shared" : " 27 GB", | 20260505 17:40:03.771 | INFO | "malloc" : " 14 GB", | 20260505 17:40:03.771 | INFO | "pinned" : " 29 GB" | 20260505 17:40:03.771 | INFO | } | 20260505 17:40:03.771 | INFO | Perf report for the last 5.14083 seconds: | 20260505 17:40:03.771 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:40:03.771 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:03.771 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:03.771 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:03.771 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 17:40:03.771 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:03.771 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:03.771 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 17:40:03.771 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:03.771 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:03.771 | INFO | Uploaded 36GiB to GPU in 2.18002s (16.5136GiB/s) | 20260505 17:40:03.771 | INFO | Downloaded 34GiB to GPU in 2.03094s (16.741GiB/s) | 20260505 17:40:08.922 | INFO | Analyzing chunk frames = [5072384,5072896) | 20260505 17:40:08.922 | INFO | MemInfo: { | 20260505 17:40:08.922 | INFO | "gpu" : " 17 GB", | 20260505 17:40:08.922 | INFO | "huge" : "", | 20260505 17:40:08.922 | INFO | "huge_shared" : " 27 GB", | 20260505 17:40:08.922 | INFO | "malloc" : " 14 GB", | 20260505 17:40:08.922 | INFO | "pinned" : " 29 GB" | 20260505 17:40:08.922 | INFO | } | 20260505 17:40:08.922 | INFO | Perf report for the last 5.15098 seconds: | 20260505 17:40:08.922 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:40:08.922 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:08.922 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:08.922 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:08.922 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260505 17:40:08.922 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:08.922 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:08.922 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:40:08.922 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:08.922 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:08.922 | INFO | Uploaded 36GiB to GPU in 2.16756s (16.6085GiB/s) | 20260505 17:40:08.922 | INFO | Downloaded 34GiB to GPU in 2.14293s (15.8661GiB/s) | 20260505 17:40:14.062 | INFO | Analyzing chunk frames = [5072896,5073408) | 20260505 17:40:14.062 | INFO | MemInfo: { | 20260505 17:40:14.062 | INFO | "gpu" : " 17 GB", | 20260505 17:40:14.062 | INFO | "huge" : "", | 20260505 17:40:14.062 | INFO | "huge_shared" : " 27 GB", | 20260505 17:40:14.062 | INFO | "malloc" : " 14 GB", | 20260505 17:40:14.062 | INFO | "pinned" : " 29 GB" | 20260505 17:40:14.062 | INFO | } | 20260505 17:40:14.062 | INFO | Perf report for the last 5.14034 seconds: | 20260505 17:40:14.062 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:40:14.062 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:14.062 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:14.062 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:14.062 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 17:40:14.062 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:14.062 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:14.062 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:40:14.062 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:14.062 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:14.062 | INFO | Uploaded 36GiB to GPU in 2.1318s (16.8872GiB/s) | 20260505 17:40:14.062 | INFO | Downloaded 34GiB to GPU in 2.05955s (16.5084GiB/s) | 20260505 17:40:19.198 | INFO | Analyzing chunk frames = [5073408,5073920) | 20260505 17:40:19.198 | INFO | MemInfo: { | 20260505 17:40:19.198 | INFO | "gpu" : " 17 GB", | 20260505 17:40:19.198 | INFO | "huge" : "", | 20260505 17:40:19.198 | INFO | "huge_shared" : " 27 GB", | 20260505 17:40:19.198 | INFO | "malloc" : " 14 GB", | 20260505 17:40:19.198 | INFO | "pinned" : " 29 GB" | 20260505 17:40:19.198 | INFO | } | 20260505 17:40:19.198 | INFO | Perf report for the last 5.13636 seconds: | 20260505 17:40:19.199 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:40:19.199 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:19.199 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:19.199 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:19.199 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 17:40:19.199 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:19.199 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:19.199 | INFO | PRE_HQ : 27.7%, 1.0, 14.3ms, 0 | 20260505 17:40:19.199 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:19.199 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:19.199 | INFO | Uploaded 36GiB to GPU in 2.14698s (16.7678GiB/s) | 20260505 17:40:19.199 | INFO | Downloaded 34GiB to GPU in 2.05483s (16.5464GiB/s) | 20260505 17:40:24.331 | INFO | Analyzing chunk frames = [5073920,5074432) | 20260505 17:40:24.331 | INFO | MemInfo: { | 20260505 17:40:24.331 | INFO | "gpu" : " 17 GB", | 20260505 17:40:24.331 | INFO | "huge" : "", | 20260505 17:40:24.331 | INFO | "huge_shared" : " 27 GB", | 20260505 17:40:24.331 | INFO | "malloc" : " 14 GB", | 20260505 17:40:24.331 | INFO | "pinned" : " 29 GB" | 20260505 17:40:24.331 | INFO | } | 20260505 17:40:24.331 | INFO | Perf report for the last 5.13266 seconds: | 20260505 17:40:24.331 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:40:24.331 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:24.331 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:24.331 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:24.331 | INFO | ANALYSIS : 63.5%, 3.0, 99.0ms, 0 | 20260505 17:40:24.331 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:24.331 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:24.331 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 17:40:24.331 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:24.331 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:24.332 | INFO | Uploaded 36GiB to GPU in 2.13578s (16.8557GiB/s) | 20260505 17:40:24.332 | INFO | Downloaded 34GiB to GPU in 2.14499s (15.8509GiB/s) | 20260505 17:40:29.469 | INFO | Analyzing chunk frames = [5074432,5074944) | 20260505 17:40:29.469 | INFO | MemInfo: { | 20260505 17:40:29.469 | INFO | "gpu" : " 17 GB", | 20260505 17:40:29.469 | INFO | "huge" : "", | 20260505 17:40:29.469 | INFO | "huge_shared" : " 27 GB", | 20260505 17:40:29.469 | INFO | "malloc" : " 14 GB", | 20260505 17:40:29.469 | INFO | "pinned" : " 29 GB" | 20260505 17:40:29.469 | INFO | } | 20260505 17:40:29.469 | INFO | Perf report for the last 5.13767 seconds: | 20260505 17:40:29.469 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:40:29.469 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:29.469 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:29.469 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:29.469 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 17:40:29.469 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:29.469 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:29.469 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:40:29.469 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:29.469 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:29.469 | INFO | Uploaded 36GiB to GPU in 2.18439s (16.4805GiB/s) | 20260505 17:40:29.469 | INFO | Downloaded 34GiB to GPU in 2.03276s (16.7261GiB/s) | 20260505 17:40:29.609 | INFO | Starting a round of Baz IO | 20260505 17:40:32.792 | INFO | Populating RTMetrics report | 20260505 17:40:32.792 | INFO | RTMetrics report populated | 20260505 17:40:32.792 | INFO | Opening tmp RTMetrics json file | 20260505 17:40:32.798 | INFO | Serializing RTMetrics to json | 20260505 17:40:32.801 | INFO | Serialization done, swapping tmp file now... | 20260505 17:40:32.802 | INFO | RTMetrics done | 20260505 17:40:32.890 | INFO | Finished a round of Baz IO: Wrote 2.124 GiB in 3.28 seconds | 20260505 17:40:32.890 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.02% / 0.00% / 77.97% | 20260505 17:40:32.890 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.288363% / 0% / 0.000103315% | 20260505 17:40:32.890 | INFO | Baz IO overall: | 20260505 17:40:32.890 | INFO | BytesWritten : 2997847129831 | 20260505 17:40:32.890 | INFO | NumEvents : 1421261498063 | 20260505 17:40:34.612 | INFO | Analyzing chunk frames = [5074944,5075456) | 20260505 17:40:34.612 | INFO | MemInfo: { | 20260505 17:40:34.612 | INFO | "gpu" : " 17 GB", | 20260505 17:40:34.612 | INFO | "huge" : "", | 20260505 17:40:34.612 | INFO | "huge_shared" : " 27 GB", | 20260505 17:40:34.612 | INFO | "malloc" : " 14 GB", | 20260505 17:40:34.612 | INFO | "pinned" : " 29 GB" | 20260505 17:40:34.612 | INFO | } | 20260505 17:40:34.612 | INFO | Perf report for the last 5.14292 seconds: | 20260505 17:40:34.612 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:40:34.612 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:34.612 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:34.612 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:34.612 | INFO | ANALYSIS : 63.5%, 3.0, 99.0ms, 0 | 20260505 17:40:34.612 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:34.612 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260505 17:40:34.612 | INFO | PRE_HQ : 60.2%, 1.8, 56.0ms, 0 | 20260505 17:40:34.612 | INFO | BAZWRITER : 63.2%, 3.9, 125.5ms, 0 | 20260505 17:40:34.612 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 17:40:34.617 | INFO | Uploaded 36GiB to GPU in 2.16301s (16.6435GiB/s) | 20260505 17:40:34.617 | INFO | Downloaded 38GiB to GPU in 2.26054s (16.8101GiB/s) | 20260505 17:40:39.752 | INFO | Analyzing chunk frames = [5075456,5075968) | 20260505 17:40:39.752 | INFO | MemInfo: { | 20260505 17:40:39.752 | INFO | "gpu" : " 17 GB", | 20260505 17:40:39.752 | INFO | "huge" : "", | 20260505 17:40:39.752 | INFO | "huge_shared" : " 27 GB", | 20260505 17:40:39.752 | INFO | "malloc" : " 14 GB", | 20260505 17:40:39.752 | INFO | "pinned" : " 29 GB" | 20260505 17:40:39.752 | INFO | } | 20260505 17:40:39.752 | INFO | Perf report for the last 5.13989 seconds: | 20260505 17:40:39.752 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:40:39.752 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:39.752 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:39.752 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:39.752 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260505 17:40:39.752 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:39.752 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:39.752 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 17:40:39.752 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:39.752 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:39.752 | INFO | Uploaded 36GiB to GPU in 2.12714s (16.9241GiB/s) | 20260505 17:40:39.752 | INFO | Downloaded 34GiB to GPU in 2.02292s (16.8074GiB/s) | 20260505 17:40:44.890 | INFO | Analyzing chunk frames = [5075968,5076480) | 20260505 17:40:44.890 | INFO | MemInfo: { | 20260505 17:40:44.890 | INFO | "gpu" : " 17 GB", | 20260505 17:40:44.890 | INFO | "huge" : "", | 20260505 17:40:44.890 | INFO | "huge_shared" : " 27 GB", | 20260505 17:40:44.890 | INFO | "malloc" : " 14 GB", | 20260505 17:40:44.890 | INFO | "pinned" : " 29 GB" | 20260505 17:40:44.890 | INFO | } | 20260505 17:40:44.890 | INFO | Perf report for the last 5.13855 seconds: | 20260505 17:40:44.890 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:40:44.890 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:44.890 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:44.890 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:44.890 | INFO | ANALYSIS : 62.7%, 3.0, 97.5ms, 0 | 20260505 17:40:44.890 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:44.890 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:44.890 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 17:40:44.890 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:44.890 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:44.891 | INFO | Uploaded 36GiB to GPU in 2.20205s (16.3484GiB/s) | 20260505 17:40:44.891 | INFO | Downloaded 34GiB to GPU in 2.03643s (16.6959GiB/s) | 20260505 17:40:50.024 | INFO | Analyzing chunk frames = [5076480,5076992) | 20260505 17:40:50.024 | INFO | MemInfo: { | 20260505 17:40:50.024 | INFO | "gpu" : " 17 GB", | 20260505 17:40:50.024 | INFO | "huge" : "", | 20260505 17:40:50.024 | INFO | "huge_shared" : " 27 GB", | 20260505 17:40:50.024 | INFO | "malloc" : " 14 GB", | 20260505 17:40:50.024 | INFO | "pinned" : " 29 GB" | 20260505 17:40:50.024 | INFO | } | 20260505 17:40:50.024 | INFO | Perf report for the last 5.13348 seconds: | 20260505 17:40:50.024 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:40:50.024 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:50.024 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:50.024 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:50.024 | INFO | ANALYSIS : 62.6%, 3.0, 97.2ms, 0 | 20260505 17:40:50.024 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:50.024 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:50.024 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 17:40:50.024 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:50.024 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:50.024 | INFO | Uploaded 36GiB to GPU in 2.19286s (16.4169GiB/s) | 20260505 17:40:50.024 | INFO | Downloaded 34GiB to GPU in 2.14256s (15.8689GiB/s) | 20260505 17:40:55.172 | INFO | Analyzing chunk frames = [5076992,5077504) | 20260505 17:40:55.172 | INFO | MemInfo: { | 20260505 17:40:55.172 | INFO | "gpu" : " 17 GB", | 20260505 17:40:55.172 | INFO | "huge" : "", | 20260505 17:40:55.172 | INFO | "huge_shared" : " 27 GB", | 20260505 17:40:55.172 | INFO | "malloc" : " 14 GB", | 20260505 17:40:55.172 | INFO | "pinned" : " 29 GB" | 20260505 17:40:55.172 | INFO | } | 20260505 17:40:55.172 | INFO | Perf report for the last 5.14875 seconds: | 20260505 17:40:55.172 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:40:55.172 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:55.172 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:55.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:55.173 | INFO | ANALYSIS : 62.1%, 3.0, 97.1ms, 0 | 20260505 17:40:55.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:55.173 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:55.173 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260505 17:40:55.173 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:55.173 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:40:55.173 | INFO | Uploaded 36GiB to GPU in 2.15337s (16.718GiB/s) | 20260505 17:40:55.173 | INFO | Downloaded 34GiB to GPU in 2.0611s (16.496GiB/s) | 20260505 17:41:00.309 | INFO | Analyzing chunk frames = [5077504,5078016) | 20260505 17:41:00.309 | INFO | MemInfo: { | 20260505 17:41:00.309 | INFO | "gpu" : " 17 GB", | 20260505 17:41:00.309 | INFO | "huge" : "", | 20260505 17:41:00.309 | INFO | "huge_shared" : " 27 GB", | 20260505 17:41:00.309 | INFO | "malloc" : " 14 GB", | 20260505 17:41:00.309 | INFO | "pinned" : " 29 GB" | 20260505 17:41:00.309 | INFO | } | 20260505 17:41:00.309 | INFO | Perf report for the last 5.13646 seconds: | 20260505 17:41:00.309 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:41:00.309 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:00.309 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:00.309 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:00.309 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260505 17:41:00.309 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:00.309 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:00.310 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:41:00.310 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:00.310 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:00.310 | INFO | Uploaded 36GiB to GPU in 2.14865s (16.7547GiB/s) | 20260505 17:41:00.310 | INFO | Downloaded 34GiB to GPU in 2.05532s (16.5425GiB/s) | 20260505 17:41:05.443 | INFO | Analyzing chunk frames = [5078016,5078528) | 20260505 17:41:05.443 | INFO | MemInfo: { | 20260505 17:41:05.443 | INFO | "gpu" : " 17 GB", | 20260505 17:41:05.443 | INFO | "huge" : "", | 20260505 17:41:05.443 | INFO | "huge_shared" : " 27 GB", | 20260505 17:41:05.443 | INFO | "malloc" : " 14 GB", | 20260505 17:41:05.443 | INFO | "pinned" : " 29 GB" | 20260505 17:41:05.443 | INFO | } | 20260505 17:41:05.444 | INFO | Perf report for the last 5.13458 seconds: | 20260505 17:41:05.444 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:41:05.444 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:05.444 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:05.444 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:05.444 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 17:41:05.444 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:05.444 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:05.444 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 17:41:05.444 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:05.444 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:05.444 | INFO | Uploaded 36GiB to GPU in 2.14417s (16.7897GiB/s) | 20260505 17:41:05.444 | INFO | Downloaded 34GiB to GPU in 2.14872s (15.8234GiB/s) | 20260505 17:41:10.583 | INFO | Analyzing chunk frames = [5078528,5079040) | 20260505 17:41:10.583 | INFO | MemInfo: { | 20260505 17:41:10.583 | INFO | "gpu" : " 17 GB", | 20260505 17:41:10.583 | INFO | "huge" : "", | 20260505 17:41:10.583 | INFO | "huge_shared" : " 27 GB", | 20260505 17:41:10.583 | INFO | "malloc" : " 14 GB", | 20260505 17:41:10.583 | INFO | "pinned" : " 29 GB" | 20260505 17:41:10.583 | INFO | } | 20260505 17:41:10.583 | INFO | Perf report for the last 5.13928 seconds: | 20260505 17:41:10.583 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:41:10.583 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:10.583 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:10.583 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:10.583 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 17:41:10.583 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:10.583 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:10.583 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 17:41:10.583 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:10.583 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:10.583 | INFO | Uploaded 36GiB to GPU in 2.17374s (16.5613GiB/s) | 20260505 17:41:10.583 | INFO | Downloaded 34GiB to GPU in 2.03722s (16.6894GiB/s) | 20260505 17:41:10.583 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 17:41:10.583 | INFO | Total mem usage: 15.3 GiB | 20260505 17:41:10.583 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 17:41:10.583 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.97%) | 20260505 17:41:10.583 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 17:41:10.583 | INFO | High Water Breakdown: | 20260505 17:41:10.583 | INFO | 61% : 8.66 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 17:41:10.583 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 17:41:10.583 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 17:41:10.583 | INFO | ------------------------------------------ | 20260505 17:41:10.583 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:41:10.584 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 17:41:10.584 | INFO | Total mem usage: 26.5 GiB | 20260505 17:41:10.584 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 17:41:10.584 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 17:41:10.584 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 17:41:10.584 | INFO | High Water Breakdown: | 20260505 17:41:10.584 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:41:10.584 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:41:10.584 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:41:10.584 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:41:10.584 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:41:10.584 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:41:10.584 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 17:41:10.584 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 17:41:10.584 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:41:10.584 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 17:41:10.584 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 17:41:10.584 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 17:41:10.584 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 17:41:10.584 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 17:41:10.584 | INFO | ------------------------------------------ | 20260505 17:41:10.584 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:41:10.584 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 17:41:10.584 | INFO | Total mem usage: 25.6 GiB | 20260505 17:41:10.584 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 17:41:10.584 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 17:41:10.584 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 17:41:10.584 | INFO | High Water Breakdown: | 20260505 17:41:10.584 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 17:41:10.584 | INFO | ------------------------------------------ | 20260505 17:41:10.584 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:41:10.584 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 17:41:10.584 | INFO | Total mem usage: 15.7 GiB | 20260505 17:41:10.584 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 17:41:10.584 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 17:41:10.584 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 17:41:10.584 | INFO | High Water Breakdown: | 20260505 17:41:10.584 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 17:41:10.584 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 17:41:10.584 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 17:41:10.584 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 17:41:10.584 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 17:41:10.584 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 17:41:10.584 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 17:41:10.584 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 17:41:10.584 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 17:41:10.584 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:41:10.584 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 17:41:10.584 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:41:10.584 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 17:41:10.584 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 17:41:10.584 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 17:41:10.584 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 17:41:10.584 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:41:10.584 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 17:41:10.584 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:41:10.584 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:41:10.584 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:41:10.584 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 17:41:10.584 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 17:41:10.584 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:41:10.584 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 17:41:10.584 | INFO | ------------------------------------------ | 20260505 17:41:10.584 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:41:10.584 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 17:41:10.584 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:41:10.585 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:41:10.585 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.49e+01, 1.36e+01 / 1.96e+00 | 20260505 17:41:10.585 | INFO | PerfReport: 0.29%, AggregateStats -- 9.26e-02 / 1.02e-01, 1.06e-02 / 6.93e-03 | 20260505 17:41:10.585 | INFO | PerfReport: 11.78%, Baseline -- 3.73e+00 / 4.50e+00, 5.74e-01 / 1.62e-01 | 20260505 17:41:10.585 | INFO | PerfReport: 3.29%, Histogram -- 1.04e+00 / 1.52e+00, 1.96e-01 / 9.40e-02 | 20260505 17:41:10.585 | INFO | PerfReport: 15.97%, DME -- 4.04e+01 / 4.49e+01, 4.41e+00 / 2.14e+00 | 20260505 17:41:10.585 | INFO | PerfReport: 56.83%, FrameLabeling -- 1.80e+01 / 1.91e+01, 1.47e+00 / 9.94e-01 | 20260505 17:41:10.585 | INFO | PerfReport: 7.31%, PulseAccumulating -- 2.31e+00 / 2.91e+00, 4.52e-01 / 1.89e-01 | 20260505 17:41:10.585 | INFO | PerfReport: 3.36%, Metrics -- 1.06e+00 / 1.38e+00, 2.95e-01 / 1.32e-01 | 20260505 17:41:10.585 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.70e-01 / 4.31e-01, 6.69e-02 / 4.51e-02 | 20260505 17:41:10.585 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 17:41:10.585 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:41:10.585 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:41:10.585 | INFO | PerfReport: Full Execution -- 9.99e+01 / -nan, 2.57e+01 / 0.00e+00 | 20260505 17:41:10.585 | INFO | PerfReport: 22.51%, Upload -- 2.25e+01 / 0.00e+00, 7.86e+00 / 0.00e+00 | 20260505 17:41:10.585 | INFO | PerfReport: 21.84%, Download -- 2.18e+01 / 0.00e+00, 7.86e+00 / 0.00e+00 | 20260505 17:41:10.585 | INFO | PerfReport: 31.65%, Compute -- 3.16e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 17:41:10.585 | INFO | PerfReport: 2.24%, LockUpload -- 2.24e+00 / 0.00e+00, 7.78e+00 / 0.00e+00 | 20260505 17:41:10.585 | INFO | PerfReport: 1.44%, LockDownload -- 1.44e+00 / 0.00e+00, 2.96e+00 / 0.00e+00 | 20260505 17:41:10.585 | INFO | PerfReport: 20.31%, LockCompute -- 2.03e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 17:41:10.585 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.11e-02 / 0.00e+00 | 20260505 17:41:10.713 | INFO | Starting a round of Baz IO | 20260505 17:41:13.916 | INFO | Populating RTMetrics report | 20260505 17:41:13.917 | INFO | RTMetrics report populated | 20260505 17:41:13.917 | INFO | Opening tmp RTMetrics json file | 20260505 17:41:13.928 | INFO | Serializing RTMetrics to json | 20260505 17:41:13.930 | INFO | Serialization done, swapping tmp file now... | 20260505 17:41:13.930 | INFO | RTMetrics done | 20260505 17:41:14.012 | INFO | Finished a round of Baz IO: Wrote 2.122 GiB in 3.299 seconds | 20260505 17:41:14.012 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.02% / 0.00% / 77.98% | 20260505 17:41:14.012 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.287012% / 0% / 0.000123183% | 20260505 17:41:14.012 | INFO | Baz IO overall: | 20260505 17:41:14.012 | INFO | BytesWritten : 3000125271776 | 20260505 17:41:14.012 | INFO | NumEvents : 1422326070186 | 20260505 17:41:15.726 | INFO | Analyzing chunk frames = [5079040,5079552) | 20260505 17:41:15.726 | INFO | MemInfo: { | 20260505 17:41:15.726 | INFO | "gpu" : " 17 GB", | 20260505 17:41:15.726 | INFO | "huge" : "", | 20260505 17:41:15.726 | INFO | "huge_shared" : " 27 GB", | 20260505 17:41:15.726 | INFO | "malloc" : " 14 GB", | 20260505 17:41:15.726 | INFO | "pinned" : " 29 GB" | 20260505 17:41:15.726 | INFO | } | 20260505 17:41:15.726 | INFO | Perf report for the last 5.14327 seconds: | 20260505 17:41:15.726 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:41:15.726 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:15.726 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:15.726 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:15.726 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 17:41:15.726 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:15.726 | INFO | RT_METRICS : 44.7%, 1.0, 23.4ms, 0 | 20260505 17:41:15.726 | INFO | PRE_HQ : 60.6%, 1.8, 56.2ms, 0 | 20260505 17:41:15.726 | INFO | BAZWRITER : 63.5%, 3.8, 125.0ms, 0 | 20260505 17:41:15.726 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 17:41:15.726 | INFO | Uploaded 36GiB to GPU in 2.1618s (16.6528GiB/s) | 20260505 17:41:15.727 | INFO | Downloaded 38GiB to GPU in 2.25716s (16.8353GiB/s) | 20260505 17:41:20.875 | INFO | Analyzing chunk frames = [5079552,5080064) | 20260505 17:41:20.875 | INFO | MemInfo: { | 20260505 17:41:20.875 | INFO | "gpu" : " 17 GB", | 20260505 17:41:20.875 | INFO | "huge" : "", | 20260505 17:41:20.875 | INFO | "huge_shared" : " 27 GB", | 20260505 17:41:20.875 | INFO | "malloc" : " 14 GB", | 20260505 17:41:20.875 | INFO | "pinned" : " 29 GB" | 20260505 17:41:20.875 | INFO | } | 20260505 17:41:20.875 | INFO | Perf report for the last 5.14908 seconds: | 20260505 17:41:20.875 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:41:20.875 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:20.875 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:20.875 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:20.875 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 17:41:20.875 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:20.875 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:20.875 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260505 17:41:20.875 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:20.875 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:20.876 | INFO | Uploaded 36GiB to GPU in 2.13807s (16.8376GiB/s) | 20260505 17:41:20.876 | INFO | Downloaded 34GiB to GPU in 2.02805s (16.7649GiB/s) | 20260505 17:41:26.010 | INFO | Analyzing chunk frames = [5080064,5080576) | 20260505 17:41:26.010 | INFO | MemInfo: { | 20260505 17:41:26.010 | INFO | "gpu" : " 17 GB", | 20260505 17:41:26.010 | INFO | "huge" : "", | 20260505 17:41:26.010 | INFO | "huge_shared" : " 27 GB", | 20260505 17:41:26.010 | INFO | "malloc" : " 14 GB", | 20260505 17:41:26.010 | INFO | "pinned" : " 29 GB" | 20260505 17:41:26.010 | INFO | } | 20260505 17:41:26.010 | INFO | Perf report for the last 5.13485 seconds: | 20260505 17:41:26.010 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:41:26.010 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:26.010 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:26.010 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:26.010 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260505 17:41:26.010 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:26.010 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:26.010 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 17:41:26.010 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:26.010 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:26.011 | INFO | Uploaded 36GiB to GPU in 2.13339s (16.8746GiB/s) | 20260505 17:41:26.011 | INFO | Downloaded 34GiB to GPU in 2.03863s (16.6779GiB/s) | 20260505 17:41:31.143 | INFO | Analyzing chunk frames = [5080576,5081088) | 20260505 17:41:31.143 | INFO | MemInfo: { | 20260505 17:41:31.143 | INFO | "gpu" : " 17 GB", | 20260505 17:41:31.143 | INFO | "huge" : "", | 20260505 17:41:31.143 | INFO | "huge_shared" : " 27 GB", | 20260505 17:41:31.143 | INFO | "malloc" : " 14 GB", | 20260505 17:41:31.143 | INFO | "pinned" : " 29 GB" | 20260505 17:41:31.143 | INFO | } | 20260505 17:41:31.143 | INFO | Perf report for the last 5.1334 seconds: | 20260505 17:41:31.144 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:41:31.144 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:31.144 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:31.144 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:31.144 | INFO | ANALYSIS : 62.5%, 3.0, 97.0ms, 0 | 20260505 17:41:31.144 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:31.144 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:31.144 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 17:41:31.144 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:31.144 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:31.144 | INFO | Uploaded 36GiB to GPU in 2.19286s (16.4169GiB/s) | 20260505 17:41:31.144 | INFO | Downloaded 34GiB to GPU in 2.04642s (16.6144GiB/s) | 20260505 17:41:36.278 | INFO | Analyzing chunk frames = [5081088,5081600) | 20260505 17:41:36.278 | INFO | MemInfo: { | 20260505 17:41:36.278 | INFO | "gpu" : " 17 GB", | 20260505 17:41:36.278 | INFO | "huge" : "", | 20260505 17:41:36.278 | INFO | "huge_shared" : " 27 GB", | 20260505 17:41:36.278 | INFO | "malloc" : " 14 GB", | 20260505 17:41:36.278 | INFO | "pinned" : " 29 GB" | 20260505 17:41:36.278 | INFO | } | 20260505 17:41:36.278 | INFO | Perf report for the last 5.13474 seconds: | 20260505 17:41:36.278 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:41:36.278 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:36.278 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:36.278 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:36.278 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 17:41:36.278 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:36.278 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:36.279 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:41:36.279 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:36.279 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:36.279 | INFO | Uploaded 36GiB to GPU in 2.16148s (16.6553GiB/s) | 20260505 17:41:36.279 | INFO | Downloaded 34GiB to GPU in 2.05732s (16.5263GiB/s) | 20260505 17:41:41.426 | INFO | Analyzing chunk frames = [5081600,5082112) | 20260505 17:41:41.426 | INFO | MemInfo: { | 20260505 17:41:41.426 | INFO | "gpu" : " 17 GB", | 20260505 17:41:41.426 | INFO | "huge" : "", | 20260505 17:41:41.426 | INFO | "huge_shared" : " 27 GB", | 20260505 17:41:41.426 | INFO | "malloc" : " 14 GB", | 20260505 17:41:41.426 | INFO | "pinned" : " 29 GB" | 20260505 17:41:41.426 | INFO | } | 20260505 17:41:41.426 | INFO | Perf report for the last 5.14815 seconds: | 20260505 17:41:41.427 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:41:41.427 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:41.427 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:41.427 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:41.427 | INFO | ANALYSIS : 62.4%, 3.0, 97.7ms, 0 | 20260505 17:41:41.427 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:41.427 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:41.427 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 17:41:41.427 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:41.427 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:41.427 | INFO | Uploaded 36GiB to GPU in 2.1442s (16.7895GiB/s) | 20260505 17:41:41.427 | INFO | Downloaded 34GiB to GPU in 2.04084s (16.6598GiB/s) | 20260505 17:41:46.570 | INFO | Analyzing chunk frames = [5082112,5082624) | 20260505 17:41:46.570 | INFO | MemInfo: { | 20260505 17:41:46.570 | INFO | "gpu" : " 17 GB", | 20260505 17:41:46.570 | INFO | "huge" : "", | 20260505 17:41:46.570 | INFO | "huge_shared" : " 27 GB", | 20260505 17:41:46.570 | INFO | "malloc" : " 14 GB", | 20260505 17:41:46.570 | INFO | "pinned" : " 29 GB" | 20260505 17:41:46.570 | INFO | } | 20260505 17:41:46.570 | INFO | Perf report for the last 5.14328 seconds: | 20260505 17:41:46.570 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:41:46.570 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:46.570 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:46.570 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:46.570 | INFO | ANALYSIS : 63.2%, 3.0, 98.8ms, 0 | 20260505 17:41:46.570 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:46.570 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:46.570 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 17:41:46.570 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:46.570 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:46.570 | INFO | Uploaded 36GiB to GPU in 2.13206s (16.8851GiB/s) | 20260505 17:41:46.570 | INFO | Downloaded 34GiB to GPU in 2.13674s (15.9121GiB/s) | 20260505 17:41:51.711 | INFO | Analyzing chunk frames = [5082624,5083136) | 20260505 17:41:51.711 | INFO | MemInfo: { | 20260505 17:41:51.711 | INFO | "gpu" : " 17 GB", | 20260505 17:41:51.711 | INFO | "huge" : "", | 20260505 17:41:51.711 | INFO | "huge_shared" : " 27 GB", | 20260505 17:41:51.711 | INFO | "malloc" : " 14 GB", | 20260505 17:41:51.711 | INFO | "pinned" : " 29 GB" | 20260505 17:41:51.711 | INFO | } | 20260505 17:41:51.711 | INFO | Perf report for the last 5.14155 seconds: | 20260505 17:41:51.711 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:41:51.711 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:51.711 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:51.711 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:51.711 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 17:41:51.711 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:51.711 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:51.712 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:41:51.712 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:51.712 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:51.712 | INFO | Uploaded 36GiB to GPU in 2.20294s (16.3418GiB/s) | 20260505 17:41:51.712 | INFO | Downloaded 34GiB to GPU in 2.0475s (16.6056GiB/s) | 20260505 17:41:51.846 | INFO | Starting a round of Baz IO | 20260505 17:41:55.030 | INFO | Populating RTMetrics report | 20260505 17:41:55.030 | INFO | RTMetrics report populated | 20260505 17:41:55.030 | INFO | Opening tmp RTMetrics json file | 20260505 17:41:55.032 | INFO | Serializing RTMetrics to json | 20260505 17:41:55.034 | INFO | Serialization done, swapping tmp file now... | 20260505 17:41:55.034 | INFO | RTMetrics done | 20260505 17:41:55.124 | INFO | Finished a round of Baz IO: Wrote 2.12 GiB in 3.278 seconds | 20260505 17:41:55.124 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.02% / 0.00% / 77.98% | 20260505 17:41:55.124 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.285172% / 3.97364e-06% / 0.000115236% | 20260505 17:41:55.124 | INFO | Baz IO overall: | 20260505 17:41:55.124 | INFO | BytesWritten : 3002402021674 | 20260505 17:41:55.124 | INFO | NumEvents : 1423389175751 | 20260505 17:41:56.841 | INFO | Analyzing chunk frames = [5083136,5083648) | 20260505 17:41:56.841 | INFO | MemInfo: { | 20260505 17:41:56.841 | INFO | "gpu" : " 17 GB", | 20260505 17:41:56.841 | INFO | "huge" : "", | 20260505 17:41:56.841 | INFO | "huge_shared" : " 27 GB", | 20260505 17:41:56.841 | INFO | "malloc" : " 14 GB", | 20260505 17:41:56.841 | INFO | "pinned" : " 29 GB" | 20260505 17:41:56.841 | INFO | } | 20260505 17:41:56.841 | INFO | Perf report for the last 5.13021 seconds: | 20260505 17:41:56.842 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:41:56.842 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:56.842 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:56.842 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:56.842 | INFO | ANALYSIS : 63.6%, 3.0, 99.0ms, 0 | 20260505 17:41:56.842 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:41:56.842 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260505 17:41:56.842 | INFO | PRE_HQ : 60.7%, 1.8, 56.1ms, 0 | 20260505 17:41:56.842 | INFO | BAZWRITER : 63.4%, 3.8, 124.5ms, 0 | 20260505 17:41:56.842 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 17:41:56.842 | INFO | Uploaded 36GiB to GPU in 2.17574s (16.5461GiB/s) | 20260505 17:41:56.842 | INFO | Downloaded 38GiB to GPU in 2.25117s (16.8801GiB/s) | 20260505 17:42:01.979 | INFO | Analyzing chunk frames = [5083648,5084160) | 20260505 17:42:01.979 | INFO | MemInfo: { | 20260505 17:42:01.979 | INFO | "gpu" : " 17 GB", | 20260505 17:42:01.979 | INFO | "huge" : "", | 20260505 17:42:01.979 | INFO | "huge_shared" : " 27 GB", | 20260505 17:42:01.979 | INFO | "malloc" : " 14 GB", | 20260505 17:42:01.979 | INFO | "pinned" : " 29 GB" | 20260505 17:42:01.979 | INFO | } | 20260505 17:42:01.979 | INFO | Perf report for the last 5.13765 seconds: | 20260505 17:42:01.980 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:42:01.980 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:01.980 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:01.980 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:01.980 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260505 17:42:01.980 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:01.980 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:01.980 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:42:01.980 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:01.980 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:01.980 | INFO | Uploaded 36GiB to GPU in 2.12636s (16.9303GiB/s) | 20260505 17:42:01.980 | INFO | Downloaded 34GiB to GPU in 2.03482s (16.7091GiB/s) | 20260505 17:42:07.122 | INFO | Analyzing chunk frames = [5084160,5084672) | 20260505 17:42:07.122 | INFO | MemInfo: { | 20260505 17:42:07.122 | INFO | "gpu" : " 17 GB", | 20260505 17:42:07.122 | INFO | "huge" : "", | 20260505 17:42:07.122 | INFO | "huge_shared" : " 27 GB", | 20260505 17:42:07.122 | INFO | "malloc" : " 14 GB", | 20260505 17:42:07.122 | INFO | "pinned" : " 29 GB" | 20260505 17:42:07.122 | INFO | } | 20260505 17:42:07.122 | INFO | Perf report for the last 5.14327 seconds: | 20260505 17:42:07.123 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:42:07.123 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:07.123 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:07.123 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:07.123 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 17:42:07.123 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:07.123 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:07.123 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:42:07.123 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:07.123 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:07.123 | INFO | Uploaded 36GiB to GPU in 2.20269s (16.3436GiB/s) | 20260505 17:42:07.123 | INFO | Downloaded 34GiB to GPU in 2.04235s (16.6475GiB/s) | 20260505 17:42:12.261 | INFO | Analyzing chunk frames = [5084672,5085184) | 20260505 17:42:12.261 | INFO | MemInfo: { | 20260505 17:42:12.261 | INFO | "gpu" : " 17 GB", | 20260505 17:42:12.261 | INFO | "huge" : "", | 20260505 17:42:12.261 | INFO | "huge_shared" : " 27 GB", | 20260505 17:42:12.261 | INFO | "malloc" : " 14 GB", | 20260505 17:42:12.261 | INFO | "pinned" : " 29 GB" | 20260505 17:42:12.261 | INFO | } | 20260505 17:42:12.261 | INFO | Perf report for the last 5.13839 seconds: | 20260505 17:42:12.261 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:42:12.261 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:12.261 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:12.261 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:12.261 | INFO | ANALYSIS : 62.8%, 3.0, 97.6ms, 0 | 20260505 17:42:12.261 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:12.261 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:12.261 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:42:12.261 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:12.261 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:12.261 | INFO | Uploaded 36GiB to GPU in 2.22666s (16.1677GiB/s) | 20260505 17:42:12.261 | INFO | Downloaded 34GiB to GPU in 2.12134s (16.0276GiB/s) | 20260505 17:42:17.396 | INFO | Analyzing chunk frames = [5085184,5085696) | 20260505 17:42:17.396 | INFO | MemInfo: { | 20260505 17:42:17.396 | INFO | "gpu" : " 17 GB", | 20260505 17:42:17.396 | INFO | "huge" : "", | 20260505 17:42:17.396 | INFO | "huge_shared" : " 27 GB", | 20260505 17:42:17.396 | INFO | "malloc" : " 14 GB", | 20260505 17:42:17.396 | INFO | "pinned" : " 29 GB" | 20260505 17:42:17.396 | INFO | } | 20260505 17:42:17.396 | INFO | Perf report for the last 5.13563 seconds: | 20260505 17:42:17.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:42:17.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:17.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:17.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:17.397 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 17:42:17.397 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:17.397 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:17.397 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:42:17.397 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:17.397 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:17.397 | INFO | Uploaded 36GiB to GPU in 2.14894s (16.7524GiB/s) | 20260505 17:42:17.397 | INFO | Downloaded 34GiB to GPU in 2.06023s (16.503GiB/s) | 20260505 17:42:22.533 | INFO | Analyzing chunk frames = [5085696,5086208) | 20260505 17:42:22.533 | INFO | MemInfo: { | 20260505 17:42:22.533 | INFO | "gpu" : " 17 GB", | 20260505 17:42:22.533 | INFO | "huge" : "", | 20260505 17:42:22.533 | INFO | "huge_shared" : " 27 GB", | 20260505 17:42:22.533 | INFO | "malloc" : " 14 GB", | 20260505 17:42:22.533 | INFO | "pinned" : " 29 GB" | 20260505 17:42:22.533 | INFO | } | 20260505 17:42:22.533 | INFO | Perf report for the last 5.13693 seconds: | 20260505 17:42:22.534 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:42:22.534 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:22.534 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:22.534 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:22.534 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260505 17:42:22.534 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:22.534 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:22.534 | INFO | PRE_HQ : 27.7%, 1.0, 14.3ms, 0 | 20260505 17:42:22.534 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:22.534 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:22.534 | INFO | Uploaded 36GiB to GPU in 2.14294s (16.7993GiB/s) | 20260505 17:42:22.534 | INFO | Downloaded 34GiB to GPU in 2.03865s (16.6777GiB/s) | 20260505 17:42:27.683 | INFO | Analyzing chunk frames = [5086208,5086720) | 20260505 17:42:27.683 | INFO | MemInfo: { | 20260505 17:42:27.683 | INFO | "gpu" : " 17 GB", | 20260505 17:42:27.683 | INFO | "huge" : "", | 20260505 17:42:27.683 | INFO | "huge_shared" : " 27 GB", | 20260505 17:42:27.683 | INFO | "malloc" : " 14 GB", | 20260505 17:42:27.683 | INFO | "pinned" : " 29 GB" | 20260505 17:42:27.683 | INFO | } | 20260505 17:42:27.683 | INFO | Perf report for the last 5.14921 seconds: | 20260505 17:42:27.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:42:27.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:27.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:27.683 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:27.683 | INFO | ANALYSIS : 63.4%, 3.0, 99.2ms, 0 | 20260505 17:42:27.683 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:27.683 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:27.683 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 17:42:27.683 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:27.683 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:27.683 | INFO | Uploaded 36GiB to GPU in 2.12586s (16.9344GiB/s) | 20260505 17:42:27.683 | INFO | Downloaded 34GiB to GPU in 2.17689s (15.6186GiB/s) | 20260505 17:42:32.825 | INFO | Analyzing chunk frames = [5086720,5087232) | 20260505 17:42:32.825 | INFO | MemInfo: { | 20260505 17:42:32.825 | INFO | "gpu" : " 17 GB", | 20260505 17:42:32.825 | INFO | "huge" : "", | 20260505 17:42:32.825 | INFO | "huge_shared" : " 27 GB", | 20260505 17:42:32.825 | INFO | "malloc" : " 14 GB", | 20260505 17:42:32.825 | INFO | "pinned" : " 29 GB" | 20260505 17:42:32.825 | INFO | } | 20260505 17:42:32.825 | INFO | Perf report for the last 5.14269 seconds: | 20260505 17:42:32.825 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:42:32.826 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:32.826 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:32.826 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:32.826 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 17:42:32.826 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:32.826 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:32.826 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 17:42:32.826 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:32.826 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:32.826 | INFO | Uploaded 36GiB to GPU in 2.19473s (16.4029GiB/s) | 20260505 17:42:32.826 | INFO | Downloaded 34GiB to GPU in 2.03072s (16.7428GiB/s) | 20260505 17:42:32.826 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 17:42:32.826 | INFO | Total mem usage: 15.3 GiB | 20260505 17:42:32.826 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 17:42:32.826 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.97%) | 20260505 17:42:32.826 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 17:42:32.826 | INFO | High Water Breakdown: | 20260505 17:42:32.826 | INFO | 61% : 8.66 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 17:42:32.826 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 17:42:32.826 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 17:42:32.826 | INFO | ------------------------------------------ | 20260505 17:42:32.826 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:42:32.826 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 17:42:32.826 | INFO | Total mem usage: 26.5 GiB | 20260505 17:42:32.826 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 17:42:32.826 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 17:42:32.826 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 17:42:32.826 | INFO | High Water Breakdown: | 20260505 17:42:32.826 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:42:32.826 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:42:32.826 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:42:32.826 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:42:32.826 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:42:32.826 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:42:32.826 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 17:42:32.826 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 17:42:32.826 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:42:32.826 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 17:42:32.827 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 17:42:32.827 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 17:42:32.827 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 17:42:32.827 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 17:42:32.827 | INFO | ------------------------------------------ | 20260505 17:42:32.827 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:42:32.827 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 17:42:32.827 | INFO | Total mem usage: 25.6 GiB | 20260505 17:42:32.827 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 17:42:32.827 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 17:42:32.827 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 17:42:32.827 | INFO | High Water Breakdown: | 20260505 17:42:32.827 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 17:42:32.827 | INFO | ------------------------------------------ | 20260505 17:42:32.827 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:42:32.827 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 17:42:32.827 | INFO | Total mem usage: 15.7 GiB | 20260505 17:42:32.827 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 17:42:32.827 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 17:42:32.827 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 17:42:32.827 | INFO | High Water Breakdown: | 20260505 17:42:32.827 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 17:42:32.827 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 17:42:32.827 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 17:42:32.827 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 17:42:32.827 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 17:42:32.827 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 17:42:32.827 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 17:42:32.827 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 17:42:32.827 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 17:42:32.827 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:42:32.827 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 17:42:32.827 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:42:32.827 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 17:42:32.827 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 17:42:32.827 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 17:42:32.827 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 17:42:32.827 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:42:32.827 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 17:42:32.827 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:42:32.827 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:42:32.827 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:42:32.827 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 17:42:32.827 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 17:42:32.827 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:42:32.827 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 17:42:32.827 | INFO | ------------------------------------------ | 20260505 17:42:32.827 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:42:32.827 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 17:42:32.827 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:42:32.827 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:42:32.827 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.48e+01, 1.35e+01 / 2.20e+00 | 20260505 17:42:32.827 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 1.04e-01, 1.07e-02 / 7.98e-03 | 20260505 17:42:32.827 | INFO | PerfReport: 11.75%, Baseline -- 3.71e+00 / 4.52e+00, 5.50e-01 / 1.14e-01 | 20260505 17:42:32.827 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.51e+00, 1.87e-01 / 1.11e-01 | 20260505 17:42:32.827 | INFO | PerfReport: 15.95%, DME -- 4.03e+01 / 4.46e+01, 4.20e+00 / 2.19e+00 | 20260505 17:42:32.827 | INFO | PerfReport: 56.88%, FrameLabeling -- 1.80e+01 / 1.93e+01, 1.47e+00 / 1.08e+00 | 20260505 17:42:32.827 | INFO | PerfReport: 7.33%, PulseAccumulating -- 2.32e+00 / 2.90e+00, 4.38e-01 / 1.79e-01 | 20260505 17:42:32.827 | INFO | PerfReport: 3.36%, Metrics -- 1.06e+00 / 1.42e+00, 2.91e-01 / 1.35e-01 | 20260505 17:42:32.827 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.69e-01 / 4.38e-01, 6.54e-02 / 3.82e-02 | 20260505 17:42:32.827 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 17:42:32.827 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:42:32.827 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:42:32.827 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.56e+01 / 0.00e+00 | 20260505 17:42:32.827 | INFO | PerfReport: 22.53%, Upload -- 2.25e+01 / 0.00e+00, 7.72e+00 / 0.00e+00 | 20260505 17:42:32.827 | INFO | PerfReport: 21.75%, Download -- 2.18e+01 / 0.00e+00, 7.65e+00 / 0.00e+00 | 20260505 17:42:32.827 | INFO | PerfReport: 31.58%, Compute -- 3.16e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 17:42:32.827 | INFO | PerfReport: 2.41%, LockUpload -- 2.41e+00 / 0.00e+00, 8.11e+00 / 0.00e+00 | 20260505 17:42:32.827 | INFO | PerfReport: 1.35%, LockDownload -- 1.35e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260505 17:42:32.827 | INFO | PerfReport: 20.37%, LockCompute -- 2.04e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 17:42:32.827 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.61e-02 / 0.00e+00 | 20260505 17:42:32.961 | INFO | Starting a round of Baz IO | 20260505 17:42:36.145 | INFO | Populating RTMetrics report | 20260505 17:42:36.145 | INFO | RTMetrics report populated | 20260505 17:42:36.145 | INFO | Opening tmp RTMetrics json file | 20260505 17:42:36.149 | INFO | Serializing RTMetrics to json | 20260505 17:42:36.150 | INFO | Serialization done, swapping tmp file now... | 20260505 17:42:36.151 | INFO | RTMetrics done | 20260505 17:42:36.244 | INFO | Finished a round of Baz IO: Wrote 2.125 GiB in 3.282 seconds | 20260505 17:42:36.244 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.02% / 0.00% / 77.98% | 20260505 17:42:36.244 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.287302% / 3.97364e-06% / 0.000107288% | 20260505 17:42:36.244 | INFO | Baz IO overall: | 20260505 17:42:36.244 | INFO | BytesWritten : 3004683481051 | 20260505 17:42:36.244 | INFO | NumEvents : 1424454225899 | 20260505 17:42:37.952 | INFO | Analyzing chunk frames = [5087232,5087744) | 20260505 17:42:37.952 | INFO | MemInfo: { | 20260505 17:42:37.952 | INFO | "gpu" : " 17 GB", | 20260505 17:42:37.952 | INFO | "huge" : "", | 20260505 17:42:37.952 | INFO | "huge_shared" : " 27 GB", | 20260505 17:42:37.952 | INFO | "malloc" : " 14 GB", | 20260505 17:42:37.952 | INFO | "pinned" : " 29 GB" | 20260505 17:42:37.952 | INFO | } | 20260505 17:42:37.952 | INFO | Perf report for the last 5.12673 seconds: | 20260505 17:42:37.952 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:42:37.952 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:37.952 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:37.952 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:37.952 | INFO | ANALYSIS : 63.7%, 3.0, 99.0ms, 0 | 20260505 17:42:37.952 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:37.952 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260505 17:42:37.952 | INFO | PRE_HQ : 60.4%, 1.8, 56.1ms, 0 | 20260505 17:42:37.952 | INFO | BAZWRITER : 63.5%, 3.8, 125.3ms, 0 | 20260505 17:42:37.952 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 17:42:37.952 | INFO | Uploaded 36GiB to GPU in 2.15195s (16.729GiB/s) | 20260505 17:42:37.953 | INFO | Downloaded 38GiB to GPU in 2.24768s (16.9063GiB/s) | 20260505 17:42:43.090 | INFO | Analyzing chunk frames = [5087744,5088256) | 20260505 17:42:43.090 | INFO | MemInfo: { | 20260505 17:42:43.090 | INFO | "gpu" : " 17 GB", | 20260505 17:42:43.090 | INFO | "huge" : "", | 20260505 17:42:43.090 | INFO | "huge_shared" : " 27 GB", | 20260505 17:42:43.090 | INFO | "malloc" : " 14 GB", | 20260505 17:42:43.090 | INFO | "pinned" : " 29 GB" | 20260505 17:42:43.090 | INFO | } | 20260505 17:42:43.090 | INFO | Perf report for the last 5.13758 seconds: | 20260505 17:42:43.090 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:42:43.090 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:43.090 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:43.090 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:43.090 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 17:42:43.090 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:43.090 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:43.090 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:42:43.090 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:43.090 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:43.090 | INFO | Uploaded 36GiB to GPU in 2.13646s (16.8503GiB/s) | 20260505 17:42:43.090 | INFO | Downloaded 34GiB to GPU in 2.03411s (16.7149GiB/s) | 20260505 17:42:48.236 | INFO | Analyzing chunk frames = [5088256,5088768) | 20260505 17:42:48.236 | INFO | MemInfo: { | 20260505 17:42:48.236 | INFO | "gpu" : " 17 GB", | 20260505 17:42:48.236 | INFO | "huge" : "", | 20260505 17:42:48.236 | INFO | "huge_shared" : " 27 GB", | 20260505 17:42:48.236 | INFO | "malloc" : " 14 GB", | 20260505 17:42:48.236 | INFO | "pinned" : " 29 GB" | 20260505 17:42:48.236 | INFO | } | 20260505 17:42:48.236 | INFO | Perf report for the last 5.14678 seconds: | 20260505 17:42:48.237 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:42:48.237 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:48.237 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:48.237 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:48.237 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 17:42:48.237 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:48.237 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:48.237 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:42:48.237 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:48.237 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:48.237 | INFO | Uploaded 36GiB to GPU in 2.18522s (16.4743GiB/s) | 20260505 17:42:48.237 | INFO | Downloaded 34GiB to GPU in 2.03497s (16.7079GiB/s) | 20260505 17:42:53.374 | INFO | Analyzing chunk frames = [5088768,5089280) | 20260505 17:42:53.374 | INFO | MemInfo: { | 20260505 17:42:53.374 | INFO | "gpu" : " 17 GB", | 20260505 17:42:53.374 | INFO | "huge" : "", | 20260505 17:42:53.374 | INFO | "huge_shared" : " 27 GB", | 20260505 17:42:53.375 | INFO | "malloc" : " 14 GB", | 20260505 17:42:53.375 | INFO | "pinned" : " 29 GB" | 20260505 17:42:53.375 | INFO | } | 20260505 17:42:53.375 | INFO | Perf report for the last 5.13818 seconds: | 20260505 17:42:53.375 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:42:53.375 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:53.375 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:53.375 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:53.375 | INFO | ANALYSIS : 62.9%, 3.0, 97.6ms, 0 | 20260505 17:42:53.375 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:53.375 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:53.375 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 17:42:53.375 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:53.375 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:53.375 | INFO | Uploaded 36GiB to GPU in 2.22474s (16.1817GiB/s) | 20260505 17:42:53.375 | INFO | Downloaded 34GiB to GPU in 2.12426s (16.0055GiB/s) | 20260505 17:42:58.519 | INFO | Analyzing chunk frames = [5089280,5089792) | 20260505 17:42:58.519 | INFO | MemInfo: { | 20260505 17:42:58.519 | INFO | "gpu" : " 17 GB", | 20260505 17:42:58.519 | INFO | "huge" : "", | 20260505 17:42:58.519 | INFO | "huge_shared" : " 27 GB", | 20260505 17:42:58.519 | INFO | "malloc" : " 14 GB", | 20260505 17:42:58.519 | INFO | "pinned" : " 29 GB" | 20260505 17:42:58.519 | INFO | } | 20260505 17:42:58.519 | INFO | Perf report for the last 5.1441 seconds: | 20260505 17:42:58.519 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:42:58.519 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:58.519 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:58.519 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:58.519 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260505 17:42:58.519 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:58.519 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:58.519 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 17:42:58.519 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:58.519 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:42:58.519 | INFO | Uploaded 36GiB to GPU in 2.143s (16.7989GiB/s) | 20260505 17:42:58.519 | INFO | Downloaded 34GiB to GPU in 2.06024s (16.503GiB/s) | 20260505 17:43:03.654 | INFO | Analyzing chunk frames = [5089792,5090304) | 20260505 17:43:03.654 | INFO | MemInfo: { | 20260505 17:43:03.654 | INFO | "gpu" : " 17 GB", | 20260505 17:43:03.654 | INFO | "huge" : "", | 20260505 17:43:03.654 | INFO | "huge_shared" : " 27 GB", | 20260505 17:43:03.654 | INFO | "malloc" : " 14 GB", | 20260505 17:43:03.654 | INFO | "pinned" : " 29 GB" | 20260505 17:43:03.654 | INFO | } | 20260505 17:43:03.654 | INFO | Perf report for the last 5.1351 seconds: | 20260505 17:43:03.654 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:43:03.654 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:03.654 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:03.654 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:03.654 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260505 17:43:03.654 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:03.654 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:03.654 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:43:03.654 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:03.654 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:03.654 | INFO | Uploaded 36GiB to GPU in 2.14427s (16.7889GiB/s) | 20260505 17:43:03.654 | INFO | Downloaded 34GiB to GPU in 2.05107s (16.5767GiB/s) | 20260505 17:43:08.787 | INFO | Analyzing chunk frames = [5090304,5090816) | 20260505 17:43:08.787 | INFO | MemInfo: { | 20260505 17:43:08.787 | INFO | "gpu" : " 17 GB", | 20260505 17:43:08.787 | INFO | "huge" : "", | 20260505 17:43:08.787 | INFO | "huge_shared" : " 27 GB", | 20260505 17:43:08.787 | INFO | "malloc" : " 14 GB", | 20260505 17:43:08.787 | INFO | "pinned" : " 29 GB" | 20260505 17:43:08.787 | INFO | } | 20260505 17:43:08.787 | INFO | Perf report for the last 5.13349 seconds: | 20260505 17:43:08.787 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:43:08.787 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:08.787 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:08.787 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:08.787 | INFO | ANALYSIS : 63.5%, 3.0, 99.0ms, 0 | 20260505 17:43:08.787 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:08.787 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:08.787 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 17:43:08.787 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:08.787 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:08.788 | INFO | Uploaded 36GiB to GPU in 2.1303s (16.899GiB/s) | 20260505 17:43:08.788 | INFO | Downloaded 34GiB to GPU in 2.14273s (15.8676GiB/s) | 20260505 17:43:13.934 | INFO | Analyzing chunk frames = [5090816,5091328) | 20260505 17:43:13.934 | INFO | MemInfo: { | 20260505 17:43:13.934 | INFO | "gpu" : " 17 GB", | 20260505 17:43:13.934 | INFO | "huge" : "", | 20260505 17:43:13.934 | INFO | "huge_shared" : " 27 GB", | 20260505 17:43:13.934 | INFO | "malloc" : " 14 GB", | 20260505 17:43:13.934 | INFO | "pinned" : " 29 GB" | 20260505 17:43:13.934 | INFO | } | 20260505 17:43:13.934 | INFO | Perf report for the last 5.14675 seconds: | 20260505 17:43:13.934 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:43:13.934 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:13.934 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:13.934 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:13.934 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 17:43:13.934 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:13.934 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:13.934 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 17:43:13.934 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:13.934 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:13.934 | INFO | Uploaded 36GiB to GPU in 2.16334s (16.6409GiB/s) | 20260505 17:43:13.934 | INFO | Downloaded 34GiB to GPU in 2.03927s (16.6726GiB/s) | 20260505 17:43:14.069 | INFO | Starting a round of Baz IO | 20260505 17:43:17.245 | INFO | Populating RTMetrics report | 20260505 17:43:17.246 | INFO | RTMetrics report populated | 20260505 17:43:17.246 | INFO | Opening tmp RTMetrics json file | 20260505 17:43:17.254 | INFO | Serializing RTMetrics to json | 20260505 17:43:17.257 | INFO | Serialization done, swapping tmp file now... | 20260505 17:43:17.258 | INFO | RTMetrics done | 20260505 17:43:17.346 | INFO | Finished a round of Baz IO: Wrote 2.128 GiB in 3.276 seconds | 20260505 17:43:17.346 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.02% / 0.00% / 77.98% | 20260505 17:43:17.346 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.287688% / 0% / 9.93411e-05% | 20260505 17:43:17.346 | INFO | Baz IO overall: | 20260505 17:43:17.346 | INFO | BytesWritten : 3006968578546 | 20260505 17:43:17.346 | INFO | NumEvents : 1425520406329 | 20260505 17:43:19.073 | INFO | Analyzing chunk frames = [5091328,5091840) | 20260505 17:43:19.073 | INFO | MemInfo: { | 20260505 17:43:19.073 | INFO | "gpu" : " 17 GB", | 20260505 17:43:19.073 | INFO | "huge" : "", | 20260505 17:43:19.073 | INFO | "huge_shared" : " 27 GB", | 20260505 17:43:19.073 | INFO | "malloc" : " 14 GB", | 20260505 17:43:19.073 | INFO | "pinned" : " 29 GB" | 20260505 17:43:19.073 | INFO | } | 20260505 17:43:19.073 | INFO | Perf report for the last 5.13885 seconds: | 20260505 17:43:19.073 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:43:19.073 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:19.073 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:19.073 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:19.073 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 17:43:19.073 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:19.073 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260505 17:43:19.073 | INFO | PRE_HQ : 60.5%, 1.8, 56.2ms, 0 | 20260505 17:43:19.073 | INFO | BAZWRITER : 63.3%, 3.9, 126.0ms, 0 | 20260505 17:43:19.073 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 17:43:19.073 | INFO | Uploaded 36GiB to GPU in 2.16198s (16.6514GiB/s) | 20260505 17:43:19.073 | INFO | Downloaded 38GiB to GPU in 2.24654s (16.9149GiB/s) | 20260505 17:43:24.212 | INFO | Analyzing chunk frames = [5091840,5092352) | 20260505 17:43:24.212 | INFO | MemInfo: { | 20260505 17:43:24.212 | INFO | "gpu" : " 17 GB", | 20260505 17:43:24.212 | INFO | "huge" : "", | 20260505 17:43:24.212 | INFO | "huge_shared" : " 27 GB", | 20260505 17:43:24.212 | INFO | "malloc" : " 14 GB", | 20260505 17:43:24.212 | INFO | "pinned" : " 29 GB" | 20260505 17:43:24.212 | INFO | } | 20260505 17:43:24.212 | INFO | Perf report for the last 5.13926 seconds: | 20260505 17:43:24.212 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:43:24.212 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:24.212 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:24.212 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:24.212 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 17:43:24.212 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:24.212 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:24.212 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 17:43:24.212 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:24.212 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:24.213 | INFO | Uploaded 36GiB to GPU in 2.14002s (16.8222GiB/s) | 20260505 17:43:24.213 | INFO | Downloaded 34GiB to GPU in 2.0307s (16.743GiB/s) | 20260505 17:43:29.346 | INFO | Analyzing chunk frames = [5092352,5092864) | 20260505 17:43:29.346 | INFO | MemInfo: { | 20260505 17:43:29.346 | INFO | "gpu" : " 17 GB", | 20260505 17:43:29.346 | INFO | "huge" : "", | 20260505 17:43:29.346 | INFO | "huge_shared" : " 27 GB", | 20260505 17:43:29.346 | INFO | "malloc" : " 14 GB", | 20260505 17:43:29.346 | INFO | "pinned" : " 29 GB" | 20260505 17:43:29.346 | INFO | } | 20260505 17:43:29.346 | INFO | Perf report for the last 5.13383 seconds: | 20260505 17:43:29.346 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:43:29.346 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:29.346 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:29.346 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:29.346 | INFO | ANALYSIS : 62.7%, 3.0, 97.5ms, 0 | 20260505 17:43:29.346 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:29.346 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:29.346 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:43:29.346 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:29.346 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:29.346 | INFO | Uploaded 36GiB to GPU in 2.19647s (16.3899GiB/s) | 20260505 17:43:29.346 | INFO | Downloaded 34GiB to GPU in 2.03771s (16.6854GiB/s) | 20260505 17:43:34.488 | INFO | Analyzing chunk frames = [5092864,5093376) | 20260505 17:43:34.488 | INFO | MemInfo: { | 20260505 17:43:34.488 | INFO | "gpu" : " 17 GB", | 20260505 17:43:34.488 | INFO | "huge" : "", | 20260505 17:43:34.488 | INFO | "huge_shared" : " 27 GB", | 20260505 17:43:34.488 | INFO | "malloc" : " 14 GB", | 20260505 17:43:34.488 | INFO | "pinned" : " 29 GB" | 20260505 17:43:34.488 | INFO | } | 20260505 17:43:34.488 | INFO | Perf report for the last 5.14213 seconds: | 20260505 17:43:34.488 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:43:34.488 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:34.488 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:34.488 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:34.488 | INFO | ANALYSIS : 62.5%, 3.0, 97.1ms, 0 | 20260505 17:43:34.488 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:34.488 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:34.488 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 17:43:34.488 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:34.488 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:34.488 | INFO | Uploaded 36GiB to GPU in 2.16424s (16.634GiB/s) | 20260505 17:43:34.489 | INFO | Downloaded 34GiB to GPU in 2.14221s (15.8715GiB/s) | 20260505 17:43:39.631 | INFO | Analyzing chunk frames = [5093376,5093888) | 20260505 17:43:39.631 | INFO | MemInfo: { | 20260505 17:43:39.631 | INFO | "gpu" : " 17 GB", | 20260505 17:43:39.631 | INFO | "huge" : "", | 20260505 17:43:39.631 | INFO | "huge_shared" : " 27 GB", | 20260505 17:43:39.631 | INFO | "malloc" : " 14 GB", | 20260505 17:43:39.631 | INFO | "pinned" : " 29 GB" | 20260505 17:43:39.631 | INFO | } | 20260505 17:43:39.631 | INFO | Perf report for the last 5.14312 seconds: | 20260505 17:43:39.631 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:43:39.631 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:39.631 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:39.631 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:39.631 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260505 17:43:39.631 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:39.631 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:39.631 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:43:39.631 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:39.631 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:39.632 | INFO | Uploaded 36GiB to GPU in 2.13875s (16.8323GiB/s) | 20260505 17:43:39.632 | INFO | Downloaded 34GiB to GPU in 2.06067s (16.4995GiB/s) | 20260505 17:43:44.770 | INFO | Analyzing chunk frames = [5093888,5094400) | 20260505 17:43:44.770 | INFO | MemInfo: { | 20260505 17:43:44.770 | INFO | "gpu" : " 17 GB", | 20260505 17:43:44.770 | INFO | "huge" : "", | 20260505 17:43:44.770 | INFO | "huge_shared" : " 27 GB", | 20260505 17:43:44.770 | INFO | "malloc" : " 14 GB", | 20260505 17:43:44.770 | INFO | "pinned" : " 29 GB" | 20260505 17:43:44.770 | INFO | } | 20260505 17:43:44.770 | INFO | Perf report for the last 5.13859 seconds: | 20260505 17:43:44.770 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:43:44.770 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:44.770 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:44.770 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:44.770 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260505 17:43:44.770 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:44.770 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:44.770 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:43:44.770 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:44.770 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:44.770 | INFO | Uploaded 36GiB to GPU in 2.14333s (16.7963GiB/s) | 20260505 17:43:44.770 | INFO | Downloaded 34GiB to GPU in 2.05219s (16.5677GiB/s) | 20260505 17:43:49.900 | INFO | Analyzing chunk frames = [5094400,5094912) | 20260505 17:43:49.900 | INFO | MemInfo: { | 20260505 17:43:49.900 | INFO | "gpu" : " 17 GB", | 20260505 17:43:49.900 | INFO | "huge" : "", | 20260505 17:43:49.900 | INFO | "huge_shared" : " 27 GB", | 20260505 17:43:49.900 | INFO | "malloc" : " 14 GB", | 20260505 17:43:49.900 | INFO | "pinned" : " 29 GB" | 20260505 17:43:49.900 | INFO | } | 20260505 17:43:49.900 | INFO | Perf report for the last 5.13049 seconds: | 20260505 17:43:49.900 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:43:49.900 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:49.900 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:49.900 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:49.901 | INFO | ANALYSIS : 63.4%, 3.0, 98.8ms, 0 | 20260505 17:43:49.901 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:49.901 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:49.901 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 17:43:49.901 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:49.901 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:49.901 | INFO | Uploaded 36GiB to GPU in 2.14263s (16.8018GiB/s) | 20260505 17:43:49.901 | INFO | Downloaded 34GiB to GPU in 2.15507s (15.7767GiB/s) | 20260505 17:43:55.042 | INFO | Analyzing chunk frames = [5094912,5095424) | 20260505 17:43:55.042 | INFO | MemInfo: { | 20260505 17:43:55.042 | INFO | "gpu" : " 17 GB", | 20260505 17:43:55.042 | INFO | "huge" : "", | 20260505 17:43:55.042 | INFO | "huge_shared" : " 27 GB", | 20260505 17:43:55.042 | INFO | "malloc" : " 14 GB", | 20260505 17:43:55.042 | INFO | "pinned" : " 29 GB" | 20260505 17:43:55.042 | INFO | } | 20260505 17:43:55.043 | INFO | Perf report for the last 5.14204 seconds: | 20260505 17:43:55.043 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:43:55.043 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:55.043 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:55.043 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:55.043 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 17:43:55.043 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:55.043 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:55.043 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 17:43:55.043 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:55.043 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:43:55.043 | INFO | Uploaded 36GiB to GPU in 2.23002s (16.1434GiB/s) | 20260505 17:43:55.043 | INFO | Downloaded 34GiB to GPU in 2.10731s (16.1343GiB/s) | 20260505 17:43:55.043 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 17:43:55.043 | INFO | Total mem usage: 15.3 GiB | 20260505 17:43:55.043 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 17:43:55.043 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -7.25%) | 20260505 17:43:55.043 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 17:43:55.043 | INFO | High Water Breakdown: | 20260505 17:43:55.043 | INFO | 61.1% : 8.74 GiB ( 91.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 17:43:55.043 | INFO | 38.7% : 5.54 GiB ( 83.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 17:43:55.043 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 17:43:55.043 | INFO | ------------------------------------------ | 20260505 17:43:55.043 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:43:55.043 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 17:43:55.043 | INFO | Total mem usage: 26.5 GiB | 20260505 17:43:55.043 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 17:43:55.043 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 17:43:55.043 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 17:43:55.043 | INFO | High Water Breakdown: | 20260505 17:43:55.043 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:43:55.043 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:43:55.043 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:43:55.043 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:43:55.043 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:43:55.043 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:43:55.043 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 17:43:55.043 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 17:43:55.043 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:43:55.043 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 17:43:55.043 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 17:43:55.043 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 17:43:55.043 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 17:43:55.043 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 17:43:55.043 | INFO | ------------------------------------------ | 20260505 17:43:55.043 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:43:55.043 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 17:43:55.043 | INFO | Total mem usage: 25.6 GiB | 20260505 17:43:55.043 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 17:43:55.043 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 17:43:55.043 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 17:43:55.043 | INFO | High Water Breakdown: | 20260505 17:43:55.043 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 17:43:55.043 | INFO | ------------------------------------------ | 20260505 17:43:55.043 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:43:55.043 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 17:43:55.044 | INFO | Total mem usage: 15.7 GiB | 20260505 17:43:55.044 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 17:43:55.044 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 17:43:55.044 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 17:43:55.044 | INFO | High Water Breakdown: | 20260505 17:43:55.044 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 17:43:55.044 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 17:43:55.044 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 17:43:55.044 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 17:43:55.044 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 17:43:55.044 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 17:43:55.044 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 17:43:55.044 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 17:43:55.044 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 17:43:55.044 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:43:55.044 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 17:43:55.044 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:43:55.044 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 17:43:55.044 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 17:43:55.044 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 17:43:55.044 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 17:43:55.044 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:43:55.044 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 17:43:55.044 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:43:55.044 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:43:55.044 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:43:55.044 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 17:43:55.044 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 17:43:55.044 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:43:55.044 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 17:43:55.044 | INFO | ------------------------------------------ | 20260505 17:43:55.044 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:43:55.044 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 17:43:55.044 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:43:55.044 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:43:55.044 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.51e+01, 1.35e+01 / 2.10e+00 | 20260505 17:43:55.044 | INFO | PerfReport: 0.29%, AggregateStats -- 9.27e-02 / 1.01e-01, 1.07e-02 / 4.16e-03 | 20260505 17:43:55.044 | INFO | PerfReport: 11.79%, Baseline -- 3.72e+00 / 4.50e+00, 5.73e-01 / 1.81e-01 | 20260505 17:43:55.044 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.54e+00, 1.95e-01 / 6.57e-02 | 20260505 17:43:55.044 | INFO | PerfReport: 15.97%, DME -- 4.04e+01 / 4.49e+01, 4.39e+00 / 2.16e+00 | 20260505 17:43:55.044 | INFO | PerfReport: 56.80%, FrameLabeling -- 1.79e+01 / 1.92e+01, 1.45e+00 / 1.12e+00 | 20260505 17:43:55.044 | INFO | PerfReport: 7.32%, PulseAccumulating -- 2.31e+00 / 2.91e+00, 4.46e-01 / 1.56e-01 | 20260505 17:43:55.044 | INFO | PerfReport: 3.37%, Metrics -- 1.06e+00 / 1.40e+00, 2.94e-01 / 1.87e-01 | 20260505 17:43:55.044 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.71e-01 / 4.29e-01, 6.48e-02 / 3.90e-02 | 20260505 17:43:55.044 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 17:43:55.044 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:43:55.044 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:43:55.044 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.59e+01 / 0.00e+00 | 20260505 17:43:55.044 | INFO | PerfReport: 22.52%, Upload -- 2.25e+01 / 0.00e+00, 7.90e+00 / 0.00e+00 | 20260505 17:43:55.044 | INFO | PerfReport: 21.85%, Download -- 2.19e+01 / 0.00e+00, 7.87e+00 / 0.00e+00 | 20260505 17:43:55.044 | INFO | PerfReport: 31.59%, Compute -- 3.16e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 17:43:55.044 | INFO | PerfReport: 2.37%, LockUpload -- 2.37e+00 / 0.00e+00, 7.95e+00 / 0.00e+00 | 20260505 17:43:55.044 | INFO | PerfReport: 1.41%, LockDownload -- 1.41e+00 / 0.00e+00, 2.92e+00 / 0.00e+00 | 20260505 17:43:55.044 | INFO | PerfReport: 20.26%, LockCompute -- 2.03e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 17:43:55.044 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.87e-02 / 0.00e+00 | 20260505 17:43:55.176 | INFO | Starting a round of Baz IO | 20260505 17:43:58.373 | INFO | Populating RTMetrics report | 20260505 17:43:58.373 | INFO | RTMetrics report populated | 20260505 17:43:58.373 | INFO | Opening tmp RTMetrics json file | 20260505 17:43:58.380 | INFO | Serializing RTMetrics to json | 20260505 17:43:58.381 | INFO | Serialization done, swapping tmp file now... | 20260505 17:43:58.381 | INFO | RTMetrics done | 20260505 17:43:58.472 | INFO | Finished a round of Baz IO: Wrote 2.126 GiB in 3.296 seconds | 20260505 17:43:58.472 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.01% / 0.00% / 77.98% | 20260505 17:43:58.472 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.286277% / 0% / 0.000111262% | 20260505 17:43:58.472 | INFO | Baz IO overall: | 20260505 17:43:58.472 | INFO | BytesWritten : 3009251365171 | 20260505 17:43:58.472 | INFO | NumEvents : 1426586100376 | 20260505 17:44:00.194 | INFO | Analyzing chunk frames = [5095424,5095936) | 20260505 17:44:00.194 | INFO | MemInfo: { | 20260505 17:44:00.194 | INFO | "gpu" : " 17 GB", | 20260505 17:44:00.194 | INFO | "huge" : "", | 20260505 17:44:00.194 | INFO | "huge_shared" : " 27 GB", | 20260505 17:44:00.194 | INFO | "malloc" : " 14 GB", | 20260505 17:44:00.194 | INFO | "pinned" : " 29 GB" | 20260505 17:44:00.194 | INFO | } | 20260505 17:44:00.194 | INFO | Perf report for the last 5.15113 seconds: | 20260505 17:44:00.194 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:44:00.194 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:00.194 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:00.194 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:00.194 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 17:44:00.194 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:00.194 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260505 17:44:00.194 | INFO | PRE_HQ : 60.4%, 1.8, 56.1ms, 0 | 20260505 17:44:00.194 | INFO | BAZWRITER : 63.3%, 3.9, 125.7ms, 0 | 20260505 17:44:00.194 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 17:44:00.194 | INFO | Uploaded 36GiB to GPU in 2.19706s (16.3855GiB/s) | 20260505 17:44:00.194 | INFO | Downloaded 38GiB to GPU in 2.25509s (16.8508GiB/s) | 20260505 17:44:05.331 | INFO | Analyzing chunk frames = [5095936,5096448) | 20260505 17:44:05.331 | INFO | MemInfo: { | 20260505 17:44:05.331 | INFO | "gpu" : " 17 GB", | 20260505 17:44:05.331 | INFO | "huge" : "", | 20260505 17:44:05.331 | INFO | "huge_shared" : " 27 GB", | 20260505 17:44:05.331 | INFO | "malloc" : " 14 GB", | 20260505 17:44:05.331 | INFO | "pinned" : " 29 GB" | 20260505 17:44:05.331 | INFO | } | 20260505 17:44:05.331 | INFO | Perf report for the last 5.13738 seconds: | 20260505 17:44:05.331 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:44:05.331 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:05.331 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:05.331 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:05.331 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 17:44:05.331 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:05.331 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:05.331 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 17:44:05.331 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:05.331 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:05.331 | INFO | Uploaded 36GiB to GPU in 2.13303s (16.8774GiB/s) | 20260505 17:44:05.332 | INFO | Downloaded 34GiB to GPU in 2.02963s (16.7518GiB/s) | 20260505 17:44:10.465 | INFO | Analyzing chunk frames = [5096448,5096960) | 20260505 17:44:10.465 | INFO | MemInfo: { | 20260505 17:44:10.465 | INFO | "gpu" : " 17 GB", | 20260505 17:44:10.465 | INFO | "huge" : "", | 20260505 17:44:10.465 | INFO | "huge_shared" : " 27 GB", | 20260505 17:44:10.465 | INFO | "malloc" : " 14 GB", | 20260505 17:44:10.465 | INFO | "pinned" : " 29 GB" | 20260505 17:44:10.465 | INFO | } | 20260505 17:44:10.465 | INFO | Perf report for the last 5.1343 seconds: | 20260505 17:44:10.465 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:44:10.465 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:10.465 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:10.465 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:10.465 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260505 17:44:10.465 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:10.465 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:10.465 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:44:10.465 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:10.465 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:10.466 | INFO | Uploaded 36GiB to GPU in 2.14123s (16.8128GiB/s) | 20260505 17:44:10.466 | INFO | Downloaded 34GiB to GPU in 2.04012s (16.6657GiB/s) | 20260505 17:44:15.599 | INFO | Analyzing chunk frames = [5096960,5097472) | 20260505 17:44:15.599 | INFO | MemInfo: { | 20260505 17:44:15.599 | INFO | "gpu" : " 17 GB", | 20260505 17:44:15.599 | INFO | "huge" : "", | 20260505 17:44:15.599 | INFO | "huge_shared" : " 27 GB", | 20260505 17:44:15.599 | INFO | "malloc" : " 14 GB", | 20260505 17:44:15.599 | INFO | "pinned" : " 29 GB" | 20260505 17:44:15.599 | INFO | } | 20260505 17:44:15.599 | INFO | Perf report for the last 5.1335 seconds: | 20260505 17:44:15.599 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:44:15.599 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:15.599 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:15.599 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:15.599 | INFO | ANALYSIS : 62.6%, 3.0, 97.2ms, 0 | 20260505 17:44:15.599 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:15.599 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:15.599 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:44:15.599 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:15.599 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:15.599 | INFO | Uploaded 36GiB to GPU in 2.17734s (16.5339GiB/s) | 20260505 17:44:15.599 | INFO | Downloaded 34GiB to GPU in 2.14242s (15.8699GiB/s) | 20260505 17:44:20.740 | INFO | Analyzing chunk frames = [5097472,5097984) | 20260505 17:44:20.740 | INFO | MemInfo: { | 20260505 17:44:20.740 | INFO | "gpu" : " 17 GB", | 20260505 17:44:20.740 | INFO | "huge" : "", | 20260505 17:44:20.740 | INFO | "huge_shared" : " 27 GB", | 20260505 17:44:20.740 | INFO | "malloc" : " 14 GB", | 20260505 17:44:20.740 | INFO | "pinned" : " 29 GB" | 20260505 17:44:20.740 | INFO | } | 20260505 17:44:20.740 | INFO | Perf report for the last 5.14139 seconds: | 20260505 17:44:20.740 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:44:20.740 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:20.740 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:20.740 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:20.741 | INFO | ANALYSIS : 62.1%, 3.0, 97.1ms, 0 | 20260505 17:44:20.741 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:20.741 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:20.741 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260505 17:44:20.741 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:20.741 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:20.741 | INFO | Uploaded 36GiB to GPU in 2.16022s (16.665GiB/s) | 20260505 17:44:20.741 | INFO | Downloaded 34GiB to GPU in 2.0579s (16.5217GiB/s) | 20260505 17:44:25.882 | INFO | Analyzing chunk frames = [5097984,5098496) | 20260505 17:44:25.882 | INFO | MemInfo: { | 20260505 17:44:25.882 | INFO | "gpu" : " 17 GB", | 20260505 17:44:25.882 | INFO | "huge" : "", | 20260505 17:44:25.882 | INFO | "huge_shared" : " 27 GB", | 20260505 17:44:25.882 | INFO | "malloc" : " 14 GB", | 20260505 17:44:25.882 | INFO | "pinned" : " 29 GB" | 20260505 17:44:25.882 | INFO | } | 20260505 17:44:25.882 | INFO | Perf report for the last 5.14198 seconds: | 20260505 17:44:25.882 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:44:25.882 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:25.882 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:25.882 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:25.883 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260505 17:44:25.883 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:25.883 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:25.883 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 17:44:25.883 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:25.883 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:25.883 | INFO | Uploaded 36GiB to GPU in 2.14332s (16.7964GiB/s) | 20260505 17:44:25.883 | INFO | Downloaded 34GiB to GPU in 2.04419s (16.6325GiB/s) | 20260505 17:44:31.028 | INFO | Analyzing chunk frames = [5098496,5099008) | 20260505 17:44:31.028 | INFO | MemInfo: { | 20260505 17:44:31.028 | INFO | "gpu" : " 17 GB", | 20260505 17:44:31.028 | INFO | "huge" : "", | 20260505 17:44:31.028 | INFO | "huge_shared" : " 27 GB", | 20260505 17:44:31.028 | INFO | "malloc" : " 14 GB", | 20260505 17:44:31.028 | INFO | "pinned" : " 29 GB" | 20260505 17:44:31.028 | INFO | } | 20260505 17:44:31.028 | INFO | Perf report for the last 5.14548 seconds: | 20260505 17:44:31.028 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:44:31.028 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:31.028 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:31.028 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:31.028 | INFO | ANALYSIS : 63.1%, 3.0, 98.7ms, 0 | 20260505 17:44:31.028 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:31.028 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:31.028 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 17:44:31.028 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:31.028 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:31.028 | INFO | Uploaded 36GiB to GPU in 2.128s (16.9173GiB/s) | 20260505 17:44:31.028 | INFO | Downloaded 34GiB to GPU in 2.13778s (15.9043GiB/s) | 20260505 17:44:36.163 | INFO | Analyzing chunk frames = [5099008,5099520) | 20260505 17:44:36.163 | INFO | MemInfo: { | 20260505 17:44:36.163 | INFO | "gpu" : " 17 GB", | 20260505 17:44:36.163 | INFO | "huge" : "", | 20260505 17:44:36.163 | INFO | "huge_shared" : " 27 GB", | 20260505 17:44:36.163 | INFO | "malloc" : " 14 GB", | 20260505 17:44:36.163 | INFO | "pinned" : " 29 GB" | 20260505 17:44:36.163 | INFO | } | 20260505 17:44:36.163 | INFO | Perf report for the last 5.13547 seconds: | 20260505 17:44:36.163 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:44:36.163 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:36.163 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:36.163 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:36.163 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 17:44:36.163 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:36.163 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:36.163 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 17:44:36.163 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:36.163 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:36.163 | INFO | Uploaded 36GiB to GPU in 2.19837s (16.3758GiB/s) | 20260505 17:44:36.163 | INFO | Downloaded 34GiB to GPU in 2.02023s (16.8298GiB/s) | 20260505 17:44:36.305 | INFO | Starting a round of Baz IO | 20260505 17:44:39.483 | INFO | Populating RTMetrics report | 20260505 17:44:39.483 | INFO | RTMetrics report populated | 20260505 17:44:39.483 | INFO | Opening tmp RTMetrics json file | 20260505 17:44:39.484 | INFO | Serializing RTMetrics to json | 20260505 17:44:39.485 | INFO | Serialization done, swapping tmp file now... | 20260505 17:44:39.486 | INFO | RTMetrics done | 20260505 17:44:39.578 | INFO | Finished a round of Baz IO: Wrote 2.124 GiB in 3.272 seconds | 20260505 17:44:39.578 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.01% / 0.00% / 77.99% | 20260505 17:44:39.578 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.285506% / 3.97364e-06% / 0.000103315% | 20260505 17:44:39.578 | INFO | Baz IO overall: | 20260505 17:44:39.578 | INFO | BytesWritten : 3011532433404 | 20260505 17:44:39.578 | INFO | NumEvents : 1427651135923 | 20260505 17:44:41.295 | INFO | Analyzing chunk frames = [5099520,5100032) | 20260505 17:44:41.295 | INFO | MemInfo: { | 20260505 17:44:41.295 | INFO | "gpu" : " 17 GB", | 20260505 17:44:41.295 | INFO | "huge" : "", | 20260505 17:44:41.295 | INFO | "huge_shared" : " 27 GB", | 20260505 17:44:41.295 | INFO | "malloc" : " 14 GB", | 20260505 17:44:41.295 | INFO | "pinned" : " 29 GB" | 20260505 17:44:41.295 | INFO | } | 20260505 17:44:41.295 | INFO | Perf report for the last 5.13217 seconds: | 20260505 17:44:41.295 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:44:41.295 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:41.295 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:41.295 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:41.295 | INFO | ANALYSIS : 63.6%, 3.0, 99.0ms, 0 | 20260505 17:44:41.295 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:41.295 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260505 17:44:41.295 | INFO | PRE_HQ : 60.4%, 1.8, 55.9ms, 0 | 20260505 17:44:41.295 | INFO | BAZWRITER : 63.2%, 3.8, 123.7ms, 0 | 20260505 17:44:41.296 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 17:44:41.296 | INFO | Uploaded 36GiB to GPU in 2.16516s (16.6269GiB/s) | 20260505 17:44:41.296 | INFO | Downloaded 38GiB to GPU in 2.263s (16.7919GiB/s) | 20260505 17:44:46.447 | INFO | Analyzing chunk frames = [5100032,5100544) | 20260505 17:44:46.447 | INFO | MemInfo: { | 20260505 17:44:46.447 | INFO | "gpu" : " 17 GB", | 20260505 17:44:46.447 | INFO | "huge" : "", | 20260505 17:44:46.447 | INFO | "huge_shared" : " 27 GB", | 20260505 17:44:46.447 | INFO | "malloc" : " 14 GB", | 20260505 17:44:46.447 | INFO | "pinned" : " 29 GB" | 20260505 17:44:46.447 | INFO | } | 20260505 17:44:46.447 | INFO | Perf report for the last 5.15126 seconds: | 20260505 17:44:46.447 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:44:46.447 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:46.447 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:46.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:46.447 | INFO | ANALYSIS : 62.6%, 3.0, 98.0ms, 0 | 20260505 17:44:46.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:46.447 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:46.447 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 17:44:46.447 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:46.447 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:46.447 | INFO | Uploaded 36GiB to GPU in 2.12731s (16.9228GiB/s) | 20260505 17:44:46.447 | INFO | Downloaded 34GiB to GPU in 2.0269s (16.7744GiB/s) | 20260505 17:44:51.581 | INFO | Analyzing chunk frames = [5100544,5101056) | 20260505 17:44:51.581 | INFO | MemInfo: { | 20260505 17:44:51.581 | INFO | "gpu" : " 17 GB", | 20260505 17:44:51.581 | INFO | "huge" : "", | 20260505 17:44:51.581 | INFO | "huge_shared" : " 27 GB", | 20260505 17:44:51.581 | INFO | "malloc" : " 14 GB", | 20260505 17:44:51.581 | INFO | "pinned" : " 29 GB" | 20260505 17:44:51.581 | INFO | } | 20260505 17:44:51.582 | INFO | Perf report for the last 5.13482 seconds: | 20260505 17:44:51.582 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:44:51.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:51.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:51.582 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:51.582 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260505 17:44:51.582 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:51.582 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:51.582 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:44:51.582 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:51.582 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:51.582 | INFO | Uploaded 36GiB to GPU in 2.13751s (16.842GiB/s) | 20260505 17:44:51.582 | INFO | Downloaded 34GiB to GPU in 2.03183s (16.7337GiB/s) | 20260505 17:44:56.712 | INFO | Analyzing chunk frames = [5101056,5101568) | 20260505 17:44:56.712 | INFO | MemInfo: { | 20260505 17:44:56.712 | INFO | "gpu" : " 17 GB", | 20260505 17:44:56.712 | INFO | "huge" : "", | 20260505 17:44:56.712 | INFO | "huge_shared" : " 27 GB", | 20260505 17:44:56.712 | INFO | "malloc" : " 14 GB", | 20260505 17:44:56.712 | INFO | "pinned" : " 29 GB" | 20260505 17:44:56.712 | INFO | } | 20260505 17:44:56.712 | INFO | Perf report for the last 5.13023 seconds: | 20260505 17:44:56.712 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:44:56.712 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:56.712 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:56.712 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:56.712 | INFO | ANALYSIS : 63.0%, 3.0, 97.7ms, 0 | 20260505 17:44:56.712 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:56.712 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:56.712 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 17:44:56.712 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:56.712 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:44:56.712 | INFO | Uploaded 36GiB to GPU in 2.22748s (16.1618GiB/s) | 20260505 17:44:56.712 | INFO | Downloaded 34GiB to GPU in 2.11052s (16.1098GiB/s) | 20260505 17:45:01.854 | INFO | Analyzing chunk frames = [5101568,5102080) | 20260505 17:45:01.854 | INFO | MemInfo: { | 20260505 17:45:01.854 | INFO | "gpu" : " 17 GB", | 20260505 17:45:01.854 | INFO | "huge" : "", | 20260505 17:45:01.854 | INFO | "huge_shared" : " 27 GB", | 20260505 17:45:01.854 | INFO | "malloc" : " 14 GB", | 20260505 17:45:01.854 | INFO | "pinned" : " 29 GB" | 20260505 17:45:01.854 | INFO | } | 20260505 17:45:01.854 | INFO | Perf report for the last 5.14241 seconds: | 20260505 17:45:01.854 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:45:01.854 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:01.854 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:01.854 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:01.854 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260505 17:45:01.854 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:01.854 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:01.854 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 17:45:01.854 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:01.854 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:01.854 | INFO | Uploaded 36GiB to GPU in 2.16074s (16.661GiB/s) | 20260505 17:45:01.855 | INFO | Downloaded 34GiB to GPU in 2.06331s (16.4784GiB/s) | 20260505 17:45:06.998 | INFO | Analyzing chunk frames = [5102080,5102592) | 20260505 17:45:06.998 | INFO | MemInfo: { | 20260505 17:45:06.998 | INFO | "gpu" : " 17 GB", | 20260505 17:45:06.998 | INFO | "huge" : "", | 20260505 17:45:06.998 | INFO | "huge_shared" : " 27 GB", | 20260505 17:45:06.998 | INFO | "malloc" : " 14 GB", | 20260505 17:45:06.998 | INFO | "pinned" : " 29 GB" | 20260505 17:45:06.998 | INFO | } | 20260505 17:45:06.998 | INFO | Perf report for the last 5.14358 seconds: | 20260505 17:45:06.998 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:45:06.998 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:06.998 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:06.998 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:06.998 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 17:45:06.998 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:06.998 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:06.998 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:45:06.998 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:06.998 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:06.998 | INFO | Uploaded 36GiB to GPU in 2.14442s (16.7878GiB/s) | 20260505 17:45:06.998 | INFO | Downloaded 34GiB to GPU in 2.03805s (16.6826GiB/s) | 20260505 17:45:12.143 | INFO | Analyzing chunk frames = [5102592,5103104) | 20260505 17:45:12.143 | INFO | MemInfo: { | 20260505 17:45:12.143 | INFO | "gpu" : " 17 GB", | 20260505 17:45:12.143 | INFO | "huge" : "", | 20260505 17:45:12.143 | INFO | "huge_shared" : " 27 GB", | 20260505 17:45:12.143 | INFO | "malloc" : " 14 GB", | 20260505 17:45:12.143 | INFO | "pinned" : " 29 GB" | 20260505 17:45:12.143 | INFO | } | 20260505 17:45:12.143 | INFO | Perf report for the last 5.14517 seconds: | 20260505 17:45:12.143 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:45:12.143 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:12.143 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:12.143 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:12.143 | INFO | ANALYSIS : 63.2%, 3.0, 98.9ms, 0 | 20260505 17:45:12.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:12.143 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:12.143 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 17:45:12.143 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:12.143 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:12.143 | INFO | Uploaded 36GiB to GPU in 2.13171s (16.8879GiB/s) | 20260505 17:45:12.143 | INFO | Downloaded 34GiB to GPU in 2.13867s (15.8977GiB/s) | 20260505 17:45:17.284 | INFO | Analyzing chunk frames = [5103104,5103616) | 20260505 17:45:17.284 | INFO | MemInfo: { | 20260505 17:45:17.284 | INFO | "gpu" : " 17 GB", | 20260505 17:45:17.284 | INFO | "huge" : "", | 20260505 17:45:17.284 | INFO | "huge_shared" : " 27 GB", | 20260505 17:45:17.284 | INFO | "malloc" : " 14 GB", | 20260505 17:45:17.284 | INFO | "pinned" : " 29 GB" | 20260505 17:45:17.284 | INFO | } | 20260505 17:45:17.284 | INFO | Perf report for the last 5.14078 seconds: | 20260505 17:45:17.284 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:45:17.284 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:17.284 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:17.284 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:17.284 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 17:45:17.284 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:17.284 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:17.284 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 17:45:17.284 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:17.284 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:17.284 | INFO | Uploaded 36GiB to GPU in 2.20068s (16.3586GiB/s) | 20260505 17:45:17.284 | INFO | Downloaded 34GiB to GPU in 2.04635s (16.6149GiB/s) | 20260505 17:45:17.284 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 17:45:17.284 | INFO | Total mem usage: 15.3 GiB | 20260505 17:45:17.284 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 17:45:17.284 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.96%) | 20260505 17:45:17.284 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 17:45:17.284 | INFO | High Water Breakdown: | 20260505 17:45:17.284 | INFO | 61% : 8.67 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 17:45:17.284 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 17:45:17.284 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 17:45:17.284 | INFO | ------------------------------------------ | 20260505 17:45:17.284 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:45:17.284 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 17:45:17.284 | INFO | Total mem usage: 26.5 GiB | 20260505 17:45:17.284 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 17:45:17.284 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 17:45:17.284 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 17:45:17.284 | INFO | High Water Breakdown: | 20260505 17:45:17.284 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:45:17.284 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:45:17.284 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:45:17.284 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:45:17.284 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:45:17.284 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:45:17.284 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 17:45:17.284 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 17:45:17.284 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:45:17.284 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 17:45:17.284 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 17:45:17.284 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 17:45:17.284 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 17:45:17.284 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 17:45:17.284 | INFO | ------------------------------------------ | 20260505 17:45:17.284 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:45:17.284 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 17:45:17.284 | INFO | Total mem usage: 25.6 GiB | 20260505 17:45:17.284 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 17:45:17.284 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 17:45:17.284 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 17:45:17.284 | INFO | High Water Breakdown: | 20260505 17:45:17.284 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 17:45:17.284 | INFO | ------------------------------------------ | 20260505 17:45:17.284 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:45:17.284 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 17:45:17.284 | INFO | Total mem usage: 15.7 GiB | 20260505 17:45:17.284 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 17:45:17.284 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 17:45:17.284 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 17:45:17.284 | INFO | High Water Breakdown: | 20260505 17:45:17.284 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 17:45:17.284 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 17:45:17.285 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 17:45:17.285 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 17:45:17.285 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 17:45:17.285 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 17:45:17.285 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 17:45:17.285 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 17:45:17.285 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 17:45:17.285 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:45:17.285 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 17:45:17.285 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:45:17.285 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 17:45:17.285 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 17:45:17.285 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 17:45:17.285 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 17:45:17.285 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:45:17.285 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 17:45:17.285 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:45:17.285 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:45:17.285 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:45:17.285 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 17:45:17.285 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 17:45:17.285 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:45:17.285 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 17:45:17.285 | INFO | ------------------------------------------ | 20260505 17:45:17.285 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:45:17.285 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 17:45:17.285 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:45:17.285 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:45:17.285 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.49e+01, 1.35e+01 / 2.15e+00 | 20260505 17:45:17.285 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.02e-01, 1.03e-02 / 4.43e-03 | 20260505 17:45:17.285 | INFO | PerfReport: 11.77%, Baseline -- 3.72e+00 / 4.51e+00, 5.66e-01 / 1.66e-01 | 20260505 17:45:17.285 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.50e+00, 1.92e-01 / 1.10e-01 | 20260505 17:45:17.285 | INFO | PerfReport: 15.96%, DME -- 4.04e+01 / 4.48e+01, 4.32e+00 / 2.31e+00 | 20260505 17:45:17.285 | INFO | PerfReport: 56.87%, FrameLabeling -- 1.80e+01 / 1.92e+01, 1.48e+00 / 1.00e+00 | 20260505 17:45:17.285 | INFO | PerfReport: 7.31%, PulseAccumulating -- 2.31e+00 / 2.91e+00, 4.44e-01 / 1.85e-01 | 20260505 17:45:17.285 | INFO | PerfReport: 3.35%, Metrics -- 1.06e+00 / 1.40e+00, 2.92e-01 / 1.64e-01 | 20260505 17:45:17.285 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.70e-01 / 4.38e-01, 6.57e-02 / 3.53e-02 | 20260505 17:45:17.285 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 17:45:17.285 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:45:17.285 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:45:17.285 | INFO | PerfReport: Full Execution -- 9.99e+01 / -nan, 2.58e+01 / 0.00e+00 | 20260505 17:45:17.285 | INFO | PerfReport: 22.53%, Upload -- 2.25e+01 / 0.00e+00, 7.84e+00 / 0.00e+00 | 20260505 17:45:17.285 | INFO | PerfReport: 21.79%, Download -- 2.18e+01 / 0.00e+00, 7.77e+00 / 0.00e+00 | 20260505 17:45:17.285 | INFO | PerfReport: 31.64%, Compute -- 3.16e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 17:45:17.285 | INFO | PerfReport: 2.34%, LockUpload -- 2.33e+00 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260505 17:45:17.285 | INFO | PerfReport: 1.35%, LockDownload -- 1.35e+00 / 0.00e+00, 2.84e+00 / 0.00e+00 | 20260505 17:45:17.285 | INFO | PerfReport: 20.34%, LockCompute -- 2.03e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 17:45:17.285 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.69e-02 / 0.00e+00 | 20260505 17:45:17.418 | INFO | Starting a round of Baz IO | 20260505 17:45:20.597 | INFO | Populating RTMetrics report | 20260505 17:45:20.597 | INFO | RTMetrics report populated | 20260505 17:45:20.597 | INFO | Opening tmp RTMetrics json file | 20260505 17:45:20.604 | INFO | Serializing RTMetrics to json | 20260505 17:45:20.605 | INFO | Serialization done, swapping tmp file now... | 20260505 17:45:20.606 | INFO | RTMetrics done | 20260505 17:45:20.690 | INFO | Finished a round of Baz IO: Wrote 2.122 GiB in 3.272 seconds | 20260505 17:45:20.691 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.01% / 0.00% / 77.99% | 20260505 17:45:20.691 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.284827% / 3.97364e-06% / 0.000111262% | 20260505 17:45:20.691 | INFO | Baz IO overall: | 20260505 17:45:20.691 | INFO | BytesWritten : 3013810416626 | 20260505 17:45:20.691 | INFO | NumEvents : 1428714468634 | 20260505 17:45:22.412 | INFO | Analyzing chunk frames = [5103616,5104128) | 20260505 17:45:22.412 | INFO | MemInfo: { | 20260505 17:45:22.412 | INFO | "gpu" : " 17 GB", | 20260505 17:45:22.412 | INFO | "huge" : "", | 20260505 17:45:22.412 | INFO | "huge_shared" : " 27 GB", | 20260505 17:45:22.412 | INFO | "malloc" : " 14 GB", | 20260505 17:45:22.412 | INFO | "pinned" : " 29 GB" | 20260505 17:45:22.412 | INFO | } | 20260505 17:45:22.412 | INFO | Perf report for the last 5.12856 seconds: | 20260505 17:45:22.412 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:45:22.412 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:22.412 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:22.412 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:22.412 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260505 17:45:22.412 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:22.412 | INFO | RT_METRICS : 44.6%, 1.0, 23.3ms, 0 | 20260505 17:45:22.412 | INFO | PRE_HQ : 60.6%, 1.8, 56.1ms, 0 | 20260505 17:45:22.412 | INFO | BAZWRITER : 63.4%, 3.9, 125.3ms, 0 | 20260505 17:45:22.412 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 17:45:22.413 | INFO | Uploaded 36GiB to GPU in 2.16073s (16.661GiB/s) | 20260505 17:45:22.413 | INFO | Downloaded 38GiB to GPU in 2.25758s (16.8322GiB/s) | 20260505 17:45:27.553 | INFO | Analyzing chunk frames = [5104128,5104640) | 20260505 17:45:27.553 | INFO | MemInfo: { | 20260505 17:45:27.553 | INFO | "gpu" : " 17 GB", | 20260505 17:45:27.553 | INFO | "huge" : "", | 20260505 17:45:27.553 | INFO | "huge_shared" : " 27 GB", | 20260505 17:45:27.553 | INFO | "malloc" : " 14 GB", | 20260505 17:45:27.553 | INFO | "pinned" : " 29 GB" | 20260505 17:45:27.553 | INFO | } | 20260505 17:45:27.553 | INFO | Perf report for the last 5.14107 seconds: | 20260505 17:45:27.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:45:27.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:27.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:27.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:27.553 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 17:45:27.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:27.553 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:27.553 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:45:27.553 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:27.553 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:27.554 | INFO | Uploaded 36GiB to GPU in 2.13579s (16.8556GiB/s) | 20260505 17:45:27.554 | INFO | Downloaded 34GiB to GPU in 2.03593s (16.7GiB/s) | 20260505 17:45:32.695 | INFO | Analyzing chunk frames = [5104640,5105152) | 20260505 17:45:32.695 | INFO | MemInfo: { | 20260505 17:45:32.695 | INFO | "gpu" : " 17 GB", | 20260505 17:45:32.695 | INFO | "huge" : "", | 20260505 17:45:32.695 | INFO | "huge_shared" : " 27 GB", | 20260505 17:45:32.695 | INFO | "malloc" : " 14 GB", | 20260505 17:45:32.695 | INFO | "pinned" : " 29 GB" | 20260505 17:45:32.695 | INFO | } | 20260505 17:45:32.695 | INFO | Perf report for the last 5.14154 seconds: | 20260505 17:45:32.695 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:45:32.695 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:32.695 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:32.695 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:32.695 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 17:45:32.695 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:32.695 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:32.695 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:45:32.695 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:32.695 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:32.695 | INFO | Uploaded 36GiB to GPU in 2.20544s (16.3232GiB/s) | 20260505 17:45:32.695 | INFO | Downloaded 34GiB to GPU in 2.04178s (16.6522GiB/s) | 20260505 17:45:37.838 | INFO | Analyzing chunk frames = [5105152,5105664) | 20260505 17:45:37.838 | INFO | MemInfo: { | 20260505 17:45:37.838 | INFO | "gpu" : " 17 GB", | 20260505 17:45:37.838 | INFO | "huge" : "", | 20260505 17:45:37.838 | INFO | "huge_shared" : " 27 GB", | 20260505 17:45:37.838 | INFO | "malloc" : " 14 GB", | 20260505 17:45:37.838 | INFO | "pinned" : " 29 GB" | 20260505 17:45:37.838 | INFO | } | 20260505 17:45:37.838 | INFO | Perf report for the last 5.14282 seconds: | 20260505 17:45:37.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:45:37.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:37.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:37.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:37.838 | INFO | ANALYSIS : 62.8%, 3.0, 97.6ms, 0 | 20260505 17:45:37.838 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:37.838 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:37.838 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:45:37.838 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:37.838 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:37.838 | INFO | Uploaded 36GiB to GPU in 2.22792s (16.1586GiB/s) | 20260505 17:45:37.838 | INFO | Downloaded 34GiB to GPU in 2.11712s (16.0595GiB/s) | 20260505 17:45:42.974 | INFO | Analyzing chunk frames = [5105664,5106176) | 20260505 17:45:42.975 | INFO | MemInfo: { | 20260505 17:45:42.975 | INFO | "gpu" : " 17 GB", | 20260505 17:45:42.975 | INFO | "huge" : "", | 20260505 17:45:42.975 | INFO | "huge_shared" : " 27 GB", | 20260505 17:45:42.975 | INFO | "malloc" : " 14 GB", | 20260505 17:45:42.975 | INFO | "pinned" : " 29 GB" | 20260505 17:45:42.975 | INFO | } | 20260505 17:45:42.975 | INFO | Perf report for the last 5.13682 seconds: | 20260505 17:45:42.975 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:45:42.975 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:42.975 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:42.975 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:42.975 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 17:45:42.975 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:42.975 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:42.975 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 17:45:42.975 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:42.975 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:42.975 | INFO | Uploaded 36GiB to GPU in 2.14963s (16.7471GiB/s) | 20260505 17:45:42.975 | INFO | Downloaded 34GiB to GPU in 2.05498s (16.5452GiB/s) | 20260505 17:45:48.107 | INFO | Analyzing chunk frames = [5106176,5106688) | 20260505 17:45:48.107 | INFO | MemInfo: { | 20260505 17:45:48.107 | INFO | "gpu" : " 17 GB", | 20260505 17:45:48.107 | INFO | "huge" : "", | 20260505 17:45:48.107 | INFO | "huge_shared" : " 27 GB", | 20260505 17:45:48.107 | INFO | "malloc" : " 14 GB", | 20260505 17:45:48.107 | INFO | "pinned" : " 29 GB" | 20260505 17:45:48.107 | INFO | } | 20260505 17:45:48.107 | INFO | Perf report for the last 5.13239 seconds: | 20260505 17:45:48.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:45:48.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:48.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:48.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:48.107 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260505 17:45:48.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:48.107 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:48.107 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:45:48.107 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:48.107 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:48.107 | INFO | Uploaded 36GiB to GPU in 2.14786s (16.7609GiB/s) | 20260505 17:45:48.107 | INFO | Downloaded 34GiB to GPU in 2.04168s (16.653GiB/s) | 20260505 17:45:53.252 | INFO | Analyzing chunk frames = [5106688,5107200) | 20260505 17:45:53.252 | INFO | MemInfo: { | 20260505 17:45:53.252 | INFO | "gpu" : " 17 GB", | 20260505 17:45:53.252 | INFO | "huge" : "", | 20260505 17:45:53.252 | INFO | "huge_shared" : " 27 GB", | 20260505 17:45:53.252 | INFO | "malloc" : " 14 GB", | 20260505 17:45:53.252 | INFO | "pinned" : " 29 GB" | 20260505 17:45:53.252 | INFO | } | 20260505 17:45:53.252 | INFO | Perf report for the last 5.14483 seconds: | 20260505 17:45:53.252 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:45:53.252 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:53.252 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:53.252 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:53.252 | INFO | ANALYSIS : 63.3%, 3.0, 99.0ms, 0 | 20260505 17:45:53.252 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:53.252 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:53.252 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 17:45:53.252 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:53.252 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:53.252 | INFO | Uploaded 36GiB to GPU in 2.13622s (16.8522GiB/s) | 20260505 17:45:53.252 | INFO | Downloaded 34GiB to GPU in 2.1803s (15.5942GiB/s) | 20260505 17:45:58.394 | INFO | Analyzing chunk frames = [5107200,5107712) | 20260505 17:45:58.394 | INFO | MemInfo: { | 20260505 17:45:58.394 | INFO | "gpu" : " 17 GB", | 20260505 17:45:58.394 | INFO | "huge" : "", | 20260505 17:45:58.394 | INFO | "huge_shared" : " 27 GB", | 20260505 17:45:58.394 | INFO | "malloc" : " 14 GB", | 20260505 17:45:58.394 | INFO | "pinned" : " 29 GB" | 20260505 17:45:58.394 | INFO | } | 20260505 17:45:58.394 | INFO | Perf report for the last 5.14195 seconds: | 20260505 17:45:58.394 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:45:58.394 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:58.394 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:58.394 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:58.394 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 17:45:58.394 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:58.394 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:58.394 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:45:58.394 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:58.394 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:45:58.394 | INFO | Uploaded 36GiB to GPU in 2.19291s (16.4165GiB/s) | 20260505 17:45:58.394 | INFO | Downloaded 34GiB to GPU in 2.03222s (16.7305GiB/s) | 20260505 17:45:58.529 | INFO | Starting a round of Baz IO | 20260505 17:46:01.705 | INFO | Populating RTMetrics report | 20260505 17:46:01.706 | INFO | RTMetrics report populated | 20260505 17:46:01.706 | INFO | Opening tmp RTMetrics json file | 20260505 17:46:01.709 | INFO | Serializing RTMetrics to json | 20260505 17:46:01.710 | INFO | Serialization done, swapping tmp file now... | 20260505 17:46:01.711 | INFO | RTMetrics done | 20260505 17:46:01.798 | INFO | Finished a round of Baz IO: Wrote 2.12 GiB in 3.268 seconds | 20260505 17:46:01.798 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.01% / 0.00% / 77.99% | 20260505 17:46:01.798 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.284517% / 3.97364e-06% / 8.34465e-05% | 20260505 17:46:01.798 | INFO | Baz IO overall: | 20260505 17:46:01.798 | INFO | BytesWritten : 3016087110788 | 20260505 17:46:01.798 | INFO | NumEvents : 1429776993406 | 20260505 17:46:03.529 | INFO | Analyzing chunk frames = [5107712,5108224) | 20260505 17:46:03.529 | INFO | MemInfo: { | 20260505 17:46:03.529 | INFO | "gpu" : " 17 GB", | 20260505 17:46:03.529 | INFO | "huge" : "", | 20260505 17:46:03.529 | INFO | "huge_shared" : " 27 GB", | 20260505 17:46:03.529 | INFO | "malloc" : " 14 GB", | 20260505 17:46:03.529 | INFO | "pinned" : " 29 GB" | 20260505 17:46:03.529 | INFO | } | 20260505 17:46:03.529 | INFO | Perf report for the last 5.13489 seconds: | 20260505 17:46:03.529 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:46:03.529 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:03.529 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:03.529 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:03.529 | INFO | ANALYSIS : 63.4%, 3.0, 98.8ms, 0 | 20260505 17:46:03.529 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:03.529 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260505 17:46:03.529 | INFO | PRE_HQ : 60.5%, 1.8, 56.1ms, 0 | 20260505 17:46:03.529 | INFO | BAZWRITER : 63.2%, 3.8, 124.2ms, 0 | 20260505 17:46:03.529 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 17:46:03.529 | INFO | Uploaded 36GiB to GPU in 2.16057s (16.6623GiB/s) | 20260505 17:46:03.529 | INFO | Downloaded 38GiB to GPU in 2.25454s (16.8548GiB/s) | 20260505 17:46:08.664 | INFO | Analyzing chunk frames = [5108224,5108736) | 20260505 17:46:08.664 | INFO | MemInfo: { | 20260505 17:46:08.664 | INFO | "gpu" : " 17 GB", | 20260505 17:46:08.664 | INFO | "huge" : "", | 20260505 17:46:08.664 | INFO | "huge_shared" : " 27 GB", | 20260505 17:46:08.664 | INFO | "malloc" : " 14 GB", | 20260505 17:46:08.664 | INFO | "pinned" : " 29 GB" | 20260505 17:46:08.664 | INFO | } | 20260505 17:46:08.664 | INFO | Perf report for the last 5.13561 seconds: | 20260505 17:46:08.664 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:46:08.664 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:08.664 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:08.664 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:08.664 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 17:46:08.664 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:08.664 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:08.664 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 17:46:08.664 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:08.664 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:08.665 | INFO | Uploaded 36GiB to GPU in 2.12688s (16.9262GiB/s) | 20260505 17:46:08.665 | INFO | Downloaded 34GiB to GPU in 2.02383s (16.7999GiB/s) | 20260505 17:46:13.803 | INFO | Analyzing chunk frames = [5108736,5109248) | 20260505 17:46:13.803 | INFO | MemInfo: { | 20260505 17:46:13.803 | INFO | "gpu" : " 17 GB", | 20260505 17:46:13.803 | INFO | "huge" : "", | 20260505 17:46:13.803 | INFO | "huge_shared" : " 27 GB", | 20260505 17:46:13.803 | INFO | "malloc" : " 14 GB", | 20260505 17:46:13.803 | INFO | "pinned" : " 29 GB" | 20260505 17:46:13.803 | INFO | } | 20260505 17:46:13.803 | INFO | Perf report for the last 5.13841 seconds: | 20260505 17:46:13.803 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:46:13.803 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:13.803 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:13.803 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:13.803 | INFO | ANALYSIS : 62.8%, 3.0, 97.6ms, 0 | 20260505 17:46:13.803 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:13.803 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:13.803 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:46:13.803 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:13.803 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:13.803 | INFO | Uploaded 36GiB to GPU in 2.18692s (16.4615GiB/s) | 20260505 17:46:13.803 | INFO | Downloaded 34GiB to GPU in 2.03035s (16.7459GiB/s) | 20260505 17:46:18.948 | INFO | Analyzing chunk frames = [5109248,5109760) | 20260505 17:46:18.948 | INFO | MemInfo: { | 20260505 17:46:18.948 | INFO | "gpu" : " 17 GB", | 20260505 17:46:18.948 | INFO | "huge" : "", | 20260505 17:46:18.948 | INFO | "huge_shared" : " 27 GB", | 20260505 17:46:18.948 | INFO | "malloc" : " 14 GB", | 20260505 17:46:18.948 | INFO | "pinned" : " 29 GB" | 20260505 17:46:18.949 | INFO | } | 20260505 17:46:18.949 | INFO | Perf report for the last 5.14577 seconds: | 20260505 17:46:18.949 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:46:18.949 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:18.949 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:18.949 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:18.949 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260505 17:46:18.949 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:18.949 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:18.949 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:46:18.949 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:18.949 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:18.949 | INFO | Uploaded 36GiB to GPU in 2.17996s (16.514GiB/s) | 20260505 17:46:18.949 | INFO | Downloaded 34GiB to GPU in 2.13433s (15.93GiB/s) | 20260505 17:46:24.097 | INFO | Analyzing chunk frames = [5109760,5110272) | 20260505 17:46:24.097 | INFO | MemInfo: { | 20260505 17:46:24.097 | INFO | "gpu" : " 17 GB", | 20260505 17:46:24.097 | INFO | "huge" : "", | 20260505 17:46:24.097 | INFO | "huge_shared" : " 27 GB", | 20260505 17:46:24.097 | INFO | "malloc" : " 14 GB", | 20260505 17:46:24.097 | INFO | "pinned" : " 29 GB" | 20260505 17:46:24.097 | INFO | } | 20260505 17:46:24.097 | INFO | Perf report for the last 5.14867 seconds: | 20260505 17:46:24.097 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:46:24.097 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:24.097 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:24.097 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:24.097 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260505 17:46:24.097 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:24.097 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:24.097 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 17:46:24.097 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:24.097 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:24.097 | INFO | Uploaded 36GiB to GPU in 2.15462s (16.7083GiB/s) | 20260505 17:46:24.097 | INFO | Downloaded 34GiB to GPU in 2.06306s (16.4803GiB/s) | 20260505 17:46:29.229 | INFO | Analyzing chunk frames = [5110272,5110784) | 20260505 17:46:29.229 | INFO | MemInfo: { | 20260505 17:46:29.229 | INFO | "gpu" : " 17 GB", | 20260505 17:46:29.229 | INFO | "huge" : "", | 20260505 17:46:29.229 | INFO | "huge_shared" : " 27 GB", | 20260505 17:46:29.229 | INFO | "malloc" : " 14 GB", | 20260505 17:46:29.229 | INFO | "pinned" : " 29 GB" | 20260505 17:46:29.229 | INFO | } | 20260505 17:46:29.229 | INFO | Perf report for the last 5.13187 seconds: | 20260505 17:46:29.229 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:46:29.229 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:29.229 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:29.229 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:29.229 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260505 17:46:29.229 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:29.229 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:29.229 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:46:29.229 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:29.229 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:29.229 | INFO | Uploaded 36GiB to GPU in 2.13298s (16.8778GiB/s) | 20260505 17:46:29.229 | INFO | Downloaded 34GiB to GPU in 2.05402s (16.5529GiB/s) | 20260505 17:46:34.359 | INFO | Analyzing chunk frames = [5110784,5111296) | 20260505 17:46:34.359 | INFO | MemInfo: { | 20260505 17:46:34.359 | INFO | "gpu" : " 17 GB", | 20260505 17:46:34.359 | INFO | "huge" : "", | 20260505 17:46:34.359 | INFO | "huge_shared" : " 27 GB", | 20260505 17:46:34.359 | INFO | "malloc" : " 14 GB", | 20260505 17:46:34.359 | INFO | "pinned" : " 29 GB" | 20260505 17:46:34.359 | INFO | } | 20260505 17:46:34.359 | INFO | Perf report for the last 5.13012 seconds: | 20260505 17:46:34.359 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:46:34.359 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:34.359 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:34.359 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:34.359 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 17:46:34.359 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:34.359 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:34.359 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 17:46:34.359 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:34.359 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:34.360 | INFO | Uploaded 36GiB to GPU in 2.13573s (16.8561GiB/s) | 20260505 17:46:34.360 | INFO | Downloaded 34GiB to GPU in 2.13698s (15.9103GiB/s) | 20260505 17:46:39.507 | INFO | Analyzing chunk frames = [5111296,5111808) | 20260505 17:46:39.507 | INFO | MemInfo: { | 20260505 17:46:39.507 | INFO | "gpu" : " 17 GB", | 20260505 17:46:39.507 | INFO | "huge" : "", | 20260505 17:46:39.507 | INFO | "huge_shared" : " 27 GB", | 20260505 17:46:39.507 | INFO | "malloc" : " 14 GB", | 20260505 17:46:39.507 | INFO | "pinned" : " 29 GB" | 20260505 17:46:39.508 | INFO | } | 20260505 17:46:39.508 | INFO | Perf report for the last 5.14831 seconds: | 20260505 17:46:39.508 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:46:39.508 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:39.508 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:39.508 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:39.508 | INFO | ANALYSIS : 63.4%, 3.0, 99.2ms, 0 | 20260505 17:46:39.508 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:39.508 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:39.508 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 17:46:39.508 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:39.508 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:39.508 | INFO | Uploaded 36GiB to GPU in 2.23134s (16.1338GiB/s) | 20260505 17:46:39.508 | INFO | Downloaded 34GiB to GPU in 2.10855s (16.1248GiB/s) | 20260505 17:46:39.508 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 17:46:39.508 | INFO | Total mem usage: 15.3 GiB | 20260505 17:46:39.508 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 17:46:39.508 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -7.52%) | 20260505 17:46:39.508 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 17:46:39.508 | INFO | High Water Breakdown: | 20260505 17:46:39.508 | INFO | 61% : 8.7 GiB ( 91.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 17:46:39.508 | INFO | 38.8% : 5.54 GiB ( 83.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 17:46:39.508 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 17:46:39.508 | INFO | ------------------------------------------ | 20260505 17:46:39.508 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:46:39.508 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 17:46:39.508 | INFO | Total mem usage: 26.5 GiB | 20260505 17:46:39.508 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 17:46:39.508 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 17:46:39.508 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 17:46:39.508 | INFO | High Water Breakdown: | 20260505 17:46:39.508 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:46:39.508 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:46:39.508 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:46:39.508 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:46:39.508 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:46:39.508 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:46:39.508 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 17:46:39.508 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 17:46:39.508 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:46:39.508 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 17:46:39.508 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 17:46:39.509 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 17:46:39.509 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 17:46:39.509 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 17:46:39.509 | INFO | ------------------------------------------ | 20260505 17:46:39.509 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:46:39.509 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 17:46:39.509 | INFO | Total mem usage: 25.6 GiB | 20260505 17:46:39.509 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 17:46:39.509 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 17:46:39.509 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 17:46:39.509 | INFO | High Water Breakdown: | 20260505 17:46:39.509 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 17:46:39.509 | INFO | ------------------------------------------ | 20260505 17:46:39.509 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:46:39.509 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 17:46:39.509 | INFO | Total mem usage: 15.7 GiB | 20260505 17:46:39.509 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 17:46:39.509 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 17:46:39.509 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 17:46:39.509 | INFO | High Water Breakdown: | 20260505 17:46:39.509 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 17:46:39.509 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 17:46:39.509 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 17:46:39.509 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 17:46:39.509 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 17:46:39.509 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 17:46:39.509 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 17:46:39.509 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 17:46:39.509 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 17:46:39.509 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:46:39.509 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 17:46:39.509 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:46:39.509 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 17:46:39.509 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 17:46:39.509 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 17:46:39.509 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 17:46:39.509 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:46:39.509 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 17:46:39.509 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:46:39.509 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:46:39.509 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:46:39.509 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 17:46:39.509 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 17:46:39.509 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:46:39.509 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 17:46:39.509 | INFO | ------------------------------------------ | 20260505 17:46:39.509 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:46:39.509 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 17:46:39.509 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:46:39.509 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:46:39.509 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.49e+01, 1.36e+01 / 2.15e+00 | 20260505 17:46:39.509 | INFO | PerfReport: 0.29%, AggregateStats -- 9.24e-02 / 1.02e-01, 1.07e-02 / 3.03e-03 | 20260505 17:46:39.509 | INFO | PerfReport: 11.78%, Baseline -- 3.72e+00 / 4.49e+00, 5.68e-01 / 1.62e-01 | 20260505 17:46:39.509 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.52e+00, 1.92e-01 / 9.38e-02 | 20260505 17:46:39.509 | INFO | PerfReport: 15.95%, DME -- 4.03e+01 / 4.47e+01, 4.26e+00 / 2.22e+00 | 20260505 17:46:39.509 | INFO | PerfReport: 56.78%, FrameLabeling -- 1.79e+01 / 1.94e+01, 1.40e+00 / 1.40e+00 | 20260505 17:46:39.509 | INFO | PerfReport: 7.36%, PulseAccumulating -- 2.32e+00 / 2.92e+00, 4.46e-01 / 1.71e-01 | 20260505 17:46:39.509 | INFO | PerfReport: 3.38%, Metrics -- 1.06e+00 / 1.37e+00, 2.91e-01 / 1.20e-01 | 20260505 17:46:39.509 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.70e-01 / 4.33e-01, 6.51e-02 / 4.07e-02 | 20260505 17:46:39.509 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 17:46:39.509 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:46:39.509 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:46:39.509 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.56e+01 / 0.00e+00 | 20260505 17:46:39.509 | INFO | PerfReport: 22.58%, Upload -- 2.26e+01 / 0.00e+00, 7.94e+00 / 0.00e+00 | 20260505 17:46:39.509 | INFO | PerfReport: 21.86%, Download -- 2.19e+01 / 0.00e+00, 7.87e+00 / 0.00e+00 | 20260505 17:46:39.509 | INFO | PerfReport: 31.56%, Compute -- 3.15e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 17:46:39.509 | INFO | PerfReport: 2.35%, LockUpload -- 2.35e+00 / 0.00e+00, 7.98e+00 / 0.00e+00 | 20260505 17:46:39.509 | INFO | PerfReport: 1.40%, LockDownload -- 1.40e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260505 17:46:39.509 | INFO | PerfReport: 20.22%, LockCompute -- 2.02e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 17:46:39.509 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.19e-02 / 0.00e+00 | 20260505 17:46:39.643 | INFO | Starting a round of Baz IO | 20260505 17:46:42.823 | INFO | Populating RTMetrics report | 20260505 17:46:42.823 | INFO | RTMetrics report populated | 20260505 17:46:42.823 | INFO | Opening tmp RTMetrics json file | 20260505 17:46:42.831 | INFO | Serializing RTMetrics to json | 20260505 17:46:42.834 | INFO | Serialization done, swapping tmp file now... | 20260505 17:46:42.836 | INFO | RTMetrics done | 20260505 17:46:42.918 | INFO | Finished a round of Baz IO: Wrote 2.12 GiB in 3.275 seconds | 20260505 17:46:42.918 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.01% / 0.00% / 77.99% | 20260505 17:46:42.918 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.283337% / 1.19209e-05% / 9.13938e-05% | 20260505 17:46:42.918 | INFO | Baz IO overall: | 20260505 17:46:42.918 | INFO | BytesWritten : 3018363372977 | 20260505 17:46:42.918 | INFO | NumEvents : 1430839611879 | 20260505 17:46:44.648 | INFO | Analyzing chunk frames = [5111808,5112320) | 20260505 17:46:44.648 | INFO | MemInfo: { | 20260505 17:46:44.648 | INFO | "gpu" : " 17 GB", | 20260505 17:46:44.648 | INFO | "huge" : "", | 20260505 17:46:44.648 | INFO | "huge_shared" : " 27 GB", | 20260505 17:46:44.648 | INFO | "malloc" : " 14 GB", | 20260505 17:46:44.648 | INFO | "pinned" : " 29 GB" | 20260505 17:46:44.648 | INFO | } | 20260505 17:46:44.648 | INFO | Perf report for the last 5.14085 seconds: | 20260505 17:46:44.649 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:46:44.649 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:44.649 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:44.649 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:44.649 | INFO | ANALYSIS : 63.4%, 3.0, 98.8ms, 0 | 20260505 17:46:44.649 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:44.649 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260505 17:46:44.649 | INFO | PRE_HQ : 60.5%, 1.8, 56.0ms, 0 | 20260505 17:46:44.649 | INFO | BAZWRITER : 63.2%, 3.8, 124.4ms, 0 | 20260505 17:46:44.649 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260505 17:46:44.649 | INFO | Uploaded 36GiB to GPU in 2.16921s (16.5959GiB/s) | 20260505 17:46:44.649 | INFO | Downloaded 38GiB to GPU in 2.25485s (16.8526GiB/s) | 20260505 17:46:49.788 | INFO | Analyzing chunk frames = [5112320,5112832) | 20260505 17:46:49.788 | INFO | MemInfo: { | 20260505 17:46:49.788 | INFO | "gpu" : " 17 GB", | 20260505 17:46:49.789 | INFO | "huge" : "", | 20260505 17:46:49.789 | INFO | "huge_shared" : " 27 GB", | 20260505 17:46:49.789 | INFO | "malloc" : " 14 GB", | 20260505 17:46:49.789 | INFO | "pinned" : " 29 GB" | 20260505 17:46:49.789 | INFO | } | 20260505 17:46:49.789 | INFO | Perf report for the last 5.14012 seconds: | 20260505 17:46:49.789 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:46:49.789 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:49.789 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:49.789 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:49.789 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 17:46:49.789 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:49.789 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:49.789 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 17:46:49.789 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:49.789 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:49.789 | INFO | Uploaded 36GiB to GPU in 2.14142s (16.8113GiB/s) | 20260505 17:46:49.789 | INFO | Downloaded 34GiB to GPU in 2.0371s (16.6904GiB/s) | 20260505 17:46:54.926 | INFO | Analyzing chunk frames = [5112832,5113344) | 20260505 17:46:54.926 | INFO | MemInfo: { | 20260505 17:46:54.926 | INFO | "gpu" : " 17 GB", | 20260505 17:46:54.926 | INFO | "huge" : "", | 20260505 17:46:54.926 | INFO | "huge_shared" : " 27 GB", | 20260505 17:46:54.926 | INFO | "malloc" : " 14 GB", | 20260505 17:46:54.926 | INFO | "pinned" : " 29 GB" | 20260505 17:46:54.926 | INFO | } | 20260505 17:46:54.926 | INFO | Perf report for the last 5.1375 seconds: | 20260505 17:46:54.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:46:54.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:54.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:54.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:54.926 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 17:46:54.926 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:54.926 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:54.926 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:46:54.926 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:54.926 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:46:54.926 | INFO | Uploaded 36GiB to GPU in 2.20662s (16.3146GiB/s) | 20260505 17:46:54.926 | INFO | Downloaded 34GiB to GPU in 2.0659s (16.4577GiB/s) | 20260505 17:47:00.062 | INFO | Analyzing chunk frames = [5113344,5113856) | 20260505 17:47:00.063 | INFO | MemInfo: { | 20260505 17:47:00.063 | INFO | "gpu" : " 17 GB", | 20260505 17:47:00.063 | INFO | "huge" : "", | 20260505 17:47:00.063 | INFO | "huge_shared" : " 27 GB", | 20260505 17:47:00.063 | INFO | "malloc" : " 14 GB", | 20260505 17:47:00.063 | INFO | "pinned" : " 29 GB" | 20260505 17:47:00.063 | INFO | } | 20260505 17:47:00.063 | INFO | Perf report for the last 5.13654 seconds: | 20260505 17:47:00.063 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:47:00.063 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:00.063 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:00.063 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:00.063 | INFO | ANALYSIS : 62.5%, 3.0, 97.1ms, 0 | 20260505 17:47:00.063 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:00.063 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:00.063 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:47:00.063 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:00.063 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:00.063 | INFO | Uploaded 36GiB to GPU in 2.18326s (16.4891GiB/s) | 20260505 17:47:00.063 | INFO | Downloaded 34GiB to GPU in 2.13793s (15.9032GiB/s) | 20260505 17:47:05.205 | INFO | Analyzing chunk frames = [5113856,5114368) | 20260505 17:47:05.205 | INFO | MemInfo: { | 20260505 17:47:05.205 | INFO | "gpu" : " 17 GB", | 20260505 17:47:05.205 | INFO | "huge" : "", | 20260505 17:47:05.205 | INFO | "huge_shared" : " 27 GB", | 20260505 17:47:05.205 | INFO | "malloc" : " 14 GB", | 20260505 17:47:05.205 | INFO | "pinned" : " 29 GB" | 20260505 17:47:05.205 | INFO | } | 20260505 17:47:05.205 | INFO | Perf report for the last 5.1425 seconds: | 20260505 17:47:05.205 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:47:05.205 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:05.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:05.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:05.205 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 17:47:05.205 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:05.205 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:05.205 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:47:05.205 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:05.205 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:05.205 | INFO | Uploaded 36GiB to GPU in 2.14948s (16.7482GiB/s) | 20260505 17:47:05.205 | INFO | Downloaded 34GiB to GPU in 2.06302s (16.4807GiB/s) | 20260505 17:47:10.343 | INFO | Analyzing chunk frames = [5114368,5114880) | 20260505 17:47:10.343 | INFO | MemInfo: { | 20260505 17:47:10.343 | INFO | "gpu" : " 17 GB", | 20260505 17:47:10.343 | INFO | "huge" : "", | 20260505 17:47:10.343 | INFO | "huge_shared" : " 27 GB", | 20260505 17:47:10.343 | INFO | "malloc" : " 14 GB", | 20260505 17:47:10.343 | INFO | "pinned" : " 29 GB" | 20260505 17:47:10.343 | INFO | } | 20260505 17:47:10.343 | INFO | Perf report for the last 5.13786 seconds: | 20260505 17:47:10.343 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:47:10.343 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:10.343 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:10.343 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:10.343 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260505 17:47:10.343 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:10.343 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:10.343 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 17:47:10.343 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:10.343 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:10.343 | INFO | Uploaded 36GiB to GPU in 2.13635s (16.8512GiB/s) | 20260505 17:47:10.343 | INFO | Downloaded 34GiB to GPU in 2.05383s (16.5545GiB/s) | 20260505 17:47:15.476 | INFO | Analyzing chunk frames = [5114880,5115392) | 20260505 17:47:15.476 | INFO | MemInfo: { | 20260505 17:47:15.476 | INFO | "gpu" : " 17 GB", | 20260505 17:47:15.476 | INFO | "huge" : "", | 20260505 17:47:15.476 | INFO | "huge_shared" : " 27 GB", | 20260505 17:47:15.476 | INFO | "malloc" : " 14 GB", | 20260505 17:47:15.476 | INFO | "pinned" : " 29 GB" | 20260505 17:47:15.476 | INFO | } | 20260505 17:47:15.476 | INFO | Perf report for the last 5.13296 seconds: | 20260505 17:47:15.476 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:47:15.476 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:15.476 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:15.476 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:15.476 | INFO | ANALYSIS : 63.6%, 3.0, 99.1ms, 0 | 20260505 17:47:15.476 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:15.476 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:15.476 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 17:47:15.476 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:15.476 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:15.476 | INFO | Uploaded 36GiB to GPU in 2.15841s (16.679GiB/s) | 20260505 17:47:15.476 | INFO | Downloaded 34GiB to GPU in 2.14839s (15.8258GiB/s) | 20260505 17:47:20.613 | INFO | Analyzing chunk frames = [5115392,5115904) | 20260505 17:47:20.613 | INFO | MemInfo: { | 20260505 17:47:20.613 | INFO | "gpu" : " 17 GB", | 20260505 17:47:20.613 | INFO | "huge" : "", | 20260505 17:47:20.613 | INFO | "huge_shared" : " 27 GB", | 20260505 17:47:20.613 | INFO | "malloc" : " 14 GB", | 20260505 17:47:20.613 | INFO | "pinned" : " 29 GB" | 20260505 17:47:20.613 | INFO | } | 20260505 17:47:20.614 | INFO | Perf report for the last 5.13752 seconds: | 20260505 17:47:20.614 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:47:20.614 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:20.614 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:20.614 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:20.614 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 17:47:20.614 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:20.614 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:20.614 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:47:20.614 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:20.614 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:20.614 | INFO | Uploaded 36GiB to GPU in 2.16686s (16.6139GiB/s) | 20260505 17:47:20.614 | INFO | Downloaded 34GiB to GPU in 2.03624s (16.6975GiB/s) | 20260505 17:47:20.744 | INFO | Starting a round of Baz IO | 20260505 17:47:23.944 | INFO | Populating RTMetrics report | 20260505 17:47:23.945 | INFO | RTMetrics report populated | 20260505 17:47:23.945 | INFO | Opening tmp RTMetrics json file | 20260505 17:47:23.951 | INFO | Serializing RTMetrics to json | 20260505 17:47:23.954 | INFO | Serialization done, swapping tmp file now... | 20260505 17:47:23.955 | INFO | RTMetrics done | 20260505 17:47:24.035 | INFO | Finished a round of Baz IO: Wrote 2.12 GiB in 3.291 seconds | 20260505 17:47:24.035 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.01% / 0.00% / 77.99% | 20260505 17:47:24.035 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.282896% / 7.94729e-06% / 0.000111262% | 20260505 17:47:24.035 | INFO | Baz IO overall: | 20260505 17:47:24.035 | INFO | BytesWritten : 3020640019026 | 20260505 17:47:24.036 | INFO | NumEvents : 1431902614057 | 20260505 17:47:25.760 | INFO | Analyzing chunk frames = [5115904,5116416) | 20260505 17:47:25.761 | INFO | MemInfo: { | 20260505 17:47:25.761 | INFO | "gpu" : " 17 GB", | 20260505 17:47:25.761 | INFO | "huge" : "", | 20260505 17:47:25.761 | INFO | "huge_shared" : " 27 GB", | 20260505 17:47:25.761 | INFO | "malloc" : " 14 GB", | 20260505 17:47:25.761 | INFO | "pinned" : " 29 GB" | 20260505 17:47:25.761 | INFO | } | 20260505 17:47:25.761 | INFO | Perf report for the last 5.14711 seconds: | 20260505 17:47:25.761 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:47:25.761 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:25.761 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:25.761 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:25.761 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 17:47:25.761 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:25.761 | INFO | RT_METRICS : 44.5%, 1.0, 23.4ms, 0 | 20260505 17:47:25.761 | INFO | PRE_HQ : 60.4%, 1.8, 55.9ms, 0 | 20260505 17:47:25.761 | INFO | BAZWRITER : 63.3%, 3.8, 124.4ms, 0 | 20260505 17:47:25.761 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 17:47:25.761 | INFO | Uploaded 36GiB to GPU in 2.15771s (16.6843GiB/s) | 20260505 17:47:25.761 | INFO | Downloaded 38GiB to GPU in 2.26104s (16.8065GiB/s) | 20260505 17:47:30.906 | INFO | Analyzing chunk frames = [5116416,5116928) | 20260505 17:47:30.906 | INFO | MemInfo: { | 20260505 17:47:30.906 | INFO | "gpu" : " 17 GB", | 20260505 17:47:30.906 | INFO | "huge" : "", | 20260505 17:47:30.906 | INFO | "huge_shared" : " 27 GB", | 20260505 17:47:30.906 | INFO | "malloc" : " 14 GB", | 20260505 17:47:30.906 | INFO | "pinned" : " 29 GB" | 20260505 17:47:30.906 | INFO | } | 20260505 17:47:30.906 | INFO | Perf report for the last 5.1456 seconds: | 20260505 17:47:30.906 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:47:30.906 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:30.906 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:30.906 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:30.906 | INFO | ANALYSIS : 62.7%, 3.0, 98.0ms, 0 | 20260505 17:47:30.906 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:30.906 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:30.906 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:47:30.906 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:30.906 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:30.906 | INFO | Uploaded 36GiB to GPU in 2.11563s (17.0162GiB/s) | 20260505 17:47:30.906 | INFO | Downloaded 34GiB to GPU in 2.03416s (16.7145GiB/s) | 20260505 17:47:36.044 | INFO | Analyzing chunk frames = [5116928,5117440) | 20260505 17:47:36.044 | INFO | MemInfo: { | 20260505 17:47:36.044 | INFO | "gpu" : " 17 GB", | 20260505 17:47:36.044 | INFO | "huge" : "", | 20260505 17:47:36.044 | INFO | "huge_shared" : " 27 GB", | 20260505 17:47:36.044 | INFO | "malloc" : " 14 GB", | 20260505 17:47:36.044 | INFO | "pinned" : " 29 GB" | 20260505 17:47:36.044 | INFO | } | 20260505 17:47:36.044 | INFO | Perf report for the last 5.1378 seconds: | 20260505 17:47:36.044 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:47:36.044 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:36.044 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:36.044 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:36.044 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 17:47:36.044 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:36.044 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:36.044 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:47:36.044 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:36.044 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:36.044 | INFO | Uploaded 36GiB to GPU in 2.20504s (16.3262GiB/s) | 20260505 17:47:36.044 | INFO | Downloaded 34GiB to GPU in 2.07354s (16.3971GiB/s) | 20260505 17:47:41.176 | INFO | Analyzing chunk frames = [5117440,5117952) | 20260505 17:47:41.176 | INFO | MemInfo: { | 20260505 17:47:41.176 | INFO | "gpu" : " 17 GB", | 20260505 17:47:41.176 | INFO | "huge" : "", | 20260505 17:47:41.176 | INFO | "huge_shared" : " 27 GB", | 20260505 17:47:41.176 | INFO | "malloc" : " 14 GB", | 20260505 17:47:41.176 | INFO | "pinned" : " 29 GB" | 20260505 17:47:41.176 | INFO | } | 20260505 17:47:41.176 | INFO | Perf report for the last 5.13204 seconds: | 20260505 17:47:41.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:47:41.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:41.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:41.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:41.176 | INFO | ANALYSIS : 62.9%, 3.0, 97.6ms, 0 | 20260505 17:47:41.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:41.176 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:41.176 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:47:41.176 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:41.176 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:41.176 | INFO | Uploaded 36GiB to GPU in 2.22478s (16.1813GiB/s) | 20260505 17:47:41.176 | INFO | Downloaded 34GiB to GPU in 2.12419s (16.0061GiB/s) | 20260505 17:47:46.312 | INFO | Analyzing chunk frames = [5117952,5118464) | 20260505 17:47:46.312 | INFO | MemInfo: { | 20260505 17:47:46.312 | INFO | "gpu" : " 17 GB", | 20260505 17:47:46.312 | INFO | "huge" : "", | 20260505 17:47:46.312 | INFO | "huge_shared" : " 27 GB", | 20260505 17:47:46.312 | INFO | "malloc" : " 14 GB", | 20260505 17:47:46.312 | INFO | "pinned" : " 29 GB" | 20260505 17:47:46.312 | INFO | } | 20260505 17:47:46.312 | INFO | Perf report for the last 5.13593 seconds: | 20260505 17:47:46.312 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:47:46.312 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:46.312 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:46.312 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:46.312 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 17:47:46.312 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:46.312 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:46.312 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:47:46.312 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:46.312 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:46.312 | INFO | Uploaded 36GiB to GPU in 2.14915s (16.7508GiB/s) | 20260505 17:47:46.312 | INFO | Downloaded 34GiB to GPU in 2.06291s (16.4816GiB/s) | 20260505 17:47:51.457 | INFO | Analyzing chunk frames = [5118464,5118976) | 20260505 17:47:51.457 | INFO | MemInfo: { | 20260505 17:47:51.457 | INFO | "gpu" : " 17 GB", | 20260505 17:47:51.458 | INFO | "huge" : "", | 20260505 17:47:51.458 | INFO | "huge_shared" : " 27 GB", | 20260505 17:47:51.458 | INFO | "malloc" : " 14 GB", | 20260505 17:47:51.458 | INFO | "pinned" : " 29 GB" | 20260505 17:47:51.458 | INFO | } | 20260505 17:47:51.458 | INFO | Perf report for the last 5.14556 seconds: | 20260505 17:47:51.458 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:47:51.458 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:51.458 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:51.458 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:51.458 | INFO | ANALYSIS : 62.3%, 3.0, 97.4ms, 0 | 20260505 17:47:51.458 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:51.458 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:51.458 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:47:51.458 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:51.458 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:51.458 | INFO | Uploaded 36GiB to GPU in 2.13671s (16.8483GiB/s) | 20260505 17:47:51.458 | INFO | Downloaded 34GiB to GPU in 2.03603s (16.6992GiB/s) | 20260505 17:47:56.602 | INFO | Analyzing chunk frames = [5118976,5119488) | 20260505 17:47:56.602 | INFO | MemInfo: { | 20260505 17:47:56.602 | INFO | "gpu" : " 17 GB", | 20260505 17:47:56.602 | INFO | "huge" : "", | 20260505 17:47:56.602 | INFO | "huge_shared" : " 27 GB", | 20260505 17:47:56.602 | INFO | "malloc" : " 14 GB", | 20260505 17:47:56.602 | INFO | "pinned" : " 29 GB" | 20260505 17:47:56.602 | INFO | } | 20260505 17:47:56.602 | INFO | Perf report for the last 5.14403 seconds: | 20260505 17:47:56.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:47:56.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:56.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:56.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:56.602 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 17:47:56.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:56.602 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:56.602 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 17:47:56.602 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:56.602 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:47:56.602 | INFO | Uploaded 36GiB to GPU in 2.15109s (16.7357GiB/s) | 20260505 17:47:56.602 | INFO | Downloaded 34GiB to GPU in 2.18426s (15.5659GiB/s) | 20260505 17:48:01.730 | INFO | Analyzing chunk frames = [5119488,5120000) | 20260505 17:48:01.730 | INFO | MemInfo: { | 20260505 17:48:01.730 | INFO | "gpu" : " 17 GB", | 20260505 17:48:01.730 | INFO | "huge" : "", | 20260505 17:48:01.730 | INFO | "huge_shared" : " 27 GB", | 20260505 17:48:01.730 | INFO | "malloc" : " 14 GB", | 20260505 17:48:01.730 | INFO | "pinned" : " 29 GB" | 20260505 17:48:01.730 | INFO | } | 20260505 17:48:01.730 | INFO | Perf report for the last 5.12802 seconds: | 20260505 17:48:01.730 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:48:01.730 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:01.730 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:01.730 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:01.730 | INFO | ANALYSIS : 63.2%, 3.0, 98.4ms, 0 | 20260505 17:48:01.730 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:01.730 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:01.730 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 17:48:01.730 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:01.730 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:01.730 | INFO | Uploaded 36GiB to GPU in 2.2035s (16.3376GiB/s) | 20260505 17:48:01.730 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 17:48:01.730 | INFO | Total mem usage: 15.3 GiB | 20260505 17:48:01.730 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 17:48:01.730 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.01%) | 20260505 17:48:01.730 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 17:48:01.730 | INFO | Downloaded 34GiB to GPU in 2.03582s (16.7009GiB/s) | 20260505 17:48:01.730 | INFO | High Water Breakdown: | 20260505 17:48:01.730 | INFO | 61% : 8.66 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 17:48:01.730 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 17:48:01.730 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 17:48:01.730 | INFO | ------------------------------------------ | 20260505 17:48:01.730 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:48:01.730 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 17:48:01.730 | INFO | Total mem usage: 26.5 GiB | 20260505 17:48:01.730 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 17:48:01.730 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 17:48:01.730 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 17:48:01.730 | INFO | High Water Breakdown: | 20260505 17:48:01.730 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:48:01.730 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:48:01.730 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:48:01.730 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:48:01.730 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:48:01.730 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:48:01.730 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 17:48:01.730 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 17:48:01.730 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:48:01.730 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 17:48:01.730 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 17:48:01.730 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 17:48:01.730 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 17:48:01.730 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 17:48:01.730 | INFO | ------------------------------------------ | 20260505 17:48:01.730 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:48:01.730 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 17:48:01.730 | INFO | Total mem usage: 25.6 GiB | 20260505 17:48:01.730 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 17:48:01.730 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 17:48:01.730 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 17:48:01.730 | INFO | High Water Breakdown: | 20260505 17:48:01.730 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 17:48:01.730 | INFO | ------------------------------------------ | 20260505 17:48:01.730 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:48:01.730 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 17:48:01.730 | INFO | Total mem usage: 15.7 GiB | 20260505 17:48:01.730 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 17:48:01.730 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 17:48:01.730 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 17:48:01.731 | INFO | High Water Breakdown: | 20260505 17:48:01.731 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 17:48:01.731 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 17:48:01.731 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 17:48:01.731 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 17:48:01.731 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 17:48:01.731 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 17:48:01.731 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 17:48:01.731 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 17:48:01.731 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 17:48:01.731 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:48:01.731 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 17:48:01.731 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:48:01.731 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 17:48:01.731 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 17:48:01.731 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 17:48:01.731 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 17:48:01.731 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:48:01.731 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 17:48:01.731 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:48:01.731 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:48:01.731 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:48:01.731 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 17:48:01.731 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 17:48:01.731 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:48:01.731 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 17:48:01.731 | INFO | ------------------------------------------ | 20260505 17:48:01.731 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:48:01.731 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 17:48:01.731 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:48:01.731 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:48:01.731 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.49e+01, 1.36e+01 / 2.01e+00 | 20260505 17:48:01.731 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.02e-01, 1.07e-02 / 4.46e-03 | 20260505 17:48:01.731 | INFO | PerfReport: 11.76%, Baseline -- 3.71e+00 / 4.54e+00, 5.64e-01 / 1.28e-01 | 20260505 17:48:01.731 | INFO | PerfReport: 3.26%, Histogram -- 1.03e+00 / 1.53e+00, 1.91e-01 / 9.47e-02 | 20260505 17:48:01.731 | INFO | PerfReport: 15.97%, DME -- 4.03e+01 / 4.47e+01, 4.26e+00 / 2.14e+00 | 20260505 17:48:01.731 | INFO | PerfReport: 56.81%, FrameLabeling -- 1.79e+01 / 1.93e+01, 1.42e+00 / 1.02e+00 | 20260505 17:48:01.731 | INFO | PerfReport: 7.36%, PulseAccumulating -- 2.32e+00 / 2.92e+00, 4.50e-01 / 1.69e-01 | 20260505 17:48:01.731 | INFO | PerfReport: 3.36%, Metrics -- 1.06e+00 / 1.40e+00, 2.93e-01 / 1.63e-01 | 20260505 17:48:01.731 | INFO | PerfReport: 1.18%, PulseSerializing -- 3.71e-01 / 4.43e-01, 6.71e-02 / 4.33e-02 | 20260505 17:48:01.731 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 17:48:01.731 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:48:01.731 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:48:01.731 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.57e+01 / 0.00e+00 | 20260505 17:48:01.731 | INFO | PerfReport: 22.56%, Upload -- 2.26e+01 / 0.00e+00, 7.95e+00 / 0.00e+00 | 20260505 17:48:01.731 | INFO | PerfReport: 21.88%, Download -- 2.19e+01 / 0.00e+00, 7.89e+00 / 0.00e+00 | 20260505 17:48:01.731 | INFO | PerfReport: 31.56%, Compute -- 3.16e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 17:48:01.731 | INFO | PerfReport: 2.35%, LockUpload -- 2.35e+00 / 0.00e+00, 7.97e+00 / 0.00e+00 | 20260505 17:48:01.731 | INFO | PerfReport: 1.37%, LockDownload -- 1.37e+00 / 0.00e+00, 2.89e+00 / 0.00e+00 | 20260505 17:48:01.731 | INFO | PerfReport: 20.27%, LockCompute -- 2.03e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 17:48:01.731 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.73e-02 / 0.00e+00 | 20260505 17:48:01.876 | INFO | Starting a round of Baz IO | 20260505 17:48:05.058 | INFO | Populating RTMetrics report | 20260505 17:48:05.059 | INFO | RTMetrics report populated | 20260505 17:48:05.059 | INFO | Opening tmp RTMetrics json file | 20260505 17:48:05.063 | INFO | Serializing RTMetrics to json | 20260505 17:48:05.065 | INFO | Serialization done, swapping tmp file now... | 20260505 17:48:05.065 | INFO | RTMetrics done | 20260505 17:48:05.152 | INFO | Finished a round of Baz IO: Wrote 2.118 GiB in 3.276 seconds | 20260505 17:48:05.152 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.00% / 0.00% / 77.99% | 20260505 17:48:05.152 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.281366% / 1.58946e-05% / 0.000139078% | 20260505 17:48:05.152 | INFO | Baz IO overall: | 20260505 17:48:05.152 | INFO | BytesWritten : 3022914450872 | 20260505 17:48:05.152 | INFO | NumEvents : 1432963901867 | 20260505 17:48:06.871 | INFO | Analyzing chunk frames = [5120000,5120512) | 20260505 17:48:06.871 | INFO | MemInfo: { | 20260505 17:48:06.871 | INFO | "gpu" : " 17 GB", | 20260505 17:48:06.871 | INFO | "huge" : "", | 20260505 17:48:06.871 | INFO | "huge_shared" : " 27 GB", | 20260505 17:48:06.871 | INFO | "malloc" : " 14 GB", | 20260505 17:48:06.871 | INFO | "pinned" : " 29 GB" | 20260505 17:48:06.871 | INFO | } | 20260505 17:48:06.871 | INFO | Perf report for the last 5.14098 seconds: | 20260505 17:48:06.871 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:48:06.871 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:06.871 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:06.871 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:06.871 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 17:48:06.871 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:06.871 | INFO | RT_METRICS : 44.7%, 1.0, 23.4ms, 0 | 20260505 17:48:06.871 | INFO | PRE_HQ : 60.4%, 1.8, 56.5ms, 0 | 20260505 17:48:06.871 | INFO | BAZWRITER : 63.2%, 3.8, 124.0ms, 0 | 20260505 17:48:06.871 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 17:48:06.871 | INFO | Uploaded 36GiB to GPU in 2.15015s (16.743GiB/s) | 20260505 17:48:06.871 | INFO | Downloaded 38GiB to GPU in 2.25594s (16.8444GiB/s) | 20260505 17:48:12.020 | INFO | Analyzing chunk frames = [5120512,5121024) | 20260505 17:48:12.020 | INFO | MemInfo: { | 20260505 17:48:12.020 | INFO | "gpu" : " 17 GB", | 20260505 17:48:12.020 | INFO | "huge" : "", | 20260505 17:48:12.020 | INFO | "huge_shared" : " 27 GB", | 20260505 17:48:12.020 | INFO | "malloc" : " 14 GB", | 20260505 17:48:12.020 | INFO | "pinned" : " 29 GB" | 20260505 17:48:12.020 | INFO | } | 20260505 17:48:12.020 | INFO | Perf report for the last 5.14965 seconds: | 20260505 17:48:12.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:48:12.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:12.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:12.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:12.020 | INFO | ANALYSIS : 62.7%, 3.0, 98.0ms, 0 | 20260505 17:48:12.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:12.020 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:12.020 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 17:48:12.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:12.020 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:12.021 | INFO | Uploaded 36GiB to GPU in 2.13067s (16.8961GiB/s) | 20260505 17:48:12.021 | INFO | Downloaded 34GiB to GPU in 2.02766s (16.7681GiB/s) | 20260505 17:48:17.156 | INFO | Analyzing chunk frames = [5121024,5121536) | 20260505 17:48:17.156 | INFO | MemInfo: { | 20260505 17:48:17.156 | INFO | "gpu" : " 17 GB", | 20260505 17:48:17.156 | INFO | "huge" : "", | 20260505 17:48:17.156 | INFO | "huge_shared" : " 27 GB", | 20260505 17:48:17.156 | INFO | "malloc" : " 14 GB", | 20260505 17:48:17.156 | INFO | "pinned" : " 29 GB" | 20260505 17:48:17.156 | INFO | } | 20260505 17:48:17.156 | INFO | Perf report for the last 5.13534 seconds: | 20260505 17:48:17.156 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:48:17.156 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:17.156 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:17.156 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:17.156 | INFO | ANALYSIS : 62.8%, 3.0, 97.6ms, 0 | 20260505 17:48:17.156 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:17.156 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:17.156 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:48:17.156 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:17.156 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:17.156 | INFO | Uploaded 36GiB to GPU in 2.20842s (16.3013GiB/s) | 20260505 17:48:17.156 | INFO | Downloaded 34GiB to GPU in 2.07478s (16.3872GiB/s) | 20260505 17:48:22.290 | INFO | Analyzing chunk frames = [5121536,5122048) | 20260505 17:48:22.290 | INFO | MemInfo: { | 20260505 17:48:22.290 | INFO | "gpu" : " 17 GB", | 20260505 17:48:22.290 | INFO | "huge" : "", | 20260505 17:48:22.290 | INFO | "huge_shared" : " 27 GB", | 20260505 17:48:22.290 | INFO | "malloc" : " 14 GB", | 20260505 17:48:22.290 | INFO | "pinned" : " 29 GB" | 20260505 17:48:22.290 | INFO | } | 20260505 17:48:22.290 | INFO | Perf report for the last 5.13461 seconds: | 20260505 17:48:22.290 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:48:22.290 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:22.290 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:22.290 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:22.290 | INFO | ANALYSIS : 62.4%, 3.0, 96.9ms, 0 | 20260505 17:48:22.290 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:22.290 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:22.290 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:48:22.290 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:22.290 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:22.291 | INFO | Uploaded 36GiB to GPU in 2.18603s (16.4682GiB/s) | 20260505 17:48:22.291 | INFO | Downloaded 34GiB to GPU in 2.05023s (16.5835GiB/s) | 20260505 17:48:27.425 | INFO | Analyzing chunk frames = [5122048,5122560) | 20260505 17:48:27.425 | INFO | MemInfo: { | 20260505 17:48:27.425 | INFO | "gpu" : " 17 GB", | 20260505 17:48:27.425 | INFO | "huge" : "", | 20260505 17:48:27.425 | INFO | "huge_shared" : " 27 GB", | 20260505 17:48:27.425 | INFO | "malloc" : " 14 GB", | 20260505 17:48:27.425 | INFO | "pinned" : " 29 GB" | 20260505 17:48:27.425 | INFO | } | 20260505 17:48:27.425 | INFO | Perf report for the last 5.13493 seconds: | 20260505 17:48:27.425 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:48:27.425 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:27.425 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:27.425 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:27.425 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 17:48:27.425 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:27.425 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:27.425 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:48:27.425 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:27.425 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:27.426 | INFO | Uploaded 36GiB to GPU in 2.15413s (16.7121GiB/s) | 20260505 17:48:27.426 | INFO | Downloaded 34GiB to GPU in 2.05216s (16.5679GiB/s) | 20260505 17:48:32.567 | INFO | Analyzing chunk frames = [5122560,5123072) | 20260505 17:48:32.567 | INFO | MemInfo: { | 20260505 17:48:32.567 | INFO | "gpu" : " 17 GB", | 20260505 17:48:32.567 | INFO | "huge" : "", | 20260505 17:48:32.567 | INFO | "huge_shared" : " 27 GB", | 20260505 17:48:32.567 | INFO | "malloc" : " 14 GB", | 20260505 17:48:32.567 | INFO | "pinned" : " 29 GB" | 20260505 17:48:32.567 | INFO | } | 20260505 17:48:32.567 | INFO | Perf report for the last 5.14208 seconds: | 20260505 17:48:32.567 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:48:32.567 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:32.567 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:32.567 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:32.567 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260505 17:48:32.567 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:32.567 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:32.567 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 17:48:32.567 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:32.567 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:32.568 | INFO | Uploaded 36GiB to GPU in 2.13685s (16.8472GiB/s) | 20260505 17:48:32.568 | INFO | Downloaded 34GiB to GPU in 2.03684s (16.6925GiB/s) | 20260505 17:48:37.712 | INFO | Analyzing chunk frames = [5123072,5123584) | 20260505 17:48:37.712 | INFO | MemInfo: { | 20260505 17:48:37.712 | INFO | "gpu" : " 17 GB", | 20260505 17:48:37.712 | INFO | "huge" : "", | 20260505 17:48:37.712 | INFO | "huge_shared" : " 27 GB", | 20260505 17:48:37.712 | INFO | "malloc" : " 14 GB", | 20260505 17:48:37.712 | INFO | "pinned" : " 29 GB" | 20260505 17:48:37.712 | INFO | } | 20260505 17:48:37.712 | INFO | Perf report for the last 5.1445 seconds: | 20260505 17:48:37.712 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:48:37.712 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:37.712 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:37.712 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:37.712 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 17:48:37.712 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:37.712 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:37.712 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 17:48:37.712 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:37.712 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:37.712 | INFO | Uploaded 36GiB to GPU in 2.14829s (16.7575GiB/s) | 20260505 17:48:37.712 | INFO | Downloaded 34GiB to GPU in 2.17857s (15.6065GiB/s) | 20260505 17:48:42.858 | INFO | Analyzing chunk frames = [5123584,5124096) | 20260505 17:48:42.858 | INFO | MemInfo: { | 20260505 17:48:42.858 | INFO | "gpu" : " 17 GB", | 20260505 17:48:42.858 | INFO | "huge" : "", | 20260505 17:48:42.858 | INFO | "huge_shared" : " 27 GB", | 20260505 17:48:42.858 | INFO | "malloc" : " 14 GB", | 20260505 17:48:42.858 | INFO | "pinned" : " 29 GB" | 20260505 17:48:42.858 | INFO | } | 20260505 17:48:42.858 | INFO | Perf report for the last 5.14596 seconds: | 20260505 17:48:42.858 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:48:42.858 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:42.858 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:42.858 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:42.858 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 17:48:42.858 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:42.858 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:42.858 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 17:48:42.858 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:42.858 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:42.858 | INFO | Uploaded 36GiB to GPU in 2.20266s (16.3439GiB/s) | 20260505 17:48:42.858 | INFO | Downloaded 34GiB to GPU in 2.0386s (16.6781GiB/s) | 20260505 17:48:42.991 | INFO | Starting a round of Baz IO | 20260505 17:48:46.174 | INFO | Populating RTMetrics report | 20260505 17:48:46.174 | INFO | RTMetrics report populated | 20260505 17:48:46.174 | INFO | Opening tmp RTMetrics json file | 20260505 17:48:46.186 | INFO | Serializing RTMetrics to json | 20260505 17:48:46.188 | INFO | Serialization done, swapping tmp file now... | 20260505 17:48:46.190 | INFO | RTMetrics done | 20260505 17:48:46.268 | INFO | Finished a round of Baz IO: Wrote 2.119 GiB in 3.276 seconds | 20260505 17:48:46.268 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.00% / 0.00% / 78.00% | 20260505 17:48:46.268 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.281886% / 3.97364e-06% / 0.000119209% | 20260505 17:48:46.268 | INFO | Baz IO overall: | 20260505 17:48:46.268 | INFO | BytesWritten : 3025189857114 | 20260505 17:48:46.268 | INFO | NumEvents : 1434026695904 | 20260505 17:48:47.990 | INFO | Analyzing chunk frames = [5124096,5124608) | 20260505 17:48:47.990 | INFO | MemInfo: { | 20260505 17:48:47.990 | INFO | "gpu" : " 17 GB", | 20260505 17:48:47.990 | INFO | "huge" : "", | 20260505 17:48:47.990 | INFO | "huge_shared" : " 27 GB", | 20260505 17:48:47.990 | INFO | "malloc" : " 14 GB", | 20260505 17:48:47.990 | INFO | "pinned" : " 29 GB" | 20260505 17:48:47.990 | INFO | } | 20260505 17:48:47.991 | INFO | Perf report for the last 5.13264 seconds: | 20260505 17:48:47.991 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:48:47.991 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:47.991 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:47.991 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:47.991 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 17:48:47.991 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:47.991 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260505 17:48:47.991 | INFO | PRE_HQ : 60.7%, 1.8, 56.7ms, 0 | 20260505 17:48:47.991 | INFO | BAZWRITER : 63.3%, 3.8, 124.8ms, 0 | 20260505 17:48:47.991 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 17:48:47.991 | INFO | Uploaded 36GiB to GPU in 2.16153s (16.6549GiB/s) | 20260505 17:48:47.991 | INFO | Downloaded 38GiB to GPU in 2.24329s (16.9394GiB/s) | 20260505 17:48:53.127 | INFO | Analyzing chunk frames = [5124608,5125120) | 20260505 17:48:53.127 | INFO | MemInfo: { | 20260505 17:48:53.127 | INFO | "gpu" : " 17 GB", | 20260505 17:48:53.127 | INFO | "huge" : "", | 20260505 17:48:53.127 | INFO | "huge_shared" : " 27 GB", | 20260505 17:48:53.127 | INFO | "malloc" : " 14 GB", | 20260505 17:48:53.127 | INFO | "pinned" : " 29 GB" | 20260505 17:48:53.127 | INFO | } | 20260505 17:48:53.127 | INFO | Perf report for the last 5.13698 seconds: | 20260505 17:48:53.128 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:48:53.128 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:53.128 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:53.128 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:53.128 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 17:48:53.128 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:53.128 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:53.128 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:48:53.128 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:53.128 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:53.128 | INFO | Uploaded 36GiB to GPU in 2.13255s (16.8812GiB/s) | 20260505 17:48:53.128 | INFO | Downloaded 34GiB to GPU in 2.02502s (16.7899GiB/s) | 20260505 17:48:58.268 | INFO | Analyzing chunk frames = [5125120,5125632) | 20260505 17:48:58.268 | INFO | MemInfo: { | 20260505 17:48:58.268 | INFO | "gpu" : " 17 GB", | 20260505 17:48:58.268 | INFO | "huge" : "", | 20260505 17:48:58.268 | INFO | "huge_shared" : " 27 GB", | 20260505 17:48:58.268 | INFO | "malloc" : " 14 GB", | 20260505 17:48:58.268 | INFO | "pinned" : " 29 GB" | 20260505 17:48:58.268 | INFO | } | 20260505 17:48:58.268 | INFO | Perf report for the last 5.14031 seconds: | 20260505 17:48:58.268 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:48:58.268 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:58.268 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:58.268 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:58.268 | INFO | ANALYSIS : 62.6%, 3.0, 97.4ms, 0 | 20260505 17:48:58.268 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:58.268 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:58.268 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:48:58.268 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:58.268 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:48:58.268 | INFO | Uploaded 36GiB to GPU in 2.19829s (16.3764GiB/s) | 20260505 17:48:58.268 | INFO | Downloaded 34GiB to GPU in 2.03666s (16.694GiB/s) | 20260505 17:49:03.407 | INFO | Analyzing chunk frames = [5125632,5126144) | 20260505 17:49:03.407 | INFO | MemInfo: { | 20260505 17:49:03.407 | INFO | "gpu" : " 17 GB", | 20260505 17:49:03.407 | INFO | "huge" : "", | 20260505 17:49:03.407 | INFO | "huge_shared" : " 27 GB", | 20260505 17:49:03.407 | INFO | "malloc" : " 14 GB", | 20260505 17:49:03.407 | INFO | "pinned" : " 29 GB" | 20260505 17:49:03.407 | INFO | } | 20260505 17:49:03.407 | INFO | Perf report for the last 5.13893 seconds: | 20260505 17:49:03.407 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:49:03.407 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:03.407 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:03.407 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:03.407 | INFO | ANALYSIS : 62.5%, 3.0, 97.0ms, 0 | 20260505 17:49:03.407 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:03.407 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:03.407 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 17:49:03.407 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:03.407 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:03.407 | INFO | Uploaded 36GiB to GPU in 2.18073s (16.5082GiB/s) | 20260505 17:49:03.407 | INFO | Downloaded 34GiB to GPU in 2.06377s (16.4747GiB/s) | 20260505 17:49:08.554 | INFO | Analyzing chunk frames = [5126144,5126656) | 20260505 17:49:08.554 | INFO | MemInfo: { | 20260505 17:49:08.554 | INFO | "gpu" : " 17 GB", | 20260505 17:49:08.554 | INFO | "huge" : "", | 20260505 17:49:08.554 | INFO | "huge_shared" : " 27 GB", | 20260505 17:49:08.554 | INFO | "malloc" : " 14 GB", | 20260505 17:49:08.554 | INFO | "pinned" : " 29 GB" | 20260505 17:49:08.554 | INFO | } | 20260505 17:49:08.554 | INFO | Perf report for the last 5.14689 seconds: | 20260505 17:49:08.554 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:49:08.554 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:08.554 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:08.554 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:08.554 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260505 17:49:08.554 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:08.554 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:08.554 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:49:08.554 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:08.554 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:08.554 | INFO | Uploaded 36GiB to GPU in 2.13815s (16.837GiB/s) | 20260505 17:49:08.554 | INFO | Downloaded 34GiB to GPU in 2.05949s (16.5089GiB/s) | 20260505 17:49:13.683 | INFO | Analyzing chunk frames = [5126656,5127168) | 20260505 17:49:13.683 | INFO | MemInfo: { | 20260505 17:49:13.683 | INFO | "gpu" : " 17 GB", | 20260505 17:49:13.683 | INFO | "huge" : "", | 20260505 17:49:13.683 | INFO | "huge_shared" : " 27 GB", | 20260505 17:49:13.683 | INFO | "malloc" : " 14 GB", | 20260505 17:49:13.683 | INFO | "pinned" : " 29 GB" | 20260505 17:49:13.683 | INFO | } | 20260505 17:49:13.683 | INFO | Perf report for the last 5.12916 seconds: | 20260505 17:49:13.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:49:13.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:13.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:13.683 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:13.683 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260505 17:49:13.683 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:13.683 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:13.683 | INFO | PRE_HQ : 27.5%, 1.0, 14.2ms, 0 | 20260505 17:49:13.683 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:13.683 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:13.683 | INFO | Uploaded 36GiB to GPU in 2.14605s (16.775GiB/s) | 20260505 17:49:13.683 | INFO | Downloaded 34GiB to GPU in 2.04732s (16.6071GiB/s) | 20260505 17:49:18.823 | INFO | Analyzing chunk frames = [5127168,5127680) | 20260505 17:49:18.823 | INFO | MemInfo: { | 20260505 17:49:18.823 | INFO | "gpu" : " 17 GB", | 20260505 17:49:18.823 | INFO | "huge" : "", | 20260505 17:49:18.823 | INFO | "huge_shared" : " 27 GB", | 20260505 17:49:18.823 | INFO | "malloc" : " 14 GB", | 20260505 17:49:18.823 | INFO | "pinned" : " 29 GB" | 20260505 17:49:18.823 | INFO | } | 20260505 17:49:18.823 | INFO | Perf report for the last 5.14015 seconds: | 20260505 17:49:18.823 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:49:18.823 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:18.823 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:18.823 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:18.823 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 17:49:18.823 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:18.823 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:18.823 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 17:49:18.823 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:18.823 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:18.823 | INFO | Uploaded 36GiB to GPU in 2.11753s (17.001GiB/s) | 20260505 17:49:18.823 | INFO | Downloaded 34GiB to GPU in 2.16961s (15.671GiB/s) | 20260505 17:49:23.970 | INFO | Analyzing chunk frames = [5127680,5128192) | 20260505 17:49:23.970 | INFO | MemInfo: { | 20260505 17:49:23.970 | INFO | "gpu" : " 17 GB", | 20260505 17:49:23.970 | INFO | "huge" : "", | 20260505 17:49:23.970 | INFO | "huge_shared" : " 27 GB", | 20260505 17:49:23.970 | INFO | "malloc" : " 14 GB", | 20260505 17:49:23.971 | INFO | "pinned" : " 29 GB" | 20260505 17:49:23.971 | INFO | } | 20260505 17:49:23.971 | INFO | Perf report for the last 5.14759 seconds: | 20260505 17:49:23.971 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:49:23.971 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:23.971 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:23.971 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:23.971 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 17:49:23.971 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:23.971 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:23.971 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260505 17:49:23.971 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:23.971 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:23.971 | INFO | Uploaded 36GiB to GPU in 2.17053s (16.5858GiB/s) | 20260505 17:49:23.971 | INFO | Downloaded 34GiB to GPU in 2.02646s (16.778GiB/s) | 20260505 17:49:23.971 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 17:49:23.971 | INFO | Total mem usage: 15.3 GiB | 20260505 17:49:23.971 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 17:49:23.971 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.98%) | 20260505 17:49:23.971 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 17:49:23.971 | INFO | High Water Breakdown: | 20260505 17:49:23.971 | INFO | 61% : 8.66 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 17:49:23.971 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 17:49:23.971 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 17:49:23.971 | INFO | ------------------------------------------ | 20260505 17:49:23.971 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:49:23.971 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 17:49:23.971 | INFO | Total mem usage: 26.5 GiB | 20260505 17:49:23.971 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 17:49:23.971 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 17:49:23.971 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 17:49:23.971 | INFO | High Water Breakdown: | 20260505 17:49:23.971 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:49:23.971 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:49:23.971 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:49:23.971 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:49:23.971 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:49:23.971 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:49:23.971 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 17:49:23.971 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 17:49:23.971 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:49:23.971 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 17:49:23.971 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 17:49:23.971 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 17:49:23.971 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 17:49:23.971 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 17:49:23.971 | INFO | ------------------------------------------ | 20260505 17:49:23.971 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:49:23.971 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 17:49:23.971 | INFO | Total mem usage: 25.6 GiB | 20260505 17:49:23.971 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 17:49:23.971 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 17:49:23.971 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 17:49:23.971 | INFO | High Water Breakdown: | 20260505 17:49:23.971 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 17:49:23.971 | INFO | ------------------------------------------ | 20260505 17:49:23.971 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:49:23.971 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 17:49:23.971 | INFO | Total mem usage: 15.7 GiB | 20260505 17:49:23.971 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 17:49:23.971 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 17:49:23.971 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 17:49:23.971 | INFO | High Water Breakdown: | 20260505 17:49:23.971 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 17:49:23.971 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 17:49:23.971 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 17:49:23.971 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 17:49:23.971 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 17:49:23.971 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 17:49:23.971 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 17:49:23.971 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 17:49:23.971 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 17:49:23.971 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:49:23.971 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 17:49:23.971 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:49:23.971 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 17:49:23.971 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 17:49:23.971 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 17:49:23.971 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 17:49:23.971 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:49:23.971 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 17:49:23.971 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:49:23.971 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:49:23.971 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:49:23.972 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 17:49:23.972 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 17:49:23.972 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:49:23.972 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 17:49:23.972 | INFO | ------------------------------------------ | 20260505 17:49:23.972 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:49:23.972 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 17:49:23.972 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:49:23.972 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:49:23.972 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.47e+01, 1.34e+01 / 2.07e+00 | 20260505 17:49:23.972 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 1.03e-01, 1.06e-02 / 6.01e-03 | 20260505 17:49:23.972 | INFO | PerfReport: 11.79%, Baseline -- 3.72e+00 / 4.51e+00, 5.64e-01 / 1.98e-01 | 20260505 17:49:23.972 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.53e+00, 1.89e-01 / 9.78e-02 | 20260505 17:49:23.972 | INFO | PerfReport: 15.89%, DME -- 4.01e+01 / 4.44e+01, 4.09e+00 / 1.87e+00 | 20260505 17:49:23.972 | INFO | PerfReport: 56.86%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.48e+00 / 1.41e+00 | 20260505 17:49:23.972 | INFO | PerfReport: 7.35%, PulseAccumulating -- 2.32e+00 / 2.91e+00, 4.43e-01 / 1.84e-01 | 20260505 17:49:23.972 | INFO | PerfReport: 3.36%, Metrics -- 1.06e+00 / 1.41e+00, 2.88e-01 / 1.56e-01 | 20260505 17:49:23.972 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.70e-01 / 4.32e-01, 6.52e-02 / 3.59e-02 | 20260505 17:49:23.972 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 17:49:23.972 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:49:23.972 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:49:23.972 | INFO | PerfReport: Full Execution -- 9.99e+01 / -nan, 2.54e+01 / 0.00e+00 | 20260505 17:49:23.972 | INFO | PerfReport: 22.53%, Upload -- 2.25e+01 / 0.00e+00, 7.72e+00 / 0.00e+00 | 20260505 17:49:23.972 | INFO | PerfReport: 21.76%, Download -- 2.17e+01 / 0.00e+00, 7.63e+00 / 0.00e+00 | 20260505 17:49:23.972 | INFO | PerfReport: 31.62%, Compute -- 3.16e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 17:49:23.972 | INFO | PerfReport: 2.31%, LockUpload -- 2.31e+00 / 0.00e+00, 7.97e+00 / 0.00e+00 | 20260505 17:49:23.972 | INFO | PerfReport: 1.43%, LockDownload -- 1.43e+00 / 0.00e+00, 3.01e+00 / 0.00e+00 | 20260505 17:49:23.972 | INFO | PerfReport: 20.33%, LockCompute -- 2.03e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 17:49:23.972 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 1.87e-02 / 0.00e+00 | 20260505 17:49:24.107 | INFO | Starting a round of Baz IO | 20260505 17:49:27.284 | INFO | Populating RTMetrics report | 20260505 17:49:27.285 | INFO | RTMetrics report populated | 20260505 17:49:27.285 | INFO | Opening tmp RTMetrics json file | 20260505 17:49:27.290 | INFO | Serializing RTMetrics to json | 20260505 17:49:27.291 | INFO | Serialization done, swapping tmp file now... | 20260505 17:49:27.291 | INFO | RTMetrics done | 20260505 17:49:27.379 | INFO | Finished a round of Baz IO: Wrote 2.116 GiB in 3.271 seconds | 20260505 17:49:27.379 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.00% / 0.00% / 78.00% | 20260505 17:49:27.379 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.280178% / 0% / 9.53674e-05% | 20260505 17:49:27.379 | INFO | Baz IO overall: | 20260505 17:49:27.379 | INFO | BytesWritten : 3027461698018 | 20260505 17:49:27.379 | INFO | NumEvents : 1435086890297 | 20260505 17:49:29.104 | INFO | Analyzing chunk frames = [5128192,5128704) | 20260505 17:49:29.104 | INFO | MemInfo: { | 20260505 17:49:29.104 | INFO | "gpu" : " 17 GB", | 20260505 17:49:29.104 | INFO | "huge" : "", | 20260505 17:49:29.104 | INFO | "huge_shared" : " 27 GB", | 20260505 17:49:29.104 | INFO | "malloc" : " 14 GB", | 20260505 17:49:29.104 | INFO | "pinned" : " 29 GB" | 20260505 17:49:29.104 | INFO | } | 20260505 17:49:29.104 | INFO | Perf report for the last 5.13383 seconds: | 20260505 17:49:29.104 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:49:29.104 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:29.104 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:29.104 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:29.105 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260505 17:49:29.105 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:29.105 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260505 17:49:29.105 | INFO | PRE_HQ : 60.5%, 1.8, 56.1ms, 0 | 20260505 17:49:29.105 | INFO | BAZWRITER : 63.2%, 3.8, 123.5ms, 0 | 20260505 17:49:29.105 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 17:49:29.105 | INFO | Uploaded 36GiB to GPU in 2.1721s (16.5738GiB/s) | 20260505 17:49:29.105 | INFO | Downloaded 38GiB to GPU in 2.24921s (16.8948GiB/s) | 20260505 17:49:34.238 | INFO | Analyzing chunk frames = [5128704,5129216) | 20260505 17:49:34.238 | INFO | MemInfo: { | 20260505 17:49:34.238 | INFO | "gpu" : " 17 GB", | 20260505 17:49:34.238 | INFO | "huge" : "", | 20260505 17:49:34.238 | INFO | "huge_shared" : " 27 GB", | 20260505 17:49:34.238 | INFO | "malloc" : " 14 GB", | 20260505 17:49:34.238 | INFO | "pinned" : " 29 GB" | 20260505 17:49:34.238 | INFO | } | 20260505 17:49:34.238 | INFO | Perf report for the last 5.13389 seconds: | 20260505 17:49:34.238 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:49:34.238 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:34.238 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:34.238 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:34.238 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260505 17:49:34.238 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:34.238 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:34.238 | INFO | PRE_HQ : 27.5%, 1.0, 14.2ms, 0 | 20260505 17:49:34.238 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:34.238 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:34.239 | INFO | Uploaded 36GiB to GPU in 2.1333s (16.8752GiB/s) | 20260505 17:49:34.239 | INFO | Downloaded 34GiB to GPU in 2.02273s (16.809GiB/s) | 20260505 17:49:39.375 | INFO | Analyzing chunk frames = [5129216,5129728) | 20260505 17:49:39.375 | INFO | MemInfo: { | 20260505 17:49:39.375 | INFO | "gpu" : " 17 GB", | 20260505 17:49:39.375 | INFO | "huge" : "", | 20260505 17:49:39.375 | INFO | "huge_shared" : " 27 GB", | 20260505 17:49:39.375 | INFO | "malloc" : " 14 GB", | 20260505 17:49:39.375 | INFO | "pinned" : " 29 GB" | 20260505 17:49:39.375 | INFO | } | 20260505 17:49:39.375 | INFO | Perf report for the last 5.13702 seconds: | 20260505 17:49:39.375 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:49:39.375 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:39.375 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:39.375 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:39.375 | INFO | ANALYSIS : 62.7%, 3.0, 97.5ms, 0 | 20260505 17:49:39.375 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:39.375 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:39.375 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 17:49:39.375 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:39.375 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:39.376 | INFO | Uploaded 36GiB to GPU in 2.18202s (16.4984GiB/s) | 20260505 17:49:39.376 | INFO | Downloaded 34GiB to GPU in 2.0308s (16.7422GiB/s) | 20260505 17:49:44.521 | INFO | Analyzing chunk frames = [5129728,5130240) | 20260505 17:49:44.521 | INFO | MemInfo: { | 20260505 17:49:44.521 | INFO | "gpu" : " 17 GB", | 20260505 17:49:44.521 | INFO | "huge" : "", | 20260505 17:49:44.521 | INFO | "huge_shared" : " 27 GB", | 20260505 17:49:44.521 | INFO | "malloc" : " 14 GB", | 20260505 17:49:44.521 | INFO | "pinned" : " 29 GB" | 20260505 17:49:44.521 | INFO | } | 20260505 17:49:44.521 | INFO | Perf report for the last 5.1459 seconds: | 20260505 17:49:44.521 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:49:44.521 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:44.521 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:44.521 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:44.521 | INFO | ANALYSIS : 62.5%, 3.0, 97.2ms, 0 | 20260505 17:49:44.521 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:44.521 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:44.521 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 17:49:44.521 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:44.521 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:44.522 | INFO | Uploaded 36GiB to GPU in 2.17919s (16.5199GiB/s) | 20260505 17:49:44.522 | INFO | Downloaded 34GiB to GPU in 2.14303s (15.8654GiB/s) | 20260505 17:49:49.667 | INFO | Analyzing chunk frames = [5130240,5130752) | 20260505 17:49:49.667 | INFO | MemInfo: { | 20260505 17:49:49.667 | INFO | "gpu" : " 17 GB", | 20260505 17:49:49.667 | INFO | "huge" : "", | 20260505 17:49:49.667 | INFO | "huge_shared" : " 27 GB", | 20260505 17:49:49.667 | INFO | "malloc" : " 14 GB", | 20260505 17:49:49.667 | INFO | "pinned" : " 29 GB" | 20260505 17:49:49.667 | INFO | } | 20260505 17:49:49.667 | INFO | Perf report for the last 5.1461 seconds: | 20260505 17:49:49.667 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:49:49.667 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:49.667 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:49.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:49.668 | INFO | ANALYSIS : 61.9%, 3.0, 96.8ms, 0 | 20260505 17:49:49.668 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:49.668 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:49.668 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 17:49:49.668 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:49.668 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:49.668 | INFO | Uploaded 36GiB to GPU in 2.14275s (16.8009GiB/s) | 20260505 17:49:49.668 | INFO | Downloaded 34GiB to GPU in 2.0591s (16.5121GiB/s) | 20260505 17:49:54.807 | INFO | Analyzing chunk frames = [5130752,5131264) | 20260505 17:49:54.807 | INFO | MemInfo: { | 20260505 17:49:54.807 | INFO | "gpu" : " 17 GB", | 20260505 17:49:54.807 | INFO | "huge" : "", | 20260505 17:49:54.807 | INFO | "huge_shared" : " 27 GB", | 20260505 17:49:54.807 | INFO | "malloc" : " 14 GB", | 20260505 17:49:54.807 | INFO | "pinned" : " 29 GB" | 20260505 17:49:54.807 | INFO | } | 20260505 17:49:54.807 | INFO | Perf report for the last 5.13947 seconds: | 20260505 17:49:54.807 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:49:54.807 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:54.807 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:54.807 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:54.807 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 17:49:54.807 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:54.807 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:54.807 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 17:49:54.807 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:54.807 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:54.807 | INFO | Uploaded 36GiB to GPU in 2.15189s (16.7295GiB/s) | 20260505 17:49:54.807 | INFO | Downloaded 34GiB to GPU in 2.05638s (16.5339GiB/s) | 20260505 17:49:59.939 | INFO | Analyzing chunk frames = [5131264,5131776) | 20260505 17:49:59.939 | INFO | MemInfo: { | 20260505 17:49:59.939 | INFO | "gpu" : " 17 GB", | 20260505 17:49:59.939 | INFO | "huge" : "", | 20260505 17:49:59.939 | INFO | "huge_shared" : " 27 GB", | 20260505 17:49:59.939 | INFO | "malloc" : " 14 GB", | 20260505 17:49:59.939 | INFO | "pinned" : " 29 GB" | 20260505 17:49:59.939 | INFO | } | 20260505 17:49:59.939 | INFO | Perf report for the last 5.13184 seconds: | 20260505 17:49:59.939 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:49:59.939 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:59.939 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:59.939 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:59.939 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 17:49:59.939 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:59.939 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:59.939 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 17:49:59.939 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:59.939 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:49:59.939 | INFO | Uploaded 36GiB to GPU in 2.14316s (16.7976GiB/s) | 20260505 17:49:59.939 | INFO | Downloaded 34GiB to GPU in 2.15014s (15.8129GiB/s) | 20260505 17:50:05.074 | INFO | Analyzing chunk frames = [5131776,5132288) | 20260505 17:50:05.074 | INFO | MemInfo: { | 20260505 17:50:05.074 | INFO | "gpu" : " 17 GB", | 20260505 17:50:05.074 | INFO | "huge" : "", | 20260505 17:50:05.074 | INFO | "huge_shared" : " 27 GB", | 20260505 17:50:05.074 | INFO | "malloc" : " 14 GB", | 20260505 17:50:05.074 | INFO | "pinned" : " 29 GB" | 20260505 17:50:05.074 | INFO | } | 20260505 17:50:05.074 | INFO | Perf report for the last 5.13576 seconds: | 20260505 17:50:05.075 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:50:05.075 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:05.075 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:05.075 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:05.075 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 17:50:05.075 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:05.075 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:05.075 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:50:05.075 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:05.075 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:05.075 | INFO | Uploaded 36GiB to GPU in 2.18622s (16.4668GiB/s) | 20260505 17:50:05.075 | INFO | Downloaded 34GiB to GPU in 2.03586s (16.7006GiB/s) | 20260505 17:50:05.213 | INFO | Starting a round of Baz IO | 20260505 17:50:08.404 | INFO | Populating RTMetrics report | 20260505 17:50:08.404 | INFO | RTMetrics report populated | 20260505 17:50:08.404 | INFO | Opening tmp RTMetrics json file | 20260505 17:50:08.411 | INFO | Serializing RTMetrics to json | 20260505 17:50:08.414 | INFO | Serialization done, swapping tmp file now... | 20260505 17:50:08.415 | INFO | RTMetrics done | 20260505 17:50:08.494 | INFO | Finished a round of Baz IO: Wrote 2.116 GiB in 3.28 seconds | 20260505 17:50:08.494 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.00% / 0.00% / 78.00% | 20260505 17:50:08.494 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.279804% / 0% / 9.13938e-05% | 20260505 17:50:08.494 | INFO | Baz IO overall: | 20260505 17:50:08.494 | INFO | BytesWritten : 3029733314328 | 20260505 17:50:08.494 | INFO | NumEvents : 1436147635923 | 20260505 17:50:10.221 | INFO | Analyzing chunk frames = [5132288,5132800) | 20260505 17:50:10.221 | INFO | MemInfo: { | 20260505 17:50:10.221 | INFO | "gpu" : " 17 GB", | 20260505 17:50:10.221 | INFO | "huge" : "", | 20260505 17:50:10.221 | INFO | "huge_shared" : " 27 GB", | 20260505 17:50:10.221 | INFO | "malloc" : " 14 GB", | 20260505 17:50:10.221 | INFO | "pinned" : " 29 GB" | 20260505 17:50:10.221 | INFO | } | 20260505 17:50:10.221 | INFO | Perf report for the last 5.1467 seconds: | 20260505 17:50:10.221 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:50:10.221 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:10.221 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:10.221 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:10.221 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 17:50:10.221 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:10.221 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260505 17:50:10.221 | INFO | PRE_HQ : 60.4%, 1.8, 56.4ms, 0 | 20260505 17:50:10.221 | INFO | BAZWRITER : 63.2%, 3.8, 123.6ms, 0 | 20260505 17:50:10.221 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 17:50:10.222 | INFO | Uploaded 36GiB to GPU in 2.1863s (16.4662GiB/s) | 20260505 17:50:10.222 | INFO | Downloaded 38GiB to GPU in 2.26646s (16.7662GiB/s) | 20260505 17:50:15.357 | INFO | Analyzing chunk frames = [5132800,5133312) | 20260505 17:50:15.357 | INFO | MemInfo: { | 20260505 17:50:15.357 | INFO | "gpu" : " 17 GB", | 20260505 17:50:15.357 | INFO | "huge" : "", | 20260505 17:50:15.357 | INFO | "huge_shared" : " 27 GB", | 20260505 17:50:15.357 | INFO | "malloc" : " 14 GB", | 20260505 17:50:15.357 | INFO | "pinned" : " 29 GB" | 20260505 17:50:15.357 | INFO | } | 20260505 17:50:15.357 | INFO | Perf report for the last 5.13547 seconds: | 20260505 17:50:15.357 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:50:15.357 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:15.357 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:15.357 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:15.357 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 17:50:15.357 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:15.357 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:15.357 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 17:50:15.357 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:15.357 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:15.357 | INFO | Uploaded 36GiB to GPU in 2.15707s (16.6893GiB/s) | 20260505 17:50:15.357 | INFO | Downloaded 34GiB to GPU in 2.04843s (16.5981GiB/s) | 20260505 17:50:20.492 | INFO | Analyzing chunk frames = [5133312,5133824) | 20260505 17:50:20.492 | INFO | MemInfo: { | 20260505 17:50:20.492 | INFO | "gpu" : " 17 GB", | 20260505 17:50:20.492 | INFO | "huge" : "", | 20260505 17:50:20.492 | INFO | "huge_shared" : " 27 GB", | 20260505 17:50:20.492 | INFO | "malloc" : " 14 GB", | 20260505 17:50:20.492 | INFO | "pinned" : " 29 GB" | 20260505 17:50:20.492 | INFO | } | 20260505 17:50:20.492 | INFO | Perf report for the last 5.13537 seconds: | 20260505 17:50:20.492 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:50:20.492 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:20.492 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:20.492 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:20.492 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 17:50:20.492 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:20.492 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:20.492 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:50:20.492 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:20.492 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:20.492 | INFO | Uploaded 36GiB to GPU in 2.2077s (16.3066GiB/s) | 20260505 17:50:20.492 | INFO | Downloaded 34GiB to GPU in 2.06666s (16.4517GiB/s) | 20260505 17:50:25.634 | INFO | Analyzing chunk frames = [5133824,5134336) | 20260505 17:50:25.634 | INFO | MemInfo: { | 20260505 17:50:25.634 | INFO | "gpu" : " 17 GB", | 20260505 17:50:25.634 | INFO | "huge" : "", | 20260505 17:50:25.634 | INFO | "huge_shared" : " 27 GB", | 20260505 17:50:25.634 | INFO | "malloc" : " 14 GB", | 20260505 17:50:25.634 | INFO | "pinned" : " 29 GB" | 20260505 17:50:25.634 | INFO | } | 20260505 17:50:25.634 | INFO | Perf report for the last 5.14183 seconds: | 20260505 17:50:25.634 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:50:25.634 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:25.634 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:25.634 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:25.634 | INFO | ANALYSIS : 62.4%, 3.0, 96.9ms, 0 | 20260505 17:50:25.634 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:25.634 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:25.634 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260505 17:50:25.634 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:25.634 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:25.634 | INFO | Uploaded 36GiB to GPU in 2.18515s (16.4748GiB/s) | 20260505 17:50:25.634 | INFO | Downloaded 34GiB to GPU in 2.13263s (15.9427GiB/s) | 20260505 17:50:30.783 | INFO | Analyzing chunk frames = [5134336,5134848) | 20260505 17:50:30.783 | INFO | MemInfo: { | 20260505 17:50:30.783 | INFO | "gpu" : " 17 GB", | 20260505 17:50:30.783 | INFO | "huge" : "", | 20260505 17:50:30.783 | INFO | "huge_shared" : " 27 GB", | 20260505 17:50:30.783 | INFO | "malloc" : " 14 GB", | 20260505 17:50:30.783 | INFO | "pinned" : " 29 GB" | 20260505 17:50:30.783 | INFO | } | 20260505 17:50:30.783 | INFO | Perf report for the last 5.14956 seconds: | 20260505 17:50:30.783 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:50:30.783 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:30.783 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:30.783 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:30.783 | INFO | ANALYSIS : 61.8%, 3.0, 96.7ms, 0 | 20260505 17:50:30.784 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:30.784 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:30.784 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260505 17:50:30.784 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:30.784 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:30.784 | INFO | Uploaded 36GiB to GPU in 2.12508s (16.9406GiB/s) | 20260505 17:50:30.784 | INFO | Downloaded 34GiB to GPU in 2.05129s (16.5749GiB/s) | 20260505 17:50:35.919 | INFO | Analyzing chunk frames = [5134848,5135360) | 20260505 17:50:35.919 | INFO | MemInfo: { | 20260505 17:50:35.919 | INFO | "gpu" : " 17 GB", | 20260505 17:50:35.919 | INFO | "huge" : "", | 20260505 17:50:35.919 | INFO | "huge_shared" : " 27 GB", | 20260505 17:50:35.919 | INFO | "malloc" : " 14 GB", | 20260505 17:50:35.920 | INFO | "pinned" : " 29 GB" | 20260505 17:50:35.920 | INFO | } | 20260505 17:50:35.920 | INFO | Perf report for the last 5.13606 seconds: | 20260505 17:50:35.920 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:50:35.920 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:35.920 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:35.920 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:35.920 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260505 17:50:35.920 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:35.920 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:35.920 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:50:35.920 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:35.920 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:35.920 | INFO | Uploaded 36GiB to GPU in 2.14674s (16.7696GiB/s) | 20260505 17:50:35.920 | INFO | Downloaded 34GiB to GPU in 2.05657s (16.5324GiB/s) | 20260505 17:50:41.049 | INFO | Analyzing chunk frames = [5135360,5135872) | 20260505 17:50:41.049 | INFO | MemInfo: { | 20260505 17:50:41.049 | INFO | "gpu" : " 17 GB", | 20260505 17:50:41.049 | INFO | "huge" : "", | 20260505 17:50:41.049 | INFO | "huge_shared" : " 27 GB", | 20260505 17:50:41.049 | INFO | "malloc" : " 14 GB", | 20260505 17:50:41.049 | INFO | "pinned" : " 29 GB" | 20260505 17:50:41.049 | INFO | } | 20260505 17:50:41.049 | INFO | Perf report for the last 5.12977 seconds: | 20260505 17:50:41.049 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:50:41.049 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:41.049 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:41.049 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:41.049 | INFO | ANALYSIS : 63.4%, 3.0, 98.8ms, 0 | 20260505 17:50:41.049 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:41.049 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:41.049 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 17:50:41.049 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:41.049 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:41.050 | INFO | Uploaded 36GiB to GPU in 2.17081s (16.5837GiB/s) | 20260505 17:50:41.050 | INFO | Downloaded 34GiB to GPU in 2.17237s (15.6511GiB/s) | 20260505 17:50:46.187 | INFO | Analyzing chunk frames = [5135872,5136384) | 20260505 17:50:46.187 | INFO | MemInfo: { | 20260505 17:50:46.187 | INFO | "gpu" : " 17 GB", | 20260505 17:50:46.187 | INFO | "huge" : "", | 20260505 17:50:46.187 | INFO | "huge_shared" : " 27 GB", | 20260505 17:50:46.187 | INFO | "malloc" : " 14 GB", | 20260505 17:50:46.187 | INFO | "pinned" : " 29 GB" | 20260505 17:50:46.187 | INFO | } | 20260505 17:50:46.187 | INFO | Perf report for the last 5.13808 seconds: | 20260505 17:50:46.187 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:50:46.187 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:46.187 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:46.187 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:46.187 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 17:50:46.187 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:46.188 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:46.188 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 17:50:46.188 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:46.188 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:46.188 | INFO | Uploaded 36GiB to GPU in 2.18034s (16.5111GiB/s) | 20260505 17:50:46.188 | INFO | Downloaded 34GiB to GPU in 2.03345s (16.7204GiB/s) | 20260505 17:50:46.188 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 17:50:46.188 | INFO | Total mem usage: 15.3 GiB | 20260505 17:50:46.188 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 17:50:46.188 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.01%) | 20260505 17:50:46.188 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 17:50:46.188 | INFO | High Water Breakdown: | 20260505 17:50:46.188 | INFO | 61% : 8.66 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 17:50:46.188 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 17:50:46.188 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 17:50:46.188 | INFO | ------------------------------------------ | 20260505 17:50:46.188 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:50:46.188 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 17:50:46.188 | INFO | Total mem usage: 26.5 GiB | 20260505 17:50:46.188 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 17:50:46.188 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 17:50:46.188 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 17:50:46.188 | INFO | High Water Breakdown: | 20260505 17:50:46.188 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:50:46.188 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:50:46.188 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:50:46.188 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:50:46.188 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:50:46.188 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:50:46.188 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 17:50:46.188 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 17:50:46.188 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:50:46.188 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 17:50:46.188 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 17:50:46.188 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 17:50:46.188 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 17:50:46.188 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 17:50:46.188 | INFO | ------------------------------------------ | 20260505 17:50:46.188 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:50:46.188 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 17:50:46.188 | INFO | Total mem usage: 25.6 GiB | 20260505 17:50:46.188 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 17:50:46.188 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 17:50:46.188 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 17:50:46.188 | INFO | High Water Breakdown: | 20260505 17:50:46.188 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 17:50:46.188 | INFO | ------------------------------------------ | 20260505 17:50:46.188 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:50:46.188 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 17:50:46.188 | INFO | Total mem usage: 15.7 GiB | 20260505 17:50:46.188 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 17:50:46.188 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 17:50:46.188 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 17:50:46.188 | INFO | High Water Breakdown: | 20260505 17:50:46.188 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 17:50:46.188 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 17:50:46.188 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 17:50:46.188 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 17:50:46.188 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 17:50:46.188 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 17:50:46.188 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 17:50:46.188 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 17:50:46.188 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 17:50:46.188 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:50:46.188 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 17:50:46.188 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:50:46.188 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 17:50:46.188 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 17:50:46.188 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 17:50:46.188 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 17:50:46.188 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:50:46.188 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 17:50:46.188 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:50:46.188 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:50:46.188 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:50:46.188 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 17:50:46.188 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 17:50:46.188 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:50:46.188 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 17:50:46.188 | INFO | ------------------------------------------ | 20260505 17:50:46.188 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:50:46.189 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 17:50:46.189 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:50:46.189 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:50:46.189 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.49e+01, 1.36e+01 / 2.22e+00 | 20260505 17:50:46.189 | INFO | PerfReport: 0.29%, AggregateStats -- 9.25e-02 / 1.01e-01, 1.04e-02 / 3.81e-03 | 20260505 17:50:46.189 | INFO | PerfReport: 11.80%, Baseline -- 3.73e+00 / 4.50e+00, 5.75e-01 / 2.07e-01 | 20260505 17:50:46.189 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.49e+00, 1.93e-01 / 1.02e-01 | 20260505 17:50:46.189 | INFO | PerfReport: 15.90%, DME -- 4.02e+01 / 4.45e+01, 4.24e+00 / 2.43e+00 | 20260505 17:50:46.189 | INFO | PerfReport: 56.90%, FrameLabeling -- 1.80e+01 / 1.96e+01, 1.50e+00 / 1.28e+00 | 20260505 17:50:46.189 | INFO | PerfReport: 7.30%, PulseAccumulating -- 2.31e+00 / 2.89e+00, 4.44e-01 / 2.09e-01 | 20260505 17:50:46.189 | INFO | PerfReport: 3.36%, Metrics -- 1.06e+00 / 1.39e+00, 2.94e-01 / 1.25e-01 | 20260505 17:50:46.189 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.70e-01 / 4.30e-01, 6.60e-02 / 3.40e-02 | 20260505 17:50:46.189 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 17:50:46.189 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:50:46.189 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:50:46.189 | INFO | PerfReport: Full Execution -- 9.99e+01 / -nan, 2.53e+01 / 0.00e+00 | 20260505 17:50:46.189 | INFO | PerfReport: 22.58%, Upload -- 2.26e+01 / 0.00e+00, 7.88e+00 / 0.00e+00 | 20260505 17:50:46.189 | INFO | PerfReport: 21.88%, Download -- 2.19e+01 / 0.00e+00, 7.86e+00 / 0.00e+00 | 20260505 17:50:46.189 | INFO | PerfReport: 31.64%, Compute -- 3.16e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 17:50:46.189 | INFO | PerfReport: 2.17%, LockUpload -- 2.17e+00 / 0.00e+00, 7.68e+00 / 0.00e+00 | 20260505 17:50:46.189 | INFO | PerfReport: 1.44%, LockDownload -- 1.44e+00 / 0.00e+00, 2.93e+00 / 0.00e+00 | 20260505 17:50:46.189 | INFO | PerfReport: 20.26%, LockCompute -- 2.02e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 17:50:46.189 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 2.01e-02 / 0.00e+00 | 20260505 17:50:46.324 | INFO | Starting a round of Baz IO | 20260505 17:50:49.513 | INFO | Populating RTMetrics report | 20260505 17:50:49.513 | INFO | RTMetrics report populated | 20260505 17:50:49.513 | INFO | Opening tmp RTMetrics json file | 20260505 17:50:49.518 | INFO | Serializing RTMetrics to json | 20260505 17:50:49.520 | INFO | Serialization done, swapping tmp file now... | 20260505 17:50:49.521 | INFO | RTMetrics done | 20260505 17:50:49.605 | INFO | Finished a round of Baz IO: Wrote 2.114 GiB in 3.281 seconds | 20260505 17:50:49.605 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 22.00% / 0.00% / 78.00% | 20260505 17:50:49.605 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.278874% / 3.97364e-06% / 0.000103315% | 20260505 17:50:49.605 | INFO | Baz IO overall: | 20260505 17:50:49.605 | INFO | BytesWritten : 3032003389517 | 20260505 17:50:49.605 | INFO | NumEvents : 1437207045197 | 20260505 17:50:51.331 | INFO | Analyzing chunk frames = [5136384,5136896) | 20260505 17:50:51.331 | INFO | MemInfo: { | 20260505 17:50:51.331 | INFO | "gpu" : " 17 GB", | 20260505 17:50:51.331 | INFO | "huge" : "", | 20260505 17:50:51.331 | INFO | "huge_shared" : " 27 GB", | 20260505 17:50:51.331 | INFO | "malloc" : " 14 GB", | 20260505 17:50:51.331 | INFO | "pinned" : " 29 GB" | 20260505 17:50:51.331 | INFO | } | 20260505 17:50:51.331 | INFO | Perf report for the last 5.14381 seconds: | 20260505 17:50:51.331 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:50:51.331 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:51.331 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:51.331 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:51.331 | INFO | ANALYSIS : 63.6%, 3.0, 99.1ms, 0 | 20260505 17:50:51.331 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:51.331 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260505 17:50:51.331 | INFO | PRE_HQ : 60.4%, 1.8, 56.2ms, 0 | 20260505 17:50:51.331 | INFO | BAZWRITER : 63.1%, 3.8, 123.2ms, 0 | 20260505 17:50:51.331 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 17:50:51.332 | INFO | Uploaded 36GiB to GPU in 2.15749s (16.686GiB/s) | 20260505 17:50:51.332 | INFO | Downloaded 38GiB to GPU in 2.24821s (16.9023GiB/s) | 20260505 17:50:56.473 | INFO | Analyzing chunk frames = [5136896,5137408) | 20260505 17:50:56.473 | INFO | MemInfo: { | 20260505 17:50:56.473 | INFO | "gpu" : " 17 GB", | 20260505 17:50:56.473 | INFO | "huge" : "", | 20260505 17:50:56.473 | INFO | "huge_shared" : " 27 GB", | 20260505 17:50:56.473 | INFO | "malloc" : " 14 GB", | 20260505 17:50:56.473 | INFO | "pinned" : " 29 GB" | 20260505 17:50:56.473 | INFO | } | 20260505 17:50:56.473 | INFO | Perf report for the last 5.14161 seconds: | 20260505 17:50:56.473 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:50:56.473 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:56.473 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:56.473 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:56.473 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 17:50:56.473 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:56.473 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:56.473 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 17:50:56.473 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:56.473 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:50:56.473 | INFO | Uploaded 36GiB to GPU in 2.14753s (16.7634GiB/s) | 20260505 17:50:56.473 | INFO | Downloaded 34GiB to GPU in 2.02852s (16.761GiB/s) | 20260505 17:51:01.620 | INFO | Analyzing chunk frames = [5137408,5137920) | 20260505 17:51:01.620 | INFO | MemInfo: { | 20260505 17:51:01.620 | INFO | "gpu" : " 17 GB", | 20260505 17:51:01.620 | INFO | "huge" : "", | 20260505 17:51:01.620 | INFO | "huge_shared" : " 27 GB", | 20260505 17:51:01.620 | INFO | "malloc" : " 14 GB", | 20260505 17:51:01.620 | INFO | "pinned" : " 29 GB" | 20260505 17:51:01.620 | INFO | } | 20260505 17:51:01.620 | INFO | Perf report for the last 5.14686 seconds: | 20260505 17:51:01.620 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:51:01.620 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:01.620 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:01.620 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:01.620 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260505 17:51:01.620 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:01.620 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:01.620 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 17:51:01.620 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:01.620 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:01.620 | INFO | Uploaded 36GiB to GPU in 2.13902s (16.8302GiB/s) | 20260505 17:51:01.620 | INFO | Downloaded 34GiB to GPU in 2.03749s (16.6872GiB/s) | 20260505 17:51:06.752 | INFO | Analyzing chunk frames = [5137920,5138432) | 20260505 17:51:06.752 | INFO | MemInfo: { | 20260505 17:51:06.752 | INFO | "gpu" : " 17 GB", | 20260505 17:51:06.752 | INFO | "huge" : "", | 20260505 17:51:06.752 | INFO | "huge_shared" : " 27 GB", | 20260505 17:51:06.752 | INFO | "malloc" : " 14 GB", | 20260505 17:51:06.752 | INFO | "pinned" : " 29 GB" | 20260505 17:51:06.752 | INFO | } | 20260505 17:51:06.752 | INFO | Perf report for the last 5.13219 seconds: | 20260505 17:51:06.752 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:51:06.752 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:06.752 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:06.752 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:06.752 | INFO | ANALYSIS : 62.4%, 3.0, 96.8ms, 0 | 20260505 17:51:06.752 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:06.752 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:06.752 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 17:51:06.752 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:06.752 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:06.752 | INFO | Uploaded 36GiB to GPU in 2.17931s (16.519GiB/s) | 20260505 17:51:06.752 | INFO | Downloaded 34GiB to GPU in 2.06035s (16.5021GiB/s) | 20260505 17:51:11.884 | INFO | Analyzing chunk frames = [5138432,5138944) | 20260505 17:51:11.884 | INFO | MemInfo: { | 20260505 17:51:11.884 | INFO | "gpu" : " 17 GB", | 20260505 17:51:11.884 | INFO | "huge" : "", | 20260505 17:51:11.884 | INFO | "huge_shared" : " 27 GB", | 20260505 17:51:11.884 | INFO | "malloc" : " 14 GB", | 20260505 17:51:11.884 | INFO | "pinned" : " 29 GB" | 20260505 17:51:11.885 | INFO | } | 20260505 17:51:11.885 | INFO | Perf report for the last 5.13261 seconds: | 20260505 17:51:11.885 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:51:11.885 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:11.885 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:11.885 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:11.885 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260505 17:51:11.885 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:11.885 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:11.885 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:51:11.885 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:11.885 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:11.885 | INFO | Uploaded 36GiB to GPU in 2.15077s (16.7382GiB/s) | 20260505 17:51:11.885 | INFO | Downloaded 34GiB to GPU in 2.06435s (16.4701GiB/s) | 20260505 17:51:17.029 | INFO | Analyzing chunk frames = [5138944,5139456) | 20260505 17:51:17.029 | INFO | MemInfo: { | 20260505 17:51:17.029 | INFO | "gpu" : " 17 GB", | 20260505 17:51:17.029 | INFO | "huge" : "", | 20260505 17:51:17.029 | INFO | "huge_shared" : " 27 GB", | 20260505 17:51:17.029 | INFO | "malloc" : " 14 GB", | 20260505 17:51:17.029 | INFO | "pinned" : " 29 GB" | 20260505 17:51:17.029 | INFO | } | 20260505 17:51:17.029 | INFO | Perf report for the last 5.14471 seconds: | 20260505 17:51:17.029 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:51:17.029 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:17.029 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:17.029 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:17.029 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260505 17:51:17.029 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:17.029 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:17.029 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260505 17:51:17.029 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:17.029 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:17.029 | INFO | Uploaded 36GiB to GPU in 2.14559s (16.7786GiB/s) | 20260505 17:51:17.029 | INFO | Downloaded 34GiB to GPU in 2.03728s (16.6889GiB/s) | 20260505 17:51:22.172 | INFO | Analyzing chunk frames = [5139456,5139968) | 20260505 17:51:22.172 | INFO | MemInfo: { | 20260505 17:51:22.172 | INFO | "gpu" : " 17 GB", | 20260505 17:51:22.172 | INFO | "huge" : "", | 20260505 17:51:22.173 | INFO | "huge_shared" : " 27 GB", | 20260505 17:51:22.173 | INFO | "malloc" : " 14 GB", | 20260505 17:51:22.173 | INFO | "pinned" : " 29 GB" | 20260505 17:51:22.173 | INFO | } | 20260505 17:51:22.173 | INFO | Perf report for the last 5.14337 seconds: | 20260505 17:51:22.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:51:22.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:22.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:22.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:22.173 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 17:51:22.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:22.173 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:22.173 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 17:51:22.173 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:22.173 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:22.173 | INFO | Uploaded 36GiB to GPU in 2.13781s (16.8396GiB/s) | 20260505 17:51:22.173 | INFO | Downloaded 34GiB to GPU in 2.14106s (15.88GiB/s) | 20260505 17:51:27.313 | INFO | Analyzing chunk frames = [5139968,5140480) | 20260505 17:51:27.313 | INFO | MemInfo: { | 20260505 17:51:27.313 | INFO | "gpu" : " 17 GB", | 20260505 17:51:27.313 | INFO | "huge" : "", | 20260505 17:51:27.313 | INFO | "huge_shared" : " 27 GB", | 20260505 17:51:27.313 | INFO | "malloc" : " 14 GB", | 20260505 17:51:27.313 | INFO | "pinned" : " 29 GB" | 20260505 17:51:27.313 | INFO | } | 20260505 17:51:27.313 | INFO | Perf report for the last 5.14039 seconds: | 20260505 17:51:27.313 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:51:27.313 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:27.313 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:27.313 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:27.313 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 17:51:27.313 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:27.313 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:27.313 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:51:27.313 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:27.313 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:27.313 | INFO | Uploaded 36GiB to GPU in 2.19768s (16.3809GiB/s) | 20260505 17:51:27.313 | INFO | Downloaded 34GiB to GPU in 2.026s (16.7818GiB/s) | 20260505 17:51:27.450 | INFO | Starting a round of Baz IO | 20260505 17:51:30.625 | INFO | Populating RTMetrics report | 20260505 17:51:30.625 | INFO | RTMetrics report populated | 20260505 17:51:30.625 | INFO | Opening tmp RTMetrics json file | 20260505 17:51:30.628 | INFO | Serializing RTMetrics to json | 20260505 17:51:30.629 | INFO | Serialization done, swapping tmp file now... | 20260505 17:51:30.630 | INFO | RTMetrics done | 20260505 17:51:30.725 | INFO | Finished a round of Baz IO: Wrote 2.116 GiB in 3.274 seconds | 20260505 17:51:30.725 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.99% / 0.00% / 78.00% | 20260505 17:51:30.725 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.278604% / 0% / 0.000103315% | 20260505 17:51:30.725 | INFO | Baz IO overall: | 20260505 17:51:30.725 | INFO | BytesWritten : 3034275233854 | 20260505 17:51:30.725 | INFO | NumEvents : 1438267731366 | 20260505 17:51:32.443 | INFO | Analyzing chunk frames = [5140480,5140992) | 20260505 17:51:32.443 | INFO | MemInfo: { | 20260505 17:51:32.443 | INFO | "gpu" : " 17 GB", | 20260505 17:51:32.443 | INFO | "huge" : "", | 20260505 17:51:32.443 | INFO | "huge_shared" : " 27 GB", | 20260505 17:51:32.443 | INFO | "malloc" : " 14 GB", | 20260505 17:51:32.443 | INFO | "pinned" : " 29 GB" | 20260505 17:51:32.443 | INFO | } | 20260505 17:51:32.443 | INFO | Perf report for the last 5.13023 seconds: | 20260505 17:51:32.443 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:51:32.443 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:32.443 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:32.443 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:32.443 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260505 17:51:32.443 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:32.443 | INFO | RT_METRICS : 44.7%, 1.0, 23.4ms, 0 | 20260505 17:51:32.443 | INFO | PRE_HQ : 60.8%, 1.8, 56.6ms, 0 | 20260505 17:51:32.443 | INFO | BAZWRITER : 63.4%, 3.8, 124.4ms, 0 | 20260505 17:51:32.443 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 17:51:32.444 | INFO | Uploaded 36GiB to GPU in 2.15901s (16.6743GiB/s) | 20260505 17:51:32.444 | INFO | Downloaded 38GiB to GPU in 2.24559s (16.9221GiB/s) | 20260505 17:51:37.583 | INFO | Analyzing chunk frames = [5140992,5141504) | 20260505 17:51:37.583 | INFO | MemInfo: { | 20260505 17:51:37.583 | INFO | "gpu" : " 17 GB", | 20260505 17:51:37.583 | INFO | "huge" : "", | 20260505 17:51:37.583 | INFO | "huge_shared" : " 27 GB", | 20260505 17:51:37.583 | INFO | "malloc" : " 14 GB", | 20260505 17:51:37.583 | INFO | "pinned" : " 29 GB" | 20260505 17:51:37.583 | INFO | } | 20260505 17:51:37.583 | INFO | Perf report for the last 5.14015 seconds: | 20260505 17:51:37.583 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:51:37.583 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:37.583 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:37.583 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:37.584 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 17:51:37.584 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:37.584 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:37.584 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 17:51:37.584 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:37.584 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:37.584 | INFO | Uploaded 36GiB to GPU in 2.13061s (16.8966GiB/s) | 20260505 17:51:37.584 | INFO | Downloaded 34GiB to GPU in 2.03369s (16.7184GiB/s) | 20260505 17:51:42.729 | INFO | Analyzing chunk frames = [5141504,5142016) | 20260505 17:51:42.729 | INFO | MemInfo: { | 20260505 17:51:42.729 | INFO | "gpu" : " 17 GB", | 20260505 17:51:42.729 | INFO | "huge" : "", | 20260505 17:51:42.729 | INFO | "huge_shared" : " 27 GB", | 20260505 17:51:42.729 | INFO | "malloc" : " 14 GB", | 20260505 17:51:42.729 | INFO | "pinned" : " 29 GB" | 20260505 17:51:42.729 | INFO | } | 20260505 17:51:42.729 | INFO | Perf report for the last 5.14538 seconds: | 20260505 17:51:42.729 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:51:42.729 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:42.729 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:42.729 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:42.729 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 17:51:42.729 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:42.729 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:42.729 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:51:42.729 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:42.729 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:42.729 | INFO | Uploaded 36GiB to GPU in 2.20646s (16.3157GiB/s) | 20260505 17:51:42.729 | INFO | Downloaded 34GiB to GPU in 2.04728s (16.6074GiB/s) | 20260505 17:51:47.871 | INFO | Analyzing chunk frames = [5142016,5142528) | 20260505 17:51:47.871 | INFO | MemInfo: { | 20260505 17:51:47.871 | INFO | "gpu" : " 17 GB", | 20260505 17:51:47.871 | INFO | "huge" : "", | 20260505 17:51:47.871 | INFO | "huge_shared" : " 27 GB", | 20260505 17:51:47.871 | INFO | "malloc" : " 14 GB", | 20260505 17:51:47.871 | INFO | "pinned" : " 29 GB" | 20260505 17:51:47.871 | INFO | } | 20260505 17:51:47.871 | INFO | Perf report for the last 5.14223 seconds: | 20260505 17:51:47.871 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:51:47.871 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:47.871 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:47.871 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:47.871 | INFO | ANALYSIS : 62.9%, 3.0, 97.7ms, 0 | 20260505 17:51:47.871 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:47.871 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:47.871 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 17:51:47.871 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:47.871 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:47.871 | INFO | Uploaded 36GiB to GPU in 2.22976s (16.1452GiB/s) | 20260505 17:51:47.871 | INFO | Downloaded 34GiB to GPU in 2.11783s (16.0542GiB/s) | 20260505 17:51:53.003 | INFO | Analyzing chunk frames = [5142528,5143040) | 20260505 17:51:53.003 | INFO | MemInfo: { | 20260505 17:51:53.003 | INFO | "gpu" : " 17 GB", | 20260505 17:51:53.003 | INFO | "huge" : "", | 20260505 17:51:53.003 | INFO | "huge_shared" : " 27 GB", | 20260505 17:51:53.003 | INFO | "malloc" : " 14 GB", | 20260505 17:51:53.003 | INFO | "pinned" : " 29 GB" | 20260505 17:51:53.004 | INFO | } | 20260505 17:51:53.004 | INFO | Perf report for the last 5.13244 seconds: | 20260505 17:51:53.004 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:51:53.004 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:53.004 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:53.004 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:53.004 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260505 17:51:53.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:53.004 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:53.004 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:51:53.004 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:53.004 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:53.004 | INFO | Uploaded 36GiB to GPU in 2.1458s (16.7769GiB/s) | 20260505 17:51:53.004 | INFO | Downloaded 34GiB to GPU in 2.05647s (16.5332GiB/s) | 20260505 17:51:58.136 | INFO | Analyzing chunk frames = [5143040,5143552) | 20260505 17:51:58.136 | INFO | MemInfo: { | 20260505 17:51:58.136 | INFO | "gpu" : " 17 GB", | 20260505 17:51:58.136 | INFO | "huge" : "", | 20260505 17:51:58.136 | INFO | "huge_shared" : " 27 GB", | 20260505 17:51:58.136 | INFO | "malloc" : " 14 GB", | 20260505 17:51:58.136 | INFO | "pinned" : " 29 GB" | 20260505 17:51:58.136 | INFO | } | 20260505 17:51:58.136 | INFO | Perf report for the last 5.13213 seconds: | 20260505 17:51:58.136 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:51:58.136 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:58.136 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:58.136 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:58.136 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 17:51:58.136 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:58.136 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:58.136 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 17:51:58.136 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:58.136 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:51:58.136 | INFO | Uploaded 36GiB to GPU in 2.1401s (16.8217GiB/s) | 20260505 17:51:58.136 | INFO | Downloaded 34GiB to GPU in 2.04289s (16.6431GiB/s) | 20260505 17:52:03.288 | INFO | Analyzing chunk frames = [5143552,5144064) | 20260505 17:52:03.288 | INFO | MemInfo: { | 20260505 17:52:03.288 | INFO | "gpu" : " 17 GB", | 20260505 17:52:03.288 | INFO | "huge" : "", | 20260505 17:52:03.288 | INFO | "huge_shared" : " 27 GB", | 20260505 17:52:03.288 | INFO | "malloc" : " 14 GB", | 20260505 17:52:03.288 | INFO | "pinned" : " 29 GB" | 20260505 17:52:03.288 | INFO | } | 20260505 17:52:03.288 | INFO | Perf report for the last 5.15222 seconds: | 20260505 17:52:03.288 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:52:03.288 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:03.288 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:03.288 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:03.288 | INFO | ANALYSIS : 63.5%, 3.0, 99.4ms, 0 | 20260505 17:52:03.288 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:03.288 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:03.288 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 17:52:03.288 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:03.288 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:03.288 | INFO | Uploaded 36GiB to GPU in 2.12961s (16.9045GiB/s) | 20260505 17:52:03.288 | INFO | Downloaded 34GiB to GPU in 2.17511s (15.6314GiB/s) | 20260505 17:52:08.429 | INFO | Analyzing chunk frames = [5144064,5144576) | 20260505 17:52:08.429 | INFO | MemInfo: { | 20260505 17:52:08.429 | INFO | "gpu" : " 17 GB", | 20260505 17:52:08.429 | INFO | "huge" : "", | 20260505 17:52:08.429 | INFO | "huge_shared" : " 27 GB", | 20260505 17:52:08.429 | INFO | "malloc" : " 14 GB", | 20260505 17:52:08.429 | INFO | "pinned" : " 29 GB" | 20260505 17:52:08.429 | INFO | } | 20260505 17:52:08.429 | INFO | Perf report for the last 5.14097 seconds: | 20260505 17:52:08.429 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:52:08.429 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:08.429 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:08.429 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:08.429 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 17:52:08.429 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:08.429 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:08.429 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 17:52:08.429 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:08.429 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:08.429 | INFO | Uploaded 36GiB to GPU in 2.19918s (16.3698GiB/s) | 20260505 17:52:08.429 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 17:52:08.429 | INFO | Total mem usage: 15.3 GiB | 20260505 17:52:08.429 | INFO | Downloaded 34GiB to GPU in 2.02293s (16.8073GiB/s) | 20260505 17:52:08.429 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 17:52:08.429 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.99%) | 20260505 17:52:08.429 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 17:52:08.429 | INFO | High Water Breakdown: | 20260505 17:52:08.429 | INFO | 61% : 8.66 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 17:52:08.429 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 17:52:08.429 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 17:52:08.429 | INFO | ------------------------------------------ | 20260505 17:52:08.429 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:52:08.429 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 17:52:08.429 | INFO | Total mem usage: 26.5 GiB | 20260505 17:52:08.430 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 17:52:08.430 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 17:52:08.430 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 17:52:08.430 | INFO | High Water Breakdown: | 20260505 17:52:08.430 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:52:08.430 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:52:08.430 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:52:08.430 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:52:08.430 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:52:08.430 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:52:08.430 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 17:52:08.430 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 17:52:08.430 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:52:08.430 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 17:52:08.430 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 17:52:08.430 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 17:52:08.430 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 17:52:08.430 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 17:52:08.430 | INFO | ------------------------------------------ | 20260505 17:52:08.430 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:52:08.430 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 17:52:08.430 | INFO | Total mem usage: 25.6 GiB | 20260505 17:52:08.430 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 17:52:08.430 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 17:52:08.430 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 17:52:08.430 | INFO | High Water Breakdown: | 20260505 17:52:08.430 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 17:52:08.430 | INFO | ------------------------------------------ | 20260505 17:52:08.430 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:52:08.430 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 17:52:08.430 | INFO | Total mem usage: 15.7 GiB | 20260505 17:52:08.430 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 17:52:08.430 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 17:52:08.430 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 17:52:08.430 | INFO | High Water Breakdown: | 20260505 17:52:08.430 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 17:52:08.430 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 17:52:08.430 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 17:52:08.430 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 17:52:08.430 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 17:52:08.430 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 17:52:08.430 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 17:52:08.430 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 17:52:08.430 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 17:52:08.430 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:52:08.430 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 17:52:08.430 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:52:08.430 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 17:52:08.430 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 17:52:08.430 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 17:52:08.430 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 17:52:08.430 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:52:08.430 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 17:52:08.430 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:52:08.430 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:52:08.430 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:52:08.430 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 17:52:08.430 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 17:52:08.430 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:52:08.430 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 17:52:08.430 | INFO | ------------------------------------------ | 20260505 17:52:08.430 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:52:08.430 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 17:52:08.430 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:52:08.430 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:52:08.430 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.48e+01, 1.36e+01 / 2.29e+00 | 20260505 17:52:08.430 | INFO | PerfReport: 0.29%, AggregateStats -- 9.27e-02 / 1.03e-01, 1.14e-02 / 5.11e-03 | 20260505 17:52:08.430 | INFO | PerfReport: 11.77%, Baseline -- 3.72e+00 / 4.50e+00, 5.66e-01 / 1.31e-01 | 20260505 17:52:08.430 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.49e+00, 1.94e-01 / 1.05e-01 | 20260505 17:52:08.430 | INFO | PerfReport: 15.95%, DME -- 4.03e+01 / 4.48e+01, 4.22e+00 / 2.29e+00 | 20260505 17:52:08.430 | INFO | PerfReport: 56.87%, FrameLabeling -- 1.80e+01 / 1.92e+01, 1.50e+00 / 1.02e+00 | 20260505 17:52:08.430 | INFO | PerfReport: 7.32%, PulseAccumulating -- 2.31e+00 / 2.94e+00, 4.46e-01 / 1.10e-01 | 20260505 17:52:08.430 | INFO | PerfReport: 3.35%, Metrics -- 1.06e+00 / 1.39e+00, 2.92e-01 / 1.57e-01 | 20260505 17:52:08.430 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.70e-01 / 4.30e-01, 6.54e-02 / 4.27e-02 | 20260505 17:52:08.430 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 17:52:08.430 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:52:08.430 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:52:08.430 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.58e+01 / 0.00e+00 | 20260505 17:52:08.430 | INFO | PerfReport: 22.53%, Upload -- 2.25e+01 / 0.00e+00, 7.86e+00 / 0.00e+00 | 20260505 17:52:08.430 | INFO | PerfReport: 21.75%, Download -- 2.17e+01 / 0.00e+00, 7.77e+00 / 0.00e+00 | 20260505 17:52:08.430 | INFO | PerfReport: 31.62%, Compute -- 3.16e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 17:52:08.430 | INFO | PerfReport: 2.36%, LockUpload -- 2.36e+00 / 0.00e+00, 8.03e+00 / 0.00e+00 | 20260505 17:52:08.430 | INFO | PerfReport: 1.36%, LockDownload -- 1.36e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260505 17:52:08.430 | INFO | PerfReport: 20.37%, LockCompute -- 2.04e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260505 17:52:08.430 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 1.98e-02 / 0.00e+00 | 20260505 17:52:08.560 | INFO | Starting a round of Baz IO | 20260505 17:52:11.740 | INFO | Populating RTMetrics report | 20260505 17:52:11.741 | INFO | RTMetrics report populated | 20260505 17:52:11.741 | INFO | Opening tmp RTMetrics json file | 20260505 17:52:11.742 | INFO | Serializing RTMetrics to json | 20260505 17:52:11.744 | INFO | Serialization done, swapping tmp file now... | 20260505 17:52:11.745 | INFO | RTMetrics done | 20260505 17:52:11.830 | INFO | Finished a round of Baz IO: Wrote 2.115 GiB in 3.27 seconds | 20260505 17:52:11.830 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.99% / 0.00% / 78.01% | 20260505 17:52:11.830 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.277778% / 0% / 9.53674e-05% | 20260505 17:52:11.830 | INFO | Baz IO overall: | 20260505 17:52:11.830 | INFO | BytesWritten : 3036546600213 | 20260505 17:52:11.830 | INFO | NumEvents : 1439327716388 | 20260505 17:52:13.563 | INFO | Analyzing chunk frames = [5144576,5145088) | 20260505 17:52:13.563 | INFO | MemInfo: { | 20260505 17:52:13.563 | INFO | "gpu" : " 17 GB", | 20260505 17:52:13.563 | INFO | "huge" : "", | 20260505 17:52:13.563 | INFO | "huge_shared" : " 27 GB", | 20260505 17:52:13.563 | INFO | "malloc" : " 14 GB", | 20260505 17:52:13.563 | INFO | "pinned" : " 29 GB" | 20260505 17:52:13.563 | INFO | } | 20260505 17:52:13.563 | INFO | Perf report for the last 5.13427 seconds: | 20260505 17:52:13.563 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:52:13.563 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:13.563 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:13.563 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:13.563 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 17:52:13.563 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:13.563 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260505 17:52:13.563 | INFO | PRE_HQ : 60.4%, 1.8, 56.1ms, 0 | 20260505 17:52:13.563 | INFO | BAZWRITER : 63.2%, 3.8, 123.4ms, 0 | 20260505 17:52:13.563 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 17:52:13.563 | INFO | Uploaded 36GiB to GPU in 2.1696s (16.5929GiB/s) | 20260505 17:52:13.563 | INFO | Downloaded 38GiB to GPU in 2.24907s (16.8959GiB/s) | 20260505 17:52:18.697 | INFO | Analyzing chunk frames = [5145088,5145600) | 20260505 17:52:18.697 | INFO | MemInfo: { | 20260505 17:52:18.697 | INFO | "gpu" : " 17 GB", | 20260505 17:52:18.697 | INFO | "huge" : "", | 20260505 17:52:18.697 | INFO | "huge_shared" : " 27 GB", | 20260505 17:52:18.697 | INFO | "malloc" : " 14 GB", | 20260505 17:52:18.697 | INFO | "pinned" : " 29 GB" | 20260505 17:52:18.697 | INFO | } | 20260505 17:52:18.698 | INFO | Perf report for the last 5.1343 seconds: | 20260505 17:52:18.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:52:18.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:18.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:18.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:18.698 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260505 17:52:18.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:18.698 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:18.698 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 17:52:18.698 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:18.698 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:18.698 | INFO | Uploaded 36GiB to GPU in 2.14133s (16.812GiB/s) | 20260505 17:52:18.698 | INFO | Downloaded 34GiB to GPU in 2.03118s (16.7391GiB/s) | 20260505 17:52:23.842 | INFO | Analyzing chunk frames = [5145600,5146112) | 20260505 17:52:23.842 | INFO | MemInfo: { | 20260505 17:52:23.842 | INFO | "gpu" : " 17 GB", | 20260505 17:52:23.842 | INFO | "huge" : "", | 20260505 17:52:23.842 | INFO | "huge_shared" : " 27 GB", | 20260505 17:52:23.842 | INFO | "malloc" : " 14 GB", | 20260505 17:52:23.842 | INFO | "pinned" : " 29 GB" | 20260505 17:52:23.842 | INFO | } | 20260505 17:52:23.842 | INFO | Perf report for the last 5.14453 seconds: | 20260505 17:52:23.842 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:52:23.842 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:23.842 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:23.842 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:23.842 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260505 17:52:23.842 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:23.842 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:23.842 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 17:52:23.842 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:23.842 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:23.842 | INFO | Uploaded 36GiB to GPU in 2.20061s (16.3591GiB/s) | 20260505 17:52:23.842 | INFO | Downloaded 34GiB to GPU in 2.04021s (16.6649GiB/s) | 20260505 17:52:28.983 | INFO | Analyzing chunk frames = [5146112,5146624) | 20260505 17:52:28.983 | INFO | MemInfo: { | 20260505 17:52:28.983 | INFO | "gpu" : " 17 GB", | 20260505 17:52:28.983 | INFO | "huge" : "", | 20260505 17:52:28.983 | INFO | "huge_shared" : " 27 GB", | 20260505 17:52:28.983 | INFO | "malloc" : " 14 GB", | 20260505 17:52:28.983 | INFO | "pinned" : " 29 GB" | 20260505 17:52:28.983 | INFO | } | 20260505 17:52:28.983 | INFO | Perf report for the last 5.14092 seconds: | 20260505 17:52:28.983 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:52:28.983 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:28.983 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:28.983 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:28.983 | INFO | ANALYSIS : 62.4%, 3.0, 97.0ms, 0 | 20260505 17:52:28.983 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:28.983 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:28.983 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:52:28.983 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:28.983 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:28.983 | INFO | Uploaded 36GiB to GPU in 2.18201s (16.4986GiB/s) | 20260505 17:52:28.983 | INFO | Downloaded 34GiB to GPU in 2.06739s (16.4458GiB/s) | 20260505 17:52:34.129 | INFO | Analyzing chunk frames = [5146624,5147136) | 20260505 17:52:34.129 | INFO | MemInfo: { | 20260505 17:52:34.129 | INFO | "gpu" : " 17 GB", | 20260505 17:52:34.129 | INFO | "huge" : "", | 20260505 17:52:34.129 | INFO | "huge_shared" : " 27 GB", | 20260505 17:52:34.129 | INFO | "malloc" : " 14 GB", | 20260505 17:52:34.129 | INFO | "pinned" : " 29 GB" | 20260505 17:52:34.129 | INFO | } | 20260505 17:52:34.129 | INFO | Perf report for the last 5.1461 seconds: | 20260505 17:52:34.129 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:52:34.129 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:34.129 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:34.129 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:34.129 | INFO | ANALYSIS : 61.9%, 3.0, 96.8ms, 0 | 20260505 17:52:34.129 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:34.129 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:34.129 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:52:34.129 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:34.129 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:34.129 | INFO | Uploaded 36GiB to GPU in 2.14746s (16.764GiB/s) | 20260505 17:52:34.129 | INFO | Downloaded 34GiB to GPU in 2.05753s (16.5247GiB/s) | 20260505 17:52:39.261 | INFO | Analyzing chunk frames = [5147136,5147648) | 20260505 17:52:39.261 | INFO | MemInfo: { | 20260505 17:52:39.261 | INFO | "gpu" : " 17 GB", | 20260505 17:52:39.261 | INFO | "huge" : "", | 20260505 17:52:39.261 | INFO | "huge_shared" : " 27 GB", | 20260505 17:52:39.261 | INFO | "malloc" : " 14 GB", | 20260505 17:52:39.261 | INFO | "pinned" : " 29 GB" | 20260505 17:52:39.261 | INFO | } | 20260505 17:52:39.261 | INFO | Perf report for the last 5.13189 seconds: | 20260505 17:52:39.261 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:52:39.261 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:39.261 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:39.261 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:39.261 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 17:52:39.261 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:39.261 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:39.261 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:52:39.261 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:39.261 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:39.261 | INFO | Uploaded 36GiB to GPU in 2.13525s (16.8599GiB/s) | 20260505 17:52:39.261 | INFO | Downloaded 34GiB to GPU in 2.04639s (16.6146GiB/s) | 20260505 17:52:44.394 | INFO | Analyzing chunk frames = [5147648,5148160) | 20260505 17:52:44.394 | INFO | MemInfo: { | 20260505 17:52:44.394 | INFO | "gpu" : " 17 GB", | 20260505 17:52:44.394 | INFO | "huge" : "", | 20260505 17:52:44.394 | INFO | "huge_shared" : " 27 GB", | 20260505 17:52:44.394 | INFO | "malloc" : " 14 GB", | 20260505 17:52:44.395 | INFO | "pinned" : " 29 GB" | 20260505 17:52:44.395 | INFO | } | 20260505 17:52:44.395 | INFO | Perf report for the last 5.13354 seconds: | 20260505 17:52:44.395 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:52:44.395 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:44.395 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:44.395 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:44.395 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 17:52:44.395 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:44.395 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:44.395 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 17:52:44.395 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:44.395 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:44.395 | INFO | Uploaded 36GiB to GPU in 2.15108s (16.7358GiB/s) | 20260505 17:52:44.395 | INFO | Downloaded 34GiB to GPU in 2.17163s (15.6564GiB/s) | 20260505 17:52:49.546 | INFO | Analyzing chunk frames = [5148160,5148672) | 20260505 17:52:49.546 | INFO | MemInfo: { | 20260505 17:52:49.546 | INFO | "gpu" : " 17 GB", | 20260505 17:52:49.546 | INFO | "huge" : "", | 20260505 17:52:49.546 | INFO | "huge_shared" : " 27 GB", | 20260505 17:52:49.546 | INFO | "malloc" : " 14 GB", | 20260505 17:52:49.546 | INFO | "pinned" : " 29 GB" | 20260505 17:52:49.546 | INFO | } | 20260505 17:52:49.546 | INFO | Perf report for the last 5.1519 seconds: | 20260505 17:52:49.546 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:52:49.546 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:49.546 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:49.546 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:49.547 | INFO | ANALYSIS : 62.8%, 3.0, 98.3ms, 0 | 20260505 17:52:49.547 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:49.547 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:49.547 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 17:52:49.547 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:49.547 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:49.547 | INFO | Uploaded 36GiB to GPU in 2.17044s (16.5865GiB/s) | 20260505 17:52:49.547 | INFO | Downloaded 34GiB to GPU in 2.02777s (16.7672GiB/s) | 20260505 17:52:49.680 | INFO | Starting a round of Baz IO | 20260505 17:52:52.861 | INFO | Populating RTMetrics report | 20260505 17:52:52.861 | INFO | RTMetrics report populated | 20260505 17:52:52.861 | INFO | Opening tmp RTMetrics json file | 20260505 17:52:52.870 | INFO | Serializing RTMetrics to json | 20260505 17:52:52.873 | INFO | Serialization done, swapping tmp file now... | 20260505 17:52:52.873 | INFO | RTMetrics done | 20260505 17:52:52.954 | INFO | Finished a round of Baz IO: Wrote 2.114 GiB in 3.273 seconds | 20260505 17:52:52.954 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.99% / 0.00% / 78.01% | 20260505 17:52:52.954 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.277384% / 0% / 0.000119209% | 20260505 17:52:52.954 | INFO | Baz IO overall: | 20260505 17:52:52.954 | INFO | BytesWritten : 3038816895351 | 20260505 17:52:52.954 | INFO | NumEvents : 1440387634745 | 20260505 17:52:54.678 | INFO | Analyzing chunk frames = [5148672,5149184) | 20260505 17:52:54.678 | INFO | MemInfo: { | 20260505 17:52:54.678 | INFO | "gpu" : " 17 GB", | 20260505 17:52:54.678 | INFO | "huge" : "", | 20260505 17:52:54.678 | INFO | "huge_shared" : " 27 GB", | 20260505 17:52:54.678 | INFO | "malloc" : " 14 GB", | 20260505 17:52:54.678 | INFO | "pinned" : " 29 GB" | 20260505 17:52:54.678 | INFO | } | 20260505 17:52:54.678 | INFO | Perf report for the last 5.13173 seconds: | 20260505 17:52:54.678 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:52:54.678 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:54.678 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:54.678 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:54.678 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260505 17:52:54.678 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:54.678 | INFO | RT_METRICS : 44.7%, 1.0, 23.4ms, 0 | 20260505 17:52:54.678 | INFO | PRE_HQ : 60.6%, 1.8, 56.0ms, 0 | 20260505 17:52:54.678 | INFO | BAZWRITER : 63.3%, 3.8, 123.1ms, 0 | 20260505 17:52:54.678 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 17:52:54.678 | INFO | Uploaded 36GiB to GPU in 2.16386s (16.6369GiB/s) | 20260505 17:52:54.678 | INFO | Downloaded 38GiB to GPU in 2.24828s (16.9018GiB/s) | 20260505 17:52:59.820 | INFO | Analyzing chunk frames = [5149184,5149696) | 20260505 17:52:59.820 | INFO | MemInfo: { | 20260505 17:52:59.820 | INFO | "gpu" : " 17 GB", | 20260505 17:52:59.820 | INFO | "huge" : "", | 20260505 17:52:59.820 | INFO | "huge_shared" : " 27 GB", | 20260505 17:52:59.820 | INFO | "malloc" : " 14 GB", | 20260505 17:52:59.820 | INFO | "pinned" : " 29 GB" | 20260505 17:52:59.820 | INFO | } | 20260505 17:52:59.820 | INFO | Perf report for the last 5.14183 seconds: | 20260505 17:52:59.820 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:52:59.820 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:59.820 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:59.820 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:59.820 | INFO | ANALYSIS : 62.7%, 3.0, 98.0ms, 0 | 20260505 17:52:59.820 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:59.820 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:59.820 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260505 17:52:59.820 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:59.820 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:52:59.820 | INFO | Uploaded 36GiB to GPU in 2.1346s (16.865GiB/s) | 20260505 17:52:59.820 | INFO | Downloaded 34GiB to GPU in 2.03039s (16.7455GiB/s) | 20260505 17:53:04.959 | INFO | Analyzing chunk frames = [5149696,5150208) | 20260505 17:53:04.959 | INFO | MemInfo: { | 20260505 17:53:04.959 | INFO | "gpu" : " 17 GB", | 20260505 17:53:04.959 | INFO | "huge" : "", | 20260505 17:53:04.959 | INFO | "huge_shared" : " 27 GB", | 20260505 17:53:04.959 | INFO | "malloc" : " 14 GB", | 20260505 17:53:04.959 | INFO | "pinned" : " 29 GB" | 20260505 17:53:04.959 | INFO | } | 20260505 17:53:04.959 | INFO | Perf report for the last 5.13927 seconds: | 20260505 17:53:04.959 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:53:04.959 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:04.959 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:04.959 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:04.959 | INFO | ANALYSIS : 62.8%, 3.0, 97.6ms, 0 | 20260505 17:53:04.959 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:04.959 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:04.959 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 17:53:04.960 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:04.960 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:04.960 | INFO | Uploaded 36GiB to GPU in 2.21052s (16.2858GiB/s) | 20260505 17:53:04.960 | INFO | Downloaded 34GiB to GPU in 2.04944s (16.5899GiB/s) | 20260505 17:53:10.093 | INFO | Analyzing chunk frames = [5150208,5150720) | 20260505 17:53:10.093 | INFO | MemInfo: { | 20260505 17:53:10.093 | INFO | "gpu" : " 17 GB", | 20260505 17:53:10.093 | INFO | "huge" : "", | 20260505 17:53:10.093 | INFO | "huge_shared" : " 27 GB", | 20260505 17:53:10.093 | INFO | "malloc" : " 14 GB", | 20260505 17:53:10.093 | INFO | "pinned" : " 29 GB" | 20260505 17:53:10.093 | INFO | } | 20260505 17:53:10.093 | INFO | Perf report for the last 5.13396 seconds: | 20260505 17:53:10.093 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:53:10.093 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:10.093 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:10.093 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:10.093 | INFO | ANALYSIS : 62.8%, 3.0, 97.4ms, 0 | 20260505 17:53:10.093 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:10.093 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:10.093 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:53:10.093 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:10.094 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:10.094 | INFO | Uploaded 36GiB to GPU in 2.22303s (16.1941GiB/s) | 20260505 17:53:10.094 | INFO | Downloaded 34GiB to GPU in 2.12803s (15.9772GiB/s) | 20260505 17:53:15.234 | INFO | Analyzing chunk frames = [5150720,5151232) | 20260505 17:53:15.234 | INFO | MemInfo: { | 20260505 17:53:15.234 | INFO | "gpu" : " 17 GB", | 20260505 17:53:15.234 | INFO | "huge" : "", | 20260505 17:53:15.234 | INFO | "huge_shared" : " 27 GB", | 20260505 17:53:15.234 | INFO | "malloc" : " 14 GB", | 20260505 17:53:15.234 | INFO | "pinned" : " 29 GB" | 20260505 17:53:15.234 | INFO | } | 20260505 17:53:15.235 | INFO | Perf report for the last 5.1412 seconds: | 20260505 17:53:15.235 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:53:15.235 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:15.235 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:15.235 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:15.235 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 17:53:15.235 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:15.235 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:15.235 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:53:15.235 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:15.235 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:15.235 | INFO | Uploaded 36GiB to GPU in 2.13087s (16.8945GiB/s) | 20260505 17:53:15.235 | INFO | Downloaded 34GiB to GPU in 2.05548s (16.5411GiB/s) | 20260505 17:53:20.376 | INFO | Analyzing chunk frames = [5151232,5151744) | 20260505 17:53:20.376 | INFO | MemInfo: { | 20260505 17:53:20.376 | INFO | "gpu" : " 17 GB", | 20260505 17:53:20.376 | INFO | "huge" : "", | 20260505 17:53:20.376 | INFO | "huge_shared" : " 27 GB", | 20260505 17:53:20.376 | INFO | "malloc" : " 14 GB", | 20260505 17:53:20.376 | INFO | "pinned" : " 29 GB" | 20260505 17:53:20.376 | INFO | } | 20260505 17:53:20.376 | INFO | Perf report for the last 5.14148 seconds: | 20260505 17:53:20.376 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:53:20.376 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:20.376 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:20.376 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:20.376 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260505 17:53:20.376 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:20.376 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:20.376 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 17:53:20.376 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:20.376 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:20.376 | INFO | Uploaded 36GiB to GPU in 2.1364s (16.8507GiB/s) | 20260505 17:53:20.376 | INFO | Downloaded 34GiB to GPU in 2.04591s (16.6185GiB/s) | 20260505 17:53:25.508 | INFO | Analyzing chunk frames = [5151744,5152256) | 20260505 17:53:25.508 | INFO | MemInfo: { | 20260505 17:53:25.508 | INFO | "gpu" : " 17 GB", | 20260505 17:53:25.508 | INFO | "huge" : "", | 20260505 17:53:25.508 | INFO | "huge_shared" : " 27 GB", | 20260505 17:53:25.509 | INFO | "malloc" : " 14 GB", | 20260505 17:53:25.509 | INFO | "pinned" : " 29 GB" | 20260505 17:53:25.509 | INFO | } | 20260505 17:53:25.509 | INFO | Perf report for the last 5.1326 seconds: | 20260505 17:53:25.509 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:53:25.509 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:25.509 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:25.509 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:25.509 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 17:53:25.509 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:25.509 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:25.509 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 17:53:25.509 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:25.509 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:25.509 | INFO | Uploaded 36GiB to GPU in 2.14195s (16.8071GiB/s) | 20260505 17:53:25.509 | INFO | Downloaded 34GiB to GPU in 2.14162s (15.8759GiB/s) | 20260505 17:53:30.646 | INFO | Analyzing chunk frames = [5152256,5152768) | 20260505 17:53:30.646 | INFO | MemInfo: { | 20260505 17:53:30.646 | INFO | "gpu" : " 17 GB", | 20260505 17:53:30.646 | INFO | "huge" : "", | 20260505 17:53:30.646 | INFO | "huge_shared" : " 27 GB", | 20260505 17:53:30.646 | INFO | "malloc" : " 14 GB", | 20260505 17:53:30.646 | INFO | "pinned" : " 29 GB" | 20260505 17:53:30.646 | INFO | } | 20260505 17:53:30.646 | INFO | Perf report for the last 5.13773 seconds: | 20260505 17:53:30.646 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:53:30.646 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:30.646 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:30.646 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:30.646 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 17:53:30.646 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:30.646 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:30.646 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:53:30.647 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:30.647 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:30.647 | INFO | Uploaded 36GiB to GPU in 2.17159s (16.5777GiB/s) | 20260505 17:53:30.647 | INFO | Downloaded 34GiB to GPU in 2.03831s (16.6805GiB/s) | 20260505 17:53:30.647 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 17:53:30.647 | INFO | Total mem usage: 15.3 GiB | 20260505 17:53:30.647 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 17:53:30.647 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.99%) | 20260505 17:53:30.647 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 17:53:30.647 | INFO | High Water Breakdown: | 20260505 17:53:30.647 | INFO | 61% : 8.66 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 17:53:30.647 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 17:53:30.647 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 17:53:30.647 | INFO | ------------------------------------------ | 20260505 17:53:30.647 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:53:30.647 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 17:53:30.647 | INFO | Total mem usage: 26.5 GiB | 20260505 17:53:30.647 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 17:53:30.647 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 17:53:30.647 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 17:53:30.647 | INFO | High Water Breakdown: | 20260505 17:53:30.647 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:53:30.647 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:53:30.647 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:53:30.647 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:53:30.647 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:53:30.647 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:53:30.647 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 17:53:30.647 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 17:53:30.647 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:53:30.647 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 17:53:30.647 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 17:53:30.647 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 17:53:30.647 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 17:53:30.647 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 17:53:30.647 | INFO | ------------------------------------------ | 20260505 17:53:30.647 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:53:30.647 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 17:53:30.647 | INFO | Total mem usage: 25.6 GiB | 20260505 17:53:30.647 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 17:53:30.647 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 17:53:30.647 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 17:53:30.647 | INFO | High Water Breakdown: | 20260505 17:53:30.647 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 17:53:30.647 | INFO | ------------------------------------------ | 20260505 17:53:30.647 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:53:30.647 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 17:53:30.647 | INFO | Total mem usage: 15.7 GiB | 20260505 17:53:30.647 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 17:53:30.647 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 17:53:30.647 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 17:53:30.647 | INFO | High Water Breakdown: | 20260505 17:53:30.647 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 17:53:30.647 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 17:53:30.647 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 17:53:30.647 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 17:53:30.647 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 17:53:30.647 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 17:53:30.647 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 17:53:30.647 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 17:53:30.647 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 17:53:30.647 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:53:30.647 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 17:53:30.647 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:53:30.647 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 17:53:30.647 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 17:53:30.647 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 17:53:30.647 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 17:53:30.647 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:53:30.647 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 17:53:30.647 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:53:30.647 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:53:30.647 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:53:30.647 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 17:53:30.648 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 17:53:30.648 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:53:30.648 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 17:53:30.648 | INFO | ------------------------------------------ | 20260505 17:53:30.648 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:53:30.648 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 17:53:30.648 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:53:30.648 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:53:30.648 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.48e+01, 1.36e+01 / 2.17e+00 | 20260505 17:53:30.648 | INFO | PerfReport: 0.29%, AggregateStats -- 9.26e-02 / 1.02e-01, 1.13e-02 / 3.49e-03 | 20260505 17:53:30.648 | INFO | PerfReport: 11.79%, Baseline -- 3.73e+00 / 4.51e+00, 5.78e-01 / 1.58e-01 | 20260505 17:53:30.648 | INFO | PerfReport: 3.26%, Histogram -- 1.03e+00 / 1.51e+00, 1.89e-01 / 8.74e-02 | 20260505 17:53:30.648 | INFO | PerfReport: 16.00%, DME -- 4.04e+01 / 4.50e+01, 4.29e+00 / 2.26e+00 | 20260505 17:53:30.648 | INFO | PerfReport: 56.78%, FrameLabeling -- 1.79e+01 / 1.90e+01, 1.43e+00 / 9.37e-01 | 20260505 17:53:30.648 | INFO | PerfReport: 7.35%, PulseAccumulating -- 2.32e+00 / 2.92e+00, 4.51e-01 / 1.72e-01 | 20260505 17:53:30.648 | INFO | PerfReport: 3.36%, Metrics -- 1.06e+00 / 1.39e+00, 2.94e-01 / 1.80e-01 | 20260505 17:53:30.648 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.70e-01 / 4.31e-01, 6.54e-02 / 4.31e-02 | 20260505 17:53:30.648 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 17:53:30.648 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:53:30.648 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:53:30.648 | INFO | PerfReport: Full Execution -- 9.99e+01 / -nan, 2.58e+01 / 0.00e+00 | 20260505 17:53:30.648 | INFO | PerfReport: 22.55%, Upload -- 2.25e+01 / 0.00e+00, 7.91e+00 / 0.00e+00 | 20260505 17:53:30.648 | INFO | PerfReport: 21.78%, Download -- 2.18e+01 / 0.00e+00, 7.79e+00 / 0.00e+00 | 20260505 17:53:30.648 | INFO | PerfReport: 31.62%, Compute -- 3.16e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 17:53:30.648 | INFO | PerfReport: 2.34%, LockUpload -- 2.34e+00 / 0.00e+00, 8.02e+00 / 0.00e+00 | 20260505 17:53:30.648 | INFO | PerfReport: 1.38%, LockDownload -- 1.38e+00 / 0.00e+00, 2.86e+00 / 0.00e+00 | 20260505 17:53:30.648 | INFO | PerfReport: 20.31%, LockCompute -- 2.03e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 17:53:30.648 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.16e-02 / 0.00e+00 | 20260505 17:53:30.780 | INFO | Starting a round of Baz IO | 20260505 17:53:33.978 | INFO | Populating RTMetrics report | 20260505 17:53:33.978 | INFO | RTMetrics report populated | 20260505 17:53:33.978 | INFO | Opening tmp RTMetrics json file | 20260505 17:53:33.980 | INFO | Serializing RTMetrics to json | 20260505 17:53:33.981 | INFO | Serialization done, swapping tmp file now... | 20260505 17:53:33.982 | INFO | RTMetrics done | 20260505 17:53:34.070 | INFO | Finished a round of Baz IO: Wrote 2.115 GiB in 3.289 seconds | 20260505 17:53:34.070 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.99% / 0.00% / 78.01% | 20260505 17:53:34.070 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.277448% / 7.94729e-06% / 8.74201e-05% | 20260505 17:53:34.070 | INFO | Baz IO overall: | 20260505 17:53:34.070 | INFO | BytesWritten : 3041088114390 | 20260505 17:53:34.070 | INFO | NumEvents : 1441448021732 | 20260505 17:53:35.800 | INFO | Analyzing chunk frames = [5152768,5153280) | 20260505 17:53:35.800 | INFO | MemInfo: { | 20260505 17:53:35.800 | INFO | "gpu" : " 17 GB", | 20260505 17:53:35.800 | INFO | "huge" : "", | 20260505 17:53:35.800 | INFO | "huge_shared" : " 27 GB", | 20260505 17:53:35.800 | INFO | "malloc" : " 14 GB", | 20260505 17:53:35.800 | INFO | "pinned" : " 29 GB" | 20260505 17:53:35.800 | INFO | } | 20260505 17:53:35.800 | INFO | Perf report for the last 5.15379 seconds: | 20260505 17:53:35.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:53:35.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:35.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:35.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:35.800 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 17:53:35.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:35.800 | INFO | RT_METRICS : 44.3%, 1.0, 23.3ms, 0 | 20260505 17:53:35.800 | INFO | PRE_HQ : 60.2%, 1.8, 55.9ms, 0 | 20260505 17:53:35.800 | INFO | BAZWRITER : 63.2%, 3.8, 123.6ms, 0 | 20260505 17:53:35.800 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 17:53:35.801 | INFO | Uploaded 36GiB to GPU in 2.16878s (16.5992GiB/s) | 20260505 17:53:35.801 | INFO | Downloaded 38GiB to GPU in 2.26079s (16.8083GiB/s) | 20260505 17:53:40.939 | INFO | Analyzing chunk frames = [5153280,5153792) | 20260505 17:53:40.939 | INFO | MemInfo: { | 20260505 17:53:40.939 | INFO | "gpu" : " 17 GB", | 20260505 17:53:40.939 | INFO | "huge" : "", | 20260505 17:53:40.939 | INFO | "huge_shared" : " 27 GB", | 20260505 17:53:40.939 | INFO | "malloc" : " 14 GB", | 20260505 17:53:40.939 | INFO | "pinned" : " 29 GB" | 20260505 17:53:40.939 | INFO | } | 20260505 17:53:40.939 | INFO | Perf report for the last 5.13929 seconds: | 20260505 17:53:40.939 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:53:40.939 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:40.939 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:40.939 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:40.939 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260505 17:53:40.940 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:40.940 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:40.940 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 17:53:40.940 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:40.940 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:40.940 | INFO | Uploaded 36GiB to GPU in 2.08368s (17.2771GiB/s) | 20260505 17:53:40.940 | INFO | Downloaded 34GiB to GPU in 2.02529s (16.7877GiB/s) | 20260505 17:53:46.072 | INFO | Analyzing chunk frames = [5153792,5154304) | 20260505 17:53:46.072 | INFO | MemInfo: { | 20260505 17:53:46.072 | INFO | "gpu" : " 17 GB", | 20260505 17:53:46.072 | INFO | "huge" : "", | 20260505 17:53:46.072 | INFO | "huge_shared" : " 27 GB", | 20260505 17:53:46.072 | INFO | "malloc" : " 14 GB", | 20260505 17:53:46.072 | INFO | "pinned" : " 29 GB" | 20260505 17:53:46.072 | INFO | } | 20260505 17:53:46.072 | INFO | Perf report for the last 5.13227 seconds: | 20260505 17:53:46.072 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:53:46.072 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:46.072 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:46.072 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:46.072 | INFO | ANALYSIS : 62.4%, 3.0, 96.9ms, 0 | 20260505 17:53:46.072 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:46.072 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:46.072 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:53:46.072 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:46.072 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:46.072 | INFO | Uploaded 36GiB to GPU in 2.12551s (16.9371GiB/s) | 20260505 17:53:46.072 | INFO | Downloaded 34GiB to GPU in 2.02785s (16.7665GiB/s) | 20260505 17:53:51.205 | INFO | Analyzing chunk frames = [5154304,5154816) | 20260505 17:53:51.205 | INFO | MemInfo: { | 20260505 17:53:51.205 | INFO | "gpu" : " 17 GB", | 20260505 17:53:51.205 | INFO | "huge" : "", | 20260505 17:53:51.205 | INFO | "huge_shared" : " 27 GB", | 20260505 17:53:51.205 | INFO | "malloc" : " 14 GB", | 20260505 17:53:51.205 | INFO | "pinned" : " 29 GB" | 20260505 17:53:51.205 | INFO | } | 20260505 17:53:51.205 | INFO | Perf report for the last 5.13346 seconds: | 20260505 17:53:51.205 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:53:51.205 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:51.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:51.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:51.205 | INFO | ANALYSIS : 62.6%, 3.0, 97.1ms, 0 | 20260505 17:53:51.205 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:51.205 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:51.205 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:53:51.205 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:51.205 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:51.205 | INFO | Uploaded 36GiB to GPU in 2.19111s (16.43GiB/s) | 20260505 17:53:51.205 | INFO | Downloaded 34GiB to GPU in 2.13922s (15.8936GiB/s) | 20260505 17:53:56.350 | INFO | Analyzing chunk frames = [5154816,5155328) | 20260505 17:53:56.350 | INFO | MemInfo: { | 20260505 17:53:56.350 | INFO | "gpu" : " 17 GB", | 20260505 17:53:56.350 | INFO | "huge" : "", | 20260505 17:53:56.350 | INFO | "huge_shared" : " 27 GB", | 20260505 17:53:56.350 | INFO | "malloc" : " 14 GB", | 20260505 17:53:56.350 | INFO | "pinned" : " 29 GB" | 20260505 17:53:56.350 | INFO | } | 20260505 17:53:56.350 | INFO | Perf report for the last 5.14499 seconds: | 20260505 17:53:56.350 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:53:56.350 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:56.350 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:56.350 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:56.350 | INFO | ANALYSIS : 62.1%, 3.0, 97.1ms, 0 | 20260505 17:53:56.350 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:56.350 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:56.350 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:53:56.350 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:56.350 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:53:56.350 | INFO | Uploaded 36GiB to GPU in 2.15222s (16.7269GiB/s) | 20260505 17:53:56.350 | INFO | Downloaded 34GiB to GPU in 2.06112s (16.4959GiB/s) | 20260505 17:54:01.490 | INFO | Analyzing chunk frames = [5155328,5155840) | 20260505 17:54:01.490 | INFO | MemInfo: { | 20260505 17:54:01.490 | INFO | "gpu" : " 17 GB", | 20260505 17:54:01.490 | INFO | "huge" : "", | 20260505 17:54:01.490 | INFO | "huge_shared" : " 27 GB", | 20260505 17:54:01.490 | INFO | "malloc" : " 14 GB", | 20260505 17:54:01.490 | INFO | "pinned" : " 29 GB" | 20260505 17:54:01.490 | INFO | } | 20260505 17:54:01.490 | INFO | Perf report for the last 5.13988 seconds: | 20260505 17:54:01.490 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:54:01.490 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:01.490 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:01.490 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:01.490 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260505 17:54:01.490 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:01.490 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:01.490 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:54:01.490 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:01.490 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:01.490 | INFO | Uploaded 36GiB to GPU in 2.14039s (16.8194GiB/s) | 20260505 17:54:01.490 | INFO | Downloaded 34GiB to GPU in 2.05356s (16.5566GiB/s) | 20260505 17:54:06.627 | INFO | Analyzing chunk frames = [5155840,5156352) | 20260505 17:54:06.627 | INFO | MemInfo: { | 20260505 17:54:06.627 | INFO | "gpu" : " 17 GB", | 20260505 17:54:06.627 | INFO | "huge" : "", | 20260505 17:54:06.628 | INFO | "huge_shared" : " 27 GB", | 20260505 17:54:06.628 | INFO | "malloc" : " 14 GB", | 20260505 17:54:06.628 | INFO | "pinned" : " 29 GB" | 20260505 17:54:06.628 | INFO | } | 20260505 17:54:06.628 | INFO | Perf report for the last 5.13749 seconds: | 20260505 17:54:06.628 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:54:06.628 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:06.628 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:06.628 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:06.628 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 17:54:06.628 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:06.628 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:06.628 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 17:54:06.628 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:06.628 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:06.628 | INFO | Uploaded 36GiB to GPU in 2.1382s (16.8366GiB/s) | 20260505 17:54:06.628 | INFO | Downloaded 34GiB to GPU in 2.14895s (15.8217GiB/s) | 20260505 17:54:11.769 | INFO | Analyzing chunk frames = [5156352,5156864) | 20260505 17:54:11.769 | INFO | MemInfo: { | 20260505 17:54:11.769 | INFO | "gpu" : " 17 GB", | 20260505 17:54:11.769 | INFO | "huge" : "", | 20260505 17:54:11.769 | INFO | "huge_shared" : " 27 GB", | 20260505 17:54:11.769 | INFO | "malloc" : " 14 GB", | 20260505 17:54:11.769 | INFO | "pinned" : " 29 GB" | 20260505 17:54:11.769 | INFO | } | 20260505 17:54:11.769 | INFO | Perf report for the last 5.14163 seconds: | 20260505 17:54:11.769 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:54:11.769 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:11.769 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:11.769 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:11.769 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 17:54:11.769 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:11.769 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:11.769 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:54:11.769 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:11.769 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:11.769 | INFO | Uploaded 36GiB to GPU in 2.19031s (16.436GiB/s) | 20260505 17:54:11.770 | INFO | Downloaded 34GiB to GPU in 2.02149s (16.8193GiB/s) | 20260505 17:54:11.905 | INFO | Starting a round of Baz IO | 20260505 17:54:15.086 | INFO | Populating RTMetrics report | 20260505 17:54:15.087 | INFO | RTMetrics report populated | 20260505 17:54:15.087 | INFO | Opening tmp RTMetrics json file | 20260505 17:54:15.097 | INFO | Serializing RTMetrics to json | 20260505 17:54:15.098 | INFO | Serialization done, swapping tmp file now... | 20260505 17:54:15.099 | INFO | RTMetrics done | 20260505 17:54:15.179 | INFO | Finished a round of Baz IO: Wrote 2.113 GiB in 3.273 seconds | 20260505 17:54:15.179 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.99% / 0.00% / 78.01% | 20260505 17:54:15.179 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.276963% / 3.97364e-06% / 8.74201e-05% | 20260505 17:54:15.179 | INFO | Baz IO overall: | 20260505 17:54:15.179 | INFO | BytesWritten : 3043357174657 | 20260505 17:54:15.179 | INFO | NumEvents : 1442506823949 | 20260505 17:54:16.906 | INFO | Analyzing chunk frames = [5156864,5157376) | 20260505 17:54:16.906 | INFO | MemInfo: { | 20260505 17:54:16.906 | INFO | "gpu" : " 17 GB", | 20260505 17:54:16.906 | INFO | "huge" : "", | 20260505 17:54:16.906 | INFO | "huge_shared" : " 27 GB", | 20260505 17:54:16.906 | INFO | "malloc" : " 14 GB", | 20260505 17:54:16.906 | INFO | "pinned" : " 29 GB" | 20260505 17:54:16.906 | INFO | } | 20260505 17:54:16.906 | INFO | Perf report for the last 5.13666 seconds: | 20260505 17:54:16.906 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:54:16.906 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:16.906 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:16.906 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:16.906 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 17:54:16.906 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:16.906 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260505 17:54:16.906 | INFO | PRE_HQ : 60.6%, 1.8, 56.7ms, 0 | 20260505 17:54:16.906 | INFO | BAZWRITER : 63.1%, 3.9, 125.1ms, 0 | 20260505 17:54:16.906 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 17:54:16.906 | INFO | Uploaded 36GiB to GPU in 2.15774s (16.6842GiB/s) | 20260505 17:54:16.906 | INFO | Downloaded 38GiB to GPU in 2.25354s (16.8624GiB/s) | 20260505 17:54:22.051 | INFO | Analyzing chunk frames = [5157376,5157888) | 20260505 17:54:22.051 | INFO | MemInfo: { | 20260505 17:54:22.051 | INFO | "gpu" : " 17 GB", | 20260505 17:54:22.051 | INFO | "huge" : "", | 20260505 17:54:22.051 | INFO | "huge_shared" : " 27 GB", | 20260505 17:54:22.051 | INFO | "malloc" : " 14 GB", | 20260505 17:54:22.051 | INFO | "pinned" : " 29 GB" | 20260505 17:54:22.051 | INFO | } | 20260505 17:54:22.051 | INFO | Perf report for the last 5.14494 seconds: | 20260505 17:54:22.051 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:54:22.051 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:22.051 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:22.051 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:22.051 | INFO | ANALYSIS : 62.7%, 3.0, 98.0ms, 0 | 20260505 17:54:22.051 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:22.051 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:22.051 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:54:22.051 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:22.051 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:22.051 | INFO | Uploaded 36GiB to GPU in 2.12388s (16.9501GiB/s) | 20260505 17:54:22.051 | INFO | Downloaded 34GiB to GPU in 2.03212s (16.7313GiB/s) | 20260505 17:54:27.186 | INFO | Analyzing chunk frames = [5157888,5158400) | 20260505 17:54:27.186 | INFO | MemInfo: { | 20260505 17:54:27.186 | INFO | "gpu" : " 17 GB", | 20260505 17:54:27.186 | INFO | "huge" : "", | 20260505 17:54:27.186 | INFO | "huge_shared" : " 27 GB", | 20260505 17:54:27.186 | INFO | "malloc" : " 14 GB", | 20260505 17:54:27.186 | INFO | "pinned" : " 29 GB" | 20260505 17:54:27.186 | INFO | } | 20260505 17:54:27.186 | INFO | Perf report for the last 5.13516 seconds: | 20260505 17:54:27.186 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:54:27.186 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:27.186 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:27.186 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:27.186 | INFO | ANALYSIS : 62.4%, 3.0, 97.0ms, 0 | 20260505 17:54:27.186 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:27.186 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:27.186 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:54:27.186 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:27.186 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:27.186 | INFO | Uploaded 36GiB to GPU in 2.1382s (16.8366GiB/s) | 20260505 17:54:27.186 | INFO | Downloaded 34GiB to GPU in 2.04623s (16.6159GiB/s) | 20260505 17:54:32.323 | INFO | Analyzing chunk frames = [5158400,5158912) | 20260505 17:54:32.323 | INFO | MemInfo: { | 20260505 17:54:32.323 | INFO | "gpu" : " 17 GB", | 20260505 17:54:32.323 | INFO | "huge" : "", | 20260505 17:54:32.323 | INFO | "huge_shared" : " 27 GB", | 20260505 17:54:32.323 | INFO | "malloc" : " 14 GB", | 20260505 17:54:32.323 | INFO | "pinned" : " 29 GB" | 20260505 17:54:32.323 | INFO | } | 20260505 17:54:32.323 | INFO | Perf report for the last 5.13717 seconds: | 20260505 17:54:32.323 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:54:32.323 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:32.323 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:32.323 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:32.323 | INFO | ANALYSIS : 62.9%, 3.0, 97.7ms, 0 | 20260505 17:54:32.323 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:32.323 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:32.323 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 17:54:32.323 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:32.323 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:32.324 | INFO | Uploaded 36GiB to GPU in 2.228s (16.158GiB/s) | 20260505 17:54:32.324 | INFO | Downloaded 34GiB to GPU in 2.12271s (16.0173GiB/s) | 20260505 17:54:37.457 | INFO | Analyzing chunk frames = [5158912,5159424) | 20260505 17:54:37.457 | INFO | MemInfo: { | 20260505 17:54:37.457 | INFO | "gpu" : " 17 GB", | 20260505 17:54:37.457 | INFO | "huge" : "", | 20260505 17:54:37.457 | INFO | "huge_shared" : " 27 GB", | 20260505 17:54:37.457 | INFO | "malloc" : " 14 GB", | 20260505 17:54:37.457 | INFO | "pinned" : " 29 GB" | 20260505 17:54:37.457 | INFO | } | 20260505 17:54:37.457 | INFO | Perf report for the last 5.13416 seconds: | 20260505 17:54:37.457 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:54:37.457 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:37.457 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:37.457 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:37.457 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 17:54:37.457 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:37.457 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:37.457 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:54:37.457 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:37.458 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:37.458 | INFO | Uploaded 36GiB to GPU in 2.16188s (16.6522GiB/s) | 20260505 17:54:37.458 | INFO | Downloaded 34GiB to GPU in 2.06169s (16.4913GiB/s) | 20260505 17:54:42.600 | INFO | Analyzing chunk frames = [5159424,5159936) | 20260505 17:54:42.600 | INFO | MemInfo: { | 20260505 17:54:42.600 | INFO | "gpu" : " 17 GB", | 20260505 17:54:42.600 | INFO | "huge" : "", | 20260505 17:54:42.600 | INFO | "huge_shared" : " 27 GB", | 20260505 17:54:42.600 | INFO | "malloc" : " 14 GB", | 20260505 17:54:42.600 | INFO | "pinned" : " 29 GB" | 20260505 17:54:42.600 | INFO | } | 20260505 17:54:42.600 | INFO | Perf report for the last 5.14302 seconds: | 20260505 17:54:42.600 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:54:42.600 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:42.600 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:42.600 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:42.600 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260505 17:54:42.600 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:42.601 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:42.601 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 17:54:42.601 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:42.601 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:42.601 | INFO | Uploaded 36GiB to GPU in 2.14333s (16.7963GiB/s) | 20260505 17:54:42.601 | INFO | Downloaded 34GiB to GPU in 2.04048s (16.6627GiB/s) | 20260505 17:54:47.744 | INFO | Analyzing chunk frames = [5159936,5160448) | 20260505 17:54:47.744 | INFO | MemInfo: { | 20260505 17:54:47.744 | INFO | "gpu" : " 17 GB", | 20260505 17:54:47.744 | INFO | "huge" : "", | 20260505 17:54:47.744 | INFO | "huge_shared" : " 27 GB", | 20260505 17:54:47.744 | INFO | "malloc" : " 14 GB", | 20260505 17:54:47.744 | INFO | "pinned" : " 29 GB" | 20260505 17:54:47.744 | INFO | } | 20260505 17:54:47.744 | INFO | Perf report for the last 5.1439 seconds: | 20260505 17:54:47.744 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:54:47.744 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:47.744 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:47.744 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:47.744 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 17:54:47.744 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:47.744 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:47.744 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 17:54:47.744 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:47.744 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:47.745 | INFO | Uploaded 36GiB to GPU in 2.15225s (16.7267GiB/s) | 20260505 17:54:47.745 | INFO | Downloaded 34GiB to GPU in 2.18212s (15.5812GiB/s) | 20260505 17:54:52.889 | INFO | Analyzing chunk frames = [5160448,5160960) | 20260505 17:54:52.889 | INFO | MemInfo: { | 20260505 17:54:52.889 | INFO | "gpu" : " 17 GB", | 20260505 17:54:52.889 | INFO | "huge" : "", | 20260505 17:54:52.889 | INFO | "huge_shared" : " 27 GB", | 20260505 17:54:52.889 | INFO | "malloc" : " 14 GB", | 20260505 17:54:52.889 | INFO | "pinned" : " 29 GB" | 20260505 17:54:52.889 | INFO | } | 20260505 17:54:52.889 | INFO | Perf report for the last 5.14496 seconds: | 20260505 17:54:52.889 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:54:52.889 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:52.889 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:52.889 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:52.889 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 17:54:52.889 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:52.889 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:52.889 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 17:54:52.889 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:52.889 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:52.890 | INFO | Uploaded 36GiB to GPU in 2.19748s (16.3824GiB/s) | 20260505 17:54:52.890 | INFO | Downloaded 34GiB to GPU in 2.03474s (16.7098GiB/s) | 20260505 17:54:52.890 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 17:54:52.890 | INFO | Total mem usage: 15.3 GiB | 20260505 17:54:52.890 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 17:54:52.890 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -6.5%) | 20260505 17:54:52.890 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 17:54:52.890 | INFO | High Water Breakdown: | 20260505 17:54:52.890 | INFO | 61.2% : 8.81 GiB ( 92.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 17:54:52.890 | INFO | 38.6% : 5.56 GiB ( 83.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 17:54:52.890 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 17:54:52.890 | INFO | ------------------------------------------ | 20260505 17:54:52.890 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:54:52.890 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 17:54:52.890 | INFO | Total mem usage: 26.5 GiB | 20260505 17:54:52.890 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 17:54:52.890 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 17:54:52.890 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 17:54:52.890 | INFO | High Water Breakdown: | 20260505 17:54:52.890 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:54:52.890 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:54:52.890 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:54:52.890 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:54:52.890 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:54:52.890 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:54:52.890 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 17:54:52.890 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 17:54:52.890 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:54:52.890 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 17:54:52.890 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 17:54:52.890 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 17:54:52.890 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 17:54:52.890 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 17:54:52.890 | INFO | ------------------------------------------ | 20260505 17:54:52.890 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:54:52.890 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 17:54:52.890 | INFO | Total mem usage: 25.6 GiB | 20260505 17:54:52.890 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 17:54:52.890 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 17:54:52.890 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 17:54:52.890 | INFO | High Water Breakdown: | 20260505 17:54:52.890 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 17:54:52.890 | INFO | ------------------------------------------ | 20260505 17:54:52.890 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:54:52.890 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 17:54:52.890 | INFO | Total mem usage: 15.7 GiB | 20260505 17:54:52.890 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 17:54:52.890 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 17:54:52.890 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 17:54:52.890 | INFO | High Water Breakdown: | 20260505 17:54:52.890 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 17:54:52.890 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 17:54:52.890 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 17:54:52.890 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 17:54:52.890 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 17:54:52.890 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 17:54:52.890 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 17:54:52.890 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 17:54:52.890 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 17:54:52.890 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:54:52.890 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 17:54:52.890 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:54:52.890 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 17:54:52.890 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 17:54:52.890 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 17:54:52.890 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 17:54:52.890 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:54:52.890 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 17:54:52.890 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:54:52.890 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:54:52.890 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:54:52.890 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 17:54:52.890 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 17:54:52.890 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:54:52.890 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 17:54:52.890 | INFO | ------------------------------------------ | 20260505 17:54:52.890 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:54:52.890 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 17:54:52.890 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:54:52.890 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:54:52.890 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.48e+01, 1.36e+01 / 1.72e+00 | 20260505 17:54:52.890 | INFO | PerfReport: 0.29%, AggregateStats -- 9.27e-02 / 1.03e-01, 1.11e-02 / 5.57e-03 | 20260505 17:54:52.890 | INFO | PerfReport: 11.77%, Baseline -- 3.72e+00 / 4.53e+00, 5.71e-01 / 1.40e-01 | 20260505 17:54:52.890 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.53e+00, 1.93e-01 / 8.01e-02 | 20260505 17:54:52.890 | INFO | PerfReport: 15.92%, DME -- 4.03e+01 / 4.45e+01, 4.06e+00 / 1.87e+00 | 20260505 17:54:52.890 | INFO | PerfReport: 56.92%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.52e+00 / 1.24e+00 | 20260505 17:54:52.890 | INFO | PerfReport: 7.30%, PulseAccumulating -- 2.31e+00 / 2.89e+00, 4.46e-01 / 1.91e-01 | 20260505 17:54:52.890 | INFO | PerfReport: 3.35%, Metrics -- 1.06e+00 / 1.40e+00, 2.95e-01 / 1.66e-01 | 20260505 17:54:52.890 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.69e-01 / 4.28e-01, 6.58e-02 / 3.37e-02 | 20260505 17:54:52.890 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 17:54:52.890 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:54:52.890 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:54:52.891 | INFO | PerfReport: Full Execution -- 9.99e+01 / -nan, 2.57e+01 / 0.00e+00 | 20260505 17:54:52.891 | INFO | PerfReport: 22.48%, Upload -- 2.25e+01 / 0.00e+00, 7.84e+00 / 0.00e+00 | 20260505 17:54:52.891 | INFO | PerfReport: 21.84%, Download -- 2.18e+01 / 0.00e+00, 7.84e+00 / 0.00e+00 | 20260505 17:54:52.891 | INFO | PerfReport: 31.64%, Compute -- 3.16e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 17:54:52.891 | INFO | PerfReport: 2.30%, LockUpload -- 2.30e+00 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260505 17:54:52.891 | INFO | PerfReport: 1.40%, LockDownload -- 1.40e+00 / 0.00e+00, 3.01e+00 / 0.00e+00 | 20260505 17:54:52.891 | INFO | PerfReport: 20.32%, LockCompute -- 2.03e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260505 17:54:52.891 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.92e-02 / 0.00e+00 | 20260505 17:54:53.022 | INFO | Starting a round of Baz IO | 20260505 17:54:56.202 | INFO | Populating RTMetrics report | 20260505 17:54:56.202 | INFO | RTMetrics report populated | 20260505 17:54:56.202 | INFO | Opening tmp RTMetrics json file | 20260505 17:54:56.204 | INFO | Serializing RTMetrics to json | 20260505 17:54:56.205 | INFO | Serialization done, swapping tmp file now... | 20260505 17:54:56.205 | INFO | RTMetrics done | 20260505 17:54:56.296 | INFO | Finished a round of Baz IO: Wrote 2.113 GiB in 3.274 seconds | 20260505 17:54:56.296 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.99% / 0.00% / 78.01% | 20260505 17:54:56.296 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.275842% / 0% / 0.000143051% | 20260505 17:54:56.296 | INFO | Baz IO overall: | 20260505 17:54:56.296 | INFO | BytesWritten : 3045626371199 | 20260505 17:54:56.296 | INFO | NumEvents : 1443565951188 | 20260505 17:54:58.020 | INFO | Analyzing chunk frames = [5160960,5161472) | 20260505 17:54:58.020 | INFO | MemInfo: { | 20260505 17:54:58.020 | INFO | "gpu" : " 17 GB", | 20260505 17:54:58.020 | INFO | "huge" : "", | 20260505 17:54:58.020 | INFO | "huge_shared" : " 27 GB", | 20260505 17:54:58.020 | INFO | "malloc" : " 14 GB", | 20260505 17:54:58.020 | INFO | "pinned" : " 29 GB" | 20260505 17:54:58.020 | INFO | } | 20260505 17:54:58.020 | INFO | Perf report for the last 5.13113 seconds: | 20260505 17:54:58.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:54:58.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:58.021 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:58.021 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:58.021 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260505 17:54:58.021 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:54:58.021 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260505 17:54:58.021 | INFO | PRE_HQ : 60.5%, 1.8, 55.9ms, 0 | 20260505 17:54:58.021 | INFO | BAZWRITER : 63.4%, 3.8, 123.6ms, 0 | 20260505 17:54:58.021 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 17:54:58.021 | INFO | Uploaded 36GiB to GPU in 2.16173s (16.6534GiB/s) | 20260505 17:54:58.021 | INFO | Downloaded 38GiB to GPU in 2.24478s (16.9281GiB/s) | 20260505 17:55:03.156 | INFO | Analyzing chunk frames = [5161472,5161984) | 20260505 17:55:03.156 | INFO | MemInfo: { | 20260505 17:55:03.156 | INFO | "gpu" : " 17 GB", | 20260505 17:55:03.156 | INFO | "huge" : "", | 20260505 17:55:03.156 | INFO | "huge_shared" : " 27 GB", | 20260505 17:55:03.156 | INFO | "malloc" : " 14 GB", | 20260505 17:55:03.156 | INFO | "pinned" : " 29 GB" | 20260505 17:55:03.156 | INFO | } | 20260505 17:55:03.156 | INFO | Perf report for the last 5.13534 seconds: | 20260505 17:55:03.156 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:55:03.156 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:03.156 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:03.156 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:03.156 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 17:55:03.156 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:03.156 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:03.156 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:55:03.156 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:03.156 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:03.156 | INFO | Uploaded 36GiB to GPU in 2.12783s (16.9187GiB/s) | 20260505 17:55:03.156 | INFO | Downloaded 34GiB to GPU in 2.03189s (16.7332GiB/s) | 20260505 17:55:08.304 | INFO | Analyzing chunk frames = [5161984,5162496) | 20260505 17:55:08.304 | INFO | MemInfo: { | 20260505 17:55:08.304 | INFO | "gpu" : " 17 GB", | 20260505 17:55:08.304 | INFO | "huge" : "", | 20260505 17:55:08.304 | INFO | "huge_shared" : " 27 GB", | 20260505 17:55:08.304 | INFO | "malloc" : " 14 GB", | 20260505 17:55:08.304 | INFO | "pinned" : " 29 GB" | 20260505 17:55:08.304 | INFO | } | 20260505 17:55:08.304 | INFO | Perf report for the last 5.14834 seconds: | 20260505 17:55:08.304 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:55:08.304 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:08.304 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:08.304 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:08.304 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260505 17:55:08.304 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:08.304 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:08.304 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260505 17:55:08.304 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:08.304 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:08.304 | INFO | Uploaded 36GiB to GPU in 2.20207s (16.3483GiB/s) | 20260505 17:55:08.304 | INFO | Downloaded 34GiB to GPU in 2.047s (16.6097GiB/s) | 20260505 17:55:13.442 | INFO | Analyzing chunk frames = [5162496,5163008) | 20260505 17:55:13.442 | INFO | MemInfo: { | 20260505 17:55:13.442 | INFO | "gpu" : " 17 GB", | 20260505 17:55:13.442 | INFO | "huge" : "", | 20260505 17:55:13.442 | INFO | "huge_shared" : " 27 GB", | 20260505 17:55:13.442 | INFO | "malloc" : " 14 GB", | 20260505 17:55:13.442 | INFO | "pinned" : " 29 GB" | 20260505 17:55:13.442 | INFO | } | 20260505 17:55:13.442 | INFO | Perf report for the last 5.13747 seconds: | 20260505 17:55:13.442 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:55:13.442 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:13.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:13.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:13.442 | INFO | ANALYSIS : 62.8%, 3.0, 97.5ms, 0 | 20260505 17:55:13.442 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:13.442 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:13.442 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:55:13.442 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:13.442 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:13.442 | INFO | Uploaded 36GiB to GPU in 2.22364s (16.1897GiB/s) | 20260505 17:55:13.442 | INFO | Downloaded 34GiB to GPU in 2.12039s (16.0348GiB/s) | 20260505 17:55:18.581 | INFO | Analyzing chunk frames = [5163008,5163520) | 20260505 17:55:18.581 | INFO | MemInfo: { | 20260505 17:55:18.581 | INFO | "gpu" : " 17 GB", | 20260505 17:55:18.581 | INFO | "huge" : "", | 20260505 17:55:18.581 | INFO | "huge_shared" : " 27 GB", | 20260505 17:55:18.581 | INFO | "malloc" : " 14 GB", | 20260505 17:55:18.581 | INFO | "pinned" : " 29 GB" | 20260505 17:55:18.581 | INFO | } | 20260505 17:55:18.581 | INFO | Perf report for the last 5.13953 seconds: | 20260505 17:55:18.581 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:55:18.581 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:18.581 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:18.581 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:18.581 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 17:55:18.581 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:18.581 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:18.581 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:55:18.581 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:18.581 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:18.582 | INFO | Uploaded 36GiB to GPU in 2.14304s (16.7986GiB/s) | 20260505 17:55:18.582 | INFO | Downloaded 34GiB to GPU in 2.06675s (16.451GiB/s) | 20260505 17:55:23.713 | INFO | Analyzing chunk frames = [5163520,5164032) | 20260505 17:55:23.713 | INFO | MemInfo: { | 20260505 17:55:23.713 | INFO | "gpu" : " 17 GB", | 20260505 17:55:23.713 | INFO | "huge" : "", | 20260505 17:55:23.713 | INFO | "huge_shared" : " 27 GB", | 20260505 17:55:23.713 | INFO | "malloc" : " 14 GB", | 20260505 17:55:23.713 | INFO | "pinned" : " 29 GB" | 20260505 17:55:23.713 | INFO | } | 20260505 17:55:23.713 | INFO | Perf report for the last 5.13159 seconds: | 20260505 17:55:23.713 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:55:23.713 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:23.713 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:23.713 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:23.713 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260505 17:55:23.713 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:23.713 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:23.713 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:55:23.713 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:23.713 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:23.713 | INFO | Uploaded 36GiB to GPU in 2.13767s (16.8408GiB/s) | 20260505 17:55:23.713 | INFO | Downloaded 34GiB to GPU in 2.04981s (16.5869GiB/s) | 20260505 17:55:28.856 | INFO | Analyzing chunk frames = [5164032,5164544) | 20260505 17:55:28.856 | INFO | MemInfo: { | 20260505 17:55:28.856 | INFO | "gpu" : " 17 GB", | 20260505 17:55:28.856 | INFO | "huge" : "", | 20260505 17:55:28.856 | INFO | "huge_shared" : " 27 GB", | 20260505 17:55:28.856 | INFO | "malloc" : " 14 GB", | 20260505 17:55:28.856 | INFO | "pinned" : " 29 GB" | 20260505 17:55:28.857 | INFO | } | 20260505 17:55:28.857 | INFO | Perf report for the last 5.14371 seconds: | 20260505 17:55:28.857 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:55:28.857 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:28.857 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:28.857 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:28.857 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 17:55:28.857 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:28.857 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:28.857 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 17:55:28.857 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:28.857 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:28.857 | INFO | Uploaded 36GiB to GPU in 2.13969s (16.8249GiB/s) | 20260505 17:55:28.857 | INFO | Downloaded 34GiB to GPU in 2.17801s (15.6106GiB/s) | 20260505 17:55:33.996 | INFO | Analyzing chunk frames = [5164544,5165056) | 20260505 17:55:33.996 | INFO | MemInfo: { | 20260505 17:55:33.996 | INFO | "gpu" : " 17 GB", | 20260505 17:55:33.996 | INFO | "huge" : "", | 20260505 17:55:33.996 | INFO | "huge_shared" : " 27 GB", | 20260505 17:55:33.996 | INFO | "malloc" : " 14 GB", | 20260505 17:55:33.996 | INFO | "pinned" : " 29 GB" | 20260505 17:55:33.996 | INFO | } | 20260505 17:55:33.996 | INFO | Perf report for the last 5.13981 seconds: | 20260505 17:55:33.996 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:55:33.996 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:33.996 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:33.996 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:33.996 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 17:55:33.996 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:33.996 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:33.996 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:55:33.996 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:33.996 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:33.997 | INFO | Uploaded 36GiB to GPU in 2.19781s (16.3799GiB/s) | 20260505 17:55:33.997 | INFO | Downloaded 34GiB to GPU in 2.02643s (16.7783GiB/s) | 20260505 17:55:34.132 | INFO | Starting a round of Baz IO | 20260505 17:55:37.309 | INFO | Populating RTMetrics report | 20260505 17:55:37.310 | INFO | RTMetrics report populated | 20260505 17:55:37.310 | INFO | Opening tmp RTMetrics json file | 20260505 17:55:37.314 | INFO | Serializing RTMetrics to json | 20260505 17:55:37.316 | INFO | Serialization done, swapping tmp file now... | 20260505 17:55:37.318 | INFO | RTMetrics done | 20260505 17:55:37.400 | INFO | Finished a round of Baz IO: Wrote 2.111 GiB in 3.267 seconds | 20260505 17:55:37.400 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.98% / 0.00% / 78.01% | 20260505 17:55:37.400 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.275393% / 0% / 9.53674e-05% | 20260505 17:55:37.400 | INFO | Baz IO overall: | 20260505 17:55:37.400 | INFO | BytesWritten : 3047893487025 | 20260505 17:55:37.400 | INFO | NumEvents : 1444623607852 | 20260505 17:55:39.138 | INFO | Analyzing chunk frames = [5165056,5165568) | 20260505 17:55:39.138 | INFO | MemInfo: { | 20260505 17:55:39.138 | INFO | "gpu" : " 17 GB", | 20260505 17:55:39.138 | INFO | "huge" : "", | 20260505 17:55:39.138 | INFO | "huge_shared" : " 27 GB", | 20260505 17:55:39.138 | INFO | "malloc" : " 14 GB", | 20260505 17:55:39.138 | INFO | "pinned" : " 29 GB" | 20260505 17:55:39.138 | INFO | } | 20260505 17:55:39.138 | INFO | Perf report for the last 5.14163 seconds: | 20260505 17:55:39.138 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:55:39.138 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:39.138 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:39.138 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:39.138 | INFO | ANALYSIS : 63.4%, 3.0, 98.8ms, 0 | 20260505 17:55:39.138 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:39.138 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260505 17:55:39.138 | INFO | PRE_HQ : 60.4%, 1.8, 55.9ms, 0 | 20260505 17:55:39.138 | INFO | BAZWRITER : 63.0%, 3.8, 124.3ms, 0 | 20260505 17:55:39.138 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 17:55:39.139 | INFO | Uploaded 36GiB to GPU in 2.15673s (16.692GiB/s) | 20260505 17:55:39.139 | INFO | Downloaded 38GiB to GPU in 2.24698s (16.9116GiB/s) | 20260505 17:55:44.277 | INFO | Analyzing chunk frames = [5165568,5166080) | 20260505 17:55:44.277 | INFO | MemInfo: { | 20260505 17:55:44.277 | INFO | "gpu" : " 17 GB", | 20260505 17:55:44.277 | INFO | "huge" : "", | 20260505 17:55:44.277 | INFO | "huge_shared" : " 27 GB", | 20260505 17:55:44.277 | INFO | "malloc" : " 14 GB", | 20260505 17:55:44.277 | INFO | "pinned" : " 29 GB" | 20260505 17:55:44.277 | INFO | } | 20260505 17:55:44.277 | INFO | Perf report for the last 5.13917 seconds: | 20260505 17:55:44.277 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:55:44.277 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:44.277 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:44.277 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:44.277 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 17:55:44.277 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:44.277 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:44.277 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 17:55:44.277 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:44.277 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:44.277 | INFO | Uploaded 36GiB to GPU in 2.14745s (16.7641GiB/s) | 20260505 17:55:44.278 | INFO | Downloaded 34GiB to GPU in 2.02767s (16.768GiB/s) | 20260505 17:55:49.408 | INFO | Analyzing chunk frames = [5166080,5166592) | 20260505 17:55:49.408 | INFO | MemInfo: { | 20260505 17:55:49.408 | INFO | "gpu" : " 17 GB", | 20260505 17:55:49.408 | INFO | "huge" : "", | 20260505 17:55:49.408 | INFO | "huge_shared" : " 27 GB", | 20260505 17:55:49.409 | INFO | "malloc" : " 14 GB", | 20260505 17:55:49.409 | INFO | "pinned" : " 29 GB" | 20260505 17:55:49.409 | INFO | } | 20260505 17:55:49.409 | INFO | Perf report for the last 5.13139 seconds: | 20260505 17:55:49.409 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:55:49.409 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:49.409 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:49.409 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:49.409 | INFO | ANALYSIS : 62.8%, 3.0, 97.6ms, 0 | 20260505 17:55:49.409 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:49.409 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:49.409 | INFO | PRE_HQ : 27.7%, 1.0, 14.3ms, 0 | 20260505 17:55:49.409 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:49.409 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:49.409 | INFO | Uploaded 36GiB to GPU in 2.17847s (16.5254GiB/s) | 20260505 17:55:49.409 | INFO | Downloaded 34GiB to GPU in 2.03843s (16.6795GiB/s) | 20260505 17:55:54.554 | INFO | Analyzing chunk frames = [5166592,5167104) | 20260505 17:55:54.554 | INFO | MemInfo: { | 20260505 17:55:54.554 | INFO | "gpu" : " 17 GB", | 20260505 17:55:54.554 | INFO | "huge" : "", | 20260505 17:55:54.554 | INFO | "huge_shared" : " 27 GB", | 20260505 17:55:54.554 | INFO | "malloc" : " 14 GB", | 20260505 17:55:54.554 | INFO | "pinned" : " 29 GB" | 20260505 17:55:54.554 | INFO | } | 20260505 17:55:54.554 | INFO | Perf report for the last 5.14565 seconds: | 20260505 17:55:54.554 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:55:54.554 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:54.554 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:54.554 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:54.554 | INFO | ANALYSIS : 62.5%, 3.0, 97.1ms, 0 | 20260505 17:55:54.554 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:54.554 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:54.554 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:55:54.554 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:54.554 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:54.555 | INFO | Uploaded 36GiB to GPU in 2.17292s (16.5676GiB/s) | 20260505 17:55:54.555 | INFO | Downloaded 34GiB to GPU in 2.14198s (15.8732GiB/s) | 20260505 17:55:59.694 | INFO | Analyzing chunk frames = [5167104,5167616) | 20260505 17:55:59.694 | INFO | MemInfo: { | 20260505 17:55:59.694 | INFO | "gpu" : " 17 GB", | 20260505 17:55:59.694 | INFO | "huge" : "", | 20260505 17:55:59.694 | INFO | "huge_shared" : " 27 GB", | 20260505 17:55:59.694 | INFO | "malloc" : " 14 GB", | 20260505 17:55:59.694 | INFO | "pinned" : " 29 GB" | 20260505 17:55:59.694 | INFO | } | 20260505 17:55:59.694 | INFO | Perf report for the last 5.13996 seconds: | 20260505 17:55:59.694 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:55:59.694 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:59.694 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:59.694 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:59.694 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 17:55:59.694 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:59.694 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:59.694 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:55:59.694 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:59.694 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:55:59.694 | INFO | Uploaded 36GiB to GPU in 2.14868s (16.7544GiB/s) | 20260505 17:55:59.695 | INFO | Downloaded 34GiB to GPU in 2.06016s (16.5036GiB/s) | 20260505 17:56:04.833 | INFO | Analyzing chunk frames = [5167616,5168128) | 20260505 17:56:04.833 | INFO | MemInfo: { | 20260505 17:56:04.833 | INFO | "gpu" : " 17 GB", | 20260505 17:56:04.833 | INFO | "huge" : "", | 20260505 17:56:04.833 | INFO | "huge_shared" : " 27 GB", | 20260505 17:56:04.833 | INFO | "malloc" : " 14 GB", | 20260505 17:56:04.833 | INFO | "pinned" : " 29 GB" | 20260505 17:56:04.833 | INFO | } | 20260505 17:56:04.833 | INFO | Perf report for the last 5.13914 seconds: | 20260505 17:56:04.833 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:56:04.833 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:04.833 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:04.834 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:04.834 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260505 17:56:04.834 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:04.834 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:04.834 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:56:04.834 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:04.834 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:04.834 | INFO | Uploaded 36GiB to GPU in 2.14279s (16.8005GiB/s) | 20260505 17:56:04.834 | INFO | Downloaded 34GiB to GPU in 2.05128s (16.575GiB/s) | 20260505 17:56:09.963 | INFO | Analyzing chunk frames = [5168128,5168640) | 20260505 17:56:09.963 | INFO | MemInfo: { | 20260505 17:56:09.963 | INFO | "gpu" : " 17 GB", | 20260505 17:56:09.963 | INFO | "huge" : "", | 20260505 17:56:09.963 | INFO | "huge_shared" : " 27 GB", | 20260505 17:56:09.963 | INFO | "malloc" : " 14 GB", | 20260505 17:56:09.964 | INFO | "pinned" : " 29 GB" | 20260505 17:56:09.964 | INFO | } | 20260505 17:56:09.964 | INFO | Perf report for the last 5.13016 seconds: | 20260505 17:56:09.964 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:56:09.964 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:09.964 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:09.964 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:09.964 | INFO | ANALYSIS : 63.4%, 3.0, 98.8ms, 0 | 20260505 17:56:09.964 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:09.964 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:09.964 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 17:56:09.964 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:09.964 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:09.964 | INFO | Uploaded 36GiB to GPU in 2.12706s (16.9248GiB/s) | 20260505 17:56:09.964 | INFO | Downloaded 34GiB to GPU in 2.13353s (15.9361GiB/s) | 20260505 17:56:15.111 | INFO | Analyzing chunk frames = [5168640,5169152) | 20260505 17:56:15.111 | INFO | MemInfo: { | 20260505 17:56:15.111 | INFO | "gpu" : " 17 GB", | 20260505 17:56:15.111 | INFO | "huge" : "", | 20260505 17:56:15.111 | INFO | "huge_shared" : " 27 GB", | 20260505 17:56:15.111 | INFO | "malloc" : " 14 GB", | 20260505 17:56:15.111 | INFO | "pinned" : " 29 GB" | 20260505 17:56:15.111 | INFO | } | 20260505 17:56:15.111 | INFO | Perf report for the last 5.14748 seconds: | 20260505 17:56:15.111 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:56:15.111 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:15.111 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:15.111 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:15.111 | INFO | ANALYSIS : 63.0%, 3.0, 98.6ms, 0 | 20260505 17:56:15.111 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:15.111 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:15.111 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260505 17:56:15.111 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:15.111 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:15.111 | INFO | Uploaded 36GiB to GPU in 2.18252s (16.4947GiB/s) | 20260505 17:56:15.111 | INFO | Downloaded 34GiB to GPU in 2.04141s (16.6552GiB/s) | 20260505 17:56:15.111 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 17:56:15.111 | INFO | Total mem usage: 15.3 GiB | 20260505 17:56:15.111 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 17:56:15.111 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.11%) | 20260505 17:56:15.111 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 17:56:15.111 | INFO | High Water Breakdown: | 20260505 17:56:15.111 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 17:56:15.111 | INFO | 38.9% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 17:56:15.111 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 17:56:15.111 | INFO | ------------------------------------------ | 20260505 17:56:15.111 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:56:15.111 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 17:56:15.111 | INFO | Total mem usage: 26.5 GiB | 20260505 17:56:15.111 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 17:56:15.111 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 17:56:15.111 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 17:56:15.111 | INFO | High Water Breakdown: | 20260505 17:56:15.111 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:56:15.112 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:56:15.112 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:56:15.112 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:56:15.112 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:56:15.112 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:56:15.112 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 17:56:15.112 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 17:56:15.112 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:56:15.112 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 17:56:15.112 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 17:56:15.112 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 17:56:15.112 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 17:56:15.112 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 17:56:15.112 | INFO | ------------------------------------------ | 20260505 17:56:15.112 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:56:15.112 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 17:56:15.112 | INFO | Total mem usage: 25.6 GiB | 20260505 17:56:15.112 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 17:56:15.112 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 17:56:15.112 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 17:56:15.112 | INFO | High Water Breakdown: | 20260505 17:56:15.112 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 17:56:15.112 | INFO | ------------------------------------------ | 20260505 17:56:15.112 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:56:15.112 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 17:56:15.112 | INFO | Total mem usage: 15.7 GiB | 20260505 17:56:15.112 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 17:56:15.112 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 17:56:15.112 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 17:56:15.112 | INFO | High Water Breakdown: | 20260505 17:56:15.112 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 17:56:15.112 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 17:56:15.112 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 17:56:15.112 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 17:56:15.112 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 17:56:15.112 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 17:56:15.112 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 17:56:15.112 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 17:56:15.112 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 17:56:15.112 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:56:15.112 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 17:56:15.112 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:56:15.112 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 17:56:15.112 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 17:56:15.112 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 17:56:15.112 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 17:56:15.112 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:56:15.112 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 17:56:15.112 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:56:15.112 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:56:15.112 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:56:15.112 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 17:56:15.112 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 17:56:15.112 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:56:15.112 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 17:56:15.112 | INFO | ------------------------------------------ | 20260505 17:56:15.112 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:56:15.112 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 17:56:15.112 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:56:15.112 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:56:15.112 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.48e+01, 1.36e+01 / 2.13e+00 | 20260505 17:56:15.112 | INFO | PerfReport: 0.29%, AggregateStats -- 9.24e-02 / 1.03e-01, 1.12e-02 / 1.15e-02 | 20260505 17:56:15.112 | INFO | PerfReport: 11.77%, Baseline -- 3.72e+00 / 4.52e+00, 5.59e-01 / 1.49e-01 | 20260505 17:56:15.112 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.53e+00, 1.92e-01 / 9.14e-02 | 20260505 17:56:15.112 | INFO | PerfReport: 15.97%, DME -- 4.03e+01 / 4.47e+01, 4.28e+00 / 2.11e+00 | 20260505 17:56:15.112 | INFO | PerfReport: 56.84%, FrameLabeling -- 1.79e+01 / 1.92e+01, 1.44e+00 / 1.19e+00 | 20260505 17:56:15.112 | INFO | PerfReport: 7.33%, PulseAccumulating -- 2.31e+00 / 2.92e+00, 4.45e-01 / 1.51e-01 | 20260505 17:56:15.112 | INFO | PerfReport: 3.35%, Metrics -- 1.06e+00 / 1.38e+00, 2.91e-01 / 1.45e-01 | 20260505 17:56:15.112 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.69e-01 / 4.32e-01, 6.51e-02 / 4.40e-02 | 20260505 17:56:15.112 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 17:56:15.112 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:56:15.112 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:56:15.112 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260505 17:56:15.112 | INFO | PerfReport: 22.53%, Upload -- 2.25e+01 / 0.00e+00, 7.85e+00 / 0.00e+00 | 20260505 17:56:15.112 | INFO | PerfReport: 21.83%, Download -- 2.18e+01 / 0.00e+00, 7.72e+00 / 0.00e+00 | 20260505 17:56:15.112 | INFO | PerfReport: 31.59%, Compute -- 3.16e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 17:56:15.112 | INFO | PerfReport: 2.32%, LockUpload -- 2.32e+00 / 0.00e+00, 7.89e+00 / 0.00e+00 | 20260505 17:56:15.112 | INFO | PerfReport: 1.42%, LockDownload -- 1.41e+00 / 0.00e+00, 2.92e+00 / 0.00e+00 | 20260505 17:56:15.112 | INFO | PerfReport: 20.30%, LockCompute -- 2.03e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260505 17:56:15.112 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.14e-02 / 0.00e+00 | 20260505 17:56:15.249 | INFO | Starting a round of Baz IO | 20260505 17:56:18.431 | INFO | Populating RTMetrics report | 20260505 17:56:18.431 | INFO | RTMetrics report populated | 20260505 17:56:18.431 | INFO | Opening tmp RTMetrics json file | 20260505 17:56:18.441 | INFO | Serializing RTMetrics to json | 20260505 17:56:18.443 | INFO | Serialization done, swapping tmp file now... | 20260505 17:56:18.444 | INFO | RTMetrics done | 20260505 17:56:18.527 | INFO | Finished a round of Baz IO: Wrote 2.112 GiB in 3.278 seconds | 20260505 17:56:18.527 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.98% / 0.00% / 78.02% | 20260505 17:56:18.527 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.274348% / 3.97364e-06% / 0.000111262% | 20260505 17:56:18.527 | INFO | Baz IO overall: | 20260505 17:56:18.527 | INFO | BytesWritten : 3050161005897 | 20260505 17:56:18.527 | INFO | NumEvents : 1445681825661 | 20260505 17:56:20.252 | INFO | Analyzing chunk frames = [5169152,5169664) | 20260505 17:56:20.252 | INFO | MemInfo: { | 20260505 17:56:20.252 | INFO | "gpu" : " 17 GB", | 20260505 17:56:20.252 | INFO | "huge" : "", | 20260505 17:56:20.252 | INFO | "huge_shared" : " 27 GB", | 20260505 17:56:20.252 | INFO | "malloc" : " 14 GB", | 20260505 17:56:20.252 | INFO | "pinned" : " 29 GB" | 20260505 17:56:20.252 | INFO | } | 20260505 17:56:20.253 | INFO | Perf report for the last 5.14148 seconds: | 20260505 17:56:20.253 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:56:20.253 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:20.253 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:20.253 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:20.253 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 17:56:20.253 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:20.253 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260505 17:56:20.253 | INFO | PRE_HQ : 60.2%, 1.8, 55.8ms, 0 | 20260505 17:56:20.253 | INFO | BAZWRITER : 63.2%, 3.9, 126.9ms, 0 | 20260505 17:56:20.253 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 17:56:20.253 | INFO | Uploaded 36GiB to GPU in 2.16241s (16.6481GiB/s) | 20260505 17:56:20.253 | INFO | Downloaded 38GiB to GPU in 2.25023s (16.8871GiB/s) | 20260505 17:56:25.395 | INFO | Analyzing chunk frames = [5169664,5170176) | 20260505 17:56:25.395 | INFO | MemInfo: { | 20260505 17:56:25.395 | INFO | "gpu" : " 17 GB", | 20260505 17:56:25.395 | INFO | "huge" : "", | 20260505 17:56:25.395 | INFO | "huge_shared" : " 27 GB", | 20260505 17:56:25.395 | INFO | "malloc" : " 14 GB", | 20260505 17:56:25.395 | INFO | "pinned" : " 29 GB" | 20260505 17:56:25.395 | INFO | } | 20260505 17:56:25.395 | INFO | Perf report for the last 5.14217 seconds: | 20260505 17:56:25.395 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:56:25.395 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:25.395 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:25.395 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:25.395 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 17:56:25.395 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:25.395 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:25.395 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 17:56:25.395 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:25.395 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:25.395 | INFO | Uploaded 36GiB to GPU in 2.13206s (16.8851GiB/s) | 20260505 17:56:25.395 | INFO | Downloaded 34GiB to GPU in 2.02395s (16.7988GiB/s) | 20260505 17:56:30.537 | INFO | Analyzing chunk frames = [5170176,5170688) | 20260505 17:56:30.537 | INFO | MemInfo: { | 20260505 17:56:30.537 | INFO | "gpu" : " 17 GB", | 20260505 17:56:30.537 | INFO | "huge" : "", | 20260505 17:56:30.537 | INFO | "huge_shared" : " 27 GB", | 20260505 17:56:30.537 | INFO | "malloc" : " 14 GB", | 20260505 17:56:30.537 | INFO | "pinned" : " 29 GB" | 20260505 17:56:30.537 | INFO | } | 20260505 17:56:30.537 | INFO | Perf report for the last 5.14203 seconds: | 20260505 17:56:30.537 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:56:30.537 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:30.537 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:30.537 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:30.537 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 17:56:30.537 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:30.537 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:30.537 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:56:30.537 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:30.537 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:30.537 | INFO | Uploaded 36GiB to GPU in 2.20974s (16.2915GiB/s) | 20260505 17:56:30.537 | INFO | Downloaded 34GiB to GPU in 2.07522s (16.3838GiB/s) | 20260505 17:56:35.666 | INFO | Analyzing chunk frames = [5170688,5171200) | 20260505 17:56:35.666 | INFO | MemInfo: { | 20260505 17:56:35.666 | INFO | "gpu" : " 17 GB", | 20260505 17:56:35.666 | INFO | "huge" : "", | 20260505 17:56:35.666 | INFO | "huge_shared" : " 27 GB", | 20260505 17:56:35.666 | INFO | "malloc" : " 14 GB", | 20260505 17:56:35.666 | INFO | "pinned" : " 29 GB" | 20260505 17:56:35.666 | INFO | } | 20260505 17:56:35.666 | INFO | Perf report for the last 5.12969 seconds: | 20260505 17:56:35.666 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:56:35.666 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:35.666 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:35.666 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:35.666 | INFO | ANALYSIS : 62.5%, 3.0, 96.9ms, 0 | 20260505 17:56:35.666 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:35.667 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:35.667 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:56:35.667 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:35.667 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:35.667 | INFO | Uploaded 36GiB to GPU in 2.16883s (16.5988GiB/s) | 20260505 17:56:35.667 | INFO | Downloaded 34GiB to GPU in 2.05469s (16.5475GiB/s) | 20260505 17:56:40.812 | INFO | Analyzing chunk frames = [5171200,5171712) | 20260505 17:56:40.812 | INFO | MemInfo: { | 20260505 17:56:40.812 | INFO | "gpu" : " 17 GB", | 20260505 17:56:40.812 | INFO | "huge" : "", | 20260505 17:56:40.812 | INFO | "huge_shared" : " 27 GB", | 20260505 17:56:40.812 | INFO | "malloc" : " 14 GB", | 20260505 17:56:40.812 | INFO | "pinned" : " 29 GB" | 20260505 17:56:40.812 | INFO | } | 20260505 17:56:40.812 | INFO | Perf report for the last 5.14533 seconds: | 20260505 17:56:40.812 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:56:40.812 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:40.812 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:40.812 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:40.812 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 17:56:40.812 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:40.812 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:40.812 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:56:40.812 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:40.812 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:40.812 | INFO | Uploaded 36GiB to GPU in 2.15027s (16.7421GiB/s) | 20260505 17:56:40.812 | INFO | Downloaded 34GiB to GPU in 2.06066s (16.4995GiB/s) | 20260505 17:56:45.948 | INFO | Analyzing chunk frames = [5171712,5172224) | 20260505 17:56:45.948 | INFO | MemInfo: { | 20260505 17:56:45.948 | INFO | "gpu" : " 17 GB", | 20260505 17:56:45.948 | INFO | "huge" : "", | 20260505 17:56:45.948 | INFO | "huge_shared" : " 27 GB", | 20260505 17:56:45.948 | INFO | "malloc" : " 14 GB", | 20260505 17:56:45.948 | INFO | "pinned" : " 29 GB" | 20260505 17:56:45.948 | INFO | } | 20260505 17:56:45.948 | INFO | Perf report for the last 5.13599 seconds: | 20260505 17:56:45.948 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:56:45.948 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:45.948 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:45.948 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:45.948 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 17:56:45.948 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:45.948 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:45.948 | INFO | PRE_HQ : 27.5%, 1.0, 14.2ms, 0 | 20260505 17:56:45.948 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:45.948 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:45.948 | INFO | Uploaded 36GiB to GPU in 2.13519s (16.8603GiB/s) | 20260505 17:56:45.948 | INFO | Downloaded 34GiB to GPU in 2.05322s (16.5594GiB/s) | 20260505 17:56:51.087 | INFO | Analyzing chunk frames = [5172224,5172736) | 20260505 17:56:51.087 | INFO | MemInfo: { | 20260505 17:56:51.087 | INFO | "gpu" : " 17 GB", | 20260505 17:56:51.087 | INFO | "huge" : "", | 20260505 17:56:51.087 | INFO | "huge_shared" : " 27 GB", | 20260505 17:56:51.087 | INFO | "malloc" : " 14 GB", | 20260505 17:56:51.087 | INFO | "pinned" : " 29 GB" | 20260505 17:56:51.087 | INFO | } | 20260505 17:56:51.087 | INFO | Perf report for the last 5.13958 seconds: | 20260505 17:56:51.087 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:56:51.087 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:51.087 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:51.087 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:51.087 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 17:56:51.087 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:51.087 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:51.087 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 17:56:51.087 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:51.087 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:51.088 | INFO | Uploaded 36GiB to GPU in 2.14426s (16.789GiB/s) | 20260505 17:56:51.088 | INFO | Downloaded 34GiB to GPU in 2.14065s (15.883GiB/s) | 20260505 17:56:56.224 | INFO | Analyzing chunk frames = [5172736,5173248) | 20260505 17:56:56.224 | INFO | MemInfo: { | 20260505 17:56:56.224 | INFO | "gpu" : " 17 GB", | 20260505 17:56:56.224 | INFO | "huge" : "", | 20260505 17:56:56.224 | INFO | "huge_shared" : " 27 GB", | 20260505 17:56:56.224 | INFO | "malloc" : " 14 GB", | 20260505 17:56:56.224 | INFO | "pinned" : " 29 GB" | 20260505 17:56:56.224 | INFO | } | 20260505 17:56:56.224 | INFO | Perf report for the last 5.13652 seconds: | 20260505 17:56:56.224 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:56:56.224 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:56.224 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:56.224 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:56.224 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 17:56:56.224 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:56.224 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:56.224 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 17:56:56.224 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:56.224 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:56:56.224 | INFO | Uploaded 36GiB to GPU in 2.19165s (16.426GiB/s) | 20260505 17:56:56.224 | INFO | Downloaded 34GiB to GPU in 2.02562s (16.785GiB/s) | 20260505 17:56:56.360 | INFO | Starting a round of Baz IO | 20260505 17:56:59.553 | INFO | Populating RTMetrics report | 20260505 17:56:59.554 | INFO | RTMetrics report populated | 20260505 17:56:59.554 | INFO | Opening tmp RTMetrics json file | 20260505 17:56:59.572 | INFO | Serializing RTMetrics to json | 20260505 17:56:59.573 | INFO | Serialization done, swapping tmp file now... | 20260505 17:56:59.573 | INFO | RTMetrics done | 20260505 17:56:59.656 | INFO | Finished a round of Baz IO: Wrote 2.109 GiB in 3.295 seconds | 20260505 17:56:59.656 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.98% / 0.00% / 78.02% | 20260505 17:56:59.656 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.272655% / 7.94729e-06% / 0.000119209% | 20260505 17:56:59.656 | INFO | Baz IO overall: | 20260505 17:56:59.656 | INFO | BytesWritten : 3052425873423 | 20260505 17:56:59.656 | INFO | NumEvents : 1446737704562 | 20260505 17:57:01.365 | INFO | Analyzing chunk frames = [5173248,5173760) | 20260505 17:57:01.365 | INFO | MemInfo: { | 20260505 17:57:01.365 | INFO | "gpu" : " 17 GB", | 20260505 17:57:01.365 | INFO | "huge" : "", | 20260505 17:57:01.365 | INFO | "huge_shared" : " 27 GB", | 20260505 17:57:01.365 | INFO | "malloc" : " 14 GB", | 20260505 17:57:01.365 | INFO | "pinned" : " 29 GB" | 20260505 17:57:01.365 | INFO | } | 20260505 17:57:01.365 | INFO | Perf report for the last 5.14097 seconds: | 20260505 17:57:01.365 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:57:01.365 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:01.365 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:01.365 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:01.365 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 17:57:01.365 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:01.365 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260505 17:57:01.365 | INFO | PRE_HQ : 60.7%, 1.8, 56.3ms, 0 | 20260505 17:57:01.365 | INFO | BAZWRITER : 63.5%, 3.8, 124.4ms, 0 | 20260505 17:57:01.365 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 17:57:01.365 | INFO | Uploaded 36GiB to GPU in 2.17107s (16.5817GiB/s) | 20260505 17:57:01.365 | INFO | Downloaded 38GiB to GPU in 2.26015s (16.8131GiB/s) | 20260505 17:57:06.509 | INFO | Analyzing chunk frames = [5173760,5174272) | 20260505 17:57:06.509 | INFO | MemInfo: { | 20260505 17:57:06.509 | INFO | "gpu" : " 17 GB", | 20260505 17:57:06.509 | INFO | "huge" : "", | 20260505 17:57:06.509 | INFO | "huge_shared" : " 27 GB", | 20260505 17:57:06.509 | INFO | "malloc" : " 14 GB", | 20260505 17:57:06.510 | INFO | "pinned" : " 29 GB" | 20260505 17:57:06.510 | INFO | } | 20260505 17:57:06.510 | INFO | Perf report for the last 5.14469 seconds: | 20260505 17:57:06.510 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:57:06.510 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:06.510 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:06.510 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:06.510 | INFO | ANALYSIS : 62.7%, 3.0, 98.0ms, 0 | 20260505 17:57:06.510 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:06.510 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:06.510 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 17:57:06.510 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:06.510 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:06.510 | INFO | Uploaded 36GiB to GPU in 2.14253s (16.8025GiB/s) | 20260505 17:57:06.510 | INFO | Downloaded 34GiB to GPU in 2.02638s (16.7787GiB/s) | 20260505 17:57:11.650 | INFO | Analyzing chunk frames = [5174272,5174784) | 20260505 17:57:11.650 | INFO | MemInfo: { | 20260505 17:57:11.650 | INFO | "gpu" : " 17 GB", | 20260505 17:57:11.650 | INFO | "huge" : "", | 20260505 17:57:11.650 | INFO | "huge_shared" : " 27 GB", | 20260505 17:57:11.650 | INFO | "malloc" : " 14 GB", | 20260505 17:57:11.650 | INFO | "pinned" : " 29 GB" | 20260505 17:57:11.650 | INFO | } | 20260505 17:57:11.650 | INFO | Perf report for the last 5.14017 seconds: | 20260505 17:57:11.650 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:57:11.650 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:11.650 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:11.650 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:11.650 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260505 17:57:11.650 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:11.650 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:11.650 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:57:11.650 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:11.650 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:11.650 | INFO | Uploaded 36GiB to GPU in 2.13901s (16.8302GiB/s) | 20260505 17:57:11.650 | INFO | Downloaded 34GiB to GPU in 2.04108s (16.6578GiB/s) | 20260505 17:57:16.779 | INFO | Analyzing chunk frames = [5174784,5175296) | 20260505 17:57:16.779 | INFO | MemInfo: { | 20260505 17:57:16.779 | INFO | "gpu" : " 17 GB", | 20260505 17:57:16.779 | INFO | "huge" : "", | 20260505 17:57:16.779 | INFO | "huge_shared" : " 27 GB", | 20260505 17:57:16.779 | INFO | "malloc" : " 14 GB", | 20260505 17:57:16.779 | INFO | "pinned" : " 29 GB" | 20260505 17:57:16.779 | INFO | } | 20260505 17:57:16.779 | INFO | Perf report for the last 5.12908 seconds: | 20260505 17:57:16.779 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:57:16.779 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:16.779 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:16.779 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:16.779 | INFO | ANALYSIS : 62.6%, 3.0, 97.0ms, 0 | 20260505 17:57:16.779 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:16.779 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:16.779 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 17:57:16.779 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:16.779 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:16.779 | INFO | Uploaded 36GiB to GPU in 2.16914s (16.5965GiB/s) | 20260505 17:57:16.779 | INFO | Downloaded 34GiB to GPU in 2.06265s (16.4836GiB/s) | 20260505 17:57:21.915 | INFO | Analyzing chunk frames = [5175296,5175808) | 20260505 17:57:21.916 | INFO | MemInfo: { | 20260505 17:57:21.916 | INFO | "gpu" : " 17 GB", | 20260505 17:57:21.916 | INFO | "huge" : "", | 20260505 17:57:21.916 | INFO | "huge_shared" : " 27 GB", | 20260505 17:57:21.916 | INFO | "malloc" : " 14 GB", | 20260505 17:57:21.916 | INFO | "pinned" : " 29 GB" | 20260505 17:57:21.916 | INFO | } | 20260505 17:57:21.916 | INFO | Perf report for the last 5.13675 seconds: | 20260505 17:57:21.916 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:57:21.916 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:21.916 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:21.916 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:21.916 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 17:57:21.916 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:21.916 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:21.916 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:57:21.916 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:21.916 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:21.916 | INFO | Uploaded 36GiB to GPU in 2.15531s (16.7029GiB/s) | 20260505 17:57:21.916 | INFO | Downloaded 34GiB to GPU in 2.06062s (16.4999GiB/s) | 20260505 17:57:27.063 | INFO | Analyzing chunk frames = [5175808,5176320) | 20260505 17:57:27.063 | INFO | MemInfo: { | 20260505 17:57:27.063 | INFO | "gpu" : " 17 GB", | 20260505 17:57:27.063 | INFO | "huge" : "", | 20260505 17:57:27.063 | INFO | "huge_shared" : " 27 GB", | 20260505 17:57:27.063 | INFO | "malloc" : " 14 GB", | 20260505 17:57:27.063 | INFO | "pinned" : " 29 GB" | 20260505 17:57:27.063 | INFO | } | 20260505 17:57:27.063 | INFO | Perf report for the last 5.14752 seconds: | 20260505 17:57:27.063 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:57:27.063 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:27.063 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:27.063 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:27.063 | INFO | ANALYSIS : 62.3%, 3.0, 97.4ms, 0 | 20260505 17:57:27.063 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:27.063 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:27.063 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260505 17:57:27.063 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:27.063 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:27.063 | INFO | Uploaded 36GiB to GPU in 2.13936s (16.8274GiB/s) | 20260505 17:57:27.063 | INFO | Downloaded 34GiB to GPU in 2.03621s (16.6977GiB/s) | 20260505 17:57:32.208 | INFO | Analyzing chunk frames = [5176320,5176832) | 20260505 17:57:32.208 | INFO | MemInfo: { | 20260505 17:57:32.208 | INFO | "gpu" : " 17 GB", | 20260505 17:57:32.208 | INFO | "huge" : "", | 20260505 17:57:32.209 | INFO | "huge_shared" : " 27 GB", | 20260505 17:57:32.209 | INFO | "malloc" : " 14 GB", | 20260505 17:57:32.209 | INFO | "pinned" : " 29 GB" | 20260505 17:57:32.209 | INFO | } | 20260505 17:57:32.209 | INFO | Perf report for the last 5.14537 seconds: | 20260505 17:57:32.209 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:57:32.209 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:32.209 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:32.209 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:32.209 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 17:57:32.209 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:32.209 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:32.209 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 17:57:32.209 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:32.209 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:32.209 | INFO | Uploaded 36GiB to GPU in 2.14324s (16.797GiB/s) | 20260505 17:57:32.209 | INFO | Downloaded 34GiB to GPU in 2.14725s (15.8342GiB/s) | 20260505 17:57:37.342 | INFO | Analyzing chunk frames = [5176832,5177344) | 20260505 17:57:37.342 | INFO | MemInfo: { | 20260505 17:57:37.342 | INFO | "gpu" : " 17 GB", | 20260505 17:57:37.342 | INFO | "huge" : "", | 20260505 17:57:37.342 | INFO | "huge_shared" : " 27 GB", | 20260505 17:57:37.342 | INFO | "malloc" : " 14 GB", | 20260505 17:57:37.342 | INFO | "pinned" : " 29 GB" | 20260505 17:57:37.342 | INFO | } | 20260505 17:57:37.342 | INFO | Perf report for the last 5.13363 seconds: | 20260505 17:57:37.342 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:57:37.342 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:37.342 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:37.342 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:37.342 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 17:57:37.342 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:37.342 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:37.342 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 17:57:37.342 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:37.342 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:37.342 | INFO | Uploaded 36GiB to GPU in 2.20063s (16.359GiB/s) | 20260505 17:57:37.342 | INFO | Downloaded 34GiB to GPU in 2.02525s (16.788GiB/s) | 20260505 17:57:37.343 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 17:57:37.343 | INFO | Total mem usage: 15.3 GiB | 20260505 17:57:37.343 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 17:57:37.343 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -7.27%) | 20260505 17:57:37.343 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 17:57:37.343 | INFO | High Water Breakdown: | 20260505 17:57:37.343 | INFO | 61.1% : 8.73 GiB ( 91.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 17:57:37.343 | INFO | 38.7% : 5.53 GiB ( 83.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 17:57:37.343 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 17:57:37.343 | INFO | ------------------------------------------ | 20260505 17:57:37.343 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:57:37.343 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 17:57:37.343 | INFO | Total mem usage: 26.5 GiB | 20260505 17:57:37.343 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 17:57:37.343 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 17:57:37.343 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 17:57:37.343 | INFO | High Water Breakdown: | 20260505 17:57:37.343 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:57:37.343 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:57:37.343 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:57:37.343 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:57:37.343 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:57:37.343 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:57:37.343 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 17:57:37.343 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 17:57:37.343 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:57:37.343 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 17:57:37.343 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 17:57:37.343 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 17:57:37.343 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 17:57:37.343 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 17:57:37.343 | INFO | ------------------------------------------ | 20260505 17:57:37.343 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:57:37.343 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 17:57:37.343 | INFO | Total mem usage: 25.6 GiB | 20260505 17:57:37.343 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 17:57:37.343 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 17:57:37.343 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 17:57:37.343 | INFO | High Water Breakdown: | 20260505 17:57:37.343 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 17:57:37.343 | INFO | ------------------------------------------ | 20260505 17:57:37.343 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:57:37.343 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 17:57:37.343 | INFO | Total mem usage: 15.7 GiB | 20260505 17:57:37.343 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 17:57:37.343 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 17:57:37.343 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 17:57:37.343 | INFO | High Water Breakdown: | 20260505 17:57:37.343 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 17:57:37.343 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 17:57:37.343 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 17:57:37.343 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 17:57:37.343 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 17:57:37.343 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 17:57:37.343 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 17:57:37.343 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 17:57:37.343 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 17:57:37.343 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:57:37.343 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 17:57:37.343 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:57:37.343 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 17:57:37.343 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 17:57:37.344 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 17:57:37.344 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 17:57:37.344 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:57:37.344 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 17:57:37.344 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:57:37.344 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:57:37.344 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:57:37.344 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 17:57:37.344 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 17:57:37.344 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:57:37.344 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 17:57:37.344 | INFO | ------------------------------------------ | 20260505 17:57:37.344 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:57:37.344 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 17:57:37.344 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:57:37.344 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:57:37.344 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.46e+01, 1.36e+01 / 1.91e+00 | 20260505 17:57:37.344 | INFO | PerfReport: 0.29%, AggregateStats -- 9.26e-02 / 1.02e-01, 1.06e-02 / 3.36e-03 | 20260505 17:57:37.344 | INFO | PerfReport: 11.78%, Baseline -- 3.72e+00 / 4.51e+00, 5.70e-01 / 1.39e-01 | 20260505 17:57:37.344 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.53e+00, 1.92e-01 / 7.35e-02 | 20260505 17:57:37.344 | INFO | PerfReport: 15.91%, DME -- 4.02e+01 / 4.44e+01, 4.14e+00 / 1.90e+00 | 20260505 17:57:37.344 | INFO | PerfReport: 56.94%, FrameLabeling -- 1.80e+01 / 1.93e+01, 1.54e+00 / 1.10e+00 | 20260505 17:57:37.344 | INFO | PerfReport: 7.29%, PulseAccumulating -- 2.30e+00 / 2.91e+00, 4.41e-01 / 1.63e-01 | 20260505 17:57:37.344 | INFO | PerfReport: 3.34%, Metrics -- 1.05e+00 / 1.42e+00, 2.92e-01 / 1.61e-01 | 20260505 17:57:37.344 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.68e-01 / 4.26e-01, 6.48e-02 / 3.35e-02 | 20260505 17:57:37.344 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 17:57:37.344 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:57:37.344 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:57:37.344 | INFO | PerfReport: Full Execution -- 9.99e+01 / -nan, 2.59e+01 / 0.00e+00 | 20260505 17:57:37.344 | INFO | PerfReport: 22.53%, Upload -- 2.25e+01 / 0.00e+00, 7.83e+00 / 0.00e+00 | 20260505 17:57:37.344 | INFO | PerfReport: 21.74%, Download -- 2.17e+01 / 0.00e+00, 7.75e+00 / 0.00e+00 | 20260505 17:57:37.344 | INFO | PerfReport: 31.65%, Compute -- 3.16e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 17:57:37.344 | INFO | PerfReport: 2.33%, LockUpload -- 2.32e+00 / 0.00e+00, 8.02e+00 / 0.00e+00 | 20260505 17:57:37.344 | INFO | PerfReport: 1.37%, LockDownload -- 1.37e+00 / 0.00e+00, 2.92e+00 / 0.00e+00 | 20260505 17:57:37.344 | INFO | PerfReport: 20.37%, LockCompute -- 2.03e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260505 17:57:37.344 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.94e-02 / 0.00e+00 | 20260505 17:57:37.481 | INFO | Starting a round of Baz IO | 20260505 17:57:40.663 | INFO | Populating RTMetrics report | 20260505 17:57:40.663 | INFO | RTMetrics report populated | 20260505 17:57:40.663 | INFO | Opening tmp RTMetrics json file | 20260505 17:57:40.667 | INFO | Serializing RTMetrics to json | 20260505 17:57:40.669 | INFO | Serialization done, swapping tmp file now... | 20260505 17:57:40.671 | INFO | RTMetrics done | 20260505 17:57:40.759 | INFO | Finished a round of Baz IO: Wrote 2.109 GiB in 3.277 seconds | 20260505 17:57:40.759 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.98% / 0.00% / 78.02% | 20260505 17:57:40.759 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.272481% / 3.97364e-06% / 0.000103315% | 20260505 17:57:40.759 | INFO | Baz IO overall: | 20260505 17:57:40.759 | INFO | BytesWritten : 3054690848194 | 20260505 17:57:40.759 | INFO | NumEvents : 1447794016671 | 20260505 17:57:42.480 | INFO | Analyzing chunk frames = [5177344,5177856) | 20260505 17:57:42.480 | INFO | MemInfo: { | 20260505 17:57:42.480 | INFO | "gpu" : " 17 GB", | 20260505 17:57:42.480 | INFO | "huge" : "", | 20260505 17:57:42.480 | INFO | "huge_shared" : " 27 GB", | 20260505 17:57:42.480 | INFO | "malloc" : " 14 GB", | 20260505 17:57:42.480 | INFO | "pinned" : " 29 GB" | 20260505 17:57:42.480 | INFO | } | 20260505 17:57:42.480 | INFO | Perf report for the last 5.13751 seconds: | 20260505 17:57:42.480 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:57:42.480 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:42.480 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:42.480 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:42.480 | INFO | ANALYSIS : 63.5%, 3.0, 99.0ms, 0 | 20260505 17:57:42.480 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:42.480 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260505 17:57:42.480 | INFO | PRE_HQ : 60.3%, 1.8, 56.0ms, 0 | 20260505 17:57:42.480 | INFO | BAZWRITER : 63.2%, 3.8, 124.2ms, 0 | 20260505 17:57:42.480 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 17:57:42.480 | INFO | Uploaded 36GiB to GPU in 2.16238s (16.6483GiB/s) | 20260505 17:57:42.480 | INFO | Downloaded 38GiB to GPU in 2.253s (16.8664GiB/s) | 20260505 17:57:47.622 | INFO | Analyzing chunk frames = [5177856,5178368) | 20260505 17:57:47.622 | INFO | MemInfo: { | 20260505 17:57:47.622 | INFO | "gpu" : " 17 GB", | 20260505 17:57:47.622 | INFO | "huge" : "", | 20260505 17:57:47.622 | INFO | "huge_shared" : " 27 GB", | 20260505 17:57:47.622 | INFO | "malloc" : " 14 GB", | 20260505 17:57:47.622 | INFO | "pinned" : " 29 GB" | 20260505 17:57:47.623 | INFO | } | 20260505 17:57:47.623 | INFO | Perf report for the last 5.14282 seconds: | 20260505 17:57:47.623 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:57:47.623 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:47.623 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:47.623 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:47.623 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 17:57:47.623 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:47.623 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:47.623 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:57:47.623 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:47.623 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:47.623 | INFO | Uploaded 36GiB to GPU in 2.13876s (16.8322GiB/s) | 20260505 17:57:47.623 | INFO | Downloaded 34GiB to GPU in 2.03458s (16.711GiB/s) | 20260505 17:57:52.761 | INFO | Analyzing chunk frames = [5178368,5178880) | 20260505 17:57:52.761 | INFO | MemInfo: { | 20260505 17:57:52.761 | INFO | "gpu" : " 17 GB", | 20260505 17:57:52.761 | INFO | "huge" : "", | 20260505 17:57:52.761 | INFO | "huge_shared" : " 27 GB", | 20260505 17:57:52.761 | INFO | "malloc" : " 14 GB", | 20260505 17:57:52.761 | INFO | "pinned" : " 29 GB" | 20260505 17:57:52.761 | INFO | } | 20260505 17:57:52.761 | INFO | Perf report for the last 5.13877 seconds: | 20260505 17:57:52.761 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:57:52.761 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:52.761 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:52.761 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:52.761 | INFO | ANALYSIS : 62.4%, 3.0, 97.0ms, 0 | 20260505 17:57:52.761 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:52.761 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:52.761 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:57:52.762 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:52.762 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:52.762 | INFO | Uploaded 36GiB to GPU in 2.12718s (16.9238GiB/s) | 20260505 17:57:52.762 | INFO | Downloaded 34GiB to GPU in 2.04861s (16.5966GiB/s) | 20260505 17:57:57.900 | INFO | Analyzing chunk frames = [5178880,5179392) | 20260505 17:57:57.900 | INFO | MemInfo: { | 20260505 17:57:57.900 | INFO | "gpu" : " 17 GB", | 20260505 17:57:57.900 | INFO | "huge" : "", | 20260505 17:57:57.900 | INFO | "huge_shared" : " 27 GB", | 20260505 17:57:57.900 | INFO | "malloc" : " 14 GB", | 20260505 17:57:57.900 | INFO | "pinned" : " 29 GB" | 20260505 17:57:57.900 | INFO | } | 20260505 17:57:57.900 | INFO | Perf report for the last 5.13887 seconds: | 20260505 17:57:57.900 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:57:57.900 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:57.900 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:57.900 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:57.900 | INFO | ANALYSIS : 62.8%, 3.0, 97.6ms, 0 | 20260505 17:57:57.900 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:57.900 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:57.900 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:57:57.900 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:57.900 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:57:57.900 | INFO | Uploaded 36GiB to GPU in 2.22443s (16.184GiB/s) | 20260505 17:57:57.900 | INFO | Downloaded 34GiB to GPU in 2.11579s (16.0697GiB/s) | 20260505 17:58:03.034 | INFO | Analyzing chunk frames = [5179392,5179904) | 20260505 17:58:03.035 | INFO | MemInfo: { | 20260505 17:58:03.035 | INFO | "gpu" : " 17 GB", | 20260505 17:58:03.035 | INFO | "huge" : "", | 20260505 17:58:03.035 | INFO | "huge_shared" : " 27 GB", | 20260505 17:58:03.035 | INFO | "malloc" : " 14 GB", | 20260505 17:58:03.035 | INFO | "pinned" : " 29 GB" | 20260505 17:58:03.035 | INFO | } | 20260505 17:58:03.035 | INFO | Perf report for the last 5.13455 seconds: | 20260505 17:58:03.035 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:58:03.035 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:03.035 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:03.035 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:03.035 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260505 17:58:03.035 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:03.035 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:03.035 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 17:58:03.035 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:03.035 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:03.035 | INFO | Uploaded 36GiB to GPU in 2.13214s (16.8844GiB/s) | 20260505 17:58:03.035 | INFO | Downloaded 34GiB to GPU in 2.06237s (16.4859GiB/s) | 20260505 17:58:08.176 | INFO | Analyzing chunk frames = [5179904,5180416) | 20260505 17:58:08.176 | INFO | MemInfo: { | 20260505 17:58:08.176 | INFO | "gpu" : " 17 GB", | 20260505 17:58:08.176 | INFO | "huge" : "", | 20260505 17:58:08.176 | INFO | "huge_shared" : " 27 GB", | 20260505 17:58:08.176 | INFO | "malloc" : " 14 GB", | 20260505 17:58:08.176 | INFO | "pinned" : " 29 GB" | 20260505 17:58:08.176 | INFO | } | 20260505 17:58:08.176 | INFO | Perf report for the last 5.14117 seconds: | 20260505 17:58:08.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:58:08.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:08.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:08.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:08.176 | INFO | ANALYSIS : 62.3%, 3.0, 97.4ms, 0 | 20260505 17:58:08.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:08.176 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:08.176 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 17:58:08.176 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:08.176 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:08.176 | INFO | Uploaded 36GiB to GPU in 2.12957s (16.9049GiB/s) | 20260505 17:58:08.176 | INFO | Downloaded 34GiB to GPU in 2.04311s (16.6413GiB/s) | 20260505 17:58:13.319 | INFO | Analyzing chunk frames = [5180416,5180928) | 20260505 17:58:13.319 | INFO | MemInfo: { | 20260505 17:58:13.319 | INFO | "gpu" : " 17 GB", | 20260505 17:58:13.319 | INFO | "huge" : "", | 20260505 17:58:13.319 | INFO | "huge_shared" : " 27 GB", | 20260505 17:58:13.319 | INFO | "malloc" : " 14 GB", | 20260505 17:58:13.319 | INFO | "pinned" : " 29 GB" | 20260505 17:58:13.319 | INFO | } | 20260505 17:58:13.319 | INFO | Perf report for the last 5.14334 seconds: | 20260505 17:58:13.319 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:58:13.319 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:13.319 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:13.319 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:13.319 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 17:58:13.319 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:13.319 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:13.319 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260505 17:58:13.319 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:13.319 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:13.320 | INFO | Uploaded 36GiB to GPU in 2.13638s (16.8509GiB/s) | 20260505 17:58:13.320 | INFO | Downloaded 34GiB to GPU in 2.15002s (15.8138GiB/s) | 20260505 17:58:18.464 | INFO | Analyzing chunk frames = [5180928,5181440) | 20260505 17:58:18.464 | INFO | MemInfo: { | 20260505 17:58:18.464 | INFO | "gpu" : " 17 GB", | 20260505 17:58:18.464 | INFO | "huge" : "", | 20260505 17:58:18.464 | INFO | "huge_shared" : " 27 GB", | 20260505 17:58:18.465 | INFO | "malloc" : " 14 GB", | 20260505 17:58:18.465 | INFO | "pinned" : " 29 GB" | 20260505 17:58:18.465 | INFO | } | 20260505 17:58:18.465 | INFO | Perf report for the last 5.14527 seconds: | 20260505 17:58:18.465 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:58:18.465 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:18.465 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:18.465 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:18.465 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 17:58:18.465 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:18.465 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:18.465 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 17:58:18.465 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:18.465 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:18.465 | INFO | Uploaded 36GiB to GPU in 2.19173s (16.4254GiB/s) | 20260505 17:58:18.465 | INFO | Downloaded 34GiB to GPU in 2.03636s (16.6964GiB/s) | 20260505 17:58:18.597 | INFO | Starting a round of Baz IO | 20260505 17:58:21.772 | INFO | Populating RTMetrics report | 20260505 17:58:21.773 | INFO | RTMetrics report populated | 20260505 17:58:21.773 | INFO | Opening tmp RTMetrics json file | 20260505 17:58:21.783 | INFO | Serializing RTMetrics to json | 20260505 17:58:21.785 | INFO | Serialization done, swapping tmp file now... | 20260505 17:58:21.785 | INFO | RTMetrics done | 20260505 17:58:21.864 | INFO | Finished a round of Baz IO: Wrote 2.112 GiB in 3.267 seconds | 20260505 17:58:21.864 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.98% / 0.00% / 78.02% | 20260505 17:58:21.865 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.272238% / 0% / 0.000107288% | 20260505 17:58:21.865 | INFO | Baz IO overall: | 20260505 17:58:21.865 | INFO | BytesWritten : 3056958074089 | 20260505 17:58:21.865 | INFO | NumEvents : 1448851904274 | 20260505 17:58:23.596 | INFO | Analyzing chunk frames = [5181440,5181952) | 20260505 17:58:23.596 | INFO | MemInfo: { | 20260505 17:58:23.596 | INFO | "gpu" : " 17 GB", | 20260505 17:58:23.596 | INFO | "huge" : "", | 20260505 17:58:23.596 | INFO | "huge_shared" : " 27 GB", | 20260505 17:58:23.597 | INFO | "malloc" : " 14 GB", | 20260505 17:58:23.597 | INFO | "pinned" : " 29 GB" | 20260505 17:58:23.597 | INFO | } | 20260505 17:58:23.597 | INFO | Perf report for the last 5.13203 seconds: | 20260505 17:58:23.597 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:58:23.597 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:23.597 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:23.597 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:23.597 | INFO | ANALYSIS : 63.4%, 3.0, 98.6ms, 0 | 20260505 17:58:23.597 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:23.597 | INFO | RT_METRICS : 44.7%, 1.0, 23.4ms, 0 | 20260505 17:58:23.597 | INFO | PRE_HQ : 60.6%, 1.8, 56.0ms, 0 | 20260505 17:58:23.597 | INFO | BAZWRITER : 63.2%, 3.8, 124.4ms, 0 | 20260505 17:58:23.597 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 17:58:23.597 | INFO | Uploaded 36GiB to GPU in 2.16527s (16.6261GiB/s) | 20260505 17:58:23.597 | INFO | Downloaded 38GiB to GPU in 2.23676s (16.9889GiB/s) | 20260505 17:58:28.725 | INFO | Analyzing chunk frames = [5181952,5182464) | 20260505 17:58:28.725 | INFO | MemInfo: { | 20260505 17:58:28.725 | INFO | "gpu" : " 17 GB", | 20260505 17:58:28.725 | INFO | "huge" : "", | 20260505 17:58:28.725 | INFO | "huge_shared" : " 27 GB", | 20260505 17:58:28.726 | INFO | "malloc" : " 14 GB", | 20260505 17:58:28.726 | INFO | "pinned" : " 29 GB" | 20260505 17:58:28.726 | INFO | } | 20260505 17:58:28.726 | INFO | Perf report for the last 5.12896 seconds: | 20260505 17:58:28.726 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:58:28.726 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:28.726 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:28.726 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:28.726 | INFO | ANALYSIS : 63.0%, 3.0, 98.1ms, 0 | 20260505 17:58:28.726 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:28.726 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:28.726 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 17:58:28.726 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:28.726 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:28.726 | INFO | Uploaded 36GiB to GPU in 2.12606s (16.9328GiB/s) | 20260505 17:58:28.726 | INFO | Downloaded 34GiB to GPU in 2.02968s (16.7514GiB/s) | 20260505 17:58:33.875 | INFO | Analyzing chunk frames = [5182464,5182976) | 20260505 17:58:33.875 | INFO | MemInfo: { | 20260505 17:58:33.875 | INFO | "gpu" : " 17 GB", | 20260505 17:58:33.875 | INFO | "huge" : "", | 20260505 17:58:33.875 | INFO | "huge_shared" : " 27 GB", | 20260505 17:58:33.875 | INFO | "malloc" : " 14 GB", | 20260505 17:58:33.875 | INFO | "pinned" : " 29 GB" | 20260505 17:58:33.875 | INFO | } | 20260505 17:58:33.875 | INFO | Perf report for the last 5.14925 seconds: | 20260505 17:58:33.875 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:58:33.875 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:33.875 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:33.875 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:33.875 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260505 17:58:33.875 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:33.875 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:33.875 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:58:33.875 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:33.875 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:33.876 | INFO | Uploaded 36GiB to GPU in 2.19047s (16.4349GiB/s) | 20260505 17:58:33.876 | INFO | Downloaded 34GiB to GPU in 2.03094s (16.741GiB/s) | 20260505 17:58:39.019 | INFO | Analyzing chunk frames = [5182976,5183488) | 20260505 17:58:39.019 | INFO | MemInfo: { | 20260505 17:58:39.019 | INFO | "gpu" : " 17 GB", | 20260505 17:58:39.019 | INFO | "huge" : "", | 20260505 17:58:39.019 | INFO | "huge_shared" : " 27 GB", | 20260505 17:58:39.019 | INFO | "malloc" : " 14 GB", | 20260505 17:58:39.019 | INFO | "pinned" : " 29 GB" | 20260505 17:58:39.019 | INFO | } | 20260505 17:58:39.019 | INFO | Perf report for the last 5.1446 seconds: | 20260505 17:58:39.019 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:58:39.019 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:39.019 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:39.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:39.020 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260505 17:58:39.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:39.020 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:39.020 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:58:39.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:39.020 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:39.020 | INFO | Uploaded 36GiB to GPU in 2.18293s (16.4916GiB/s) | 20260505 17:58:39.020 | INFO | Downloaded 34GiB to GPU in 2.04773s (16.6037GiB/s) | 20260505 17:58:44.158 | INFO | Analyzing chunk frames = [5183488,5184000) | 20260505 17:58:44.158 | INFO | MemInfo: { | 20260505 17:58:44.158 | INFO | "gpu" : " 17 GB", | 20260505 17:58:44.158 | INFO | "huge" : "", | 20260505 17:58:44.158 | INFO | "huge_shared" : " 27 GB", | 20260505 17:58:44.158 | INFO | "malloc" : " 14 GB", | 20260505 17:58:44.158 | INFO | "pinned" : " 29 GB" | 20260505 17:58:44.158 | INFO | } | 20260505 17:58:44.158 | INFO | Perf report for the last 5.13863 seconds: | 20260505 17:58:44.158 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:58:44.158 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:44.158 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:44.158 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:44.158 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260505 17:58:44.158 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:44.158 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:44.158 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:58:44.158 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:44.158 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:44.158 | INFO | Uploaded 36GiB to GPU in 2.13821s (16.8365GiB/s) | 20260505 17:58:44.158 | INFO | Downloaded 34GiB to GPU in 2.05679s (16.5306GiB/s) | 20260505 17:58:49.291 | INFO | Analyzing chunk frames = [5184000,5184512) | 20260505 17:58:49.291 | INFO | MemInfo: { | 20260505 17:58:49.291 | INFO | "gpu" : " 17 GB", | 20260505 17:58:49.291 | INFO | "huge" : "", | 20260505 17:58:49.291 | INFO | "huge_shared" : " 27 GB", | 20260505 17:58:49.291 | INFO | "malloc" : " 14 GB", | 20260505 17:58:49.291 | INFO | "pinned" : " 29 GB" | 20260505 17:58:49.291 | INFO | } | 20260505 17:58:49.291 | INFO | Perf report for the last 5.13315 seconds: | 20260505 17:58:49.291 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:58:49.291 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:49.291 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:49.291 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:49.291 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260505 17:58:49.291 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:49.291 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:49.291 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 17:58:49.291 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:49.291 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:49.292 | INFO | Uploaded 36GiB to GPU in 2.15323s (16.719GiB/s) | 20260505 17:58:49.292 | INFO | Downloaded 34GiB to GPU in 2.0568s (16.5305GiB/s) | 20260505 17:58:54.427 | INFO | Analyzing chunk frames = [5184512,5185024) | 20260505 17:58:54.427 | INFO | MemInfo: { | 20260505 17:58:54.427 | INFO | "gpu" : " 17 GB", | 20260505 17:58:54.427 | INFO | "huge" : "", | 20260505 17:58:54.427 | INFO | "huge_shared" : " 27 GB", | 20260505 17:58:54.427 | INFO | "malloc" : " 14 GB", | 20260505 17:58:54.427 | INFO | "pinned" : " 29 GB" | 20260505 17:58:54.427 | INFO | } | 20260505 17:58:54.427 | INFO | Perf report for the last 5.13555 seconds: | 20260505 17:58:54.427 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:58:54.427 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:54.427 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:54.427 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:54.427 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 17:58:54.427 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:54.427 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:54.427 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 17:58:54.427 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:54.427 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:54.427 | INFO | Uploaded 36GiB to GPU in 2.11004s (17.0613GiB/s) | 20260505 17:58:54.427 | INFO | Downloaded 34GiB to GPU in 2.17422s (15.6378GiB/s) | 20260505 17:58:59.575 | INFO | Analyzing chunk frames = [5185024,5185536) | 20260505 17:58:59.575 | INFO | MemInfo: { | 20260505 17:58:59.575 | INFO | "gpu" : " 17 GB", | 20260505 17:58:59.575 | INFO | "huge" : "", | 20260505 17:58:59.575 | INFO | "huge_shared" : " 27 GB", | 20260505 17:58:59.575 | INFO | "malloc" : " 14 GB", | 20260505 17:58:59.575 | INFO | "pinned" : " 29 GB" | 20260505 17:58:59.575 | INFO | } | 20260505 17:58:59.575 | INFO | Perf report for the last 5.14846 seconds: | 20260505 17:58:59.575 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:58:59.575 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:59.575 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:59.575 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:59.575 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 17:58:59.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:59.575 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:59.575 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 17:58:59.575 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:59.575 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:58:59.576 | INFO | Uploaded 36GiB to GPU in 2.18768s (16.4558GiB/s) | 20260505 17:58:59.576 | INFO | Downloaded 34GiB to GPU in 2.03289s (16.725GiB/s) | 20260505 17:58:59.576 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 17:58:59.576 | INFO | Total mem usage: 15.3 GiB | 20260505 17:58:59.576 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 17:58:59.576 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.99%) | 20260505 17:58:59.576 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 17:58:59.576 | INFO | High Water Breakdown: | 20260505 17:58:59.576 | INFO | 61% : 8.66 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 17:58:59.576 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 17:58:59.576 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 17:58:59.576 | INFO | ------------------------------------------ | 20260505 17:58:59.576 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:58:59.576 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 17:58:59.576 | INFO | Total mem usage: 26.5 GiB | 20260505 17:58:59.576 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 17:58:59.576 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 17:58:59.576 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 17:58:59.576 | INFO | High Water Breakdown: | 20260505 17:58:59.576 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:58:59.576 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:58:59.576 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:58:59.576 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:58:59.576 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:58:59.576 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:58:59.576 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 17:58:59.576 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 17:58:59.576 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:58:59.576 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 17:58:59.576 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 17:58:59.576 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 17:58:59.576 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 17:58:59.576 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 17:58:59.576 | INFO | ------------------------------------------ | 20260505 17:58:59.576 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:58:59.576 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 17:58:59.576 | INFO | Total mem usage: 25.6 GiB | 20260505 17:58:59.576 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 17:58:59.576 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 17:58:59.576 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 17:58:59.576 | INFO | High Water Breakdown: | 20260505 17:58:59.576 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 17:58:59.576 | INFO | ------------------------------------------ | 20260505 17:58:59.576 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:58:59.576 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 17:58:59.576 | INFO | Total mem usage: 15.7 GiB | 20260505 17:58:59.576 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 17:58:59.576 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 17:58:59.576 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 17:58:59.576 | INFO | High Water Breakdown: | 20260505 17:58:59.576 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 17:58:59.576 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 17:58:59.576 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 17:58:59.576 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 17:58:59.576 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 17:58:59.576 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 17:58:59.576 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 17:58:59.576 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 17:58:59.576 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 17:58:59.576 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 17:58:59.576 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 17:58:59.576 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 17:58:59.576 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 17:58:59.577 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 17:58:59.577 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 17:58:59.577 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 17:58:59.577 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 17:58:59.577 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 17:58:59.577 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 17:58:59.577 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 17:58:59.577 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 17:58:59.577 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 17:58:59.577 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 17:58:59.577 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 17:58:59.577 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 17:58:59.577 | INFO | ------------------------------------------ | 20260505 17:58:59.577 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 17:58:59.577 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 17:58:59.577 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:58:59.577 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:58:59.577 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.47e+01, 1.34e+01 / 1.73e+00 | 20260505 17:58:59.577 | INFO | PerfReport: 0.29%, AggregateStats -- 9.27e-02 / 1.02e-01, 1.09e-02 / 5.16e-03 | 20260505 17:58:59.577 | INFO | PerfReport: 11.79%, Baseline -- 3.72e+00 / 4.48e+00, 5.75e-01 / 1.53e-01 | 20260505 17:58:59.577 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.53e+00, 1.92e-01 / 9.35e-02 | 20260505 17:58:59.577 | INFO | PerfReport: 15.91%, DME -- 4.02e+01 / 4.44e+01, 4.06e+00 / 1.71e+00 | 20260505 17:58:59.577 | INFO | PerfReport: 56.93%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.51e+00 / 1.31e+00 | 20260505 17:58:59.577 | INFO | PerfReport: 7.30%, PulseAccumulating -- 2.31e+00 / 2.85e+00, 4.43e-01 / 2.34e-01 | 20260505 17:58:59.577 | INFO | PerfReport: 3.35%, Metrics -- 1.06e+00 / 1.40e+00, 2.91e-01 / 1.58e-01 | 20260505 17:58:59.577 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.69e-01 / 4.31e-01, 6.62e-02 / 3.91e-02 | 20260505 17:58:59.577 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 17:58:59.577 | INFO | PerfReport: Gathered 1536 samples. | 20260505 17:58:59.577 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 17:58:59.577 | INFO | PerfReport: Full Execution -- 9.99e+01 / -nan, 2.55e+01 / 0.00e+00 | 20260505 17:58:59.577 | INFO | PerfReport: 22.49%, Upload -- 2.25e+01 / 0.00e+00, 7.81e+00 / 0.00e+00 | 20260505 17:58:59.577 | INFO | PerfReport: 21.78%, Download -- 2.18e+01 / 0.00e+00, 7.77e+00 / 0.00e+00 | 20260505 17:58:59.577 | INFO | PerfReport: 31.65%, Compute -- 3.16e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 17:58:59.577 | INFO | PerfReport: 2.32%, LockUpload -- 2.31e+00 / 0.00e+00, 7.94e+00 / 0.00e+00 | 20260505 17:58:59.577 | INFO | PerfReport: 1.42%, LockDownload -- 1.41e+00 / 0.00e+00, 2.99e+00 / 0.00e+00 | 20260505 17:58:59.577 | INFO | PerfReport: 20.33%, LockCompute -- 2.03e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 17:58:59.577 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.58e-02 / 0.00e+00 | 20260505 17:58:59.708 | INFO | Starting a round of Baz IO | 20260505 17:59:02.890 | INFO | Populating RTMetrics report | 20260505 17:59:02.890 | INFO | RTMetrics report populated | 20260505 17:59:02.890 | INFO | Opening tmp RTMetrics json file | 20260505 17:59:02.897 | INFO | Serializing RTMetrics to json | 20260505 17:59:02.898 | INFO | Serialization done, swapping tmp file now... | 20260505 17:59:02.898 | INFO | RTMetrics done | 20260505 17:59:02.986 | INFO | Finished a round of Baz IO: Wrote 2.11 GiB in 3.277 seconds | 20260505 17:59:02.986 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.98% / 0.00% / 78.02% | 20260505 17:59:02.986 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.272759% / 0% / 9.13938e-05% | 20260505 17:59:02.986 | INFO | Baz IO overall: | 20260505 17:59:02.986 | INFO | BytesWritten : 3059224142008 | 20260505 17:59:02.986 | INFO | NumEvents : 1449909367957 | 20260505 17:59:04.712 | INFO | Analyzing chunk frames = [5185536,5186048) | 20260505 17:59:04.712 | INFO | MemInfo: { | 20260505 17:59:04.712 | INFO | "gpu" : " 17 GB", | 20260505 17:59:04.712 | INFO | "huge" : "", | 20260505 17:59:04.712 | INFO | "huge_shared" : " 27 GB", | 20260505 17:59:04.712 | INFO | "malloc" : " 14 GB", | 20260505 17:59:04.712 | INFO | "pinned" : " 29 GB" | 20260505 17:59:04.712 | INFO | } | 20260505 17:59:04.712 | INFO | Perf report for the last 5.13681 seconds: | 20260505 17:59:04.712 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:59:04.712 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:04.712 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:04.712 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:04.712 | INFO | ANALYSIS : 63.4%, 3.0, 98.8ms, 0 | 20260505 17:59:04.712 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:04.712 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260505 17:59:04.712 | INFO | PRE_HQ : 60.6%, 1.8, 56.1ms, 0 | 20260505 17:59:04.712 | INFO | BAZWRITER : 63.4%, 3.8, 123.4ms, 0 | 20260505 17:59:04.712 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 17:59:04.713 | INFO | Uploaded 36GiB to GPU in 2.17275s (16.5688GiB/s) | 20260505 17:59:04.713 | INFO | Downloaded 38GiB to GPU in 2.2455s (16.9227GiB/s) | 20260505 17:59:09.849 | INFO | Analyzing chunk frames = [5186048,5186560) | 20260505 17:59:09.849 | INFO | MemInfo: { | 20260505 17:59:09.849 | INFO | "gpu" : " 17 GB", | 20260505 17:59:09.849 | INFO | "huge" : "", | 20260505 17:59:09.849 | INFO | "huge_shared" : " 27 GB", | 20260505 17:59:09.849 | INFO | "malloc" : " 14 GB", | 20260505 17:59:09.849 | INFO | "pinned" : " 29 GB" | 20260505 17:59:09.850 | INFO | } | 20260505 17:59:09.850 | INFO | Perf report for the last 5.1374 seconds: | 20260505 17:59:09.850 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:59:09.850 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:09.850 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:09.850 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:09.850 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 17:59:09.850 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:09.850 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:09.850 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 17:59:09.850 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:09.850 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:09.850 | INFO | Uploaded 36GiB to GPU in 2.13986s (16.8236GiB/s) | 20260505 17:59:09.850 | INFO | Downloaded 34GiB to GPU in 2.02952s (16.7527GiB/s) | 20260505 17:59:14.982 | INFO | Analyzing chunk frames = [5186560,5187072) | 20260505 17:59:14.982 | INFO | MemInfo: { | 20260505 17:59:14.982 | INFO | "gpu" : " 17 GB", | 20260505 17:59:14.982 | INFO | "huge" : "", | 20260505 17:59:14.982 | INFO | "huge_shared" : " 27 GB", | 20260505 17:59:14.982 | INFO | "malloc" : " 14 GB", | 20260505 17:59:14.982 | INFO | "pinned" : " 29 GB" | 20260505 17:59:14.982 | INFO | } | 20260505 17:59:14.982 | INFO | Perf report for the last 5.13237 seconds: | 20260505 17:59:14.982 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:59:14.982 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:14.982 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:14.982 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:14.982 | INFO | ANALYSIS : 62.9%, 3.0, 97.7ms, 0 | 20260505 17:59:14.982 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:14.982 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:14.982 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:59:14.982 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:14.982 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:14.982 | INFO | Uploaded 36GiB to GPU in 2.2057s (16.3214GiB/s) | 20260505 17:59:14.982 | INFO | Downloaded 34GiB to GPU in 2.03802s (16.6829GiB/s) | 20260505 17:59:20.132 | INFO | Analyzing chunk frames = [5187072,5187584) | 20260505 17:59:20.132 | INFO | MemInfo: { | 20260505 17:59:20.132 | INFO | "gpu" : " 17 GB", | 20260505 17:59:20.132 | INFO | "huge" : "", | 20260505 17:59:20.132 | INFO | "huge_shared" : " 27 GB", | 20260505 17:59:20.132 | INFO | "malloc" : " 14 GB", | 20260505 17:59:20.132 | INFO | "pinned" : " 29 GB" | 20260505 17:59:20.132 | INFO | } | 20260505 17:59:20.132 | INFO | Perf report for the last 5.15035 seconds: | 20260505 17:59:20.132 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:59:20.132 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:20.132 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:20.132 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:20.132 | INFO | ANALYSIS : 62.5%, 3.0, 97.2ms, 0 | 20260505 17:59:20.132 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:20.132 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:20.132 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260505 17:59:20.132 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:20.132 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:20.133 | INFO | Uploaded 36GiB to GPU in 2.21362s (16.263GiB/s) | 20260505 17:59:20.133 | INFO | Downloaded 34GiB to GPU in 2.14212s (15.8721GiB/s) | 20260505 17:59:25.273 | INFO | Analyzing chunk frames = [5187584,5188096) | 20260505 17:59:25.273 | INFO | MemInfo: { | 20260505 17:59:25.273 | INFO | "gpu" : " 17 GB", | 20260505 17:59:25.273 | INFO | "huge" : "", | 20260505 17:59:25.273 | INFO | "huge_shared" : " 27 GB", | 20260505 17:59:25.273 | INFO | "malloc" : " 14 GB", | 20260505 17:59:25.273 | INFO | "pinned" : " 29 GB" | 20260505 17:59:25.273 | INFO | } | 20260505 17:59:25.273 | INFO | Perf report for the last 5.14105 seconds: | 20260505 17:59:25.273 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:59:25.273 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:25.273 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:25.273 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:25.273 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 17:59:25.273 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:25.273 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:25.274 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 17:59:25.274 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:25.274 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:25.274 | INFO | Uploaded 36GiB to GPU in 2.14233s (16.8042GiB/s) | 20260505 17:59:25.274 | INFO | Downloaded 34GiB to GPU in 2.06035s (16.5021GiB/s) | 20260505 17:59:30.409 | INFO | Analyzing chunk frames = [5188096,5188608) | 20260505 17:59:30.409 | INFO | MemInfo: { | 20260505 17:59:30.409 | INFO | "gpu" : " 17 GB", | 20260505 17:59:30.409 | INFO | "huge" : "", | 20260505 17:59:30.409 | INFO | "huge_shared" : " 27 GB", | 20260505 17:59:30.409 | INFO | "malloc" : " 14 GB", | 20260505 17:59:30.409 | INFO | "pinned" : " 29 GB" | 20260505 17:59:30.409 | INFO | } | 20260505 17:59:30.409 | INFO | Perf report for the last 5.13565 seconds: | 20260505 17:59:30.409 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:59:30.409 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:30.409 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:30.409 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:30.409 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260505 17:59:30.409 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:30.409 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:30.409 | INFO | PRE_HQ : 27.5%, 1.0, 14.2ms, 0 | 20260505 17:59:30.409 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:30.409 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:30.409 | INFO | Uploaded 36GiB to GPU in 2.14227s (16.8046GiB/s) | 20260505 17:59:30.409 | INFO | Downloaded 34GiB to GPU in 2.05437s (16.5501GiB/s) | 20260505 17:59:35.540 | INFO | Analyzing chunk frames = [5188608,5189120) | 20260505 17:59:35.540 | INFO | MemInfo: { | 20260505 17:59:35.540 | INFO | "gpu" : " 17 GB", | 20260505 17:59:35.540 | INFO | "huge" : "", | 20260505 17:59:35.540 | INFO | "huge_shared" : " 27 GB", | 20260505 17:59:35.540 | INFO | "malloc" : " 14 GB", | 20260505 17:59:35.540 | INFO | "pinned" : " 29 GB" | 20260505 17:59:35.540 | INFO | } | 20260505 17:59:35.540 | INFO | Perf report for the last 5.13099 seconds: | 20260505 17:59:35.540 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:59:35.540 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:35.540 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:35.540 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:35.540 | INFO | ANALYSIS : 63.4%, 3.0, 98.8ms, 0 | 20260505 17:59:35.540 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:35.540 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:35.540 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 17:59:35.540 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:35.540 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:35.540 | INFO | Uploaded 36GiB to GPU in 2.13419s (16.8682GiB/s) | 20260505 17:59:35.540 | INFO | Downloaded 34GiB to GPU in 2.13574s (15.9196GiB/s) | 20260505 17:59:40.679 | INFO | Analyzing chunk frames = [5189120,5189632) | 20260505 17:59:40.680 | INFO | MemInfo: { | 20260505 17:59:40.680 | INFO | "gpu" : " 17 GB", | 20260505 17:59:40.680 | INFO | "huge" : "", | 20260505 17:59:40.680 | INFO | "huge_shared" : " 27 GB", | 20260505 17:59:40.680 | INFO | "malloc" : " 14 GB", | 20260505 17:59:40.680 | INFO | "pinned" : " 29 GB" | 20260505 17:59:40.680 | INFO | } | 20260505 17:59:40.680 | INFO | Perf report for the last 5.13966 seconds: | 20260505 17:59:40.680 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:59:40.680 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:40.680 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:40.680 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:40.680 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 17:59:40.680 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:40.680 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:40.680 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 17:59:40.680 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:40.680 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:40.680 | INFO | Uploaded 36GiB to GPU in 2.17242s (16.5714GiB/s) | 20260505 17:59:40.680 | INFO | Downloaded 34GiB to GPU in 2.03582s (16.7009GiB/s) | 20260505 17:59:40.821 | INFO | Starting a round of Baz IO | 20260505 17:59:44.010 | INFO | Populating RTMetrics report | 20260505 17:59:44.010 | INFO | RTMetrics report populated | 20260505 17:59:44.010 | INFO | Opening tmp RTMetrics json file | 20260505 17:59:44.017 | INFO | Serializing RTMetrics to json | 20260505 17:59:44.019 | INFO | Serialization done, swapping tmp file now... | 20260505 17:59:44.019 | INFO | RTMetrics done | 20260505 17:59:44.106 | INFO | Finished a round of Baz IO: Wrote 2.111 GiB in 3.285 seconds | 20260505 17:59:44.106 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.97% / 0.00% / 78.02% | 20260505 17:59:44.107 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.271968% / 3.97364e-06% / 7.15256e-05% | 20260505 17:59:44.107 | INFO | Baz IO overall: | 20260505 17:59:44.107 | INFO | BytesWritten : 3061490748222 | 20260505 17:59:44.107 | INFO | NumEvents : 1450967514922 | 20260505 17:59:45.825 | INFO | Analyzing chunk frames = [5189632,5190144) | 20260505 17:59:45.825 | INFO | MemInfo: { | 20260505 17:59:45.825 | INFO | "gpu" : " 17 GB", | 20260505 17:59:45.825 | INFO | "huge" : "", | 20260505 17:59:45.825 | INFO | "huge_shared" : " 27 GB", | 20260505 17:59:45.825 | INFO | "malloc" : " 14 GB", | 20260505 17:59:45.825 | INFO | "pinned" : " 29 GB" | 20260505 17:59:45.825 | INFO | } | 20260505 17:59:45.825 | INFO | Perf report for the last 5.14564 seconds: | 20260505 17:59:45.825 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:59:45.825 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:45.825 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:45.825 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:45.825 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 17:59:45.825 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:45.825 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260505 17:59:45.825 | INFO | PRE_HQ : 60.4%, 1.8, 56.0ms, 0 | 20260505 17:59:45.825 | INFO | BAZWRITER : 63.3%, 3.8, 123.3ms, 0 | 20260505 17:59:45.826 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 17:59:45.826 | INFO | Uploaded 36GiB to GPU in 2.16011s (16.6659GiB/s) | 20260505 17:59:45.826 | INFO | Downloaded 38GiB to GPU in 2.25273s (16.8684GiB/s) | 20260505 17:59:50.969 | INFO | Analyzing chunk frames = [5190144,5190656) | 20260505 17:59:50.969 | INFO | MemInfo: { | 20260505 17:59:50.969 | INFO | "gpu" : " 17 GB", | 20260505 17:59:50.969 | INFO | "huge" : "", | 20260505 17:59:50.969 | INFO | "huge_shared" : " 27 GB", | 20260505 17:59:50.969 | INFO | "malloc" : " 14 GB", | 20260505 17:59:50.969 | INFO | "pinned" : " 29 GB" | 20260505 17:59:50.969 | INFO | } | 20260505 17:59:50.969 | INFO | Perf report for the last 5.14398 seconds: | 20260505 17:59:50.969 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:59:50.969 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:50.969 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:50.969 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:50.969 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 17:59:50.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:50.969 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:50.969 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260505 17:59:50.969 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:50.969 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:50.970 | INFO | Uploaded 36GiB to GPU in 2.14032s (16.8199GiB/s) | 20260505 17:59:50.970 | INFO | Downloaded 34GiB to GPU in 2.03947s (16.671GiB/s) | 20260505 17:59:56.109 | INFO | Analyzing chunk frames = [5190656,5191168) | 20260505 17:59:56.109 | INFO | MemInfo: { | 20260505 17:59:56.109 | INFO | "gpu" : " 17 GB", | 20260505 17:59:56.109 | INFO | "huge" : "", | 20260505 17:59:56.109 | INFO | "huge_shared" : " 27 GB", | 20260505 17:59:56.109 | INFO | "malloc" : " 14 GB", | 20260505 17:59:56.109 | INFO | "pinned" : " 29 GB" | 20260505 17:59:56.109 | INFO | } | 20260505 17:59:56.109 | INFO | Perf report for the last 5.13963 seconds: | 20260505 17:59:56.109 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 17:59:56.109 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:56.109 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:56.109 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:56.109 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260505 17:59:56.109 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:56.109 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:56.109 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 17:59:56.109 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:56.109 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 17:59:56.109 | INFO | Uploaded 36GiB to GPU in 2.13023s (16.8996GiB/s) | 20260505 17:59:56.109 | INFO | Downloaded 34GiB to GPU in 2.03126s (16.7384GiB/s) | 20260505 18:00:01.242 | INFO | Analyzing chunk frames = [5191168,5191680) | 20260505 18:00:01.242 | INFO | MemInfo: { | 20260505 18:00:01.242 | INFO | "gpu" : " 17 GB", | 20260505 18:00:01.242 | INFO | "huge" : "", | 20260505 18:00:01.242 | INFO | "huge_shared" : " 27 GB", | 20260505 18:00:01.242 | INFO | "malloc" : " 14 GB", | 20260505 18:00:01.242 | INFO | "pinned" : " 29 GB" | 20260505 18:00:01.242 | INFO | } | 20260505 18:00:01.242 | INFO | Perf report for the last 5.13345 seconds: | 20260505 18:00:01.242 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:00:01.242 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:01.242 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:01.243 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:01.243 | INFO | ANALYSIS : 62.9%, 3.0, 97.6ms, 0 | 20260505 18:00:01.243 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:01.243 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:01.243 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:00:01.243 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:01.243 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:01.243 | INFO | Uploaded 36GiB to GPU in 2.22808s (16.1574GiB/s) | 20260505 18:00:01.243 | INFO | Downloaded 34GiB to GPU in 2.10982s (16.1151GiB/s) | 20260505 18:00:06.387 | INFO | Analyzing chunk frames = [5191680,5192192) | 20260505 18:00:06.387 | INFO | MemInfo: { | 20260505 18:00:06.387 | INFO | "gpu" : " 17 GB", | 20260505 18:00:06.387 | INFO | "huge" : "", | 20260505 18:00:06.387 | INFO | "huge_shared" : " 27 GB", | 20260505 18:00:06.387 | INFO | "malloc" : " 14 GB", | 20260505 18:00:06.387 | INFO | "pinned" : " 29 GB" | 20260505 18:00:06.387 | INFO | } | 20260505 18:00:06.387 | INFO | Perf report for the last 5.14499 seconds: | 20260505 18:00:06.387 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:00:06.387 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:06.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:06.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:06.388 | INFO | ANALYSIS : 62.1%, 3.0, 97.1ms, 0 | 20260505 18:00:06.388 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:06.388 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:06.388 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:00:06.388 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:06.388 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:06.388 | INFO | Uploaded 36GiB to GPU in 2.15461s (16.7083GiB/s) | 20260505 18:00:06.388 | INFO | Downloaded 34GiB to GPU in 2.06096s (16.4972GiB/s) | 20260505 18:00:11.525 | INFO | Analyzing chunk frames = [5192192,5192704) | 20260505 18:00:11.525 | INFO | MemInfo: { | 20260505 18:00:11.525 | INFO | "gpu" : " 17 GB", | 20260505 18:00:11.525 | INFO | "huge" : "", | 20260505 18:00:11.525 | INFO | "huge_shared" : " 27 GB", | 20260505 18:00:11.525 | INFO | "malloc" : " 14 GB", | 20260505 18:00:11.525 | INFO | "pinned" : " 29 GB" | 20260505 18:00:11.525 | INFO | } | 20260505 18:00:11.525 | INFO | Perf report for the last 5.13772 seconds: | 20260505 18:00:11.525 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:00:11.525 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:11.525 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:11.525 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:11.525 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 18:00:11.525 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:11.525 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:11.525 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 18:00:11.525 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:11.525 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:11.526 | INFO | Uploaded 36GiB to GPU in 2.13678s (16.8478GiB/s) | 20260505 18:00:11.526 | INFO | Downloaded 34GiB to GPU in 2.04686s (16.6108GiB/s) | 20260505 18:00:16.660 | INFO | Analyzing chunk frames = [5192704,5193216) | 20260505 18:00:16.660 | INFO | MemInfo: { | 20260505 18:00:16.660 | INFO | "gpu" : " 17 GB", | 20260505 18:00:16.660 | INFO | "huge" : "", | 20260505 18:00:16.660 | INFO | "huge_shared" : " 27 GB", | 20260505 18:00:16.660 | INFO | "malloc" : " 14 GB", | 20260505 18:00:16.660 | INFO | "pinned" : " 29 GB" | 20260505 18:00:16.660 | INFO | } | 20260505 18:00:16.660 | INFO | Perf report for the last 5.13488 seconds: | 20260505 18:00:16.660 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:00:16.660 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:16.660 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:16.660 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:16.660 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 18:00:16.660 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:16.660 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:16.660 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 18:00:16.660 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:16.660 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:16.660 | INFO | Uploaded 36GiB to GPU in 2.12721s (16.9236GiB/s) | 20260505 18:00:16.660 | INFO | Downloaded 34GiB to GPU in 2.1248s (16.0015GiB/s) | 20260505 18:00:21.798 | INFO | Analyzing chunk frames = [5193216,5193728) | 20260505 18:00:21.798 | INFO | MemInfo: { | 20260505 18:00:21.798 | INFO | "gpu" : " 17 GB", | 20260505 18:00:21.798 | INFO | "huge" : "", | 20260505 18:00:21.798 | INFO | "huge_shared" : " 27 GB", | 20260505 18:00:21.798 | INFO | "malloc" : " 14 GB", | 20260505 18:00:21.798 | INFO | "pinned" : " 29 GB" | 20260505 18:00:21.798 | INFO | } | 20260505 18:00:21.799 | INFO | Perf report for the last 5.13848 seconds: | 20260505 18:00:21.799 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:00:21.799 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:21.799 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:21.799 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:21.799 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 18:00:21.799 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:21.799 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:21.799 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 18:00:21.799 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:21.799 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:21.799 | INFO | Uploaded 36GiB to GPU in 2.19926s (16.3692GiB/s) | 20260505 18:00:21.799 | INFO | Downloaded 34GiB to GPU in 2.03514s (16.7065GiB/s) | 20260505 18:00:21.799 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 18:00:21.799 | INFO | Total mem usage: 15.3 GiB | 20260505 18:00:21.799 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 18:00:21.799 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8%) | 20260505 18:00:21.799 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 18:00:21.799 | INFO | High Water Breakdown: | 20260505 18:00:21.799 | INFO | 61% : 8.66 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 18:00:21.799 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 18:00:21.799 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 18:00:21.799 | INFO | ------------------------------------------ | 20260505 18:00:21.799 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:00:21.799 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 18:00:21.799 | INFO | Total mem usage: 26.5 GiB | 20260505 18:00:21.799 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 18:00:21.799 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 18:00:21.799 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 18:00:21.799 | INFO | High Water Breakdown: | 20260505 18:00:21.799 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:00:21.799 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:00:21.799 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:00:21.799 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:00:21.799 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:00:21.799 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:00:21.799 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 18:00:21.799 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 18:00:21.799 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:00:21.799 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 18:00:21.799 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 18:00:21.799 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 18:00:21.799 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 18:00:21.799 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 18:00:21.799 | INFO | ------------------------------------------ | 20260505 18:00:21.799 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:00:21.800 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 18:00:21.800 | INFO | Total mem usage: 25.6 GiB | 20260505 18:00:21.800 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 18:00:21.800 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 18:00:21.800 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 18:00:21.800 | INFO | High Water Breakdown: | 20260505 18:00:21.800 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 18:00:21.800 | INFO | ------------------------------------------ | 20260505 18:00:21.800 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:00:21.800 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 18:00:21.800 | INFO | Total mem usage: 15.7 GiB | 20260505 18:00:21.800 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 18:00:21.800 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 18:00:21.800 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 18:00:21.800 | INFO | High Water Breakdown: | 20260505 18:00:21.800 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 18:00:21.800 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 18:00:21.800 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 18:00:21.800 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 18:00:21.800 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 18:00:21.800 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 18:00:21.800 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 18:00:21.800 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 18:00:21.800 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 18:00:21.800 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:00:21.800 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 18:00:21.800 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:00:21.800 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 18:00:21.800 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 18:00:21.800 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 18:00:21.800 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 18:00:21.800 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:00:21.800 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 18:00:21.800 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:00:21.800 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:00:21.800 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:00:21.800 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 18:00:21.800 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 18:00:21.800 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:00:21.800 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 18:00:21.800 | INFO | ------------------------------------------ | 20260505 18:00:21.800 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:00:21.800 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 18:00:21.800 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:00:21.800 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:00:21.800 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.48e+01, 1.36e+01 / 1.94e+00 | 20260505 18:00:21.800 | INFO | PerfReport: 0.29%, AggregateStats -- 9.25e-02 / 1.03e-01, 1.10e-02 / 6.24e-03 | 20260505 18:00:21.800 | INFO | PerfReport: 11.76%, Baseline -- 3.71e+00 / 4.53e+00, 5.60e-01 / 1.17e-01 | 20260505 18:00:21.800 | INFO | PerfReport: 3.26%, Histogram -- 1.03e+00 / 1.52e+00, 1.90e-01 / 9.57e-02 | 20260505 18:00:21.800 | INFO | PerfReport: 15.97%, DME -- 4.03e+01 / 4.47e+01, 4.12e+00 / 1.94e+00 | 20260505 18:00:21.800 | INFO | PerfReport: 56.86%, FrameLabeling -- 1.79e+01 / 1.92e+01, 1.44e+00 / 9.55e-01 | 20260505 18:00:21.800 | INFO | PerfReport: 7.33%, PulseAccumulating -- 2.31e+00 / 2.91e+00, 4.45e-01 / 1.48e-01 | 20260505 18:00:21.800 | INFO | PerfReport: 3.35%, Metrics -- 1.06e+00 / 1.38e+00, 2.90e-01 / 1.42e-01 | 20260505 18:00:21.800 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.69e-01 / 4.40e-01, 6.58e-02 / 4.42e-02 | 20260505 18:00:21.800 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 18:00:21.800 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:00:21.800 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:00:21.800 | INFO | PerfReport: Full Execution -- 9.99e+01 / -nan, 2.57e+01 / 0.00e+00 | 20260505 18:00:21.800 | INFO | PerfReport: 22.54%, Upload -- 2.25e+01 / 0.00e+00, 7.94e+00 / 0.00e+00 | 20260505 18:00:21.800 | INFO | PerfReport: 21.79%, Download -- 2.18e+01 / 0.00e+00, 7.80e+00 / 0.00e+00 | 20260505 18:00:21.800 | INFO | PerfReport: 31.58%, Compute -- 3.16e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 18:00:21.800 | INFO | PerfReport: 2.42%, LockUpload -- 2.42e+00 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260505 18:00:21.800 | INFO | PerfReport: 1.33%, LockDownload -- 1.33e+00 / 0.00e+00, 2.81e+00 / 0.00e+00 | 20260505 18:00:21.800 | INFO | PerfReport: 20.33%, LockCompute -- 2.03e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260505 18:00:21.800 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.98e-02 / 0.00e+00 | 20260505 18:00:21.940 | INFO | Starting a round of Baz IO | 20260505 18:00:25.126 | INFO | Populating RTMetrics report | 20260505 18:00:25.126 | INFO | RTMetrics report populated | 20260505 18:00:25.126 | INFO | Opening tmp RTMetrics json file | 20260505 18:00:25.138 | INFO | Serializing RTMetrics to json | 20260505 18:00:25.144 | INFO | Serialization done, swapping tmp file now... | 20260505 18:00:25.156 | INFO | RTMetrics done | 20260505 18:00:25.222 | INFO | Finished a round of Baz IO: Wrote 2.11 GiB in 3.281 seconds | 20260505 18:00:25.222 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.97% / 0.00% / 78.03% | 20260505 18:00:25.222 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.271149% / 3.97364e-06% / 0.000111262% | 20260505 18:00:25.222 | INFO | Baz IO overall: | 20260505 18:00:25.222 | INFO | BytesWritten : 3063755984665 | 20260505 18:00:25.222 | INFO | NumEvents : 1452024962747 | 20260505 18:00:26.933 | INFO | Analyzing chunk frames = [5193728,5194240) | 20260505 18:00:26.933 | INFO | MemInfo: { | 20260505 18:00:26.933 | INFO | "gpu" : " 17 GB", | 20260505 18:00:26.933 | INFO | "huge" : "", | 20260505 18:00:26.933 | INFO | "huge_shared" : " 27 GB", | 20260505 18:00:26.933 | INFO | "malloc" : " 14 GB", | 20260505 18:00:26.933 | INFO | "pinned" : " 29 GB" | 20260505 18:00:26.933 | INFO | } | 20260505 18:00:26.933 | INFO | Perf report for the last 5.13465 seconds: | 20260505 18:00:26.933 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:00:26.933 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:26.933 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:26.933 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:26.933 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 18:00:26.933 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:26.933 | INFO | RT_METRICS : 44.9%, 1.0, 23.5ms, 0 | 20260505 18:00:26.934 | INFO | PRE_HQ : 60.5%, 1.8, 56.1ms, 0 | 20260505 18:00:26.934 | INFO | BAZWRITER : 63.3%, 3.8, 123.8ms, 0 | 20260505 18:00:26.934 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260505 18:00:26.934 | INFO | Uploaded 36GiB to GPU in 2.17278s (16.5687GiB/s) | 20260505 18:00:26.934 | INFO | Downloaded 38GiB to GPU in 2.25445s (16.8556GiB/s) | 20260505 18:00:32.080 | INFO | Analyzing chunk frames = [5194240,5194752) | 20260505 18:00:32.080 | INFO | MemInfo: { | 20260505 18:00:32.080 | INFO | "gpu" : " 17 GB", | 20260505 18:00:32.080 | INFO | "huge" : "", | 20260505 18:00:32.080 | INFO | "huge_shared" : " 27 GB", | 20260505 18:00:32.080 | INFO | "malloc" : " 14 GB", | 20260505 18:00:32.080 | INFO | "pinned" : " 29 GB" | 20260505 18:00:32.081 | INFO | } | 20260505 18:00:32.081 | INFO | Perf report for the last 5.1474 seconds: | 20260505 18:00:32.081 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:00:32.081 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:32.081 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:32.081 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:32.081 | INFO | ANALYSIS : 62.6%, 3.0, 97.9ms, 0 | 20260505 18:00:32.081 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:32.081 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:32.081 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:00:32.081 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:32.081 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:32.081 | INFO | Uploaded 36GiB to GPU in 2.1227s (16.9595GiB/s) | 20260505 18:00:32.081 | INFO | Downloaded 34GiB to GPU in 2.02057s (16.827GiB/s) | 20260505 18:00:37.227 | INFO | Analyzing chunk frames = [5194752,5195264) | 20260505 18:00:37.227 | INFO | MemInfo: { | 20260505 18:00:37.227 | INFO | "gpu" : " 17 GB", | 20260505 18:00:37.227 | INFO | "huge" : "", | 20260505 18:00:37.227 | INFO | "huge_shared" : " 27 GB", | 20260505 18:00:37.227 | INFO | "malloc" : " 14 GB", | 20260505 18:00:37.227 | INFO | "pinned" : " 29 GB" | 20260505 18:00:37.227 | INFO | } | 20260505 18:00:37.227 | INFO | Perf report for the last 5.14626 seconds: | 20260505 18:00:37.227 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:00:37.227 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:37.227 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:37.227 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:37.227 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260505 18:00:37.227 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:37.227 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:37.227 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:00:37.227 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:37.227 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:37.227 | INFO | Uploaded 36GiB to GPU in 2.13664s (16.8489GiB/s) | 20260505 18:00:37.227 | INFO | Downloaded 34GiB to GPU in 2.03614s (16.6982GiB/s) | 20260505 18:00:42.357 | INFO | Analyzing chunk frames = [5195264,5195776) | 20260505 18:00:42.357 | INFO | MemInfo: { | 20260505 18:00:42.357 | INFO | "gpu" : " 17 GB", | 20260505 18:00:42.357 | INFO | "huge" : "", | 20260505 18:00:42.357 | INFO | "huge_shared" : " 27 GB", | 20260505 18:00:42.357 | INFO | "malloc" : " 14 GB", | 20260505 18:00:42.357 | INFO | "pinned" : " 29 GB" | 20260505 18:00:42.357 | INFO | } | 20260505 18:00:42.357 | INFO | Perf report for the last 5.12994 seconds: | 20260505 18:00:42.357 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:00:42.357 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:42.357 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:42.357 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:42.357 | INFO | ANALYSIS : 62.4%, 3.0, 96.8ms, 0 | 20260505 18:00:42.357 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:42.357 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:42.357 | INFO | PRE_HQ : 27.7%, 1.0, 14.3ms, 0 | 20260505 18:00:42.357 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:42.357 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:42.357 | INFO | Uploaded 36GiB to GPU in 2.16784s (16.6064GiB/s) | 20260505 18:00:42.357 | INFO | Downloaded 34GiB to GPU in 2.03754s (16.6867GiB/s) | 20260505 18:00:47.490 | INFO | Analyzing chunk frames = [5195776,5196288) | 20260505 18:00:47.490 | INFO | MemInfo: { | 20260505 18:00:47.490 | INFO | "gpu" : " 17 GB", | 20260505 18:00:47.490 | INFO | "huge" : "", | 20260505 18:00:47.490 | INFO | "huge_shared" : " 27 GB", | 20260505 18:00:47.490 | INFO | "malloc" : " 14 GB", | 20260505 18:00:47.490 | INFO | "pinned" : " 29 GB" | 20260505 18:00:47.490 | INFO | } | 20260505 18:00:47.490 | INFO | Perf report for the last 5.13311 seconds: | 20260505 18:00:47.490 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:00:47.490 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:47.490 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:47.490 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:47.490 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 18:00:47.490 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:47.490 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:47.490 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 18:00:47.490 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:47.490 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:47.490 | INFO | Uploaded 36GiB to GPU in 2.14828s (16.7576GiB/s) | 20260505 18:00:47.490 | INFO | Downloaded 34GiB to GPU in 2.0625s (16.4849GiB/s) | 20260505 18:00:52.636 | INFO | Analyzing chunk frames = [5196288,5196800) | 20260505 18:00:52.636 | INFO | MemInfo: { | 20260505 18:00:52.636 | INFO | "gpu" : " 17 GB", | 20260505 18:00:52.636 | INFO | "huge" : "", | 20260505 18:00:52.636 | INFO | "huge_shared" : " 27 GB", | 20260505 18:00:52.636 | INFO | "malloc" : " 14 GB", | 20260505 18:00:52.636 | INFO | "pinned" : " 29 GB" | 20260505 18:00:52.636 | INFO | } | 20260505 18:00:52.636 | INFO | Perf report for the last 5.14628 seconds: | 20260505 18:00:52.636 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:00:52.636 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:52.636 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:52.636 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:52.636 | INFO | ANALYSIS : 62.4%, 3.0, 97.6ms, 0 | 20260505 18:00:52.636 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:52.636 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:52.636 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 18:00:52.636 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:52.636 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:52.637 | INFO | Uploaded 36GiB to GPU in 2.13838s (16.8352GiB/s) | 20260505 18:00:52.637 | INFO | Downloaded 34GiB to GPU in 2.04385s (16.6353GiB/s) | 20260505 18:00:57.780 | INFO | Analyzing chunk frames = [5196800,5197312) | 20260505 18:00:57.780 | INFO | MemInfo: { | 20260505 18:00:57.780 | INFO | "gpu" : " 17 GB", | 20260505 18:00:57.780 | INFO | "huge" : "", | 20260505 18:00:57.780 | INFO | "huge_shared" : " 27 GB", | 20260505 18:00:57.780 | INFO | "malloc" : " 14 GB", | 20260505 18:00:57.780 | INFO | "pinned" : " 29 GB" | 20260505 18:00:57.781 | INFO | } | 20260505 18:00:57.781 | INFO | Perf report for the last 5.14427 seconds: | 20260505 18:00:57.781 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:00:57.781 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:57.781 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:57.781 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:57.781 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 18:00:57.781 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:57.781 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:57.781 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 18:00:57.781 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:57.781 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:00:57.781 | INFO | Uploaded 36GiB to GPU in 2.14258s (16.8022GiB/s) | 20260505 18:00:57.781 | INFO | Downloaded 34GiB to GPU in 2.14971s (15.8161GiB/s) | 20260505 18:01:02.915 | INFO | Analyzing chunk frames = [5197312,5197824) | 20260505 18:01:02.915 | INFO | MemInfo: { | 20260505 18:01:02.915 | INFO | "gpu" : " 17 GB", | 20260505 18:01:02.915 | INFO | "huge" : "", | 20260505 18:01:02.915 | INFO | "huge_shared" : " 27 GB", | 20260505 18:01:02.915 | INFO | "malloc" : " 14 GB", | 20260505 18:01:02.915 | INFO | "pinned" : " 29 GB" | 20260505 18:01:02.915 | INFO | } | 20260505 18:01:02.916 | INFO | Perf report for the last 5.13499 seconds: | 20260505 18:01:02.916 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:01:02.916 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:02.916 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:02.916 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:02.916 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 18:01:02.916 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:02.916 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:02.916 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 18:01:02.916 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:02.916 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:02.916 | INFO | Uploaded 36GiB to GPU in 2.1943s (16.4062GiB/s) | 20260505 18:01:02.916 | INFO | Downloaded 34GiB to GPU in 2.03217s (16.7308GiB/s) | 20260505 18:01:03.051 | INFO | Starting a round of Baz IO | 20260505 18:01:06.227 | INFO | Populating RTMetrics report | 20260505 18:01:06.227 | INFO | RTMetrics report populated | 20260505 18:01:06.227 | INFO | Opening tmp RTMetrics json file | 20260505 18:01:06.229 | INFO | Serializing RTMetrics to json | 20260505 18:01:06.230 | INFO | Serialization done, swapping tmp file now... | 20260505 18:01:06.231 | INFO | RTMetrics done | 20260505 18:01:06.321 | INFO | Finished a round of Baz IO: Wrote 2.108 GiB in 3.27 seconds | 20260505 18:01:06.321 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.97% / 0.00% / 78.03% | 20260505 18:01:06.321 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.270903% / 0% / 0.000107288% | 20260505 18:01:06.321 | INFO | Baz IO overall: | 20260505 18:01:06.321 | INFO | BytesWritten : 3066019223822 | 20260505 18:01:06.321 | INFO | NumEvents : 1453080990226 | 20260505 18:01:08.055 | INFO | Analyzing chunk frames = [5197824,5198336) | 20260505 18:01:08.055 | INFO | MemInfo: { | 20260505 18:01:08.055 | INFO | "gpu" : " 17 GB", | 20260505 18:01:08.055 | INFO | "huge" : "", | 20260505 18:01:08.055 | INFO | "huge_shared" : " 27 GB", | 20260505 18:01:08.055 | INFO | "malloc" : " 14 GB", | 20260505 18:01:08.055 | INFO | "pinned" : " 29 GB" | 20260505 18:01:08.055 | INFO | } | 20260505 18:01:08.055 | INFO | Perf report for the last 5.13918 seconds: | 20260505 18:01:08.055 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:01:08.055 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:08.055 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:08.055 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:08.055 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 18:01:08.055 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:08.055 | INFO | RT_METRICS : 44.3%, 1.0, 23.3ms, 0 | 20260505 18:01:08.055 | INFO | PRE_HQ : 60.4%, 1.8, 55.8ms, 0 | 20260505 18:01:08.055 | INFO | BAZWRITER : 63.1%, 3.8, 123.2ms, 0 | 20260505 18:01:08.055 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 18:01:08.055 | INFO | Uploaded 36GiB to GPU in 2.16356s (16.6392GiB/s) | 20260505 18:01:08.055 | INFO | Downloaded 38GiB to GPU in 2.24353s (16.9376GiB/s) | 20260505 18:01:13.192 | INFO | Analyzing chunk frames = [5198336,5198848) | 20260505 18:01:13.192 | INFO | MemInfo: { | 20260505 18:01:13.192 | INFO | "gpu" : " 17 GB", | 20260505 18:01:13.192 | INFO | "huge" : "", | 20260505 18:01:13.192 | INFO | "huge_shared" : " 27 GB", | 20260505 18:01:13.192 | INFO | "malloc" : " 14 GB", | 20260505 18:01:13.192 | INFO | "pinned" : " 29 GB" | 20260505 18:01:13.192 | INFO | } | 20260505 18:01:13.192 | INFO | Perf report for the last 5.13769 seconds: | 20260505 18:01:13.192 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:01:13.192 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:13.193 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:13.193 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:13.193 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 18:01:13.193 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:13.193 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:13.193 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:01:13.193 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:13.193 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:13.193 | INFO | Uploaded 36GiB to GPU in 2.13096s (16.8938GiB/s) | 20260505 18:01:13.193 | INFO | Downloaded 34GiB to GPU in 2.02842s (16.7618GiB/s) | 20260505 18:01:18.335 | INFO | Analyzing chunk frames = [5198848,5199360) | 20260505 18:01:18.335 | INFO | MemInfo: { | 20260505 18:01:18.335 | INFO | "gpu" : " 17 GB", | 20260505 18:01:18.335 | INFO | "huge" : "", | 20260505 18:01:18.335 | INFO | "huge_shared" : " 27 GB", | 20260505 18:01:18.335 | INFO | "malloc" : " 14 GB", | 20260505 18:01:18.335 | INFO | "pinned" : " 29 GB" | 20260505 18:01:18.335 | INFO | } | 20260505 18:01:18.335 | INFO | Perf report for the last 5.14269 seconds: | 20260505 18:01:18.335 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:01:18.335 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:18.335 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:18.335 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:18.335 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 18:01:18.335 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:18.335 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:18.335 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:01:18.335 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:18.335 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:18.335 | INFO | Uploaded 36GiB to GPU in 2.20867s (16.2994GiB/s) | 20260505 18:01:18.335 | INFO | Downloaded 34GiB to GPU in 2.04908s (16.5928GiB/s) | 20260505 18:01:23.474 | INFO | Analyzing chunk frames = [5199360,5199872) | 20260505 18:01:23.474 | INFO | MemInfo: { | 20260505 18:01:23.474 | INFO | "gpu" : " 17 GB", | 20260505 18:01:23.474 | INFO | "huge" : "", | 20260505 18:01:23.474 | INFO | "huge_shared" : " 27 GB", | 20260505 18:01:23.474 | INFO | "malloc" : " 14 GB", | 20260505 18:01:23.474 | INFO | "pinned" : " 29 GB" | 20260505 18:01:23.474 | INFO | } | 20260505 18:01:23.475 | INFO | Perf report for the last 5.13939 seconds: | 20260505 18:01:23.475 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:01:23.475 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:23.475 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:23.475 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:23.475 | INFO | ANALYSIS : 62.9%, 3.0, 97.7ms, 0 | 20260505 18:01:23.475 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:23.475 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:23.475 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:01:23.475 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:23.475 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:23.475 | INFO | Uploaded 36GiB to GPU in 2.22708s (16.1646GiB/s) | 20260505 18:01:23.475 | INFO | Downloaded 34GiB to GPU in 2.11866s (16.0479GiB/s) | 20260505 18:01:28.610 | INFO | Analyzing chunk frames = [5199872,5200384) | 20260505 18:01:28.610 | INFO | MemInfo: { | 20260505 18:01:28.610 | INFO | "gpu" : " 17 GB", | 20260505 18:01:28.610 | INFO | "huge" : "", | 20260505 18:01:28.610 | INFO | "huge_shared" : " 27 GB", | 20260505 18:01:28.610 | INFO | "malloc" : " 14 GB", | 20260505 18:01:28.610 | INFO | "pinned" : " 29 GB" | 20260505 18:01:28.610 | INFO | } | 20260505 18:01:28.610 | INFO | Perf report for the last 5.1358 seconds: | 20260505 18:01:28.610 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:01:28.610 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:28.610 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:28.610 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:28.610 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 18:01:28.611 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:28.611 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:28.611 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 18:01:28.611 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:28.611 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:28.611 | INFO | Uploaded 36GiB to GPU in 2.13684s (16.8473GiB/s) | 20260505 18:01:28.611 | INFO | Downloaded 34GiB to GPU in 2.059s (16.5128GiB/s) | 20260505 18:01:33.748 | INFO | Analyzing chunk frames = [5200384,5200896) | 20260505 18:01:33.748 | INFO | MemInfo: { | 20260505 18:01:33.748 | INFO | "gpu" : " 17 GB", | 20260505 18:01:33.748 | INFO | "huge" : "", | 20260505 18:01:33.748 | INFO | "huge_shared" : " 27 GB", | 20260505 18:01:33.748 | INFO | "malloc" : " 14 GB", | 20260505 18:01:33.748 | INFO | "pinned" : " 29 GB" | 20260505 18:01:33.748 | INFO | } | 20260505 18:01:33.748 | INFO | Perf report for the last 5.13756 seconds: | 20260505 18:01:33.748 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:01:33.748 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:33.748 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:33.748 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:33.748 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260505 18:01:33.748 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:33.748 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:33.748 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:01:33.748 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:33.748 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:33.748 | INFO | Uploaded 36GiB to GPU in 2.14143s (16.8112GiB/s) | 20260505 18:01:33.748 | INFO | Downloaded 34GiB to GPU in 2.04371s (16.6364GiB/s) | 20260505 18:01:38.895 | INFO | Analyzing chunk frames = [5200896,5201408) | 20260505 18:01:38.895 | INFO | MemInfo: { | 20260505 18:01:38.895 | INFO | "gpu" : " 17 GB", | 20260505 18:01:38.895 | INFO | "huge" : "", | 20260505 18:01:38.895 | INFO | "huge_shared" : " 27 GB", | 20260505 18:01:38.895 | INFO | "malloc" : " 14 GB", | 20260505 18:01:38.895 | INFO | "pinned" : " 29 GB" | 20260505 18:01:38.895 | INFO | } | 20260505 18:01:38.895 | INFO | Perf report for the last 5.14717 seconds: | 20260505 18:01:38.895 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:01:38.895 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:38.895 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:38.895 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:38.895 | INFO | ANALYSIS : 63.2%, 3.0, 98.8ms, 0 | 20260505 18:01:38.895 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:38.895 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:38.895 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 18:01:38.895 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:38.895 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:38.895 | INFO | Uploaded 36GiB to GPU in 2.16073s (16.661GiB/s) | 20260505 18:01:38.896 | INFO | Downloaded 34GiB to GPU in 2.17399s (15.6394GiB/s) | 20260505 18:01:44.031 | INFO | Analyzing chunk frames = [5201408,5201920) | 20260505 18:01:44.031 | INFO | MemInfo: { | 20260505 18:01:44.031 | INFO | "gpu" : " 17 GB", | 20260505 18:01:44.031 | INFO | "huge" : "", | 20260505 18:01:44.031 | INFO | "huge_shared" : " 27 GB", | 20260505 18:01:44.031 | INFO | "malloc" : " 14 GB", | 20260505 18:01:44.031 | INFO | "pinned" : " 29 GB" | 20260505 18:01:44.031 | INFO | } | 20260505 18:01:44.031 | INFO | Perf report for the last 5.13565 seconds: | 20260505 18:01:44.031 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:01:44.031 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:44.031 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:44.031 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:44.031 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 18:01:44.031 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:44.031 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:44.031 | INFO | PRE_HQ : 27.7%, 1.0, 14.3ms, 0 | 20260505 18:01:44.031 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:44.031 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:44.031 | INFO | Uploaded 36GiB to GPU in 2.20034s (16.3611GiB/s) | 20260505 18:01:44.031 | INFO | Downloaded 34GiB to GPU in 2.0429s (16.643GiB/s) | 20260505 18:01:44.031 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 18:01:44.031 | INFO | Total mem usage: 15.3 GiB | 20260505 18:01:44.031 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 18:01:44.031 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.01%) | 20260505 18:01:44.031 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 18:01:44.031 | INFO | High Water Breakdown: | 20260505 18:01:44.031 | INFO | 61% : 8.66 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 18:01:44.031 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 18:01:44.031 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 18:01:44.031 | INFO | ------------------------------------------ | 20260505 18:01:44.031 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:01:44.031 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 18:01:44.031 | INFO | Total mem usage: 26.5 GiB | 20260505 18:01:44.031 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 18:01:44.031 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 18:01:44.031 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 18:01:44.031 | INFO | High Water Breakdown: | 20260505 18:01:44.031 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:01:44.031 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:01:44.031 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:01:44.031 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:01:44.031 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:01:44.031 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:01:44.031 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 18:01:44.031 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 18:01:44.031 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:01:44.031 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 18:01:44.031 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 18:01:44.031 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 18:01:44.031 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 18:01:44.031 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 18:01:44.031 | INFO | ------------------------------------------ | 20260505 18:01:44.032 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:01:44.032 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 18:01:44.032 | INFO | Total mem usage: 25.6 GiB | 20260505 18:01:44.032 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 18:01:44.032 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 18:01:44.032 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 18:01:44.032 | INFO | High Water Breakdown: | 20260505 18:01:44.032 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 18:01:44.032 | INFO | ------------------------------------------ | 20260505 18:01:44.032 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:01:44.032 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 18:01:44.032 | INFO | Total mem usage: 15.7 GiB | 20260505 18:01:44.032 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 18:01:44.032 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 18:01:44.032 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 18:01:44.032 | INFO | High Water Breakdown: | 20260505 18:01:44.032 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 18:01:44.032 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 18:01:44.032 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 18:01:44.032 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 18:01:44.032 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 18:01:44.032 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 18:01:44.032 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 18:01:44.032 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 18:01:44.032 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 18:01:44.032 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:01:44.032 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 18:01:44.032 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:01:44.032 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 18:01:44.032 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 18:01:44.032 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 18:01:44.032 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 18:01:44.032 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:01:44.032 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 18:01:44.032 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:01:44.032 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:01:44.032 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:01:44.032 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 18:01:44.032 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 18:01:44.032 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:01:44.032 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 18:01:44.032 | INFO | ------------------------------------------ | 20260505 18:01:44.032 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:01:44.032 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 18:01:44.032 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:01:44.032 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:01:44.032 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.49e+01, 1.36e+01 / 2.31e+00 | 20260505 18:01:44.032 | INFO | PerfReport: 0.29%, AggregateStats -- 9.26e-02 / 1.04e-01, 1.10e-02 / 5.47e-03 | 20260505 18:01:44.032 | INFO | PerfReport: 11.76%, Baseline -- 3.71e+00 / 4.52e+00, 5.61e-01 / 1.56e-01 | 20260505 18:01:44.034 | INFO | PerfReport: 3.26%, Histogram -- 1.03e+00 / 1.52e+00, 1.90e-01 / 1.07e-01 | 20260505 18:01:44.034 | INFO | PerfReport: 15.95%, DME -- 4.03e+01 / 4.48e+01, 4.17e+00 / 2.35e+00 | 20260505 18:01:44.034 | INFO | PerfReport: 56.92%, FrameLabeling -- 1.80e+01 / 1.93e+01, 1.50e+00 / 8.76e-01 | 20260505 18:01:44.034 | INFO | PerfReport: 7.31%, PulseAccumulating -- 2.31e+00 / 2.91e+00, 4.48e-01 / 1.88e-01 | 20260505 18:01:44.034 | INFO | PerfReport: 3.34%, Metrics -- 1.05e+00 / 1.38e+00, 2.92e-01 / 1.29e-01 | 20260505 18:01:44.034 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.68e-01 / 4.32e-01, 6.56e-02 / 4.19e-02 | 20260505 18:01:44.034 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 18:01:44.034 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:01:44.034 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:01:44.034 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.59e+01 / 0.00e+00 | 20260505 18:01:44.034 | INFO | PerfReport: 22.53%, Upload -- 2.25e+01 / 0.00e+00, 7.89e+00 / 0.00e+00 | 20260505 18:01:44.034 | INFO | PerfReport: 21.75%, Download -- 2.17e+01 / 0.00e+00, 7.77e+00 / 0.00e+00 | 20260505 18:01:44.034 | INFO | PerfReport: 31.62%, Compute -- 3.16e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 18:01:44.034 | INFO | PerfReport: 2.36%, LockUpload -- 2.36e+00 / 0.00e+00, 8.05e+00 / 0.00e+00 | 20260505 18:01:44.034 | INFO | PerfReport: 1.35%, LockDownload -- 1.35e+00 / 0.00e+00, 2.82e+00 / 0.00e+00 | 20260505 18:01:44.034 | INFO | PerfReport: 20.37%, LockCompute -- 2.04e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260505 18:01:44.034 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.85e-02 / 0.00e+00 | 20260505 18:01:44.166 | INFO | Starting a round of Baz IO | 20260505 18:01:47.341 | INFO | Populating RTMetrics report | 20260505 18:01:47.341 | INFO | RTMetrics report populated | 20260505 18:01:47.341 | INFO | Opening tmp RTMetrics json file | 20260505 18:01:47.344 | INFO | Serializing RTMetrics to json | 20260505 18:01:47.346 | INFO | Serialization done, swapping tmp file now... | 20260505 18:01:47.346 | INFO | RTMetrics done | 20260505 18:01:47.437 | INFO | Finished a round of Baz IO: Wrote 2.107 GiB in 3.27 seconds | 20260505 18:01:47.437 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.97% / 0.00% / 78.03% | 20260505 18:01:47.437 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.269926% / 3.97364e-06% / 8.34465e-05% | 20260505 18:01:47.437 | INFO | Baz IO overall: | 20260505 18:01:47.437 | INFO | BytesWritten : 3068281777389 | 20260505 18:01:47.437 | INFO | NumEvents : 1454136561982 | 20260505 18:01:49.168 | INFO | Analyzing chunk frames = [5201920,5202432) | 20260505 18:01:49.168 | INFO | MemInfo: { | 20260505 18:01:49.168 | INFO | "gpu" : " 17 GB", | 20260505 18:01:49.168 | INFO | "huge" : "", | 20260505 18:01:49.168 | INFO | "huge_shared" : " 27 GB", | 20260505 18:01:49.168 | INFO | "malloc" : " 14 GB", | 20260505 18:01:49.168 | INFO | "pinned" : " 29 GB" | 20260505 18:01:49.168 | INFO | } | 20260505 18:01:49.168 | INFO | Perf report for the last 5.13738 seconds: | 20260505 18:01:49.168 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:01:49.168 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:49.168 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:49.168 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:49.168 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 18:01:49.168 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:49.168 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260505 18:01:49.168 | INFO | PRE_HQ : 60.5%, 1.8, 56.1ms, 0 | 20260505 18:01:49.168 | INFO | BAZWRITER : 63.2%, 3.8, 122.6ms, 0 | 20260505 18:01:49.168 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 18:01:49.168 | INFO | Uploaded 36GiB to GPU in 2.15831s (16.6797GiB/s) | 20260505 18:01:49.169 | INFO | Downloaded 38GiB to GPU in 2.24089s (16.9576GiB/s) | 20260505 18:01:54.305 | INFO | Analyzing chunk frames = [5202432,5202944) | 20260505 18:01:54.305 | INFO | MemInfo: { | 20260505 18:01:54.305 | INFO | "gpu" : " 17 GB", | 20260505 18:01:54.305 | INFO | "huge" : "", | 20260505 18:01:54.305 | INFO | "huge_shared" : " 27 GB", | 20260505 18:01:54.305 | INFO | "malloc" : " 14 GB", | 20260505 18:01:54.305 | INFO | "pinned" : " 29 GB" | 20260505 18:01:54.305 | INFO | } | 20260505 18:01:54.305 | INFO | Perf report for the last 5.13677 seconds: | 20260505 18:01:54.305 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:01:54.305 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:54.305 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:54.305 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:54.305 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 18:01:54.305 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:54.305 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:54.305 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:01:54.305 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:54.305 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:54.305 | INFO | Uploaded 36GiB to GPU in 2.13881s (16.8318GiB/s) | 20260505 18:01:54.305 | INFO | Downloaded 34GiB to GPU in 2.02641s (16.7785GiB/s) | 20260505 18:01:59.447 | INFO | Analyzing chunk frames = [5202944,5203456) | 20260505 18:01:59.447 | INFO | MemInfo: { | 20260505 18:01:59.447 | INFO | "gpu" : " 17 GB", | 20260505 18:01:59.447 | INFO | "huge" : "", | 20260505 18:01:59.447 | INFO | "huge_shared" : " 27 GB", | 20260505 18:01:59.447 | INFO | "malloc" : " 14 GB", | 20260505 18:01:59.447 | INFO | "pinned" : " 29 GB" | 20260505 18:01:59.447 | INFO | } | 20260505 18:01:59.447 | INFO | Perf report for the last 5.14185 seconds: | 20260505 18:01:59.447 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:01:59.447 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:59.447 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:59.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:59.447 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260505 18:01:59.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:59.447 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:59.447 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:01:59.447 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:59.447 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:01:59.447 | INFO | Uploaded 36GiB to GPU in 2.20355s (16.3373GiB/s) | 20260505 18:01:59.447 | INFO | Downloaded 34GiB to GPU in 2.03788s (16.684GiB/s) | 20260505 18:02:04.589 | INFO | Analyzing chunk frames = [5203456,5203968) | 20260505 18:02:04.589 | INFO | MemInfo: { | 20260505 18:02:04.589 | INFO | "gpu" : " 17 GB", | 20260505 18:02:04.589 | INFO | "huge" : "", | 20260505 18:02:04.589 | INFO | "huge_shared" : " 27 GB", | 20260505 18:02:04.589 | INFO | "malloc" : " 14 GB", | 20260505 18:02:04.589 | INFO | "pinned" : " 29 GB" | 20260505 18:02:04.589 | INFO | } | 20260505 18:02:04.589 | INFO | Perf report for the last 5.14213 seconds: | 20260505 18:02:04.589 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:02:04.589 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:04.589 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:04.589 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:04.589 | INFO | ANALYSIS : 62.5%, 3.0, 97.1ms, 0 | 20260505 18:02:04.589 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:04.589 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:04.589 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:02:04.589 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:04.589 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:04.589 | INFO | Uploaded 36GiB to GPU in 2.1824s (16.4956GiB/s) | 20260505 18:02:04.589 | INFO | Downloaded 34GiB to GPU in 2.14081s (15.8819GiB/s) | 20260505 18:02:09.730 | INFO | Analyzing chunk frames = [5203968,5204480) | 20260505 18:02:09.730 | INFO | MemInfo: { | 20260505 18:02:09.730 | INFO | "gpu" : " 17 GB", | 20260505 18:02:09.730 | INFO | "huge" : "", | 20260505 18:02:09.730 | INFO | "huge_shared" : " 27 GB", | 20260505 18:02:09.730 | INFO | "malloc" : " 14 GB", | 20260505 18:02:09.730 | INFO | "pinned" : " 29 GB" | 20260505 18:02:09.730 | INFO | } | 20260505 18:02:09.731 | INFO | Perf report for the last 5.14163 seconds: | 20260505 18:02:09.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:02:09.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:09.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:09.731 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:09.731 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 18:02:09.731 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:09.731 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:09.731 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:02:09.731 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:09.731 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:09.731 | INFO | Uploaded 36GiB to GPU in 2.15262s (16.7238GiB/s) | 20260505 18:02:09.731 | INFO | Downloaded 34GiB to GPU in 2.05929s (16.5105GiB/s) | 20260505 18:02:14.864 | INFO | Analyzing chunk frames = [5204480,5204992) | 20260505 18:02:14.864 | INFO | MemInfo: { | 20260505 18:02:14.864 | INFO | "gpu" : " 17 GB", | 20260505 18:02:14.864 | INFO | "huge" : "", | 20260505 18:02:14.864 | INFO | "huge_shared" : " 27 GB", | 20260505 18:02:14.864 | INFO | "malloc" : " 14 GB", | 20260505 18:02:14.864 | INFO | "pinned" : " 29 GB" | 20260505 18:02:14.864 | INFO | } | 20260505 18:02:14.865 | INFO | Perf report for the last 5.1339 seconds: | 20260505 18:02:14.865 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:02:14.865 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:14.865 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:14.865 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:14.865 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260505 18:02:14.865 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:14.865 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:14.865 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:02:14.865 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:14.865 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:14.865 | INFO | Uploaded 36GiB to GPU in 2.14363s (16.7939GiB/s) | 20260505 18:02:14.865 | INFO | Downloaded 34GiB to GPU in 2.058s (16.5209GiB/s) | 20260505 18:02:19.998 | INFO | Analyzing chunk frames = [5204992,5205504) | 20260505 18:02:19.998 | INFO | MemInfo: { | 20260505 18:02:19.998 | INFO | "gpu" : " 17 GB", | 20260505 18:02:19.998 | INFO | "huge" : "", | 20260505 18:02:19.998 | INFO | "huge_shared" : " 27 GB", | 20260505 18:02:19.998 | INFO | "malloc" : " 14 GB", | 20260505 18:02:19.998 | INFO | "pinned" : " 29 GB" | 20260505 18:02:19.998 | INFO | } | 20260505 18:02:19.998 | INFO | Perf report for the last 5.13343 seconds: | 20260505 18:02:19.998 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:02:19.998 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:19.998 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:19.998 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:19.998 | INFO | ANALYSIS : 63.5%, 3.0, 99.0ms, 0 | 20260505 18:02:19.998 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:19.998 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:19.998 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 18:02:19.998 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:19.998 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:19.998 | INFO | Uploaded 36GiB to GPU in 2.16862s (16.6004GiB/s) | 20260505 18:02:19.998 | INFO | Downloaded 34GiB to GPU in 2.18005s (15.596GiB/s) | 20260505 18:02:25.144 | INFO | Analyzing chunk frames = [5205504,5206016) | 20260505 18:02:25.144 | INFO | MemInfo: { | 20260505 18:02:25.144 | INFO | "gpu" : " 17 GB", | 20260505 18:02:25.144 | INFO | "huge" : "", | 20260505 18:02:25.144 | INFO | "huge_shared" : " 27 GB", | 20260505 18:02:25.144 | INFO | "malloc" : " 14 GB", | 20260505 18:02:25.144 | INFO | "pinned" : " 29 GB" | 20260505 18:02:25.144 | INFO | } | 20260505 18:02:25.144 | INFO | Perf report for the last 5.14605 seconds: | 20260505 18:02:25.144 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:02:25.144 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:25.144 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:25.144 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:25.144 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 18:02:25.144 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:25.144 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:25.144 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 18:02:25.144 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:25.144 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:25.144 | INFO | Uploaded 36GiB to GPU in 2.18032s (16.5113GiB/s) | 20260505 18:02:25.144 | INFO | Downloaded 34GiB to GPU in 2.03946s (16.671GiB/s) | 20260505 18:02:25.289 | INFO | Starting a round of Baz IO | 20260505 18:02:28.454 | INFO | Populating RTMetrics report | 20260505 18:02:28.454 | INFO | RTMetrics report populated | 20260505 18:02:28.454 | INFO | Opening tmp RTMetrics json file | 20260505 18:02:28.457 | INFO | Serializing RTMetrics to json | 20260505 18:02:28.459 | INFO | Serialization done, swapping tmp file now... | 20260505 18:02:28.462 | INFO | RTMetrics done | 20260505 18:02:28.547 | INFO | Finished a round of Baz IO: Wrote 2.107 GiB in 3.258 seconds | 20260505 18:02:28.547 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.97% / 0.00% / 78.03% | 20260505 18:02:28.547 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.269727% / 0% / 0.000103315% | 20260505 18:02:28.547 | INFO | Baz IO overall: | 20260505 18:02:28.547 | INFO | BytesWritten : 3070544085500 | 20260505 18:02:28.547 | INFO | NumEvents : 1455191869834 | 20260505 18:02:30.286 | INFO | Analyzing chunk frames = [5206016,5206528) | 20260505 18:02:30.286 | INFO | MemInfo: { | 20260505 18:02:30.286 | INFO | "gpu" : " 17 GB", | 20260505 18:02:30.286 | INFO | "huge" : "", | 20260505 18:02:30.286 | INFO | "huge_shared" : " 27 GB", | 20260505 18:02:30.286 | INFO | "malloc" : " 14 GB", | 20260505 18:02:30.286 | INFO | "pinned" : " 29 GB" | 20260505 18:02:30.286 | INFO | } | 20260505 18:02:30.286 | INFO | Perf report for the last 5.14158 seconds: | 20260505 18:02:30.286 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:02:30.286 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:30.286 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:30.286 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:30.286 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 18:02:30.286 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:30.286 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260505 18:02:30.286 | INFO | PRE_HQ : 60.3%, 1.8, 56.2ms, 0 | 20260505 18:02:30.286 | INFO | BAZWRITER : 63.1%, 3.8, 123.1ms, 0 | 20260505 18:02:30.286 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 18:02:30.286 | INFO | Uploaded 36GiB to GPU in 2.16743s (16.6095GiB/s) | 20260505 18:02:30.286 | INFO | Downloaded 38GiB to GPU in 2.24655s (16.9149GiB/s) | 20260505 18:02:35.424 | INFO | Analyzing chunk frames = [5206528,5207040) | 20260505 18:02:35.424 | INFO | MemInfo: { | 20260505 18:02:35.424 | INFO | "gpu" : " 17 GB", | 20260505 18:02:35.424 | INFO | "huge" : "", | 20260505 18:02:35.424 | INFO | "huge_shared" : " 27 GB", | 20260505 18:02:35.424 | INFO | "malloc" : " 14 GB", | 20260505 18:02:35.424 | INFO | "pinned" : " 29 GB" | 20260505 18:02:35.424 | INFO | } | 20260505 18:02:35.424 | INFO | Perf report for the last 5.13848 seconds: | 20260505 18:02:35.424 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:02:35.424 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:35.424 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:35.424 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:35.424 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 18:02:35.424 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:35.424 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:35.424 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:02:35.424 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:35.424 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:35.425 | INFO | Uploaded 36GiB to GPU in 2.13047s (16.8977GiB/s) | 20260505 18:02:35.425 | INFO | Downloaded 34GiB to GPU in 2.0233s (16.8042GiB/s) | 20260505 18:02:40.560 | INFO | Analyzing chunk frames = [5207040,5207552) | 20260505 18:02:40.560 | INFO | MemInfo: { | 20260505 18:02:40.560 | INFO | "gpu" : " 17 GB", | 20260505 18:02:40.560 | INFO | "huge" : "", | 20260505 18:02:40.560 | INFO | "huge_shared" : " 27 GB", | 20260505 18:02:40.560 | INFO | "malloc" : " 14 GB", | 20260505 18:02:40.560 | INFO | "pinned" : " 29 GB" | 20260505 18:02:40.560 | INFO | } | 20260505 18:02:40.560 | INFO | Perf report for the last 5.13611 seconds: | 20260505 18:02:40.560 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:02:40.560 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:40.560 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:40.560 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:40.560 | INFO | ANALYSIS : 62.7%, 3.0, 97.5ms, 0 | 20260505 18:02:40.560 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:40.560 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:40.560 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:02:40.561 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:40.561 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:40.561 | INFO | Uploaded 36GiB to GPU in 2.20161s (16.3517GiB/s) | 20260505 18:02:40.561 | INFO | Downloaded 34GiB to GPU in 2.04361s (16.6372GiB/s) | 20260505 18:02:45.699 | INFO | Analyzing chunk frames = [5207552,5208064) | 20260505 18:02:45.699 | INFO | MemInfo: { | 20260505 18:02:45.699 | INFO | "gpu" : " 17 GB", | 20260505 18:02:45.699 | INFO | "huge" : "", | 20260505 18:02:45.699 | INFO | "huge_shared" : " 27 GB", | 20260505 18:02:45.699 | INFO | "malloc" : " 14 GB", | 20260505 18:02:45.699 | INFO | "pinned" : " 29 GB" | 20260505 18:02:45.699 | INFO | } | 20260505 18:02:45.699 | INFO | Perf report for the last 5.13845 seconds: | 20260505 18:02:45.699 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:02:45.699 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:45.699 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:45.699 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:45.699 | INFO | ANALYSIS : 62.6%, 3.0, 97.3ms, 0 | 20260505 18:02:45.699 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:45.699 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:45.699 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 18:02:45.699 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:45.699 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:45.700 | INFO | Uploaded 36GiB to GPU in 2.2107s (16.2844GiB/s) | 20260505 18:02:45.700 | INFO | Downloaded 34GiB to GPU in 2.14368s (15.8606GiB/s) | 20260505 18:02:50.843 | INFO | Analyzing chunk frames = [5208064,5208576) | 20260505 18:02:50.843 | INFO | MemInfo: { | 20260505 18:02:50.843 | INFO | "gpu" : " 17 GB", | 20260505 18:02:50.843 | INFO | "huge" : "", | 20260505 18:02:50.843 | INFO | "huge_shared" : " 27 GB", | 20260505 18:02:50.843 | INFO | "malloc" : " 14 GB", | 20260505 18:02:50.843 | INFO | "pinned" : " 29 GB" | 20260505 18:02:50.843 | INFO | } | 20260505 18:02:50.843 | INFO | Perf report for the last 5.14417 seconds: | 20260505 18:02:50.843 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:02:50.843 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:50.843 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:50.843 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:50.843 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 18:02:50.843 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:50.843 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:50.843 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 18:02:50.843 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:50.843 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:50.843 | INFO | Uploaded 36GiB to GPU in 2.14914s (16.7509GiB/s) | 20260505 18:02:50.843 | INFO | Downloaded 34GiB to GPU in 2.06329s (16.4785GiB/s) | 20260505 18:02:55.986 | INFO | Analyzing chunk frames = [5208576,5209088) | 20260505 18:02:55.986 | INFO | MemInfo: { | 20260505 18:02:55.986 | INFO | "gpu" : " 17 GB", | 20260505 18:02:55.986 | INFO | "huge" : "", | 20260505 18:02:55.986 | INFO | "huge_shared" : " 27 GB", | 20260505 18:02:55.986 | INFO | "malloc" : " 14 GB", | 20260505 18:02:55.986 | INFO | "pinned" : " 29 GB" | 20260505 18:02:55.986 | INFO | } | 20260505 18:02:55.986 | INFO | Perf report for the last 5.14283 seconds: | 20260505 18:02:55.986 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:02:55.986 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:55.986 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:55.986 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:55.986 | INFO | ANALYSIS : 62.1%, 3.0, 97.1ms, 0 | 20260505 18:02:55.986 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:55.986 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:55.986 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:02:55.986 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:55.986 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:02:55.986 | INFO | Uploaded 36GiB to GPU in 2.14256s (16.8024GiB/s) | 20260505 18:02:55.986 | INFO | Downloaded 34GiB to GPU in 2.06132s (16.4943GiB/s) | 20260505 18:03:01.114 | INFO | Analyzing chunk frames = [5209088,5209600) | 20260505 18:03:01.114 | INFO | MemInfo: { | 20260505 18:03:01.114 | INFO | "gpu" : " 17 GB", | 20260505 18:03:01.115 | INFO | "huge" : "", | 20260505 18:03:01.115 | INFO | "huge_shared" : " 27 GB", | 20260505 18:03:01.115 | INFO | "malloc" : " 14 GB", | 20260505 18:03:01.115 | INFO | "pinned" : " 29 GB" | 20260505 18:03:01.115 | INFO | } | 20260505 18:03:01.115 | INFO | Perf report for the last 5.12905 seconds: | 20260505 18:03:01.115 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:03:01.115 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:01.115 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:01.115 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:01.115 | INFO | ANALYSIS : 63.4%, 3.0, 98.8ms, 0 | 20260505 18:03:01.115 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:01.115 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:01.115 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 18:03:01.115 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:01.115 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:01.115 | INFO | Uploaded 36GiB to GPU in 2.13213s (16.8846GiB/s) | 20260505 18:03:01.115 | INFO | Downloaded 34GiB to GPU in 2.13306s (15.9395GiB/s) | 20260505 18:03:06.252 | INFO | Analyzing chunk frames = [5209600,5210112) | 20260505 18:03:06.252 | INFO | MemInfo: { | 20260505 18:03:06.252 | INFO | "gpu" : " 17 GB", | 20260505 18:03:06.252 | INFO | "huge" : "", | 20260505 18:03:06.252 | INFO | "huge_shared" : " 27 GB", | 20260505 18:03:06.252 | INFO | "malloc" : " 14 GB", | 20260505 18:03:06.252 | INFO | "pinned" : " 29 GB" | 20260505 18:03:06.252 | INFO | } | 20260505 18:03:06.252 | INFO | Perf report for the last 5.13726 seconds: | 20260505 18:03:06.252 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:03:06.252 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:06.252 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:06.252 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:06.252 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 18:03:06.252 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:06.252 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:06.252 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 18:03:06.252 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:06.252 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:06.252 | INFO | Uploaded 36GiB to GPU in 2.1681s (16.6044GiB/s) | 20260505 18:03:06.253 | INFO | Downloaded 34GiB to GPU in 2.03848s (16.6791GiB/s) | 20260505 18:03:06.253 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 18:03:06.253 | INFO | Total mem usage: 15.3 GiB | 20260505 18:03:06.253 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 18:03:06.253 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.04%) | 20260505 18:03:06.253 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 18:03:06.253 | INFO | High Water Breakdown: | 20260505 18:03:06.253 | INFO | 61% : 8.66 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 18:03:06.253 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 18:03:06.253 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 18:03:06.253 | INFO | ------------------------------------------ | 20260505 18:03:06.253 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:03:06.253 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 18:03:06.253 | INFO | Total mem usage: 26.5 GiB | 20260505 18:03:06.253 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 18:03:06.253 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 18:03:06.253 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 18:03:06.254 | INFO | High Water Breakdown: | 20260505 18:03:06.254 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:03:06.254 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:03:06.254 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:03:06.254 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:03:06.254 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:03:06.254 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:03:06.254 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 18:03:06.254 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 18:03:06.254 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:03:06.254 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 18:03:06.254 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 18:03:06.254 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 18:03:06.254 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 18:03:06.254 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 18:03:06.254 | INFO | ------------------------------------------ | 20260505 18:03:06.254 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:03:06.254 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 18:03:06.254 | INFO | Total mem usage: 25.6 GiB | 20260505 18:03:06.254 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 18:03:06.254 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 18:03:06.254 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 18:03:06.254 | INFO | High Water Breakdown: | 20260505 18:03:06.254 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 18:03:06.254 | INFO | ------------------------------------------ | 20260505 18:03:06.254 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:03:06.254 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 18:03:06.254 | INFO | Total mem usage: 15.7 GiB | 20260505 18:03:06.254 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 18:03:06.254 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 18:03:06.254 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 18:03:06.254 | INFO | High Water Breakdown: | 20260505 18:03:06.254 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 18:03:06.254 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 18:03:06.254 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 18:03:06.254 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 18:03:06.254 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 18:03:06.254 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 18:03:06.254 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 18:03:06.254 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 18:03:06.254 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 18:03:06.254 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:03:06.254 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 18:03:06.254 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:03:06.254 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 18:03:06.254 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 18:03:06.254 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 18:03:06.254 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 18:03:06.254 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:03:06.254 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 18:03:06.254 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:03:06.254 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:03:06.254 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:03:06.254 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 18:03:06.254 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 18:03:06.254 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:03:06.254 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 18:03:06.254 | INFO | ------------------------------------------ | 20260505 18:03:06.254 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:03:06.255 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 18:03:06.255 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:03:06.255 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:03:06.255 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.50e+01, 1.35e+01 / 2.03e+00 | 20260505 18:03:06.255 | INFO | PerfReport: 0.29%, AggregateStats -- 9.26e-02 / 1.01e-01, 1.04e-02 / 3.35e-03 | 20260505 18:03:06.255 | INFO | PerfReport: 11.80%, Baseline -- 3.72e+00 / 4.52e+00, 5.75e-01 / 1.24e-01 | 20260505 18:03:06.255 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.53e+00, 1.94e-01 / 7.22e-02 | 20260505 18:03:06.255 | INFO | PerfReport: 16.00%, DME -- 4.04e+01 / 4.50e+01, 4.38e+00 / 2.07e+00 | 20260505 18:03:06.255 | INFO | PerfReport: 56.80%, FrameLabeling -- 1.79e+01 / 1.92e+01, 1.43e+00 / 9.66e-01 | 20260505 18:03:06.255 | INFO | PerfReport: 7.31%, PulseAccumulating -- 2.31e+00 / 2.90e+00, 4.46e-01 / 1.45e-01 | 20260505 18:03:06.255 | INFO | PerfReport: 3.35%, Metrics -- 1.06e+00 / 1.38e+00, 2.92e-01 / 1.66e-01 | 20260505 18:03:06.255 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.68e-01 / 4.26e-01, 6.53e-02 / 3.75e-02 | 20260505 18:03:06.255 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 18:03:06.255 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:03:06.255 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:03:06.255 | INFO | PerfReport: Full Execution -- 9.99e+01 / -nan, 2.55e+01 / 0.00e+00 | 20260505 18:03:06.255 | INFO | PerfReport: 22.57%, Upload -- 2.25e+01 / 0.00e+00, 7.93e+00 / 0.00e+00 | 20260505 18:03:06.255 | INFO | PerfReport: 21.86%, Download -- 2.18e+01 / 0.00e+00, 7.88e+00 / 0.00e+00 | 20260505 18:03:06.255 | INFO | PerfReport: 31.60%, Compute -- 3.16e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 18:03:06.255 | INFO | PerfReport: 2.29%, LockUpload -- 2.29e+00 / 0.00e+00, 7.84e+00 / 0.00e+00 | 20260505 18:03:06.255 | INFO | PerfReport: 1.41%, LockDownload -- 1.41e+00 / 0.00e+00, 2.87e+00 / 0.00e+00 | 20260505 18:03:06.255 | INFO | PerfReport: 20.26%, LockCompute -- 2.02e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 18:03:06.255 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.60e-02 / 0.00e+00 | 20260505 18:03:06.381 | INFO | Starting a round of Baz IO | 20260505 18:03:09.593 | INFO | Populating RTMetrics report | 20260505 18:03:09.593 | INFO | RTMetrics report populated | 20260505 18:03:09.593 | INFO | Opening tmp RTMetrics json file | 20260505 18:03:09.594 | INFO | Serializing RTMetrics to json | 20260505 18:03:09.595 | INFO | Serialization done, swapping tmp file now... | 20260505 18:03:09.595 | INFO | RTMetrics done | 20260505 18:03:09.688 | INFO | Finished a round of Baz IO: Wrote 2.105 GiB in 3.307 seconds | 20260505 18:03:09.688 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.96% / 0.00% / 78.03% | 20260505 18:03:09.688 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.268245% / 3.97364e-06% / 9.13938e-05% | 20260505 18:03:09.688 | INFO | Baz IO overall: | 20260505 18:03:09.688 | INFO | BytesWritten : 3072804007936 | 20260505 18:03:09.688 | INFO | NumEvents : 1456245548365 | 20260505 18:03:11.402 | INFO | Analyzing chunk frames = [5210112,5210624) | 20260505 18:03:11.402 | INFO | MemInfo: { | 20260505 18:03:11.402 | INFO | "gpu" : " 17 GB", | 20260505 18:03:11.402 | INFO | "huge" : "", | 20260505 18:03:11.402 | INFO | "huge_shared" : " 27 GB", | 20260505 18:03:11.402 | INFO | "malloc" : " 14 GB", | 20260505 18:03:11.402 | INFO | "pinned" : " 29 GB" | 20260505 18:03:11.402 | INFO | } | 20260505 18:03:11.402 | INFO | Perf report for the last 5.15022 seconds: | 20260505 18:03:11.402 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:03:11.402 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:11.402 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:11.402 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:11.402 | INFO | ANALYSIS : 63.8%, 3.0, 99.6ms, 0 | 20260505 18:03:11.403 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:11.403 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260505 18:03:11.403 | INFO | PRE_HQ : 60.4%, 1.8, 56.1ms, 0 | 20260505 18:03:11.403 | INFO | BAZWRITER : 63.6%, 3.8, 123.6ms, 0 | 20260505 18:03:11.403 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 18:03:11.403 | INFO | Uploaded 36GiB to GPU in 2.19248s (16.4198GiB/s) | 20260505 18:03:11.403 | INFO | Downloaded 38GiB to GPU in 2.26235s (16.7967GiB/s) | 20260505 18:03:16.543 | INFO | Analyzing chunk frames = [5210624,5211136) | 20260505 18:03:16.543 | INFO | MemInfo: { | 20260505 18:03:16.543 | INFO | "gpu" : " 17 GB", | 20260505 18:03:16.543 | INFO | "huge" : "", | 20260505 18:03:16.543 | INFO | "huge_shared" : " 27 GB", | 20260505 18:03:16.543 | INFO | "malloc" : " 14 GB", | 20260505 18:03:16.543 | INFO | "pinned" : " 29 GB" | 20260505 18:03:16.543 | INFO | } | 20260505 18:03:16.544 | INFO | Perf report for the last 5.14124 seconds: | 20260505 18:03:16.544 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:03:16.544 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:16.544 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:16.544 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:16.544 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 18:03:16.544 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:16.544 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:16.544 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:03:16.544 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:16.544 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:16.544 | INFO | Uploaded 36GiB to GPU in 2.13541s (16.8586GiB/s) | 20260505 18:03:16.544 | INFO | Downloaded 34GiB to GPU in 2.02575s (16.7839GiB/s) | 20260505 18:03:21.678 | INFO | Analyzing chunk frames = [5211136,5211648) | 20260505 18:03:21.678 | INFO | MemInfo: { | 20260505 18:03:21.678 | INFO | "gpu" : " 17 GB", | 20260505 18:03:21.678 | INFO | "huge" : "", | 20260505 18:03:21.678 | INFO | "huge_shared" : " 27 GB", | 20260505 18:03:21.678 | INFO | "malloc" : " 14 GB", | 20260505 18:03:21.679 | INFO | "pinned" : " 29 GB" | 20260505 18:03:21.679 | INFO | } | 20260505 18:03:21.679 | INFO | Perf report for the last 5.13506 seconds: | 20260505 18:03:21.679 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:03:21.679 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:21.679 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:21.679 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:21.679 | INFO | ANALYSIS : 62.4%, 3.0, 97.0ms, 0 | 20260505 18:03:21.679 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:21.679 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:21.679 | INFO | PRE_HQ : 27.7%, 1.0, 14.3ms, 0 | 20260505 18:03:21.679 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:21.679 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:21.679 | INFO | Uploaded 36GiB to GPU in 2.13655s (16.8496GiB/s) | 20260505 18:03:21.679 | INFO | Downloaded 34GiB to GPU in 2.04297s (16.6425GiB/s) | 20260505 18:03:26.814 | INFO | Analyzing chunk frames = [5211648,5212160) | 20260505 18:03:26.814 | INFO | MemInfo: { | 20260505 18:03:26.814 | INFO | "gpu" : " 17 GB", | 20260505 18:03:26.814 | INFO | "huge" : "", | 20260505 18:03:26.814 | INFO | "huge_shared" : " 27 GB", | 20260505 18:03:26.814 | INFO | "malloc" : " 14 GB", | 20260505 18:03:26.814 | INFO | "pinned" : " 29 GB" | 20260505 18:03:26.814 | INFO | } | 20260505 18:03:26.814 | INFO | Perf report for the last 5.13529 seconds: | 20260505 18:03:26.814 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:03:26.814 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:26.814 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:26.814 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:26.814 | INFO | ANALYSIS : 62.8%, 3.0, 97.5ms, 0 | 20260505 18:03:26.814 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:26.814 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:26.814 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 18:03:26.814 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:26.814 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:26.814 | INFO | Uploaded 36GiB to GPU in 2.22658s (16.1683GiB/s) | 20260505 18:03:26.814 | INFO | Downloaded 34GiB to GPU in 2.12253s (16.0186GiB/s) | 20260505 18:03:31.953 | INFO | Analyzing chunk frames = [5212160,5212672) | 20260505 18:03:31.953 | INFO | MemInfo: { | 20260505 18:03:31.953 | INFO | "gpu" : " 17 GB", | 20260505 18:03:31.953 | INFO | "huge" : "", | 20260505 18:03:31.953 | INFO | "huge_shared" : " 27 GB", | 20260505 18:03:31.953 | INFO | "malloc" : " 14 GB", | 20260505 18:03:31.953 | INFO | "pinned" : " 29 GB" | 20260505 18:03:31.953 | INFO | } | 20260505 18:03:31.953 | INFO | Perf report for the last 5.13905 seconds: | 20260505 18:03:31.953 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:03:31.953 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:31.953 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:31.953 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:31.953 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 18:03:31.953 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:31.953 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:31.953 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 18:03:31.953 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:31.953 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:31.953 | INFO | Uploaded 36GiB to GPU in 2.15534s (16.7027GiB/s) | 20260505 18:03:31.953 | INFO | Downloaded 34GiB to GPU in 2.05646s (16.5333GiB/s) | 20260505 18:03:37.095 | INFO | Analyzing chunk frames = [5212672,5213184) | 20260505 18:03:37.095 | INFO | MemInfo: { | 20260505 18:03:37.095 | INFO | "gpu" : " 17 GB", | 20260505 18:03:37.095 | INFO | "huge" : "", | 20260505 18:03:37.095 | INFO | "huge_shared" : " 27 GB", | 20260505 18:03:37.095 | INFO | "malloc" : " 14 GB", | 20260505 18:03:37.095 | INFO | "pinned" : " 29 GB" | 20260505 18:03:37.095 | INFO | } | 20260505 18:03:37.095 | INFO | Perf report for the last 5.14236 seconds: | 20260505 18:03:37.096 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:03:37.096 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:37.096 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:37.096 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:37.096 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260505 18:03:37.096 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:37.096 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:37.096 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:03:37.096 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:37.096 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:37.096 | INFO | Uploaded 36GiB to GPU in 2.13656s (16.8495GiB/s) | 20260505 18:03:37.096 | INFO | Downloaded 34GiB to GPU in 2.04173s (16.6526GiB/s) | 20260505 18:03:42.233 | INFO | Analyzing chunk frames = [5213184,5213696) | 20260505 18:03:42.233 | INFO | MemInfo: { | 20260505 18:03:42.233 | INFO | "gpu" : " 17 GB", | 20260505 18:03:42.233 | INFO | "huge" : "", | 20260505 18:03:42.233 | INFO | "huge_shared" : " 27 GB", | 20260505 18:03:42.233 | INFO | "malloc" : " 14 GB", | 20260505 18:03:42.233 | INFO | "pinned" : " 29 GB" | 20260505 18:03:42.233 | INFO | } | 20260505 18:03:42.233 | INFO | Perf report for the last 5.13792 seconds: | 20260505 18:03:42.233 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:03:42.233 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:42.233 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:42.233 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:42.234 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 18:03:42.234 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:42.234 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:42.234 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 18:03:42.234 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:42.234 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:42.234 | INFO | Uploaded 36GiB to GPU in 2.13424s (16.8679GiB/s) | 20260505 18:03:42.234 | INFO | Downloaded 34GiB to GPU in 2.14416s (15.857GiB/s) | 20260505 18:03:47.373 | INFO | Analyzing chunk frames = [5213696,5214208) | 20260505 18:03:47.373 | INFO | MemInfo: { | 20260505 18:03:47.373 | INFO | "gpu" : " 17 GB", | 20260505 18:03:47.373 | INFO | "huge" : "", | 20260505 18:03:47.373 | INFO | "huge_shared" : " 27 GB", | 20260505 18:03:47.373 | INFO | "malloc" : " 14 GB", | 20260505 18:03:47.373 | INFO | "pinned" : " 29 GB" | 20260505 18:03:47.373 | INFO | } | 20260505 18:03:47.373 | INFO | Perf report for the last 5.13997 seconds: | 20260505 18:03:47.373 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:03:47.373 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:47.373 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:47.373 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:47.373 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 18:03:47.373 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:47.373 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:47.373 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 18:03:47.373 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:47.373 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:47.374 | INFO | Uploaded 36GiB to GPU in 2.19317s (16.4146GiB/s) | 20260505 18:03:47.374 | INFO | Downloaded 34GiB to GPU in 2.02866s (16.7598GiB/s) | 20260505 18:03:47.513 | INFO | Starting a round of Baz IO | 20260505 18:03:50.695 | INFO | Populating RTMetrics report | 20260505 18:03:50.695 | INFO | RTMetrics report populated | 20260505 18:03:50.695 | INFO | Opening tmp RTMetrics json file | 20260505 18:03:50.700 | INFO | Serializing RTMetrics to json | 20260505 18:03:50.702 | INFO | Serialization done, swapping tmp file now... | 20260505 18:03:50.702 | INFO | RTMetrics done | 20260505 18:03:50.788 | INFO | Finished a round of Baz IO: Wrote 2.107 GiB in 3.275 seconds | 20260505 18:03:50.788 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.96% / 0.00% / 78.04% | 20260505 18:03:50.788 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.268161% / 0% / 0.000111262% | 20260505 18:03:50.788 | INFO | Baz IO overall: | 20260505 18:03:50.788 | INFO | BytesWritten : 3075066690171 | 20260505 18:03:50.788 | INFO | NumEvents : 1457301305260 | 20260505 18:03:52.513 | INFO | Analyzing chunk frames = [5214208,5214720) | 20260505 18:03:52.513 | INFO | MemInfo: { | 20260505 18:03:52.513 | INFO | "gpu" : " 17 GB", | 20260505 18:03:52.513 | INFO | "huge" : "", | 20260505 18:03:52.513 | INFO | "huge_shared" : " 27 GB", | 20260505 18:03:52.513 | INFO | "malloc" : " 14 GB", | 20260505 18:03:52.513 | INFO | "pinned" : " 29 GB" | 20260505 18:03:52.513 | INFO | } | 20260505 18:03:52.513 | INFO | Perf report for the last 5.14011 seconds: | 20260505 18:03:52.513 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:03:52.513 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:52.513 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:52.513 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:52.513 | INFO | ANALYSIS : 63.5%, 3.0, 99.0ms, 0 | 20260505 18:03:52.513 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:52.513 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260505 18:03:52.513 | INFO | PRE_HQ : 60.4%, 1.8, 56.1ms, 0 | 20260505 18:03:52.514 | INFO | BAZWRITER : 63.1%, 3.8, 122.7ms, 0 | 20260505 18:03:52.514 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 18:03:52.514 | INFO | Uploaded 36GiB to GPU in 2.16285s (16.6447GiB/s) | 20260505 18:03:52.514 | INFO | Downloaded 38GiB to GPU in 2.26155s (16.8027GiB/s) | 20260505 18:03:57.660 | INFO | Analyzing chunk frames = [5214720,5215232) | 20260505 18:03:57.660 | INFO | MemInfo: { | 20260505 18:03:57.660 | INFO | "gpu" : " 17 GB", | 20260505 18:03:57.660 | INFO | "huge" : "", | 20260505 18:03:57.660 | INFO | "huge_shared" : " 27 GB", | 20260505 18:03:57.660 | INFO | "malloc" : " 14 GB", | 20260505 18:03:57.660 | INFO | "pinned" : " 29 GB" | 20260505 18:03:57.660 | INFO | } | 20260505 18:03:57.660 | INFO | Perf report for the last 5.14638 seconds: | 20260505 18:03:57.660 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:03:57.660 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:57.660 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:57.660 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:57.660 | INFO | ANALYSIS : 62.7%, 3.0, 98.1ms, 0 | 20260505 18:03:57.660 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:57.660 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:57.660 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 18:03:57.660 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:57.660 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:03:57.660 | INFO | Uploaded 36GiB to GPU in 2.14151s (16.8106GiB/s) | 20260505 18:03:57.660 | INFO | Downloaded 34GiB to GPU in 2.02862s (16.7602GiB/s) | 20260505 18:04:02.801 | INFO | Analyzing chunk frames = [5215232,5215744) | 20260505 18:04:02.801 | INFO | MemInfo: { | 20260505 18:04:02.801 | INFO | "gpu" : " 17 GB", | 20260505 18:04:02.801 | INFO | "huge" : "", | 20260505 18:04:02.801 | INFO | "huge_shared" : " 27 GB", | 20260505 18:04:02.801 | INFO | "malloc" : " 14 GB", | 20260505 18:04:02.801 | INFO | "pinned" : " 29 GB" | 20260505 18:04:02.801 | INFO | } | 20260505 18:04:02.801 | INFO | Perf report for the last 5.14157 seconds: | 20260505 18:04:02.801 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:04:02.801 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:02.801 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:02.801 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:02.801 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260505 18:04:02.801 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:02.801 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:02.801 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:04:02.801 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:02.801 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:02.802 | INFO | Uploaded 36GiB to GPU in 2.12319s (16.9557GiB/s) | 20260505 18:04:02.802 | INFO | Downloaded 34GiB to GPU in 2.02259s (16.8102GiB/s) | 20260505 18:04:07.929 | INFO | Analyzing chunk frames = [5215744,5216256) | 20260505 18:04:07.929 | INFO | MemInfo: { | 20260505 18:04:07.929 | INFO | "gpu" : " 17 GB", | 20260505 18:04:07.929 | INFO | "huge" : "", | 20260505 18:04:07.929 | INFO | "huge_shared" : " 27 GB", | 20260505 18:04:07.929 | INFO | "malloc" : " 14 GB", | 20260505 18:04:07.929 | INFO | "pinned" : " 29 GB" | 20260505 18:04:07.929 | INFO | } | 20260505 18:04:07.929 | INFO | Perf report for the last 5.12747 seconds: | 20260505 18:04:07.929 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:04:07.929 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:07.929 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:07.929 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:07.929 | INFO | ANALYSIS : 62.6%, 3.0, 97.0ms, 0 | 20260505 18:04:07.929 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:07.929 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:07.929 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 18:04:07.929 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:07.929 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:07.929 | INFO | Uploaded 36GiB to GPU in 2.20795s (16.3047GiB/s) | 20260505 18:04:07.929 | INFO | Downloaded 34GiB to GPU in 2.03928s (16.6725GiB/s) | 20260505 18:04:13.063 | INFO | Analyzing chunk frames = [5216256,5216768) | 20260505 18:04:13.063 | INFO | MemInfo: { | 20260505 18:04:13.063 | INFO | "gpu" : " 17 GB", | 20260505 18:04:13.063 | INFO | "huge" : "", | 20260505 18:04:13.063 | INFO | "huge_shared" : " 27 GB", | 20260505 18:04:13.063 | INFO | "malloc" : " 14 GB", | 20260505 18:04:13.063 | INFO | "pinned" : " 29 GB" | 20260505 18:04:13.063 | INFO | } | 20260505 18:04:13.063 | INFO | Perf report for the last 5.1341 seconds: | 20260505 18:04:13.063 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:04:13.063 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:13.063 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:13.063 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:13.063 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 18:04:13.063 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:13.063 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:13.063 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:04:13.063 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:13.063 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:13.063 | INFO | Uploaded 36GiB to GPU in 2.15748s (16.6861GiB/s) | 20260505 18:04:13.063 | INFO | Downloaded 34GiB to GPU in 2.06106s (16.4964GiB/s) | 20260505 18:04:18.207 | INFO | Analyzing chunk frames = [5216768,5217280) | 20260505 18:04:18.207 | INFO | MemInfo: { | 20260505 18:04:18.207 | INFO | "gpu" : " 17 GB", | 20260505 18:04:18.207 | INFO | "huge" : "", | 20260505 18:04:18.207 | INFO | "huge_shared" : " 27 GB", | 20260505 18:04:18.207 | INFO | "malloc" : " 14 GB", | 20260505 18:04:18.207 | INFO | "pinned" : " 29 GB" | 20260505 18:04:18.207 | INFO | } | 20260505 18:04:18.207 | INFO | Perf report for the last 5.14422 seconds: | 20260505 18:04:18.207 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:04:18.207 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:18.207 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:18.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:18.207 | INFO | ANALYSIS : 62.4%, 3.0, 97.6ms, 0 | 20260505 18:04:18.207 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:18.207 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:18.207 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:04:18.207 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:18.207 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:18.207 | INFO | Uploaded 36GiB to GPU in 2.14126s (16.8125GiB/s) | 20260505 18:04:18.207 | INFO | Downloaded 34GiB to GPU in 2.03841s (16.6796GiB/s) | 20260505 18:04:23.349 | INFO | Analyzing chunk frames = [5217280,5217792) | 20260505 18:04:23.349 | INFO | MemInfo: { | 20260505 18:04:23.349 | INFO | "gpu" : " 17 GB", | 20260505 18:04:23.349 | INFO | "huge" : "", | 20260505 18:04:23.349 | INFO | "huge_shared" : " 27 GB", | 20260505 18:04:23.349 | INFO | "malloc" : " 14 GB", | 20260505 18:04:23.349 | INFO | "pinned" : " 29 GB" | 20260505 18:04:23.349 | INFO | } | 20260505 18:04:23.349 | INFO | Perf report for the last 5.14192 seconds: | 20260505 18:04:23.349 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:04:23.349 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:23.349 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:23.349 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:23.349 | INFO | ANALYSIS : 63.2%, 3.0, 98.8ms, 0 | 20260505 18:04:23.349 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:23.349 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:23.349 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 18:04:23.349 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:23.349 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:23.349 | INFO | Uploaded 36GiB to GPU in 2.14733s (16.765GiB/s) | 20260505 18:04:23.349 | INFO | Downloaded 34GiB to GPU in 2.15878s (15.7496GiB/s) | 20260505 18:04:28.495 | INFO | Analyzing chunk frames = [5217792,5218304) | 20260505 18:04:28.495 | INFO | MemInfo: { | 20260505 18:04:28.495 | INFO | "gpu" : " 17 GB", | 20260505 18:04:28.495 | INFO | "huge" : "", | 20260505 18:04:28.495 | INFO | "huge_shared" : " 27 GB", | 20260505 18:04:28.495 | INFO | "malloc" : " 14 GB", | 20260505 18:04:28.495 | INFO | "pinned" : " 29 GB" | 20260505 18:04:28.495 | INFO | } | 20260505 18:04:28.495 | INFO | Perf report for the last 5.14565 seconds: | 20260505 18:04:28.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:04:28.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:28.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:28.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:28.495 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 18:04:28.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:28.495 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:28.495 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 18:04:28.495 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:28.495 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:28.495 | INFO | Uploaded 36GiB to GPU in 2.19826s (16.3766GiB/s) | 20260505 18:04:28.495 | INFO | Downloaded 34GiB to GPU in 2.03942s (16.6714GiB/s) | 20260505 18:04:28.495 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 18:04:28.495 | INFO | Total mem usage: 15.3 GiB | 20260505 18:04:28.495 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 18:04:28.495 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.01%) | 20260505 18:04:28.495 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 18:04:28.495 | INFO | High Water Breakdown: | 20260505 18:04:28.495 | INFO | 61% : 8.66 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 18:04:28.495 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 18:04:28.495 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 18:04:28.495 | INFO | ------------------------------------------ | 20260505 18:04:28.495 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:04:28.495 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 18:04:28.495 | INFO | Total mem usage: 26.5 GiB | 20260505 18:04:28.495 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 18:04:28.495 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 18:04:28.495 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 18:04:28.495 | INFO | High Water Breakdown: | 20260505 18:04:28.495 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:04:28.495 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:04:28.495 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:04:28.495 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:04:28.495 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:04:28.495 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:04:28.495 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 18:04:28.495 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 18:04:28.495 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:04:28.495 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 18:04:28.496 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 18:04:28.496 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 18:04:28.496 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 18:04:28.496 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 18:04:28.496 | INFO | ------------------------------------------ | 20260505 18:04:28.496 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:04:28.496 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 18:04:28.496 | INFO | Total mem usage: 25.6 GiB | 20260505 18:04:28.496 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 18:04:28.496 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 18:04:28.496 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 18:04:28.496 | INFO | High Water Breakdown: | 20260505 18:04:28.496 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 18:04:28.496 | INFO | ------------------------------------------ | 20260505 18:04:28.496 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:04:28.496 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 18:04:28.496 | INFO | Total mem usage: 15.7 GiB | 20260505 18:04:28.496 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 18:04:28.496 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 18:04:28.496 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 18:04:28.496 | INFO | High Water Breakdown: | 20260505 18:04:28.496 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 18:04:28.496 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 18:04:28.496 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 18:04:28.496 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 18:04:28.496 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 18:04:28.496 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 18:04:28.496 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 18:04:28.496 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 18:04:28.496 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 18:04:28.496 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:04:28.496 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 18:04:28.496 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:04:28.496 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 18:04:28.496 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 18:04:28.496 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 18:04:28.496 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 18:04:28.496 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:04:28.496 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 18:04:28.496 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:04:28.496 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:04:28.496 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:04:28.496 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 18:04:28.496 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 18:04:28.496 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:04:28.496 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 18:04:28.496 | INFO | ------------------------------------------ | 20260505 18:04:28.496 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:04:28.496 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 18:04:28.496 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:04:28.496 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:04:28.496 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.43e+01, 1.33e+01 / 1.73e+00 | 20260505 18:04:28.496 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 1.03e-01, 1.06e-02 / 5.45e-03 | 20260505 18:04:28.496 | INFO | PerfReport: 11.78%, Baseline -- 3.72e+00 / 4.54e+00, 5.69e-01 / 1.16e-01 | 20260505 18:04:28.496 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.52e+00, 1.92e-01 / 9.82e-02 | 20260505 18:04:28.496 | INFO | PerfReport: 15.91%, DME -- 4.02e+01 / 4.42e+01, 3.96e+00 / 1.72e+00 | 20260505 18:04:28.496 | INFO | PerfReport: 56.88%, FrameLabeling -- 1.80e+01 / 1.92e+01, 1.48e+00 / 9.25e-01 | 20260505 18:04:28.496 | INFO | PerfReport: 7.33%, PulseAccumulating -- 2.31e+00 / 2.91e+00, 4.47e-01 / 1.53e-01 | 20260505 18:04:28.496 | INFO | PerfReport: 3.35%, Metrics -- 1.06e+00 / 1.41e+00, 2.96e-01 / 1.72e-01 | 20260505 18:04:28.496 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.69e-01 / 4.30e-01, 6.54e-02 / 3.99e-02 | 20260505 18:04:28.496 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 18:04:28.496 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:04:28.496 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:04:28.496 | INFO | PerfReport: Full Execution -- 9.99e+01 / -nan, 2.59e+01 / 0.00e+00 | 20260505 18:04:28.496 | INFO | PerfReport: 22.54%, Upload -- 2.25e+01 / 0.00e+00, 7.87e+00 / 0.00e+00 | 20260505 18:04:28.496 | INFO | PerfReport: 21.75%, Download -- 2.17e+01 / 0.00e+00, 7.79e+00 / 0.00e+00 | 20260505 18:04:28.496 | INFO | PerfReport: 31.60%, Compute -- 3.16e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260505 18:04:28.496 | INFO | PerfReport: 2.39%, LockUpload -- 2.38e+00 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260505 18:04:28.496 | INFO | PerfReport: 1.37%, LockDownload -- 1.37e+00 / 0.00e+00, 2.92e+00 / 0.00e+00 | 20260505 18:04:28.497 | INFO | PerfReport: 20.33%, LockCompute -- 2.03e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260505 18:04:28.497 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.44e-02 / 0.00e+00 | 20260505 18:04:28.628 | INFO | Starting a round of Baz IO | 20260505 18:04:31.805 | INFO | Populating RTMetrics report | 20260505 18:04:31.805 | INFO | RTMetrics report populated | 20260505 18:04:31.805 | INFO | Opening tmp RTMetrics json file | 20260505 18:04:31.807 | INFO | Serializing RTMetrics to json | 20260505 18:04:31.809 | INFO | Serialization done, swapping tmp file now... | 20260505 18:04:31.810 | INFO | RTMetrics done | 20260505 18:04:31.900 | INFO | Finished a round of Baz IO: Wrote 2.107 GiB in 3.272 seconds | 20260505 18:04:31.900 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.96% / 0.00% / 78.04% | 20260505 18:04:31.900 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.268773% / 3.97364e-06% / 0.000115236% | 20260505 18:04:31.900 | INFO | Baz IO overall: | 20260505 18:04:31.900 | INFO | BytesWritten : 3077329411074 | 20260505 18:04:31.900 | INFO | NumEvents : 1458357107716 | 20260505 18:04:33.626 | INFO | Analyzing chunk frames = [5218304,5218816) | 20260505 18:04:33.626 | INFO | MemInfo: { | 20260505 18:04:33.626 | INFO | "gpu" : " 17 GB", | 20260505 18:04:33.626 | INFO | "huge" : "", | 20260505 18:04:33.626 | INFO | "huge_shared" : " 27 GB", | 20260505 18:04:33.626 | INFO | "malloc" : " 14 GB", | 20260505 18:04:33.626 | INFO | "pinned" : " 29 GB" | 20260505 18:04:33.626 | INFO | } | 20260505 18:04:33.626 | INFO | Perf report for the last 5.13165 seconds: | 20260505 18:04:33.627 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:04:33.627 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:33.627 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:33.627 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:33.627 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 18:04:33.627 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:33.627 | INFO | RT_METRICS : 44.8%, 1.0, 23.4ms, 0 | 20260505 18:04:33.627 | INFO | PRE_HQ : 60.5%, 1.8, 56.1ms, 0 | 20260505 18:04:33.627 | INFO | BAZWRITER : 63.3%, 3.8, 123.1ms, 0 | 20260505 18:04:33.627 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 18:04:33.627 | INFO | Uploaded 36GiB to GPU in 2.16303s (16.6433GiB/s) | 20260505 18:04:33.627 | INFO | Downloaded 38GiB to GPU in 2.25069s (16.8837GiB/s) | 20260505 18:04:38.761 | INFO | Analyzing chunk frames = [5218816,5219328) | 20260505 18:04:38.761 | INFO | MemInfo: { | 20260505 18:04:38.761 | INFO | "gpu" : " 17 GB", | 20260505 18:04:38.761 | INFO | "huge" : "", | 20260505 18:04:38.761 | INFO | "huge_shared" : " 27 GB", | 20260505 18:04:38.761 | INFO | "malloc" : " 14 GB", | 20260505 18:04:38.761 | INFO | "pinned" : " 29 GB" | 20260505 18:04:38.761 | INFO | } | 20260505 18:04:38.761 | INFO | Perf report for the last 5.13499 seconds: | 20260505 18:04:38.761 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:04:38.761 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:38.761 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:38.761 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:38.762 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 18:04:38.762 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:38.762 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:38.762 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:04:38.762 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:38.762 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:38.762 | INFO | Uploaded 36GiB to GPU in 2.13739s (16.843GiB/s) | 20260505 18:04:38.762 | INFO | Downloaded 34GiB to GPU in 2.03145s (16.7368GiB/s) | 20260505 18:04:43.907 | INFO | Analyzing chunk frames = [5219328,5219840) | 20260505 18:04:43.907 | INFO | MemInfo: { | 20260505 18:04:43.907 | INFO | "gpu" : " 17 GB", | 20260505 18:04:43.907 | INFO | "huge" : "", | 20260505 18:04:43.907 | INFO | "huge_shared" : " 27 GB", | 20260505 18:04:43.907 | INFO | "malloc" : " 14 GB", | 20260505 18:04:43.907 | INFO | "pinned" : " 29 GB" | 20260505 18:04:43.907 | INFO | } | 20260505 18:04:43.907 | INFO | Perf report for the last 5.14593 seconds: | 20260505 18:04:43.907 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:04:43.907 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:43.907 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:43.907 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:43.907 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 18:04:43.907 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:43.907 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:43.907 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260505 18:04:43.907 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:43.907 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:43.908 | INFO | Uploaded 36GiB to GPU in 2.2039s (16.3347GiB/s) | 20260505 18:04:43.908 | INFO | Downloaded 34GiB to GPU in 2.03925s (16.6728GiB/s) | 20260505 18:04:49.047 | INFO | Analyzing chunk frames = [5219840,5220352) | 20260505 18:04:49.047 | INFO | MemInfo: { | 20260505 18:04:49.047 | INFO | "gpu" : " 17 GB", | 20260505 18:04:49.047 | INFO | "huge" : "", | 20260505 18:04:49.047 | INFO | "huge_shared" : " 27 GB", | 20260505 18:04:49.047 | INFO | "malloc" : " 14 GB", | 20260505 18:04:49.047 | INFO | "pinned" : " 29 GB" | 20260505 18:04:49.047 | INFO | } | 20260505 18:04:49.047 | INFO | Perf report for the last 5.13955 seconds: | 20260505 18:04:49.047 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:04:49.047 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:49.047 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:49.047 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:49.047 | INFO | ANALYSIS : 62.8%, 3.0, 97.5ms, 0 | 20260505 18:04:49.047 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:49.047 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:49.047 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:04:49.047 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:49.047 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:49.047 | INFO | Uploaded 36GiB to GPU in 2.22747s (16.1618GiB/s) | 20260505 18:04:49.047 | INFO | Downloaded 34GiB to GPU in 2.11992s (16.0384GiB/s) | 20260505 18:04:54.182 | INFO | Analyzing chunk frames = [5220352,5220864) | 20260505 18:04:54.182 | INFO | MemInfo: { | 20260505 18:04:54.182 | INFO | "gpu" : " 17 GB", | 20260505 18:04:54.182 | INFO | "huge" : "", | 20260505 18:04:54.182 | INFO | "huge_shared" : " 27 GB", | 20260505 18:04:54.182 | INFO | "malloc" : " 14 GB", | 20260505 18:04:54.182 | INFO | "pinned" : " 29 GB" | 20260505 18:04:54.182 | INFO | } | 20260505 18:04:54.182 | INFO | Perf report for the last 5.13494 seconds: | 20260505 18:04:54.182 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:04:54.182 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:54.182 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:54.182 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:54.182 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260505 18:04:54.182 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:54.182 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:54.182 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:04:54.182 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:54.182 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:54.182 | INFO | Uploaded 36GiB to GPU in 2.14489s (16.7841GiB/s) | 20260505 18:04:54.182 | INFO | Downloaded 34GiB to GPU in 2.06044s (16.5013GiB/s) | 20260505 18:04:59.317 | INFO | Analyzing chunk frames = [5220864,5221376) | 20260505 18:04:59.317 | INFO | MemInfo: { | 20260505 18:04:59.317 | INFO | "gpu" : " 17 GB", | 20260505 18:04:59.317 | INFO | "huge" : "", | 20260505 18:04:59.317 | INFO | "huge_shared" : " 27 GB", | 20260505 18:04:59.317 | INFO | "malloc" : " 14 GB", | 20260505 18:04:59.317 | INFO | "pinned" : " 29 GB" | 20260505 18:04:59.317 | INFO | } | 20260505 18:04:59.317 | INFO | Perf report for the last 5.13498 seconds: | 20260505 18:04:59.317 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:04:59.317 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:59.317 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:59.317 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:59.317 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260505 18:04:59.317 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:59.317 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:59.317 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 18:04:59.317 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:59.317 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:04:59.317 | INFO | Uploaded 36GiB to GPU in 2.14934s (16.7493GiB/s) | 20260505 18:04:59.317 | INFO | Downloaded 34GiB to GPU in 2.04204s (16.65GiB/s) | 20260505 18:05:04.464 | INFO | Analyzing chunk frames = [5221376,5221888) | 20260505 18:05:04.464 | INFO | MemInfo: { | 20260505 18:05:04.464 | INFO | "gpu" : " 17 GB", | 20260505 18:05:04.464 | INFO | "huge" : "", | 20260505 18:05:04.464 | INFO | "huge_shared" : " 27 GB", | 20260505 18:05:04.464 | INFO | "malloc" : " 14 GB", | 20260505 18:05:04.464 | INFO | "pinned" : " 29 GB" | 20260505 18:05:04.464 | INFO | } | 20260505 18:05:04.464 | INFO | Perf report for the last 5.14742 seconds: | 20260505 18:05:04.464 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:05:04.464 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:04.464 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:04.464 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:04.464 | INFO | ANALYSIS : 63.3%, 3.0, 99.1ms, 0 | 20260505 18:05:04.464 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:04.464 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:04.464 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 18:05:04.464 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:04.464 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:04.465 | INFO | Uploaded 36GiB to GPU in 2.16227s (16.6492GiB/s) | 20260505 18:05:04.465 | INFO | Downloaded 34GiB to GPU in 2.16516s (15.7033GiB/s) | 20260505 18:05:09.609 | INFO | Analyzing chunk frames = [5221888,5222400) | 20260505 18:05:09.609 | INFO | MemInfo: { | 20260505 18:05:09.609 | INFO | "gpu" : " 17 GB", | 20260505 18:05:09.609 | INFO | "huge" : "", | 20260505 18:05:09.609 | INFO | "huge_shared" : " 27 GB", | 20260505 18:05:09.609 | INFO | "malloc" : " 14 GB", | 20260505 18:05:09.609 | INFO | "pinned" : " 29 GB" | 20260505 18:05:09.609 | INFO | } | 20260505 18:05:09.609 | INFO | Perf report for the last 5.14489 seconds: | 20260505 18:05:09.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:05:09.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:09.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:09.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:09.609 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 18:05:09.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:09.609 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:09.609 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:05:09.609 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:09.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:09.609 | INFO | Uploaded 36GiB to GPU in 2.1997s (16.3659GiB/s) | 20260505 18:05:09.609 | INFO | Downloaded 34GiB to GPU in 2.02533s (16.7874GiB/s) | 20260505 18:05:09.740 | INFO | Starting a round of Baz IO | 20260505 18:05:12.919 | INFO | Populating RTMetrics report | 20260505 18:05:12.919 | INFO | RTMetrics report populated | 20260505 18:05:12.919 | INFO | Opening tmp RTMetrics json file | 20260505 18:05:12.924 | INFO | Serializing RTMetrics to json | 20260505 18:05:12.926 | INFO | Serialization done, swapping tmp file now... | 20260505 18:05:12.927 | INFO | RTMetrics done | 20260505 18:05:13.011 | INFO | Finished a round of Baz IO: Wrote 2.104 GiB in 3.27 seconds | 20260505 18:05:13.011 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.96% / 0.00% / 78.04% | 20260505 18:05:13.011 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.267514% / 3.97364e-06% / 9.93411e-05% | 20260505 18:05:13.011 | INFO | Baz IO overall: | 20260505 18:05:13.011 | INFO | BytesWritten : 3079588625362 | 20260505 18:05:13.011 | INFO | NumEvents : 1459410381608 | 20260505 18:05:14.752 | INFO | Analyzing chunk frames = [5222400,5222912) | 20260505 18:05:14.752 | INFO | MemInfo: { | 20260505 18:05:14.752 | INFO | "gpu" : " 17 GB", | 20260505 18:05:14.752 | INFO | "huge" : "", | 20260505 18:05:14.752 | INFO | "huge_shared" : " 27 GB", | 20260505 18:05:14.752 | INFO | "malloc" : " 14 GB", | 20260505 18:05:14.752 | INFO | "pinned" : " 29 GB" | 20260505 18:05:14.752 | INFO | } | 20260505 18:05:14.752 | INFO | Perf report for the last 5.143 seconds: | 20260505 18:05:14.752 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:05:14.752 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:14.752 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:14.752 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:14.752 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 18:05:14.752 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:14.752 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260505 18:05:14.752 | INFO | PRE_HQ : 60.3%, 1.8, 55.7ms, 0 | 20260505 18:05:14.752 | INFO | BAZWRITER : 63.1%, 3.8, 123.5ms, 0 | 20260505 18:05:14.752 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 18:05:14.752 | INFO | Uploaded 36GiB to GPU in 2.16741s (16.6097GiB/s) | 20260505 18:05:14.753 | INFO | Downloaded 38GiB to GPU in 2.24731s (16.9091GiB/s) | 20260505 18:05:19.877 | INFO | Analyzing chunk frames = [5222912,5223424) | 20260505 18:05:19.877 | INFO | MemInfo: { | 20260505 18:05:19.877 | INFO | "gpu" : " 17 GB", | 20260505 18:05:19.877 | INFO | "huge" : "", | 20260505 18:05:19.877 | INFO | "huge_shared" : " 27 GB", | 20260505 18:05:19.877 | INFO | "malloc" : " 14 GB", | 20260505 18:05:19.877 | INFO | "pinned" : " 29 GB" | 20260505 18:05:19.878 | INFO | } | 20260505 18:05:19.878 | INFO | Perf report for the last 5.12537 seconds: | 20260505 18:05:19.878 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:05:19.878 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:19.878 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:19.878 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:19.878 | INFO | ANALYSIS : 62.9%, 3.0, 97.9ms, 0 | 20260505 18:05:19.878 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:19.878 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:19.878 | INFO | PRE_HQ : 27.5%, 1.0, 14.2ms, 0 | 20260505 18:05:19.878 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:19.878 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:19.878 | INFO | Uploaded 36GiB to GPU in 2.15886s (16.6755GiB/s) | 20260505 18:05:19.878 | INFO | Downloaded 34GiB to GPU in 2.05288s (16.5621GiB/s) | 20260505 18:05:25.014 | INFO | Analyzing chunk frames = [5223424,5223936) | 20260505 18:05:25.014 | INFO | MemInfo: { | 20260505 18:05:25.014 | INFO | "gpu" : " 17 GB", | 20260505 18:05:25.014 | INFO | "huge" : "", | 20260505 18:05:25.014 | INFO | "huge_shared" : " 27 GB", | 20260505 18:05:25.014 | INFO | "malloc" : " 14 GB", | 20260505 18:05:25.014 | INFO | "pinned" : " 29 GB" | 20260505 18:05:25.014 | INFO | } | 20260505 18:05:25.015 | INFO | Perf report for the last 5.13693 seconds: | 20260505 18:05:25.015 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:05:25.015 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:25.015 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:25.015 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:25.015 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 18:05:25.015 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:25.015 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:25.015 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:05:25.015 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:25.015 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:25.015 | INFO | Uploaded 36GiB to GPU in 2.20398s (16.3341GiB/s) | 20260505 18:05:25.015 | INFO | Downloaded 34GiB to GPU in 2.05156s (16.5727GiB/s) | 20260505 18:05:30.158 | INFO | Analyzing chunk frames = [5223936,5224448) | 20260505 18:05:30.158 | INFO | MemInfo: { | 20260505 18:05:30.158 | INFO | "gpu" : " 17 GB", | 20260505 18:05:30.158 | INFO | "huge" : "", | 20260505 18:05:30.158 | INFO | "huge_shared" : " 27 GB", | 20260505 18:05:30.158 | INFO | "malloc" : " 14 GB", | 20260505 18:05:30.158 | INFO | "pinned" : " 29 GB" | 20260505 18:05:30.159 | INFO | } | 20260505 18:05:30.159 | INFO | Perf report for the last 5.14416 seconds: | 20260505 18:05:30.159 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:05:30.159 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:30.159 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:30.159 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:30.159 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260505 18:05:30.159 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:30.159 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:30.159 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260505 18:05:30.159 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:30.159 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:30.159 | INFO | Uploaded 36GiB to GPU in 2.1911s (16.4301GiB/s) | 20260505 18:05:30.159 | INFO | Downloaded 34GiB to GPU in 2.13433s (15.93GiB/s) | 20260505 18:05:35.309 | INFO | Analyzing chunk frames = [5224448,5224960) | 20260505 18:05:35.309 | INFO | MemInfo: { | 20260505 18:05:35.309 | INFO | "gpu" : " 17 GB", | 20260505 18:05:35.309 | INFO | "huge" : "", | 20260505 18:05:35.309 | INFO | "huge_shared" : " 27 GB", | 20260505 18:05:35.309 | INFO | "malloc" : " 14 GB", | 20260505 18:05:35.309 | INFO | "pinned" : " 29 GB" | 20260505 18:05:35.309 | INFO | } | 20260505 18:05:35.309 | INFO | Perf report for the last 5.15056 seconds: | 20260505 18:05:35.309 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:05:35.309 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:35.309 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:35.309 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:35.309 | INFO | ANALYSIS : 61.9%, 3.0, 96.8ms, 0 | 20260505 18:05:35.309 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:35.309 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:35.309 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:05:35.309 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:35.309 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:35.310 | INFO | Uploaded 36GiB to GPU in 2.14626s (16.7733GiB/s) | 20260505 18:05:35.310 | INFO | Downloaded 34GiB to GPU in 2.05955s (16.5084GiB/s) | 20260505 18:05:40.438 | INFO | Analyzing chunk frames = [5224960,5225472) | 20260505 18:05:40.439 | INFO | MemInfo: { | 20260505 18:05:40.439 | INFO | "gpu" : " 17 GB", | 20260505 18:05:40.439 | INFO | "huge" : "", | 20260505 18:05:40.439 | INFO | "huge_shared" : " 27 GB", | 20260505 18:05:40.439 | INFO | "malloc" : " 14 GB", | 20260505 18:05:40.439 | INFO | "pinned" : " 29 GB" | 20260505 18:05:40.439 | INFO | } | 20260505 18:05:40.439 | INFO | Perf report for the last 5.12928 seconds: | 20260505 18:05:40.439 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:05:40.439 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:40.439 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:40.439 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:40.439 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260505 18:05:40.439 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:40.439 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:40.439 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 18:05:40.439 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:40.439 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:40.439 | INFO | Uploaded 36GiB to GPU in 2.15873s (16.6765GiB/s) | 20260505 18:05:40.439 | INFO | Downloaded 34GiB to GPU in 2.05847s (16.5171GiB/s) | 20260505 18:05:45.576 | INFO | Analyzing chunk frames = [5225472,5225984) | 20260505 18:05:45.576 | INFO | MemInfo: { | 20260505 18:05:45.576 | INFO | "gpu" : " 17 GB", | 20260505 18:05:45.576 | INFO | "huge" : "", | 20260505 18:05:45.576 | INFO | "huge_shared" : " 27 GB", | 20260505 18:05:45.576 | INFO | "malloc" : " 14 GB", | 20260505 18:05:45.576 | INFO | "pinned" : " 29 GB" | 20260505 18:05:45.576 | INFO | } | 20260505 18:05:45.576 | INFO | Perf report for the last 5.137 seconds: | 20260505 18:05:45.576 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:05:45.576 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:45.576 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:45.576 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:45.576 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 18:05:45.576 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:45.576 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:45.576 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 18:05:45.576 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:45.576 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:45.576 | INFO | Uploaded 36GiB to GPU in 2.17388s (16.5602GiB/s) | 20260505 18:05:45.576 | INFO | Downloaded 34GiB to GPU in 2.18825s (15.5375GiB/s) | 20260505 18:05:50.719 | INFO | Analyzing chunk frames = [5225984,5226496) | 20260505 18:05:50.719 | INFO | MemInfo: { | 20260505 18:05:50.719 | INFO | "gpu" : " 17 GB", | 20260505 18:05:50.719 | INFO | "huge" : "", | 20260505 18:05:50.719 | INFO | "huge_shared" : " 27 GB", | 20260505 18:05:50.719 | INFO | "malloc" : " 14 GB", | 20260505 18:05:50.719 | INFO | "pinned" : " 29 GB" | 20260505 18:05:50.719 | INFO | } | 20260505 18:05:50.719 | INFO | Perf report for the last 5.14366 seconds: | 20260505 18:05:50.719 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:05:50.719 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:50.719 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:50.719 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:50.719 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 18:05:50.719 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:50.719 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:50.719 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 18:05:50.720 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:50.720 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:50.720 | INFO | Uploaded 36GiB to GPU in 2.16579s (16.6221GiB/s) | 20260505 18:05:50.720 | INFO | Downloaded 34GiB to GPU in 2.02577s (16.7837GiB/s) | 20260505 18:05:50.720 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 18:05:50.720 | INFO | Total mem usage: 15.3 GiB | 20260505 18:05:50.720 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 18:05:50.720 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.01%) | 20260505 18:05:50.720 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 18:05:50.720 | INFO | High Water Breakdown: | 20260505 18:05:50.720 | INFO | 61% : 8.66 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 18:05:50.720 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 18:05:50.720 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 18:05:50.720 | INFO | ------------------------------------------ | 20260505 18:05:50.720 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:05:50.720 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 18:05:50.720 | INFO | Total mem usage: 26.5 GiB | 20260505 18:05:50.720 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 18:05:50.720 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 18:05:50.720 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 18:05:50.720 | INFO | High Water Breakdown: | 20260505 18:05:50.720 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:05:50.720 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:05:50.720 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:05:50.720 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:05:50.720 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:05:50.720 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:05:50.720 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 18:05:50.720 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 18:05:50.720 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:05:50.720 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 18:05:50.720 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 18:05:50.720 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 18:05:50.720 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 18:05:50.720 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 18:05:50.720 | INFO | ------------------------------------------ | 20260505 18:05:50.720 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:05:50.720 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 18:05:50.720 | INFO | Total mem usage: 25.6 GiB | 20260505 18:05:50.720 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 18:05:50.720 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 18:05:50.720 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 18:05:50.720 | INFO | High Water Breakdown: | 20260505 18:05:50.720 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 18:05:50.720 | INFO | ------------------------------------------ | 20260505 18:05:50.720 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:05:50.720 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 18:05:50.720 | INFO | Total mem usage: 15.7 GiB | 20260505 18:05:50.720 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 18:05:50.720 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 18:05:50.720 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 18:05:50.720 | INFO | High Water Breakdown: | 20260505 18:05:50.720 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 18:05:50.720 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 18:05:50.720 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 18:05:50.720 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 18:05:50.720 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 18:05:50.720 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 18:05:50.720 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 18:05:50.721 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 18:05:50.721 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 18:05:50.721 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:05:50.721 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 18:05:50.721 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:05:50.721 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 18:05:50.721 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 18:05:50.721 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 18:05:50.721 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 18:05:50.721 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:05:50.721 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 18:05:50.721 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:05:50.721 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:05:50.721 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:05:50.721 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 18:05:50.721 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 18:05:50.721 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:05:50.721 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 18:05:50.721 | INFO | ------------------------------------------ | 20260505 18:05:50.721 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:05:50.721 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 18:05:50.721 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:05:50.721 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:05:50.721 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.46e+01, 1.36e+01 / 1.64e+00 | 20260505 18:05:50.721 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 1.02e-01, 1.06e-02 / 2.99e-03 | 20260505 18:05:50.721 | INFO | PerfReport: 11.77%, Baseline -- 3.71e+00 / 4.54e+00, 5.62e-01 / 1.25e-01 | 20260505 18:05:50.721 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.51e+00, 1.91e-01 / 1.03e-01 | 20260505 18:05:50.721 | INFO | PerfReport: 15.92%, DME -- 4.02e+01 / 4.43e+01, 4.02e+00 / 1.86e+00 | 20260505 18:05:50.721 | INFO | PerfReport: 56.89%, FrameLabeling -- 1.79e+01 / 1.95e+01, 1.46e+00 / 1.11e+00 | 20260505 18:05:50.721 | INFO | PerfReport: 7.33%, PulseAccumulating -- 2.31e+00 / 2.89e+00, 4.44e-01 / 1.53e-01 | 20260505 18:05:50.721 | INFO | PerfReport: 3.36%, Metrics -- 1.06e+00 / 1.38e+00, 2.93e-01 / 1.52e-01 | 20260505 18:05:50.721 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.68e-01 / 4.29e-01, 6.53e-02 / 3.98e-02 | 20260505 18:05:50.721 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 18:05:50.721 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:05:50.721 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:05:50.721 | INFO | PerfReport: Full Execution -- 9.99e+01 / -nan, 2.55e+01 / 0.00e+00 | 20260505 18:05:50.721 | INFO | PerfReport: 22.65%, Upload -- 2.26e+01 / 0.00e+00, 7.96e+00 / 0.00e+00 | 20260505 18:05:50.721 | INFO | PerfReport: 21.87%, Download -- 2.18e+01 / 0.00e+00, 7.85e+00 / 0.00e+00 | 20260505 18:05:50.721 | INFO | PerfReport: 31.58%, Compute -- 3.16e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 18:05:50.721 | INFO | PerfReport: 2.30%, LockUpload -- 2.29e+00 / 0.00e+00, 7.81e+00 / 0.00e+00 | 20260505 18:05:50.721 | INFO | PerfReport: 1.34%, LockDownload -- 1.34e+00 / 0.00e+00, 2.77e+00 / 0.00e+00 | 20260505 18:05:50.721 | INFO | PerfReport: 20.25%, LockCompute -- 2.02e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 18:05:50.721 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.62e-02 / 0.00e+00 | 20260505 18:05:50.854 | INFO | Starting a round of Baz IO | 20260505 18:05:54.032 | INFO | Populating RTMetrics report | 20260505 18:05:54.032 | INFO | RTMetrics report populated | 20260505 18:05:54.032 | INFO | Opening tmp RTMetrics json file | 20260505 18:05:54.036 | INFO | Serializing RTMetrics to json | 20260505 18:05:54.037 | INFO | Serialization done, swapping tmp file now... | 20260505 18:05:54.038 | INFO | RTMetrics done | 20260505 18:05:54.120 | INFO | Finished a round of Baz IO: Wrote 2.106 GiB in 3.266 seconds | 20260505 18:05:54.120 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.96% / 0.00% / 78.04% | 20260505 18:05:54.120 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.266476% / 7.94729e-06% / 8.74201e-05% | 20260505 18:05:54.120 | INFO | Baz IO overall: | 20260505 18:05:54.120 | INFO | BytesWritten : 3081849517642 | 20260505 18:05:54.120 | INFO | NumEvents : 1460465043759 | 20260505 18:05:55.860 | INFO | Analyzing chunk frames = [5226496,5227008) | 20260505 18:05:55.860 | INFO | MemInfo: { | 20260505 18:05:55.860 | INFO | "gpu" : " 17 GB", | 20260505 18:05:55.860 | INFO | "huge" : "", | 20260505 18:05:55.860 | INFO | "huge_shared" : " 27 GB", | 20260505 18:05:55.860 | INFO | "malloc" : " 14 GB", | 20260505 18:05:55.860 | INFO | "pinned" : " 29 GB" | 20260505 18:05:55.860 | INFO | } | 20260505 18:05:55.860 | INFO | Perf report for the last 5.14119 seconds: | 20260505 18:05:55.860 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:05:55.860 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:55.860 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:55.860 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:55.861 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 18:05:55.861 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:05:55.861 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260505 18:05:55.861 | INFO | PRE_HQ : 60.3%, 1.8, 56.0ms, 0 | 20260505 18:05:55.861 | INFO | BAZWRITER : 63.1%, 3.8, 123.6ms, 0 | 20260505 18:05:55.861 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 18:05:55.861 | INFO | Uploaded 36GiB to GPU in 2.1564s (16.6945GiB/s) | 20260505 18:05:55.861 | INFO | Downloaded 38GiB to GPU in 2.25396s (16.8592GiB/s) | 20260505 18:06:00.995 | INFO | Analyzing chunk frames = [5227008,5227520) | 20260505 18:06:00.995 | INFO | MemInfo: { | 20260505 18:06:00.995 | INFO | "gpu" : " 17 GB", | 20260505 18:06:00.995 | INFO | "huge" : "", | 20260505 18:06:00.995 | INFO | "huge_shared" : " 27 GB", | 20260505 18:06:00.995 | INFO | "malloc" : " 14 GB", | 20260505 18:06:00.996 | INFO | "pinned" : " 29 GB" | 20260505 18:06:00.996 | INFO | } | 20260505 18:06:00.996 | INFO | Perf report for the last 5.13513 seconds: | 20260505 18:06:00.996 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:06:00.996 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:00.996 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:00.996 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:00.996 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260505 18:06:00.996 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:00.996 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:00.996 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:06:00.996 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:00.996 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:00.996 | INFO | Uploaded 36GiB to GPU in 2.15136s (16.7336GiB/s) | 20260505 18:06:00.996 | INFO | Downloaded 34GiB to GPU in 2.04799s (16.6016GiB/s) | 20260505 18:06:06.133 | INFO | Analyzing chunk frames = [5227520,5228032) | 20260505 18:06:06.133 | INFO | MemInfo: { | 20260505 18:06:06.133 | INFO | "gpu" : " 17 GB", | 20260505 18:06:06.133 | INFO | "huge" : "", | 20260505 18:06:06.133 | INFO | "huge_shared" : " 27 GB", | 20260505 18:06:06.133 | INFO | "malloc" : " 14 GB", | 20260505 18:06:06.133 | INFO | "pinned" : " 29 GB" | 20260505 18:06:06.133 | INFO | } | 20260505 18:06:06.133 | INFO | Perf report for the last 5.13707 seconds: | 20260505 18:06:06.133 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:06:06.133 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:06.133 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:06.133 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:06.133 | INFO | ANALYSIS : 62.7%, 3.0, 97.5ms, 0 | 20260505 18:06:06.133 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:06.133 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:06.133 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 18:06:06.133 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:06.133 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:06.133 | INFO | Uploaded 36GiB to GPU in 2.19802s (16.3784GiB/s) | 20260505 18:06:06.133 | INFO | Downloaded 34GiB to GPU in 2.04226s (16.6483GiB/s) | 20260505 18:06:11.270 | INFO | Analyzing chunk frames = [5228032,5228544) | 20260505 18:06:11.270 | INFO | MemInfo: { | 20260505 18:06:11.270 | INFO | "gpu" : " 17 GB", | 20260505 18:06:11.270 | INFO | "huge" : "", | 20260505 18:06:11.270 | INFO | "huge_shared" : " 27 GB", | 20260505 18:06:11.270 | INFO | "malloc" : " 14 GB", | 20260505 18:06:11.270 | INFO | "pinned" : " 29 GB" | 20260505 18:06:11.270 | INFO | } | 20260505 18:06:11.270 | INFO | Perf report for the last 5.13755 seconds: | 20260505 18:06:11.270 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:06:11.270 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:11.270 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:11.270 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:11.270 | INFO | ANALYSIS : 62.5%, 3.0, 97.0ms, 0 | 20260505 18:06:11.270 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:11.270 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:11.270 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:06:11.270 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:11.270 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:11.271 | INFO | Uploaded 36GiB to GPU in 2.16429s (16.6336GiB/s) | 20260505 18:06:11.271 | INFO | Downloaded 34GiB to GPU in 2.14424s (15.8564GiB/s) | 20260505 18:06:16.419 | INFO | Analyzing chunk frames = [5228544,5229056) | 20260505 18:06:16.419 | INFO | MemInfo: { | 20260505 18:06:16.419 | INFO | "gpu" : " 17 GB", | 20260505 18:06:16.419 | INFO | "huge" : "", | 20260505 18:06:16.419 | INFO | "huge_shared" : " 27 GB", | 20260505 18:06:16.419 | INFO | "malloc" : " 14 GB", | 20260505 18:06:16.420 | INFO | "pinned" : " 29 GB" | 20260505 18:06:16.420 | INFO | } | 20260505 18:06:16.420 | INFO | Perf report for the last 5.14945 seconds: | 20260505 18:06:16.420 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:06:16.420 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:16.420 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:16.420 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:16.420 | INFO | ANALYSIS : 62.1%, 3.0, 97.2ms, 0 | 20260505 18:06:16.420 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:16.420 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:16.420 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 18:06:16.420 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:16.420 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:16.420 | INFO | Uploaded 36GiB to GPU in 2.16037s (16.6638GiB/s) | 20260505 18:06:16.420 | INFO | Downloaded 34GiB to GPU in 2.06339s (16.4777GiB/s) | 20260505 18:06:21.563 | INFO | Analyzing chunk frames = [5229056,5229568) | 20260505 18:06:21.563 | INFO | MemInfo: { | 20260505 18:06:21.563 | INFO | "gpu" : " 17 GB", | 20260505 18:06:21.563 | INFO | "huge" : "", | 20260505 18:06:21.563 | INFO | "huge_shared" : " 27 GB", | 20260505 18:06:21.563 | INFO | "malloc" : " 14 GB", | 20260505 18:06:21.563 | INFO | "pinned" : " 29 GB" | 20260505 18:06:21.563 | INFO | } | 20260505 18:06:21.563 | INFO | Perf report for the last 5.14309 seconds: | 20260505 18:06:21.563 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:06:21.563 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:21.563 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:21.563 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:21.563 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260505 18:06:21.563 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:21.563 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:21.563 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260505 18:06:21.563 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:21.563 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:21.563 | INFO | Uploaded 36GiB to GPU in 2.13627s (16.8518GiB/s) | 20260505 18:06:21.563 | INFO | Downloaded 34GiB to GPU in 2.05106s (16.5768GiB/s) | 20260505 18:06:26.697 | INFO | Analyzing chunk frames = [5229568,5230080) | 20260505 18:06:26.697 | INFO | MemInfo: { | 20260505 18:06:26.697 | INFO | "gpu" : " 17 GB", | 20260505 18:06:26.697 | INFO | "huge" : "", | 20260505 18:06:26.697 | INFO | "huge_shared" : " 27 GB", | 20260505 18:06:26.697 | INFO | "malloc" : " 14 GB", | 20260505 18:06:26.697 | INFO | "pinned" : " 29 GB" | 20260505 18:06:26.697 | INFO | } | 20260505 18:06:26.697 | INFO | Perf report for the last 5.13464 seconds: | 20260505 18:06:26.697 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:06:26.697 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:26.697 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:26.697 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:26.698 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 18:06:26.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:26.698 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:26.698 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 18:06:26.698 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:26.698 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:26.698 | INFO | Uploaded 36GiB to GPU in 2.13233s (16.8829GiB/s) | 20260505 18:06:26.698 | INFO | Downloaded 34GiB to GPU in 2.12231s (16.0203GiB/s) | 20260505 18:06:31.830 | INFO | Analyzing chunk frames = [5230080,5230592) | 20260505 18:06:31.830 | INFO | MemInfo: { | 20260505 18:06:31.830 | INFO | "gpu" : " 17 GB", | 20260505 18:06:31.830 | INFO | "huge" : "", | 20260505 18:06:31.830 | INFO | "huge_shared" : " 27 GB", | 20260505 18:06:31.830 | INFO | "malloc" : " 14 GB", | 20260505 18:06:31.830 | INFO | "pinned" : " 29 GB" | 20260505 18:06:31.830 | INFO | } | 20260505 18:06:31.830 | INFO | Perf report for the last 5.1326 seconds: | 20260505 18:06:31.830 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:06:31.830 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:31.830 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:31.830 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:31.830 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 18:06:31.830 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:31.830 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:31.830 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 18:06:31.830 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:31.830 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:31.830 | INFO | Uploaded 36GiB to GPU in 2.19676s (16.3878GiB/s) | 20260505 18:06:31.830 | INFO | Downloaded 34GiB to GPU in 2.03608s (16.6988GiB/s) | 20260505 18:06:31.970 | INFO | Starting a round of Baz IO | 20260505 18:06:35.152 | INFO | Populating RTMetrics report | 20260505 18:06:35.152 | INFO | RTMetrics report populated | 20260505 18:06:35.152 | INFO | Opening tmp RTMetrics json file | 20260505 18:06:35.153 | INFO | Serializing RTMetrics to json | 20260505 18:06:35.156 | INFO | Serialization done, swapping tmp file now... | 20260505 18:06:35.157 | INFO | RTMetrics done | 20260505 18:06:35.247 | INFO | Finished a round of Baz IO: Wrote 2.106 GiB in 3.277 seconds | 20260505 18:06:35.247 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.96% / 0.00% / 78.04% | 20260505 18:06:35.248 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.265574% / 3.97364e-06% / 9.93411e-05% | 20260505 18:06:35.248 | INFO | Baz IO overall: | 20260505 18:06:35.248 | INFO | BytesWritten : 3084110445649 | 20260505 18:06:35.248 | INFO | NumEvents : 1461519878668 | 20260505 18:06:36.970 | INFO | Analyzing chunk frames = [5230592,5231104) | 20260505 18:06:36.971 | INFO | MemInfo: { | 20260505 18:06:36.971 | INFO | "gpu" : " 17 GB", | 20260505 18:06:36.971 | INFO | "huge" : "", | 20260505 18:06:36.971 | INFO | "huge_shared" : " 27 GB", | 20260505 18:06:36.971 | INFO | "malloc" : " 14 GB", | 20260505 18:06:36.971 | INFO | "pinned" : " 29 GB" | 20260505 18:06:36.971 | INFO | } | 20260505 18:06:36.971 | INFO | Perf report for the last 5.1406 seconds: | 20260505 18:06:36.971 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:06:36.971 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:36.971 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:36.971 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:36.971 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 18:06:36.971 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:36.971 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260505 18:06:36.971 | INFO | PRE_HQ : 59.9%, 1.8, 55.7ms, 0 | 20260505 18:06:36.971 | INFO | BAZWRITER : 63.2%, 3.8, 123.1ms, 0 | 20260505 18:06:36.971 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 18:06:36.971 | INFO | Uploaded 36GiB to GPU in 2.15055s (16.7399GiB/s) | 20260505 18:06:36.971 | INFO | Downloaded 38GiB to GPU in 2.25164s (16.8766GiB/s) | 20260505 18:06:42.112 | INFO | Analyzing chunk frames = [5231104,5231616) | 20260505 18:06:42.112 | INFO | MemInfo: { | 20260505 18:06:42.112 | INFO | "gpu" : " 17 GB", | 20260505 18:06:42.112 | INFO | "huge" : "", | 20260505 18:06:42.112 | INFO | "huge_shared" : " 27 GB", | 20260505 18:06:42.112 | INFO | "malloc" : " 14 GB", | 20260505 18:06:42.112 | INFO | "pinned" : " 29 GB" | 20260505 18:06:42.112 | INFO | } | 20260505 18:06:42.112 | INFO | Perf report for the last 5.14175 seconds: | 20260505 18:06:42.112 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:06:42.112 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:42.113 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:42.113 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:42.113 | INFO | ANALYSIS : 62.7%, 3.0, 98.0ms, 0 | 20260505 18:06:42.113 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:42.113 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:42.113 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:06:42.113 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:42.113 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:42.113 | INFO | Uploaded 36GiB to GPU in 2.13787s (16.8392GiB/s) | 20260505 18:06:42.113 | INFO | Downloaded 34GiB to GPU in 2.03062s (16.7437GiB/s) | 20260505 18:06:47.261 | INFO | Analyzing chunk frames = [5231616,5232128) | 20260505 18:06:47.261 | INFO | MemInfo: { | 20260505 18:06:47.261 | INFO | "gpu" : " 17 GB", | 20260505 18:06:47.261 | INFO | "huge" : "", | 20260505 18:06:47.261 | INFO | "huge_shared" : " 27 GB", | 20260505 18:06:47.261 | INFO | "malloc" : " 14 GB", | 20260505 18:06:47.261 | INFO | "pinned" : " 29 GB" | 20260505 18:06:47.261 | INFO | } | 20260505 18:06:47.261 | INFO | Perf report for the last 5.14894 seconds: | 20260505 18:06:47.261 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:06:47.261 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:47.261 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:47.261 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:47.261 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 18:06:47.261 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:47.261 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:47.261 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260505 18:06:47.261 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:47.261 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:47.262 | INFO | Uploaded 36GiB to GPU in 2.20781s (16.3058GiB/s) | 20260505 18:06:47.262 | INFO | Downloaded 34GiB to GPU in 2.07257s (16.4048GiB/s) | 20260505 18:06:52.392 | INFO | Analyzing chunk frames = [5232128,5232640) | 20260505 18:06:52.392 | INFO | MemInfo: { | 20260505 18:06:52.392 | INFO | "gpu" : " 17 GB", | 20260505 18:06:52.392 | INFO | "huge" : "", | 20260505 18:06:52.392 | INFO | "huge_shared" : " 27 GB", | 20260505 18:06:52.392 | INFO | "malloc" : " 14 GB", | 20260505 18:06:52.392 | INFO | "pinned" : " 29 GB" | 20260505 18:06:52.392 | INFO | } | 20260505 18:06:52.392 | INFO | Perf report for the last 5.13081 seconds: | 20260505 18:06:52.392 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:06:52.392 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:52.392 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:52.392 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:52.392 | INFO | ANALYSIS : 62.4%, 3.0, 96.8ms, 0 | 20260505 18:06:52.392 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:52.392 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:52.392 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:06:52.392 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:52.392 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:52.392 | INFO | Uploaded 36GiB to GPU in 2.18458s (16.4792GiB/s) | 20260505 18:06:52.393 | INFO | Downloaded 34GiB to GPU in 2.05629s (16.5347GiB/s) | 20260505 18:06:57.523 | INFO | Analyzing chunk frames = [5232640,5233152) | 20260505 18:06:57.523 | INFO | MemInfo: { | 20260505 18:06:57.523 | INFO | "gpu" : " 17 GB", | 20260505 18:06:57.523 | INFO | "huge" : "", | 20260505 18:06:57.523 | INFO | "huge_shared" : " 27 GB", | 20260505 18:06:57.523 | INFO | "malloc" : " 14 GB", | 20260505 18:06:57.523 | INFO | "pinned" : " 29 GB" | 20260505 18:06:57.523 | INFO | } | 20260505 18:06:57.523 | INFO | Perf report for the last 5.13061 seconds: | 20260505 18:06:57.523 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:06:57.523 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:57.523 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:57.523 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:57.523 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260505 18:06:57.523 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:57.523 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:57.523 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:06:57.523 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:57.523 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:06:57.523 | INFO | Uploaded 36GiB to GPU in 2.15288s (16.7218GiB/s) | 20260505 18:06:57.523 | INFO | Downloaded 34GiB to GPU in 2.06383s (16.4742GiB/s) | 20260505 18:07:02.674 | INFO | Analyzing chunk frames = [5233152,5233664) | 20260505 18:07:02.674 | INFO | MemInfo: { | 20260505 18:07:02.674 | INFO | "gpu" : " 17 GB", | 20260505 18:07:02.674 | INFO | "huge" : "", | 20260505 18:07:02.674 | INFO | "huge_shared" : " 27 GB", | 20260505 18:07:02.674 | INFO | "malloc" : " 14 GB", | 20260505 18:07:02.674 | INFO | "pinned" : " 29 GB" | 20260505 18:07:02.674 | INFO | } | 20260505 18:07:02.674 | INFO | Perf report for the last 5.15097 seconds: | 20260505 18:07:02.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:07:02.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:02.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:02.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:02.674 | INFO | ANALYSIS : 62.2%, 3.0, 97.4ms, 0 | 20260505 18:07:02.674 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:02.674 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:02.674 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260505 18:07:02.674 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:02.674 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:02.674 | INFO | Uploaded 36GiB to GPU in 2.14004s (16.8221GiB/s) | 20260505 18:07:02.674 | INFO | Downloaded 34GiB to GPU in 2.04406s (16.6336GiB/s) | 20260505 18:07:07.814 | INFO | Analyzing chunk frames = [5233664,5234176) | 20260505 18:07:07.814 | INFO | MemInfo: { | 20260505 18:07:07.814 | INFO | "gpu" : " 17 GB", | 20260505 18:07:07.814 | INFO | "huge" : "", | 20260505 18:07:07.814 | INFO | "huge_shared" : " 27 GB", | 20260505 18:07:07.814 | INFO | "malloc" : " 14 GB", | 20260505 18:07:07.814 | INFO | "pinned" : " 29 GB" | 20260505 18:07:07.814 | INFO | } | 20260505 18:07:07.814 | INFO | Perf report for the last 5.14052 seconds: | 20260505 18:07:07.814 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:07:07.814 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:07.814 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:07.814 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:07.814 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 18:07:07.814 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:07.814 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:07.814 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 18:07:07.814 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:07.814 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:07.815 | INFO | Uploaded 36GiB to GPU in 2.12331s (16.9546GiB/s) | 20260505 18:07:07.815 | INFO | Downloaded 34GiB to GPU in 2.14445s (15.8548GiB/s) | 20260505 18:07:12.943 | INFO | Analyzing chunk frames = [5234176,5234688) | 20260505 18:07:12.943 | INFO | MemInfo: { | 20260505 18:07:12.943 | INFO | "gpu" : " 17 GB", | 20260505 18:07:12.943 | INFO | "huge" : "", | 20260505 18:07:12.943 | INFO | "huge_shared" : " 27 GB", | 20260505 18:07:12.943 | INFO | "malloc" : " 14 GB", | 20260505 18:07:12.944 | INFO | "pinned" : " 29 GB" | 20260505 18:07:12.944 | INFO | } | 20260505 18:07:12.944 | INFO | Perf report for the last 5.12937 seconds: | 20260505 18:07:12.944 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:07:12.944 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:12.944 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:12.944 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:12.944 | INFO | ANALYSIS : 62.9%, 3.0, 98.0ms, 0 | 20260505 18:07:12.944 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:12.944 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:12.944 | INFO | PRE_HQ : 27.7%, 1.0, 14.3ms, 0 | 20260505 18:07:12.944 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:12.944 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:12.944 | INFO | Uploaded 36GiB to GPU in 2.19563s (16.3962GiB/s) | 20260505 18:07:12.944 | INFO | Downloaded 34GiB to GPU in 2.0274s (16.7703GiB/s) | 20260505 18:07:12.944 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 18:07:12.944 | INFO | Total mem usage: 15.3 GiB | 20260505 18:07:12.944 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 18:07:12.944 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.04%) | 20260505 18:07:12.944 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 18:07:12.944 | INFO | High Water Breakdown: | 20260505 18:07:12.944 | INFO | 61% : 8.66 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 18:07:12.944 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 18:07:12.944 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 18:07:12.944 | INFO | ------------------------------------------ | 20260505 18:07:12.944 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:07:12.944 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 18:07:12.944 | INFO | Total mem usage: 26.5 GiB | 20260505 18:07:12.944 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 18:07:12.944 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 18:07:12.944 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 18:07:12.944 | INFO | High Water Breakdown: | 20260505 18:07:12.944 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:07:12.944 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:07:12.944 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:07:12.945 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:07:12.945 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:07:12.945 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:07:12.945 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 18:07:12.945 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 18:07:12.945 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:07:12.945 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 18:07:12.945 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 18:07:12.945 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 18:07:12.945 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 18:07:12.945 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 18:07:12.945 | INFO | ------------------------------------------ | 20260505 18:07:12.945 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:07:12.945 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 18:07:12.945 | INFO | Total mem usage: 25.6 GiB | 20260505 18:07:12.945 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 18:07:12.945 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 18:07:12.945 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 18:07:12.945 | INFO | High Water Breakdown: | 20260505 18:07:12.945 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 18:07:12.945 | INFO | ------------------------------------------ | 20260505 18:07:12.945 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:07:12.945 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 18:07:12.945 | INFO | Total mem usage: 15.7 GiB | 20260505 18:07:12.945 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 18:07:12.945 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 18:07:12.945 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 18:07:12.945 | INFO | High Water Breakdown: | 20260505 18:07:12.945 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 18:07:12.945 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 18:07:12.945 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 18:07:12.945 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 18:07:12.945 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 18:07:12.945 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 18:07:12.945 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 18:07:12.945 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 18:07:12.945 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 18:07:12.945 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:07:12.945 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 18:07:12.945 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:07:12.945 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 18:07:12.945 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 18:07:12.945 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 18:07:12.945 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 18:07:12.945 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:07:12.945 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 18:07:12.945 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:07:12.945 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:07:12.945 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:07:12.945 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 18:07:12.945 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 18:07:12.945 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:07:12.945 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 18:07:12.945 | INFO | ------------------------------------------ | 20260505 18:07:12.945 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:07:12.945 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 18:07:12.945 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:07:12.945 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:07:12.945 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.46e+01, 1.37e+01 / 2.01e+00 | 20260505 18:07:12.945 | INFO | PerfReport: 0.29%, AggregateStats -- 9.27e-02 / 1.02e-01, 1.07e-02 / 3.62e-03 | 20260505 18:07:12.945 | INFO | PerfReport: 11.79%, Baseline -- 3.72e+00 / 4.56e+00, 5.66e-01 / 5.29e-02 | 20260505 18:07:12.945 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.54e+00, 1.92e-01 / 7.24e-02 | 20260505 18:07:12.945 | INFO | PerfReport: 15.92%, DME -- 4.01e+01 / 4.44e+01, 3.91e+00 / 1.96e+00 | 20260505 18:07:12.945 | INFO | PerfReport: 56.84%, FrameLabeling -- 1.79e+01 / 1.92e+01, 1.40e+00 / 9.58e-01 | 20260505 18:07:12.945 | INFO | PerfReport: 7.35%, PulseAccumulating -- 2.32e+00 / 2.90e+00, 4.47e-01 / 1.73e-01 | 20260505 18:07:12.945 | INFO | PerfReport: 3.36%, Metrics -- 1.06e+00 / 1.42e+00, 2.94e-01 / 1.49e-01 | 20260505 18:07:12.945 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.68e-01 / 4.37e-01, 6.46e-02 / 3.52e-02 | 20260505 18:07:12.945 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 18:07:12.945 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:07:12.945 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:07:12.945 | INFO | PerfReport: Full Execution -- 9.98e+01 / -nan, 2.57e+01 / 0.00e+00 | 20260505 18:07:12.945 | INFO | PerfReport: 22.55%, Upload -- 2.25e+01 / 0.00e+00, 7.85e+00 / 0.00e+00 | 20260505 18:07:12.945 | INFO | PerfReport: 21.81%, Download -- 2.18e+01 / 0.00e+00, 7.80e+00 / 0.00e+00 | 20260505 18:07:12.945 | INFO | PerfReport: 31.58%, Compute -- 3.15e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 18:07:12.945 | INFO | PerfReport: 2.31%, LockUpload -- 2.31e+00 / 0.00e+00, 8.04e+00 / 0.00e+00 | 20260505 18:07:12.945 | INFO | PerfReport: 1.42%, LockDownload -- 1.42e+00 / 0.00e+00, 2.93e+00 / 0.00e+00 | 20260505 18:07:12.946 | INFO | PerfReport: 20.30%, LockCompute -- 2.03e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260505 18:07:12.946 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.30e-02 / 0.00e+00 | 20260505 18:07:13.086 | INFO | Starting a round of Baz IO | 20260505 18:07:16.264 | INFO | Populating RTMetrics report | 20260505 18:07:16.265 | INFO | RTMetrics report populated | 20260505 18:07:16.265 | INFO | Opening tmp RTMetrics json file | 20260505 18:07:16.273 | INFO | Serializing RTMetrics to json | 20260505 18:07:16.274 | INFO | Serialization done, swapping tmp file now... | 20260505 18:07:16.275 | INFO | RTMetrics done | 20260505 18:07:16.358 | INFO | Finished a round of Baz IO: Wrote 2.103 GiB in 3.271 seconds | 20260505 18:07:16.358 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.95% / 0.00% / 78.04% | 20260505 18:07:16.358 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.264867% / 0% / 8.34465e-05% | 20260505 18:07:16.358 | INFO | Baz IO overall: | 20260505 18:07:16.358 | INFO | BytesWritten : 3086368907400 | 20260505 18:07:16.358 | INFO | NumEvents : 1462572763013 | 20260505 18:07:18.082 | INFO | Analyzing chunk frames = [5234688,5235200) | 20260505 18:07:18.082 | INFO | MemInfo: { | 20260505 18:07:18.082 | INFO | "gpu" : " 17 GB", | 20260505 18:07:18.082 | INFO | "huge" : "", | 20260505 18:07:18.082 | INFO | "huge_shared" : " 27 GB", | 20260505 18:07:18.082 | INFO | "malloc" : " 14 GB", | 20260505 18:07:18.082 | INFO | "pinned" : " 29 GB" | 20260505 18:07:18.082 | INFO | } | 20260505 18:07:18.082 | INFO | Perf report for the last 5.13782 seconds: | 20260505 18:07:18.082 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:07:18.082 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:18.082 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:18.082 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:18.082 | INFO | ANALYSIS : 63.6%, 3.0, 99.0ms, 0 | 20260505 18:07:18.082 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:18.082 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260505 18:07:18.082 | INFO | PRE_HQ : 60.2%, 1.8, 56.0ms, 0 | 20260505 18:07:18.082 | INFO | BAZWRITER : 63.0%, 3.8, 124.3ms, 0 | 20260505 18:07:18.082 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 18:07:18.082 | INFO | Uploaded 36GiB to GPU in 2.16192s (16.6519GiB/s) | 20260505 18:07:18.082 | INFO | Downloaded 38GiB to GPU in 2.25244s (16.8706GiB/s) | 20260505 18:07:23.228 | INFO | Analyzing chunk frames = [5235200,5235712) | 20260505 18:07:23.228 | INFO | MemInfo: { | 20260505 18:07:23.228 | INFO | "gpu" : " 17 GB", | 20260505 18:07:23.229 | INFO | "huge" : "", | 20260505 18:07:23.229 | INFO | "huge_shared" : " 27 GB", | 20260505 18:07:23.229 | INFO | "malloc" : " 14 GB", | 20260505 18:07:23.229 | INFO | "pinned" : " 29 GB" | 20260505 18:07:23.229 | INFO | } | 20260505 18:07:23.229 | INFO | Perf report for the last 5.14706 seconds: | 20260505 18:07:23.229 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:07:23.229 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:23.229 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:23.229 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:23.229 | INFO | ANALYSIS : 62.7%, 3.0, 98.0ms, 0 | 20260505 18:07:23.229 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:23.229 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:23.229 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:07:23.229 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:23.229 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:23.229 | INFO | Uploaded 36GiB to GPU in 2.13208s (16.8849GiB/s) | 20260505 18:07:23.229 | INFO | Downloaded 34GiB to GPU in 2.02904s (16.7567GiB/s) | 20260505 18:07:28.369 | INFO | Analyzing chunk frames = [5235712,5236224) | 20260505 18:07:28.369 | INFO | MemInfo: { | 20260505 18:07:28.369 | INFO | "gpu" : " 17 GB", | 20260505 18:07:28.369 | INFO | "huge" : "", | 20260505 18:07:28.369 | INFO | "huge_shared" : " 27 GB", | 20260505 18:07:28.369 | INFO | "malloc" : " 14 GB", | 20260505 18:07:28.369 | INFO | "pinned" : " 29 GB" | 20260505 18:07:28.369 | INFO | } | 20260505 18:07:28.369 | INFO | Perf report for the last 5.14056 seconds: | 20260505 18:07:28.369 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:07:28.369 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:28.369 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:28.369 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:28.369 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 18:07:28.369 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:28.369 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:28.369 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:07:28.369 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:28.369 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:28.370 | INFO | Uploaded 36GiB to GPU in 2.20712s (16.3108GiB/s) | 20260505 18:07:28.370 | INFO | Downloaded 34GiB to GPU in 2.06319s (16.4794GiB/s) | 20260505 18:07:33.506 | INFO | Analyzing chunk frames = [5236224,5236736) | 20260505 18:07:33.506 | INFO | MemInfo: { | 20260505 18:07:33.506 | INFO | "gpu" : " 17 GB", | 20260505 18:07:33.506 | INFO | "huge" : "", | 20260505 18:07:33.506 | INFO | "huge_shared" : " 27 GB", | 20260505 18:07:33.506 | INFO | "malloc" : " 14 GB", | 20260505 18:07:33.506 | INFO | "pinned" : " 29 GB" | 20260505 18:07:33.506 | INFO | } | 20260505 18:07:33.506 | INFO | Perf report for the last 5.13721 seconds: | 20260505 18:07:33.506 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:07:33.506 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:33.506 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:33.506 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:33.506 | INFO | ANALYSIS : 62.4%, 3.0, 97.0ms, 0 | 20260505 18:07:33.506 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:33.506 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:33.507 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:07:33.507 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:33.507 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:33.507 | INFO | Uploaded 36GiB to GPU in 2.17456s (16.5551GiB/s) | 20260505 18:07:33.507 | INFO | Downloaded 34GiB to GPU in 2.0645s (16.4689GiB/s) | 20260505 18:07:38.637 | INFO | Analyzing chunk frames = [5236736,5237248) | 20260505 18:07:38.637 | INFO | MemInfo: { | 20260505 18:07:38.637 | INFO | "gpu" : " 17 GB", | 20260505 18:07:38.637 | INFO | "huge" : "", | 20260505 18:07:38.637 | INFO | "huge_shared" : " 27 GB", | 20260505 18:07:38.637 | INFO | "malloc" : " 14 GB", | 20260505 18:07:38.637 | INFO | "pinned" : " 29 GB" | 20260505 18:07:38.637 | INFO | } | 20260505 18:07:38.637 | INFO | Perf report for the last 5.13083 seconds: | 20260505 18:07:38.637 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:07:38.637 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:38.637 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:38.637 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:38.637 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260505 18:07:38.637 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:38.637 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:38.637 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:07:38.637 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:38.637 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:38.637 | INFO | Uploaded 36GiB to GPU in 2.13877s (16.8321GiB/s) | 20260505 18:07:38.638 | INFO | Downloaded 34GiB to GPU in 2.0607s (16.4992GiB/s) | 20260505 18:07:43.779 | INFO | Analyzing chunk frames = [5237248,5237760) | 20260505 18:07:43.779 | INFO | MemInfo: { | 20260505 18:07:43.779 | INFO | "gpu" : " 17 GB", | 20260505 18:07:43.779 | INFO | "huge" : "", | 20260505 18:07:43.779 | INFO | "huge_shared" : " 27 GB", | 20260505 18:07:43.779 | INFO | "malloc" : " 14 GB", | 20260505 18:07:43.779 | INFO | "pinned" : " 29 GB" | 20260505 18:07:43.779 | INFO | } | 20260505 18:07:43.779 | INFO | Perf report for the last 5.14168 seconds: | 20260505 18:07:43.779 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:07:43.779 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:43.779 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:43.779 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:43.779 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260505 18:07:43.779 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:43.779 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:43.779 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 18:07:43.779 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:43.779 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:43.779 | INFO | Uploaded 36GiB to GPU in 2.13222s (16.8838GiB/s) | 20260505 18:07:43.779 | INFO | Downloaded 34GiB to GPU in 2.04077s (16.6604GiB/s) | 20260505 18:07:48.923 | INFO | Analyzing chunk frames = [5237760,5238272) | 20260505 18:07:48.923 | INFO | MemInfo: { | 20260505 18:07:48.923 | INFO | "gpu" : " 17 GB", | 20260505 18:07:48.923 | INFO | "huge" : "", | 20260505 18:07:48.923 | INFO | "huge_shared" : " 27 GB", | 20260505 18:07:48.924 | INFO | "malloc" : " 14 GB", | 20260505 18:07:48.924 | INFO | "pinned" : " 29 GB" | 20260505 18:07:48.924 | INFO | } | 20260505 18:07:48.924 | INFO | Perf report for the last 5.14459 seconds: | 20260505 18:07:48.924 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:07:48.924 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:48.924 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:48.924 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:48.924 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 18:07:48.924 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:48.924 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:48.924 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 18:07:48.924 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:48.924 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:48.924 | INFO | Uploaded 36GiB to GPU in 2.17908s (16.5207GiB/s) | 20260505 18:07:48.924 | INFO | Downloaded 34GiB to GPU in 2.18036s (15.5937GiB/s) | 20260505 18:07:54.067 | INFO | Analyzing chunk frames = [5238272,5238784) | 20260505 18:07:54.067 | INFO | MemInfo: { | 20260505 18:07:54.067 | INFO | "gpu" : " 17 GB", | 20260505 18:07:54.067 | INFO | "huge" : "", | 20260505 18:07:54.067 | INFO | "huge_shared" : " 27 GB", | 20260505 18:07:54.068 | INFO | "malloc" : " 14 GB", | 20260505 18:07:54.068 | INFO | "pinned" : " 29 GB" | 20260505 18:07:54.068 | INFO | } | 20260505 18:07:54.068 | INFO | Perf report for the last 5.14402 seconds: | 20260505 18:07:54.068 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:07:54.068 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:54.068 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:54.068 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:54.068 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 18:07:54.068 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:54.068 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:54.068 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:07:54.068 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:54.068 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:54.068 | INFO | Uploaded 36GiB to GPU in 2.18977s (16.4401GiB/s) | 20260505 18:07:54.068 | INFO | Downloaded 34GiB to GPU in 2.03171s (16.7347GiB/s) | 20260505 18:07:54.199 | INFO | Starting a round of Baz IO | 20260505 18:07:57.378 | INFO | Populating RTMetrics report | 20260505 18:07:57.378 | INFO | RTMetrics report populated | 20260505 18:07:57.378 | INFO | Opening tmp RTMetrics json file | 20260505 18:07:57.379 | INFO | Serializing RTMetrics to json | 20260505 18:07:57.381 | INFO | Serialization done, swapping tmp file now... | 20260505 18:07:57.381 | INFO | RTMetrics done | 20260505 18:07:57.467 | INFO | Finished a round of Baz IO: Wrote 2.103 GiB in 3.267 seconds | 20260505 18:07:57.467 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.95% / 0.00% / 78.05% | 20260505 18:07:57.467 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.264275% / 1.19209e-05% / 9.93411e-05% | 20260505 18:07:57.467 | INFO | Baz IO overall: | 20260505 18:07:57.467 | INFO | BytesWritten : 3088626698441 | 20260505 18:07:57.467 | INFO | NumEvents : 1463625227598 | 20260505 18:07:59.205 | INFO | Analyzing chunk frames = [5238784,5239296) | 20260505 18:07:59.205 | INFO | MemInfo: { | 20260505 18:07:59.205 | INFO | "gpu" : " 17 GB", | 20260505 18:07:59.206 | INFO | "huge" : "", | 20260505 18:07:59.206 | INFO | "huge_shared" : " 27 GB", | 20260505 18:07:59.206 | INFO | "malloc" : " 14 GB", | 20260505 18:07:59.206 | INFO | "pinned" : " 29 GB" | 20260505 18:07:59.206 | INFO | } | 20260505 18:07:59.206 | INFO | Perf report for the last 5.138 seconds: | 20260505 18:07:59.206 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:07:59.206 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:59.206 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:59.206 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:59.206 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 18:07:59.206 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:07:59.206 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260505 18:07:59.206 | INFO | PRE_HQ : 60.5%, 1.8, 55.8ms, 0 | 20260505 18:07:59.206 | INFO | BAZWRITER : 63.1%, 3.8, 122.7ms, 0 | 20260505 18:07:59.206 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 18:07:59.206 | INFO | Uploaded 36GiB to GPU in 2.1684s (16.6021GiB/s) | 20260505 18:07:59.206 | INFO | Downloaded 38GiB to GPU in 2.25217s (16.8726GiB/s) | 20260505 18:08:04.336 | INFO | Analyzing chunk frames = [5239296,5239808) | 20260505 18:08:04.336 | INFO | MemInfo: { | 20260505 18:08:04.336 | INFO | "gpu" : " 17 GB", | 20260505 18:08:04.336 | INFO | "huge" : "", | 20260505 18:08:04.336 | INFO | "huge_shared" : " 27 GB", | 20260505 18:08:04.336 | INFO | "malloc" : " 14 GB", | 20260505 18:08:04.336 | INFO | "pinned" : " 29 GB" | 20260505 18:08:04.336 | INFO | } | 20260505 18:08:04.336 | INFO | Perf report for the last 5.13065 seconds: | 20260505 18:08:04.336 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:08:04.336 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:04.336 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:04.336 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:04.336 | INFO | ANALYSIS : 62.9%, 3.0, 98.0ms, 0 | 20260505 18:08:04.336 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:04.336 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:04.336 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:08:04.336 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:04.336 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:04.337 | INFO | Uploaded 36GiB to GPU in 2.1364s (16.8508GiB/s) | 20260505 18:08:04.337 | INFO | Downloaded 34GiB to GPU in 2.02439s (16.7952GiB/s) | 20260505 18:08:09.482 | INFO | Analyzing chunk frames = [5239808,5240320) | 20260505 18:08:09.482 | INFO | MemInfo: { | 20260505 18:08:09.482 | INFO | "gpu" : " 17 GB", | 20260505 18:08:09.482 | INFO | "huge" : "", | 20260505 18:08:09.482 | INFO | "huge_shared" : " 27 GB", | 20260505 18:08:09.482 | INFO | "malloc" : " 14 GB", | 20260505 18:08:09.482 | INFO | "pinned" : " 29 GB" | 20260505 18:08:09.482 | INFO | } | 20260505 18:08:09.482 | INFO | Perf report for the last 5.14559 seconds: | 20260505 18:08:09.482 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:08:09.482 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:09.482 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:09.482 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:09.482 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260505 18:08:09.482 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:09.482 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:09.482 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:08:09.482 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:09.482 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:09.482 | INFO | Uploaded 36GiB to GPU in 2.19954s (16.367GiB/s) | 20260505 18:08:09.482 | INFO | Downloaded 34GiB to GPU in 2.04126s (16.6564GiB/s) | 20260505 18:08:14.620 | INFO | Analyzing chunk frames = [5240320,5240832) | 20260505 18:08:14.620 | INFO | MemInfo: { | 20260505 18:08:14.620 | INFO | "gpu" : " 17 GB", | 20260505 18:08:14.621 | INFO | "huge" : "", | 20260505 18:08:14.621 | INFO | "huge_shared" : " 27 GB", | 20260505 18:08:14.621 | INFO | "malloc" : " 14 GB", | 20260505 18:08:14.621 | INFO | "pinned" : " 29 GB" | 20260505 18:08:14.621 | INFO | } | 20260505 18:08:14.621 | INFO | Perf report for the last 5.13879 seconds: | 20260505 18:08:14.621 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:08:14.621 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:14.621 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:14.621 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:14.621 | INFO | ANALYSIS : 62.8%, 3.0, 97.6ms, 0 | 20260505 18:08:14.621 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:14.621 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:14.621 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:08:14.621 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:14.621 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:14.621 | INFO | Uploaded 36GiB to GPU in 2.22771s (16.1601GiB/s) | 20260505 18:08:14.621 | INFO | Downloaded 34GiB to GPU in 2.117s (16.0605GiB/s) | 20260505 18:08:19.760 | INFO | Analyzing chunk frames = [5240832,5241344) | 20260505 18:08:19.760 | INFO | MemInfo: { | 20260505 18:08:19.760 | INFO | "gpu" : " 17 GB", | 20260505 18:08:19.760 | INFO | "huge" : "", | 20260505 18:08:19.760 | INFO | "huge_shared" : " 27 GB", | 20260505 18:08:19.760 | INFO | "malloc" : " 14 GB", | 20260505 18:08:19.760 | INFO | "pinned" : " 29 GB" | 20260505 18:08:19.760 | INFO | } | 20260505 18:08:19.761 | INFO | Perf report for the last 5.13982 seconds: | 20260505 18:08:19.761 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:08:19.761 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:19.761 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:19.761 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:19.761 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 18:08:19.761 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:19.761 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:19.761 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:08:19.761 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:19.761 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:19.761 | INFO | Uploaded 36GiB to GPU in 2.13338s (16.8746GiB/s) | 20260505 18:08:19.761 | INFO | Downloaded 34GiB to GPU in 2.06114s (16.4957GiB/s) | 20260505 18:08:24.896 | INFO | Analyzing chunk frames = [5241344,5241856) | 20260505 18:08:24.896 | INFO | MemInfo: { | 20260505 18:08:24.896 | INFO | "gpu" : " 17 GB", | 20260505 18:08:24.896 | INFO | "huge" : "", | 20260505 18:08:24.896 | INFO | "huge_shared" : " 27 GB", | 20260505 18:08:24.896 | INFO | "malloc" : " 14 GB", | 20260505 18:08:24.896 | INFO | "pinned" : " 29 GB" | 20260505 18:08:24.896 | INFO | } | 20260505 18:08:24.896 | INFO | Perf report for the last 5.13586 seconds: | 20260505 18:08:24.897 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:08:24.897 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:24.897 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:24.897 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:24.897 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260505 18:08:24.897 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:24.897 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:24.897 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 18:08:24.897 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:24.897 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:24.897 | INFO | Uploaded 36GiB to GPU in 2.13843s (16.8348GiB/s) | 20260505 18:08:24.897 | INFO | Downloaded 34GiB to GPU in 2.04366s (16.6368GiB/s) | 20260505 18:08:30.032 | INFO | Analyzing chunk frames = [5241856,5242368) | 20260505 18:08:30.032 | INFO | MemInfo: { | 20260505 18:08:30.032 | INFO | "gpu" : " 17 GB", | 20260505 18:08:30.032 | INFO | "huge" : "", | 20260505 18:08:30.032 | INFO | "huge_shared" : " 27 GB", | 20260505 18:08:30.032 | INFO | "malloc" : " 14 GB", | 20260505 18:08:30.032 | INFO | "pinned" : " 29 GB" | 20260505 18:08:30.032 | INFO | } | 20260505 18:08:30.032 | INFO | Perf report for the last 5.13517 seconds: | 20260505 18:08:30.032 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:08:30.032 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:30.032 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:30.032 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:30.032 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 18:08:30.032 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:30.032 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:30.032 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 18:08:30.032 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:30.032 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:30.032 | INFO | Uploaded 36GiB to GPU in 2.16527s (16.6261GiB/s) | 20260505 18:08:30.032 | INFO | Downloaded 34GiB to GPU in 2.17425s (15.6376GiB/s) | 20260505 18:08:35.178 | INFO | Analyzing chunk frames = [5242368,5242880) | 20260505 18:08:35.178 | INFO | MemInfo: { | 20260505 18:08:35.178 | INFO | "gpu" : " 17 GB", | 20260505 18:08:35.178 | INFO | "huge" : "", | 20260505 18:08:35.178 | INFO | "huge_shared" : " 27 GB", | 20260505 18:08:35.178 | INFO | "malloc" : " 14 GB", | 20260505 18:08:35.178 | INFO | "pinned" : " 29 GB" | 20260505 18:08:35.178 | INFO | } | 20260505 18:08:35.178 | INFO | Perf report for the last 5.14637 seconds: | 20260505 18:08:35.178 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:08:35.178 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:35.178 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:35.178 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:35.178 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 18:08:35.178 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:35.178 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:35.178 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:08:35.178 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:35.178 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:35.178 | INFO | Uploaded 36GiB to GPU in 2.18415s (16.4824GiB/s) | 20260505 18:08:35.178 | INFO | Downloaded 34GiB to GPU in 2.02365s (16.8013GiB/s) | 20260505 18:08:35.178 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 18:08:35.178 | INFO | Total mem usage: 15.3 GiB | 20260505 18:08:35.178 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 18:08:35.178 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.01%) | 20260505 18:08:35.178 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 18:08:35.178 | INFO | High Water Breakdown: | 20260505 18:08:35.178 | INFO | 61% : 8.66 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 18:08:35.178 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 18:08:35.178 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 18:08:35.178 | INFO | ------------------------------------------ | 20260505 18:08:35.178 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:08:35.178 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 18:08:35.178 | INFO | Total mem usage: 26.5 GiB | 20260505 18:08:35.178 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 18:08:35.179 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 18:08:35.179 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 18:08:35.179 | INFO | High Water Breakdown: | 20260505 18:08:35.179 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:08:35.179 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:08:35.179 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:08:35.179 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:08:35.179 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:08:35.179 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:08:35.179 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 18:08:35.179 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 18:08:35.179 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:08:35.179 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 18:08:35.179 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 18:08:35.179 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 18:08:35.179 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 18:08:35.179 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 18:08:35.179 | INFO | ------------------------------------------ | 20260505 18:08:35.179 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:08:35.179 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 18:08:35.179 | INFO | Total mem usage: 25.6 GiB | 20260505 18:08:35.179 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 18:08:35.179 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 18:08:35.179 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 18:08:35.179 | INFO | High Water Breakdown: | 20260505 18:08:35.179 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 18:08:35.179 | INFO | ------------------------------------------ | 20260505 18:08:35.179 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:08:35.179 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 18:08:35.179 | INFO | Total mem usage: 15.7 GiB | 20260505 18:08:35.179 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 18:08:35.179 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 18:08:35.179 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 18:08:35.179 | INFO | High Water Breakdown: | 20260505 18:08:35.179 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 18:08:35.179 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 18:08:35.179 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 18:08:35.179 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 18:08:35.179 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 18:08:35.179 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 18:08:35.179 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 18:08:35.179 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 18:08:35.179 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 18:08:35.179 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:08:35.179 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 18:08:35.179 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:08:35.179 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 18:08:35.179 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 18:08:35.179 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 18:08:35.180 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 18:08:35.180 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:08:35.180 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 18:08:35.180 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:08:35.180 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:08:35.180 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:08:35.180 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 18:08:35.180 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 18:08:35.180 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:08:35.180 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 18:08:35.180 | INFO | ------------------------------------------ | 20260505 18:08:35.180 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:08:35.180 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 18:08:35.180 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:08:35.180 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:08:35.180 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.46e+01, 1.34e+01 / 2.45e+00 | 20260505 18:08:35.180 | INFO | PerfReport: 0.29%, AggregateStats -- 9.25e-02 / 1.02e-01, 1.05e-02 / 4.28e-03 | 20260505 18:08:35.180 | INFO | PerfReport: 11.76%, Baseline -- 3.71e+00 / 4.50e+00, 5.58e-01 / 1.97e-01 | 20260505 18:08:35.180 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.53e+00, 1.90e-01 / 9.31e-02 | 20260505 18:08:35.180 | INFO | PerfReport: 15.94%, DME -- 4.02e+01 / 4.45e+01, 4.05e+00 / 2.52e+00 | 20260505 18:08:35.180 | INFO | PerfReport: 56.89%, FrameLabeling -- 1.79e+01 / 1.93e+01, 1.44e+00 / 1.04e+00 | 20260505 18:08:35.180 | INFO | PerfReport: 7.34%, PulseAccumulating -- 2.31e+00 / 2.91e+00, 4.46e-01 / 1.67e-01 | 20260505 18:08:35.180 | INFO | PerfReport: 3.34%, Metrics -- 1.05e+00 / 1.38e+00, 2.89e-01 / 1.49e-01 | 20260505 18:08:35.180 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.67e-01 / 4.33e-01, 6.44e-02 / 4.08e-02 | 20260505 18:08:35.180 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 18:08:35.180 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:08:35.180 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:08:35.180 | INFO | PerfReport: Full Execution -- 9.99e+01 / -nan, 2.56e+01 / 0.00e+00 | 20260505 18:08:35.180 | INFO | PerfReport: 22.59%, Upload -- 2.26e+01 / 0.00e+00, 7.83e+00 / 0.00e+00 | 20260505 18:08:35.180 | INFO | PerfReport: 21.81%, Download -- 2.18e+01 / 0.00e+00, 7.78e+00 / 0.00e+00 | 20260505 18:08:35.180 | INFO | PerfReport: 31.57%, Compute -- 3.15e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 18:08:35.180 | INFO | PerfReport: 2.34%, LockUpload -- 2.34e+00 / 0.00e+00, 7.97e+00 / 0.00e+00 | 20260505 18:08:35.180 | INFO | PerfReport: 1.40%, LockDownload -- 1.39e+00 / 0.00e+00, 2.89e+00 / 0.00e+00 | 20260505 18:08:35.180 | INFO | PerfReport: 20.28%, LockCompute -- 2.03e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 18:08:35.180 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 2.25e-02 / 0.00e+00 | 20260505 18:08:35.313 | INFO | Starting a round of Baz IO | 20260505 18:08:38.482 | INFO | Populating RTMetrics report | 20260505 18:08:38.483 | INFO | RTMetrics report populated | 20260505 18:08:38.483 | INFO | Opening tmp RTMetrics json file | 20260505 18:08:38.484 | INFO | Serializing RTMetrics to json | 20260505 18:08:38.485 | INFO | Serialization done, swapping tmp file now... | 20260505 18:08:38.486 | INFO | RTMetrics done | 20260505 18:08:38.577 | INFO | Finished a round of Baz IO: Wrote 2.103 GiB in 3.263 seconds | 20260505 18:08:38.577 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.95% / 0.00% / 78.05% | 20260505 18:08:38.577 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.264196% / 7.94729e-06% / 8.34465e-05% | 20260505 18:08:38.577 | INFO | Baz IO overall: | 20260505 18:08:38.577 | INFO | BytesWritten : 3090884704607 | 20260505 18:08:38.577 | INFO | NumEvents : 1464678174490 | 20260505 18:08:40.317 | INFO | Analyzing chunk frames = [5242880,5243392) | 20260505 18:08:40.317 | INFO | MemInfo: { | 20260505 18:08:40.317 | INFO | "gpu" : " 17 GB", | 20260505 18:08:40.317 | INFO | "huge" : "", | 20260505 18:08:40.317 | INFO | "huge_shared" : " 27 GB", | 20260505 18:08:40.317 | INFO | "malloc" : " 14 GB", | 20260505 18:08:40.317 | INFO | "pinned" : " 29 GB" | 20260505 18:08:40.317 | INFO | } | 20260505 18:08:40.317 | INFO | Perf report for the last 5.13941 seconds: | 20260505 18:08:40.317 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:08:40.317 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:40.317 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:40.317 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:40.317 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 18:08:40.317 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:40.317 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260505 18:08:40.317 | INFO | PRE_HQ : 60.3%, 1.8, 55.9ms, 0 | 20260505 18:08:40.317 | INFO | BAZWRITER : 63.1%, 3.8, 123.1ms, 0 | 20260505 18:08:40.317 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 18:08:40.318 | INFO | Uploaded 36GiB to GPU in 2.16266s (16.6462GiB/s) | 20260505 18:08:40.318 | INFO | Downloaded 38GiB to GPU in 2.24842s (16.9007GiB/s) | 20260505 18:08:45.454 | INFO | Analyzing chunk frames = [5243392,5243904) | 20260505 18:08:45.454 | INFO | MemInfo: { | 20260505 18:08:45.454 | INFO | "gpu" : " 17 GB", | 20260505 18:08:45.454 | INFO | "huge" : "", | 20260505 18:08:45.454 | INFO | "huge_shared" : " 27 GB", | 20260505 18:08:45.454 | INFO | "malloc" : " 14 GB", | 20260505 18:08:45.454 | INFO | "pinned" : " 29 GB" | 20260505 18:08:45.454 | INFO | } | 20260505 18:08:45.454 | INFO | Perf report for the last 5.13655 seconds: | 20260505 18:08:45.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:08:45.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:45.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:45.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:45.454 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260505 18:08:45.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:45.454 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:45.454 | INFO | PRE_HQ : 27.5%, 1.0, 14.2ms, 0 | 20260505 18:08:45.454 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:45.454 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:45.454 | INFO | Uploaded 36GiB to GPU in 2.13072s (16.8957GiB/s) | 20260505 18:08:45.454 | INFO | Downloaded 34GiB to GPU in 2.03011s (16.7479GiB/s) | 20260505 18:08:50.588 | INFO | Analyzing chunk frames = [5243904,5244416) | 20260505 18:08:50.588 | INFO | MemInfo: { | 20260505 18:08:50.588 | INFO | "gpu" : " 17 GB", | 20260505 18:08:50.588 | INFO | "huge" : "", | 20260505 18:08:50.588 | INFO | "huge_shared" : " 27 GB", | 20260505 18:08:50.588 | INFO | "malloc" : " 14 GB", | 20260505 18:08:50.588 | INFO | "pinned" : " 29 GB" | 20260505 18:08:50.589 | INFO | } | 20260505 18:08:50.589 | INFO | Perf report for the last 5.13466 seconds: | 20260505 18:08:50.589 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:08:50.589 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:50.589 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:50.589 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:50.589 | INFO | ANALYSIS : 62.8%, 3.0, 97.6ms, 0 | 20260505 18:08:50.589 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:50.589 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:50.589 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:08:50.589 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:50.589 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:50.589 | INFO | Uploaded 36GiB to GPU in 2.20365s (16.3366GiB/s) | 20260505 18:08:50.589 | INFO | Downloaded 34GiB to GPU in 2.04263s (16.6452GiB/s) | 20260505 18:08:55.737 | INFO | Analyzing chunk frames = [5244416,5244928) | 20260505 18:08:55.737 | INFO | MemInfo: { | 20260505 18:08:55.737 | INFO | "gpu" : " 17 GB", | 20260505 18:08:55.737 | INFO | "huge" : "", | 20260505 18:08:55.737 | INFO | "huge_shared" : " 27 GB", | 20260505 18:08:55.737 | INFO | "malloc" : " 14 GB", | 20260505 18:08:55.737 | INFO | "pinned" : " 29 GB" | 20260505 18:08:55.737 | INFO | } | 20260505 18:08:55.737 | INFO | Perf report for the last 5.14826 seconds: | 20260505 18:08:55.737 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:08:55.737 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:55.737 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:55.737 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:55.737 | INFO | ANALYSIS : 62.5%, 3.0, 97.2ms, 0 | 20260505 18:08:55.737 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:55.737 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:55.737 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260505 18:08:55.737 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:55.737 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:08:55.737 | INFO | Uploaded 36GiB to GPU in 2.1739s (16.5601GiB/s) | 20260505 18:08:55.737 | INFO | Downloaded 34GiB to GPU in 2.1419s (15.8737GiB/s) | 20260505 18:09:00.881 | INFO | Analyzing chunk frames = [5244928,5245440) | 20260505 18:09:00.881 | INFO | MemInfo: { | 20260505 18:09:00.881 | INFO | "gpu" : " 17 GB", | 20260505 18:09:00.881 | INFO | "huge" : "", | 20260505 18:09:00.881 | INFO | "huge_shared" : " 27 GB", | 20260505 18:09:00.881 | INFO | "malloc" : " 14 GB", | 20260505 18:09:00.881 | INFO | "pinned" : " 29 GB" | 20260505 18:09:00.881 | INFO | } | 20260505 18:09:00.881 | INFO | Perf report for the last 5.14417 seconds: | 20260505 18:09:00.881 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:09:00.881 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:00.881 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:00.881 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:00.881 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260505 18:09:00.881 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:00.881 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:00.881 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 18:09:00.881 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:00.881 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:00.881 | INFO | Uploaded 36GiB to GPU in 2.12832s (16.9148GiB/s) | 20260505 18:09:00.881 | INFO | Downloaded 34GiB to GPU in 2.05585s (16.5382GiB/s) | 20260505 18:09:06.018 | INFO | Analyzing chunk frames = [5245440,5245952) | 20260505 18:09:06.018 | INFO | MemInfo: { | 20260505 18:09:06.018 | INFO | "gpu" : " 17 GB", | 20260505 18:09:06.018 | INFO | "huge" : "", | 20260505 18:09:06.018 | INFO | "huge_shared" : " 27 GB", | 20260505 18:09:06.018 | INFO | "malloc" : " 14 GB", | 20260505 18:09:06.018 | INFO | "pinned" : " 29 GB" | 20260505 18:09:06.018 | INFO | } | 20260505 18:09:06.018 | INFO | Perf report for the last 5.13671 seconds: | 20260505 18:09:06.018 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:09:06.018 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:06.018 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:06.018 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:06.018 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 18:09:06.018 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:06.018 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:06.018 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:09:06.018 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:06.018 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:06.018 | INFO | Uploaded 36GiB to GPU in 2.13709s (16.8453GiB/s) | 20260505 18:09:06.018 | INFO | Downloaded 34GiB to GPU in 2.05453s (16.5488GiB/s) | 20260505 18:09:11.152 | INFO | Analyzing chunk frames = [5245952,5246464) | 20260505 18:09:11.152 | INFO | MemInfo: { | 20260505 18:09:11.152 | INFO | "gpu" : " 17 GB", | 20260505 18:09:11.152 | INFO | "huge" : "", | 20260505 18:09:11.152 | INFO | "huge_shared" : " 27 GB", | 20260505 18:09:11.152 | INFO | "malloc" : " 14 GB", | 20260505 18:09:11.152 | INFO | "pinned" : " 29 GB" | 20260505 18:09:11.152 | INFO | } | 20260505 18:09:11.152 | INFO | Perf report for the last 5.13414 seconds: | 20260505 18:09:11.152 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:09:11.152 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:11.152 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:11.152 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:11.152 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 18:09:11.152 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:11.152 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:11.152 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 18:09:11.152 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:11.152 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:11.152 | INFO | Uploaded 36GiB to GPU in 2.13308s (16.877GiB/s) | 20260505 18:09:11.152 | INFO | Downloaded 34GiB to GPU in 2.1343s (15.9303GiB/s) | 20260505 18:09:16.283 | INFO | Analyzing chunk frames = [5246464,5246976) | 20260505 18:09:16.283 | INFO | MemInfo: { | 20260505 18:09:16.283 | INFO | "gpu" : " 17 GB", | 20260505 18:09:16.283 | INFO | "huge" : "", | 20260505 18:09:16.283 | INFO | "huge_shared" : " 27 GB", | 20260505 18:09:16.283 | INFO | "malloc" : " 14 GB", | 20260505 18:09:16.283 | INFO | "pinned" : " 29 GB" | 20260505 18:09:16.283 | INFO | } | 20260505 18:09:16.283 | INFO | Perf report for the last 5.13141 seconds: | 20260505 18:09:16.283 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:09:16.283 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:16.283 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:16.283 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:16.283 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 18:09:16.283 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:16.283 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:16.283 | INFO | PRE_HQ : 27.7%, 1.0, 14.3ms, 0 | 20260505 18:09:16.283 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:16.283 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:16.284 | INFO | Uploaded 36GiB to GPU in 2.19855s (16.3744GiB/s) | 20260505 18:09:16.284 | INFO | Downloaded 34GiB to GPU in 2.03196s (16.7326GiB/s) | 20260505 18:09:16.423 | INFO | Starting a round of Baz IO | 20260505 18:09:19.609 | INFO | Populating RTMetrics report | 20260505 18:09:19.610 | INFO | RTMetrics report populated | 20260505 18:09:19.610 | INFO | Opening tmp RTMetrics json file | 20260505 18:09:19.612 | INFO | Serializing RTMetrics to json | 20260505 18:09:19.615 | INFO | Serialization done, swapping tmp file now... | 20260505 18:09:19.617 | INFO | RTMetrics done | 20260505 18:09:19.705 | INFO | Finished a round of Baz IO: Wrote 2.102 GiB in 3.281 seconds | 20260505 18:09:19.705 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.95% / 0.00% / 78.05% | 20260505 18:09:19.705 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.263198% / 0% / 8.34465e-05% | 20260505 18:09:19.705 | INFO | Baz IO overall: | 20260505 18:09:19.705 | INFO | BytesWritten : 3093141704460 | 20260505 18:09:19.705 | INFO | NumEvents : 1465730145996 | 20260505 18:09:21.436 | INFO | Analyzing chunk frames = [5246976,5247488) | 20260505 18:09:21.436 | INFO | MemInfo: { | 20260505 18:09:21.436 | INFO | "gpu" : " 17 GB", | 20260505 18:09:21.436 | INFO | "huge" : "", | 20260505 18:09:21.436 | INFO | "huge_shared" : " 27 GB", | 20260505 18:09:21.436 | INFO | "malloc" : " 14 GB", | 20260505 18:09:21.436 | INFO | "pinned" : " 29 GB" | 20260505 18:09:21.436 | INFO | } | 20260505 18:09:21.437 | INFO | Perf report for the last 5.15314 seconds: | 20260505 18:09:21.437 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:09:21.437 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:21.437 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:21.437 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:21.437 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 18:09:21.437 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:21.437 | INFO | RT_METRICS : 44.6%, 1.0, 23.5ms, 0 | 20260505 18:09:21.437 | INFO | PRE_HQ : 60.1%, 1.8, 55.6ms, 0 | 20260505 18:09:21.437 | INFO | BAZWRITER : 63.1%, 3.8, 123.4ms, 0 | 20260505 18:09:21.437 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 18:09:21.437 | INFO | Uploaded 36GiB to GPU in 2.19249s (16.4197GiB/s) | 20260505 18:09:21.437 | INFO | Downloaded 38GiB to GPU in 2.26837s (16.7521GiB/s) | 20260505 18:09:26.580 | INFO | Analyzing chunk frames = [5247488,5248000) | 20260505 18:09:26.580 | INFO | MemInfo: { | 20260505 18:09:26.580 | INFO | "gpu" : " 17 GB", | 20260505 18:09:26.580 | INFO | "huge" : "", | 20260505 18:09:26.580 | INFO | "huge_shared" : " 27 GB", | 20260505 18:09:26.580 | INFO | "malloc" : " 14 GB", | 20260505 18:09:26.580 | INFO | "pinned" : " 29 GB" | 20260505 18:09:26.580 | INFO | } | 20260505 18:09:26.580 | INFO | Perf report for the last 5.14369 seconds: | 20260505 18:09:26.580 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:09:26.580 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:26.580 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:26.580 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:26.580 | INFO | ANALYSIS : 62.7%, 3.0, 98.0ms, 0 | 20260505 18:09:26.580 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:26.580 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:26.580 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:09:26.580 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:26.580 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:26.580 | INFO | Uploaded 36GiB to GPU in 2.13678s (16.8478GiB/s) | 20260505 18:09:26.581 | INFO | Downloaded 34GiB to GPU in 2.02946s (16.7532GiB/s) | 20260505 18:09:31.708 | INFO | Analyzing chunk frames = [5248000,5248512) | 20260505 18:09:31.708 | INFO | MemInfo: { | 20260505 18:09:31.708 | INFO | "gpu" : " 17 GB", | 20260505 18:09:31.708 | INFO | "huge" : "", | 20260505 18:09:31.708 | INFO | "huge_shared" : " 27 GB", | 20260505 18:09:31.708 | INFO | "malloc" : " 14 GB", | 20260505 18:09:31.708 | INFO | "pinned" : " 29 GB" | 20260505 18:09:31.708 | INFO | } | 20260505 18:09:31.708 | INFO | Perf report for the last 5.12818 seconds: | 20260505 18:09:31.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:09:31.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:31.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:31.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:31.709 | INFO | ANALYSIS : 62.8%, 3.0, 97.6ms, 0 | 20260505 18:09:31.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:31.709 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:31.709 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:09:31.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:31.709 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:31.709 | INFO | Uploaded 36GiB to GPU in 2.20628s (16.3171GiB/s) | 20260505 18:09:31.709 | INFO | Downloaded 34GiB to GPU in 2.07013s (16.4241GiB/s) | 20260505 18:09:36.844 | INFO | Analyzing chunk frames = [5248512,5249024) | 20260505 18:09:36.844 | INFO | MemInfo: { | 20260505 18:09:36.844 | INFO | "gpu" : " 17 GB", | 20260505 18:09:36.844 | INFO | "huge" : "", | 20260505 18:09:36.844 | INFO | "huge_shared" : " 27 GB", | 20260505 18:09:36.844 | INFO | "malloc" : " 14 GB", | 20260505 18:09:36.844 | INFO | "pinned" : " 29 GB" | 20260505 18:09:36.844 | INFO | } | 20260505 18:09:36.844 | INFO | Perf report for the last 5.13525 seconds: | 20260505 18:09:36.844 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:09:36.844 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:36.844 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:36.844 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:36.844 | INFO | ANALYSIS : 62.5%, 3.0, 97.0ms, 0 | 20260505 18:09:36.844 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:36.844 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:36.844 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:09:36.844 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:36.844 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:36.844 | INFO | Uploaded 36GiB to GPU in 2.18753s (16.4569GiB/s) | 20260505 18:09:36.844 | INFO | Downloaded 34GiB to GPU in 2.14196s (15.8733GiB/s) | 20260505 18:09:41.988 | INFO | Analyzing chunk frames = [5249024,5249536) | 20260505 18:09:41.988 | INFO | MemInfo: { | 20260505 18:09:41.988 | INFO | "gpu" : " 17 GB", | 20260505 18:09:41.988 | INFO | "huge" : "", | 20260505 18:09:41.988 | INFO | "huge_shared" : " 27 GB", | 20260505 18:09:41.988 | INFO | "malloc" : " 14 GB", | 20260505 18:09:41.988 | INFO | "pinned" : " 29 GB" | 20260505 18:09:41.988 | INFO | } | 20260505 18:09:41.988 | INFO | Perf report for the last 5.14435 seconds: | 20260505 18:09:41.988 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:09:41.988 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:41.988 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:41.988 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:41.988 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260505 18:09:41.988 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:41.988 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:41.988 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:09:41.988 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:41.988 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:41.988 | INFO | Uploaded 36GiB to GPU in 2.15593s (16.6981GiB/s) | 20260505 18:09:41.988 | INFO | Downloaded 34GiB to GPU in 2.06094s (16.4974GiB/s) | 20260505 18:09:47.131 | INFO | Analyzing chunk frames = [5249536,5250048) | 20260505 18:09:47.131 | INFO | MemInfo: { | 20260505 18:09:47.131 | INFO | "gpu" : " 17 GB", | 20260505 18:09:47.131 | INFO | "huge" : "", | 20260505 18:09:47.131 | INFO | "huge_shared" : " 27 GB", | 20260505 18:09:47.131 | INFO | "malloc" : " 14 GB", | 20260505 18:09:47.131 | INFO | "pinned" : " 29 GB" | 20260505 18:09:47.131 | INFO | } | 20260505 18:09:47.131 | INFO | Perf report for the last 5.14313 seconds: | 20260505 18:09:47.131 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:09:47.131 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:47.131 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:47.131 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:47.131 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260505 18:09:47.131 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:47.131 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:47.131 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 18:09:47.131 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:47.131 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:47.132 | INFO | Uploaded 36GiB to GPU in 2.1374s (16.8429GiB/s) | 20260505 18:09:47.132 | INFO | Downloaded 34GiB to GPU in 2.0542s (16.5514GiB/s) | 20260505 18:09:52.264 | INFO | Analyzing chunk frames = [5250048,5250560) | 20260505 18:09:52.264 | INFO | MemInfo: { | 20260505 18:09:52.264 | INFO | "gpu" : " 17 GB", | 20260505 18:09:52.264 | INFO | "huge" : "", | 20260505 18:09:52.264 | INFO | "huge_shared" : " 27 GB", | 20260505 18:09:52.265 | INFO | "malloc" : " 14 GB", | 20260505 18:09:52.265 | INFO | "pinned" : " 29 GB" | 20260505 18:09:52.265 | INFO | } | 20260505 18:09:52.265 | INFO | Perf report for the last 5.13343 seconds: | 20260505 18:09:52.265 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:09:52.265 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:52.265 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:52.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:52.265 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 18:09:52.265 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:52.265 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:52.265 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 18:09:52.265 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:52.265 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:52.265 | INFO | Uploaded 36GiB to GPU in 2.13015s (16.9002GiB/s) | 20260505 18:09:52.265 | INFO | Downloaded 34GiB to GPU in 2.14195s (15.8734GiB/s) | 20260505 18:09:57.402 | INFO | Analyzing chunk frames = [5250560,5251072) | 20260505 18:09:57.402 | INFO | MemInfo: { | 20260505 18:09:57.402 | INFO | "gpu" : " 17 GB", | 20260505 18:09:57.402 | INFO | "huge" : "", | 20260505 18:09:57.402 | INFO | "huge_shared" : " 27 GB", | 20260505 18:09:57.402 | INFO | "malloc" : " 14 GB", | 20260505 18:09:57.402 | INFO | "pinned" : " 29 GB" | 20260505 18:09:57.402 | INFO | } | 20260505 18:09:57.402 | INFO | Perf report for the last 5.13774 seconds: | 20260505 18:09:57.402 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:09:57.402 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:57.402 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:57.402 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:57.402 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 18:09:57.402 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:57.402 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:57.402 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 18:09:57.402 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:57.402 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:09:57.403 | INFO | Uploaded 36GiB to GPU in 2.19233s (16.4209GiB/s) | 20260505 18:09:57.403 | INFO | Downloaded 34GiB to GPU in 2.02198s (16.8152GiB/s) | 20260505 18:09:57.403 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 18:09:57.403 | INFO | Total mem usage: 15.3 GiB | 20260505 18:09:57.403 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 18:09:57.403 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.01%) | 20260505 18:09:57.403 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 18:09:57.403 | INFO | High Water Breakdown: | 20260505 18:09:57.403 | INFO | 61% : 8.66 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 18:09:57.403 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 18:09:57.403 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 18:09:57.403 | INFO | ------------------------------------------ | 20260505 18:09:57.403 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:09:57.403 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 18:09:57.403 | INFO | Total mem usage: 26.5 GiB | 20260505 18:09:57.403 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 18:09:57.403 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 18:09:57.403 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 18:09:57.403 | INFO | High Water Breakdown: | 20260505 18:09:57.403 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:09:57.403 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:09:57.403 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:09:57.403 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:09:57.403 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:09:57.403 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:09:57.403 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 18:09:57.403 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 18:09:57.403 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:09:57.403 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 18:09:57.403 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 18:09:57.403 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 18:09:57.403 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 18:09:57.403 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 18:09:57.403 | INFO | ------------------------------------------ | 20260505 18:09:57.403 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:09:57.403 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 18:09:57.403 | INFO | Total mem usage: 25.6 GiB | 20260505 18:09:57.403 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 18:09:57.403 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 18:09:57.403 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 18:09:57.403 | INFO | High Water Breakdown: | 20260505 18:09:57.403 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 18:09:57.403 | INFO | ------------------------------------------ | 20260505 18:09:57.403 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:09:57.403 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 18:09:57.403 | INFO | Total mem usage: 15.7 GiB | 20260505 18:09:57.403 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 18:09:57.403 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 18:09:57.403 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 18:09:57.403 | INFO | High Water Breakdown: | 20260505 18:09:57.403 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 18:09:57.403 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 18:09:57.403 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 18:09:57.403 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 18:09:57.403 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 18:09:57.403 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 18:09:57.403 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 18:09:57.403 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 18:09:57.403 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 18:09:57.403 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:09:57.403 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 18:09:57.403 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:09:57.403 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 18:09:57.403 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 18:09:57.403 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 18:09:57.403 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 18:09:57.403 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:09:57.403 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 18:09:57.403 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:09:57.403 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:09:57.403 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:09:57.403 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 18:09:57.403 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 18:09:57.403 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:09:57.403 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 18:09:57.403 | INFO | ------------------------------------------ | 20260505 18:09:57.403 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:09:57.403 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 18:09:57.403 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:09:57.403 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:09:57.403 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.48e+01, 1.34e+01 / 2.35e+00 | 20260505 18:09:57.403 | INFO | PerfReport: 0.29%, AggregateStats -- 9.26e-02 / 1.02e-01, 1.06e-02 / 5.06e-03 | 20260505 18:09:57.403 | INFO | PerfReport: 11.82%, Baseline -- 3.72e+00 / 4.55e+00, 5.71e-01 / 1.09e-01 | 20260505 18:09:57.403 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.51e+00, 1.88e-01 / 9.69e-02 | 20260505 18:09:57.403 | INFO | PerfReport: 15.98%, DME -- 4.03e+01 / 4.48e+01, 4.18e+00 / 2.40e+00 | 20260505 18:09:57.403 | INFO | PerfReport: 56.75%, FrameLabeling -- 1.79e+01 / 1.91e+01, 1.34e+00 / 9.38e-01 | 20260505 18:09:57.403 | INFO | PerfReport: 7.36%, PulseAccumulating -- 2.32e+00 / 2.93e+00, 4.48e-01 / 1.46e-01 | 20260505 18:09:57.403 | INFO | PerfReport: 3.35%, Metrics -- 1.06e+00 / 1.39e+00, 2.87e-01 / 1.46e-01 | 20260505 18:09:57.404 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.67e-01 / 4.26e-01, 6.30e-02 / 3.04e-02 | 20260505 18:09:57.404 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 18:09:57.404 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:09:57.404 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:09:57.404 | INFO | PerfReport: Full Execution -- 9.98e+01 / -nan, 2.55e+01 / 0.00e+00 | 20260505 18:09:57.404 | INFO | PerfReport: 22.57%, Upload -- 2.25e+01 / 0.00e+00, 7.86e+00 / 0.00e+00 | 20260505 18:09:57.404 | INFO | PerfReport: 21.87%, Download -- 2.18e+01 / 0.00e+00, 7.82e+00 / 0.00e+00 | 20260505 18:09:57.404 | INFO | PerfReport: 31.57%, Compute -- 3.15e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 18:09:57.404 | INFO | PerfReport: 2.25%, LockUpload -- 2.24e+00 / 0.00e+00, 7.84e+00 / 0.00e+00 | 20260505 18:09:57.404 | INFO | PerfReport: 1.46%, LockDownload -- 1.46e+00 / 0.00e+00, 2.96e+00 / 0.00e+00 | 20260505 18:09:57.404 | INFO | PerfReport: 20.26%, LockCompute -- 2.02e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 18:09:57.404 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.39e-02 / 0.00e+00 | 20260505 18:09:57.537 | INFO | Starting a round of Baz IO | 20260505 18:10:00.731 | INFO | Populating RTMetrics report | 20260505 18:10:00.731 | INFO | RTMetrics report populated | 20260505 18:10:00.731 | INFO | Opening tmp RTMetrics json file | 20260505 18:10:00.736 | INFO | Serializing RTMetrics to json | 20260505 18:10:00.737 | INFO | Serialization done, swapping tmp file now... | 20260505 18:10:00.738 | INFO | RTMetrics done | 20260505 18:10:00.825 | INFO | Finished a round of Baz IO: Wrote 2.104 GiB in 3.288 seconds | 20260505 18:10:00.825 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.95% / 0.00% / 78.05% | 20260505 18:10:00.825 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.263532% / 0% / 0.000123183% | 20260505 18:10:00.825 | INFO | Baz IO overall: | 20260505 18:10:00.826 | INFO | BytesWritten : 3095400487596 | 20260505 18:10:00.826 | INFO | NumEvents : 1466783700846 | 20260505 18:10:02.538 | INFO | Analyzing chunk frames = [5251072,5251584) | 20260505 18:10:02.538 | INFO | MemInfo: { | 20260505 18:10:02.538 | INFO | "gpu" : " 17 GB", | 20260505 18:10:02.538 | INFO | "huge" : "", | 20260505 18:10:02.538 | INFO | "huge_shared" : " 27 GB", | 20260505 18:10:02.538 | INFO | "malloc" : " 14 GB", | 20260505 18:10:02.538 | INFO | "pinned" : " 29 GB" | 20260505 18:10:02.538 | INFO | } | 20260505 18:10:02.539 | INFO | Perf report for the last 5.13615 seconds: | 20260505 18:10:02.539 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:10:02.539 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:02.539 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:02.539 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:02.539 | INFO | ANALYSIS : 63.7%, 3.0, 99.2ms, 0 | 20260505 18:10:02.539 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:02.539 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260505 18:10:02.539 | INFO | PRE_HQ : 60.6%, 1.8, 56.1ms, 0 | 20260505 18:10:02.539 | INFO | BAZWRITER : 63.4%, 3.9, 125.4ms, 0 | 20260505 18:10:02.539 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 18:10:02.539 | INFO | Uploaded 36GiB to GPU in 2.16595s (16.6209GiB/s) | 20260505 18:10:02.539 | INFO | Downloaded 38GiB to GPU in 2.25757s (16.8323GiB/s) | 20260505 18:10:07.684 | INFO | Analyzing chunk frames = [5251584,5252096) | 20260505 18:10:07.684 | INFO | MemInfo: { | 20260505 18:10:07.684 | INFO | "gpu" : " 17 GB", | 20260505 18:10:07.684 | INFO | "huge" : "", | 20260505 18:10:07.685 | INFO | "huge_shared" : " 27 GB", | 20260505 18:10:07.685 | INFO | "malloc" : " 14 GB", | 20260505 18:10:07.685 | INFO | "pinned" : " 29 GB" | 20260505 18:10:07.685 | INFO | } | 20260505 18:10:07.685 | INFO | Perf report for the last 5.14612 seconds: | 20260505 18:10:07.685 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:10:07.685 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:07.685 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:07.685 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:07.685 | INFO | ANALYSIS : 62.7%, 3.0, 98.0ms, 0 | 20260505 18:10:07.685 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:07.685 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:07.685 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260505 18:10:07.685 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:07.685 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:07.685 | INFO | Uploaded 36GiB to GPU in 2.12768s (16.9198GiB/s) | 20260505 18:10:07.685 | INFO | Downloaded 34GiB to GPU in 2.03233s (16.7296GiB/s) | 20260505 18:10:12.834 | INFO | Analyzing chunk frames = [5252096,5252608) | 20260505 18:10:12.834 | INFO | MemInfo: { | 20260505 18:10:12.834 | INFO | "gpu" : " 17 GB", | 20260505 18:10:12.834 | INFO | "huge" : "", | 20260505 18:10:12.834 | INFO | "huge_shared" : " 27 GB", | 20260505 18:10:12.834 | INFO | "malloc" : " 14 GB", | 20260505 18:10:12.834 | INFO | "pinned" : " 29 GB" | 20260505 18:10:12.834 | INFO | } | 20260505 18:10:12.834 | INFO | Perf report for the last 5.14927 seconds: | 20260505 18:10:12.834 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:10:12.834 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:12.834 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:12.834 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:12.834 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260505 18:10:12.834 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:12.834 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:12.834 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260505 18:10:12.834 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:12.834 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:12.834 | INFO | Uploaded 36GiB to GPU in 2.20767s (16.3068GiB/s) | 20260505 18:10:12.834 | INFO | Downloaded 34GiB to GPU in 2.05929s (16.5105GiB/s) | 20260505 18:10:17.962 | INFO | Analyzing chunk frames = [5252608,5253120) | 20260505 18:10:17.962 | INFO | MemInfo: { | 20260505 18:10:17.962 | INFO | "gpu" : " 17 GB", | 20260505 18:10:17.962 | INFO | "huge" : "", | 20260505 18:10:17.962 | INFO | "huge_shared" : " 27 GB", | 20260505 18:10:17.962 | INFO | "malloc" : " 14 GB", | 20260505 18:10:17.962 | INFO | "pinned" : " 29 GB" | 20260505 18:10:17.962 | INFO | } | 20260505 18:10:17.962 | INFO | Perf report for the last 5.12847 seconds: | 20260505 18:10:17.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:10:17.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:17.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:17.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:17.962 | INFO | ANALYSIS : 62.6%, 3.0, 97.0ms, 0 | 20260505 18:10:17.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:17.962 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:17.962 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 18:10:17.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:17.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:17.963 | INFO | Uploaded 36GiB to GPU in 2.2049s (16.3273GiB/s) | 20260505 18:10:17.963 | INFO | Downloaded 34GiB to GPU in 2.03803s (16.6827GiB/s) | 20260505 18:10:23.097 | INFO | Analyzing chunk frames = [5253120,5253632) | 20260505 18:10:23.097 | INFO | MemInfo: { | 20260505 18:10:23.097 | INFO | "gpu" : " 17 GB", | 20260505 18:10:23.097 | INFO | "huge" : "", | 20260505 18:10:23.097 | INFO | "huge_shared" : " 27 GB", | 20260505 18:10:23.098 | INFO | "malloc" : " 14 GB", | 20260505 18:10:23.098 | INFO | "pinned" : " 29 GB" | 20260505 18:10:23.098 | INFO | } | 20260505 18:10:23.098 | INFO | Perf report for the last 5.13524 seconds: | 20260505 18:10:23.098 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:10:23.098 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:23.098 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:23.098 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:23.098 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 18:10:23.098 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:23.098 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:23.098 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:10:23.098 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:23.098 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:23.098 | INFO | Uploaded 36GiB to GPU in 2.14987s (16.7452GiB/s) | 20260505 18:10:23.098 | INFO | Downloaded 34GiB to GPU in 2.06208s (16.4882GiB/s) | 20260505 18:10:28.240 | INFO | Analyzing chunk frames = [5253632,5254144) | 20260505 18:10:28.240 | INFO | MemInfo: { | 20260505 18:10:28.240 | INFO | "gpu" : " 17 GB", | 20260505 18:10:28.240 | INFO | "huge" : "", | 20260505 18:10:28.240 | INFO | "huge_shared" : " 27 GB", | 20260505 18:10:28.240 | INFO | "malloc" : " 14 GB", | 20260505 18:10:28.240 | INFO | "pinned" : " 29 GB" | 20260505 18:10:28.240 | INFO | } | 20260505 18:10:28.240 | INFO | Perf report for the last 5.14249 seconds: | 20260505 18:10:28.240 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:10:28.240 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:28.240 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:28.240 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:28.240 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260505 18:10:28.240 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:28.240 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:28.240 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:10:28.240 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:28.240 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:28.240 | INFO | Uploaded 36GiB to GPU in 2.14369s (16.7934GiB/s) | 20260505 18:10:28.240 | INFO | Downloaded 34GiB to GPU in 2.04486s (16.627GiB/s) | 20260505 18:10:33.388 | INFO | Analyzing chunk frames = [5254144,5254656) | 20260505 18:10:33.388 | INFO | MemInfo: { | 20260505 18:10:33.388 | INFO | "gpu" : " 17 GB", | 20260505 18:10:33.388 | INFO | "huge" : "", | 20260505 18:10:33.388 | INFO | "huge_shared" : " 27 GB", | 20260505 18:10:33.388 | INFO | "malloc" : " 14 GB", | 20260505 18:10:33.388 | INFO | "pinned" : " 29 GB" | 20260505 18:10:33.388 | INFO | } | 20260505 18:10:33.388 | INFO | Perf report for the last 5.14753 seconds: | 20260505 18:10:33.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:10:33.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:33.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:33.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:33.388 | INFO | ANALYSIS : 63.3%, 3.0, 99.1ms, 0 | 20260505 18:10:33.388 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:33.388 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:33.388 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 18:10:33.388 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:33.388 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:33.388 | INFO | Uploaded 36GiB to GPU in 2.15994s (16.6671GiB/s) | 20260505 18:10:33.388 | INFO | Downloaded 34GiB to GPU in 2.17896s (15.6038GiB/s) | 20260505 18:10:38.528 | INFO | Analyzing chunk frames = [5254656,5255168) | 20260505 18:10:38.528 | INFO | MemInfo: { | 20260505 18:10:38.528 | INFO | "gpu" : " 17 GB", | 20260505 18:10:38.528 | INFO | "huge" : "", | 20260505 18:10:38.528 | INFO | "huge_shared" : " 27 GB", | 20260505 18:10:38.528 | INFO | "malloc" : " 14 GB", | 20260505 18:10:38.528 | INFO | "pinned" : " 29 GB" | 20260505 18:10:38.528 | INFO | } | 20260505 18:10:38.528 | INFO | Perf report for the last 5.14073 seconds: | 20260505 18:10:38.528 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:10:38.528 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:38.528 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:38.528 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:38.529 | INFO | ANALYSIS : 62.5%, 3.0, 97.7ms, 0 | 20260505 18:10:38.529 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:38.529 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:38.529 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 18:10:38.529 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:38.529 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:38.529 | INFO | Uploaded 36GiB to GPU in 2.12667s (16.9279GiB/s) | 20260505 18:10:38.529 | INFO | Downloaded 34GiB to GPU in 2.01037s (16.9123GiB/s) | 20260505 18:10:38.663 | INFO | Starting a round of Baz IO | 20260505 18:10:41.837 | INFO | Populating RTMetrics report | 20260505 18:10:41.838 | INFO | RTMetrics report populated | 20260505 18:10:41.838 | INFO | Opening tmp RTMetrics json file | 20260505 18:10:41.846 | INFO | Serializing RTMetrics to json | 20260505 18:10:41.848 | INFO | Serialization done, swapping tmp file now... | 20260505 18:10:41.850 | INFO | RTMetrics done | 20260505 18:10:41.932 | INFO | Finished a round of Baz IO: Wrote 2.1 GiB in 3.269 seconds | 20260505 18:10:41.932 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.95% / 0.00% / 78.05% | 20260505 18:10:41.932 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.262459% / 0% / 0.000103315% | 20260505 18:10:41.932 | INFO | Baz IO overall: | 20260505 18:10:41.932 | INFO | BytesWritten : 3097655732477 | 20260505 18:10:41.932 | INFO | NumEvents : 1467835063625 | 20260505 18:10:43.657 | INFO | Analyzing chunk frames = [5255168,5255680) | 20260505 18:10:43.657 | INFO | MemInfo: { | 20260505 18:10:43.657 | INFO | "gpu" : " 17 GB", | 20260505 18:10:43.657 | INFO | "huge" : "", | 20260505 18:10:43.657 | INFO | "huge_shared" : " 27 GB", | 20260505 18:10:43.657 | INFO | "malloc" : " 14 GB", | 20260505 18:10:43.657 | INFO | "pinned" : " 29 GB" | 20260505 18:10:43.657 | INFO | } | 20260505 18:10:43.657 | INFO | Perf report for the last 5.12886 seconds: | 20260505 18:10:43.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:10:43.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:43.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:43.657 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:43.657 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 18:10:43.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:43.657 | INFO | RT_METRICS : 44.7%, 1.0, 23.4ms, 0 | 20260505 18:10:43.657 | INFO | PRE_HQ : 60.5%, 1.8, 55.9ms, 0 | 20260505 18:10:43.657 | INFO | BAZWRITER : 63.3%, 3.8, 122.7ms, 0 | 20260505 18:10:43.657 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 18:10:43.657 | INFO | Uploaded 36GiB to GPU in 2.15734s (16.6872GiB/s) | 20260505 18:10:43.658 | INFO | Downloaded 38GiB to GPU in 2.24414s (16.933GiB/s) | 20260505 18:10:48.797 | INFO | Analyzing chunk frames = [5255680,5256192) | 20260505 18:10:48.797 | INFO | MemInfo: { | 20260505 18:10:48.797 | INFO | "gpu" : " 17 GB", | 20260505 18:10:48.797 | INFO | "huge" : "", | 20260505 18:10:48.797 | INFO | "huge_shared" : " 27 GB", | 20260505 18:10:48.797 | INFO | "malloc" : " 14 GB", | 20260505 18:10:48.798 | INFO | "pinned" : " 29 GB" | 20260505 18:10:48.798 | INFO | } | 20260505 18:10:48.798 | INFO | Perf report for the last 5.14024 seconds: | 20260505 18:10:48.798 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:10:48.798 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:48.798 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:48.798 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:48.798 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 18:10:48.798 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:48.798 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:48.798 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 18:10:48.798 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:48.798 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:48.798 | INFO | Uploaded 36GiB to GPU in 2.14795s (16.7601GiB/s) | 20260505 18:10:48.798 | INFO | Downloaded 34GiB to GPU in 2.03512s (16.7066GiB/s) | 20260505 18:10:53.944 | INFO | Analyzing chunk frames = [5256192,5256704) | 20260505 18:10:53.944 | INFO | MemInfo: { | 20260505 18:10:53.944 | INFO | "gpu" : " 17 GB", | 20260505 18:10:53.944 | INFO | "huge" : "", | 20260505 18:10:53.944 | INFO | "huge_shared" : " 27 GB", | 20260505 18:10:53.944 | INFO | "malloc" : " 14 GB", | 20260505 18:10:53.944 | INFO | "pinned" : " 29 GB" | 20260505 18:10:53.944 | INFO | } | 20260505 18:10:53.944 | INFO | Perf report for the last 5.14621 seconds: | 20260505 18:10:53.944 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:10:53.944 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:53.944 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:53.944 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:53.944 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 18:10:53.944 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:53.944 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:53.944 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:10:53.944 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:53.944 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:53.944 | INFO | Uploaded 36GiB to GPU in 2.20496s (16.3269GiB/s) | 20260505 18:10:53.944 | INFO | Downloaded 34GiB to GPU in 2.04932s (16.5909GiB/s) | 20260505 18:10:59.079 | INFO | Analyzing chunk frames = [5256704,5257216) | 20260505 18:10:59.079 | INFO | MemInfo: { | 20260505 18:10:59.079 | INFO | "gpu" : " 17 GB", | 20260505 18:10:59.079 | INFO | "huge" : "", | 20260505 18:10:59.079 | INFO | "huge_shared" : " 27 GB", | 20260505 18:10:59.079 | INFO | "malloc" : " 14 GB", | 20260505 18:10:59.079 | INFO | "pinned" : " 29 GB" | 20260505 18:10:59.079 | INFO | } | 20260505 18:10:59.079 | INFO | Perf report for the last 5.13532 seconds: | 20260505 18:10:59.079 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:10:59.079 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:59.079 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:59.079 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:59.079 | INFO | ANALYSIS : 62.4%, 3.0, 96.8ms, 0 | 20260505 18:10:59.079 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:59.079 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:59.079 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:10:59.079 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:59.079 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:10:59.079 | INFO | Uploaded 36GiB to GPU in 2.1812s (16.5046GiB/s) | 20260505 18:10:59.079 | INFO | Downloaded 34GiB to GPU in 2.06467s (16.4675GiB/s) | 20260505 18:11:04.213 | INFO | Analyzing chunk frames = [5257216,5257728) | 20260505 18:11:04.213 | INFO | MemInfo: { | 20260505 18:11:04.213 | INFO | "gpu" : " 17 GB", | 20260505 18:11:04.213 | INFO | "huge" : "", | 20260505 18:11:04.213 | INFO | "huge_shared" : " 27 GB", | 20260505 18:11:04.213 | INFO | "malloc" : " 14 GB", | 20260505 18:11:04.213 | INFO | "pinned" : " 29 GB" | 20260505 18:11:04.213 | INFO | } | 20260505 18:11:04.214 | INFO | Perf report for the last 5.13438 seconds: | 20260505 18:11:04.214 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:11:04.214 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:04.214 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:04.214 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:04.214 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 18:11:04.214 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:04.214 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:04.214 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 18:11:04.214 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:04.214 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:04.214 | INFO | Uploaded 36GiB to GPU in 2.13124s (16.8916GiB/s) | 20260505 18:11:04.214 | INFO | Downloaded 34GiB to GPU in 2.05849s (16.5169GiB/s) | 20260505 18:11:09.354 | INFO | Analyzing chunk frames = [5257728,5258240) | 20260505 18:11:09.354 | INFO | MemInfo: { | 20260505 18:11:09.354 | INFO | "gpu" : " 17 GB", | 20260505 18:11:09.354 | INFO | "huge" : "", | 20260505 18:11:09.354 | INFO | "huge_shared" : " 27 GB", | 20260505 18:11:09.354 | INFO | "malloc" : " 14 GB", | 20260505 18:11:09.354 | INFO | "pinned" : " 29 GB" | 20260505 18:11:09.354 | INFO | } | 20260505 18:11:09.354 | INFO | Perf report for the last 5.14052 seconds: | 20260505 18:11:09.354 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:11:09.354 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:09.354 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:09.354 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:09.354 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260505 18:11:09.354 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:09.354 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:09.354 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:11:09.354 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:09.354 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:09.354 | INFO | Uploaded 36GiB to GPU in 2.14054s (16.8182GiB/s) | 20260505 18:11:09.354 | INFO | Downloaded 34GiB to GPU in 2.03578s (16.7012GiB/s) | 20260505 18:11:14.499 | INFO | Analyzing chunk frames = [5258240,5258752) | 20260505 18:11:14.499 | INFO | MemInfo: { | 20260505 18:11:14.499 | INFO | "gpu" : " 17 GB", | 20260505 18:11:14.499 | INFO | "huge" : "", | 20260505 18:11:14.499 | INFO | "huge_shared" : " 27 GB", | 20260505 18:11:14.499 | INFO | "malloc" : " 14 GB", | 20260505 18:11:14.499 | INFO | "pinned" : " 29 GB" | 20260505 18:11:14.499 | INFO | } | 20260505 18:11:14.499 | INFO | Perf report for the last 5.14492 seconds: | 20260505 18:11:14.499 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:11:14.499 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:14.499 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:14.499 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:14.499 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 18:11:14.499 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:14.499 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:14.499 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 18:11:14.499 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:14.499 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:14.499 | INFO | Uploaded 36GiB to GPU in 2.17015s (16.5887GiB/s) | 20260505 18:11:14.499 | INFO | Downloaded 34GiB to GPU in 2.18605s (15.5532GiB/s) | 20260505 18:11:19.636 | INFO | Analyzing chunk frames = [5258752,5259264) | 20260505 18:11:19.636 | INFO | MemInfo: { | 20260505 18:11:19.636 | INFO | "gpu" : " 17 GB", | 20260505 18:11:19.636 | INFO | "huge" : "", | 20260505 18:11:19.636 | INFO | "huge_shared" : " 27 GB", | 20260505 18:11:19.636 | INFO | "malloc" : " 14 GB", | 20260505 18:11:19.636 | INFO | "pinned" : " 29 GB" | 20260505 18:11:19.636 | INFO | } | 20260505 18:11:19.636 | INFO | Perf report for the last 5.13699 seconds: | 20260505 18:11:19.636 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:11:19.636 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:19.636 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:19.636 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:19.636 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 18:11:19.636 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:19.636 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:19.636 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 18:11:19.636 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:19.636 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:19.636 | INFO | Uploaded 36GiB to GPU in 2.19923s (16.3694GiB/s) | 20260505 18:11:19.636 | INFO | Downloaded 34GiB to GPU in 2.04032s (16.6641GiB/s) | 20260505 18:11:19.636 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 18:11:19.636 | INFO | Total mem usage: 15.3 GiB | 20260505 18:11:19.636 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 18:11:19.636 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.04%) | 20260505 18:11:19.636 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 18:11:19.636 | INFO | High Water Breakdown: | 20260505 18:11:19.636 | INFO | 61% : 8.66 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 18:11:19.636 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 18:11:19.636 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 18:11:19.636 | INFO | ------------------------------------------ | 20260505 18:11:19.636 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:11:19.636 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 18:11:19.636 | INFO | Total mem usage: 26.5 GiB | 20260505 18:11:19.636 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 18:11:19.636 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 18:11:19.636 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260505 18:11:19.636 | INFO | High Water Breakdown: | 20260505 18:11:19.636 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:11:19.636 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:11:19.636 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:11:19.636 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:11:19.636 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:11:19.636 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:11:19.636 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 18:11:19.636 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 18:11:19.636 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:11:19.637 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 18:11:19.637 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 18:11:19.637 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 18:11:19.637 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 18:11:19.637 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 18:11:19.637 | INFO | ------------------------------------------ | 20260505 18:11:19.637 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:11:19.637 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 18:11:19.637 | INFO | Total mem usage: 25.6 GiB | 20260505 18:11:19.637 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 18:11:19.637 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 18:11:19.637 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 18:11:19.637 | INFO | High Water Breakdown: | 20260505 18:11:19.637 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 18:11:19.637 | INFO | ------------------------------------------ | 20260505 18:11:19.637 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:11:19.637 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 18:11:19.637 | INFO | Total mem usage: 15.7 GiB | 20260505 18:11:19.637 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 18:11:19.637 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 18:11:19.637 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 18:11:19.637 | INFO | High Water Breakdown: | 20260505 18:11:19.637 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 18:11:19.637 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 18:11:19.637 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 18:11:19.637 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 18:11:19.637 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 18:11:19.637 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 18:11:19.637 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 18:11:19.637 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 18:11:19.637 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 18:11:19.637 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:11:19.637 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 18:11:19.637 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:11:19.637 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 18:11:19.637 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 18:11:19.637 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 18:11:19.637 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 18:11:19.637 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:11:19.637 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 18:11:19.637 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:11:19.637 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:11:19.637 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:11:19.637 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 18:11:19.637 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 18:11:19.637 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:11:19.637 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 18:11:19.637 | INFO | ------------------------------------------ | 20260505 18:11:19.637 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:11:19.637 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 18:11:19.637 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:11:19.637 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:11:19.637 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.47e+01, 1.35e+01 / 2.25e+00 | 20260505 18:11:19.637 | INFO | PerfReport: 0.29%, AggregateStats -- 9.30e-02 / 1.03e-01, 1.11e-02 / 2.90e-03 | 20260505 18:11:19.637 | INFO | PerfReport: 11.77%, Baseline -- 3.71e+00 / 4.54e+00, 5.61e-01 / 1.48e-01 | 20260505 18:11:19.637 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.54e+00, 1.93e-01 / 7.20e-02 | 20260505 18:11:19.637 | INFO | PerfReport: 15.93%, DME -- 4.02e+01 / 4.45e+01, 4.20e+00 / 2.36e+00 | 20260505 18:11:19.637 | INFO | PerfReport: 56.89%, FrameLabeling -- 1.80e+01 / 1.93e+01, 1.45e+00 / 1.07e+00 | 20260505 18:11:19.637 | INFO | PerfReport: 7.33%, PulseAccumulating -- 2.31e+00 / 2.91e+00, 4.45e-01 / 1.56e-01 | 20260505 18:11:19.637 | INFO | PerfReport: 3.34%, Metrics -- 1.06e+00 / 1.40e+00, 2.89e-01 / 1.50e-01 | 20260505 18:11:19.637 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.67e-01 / 4.31e-01, 6.44e-02 / 4.30e-02 | 20260505 18:11:19.637 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 18:11:19.637 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:11:19.637 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:11:19.637 | INFO | PerfReport: Full Execution -- 9.99e+01 / -nan, 2.58e+01 / 0.00e+00 | 20260505 18:11:19.637 | INFO | PerfReport: 22.57%, Upload -- 2.25e+01 / 0.00e+00, 7.81e+00 / 0.00e+00 | 20260505 18:11:19.637 | INFO | PerfReport: 21.77%, Download -- 2.17e+01 / 0.00e+00, 7.75e+00 / 0.00e+00 | 20260505 18:11:19.637 | INFO | PerfReport: 31.60%, Compute -- 3.16e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 18:11:19.637 | INFO | PerfReport: 2.34%, LockUpload -- 2.34e+00 / 0.00e+00, 8.16e+00 / 0.00e+00 | 20260505 18:11:19.637 | INFO | PerfReport: 1.38%, LockDownload -- 1.38e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260505 18:11:19.637 | INFO | PerfReport: 20.33%, LockCompute -- 2.03e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260505 18:11:19.637 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.54e-02 / 0.00e+00 | 20260505 18:11:19.769 | INFO | Starting a round of Baz IO | 20260505 18:11:22.949 | INFO | Populating RTMetrics report | 20260505 18:11:22.949 | INFO | RTMetrics report populated | 20260505 18:11:22.949 | INFO | Opening tmp RTMetrics json file | 20260505 18:11:22.953 | INFO | Serializing RTMetrics to json | 20260505 18:11:22.954 | INFO | Serialization done, swapping tmp file now... | 20260505 18:11:22.955 | INFO | RTMetrics done | 20260505 18:11:23.043 | INFO | Finished a round of Baz IO: Wrote 2.102 GiB in 3.274 seconds | 20260505 18:11:23.043 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.94% / 0.00% / 78.05% | 20260505 18:11:23.043 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.262686% / 0% / 0.000111262% | 20260505 18:11:23.043 | INFO | Baz IO overall: | 20260505 18:11:23.043 | INFO | BytesWritten : 3099912554726 | 20260505 18:11:23.043 | INFO | NumEvents : 1468887527325 | 20260505 18:11:24.775 | INFO | Analyzing chunk frames = [5259264,5259776) | 20260505 18:11:24.775 | INFO | MemInfo: { | 20260505 18:11:24.775 | INFO | "gpu" : " 17 GB", | 20260505 18:11:24.775 | INFO | "huge" : "", | 20260505 18:11:24.775 | INFO | "huge_shared" : " 27 GB", | 20260505 18:11:24.775 | INFO | "malloc" : " 14 GB", | 20260505 18:11:24.775 | INFO | "pinned" : " 29 GB" | 20260505 18:11:24.775 | INFO | } | 20260505 18:11:24.775 | INFO | Perf report for the last 5.13909 seconds: | 20260505 18:11:24.775 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:11:24.775 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:24.775 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:24.775 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:24.775 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 18:11:24.775 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:24.775 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260505 18:11:24.775 | INFO | PRE_HQ : 60.6%, 1.8, 56.1ms, 0 | 20260505 18:11:24.775 | INFO | BAZWRITER : 63.2%, 3.8, 123.5ms, 0 | 20260505 18:11:24.775 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 18:11:24.775 | INFO | Uploaded 36GiB to GPU in 2.16059s (16.6621GiB/s) | 20260505 18:11:24.775 | INFO | Downloaded 38GiB to GPU in 2.23214s (17.024GiB/s) | 20260505 18:11:29.909 | INFO | Analyzing chunk frames = [5259776,5260288) | 20260505 18:11:29.909 | INFO | MemInfo: { | 20260505 18:11:29.909 | INFO | "gpu" : " 17 GB", | 20260505 18:11:29.909 | INFO | "huge" : "", | 20260505 18:11:29.909 | INFO | "huge_shared" : " 27 GB", | 20260505 18:11:29.909 | INFO | "malloc" : " 14 GB", | 20260505 18:11:29.909 | INFO | "pinned" : " 29 GB" | 20260505 18:11:29.909 | INFO | } | 20260505 18:11:29.909 | INFO | Perf report for the last 5.13424 seconds: | 20260505 18:11:29.909 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:11:29.909 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:29.909 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:29.909 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:29.909 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 18:11:29.909 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:29.909 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:29.909 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:11:29.909 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:29.909 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:29.909 | INFO | Uploaded 36GiB to GPU in 2.13941s (16.8271GiB/s) | 20260505 18:11:29.910 | INFO | Downloaded 34GiB to GPU in 2.02617s (16.7804GiB/s) | 20260505 18:11:35.050 | INFO | Analyzing chunk frames = [5260288,5260800) | 20260505 18:11:35.050 | INFO | MemInfo: { | 20260505 18:11:35.050 | INFO | "gpu" : " 17 GB", | 20260505 18:11:35.050 | INFO | "huge" : "", | 20260505 18:11:35.050 | INFO | "huge_shared" : " 27 GB", | 20260505 18:11:35.050 | INFO | "malloc" : " 14 GB", | 20260505 18:11:35.050 | INFO | "pinned" : " 29 GB" | 20260505 18:11:35.050 | INFO | } | 20260505 18:11:35.050 | INFO | Perf report for the last 5.1406 seconds: | 20260505 18:11:35.050 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:11:35.050 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:35.050 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:35.050 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:35.050 | INFO | ANALYSIS : 62.6%, 3.0, 97.4ms, 0 | 20260505 18:11:35.050 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:35.050 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:35.050 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:11:35.050 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:35.050 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:35.050 | INFO | Uploaded 36GiB to GPU in 2.20111s (16.3554GiB/s) | 20260505 18:11:35.050 | INFO | Downloaded 34GiB to GPU in 2.02893s (16.7576GiB/s) | 20260505 18:11:40.195 | INFO | Analyzing chunk frames = [5260800,5261312) | 20260505 18:11:40.195 | INFO | MemInfo: { | 20260505 18:11:40.195 | INFO | "gpu" : " 17 GB", | 20260505 18:11:40.195 | INFO | "huge" : "", | 20260505 18:11:40.195 | INFO | "huge_shared" : " 27 GB", | 20260505 18:11:40.195 | INFO | "malloc" : " 14 GB", | 20260505 18:11:40.195 | INFO | "pinned" : " 29 GB" | 20260505 18:11:40.195 | INFO | } | 20260505 18:11:40.195 | INFO | Perf report for the last 5.14524 seconds: | 20260505 18:11:40.195 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:11:40.195 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:40.195 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:40.195 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:40.195 | INFO | ANALYSIS : 62.8%, 3.0, 97.6ms, 0 | 20260505 18:11:40.195 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:40.195 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:40.195 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 18:11:40.195 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:40.195 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:40.195 | INFO | Uploaded 36GiB to GPU in 2.22696s (16.1655GiB/s) | 20260505 18:11:40.195 | INFO | Downloaded 34GiB to GPU in 2.12171s (16.0248GiB/s) | 20260505 18:11:45.338 | INFO | Analyzing chunk frames = [5261312,5261824) | 20260505 18:11:45.338 | INFO | MemInfo: { | 20260505 18:11:45.338 | INFO | "gpu" : " 17 GB", | 20260505 18:11:45.338 | INFO | "huge" : "", | 20260505 18:11:45.338 | INFO | "huge_shared" : " 27 GB", | 20260505 18:11:45.338 | INFO | "malloc" : " 14 GB", | 20260505 18:11:45.338 | INFO | "pinned" : " 29 GB" | 20260505 18:11:45.338 | INFO | } | 20260505 18:11:45.338 | INFO | Perf report for the last 5.14282 seconds: | 20260505 18:11:45.338 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:11:45.338 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:45.338 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:45.338 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:45.338 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260505 18:11:45.338 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:45.338 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:45.338 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:11:45.338 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:45.338 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:45.338 | INFO | Uploaded 36GiB to GPU in 2.14588s (16.7763GiB/s) | 20260505 18:11:45.338 | INFO | Downloaded 34GiB to GPU in 2.06456s (16.4684GiB/s) | 20260505 18:11:50.478 | INFO | Analyzing chunk frames = [5261824,5262336) | 20260505 18:11:50.478 | INFO | MemInfo: { | 20260505 18:11:50.478 | INFO | "gpu" : " 17 GB", | 20260505 18:11:50.478 | INFO | "huge" : "", | 20260505 18:11:50.478 | INFO | "huge_shared" : " 27 GB", | 20260505 18:11:50.478 | INFO | "malloc" : " 14 GB", | 20260505 18:11:50.478 | INFO | "pinned" : " 29 GB" | 20260505 18:11:50.478 | INFO | } | 20260505 18:11:50.478 | INFO | Perf report for the last 5.14033 seconds: | 20260505 18:11:50.478 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:11:50.478 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:50.478 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:50.478 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:50.478 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 18:11:50.478 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:50.478 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:50.478 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:11:50.478 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:50.478 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:50.479 | INFO | Uploaded 36GiB to GPU in 2.14025s (16.8205GiB/s) | 20260505 18:11:50.479 | INFO | Downloaded 34GiB to GPU in 2.05198s (16.5693GiB/s) | 20260505 18:11:55.601 | INFO | Analyzing chunk frames = [5262336,5262848) | 20260505 18:11:55.601 | INFO | MemInfo: { | 20260505 18:11:55.601 | INFO | "gpu" : " 17 GB", | 20260505 18:11:55.601 | INFO | "huge" : "", | 20260505 18:11:55.601 | INFO | "huge_shared" : " 27 GB", | 20260505 18:11:55.601 | INFO | "malloc" : " 14 GB", | 20260505 18:11:55.601 | INFO | "pinned" : " 29 GB" | 20260505 18:11:55.601 | INFO | } | 20260505 18:11:55.601 | INFO | Perf report for the last 5.12297 seconds: | 20260505 18:11:55.601 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:11:55.601 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:55.601 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:55.601 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:55.601 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 18:11:55.601 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:55.601 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:55.601 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 18:11:55.601 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:55.601 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:11:55.602 | INFO | Uploaded 36GiB to GPU in 2.11483s (17.0226GiB/s) | 20260505 18:11:55.602 | INFO | Downloaded 34GiB to GPU in 2.13605s (15.9173GiB/s) | 20260505 18:12:00.756 | INFO | Analyzing chunk frames = [5262848,5263360) | 20260505 18:12:00.756 | INFO | MemInfo: { | 20260505 18:12:00.756 | INFO | "gpu" : " 17 GB", | 20260505 18:12:00.756 | INFO | "huge" : "", | 20260505 18:12:00.756 | INFO | "huge_shared" : " 27 GB", | 20260505 18:12:00.756 | INFO | "malloc" : " 14 GB", | 20260505 18:12:00.756 | INFO | "pinned" : " 29 GB" | 20260505 18:12:00.756 | INFO | } | 20260505 18:12:00.756 | INFO | Perf report for the last 5.15451 seconds: | 20260505 18:12:00.756 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:12:00.756 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:00.756 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:00.756 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:00.756 | INFO | ANALYSIS : 63.0%, 3.0, 98.6ms, 0 | 20260505 18:12:00.756 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:00.756 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:00.756 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 18:12:00.756 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:00.756 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:00.756 | INFO | Uploaded 36GiB to GPU in 2.172s (16.5745GiB/s) | 20260505 18:12:00.756 | INFO | Downloaded 34GiB to GPU in 2.03462s (16.7108GiB/s) | 20260505 18:12:00.890 | INFO | Starting a round of Baz IO | 20260505 18:12:04.066 | INFO | Populating RTMetrics report | 20260505 18:12:04.066 | INFO | RTMetrics report populated | 20260505 18:12:04.066 | INFO | Opening tmp RTMetrics json file | 20260505 18:12:04.068 | INFO | Serializing RTMetrics to json | 20260505 18:12:04.070 | INFO | Serialization done, swapping tmp file now... | 20260505 18:12:04.070 | INFO | RTMetrics done | 20260505 18:12:04.159 | INFO | Finished a round of Baz IO: Wrote 2.101 GiB in 3.269 seconds | 20260505 18:12:04.159 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.94% / 0.00% / 78.06% | 20260505 18:12:04.159 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.262642% / 0% / 9.93411e-05% | 20260505 18:12:04.159 | INFO | Baz IO overall: | 20260505 18:12:04.159 | INFO | BytesWritten : 3102167963462 | 20260505 18:12:04.159 | INFO | NumEvents : 1469939347575 | 20260505 18:12:05.899 | INFO | Analyzing chunk frames = [5263360,5263872) | 20260505 18:12:05.899 | INFO | MemInfo: { | 20260505 18:12:05.899 | INFO | "gpu" : " 17 GB", | 20260505 18:12:05.899 | INFO | "huge" : "", | 20260505 18:12:05.899 | INFO | "huge_shared" : " 27 GB", | 20260505 18:12:05.899 | INFO | "malloc" : " 14 GB", | 20260505 18:12:05.899 | INFO | "pinned" : " 29 GB" | 20260505 18:12:05.899 | INFO | } | 20260505 18:12:05.899 | INFO | Perf report for the last 5.1429 seconds: | 20260505 18:12:05.899 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:12:05.899 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:05.899 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:05.899 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:05.899 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 18:12:05.899 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:05.899 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260505 18:12:05.899 | INFO | PRE_HQ : 60.4%, 1.8, 56.0ms, 0 | 20260505 18:12:05.899 | INFO | BAZWRITER : 63.1%, 3.8, 122.8ms, 0 | 20260505 18:12:05.899 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 18:12:05.899 | INFO | Uploaded 36GiB to GPU in 2.1643s (16.6335GiB/s) | 20260505 18:12:05.899 | INFO | Downloaded 38GiB to GPU in 2.23734s (16.9845GiB/s) | 20260505 18:12:11.027 | INFO | Analyzing chunk frames = [5263872,5264384) | 20260505 18:12:11.027 | INFO | MemInfo: { | 20260505 18:12:11.027 | INFO | "gpu" : " 17 GB", | 20260505 18:12:11.027 | INFO | "huge" : "", | 20260505 18:12:11.027 | INFO | "huge_shared" : " 27 GB", | 20260505 18:12:11.027 | INFO | "malloc" : " 14 GB", | 20260505 18:12:11.027 | INFO | "pinned" : " 29 GB" | 20260505 18:12:11.027 | INFO | } | 20260505 18:12:11.027 | INFO | Perf report for the last 5.12847 seconds: | 20260505 18:12:11.027 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:12:11.027 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:11.027 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:11.027 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:11.027 | INFO | ANALYSIS : 62.9%, 3.0, 98.0ms, 0 | 20260505 18:12:11.027 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:11.027 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:11.027 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:12:11.027 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:11.027 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:11.028 | INFO | Uploaded 36GiB to GPU in 2.14861s (16.755GiB/s) | 20260505 18:12:11.028 | INFO | Downloaded 34GiB to GPU in 2.0396s (16.6699GiB/s) | 20260505 18:12:16.166 | INFO | Analyzing chunk frames = [5264384,5264896) | 20260505 18:12:16.166 | INFO | MemInfo: { | 20260505 18:12:16.166 | INFO | "gpu" : " 17 GB", | 20260505 18:12:16.166 | INFO | "huge" : "", | 20260505 18:12:16.166 | INFO | "huge_shared" : " 27 GB", | 20260505 18:12:16.166 | INFO | "malloc" : " 14 GB", | 20260505 18:12:16.166 | INFO | "pinned" : " 29 GB" | 20260505 18:12:16.166 | INFO | } | 20260505 18:12:16.166 | INFO | Perf report for the last 5.13863 seconds: | 20260505 18:12:16.166 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:12:16.166 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:16.166 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:16.166 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:16.166 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 18:12:16.166 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:16.166 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:16.166 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 18:12:16.166 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:16.166 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:16.166 | INFO | Uploaded 36GiB to GPU in 2.20476s (16.3283GiB/s) | 20260505 18:12:16.166 | INFO | Downloaded 34GiB to GPU in 2.04175s (16.6524GiB/s) | 20260505 18:12:21.308 | INFO | Analyzing chunk frames = [5264896,5265408) | 20260505 18:12:21.308 | INFO | MemInfo: { | 20260505 18:12:21.308 | INFO | "gpu" : " 17 GB", | 20260505 18:12:21.308 | INFO | "huge" : "", | 20260505 18:12:21.308 | INFO | "huge_shared" : " 27 GB", | 20260505 18:12:21.308 | INFO | "malloc" : " 14 GB", | 20260505 18:12:21.308 | INFO | "pinned" : " 29 GB" | 20260505 18:12:21.308 | INFO | } | 20260505 18:12:21.308 | INFO | Perf report for the last 5.14225 seconds: | 20260505 18:12:21.308 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:12:21.308 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:21.308 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:21.308 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:21.308 | INFO | ANALYSIS : 62.5%, 3.0, 97.1ms, 0 | 20260505 18:12:21.308 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:21.308 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:21.308 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 18:12:21.308 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:21.308 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:21.308 | INFO | Uploaded 36GiB to GPU in 2.17693s (16.5371GiB/s) | 20260505 18:12:21.308 | INFO | Downloaded 34GiB to GPU in 2.14084s (15.8816GiB/s) | 20260505 18:12:26.452 | INFO | Analyzing chunk frames = [5265408,5265920) | 20260505 18:12:26.452 | INFO | MemInfo: { | 20260505 18:12:26.452 | INFO | "gpu" : " 17 GB", | 20260505 18:12:26.452 | INFO | "huge" : "", | 20260505 18:12:26.452 | INFO | "huge_shared" : " 27 GB", | 20260505 18:12:26.452 | INFO | "malloc" : " 14 GB", | 20260505 18:12:26.452 | INFO | "pinned" : " 29 GB" | 20260505 18:12:26.452 | INFO | } | 20260505 18:12:26.452 | INFO | Perf report for the last 5.14409 seconds: | 20260505 18:12:26.452 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:12:26.452 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:26.452 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:26.452 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:26.452 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 18:12:26.452 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:26.452 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:26.452 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:12:26.452 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:26.452 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:26.453 | INFO | Uploaded 36GiB to GPU in 2.14455s (16.7868GiB/s) | 20260505 18:12:26.453 | INFO | Downloaded 34GiB to GPU in 2.07092s (16.4178GiB/s) | 20260505 18:12:31.586 | INFO | Analyzing chunk frames = [5265920,5266432) | 20260505 18:12:31.586 | INFO | MemInfo: { | 20260505 18:12:31.586 | INFO | "gpu" : " 17 GB", | 20260505 18:12:31.586 | INFO | "huge" : "", | 20260505 18:12:31.586 | INFO | "huge_shared" : " 27 GB", | 20260505 18:12:31.586 | INFO | "malloc" : " 14 GB", | 20260505 18:12:31.586 | INFO | "pinned" : " 29 GB" | 20260505 18:12:31.586 | INFO | } | 20260505 18:12:31.586 | INFO | Perf report for the last 5.13384 seconds: | 20260505 18:12:31.586 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:12:31.586 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:31.586 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:31.586 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:31.586 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260505 18:12:31.586 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:31.586 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:31.586 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:12:31.586 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:31.586 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:31.586 | INFO | Uploaded 36GiB to GPU in 2.14257s (16.8023GiB/s) | 20260505 18:12:31.587 | INFO | Downloaded 34GiB to GPU in 2.0588s (16.5145GiB/s) | 20260505 18:12:36.721 | INFO | Analyzing chunk frames = [5266432,5266944) | 20260505 18:12:36.721 | INFO | MemInfo: { | 20260505 18:12:36.721 | INFO | "gpu" : " 17 GB", | 20260505 18:12:36.721 | INFO | "huge" : "", | 20260505 18:12:36.721 | INFO | "huge_shared" : " 27 GB", | 20260505 18:12:36.721 | INFO | "malloc" : " 14 GB", | 20260505 18:12:36.721 | INFO | "pinned" : " 29 GB" | 20260505 18:12:36.721 | INFO | } | 20260505 18:12:36.721 | INFO | Perf report for the last 5.13474 seconds: | 20260505 18:12:36.721 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:12:36.721 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:36.721 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:36.721 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:36.721 | INFO | ANALYSIS : 63.5%, 3.0, 99.0ms, 0 | 20260505 18:12:36.721 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:36.721 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:36.721 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 18:12:36.721 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:36.721 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:36.721 | INFO | Uploaded 36GiB to GPU in 2.14206s (16.8062GiB/s) | 20260505 18:12:36.721 | INFO | Downloaded 34GiB to GPU in 2.17911s (15.6027GiB/s) | 20260505 18:12:41.861 | INFO | Analyzing chunk frames = [5266944,5267456) | 20260505 18:12:41.861 | INFO | MemInfo: { | 20260505 18:12:41.861 | INFO | "gpu" : " 17 GB", | 20260505 18:12:41.861 | INFO | "huge" : "", | 20260505 18:12:41.861 | INFO | "huge_shared" : " 27 GB", | 20260505 18:12:41.861 | INFO | "malloc" : " 14 GB", | 20260505 18:12:41.861 | INFO | "pinned" : " 29 GB" | 20260505 18:12:41.861 | INFO | } | 20260505 18:12:41.861 | INFO | Perf report for the last 5.14013 seconds: | 20260505 18:12:41.861 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:12:41.861 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:41.861 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:41.861 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:41.861 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 18:12:41.861 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:41.861 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:41.861 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 18:12:41.861 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:41.861 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:41.861 | INFO | Uploaded 36GiB to GPU in 2.17485s (16.5529GiB/s) | 20260505 18:12:41.861 | INFO | Downloaded 34GiB to GPU in 2.03657s (16.6947GiB/s) | 20260505 18:12:41.861 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 18:12:41.861 | INFO | Total mem usage: 15.3 GiB | 20260505 18:12:41.861 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 18:12:41.861 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.01%) | 20260505 18:12:41.861 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 18:12:41.861 | INFO | High Water Breakdown: | 20260505 18:12:41.861 | INFO | 61% : 8.66 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 18:12:41.862 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 18:12:41.862 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 18:12:41.862 | INFO | ------------------------------------------ | 20260505 18:12:41.862 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:12:41.862 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 18:12:41.862 | INFO | Total mem usage: 26.5 GiB | 20260505 18:12:41.862 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 18:12:41.862 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 18:12:41.862 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 18:12:41.862 | INFO | High Water Breakdown: | 20260505 18:12:41.862 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:12:41.862 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:12:41.862 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:12:41.862 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:12:41.862 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:12:41.862 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:12:41.862 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 18:12:41.862 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 18:12:41.862 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:12:41.862 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 18:12:41.862 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 18:12:41.862 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 18:12:41.862 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 18:12:41.862 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 18:12:41.862 | INFO | ------------------------------------------ | 20260505 18:12:41.862 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:12:41.862 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 18:12:41.862 | INFO | Total mem usage: 25.6 GiB | 20260505 18:12:41.862 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 18:12:41.862 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 18:12:41.862 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 18:12:41.862 | INFO | High Water Breakdown: | 20260505 18:12:41.862 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 18:12:41.862 | INFO | ------------------------------------------ | 20260505 18:12:41.862 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:12:41.862 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 18:12:41.862 | INFO | Total mem usage: 15.7 GiB | 20260505 18:12:41.862 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 18:12:41.862 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 18:12:41.862 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 18:12:41.862 | INFO | High Water Breakdown: | 20260505 18:12:41.862 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 18:12:41.862 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 18:12:41.862 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 18:12:41.862 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 18:12:41.862 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 18:12:41.862 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 18:12:41.862 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 18:12:41.862 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 18:12:41.862 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 18:12:41.862 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:12:41.862 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 18:12:41.862 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:12:41.862 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 18:12:41.862 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 18:12:41.862 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 18:12:41.862 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 18:12:41.862 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:12:41.862 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 18:12:41.862 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:12:41.862 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:12:41.862 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:12:41.862 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 18:12:41.862 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 18:12:41.862 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:12:41.862 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 18:12:41.862 | INFO | ------------------------------------------ | 20260505 18:12:41.862 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:12:41.862 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 18:12:41.862 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:12:41.862 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:12:41.862 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.51e+01, 1.35e+01 / 2.14e+00 | 20260505 18:12:41.862 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.01e-01, 1.03e-02 / 3.81e-03 | 20260505 18:12:41.862 | INFO | PerfReport: 11.78%, Baseline -- 3.72e+00 / 4.53e+00, 5.70e-01 / 1.37e-01 | 20260505 18:12:41.862 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.54e+00, 1.91e-01 / 7.98e-02 | 20260505 18:12:41.862 | INFO | PerfReport: 15.99%, DME -- 4.04e+01 / 4.49e+01, 4.36e+00 / 2.10e+00 | 20260505 18:12:41.862 | INFO | PerfReport: 56.84%, FrameLabeling -- 1.79e+01 / 1.93e+01, 1.43e+00 / 1.25e+00 | 20260505 18:12:41.863 | INFO | PerfReport: 7.31%, PulseAccumulating -- 2.31e+00 / 2.89e+00, 4.45e-01 / 1.88e-01 | 20260505 18:12:41.863 | INFO | PerfReport: 3.35%, Metrics -- 1.06e+00 / 1.40e+00, 2.90e-01 / 1.55e-01 | 20260505 18:12:41.863 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.68e-01 / 4.29e-01, 6.52e-02 / 3.63e-02 | 20260505 18:12:41.863 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 18:12:41.863 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:12:41.863 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:12:41.863 | INFO | PerfReport: Full Execution -- 9.99e+01 / -nan, 2.57e+01 / 0.00e+00 | 20260505 18:12:41.863 | INFO | PerfReport: 22.55%, Upload -- 2.25e+01 / 0.00e+00, 7.87e+00 / 0.00e+00 | 20260505 18:12:41.863 | INFO | PerfReport: 21.83%, Download -- 2.18e+01 / 0.00e+00, 7.85e+00 / 0.00e+00 | 20260505 18:12:41.863 | INFO | PerfReport: 31.59%, Compute -- 3.16e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 18:12:41.863 | INFO | PerfReport: 2.32%, LockUpload -- 2.32e+00 / 0.00e+00, 7.90e+00 / 0.00e+00 | 20260505 18:12:41.863 | INFO | PerfReport: 1.41%, LockDownload -- 1.41e+00 / 0.00e+00, 2.92e+00 / 0.00e+00 | 20260505 18:12:41.863 | INFO | PerfReport: 20.29%, LockCompute -- 2.03e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 18:12:41.863 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.65e-02 / 0.00e+00 | 20260505 18:12:41.996 | INFO | Starting a round of Baz IO | 20260505 18:12:45.192 | INFO | Populating RTMetrics report | 20260505 18:12:45.192 | INFO | RTMetrics report populated | 20260505 18:12:45.192 | INFO | Opening tmp RTMetrics json file | 20260505 18:12:45.197 | INFO | Serializing RTMetrics to json | 20260505 18:12:45.198 | INFO | Serialization done, swapping tmp file now... | 20260505 18:12:45.199 | INFO | RTMetrics done | 20260505 18:12:45.288 | INFO | Finished a round of Baz IO: Wrote 2.101 GiB in 3.291 seconds | 20260505 18:12:45.288 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.94% / 0.00% / 78.06% | 20260505 18:12:45.288 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.262002% / 1.19209e-05% / 0.000107288% | 20260505 18:12:45.288 | INFO | Baz IO overall: | 20260505 18:12:45.288 | INFO | BytesWritten : 3104424141861 | 20260505 18:12:45.288 | INFO | NumEvents : 1470991857220 | 20260505 18:12:47.004 | INFO | Analyzing chunk frames = [5267456,5267968) | 20260505 18:12:47.004 | INFO | MemInfo: { | 20260505 18:12:47.004 | INFO | "gpu" : " 17 GB", | 20260505 18:12:47.004 | INFO | "huge" : "", | 20260505 18:12:47.004 | INFO | "huge_shared" : " 27 GB", | 20260505 18:12:47.004 | INFO | "malloc" : " 14 GB", | 20260505 18:12:47.005 | INFO | "pinned" : " 29 GB" | 20260505 18:12:47.005 | INFO | } | 20260505 18:12:47.005 | INFO | Perf report for the last 5.14365 seconds: | 20260505 18:12:47.005 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:12:47.005 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:47.005 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:47.005 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:47.005 | INFO | ANALYSIS : 63.7%, 3.0, 99.4ms, 0 | 20260505 18:12:47.005 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:47.005 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260505 18:12:47.005 | INFO | PRE_HQ : 60.6%, 1.8, 56.3ms, 0 | 20260505 18:12:47.005 | INFO | BAZWRITER : 63.3%, 3.8, 123.3ms, 0 | 20260505 18:12:47.005 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 18:12:47.005 | INFO | Uploaded 36GiB to GPU in 2.17073s (16.5842GiB/s) | 20260505 18:12:47.005 | INFO | Downloaded 38GiB to GPU in 2.26397s (16.7847GiB/s) | 20260505 18:12:52.153 | INFO | Analyzing chunk frames = [5267968,5268480) | 20260505 18:12:52.153 | INFO | MemInfo: { | 20260505 18:12:52.153 | INFO | "gpu" : " 17 GB", | 20260505 18:12:52.153 | INFO | "huge" : "", | 20260505 18:12:52.153 | INFO | "huge_shared" : " 27 GB", | 20260505 18:12:52.153 | INFO | "malloc" : " 14 GB", | 20260505 18:12:52.153 | INFO | "pinned" : " 29 GB" | 20260505 18:12:52.153 | INFO | } | 20260505 18:12:52.153 | INFO | Perf report for the last 5.14852 seconds: | 20260505 18:12:52.153 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:12:52.153 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:52.153 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:52.153 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:52.153 | INFO | ANALYSIS : 62.6%, 3.0, 98.0ms, 0 | 20260505 18:12:52.153 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:52.153 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:52.153 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:12:52.153 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:52.153 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:52.154 | INFO | Uploaded 36GiB to GPU in 2.14035s (16.8197GiB/s) | 20260505 18:12:52.154 | INFO | Downloaded 34GiB to GPU in 2.0325s (16.7282GiB/s) | 20260505 18:12:57.283 | INFO | Analyzing chunk frames = [5268480,5268992) | 20260505 18:12:57.283 | INFO | MemInfo: { | 20260505 18:12:57.283 | INFO | "gpu" : " 17 GB", | 20260505 18:12:57.283 | INFO | "huge" : "", | 20260505 18:12:57.283 | INFO | "huge_shared" : " 27 GB", | 20260505 18:12:57.283 | INFO | "malloc" : " 14 GB", | 20260505 18:12:57.283 | INFO | "pinned" : " 29 GB" | 20260505 18:12:57.283 | INFO | } | 20260505 18:12:57.283 | INFO | Perf report for the last 5.12957 seconds: | 20260505 18:12:57.283 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:12:57.283 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:57.283 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:57.283 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:57.283 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260505 18:12:57.283 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:57.283 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:57.283 | INFO | PRE_HQ : 27.7%, 1.0, 14.3ms, 0 | 20260505 18:12:57.283 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:57.283 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:12:57.283 | INFO | Uploaded 36GiB to GPU in 2.11221s (17.0438GiB/s) | 20260505 18:12:57.283 | INFO | Downloaded 34GiB to GPU in 2.02315s (16.8055GiB/s) | 20260505 18:13:02.418 | INFO | Analyzing chunk frames = [5268992,5269504) | 20260505 18:13:02.418 | INFO | MemInfo: { | 20260505 18:13:02.418 | INFO | "gpu" : " 17 GB", | 20260505 18:13:02.418 | INFO | "huge" : "", | 20260505 18:13:02.418 | INFO | "huge_shared" : " 27 GB", | 20260505 18:13:02.418 | INFO | "malloc" : " 14 GB", | 20260505 18:13:02.418 | INFO | "pinned" : " 29 GB" | 20260505 18:13:02.418 | INFO | } | 20260505 18:13:02.418 | INFO | Perf report for the last 5.13513 seconds: | 20260505 18:13:02.418 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:13:02.418 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:02.418 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:02.418 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:02.418 | INFO | ANALYSIS : 62.9%, 3.0, 97.6ms, 0 | 20260505 18:13:02.418 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:02.418 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:02.418 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 18:13:02.418 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:02.418 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:02.418 | INFO | Uploaded 36GiB to GPU in 2.22512s (16.1789GiB/s) | 20260505 18:13:02.418 | INFO | Downloaded 34GiB to GPU in 2.12181s (16.0241GiB/s) | 20260505 18:13:07.558 | INFO | Analyzing chunk frames = [5269504,5270016) | 20260505 18:13:07.558 | INFO | MemInfo: { | 20260505 18:13:07.558 | INFO | "gpu" : " 17 GB", | 20260505 18:13:07.558 | INFO | "huge" : "", | 20260505 18:13:07.558 | INFO | "huge_shared" : " 27 GB", | 20260505 18:13:07.558 | INFO | "malloc" : " 14 GB", | 20260505 18:13:07.558 | INFO | "pinned" : " 29 GB" | 20260505 18:13:07.558 | INFO | } | 20260505 18:13:07.558 | INFO | Perf report for the last 5.14034 seconds: | 20260505 18:13:07.558 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:13:07.558 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:07.558 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:07.558 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:07.558 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 18:13:07.558 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:07.558 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:07.558 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 18:13:07.558 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:07.558 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:07.559 | INFO | Uploaded 36GiB to GPU in 2.14967s (16.7468GiB/s) | 20260505 18:13:07.559 | INFO | Downloaded 34GiB to GPU in 2.06039s (16.5017GiB/s) | 20260505 18:13:12.701 | INFO | Analyzing chunk frames = [5270016,5270528) | 20260505 18:13:12.701 | INFO | MemInfo: { | 20260505 18:13:12.701 | INFO | "gpu" : " 17 GB", | 20260505 18:13:12.701 | INFO | "huge" : "", | 20260505 18:13:12.702 | INFO | "huge_shared" : " 27 GB", | 20260505 18:13:12.702 | INFO | "malloc" : " 14 GB", | 20260505 18:13:12.702 | INFO | "pinned" : " 29 GB" | 20260505 18:13:12.702 | INFO | } | 20260505 18:13:12.702 | INFO | Perf report for the last 5.14335 seconds: | 20260505 18:13:12.702 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:13:12.702 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:12.702 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:12.702 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:12.702 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260505 18:13:12.702 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:12.702 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:12.702 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260505 18:13:12.702 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:12.702 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:12.702 | INFO | Uploaded 36GiB to GPU in 2.13405s (16.8693GiB/s) | 20260505 18:13:12.702 | INFO | Downloaded 34GiB to GPU in 2.04654s (16.6134GiB/s) | 20260505 18:13:17.850 | INFO | Analyzing chunk frames = [5270528,5271040) | 20260505 18:13:17.850 | INFO | MemInfo: { | 20260505 18:13:17.850 | INFO | "gpu" : " 17 GB", | 20260505 18:13:17.850 | INFO | "huge" : "", | 20260505 18:13:17.850 | INFO | "huge_shared" : " 27 GB", | 20260505 18:13:17.850 | INFO | "malloc" : " 14 GB", | 20260505 18:13:17.850 | INFO | "pinned" : " 29 GB" | 20260505 18:13:17.850 | INFO | } | 20260505 18:13:17.850 | INFO | Perf report for the last 5.14814 seconds: | 20260505 18:13:17.850 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:13:17.850 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:17.850 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:17.850 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:17.850 | INFO | ANALYSIS : 63.1%, 3.0, 98.7ms, 0 | 20260505 18:13:17.850 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:17.850 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:17.850 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 18:13:17.850 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:17.850 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:17.850 | INFO | Uploaded 36GiB to GPU in 2.14253s (16.8026GiB/s) | 20260505 18:13:17.850 | INFO | Downloaded 34GiB to GPU in 2.1521s (15.7985GiB/s) | 20260505 18:13:22.978 | INFO | Analyzing chunk frames = [5271040,5271552) | 20260505 18:13:22.978 | INFO | MemInfo: { | 20260505 18:13:22.978 | INFO | "gpu" : " 17 GB", | 20260505 18:13:22.978 | INFO | "huge" : "", | 20260505 18:13:22.978 | INFO | "huge_shared" : " 27 GB", | 20260505 18:13:22.978 | INFO | "malloc" : " 14 GB", | 20260505 18:13:22.978 | INFO | "pinned" : " 29 GB" | 20260505 18:13:22.978 | INFO | } | 20260505 18:13:22.978 | INFO | Perf report for the last 5.12842 seconds: | 20260505 18:13:22.978 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:13:22.978 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:22.978 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:22.978 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:22.978 | INFO | ANALYSIS : 63.0%, 3.0, 98.1ms, 0 | 20260505 18:13:22.978 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:22.978 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:22.978 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 18:13:22.978 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:22.978 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:22.979 | INFO | Uploaded 36GiB to GPU in 2.19864s (16.3738GiB/s) | 20260505 18:13:22.979 | INFO | Downloaded 34GiB to GPU in 2.03925s (16.6728GiB/s) | 20260505 18:13:23.114 | INFO | Starting a round of Baz IO | 20260505 18:13:26.293 | INFO | Populating RTMetrics report | 20260505 18:13:26.293 | INFO | RTMetrics report populated | 20260505 18:13:26.293 | INFO | Opening tmp RTMetrics json file | 20260505 18:13:26.297 | INFO | Serializing RTMetrics to json | 20260505 18:13:26.298 | INFO | Serialization done, swapping tmp file now... | 20260505 18:13:26.299 | INFO | RTMetrics done | 20260505 18:13:26.385 | INFO | Finished a round of Baz IO: Wrote 2.1 GiB in 3.27 seconds | 20260505 18:13:26.385 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.94% / 0.00% / 78.06% | 20260505 18:13:26.385 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.261013% / 7.94729e-06% / 0.00013113% | 20260505 18:13:26.385 | INFO | Baz IO overall: | 20260505 18:13:26.385 | INFO | BytesWritten : 3106678619323 | 20260505 18:13:26.385 | INFO | NumEvents : 1472043360069 | 20260505 18:13:28.118 | INFO | Analyzing chunk frames = [5271552,5272064) | 20260505 18:13:28.118 | INFO | MemInfo: { | 20260505 18:13:28.118 | INFO | "gpu" : " 17 GB", | 20260505 18:13:28.118 | INFO | "huge" : "", | 20260505 18:13:28.118 | INFO | "huge_shared" : " 27 GB", | 20260505 18:13:28.118 | INFO | "malloc" : " 14 GB", | 20260505 18:13:28.118 | INFO | "pinned" : " 29 GB" | 20260505 18:13:28.118 | INFO | } | 20260505 18:13:28.118 | INFO | Perf report for the last 5.13966 seconds: | 20260505 18:13:28.118 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:13:28.118 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:28.118 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:28.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:28.118 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 18:13:28.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:28.118 | INFO | RT_METRICS : 44.3%, 1.0, 23.3ms, 0 | 20260505 18:13:28.118 | INFO | PRE_HQ : 60.2%, 1.8, 55.6ms, 0 | 20260505 18:13:28.118 | INFO | BAZWRITER : 63.1%, 3.8, 122.5ms, 0 | 20260505 18:13:28.118 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 18:13:28.118 | INFO | Uploaded 36GiB to GPU in 2.16306s (16.6431GiB/s) | 20260505 18:13:28.118 | INFO | Downloaded 38GiB to GPU in 2.24707s (16.9109GiB/s) | 20260505 18:13:33.263 | INFO | Analyzing chunk frames = [5272064,5272576) | 20260505 18:13:33.263 | INFO | MemInfo: { | 20260505 18:13:33.263 | INFO | "gpu" : " 17 GB", | 20260505 18:13:33.263 | INFO | "huge" : "", | 20260505 18:13:33.263 | INFO | "huge_shared" : " 27 GB", | 20260505 18:13:33.263 | INFO | "malloc" : " 14 GB", | 20260505 18:13:33.263 | INFO | "pinned" : " 29 GB" | 20260505 18:13:33.263 | INFO | } | 20260505 18:13:33.263 | INFO | Perf report for the last 5.14488 seconds: | 20260505 18:13:33.263 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:13:33.263 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:33.263 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:33.263 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:33.263 | INFO | ANALYSIS : 62.7%, 3.0, 98.0ms, 0 | 20260505 18:13:33.263 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:33.263 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:33.263 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:13:33.263 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:33.263 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:33.263 | INFO | Uploaded 36GiB to GPU in 2.1423s (16.8043GiB/s) | 20260505 18:13:33.263 | INFO | Downloaded 34GiB to GPU in 2.02962s (16.7519GiB/s) | 20260505 18:13:38.399 | INFO | Analyzing chunk frames = [5272576,5273088) | 20260505 18:13:38.399 | INFO | MemInfo: { | 20260505 18:13:38.399 | INFO | "gpu" : " 17 GB", | 20260505 18:13:38.399 | INFO | "huge" : "", | 20260505 18:13:38.399 | INFO | "huge_shared" : " 27 GB", | 20260505 18:13:38.399 | INFO | "malloc" : " 14 GB", | 20260505 18:13:38.399 | INFO | "pinned" : " 29 GB" | 20260505 18:13:38.399 | INFO | } | 20260505 18:13:38.399 | INFO | Perf report for the last 5.13669 seconds: | 20260505 18:13:38.400 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:13:38.400 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:38.400 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:38.400 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:38.400 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 18:13:38.400 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:38.400 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:38.400 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 18:13:38.400 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:38.400 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:38.400 | INFO | Uploaded 36GiB to GPU in 2.21035s (16.287GiB/s) | 20260505 18:13:38.400 | INFO | Downloaded 34GiB to GPU in 2.07401s (16.3934GiB/s) | 20260505 18:13:43.536 | INFO | Analyzing chunk frames = [5273088,5273600) | 20260505 18:13:43.536 | INFO | MemInfo: { | 20260505 18:13:43.536 | INFO | "gpu" : " 17 GB", | 20260505 18:13:43.536 | INFO | "huge" : "", | 20260505 18:13:43.536 | INFO | "huge_shared" : " 27 GB", | 20260505 18:13:43.536 | INFO | "malloc" : " 14 GB", | 20260505 18:13:43.536 | INFO | "pinned" : " 29 GB" | 20260505 18:13:43.536 | INFO | } | 20260505 18:13:43.536 | INFO | Perf report for the last 5.13675 seconds: | 20260505 18:13:43.536 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:13:43.536 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:43.536 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:43.536 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:43.536 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260505 18:13:43.536 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:43.536 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:43.536 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:13:43.537 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:43.537 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:43.537 | INFO | Uploaded 36GiB to GPU in 2.18101s (16.5061GiB/s) | 20260505 18:13:43.537 | INFO | Downloaded 34GiB to GPU in 2.05871s (16.5152GiB/s) | 20260505 18:13:48.673 | INFO | Analyzing chunk frames = [5273600,5274112) | 20260505 18:13:48.673 | INFO | MemInfo: { | 20260505 18:13:48.673 | INFO | "gpu" : " 17 GB", | 20260505 18:13:48.673 | INFO | "huge" : "", | 20260505 18:13:48.673 | INFO | "huge_shared" : " 27 GB", | 20260505 18:13:48.673 | INFO | "malloc" : " 14 GB", | 20260505 18:13:48.673 | INFO | "pinned" : " 29 GB" | 20260505 18:13:48.673 | INFO | } | 20260505 18:13:48.673 | INFO | Perf report for the last 5.1367 seconds: | 20260505 18:13:48.673 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:13:48.673 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:48.673 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:48.673 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:48.673 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 18:13:48.673 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:48.673 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:48.673 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 18:13:48.673 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:48.673 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:48.673 | INFO | Uploaded 36GiB to GPU in 2.14329s (16.7966GiB/s) | 20260505 18:13:48.673 | INFO | Downloaded 34GiB to GPU in 2.05849s (16.5169GiB/s) | 20260505 18:13:53.814 | INFO | Analyzing chunk frames = [5274112,5274624) | 20260505 18:13:53.814 | INFO | MemInfo: { | 20260505 18:13:53.814 | INFO | "gpu" : " 17 GB", | 20260505 18:13:53.814 | INFO | "huge" : "", | 20260505 18:13:53.814 | INFO | "huge_shared" : " 27 GB", | 20260505 18:13:53.814 | INFO | "malloc" : " 14 GB", | 20260505 18:13:53.814 | INFO | "pinned" : " 29 GB" | 20260505 18:13:53.814 | INFO | } | 20260505 18:13:53.815 | INFO | Perf report for the last 5.14157 seconds: | 20260505 18:13:53.815 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:13:53.815 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:53.815 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:53.815 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:53.815 | INFO | ANALYSIS : 62.3%, 3.0, 97.4ms, 0 | 20260505 18:13:53.815 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:53.815 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:53.815 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 18:13:53.815 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:53.815 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:53.815 | INFO | Uploaded 36GiB to GPU in 2.13796s (16.8385GiB/s) | 20260505 18:13:53.815 | INFO | Downloaded 34GiB to GPU in 2.03837s (16.68GiB/s) | 20260505 18:13:58.957 | INFO | Analyzing chunk frames = [5274624,5275136) | 20260505 18:13:58.957 | INFO | MemInfo: { | 20260505 18:13:58.957 | INFO | "gpu" : " 17 GB", | 20260505 18:13:58.957 | INFO | "huge" : "", | 20260505 18:13:58.957 | INFO | "huge_shared" : " 27 GB", | 20260505 18:13:58.957 | INFO | "malloc" : " 14 GB", | 20260505 18:13:58.958 | INFO | "pinned" : " 29 GB" | 20260505 18:13:58.958 | INFO | } | 20260505 18:13:58.958 | INFO | Perf report for the last 5.14299 seconds: | 20260505 18:13:58.958 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:13:58.958 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:58.958 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:58.958 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:58.958 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 18:13:58.958 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:58.958 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:58.958 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 18:13:58.958 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:58.958 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:13:58.958 | INFO | Uploaded 36GiB to GPU in 2.15211s (16.7278GiB/s) | 20260505 18:13:58.958 | INFO | Downloaded 34GiB to GPU in 2.15829s (15.7532GiB/s) | 20260505 18:14:04.103 | INFO | Analyzing chunk frames = [5275136,5275648) | 20260505 18:14:04.103 | INFO | MemInfo: { | 20260505 18:14:04.103 | INFO | "gpu" : " 17 GB", | 20260505 18:14:04.103 | INFO | "huge" : "", | 20260505 18:14:04.103 | INFO | "huge_shared" : " 27 GB", | 20260505 18:14:04.103 | INFO | "malloc" : " 14 GB", | 20260505 18:14:04.103 | INFO | "pinned" : " 29 GB" | 20260505 18:14:04.103 | INFO | } | 20260505 18:14:04.103 | INFO | Perf report for the last 5.14586 seconds: | 20260505 18:14:04.103 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:14:04.103 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:04.103 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:04.103 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:04.103 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 18:14:04.103 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:04.103 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:04.103 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 18:14:04.103 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:04.103 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:04.104 | INFO | Uploaded 36GiB to GPU in 2.19709s (16.3853GiB/s) | 20260505 18:14:04.104 | INFO | Downloaded 34GiB to GPU in 2.03949s (16.6709GiB/s) | 20260505 18:14:04.104 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 18:14:04.104 | INFO | Total mem usage: 15.3 GiB | 20260505 18:14:04.104 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 18:14:04.104 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.04%) | 20260505 18:14:04.104 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 18:14:04.104 | INFO | High Water Breakdown: | 20260505 18:14:04.104 | INFO | 61% : 8.66 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 18:14:04.104 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 18:14:04.104 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 18:14:04.104 | INFO | ------------------------------------------ | 20260505 18:14:04.104 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:14:04.104 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 18:14:04.104 | INFO | Total mem usage: 26.5 GiB | 20260505 18:14:04.104 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 18:14:04.104 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 18:14:04.104 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 18:14:04.104 | INFO | High Water Breakdown: | 20260505 18:14:04.104 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:14:04.104 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:14:04.104 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:14:04.104 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:14:04.104 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:14:04.104 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:14:04.104 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 18:14:04.104 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 18:14:04.105 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:14:04.105 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 18:14:04.105 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 18:14:04.105 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 18:14:04.105 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 18:14:04.105 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 18:14:04.105 | INFO | ------------------------------------------ | 20260505 18:14:04.105 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:14:04.105 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 18:14:04.105 | INFO | Total mem usage: 25.6 GiB | 20260505 18:14:04.105 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 18:14:04.105 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 18:14:04.105 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 18:14:04.105 | INFO | High Water Breakdown: | 20260505 18:14:04.105 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 18:14:04.105 | INFO | ------------------------------------------ | 20260505 18:14:04.105 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:14:04.105 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 18:14:04.105 | INFO | Total mem usage: 15.7 GiB | 20260505 18:14:04.105 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 18:14:04.105 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 18:14:04.105 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 18:14:04.105 | INFO | High Water Breakdown: | 20260505 18:14:04.105 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 18:14:04.105 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 18:14:04.105 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 18:14:04.105 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 18:14:04.105 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 18:14:04.105 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 18:14:04.105 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 18:14:04.105 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 18:14:04.105 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 18:14:04.105 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:14:04.105 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 18:14:04.105 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:14:04.105 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 18:14:04.105 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 18:14:04.105 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 18:14:04.105 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 18:14:04.105 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:14:04.105 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 18:14:04.105 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:14:04.105 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:14:04.105 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:14:04.105 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 18:14:04.105 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 18:14:04.105 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:14:04.105 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 18:14:04.105 | INFO | ------------------------------------------ | 20260505 18:14:04.105 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:14:04.105 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 18:14:04.105 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:14:04.105 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:14:04.105 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.47e+01, 1.35e+01 / 1.90e+00 | 20260505 18:14:04.105 | INFO | PerfReport: 0.29%, AggregateStats -- 9.24e-02 / 1.04e-01, 1.03e-02 / 6.95e-03 | 20260505 18:14:04.105 | INFO | PerfReport: 11.79%, Baseline -- 3.72e+00 / 4.53e+00, 5.65e-01 / 1.49e-01 | 20260505 18:14:04.105 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.55e+00, 1.90e-01 / 6.84e-02 | 20260505 18:14:04.105 | INFO | PerfReport: 15.95%, DME -- 4.02e+01 / 4.45e+01, 3.78e+00 / 1.93e+00 | 20260505 18:14:04.105 | INFO | PerfReport: 56.86%, FrameLabeling -- 1.79e+01 / 1.92e+01, 1.42e+00 / 9.52e-01 | 20260505 18:14:04.105 | INFO | PerfReport: 7.33%, PulseAccumulating -- 2.31e+00 / 2.90e+00, 4.45e-01 / 1.75e-01 | 20260505 18:14:04.105 | INFO | PerfReport: 3.34%, Metrics -- 1.05e+00 / 1.40e+00, 2.89e-01 / 1.50e-01 | 20260505 18:14:04.105 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.67e-01 / 4.30e-01, 6.32e-02 / 3.87e-02 | 20260505 18:14:04.105 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 18:14:04.105 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:14:04.105 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:14:04.105 | INFO | PerfReport: Full Execution -- 9.99e+01 / -nan, 2.59e+01 / 0.00e+00 | 20260505 18:14:04.105 | INFO | PerfReport: 22.55%, Upload -- 2.25e+01 / 0.00e+00, 7.85e+00 / 0.00e+00 | 20260505 18:14:04.105 | INFO | PerfReport: 21.80%, Download -- 2.18e+01 / 0.00e+00, 7.76e+00 / 0.00e+00 | 20260505 18:14:04.105 | INFO | PerfReport: 31.57%, Compute -- 3.15e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 18:14:04.105 | INFO | PerfReport: 2.36%, LockUpload -- 2.36e+00 / 0.00e+00, 8.12e+00 / 0.00e+00 | 20260505 18:14:04.105 | INFO | PerfReport: 1.37%, LockDownload -- 1.37e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260505 18:14:04.105 | INFO | PerfReport: 20.33%, LockCompute -- 2.03e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 18:14:04.105 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.97e-02 / 0.00e+00 | 20260505 18:14:04.237 | INFO | Starting a round of Baz IO | 20260505 18:14:07.420 | INFO | Populating RTMetrics report | 20260505 18:14:07.421 | INFO | RTMetrics report populated | 20260505 18:14:07.421 | INFO | Opening tmp RTMetrics json file | 20260505 18:14:07.425 | INFO | Serializing RTMetrics to json | 20260505 18:14:07.426 | INFO | Serialization done, swapping tmp file now... | 20260505 18:14:07.427 | INFO | RTMetrics done | 20260505 18:14:07.516 | INFO | Finished a round of Baz IO: Wrote 2.097 GiB in 3.278 seconds | 20260505 18:14:07.516 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.94% / 0.00% / 78.06% | 20260505 18:14:07.516 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.259729% / 0% / 0.000115236% | 20260505 18:14:07.516 | INFO | Baz IO overall: | 20260505 18:14:07.516 | INFO | BytesWritten : 3108930315270 | 20260505 18:14:07.516 | INFO | NumEvents : 1473092655100 | 20260505 18:14:09.232 | INFO | Analyzing chunk frames = [5275648,5276160) | 20260505 18:14:09.232 | INFO | MemInfo: { | 20260505 18:14:09.232 | INFO | "gpu" : " 17 GB", | 20260505 18:14:09.232 | INFO | "huge" : "", | 20260505 18:14:09.232 | INFO | "huge_shared" : " 27 GB", | 20260505 18:14:09.232 | INFO | "malloc" : " 14 GB", | 20260505 18:14:09.232 | INFO | "pinned" : " 29 GB" | 20260505 18:14:09.232 | INFO | } | 20260505 18:14:09.232 | INFO | Perf report for the last 5.1284 seconds: | 20260505 18:14:09.232 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:14:09.232 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:09.232 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:09.232 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:09.232 | INFO | ANALYSIS : 63.6%, 3.0, 98.9ms, 0 | 20260505 18:14:09.232 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:09.232 | INFO | RT_METRICS : 44.7%, 1.0, 23.4ms, 0 | 20260505 18:14:09.232 | INFO | PRE_HQ : 60.6%, 1.8, 56.2ms, 0 | 20260505 18:14:09.232 | INFO | BAZWRITER : 63.4%, 3.8, 123.7ms, 0 | 20260505 18:14:09.232 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 18:14:09.232 | INFO | Uploaded 36GiB to GPU in 2.17347s (16.5633GiB/s) | 20260505 18:14:09.232 | INFO | Downloaded 38GiB to GPU in 2.24826s (16.9019GiB/s) | 20260505 18:14:14.367 | INFO | Analyzing chunk frames = [5276160,5276672) | 20260505 18:14:14.367 | INFO | MemInfo: { | 20260505 18:14:14.367 | INFO | "gpu" : " 17 GB", | 20260505 18:14:14.367 | INFO | "huge" : "", | 20260505 18:14:14.367 | INFO | "huge_shared" : " 27 GB", | 20260505 18:14:14.367 | INFO | "malloc" : " 14 GB", | 20260505 18:14:14.367 | INFO | "pinned" : " 29 GB" | 20260505 18:14:14.367 | INFO | } | 20260505 18:14:14.367 | INFO | Perf report for the last 5.13508 seconds: | 20260505 18:14:14.367 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:14:14.367 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:14.367 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:14.367 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:14.367 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 18:14:14.367 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:14.367 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:14.367 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:14:14.367 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:14.367 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:14.367 | INFO | Uploaded 36GiB to GPU in 2.13011s (16.9006GiB/s) | 20260505 18:14:14.367 | INFO | Downloaded 34GiB to GPU in 2.03249s (16.7282GiB/s) | 20260505 18:14:19.516 | INFO | Analyzing chunk frames = [5276672,5277184) | 20260505 18:14:19.516 | INFO | MemInfo: { | 20260505 18:14:19.516 | INFO | "gpu" : " 17 GB", | 20260505 18:14:19.516 | INFO | "huge" : "", | 20260505 18:14:19.516 | INFO | "huge_shared" : " 27 GB", | 20260505 18:14:19.516 | INFO | "malloc" : " 14 GB", | 20260505 18:14:19.516 | INFO | "pinned" : " 29 GB" | 20260505 18:14:19.516 | INFO | } | 20260505 18:14:19.516 | INFO | Perf report for the last 5.14906 seconds: | 20260505 18:14:19.516 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:14:19.516 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:19.516 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:19.516 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:19.516 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260505 18:14:19.516 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:19.516 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:19.516 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:14:19.516 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:19.516 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:19.516 | INFO | Uploaded 36GiB to GPU in 2.19534s (16.3983GiB/s) | 20260505 18:14:19.516 | INFO | Downloaded 34GiB to GPU in 2.0394s (16.6716GiB/s) | 20260505 18:14:24.656 | INFO | Analyzing chunk frames = [5277184,5277696) | 20260505 18:14:24.656 | INFO | MemInfo: { | 20260505 18:14:24.656 | INFO | "gpu" : " 17 GB", | 20260505 18:14:24.656 | INFO | "huge" : "", | 20260505 18:14:24.656 | INFO | "huge_shared" : " 27 GB", | 20260505 18:14:24.656 | INFO | "malloc" : " 14 GB", | 20260505 18:14:24.656 | INFO | "pinned" : " 29 GB" | 20260505 18:14:24.656 | INFO | } | 20260505 18:14:24.657 | INFO | Perf report for the last 5.14049 seconds: | 20260505 18:14:24.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:14:24.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:24.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:24.657 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:24.657 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260505 18:14:24.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:24.657 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:24.657 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:14:24.657 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:24.657 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:24.657 | INFO | Uploaded 36GiB to GPU in 2.1853s (16.4737GiB/s) | 20260505 18:14:24.657 | INFO | Downloaded 34GiB to GPU in 2.04945s (16.5898GiB/s) | 20260505 18:14:29.796 | INFO | Analyzing chunk frames = [5277696,5278208) | 20260505 18:14:29.796 | INFO | MemInfo: { | 20260505 18:14:29.796 | INFO | "gpu" : " 17 GB", | 20260505 18:14:29.796 | INFO | "huge" : "", | 20260505 18:14:29.796 | INFO | "huge_shared" : " 27 GB", | 20260505 18:14:29.796 | INFO | "malloc" : " 14 GB", | 20260505 18:14:29.796 | INFO | "pinned" : " 29 GB" | 20260505 18:14:29.796 | INFO | } | 20260505 18:14:29.796 | INFO | Perf report for the last 5.13983 seconds: | 20260505 18:14:29.796 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:14:29.796 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:29.796 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:29.796 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:29.796 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 18:14:29.796 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:29.796 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:29.796 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:14:29.796 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:29.796 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:29.797 | INFO | Uploaded 36GiB to GPU in 2.13702s (16.8459GiB/s) | 20260505 18:14:29.797 | INFO | Downloaded 34GiB to GPU in 2.0563s (16.5346GiB/s) | 20260505 18:14:34.929 | INFO | Analyzing chunk frames = [5278208,5278720) | 20260505 18:14:34.929 | INFO | MemInfo: { | 20260505 18:14:34.929 | INFO | "gpu" : " 17 GB", | 20260505 18:14:34.929 | INFO | "huge" : "", | 20260505 18:14:34.929 | INFO | "huge_shared" : " 27 GB", | 20260505 18:14:34.929 | INFO | "malloc" : " 14 GB", | 20260505 18:14:34.929 | INFO | "pinned" : " 29 GB" | 20260505 18:14:34.929 | INFO | } | 20260505 18:14:34.929 | INFO | Perf report for the last 5.13246 seconds: | 20260505 18:14:34.929 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:14:34.929 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:34.929 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:34.929 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:34.929 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 18:14:34.929 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:34.929 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:34.929 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 18:14:34.929 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:34.929 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:34.929 | INFO | Uploaded 36GiB to GPU in 2.14419s (16.7896GiB/s) | 20260505 18:14:34.929 | INFO | Downloaded 34GiB to GPU in 2.05733s (16.5263GiB/s) | 20260505 18:14:40.071 | INFO | Analyzing chunk frames = [5278720,5279232) | 20260505 18:14:40.071 | INFO | MemInfo: { | 20260505 18:14:40.071 | INFO | "gpu" : " 17 GB", | 20260505 18:14:40.071 | INFO | "huge" : "", | 20260505 18:14:40.071 | INFO | "huge_shared" : " 27 GB", | 20260505 18:14:40.071 | INFO | "malloc" : " 14 GB", | 20260505 18:14:40.072 | INFO | "pinned" : " 29 GB" | 20260505 18:14:40.072 | INFO | } | 20260505 18:14:40.072 | INFO | Perf report for the last 5.14278 seconds: | 20260505 18:14:40.072 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:14:40.072 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:40.072 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:40.072 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:40.072 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 18:14:40.072 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:40.072 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:40.072 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 18:14:40.072 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:40.072 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:40.072 | INFO | Uploaded 36GiB to GPU in 2.16929s (16.5953GiB/s) | 20260505 18:14:40.072 | INFO | Downloaded 34GiB to GPU in 2.17816s (15.6095GiB/s) | 20260505 18:14:45.211 | INFO | Analyzing chunk frames = [5279232,5279744) | 20260505 18:14:45.211 | INFO | MemInfo: { | 20260505 18:14:45.211 | INFO | "gpu" : " 17 GB", | 20260505 18:14:45.211 | INFO | "huge" : "", | 20260505 18:14:45.211 | INFO | "huge_shared" : " 27 GB", | 20260505 18:14:45.211 | INFO | "malloc" : " 14 GB", | 20260505 18:14:45.211 | INFO | "pinned" : " 29 GB" | 20260505 18:14:45.211 | INFO | } | 20260505 18:14:45.211 | INFO | Perf report for the last 5.13965 seconds: | 20260505 18:14:45.211 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:14:45.211 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:45.211 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:45.211 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:45.211 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 18:14:45.211 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:45.211 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:45.211 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:14:45.211 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:45.211 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:45.212 | INFO | Uploaded 36GiB to GPU in 2.19972s (16.3657GiB/s) | 20260505 18:14:45.212 | INFO | Downloaded 34GiB to GPU in 2.04061s (16.6617GiB/s) | 20260505 18:14:45.345 | INFO | Starting a round of Baz IO | 20260505 18:14:48.525 | INFO | Populating RTMetrics report | 20260505 18:14:48.525 | INFO | RTMetrics report populated | 20260505 18:14:48.525 | INFO | Opening tmp RTMetrics json file | 20260505 18:14:48.529 | INFO | Serializing RTMetrics to json | 20260505 18:14:48.532 | INFO | Serialization done, swapping tmp file now... | 20260505 18:14:48.533 | INFO | RTMetrics done | 20260505 18:14:48.618 | INFO | Finished a round of Baz IO: Wrote 2.098 GiB in 3.273 seconds | 20260505 18:14:48.618 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.94% / 0.00% / 78.06% | 20260505 18:14:48.618 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.258744% / 0% / 0.000123183% | 20260505 18:14:48.618 | INFO | Baz IO overall: | 20260505 18:14:48.618 | INFO | BytesWritten : 3111183260035 | 20260505 18:14:48.618 | INFO | NumEvents : 1474143442545 | 20260505 18:14:50.349 | INFO | Analyzing chunk frames = [5279744,5280256) | 20260505 18:14:50.349 | INFO | MemInfo: { | 20260505 18:14:50.349 | INFO | "gpu" : " 17 GB", | 20260505 18:14:50.349 | INFO | "huge" : "", | 20260505 18:14:50.349 | INFO | "huge_shared" : " 27 GB", | 20260505 18:14:50.349 | INFO | "malloc" : " 14 GB", | 20260505 18:14:50.349 | INFO | "pinned" : " 29 GB" | 20260505 18:14:50.349 | INFO | } | 20260505 18:14:50.349 | INFO | Perf report for the last 5.13789 seconds: | 20260505 18:14:50.349 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:14:50.349 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:50.349 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:50.349 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:50.349 | INFO | ANALYSIS : 63.4%, 3.0, 98.8ms, 0 | 20260505 18:14:50.349 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:50.349 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260505 18:14:50.349 | INFO | PRE_HQ : 60.4%, 1.8, 56.0ms, 0 | 20260505 18:14:50.349 | INFO | BAZWRITER : 63.2%, 3.8, 122.8ms, 0 | 20260505 18:14:50.349 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 18:14:50.350 | INFO | Uploaded 36GiB to GPU in 2.16237s (16.6484GiB/s) | 20260505 18:14:50.350 | INFO | Downloaded 38GiB to GPU in 2.25404s (16.8586GiB/s) | 20260505 18:14:55.483 | INFO | Analyzing chunk frames = [5280256,5280768) | 20260505 18:14:55.483 | INFO | MemInfo: { | 20260505 18:14:55.483 | INFO | "gpu" : " 17 GB", | 20260505 18:14:55.483 | INFO | "huge" : "", | 20260505 18:14:55.483 | INFO | "huge_shared" : " 27 GB", | 20260505 18:14:55.483 | INFO | "malloc" : " 14 GB", | 20260505 18:14:55.483 | INFO | "pinned" : " 29 GB" | 20260505 18:14:55.483 | INFO | } | 20260505 18:14:55.483 | INFO | Perf report for the last 5.1341 seconds: | 20260505 18:14:55.483 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:14:55.483 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:55.483 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:55.483 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:55.483 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260505 18:14:55.483 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:55.483 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:55.483 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:14:55.483 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:55.483 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:14:55.484 | INFO | Uploaded 36GiB to GPU in 2.13418s (16.8683GiB/s) | 20260505 18:14:55.484 | INFO | Downloaded 34GiB to GPU in 2.0302s (16.7471GiB/s) | 20260505 18:15:00.623 | INFO | Analyzing chunk frames = [5280768,5281280) | 20260505 18:15:00.623 | INFO | MemInfo: { | 20260505 18:15:00.623 | INFO | "gpu" : " 17 GB", | 20260505 18:15:00.623 | INFO | "huge" : "", | 20260505 18:15:00.623 | INFO | "huge_shared" : " 27 GB", | 20260505 18:15:00.624 | INFO | "malloc" : " 14 GB", | 20260505 18:15:00.624 | INFO | "pinned" : " 29 GB" | 20260505 18:15:00.624 | INFO | } | 20260505 18:15:00.624 | INFO | Perf report for the last 5.14035 seconds: | 20260505 18:15:00.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:15:00.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:00.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:00.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:00.624 | INFO | ANALYSIS : 62.7%, 3.0, 97.5ms, 0 | 20260505 18:15:00.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:00.624 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:00.624 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:15:00.624 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:00.624 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:00.624 | INFO | Uploaded 36GiB to GPU in 2.20281s (16.3428GiB/s) | 20260505 18:15:00.624 | INFO | Downloaded 34GiB to GPU in 2.04165s (16.6532GiB/s) | 20260505 18:15:05.766 | INFO | Analyzing chunk frames = [5281280,5281792) | 20260505 18:15:05.766 | INFO | MemInfo: { | 20260505 18:15:05.766 | INFO | "gpu" : " 17 GB", | 20260505 18:15:05.766 | INFO | "huge" : "", | 20260505 18:15:05.766 | INFO | "huge_shared" : " 27 GB", | 20260505 18:15:05.766 | INFO | "malloc" : " 14 GB", | 20260505 18:15:05.766 | INFO | "pinned" : " 29 GB" | 20260505 18:15:05.766 | INFO | } | 20260505 18:15:05.767 | INFO | Perf report for the last 5.14293 seconds: | 20260505 18:15:05.767 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:15:05.767 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:05.767 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:05.767 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:05.767 | INFO | ANALYSIS : 62.5%, 3.0, 97.1ms, 0 | 20260505 18:15:05.767 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:05.767 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:05.767 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:15:05.767 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:05.767 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:05.767 | INFO | Uploaded 36GiB to GPU in 2.1934s (16.4129GiB/s) | 20260505 18:15:05.767 | INFO | Downloaded 34GiB to GPU in 2.14147s (15.8769GiB/s) | 20260505 18:15:10.911 | INFO | Analyzing chunk frames = [5281792,5282304) | 20260505 18:15:10.911 | INFO | MemInfo: { | 20260505 18:15:10.911 | INFO | "gpu" : " 17 GB", | 20260505 18:15:10.911 | INFO | "huge" : "", | 20260505 18:15:10.911 | INFO | "huge_shared" : " 27 GB", | 20260505 18:15:10.912 | INFO | "malloc" : " 14 GB", | 20260505 18:15:10.912 | INFO | "pinned" : " 29 GB" | 20260505 18:15:10.912 | INFO | } | 20260505 18:15:10.912 | INFO | Perf report for the last 5.14508 seconds: | 20260505 18:15:10.912 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:15:10.912 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:10.912 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:10.912 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:10.912 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260505 18:15:10.912 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:10.912 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:10.912 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:15:10.912 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:10.912 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:10.912 | INFO | Uploaded 36GiB to GPU in 2.13633s (16.8513GiB/s) | 20260505 18:15:10.912 | INFO | Downloaded 34GiB to GPU in 2.06229s (16.4865GiB/s) | 20260505 18:15:16.047 | INFO | Analyzing chunk frames = [5282304,5282816) | 20260505 18:15:16.047 | INFO | MemInfo: { | 20260505 18:15:16.047 | INFO | "gpu" : " 17 GB", | 20260505 18:15:16.047 | INFO | "huge" : "", | 20260505 18:15:16.047 | INFO | "huge_shared" : " 27 GB", | 20260505 18:15:16.047 | INFO | "malloc" : " 14 GB", | 20260505 18:15:16.047 | INFO | "pinned" : " 29 GB" | 20260505 18:15:16.047 | INFO | } | 20260505 18:15:16.047 | INFO | Perf report for the last 5.13573 seconds: | 20260505 18:15:16.047 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:15:16.047 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:16.047 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:16.047 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:16.048 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260505 18:15:16.048 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:16.048 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:16.048 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:15:16.048 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:16.048 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:16.048 | INFO | Uploaded 36GiB to GPU in 2.13981s (16.8239GiB/s) | 20260505 18:15:16.048 | INFO | Downloaded 34GiB to GPU in 2.0569s (16.5298GiB/s) | 20260505 18:15:21.179 | INFO | Analyzing chunk frames = [5282816,5283328) | 20260505 18:15:21.179 | INFO | MemInfo: { | 20260505 18:15:21.179 | INFO | "gpu" : " 17 GB", | 20260505 18:15:21.179 | INFO | "huge" : "", | 20260505 18:15:21.179 | INFO | "huge_shared" : " 27 GB", | 20260505 18:15:21.179 | INFO | "malloc" : " 14 GB", | 20260505 18:15:21.179 | INFO | "pinned" : " 29 GB" | 20260505 18:15:21.179 | INFO | } | 20260505 18:15:21.179 | INFO | Perf report for the last 5.13176 seconds: | 20260505 18:15:21.179 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:15:21.179 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:21.179 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:21.179 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:21.179 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 18:15:21.179 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:21.179 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:21.179 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 18:15:21.179 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:21.179 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:21.179 | INFO | Uploaded 36GiB to GPU in 2.14397s (16.7913GiB/s) | 20260505 18:15:21.179 | INFO | Downloaded 34GiB to GPU in 2.14377s (15.8599GiB/s) | 20260505 18:15:26.323 | INFO | Analyzing chunk frames = [5283328,5283840) | 20260505 18:15:26.323 | INFO | MemInfo: { | 20260505 18:15:26.323 | INFO | "gpu" : " 17 GB", | 20260505 18:15:26.323 | INFO | "huge" : "", | 20260505 18:15:26.323 | INFO | "huge_shared" : " 27 GB", | 20260505 18:15:26.324 | INFO | "malloc" : " 14 GB", | 20260505 18:15:26.324 | INFO | "pinned" : " 29 GB" | 20260505 18:15:26.324 | INFO | } | 20260505 18:15:26.324 | INFO | Perf report for the last 5.14443 seconds: | 20260505 18:15:26.324 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:15:26.324 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:26.324 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:26.324 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:26.324 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 18:15:26.324 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:26.324 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:26.324 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 18:15:26.324 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:26.324 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:26.324 | INFO | Uploaded 36GiB to GPU in 2.17135s (16.5795GiB/s) | 20260505 18:15:26.324 | INFO | Downloaded 34GiB to GPU in 2.02733s (16.7709GiB/s) | 20260505 18:15:26.324 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 18:15:26.324 | INFO | Total mem usage: 15.3 GiB | 20260505 18:15:26.324 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 18:15:26.324 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.12%) | 20260505 18:15:26.324 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 18:15:26.324 | INFO | High Water Breakdown: | 20260505 18:15:26.324 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 18:15:26.324 | INFO | 38.9% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 18:15:26.324 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 18:15:26.324 | INFO | ------------------------------------------ | 20260505 18:15:26.324 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:15:26.324 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 18:15:26.324 | INFO | Total mem usage: 26.5 GiB | 20260505 18:15:26.324 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 18:15:26.324 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 18:15:26.324 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 18:15:26.324 | INFO | High Water Breakdown: | 20260505 18:15:26.325 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:15:26.325 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:15:26.325 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:15:26.325 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:15:26.325 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:15:26.325 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:15:26.325 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 18:15:26.325 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 18:15:26.325 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:15:26.325 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 18:15:26.325 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 18:15:26.325 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 18:15:26.325 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 18:15:26.325 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 18:15:26.325 | INFO | ------------------------------------------ | 20260505 18:15:26.325 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:15:26.325 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 18:15:26.325 | INFO | Total mem usage: 25.6 GiB | 20260505 18:15:26.325 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 18:15:26.325 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 18:15:26.325 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 18:15:26.325 | INFO | High Water Breakdown: | 20260505 18:15:26.325 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 18:15:26.325 | INFO | ------------------------------------------ | 20260505 18:15:26.325 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:15:26.325 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 18:15:26.325 | INFO | Total mem usage: 15.7 GiB | 20260505 18:15:26.325 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 18:15:26.325 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 18:15:26.325 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 18:15:26.325 | INFO | High Water Breakdown: | 20260505 18:15:26.325 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 18:15:26.325 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 18:15:26.325 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 18:15:26.325 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 18:15:26.325 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 18:15:26.325 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 18:15:26.325 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 18:15:26.325 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 18:15:26.325 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 18:15:26.325 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:15:26.325 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 18:15:26.325 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:15:26.325 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 18:15:26.325 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 18:15:26.325 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 18:15:26.325 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 18:15:26.325 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:15:26.325 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 18:15:26.325 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:15:26.325 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:15:26.325 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:15:26.325 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 18:15:26.325 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 18:15:26.325 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:15:26.325 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 18:15:26.325 | INFO | ------------------------------------------ | 20260505 18:15:26.325 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:15:26.325 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 18:15:26.325 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:15:26.325 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:15:26.325 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.47e+01, 1.35e+01 / 2.06e+00 | 20260505 18:15:26.325 | INFO | PerfReport: 0.29%, AggregateStats -- 9.27e-02 / 1.01e-01, 1.16e-02 / 3.67e-03 | 20260505 18:15:26.325 | INFO | PerfReport: 11.81%, Baseline -- 3.72e+00 / 4.52e+00, 5.69e-01 / 1.65e-01 | 20260505 18:15:26.325 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.52e+00, 1.88e-01 / 9.62e-02 | 20260505 18:15:26.325 | INFO | PerfReport: 15.96%, DME -- 4.03e+01 / 4.47e+01, 4.23e+00 / 2.21e+00 | 20260505 18:15:26.325 | INFO | PerfReport: 56.82%, FrameLabeling -- 1.79e+01 / 1.92e+01, 1.41e+00 / 9.01e-01 | 20260505 18:15:26.325 | INFO | PerfReport: 7.33%, PulseAccumulating -- 2.31e+00 / 2.89e+00, 4.48e-01 / 1.81e-01 | 20260505 18:15:26.325 | INFO | PerfReport: 3.35%, Metrics -- 1.06e+00 / 1.38e+00, 2.94e-01 / 1.10e-01 | 20260505 18:15:26.325 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.67e-01 / 4.30e-01, 6.47e-02 / 4.84e-02 | 20260505 18:15:26.325 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 18:15:26.325 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:15:26.325 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:15:26.325 | INFO | PerfReport: Full Execution -- 9.98e+01 / -nan, 2.55e+01 / 0.00e+00 | 20260505 18:15:26.325 | INFO | PerfReport: 22.58%, Upload -- 2.25e+01 / 0.00e+00, 7.87e+00 / 0.00e+00 | 20260505 18:15:26.325 | INFO | PerfReport: 21.82%, Download -- 2.18e+01 / 0.00e+00, 7.80e+00 / 0.00e+00 | 20260505 18:15:26.325 | INFO | PerfReport: 31.59%, Compute -- 3.15e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 18:15:26.325 | INFO | PerfReport: 2.27%, LockUpload -- 2.27e+00 / 0.00e+00, 7.86e+00 / 0.00e+00 | 20260505 18:15:26.325 | INFO | PerfReport: 1.43%, LockDownload -- 1.43e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260505 18:15:26.325 | INFO | PerfReport: 20.30%, LockCompute -- 2.03e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 18:15:26.325 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.94e-02 / 0.00e+00 | 20260505 18:15:26.458 | INFO | Starting a round of Baz IO | 20260505 18:15:29.627 | INFO | Populating RTMetrics report | 20260505 18:15:29.627 | INFO | RTMetrics report populated | 20260505 18:15:29.627 | INFO | Opening tmp RTMetrics json file | 20260505 18:15:29.635 | INFO | Serializing RTMetrics to json | 20260505 18:15:29.638 | INFO | Serialization done, swapping tmp file now... | 20260505 18:15:29.640 | INFO | RTMetrics done | 20260505 18:15:29.714 | INFO | Finished a round of Baz IO: Wrote 2.098 GiB in 3.255 seconds | 20260505 18:15:29.714 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.93% / 0.00% / 78.06% | 20260505 18:15:29.714 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.257671% / 0% / 0.000115236% | 20260505 18:15:29.714 | INFO | Baz IO overall: | 20260505 18:15:29.714 | INFO | BytesWritten : 3113435964923 | 20260505 18:15:29.714 | INFO | NumEvents : 1475193915372 | 20260505 18:15:31.466 | INFO | Analyzing chunk frames = [5283840,5284352) | 20260505 18:15:31.466 | INFO | MemInfo: { | 20260505 18:15:31.466 | INFO | "gpu" : " 17 GB", | 20260505 18:15:31.466 | INFO | "huge" : "", | 20260505 18:15:31.466 | INFO | "huge_shared" : " 27 GB", | 20260505 18:15:31.466 | INFO | "malloc" : " 14 GB", | 20260505 18:15:31.466 | INFO | "pinned" : " 29 GB" | 20260505 18:15:31.466 | INFO | } | 20260505 18:15:31.466 | INFO | Perf report for the last 5.14256 seconds: | 20260505 18:15:31.466 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:15:31.466 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:31.466 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:31.466 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:31.466 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 18:15:31.466 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:31.466 | INFO | RT_METRICS : 44.7%, 1.0, 23.5ms, 0 | 20260505 18:15:31.466 | INFO | PRE_HQ : 60.5%, 1.8, 56.4ms, 0 | 20260505 18:15:31.466 | INFO | BAZWRITER : 62.8%, 3.8, 123.4ms, 0 | 20260505 18:15:31.466 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 18:15:31.467 | INFO | Uploaded 36GiB to GPU in 2.16147s (16.6553GiB/s) | 20260505 18:15:31.467 | INFO | Downloaded 38GiB to GPU in 2.24494s (16.927GiB/s) | 20260505 18:15:36.608 | INFO | Analyzing chunk frames = [5284352,5284864) | 20260505 18:15:36.608 | INFO | MemInfo: { | 20260505 18:15:36.609 | INFO | "gpu" : " 17 GB", | 20260505 18:15:36.609 | INFO | "huge" : "", | 20260505 18:15:36.609 | INFO | "huge_shared" : " 27 GB", | 20260505 18:15:36.609 | INFO | "malloc" : " 14 GB", | 20260505 18:15:36.609 | INFO | "pinned" : " 29 GB" | 20260505 18:15:36.609 | INFO | } | 20260505 18:15:36.609 | INFO | Perf report for the last 5.14253 seconds: | 20260505 18:15:36.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:15:36.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:36.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:36.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:36.609 | INFO | ANALYSIS : 62.7%, 3.0, 98.0ms, 0 | 20260505 18:15:36.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:36.609 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:36.609 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:15:36.609 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:36.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:36.609 | INFO | Uploaded 36GiB to GPU in 2.14653s (16.7713GiB/s) | 20260505 18:15:36.609 | INFO | Downloaded 34GiB to GPU in 2.03988s (16.6677GiB/s) | 20260505 18:15:41.740 | INFO | Analyzing chunk frames = [5284864,5285376) | 20260505 18:15:41.740 | INFO | MemInfo: { | 20260505 18:15:41.740 | INFO | "gpu" : " 17 GB", | 20260505 18:15:41.740 | INFO | "huge" : "", | 20260505 18:15:41.740 | INFO | "huge_shared" : " 27 GB", | 20260505 18:15:41.740 | INFO | "malloc" : " 14 GB", | 20260505 18:15:41.740 | INFO | "pinned" : " 29 GB" | 20260505 18:15:41.740 | INFO | } | 20260505 18:15:41.740 | INFO | Perf report for the last 5.13145 seconds: | 20260505 18:15:41.740 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:15:41.740 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:41.740 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:41.740 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:41.740 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260505 18:15:41.740 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:41.740 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:41.740 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:15:41.740 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:41.740 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:41.741 | INFO | Uploaded 36GiB to GPU in 2.13298s (16.8778GiB/s) | 20260505 18:15:41.741 | INFO | Downloaded 34GiB to GPU in 2.02463s (16.7932GiB/s) | 20260505 18:15:46.878 | INFO | Analyzing chunk frames = [5285376,5285888) | 20260505 18:15:46.878 | INFO | MemInfo: { | 20260505 18:15:46.878 | INFO | "gpu" : " 17 GB", | 20260505 18:15:46.878 | INFO | "huge" : "", | 20260505 18:15:46.878 | INFO | "huge_shared" : " 27 GB", | 20260505 18:15:46.878 | INFO | "malloc" : " 14 GB", | 20260505 18:15:46.878 | INFO | "pinned" : " 29 GB" | 20260505 18:15:46.878 | INFO | } | 20260505 18:15:46.878 | INFO | Perf report for the last 5.1382 seconds: | 20260505 18:15:46.878 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:15:46.878 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:46.878 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:46.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:46.879 | INFO | ANALYSIS : 62.6%, 3.0, 97.2ms, 0 | 20260505 18:15:46.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:46.879 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:46.879 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 18:15:46.879 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:46.879 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:46.879 | INFO | Uploaded 36GiB to GPU in 2.20027s (16.3616GiB/s) | 20260505 18:15:46.879 | INFO | Downloaded 34GiB to GPU in 2.13967s (15.8903GiB/s) | 20260505 18:15:52.023 | INFO | Analyzing chunk frames = [5285888,5286400) | 20260505 18:15:52.023 | INFO | MemInfo: { | 20260505 18:15:52.023 | INFO | "gpu" : " 17 GB", | 20260505 18:15:52.023 | INFO | "huge" : "", | 20260505 18:15:52.023 | INFO | "huge_shared" : " 27 GB", | 20260505 18:15:52.023 | INFO | "malloc" : " 14 GB", | 20260505 18:15:52.023 | INFO | "pinned" : " 29 GB" | 20260505 18:15:52.023 | INFO | } | 20260505 18:15:52.023 | INFO | Perf report for the last 5.14495 seconds: | 20260505 18:15:52.023 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:15:52.023 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:52.023 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:52.023 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:52.023 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260505 18:15:52.023 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:52.024 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:52.024 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:15:52.024 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:52.024 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:52.024 | INFO | Uploaded 36GiB to GPU in 2.14976s (16.7461GiB/s) | 20260505 18:15:52.024 | INFO | Downloaded 34GiB to GPU in 2.06421s (16.4712GiB/s) | 20260505 18:15:57.161 | INFO | Analyzing chunk frames = [5286400,5286912) | 20260505 18:15:57.161 | INFO | MemInfo: { | 20260505 18:15:57.161 | INFO | "gpu" : " 17 GB", | 20260505 18:15:57.161 | INFO | "huge" : "", | 20260505 18:15:57.161 | INFO | "huge_shared" : " 27 GB", | 20260505 18:15:57.161 | INFO | "malloc" : " 14 GB", | 20260505 18:15:57.161 | INFO | "pinned" : " 29 GB" | 20260505 18:15:57.161 | INFO | } | 20260505 18:15:57.161 | INFO | Perf report for the last 5.13801 seconds: | 20260505 18:15:57.161 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:15:57.161 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:57.161 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:57.161 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:57.161 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260505 18:15:57.162 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:57.162 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:57.162 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:15:57.162 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:57.162 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:15:57.162 | INFO | Uploaded 36GiB to GPU in 2.13962s (16.8254GiB/s) | 20260505 18:15:57.162 | INFO | Downloaded 34GiB to GPU in 2.04864s (16.5964GiB/s) | 20260505 18:16:02.301 | INFO | Analyzing chunk frames = [5286912,5287424) | 20260505 18:16:02.301 | INFO | MemInfo: { | 20260505 18:16:02.301 | INFO | "gpu" : " 17 GB", | 20260505 18:16:02.301 | INFO | "huge" : "", | 20260505 18:16:02.301 | INFO | "huge_shared" : " 27 GB", | 20260505 18:16:02.301 | INFO | "malloc" : " 14 GB", | 20260505 18:16:02.301 | INFO | "pinned" : " 29 GB" | 20260505 18:16:02.301 | INFO | } | 20260505 18:16:02.301 | INFO | Perf report for the last 5.13975 seconds: | 20260505 18:16:02.301 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:16:02.301 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:02.301 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:02.301 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:02.301 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 18:16:02.301 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:02.301 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:02.301 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 18:16:02.301 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:02.301 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:02.302 | INFO | Uploaded 36GiB to GPU in 2.14724s (16.7657GiB/s) | 20260505 18:16:02.302 | INFO | Downloaded 34GiB to GPU in 2.14781s (15.8301GiB/s) | 20260505 18:16:07.434 | INFO | Analyzing chunk frames = [5287424,5287936) | 20260505 18:16:07.434 | INFO | MemInfo: { | 20260505 18:16:07.434 | INFO | "gpu" : " 17 GB", | 20260505 18:16:07.434 | INFO | "huge" : "", | 20260505 18:16:07.434 | INFO | "huge_shared" : " 27 GB", | 20260505 18:16:07.434 | INFO | "malloc" : " 14 GB", | 20260505 18:16:07.434 | INFO | "pinned" : " 29 GB" | 20260505 18:16:07.434 | INFO | } | 20260505 18:16:07.434 | INFO | Perf report for the last 5.13326 seconds: | 20260505 18:16:07.434 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:16:07.434 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:07.435 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:07.435 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:07.435 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 18:16:07.435 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:07.435 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:07.435 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 18:16:07.435 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:07.435 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:07.435 | INFO | Uploaded 36GiB to GPU in 2.19882s (16.3724GiB/s) | 20260505 18:16:07.435 | INFO | Downloaded 34GiB to GPU in 2.02293s (16.8073GiB/s) | 20260505 18:16:07.584 | INFO | Starting a round of Baz IO | 20260505 18:16:10.759 | INFO | Populating RTMetrics report | 20260505 18:16:10.759 | INFO | RTMetrics report populated | 20260505 18:16:10.759 | INFO | Opening tmp RTMetrics json file | 20260505 18:16:10.760 | INFO | Serializing RTMetrics to json | 20260505 18:16:10.763 | INFO | Serialization done, swapping tmp file now... | 20260505 18:16:10.765 | INFO | RTMetrics done | 20260505 18:16:10.856 | INFO | Finished a round of Baz IO: Wrote 2.099 GiB in 3.271 seconds | 20260505 18:16:10.856 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.93% / 0.00% / 78.07% | 20260505 18:16:10.856 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.257532% / 3.97364e-06% / 9.53674e-05% | 20260505 18:16:10.856 | INFO | Baz IO overall: | 20260505 18:16:10.856 | INFO | BytesWritten : 3115689540352 | 20260505 18:16:10.856 | INFO | NumEvents : 1476245095324 | 20260505 18:16:12.573 | INFO | Analyzing chunk frames = [5287936,5288448) | 20260505 18:16:12.573 | INFO | MemInfo: { | 20260505 18:16:12.573 | INFO | "gpu" : " 17 GB", | 20260505 18:16:12.573 | INFO | "huge" : "", | 20260505 18:16:12.573 | INFO | "huge_shared" : " 27 GB", | 20260505 18:16:12.573 | INFO | "malloc" : " 14 GB", | 20260505 18:16:12.573 | INFO | "pinned" : " 29 GB" | 20260505 18:16:12.573 | INFO | } | 20260505 18:16:12.573 | INFO | Perf report for the last 5.13863 seconds: | 20260505 18:16:12.573 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:16:12.573 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:12.573 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:12.573 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:12.573 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 18:16:12.573 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:12.573 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260505 18:16:12.573 | INFO | PRE_HQ : 60.2%, 1.8, 55.7ms, 0 | 20260505 18:16:12.573 | INFO | BAZWRITER : 63.4%, 3.8, 122.6ms, 0 | 20260505 18:16:12.573 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 18:16:12.573 | INFO | Uploaded 36GiB to GPU in 2.1647s (16.6305GiB/s) | 20260505 18:16:12.573 | INFO | Downloaded 38GiB to GPU in 2.25682s (16.8379GiB/s) | 20260505 18:16:17.722 | INFO | Analyzing chunk frames = [5288448,5288960) | 20260505 18:16:17.722 | INFO | MemInfo: { | 20260505 18:16:17.722 | INFO | "gpu" : " 17 GB", | 20260505 18:16:17.722 | INFO | "huge" : "", | 20260505 18:16:17.722 | INFO | "huge_shared" : " 27 GB", | 20260505 18:16:17.722 | INFO | "malloc" : " 14 GB", | 20260505 18:16:17.722 | INFO | "pinned" : " 29 GB" | 20260505 18:16:17.722 | INFO | } | 20260505 18:16:17.722 | INFO | Perf report for the last 5.14906 seconds: | 20260505 18:16:17.722 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:16:17.722 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:17.722 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:17.722 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:17.722 | INFO | ANALYSIS : 62.7%, 3.0, 98.1ms, 0 | 20260505 18:16:17.722 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:17.722 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:17.722 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260505 18:16:17.722 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:17.722 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:17.722 | INFO | Uploaded 36GiB to GPU in 2.13465s (16.8646GiB/s) | 20260505 18:16:17.722 | INFO | Downloaded 34GiB to GPU in 2.03224s (16.7303GiB/s) | 20260505 18:16:22.861 | INFO | Analyzing chunk frames = [5288960,5289472) | 20260505 18:16:22.861 | INFO | MemInfo: { | 20260505 18:16:22.861 | INFO | "gpu" : " 17 GB", | 20260505 18:16:22.861 | INFO | "huge" : "", | 20260505 18:16:22.861 | INFO | "huge_shared" : " 27 GB", | 20260505 18:16:22.861 | INFO | "malloc" : " 14 GB", | 20260505 18:16:22.861 | INFO | "pinned" : " 29 GB" | 20260505 18:16:22.861 | INFO | } | 20260505 18:16:22.861 | INFO | Perf report for the last 5.13919 seconds: | 20260505 18:16:22.861 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:16:22.861 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:22.861 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:22.861 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:22.861 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260505 18:16:22.861 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:22.861 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:22.861 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:16:22.861 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:22.861 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:22.862 | INFO | Uploaded 36GiB to GPU in 2.11509s (17.0206GiB/s) | 20260505 18:16:22.862 | INFO | Downloaded 34GiB to GPU in 2.02428s (16.7961GiB/s) | 20260505 18:16:27.992 | INFO | Analyzing chunk frames = [5289472,5289984) | 20260505 18:16:27.992 | INFO | MemInfo: { | 20260505 18:16:27.992 | INFO | "gpu" : " 17 GB", | 20260505 18:16:27.992 | INFO | "huge" : "", | 20260505 18:16:27.992 | INFO | "huge_shared" : " 27 GB", | 20260505 18:16:27.992 | INFO | "malloc" : " 14 GB", | 20260505 18:16:27.992 | INFO | "pinned" : " 29 GB" | 20260505 18:16:27.992 | INFO | } | 20260505 18:16:27.992 | INFO | Perf report for the last 5.1304 seconds: | 20260505 18:16:27.992 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:16:27.992 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:27.992 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:27.992 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:27.992 | INFO | ANALYSIS : 62.5%, 3.0, 96.9ms, 0 | 20260505 18:16:27.992 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:27.992 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:27.992 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:16:27.992 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:27.992 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:27.992 | INFO | Uploaded 36GiB to GPU in 2.18805s (16.453GiB/s) | 20260505 18:16:27.992 | INFO | Downloaded 34GiB to GPU in 2.05491s (16.5457GiB/s) | 20260505 18:16:33.128 | INFO | Analyzing chunk frames = [5289984,5290496) | 20260505 18:16:33.128 | INFO | MemInfo: { | 20260505 18:16:33.128 | INFO | "gpu" : " 17 GB", | 20260505 18:16:33.128 | INFO | "huge" : "", | 20260505 18:16:33.128 | INFO | "huge_shared" : " 27 GB", | 20260505 18:16:33.128 | INFO | "malloc" : " 14 GB", | 20260505 18:16:33.128 | INFO | "pinned" : " 29 GB" | 20260505 18:16:33.128 | INFO | } | 20260505 18:16:33.128 | INFO | Perf report for the last 5.13649 seconds: | 20260505 18:16:33.128 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:16:33.128 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:33.128 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:33.128 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:33.128 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 18:16:33.128 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:33.128 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:33.128 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:16:33.128 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:33.128 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:33.129 | INFO | Uploaded 36GiB to GPU in 2.14699s (16.7677GiB/s) | 20260505 18:16:33.129 | INFO | Downloaded 34GiB to GPU in 2.06471s (16.4672GiB/s) | 20260505 18:16:38.277 | INFO | Analyzing chunk frames = [5290496,5291008) | 20260505 18:16:38.277 | INFO | MemInfo: { | 20260505 18:16:38.277 | INFO | "gpu" : " 17 GB", | 20260505 18:16:38.277 | INFO | "huge" : "", | 20260505 18:16:38.277 | INFO | "huge_shared" : " 27 GB", | 20260505 18:16:38.277 | INFO | "malloc" : " 14 GB", | 20260505 18:16:38.277 | INFO | "pinned" : " 29 GB" | 20260505 18:16:38.277 | INFO | } | 20260505 18:16:38.277 | INFO | Perf report for the last 5.14891 seconds: | 20260505 18:16:38.277 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:16:38.277 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:38.277 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:38.277 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:38.277 | INFO | ANALYSIS : 62.3%, 3.0, 97.4ms, 0 | 20260505 18:16:38.277 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:38.277 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:38.277 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 18:16:38.277 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:38.277 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:38.277 | INFO | Uploaded 36GiB to GPU in 2.14027s (16.8203GiB/s) | 20260505 18:16:38.277 | INFO | Downloaded 34GiB to GPU in 2.04454s (16.6296GiB/s) | 20260505 18:16:43.425 | INFO | Analyzing chunk frames = [5291008,5291520) | 20260505 18:16:43.425 | INFO | MemInfo: { | 20260505 18:16:43.425 | INFO | "gpu" : " 17 GB", | 20260505 18:16:43.425 | INFO | "huge" : "", | 20260505 18:16:43.425 | INFO | "huge_shared" : " 27 GB", | 20260505 18:16:43.425 | INFO | "malloc" : " 14 GB", | 20260505 18:16:43.425 | INFO | "pinned" : " 29 GB" | 20260505 18:16:43.425 | INFO | } | 20260505 18:16:43.425 | INFO | Perf report for the last 5.14761 seconds: | 20260505 18:16:43.425 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:16:43.425 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:43.425 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:43.425 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:43.425 | INFO | ANALYSIS : 63.2%, 3.0, 98.9ms, 0 | 20260505 18:16:43.425 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:43.425 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:43.425 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 18:16:43.425 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:43.425 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:43.425 | INFO | Uploaded 36GiB to GPU in 2.14229s (16.8045GiB/s) | 20260505 18:16:43.425 | INFO | Downloaded 34GiB to GPU in 2.13823s (15.901GiB/s) | 20260505 18:16:48.552 | INFO | Analyzing chunk frames = [5291520,5292032) | 20260505 18:16:48.552 | INFO | MemInfo: { | 20260505 18:16:48.552 | INFO | "gpu" : " 17 GB", | 20260505 18:16:48.552 | INFO | "huge" : "", | 20260505 18:16:48.553 | INFO | "huge_shared" : " 27 GB", | 20260505 18:16:48.553 | INFO | "malloc" : " 14 GB", | 20260505 18:16:48.553 | INFO | "pinned" : " 29 GB" | 20260505 18:16:48.553 | INFO | } | 20260505 18:16:48.553 | INFO | Perf report for the last 5.12779 seconds: | 20260505 18:16:48.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:16:48.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:48.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:48.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:48.553 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260505 18:16:48.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:48.553 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:48.553 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 18:16:48.553 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:48.553 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:48.553 | INFO | Uploaded 36GiB to GPU in 2.1282s (16.9157GiB/s) | 20260505 18:16:48.553 | INFO | Downloaded 34GiB to GPU in 2.02048s (16.8277GiB/s) | 20260505 18:16:48.553 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 18:16:48.553 | INFO | Total mem usage: 15.3 GiB | 20260505 18:16:48.553 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 18:16:48.553 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.01%) | 20260505 18:16:48.553 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 18:16:48.553 | INFO | High Water Breakdown: | 20260505 18:16:48.553 | INFO | 61% : 8.66 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 18:16:48.553 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 18:16:48.553 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 18:16:48.553 | INFO | ------------------------------------------ | 20260505 18:16:48.553 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:16:48.553 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 18:16:48.553 | INFO | Total mem usage: 26.5 GiB | 20260505 18:16:48.553 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 18:16:48.553 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 18:16:48.553 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 18:16:48.553 | INFO | High Water Breakdown: | 20260505 18:16:48.553 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:16:48.553 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:16:48.553 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:16:48.553 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:16:48.553 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:16:48.553 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:16:48.553 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 18:16:48.553 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 18:16:48.553 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:16:48.553 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 18:16:48.553 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 18:16:48.553 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 18:16:48.553 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 18:16:48.553 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 18:16:48.553 | INFO | ------------------------------------------ | 20260505 18:16:48.554 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:16:48.554 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 18:16:48.554 | INFO | Total mem usage: 25.6 GiB | 20260505 18:16:48.554 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 18:16:48.554 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 18:16:48.554 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 18:16:48.554 | INFO | High Water Breakdown: | 20260505 18:16:48.554 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 18:16:48.554 | INFO | ------------------------------------------ | 20260505 18:16:48.554 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:16:48.554 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 18:16:48.554 | INFO | Total mem usage: 15.7 GiB | 20260505 18:16:48.554 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 18:16:48.554 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 18:16:48.554 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 18:16:48.554 | INFO | High Water Breakdown: | 20260505 18:16:48.554 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 18:16:48.554 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 18:16:48.554 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 18:16:48.554 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 18:16:48.554 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 18:16:48.554 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 18:16:48.554 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 18:16:48.554 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 18:16:48.554 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 18:16:48.554 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:16:48.554 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 18:16:48.554 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:16:48.554 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 18:16:48.554 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 18:16:48.554 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 18:16:48.554 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 18:16:48.554 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:16:48.554 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 18:16:48.554 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:16:48.554 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:16:48.554 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:16:48.554 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 18:16:48.554 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 18:16:48.554 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:16:48.554 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 18:16:48.554 | INFO | ------------------------------------------ | 20260505 18:16:48.554 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:16:48.554 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 18:16:48.554 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:16:48.554 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:16:48.554 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.49e+01, 1.37e+01 / 2.02e+00 | 20260505 18:16:48.554 | INFO | PerfReport: 0.29%, AggregateStats -- 9.25e-02 / 1.03e-01, 1.05e-02 / 5.95e-03 | 20260505 18:16:48.554 | INFO | PerfReport: 11.79%, Baseline -- 3.73e+00 / 4.53e+00, 5.73e-01 / 1.45e-01 | 20260505 18:16:48.554 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.52e+00, 1.94e-01 / 1.04e-01 | 20260505 18:16:48.554 | INFO | PerfReport: 15.94%, DME -- 4.03e+01 / 4.47e+01, 4.31e+00 / 2.05e+00 | 20260505 18:16:48.554 | INFO | PerfReport: 56.92%, FrameLabeling -- 1.80e+01 / 1.93e+01, 1.50e+00 / 9.78e-01 | 20260505 18:16:48.554 | INFO | PerfReport: 7.28%, PulseAccumulating -- 2.30e+00 / 2.89e+00, 4.43e-01 / 1.65e-01 | 20260505 18:16:48.554 | INFO | PerfReport: 3.33%, Metrics -- 1.05e+00 / 1.39e+00, 2.91e-01 / 1.19e-01 | 20260505 18:16:48.554 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.67e-01 / 4.29e-01, 6.55e-02 / 3.48e-02 | 20260505 18:16:48.554 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 18:16:48.554 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:16:48.554 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:16:48.554 | INFO | PerfReport: Full Execution -- 9.98e+01 / -nan, 2.57e+01 / 0.00e+00 | 20260505 18:16:48.554 | INFO | PerfReport: 22.48%, Upload -- 2.24e+01 / 0.00e+00, 7.75e+00 / 0.00e+00 | 20260505 18:16:48.554 | INFO | PerfReport: 21.78%, Download -- 2.17e+01 / 0.00e+00, 7.74e+00 / 0.00e+00 | 20260505 18:16:48.554 | INFO | PerfReport: 31.68%, Compute -- 3.16e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 18:16:48.554 | INFO | PerfReport: 2.27%, LockUpload -- 2.26e+00 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260505 18:16:48.554 | INFO | PerfReport: 1.39%, LockDownload -- 1.39e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260505 18:16:48.554 | INFO | PerfReport: 20.40%, LockCompute -- 2.03e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260505 18:16:48.554 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.81e-02 / 0.00e+00 | 20260505 18:16:48.688 | INFO | Starting a round of Baz IO | 20260505 18:16:51.872 | INFO | Populating RTMetrics report | 20260505 18:16:51.873 | INFO | RTMetrics report populated | 20260505 18:16:51.873 | INFO | Opening tmp RTMetrics json file | 20260505 18:16:51.890 | INFO | Serializing RTMetrics to json | 20260505 18:16:51.892 | INFO | Serialization done, swapping tmp file now... | 20260505 18:16:51.895 | INFO | RTMetrics done | 20260505 18:16:51.992 | INFO | Finished a round of Baz IO: Wrote 2.096 GiB in 3.303 seconds | 20260505 18:16:51.992 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.93% / 0.00% / 78.07% | 20260505 18:16:51.992 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.256427% / 3.97364e-06% / 0.000111262% | 20260505 18:16:51.992 | INFO | Baz IO overall: | 20260505 18:16:51.992 | INFO | BytesWritten : 3117939882753 | 20260505 18:16:51.992 | INFO | NumEvents : 1477293787815 | 20260505 18:16:53.687 | INFO | Analyzing chunk frames = [5292032,5292544) | 20260505 18:16:53.687 | INFO | MemInfo: { | 20260505 18:16:53.687 | INFO | "gpu" : " 17 GB", | 20260505 18:16:53.687 | INFO | "huge" : "", | 20260505 18:16:53.687 | INFO | "huge_shared" : " 27 GB", | 20260505 18:16:53.687 | INFO | "malloc" : " 14 GB", | 20260505 18:16:53.687 | INFO | "pinned" : " 29 GB" | 20260505 18:16:53.687 | INFO | } | 20260505 18:16:53.688 | INFO | Perf report for the last 5.13503 seconds: | 20260505 18:16:53.688 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:16:53.688 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:53.688 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:53.688 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:53.688 | INFO | ANALYSIS : 63.6%, 2.9, 97.4ms, 0 | 20260505 18:16:53.688 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:53.688 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260505 18:16:53.688 | INFO | PRE_HQ : 60.4%, 1.8, 55.5ms, 0 | 20260505 18:16:53.688 | INFO | BAZWRITER : 63.6%, 4.5, 146.2ms, 0 | 20260505 18:16:53.688 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 18:16:53.688 | INFO | Uploaded 36GiB to GPU in 2.1812s (16.5047GiB/s) | 20260505 18:16:53.688 | INFO | Downloaded 38GiB to GPU in 2.26976s (16.7419GiB/s) | 20260505 18:16:58.834 | INFO | Analyzing chunk frames = [5292544,5293056) | 20260505 18:16:58.834 | INFO | MemInfo: { | 20260505 18:16:58.834 | INFO | "gpu" : " 17 GB", | 20260505 18:16:58.834 | INFO | "huge" : "", | 20260505 18:16:58.834 | INFO | "huge_shared" : " 27 GB", | 20260505 18:16:58.834 | INFO | "malloc" : " 14 GB", | 20260505 18:16:58.834 | INFO | "pinned" : " 29 GB" | 20260505 18:16:58.834 | INFO | } | 20260505 18:16:58.834 | INFO | Perf report for the last 5.14653 seconds: | 20260505 18:16:58.834 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:16:58.834 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:58.834 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:58.834 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:58.834 | INFO | ANALYSIS : 62.7%, 3.0, 98.0ms, 0 | 20260505 18:16:58.834 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:58.834 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:58.834 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 18:16:58.834 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:58.834 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:16:58.834 | INFO | Uploaded 36GiB to GPU in 2.11773s (16.9994GiB/s) | 20260505 18:16:58.834 | INFO | Downloaded 34GiB to GPU in 2.02027s (16.8294GiB/s) | 20260505 18:17:03.974 | INFO | Analyzing chunk frames = [5293056,5293568) | 20260505 18:17:03.974 | INFO | MemInfo: { | 20260505 18:17:03.974 | INFO | "gpu" : " 17 GB", | 20260505 18:17:03.974 | INFO | "huge" : "", | 20260505 18:17:03.974 | INFO | "huge_shared" : " 27 GB", | 20260505 18:17:03.974 | INFO | "malloc" : " 14 GB", | 20260505 18:17:03.974 | INFO | "pinned" : " 29 GB" | 20260505 18:17:03.974 | INFO | } | 20260505 18:17:03.974 | INFO | Perf report for the last 5.13963 seconds: | 20260505 18:17:03.974 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:17:03.974 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:03.974 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:03.974 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:03.974 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 18:17:03.974 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:03.974 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:03.974 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 18:17:03.974 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:03.974 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:03.974 | INFO | Uploaded 36GiB to GPU in 2.20702s (16.3116GiB/s) | 20260505 18:17:03.974 | INFO | Downloaded 34GiB to GPU in 2.05934s (16.5102GiB/s) | 20260505 18:17:09.117 | INFO | Analyzing chunk frames = [5293568,5294080) | 20260505 18:17:09.117 | INFO | MemInfo: { | 20260505 18:17:09.117 | INFO | "gpu" : " 17 GB", | 20260505 18:17:09.117 | INFO | "huge" : "", | 20260505 18:17:09.117 | INFO | "huge_shared" : " 27 GB", | 20260505 18:17:09.118 | INFO | "malloc" : " 14 GB", | 20260505 18:17:09.118 | INFO | "pinned" : " 29 GB" | 20260505 18:17:09.118 | INFO | } | 20260505 18:17:09.118 | INFO | Perf report for the last 5.14382 seconds: | 20260505 18:17:09.118 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:17:09.118 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:09.118 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:09.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:09.118 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260505 18:17:09.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:09.118 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:09.118 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 18:17:09.118 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:09.118 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:09.118 | INFO | Uploaded 36GiB to GPU in 2.17499s (16.5518GiB/s) | 20260505 18:17:09.118 | INFO | Downloaded 34GiB to GPU in 2.04523s (16.624GiB/s) | 20260505 18:17:14.253 | INFO | Analyzing chunk frames = [5294080,5294592) | 20260505 18:17:14.253 | INFO | MemInfo: { | 20260505 18:17:14.253 | INFO | "gpu" : " 17 GB", | 20260505 18:17:14.253 | INFO | "huge" : "", | 20260505 18:17:14.253 | INFO | "huge_shared" : " 27 GB", | 20260505 18:17:14.253 | INFO | "malloc" : " 14 GB", | 20260505 18:17:14.253 | INFO | "pinned" : " 29 GB" | 20260505 18:17:14.254 | INFO | } | 20260505 18:17:14.254 | INFO | Perf report for the last 5.13591 seconds: | 20260505 18:17:14.254 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:17:14.254 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:14.254 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:14.254 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:14.254 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 18:17:14.254 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:14.254 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:14.254 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 18:17:14.254 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:14.254 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:14.254 | INFO | Uploaded 36GiB to GPU in 2.11816s (16.9959GiB/s) | 20260505 18:17:14.254 | INFO | Downloaded 34GiB to GPU in 2.05406s (16.5526GiB/s) | 20260505 18:17:19.384 | INFO | Analyzing chunk frames = [5294592,5295104) | 20260505 18:17:19.384 | INFO | MemInfo: { | 20260505 18:17:19.384 | INFO | "gpu" : " 17 GB", | 20260505 18:17:19.384 | INFO | "huge" : "", | 20260505 18:17:19.384 | INFO | "huge_shared" : " 27 GB", | 20260505 18:17:19.384 | INFO | "malloc" : " 14 GB", | 20260505 18:17:19.384 | INFO | "pinned" : " 29 GB" | 20260505 18:17:19.384 | INFO | } | 20260505 18:17:19.384 | INFO | Perf report for the last 5.13025 seconds: | 20260505 18:17:19.384 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:17:19.384 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:19.384 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:19.384 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:19.384 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260505 18:17:19.384 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:19.384 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:19.384 | INFO | PRE_HQ : 27.7%, 1.0, 14.3ms, 0 | 20260505 18:17:19.384 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:19.384 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:19.384 | INFO | Uploaded 36GiB to GPU in 2.14447s (16.7873GiB/s) | 20260505 18:17:19.384 | INFO | Downloaded 34GiB to GPU in 2.04122s (16.6567GiB/s) | 20260505 18:17:24.534 | INFO | Analyzing chunk frames = [5295104,5295616) | 20260505 18:17:24.534 | INFO | MemInfo: { | 20260505 18:17:24.534 | INFO | "gpu" : " 17 GB", | 20260505 18:17:24.534 | INFO | "huge" : "", | 20260505 18:17:24.534 | INFO | "huge_shared" : " 27 GB", | 20260505 18:17:24.534 | INFO | "malloc" : " 14 GB", | 20260505 18:17:24.534 | INFO | "pinned" : " 29 GB" | 20260505 18:17:24.535 | INFO | } | 20260505 18:17:24.535 | INFO | Perf report for the last 5.15123 seconds: | 20260505 18:17:24.535 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:17:24.535 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:24.535 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:24.535 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:24.535 | INFO | ANALYSIS : 63.2%, 3.0, 99.0ms, 0 | 20260505 18:17:24.535 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:24.535 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:24.535 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 18:17:24.535 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:24.535 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:24.535 | INFO | Uploaded 36GiB to GPU in 2.16192s (16.6518GiB/s) | 20260505 18:17:24.535 | INFO | Downloaded 34GiB to GPU in 2.17328s (15.6446GiB/s) | 20260505 18:17:29.674 | INFO | Analyzing chunk frames = [5295616,5296128) | 20260505 18:17:29.674 | INFO | MemInfo: { | 20260505 18:17:29.674 | INFO | "gpu" : " 17 GB", | 20260505 18:17:29.674 | INFO | "huge" : "", | 20260505 18:17:29.674 | INFO | "huge_shared" : " 27 GB", | 20260505 18:17:29.674 | INFO | "malloc" : " 14 GB", | 20260505 18:17:29.674 | INFO | "pinned" : " 29 GB" | 20260505 18:17:29.674 | INFO | } | 20260505 18:17:29.674 | INFO | Perf report for the last 5.13861 seconds: | 20260505 18:17:29.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:17:29.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:29.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:29.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:29.674 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 18:17:29.674 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:29.674 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:29.674 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:17:29.674 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:29.674 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:29.674 | INFO | Uploaded 36GiB to GPU in 2.20276s (16.3431GiB/s) | 20260505 18:17:29.674 | INFO | Downloaded 34GiB to GPU in 2.04263s (16.6452GiB/s) | 20260505 18:17:29.805 | INFO | Starting a round of Baz IO | 20260505 18:17:32.994 | INFO | Populating RTMetrics report | 20260505 18:17:32.994 | INFO | RTMetrics report populated | 20260505 18:17:32.994 | INFO | Opening tmp RTMetrics json file | 20260505 18:17:32.996 | INFO | Serializing RTMetrics to json | 20260505 18:17:32.997 | INFO | Serialization done, swapping tmp file now... | 20260505 18:17:32.998 | INFO | RTMetrics done | 20260505 18:17:33.086 | INFO | Finished a round of Baz IO: Wrote 2.096 GiB in 3.28 seconds | 20260505 18:17:33.086 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.93% / 0.00% / 78.07% | 20260505 18:17:33.086 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.25632% / 0% / 7.94729e-05% | 20260505 18:17:33.086 | INFO | Baz IO overall: | 20260505 18:17:33.086 | INFO | BytesWritten : 3120190740929 | 20260505 18:17:33.086 | INFO | NumEvents : 1478342978406 | 20260505 18:17:34.806 | INFO | Analyzing chunk frames = [5296128,5296640) | 20260505 18:17:34.806 | INFO | MemInfo: { | 20260505 18:17:34.806 | INFO | "gpu" : " 17 GB", | 20260505 18:17:34.806 | INFO | "huge" : "", | 20260505 18:17:34.806 | INFO | "huge_shared" : " 27 GB", | 20260505 18:17:34.806 | INFO | "malloc" : " 14 GB", | 20260505 18:17:34.806 | INFO | "pinned" : " 29 GB" | 20260505 18:17:34.806 | INFO | } | 20260505 18:17:34.806 | INFO | Perf report for the last 5.13227 seconds: | 20260505 18:17:34.806 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:17:34.806 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:34.806 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:34.806 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:34.806 | INFO | ANALYSIS : 63.6%, 3.0, 99.0ms, 0 | 20260505 18:17:34.806 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:34.806 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260505 18:17:34.806 | INFO | PRE_HQ : 60.6%, 1.8, 55.8ms, 0 | 20260505 18:17:34.806 | INFO | BAZWRITER : 63.5%, 3.7, 121.9ms, 0 | 20260505 18:17:34.806 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 18:17:34.806 | INFO | Uploaded 36GiB to GPU in 2.18473s (16.478GiB/s) | 20260505 18:17:34.806 | INFO | Downloaded 38GiB to GPU in 2.25891s (16.8223GiB/s) | 20260505 18:17:39.938 | INFO | Analyzing chunk frames = [5296640,5297152) | 20260505 18:17:39.938 | INFO | MemInfo: { | 20260505 18:17:39.938 | INFO | "gpu" : " 17 GB", | 20260505 18:17:39.938 | INFO | "huge" : "", | 20260505 18:17:39.938 | INFO | "huge_shared" : " 27 GB", | 20260505 18:17:39.938 | INFO | "malloc" : " 14 GB", | 20260505 18:17:39.938 | INFO | "pinned" : " 29 GB" | 20260505 18:17:39.938 | INFO | } | 20260505 18:17:39.938 | INFO | Perf report for the last 5.13236 seconds: | 20260505 18:17:39.939 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:17:39.939 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:39.939 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:39.939 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:39.939 | INFO | ANALYSIS : 62.9%, 3.0, 98.0ms, 0 | 20260505 18:17:39.939 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:39.939 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:39.939 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:17:39.939 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:39.939 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:39.939 | INFO | Uploaded 36GiB to GPU in 2.1412s (16.813GiB/s) | 20260505 18:17:39.939 | INFO | Downloaded 34GiB to GPU in 2.02931s (16.7545GiB/s) | 20260505 18:17:45.084 | INFO | Analyzing chunk frames = [5297152,5297664) | 20260505 18:17:45.084 | INFO | MemInfo: { | 20260505 18:17:45.084 | INFO | "gpu" : " 17 GB", | 20260505 18:17:45.084 | INFO | "huge" : "", | 20260505 18:17:45.084 | INFO | "huge_shared" : " 27 GB", | 20260505 18:17:45.084 | INFO | "malloc" : " 14 GB", | 20260505 18:17:45.084 | INFO | "pinned" : " 29 GB" | 20260505 18:17:45.084 | INFO | } | 20260505 18:17:45.084 | INFO | Perf report for the last 5.14593 seconds: | 20260505 18:17:45.084 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:17:45.084 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:45.084 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:45.084 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:45.084 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 18:17:45.084 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:45.084 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:45.084 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:17:45.084 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:45.084 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:45.085 | INFO | Uploaded 36GiB to GPU in 2.18793s (16.4539GiB/s) | 20260505 18:17:45.085 | INFO | Downloaded 34GiB to GPU in 2.03247s (16.7284GiB/s) | 20260505 18:17:50.230 | INFO | Analyzing chunk frames = [5297664,5298176) | 20260505 18:17:50.230 | INFO | MemInfo: { | 20260505 18:17:50.230 | INFO | "gpu" : " 17 GB", | 20260505 18:17:50.230 | INFO | "huge" : "", | 20260505 18:17:50.230 | INFO | "huge_shared" : " 27 GB", | 20260505 18:17:50.230 | INFO | "malloc" : " 14 GB", | 20260505 18:17:50.230 | INFO | "pinned" : " 29 GB" | 20260505 18:17:50.230 | INFO | } | 20260505 18:17:50.230 | INFO | Perf report for the last 5.14585 seconds: | 20260505 18:17:50.230 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:17:50.230 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:50.230 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:50.230 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:50.230 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 18:17:50.230 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:50.230 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:50.230 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:17:50.230 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:50.230 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:50.231 | INFO | Uploaded 36GiB to GPU in 2.22385s (16.1882GiB/s) | 20260505 18:17:50.231 | INFO | Downloaded 34GiB to GPU in 2.12161s (16.0256GiB/s) | 20260505 18:17:55.371 | INFO | Analyzing chunk frames = [5298176,5298688) | 20260505 18:17:55.371 | INFO | MemInfo: { | 20260505 18:17:55.371 | INFO | "gpu" : " 17 GB", | 20260505 18:17:55.371 | INFO | "huge" : "", | 20260505 18:17:55.371 | INFO | "huge_shared" : " 27 GB", | 20260505 18:17:55.371 | INFO | "malloc" : " 14 GB", | 20260505 18:17:55.372 | INFO | "pinned" : " 29 GB" | 20260505 18:17:55.372 | INFO | } | 20260505 18:17:55.372 | INFO | Perf report for the last 5.14139 seconds: | 20260505 18:17:55.372 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:17:55.372 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:55.372 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:55.372 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:55.372 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 18:17:55.372 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:55.372 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:55.372 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:17:55.372 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:55.372 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:17:55.372 | INFO | Uploaded 36GiB to GPU in 2.13916s (16.829GiB/s) | 20260505 18:17:55.372 | INFO | Downloaded 34GiB to GPU in 2.06449s (16.4689GiB/s) | 20260505 18:18:00.506 | INFO | Analyzing chunk frames = [5298688,5299200) | 20260505 18:18:00.506 | INFO | MemInfo: { | 20260505 18:18:00.506 | INFO | "gpu" : " 17 GB", | 20260505 18:18:00.506 | INFO | "huge" : "", | 20260505 18:18:00.506 | INFO | "huge_shared" : " 27 GB", | 20260505 18:18:00.506 | INFO | "malloc" : " 14 GB", | 20260505 18:18:00.506 | INFO | "pinned" : " 29 GB" | 20260505 18:18:00.506 | INFO | } | 20260505 18:18:00.506 | INFO | Perf report for the last 5.1344 seconds: | 20260505 18:18:00.506 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:18:00.506 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:00.506 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:00.506 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:00.506 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 18:18:00.506 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:00.506 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:00.506 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:18:00.506 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:00.506 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:00.506 | INFO | Uploaded 36GiB to GPU in 2.15085s (16.7376GiB/s) | 20260505 18:18:00.506 | INFO | Downloaded 34GiB to GPU in 2.05447s (16.5493GiB/s) | 20260505 18:18:05.638 | INFO | Analyzing chunk frames = [5299200,5299712) | 20260505 18:18:05.638 | INFO | MemInfo: { | 20260505 18:18:05.638 | INFO | "gpu" : " 17 GB", | 20260505 18:18:05.638 | INFO | "huge" : "", | 20260505 18:18:05.638 | INFO | "huge_shared" : " 27 GB", | 20260505 18:18:05.638 | INFO | "malloc" : " 14 GB", | 20260505 18:18:05.638 | INFO | "pinned" : " 29 GB" | 20260505 18:18:05.639 | INFO | } | 20260505 18:18:05.639 | INFO | Perf report for the last 5.13255 seconds: | 20260505 18:18:05.639 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:18:05.639 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:05.639 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:05.639 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:05.639 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 18:18:05.639 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:05.639 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:05.639 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 18:18:05.639 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:05.639 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:05.639 | INFO | Uploaded 36GiB to GPU in 2.14186s (16.8078GiB/s) | 20260505 18:18:05.639 | INFO | Downloaded 34GiB to GPU in 2.14561s (15.8463GiB/s) | 20260505 18:18:10.788 | INFO | Analyzing chunk frames = [5299712,5300224) | 20260505 18:18:10.788 | INFO | MemInfo: { | 20260505 18:18:10.788 | INFO | "gpu" : " 17 GB", | 20260505 18:18:10.788 | INFO | "huge" : "", | 20260505 18:18:10.788 | INFO | "huge_shared" : " 27 GB", | 20260505 18:18:10.788 | INFO | "malloc" : " 14 GB", | 20260505 18:18:10.788 | INFO | "pinned" : " 29 GB" | 20260505 18:18:10.788 | INFO | } | 20260505 18:18:10.788 | INFO | Perf report for the last 5.14923 seconds: | 20260505 18:18:10.788 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:18:10.788 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:10.788 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:10.788 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:10.788 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 18:18:10.788 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:10.788 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:10.788 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 18:18:10.788 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:10.788 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:10.788 | INFO | Uploaded 36GiB to GPU in 2.19023s (16.4366GiB/s) | 20260505 18:18:10.788 | INFO | Downloaded 34GiB to GPU in 2.0268s (16.7752GiB/s) | 20260505 18:18:10.788 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 18:18:10.788 | INFO | Total mem usage: 15.3 GiB | 20260505 18:18:10.788 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 18:18:10.788 | INFO | Recent High Water Marks Sum: 14.7 GiB (Savings: -4.05%) | 20260505 18:18:10.788 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 18:18:10.788 | INFO | High Water Breakdown: | 20260505 18:18:10.788 | INFO | 61.6% : 9.07 GiB ( 95.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 18:18:10.788 | INFO | 38.3% : 5.64 GiB ( 84.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 18:18:10.788 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 18:18:10.788 | INFO | ------------------------------------------ | 20260505 18:18:10.788 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:18:10.788 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 18:18:10.788 | INFO | Total mem usage: 26.5 GiB | 20260505 18:18:10.788 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 18:18:10.788 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 18:18:10.788 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 18:18:10.788 | INFO | High Water Breakdown: | 20260505 18:18:10.788 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:18:10.789 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:18:10.789 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:18:10.789 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:18:10.789 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:18:10.789 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:18:10.789 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 18:18:10.789 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 18:18:10.789 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:18:10.789 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 18:18:10.789 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 18:18:10.789 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 18:18:10.789 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 18:18:10.789 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 18:18:10.789 | INFO | ------------------------------------------ | 20260505 18:18:10.789 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:18:10.789 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 18:18:10.789 | INFO | Total mem usage: 25.6 GiB | 20260505 18:18:10.789 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 18:18:10.789 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 18:18:10.789 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 18:18:10.789 | INFO | High Water Breakdown: | 20260505 18:18:10.789 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 18:18:10.789 | INFO | ------------------------------------------ | 20260505 18:18:10.789 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:18:10.789 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 18:18:10.789 | INFO | Total mem usage: 15.7 GiB | 20260505 18:18:10.789 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 18:18:10.789 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 18:18:10.789 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 18:18:10.789 | INFO | High Water Breakdown: | 20260505 18:18:10.789 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 18:18:10.789 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 18:18:10.789 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 18:18:10.789 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 18:18:10.789 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 18:18:10.789 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 18:18:10.789 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 18:18:10.789 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 18:18:10.789 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 18:18:10.789 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:18:10.789 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 18:18:10.789 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:18:10.789 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 18:18:10.789 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 18:18:10.789 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 18:18:10.789 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 18:18:10.789 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:18:10.789 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 18:18:10.789 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:18:10.789 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:18:10.789 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:18:10.789 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 18:18:10.789 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 18:18:10.789 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:18:10.789 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 18:18:10.789 | INFO | ------------------------------------------ | 20260505 18:18:10.789 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:18:10.789 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 18:18:10.789 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:18:10.789 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:18:10.789 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.45e+01, 1.35e+01 / 1.86e+00 | 20260505 18:18:10.789 | INFO | PerfReport: 0.29%, AggregateStats -- 9.24e-02 / 1.01e-01, 1.07e-02 / 4.01e-03 | 20260505 18:18:10.789 | INFO | PerfReport: 11.80%, Baseline -- 3.72e+00 / 4.51e+00, 5.66e-01 / 1.71e-01 | 20260505 18:18:10.789 | INFO | PerfReport: 3.26%, Histogram -- 1.03e+00 / 1.50e+00, 1.86e-01 / 1.09e-01 | 20260505 18:18:10.789 | INFO | PerfReport: 15.94%, DME -- 4.02e+01 / 4.44e+01, 3.97e+00 / 1.83e+00 | 20260505 18:18:10.789 | INFO | PerfReport: 56.88%, FrameLabeling -- 1.79e+01 / 1.93e+01, 1.45e+00 / 1.14e+00 | 20260505 18:18:10.789 | INFO | PerfReport: 7.33%, PulseAccumulating -- 2.31e+00 / 2.90e+00, 4.49e-01 / 1.71e-01 | 20260505 18:18:10.789 | INFO | PerfReport: 3.33%, Metrics -- 1.05e+00 / 1.38e+00, 2.88e-01 / 1.06e-01 | 20260505 18:18:10.789 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.66e-01 / 4.27e-01, 6.49e-02 / 3.70e-02 | 20260505 18:18:10.789 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 18:18:10.789 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:18:10.789 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:18:10.789 | INFO | PerfReport: Full Execution -- 9.98e+01 / -nan, 2.57e+01 / 0.00e+00 | 20260505 18:18:10.789 | INFO | PerfReport: 22.62%, Upload -- 2.26e+01 / 0.00e+00, 7.86e+00 / 0.00e+00 | 20260505 18:18:10.789 | INFO | PerfReport: 21.82%, Download -- 2.18e+01 / 0.00e+00, 7.76e+00 / 0.00e+00 | 20260505 18:18:10.789 | INFO | PerfReport: 31.62%, Compute -- 3.15e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 18:18:10.789 | INFO | PerfReport: 2.37%, LockUpload -- 2.36e+00 / 0.00e+00, 7.97e+00 / 0.00e+00 | 20260505 18:18:10.789 | INFO | PerfReport: 1.38%, LockDownload -- 1.38e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260505 18:18:10.789 | INFO | PerfReport: 20.18%, LockCompute -- 2.01e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 18:18:10.789 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.19e-02 / 0.00e+00 | 20260505 18:18:10.917 | INFO | Starting a round of Baz IO | 20260505 18:18:14.097 | INFO | Populating RTMetrics report | 20260505 18:18:14.097 | INFO | RTMetrics report populated | 20260505 18:18:14.097 | INFO | Opening tmp RTMetrics json file | 20260505 18:18:14.112 | INFO | Serializing RTMetrics to json | 20260505 18:18:14.123 | INFO | Serialization done, swapping tmp file now... | 20260505 18:18:14.136 | INFO | RTMetrics done | 20260505 18:18:14.195 | INFO | Finished a round of Baz IO: Wrote 2.094 GiB in 3.277 seconds | 20260505 18:18:14.195 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.93% / 0.00% / 78.07% | 20260505 18:18:14.195 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.254798% / 3.97364e-06% / 8.74201e-05% | 20260505 18:18:14.195 | INFO | Baz IO overall: | 20260505 18:18:14.195 | INFO | BytesWritten : 3122439326058 | 20260505 18:18:14.195 | INFO | NumEvents : 1479390204354 | 20260505 18:18:15.928 | INFO | Analyzing chunk frames = [5300224,5300736) | 20260505 18:18:15.928 | INFO | MemInfo: { | 20260505 18:18:15.928 | INFO | "gpu" : " 17 GB", | 20260505 18:18:15.928 | INFO | "huge" : "", | 20260505 18:18:15.928 | INFO | "huge_shared" : " 27 GB", | 20260505 18:18:15.928 | INFO | "malloc" : " 14 GB", | 20260505 18:18:15.928 | INFO | "pinned" : " 29 GB" | 20260505 18:18:15.928 | INFO | } | 20260505 18:18:15.928 | INFO | Perf report for the last 5.14065 seconds: | 20260505 18:18:15.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:18:15.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:15.929 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:15.929 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:15.929 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 18:18:15.929 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:15.929 | INFO | RT_METRICS : 45.1%, 1.0, 23.7ms, 0 | 20260505 18:18:15.929 | INFO | PRE_HQ : 60.4%, 1.8, 56.1ms, 0 | 20260505 18:18:15.929 | INFO | BAZWRITER : 63.3%, 3.8, 123.5ms, 0 | 20260505 18:18:15.929 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 18:18:15.929 | INFO | Uploaded 36GiB to GPU in 2.17038s (16.5869GiB/s) | 20260505 18:18:15.929 | INFO | Downloaded 38GiB to GPU in 2.24855s (16.8998GiB/s) | 20260505 18:18:21.062 | INFO | Analyzing chunk frames = [5300736,5301248) | 20260505 18:18:21.062 | INFO | MemInfo: { | 20260505 18:18:21.062 | INFO | "gpu" : " 17 GB", | 20260505 18:18:21.062 | INFO | "huge" : "", | 20260505 18:18:21.062 | INFO | "huge_shared" : " 27 GB", | 20260505 18:18:21.062 | INFO | "malloc" : " 14 GB", | 20260505 18:18:21.062 | INFO | "pinned" : " 29 GB" | 20260505 18:18:21.062 | INFO | } | 20260505 18:18:21.062 | INFO | Perf report for the last 5.13337 seconds: | 20260505 18:18:21.062 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:18:21.062 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:21.062 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:21.062 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:21.062 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260505 18:18:21.062 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:21.062 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:21.062 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:18:21.062 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:21.062 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:21.062 | INFO | Uploaded 36GiB to GPU in 2.12178s (16.9669GiB/s) | 20260505 18:18:21.062 | INFO | Downloaded 34GiB to GPU in 2.02193s (16.8156GiB/s) | 20260505 18:18:26.195 | INFO | Analyzing chunk frames = [5301248,5301760) | 20260505 18:18:26.195 | INFO | MemInfo: { | 20260505 18:18:26.195 | INFO | "gpu" : " 17 GB", | 20260505 18:18:26.195 | INFO | "huge" : "", | 20260505 18:18:26.195 | INFO | "huge_shared" : " 27 GB", | 20260505 18:18:26.195 | INFO | "malloc" : " 14 GB", | 20260505 18:18:26.195 | INFO | "pinned" : " 29 GB" | 20260505 18:18:26.195 | INFO | } | 20260505 18:18:26.195 | INFO | Perf report for the last 5.13331 seconds: | 20260505 18:18:26.195 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:18:26.195 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:26.195 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:26.195 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:26.195 | INFO | ANALYSIS : 62.7%, 3.0, 97.5ms, 0 | 20260505 18:18:26.195 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:26.195 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:26.195 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:18:26.195 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:26.195 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:26.195 | INFO | Uploaded 36GiB to GPU in 2.20197s (16.349GiB/s) | 20260505 18:18:26.196 | INFO | Downloaded 34GiB to GPU in 2.04646s (16.614GiB/s) | 20260505 18:18:31.338 | INFO | Analyzing chunk frames = [5301760,5302272) | 20260505 18:18:31.338 | INFO | MemInfo: { | 20260505 18:18:31.338 | INFO | "gpu" : " 17 GB", | 20260505 18:18:31.338 | INFO | "huge" : "", | 20260505 18:18:31.338 | INFO | "huge_shared" : " 27 GB", | 20260505 18:18:31.338 | INFO | "malloc" : " 14 GB", | 20260505 18:18:31.338 | INFO | "pinned" : " 29 GB" | 20260505 18:18:31.338 | INFO | } | 20260505 18:18:31.338 | INFO | Perf report for the last 5.14326 seconds: | 20260505 18:18:31.338 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:18:31.338 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:31.339 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:31.339 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:31.339 | INFO | ANALYSIS : 62.6%, 3.0, 97.3ms, 0 | 20260505 18:18:31.339 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:31.339 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:31.339 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 18:18:31.339 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:31.339 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:31.339 | INFO | Uploaded 36GiB to GPU in 2.17561s (16.5471GiB/s) | 20260505 18:18:31.339 | INFO | Downloaded 34GiB to GPU in 2.14274s (15.8675GiB/s) | 20260505 18:18:36.484 | INFO | Analyzing chunk frames = [5302272,5302784) | 20260505 18:18:36.484 | INFO | MemInfo: { | 20260505 18:18:36.484 | INFO | "gpu" : " 17 GB", | 20260505 18:18:36.484 | INFO | "huge" : "", | 20260505 18:18:36.484 | INFO | "huge_shared" : " 27 GB", | 20260505 18:18:36.484 | INFO | "malloc" : " 14 GB", | 20260505 18:18:36.484 | INFO | "pinned" : " 29 GB" | 20260505 18:18:36.484 | INFO | } | 20260505 18:18:36.484 | INFO | Perf report for the last 5.14576 seconds: | 20260505 18:18:36.484 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:18:36.484 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:36.484 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:36.484 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:36.484 | INFO | ANALYSIS : 61.9%, 3.0, 96.8ms, 0 | 20260505 18:18:36.484 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:36.484 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:36.484 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:18:36.484 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:36.484 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:36.484 | INFO | Uploaded 36GiB to GPU in 2.13096s (16.8938GiB/s) | 20260505 18:18:36.485 | INFO | Downloaded 34GiB to GPU in 2.0608s (16.4985GiB/s) | 20260505 18:18:41.621 | INFO | Analyzing chunk frames = [5302784,5303296) | 20260505 18:18:41.621 | INFO | MemInfo: { | 20260505 18:18:41.621 | INFO | "gpu" : " 17 GB", | 20260505 18:18:41.621 | INFO | "huge" : "", | 20260505 18:18:41.621 | INFO | "huge_shared" : " 27 GB", | 20260505 18:18:41.621 | INFO | "malloc" : " 14 GB", | 20260505 18:18:41.621 | INFO | "pinned" : " 29 GB" | 20260505 18:18:41.621 | INFO | } | 20260505 18:18:41.622 | INFO | Perf report for the last 5.13736 seconds: | 20260505 18:18:41.622 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:18:41.622 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:41.622 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:41.622 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:41.622 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 18:18:41.622 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:41.622 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:41.622 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:18:41.622 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:41.622 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:41.622 | INFO | Uploaded 36GiB to GPU in 2.12785s (16.9185GiB/s) | 20260505 18:18:41.622 | INFO | Downloaded 34GiB to GPU in 2.053s (16.5612GiB/s) | 20260505 18:18:46.748 | INFO | Analyzing chunk frames = [5303296,5303808) | 20260505 18:18:46.748 | INFO | MemInfo: { | 20260505 18:18:46.748 | INFO | "gpu" : " 17 GB", | 20260505 18:18:46.748 | INFO | "huge" : "", | 20260505 18:18:46.748 | INFO | "huge_shared" : " 27 GB", | 20260505 18:18:46.748 | INFO | "malloc" : " 14 GB", | 20260505 18:18:46.749 | INFO | "pinned" : " 29 GB" | 20260505 18:18:46.749 | INFO | } | 20260505 18:18:46.749 | INFO | Perf report for the last 5.12706 seconds: | 20260505 18:18:46.749 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:18:46.749 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:46.749 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:46.749 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:46.749 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 18:18:46.749 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:46.749 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:46.749 | INFO | PRE_HQ : 28.5%, 1.0, 14.7ms, 0 | 20260505 18:18:46.749 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:46.749 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:46.749 | INFO | Uploaded 36GiB to GPU in 2.14149s (16.8107GiB/s) | 20260505 18:18:46.749 | INFO | Downloaded 34GiB to GPU in 2.1465s (15.8398GiB/s) | 20260505 18:18:51.892 | INFO | Analyzing chunk frames = [5303808,5304320) | 20260505 18:18:51.892 | INFO | MemInfo: { | 20260505 18:18:51.892 | INFO | "gpu" : " 17 GB", | 20260505 18:18:51.892 | INFO | "huge" : "", | 20260505 18:18:51.892 | INFO | "huge_shared" : " 27 GB", | 20260505 18:18:51.892 | INFO | "malloc" : " 14 GB", | 20260505 18:18:51.893 | INFO | "pinned" : " 29 GB" | 20260505 18:18:51.893 | INFO | } | 20260505 18:18:51.893 | INFO | Perf report for the last 5.14399 seconds: | 20260505 18:18:51.893 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:18:51.893 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:51.893 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:51.893 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:51.893 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 18:18:51.893 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:51.893 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:51.893 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260505 18:18:51.893 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:51.893 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:51.893 | INFO | Uploaded 36GiB to GPU in 2.15597s (16.6978GiB/s) | 20260505 18:18:51.893 | INFO | Downloaded 34GiB to GPU in 2.02497s (16.7904GiB/s) | 20260505 18:18:52.034 | INFO | Starting a round of Baz IO | 20260505 18:18:55.215 | INFO | Populating RTMetrics report | 20260505 18:18:55.215 | INFO | RTMetrics report populated | 20260505 18:18:55.215 | INFO | Opening tmp RTMetrics json file | 20260505 18:18:55.216 | INFO | Serializing RTMetrics to json | 20260505 18:18:55.217 | INFO | Serialization done, swapping tmp file now... | 20260505 18:18:55.217 | INFO | RTMetrics done | 20260505 18:18:55.309 | INFO | Finished a round of Baz IO: Wrote 2.096 GiB in 3.274 seconds | 20260505 18:18:55.309 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.93% / 0.00% / 78.07% | 20260505 18:18:55.309 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.255855% / 0% / 0.000119209% | 20260505 18:18:55.309 | INFO | Baz IO overall: | 20260505 18:18:55.309 | INFO | BytesWritten : 3124689964603 | 20260505 18:18:55.309 | INFO | NumEvents : 1480439152335 | 20260505 18:18:57.039 | INFO | Analyzing chunk frames = [5304320,5304832) | 20260505 18:18:57.039 | INFO | MemInfo: { | 20260505 18:18:57.039 | INFO | "gpu" : " 17 GB", | 20260505 18:18:57.039 | INFO | "huge" : "", | 20260505 18:18:57.039 | INFO | "huge_shared" : " 27 GB", | 20260505 18:18:57.039 | INFO | "malloc" : " 14 GB", | 20260505 18:18:57.039 | INFO | "pinned" : " 29 GB" | 20260505 18:18:57.039 | INFO | } | 20260505 18:18:57.039 | INFO | Perf report for the last 5.1468 seconds: | 20260505 18:18:57.039 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:18:57.039 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:57.040 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:57.040 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:57.040 | INFO | ANALYSIS : 63.5%, 3.0, 99.0ms, 0 | 20260505 18:18:57.040 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:18:57.040 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260505 18:18:57.040 | INFO | PRE_HQ : 59.9%, 1.8, 55.8ms, 0 | 20260505 18:18:57.040 | INFO | BAZWRITER : 63.1%, 3.8, 123.2ms, 0 | 20260505 18:18:57.040 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 18:18:57.040 | INFO | Uploaded 36GiB to GPU in 2.15004s (16.7438GiB/s) | 20260505 18:18:57.040 | INFO | Downloaded 38GiB to GPU in 2.25607s (16.8434GiB/s) | 20260505 18:19:02.186 | INFO | Analyzing chunk frames = [5304832,5305344) | 20260505 18:19:02.186 | INFO | MemInfo: { | 20260505 18:19:02.186 | INFO | "gpu" : " 17 GB", | 20260505 18:19:02.186 | INFO | "huge" : "", | 20260505 18:19:02.186 | INFO | "huge_shared" : " 27 GB", | 20260505 18:19:02.186 | INFO | "malloc" : " 14 GB", | 20260505 18:19:02.186 | INFO | "pinned" : " 29 GB" | 20260505 18:19:02.186 | INFO | } | 20260505 18:19:02.186 | INFO | Perf report for the last 5.14634 seconds: | 20260505 18:19:02.186 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:19:02.186 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:02.186 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:02.186 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:02.186 | INFO | ANALYSIS : 62.6%, 3.0, 97.9ms, 0 | 20260505 18:19:02.186 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:02.186 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:02.186 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:19:02.186 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:02.186 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:02.186 | INFO | Uploaded 36GiB to GPU in 2.1227s (16.9595GiB/s) | 20260505 18:19:02.186 | INFO | Downloaded 34GiB to GPU in 2.02522s (16.7883GiB/s) | 20260505 18:19:07.319 | INFO | Analyzing chunk frames = [5305344,5305856) | 20260505 18:19:07.319 | INFO | MemInfo: { | 20260505 18:19:07.319 | INFO | "gpu" : " 17 GB", | 20260505 18:19:07.319 | INFO | "huge" : "", | 20260505 18:19:07.319 | INFO | "huge_shared" : " 27 GB", | 20260505 18:19:07.319 | INFO | "malloc" : " 14 GB", | 20260505 18:19:07.319 | INFO | "pinned" : " 29 GB" | 20260505 18:19:07.319 | INFO | } | 20260505 18:19:07.319 | INFO | Perf report for the last 5.13315 seconds: | 20260505 18:19:07.319 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:19:07.319 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:07.319 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:07.319 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:07.319 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260505 18:19:07.319 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:07.319 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:07.319 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:19:07.319 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:07.319 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:07.319 | INFO | Uploaded 36GiB to GPU in 2.1274s (16.922GiB/s) | 20260505 18:19:07.319 | INFO | Downloaded 34GiB to GPU in 2.0266s (16.7768GiB/s) | 20260505 18:19:12.455 | INFO | Analyzing chunk frames = [5305856,5306368) | 20260505 18:19:12.455 | INFO | MemInfo: { | 20260505 18:19:12.455 | INFO | "gpu" : " 17 GB", | 20260505 18:19:12.455 | INFO | "huge" : "", | 20260505 18:19:12.455 | INFO | "huge_shared" : " 27 GB", | 20260505 18:19:12.455 | INFO | "malloc" : " 14 GB", | 20260505 18:19:12.455 | INFO | "pinned" : " 29 GB" | 20260505 18:19:12.455 | INFO | } | 20260505 18:19:12.455 | INFO | Perf report for the last 5.13616 seconds: | 20260505 18:19:12.455 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:19:12.455 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:12.455 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:12.455 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:12.455 | INFO | ANALYSIS : 62.9%, 3.0, 97.6ms, 0 | 20260505 18:19:12.455 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:12.455 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:12.455 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 18:19:12.455 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:12.455 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:12.455 | INFO | Uploaded 36GiB to GPU in 2.22488s (16.1806GiB/s) | 20260505 18:19:12.455 | INFO | Downloaded 34GiB to GPU in 2.12207s (16.0221GiB/s) | 20260505 18:19:17.599 | INFO | Analyzing chunk frames = [5306368,5306880) | 20260505 18:19:17.599 | INFO | MemInfo: { | 20260505 18:19:17.599 | INFO | "gpu" : " 17 GB", | 20260505 18:19:17.599 | INFO | "huge" : "", | 20260505 18:19:17.599 | INFO | "huge_shared" : " 27 GB", | 20260505 18:19:17.599 | INFO | "malloc" : " 14 GB", | 20260505 18:19:17.599 | INFO | "pinned" : " 29 GB" | 20260505 18:19:17.599 | INFO | } | 20260505 18:19:17.599 | INFO | Perf report for the last 5.14414 seconds: | 20260505 18:19:17.599 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:19:17.599 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:17.599 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:17.599 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:17.599 | INFO | ANALYSIS : 61.9%, 3.0, 96.8ms, 0 | 20260505 18:19:17.599 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:17.599 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:17.599 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 18:19:17.599 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:17.599 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:17.600 | INFO | Uploaded 36GiB to GPU in 2.13872s (16.8325GiB/s) | 20260505 18:19:17.600 | INFO | Downloaded 34GiB to GPU in 2.061s (16.4968GiB/s) | 20260505 18:19:22.732 | INFO | Analyzing chunk frames = [5306880,5307392) | 20260505 18:19:22.732 | INFO | MemInfo: { | 20260505 18:19:22.732 | INFO | "gpu" : " 17 GB", | 20260505 18:19:22.732 | INFO | "huge" : "", | 20260505 18:19:22.732 | INFO | "huge_shared" : " 27 GB", | 20260505 18:19:22.732 | INFO | "malloc" : " 14 GB", | 20260505 18:19:22.732 | INFO | "pinned" : " 29 GB" | 20260505 18:19:22.732 | INFO | } | 20260505 18:19:22.732 | INFO | Perf report for the last 5.13303 seconds: | 20260505 18:19:22.732 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:19:22.732 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:22.732 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:22.732 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:22.732 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260505 18:19:22.732 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:22.732 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:22.732 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 18:19:22.733 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:22.733 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:22.733 | INFO | Uploaded 36GiB to GPU in 2.13349s (16.8737GiB/s) | 20260505 18:19:22.733 | INFO | Downloaded 34GiB to GPU in 2.05323s (16.5593GiB/s) | 20260505 18:19:27.876 | INFO | Analyzing chunk frames = [5307392,5307904) | 20260505 18:19:27.876 | INFO | MemInfo: { | 20260505 18:19:27.876 | INFO | "gpu" : " 17 GB", | 20260505 18:19:27.876 | INFO | "huge" : "", | 20260505 18:19:27.876 | INFO | "huge_shared" : " 27 GB", | 20260505 18:19:27.876 | INFO | "malloc" : " 14 GB", | 20260505 18:19:27.876 | INFO | "pinned" : " 29 GB" | 20260505 18:19:27.876 | INFO | } | 20260505 18:19:27.876 | INFO | Perf report for the last 5.14379 seconds: | 20260505 18:19:27.876 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:19:27.876 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:27.876 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:27.876 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:27.876 | INFO | ANALYSIS : 63.5%, 3.0, 99.2ms, 0 | 20260505 18:19:27.876 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:27.876 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:27.876 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 18:19:27.876 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:27.876 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:27.877 | INFO | Uploaded 36GiB to GPU in 2.15864s (16.6771GiB/s) | 20260505 18:19:27.877 | INFO | Downloaded 34GiB to GPU in 2.16222s (15.7246GiB/s) | 20260505 18:19:33.016 | INFO | Analyzing chunk frames = [5307904,5308416) | 20260505 18:19:33.016 | INFO | MemInfo: { | 20260505 18:19:33.016 | INFO | "gpu" : " 17 GB", | 20260505 18:19:33.016 | INFO | "huge" : "", | 20260505 18:19:33.016 | INFO | "huge_shared" : " 27 GB", | 20260505 18:19:33.016 | INFO | "malloc" : " 14 GB", | 20260505 18:19:33.016 | INFO | "pinned" : " 29 GB" | 20260505 18:19:33.016 | INFO | } | 20260505 18:19:33.017 | INFO | Perf report for the last 5.14037 seconds: | 20260505 18:19:33.017 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:19:33.017 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:33.017 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:33.017 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:33.017 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 18:19:33.017 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:33.017 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:33.017 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 18:19:33.017 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:33.017 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:33.017 | INFO | Uploaded 36GiB to GPU in 2.20282s (16.3427GiB/s) | 20260505 18:19:33.017 | INFO | Downloaded 34GiB to GPU in 2.03381s (16.7174GiB/s) | 20260505 18:19:33.017 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 18:19:33.017 | INFO | Total mem usage: 15.3 GiB | 20260505 18:19:33.017 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 18:19:33.017 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.01%) | 20260505 18:19:33.017 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 18:19:33.017 | INFO | High Water Breakdown: | 20260505 18:19:33.017 | INFO | 61% : 8.66 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 18:19:33.017 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 18:19:33.017 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 18:19:33.017 | INFO | ------------------------------------------ | 20260505 18:19:33.017 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:19:33.017 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 18:19:33.017 | INFO | Total mem usage: 26.5 GiB | 20260505 18:19:33.017 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 18:19:33.017 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 18:19:33.017 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 18:19:33.017 | INFO | High Water Breakdown: | 20260505 18:19:33.017 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:19:33.017 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:19:33.017 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:19:33.017 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:19:33.017 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:19:33.017 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:19:33.017 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 18:19:33.017 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 18:19:33.017 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:19:33.017 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 18:19:33.017 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 18:19:33.017 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 18:19:33.017 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 18:19:33.017 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 18:19:33.017 | INFO | ------------------------------------------ | 20260505 18:19:33.017 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:19:33.017 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 18:19:33.018 | INFO | Total mem usage: 25.6 GiB | 20260505 18:19:33.018 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 18:19:33.018 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 18:19:33.018 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 18:19:33.018 | INFO | High Water Breakdown: | 20260505 18:19:33.018 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 18:19:33.018 | INFO | ------------------------------------------ | 20260505 18:19:33.018 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:19:33.018 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 18:19:33.018 | INFO | Total mem usage: 15.7 GiB | 20260505 18:19:33.018 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 18:19:33.018 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 18:19:33.018 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 18:19:33.018 | INFO | High Water Breakdown: | 20260505 18:19:33.018 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 18:19:33.018 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 18:19:33.018 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 18:19:33.018 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 18:19:33.018 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 18:19:33.018 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 18:19:33.018 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 18:19:33.018 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 18:19:33.018 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 18:19:33.018 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:19:33.018 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 18:19:33.018 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:19:33.018 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 18:19:33.018 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 18:19:33.018 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 18:19:33.018 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 18:19:33.018 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:19:33.018 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 18:19:33.018 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:19:33.018 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:19:33.018 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:19:33.018 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 18:19:33.018 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 18:19:33.018 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:19:33.018 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 18:19:33.018 | INFO | ------------------------------------------ | 20260505 18:19:33.018 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:19:33.018 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 18:19:33.018 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:19:33.018 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:19:33.018 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.50e+01, 1.35e+01 / 2.33e+00 | 20260505 18:19:33.018 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 1.02e-01, 1.07e-02 / 6.04e-03 | 20260505 18:19:33.018 | INFO | PerfReport: 11.80%, Baseline -- 3.72e+00 / 4.52e+00, 5.71e-01 / 1.46e-01 | 20260505 18:19:33.018 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.52e+00, 1.90e-01 / 9.10e-02 | 20260505 18:19:33.018 | INFO | PerfReport: 15.98%, DME -- 4.04e+01 / 4.49e+01, 4.26e+00 / 2.33e+00 | 20260505 18:19:33.018 | INFO | PerfReport: 56.86%, FrameLabeling -- 1.80e+01 / 1.93e+01, 1.46e+00 / 1.10e+00 | 20260505 18:19:33.018 | INFO | PerfReport: 7.30%, PulseAccumulating -- 2.31e+00 / 2.91e+00, 4.44e-01 / 1.67e-01 | 20260505 18:19:33.018 | INFO | PerfReport: 3.33%, Metrics -- 1.05e+00 / 1.39e+00, 2.90e-01 / 1.55e-01 | 20260505 18:19:33.018 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.66e-01 / 4.25e-01, 6.45e-02 / 3.78e-02 | 20260505 18:19:33.018 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 18:19:33.018 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:19:33.018 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:19:33.018 | INFO | PerfReport: Full Execution -- 9.98e+01 / -nan, 2.55e+01 / 0.00e+00 | 20260505 18:19:33.018 | INFO | PerfReport: 22.49%, Upload -- 2.25e+01 / 0.00e+00, 7.74e+00 / 0.00e+00 | 20260505 18:19:33.018 | INFO | PerfReport: 21.84%, Download -- 2.18e+01 / 0.00e+00, 7.75e+00 / 0.00e+00 | 20260505 18:19:33.018 | INFO | PerfReport: 31.64%, Compute -- 3.16e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 18:19:33.018 | INFO | PerfReport: 2.25%, LockUpload -- 2.25e+00 / 0.00e+00, 7.81e+00 / 0.00e+00 | 20260505 18:19:33.018 | INFO | PerfReport: 1.42%, LockDownload -- 1.42e+00 / 0.00e+00, 2.96e+00 / 0.00e+00 | 20260505 18:19:33.018 | INFO | PerfReport: 20.34%, LockCompute -- 2.03e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 18:19:33.018 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.06e-02 / 0.00e+00 | 20260505 18:19:33.151 | INFO | Starting a round of Baz IO | 20260505 18:19:36.329 | INFO | Populating RTMetrics report | 20260505 18:19:36.329 | INFO | RTMetrics report populated | 20260505 18:19:36.329 | INFO | Opening tmp RTMetrics json file | 20260505 18:19:36.333 | INFO | Serializing RTMetrics to json | 20260505 18:19:36.335 | INFO | Serialization done, swapping tmp file now... | 20260505 18:19:36.337 | INFO | RTMetrics done | 20260505 18:19:36.423 | INFO | Finished a round of Baz IO: Wrote 2.096 GiB in 3.272 seconds | 20260505 18:19:36.423 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.92% / 0.00% / 78.07% | 20260505 18:19:36.423 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.255175% / 7.94729e-06% / 9.13938e-05% | 20260505 18:19:36.423 | INFO | Baz IO overall: | 20260505 18:19:36.423 | INFO | BytesWritten : 3126940798244 | 20260505 18:19:36.423 | INFO | NumEvents : 1481488501422 | 20260505 18:19:38.146 | INFO | Analyzing chunk frames = [5308416,5308928) | 20260505 18:19:38.146 | INFO | MemInfo: { | 20260505 18:19:38.146 | INFO | "gpu" : " 17 GB", | 20260505 18:19:38.146 | INFO | "huge" : "", | 20260505 18:19:38.146 | INFO | "huge_shared" : " 27 GB", | 20260505 18:19:38.146 | INFO | "malloc" : " 14 GB", | 20260505 18:19:38.146 | INFO | "pinned" : " 29 GB" | 20260505 18:19:38.146 | INFO | } | 20260505 18:19:38.146 | INFO | Perf report for the last 5.12923 seconds: | 20260505 18:19:38.146 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:19:38.146 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:38.146 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:38.146 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:38.146 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260505 18:19:38.146 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:38.146 | INFO | RT_METRICS : 44.6%, 1.0, 23.3ms, 0 | 20260505 18:19:38.146 | INFO | PRE_HQ : 60.3%, 1.8, 55.8ms, 0 | 20260505 18:19:38.146 | INFO | BAZWRITER : 63.4%, 3.8, 124.0ms, 0 | 20260505 18:19:38.146 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 18:19:38.146 | INFO | Uploaded 36GiB to GPU in 2.15767s (16.6847GiB/s) | 20260505 18:19:38.146 | INFO | Downloaded 38GiB to GPU in 2.23696s (16.9874GiB/s) | 20260505 18:19:43.294 | INFO | Analyzing chunk frames = [5308928,5309440) | 20260505 18:19:43.294 | INFO | MemInfo: { | 20260505 18:19:43.294 | INFO | "gpu" : " 17 GB", | 20260505 18:19:43.294 | INFO | "huge" : "", | 20260505 18:19:43.294 | INFO | "huge_shared" : " 27 GB", | 20260505 18:19:43.294 | INFO | "malloc" : " 14 GB", | 20260505 18:19:43.295 | INFO | "pinned" : " 29 GB" | 20260505 18:19:43.295 | INFO | } | 20260505 18:19:43.295 | INFO | Perf report for the last 5.14882 seconds: | 20260505 18:19:43.295 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:19:43.295 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:43.295 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:43.295 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:43.295 | INFO | ANALYSIS : 62.7%, 3.0, 98.1ms, 0 | 20260505 18:19:43.295 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:43.295 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:43.295 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:19:43.295 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:43.295 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:43.295 | INFO | Uploaded 36GiB to GPU in 2.13627s (16.8518GiB/s) | 20260505 18:19:43.295 | INFO | Downloaded 34GiB to GPU in 2.03677s (16.6931GiB/s) | 20260505 18:19:48.436 | INFO | Analyzing chunk frames = [5309440,5309952) | 20260505 18:19:48.436 | INFO | MemInfo: { | 20260505 18:19:48.436 | INFO | "gpu" : " 17 GB", | 20260505 18:19:48.436 | INFO | "huge" : "", | 20260505 18:19:48.436 | INFO | "huge_shared" : " 27 GB", | 20260505 18:19:48.436 | INFO | "malloc" : " 14 GB", | 20260505 18:19:48.437 | INFO | "pinned" : " 29 GB" | 20260505 18:19:48.437 | INFO | } | 20260505 18:19:48.437 | INFO | Perf report for the last 5.14201 seconds: | 20260505 18:19:48.437 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:19:48.437 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:48.437 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:48.437 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:48.437 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260505 18:19:48.437 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:48.437 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:48.437 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:19:48.437 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:48.437 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:48.437 | INFO | Uploaded 36GiB to GPU in 2.13733s (16.8435GiB/s) | 20260505 18:19:48.437 | INFO | Downloaded 34GiB to GPU in 2.04322s (16.6404GiB/s) | 20260505 18:19:53.564 | INFO | Analyzing chunk frames = [5309952,5310464) | 20260505 18:19:53.564 | INFO | MemInfo: { | 20260505 18:19:53.564 | INFO | "gpu" : " 17 GB", | 20260505 18:19:53.564 | INFO | "huge" : "", | 20260505 18:19:53.564 | INFO | "huge_shared" : " 27 GB", | 20260505 18:19:53.564 | INFO | "malloc" : " 14 GB", | 20260505 18:19:53.565 | INFO | "pinned" : " 29 GB" | 20260505 18:19:53.565 | INFO | } | 20260505 18:19:53.565 | INFO | Perf report for the last 5.12798 seconds: | 20260505 18:19:53.565 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:19:53.565 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:53.565 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:53.565 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:53.565 | INFO | ANALYSIS : 62.5%, 3.0, 96.8ms, 0 | 20260505 18:19:53.565 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:53.565 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:53.565 | INFO | PRE_HQ : 27.5%, 1.0, 14.2ms, 0 | 20260505 18:19:53.565 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:53.565 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:53.565 | INFO | Uploaded 36GiB to GPU in 2.19845s (16.3752GiB/s) | 20260505 18:19:53.565 | INFO | Downloaded 34GiB to GPU in 2.03954s (16.6705GiB/s) | 20260505 18:19:58.698 | INFO | Analyzing chunk frames = [5310464,5310976) | 20260505 18:19:58.698 | INFO | MemInfo: { | 20260505 18:19:58.698 | INFO | "gpu" : " 17 GB", | 20260505 18:19:58.698 | INFO | "huge" : "", | 20260505 18:19:58.698 | INFO | "huge_shared" : " 27 GB", | 20260505 18:19:58.698 | INFO | "malloc" : " 14 GB", | 20260505 18:19:58.698 | INFO | "pinned" : " 29 GB" | 20260505 18:19:58.698 | INFO | } | 20260505 18:19:58.698 | INFO | Perf report for the last 5.13348 seconds: | 20260505 18:19:58.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:19:58.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:58.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:58.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:58.698 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 18:19:58.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:58.698 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:58.698 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:19:58.698 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:58.698 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:19:58.698 | INFO | Uploaded 36GiB to GPU in 2.15512s (16.7044GiB/s) | 20260505 18:19:58.698 | INFO | Downloaded 34GiB to GPU in 2.05669s (16.5314GiB/s) | 20260505 18:20:03.846 | INFO | Analyzing chunk frames = [5310976,5311488) | 20260505 18:20:03.846 | INFO | MemInfo: { | 20260505 18:20:03.846 | INFO | "gpu" : " 17 GB", | 20260505 18:20:03.846 | INFO | "huge" : "", | 20260505 18:20:03.846 | INFO | "huge_shared" : " 27 GB", | 20260505 18:20:03.846 | INFO | "malloc" : " 14 GB", | 20260505 18:20:03.846 | INFO | "pinned" : " 29 GB" | 20260505 18:20:03.846 | INFO | } | 20260505 18:20:03.846 | INFO | Perf report for the last 5.14796 seconds: | 20260505 18:20:03.846 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:20:03.846 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:03.846 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:03.846 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:03.846 | INFO | ANALYSIS : 62.4%, 3.0, 97.6ms, 0 | 20260505 18:20:03.846 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:03.846 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:03.846 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 18:20:03.846 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:03.846 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:03.846 | INFO | Uploaded 36GiB to GPU in 2.15114s (16.7353GiB/s) | 20260505 18:20:03.846 | INFO | Downloaded 34GiB to GPU in 2.04409s (16.6333GiB/s) | 20260505 18:20:08.991 | INFO | Analyzing chunk frames = [5311488,5312000) | 20260505 18:20:08.991 | INFO | MemInfo: { | 20260505 18:20:08.991 | INFO | "gpu" : " 17 GB", | 20260505 18:20:08.991 | INFO | "huge" : "", | 20260505 18:20:08.992 | INFO | "huge_shared" : " 27 GB", | 20260505 18:20:08.992 | INFO | "malloc" : " 14 GB", | 20260505 18:20:08.992 | INFO | "pinned" : " 29 GB" | 20260505 18:20:08.992 | INFO | } | 20260505 18:20:08.992 | INFO | Perf report for the last 5.14562 seconds: | 20260505 18:20:08.992 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:20:08.992 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:08.992 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:08.992 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:08.992 | INFO | ANALYSIS : 63.3%, 3.0, 99.0ms, 0 | 20260505 18:20:08.992 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:08.992 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:08.992 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260505 18:20:08.992 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:08.992 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:08.992 | INFO | Uploaded 36GiB to GPU in 2.1428s (16.8004GiB/s) | 20260505 18:20:08.992 | INFO | Downloaded 34GiB to GPU in 2.14159s (15.8761GiB/s) | 20260505 18:20:14.132 | INFO | Analyzing chunk frames = [5312000,5312512) | 20260505 18:20:14.132 | INFO | MemInfo: { | 20260505 18:20:14.132 | INFO | "gpu" : " 17 GB", | 20260505 18:20:14.132 | INFO | "huge" : "", | 20260505 18:20:14.132 | INFO | "huge_shared" : " 27 GB", | 20260505 18:20:14.132 | INFO | "malloc" : " 14 GB", | 20260505 18:20:14.132 | INFO | "pinned" : " 29 GB" | 20260505 18:20:14.132 | INFO | } | 20260505 18:20:14.132 | INFO | Perf report for the last 5.14051 seconds: | 20260505 18:20:14.132 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:20:14.132 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:14.132 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:14.132 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:14.132 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 18:20:14.132 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:14.132 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:14.132 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:20:14.132 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:14.132 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:14.133 | INFO | Uploaded 36GiB to GPU in 2.19329s (16.4137GiB/s) | 20260505 18:20:14.133 | INFO | Downloaded 34GiB to GPU in 2.0299s (16.7496GiB/s) | 20260505 18:20:14.264 | INFO | Starting a round of Baz IO | 20260505 18:20:17.446 | INFO | Populating RTMetrics report | 20260505 18:20:17.446 | INFO | RTMetrics report populated | 20260505 18:20:17.446 | INFO | Opening tmp RTMetrics json file | 20260505 18:20:17.449 | INFO | Serializing RTMetrics to json | 20260505 18:20:17.451 | INFO | Serialization done, swapping tmp file now... | 20260505 18:20:17.451 | INFO | RTMetrics done | 20260505 18:20:17.538 | INFO | Finished a round of Baz IO: Wrote 2.096 GiB in 3.273 seconds | 20260505 18:20:17.538 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.92% / 0.00% / 78.08% | 20260505 18:20:17.538 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.254186% / 3.97364e-06% / 9.93411e-05% | 20260505 18:20:17.539 | INFO | Baz IO overall: | 20260505 18:20:17.539 | INFO | BytesWritten : 3129191156032 | 20260505 18:20:17.539 | INFO | NumEvents : 1482537499682 | 20260505 18:20:19.264 | INFO | Analyzing chunk frames = [5312512,5313024) | 20260505 18:20:19.264 | INFO | MemInfo: { | 20260505 18:20:19.264 | INFO | "gpu" : " 17 GB", | 20260505 18:20:19.264 | INFO | "huge" : "", | 20260505 18:20:19.264 | INFO | "huge_shared" : " 27 GB", | 20260505 18:20:19.264 | INFO | "malloc" : " 14 GB", | 20260505 18:20:19.264 | INFO | "pinned" : " 29 GB" | 20260505 18:20:19.264 | INFO | } | 20260505 18:20:19.264 | INFO | Perf report for the last 5.13186 seconds: | 20260505 18:20:19.264 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:20:19.264 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:19.264 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:19.264 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:19.264 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260505 18:20:19.264 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:19.264 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260505 18:20:19.264 | INFO | PRE_HQ : 60.6%, 1.8, 55.9ms, 0 | 20260505 18:20:19.264 | INFO | BAZWRITER : 63.3%, 3.8, 123.9ms, 0 | 20260505 18:20:19.264 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 18:20:19.264 | INFO | Uploaded 36GiB to GPU in 2.16115s (16.6578GiB/s) | 20260505 18:20:19.265 | INFO | Downloaded 38GiB to GPU in 2.24283s (16.9429GiB/s) | 20260505 18:20:24.405 | INFO | Analyzing chunk frames = [5313024,5313536) | 20260505 18:20:24.405 | INFO | MemInfo: { | 20260505 18:20:24.405 | INFO | "gpu" : " 17 GB", | 20260505 18:20:24.405 | INFO | "huge" : "", | 20260505 18:20:24.406 | INFO | "huge_shared" : " 27 GB", | 20260505 18:20:24.406 | INFO | "malloc" : " 14 GB", | 20260505 18:20:24.406 | INFO | "pinned" : " 29 GB" | 20260505 18:20:24.406 | INFO | } | 20260505 18:20:24.406 | INFO | Perf report for the last 5.14152 seconds: | 20260505 18:20:24.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:20:24.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:24.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:24.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:24.406 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 18:20:24.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:24.406 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:24.406 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:20:24.406 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:24.406 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:24.406 | INFO | Uploaded 36GiB to GPU in 2.12856s (16.9128GiB/s) | 20260505 18:20:24.406 | INFO | Downloaded 34GiB to GPU in 2.02458s (16.7936GiB/s) | 20260505 18:20:29.552 | INFO | Analyzing chunk frames = [5313536,5314048) | 20260505 18:20:29.552 | INFO | MemInfo: { | 20260505 18:20:29.552 | INFO | "gpu" : " 17 GB", | 20260505 18:20:29.552 | INFO | "huge" : "", | 20260505 18:20:29.552 | INFO | "huge_shared" : " 27 GB", | 20260505 18:20:29.552 | INFO | "malloc" : " 14 GB", | 20260505 18:20:29.552 | INFO | "pinned" : " 29 GB" | 20260505 18:20:29.552 | INFO | } | 20260505 18:20:29.552 | INFO | Perf report for the last 5.14643 seconds: | 20260505 18:20:29.552 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:20:29.552 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:29.552 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:29.552 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:29.552 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 18:20:29.552 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:29.552 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:29.552 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260505 18:20:29.552 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:29.552 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:29.552 | INFO | Uploaded 36GiB to GPU in 2.21061s (16.2851GiB/s) | 20260505 18:20:29.554 | INFO | Downloaded 34GiB to GPU in 2.074s (16.3935GiB/s) | 20260505 18:20:34.687 | INFO | Analyzing chunk frames = [5314048,5314560) | 20260505 18:20:34.687 | INFO | MemInfo: { | 20260505 18:20:34.687 | INFO | "gpu" : " 17 GB", | 20260505 18:20:34.687 | INFO | "huge" : "", | 20260505 18:20:34.687 | INFO | "huge_shared" : " 27 GB", | 20260505 18:20:34.687 | INFO | "malloc" : " 14 GB", | 20260505 18:20:34.687 | INFO | "pinned" : " 29 GB" | 20260505 18:20:34.687 | INFO | } | 20260505 18:20:34.687 | INFO | Perf report for the last 5.13503 seconds: | 20260505 18:20:34.687 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:20:34.687 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:34.687 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:34.687 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:34.687 | INFO | ANALYSIS : 62.4%, 3.0, 96.9ms, 0 | 20260505 18:20:34.687 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:34.687 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:34.687 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:20:34.687 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:34.687 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:34.687 | INFO | Uploaded 36GiB to GPU in 2.19868s (16.3735GiB/s) | 20260505 18:20:34.687 | INFO | Downloaded 34GiB to GPU in 2.03282s (16.7255GiB/s) | 20260505 18:20:39.827 | INFO | Analyzing chunk frames = [5314560,5315072) | 20260505 18:20:39.827 | INFO | MemInfo: { | 20260505 18:20:39.827 | INFO | "gpu" : " 17 GB", | 20260505 18:20:39.827 | INFO | "huge" : "", | 20260505 18:20:39.827 | INFO | "huge_shared" : " 27 GB", | 20260505 18:20:39.827 | INFO | "malloc" : " 14 GB", | 20260505 18:20:39.827 | INFO | "pinned" : " 29 GB" | 20260505 18:20:39.827 | INFO | } | 20260505 18:20:39.827 | INFO | Perf report for the last 5.13967 seconds: | 20260505 18:20:39.827 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:20:39.827 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:39.827 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:39.827 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:39.827 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260505 18:20:39.827 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:39.827 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:39.827 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:20:39.827 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:39.827 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:39.827 | INFO | Uploaded 36GiB to GPU in 2.13821s (16.8365GiB/s) | 20260505 18:20:39.827 | INFO | Downloaded 34GiB to GPU in 2.05565s (16.5398GiB/s) | 20260505 18:20:44.962 | INFO | Analyzing chunk frames = [5315072,5315584) | 20260505 18:20:44.962 | INFO | MemInfo: { | 20260505 18:20:44.962 | INFO | "gpu" : " 17 GB", | 20260505 18:20:44.962 | INFO | "huge" : "", | 20260505 18:20:44.962 | INFO | "huge_shared" : " 27 GB", | 20260505 18:20:44.962 | INFO | "malloc" : " 14 GB", | 20260505 18:20:44.962 | INFO | "pinned" : " 29 GB" | 20260505 18:20:44.962 | INFO | } | 20260505 18:20:44.962 | INFO | Perf report for the last 5.13514 seconds: | 20260505 18:20:44.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:20:44.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:44.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:44.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:44.962 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260505 18:20:44.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:44.962 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:44.962 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 18:20:44.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:44.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:44.962 | INFO | Uploaded 36GiB to GPU in 2.13896s (16.8306GiB/s) | 20260505 18:20:44.962 | INFO | Downloaded 34GiB to GPU in 2.0453s (16.6235GiB/s) | 20260505 18:20:50.101 | INFO | Analyzing chunk frames = [5315584,5316096) | 20260505 18:20:50.101 | INFO | MemInfo: { | 20260505 18:20:50.101 | INFO | "gpu" : " 17 GB", | 20260505 18:20:50.101 | INFO | "huge" : "", | 20260505 18:20:50.101 | INFO | "huge_shared" : " 27 GB", | 20260505 18:20:50.101 | INFO | "malloc" : " 14 GB", | 20260505 18:20:50.101 | INFO | "pinned" : " 29 GB" | 20260505 18:20:50.101 | INFO | } | 20260505 18:20:50.101 | INFO | Perf report for the last 5.13937 seconds: | 20260505 18:20:50.101 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:20:50.101 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:50.101 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:50.101 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:50.101 | INFO | ANALYSIS : 63.1%, 3.0, 97.6ms, 0 | 20260505 18:20:50.101 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:50.101 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:50.101 | INFO | PRE_HQ : 28.1%, 1.0, 14.5ms, 0 | 20260505 18:20:50.102 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:50.102 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:50.102 | INFO | Uploaded 36GiB to GPU in 2.13215s (16.8844GiB/s) | 20260505 18:20:50.102 | INFO | Downloaded 34GiB to GPU in 2.1438s (15.8597GiB/s) | 20260505 18:20:55.246 | INFO | Analyzing chunk frames = [5316096,5316608) | 20260505 18:20:55.246 | INFO | MemInfo: { | 20260505 18:20:55.246 | INFO | "gpu" : " 17 GB", | 20260505 18:20:55.246 | INFO | "huge" : "", | 20260505 18:20:55.246 | INFO | "huge_shared" : " 27 GB", | 20260505 18:20:55.246 | INFO | "malloc" : " 14 GB", | 20260505 18:20:55.246 | INFO | "pinned" : " 29 GB" | 20260505 18:20:55.247 | INFO | } | 20260505 18:20:55.247 | INFO | Perf report for the last 5.14534 seconds: | 20260505 18:20:55.247 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:20:55.247 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:55.247 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:55.247 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:55.247 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 18:20:55.247 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:55.247 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:55.247 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:20:55.247 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:55.247 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:20:55.247 | INFO | Uploaded 36GiB to GPU in 2.19918s (16.3698GiB/s) | 20260505 18:20:55.247 | INFO | Downloaded 34GiB to GPU in 2.02157s (16.8186GiB/s) | 20260505 18:20:55.247 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 18:20:55.247 | INFO | Total mem usage: 15.3 GiB | 20260505 18:20:55.247 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 18:20:55.247 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.01%) | 20260505 18:20:55.247 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 18:20:55.247 | INFO | High Water Breakdown: | 20260505 18:20:55.247 | INFO | 61% : 8.66 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 18:20:55.247 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 18:20:55.247 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 18:20:55.247 | INFO | ------------------------------------------ | 20260505 18:20:55.247 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:20:55.247 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 18:20:55.247 | INFO | Total mem usage: 26.5 GiB | 20260505 18:20:55.247 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 18:20:55.247 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 18:20:55.247 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 18:20:55.247 | INFO | High Water Breakdown: | 20260505 18:20:55.247 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:20:55.247 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:20:55.247 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:20:55.247 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:20:55.247 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:20:55.247 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:20:55.247 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 18:20:55.248 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 18:20:55.248 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:20:55.248 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 18:20:55.248 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 18:20:55.248 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 18:20:55.248 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 18:20:55.248 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 18:20:55.248 | INFO | ------------------------------------------ | 20260505 18:20:55.248 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:20:55.248 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 18:20:55.248 | INFO | Total mem usage: 25.6 GiB | 20260505 18:20:55.248 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 18:20:55.248 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 18:20:55.248 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 18:20:55.248 | INFO | High Water Breakdown: | 20260505 18:20:55.248 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 18:20:55.248 | INFO | ------------------------------------------ | 20260505 18:20:55.248 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:20:55.248 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 18:20:55.248 | INFO | Total mem usage: 15.7 GiB | 20260505 18:20:55.248 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 18:20:55.248 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 18:20:55.248 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 18:20:55.248 | INFO | High Water Breakdown: | 20260505 18:20:55.248 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 18:20:55.248 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 18:20:55.248 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 18:20:55.248 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 18:20:55.248 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 18:20:55.248 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 18:20:55.248 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 18:20:55.248 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 18:20:55.248 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 18:20:55.248 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:20:55.248 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 18:20:55.248 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:20:55.248 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 18:20:55.248 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 18:20:55.248 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 18:20:55.248 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 18:20:55.248 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:20:55.248 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 18:20:55.248 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:20:55.248 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:20:55.248 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:20:55.248 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 18:20:55.248 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 18:20:55.248 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:20:55.248 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 18:20:55.248 | INFO | ------------------------------------------ | 20260505 18:20:55.248 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:20:55.248 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 18:20:55.248 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:20:55.248 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:20:55.248 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.42e+01, 1.31e+01 / 1.62e+00 | 20260505 18:20:55.248 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 1.02e-01, 1.10e-02 / 2.74e-03 | 20260505 18:20:55.248 | INFO | PerfReport: 11.82%, Baseline -- 3.73e+00 / 4.51e+00, 5.78e-01 / 2.16e-01 | 20260505 18:20:55.248 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.54e+00, 1.91e-01 / 9.07e-02 | 20260505 18:20:55.248 | INFO | PerfReport: 15.92%, DME -- 4.02e+01 / 4.42e+01, 3.96e+00 / 1.70e+00 | 20260505 18:20:55.248 | INFO | PerfReport: 56.86%, FrameLabeling -- 1.79e+01 / 1.91e+01, 1.48e+00 / 8.03e-01 | 20260505 18:20:55.248 | INFO | PerfReport: 7.32%, PulseAccumulating -- 2.31e+00 / 2.92e+00, 4.44e-01 / 1.39e-01 | 20260505 18:20:55.248 | INFO | PerfReport: 3.34%, Metrics -- 1.05e+00 / 1.39e+00, 2.91e-01 / 1.38e-01 | 20260505 18:20:55.248 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.67e-01 / 4.29e-01, 6.55e-02 / 3.17e-02 | 20260505 18:20:55.248 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 18:20:55.248 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:20:55.248 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:20:55.248 | INFO | PerfReport: Full Execution -- 9.98e+01 / -nan, 2.57e+01 / 0.00e+00 | 20260505 18:20:55.248 | INFO | PerfReport: 22.56%, Upload -- 2.25e+01 / 0.00e+00, 7.81e+00 / 0.00e+00 | 20260505 18:20:55.248 | INFO | PerfReport: 21.70%, Download -- 2.17e+01 / 0.00e+00, 7.68e+00 / 0.00e+00 | 20260505 18:20:55.248 | INFO | PerfReport: 31.63%, Compute -- 3.16e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260505 18:20:55.248 | INFO | PerfReport: 2.36%, LockUpload -- 2.35e+00 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260505 18:20:55.248 | INFO | PerfReport: 1.36%, LockDownload -- 1.36e+00 / 0.00e+00, 2.84e+00 / 0.00e+00 | 20260505 18:20:55.248 | INFO | PerfReport: 20.37%, LockCompute -- 2.03e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 18:20:55.248 | INFO | PerfReport: 0.02%, Misc -- 1.68e-02 / 0.00e+00, 7.19e-02 / 0.00e+00 | 20260505 18:20:55.379 | INFO | Starting a round of Baz IO | 20260505 18:20:58.553 | INFO | Populating RTMetrics report | 20260505 18:20:58.554 | INFO | RTMetrics report populated | 20260505 18:20:58.554 | INFO | Opening tmp RTMetrics json file | 20260505 18:20:58.556 | INFO | Serializing RTMetrics to json | 20260505 18:20:58.558 | INFO | Serialization done, swapping tmp file now... | 20260505 18:20:58.559 | INFO | RTMetrics done | 20260505 18:20:58.645 | INFO | Finished a round of Baz IO: Wrote 2.095 GiB in 3.265 seconds | 20260505 18:20:58.645 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.92% / 0.00% / 78.08% | 20260505 18:20:58.645 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.254083% / 0% / 0.000127157% | 20260505 18:20:58.645 | INFO | Baz IO overall: | 20260505 18:20:58.645 | INFO | BytesWritten : 3131440965065 | 20260505 18:20:58.645 | INFO | NumEvents : 1483586400063 | 20260505 18:21:00.380 | INFO | Analyzing chunk frames = [5316608,5317120) | 20260505 18:21:00.380 | INFO | MemInfo: { | 20260505 18:21:00.380 | INFO | "gpu" : " 17 GB", | 20260505 18:21:00.380 | INFO | "huge" : "", | 20260505 18:21:00.380 | INFO | "huge_shared" : " 27 GB", | 20260505 18:21:00.380 | INFO | "malloc" : " 14 GB", | 20260505 18:21:00.380 | INFO | "pinned" : " 29 GB" | 20260505 18:21:00.380 | INFO | } | 20260505 18:21:00.380 | INFO | Perf report for the last 5.13343 seconds: | 20260505 18:21:00.380 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:21:00.380 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:00.380 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:00.380 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:00.380 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 18:21:00.380 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:00.380 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260505 18:21:00.380 | INFO | PRE_HQ : 60.3%, 1.8, 55.6ms, 0 | 20260505 18:21:00.380 | INFO | BAZWRITER : 63.2%, 3.8, 124.2ms, 0 | 20260505 18:21:00.380 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 18:21:00.380 | INFO | Uploaded 36GiB to GPU in 2.1611s (16.6582GiB/s) | 20260505 18:21:00.380 | INFO | Downloaded 38GiB to GPU in 2.24444s (16.9307GiB/s) | 20260505 18:21:05.514 | INFO | Analyzing chunk frames = [5317120,5317632) | 20260505 18:21:05.514 | INFO | MemInfo: { | 20260505 18:21:05.514 | INFO | "gpu" : " 17 GB", | 20260505 18:21:05.514 | INFO | "huge" : "", | 20260505 18:21:05.515 | INFO | "huge_shared" : " 27 GB", | 20260505 18:21:05.515 | INFO | "malloc" : " 14 GB", | 20260505 18:21:05.515 | INFO | "pinned" : " 29 GB" | 20260505 18:21:05.515 | INFO | } | 20260505 18:21:05.515 | INFO | Perf report for the last 5.13457 seconds: | 20260505 18:21:05.515 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:21:05.515 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:05.515 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:05.515 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:05.515 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 18:21:05.515 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:05.515 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:05.515 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:21:05.515 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:05.515 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:05.515 | INFO | Uploaded 36GiB to GPU in 2.14189s (16.8076GiB/s) | 20260505 18:21:05.515 | INFO | Downloaded 34GiB to GPU in 2.0332s (16.7224GiB/s) | 20260505 18:21:10.653 | INFO | Analyzing chunk frames = [5317632,5318144) | 20260505 18:21:10.653 | INFO | MemInfo: { | 20260505 18:21:10.653 | INFO | "gpu" : " 17 GB", | 20260505 18:21:10.653 | INFO | "huge" : "", | 20260505 18:21:10.653 | INFO | "huge_shared" : " 27 GB", | 20260505 18:21:10.654 | INFO | "malloc" : " 14 GB", | 20260505 18:21:10.654 | INFO | "pinned" : " 29 GB" | 20260505 18:21:10.654 | INFO | } | 20260505 18:21:10.654 | INFO | Perf report for the last 5.139 seconds: | 20260505 18:21:10.654 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:21:10.654 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:10.654 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:10.654 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:10.654 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260505 18:21:10.654 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:10.654 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:10.654 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 18:21:10.654 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:10.654 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:10.654 | INFO | Uploaded 36GiB to GPU in 2.19159s (16.4265GiB/s) | 20260505 18:21:10.654 | INFO | Downloaded 34GiB to GPU in 2.0357s (16.7019GiB/s) | 20260505 18:21:15.803 | INFO | Analyzing chunk frames = [5318144,5318656) | 20260505 18:21:15.803 | INFO | MemInfo: { | 20260505 18:21:15.803 | INFO | "gpu" : " 17 GB", | 20260505 18:21:15.803 | INFO | "huge" : "", | 20260505 18:21:15.803 | INFO | "huge_shared" : " 27 GB", | 20260505 18:21:15.803 | INFO | "malloc" : " 14 GB", | 20260505 18:21:15.803 | INFO | "pinned" : " 29 GB" | 20260505 18:21:15.803 | INFO | } | 20260505 18:21:15.803 | INFO | Perf report for the last 5.14906 seconds: | 20260505 18:21:15.803 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:21:15.803 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:15.803 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:15.803 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:15.803 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260505 18:21:15.803 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:15.803 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:15.803 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260505 18:21:15.803 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:15.803 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:15.803 | INFO | Uploaded 36GiB to GPU in 2.20498s (16.3267GiB/s) | 20260505 18:21:15.803 | INFO | Downloaded 34GiB to GPU in 2.14151s (15.8767GiB/s) | 20260505 18:21:20.948 | INFO | Analyzing chunk frames = [5318656,5319168) | 20260505 18:21:20.948 | INFO | MemInfo: { | 20260505 18:21:20.948 | INFO | "gpu" : " 17 GB", | 20260505 18:21:20.948 | INFO | "huge" : "", | 20260505 18:21:20.948 | INFO | "huge_shared" : " 27 GB", | 20260505 18:21:20.948 | INFO | "malloc" : " 14 GB", | 20260505 18:21:20.948 | INFO | "pinned" : " 29 GB" | 20260505 18:21:20.953 | INFO | } | 20260505 18:21:20.953 | INFO | Perf report for the last 5.14988 seconds: | 20260505 18:21:20.953 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:21:20.953 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:20.953 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:20.953 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:20.953 | INFO | ANALYSIS : 61.8%, 3.0, 96.6ms, 0 | 20260505 18:21:20.953 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:20.953 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:20.953 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:21:20.953 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:20.953 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:20.953 | INFO | Uploaded 36GiB to GPU in 2.13947s (16.8266GiB/s) | 20260505 18:21:20.953 | INFO | Downloaded 34GiB to GPU in 2.05955s (16.5085GiB/s) | 20260505 18:21:26.082 | INFO | Analyzing chunk frames = [5319168,5319680) | 20260505 18:21:26.082 | INFO | MemInfo: { | 20260505 18:21:26.082 | INFO | "gpu" : " 17 GB", | 20260505 18:21:26.082 | INFO | "huge" : "", | 20260505 18:21:26.082 | INFO | "huge_shared" : " 27 GB", | 20260505 18:21:26.082 | INFO | "malloc" : " 14 GB", | 20260505 18:21:26.082 | INFO | "pinned" : " 29 GB" | 20260505 18:21:26.082 | INFO | } | 20260505 18:21:26.082 | INFO | Perf report for the last 5.12953 seconds: | 20260505 18:21:26.082 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:21:26.082 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:26.082 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:26.082 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:26.082 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260505 18:21:26.082 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:26.082 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:26.082 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:21:26.082 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:26.082 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:26.082 | INFO | Uploaded 36GiB to GPU in 2.16807s (16.6046GiB/s) | 20260505 18:21:26.082 | INFO | Downloaded 34GiB to GPU in 2.07189s (16.4101GiB/s) | 20260505 18:21:31.210 | INFO | Analyzing chunk frames = [5319680,5320192) | 20260505 18:21:31.210 | INFO | MemInfo: { | 20260505 18:21:31.210 | INFO | "gpu" : " 17 GB", | 20260505 18:21:31.210 | INFO | "huge" : "", | 20260505 18:21:31.210 | INFO | "huge_shared" : " 27 GB", | 20260505 18:21:31.210 | INFO | "malloc" : " 14 GB", | 20260505 18:21:31.210 | INFO | "pinned" : " 29 GB" | 20260505 18:21:31.210 | INFO | } | 20260505 18:21:31.210 | INFO | Perf report for the last 5.12814 seconds: | 20260505 18:21:31.210 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:21:31.210 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:31.210 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:31.210 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:31.210 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 18:21:31.210 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:31.210 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:31.210 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 18:21:31.211 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:31.211 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:31.211 | INFO | Uploaded 36GiB to GPU in 2.14684s (16.7689GiB/s) | 20260505 18:21:31.211 | INFO | Downloaded 34GiB to GPU in 2.13835s (15.9001GiB/s) | 20260505 18:21:36.359 | INFO | Analyzing chunk frames = [5320192,5320704) | 20260505 18:21:36.359 | INFO | MemInfo: { | 20260505 18:21:36.359 | INFO | "gpu" : " 17 GB", | 20260505 18:21:36.359 | INFO | "huge" : "", | 20260505 18:21:36.359 | INFO | "huge_shared" : " 27 GB", | 20260505 18:21:36.359 | INFO | "malloc" : " 14 GB", | 20260505 18:21:36.359 | INFO | "pinned" : " 29 GB" | 20260505 18:21:36.359 | INFO | } | 20260505 18:21:36.359 | INFO | Perf report for the last 5.14899 seconds: | 20260505 18:21:36.359 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:21:36.359 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:36.359 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:36.359 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:36.359 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 18:21:36.359 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:36.359 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:36.360 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 18:21:36.360 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:36.360 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:36.360 | INFO | Uploaded 36GiB to GPU in 2.15539s (16.7023GiB/s) | 20260505 18:21:36.360 | INFO | Downloaded 34GiB to GPU in 2.03909s (16.6741GiB/s) | 20260505 18:21:36.492 | INFO | Starting a round of Baz IO | 20260505 18:21:39.669 | INFO | Populating RTMetrics report | 20260505 18:21:39.669 | INFO | RTMetrics report populated | 20260505 18:21:39.669 | INFO | Opening tmp RTMetrics json file | 20260505 18:21:39.672 | INFO | Serializing RTMetrics to json | 20260505 18:21:39.673 | INFO | Serialization done, swapping tmp file now... | 20260505 18:21:39.674 | INFO | RTMetrics done | 20260505 18:21:39.759 | INFO | Finished a round of Baz IO: Wrote 2.091 GiB in 3.267 seconds | 20260505 18:21:39.759 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.92% / 0.00% / 78.08% | 20260505 18:21:39.759 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.253034% / 0% / 7.94729e-05% | 20260505 18:21:39.759 | INFO | Baz IO overall: | 20260505 18:21:39.759 | INFO | BytesWritten : 3133686552454 | 20260505 18:21:39.759 | INFO | NumEvents : 1484632364347 | 20260505 18:21:41.501 | INFO | Analyzing chunk frames = [5320704,5321216) | 20260505 18:21:41.501 | INFO | MemInfo: { | 20260505 18:21:41.501 | INFO | "gpu" : " 17 GB", | 20260505 18:21:41.501 | INFO | "huge" : "", | 20260505 18:21:41.501 | INFO | "huge_shared" : " 27 GB", | 20260505 18:21:41.501 | INFO | "malloc" : " 14 GB", | 20260505 18:21:41.501 | INFO | "pinned" : " 29 GB" | 20260505 18:21:41.501 | INFO | } | 20260505 18:21:41.501 | INFO | Perf report for the last 5.14178 seconds: | 20260505 18:21:41.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:21:41.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:41.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:41.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:41.501 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 18:21:41.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:41.501 | INFO | RT_METRICS : 44.5%, 1.0, 23.4ms, 0 | 20260505 18:21:41.501 | INFO | PRE_HQ : 60.3%, 1.8, 56.4ms, 0 | 20260505 18:21:41.501 | INFO | BAZWRITER : 63.1%, 3.8, 123.0ms, 0 | 20260505 18:21:41.501 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 18:21:41.501 | INFO | Uploaded 36GiB to GPU in 2.15523s (16.7036GiB/s) | 20260505 18:21:41.502 | INFO | Downloaded 38GiB to GPU in 2.24289s (16.9424GiB/s) | 20260505 18:21:46.637 | INFO | Analyzing chunk frames = [5321216,5321728) | 20260505 18:21:46.637 | INFO | MemInfo: { | 20260505 18:21:46.637 | INFO | "gpu" : " 17 GB", | 20260505 18:21:46.637 | INFO | "huge" : "", | 20260505 18:21:46.637 | INFO | "huge_shared" : " 27 GB", | 20260505 18:21:46.637 | INFO | "malloc" : " 14 GB", | 20260505 18:21:46.637 | INFO | "pinned" : " 29 GB" | 20260505 18:21:46.637 | INFO | } | 20260505 18:21:46.637 | INFO | Perf report for the last 5.13617 seconds: | 20260505 18:21:46.637 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:21:46.637 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:46.637 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:46.637 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:46.637 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 18:21:46.637 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:46.638 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:46.638 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:21:46.638 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:46.638 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:46.638 | INFO | Uploaded 36GiB to GPU in 2.15192s (16.7293GiB/s) | 20260505 18:21:46.638 | INFO | Downloaded 34GiB to GPU in 2.04696s (16.61GiB/s) | 20260505 18:21:51.775 | INFO | Analyzing chunk frames = [5321728,5322240) | 20260505 18:21:51.775 | INFO | MemInfo: { | 20260505 18:21:51.775 | INFO | "gpu" : " 17 GB", | 20260505 18:21:51.775 | INFO | "huge" : "", | 20260505 18:21:51.775 | INFO | "huge_shared" : " 27 GB", | 20260505 18:21:51.775 | INFO | "malloc" : " 14 GB", | 20260505 18:21:51.775 | INFO | "pinned" : " 29 GB" | 20260505 18:21:51.775 | INFO | } | 20260505 18:21:51.775 | INFO | Perf report for the last 5.13796 seconds: | 20260505 18:21:51.775 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:21:51.775 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:51.775 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:51.775 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:51.775 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260505 18:21:51.775 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:51.775 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:51.775 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:21:51.776 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:51.776 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:51.776 | INFO | Uploaded 36GiB to GPU in 2.12987s (16.9025GiB/s) | 20260505 18:21:51.776 | INFO | Downloaded 34GiB to GPU in 2.0452s (16.6243GiB/s) | 20260505 18:21:56.910 | INFO | Analyzing chunk frames = [5322240,5322752) | 20260505 18:21:56.910 | INFO | MemInfo: { | 20260505 18:21:56.910 | INFO | "gpu" : " 17 GB", | 20260505 18:21:56.910 | INFO | "huge" : "", | 20260505 18:21:56.910 | INFO | "huge_shared" : " 27 GB", | 20260505 18:21:56.910 | INFO | "malloc" : " 14 GB", | 20260505 18:21:56.910 | INFO | "pinned" : " 29 GB" | 20260505 18:21:56.910 | INFO | } | 20260505 18:21:56.911 | INFO | Perf report for the last 5.13521 seconds: | 20260505 18:21:56.911 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:21:56.911 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:56.911 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:56.911 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:56.911 | INFO | ANALYSIS : 62.9%, 3.0, 97.6ms, 0 | 20260505 18:21:56.911 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:56.911 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:56.911 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:21:56.911 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:56.911 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:21:56.911 | INFO | Uploaded 36GiB to GPU in 2.22479s (16.1813GiB/s) | 20260505 18:21:56.911 | INFO | Downloaded 34GiB to GPU in 2.12203s (16.0224GiB/s) | 20260505 18:22:02.058 | INFO | Analyzing chunk frames = [5322752,5323264) | 20260505 18:22:02.058 | INFO | MemInfo: { | 20260505 18:22:02.058 | INFO | "gpu" : " 17 GB", | 20260505 18:22:02.058 | INFO | "huge" : "", | 20260505 18:22:02.058 | INFO | "huge_shared" : " 27 GB", | 20260505 18:22:02.058 | INFO | "malloc" : " 14 GB", | 20260505 18:22:02.058 | INFO | "pinned" : " 29 GB" | 20260505 18:22:02.058 | INFO | } | 20260505 18:22:02.058 | INFO | Perf report for the last 5.14726 seconds: | 20260505 18:22:02.058 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:22:02.058 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:02.058 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:02.058 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:02.058 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260505 18:22:02.058 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:02.058 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:02.058 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260505 18:22:02.058 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:02.058 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:02.058 | INFO | Uploaded 36GiB to GPU in 2.14383s (16.7924GiB/s) | 20260505 18:22:02.058 | INFO | Downloaded 34GiB to GPU in 2.06195s (16.4892GiB/s) | 20260505 18:22:07.201 | INFO | Analyzing chunk frames = [5323264,5323776) | 20260505 18:22:07.201 | INFO | MemInfo: { | 20260505 18:22:07.201 | INFO | "gpu" : " 17 GB", | 20260505 18:22:07.201 | INFO | "huge" : "", | 20260505 18:22:07.201 | INFO | "huge_shared" : " 27 GB", | 20260505 18:22:07.201 | INFO | "malloc" : " 14 GB", | 20260505 18:22:07.201 | INFO | "pinned" : " 29 GB" | 20260505 18:22:07.202 | INFO | } | 20260505 18:22:07.202 | INFO | Perf report for the last 5.14372 seconds: | 20260505 18:22:07.202 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:22:07.202 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:07.202 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:07.202 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:07.202 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 18:22:07.202 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:07.202 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:07.202 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:22:07.202 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:07.202 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:07.202 | INFO | Uploaded 36GiB to GPU in 2.14256s (16.8023GiB/s) | 20260505 18:22:07.202 | INFO | Downloaded 34GiB to GPU in 2.05634s (16.5342GiB/s) | 20260505 18:22:12.326 | INFO | Analyzing chunk frames = [5323776,5324288) | 20260505 18:22:12.326 | INFO | MemInfo: { | 20260505 18:22:12.326 | INFO | "gpu" : " 17 GB", | 20260505 18:22:12.326 | INFO | "huge" : "", | 20260505 18:22:12.326 | INFO | "huge_shared" : " 27 GB", | 20260505 18:22:12.326 | INFO | "malloc" : " 14 GB", | 20260505 18:22:12.326 | INFO | "pinned" : " 29 GB" | 20260505 18:22:12.326 | INFO | } | 20260505 18:22:12.326 | INFO | Perf report for the last 5.12429 seconds: | 20260505 18:22:12.326 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:22:12.326 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:12.326 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:12.326 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:12.326 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 18:22:12.326 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:12.326 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:12.326 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 18:22:12.326 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:12.326 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:12.326 | INFO | Uploaded 36GiB to GPU in 2.12759s (16.9206GiB/s) | 20260505 18:22:12.326 | INFO | Downloaded 34GiB to GPU in 2.13833s (15.9003GiB/s) | 20260505 18:22:17.463 | INFO | Analyzing chunk frames = [5324288,5324800) | 20260505 18:22:17.463 | INFO | MemInfo: { | 20260505 18:22:17.463 | INFO | "gpu" : " 17 GB", | 20260505 18:22:17.463 | INFO | "huge" : "", | 20260505 18:22:17.463 | INFO | "huge_shared" : " 27 GB", | 20260505 18:22:17.463 | INFO | "malloc" : " 14 GB", | 20260505 18:22:17.463 | INFO | "pinned" : " 29 GB" | 20260505 18:22:17.463 | INFO | } | 20260505 18:22:17.463 | INFO | Perf report for the last 5.13699 seconds: | 20260505 18:22:17.463 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:22:17.463 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:17.463 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:17.463 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:17.463 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 18:22:17.463 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:17.463 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:17.463 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 18:22:17.463 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:17.463 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:17.463 | INFO | Uploaded 36GiB to GPU in 2.16033s (16.6641GiB/s) | 20260505 18:22:17.463 | INFO | Downloaded 34GiB to GPU in 2.03542s (16.7041GiB/s) | 20260505 18:22:17.463 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 18:22:17.463 | INFO | Total mem usage: 15.3 GiB | 20260505 18:22:17.463 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 18:22:17.463 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -7.64%) | 20260505 18:22:17.463 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 18:22:17.463 | INFO | High Water Breakdown: | 20260505 18:22:17.463 | INFO | 61.1% : 8.71 GiB ( 91.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 18:22:17.463 | INFO | 38.7% : 5.51 GiB ( 83% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 18:22:17.463 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 18:22:17.463 | INFO | ------------------------------------------ | 20260505 18:22:17.463 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:22:17.463 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 18:22:17.463 | INFO | Total mem usage: 26.5 GiB | 20260505 18:22:17.463 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 18:22:17.463 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 18:22:17.463 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 18:22:17.463 | INFO | High Water Breakdown: | 20260505 18:22:17.463 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:22:17.463 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:22:17.463 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:22:17.463 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:22:17.463 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:22:17.463 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:22:17.464 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 18:22:17.464 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 18:22:17.464 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:22:17.464 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 18:22:17.464 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 18:22:17.464 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 18:22:17.464 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 18:22:17.464 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 18:22:17.464 | INFO | ------------------------------------------ | 20260505 18:22:17.464 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:22:17.464 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 18:22:17.464 | INFO | Total mem usage: 25.6 GiB | 20260505 18:22:17.464 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 18:22:17.464 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 18:22:17.464 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 18:22:17.464 | INFO | High Water Breakdown: | 20260505 18:22:17.464 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 18:22:17.464 | INFO | ------------------------------------------ | 20260505 18:22:17.464 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:22:17.464 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 18:22:17.464 | INFO | Total mem usage: 15.7 GiB | 20260505 18:22:17.464 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 18:22:17.464 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 18:22:17.464 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 18:22:17.464 | INFO | High Water Breakdown: | 20260505 18:22:17.464 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 18:22:17.464 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 18:22:17.464 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 18:22:17.464 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 18:22:17.464 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 18:22:17.464 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 18:22:17.464 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 18:22:17.464 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 18:22:17.464 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 18:22:17.464 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:22:17.464 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 18:22:17.464 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:22:17.464 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 18:22:17.464 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 18:22:17.464 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 18:22:17.464 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 18:22:17.464 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:22:17.464 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 18:22:17.464 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:22:17.464 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:22:17.464 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:22:17.464 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 18:22:17.464 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 18:22:17.464 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:22:17.464 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 18:22:17.464 | INFO | ------------------------------------------ | 20260505 18:22:17.464 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:22:17.464 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 18:22:17.464 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:22:17.464 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:22:17.464 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.51e+01, 1.35e+01 / 2.31e+00 | 20260505 18:22:17.464 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.02e-01, 1.06e-02 / 4.88e-03 | 20260505 18:22:17.464 | INFO | PerfReport: 11.79%, Baseline -- 3.72e+00 / 4.53e+00, 5.72e-01 / 1.33e-01 | 20260505 18:22:17.464 | INFO | PerfReport: 3.26%, Histogram -- 1.03e+00 / 1.52e+00, 1.88e-01 / 9.12e-02 | 20260505 18:22:17.464 | INFO | PerfReport: 16.00%, DME -- 4.04e+01 / 4.50e+01, 4.30e+00 / 2.48e+00 | 20260505 18:22:17.464 | INFO | PerfReport: 56.84%, FrameLabeling -- 1.79e+01 / 1.93e+01, 1.43e+00 / 1.07e+00 | 20260505 18:22:17.464 | INFO | PerfReport: 7.31%, PulseAccumulating -- 2.31e+00 / 2.89e+00, 4.44e-01 / 1.86e-01 | 20260505 18:22:17.464 | INFO | PerfReport: 3.34%, Metrics -- 1.05e+00 / 1.39e+00, 2.92e-01 / 1.55e-01 | 20260505 18:22:17.464 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.66e-01 / 4.30e-01, 6.47e-02 / 3.43e-02 | 20260505 18:22:17.464 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 18:22:17.464 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:22:17.464 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:22:17.464 | INFO | PerfReport: Full Execution -- 9.98e+01 / -nan, 2.56e+01 / 0.00e+00 | 20260505 18:22:17.464 | INFO | PerfReport: 22.53%, Upload -- 2.25e+01 / 0.00e+00, 7.89e+00 / 0.00e+00 | 20260505 18:22:17.464 | INFO | PerfReport: 21.86%, Download -- 2.18e+01 / 0.00e+00, 7.82e+00 / 0.00e+00 | 20260505 18:22:17.464 | INFO | PerfReport: 31.60%, Compute -- 3.15e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 18:22:17.464 | INFO | PerfReport: 2.30%, LockUpload -- 2.30e+00 / 0.00e+00, 7.84e+00 / 0.00e+00 | 20260505 18:22:17.464 | INFO | PerfReport: 1.38%, LockDownload -- 1.38e+00 / 0.00e+00, 2.85e+00 / 0.00e+00 | 20260505 18:22:17.464 | INFO | PerfReport: 20.31%, LockCompute -- 2.03e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260505 18:22:17.464 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.71e-02 / 0.00e+00 | 20260505 18:22:17.597 | INFO | Starting a round of Baz IO | 20260505 18:22:20.799 | INFO | Populating RTMetrics report | 20260505 18:22:20.799 | INFO | RTMetrics report populated | 20260505 18:22:20.799 | INFO | Opening tmp RTMetrics json file | 20260505 18:22:20.801 | INFO | Serializing RTMetrics to json | 20260505 18:22:20.802 | INFO | Serialization done, swapping tmp file now... | 20260505 18:22:20.803 | INFO | RTMetrics done | 20260505 18:22:20.892 | INFO | Finished a round of Baz IO: Wrote 2.095 GiB in 3.295 seconds | 20260505 18:22:20.892 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.92% / 0.00% / 78.08% | 20260505 18:22:20.892 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.253554% / 0% / 6.75519e-05% | 20260505 18:22:20.892 | INFO | Baz IO overall: | 20260505 18:22:20.893 | INFO | BytesWritten : 3135936104916 | 20260505 18:22:20.893 | INFO | NumEvents : 1485681281884 | 20260505 18:22:22.610 | INFO | Analyzing chunk frames = [5324800,5325312) | 20260505 18:22:22.610 | INFO | MemInfo: { | 20260505 18:22:22.610 | INFO | "gpu" : " 17 GB", | 20260505 18:22:22.610 | INFO | "huge" : "", | 20260505 18:22:22.610 | INFO | "huge_shared" : " 27 GB", | 20260505 18:22:22.610 | INFO | "malloc" : " 14 GB", | 20260505 18:22:22.610 | INFO | "pinned" : " 29 GB" | 20260505 18:22:22.610 | INFO | } | 20260505 18:22:22.610 | INFO | Perf report for the last 5.14752 seconds: | 20260505 18:22:22.611 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:22:22.611 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:22.611 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:22.611 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:22.611 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 18:22:22.611 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:22.611 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260505 18:22:22.611 | INFO | PRE_HQ : 60.4%, 1.8, 55.8ms, 0 | 20260505 18:22:22.611 | INFO | BAZWRITER : 63.4%, 3.8, 122.7ms, 0 | 20260505 18:22:22.611 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 18:22:22.611 | INFO | Uploaded 36GiB to GPU in 2.18502s (16.4758GiB/s) | 20260505 18:22:22.611 | INFO | Downloaded 38GiB to GPU in 2.2687s (16.7497GiB/s) | 20260505 18:22:27.755 | INFO | Analyzing chunk frames = [5325312,5325824) | 20260505 18:22:27.755 | INFO | MemInfo: { | 20260505 18:22:27.755 | INFO | "gpu" : " 17 GB", | 20260505 18:22:27.755 | INFO | "huge" : "", | 20260505 18:22:27.755 | INFO | "huge_shared" : " 27 GB", | 20260505 18:22:27.755 | INFO | "malloc" : " 14 GB", | 20260505 18:22:27.756 | INFO | "pinned" : " 29 GB" | 20260505 18:22:27.756 | INFO | } | 20260505 18:22:27.756 | INFO | Perf report for the last 5.14521 seconds: | 20260505 18:22:27.756 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:22:27.756 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:27.756 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:27.756 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:27.756 | INFO | ANALYSIS : 62.6%, 3.0, 97.9ms, 0 | 20260505 18:22:27.756 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:27.756 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:27.756 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 18:22:27.756 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:27.756 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:27.756 | INFO | Uploaded 36GiB to GPU in 2.12805s (16.9169GiB/s) | 20260505 18:22:27.756 | INFO | Downloaded 34GiB to GPU in 2.02499s (16.7902GiB/s) | 20260505 18:22:32.892 | INFO | Analyzing chunk frames = [5325824,5326336) | 20260505 18:22:32.892 | INFO | MemInfo: { | 20260505 18:22:32.892 | INFO | "gpu" : " 17 GB", | 20260505 18:22:32.892 | INFO | "huge" : "", | 20260505 18:22:32.892 | INFO | "huge_shared" : " 27 GB", | 20260505 18:22:32.892 | INFO | "malloc" : " 14 GB", | 20260505 18:22:32.892 | INFO | "pinned" : " 29 GB" | 20260505 18:22:32.892 | INFO | } | 20260505 18:22:32.892 | INFO | Perf report for the last 5.13633 seconds: | 20260505 18:22:32.892 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:22:32.892 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:32.892 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:32.892 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:32.892 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260505 18:22:32.892 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:32.892 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:32.892 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 18:22:32.892 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:32.892 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:32.892 | INFO | Uploaded 36GiB to GPU in 2.12701s (16.9252GiB/s) | 20260505 18:22:32.892 | INFO | Downloaded 34GiB to GPU in 2.01257s (16.8938GiB/s) | 20260505 18:22:38.025 | INFO | Analyzing chunk frames = [5326336,5326848) | 20260505 18:22:38.025 | INFO | MemInfo: { | 20260505 18:22:38.025 | INFO | "gpu" : " 17 GB", | 20260505 18:22:38.025 | INFO | "huge" : "", | 20260505 18:22:38.025 | INFO | "huge_shared" : " 27 GB", | 20260505 18:22:38.025 | INFO | "malloc" : " 14 GB", | 20260505 18:22:38.025 | INFO | "pinned" : " 29 GB" | 20260505 18:22:38.026 | INFO | } | 20260505 18:22:38.026 | INFO | Perf report for the last 5.13357 seconds: | 20260505 18:22:38.026 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:22:38.026 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:38.026 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:38.026 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:38.026 | INFO | ANALYSIS : 62.9%, 3.0, 97.6ms, 0 | 20260505 18:22:38.026 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:38.026 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:38.026 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:22:38.026 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:38.026 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:38.026 | INFO | Uploaded 36GiB to GPU in 2.22911s (16.1499GiB/s) | 20260505 18:22:38.026 | INFO | Downloaded 34GiB to GPU in 2.12002s (16.0376GiB/s) | 20260505 18:22:43.164 | INFO | Analyzing chunk frames = [5326848,5327360) | 20260505 18:22:43.164 | INFO | MemInfo: { | 20260505 18:22:43.164 | INFO | "gpu" : " 17 GB", | 20260505 18:22:43.164 | INFO | "huge" : "", | 20260505 18:22:43.164 | INFO | "huge_shared" : " 27 GB", | 20260505 18:22:43.164 | INFO | "malloc" : " 14 GB", | 20260505 18:22:43.164 | INFO | "pinned" : " 29 GB" | 20260505 18:22:43.164 | INFO | } | 20260505 18:22:43.164 | INFO | Perf report for the last 5.13888 seconds: | 20260505 18:22:43.164 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:22:43.164 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:43.164 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:43.165 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:43.165 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 18:22:43.165 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:43.165 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:43.165 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:22:43.165 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:43.165 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:43.165 | INFO | Uploaded 36GiB to GPU in 2.14539s (16.7802GiB/s) | 20260505 18:22:43.165 | INFO | Downloaded 34GiB to GPU in 2.05313s (16.5601GiB/s) | 20260505 18:22:48.309 | INFO | Analyzing chunk frames = [5327360,5327872) | 20260505 18:22:48.309 | INFO | MemInfo: { | 20260505 18:22:48.309 | INFO | "gpu" : " 17 GB", | 20260505 18:22:48.309 | INFO | "huge" : "", | 20260505 18:22:48.309 | INFO | "huge_shared" : " 27 GB", | 20260505 18:22:48.309 | INFO | "malloc" : " 14 GB", | 20260505 18:22:48.309 | INFO | "pinned" : " 29 GB" | 20260505 18:22:48.309 | INFO | } | 20260505 18:22:48.309 | INFO | Perf report for the last 5.14498 seconds: | 20260505 18:22:48.309 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:22:48.309 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:48.309 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:48.309 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:48.309 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260505 18:22:48.310 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:48.310 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:48.310 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:22:48.310 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:48.310 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:48.310 | INFO | Uploaded 36GiB to GPU in 2.13877s (16.8321GiB/s) | 20260505 18:22:48.310 | INFO | Downloaded 34GiB to GPU in 2.04804s (16.6012GiB/s) | 20260505 18:22:53.454 | INFO | Analyzing chunk frames = [5327872,5328384) | 20260505 18:22:53.454 | INFO | MemInfo: { | 20260505 18:22:53.454 | INFO | "gpu" : " 17 GB", | 20260505 18:22:53.454 | INFO | "huge" : "", | 20260505 18:22:53.454 | INFO | "huge_shared" : " 27 GB", | 20260505 18:22:53.454 | INFO | "malloc" : " 14 GB", | 20260505 18:22:53.454 | INFO | "pinned" : " 29 GB" | 20260505 18:22:53.454 | INFO | } | 20260505 18:22:53.454 | INFO | Perf report for the last 5.14486 seconds: | 20260505 18:22:53.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:22:53.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:53.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:53.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:53.454 | INFO | ANALYSIS : 63.3%, 3.0, 99.0ms, 0 | 20260505 18:22:53.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:53.454 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:53.454 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 18:22:53.454 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:53.454 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:53.455 | INFO | Uploaded 36GiB to GPU in 2.13224s (16.8837GiB/s) | 20260505 18:22:53.455 | INFO | Downloaded 34GiB to GPU in 2.16227s (15.7242GiB/s) | 20260505 18:22:58.587 | INFO | Analyzing chunk frames = [5328384,5328896) | 20260505 18:22:58.587 | INFO | MemInfo: { | 20260505 18:22:58.587 | INFO | "gpu" : " 17 GB", | 20260505 18:22:58.587 | INFO | "huge" : "", | 20260505 18:22:58.587 | INFO | "huge_shared" : " 27 GB", | 20260505 18:22:58.587 | INFO | "malloc" : " 14 GB", | 20260505 18:22:58.587 | INFO | "pinned" : " 29 GB" | 20260505 18:22:58.587 | INFO | } | 20260505 18:22:58.587 | INFO | Perf report for the last 5.13302 seconds: | 20260505 18:22:58.587 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:22:58.587 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:58.587 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:58.587 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:58.587 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 18:22:58.587 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:58.587 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:58.588 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 18:22:58.588 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:58.588 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:22:58.588 | INFO | Uploaded 36GiB to GPU in 2.20073s (16.3582GiB/s) | 20260505 18:22:58.588 | INFO | Downloaded 34GiB to GPU in 2.04602s (16.6177GiB/s) | 20260505 18:22:58.723 | INFO | Starting a round of Baz IO | 20260505 18:23:01.913 | INFO | Populating RTMetrics report | 20260505 18:23:01.914 | INFO | RTMetrics report populated | 20260505 18:23:01.914 | INFO | Opening tmp RTMetrics json file | 20260505 18:23:01.914 | INFO | Serializing RTMetrics to json | 20260505 18:23:01.916 | INFO | Serialization done, swapping tmp file now... | 20260505 18:23:01.916 | INFO | RTMetrics done | 20260505 18:23:02.006 | INFO | Finished a round of Baz IO: Wrote 2.094 GiB in 3.282 seconds | 20260505 18:23:02.006 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.92% / 0.00% / 78.08% | 20260505 18:23:02.006 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.252036% / 7.94729e-06% / 9.93411e-05% | 20260505 18:23:02.006 | INFO | Baz IO overall: | 20260505 18:23:02.006 | INFO | BytesWritten : 3138184217062 | 20260505 18:23:02.006 | INFO | NumEvents : 1486729066000 | 20260505 18:23:03.724 | INFO | Analyzing chunk frames = [5328896,5329408) | 20260505 18:23:03.724 | INFO | MemInfo: { | 20260505 18:23:03.724 | INFO | "gpu" : " 17 GB", | 20260505 18:23:03.724 | INFO | "huge" : "", | 20260505 18:23:03.724 | INFO | "huge_shared" : " 27 GB", | 20260505 18:23:03.724 | INFO | "malloc" : " 14 GB", | 20260505 18:23:03.724 | INFO | "pinned" : " 29 GB" | 20260505 18:23:03.724 | INFO | } | 20260505 18:23:03.724 | INFO | Perf report for the last 5.13685 seconds: | 20260505 18:23:03.724 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:23:03.724 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:03.724 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:03.724 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:03.724 | INFO | ANALYSIS : 63.6%, 3.0, 99.2ms, 0 | 20260505 18:23:03.724 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:03.724 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260505 18:23:03.724 | INFO | PRE_HQ : 60.7%, 1.8, 55.9ms, 0 | 20260505 18:23:03.724 | INFO | BAZWRITER : 63.4%, 3.8, 123.7ms, 0 | 20260505 18:23:03.724 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 18:23:03.725 | INFO | Uploaded 36GiB to GPU in 2.18241s (16.4955GiB/s) | 20260505 18:23:03.725 | INFO | Downloaded 38GiB to GPU in 2.26317s (16.7906GiB/s) | 20260505 18:23:08.870 | INFO | Analyzing chunk frames = [5329408,5329920) | 20260505 18:23:08.870 | INFO | MemInfo: { | 20260505 18:23:08.870 | INFO | "gpu" : " 17 GB", | 20260505 18:23:08.870 | INFO | "huge" : "", | 20260505 18:23:08.870 | INFO | "huge_shared" : " 27 GB", | 20260505 18:23:08.870 | INFO | "malloc" : " 14 GB", | 20260505 18:23:08.870 | INFO | "pinned" : " 29 GB" | 20260505 18:23:08.870 | INFO | } | 20260505 18:23:08.870 | INFO | Perf report for the last 5.14605 seconds: | 20260505 18:23:08.870 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:23:08.870 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:08.870 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:08.870 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:08.870 | INFO | ANALYSIS : 62.7%, 3.0, 98.0ms, 0 | 20260505 18:23:08.870 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:08.870 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:08.870 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 18:23:08.870 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:08.870 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:08.871 | INFO | Uploaded 36GiB to GPU in 2.14776s (16.7616GiB/s) | 20260505 18:23:08.871 | INFO | Downloaded 34GiB to GPU in 2.03145s (16.7368GiB/s) | 20260505 18:23:14.005 | INFO | Analyzing chunk frames = [5329920,5330432) | 20260505 18:23:14.005 | INFO | MemInfo: { | 20260505 18:23:14.005 | INFO | "gpu" : " 17 GB", | 20260505 18:23:14.005 | INFO | "huge" : "", | 20260505 18:23:14.005 | INFO | "huge_shared" : " 27 GB", | 20260505 18:23:14.005 | INFO | "malloc" : " 14 GB", | 20260505 18:23:14.005 | INFO | "pinned" : " 29 GB" | 20260505 18:23:14.005 | INFO | } | 20260505 18:23:14.005 | INFO | Perf report for the last 5.13507 seconds: | 20260505 18:23:14.005 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:23:14.005 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:14.005 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:14.005 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:14.005 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260505 18:23:14.005 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:14.005 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:14.006 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:23:14.006 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:14.006 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:14.006 | INFO | Uploaded 36GiB to GPU in 2.13474s (16.8639GiB/s) | 20260505 18:23:14.006 | INFO | Downloaded 34GiB to GPU in 2.03065s (16.7434GiB/s) | 20260505 18:23:19.147 | INFO | Analyzing chunk frames = [5330432,5330944) | 20260505 18:23:19.147 | INFO | MemInfo: { | 20260505 18:23:19.147 | INFO | "gpu" : " 17 GB", | 20260505 18:23:19.147 | INFO | "huge" : "", | 20260505 18:23:19.147 | INFO | "huge_shared" : " 27 GB", | 20260505 18:23:19.147 | INFO | "malloc" : " 14 GB", | 20260505 18:23:19.147 | INFO | "pinned" : " 29 GB" | 20260505 18:23:19.147 | INFO | } | 20260505 18:23:19.147 | INFO | Perf report for the last 5.14201 seconds: | 20260505 18:23:19.147 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:23:19.147 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:19.147 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:19.147 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:19.147 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260505 18:23:19.147 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:19.147 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:19.147 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:23:19.147 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:19.147 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:19.148 | INFO | Uploaded 36GiB to GPU in 2.17143s (16.5789GiB/s) | 20260505 18:23:19.148 | INFO | Downloaded 34GiB to GPU in 2.05475s (16.547GiB/s) | 20260505 18:23:24.285 | INFO | Analyzing chunk frames = [5330944,5331456) | 20260505 18:23:24.285 | INFO | MemInfo: { | 20260505 18:23:24.285 | INFO | "gpu" : " 17 GB", | 20260505 18:23:24.285 | INFO | "huge" : "", | 20260505 18:23:24.285 | INFO | "huge_shared" : " 27 GB", | 20260505 18:23:24.285 | INFO | "malloc" : " 14 GB", | 20260505 18:23:24.285 | INFO | "pinned" : " 29 GB" | 20260505 18:23:24.285 | INFO | } | 20260505 18:23:24.285 | INFO | Perf report for the last 5.1373 seconds: | 20260505 18:23:24.285 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:23:24.285 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:24.285 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:24.285 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:24.285 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260505 18:23:24.285 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:24.285 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:24.285 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:23:24.285 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:24.285 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:24.285 | INFO | Uploaded 36GiB to GPU in 2.1382s (16.8366GiB/s) | 20260505 18:23:24.285 | INFO | Downloaded 34GiB to GPU in 2.05792s (16.5215GiB/s) | 20260505 18:23:29.418 | INFO | Analyzing chunk frames = [5331456,5331968) | 20260505 18:23:29.418 | INFO | MemInfo: { | 20260505 18:23:29.418 | INFO | "gpu" : " 17 GB", | 20260505 18:23:29.418 | INFO | "huge" : "", | 20260505 18:23:29.418 | INFO | "huge_shared" : " 27 GB", | 20260505 18:23:29.418 | INFO | "malloc" : " 14 GB", | 20260505 18:23:29.418 | INFO | "pinned" : " 29 GB" | 20260505 18:23:29.418 | INFO | } | 20260505 18:23:29.418 | INFO | Perf report for the last 5.13366 seconds: | 20260505 18:23:29.418 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:23:29.418 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:29.418 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:29.418 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:29.418 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260505 18:23:29.419 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:29.419 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:29.419 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 18:23:29.419 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:29.419 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:29.419 | INFO | Uploaded 36GiB to GPU in 2.14592s (16.776GiB/s) | 20260505 18:23:29.419 | INFO | Downloaded 34GiB to GPU in 2.04642s (16.6144GiB/s) | 20260505 18:23:34.565 | INFO | Analyzing chunk frames = [5331968,5332480) | 20260505 18:23:34.565 | INFO | MemInfo: { | 20260505 18:23:34.565 | INFO | "gpu" : " 17 GB", | 20260505 18:23:34.565 | INFO | "huge" : "", | 20260505 18:23:34.565 | INFO | "huge_shared" : " 27 GB", | 20260505 18:23:34.565 | INFO | "malloc" : " 14 GB", | 20260505 18:23:34.565 | INFO | "pinned" : " 29 GB" | 20260505 18:23:34.565 | INFO | } | 20260505 18:23:34.565 | INFO | Perf report for the last 5.14646 seconds: | 20260505 18:23:34.565 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:23:34.565 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:34.565 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:34.565 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:34.565 | INFO | ANALYSIS : 63.4%, 3.0, 98.1ms, 0 | 20260505 18:23:34.565 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:34.565 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:34.565 | INFO | PRE_HQ : 28.3%, 1.0, 14.6ms, 0 | 20260505 18:23:34.565 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:34.565 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:34.565 | INFO | Uploaded 36GiB to GPU in 2.16219s (16.6498GiB/s) | 20260505 18:23:34.565 | INFO | Downloaded 34GiB to GPU in 2.17948s (15.6001GiB/s) | 20260505 18:23:39.709 | INFO | Analyzing chunk frames = [5332480,5332992) | 20260505 18:23:39.709 | INFO | MemInfo: { | 20260505 18:23:39.709 | INFO | "gpu" : " 17 GB", | 20260505 18:23:39.709 | INFO | "huge" : "", | 20260505 18:23:39.709 | INFO | "huge_shared" : " 27 GB", | 20260505 18:23:39.709 | INFO | "malloc" : " 14 GB", | 20260505 18:23:39.709 | INFO | "pinned" : " 29 GB" | 20260505 18:23:39.709 | INFO | } | 20260505 18:23:39.709 | INFO | Perf report for the last 5.14452 seconds: | 20260505 18:23:39.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:23:39.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:39.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:39.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:39.709 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 18:23:39.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:39.710 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:39.710 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 18:23:39.710 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:39.710 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:39.710 | INFO | Uploaded 36GiB to GPU in 2.19753s (16.382GiB/s) | 20260505 18:23:39.710 | INFO | Downloaded 34GiB to GPU in 2.026s (16.7819GiB/s) | 20260505 18:23:39.710 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 18:23:39.710 | INFO | Total mem usage: 15.3 GiB | 20260505 18:23:39.710 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 18:23:39.710 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.01%) | 20260505 18:23:39.710 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 18:23:39.710 | INFO | High Water Breakdown: | 20260505 18:23:39.710 | INFO | 61% : 8.66 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 18:23:39.710 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 18:23:39.710 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 18:23:39.710 | INFO | ------------------------------------------ | 20260505 18:23:39.710 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:23:39.710 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 18:23:39.710 | INFO | Total mem usage: 26.5 GiB | 20260505 18:23:39.710 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 18:23:39.710 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 18:23:39.710 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 18:23:39.710 | INFO | High Water Breakdown: | 20260505 18:23:39.710 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:23:39.710 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:23:39.710 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:23:39.710 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:23:39.710 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:23:39.710 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:23:39.710 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 18:23:39.710 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 18:23:39.710 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:23:39.710 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 18:23:39.710 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 18:23:39.710 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 18:23:39.710 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 18:23:39.710 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 18:23:39.710 | INFO | ------------------------------------------ | 20260505 18:23:39.710 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:23:39.710 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 18:23:39.710 | INFO | Total mem usage: 25.6 GiB | 20260505 18:23:39.710 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 18:23:39.710 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 18:23:39.710 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 18:23:39.710 | INFO | High Water Breakdown: | 20260505 18:23:39.710 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 18:23:39.710 | INFO | ------------------------------------------ | 20260505 18:23:39.710 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:23:39.710 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 18:23:39.710 | INFO | Total mem usage: 15.7 GiB | 20260505 18:23:39.710 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 18:23:39.710 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 18:23:39.710 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 18:23:39.710 | INFO | High Water Breakdown: | 20260505 18:23:39.710 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 18:23:39.711 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 18:23:39.711 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 18:23:39.711 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 18:23:39.711 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 18:23:39.711 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 18:23:39.711 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 18:23:39.711 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 18:23:39.711 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 18:23:39.711 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:23:39.711 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 18:23:39.711 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:23:39.711 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 18:23:39.711 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 18:23:39.711 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 18:23:39.711 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 18:23:39.711 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:23:39.711 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 18:23:39.711 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:23:39.711 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:23:39.711 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:23:39.711 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 18:23:39.711 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 18:23:39.711 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:23:39.711 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 18:23:39.711 | INFO | ------------------------------------------ | 20260505 18:23:39.711 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:23:39.711 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 18:23:39.711 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:23:39.711 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:23:39.711 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.48e+01, 1.36e+01 / 2.32e+00 | 20260505 18:23:39.711 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 1.03e-01, 1.08e-02 / 5.29e-03 | 20260505 18:23:39.711 | INFO | PerfReport: 11.78%, Baseline -- 3.72e+00 / 4.52e+00, 5.69e-01 / 1.39e-01 | 20260505 18:23:39.711 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.50e+00, 1.89e-01 / 9.87e-02 | 20260505 18:23:39.711 | INFO | PerfReport: 15.99%, DME -- 4.04e+01 / 4.48e+01, 4.26e+00 / 2.28e+00 | 20260505 18:23:39.711 | INFO | PerfReport: 56.84%, FrameLabeling -- 1.79e+01 / 1.91e+01, 1.43e+00 / 9.01e-01 | 20260505 18:23:39.711 | INFO | PerfReport: 7.32%, PulseAccumulating -- 2.31e+00 / 2.94e+00, 4.46e-01 / 1.34e-01 | 20260505 18:23:39.711 | INFO | PerfReport: 3.33%, Metrics -- 1.05e+00 / 1.38e+00, 2.85e-01 / 1.59e-01 | 20260505 18:23:39.711 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.65e-01 / 4.20e-01, 6.40e-02 / 3.85e-02 | 20260505 18:23:39.711 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 18:23:39.711 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:23:39.711 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:23:39.711 | INFO | PerfReport: Full Execution -- 9.99e+01 / -nan, 2.56e+01 / 0.00e+00 | 20260505 18:23:39.711 | INFO | PerfReport: 22.54%, Upload -- 2.25e+01 / 0.00e+00, 7.84e+00 / 0.00e+00 | 20260505 18:23:39.711 | INFO | PerfReport: 21.79%, Download -- 2.18e+01 / 0.00e+00, 7.73e+00 / 0.00e+00 | 20260505 18:23:39.711 | INFO | PerfReport: 31.60%, Compute -- 3.16e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 18:23:39.711 | INFO | PerfReport: 2.34%, LockUpload -- 2.34e+00 / 0.00e+00, 8.06e+00 / 0.00e+00 | 20260505 18:23:39.711 | INFO | PerfReport: 1.36%, LockDownload -- 1.36e+00 / 0.00e+00, 2.90e+00 / 0.00e+00 | 20260505 18:23:39.711 | INFO | PerfReport: 20.34%, LockCompute -- 2.03e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260505 18:23:39.711 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 2.19e-02 / 0.00e+00 | 20260505 18:23:39.844 | INFO | Starting a round of Baz IO | 20260505 18:23:43.018 | INFO | Populating RTMetrics report | 20260505 18:23:43.018 | INFO | RTMetrics report populated | 20260505 18:23:43.018 | INFO | Opening tmp RTMetrics json file | 20260505 18:23:43.024 | INFO | Serializing RTMetrics to json | 20260505 18:23:43.026 | INFO | Serialization done, swapping tmp file now... | 20260505 18:23:43.028 | INFO | RTMetrics done | 20260505 18:23:43.109 | INFO | Finished a round of Baz IO: Wrote 2.094 GiB in 3.264 seconds | 20260505 18:23:43.109 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.91% / 0.00% / 78.08% | 20260505 18:23:43.109 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.252012% / 0% / 9.13938e-05% | 20260505 18:23:43.109 | INFO | Baz IO overall: | 20260505 18:23:43.109 | INFO | BytesWritten : 3140432686194 | 20260505 18:23:43.109 | INFO | NumEvents : 1487777231171 | 20260505 18:23:44.841 | INFO | Analyzing chunk frames = [5332992,5333504) | 20260505 18:23:44.841 | INFO | MemInfo: { | 20260505 18:23:44.841 | INFO | "gpu" : " 17 GB", | 20260505 18:23:44.841 | INFO | "huge" : "", | 20260505 18:23:44.841 | INFO | "huge_shared" : " 27 GB", | 20260505 18:23:44.841 | INFO | "malloc" : " 14 GB", | 20260505 18:23:44.841 | INFO | "pinned" : " 29 GB" | 20260505 18:23:44.841 | INFO | } | 20260505 18:23:44.841 | INFO | Perf report for the last 5.13119 seconds: | 20260505 18:23:44.841 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:23:44.841 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:44.841 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:44.841 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:44.841 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 18:23:44.841 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:44.841 | INFO | RT_METRICS : 44.6%, 1.0, 23.3ms, 0 | 20260505 18:23:44.841 | INFO | PRE_HQ : 60.4%, 1.8, 55.8ms, 0 | 20260505 18:23:44.841 | INFO | BAZWRITER : 63.2%, 3.8, 122.8ms, 0 | 20260505 18:23:44.841 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 18:23:44.841 | INFO | Uploaded 36GiB to GPU in 2.17434s (16.5568GiB/s) | 20260505 18:23:44.841 | INFO | Downloaded 38GiB to GPU in 2.24197s (16.9494GiB/s) | 20260505 18:23:49.973 | INFO | Analyzing chunk frames = [5333504,5334016) | 20260505 18:23:49.973 | INFO | MemInfo: { | 20260505 18:23:49.973 | INFO | "gpu" : " 17 GB", | 20260505 18:23:49.973 | INFO | "huge" : "", | 20260505 18:23:49.973 | INFO | "huge_shared" : " 27 GB", | 20260505 18:23:49.973 | INFO | "malloc" : " 14 GB", | 20260505 18:23:49.973 | INFO | "pinned" : " 29 GB" | 20260505 18:23:49.973 | INFO | } | 20260505 18:23:49.973 | INFO | Perf report for the last 5.13254 seconds: | 20260505 18:23:49.973 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:23:49.973 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:49.973 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:49.973 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:49.973 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260505 18:23:49.973 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:49.973 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:49.973 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:23:49.973 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:49.973 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:49.973 | INFO | Uploaded 36GiB to GPU in 2.13352s (16.8735GiB/s) | 20260505 18:23:49.974 | INFO | Downloaded 34GiB to GPU in 2.0276s (16.7686GiB/s) | 20260505 18:23:55.117 | INFO | Analyzing chunk frames = [5334016,5334528) | 20260505 18:23:55.117 | INFO | MemInfo: { | 20260505 18:23:55.117 | INFO | "gpu" : " 17 GB", | 20260505 18:23:55.117 | INFO | "huge" : "", | 20260505 18:23:55.117 | INFO | "huge_shared" : " 27 GB", | 20260505 18:23:55.117 | INFO | "malloc" : " 14 GB", | 20260505 18:23:55.117 | INFO | "pinned" : " 29 GB" | 20260505 18:23:55.117 | INFO | } | 20260505 18:23:55.117 | INFO | Perf report for the last 5.14406 seconds: | 20260505 18:23:55.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:23:55.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:55.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:55.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:55.117 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260505 18:23:55.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:55.117 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:55.117 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:23:55.117 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:55.117 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:23:55.118 | INFO | Uploaded 36GiB to GPU in 2.20577s (16.3209GiB/s) | 20260505 18:23:55.118 | INFO | Downloaded 34GiB to GPU in 2.0343s (16.7133GiB/s) | 20260505 18:24:00.262 | INFO | Analyzing chunk frames = [5334528,5335040) | 20260505 18:24:00.262 | INFO | MemInfo: { | 20260505 18:24:00.262 | INFO | "gpu" : " 17 GB", | 20260505 18:24:00.262 | INFO | "huge" : "", | 20260505 18:24:00.262 | INFO | "huge_shared" : " 27 GB", | 20260505 18:24:00.262 | INFO | "malloc" : " 14 GB", | 20260505 18:24:00.262 | INFO | "pinned" : " 29 GB" | 20260505 18:24:00.262 | INFO | } | 20260505 18:24:00.263 | INFO | Perf report for the last 5.14527 seconds: | 20260505 18:24:00.263 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:24:00.263 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:00.263 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:00.263 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:00.263 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260505 18:24:00.263 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:00.263 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:00.263 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:24:00.263 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:00.263 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:00.263 | INFO | Uploaded 36GiB to GPU in 2.18498s (16.4761GiB/s) | 20260505 18:24:00.263 | INFO | Downloaded 34GiB to GPU in 2.06632s (16.4544GiB/s) | 20260505 18:24:05.401 | INFO | Analyzing chunk frames = [5335040,5335552) | 20260505 18:24:05.401 | INFO | MemInfo: { | 20260505 18:24:05.401 | INFO | "gpu" : " 17 GB", | 20260505 18:24:05.401 | INFO | "huge" : "", | 20260505 18:24:05.401 | INFO | "huge_shared" : " 27 GB", | 20260505 18:24:05.401 | INFO | "malloc" : " 14 GB", | 20260505 18:24:05.401 | INFO | "pinned" : " 29 GB" | 20260505 18:24:05.401 | INFO | } | 20260505 18:24:05.401 | INFO | Perf report for the last 5.13871 seconds: | 20260505 18:24:05.401 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:24:05.401 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:05.401 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:05.401 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:05.401 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 18:24:05.401 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:05.401 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:05.401 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:24:05.401 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:05.401 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:05.402 | INFO | Uploaded 36GiB to GPU in 2.14753s (16.7634GiB/s) | 20260505 18:24:05.402 | INFO | Downloaded 34GiB to GPU in 2.06145s (16.4933GiB/s) | 20260505 18:24:10.535 | INFO | Analyzing chunk frames = [5335552,5336064) | 20260505 18:24:10.535 | INFO | MemInfo: { | 20260505 18:24:10.535 | INFO | "gpu" : " 17 GB", | 20260505 18:24:10.535 | INFO | "huge" : "", | 20260505 18:24:10.535 | INFO | "huge_shared" : " 27 GB", | 20260505 18:24:10.535 | INFO | "malloc" : " 14 GB", | 20260505 18:24:10.535 | INFO | "pinned" : " 29 GB" | 20260505 18:24:10.535 | INFO | } | 20260505 18:24:10.535 | INFO | Perf report for the last 5.13354 seconds: | 20260505 18:24:10.535 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:24:10.535 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:10.535 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:10.535 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:10.535 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260505 18:24:10.535 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:10.535 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:10.535 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 18:24:10.535 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:10.535 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:10.535 | INFO | Uploaded 36GiB to GPU in 2.13839s (16.8351GiB/s) | 20260505 18:24:10.535 | INFO | Downloaded 34GiB to GPU in 2.05475s (16.547GiB/s) | 20260505 18:24:15.674 | INFO | Analyzing chunk frames = [5336064,5336576) | 20260505 18:24:15.674 | INFO | MemInfo: { | 20260505 18:24:15.674 | INFO | "gpu" : " 17 GB", | 20260505 18:24:15.674 | INFO | "huge" : "", | 20260505 18:24:15.674 | INFO | "huge_shared" : " 27 GB", | 20260505 18:24:15.674 | INFO | "malloc" : " 14 GB", | 20260505 18:24:15.674 | INFO | "pinned" : " 29 GB" | 20260505 18:24:15.674 | INFO | } | 20260505 18:24:15.674 | INFO | Perf report for the last 5.13943 seconds: | 20260505 18:24:15.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:24:15.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:15.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:15.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:15.674 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 18:24:15.674 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:15.674 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:15.674 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 18:24:15.674 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:15.674 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:15.675 | INFO | Uploaded 36GiB to GPU in 2.13672s (16.8483GiB/s) | 20260505 18:24:15.675 | INFO | Downloaded 34GiB to GPU in 2.13733s (15.9077GiB/s) | 20260505 18:24:20.820 | INFO | Analyzing chunk frames = [5336576,5337088) | 20260505 18:24:20.820 | INFO | MemInfo: { | 20260505 18:24:20.820 | INFO | "gpu" : " 17 GB", | 20260505 18:24:20.820 | INFO | "huge" : "", | 20260505 18:24:20.820 | INFO | "huge_shared" : " 27 GB", | 20260505 18:24:20.820 | INFO | "malloc" : " 14 GB", | 20260505 18:24:20.820 | INFO | "pinned" : " 29 GB" | 20260505 18:24:20.820 | INFO | } | 20260505 18:24:20.820 | INFO | Perf report for the last 5.14572 seconds: | 20260505 18:24:20.820 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:24:20.820 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:20.820 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:20.820 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:20.820 | INFO | ANALYSIS : 62.7%, 3.0, 98.1ms, 0 | 20260505 18:24:20.820 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:20.820 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:20.820 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 18:24:20.820 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:20.820 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:20.820 | INFO | Uploaded 36GiB to GPU in 2.19687s (16.3869GiB/s) | 20260505 18:24:20.820 | INFO | Downloaded 34GiB to GPU in 2.02956s (16.7524GiB/s) | 20260505 18:24:20.952 | INFO | Starting a round of Baz IO | 20260505 18:24:24.131 | INFO | Populating RTMetrics report | 20260505 18:24:24.132 | INFO | RTMetrics report populated | 20260505 18:24:24.132 | INFO | Opening tmp RTMetrics json file | 20260505 18:24:24.139 | INFO | Serializing RTMetrics to json | 20260505 18:24:24.140 | INFO | Serialization done, swapping tmp file now... | 20260505 18:24:24.140 | INFO | RTMetrics done | 20260505 18:24:24.223 | INFO | Finished a round of Baz IO: Wrote 2.093 GiB in 3.271 seconds | 20260505 18:24:24.224 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.91% / 0.00% / 78.09% | 20260505 18:24:24.224 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.25173% / 3.97364e-06% / 6.75519e-05% | 20260505 18:24:24.224 | INFO | Baz IO overall: | 20260505 18:24:24.224 | INFO | BytesWritten : 3142680134269 | 20260505 18:24:24.224 | INFO | NumEvents : 1488825075253 | 20260505 18:24:25.956 | INFO | Analyzing chunk frames = [5337088,5337600) | 20260505 18:24:25.956 | INFO | MemInfo: { | 20260505 18:24:25.956 | INFO | "gpu" : " 17 GB", | 20260505 18:24:25.956 | INFO | "huge" : "", | 20260505 18:24:25.956 | INFO | "huge_shared" : " 27 GB", | 20260505 18:24:25.956 | INFO | "malloc" : " 14 GB", | 20260505 18:24:25.956 | INFO | "pinned" : " 29 GB" | 20260505 18:24:25.956 | INFO | } | 20260505 18:24:25.956 | INFO | Perf report for the last 5.136 seconds: | 20260505 18:24:25.956 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:24:25.956 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:25.956 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:25.956 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:25.956 | INFO | ANALYSIS : 63.4%, 3.0, 98.8ms, 0 | 20260505 18:24:25.956 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:25.956 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260505 18:24:25.956 | INFO | PRE_HQ : 60.3%, 1.8, 56.1ms, 0 | 20260505 18:24:25.956 | INFO | BAZWRITER : 63.3%, 3.8, 124.1ms, 0 | 20260505 18:24:25.956 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 18:24:25.956 | INFO | Uploaded 36GiB to GPU in 2.16179s (16.6528GiB/s) | 20260505 18:24:25.956 | INFO | Downloaded 38GiB to GPU in 2.2522s (16.8724GiB/s) | 20260505 18:24:31.095 | INFO | Analyzing chunk frames = [5337600,5338112) | 20260505 18:24:31.095 | INFO | MemInfo: { | 20260505 18:24:31.095 | INFO | "gpu" : " 17 GB", | 20260505 18:24:31.095 | INFO | "huge" : "", | 20260505 18:24:31.095 | INFO | "huge_shared" : " 27 GB", | 20260505 18:24:31.095 | INFO | "malloc" : " 14 GB", | 20260505 18:24:31.095 | INFO | "pinned" : " 29 GB" | 20260505 18:24:31.095 | INFO | } | 20260505 18:24:31.095 | INFO | Perf report for the last 5.13877 seconds: | 20260505 18:24:31.095 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:24:31.095 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:31.095 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:31.095 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:31.095 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 18:24:31.095 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:31.095 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:31.095 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 18:24:31.095 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:31.095 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:31.095 | INFO | Uploaded 36GiB to GPU in 2.12966s (16.9041GiB/s) | 20260505 18:24:31.095 | INFO | Downloaded 34GiB to GPU in 2.02315s (16.8054GiB/s) | 20260505 18:24:36.236 | INFO | Analyzing chunk frames = [5338112,5338624) | 20260505 18:24:36.236 | INFO | MemInfo: { | 20260505 18:24:36.236 | INFO | "gpu" : " 17 GB", | 20260505 18:24:36.236 | INFO | "huge" : "", | 20260505 18:24:36.236 | INFO | "huge_shared" : " 27 GB", | 20260505 18:24:36.236 | INFO | "malloc" : " 14 GB", | 20260505 18:24:36.236 | INFO | "pinned" : " 29 GB" | 20260505 18:24:36.236 | INFO | } | 20260505 18:24:36.236 | INFO | Perf report for the last 5.14152 seconds: | 20260505 18:24:36.236 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:24:36.236 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:36.236 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:36.236 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:36.236 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 18:24:36.236 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:36.236 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:36.236 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:24:36.236 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:36.236 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:36.237 | INFO | Uploaded 36GiB to GPU in 2.20416s (16.3327GiB/s) | 20260505 18:24:36.237 | INFO | Downloaded 34GiB to GPU in 2.04676s (16.6116GiB/s) | 20260505 18:24:41.375 | INFO | Analyzing chunk frames = [5338624,5339136) | 20260505 18:24:41.375 | INFO | MemInfo: { | 20260505 18:24:41.375 | INFO | "gpu" : " 17 GB", | 20260505 18:24:41.375 | INFO | "huge" : "", | 20260505 18:24:41.375 | INFO | "huge_shared" : " 27 GB", | 20260505 18:24:41.375 | INFO | "malloc" : " 14 GB", | 20260505 18:24:41.375 | INFO | "pinned" : " 29 GB" | 20260505 18:24:41.375 | INFO | } | 20260505 18:24:41.375 | INFO | Perf report for the last 5.13875 seconds: | 20260505 18:24:41.375 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:24:41.375 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:41.375 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:41.375 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:41.375 | INFO | ANALYSIS : 62.8%, 3.0, 97.5ms, 0 | 20260505 18:24:41.375 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:41.375 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:41.375 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:24:41.375 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:41.375 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:41.375 | INFO | Uploaded 36GiB to GPU in 2.22825s (16.1562GiB/s) | 20260505 18:24:41.375 | INFO | Downloaded 34GiB to GPU in 2.12048s (16.0341GiB/s) | 20260505 18:24:46.515 | INFO | Analyzing chunk frames = [5339136,5339648) | 20260505 18:24:46.515 | INFO | MemInfo: { | 20260505 18:24:46.515 | INFO | "gpu" : " 17 GB", | 20260505 18:24:46.515 | INFO | "huge" : "", | 20260505 18:24:46.515 | INFO | "huge_shared" : " 27 GB", | 20260505 18:24:46.515 | INFO | "malloc" : " 14 GB", | 20260505 18:24:46.515 | INFO | "pinned" : " 29 GB" | 20260505 18:24:46.515 | INFO | } | 20260505 18:24:46.516 | INFO | Perf report for the last 5.14043 seconds: | 20260505 18:24:46.516 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:24:46.516 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:46.516 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:46.516 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:46.516 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260505 18:24:46.516 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:46.516 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:46.516 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:24:46.516 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:46.516 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:46.516 | INFO | Uploaded 36GiB to GPU in 2.13582s (16.8554GiB/s) | 20260505 18:24:46.516 | INFO | Downloaded 34GiB to GPU in 2.05572s (16.5392GiB/s) | 20260505 18:24:51.655 | INFO | Analyzing chunk frames = [5339648,5340160) | 20260505 18:24:51.655 | INFO | MemInfo: { | 20260505 18:24:51.655 | INFO | "gpu" : " 17 GB", | 20260505 18:24:51.655 | INFO | "huge" : "", | 20260505 18:24:51.655 | INFO | "huge_shared" : " 27 GB", | 20260505 18:24:51.655 | INFO | "malloc" : " 14 GB", | 20260505 18:24:51.655 | INFO | "pinned" : " 29 GB" | 20260505 18:24:51.655 | INFO | } | 20260505 18:24:51.655 | INFO | Perf report for the last 5.13971 seconds: | 20260505 18:24:51.655 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:24:51.655 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:51.655 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:51.655 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:51.655 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 18:24:51.655 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:51.655 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:51.655 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 18:24:51.655 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:51.655 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:51.656 | INFO | Uploaded 36GiB to GPU in 2.14133s (16.812GiB/s) | 20260505 18:24:51.656 | INFO | Downloaded 34GiB to GPU in 2.05203s (16.5689GiB/s) | 20260505 18:24:56.786 | INFO | Analyzing chunk frames = [5340160,5340672) | 20260505 18:24:56.786 | INFO | MemInfo: { | 20260505 18:24:56.786 | INFO | "gpu" : " 17 GB", | 20260505 18:24:56.786 | INFO | "huge" : "", | 20260505 18:24:56.786 | INFO | "huge_shared" : " 27 GB", | 20260505 18:24:56.786 | INFO | "malloc" : " 14 GB", | 20260505 18:24:56.786 | INFO | "pinned" : " 29 GB" | 20260505 18:24:56.786 | INFO | } | 20260505 18:24:56.786 | INFO | Perf report for the last 5.1308 seconds: | 20260505 18:24:56.786 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:24:56.786 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:56.786 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:56.786 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:56.786 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 18:24:56.786 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:56.786 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:56.786 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 18:24:56.786 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:56.786 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:24:56.786 | INFO | Uploaded 36GiB to GPU in 2.13354s (16.8733GiB/s) | 20260505 18:24:56.786 | INFO | Downloaded 34GiB to GPU in 2.14553s (15.8469GiB/s) | 20260505 18:25:01.930 | INFO | Analyzing chunk frames = [5340672,5341184) | 20260505 18:25:01.930 | INFO | MemInfo: { | 20260505 18:25:01.930 | INFO | "gpu" : " 17 GB", | 20260505 18:25:01.930 | INFO | "huge" : "", | 20260505 18:25:01.930 | INFO | "huge_shared" : " 27 GB", | 20260505 18:25:01.930 | INFO | "malloc" : " 14 GB", | 20260505 18:25:01.930 | INFO | "pinned" : " 29 GB" | 20260505 18:25:01.930 | INFO | } | 20260505 18:25:01.930 | INFO | Perf report for the last 5.14437 seconds: | 20260505 18:25:01.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:25:01.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:01.930 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:01.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:01.930 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 18:25:01.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:01.930 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:01.931 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 18:25:01.931 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:01.931 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:01.931 | INFO | Uploaded 36GiB to GPU in 2.17831s (16.5266GiB/s) | 20260505 18:25:01.931 | INFO | Downloaded 34GiB to GPU in 2.02909s (16.7563GiB/s) | 20260505 18:25:01.931 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 18:25:01.931 | INFO | Total mem usage: 15.3 GiB | 20260505 18:25:01.931 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 18:25:01.931 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.04%) | 20260505 18:25:01.931 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 18:25:01.931 | INFO | High Water Breakdown: | 20260505 18:25:01.931 | INFO | 61% : 8.66 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 18:25:01.931 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 18:25:01.931 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 18:25:01.931 | INFO | ------------------------------------------ | 20260505 18:25:01.931 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:25:01.931 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 18:25:01.931 | INFO | Total mem usage: 26.5 GiB | 20260505 18:25:01.931 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 18:25:01.931 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 18:25:01.931 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 18:25:01.931 | INFO | High Water Breakdown: | 20260505 18:25:01.931 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:25:01.931 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:25:01.931 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:25:01.931 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:25:01.931 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:25:01.931 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:25:01.931 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 18:25:01.931 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 18:25:01.931 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:25:01.931 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 18:25:01.931 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 18:25:01.931 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 18:25:01.931 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 18:25:01.931 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 18:25:01.931 | INFO | ------------------------------------------ | 20260505 18:25:01.931 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:25:01.931 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 18:25:01.931 | INFO | Total mem usage: 25.6 GiB | 20260505 18:25:01.931 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 18:25:01.931 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 18:25:01.931 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 18:25:01.931 | INFO | High Water Breakdown: | 20260505 18:25:01.931 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 18:25:01.931 | INFO | ------------------------------------------ | 20260505 18:25:01.931 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:25:01.931 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 18:25:01.931 | INFO | Total mem usage: 15.7 GiB | 20260505 18:25:01.931 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 18:25:01.931 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 18:25:01.931 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 18:25:01.931 | INFO | High Water Breakdown: | 20260505 18:25:01.931 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 18:25:01.931 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 18:25:01.931 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 18:25:01.931 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 18:25:01.932 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 18:25:01.932 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 18:25:01.932 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 18:25:01.932 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 18:25:01.932 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 18:25:01.932 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:25:01.932 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 18:25:01.932 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:25:01.932 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 18:25:01.932 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 18:25:01.932 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 18:25:01.932 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 18:25:01.932 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:25:01.932 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 18:25:01.932 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:25:01.932 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:25:01.932 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:25:01.932 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 18:25:01.932 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 18:25:01.932 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:25:01.932 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 18:25:01.932 | INFO | ------------------------------------------ | 20260505 18:25:01.932 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:25:01.932 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 18:25:01.932 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:25:01.932 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:25:01.932 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.47e+01, 1.37e+01 / 2.07e+00 | 20260505 18:25:01.932 | INFO | PerfReport: 0.29%, AggregateStats -- 9.25e-02 / 1.01e-01, 1.03e-02 / 3.39e-03 | 20260505 18:25:01.932 | INFO | PerfReport: 11.79%, Baseline -- 3.71e+00 / 4.56e+00, 5.63e-01 / 1.10e-01 | 20260505 18:25:01.932 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.53e+00, 1.92e-01 / 8.22e-02 | 20260505 18:25:01.932 | INFO | PerfReport: 15.95%, DME -- 4.02e+01 / 4.45e+01, 4.10e+00 / 2.05e+00 | 20260505 18:25:01.932 | INFO | PerfReport: 56.86%, FrameLabeling -- 1.79e+01 / 1.93e+01, 1.42e+00 / 1.04e+00 | 20260505 18:25:01.932 | INFO | PerfReport: 7.34%, PulseAccumulating -- 2.31e+00 / 2.89e+00, 4.43e-01 / 1.75e-01 | 20260505 18:25:01.932 | INFO | PerfReport: 3.34%, Metrics -- 1.05e+00 / 1.42e+00, 2.89e-01 / 1.85e-01 | 20260505 18:25:01.932 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.65e-01 / 4.29e-01, 6.35e-02 / 3.22e-02 | 20260505 18:25:01.932 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 18:25:01.932 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:25:01.932 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:25:01.932 | INFO | PerfReport: Full Execution -- 9.98e+01 / -nan, 2.58e+01 / 0.00e+00 | 20260505 18:25:01.932 | INFO | PerfReport: 22.59%, Upload -- 2.25e+01 / 0.00e+00, 7.88e+00 / 0.00e+00 | 20260505 18:25:01.932 | INFO | PerfReport: 21.77%, Download -- 2.17e+01 / 0.00e+00, 7.77e+00 / 0.00e+00 | 20260505 18:25:01.932 | INFO | PerfReport: 31.56%, Compute -- 3.15e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 18:25:01.932 | INFO | PerfReport: 2.44%, LockUpload -- 2.43e+00 / 0.00e+00, 8.03e+00 / 0.00e+00 | 20260505 18:25:01.932 | INFO | PerfReport: 1.35%, LockDownload -- 1.35e+00 / 0.00e+00, 2.81e+00 / 0.00e+00 | 20260505 18:25:01.932 | INFO | PerfReport: 20.27%, LockCompute -- 2.02e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260505 18:25:01.932 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.77e-02 / 0.00e+00 | 20260505 18:25:02.065 | INFO | Starting a round of Baz IO | 20260505 18:25:05.236 | INFO | Populating RTMetrics report | 20260505 18:25:05.237 | INFO | RTMetrics report populated | 20260505 18:25:05.237 | INFO | Opening tmp RTMetrics json file | 20260505 18:25:05.239 | INFO | Serializing RTMetrics to json | 20260505 18:25:05.242 | INFO | Serialization done, swapping tmp file now... | 20260505 18:25:05.242 | INFO | RTMetrics done | 20260505 18:25:05.323 | INFO | Finished a round of Baz IO: Wrote 2.093 GiB in 3.258 seconds | 20260505 18:25:05.323 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.91% / 0.00% / 78.09% | 20260505 18:25:05.323 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.251341% / 0% / 6.35783e-05% | 20260505 18:25:05.323 | INFO | Baz IO overall: | 20260505 18:25:05.323 | INFO | BytesWritten : 3144926953842 | 20260505 18:25:05.323 | INFO | NumEvents : 1489872320000 | 20260505 18:25:07.071 | INFO | Analyzing chunk frames = [5341184,5341696) | 20260505 18:25:07.071 | INFO | MemInfo: { | 20260505 18:25:07.071 | INFO | "gpu" : " 17 GB", | 20260505 18:25:07.071 | INFO | "huge" : "", | 20260505 18:25:07.071 | INFO | "huge_shared" : " 27 GB", | 20260505 18:25:07.071 | INFO | "malloc" : " 14 GB", | 20260505 18:25:07.071 | INFO | "pinned" : " 29 GB" | 20260505 18:25:07.071 | INFO | } | 20260505 18:25:07.071 | INFO | Perf report for the last 5.14098 seconds: | 20260505 18:25:07.071 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:25:07.072 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:07.072 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:07.072 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:07.072 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 18:25:07.072 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:07.072 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260505 18:25:07.072 | INFO | PRE_HQ : 60.2%, 1.8, 55.5ms, 0 | 20260505 18:25:07.072 | INFO | BAZWRITER : 62.9%, 3.8, 122.2ms, 0 | 20260505 18:25:07.072 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 18:25:07.072 | INFO | Uploaded 36GiB to GPU in 2.15562s (16.7005GiB/s) | 20260505 18:25:07.072 | INFO | Downloaded 38GiB to GPU in 2.23993s (16.9648GiB/s) | 20260505 18:25:12.215 | INFO | Analyzing chunk frames = [5341696,5342208) | 20260505 18:25:12.215 | INFO | MemInfo: { | 20260505 18:25:12.215 | INFO | "gpu" : " 17 GB", | 20260505 18:25:12.215 | INFO | "huge" : "", | 20260505 18:25:12.215 | INFO | "huge_shared" : " 27 GB", | 20260505 18:25:12.215 | INFO | "malloc" : " 14 GB", | 20260505 18:25:12.215 | INFO | "pinned" : " 29 GB" | 20260505 18:25:12.215 | INFO | } | 20260505 18:25:12.215 | INFO | Perf report for the last 5.14405 seconds: | 20260505 18:25:12.216 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:25:12.216 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:12.216 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:12.216 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:12.216 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 18:25:12.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:12.216 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:12.216 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 18:25:12.216 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:12.216 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:12.216 | INFO | Uploaded 36GiB to GPU in 2.14459s (16.7864GiB/s) | 20260505 18:25:12.216 | INFO | Downloaded 34GiB to GPU in 2.04061s (16.6617GiB/s) | 20260505 18:25:17.348 | INFO | Analyzing chunk frames = [5342208,5342720) | 20260505 18:25:17.348 | INFO | MemInfo: { | 20260505 18:25:17.348 | INFO | "gpu" : " 17 GB", | 20260505 18:25:17.348 | INFO | "huge" : "", | 20260505 18:25:17.348 | INFO | "huge_shared" : " 27 GB", | 20260505 18:25:17.348 | INFO | "malloc" : " 14 GB", | 20260505 18:25:17.348 | INFO | "pinned" : " 29 GB" | 20260505 18:25:17.348 | INFO | } | 20260505 18:25:17.348 | INFO | Perf report for the last 5.13239 seconds: | 20260505 18:25:17.348 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:25:17.348 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:17.348 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:17.348 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:17.348 | INFO | ANALYSIS : 62.7%, 3.0, 97.5ms, 0 | 20260505 18:25:17.348 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:17.348 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:17.348 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 18:25:17.348 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:17.348 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:17.348 | INFO | Uploaded 36GiB to GPU in 2.20712s (16.3109GiB/s) | 20260505 18:25:17.348 | INFO | Downloaded 34GiB to GPU in 2.07233s (16.4066GiB/s) | 20260505 18:25:22.485 | INFO | Analyzing chunk frames = [5342720,5343232) | 20260505 18:25:22.485 | INFO | MemInfo: { | 20260505 18:25:22.485 | INFO | "gpu" : " 17 GB", | 20260505 18:25:22.485 | INFO | "huge" : "", | 20260505 18:25:22.485 | INFO | "huge_shared" : " 27 GB", | 20260505 18:25:22.485 | INFO | "malloc" : " 14 GB", | 20260505 18:25:22.485 | INFO | "pinned" : " 29 GB" | 20260505 18:25:22.485 | INFO | } | 20260505 18:25:22.486 | INFO | Perf report for the last 5.13768 seconds: | 20260505 18:25:22.486 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:25:22.486 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:22.486 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:22.486 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:22.486 | INFO | ANALYSIS : 62.6%, 3.0, 97.1ms, 0 | 20260505 18:25:22.486 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:22.486 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:22.486 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:25:22.486 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:22.486 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:22.486 | INFO | Uploaded 36GiB to GPU in 2.16256s (16.6469GiB/s) | 20260505 18:25:22.486 | INFO | Downloaded 34GiB to GPU in 2.1452s (15.8493GiB/s) | 20260505 18:25:27.634 | INFO | Analyzing chunk frames = [5343232,5343744) | 20260505 18:25:27.634 | INFO | MemInfo: { | 20260505 18:25:27.634 | INFO | "gpu" : " 17 GB", | 20260505 18:25:27.634 | INFO | "huge" : "", | 20260505 18:25:27.634 | INFO | "huge_shared" : " 27 GB", | 20260505 18:25:27.634 | INFO | "malloc" : " 14 GB", | 20260505 18:25:27.634 | INFO | "pinned" : " 29 GB" | 20260505 18:25:27.634 | INFO | } | 20260505 18:25:27.634 | INFO | Perf report for the last 5.14842 seconds: | 20260505 18:25:27.634 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:25:27.634 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:27.634 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:27.634 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:27.634 | INFO | ANALYSIS : 62.0%, 3.0, 97.0ms, 0 | 20260505 18:25:27.634 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:27.634 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:27.634 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:25:27.634 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:27.634 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:27.634 | INFO | Uploaded 36GiB to GPU in 2.15244s (16.7252GiB/s) | 20260505 18:25:27.634 | INFO | Downloaded 34GiB to GPU in 2.05706s (16.5284GiB/s) | 20260505 18:25:32.773 | INFO | Analyzing chunk frames = [5343744,5344256) | 20260505 18:25:32.773 | INFO | MemInfo: { | 20260505 18:25:32.773 | INFO | "gpu" : " 17 GB", | 20260505 18:25:32.773 | INFO | "huge" : "", | 20260505 18:25:32.773 | INFO | "huge_shared" : " 27 GB", | 20260505 18:25:32.773 | INFO | "malloc" : " 14 GB", | 20260505 18:25:32.773 | INFO | "pinned" : " 29 GB" | 20260505 18:25:32.773 | INFO | } | 20260505 18:25:32.773 | INFO | Perf report for the last 5.1392 seconds: | 20260505 18:25:32.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:25:32.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:32.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:32.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:32.774 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 18:25:32.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:32.774 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:32.774 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:25:32.774 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:32.774 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:32.774 | INFO | Uploaded 36GiB to GPU in 2.15452s (16.7091GiB/s) | 20260505 18:25:32.774 | INFO | Downloaded 34GiB to GPU in 2.06175s (16.4908GiB/s) | 20260505 18:25:37.903 | INFO | Analyzing chunk frames = [5344256,5344768) | 20260505 18:25:37.903 | INFO | MemInfo: { | 20260505 18:25:37.903 | INFO | "gpu" : " 17 GB", | 20260505 18:25:37.903 | INFO | "huge" : "", | 20260505 18:25:37.903 | INFO | "huge_shared" : " 27 GB", | 20260505 18:25:37.903 | INFO | "malloc" : " 14 GB", | 20260505 18:25:37.903 | INFO | "pinned" : " 29 GB" | 20260505 18:25:37.903 | INFO | } | 20260505 18:25:37.903 | INFO | Perf report for the last 5.13007 seconds: | 20260505 18:25:37.903 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:25:37.903 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:37.903 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:37.903 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:37.903 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 18:25:37.903 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:37.903 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:37.903 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 18:25:37.903 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:37.903 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:37.904 | INFO | Uploaded 36GiB to GPU in 2.11066s (17.0563GiB/s) | 20260505 18:25:37.904 | INFO | Downloaded 34GiB to GPU in 2.13751s (15.9064GiB/s) | 20260505 18:25:43.040 | INFO | Analyzing chunk frames = [5344768,5345280) | 20260505 18:25:43.040 | INFO | MemInfo: { | 20260505 18:25:43.040 | INFO | "gpu" : " 17 GB", | 20260505 18:25:43.040 | INFO | "huge" : "", | 20260505 18:25:43.040 | INFO | "huge_shared" : " 27 GB", | 20260505 18:25:43.040 | INFO | "malloc" : " 14 GB", | 20260505 18:25:43.040 | INFO | "pinned" : " 29 GB" | 20260505 18:25:43.040 | INFO | } | 20260505 18:25:43.040 | INFO | Perf report for the last 5.13665 seconds: | 20260505 18:25:43.040 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:25:43.040 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:43.040 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:43.040 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:43.040 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 18:25:43.040 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:43.040 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:43.040 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:25:43.040 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:43.040 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:43.040 | INFO | Uploaded 36GiB to GPU in 2.20086s (16.3573GiB/s) | 20260505 18:25:43.040 | INFO | Downloaded 34GiB to GPU in 2.03893s (16.6754GiB/s) | 20260505 18:25:43.180 | INFO | Starting a round of Baz IO | 20260505 18:25:46.363 | INFO | Populating RTMetrics report | 20260505 18:25:46.363 | INFO | RTMetrics report populated | 20260505 18:25:46.363 | INFO | Opening tmp RTMetrics json file | 20260505 18:25:46.372 | INFO | Serializing RTMetrics to json | 20260505 18:25:46.375 | INFO | Serialization done, swapping tmp file now... | 20260505 18:25:46.375 | INFO | RTMetrics done | 20260505 18:25:46.451 | INFO | Finished a round of Baz IO: Wrote 2.091 GiB in 3.27 seconds | 20260505 18:25:46.451 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.91% / 0.00% / 78.09% | 20260505 18:25:46.451 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.249998% / 7.94729e-06% / 6.35783e-05% | 20260505 18:25:46.451 | INFO | Baz IO overall: | 20260505 18:25:46.451 | INFO | BytesWritten : 3147171691045 | 20260505 18:25:46.451 | INFO | NumEvents : 1490917936811 | 20260505 18:25:48.185 | INFO | Analyzing chunk frames = [5345280,5345792) | 20260505 18:25:48.185 | INFO | MemInfo: { | 20260505 18:25:48.185 | INFO | "gpu" : " 17 GB", | 20260505 18:25:48.185 | INFO | "huge" : "", | 20260505 18:25:48.185 | INFO | "huge_shared" : " 27 GB", | 20260505 18:25:48.185 | INFO | "malloc" : " 14 GB", | 20260505 18:25:48.185 | INFO | "pinned" : " 29 GB" | 20260505 18:25:48.185 | INFO | } | 20260505 18:25:48.185 | INFO | Perf report for the last 5.14519 seconds: | 20260505 18:25:48.185 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:25:48.185 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:48.185 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:48.185 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:48.185 | INFO | ANALYSIS : 63.5%, 3.0, 99.0ms, 0 | 20260505 18:25:48.185 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:48.185 | INFO | RT_METRICS : 44.5%, 1.0, 23.4ms, 0 | 20260505 18:25:48.185 | INFO | PRE_HQ : 60.3%, 1.8, 56.0ms, 0 | 20260505 18:25:48.185 | INFO | BAZWRITER : 63.0%, 3.8, 122.7ms, 0 | 20260505 18:25:48.185 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260505 18:25:48.186 | INFO | Uploaded 36GiB to GPU in 2.18175s (16.5005GiB/s) | 20260505 18:25:48.186 | INFO | Downloaded 38GiB to GPU in 2.26951s (16.7437GiB/s) | 20260505 18:25:53.325 | INFO | Analyzing chunk frames = [5345792,5346304) | 20260505 18:25:53.325 | INFO | MemInfo: { | 20260505 18:25:53.325 | INFO | "gpu" : " 17 GB", | 20260505 18:25:53.325 | INFO | "huge" : "", | 20260505 18:25:53.325 | INFO | "huge_shared" : " 27 GB", | 20260505 18:25:53.325 | INFO | "malloc" : " 14 GB", | 20260505 18:25:53.325 | INFO | "pinned" : " 29 GB" | 20260505 18:25:53.325 | INFO | } | 20260505 18:25:53.325 | INFO | Perf report for the last 5.13985 seconds: | 20260505 18:25:53.325 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:25:53.325 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:53.325 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:53.325 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:53.325 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 18:25:53.325 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:53.325 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:53.325 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 18:25:53.325 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:53.325 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:53.325 | INFO | Uploaded 36GiB to GPU in 2.13414s (16.8686GiB/s) | 20260505 18:25:53.325 | INFO | Downloaded 34GiB to GPU in 2.02707s (16.773GiB/s) | 20260505 18:25:58.467 | INFO | Analyzing chunk frames = [5346304,5346816) | 20260505 18:25:58.467 | INFO | MemInfo: { | 20260505 18:25:58.467 | INFO | "gpu" : " 17 GB", | 20260505 18:25:58.467 | INFO | "huge" : "", | 20260505 18:25:58.467 | INFO | "huge_shared" : " 27 GB", | 20260505 18:25:58.467 | INFO | "malloc" : " 14 GB", | 20260505 18:25:58.467 | INFO | "pinned" : " 29 GB" | 20260505 18:25:58.467 | INFO | } | 20260505 18:25:58.467 | INFO | Perf report for the last 5.14182 seconds: | 20260505 18:25:58.467 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:25:58.467 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:58.467 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:58.467 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:58.467 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260505 18:25:58.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:58.467 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:58.467 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:25:58.467 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:58.467 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:25:58.467 | INFO | Uploaded 36GiB to GPU in 2.14005s (16.8221GiB/s) | 20260505 18:25:58.467 | INFO | Downloaded 34GiB to GPU in 2.03791s (16.6837GiB/s) | 20260505 18:26:03.601 | INFO | Analyzing chunk frames = [5346816,5347328) | 20260505 18:26:03.601 | INFO | MemInfo: { | 20260505 18:26:03.601 | INFO | "gpu" : " 17 GB", | 20260505 18:26:03.601 | INFO | "huge" : "", | 20260505 18:26:03.601 | INFO | "huge_shared" : " 27 GB", | 20260505 18:26:03.601 | INFO | "malloc" : " 14 GB", | 20260505 18:26:03.601 | INFO | "pinned" : " 29 GB" | 20260505 18:26:03.602 | INFO | } | 20260505 18:26:03.602 | INFO | Perf report for the last 5.13474 seconds: | 20260505 18:26:03.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:26:03.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:03.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:03.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:03.602 | INFO | ANALYSIS : 62.4%, 3.0, 96.9ms, 0 | 20260505 18:26:03.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:03.602 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:03.602 | INFO | PRE_HQ : 27.7%, 1.0, 14.3ms, 0 | 20260505 18:26:03.602 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:03.602 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:03.602 | INFO | Uploaded 36GiB to GPU in 2.1953s (16.3986GiB/s) | 20260505 18:26:03.602 | INFO | Downloaded 34GiB to GPU in 2.06448s (16.4691GiB/s) | 20260505 18:26:08.735 | INFO | Analyzing chunk frames = [5347328,5347840) | 20260505 18:26:08.735 | INFO | MemInfo: { | 20260505 18:26:08.735 | INFO | "gpu" : " 17 GB", | 20260505 18:26:08.735 | INFO | "huge" : "", | 20260505 18:26:08.735 | INFO | "huge_shared" : " 27 GB", | 20260505 18:26:08.735 | INFO | "malloc" : " 14 GB", | 20260505 18:26:08.735 | INFO | "pinned" : " 29 GB" | 20260505 18:26:08.735 | INFO | } | 20260505 18:26:08.735 | INFO | Perf report for the last 5.13345 seconds: | 20260505 18:26:08.735 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:26:08.735 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:08.735 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:08.735 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:08.735 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260505 18:26:08.735 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:08.735 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:08.735 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:26:08.735 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:08.735 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:08.735 | INFO | Uploaded 36GiB to GPU in 2.12977s (16.9032GiB/s) | 20260505 18:26:08.735 | INFO | Downloaded 34GiB to GPU in 2.05711s (16.5281GiB/s) | 20260505 18:26:13.882 | INFO | Analyzing chunk frames = [5347840,5348352) | 20260505 18:26:13.882 | INFO | MemInfo: { | 20260505 18:26:13.882 | INFO | "gpu" : " 17 GB", | 20260505 18:26:13.882 | INFO | "huge" : "", | 20260505 18:26:13.882 | INFO | "huge_shared" : " 27 GB", | 20260505 18:26:13.882 | INFO | "malloc" : " 14 GB", | 20260505 18:26:13.882 | INFO | "pinned" : " 29 GB" | 20260505 18:26:13.882 | INFO | } | 20260505 18:26:13.882 | INFO | Perf report for the last 5.14688 seconds: | 20260505 18:26:13.882 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:26:13.882 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:13.882 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:13.882 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:13.882 | INFO | ANALYSIS : 62.3%, 3.0, 97.5ms, 0 | 20260505 18:26:13.882 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:13.882 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:13.882 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:26:13.882 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:13.882 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:13.882 | INFO | Uploaded 36GiB to GPU in 2.14692s (16.7682GiB/s) | 20260505 18:26:13.882 | INFO | Downloaded 34GiB to GPU in 2.04586s (16.6189GiB/s) | 20260505 18:26:19.024 | INFO | Analyzing chunk frames = [5348352,5348864) | 20260505 18:26:19.024 | INFO | MemInfo: { | 20260505 18:26:19.024 | INFO | "gpu" : " 17 GB", | 20260505 18:26:19.024 | INFO | "huge" : "", | 20260505 18:26:19.024 | INFO | "huge_shared" : " 27 GB", | 20260505 18:26:19.025 | INFO | "malloc" : " 14 GB", | 20260505 18:26:19.025 | INFO | "pinned" : " 29 GB" | 20260505 18:26:19.025 | INFO | } | 20260505 18:26:19.025 | INFO | Perf report for the last 5.1427 seconds: | 20260505 18:26:19.025 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:26:19.025 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:19.025 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:19.025 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:19.025 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 18:26:19.025 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:19.025 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:19.025 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 18:26:19.025 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:19.025 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:19.025 | INFO | Uploaded 36GiB to GPU in 2.12621s (16.9316GiB/s) | 20260505 18:26:19.025 | INFO | Downloaded 34GiB to GPU in 2.12818s (15.9761GiB/s) | 20260505 18:26:24.161 | INFO | Analyzing chunk frames = [5348864,5349376) | 20260505 18:26:24.161 | INFO | MemInfo: { | 20260505 18:26:24.161 | INFO | "gpu" : " 17 GB", | 20260505 18:26:24.161 | INFO | "huge" : "", | 20260505 18:26:24.161 | INFO | "huge_shared" : " 27 GB", | 20260505 18:26:24.162 | INFO | "malloc" : " 14 GB", | 20260505 18:26:24.162 | INFO | "pinned" : " 29 GB" | 20260505 18:26:24.162 | INFO | } | 20260505 18:26:24.162 | INFO | Perf report for the last 5.1369 seconds: | 20260505 18:26:24.162 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:26:24.162 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:24.162 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:24.162 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:24.162 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 18:26:24.162 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:24.162 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:24.162 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 18:26:24.162 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:24.162 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:24.162 | INFO | Uploaded 36GiB to GPU in 2.19339s (16.4129GiB/s) | 20260505 18:26:24.162 | INFO | Downloaded 34GiB to GPU in 2.01831s (16.8458GiB/s) | 20260505 18:26:24.162 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 18:26:24.162 | INFO | Total mem usage: 15.3 GiB | 20260505 18:26:24.162 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 18:26:24.162 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.01%) | 20260505 18:26:24.162 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 18:26:24.162 | INFO | High Water Breakdown: | 20260505 18:26:24.162 | INFO | 61% : 8.66 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 18:26:24.162 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 18:26:24.162 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 18:26:24.162 | INFO | ------------------------------------------ | 20260505 18:26:24.162 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:26:24.162 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 18:26:24.162 | INFO | Total mem usage: 26.5 GiB | 20260505 18:26:24.162 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 18:26:24.162 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 18:26:24.162 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 18:26:24.162 | INFO | High Water Breakdown: | 20260505 18:26:24.162 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:26:24.162 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:26:24.162 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:26:24.162 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:26:24.162 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:26:24.162 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:26:24.162 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 18:26:24.162 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 18:26:24.162 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:26:24.162 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 18:26:24.162 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 18:26:24.162 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 18:26:24.162 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 18:26:24.162 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 18:26:24.162 | INFO | ------------------------------------------ | 20260505 18:26:24.163 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:26:24.163 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 18:26:24.163 | INFO | Total mem usage: 25.6 GiB | 20260505 18:26:24.163 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 18:26:24.163 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 18:26:24.163 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 18:26:24.163 | INFO | High Water Breakdown: | 20260505 18:26:24.163 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 18:26:24.163 | INFO | ------------------------------------------ | 20260505 18:26:24.163 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:26:24.163 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 18:26:24.163 | INFO | Total mem usage: 15.7 GiB | 20260505 18:26:24.163 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 18:26:24.163 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 18:26:24.163 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 18:26:24.163 | INFO | High Water Breakdown: | 20260505 18:26:24.163 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 18:26:24.163 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 18:26:24.163 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 18:26:24.163 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 18:26:24.163 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 18:26:24.163 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 18:26:24.163 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 18:26:24.163 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 18:26:24.163 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 18:26:24.163 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:26:24.163 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 18:26:24.163 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:26:24.163 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 18:26:24.163 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 18:26:24.163 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 18:26:24.163 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 18:26:24.163 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:26:24.163 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 18:26:24.163 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:26:24.163 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:26:24.163 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:26:24.163 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 18:26:24.163 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 18:26:24.163 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:26:24.163 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 18:26:24.163 | INFO | ------------------------------------------ | 20260505 18:26:24.163 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:26:24.163 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 18:26:24.163 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:26:24.163 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:26:24.163 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.48e+01, 1.35e+01 / 2.24e+00 | 20260505 18:26:24.163 | INFO | PerfReport: 0.29%, AggregateStats -- 9.26e-02 / 1.03e-01, 1.08e-02 / 5.50e-03 | 20260505 18:26:24.163 | INFO | PerfReport: 11.78%, Baseline -- 3.72e+00 / 4.53e+00, 5.64e-01 / 1.75e-01 | 20260505 18:26:24.163 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.50e+00, 1.92e-01 / 1.07e-01 | 20260505 18:26:24.163 | INFO | PerfReport: 15.94%, DME -- 4.02e+01 / 4.46e+01, 4.09e+00 / 2.44e+00 | 20260505 18:26:24.163 | INFO | PerfReport: 56.88%, FrameLabeling -- 1.79e+01 / 1.93e+01, 1.43e+00 / 9.56e-01 | 20260505 18:26:24.163 | INFO | PerfReport: 7.32%, PulseAccumulating -- 2.31e+00 / 2.89e+00, 4.45e-01 / 2.05e-01 | 20260505 18:26:24.163 | INFO | PerfReport: 3.33%, Metrics -- 1.05e+00 / 1.38e+00, 2.89e-01 / 1.17e-01 | 20260505 18:26:24.163 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.66e-01 / 4.31e-01, 6.46e-02 / 3.19e-02 | 20260505 18:26:24.163 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 18:26:24.163 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:26:24.163 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:26:24.163 | INFO | PerfReport: Full Execution -- 9.98e+01 / -nan, 2.57e+01 / 0.00e+00 | 20260505 18:26:24.163 | INFO | PerfReport: 22.54%, Upload -- 2.25e+01 / 0.00e+00, 7.84e+00 / 0.00e+00 | 20260505 18:26:24.163 | INFO | PerfReport: 21.83%, Download -- 2.18e+01 / 0.00e+00, 7.79e+00 / 0.00e+00 | 20260505 18:26:24.163 | INFO | PerfReport: 31.61%, Compute -- 3.15e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 18:26:24.163 | INFO | PerfReport: 2.26%, LockUpload -- 2.25e+00 / 0.00e+00, 7.91e+00 / 0.00e+00 | 20260505 18:26:24.163 | INFO | PerfReport: 1.41%, LockDownload -- 1.41e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260505 18:26:24.163 | INFO | PerfReport: 20.34%, LockCompute -- 2.03e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260505 18:26:24.163 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.98e-02 / 0.00e+00 | 20260505 18:26:24.297 | INFO | Starting a round of Baz IO | 20260505 18:26:27.465 | INFO | Populating RTMetrics report | 20260505 18:26:27.465 | INFO | RTMetrics report populated | 20260505 18:26:27.465 | INFO | Opening tmp RTMetrics json file | 20260505 18:26:27.470 | INFO | Serializing RTMetrics to json | 20260505 18:26:27.471 | INFO | Serialization done, swapping tmp file now... | 20260505 18:26:27.472 | INFO | RTMetrics done | 20260505 18:26:27.561 | INFO | Finished a round of Baz IO: Wrote 2.092 GiB in 3.263 seconds | 20260505 18:26:27.561 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.91% / 0.00% / 78.09% | 20260505 18:26:27.561 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.249958% / 0% / 8.34465e-05% | 20260505 18:26:27.561 | INFO | Baz IO overall: | 20260505 18:26:27.561 | INFO | BytesWritten : 3149417577696 | 20260505 18:26:27.561 | INFO | NumEvents : 1491964369194 | 20260505 18:26:29.296 | INFO | Analyzing chunk frames = [5349376,5349888) | 20260505 18:26:29.296 | INFO | MemInfo: { | 20260505 18:26:29.296 | INFO | "gpu" : " 17 GB", | 20260505 18:26:29.296 | INFO | "huge" : "", | 20260505 18:26:29.296 | INFO | "huge_shared" : " 27 GB", | 20260505 18:26:29.296 | INFO | "malloc" : " 14 GB", | 20260505 18:26:29.296 | INFO | "pinned" : " 29 GB" | 20260505 18:26:29.296 | INFO | } | 20260505 18:26:29.296 | INFO | Perf report for the last 5.1343 seconds: | 20260505 18:26:29.296 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:26:29.296 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:29.296 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:29.296 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:29.296 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 18:26:29.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:29.296 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260505 18:26:29.296 | INFO | PRE_HQ : 60.5%, 1.8, 56.1ms, 0 | 20260505 18:26:29.296 | INFO | BAZWRITER : 63.0%, 3.8, 122.3ms, 0 | 20260505 18:26:29.296 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 18:26:29.296 | INFO | Uploaded 36GiB to GPU in 2.13208s (16.8849GiB/s) | 20260505 18:26:29.296 | INFO | Downloaded 38GiB to GPU in 2.23964s (16.967GiB/s) | 20260505 18:26:34.439 | INFO | Analyzing chunk frames = [5349888,5350400) | 20260505 18:26:34.439 | INFO | MemInfo: { | 20260505 18:26:34.439 | INFO | "gpu" : " 17 GB", | 20260505 18:26:34.439 | INFO | "huge" : "", | 20260505 18:26:34.439 | INFO | "huge_shared" : " 27 GB", | 20260505 18:26:34.439 | INFO | "malloc" : " 14 GB", | 20260505 18:26:34.439 | INFO | "pinned" : " 29 GB" | 20260505 18:26:34.439 | INFO | } | 20260505 18:26:34.439 | INFO | Perf report for the last 5.1429 seconds: | 20260505 18:26:34.439 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:26:34.439 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:34.439 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:34.439 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:34.439 | INFO | ANALYSIS : 62.7%, 3.0, 98.0ms, 0 | 20260505 18:26:34.439 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:34.439 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:34.439 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:26:34.439 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:34.439 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:34.439 | INFO | Uploaded 36GiB to GPU in 2.12899s (16.9094GiB/s) | 20260505 18:26:34.439 | INFO | Downloaded 34GiB to GPU in 2.02972s (16.7511GiB/s) | 20260505 18:26:39.582 | INFO | Analyzing chunk frames = [5350400,5350912) | 20260505 18:26:39.582 | INFO | MemInfo: { | 20260505 18:26:39.582 | INFO | "gpu" : " 17 GB", | 20260505 18:26:39.582 | INFO | "huge" : "", | 20260505 18:26:39.582 | INFO | "huge_shared" : " 27 GB", | 20260505 18:26:39.582 | INFO | "malloc" : " 14 GB", | 20260505 18:26:39.582 | INFO | "pinned" : " 29 GB" | 20260505 18:26:39.582 | INFO | } | 20260505 18:26:39.583 | INFO | Perf report for the last 5.14383 seconds: | 20260505 18:26:39.583 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:26:39.583 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:39.583 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:39.583 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:39.583 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260505 18:26:39.583 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:39.583 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:39.583 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:26:39.583 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:39.583 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:39.583 | INFO | Uploaded 36GiB to GPU in 2.20308s (16.3408GiB/s) | 20260505 18:26:39.583 | INFO | Downloaded 34GiB to GPU in 2.03095s (16.741GiB/s) | 20260505 18:26:44.719 | INFO | Analyzing chunk frames = [5350912,5351424) | 20260505 18:26:44.719 | INFO | MemInfo: { | 20260505 18:26:44.719 | INFO | "gpu" : " 17 GB", | 20260505 18:26:44.719 | INFO | "huge" : "", | 20260505 18:26:44.719 | INFO | "huge_shared" : " 27 GB", | 20260505 18:26:44.719 | INFO | "malloc" : " 14 GB", | 20260505 18:26:44.719 | INFO | "pinned" : " 29 GB" | 20260505 18:26:44.719 | INFO | } | 20260505 18:26:44.719 | INFO | Perf report for the last 5.13669 seconds: | 20260505 18:26:44.719 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:26:44.719 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:44.719 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:44.719 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:44.719 | INFO | ANALYSIS : 62.5%, 3.0, 97.0ms, 0 | 20260505 18:26:44.719 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:44.719 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:44.719 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 18:26:44.719 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:44.719 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:44.720 | INFO | Uploaded 36GiB to GPU in 2.2049s (16.3273GiB/s) | 20260505 18:26:44.720 | INFO | Downloaded 34GiB to GPU in 2.04032s (16.6641GiB/s) | 20260505 18:26:49.854 | INFO | Analyzing chunk frames = [5351424,5351936) | 20260505 18:26:49.854 | INFO | MemInfo: { | 20260505 18:26:49.854 | INFO | "gpu" : " 17 GB", | 20260505 18:26:49.854 | INFO | "huge" : "", | 20260505 18:26:49.854 | INFO | "huge_shared" : " 27 GB", | 20260505 18:26:49.854 | INFO | "malloc" : " 14 GB", | 20260505 18:26:49.854 | INFO | "pinned" : " 29 GB" | 20260505 18:26:49.854 | INFO | } | 20260505 18:26:49.855 | INFO | Perf report for the last 5.13529 seconds: | 20260505 18:26:49.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:26:49.855 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:49.855 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:49.855 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:49.855 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 18:26:49.855 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:49.855 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:49.855 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 18:26:49.855 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:49.855 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:49.855 | INFO | Uploaded 36GiB to GPU in 2.14898s (16.7522GiB/s) | 20260505 18:26:49.855 | INFO | Downloaded 34GiB to GPU in 2.06036s (16.5019GiB/s) | 20260505 18:26:54.991 | INFO | Analyzing chunk frames = [5351936,5352448) | 20260505 18:26:54.991 | INFO | MemInfo: { | 20260505 18:26:54.991 | INFO | "gpu" : " 17 GB", | 20260505 18:26:54.991 | INFO | "huge" : "", | 20260505 18:26:54.992 | INFO | "huge_shared" : " 27 GB", | 20260505 18:26:54.992 | INFO | "malloc" : " 14 GB", | 20260505 18:26:54.992 | INFO | "pinned" : " 29 GB" | 20260505 18:26:54.992 | INFO | } | 20260505 18:26:54.992 | INFO | Perf report for the last 5.13702 seconds: | 20260505 18:26:54.992 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:26:54.992 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:54.992 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:54.992 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:54.992 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 18:26:54.992 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:54.992 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:54.992 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 18:26:54.992 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:54.992 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:26:54.992 | INFO | Uploaded 36GiB to GPU in 2.1455s (16.7793GiB/s) | 20260505 18:26:54.992 | INFO | Downloaded 34GiB to GPU in 2.04991s (16.5861GiB/s) | 20260505 18:27:00.141 | INFO | Analyzing chunk frames = [5352448,5352960) | 20260505 18:27:00.141 | INFO | MemInfo: { | 20260505 18:27:00.141 | INFO | "gpu" : " 17 GB", | 20260505 18:27:00.141 | INFO | "huge" : "", | 20260505 18:27:00.141 | INFO | "huge_shared" : " 27 GB", | 20260505 18:27:00.141 | INFO | "malloc" : " 14 GB", | 20260505 18:27:00.141 | INFO | "pinned" : " 29 GB" | 20260505 18:27:00.141 | INFO | } | 20260505 18:27:00.141 | INFO | Perf report for the last 5.14956 seconds: | 20260505 18:27:00.141 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:27:00.141 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:00.141 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:00.141 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:00.141 | INFO | ANALYSIS : 63.1%, 3.0, 98.8ms, 0 | 20260505 18:27:00.141 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:00.141 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:00.141 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 18:27:00.141 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:00.141 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:00.142 | INFO | Uploaded 36GiB to GPU in 2.16015s (16.6655GiB/s) | 20260505 18:27:00.142 | INFO | Downloaded 34GiB to GPU in 2.17086s (15.662GiB/s) | 20260505 18:27:05.279 | INFO | Analyzing chunk frames = [5352960,5353472) | 20260505 18:27:05.279 | INFO | MemInfo: { | 20260505 18:27:05.279 | INFO | "gpu" : " 17 GB", | 20260505 18:27:05.279 | INFO | "huge" : "", | 20260505 18:27:05.280 | INFO | "huge_shared" : " 27 GB", | 20260505 18:27:05.280 | INFO | "malloc" : " 14 GB", | 20260505 18:27:05.280 | INFO | "pinned" : " 29 GB" | 20260505 18:27:05.280 | INFO | } | 20260505 18:27:05.280 | INFO | Perf report for the last 5.13844 seconds: | 20260505 18:27:05.280 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:27:05.280 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:05.280 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:05.280 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:05.280 | INFO | ANALYSIS : 63.0%, 3.0, 97.3ms, 0 | 20260505 18:27:05.280 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:05.280 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:05.280 | INFO | PRE_HQ : 27.7%, 1.0, 14.3ms, 0 | 20260505 18:27:05.280 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:05.280 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:05.280 | INFO | Uploaded 36GiB to GPU in 2.20489s (16.3273GiB/s) | 20260505 18:27:05.280 | INFO | Downloaded 34GiB to GPU in 2.0432s (16.6405GiB/s) | 20260505 18:27:05.411 | INFO | Starting a round of Baz IO | 20260505 18:27:08.591 | INFO | Populating RTMetrics report | 20260505 18:27:08.592 | INFO | RTMetrics report populated | 20260505 18:27:08.592 | INFO | Opening tmp RTMetrics json file | 20260505 18:27:08.593 | INFO | Serializing RTMetrics to json | 20260505 18:27:08.594 | INFO | Serialization done, swapping tmp file now... | 20260505 18:27:08.594 | INFO | RTMetrics done | 20260505 18:27:08.685 | INFO | Finished a round of Baz IO: Wrote 2.092 GiB in 3.273 seconds | 20260505 18:27:08.685 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.91% / 0.00% / 78.09% | 20260505 18:27:08.685 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.249771% / 0% / 9.13938e-05% | 20260505 18:27:08.685 | INFO | Baz IO overall: | 20260505 18:27:08.685 | INFO | BytesWritten : 3151663821386 | 20260505 18:27:08.685 | INFO | NumEvents : 1493011248142 | 20260505 18:27:10.413 | INFO | Analyzing chunk frames = [5353472,5353984) | 20260505 18:27:10.413 | INFO | MemInfo: { | 20260505 18:27:10.413 | INFO | "gpu" : " 17 GB", | 20260505 18:27:10.413 | INFO | "huge" : "", | 20260505 18:27:10.413 | INFO | "huge_shared" : " 27 GB", | 20260505 18:27:10.413 | INFO | "malloc" : " 14 GB", | 20260505 18:27:10.413 | INFO | "pinned" : " 29 GB" | 20260505 18:27:10.413 | INFO | } | 20260505 18:27:10.413 | INFO | Perf report for the last 5.1335 seconds: | 20260505 18:27:10.413 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:27:10.413 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:10.413 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:10.413 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:10.413 | INFO | ANALYSIS : 63.4%, 3.0, 98.8ms, 0 | 20260505 18:27:10.413 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:10.413 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260505 18:27:10.413 | INFO | PRE_HQ : 60.8%, 1.8, 56.3ms, 0 | 20260505 18:27:10.413 | INFO | BAZWRITER : 63.3%, 3.7, 121.3ms, 0 | 20260505 18:27:10.413 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 18:27:10.413 | INFO | Uploaded 36GiB to GPU in 2.16058s (16.6622GiB/s) | 20260505 18:27:10.413 | INFO | Downloaded 38GiB to GPU in 2.25389s (16.8597GiB/s) | 20260505 18:27:15.552 | INFO | Analyzing chunk frames = [5353984,5354496) | 20260505 18:27:15.552 | INFO | MemInfo: { | 20260505 18:27:15.552 | INFO | "gpu" : " 17 GB", | 20260505 18:27:15.552 | INFO | "huge" : "", | 20260505 18:27:15.552 | INFO | "huge_shared" : " 27 GB", | 20260505 18:27:15.552 | INFO | "malloc" : " 14 GB", | 20260505 18:27:15.552 | INFO | "pinned" : " 29 GB" | 20260505 18:27:15.552 | INFO | } | 20260505 18:27:15.552 | INFO | Perf report for the last 5.13917 seconds: | 20260505 18:27:15.552 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:27:15.552 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:15.552 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:15.552 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:15.552 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 18:27:15.552 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:15.552 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:15.552 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:27:15.552 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:15.552 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:15.553 | INFO | Uploaded 36GiB to GPU in 2.14476s (16.7851GiB/s) | 20260505 18:27:15.553 | INFO | Downloaded 34GiB to GPU in 2.02823s (16.7634GiB/s) | 20260505 18:27:20.689 | INFO | Analyzing chunk frames = [5354496,5355008) | 20260505 18:27:20.689 | INFO | MemInfo: { | 20260505 18:27:20.689 | INFO | "gpu" : " 17 GB", | 20260505 18:27:20.689 | INFO | "huge" : "", | 20260505 18:27:20.689 | INFO | "huge_shared" : " 27 GB", | 20260505 18:27:20.689 | INFO | "malloc" : " 14 GB", | 20260505 18:27:20.689 | INFO | "pinned" : " 29 GB" | 20260505 18:27:20.689 | INFO | } | 20260505 18:27:20.689 | INFO | Perf report for the last 5.13683 seconds: | 20260505 18:27:20.689 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:27:20.689 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:20.689 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:20.689 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:20.689 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 18:27:20.689 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:20.689 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:20.689 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 18:27:20.689 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:20.690 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:20.690 | INFO | Uploaded 36GiB to GPU in 2.2024s (16.3458GiB/s) | 20260505 18:27:20.690 | INFO | Downloaded 34GiB to GPU in 2.03861s (16.678GiB/s) | 20260505 18:27:25.830 | INFO | Analyzing chunk frames = [5355008,5355520) | 20260505 18:27:25.830 | INFO | MemInfo: { | 20260505 18:27:25.830 | INFO | "gpu" : " 17 GB", | 20260505 18:27:25.830 | INFO | "huge" : "", | 20260505 18:27:25.830 | INFO | "huge_shared" : " 27 GB", | 20260505 18:27:25.830 | INFO | "malloc" : " 14 GB", | 20260505 18:27:25.830 | INFO | "pinned" : " 29 GB" | 20260505 18:27:25.830 | INFO | } | 20260505 18:27:25.830 | INFO | Perf report for the last 5.14037 seconds: | 20260505 18:27:25.830 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:27:25.830 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:25.830 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:25.830 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:25.830 | INFO | ANALYSIS : 62.7%, 3.0, 97.5ms, 0 | 20260505 18:27:25.830 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:25.830 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:25.830 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 18:27:25.830 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:25.830 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:25.830 | INFO | Uploaded 36GiB to GPU in 2.22742s (16.1622GiB/s) | 20260505 18:27:25.830 | INFO | Downloaded 34GiB to GPU in 2.126s (15.9925GiB/s) | 20260505 18:27:30.977 | INFO | Analyzing chunk frames = [5355520,5356032) | 20260505 18:27:30.977 | INFO | MemInfo: { | 20260505 18:27:30.978 | INFO | "gpu" : " 17 GB", | 20260505 18:27:30.978 | INFO | "huge" : "", | 20260505 18:27:30.978 | INFO | "huge_shared" : " 27 GB", | 20260505 18:27:30.978 | INFO | "malloc" : " 14 GB", | 20260505 18:27:30.978 | INFO | "pinned" : " 29 GB" | 20260505 18:27:30.978 | INFO | } | 20260505 18:27:30.978 | INFO | Perf report for the last 5.14812 seconds: | 20260505 18:27:30.978 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:27:30.978 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:30.978 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:30.978 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:30.978 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260505 18:27:30.978 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:30.978 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:30.978 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:27:30.978 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:30.978 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:30.978 | INFO | Uploaded 36GiB to GPU in 2.13527s (16.8597GiB/s) | 20260505 18:27:30.978 | INFO | Downloaded 34GiB to GPU in 2.05955s (16.5084GiB/s) | 20260505 18:27:36.110 | INFO | Analyzing chunk frames = [5356032,5356544) | 20260505 18:27:36.110 | INFO | MemInfo: { | 20260505 18:27:36.110 | INFO | "gpu" : " 17 GB", | 20260505 18:27:36.110 | INFO | "huge" : "", | 20260505 18:27:36.110 | INFO | "huge_shared" : " 27 GB", | 20260505 18:27:36.110 | INFO | "malloc" : " 14 GB", | 20260505 18:27:36.110 | INFO | "pinned" : " 29 GB" | 20260505 18:27:36.110 | INFO | } | 20260505 18:27:36.110 | INFO | Perf report for the last 5.13203 seconds: | 20260505 18:27:36.110 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:27:36.110 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:36.110 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:36.110 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:36.110 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260505 18:27:36.110 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:36.110 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:36.110 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:27:36.110 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:36.110 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:36.110 | INFO | Uploaded 36GiB to GPU in 2.15874s (16.6764GiB/s) | 20260505 18:27:36.110 | INFO | Downloaded 34GiB to GPU in 2.06154s (16.4925GiB/s) | 20260505 18:27:41.242 | INFO | Analyzing chunk frames = [5356544,5357056) | 20260505 18:27:41.242 | INFO | MemInfo: { | 20260505 18:27:41.242 | INFO | "gpu" : " 17 GB", | 20260505 18:27:41.242 | INFO | "huge" : "", | 20260505 18:27:41.242 | INFO | "huge_shared" : " 27 GB", | 20260505 18:27:41.242 | INFO | "malloc" : " 14 GB", | 20260505 18:27:41.242 | INFO | "pinned" : " 29 GB" | 20260505 18:27:41.242 | INFO | } | 20260505 18:27:41.242 | INFO | Perf report for the last 5.13203 seconds: | 20260505 18:27:41.242 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:27:41.242 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:41.242 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:41.242 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:41.242 | INFO | ANALYSIS : 63.5%, 3.0, 99.0ms, 0 | 20260505 18:27:41.242 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:41.242 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:41.242 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 18:27:41.242 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:41.242 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:41.242 | INFO | Uploaded 36GiB to GPU in 2.14707s (16.767GiB/s) | 20260505 18:27:41.242 | INFO | Downloaded 34GiB to GPU in 2.14235s (15.8704GiB/s) | 20260505 18:27:46.393 | INFO | Analyzing chunk frames = [5357056,5357568) | 20260505 18:27:46.393 | INFO | MemInfo: { | 20260505 18:27:46.393 | INFO | "gpu" : " 17 GB", | 20260505 18:27:46.393 | INFO | "huge" : "", | 20260505 18:27:46.393 | INFO | "huge_shared" : " 27 GB", | 20260505 18:27:46.393 | INFO | "malloc" : " 14 GB", | 20260505 18:27:46.393 | INFO | "pinned" : " 29 GB" | 20260505 18:27:46.393 | INFO | } | 20260505 18:27:46.393 | INFO | Perf report for the last 5.15122 seconds: | 20260505 18:27:46.393 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:27:46.393 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:46.393 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:46.393 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:46.393 | INFO | ANALYSIS : 62.7%, 3.0, 98.2ms, 0 | 20260505 18:27:46.393 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:46.393 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:46.393 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260505 18:27:46.393 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:46.393 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:46.393 | INFO | Uploaded 36GiB to GPU in 2.16491s (16.6289GiB/s) | 20260505 18:27:46.393 | INFO | Downloaded 34GiB to GPU in 2.02402s (16.7983GiB/s) | 20260505 18:27:46.394 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 18:27:46.394 | INFO | Total mem usage: 15.3 GiB | 20260505 18:27:46.394 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 18:27:46.394 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.01%) | 20260505 18:27:46.394 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 18:27:46.394 | INFO | High Water Breakdown: | 20260505 18:27:46.394 | INFO | 61% : 8.66 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 18:27:46.394 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 18:27:46.394 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 18:27:46.394 | INFO | ------------------------------------------ | 20260505 18:27:46.394 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:27:46.394 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 18:27:46.394 | INFO | Total mem usage: 26.5 GiB | 20260505 18:27:46.394 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 18:27:46.394 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 18:27:46.394 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 18:27:46.394 | INFO | High Water Breakdown: | 20260505 18:27:46.394 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:27:46.394 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:27:46.394 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:27:46.394 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:27:46.394 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:27:46.394 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:27:46.394 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 18:27:46.394 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 18:27:46.394 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:27:46.394 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 18:27:46.394 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 18:27:46.394 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 18:27:46.394 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 18:27:46.394 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 18:27:46.394 | INFO | ------------------------------------------ | 20260505 18:27:46.394 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:27:46.394 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 18:27:46.394 | INFO | Total mem usage: 25.6 GiB | 20260505 18:27:46.394 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 18:27:46.394 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 18:27:46.394 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 18:27:46.394 | INFO | High Water Breakdown: | 20260505 18:27:46.394 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 18:27:46.394 | INFO | ------------------------------------------ | 20260505 18:27:46.394 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:27:46.394 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 18:27:46.394 | INFO | Total mem usage: 15.7 GiB | 20260505 18:27:46.394 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 18:27:46.394 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 18:27:46.394 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 18:27:46.394 | INFO | High Water Breakdown: | 20260505 18:27:46.394 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 18:27:46.394 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 18:27:46.394 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 18:27:46.394 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 18:27:46.394 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 18:27:46.394 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 18:27:46.394 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 18:27:46.394 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 18:27:46.394 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 18:27:46.394 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:27:46.394 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 18:27:46.394 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:27:46.394 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 18:27:46.394 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 18:27:46.394 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 18:27:46.394 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 18:27:46.394 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:27:46.394 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 18:27:46.394 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:27:46.394 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:27:46.394 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:27:46.394 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 18:27:46.394 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 18:27:46.394 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:27:46.394 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 18:27:46.394 | INFO | ------------------------------------------ | 20260505 18:27:46.394 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:27:46.395 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 18:27:46.395 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:27:46.395 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:27:46.395 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.47e+01, 1.36e+01 / 2.03e+00 | 20260505 18:27:46.395 | INFO | PerfReport: 0.29%, AggregateStats -- 9.24e-02 / 1.02e-01, 1.07e-02 / 3.74e-03 | 20260505 18:27:46.395 | INFO | PerfReport: 11.78%, Baseline -- 3.71e+00 / 4.54e+00, 5.51e-01 / 1.35e-01 | 20260505 18:27:46.395 | INFO | PerfReport: 3.26%, Histogram -- 1.03e+00 / 1.54e+00, 1.88e-01 / 8.43e-02 | 20260505 18:27:46.395 | INFO | PerfReport: 15.96%, DME -- 4.02e+01 / 4.45e+01, 4.07e+00 / 1.89e+00 | 20260505 18:27:46.395 | INFO | PerfReport: 56.84%, FrameLabeling -- 1.79e+01 / 1.92e+01, 1.39e+00 / 1.10e+00 | 20260505 18:27:46.395 | INFO | PerfReport: 7.36%, PulseAccumulating -- 2.32e+00 / 2.92e+00, 4.45e-01 / 1.52e-01 | 20260505 18:27:46.395 | INFO | PerfReport: 3.35%, Metrics -- 1.05e+00 / 1.40e+00, 2.91e-01 / 1.61e-01 | 20260505 18:27:46.395 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.66e-01 / 4.29e-01, 6.44e-02 / 3.88e-02 | 20260505 18:27:46.395 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 18:27:46.395 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:27:46.395 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:27:46.395 | INFO | PerfReport: Full Execution -- 9.98e+01 / -nan, 2.55e+01 / 0.00e+00 | 20260505 18:27:46.395 | INFO | PerfReport: 22.61%, Upload -- 2.26e+01 / 0.00e+00, 7.85e+00 / 0.00e+00 | 20260505 18:27:46.395 | INFO | PerfReport: 21.78%, Download -- 2.17e+01 / 0.00e+00, 7.67e+00 / 0.00e+00 | 20260505 18:27:46.395 | INFO | PerfReport: 31.54%, Compute -- 3.15e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 18:27:46.395 | INFO | PerfReport: 2.44%, LockUpload -- 2.44e+00 / 0.00e+00, 8.07e+00 / 0.00e+00 | 20260505 18:27:46.395 | INFO | PerfReport: 1.35%, LockDownload -- 1.34e+00 / 0.00e+00, 2.82e+00 / 0.00e+00 | 20260505 18:27:46.395 | INFO | PerfReport: 20.27%, LockCompute -- 2.02e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 18:27:46.395 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.47e-02 / 0.00e+00 | 20260505 18:27:46.527 | INFO | Starting a round of Baz IO | 20260505 18:27:49.708 | INFO | Populating RTMetrics report | 20260505 18:27:49.709 | INFO | RTMetrics report populated | 20260505 18:27:49.709 | INFO | Opening tmp RTMetrics json file | 20260505 18:27:49.715 | INFO | Serializing RTMetrics to json | 20260505 18:27:49.716 | INFO | Serialization done, swapping tmp file now... | 20260505 18:27:49.716 | INFO | RTMetrics done | 20260505 18:27:49.801 | INFO | Finished a round of Baz IO: Wrote 2.092 GiB in 3.273 seconds | 20260505 18:27:49.801 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.90% / 0.00% / 78.09% | 20260505 18:27:49.801 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.249592% / 0% / 7.54992e-05% | 20260505 18:27:49.801 | INFO | Baz IO overall: | 20260505 18:27:49.801 | INFO | BytesWritten : 3153910215132 | 20260505 18:27:49.801 | INFO | NumEvents : 1494058562096 | 20260505 18:27:51.532 | INFO | Analyzing chunk frames = [5357568,5358080) | 20260505 18:27:51.532 | INFO | MemInfo: { | 20260505 18:27:51.532 | INFO | "gpu" : " 17 GB", | 20260505 18:27:51.532 | INFO | "huge" : "", | 20260505 18:27:51.532 | INFO | "huge_shared" : " 27 GB", | 20260505 18:27:51.532 | INFO | "malloc" : " 14 GB", | 20260505 18:27:51.532 | INFO | "pinned" : " 29 GB" | 20260505 18:27:51.532 | INFO | } | 20260505 18:27:51.532 | INFO | Perf report for the last 5.13934 seconds: | 20260505 18:27:51.533 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:27:51.533 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:51.533 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:51.533 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:51.533 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 18:27:51.533 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:51.533 | INFO | RT_METRICS : 44.3%, 1.0, 23.3ms, 0 | 20260505 18:27:51.533 | INFO | PRE_HQ : 60.4%, 1.8, 55.5ms, 0 | 20260505 18:27:51.533 | INFO | BAZWRITER : 63.3%, 3.8, 123.6ms, 0 | 20260505 18:27:51.533 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 18:27:51.533 | INFO | Uploaded 36GiB to GPU in 2.16825s (16.6032GiB/s) | 20260505 18:27:51.533 | INFO | Downloaded 38GiB to GPU in 2.24411s (16.9332GiB/s) | 20260505 18:27:56.665 | INFO | Analyzing chunk frames = [5358080,5358592) | 20260505 18:27:56.665 | INFO | MemInfo: { | 20260505 18:27:56.665 | INFO | "gpu" : " 17 GB", | 20260505 18:27:56.665 | INFO | "huge" : "", | 20260505 18:27:56.665 | INFO | "huge_shared" : " 27 GB", | 20260505 18:27:56.665 | INFO | "malloc" : " 14 GB", | 20260505 18:27:56.665 | INFO | "pinned" : " 29 GB" | 20260505 18:27:56.665 | INFO | } | 20260505 18:27:56.665 | INFO | Perf report for the last 5.13257 seconds: | 20260505 18:27:56.665 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:27:56.665 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:56.665 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:56.665 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:56.665 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260505 18:27:56.665 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:56.665 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:56.665 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:27:56.665 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:56.665 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:27:56.665 | INFO | Uploaded 36GiB to GPU in 2.14201s (16.8066GiB/s) | 20260505 18:27:56.665 | INFO | Downloaded 34GiB to GPU in 2.03249s (16.7283GiB/s) | 20260505 18:28:01.804 | INFO | Analyzing chunk frames = [5358592,5359104) | 20260505 18:28:01.804 | INFO | MemInfo: { | 20260505 18:28:01.804 | INFO | "gpu" : " 17 GB", | 20260505 18:28:01.804 | INFO | "huge" : "", | 20260505 18:28:01.804 | INFO | "huge_shared" : " 27 GB", | 20260505 18:28:01.804 | INFO | "malloc" : " 14 GB", | 20260505 18:28:01.804 | INFO | "pinned" : " 29 GB" | 20260505 18:28:01.804 | INFO | } | 20260505 18:28:01.804 | INFO | Perf report for the last 5.13906 seconds: | 20260505 18:28:01.804 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:28:01.804 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:01.804 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:01.804 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:01.804 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 18:28:01.804 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:01.804 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:01.804 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:28:01.804 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:01.804 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:01.804 | INFO | Uploaded 36GiB to GPU in 2.20726s (16.3098GiB/s) | 20260505 18:28:01.804 | INFO | Downloaded 34GiB to GPU in 2.06309s (16.4802GiB/s) | 20260505 18:28:06.943 | INFO | Analyzing chunk frames = [5359104,5359616) | 20260505 18:28:06.943 | INFO | MemInfo: { | 20260505 18:28:06.943 | INFO | "gpu" : " 17 GB", | 20260505 18:28:06.943 | INFO | "huge" : "", | 20260505 18:28:06.943 | INFO | "huge_shared" : " 27 GB", | 20260505 18:28:06.943 | INFO | "malloc" : " 14 GB", | 20260505 18:28:06.943 | INFO | "pinned" : " 29 GB" | 20260505 18:28:06.943 | INFO | } | 20260505 18:28:06.943 | INFO | Perf report for the last 5.13892 seconds: | 20260505 18:28:06.943 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:28:06.943 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:06.943 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:06.943 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:06.943 | INFO | ANALYSIS : 62.6%, 3.0, 97.2ms, 0 | 20260505 18:28:06.943 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:06.943 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:06.943 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:28:06.943 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:06.943 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:06.943 | INFO | Uploaded 36GiB to GPU in 2.21307s (16.267GiB/s) | 20260505 18:28:06.943 | INFO | Downloaded 34GiB to GPU in 2.13725s (15.9083GiB/s) | 20260505 18:28:12.089 | INFO | Analyzing chunk frames = [5359616,5360128) | 20260505 18:28:12.089 | INFO | MemInfo: { | 20260505 18:28:12.089 | INFO | "gpu" : " 17 GB", | 20260505 18:28:12.089 | INFO | "huge" : "", | 20260505 18:28:12.089 | INFO | "huge_shared" : " 27 GB", | 20260505 18:28:12.089 | INFO | "malloc" : " 14 GB", | 20260505 18:28:12.089 | INFO | "pinned" : " 29 GB" | 20260505 18:28:12.089 | INFO | } | 20260505 18:28:12.090 | INFO | Perf report for the last 5.14648 seconds: | 20260505 18:28:12.090 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:28:12.090 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:12.090 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:12.090 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:12.090 | INFO | ANALYSIS : 61.9%, 3.0, 96.8ms, 0 | 20260505 18:28:12.090 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:12.090 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:12.090 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260505 18:28:12.090 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:12.090 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:12.090 | INFO | Uploaded 36GiB to GPU in 2.14187s (16.8077GiB/s) | 20260505 18:28:12.090 | INFO | Downloaded 34GiB to GPU in 2.06s (16.5049GiB/s) | 20260505 18:28:17.224 | INFO | Analyzing chunk frames = [5360128,5360640) | 20260505 18:28:17.224 | INFO | MemInfo: { | 20260505 18:28:17.224 | INFO | "gpu" : " 17 GB", | 20260505 18:28:17.224 | INFO | "huge" : "", | 20260505 18:28:17.224 | INFO | "huge_shared" : " 27 GB", | 20260505 18:28:17.224 | INFO | "malloc" : " 14 GB", | 20260505 18:28:17.224 | INFO | "pinned" : " 29 GB" | 20260505 18:28:17.224 | INFO | } | 20260505 18:28:17.224 | INFO | Perf report for the last 5.13498 seconds: | 20260505 18:28:17.224 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:28:17.224 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:17.225 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:17.225 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:17.225 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 18:28:17.225 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:17.225 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:17.225 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:28:17.225 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:17.225 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:17.225 | INFO | Uploaded 36GiB to GPU in 2.13282s (16.8791GiB/s) | 20260505 18:28:17.225 | INFO | Downloaded 34GiB to GPU in 2.05604s (16.5366GiB/s) | 20260505 18:28:22.358 | INFO | Analyzing chunk frames = [5360640,5361152) | 20260505 18:28:22.358 | INFO | MemInfo: { | 20260505 18:28:22.358 | INFO | "gpu" : " 17 GB", | 20260505 18:28:22.358 | INFO | "huge" : "", | 20260505 18:28:22.358 | INFO | "huge_shared" : " 27 GB", | 20260505 18:28:22.358 | INFO | "malloc" : " 14 GB", | 20260505 18:28:22.358 | INFO | "pinned" : " 29 GB" | 20260505 18:28:22.358 | INFO | } | 20260505 18:28:22.358 | INFO | Perf report for the last 5.1336 seconds: | 20260505 18:28:22.358 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:28:22.358 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:22.358 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:22.358 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:22.358 | INFO | ANALYSIS : 63.5%, 3.0, 99.0ms, 0 | 20260505 18:28:22.358 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:22.358 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:22.358 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 18:28:22.358 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:22.358 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:22.358 | INFO | Uploaded 36GiB to GPU in 2.14624s (16.7735GiB/s) | 20260505 18:28:22.359 | INFO | Downloaded 34GiB to GPU in 2.1419s (15.8738GiB/s) | 20260505 18:28:27.499 | INFO | Analyzing chunk frames = [5361152,5361664) | 20260505 18:28:27.499 | INFO | MemInfo: { | 20260505 18:28:27.499 | INFO | "gpu" : " 17 GB", | 20260505 18:28:27.499 | INFO | "huge" : "", | 20260505 18:28:27.499 | INFO | "huge_shared" : " 27 GB", | 20260505 18:28:27.499 | INFO | "malloc" : " 14 GB", | 20260505 18:28:27.499 | INFO | "pinned" : " 29 GB" | 20260505 18:28:27.499 | INFO | } | 20260505 18:28:27.499 | INFO | Perf report for the last 5.14083 seconds: | 20260505 18:28:27.499 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:28:27.499 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:27.499 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:27.499 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:27.499 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 18:28:27.499 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:27.499 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:27.499 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260505 18:28:27.499 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:27.499 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:27.499 | INFO | Uploaded 36GiB to GPU in 2.17737s (16.5337GiB/s) | 20260505 18:28:27.499 | INFO | Downloaded 34GiB to GPU in 2.03979s (16.6684GiB/s) | 20260505 18:28:27.640 | INFO | Starting a round of Baz IO | 20260505 18:28:30.823 | INFO | Populating RTMetrics report | 20260505 18:28:30.823 | INFO | RTMetrics report populated | 20260505 18:28:30.823 | INFO | Opening tmp RTMetrics json file | 20260505 18:28:30.824 | INFO | Serializing RTMetrics to json | 20260505 18:28:30.826 | INFO | Serialization done, swapping tmp file now... | 20260505 18:28:30.826 | INFO | RTMetrics done | 20260505 18:28:30.912 | INFO | Finished a round of Baz IO: Wrote 2.092 GiB in 3.272 seconds | 20260505 18:28:30.912 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.90% / 0.00% / 78.10% | 20260505 18:28:30.912 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.248587% / 3.97364e-06% / 8.74201e-05% | 20260505 18:28:30.912 | INFO | Baz IO overall: | 20260505 18:28:30.912 | INFO | BytesWritten : 3156156998741 | 20260505 18:28:30.912 | INFO | NumEvents : 1495106182013 | 20260505 18:28:32.643 | INFO | Analyzing chunk frames = [5361664,5362176) | 20260505 18:28:32.643 | INFO | MemInfo: { | 20260505 18:28:32.643 | INFO | "gpu" : " 17 GB", | 20260505 18:28:32.643 | INFO | "huge" : "", | 20260505 18:28:32.643 | INFO | "huge_shared" : " 27 GB", | 20260505 18:28:32.643 | INFO | "malloc" : " 14 GB", | 20260505 18:28:32.643 | INFO | "pinned" : " 29 GB" | 20260505 18:28:32.643 | INFO | } | 20260505 18:28:32.643 | INFO | Perf report for the last 5.14449 seconds: | 20260505 18:28:32.643 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:28:32.643 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:32.643 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:32.643 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:32.644 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 18:28:32.644 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:32.644 | INFO | RT_METRICS : 44.3%, 1.0, 23.3ms, 0 | 20260505 18:28:32.644 | INFO | PRE_HQ : 60.1%, 1.8, 55.7ms, 0 | 20260505 18:28:32.644 | INFO | BAZWRITER : 63.0%, 3.8, 121.8ms, 0 | 20260505 18:28:32.644 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 18:28:32.644 | INFO | Uploaded 36GiB to GPU in 2.15886s (16.6755GiB/s) | 20260505 18:28:32.644 | INFO | Downloaded 38GiB to GPU in 2.25476s (16.8532GiB/s) | 20260505 18:28:37.790 | INFO | Analyzing chunk frames = [5362176,5362688) | 20260505 18:28:37.790 | INFO | MemInfo: { | 20260505 18:28:37.790 | INFO | "gpu" : " 17 GB", | 20260505 18:28:37.790 | INFO | "huge" : "", | 20260505 18:28:37.790 | INFO | "huge_shared" : " 27 GB", | 20260505 18:28:37.790 | INFO | "malloc" : " 14 GB", | 20260505 18:28:37.790 | INFO | "pinned" : " 29 GB" | 20260505 18:28:37.790 | INFO | } | 20260505 18:28:37.790 | INFO | Perf report for the last 5.14624 seconds: | 20260505 18:28:37.790 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:28:37.790 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:37.790 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:37.790 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:37.790 | INFO | ANALYSIS : 62.6%, 3.0, 97.9ms, 0 | 20260505 18:28:37.790 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:37.790 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:37.790 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 18:28:37.790 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:37.790 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:37.790 | INFO | Uploaded 36GiB to GPU in 2.14168s (16.8092GiB/s) | 20260505 18:28:37.790 | INFO | Downloaded 34GiB to GPU in 2.02834s (16.7625GiB/s) | 20260505 18:28:42.924 | INFO | Analyzing chunk frames = [5362688,5363200) | 20260505 18:28:42.924 | INFO | MemInfo: { | 20260505 18:28:42.924 | INFO | "gpu" : " 17 GB", | 20260505 18:28:42.924 | INFO | "huge" : "", | 20260505 18:28:42.924 | INFO | "huge_shared" : " 27 GB", | 20260505 18:28:42.924 | INFO | "malloc" : " 14 GB", | 20260505 18:28:42.924 | INFO | "pinned" : " 29 GB" | 20260505 18:28:42.924 | INFO | } | 20260505 18:28:42.924 | INFO | Perf report for the last 5.13434 seconds: | 20260505 18:28:42.924 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:28:42.924 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:42.924 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:42.924 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:42.924 | INFO | ANALYSIS : 62.1%, 3.0, 96.5ms, 0 | 20260505 18:28:42.924 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:42.924 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:42.924 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:28:42.924 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:42.924 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:42.924 | INFO | Uploaded 36GiB to GPU in 2.12113s (16.9721GiB/s) | 20260505 18:28:42.924 | INFO | Downloaded 34GiB to GPU in 2.01083s (16.9084GiB/s) | 20260505 18:28:48.059 | INFO | Analyzing chunk frames = [5363200,5363712) | 20260505 18:28:48.059 | INFO | MemInfo: { | 20260505 18:28:48.059 | INFO | "gpu" : " 17 GB", | 20260505 18:28:48.059 | INFO | "huge" : "", | 20260505 18:28:48.059 | INFO | "huge_shared" : " 27 GB", | 20260505 18:28:48.059 | INFO | "malloc" : " 14 GB", | 20260505 18:28:48.059 | INFO | "pinned" : " 29 GB" | 20260505 18:28:48.059 | INFO | } | 20260505 18:28:48.060 | INFO | Perf report for the last 5.13543 seconds: | 20260505 18:28:48.060 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:28:48.060 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:48.060 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:48.060 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:48.060 | INFO | ANALYSIS : 62.5%, 3.0, 97.0ms, 0 | 20260505 18:28:48.060 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:48.060 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:48.060 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:28:48.060 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:48.060 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:48.060 | INFO | Uploaded 36GiB to GPU in 2.16672s (16.615GiB/s) | 20260505 18:28:48.060 | INFO | Downloaded 34GiB to GPU in 2.06168s (16.4914GiB/s) | 20260505 18:28:53.201 | INFO | Analyzing chunk frames = [5363712,5364224) | 20260505 18:28:53.201 | INFO | MemInfo: { | 20260505 18:28:53.201 | INFO | "gpu" : " 17 GB", | 20260505 18:28:53.201 | INFO | "huge" : "", | 20260505 18:28:53.201 | INFO | "huge_shared" : " 27 GB", | 20260505 18:28:53.201 | INFO | "malloc" : " 14 GB", | 20260505 18:28:53.201 | INFO | "pinned" : " 29 GB" | 20260505 18:28:53.201 | INFO | } | 20260505 18:28:53.201 | INFO | Perf report for the last 5.14192 seconds: | 20260505 18:28:53.201 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:28:53.201 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:53.201 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:53.201 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:53.201 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260505 18:28:53.201 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:53.201 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:53.202 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:28:53.202 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:53.202 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:53.202 | INFO | Uploaded 36GiB to GPU in 2.148s (16.7598GiB/s) | 20260505 18:28:53.202 | INFO | Downloaded 34GiB to GPU in 2.0641s (16.4721GiB/s) | 20260505 18:28:58.342 | INFO | Analyzing chunk frames = [5364224,5364736) | 20260505 18:28:58.342 | INFO | MemInfo: { | 20260505 18:28:58.342 | INFO | "gpu" : " 17 GB", | 20260505 18:28:58.342 | INFO | "huge" : "", | 20260505 18:28:58.342 | INFO | "huge_shared" : " 27 GB", | 20260505 18:28:58.342 | INFO | "malloc" : " 14 GB", | 20260505 18:28:58.342 | INFO | "pinned" : " 29 GB" | 20260505 18:28:58.342 | INFO | } | 20260505 18:28:58.342 | INFO | Perf report for the last 5.14042 seconds: | 20260505 18:28:58.342 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:28:58.342 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:58.342 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:58.342 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:58.342 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 18:28:58.342 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:58.342 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:58.342 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 18:28:58.342 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:58.342 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:28:58.342 | INFO | Uploaded 36GiB to GPU in 2.14942s (16.7487GiB/s) | 20260505 18:28:58.342 | INFO | Downloaded 34GiB to GPU in 2.04843s (16.5981GiB/s) | 20260505 18:29:03.482 | INFO | Analyzing chunk frames = [5364736,5365248) | 20260505 18:29:03.482 | INFO | MemInfo: { | 20260505 18:29:03.482 | INFO | "gpu" : " 17 GB", | 20260505 18:29:03.482 | INFO | "huge" : "", | 20260505 18:29:03.482 | INFO | "huge_shared" : " 27 GB", | 20260505 18:29:03.482 | INFO | "malloc" : " 14 GB", | 20260505 18:29:03.482 | INFO | "pinned" : " 29 GB" | 20260505 18:29:03.482 | INFO | } | 20260505 18:29:03.482 | INFO | Perf report for the last 5.1402 seconds: | 20260505 18:29:03.482 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:29:03.482 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:03.482 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:03.482 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:03.482 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 18:29:03.482 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:03.482 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:03.482 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 18:29:03.482 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:03.482 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:03.482 | INFO | Uploaded 36GiB to GPU in 2.13746s (16.8425GiB/s) | 20260505 18:29:03.482 | INFO | Downloaded 34GiB to GPU in 2.14972s (15.816GiB/s) | 20260505 18:29:08.615 | INFO | Analyzing chunk frames = [5365248,5365760) | 20260505 18:29:08.615 | INFO | MemInfo: { | 20260505 18:29:08.615 | INFO | "gpu" : " 17 GB", | 20260505 18:29:08.615 | INFO | "huge" : "", | 20260505 18:29:08.615 | INFO | "huge_shared" : " 27 GB", | 20260505 18:29:08.615 | INFO | "malloc" : " 14 GB", | 20260505 18:29:08.615 | INFO | "pinned" : " 29 GB" | 20260505 18:29:08.615 | INFO | } | 20260505 18:29:08.615 | INFO | Perf report for the last 5.13262 seconds: | 20260505 18:29:08.615 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:29:08.615 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:08.615 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:08.615 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:08.615 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260505 18:29:08.615 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:08.615 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:08.615 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 18:29:08.615 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:08.615 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:08.615 | INFO | Uploaded 36GiB to GPU in 2.19425s (16.4065GiB/s) | 20260505 18:29:08.615 | INFO | Downloaded 34GiB to GPU in 2.026s (16.7818GiB/s) | 20260505 18:29:08.615 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 18:29:08.615 | INFO | Total mem usage: 15.3 GiB | 20260505 18:29:08.615 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 18:29:08.615 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.31%) | 20260505 18:29:08.615 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 18:29:08.615 | INFO | High Water Breakdown: | 20260505 18:29:08.615 | INFO | 60.9% : 8.62 GiB ( 90.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 18:29:08.615 | INFO | 38.9% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 18:29:08.615 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 18:29:08.615 | INFO | ------------------------------------------ | 20260505 18:29:08.615 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:29:08.615 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 18:29:08.615 | INFO | Total mem usage: 26.5 GiB | 20260505 18:29:08.615 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 18:29:08.615 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 18:29:08.615 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 18:29:08.615 | INFO | High Water Breakdown: | 20260505 18:29:08.615 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:29:08.615 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:29:08.615 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:29:08.615 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:29:08.615 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:29:08.615 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:29:08.615 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 18:29:08.615 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 18:29:08.615 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:29:08.615 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 18:29:08.615 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 18:29:08.615 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 18:29:08.615 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 18:29:08.615 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 18:29:08.615 | INFO | ------------------------------------------ | 20260505 18:29:08.615 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:29:08.615 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 18:29:08.615 | INFO | Total mem usage: 25.6 GiB | 20260505 18:29:08.615 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 18:29:08.615 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 18:29:08.615 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 18:29:08.616 | INFO | High Water Breakdown: | 20260505 18:29:08.616 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 18:29:08.616 | INFO | ------------------------------------------ | 20260505 18:29:08.616 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:29:08.616 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 18:29:08.616 | INFO | Total mem usage: 15.7 GiB | 20260505 18:29:08.616 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 18:29:08.616 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 18:29:08.616 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 18:29:08.616 | INFO | High Water Breakdown: | 20260505 18:29:08.616 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 18:29:08.616 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 18:29:08.616 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 18:29:08.616 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 18:29:08.616 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 18:29:08.616 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 18:29:08.616 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 18:29:08.616 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 18:29:08.616 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 18:29:08.616 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:29:08.616 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 18:29:08.616 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:29:08.616 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 18:29:08.616 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 18:29:08.616 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 18:29:08.616 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 18:29:08.616 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:29:08.616 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 18:29:08.616 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:29:08.616 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:29:08.616 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:29:08.616 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 18:29:08.616 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 18:29:08.616 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:29:08.616 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 18:29:08.616 | INFO | ------------------------------------------ | 20260505 18:29:08.616 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:29:08.616 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 18:29:08.616 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:29:08.616 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:29:08.616 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.47e+01, 1.33e+01 / 2.12e+00 | 20260505 18:29:08.616 | INFO | PerfReport: 0.29%, AggregateStats -- 9.25e-02 / 1.02e-01, 1.05e-02 / 4.56e-03 | 20260505 18:29:08.616 | INFO | PerfReport: 11.79%, Baseline -- 3.72e+00 / 4.52e+00, 5.66e-01 / 1.68e-01 | 20260505 18:29:08.616 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.52e+00, 1.88e-01 / 9.11e-02 | 20260505 18:29:08.616 | INFO | PerfReport: 15.97%, DME -- 4.03e+01 / 4.47e+01, 4.01e+00 / 2.17e+00 | 20260505 18:29:08.616 | INFO | PerfReport: 56.89%, FrameLabeling -- 1.79e+01 / 1.92e+01, 1.46e+00 / 1.02e+00 | 20260505 18:29:08.616 | INFO | PerfReport: 7.31%, PulseAccumulating -- 2.31e+00 / 2.91e+00, 4.41e-01 / 1.70e-01 | 20260505 18:29:08.616 | INFO | PerfReport: 3.32%, Metrics -- 1.05e+00 / 1.39e+00, 2.85e-01 / 1.48e-01 | 20260505 18:29:08.616 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.65e-01 / 4.29e-01, 6.48e-02 / 3.68e-02 | 20260505 18:29:08.616 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 18:29:08.616 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:29:08.616 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:29:08.616 | INFO | PerfReport: Full Execution -- 9.98e+01 / -nan, 2.55e+01 / 0.00e+00 | 20260505 18:29:08.616 | INFO | PerfReport: 22.53%, Upload -- 2.25e+01 / 0.00e+00, 7.83e+00 / 0.00e+00 | 20260505 18:29:08.616 | INFO | PerfReport: 21.80%, Download -- 2.18e+01 / 0.00e+00, 7.77e+00 / 0.00e+00 | 20260505 18:29:08.616 | INFO | PerfReport: 31.61%, Compute -- 3.16e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260505 18:29:08.616 | INFO | PerfReport: 2.33%, LockUpload -- 2.33e+00 / 0.00e+00, 7.95e+00 / 0.00e+00 | 20260505 18:29:08.616 | INFO | PerfReport: 1.39%, LockDownload -- 1.39e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260505 18:29:08.616 | INFO | PerfReport: 20.32%, LockCompute -- 2.03e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260505 18:29:08.616 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.65e-02 / 0.00e+00 | 20260505 18:29:08.752 | INFO | Starting a round of Baz IO | 20260505 18:29:11.927 | INFO | Populating RTMetrics report | 20260505 18:29:11.927 | INFO | RTMetrics report populated | 20260505 18:29:11.927 | INFO | Opening tmp RTMetrics json file | 20260505 18:29:11.932 | INFO | Serializing RTMetrics to json | 20260505 18:29:11.934 | INFO | Serialization done, swapping tmp file now... | 20260505 18:29:11.934 | INFO | RTMetrics done | 20260505 18:29:12.023 | INFO | Finished a round of Baz IO: Wrote 2.089 GiB in 3.27 seconds | 20260505 18:29:12.023 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.90% / 0.00% / 78.10% | 20260505 18:29:12.023 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.247705% / 3.97364e-06% / 0.000111262% | 20260505 18:29:12.023 | INFO | Baz IO overall: | 20260505 18:29:12.023 | INFO | BytesWritten : 3158400539773 | 20260505 18:29:12.023 | INFO | NumEvents : 1496151269106 | 20260505 18:29:13.756 | INFO | Analyzing chunk frames = [5365760,5366272) | 20260505 18:29:13.756 | INFO | MemInfo: { | 20260505 18:29:13.756 | INFO | "gpu" : " 17 GB", | 20260505 18:29:13.756 | INFO | "huge" : "", | 20260505 18:29:13.756 | INFO | "huge_shared" : " 27 GB", | 20260505 18:29:13.756 | INFO | "malloc" : " 14 GB", | 20260505 18:29:13.756 | INFO | "pinned" : " 29 GB" | 20260505 18:29:13.756 | INFO | } | 20260505 18:29:13.756 | INFO | Perf report for the last 5.14115 seconds: | 20260505 18:29:13.756 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:29:13.756 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:13.756 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:13.756 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:13.756 | INFO | ANALYSIS : 63.4%, 3.0, 98.8ms, 0 | 20260505 18:29:13.756 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:13.756 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260505 18:29:13.756 | INFO | PRE_HQ : 60.2%, 1.8, 55.9ms, 0 | 20260505 18:29:13.756 | INFO | BAZWRITER : 63.1%, 3.8, 121.7ms, 0 | 20260505 18:29:13.756 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 18:29:13.756 | INFO | Uploaded 36GiB to GPU in 2.14717s (16.7662GiB/s) | 20260505 18:29:13.756 | INFO | Downloaded 38GiB to GPU in 2.25567s (16.8465GiB/s) | 20260505 18:29:18.903 | INFO | Analyzing chunk frames = [5366272,5366784) | 20260505 18:29:18.903 | INFO | MemInfo: { | 20260505 18:29:18.903 | INFO | "gpu" : " 17 GB", | 20260505 18:29:18.903 | INFO | "huge" : "", | 20260505 18:29:18.903 | INFO | "huge_shared" : " 27 GB", | 20260505 18:29:18.903 | INFO | "malloc" : " 14 GB", | 20260505 18:29:18.903 | INFO | "pinned" : " 29 GB" | 20260505 18:29:18.903 | INFO | } | 20260505 18:29:18.903 | INFO | Perf report for the last 5.14686 seconds: | 20260505 18:29:18.903 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:29:18.903 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:18.903 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:18.903 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:18.903 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260505 18:29:18.903 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:18.903 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:18.903 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 18:29:18.903 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:18.903 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:18.903 | INFO | Uploaded 36GiB to GPU in 2.13036s (16.8985GiB/s) | 20260505 18:29:18.903 | INFO | Downloaded 34GiB to GPU in 2.02876s (16.759GiB/s) | 20260505 18:29:24.041 | INFO | Analyzing chunk frames = [5366784,5367296) | 20260505 18:29:24.041 | INFO | MemInfo: { | 20260505 18:29:24.041 | INFO | "gpu" : " 17 GB", | 20260505 18:29:24.041 | INFO | "huge" : "", | 20260505 18:29:24.041 | INFO | "huge_shared" : " 27 GB", | 20260505 18:29:24.041 | INFO | "malloc" : " 14 GB", | 20260505 18:29:24.041 | INFO | "pinned" : " 29 GB" | 20260505 18:29:24.041 | INFO | } | 20260505 18:29:24.041 | INFO | Perf report for the last 5.1385 seconds: | 20260505 18:29:24.041 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:29:24.041 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:24.041 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:24.041 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:24.041 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260505 18:29:24.041 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:24.041 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:24.041 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 18:29:24.041 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:24.041 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:24.042 | INFO | Uploaded 36GiB to GPU in 2.12179s (16.9668GiB/s) | 20260505 18:29:24.042 | INFO | Downloaded 34GiB to GPU in 2.02123s (16.8214GiB/s) | 20260505 18:29:29.177 | INFO | Analyzing chunk frames = [5367296,5367808) | 20260505 18:29:29.177 | INFO | MemInfo: { | 20260505 18:29:29.177 | INFO | "gpu" : " 17 GB", | 20260505 18:29:29.177 | INFO | "huge" : "", | 20260505 18:29:29.177 | INFO | "huge_shared" : " 27 GB", | 20260505 18:29:29.177 | INFO | "malloc" : " 14 GB", | 20260505 18:29:29.177 | INFO | "pinned" : " 29 GB" | 20260505 18:29:29.177 | INFO | } | 20260505 18:29:29.177 | INFO | Perf report for the last 5.13554 seconds: | 20260505 18:29:29.177 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:29:29.177 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:29.177 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:29.177 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:29.177 | INFO | ANALYSIS : 62.4%, 3.0, 96.8ms, 0 | 20260505 18:29:29.177 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:29.177 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:29.177 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:29:29.177 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:29.177 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:29.177 | INFO | Uploaded 36GiB to GPU in 2.17494s (16.5522GiB/s) | 20260505 18:29:29.177 | INFO | Downloaded 34GiB to GPU in 2.05911s (16.512GiB/s) | 20260505 18:29:34.306 | INFO | Analyzing chunk frames = [5367808,5368320) | 20260505 18:29:34.306 | INFO | MemInfo: { | 20260505 18:29:34.306 | INFO | "gpu" : " 17 GB", | 20260505 18:29:34.306 | INFO | "huge" : "", | 20260505 18:29:34.306 | INFO | "huge_shared" : " 27 GB", | 20260505 18:29:34.306 | INFO | "malloc" : " 14 GB", | 20260505 18:29:34.306 | INFO | "pinned" : " 29 GB" | 20260505 18:29:34.306 | INFO | } | 20260505 18:29:34.306 | INFO | Perf report for the last 5.12929 seconds: | 20260505 18:29:34.306 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:29:34.306 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:34.306 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:34.306 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:34.306 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260505 18:29:34.306 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:34.306 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:34.306 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:29:34.306 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:34.306 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:34.306 | INFO | Uploaded 36GiB to GPU in 2.12758s (16.9206GiB/s) | 20260505 18:29:34.306 | INFO | Downloaded 34GiB to GPU in 2.05894s (16.5134GiB/s) | 20260505 18:29:39.454 | INFO | Analyzing chunk frames = [5368320,5368832) | 20260505 18:29:39.454 | INFO | MemInfo: { | 20260505 18:29:39.454 | INFO | "gpu" : " 17 GB", | 20260505 18:29:39.454 | INFO | "huge" : "", | 20260505 18:29:39.454 | INFO | "huge_shared" : " 27 GB", | 20260505 18:29:39.454 | INFO | "malloc" : " 14 GB", | 20260505 18:29:39.454 | INFO | "pinned" : " 29 GB" | 20260505 18:29:39.454 | INFO | } | 20260505 18:29:39.454 | INFO | Perf report for the last 5.14779 seconds: | 20260505 18:29:39.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:29:39.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:39.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:39.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:39.454 | INFO | ANALYSIS : 62.3%, 3.0, 97.4ms, 0 | 20260505 18:29:39.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:39.454 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:39.454 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 18:29:39.454 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:39.454 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:39.454 | INFO | Uploaded 36GiB to GPU in 2.13907s (16.8298GiB/s) | 20260505 18:29:39.454 | INFO | Downloaded 34GiB to GPU in 2.03994s (16.6672GiB/s) | 20260505 18:29:44.595 | INFO | Analyzing chunk frames = [5368832,5369344) | 20260505 18:29:44.595 | INFO | MemInfo: { | 20260505 18:29:44.595 | INFO | "gpu" : " 17 GB", | 20260505 18:29:44.595 | INFO | "huge" : "", | 20260505 18:29:44.595 | INFO | "huge_shared" : " 27 GB", | 20260505 18:29:44.595 | INFO | "malloc" : " 14 GB", | 20260505 18:29:44.595 | INFO | "pinned" : " 29 GB" | 20260505 18:29:44.595 | INFO | } | 20260505 18:29:44.595 | INFO | Perf report for the last 5.14114 seconds: | 20260505 18:29:44.595 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:29:44.595 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:44.595 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:44.595 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:44.595 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 18:29:44.595 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:44.595 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:44.595 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 18:29:44.595 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:44.595 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:44.595 | INFO | Uploaded 36GiB to GPU in 2.13197s (16.8858GiB/s) | 20260505 18:29:44.595 | INFO | Downloaded 34GiB to GPU in 2.13497s (15.9253GiB/s) | 20260505 18:29:49.737 | INFO | Analyzing chunk frames = [5369344,5369856) | 20260505 18:29:49.737 | INFO | MemInfo: { | 20260505 18:29:49.737 | INFO | "gpu" : " 17 GB", | 20260505 18:29:49.737 | INFO | "huge" : "", | 20260505 18:29:49.737 | INFO | "huge_shared" : " 27 GB", | 20260505 18:29:49.737 | INFO | "malloc" : " 14 GB", | 20260505 18:29:49.737 | INFO | "pinned" : " 29 GB" | 20260505 18:29:49.737 | INFO | } | 20260505 18:29:49.737 | INFO | Perf report for the last 5.14216 seconds: | 20260505 18:29:49.737 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:29:49.737 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:49.737 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:49.737 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:49.737 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 18:29:49.737 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:49.737 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:49.737 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:29:49.737 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:49.737 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:49.737 | INFO | Uploaded 36GiB to GPU in 2.20004s (16.3634GiB/s) | 20260505 18:29:49.738 | INFO | Downloaded 34GiB to GPU in 2.03469s (16.7102GiB/s) | 20260505 18:29:49.867 | INFO | Starting a round of Baz IO | 20260505 18:29:53.045 | INFO | Populating RTMetrics report | 20260505 18:29:53.045 | INFO | RTMetrics report populated | 20260505 18:29:53.045 | INFO | Opening tmp RTMetrics json file | 20260505 18:29:53.048 | INFO | Serializing RTMetrics to json | 20260505 18:29:53.050 | INFO | Serialization done, swapping tmp file now... | 20260505 18:29:53.051 | INFO | RTMetrics done | 20260505 18:29:53.136 | INFO | Finished a round of Baz IO: Wrote 2.09 GiB in 3.268 seconds | 20260505 18:29:53.136 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.90% / 0.00% / 78.10% | 20260505 18:29:53.136 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.247105% / 0% / 9.93411e-05% | 20260505 18:29:53.136 | INFO | Baz IO overall: | 20260505 18:29:53.136 | INFO | BytesWritten : 3160644724243 | 20260505 18:29:53.136 | INFO | NumEvents : 1497197011980 | 20260505 18:29:54.871 | INFO | Analyzing chunk frames = [5369856,5370368) | 20260505 18:29:54.871 | INFO | MemInfo: { | 20260505 18:29:54.871 | INFO | "gpu" : " 17 GB", | 20260505 18:29:54.871 | INFO | "huge" : "", | 20260505 18:29:54.871 | INFO | "huge_shared" : " 27 GB", | 20260505 18:29:54.871 | INFO | "malloc" : " 14 GB", | 20260505 18:29:54.871 | INFO | "pinned" : " 29 GB" | 20260505 18:29:54.871 | INFO | } | 20260505 18:29:54.871 | INFO | Perf report for the last 5.13374 seconds: | 20260505 18:29:54.871 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:29:54.871 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:54.871 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:54.871 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:54.871 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 18:29:54.871 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:29:54.871 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260505 18:29:54.871 | INFO | PRE_HQ : 60.4%, 1.8, 56.0ms, 0 | 20260505 18:29:54.871 | INFO | BAZWRITER : 63.2%, 3.8, 122.7ms, 0 | 20260505 18:29:54.871 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 18:29:54.871 | INFO | Uploaded 36GiB to GPU in 2.16246s (16.6477GiB/s) | 20260505 18:29:54.871 | INFO | Downloaded 38GiB to GPU in 2.24925s (16.8945GiB/s) | 20260505 18:30:00.007 | INFO | Analyzing chunk frames = [5370368,5370880) | 20260505 18:30:00.007 | INFO | MemInfo: { | 20260505 18:30:00.007 | INFO | "gpu" : " 17 GB", | 20260505 18:30:00.007 | INFO | "huge" : "", | 20260505 18:30:00.007 | INFO | "huge_shared" : " 27 GB", | 20260505 18:30:00.007 | INFO | "malloc" : " 14 GB", | 20260505 18:30:00.007 | INFO | "pinned" : " 29 GB" | 20260505 18:30:00.007 | INFO | } | 20260505 18:30:00.007 | INFO | Perf report for the last 5.13617 seconds: | 20260505 18:30:00.007 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:30:00.007 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:00.007 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:00.007 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:00.007 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 18:30:00.007 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:00.007 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:00.007 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:30:00.007 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:00.007 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:00.007 | INFO | Uploaded 36GiB to GPU in 2.14795s (16.7602GiB/s) | 20260505 18:30:00.007 | INFO | Downloaded 34GiB to GPU in 2.03461s (16.7108GiB/s) | 20260505 18:30:05.159 | INFO | Analyzing chunk frames = [5370880,5371392) | 20260505 18:30:05.159 | INFO | MemInfo: { | 20260505 18:30:05.159 | INFO | "gpu" : " 17 GB", | 20260505 18:30:05.159 | INFO | "huge" : "", | 20260505 18:30:05.159 | INFO | "huge_shared" : " 27 GB", | 20260505 18:30:05.159 | INFO | "malloc" : " 14 GB", | 20260505 18:30:05.159 | INFO | "pinned" : " 29 GB" | 20260505 18:30:05.159 | INFO | } | 20260505 18:30:05.159 | INFO | Perf report for the last 5.15175 seconds: | 20260505 18:30:05.159 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:30:05.159 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:05.159 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:05.159 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:05.159 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260505 18:30:05.159 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:05.159 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:05.159 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:30:05.159 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:05.159 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:05.159 | INFO | Uploaded 36GiB to GPU in 2.20011s (16.3628GiB/s) | 20260505 18:30:05.160 | INFO | Downloaded 34GiB to GPU in 2.04103s (16.6582GiB/s) | 20260505 18:30:10.292 | INFO | Analyzing chunk frames = [5371392,5371904) | 20260505 18:30:10.292 | INFO | MemInfo: { | 20260505 18:30:10.292 | INFO | "gpu" : " 17 GB", | 20260505 18:30:10.292 | INFO | "huge" : "", | 20260505 18:30:10.292 | INFO | "huge_shared" : " 27 GB", | 20260505 18:30:10.292 | INFO | "malloc" : " 14 GB", | 20260505 18:30:10.292 | INFO | "pinned" : " 29 GB" | 20260505 18:30:10.292 | INFO | } | 20260505 18:30:10.292 | INFO | Perf report for the last 5.13342 seconds: | 20260505 18:30:10.292 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:30:10.292 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:10.292 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:10.292 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:10.292 | INFO | ANALYSIS : 62.4%, 3.0, 96.9ms, 0 | 20260505 18:30:10.292 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:10.293 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:10.293 | INFO | PRE_HQ : 27.7%, 1.0, 14.3ms, 0 | 20260505 18:30:10.293 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:10.293 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:10.293 | INFO | Uploaded 36GiB to GPU in 2.1903s (16.4361GiB/s) | 20260505 18:30:10.293 | INFO | Downloaded 34GiB to GPU in 2.0419s (16.6511GiB/s) | 20260505 18:30:15.425 | INFO | Analyzing chunk frames = [5371904,5372416) | 20260505 18:30:15.426 | INFO | MemInfo: { | 20260505 18:30:15.426 | INFO | "gpu" : " 17 GB", | 20260505 18:30:15.426 | INFO | "huge" : "", | 20260505 18:30:15.426 | INFO | "huge_shared" : " 27 GB", | 20260505 18:30:15.426 | INFO | "malloc" : " 14 GB", | 20260505 18:30:15.426 | INFO | "pinned" : " 29 GB" | 20260505 18:30:15.426 | INFO | } | 20260505 18:30:15.426 | INFO | Perf report for the last 5.13326 seconds: | 20260505 18:30:15.426 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:30:15.426 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:15.426 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:15.426 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:15.426 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260505 18:30:15.426 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:15.426 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:15.426 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:30:15.426 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:15.426 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:15.426 | INFO | Uploaded 36GiB to GPU in 2.14326s (16.7968GiB/s) | 20260505 18:30:15.426 | INFO | Downloaded 34GiB to GPU in 2.06229s (16.4866GiB/s) | 20260505 18:30:20.566 | INFO | Analyzing chunk frames = [5372416,5372928) | 20260505 18:30:20.566 | INFO | MemInfo: { | 20260505 18:30:20.566 | INFO | "gpu" : " 17 GB", | 20260505 18:30:20.566 | INFO | "huge" : "", | 20260505 18:30:20.566 | INFO | "huge_shared" : " 27 GB", | 20260505 18:30:20.566 | INFO | "malloc" : " 14 GB", | 20260505 18:30:20.566 | INFO | "pinned" : " 29 GB" | 20260505 18:30:20.566 | INFO | } | 20260505 18:30:20.566 | INFO | Perf report for the last 5.14063 seconds: | 20260505 18:30:20.566 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:30:20.566 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:20.566 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:20.566 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:20.566 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260505 18:30:20.566 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:20.566 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:20.566 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 18:30:20.566 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:20.566 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:20.567 | INFO | Uploaded 36GiB to GPU in 2.14087s (16.8156GiB/s) | 20260505 18:30:20.567 | INFO | Downloaded 34GiB to GPU in 2.04915s (16.5922GiB/s) | 20260505 18:30:25.712 | INFO | Analyzing chunk frames = [5372928,5373440) | 20260505 18:30:25.712 | INFO | MemInfo: { | 20260505 18:30:25.712 | INFO | "gpu" : " 17 GB", | 20260505 18:30:25.712 | INFO | "huge" : "", | 20260505 18:30:25.712 | INFO | "huge_shared" : " 27 GB", | 20260505 18:30:25.712 | INFO | "malloc" : " 14 GB", | 20260505 18:30:25.712 | INFO | "pinned" : " 29 GB" | 20260505 18:30:25.712 | INFO | } | 20260505 18:30:25.712 | INFO | Perf report for the last 5.14555 seconds: | 20260505 18:30:25.712 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:30:25.712 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:25.712 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:25.712 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:25.712 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 18:30:25.712 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:25.712 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:25.712 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 18:30:25.712 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:25.712 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:25.712 | INFO | Uploaded 36GiB to GPU in 2.16306s (16.6431GiB/s) | 20260505 18:30:25.712 | INFO | Downloaded 34GiB to GPU in 2.18974s (15.5269GiB/s) | 20260505 18:30:30.851 | INFO | Analyzing chunk frames = [5373440,5373952) | 20260505 18:30:30.851 | INFO | MemInfo: { | 20260505 18:30:30.851 | INFO | "gpu" : " 17 GB", | 20260505 18:30:30.851 | INFO | "huge" : "", | 20260505 18:30:30.852 | INFO | "huge_shared" : " 27 GB", | 20260505 18:30:30.852 | INFO | "malloc" : " 14 GB", | 20260505 18:30:30.852 | INFO | "pinned" : " 29 GB" | 20260505 18:30:30.852 | INFO | } | 20260505 18:30:30.852 | INFO | Perf report for the last 5.13975 seconds: | 20260505 18:30:30.852 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:30:30.852 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:30.852 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:30.852 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:30.852 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 18:30:30.852 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:30.852 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:30.852 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 18:30:30.852 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:30.852 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:30.852 | INFO | Uploaded 36GiB to GPU in 2.19898s (16.3712GiB/s) | 20260505 18:30:30.852 | INFO | Downloaded 34GiB to GPU in 2.04196s (16.6507GiB/s) | 20260505 18:30:30.852 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 18:30:30.852 | INFO | Total mem usage: 15.3 GiB | 20260505 18:30:30.852 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 18:30:30.852 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.01%) | 20260505 18:30:30.852 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 18:30:30.852 | INFO | High Water Breakdown: | 20260505 18:30:30.852 | INFO | 61% : 8.66 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 18:30:30.852 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 18:30:30.852 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 18:30:30.852 | INFO | ------------------------------------------ | 20260505 18:30:30.852 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:30:30.852 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 18:30:30.852 | INFO | Total mem usage: 26.5 GiB | 20260505 18:30:30.852 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 18:30:30.852 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 18:30:30.852 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 18:30:30.852 | INFO | High Water Breakdown: | 20260505 18:30:30.852 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:30:30.852 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:30:30.852 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:30:30.852 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:30:30.852 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:30:30.852 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:30:30.852 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 18:30:30.852 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 18:30:30.852 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:30:30.852 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 18:30:30.852 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 18:30:30.852 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 18:30:30.852 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 18:30:30.852 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 18:30:30.853 | INFO | ------------------------------------------ | 20260505 18:30:30.853 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:30:30.853 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 18:30:30.853 | INFO | Total mem usage: 25.6 GiB | 20260505 18:30:30.853 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 18:30:30.853 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 18:30:30.853 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 18:30:30.853 | INFO | High Water Breakdown: | 20260505 18:30:30.853 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 18:30:30.853 | INFO | ------------------------------------------ | 20260505 18:30:30.853 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:30:30.853 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 18:30:30.853 | INFO | Total mem usage: 15.7 GiB | 20260505 18:30:30.853 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 18:30:30.853 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 18:30:30.853 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 18:30:30.853 | INFO | High Water Breakdown: | 20260505 18:30:30.853 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 18:30:30.853 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 18:30:30.853 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 18:30:30.853 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 18:30:30.853 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 18:30:30.853 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 18:30:30.853 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 18:30:30.853 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 18:30:30.853 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 18:30:30.853 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:30:30.853 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 18:30:30.853 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:30:30.853 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 18:30:30.853 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 18:30:30.853 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 18:30:30.853 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 18:30:30.853 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:30:30.853 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 18:30:30.853 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:30:30.853 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:30:30.853 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:30:30.853 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 18:30:30.853 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 18:30:30.853 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:30:30.853 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 18:30:30.853 | INFO | ------------------------------------------ | 20260505 18:30:30.853 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:30:30.853 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 18:30:30.853 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:30:30.853 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:30:30.853 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.45e+01, 1.37e+01 / 1.99e+00 | 20260505 18:30:30.853 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 1.02e-01, 1.07e-02 / 2.75e-03 | 20260505 18:30:30.853 | INFO | PerfReport: 11.81%, Baseline -- 3.72e+00 / 4.53e+00, 5.70e-01 / 1.24e-01 | 20260505 18:30:30.853 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.53e+00, 1.91e-01 / 8.70e-02 | 20260505 18:30:30.853 | INFO | PerfReport: 15.95%, DME -- 4.02e+01 / 4.45e+01, 4.12e+00 / 1.93e+00 | 20260505 18:30:30.853 | INFO | PerfReport: 56.87%, FrameLabeling -- 1.79e+01 / 1.91e+01, 1.43e+00 / 8.95e-01 | 20260505 18:30:30.853 | INFO | PerfReport: 7.31%, PulseAccumulating -- 2.31e+00 / 2.90e+00, 4.47e-01 / 1.62e-01 | 20260505 18:30:30.853 | INFO | PerfReport: 3.33%, Metrics -- 1.05e+00 / 1.38e+00, 2.89e-01 / 1.29e-01 | 20260505 18:30:30.853 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.65e-01 / 4.27e-01, 6.45e-02 / 3.76e-02 | 20260505 18:30:30.853 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 18:30:30.853 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:30:30.853 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:30:30.853 | INFO | PerfReport: Full Execution -- 9.97e+01 / -nan, 2.57e+01 / 0.00e+00 | 20260505 18:30:30.853 | INFO | PerfReport: 22.53%, Upload -- 2.25e+01 / 0.00e+00, 7.77e+00 / 0.00e+00 | 20260505 18:30:30.853 | INFO | PerfReport: 21.76%, Download -- 2.17e+01 / 0.00e+00, 7.74e+00 / 0.00e+00 | 20260505 18:30:30.853 | INFO | PerfReport: 31.62%, Compute -- 3.15e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 18:30:30.853 | INFO | PerfReport: 2.31%, LockUpload -- 2.30e+00 / 0.00e+00, 8.12e+00 / 0.00e+00 | 20260505 18:30:30.853 | INFO | PerfReport: 1.42%, LockDownload -- 1.41e+00 / 0.00e+00, 2.97e+00 / 0.00e+00 | 20260505 18:30:30.853 | INFO | PerfReport: 20.35%, LockCompute -- 2.03e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 18:30:30.853 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.14e-02 / 0.00e+00 | 20260505 18:30:30.983 | INFO | Starting a round of Baz IO | 20260505 18:30:34.159 | INFO | Populating RTMetrics report | 20260505 18:30:34.159 | INFO | RTMetrics report populated | 20260505 18:30:34.159 | INFO | Opening tmp RTMetrics json file | 20260505 18:30:34.160 | INFO | Serializing RTMetrics to json | 20260505 18:30:34.161 | INFO | Serialization done, swapping tmp file now... | 20260505 18:30:34.162 | INFO | RTMetrics done | 20260505 18:30:34.255 | INFO | Finished a round of Baz IO: Wrote 2.088 GiB in 3.272 seconds | 20260505 18:30:34.255 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.90% / 0.00% / 78.10% | 20260505 18:30:34.255 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.246338% / 3.97364e-06% / 7.94729e-05% | 20260505 18:30:34.256 | INFO | Baz IO overall: | 20260505 18:30:34.256 | INFO | BytesWritten : 3162886977045 | 20260505 18:30:34.256 | INFO | NumEvents : 1498241114068 | 20260505 18:30:35.990 | INFO | Analyzing chunk frames = [5373952,5374464) | 20260505 18:30:35.990 | INFO | MemInfo: { | 20260505 18:30:35.990 | INFO | "gpu" : " 17 GB", | 20260505 18:30:35.990 | INFO | "huge" : "", | 20260505 18:30:35.990 | INFO | "huge_shared" : " 27 GB", | 20260505 18:30:35.990 | INFO | "malloc" : " 14 GB", | 20260505 18:30:35.990 | INFO | "pinned" : " 29 GB" | 20260505 18:30:35.990 | INFO | } | 20260505 18:30:35.990 | INFO | Perf report for the last 5.13804 seconds: | 20260505 18:30:35.990 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:30:35.990 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:35.990 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:35.990 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:35.990 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 18:30:35.990 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:35.990 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260505 18:30:35.990 | INFO | PRE_HQ : 60.3%, 1.8, 56.0ms, 0 | 20260505 18:30:35.990 | INFO | BAZWRITER : 63.2%, 3.8, 123.0ms, 0 | 20260505 18:30:35.990 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 18:30:35.990 | INFO | Uploaded 36GiB to GPU in 2.16697s (16.6131GiB/s) | 20260505 18:30:35.990 | INFO | Downloaded 38GiB to GPU in 2.25322s (16.8648GiB/s) | 20260505 18:30:41.123 | INFO | Analyzing chunk frames = [5374464,5374976) | 20260505 18:30:41.123 | INFO | MemInfo: { | 20260505 18:30:41.123 | INFO | "gpu" : " 17 GB", | 20260505 18:30:41.123 | INFO | "huge" : "", | 20260505 18:30:41.123 | INFO | "huge_shared" : " 27 GB", | 20260505 18:30:41.123 | INFO | "malloc" : " 14 GB", | 20260505 18:30:41.123 | INFO | "pinned" : " 29 GB" | 20260505 18:30:41.123 | INFO | } | 20260505 18:30:41.123 | INFO | Perf report for the last 5.1333 seconds: | 20260505 18:30:41.123 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:30:41.123 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:41.123 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:41.123 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:41.123 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 18:30:41.123 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:41.123 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:41.123 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:30:41.123 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:41.123 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:41.123 | INFO | Uploaded 36GiB to GPU in 2.12713s (16.9242GiB/s) | 20260505 18:30:41.123 | INFO | Downloaded 34GiB to GPU in 2.03008s (16.7481GiB/s) | 20260505 18:30:46.260 | INFO | Analyzing chunk frames = [5374976,5375488) | 20260505 18:30:46.260 | INFO | MemInfo: { | 20260505 18:30:46.260 | INFO | "gpu" : " 17 GB", | 20260505 18:30:46.260 | INFO | "huge" : "", | 20260505 18:30:46.260 | INFO | "huge_shared" : " 27 GB", | 20260505 18:30:46.260 | INFO | "malloc" : " 14 GB", | 20260505 18:30:46.260 | INFO | "pinned" : " 29 GB" | 20260505 18:30:46.260 | INFO | } | 20260505 18:30:46.260 | INFO | Perf report for the last 5.13705 seconds: | 20260505 18:30:46.260 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:30:46.260 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:46.260 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:46.260 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:46.260 | INFO | ANALYSIS : 62.8%, 3.0, 97.6ms, 0 | 20260505 18:30:46.260 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:46.260 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:46.260 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:30:46.260 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:46.260 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:46.260 | INFO | Uploaded 36GiB to GPU in 2.18844s (16.4501GiB/s) | 20260505 18:30:46.260 | INFO | Downloaded 34GiB to GPU in 2.03783s (16.6844GiB/s) | 20260505 18:30:51.411 | INFO | Analyzing chunk frames = [5375488,5376000) | 20260505 18:30:51.411 | INFO | MemInfo: { | 20260505 18:30:51.411 | INFO | "gpu" : " 17 GB", | 20260505 18:30:51.411 | INFO | "huge" : "", | 20260505 18:30:51.411 | INFO | "huge_shared" : " 27 GB", | 20260505 18:30:51.411 | INFO | "malloc" : " 14 GB", | 20260505 18:30:51.411 | INFO | "pinned" : " 29 GB" | 20260505 18:30:51.411 | INFO | } | 20260505 18:30:51.411 | INFO | Perf report for the last 5.15129 seconds: | 20260505 18:30:51.411 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:30:51.411 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:51.411 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:51.411 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:51.411 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260505 18:30:51.411 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:51.411 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:51.411 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 18:30:51.411 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:51.411 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:51.412 | INFO | Uploaded 36GiB to GPU in 2.18266s (16.4936GiB/s) | 20260505 18:30:51.412 | INFO | Downloaded 34GiB to GPU in 2.14145s (15.8771GiB/s) | 20260505 18:30:56.553 | INFO | Analyzing chunk frames = [5376000,5376512) | 20260505 18:30:56.553 | INFO | MemInfo: { | 20260505 18:30:56.553 | INFO | "gpu" : " 17 GB", | 20260505 18:30:56.553 | INFO | "huge" : "", | 20260505 18:30:56.553 | INFO | "huge_shared" : " 27 GB", | 20260505 18:30:56.553 | INFO | "malloc" : " 14 GB", | 20260505 18:30:56.553 | INFO | "pinned" : " 29 GB" | 20260505 18:30:56.553 | INFO | } | 20260505 18:30:56.553 | INFO | Perf report for the last 5.14182 seconds: | 20260505 18:30:56.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:30:56.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:56.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:56.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:56.553 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 18:30:56.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:56.553 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:56.553 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 18:30:56.553 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:56.554 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:30:56.554 | INFO | Uploaded 36GiB to GPU in 2.1298s (16.903GiB/s) | 20260505 18:30:56.554 | INFO | Downloaded 34GiB to GPU in 2.05559s (16.5403GiB/s) | 20260505 18:31:01.686 | INFO | Analyzing chunk frames = [5376512,5377024) | 20260505 18:31:01.686 | INFO | MemInfo: { | 20260505 18:31:01.686 | INFO | "gpu" : " 17 GB", | 20260505 18:31:01.686 | INFO | "huge" : "", | 20260505 18:31:01.686 | INFO | "huge_shared" : " 27 GB", | 20260505 18:31:01.686 | INFO | "malloc" : " 14 GB", | 20260505 18:31:01.686 | INFO | "pinned" : " 29 GB" | 20260505 18:31:01.686 | INFO | } | 20260505 18:31:01.686 | INFO | Perf report for the last 5.13281 seconds: | 20260505 18:31:01.686 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:31:01.686 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:01.686 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:01.686 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:01.686 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 18:31:01.686 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:01.686 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:01.686 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:31:01.686 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:01.686 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:01.686 | INFO | Uploaded 36GiB to GPU in 2.13408s (16.8691GiB/s) | 20260505 18:31:01.686 | INFO | Downloaded 34GiB to GPU in 2.05488s (16.5459GiB/s) | 20260505 18:31:06.819 | INFO | Analyzing chunk frames = [5377024,5377536) | 20260505 18:31:06.819 | INFO | MemInfo: { | 20260505 18:31:06.819 | INFO | "gpu" : " 17 GB", | 20260505 18:31:06.819 | INFO | "huge" : "", | 20260505 18:31:06.819 | INFO | "huge_shared" : " 27 GB", | 20260505 18:31:06.819 | INFO | "malloc" : " 14 GB", | 20260505 18:31:06.819 | INFO | "pinned" : " 29 GB" | 20260505 18:31:06.820 | INFO | } | 20260505 18:31:06.820 | INFO | Perf report for the last 5.13354 seconds: | 20260505 18:31:06.820 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:31:06.820 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:06.820 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:06.820 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:06.820 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 18:31:06.820 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:06.820 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:06.820 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 18:31:06.820 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:06.820 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:06.820 | INFO | Uploaded 36GiB to GPU in 2.13628s (16.8517GiB/s) | 20260505 18:31:06.820 | INFO | Downloaded 34GiB to GPU in 2.15902s (15.7479GiB/s) | 20260505 18:31:11.964 | INFO | Analyzing chunk frames = [5377536,5378048) | 20260505 18:31:11.964 | INFO | MemInfo: { | 20260505 18:31:11.964 | INFO | "gpu" : " 17 GB", | 20260505 18:31:11.964 | INFO | "huge" : "", | 20260505 18:31:11.964 | INFO | "huge_shared" : " 27 GB", | 20260505 18:31:11.964 | INFO | "malloc" : " 14 GB", | 20260505 18:31:11.964 | INFO | "pinned" : " 29 GB" | 20260505 18:31:11.964 | INFO | } | 20260505 18:31:11.964 | INFO | Perf report for the last 5.14485 seconds: | 20260505 18:31:11.964 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:31:11.964 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:11.964 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:11.964 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:11.964 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 18:31:11.964 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:11.964 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:11.964 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 18:31:11.964 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:11.964 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:11.965 | INFO | Uploaded 36GiB to GPU in 2.20214s (16.3477GiB/s) | 20260505 18:31:11.965 | INFO | Downloaded 34GiB to GPU in 2.02523s (16.7883GiB/s) | 20260505 18:31:12.097 | INFO | Starting a round of Baz IO | 20260505 18:31:15.280 | INFO | Populating RTMetrics report | 20260505 18:31:15.281 | INFO | RTMetrics report populated | 20260505 18:31:15.281 | INFO | Opening tmp RTMetrics json file | 20260505 18:31:15.282 | INFO | Serializing RTMetrics to json | 20260505 18:31:15.284 | INFO | Serialization done, swapping tmp file now... | 20260505 18:31:15.284 | INFO | RTMetrics done | 20260505 18:31:15.377 | INFO | Finished a round of Baz IO: Wrote 2.088 GiB in 3.279 seconds | 20260505 18:31:15.377 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.90% / 0.00% / 78.10% | 20260505 18:31:15.377 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.245806% / 3.97364e-06% / 7.94729e-05% | 20260505 18:31:15.377 | INFO | Baz IO overall: | 20260505 18:31:15.377 | INFO | BytesWritten : 3165128801532 | 20260505 18:31:15.377 | INFO | NumEvents : 1499285117646 | 20260505 18:31:17.103 | INFO | Analyzing chunk frames = [5378048,5378560) | 20260505 18:31:17.103 | INFO | MemInfo: { | 20260505 18:31:17.103 | INFO | "gpu" : " 17 GB", | 20260505 18:31:17.103 | INFO | "huge" : "", | 20260505 18:31:17.103 | INFO | "huge_shared" : " 27 GB", | 20260505 18:31:17.103 | INFO | "malloc" : " 14 GB", | 20260505 18:31:17.103 | INFO | "pinned" : " 29 GB" | 20260505 18:31:17.103 | INFO | } | 20260505 18:31:17.104 | INFO | Perf report for the last 5.13909 seconds: | 20260505 18:31:17.104 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:31:17.104 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:17.104 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:17.104 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:17.104 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 18:31:17.104 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:17.104 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260505 18:31:17.104 | INFO | PRE_HQ : 60.6%, 1.8, 56.2ms, 0 | 20260505 18:31:17.104 | INFO | BAZWRITER : 63.3%, 3.8, 123.5ms, 0 | 20260505 18:31:17.104 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 18:31:17.104 | INFO | Uploaded 36GiB to GPU in 2.1745s (16.5555GiB/s) | 20260505 18:31:17.104 | INFO | Downloaded 38GiB to GPU in 2.2405s (16.9605GiB/s) | 20260505 18:31:22.240 | INFO | Analyzing chunk frames = [5378560,5379072) | 20260505 18:31:22.240 | INFO | MemInfo: { | 20260505 18:31:22.240 | INFO | "gpu" : " 17 GB", | 20260505 18:31:22.240 | INFO | "huge" : "", | 20260505 18:31:22.240 | INFO | "huge_shared" : " 27 GB", | 20260505 18:31:22.240 | INFO | "malloc" : " 14 GB", | 20260505 18:31:22.240 | INFO | "pinned" : " 29 GB" | 20260505 18:31:22.240 | INFO | } | 20260505 18:31:22.240 | INFO | Perf report for the last 5.13692 seconds: | 20260505 18:31:22.240 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:31:22.240 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:22.240 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:22.240 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:22.240 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 18:31:22.240 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:22.240 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:22.241 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:31:22.241 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:22.241 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:22.241 | INFO | Uploaded 36GiB to GPU in 2.1552s (16.7038GiB/s) | 20260505 18:31:22.241 | INFO | Downloaded 34GiB to GPU in 2.05132s (16.5747GiB/s) | 20260505 18:31:27.380 | INFO | Analyzing chunk frames = [5379072,5379584) | 20260505 18:31:27.380 | INFO | MemInfo: { | 20260505 18:31:27.380 | INFO | "gpu" : " 17 GB", | 20260505 18:31:27.380 | INFO | "huge" : "", | 20260505 18:31:27.380 | INFO | "huge_shared" : " 27 GB", | 20260505 18:31:27.380 | INFO | "malloc" : " 14 GB", | 20260505 18:31:27.380 | INFO | "pinned" : " 29 GB" | 20260505 18:31:27.380 | INFO | } | 20260505 18:31:27.380 | INFO | Perf report for the last 5.13937 seconds: | 20260505 18:31:27.380 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:31:27.380 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:27.380 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:27.380 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:27.380 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 18:31:27.380 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:27.380 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:27.380 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:31:27.380 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:27.380 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:27.380 | INFO | Uploaded 36GiB to GPU in 2.202s (16.3487GiB/s) | 20260505 18:31:27.380 | INFO | Downloaded 34GiB to GPU in 2.04286s (16.6433GiB/s) | 20260505 18:31:32.515 | INFO | Analyzing chunk frames = [5379584,5380096) | 20260505 18:31:32.515 | INFO | MemInfo: { | 20260505 18:31:32.515 | INFO | "gpu" : " 17 GB", | 20260505 18:31:32.515 | INFO | "huge" : "", | 20260505 18:31:32.515 | INFO | "huge_shared" : " 27 GB", | 20260505 18:31:32.515 | INFO | "malloc" : " 14 GB", | 20260505 18:31:32.515 | INFO | "pinned" : " 29 GB" | 20260505 18:31:32.515 | INFO | } | 20260505 18:31:32.515 | INFO | Perf report for the last 5.13536 seconds: | 20260505 18:31:32.515 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:31:32.515 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:32.515 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:32.515 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:32.515 | INFO | ANALYSIS : 62.8%, 3.0, 97.5ms, 0 | 20260505 18:31:32.515 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:32.515 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:32.515 | INFO | PRE_HQ : 27.7%, 1.0, 14.3ms, 0 | 20260505 18:31:32.515 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:32.515 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:32.515 | INFO | Uploaded 36GiB to GPU in 2.22498s (16.1799GiB/s) | 20260505 18:31:32.515 | INFO | Downloaded 34GiB to GPU in 2.1281s (15.9767GiB/s) | 20260505 18:31:37.665 | INFO | Analyzing chunk frames = [5380096,5380608) | 20260505 18:31:37.665 | INFO | MemInfo: { | 20260505 18:31:37.665 | INFO | "gpu" : " 17 GB", | 20260505 18:31:37.665 | INFO | "huge" : "", | 20260505 18:31:37.665 | INFO | "huge_shared" : " 27 GB", | 20260505 18:31:37.665 | INFO | "malloc" : " 14 GB", | 20260505 18:31:37.665 | INFO | "pinned" : " 29 GB" | 20260505 18:31:37.665 | INFO | } | 20260505 18:31:37.665 | INFO | Perf report for the last 5.14984 seconds: | 20260505 18:31:37.665 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:31:37.665 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:37.665 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:37.665 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:37.665 | INFO | ANALYSIS : 62.0%, 3.0, 97.0ms, 0 | 20260505 18:31:37.665 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:37.665 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:37.666 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260505 18:31:37.666 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:37.666 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:37.666 | INFO | Uploaded 36GiB to GPU in 2.15139s (16.7334GiB/s) | 20260505 18:31:37.666 | INFO | Downloaded 34GiB to GPU in 2.05657s (16.5324GiB/s) | 20260505 18:31:42.810 | INFO | Analyzing chunk frames = [5380608,5381120) | 20260505 18:31:42.810 | INFO | MemInfo: { | 20260505 18:31:42.810 | INFO | "gpu" : " 17 GB", | 20260505 18:31:42.810 | INFO | "huge" : "", | 20260505 18:31:42.810 | INFO | "huge_shared" : " 27 GB", | 20260505 18:31:42.810 | INFO | "malloc" : " 14 GB", | 20260505 18:31:42.810 | INFO | "pinned" : " 29 GB" | 20260505 18:31:42.810 | INFO | } | 20260505 18:31:42.810 | INFO | Perf report for the last 5.14495 seconds: | 20260505 18:31:42.810 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:31:42.810 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:42.810 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:42.810 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:42.810 | INFO | ANALYSIS : 62.0%, 3.0, 97.0ms, 0 | 20260505 18:31:42.810 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:42.810 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:42.810 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 18:31:42.810 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:42.810 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:42.810 | INFO | Uploaded 36GiB to GPU in 2.13252s (16.8815GiB/s) | 20260505 18:31:42.810 | INFO | Downloaded 34GiB to GPU in 2.04991s (16.5861GiB/s) | 20260505 18:31:47.935 | INFO | Analyzing chunk frames = [5381120,5381632) | 20260505 18:31:47.935 | INFO | MemInfo: { | 20260505 18:31:47.935 | INFO | "gpu" : " 17 GB", | 20260505 18:31:47.935 | INFO | "huge" : "", | 20260505 18:31:47.935 | INFO | "huge_shared" : " 27 GB", | 20260505 18:31:47.935 | INFO | "malloc" : " 14 GB", | 20260505 18:31:47.935 | INFO | "pinned" : " 29 GB" | 20260505 18:31:47.935 | INFO | } | 20260505 18:31:47.935 | INFO | Perf report for the last 5.12495 seconds: | 20260505 18:31:47.935 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:31:47.935 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:47.935 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:47.935 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:47.935 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 18:31:47.935 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:47.935 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:47.935 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260505 18:31:47.935 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:47.935 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:47.935 | INFO | Uploaded 36GiB to GPU in 2.13948s (16.8265GiB/s) | 20260505 18:31:47.935 | INFO | Downloaded 34GiB to GPU in 2.14023s (15.8862GiB/s) | 20260505 18:31:53.070 | INFO | Analyzing chunk frames = [5381632,5382144) | 20260505 18:31:53.070 | INFO | MemInfo: { | 20260505 18:31:53.070 | INFO | "gpu" : " 17 GB", | 20260505 18:31:53.070 | INFO | "huge" : "", | 20260505 18:31:53.070 | INFO | "huge_shared" : " 27 GB", | 20260505 18:31:53.070 | INFO | "malloc" : " 14 GB", | 20260505 18:31:53.070 | INFO | "pinned" : " 29 GB" | 20260505 18:31:53.070 | INFO | } | 20260505 18:31:53.070 | INFO | Perf report for the last 5.13516 seconds: | 20260505 18:31:53.070 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:31:53.070 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:53.070 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:53.070 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:53.070 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 18:31:53.070 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:53.070 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:53.070 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 18:31:53.070 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:53.070 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:53.070 | INFO | Uploaded 36GiB to GPU in 2.18841s (16.4503GiB/s) | 20260505 18:31:53.070 | INFO | Downloaded 34GiB to GPU in 2.03282s (16.7255GiB/s) | 20260505 18:31:53.070 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 18:31:53.070 | INFO | Total mem usage: 15.3 GiB | 20260505 18:31:53.071 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 18:31:53.071 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.01%) | 20260505 18:31:53.071 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 18:31:53.071 | INFO | High Water Breakdown: | 20260505 18:31:53.071 | INFO | 61% : 8.66 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 18:31:53.071 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 18:31:53.071 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 18:31:53.071 | INFO | ------------------------------------------ | 20260505 18:31:53.071 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:31:53.071 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 18:31:53.071 | INFO | Total mem usage: 26.5 GiB | 20260505 18:31:53.071 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 18:31:53.071 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 18:31:53.071 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 18:31:53.071 | INFO | High Water Breakdown: | 20260505 18:31:53.071 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:31:53.071 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:31:53.071 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:31:53.071 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:31:53.071 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:31:53.071 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:31:53.071 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 18:31:53.071 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 18:31:53.071 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:31:53.071 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 18:31:53.071 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 18:31:53.071 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 18:31:53.071 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 18:31:53.071 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 18:31:53.071 | INFO | ------------------------------------------ | 20260505 18:31:53.071 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:31:53.071 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 18:31:53.071 | INFO | Total mem usage: 25.6 GiB | 20260505 18:31:53.071 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 18:31:53.071 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 18:31:53.071 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 18:31:53.071 | INFO | High Water Breakdown: | 20260505 18:31:53.071 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 18:31:53.071 | INFO | ------------------------------------------ | 20260505 18:31:53.071 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:31:53.071 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 18:31:53.071 | INFO | Total mem usage: 15.7 GiB | 20260505 18:31:53.071 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 18:31:53.071 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 18:31:53.071 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 18:31:53.071 | INFO | High Water Breakdown: | 20260505 18:31:53.071 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 18:31:53.071 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 18:31:53.071 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 18:31:53.071 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 18:31:53.071 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 18:31:53.071 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 18:31:53.071 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 18:31:53.071 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 18:31:53.071 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 18:31:53.071 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:31:53.071 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 18:31:53.071 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:31:53.071 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 18:31:53.071 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 18:31:53.071 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 18:31:53.071 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 18:31:53.071 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:31:53.071 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 18:31:53.071 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:31:53.071 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:31:53.071 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:31:53.071 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 18:31:53.071 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 18:31:53.071 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:31:53.071 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 18:31:53.071 | INFO | ------------------------------------------ | 20260505 18:31:53.071 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:31:53.071 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 18:31:53.071 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:31:53.071 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:31:53.071 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.50e+01, 1.36e+01 / 2.05e+00 | 20260505 18:31:53.071 | INFO | PerfReport: 0.29%, AggregateStats -- 9.26e-02 / 1.02e-01, 1.08e-02 / 3.41e-03 | 20260505 18:31:53.071 | INFO | PerfReport: 11.81%, Baseline -- 3.72e+00 / 4.55e+00, 5.70e-01 / 1.11e-01 | 20260505 18:31:53.071 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.54e+00, 1.91e-01 / 6.37e-02 | 20260505 18:31:53.071 | INFO | PerfReport: 16.03%, DME -- 4.04e+01 / 4.50e+01, 4.32e+00 / 2.25e+00 | 20260505 18:31:53.071 | INFO | PerfReport: 56.75%, FrameLabeling -- 1.79e+01 / 1.92e+01, 1.35e+00 / 9.74e-01 | 20260505 18:31:53.071 | INFO | PerfReport: 7.34%, PulseAccumulating -- 2.31e+00 / 2.89e+00, 4.47e-01 / 1.66e-01 | 20260505 18:31:53.072 | INFO | PerfReport: 3.34%, Metrics -- 1.05e+00 / 1.39e+00, 2.90e-01 / 1.70e-01 | 20260505 18:31:53.072 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.65e-01 / 4.29e-01, 6.40e-02 / 3.80e-02 | 20260505 18:31:53.072 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 18:31:53.072 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:31:53.072 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:31:53.072 | INFO | PerfReport: Full Execution -- 9.98e+01 / -nan, 2.57e+01 / 0.00e+00 | 20260505 18:31:53.072 | INFO | PerfReport: 22.59%, Upload -- 2.25e+01 / 0.00e+00, 7.90e+00 / 0.00e+00 | 20260505 18:31:53.072 | INFO | PerfReport: 21.85%, Download -- 2.18e+01 / 0.00e+00, 7.85e+00 / 0.00e+00 | 20260505 18:31:53.072 | INFO | PerfReport: 31.55%, Compute -- 3.15e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 18:31:53.072 | INFO | PerfReport: 2.34%, LockUpload -- 2.34e+00 / 0.00e+00, 7.91e+00 / 0.00e+00 | 20260505 18:31:53.072 | INFO | PerfReport: 1.41%, LockDownload -- 1.41e+00 / 0.00e+00, 2.87e+00 / 0.00e+00 | 20260505 18:31:53.072 | INFO | PerfReport: 20.25%, LockCompute -- 2.02e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 18:31:53.072 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.73e-02 / 0.00e+00 | 20260505 18:31:53.206 | INFO | Starting a round of Baz IO | 20260505 18:31:56.404 | INFO | Populating RTMetrics report | 20260505 18:31:56.405 | INFO | RTMetrics report populated | 20260505 18:31:56.405 | INFO | Opening tmp RTMetrics json file | 20260505 18:31:56.406 | INFO | Serializing RTMetrics to json | 20260505 18:31:56.407 | INFO | Serialization done, swapping tmp file now... | 20260505 18:31:56.407 | INFO | RTMetrics done | 20260505 18:31:56.497 | INFO | Finished a round of Baz IO: Wrote 2.087 GiB in 3.29 seconds | 20260505 18:31:56.497 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.90% / 0.00% / 78.10% | 20260505 18:31:56.497 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.245706% / 0% / 7.15256e-05% | 20260505 18:31:56.497 | INFO | Baz IO overall: | 20260505 18:31:56.497 | INFO | BytesWritten : 3167369969706 | 20260505 18:31:56.497 | INFO | NumEvents : 1500328908444 | 20260505 18:31:58.220 | INFO | Analyzing chunk frames = [5382144,5382656) | 20260505 18:31:58.220 | INFO | MemInfo: { | 20260505 18:31:58.220 | INFO | "gpu" : " 17 GB", | 20260505 18:31:58.220 | INFO | "huge" : "", | 20260505 18:31:58.220 | INFO | "huge_shared" : " 27 GB", | 20260505 18:31:58.220 | INFO | "malloc" : " 14 GB", | 20260505 18:31:58.220 | INFO | "pinned" : " 29 GB" | 20260505 18:31:58.220 | INFO | } | 20260505 18:31:58.220 | INFO | Perf report for the last 5.1496 seconds: | 20260505 18:31:58.220 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:31:58.220 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:58.220 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:58.220 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:58.220 | INFO | ANALYSIS : 63.7%, 3.0, 99.5ms, 0 | 20260505 18:31:58.220 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:31:58.220 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260505 18:31:58.220 | INFO | PRE_HQ : 60.3%, 1.8, 55.7ms, 0 | 20260505 18:31:58.220 | INFO | BAZWRITER : 63.3%, 3.8, 122.5ms, 0 | 20260505 18:31:58.220 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 18:31:58.220 | INFO | Uploaded 36GiB to GPU in 2.18231s (16.4963GiB/s) | 20260505 18:31:58.220 | INFO | Downloaded 38GiB to GPU in 2.2747s (16.7055GiB/s) | 20260505 18:32:03.359 | INFO | Analyzing chunk frames = [5382656,5383168) | 20260505 18:32:03.359 | INFO | MemInfo: { | 20260505 18:32:03.359 | INFO | "gpu" : " 17 GB", | 20260505 18:32:03.359 | INFO | "huge" : "", | 20260505 18:32:03.359 | INFO | "huge_shared" : " 27 GB", | 20260505 18:32:03.359 | INFO | "malloc" : " 14 GB", | 20260505 18:32:03.359 | INFO | "pinned" : " 29 GB" | 20260505 18:32:03.359 | INFO | } | 20260505 18:32:03.359 | INFO | Perf report for the last 5.13912 seconds: | 20260505 18:32:03.359 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:32:03.359 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:03.359 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:03.359 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:03.359 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 18:32:03.359 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:03.359 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:03.359 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:32:03.359 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:03.359 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:03.359 | INFO | Uploaded 36GiB to GPU in 2.12568s (16.9357GiB/s) | 20260505 18:32:03.359 | INFO | Downloaded 34GiB to GPU in 2.02944s (16.7534GiB/s) | 20260505 18:32:08.498 | INFO | Analyzing chunk frames = [5383168,5383680) | 20260505 18:32:08.498 | INFO | MemInfo: { | 20260505 18:32:08.498 | INFO | "gpu" : " 17 GB", | 20260505 18:32:08.498 | INFO | "huge" : "", | 20260505 18:32:08.498 | INFO | "huge_shared" : " 27 GB", | 20260505 18:32:08.498 | INFO | "malloc" : " 14 GB", | 20260505 18:32:08.498 | INFO | "pinned" : " 29 GB" | 20260505 18:32:08.498 | INFO | } | 20260505 18:32:08.498 | INFO | Perf report for the last 5.13881 seconds: | 20260505 18:32:08.498 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:32:08.498 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:08.498 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:08.498 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:08.498 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 18:32:08.498 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:08.498 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:08.498 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:32:08.498 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:08.498 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:08.498 | INFO | Uploaded 36GiB to GPU in 2.20542s (16.3234GiB/s) | 20260505 18:32:08.498 | INFO | Downloaded 34GiB to GPU in 2.07031s (16.4227GiB/s) | 20260505 18:32:13.630 | INFO | Analyzing chunk frames = [5383680,5384192) | 20260505 18:32:13.630 | INFO | MemInfo: { | 20260505 18:32:13.630 | INFO | "gpu" : " 17 GB", | 20260505 18:32:13.630 | INFO | "huge" : "", | 20260505 18:32:13.630 | INFO | "huge_shared" : " 27 GB", | 20260505 18:32:13.630 | INFO | "malloc" : " 14 GB", | 20260505 18:32:13.630 | INFO | "pinned" : " 29 GB" | 20260505 18:32:13.630 | INFO | } | 20260505 18:32:13.630 | INFO | Perf report for the last 5.13208 seconds: | 20260505 18:32:13.630 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:32:13.630 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:13.630 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:13.630 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:13.630 | INFO | ANALYSIS : 62.9%, 3.0, 97.6ms, 0 | 20260505 18:32:13.630 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:13.630 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:13.630 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 18:32:13.630 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:13.630 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:13.630 | INFO | Uploaded 36GiB to GPU in 2.22499s (16.1798GiB/s) | 20260505 18:32:13.630 | INFO | Downloaded 34GiB to GPU in 2.11958s (16.0409GiB/s) | 20260505 18:32:18.768 | INFO | Analyzing chunk frames = [5384192,5384704) | 20260505 18:32:18.768 | INFO | MemInfo: { | 20260505 18:32:18.768 | INFO | "gpu" : " 17 GB", | 20260505 18:32:18.768 | INFO | "huge" : "", | 20260505 18:32:18.769 | INFO | "huge_shared" : " 27 GB", | 20260505 18:32:18.769 | INFO | "malloc" : " 14 GB", | 20260505 18:32:18.769 | INFO | "pinned" : " 29 GB" | 20260505 18:32:18.769 | INFO | } | 20260505 18:32:18.769 | INFO | Perf report for the last 5.13879 seconds: | 20260505 18:32:18.769 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:32:18.769 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:18.769 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:18.769 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:18.769 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 18:32:18.769 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:18.769 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:18.769 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 18:32:18.769 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:18.769 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:18.769 | INFO | Uploaded 36GiB to GPU in 2.14912s (16.7511GiB/s) | 20260505 18:32:18.769 | INFO | Downloaded 34GiB to GPU in 2.06359s (16.4761GiB/s) | 20260505 18:32:23.920 | INFO | Analyzing chunk frames = [5384704,5385216) | 20260505 18:32:23.920 | INFO | MemInfo: { | 20260505 18:32:23.920 | INFO | "gpu" : " 17 GB", | 20260505 18:32:23.920 | INFO | "huge" : "", | 20260505 18:32:23.920 | INFO | "huge_shared" : " 27 GB", | 20260505 18:32:23.920 | INFO | "malloc" : " 14 GB", | 20260505 18:32:23.920 | INFO | "pinned" : " 29 GB" | 20260505 18:32:23.920 | INFO | } | 20260505 18:32:23.920 | INFO | Perf report for the last 5.15131 seconds: | 20260505 18:32:23.920 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:32:23.920 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:23.920 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:23.920 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:23.920 | INFO | ANALYSIS : 62.3%, 3.0, 97.4ms, 0 | 20260505 18:32:23.920 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:23.920 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:23.920 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260505 18:32:23.920 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:23.920 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:23.920 | INFO | Uploaded 36GiB to GPU in 2.14961s (16.7472GiB/s) | 20260505 18:32:23.920 | INFO | Downloaded 34GiB to GPU in 2.04725s (16.6076GiB/s) | 20260505 18:32:29.056 | INFO | Analyzing chunk frames = [5385216,5385728) | 20260505 18:32:29.056 | INFO | MemInfo: { | 20260505 18:32:29.056 | INFO | "gpu" : " 17 GB", | 20260505 18:32:29.056 | INFO | "huge" : "", | 20260505 18:32:29.056 | INFO | "huge_shared" : " 27 GB", | 20260505 18:32:29.056 | INFO | "malloc" : " 14 GB", | 20260505 18:32:29.056 | INFO | "pinned" : " 29 GB" | 20260505 18:32:29.056 | INFO | } | 20260505 18:32:29.056 | INFO | Perf report for the last 5.13626 seconds: | 20260505 18:32:29.056 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:32:29.056 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:29.056 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:29.056 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:29.056 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 18:32:29.056 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:29.056 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:29.056 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 18:32:29.056 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:29.056 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:29.057 | INFO | Uploaded 36GiB to GPU in 2.12885s (16.9105GiB/s) | 20260505 18:32:29.057 | INFO | Downloaded 34GiB to GPU in 2.13698s (15.9103GiB/s) | 20260505 18:32:34.188 | INFO | Analyzing chunk frames = [5385728,5386240) | 20260505 18:32:34.188 | INFO | MemInfo: { | 20260505 18:32:34.188 | INFO | "gpu" : " 17 GB", | 20260505 18:32:34.188 | INFO | "huge" : "", | 20260505 18:32:34.188 | INFO | "huge_shared" : " 27 GB", | 20260505 18:32:34.188 | INFO | "malloc" : " 14 GB", | 20260505 18:32:34.188 | INFO | "pinned" : " 29 GB" | 20260505 18:32:34.188 | INFO | } | 20260505 18:32:34.188 | INFO | Perf report for the last 5.13215 seconds: | 20260505 18:32:34.188 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:32:34.188 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:34.188 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:34.188 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:34.188 | INFO | ANALYSIS : 63.1%, 3.0, 98.3ms, 0 | 20260505 18:32:34.188 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:34.188 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:34.188 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 18:32:34.188 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:34.188 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:34.189 | INFO | Uploaded 36GiB to GPU in 2.20453s (16.33GiB/s) | 20260505 18:32:34.189 | INFO | Downloaded 34GiB to GPU in 2.04755s (16.6052GiB/s) | 20260505 18:32:34.326 | INFO | Starting a round of Baz IO | 20260505 18:32:37.512 | INFO | Populating RTMetrics report | 20260505 18:32:37.512 | INFO | RTMetrics report populated | 20260505 18:32:37.512 | INFO | Opening tmp RTMetrics json file | 20260505 18:32:37.516 | INFO | Serializing RTMetrics to json | 20260505 18:32:37.517 | INFO | Serialization done, swapping tmp file now... | 20260505 18:32:37.517 | INFO | RTMetrics done | 20260505 18:32:37.604 | INFO | Finished a round of Baz IO: Wrote 2.089 GiB in 3.277 seconds | 20260505 18:32:37.604 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.89% / 0.00% / 78.10% | 20260505 18:32:37.604 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.244935% / 0% / 7.54992e-05% | 20260505 18:32:37.604 | INFO | Baz IO overall: | 20260505 18:32:37.604 | INFO | BytesWritten : 3169612706037 | 20260505 18:32:37.604 | INFO | NumEvents : 1501373933788 | 20260505 18:32:39.329 | INFO | Analyzing chunk frames = [5386240,5386752) | 20260505 18:32:39.329 | INFO | MemInfo: { | 20260505 18:32:39.329 | INFO | "gpu" : " 17 GB", | 20260505 18:32:39.329 | INFO | "huge" : "", | 20260505 18:32:39.329 | INFO | "huge_shared" : " 27 GB", | 20260505 18:32:39.329 | INFO | "malloc" : " 14 GB", | 20260505 18:32:39.329 | INFO | "pinned" : " 29 GB" | 20260505 18:32:39.329 | INFO | } | 20260505 18:32:39.329 | INFO | Perf report for the last 5.14064 seconds: | 20260505 18:32:39.329 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:32:39.329 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:39.329 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:39.329 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:39.329 | INFO | ANALYSIS : 63.6%, 3.0, 99.1ms, 0 | 20260505 18:32:39.329 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:39.329 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260505 18:32:39.329 | INFO | PRE_HQ : 60.2%, 1.8, 55.8ms, 0 | 20260505 18:32:39.329 | INFO | BAZWRITER : 63.2%, 3.8, 122.3ms, 0 | 20260505 18:32:39.329 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 18:32:39.329 | INFO | Uploaded 36GiB to GPU in 2.16777s (16.607GiB/s) | 20260505 18:32:39.329 | INFO | Downloaded 38GiB to GPU in 2.26249s (16.7956GiB/s) | 20260505 18:32:44.475 | INFO | Analyzing chunk frames = [5386752,5387264) | 20260505 18:32:44.475 | INFO | MemInfo: { | 20260505 18:32:44.475 | INFO | "gpu" : " 17 GB", | 20260505 18:32:44.475 | INFO | "huge" : "", | 20260505 18:32:44.475 | INFO | "huge_shared" : " 27 GB", | 20260505 18:32:44.475 | INFO | "malloc" : " 14 GB", | 20260505 18:32:44.475 | INFO | "pinned" : " 29 GB" | 20260505 18:32:44.475 | INFO | } | 20260505 18:32:44.475 | INFO | Perf report for the last 5.1457 seconds: | 20260505 18:32:44.475 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:32:44.475 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:44.475 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:44.475 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:44.475 | INFO | ANALYSIS : 62.7%, 3.0, 98.0ms, 0 | 20260505 18:32:44.475 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:44.475 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:44.475 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 18:32:44.475 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:44.475 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:44.475 | INFO | Uploaded 36GiB to GPU in 2.13463s (16.8648GiB/s) | 20260505 18:32:44.475 | INFO | Downloaded 34GiB to GPU in 2.03495s (16.708GiB/s) | 20260505 18:32:49.615 | INFO | Analyzing chunk frames = [5387264,5387776) | 20260505 18:32:49.615 | INFO | MemInfo: { | 20260505 18:32:49.615 | INFO | "gpu" : " 17 GB", | 20260505 18:32:49.615 | INFO | "huge" : "", | 20260505 18:32:49.615 | INFO | "huge_shared" : " 27 GB", | 20260505 18:32:49.615 | INFO | "malloc" : " 14 GB", | 20260505 18:32:49.615 | INFO | "pinned" : " 29 GB" | 20260505 18:32:49.615 | INFO | } | 20260505 18:32:49.616 | INFO | Perf report for the last 5.14078 seconds: | 20260505 18:32:49.616 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:32:49.616 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:49.616 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:49.616 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:49.616 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260505 18:32:49.616 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:49.616 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:49.616 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:32:49.616 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:49.616 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:49.616 | INFO | Uploaded 36GiB to GPU in 2.13309s (16.8769GiB/s) | 20260505 18:32:49.616 | INFO | Downloaded 34GiB to GPU in 2.03415s (16.7146GiB/s) | 20260505 18:32:54.758 | INFO | Analyzing chunk frames = [5387776,5388288) | 20260505 18:32:54.758 | INFO | MemInfo: { | 20260505 18:32:54.758 | INFO | "gpu" : " 17 GB", | 20260505 18:32:54.758 | INFO | "huge" : "", | 20260505 18:32:54.758 | INFO | "huge_shared" : " 27 GB", | 20260505 18:32:54.758 | INFO | "malloc" : " 14 GB", | 20260505 18:32:54.758 | INFO | "pinned" : " 29 GB" | 20260505 18:32:54.758 | INFO | } | 20260505 18:32:54.758 | INFO | Perf report for the last 5.14238 seconds: | 20260505 18:32:54.758 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:32:54.758 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:54.758 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:54.758 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:54.758 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260505 18:32:54.758 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:54.758 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:54.758 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 18:32:54.758 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:54.758 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:54.758 | INFO | Uploaded 36GiB to GPU in 2.19128s (16.4287GiB/s) | 20260505 18:32:54.758 | INFO | Downloaded 34GiB to GPU in 2.04589s (16.6186GiB/s) | 20260505 18:32:59.887 | INFO | Analyzing chunk frames = [5388288,5388800) | 20260505 18:32:59.887 | INFO | MemInfo: { | 20260505 18:32:59.887 | INFO | "gpu" : " 17 GB", | 20260505 18:32:59.887 | INFO | "huge" : "", | 20260505 18:32:59.887 | INFO | "huge_shared" : " 27 GB", | 20260505 18:32:59.887 | INFO | "malloc" : " 14 GB", | 20260505 18:32:59.887 | INFO | "pinned" : " 29 GB" | 20260505 18:32:59.887 | INFO | } | 20260505 18:32:59.887 | INFO | Perf report for the last 5.12906 seconds: | 20260505 18:32:59.887 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:32:59.887 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:59.887 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:59.887 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:59.887 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260505 18:32:59.887 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:59.887 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:59.887 | INFO | PRE_HQ : 27.9%, 1.0, 14.4ms, 0 | 20260505 18:32:59.887 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:59.887 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:32:59.887 | INFO | Uploaded 36GiB to GPU in 2.12894s (16.9098GiB/s) | 20260505 18:32:59.887 | INFO | Downloaded 34GiB to GPU in 2.06295s (16.4813GiB/s) | 20260505 18:33:05.024 | INFO | Analyzing chunk frames = [5388800,5389312) | 20260505 18:33:05.024 | INFO | MemInfo: { | 20260505 18:33:05.024 | INFO | "gpu" : " 17 GB", | 20260505 18:33:05.024 | INFO | "huge" : "", | 20260505 18:33:05.024 | INFO | "huge_shared" : " 27 GB", | 20260505 18:33:05.024 | INFO | "malloc" : " 14 GB", | 20260505 18:33:05.024 | INFO | "pinned" : " 29 GB" | 20260505 18:33:05.024 | INFO | } | 20260505 18:33:05.024 | INFO | Perf report for the last 5.13721 seconds: | 20260505 18:33:05.024 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:33:05.024 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:05.024 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:05.024 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:05.024 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260505 18:33:05.024 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:05.024 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:05.024 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 18:33:05.024 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:05.024 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:05.024 | INFO | Uploaded 36GiB to GPU in 2.13952s (16.8262GiB/s) | 20260505 18:33:05.024 | INFO | Downloaded 34GiB to GPU in 2.04747s (16.6059GiB/s) | 20260505 18:33:10.173 | INFO | Analyzing chunk frames = [5389312,5389824) | 20260505 18:33:10.173 | INFO | MemInfo: { | 20260505 18:33:10.173 | INFO | "gpu" : " 17 GB", | 20260505 18:33:10.173 | INFO | "huge" : "", | 20260505 18:33:10.173 | INFO | "huge_shared" : " 27 GB", | 20260505 18:33:10.173 | INFO | "malloc" : " 14 GB", | 20260505 18:33:10.173 | INFO | "pinned" : " 29 GB" | 20260505 18:33:10.173 | INFO | } | 20260505 18:33:10.173 | INFO | Perf report for the last 5.14909 seconds: | 20260505 18:33:10.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:33:10.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:10.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:10.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:10.173 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 18:33:10.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:10.173 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:10.173 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 18:33:10.173 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:10.173 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:10.173 | INFO | Uploaded 36GiB to GPU in 2.15312s (16.7199GiB/s) | 20260505 18:33:10.174 | INFO | Downloaded 34GiB to GPU in 2.16286s (15.7199GiB/s) | 20260505 18:33:15.310 | INFO | Analyzing chunk frames = [5389824,5390336) | 20260505 18:33:15.310 | INFO | MemInfo: { | 20260505 18:33:15.310 | INFO | "gpu" : " 17 GB", | 20260505 18:33:15.310 | INFO | "huge" : "", | 20260505 18:33:15.310 | INFO | "huge_shared" : " 27 GB", | 20260505 18:33:15.310 | INFO | "malloc" : " 14 GB", | 20260505 18:33:15.310 | INFO | "pinned" : " 29 GB" | 20260505 18:33:15.310 | INFO | } | 20260505 18:33:15.310 | INFO | Perf report for the last 5.13715 seconds: | 20260505 18:33:15.310 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:33:15.310 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:15.310 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:15.310 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:15.310 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260505 18:33:15.310 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:15.310 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:15.311 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 18:33:15.311 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:15.311 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:15.311 | INFO | Uploaded 36GiB to GPU in 2.19362s (16.4112GiB/s) | 20260505 18:33:15.311 | INFO | Downloaded 34GiB to GPU in 2.02818s (16.7638GiB/s) | 20260505 18:33:15.311 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 18:33:15.311 | INFO | Total mem usage: 15.3 GiB | 20260505 18:33:15.311 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 18:33:15.311 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.02%) | 20260505 18:33:15.311 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 18:33:15.311 | INFO | High Water Breakdown: | 20260505 18:33:15.311 | INFO | 61% : 8.66 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 18:33:15.311 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 18:33:15.311 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 18:33:15.311 | INFO | ------------------------------------------ | 20260505 18:33:15.311 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:33:15.311 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 18:33:15.311 | INFO | Total mem usage: 26.5 GiB | 20260505 18:33:15.311 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 18:33:15.311 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 18:33:15.311 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 18:33:15.311 | INFO | High Water Breakdown: | 20260505 18:33:15.311 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:33:15.311 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:33:15.311 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:33:15.311 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:33:15.311 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:33:15.311 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:33:15.311 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 18:33:15.311 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 18:33:15.311 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:33:15.311 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 18:33:15.311 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 18:33:15.311 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 18:33:15.311 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 18:33:15.311 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 18:33:15.311 | INFO | ------------------------------------------ | 20260505 18:33:15.311 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:33:15.311 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 18:33:15.311 | INFO | Total mem usage: 25.6 GiB | 20260505 18:33:15.311 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 18:33:15.311 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 18:33:15.311 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 18:33:15.311 | INFO | High Water Breakdown: | 20260505 18:33:15.311 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 18:33:15.311 | INFO | ------------------------------------------ | 20260505 18:33:15.311 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:33:15.311 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 18:33:15.311 | INFO | Total mem usage: 15.7 GiB | 20260505 18:33:15.311 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 18:33:15.311 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 18:33:15.311 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 18:33:15.311 | INFO | High Water Breakdown: | 20260505 18:33:15.311 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 18:33:15.311 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 18:33:15.311 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 18:33:15.311 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 18:33:15.311 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 18:33:15.311 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 18:33:15.311 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 18:33:15.312 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 18:33:15.312 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 18:33:15.312 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:33:15.312 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 18:33:15.312 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:33:15.312 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 18:33:15.312 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 18:33:15.312 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 18:33:15.312 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 18:33:15.312 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:33:15.312 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 18:33:15.312 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:33:15.312 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:33:15.312 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:33:15.312 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 18:33:15.312 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 18:33:15.312 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:33:15.312 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 18:33:15.312 | INFO | ------------------------------------------ | 20260505 18:33:15.312 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:33:15.312 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 18:33:15.312 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:33:15.312 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:33:15.312 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.46e+01, 1.34e+01 / 2.10e+00 | 20260505 18:33:15.312 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.02e-01, 1.11e-02 / 3.56e-03 | 20260505 18:33:15.312 | INFO | PerfReport: 11.76%, Baseline -- 3.71e+00 / 4.54e+00, 5.57e-01 / 1.32e-01 | 20260505 18:33:15.312 | INFO | PerfReport: 3.26%, Histogram -- 1.03e+00 / 1.51e+00, 1.87e-01 / 1.21e-01 | 20260505 18:33:15.312 | INFO | PerfReport: 15.96%, DME -- 4.02e+01 / 4.45e+01, 4.24e+00 / 2.13e+00 | 20260505 18:33:15.312 | INFO | PerfReport: 56.92%, FrameLabeling -- 1.79e+01 / 1.93e+01, 1.41e+00 / 1.01e+00 | 20260505 18:33:15.312 | INFO | PerfReport: 7.32%, PulseAccumulating -- 2.31e+00 / 2.90e+00, 4.45e-01 / 1.80e-01 | 20260505 18:33:15.312 | INFO | PerfReport: 3.33%, Metrics -- 1.05e+00 / 1.38e+00, 2.87e-01 / 1.17e-01 | 20260505 18:33:15.312 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.65e-01 / 4.33e-01, 6.44e-02 / 3.66e-02 | 20260505 18:33:15.312 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 18:33:15.312 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:33:15.312 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:33:15.312 | INFO | PerfReport: Full Execution -- 9.99e+01 / -nan, 2.59e+01 / 0.00e+00 | 20260505 18:33:15.312 | INFO | PerfReport: 22.56%, Upload -- 2.25e+01 / 0.00e+00, 7.87e+00 / 0.00e+00 | 20260505 18:33:15.312 | INFO | PerfReport: 21.81%, Download -- 2.18e+01 / 0.00e+00, 7.81e+00 / 0.00e+00 | 20260505 18:33:15.312 | INFO | PerfReport: 31.55%, Compute -- 3.15e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 18:33:15.312 | INFO | PerfReport: 2.37%, LockUpload -- 2.37e+00 / 0.00e+00, 8.03e+00 / 0.00e+00 | 20260505 18:33:15.312 | INFO | PerfReport: 1.39%, LockDownload -- 1.39e+00 / 0.00e+00, 2.93e+00 / 0.00e+00 | 20260505 18:33:15.312 | INFO | PerfReport: 20.30%, LockCompute -- 2.03e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260505 18:33:15.312 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.81e-02 / 0.00e+00 | 20260505 18:33:15.446 | INFO | Starting a round of Baz IO | 20260505 18:33:18.617 | INFO | Populating RTMetrics report | 20260505 18:33:18.617 | INFO | RTMetrics report populated | 20260505 18:33:18.617 | INFO | Opening tmp RTMetrics json file | 20260505 18:33:18.621 | INFO | Serializing RTMetrics to json | 20260505 18:33:18.623 | INFO | Serialization done, swapping tmp file now... | 20260505 18:33:18.624 | INFO | RTMetrics done | 20260505 18:33:18.707 | INFO | Finished a round of Baz IO: Wrote 2.086 GiB in 3.261 seconds | 20260505 18:33:18.708 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.89% / 0.00% / 78.11% | 20260505 18:33:18.708 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.244737% / 0% / 5.5631e-05% | 20260505 18:33:18.708 | INFO | Baz IO overall: | 20260505 18:33:18.708 | INFO | BytesWritten : 3171852895597 | 20260505 18:33:18.708 | INFO | NumEvents : 1502417470287 | 20260505 18:33:20.446 | INFO | Analyzing chunk frames = [5390336,5390848) | 20260505 18:33:20.446 | INFO | MemInfo: { | 20260505 18:33:20.446 | INFO | "gpu" : " 17 GB", | 20260505 18:33:20.446 | INFO | "huge" : "", | 20260505 18:33:20.446 | INFO | "huge_shared" : " 27 GB", | 20260505 18:33:20.446 | INFO | "malloc" : " 14 GB", | 20260505 18:33:20.446 | INFO | "pinned" : " 29 GB" | 20260505 18:33:20.446 | INFO | } | 20260505 18:33:20.446 | INFO | Perf report for the last 5.13533 seconds: | 20260505 18:33:20.446 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:33:20.446 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:20.446 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:20.446 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:20.446 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 18:33:20.446 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:20.446 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260505 18:33:20.446 | INFO | PRE_HQ : 60.4%, 1.8, 55.9ms, 0 | 20260505 18:33:20.446 | INFO | BAZWRITER : 63.0%, 3.8, 122.6ms, 0 | 20260505 18:33:20.446 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 18:33:20.446 | INFO | Uploaded 36GiB to GPU in 2.16982s (16.5913GiB/s) | 20260505 18:33:20.446 | INFO | Downloaded 38GiB to GPU in 2.25203s (16.8737GiB/s) | 20260505 18:33:25.582 | INFO | Analyzing chunk frames = [5390848,5391360) | 20260505 18:33:25.582 | INFO | MemInfo: { | 20260505 18:33:25.582 | INFO | "gpu" : " 17 GB", | 20260505 18:33:25.582 | INFO | "huge" : "", | 20260505 18:33:25.582 | INFO | "huge_shared" : " 27 GB", | 20260505 18:33:25.582 | INFO | "malloc" : " 14 GB", | 20260505 18:33:25.582 | INFO | "pinned" : " 29 GB" | 20260505 18:33:25.582 | INFO | } | 20260505 18:33:25.582 | INFO | Perf report for the last 5.13625 seconds: | 20260505 18:33:25.582 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:33:25.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:25.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:25.582 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:25.582 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 18:33:25.582 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:25.582 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:25.582 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:33:25.582 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:25.582 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:25.582 | INFO | Uploaded 36GiB to GPU in 2.13713s (16.845GiB/s) | 20260505 18:33:25.583 | INFO | Downloaded 34GiB to GPU in 2.03407s (16.7152GiB/s) | 20260505 18:33:30.729 | INFO | Analyzing chunk frames = [5391360,5391872) | 20260505 18:33:30.729 | INFO | MemInfo: { | 20260505 18:33:30.729 | INFO | "gpu" : " 17 GB", | 20260505 18:33:30.729 | INFO | "huge" : "", | 20260505 18:33:30.729 | INFO | "huge_shared" : " 27 GB", | 20260505 18:33:30.729 | INFO | "malloc" : " 14 GB", | 20260505 18:33:30.729 | INFO | "pinned" : " 29 GB" | 20260505 18:33:30.729 | INFO | } | 20260505 18:33:30.729 | INFO | Perf report for the last 5.14712 seconds: | 20260505 18:33:30.729 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:33:30.729 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:30.729 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:30.729 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:30.729 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260505 18:33:30.729 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:30.729 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:30.729 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:33:30.729 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:30.729 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:30.729 | INFO | Uploaded 36GiB to GPU in 2.19801s (16.3785GiB/s) | 20260505 18:33:30.729 | INFO | Downloaded 34GiB to GPU in 2.03912s (16.6739GiB/s) | 20260505 18:33:35.866 | INFO | Analyzing chunk frames = [5391872,5392384) | 20260505 18:33:35.866 | INFO | MemInfo: { | 20260505 18:33:35.866 | INFO | "gpu" : " 17 GB", | 20260505 18:33:35.866 | INFO | "huge" : "", | 20260505 18:33:35.866 | INFO | "huge_shared" : " 27 GB", | 20260505 18:33:35.866 | INFO | "malloc" : " 14 GB", | 20260505 18:33:35.866 | INFO | "pinned" : " 29 GB" | 20260505 18:33:35.866 | INFO | } | 20260505 18:33:35.866 | INFO | Perf report for the last 5.13701 seconds: | 20260505 18:33:35.866 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:33:35.866 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:35.866 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:35.866 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:35.866 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260505 18:33:35.866 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:35.866 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:35.866 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 18:33:35.866 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:35.866 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:35.866 | INFO | Uploaded 36GiB to GPU in 2.19344s (16.4126GiB/s) | 20260505 18:33:35.867 | INFO | Downloaded 34GiB to GPU in 2.03612s (16.6984GiB/s) | 20260505 18:33:41.003 | INFO | Analyzing chunk frames = [5392384,5392896) | 20260505 18:33:41.003 | INFO | MemInfo: { | 20260505 18:33:41.003 | INFO | "gpu" : " 17 GB", | 20260505 18:33:41.003 | INFO | "huge" : "", | 20260505 18:33:41.003 | INFO | "huge_shared" : " 27 GB", | 20260505 18:33:41.003 | INFO | "malloc" : " 14 GB", | 20260505 18:33:41.003 | INFO | "pinned" : " 29 GB" | 20260505 18:33:41.003 | INFO | } | 20260505 18:33:41.003 | INFO | Perf report for the last 5.1366 seconds: | 20260505 18:33:41.003 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:33:41.003 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:41.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:41.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:41.003 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 18:33:41.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:41.003 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:41.003 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 18:33:41.003 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:41.003 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:41.003 | INFO | Uploaded 36GiB to GPU in 2.14226s (16.8047GiB/s) | 20260505 18:33:41.003 | INFO | Downloaded 34GiB to GPU in 2.05731s (16.5264GiB/s) | 20260505 18:33:46.136 | INFO | Analyzing chunk frames = [5392896,5393408) | 20260505 18:33:46.136 | INFO | MemInfo: { | 20260505 18:33:46.136 | INFO | "gpu" : " 17 GB", | 20260505 18:33:46.136 | INFO | "huge" : "", | 20260505 18:33:46.136 | INFO | "huge_shared" : " 27 GB", | 20260505 18:33:46.136 | INFO | "malloc" : " 14 GB", | 20260505 18:33:46.136 | INFO | "pinned" : " 29 GB" | 20260505 18:33:46.136 | INFO | } | 20260505 18:33:46.136 | INFO | Perf report for the last 5.13301 seconds: | 20260505 18:33:46.136 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:33:46.136 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:46.136 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:46.136 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:46.136 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260505 18:33:46.136 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:46.136 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:46.136 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:33:46.136 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:46.136 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:46.136 | INFO | Uploaded 36GiB to GPU in 2.14703s (16.7674GiB/s) | 20260505 18:33:46.136 | INFO | Downloaded 34GiB to GPU in 2.05333s (16.5584GiB/s) | 20260505 18:33:51.277 | INFO | Analyzing chunk frames = [5393408,5393920) | 20260505 18:33:51.277 | INFO | MemInfo: { | 20260505 18:33:51.277 | INFO | "gpu" : " 17 GB", | 20260505 18:33:51.277 | INFO | "huge" : "", | 20260505 18:33:51.278 | INFO | "huge_shared" : " 27 GB", | 20260505 18:33:51.278 | INFO | "malloc" : " 14 GB", | 20260505 18:33:51.278 | INFO | "pinned" : " 29 GB" | 20260505 18:33:51.278 | INFO | } | 20260505 18:33:51.278 | INFO | Perf report for the last 5.14169 seconds: | 20260505 18:33:51.278 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:33:51.278 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:51.278 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:51.278 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:51.278 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 18:33:51.278 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:51.278 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:51.278 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 18:33:51.278 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:51.278 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:51.278 | INFO | Uploaded 36GiB to GPU in 2.15271s (16.7231GiB/s) | 20260505 18:33:51.278 | INFO | Downloaded 34GiB to GPU in 2.15127s (15.8046GiB/s) | 20260505 18:33:56.428 | INFO | Analyzing chunk frames = [5393920,5394432) | 20260505 18:33:56.428 | INFO | MemInfo: { | 20260505 18:33:56.428 | INFO | "gpu" : " 17 GB", | 20260505 18:33:56.428 | INFO | "huge" : "", | 20260505 18:33:56.428 | INFO | "huge_shared" : " 27 GB", | 20260505 18:33:56.428 | INFO | "malloc" : " 14 GB", | 20260505 18:33:56.428 | INFO | "pinned" : " 29 GB" | 20260505 18:33:56.428 | INFO | } | 20260505 18:33:56.428 | INFO | Perf report for the last 5.15039 seconds: | 20260505 18:33:56.428 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:33:56.428 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:56.428 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:56.428 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:56.428 | INFO | ANALYSIS : 62.7%, 3.0, 98.1ms, 0 | 20260505 18:33:56.428 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:56.428 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:56.428 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 18:33:56.428 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:56.428 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:33:56.428 | INFO | Uploaded 36GiB to GPU in 2.17923s (16.5196GiB/s) | 20260505 18:33:56.428 | INFO | Downloaded 34GiB to GPU in 2.03068s (16.7432GiB/s) | 20260505 18:33:56.558 | INFO | Starting a round of Baz IO | 20260505 18:33:59.737 | INFO | Populating RTMetrics report | 20260505 18:33:59.738 | INFO | RTMetrics report populated | 20260505 18:33:59.738 | INFO | Opening tmp RTMetrics json file | 20260505 18:33:59.741 | INFO | Serializing RTMetrics to json | 20260505 18:33:59.742 | INFO | Serialization done, swapping tmp file now... | 20260505 18:33:59.742 | INFO | RTMetrics done | 20260505 18:33:59.833 | INFO | Finished a round of Baz IO: Wrote 2.084 GiB in 3.274 seconds | 20260505 18:33:59.833 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.89% / 0.00% / 78.11% | 20260505 18:33:59.833 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.243417% / 0% / 8.34465e-05% | 20260505 18:33:59.833 | INFO | Baz IO overall: | 20260505 18:33:59.833 | INFO | BytesWritten : 3174090389458 | 20260505 18:33:59.833 | INFO | NumEvents : 1503459261910 | 20260505 18:34:01.569 | INFO | Analyzing chunk frames = [5394432,5394944) | 20260505 18:34:01.569 | INFO | MemInfo: { | 20260505 18:34:01.569 | INFO | "gpu" : " 17 GB", | 20260505 18:34:01.569 | INFO | "huge" : "", | 20260505 18:34:01.569 | INFO | "huge_shared" : " 27 GB", | 20260505 18:34:01.569 | INFO | "malloc" : " 14 GB", | 20260505 18:34:01.569 | INFO | "pinned" : " 29 GB" | 20260505 18:34:01.569 | INFO | } | 20260505 18:34:01.570 | INFO | Perf report for the last 5.14155 seconds: | 20260505 18:34:01.570 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:34:01.570 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:01.570 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:01.570 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:01.570 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 18:34:01.570 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:01.570 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260505 18:34:01.570 | INFO | PRE_HQ : 60.6%, 1.8, 55.9ms, 0 | 20260505 18:34:01.570 | INFO | BAZWRITER : 63.2%, 3.7, 121.9ms, 0 | 20260505 18:34:01.570 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 18:34:01.570 | INFO | Uploaded 36GiB to GPU in 2.15765s (16.6848GiB/s) | 20260505 18:34:01.570 | INFO | Downloaded 38GiB to GPU in 2.23607s (16.9941GiB/s) | 20260505 18:34:06.702 | INFO | Analyzing chunk frames = [5394944,5395456) | 20260505 18:34:06.702 | INFO | MemInfo: { | 20260505 18:34:06.702 | INFO | "gpu" : " 17 GB", | 20260505 18:34:06.702 | INFO | "huge" : "", | 20260505 18:34:06.702 | INFO | "huge_shared" : " 27 GB", | 20260505 18:34:06.702 | INFO | "malloc" : " 14 GB", | 20260505 18:34:06.702 | INFO | "pinned" : " 29 GB" | 20260505 18:34:06.702 | INFO | } | 20260505 18:34:06.702 | INFO | Perf report for the last 5.13265 seconds: | 20260505 18:34:06.702 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:34:06.702 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:06.702 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:06.702 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:06.702 | INFO | ANALYSIS : 62.9%, 3.0, 98.0ms, 0 | 20260505 18:34:06.702 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:06.702 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:06.702 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:34:06.702 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:06.702 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:06.702 | INFO | Uploaded 36GiB to GPU in 2.16427s (16.6338GiB/s) | 20260505 18:34:06.702 | INFO | Downloaded 34GiB to GPU in 2.0575s (16.5249GiB/s) | 20260505 18:34:11.831 | INFO | Analyzing chunk frames = [5395456,5395968) | 20260505 18:34:11.831 | INFO | MemInfo: { | 20260505 18:34:11.831 | INFO | "gpu" : " 17 GB", | 20260505 18:34:11.831 | INFO | "huge" : "", | 20260505 18:34:11.831 | INFO | "huge_shared" : " 27 GB", | 20260505 18:34:11.831 | INFO | "malloc" : " 14 GB", | 20260505 18:34:11.831 | INFO | "pinned" : " 29 GB" | 20260505 18:34:11.831 | INFO | } | 20260505 18:34:11.831 | INFO | Perf report for the last 5.12865 seconds: | 20260505 18:34:11.831 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:34:11.831 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:11.831 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:11.831 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:11.831 | INFO | ANALYSIS : 62.8%, 3.0, 97.6ms, 0 | 20260505 18:34:11.831 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:11.831 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:11.831 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:34:11.831 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:11.831 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:11.831 | INFO | Uploaded 36GiB to GPU in 2.20775s (16.3062GiB/s) | 20260505 18:34:11.831 | INFO | Downloaded 34GiB to GPU in 2.06586s (16.4581GiB/s) | 20260505 18:34:16.979 | INFO | Analyzing chunk frames = [5395968,5396480) | 20260505 18:34:16.979 | INFO | MemInfo: { | 20260505 18:34:16.979 | INFO | "gpu" : " 17 GB", | 20260505 18:34:16.979 | INFO | "huge" : "", | 20260505 18:34:16.979 | INFO | "huge_shared" : " 27 GB", | 20260505 18:34:16.979 | INFO | "malloc" : " 14 GB", | 20260505 18:34:16.979 | INFO | "pinned" : " 29 GB" | 20260505 18:34:16.979 | INFO | } | 20260505 18:34:16.979 | INFO | Perf report for the last 5.14785 seconds: | 20260505 18:34:16.979 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:34:16.979 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:16.979 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:16.979 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:16.979 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260505 18:34:16.979 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:16.979 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:16.979 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 18:34:16.979 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:16.979 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:16.979 | INFO | Uploaded 36GiB to GPU in 2.18276s (16.4929GiB/s) | 20260505 18:34:16.979 | INFO | Downloaded 34GiB to GPU in 2.14186s (15.874GiB/s) | 20260505 18:34:22.119 | INFO | Analyzing chunk frames = [5396480,5396992) | 20260505 18:34:22.119 | INFO | MemInfo: { | 20260505 18:34:22.119 | INFO | "gpu" : " 17 GB", | 20260505 18:34:22.119 | INFO | "huge" : "", | 20260505 18:34:22.119 | INFO | "huge_shared" : " 27 GB", | 20260505 18:34:22.119 | INFO | "malloc" : " 14 GB", | 20260505 18:34:22.119 | INFO | "pinned" : " 29 GB" | 20260505 18:34:22.119 | INFO | } | 20260505 18:34:22.120 | INFO | Perf report for the last 5.14078 seconds: | 20260505 18:34:22.120 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:34:22.120 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:22.120 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:22.120 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:22.120 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 18:34:22.120 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:22.120 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:22.120 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:34:22.120 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:22.120 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:22.120 | INFO | Uploaded 36GiB to GPU in 2.1406s (16.8177GiB/s) | 20260505 18:34:22.120 | INFO | Downloaded 34GiB to GPU in 2.062s (16.4889GiB/s) | 20260505 18:34:27.257 | INFO | Analyzing chunk frames = [5396992,5397504) | 20260505 18:34:27.257 | INFO | MemInfo: { | 20260505 18:34:27.257 | INFO | "gpu" : " 17 GB", | 20260505 18:34:27.257 | INFO | "huge" : "", | 20260505 18:34:27.257 | INFO | "huge_shared" : " 27 GB", | 20260505 18:34:27.257 | INFO | "malloc" : " 14 GB", | 20260505 18:34:27.257 | INFO | "pinned" : " 29 GB" | 20260505 18:34:27.257 | INFO | } | 20260505 18:34:27.257 | INFO | Perf report for the last 5.13763 seconds: | 20260505 18:34:27.257 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:34:27.257 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:27.257 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:27.257 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:27.257 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 18:34:27.257 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:27.257 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:27.257 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:34:27.257 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:27.257 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:27.257 | INFO | Uploaded 36GiB to GPU in 2.14432s (16.7885GiB/s) | 20260505 18:34:27.257 | INFO | Downloaded 34GiB to GPU in 2.05734s (16.5262GiB/s) | 20260505 18:34:32.399 | INFO | Analyzing chunk frames = [5397504,5398016) | 20260505 18:34:32.399 | INFO | MemInfo: { | 20260505 18:34:32.399 | INFO | "gpu" : " 17 GB", | 20260505 18:34:32.399 | INFO | "huge" : "", | 20260505 18:34:32.399 | INFO | "huge_shared" : " 27 GB", | 20260505 18:34:32.399 | INFO | "malloc" : " 14 GB", | 20260505 18:34:32.399 | INFO | "pinned" : " 29 GB" | 20260505 18:34:32.399 | INFO | } | 20260505 18:34:32.399 | INFO | Perf report for the last 5.14201 seconds: | 20260505 18:34:32.399 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:34:32.399 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:32.399 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:32.399 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:32.399 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 18:34:32.399 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:32.399 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:32.399 | INFO | PRE_HQ : 28.5%, 1.0, 14.9ms, 0 | 20260505 18:34:32.399 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:32.399 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:32.399 | INFO | Uploaded 36GiB to GPU in 2.13472s (16.8641GiB/s) | 20260505 18:34:32.399 | INFO | Downloaded 34GiB to GPU in 2.15021s (15.8124GiB/s) | 20260505 18:34:37.536 | INFO | Analyzing chunk frames = [5398016,5398528) | 20260505 18:34:37.536 | INFO | MemInfo: { | 20260505 18:34:37.536 | INFO | "gpu" : " 17 GB", | 20260505 18:34:37.536 | INFO | "huge" : "", | 20260505 18:34:37.536 | INFO | "huge_shared" : " 27 GB", | 20260505 18:34:37.537 | INFO | "malloc" : " 14 GB", | 20260505 18:34:37.537 | INFO | "pinned" : " 29 GB" | 20260505 18:34:37.537 | INFO | } | 20260505 18:34:37.537 | INFO | Perf report for the last 5.13736 seconds: | 20260505 18:34:37.537 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:34:37.537 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:37.537 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:37.537 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:37.537 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 18:34:37.537 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:37.537 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:37.537 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:34:37.537 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:37.537 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:37.537 | INFO | Uploaded 36GiB to GPU in 2.20093s (16.3568GiB/s) | 20260505 18:34:37.537 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 18:34:37.537 | INFO | Total mem usage: 15.3 GiB | 20260505 18:34:37.537 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 18:34:37.537 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.05%) | 20260505 18:34:37.537 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 18:34:37.537 | INFO | High Water Breakdown: | 20260505 18:34:37.537 | INFO | 61% : 8.66 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 18:34:37.537 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 18:34:37.537 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 18:34:37.537 | INFO | ------------------------------------------ | 20260505 18:34:37.537 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:34:37.537 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 18:34:37.537 | INFO | Total mem usage: 26.5 GiB | 20260505 18:34:37.537 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 18:34:37.537 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 18:34:37.537 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 18:34:37.537 | INFO | High Water Breakdown: | 20260505 18:34:37.537 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:34:37.537 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:34:37.537 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:34:37.537 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:34:37.537 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:34:37.537 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:34:37.537 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 18:34:37.537 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 18:34:37.537 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:34:37.537 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 18:34:37.537 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 18:34:37.537 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 18:34:37.537 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 18:34:37.537 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 18:34:37.537 | INFO | ------------------------------------------ | 20260505 18:34:37.537 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:34:37.537 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 18:34:37.537 | INFO | Total mem usage: 25.6 GiB | 20260505 18:34:37.537 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 18:34:37.537 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 18:34:37.537 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 18:34:37.537 | INFO | High Water Breakdown: | 20260505 18:34:37.537 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 18:34:37.537 | INFO | ------------------------------------------ | 20260505 18:34:37.537 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:34:37.537 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 18:34:37.537 | INFO | Total mem usage: 15.7 GiB | 20260505 18:34:37.537 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 18:34:37.537 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 18:34:37.537 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 18:34:37.537 | INFO | High Water Breakdown: | 20260505 18:34:37.537 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 18:34:37.537 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 18:34:37.537 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 18:34:37.537 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 18:34:37.537 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 18:34:37.537 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 18:34:37.537 | INFO | Downloaded 34GiB to GPU in 2.02971s (16.7512GiB/s) | 20260505 18:34:37.537 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 18:34:37.537 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 18:34:37.537 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 18:34:37.537 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:34:37.537 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 18:34:37.537 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:34:37.537 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 18:34:37.537 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 18:34:37.537 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 18:34:37.537 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 18:34:37.537 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:34:37.537 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 18:34:37.537 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:34:37.537 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:34:37.537 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:34:37.537 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 18:34:37.537 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 18:34:37.537 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:34:37.537 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 18:34:37.537 | INFO | ------------------------------------------ | 20260505 18:34:37.537 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:34:37.538 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 18:34:37.538 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:34:37.538 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:34:37.538 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.44e+01, 1.33e+01 / 1.74e+00 | 20260505 18:34:37.538 | INFO | PerfReport: 0.29%, AggregateStats -- 9.26e-02 / 1.03e-01, 1.05e-02 / 3.14e-03 | 20260505 18:34:37.538 | INFO | PerfReport: 11.83%, Baseline -- 3.72e+00 / 4.55e+00, 5.70e-01 / 1.05e-01 | 20260505 18:34:37.538 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.54e+00, 1.86e-01 / 6.07e-02 | 20260505 18:34:37.538 | INFO | PerfReport: 15.91%, DME -- 4.01e+01 / 4.43e+01, 3.91e+00 / 1.57e+00 | 20260505 18:34:37.538 | INFO | PerfReport: 56.86%, FrameLabeling -- 1.79e+01 / 1.92e+01, 1.41e+00 / 1.04e+00 | 20260505 18:34:37.538 | INFO | PerfReport: 7.33%, PulseAccumulating -- 2.31e+00 / 2.87e+00, 4.42e-01 / 1.67e-01 | 20260505 18:34:37.538 | INFO | PerfReport: 3.34%, Metrics -- 1.05e+00 / 1.41e+00, 2.89e-01 / 1.47e-01 | 20260505 18:34:37.538 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.65e-01 / 4.32e-01, 6.39e-02 / 4.06e-02 | 20260505 18:34:37.538 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 18:34:37.538 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:34:37.538 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:34:37.538 | INFO | PerfReport: Full Execution -- 9.97e+01 / -nan, 2.53e+01 / 0.00e+00 | 20260505 18:34:37.538 | INFO | PerfReport: 22.62%, Upload -- 2.26e+01 / 0.00e+00, 7.81e+00 / 0.00e+00 | 20260505 18:34:37.538 | INFO | PerfReport: 21.84%, Download -- 2.18e+01 / 0.00e+00, 7.76e+00 / 0.00e+00 | 20260505 18:34:37.538 | INFO | PerfReport: 31.58%, Compute -- 3.15e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260505 18:34:37.538 | INFO | PerfReport: 2.27%, LockUpload -- 2.26e+00 / 0.00e+00, 7.94e+00 / 0.00e+00 | 20260505 18:34:37.538 | INFO | PerfReport: 1.44%, LockDownload -- 1.43e+00 / 0.00e+00, 2.89e+00 / 0.00e+00 | 20260505 18:34:37.538 | INFO | PerfReport: 20.23%, LockCompute -- 2.02e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 18:34:37.538 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.04e-02 / 0.00e+00 | 20260505 18:34:37.673 | INFO | Starting a round of Baz IO | 20260505 18:34:40.846 | INFO | Populating RTMetrics report | 20260505 18:34:40.846 | INFO | RTMetrics report populated | 20260505 18:34:40.846 | INFO | Opening tmp RTMetrics json file | 20260505 18:34:40.850 | INFO | Serializing RTMetrics to json | 20260505 18:34:40.851 | INFO | Serialization done, swapping tmp file now... | 20260505 18:34:40.851 | INFO | RTMetrics done | 20260505 18:34:40.939 | INFO | Finished a round of Baz IO: Wrote 2.084 GiB in 3.266 seconds | 20260505 18:34:40.939 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.89% / 0.00% / 78.11% | 20260505 18:34:40.939 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.244077% / 0% / 6.35783e-05% | 20260505 18:34:40.939 | INFO | Baz IO overall: | 20260505 18:34:40.939 | INFO | BytesWritten : 3176328266569 | 20260505 18:34:40.939 | INFO | NumEvents : 1504501594541 | 20260505 18:34:42.681 | INFO | Analyzing chunk frames = [5398528,5399040) | 20260505 18:34:42.681 | INFO | MemInfo: { | 20260505 18:34:42.681 | INFO | "gpu" : " 17 GB", | 20260505 18:34:42.681 | INFO | "huge" : "", | 20260505 18:34:42.681 | INFO | "huge_shared" : " 27 GB", | 20260505 18:34:42.681 | INFO | "malloc" : " 14 GB", | 20260505 18:34:42.681 | INFO | "pinned" : " 29 GB" | 20260505 18:34:42.681 | INFO | } | 20260505 18:34:42.681 | INFO | Perf report for the last 5.1447 seconds: | 20260505 18:34:42.681 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:34:42.681 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:42.681 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:42.681 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:42.681 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 18:34:42.681 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:42.681 | INFO | RT_METRICS : 44.3%, 1.0, 23.3ms, 0 | 20260505 18:34:42.681 | INFO | PRE_HQ : 60.4%, 1.8, 55.9ms, 0 | 20260505 18:34:42.681 | INFO | BAZWRITER : 63.0%, 3.8, 122.3ms, 0 | 20260505 18:34:42.681 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 18:34:42.682 | INFO | Uploaded 36GiB to GPU in 2.15411s (16.7122GiB/s) | 20260505 18:34:42.682 | INFO | Downloaded 38GiB to GPU in 2.24378s (16.9357GiB/s) | 20260505 18:34:47.816 | INFO | Analyzing chunk frames = [5399040,5399552) | 20260505 18:34:47.816 | INFO | MemInfo: { | 20260505 18:34:47.816 | INFO | "gpu" : " 17 GB", | 20260505 18:34:47.816 | INFO | "huge" : "", | 20260505 18:34:47.816 | INFO | "huge_shared" : " 27 GB", | 20260505 18:34:47.816 | INFO | "malloc" : " 14 GB", | 20260505 18:34:47.816 | INFO | "pinned" : " 29 GB" | 20260505 18:34:47.816 | INFO | } | 20260505 18:34:47.816 | INFO | Perf report for the last 5.13494 seconds: | 20260505 18:34:47.816 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:34:47.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:47.816 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:47.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:47.816 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260505 18:34:47.816 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:47.816 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:47.816 | INFO | PRE_HQ : 27.5%, 1.0, 14.2ms, 0 | 20260505 18:34:47.816 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:47.816 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:47.816 | INFO | Uploaded 36GiB to GPU in 2.09779s (17.1609GiB/s) | 20260505 18:34:47.817 | INFO | Downloaded 34GiB to GPU in 2.02373s (16.8007GiB/s) | 20260505 18:34:52.949 | INFO | Analyzing chunk frames = [5399552,5400064) | 20260505 18:34:52.949 | INFO | MemInfo: { | 20260505 18:34:52.949 | INFO | "gpu" : " 17 GB", | 20260505 18:34:52.950 | INFO | "huge" : "", | 20260505 18:34:52.950 | INFO | "huge_shared" : " 27 GB", | 20260505 18:34:52.950 | INFO | "malloc" : " 14 GB", | 20260505 18:34:52.950 | INFO | "pinned" : " 29 GB" | 20260505 18:34:52.950 | INFO | } | 20260505 18:34:52.950 | INFO | Perf report for the last 5.13335 seconds: | 20260505 18:34:52.950 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:34:52.950 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:52.950 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:52.950 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:52.950 | INFO | ANALYSIS : 62.8%, 3.0, 97.6ms, 0 | 20260505 18:34:52.950 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:52.950 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:52.950 | INFO | PRE_HQ : 27.5%, 1.0, 14.2ms, 0 | 20260505 18:34:52.950 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:52.950 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:52.950 | INFO | Uploaded 36GiB to GPU in 2.20938s (16.2941GiB/s) | 20260505 18:34:52.950 | INFO | Downloaded 34GiB to GPU in 2.07391s (16.3941GiB/s) | 20260505 18:34:58.086 | INFO | Analyzing chunk frames = [5400064,5400576) | 20260505 18:34:58.086 | INFO | MemInfo: { | 20260505 18:34:58.086 | INFO | "gpu" : " 17 GB", | 20260505 18:34:58.086 | INFO | "huge" : "", | 20260505 18:34:58.086 | INFO | "huge_shared" : " 27 GB", | 20260505 18:34:58.086 | INFO | "malloc" : " 14 GB", | 20260505 18:34:58.086 | INFO | "pinned" : " 29 GB" | 20260505 18:34:58.086 | INFO | } | 20260505 18:34:58.086 | INFO | Perf report for the last 5.13649 seconds: | 20260505 18:34:58.086 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:34:58.086 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:58.086 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:58.086 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:58.086 | INFO | ANALYSIS : 62.4%, 3.0, 96.9ms, 0 | 20260505 18:34:58.086 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:58.086 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:58.086 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:34:58.086 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:58.086 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:34:58.086 | INFO | Uploaded 36GiB to GPU in 2.18029s (16.5116GiB/s) | 20260505 18:34:58.086 | INFO | Downloaded 34GiB to GPU in 2.13896s (15.8956GiB/s) | 20260505 18:35:03.233 | INFO | Analyzing chunk frames = [5400576,5401088) | 20260505 18:35:03.233 | INFO | MemInfo: { | 20260505 18:35:03.233 | INFO | "gpu" : " 17 GB", | 20260505 18:35:03.233 | INFO | "huge" : "", | 20260505 18:35:03.233 | INFO | "huge_shared" : " 27 GB", | 20260505 18:35:03.233 | INFO | "malloc" : " 14 GB", | 20260505 18:35:03.233 | INFO | "pinned" : " 29 GB" | 20260505 18:35:03.233 | INFO | } | 20260505 18:35:03.233 | INFO | Perf report for the last 5.14688 seconds: | 20260505 18:35:03.233 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:35:03.233 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:03.233 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:03.233 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:03.233 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260505 18:35:03.233 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:03.233 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:03.233 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260505 18:35:03.233 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:03.233 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:03.233 | INFO | Uploaded 36GiB to GPU in 2.15914s (16.6733GiB/s) | 20260505 18:35:03.233 | INFO | Downloaded 34GiB to GPU in 2.06307s (16.4803GiB/s) | 20260505 18:35:08.380 | INFO | Analyzing chunk frames = [5401088,5401600) | 20260505 18:35:08.380 | INFO | MemInfo: { | 20260505 18:35:08.380 | INFO | "gpu" : " 17 GB", | 20260505 18:35:08.380 | INFO | "huge" : "", | 20260505 18:35:08.381 | INFO | "huge_shared" : " 27 GB", | 20260505 18:35:08.381 | INFO | "malloc" : " 14 GB", | 20260505 18:35:08.381 | INFO | "pinned" : " 29 GB" | 20260505 18:35:08.381 | INFO | } | 20260505 18:35:08.381 | INFO | Perf report for the last 5.1476 seconds: | 20260505 18:35:08.381 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:35:08.381 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:08.381 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:08.381 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:08.381 | INFO | ANALYSIS : 62.1%, 3.0, 97.1ms, 0 | 20260505 18:35:08.381 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:08.381 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:08.381 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 18:35:08.381 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:08.381 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:08.381 | INFO | Uploaded 36GiB to GPU in 2.14275s (16.8009GiB/s) | 20260505 18:35:08.381 | INFO | Downloaded 34GiB to GPU in 2.05186s (16.5703GiB/s) | 20260505 18:35:13.514 | INFO | Analyzing chunk frames = [5401600,5402112) | 20260505 18:35:13.514 | INFO | MemInfo: { | 20260505 18:35:13.514 | INFO | "gpu" : " 17 GB", | 20260505 18:35:13.514 | INFO | "huge" : "", | 20260505 18:35:13.514 | INFO | "huge_shared" : " 27 GB", | 20260505 18:35:13.514 | INFO | "malloc" : " 14 GB", | 20260505 18:35:13.514 | INFO | "pinned" : " 29 GB" | 20260505 18:35:13.514 | INFO | } | 20260505 18:35:13.514 | INFO | Perf report for the last 5.13371 seconds: | 20260505 18:35:13.514 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:35:13.514 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:13.514 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:13.514 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:13.514 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 18:35:13.514 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:13.515 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:13.515 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 18:35:13.515 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:13.515 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:13.515 | INFO | Uploaded 36GiB to GPU in 2.12818s (16.9158GiB/s) | 20260505 18:35:13.515 | INFO | Downloaded 34GiB to GPU in 2.14936s (15.8186GiB/s) | 20260505 18:35:18.651 | INFO | Analyzing chunk frames = [5402112,5402624) | 20260505 18:35:18.651 | INFO | MemInfo: { | 20260505 18:35:18.651 | INFO | "gpu" : " 17 GB", | 20260505 18:35:18.651 | INFO | "huge" : "", | 20260505 18:35:18.651 | INFO | "huge_shared" : " 27 GB", | 20260505 18:35:18.651 | INFO | "malloc" : " 14 GB", | 20260505 18:35:18.651 | INFO | "pinned" : " 29 GB" | 20260505 18:35:18.651 | INFO | } | 20260505 18:35:18.651 | INFO | Perf report for the last 5.13704 seconds: | 20260505 18:35:18.651 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:35:18.651 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:18.651 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:18.651 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:18.651 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260505 18:35:18.651 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:18.651 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:18.651 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 18:35:18.651 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:18.652 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:18.652 | INFO | Uploaded 36GiB to GPU in 2.19767s (16.381GiB/s) | 20260505 18:35:18.652 | INFO | Downloaded 34GiB to GPU in 2.02128s (16.821GiB/s) | 20260505 18:35:18.786 | INFO | Starting a round of Baz IO | 20260505 18:35:21.961 | INFO | Populating RTMetrics report | 20260505 18:35:21.962 | INFO | RTMetrics report populated | 20260505 18:35:21.962 | INFO | Opening tmp RTMetrics json file | 20260505 18:35:21.963 | INFO | Serializing RTMetrics to json | 20260505 18:35:21.965 | INFO | Serialization done, swapping tmp file now... | 20260505 18:35:21.966 | INFO | RTMetrics done | 20260505 18:35:22.054 | INFO | Finished a round of Baz IO: Wrote 2.084 GiB in 3.267 seconds | 20260505 18:35:22.054 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.89% / 0.00% / 78.11% | 20260505 18:35:22.054 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.242746% / 7.94729e-06% / 8.74201e-05% | 20260505 18:35:22.054 | INFO | Baz IO overall: | 20260505 18:35:22.054 | INFO | BytesWritten : 3178566163398 | 20260505 18:35:22.054 | INFO | NumEvents : 1505544005397 | 20260505 18:35:23.788 | INFO | Analyzing chunk frames = [5402624,5403136) | 20260505 18:35:23.788 | INFO | MemInfo: { | 20260505 18:35:23.788 | INFO | "gpu" : " 17 GB", | 20260505 18:35:23.788 | INFO | "huge" : "", | 20260505 18:35:23.788 | INFO | "huge_shared" : " 27 GB", | 20260505 18:35:23.788 | INFO | "malloc" : " 14 GB", | 20260505 18:35:23.788 | INFO | "pinned" : " 29 GB" | 20260505 18:35:23.788 | INFO | } | 20260505 18:35:23.788 | INFO | Perf report for the last 5.13701 seconds: | 20260505 18:35:23.788 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:35:23.788 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:23.788 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:23.788 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:23.788 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 18:35:23.788 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:23.788 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260505 18:35:23.788 | INFO | PRE_HQ : 60.3%, 1.8, 55.9ms, 0 | 20260505 18:35:23.789 | INFO | BAZWRITER : 63.2%, 3.8, 123.1ms, 0 | 20260505 18:35:23.789 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 18:35:23.789 | INFO | Uploaded 36GiB to GPU in 2.16271s (16.6457GiB/s) | 20260505 18:35:23.789 | INFO | Downloaded 38GiB to GPU in 2.24591s (16.9196GiB/s) | 20260505 18:35:28.928 | INFO | Analyzing chunk frames = [5403136,5403648) | 20260505 18:35:28.928 | INFO | MemInfo: { | 20260505 18:35:28.928 | INFO | "gpu" : " 17 GB", | 20260505 18:35:28.928 | INFO | "huge" : "", | 20260505 18:35:28.928 | INFO | "huge_shared" : " 27 GB", | 20260505 18:35:28.928 | INFO | "malloc" : " 14 GB", | 20260505 18:35:28.928 | INFO | "pinned" : " 29 GB" | 20260505 18:35:28.928 | INFO | } | 20260505 18:35:28.928 | INFO | Perf report for the last 5.13951 seconds: | 20260505 18:35:28.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:35:28.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:28.928 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:28.928 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:28.928 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 18:35:28.928 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:28.928 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:28.928 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:35:28.928 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:28.928 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:28.928 | INFO | Uploaded 36GiB to GPU in 2.13307s (16.8771GiB/s) | 20260505 18:35:28.928 | INFO | Downloaded 34GiB to GPU in 2.02525s (16.7881GiB/s) | 20260505 18:35:34.074 | INFO | Analyzing chunk frames = [5403648,5404160) | 20260505 18:35:34.074 | INFO | MemInfo: { | 20260505 18:35:34.074 | INFO | "gpu" : " 17 GB", | 20260505 18:35:34.074 | INFO | "huge" : "", | 20260505 18:35:34.075 | INFO | "huge_shared" : " 27 GB", | 20260505 18:35:34.075 | INFO | "malloc" : " 14 GB", | 20260505 18:35:34.075 | INFO | "pinned" : " 29 GB" | 20260505 18:35:34.075 | INFO | } | 20260505 18:35:34.075 | INFO | Perf report for the last 5.1467 seconds: | 20260505 18:35:34.075 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:35:34.075 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:34.075 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:34.075 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:34.075 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260505 18:35:34.075 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:34.075 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:34.075 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 18:35:34.075 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:34.075 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:34.075 | INFO | Uploaded 36GiB to GPU in 2.13023s (16.8996GiB/s) | 20260505 18:35:34.075 | INFO | Downloaded 34GiB to GPU in 2.0489s (16.5943GiB/s) | 20260505 18:35:39.205 | INFO | Analyzing chunk frames = [5404160,5404672) | 20260505 18:35:39.205 | INFO | MemInfo: { | 20260505 18:35:39.205 | INFO | "gpu" : " 17 GB", | 20260505 18:35:39.205 | INFO | "huge" : "", | 20260505 18:35:39.205 | INFO | "huge_shared" : " 27 GB", | 20260505 18:35:39.205 | INFO | "malloc" : " 14 GB", | 20260505 18:35:39.205 | INFO | "pinned" : " 29 GB" | 20260505 18:35:39.205 | INFO | } | 20260505 18:35:39.205 | INFO | Perf report for the last 5.13056 seconds: | 20260505 18:35:39.205 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:35:39.205 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:39.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:39.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:39.205 | INFO | ANALYSIS : 62.4%, 3.0, 96.7ms, 0 | 20260505 18:35:39.205 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:39.205 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:39.205 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:35:39.205 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:39.205 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:39.205 | INFO | Uploaded 36GiB to GPU in 2.1899s (16.4391GiB/s) | 20260505 18:35:39.205 | INFO | Downloaded 34GiB to GPU in 2.03704s (16.6909GiB/s) | 20260505 18:35:44.341 | INFO | Analyzing chunk frames = [5404672,5405184) | 20260505 18:35:44.341 | INFO | MemInfo: { | 20260505 18:35:44.341 | INFO | "gpu" : " 17 GB", | 20260505 18:35:44.341 | INFO | "huge" : "", | 20260505 18:35:44.341 | INFO | "huge_shared" : " 27 GB", | 20260505 18:35:44.341 | INFO | "malloc" : " 14 GB", | 20260505 18:35:44.341 | INFO | "pinned" : " 29 GB" | 20260505 18:35:44.341 | INFO | } | 20260505 18:35:44.341 | INFO | Perf report for the last 5.13621 seconds: | 20260505 18:35:44.341 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:35:44.341 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:44.341 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:44.341 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:44.341 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 18:35:44.341 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:44.341 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:44.341 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:35:44.341 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:44.341 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:44.342 | INFO | Uploaded 36GiB to GPU in 2.14929s (16.7497GiB/s) | 20260505 18:35:44.342 | INFO | Downloaded 34GiB to GPU in 2.05889s (16.5137GiB/s) | 20260505 18:35:49.489 | INFO | Analyzing chunk frames = [5405184,5405696) | 20260505 18:35:49.489 | INFO | MemInfo: { | 20260505 18:35:49.489 | INFO | "gpu" : " 17 GB", | 20260505 18:35:49.489 | INFO | "huge" : "", | 20260505 18:35:49.489 | INFO | "huge_shared" : " 27 GB", | 20260505 18:35:49.489 | INFO | "malloc" : " 14 GB", | 20260505 18:35:49.489 | INFO | "pinned" : " 29 GB" | 20260505 18:35:49.489 | INFO | } | 20260505 18:35:49.489 | INFO | Perf report for the last 5.14725 seconds: | 20260505 18:35:49.489 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:35:49.489 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:49.489 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:49.489 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:49.489 | INFO | ANALYSIS : 62.3%, 3.0, 97.5ms, 0 | 20260505 18:35:49.489 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:49.489 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:49.489 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:35:49.489 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:49.489 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:49.489 | INFO | Uploaded 36GiB to GPU in 2.1437s (16.7934GiB/s) | 20260505 18:35:49.489 | INFO | Downloaded 34GiB to GPU in 2.04715s (16.6084GiB/s) | 20260505 18:35:54.628 | INFO | Analyzing chunk frames = [5405696,5406208) | 20260505 18:35:54.628 | INFO | MemInfo: { | 20260505 18:35:54.628 | INFO | "gpu" : " 17 GB", | 20260505 18:35:54.628 | INFO | "huge" : "", | 20260505 18:35:54.628 | INFO | "huge_shared" : " 27 GB", | 20260505 18:35:54.628 | INFO | "malloc" : " 14 GB", | 20260505 18:35:54.629 | INFO | "pinned" : " 29 GB" | 20260505 18:35:54.629 | INFO | } | 20260505 18:35:54.629 | INFO | Perf report for the last 5.13986 seconds: | 20260505 18:35:54.629 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:35:54.629 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:54.629 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:54.629 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:54.629 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 18:35:54.629 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:54.629 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:54.629 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 18:35:54.629 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:54.629 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:54.629 | INFO | Uploaded 36GiB to GPU in 2.15284s (16.7221GiB/s) | 20260505 18:35:54.629 | INFO | Downloaded 34GiB to GPU in 2.1225s (16.0189GiB/s) | 20260505 18:35:59.763 | INFO | Analyzing chunk frames = [5406208,5406720) | 20260505 18:35:59.763 | INFO | MemInfo: { | 20260505 18:35:59.763 | INFO | "gpu" : " 17 GB", | 20260505 18:35:59.763 | INFO | "huge" : "", | 20260505 18:35:59.763 | INFO | "huge_shared" : " 27 GB", | 20260505 18:35:59.763 | INFO | "malloc" : " 14 GB", | 20260505 18:35:59.763 | INFO | "pinned" : " 29 GB" | 20260505 18:35:59.763 | INFO | } | 20260505 18:35:59.763 | INFO | Perf report for the last 5.1348 seconds: | 20260505 18:35:59.763 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:35:59.763 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:59.763 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:59.763 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:59.763 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260505 18:35:59.763 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:59.763 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:59.763 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 18:35:59.763 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:59.763 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:35:59.764 | INFO | Uploaded 36GiB to GPU in 2.19749s (16.3824GiB/s) | 20260505 18:35:59.764 | INFO | Downloaded 34GiB to GPU in 2.02002s (16.8315GiB/s) | 20260505 18:35:59.764 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 18:35:59.764 | INFO | Total mem usage: 15.3 GiB | 20260505 18:35:59.764 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 18:35:59.764 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.02%) | 20260505 18:35:59.764 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 18:35:59.764 | INFO | High Water Breakdown: | 20260505 18:35:59.764 | INFO | 61% : 8.66 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 18:35:59.764 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 18:35:59.764 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 18:35:59.764 | INFO | ------------------------------------------ | 20260505 18:35:59.764 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:35:59.764 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 18:35:59.764 | INFO | Total mem usage: 26.5 GiB | 20260505 18:35:59.764 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 18:35:59.764 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 18:35:59.764 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 18:35:59.764 | INFO | High Water Breakdown: | 20260505 18:35:59.764 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:35:59.764 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:35:59.764 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:35:59.764 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:35:59.764 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:35:59.764 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:35:59.764 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 18:35:59.764 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 18:35:59.764 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:35:59.764 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 18:35:59.764 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 18:35:59.764 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 18:35:59.764 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 18:35:59.764 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 18:35:59.764 | INFO | ------------------------------------------ | 20260505 18:35:59.764 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:35:59.764 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 18:35:59.764 | INFO | Total mem usage: 25.6 GiB | 20260505 18:35:59.764 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 18:35:59.764 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 18:35:59.764 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 18:35:59.764 | INFO | High Water Breakdown: | 20260505 18:35:59.764 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 18:35:59.764 | INFO | ------------------------------------------ | 20260505 18:35:59.764 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:35:59.764 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 18:35:59.764 | INFO | Total mem usage: 15.7 GiB | 20260505 18:35:59.764 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 18:35:59.764 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 18:35:59.764 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 18:35:59.764 | INFO | High Water Breakdown: | 20260505 18:35:59.764 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 18:35:59.764 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 18:35:59.764 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 18:35:59.764 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 18:35:59.764 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 18:35:59.764 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 18:35:59.764 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 18:35:59.764 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 18:35:59.764 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 18:35:59.764 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:35:59.764 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 18:35:59.764 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:35:59.764 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 18:35:59.764 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 18:35:59.764 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 18:35:59.764 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 18:35:59.764 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:35:59.764 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 18:35:59.764 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:35:59.764 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:35:59.764 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:35:59.764 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 18:35:59.764 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 18:35:59.764 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:35:59.764 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 18:35:59.764 | INFO | ------------------------------------------ | 20260505 18:35:59.764 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:35:59.764 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 18:35:59.764 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:35:59.764 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:35:59.764 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.45e+01, 1.35e+01 / 1.26e+00 | 20260505 18:35:59.764 | INFO | PerfReport: 0.29%, AggregateStats -- 9.26e-02 / 1.03e-01, 1.04e-02 / 3.75e-03 | 20260505 18:35:59.764 | INFO | PerfReport: 11.79%, Baseline -- 3.72e+00 / 4.52e+00, 5.66e-01 / 1.37e-01 | 20260505 18:35:59.764 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.54e+00, 1.93e-01 / 6.47e-02 | 20260505 18:35:59.764 | INFO | PerfReport: 15.90%, DME -- 4.01e+01 / 4.42e+01, 3.99e+00 / 1.23e+00 | 20260505 18:35:59.764 | INFO | PerfReport: 56.95%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.52e+00 / 1.17e+00 | 20260505 18:35:59.764 | INFO | PerfReport: 7.29%, PulseAccumulating -- 2.30e+00 / 2.89e+00, 4.39e-01 / 1.39e-01 | 20260505 18:35:59.764 | INFO | PerfReport: 3.32%, Metrics -- 1.05e+00 / 1.41e+00, 2.89e-01 / 1.37e-01 | 20260505 18:35:59.764 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.64e-01 / 4.30e-01, 6.39e-02 / 3.61e-02 | 20260505 18:35:59.765 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 18:35:59.765 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:35:59.765 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:35:59.765 | INFO | PerfReport: Full Execution -- 9.97e+01 / -nan, 2.56e+01 / 0.00e+00 | 20260505 18:35:59.765 | INFO | PerfReport: 22.54%, Upload -- 2.25e+01 / 0.00e+00, 7.83e+00 / 0.00e+00 | 20260505 18:35:59.765 | INFO | PerfReport: 21.79%, Download -- 2.17e+01 / 0.00e+00, 7.74e+00 / 0.00e+00 | 20260505 18:35:59.765 | INFO | PerfReport: 31.64%, Compute -- 3.16e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 18:35:59.765 | INFO | PerfReport: 2.29%, LockUpload -- 2.28e+00 / 0.00e+00, 8.01e+00 / 0.00e+00 | 20260505 18:35:59.765 | INFO | PerfReport: 1.38%, LockDownload -- 1.38e+00 / 0.00e+00, 2.87e+00 / 0.00e+00 | 20260505 18:35:59.765 | INFO | PerfReport: 20.34%, LockCompute -- 2.03e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 18:35:59.765 | INFO | PerfReport: 0.02%, Misc -- 1.51e-02 / 0.00e+00, 3.21e-02 / 0.00e+00 | 20260505 18:35:59.900 | INFO | Starting a round of Baz IO | 20260505 18:36:03.073 | INFO | Populating RTMetrics report | 20260505 18:36:03.073 | INFO | RTMetrics report populated | 20260505 18:36:03.073 | INFO | Opening tmp RTMetrics json file | 20260505 18:36:03.078 | INFO | Serializing RTMetrics to json | 20260505 18:36:03.079 | INFO | Serialization done, swapping tmp file now... | 20260505 18:36:03.079 | INFO | RTMetrics done | 20260505 18:36:03.167 | INFO | Finished a round of Baz IO: Wrote 2.084 GiB in 3.267 seconds | 20260505 18:36:03.167 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.89% / 0.00% / 78.11% | 20260505 18:36:03.167 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.242869% / 3.97364e-06% / 8.74201e-05% | 20260505 18:36:03.167 | INFO | Baz IO overall: | 20260505 18:36:03.167 | INFO | BytesWritten : 3180803922427 | 20260505 18:36:03.167 | INFO | NumEvents : 1506586524269 | 20260505 18:36:04.897 | INFO | Analyzing chunk frames = [5406720,5407232) | 20260505 18:36:04.897 | INFO | MemInfo: { | 20260505 18:36:04.897 | INFO | "gpu" : " 17 GB", | 20260505 18:36:04.897 | INFO | "huge" : "", | 20260505 18:36:04.897 | INFO | "huge_shared" : " 27 GB", | 20260505 18:36:04.897 | INFO | "malloc" : " 14 GB", | 20260505 18:36:04.897 | INFO | "pinned" : " 29 GB" | 20260505 18:36:04.897 | INFO | } | 20260505 18:36:04.897 | INFO | Perf report for the last 5.13334 seconds: | 20260505 18:36:04.897 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:36:04.897 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:04.897 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:04.897 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:04.897 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 18:36:04.897 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:04.897 | INFO | RT_METRICS : 44.4%, 1.0, 23.2ms, 0 | 20260505 18:36:04.897 | INFO | PRE_HQ : 60.2%, 1.8, 55.7ms, 0 | 20260505 18:36:04.897 | INFO | BAZWRITER : 63.1%, 3.8, 121.8ms, 0 | 20260505 18:36:04.897 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 18:36:04.897 | INFO | Uploaded 36GiB to GPU in 2.15301s (16.7208GiB/s) | 20260505 18:36:04.897 | INFO | Downloaded 38GiB to GPU in 2.23982s (16.9656GiB/s) | 20260505 18:36:10.041 | INFO | Analyzing chunk frames = [5407232,5407744) | 20260505 18:36:10.041 | INFO | MemInfo: { | 20260505 18:36:10.041 | INFO | "gpu" : " 17 GB", | 20260505 18:36:10.041 | INFO | "huge" : "", | 20260505 18:36:10.041 | INFO | "huge_shared" : " 27 GB", | 20260505 18:36:10.041 | INFO | "malloc" : " 14 GB", | 20260505 18:36:10.041 | INFO | "pinned" : " 29 GB" | 20260505 18:36:10.041 | INFO | } | 20260505 18:36:10.041 | INFO | Perf report for the last 5.14423 seconds: | 20260505 18:36:10.041 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:36:10.041 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:10.041 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:10.041 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:10.041 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 18:36:10.041 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:10.041 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:10.041 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:36:10.041 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:10.041 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:10.041 | INFO | Uploaded 36GiB to GPU in 2.13557s (16.8574GiB/s) | 20260505 18:36:10.041 | INFO | Downloaded 34GiB to GPU in 2.03965s (16.6695GiB/s) | 20260505 18:36:15.184 | INFO | Analyzing chunk frames = [5407744,5408256) | 20260505 18:36:15.184 | INFO | MemInfo: { | 20260505 18:36:15.184 | INFO | "gpu" : " 17 GB", | 20260505 18:36:15.184 | INFO | "huge" : "", | 20260505 18:36:15.184 | INFO | "huge_shared" : " 27 GB", | 20260505 18:36:15.185 | INFO | "malloc" : " 14 GB", | 20260505 18:36:15.185 | INFO | "pinned" : " 29 GB" | 20260505 18:36:15.185 | INFO | } | 20260505 18:36:15.185 | INFO | Perf report for the last 5.1436 seconds: | 20260505 18:36:15.185 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:36:15.185 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:15.185 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:15.185 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:15.185 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260505 18:36:15.185 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:15.185 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:15.185 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:36:15.185 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:15.185 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:15.185 | INFO | Uploaded 36GiB to GPU in 2.14256s (16.8024GiB/s) | 20260505 18:36:15.185 | INFO | Downloaded 34GiB to GPU in 2.04495s (16.6263GiB/s) | 20260505 18:36:20.335 | INFO | Analyzing chunk frames = [5408256,5408768) | 20260505 18:36:20.335 | INFO | MemInfo: { | 20260505 18:36:20.335 | INFO | "gpu" : " 17 GB", | 20260505 18:36:20.335 | INFO | "huge" : "", | 20260505 18:36:20.335 | INFO | "huge_shared" : " 27 GB", | 20260505 18:36:20.335 | INFO | "malloc" : " 14 GB", | 20260505 18:36:20.335 | INFO | "pinned" : " 29 GB" | 20260505 18:36:20.335 | INFO | } | 20260505 18:36:20.335 | INFO | Perf report for the last 5.15023 seconds: | 20260505 18:36:20.335 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:36:20.335 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:20.335 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:20.335 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:20.335 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260505 18:36:20.335 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:20.335 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:20.335 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:36:20.335 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:20.335 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:20.335 | INFO | Uploaded 36GiB to GPU in 2.17346s (16.5634GiB/s) | 20260505 18:36:20.335 | INFO | Downloaded 34GiB to GPU in 2.05038s (16.5823GiB/s) | 20260505 18:36:25.465 | INFO | Analyzing chunk frames = [5408768,5409280) | 20260505 18:36:25.465 | INFO | MemInfo: { | 20260505 18:36:25.465 | INFO | "gpu" : " 17 GB", | 20260505 18:36:25.465 | INFO | "huge" : "", | 20260505 18:36:25.465 | INFO | "huge_shared" : " 27 GB", | 20260505 18:36:25.465 | INFO | "malloc" : " 14 GB", | 20260505 18:36:25.465 | INFO | "pinned" : " 29 GB" | 20260505 18:36:25.465 | INFO | } | 20260505 18:36:25.465 | INFO | Perf report for the last 5.13017 seconds: | 20260505 18:36:25.465 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:36:25.465 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:25.465 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:25.465 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:25.465 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260505 18:36:25.465 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:25.465 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:25.465 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:36:25.465 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:25.465 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:25.465 | INFO | Uploaded 36GiB to GPU in 2.1398s (16.824GiB/s) | 20260505 18:36:25.465 | INFO | Downloaded 34GiB to GPU in 2.06347s (16.4771GiB/s) | 20260505 18:36:30.595 | INFO | Analyzing chunk frames = [5409280,5409792) | 20260505 18:36:30.595 | INFO | MemInfo: { | 20260505 18:36:30.595 | INFO | "gpu" : " 17 GB", | 20260505 18:36:30.595 | INFO | "huge" : "", | 20260505 18:36:30.595 | INFO | "huge_shared" : " 27 GB", | 20260505 18:36:30.595 | INFO | "malloc" : " 14 GB", | 20260505 18:36:30.595 | INFO | "pinned" : " 29 GB" | 20260505 18:36:30.595 | INFO | } | 20260505 18:36:30.595 | INFO | Perf report for the last 5.13046 seconds: | 20260505 18:36:30.595 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:36:30.595 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:30.595 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:30.596 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:30.596 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260505 18:36:30.596 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:30.596 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:30.596 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:36:30.596 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:30.596 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:30.596 | INFO | Uploaded 36GiB to GPU in 2.14409s (16.7903GiB/s) | 20260505 18:36:30.596 | INFO | Downloaded 34GiB to GPU in 2.04372s (16.6363GiB/s) | 20260505 18:36:35.746 | INFO | Analyzing chunk frames = [5409792,5410304) | 20260505 18:36:35.746 | INFO | MemInfo: { | 20260505 18:36:35.746 | INFO | "gpu" : " 17 GB", | 20260505 18:36:35.746 | INFO | "huge" : "", | 20260505 18:36:35.746 | INFO | "huge_shared" : " 27 GB", | 20260505 18:36:35.746 | INFO | "malloc" : " 14 GB", | 20260505 18:36:35.746 | INFO | "pinned" : " 29 GB" | 20260505 18:36:35.746 | INFO | } | 20260505 18:36:35.746 | INFO | Perf report for the last 5.15049 seconds: | 20260505 18:36:35.746 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:36:35.746 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:35.746 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:35.746 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:35.746 | INFO | ANALYSIS : 63.4%, 3.0, 99.2ms, 0 | 20260505 18:36:35.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:35.746 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:35.746 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 18:36:35.746 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:35.746 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:35.746 | INFO | Uploaded 36GiB to GPU in 2.16355s (16.6393GiB/s) | 20260505 18:36:35.746 | INFO | Downloaded 34GiB to GPU in 2.16746s (15.6866GiB/s) | 20260505 18:36:40.882 | INFO | Analyzing chunk frames = [5410304,5410816) | 20260505 18:36:40.882 | INFO | MemInfo: { | 20260505 18:36:40.882 | INFO | "gpu" : " 17 GB", | 20260505 18:36:40.882 | INFO | "huge" : "", | 20260505 18:36:40.882 | INFO | "huge_shared" : " 27 GB", | 20260505 18:36:40.882 | INFO | "malloc" : " 14 GB", | 20260505 18:36:40.882 | INFO | "pinned" : " 29 GB" | 20260505 18:36:40.882 | INFO | } | 20260505 18:36:40.882 | INFO | Perf report for the last 5.13583 seconds: | 20260505 18:36:40.882 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:36:40.882 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:40.882 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:40.882 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:40.882 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 18:36:40.882 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:40.882 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:40.882 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 18:36:40.882 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:40.882 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:40.882 | INFO | Uploaded 36GiB to GPU in 2.2006s (16.3592GiB/s) | 20260505 18:36:40.882 | INFO | Downloaded 34GiB to GPU in 2.04335s (16.6394GiB/s) | 20260505 18:36:41.014 | INFO | Starting a round of Baz IO | 20260505 18:36:44.195 | INFO | Populating RTMetrics report | 20260505 18:36:44.195 | INFO | RTMetrics report populated | 20260505 18:36:44.195 | INFO | Opening tmp RTMetrics json file | 20260505 18:36:44.199 | INFO | Serializing RTMetrics to json | 20260505 18:36:44.201 | INFO | Serialization done, swapping tmp file now... | 20260505 18:36:44.202 | INFO | RTMetrics done | 20260505 18:36:44.286 | INFO | Finished a round of Baz IO: Wrote 2.083 GiB in 3.272 seconds | 20260505 18:36:44.286 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.88% / 0.00% / 78.11% | 20260505 18:36:44.286 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.24182% / 7.94729e-06% / 7.15256e-05% | 20260505 18:36:44.286 | INFO | Baz IO overall: | 20260505 18:36:44.286 | INFO | BytesWritten : 3183040023982 | 20260505 18:36:44.286 | INFO | NumEvents : 1507627632160 | 20260505 18:36:46.021 | INFO | Analyzing chunk frames = [5410816,5411328) | 20260505 18:36:46.021 | INFO | MemInfo: { | 20260505 18:36:46.021 | INFO | "gpu" : " 17 GB", | 20260505 18:36:46.021 | INFO | "huge" : "", | 20260505 18:36:46.021 | INFO | "huge_shared" : " 27 GB", | 20260505 18:36:46.021 | INFO | "malloc" : " 14 GB", | 20260505 18:36:46.021 | INFO | "pinned" : " 29 GB" | 20260505 18:36:46.021 | INFO | } | 20260505 18:36:46.021 | INFO | Perf report for the last 5.13958 seconds: | 20260505 18:36:46.021 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:36:46.021 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:46.021 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:46.021 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:46.021 | INFO | ANALYSIS : 63.4%, 3.0, 98.8ms, 0 | 20260505 18:36:46.022 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:46.022 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260505 18:36:46.022 | INFO | PRE_HQ : 60.4%, 1.8, 56.1ms, 0 | 20260505 18:36:46.022 | INFO | BAZWRITER : 63.2%, 3.8, 123.2ms, 0 | 20260505 18:36:46.022 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 18:36:46.022 | INFO | Uploaded 36GiB to GPU in 2.17986s (16.5148GiB/s) | 20260505 18:36:46.022 | INFO | Downloaded 38GiB to GPU in 2.25434s (16.8564GiB/s) | 20260505 18:36:51.160 | INFO | Analyzing chunk frames = [5411328,5411840) | 20260505 18:36:51.160 | INFO | MemInfo: { | 20260505 18:36:51.160 | INFO | "gpu" : " 17 GB", | 20260505 18:36:51.160 | INFO | "huge" : "", | 20260505 18:36:51.160 | INFO | "huge_shared" : " 27 GB", | 20260505 18:36:51.160 | INFO | "malloc" : " 14 GB", | 20260505 18:36:51.160 | INFO | "pinned" : " 29 GB" | 20260505 18:36:51.160 | INFO | } | 20260505 18:36:51.160 | INFO | Perf report for the last 5.13848 seconds: | 20260505 18:36:51.160 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:36:51.160 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:51.160 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:51.160 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:51.160 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 18:36:51.160 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:51.160 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:51.160 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:36:51.160 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:51.160 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:51.161 | INFO | Uploaded 36GiB to GPU in 2.1379s (16.839GiB/s) | 20260505 18:36:51.161 | INFO | Downloaded 34GiB to GPU in 2.03007s (16.7482GiB/s) | 20260505 18:36:56.296 | INFO | Analyzing chunk frames = [5411840,5412352) | 20260505 18:36:56.296 | INFO | MemInfo: { | 20260505 18:36:56.296 | INFO | "gpu" : " 17 GB", | 20260505 18:36:56.296 | INFO | "huge" : "", | 20260505 18:36:56.296 | INFO | "huge_shared" : " 27 GB", | 20260505 18:36:56.296 | INFO | "malloc" : " 14 GB", | 20260505 18:36:56.296 | INFO | "pinned" : " 29 GB" | 20260505 18:36:56.296 | INFO | } | 20260505 18:36:56.296 | INFO | Perf report for the last 5.13607 seconds: | 20260505 18:36:56.296 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:36:56.296 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:56.296 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:56.296 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:56.296 | INFO | ANALYSIS : 62.7%, 3.0, 97.5ms, 0 | 20260505 18:36:56.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:56.296 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:56.296 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 18:36:56.296 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:56.296 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:36:56.296 | INFO | Uploaded 36GiB to GPU in 2.209s (16.297GiB/s) | 20260505 18:36:56.296 | INFO | Downloaded 34GiB to GPU in 2.06638s (16.4539GiB/s) | 20260505 18:37:01.439 | INFO | Analyzing chunk frames = [5412352,5412864) | 20260505 18:37:01.439 | INFO | MemInfo: { | 20260505 18:37:01.439 | INFO | "gpu" : " 17 GB", | 20260505 18:37:01.439 | INFO | "huge" : "", | 20260505 18:37:01.439 | INFO | "huge_shared" : " 27 GB", | 20260505 18:37:01.439 | INFO | "malloc" : " 14 GB", | 20260505 18:37:01.439 | INFO | "pinned" : " 29 GB" | 20260505 18:37:01.439 | INFO | } | 20260505 18:37:01.439 | INFO | Perf report for the last 5.14286 seconds: | 20260505 18:37:01.439 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:37:01.439 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:01.439 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:01.439 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:01.439 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260505 18:37:01.439 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:01.439 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:01.439 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 18:37:01.439 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:01.439 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:01.439 | INFO | Uploaded 36GiB to GPU in 2.2284s (16.1551GiB/s) | 20260505 18:37:01.439 | INFO | Downloaded 34GiB to GPU in 2.12281s (16.0165GiB/s) | 20260505 18:37:06.577 | INFO | Analyzing chunk frames = [5412864,5413376) | 20260505 18:37:06.577 | INFO | MemInfo: { | 20260505 18:37:06.577 | INFO | "gpu" : " 17 GB", | 20260505 18:37:06.577 | INFO | "huge" : "", | 20260505 18:37:06.577 | INFO | "huge_shared" : " 27 GB", | 20260505 18:37:06.577 | INFO | "malloc" : " 14 GB", | 20260505 18:37:06.577 | INFO | "pinned" : " 29 GB" | 20260505 18:37:06.577 | INFO | } | 20260505 18:37:06.577 | INFO | Perf report for the last 5.13847 seconds: | 20260505 18:37:06.577 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:37:06.577 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:06.577 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:06.577 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:06.577 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 18:37:06.577 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:06.577 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:06.577 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 18:37:06.577 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:06.577 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:06.578 | INFO | Uploaded 36GiB to GPU in 2.13879s (16.8319GiB/s) | 20260505 18:37:06.578 | INFO | Downloaded 34GiB to GPU in 2.05589s (16.5379GiB/s) | 20260505 18:37:11.712 | INFO | Analyzing chunk frames = [5413376,5413888) | 20260505 18:37:11.712 | INFO | MemInfo: { | 20260505 18:37:11.712 | INFO | "gpu" : " 17 GB", | 20260505 18:37:11.712 | INFO | "huge" : "", | 20260505 18:37:11.712 | INFO | "huge_shared" : " 27 GB", | 20260505 18:37:11.712 | INFO | "malloc" : " 14 GB", | 20260505 18:37:11.712 | INFO | "pinned" : " 29 GB" | 20260505 18:37:11.712 | INFO | } | 20260505 18:37:11.712 | INFO | Perf report for the last 5.13466 seconds: | 20260505 18:37:11.712 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:37:11.712 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:11.712 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:11.712 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:11.712 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 18:37:11.712 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:11.712 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:11.712 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 18:37:11.712 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:11.712 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:11.712 | INFO | Uploaded 36GiB to GPU in 2.1338s (16.8713GiB/s) | 20260505 18:37:11.712 | INFO | Downloaded 34GiB to GPU in 2.05131s (16.5748GiB/s) | 20260505 18:37:16.845 | INFO | Analyzing chunk frames = [5413888,5414400) | 20260505 18:37:16.845 | INFO | MemInfo: { | 20260505 18:37:16.845 | INFO | "gpu" : " 17 GB", | 20260505 18:37:16.845 | INFO | "huge" : "", | 20260505 18:37:16.845 | INFO | "huge_shared" : " 27 GB", | 20260505 18:37:16.845 | INFO | "malloc" : " 14 GB", | 20260505 18:37:16.845 | INFO | "pinned" : " 29 GB" | 20260505 18:37:16.845 | INFO | } | 20260505 18:37:16.845 | INFO | Perf report for the last 5.13332 seconds: | 20260505 18:37:16.845 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:37:16.845 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:16.845 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:16.845 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:16.845 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 18:37:16.845 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:16.845 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:16.845 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260505 18:37:16.845 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:16.845 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:16.846 | INFO | Uploaded 36GiB to GPU in 2.13354s (16.8733GiB/s) | 20260505 18:37:16.846 | INFO | Downloaded 34GiB to GPU in 2.17759s (15.6136GiB/s) | 20260505 18:37:21.993 | INFO | Analyzing chunk frames = [5414400,5414912) | 20260505 18:37:21.993 | INFO | MemInfo: { | 20260505 18:37:21.993 | INFO | "gpu" : " 17 GB", | 20260505 18:37:21.993 | INFO | "huge" : "", | 20260505 18:37:21.993 | INFO | "huge_shared" : " 27 GB", | 20260505 18:37:21.993 | INFO | "malloc" : " 14 GB", | 20260505 18:37:21.993 | INFO | "pinned" : " 29 GB" | 20260505 18:37:21.993 | INFO | } | 20260505 18:37:21.993 | INFO | Perf report for the last 5.14736 seconds: | 20260505 18:37:21.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:37:21.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:21.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:21.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:21.993 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 18:37:21.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:21.993 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:21.993 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 18:37:21.993 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:21.993 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:21.993 | INFO | Uploaded 36GiB to GPU in 2.15852s (16.6781GiB/s) | 20260505 18:37:21.993 | INFO | Downloaded 34GiB to GPU in 2.03753s (16.6869GiB/s) | 20260505 18:37:21.993 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 18:37:21.993 | INFO | Total mem usage: 15.3 GiB | 20260505 18:37:21.993 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 18:37:21.993 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.02%) | 20260505 18:37:21.993 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 18:37:21.993 | INFO | High Water Breakdown: | 20260505 18:37:21.993 | INFO | 61% : 8.66 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 18:37:21.993 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 18:37:21.993 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 18:37:21.993 | INFO | ------------------------------------------ | 20260505 18:37:21.993 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:37:21.993 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 18:37:21.993 | INFO | Total mem usage: 26.5 GiB | 20260505 18:37:21.993 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 18:37:21.993 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 18:37:21.993 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 18:37:21.993 | INFO | High Water Breakdown: | 20260505 18:37:21.993 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:37:21.993 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:37:21.993 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:37:21.993 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:37:21.993 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:37:21.993 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:37:21.993 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 18:37:21.993 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 18:37:21.993 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:37:21.993 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 18:37:21.993 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 18:37:21.993 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 18:37:21.993 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 18:37:21.993 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 18:37:21.993 | INFO | ------------------------------------------ | 20260505 18:37:21.993 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:37:21.993 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 18:37:21.993 | INFO | Total mem usage: 25.6 GiB | 20260505 18:37:21.993 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 18:37:21.993 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 18:37:21.993 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 18:37:21.993 | INFO | High Water Breakdown: | 20260505 18:37:21.993 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 18:37:21.993 | INFO | ------------------------------------------ | 20260505 18:37:21.993 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:37:21.993 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 18:37:21.993 | INFO | Total mem usage: 15.7 GiB | 20260505 18:37:21.993 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 18:37:21.993 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 18:37:21.993 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 18:37:21.993 | INFO | High Water Breakdown: | 20260505 18:37:21.993 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 18:37:21.994 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 18:37:21.994 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 18:37:21.994 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 18:37:21.994 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 18:37:21.994 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 18:37:21.994 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 18:37:21.994 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 18:37:21.994 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 18:37:21.994 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:37:21.994 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 18:37:21.994 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:37:21.994 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 18:37:21.994 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 18:37:21.994 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 18:37:21.994 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 18:37:21.994 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:37:21.994 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 18:37:21.994 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:37:21.994 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:37:21.994 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:37:21.994 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 18:37:21.994 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 18:37:21.994 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:37:21.994 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 18:37:21.994 | INFO | ------------------------------------------ | 20260505 18:37:21.994 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:37:21.994 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 18:37:21.994 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:37:21.994 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:37:21.994 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.49e+01, 1.35e+01 / 2.21e+00 | 20260505 18:37:21.994 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.02e-01, 1.05e-02 / 3.41e-03 | 20260505 18:37:21.994 | INFO | PerfReport: 11.77%, Baseline -- 3.72e+00 / 4.49e+00, 5.67e-01 / 1.84e-01 | 20260505 18:37:21.994 | INFO | PerfReport: 3.26%, Histogram -- 1.03e+00 / 1.51e+00, 1.89e-01 / 9.58e-02 | 20260505 18:37:21.994 | INFO | PerfReport: 15.97%, DME -- 4.03e+01 / 4.47e+01, 4.38e+00 / 2.36e+00 | 20260505 18:37:21.994 | INFO | PerfReport: 56.94%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.48e+00 / 1.07e+00 | 20260505 18:37:21.994 | INFO | PerfReport: 7.29%, PulseAccumulating -- 2.30e+00 / 2.89e+00, 4.45e-01 / 1.84e-01 | 20260505 18:37:21.994 | INFO | PerfReport: 3.32%, Metrics -- 1.05e+00 / 1.38e+00, 2.91e-01 / 1.56e-01 | 20260505 18:37:21.994 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.65e-01 / 4.36e-01, 6.56e-02 / 4.68e-02 | 20260505 18:37:21.994 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 18:37:21.994 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:37:21.994 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:37:21.994 | INFO | PerfReport: Full Execution -- 9.98e+01 / -nan, 2.56e+01 / 0.00e+00 | 20260505 18:37:21.994 | INFO | PerfReport: 22.55%, Upload -- 2.25e+01 / 0.00e+00, 7.86e+00 / 0.00e+00 | 20260505 18:37:21.994 | INFO | PerfReport: 21.85%, Download -- 2.18e+01 / 0.00e+00, 7.81e+00 / 0.00e+00 | 20260505 18:37:21.994 | INFO | PerfReport: 31.63%, Compute -- 3.16e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 18:37:21.994 | INFO | PerfReport: 2.30%, LockUpload -- 2.29e+00 / 0.00e+00, 7.87e+00 / 0.00e+00 | 20260505 18:37:21.994 | INFO | PerfReport: 1.37%, LockDownload -- 1.37e+00 / 0.00e+00, 2.90e+00 / 0.00e+00 | 20260505 18:37:21.994 | INFO | PerfReport: 20.29%, LockCompute -- 2.03e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 18:37:21.994 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.92e-02 / 0.00e+00 | 20260505 18:37:22.128 | INFO | Starting a round of Baz IO | 20260505 18:37:25.308 | INFO | Populating RTMetrics report | 20260505 18:37:25.308 | INFO | RTMetrics report populated | 20260505 18:37:25.308 | INFO | Opening tmp RTMetrics json file | 20260505 18:37:25.312 | INFO | Serializing RTMetrics to json | 20260505 18:37:25.314 | INFO | Serialization done, swapping tmp file now... | 20260505 18:37:25.317 | INFO | RTMetrics done | 20260505 18:37:25.400 | INFO | Finished a round of Baz IO: Wrote 2.084 GiB in 3.272 seconds | 20260505 18:37:25.400 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.88% / 0.00% / 78.12% | 20260505 18:37:25.400 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.241617% / 3.97364e-06% / 9.53674e-05% | 20260505 18:37:25.400 | INFO | Baz IO overall: | 20260505 18:37:25.400 | INFO | BytesWritten : 3185277558435 | 20260505 18:37:25.400 | INFO | NumEvents : 1508669913055 | 20260505 18:37:27.141 | INFO | Analyzing chunk frames = [5414912,5415424) | 20260505 18:37:27.141 | INFO | MemInfo: { | 20260505 18:37:27.141 | INFO | "gpu" : " 17 GB", | 20260505 18:37:27.141 | INFO | "huge" : "", | 20260505 18:37:27.141 | INFO | "huge_shared" : " 27 GB", | 20260505 18:37:27.141 | INFO | "malloc" : " 14 GB", | 20260505 18:37:27.141 | INFO | "pinned" : " 29 GB" | 20260505 18:37:27.141 | INFO | } | 20260505 18:37:27.142 | INFO | Perf report for the last 5.14878 seconds: | 20260505 18:37:27.142 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:37:27.142 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:27.142 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:27.142 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:27.142 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 18:37:27.142 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:27.142 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260505 18:37:27.142 | INFO | PRE_HQ : 60.2%, 1.8, 55.7ms, 0 | 20260505 18:37:27.142 | INFO | BAZWRITER : 63.1%, 3.8, 122.0ms, 0 | 20260505 18:37:27.142 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 18:37:27.142 | INFO | Uploaded 36GiB to GPU in 2.1542s (16.7116GiB/s) | 20260505 18:37:27.142 | INFO | Downloaded 38GiB to GPU in 2.23818s (16.9781GiB/s) | 20260505 18:37:32.276 | INFO | Analyzing chunk frames = [5415424,5415936) | 20260505 18:37:32.276 | INFO | MemInfo: { | 20260505 18:37:32.276 | INFO | "gpu" : " 17 GB", | 20260505 18:37:32.276 | INFO | "huge" : "", | 20260505 18:37:32.276 | INFO | "huge_shared" : " 27 GB", | 20260505 18:37:32.276 | INFO | "malloc" : " 14 GB", | 20260505 18:37:32.276 | INFO | "pinned" : " 29 GB" | 20260505 18:37:32.276 | INFO | } | 20260505 18:37:32.276 | INFO | Perf report for the last 5.13469 seconds: | 20260505 18:37:32.276 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:37:32.276 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:32.276 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:32.276 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:32.276 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260505 18:37:32.276 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:32.276 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:32.276 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:37:32.276 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:32.276 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:32.277 | INFO | Uploaded 36GiB to GPU in 2.08766s (17.2442GiB/s) | 20260505 18:37:32.277 | INFO | Downloaded 34GiB to GPU in 2.02347s (16.8029GiB/s) | 20260505 18:37:37.406 | INFO | Analyzing chunk frames = [5415936,5416448) | 20260505 18:37:37.406 | INFO | MemInfo: { | 20260505 18:37:37.406 | INFO | "gpu" : " 17 GB", | 20260505 18:37:37.406 | INFO | "huge" : "", | 20260505 18:37:37.406 | INFO | "huge_shared" : " 27 GB", | 20260505 18:37:37.406 | INFO | "malloc" : " 14 GB", | 20260505 18:37:37.406 | INFO | "pinned" : " 29 GB" | 20260505 18:37:37.406 | INFO | } | 20260505 18:37:37.406 | INFO | Perf report for the last 5.12951 seconds: | 20260505 18:37:37.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:37:37.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:37.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:37.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:37.406 | INFO | ANALYSIS : 62.8%, 3.0, 97.5ms, 0 | 20260505 18:37:37.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:37.406 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:37.406 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:37:37.406 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:37.406 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:37.406 | INFO | Uploaded 36GiB to GPU in 2.20564s (16.3218GiB/s) | 20260505 18:37:37.406 | INFO | Downloaded 34GiB to GPU in 2.0574s (16.5257GiB/s) | 20260505 18:37:42.550 | INFO | Analyzing chunk frames = [5416448,5416960) | 20260505 18:37:42.550 | INFO | MemInfo: { | 20260505 18:37:42.550 | INFO | "gpu" : " 17 GB", | 20260505 18:37:42.550 | INFO | "huge" : "", | 20260505 18:37:42.550 | INFO | "huge_shared" : " 27 GB", | 20260505 18:37:42.550 | INFO | "malloc" : " 14 GB", | 20260505 18:37:42.550 | INFO | "pinned" : " 29 GB" | 20260505 18:37:42.550 | INFO | } | 20260505 18:37:42.550 | INFO | Perf report for the last 5.1443 seconds: | 20260505 18:37:42.550 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:37:42.550 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:42.550 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:42.550 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:42.550 | INFO | ANALYSIS : 62.4%, 3.0, 97.0ms, 0 | 20260505 18:37:42.550 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:42.550 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:42.550 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:37:42.550 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:42.550 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:42.550 | INFO | Uploaded 36GiB to GPU in 2.17511s (16.5509GiB/s) | 20260505 18:37:42.550 | INFO | Downloaded 34GiB to GPU in 2.14416s (15.8571GiB/s) | 20260505 18:37:47.697 | INFO | Analyzing chunk frames = [5416960,5417472) | 20260505 18:37:47.697 | INFO | MemInfo: { | 20260505 18:37:47.697 | INFO | "gpu" : " 17 GB", | 20260505 18:37:47.697 | INFO | "huge" : "", | 20260505 18:37:47.697 | INFO | "huge_shared" : " 27 GB", | 20260505 18:37:47.697 | INFO | "malloc" : " 14 GB", | 20260505 18:37:47.697 | INFO | "pinned" : " 29 GB" | 20260505 18:37:47.697 | INFO | } | 20260505 18:37:47.697 | INFO | Perf report for the last 5.14707 seconds: | 20260505 18:37:47.697 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:37:47.697 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:47.697 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:47.697 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:47.697 | INFO | ANALYSIS : 61.9%, 3.0, 96.8ms, 0 | 20260505 18:37:47.697 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:47.697 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:47.697 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:37:47.697 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:47.697 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:47.697 | INFO | Uploaded 36GiB to GPU in 2.14041s (16.8192GiB/s) | 20260505 18:37:47.697 | INFO | Downloaded 34GiB to GPU in 2.0612s (16.4953GiB/s) | 20260505 18:37:52.836 | INFO | Analyzing chunk frames = [5417472,5417984) | 20260505 18:37:52.836 | INFO | MemInfo: { | 20260505 18:37:52.836 | INFO | "gpu" : " 17 GB", | 20260505 18:37:52.836 | INFO | "huge" : "", | 20260505 18:37:52.836 | INFO | "huge_shared" : " 27 GB", | 20260505 18:37:52.836 | INFO | "malloc" : " 14 GB", | 20260505 18:37:52.836 | INFO | "pinned" : " 29 GB" | 20260505 18:37:52.836 | INFO | } | 20260505 18:37:52.836 | INFO | Perf report for the last 5.13923 seconds: | 20260505 18:37:52.836 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:37:52.836 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:52.836 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:52.836 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:52.836 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 18:37:52.836 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:52.836 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:52.836 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 18:37:52.837 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:52.837 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:52.837 | INFO | Uploaded 36GiB to GPU in 2.15042s (16.7409GiB/s) | 20260505 18:37:52.837 | INFO | Downloaded 34GiB to GPU in 2.05756s (16.5244GiB/s) | 20260505 18:37:57.972 | INFO | Analyzing chunk frames = [5417984,5418496) | 20260505 18:37:57.972 | INFO | MemInfo: { | 20260505 18:37:57.972 | INFO | "gpu" : " 17 GB", | 20260505 18:37:57.972 | INFO | "huge" : "", | 20260505 18:37:57.972 | INFO | "huge_shared" : " 27 GB", | 20260505 18:37:57.972 | INFO | "malloc" : " 14 GB", | 20260505 18:37:57.972 | INFO | "pinned" : " 29 GB" | 20260505 18:37:57.972 | INFO | } | 20260505 18:37:57.972 | INFO | Perf report for the last 5.13569 seconds: | 20260505 18:37:57.972 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:37:57.972 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:57.972 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:57.972 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:57.972 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 18:37:57.972 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:57.972 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:57.972 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 18:37:57.972 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:57.972 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:37:57.972 | INFO | Uploaded 36GiB to GPU in 2.14427s (16.7889GiB/s) | 20260505 18:37:57.972 | INFO | Downloaded 34GiB to GPU in 2.16335s (15.7164GiB/s) | 20260505 18:38:03.107 | INFO | Analyzing chunk frames = [5418496,5419008) | 20260505 18:38:03.107 | INFO | MemInfo: { | 20260505 18:38:03.107 | INFO | "gpu" : " 17 GB", | 20260505 18:38:03.107 | INFO | "huge" : "", | 20260505 18:38:03.107 | INFO | "huge_shared" : " 27 GB", | 20260505 18:38:03.107 | INFO | "malloc" : " 14 GB", | 20260505 18:38:03.107 | INFO | "pinned" : " 29 GB" | 20260505 18:38:03.107 | INFO | } | 20260505 18:38:03.107 | INFO | Perf report for the last 5.13498 seconds: | 20260505 18:38:03.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:38:03.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:03.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:03.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:03.107 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 18:38:03.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:03.107 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:03.107 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 18:38:03.107 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:03.107 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:03.107 | INFO | Uploaded 36GiB to GPU in 2.19767s (16.381GiB/s) | 20260505 18:38:03.107 | INFO | Downloaded 34GiB to GPU in 2.03532s (16.705GiB/s) | 20260505 18:38:03.243 | INFO | Starting a round of Baz IO | 20260505 18:38:06.422 | INFO | Populating RTMetrics report | 20260505 18:38:06.422 | INFO | RTMetrics report populated | 20260505 18:38:06.422 | INFO | Opening tmp RTMetrics json file | 20260505 18:38:06.424 | INFO | Serializing RTMetrics to json | 20260505 18:38:06.427 | INFO | Serialization done, swapping tmp file now... | 20260505 18:38:06.427 | INFO | RTMetrics done | 20260505 18:38:06.512 | INFO | Finished a round of Baz IO: Wrote 2.081 GiB in 3.268 seconds | 20260505 18:38:06.512 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.88% / 0.00% / 78.12% | 20260505 18:38:06.512 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.24122% / 3.97364e-06% / 8.34465e-05% | 20260505 18:38:06.512 | INFO | Baz IO overall: | 20260505 18:38:06.512 | INFO | BytesWritten : 3187512106975 | 20260505 18:38:06.512 | INFO | NumEvents : 1509710102544 | 20260505 18:38:08.249 | INFO | Analyzing chunk frames = [5419008,5419520) | 20260505 18:38:08.249 | INFO | MemInfo: { | 20260505 18:38:08.249 | INFO | "gpu" : " 17 GB", | 20260505 18:38:08.249 | INFO | "huge" : "", | 20260505 18:38:08.249 | INFO | "huge_shared" : " 27 GB", | 20260505 18:38:08.249 | INFO | "malloc" : " 14 GB", | 20260505 18:38:08.249 | INFO | "pinned" : " 29 GB" | 20260505 18:38:08.249 | INFO | } | 20260505 18:38:08.249 | INFO | Perf report for the last 5.14225 seconds: | 20260505 18:38:08.249 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:38:08.249 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:08.249 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:08.249 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:08.249 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 18:38:08.249 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:08.249 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260505 18:38:08.249 | INFO | PRE_HQ : 60.2%, 1.8, 55.7ms, 0 | 20260505 18:38:08.249 | INFO | BAZWRITER : 63.0%, 3.8, 122.0ms, 0 | 20260505 18:38:08.249 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 18:38:08.250 | INFO | Uploaded 36GiB to GPU in 2.16448s (16.6322GiB/s) | 20260505 18:38:08.250 | INFO | Downloaded 38GiB to GPU in 2.26232s (16.7969GiB/s) | 20260505 18:38:13.392 | INFO | Analyzing chunk frames = [5419520,5420032) | 20260505 18:38:13.392 | INFO | MemInfo: { | 20260505 18:38:13.392 | INFO | "gpu" : " 17 GB", | 20260505 18:38:13.392 | INFO | "huge" : "", | 20260505 18:38:13.392 | INFO | "huge_shared" : " 27 GB", | 20260505 18:38:13.392 | INFO | "malloc" : " 14 GB", | 20260505 18:38:13.392 | INFO | "pinned" : " 29 GB" | 20260505 18:38:13.392 | INFO | } | 20260505 18:38:13.392 | INFO | Perf report for the last 5.14261 seconds: | 20260505 18:38:13.392 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:38:13.392 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:13.392 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:13.392 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:13.392 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260505 18:38:13.392 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:13.392 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:13.392 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 18:38:13.392 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:13.392 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:13.392 | INFO | Uploaded 36GiB to GPU in 2.14181s (16.8082GiB/s) | 20260505 18:38:13.392 | INFO | Downloaded 34GiB to GPU in 2.04126s (16.6564GiB/s) | 20260505 18:38:18.525 | INFO | Analyzing chunk frames = [5420032,5420544) | 20260505 18:38:18.525 | INFO | MemInfo: { | 20260505 18:38:18.525 | INFO | "gpu" : " 17 GB", | 20260505 18:38:18.525 | INFO | "huge" : "", | 20260505 18:38:18.525 | INFO | "huge_shared" : " 27 GB", | 20260505 18:38:18.525 | INFO | "malloc" : " 14 GB", | 20260505 18:38:18.525 | INFO | "pinned" : " 29 GB" | 20260505 18:38:18.525 | INFO | } | 20260505 18:38:18.525 | INFO | Perf report for the last 5.13292 seconds: | 20260505 18:38:18.525 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:38:18.525 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:18.525 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:18.525 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:18.525 | INFO | ANALYSIS : 62.4%, 3.0, 96.9ms, 0 | 20260505 18:38:18.525 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:18.525 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:18.525 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:38:18.525 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:18.525 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:18.525 | INFO | Uploaded 36GiB to GPU in 2.12422s (16.9474GiB/s) | 20260505 18:38:18.525 | INFO | Downloaded 34GiB to GPU in 2.03882s (16.6763GiB/s) | 20260505 18:38:23.664 | INFO | Analyzing chunk frames = [5420544,5421056) | 20260505 18:38:23.665 | INFO | MemInfo: { | 20260505 18:38:23.665 | INFO | "gpu" : " 17 GB", | 20260505 18:38:23.665 | INFO | "huge" : "", | 20260505 18:38:23.665 | INFO | "huge_shared" : " 27 GB", | 20260505 18:38:23.665 | INFO | "malloc" : " 14 GB", | 20260505 18:38:23.665 | INFO | "pinned" : " 29 GB" | 20260505 18:38:23.665 | INFO | } | 20260505 18:38:23.665 | INFO | Perf report for the last 5.13972 seconds: | 20260505 18:38:23.665 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:38:23.665 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:23.665 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:23.665 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:23.665 | INFO | ANALYSIS : 62.6%, 3.0, 97.3ms, 0 | 20260505 18:38:23.665 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:23.665 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:23.665 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:38:23.665 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:23.665 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:23.665 | INFO | Uploaded 36GiB to GPU in 2.22219s (16.2002GiB/s) | 20260505 18:38:23.665 | INFO | Downloaded 34GiB to GPU in 2.13124s (15.9531GiB/s) | 20260505 18:38:28.801 | INFO | Analyzing chunk frames = [5421056,5421568) | 20260505 18:38:28.801 | INFO | MemInfo: { | 20260505 18:38:28.801 | INFO | "gpu" : " 17 GB", | 20260505 18:38:28.801 | INFO | "huge" : "", | 20260505 18:38:28.801 | INFO | "huge_shared" : " 27 GB", | 20260505 18:38:28.801 | INFO | "malloc" : " 14 GB", | 20260505 18:38:28.801 | INFO | "pinned" : " 29 GB" | 20260505 18:38:28.801 | INFO | } | 20260505 18:38:28.801 | INFO | Perf report for the last 5.13657 seconds: | 20260505 18:38:28.801 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:38:28.801 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:28.801 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:28.801 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:28.801 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 18:38:28.801 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:28.801 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:28.801 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 18:38:28.801 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:28.801 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:28.802 | INFO | Uploaded 36GiB to GPU in 2.16064s (16.6618GiB/s) | 20260505 18:38:28.802 | INFO | Downloaded 34GiB to GPU in 2.05836s (16.518GiB/s) | 20260505 18:38:33.945 | INFO | Analyzing chunk frames = [5421568,5422080) | 20260505 18:38:33.945 | INFO | MemInfo: { | 20260505 18:38:33.945 | INFO | "gpu" : " 17 GB", | 20260505 18:38:33.945 | INFO | "huge" : "", | 20260505 18:38:33.945 | INFO | "huge_shared" : " 27 GB", | 20260505 18:38:33.945 | INFO | "malloc" : " 14 GB", | 20260505 18:38:33.945 | INFO | "pinned" : " 29 GB" | 20260505 18:38:33.945 | INFO | } | 20260505 18:38:33.946 | INFO | Perf report for the last 5.14429 seconds: | 20260505 18:38:33.946 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:38:33.946 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:33.946 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:33.946 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:33.946 | INFO | ANALYSIS : 62.2%, 3.0, 97.3ms, 0 | 20260505 18:38:33.946 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:33.946 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:33.946 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 18:38:33.946 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:33.946 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:33.946 | INFO | Uploaded 36GiB to GPU in 2.14435s (16.7883GiB/s) | 20260505 18:38:33.946 | INFO | Downloaded 34GiB to GPU in 2.05314s (16.56GiB/s) | 20260505 18:38:39.089 | INFO | Analyzing chunk frames = [5422080,5422592) | 20260505 18:38:39.089 | INFO | MemInfo: { | 20260505 18:38:39.089 | INFO | "gpu" : " 17 GB", | 20260505 18:38:39.089 | INFO | "huge" : "", | 20260505 18:38:39.089 | INFO | "huge_shared" : " 27 GB", | 20260505 18:38:39.089 | INFO | "malloc" : " 14 GB", | 20260505 18:38:39.089 | INFO | "pinned" : " 29 GB" | 20260505 18:38:39.089 | INFO | } | 20260505 18:38:39.089 | INFO | Perf report for the last 5.14391 seconds: | 20260505 18:38:39.089 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:38:39.089 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:39.089 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:39.089 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:39.089 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 18:38:39.090 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:39.090 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:39.090 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 18:38:39.090 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:39.090 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:39.090 | INFO | Uploaded 36GiB to GPU in 2.12172s (16.9674GiB/s) | 20260505 18:38:39.090 | INFO | Downloaded 34GiB to GPU in 2.13481s (15.9265GiB/s) | 20260505 18:38:44.222 | INFO | Analyzing chunk frames = [5422592,5423104) | 20260505 18:38:44.222 | INFO | MemInfo: { | 20260505 18:38:44.222 | INFO | "gpu" : " 17 GB", | 20260505 18:38:44.222 | INFO | "huge" : "", | 20260505 18:38:44.222 | INFO | "huge_shared" : " 27 GB", | 20260505 18:38:44.222 | INFO | "malloc" : " 14 GB", | 20260505 18:38:44.222 | INFO | "pinned" : " 29 GB" | 20260505 18:38:44.222 | INFO | } | 20260505 18:38:44.222 | INFO | Perf report for the last 5.13287 seconds: | 20260505 18:38:44.222 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:38:44.222 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:44.222 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:44.222 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:44.222 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 18:38:44.222 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:44.222 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:44.222 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 18:38:44.222 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:44.222 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:44.222 | INFO | Uploaded 36GiB to GPU in 2.20022s (16.362GiB/s) | 20260505 18:38:44.223 | INFO | Downloaded 34GiB to GPU in 2.05043s (16.5819GiB/s) | 20260505 18:38:44.223 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 18:38:44.223 | INFO | Total mem usage: 15.3 GiB | 20260505 18:38:44.223 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 18:38:44.223 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.32%) | 20260505 18:38:44.223 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 18:38:44.223 | INFO | High Water Breakdown: | 20260505 18:38:44.223 | INFO | 60.9% : 8.62 GiB ( 90.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 18:38:44.223 | INFO | 38.9% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 18:38:44.223 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 18:38:44.223 | INFO | ------------------------------------------ | 20260505 18:38:44.223 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:38:44.223 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 18:38:44.223 | INFO | Total mem usage: 26.5 GiB | 20260505 18:38:44.223 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 18:38:44.223 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 18:38:44.223 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 18:38:44.223 | INFO | High Water Breakdown: | 20260505 18:38:44.223 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:38:44.223 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:38:44.223 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:38:44.223 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:38:44.223 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:38:44.223 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:38:44.223 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 18:38:44.223 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 18:38:44.223 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:38:44.223 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 18:38:44.223 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 18:38:44.223 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 18:38:44.223 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 18:38:44.223 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 18:38:44.223 | INFO | ------------------------------------------ | 20260505 18:38:44.223 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:38:44.223 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 18:38:44.223 | INFO | Total mem usage: 25.6 GiB | 20260505 18:38:44.223 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 18:38:44.223 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 18:38:44.223 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 18:38:44.223 | INFO | High Water Breakdown: | 20260505 18:38:44.223 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 18:38:44.223 | INFO | ------------------------------------------ | 20260505 18:38:44.223 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:38:44.223 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 18:38:44.223 | INFO | Total mem usage: 15.7 GiB | 20260505 18:38:44.223 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 18:38:44.223 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 18:38:44.223 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 18:38:44.223 | INFO | High Water Breakdown: | 20260505 18:38:44.223 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 18:38:44.223 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 18:38:44.223 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 18:38:44.223 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 18:38:44.223 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 18:38:44.223 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 18:38:44.223 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 18:38:44.223 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 18:38:44.223 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 18:38:44.223 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:38:44.223 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 18:38:44.223 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:38:44.223 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 18:38:44.223 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 18:38:44.223 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 18:38:44.223 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 18:38:44.223 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:38:44.223 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 18:38:44.223 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:38:44.223 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:38:44.223 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:38:44.223 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 18:38:44.223 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 18:38:44.223 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:38:44.223 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 18:38:44.223 | INFO | ------------------------------------------ | 20260505 18:38:44.223 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:38:44.223 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 18:38:44.223 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:38:44.223 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:38:44.223 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.49e+01, 1.35e+01 / 2.12e+00 | 20260505 18:38:44.223 | INFO | PerfReport: 0.29%, AggregateStats -- 9.24e-02 / 1.02e-01, 1.09e-02 / 4.53e-03 | 20260505 18:38:44.223 | INFO | PerfReport: 11.79%, Baseline -- 3.72e+00 / 4.51e+00, 5.70e-01 / 1.59e-01 | 20260505 18:38:44.223 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.52e+00, 1.89e-01 / 9.22e-02 | 20260505 18:38:44.223 | INFO | PerfReport: 15.98%, DME -- 4.03e+01 / 4.47e+01, 4.25e+00 / 2.21e+00 | 20260505 18:38:44.223 | INFO | PerfReport: 56.90%, FrameLabeling -- 1.79e+01 / 1.93e+01, 1.45e+00 / 1.20e+00 | 20260505 18:38:44.223 | INFO | PerfReport: 7.29%, PulseAccumulating -- 2.30e+00 / 2.87e+00, 4.42e-01 / 2.07e-01 | 20260505 18:38:44.223 | INFO | PerfReport: 3.31%, Metrics -- 1.04e+00 / 1.39e+00, 2.88e-01 / 1.62e-01 | 20260505 18:38:44.223 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.64e-01 / 4.24e-01, 6.45e-02 / 3.95e-02 | 20260505 18:38:44.224 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 18:38:44.224 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:38:44.224 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:38:44.224 | INFO | PerfReport: Full Execution -- 9.98e+01 / -nan, 2.57e+01 / 0.00e+00 | 20260505 18:38:44.224 | INFO | PerfReport: 22.54%, Upload -- 2.25e+01 / 0.00e+00, 7.89e+00 / 0.00e+00 | 20260505 18:38:44.224 | INFO | PerfReport: 21.90%, Download -- 2.18e+01 / 0.00e+00, 7.85e+00 / 0.00e+00 | 20260505 18:38:44.224 | INFO | PerfReport: 31.61%, Compute -- 3.15e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 18:38:44.224 | INFO | PerfReport: 2.25%, LockUpload -- 2.25e+00 / 0.00e+00, 7.86e+00 / 0.00e+00 | 20260505 18:38:44.224 | INFO | PerfReport: 1.39%, LockDownload -- 1.39e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260505 18:38:44.224 | INFO | PerfReport: 20.29%, LockCompute -- 2.02e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 18:38:44.224 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.95e-02 / 0.00e+00 | 20260505 18:38:44.357 | INFO | Starting a round of Baz IO | 20260505 18:38:47.535 | INFO | Populating RTMetrics report | 20260505 18:38:47.535 | INFO | RTMetrics report populated | 20260505 18:38:47.535 | INFO | Opening tmp RTMetrics json file | 20260505 18:38:47.538 | INFO | Serializing RTMetrics to json | 20260505 18:38:47.540 | INFO | Serialization done, swapping tmp file now... | 20260505 18:38:47.541 | INFO | RTMetrics done | 20260505 18:38:47.628 | INFO | Finished a round of Baz IO: Wrote 2.08 GiB in 3.27 seconds | 20260505 18:38:47.628 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.88% / 0.00% / 78.12% | 20260505 18:38:47.628 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.239336% / 7.94729e-06% / 6.75519e-05% | 20260505 18:38:47.628 | INFO | Baz IO overall: | 20260505 18:38:47.628 | INFO | BytesWritten : 3189745294237 | 20260505 18:38:47.628 | INFO | NumEvents : 1510749207561 | 20260505 18:38:49.356 | INFO | Analyzing chunk frames = [5423104,5423616) | 20260505 18:38:49.356 | INFO | MemInfo: { | 20260505 18:38:49.356 | INFO | "gpu" : " 17 GB", | 20260505 18:38:49.356 | INFO | "huge" : "", | 20260505 18:38:49.356 | INFO | "huge_shared" : " 27 GB", | 20260505 18:38:49.356 | INFO | "malloc" : " 14 GB", | 20260505 18:38:49.356 | INFO | "pinned" : " 29 GB" | 20260505 18:38:49.356 | INFO | } | 20260505 18:38:49.356 | INFO | Perf report for the last 5.13406 seconds: | 20260505 18:38:49.356 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:38:49.356 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:49.356 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:49.356 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:49.356 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260505 18:38:49.356 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:49.356 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260505 18:38:49.357 | INFO | PRE_HQ : 60.4%, 1.8, 55.6ms, 0 | 20260505 18:38:49.357 | INFO | BAZWRITER : 63.2%, 3.8, 121.7ms, 0 | 20260505 18:38:49.357 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 18:38:49.357 | INFO | Uploaded 36GiB to GPU in 2.15711s (16.689GiB/s) | 20260505 18:38:49.357 | INFO | Downloaded 38GiB to GPU in 2.2359s (16.9954GiB/s) | 20260505 18:38:54.508 | INFO | Analyzing chunk frames = [5423616,5424128) | 20260505 18:38:54.508 | INFO | MemInfo: { | 20260505 18:38:54.508 | INFO | "gpu" : " 17 GB", | 20260505 18:38:54.508 | INFO | "huge" : "", | 20260505 18:38:54.508 | INFO | "huge_shared" : " 27 GB", | 20260505 18:38:54.508 | INFO | "malloc" : " 14 GB", | 20260505 18:38:54.508 | INFO | "pinned" : " 29 GB" | 20260505 18:38:54.508 | INFO | } | 20260505 18:38:54.508 | INFO | Perf report for the last 5.15175 seconds: | 20260505 18:38:54.508 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:38:54.508 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:54.508 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:54.508 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:54.508 | INFO | ANALYSIS : 62.6%, 3.0, 98.0ms, 0 | 20260505 18:38:54.508 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:54.508 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:54.508 | INFO | PRE_HQ : 27.2%, 1.0, 14.2ms, 0 | 20260505 18:38:54.508 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:54.508 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:54.508 | INFO | Uploaded 36GiB to GPU in 2.14884s (16.7533GiB/s) | 20260505 18:38:54.508 | INFO | Downloaded 34GiB to GPU in 2.03527s (16.7054GiB/s) | 20260505 18:38:59.650 | INFO | Analyzing chunk frames = [5424128,5424640) | 20260505 18:38:59.650 | INFO | MemInfo: { | 20260505 18:38:59.650 | INFO | "gpu" : " 17 GB", | 20260505 18:38:59.650 | INFO | "huge" : "", | 20260505 18:38:59.650 | INFO | "huge_shared" : " 27 GB", | 20260505 18:38:59.650 | INFO | "malloc" : " 14 GB", | 20260505 18:38:59.650 | INFO | "pinned" : " 29 GB" | 20260505 18:38:59.650 | INFO | } | 20260505 18:38:59.650 | INFO | Perf report for the last 5.14223 seconds: | 20260505 18:38:59.650 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:38:59.650 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:59.650 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:59.650 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:59.650 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260505 18:38:59.650 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:59.650 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:59.650 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260505 18:38:59.650 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:59.650 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:38:59.651 | INFO | Uploaded 36GiB to GPU in 2.13453s (16.8656GiB/s) | 20260505 18:38:59.651 | INFO | Downloaded 34GiB to GPU in 2.04167s (16.6531GiB/s) | 20260505 18:39:04.788 | INFO | Analyzing chunk frames = [5424640,5425152) | 20260505 18:39:04.788 | INFO | MemInfo: { | 20260505 18:39:04.788 | INFO | "gpu" : " 17 GB", | 20260505 18:39:04.788 | INFO | "huge" : "", | 20260505 18:39:04.788 | INFO | "huge_shared" : " 27 GB", | 20260505 18:39:04.788 | INFO | "malloc" : " 14 GB", | 20260505 18:39:04.788 | INFO | "pinned" : " 29 GB" | 20260505 18:39:04.788 | INFO | } | 20260505 18:39:04.789 | INFO | Perf report for the last 5.13812 seconds: | 20260505 18:39:04.789 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:39:04.789 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:04.789 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:04.789 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:04.789 | INFO | ANALYSIS : 62.4%, 3.0, 96.9ms, 0 | 20260505 18:39:04.789 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:04.789 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:04.789 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:39:04.789 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:04.789 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:04.789 | INFO | Uploaded 36GiB to GPU in 2.19871s (16.3732GiB/s) | 20260505 18:39:04.789 | INFO | Downloaded 34GiB to GPU in 2.03771s (16.6854GiB/s) | 20260505 18:39:09.918 | INFO | Analyzing chunk frames = [5425152,5425664) | 20260505 18:39:09.918 | INFO | MemInfo: { | 20260505 18:39:09.918 | INFO | "gpu" : " 17 GB", | 20260505 18:39:09.918 | INFO | "huge" : "", | 20260505 18:39:09.918 | INFO | "huge_shared" : " 27 GB", | 20260505 18:39:09.918 | INFO | "malloc" : " 14 GB", | 20260505 18:39:09.918 | INFO | "pinned" : " 29 GB" | 20260505 18:39:09.918 | INFO | } | 20260505 18:39:09.918 | INFO | Perf report for the last 5.1298 seconds: | 20260505 18:39:09.918 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:39:09.918 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:09.918 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:09.918 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:09.918 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260505 18:39:09.918 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:09.919 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:09.919 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:39:09.919 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:09.919 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:09.919 | INFO | Uploaded 36GiB to GPU in 2.13239s (16.8825GiB/s) | 20260505 18:39:09.919 | INFO | Downloaded 34GiB to GPU in 2.06235s (16.486GiB/s) | 20260505 18:39:15.064 | INFO | Analyzing chunk frames = [5425664,5426176) | 20260505 18:39:15.064 | INFO | MemInfo: { | 20260505 18:39:15.064 | INFO | "gpu" : " 17 GB", | 20260505 18:39:15.064 | INFO | "huge" : "", | 20260505 18:39:15.064 | INFO | "huge_shared" : " 27 GB", | 20260505 18:39:15.064 | INFO | "malloc" : " 14 GB", | 20260505 18:39:15.064 | INFO | "pinned" : " 29 GB" | 20260505 18:39:15.064 | INFO | } | 20260505 18:39:15.064 | INFO | Perf report for the last 5.14561 seconds: | 20260505 18:39:15.064 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:39:15.064 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:15.064 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:15.064 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:15.064 | INFO | ANALYSIS : 62.3%, 3.0, 97.4ms, 0 | 20260505 18:39:15.064 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:15.064 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:15.064 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 18:39:15.064 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:15.064 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:15.064 | INFO | Uploaded 36GiB to GPU in 2.14823s (16.758GiB/s) | 20260505 18:39:15.064 | INFO | Downloaded 34GiB to GPU in 2.05135s (16.5744GiB/s) | 20260505 18:39:20.199 | INFO | Analyzing chunk frames = [5426176,5426688) | 20260505 18:39:20.199 | INFO | MemInfo: { | 20260505 18:39:20.199 | INFO | "gpu" : " 17 GB", | 20260505 18:39:20.199 | INFO | "huge" : "", | 20260505 18:39:20.199 | INFO | "huge_shared" : " 27 GB", | 20260505 18:39:20.199 | INFO | "malloc" : " 14 GB", | 20260505 18:39:20.199 | INFO | "pinned" : " 29 GB" | 20260505 18:39:20.199 | INFO | } | 20260505 18:39:20.199 | INFO | Perf report for the last 5.13521 seconds: | 20260505 18:39:20.199 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:39:20.199 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:20.199 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:20.199 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:20.199 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 18:39:20.199 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:20.199 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:20.199 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 18:39:20.199 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:20.199 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:20.199 | INFO | Uploaded 36GiB to GPU in 2.11422s (17.0275GiB/s) | 20260505 18:39:20.199 | INFO | Downloaded 34GiB to GPU in 2.11625s (16.0661GiB/s) | 20260505 18:39:25.341 | INFO | Analyzing chunk frames = [5426688,5427200) | 20260505 18:39:25.341 | INFO | MemInfo: { | 20260505 18:39:25.341 | INFO | "gpu" : " 17 GB", | 20260505 18:39:25.342 | INFO | "huge" : "", | 20260505 18:39:25.342 | INFO | "huge_shared" : " 27 GB", | 20260505 18:39:25.342 | INFO | "malloc" : " 14 GB", | 20260505 18:39:25.342 | INFO | "pinned" : " 29 GB" | 20260505 18:39:25.342 | INFO | } | 20260505 18:39:25.342 | INFO | Perf report for the last 5.14259 seconds: | 20260505 18:39:25.342 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:39:25.342 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:25.342 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:25.342 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:25.342 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 18:39:25.342 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:25.342 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:25.342 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:39:25.342 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:25.342 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:25.342 | INFO | Uploaded 36GiB to GPU in 2.19506s (16.4004GiB/s) | 20260505 18:39:25.342 | INFO | Downloaded 34GiB to GPU in 2.03452s (16.7116GiB/s) | 20260505 18:39:25.479 | INFO | Starting a round of Baz IO | 20260505 18:39:28.652 | INFO | Populating RTMetrics report | 20260505 18:39:28.652 | INFO | RTMetrics report populated | 20260505 18:39:28.652 | INFO | Opening tmp RTMetrics json file | 20260505 18:39:28.653 | INFO | Serializing RTMetrics to json | 20260505 18:39:28.655 | INFO | Serialization done, swapping tmp file now... | 20260505 18:39:28.655 | INFO | RTMetrics done | 20260505 18:39:28.741 | INFO | Finished a round of Baz IO: Wrote 2.081 GiB in 3.262 seconds | 20260505 18:39:28.741 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.88% / 0.00% / 78.12% | 20260505 18:39:28.741 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.239674% / 0% / 8.34465e-05% | 20260505 18:39:28.741 | INFO | Baz IO overall: | 20260505 18:39:28.741 | INFO | BytesWritten : 3191979588328 | 20260505 18:39:28.741 | INFO | NumEvents : 1511789259818 | 20260505 18:39:30.481 | INFO | Analyzing chunk frames = [5427200,5427712) | 20260505 18:39:30.481 | INFO | MemInfo: { | 20260505 18:39:30.481 | INFO | "gpu" : " 17 GB", | 20260505 18:39:30.481 | INFO | "huge" : "", | 20260505 18:39:30.481 | INFO | "huge_shared" : " 27 GB", | 20260505 18:39:30.481 | INFO | "malloc" : " 14 GB", | 20260505 18:39:30.481 | INFO | "pinned" : " 29 GB" | 20260505 18:39:30.481 | INFO | } | 20260505 18:39:30.481 | INFO | Perf report for the last 5.13948 seconds: | 20260505 18:39:30.481 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:39:30.481 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:30.481 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:30.481 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:30.481 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 18:39:30.481 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:30.481 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260505 18:39:30.481 | INFO | PRE_HQ : 60.2%, 1.8, 56.0ms, 0 | 20260505 18:39:30.481 | INFO | BAZWRITER : 62.9%, 3.8, 121.5ms, 0 | 20260505 18:39:30.481 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 18:39:30.481 | INFO | Uploaded 36GiB to GPU in 2.15047s (16.7405GiB/s) | 20260505 18:39:30.482 | INFO | Downloaded 38GiB to GPU in 2.24216s (16.948GiB/s) | 20260505 18:39:35.611 | INFO | Analyzing chunk frames = [5427712,5428224) | 20260505 18:39:35.611 | INFO | MemInfo: { | 20260505 18:39:35.611 | INFO | "gpu" : " 17 GB", | 20260505 18:39:35.611 | INFO | "huge" : "", | 20260505 18:39:35.611 | INFO | "huge_shared" : " 27 GB", | 20260505 18:39:35.611 | INFO | "malloc" : " 14 GB", | 20260505 18:39:35.611 | INFO | "pinned" : " 29 GB" | 20260505 18:39:35.611 | INFO | } | 20260505 18:39:35.611 | INFO | Perf report for the last 5.13012 seconds: | 20260505 18:39:35.611 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:39:35.611 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:35.611 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:35.611 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:35.611 | INFO | ANALYSIS : 62.9%, 3.0, 98.0ms, 0 | 20260505 18:39:35.611 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:35.611 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:35.612 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:39:35.612 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:35.612 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:35.612 | INFO | Uploaded 36GiB to GPU in 2.13714s (16.8449GiB/s) | 20260505 18:39:35.612 | INFO | Downloaded 34GiB to GPU in 2.02873s (16.7593GiB/s) | 20260505 18:39:40.759 | INFO | Analyzing chunk frames = [5428224,5428736) | 20260505 18:39:40.759 | INFO | MemInfo: { | 20260505 18:39:40.759 | INFO | "gpu" : " 17 GB", | 20260505 18:39:40.759 | INFO | "huge" : "", | 20260505 18:39:40.759 | INFO | "huge_shared" : " 27 GB", | 20260505 18:39:40.759 | INFO | "malloc" : " 14 GB", | 20260505 18:39:40.759 | INFO | "pinned" : " 29 GB" | 20260505 18:39:40.759 | INFO | } | 20260505 18:39:40.759 | INFO | Perf report for the last 5.14749 seconds: | 20260505 18:39:40.759 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:39:40.759 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:40.759 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:40.759 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:40.759 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260505 18:39:40.759 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:40.759 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:40.759 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:39:40.759 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:40.759 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:40.759 | INFO | Uploaded 36GiB to GPU in 2.20557s (16.3223GiB/s) | 20260505 18:39:40.759 | INFO | Downloaded 34GiB to GPU in 2.04498s (16.6261GiB/s) | 20260505 18:39:45.907 | INFO | Analyzing chunk frames = [5428736,5429248) | 20260505 18:39:45.907 | INFO | MemInfo: { | 20260505 18:39:45.907 | INFO | "gpu" : " 17 GB", | 20260505 18:39:45.907 | INFO | "huge" : "", | 20260505 18:39:45.907 | INFO | "huge_shared" : " 27 GB", | 20260505 18:39:45.907 | INFO | "malloc" : " 14 GB", | 20260505 18:39:45.907 | INFO | "pinned" : " 29 GB" | 20260505 18:39:45.907 | INFO | } | 20260505 18:39:45.907 | INFO | Perf report for the last 5.14782 seconds: | 20260505 18:39:45.907 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:39:45.907 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:45.907 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:45.907 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:45.907 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260505 18:39:45.907 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:45.907 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:45.907 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 18:39:45.907 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:45.907 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:45.907 | INFO | Uploaded 36GiB to GPU in 2.18235s (16.496GiB/s) | 20260505 18:39:45.907 | INFO | Downloaded 34GiB to GPU in 2.05975s (16.5069GiB/s) | 20260505 18:39:51.034 | INFO | Analyzing chunk frames = [5429248,5429760) | 20260505 18:39:51.034 | INFO | MemInfo: { | 20260505 18:39:51.034 | INFO | "gpu" : " 17 GB", | 20260505 18:39:51.034 | INFO | "huge" : "", | 20260505 18:39:51.034 | INFO | "huge_shared" : " 27 GB", | 20260505 18:39:51.034 | INFO | "malloc" : " 14 GB", | 20260505 18:39:51.034 | INFO | "pinned" : " 29 GB" | 20260505 18:39:51.034 | INFO | } | 20260505 18:39:51.034 | INFO | Perf report for the last 5.12748 seconds: | 20260505 18:39:51.034 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:39:51.034 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:51.034 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:51.034 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:51.034 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260505 18:39:51.034 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:51.034 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:51.034 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 18:39:51.034 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:51.034 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:51.035 | INFO | Uploaded 36GiB to GPU in 2.13211s (16.8847GiB/s) | 20260505 18:39:51.035 | INFO | Downloaded 34GiB to GPU in 2.06376s (16.4748GiB/s) | 20260505 18:39:56.167 | INFO | Analyzing chunk frames = [5429760,5430272) | 20260505 18:39:56.167 | INFO | MemInfo: { | 20260505 18:39:56.167 | INFO | "gpu" : " 17 GB", | 20260505 18:39:56.167 | INFO | "huge" : "", | 20260505 18:39:56.167 | INFO | "huge_shared" : " 27 GB", | 20260505 18:39:56.167 | INFO | "malloc" : " 14 GB", | 20260505 18:39:56.167 | INFO | "pinned" : " 29 GB" | 20260505 18:39:56.167 | INFO | } | 20260505 18:39:56.167 | INFO | Perf report for the last 5.13251 seconds: | 20260505 18:39:56.167 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:39:56.167 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:56.167 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:56.167 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:56.167 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260505 18:39:56.167 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:56.167 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:56.167 | INFO | PRE_HQ : 27.5%, 1.0, 14.2ms, 0 | 20260505 18:39:56.167 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:56.167 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:39:56.167 | INFO | Uploaded 36GiB to GPU in 2.14417s (16.7897GiB/s) | 20260505 18:39:56.167 | INFO | Downloaded 34GiB to GPU in 2.05342s (16.5578GiB/s) | 20260505 18:40:01.312 | INFO | Analyzing chunk frames = [5430272,5430784) | 20260505 18:40:01.312 | INFO | MemInfo: { | 20260505 18:40:01.312 | INFO | "gpu" : " 17 GB", | 20260505 18:40:01.312 | INFO | "huge" : "", | 20260505 18:40:01.312 | INFO | "huge_shared" : " 27 GB", | 20260505 18:40:01.312 | INFO | "malloc" : " 14 GB", | 20260505 18:40:01.312 | INFO | "pinned" : " 29 GB" | 20260505 18:40:01.312 | INFO | } | 20260505 18:40:01.312 | INFO | Perf report for the last 5.14533 seconds: | 20260505 18:40:01.312 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:40:01.312 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:01.312 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:01.312 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:01.312 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 18:40:01.312 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:01.312 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:01.312 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 18:40:01.312 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:01.312 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:01.312 | INFO | Uploaded 36GiB to GPU in 2.12388s (16.9501GiB/s) | 20260505 18:40:01.312 | INFO | Downloaded 34GiB to GPU in 2.17077s (15.6626GiB/s) | 20260505 18:40:06.454 | INFO | Analyzing chunk frames = [5430784,5431296) | 20260505 18:40:06.454 | INFO | MemInfo: { | 20260505 18:40:06.454 | INFO | "gpu" : " 17 GB", | 20260505 18:40:06.455 | INFO | "huge" : "", | 20260505 18:40:06.455 | INFO | "huge_shared" : " 27 GB", | 20260505 18:40:06.455 | INFO | "malloc" : " 14 GB", | 20260505 18:40:06.455 | INFO | "pinned" : " 29 GB" | 20260505 18:40:06.455 | INFO | } | 20260505 18:40:06.455 | INFO | Perf report for the last 5.1425 seconds: | 20260505 18:40:06.455 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:40:06.455 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:06.455 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:06.455 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:06.455 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 18:40:06.455 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:06.455 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:06.455 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:40:06.455 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:06.455 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:06.455 | INFO | Uploaded 36GiB to GPU in 2.19743s (16.3828GiB/s) | 20260505 18:40:06.455 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 18:40:06.455 | INFO | Total mem usage: 15.3 GiB | 20260505 18:40:06.455 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 18:40:06.455 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.05%) | 20260505 18:40:06.455 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 18:40:06.455 | INFO | High Water Breakdown: | 20260505 18:40:06.455 | INFO | 61% : 8.66 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 18:40:06.455 | INFO | Downloaded 34GiB to GPU in 2.03744s (16.6876GiB/s) | 20260505 18:40:06.455 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 18:40:06.455 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 18:40:06.455 | INFO | ------------------------------------------ | 20260505 18:40:06.455 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:40:06.455 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 18:40:06.455 | INFO | Total mem usage: 26.5 GiB | 20260505 18:40:06.455 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 18:40:06.455 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 18:40:06.455 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 18:40:06.455 | INFO | High Water Breakdown: | 20260505 18:40:06.455 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:40:06.455 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:40:06.455 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:40:06.455 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:40:06.455 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:40:06.455 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:40:06.455 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 18:40:06.455 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 18:40:06.455 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:40:06.455 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 18:40:06.455 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 18:40:06.455 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 18:40:06.455 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 18:40:06.455 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 18:40:06.455 | INFO | ------------------------------------------ | 20260505 18:40:06.455 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:40:06.455 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 18:40:06.455 | INFO | Total mem usage: 25.6 GiB | 20260505 18:40:06.455 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 18:40:06.455 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 18:40:06.455 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 18:40:06.455 | INFO | High Water Breakdown: | 20260505 18:40:06.455 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 18:40:06.455 | INFO | ------------------------------------------ | 20260505 18:40:06.455 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:40:06.455 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 18:40:06.455 | INFO | Total mem usage: 15.7 GiB | 20260505 18:40:06.455 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 18:40:06.455 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 18:40:06.455 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 18:40:06.455 | INFO | High Water Breakdown: | 20260505 18:40:06.455 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 18:40:06.455 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 18:40:06.455 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 18:40:06.455 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 18:40:06.455 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 18:40:06.455 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 18:40:06.455 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 18:40:06.455 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 18:40:06.455 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 18:40:06.455 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:40:06.455 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 18:40:06.455 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:40:06.456 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 18:40:06.456 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 18:40:06.456 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 18:40:06.456 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 18:40:06.456 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:40:06.456 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 18:40:06.456 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:40:06.456 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:40:06.456 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:40:06.456 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 18:40:06.456 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 18:40:06.456 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:40:06.456 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 18:40:06.456 | INFO | ------------------------------------------ | 20260505 18:40:06.456 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:40:06.456 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 18:40:06.456 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:40:06.456 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:40:06.456 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.45e+01, 1.35e+01 / 1.65e+00 | 20260505 18:40:06.456 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 1.03e-01, 1.11e-02 / 3.72e-03 | 20260505 18:40:06.456 | INFO | PerfReport: 11.81%, Baseline -- 3.72e+00 / 4.53e+00, 5.71e-01 / 1.60e-01 | 20260505 18:40:06.456 | INFO | PerfReport: 3.29%, Histogram -- 1.04e+00 / 1.54e+00, 1.95e-01 / 8.72e-02 | 20260505 18:40:06.456 | INFO | PerfReport: 15.90%, DME -- 4.01e+01 / 4.42e+01, 3.91e+00 / 1.55e+00 | 20260505 18:40:06.456 | INFO | PerfReport: 56.93%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.50e+00 / 1.22e+00 | 20260505 18:40:06.456 | INFO | PerfReport: 7.30%, PulseAccumulating -- 2.30e+00 / 2.89e+00, 4.42e-01 / 1.56e-01 | 20260505 18:40:06.456 | INFO | PerfReport: 3.33%, Metrics -- 1.05e+00 / 1.39e+00, 2.91e-01 / 1.57e-01 | 20260505 18:40:06.456 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.64e-01 / 4.32e-01, 6.42e-02 / 4.15e-02 | 20260505 18:40:06.456 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 18:40:06.456 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:40:06.456 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:40:06.456 | INFO | PerfReport: Full Execution -- 9.97e+01 / -nan, 2.56e+01 / 0.00e+00 | 20260505 18:40:06.456 | INFO | PerfReport: 22.53%, Upload -- 2.25e+01 / 0.00e+00, 7.79e+00 / 0.00e+00 | 20260505 18:40:06.456 | INFO | PerfReport: 21.76%, Download -- 2.17e+01 / 0.00e+00, 7.70e+00 / 0.00e+00 | 20260505 18:40:06.456 | INFO | PerfReport: 31.63%, Compute -- 3.15e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 18:40:06.456 | INFO | PerfReport: 2.29%, LockUpload -- 2.28e+00 / 0.00e+00, 8.04e+00 / 0.00e+00 | 20260505 18:40:06.456 | INFO | PerfReport: 1.40%, LockDownload -- 1.39e+00 / 0.00e+00, 2.94e+00 / 0.00e+00 | 20260505 18:40:06.456 | INFO | PerfReport: 20.38%, LockCompute -- 2.03e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260505 18:40:06.456 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.36e-02 / 0.00e+00 | 20260505 18:40:06.589 | INFO | Starting a round of Baz IO | 20260505 18:40:09.763 | INFO | Populating RTMetrics report | 20260505 18:40:09.764 | INFO | RTMetrics report populated | 20260505 18:40:09.764 | INFO | Opening tmp RTMetrics json file | 20260505 18:40:09.764 | INFO | Serializing RTMetrics to json | 20260505 18:40:09.765 | INFO | Serialization done, swapping tmp file now... | 20260505 18:40:09.766 | INFO | RTMetrics done | 20260505 18:40:09.857 | INFO | Finished a round of Baz IO: Wrote 2.081 GiB in 3.267 seconds | 20260505 18:40:09.857 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.88% / 0.00% / 78.12% | 20260505 18:40:09.857 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.239114% / 3.97364e-06% / 8.74201e-05% | 20260505 18:40:09.857 | INFO | Baz IO overall: | 20260505 18:40:09.857 | INFO | BytesWritten : 3194214033962 | 20260505 18:40:09.857 | INFO | NumEvents : 1512829672325 | 20260505 18:40:11.596 | INFO | Analyzing chunk frames = [5431296,5431808) | 20260505 18:40:11.596 | INFO | MemInfo: { | 20260505 18:40:11.596 | INFO | "gpu" : " 17 GB", | 20260505 18:40:11.596 | INFO | "huge" : "", | 20260505 18:40:11.596 | INFO | "huge_shared" : " 27 GB", | 20260505 18:40:11.596 | INFO | "malloc" : " 14 GB", | 20260505 18:40:11.596 | INFO | "pinned" : " 29 GB" | 20260505 18:40:11.596 | INFO | } | 20260505 18:40:11.596 | INFO | Perf report for the last 5.14162 seconds: | 20260505 18:40:11.596 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:40:11.596 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:11.596 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:11.596 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:11.596 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 18:40:11.596 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:11.596 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260505 18:40:11.596 | INFO | PRE_HQ : 60.1%, 1.8, 55.9ms, 0 | 20260505 18:40:11.596 | INFO | BAZWRITER : 63.0%, 3.8, 122.0ms, 0 | 20260505 18:40:11.596 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 18:40:11.597 | INFO | Uploaded 36GiB to GPU in 2.1515s (16.7325GiB/s) | 20260505 18:40:11.597 | INFO | Downloaded 38GiB to GPU in 2.24403s (16.9339GiB/s) | 20260505 18:40:16.736 | INFO | Analyzing chunk frames = [5431808,5432320) | 20260505 18:40:16.736 | INFO | MemInfo: { | 20260505 18:40:16.736 | INFO | "gpu" : " 17 GB", | 20260505 18:40:16.736 | INFO | "huge" : "", | 20260505 18:40:16.736 | INFO | "huge_shared" : " 27 GB", | 20260505 18:40:16.736 | INFO | "malloc" : " 14 GB", | 20260505 18:40:16.736 | INFO | "pinned" : " 29 GB" | 20260505 18:40:16.736 | INFO | } | 20260505 18:40:16.736 | INFO | Perf report for the last 5.13966 seconds: | 20260505 18:40:16.736 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:40:16.736 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:16.736 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:16.736 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:16.736 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 18:40:16.736 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:16.736 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:16.736 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:40:16.736 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:16.736 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:16.736 | INFO | Uploaded 36GiB to GPU in 2.14251s (16.8027GiB/s) | 20260505 18:40:16.736 | INFO | Downloaded 34GiB to GPU in 2.04218s (16.6488GiB/s) | 20260505 18:40:21.868 | INFO | Analyzing chunk frames = [5432320,5432832) | 20260505 18:40:21.868 | INFO | MemInfo: { | 20260505 18:40:21.868 | INFO | "gpu" : " 17 GB", | 20260505 18:40:21.868 | INFO | "huge" : "", | 20260505 18:40:21.868 | INFO | "huge_shared" : " 27 GB", | 20260505 18:40:21.868 | INFO | "malloc" : " 14 GB", | 20260505 18:40:21.868 | INFO | "pinned" : " 29 GB" | 20260505 18:40:21.868 | INFO | } | 20260505 18:40:21.868 | INFO | Perf report for the last 5.13237 seconds: | 20260505 18:40:21.868 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:40:21.868 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:21.868 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:21.868 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:21.868 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260505 18:40:21.868 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:21.868 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:21.868 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:40:21.868 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:21.868 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:21.868 | INFO | Uploaded 36GiB to GPU in 2.13797s (16.8384GiB/s) | 20260505 18:40:21.869 | INFO | Downloaded 34GiB to GPU in 2.04118s (16.657GiB/s) | 20260505 18:40:27.018 | INFO | Analyzing chunk frames = [5432832,5433344) | 20260505 18:40:27.018 | INFO | MemInfo: { | 20260505 18:40:27.018 | INFO | "gpu" : " 17 GB", | 20260505 18:40:27.018 | INFO | "huge" : "", | 20260505 18:40:27.018 | INFO | "huge_shared" : " 27 GB", | 20260505 18:40:27.018 | INFO | "malloc" : " 14 GB", | 20260505 18:40:27.018 | INFO | "pinned" : " 29 GB" | 20260505 18:40:27.018 | INFO | } | 20260505 18:40:27.018 | INFO | Perf report for the last 5.15005 seconds: | 20260505 18:40:27.018 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:40:27.018 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:27.018 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:27.018 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:27.018 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260505 18:40:27.018 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:27.018 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:27.018 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:40:27.018 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:27.019 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:27.019 | INFO | Uploaded 36GiB to GPU in 2.22612s (16.1717GiB/s) | 20260505 18:40:27.019 | INFO | Downloaded 34GiB to GPU in 2.1238s (16.009GiB/s) | 20260505 18:40:32.150 | INFO | Analyzing chunk frames = [5433344,5433856) | 20260505 18:40:32.150 | INFO | MemInfo: { | 20260505 18:40:32.150 | INFO | "gpu" : " 17 GB", | 20260505 18:40:32.150 | INFO | "huge" : "", | 20260505 18:40:32.150 | INFO | "huge_shared" : " 27 GB", | 20260505 18:40:32.150 | INFO | "malloc" : " 14 GB", | 20260505 18:40:32.150 | INFO | "pinned" : " 29 GB" | 20260505 18:40:32.150 | INFO | } | 20260505 18:40:32.150 | INFO | Perf report for the last 5.13201 seconds: | 20260505 18:40:32.150 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:40:32.150 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:32.150 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:32.150 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:32.150 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260505 18:40:32.150 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:32.150 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:32.150 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 18:40:32.150 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:32.150 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:32.151 | INFO | Uploaded 36GiB to GPU in 2.13222s (16.8838GiB/s) | 20260505 18:40:32.151 | INFO | Downloaded 34GiB to GPU in 2.06315s (16.4796GiB/s) | 20260505 18:40:37.289 | INFO | Analyzing chunk frames = [5433856,5434368) | 20260505 18:40:37.289 | INFO | MemInfo: { | 20260505 18:40:37.289 | INFO | "gpu" : " 17 GB", | 20260505 18:40:37.289 | INFO | "huge" : "", | 20260505 18:40:37.290 | INFO | "huge_shared" : " 27 GB", | 20260505 18:40:37.290 | INFO | "malloc" : " 14 GB", | 20260505 18:40:37.290 | INFO | "pinned" : " 29 GB" | 20260505 18:40:37.290 | INFO | } | 20260505 18:40:37.290 | INFO | Perf report for the last 5.13919 seconds: | 20260505 18:40:37.290 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:40:37.290 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:37.290 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:37.290 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:37.290 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260505 18:40:37.290 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:37.290 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:37.290 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:40:37.290 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:37.290 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:37.290 | INFO | Uploaded 36GiB to GPU in 2.13505s (16.8615GiB/s) | 20260505 18:40:37.290 | INFO | Downloaded 34GiB to GPU in 2.0519s (16.57GiB/s) | 20260505 18:40:42.431 | INFO | Analyzing chunk frames = [5434368,5434880) | 20260505 18:40:42.431 | INFO | MemInfo: { | 20260505 18:40:42.431 | INFO | "gpu" : " 17 GB", | 20260505 18:40:42.431 | INFO | "huge" : "", | 20260505 18:40:42.431 | INFO | "huge_shared" : " 27 GB", | 20260505 18:40:42.431 | INFO | "malloc" : " 14 GB", | 20260505 18:40:42.431 | INFO | "pinned" : " 29 GB" | 20260505 18:40:42.431 | INFO | } | 20260505 18:40:42.432 | INFO | Perf report for the last 5.14204 seconds: | 20260505 18:40:42.432 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:40:42.432 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:42.432 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:42.432 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:42.432 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 18:40:42.432 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:42.432 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:42.432 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 18:40:42.432 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:42.432 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:42.432 | INFO | Uploaded 36GiB to GPU in 2.14552s (16.7792GiB/s) | 20260505 18:40:42.432 | INFO | Downloaded 34GiB to GPU in 2.14185s (15.8742GiB/s) | 20260505 18:40:47.569 | INFO | Analyzing chunk frames = [5434880,5435392) | 20260505 18:40:47.569 | INFO | MemInfo: { | 20260505 18:40:47.569 | INFO | "gpu" : " 17 GB", | 20260505 18:40:47.569 | INFO | "huge" : "", | 20260505 18:40:47.569 | INFO | "huge_shared" : " 27 GB", | 20260505 18:40:47.569 | INFO | "malloc" : " 14 GB", | 20260505 18:40:47.569 | INFO | "pinned" : " 29 GB" | 20260505 18:40:47.569 | INFO | } | 20260505 18:40:47.569 | INFO | Perf report for the last 5.1375 seconds: | 20260505 18:40:47.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:40:47.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:47.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:47.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:47.569 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260505 18:40:47.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:47.569 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:47.569 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 18:40:47.569 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:47.569 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:47.569 | INFO | Uploaded 36GiB to GPU in 2.19529s (16.3987GiB/s) | 20260505 18:40:47.569 | INFO | Downloaded 34GiB to GPU in 2.02976s (16.7507GiB/s) | 20260505 18:40:47.712 | INFO | Starting a round of Baz IO | 20260505 18:40:50.883 | INFO | Populating RTMetrics report | 20260505 18:40:50.883 | INFO | RTMetrics report populated | 20260505 18:40:50.883 | INFO | Opening tmp RTMetrics json file | 20260505 18:40:50.884 | INFO | Serializing RTMetrics to json | 20260505 18:40:50.885 | INFO | Serialization done, swapping tmp file now... | 20260505 18:40:50.885 | INFO | RTMetrics done | 20260505 18:40:50.973 | INFO | Finished a round of Baz IO: Wrote 2.08 GiB in 3.261 seconds | 20260505 18:40:50.973 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.88% / 0.00% / 78.12% | 20260505 18:40:50.973 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.238911% / 7.94729e-06% / 8.74201e-05% | 20260505 18:40:50.973 | INFO | Baz IO overall: | 20260505 18:40:50.973 | INFO | BytesWritten : 3196447233374 | 20260505 18:40:50.973 | INFO | NumEvents : 1513869251148 | 20260505 18:40:52.711 | INFO | Analyzing chunk frames = [5435392,5435904) | 20260505 18:40:52.711 | INFO | MemInfo: { | 20260505 18:40:52.711 | INFO | "gpu" : " 17 GB", | 20260505 18:40:52.711 | INFO | "huge" : "", | 20260505 18:40:52.711 | INFO | "huge_shared" : " 27 GB", | 20260505 18:40:52.711 | INFO | "malloc" : " 14 GB", | 20260505 18:40:52.711 | INFO | "pinned" : " 29 GB" | 20260505 18:40:52.711 | INFO | } | 20260505 18:40:52.711 | INFO | Perf report for the last 5.14185 seconds: | 20260505 18:40:52.711 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:40:52.711 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:52.711 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:52.711 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:52.711 | INFO | ANALYSIS : 63.4%, 3.0, 98.8ms, 0 | 20260505 18:40:52.711 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:52.711 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260505 18:40:52.711 | INFO | PRE_HQ : 60.5%, 1.8, 56.0ms, 0 | 20260505 18:40:52.711 | INFO | BAZWRITER : 63.2%, 3.8, 122.0ms, 0 | 20260505 18:40:52.711 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 18:40:52.711 | INFO | Uploaded 36GiB to GPU in 2.16623s (16.6187GiB/s) | 20260505 18:40:52.711 | INFO | Downloaded 38GiB to GPU in 2.24746s (16.908GiB/s) | 20260505 18:40:57.849 | INFO | Analyzing chunk frames = [5435904,5436416) | 20260505 18:40:57.849 | INFO | MemInfo: { | 20260505 18:40:57.849 | INFO | "gpu" : " 17 GB", | 20260505 18:40:57.849 | INFO | "huge" : "", | 20260505 18:40:57.849 | INFO | "huge_shared" : " 27 GB", | 20260505 18:40:57.850 | INFO | "malloc" : " 14 GB", | 20260505 18:40:57.850 | INFO | "pinned" : " 29 GB" | 20260505 18:40:57.850 | INFO | } | 20260505 18:40:57.850 | INFO | Perf report for the last 5.13872 seconds: | 20260505 18:40:57.850 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:40:57.850 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:57.850 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:57.850 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:57.850 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 18:40:57.850 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:57.850 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:57.850 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:40:57.850 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:57.850 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:40:57.850 | INFO | Uploaded 36GiB to GPU in 2.15102s (16.7362GiB/s) | 20260505 18:40:57.850 | INFO | Downloaded 34GiB to GPU in 2.04692s (16.6103GiB/s) | 20260505 18:41:02.980 | INFO | Analyzing chunk frames = [5436416,5436928) | 20260505 18:41:02.980 | INFO | MemInfo: { | 20260505 18:41:02.980 | INFO | "gpu" : " 17 GB", | 20260505 18:41:02.980 | INFO | "huge" : "", | 20260505 18:41:02.980 | INFO | "huge_shared" : " 27 GB", | 20260505 18:41:02.980 | INFO | "malloc" : " 14 GB", | 20260505 18:41:02.980 | INFO | "pinned" : " 29 GB" | 20260505 18:41:02.980 | INFO | } | 20260505 18:41:02.980 | INFO | Perf report for the last 5.13055 seconds: | 20260505 18:41:02.980 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:41:02.980 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:02.980 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:02.980 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:02.980 | INFO | ANALYSIS : 62.8%, 3.0, 97.5ms, 0 | 20260505 18:41:02.980 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:02.980 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:02.980 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:41:02.980 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:02.980 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:02.981 | INFO | Uploaded 36GiB to GPU in 2.20509s (16.3258GiB/s) | 20260505 18:41:02.981 | INFO | Downloaded 34GiB to GPU in 2.0689s (16.4338GiB/s) | 20260505 18:41:08.117 | INFO | Analyzing chunk frames = [5436928,5437440) | 20260505 18:41:08.117 | INFO | MemInfo: { | 20260505 18:41:08.117 | INFO | "gpu" : " 17 GB", | 20260505 18:41:08.117 | INFO | "huge" : "", | 20260505 18:41:08.117 | INFO | "huge_shared" : " 27 GB", | 20260505 18:41:08.117 | INFO | "malloc" : " 14 GB", | 20260505 18:41:08.117 | INFO | "pinned" : " 29 GB" | 20260505 18:41:08.117 | INFO | } | 20260505 18:41:08.117 | INFO | Perf report for the last 5.13715 seconds: | 20260505 18:41:08.118 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:41:08.118 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:08.118 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:08.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:08.118 | INFO | ANALYSIS : 62.5%, 3.0, 97.1ms, 0 | 20260505 18:41:08.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:08.118 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:08.118 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 18:41:08.118 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:08.118 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:08.118 | INFO | Uploaded 36GiB to GPU in 2.19588s (16.3943GiB/s) | 20260505 18:41:08.118 | INFO | Downloaded 34GiB to GPU in 2.13853s (15.8988GiB/s) | 20260505 18:41:13.268 | INFO | Analyzing chunk frames = [5437440,5437952) | 20260505 18:41:13.268 | INFO | MemInfo: { | 20260505 18:41:13.268 | INFO | "gpu" : " 17 GB", | 20260505 18:41:13.268 | INFO | "huge" : "", | 20260505 18:41:13.268 | INFO | "huge_shared" : " 27 GB", | 20260505 18:41:13.268 | INFO | "malloc" : " 14 GB", | 20260505 18:41:13.268 | INFO | "pinned" : " 29 GB" | 20260505 18:41:13.268 | INFO | } | 20260505 18:41:13.269 | INFO | Perf report for the last 5.15103 seconds: | 20260505 18:41:13.269 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:41:13.269 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:13.269 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:13.269 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:13.269 | INFO | ANALYSIS : 62.0%, 3.0, 97.1ms, 0 | 20260505 18:41:13.269 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:13.269 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:13.269 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260505 18:41:13.269 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:13.269 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:13.269 | INFO | Uploaded 36GiB to GPU in 2.15272s (16.7231GiB/s) | 20260505 18:41:13.269 | INFO | Downloaded 34GiB to GPU in 2.05866s (16.5156GiB/s) | 20260505 18:41:18.417 | INFO | Analyzing chunk frames = [5437952,5438464) | 20260505 18:41:18.417 | INFO | MemInfo: { | 20260505 18:41:18.417 | INFO | "gpu" : " 17 GB", | 20260505 18:41:18.418 | INFO | "huge" : "", | 20260505 18:41:18.418 | INFO | "huge_shared" : " 27 GB", | 20260505 18:41:18.418 | INFO | "malloc" : " 14 GB", | 20260505 18:41:18.418 | INFO | "pinned" : " 29 GB" | 20260505 18:41:18.418 | INFO | } | 20260505 18:41:18.418 | INFO | Perf report for the last 5.14921 seconds: | 20260505 18:41:18.418 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:41:18.418 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:18.418 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:18.418 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:18.418 | INFO | ANALYSIS : 62.0%, 3.0, 97.0ms, 0 | 20260505 18:41:18.418 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:18.418 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:18.418 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:41:18.418 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:18.418 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:18.418 | INFO | Uploaded 36GiB to GPU in 2.13219s (16.8841GiB/s) | 20260505 18:41:18.418 | INFO | Downloaded 34GiB to GPU in 2.05769s (16.5234GiB/s) | 20260505 18:41:23.546 | INFO | Analyzing chunk frames = [5438464,5438976) | 20260505 18:41:23.546 | INFO | MemInfo: { | 20260505 18:41:23.546 | INFO | "gpu" : " 17 GB", | 20260505 18:41:23.546 | INFO | "huge" : "", | 20260505 18:41:23.546 | INFO | "huge_shared" : " 27 GB", | 20260505 18:41:23.547 | INFO | "malloc" : " 14 GB", | 20260505 18:41:23.547 | INFO | "pinned" : " 29 GB" | 20260505 18:41:23.547 | INFO | } | 20260505 18:41:23.547 | INFO | Perf report for the last 5.12882 seconds: | 20260505 18:41:23.547 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:41:23.547 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:23.547 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:23.547 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:23.547 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 18:41:23.547 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:23.547 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:23.547 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 18:41:23.547 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:23.547 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:23.547 | INFO | Uploaded 36GiB to GPU in 2.16061s (16.6619GiB/s) | 20260505 18:41:23.547 | INFO | Downloaded 34GiB to GPU in 2.12202s (16.0225GiB/s) | 20260505 18:41:28.678 | INFO | Analyzing chunk frames = [5438976,5439488) | 20260505 18:41:28.679 | INFO | MemInfo: { | 20260505 18:41:28.679 | INFO | "gpu" : " 17 GB", | 20260505 18:41:28.679 | INFO | "huge" : "", | 20260505 18:41:28.679 | INFO | "huge_shared" : " 27 GB", | 20260505 18:41:28.679 | INFO | "malloc" : " 14 GB", | 20260505 18:41:28.679 | INFO | "pinned" : " 29 GB" | 20260505 18:41:28.679 | INFO | } | 20260505 18:41:28.679 | INFO | Perf report for the last 5.13217 seconds: | 20260505 18:41:28.679 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:41:28.679 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:28.679 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:28.679 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:28.679 | INFO | ANALYSIS : 62.9%, 3.0, 98.0ms, 0 | 20260505 18:41:28.679 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:28.679 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:28.679 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 18:41:28.679 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:28.679 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:28.679 | INFO | Uploaded 36GiB to GPU in 2.19944s (16.3678GiB/s) | 20260505 18:41:28.679 | INFO | Downloaded 34GiB to GPU in 2.02497s (16.7904GiB/s) | 20260505 18:41:28.679 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 18:41:28.679 | INFO | Total mem usage: 15.3 GiB | 20260505 18:41:28.679 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 18:41:28.679 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.02%) | 20260505 18:41:28.679 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 18:41:28.679 | INFO | High Water Breakdown: | 20260505 18:41:28.679 | INFO | 61% : 8.66 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 18:41:28.679 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 18:41:28.679 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 18:41:28.679 | INFO | ------------------------------------------ | 20260505 18:41:28.679 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:41:28.679 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 18:41:28.679 | INFO | Total mem usage: 26.5 GiB | 20260505 18:41:28.679 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 18:41:28.679 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 18:41:28.679 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 18:41:28.679 | INFO | High Water Breakdown: | 20260505 18:41:28.679 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:41:28.679 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:41:28.679 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:41:28.679 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:41:28.679 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:41:28.680 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:41:28.680 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 18:41:28.680 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 18:41:28.680 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:41:28.680 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 18:41:28.680 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 18:41:28.680 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 18:41:28.680 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 18:41:28.680 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 18:41:28.680 | INFO | ------------------------------------------ | 20260505 18:41:28.680 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:41:28.680 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 18:41:28.680 | INFO | Total mem usage: 25.6 GiB | 20260505 18:41:28.680 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 18:41:28.680 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 18:41:28.680 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 18:41:28.680 | INFO | High Water Breakdown: | 20260505 18:41:28.680 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 18:41:28.680 | INFO | ------------------------------------------ | 20260505 18:41:28.680 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:41:28.680 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 18:41:28.680 | INFO | Total mem usage: 15.7 GiB | 20260505 18:41:28.680 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 18:41:28.680 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 18:41:28.680 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 18:41:28.680 | INFO | High Water Breakdown: | 20260505 18:41:28.680 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 18:41:28.680 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 18:41:28.680 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 18:41:28.680 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 18:41:28.680 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 18:41:28.680 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 18:41:28.680 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 18:41:28.680 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 18:41:28.680 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 18:41:28.680 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:41:28.680 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 18:41:28.680 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:41:28.680 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 18:41:28.680 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 18:41:28.680 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 18:41:28.680 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 18:41:28.680 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:41:28.680 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 18:41:28.680 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:41:28.680 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:41:28.680 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:41:28.680 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 18:41:28.680 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 18:41:28.680 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:41:28.680 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 18:41:28.680 | INFO | ------------------------------------------ | 20260505 18:41:28.680 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:41:28.680 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 18:41:28.680 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:41:28.680 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:41:28.680 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.46e+01, 1.34e+01 / 1.96e+00 | 20260505 18:41:28.680 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.03e-01, 1.04e-02 / 8.59e-03 | 20260505 18:41:28.680 | INFO | PerfReport: 11.77%, Baseline -- 3.71e+00 / 4.54e+00, 5.58e-01 / 1.08e-01 | 20260505 18:41:28.680 | INFO | PerfReport: 3.26%, Histogram -- 1.03e+00 / 1.51e+00, 1.86e-01 / 9.78e-02 | 20260505 18:41:28.680 | INFO | PerfReport: 15.97%, DME -- 4.02e+01 / 4.46e+01, 4.25e+00 / 2.05e+00 | 20260505 18:41:28.680 | INFO | PerfReport: 56.89%, FrameLabeling -- 1.79e+01 / 1.91e+01, 1.41e+00 / 9.48e-01 | 20260505 18:41:28.680 | INFO | PerfReport: 7.32%, PulseAccumulating -- 2.31e+00 / 2.89e+00, 4.42e-01 / 1.68e-01 | 20260505 18:41:28.680 | INFO | PerfReport: 3.33%, Metrics -- 1.05e+00 / 1.37e+00, 2.88e-01 / 1.27e-01 | 20260505 18:41:28.680 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.64e-01 / 4.25e-01, 6.37e-02 / 3.43e-02 | 20260505 18:41:28.680 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 18:41:28.680 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:41:28.680 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:41:28.680 | INFO | PerfReport: Full Execution -- 9.97e+01 / -nan, 2.56e+01 / 0.00e+00 | 20260505 18:41:28.680 | INFO | PerfReport: 22.61%, Upload -- 2.25e+01 / 0.00e+00, 7.91e+00 / 0.00e+00 | 20260505 18:41:28.680 | INFO | PerfReport: 21.87%, Download -- 2.18e+01 / 0.00e+00, 7.75e+00 / 0.00e+00 | 20260505 18:41:28.680 | INFO | PerfReport: 31.58%, Compute -- 3.15e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 18:41:28.680 | INFO | PerfReport: 2.33%, LockUpload -- 2.32e+00 / 0.00e+00, 7.92e+00 / 0.00e+00 | 20260505 18:41:28.680 | INFO | PerfReport: 1.36%, LockDownload -- 1.35e+00 / 0.00e+00, 2.78e+00 / 0.00e+00 | 20260505 18:41:28.680 | INFO | PerfReport: 20.25%, LockCompute -- 2.02e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 18:41:28.680 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.87e-02 / 0.00e+00 | 20260505 18:41:28.818 | INFO | Starting a round of Baz IO | 20260505 18:41:32.001 | INFO | Populating RTMetrics report | 20260505 18:41:32.001 | INFO | RTMetrics report populated | 20260505 18:41:32.001 | INFO | Opening tmp RTMetrics json file | 20260505 18:41:32.002 | INFO | Serializing RTMetrics to json | 20260505 18:41:32.004 | INFO | Serialization done, swapping tmp file now... | 20260505 18:41:32.004 | INFO | RTMetrics done | 20260505 18:41:32.091 | INFO | Finished a round of Baz IO: Wrote 2.078 GiB in 3.272 seconds | 20260505 18:41:32.091 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.87% / 0.00% / 78.12% | 20260505 18:41:32.091 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.238093% / 7.94729e-06% / 7.94729e-05% | 20260505 18:41:32.091 | INFO | Baz IO overall: | 20260505 18:41:32.091 | INFO | BytesWritten : 3198678957417 | 20260505 18:41:32.092 | INFO | NumEvents : 1514907755737 | 20260505 18:41:33.824 | INFO | Analyzing chunk frames = [5439488,5440000) | 20260505 18:41:33.824 | INFO | MemInfo: { | 20260505 18:41:33.824 | INFO | "gpu" : " 17 GB", | 20260505 18:41:33.824 | INFO | "huge" : "", | 20260505 18:41:33.824 | INFO | "huge_shared" : " 27 GB", | 20260505 18:41:33.824 | INFO | "malloc" : " 14 GB", | 20260505 18:41:33.824 | INFO | "pinned" : " 29 GB" | 20260505 18:41:33.824 | INFO | } | 20260505 18:41:33.824 | INFO | Perf report for the last 5.14564 seconds: | 20260505 18:41:33.824 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:41:33.825 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:33.825 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:33.825 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:33.825 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 18:41:33.825 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:33.825 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260505 18:41:33.825 | INFO | PRE_HQ : 60.2%, 1.8, 55.7ms, 0 | 20260505 18:41:33.825 | INFO | BAZWRITER : 63.0%, 3.8, 121.7ms, 0 | 20260505 18:41:33.825 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 18:41:33.825 | INFO | Uploaded 36GiB to GPU in 2.16214s (16.6502GiB/s) | 20260505 18:41:33.825 | INFO | Downloaded 38GiB to GPU in 2.25183s (16.8752GiB/s) | 20260505 18:41:38.964 | INFO | Analyzing chunk frames = [5440000,5440512) | 20260505 18:41:38.964 | INFO | MemInfo: { | 20260505 18:41:38.964 | INFO | "gpu" : " 17 GB", | 20260505 18:41:38.964 | INFO | "huge" : "", | 20260505 18:41:38.964 | INFO | "huge_shared" : " 27 GB", | 20260505 18:41:38.964 | INFO | "malloc" : " 14 GB", | 20260505 18:41:38.964 | INFO | "pinned" : " 29 GB" | 20260505 18:41:38.964 | INFO | } | 20260505 18:41:38.964 | INFO | Perf report for the last 5.13964 seconds: | 20260505 18:41:38.964 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:41:38.964 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:38.964 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:38.964 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:38.964 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 18:41:38.964 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:38.964 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:38.964 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:41:38.964 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:38.964 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:38.964 | INFO | Uploaded 36GiB to GPU in 2.13379s (16.8714GiB/s) | 20260505 18:41:38.964 | INFO | Downloaded 34GiB to GPU in 2.02392s (16.7991GiB/s) | 20260505 18:41:44.103 | INFO | Analyzing chunk frames = [5440512,5441024) | 20260505 18:41:44.103 | INFO | MemInfo: { | 20260505 18:41:44.103 | INFO | "gpu" : " 17 GB", | 20260505 18:41:44.103 | INFO | "huge" : "", | 20260505 18:41:44.103 | INFO | "huge_shared" : " 27 GB", | 20260505 18:41:44.103 | INFO | "malloc" : " 14 GB", | 20260505 18:41:44.103 | INFO | "pinned" : " 29 GB" | 20260505 18:41:44.103 | INFO | } | 20260505 18:41:44.103 | INFO | Perf report for the last 5.13897 seconds: | 20260505 18:41:44.103 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:41:44.103 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:44.103 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:44.103 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:44.103 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260505 18:41:44.103 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:44.103 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:44.103 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:41:44.103 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:44.103 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:44.103 | INFO | Uploaded 36GiB to GPU in 2.13427s (16.8676GiB/s) | 20260505 18:41:44.103 | INFO | Downloaded 34GiB to GPU in 2.0392s (16.6732GiB/s) | 20260505 18:41:49.241 | INFO | Analyzing chunk frames = [5441024,5441536) | 20260505 18:41:49.241 | INFO | MemInfo: { | 20260505 18:41:49.241 | INFO | "gpu" : " 17 GB", | 20260505 18:41:49.241 | INFO | "huge" : "", | 20260505 18:41:49.241 | INFO | "huge_shared" : " 27 GB", | 20260505 18:41:49.241 | INFO | "malloc" : " 14 GB", | 20260505 18:41:49.241 | INFO | "pinned" : " 29 GB" | 20260505 18:41:49.241 | INFO | } | 20260505 18:41:49.241 | INFO | Perf report for the last 5.13794 seconds: | 20260505 18:41:49.241 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:41:49.241 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:49.241 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:49.241 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:49.241 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260505 18:41:49.241 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:49.241 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:49.241 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:41:49.241 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:49.241 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:49.241 | INFO | Uploaded 36GiB to GPU in 2.19391s (16.409GiB/s) | 20260505 18:41:49.241 | INFO | Downloaded 34GiB to GPU in 2.05333s (16.5585GiB/s) | 20260505 18:41:54.378 | INFO | Analyzing chunk frames = [5441536,5442048) | 20260505 18:41:54.378 | INFO | MemInfo: { | 20260505 18:41:54.378 | INFO | "gpu" : " 17 GB", | 20260505 18:41:54.378 | INFO | "huge" : "", | 20260505 18:41:54.378 | INFO | "huge_shared" : " 27 GB", | 20260505 18:41:54.378 | INFO | "malloc" : " 14 GB", | 20260505 18:41:54.378 | INFO | "pinned" : " 29 GB" | 20260505 18:41:54.378 | INFO | } | 20260505 18:41:54.378 | INFO | Perf report for the last 5.13666 seconds: | 20260505 18:41:54.378 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:41:54.378 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:54.378 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:54.378 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:54.378 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 18:41:54.378 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:54.378 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:54.378 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:41:54.378 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:54.378 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:54.378 | INFO | Uploaded 36GiB to GPU in 2.14899s (16.752GiB/s) | 20260505 18:41:54.378 | INFO | Downloaded 34GiB to GPU in 2.06288s (16.4818GiB/s) | 20260505 18:41:59.522 | INFO | Analyzing chunk frames = [5442048,5442560) | 20260505 18:41:59.522 | INFO | MemInfo: { | 20260505 18:41:59.522 | INFO | "gpu" : " 17 GB", | 20260505 18:41:59.522 | INFO | "huge" : "", | 20260505 18:41:59.522 | INFO | "huge_shared" : " 27 GB", | 20260505 18:41:59.522 | INFO | "malloc" : " 14 GB", | 20260505 18:41:59.522 | INFO | "pinned" : " 29 GB" | 20260505 18:41:59.522 | INFO | } | 20260505 18:41:59.522 | INFO | Perf report for the last 5.14424 seconds: | 20260505 18:41:59.522 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:41:59.522 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:59.522 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:59.522 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:59.522 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260505 18:41:59.522 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:59.522 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:59.522 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 18:41:59.522 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:59.522 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:41:59.522 | INFO | Uploaded 36GiB to GPU in 2.13779s (16.8398GiB/s) | 20260505 18:41:59.522 | INFO | Downloaded 34GiB to GPU in 2.05211s (16.5683GiB/s) | 20260505 18:42:04.662 | INFO | Analyzing chunk frames = [5442560,5443072) | 20260505 18:42:04.662 | INFO | MemInfo: { | 20260505 18:42:04.662 | INFO | "gpu" : " 17 GB", | 20260505 18:42:04.662 | INFO | "huge" : "", | 20260505 18:42:04.662 | INFO | "huge_shared" : " 27 GB", | 20260505 18:42:04.662 | INFO | "malloc" : " 14 GB", | 20260505 18:42:04.662 | INFO | "pinned" : " 29 GB" | 20260505 18:42:04.662 | INFO | } | 20260505 18:42:04.662 | INFO | Perf report for the last 5.14021 seconds: | 20260505 18:42:04.662 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:42:04.662 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:04.662 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:04.662 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:04.662 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 18:42:04.662 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:04.662 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:04.662 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 18:42:04.662 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:04.662 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:04.662 | INFO | Uploaded 36GiB to GPU in 2.13408s (16.8691GiB/s) | 20260505 18:42:04.663 | INFO | Downloaded 34GiB to GPU in 2.13864s (15.8979GiB/s) | 20260505 18:42:09.792 | INFO | Analyzing chunk frames = [5443072,5443584) | 20260505 18:42:09.792 | INFO | MemInfo: { | 20260505 18:42:09.792 | INFO | "gpu" : " 17 GB", | 20260505 18:42:09.792 | INFO | "huge" : "", | 20260505 18:42:09.792 | INFO | "huge_shared" : " 27 GB", | 20260505 18:42:09.792 | INFO | "malloc" : " 14 GB", | 20260505 18:42:09.792 | INFO | "pinned" : " 29 GB" | 20260505 18:42:09.792 | INFO | } | 20260505 18:42:09.793 | INFO | Perf report for the last 5.13036 seconds: | 20260505 18:42:09.793 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:42:09.793 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:09.793 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:09.793 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:09.793 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 18:42:09.793 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:09.793 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:09.793 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 18:42:09.793 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:09.793 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:09.793 | INFO | Uploaded 36GiB to GPU in 2.19853s (16.3746GiB/s) | 20260505 18:42:09.793 | INFO | Downloaded 34GiB to GPU in 2.03753s (16.6869GiB/s) | 20260505 18:42:09.933 | INFO | Starting a round of Baz IO | 20260505 18:42:13.113 | INFO | Populating RTMetrics report | 20260505 18:42:13.114 | INFO | RTMetrics report populated | 20260505 18:42:13.114 | INFO | Opening tmp RTMetrics json file | 20260505 18:42:13.114 | INFO | Serializing RTMetrics to json | 20260505 18:42:13.116 | INFO | Serialization done, swapping tmp file now... | 20260505 18:42:13.116 | INFO | RTMetrics done | 20260505 18:42:13.206 | INFO | Finished a round of Baz IO: Wrote 2.077 GiB in 3.272 seconds | 20260505 18:42:13.206 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.87% / 0.00% / 78.13% | 20260505 18:42:13.206 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.236877% / 7.94729e-06% / 0.000107288% | 20260505 18:42:13.206 | INFO | Baz IO overall: | 20260505 18:42:13.206 | INFO | BytesWritten : 3200908679481 | 20260505 18:42:13.206 | INFO | NumEvents : 1515944630776 | 20260505 18:42:14.928 | INFO | Analyzing chunk frames = [5443584,5444096) | 20260505 18:42:14.928 | INFO | MemInfo: { | 20260505 18:42:14.928 | INFO | "gpu" : " 17 GB", | 20260505 18:42:14.928 | INFO | "huge" : "", | 20260505 18:42:14.928 | INFO | "huge_shared" : " 27 GB", | 20260505 18:42:14.928 | INFO | "malloc" : " 14 GB", | 20260505 18:42:14.928 | INFO | "pinned" : " 29 GB" | 20260505 18:42:14.928 | INFO | } | 20260505 18:42:14.928 | INFO | Perf report for the last 5.13587 seconds: | 20260505 18:42:14.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:42:14.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:14.928 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:14.928 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:14.928 | INFO | ANALYSIS : 63.6%, 3.0, 99.0ms, 0 | 20260505 18:42:14.929 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:14.929 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260505 18:42:14.929 | INFO | PRE_HQ : 60.1%, 1.8, 55.7ms, 0 | 20260505 18:42:14.929 | INFO | BAZWRITER : 63.2%, 3.7, 120.8ms, 0 | 20260505 18:42:14.929 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 18:42:14.929 | INFO | Uploaded 36GiB to GPU in 2.16703s (16.6126GiB/s) | 20260505 18:42:14.929 | INFO | Downloaded 38GiB to GPU in 2.26631s (16.7674GiB/s) | 20260505 18:42:20.074 | INFO | Analyzing chunk frames = [5444096,5444608) | 20260505 18:42:20.074 | INFO | MemInfo: { | 20260505 18:42:20.074 | INFO | "gpu" : " 17 GB", | 20260505 18:42:20.074 | INFO | "huge" : "", | 20260505 18:42:20.074 | INFO | "huge_shared" : " 27 GB", | 20260505 18:42:20.074 | INFO | "malloc" : " 14 GB", | 20260505 18:42:20.074 | INFO | "pinned" : " 29 GB" | 20260505 18:42:20.074 | INFO | } | 20260505 18:42:20.074 | INFO | Perf report for the last 5.1458 seconds: | 20260505 18:42:20.074 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:42:20.074 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:20.074 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:20.074 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:20.074 | INFO | ANALYSIS : 62.7%, 3.0, 97.0ms, 0 | 20260505 18:42:20.074 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:20.074 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:20.074 | INFO | PRE_HQ : 27.3%, 1.0, 14.0ms, 0 | 20260505 18:42:20.074 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:20.074 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:20.075 | INFO | Uploaded 36GiB to GPU in 2.12643s (16.9298GiB/s) | 20260505 18:42:20.075 | INFO | Downloaded 34GiB to GPU in 2.03321s (16.7223GiB/s) | 20260505 18:42:25.218 | INFO | Analyzing chunk frames = [5444608,5445120) | 20260505 18:42:25.218 | INFO | MemInfo: { | 20260505 18:42:25.218 | INFO | "gpu" : " 17 GB", | 20260505 18:42:25.218 | INFO | "huge" : "", | 20260505 18:42:25.218 | INFO | "huge_shared" : " 27 GB", | 20260505 18:42:25.218 | INFO | "malloc" : " 14 GB", | 20260505 18:42:25.218 | INFO | "pinned" : " 29 GB" | 20260505 18:42:25.218 | INFO | } | 20260505 18:42:25.218 | INFO | Perf report for the last 5.14374 seconds: | 20260505 18:42:25.218 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:42:25.218 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:25.218 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:25.218 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:25.218 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 18:42:25.218 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:25.218 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:25.218 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:42:25.218 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:25.218 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:25.218 | INFO | Uploaded 36GiB to GPU in 2.2074s (16.3088GiB/s) | 20260505 18:42:25.218 | INFO | Downloaded 34GiB to GPU in 2.07124s (16.4152GiB/s) | 20260505 18:42:30.359 | INFO | Analyzing chunk frames = [5445120,5445632) | 20260505 18:42:30.359 | INFO | MemInfo: { | 20260505 18:42:30.359 | INFO | "gpu" : " 17 GB", | 20260505 18:42:30.359 | INFO | "huge" : "", | 20260505 18:42:30.359 | INFO | "huge_shared" : " 27 GB", | 20260505 18:42:30.359 | INFO | "malloc" : " 14 GB", | 20260505 18:42:30.359 | INFO | "pinned" : " 29 GB" | 20260505 18:42:30.359 | INFO | } | 20260505 18:42:30.359 | INFO | Perf report for the last 5.14084 seconds: | 20260505 18:42:30.359 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:42:30.359 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:30.359 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:30.359 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:30.359 | INFO | ANALYSIS : 62.4%, 3.0, 96.9ms, 0 | 20260505 18:42:30.359 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:30.359 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:30.359 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:42:30.359 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:30.359 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:30.359 | INFO | Uploaded 36GiB to GPU in 2.18781s (16.4548GiB/s) | 20260505 18:42:30.359 | INFO | Downloaded 34GiB to GPU in 2.04885s (16.5947GiB/s) | 20260505 18:42:35.493 | INFO | Analyzing chunk frames = [5445632,5446144) | 20260505 18:42:35.493 | INFO | MemInfo: { | 20260505 18:42:35.493 | INFO | "gpu" : " 17 GB", | 20260505 18:42:35.493 | INFO | "huge" : "", | 20260505 18:42:35.493 | INFO | "huge_shared" : " 27 GB", | 20260505 18:42:35.493 | INFO | "malloc" : " 14 GB", | 20260505 18:42:35.493 | INFO | "pinned" : " 29 GB" | 20260505 18:42:35.493 | INFO | } | 20260505 18:42:35.493 | INFO | Perf report for the last 5.1341 seconds: | 20260505 18:42:35.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:42:35.493 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:35.493 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:35.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:35.493 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260505 18:42:35.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:35.493 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:35.493 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:42:35.493 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:35.493 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:35.493 | INFO | Uploaded 36GiB to GPU in 2.12906s (16.9089GiB/s) | 20260505 18:42:35.493 | INFO | Downloaded 34GiB to GPU in 2.06123s (16.495GiB/s) | 20260505 18:42:40.631 | INFO | Analyzing chunk frames = [5446144,5446656) | 20260505 18:42:40.631 | INFO | MemInfo: { | 20260505 18:42:40.631 | INFO | "gpu" : " 17 GB", | 20260505 18:42:40.631 | INFO | "huge" : "", | 20260505 18:42:40.631 | INFO | "huge_shared" : " 27 GB", | 20260505 18:42:40.631 | INFO | "malloc" : " 14 GB", | 20260505 18:42:40.632 | INFO | "pinned" : " 29 GB" | 20260505 18:42:40.632 | INFO | } | 20260505 18:42:40.632 | INFO | Perf report for the last 5.13861 seconds: | 20260505 18:42:40.632 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:42:40.632 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:40.632 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:40.632 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:40.632 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260505 18:42:40.632 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:40.632 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:40.632 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:42:40.632 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:40.632 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:40.632 | INFO | Uploaded 36GiB to GPU in 2.14242s (16.8034GiB/s) | 20260505 18:42:40.632 | INFO | Downloaded 34GiB to GPU in 2.04577s (16.6197GiB/s) | 20260505 18:42:45.776 | INFO | Analyzing chunk frames = [5446656,5447168) | 20260505 18:42:45.776 | INFO | MemInfo: { | 20260505 18:42:45.776 | INFO | "gpu" : " 17 GB", | 20260505 18:42:45.776 | INFO | "huge" : "", | 20260505 18:42:45.776 | INFO | "huge_shared" : " 27 GB", | 20260505 18:42:45.776 | INFO | "malloc" : " 14 GB", | 20260505 18:42:45.776 | INFO | "pinned" : " 29 GB" | 20260505 18:42:45.776 | INFO | } | 20260505 18:42:45.776 | INFO | Perf report for the last 5.14433 seconds: | 20260505 18:42:45.776 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:42:45.776 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:45.776 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:45.776 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:45.776 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 18:42:45.776 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:45.776 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:45.776 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 18:42:45.776 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:45.776 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:45.776 | INFO | Uploaded 36GiB to GPU in 2.13902s (16.8301GiB/s) | 20260505 18:42:45.776 | INFO | Downloaded 34GiB to GPU in 2.16487s (15.7053GiB/s) | 20260505 18:42:50.917 | INFO | Analyzing chunk frames = [5447168,5447680) | 20260505 18:42:50.917 | INFO | MemInfo: { | 20260505 18:42:50.917 | INFO | "gpu" : " 17 GB", | 20260505 18:42:50.917 | INFO | "huge" : "", | 20260505 18:42:50.917 | INFO | "huge_shared" : " 27 GB", | 20260505 18:42:50.917 | INFO | "malloc" : " 14 GB", | 20260505 18:42:50.918 | INFO | "pinned" : " 29 GB" | 20260505 18:42:50.918 | INFO | } | 20260505 18:42:50.918 | INFO | Perf report for the last 5.14172 seconds: | 20260505 18:42:50.918 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:42:50.918 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:50.918 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:50.918 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:50.918 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 18:42:50.918 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:50.918 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:50.918 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 18:42:50.918 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:50.918 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:50.918 | INFO | Uploaded 36GiB to GPU in 2.19525s (16.399GiB/s) | 20260505 18:42:50.918 | INFO | Downloaded 34GiB to GPU in 2.0442s (16.6325GiB/s) | 20260505 18:42:50.918 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 18:42:50.918 | INFO | Total mem usage: 15.3 GiB | 20260505 18:42:50.918 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 18:42:50.918 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.02%) | 20260505 18:42:50.918 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 18:42:50.918 | INFO | High Water Breakdown: | 20260505 18:42:50.918 | INFO | 61% : 8.66 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 18:42:50.918 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 18:42:50.918 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 18:42:50.918 | INFO | ------------------------------------------ | 20260505 18:42:50.918 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:42:50.918 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 18:42:50.918 | INFO | Total mem usage: 26.5 GiB | 20260505 18:42:50.918 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 18:42:50.918 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 18:42:50.918 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 18:42:50.918 | INFO | High Water Breakdown: | 20260505 18:42:50.918 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:42:50.918 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:42:50.918 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:42:50.918 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:42:50.918 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:42:50.918 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:42:50.918 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 18:42:50.918 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 18:42:50.919 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:42:50.919 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 18:42:50.919 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 18:42:50.919 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 18:42:50.919 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 18:42:50.919 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 18:42:50.919 | INFO | ------------------------------------------ | 20260505 18:42:50.919 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:42:50.919 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 18:42:50.919 | INFO | Total mem usage: 25.6 GiB | 20260505 18:42:50.919 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 18:42:50.919 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 18:42:50.919 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 18:42:50.919 | INFO | High Water Breakdown: | 20260505 18:42:50.919 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 18:42:50.919 | INFO | ------------------------------------------ | 20260505 18:42:50.919 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:42:50.919 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 18:42:50.919 | INFO | Total mem usage: 15.7 GiB | 20260505 18:42:50.919 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 18:42:50.919 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 18:42:50.919 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 18:42:50.919 | INFO | High Water Breakdown: | 20260505 18:42:50.919 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 18:42:50.919 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 18:42:50.919 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 18:42:50.919 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 18:42:50.919 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 18:42:50.919 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 18:42:50.919 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 18:42:50.919 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 18:42:50.919 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 18:42:50.919 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:42:50.919 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 18:42:50.919 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:42:50.919 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 18:42:50.919 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 18:42:50.919 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 18:42:50.919 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 18:42:50.919 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:42:50.919 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 18:42:50.919 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:42:50.919 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:42:50.919 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:42:50.919 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 18:42:50.919 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 18:42:50.919 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:42:50.919 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 18:42:50.919 | INFO | ------------------------------------------ | 20260505 18:42:50.919 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:42:50.919 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 18:42:50.919 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:42:50.919 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:42:50.919 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.47e+01, 1.34e+01 / 2.53e+00 | 20260505 18:42:50.919 | INFO | PerfReport: 0.29%, AggregateStats -- 9.25e-02 / 1.03e-01, 1.03e-02 / 7.75e-03 | 20260505 18:42:50.919 | INFO | PerfReport: 11.77%, Baseline -- 3.72e+00 / 4.54e+00, 5.63e-01 / 1.80e-01 | 20260505 18:42:50.919 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.49e+00, 1.87e-01 / 1.32e-01 | 20260505 18:42:50.919 | INFO | PerfReport: 15.94%, DME -- 4.02e+01 / 4.45e+01, 4.13e+00 / 2.57e+00 | 20260505 18:42:50.919 | INFO | PerfReport: 56.98%, FrameLabeling -- 1.80e+01 / 1.93e+01, 1.50e+00 / 1.08e+00 | 20260505 18:42:50.919 | INFO | PerfReport: 7.28%, PulseAccumulating -- 2.30e+00 / 2.89e+00, 4.42e-01 / 2.19e-01 | 20260505 18:42:50.919 | INFO | PerfReport: 3.31%, Metrics -- 1.04e+00 / 1.38e+00, 2.87e-01 / 1.59e-01 | 20260505 18:42:50.919 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.65e-01 / 4.27e-01, 6.48e-02 / 3.96e-02 | 20260505 18:42:50.919 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 18:42:50.919 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:42:50.919 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:42:50.919 | INFO | PerfReport: Full Execution -- 9.98e+01 / -nan, 2.54e+01 / 0.00e+00 | 20260505 18:42:50.919 | INFO | PerfReport: 22.53%, Upload -- 2.25e+01 / 0.00e+00, 7.79e+00 / 0.00e+00 | 20260505 18:42:50.919 | INFO | PerfReport: 21.79%, Download -- 2.17e+01 / 0.00e+00, 7.75e+00 / 0.00e+00 | 20260505 18:42:50.919 | INFO | PerfReport: 31.62%, Compute -- 3.16e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 18:42:50.919 | INFO | PerfReport: 2.30%, LockUpload -- 2.30e+00 / 0.00e+00, 7.98e+00 / 0.00e+00 | 20260505 18:42:50.919 | INFO | PerfReport: 1.42%, LockDownload -- 1.42e+00 / 0.00e+00, 2.97e+00 / 0.00e+00 | 20260505 18:42:50.919 | INFO | PerfReport: 20.33%, LockCompute -- 2.03e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 18:42:50.919 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.97e-02 / 0.00e+00 | 20260505 18:42:51.046 | INFO | Starting a round of Baz IO | 20260505 18:42:54.227 | INFO | Populating RTMetrics report | 20260505 18:42:54.228 | INFO | RTMetrics report populated | 20260505 18:42:54.228 | INFO | Opening tmp RTMetrics json file | 20260505 18:42:54.233 | INFO | Serializing RTMetrics to json | 20260505 18:42:54.235 | INFO | Serialization done, swapping tmp file now... | 20260505 18:42:54.235 | INFO | RTMetrics done | 20260505 18:42:54.312 | INFO | Finished a round of Baz IO: Wrote 2.086 GiB in 3.265 seconds | 20260505 18:42:54.312 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.87% / 0.00% / 78.13% | 20260505 18:42:54.312 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.239356% / 3.97364e-06% / 0.000115236% | 20260505 18:42:54.312 | INFO | Baz IO overall: | 20260505 18:42:54.312 | INFO | BytesWritten : 3203148692687 | 20260505 18:42:54.312 | INFO | NumEvents : 1516986570409 | 20260505 18:42:56.062 | INFO | Analyzing chunk frames = [5447680,5448192) | 20260505 18:42:56.062 | INFO | MemInfo: { | 20260505 18:42:56.062 | INFO | "gpu" : " 17 GB", | 20260505 18:42:56.062 | INFO | "huge" : "", | 20260505 18:42:56.062 | INFO | "huge_shared" : " 27 GB", | 20260505 18:42:56.062 | INFO | "malloc" : " 14 GB", | 20260505 18:42:56.062 | INFO | "pinned" : " 29 GB" | 20260505 18:42:56.062 | INFO | } | 20260505 18:42:56.062 | INFO | Perf report for the last 5.14436 seconds: | 20260505 18:42:56.062 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:42:56.062 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:56.062 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:56.062 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:56.062 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 18:42:56.062 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:42:56.062 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260505 18:42:56.062 | INFO | PRE_HQ : 60.5%, 1.8, 56.2ms, 0 | 20260505 18:42:56.062 | INFO | BAZWRITER : 63.0%, 3.8, 122.3ms, 0 | 20260505 18:42:56.062 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 18:42:56.062 | INFO | Uploaded 36GiB to GPU in 2.16266s (16.6462GiB/s) | 20260505 18:42:56.062 | INFO | Downloaded 38GiB to GPU in 2.24725s (16.9095GiB/s) | 20260505 18:43:01.187 | INFO | Analyzing chunk frames = [5448192,5448704) | 20260505 18:43:01.187 | INFO | MemInfo: { | 20260505 18:43:01.187 | INFO | "gpu" : " 17 GB", | 20260505 18:43:01.187 | INFO | "huge" : "", | 20260505 18:43:01.187 | INFO | "huge_shared" : " 27 GB", | 20260505 18:43:01.187 | INFO | "malloc" : " 14 GB", | 20260505 18:43:01.187 | INFO | "pinned" : " 29 GB" | 20260505 18:43:01.187 | INFO | } | 20260505 18:43:01.187 | INFO | Perf report for the last 5.12506 seconds: | 20260505 18:43:01.187 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:43:01.187 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:01.187 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:01.187 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:01.187 | INFO | ANALYSIS : 62.9%, 3.0, 98.0ms, 0 | 20260505 18:43:01.187 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:01.187 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:01.187 | INFO | PRE_HQ : 27.7%, 1.0, 14.3ms, 0 | 20260505 18:43:01.187 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:01.187 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:01.187 | INFO | Uploaded 36GiB to GPU in 2.15139s (16.7334GiB/s) | 20260505 18:43:01.187 | INFO | Downloaded 34GiB to GPU in 2.049s (16.5934GiB/s) | 20260505 18:43:06.332 | INFO | Analyzing chunk frames = [5448704,5449216) | 20260505 18:43:06.332 | INFO | MemInfo: { | 20260505 18:43:06.332 | INFO | "gpu" : " 17 GB", | 20260505 18:43:06.332 | INFO | "huge" : "", | 20260505 18:43:06.332 | INFO | "huge_shared" : " 27 GB", | 20260505 18:43:06.332 | INFO | "malloc" : " 14 GB", | 20260505 18:43:06.332 | INFO | "pinned" : " 29 GB" | 20260505 18:43:06.332 | INFO | } | 20260505 18:43:06.332 | INFO | Perf report for the last 5.14501 seconds: | 20260505 18:43:06.332 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:43:06.332 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:06.332 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:06.332 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:06.332 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260505 18:43:06.332 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:06.332 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:06.332 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:43:06.332 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:06.332 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:06.332 | INFO | Uploaded 36GiB to GPU in 2.19822s (16.3769GiB/s) | 20260505 18:43:06.332 | INFO | Downloaded 34GiB to GPU in 2.03826s (16.6809GiB/s) | 20260505 18:43:11.478 | INFO | Analyzing chunk frames = [5449216,5449728) | 20260505 18:43:11.478 | INFO | MemInfo: { | 20260505 18:43:11.478 | INFO | "gpu" : " 17 GB", | 20260505 18:43:11.478 | INFO | "huge" : "", | 20260505 18:43:11.478 | INFO | "huge_shared" : " 27 GB", | 20260505 18:43:11.478 | INFO | "malloc" : " 14 GB", | 20260505 18:43:11.478 | INFO | "pinned" : " 29 GB" | 20260505 18:43:11.478 | INFO | } | 20260505 18:43:11.478 | INFO | Perf report for the last 5.14573 seconds: | 20260505 18:43:11.478 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:43:11.478 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:11.478 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:11.478 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:11.478 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260505 18:43:11.478 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:11.478 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:11.478 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260505 18:43:11.478 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:11.478 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:11.478 | INFO | Uploaded 36GiB to GPU in 2.19456s (16.4042GiB/s) | 20260505 18:43:11.478 | INFO | Downloaded 34GiB to GPU in 2.05226s (16.5671GiB/s) | 20260505 18:43:16.608 | INFO | Analyzing chunk frames = [5449728,5450240) | 20260505 18:43:16.608 | INFO | MemInfo: { | 20260505 18:43:16.608 | INFO | "gpu" : " 17 GB", | 20260505 18:43:16.608 | INFO | "huge" : "", | 20260505 18:43:16.608 | INFO | "huge_shared" : " 27 GB", | 20260505 18:43:16.608 | INFO | "malloc" : " 14 GB", | 20260505 18:43:16.608 | INFO | "pinned" : " 29 GB" | 20260505 18:43:16.608 | INFO | } | 20260505 18:43:16.608 | INFO | Perf report for the last 5.1305 seconds: | 20260505 18:43:16.608 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:43:16.608 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:16.608 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:16.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:16.608 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260505 18:43:16.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:16.608 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:16.608 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 18:43:16.609 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:16.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:16.609 | INFO | Uploaded 36GiB to GPU in 2.12595s (16.9336GiB/s) | 20260505 18:43:16.609 | INFO | Downloaded 34GiB to GPU in 2.05869s (16.5154GiB/s) | 20260505 18:43:21.745 | INFO | Analyzing chunk frames = [5450240,5450752) | 20260505 18:43:21.745 | INFO | MemInfo: { | 20260505 18:43:21.745 | INFO | "gpu" : " 17 GB", | 20260505 18:43:21.745 | INFO | "huge" : "", | 20260505 18:43:21.745 | INFO | "huge_shared" : " 27 GB", | 20260505 18:43:21.745 | INFO | "malloc" : " 14 GB", | 20260505 18:43:21.745 | INFO | "pinned" : " 29 GB" | 20260505 18:43:21.745 | INFO | } | 20260505 18:43:21.745 | INFO | Perf report for the last 5.13711 seconds: | 20260505 18:43:21.746 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:43:21.746 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:21.746 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:21.746 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:21.746 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 18:43:21.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:21.746 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:21.746 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:43:21.746 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:21.746 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:21.746 | INFO | Uploaded 36GiB to GPU in 2.13359s (16.873GiB/s) | 20260505 18:43:21.746 | INFO | Downloaded 34GiB to GPU in 2.04491s (16.6267GiB/s) | 20260505 18:43:26.885 | INFO | Analyzing chunk frames = [5450752,5451264) | 20260505 18:43:26.885 | INFO | MemInfo: { | 20260505 18:43:26.885 | INFO | "gpu" : " 17 GB", | 20260505 18:43:26.885 | INFO | "huge" : "", | 20260505 18:43:26.885 | INFO | "huge_shared" : " 27 GB", | 20260505 18:43:26.885 | INFO | "malloc" : " 14 GB", | 20260505 18:43:26.885 | INFO | "pinned" : " 29 GB" | 20260505 18:43:26.885 | INFO | } | 20260505 18:43:26.885 | INFO | Perf report for the last 5.13965 seconds: | 20260505 18:43:26.885 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:43:26.885 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:26.885 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:26.885 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:26.885 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 18:43:26.885 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:26.885 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:26.885 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 18:43:26.885 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:26.885 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:26.885 | INFO | Uploaded 36GiB to GPU in 2.13424s (16.8678GiB/s) | 20260505 18:43:26.886 | INFO | Downloaded 34GiB to GPU in 2.13891s (15.896GiB/s) | 20260505 18:43:32.031 | INFO | Analyzing chunk frames = [5451264,5451776) | 20260505 18:43:32.031 | INFO | MemInfo: { | 20260505 18:43:32.031 | INFO | "gpu" : " 17 GB", | 20260505 18:43:32.031 | INFO | "huge" : "", | 20260505 18:43:32.031 | INFO | "huge_shared" : " 27 GB", | 20260505 18:43:32.031 | INFO | "malloc" : " 14 GB", | 20260505 18:43:32.031 | INFO | "pinned" : " 29 GB" | 20260505 18:43:32.031 | INFO | } | 20260505 18:43:32.031 | INFO | Perf report for the last 5.14587 seconds: | 20260505 18:43:32.031 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:43:32.031 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:32.031 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:32.031 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:32.031 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 18:43:32.031 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:32.031 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:32.031 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 18:43:32.031 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:32.031 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:32.031 | INFO | Uploaded 36GiB to GPU in 2.199s (16.3711GiB/s) | 20260505 18:43:32.031 | INFO | Downloaded 34GiB to GPU in 2.03764s (16.686GiB/s) | 20260505 18:43:32.161 | INFO | Starting a round of Baz IO | 20260505 18:43:35.336 | INFO | Populating RTMetrics report | 20260505 18:43:35.337 | INFO | RTMetrics report populated | 20260505 18:43:35.337 | INFO | Opening tmp RTMetrics json file | 20260505 18:43:35.340 | INFO | Serializing RTMetrics to json | 20260505 18:43:35.341 | INFO | Serialization done, swapping tmp file now... | 20260505 18:43:35.341 | INFO | RTMetrics done | 20260505 18:43:35.431 | INFO | Finished a round of Baz IO: Wrote 2.085 GiB in 3.27 seconds | 20260505 18:43:35.431 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.87% / 0.00% / 78.13% | 20260505 18:43:35.431 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.239742% / 0% / 0.000107288% | 20260505 18:43:35.431 | INFO | Baz IO overall: | 20260505 18:43:35.431 | INFO | BytesWritten : 3205387150947 | 20260505 18:43:35.431 | INFO | NumEvents : 1518028047830 | 20260505 18:43:37.171 | INFO | Analyzing chunk frames = [5451776,5452288) | 20260505 18:43:37.171 | INFO | MemInfo: { | 20260505 18:43:37.171 | INFO | "gpu" : " 17 GB", | 20260505 18:43:37.171 | INFO | "huge" : "", | 20260505 18:43:37.171 | INFO | "huge_shared" : " 27 GB", | 20260505 18:43:37.171 | INFO | "malloc" : " 14 GB", | 20260505 18:43:37.171 | INFO | "pinned" : " 29 GB" | 20260505 18:43:37.171 | INFO | } | 20260505 18:43:37.171 | INFO | Perf report for the last 5.13987 seconds: | 20260505 18:43:37.171 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:43:37.171 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:37.171 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:37.171 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:37.171 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 18:43:37.171 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:37.171 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260505 18:43:37.171 | INFO | PRE_HQ : 60.3%, 1.8, 55.9ms, 0 | 20260505 18:43:37.171 | INFO | BAZWRITER : 63.1%, 3.8, 121.9ms, 0 | 20260505 18:43:37.171 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 18:43:37.171 | INFO | Uploaded 36GiB to GPU in 2.16932s (16.5951GiB/s) | 20260505 18:43:37.171 | INFO | Downloaded 38GiB to GPU in 2.24514s (16.9254GiB/s) | 20260505 18:43:42.301 | INFO | Analyzing chunk frames = [5452288,5452800) | 20260505 18:43:42.301 | INFO | MemInfo: { | 20260505 18:43:42.301 | INFO | "gpu" : " 17 GB", | 20260505 18:43:42.301 | INFO | "huge" : "", | 20260505 18:43:42.301 | INFO | "huge_shared" : " 27 GB", | 20260505 18:43:42.301 | INFO | "malloc" : " 14 GB", | 20260505 18:43:42.301 | INFO | "pinned" : " 29 GB" | 20260505 18:43:42.301 | INFO | } | 20260505 18:43:42.301 | INFO | Perf report for the last 5.13033 seconds: | 20260505 18:43:42.301 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:43:42.301 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:42.301 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:42.301 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:42.301 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260505 18:43:42.301 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:42.301 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:42.301 | INFO | PRE_HQ : 27.5%, 1.0, 14.2ms, 0 | 20260505 18:43:42.301 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:42.301 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:42.302 | INFO | Uploaded 36GiB to GPU in 2.16353s (16.6395GiB/s) | 20260505 18:43:42.302 | INFO | Downloaded 34GiB to GPU in 2.05295s (16.5615GiB/s) | 20260505 18:43:47.438 | INFO | Analyzing chunk frames = [5452800,5453312) | 20260505 18:43:47.438 | INFO | MemInfo: { | 20260505 18:43:47.438 | INFO | "gpu" : " 17 GB", | 20260505 18:43:47.438 | INFO | "huge" : "", | 20260505 18:43:47.438 | INFO | "huge_shared" : " 27 GB", | 20260505 18:43:47.438 | INFO | "malloc" : " 14 GB", | 20260505 18:43:47.438 | INFO | "pinned" : " 29 GB" | 20260505 18:43:47.438 | INFO | } | 20260505 18:43:47.438 | INFO | Perf report for the last 5.13657 seconds: | 20260505 18:43:47.438 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:43:47.438 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:47.438 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:47.438 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:47.438 | INFO | ANALYSIS : 62.6%, 3.0, 97.4ms, 0 | 20260505 18:43:47.438 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:47.438 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:47.438 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:43:47.438 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:47.438 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:47.438 | INFO | Uploaded 36GiB to GPU in 2.20838s (16.3016GiB/s) | 20260505 18:43:47.438 | INFO | Downloaded 34GiB to GPU in 2.06897s (16.4333GiB/s) | 20260505 18:43:52.586 | INFO | Analyzing chunk frames = [5453312,5453824) | 20260505 18:43:52.586 | INFO | MemInfo: { | 20260505 18:43:52.586 | INFO | "gpu" : " 17 GB", | 20260505 18:43:52.586 | INFO | "huge" : "", | 20260505 18:43:52.586 | INFO | "huge_shared" : " 27 GB", | 20260505 18:43:52.586 | INFO | "malloc" : " 14 GB", | 20260505 18:43:52.586 | INFO | "pinned" : " 29 GB" | 20260505 18:43:52.586 | INFO | } | 20260505 18:43:52.586 | INFO | Perf report for the last 5.14796 seconds: | 20260505 18:43:52.586 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:43:52.586 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:52.586 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:52.586 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:52.586 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260505 18:43:52.586 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:52.586 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:52.586 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260505 18:43:52.586 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:52.586 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:52.586 | INFO | Uploaded 36GiB to GPU in 2.14966s (16.7468GiB/s) | 20260505 18:43:52.586 | INFO | Downloaded 34GiB to GPU in 2.12805s (15.9771GiB/s) | 20260505 18:43:57.724 | INFO | Analyzing chunk frames = [5453824,5454336) | 20260505 18:43:57.724 | INFO | MemInfo: { | 20260505 18:43:57.724 | INFO | "gpu" : " 17 GB", | 20260505 18:43:57.724 | INFO | "huge" : "", | 20260505 18:43:57.724 | INFO | "huge_shared" : " 27 GB", | 20260505 18:43:57.724 | INFO | "malloc" : " 14 GB", | 20260505 18:43:57.724 | INFO | "pinned" : " 29 GB" | 20260505 18:43:57.724 | INFO | } | 20260505 18:43:57.724 | INFO | Perf report for the last 5.13852 seconds: | 20260505 18:43:57.724 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:43:57.724 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:57.724 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:57.724 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:57.724 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 18:43:57.724 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:57.724 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:57.725 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:43:57.725 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:57.725 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:43:57.725 | INFO | Uploaded 36GiB to GPU in 2.14145s (16.811GiB/s) | 20260505 18:43:57.725 | INFO | Downloaded 34GiB to GPU in 2.05716s (16.5276GiB/s) | 20260505 18:44:02.868 | INFO | Analyzing chunk frames = [5454336,5454848) | 20260505 18:44:02.868 | INFO | MemInfo: { | 20260505 18:44:02.868 | INFO | "gpu" : " 17 GB", | 20260505 18:44:02.868 | INFO | "huge" : "", | 20260505 18:44:02.868 | INFO | "huge_shared" : " 27 GB", | 20260505 18:44:02.868 | INFO | "malloc" : " 14 GB", | 20260505 18:44:02.868 | INFO | "pinned" : " 29 GB" | 20260505 18:44:02.869 | INFO | } | 20260505 18:44:02.869 | INFO | Perf report for the last 5.14421 seconds: | 20260505 18:44:02.869 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:44:02.869 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:02.869 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:02.869 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:02.869 | INFO | ANALYSIS : 62.1%, 3.0, 97.1ms, 0 | 20260505 18:44:02.869 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:02.869 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:02.869 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:44:02.869 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:02.869 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:02.869 | INFO | Uploaded 36GiB to GPU in 2.13129s (16.8911GiB/s) | 20260505 18:44:02.869 | INFO | Downloaded 34GiB to GPU in 2.04934s (16.5907GiB/s) | 20260505 18:44:08.008 | INFO | Analyzing chunk frames = [5454848,5455360) | 20260505 18:44:08.008 | INFO | MemInfo: { | 20260505 18:44:08.008 | INFO | "gpu" : " 17 GB", | 20260505 18:44:08.008 | INFO | "huge" : "", | 20260505 18:44:08.008 | INFO | "huge_shared" : " 27 GB", | 20260505 18:44:08.008 | INFO | "malloc" : " 14 GB", | 20260505 18:44:08.008 | INFO | "pinned" : " 29 GB" | 20260505 18:44:08.008 | INFO | } | 20260505 18:44:08.008 | INFO | Perf report for the last 5.13946 seconds: | 20260505 18:44:08.008 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:44:08.008 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:08.008 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:08.008 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:08.008 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 18:44:08.008 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:08.008 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:08.008 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 18:44:08.008 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:08.008 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:08.008 | INFO | Uploaded 36GiB to GPU in 2.12934s (16.9066GiB/s) | 20260505 18:44:08.008 | INFO | Downloaded 34GiB to GPU in 2.14588s (15.8443GiB/s) | 20260505 18:44:13.141 | INFO | Analyzing chunk frames = [5455360,5455872) | 20260505 18:44:13.141 | INFO | MemInfo: { | 20260505 18:44:13.141 | INFO | "gpu" : " 17 GB", | 20260505 18:44:13.141 | INFO | "huge" : "", | 20260505 18:44:13.141 | INFO | "huge_shared" : " 27 GB", | 20260505 18:44:13.141 | INFO | "malloc" : " 14 GB", | 20260505 18:44:13.141 | INFO | "pinned" : " 29 GB" | 20260505 18:44:13.141 | INFO | } | 20260505 18:44:13.141 | INFO | Perf report for the last 5.13326 seconds: | 20260505 18:44:13.141 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:44:13.141 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:13.141 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:13.141 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:13.141 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 18:44:13.141 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:13.141 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:13.141 | INFO | PRE_HQ : 27.7%, 1.0, 14.3ms, 0 | 20260505 18:44:13.141 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:13.141 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:13.142 | INFO | Uploaded 36GiB to GPU in 2.19746s (16.3826GiB/s) | 20260505 18:44:13.142 | INFO | Downloaded 34GiB to GPU in 2.02639s (16.7786GiB/s) | 20260505 18:44:13.142 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 18:44:13.142 | INFO | Total mem usage: 15.3 GiB | 20260505 18:44:13.142 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 18:44:13.142 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.06%) | 20260505 18:44:13.142 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 18:44:13.142 | INFO | High Water Breakdown: | 20260505 18:44:13.142 | INFO | 61% : 8.66 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 18:44:13.142 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 18:44:13.142 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 18:44:13.142 | INFO | ------------------------------------------ | 20260505 18:44:13.142 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:44:13.142 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 18:44:13.142 | INFO | Total mem usage: 26.5 GiB | 20260505 18:44:13.142 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 18:44:13.142 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 18:44:13.142 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 18:44:13.142 | INFO | High Water Breakdown: | 20260505 18:44:13.142 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:44:13.142 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:44:13.142 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:44:13.142 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:44:13.142 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:44:13.142 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:44:13.142 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 18:44:13.142 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 18:44:13.142 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:44:13.142 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 18:44:13.142 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 18:44:13.142 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 18:44:13.142 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 18:44:13.142 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 18:44:13.142 | INFO | ------------------------------------------ | 20260505 18:44:13.142 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:44:13.142 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 18:44:13.142 | INFO | Total mem usage: 25.6 GiB | 20260505 18:44:13.142 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 18:44:13.142 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 18:44:13.142 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 18:44:13.142 | INFO | High Water Breakdown: | 20260505 18:44:13.142 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 18:44:13.142 | INFO | ------------------------------------------ | 20260505 18:44:13.142 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:44:13.142 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 18:44:13.142 | INFO | Total mem usage: 15.7 GiB | 20260505 18:44:13.142 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 18:44:13.142 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 18:44:13.142 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 18:44:13.142 | INFO | High Water Breakdown: | 20260505 18:44:13.142 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 18:44:13.142 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 18:44:13.142 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 18:44:13.142 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 18:44:13.142 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 18:44:13.142 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 18:44:13.142 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 18:44:13.142 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 18:44:13.142 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 18:44:13.142 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:44:13.142 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 18:44:13.142 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:44:13.142 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 18:44:13.142 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 18:44:13.142 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 18:44:13.142 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 18:44:13.142 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:44:13.142 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 18:44:13.142 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:44:13.142 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:44:13.142 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:44:13.142 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 18:44:13.142 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 18:44:13.142 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:44:13.142 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 18:44:13.142 | INFO | ------------------------------------------ | 20260505 18:44:13.142 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:44:13.143 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 18:44:13.143 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:44:13.143 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:44:13.143 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.45e+01, 1.36e+01 / 2.38e+00 | 20260505 18:44:13.143 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 1.02e-01, 1.12e-02 / 4.17e-03 | 20260505 18:44:13.143 | INFO | PerfReport: 11.81%, Baseline -- 3.72e+00 / 4.57e+00, 5.68e-01 / 8.67e-02 | 20260505 18:44:13.143 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.54e+00, 1.91e-01 / 9.48e-02 | 20260505 18:44:13.143 | INFO | PerfReport: 15.93%, DME -- 4.01e+01 / 4.45e+01, 4.17e+00 / 2.40e+00 | 20260505 18:44:13.143 | INFO | PerfReport: 56.83%, FrameLabeling -- 1.79e+01 / 1.91e+01, 1.38e+00 / 8.26e-01 | 20260505 18:44:13.143 | INFO | PerfReport: 7.35%, PulseAccumulating -- 2.32e+00 / 2.91e+00, 4.47e-01 / 1.68e-01 | 20260505 18:44:13.143 | INFO | PerfReport: 3.33%, Metrics -- 1.05e+00 / 1.39e+00, 2.91e-01 / 1.76e-01 | 20260505 18:44:13.143 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.65e-01 / 4.30e-01, 6.43e-02 / 4.22e-02 | 20260505 18:44:13.143 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 18:44:13.143 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:44:13.143 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:44:13.143 | INFO | PerfReport: Full Execution -- 9.97e+01 / -nan, 2.55e+01 / 0.00e+00 | 20260505 18:44:13.143 | INFO | PerfReport: 22.59%, Upload -- 2.25e+01 / 0.00e+00, 7.80e+00 / 0.00e+00 | 20260505 18:44:13.143 | INFO | PerfReport: 21.84%, Download -- 2.18e+01 / 0.00e+00, 7.79e+00 / 0.00e+00 | 20260505 18:44:13.143 | INFO | PerfReport: 31.60%, Compute -- 3.15e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 18:44:13.143 | INFO | PerfReport: 2.26%, LockUpload -- 2.25e+00 / 0.00e+00, 7.90e+00 / 0.00e+00 | 20260505 18:44:13.143 | INFO | PerfReport: 1.42%, LockDownload -- 1.41e+00 / 0.00e+00, 2.86e+00 / 0.00e+00 | 20260505 18:44:13.143 | INFO | PerfReport: 20.28%, LockCompute -- 2.02e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 18:44:13.143 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.74e-02 / 0.00e+00 | 20260505 18:44:13.274 | INFO | Starting a round of Baz IO | 20260505 18:44:16.453 | INFO | Populating RTMetrics report | 20260505 18:44:16.454 | INFO | RTMetrics report populated | 20260505 18:44:16.454 | INFO | Opening tmp RTMetrics json file | 20260505 18:44:16.455 | INFO | Serializing RTMetrics to json | 20260505 18:44:16.456 | INFO | Serialization done, swapping tmp file now... | 20260505 18:44:16.456 | INFO | RTMetrics done | 20260505 18:44:16.547 | INFO | Finished a round of Baz IO: Wrote 2.084 GiB in 3.272 seconds | 20260505 18:44:16.547 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.87% / 0.00% / 78.13% | 20260505 18:44:16.547 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.239495% / 7.94729e-06% / 9.53674e-05% | 20260505 18:44:16.547 | INFO | Baz IO overall: | 20260505 18:44:16.547 | INFO | BytesWritten : 3207625000142 | 20260505 18:44:16.547 | INFO | NumEvents : 1519069862625 | 20260505 18:44:18.285 | INFO | Analyzing chunk frames = [5455872,5456384) | 20260505 18:44:18.285 | INFO | MemInfo: { | 20260505 18:44:18.285 | INFO | "gpu" : " 17 GB", | 20260505 18:44:18.285 | INFO | "huge" : "", | 20260505 18:44:18.285 | INFO | "huge_shared" : " 27 GB", | 20260505 18:44:18.285 | INFO | "malloc" : " 14 GB", | 20260505 18:44:18.285 | INFO | "pinned" : " 29 GB" | 20260505 18:44:18.285 | INFO | } | 20260505 18:44:18.285 | INFO | Perf report for the last 5.14353 seconds: | 20260505 18:44:18.285 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:44:18.285 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:18.285 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:18.285 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:18.285 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 18:44:18.285 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:18.285 | INFO | RT_METRICS : 44.3%, 1.0, 23.3ms, 0 | 20260505 18:44:18.285 | INFO | PRE_HQ : 60.4%, 1.8, 55.8ms, 0 | 20260505 18:44:18.285 | INFO | BAZWRITER : 63.1%, 3.8, 122.8ms, 0 | 20260505 18:44:18.285 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 18:44:18.285 | INFO | Uploaded 36GiB to GPU in 2.16662s (16.6158GiB/s) | 20260505 18:44:18.285 | INFO | Downloaded 38GiB to GPU in 2.24431s (16.9317GiB/s) | 20260505 18:44:23.424 | INFO | Analyzing chunk frames = [5456384,5456896) | 20260505 18:44:23.424 | INFO | MemInfo: { | 20260505 18:44:23.424 | INFO | "gpu" : " 17 GB", | 20260505 18:44:23.424 | INFO | "huge" : "", | 20260505 18:44:23.424 | INFO | "huge_shared" : " 27 GB", | 20260505 18:44:23.424 | INFO | "malloc" : " 14 GB", | 20260505 18:44:23.424 | INFO | "pinned" : " 29 GB" | 20260505 18:44:23.424 | INFO | } | 20260505 18:44:23.424 | INFO | Perf report for the last 5.13926 seconds: | 20260505 18:44:23.424 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:44:23.424 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:23.424 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:23.424 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:23.424 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260505 18:44:23.424 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:23.424 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:23.424 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 18:44:23.424 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:23.424 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:23.424 | INFO | Uploaded 36GiB to GPU in 2.14975s (16.7461GiB/s) | 20260505 18:44:23.424 | INFO | Downloaded 34GiB to GPU in 2.0292s (16.7554GiB/s) | 20260505 18:44:28.558 | INFO | Analyzing chunk frames = [5456896,5457408) | 20260505 18:44:28.558 | INFO | MemInfo: { | 20260505 18:44:28.558 | INFO | "gpu" : " 17 GB", | 20260505 18:44:28.558 | INFO | "huge" : "", | 20260505 18:44:28.558 | INFO | "huge_shared" : " 27 GB", | 20260505 18:44:28.558 | INFO | "malloc" : " 14 GB", | 20260505 18:44:28.558 | INFO | "pinned" : " 29 GB" | 20260505 18:44:28.558 | INFO | } | 20260505 18:44:28.558 | INFO | Perf report for the last 5.13403 seconds: | 20260505 18:44:28.558 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:44:28.558 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:28.558 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:28.558 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:28.558 | INFO | ANALYSIS : 62.7%, 3.0, 97.5ms, 0 | 20260505 18:44:28.558 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:28.558 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:28.558 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:44:28.558 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:28.558 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:28.558 | INFO | Uploaded 36GiB to GPU in 2.21046s (16.2862GiB/s) | 20260505 18:44:28.558 | INFO | Downloaded 34GiB to GPU in 2.07423s (16.3916GiB/s) | 20260505 18:44:33.696 | INFO | Analyzing chunk frames = [5457408,5457920) | 20260505 18:44:33.696 | INFO | MemInfo: { | 20260505 18:44:33.696 | INFO | "gpu" : " 17 GB", | 20260505 18:44:33.696 | INFO | "huge" : "", | 20260505 18:44:33.696 | INFO | "huge_shared" : " 27 GB", | 20260505 18:44:33.696 | INFO | "malloc" : " 14 GB", | 20260505 18:44:33.696 | INFO | "pinned" : " 29 GB" | 20260505 18:44:33.696 | INFO | } | 20260505 18:44:33.697 | INFO | Perf report for the last 5.13834 seconds: | 20260505 18:44:33.697 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:44:33.697 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:33.697 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:33.697 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:33.697 | INFO | ANALYSIS : 62.8%, 3.0, 97.5ms, 0 | 20260505 18:44:33.697 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:33.697 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:33.697 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:44:33.697 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:33.697 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:33.697 | INFO | Uploaded 36GiB to GPU in 2.22764s (16.1606GiB/s) | 20260505 18:44:33.697 | INFO | Downloaded 34GiB to GPU in 2.12334s (16.0125GiB/s) | 20260505 18:44:38.842 | INFO | Analyzing chunk frames = [5457920,5458432) | 20260505 18:44:38.842 | INFO | MemInfo: { | 20260505 18:44:38.842 | INFO | "gpu" : " 17 GB", | 20260505 18:44:38.842 | INFO | "huge" : "", | 20260505 18:44:38.842 | INFO | "huge_shared" : " 27 GB", | 20260505 18:44:38.842 | INFO | "malloc" : " 14 GB", | 20260505 18:44:38.842 | INFO | "pinned" : " 29 GB" | 20260505 18:44:38.842 | INFO | } | 20260505 18:44:38.843 | INFO | Perf report for the last 5.14597 seconds: | 20260505 18:44:38.843 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:44:38.843 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:38.843 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:38.843 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:38.843 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 18:44:38.843 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:38.843 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:38.843 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:44:38.843 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:38.843 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:38.843 | INFO | Uploaded 36GiB to GPU in 2.15195s (16.729GiB/s) | 20260505 18:44:38.843 | INFO | Downloaded 34GiB to GPU in 2.06097s (16.4971GiB/s) | 20260505 18:44:43.985 | INFO | Analyzing chunk frames = [5458432,5458944) | 20260505 18:44:43.985 | INFO | MemInfo: { | 20260505 18:44:43.985 | INFO | "gpu" : " 17 GB", | 20260505 18:44:43.985 | INFO | "huge" : "", | 20260505 18:44:43.985 | INFO | "huge_shared" : " 27 GB", | 20260505 18:44:43.985 | INFO | "malloc" : " 14 GB", | 20260505 18:44:43.985 | INFO | "pinned" : " 29 GB" | 20260505 18:44:43.985 | INFO | } | 20260505 18:44:43.985 | INFO | Perf report for the last 5.14272 seconds: | 20260505 18:44:43.985 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:44:43.985 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:43.985 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:43.985 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:43.985 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 18:44:43.985 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:43.985 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:43.985 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 18:44:43.985 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:43.985 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:43.985 | INFO | Uploaded 36GiB to GPU in 2.14859s (16.7552GiB/s) | 20260505 18:44:43.985 | INFO | Downloaded 34GiB to GPU in 2.05792s (16.5215GiB/s) | 20260505 18:44:49.119 | INFO | Analyzing chunk frames = [5458944,5459456) | 20260505 18:44:49.119 | INFO | MemInfo: { | 20260505 18:44:49.119 | INFO | "gpu" : " 17 GB", | 20260505 18:44:49.119 | INFO | "huge" : "", | 20260505 18:44:49.119 | INFO | "huge_shared" : " 27 GB", | 20260505 18:44:49.119 | INFO | "malloc" : " 14 GB", | 20260505 18:44:49.119 | INFO | "pinned" : " 29 GB" | 20260505 18:44:49.120 | INFO | } | 20260505 18:44:49.120 | INFO | Perf report for the last 5.13431 seconds: | 20260505 18:44:49.120 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:44:49.120 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:49.120 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:49.120 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:49.120 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 18:44:49.120 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:49.120 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:49.120 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 18:44:49.120 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:49.120 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:49.120 | INFO | Uploaded 36GiB to GPU in 2.12545s (16.9376GiB/s) | 20260505 18:44:49.120 | INFO | Downloaded 34GiB to GPU in 2.14694s (15.8365GiB/s) | 20260505 18:44:54.253 | INFO | Analyzing chunk frames = [5459456,5459968) | 20260505 18:44:54.253 | INFO | MemInfo: { | 20260505 18:44:54.253 | INFO | "gpu" : " 17 GB", | 20260505 18:44:54.253 | INFO | "huge" : "", | 20260505 18:44:54.253 | INFO | "huge_shared" : " 27 GB", | 20260505 18:44:54.253 | INFO | "malloc" : " 14 GB", | 20260505 18:44:54.253 | INFO | "pinned" : " 29 GB" | 20260505 18:44:54.253 | INFO | } | 20260505 18:44:54.253 | INFO | Perf report for the last 5.13353 seconds: | 20260505 18:44:54.253 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:44:54.253 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:54.253 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:54.253 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:54.253 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260505 18:44:54.253 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:54.253 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:54.253 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 18:44:54.253 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:54.253 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:54.253 | INFO | Uploaded 36GiB to GPU in 2.20184s (16.3499GiB/s) | 20260505 18:44:54.253 | INFO | Downloaded 34GiB to GPU in 2.02538s (16.787GiB/s) | 20260505 18:44:54.389 | INFO | Starting a round of Baz IO | 20260505 18:44:57.570 | INFO | Populating RTMetrics report | 20260505 18:44:57.570 | INFO | RTMetrics report populated | 20260505 18:44:57.570 | INFO | Opening tmp RTMetrics json file | 20260505 18:44:57.571 | INFO | Serializing RTMetrics to json | 20260505 18:44:57.572 | INFO | Serialization done, swapping tmp file now... | 20260505 18:44:57.573 | INFO | RTMetrics done | 20260505 18:44:57.662 | INFO | Finished a round of Baz IO: Wrote 2.081 GiB in 3.273 seconds | 20260505 18:44:57.662 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.87% / 0.00% / 78.13% | 20260505 18:44:57.662 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.238355% / 0% / 8.34465e-05% | 20260505 18:44:57.662 | INFO | Baz IO overall: | 20260505 18:44:57.662 | INFO | BytesWritten : 3209859170675 | 20260505 18:44:57.662 | INFO | NumEvents : 1520109727282 | 20260505 18:44:59.393 | INFO | Analyzing chunk frames = [5459968,5460480) | 20260505 18:44:59.393 | INFO | MemInfo: { | 20260505 18:44:59.393 | INFO | "gpu" : " 17 GB", | 20260505 18:44:59.393 | INFO | "huge" : "", | 20260505 18:44:59.393 | INFO | "huge_shared" : " 27 GB", | 20260505 18:44:59.393 | INFO | "malloc" : " 14 GB", | 20260505 18:44:59.393 | INFO | "pinned" : " 29 GB" | 20260505 18:44:59.393 | INFO | } | 20260505 18:44:59.393 | INFO | Perf report for the last 5.13992 seconds: | 20260505 18:44:59.393 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:44:59.393 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:59.393 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:59.393 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:59.393 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 18:44:59.393 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:44:59.393 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260505 18:44:59.393 | INFO | PRE_HQ : 60.2%, 1.8, 55.8ms, 0 | 20260505 18:44:59.393 | INFO | BAZWRITER : 63.1%, 3.8, 122.0ms, 0 | 20260505 18:44:59.393 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 18:44:59.393 | INFO | Uploaded 36GiB to GPU in 2.16774s (16.6071GiB/s) | 20260505 18:44:59.393 | INFO | Downloaded 38GiB to GPU in 2.25597s (16.8442GiB/s) | 20260505 18:45:04.534 | INFO | Analyzing chunk frames = [5460480,5460992) | 20260505 18:45:04.534 | INFO | MemInfo: { | 20260505 18:45:04.534 | INFO | "gpu" : " 17 GB", | 20260505 18:45:04.534 | INFO | "huge" : "", | 20260505 18:45:04.534 | INFO | "huge_shared" : " 27 GB", | 20260505 18:45:04.534 | INFO | "malloc" : " 14 GB", | 20260505 18:45:04.534 | INFO | "pinned" : " 29 GB" | 20260505 18:45:04.534 | INFO | } | 20260505 18:45:04.534 | INFO | Perf report for the last 5.14133 seconds: | 20260505 18:45:04.534 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:45:04.534 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:04.534 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:04.534 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:04.534 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 18:45:04.534 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:04.535 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:04.535 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:45:04.535 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:04.535 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:04.535 | INFO | Uploaded 36GiB to GPU in 2.13311s (16.8767GiB/s) | 20260505 18:45:04.535 | INFO | Downloaded 34GiB to GPU in 2.02517s (16.7887GiB/s) | 20260505 18:45:09.681 | INFO | Analyzing chunk frames = [5460992,5461504) | 20260505 18:45:09.681 | INFO | MemInfo: { | 20260505 18:45:09.681 | INFO | "gpu" : " 17 GB", | 20260505 18:45:09.681 | INFO | "huge" : "", | 20260505 18:45:09.681 | INFO | "huge_shared" : " 27 GB", | 20260505 18:45:09.681 | INFO | "malloc" : " 14 GB", | 20260505 18:45:09.681 | INFO | "pinned" : " 29 GB" | 20260505 18:45:09.681 | INFO | } | 20260505 18:45:09.681 | INFO | Perf report for the last 5.14673 seconds: | 20260505 18:45:09.681 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:45:09.681 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:09.681 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:09.681 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:09.681 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260505 18:45:09.681 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:09.681 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:09.681 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:45:09.681 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:09.681 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:09.681 | INFO | Uploaded 36GiB to GPU in 2.14626s (16.7734GiB/s) | 20260505 18:45:09.682 | INFO | Downloaded 34GiB to GPU in 2.04517s (16.6245GiB/s) | 20260505 18:45:14.813 | INFO | Analyzing chunk frames = [5461504,5462016) | 20260505 18:45:14.813 | INFO | MemInfo: { | 20260505 18:45:14.813 | INFO | "gpu" : " 17 GB", | 20260505 18:45:14.813 | INFO | "huge" : "", | 20260505 18:45:14.813 | INFO | "huge_shared" : " 27 GB", | 20260505 18:45:14.813 | INFO | "malloc" : " 14 GB", | 20260505 18:45:14.813 | INFO | "pinned" : " 29 GB" | 20260505 18:45:14.813 | INFO | } | 20260505 18:45:14.813 | INFO | Perf report for the last 5.13199 seconds: | 20260505 18:45:14.813 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:45:14.813 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:14.813 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:14.813 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:14.813 | INFO | ANALYSIS : 62.4%, 3.0, 96.8ms, 0 | 20260505 18:45:14.813 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:14.813 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:14.813 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:45:14.813 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:14.813 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:14.813 | INFO | Uploaded 36GiB to GPU in 2.20443s (16.3308GiB/s) | 20260505 18:45:14.813 | INFO | Downloaded 34GiB to GPU in 2.03679s (16.693GiB/s) | 20260505 18:45:19.952 | INFO | Analyzing chunk frames = [5462016,5462528) | 20260505 18:45:19.952 | INFO | MemInfo: { | 20260505 18:45:19.952 | INFO | "gpu" : " 17 GB", | 20260505 18:45:19.952 | INFO | "huge" : "", | 20260505 18:45:19.952 | INFO | "huge_shared" : " 27 GB", | 20260505 18:45:19.952 | INFO | "malloc" : " 14 GB", | 20260505 18:45:19.952 | INFO | "pinned" : " 29 GB" | 20260505 18:45:19.952 | INFO | } | 20260505 18:45:19.952 | INFO | Perf report for the last 5.13915 seconds: | 20260505 18:45:19.952 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:45:19.952 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:19.952 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:19.952 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:19.952 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 18:45:19.952 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:19.952 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:19.952 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:45:19.952 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:19.952 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:19.953 | INFO | Uploaded 36GiB to GPU in 2.14927s (16.7499GiB/s) | 20260505 18:45:19.953 | INFO | Downloaded 34GiB to GPU in 2.05959s (16.5081GiB/s) | 20260505 18:45:25.098 | INFO | Analyzing chunk frames = [5462528,5463040) | 20260505 18:45:25.098 | INFO | MemInfo: { | 20260505 18:45:25.098 | INFO | "gpu" : " 17 GB", | 20260505 18:45:25.098 | INFO | "huge" : "", | 20260505 18:45:25.098 | INFO | "huge_shared" : " 27 GB", | 20260505 18:45:25.098 | INFO | "malloc" : " 14 GB", | 20260505 18:45:25.098 | INFO | "pinned" : " 29 GB" | 20260505 18:45:25.098 | INFO | } | 20260505 18:45:25.098 | INFO | Perf report for the last 5.14616 seconds: | 20260505 18:45:25.098 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:45:25.098 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:25.098 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:25.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:25.099 | INFO | ANALYSIS : 62.3%, 3.0, 97.4ms, 0 | 20260505 18:45:25.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:25.099 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:25.099 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 18:45:25.099 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:25.099 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:25.099 | INFO | Uploaded 36GiB to GPU in 2.1465s (16.7715GiB/s) | 20260505 18:45:25.099 | INFO | Downloaded 34GiB to GPU in 2.04973s (16.5876GiB/s) | 20260505 18:45:30.234 | INFO | Analyzing chunk frames = [5463040,5463552) | 20260505 18:45:30.234 | INFO | MemInfo: { | 20260505 18:45:30.234 | INFO | "gpu" : " 17 GB", | 20260505 18:45:30.234 | INFO | "huge" : "", | 20260505 18:45:30.234 | INFO | "huge_shared" : " 27 GB", | 20260505 18:45:30.234 | INFO | "malloc" : " 14 GB", | 20260505 18:45:30.234 | INFO | "pinned" : " 29 GB" | 20260505 18:45:30.234 | INFO | } | 20260505 18:45:30.234 | INFO | Perf report for the last 5.13587 seconds: | 20260505 18:45:30.234 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:45:30.234 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:30.234 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:30.234 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:30.234 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 18:45:30.234 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:30.234 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:30.234 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260505 18:45:30.234 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:30.234 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:30.235 | INFO | Uploaded 36GiB to GPU in 2.12607s (16.9326GiB/s) | 20260505 18:45:30.235 | INFO | Downloaded 34GiB to GPU in 2.14622s (15.8418GiB/s) | 20260505 18:45:35.370 | INFO | Analyzing chunk frames = [5463552,5464064) | 20260505 18:45:35.370 | INFO | MemInfo: { | 20260505 18:45:35.370 | INFO | "gpu" : " 17 GB", | 20260505 18:45:35.370 | INFO | "huge" : "", | 20260505 18:45:35.370 | INFO | "huge_shared" : " 27 GB", | 20260505 18:45:35.370 | INFO | "malloc" : " 14 GB", | 20260505 18:45:35.370 | INFO | "pinned" : " 29 GB" | 20260505 18:45:35.370 | INFO | } | 20260505 18:45:35.370 | INFO | Perf report for the last 5.13599 seconds: | 20260505 18:45:35.370 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:45:35.370 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:35.370 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:35.370 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:35.370 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 18:45:35.370 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:35.370 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:35.370 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 18:45:35.370 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:35.371 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:35.371 | INFO | Uploaded 36GiB to GPU in 2.20446s (16.3305GiB/s) | 20260505 18:45:35.371 | INFO | Downloaded 34GiB to GPU in 2.04873s (16.5956GiB/s) | 20260505 18:45:35.371 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 18:45:35.371 | INFO | Total mem usage: 15.3 GiB | 20260505 18:45:35.371 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 18:45:35.371 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.06%) | 20260505 18:45:35.371 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 18:45:35.371 | INFO | High Water Breakdown: | 20260505 18:45:35.371 | INFO | 61% : 8.66 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 18:45:35.371 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 18:45:35.371 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 18:45:35.371 | INFO | ------------------------------------------ | 20260505 18:45:35.371 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:45:35.371 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 18:45:35.371 | INFO | Total mem usage: 26.5 GiB | 20260505 18:45:35.371 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 18:45:35.371 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 18:45:35.371 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260505 18:45:35.371 | INFO | High Water Breakdown: | 20260505 18:45:35.371 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:45:35.371 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:45:35.371 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:45:35.371 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:45:35.371 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:45:35.371 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:45:35.371 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 18:45:35.371 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 18:45:35.371 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:45:35.371 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 18:45:35.371 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 18:45:35.371 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 18:45:35.371 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 18:45:35.371 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 18:45:35.371 | INFO | ------------------------------------------ | 20260505 18:45:35.371 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:45:35.371 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 18:45:35.371 | INFO | Total mem usage: 25.6 GiB | 20260505 18:45:35.371 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 18:45:35.371 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 18:45:35.371 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 18:45:35.371 | INFO | High Water Breakdown: | 20260505 18:45:35.371 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 18:45:35.371 | INFO | ------------------------------------------ | 20260505 18:45:35.371 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:45:35.371 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 18:45:35.371 | INFO | Total mem usage: 15.7 GiB | 20260505 18:45:35.371 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 18:45:35.371 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 18:45:35.371 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 18:45:35.371 | INFO | High Water Breakdown: | 20260505 18:45:35.371 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 18:45:35.371 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 18:45:35.371 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 18:45:35.371 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 18:45:35.371 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 18:45:35.371 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 18:45:35.371 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 18:45:35.371 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 18:45:35.371 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 18:45:35.371 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:45:35.371 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 18:45:35.371 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:45:35.371 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 18:45:35.371 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 18:45:35.371 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 18:45:35.371 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 18:45:35.371 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:45:35.371 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 18:45:35.372 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:45:35.372 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:45:35.372 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:45:35.372 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 18:45:35.372 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 18:45:35.372 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:45:35.372 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 18:45:35.372 | INFO | ------------------------------------------ | 20260505 18:45:35.372 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:45:35.372 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 18:45:35.372 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:45:35.372 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:45:35.372 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.43e+01, 1.34e+01 / 1.72e+00 | 20260505 18:45:35.372 | INFO | PerfReport: 0.29%, AggregateStats -- 9.25e-02 / 1.03e-01, 1.04e-02 / 4.36e-03 | 20260505 18:45:35.372 | INFO | PerfReport: 11.78%, Baseline -- 3.71e+00 / 4.53e+00, 5.59e-01 / 1.52e-01 | 20260505 18:45:35.372 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.54e+00, 1.89e-01 / 7.92e-02 | 20260505 18:45:35.372 | INFO | PerfReport: 15.91%, DME -- 4.01e+01 / 4.42e+01, 3.98e+00 / 1.71e+00 | 20260505 18:45:35.372 | INFO | PerfReport: 56.92%, FrameLabeling -- 1.79e+01 / 1.93e+01, 1.42e+00 / 1.07e+00 | 20260505 18:45:35.372 | INFO | PerfReport: 7.34%, PulseAccumulating -- 2.31e+00 / 2.90e+00, 4.48e-01 / 1.92e-01 | 20260505 18:45:35.372 | INFO | PerfReport: 3.33%, Metrics -- 1.05e+00 / 1.38e+00, 2.90e-01 / 1.41e-01 | 20260505 18:45:35.372 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.64e-01 / 4.27e-01, 6.41e-02 / 3.09e-02 | 20260505 18:45:35.372 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 18:45:35.372 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:45:35.372 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:45:35.372 | INFO | PerfReport: Full Execution -- 9.98e+01 / -nan, 2.57e+01 / 0.00e+00 | 20260505 18:45:35.372 | INFO | PerfReport: 22.62%, Upload -- 2.26e+01 / 0.00e+00, 7.94e+00 / 0.00e+00 | 20260505 18:45:35.372 | INFO | PerfReport: 21.81%, Download -- 2.18e+01 / 0.00e+00, 7.80e+00 / 0.00e+00 | 20260505 18:45:35.372 | INFO | PerfReport: 31.54%, Compute -- 3.15e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 18:45:35.372 | INFO | PerfReport: 2.43%, LockUpload -- 2.42e+00 / 0.00e+00, 8.13e+00 / 0.00e+00 | 20260505 18:45:35.372 | INFO | PerfReport: 1.34%, LockDownload -- 1.33e+00 / 0.00e+00, 2.85e+00 / 0.00e+00 | 20260505 18:45:35.372 | INFO | PerfReport: 20.25%, LockCompute -- 2.02e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260505 18:45:35.372 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.51e-02 / 0.00e+00 | 20260505 18:45:35.506 | INFO | Starting a round of Baz IO | 20260505 18:45:38.698 | INFO | Populating RTMetrics report | 20260505 18:45:38.699 | INFO | RTMetrics report populated | 20260505 18:45:38.699 | INFO | Opening tmp RTMetrics json file | 20260505 18:45:38.704 | INFO | Serializing RTMetrics to json | 20260505 18:45:38.705 | INFO | Serialization done, swapping tmp file now... | 20260505 18:45:38.706 | INFO | RTMetrics done | 20260505 18:45:38.784 | INFO | Finished a round of Baz IO: Wrote 2.079 GiB in 3.278 seconds | 20260505 18:45:38.784 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.86% / 0.00% / 78.13% | 20260505 18:45:38.784 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.236615% / 0% / 9.53674e-05% | 20260505 18:45:38.784 | INFO | Baz IO overall: | 20260505 18:45:38.785 | INFO | BytesWritten : 3212091776220 | 20260505 18:45:38.785 | INFO | NumEvents : 1521148591446 | 20260505 18:45:40.506 | INFO | Analyzing chunk frames = [5464064,5464576) | 20260505 18:45:40.506 | INFO | MemInfo: { | 20260505 18:45:40.506 | INFO | "gpu" : " 17 GB", | 20260505 18:45:40.506 | INFO | "huge" : "", | 20260505 18:45:40.506 | INFO | "huge_shared" : " 27 GB", | 20260505 18:45:40.506 | INFO | "malloc" : " 14 GB", | 20260505 18:45:40.506 | INFO | "pinned" : " 29 GB" | 20260505 18:45:40.506 | INFO | } | 20260505 18:45:40.506 | INFO | Perf report for the last 5.13553 seconds: | 20260505 18:45:40.506 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:45:40.506 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:40.506 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:40.506 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:40.506 | INFO | ANALYSIS : 63.7%, 3.0, 99.3ms, 0 | 20260505 18:45:40.506 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:40.506 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260505 18:45:40.506 | INFO | PRE_HQ : 60.6%, 1.8, 55.9ms, 0 | 20260505 18:45:40.506 | INFO | BAZWRITER : 63.3%, 3.8, 121.9ms, 0 | 20260505 18:45:40.506 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 18:45:40.506 | INFO | Uploaded 36GiB to GPU in 2.19885s (16.3722GiB/s) | 20260505 18:45:40.506 | INFO | Downloaded 38GiB to GPU in 2.26576s (16.7714GiB/s) | 20260505 18:45:45.650 | INFO | Analyzing chunk frames = [5464576,5465088) | 20260505 18:45:45.650 | INFO | MemInfo: { | 20260505 18:45:45.650 | INFO | "gpu" : " 17 GB", | 20260505 18:45:45.650 | INFO | "huge" : "", | 20260505 18:45:45.650 | INFO | "huge_shared" : " 27 GB", | 20260505 18:45:45.650 | INFO | "malloc" : " 14 GB", | 20260505 18:45:45.650 | INFO | "pinned" : " 29 GB" | 20260505 18:45:45.650 | INFO | } | 20260505 18:45:45.650 | INFO | Perf report for the last 5.14396 seconds: | 20260505 18:45:45.650 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:45:45.650 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:45.650 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:45.650 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:45.650 | INFO | ANALYSIS : 62.7%, 3.0, 98.0ms, 0 | 20260505 18:45:45.650 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:45.650 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:45.650 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:45:45.650 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:45.650 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:45.650 | INFO | Uploaded 36GiB to GPU in 2.12785s (16.9185GiB/s) | 20260505 18:45:45.650 | INFO | Downloaded 34GiB to GPU in 2.03319s (16.7225GiB/s) | 20260505 18:45:50.798 | INFO | Analyzing chunk frames = [5465088,5465600) | 20260505 18:45:50.798 | INFO | MemInfo: { | 20260505 18:45:50.798 | INFO | "gpu" : " 17 GB", | 20260505 18:45:50.798 | INFO | "huge" : "", | 20260505 18:45:50.798 | INFO | "huge_shared" : " 27 GB", | 20260505 18:45:50.798 | INFO | "malloc" : " 14 GB", | 20260505 18:45:50.798 | INFO | "pinned" : " 29 GB" | 20260505 18:45:50.798 | INFO | } | 20260505 18:45:50.798 | INFO | Perf report for the last 5.14793 seconds: | 20260505 18:45:50.798 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:45:50.798 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:50.798 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:50.798 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:50.798 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 18:45:50.798 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:50.798 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:50.798 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 18:45:50.798 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:50.798 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:50.798 | INFO | Uploaded 36GiB to GPU in 2.12947s (16.9056GiB/s) | 20260505 18:45:50.798 | INFO | Downloaded 34GiB to GPU in 2.04743s (16.6062GiB/s) | 20260505 18:45:55.934 | INFO | Analyzing chunk frames = [5465600,5466112) | 20260505 18:45:55.934 | INFO | MemInfo: { | 20260505 18:45:55.934 | INFO | "gpu" : " 17 GB", | 20260505 18:45:55.934 | INFO | "huge" : "", | 20260505 18:45:55.934 | INFO | "huge_shared" : " 27 GB", | 20260505 18:45:55.934 | INFO | "malloc" : " 14 GB", | 20260505 18:45:55.934 | INFO | "pinned" : " 29 GB" | 20260505 18:45:55.934 | INFO | } | 20260505 18:45:55.934 | INFO | Perf report for the last 5.13635 seconds: | 20260505 18:45:55.934 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:45:55.934 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:55.934 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:55.934 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:55.934 | INFO | ANALYSIS : 62.4%, 3.0, 96.9ms, 0 | 20260505 18:45:55.934 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:55.934 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:55.934 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 18:45:55.934 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:55.934 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:45:55.934 | INFO | Uploaded 36GiB to GPU in 2.20393s (16.3345GiB/s) | 20260505 18:45:55.934 | INFO | Downloaded 34GiB to GPU in 2.04866s (16.5962GiB/s) | 20260505 18:46:01.063 | INFO | Analyzing chunk frames = [5466112,5466624) | 20260505 18:46:01.063 | INFO | MemInfo: { | 20260505 18:46:01.063 | INFO | "gpu" : " 17 GB", | 20260505 18:46:01.063 | INFO | "huge" : "", | 20260505 18:46:01.063 | INFO | "huge_shared" : " 27 GB", | 20260505 18:46:01.063 | INFO | "malloc" : " 14 GB", | 20260505 18:46:01.063 | INFO | "pinned" : " 29 GB" | 20260505 18:46:01.063 | INFO | } | 20260505 18:46:01.063 | INFO | Perf report for the last 5.12914 seconds: | 20260505 18:46:01.063 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:46:01.063 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:01.063 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:01.063 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:01.063 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260505 18:46:01.063 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:01.063 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:01.063 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 18:46:01.064 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:01.064 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:01.064 | INFO | Uploaded 36GiB to GPU in 2.13786s (16.8392GiB/s) | 20260505 18:46:01.064 | INFO | Downloaded 34GiB to GPU in 2.06126s (16.4947GiB/s) | 20260505 18:46:06.200 | INFO | Analyzing chunk frames = [5466624,5467136) | 20260505 18:46:06.200 | INFO | MemInfo: { | 20260505 18:46:06.200 | INFO | "gpu" : " 17 GB", | 20260505 18:46:06.200 | INFO | "huge" : "", | 20260505 18:46:06.200 | INFO | "huge_shared" : " 27 GB", | 20260505 18:46:06.200 | INFO | "malloc" : " 14 GB", | 20260505 18:46:06.200 | INFO | "pinned" : " 29 GB" | 20260505 18:46:06.200 | INFO | } | 20260505 18:46:06.200 | INFO | Perf report for the last 5.13696 seconds: | 20260505 18:46:06.200 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:46:06.200 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:06.200 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:06.200 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:06.200 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260505 18:46:06.200 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:06.200 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:06.200 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:46:06.200 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:06.200 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:06.201 | INFO | Uploaded 36GiB to GPU in 2.13587s (16.8549GiB/s) | 20260505 18:46:06.201 | INFO | Downloaded 34GiB to GPU in 2.03924s (16.6729GiB/s) | 20260505 18:46:11.348 | INFO | Analyzing chunk frames = [5467136,5467648) | 20260505 18:46:11.348 | INFO | MemInfo: { | 20260505 18:46:11.348 | INFO | "gpu" : " 17 GB", | 20260505 18:46:11.348 | INFO | "huge" : "", | 20260505 18:46:11.348 | INFO | "huge_shared" : " 27 GB", | 20260505 18:46:11.348 | INFO | "malloc" : " 14 GB", | 20260505 18:46:11.349 | INFO | "pinned" : " 29 GB" | 20260505 18:46:11.349 | INFO | } | 20260505 18:46:11.349 | INFO | Perf report for the last 5.14831 seconds: | 20260505 18:46:11.349 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:46:11.349 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:11.349 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:11.349 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:11.349 | INFO | ANALYSIS : 63.3%, 3.0, 99.0ms, 0 | 20260505 18:46:11.349 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:11.349 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:11.349 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 18:46:11.349 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:11.349 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:11.349 | INFO | Uploaded 36GiB to GPU in 2.16607s (16.6199GiB/s) | 20260505 18:46:11.349 | INFO | Downloaded 34GiB to GPU in 2.18523s (15.559GiB/s) | 20260505 18:46:16.488 | INFO | Analyzing chunk frames = [5467648,5468160) | 20260505 18:46:16.488 | INFO | MemInfo: { | 20260505 18:46:16.488 | INFO | "gpu" : " 17 GB", | 20260505 18:46:16.488 | INFO | "huge" : "", | 20260505 18:46:16.488 | INFO | "huge_shared" : " 27 GB", | 20260505 18:46:16.488 | INFO | "malloc" : " 14 GB", | 20260505 18:46:16.488 | INFO | "pinned" : " 29 GB" | 20260505 18:46:16.488 | INFO | } | 20260505 18:46:16.488 | INFO | Perf report for the last 5.13911 seconds: | 20260505 18:46:16.488 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:46:16.488 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:16.488 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:16.488 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:16.488 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 18:46:16.488 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:16.488 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:16.488 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 18:46:16.488 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:16.488 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:16.488 | INFO | Uploaded 36GiB to GPU in 2.19803s (16.3783GiB/s) | 20260505 18:46:16.488 | INFO | Downloaded 34GiB to GPU in 2.02988s (16.7498GiB/s) | 20260505 18:46:16.620 | INFO | Starting a round of Baz IO | 20260505 18:46:19.796 | INFO | Populating RTMetrics report | 20260505 18:46:19.797 | INFO | RTMetrics report populated | 20260505 18:46:19.797 | INFO | Opening tmp RTMetrics json file | 20260505 18:46:19.802 | INFO | Serializing RTMetrics to json | 20260505 18:46:19.804 | INFO | Serialization done, swapping tmp file now... | 20260505 18:46:19.805 | INFO | RTMetrics done | 20260505 18:46:19.882 | INFO | Finished a round of Baz IO: Wrote 2.079 GiB in 3.262 seconds | 20260505 18:46:19.882 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.86% / 0.00% / 78.14% | 20260505 18:46:19.883 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.23605% / 0% / 0.000115236% | 20260505 18:46:19.883 | INFO | Baz IO overall: | 20260505 18:46:19.883 | INFO | BytesWritten : 3214323942655 | 20260505 18:46:19.883 | INFO | NumEvents : 1522187529746 | 20260505 18:46:21.632 | INFO | Analyzing chunk frames = [5468160,5468672) | 20260505 18:46:21.632 | INFO | MemInfo: { | 20260505 18:46:21.632 | INFO | "gpu" : " 17 GB", | 20260505 18:46:21.632 | INFO | "huge" : "", | 20260505 18:46:21.632 | INFO | "huge_shared" : " 27 GB", | 20260505 18:46:21.632 | INFO | "malloc" : " 14 GB", | 20260505 18:46:21.632 | INFO | "pinned" : " 29 GB" | 20260505 18:46:21.632 | INFO | } | 20260505 18:46:21.632 | INFO | Perf report for the last 5.14471 seconds: | 20260505 18:46:21.632 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:46:21.632 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:21.632 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:21.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:21.633 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 18:46:21.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:21.633 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260505 18:46:21.633 | INFO | PRE_HQ : 60.2%, 1.8, 55.5ms, 0 | 20260505 18:46:21.633 | INFO | BAZWRITER : 63.0%, 3.8, 122.2ms, 0 | 20260505 18:46:21.633 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 18:46:21.633 | INFO | Uploaded 36GiB to GPU in 2.16543s (16.6248GiB/s) | 20260505 18:46:21.633 | INFO | Downloaded 38GiB to GPU in 2.24169s (16.9515GiB/s) | 20260505 18:46:26.768 | INFO | Analyzing chunk frames = [5468672,5469184) | 20260505 18:46:26.768 | INFO | MemInfo: { | 20260505 18:46:26.768 | INFO | "gpu" : " 17 GB", | 20260505 18:46:26.768 | INFO | "huge" : "", | 20260505 18:46:26.769 | INFO | "huge_shared" : " 27 GB", | 20260505 18:46:26.769 | INFO | "malloc" : " 14 GB", | 20260505 18:46:26.769 | INFO | "pinned" : " 29 GB" | 20260505 18:46:26.769 | INFO | } | 20260505 18:46:26.769 | INFO | Perf report for the last 5.13618 seconds: | 20260505 18:46:26.769 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:46:26.769 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:26.769 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:26.769 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:26.769 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 18:46:26.769 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:26.769 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:26.769 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260505 18:46:26.769 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:26.769 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:26.769 | INFO | Uploaded 36GiB to GPU in 2.13526s (16.8597GiB/s) | 20260505 18:46:26.769 | INFO | Downloaded 34GiB to GPU in 2.02878s (16.7588GiB/s) | 20260505 18:46:31.902 | INFO | Analyzing chunk frames = [5469184,5469696) | 20260505 18:46:31.902 | INFO | MemInfo: { | 20260505 18:46:31.902 | INFO | "gpu" : " 17 GB", | 20260505 18:46:31.902 | INFO | "huge" : "", | 20260505 18:46:31.902 | INFO | "huge_shared" : " 27 GB", | 20260505 18:46:31.902 | INFO | "malloc" : " 14 GB", | 20260505 18:46:31.902 | INFO | "pinned" : " 29 GB" | 20260505 18:46:31.902 | INFO | } | 20260505 18:46:31.902 | INFO | Perf report for the last 5.13315 seconds: | 20260505 18:46:31.902 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:46:31.902 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:31.902 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:31.902 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:31.902 | INFO | ANALYSIS : 62.4%, 3.0, 97.0ms, 0 | 20260505 18:46:31.902 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:31.902 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:31.902 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:46:31.902 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:31.902 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:31.902 | INFO | Uploaded 36GiB to GPU in 2.1375s (16.8421GiB/s) | 20260505 18:46:31.902 | INFO | Downloaded 34GiB to GPU in 2.04307s (16.6416GiB/s) | 20260505 18:46:37.049 | INFO | Analyzing chunk frames = [5469696,5470208) | 20260505 18:46:37.049 | INFO | MemInfo: { | 20260505 18:46:37.049 | INFO | "gpu" : " 17 GB", | 20260505 18:46:37.049 | INFO | "huge" : "", | 20260505 18:46:37.049 | INFO | "huge_shared" : " 27 GB", | 20260505 18:46:37.049 | INFO | "malloc" : " 14 GB", | 20260505 18:46:37.049 | INFO | "pinned" : " 29 GB" | 20260505 18:46:37.049 | INFO | } | 20260505 18:46:37.049 | INFO | Perf report for the last 5.14704 seconds: | 20260505 18:46:37.049 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:46:37.049 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:37.049 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:37.049 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:37.049 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 18:46:37.049 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:37.049 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:37.049 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:46:37.049 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:37.049 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:37.049 | INFO | Uploaded 36GiB to GPU in 2.22891s (16.1514GiB/s) | 20260505 18:46:37.049 | INFO | Downloaded 34GiB to GPU in 2.11965s (16.0404GiB/s) | 20260505 18:46:42.184 | INFO | Analyzing chunk frames = [5470208,5470720) | 20260505 18:46:42.184 | INFO | MemInfo: { | 20260505 18:46:42.184 | INFO | "gpu" : " 17 GB", | 20260505 18:46:42.184 | INFO | "huge" : "", | 20260505 18:46:42.184 | INFO | "huge_shared" : " 27 GB", | 20260505 18:46:42.184 | INFO | "malloc" : " 14 GB", | 20260505 18:46:42.184 | INFO | "pinned" : " 29 GB" | 20260505 18:46:42.184 | INFO | } | 20260505 18:46:42.184 | INFO | Perf report for the last 5.13537 seconds: | 20260505 18:46:42.184 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:46:42.184 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:42.184 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:42.184 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:42.184 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 18:46:42.184 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:42.184 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:42.184 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:46:42.184 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:42.184 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:42.184 | INFO | Uploaded 36GiB to GPU in 2.14578s (16.7771GiB/s) | 20260505 18:46:42.185 | INFO | Downloaded 34GiB to GPU in 2.05922s (16.5111GiB/s) | 20260505 18:46:47.321 | INFO | Analyzing chunk frames = [5470720,5471232) | 20260505 18:46:47.321 | INFO | MemInfo: { | 20260505 18:46:47.321 | INFO | "gpu" : " 17 GB", | 20260505 18:46:47.321 | INFO | "huge" : "", | 20260505 18:46:47.322 | INFO | "huge_shared" : " 27 GB", | 20260505 18:46:47.322 | INFO | "malloc" : " 14 GB", | 20260505 18:46:47.322 | INFO | "pinned" : " 29 GB" | 20260505 18:46:47.322 | INFO | } | 20260505 18:46:47.322 | INFO | Perf report for the last 5.13753 seconds: | 20260505 18:46:47.322 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:46:47.322 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:47.322 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:47.322 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:47.322 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 18:46:47.322 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:47.322 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:47.322 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:46:47.322 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:47.322 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:47.322 | INFO | Uploaded 36GiB to GPU in 2.1375s (16.8421GiB/s) | 20260505 18:46:47.322 | INFO | Downloaded 34GiB to GPU in 2.05514s (16.5439GiB/s) | 20260505 18:46:52.456 | INFO | Analyzing chunk frames = [5471232,5471744) | 20260505 18:46:52.456 | INFO | MemInfo: { | 20260505 18:46:52.456 | INFO | "gpu" : " 17 GB", | 20260505 18:46:52.456 | INFO | "huge" : "", | 20260505 18:46:52.456 | INFO | "huge_shared" : " 27 GB", | 20260505 18:46:52.456 | INFO | "malloc" : " 14 GB", | 20260505 18:46:52.456 | INFO | "pinned" : " 29 GB" | 20260505 18:46:52.456 | INFO | } | 20260505 18:46:52.456 | INFO | Perf report for the last 5.13445 seconds: | 20260505 18:46:52.456 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:46:52.456 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:52.456 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:52.456 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:52.456 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 18:46:52.456 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:52.456 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:52.456 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 18:46:52.456 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:52.456 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:52.457 | INFO | Uploaded 36GiB to GPU in 2.15825s (16.6802GiB/s) | 20260505 18:46:52.457 | INFO | Downloaded 34GiB to GPU in 2.16403s (15.7114GiB/s) | 20260505 18:46:57.605 | INFO | Analyzing chunk frames = [5471744,5472256) | 20260505 18:46:57.605 | INFO | MemInfo: { | 20260505 18:46:57.605 | INFO | "gpu" : " 17 GB", | 20260505 18:46:57.605 | INFO | "huge" : "", | 20260505 18:46:57.605 | INFO | "huge_shared" : " 27 GB", | 20260505 18:46:57.605 | INFO | "malloc" : " 14 GB", | 20260505 18:46:57.605 | INFO | "pinned" : " 29 GB" | 20260505 18:46:57.605 | INFO | } | 20260505 18:46:57.606 | INFO | Perf report for the last 5.14931 seconds: | 20260505 18:46:57.606 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:46:57.606 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:57.606 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:57.606 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:57.606 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 18:46:57.606 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:57.606 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:57.606 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:46:57.606 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:57.606 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:46:57.606 | INFO | Uploaded 36GiB to GPU in 2.2024s (16.3458GiB/s) | 20260505 18:46:57.606 | INFO | Downloaded 34GiB to GPU in 2.02518s (16.7886GiB/s) | 20260505 18:46:57.606 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 18:46:57.606 | INFO | Total mem usage: 15.3 GiB | 20260505 18:46:57.606 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 18:46:57.606 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.22%) | 20260505 18:46:57.606 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 18:46:57.606 | INFO | High Water Breakdown: | 20260505 18:46:57.606 | INFO | 61% : 8.64 GiB ( 90.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 18:46:57.606 | INFO | 38.8% : 5.5 GiB ( 82.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 18:46:57.606 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 18:46:57.606 | INFO | ------------------------------------------ | 20260505 18:46:57.606 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:46:57.606 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 18:46:57.606 | INFO | Total mem usage: 26.5 GiB | 20260505 18:46:57.606 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 18:46:57.606 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 18:46:57.606 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 18:46:57.606 | INFO | High Water Breakdown: | 20260505 18:46:57.606 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:46:57.606 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:46:57.606 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:46:57.606 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:46:57.606 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:46:57.606 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:46:57.606 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 18:46:57.606 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 18:46:57.606 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:46:57.606 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 18:46:57.606 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 18:46:57.606 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 18:46:57.606 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 18:46:57.606 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 18:46:57.606 | INFO | ------------------------------------------ | 20260505 18:46:57.606 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:46:57.606 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 18:46:57.606 | INFO | Total mem usage: 25.6 GiB | 20260505 18:46:57.606 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 18:46:57.606 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 18:46:57.606 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 18:46:57.606 | INFO | High Water Breakdown: | 20260505 18:46:57.606 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 18:46:57.606 | INFO | ------------------------------------------ | 20260505 18:46:57.606 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:46:57.606 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 18:46:57.606 | INFO | Total mem usage: 15.7 GiB | 20260505 18:46:57.606 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 18:46:57.606 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 18:46:57.606 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 18:46:57.606 | INFO | High Water Breakdown: | 20260505 18:46:57.606 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 18:46:57.606 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 18:46:57.606 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 18:46:57.606 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 18:46:57.606 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 18:46:57.606 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 18:46:57.606 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 18:46:57.607 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 18:46:57.607 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 18:46:57.607 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:46:57.607 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 18:46:57.607 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:46:57.607 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 18:46:57.607 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 18:46:57.607 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 18:46:57.607 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 18:46:57.607 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:46:57.607 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 18:46:57.607 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:46:57.607 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:46:57.607 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:46:57.607 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 18:46:57.607 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 18:46:57.607 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:46:57.607 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 18:46:57.607 | INFO | ------------------------------------------ | 20260505 18:46:57.607 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:46:57.607 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 18:46:57.607 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:46:57.607 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:46:57.607 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.48e+01, 1.36e+01 / 2.34e+00 | 20260505 18:46:57.607 | INFO | PerfReport: 0.29%, AggregateStats -- 9.25e-02 / 1.03e-01, 1.08e-02 / 5.63e-03 | 20260505 18:46:57.607 | INFO | PerfReport: 11.76%, Baseline -- 3.71e+00 / 4.52e+00, 5.66e-01 / 1.58e-01 | 20260505 18:46:57.607 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.51e+00, 1.91e-01 / 1.15e-01 | 20260505 18:46:57.607 | INFO | PerfReport: 15.97%, DME -- 4.03e+01 / 4.47e+01, 4.16e+00 / 2.38e+00 | 20260505 18:46:57.607 | INFO | PerfReport: 56.96%, FrameLabeling -- 1.80e+01 / 1.92e+01, 1.51e+00 / 9.78e-01 | 20260505 18:46:57.607 | INFO | PerfReport: 7.28%, PulseAccumulating -- 2.30e+00 / 2.91e+00, 4.44e-01 / 1.69e-01 | 20260505 18:46:57.607 | INFO | PerfReport: 3.30%, Metrics -- 1.04e+00 / 1.36e+00, 2.88e-01 / 1.51e-01 | 20260505 18:46:57.607 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.64e-01 / 4.25e-01, 6.43e-02 / 4.09e-02 | 20260505 18:46:57.607 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 18:46:57.607 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:46:57.607 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:46:57.607 | INFO | PerfReport: Full Execution -- 9.98e+01 / -nan, 2.58e+01 / 0.00e+00 | 20260505 18:46:57.607 | INFO | PerfReport: 22.58%, Upload -- 2.25e+01 / 0.00e+00, 7.89e+00 / 0.00e+00 | 20260505 18:46:57.607 | INFO | PerfReport: 21.82%, Download -- 2.18e+01 / 0.00e+00, 7.81e+00 / 0.00e+00 | 20260505 18:46:57.607 | INFO | PerfReport: 31.64%, Compute -- 3.16e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 18:46:57.607 | INFO | PerfReport: 2.29%, LockUpload -- 2.28e+00 / 0.00e+00, 7.91e+00 / 0.00e+00 | 20260505 18:46:57.607 | INFO | PerfReport: 1.35%, LockDownload -- 1.34e+00 / 0.00e+00, 2.83e+00 / 0.00e+00 | 20260505 18:46:57.607 | INFO | PerfReport: 20.32%, LockCompute -- 2.03e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 18:46:57.607 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.74e-02 / 0.00e+00 | 20260505 18:46:57.735 | INFO | Starting a round of Baz IO | 20260505 18:47:00.911 | INFO | Populating RTMetrics report | 20260505 18:47:00.911 | INFO | RTMetrics report populated | 20260505 18:47:00.911 | INFO | Opening tmp RTMetrics json file | 20260505 18:47:00.912 | INFO | Serializing RTMetrics to json | 20260505 18:47:00.913 | INFO | Serialization done, swapping tmp file now... | 20260505 18:47:00.914 | INFO | RTMetrics done | 20260505 18:47:01.003 | INFO | Finished a round of Baz IO: Wrote 2.078 GiB in 3.267 seconds | 20260505 18:47:01.003 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.86% / 0.00% / 78.14% | 20260505 18:47:01.003 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.235399% / 3.97364e-06% / 6.75519e-05% | 20260505 18:47:01.003 | INFO | Baz IO overall: | 20260505 18:47:01.003 | INFO | BytesWritten : 3216555231046 | 20260505 18:47:01.003 | INFO | NumEvents : 1523225941080 | 20260505 18:47:02.746 | INFO | Analyzing chunk frames = [5472256,5472768) | 20260505 18:47:02.746 | INFO | MemInfo: { | 20260505 18:47:02.746 | INFO | "gpu" : " 17 GB", | 20260505 18:47:02.746 | INFO | "huge" : "", | 20260505 18:47:02.746 | INFO | "huge_shared" : " 27 GB", | 20260505 18:47:02.746 | INFO | "malloc" : " 14 GB", | 20260505 18:47:02.746 | INFO | "pinned" : " 29 GB" | 20260505 18:47:02.746 | INFO | } | 20260505 18:47:02.747 | INFO | Perf report for the last 5.14094 seconds: | 20260505 18:47:02.747 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:47:02.747 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:02.747 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:02.747 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:02.747 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 18:47:02.747 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:02.747 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260505 18:47:02.747 | INFO | PRE_HQ : 60.3%, 1.8, 55.6ms, 0 | 20260505 18:47:02.747 | INFO | BAZWRITER : 63.2%, 3.7, 121.4ms, 0 | 20260505 18:47:02.747 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 18:47:02.747 | INFO | Uploaded 36GiB to GPU in 2.16568s (16.6229GiB/s) | 20260505 18:47:02.747 | INFO | Downloaded 38GiB to GPU in 2.25616s (16.8428GiB/s) | 20260505 18:47:07.877 | INFO | Analyzing chunk frames = [5472768,5473280) | 20260505 18:47:07.877 | INFO | MemInfo: { | 20260505 18:47:07.877 | INFO | "gpu" : " 17 GB", | 20260505 18:47:07.877 | INFO | "huge" : "", | 20260505 18:47:07.877 | INFO | "huge_shared" : " 27 GB", | 20260505 18:47:07.877 | INFO | "malloc" : " 14 GB", | 20260505 18:47:07.877 | INFO | "pinned" : " 29 GB" | 20260505 18:47:07.877 | INFO | } | 20260505 18:47:07.877 | INFO | Perf report for the last 5.13078 seconds: | 20260505 18:47:07.877 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:47:07.877 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:07.877 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:07.877 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:07.877 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260505 18:47:07.877 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:07.877 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:07.877 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260505 18:47:07.877 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:07.877 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:07.878 | INFO | Uploaded 36GiB to GPU in 2.1605s (16.6628GiB/s) | 20260505 18:47:07.878 | INFO | Downloaded 34GiB to GPU in 2.05167s (16.5719GiB/s) | 20260505 18:47:13.012 | INFO | Analyzing chunk frames = [5473280,5473792) | 20260505 18:47:13.012 | INFO | MemInfo: { | 20260505 18:47:13.012 | INFO | "gpu" : " 17 GB", | 20260505 18:47:13.012 | INFO | "huge" : "", | 20260505 18:47:13.012 | INFO | "huge_shared" : " 27 GB", | 20260505 18:47:13.012 | INFO | "malloc" : " 14 GB", | 20260505 18:47:13.012 | INFO | "pinned" : " 29 GB" | 20260505 18:47:13.012 | INFO | } | 20260505 18:47:13.012 | INFO | Perf report for the last 5.13463 seconds: | 20260505 18:47:13.012 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:47:13.012 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:13.012 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:13.012 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:13.012 | INFO | ANALYSIS : 62.7%, 3.0, 97.5ms, 0 | 20260505 18:47:13.012 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:13.012 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:13.012 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 18:47:13.012 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:13.012 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:13.012 | INFO | Uploaded 36GiB to GPU in 2.20499s (16.3266GiB/s) | 20260505 18:47:13.012 | INFO | Downloaded 34GiB to GPU in 2.04543s (16.6225GiB/s) | 20260505 18:47:18.157 | INFO | Analyzing chunk frames = [5473792,5474304) | 20260505 18:47:18.157 | INFO | MemInfo: { | 20260505 18:47:18.157 | INFO | "gpu" : " 17 GB", | 20260505 18:47:18.157 | INFO | "huge" : "", | 20260505 18:47:18.157 | INFO | "huge_shared" : " 27 GB", | 20260505 18:47:18.157 | INFO | "malloc" : " 14 GB", | 20260505 18:47:18.157 | INFO | "pinned" : " 29 GB" | 20260505 18:47:18.157 | INFO | } | 20260505 18:47:18.157 | INFO | Perf report for the last 5.1455 seconds: | 20260505 18:47:18.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:47:18.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:18.158 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:18.158 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:18.158 | INFO | ANALYSIS : 62.5%, 3.0, 97.2ms, 0 | 20260505 18:47:18.158 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:18.158 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:18.158 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260505 18:47:18.158 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:18.158 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:18.158 | INFO | Uploaded 36GiB to GPU in 2.17969s (16.5161GiB/s) | 20260505 18:47:18.158 | INFO | Downloaded 34GiB to GPU in 2.14567s (15.8459GiB/s) | 20260505 18:47:23.296 | INFO | Analyzing chunk frames = [5474304,5474816) | 20260505 18:47:23.296 | INFO | MemInfo: { | 20260505 18:47:23.296 | INFO | "gpu" : " 17 GB", | 20260505 18:47:23.296 | INFO | "huge" : "", | 20260505 18:47:23.296 | INFO | "huge_shared" : " 27 GB", | 20260505 18:47:23.296 | INFO | "malloc" : " 14 GB", | 20260505 18:47:23.296 | INFO | "pinned" : " 29 GB" | 20260505 18:47:23.296 | INFO | } | 20260505 18:47:23.296 | INFO | Perf report for the last 5.13866 seconds: | 20260505 18:47:23.296 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:47:23.296 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:23.296 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:23.296 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:23.296 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260505 18:47:23.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:23.296 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:23.296 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:47:23.296 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:23.296 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:23.296 | INFO | Uploaded 36GiB to GPU in 2.14167s (16.8093GiB/s) | 20260505 18:47:23.297 | INFO | Downloaded 34GiB to GPU in 2.06306s (16.4804GiB/s) | 20260505 18:47:28.444 | INFO | Analyzing chunk frames = [5474816,5475328) | 20260505 18:47:28.444 | INFO | MemInfo: { | 20260505 18:47:28.444 | INFO | "gpu" : " 17 GB", | 20260505 18:47:28.444 | INFO | "huge" : "", | 20260505 18:47:28.444 | INFO | "huge_shared" : " 27 GB", | 20260505 18:47:28.444 | INFO | "malloc" : " 14 GB", | 20260505 18:47:28.444 | INFO | "pinned" : " 29 GB" | 20260505 18:47:28.444 | INFO | } | 20260505 18:47:28.444 | INFO | Perf report for the last 5.14789 seconds: | 20260505 18:47:28.444 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:47:28.444 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:28.444 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:28.444 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:28.444 | INFO | ANALYSIS : 62.2%, 3.0, 97.3ms, 0 | 20260505 18:47:28.444 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:28.444 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:28.444 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260505 18:47:28.444 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:28.444 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:28.444 | INFO | Uploaded 36GiB to GPU in 2.13626s (16.8519GiB/s) | 20260505 18:47:28.444 | INFO | Downloaded 34GiB to GPU in 2.05486s (16.5461GiB/s) | 20260505 18:47:33.576 | INFO | Analyzing chunk frames = [5475328,5475840) | 20260505 18:47:33.576 | INFO | MemInfo: { | 20260505 18:47:33.576 | INFO | "gpu" : " 17 GB", | 20260505 18:47:33.576 | INFO | "huge" : "", | 20260505 18:47:33.576 | INFO | "huge_shared" : " 27 GB", | 20260505 18:47:33.576 | INFO | "malloc" : " 14 GB", | 20260505 18:47:33.576 | INFO | "pinned" : " 29 GB" | 20260505 18:47:33.576 | INFO | } | 20260505 18:47:33.576 | INFO | Perf report for the last 5.13193 seconds: | 20260505 18:47:33.576 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:47:33.576 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:33.576 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:33.576 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:33.576 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 18:47:33.576 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:33.576 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:33.576 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 18:47:33.576 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:33.576 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:33.576 | INFO | Uploaded 36GiB to GPU in 2.09738s (17.1643GiB/s) | 20260505 18:47:33.576 | INFO | Downloaded 34GiB to GPU in 2.13315s (15.9389GiB/s) | 20260505 18:47:38.713 | INFO | Analyzing chunk frames = [5475840,5476352) | 20260505 18:47:38.713 | INFO | MemInfo: { | 20260505 18:47:38.713 | INFO | "gpu" : " 17 GB", | 20260505 18:47:38.713 | INFO | "huge" : "", | 20260505 18:47:38.713 | INFO | "huge_shared" : " 27 GB", | 20260505 18:47:38.713 | INFO | "malloc" : " 14 GB", | 20260505 18:47:38.713 | INFO | "pinned" : " 29 GB" | 20260505 18:47:38.713 | INFO | } | 20260505 18:47:38.713 | INFO | Perf report for the last 5.13693 seconds: | 20260505 18:47:38.713 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:47:38.713 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:38.713 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:38.713 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:38.713 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 18:47:38.713 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:38.713 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:38.713 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 18:47:38.713 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:38.713 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:38.713 | INFO | Uploaded 36GiB to GPU in 2.2041s (16.3332GiB/s) | 20260505 18:47:38.713 | INFO | Downloaded 34GiB to GPU in 2.04103s (16.6583GiB/s) | 20260505 18:47:38.847 | INFO | Starting a round of Baz IO | 20260505 18:47:42.023 | INFO | Populating RTMetrics report | 20260505 18:47:42.024 | INFO | RTMetrics report populated | 20260505 18:47:42.024 | INFO | Opening tmp RTMetrics json file | 20260505 18:47:42.024 | INFO | Serializing RTMetrics to json | 20260505 18:47:42.026 | INFO | Serialization done, swapping tmp file now... | 20260505 18:47:42.026 | INFO | RTMetrics done | 20260505 18:47:42.115 | INFO | Finished a round of Baz IO: Wrote 2.078 GiB in 3.267 seconds | 20260505 18:47:42.115 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.86% / 0.00% / 78.14% | 20260505 18:47:42.115 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.234938% / 0% / 6.35783e-05% | 20260505 18:47:42.115 | INFO | Baz IO overall: | 20260505 18:47:42.115 | INFO | BytesWritten : 3218786579864 | 20260505 18:47:42.115 | INFO | NumEvents : 1524264259626 | 20260505 18:47:43.858 | INFO | Analyzing chunk frames = [5476352,5476864) | 20260505 18:47:43.858 | INFO | MemInfo: { | 20260505 18:47:43.858 | INFO | "gpu" : " 17 GB", | 20260505 18:47:43.858 | INFO | "huge" : "", | 20260505 18:47:43.858 | INFO | "huge_shared" : " 27 GB", | 20260505 18:47:43.858 | INFO | "malloc" : " 14 GB", | 20260505 18:47:43.858 | INFO | "pinned" : " 29 GB" | 20260505 18:47:43.858 | INFO | } | 20260505 18:47:43.858 | INFO | Perf report for the last 5.14549 seconds: | 20260505 18:47:43.858 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:47:43.858 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:43.858 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:43.858 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:43.858 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 18:47:43.858 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:43.859 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260505 18:47:43.859 | INFO | PRE_HQ : 60.3%, 1.8, 55.6ms, 0 | 20260505 18:47:43.859 | INFO | BAZWRITER : 63.0%, 3.7, 121.2ms, 0 | 20260505 18:47:43.859 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 18:47:43.859 | INFO | Uploaded 36GiB to GPU in 2.16731s (16.6105GiB/s) | 20260505 18:47:43.859 | INFO | Downloaded 38GiB to GPU in 2.24566s (16.9215GiB/s) | 20260505 18:47:49.000 | INFO | Analyzing chunk frames = [5476864,5477376) | 20260505 18:47:49.000 | INFO | MemInfo: { | 20260505 18:47:49.000 | INFO | "gpu" : " 17 GB", | 20260505 18:47:49.000 | INFO | "huge" : "", | 20260505 18:47:49.000 | INFO | "huge_shared" : " 27 GB", | 20260505 18:47:49.000 | INFO | "malloc" : " 14 GB", | 20260505 18:47:49.000 | INFO | "pinned" : " 29 GB" | 20260505 18:47:49.000 | INFO | } | 20260505 18:47:49.000 | INFO | Perf report for the last 5.14132 seconds: | 20260505 18:47:49.000 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:47:49.000 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:49.000 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:49.000 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:49.000 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 18:47:49.000 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:49.000 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:49.000 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:47:49.000 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:49.000 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:49.000 | INFO | Uploaded 36GiB to GPU in 2.14715s (16.7664GiB/s) | 20260505 18:47:49.000 | INFO | Downloaded 34GiB to GPU in 2.03725s (16.6892GiB/s) | 20260505 18:47:54.134 | INFO | Analyzing chunk frames = [5477376,5477888) | 20260505 18:47:54.134 | INFO | MemInfo: { | 20260505 18:47:54.134 | INFO | "gpu" : " 17 GB", | 20260505 18:47:54.134 | INFO | "huge" : "", | 20260505 18:47:54.134 | INFO | "huge_shared" : " 27 GB", | 20260505 18:47:54.134 | INFO | "malloc" : " 14 GB", | 20260505 18:47:54.134 | INFO | "pinned" : " 29 GB" | 20260505 18:47:54.134 | INFO | } | 20260505 18:47:54.134 | INFO | Perf report for the last 5.13414 seconds: | 20260505 18:47:54.134 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:47:54.134 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:54.134 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:54.134 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:54.134 | INFO | ANALYSIS : 62.8%, 3.0, 97.6ms, 0 | 20260505 18:47:54.134 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:54.134 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:54.134 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:47:54.134 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:54.134 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:54.134 | INFO | Uploaded 36GiB to GPU in 2.2081s (16.3036GiB/s) | 20260505 18:47:54.134 | INFO | Downloaded 34GiB to GPU in 2.0735s (16.3974GiB/s) | 20260505 18:47:59.271 | INFO | Analyzing chunk frames = [5477888,5478400) | 20260505 18:47:59.271 | INFO | MemInfo: { | 20260505 18:47:59.271 | INFO | "gpu" : " 17 GB", | 20260505 18:47:59.271 | INFO | "huge" : "", | 20260505 18:47:59.271 | INFO | "huge_shared" : " 27 GB", | 20260505 18:47:59.271 | INFO | "malloc" : " 14 GB", | 20260505 18:47:59.271 | INFO | "pinned" : " 29 GB" | 20260505 18:47:59.271 | INFO | } | 20260505 18:47:59.271 | INFO | Perf report for the last 5.13748 seconds: | 20260505 18:47:59.271 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:47:59.271 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:59.271 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:59.271 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:59.271 | INFO | ANALYSIS : 62.8%, 3.0, 97.6ms, 0 | 20260505 18:47:59.271 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:59.272 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:59.272 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:47:59.272 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:59.272 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:47:59.272 | INFO | Uploaded 36GiB to GPU in 2.22825s (16.1562GiB/s) | 20260505 18:47:59.272 | INFO | Downloaded 34GiB to GPU in 2.12173s (16.0246GiB/s) | 20260505 18:48:04.410 | INFO | Analyzing chunk frames = [5478400,5478912) | 20260505 18:48:04.410 | INFO | MemInfo: { | 20260505 18:48:04.410 | INFO | "gpu" : " 17 GB", | 20260505 18:48:04.410 | INFO | "huge" : "", | 20260505 18:48:04.410 | INFO | "huge_shared" : " 27 GB", | 20260505 18:48:04.410 | INFO | "malloc" : " 14 GB", | 20260505 18:48:04.410 | INFO | "pinned" : " 29 GB" | 20260505 18:48:04.410 | INFO | } | 20260505 18:48:04.410 | INFO | Perf report for the last 5.13888 seconds: | 20260505 18:48:04.410 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:48:04.410 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:04.410 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:04.410 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:04.410 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 18:48:04.410 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:04.410 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:04.410 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:48:04.410 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:04.410 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:04.411 | INFO | Uploaded 36GiB to GPU in 2.14998s (16.7443GiB/s) | 20260505 18:48:04.411 | INFO | Downloaded 34GiB to GPU in 2.06287s (16.4819GiB/s) | 20260505 18:48:09.553 | INFO | Analyzing chunk frames = [5478912,5479424) | 20260505 18:48:09.553 | INFO | MemInfo: { | 20260505 18:48:09.553 | INFO | "gpu" : " 17 GB", | 20260505 18:48:09.553 | INFO | "huge" : "", | 20260505 18:48:09.553 | INFO | "huge_shared" : " 27 GB", | 20260505 18:48:09.553 | INFO | "malloc" : " 14 GB", | 20260505 18:48:09.553 | INFO | "pinned" : " 29 GB" | 20260505 18:48:09.553 | INFO | } | 20260505 18:48:09.553 | INFO | Perf report for the last 5.14273 seconds: | 20260505 18:48:09.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:48:09.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:09.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:09.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:09.553 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260505 18:48:09.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:09.553 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:09.553 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:48:09.553 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:09.553 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:09.553 | INFO | Uploaded 36GiB to GPU in 2.13571s (16.8562GiB/s) | 20260505 18:48:09.553 | INFO | Downloaded 34GiB to GPU in 2.05177s (16.571GiB/s) | 20260505 18:48:14.696 | INFO | Analyzing chunk frames = [5479424,5479936) | 20260505 18:48:14.697 | INFO | MemInfo: { | 20260505 18:48:14.697 | INFO | "gpu" : " 17 GB", | 20260505 18:48:14.697 | INFO | "huge" : "", | 20260505 18:48:14.697 | INFO | "huge_shared" : " 27 GB", | 20260505 18:48:14.697 | INFO | "malloc" : " 14 GB", | 20260505 18:48:14.697 | INFO | "pinned" : " 29 GB" | 20260505 18:48:14.697 | INFO | } | 20260505 18:48:14.697 | INFO | Perf report for the last 5.14377 seconds: | 20260505 18:48:14.697 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:48:14.697 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:14.697 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:14.697 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:14.697 | INFO | ANALYSIS : 63.1%, 3.0, 98.7ms, 0 | 20260505 18:48:14.697 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:14.697 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:14.697 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 18:48:14.697 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:14.697 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:14.697 | INFO | Uploaded 36GiB to GPU in 2.13106s (16.893GiB/s) | 20260505 18:48:14.697 | INFO | Downloaded 34GiB to GPU in 2.13299s (15.94GiB/s) | 20260505 18:48:19.830 | INFO | Analyzing chunk frames = [5479936,5480448) | 20260505 18:48:19.831 | INFO | MemInfo: { | 20260505 18:48:19.831 | INFO | "gpu" : " 17 GB", | 20260505 18:48:19.831 | INFO | "huge" : "", | 20260505 18:48:19.831 | INFO | "huge_shared" : " 27 GB", | 20260505 18:48:19.831 | INFO | "malloc" : " 14 GB", | 20260505 18:48:19.831 | INFO | "pinned" : " 29 GB" | 20260505 18:48:19.831 | INFO | } | 20260505 18:48:19.831 | INFO | Perf report for the last 5.13395 seconds: | 20260505 18:48:19.831 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:48:19.831 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:19.831 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:19.831 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:19.831 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260505 18:48:19.831 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:19.831 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:19.831 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:48:19.831 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:19.831 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:19.831 | INFO | Uploaded 36GiB to GPU in 2.11974s (16.9832GiB/s) | 20260505 18:48:19.831 | INFO | Downloaded 34GiB to GPU in 2.01278s (16.8921GiB/s) | 20260505 18:48:19.831 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 18:48:19.831 | INFO | Total mem usage: 15.3 GiB | 20260505 18:48:19.831 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 18:48:19.831 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.03%) | 20260505 18:48:19.831 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 18:48:19.831 | INFO | High Water Breakdown: | 20260505 18:48:19.831 | INFO | 61% : 8.66 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 18:48:19.831 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 18:48:19.831 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 18:48:19.831 | INFO | ------------------------------------------ | 20260505 18:48:19.832 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:48:19.832 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 18:48:19.832 | INFO | Total mem usage: 26.5 GiB | 20260505 18:48:19.832 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 18:48:19.832 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 18:48:19.832 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 18:48:19.832 | INFO | High Water Breakdown: | 20260505 18:48:19.832 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:48:19.832 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:48:19.832 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:48:19.832 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:48:19.832 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:48:19.832 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:48:19.832 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 18:48:19.832 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 18:48:19.832 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:48:19.832 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 18:48:19.832 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 18:48:19.832 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 18:48:19.832 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 18:48:19.832 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 18:48:19.832 | INFO | ------------------------------------------ | 20260505 18:48:19.832 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:48:19.832 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 18:48:19.832 | INFO | Total mem usage: 25.6 GiB | 20260505 18:48:19.832 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 18:48:19.832 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 18:48:19.832 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 18:48:19.832 | INFO | High Water Breakdown: | 20260505 18:48:19.832 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 18:48:19.832 | INFO | ------------------------------------------ | 20260505 18:48:19.832 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:48:19.832 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 18:48:19.832 | INFO | Total mem usage: 15.7 GiB | 20260505 18:48:19.832 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 18:48:19.832 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 18:48:19.832 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 18:48:19.832 | INFO | High Water Breakdown: | 20260505 18:48:19.832 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 18:48:19.832 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 18:48:19.832 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 18:48:19.832 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 18:48:19.832 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 18:48:19.832 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 18:48:19.832 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 18:48:19.832 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 18:48:19.832 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 18:48:19.832 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:48:19.832 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 18:48:19.832 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:48:19.832 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 18:48:19.832 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 18:48:19.832 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 18:48:19.832 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 18:48:19.832 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:48:19.832 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 18:48:19.832 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:48:19.832 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:48:19.832 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:48:19.832 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 18:48:19.832 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 18:48:19.832 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:48:19.832 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 18:48:19.832 | INFO | ------------------------------------------ | 20260505 18:48:19.832 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:48:19.832 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 18:48:19.832 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:48:19.832 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:48:19.832 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.47e+01, 1.36e+01 / 2.16e+00 | 20260505 18:48:19.832 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.01e-01, 1.12e-02 / 4.97e-03 | 20260505 18:48:19.832 | INFO | PerfReport: 11.75%, Baseline -- 3.70e+00 / 4.53e+00, 5.51e-01 / 1.19e-01 | 20260505 18:48:19.832 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.54e+00, 1.90e-01 / 7.47e-02 | 20260505 18:48:19.832 | INFO | PerfReport: 16.00%, DME -- 4.03e+01 / 4.46e+01, 4.26e+00 / 2.29e+00 | 20260505 18:48:19.832 | INFO | PerfReport: 56.86%, FrameLabeling -- 1.79e+01 / 1.92e+01, 1.36e+00 / 7.11e-01 | 20260505 18:48:19.832 | INFO | PerfReport: 7.34%, PulseAccumulating -- 2.31e+00 / 2.91e+00, 4.44e-01 / 8.98e-02 | 20260505 18:48:19.832 | INFO | PerfReport: 3.33%, Metrics -- 1.05e+00 / 1.38e+00, 2.87e-01 / 1.33e-01 | 20260505 18:48:19.832 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.65e-01 / 4.34e-01, 6.42e-02 / 3.83e-02 | 20260505 18:48:19.833 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 18:48:19.833 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:48:19.833 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:48:19.833 | INFO | PerfReport: Full Execution -- 9.98e+01 / -nan, 2.56e+01 / 0.00e+00 | 20260505 18:48:19.833 | INFO | PerfReport: 22.56%, Upload -- 2.25e+01 / 0.00e+00, 7.88e+00 / 0.00e+00 | 20260505 18:48:19.833 | INFO | PerfReport: 21.87%, Download -- 2.18e+01 / 0.00e+00, 7.78e+00 / 0.00e+00 | 20260505 18:48:19.833 | INFO | PerfReport: 31.55%, Compute -- 3.15e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 18:48:19.833 | INFO | PerfReport: 2.36%, LockUpload -- 2.35e+00 / 0.00e+00, 7.93e+00 / 0.00e+00 | 20260505 18:48:19.833 | INFO | PerfReport: 1.37%, LockDownload -- 1.37e+00 / 0.00e+00, 2.89e+00 / 0.00e+00 | 20260505 18:48:19.833 | INFO | PerfReport: 20.28%, LockCompute -- 2.02e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 18:48:19.833 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.90e-02 / 0.00e+00 | 20260505 18:48:19.966 | INFO | Starting a round of Baz IO | 20260505 18:48:23.142 | INFO | Populating RTMetrics report | 20260505 18:48:23.142 | INFO | RTMetrics report populated | 20260505 18:48:23.142 | INFO | Opening tmp RTMetrics json file | 20260505 18:48:23.144 | INFO | Serializing RTMetrics to json | 20260505 18:48:23.145 | INFO | Serialization done, swapping tmp file now... | 20260505 18:48:23.146 | INFO | RTMetrics done | 20260505 18:48:23.236 | INFO | Finished a round of Baz IO: Wrote 2.078 GiB in 3.269 seconds | 20260505 18:48:23.236 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.86% / 0.00% / 78.14% | 20260505 18:48:23.236 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.234981% / 0% / 6.75519e-05% | 20260505 18:48:23.236 | INFO | Baz IO overall: | 20260505 18:48:23.236 | INFO | BytesWritten : 3221017320898 | 20260505 18:48:23.236 | INFO | NumEvents : 1525302558230 | 20260505 18:48:24.963 | INFO | Analyzing chunk frames = [5480448,5480960) | 20260505 18:48:24.963 | INFO | MemInfo: { | 20260505 18:48:24.963 | INFO | "gpu" : " 17 GB", | 20260505 18:48:24.963 | INFO | "huge" : "", | 20260505 18:48:24.963 | INFO | "huge_shared" : " 27 GB", | 20260505 18:48:24.963 | INFO | "malloc" : " 14 GB", | 20260505 18:48:24.963 | INFO | "pinned" : " 29 GB" | 20260505 18:48:24.963 | INFO | } | 20260505 18:48:24.963 | INFO | Perf report for the last 5.13234 seconds: | 20260505 18:48:24.963 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:48:24.963 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:24.963 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:24.963 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:24.963 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 18:48:24.963 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:24.963 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260505 18:48:24.963 | INFO | PRE_HQ : 60.2%, 1.8, 55.7ms, 0 | 20260505 18:48:24.963 | INFO | BAZWRITER : 63.1%, 3.8, 122.0ms, 0 | 20260505 18:48:24.963 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 18:48:24.964 | INFO | Uploaded 36GiB to GPU in 2.13937s (16.8274GiB/s) | 20260505 18:48:24.964 | INFO | Downloaded 38GiB to GPU in 2.24258s (16.9448GiB/s) | 20260505 18:48:30.112 | INFO | Analyzing chunk frames = [5480960,5481472) | 20260505 18:48:30.112 | INFO | MemInfo: { | 20260505 18:48:30.112 | INFO | "gpu" : " 17 GB", | 20260505 18:48:30.112 | INFO | "huge" : "", | 20260505 18:48:30.112 | INFO | "huge_shared" : " 27 GB", | 20260505 18:48:30.112 | INFO | "malloc" : " 14 GB", | 20260505 18:48:30.112 | INFO | "pinned" : " 29 GB" | 20260505 18:48:30.112 | INFO | } | 20260505 18:48:30.112 | INFO | Perf report for the last 5.14915 seconds: | 20260505 18:48:30.112 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:48:30.112 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:30.112 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:30.112 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:30.112 | INFO | ANALYSIS : 62.7%, 3.0, 98.1ms, 0 | 20260505 18:48:30.112 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:30.112 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:30.112 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 18:48:30.112 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:30.112 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:30.113 | INFO | Uploaded 36GiB to GPU in 2.14247s (16.8031GiB/s) | 20260505 18:48:30.113 | INFO | Downloaded 34GiB to GPU in 2.0365s (16.6953GiB/s) | 20260505 18:48:35.250 | INFO | Analyzing chunk frames = [5481472,5481984) | 20260505 18:48:35.250 | INFO | MemInfo: { | 20260505 18:48:35.250 | INFO | "gpu" : " 17 GB", | 20260505 18:48:35.250 | INFO | "huge" : "", | 20260505 18:48:35.250 | INFO | "huge_shared" : " 27 GB", | 20260505 18:48:35.250 | INFO | "malloc" : " 14 GB", | 20260505 18:48:35.250 | INFO | "pinned" : " 29 GB" | 20260505 18:48:35.250 | INFO | } | 20260505 18:48:35.251 | INFO | Perf report for the last 5.13825 seconds: | 20260505 18:48:35.251 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:48:35.251 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:35.251 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:35.251 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:35.251 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260505 18:48:35.251 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:35.251 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:35.251 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:48:35.251 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:35.251 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:35.251 | INFO | Uploaded 36GiB to GPU in 2.12761s (16.9204GiB/s) | 20260505 18:48:35.251 | INFO | Downloaded 34GiB to GPU in 2.03589s (16.7003GiB/s) | 20260505 18:48:40.390 | INFO | Analyzing chunk frames = [5481984,5482496) | 20260505 18:48:40.390 | INFO | MemInfo: { | 20260505 18:48:40.390 | INFO | "gpu" : " 17 GB", | 20260505 18:48:40.390 | INFO | "huge" : "", | 20260505 18:48:40.390 | INFO | "huge_shared" : " 27 GB", | 20260505 18:48:40.390 | INFO | "malloc" : " 14 GB", | 20260505 18:48:40.390 | INFO | "pinned" : " 29 GB" | 20260505 18:48:40.390 | INFO | } | 20260505 18:48:40.390 | INFO | Perf report for the last 5.13967 seconds: | 20260505 18:48:40.390 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:48:40.390 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:40.390 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:40.390 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:40.390 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260505 18:48:40.390 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:40.390 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:40.390 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:48:40.390 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:40.390 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:40.390 | INFO | Uploaded 36GiB to GPU in 2.19859s (16.3741GiB/s) | 20260505 18:48:40.390 | INFO | Downloaded 34GiB to GPU in 2.05092s (16.5779GiB/s) | 20260505 18:48:45.527 | INFO | Analyzing chunk frames = [5482496,5483008) | 20260505 18:48:45.527 | INFO | MemInfo: { | 20260505 18:48:45.527 | INFO | "gpu" : " 17 GB", | 20260505 18:48:45.527 | INFO | "huge" : "", | 20260505 18:48:45.527 | INFO | "huge_shared" : " 27 GB", | 20260505 18:48:45.527 | INFO | "malloc" : " 14 GB", | 20260505 18:48:45.527 | INFO | "pinned" : " 29 GB" | 20260505 18:48:45.527 | INFO | } | 20260505 18:48:45.527 | INFO | Perf report for the last 5.137 seconds: | 20260505 18:48:45.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:48:45.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:45.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:45.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:45.527 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260505 18:48:45.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:45.527 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:45.527 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:48:45.527 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:45.527 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:45.528 | INFO | Uploaded 36GiB to GPU in 2.1386s (16.8334GiB/s) | 20260505 18:48:45.528 | INFO | Downloaded 34GiB to GPU in 2.06126s (16.4948GiB/s) | 20260505 18:48:50.668 | INFO | Analyzing chunk frames = [5483008,5483520) | 20260505 18:48:50.668 | INFO | MemInfo: { | 20260505 18:48:50.668 | INFO | "gpu" : " 17 GB", | 20260505 18:48:50.668 | INFO | "huge" : "", | 20260505 18:48:50.668 | INFO | "huge_shared" : " 27 GB", | 20260505 18:48:50.668 | INFO | "malloc" : " 14 GB", | 20260505 18:48:50.668 | INFO | "pinned" : " 29 GB" | 20260505 18:48:50.668 | INFO | } | 20260505 18:48:50.668 | INFO | Perf report for the last 5.14121 seconds: | 20260505 18:48:50.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:48:50.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:50.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:50.669 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:50.669 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260505 18:48:50.669 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:50.669 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:50.669 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:48:50.669 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:50.669 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:50.669 | INFO | Uploaded 36GiB to GPU in 2.14021s (16.8208GiB/s) | 20260505 18:48:50.669 | INFO | Downloaded 34GiB to GPU in 2.05206s (16.5687GiB/s) | 20260505 18:48:55.806 | INFO | Analyzing chunk frames = [5483520,5484032) | 20260505 18:48:55.807 | INFO | MemInfo: { | 20260505 18:48:55.807 | INFO | "gpu" : " 17 GB", | 20260505 18:48:55.807 | INFO | "huge" : "", | 20260505 18:48:55.807 | INFO | "huge_shared" : " 27 GB", | 20260505 18:48:55.807 | INFO | "malloc" : " 14 GB", | 20260505 18:48:55.807 | INFO | "pinned" : " 29 GB" | 20260505 18:48:55.807 | INFO | } | 20260505 18:48:55.807 | INFO | Perf report for the last 5.13829 seconds: | 20260505 18:48:55.807 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:48:55.807 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:55.807 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:55.807 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:55.807 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 18:48:55.807 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:55.807 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:55.807 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 18:48:55.807 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:55.807 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:48:55.807 | INFO | Uploaded 36GiB to GPU in 2.11866s (16.9919GiB/s) | 20260505 18:48:55.807 | INFO | Downloaded 34GiB to GPU in 2.13316s (15.9388GiB/s) | 20260505 18:49:00.947 | INFO | Analyzing chunk frames = [5484032,5484544) | 20260505 18:49:00.947 | INFO | MemInfo: { | 20260505 18:49:00.947 | INFO | "gpu" : " 17 GB", | 20260505 18:49:00.947 | INFO | "huge" : "", | 20260505 18:49:00.947 | INFO | "huge_shared" : " 27 GB", | 20260505 18:49:00.947 | INFO | "malloc" : " 14 GB", | 20260505 18:49:00.947 | INFO | "pinned" : " 29 GB" | 20260505 18:49:00.947 | INFO | } | 20260505 18:49:00.947 | INFO | Perf report for the last 5.14024 seconds: | 20260505 18:49:00.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:49:00.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:00.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:00.947 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:00.947 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 18:49:00.947 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:00.947 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:00.947 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260505 18:49:00.947 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:00.947 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:00.947 | INFO | Uploaded 36GiB to GPU in 2.19549s (16.3973GiB/s) | 20260505 18:49:00.947 | INFO | Downloaded 34GiB to GPU in 2.03802s (16.6829GiB/s) | 20260505 18:49:01.082 | INFO | Starting a round of Baz IO | 20260505 18:49:04.262 | INFO | Populating RTMetrics report | 20260505 18:49:04.263 | INFO | RTMetrics report populated | 20260505 18:49:04.263 | INFO | Opening tmp RTMetrics json file | 20260505 18:49:04.266 | INFO | Serializing RTMetrics to json | 20260505 18:49:04.267 | INFO | Serialization done, swapping tmp file now... | 20260505 18:49:04.267 | INFO | RTMetrics done | 20260505 18:49:04.367 | INFO | Finished a round of Baz IO: Wrote 2.076 GiB in 3.285 seconds | 20260505 18:49:04.367 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.86% / 0.00% / 78.14% | 20260505 18:49:04.367 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.234572% / 3.97364e-06% / 8.34465e-05% | 20260505 18:49:04.367 | INFO | Baz IO overall: | 20260505 18:49:04.367 | INFO | BytesWritten : 3223246073427 | 20260505 18:49:04.367 | INFO | NumEvents : 1526339660528 | 20260505 18:49:06.083 | INFO | Analyzing chunk frames = [5484544,5485056) | 20260505 18:49:06.083 | INFO | MemInfo: { | 20260505 18:49:06.083 | INFO | "gpu" : " 17 GB", | 20260505 18:49:06.083 | INFO | "huge" : "", | 20260505 18:49:06.083 | INFO | "huge_shared" : " 27 GB", | 20260505 18:49:06.083 | INFO | "malloc" : " 14 GB", | 20260505 18:49:06.083 | INFO | "pinned" : " 29 GB" | 20260505 18:49:06.083 | INFO | } | 20260505 18:49:06.083 | INFO | Perf report for the last 5.13611 seconds: | 20260505 18:49:06.083 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:49:06.083 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:06.083 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:06.083 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:06.083 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260505 18:49:06.083 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:06.083 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260505 18:49:06.083 | INFO | PRE_HQ : 60.4%, 1.8, 55.6ms, 0 | 20260505 18:49:06.083 | INFO | BAZWRITER : 63.5%, 3.8, 123.3ms, 0 | 20260505 18:49:06.083 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 18:49:06.084 | INFO | Uploaded 36GiB to GPU in 2.15913s (16.6734GiB/s) | 20260505 18:49:06.084 | INFO | Downloaded 38GiB to GPU in 2.24149s (16.953GiB/s) | 20260505 18:49:11.222 | INFO | Analyzing chunk frames = [5485056,5485568) | 20260505 18:49:11.223 | INFO | MemInfo: { | 20260505 18:49:11.223 | INFO | "gpu" : " 17 GB", | 20260505 18:49:11.223 | INFO | "huge" : "", | 20260505 18:49:11.223 | INFO | "huge_shared" : " 27 GB", | 20260505 18:49:11.223 | INFO | "malloc" : " 14 GB", | 20260505 18:49:11.223 | INFO | "pinned" : " 29 GB" | 20260505 18:49:11.223 | INFO | } | 20260505 18:49:11.223 | INFO | Perf report for the last 5.13961 seconds: | 20260505 18:49:11.223 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:49:11.223 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:11.223 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:11.223 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:11.223 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 18:49:11.223 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:11.223 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:11.223 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 18:49:11.223 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:11.223 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:11.223 | INFO | Uploaded 36GiB to GPU in 2.1297s (16.9038GiB/s) | 20260505 18:49:11.223 | INFO | Downloaded 34GiB to GPU in 2.02125s (16.8213GiB/s) | 20260505 18:49:16.368 | INFO | Analyzing chunk frames = [5485568,5486080) | 20260505 18:49:16.368 | INFO | MemInfo: { | 20260505 18:49:16.368 | INFO | "gpu" : " 17 GB", | 20260505 18:49:16.368 | INFO | "huge" : "", | 20260505 18:49:16.368 | INFO | "huge_shared" : " 27 GB", | 20260505 18:49:16.368 | INFO | "malloc" : " 14 GB", | 20260505 18:49:16.369 | INFO | "pinned" : " 29 GB" | 20260505 18:49:16.369 | INFO | } | 20260505 18:49:16.369 | INFO | Perf report for the last 5.14588 seconds: | 20260505 18:49:16.369 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:49:16.369 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:16.369 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:16.369 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:16.369 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260505 18:49:16.369 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:16.369 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:16.369 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:49:16.369 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:16.369 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:16.369 | INFO | Uploaded 36GiB to GPU in 2.13518s (16.8604GiB/s) | 20260505 18:49:16.369 | INFO | Downloaded 34GiB to GPU in 2.04673s (16.6119GiB/s) | 20260505 18:49:21.506 | INFO | Analyzing chunk frames = [5486080,5486592) | 20260505 18:49:21.506 | INFO | MemInfo: { | 20260505 18:49:21.506 | INFO | "gpu" : " 17 GB", | 20260505 18:49:21.506 | INFO | "huge" : "", | 20260505 18:49:21.506 | INFO | "huge_shared" : " 27 GB", | 20260505 18:49:21.506 | INFO | "malloc" : " 14 GB", | 20260505 18:49:21.506 | INFO | "pinned" : " 29 GB" | 20260505 18:49:21.506 | INFO | } | 20260505 18:49:21.507 | INFO | Perf report for the last 5.13784 seconds: | 20260505 18:49:21.507 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:49:21.507 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:21.507 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:21.507 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:21.507 | INFO | ANALYSIS : 62.4%, 3.0, 96.8ms, 0 | 20260505 18:49:21.507 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:21.507 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:21.507 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:49:21.507 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:21.507 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:21.507 | INFO | Uploaded 36GiB to GPU in 2.19864s (16.3738GiB/s) | 20260505 18:49:21.507 | INFO | Downloaded 34GiB to GPU in 2.04815s (16.6004GiB/s) | 20260505 18:49:26.639 | INFO | Analyzing chunk frames = [5486592,5487104) | 20260505 18:49:26.639 | INFO | MemInfo: { | 20260505 18:49:26.639 | INFO | "gpu" : " 17 GB", | 20260505 18:49:26.639 | INFO | "huge" : "", | 20260505 18:49:26.639 | INFO | "huge_shared" : " 27 GB", | 20260505 18:49:26.639 | INFO | "malloc" : " 14 GB", | 20260505 18:49:26.639 | INFO | "pinned" : " 29 GB" | 20260505 18:49:26.639 | INFO | } | 20260505 18:49:26.639 | INFO | Perf report for the last 5.13233 seconds: | 20260505 18:49:26.639 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:49:26.639 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:26.639 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:26.639 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:26.639 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260505 18:49:26.639 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:26.639 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:26.639 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:49:26.639 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:26.639 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:26.639 | INFO | Uploaded 36GiB to GPU in 2.1399s (16.8232GiB/s) | 20260505 18:49:26.639 | INFO | Downloaded 34GiB to GPU in 2.06386s (16.474GiB/s) | 20260505 18:49:31.782 | INFO | Analyzing chunk frames = [5487104,5487616) | 20260505 18:49:31.782 | INFO | MemInfo: { | 20260505 18:49:31.782 | INFO | "gpu" : " 17 GB", | 20260505 18:49:31.782 | INFO | "huge" : "", | 20260505 18:49:31.782 | INFO | "huge_shared" : " 27 GB", | 20260505 18:49:31.782 | INFO | "malloc" : " 14 GB", | 20260505 18:49:31.782 | INFO | "pinned" : " 29 GB" | 20260505 18:49:31.782 | INFO | } | 20260505 18:49:31.782 | INFO | Perf report for the last 5.14353 seconds: | 20260505 18:49:31.783 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:49:31.783 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:31.783 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:31.783 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:31.783 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260505 18:49:31.783 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:31.783 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:31.783 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:49:31.783 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:31.783 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:31.783 | INFO | Uploaded 36GiB to GPU in 2.14005s (16.8221GiB/s) | 20260505 18:49:31.783 | INFO | Downloaded 34GiB to GPU in 2.04366s (16.6369GiB/s) | 20260505 18:49:36.919 | INFO | Analyzing chunk frames = [5487616,5488128) | 20260505 18:49:36.919 | INFO | MemInfo: { | 20260505 18:49:36.919 | INFO | "gpu" : " 17 GB", | 20260505 18:49:36.919 | INFO | "huge" : "", | 20260505 18:49:36.919 | INFO | "huge_shared" : " 27 GB", | 20260505 18:49:36.919 | INFO | "malloc" : " 14 GB", | 20260505 18:49:36.919 | INFO | "pinned" : " 29 GB" | 20260505 18:49:36.919 | INFO | } | 20260505 18:49:36.919 | INFO | Perf report for the last 5.13665 seconds: | 20260505 18:49:36.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:49:36.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:36.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:36.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:36.919 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 18:49:36.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:36.919 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:36.919 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 18:49:36.919 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:36.919 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:36.919 | INFO | Uploaded 36GiB to GPU in 2.15722s (16.6881GiB/s) | 20260505 18:49:36.919 | INFO | Downloaded 34GiB to GPU in 2.17286s (15.6476GiB/s) | 20260505 18:49:42.059 | INFO | Analyzing chunk frames = [5488128,5488640) | 20260505 18:49:42.059 | INFO | MemInfo: { | 20260505 18:49:42.059 | INFO | "gpu" : " 17 GB", | 20260505 18:49:42.059 | INFO | "huge" : "", | 20260505 18:49:42.060 | INFO | "huge_shared" : " 27 GB", | 20260505 18:49:42.060 | INFO | "malloc" : " 14 GB", | 20260505 18:49:42.060 | INFO | "pinned" : " 29 GB" | 20260505 18:49:42.060 | INFO | } | 20260505 18:49:42.060 | INFO | Perf report for the last 5.14058 seconds: | 20260505 18:49:42.060 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:49:42.060 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:42.060 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:42.060 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:42.060 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 18:49:42.060 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:42.060 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:42.060 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:49:42.060 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:42.060 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:42.060 | INFO | Uploaded 36GiB to GPU in 2.18125s (16.5043GiB/s) | 20260505 18:49:42.060 | INFO | Downloaded 34GiB to GPU in 2.02471s (16.7925GiB/s) | 20260505 18:49:42.060 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 18:49:42.060 | INFO | Total mem usage: 15.3 GiB | 20260505 18:49:42.060 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 18:49:42.060 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.15%) | 20260505 18:49:42.060 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 18:49:42.060 | INFO | High Water Breakdown: | 20260505 18:49:42.060 | INFO | 61% : 8.64 GiB ( 90.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 18:49:42.060 | INFO | 38.9% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 18:49:42.060 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 18:49:42.060 | INFO | ------------------------------------------ | 20260505 18:49:42.060 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:49:42.060 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 18:49:42.060 | INFO | Total mem usage: 26.5 GiB | 20260505 18:49:42.060 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 18:49:42.060 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 18:49:42.060 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 18:49:42.060 | INFO | High Water Breakdown: | 20260505 18:49:42.060 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:49:42.061 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:49:42.061 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:49:42.061 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:49:42.061 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:49:42.061 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:49:42.061 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 18:49:42.061 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 18:49:42.061 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:49:42.061 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 18:49:42.061 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 18:49:42.061 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 18:49:42.061 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 18:49:42.061 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 18:49:42.061 | INFO | ------------------------------------------ | 20260505 18:49:42.061 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:49:42.061 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 18:49:42.061 | INFO | Total mem usage: 25.6 GiB | 20260505 18:49:42.061 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 18:49:42.061 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 18:49:42.061 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 18:49:42.061 | INFO | High Water Breakdown: | 20260505 18:49:42.061 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 18:49:42.061 | INFO | ------------------------------------------ | 20260505 18:49:42.061 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:49:42.061 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 18:49:42.061 | INFO | Total mem usage: 15.7 GiB | 20260505 18:49:42.061 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 18:49:42.061 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 18:49:42.061 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 18:49:42.061 | INFO | High Water Breakdown: | 20260505 18:49:42.061 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 18:49:42.061 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 18:49:42.061 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 18:49:42.061 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 18:49:42.061 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 18:49:42.061 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 18:49:42.061 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 18:49:42.061 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 18:49:42.061 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 18:49:42.061 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:49:42.061 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 18:49:42.061 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:49:42.061 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 18:49:42.061 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 18:49:42.061 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 18:49:42.061 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 18:49:42.061 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:49:42.061 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 18:49:42.061 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:49:42.061 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:49:42.061 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:49:42.061 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 18:49:42.061 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 18:49:42.061 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:49:42.061 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 18:49:42.061 | INFO | ------------------------------------------ | 20260505 18:49:42.061 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:49:42.061 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 18:49:42.061 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:49:42.061 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:49:42.061 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.47e+01, 1.35e+01 / 1.94e+00 | 20260505 18:49:42.061 | INFO | PerfReport: 0.29%, AggregateStats -- 9.25e-02 / 1.04e-01, 1.01e-02 / 5.89e-03 | 20260505 18:49:42.061 | INFO | PerfReport: 11.80%, Baseline -- 3.73e+00 / 4.51e+00, 5.64e-01 / 1.72e-01 | 20260505 18:49:42.061 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.53e+00, 1.90e-01 / 8.70e-02 | 20260505 18:49:42.061 | INFO | PerfReport: 15.90%, DME -- 4.02e+01 / 4.44e+01, 4.11e+00 / 2.02e+00 | 20260505 18:49:42.061 | INFO | PerfReport: 57.01%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.56e+00 / 1.26e+00 | 20260505 18:49:42.061 | INFO | PerfReport: 7.26%, PulseAccumulating -- 2.29e+00 / 2.88e+00, 4.36e-01 / 1.69e-01 | 20260505 18:49:42.061 | INFO | PerfReport: 3.31%, Metrics -- 1.04e+00 / 1.38e+00, 2.89e-01 / 1.44e-01 | 20260505 18:49:42.061 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.64e-01 / 4.25e-01, 6.42e-02 / 3.91e-02 | 20260505 18:49:42.061 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 18:49:42.061 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:49:42.061 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:49:42.061 | INFO | PerfReport: Full Execution -- 9.97e+01 / -nan, 2.54e+01 / 0.00e+00 | 20260505 18:49:42.061 | INFO | PerfReport: 22.49%, Upload -- 2.24e+01 / 0.00e+00, 7.66e+00 / 0.00e+00 | 20260505 18:49:42.061 | INFO | PerfReport: 21.75%, Download -- 2.17e+01 / 0.00e+00, 7.62e+00 / 0.00e+00 | 20260505 18:49:42.061 | INFO | PerfReport: 31.69%, Compute -- 3.16e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 18:49:42.061 | INFO | PerfReport: 2.26%, LockUpload -- 2.25e+00 / 0.00e+00, 7.91e+00 / 0.00e+00 | 20260505 18:49:42.061 | INFO | PerfReport: 1.37%, LockDownload -- 1.37e+00 / 0.00e+00, 2.89e+00 / 0.00e+00 | 20260505 18:49:42.061 | INFO | PerfReport: 20.42%, LockCompute -- 2.04e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260505 18:49:42.061 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.58e-02 / 0.00e+00 | 20260505 18:49:42.197 | INFO | Starting a round of Baz IO | 20260505 18:49:45.367 | INFO | Populating RTMetrics report | 20260505 18:49:45.367 | INFO | RTMetrics report populated | 20260505 18:49:45.367 | INFO | Opening tmp RTMetrics json file | 20260505 18:49:45.370 | INFO | Serializing RTMetrics to json | 20260505 18:49:45.372 | INFO | Serialization done, swapping tmp file now... | 20260505 18:49:45.372 | INFO | RTMetrics done | 20260505 18:49:45.462 | INFO | Finished a round of Baz IO: Wrote 2.077 GiB in 3.265 seconds | 20260505 18:49:45.462 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.86% / 0.00% / 78.14% | 20260505 18:49:45.462 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.233241% / 0% / 7.54992e-05% | 20260505 18:49:45.462 | INFO | Baz IO overall: | 20260505 18:49:45.462 | INFO | BytesWritten : 3225476151135 | 20260505 18:49:45.462 | INFO | NumEvents : 1527377381969 | 20260505 18:49:47.204 | INFO | Analyzing chunk frames = [5488640,5489152) | 20260505 18:49:47.204 | INFO | MemInfo: { | 20260505 18:49:47.204 | INFO | "gpu" : " 17 GB", | 20260505 18:49:47.204 | INFO | "huge" : "", | 20260505 18:49:47.204 | INFO | "huge_shared" : " 27 GB", | 20260505 18:49:47.204 | INFO | "malloc" : " 14 GB", | 20260505 18:49:47.204 | INFO | "pinned" : " 29 GB" | 20260505 18:49:47.204 | INFO | } | 20260505 18:49:47.204 | INFO | Perf report for the last 5.14396 seconds: | 20260505 18:49:47.204 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:49:47.204 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:47.204 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:47.204 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:47.204 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 18:49:47.204 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:47.204 | INFO | RT_METRICS : 44.3%, 1.0, 23.3ms, 0 | 20260505 18:49:47.204 | INFO | PRE_HQ : 60.1%, 1.8, 56.2ms, 0 | 20260505 18:49:47.204 | INFO | BAZWRITER : 63.1%, 3.8, 123.1ms, 0 | 20260505 18:49:47.204 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 18:49:47.204 | INFO | Uploaded 36GiB to GPU in 2.15216s (16.7274GiB/s) | 20260505 18:49:47.204 | INFO | Downloaded 38GiB to GPU in 2.2373s (16.9848GiB/s) | 20260505 18:49:52.339 | INFO | Analyzing chunk frames = [5489152,5489664) | 20260505 18:49:52.339 | INFO | MemInfo: { | 20260505 18:49:52.339 | INFO | "gpu" : " 17 GB", | 20260505 18:49:52.339 | INFO | "huge" : "", | 20260505 18:49:52.339 | INFO | "huge_shared" : " 27 GB", | 20260505 18:49:52.339 | INFO | "malloc" : " 14 GB", | 20260505 18:49:52.339 | INFO | "pinned" : " 29 GB" | 20260505 18:49:52.339 | INFO | } | 20260505 18:49:52.340 | INFO | Perf report for the last 5.13585 seconds: | 20260505 18:49:52.340 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:49:52.340 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:52.340 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:52.340 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:52.340 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 18:49:52.340 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:52.340 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:52.340 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 18:49:52.340 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:52.340 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:52.340 | INFO | Uploaded 36GiB to GPU in 2.14059s (16.8178GiB/s) | 20260505 18:49:52.340 | INFO | Downloaded 34GiB to GPU in 2.03231s (16.7298GiB/s) | 20260505 18:49:57.472 | INFO | Analyzing chunk frames = [5489664,5490176) | 20260505 18:49:57.472 | INFO | MemInfo: { | 20260505 18:49:57.472 | INFO | "gpu" : " 17 GB", | 20260505 18:49:57.472 | INFO | "huge" : "", | 20260505 18:49:57.472 | INFO | "huge_shared" : " 27 GB", | 20260505 18:49:57.472 | INFO | "malloc" : " 14 GB", | 20260505 18:49:57.472 | INFO | "pinned" : " 29 GB" | 20260505 18:49:57.473 | INFO | } | 20260505 18:49:57.473 | INFO | Perf report for the last 5.1332 seconds: | 20260505 18:49:57.473 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:49:57.473 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:57.473 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:57.473 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:57.473 | INFO | ANALYSIS : 62.7%, 3.0, 97.4ms, 0 | 20260505 18:49:57.473 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:57.473 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:57.473 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 18:49:57.473 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:57.473 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:49:57.473 | INFO | Uploaded 36GiB to GPU in 2.20627s (16.3171GiB/s) | 20260505 18:49:57.473 | INFO | Downloaded 34GiB to GPU in 2.0624s (16.4857GiB/s) | 20260505 18:50:02.622 | INFO | Analyzing chunk frames = [5490176,5490688) | 20260505 18:50:02.622 | INFO | MemInfo: { | 20260505 18:50:02.622 | INFO | "gpu" : " 17 GB", | 20260505 18:50:02.622 | INFO | "huge" : "", | 20260505 18:50:02.622 | INFO | "huge_shared" : " 27 GB", | 20260505 18:50:02.622 | INFO | "malloc" : " 14 GB", | 20260505 18:50:02.622 | INFO | "pinned" : " 29 GB" | 20260505 18:50:02.622 | INFO | } | 20260505 18:50:02.622 | INFO | Perf report for the last 5.14914 seconds: | 20260505 18:50:02.622 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:50:02.622 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:02.622 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:02.622 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:02.622 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260505 18:50:02.622 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:02.622 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:02.622 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 18:50:02.622 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:02.622 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:02.622 | INFO | Uploaded 36GiB to GPU in 2.19846s (16.3751GiB/s) | 20260505 18:50:02.622 | INFO | Downloaded 34GiB to GPU in 2.14103s (15.8802GiB/s) | 20260505 18:50:07.758 | INFO | Analyzing chunk frames = [5490688,5491200) | 20260505 18:50:07.758 | INFO | MemInfo: { | 20260505 18:50:07.758 | INFO | "gpu" : " 17 GB", | 20260505 18:50:07.758 | INFO | "huge" : "", | 20260505 18:50:07.758 | INFO | "huge_shared" : " 27 GB", | 20260505 18:50:07.758 | INFO | "malloc" : " 14 GB", | 20260505 18:50:07.758 | INFO | "pinned" : " 29 GB" | 20260505 18:50:07.758 | INFO | } | 20260505 18:50:07.759 | INFO | Perf report for the last 5.13666 seconds: | 20260505 18:50:07.759 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:50:07.759 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:07.759 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:07.759 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:07.759 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260505 18:50:07.759 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:07.759 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:07.759 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:50:07.759 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:07.759 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:07.759 | INFO | Uploaded 36GiB to GPU in 2.14083s (16.8159GiB/s) | 20260505 18:50:07.759 | INFO | Downloaded 34GiB to GPU in 2.05971s (16.5071GiB/s) | 20260505 18:50:12.895 | INFO | Analyzing chunk frames = [5491200,5491712) | 20260505 18:50:12.895 | INFO | MemInfo: { | 20260505 18:50:12.895 | INFO | "gpu" : " 17 GB", | 20260505 18:50:12.895 | INFO | "huge" : "", | 20260505 18:50:12.895 | INFO | "huge_shared" : " 27 GB", | 20260505 18:50:12.895 | INFO | "malloc" : " 14 GB", | 20260505 18:50:12.895 | INFO | "pinned" : " 29 GB" | 20260505 18:50:12.895 | INFO | } | 20260505 18:50:12.895 | INFO | Perf report for the last 5.13657 seconds: | 20260505 18:50:12.895 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:50:12.895 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:12.895 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:12.895 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:12.895 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260505 18:50:12.895 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:12.895 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:12.895 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:50:12.895 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:12.895 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:12.896 | INFO | Uploaded 36GiB to GPU in 2.14171s (16.809GiB/s) | 20260505 18:50:12.896 | INFO | Downloaded 34GiB to GPU in 2.05191s (16.5699GiB/s) | 20260505 18:50:18.034 | INFO | Analyzing chunk frames = [5491712,5492224) | 20260505 18:50:18.034 | INFO | MemInfo: { | 20260505 18:50:18.034 | INFO | "gpu" : " 17 GB", | 20260505 18:50:18.034 | INFO | "huge" : "", | 20260505 18:50:18.034 | INFO | "huge_shared" : " 27 GB", | 20260505 18:50:18.034 | INFO | "malloc" : " 14 GB", | 20260505 18:50:18.034 | INFO | "pinned" : " 29 GB" | 20260505 18:50:18.034 | INFO | } | 20260505 18:50:18.034 | INFO | Perf report for the last 5.13875 seconds: | 20260505 18:50:18.034 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:50:18.034 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:18.034 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:18.034 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:18.034 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 18:50:18.034 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:18.034 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:18.034 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 18:50:18.034 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:18.034 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:18.034 | INFO | Uploaded 36GiB to GPU in 2.15742s (16.6866GiB/s) | 20260505 18:50:18.034 | INFO | Downloaded 34GiB to GPU in 2.18378s (15.5694GiB/s) | 20260505 18:50:23.175 | INFO | Analyzing chunk frames = [5492224,5492736) | 20260505 18:50:23.175 | INFO | MemInfo: { | 20260505 18:50:23.175 | INFO | "gpu" : " 17 GB", | 20260505 18:50:23.175 | INFO | "huge" : "", | 20260505 18:50:23.175 | INFO | "huge_shared" : " 27 GB", | 20260505 18:50:23.175 | INFO | "malloc" : " 14 GB", | 20260505 18:50:23.176 | INFO | "pinned" : " 29 GB" | 20260505 18:50:23.176 | INFO | } | 20260505 18:50:23.176 | INFO | Perf report for the last 5.14188 seconds: | 20260505 18:50:23.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:50:23.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:23.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:23.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:23.176 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 18:50:23.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:23.176 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:23.176 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:50:23.176 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:23.176 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:23.176 | INFO | Uploaded 36GiB to GPU in 2.20039s (16.3607GiB/s) | 20260505 18:50:23.176 | INFO | Downloaded 34GiB to GPU in 2.03677s (16.6931GiB/s) | 20260505 18:50:23.307 | INFO | Starting a round of Baz IO | 20260505 18:50:26.491 | INFO | Populating RTMetrics report | 20260505 18:50:26.491 | INFO | RTMetrics report populated | 20260505 18:50:26.491 | INFO | Opening tmp RTMetrics json file | 20260505 18:50:26.494 | INFO | Serializing RTMetrics to json | 20260505 18:50:26.495 | INFO | Serialization done, swapping tmp file now... | 20260505 18:50:26.496 | INFO | RTMetrics done | 20260505 18:50:26.585 | INFO | Finished a round of Baz IO: Wrote 2.075 GiB in 3.277 seconds | 20260505 18:50:26.585 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.85% / 0.00% / 78.14% | 20260505 18:50:26.585 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.23338% / 3.97364e-06% / 9.13938e-05% | 20260505 18:50:26.585 | INFO | Baz IO overall: | 20260505 18:50:26.585 | INFO | BytesWritten : 3227704362612 | 20260505 18:50:26.585 | INFO | NumEvents : 1528414137402 | 20260505 18:50:28.322 | INFO | Analyzing chunk frames = [5492736,5493248) | 20260505 18:50:28.322 | INFO | MemInfo: { | 20260505 18:50:28.322 | INFO | "gpu" : " 17 GB", | 20260505 18:50:28.322 | INFO | "huge" : "", | 20260505 18:50:28.322 | INFO | "huge_shared" : " 27 GB", | 20260505 18:50:28.322 | INFO | "malloc" : " 14 GB", | 20260505 18:50:28.322 | INFO | "pinned" : " 29 GB" | 20260505 18:50:28.322 | INFO | } | 20260505 18:50:28.322 | INFO | Perf report for the last 5.14643 seconds: | 20260505 18:50:28.322 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:50:28.322 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:28.322 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:28.322 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:28.322 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 18:50:28.322 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:28.322 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260505 18:50:28.322 | INFO | PRE_HQ : 60.4%, 1.8, 55.6ms, 0 | 20260505 18:50:28.322 | INFO | BAZWRITER : 63.3%, 3.8, 122.6ms, 0 | 20260505 18:50:28.322 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 18:50:28.323 | INFO | Uploaded 36GiB to GPU in 2.18715s (16.4598GiB/s) | 20260505 18:50:28.323 | INFO | Downloaded 38GiB to GPU in 2.26057s (16.8099GiB/s) | 20260505 18:50:33.456 | INFO | Analyzing chunk frames = [5493248,5493760) | 20260505 18:50:33.456 | INFO | MemInfo: { | 20260505 18:50:33.456 | INFO | "gpu" : " 17 GB", | 20260505 18:50:33.456 | INFO | "huge" : "", | 20260505 18:50:33.456 | INFO | "huge_shared" : " 27 GB", | 20260505 18:50:33.456 | INFO | "malloc" : " 14 GB", | 20260505 18:50:33.457 | INFO | "pinned" : " 29 GB" | 20260505 18:50:33.457 | INFO | } | 20260505 18:50:33.457 | INFO | Perf report for the last 5.13442 seconds: | 20260505 18:50:33.457 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:50:33.457 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:33.457 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:33.457 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:33.457 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260505 18:50:33.457 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:33.457 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:33.457 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:50:33.457 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:33.457 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:33.457 | INFO | Uploaded 36GiB to GPU in 2.09441s (17.1886GiB/s) | 20260505 18:50:33.457 | INFO | Downloaded 34GiB to GPU in 2.0254s (16.7868GiB/s) | 20260505 18:50:38.589 | INFO | Analyzing chunk frames = [5493760,5494272) | 20260505 18:50:38.589 | INFO | MemInfo: { | 20260505 18:50:38.589 | INFO | "gpu" : " 17 GB", | 20260505 18:50:38.589 | INFO | "huge" : "", | 20260505 18:50:38.589 | INFO | "huge_shared" : " 27 GB", | 20260505 18:50:38.589 | INFO | "malloc" : " 14 GB", | 20260505 18:50:38.589 | INFO | "pinned" : " 29 GB" | 20260505 18:50:38.589 | INFO | } | 20260505 18:50:38.589 | INFO | Perf report for the last 5.13236 seconds: | 20260505 18:50:38.589 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:50:38.589 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:38.589 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:38.589 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:38.589 | INFO | ANALYSIS : 62.4%, 3.0, 97.0ms, 0 | 20260505 18:50:38.589 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:38.589 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:38.589 | INFO | PRE_HQ : 27.5%, 1.0, 14.2ms, 0 | 20260505 18:50:38.589 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:38.589 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:38.589 | INFO | Uploaded 36GiB to GPU in 2.13755s (16.8417GiB/s) | 20260505 18:50:38.589 | INFO | Downloaded 34GiB to GPU in 2.04113s (16.6575GiB/s) | 20260505 18:50:43.728 | INFO | Analyzing chunk frames = [5494272,5494784) | 20260505 18:50:43.728 | INFO | MemInfo: { | 20260505 18:50:43.728 | INFO | "gpu" : " 17 GB", | 20260505 18:50:43.728 | INFO | "huge" : "", | 20260505 18:50:43.728 | INFO | "huge_shared" : " 27 GB", | 20260505 18:50:43.728 | INFO | "malloc" : " 14 GB", | 20260505 18:50:43.729 | INFO | "pinned" : " 29 GB" | 20260505 18:50:43.729 | INFO | } | 20260505 18:50:43.729 | INFO | Perf report for the last 5.13972 seconds: | 20260505 18:50:43.729 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:50:43.729 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:43.729 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:43.729 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:43.729 | INFO | ANALYSIS : 62.6%, 3.0, 97.2ms, 0 | 20260505 18:50:43.729 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:43.729 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:43.729 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:50:43.729 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:43.729 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:43.729 | INFO | Uploaded 36GiB to GPU in 2.1731s (16.5662GiB/s) | 20260505 18:50:43.729 | INFO | Downloaded 34GiB to GPU in 2.14322s (15.864GiB/s) | 20260505 18:50:48.876 | INFO | Analyzing chunk frames = [5494784,5495296) | 20260505 18:50:48.876 | INFO | MemInfo: { | 20260505 18:50:48.876 | INFO | "gpu" : " 17 GB", | 20260505 18:50:48.876 | INFO | "huge" : "", | 20260505 18:50:48.876 | INFO | "huge_shared" : " 27 GB", | 20260505 18:50:48.876 | INFO | "malloc" : " 14 GB", | 20260505 18:50:48.876 | INFO | "pinned" : " 29 GB" | 20260505 18:50:48.876 | INFO | } | 20260505 18:50:48.876 | INFO | Perf report for the last 5.14692 seconds: | 20260505 18:50:48.876 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:50:48.876 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:48.876 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:48.876 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:48.876 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260505 18:50:48.876 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:48.876 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:48.876 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:50:48.876 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:48.876 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:48.876 | INFO | Uploaded 36GiB to GPU in 2.14227s (16.8046GiB/s) | 20260505 18:50:48.876 | INFO | Downloaded 34GiB to GPU in 2.06324s (16.4789GiB/s) | 20260505 18:50:54.012 | INFO | Analyzing chunk frames = [5495296,5495808) | 20260505 18:50:54.012 | INFO | MemInfo: { | 20260505 18:50:54.012 | INFO | "gpu" : " 17 GB", | 20260505 18:50:54.012 | INFO | "huge" : "", | 20260505 18:50:54.012 | INFO | "huge_shared" : " 27 GB", | 20260505 18:50:54.012 | INFO | "malloc" : " 14 GB", | 20260505 18:50:54.012 | INFO | "pinned" : " 29 GB" | 20260505 18:50:54.012 | INFO | } | 20260505 18:50:54.012 | INFO | Perf report for the last 5.13637 seconds: | 20260505 18:50:54.012 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:50:54.012 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:54.012 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:54.012 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:54.012 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 18:50:54.012 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:54.012 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:54.012 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:50:54.012 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:54.012 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:54.013 | INFO | Uploaded 36GiB to GPU in 2.15301s (16.7208GiB/s) | 20260505 18:50:54.013 | INFO | Downloaded 34GiB to GPU in 2.05365s (16.5559GiB/s) | 20260505 18:50:59.151 | INFO | Analyzing chunk frames = [5495808,5496320) | 20260505 18:50:59.151 | INFO | MemInfo: { | 20260505 18:50:59.151 | INFO | "gpu" : " 17 GB", | 20260505 18:50:59.151 | INFO | "huge" : "", | 20260505 18:50:59.152 | INFO | "huge_shared" : " 27 GB", | 20260505 18:50:59.152 | INFO | "malloc" : " 14 GB", | 20260505 18:50:59.152 | INFO | "pinned" : " 29 GB" | 20260505 18:50:59.152 | INFO | } | 20260505 18:50:59.152 | INFO | Perf report for the last 5.13954 seconds: | 20260505 18:50:59.152 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:50:59.152 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:59.152 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:59.152 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:59.152 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 18:50:59.152 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:59.152 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:59.152 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 18:50:59.152 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:59.152 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:50:59.152 | INFO | Uploaded 36GiB to GPU in 2.13559s (16.8571GiB/s) | 20260505 18:50:59.152 | INFO | Downloaded 34GiB to GPU in 2.13342s (15.9369GiB/s) | 20260505 18:51:04.287 | INFO | Analyzing chunk frames = [5496320,5496832) | 20260505 18:51:04.287 | INFO | MemInfo: { | 20260505 18:51:04.287 | INFO | "gpu" : " 17 GB", | 20260505 18:51:04.287 | INFO | "huge" : "", | 20260505 18:51:04.287 | INFO | "huge_shared" : " 27 GB", | 20260505 18:51:04.287 | INFO | "malloc" : " 14 GB", | 20260505 18:51:04.287 | INFO | "pinned" : " 29 GB" | 20260505 18:51:04.288 | INFO | } | 20260505 18:51:04.288 | INFO | Perf report for the last 5.13603 seconds: | 20260505 18:51:04.288 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:51:04.288 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:04.288 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:04.288 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:04.288 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 18:51:04.288 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:04.288 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:04.288 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 18:51:04.288 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:04.288 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:04.288 | INFO | Uploaded 36GiB to GPU in 2.19609s (16.3927GiB/s) | 20260505 18:51:04.288 | INFO | Downloaded 34GiB to GPU in 2.03875s (16.6769GiB/s) | 20260505 18:51:04.288 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 18:51:04.288 | INFO | Total mem usage: 15.3 GiB | 20260505 18:51:04.288 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 18:51:04.288 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.13%) | 20260505 18:51:04.288 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 18:51:04.288 | INFO | High Water Breakdown: | 20260505 18:51:04.288 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 18:51:04.288 | INFO | 38.9% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 18:51:04.288 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 18:51:04.288 | INFO | ------------------------------------------ | 20260505 18:51:04.288 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:51:04.288 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 18:51:04.288 | INFO | Total mem usage: 26.5 GiB | 20260505 18:51:04.288 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 18:51:04.288 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 18:51:04.288 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 18:51:04.289 | INFO | High Water Breakdown: | 20260505 18:51:04.289 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:51:04.289 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:51:04.289 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:51:04.289 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:51:04.289 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:51:04.289 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:51:04.289 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 18:51:04.289 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 18:51:04.289 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:51:04.289 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 18:51:04.289 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 18:51:04.289 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 18:51:04.289 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 18:51:04.289 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 18:51:04.289 | INFO | ------------------------------------------ | 20260505 18:51:04.289 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:51:04.289 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 18:51:04.289 | INFO | Total mem usage: 25.6 GiB | 20260505 18:51:04.289 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 18:51:04.289 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 18:51:04.289 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 18:51:04.289 | INFO | High Water Breakdown: | 20260505 18:51:04.289 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 18:51:04.289 | INFO | ------------------------------------------ | 20260505 18:51:04.289 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:51:04.289 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 18:51:04.289 | INFO | Total mem usage: 15.7 GiB | 20260505 18:51:04.289 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 18:51:04.289 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 18:51:04.289 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 18:51:04.289 | INFO | High Water Breakdown: | 20260505 18:51:04.289 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 18:51:04.289 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 18:51:04.289 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 18:51:04.289 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 18:51:04.289 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 18:51:04.289 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 18:51:04.289 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 18:51:04.289 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 18:51:04.289 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 18:51:04.289 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:51:04.289 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 18:51:04.289 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:51:04.289 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 18:51:04.289 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 18:51:04.289 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 18:51:04.289 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 18:51:04.289 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:51:04.289 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 18:51:04.289 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:51:04.289 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:51:04.289 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:51:04.289 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 18:51:04.289 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 18:51:04.289 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:51:04.289 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 18:51:04.289 | INFO | ------------------------------------------ | 20260505 18:51:04.289 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:51:04.289 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 18:51:04.289 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:51:04.289 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:51:04.289 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.50e+01, 1.37e+01 / 2.16e+00 | 20260505 18:51:04.289 | INFO | PerfReport: 0.29%, AggregateStats -- 9.26e-02 / 1.01e-01, 1.06e-02 / 3.94e-03 | 20260505 18:51:04.289 | INFO | PerfReport: 11.77%, Baseline -- 3.71e+00 / 4.53e+00, 5.62e-01 / 1.25e-01 | 20260505 18:51:04.289 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.51e+00, 1.92e-01 / 1.00e-01 | 20260505 18:51:04.289 | INFO | PerfReport: 15.97%, DME -- 4.03e+01 / 4.48e+01, 4.33e+00 / 2.30e+00 | 20260505 18:51:04.289 | INFO | PerfReport: 56.93%, FrameLabeling -- 1.80e+01 / 1.93e+01, 1.46e+00 / 1.14e+00 | 20260505 18:51:04.289 | INFO | PerfReport: 7.29%, PulseAccumulating -- 2.30e+00 / 2.88e+00, 4.40e-01 / 2.14e-01 | 20260505 18:51:04.289 | INFO | PerfReport: 3.31%, Metrics -- 1.04e+00 / 1.37e+00, 2.88e-01 / 1.12e-01 | 20260505 18:51:04.289 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.64e-01 / 4.26e-01, 6.44e-02 / 3.19e-02 | 20260505 18:51:04.289 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 18:51:04.289 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:51:04.289 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:51:04.289 | INFO | PerfReport: Full Execution -- 9.97e+01 / -nan, 2.55e+01 / 0.00e+00 | 20260505 18:51:04.290 | INFO | PerfReport: 22.56%, Upload -- 2.25e+01 / 0.00e+00, 7.80e+00 / 0.00e+00 | 20260505 18:51:04.290 | INFO | PerfReport: 21.91%, Download -- 2.19e+01 / 0.00e+00, 7.78e+00 / 0.00e+00 | 20260505 18:51:04.290 | INFO | PerfReport: 31.63%, Compute -- 3.15e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 18:51:04.290 | INFO | PerfReport: 2.22%, LockUpload -- 2.21e+00 / 0.00e+00, 7.75e+00 / 0.00e+00 | 20260505 18:51:04.290 | INFO | PerfReport: 1.41%, LockDownload -- 1.41e+00 / 0.00e+00, 2.87e+00 / 0.00e+00 | 20260505 18:51:04.290 | INFO | PerfReport: 20.27%, LockCompute -- 2.02e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 18:51:04.290 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.73e-02 / 0.00e+00 | 20260505 18:51:04.423 | INFO | Starting a round of Baz IO | 20260505 18:51:07.602 | INFO | Populating RTMetrics report | 20260505 18:51:07.602 | INFO | RTMetrics report populated | 20260505 18:51:07.602 | INFO | Opening tmp RTMetrics json file | 20260505 18:51:07.606 | INFO | Serializing RTMetrics to json | 20260505 18:51:07.608 | INFO | Serialization done, swapping tmp file now... | 20260505 18:51:07.609 | INFO | RTMetrics done | 20260505 18:51:07.695 | INFO | Finished a round of Baz IO: Wrote 2.074 GiB in 3.272 seconds | 20260505 18:51:07.695 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.85% / 0.00% / 78.15% | 20260505 18:51:07.695 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.232359% / 3.97364e-06% / 5.5631e-05% | 20260505 18:51:07.695 | INFO | Baz IO overall: | 20260505 18:51:07.695 | INFO | BytesWritten : 3229931768125 | 20260505 18:51:07.695 | INFO | NumEvents : 1529450174012 | 20260505 18:51:09.428 | INFO | Analyzing chunk frames = [5496832,5497344) | 20260505 18:51:09.428 | INFO | MemInfo: { | 20260505 18:51:09.428 | INFO | "gpu" : " 17 GB", | 20260505 18:51:09.428 | INFO | "huge" : "", | 20260505 18:51:09.428 | INFO | "huge_shared" : " 27 GB", | 20260505 18:51:09.428 | INFO | "malloc" : " 14 GB", | 20260505 18:51:09.428 | INFO | "pinned" : " 29 GB" | 20260505 18:51:09.428 | INFO | } | 20260505 18:51:09.428 | INFO | Perf report for the last 5.14059 seconds: | 20260505 18:51:09.428 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:51:09.428 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:09.428 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:09.428 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:09.428 | INFO | ANALYSIS : 63.4%, 3.0, 98.8ms, 0 | 20260505 18:51:09.428 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:09.428 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260505 18:51:09.428 | INFO | PRE_HQ : 60.4%, 1.8, 55.9ms, 0 | 20260505 18:51:09.428 | INFO | BAZWRITER : 62.9%, 3.8, 123.3ms, 0 | 20260505 18:51:09.428 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 18:51:09.429 | INFO | Uploaded 36GiB to GPU in 2.17274s (16.569GiB/s) | 20260505 18:51:09.429 | INFO | Downloaded 38GiB to GPU in 2.24916s (16.8952GiB/s) | 20260505 18:51:14.568 | INFO | Analyzing chunk frames = [5497344,5497856) | 20260505 18:51:14.568 | INFO | MemInfo: { | 20260505 18:51:14.568 | INFO | "gpu" : " 17 GB", | 20260505 18:51:14.568 | INFO | "huge" : "", | 20260505 18:51:14.568 | INFO | "huge_shared" : " 27 GB", | 20260505 18:51:14.568 | INFO | "malloc" : " 14 GB", | 20260505 18:51:14.568 | INFO | "pinned" : " 29 GB" | 20260505 18:51:14.569 | INFO | } | 20260505 18:51:14.569 | INFO | Perf report for the last 5.14028 seconds: | 20260505 18:51:14.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:51:14.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:14.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:14.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:14.569 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 18:51:14.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:14.569 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:14.569 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 18:51:14.569 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:14.569 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:14.569 | INFO | Uploaded 36GiB to GPU in 2.15678s (16.6916GiB/s) | 20260505 18:51:14.569 | INFO | Downloaded 34GiB to GPU in 2.04668s (16.6123GiB/s) | 20260505 18:51:19.713 | INFO | Analyzing chunk frames = [5497856,5498368) | 20260505 18:51:19.713 | INFO | MemInfo: { | 20260505 18:51:19.713 | INFO | "gpu" : " 17 GB", | 20260505 18:51:19.713 | INFO | "huge" : "", | 20260505 18:51:19.713 | INFO | "huge_shared" : " 27 GB", | 20260505 18:51:19.713 | INFO | "malloc" : " 14 GB", | 20260505 18:51:19.713 | INFO | "pinned" : " 29 GB" | 20260505 18:51:19.713 | INFO | } | 20260505 18:51:19.713 | INFO | Perf report for the last 5.14443 seconds: | 20260505 18:51:19.713 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:51:19.713 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:19.713 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:19.713 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:19.713 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260505 18:51:19.713 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:19.713 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:19.713 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:51:19.713 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:19.713 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:19.713 | INFO | Uploaded 36GiB to GPU in 2.14006s (16.822GiB/s) | 20260505 18:51:19.714 | INFO | Downloaded 34GiB to GPU in 2.03783s (16.6844GiB/s) | 20260505 18:51:24.844 | INFO | Analyzing chunk frames = [5498368,5498880) | 20260505 18:51:24.844 | INFO | MemInfo: { | 20260505 18:51:24.844 | INFO | "gpu" : " 17 GB", | 20260505 18:51:24.844 | INFO | "huge" : "", | 20260505 18:51:24.844 | INFO | "huge_shared" : " 27 GB", | 20260505 18:51:24.844 | INFO | "malloc" : " 14 GB", | 20260505 18:51:24.844 | INFO | "pinned" : " 29 GB" | 20260505 18:51:24.844 | INFO | } | 20260505 18:51:24.844 | INFO | Perf report for the last 5.13132 seconds: | 20260505 18:51:24.844 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:51:24.844 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:24.844 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:24.844 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:24.844 | INFO | ANALYSIS : 62.4%, 3.0, 96.7ms, 0 | 20260505 18:51:24.844 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:24.844 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:24.844 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:51:24.844 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:24.844 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:24.845 | INFO | Uploaded 36GiB to GPU in 2.19013s (16.4374GiB/s) | 20260505 18:51:24.845 | INFO | Downloaded 34GiB to GPU in 2.04938s (16.5904GiB/s) | 20260505 18:51:29.982 | INFO | Analyzing chunk frames = [5498880,5499392) | 20260505 18:51:29.982 | INFO | MemInfo: { | 20260505 18:51:29.982 | INFO | "gpu" : " 17 GB", | 20260505 18:51:29.982 | INFO | "huge" : "", | 20260505 18:51:29.982 | INFO | "huge_shared" : " 27 GB", | 20260505 18:51:29.982 | INFO | "malloc" : " 14 GB", | 20260505 18:51:29.982 | INFO | "pinned" : " 29 GB" | 20260505 18:51:29.982 | INFO | } | 20260505 18:51:29.982 | INFO | Perf report for the last 5.13791 seconds: | 20260505 18:51:29.982 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:51:29.982 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:29.982 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:29.982 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:29.982 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 18:51:29.982 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:29.982 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:29.982 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:51:29.982 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:29.982 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:29.983 | INFO | Uploaded 36GiB to GPU in 2.1565s (16.6937GiB/s) | 20260505 18:51:29.983 | INFO | Downloaded 34GiB to GPU in 2.06261s (16.484GiB/s) | 20260505 18:51:35.131 | INFO | Analyzing chunk frames = [5499392,5499904) | 20260505 18:51:35.131 | INFO | MemInfo: { | 20260505 18:51:35.131 | INFO | "gpu" : " 17 GB", | 20260505 18:51:35.131 | INFO | "huge" : "", | 20260505 18:51:35.131 | INFO | "huge_shared" : " 27 GB", | 20260505 18:51:35.131 | INFO | "malloc" : " 14 GB", | 20260505 18:51:35.131 | INFO | "pinned" : " 29 GB" | 20260505 18:51:35.131 | INFO | } | 20260505 18:51:35.132 | INFO | Perf report for the last 5.14927 seconds: | 20260505 18:51:35.132 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:51:35.132 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:35.132 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:35.132 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:35.132 | INFO | ANALYSIS : 62.2%, 3.0, 97.3ms, 0 | 20260505 18:51:35.132 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:35.132 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:35.132 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 18:51:35.132 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:35.132 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:35.132 | INFO | Uploaded 36GiB to GPU in 2.14876s (16.7539GiB/s) | 20260505 18:51:35.132 | INFO | Downloaded 34GiB to GPU in 2.04968s (16.5879GiB/s) | 20260505 18:51:40.273 | INFO | Analyzing chunk frames = [5499904,5500416) | 20260505 18:51:40.273 | INFO | MemInfo: { | 20260505 18:51:40.273 | INFO | "gpu" : " 17 GB", | 20260505 18:51:40.273 | INFO | "huge" : "", | 20260505 18:51:40.273 | INFO | "huge_shared" : " 27 GB", | 20260505 18:51:40.273 | INFO | "malloc" : " 14 GB", | 20260505 18:51:40.273 | INFO | "pinned" : " 29 GB" | 20260505 18:51:40.274 | INFO | } | 20260505 18:51:40.274 | INFO | Perf report for the last 5.14219 seconds: | 20260505 18:51:40.274 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:51:40.274 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:40.274 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:40.274 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:40.274 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 18:51:40.274 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:40.274 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:40.274 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 18:51:40.274 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:40.274 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:40.274 | INFO | Uploaded 36GiB to GPU in 2.14404s (16.7907GiB/s) | 20260505 18:51:40.274 | INFO | Downloaded 34GiB to GPU in 2.15496s (15.7775GiB/s) | 20260505 18:51:45.403 | INFO | Analyzing chunk frames = [5500416,5500928) | 20260505 18:51:45.403 | INFO | MemInfo: { | 20260505 18:51:45.403 | INFO | "gpu" : " 17 GB", | 20260505 18:51:45.404 | INFO | "huge" : "", | 20260505 18:51:45.404 | INFO | "huge_shared" : " 27 GB", | 20260505 18:51:45.404 | INFO | "malloc" : " 14 GB", | 20260505 18:51:45.404 | INFO | "pinned" : " 29 GB" | 20260505 18:51:45.404 | INFO | } | 20260505 18:51:45.404 | INFO | Perf report for the last 5.13001 seconds: | 20260505 18:51:45.404 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:51:45.404 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:45.404 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:45.404 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:45.404 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260505 18:51:45.404 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:45.404 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:45.404 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 18:51:45.404 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:45.404 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:45.404 | INFO | Uploaded 36GiB to GPU in 2.11906s (16.9887GiB/s) | 20260505 18:51:45.404 | INFO | Downloaded 34GiB to GPU in 2.00705s (16.9403GiB/s) | 20260505 18:51:45.538 | INFO | Starting a round of Baz IO | 20260505 18:51:48.715 | INFO | Populating RTMetrics report | 20260505 18:51:48.715 | INFO | RTMetrics report populated | 20260505 18:51:48.715 | INFO | Opening tmp RTMetrics json file | 20260505 18:51:48.723 | INFO | Serializing RTMetrics to json | 20260505 18:51:48.724 | INFO | Serialization done, swapping tmp file now... | 20260505 18:51:48.724 | INFO | RTMetrics done | 20260505 18:51:48.815 | INFO | Finished a round of Baz IO: Wrote 2.074 GiB in 3.276 seconds | 20260505 18:51:48.815 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.85% / 0.00% / 78.15% | 20260505 18:51:48.815 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.232172% / 3.97364e-06% / 0.000103315% | 20260505 18:51:48.815 | INFO | Baz IO overall: | 20260505 18:51:48.815 | INFO | BytesWritten : 3232158932431 | 20260505 18:51:48.815 | INFO | NumEvents : 1530486197020 | 20260505 18:51:50.535 | INFO | Analyzing chunk frames = [5500928,5501440) | 20260505 18:51:50.535 | INFO | MemInfo: { | 20260505 18:51:50.535 | INFO | "gpu" : " 17 GB", | 20260505 18:51:50.535 | INFO | "huge" : "", | 20260505 18:51:50.535 | INFO | "huge_shared" : " 27 GB", | 20260505 18:51:50.535 | INFO | "malloc" : " 14 GB", | 20260505 18:51:50.535 | INFO | "pinned" : " 29 GB" | 20260505 18:51:50.535 | INFO | } | 20260505 18:51:50.536 | INFO | Perf report for the last 5.13173 seconds: | 20260505 18:51:50.536 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:51:50.536 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:50.536 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:50.536 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:50.536 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260505 18:51:50.536 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:50.536 | INFO | RT_METRICS : 44.6%, 1.0, 23.3ms, 0 | 20260505 18:51:50.536 | INFO | PRE_HQ : 60.4%, 1.8, 55.9ms, 0 | 20260505 18:51:50.536 | INFO | BAZWRITER : 63.4%, 3.8, 122.7ms, 0 | 20260505 18:51:50.536 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 18:51:50.536 | INFO | Uploaded 36GiB to GPU in 2.16879s (16.5991GiB/s) | 20260505 18:51:50.536 | INFO | Downloaded 38GiB to GPU in 2.25211s (16.8731GiB/s) | 20260505 18:51:55.683 | INFO | Analyzing chunk frames = [5501440,5501952) | 20260505 18:51:55.683 | INFO | MemInfo: { | 20260505 18:51:55.683 | INFO | "gpu" : " 17 GB", | 20260505 18:51:55.683 | INFO | "huge" : "", | 20260505 18:51:55.683 | INFO | "huge_shared" : " 27 GB", | 20260505 18:51:55.683 | INFO | "malloc" : " 14 GB", | 20260505 18:51:55.683 | INFO | "pinned" : " 29 GB" | 20260505 18:51:55.683 | INFO | } | 20260505 18:51:55.683 | INFO | Perf report for the last 5.14746 seconds: | 20260505 18:51:55.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:51:55.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:55.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:55.683 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:55.683 | INFO | ANALYSIS : 62.7%, 3.0, 98.0ms, 0 | 20260505 18:51:55.683 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:55.683 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:55.683 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:51:55.683 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:55.683 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:51:55.683 | INFO | Uploaded 36GiB to GPU in 2.13053s (16.8972GiB/s) | 20260505 18:51:55.683 | INFO | Downloaded 34GiB to GPU in 2.02996s (16.7491GiB/s) | 20260505 18:52:00.827 | INFO | Analyzing chunk frames = [5501952,5502464) | 20260505 18:52:00.827 | INFO | MemInfo: { | 20260505 18:52:00.827 | INFO | "gpu" : " 17 GB", | 20260505 18:52:00.827 | INFO | "huge" : "", | 20260505 18:52:00.827 | INFO | "huge_shared" : " 27 GB", | 20260505 18:52:00.827 | INFO | "malloc" : " 14 GB", | 20260505 18:52:00.827 | INFO | "pinned" : " 29 GB" | 20260505 18:52:00.827 | INFO | } | 20260505 18:52:00.827 | INFO | Perf report for the last 5.144 seconds: | 20260505 18:52:00.827 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:52:00.827 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:00.827 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:00.827 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:00.827 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 18:52:00.827 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:00.827 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:00.827 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 18:52:00.827 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:00.827 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:00.827 | INFO | Uploaded 36GiB to GPU in 2.20888s (16.2979GiB/s) | 20260505 18:52:00.827 | INFO | Downloaded 34GiB to GPU in 2.0733s (16.399GiB/s) | 20260505 18:52:05.968 | INFO | Analyzing chunk frames = [5502464,5502976) | 20260505 18:52:05.968 | INFO | MemInfo: { | 20260505 18:52:05.968 | INFO | "gpu" : " 17 GB", | 20260505 18:52:05.968 | INFO | "huge" : "", | 20260505 18:52:05.968 | INFO | "huge_shared" : " 27 GB", | 20260505 18:52:05.968 | INFO | "malloc" : " 14 GB", | 20260505 18:52:05.968 | INFO | "pinned" : " 29 GB" | 20260505 18:52:05.968 | INFO | } | 20260505 18:52:05.968 | INFO | Perf report for the last 5.14136 seconds: | 20260505 18:52:05.968 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:52:05.968 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:05.968 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:05.968 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:05.968 | INFO | ANALYSIS : 62.4%, 3.0, 97.0ms, 0 | 20260505 18:52:05.968 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:05.968 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:05.968 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 18:52:05.969 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:05.969 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:05.969 | INFO | Uploaded 36GiB to GPU in 2.208s (16.3043GiB/s) | 20260505 18:52:05.969 | INFO | Downloaded 34GiB to GPU in 2.04284s (16.6435GiB/s) | 20260505 18:52:11.102 | INFO | Analyzing chunk frames = [5502976,5503488) | 20260505 18:52:11.102 | INFO | MemInfo: { | 20260505 18:52:11.102 | INFO | "gpu" : " 17 GB", | 20260505 18:52:11.102 | INFO | "huge" : "", | 20260505 18:52:11.102 | INFO | "huge_shared" : " 27 GB", | 20260505 18:52:11.102 | INFO | "malloc" : " 14 GB", | 20260505 18:52:11.102 | INFO | "pinned" : " 29 GB" | 20260505 18:52:11.102 | INFO | } | 20260505 18:52:11.102 | INFO | Perf report for the last 5.13356 seconds: | 20260505 18:52:11.102 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:52:11.102 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:11.102 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:11.102 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:11.102 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260505 18:52:11.102 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:11.102 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:11.102 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:52:11.102 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:11.102 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:11.102 | INFO | Uploaded 36GiB to GPU in 2.14164s (16.8096GiB/s) | 20260505 18:52:11.102 | INFO | Downloaded 34GiB to GPU in 2.06251s (16.4848GiB/s) | 20260505 18:52:16.236 | INFO | Analyzing chunk frames = [5503488,5504000) | 20260505 18:52:16.236 | INFO | MemInfo: { | 20260505 18:52:16.236 | INFO | "gpu" : " 17 GB", | 20260505 18:52:16.236 | INFO | "huge" : "", | 20260505 18:52:16.236 | INFO | "huge_shared" : " 27 GB", | 20260505 18:52:16.236 | INFO | "malloc" : " 14 GB", | 20260505 18:52:16.236 | INFO | "pinned" : " 29 GB" | 20260505 18:52:16.236 | INFO | } | 20260505 18:52:16.236 | INFO | Perf report for the last 5.13448 seconds: | 20260505 18:52:16.236 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:52:16.236 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:16.236 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:16.237 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:16.237 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 18:52:16.237 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:16.237 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:16.237 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:52:16.237 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:16.237 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:16.237 | INFO | Uploaded 36GiB to GPU in 2.13881s (16.8318GiB/s) | 20260505 18:52:16.237 | INFO | Downloaded 34GiB to GPU in 2.05073s (16.5795GiB/s) | 20260505 18:52:21.382 | INFO | Analyzing chunk frames = [5504000,5504512) | 20260505 18:52:21.382 | INFO | MemInfo: { | 20260505 18:52:21.382 | INFO | "gpu" : " 17 GB", | 20260505 18:52:21.382 | INFO | "huge" : "", | 20260505 18:52:21.382 | INFO | "huge_shared" : " 27 GB", | 20260505 18:52:21.382 | INFO | "malloc" : " 14 GB", | 20260505 18:52:21.382 | INFO | "pinned" : " 29 GB" | 20260505 18:52:21.382 | INFO | } | 20260505 18:52:21.382 | INFO | Perf report for the last 5.14548 seconds: | 20260505 18:52:21.382 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:52:21.382 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:21.382 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:21.382 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:21.382 | INFO | ANALYSIS : 63.2%, 3.0, 98.8ms, 0 | 20260505 18:52:21.382 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:21.382 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:21.382 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 18:52:21.382 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:21.382 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:21.382 | INFO | Uploaded 36GiB to GPU in 2.16069s (16.6614GiB/s) | 20260505 18:52:21.382 | INFO | Downloaded 34GiB to GPU in 2.15767s (15.7577GiB/s) | 20260505 18:52:26.521 | INFO | Analyzing chunk frames = [5504512,5505024) | 20260505 18:52:26.521 | INFO | MemInfo: { | 20260505 18:52:26.521 | INFO | "gpu" : " 17 GB", | 20260505 18:52:26.521 | INFO | "huge" : "", | 20260505 18:52:26.521 | INFO | "huge_shared" : " 27 GB", | 20260505 18:52:26.521 | INFO | "malloc" : " 14 GB", | 20260505 18:52:26.521 | INFO | "pinned" : " 29 GB" | 20260505 18:52:26.521 | INFO | } | 20260505 18:52:26.521 | INFO | Perf report for the last 5.13899 seconds: | 20260505 18:52:26.521 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:52:26.521 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:26.521 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:26.521 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:26.521 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 18:52:26.521 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:26.521 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:26.521 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 18:52:26.521 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:26.521 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:26.521 | INFO | Uploaded 36GiB to GPU in 2.19987s (16.3646GiB/s) | 20260505 18:52:26.521 | INFO | Downloaded 34GiB to GPU in 2.04643s (16.6143GiB/s) | 20260505 18:52:26.521 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 18:52:26.521 | INFO | Total mem usage: 15.3 GiB | 20260505 18:52:26.521 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 18:52:26.521 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.03%) | 20260505 18:52:26.521 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 18:52:26.521 | INFO | High Water Breakdown: | 20260505 18:52:26.521 | INFO | 61% : 8.66 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 18:52:26.521 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 18:52:26.521 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 18:52:26.522 | INFO | ------------------------------------------ | 20260505 18:52:26.522 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:52:26.522 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 18:52:26.522 | INFO | Total mem usage: 26.5 GiB | 20260505 18:52:26.522 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 18:52:26.522 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 18:52:26.522 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 18:52:26.522 | INFO | High Water Breakdown: | 20260505 18:52:26.522 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:52:26.522 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:52:26.522 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:52:26.522 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:52:26.522 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:52:26.522 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:52:26.522 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 18:52:26.522 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 18:52:26.522 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:52:26.522 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 18:52:26.522 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 18:52:26.522 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 18:52:26.522 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 18:52:26.522 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 18:52:26.522 | INFO | ------------------------------------------ | 20260505 18:52:26.522 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:52:26.522 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 18:52:26.522 | INFO | Total mem usage: 25.6 GiB | 20260505 18:52:26.522 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 18:52:26.522 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 18:52:26.522 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 18:52:26.522 | INFO | High Water Breakdown: | 20260505 18:52:26.522 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 18:52:26.522 | INFO | ------------------------------------------ | 20260505 18:52:26.522 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:52:26.522 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 18:52:26.522 | INFO | Total mem usage: 15.7 GiB | 20260505 18:52:26.522 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 18:52:26.522 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 18:52:26.522 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 18:52:26.522 | INFO | High Water Breakdown: | 20260505 18:52:26.522 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 18:52:26.522 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 18:52:26.522 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 18:52:26.522 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 18:52:26.522 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 18:52:26.522 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 18:52:26.522 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 18:52:26.522 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 18:52:26.522 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 18:52:26.522 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:52:26.522 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 18:52:26.522 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:52:26.522 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 18:52:26.522 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 18:52:26.522 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 18:52:26.522 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 18:52:26.522 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:52:26.522 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 18:52:26.522 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:52:26.522 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:52:26.522 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:52:26.522 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 18:52:26.522 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 18:52:26.522 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:52:26.522 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 18:52:26.522 | INFO | ------------------------------------------ | 20260505 18:52:26.522 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:52:26.522 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 18:52:26.522 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:52:26.522 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:52:26.522 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.44e+01, 1.33e+01 / 1.99e+00 | 20260505 18:52:26.522 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.03e-01, 1.05e-02 / 4.65e-03 | 20260505 18:52:26.522 | INFO | PerfReport: 11.78%, Baseline -- 3.71e+00 / 4.53e+00, 5.65e-01 / 1.67e-01 | 20260505 18:52:26.522 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.53e+00, 1.89e-01 / 9.06e-02 | 20260505 18:52:26.522 | INFO | PerfReport: 15.93%, DME -- 4.02e+01 / 4.43e+01, 4.15e+00 / 2.06e+00 | 20260505 18:52:26.522 | INFO | PerfReport: 56.97%, FrameLabeling -- 1.80e+01 / 1.92e+01, 1.48e+00 / 9.27e-01 | 20260505 18:52:26.523 | INFO | PerfReport: 7.29%, PulseAccumulating -- 2.30e+00 / 2.89e+00, 4.42e-01 / 1.93e-01 | 20260505 18:52:26.523 | INFO | PerfReport: 3.31%, Metrics -- 1.04e+00 / 1.38e+00, 2.92e-01 / 1.57e-01 | 20260505 18:52:26.523 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.64e-01 / 4.36e-01, 6.52e-02 / 4.42e-02 | 20260505 18:52:26.523 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 18:52:26.523 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:52:26.523 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:52:26.523 | INFO | PerfReport: Full Execution -- 9.97e+01 / -nan, 2.57e+01 / 0.00e+00 | 20260505 18:52:26.523 | INFO | PerfReport: 22.57%, Upload -- 2.25e+01 / 0.00e+00, 7.87e+00 / 0.00e+00 | 20260505 18:52:26.523 | INFO | PerfReport: 21.78%, Download -- 2.17e+01 / 0.00e+00, 7.75e+00 / 0.00e+00 | 20260505 18:52:26.523 | INFO | PerfReport: 31.62%, Compute -- 3.15e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260505 18:52:26.523 | INFO | PerfReport: 2.35%, LockUpload -- 2.35e+00 / 0.00e+00, 8.14e+00 / 0.00e+00 | 20260505 18:52:26.523 | INFO | PerfReport: 1.34%, LockDownload -- 1.33e+00 / 0.00e+00, 2.83e+00 / 0.00e+00 | 20260505 18:52:26.523 | INFO | PerfReport: 20.32%, LockCompute -- 2.03e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 18:52:26.523 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.83e-02 / 0.00e+00 | 20260505 18:52:26.656 | INFO | Starting a round of Baz IO | 20260505 18:52:29.827 | INFO | Populating RTMetrics report | 20260505 18:52:29.827 | INFO | RTMetrics report populated | 20260505 18:52:29.827 | INFO | Opening tmp RTMetrics json file | 20260505 18:52:29.833 | INFO | Serializing RTMetrics to json | 20260505 18:52:29.835 | INFO | Serialization done, swapping tmp file now... | 20260505 18:52:29.837 | INFO | RTMetrics done | 20260505 18:52:29.918 | INFO | Finished a round of Baz IO: Wrote 2.074 GiB in 3.262 seconds | 20260505 18:52:29.918 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.85% / 0.00% / 78.15% | 20260505 18:52:29.918 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.231548% / 3.97364e-06% / 7.94729e-05% | 20260505 18:52:29.918 | INFO | Baz IO overall: | 20260505 18:52:29.918 | INFO | BytesWritten : 3234386109103 | 20260505 18:52:29.918 | INFO | NumEvents : 1531522055083 | 20260505 18:52:31.659 | INFO | Analyzing chunk frames = [5505024,5505536) | 20260505 18:52:31.659 | INFO | MemInfo: { | 20260505 18:52:31.659 | INFO | "gpu" : " 17 GB", | 20260505 18:52:31.659 | INFO | "huge" : "", | 20260505 18:52:31.659 | INFO | "huge_shared" : " 27 GB", | 20260505 18:52:31.659 | INFO | "malloc" : " 14 GB", | 20260505 18:52:31.659 | INFO | "pinned" : " 29 GB" | 20260505 18:52:31.659 | INFO | } | 20260505 18:52:31.659 | INFO | Perf report for the last 5.1385 seconds: | 20260505 18:52:31.659 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:52:31.659 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:31.660 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:31.660 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:31.660 | INFO | ANALYSIS : 63.2%, 3.0, 98.4ms, 0 | 20260505 18:52:31.660 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:31.660 | INFO | RT_METRICS : 44.7%, 1.0, 23.4ms, 0 | 20260505 18:52:31.660 | INFO | PRE_HQ : 60.2%, 1.8, 56.0ms, 0 | 20260505 18:52:31.660 | INFO | BAZWRITER : 63.1%, 3.8, 123.1ms, 0 | 20260505 18:52:31.660 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 18:52:31.660 | INFO | Uploaded 36GiB to GPU in 2.15178s (16.7303GiB/s) | 20260505 18:52:31.660 | INFO | Downloaded 38GiB to GPU in 2.24324s (16.9398GiB/s) | 20260505 18:52:36.797 | INFO | Analyzing chunk frames = [5505536,5506048) | 20260505 18:52:36.797 | INFO | MemInfo: { | 20260505 18:52:36.797 | INFO | "gpu" : " 17 GB", | 20260505 18:52:36.797 | INFO | "huge" : "", | 20260505 18:52:36.797 | INFO | "huge_shared" : " 27 GB", | 20260505 18:52:36.797 | INFO | "malloc" : " 14 GB", | 20260505 18:52:36.797 | INFO | "pinned" : " 29 GB" | 20260505 18:52:36.797 | INFO | } | 20260505 18:52:36.797 | INFO | Perf report for the last 5.13727 seconds: | 20260505 18:52:36.797 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:52:36.797 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:36.797 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:36.797 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:36.797 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 18:52:36.797 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:36.797 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:36.797 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:52:36.797 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:36.797 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:36.797 | INFO | Uploaded 36GiB to GPU in 2.14192s (16.8074GiB/s) | 20260505 18:52:36.797 | INFO | Downloaded 34GiB to GPU in 2.04203s (16.6501GiB/s) | 20260505 18:52:41.944 | INFO | Analyzing chunk frames = [5506048,5506560) | 20260505 18:52:41.944 | INFO | MemInfo: { | 20260505 18:52:41.944 | INFO | "gpu" : " 17 GB", | 20260505 18:52:41.944 | INFO | "huge" : "", | 20260505 18:52:41.944 | INFO | "huge_shared" : " 27 GB", | 20260505 18:52:41.944 | INFO | "malloc" : " 14 GB", | 20260505 18:52:41.944 | INFO | "pinned" : " 29 GB" | 20260505 18:52:41.944 | INFO | } | 20260505 18:52:41.944 | INFO | Perf report for the last 5.14724 seconds: | 20260505 18:52:41.944 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:52:41.944 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:41.944 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:41.944 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:41.944 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260505 18:52:41.944 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:41.944 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:41.944 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:52:41.944 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:41.944 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:41.944 | INFO | Uploaded 36GiB to GPU in 2.1936s (16.4114GiB/s) | 20260505 18:52:41.944 | INFO | Downloaded 34GiB to GPU in 2.03333s (16.7214GiB/s) | 20260505 18:52:47.080 | INFO | Analyzing chunk frames = [5506560,5507072) | 20260505 18:52:47.080 | INFO | MemInfo: { | 20260505 18:52:47.080 | INFO | "gpu" : " 17 GB", | 20260505 18:52:47.080 | INFO | "huge" : "", | 20260505 18:52:47.080 | INFO | "huge_shared" : " 27 GB", | 20260505 18:52:47.080 | INFO | "malloc" : " 14 GB", | 20260505 18:52:47.080 | INFO | "pinned" : " 29 GB" | 20260505 18:52:47.080 | INFO | } | 20260505 18:52:47.080 | INFO | Perf report for the last 5.13587 seconds: | 20260505 18:52:47.080 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:52:47.080 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:47.080 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:47.080 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:47.080 | INFO | ANALYSIS : 62.4%, 3.0, 96.8ms, 0 | 20260505 18:52:47.080 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:47.080 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:47.080 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:52:47.080 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:47.080 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:47.080 | INFO | Uploaded 36GiB to GPU in 2.19817s (16.3772GiB/s) | 20260505 18:52:47.080 | INFO | Downloaded 34GiB to GPU in 2.03935s (16.672GiB/s) | 20260505 18:52:52.221 | INFO | Analyzing chunk frames = [5507072,5507584) | 20260505 18:52:52.221 | INFO | MemInfo: { | 20260505 18:52:52.221 | INFO | "gpu" : " 17 GB", | 20260505 18:52:52.221 | INFO | "huge" : "", | 20260505 18:52:52.221 | INFO | "huge_shared" : " 27 GB", | 20260505 18:52:52.221 | INFO | "malloc" : " 14 GB", | 20260505 18:52:52.221 | INFO | "pinned" : " 29 GB" | 20260505 18:52:52.221 | INFO | } | 20260505 18:52:52.221 | INFO | Perf report for the last 5.14089 seconds: | 20260505 18:52:52.221 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:52:52.221 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:52.221 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:52.221 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:52.221 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260505 18:52:52.221 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:52.221 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:52.221 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:52:52.221 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:52.221 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:52.221 | INFO | Uploaded 36GiB to GPU in 2.12711s (16.9244GiB/s) | 20260505 18:52:52.221 | INFO | Downloaded 34GiB to GPU in 2.06036s (16.502GiB/s) | 20260505 18:52:57.349 | INFO | Analyzing chunk frames = [5507584,5508096) | 20260505 18:52:57.349 | INFO | MemInfo: { | 20260505 18:52:57.349 | INFO | "gpu" : " 17 GB", | 20260505 18:52:57.349 | INFO | "huge" : "", | 20260505 18:52:57.349 | INFO | "huge_shared" : " 27 GB", | 20260505 18:52:57.349 | INFO | "malloc" : " 14 GB", | 20260505 18:52:57.349 | INFO | "pinned" : " 29 GB" | 20260505 18:52:57.349 | INFO | } | 20260505 18:52:57.350 | INFO | Perf report for the last 5.12884 seconds: | 20260505 18:52:57.350 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:52:57.350 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:57.350 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:57.350 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:57.350 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260505 18:52:57.350 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:57.350 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:57.350 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260505 18:52:57.350 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:57.350 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:52:57.350 | INFO | Uploaded 36GiB to GPU in 2.13472s (16.864GiB/s) | 20260505 18:52:57.350 | INFO | Downloaded 34GiB to GPU in 2.05086s (16.5784GiB/s) | 20260505 18:53:02.486 | INFO | Analyzing chunk frames = [5508096,5508608) | 20260505 18:53:02.486 | INFO | MemInfo: { | 20260505 18:53:02.486 | INFO | "gpu" : " 17 GB", | 20260505 18:53:02.486 | INFO | "huge" : "", | 20260505 18:53:02.486 | INFO | "huge_shared" : " 27 GB", | 20260505 18:53:02.486 | INFO | "malloc" : " 14 GB", | 20260505 18:53:02.486 | INFO | "pinned" : " 29 GB" | 20260505 18:53:02.486 | INFO | } | 20260505 18:53:02.486 | INFO | Perf report for the last 5.13649 seconds: | 20260505 18:53:02.486 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:53:02.486 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:02.486 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:02.486 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:02.486 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 18:53:02.486 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:02.486 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:02.486 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 18:53:02.486 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:02.486 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:02.486 | INFO | Uploaded 36GiB to GPU in 2.15217s (16.7273GiB/s) | 20260505 18:53:02.486 | INFO | Downloaded 34GiB to GPU in 2.15883s (15.7492GiB/s) | 20260505 18:53:07.637 | INFO | Analyzing chunk frames = [5508608,5509120) | 20260505 18:53:07.637 | INFO | MemInfo: { | 20260505 18:53:07.637 | INFO | "gpu" : " 17 GB", | 20260505 18:53:07.637 | INFO | "huge" : "", | 20260505 18:53:07.637 | INFO | "huge_shared" : " 27 GB", | 20260505 18:53:07.637 | INFO | "malloc" : " 14 GB", | 20260505 18:53:07.637 | INFO | "pinned" : " 29 GB" | 20260505 18:53:07.637 | INFO | } | 20260505 18:53:07.637 | INFO | Perf report for the last 5.15074 seconds: | 20260505 18:53:07.637 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:53:07.637 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:07.637 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:07.637 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:07.637 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 18:53:07.637 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:07.637 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:07.637 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 18:53:07.637 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:07.637 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:07.637 | INFO | Uploaded 36GiB to GPU in 2.19127s (16.4289GiB/s) | 20260505 18:53:07.637 | INFO | Downloaded 34GiB to GPU in 2.02465s (16.7931GiB/s) | 20260505 18:53:07.763 | INFO | Starting a round of Baz IO | 20260505 18:53:10.938 | INFO | Populating RTMetrics report | 20260505 18:53:10.939 | INFO | RTMetrics report populated | 20260505 18:53:10.939 | INFO | Opening tmp RTMetrics json file | 20260505 18:53:10.948 | INFO | Serializing RTMetrics to json | 20260505 18:53:10.950 | INFO | Serialization done, swapping tmp file now... | 20260505 18:53:10.952 | INFO | RTMetrics done | 20260505 18:53:11.045 | INFO | Finished a round of Baz IO: Wrote 2.074 GiB in 3.282 seconds | 20260505 18:53:11.045 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.85% / 0.00% / 78.15% | 20260505 18:53:11.045 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.23127% / 0% / 9.93411e-05% | 20260505 18:53:11.045 | INFO | Baz IO overall: | 20260505 18:53:11.045 | INFO | BytesWritten : 3236613334756 | 20260505 18:53:11.045 | INFO | NumEvents : 1532558414523 | 20260505 18:53:12.782 | INFO | Analyzing chunk frames = [5509120,5509632) | 20260505 18:53:12.782 | INFO | MemInfo: { | 20260505 18:53:12.782 | INFO | "gpu" : " 17 GB", | 20260505 18:53:12.782 | INFO | "huge" : "", | 20260505 18:53:12.782 | INFO | "huge_shared" : " 27 GB", | 20260505 18:53:12.782 | INFO | "malloc" : " 14 GB", | 20260505 18:53:12.782 | INFO | "pinned" : " 29 GB" | 20260505 18:53:12.782 | INFO | } | 20260505 18:53:12.783 | INFO | Perf report for the last 5.14567 seconds: | 20260505 18:53:12.783 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:53:12.783 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:12.783 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:12.783 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:12.783 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 18:53:12.783 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:12.783 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260505 18:53:12.783 | INFO | PRE_HQ : 60.2%, 1.8, 55.8ms, 0 | 20260505 18:53:12.783 | INFO | BAZWRITER : 63.3%, 3.8, 123.5ms, 0 | 20260505 18:53:12.783 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 18:53:12.783 | INFO | Uploaded 36GiB to GPU in 2.15521s (16.7037GiB/s) | 20260505 18:53:12.783 | INFO | Downloaded 38GiB to GPU in 2.23868s (16.9743GiB/s) | 20260505 18:53:17.915 | INFO | Analyzing chunk frames = [5509632,5510144) | 20260505 18:53:17.915 | INFO | MemInfo: { | 20260505 18:53:17.915 | INFO | "gpu" : " 17 GB", | 20260505 18:53:17.915 | INFO | "huge" : "", | 20260505 18:53:17.915 | INFO | "huge_shared" : " 27 GB", | 20260505 18:53:17.915 | INFO | "malloc" : " 14 GB", | 20260505 18:53:17.916 | INFO | "pinned" : " 29 GB" | 20260505 18:53:17.916 | INFO | } | 20260505 18:53:17.916 | INFO | Perf report for the last 5.1331 seconds: | 20260505 18:53:17.916 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:53:17.916 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:17.916 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:17.916 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:17.916 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260505 18:53:17.916 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:17.916 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:17.916 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:53:17.916 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:17.916 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:17.916 | INFO | Uploaded 36GiB to GPU in 2.08815s (17.2402GiB/s) | 20260505 18:53:17.916 | INFO | Downloaded 34GiB to GPU in 2.02602s (16.7817GiB/s) | 20260505 18:53:23.049 | INFO | Analyzing chunk frames = [5510144,5510656) | 20260505 18:53:23.049 | INFO | MemInfo: { | 20260505 18:53:23.049 | INFO | "gpu" : " 17 GB", | 20260505 18:53:23.049 | INFO | "huge" : "", | 20260505 18:53:23.049 | INFO | "huge_shared" : " 27 GB", | 20260505 18:53:23.049 | INFO | "malloc" : " 14 GB", | 20260505 18:53:23.049 | INFO | "pinned" : " 29 GB" | 20260505 18:53:23.050 | INFO | } | 20260505 18:53:23.050 | INFO | Perf report for the last 5.13429 seconds: | 20260505 18:53:23.050 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:53:23.050 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:23.050 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:23.050 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:23.050 | INFO | ANALYSIS : 62.4%, 3.0, 97.0ms, 0 | 20260505 18:53:23.050 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:23.050 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:23.050 | INFO | PRE_HQ : 27.9%, 1.0, 14.4ms, 0 | 20260505 18:53:23.050 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:23.050 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:23.050 | INFO | Uploaded 36GiB to GPU in 2.144s (16.791GiB/s) | 20260505 18:53:23.050 | INFO | Downloaded 34GiB to GPU in 2.04605s (16.6174GiB/s) | 20260505 18:53:28.190 | INFO | Analyzing chunk frames = [5510656,5511168) | 20260505 18:53:28.190 | INFO | MemInfo: { | 20260505 18:53:28.190 | INFO | "gpu" : " 17 GB", | 20260505 18:53:28.190 | INFO | "huge" : "", | 20260505 18:53:28.190 | INFO | "huge_shared" : " 27 GB", | 20260505 18:53:28.191 | INFO | "malloc" : " 14 GB", | 20260505 18:53:28.191 | INFO | "pinned" : " 29 GB" | 20260505 18:53:28.191 | INFO | } | 20260505 18:53:28.191 | INFO | Perf report for the last 5.14073 seconds: | 20260505 18:53:28.191 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:53:28.191 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:28.191 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:28.191 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:28.191 | INFO | ANALYSIS : 62.5%, 3.0, 97.1ms, 0 | 20260505 18:53:28.191 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:28.191 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:28.191 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260505 18:53:28.191 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:28.191 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:28.191 | INFO | Uploaded 36GiB to GPU in 2.20456s (16.3298GiB/s) | 20260505 18:53:28.191 | INFO | Downloaded 34GiB to GPU in 2.13717s (15.9089GiB/s) | 20260505 18:53:33.330 | INFO | Analyzing chunk frames = [5511168,5511680) | 20260505 18:53:33.330 | INFO | MemInfo: { | 20260505 18:53:33.330 | INFO | "gpu" : " 17 GB", | 20260505 18:53:33.330 | INFO | "huge" : "", | 20260505 18:53:33.330 | INFO | "huge_shared" : " 27 GB", | 20260505 18:53:33.330 | INFO | "malloc" : " 14 GB", | 20260505 18:53:33.330 | INFO | "pinned" : " 29 GB" | 20260505 18:53:33.330 | INFO | } | 20260505 18:53:33.330 | INFO | Perf report for the last 5.13976 seconds: | 20260505 18:53:33.330 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:53:33.330 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:33.330 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:33.331 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:33.331 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260505 18:53:33.331 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:33.331 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:33.331 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:53:33.331 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:33.331 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:33.331 | INFO | Uploaded 36GiB to GPU in 2.1328s (16.8792GiB/s) | 20260505 18:53:33.331 | INFO | Downloaded 34GiB to GPU in 2.06469s (16.4674GiB/s) | 20260505 18:53:38.470 | INFO | Analyzing chunk frames = [5511680,5512192) | 20260505 18:53:38.470 | INFO | MemInfo: { | 20260505 18:53:38.470 | INFO | "gpu" : " 17 GB", | 20260505 18:53:38.470 | INFO | "huge" : "", | 20260505 18:53:38.470 | INFO | "huge_shared" : " 27 GB", | 20260505 18:53:38.470 | INFO | "malloc" : " 14 GB", | 20260505 18:53:38.470 | INFO | "pinned" : " 29 GB" | 20260505 18:53:38.470 | INFO | } | 20260505 18:53:38.471 | INFO | Perf report for the last 5.14001 seconds: | 20260505 18:53:38.471 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:53:38.471 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:38.471 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:38.471 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:38.471 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 18:53:38.471 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:38.471 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:38.471 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 18:53:38.471 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:38.471 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:38.471 | INFO | Uploaded 36GiB to GPU in 2.13302s (16.8775GiB/s) | 20260505 18:53:38.471 | INFO | Downloaded 34GiB to GPU in 2.05369s (16.5556GiB/s) | 20260505 18:53:43.608 | INFO | Analyzing chunk frames = [5512192,5512704) | 20260505 18:53:43.608 | INFO | MemInfo: { | 20260505 18:53:43.608 | INFO | "gpu" : " 17 GB", | 20260505 18:53:43.608 | INFO | "huge" : "", | 20260505 18:53:43.608 | INFO | "huge_shared" : " 27 GB", | 20260505 18:53:43.608 | INFO | "malloc" : " 14 GB", | 20260505 18:53:43.608 | INFO | "pinned" : " 29 GB" | 20260505 18:53:43.608 | INFO | } | 20260505 18:53:43.608 | INFO | Perf report for the last 5.13737 seconds: | 20260505 18:53:43.608 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:53:43.608 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:43.608 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:43.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:43.608 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 18:53:43.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:43.608 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:43.608 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 18:53:43.608 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:43.608 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:43.608 | INFO | Uploaded 36GiB to GPU in 2.13014s (16.9003GiB/s) | 20260505 18:53:43.608 | INFO | Downloaded 34GiB to GPU in 2.1369s (15.9109GiB/s) | 20260505 18:53:48.743 | INFO | Analyzing chunk frames = [5512704,5513216) | 20260505 18:53:48.743 | INFO | MemInfo: { | 20260505 18:53:48.743 | INFO | "gpu" : " 17 GB", | 20260505 18:53:48.743 | INFO | "huge" : "", | 20260505 18:53:48.743 | INFO | "huge_shared" : " 27 GB", | 20260505 18:53:48.743 | INFO | "malloc" : " 14 GB", | 20260505 18:53:48.743 | INFO | "pinned" : " 29 GB" | 20260505 18:53:48.744 | INFO | } | 20260505 18:53:48.744 | INFO | Perf report for the last 5.13566 seconds: | 20260505 18:53:48.744 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:53:48.744 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:48.744 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:48.744 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:48.744 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260505 18:53:48.744 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:48.744 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:48.744 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 18:53:48.744 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:48.744 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:48.744 | INFO | Uploaded 36GiB to GPU in 2.19774s (16.3804GiB/s) | 20260505 18:53:48.744 | INFO | Downloaded 34GiB to GPU in 2.02745s (16.7698GiB/s) | 20260505 18:53:48.744 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 18:53:48.744 | INFO | Total mem usage: 15.3 GiB | 20260505 18:53:48.744 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 18:53:48.744 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.06%) | 20260505 18:53:48.744 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 18:53:48.744 | INFO | High Water Breakdown: | 20260505 18:53:48.744 | INFO | 61% : 8.66 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 18:53:48.744 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 18:53:48.744 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 18:53:48.744 | INFO | ------------------------------------------ | 20260505 18:53:48.744 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:53:48.744 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 18:53:48.744 | INFO | Total mem usage: 26.5 GiB | 20260505 18:53:48.744 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 18:53:48.744 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 18:53:48.744 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 18:53:48.744 | INFO | High Water Breakdown: | 20260505 18:53:48.744 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:53:48.744 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:53:48.744 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:53:48.744 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:53:48.744 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:53:48.744 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:53:48.744 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 18:53:48.744 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 18:53:48.744 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:53:48.744 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 18:53:48.744 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 18:53:48.745 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 18:53:48.745 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 18:53:48.745 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 18:53:48.745 | INFO | ------------------------------------------ | 20260505 18:53:48.745 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:53:48.745 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 18:53:48.745 | INFO | Total mem usage: 25.6 GiB | 20260505 18:53:48.745 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 18:53:48.745 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 18:53:48.745 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 18:53:48.745 | INFO | High Water Breakdown: | 20260505 18:53:48.745 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 18:53:48.745 | INFO | ------------------------------------------ | 20260505 18:53:48.745 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:53:48.745 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 18:53:48.745 | INFO | Total mem usage: 15.7 GiB | 20260505 18:53:48.745 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 18:53:48.745 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 18:53:48.745 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 18:53:48.745 | INFO | High Water Breakdown: | 20260505 18:53:48.745 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 18:53:48.745 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 18:53:48.745 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 18:53:48.745 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 18:53:48.745 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 18:53:48.745 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 18:53:48.745 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 18:53:48.745 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 18:53:48.745 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 18:53:48.745 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:53:48.745 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 18:53:48.745 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:53:48.745 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 18:53:48.745 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 18:53:48.745 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 18:53:48.745 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 18:53:48.745 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:53:48.745 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 18:53:48.745 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:53:48.745 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:53:48.745 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:53:48.745 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 18:53:48.745 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 18:53:48.745 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:53:48.745 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 18:53:48.745 | INFO | ------------------------------------------ | 20260505 18:53:48.745 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:53:48.745 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 18:53:48.745 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:53:48.745 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:53:48.745 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.48e+01, 1.35e+01 / 1.92e+00 | 20260505 18:53:48.745 | INFO | PerfReport: 0.29%, AggregateStats -- 9.27e-02 / 1.04e-01, 1.09e-02 / 6.51e-03 | 20260505 18:53:48.745 | INFO | PerfReport: 11.82%, Baseline -- 3.73e+00 / 4.55e+00, 5.65e-01 / 1.34e-01 | 20260505 18:53:48.745 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.54e+00, 1.89e-01 / 8.54e-02 | 20260505 18:53:48.745 | INFO | PerfReport: 15.95%, DME -- 4.02e+01 / 4.46e+01, 3.98e+00 / 1.76e+00 | 20260505 18:53:48.745 | INFO | PerfReport: 56.88%, FrameLabeling -- 1.79e+01 / 1.93e+01, 1.42e+00 / 1.09e+00 | 20260505 18:53:48.745 | INFO | PerfReport: 7.31%, PulseAccumulating -- 2.30e+00 / 2.88e+00, 4.39e-01 / 1.62e-01 | 20260505 18:53:48.745 | INFO | PerfReport: 3.32%, Metrics -- 1.05e+00 / 1.38e+00, 2.91e-01 / 1.46e-01 | 20260505 18:53:48.745 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.63e-01 / 4.28e-01, 6.45e-02 / 3.98e-02 | 20260505 18:53:48.745 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 18:53:48.745 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:53:48.745 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:53:48.745 | INFO | PerfReport: Full Execution -- 9.96e+01 / -nan, 2.52e+01 / 0.00e+00 | 20260505 18:53:48.745 | INFO | PerfReport: 22.53%, Upload -- 2.24e+01 / 0.00e+00, 7.77e+00 / 0.00e+00 | 20260505 18:53:48.745 | INFO | PerfReport: 21.81%, Download -- 2.17e+01 / 0.00e+00, 7.63e+00 / 0.00e+00 | 20260505 18:53:48.745 | INFO | PerfReport: 31.65%, Compute -- 3.15e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 18:53:48.745 | INFO | PerfReport: 2.25%, LockUpload -- 2.25e+00 / 0.00e+00, 7.87e+00 / 0.00e+00 | 20260505 18:53:48.745 | INFO | PerfReport: 1.41%, LockDownload -- 1.41e+00 / 0.00e+00, 2.90e+00 / 0.00e+00 | 20260505 18:53:48.745 | INFO | PerfReport: 20.33%, LockCompute -- 2.03e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 18:53:48.745 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.49e-02 / 0.00e+00 | 20260505 18:53:48.881 | INFO | Starting a round of Baz IO | 20260505 18:53:52.058 | INFO | Populating RTMetrics report | 20260505 18:53:52.058 | INFO | RTMetrics report populated | 20260505 18:53:52.058 | INFO | Opening tmp RTMetrics json file | 20260505 18:53:52.060 | INFO | Serializing RTMetrics to json | 20260505 18:53:52.063 | INFO | Serialization done, swapping tmp file now... | 20260505 18:53:52.065 | INFO | RTMetrics done | 20260505 18:53:52.153 | INFO | Finished a round of Baz IO: Wrote 2.074 GiB in 3.272 seconds | 20260505 18:53:52.153 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.85% / 0.00% / 78.15% | 20260505 18:53:52.154 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.231449% / 0% / 9.13938e-05% | 20260505 18:53:52.154 | INFO | Baz IO overall: | 20260505 18:53:52.154 | INFO | BytesWritten : 3238840233237 | 20260505 18:53:52.154 | INFO | NumEvents : 1533594888328 | 20260505 18:53:53.896 | INFO | Analyzing chunk frames = [5513216,5513728) | 20260505 18:53:53.896 | INFO | MemInfo: { | 20260505 18:53:53.896 | INFO | "gpu" : " 17 GB", | 20260505 18:53:53.896 | INFO | "huge" : "", | 20260505 18:53:53.896 | INFO | "huge_shared" : " 27 GB", | 20260505 18:53:53.896 | INFO | "malloc" : " 14 GB", | 20260505 18:53:53.896 | INFO | "pinned" : " 29 GB" | 20260505 18:53:53.896 | INFO | } | 20260505 18:53:53.896 | INFO | Perf report for the last 5.15265 seconds: | 20260505 18:53:53.896 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:53:53.896 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:53.896 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:53.896 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:53.896 | INFO | ANALYSIS : 63.2%, 3.0, 98.8ms, 0 | 20260505 18:53:53.896 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:53.896 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260505 18:53:53.896 | INFO | PRE_HQ : 60.0%, 1.8, 55.6ms, 0 | 20260505 18:53:53.896 | INFO | BAZWRITER : 63.0%, 3.8, 123.2ms, 0 | 20260505 18:53:53.896 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 18:53:53.897 | INFO | Uploaded 36GiB to GPU in 2.14186s (16.8078GiB/s) | 20260505 18:53:53.897 | INFO | Downloaded 38GiB to GPU in 2.24679s (16.913GiB/s) | 20260505 18:53:59.033 | INFO | Analyzing chunk frames = [5513728,5514240) | 20260505 18:53:59.033 | INFO | MemInfo: { | 20260505 18:53:59.033 | INFO | "gpu" : " 17 GB", | 20260505 18:53:59.033 | INFO | "huge" : "", | 20260505 18:53:59.033 | INFO | "huge_shared" : " 27 GB", | 20260505 18:53:59.033 | INFO | "malloc" : " 14 GB", | 20260505 18:53:59.033 | INFO | "pinned" : " 29 GB" | 20260505 18:53:59.033 | INFO | } | 20260505 18:53:59.033 | INFO | Perf report for the last 5.13695 seconds: | 20260505 18:53:59.033 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:53:59.033 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:59.033 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:59.033 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:59.033 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260505 18:53:59.033 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:59.033 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:59.033 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:53:59.033 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:59.033 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:53:59.034 | INFO | Uploaded 36GiB to GPU in 2.10299s (17.1185GiB/s) | 20260505 18:53:59.034 | INFO | Downloaded 34GiB to GPU in 2.02295s (16.8071GiB/s) | 20260505 18:54:04.161 | INFO | Analyzing chunk frames = [5514240,5514752) | 20260505 18:54:04.161 | INFO | MemInfo: { | 20260505 18:54:04.161 | INFO | "gpu" : " 17 GB", | 20260505 18:54:04.161 | INFO | "huge" : "", | 20260505 18:54:04.161 | INFO | "huge_shared" : " 27 GB", | 20260505 18:54:04.161 | INFO | "malloc" : " 14 GB", | 20260505 18:54:04.161 | INFO | "pinned" : " 29 GB" | 20260505 18:54:04.161 | INFO | } | 20260505 18:54:04.161 | INFO | Perf report for the last 5.12824 seconds: | 20260505 18:54:04.161 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:54:04.161 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:04.161 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:04.161 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:04.162 | INFO | ANALYSIS : 62.4%, 3.0, 96.9ms, 0 | 20260505 18:54:04.162 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:04.162 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:04.162 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:54:04.162 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:04.162 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:04.162 | INFO | Uploaded 36GiB to GPU in 2.1409s (16.8154GiB/s) | 20260505 18:54:04.162 | INFO | Downloaded 34GiB to GPU in 2.03398s (16.716GiB/s) | 20260505 18:54:09.301 | INFO | Analyzing chunk frames = [5514752,5515264) | 20260505 18:54:09.301 | INFO | MemInfo: { | 20260505 18:54:09.301 | INFO | "gpu" : " 17 GB", | 20260505 18:54:09.301 | INFO | "huge" : "", | 20260505 18:54:09.301 | INFO | "huge_shared" : " 27 GB", | 20260505 18:54:09.301 | INFO | "malloc" : " 14 GB", | 20260505 18:54:09.301 | INFO | "pinned" : " 29 GB" | 20260505 18:54:09.301 | INFO | } | 20260505 18:54:09.302 | INFO | Perf report for the last 5.14006 seconds: | 20260505 18:54:09.302 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:54:09.302 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:09.302 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:09.302 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:09.302 | INFO | ANALYSIS : 62.5%, 3.0, 97.1ms, 0 | 20260505 18:54:09.302 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:09.302 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:09.302 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:54:09.302 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:09.302 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:09.302 | INFO | Uploaded 36GiB to GPU in 2.19909s (16.3704GiB/s) | 20260505 18:54:09.302 | INFO | Downloaded 34GiB to GPU in 2.14104s (15.8802GiB/s) | 20260505 18:54:14.444 | INFO | Analyzing chunk frames = [5515264,5515776) | 20260505 18:54:14.444 | INFO | MemInfo: { | 20260505 18:54:14.444 | INFO | "gpu" : " 17 GB", | 20260505 18:54:14.444 | INFO | "huge" : "", | 20260505 18:54:14.444 | INFO | "huge_shared" : " 27 GB", | 20260505 18:54:14.444 | INFO | "malloc" : " 14 GB", | 20260505 18:54:14.444 | INFO | "pinned" : " 29 GB" | 20260505 18:54:14.444 | INFO | } | 20260505 18:54:14.444 | INFO | Perf report for the last 5.14222 seconds: | 20260505 18:54:14.444 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:54:14.444 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:14.444 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:14.444 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:14.444 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260505 18:54:14.444 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:14.444 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:14.444 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:54:14.444 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:14.444 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:14.444 | INFO | Uploaded 36GiB to GPU in 2.14298s (16.799GiB/s) | 20260505 18:54:14.444 | INFO | Downloaded 34GiB to GPU in 2.06305s (16.4804GiB/s) | 20260505 18:54:19.586 | INFO | Analyzing chunk frames = [5515776,5516288) | 20260505 18:54:19.586 | INFO | MemInfo: { | 20260505 18:54:19.586 | INFO | "gpu" : " 17 GB", | 20260505 18:54:19.586 | INFO | "huge" : "", | 20260505 18:54:19.586 | INFO | "huge_shared" : " 27 GB", | 20260505 18:54:19.587 | INFO | "malloc" : " 14 GB", | 20260505 18:54:19.587 | INFO | "pinned" : " 29 GB" | 20260505 18:54:19.587 | INFO | } | 20260505 18:54:19.587 | INFO | Perf report for the last 5.14285 seconds: | 20260505 18:54:19.587 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:54:19.587 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:19.587 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:19.587 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:19.587 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260505 18:54:19.587 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:19.587 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:19.587 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 18:54:19.587 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:19.587 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:19.587 | INFO | Uploaded 36GiB to GPU in 2.1351s (16.861GiB/s) | 20260505 18:54:19.587 | INFO | Downloaded 34GiB to GPU in 2.05232s (16.5666GiB/s) | 20260505 18:54:24.725 | INFO | Analyzing chunk frames = [5516288,5516800) | 20260505 18:54:24.725 | INFO | MemInfo: { | 20260505 18:54:24.725 | INFO | "gpu" : " 17 GB", | 20260505 18:54:24.725 | INFO | "huge" : "", | 20260505 18:54:24.725 | INFO | "huge_shared" : " 27 GB", | 20260505 18:54:24.725 | INFO | "malloc" : " 14 GB", | 20260505 18:54:24.725 | INFO | "pinned" : " 29 GB" | 20260505 18:54:24.726 | INFO | } | 20260505 18:54:24.726 | INFO | Perf report for the last 5.13904 seconds: | 20260505 18:54:24.726 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:54:24.726 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:24.726 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:24.726 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:24.726 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 18:54:24.726 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:24.726 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:24.726 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 18:54:24.726 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:24.726 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:24.726 | INFO | Uploaded 36GiB to GPU in 2.12998s (16.9016GiB/s) | 20260505 18:54:24.726 | INFO | Downloaded 34GiB to GPU in 2.1491s (15.8206GiB/s) | 20260505 18:54:29.865 | INFO | Analyzing chunk frames = [5516800,5517312) | 20260505 18:54:29.865 | INFO | MemInfo: { | 20260505 18:54:29.865 | INFO | "gpu" : " 17 GB", | 20260505 18:54:29.865 | INFO | "huge" : "", | 20260505 18:54:29.865 | INFO | "huge_shared" : " 27 GB", | 20260505 18:54:29.865 | INFO | "malloc" : " 14 GB", | 20260505 18:54:29.866 | INFO | "pinned" : " 29 GB" | 20260505 18:54:29.866 | INFO | } | 20260505 18:54:29.866 | INFO | Perf report for the last 5.13988 seconds: | 20260505 18:54:29.866 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:54:29.866 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:29.866 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:29.866 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:29.866 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 18:54:29.866 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:29.866 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:29.866 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:54:29.866 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:29.866 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:29.866 | INFO | Uploaded 36GiB to GPU in 2.19952s (16.3672GiB/s) | 20260505 18:54:29.866 | INFO | Downloaded 34GiB to GPU in 2.04434s (16.6313GiB/s) | 20260505 18:54:30.000 | INFO | Starting a round of Baz IO | 20260505 18:54:33.179 | INFO | Populating RTMetrics report | 20260505 18:54:33.180 | INFO | RTMetrics report populated | 20260505 18:54:33.180 | INFO | Opening tmp RTMetrics json file | 20260505 18:54:33.182 | INFO | Serializing RTMetrics to json | 20260505 18:54:33.184 | INFO | Serialization done, swapping tmp file now... | 20260505 18:54:33.184 | INFO | RTMetrics done | 20260505 18:54:33.272 | INFO | Finished a round of Baz IO: Wrote 2.072 GiB in 3.272 seconds | 20260505 18:54:33.272 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.85% / 0.00% / 78.15% | 20260505 18:54:33.272 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.230646% / 3.97364e-06% / 9.53674e-05% | 20260505 18:54:33.272 | INFO | Baz IO overall: | 20260505 18:54:33.272 | INFO | BytesWritten : 3241064592675 | 20260505 18:54:33.272 | INFO | NumEvents : 1534629669901 | 20260505 18:54:35.003 | INFO | Analyzing chunk frames = [5517312,5517824) | 20260505 18:54:35.003 | INFO | MemInfo: { | 20260505 18:54:35.003 | INFO | "gpu" : " 17 GB", | 20260505 18:54:35.003 | INFO | "huge" : "", | 20260505 18:54:35.003 | INFO | "huge_shared" : " 27 GB", | 20260505 18:54:35.003 | INFO | "malloc" : " 14 GB", | 20260505 18:54:35.003 | INFO | "pinned" : " 29 GB" | 20260505 18:54:35.003 | INFO | } | 20260505 18:54:35.003 | INFO | Perf report for the last 5.1371 seconds: | 20260505 18:54:35.003 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:54:35.003 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:35.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:35.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:35.003 | INFO | ANALYSIS : 63.4%, 3.0, 98.8ms, 0 | 20260505 18:54:35.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:35.003 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260505 18:54:35.003 | INFO | PRE_HQ : 60.4%, 1.8, 55.7ms, 0 | 20260505 18:54:35.003 | INFO | BAZWRITER : 63.2%, 3.7, 121.3ms, 0 | 20260505 18:54:35.003 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 18:54:35.003 | INFO | Uploaded 36GiB to GPU in 2.17195s (16.575GiB/s) | 20260505 18:54:35.003 | INFO | Downloaded 38GiB to GPU in 2.25205s (16.8736GiB/s) | 20260505 18:54:40.144 | INFO | Analyzing chunk frames = [5517824,5518336) | 20260505 18:54:40.144 | INFO | MemInfo: { | 20260505 18:54:40.144 | INFO | "gpu" : " 17 GB", | 20260505 18:54:40.144 | INFO | "huge" : "", | 20260505 18:54:40.144 | INFO | "huge_shared" : " 27 GB", | 20260505 18:54:40.144 | INFO | "malloc" : " 14 GB", | 20260505 18:54:40.144 | INFO | "pinned" : " 29 GB" | 20260505 18:54:40.144 | INFO | } | 20260505 18:54:40.144 | INFO | Perf report for the last 5.14135 seconds: | 20260505 18:54:40.144 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:54:40.144 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:40.144 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:40.144 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:40.144 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 18:54:40.144 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:40.144 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:40.144 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:54:40.144 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:40.144 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:40.144 | INFO | Uploaded 36GiB to GPU in 2.14491s (16.7839GiB/s) | 20260505 18:54:40.144 | INFO | Downloaded 34GiB to GPU in 2.03547s (16.7038GiB/s) | 20260505 18:54:45.283 | INFO | Analyzing chunk frames = [5518336,5518848) | 20260505 18:54:45.283 | INFO | MemInfo: { | 20260505 18:54:45.284 | INFO | "gpu" : " 17 GB", | 20260505 18:54:45.284 | INFO | "huge" : "", | 20260505 18:54:45.284 | INFO | "huge_shared" : " 27 GB", | 20260505 18:54:45.284 | INFO | "malloc" : " 14 GB", | 20260505 18:54:45.284 | INFO | "pinned" : " 29 GB" | 20260505 18:54:45.284 | INFO | } | 20260505 18:54:45.284 | INFO | Perf report for the last 5.13968 seconds: | 20260505 18:54:45.284 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:54:45.284 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:45.284 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:45.284 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:45.284 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260505 18:54:45.284 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:45.284 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:45.284 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 18:54:45.284 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:45.284 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:45.284 | INFO | Uploaded 36GiB to GPU in 2.14017s (16.8211GiB/s) | 20260505 18:54:45.284 | INFO | Downloaded 34GiB to GPU in 2.039s (16.6748GiB/s) | 20260505 18:54:50.417 | INFO | Analyzing chunk frames = [5518848,5519360) | 20260505 18:54:50.417 | INFO | MemInfo: { | 20260505 18:54:50.417 | INFO | "gpu" : " 17 GB", | 20260505 18:54:50.417 | INFO | "huge" : "", | 20260505 18:54:50.417 | INFO | "huge_shared" : " 27 GB", | 20260505 18:54:50.417 | INFO | "malloc" : " 14 GB", | 20260505 18:54:50.417 | INFO | "pinned" : " 29 GB" | 20260505 18:54:50.417 | INFO | } | 20260505 18:54:50.417 | INFO | Perf report for the last 5.13311 seconds: | 20260505 18:54:50.417 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:54:50.417 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:50.417 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:50.417 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:50.417 | INFO | ANALYSIS : 62.3%, 3.0, 96.7ms, 0 | 20260505 18:54:50.417 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:50.417 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:50.417 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:54:50.417 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:50.417 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:50.417 | INFO | Uploaded 36GiB to GPU in 2.18954s (16.4418GiB/s) | 20260505 18:54:50.417 | INFO | Downloaded 34GiB to GPU in 2.05255s (16.5648GiB/s) | 20260505 18:54:55.555 | INFO | Analyzing chunk frames = [5519360,5519872) | 20260505 18:54:55.555 | INFO | MemInfo: { | 20260505 18:54:55.555 | INFO | "gpu" : " 17 GB", | 20260505 18:54:55.555 | INFO | "huge" : "", | 20260505 18:54:55.555 | INFO | "huge_shared" : " 27 GB", | 20260505 18:54:55.555 | INFO | "malloc" : " 14 GB", | 20260505 18:54:55.555 | INFO | "pinned" : " 29 GB" | 20260505 18:54:55.555 | INFO | } | 20260505 18:54:55.556 | INFO | Perf report for the last 5.13871 seconds: | 20260505 18:54:55.556 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:54:55.556 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:55.556 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:55.556 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:55.556 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 18:54:55.556 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:55.556 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:55.556 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:54:55.556 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:55.556 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:54:55.556 | INFO | Uploaded 36GiB to GPU in 2.15821s (16.6805GiB/s) | 20260505 18:54:55.556 | INFO | Downloaded 34GiB to GPU in 2.06546s (16.4612GiB/s) | 20260505 18:55:00.702 | INFO | Analyzing chunk frames = [5519872,5520384) | 20260505 18:55:00.702 | INFO | MemInfo: { | 20260505 18:55:00.702 | INFO | "gpu" : " 17 GB", | 20260505 18:55:00.702 | INFO | "huge" : "", | 20260505 18:55:00.702 | INFO | "huge_shared" : " 27 GB", | 20260505 18:55:00.702 | INFO | "malloc" : " 14 GB", | 20260505 18:55:00.702 | INFO | "pinned" : " 29 GB" | 20260505 18:55:00.702 | INFO | } | 20260505 18:55:00.702 | INFO | Perf report for the last 5.14658 seconds: | 20260505 18:55:00.702 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:55:00.702 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:00.702 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:00.702 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:00.702 | INFO | ANALYSIS : 62.2%, 3.0, 97.3ms, 0 | 20260505 18:55:00.702 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:00.702 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:00.702 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 18:55:00.702 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:00.702 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:00.702 | INFO | Uploaded 36GiB to GPU in 2.1403s (16.8201GiB/s) | 20260505 18:55:00.703 | INFO | Downloaded 34GiB to GPU in 2.05662s (16.532GiB/s) | 20260505 18:55:05.846 | INFO | Analyzing chunk frames = [5520384,5520896) | 20260505 18:55:05.846 | INFO | MemInfo: { | 20260505 18:55:05.846 | INFO | "gpu" : " 17 GB", | 20260505 18:55:05.846 | INFO | "huge" : "", | 20260505 18:55:05.846 | INFO | "huge_shared" : " 27 GB", | 20260505 18:55:05.846 | INFO | "malloc" : " 14 GB", | 20260505 18:55:05.846 | INFO | "pinned" : " 29 GB" | 20260505 18:55:05.846 | INFO | } | 20260505 18:55:05.847 | INFO | Perf report for the last 5.1443 seconds: | 20260505 18:55:05.847 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:55:05.847 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:05.847 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:05.847 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:05.847 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 18:55:05.847 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:05.847 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:05.847 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 18:55:05.847 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:05.847 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:05.847 | INFO | Uploaded 36GiB to GPU in 2.12198s (16.9653GiB/s) | 20260505 18:55:05.847 | INFO | Downloaded 34GiB to GPU in 2.13874s (15.8972GiB/s) | 20260505 18:55:10.982 | INFO | Analyzing chunk frames = [5520896,5521408) | 20260505 18:55:10.982 | INFO | MemInfo: { | 20260505 18:55:10.982 | INFO | "gpu" : " 17 GB", | 20260505 18:55:10.982 | INFO | "huge" : "", | 20260505 18:55:10.982 | INFO | "huge_shared" : " 27 GB", | 20260505 18:55:10.982 | INFO | "malloc" : " 14 GB", | 20260505 18:55:10.982 | INFO | "pinned" : " 29 GB" | 20260505 18:55:10.982 | INFO | } | 20260505 18:55:10.982 | INFO | Perf report for the last 5.1354 seconds: | 20260505 18:55:10.982 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:55:10.982 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:10.982 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:10.982 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:10.982 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 18:55:10.982 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:10.982 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:10.982 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 18:55:10.982 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:10.982 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:10.982 | INFO | Uploaded 36GiB to GPU in 2.12294s (16.9577GiB/s) | 20260505 18:55:10.982 | INFO | Downloaded 34GiB to GPU in 2.02033s (16.8289GiB/s) | 20260505 18:55:10.982 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 18:55:10.982 | INFO | Total mem usage: 15.3 GiB | 20260505 18:55:10.982 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 18:55:10.982 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.03%) | 20260505 18:55:10.982 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 18:55:10.982 | INFO | High Water Breakdown: | 20260505 18:55:10.982 | INFO | 61% : 8.66 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 18:55:10.982 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 18:55:10.982 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 18:55:10.982 | INFO | ------------------------------------------ | 20260505 18:55:10.982 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:55:10.983 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 18:55:10.983 | INFO | Total mem usage: 26.5 GiB | 20260505 18:55:10.983 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 18:55:10.983 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 18:55:10.983 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 18:55:10.983 | INFO | High Water Breakdown: | 20260505 18:55:10.983 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:55:10.983 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:55:10.983 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:55:10.983 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:55:10.983 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:55:10.983 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:55:10.983 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 18:55:10.983 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 18:55:10.983 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:55:10.983 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 18:55:10.983 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 18:55:10.983 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 18:55:10.983 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 18:55:10.983 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 18:55:10.983 | INFO | ------------------------------------------ | 20260505 18:55:10.983 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:55:10.983 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 18:55:10.983 | INFO | Total mem usage: 25.6 GiB | 20260505 18:55:10.983 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 18:55:10.983 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 18:55:10.983 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 18:55:10.983 | INFO | High Water Breakdown: | 20260505 18:55:10.983 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 18:55:10.983 | INFO | ------------------------------------------ | 20260505 18:55:10.983 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:55:10.983 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 18:55:10.983 | INFO | Total mem usage: 15.7 GiB | 20260505 18:55:10.983 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 18:55:10.983 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 18:55:10.983 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 18:55:10.983 | INFO | High Water Breakdown: | 20260505 18:55:10.983 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 18:55:10.983 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 18:55:10.983 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 18:55:10.983 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 18:55:10.983 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 18:55:10.983 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 18:55:10.983 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 18:55:10.983 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 18:55:10.983 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 18:55:10.983 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:55:10.983 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 18:55:10.983 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:55:10.983 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 18:55:10.983 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 18:55:10.983 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 18:55:10.983 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 18:55:10.983 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:55:10.983 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 18:55:10.983 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:55:10.983 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:55:10.983 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:55:10.983 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 18:55:10.983 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 18:55:10.983 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:55:10.983 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 18:55:10.984 | INFO | ------------------------------------------ | 20260505 18:55:10.984 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:55:10.984 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 18:55:10.984 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:55:10.984 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:55:10.984 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.48e+01, 1.35e+01 / 1.93e+00 | 20260505 18:55:10.984 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.02e-01, 1.05e-02 / 4.12e-03 | 20260505 18:55:10.984 | INFO | PerfReport: 11.80%, Baseline -- 3.73e+00 / 4.54e+00, 5.69e-01 / 1.24e-01 | 20260505 18:55:10.984 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.52e+00, 1.92e-01 / 7.23e-02 | 20260505 18:55:10.984 | INFO | PerfReport: 15.93%, DME -- 4.02e+01 / 4.46e+01, 4.12e+00 / 2.01e+00 | 20260505 18:55:10.984 | INFO | PerfReport: 56.96%, FrameLabeling -- 1.80e+01 / 1.93e+01, 1.49e+00 / 9.44e-01 | 20260505 18:55:10.984 | INFO | PerfReport: 7.26%, PulseAccumulating -- 2.29e+00 / 2.89e+00, 4.32e-01 / 1.39e-01 | 20260505 18:55:10.984 | INFO | PerfReport: 3.31%, Metrics -- 1.05e+00 / 1.39e+00, 2.90e-01 / 1.47e-01 | 20260505 18:55:10.984 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.64e-01 / 4.33e-01, 6.53e-02 / 3.87e-02 | 20260505 18:55:10.984 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 18:55:10.984 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:55:10.984 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:55:10.984 | INFO | PerfReport: Full Execution -- 9.96e+01 / -nan, 2.54e+01 / 0.00e+00 | 20260505 18:55:10.984 | INFO | PerfReport: 22.46%, Upload -- 2.24e+01 / 0.00e+00, 7.69e+00 / 0.00e+00 | 20260505 18:55:10.984 | INFO | PerfReport: 21.83%, Download -- 2.18e+01 / 0.00e+00, 7.71e+00 / 0.00e+00 | 20260505 18:55:10.984 | INFO | PerfReport: 31.69%, Compute -- 3.16e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 18:55:10.984 | INFO | PerfReport: 2.25%, LockUpload -- 2.24e+00 / 0.00e+00, 7.90e+00 / 0.00e+00 | 20260505 18:55:10.984 | INFO | PerfReport: 1.39%, LockDownload -- 1.38e+00 / 0.00e+00, 2.93e+00 / 0.00e+00 | 20260505 18:55:10.984 | INFO | PerfReport: 20.37%, LockCompute -- 2.03e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 18:55:10.984 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.81e-02 / 0.00e+00 | 20260505 18:55:11.115 | INFO | Starting a round of Baz IO | 20260505 18:55:14.295 | INFO | Populating RTMetrics report | 20260505 18:55:14.295 | INFO | RTMetrics report populated | 20260505 18:55:14.295 | INFO | Opening tmp RTMetrics json file | 20260505 18:55:14.310 | INFO | Serializing RTMetrics to json | 20260505 18:55:14.331 | INFO | Serialization done, swapping tmp file now... | 20260505 18:55:14.365 | INFO | RTMetrics done | 20260505 18:55:14.382 | INFO | Finished a round of Baz IO: Wrote 2.074 GiB in 3.266 seconds | 20260505 18:55:14.382 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.84% / 0.00% / 78.15% | 20260505 18:55:14.382 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.230034% / 3.97364e-06% / 7.54992e-05% | 20260505 18:55:14.382 | INFO | Baz IO overall: | 20260505 18:55:14.382 | INFO | BytesWritten : 3243291696100 | 20260505 18:55:14.382 | INFO | NumEvents : 1535666885373 | 20260505 18:55:16.112 | INFO | Analyzing chunk frames = [5521408,5521920) | 20260505 18:55:16.112 | INFO | MemInfo: { | 20260505 18:55:16.112 | INFO | "gpu" : " 17 GB", | 20260505 18:55:16.112 | INFO | "huge" : "", | 20260505 18:55:16.112 | INFO | "huge_shared" : " 27 GB", | 20260505 18:55:16.112 | INFO | "malloc" : " 14 GB", | 20260505 18:55:16.112 | INFO | "pinned" : " 29 GB" | 20260505 18:55:16.113 | INFO | } | 20260505 18:55:16.113 | INFO | Perf report for the last 5.13068 seconds: | 20260505 18:55:16.113 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:55:16.113 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:16.113 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:16.113 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:16.113 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260505 18:55:16.113 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:16.113 | INFO | RT_METRICS : 45.7%, 1.0, 23.9ms, 0 | 20260505 18:55:16.113 | INFO | PRE_HQ : 60.5%, 1.8, 55.9ms, 0 | 20260505 18:55:16.113 | INFO | BAZWRITER : 63.2%, 3.8, 123.2ms, 0 | 20260505 18:55:16.113 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260505 18:55:16.113 | INFO | Uploaded 36GiB to GPU in 2.14839s (16.7568GiB/s) | 20260505 18:55:16.113 | INFO | Downloaded 38GiB to GPU in 2.24132s (16.9543GiB/s) | 20260505 18:55:21.258 | INFO | Analyzing chunk frames = [5521920,5522432) | 20260505 18:55:21.258 | INFO | MemInfo: { | 20260505 18:55:21.258 | INFO | "gpu" : " 17 GB", | 20260505 18:55:21.258 | INFO | "huge" : "", | 20260505 18:55:21.258 | INFO | "huge_shared" : " 27 GB", | 20260505 18:55:21.258 | INFO | "malloc" : " 14 GB", | 20260505 18:55:21.258 | INFO | "pinned" : " 29 GB" | 20260505 18:55:21.258 | INFO | } | 20260505 18:55:21.258 | INFO | Perf report for the last 5.14526 seconds: | 20260505 18:55:21.258 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:55:21.258 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:21.258 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:21.258 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:21.258 | INFO | ANALYSIS : 62.6%, 3.0, 97.9ms, 0 | 20260505 18:55:21.258 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:21.258 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:21.258 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 18:55:21.258 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:21.258 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:21.258 | INFO | Uploaded 36GiB to GPU in 2.13057s (16.8969GiB/s) | 20260505 18:55:21.258 | INFO | Downloaded 34GiB to GPU in 2.02645s (16.7781GiB/s) | 20260505 18:55:26.398 | INFO | Analyzing chunk frames = [5522432,5522944) | 20260505 18:55:26.398 | INFO | MemInfo: { | 20260505 18:55:26.398 | INFO | "gpu" : " 17 GB", | 20260505 18:55:26.398 | INFO | "huge" : "", | 20260505 18:55:26.398 | INFO | "huge_shared" : " 27 GB", | 20260505 18:55:26.398 | INFO | "malloc" : " 14 GB", | 20260505 18:55:26.398 | INFO | "pinned" : " 29 GB" | 20260505 18:55:26.398 | INFO | } | 20260505 18:55:26.398 | INFO | Perf report for the last 5.14008 seconds: | 20260505 18:55:26.398 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:55:26.398 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:26.398 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:26.398 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:26.398 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260505 18:55:26.398 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:26.398 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:26.398 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 18:55:26.398 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:26.398 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:26.398 | INFO | Uploaded 36GiB to GPU in 2.14224s (16.8048GiB/s) | 20260505 18:55:26.398 | INFO | Downloaded 34GiB to GPU in 2.04122s (16.6567GiB/s) | 20260505 18:55:31.542 | INFO | Analyzing chunk frames = [5522944,5523456) | 20260505 18:55:31.542 | INFO | MemInfo: { | 20260505 18:55:31.542 | INFO | "gpu" : " 17 GB", | 20260505 18:55:31.542 | INFO | "huge" : "", | 20260505 18:55:31.542 | INFO | "huge_shared" : " 27 GB", | 20260505 18:55:31.542 | INFO | "malloc" : " 14 GB", | 20260505 18:55:31.542 | INFO | "pinned" : " 29 GB" | 20260505 18:55:31.542 | INFO | } | 20260505 18:55:31.542 | INFO | Perf report for the last 5.14439 seconds: | 20260505 18:55:31.542 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:55:31.542 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:31.542 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:31.542 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:31.542 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260505 18:55:31.542 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:31.542 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:31.542 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:55:31.543 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:31.543 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:31.543 | INFO | Uploaded 36GiB to GPU in 2.19875s (16.3729GiB/s) | 20260505 18:55:31.543 | INFO | Downloaded 34GiB to GPU in 2.04056s (16.6621GiB/s) | 20260505 18:55:36.678 | INFO | Analyzing chunk frames = [5523456,5523968) | 20260505 18:55:36.678 | INFO | MemInfo: { | 20260505 18:55:36.678 | INFO | "gpu" : " 17 GB", | 20260505 18:55:36.678 | INFO | "huge" : "", | 20260505 18:55:36.678 | INFO | "huge_shared" : " 27 GB", | 20260505 18:55:36.678 | INFO | "malloc" : " 14 GB", | 20260505 18:55:36.678 | INFO | "pinned" : " 29 GB" | 20260505 18:55:36.678 | INFO | } | 20260505 18:55:36.678 | INFO | Perf report for the last 5.13565 seconds: | 20260505 18:55:36.678 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:55:36.678 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:36.678 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:36.678 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:36.678 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260505 18:55:36.678 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:36.678 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:36.678 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:55:36.678 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:36.678 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:36.678 | INFO | Uploaded 36GiB to GPU in 2.13087s (16.8945GiB/s) | 20260505 18:55:36.678 | INFO | Downloaded 34GiB to GPU in 2.05782s (16.5223GiB/s) | 20260505 18:55:41.810 | INFO | Analyzing chunk frames = [5523968,5524480) | 20260505 18:55:41.810 | INFO | MemInfo: { | 20260505 18:55:41.810 | INFO | "gpu" : " 17 GB", | 20260505 18:55:41.810 | INFO | "huge" : "", | 20260505 18:55:41.810 | INFO | "huge_shared" : " 27 GB", | 20260505 18:55:41.810 | INFO | "malloc" : " 14 GB", | 20260505 18:55:41.810 | INFO | "pinned" : " 29 GB" | 20260505 18:55:41.810 | INFO | } | 20260505 18:55:41.810 | INFO | Perf report for the last 5.1318 seconds: | 20260505 18:55:41.810 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:55:41.810 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:41.810 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:41.810 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:41.810 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260505 18:55:41.810 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:41.810 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:41.810 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 18:55:41.810 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:41.810 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:41.810 | INFO | Uploaded 36GiB to GPU in 2.15797s (16.6823GiB/s) | 20260505 18:55:41.810 | INFO | Downloaded 34GiB to GPU in 2.05608s (16.5363GiB/s) | 20260505 18:55:46.954 | INFO | Analyzing chunk frames = [5524480,5524992) | 20260505 18:55:46.954 | INFO | MemInfo: { | 20260505 18:55:46.954 | INFO | "gpu" : " 17 GB", | 20260505 18:55:46.954 | INFO | "huge" : "", | 20260505 18:55:46.954 | INFO | "huge_shared" : " 27 GB", | 20260505 18:55:46.954 | INFO | "malloc" : " 14 GB", | 20260505 18:55:46.954 | INFO | "pinned" : " 29 GB" | 20260505 18:55:46.954 | INFO | } | 20260505 18:55:46.955 | INFO | Perf report for the last 5.14467 seconds: | 20260505 18:55:46.955 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:55:46.955 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:46.955 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:46.955 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:46.955 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 18:55:46.955 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:46.955 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:46.955 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 18:55:46.955 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:46.955 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:46.955 | INFO | Uploaded 36GiB to GPU in 2.15027s (16.7421GiB/s) | 20260505 18:55:46.955 | INFO | Downloaded 34GiB to GPU in 2.14263s (15.8683GiB/s) | 20260505 18:55:52.094 | INFO | Analyzing chunk frames = [5524992,5525504) | 20260505 18:55:52.094 | INFO | MemInfo: { | 20260505 18:55:52.094 | INFO | "gpu" : " 17 GB", | 20260505 18:55:52.094 | INFO | "huge" : "", | 20260505 18:55:52.094 | INFO | "huge_shared" : " 27 GB", | 20260505 18:55:52.094 | INFO | "malloc" : " 14 GB", | 20260505 18:55:52.095 | INFO | "pinned" : " 29 GB" | 20260505 18:55:52.095 | INFO | } | 20260505 18:55:52.095 | INFO | Perf report for the last 5.14006 seconds: | 20260505 18:55:52.095 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:55:52.095 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:52.095 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:52.095 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:52.095 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 18:55:52.095 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:52.095 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:52.095 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 18:55:52.095 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:52.095 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:52.095 | INFO | Uploaded 36GiB to GPU in 2.19512s (16.4GiB/s) | 20260505 18:55:52.095 | INFO | Downloaded 34GiB to GPU in 2.04157s (16.6539GiB/s) | 20260505 18:55:52.227 | INFO | Starting a round of Baz IO | 20260505 18:55:55.406 | INFO | Populating RTMetrics report | 20260505 18:55:55.406 | INFO | RTMetrics report populated | 20260505 18:55:55.406 | INFO | Opening tmp RTMetrics json file | 20260505 18:55:55.409 | INFO | Serializing RTMetrics to json | 20260505 18:55:55.411 | INFO | Serialization done, swapping tmp file now... | 20260505 18:55:55.411 | INFO | RTMetrics done | 20260505 18:55:55.501 | INFO | Finished a round of Baz IO: Wrote 2.073 GiB in 3.273 seconds | 20260505 18:55:55.501 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.84% / 0.00% / 78.16% | 20260505 18:55:55.501 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.229589% / 7.94729e-06% / 9.13938e-05% | 20260505 18:55:55.501 | INFO | Baz IO overall: | 20260505 18:55:55.501 | INFO | BytesWritten : 3245517221669 | 20260505 18:55:55.501 | INFO | NumEvents : 1536703012134 | 20260505 18:55:57.229 | INFO | Analyzing chunk frames = [5525504,5526016) | 20260505 18:55:57.229 | INFO | MemInfo: { | 20260505 18:55:57.229 | INFO | "gpu" : " 17 GB", | 20260505 18:55:57.229 | INFO | "huge" : "", | 20260505 18:55:57.229 | INFO | "huge_shared" : " 27 GB", | 20260505 18:55:57.229 | INFO | "malloc" : " 14 GB", | 20260505 18:55:57.229 | INFO | "pinned" : " 29 GB" | 20260505 18:55:57.229 | INFO | } | 20260505 18:55:57.229 | INFO | Perf report for the last 5.13435 seconds: | 20260505 18:55:57.229 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:55:57.229 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:57.229 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:57.229 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:57.229 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 18:55:57.229 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:55:57.229 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260505 18:55:57.229 | INFO | PRE_HQ : 60.3%, 1.8, 55.6ms, 0 | 20260505 18:55:57.229 | INFO | BAZWRITER : 63.3%, 3.8, 123.0ms, 0 | 20260505 18:55:57.229 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 18:55:57.229 | INFO | Uploaded 36GiB to GPU in 2.15948s (16.6707GiB/s) | 20260505 18:55:57.229 | INFO | Downloaded 38GiB to GPU in 2.23865s (16.9745GiB/s) | 20260505 18:56:02.364 | INFO | Analyzing chunk frames = [5526016,5526528) | 20260505 18:56:02.364 | INFO | MemInfo: { | 20260505 18:56:02.364 | INFO | "gpu" : " 17 GB", | 20260505 18:56:02.364 | INFO | "huge" : "", | 20260505 18:56:02.364 | INFO | "huge_shared" : " 27 GB", | 20260505 18:56:02.364 | INFO | "malloc" : " 14 GB", | 20260505 18:56:02.364 | INFO | "pinned" : " 29 GB" | 20260505 18:56:02.364 | INFO | } | 20260505 18:56:02.364 | INFO | Perf report for the last 5.13535 seconds: | 20260505 18:56:02.364 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:56:02.365 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:02.365 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:02.365 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:02.365 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260505 18:56:02.365 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:02.365 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:02.365 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:56:02.365 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:02.365 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:02.365 | INFO | Uploaded 36GiB to GPU in 2.13952s (16.8262GiB/s) | 20260505 18:56:02.365 | INFO | Downloaded 34GiB to GPU in 2.03518s (16.7062GiB/s) | 20260505 18:56:07.505 | INFO | Analyzing chunk frames = [5526528,5527040) | 20260505 18:56:07.505 | INFO | MemInfo: { | 20260505 18:56:07.505 | INFO | "gpu" : " 17 GB", | 20260505 18:56:07.505 | INFO | "huge" : "", | 20260505 18:56:07.505 | INFO | "huge_shared" : " 27 GB", | 20260505 18:56:07.505 | INFO | "malloc" : " 14 GB", | 20260505 18:56:07.505 | INFO | "pinned" : " 29 GB" | 20260505 18:56:07.505 | INFO | } | 20260505 18:56:07.505 | INFO | Perf report for the last 5.14084 seconds: | 20260505 18:56:07.505 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:56:07.505 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:07.505 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:07.505 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:07.505 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 18:56:07.505 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:07.505 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:07.505 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:56:07.505 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:07.505 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:07.505 | INFO | Uploaded 36GiB to GPU in 2.20806s (16.3039GiB/s) | 20260505 18:56:07.506 | INFO | Downloaded 34GiB to GPU in 2.04847s (16.5977GiB/s) | 20260505 18:56:12.653 | INFO | Analyzing chunk frames = [5527040,5527552) | 20260505 18:56:12.653 | INFO | MemInfo: { | 20260505 18:56:12.653 | INFO | "gpu" : " 17 GB", | 20260505 18:56:12.653 | INFO | "huge" : "", | 20260505 18:56:12.653 | INFO | "huge_shared" : " 27 GB", | 20260505 18:56:12.653 | INFO | "malloc" : " 14 GB", | 20260505 18:56:12.653 | INFO | "pinned" : " 29 GB" | 20260505 18:56:12.653 | INFO | } | 20260505 18:56:12.653 | INFO | Perf report for the last 5.14762 seconds: | 20260505 18:56:12.653 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:56:12.653 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:12.653 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:12.653 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:12.653 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260505 18:56:12.653 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:12.653 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:12.653 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:56:12.653 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:12.653 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:12.653 | INFO | Uploaded 36GiB to GPU in 2.18184s (16.4999GiB/s) | 20260505 18:56:12.653 | INFO | Downloaded 34GiB to GPU in 2.0655s (16.4609GiB/s) | 20260505 18:56:17.795 | INFO | Analyzing chunk frames = [5527552,5528064) | 20260505 18:56:17.795 | INFO | MemInfo: { | 20260505 18:56:17.795 | INFO | "gpu" : " 17 GB", | 20260505 18:56:17.795 | INFO | "huge" : "", | 20260505 18:56:17.795 | INFO | "huge_shared" : " 27 GB", | 20260505 18:56:17.795 | INFO | "malloc" : " 14 GB", | 20260505 18:56:17.795 | INFO | "pinned" : " 29 GB" | 20260505 18:56:17.795 | INFO | } | 20260505 18:56:17.796 | INFO | Perf report for the last 5.1427 seconds: | 20260505 18:56:17.796 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:56:17.796 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:17.796 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:17.796 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:17.796 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 18:56:17.796 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:17.796 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:17.796 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 18:56:17.796 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:17.796 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:17.796 | INFO | Uploaded 36GiB to GPU in 2.13277s (16.8795GiB/s) | 20260505 18:56:17.796 | INFO | Downloaded 34GiB to GPU in 2.06112s (16.4959GiB/s) | 20260505 18:56:22.925 | INFO | Analyzing chunk frames = [5528064,5528576) | 20260505 18:56:22.925 | INFO | MemInfo: { | 20260505 18:56:22.925 | INFO | "gpu" : " 17 GB", | 20260505 18:56:22.925 | INFO | "huge" : "", | 20260505 18:56:22.925 | INFO | "huge_shared" : " 27 GB", | 20260505 18:56:22.925 | INFO | "malloc" : " 14 GB", | 20260505 18:56:22.925 | INFO | "pinned" : " 29 GB" | 20260505 18:56:22.925 | INFO | } | 20260505 18:56:22.925 | INFO | Perf report for the last 5.12941 seconds: | 20260505 18:56:22.925 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:56:22.925 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:22.925 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:22.925 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:22.925 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260505 18:56:22.925 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:22.925 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:22.925 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 18:56:22.925 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:22.925 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:22.925 | INFO | Uploaded 36GiB to GPU in 2.1489s (16.7527GiB/s) | 20260505 18:56:22.925 | INFO | Downloaded 34GiB to GPU in 2.06047s (16.5011GiB/s) | 20260505 18:56:28.061 | INFO | Analyzing chunk frames = [5528576,5529088) | 20260505 18:56:28.061 | INFO | MemInfo: { | 20260505 18:56:28.061 | INFO | "gpu" : " 17 GB", | 20260505 18:56:28.061 | INFO | "huge" : "", | 20260505 18:56:28.061 | INFO | "huge_shared" : " 27 GB", | 20260505 18:56:28.061 | INFO | "malloc" : " 14 GB", | 20260505 18:56:28.061 | INFO | "pinned" : " 29 GB" | 20260505 18:56:28.061 | INFO | } | 20260505 18:56:28.061 | INFO | Perf report for the last 5.13586 seconds: | 20260505 18:56:28.061 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:56:28.061 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:28.061 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:28.061 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:28.061 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 18:56:28.061 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:28.061 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:28.061 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 18:56:28.061 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:28.061 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:28.061 | INFO | Uploaded 36GiB to GPU in 2.14171s (16.809GiB/s) | 20260505 18:56:28.061 | INFO | Downloaded 34GiB to GPU in 2.13433s (15.9301GiB/s) | 20260505 18:56:33.210 | INFO | Analyzing chunk frames = [5529088,5529600) | 20260505 18:56:33.210 | INFO | MemInfo: { | 20260505 18:56:33.210 | INFO | "gpu" : " 17 GB", | 20260505 18:56:33.210 | INFO | "huge" : "", | 20260505 18:56:33.210 | INFO | "huge_shared" : " 27 GB", | 20260505 18:56:33.210 | INFO | "malloc" : " 14 GB", | 20260505 18:56:33.210 | INFO | "pinned" : " 29 GB" | 20260505 18:56:33.210 | INFO | } | 20260505 18:56:33.210 | INFO | Perf report for the last 5.14926 seconds: | 20260505 18:56:33.210 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:56:33.210 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:33.210 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:33.210 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:33.210 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 18:56:33.210 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:33.210 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:33.210 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 18:56:33.210 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:33.210 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:33.210 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 18:56:33.210 | INFO | Total mem usage: 15.3 GiB | 20260505 18:56:33.210 | INFO | Uploaded 36GiB to GPU in 2.19327s (16.4138GiB/s) | 20260505 18:56:33.211 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 18:56:33.211 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.06%) | 20260505 18:56:33.211 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 18:56:33.211 | INFO | Downloaded 34GiB to GPU in 2.02551s (16.7859GiB/s) | 20260505 18:56:33.211 | INFO | High Water Breakdown: | 20260505 18:56:33.211 | INFO | 61% : 8.66 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 18:56:33.211 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 18:56:33.211 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 18:56:33.211 | INFO | ------------------------------------------ | 20260505 18:56:33.211 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:56:33.211 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 18:56:33.211 | INFO | Total mem usage: 26.5 GiB | 20260505 18:56:33.211 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 18:56:33.211 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 18:56:33.211 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 18:56:33.211 | INFO | High Water Breakdown: | 20260505 18:56:33.211 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:56:33.211 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:56:33.211 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:56:33.211 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:56:33.211 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:56:33.211 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:56:33.211 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 18:56:33.211 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 18:56:33.211 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:56:33.211 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 18:56:33.211 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 18:56:33.211 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 18:56:33.211 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 18:56:33.211 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 18:56:33.211 | INFO | ------------------------------------------ | 20260505 18:56:33.211 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:56:33.211 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 18:56:33.211 | INFO | Total mem usage: 25.6 GiB | 20260505 18:56:33.211 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 18:56:33.211 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 18:56:33.211 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 18:56:33.211 | INFO | High Water Breakdown: | 20260505 18:56:33.211 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 18:56:33.211 | INFO | ------------------------------------------ | 20260505 18:56:33.211 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:56:33.211 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 18:56:33.211 | INFO | Total mem usage: 15.7 GiB | 20260505 18:56:33.211 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 18:56:33.211 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 18:56:33.211 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 18:56:33.211 | INFO | High Water Breakdown: | 20260505 18:56:33.211 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 18:56:33.211 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 18:56:33.211 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 18:56:33.211 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 18:56:33.211 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 18:56:33.211 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 18:56:33.211 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 18:56:33.211 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 18:56:33.211 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 18:56:33.211 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:56:33.211 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 18:56:33.212 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:56:33.212 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 18:56:33.212 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 18:56:33.212 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 18:56:33.212 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 18:56:33.212 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:56:33.212 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 18:56:33.212 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:56:33.212 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:56:33.212 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:56:33.212 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 18:56:33.212 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 18:56:33.212 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:56:33.212 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 18:56:33.212 | INFO | ------------------------------------------ | 20260505 18:56:33.212 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:56:33.212 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 18:56:33.212 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:56:33.212 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:56:33.212 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.46e+01, 1.36e+01 / 2.12e+00 | 20260505 18:56:33.212 | INFO | PerfReport: 0.29%, AggregateStats -- 9.26e-02 / 1.04e-01, 1.08e-02 / 6.50e-03 | 20260505 18:56:33.212 | INFO | PerfReport: 11.80%, Baseline -- 3.72e+00 / 4.52e+00, 5.63e-01 / 1.30e-01 | 20260505 18:56:33.212 | INFO | PerfReport: 3.29%, Histogram -- 1.04e+00 / 1.53e+00, 1.91e-01 / 8.16e-02 | 20260505 18:56:33.212 | INFO | PerfReport: 15.95%, DME -- 4.02e+01 / 4.45e+01, 4.09e+00 / 2.12e+00 | 20260505 18:56:33.212 | INFO | PerfReport: 56.93%, FrameLabeling -- 1.80e+01 / 1.93e+01, 1.47e+00 / 1.18e+00 | 20260505 18:56:33.212 | INFO | PerfReport: 7.29%, PulseAccumulating -- 2.30e+00 / 2.89e+00, 4.39e-01 / 1.63e-01 | 20260505 18:56:33.212 | INFO | PerfReport: 3.31%, Metrics -- 1.04e+00 / 1.38e+00, 2.86e-01 / 1.36e-01 | 20260505 18:56:33.212 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.63e-01 / 4.28e-01, 6.42e-02 / 3.97e-02 | 20260505 18:56:33.212 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 18:56:33.212 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:56:33.212 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:56:33.212 | INFO | PerfReport: Full Execution -- 9.97e+01 / -nan, 2.56e+01 / 0.00e+00 | 20260505 18:56:33.212 | INFO | PerfReport: 22.56%, Upload -- 2.25e+01 / 0.00e+00, 7.82e+00 / 0.00e+00 | 20260505 18:56:33.212 | INFO | PerfReport: 21.75%, Download -- 2.17e+01 / 0.00e+00, 7.68e+00 / 0.00e+00 | 20260505 18:56:33.212 | INFO | PerfReport: 31.63%, Compute -- 3.15e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 18:56:33.212 | INFO | PerfReport: 2.33%, LockUpload -- 2.32e+00 / 0.00e+00, 7.93e+00 / 0.00e+00 | 20260505 18:56:33.212 | INFO | PerfReport: 1.34%, LockDownload -- 1.33e+00 / 0.00e+00, 2.78e+00 / 0.00e+00 | 20260505 18:56:33.212 | INFO | PerfReport: 20.38%, LockCompute -- 2.03e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260505 18:56:33.212 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.79e-02 / 0.00e+00 | 20260505 18:56:33.339 | INFO | Starting a round of Baz IO | 20260505 18:56:36.517 | INFO | Populating RTMetrics report | 20260505 18:56:36.517 | INFO | RTMetrics report populated | 20260505 18:56:36.517 | INFO | Opening tmp RTMetrics json file | 20260505 18:56:36.520 | INFO | Serializing RTMetrics to json | 20260505 18:56:36.522 | INFO | Serialization done, swapping tmp file now... | 20260505 18:56:36.522 | INFO | RTMetrics done | 20260505 18:56:36.611 | INFO | Finished a round of Baz IO: Wrote 2.072 GiB in 3.271 seconds | 20260505 18:56:36.611 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.84% / 0.00% / 78.16% | 20260505 18:56:36.611 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.229112% / 0% / 6.35783e-05% | 20260505 18:56:36.611 | INFO | Baz IO overall: | 20260505 18:56:36.611 | INFO | BytesWritten : 3247742148304 | 20260505 18:56:36.611 | INFO | NumEvents : 1537738861725 | 20260505 18:56:38.350 | INFO | Analyzing chunk frames = [5529600,5530112) | 20260505 18:56:38.350 | INFO | MemInfo: { | 20260505 18:56:38.350 | INFO | "gpu" : " 17 GB", | 20260505 18:56:38.350 | INFO | "huge" : "", | 20260505 18:56:38.350 | INFO | "huge_shared" : " 27 GB", | 20260505 18:56:38.350 | INFO | "malloc" : " 14 GB", | 20260505 18:56:38.350 | INFO | "pinned" : " 29 GB" | 20260505 18:56:38.350 | INFO | } | 20260505 18:56:38.350 | INFO | Perf report for the last 5.13954 seconds: | 20260505 18:56:38.350 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:56:38.350 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:38.350 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:38.350 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:38.350 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 18:56:38.350 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:38.350 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260505 18:56:38.350 | INFO | PRE_HQ : 60.4%, 1.8, 55.7ms, 0 | 20260505 18:56:38.350 | INFO | BAZWRITER : 63.2%, 3.8, 123.0ms, 0 | 20260505 18:56:38.350 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 18:56:38.350 | INFO | Uploaded 36GiB to GPU in 2.16744s (16.6094GiB/s) | 20260505 18:56:38.350 | INFO | Downloaded 38GiB to GPU in 2.25196s (16.8742GiB/s) | 20260505 18:56:43.489 | INFO | Analyzing chunk frames = [5530112,5530624) | 20260505 18:56:43.489 | INFO | MemInfo: { | 20260505 18:56:43.489 | INFO | "gpu" : " 17 GB", | 20260505 18:56:43.489 | INFO | "huge" : "", | 20260505 18:56:43.489 | INFO | "huge_shared" : " 27 GB", | 20260505 18:56:43.489 | INFO | "malloc" : " 14 GB", | 20260505 18:56:43.489 | INFO | "pinned" : " 29 GB" | 20260505 18:56:43.489 | INFO | } | 20260505 18:56:43.489 | INFO | Perf report for the last 5.1396 seconds: | 20260505 18:56:43.489 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:56:43.489 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:43.489 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:43.489 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:43.489 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260505 18:56:43.489 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:43.489 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:43.489 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 18:56:43.489 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:43.489 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:43.490 | INFO | Uploaded 36GiB to GPU in 2.09045s (17.2212GiB/s) | 20260505 18:56:43.490 | INFO | Downloaded 34GiB to GPU in 2.02602s (16.7817GiB/s) | 20260505 18:56:48.626 | INFO | Analyzing chunk frames = [5530624,5531136) | 20260505 18:56:48.626 | INFO | MemInfo: { | 20260505 18:56:48.626 | INFO | "gpu" : " 17 GB", | 20260505 18:56:48.626 | INFO | "huge" : "", | 20260505 18:56:48.626 | INFO | "huge_shared" : " 27 GB", | 20260505 18:56:48.626 | INFO | "malloc" : " 14 GB", | 20260505 18:56:48.627 | INFO | "pinned" : " 29 GB" | 20260505 18:56:48.627 | INFO | } | 20260505 18:56:48.627 | INFO | Perf report for the last 5.13733 seconds: | 20260505 18:56:48.627 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:56:48.627 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:48.627 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:48.627 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:48.627 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260505 18:56:48.627 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:48.627 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:48.627 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:56:48.627 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:48.627 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:48.627 | INFO | Uploaded 36GiB to GPU in 2.12072s (16.9753GiB/s) | 20260505 18:56:48.627 | INFO | Downloaded 34GiB to GPU in 2.0305s (16.7446GiB/s) | 20260505 18:56:53.765 | INFO | Analyzing chunk frames = [5531136,5531648) | 20260505 18:56:53.765 | INFO | MemInfo: { | 20260505 18:56:53.765 | INFO | "gpu" : " 17 GB", | 20260505 18:56:53.765 | INFO | "huge" : "", | 20260505 18:56:53.765 | INFO | "huge_shared" : " 27 GB", | 20260505 18:56:53.765 | INFO | "malloc" : " 14 GB", | 20260505 18:56:53.765 | INFO | "pinned" : " 29 GB" | 20260505 18:56:53.765 | INFO | } | 20260505 18:56:53.765 | INFO | Perf report for the last 5.13843 seconds: | 20260505 18:56:53.765 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:56:53.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:53.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:53.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:53.765 | INFO | ANALYSIS : 62.8%, 3.0, 97.6ms, 0 | 20260505 18:56:53.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:53.765 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:53.765 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:56:53.765 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:53.765 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:53.765 | INFO | Uploaded 36GiB to GPU in 2.22658s (16.1683GiB/s) | 20260505 18:56:53.765 | INFO | Downloaded 34GiB to GPU in 2.12264s (16.0178GiB/s) | 20260505 18:56:58.906 | INFO | Analyzing chunk frames = [5531648,5532160) | 20260505 18:56:58.906 | INFO | MemInfo: { | 20260505 18:56:58.906 | INFO | "gpu" : " 17 GB", | 20260505 18:56:58.906 | INFO | "huge" : "", | 20260505 18:56:58.906 | INFO | "huge_shared" : " 27 GB", | 20260505 18:56:58.906 | INFO | "malloc" : " 14 GB", | 20260505 18:56:58.906 | INFO | "pinned" : " 29 GB" | 20260505 18:56:58.906 | INFO | } | 20260505 18:56:58.907 | INFO | Perf report for the last 5.14147 seconds: | 20260505 18:56:58.907 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:56:58.907 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:58.907 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:58.907 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:58.907 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 18:56:58.907 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:58.907 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:58.907 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:56:58.907 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:58.907 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:56:58.907 | INFO | Uploaded 36GiB to GPU in 2.14027s (16.8203GiB/s) | 20260505 18:56:58.907 | INFO | Downloaded 34GiB to GPU in 2.06466s (16.4676GiB/s) | 20260505 18:57:04.046 | INFO | Analyzing chunk frames = [5532160,5532672) | 20260505 18:57:04.046 | INFO | MemInfo: { | 20260505 18:57:04.046 | INFO | "gpu" : " 17 GB", | 20260505 18:57:04.046 | INFO | "huge" : "", | 20260505 18:57:04.046 | INFO | "huge_shared" : " 27 GB", | 20260505 18:57:04.046 | INFO | "malloc" : " 14 GB", | 20260505 18:57:04.046 | INFO | "pinned" : " 29 GB" | 20260505 18:57:04.046 | INFO | } | 20260505 18:57:04.046 | INFO | Perf report for the last 5.13932 seconds: | 20260505 18:57:04.046 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:57:04.046 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:04.046 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:04.046 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:04.046 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 18:57:04.046 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:04.046 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:04.046 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:57:04.046 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:04.046 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:04.046 | INFO | Uploaded 36GiB to GPU in 2.17025s (16.588GiB/s) | 20260505 18:57:04.046 | INFO | Downloaded 34GiB to GPU in 2.06716s (16.4477GiB/s) | 20260505 18:57:09.180 | INFO | Analyzing chunk frames = [5532672,5533184) | 20260505 18:57:09.180 | INFO | MemInfo: { | 20260505 18:57:09.180 | INFO | "gpu" : " 17 GB", | 20260505 18:57:09.180 | INFO | "huge" : "", | 20260505 18:57:09.180 | INFO | "huge_shared" : " 27 GB", | 20260505 18:57:09.180 | INFO | "malloc" : " 14 GB", | 20260505 18:57:09.180 | INFO | "pinned" : " 29 GB" | 20260505 18:57:09.180 | INFO | } | 20260505 18:57:09.180 | INFO | Perf report for the last 5.13429 seconds: | 20260505 18:57:09.180 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:57:09.180 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:09.180 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:09.180 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:09.180 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 18:57:09.180 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:09.180 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:09.180 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 18:57:09.180 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:09.180 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:09.180 | INFO | Uploaded 36GiB to GPU in 2.13572s (16.8562GiB/s) | 20260505 18:57:09.181 | INFO | Downloaded 34GiB to GPU in 2.15253s (15.7953GiB/s) | 20260505 18:57:14.315 | INFO | Analyzing chunk frames = [5533184,5533696) | 20260505 18:57:14.315 | INFO | MemInfo: { | 20260505 18:57:14.315 | INFO | "gpu" : " 17 GB", | 20260505 18:57:14.315 | INFO | "huge" : "", | 20260505 18:57:14.315 | INFO | "huge_shared" : " 27 GB", | 20260505 18:57:14.315 | INFO | "malloc" : " 14 GB", | 20260505 18:57:14.315 | INFO | "pinned" : " 29 GB" | 20260505 18:57:14.315 | INFO | } | 20260505 18:57:14.315 | INFO | Perf report for the last 5.13491 seconds: | 20260505 18:57:14.315 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:57:14.315 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:14.315 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:14.315 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:14.315 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 18:57:14.315 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:14.315 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:14.315 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 18:57:14.315 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:14.315 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:14.315 | INFO | Uploaded 36GiB to GPU in 2.19246s (16.4199GiB/s) | 20260505 18:57:14.315 | INFO | Downloaded 34GiB to GPU in 2.02755s (16.769GiB/s) | 20260505 18:57:14.455 | INFO | Starting a round of Baz IO | 20260505 18:57:17.635 | INFO | Populating RTMetrics report | 20260505 18:57:17.635 | INFO | RTMetrics report populated | 20260505 18:57:17.635 | INFO | Opening tmp RTMetrics json file | 20260505 18:57:17.649 | INFO | Serializing RTMetrics to json | 20260505 18:57:17.668 | INFO | Serialization done, swapping tmp file now... | 20260505 18:57:17.671 | INFO | RTMetrics done | 20260505 18:57:17.732 | INFO | Finished a round of Baz IO: Wrote 2.071 GiB in 3.276 seconds | 20260505 18:57:17.732 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.84% / 0.00% / 78.16% | 20260505 18:57:17.732 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.228266% / 0% / 7.94729e-05% | 20260505 18:57:17.732 | INFO | Baz IO overall: | 20260505 18:57:17.732 | INFO | BytesWritten : 3249966234474 | 20260505 18:57:17.732 | INFO | NumEvents : 1538773959974 | 20260505 18:57:19.461 | INFO | Analyzing chunk frames = [5533696,5534208) | 20260505 18:57:19.461 | INFO | MemInfo: { | 20260505 18:57:19.461 | INFO | "gpu" : " 17 GB", | 20260505 18:57:19.461 | INFO | "huge" : "", | 20260505 18:57:19.461 | INFO | "huge_shared" : " 27 GB", | 20260505 18:57:19.461 | INFO | "malloc" : " 14 GB", | 20260505 18:57:19.461 | INFO | "pinned" : " 29 GB" | 20260505 18:57:19.461 | INFO | } | 20260505 18:57:19.461 | INFO | Perf report for the last 5.14618 seconds: | 20260505 18:57:19.461 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:57:19.461 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:19.461 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:19.461 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:19.461 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 18:57:19.461 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:19.461 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260505 18:57:19.461 | INFO | PRE_HQ : 60.1%, 1.8, 55.6ms, 0 | 20260505 18:57:19.461 | INFO | BAZWRITER : 63.1%, 3.8, 123.2ms, 0 | 20260505 18:57:19.462 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 18:57:19.462 | INFO | Uploaded 36GiB to GPU in 2.1658s (16.6221GiB/s) | 20260505 18:57:19.462 | INFO | Downloaded 38GiB to GPU in 2.25142s (16.8782GiB/s) | 20260505 18:57:24.609 | INFO | Analyzing chunk frames = [5534208,5534720) | 20260505 18:57:24.609 | INFO | MemInfo: { | 20260505 18:57:24.609 | INFO | "gpu" : " 17 GB", | 20260505 18:57:24.609 | INFO | "huge" : "", | 20260505 18:57:24.609 | INFO | "huge_shared" : " 27 GB", | 20260505 18:57:24.609 | INFO | "malloc" : " 14 GB", | 20260505 18:57:24.609 | INFO | "pinned" : " 29 GB" | 20260505 18:57:24.609 | INFO | } | 20260505 18:57:24.609 | INFO | Perf report for the last 5.14784 seconds: | 20260505 18:57:24.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:57:24.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:24.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:24.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:24.609 | INFO | ANALYSIS : 62.6%, 3.0, 97.9ms, 0 | 20260505 18:57:24.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:24.609 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:24.609 | INFO | PRE_HQ : 27.2%, 1.0, 14.2ms, 0 | 20260505 18:57:24.609 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:24.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:24.609 | INFO | Uploaded 36GiB to GPU in 2.15361s (16.7161GiB/s) | 20260505 18:57:24.609 | INFO | Downloaded 34GiB to GPU in 2.0464s (16.6146GiB/s) | 20260505 18:57:29.742 | INFO | Analyzing chunk frames = [5534720,5535232) | 20260505 18:57:29.742 | INFO | MemInfo: { | 20260505 18:57:29.742 | INFO | "gpu" : " 17 GB", | 20260505 18:57:29.742 | INFO | "huge" : "", | 20260505 18:57:29.742 | INFO | "huge_shared" : " 27 GB", | 20260505 18:57:29.742 | INFO | "malloc" : " 14 GB", | 20260505 18:57:29.742 | INFO | "pinned" : " 29 GB" | 20260505 18:57:29.742 | INFO | } | 20260505 18:57:29.742 | INFO | Perf report for the last 5.13267 seconds: | 20260505 18:57:29.742 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:57:29.742 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:29.742 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:29.742 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:29.742 | INFO | ANALYSIS : 62.2%, 3.0, 96.6ms, 0 | 20260505 18:57:29.742 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:29.742 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:29.742 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:57:29.742 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:29.742 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:29.742 | INFO | Uploaded 36GiB to GPU in 2.12943s (16.9059GiB/s) | 20260505 18:57:29.742 | INFO | Downloaded 34GiB to GPU in 2.01571s (16.8675GiB/s) | 20260505 18:57:34.871 | INFO | Analyzing chunk frames = [5535232,5535744) | 20260505 18:57:34.871 | INFO | MemInfo: { | 20260505 18:57:34.871 | INFO | "gpu" : " 17 GB", | 20260505 18:57:34.871 | INFO | "huge" : "", | 20260505 18:57:34.871 | INFO | "huge_shared" : " 27 GB", | 20260505 18:57:34.871 | INFO | "malloc" : " 14 GB", | 20260505 18:57:34.871 | INFO | "pinned" : " 29 GB" | 20260505 18:57:34.871 | INFO | } | 20260505 18:57:34.871 | INFO | Perf report for the last 5.12951 seconds: | 20260505 18:57:34.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:57:34.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:34.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:34.872 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:34.872 | INFO | ANALYSIS : 62.4%, 3.0, 96.8ms, 0 | 20260505 18:57:34.872 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:34.872 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:34.872 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:57:34.872 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:34.872 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:34.872 | INFO | Uploaded 36GiB to GPU in 2.17714s (16.5354GiB/s) | 20260505 18:57:34.872 | INFO | Downloaded 34GiB to GPU in 2.05799s (16.521GiB/s) | 20260505 18:57:40.015 | INFO | Analyzing chunk frames = [5535744,5536256) | 20260505 18:57:40.015 | INFO | MemInfo: { | 20260505 18:57:40.015 | INFO | "gpu" : " 17 GB", | 20260505 18:57:40.015 | INFO | "huge" : "", | 20260505 18:57:40.015 | INFO | "huge_shared" : " 27 GB", | 20260505 18:57:40.015 | INFO | "malloc" : " 14 GB", | 20260505 18:57:40.015 | INFO | "pinned" : " 29 GB" | 20260505 18:57:40.015 | INFO | } | 20260505 18:57:40.015 | INFO | Perf report for the last 5.14333 seconds: | 20260505 18:57:40.015 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:57:40.015 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:40.015 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:40.015 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:40.015 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 18:57:40.015 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:40.015 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:40.015 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 18:57:40.015 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:40.015 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:40.015 | INFO | Uploaded 36GiB to GPU in 2.15102s (16.7363GiB/s) | 20260505 18:57:40.015 | INFO | Downloaded 34GiB to GPU in 2.06136s (16.4939GiB/s) | 20260505 18:57:45.161 | INFO | Analyzing chunk frames = [5536256,5536768) | 20260505 18:57:45.161 | INFO | MemInfo: { | 20260505 18:57:45.161 | INFO | "gpu" : " 17 GB", | 20260505 18:57:45.161 | INFO | "huge" : "", | 20260505 18:57:45.161 | INFO | "huge_shared" : " 27 GB", | 20260505 18:57:45.161 | INFO | "malloc" : " 14 GB", | 20260505 18:57:45.161 | INFO | "pinned" : " 29 GB" | 20260505 18:57:45.161 | INFO | } | 20260505 18:57:45.161 | INFO | Perf report for the last 5.14654 seconds: | 20260505 18:57:45.161 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:57:45.161 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:45.161 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:45.161 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:45.161 | INFO | ANALYSIS : 62.3%, 3.0, 97.5ms, 0 | 20260505 18:57:45.161 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:45.161 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:45.161 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:57:45.161 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:45.161 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:45.162 | INFO | Uploaded 36GiB to GPU in 2.14616s (16.7741GiB/s) | 20260505 18:57:45.162 | INFO | Downloaded 34GiB to GPU in 2.04736s (16.6068GiB/s) | 20260505 18:57:50.304 | INFO | Analyzing chunk frames = [5536768,5537280) | 20260505 18:57:50.304 | INFO | MemInfo: { | 20260505 18:57:50.304 | INFO | "gpu" : " 17 GB", | 20260505 18:57:50.304 | INFO | "huge" : "", | 20260505 18:57:50.304 | INFO | "huge_shared" : " 27 GB", | 20260505 18:57:50.304 | INFO | "malloc" : " 14 GB", | 20260505 18:57:50.304 | INFO | "pinned" : " 29 GB" | 20260505 18:57:50.304 | INFO | } | 20260505 18:57:50.304 | INFO | Perf report for the last 5.14305 seconds: | 20260505 18:57:50.304 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:57:50.304 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:50.304 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:50.304 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:50.304 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 18:57:50.304 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:50.304 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:50.304 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 18:57:50.304 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:50.304 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:50.305 | INFO | Uploaded 36GiB to GPU in 2.14214s (16.8056GiB/s) | 20260505 18:57:50.305 | INFO | Downloaded 34GiB to GPU in 2.12601s (15.9924GiB/s) | 20260505 18:57:55.436 | INFO | Analyzing chunk frames = [5537280,5537792) | 20260505 18:57:55.436 | INFO | MemInfo: { | 20260505 18:57:55.436 | INFO | "gpu" : " 17 GB", | 20260505 18:57:55.436 | INFO | "huge" : "", | 20260505 18:57:55.436 | INFO | "huge_shared" : " 27 GB", | 20260505 18:57:55.436 | INFO | "malloc" : " 14 GB", | 20260505 18:57:55.436 | INFO | "pinned" : " 29 GB" | 20260505 18:57:55.436 | INFO | } | 20260505 18:57:55.436 | INFO | Perf report for the last 5.1318 seconds: | 20260505 18:57:55.436 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:57:55.436 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:55.436 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:55.436 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:55.436 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260505 18:57:55.436 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:55.436 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:55.436 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 18:57:55.436 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:55.436 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:57:55.436 | INFO | Uploaded 36GiB to GPU in 2.1174s (17.002GiB/s) | 20260505 18:57:55.436 | INFO | Downloaded 34GiB to GPU in 2.00405s (16.9656GiB/s) | 20260505 18:57:55.437 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 18:57:55.437 | INFO | Total mem usage: 15.3 GiB | 20260505 18:57:55.437 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 18:57:55.437 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.14%) | 20260505 18:57:55.437 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 18:57:55.437 | INFO | High Water Breakdown: | 20260505 18:57:55.437 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 18:57:55.437 | INFO | 38.9% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 18:57:55.437 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 18:57:55.437 | INFO | ------------------------------------------ | 20260505 18:57:55.437 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:57:55.437 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 18:57:55.437 | INFO | Total mem usage: 26.5 GiB | 20260505 18:57:55.437 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 18:57:55.437 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 18:57:55.437 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 18:57:55.437 | INFO | High Water Breakdown: | 20260505 18:57:55.437 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:57:55.437 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:57:55.437 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:57:55.437 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:57:55.437 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:57:55.437 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:57:55.437 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 18:57:55.437 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 18:57:55.437 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:57:55.437 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 18:57:55.437 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 18:57:55.437 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 18:57:55.437 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 18:57:55.437 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 18:57:55.437 | INFO | ------------------------------------------ | 20260505 18:57:55.437 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:57:55.437 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 18:57:55.437 | INFO | Total mem usage: 25.6 GiB | 20260505 18:57:55.437 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 18:57:55.437 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 18:57:55.437 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 18:57:55.437 | INFO | High Water Breakdown: | 20260505 18:57:55.437 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 18:57:55.437 | INFO | ------------------------------------------ | 20260505 18:57:55.437 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:57:55.437 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 18:57:55.437 | INFO | Total mem usage: 15.7 GiB | 20260505 18:57:55.437 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 18:57:55.437 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 18:57:55.437 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 18:57:55.437 | INFO | High Water Breakdown: | 20260505 18:57:55.437 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 18:57:55.437 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 18:57:55.437 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 18:57:55.437 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 18:57:55.437 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 18:57:55.437 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 18:57:55.437 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 18:57:55.437 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 18:57:55.437 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 18:57:55.437 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:57:55.437 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 18:57:55.437 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:57:55.438 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 18:57:55.438 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 18:57:55.438 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 18:57:55.438 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 18:57:55.438 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:57:55.438 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 18:57:55.438 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:57:55.438 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:57:55.438 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:57:55.438 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 18:57:55.438 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 18:57:55.438 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:57:55.438 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 18:57:55.438 | INFO | ------------------------------------------ | 20260505 18:57:55.438 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:57:55.438 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 18:57:55.438 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:57:55.438 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:57:55.438 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.45e+01, 1.35e+01 / 1.64e+00 | 20260505 18:57:55.438 | INFO | PerfReport: 0.29%, AggregateStats -- 9.27e-02 / 1.03e-01, 1.11e-02 / 3.24e-03 | 20260505 18:57:55.438 | INFO | PerfReport: 11.77%, Baseline -- 3.71e+00 / 4.52e+00, 5.66e-01 / 1.56e-01 | 20260505 18:57:55.438 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.51e+00, 1.90e-01 / 1.06e-01 | 20260505 18:57:55.438 | INFO | PerfReport: 15.93%, DME -- 4.02e+01 / 4.44e+01, 4.03e+00 / 1.63e+00 | 20260505 18:57:55.438 | INFO | PerfReport: 57.01%, FrameLabeling -- 1.80e+01 / 1.93e+01, 1.51e+00 / 1.10e+00 | 20260505 18:57:55.438 | INFO | PerfReport: 7.27%, PulseAccumulating -- 2.29e+00 / 2.89e+00, 4.39e-01 / 1.89e-01 | 20260505 18:57:55.438 | INFO | PerfReport: 3.30%, Metrics -- 1.04e+00 / 1.36e+00, 2.85e-01 / 1.26e-01 | 20260505 18:57:55.438 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.64e-01 / 4.35e-01, 6.47e-02 / 3.83e-02 | 20260505 18:57:55.438 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 18:57:55.438 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:57:55.438 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:57:55.438 | INFO | PerfReport: Full Execution -- 9.97e+01 / -nan, 2.56e+01 / 0.00e+00 | 20260505 18:57:55.438 | INFO | PerfReport: 22.48%, Upload -- 2.24e+01 / 0.00e+00, 7.79e+00 / 0.00e+00 | 20260505 18:57:55.438 | INFO | PerfReport: 21.78%, Download -- 2.17e+01 / 0.00e+00, 7.73e+00 / 0.00e+00 | 20260505 18:57:55.438 | INFO | PerfReport: 31.65%, Compute -- 3.16e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 18:57:55.438 | INFO | PerfReport: 2.35%, LockUpload -- 2.35e+00 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260505 18:57:55.438 | INFO | PerfReport: 1.33%, LockDownload -- 1.32e+00 / 0.00e+00, 2.84e+00 / 0.00e+00 | 20260505 18:57:55.438 | INFO | PerfReport: 20.39%, LockCompute -- 2.03e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 18:57:55.438 | INFO | PerfReport: 0.02%, Misc -- 1.50e-02 / 0.00e+00, 2.93e-02 / 0.00e+00 | 20260505 18:57:55.572 | INFO | Starting a round of Baz IO | 20260505 18:57:58.745 | INFO | Populating RTMetrics report | 20260505 18:57:58.746 | INFO | RTMetrics report populated | 20260505 18:57:58.746 | INFO | Opening tmp RTMetrics json file | 20260505 18:57:58.750 | INFO | Serializing RTMetrics to json | 20260505 18:57:58.752 | INFO | Serialization done, swapping tmp file now... | 20260505 18:57:58.754 | INFO | RTMetrics done | 20260505 18:57:58.837 | INFO | Finished a round of Baz IO: Wrote 2.071 GiB in 3.265 seconds | 20260505 18:57:58.837 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.84% / 0.00% / 78.16% | 20260505 18:57:58.837 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.228075% / 3.97364e-06% / 7.54992e-05% | 20260505 18:57:58.837 | INFO | Baz IO overall: | 20260505 18:57:58.837 | INFO | BytesWritten : 3252189591228 | 20260505 18:57:58.837 | INFO | NumEvents : 1539808849026 | 20260505 18:58:00.573 | INFO | Analyzing chunk frames = [5537792,5538304) | 20260505 18:58:00.573 | INFO | MemInfo: { | 20260505 18:58:00.573 | INFO | "gpu" : " 17 GB", | 20260505 18:58:00.573 | INFO | "huge" : "", | 20260505 18:58:00.573 | INFO | "huge_shared" : " 27 GB", | 20260505 18:58:00.573 | INFO | "malloc" : " 14 GB", | 20260505 18:58:00.573 | INFO | "pinned" : " 29 GB" | 20260505 18:58:00.573 | INFO | } | 20260505 18:58:00.573 | INFO | Perf report for the last 5.13682 seconds: | 20260505 18:58:00.573 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:58:00.573 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:00.573 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:00.573 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:00.573 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 18:58:00.573 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:00.573 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260505 18:58:00.573 | INFO | PRE_HQ : 60.4%, 1.8, 55.8ms, 0 | 20260505 18:58:00.573 | INFO | BAZWRITER : 63.1%, 3.8, 122.9ms, 0 | 20260505 18:58:00.573 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 18:58:00.573 | INFO | Uploaded 36GiB to GPU in 2.16759s (16.6083GiB/s) | 20260505 18:58:00.573 | INFO | Downloaded 38GiB to GPU in 2.23803s (16.9792GiB/s) | 20260505 18:58:05.719 | INFO | Analyzing chunk frames = [5538304,5538816) | 20260505 18:58:05.719 | INFO | MemInfo: { | 20260505 18:58:05.719 | INFO | "gpu" : " 17 GB", | 20260505 18:58:05.719 | INFO | "huge" : "", | 20260505 18:58:05.719 | INFO | "huge_shared" : " 27 GB", | 20260505 18:58:05.719 | INFO | "malloc" : " 14 GB", | 20260505 18:58:05.719 | INFO | "pinned" : " 29 GB" | 20260505 18:58:05.719 | INFO | } | 20260505 18:58:05.719 | INFO | Perf report for the last 5.14636 seconds: | 20260505 18:58:05.719 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:58:05.719 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:05.719 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:05.719 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:05.719 | INFO | ANALYSIS : 62.7%, 3.0, 98.0ms, 0 | 20260505 18:58:05.719 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:05.719 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:05.719 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 18:58:05.719 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:05.719 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:05.720 | INFO | Uploaded 36GiB to GPU in 2.13518s (16.8604GiB/s) | 20260505 18:58:05.720 | INFO | Downloaded 34GiB to GPU in 2.02322s (16.8049GiB/s) | 20260505 18:58:10.856 | INFO | Analyzing chunk frames = [5538816,5539328) | 20260505 18:58:10.856 | INFO | MemInfo: { | 20260505 18:58:10.856 | INFO | "gpu" : " 17 GB", | 20260505 18:58:10.856 | INFO | "huge" : "", | 20260505 18:58:10.856 | INFO | "huge_shared" : " 27 GB", | 20260505 18:58:10.856 | INFO | "malloc" : " 14 GB", | 20260505 18:58:10.856 | INFO | "pinned" : " 29 GB" | 20260505 18:58:10.856 | INFO | } | 20260505 18:58:10.856 | INFO | Perf report for the last 5.13658 seconds: | 20260505 18:58:10.856 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:58:10.856 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:10.856 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:10.856 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:10.856 | INFO | ANALYSIS : 62.4%, 3.0, 97.0ms, 0 | 20260505 18:58:10.856 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:10.856 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:10.856 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:58:10.856 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:10.856 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:10.856 | INFO | Uploaded 36GiB to GPU in 2.13194s (16.886GiB/s) | 20260505 18:58:10.856 | INFO | Downloaded 34GiB to GPU in 2.03308s (16.7234GiB/s) | 20260505 18:58:15.988 | INFO | Analyzing chunk frames = [5539328,5539840) | 20260505 18:58:15.988 | INFO | MemInfo: { | 20260505 18:58:15.988 | INFO | "gpu" : " 17 GB", | 20260505 18:58:15.988 | INFO | "huge" : "", | 20260505 18:58:15.988 | INFO | "huge_shared" : " 27 GB", | 20260505 18:58:15.988 | INFO | "malloc" : " 14 GB", | 20260505 18:58:15.988 | INFO | "pinned" : " 29 GB" | 20260505 18:58:15.988 | INFO | } | 20260505 18:58:15.988 | INFO | Perf report for the last 5.13247 seconds: | 20260505 18:58:15.988 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:58:15.988 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:15.988 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:15.988 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:15.988 | INFO | ANALYSIS : 62.3%, 3.0, 96.7ms, 0 | 20260505 18:58:15.988 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:15.988 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:15.988 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:58:15.988 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:15.988 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:15.989 | INFO | Uploaded 36GiB to GPU in 2.17936s (16.5186GiB/s) | 20260505 18:58:15.989 | INFO | Downloaded 34GiB to GPU in 2.05296s (16.5615GiB/s) | 20260505 18:58:21.130 | INFO | Analyzing chunk frames = [5539840,5540352) | 20260505 18:58:21.130 | INFO | MemInfo: { | 20260505 18:58:21.130 | INFO | "gpu" : " 17 GB", | 20260505 18:58:21.130 | INFO | "huge" : "", | 20260505 18:58:21.130 | INFO | "huge_shared" : " 27 GB", | 20260505 18:58:21.130 | INFO | "malloc" : " 14 GB", | 20260505 18:58:21.130 | INFO | "pinned" : " 29 GB" | 20260505 18:58:21.130 | INFO | } | 20260505 18:58:21.130 | INFO | Perf report for the last 5.14178 seconds: | 20260505 18:58:21.130 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:58:21.130 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:21.130 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:21.130 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:21.130 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260505 18:58:21.130 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:21.130 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:21.130 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 18:58:21.130 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:21.130 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:21.130 | INFO | Uploaded 36GiB to GPU in 2.15942s (16.6711GiB/s) | 20260505 18:58:21.131 | INFO | Downloaded 34GiB to GPU in 2.06039s (16.5017GiB/s) | 20260505 18:58:26.268 | INFO | Analyzing chunk frames = [5540352,5540864) | 20260505 18:58:26.268 | INFO | MemInfo: { | 20260505 18:58:26.268 | INFO | "gpu" : " 17 GB", | 20260505 18:58:26.268 | INFO | "huge" : "", | 20260505 18:58:26.268 | INFO | "huge_shared" : " 27 GB", | 20260505 18:58:26.268 | INFO | "malloc" : " 14 GB", | 20260505 18:58:26.268 | INFO | "pinned" : " 29 GB" | 20260505 18:58:26.268 | INFO | } | 20260505 18:58:26.268 | INFO | Perf report for the last 5.1382 seconds: | 20260505 18:58:26.268 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:58:26.268 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:26.268 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:26.268 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:26.268 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 18:58:26.268 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:26.269 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:26.269 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 18:58:26.269 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:26.269 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:26.269 | INFO | Uploaded 36GiB to GPU in 2.14068s (16.8171GiB/s) | 20260505 18:58:26.269 | INFO | Downloaded 34GiB to GPU in 2.04123s (16.6567GiB/s) | 20260505 18:58:31.413 | INFO | Analyzing chunk frames = [5540864,5541376) | 20260505 18:58:31.413 | INFO | MemInfo: { | 20260505 18:58:31.413 | INFO | "gpu" : " 17 GB", | 20260505 18:58:31.413 | INFO | "huge" : "", | 20260505 18:58:31.413 | INFO | "huge_shared" : " 27 GB", | 20260505 18:58:31.413 | INFO | "malloc" : " 14 GB", | 20260505 18:58:31.413 | INFO | "pinned" : " 29 GB" | 20260505 18:58:31.413 | INFO | } | 20260505 18:58:31.414 | INFO | Perf report for the last 5.14511 seconds: | 20260505 18:58:31.414 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:58:31.414 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:31.414 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:31.414 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:31.414 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 18:58:31.414 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:31.414 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:31.414 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 18:58:31.414 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:31.414 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:31.414 | INFO | Uploaded 36GiB to GPU in 2.1476s (16.7629GiB/s) | 20260505 18:58:31.414 | INFO | Downloaded 34GiB to GPU in 2.14799s (15.8288GiB/s) | 20260505 18:58:36.558 | INFO | Analyzing chunk frames = [5541376,5541888) | 20260505 18:58:36.558 | INFO | MemInfo: { | 20260505 18:58:36.558 | INFO | "gpu" : " 17 GB", | 20260505 18:58:36.558 | INFO | "huge" : "", | 20260505 18:58:36.558 | INFO | "huge_shared" : " 27 GB", | 20260505 18:58:36.559 | INFO | "malloc" : " 14 GB", | 20260505 18:58:36.559 | INFO | "pinned" : " 29 GB" | 20260505 18:58:36.559 | INFO | } | 20260505 18:58:36.559 | INFO | Perf report for the last 5.14523 seconds: | 20260505 18:58:36.559 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:58:36.559 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:36.559 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:36.559 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:36.559 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 18:58:36.559 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:36.559 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:36.559 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 18:58:36.559 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:36.559 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:36.559 | INFO | Uploaded 36GiB to GPU in 2.2033s (16.3391GiB/s) | 20260505 18:58:36.559 | INFO | Downloaded 34GiB to GPU in 2.05035s (16.5825GiB/s) | 20260505 18:58:36.692 | INFO | Starting a round of Baz IO | 20260505 18:58:39.863 | INFO | Populating RTMetrics report | 20260505 18:58:39.863 | INFO | RTMetrics report populated | 20260505 18:58:39.863 | INFO | Opening tmp RTMetrics json file | 20260505 18:58:39.864 | INFO | Serializing RTMetrics to json | 20260505 18:58:39.868 | INFO | Serialization done, swapping tmp file now... | 20260505 18:58:39.869 | INFO | RTMetrics done | 20260505 18:58:39.954 | INFO | Finished a round of Baz IO: Wrote 2.07 GiB in 3.261 seconds | 20260505 18:58:39.954 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.84% / 0.00% / 78.16% | 20260505 18:58:39.954 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.227789% / 0% / 6.75519e-05% | 20260505 18:58:39.954 | INFO | Baz IO overall: | 20260505 18:58:39.954 | INFO | BytesWritten : 3254412736323 | 20260505 18:58:39.954 | INFO | NumEvents : 1540843716434 | 20260505 18:58:41.689 | INFO | Analyzing chunk frames = [5541888,5542400) | 20260505 18:58:41.689 | INFO | MemInfo: { | 20260505 18:58:41.689 | INFO | "gpu" : " 17 GB", | 20260505 18:58:41.689 | INFO | "huge" : "", | 20260505 18:58:41.689 | INFO | "huge_shared" : " 27 GB", | 20260505 18:58:41.689 | INFO | "malloc" : " 14 GB", | 20260505 18:58:41.689 | INFO | "pinned" : " 29 GB" | 20260505 18:58:41.689 | INFO | } | 20260505 18:58:41.690 | INFO | Perf report for the last 5.13076 seconds: | 20260505 18:58:41.690 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:58:41.690 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:41.690 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:41.690 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:41.690 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 18:58:41.690 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:41.690 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260505 18:58:41.690 | INFO | PRE_HQ : 60.2%, 1.8, 55.4ms, 0 | 20260505 18:58:41.690 | INFO | BAZWRITER : 63.0%, 3.7, 120.4ms, 0 | 20260505 18:58:41.690 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 18:58:41.690 | INFO | Uploaded 36GiB to GPU in 2.17449s (16.5556GiB/s) | 20260505 18:58:41.690 | INFO | Downloaded 38GiB to GPU in 2.25629s (16.8418GiB/s) | 20260505 18:58:46.825 | INFO | Analyzing chunk frames = [5542400,5542912) | 20260505 18:58:46.825 | INFO | MemInfo: { | 20260505 18:58:46.825 | INFO | "gpu" : " 17 GB", | 20260505 18:58:46.825 | INFO | "huge" : "", | 20260505 18:58:46.825 | INFO | "huge_shared" : " 27 GB", | 20260505 18:58:46.825 | INFO | "malloc" : " 14 GB", | 20260505 18:58:46.825 | INFO | "pinned" : " 29 GB" | 20260505 18:58:46.825 | INFO | } | 20260505 18:58:46.826 | INFO | Perf report for the last 5.13599 seconds: | 20260505 18:58:46.826 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:58:46.826 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:46.826 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:46.826 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:46.826 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260505 18:58:46.826 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:46.826 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:46.826 | INFO | PRE_HQ : 27.5%, 1.0, 14.2ms, 0 | 20260505 18:58:46.826 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:46.826 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:46.826 | INFO | Uploaded 36GiB to GPU in 2.11723s (17.0034GiB/s) | 20260505 18:58:46.826 | INFO | Downloaded 34GiB to GPU in 2.02889s (16.7579GiB/s) | 20260505 18:58:51.971 | INFO | Analyzing chunk frames = [5542912,5543424) | 20260505 18:58:51.971 | INFO | MemInfo: { | 20260505 18:58:51.971 | INFO | "gpu" : " 17 GB", | 20260505 18:58:51.971 | INFO | "huge" : "", | 20260505 18:58:51.971 | INFO | "huge_shared" : " 27 GB", | 20260505 18:58:51.971 | INFO | "malloc" : " 14 GB", | 20260505 18:58:51.971 | INFO | "pinned" : " 29 GB" | 20260505 18:58:51.971 | INFO | } | 20260505 18:58:51.971 | INFO | Perf report for the last 5.14561 seconds: | 20260505 18:58:51.971 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:58:51.971 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:51.971 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:51.971 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:51.971 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 18:58:51.971 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:51.971 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:51.971 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 18:58:51.971 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:51.971 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:51.972 | INFO | Uploaded 36GiB to GPU in 2.20629s (16.317GiB/s) | 20260505 18:58:51.972 | INFO | Downloaded 34GiB to GPU in 2.04922s (16.5917GiB/s) | 20260505 18:58:57.116 | INFO | Analyzing chunk frames = [5543424,5543936) | 20260505 18:58:57.116 | INFO | MemInfo: { | 20260505 18:58:57.116 | INFO | "gpu" : " 17 GB", | 20260505 18:58:57.116 | INFO | "huge" : "", | 20260505 18:58:57.116 | INFO | "huge_shared" : " 27 GB", | 20260505 18:58:57.116 | INFO | "malloc" : " 14 GB", | 20260505 18:58:57.116 | INFO | "pinned" : " 29 GB" | 20260505 18:58:57.116 | INFO | } | 20260505 18:58:57.116 | INFO | Perf report for the last 5.14464 seconds: | 20260505 18:58:57.116 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:58:57.116 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:57.116 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:57.116 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:57.116 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260505 18:58:57.116 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:57.116 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:57.116 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 18:58:57.116 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:57.116 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:58:57.116 | INFO | Uploaded 36GiB to GPU in 2.18693s (16.4614GiB/s) | 20260505 18:58:57.116 | INFO | Downloaded 34GiB to GPU in 2.05836s (16.518GiB/s) | 20260505 18:59:02.256 | INFO | Analyzing chunk frames = [5543936,5544448) | 20260505 18:59:02.256 | INFO | MemInfo: { | 20260505 18:59:02.256 | INFO | "gpu" : " 17 GB", | 20260505 18:59:02.256 | INFO | "huge" : "", | 20260505 18:59:02.256 | INFO | "huge_shared" : " 27 GB", | 20260505 18:59:02.256 | INFO | "malloc" : " 14 GB", | 20260505 18:59:02.256 | INFO | "pinned" : " 29 GB" | 20260505 18:59:02.256 | INFO | } | 20260505 18:59:02.256 | INFO | Perf report for the last 5.14034 seconds: | 20260505 18:59:02.256 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:59:02.256 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:02.256 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:02.256 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:02.256 | INFO | ANALYSIS : 61.8%, 3.0, 96.6ms, 0 | 20260505 18:59:02.256 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:02.256 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:02.256 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:59:02.257 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:02.257 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:02.257 | INFO | Uploaded 36GiB to GPU in 2.13918s (16.8289GiB/s) | 20260505 18:59:02.257 | INFO | Downloaded 34GiB to GPU in 2.0656s (16.4601GiB/s) | 20260505 18:59:07.385 | INFO | Analyzing chunk frames = [5544448,5544960) | 20260505 18:59:07.385 | INFO | MemInfo: { | 20260505 18:59:07.385 | INFO | "gpu" : " 17 GB", | 20260505 18:59:07.385 | INFO | "huge" : "", | 20260505 18:59:07.385 | INFO | "huge_shared" : " 27 GB", | 20260505 18:59:07.385 | INFO | "malloc" : " 14 GB", | 20260505 18:59:07.385 | INFO | "pinned" : " 29 GB" | 20260505 18:59:07.385 | INFO | } | 20260505 18:59:07.385 | INFO | Perf report for the last 5.12887 seconds: | 20260505 18:59:07.385 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:59:07.385 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:07.385 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:07.385 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:07.385 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260505 18:59:07.385 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:07.385 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:07.385 | INFO | PRE_HQ : 27.5%, 1.0, 14.2ms, 0 | 20260505 18:59:07.385 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:07.385 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:07.386 | INFO | Uploaded 36GiB to GPU in 2.14557s (16.7788GiB/s) | 20260505 18:59:07.386 | INFO | Downloaded 34GiB to GPU in 2.05609s (16.5362GiB/s) | 20260505 18:59:12.527 | INFO | Analyzing chunk frames = [5544960,5545472) | 20260505 18:59:12.527 | INFO | MemInfo: { | 20260505 18:59:12.527 | INFO | "gpu" : " 17 GB", | 20260505 18:59:12.527 | INFO | "huge" : "", | 20260505 18:59:12.527 | INFO | "huge_shared" : " 27 GB", | 20260505 18:59:12.527 | INFO | "malloc" : " 14 GB", | 20260505 18:59:12.527 | INFO | "pinned" : " 29 GB" | 20260505 18:59:12.527 | INFO | } | 20260505 18:59:12.527 | INFO | Perf report for the last 5.14203 seconds: | 20260505 18:59:12.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:59:12.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:12.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:12.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:12.527 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 18:59:12.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:12.527 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:12.527 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 18:59:12.527 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:12.527 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:12.527 | INFO | Uploaded 36GiB to GPU in 2.13108s (16.8928GiB/s) | 20260505 18:59:12.527 | INFO | Downloaded 34GiB to GPU in 2.14995s (15.8143GiB/s) | 20260505 18:59:17.666 | INFO | Analyzing chunk frames = [5545472,5545984) | 20260505 18:59:17.666 | INFO | MemInfo: { | 20260505 18:59:17.666 | INFO | "gpu" : " 17 GB", | 20260505 18:59:17.666 | INFO | "huge" : "", | 20260505 18:59:17.667 | INFO | "huge_shared" : " 27 GB", | 20260505 18:59:17.667 | INFO | "malloc" : " 14 GB", | 20260505 18:59:17.667 | INFO | "pinned" : " 29 GB" | 20260505 18:59:17.667 | INFO | } | 20260505 18:59:17.667 | INFO | Perf report for the last 5.13968 seconds: | 20260505 18:59:17.667 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:59:17.667 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:17.667 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:17.667 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:17.667 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 18:59:17.667 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:17.667 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:17.667 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 18:59:17.667 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:17.667 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:17.667 | INFO | Uploaded 36GiB to GPU in 2.20081s (16.3576GiB/s) | 20260505 18:59:17.667 | INFO | Downloaded 34GiB to GPU in 2.04166s (16.6531GiB/s) | 20260505 18:59:17.667 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 18:59:17.667 | INFO | Total mem usage: 15.3 GiB | 20260505 18:59:17.667 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 18:59:17.667 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.14%) | 20260505 18:59:17.667 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 18:59:17.667 | INFO | High Water Breakdown: | 20260505 18:59:17.667 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 18:59:17.667 | INFO | 38.9% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 18:59:17.667 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 18:59:17.667 | INFO | ------------------------------------------ | 20260505 18:59:17.667 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:59:17.667 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 18:59:17.667 | INFO | Total mem usage: 26.5 GiB | 20260505 18:59:17.667 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 18:59:17.667 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 18:59:17.667 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 18:59:17.667 | INFO | High Water Breakdown: | 20260505 18:59:17.667 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:59:17.667 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:59:17.667 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:59:17.667 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:59:17.667 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:59:17.667 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:59:17.667 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 18:59:17.667 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 18:59:17.667 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:59:17.667 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 18:59:17.667 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 18:59:17.667 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 18:59:17.667 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 18:59:17.668 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 18:59:17.668 | INFO | ------------------------------------------ | 20260505 18:59:17.668 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:59:17.668 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 18:59:17.668 | INFO | Total mem usage: 25.6 GiB | 20260505 18:59:17.668 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 18:59:17.668 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 18:59:17.668 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 18:59:17.668 | INFO | High Water Breakdown: | 20260505 18:59:17.668 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 18:59:17.668 | INFO | ------------------------------------------ | 20260505 18:59:17.668 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:59:17.668 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 18:59:17.668 | INFO | Total mem usage: 15.7 GiB | 20260505 18:59:17.668 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 18:59:17.668 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 18:59:17.668 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 18:59:17.668 | INFO | High Water Breakdown: | 20260505 18:59:17.668 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 18:59:17.668 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 18:59:17.668 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 18:59:17.668 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 18:59:17.668 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 18:59:17.668 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 18:59:17.668 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 18:59:17.668 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 18:59:17.668 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 18:59:17.668 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 18:59:17.668 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 18:59:17.668 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 18:59:17.668 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 18:59:17.668 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 18:59:17.668 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 18:59:17.668 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 18:59:17.668 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 18:59:17.668 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 18:59:17.668 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 18:59:17.668 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 18:59:17.668 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 18:59:17.668 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 18:59:17.668 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 18:59:17.668 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 18:59:17.668 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 18:59:17.668 | INFO | ------------------------------------------ | 20260505 18:59:17.668 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 18:59:17.668 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 18:59:17.668 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:59:17.668 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:59:17.668 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.41e+01, 1.35e+01 / 1.51e+00 | 20260505 18:59:17.668 | INFO | PerfReport: 0.29%, AggregateStats -- 9.24e-02 / 1.03e-01, 1.07e-02 / 4.37e-03 | 20260505 18:59:17.668 | INFO | PerfReport: 11.82%, Baseline -- 3.72e+00 / 4.55e+00, 5.74e-01 / 1.09e-01 | 20260505 18:59:17.668 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.55e+00, 1.90e-01 / 5.57e-02 | 20260505 18:59:17.668 | INFO | PerfReport: 15.91%, DME -- 4.01e+01 / 4.41e+01, 4.03e+00 / 1.66e+00 | 20260505 18:59:17.668 | INFO | PerfReport: 56.95%, FrameLabeling -- 1.79e+01 / 1.92e+01, 1.45e+00 / 8.75e-01 | 20260505 18:59:17.668 | INFO | PerfReport: 7.30%, PulseAccumulating -- 2.30e+00 / 2.90e+00, 4.45e-01 / 1.55e-01 | 20260505 18:59:17.668 | INFO | PerfReport: 3.31%, Metrics -- 1.04e+00 / 1.37e+00, 2.89e-01 / 1.63e-01 | 20260505 18:59:17.668 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.62e-01 / 4.27e-01, 6.39e-02 / 3.64e-02 | 20260505 18:59:17.668 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 18:59:17.668 | INFO | PerfReport: Gathered 1536 samples. | 20260505 18:59:17.668 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 18:59:17.668 | INFO | PerfReport: Full Execution -- 9.97e+01 / -nan, 2.57e+01 / 0.00e+00 | 20260505 18:59:17.668 | INFO | PerfReport: 22.57%, Upload -- 2.25e+01 / 0.00e+00, 7.84e+00 / 0.00e+00 | 20260505 18:59:17.668 | INFO | PerfReport: 21.78%, Download -- 2.17e+01 / 0.00e+00, 7.74e+00 / 0.00e+00 | 20260505 18:59:17.668 | INFO | PerfReport: 31.61%, Compute -- 3.15e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 18:59:17.668 | INFO | PerfReport: 2.31%, LockUpload -- 2.30e+00 / 0.00e+00, 8.02e+00 / 0.00e+00 | 20260505 18:59:17.668 | INFO | PerfReport: 1.39%, LockDownload -- 1.39e+00 / 0.00e+00, 2.87e+00 / 0.00e+00 | 20260505 18:59:17.668 | INFO | PerfReport: 20.33%, LockCompute -- 2.03e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260505 18:59:17.668 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.91e-02 / 0.00e+00 | 20260505 18:59:17.800 | INFO | Starting a round of Baz IO | 20260505 18:59:20.976 | INFO | Populating RTMetrics report | 20260505 18:59:20.977 | INFO | RTMetrics report populated | 20260505 18:59:20.977 | INFO | Opening tmp RTMetrics json file | 20260505 18:59:20.978 | INFO | Serializing RTMetrics to json | 20260505 18:59:20.979 | INFO | Serialization done, swapping tmp file now... | 20260505 18:59:20.979 | INFO | RTMetrics done | 20260505 18:59:21.268 | INFO | Finished a round of Baz IO: Wrote 2.068 GiB in 3.468 seconds | 20260505 18:59:21.268 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.83% / 0.00% / 78.16% | 20260505 18:59:21.268 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.226156% / 3.97364e-06% / 7.94729e-05% | 20260505 18:59:21.268 | INFO | Baz IO overall: | 20260505 18:59:21.268 | INFO | BytesWritten : 3256633723916 | 20260505 18:59:21.268 | INFO | NumEvents : 1541876869219 | 20260505 18:59:22.808 | INFO | Analyzing chunk frames = [5545984,5546496) | 20260505 18:59:22.809 | INFO | MemInfo: { | 20260505 18:59:22.809 | INFO | "gpu" : " 17 GB", | 20260505 18:59:22.809 | INFO | "huge" : "", | 20260505 18:59:22.809 | INFO | "huge_shared" : " 27 GB", | 20260505 18:59:22.809 | INFO | "malloc" : " 14 GB", | 20260505 18:59:22.809 | INFO | "pinned" : " 29 GB" | 20260505 18:59:22.809 | INFO | } | 20260505 18:59:22.809 | INFO | Perf report for the last 5.1418 seconds: | 20260505 18:59:22.809 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:59:22.809 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:22.809 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:22.809 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:22.809 | INFO | ANALYSIS : 63.3%, 2.9, 97.7ms, 0 | 20260505 18:59:22.809 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:22.809 | INFO | RT_METRICS : 44.3%, 1.0, 23.3ms, 0 | 20260505 18:59:22.809 | INFO | PRE_HQ : 60.4%, 1.8, 55.8ms, 0 | 20260505 18:59:22.809 | INFO | BAZWRITER : 67.0%, 4.0, 138.0ms, 0 | 20260505 18:59:22.809 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 18:59:22.809 | INFO | Uploaded 36GiB to GPU in 2.18225s (16.4968GiB/s) | 20260505 18:59:22.809 | INFO | Downloaded 38GiB to GPU in 2.26098s (16.8069GiB/s) | 20260505 18:59:27.944 | INFO | Analyzing chunk frames = [5546496,5547008) | 20260505 18:59:27.944 | INFO | MemInfo: { | 20260505 18:59:27.944 | INFO | "gpu" : " 17 GB", | 20260505 18:59:27.944 | INFO | "huge" : "", | 20260505 18:59:27.944 | INFO | "huge_shared" : " 27 GB", | 20260505 18:59:27.944 | INFO | "malloc" : " 14 GB", | 20260505 18:59:27.944 | INFO | "pinned" : " 29 GB" | 20260505 18:59:27.944 | INFO | } | 20260505 18:59:27.945 | INFO | Perf report for the last 5.13595 seconds: | 20260505 18:59:27.945 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:59:27.945 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:27.945 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:27.945 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:27.945 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 18:59:27.945 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:27.945 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:27.945 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 18:59:27.945 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:27.945 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:27.945 | INFO | Uploaded 36GiB to GPU in 2.13764s (16.841GiB/s) | 20260505 18:59:27.945 | INFO | Downloaded 34GiB to GPU in 2.03364s (16.7188GiB/s) | 20260505 18:59:33.075 | INFO | Analyzing chunk frames = [5547008,5547520) | 20260505 18:59:33.075 | INFO | MemInfo: { | 20260505 18:59:33.075 | INFO | "gpu" : " 17 GB", | 20260505 18:59:33.075 | INFO | "huge" : "", | 20260505 18:59:33.075 | INFO | "huge_shared" : " 27 GB", | 20260505 18:59:33.075 | INFO | "malloc" : " 14 GB", | 20260505 18:59:33.075 | INFO | "pinned" : " 29 GB" | 20260505 18:59:33.075 | INFO | } | 20260505 18:59:33.075 | INFO | Perf report for the last 5.13079 seconds: | 20260505 18:59:33.075 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:59:33.075 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:33.075 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:33.075 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:33.075 | INFO | ANALYSIS : 62.8%, 3.0, 97.6ms, 0 | 20260505 18:59:33.075 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:33.075 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:33.075 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 18:59:33.076 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:33.076 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:33.076 | INFO | Uploaded 36GiB to GPU in 2.20767s (16.3068GiB/s) | 20260505 18:59:33.076 | INFO | Downloaded 34GiB to GPU in 2.05791s (16.5217GiB/s) | 20260505 18:59:38.225 | INFO | Analyzing chunk frames = [5547520,5548032) | 20260505 18:59:38.225 | INFO | MemInfo: { | 20260505 18:59:38.225 | INFO | "gpu" : " 17 GB", | 20260505 18:59:38.225 | INFO | "huge" : "", | 20260505 18:59:38.225 | INFO | "huge_shared" : " 27 GB", | 20260505 18:59:38.225 | INFO | "malloc" : " 14 GB", | 20260505 18:59:38.225 | INFO | "pinned" : " 29 GB" | 20260505 18:59:38.225 | INFO | } | 20260505 18:59:38.225 | INFO | Perf report for the last 5.14992 seconds: | 20260505 18:59:38.225 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:59:38.225 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:38.225 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:38.225 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:38.225 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260505 18:59:38.225 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:38.225 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:38.225 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 18:59:38.225 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:38.225 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:38.226 | INFO | Uploaded 36GiB to GPU in 2.16879s (16.5991GiB/s) | 20260505 18:59:38.226 | INFO | Downloaded 34GiB to GPU in 2.1434s (15.8626GiB/s) | 20260505 18:59:43.371 | INFO | Analyzing chunk frames = [5548032,5548544) | 20260505 18:59:43.371 | INFO | MemInfo: { | 20260505 18:59:43.371 | INFO | "gpu" : " 17 GB", | 20260505 18:59:43.371 | INFO | "huge" : "", | 20260505 18:59:43.371 | INFO | "huge_shared" : " 27 GB", | 20260505 18:59:43.371 | INFO | "malloc" : " 14 GB", | 20260505 18:59:43.371 | INFO | "pinned" : " 29 GB" | 20260505 18:59:43.371 | INFO | } | 20260505 18:59:43.371 | INFO | Perf report for the last 5.14576 seconds: | 20260505 18:59:43.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:59:43.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:43.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:43.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:43.371 | INFO | ANALYSIS : 61.7%, 3.0, 96.5ms, 0 | 20260505 18:59:43.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:43.371 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:43.371 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 18:59:43.371 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:43.371 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:43.371 | INFO | Uploaded 36GiB to GPU in 2.12486s (16.9423GiB/s) | 20260505 18:59:43.371 | INFO | Downloaded 34GiB to GPU in 2.05884s (16.5142GiB/s) | 20260505 18:59:48.501 | INFO | Analyzing chunk frames = [5548544,5549056) | 20260505 18:59:48.501 | INFO | MemInfo: { | 20260505 18:59:48.501 | INFO | "gpu" : " 17 GB", | 20260505 18:59:48.501 | INFO | "huge" : "", | 20260505 18:59:48.501 | INFO | "huge_shared" : " 27 GB", | 20260505 18:59:48.501 | INFO | "malloc" : " 14 GB", | 20260505 18:59:48.501 | INFO | "pinned" : " 29 GB" | 20260505 18:59:48.501 | INFO | } | 20260505 18:59:48.501 | INFO | Perf report for the last 5.12986 seconds: | 20260505 18:59:48.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:59:48.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:48.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:48.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:48.501 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260505 18:59:48.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:48.501 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:48.501 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 18:59:48.501 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:48.501 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:48.501 | INFO | Uploaded 36GiB to GPU in 2.16152s (16.655GiB/s) | 20260505 18:59:48.501 | INFO | Downloaded 34GiB to GPU in 2.06912s (16.4321GiB/s) | 20260505 18:59:53.636 | INFO | Analyzing chunk frames = [5549056,5549568) | 20260505 18:59:53.636 | INFO | MemInfo: { | 20260505 18:59:53.636 | INFO | "gpu" : " 17 GB", | 20260505 18:59:53.636 | INFO | "huge" : "", | 20260505 18:59:53.636 | INFO | "huge_shared" : " 27 GB", | 20260505 18:59:53.636 | INFO | "malloc" : " 14 GB", | 20260505 18:59:53.636 | INFO | "pinned" : " 29 GB" | 20260505 18:59:53.636 | INFO | } | 20260505 18:59:53.636 | INFO | Perf report for the last 5.1355 seconds: | 20260505 18:59:53.636 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:59:53.636 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:53.636 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:53.636 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:53.637 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 18:59:53.637 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:53.637 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:53.637 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 18:59:53.637 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:53.637 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:53.637 | INFO | Uploaded 36GiB to GPU in 2.14509s (16.7825GiB/s) | 20260505 18:59:53.637 | INFO | Downloaded 34GiB to GPU in 2.16611s (15.6963GiB/s) | 20260505 18:59:58.781 | INFO | Analyzing chunk frames = [5549568,5550080) | 20260505 18:59:58.781 | INFO | MemInfo: { | 20260505 18:59:58.781 | INFO | "gpu" : " 17 GB", | 20260505 18:59:58.781 | INFO | "huge" : "", | 20260505 18:59:58.781 | INFO | "huge_shared" : " 27 GB", | 20260505 18:59:58.781 | INFO | "malloc" : " 14 GB", | 20260505 18:59:58.781 | INFO | "pinned" : " 29 GB" | 20260505 18:59:58.781 | INFO | } | 20260505 18:59:58.781 | INFO | Perf report for the last 5.14444 seconds: | 20260505 18:59:58.781 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 18:59:58.781 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:58.781 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:58.781 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:58.781 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 18:59:58.781 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:58.781 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:58.781 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 18:59:58.781 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:58.781 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 18:59:58.781 | INFO | Uploaded 36GiB to GPU in 2.1593s (16.6721GiB/s) | 20260505 18:59:58.781 | INFO | Downloaded 34GiB to GPU in 2.02251s (16.8108GiB/s) | 20260505 18:59:58.916 | INFO | Starting a round of Baz IO | 20260505 19:00:02.084 | INFO | Populating RTMetrics report | 20260505 19:00:02.084 | INFO | RTMetrics report populated | 20260505 19:00:02.084 | INFO | Opening tmp RTMetrics json file | 20260505 19:00:02.087 | INFO | Serializing RTMetrics to json | 20260505 19:00:02.106 | INFO | Serialization done, swapping tmp file now... | 20260505 19:00:02.107 | INFO | RTMetrics done | 20260505 19:00:02.316 | INFO | Finished a round of Baz IO: Wrote 2.069 GiB in 3.4 seconds | 20260505 19:00:02.316 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.83% / 0.00% / 78.17% | 20260505 19:00:02.316 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.226378% / 0% / 8.34465e-05% | 20260505 19:00:02.316 | INFO | Baz IO overall: | 20260505 19:00:02.316 | INFO | BytesWritten : 3258855029283 | 20260505 19:00:02.316 | INFO | NumEvents : 1542910317736 | 20260505 19:00:03.921 | INFO | Analyzing chunk frames = [5550080,5550592) | 20260505 19:00:03.921 | INFO | MemInfo: { | 20260505 19:00:03.921 | INFO | "gpu" : " 17 GB", | 20260505 19:00:03.921 | INFO | "huge" : "", | 20260505 19:00:03.921 | INFO | "huge_shared" : " 27 GB", | 20260505 19:00:03.921 | INFO | "malloc" : " 14 GB", | 20260505 19:00:03.921 | INFO | "pinned" : " 29 GB" | 20260505 19:00:03.921 | INFO | } | 20260505 19:00:03.921 | INFO | Perf report for the last 5.13991 seconds: | 20260505 19:00:03.921 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:00:03.921 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:03.921 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:03.921 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:03.921 | INFO | ANALYSIS : 63.2%, 2.9, 97.1ms, 0 | 20260505 19:00:03.921 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:03.921 | INFO | RT_METRICS : 44.7%, 1.0, 23.5ms, 0 | 20260505 19:00:03.921 | INFO | PRE_HQ : 60.4%, 1.8, 55.1ms, 0 | 20260505 19:00:03.921 | INFO | BAZWRITER : 65.7%, 4.2, 140.7ms, 0 | 20260505 19:00:03.921 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 19:00:03.921 | INFO | Uploaded 36GiB to GPU in 2.19102s (16.4307GiB/s) | 20260505 19:00:03.921 | INFO | Downloaded 38GiB to GPU in 2.27218s (16.724GiB/s) | 20260505 19:00:09.066 | INFO | Analyzing chunk frames = [5550592,5551104) | 20260505 19:00:09.066 | INFO | MemInfo: { | 20260505 19:00:09.066 | INFO | "gpu" : " 17 GB", | 20260505 19:00:09.066 | INFO | "huge" : "", | 20260505 19:00:09.066 | INFO | "huge_shared" : " 27 GB", | 20260505 19:00:09.066 | INFO | "malloc" : " 14 GB", | 20260505 19:00:09.066 | INFO | "pinned" : " 29 GB" | 20260505 19:00:09.066 | INFO | } | 20260505 19:00:09.066 | INFO | Perf report for the last 5.14514 seconds: | 20260505 19:00:09.066 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:00:09.066 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:09.066 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:09.066 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:09.066 | INFO | ANALYSIS : 62.6%, 3.0, 97.9ms, 0 | 20260505 19:00:09.066 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:09.066 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:09.066 | INFO | PRE_HQ : 27.2%, 1.0, 14.2ms, 0 | 20260505 19:00:09.066 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:09.066 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:09.066 | INFO | Uploaded 36GiB to GPU in 2.13443s (16.8663GiB/s) | 20260505 19:00:09.066 | INFO | Downloaded 34GiB to GPU in 2.03355s (16.7195GiB/s) | 20260505 19:00:14.204 | INFO | Analyzing chunk frames = [5551104,5551616) | 20260505 19:00:14.204 | INFO | MemInfo: { | 20260505 19:00:14.204 | INFO | "gpu" : " 17 GB", | 20260505 19:00:14.204 | INFO | "huge" : "", | 20260505 19:00:14.204 | INFO | "huge_shared" : " 27 GB", | 20260505 19:00:14.204 | INFO | "malloc" : " 14 GB", | 20260505 19:00:14.204 | INFO | "pinned" : " 29 GB" | 20260505 19:00:14.204 | INFO | } | 20260505 19:00:14.204 | INFO | Perf report for the last 5.13807 seconds: | 20260505 19:00:14.204 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:00:14.204 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:14.204 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:14.204 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:14.204 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260505 19:00:14.204 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:14.204 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:14.204 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:00:14.204 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:14.204 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:14.204 | INFO | Uploaded 36GiB to GPU in 2.12255s (16.9607GiB/s) | 20260505 19:00:14.204 | INFO | Downloaded 34GiB to GPU in 2.01503s (16.8732GiB/s) | 20260505 19:00:19.335 | INFO | Analyzing chunk frames = [5551616,5552128) | 20260505 19:00:19.335 | INFO | MemInfo: { | 20260505 19:00:19.335 | INFO | "gpu" : " 17 GB", | 20260505 19:00:19.335 | INFO | "huge" : "", | 20260505 19:00:19.335 | INFO | "huge_shared" : " 27 GB", | 20260505 19:00:19.335 | INFO | "malloc" : " 14 GB", | 20260505 19:00:19.335 | INFO | "pinned" : " 29 GB" | 20260505 19:00:19.335 | INFO | } | 20260505 19:00:19.335 | INFO | Perf report for the last 5.13128 seconds: | 20260505 19:00:19.335 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:00:19.335 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:19.335 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:19.335 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:19.335 | INFO | ANALYSIS : 62.5%, 3.0, 97.0ms, 0 | 20260505 19:00:19.335 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:19.335 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:19.335 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 19:00:19.335 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:19.336 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:19.336 | INFO | Uploaded 36GiB to GPU in 2.17554s (16.5476GiB/s) | 20260505 19:00:19.336 | INFO | Downloaded 34GiB to GPU in 2.06453s (16.4687GiB/s) | 20260505 19:00:24.486 | INFO | Analyzing chunk frames = [5552128,5552640) | 20260505 19:00:24.486 | INFO | MemInfo: { | 20260505 19:00:24.486 | INFO | "gpu" : " 17 GB", | 20260505 19:00:24.486 | INFO | "huge" : "", | 20260505 19:00:24.486 | INFO | "huge_shared" : " 27 GB", | 20260505 19:00:24.486 | INFO | "malloc" : " 14 GB", | 20260505 19:00:24.486 | INFO | "pinned" : " 29 GB" | 20260505 19:00:24.486 | INFO | } | 20260505 19:00:24.486 | INFO | Perf report for the last 5.15057 seconds: | 20260505 19:00:24.486 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:00:24.486 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:24.486 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:24.486 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:24.486 | INFO | ANALYSIS : 61.8%, 3.0, 96.7ms, 0 | 20260505 19:00:24.486 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:24.486 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:24.486 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:00:24.486 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:24.486 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:24.486 | INFO | Uploaded 36GiB to GPU in 2.14363s (16.7939GiB/s) | 20260505 19:00:24.486 | INFO | Downloaded 34GiB to GPU in 2.06325s (16.4788GiB/s) | 20260505 19:00:29.621 | INFO | Analyzing chunk frames = [5552640,5553152) | 20260505 19:00:29.621 | INFO | MemInfo: { | 20260505 19:00:29.621 | INFO | "gpu" : " 17 GB", | 20260505 19:00:29.621 | INFO | "huge" : "", | 20260505 19:00:29.621 | INFO | "huge_shared" : " 27 GB", | 20260505 19:00:29.621 | INFO | "malloc" : " 14 GB", | 20260505 19:00:29.621 | INFO | "pinned" : " 29 GB" | 20260505 19:00:29.621 | INFO | } | 20260505 19:00:29.621 | INFO | Perf report for the last 5.13491 seconds: | 20260505 19:00:29.621 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:00:29.621 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:29.621 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:29.621 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:29.621 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 19:00:29.621 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:29.621 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:29.621 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:00:29.621 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:29.621 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:29.621 | INFO | Uploaded 36GiB to GPU in 2.14931s (16.7496GiB/s) | 20260505 19:00:29.621 | INFO | Downloaded 34GiB to GPU in 2.05449s (16.5491GiB/s) | 20260505 19:00:34.757 | INFO | Analyzing chunk frames = [5553152,5553664) | 20260505 19:00:34.757 | INFO | MemInfo: { | 20260505 19:00:34.757 | INFO | "gpu" : " 17 GB", | 20260505 19:00:34.757 | INFO | "huge" : "", | 20260505 19:00:34.757 | INFO | "huge_shared" : " 27 GB", | 20260505 19:00:34.757 | INFO | "malloc" : " 14 GB", | 20260505 19:00:34.757 | INFO | "pinned" : " 29 GB" | 20260505 19:00:34.757 | INFO | } | 20260505 19:00:34.757 | INFO | Perf report for the last 5.13593 seconds: | 20260505 19:00:34.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:00:34.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:34.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:34.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:34.757 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 19:00:34.757 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:34.757 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:34.757 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 19:00:34.757 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:34.757 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:34.757 | INFO | Uploaded 36GiB to GPU in 2.13871s (16.8326GiB/s) | 20260505 19:00:34.757 | INFO | Downloaded 34GiB to GPU in 2.14411s (15.8574GiB/s) | 20260505 19:00:39.895 | INFO | Analyzing chunk frames = [5553664,5554176) | 20260505 19:00:39.895 | INFO | MemInfo: { | 20260505 19:00:39.895 | INFO | "gpu" : " 17 GB", | 20260505 19:00:39.895 | INFO | "huge" : "", | 20260505 19:00:39.895 | INFO | "huge_shared" : " 27 GB", | 20260505 19:00:39.895 | INFO | "malloc" : " 14 GB", | 20260505 19:00:39.895 | INFO | "pinned" : " 29 GB" | 20260505 19:00:39.895 | INFO | } | 20260505 19:00:39.895 | INFO | Perf report for the last 5.13859 seconds: | 20260505 19:00:39.895 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:00:39.895 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:39.895 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:39.895 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:39.895 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 19:00:39.896 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:39.896 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:39.896 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 19:00:39.896 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:39.896 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:39.896 | INFO | Uploaded 36GiB to GPU in 2.20129s (16.354GiB/s) | 20260505 19:00:39.896 | INFO | Downloaded 34GiB to GPU in 2.0412s (16.6568GiB/s) | 20260505 19:00:39.896 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 19:00:39.896 | INFO | Total mem usage: 15.3 GiB | 20260505 19:00:39.896 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 19:00:39.896 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -2.01%) | 20260505 19:00:39.896 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 19:00:39.896 | INFO | High Water Breakdown: | 20260505 19:00:39.896 | INFO | 62% : 9.32 GiB ( 97.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 19:00:39.896 | INFO | 37.9% : 5.69 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 19:00:39.896 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 19:00:39.896 | INFO | ------------------------------------------ | 20260505 19:00:39.896 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:00:39.896 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 19:00:39.896 | INFO | Total mem usage: 26.5 GiB | 20260505 19:00:39.896 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 19:00:39.896 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 19:00:39.896 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 19:00:39.896 | INFO | High Water Breakdown: | 20260505 19:00:39.896 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:00:39.896 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:00:39.896 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:00:39.896 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:00:39.896 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:00:39.896 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:00:39.896 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 19:00:39.896 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 19:00:39.896 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:00:39.896 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 19:00:39.896 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 19:00:39.896 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 19:00:39.896 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 19:00:39.896 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 19:00:39.896 | INFO | ------------------------------------------ | 20260505 19:00:39.896 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:00:39.896 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 19:00:39.896 | INFO | Total mem usage: 25.6 GiB | 20260505 19:00:39.896 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 19:00:39.896 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 19:00:39.896 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 19:00:39.896 | INFO | High Water Breakdown: | 20260505 19:00:39.896 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 19:00:39.896 | INFO | ------------------------------------------ | 20260505 19:00:39.896 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:00:39.896 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 19:00:39.896 | INFO | Total mem usage: 15.7 GiB | 20260505 19:00:39.896 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 19:00:39.896 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 19:00:39.896 | INFO | Unused Cache: 914 MiB (Waste: 5.7%) | 20260505 19:00:39.896 | INFO | High Water Breakdown: | 20260505 19:00:39.896 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 19:00:39.896 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 19:00:39.896 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 19:00:39.897 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 19:00:39.897 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 19:00:39.897 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 19:00:39.897 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 19:00:39.897 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 19:00:39.897 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 19:00:39.897 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:00:39.897 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 19:00:39.897 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:00:39.897 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 19:00:39.897 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 19:00:39.897 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 19:00:39.897 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 19:00:39.897 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:00:39.897 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 19:00:39.897 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:00:39.897 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:00:39.897 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:00:39.897 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 19:00:39.897 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 19:00:39.897 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:00:39.897 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 19:00:39.897 | INFO | ------------------------------------------ | 20260505 19:00:39.897 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:00:39.897 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 19:00:39.897 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:00:39.897 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:00:39.897 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.46e+01, 1.33e+01 / 1.88e+00 | 20260505 19:00:39.897 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.03e-01, 1.10e-02 / 4.91e-03 | 20260505 19:00:39.897 | INFO | PerfReport: 11.80%, Baseline -- 3.72e+00 / 4.52e+00, 5.65e-01 / 1.61e-01 | 20260505 19:00:39.897 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.54e+00, 1.91e-01 / 5.40e-02 | 20260505 19:00:39.897 | INFO | PerfReport: 15.98%, DME -- 4.03e+01 / 4.45e+01, 4.01e+00 / 1.89e+00 | 20260505 19:00:39.897 | INFO | PerfReport: 56.87%, FrameLabeling -- 1.79e+01 / 1.92e+01, 1.39e+00 / 1.16e+00 | 20260505 19:00:39.897 | INFO | PerfReport: 7.31%, PulseAccumulating -- 2.30e+00 / 2.89e+00, 4.43e-01 / 1.59e-01 | 20260505 19:00:39.897 | INFO | PerfReport: 3.31%, Metrics -- 1.04e+00 / 1.37e+00, 2.90e-01 / 1.47e-01 | 20260505 19:00:39.897 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.63e-01 / 4.18e-01, 6.38e-02 / 3.72e-02 | 20260505 19:00:39.897 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 19:00:39.897 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:00:39.897 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:00:39.897 | INFO | PerfReport: Full Execution -- 9.95e+01 / -nan, 2.55e+01 / 0.00e+00 | 20260505 19:00:39.897 | INFO | PerfReport: 22.60%, Upload -- 2.25e+01 / 0.00e+00, 7.81e+00 / 0.00e+00 | 20260505 19:00:39.897 | INFO | PerfReport: 21.92%, Download -- 2.18e+01 / 0.00e+00, 7.75e+00 / 0.00e+00 | 20260505 19:00:39.897 | INFO | PerfReport: 31.66%, Compute -- 3.15e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260505 19:00:39.897 | INFO | PerfReport: 2.26%, LockUpload -- 2.25e+00 / 0.00e+00, 7.73e+00 / 0.00e+00 | 20260505 19:00:39.897 | INFO | PerfReport: 1.40%, LockDownload -- 1.40e+00 / 0.00e+00, 2.86e+00 / 0.00e+00 | 20260505 19:00:39.897 | INFO | PerfReport: 20.14%, LockCompute -- 2.00e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 19:00:39.897 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.33e-02 / 0.00e+00 | 20260505 19:00:40.030 | INFO | Starting a round of Baz IO | 20260505 19:00:43.203 | INFO | Populating RTMetrics report | 20260505 19:00:43.203 | INFO | RTMetrics report populated | 20260505 19:00:43.203 | INFO | Opening tmp RTMetrics json file | 20260505 19:00:43.204 | INFO | Serializing RTMetrics to json | 20260505 19:00:43.207 | INFO | Serialization done, swapping tmp file now... | 20260505 19:00:43.209 | INFO | RTMetrics done | 20260505 19:00:43.293 | INFO | Finished a round of Baz IO: Wrote 2.068 GiB in 3.262 seconds | 20260505 19:00:43.293 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.83% / 0.00% / 78.17% | 20260505 19:00:43.293 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.225302% / 0% / 0.000111262% | 20260505 19:00:43.293 | INFO | Baz IO overall: | 20260505 19:00:43.293 | INFO | BytesWritten : 3261075877894 | 20260505 19:00:43.293 | INFO | NumEvents : 1543943299973 | 20260505 19:00:45.036 | INFO | Analyzing chunk frames = [5554176,5554688) | 20260505 19:00:45.036 | INFO | MemInfo: { | 20260505 19:00:45.036 | INFO | "gpu" : " 17 GB", | 20260505 19:00:45.036 | INFO | "huge" : "", | 20260505 19:00:45.036 | INFO | "huge_shared" : " 27 GB", | 20260505 19:00:45.036 | INFO | "malloc" : " 14 GB", | 20260505 19:00:45.036 | INFO | "pinned" : " 29 GB" | 20260505 19:00:45.036 | INFO | } | 20260505 19:00:45.036 | INFO | Perf report for the last 5.14028 seconds: | 20260505 19:00:45.036 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:00:45.036 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:45.036 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:45.036 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:45.036 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 19:00:45.036 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:45.036 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260505 19:00:45.036 | INFO | PRE_HQ : 60.2%, 1.8, 55.5ms, 0 | 20260505 19:00:45.036 | INFO | BAZWRITER : 63.0%, 3.8, 122.8ms, 0 | 20260505 19:00:45.036 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 19:00:45.036 | INFO | Uploaded 36GiB to GPU in 2.17154s (16.5781GiB/s) | 20260505 19:00:45.036 | INFO | Downloaded 38GiB to GPU in 2.24808s (16.9033GiB/s) | 20260505 19:00:50.177 | INFO | Analyzing chunk frames = [5554688,5555200) | 20260505 19:00:50.177 | INFO | MemInfo: { | 20260505 19:00:50.177 | INFO | "gpu" : " 17 GB", | 20260505 19:00:50.177 | INFO | "huge" : "", | 20260505 19:00:50.177 | INFO | "huge_shared" : " 27 GB", | 20260505 19:00:50.177 | INFO | "malloc" : " 14 GB", | 20260505 19:00:50.177 | INFO | "pinned" : " 29 GB" | 20260505 19:00:50.177 | INFO | } | 20260505 19:00:50.178 | INFO | Perf report for the last 5.1418 seconds: | 20260505 19:00:50.178 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:00:50.178 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:50.178 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:50.178 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:50.178 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 19:00:50.178 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:50.178 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:50.178 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:00:50.178 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:50.178 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:50.178 | INFO | Uploaded 36GiB to GPU in 2.12479s (16.9429GiB/s) | 20260505 19:00:50.178 | INFO | Downloaded 34GiB to GPU in 2.02152s (16.819GiB/s) | 20260505 19:00:55.318 | INFO | Analyzing chunk frames = [5555200,5555712) | 20260505 19:00:55.318 | INFO | MemInfo: { | 20260505 19:00:55.318 | INFO | "gpu" : " 17 GB", | 20260505 19:00:55.318 | INFO | "huge" : "", | 20260505 19:00:55.319 | INFO | "huge_shared" : " 27 GB", | 20260505 19:00:55.319 | INFO | "malloc" : " 14 GB", | 20260505 19:00:55.319 | INFO | "pinned" : " 29 GB" | 20260505 19:00:55.319 | INFO | } | 20260505 19:00:55.319 | INFO | Perf report for the last 5.14124 seconds: | 20260505 19:00:55.319 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:00:55.319 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:55.319 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:55.319 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:55.319 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260505 19:00:55.319 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:55.319 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:55.319 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:00:55.319 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:55.319 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:00:55.319 | INFO | Uploaded 36GiB to GPU in 2.11778s (16.9989GiB/s) | 20260505 19:00:55.319 | INFO | Downloaded 34GiB to GPU in 2.00904s (16.9235GiB/s) | 20260505 19:01:00.453 | INFO | Analyzing chunk frames = [5555712,5556224) | 20260505 19:01:00.453 | INFO | MemInfo: { | 20260505 19:01:00.453 | INFO | "gpu" : " 17 GB", | 20260505 19:01:00.453 | INFO | "huge" : "", | 20260505 19:01:00.453 | INFO | "huge_shared" : " 27 GB", | 20260505 19:01:00.453 | INFO | "malloc" : " 14 GB", | 20260505 19:01:00.453 | INFO | "pinned" : " 29 GB" | 20260505 19:01:00.453 | INFO | } | 20260505 19:01:00.453 | INFO | Perf report for the last 5.13424 seconds: | 20260505 19:01:00.453 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:01:00.453 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:00.453 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:00.453 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:00.453 | INFO | ANALYSIS : 62.5%, 3.0, 97.0ms, 0 | 20260505 19:01:00.453 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:00.453 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:00.453 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 19:01:00.453 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:00.453 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:00.453 | INFO | Uploaded 36GiB to GPU in 2.20104s (16.3559GiB/s) | 20260505 19:01:00.453 | INFO | Downloaded 34GiB to GPU in 2.03902s (16.6747GiB/s) | 20260505 19:01:05.584 | INFO | Analyzing chunk frames = [5556224,5556736) | 20260505 19:01:05.584 | INFO | MemInfo: { | 20260505 19:01:05.584 | INFO | "gpu" : " 17 GB", | 20260505 19:01:05.584 | INFO | "huge" : "", | 20260505 19:01:05.584 | INFO | "huge_shared" : " 27 GB", | 20260505 19:01:05.584 | INFO | "malloc" : " 14 GB", | 20260505 19:01:05.584 | INFO | "pinned" : " 29 GB" | 20260505 19:01:05.584 | INFO | } | 20260505 19:01:05.584 | INFO | Perf report for the last 5.13136 seconds: | 20260505 19:01:05.584 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:01:05.584 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:05.584 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:05.584 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:05.585 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260505 19:01:05.585 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:05.585 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:05.585 | INFO | PRE_HQ : 28.1%, 1.0, 14.5ms, 0 | 20260505 19:01:05.585 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:05.585 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:05.585 | INFO | Uploaded 36GiB to GPU in 2.13546s (16.8582GiB/s) | 20260505 19:01:05.585 | INFO | Downloaded 34GiB to GPU in 2.06263s (16.4838GiB/s) | 20260505 19:01:10.734 | INFO | Analyzing chunk frames = [5556736,5557248) | 20260505 19:01:10.734 | INFO | MemInfo: { | 20260505 19:01:10.734 | INFO | "gpu" : " 17 GB", | 20260505 19:01:10.734 | INFO | "huge" : "", | 20260505 19:01:10.734 | INFO | "huge_shared" : " 27 GB", | 20260505 19:01:10.734 | INFO | "malloc" : " 14 GB", | 20260505 19:01:10.734 | INFO | "pinned" : " 29 GB" | 20260505 19:01:10.734 | INFO | } | 20260505 19:01:10.734 | INFO | Perf report for the last 5.14921 seconds: | 20260505 19:01:10.734 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:01:10.734 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:10.734 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:10.734 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:10.734 | INFO | ANALYSIS : 62.3%, 3.0, 97.4ms, 0 | 20260505 19:01:10.734 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:10.734 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:10.734 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 19:01:10.734 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:10.734 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:10.734 | INFO | Uploaded 36GiB to GPU in 2.14597s (16.7757GiB/s) | 20260505 19:01:10.734 | INFO | Downloaded 34GiB to GPU in 2.04432s (16.6314GiB/s) | 20260505 19:01:15.878 | INFO | Analyzing chunk frames = [5557248,5557760) | 20260505 19:01:15.878 | INFO | MemInfo: { | 20260505 19:01:15.878 | INFO | "gpu" : " 17 GB", | 20260505 19:01:15.878 | INFO | "huge" : "", | 20260505 19:01:15.878 | INFO | "huge_shared" : " 27 GB", | 20260505 19:01:15.878 | INFO | "malloc" : " 14 GB", | 20260505 19:01:15.878 | INFO | "pinned" : " 29 GB" | 20260505 19:01:15.878 | INFO | } | 20260505 19:01:15.878 | INFO | Perf report for the last 5.14443 seconds: | 20260505 19:01:15.878 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:01:15.878 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:15.878 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:15.878 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:15.878 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 19:01:15.878 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:15.878 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:15.878 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 19:01:15.878 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:15.878 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:15.878 | INFO | Uploaded 36GiB to GPU in 2.14894s (16.7525GiB/s) | 20260505 19:01:15.878 | INFO | Downloaded 34GiB to GPU in 2.12986s (15.9635GiB/s) | 20260505 19:01:21.012 | INFO | Analyzing chunk frames = [5557760,5558272) | 20260505 19:01:21.012 | INFO | MemInfo: { | 20260505 19:01:21.012 | INFO | "gpu" : " 17 GB", | 20260505 19:01:21.012 | INFO | "huge" : "", | 20260505 19:01:21.012 | INFO | "huge_shared" : " 27 GB", | 20260505 19:01:21.013 | INFO | "malloc" : " 14 GB", | 20260505 19:01:21.013 | INFO | "pinned" : " 29 GB" | 20260505 19:01:21.013 | INFO | } | 20260505 19:01:21.013 | INFO | Perf report for the last 5.13449 seconds: | 20260505 19:01:21.013 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:01:21.013 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:21.013 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:21.013 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:21.013 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260505 19:01:21.013 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:21.013 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:21.013 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 19:01:21.013 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:21.013 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:21.013 | INFO | Uploaded 36GiB to GPU in 2.11884s (16.9904GiB/s) | 20260505 19:01:21.013 | INFO | Downloaded 34GiB to GPU in 2.00714s (16.9395GiB/s) | 20260505 19:01:21.149 | INFO | Starting a round of Baz IO | 20260505 19:01:24.326 | INFO | Populating RTMetrics report | 20260505 19:01:24.326 | INFO | RTMetrics report populated | 20260505 19:01:24.326 | INFO | Opening tmp RTMetrics json file | 20260505 19:01:24.334 | INFO | Serializing RTMetrics to json | 20260505 19:01:24.338 | INFO | Serialization done, swapping tmp file now... | 20260505 19:01:24.338 | INFO | RTMetrics done | 20260505 19:01:24.425 | INFO | Finished a round of Baz IO: Wrote 2.067 GiB in 3.276 seconds | 20260505 19:01:24.425 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.83% / 0.00% / 78.17% | 20260505 19:01:24.425 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.224694% / 3.97364e-06% / 0.000115236% | 20260505 19:01:24.425 | INFO | Baz IO overall: | 20260505 19:01:24.425 | INFO | BytesWritten : 3263295353738 | 20260505 19:01:24.425 | INFO | NumEvents : 1544975509072 | 20260505 19:01:26.149 | INFO | Analyzing chunk frames = [5558272,5558784) | 20260505 19:01:26.149 | INFO | MemInfo: { | 20260505 19:01:26.149 | INFO | "gpu" : " 17 GB", | 20260505 19:01:26.149 | INFO | "huge" : "", | 20260505 19:01:26.149 | INFO | "huge_shared" : " 27 GB", | 20260505 19:01:26.149 | INFO | "malloc" : " 14 GB", | 20260505 19:01:26.149 | INFO | "pinned" : " 29 GB" | 20260505 19:01:26.149 | INFO | } | 20260505 19:01:26.149 | INFO | Perf report for the last 5.13667 seconds: | 20260505 19:01:26.149 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:01:26.149 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:26.149 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:26.149 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:26.149 | INFO | ANALYSIS : 63.4%, 3.0, 98.8ms, 0 | 20260505 19:01:26.149 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:26.149 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260505 19:01:26.149 | INFO | PRE_HQ : 60.5%, 1.8, 55.8ms, 0 | 20260505 19:01:26.149 | INFO | BAZWRITER : 63.3%, 3.8, 123.0ms, 0 | 20260505 19:01:26.149 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 19:01:26.149 | INFO | Uploaded 36GiB to GPU in 2.17321s (16.5653GiB/s) | 20260505 19:01:26.149 | INFO | Downloaded 38GiB to GPU in 2.24835s (16.9013GiB/s) | 20260505 19:01:31.288 | INFO | Analyzing chunk frames = [5558784,5559296) | 20260505 19:01:31.288 | INFO | MemInfo: { | 20260505 19:01:31.288 | INFO | "gpu" : " 17 GB", | 20260505 19:01:31.288 | INFO | "huge" : "", | 20260505 19:01:31.288 | INFO | "huge_shared" : " 27 GB", | 20260505 19:01:31.288 | INFO | "malloc" : " 14 GB", | 20260505 19:01:31.288 | INFO | "pinned" : " 29 GB" | 20260505 19:01:31.288 | INFO | } | 20260505 19:01:31.288 | INFO | Perf report for the last 5.139 seconds: | 20260505 19:01:31.288 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:01:31.288 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:31.288 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:31.288 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:31.288 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 19:01:31.288 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:31.288 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:31.288 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:01:31.288 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:31.288 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:31.289 | INFO | Uploaded 36GiB to GPU in 2.13793s (16.8387GiB/s) | 20260505 19:01:31.289 | INFO | Downloaded 34GiB to GPU in 2.03294s (16.7245GiB/s) | 20260505 19:01:36.430 | INFO | Analyzing chunk frames = [5559296,5559808) | 20260505 19:01:36.430 | INFO | MemInfo: { | 20260505 19:01:36.430 | INFO | "gpu" : " 17 GB", | 20260505 19:01:36.430 | INFO | "huge" : "", | 20260505 19:01:36.430 | INFO | "huge_shared" : " 27 GB", | 20260505 19:01:36.430 | INFO | "malloc" : " 14 GB", | 20260505 19:01:36.430 | INFO | "pinned" : " 29 GB" | 20260505 19:01:36.430 | INFO | } | 20260505 19:01:36.430 | INFO | Perf report for the last 5.14185 seconds: | 20260505 19:01:36.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:01:36.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:36.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:36.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:36.430 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260505 19:01:36.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:36.430 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:36.430 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 19:01:36.430 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:36.430 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:36.430 | INFO | Uploaded 36GiB to GPU in 2.13165s (16.8883GiB/s) | 20260505 19:01:36.430 | INFO | Downloaded 34GiB to GPU in 2.03292s (16.7248GiB/s) | 20260505 19:01:41.570 | INFO | Analyzing chunk frames = [5559808,5560320) | 20260505 19:01:41.570 | INFO | MemInfo: { | 20260505 19:01:41.570 | INFO | "gpu" : " 17 GB", | 20260505 19:01:41.570 | INFO | "huge" : "", | 20260505 19:01:41.570 | INFO | "huge_shared" : " 27 GB", | 20260505 19:01:41.570 | INFO | "malloc" : " 14 GB", | 20260505 19:01:41.570 | INFO | "pinned" : " 29 GB" | 20260505 19:01:41.570 | INFO | } | 20260505 19:01:41.570 | INFO | Perf report for the last 5.13976 seconds: | 20260505 19:01:41.570 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:01:41.570 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:41.570 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:41.570 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:41.570 | INFO | ANALYSIS : 62.4%, 3.0, 96.9ms, 0 | 20260505 19:01:41.570 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:41.570 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:41.570 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 19:01:41.570 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:41.570 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:41.570 | INFO | Uploaded 36GiB to GPU in 2.20162s (16.3516GiB/s) | 20260505 19:01:41.570 | INFO | Downloaded 34GiB to GPU in 2.04265s (16.645GiB/s) | 20260505 19:01:46.705 | INFO | Analyzing chunk frames = [5560320,5560832) | 20260505 19:01:46.705 | INFO | MemInfo: { | 20260505 19:01:46.705 | INFO | "gpu" : " 17 GB", | 20260505 19:01:46.705 | INFO | "huge" : "", | 20260505 19:01:46.705 | INFO | "huge_shared" : " 27 GB", | 20260505 19:01:46.705 | INFO | "malloc" : " 14 GB", | 20260505 19:01:46.705 | INFO | "pinned" : " 29 GB" | 20260505 19:01:46.705 | INFO | } | 20260505 19:01:46.705 | INFO | Perf report for the last 5.13505 seconds: | 20260505 19:01:46.705 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:01:46.705 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:46.705 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:46.705 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:46.705 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 19:01:46.705 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:46.705 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:46.705 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 19:01:46.705 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:46.705 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:46.705 | INFO | Uploaded 36GiB to GPU in 2.14768s (16.7623GiB/s) | 20260505 19:01:46.705 | INFO | Downloaded 34GiB to GPU in 2.06351s (16.4768GiB/s) | 20260505 19:01:51.844 | INFO | Analyzing chunk frames = [5560832,5561344) | 20260505 19:01:51.844 | INFO | MemInfo: { | 20260505 19:01:51.844 | INFO | "gpu" : " 17 GB", | 20260505 19:01:51.844 | INFO | "huge" : "", | 20260505 19:01:51.844 | INFO | "huge_shared" : " 27 GB", | 20260505 19:01:51.844 | INFO | "malloc" : " 14 GB", | 20260505 19:01:51.844 | INFO | "pinned" : " 29 GB" | 20260505 19:01:51.845 | INFO | } | 20260505 19:01:51.845 | INFO | Perf report for the last 5.13966 seconds: | 20260505 19:01:51.845 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:01:51.845 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:51.845 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:51.845 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:51.845 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260505 19:01:51.845 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:51.845 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:51.845 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 19:01:51.845 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:51.845 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:51.845 | INFO | Uploaded 36GiB to GPU in 2.14551s (16.7792GiB/s) | 20260505 19:01:51.845 | INFO | Downloaded 34GiB to GPU in 2.04698s (16.6098GiB/s) | 20260505 19:01:56.992 | INFO | Analyzing chunk frames = [5561344,5561856) | 20260505 19:01:56.992 | INFO | MemInfo: { | 20260505 19:01:56.992 | INFO | "gpu" : " 17 GB", | 20260505 19:01:56.992 | INFO | "huge" : "", | 20260505 19:01:56.992 | INFO | "huge_shared" : " 27 GB", | 20260505 19:01:56.992 | INFO | "malloc" : " 14 GB", | 20260505 19:01:56.992 | INFO | "pinned" : " 29 GB" | 20260505 19:01:56.992 | INFO | } | 20260505 19:01:56.992 | INFO | Perf report for the last 5.14736 seconds: | 20260505 19:01:56.992 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:01:56.992 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:56.992 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:56.992 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:56.992 | INFO | ANALYSIS : 63.2%, 3.0, 98.9ms, 0 | 20260505 19:01:56.992 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:56.992 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:56.992 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 19:01:56.992 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:56.992 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:01:56.992 | INFO | Uploaded 36GiB to GPU in 2.14077s (16.8164GiB/s) | 20260505 19:01:56.992 | INFO | Downloaded 34GiB to GPU in 2.14137s (15.8777GiB/s) | 20260505 19:02:02.132 | INFO | Analyzing chunk frames = [5561856,5562368) | 20260505 19:02:02.132 | INFO | MemInfo: { | 20260505 19:02:02.132 | INFO | "gpu" : " 17 GB", | 20260505 19:02:02.132 | INFO | "huge" : "", | 20260505 19:02:02.132 | INFO | "huge_shared" : " 27 GB", | 20260505 19:02:02.132 | INFO | "malloc" : " 14 GB", | 20260505 19:02:02.132 | INFO | "pinned" : " 29 GB" | 20260505 19:02:02.132 | INFO | } | 20260505 19:02:02.132 | INFO | Perf report for the last 5.13982 seconds: | 20260505 19:02:02.132 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:02:02.132 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:02.132 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:02.132 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:02.132 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 19:02:02.132 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:02.132 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:02.132 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 19:02:02.132 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:02.132 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:02.132 | INFO | Uploaded 36GiB to GPU in 2.20225s (16.3469GiB/s) | 20260505 19:02:02.132 | INFO | Downloaded 34GiB to GPU in 2.04448s (16.6301GiB/s) | 20260505 19:02:02.132 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 19:02:02.132 | INFO | Total mem usage: 15.3 GiB | 20260505 19:02:02.132 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 19:02:02.132 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.14%) | 20260505 19:02:02.132 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 19:02:02.132 | INFO | High Water Breakdown: | 20260505 19:02:02.132 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 19:02:02.132 | INFO | 38.9% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 19:02:02.132 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 19:02:02.132 | INFO | ------------------------------------------ | 20260505 19:02:02.132 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:02:02.132 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 19:02:02.132 | INFO | Total mem usage: 26.5 GiB | 20260505 19:02:02.133 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 19:02:02.133 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 19:02:02.133 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 19:02:02.133 | INFO | High Water Breakdown: | 20260505 19:02:02.133 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:02:02.133 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:02:02.133 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:02:02.133 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:02:02.133 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:02:02.133 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:02:02.133 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 19:02:02.133 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 19:02:02.133 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:02:02.133 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 19:02:02.133 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 19:02:02.133 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 19:02:02.133 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 19:02:02.133 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 19:02:02.133 | INFO | ------------------------------------------ | 20260505 19:02:02.133 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:02:02.133 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 19:02:02.133 | INFO | Total mem usage: 25.6 GiB | 20260505 19:02:02.133 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 19:02:02.133 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 19:02:02.133 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 19:02:02.133 | INFO | High Water Breakdown: | 20260505 19:02:02.133 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 19:02:02.133 | INFO | ------------------------------------------ | 20260505 19:02:02.133 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:02:02.133 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 19:02:02.133 | INFO | Total mem usage: 15.7 GiB | 20260505 19:02:02.133 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 19:02:02.133 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 19:02:02.133 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 19:02:02.133 | INFO | High Water Breakdown: | 20260505 19:02:02.133 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 19:02:02.133 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 19:02:02.133 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 19:02:02.133 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 19:02:02.133 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 19:02:02.133 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 19:02:02.133 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 19:02:02.133 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 19:02:02.133 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 19:02:02.133 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:02:02.133 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 19:02:02.133 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:02:02.133 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 19:02:02.133 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 19:02:02.133 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 19:02:02.133 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 19:02:02.133 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:02:02.133 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 19:02:02.133 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:02:02.133 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:02:02.133 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:02:02.133 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 19:02:02.133 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 19:02:02.133 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:02:02.133 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 19:02:02.133 | INFO | ------------------------------------------ | 20260505 19:02:02.133 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:02:02.133 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 19:02:02.133 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:02:02.133 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:02:02.133 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.45e+01, 1.34e+01 / 1.86e+00 | 20260505 19:02:02.133 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 1.02e-01, 1.13e-02 / 3.63e-03 | 20260505 19:02:02.133 | INFO | PerfReport: 11.78%, Baseline -- 3.71e+00 / 4.56e+00, 5.62e-01 / 1.19e-01 | 20260505 19:02:02.133 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.54e+00, 1.91e-01 / 8.98e-02 | 20260505 19:02:02.133 | INFO | PerfReport: 15.93%, DME -- 4.02e+01 / 4.43e+01, 4.05e+00 / 1.87e+00 | 20260505 19:02:02.133 | INFO | PerfReport: 56.95%, FrameLabeling -- 1.80e+01 / 1.93e+01, 1.47e+00 / 1.12e+00 | 20260505 19:02:02.133 | INFO | PerfReport: 7.29%, PulseAccumulating -- 2.30e+00 / 2.88e+00, 4.40e-01 / 1.92e-01 | 20260505 19:02:02.133 | INFO | PerfReport: 3.31%, Metrics -- 1.04e+00 / 1.39e+00, 2.87e-01 / 1.58e-01 | 20260505 19:02:02.134 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.63e-01 / 4.29e-01, 6.43e-02 / 3.92e-02 | 20260505 19:02:02.134 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 19:02:02.134 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:02:02.134 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:02:02.134 | INFO | PerfReport: Full Execution -- 9.96e+01 / -nan, 2.56e+01 / 0.00e+00 | 20260505 19:02:02.134 | INFO | PerfReport: 22.51%, Upload -- 2.24e+01 / 0.00e+00, 7.82e+00 / 0.00e+00 | 20260505 19:02:02.134 | INFO | PerfReport: 21.70%, Download -- 2.16e+01 / 0.00e+00, 7.69e+00 / 0.00e+00 | 20260505 19:02:02.134 | INFO | PerfReport: 31.64%, Compute -- 3.15e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 19:02:02.134 | INFO | PerfReport: 2.38%, LockUpload -- 2.37e+00 / 0.00e+00, 8.18e+00 / 0.00e+00 | 20260505 19:02:02.134 | INFO | PerfReport: 1.31%, LockDownload -- 1.30e+00 / 0.00e+00, 2.78e+00 / 0.00e+00 | 20260505 19:02:02.134 | INFO | PerfReport: 20.46%, LockCompute -- 2.04e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 19:02:02.134 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.48e-02 / 0.00e+00 | 20260505 19:02:02.262 | INFO | Starting a round of Baz IO | 20260505 19:02:05.434 | INFO | Populating RTMetrics report | 20260505 19:02:05.434 | INFO | RTMetrics report populated | 20260505 19:02:05.434 | INFO | Opening tmp RTMetrics json file | 20260505 19:02:05.437 | INFO | Serializing RTMetrics to json | 20260505 19:02:05.439 | INFO | Serialization done, swapping tmp file now... | 20260505 19:02:05.439 | INFO | RTMetrics done | 20260505 19:02:05.525 | INFO | Finished a round of Baz IO: Wrote 2.068 GiB in 3.263 seconds | 20260505 19:02:05.525 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.83% / 0.00% / 78.17% | 20260505 19:02:05.525 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.224356% / 0% / 9.93411e-05% | 20260505 19:02:05.525 | INFO | Baz IO overall: | 20260505 19:02:05.525 | INFO | BytesWritten : 3265516093910 | 20260505 19:02:05.525 | INFO | NumEvents : 1546008520641 | 20260505 19:02:07.263 | INFO | Analyzing chunk frames = [5562368,5562880) | 20260505 19:02:07.263 | INFO | MemInfo: { | 20260505 19:02:07.263 | INFO | "gpu" : " 17 GB", | 20260505 19:02:07.263 | INFO | "huge" : "", | 20260505 19:02:07.263 | INFO | "huge_shared" : " 27 GB", | 20260505 19:02:07.263 | INFO | "malloc" : " 14 GB", | 20260505 19:02:07.263 | INFO | "pinned" : " 29 GB" | 20260505 19:02:07.263 | INFO | } | 20260505 19:02:07.264 | INFO | Perf report for the last 5.13176 seconds: | 20260505 19:02:07.264 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:02:07.264 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:07.264 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:07.264 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:07.264 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 19:02:07.264 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:07.264 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260505 19:02:07.264 | INFO | PRE_HQ : 60.4%, 1.8, 55.6ms, 0 | 20260505 19:02:07.264 | INFO | BAZWRITER : 63.1%, 3.8, 122.5ms, 0 | 20260505 19:02:07.264 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 19:02:07.264 | INFO | Uploaded 36GiB to GPU in 2.16399s (16.636GiB/s) | 20260505 19:02:07.264 | INFO | Downloaded 38GiB to GPU in 2.2429s (16.9424GiB/s) | 20260505 19:02:12.399 | INFO | Analyzing chunk frames = [5562880,5563392) | 20260505 19:02:12.399 | INFO | MemInfo: { | 20260505 19:02:12.399 | INFO | "gpu" : " 17 GB", | 20260505 19:02:12.399 | INFO | "huge" : "", | 20260505 19:02:12.399 | INFO | "huge_shared" : " 27 GB", | 20260505 19:02:12.399 | INFO | "malloc" : " 14 GB", | 20260505 19:02:12.399 | INFO | "pinned" : " 29 GB" | 20260505 19:02:12.399 | INFO | } | 20260505 19:02:12.399 | INFO | Perf report for the last 5.13542 seconds: | 20260505 19:02:12.399 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:02:12.399 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:12.399 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:12.399 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:12.399 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260505 19:02:12.399 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:12.399 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:12.399 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:02:12.399 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:12.399 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:12.399 | INFO | Uploaded 36GiB to GPU in 2.14144s (16.8111GiB/s) | 20260505 19:02:12.399 | INFO | Downloaded 34GiB to GPU in 2.03931s (16.6723GiB/s) | 20260505 19:02:17.540 | INFO | Analyzing chunk frames = [5563392,5563904) | 20260505 19:02:17.540 | INFO | MemInfo: { | 20260505 19:02:17.540 | INFO | "gpu" : " 17 GB", | 20260505 19:02:17.540 | INFO | "huge" : "", | 20260505 19:02:17.540 | INFO | "huge_shared" : " 27 GB", | 20260505 19:02:17.540 | INFO | "malloc" : " 14 GB", | 20260505 19:02:17.541 | INFO | "pinned" : " 29 GB" | 20260505 19:02:17.541 | INFO | } | 20260505 19:02:17.541 | INFO | Perf report for the last 5.14171 seconds: | 20260505 19:02:17.541 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:02:17.541 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:17.541 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:17.541 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:17.541 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 19:02:17.541 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:17.541 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:17.541 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 19:02:17.541 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:17.541 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:17.541 | INFO | Uploaded 36GiB to GPU in 2.20983s (16.2908GiB/s) | 20260505 19:02:17.541 | INFO | Downloaded 34GiB to GPU in 2.05152s (16.5731GiB/s) | 20260505 19:02:22.681 | INFO | Analyzing chunk frames = [5563904,5564416) | 20260505 19:02:22.681 | INFO | MemInfo: { | 20260505 19:02:22.681 | INFO | "gpu" : " 17 GB", | 20260505 19:02:22.681 | INFO | "huge" : "", | 20260505 19:02:22.681 | INFO | "huge_shared" : " 27 GB", | 20260505 19:02:22.681 | INFO | "malloc" : " 14 GB", | 20260505 19:02:22.681 | INFO | "pinned" : " 29 GB" | 20260505 19:02:22.681 | INFO | } | 20260505 19:02:22.681 | INFO | Perf report for the last 5.14051 seconds: | 20260505 19:02:22.681 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:02:22.681 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:22.681 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:22.681 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:22.681 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260505 19:02:22.681 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:22.682 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:22.682 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:02:22.682 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:22.682 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:22.682 | INFO | Uploaded 36GiB to GPU in 2.18526s (16.474GiB/s) | 20260505 19:02:22.682 | INFO | Downloaded 34GiB to GPU in 2.05461s (16.5482GiB/s) | 20260505 19:02:27.826 | INFO | Analyzing chunk frames = [5564416,5564928) | 20260505 19:02:27.826 | INFO | MemInfo: { | 20260505 19:02:27.826 | INFO | "gpu" : " 17 GB", | 20260505 19:02:27.826 | INFO | "huge" : "", | 20260505 19:02:27.826 | INFO | "huge_shared" : " 27 GB", | 20260505 19:02:27.826 | INFO | "malloc" : " 14 GB", | 20260505 19:02:27.826 | INFO | "pinned" : " 29 GB" | 20260505 19:02:27.826 | INFO | } | 20260505 19:02:27.826 | INFO | Perf report for the last 5.14481 seconds: | 20260505 19:02:27.826 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:02:27.826 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:27.826 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:27.826 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:27.826 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260505 19:02:27.826 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:27.827 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:27.827 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 19:02:27.827 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:27.827 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:27.827 | INFO | Uploaded 36GiB to GPU in 2.14502s (16.7831GiB/s) | 20260505 19:02:27.827 | INFO | Downloaded 34GiB to GPU in 2.05691s (16.5296GiB/s) | 20260505 19:02:32.963 | INFO | Analyzing chunk frames = [5564928,5565440) | 20260505 19:02:32.963 | INFO | MemInfo: { | 20260505 19:02:32.963 | INFO | "gpu" : " 17 GB", | 20260505 19:02:32.963 | INFO | "huge" : "", | 20260505 19:02:32.963 | INFO | "huge_shared" : " 27 GB", | 20260505 19:02:32.963 | INFO | "malloc" : " 14 GB", | 20260505 19:02:32.963 | INFO | "pinned" : " 29 GB" | 20260505 19:02:32.963 | INFO | } | 20260505 19:02:32.963 | INFO | Perf report for the last 5.13671 seconds: | 20260505 19:02:32.963 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:02:32.963 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:32.963 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:32.963 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:32.963 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 19:02:32.963 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:32.963 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:32.963 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:02:32.963 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:32.963 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:32.963 | INFO | Uploaded 36GiB to GPU in 2.1407s (16.8169GiB/s) | 20260505 19:02:32.963 | INFO | Downloaded 34GiB to GPU in 2.05857s (16.5163GiB/s) | 20260505 19:02:38.097 | INFO | Analyzing chunk frames = [5565440,5565952) | 20260505 19:02:38.097 | INFO | MemInfo: { | 20260505 19:02:38.097 | INFO | "gpu" : " 17 GB", | 20260505 19:02:38.097 | INFO | "huge" : "", | 20260505 19:02:38.097 | INFO | "huge_shared" : " 27 GB", | 20260505 19:02:38.097 | INFO | "malloc" : " 14 GB", | 20260505 19:02:38.097 | INFO | "pinned" : " 29 GB" | 20260505 19:02:38.097 | INFO | } | 20260505 19:02:38.097 | INFO | Perf report for the last 5.13466 seconds: | 20260505 19:02:38.097 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:02:38.097 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:38.097 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:38.098 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:38.098 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 19:02:38.098 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:38.098 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:38.098 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 19:02:38.098 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:38.098 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:38.098 | INFO | Uploaded 36GiB to GPU in 2.13096s (16.8938GiB/s) | 20260505 19:02:38.098 | INFO | Downloaded 34GiB to GPU in 2.13728s (15.9081GiB/s) | 20260505 19:02:43.249 | INFO | Analyzing chunk frames = [5565952,5566464) | 20260505 19:02:43.249 | INFO | MemInfo: { | 20260505 19:02:43.249 | INFO | "gpu" : " 17 GB", | 20260505 19:02:43.249 | INFO | "huge" : "", | 20260505 19:02:43.249 | INFO | "huge_shared" : " 27 GB", | 20260505 19:02:43.249 | INFO | "malloc" : " 14 GB", | 20260505 19:02:43.249 | INFO | "pinned" : " 29 GB" | 20260505 19:02:43.249 | INFO | } | 20260505 19:02:43.249 | INFO | Perf report for the last 5.15168 seconds: | 20260505 19:02:43.249 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:02:43.249 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:43.249 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:43.249 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:43.249 | INFO | ANALYSIS : 62.7%, 3.0, 98.2ms, 0 | 20260505 19:02:43.249 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:43.249 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:43.249 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260505 19:02:43.249 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:43.249 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:43.250 | INFO | Uploaded 36GiB to GPU in 2.19801s (16.3784GiB/s) | 20260505 19:02:43.250 | INFO | Downloaded 34GiB to GPU in 2.04367s (16.6367GiB/s) | 20260505 19:02:43.384 | INFO | Starting a round of Baz IO | 20260505 19:02:46.544 | INFO | Populating RTMetrics report | 20260505 19:02:46.545 | INFO | RTMetrics report populated | 20260505 19:02:46.545 | INFO | Opening tmp RTMetrics json file | 20260505 19:02:46.547 | INFO | Serializing RTMetrics to json | 20260505 19:02:46.549 | INFO | Serialization done, swapping tmp file now... | 20260505 19:02:46.549 | INFO | RTMetrics done | 20260505 19:02:46.639 | INFO | Finished a round of Baz IO: Wrote 2.068 GiB in 3.255 seconds | 20260505 19:02:46.639 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.83% / 0.00% / 78.17% | 20260505 19:02:46.639 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.224618% / 3.97364e-06% / 7.94729e-05% | 20260505 19:02:46.639 | INFO | Baz IO overall: | 20260505 19:02:46.639 | INFO | BytesWritten : 3267736833689 | 20260505 19:02:46.639 | INFO | NumEvents : 1547041881128 | 20260505 19:02:48.384 | INFO | Analyzing chunk frames = [5566464,5566976) | 20260505 19:02:48.384 | INFO | MemInfo: { | 20260505 19:02:48.384 | INFO | "gpu" : " 17 GB", | 20260505 19:02:48.384 | INFO | "huge" : "", | 20260505 19:02:48.384 | INFO | "huge_shared" : " 27 GB", | 20260505 19:02:48.384 | INFO | "malloc" : " 14 GB", | 20260505 19:02:48.384 | INFO | "pinned" : " 29 GB" | 20260505 19:02:48.384 | INFO | } | 20260505 19:02:48.384 | INFO | Perf report for the last 5.13533 seconds: | 20260505 19:02:48.384 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:02:48.384 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:48.384 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:48.384 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:48.385 | INFO | ANALYSIS : 63.1%, 3.0, 98.2ms, 0 | 20260505 19:02:48.385 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:48.385 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260505 19:02:48.385 | INFO | PRE_HQ : 60.1%, 1.8, 55.8ms, 0 | 20260505 19:02:48.385 | INFO | BAZWRITER : 62.9%, 3.8, 122.9ms, 0 | 20260505 19:02:48.385 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 19:02:48.385 | INFO | Uploaded 36GiB to GPU in 2.14477s (16.785GiB/s) | 20260505 19:02:48.385 | INFO | Downloaded 38GiB to GPU in 2.23702s (16.9869GiB/s) | 20260505 19:02:53.513 | INFO | Analyzing chunk frames = [5566976,5567488) | 20260505 19:02:53.513 | INFO | MemInfo: { | 20260505 19:02:53.513 | INFO | "gpu" : " 17 GB", | 20260505 19:02:53.513 | INFO | "huge" : "", | 20260505 19:02:53.513 | INFO | "huge_shared" : " 27 GB", | 20260505 19:02:53.513 | INFO | "malloc" : " 14 GB", | 20260505 19:02:53.513 | INFO | "pinned" : " 29 GB" | 20260505 19:02:53.513 | INFO | } | 20260505 19:02:53.513 | INFO | Perf report for the last 5.12837 seconds: | 20260505 19:02:53.513 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:02:53.513 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:53.513 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:53.513 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:53.513 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260505 19:02:53.513 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:53.513 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:53.513 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:02:53.513 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:53.513 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:53.513 | INFO | Uploaded 36GiB to GPU in 2.09838s (17.1561GiB/s) | 20260505 19:02:53.513 | INFO | Downloaded 34GiB to GPU in 2.02623s (16.78GiB/s) | 20260505 19:02:58.655 | INFO | Analyzing chunk frames = [5567488,5568000) | 20260505 19:02:58.655 | INFO | MemInfo: { | 20260505 19:02:58.655 | INFO | "gpu" : " 17 GB", | 20260505 19:02:58.655 | INFO | "huge" : "", | 20260505 19:02:58.655 | INFO | "huge_shared" : " 27 GB", | 20260505 19:02:58.655 | INFO | "malloc" : " 14 GB", | 20260505 19:02:58.655 | INFO | "pinned" : " 29 GB" | 20260505 19:02:58.655 | INFO | } | 20260505 19:02:58.656 | INFO | Perf report for the last 5.14274 seconds: | 20260505 19:02:58.656 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:02:58.656 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:58.656 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:58.656 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:58.656 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 19:02:58.656 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:58.656 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:58.656 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:02:58.656 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:58.656 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:02:58.656 | INFO | Uploaded 36GiB to GPU in 2.20567s (16.3216GiB/s) | 20260505 19:02:58.656 | INFO | Downloaded 34GiB to GPU in 2.0535s (16.5571GiB/s) | 20260505 19:03:03.796 | INFO | Analyzing chunk frames = [5568000,5568512) | 20260505 19:03:03.796 | INFO | MemInfo: { | 20260505 19:03:03.796 | INFO | "gpu" : " 17 GB", | 20260505 19:03:03.796 | INFO | "huge" : "", | 20260505 19:03:03.796 | INFO | "huge_shared" : " 27 GB", | 20260505 19:03:03.796 | INFO | "malloc" : " 14 GB", | 20260505 19:03:03.796 | INFO | "pinned" : " 29 GB" | 20260505 19:03:03.796 | INFO | } | 20260505 19:03:03.796 | INFO | Perf report for the last 5.14059 seconds: | 20260505 19:03:03.796 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:03:03.796 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:03.796 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:03.796 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:03.796 | INFO | ANALYSIS : 62.8%, 3.0, 97.6ms, 0 | 20260505 19:03:03.796 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:03.796 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:03.797 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 19:03:03.797 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:03.797 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:03.797 | INFO | Uploaded 36GiB to GPU in 2.22666s (16.1677GiB/s) | 20260505 19:03:03.797 | INFO | Downloaded 34GiB to GPU in 2.12788s (15.9783GiB/s) | 20260505 19:03:08.937 | INFO | Analyzing chunk frames = [5568512,5569024) | 20260505 19:03:08.937 | INFO | MemInfo: { | 20260505 19:03:08.937 | INFO | "gpu" : " 17 GB", | 20260505 19:03:08.937 | INFO | "huge" : "", | 20260505 19:03:08.937 | INFO | "huge_shared" : " 27 GB", | 20260505 19:03:08.937 | INFO | "malloc" : " 14 GB", | 20260505 19:03:08.937 | INFO | "pinned" : " 29 GB" | 20260505 19:03:08.937 | INFO | } | 20260505 19:03:08.937 | INFO | Perf report for the last 5.14041 seconds: | 20260505 19:03:08.937 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:03:08.937 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:08.937 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:08.937 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:08.937 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 19:03:08.937 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:08.937 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:08.937 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:03:08.937 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:08.937 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:08.937 | INFO | Uploaded 36GiB to GPU in 2.1474s (16.7645GiB/s) | 20260505 19:03:08.937 | INFO | Downloaded 34GiB to GPU in 2.0637s (16.4753GiB/s) | 20260505 19:03:14.079 | INFO | Analyzing chunk frames = [5569024,5569536) | 20260505 19:03:14.079 | INFO | MemInfo: { | 20260505 19:03:14.079 | INFO | "gpu" : " 17 GB", | 20260505 19:03:14.079 | INFO | "huge" : "", | 20260505 19:03:14.079 | INFO | "huge_shared" : " 27 GB", | 20260505 19:03:14.079 | INFO | "malloc" : " 14 GB", | 20260505 19:03:14.079 | INFO | "pinned" : " 29 GB" | 20260505 19:03:14.079 | INFO | } | 20260505 19:03:14.079 | INFO | Perf report for the last 5.14241 seconds: | 20260505 19:03:14.079 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:03:14.079 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:14.079 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:14.079 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:14.079 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260505 19:03:14.079 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:14.079 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:14.079 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:03:14.079 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:14.079 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:14.079 | INFO | Uploaded 36GiB to GPU in 2.13851s (16.8341GiB/s) | 20260505 19:03:14.079 | INFO | Downloaded 34GiB to GPU in 2.05452s (16.5489GiB/s) | 20260505 19:03:19.214 | INFO | Analyzing chunk frames = [5569536,5570048) | 20260505 19:03:19.214 | INFO | MemInfo: { | 20260505 19:03:19.214 | INFO | "gpu" : " 17 GB", | 20260505 19:03:19.214 | INFO | "huge" : "", | 20260505 19:03:19.214 | INFO | "huge_shared" : " 27 GB", | 20260505 19:03:19.214 | INFO | "malloc" : " 14 GB", | 20260505 19:03:19.214 | INFO | "pinned" : " 29 GB" | 20260505 19:03:19.214 | INFO | } | 20260505 19:03:19.214 | INFO | Perf report for the last 5.13489 seconds: | 20260505 19:03:19.214 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:03:19.214 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:19.214 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:19.214 | INFO | SAVE_TRACE : 0.1%, 1.0, 0.0ms, 0 | 20260505 19:03:19.214 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 19:03:19.214 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:19.214 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:19.214 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 19:03:19.214 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:19.214 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:19.214 | INFO | Uploaded 36GiB to GPU in 2.13608s (16.8533GiB/s) | 20260505 19:03:19.214 | INFO | Downloaded 34GiB to GPU in 2.13201s (15.9474GiB/s) | 20260505 19:03:24.352 | INFO | Analyzing chunk frames = [5570048,5570560) | 20260505 19:03:24.352 | INFO | MemInfo: { | 20260505 19:03:24.352 | INFO | "gpu" : " 17 GB", | 20260505 19:03:24.352 | INFO | "huge" : "", | 20260505 19:03:24.352 | INFO | "huge_shared" : " 27 GB", | 20260505 19:03:24.352 | INFO | "malloc" : " 14 GB", | 20260505 19:03:24.352 | INFO | "pinned" : " 29 GB" | 20260505 19:03:24.352 | INFO | } | 20260505 19:03:24.352 | INFO | Perf report for the last 5.13789 seconds: | 20260505 19:03:24.352 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:03:24.352 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:24.352 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:24.352 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:24.352 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 19:03:24.352 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:24.352 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:24.352 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 19:03:24.352 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:24.352 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:24.352 | INFO | Uploaded 36GiB to GPU in 2.19882s (16.3724GiB/s) | 20260505 19:03:24.352 | INFO | Downloaded 34GiB to GPU in 2.02809s (16.7645GiB/s) | 20260505 19:03:24.352 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 19:03:24.352 | INFO | Total mem usage: 15.3 GiB | 20260505 19:03:24.352 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 19:03:24.352 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.06%) | 20260505 19:03:24.352 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 19:03:24.352 | INFO | High Water Breakdown: | 20260505 19:03:24.352 | INFO | 61% : 8.66 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 19:03:24.352 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 19:03:24.352 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 19:03:24.352 | INFO | ------------------------------------------ | 20260505 19:03:24.352 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:03:24.352 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 19:03:24.352 | INFO | Total mem usage: 26.5 GiB | 20260505 19:03:24.353 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 19:03:24.353 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 19:03:24.353 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 19:03:24.353 | INFO | High Water Breakdown: | 20260505 19:03:24.353 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:03:24.353 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:03:24.353 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:03:24.353 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:03:24.353 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:03:24.353 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:03:24.353 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 19:03:24.353 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 19:03:24.353 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:03:24.353 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 19:03:24.353 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 19:03:24.353 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 19:03:24.353 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 19:03:24.353 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 19:03:24.353 | INFO | ------------------------------------------ | 20260505 19:03:24.353 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:03:24.353 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 19:03:24.353 | INFO | Total mem usage: 25.6 GiB | 20260505 19:03:24.353 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 19:03:24.353 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 19:03:24.353 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 19:03:24.353 | INFO | High Water Breakdown: | 20260505 19:03:24.353 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 19:03:24.353 | INFO | ------------------------------------------ | 20260505 19:03:24.353 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:03:24.353 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 19:03:24.353 | INFO | Total mem usage: 15.7 GiB | 20260505 19:03:24.353 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 19:03:24.353 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 19:03:24.353 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 19:03:24.353 | INFO | High Water Breakdown: | 20260505 19:03:24.353 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 19:03:24.353 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 19:03:24.353 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 19:03:24.353 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 19:03:24.353 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 19:03:24.353 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 19:03:24.353 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 19:03:24.353 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 19:03:24.353 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 19:03:24.353 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:03:24.353 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 19:03:24.353 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:03:24.353 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 19:03:24.353 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 19:03:24.353 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 19:03:24.353 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 19:03:24.353 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:03:24.353 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 19:03:24.353 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:03:24.353 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:03:24.353 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:03:24.353 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 19:03:24.353 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 19:03:24.353 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:03:24.353 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 19:03:24.353 | INFO | ------------------------------------------ | 20260505 19:03:24.353 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:03:24.353 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 19:03:24.353 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:03:24.353 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:03:24.353 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.46e+01, 1.35e+01 / 2.14e+00 | 20260505 19:03:24.353 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.03e-01, 1.03e-02 / 2.97e-03 | 20260505 19:03:24.353 | INFO | PerfReport: 11.77%, Baseline -- 3.71e+00 / 4.52e+00, 5.59e-01 / 1.72e-01 | 20260505 19:03:24.353 | INFO | PerfReport: 3.26%, Histogram -- 1.03e+00 / 1.54e+00, 1.89e-01 / 6.52e-02 | 20260505 19:03:24.353 | INFO | PerfReport: 15.96%, DME -- 4.02e+01 / 4.44e+01, 4.21e+00 / 2.08e+00 | 20260505 19:03:24.353 | INFO | PerfReport: 56.95%, FrameLabeling -- 1.79e+01 / 1.94e+01, 1.46e+00 / 1.19e+00 | 20260505 19:03:24.353 | INFO | PerfReport: 7.30%, PulseAccumulating -- 2.30e+00 / 2.86e+00, 4.41e-01 / 2.07e-01 | 20260505 19:03:24.353 | INFO | PerfReport: 3.31%, Metrics -- 1.04e+00 / 1.38e+00, 2.87e-01 / 1.50e-01 | 20260505 19:03:24.353 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.63e-01 / 4.28e-01, 6.36e-02 / 3.22e-02 | 20260505 19:03:24.353 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 19:03:24.353 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:03:24.353 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:03:24.353 | INFO | PerfReport: Full Execution -- 9.97e+01 / -nan, 2.57e+01 / 0.00e+00 | 20260505 19:03:24.353 | INFO | PerfReport: 22.60%, Upload -- 2.25e+01 / 0.00e+00, 7.86e+00 / 0.00e+00 | 20260505 19:03:24.354 | INFO | PerfReport: 21.82%, Download -- 2.18e+01 / 0.00e+00, 7.79e+00 / 0.00e+00 | 20260505 19:03:24.354 | INFO | PerfReport: 31.59%, Compute -- 3.15e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 19:03:24.354 | INFO | PerfReport: 2.42%, LockUpload -- 2.41e+00 / 0.00e+00, 8.04e+00 / 0.00e+00 | 20260505 19:03:24.354 | INFO | PerfReport: 1.31%, LockDownload -- 1.31e+00 / 0.00e+00, 2.76e+00 / 0.00e+00 | 20260505 19:03:24.354 | INFO | PerfReport: 20.25%, LockCompute -- 2.02e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 19:03:24.354 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.77e-02 / 0.00e+00 | 20260505 19:03:24.486 | INFO | Starting a round of Baz IO | 20260505 19:03:27.660 | INFO | Populating RTMetrics report | 20260505 19:03:27.660 | INFO | RTMetrics report populated | 20260505 19:03:27.660 | INFO | Opening tmp RTMetrics json file | 20260505 19:03:27.663 | INFO | Serializing RTMetrics to json | 20260505 19:03:27.664 | INFO | Serialization done, swapping tmp file now... | 20260505 19:03:27.665 | INFO | RTMetrics done | 20260505 19:03:27.753 | INFO | Finished a round of Baz IO: Wrote 2.069 GiB in 3.266 seconds | 20260505 19:03:27.753 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.83% / 0.00% / 78.17% | 20260505 19:03:27.753 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.223819% / 0% / 5.16574e-05% | 20260505 19:03:27.753 | INFO | Baz IO overall: | 20260505 19:03:27.753 | INFO | BytesWritten : 3269958008237 | 20260505 19:03:27.753 | INFO | NumEvents : 1548075679618 | 20260505 19:03:29.495 | INFO | Analyzing chunk frames = [5570560,5571072) | 20260505 19:03:29.495 | INFO | MemInfo: { | 20260505 19:03:29.495 | INFO | "gpu" : " 17 GB", | 20260505 19:03:29.495 | INFO | "huge" : "", | 20260505 19:03:29.495 | INFO | "huge_shared" : " 27 GB", | 20260505 19:03:29.495 | INFO | "malloc" : " 14 GB", | 20260505 19:03:29.495 | INFO | "pinned" : " 29 GB" | 20260505 19:03:29.495 | INFO | } | 20260505 19:03:29.495 | INFO | Perf report for the last 5.14338 seconds: | 20260505 19:03:29.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:03:29.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:29.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:29.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:29.495 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 19:03:29.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:29.495 | INFO | RT_METRICS : 44.3%, 1.0, 23.3ms, 0 | 20260505 19:03:29.495 | INFO | PRE_HQ : 60.3%, 1.8, 55.6ms, 0 | 20260505 19:03:29.495 | INFO | BAZWRITER : 63.0%, 3.8, 122.2ms, 0 | 20260505 19:03:29.495 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 19:03:29.496 | INFO | Uploaded 36GiB to GPU in 2.16523s (16.6264GiB/s) | 20260505 19:03:29.496 | INFO | Downloaded 38GiB to GPU in 2.24564s (16.9217GiB/s) | 20260505 19:03:34.635 | INFO | Analyzing chunk frames = [5571072,5571584) | 20260505 19:03:34.635 | INFO | MemInfo: { | 20260505 19:03:34.635 | INFO | "gpu" : " 17 GB", | 20260505 19:03:34.635 | INFO | "huge" : "", | 20260505 19:03:34.635 | INFO | "huge_shared" : " 27 GB", | 20260505 19:03:34.635 | INFO | "malloc" : " 14 GB", | 20260505 19:03:34.635 | INFO | "pinned" : " 29 GB" | 20260505 19:03:34.635 | INFO | } | 20260505 19:03:34.635 | INFO | Perf report for the last 5.1397 seconds: | 20260505 19:03:34.635 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:03:34.635 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:34.635 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:34.635 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:34.635 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260505 19:03:34.635 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:34.635 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:34.635 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:03:34.635 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:34.635 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:34.635 | INFO | Uploaded 36GiB to GPU in 2.10458s (17.1056GiB/s) | 20260505 19:03:34.635 | INFO | Downloaded 34GiB to GPU in 2.02854s (16.7609GiB/s) | 20260505 19:03:39.774 | INFO | Analyzing chunk frames = [5571584,5572096) | 20260505 19:03:39.774 | INFO | MemInfo: { | 20260505 19:03:39.774 | INFO | "gpu" : " 17 GB", | 20260505 19:03:39.774 | INFO | "huge" : "", | 20260505 19:03:39.774 | INFO | "huge_shared" : " 27 GB", | 20260505 19:03:39.774 | INFO | "malloc" : " 14 GB", | 20260505 19:03:39.774 | INFO | "pinned" : " 29 GB" | 20260505 19:03:39.774 | INFO | } | 20260505 19:03:39.774 | INFO | Perf report for the last 5.13932 seconds: | 20260505 19:03:39.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:03:39.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:39.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:39.775 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:39.775 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260505 19:03:39.775 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:39.775 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:39.775 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:03:39.775 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:39.775 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:39.775 | INFO | Uploaded 36GiB to GPU in 2.14475s (16.7852GiB/s) | 20260505 19:03:39.775 | INFO | Downloaded 34GiB to GPU in 2.04609s (16.6171GiB/s) | 20260505 19:03:44.908 | INFO | Analyzing chunk frames = [5572096,5572608) | 20260505 19:03:44.908 | INFO | MemInfo: { | 20260505 19:03:44.908 | INFO | "gpu" : " 17 GB", | 20260505 19:03:44.908 | INFO | "huge" : "", | 20260505 19:03:44.908 | INFO | "huge_shared" : " 27 GB", | 20260505 19:03:44.908 | INFO | "malloc" : " 14 GB", | 20260505 19:03:44.908 | INFO | "pinned" : " 29 GB" | 20260505 19:03:44.908 | INFO | } | 20260505 19:03:44.908 | INFO | Perf report for the last 5.13388 seconds: | 20260505 19:03:44.908 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:03:44.908 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:44.908 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:44.908 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:44.908 | INFO | ANALYSIS : 62.5%, 3.0, 97.0ms, 0 | 20260505 19:03:44.908 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:44.908 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:44.908 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:03:44.908 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:44.908 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:44.909 | INFO | Uploaded 36GiB to GPU in 2.18969s (16.4407GiB/s) | 20260505 19:03:44.909 | INFO | Downloaded 34GiB to GPU in 2.14212s (15.8721GiB/s) | 20260505 19:03:50.055 | INFO | Analyzing chunk frames = [5572608,5573120) | 20260505 19:03:50.055 | INFO | MemInfo: { | 20260505 19:03:50.055 | INFO | "gpu" : " 17 GB", | 20260505 19:03:50.055 | INFO | "huge" : "", | 20260505 19:03:50.055 | INFO | "huge_shared" : " 27 GB", | 20260505 19:03:50.055 | INFO | "malloc" : " 14 GB", | 20260505 19:03:50.055 | INFO | "pinned" : " 29 GB" | 20260505 19:03:50.055 | INFO | } | 20260505 19:03:50.056 | INFO | Perf report for the last 5.14733 seconds: | 20260505 19:03:50.056 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:03:50.056 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:50.056 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:50.056 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:50.056 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260505 19:03:50.056 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:50.056 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:50.056 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:03:50.056 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:50.056 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:50.056 | INFO | Uploaded 36GiB to GPU in 2.14468s (16.7857GiB/s) | 20260505 19:03:50.056 | INFO | Downloaded 34GiB to GPU in 2.05913s (16.5118GiB/s) | 20260505 19:03:55.197 | INFO | Analyzing chunk frames = [5573120,5573632) | 20260505 19:03:55.197 | INFO | MemInfo: { | 20260505 19:03:55.197 | INFO | "gpu" : " 17 GB", | 20260505 19:03:55.198 | INFO | "huge" : "", | 20260505 19:03:55.198 | INFO | "huge_shared" : " 27 GB", | 20260505 19:03:55.198 | INFO | "malloc" : " 14 GB", | 20260505 19:03:55.198 | INFO | "pinned" : " 29 GB" | 20260505 19:03:55.198 | INFO | } | 20260505 19:03:55.198 | INFO | Perf report for the last 5.14222 seconds: | 20260505 19:03:55.198 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:03:55.198 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:55.198 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:55.198 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:55.198 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 19:03:55.198 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:55.198 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:55.198 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:03:55.198 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:55.198 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:03:55.198 | INFO | Uploaded 36GiB to GPU in 2.13994s (16.8229GiB/s) | 20260505 19:03:55.198 | INFO | Downloaded 34GiB to GPU in 2.05182s (16.5706GiB/s) | 20260505 19:04:00.330 | INFO | Analyzing chunk frames = [5573632,5574144) | 20260505 19:04:00.330 | INFO | MemInfo: { | 20260505 19:04:00.330 | INFO | "gpu" : " 17 GB", | 20260505 19:04:00.330 | INFO | "huge" : "", | 20260505 19:04:00.330 | INFO | "huge_shared" : " 27 GB", | 20260505 19:04:00.330 | INFO | "malloc" : " 14 GB", | 20260505 19:04:00.330 | INFO | "pinned" : " 29 GB" | 20260505 19:04:00.330 | INFO | } | 20260505 19:04:00.330 | INFO | Perf report for the last 5.13192 seconds: | 20260505 19:04:00.330 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:04:00.330 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:00.330 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:00.330 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:00.330 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 19:04:00.330 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:00.330 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:00.330 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 19:04:00.330 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:00.330 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:00.330 | INFO | Uploaded 36GiB to GPU in 2.13826s (16.8361GiB/s) | 20260505 19:04:00.330 | INFO | Downloaded 34GiB to GPU in 2.12939s (15.967GiB/s) | 20260505 19:04:05.461 | INFO | Analyzing chunk frames = [5574144,5574656) | 20260505 19:04:05.461 | INFO | MemInfo: { | 20260505 19:04:05.461 | INFO | "gpu" : " 17 GB", | 20260505 19:04:05.461 | INFO | "huge" : "", | 20260505 19:04:05.461 | INFO | "huge_shared" : " 27 GB", | 20260505 19:04:05.461 | INFO | "malloc" : " 14 GB", | 20260505 19:04:05.461 | INFO | "pinned" : " 29 GB" | 20260505 19:04:05.461 | INFO | } | 20260505 19:04:05.461 | INFO | Perf report for the last 5.13131 seconds: | 20260505 19:04:05.461 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:04:05.461 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:05.461 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:05.461 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:05.461 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 19:04:05.461 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:05.461 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:05.461 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 19:04:05.462 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:05.462 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:05.462 | INFO | Uploaded 36GiB to GPU in 2.20058s (16.3593GiB/s) | 20260505 19:04:05.462 | INFO | Downloaded 34GiB to GPU in 2.03621s (16.6977GiB/s) | 20260505 19:04:05.594 | INFO | Starting a round of Baz IO | 20260505 19:04:08.786 | INFO | Populating RTMetrics report | 20260505 19:04:08.787 | INFO | RTMetrics report populated | 20260505 19:04:08.787 | INFO | Opening tmp RTMetrics json file | 20260505 19:04:08.825 | INFO | Serializing RTMetrics to json | 20260505 19:04:08.872 | INFO | Serialization done, swapping tmp file now... | 20260505 19:04:08.874 | INFO | RTMetrics done | 20260505 19:04:09.009 | INFO | Finished a round of Baz IO: Wrote 2.067 GiB in 3.414 seconds | 20260505 19:04:09.009 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.82% / 0.00% / 78.17% | 20260505 19:04:09.009 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.223672% / 0% / 6.75519e-05% | 20260505 19:04:09.009 | INFO | Baz IO overall: | 20260505 19:04:09.009 | INFO | BytesWritten : 3272177192472 | 20260505 19:04:09.009 | INFO | NumEvents : 1549108065943 | 20260505 19:04:10.603 | INFO | Analyzing chunk frames = [5574656,5575168) | 20260505 19:04:10.603 | INFO | MemInfo: { | 20260505 19:04:10.603 | INFO | "gpu" : " 17 GB", | 20260505 19:04:10.603 | INFO | "huge" : "", | 20260505 19:04:10.603 | INFO | "huge_shared" : " 27 GB", | 20260505 19:04:10.603 | INFO | "malloc" : " 14 GB", | 20260505 19:04:10.603 | INFO | "pinned" : " 29 GB" | 20260505 19:04:10.603 | INFO | } | 20260505 19:04:10.603 | INFO | Perf report for the last 5.14192 seconds: | 20260505 19:04:10.603 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:04:10.603 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:10.603 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:10.603 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:10.603 | INFO | ANALYSIS : 63.6%, 2.9, 97.9ms, 0 | 20260505 19:04:10.603 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:10.603 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260505 19:04:10.603 | INFO | PRE_HQ : 60.5%, 1.8, 55.1ms, 0 | 20260505 19:04:10.603 | INFO | BAZWRITER : 65.7%, 4.1, 139.9ms, 0 | 20260505 19:04:10.603 | INFO | BAZ_SAMPLING : 2.1%, 1.0, 1.1ms, 0 | 20260505 19:04:10.603 | INFO | Uploaded 36GiB to GPU in 2.19524s (16.3991GiB/s) | 20260505 19:04:10.603 | INFO | Downloaded 38GiB to GPU in 2.28168s (16.6544GiB/s) | 20260505 19:04:15.750 | INFO | Analyzing chunk frames = [5575168,5575680) | 20260505 19:04:15.750 | INFO | MemInfo: { | 20260505 19:04:15.750 | INFO | "gpu" : " 17 GB", | 20260505 19:04:15.750 | INFO | "huge" : "", | 20260505 19:04:15.750 | INFO | "huge_shared" : " 27 GB", | 20260505 19:04:15.750 | INFO | "malloc" : " 14 GB", | 20260505 19:04:15.750 | INFO | "pinned" : " 29 GB" | 20260505 19:04:15.750 | INFO | } | 20260505 19:04:15.750 | INFO | Perf report for the last 5.14729 seconds: | 20260505 19:04:15.750 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:04:15.750 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:15.750 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:15.750 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:15.750 | INFO | ANALYSIS : 62.7%, 3.0, 98.0ms, 0 | 20260505 19:04:15.750 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:15.750 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:15.750 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:04:15.750 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:15.750 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:15.751 | INFO | Uploaded 36GiB to GPU in 2.12982s (16.9028GiB/s) | 20260505 19:04:15.751 | INFO | Downloaded 34GiB to GPU in 2.02217s (16.8136GiB/s) | 20260505 19:04:20.890 | INFO | Analyzing chunk frames = [5575680,5576192) | 20260505 19:04:20.890 | INFO | MemInfo: { | 20260505 19:04:20.890 | INFO | "gpu" : " 17 GB", | 20260505 19:04:20.890 | INFO | "huge" : "", | 20260505 19:04:20.890 | INFO | "huge_shared" : " 27 GB", | 20260505 19:04:20.890 | INFO | "malloc" : " 14 GB", | 20260505 19:04:20.890 | INFO | "pinned" : " 29 GB" | 20260505 19:04:20.890 | INFO | } | 20260505 19:04:20.890 | INFO | Perf report for the last 5.13966 seconds: | 20260505 19:04:20.890 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:04:20.890 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:20.890 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:20.890 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:20.890 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260505 19:04:20.890 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:20.890 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:20.890 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:04:20.890 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:20.890 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:20.890 | INFO | Uploaded 36GiB to GPU in 2.12067s (16.9758GiB/s) | 20260505 19:04:20.890 | INFO | Downloaded 34GiB to GPU in 2.02011s (16.8308GiB/s) | 20260505 19:04:26.028 | INFO | Analyzing chunk frames = [5576192,5576704) | 20260505 19:04:26.028 | INFO | MemInfo: { | 20260505 19:04:26.028 | INFO | "gpu" : " 17 GB", | 20260505 19:04:26.028 | INFO | "huge" : "", | 20260505 19:04:26.028 | INFO | "huge_shared" : " 27 GB", | 20260505 19:04:26.028 | INFO | "malloc" : " 14 GB", | 20260505 19:04:26.028 | INFO | "pinned" : " 29 GB" | 20260505 19:04:26.028 | INFO | } | 20260505 19:04:26.028 | INFO | Perf report for the last 5.13785 seconds: | 20260505 19:04:26.028 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:04:26.028 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:26.028 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:26.028 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:26.028 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260505 19:04:26.028 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:26.028 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:26.028 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:04:26.028 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:26.028 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:26.028 | INFO | Uploaded 36GiB to GPU in 2.18787s (16.4544GiB/s) | 20260505 19:04:26.028 | INFO | Downloaded 34GiB to GPU in 2.04505s (16.6255GiB/s) | 20260505 19:04:31.160 | INFO | Analyzing chunk frames = [5576704,5577216) | 20260505 19:04:31.160 | INFO | MemInfo: { | 20260505 19:04:31.160 | INFO | "gpu" : " 17 GB", | 20260505 19:04:31.160 | INFO | "huge" : "", | 20260505 19:04:31.160 | INFO | "huge_shared" : " 27 GB", | 20260505 19:04:31.160 | INFO | "malloc" : " 14 GB", | 20260505 19:04:31.160 | INFO | "pinned" : " 29 GB" | 20260505 19:04:31.160 | INFO | } | 20260505 19:04:31.160 | INFO | Perf report for the last 5.13253 seconds: | 20260505 19:04:31.160 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:04:31.160 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:31.160 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:31.161 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:31.161 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260505 19:04:31.161 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:31.161 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:31.161 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260505 19:04:31.161 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:31.161 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:31.161 | INFO | Uploaded 36GiB to GPU in 2.14496s (16.7835GiB/s) | 20260505 19:04:31.161 | INFO | Downloaded 34GiB to GPU in 2.06203s (16.4886GiB/s) | 20260505 19:04:36.307 | INFO | Analyzing chunk frames = [5577216,5577728) | 20260505 19:04:36.307 | INFO | MemInfo: { | 20260505 19:04:36.307 | INFO | "gpu" : " 17 GB", | 20260505 19:04:36.307 | INFO | "huge" : "", | 20260505 19:04:36.307 | INFO | "huge_shared" : " 27 GB", | 20260505 19:04:36.307 | INFO | "malloc" : " 14 GB", | 20260505 19:04:36.307 | INFO | "pinned" : " 29 GB" | 20260505 19:04:36.307 | INFO | } | 20260505 19:04:36.307 | INFO | Perf report for the last 5.14657 seconds: | 20260505 19:04:36.307 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:04:36.307 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:36.307 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:36.307 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:36.307 | INFO | ANALYSIS : 62.3%, 3.0, 97.4ms, 0 | 20260505 19:04:36.307 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:36.307 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:36.307 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 19:04:36.307 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:36.307 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:36.307 | INFO | Uploaded 36GiB to GPU in 2.14482s (16.7846GiB/s) | 20260505 19:04:36.307 | INFO | Downloaded 34GiB to GPU in 2.04769s (16.6041GiB/s) | 20260505 19:04:41.441 | INFO | Analyzing chunk frames = [5577728,5578240) | 20260505 19:04:41.441 | INFO | MemInfo: { | 20260505 19:04:41.441 | INFO | "gpu" : " 17 GB", | 20260505 19:04:41.441 | INFO | "huge" : "", | 20260505 19:04:41.441 | INFO | "huge_shared" : " 27 GB", | 20260505 19:04:41.441 | INFO | "malloc" : " 14 GB", | 20260505 19:04:41.441 | INFO | "pinned" : " 29 GB" | 20260505 19:04:41.442 | INFO | } | 20260505 19:04:41.442 | INFO | Perf report for the last 5.13474 seconds: | 20260505 19:04:41.442 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:04:41.442 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:41.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:41.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:41.442 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 19:04:41.442 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:41.442 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:41.442 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 19:04:41.442 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:41.442 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:41.442 | INFO | Uploaded 36GiB to GPU in 2.13447s (16.866GiB/s) | 20260505 19:04:41.442 | INFO | Downloaded 34GiB to GPU in 2.14859s (15.8243GiB/s) | 20260505 19:04:46.584 | INFO | Analyzing chunk frames = [5578240,5578752) | 20260505 19:04:46.584 | INFO | MemInfo: { | 20260505 19:04:46.584 | INFO | "gpu" : " 17 GB", | 20260505 19:04:46.584 | INFO | "huge" : "", | 20260505 19:04:46.584 | INFO | "huge_shared" : " 27 GB", | 20260505 19:04:46.584 | INFO | "malloc" : " 14 GB", | 20260505 19:04:46.584 | INFO | "pinned" : " 29 GB" | 20260505 19:04:46.584 | INFO | } | 20260505 19:04:46.584 | INFO | Perf report for the last 5.14223 seconds: | 20260505 19:04:46.584 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:04:46.584 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:46.584 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:46.584 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:46.584 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 19:04:46.584 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:46.584 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:46.584 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 19:04:46.584 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:46.584 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:46.584 | INFO | Uploaded 36GiB to GPU in 2.19501s (16.4009GiB/s) | 20260505 19:04:46.584 | INFO | Downloaded 34GiB to GPU in 2.02922s (16.7552GiB/s) | 20260505 19:04:46.584 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 19:04:46.584 | INFO | Total mem usage: 15.3 GiB | 20260505 19:04:46.584 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 19:04:46.584 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -1.83%) | 20260505 19:04:46.584 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 19:04:46.584 | INFO | High Water Breakdown: | 20260505 19:04:46.584 | INFO | 61.9% : 9.32 GiB ( 98% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 19:04:46.584 | INFO | 37.9% : 5.71 GiB ( 85.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 19:04:46.584 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 19:04:46.585 | INFO | ------------------------------------------ | 20260505 19:04:46.585 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:04:46.585 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 19:04:46.585 | INFO | Total mem usage: 26.5 GiB | 20260505 19:04:46.585 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 19:04:46.585 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 19:04:46.585 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 19:04:46.585 | INFO | High Water Breakdown: | 20260505 19:04:46.585 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:04:46.585 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:04:46.585 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:04:46.585 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:04:46.585 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:04:46.585 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:04:46.585 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 19:04:46.585 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 19:04:46.585 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:04:46.585 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 19:04:46.585 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 19:04:46.585 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 19:04:46.585 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 19:04:46.585 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 19:04:46.585 | INFO | ------------------------------------------ | 20260505 19:04:46.585 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:04:46.585 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 19:04:46.585 | INFO | Total mem usage: 25.6 GiB | 20260505 19:04:46.585 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 19:04:46.585 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 19:04:46.585 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 19:04:46.585 | INFO | High Water Breakdown: | 20260505 19:04:46.585 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 19:04:46.585 | INFO | ------------------------------------------ | 20260505 19:04:46.585 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:04:46.585 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 19:04:46.585 | INFO | Total mem usage: 15.7 GiB | 20260505 19:04:46.586 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 19:04:46.586 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 19:04:46.586 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 19:04:46.586 | INFO | High Water Breakdown: | 20260505 19:04:46.586 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 19:04:46.586 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 19:04:46.586 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 19:04:46.586 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 19:04:46.586 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 19:04:46.586 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 19:04:46.586 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 19:04:46.586 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 19:04:46.586 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 19:04:46.586 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:04:46.586 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 19:04:46.586 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:04:46.586 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 19:04:46.586 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 19:04:46.586 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 19:04:46.586 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 19:04:46.586 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:04:46.586 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 19:04:46.586 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:04:46.586 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:04:46.586 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:04:46.586 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 19:04:46.586 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 19:04:46.586 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:04:46.586 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 19:04:46.586 | INFO | ------------------------------------------ | 20260505 19:04:46.586 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:04:46.586 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 19:04:46.586 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:04:46.586 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:04:46.586 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.48e+01, 1.36e+01 / 1.57e+00 | 20260505 19:04:46.586 | INFO | PerfReport: 0.29%, AggregateStats -- 9.24e-02 / 1.02e-01, 1.09e-02 / 4.35e-03 | 20260505 19:04:46.586 | INFO | PerfReport: 11.80%, Baseline -- 3.72e+00 / 4.57e+00, 5.76e-01 / 1.02e-01 | 20260505 19:04:46.586 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.54e+00, 1.91e-01 / 7.26e-02 | 20260505 19:04:46.586 | INFO | PerfReport: 15.94%, DME -- 4.02e+01 / 4.46e+01, 4.00e+00 / 1.72e+00 | 20260505 19:04:46.586 | INFO | PerfReport: 56.96%, FrameLabeling -- 1.80e+01 / 1.93e+01, 1.51e+00 / 1.13e+00 | 20260505 19:04:46.586 | INFO | PerfReport: 7.27%, PulseAccumulating -- 2.29e+00 / 2.89e+00, 4.38e-01 / 1.70e-01 | 20260505 19:04:46.586 | INFO | PerfReport: 3.31%, Metrics -- 1.04e+00 / 1.40e+00, 2.88e-01 / 1.87e-01 | 20260505 19:04:46.586 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.62e-01 / 4.26e-01, 6.39e-02 / 3.57e-02 | 20260505 19:04:46.586 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 19:04:46.586 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:04:46.586 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:04:46.586 | INFO | PerfReport: Full Execution -- 9.96e+01 / -nan, 2.57e+01 / 0.00e+00 | 20260505 19:04:46.586 | INFO | PerfReport: 22.54%, Upload -- 2.24e+01 / 0.00e+00, 7.77e+00 / 0.00e+00 | 20260505 19:04:46.586 | INFO | PerfReport: 21.83%, Download -- 2.17e+01 / 0.00e+00, 7.72e+00 / 0.00e+00 | 20260505 19:04:46.586 | INFO | PerfReport: 31.69%, Compute -- 3.16e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 19:04:46.586 | INFO | PerfReport: 2.28%, LockUpload -- 2.27e+00 / 0.00e+00, 7.93e+00 / 0.00e+00 | 20260505 19:04:46.586 | INFO | PerfReport: 1.40%, LockDownload -- 1.40e+00 / 0.00e+00, 2.95e+00 / 0.00e+00 | 20260505 19:04:46.586 | INFO | PerfReport: 20.24%, LockCompute -- 2.02e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260505 19:04:46.586 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.82e-02 / 0.00e+00 | 20260505 19:04:46.716 | INFO | Starting a round of Baz IO | 20260505 19:04:49.893 | INFO | Populating RTMetrics report | 20260505 19:04:49.893 | INFO | RTMetrics report populated | 20260505 19:04:49.893 | INFO | Opening tmp RTMetrics json file | 20260505 19:04:49.894 | INFO | Serializing RTMetrics to json | 20260505 19:04:49.895 | INFO | Serialization done, swapping tmp file now... | 20260505 19:04:49.896 | INFO | RTMetrics done | 20260505 19:04:49.984 | INFO | Finished a round of Baz IO: Wrote 2.065 GiB in 3.268 seconds | 20260505 19:04:49.984 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.82% / 0.00% / 78.18% | 20260505 19:04:49.984 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.222492% / 0% / 8.74201e-05% | 20260505 19:04:49.985 | INFO | Baz IO overall: | 20260505 19:04:49.985 | INFO | BytesWritten : 3274393986307 | 20260505 19:04:49.985 | INFO | NumEvents : 1550138923814 | 20260505 19:04:51.722 | INFO | Analyzing chunk frames = [5578752,5579264) | 20260505 19:04:51.722 | INFO | MemInfo: { | 20260505 19:04:51.722 | INFO | "gpu" : " 17 GB", | 20260505 19:04:51.722 | INFO | "huge" : "", | 20260505 19:04:51.722 | INFO | "huge_shared" : " 27 GB", | 20260505 19:04:51.722 | INFO | "malloc" : " 14 GB", | 20260505 19:04:51.722 | INFO | "pinned" : " 29 GB" | 20260505 19:04:51.722 | INFO | } | 20260505 19:04:51.722 | INFO | Perf report for the last 5.1384 seconds: | 20260505 19:04:51.722 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:04:51.722 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:51.722 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:51.722 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:51.722 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 19:04:51.722 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:51.722 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260505 19:04:51.722 | INFO | PRE_HQ : 60.4%, 1.8, 55.5ms, 0 | 20260505 19:04:51.722 | INFO | BAZWRITER : 63.1%, 3.8, 122.3ms, 0 | 20260505 19:04:51.722 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 19:04:51.723 | INFO | Uploaded 36GiB to GPU in 2.15637s (16.6947GiB/s) | 20260505 19:04:51.723 | INFO | Downloaded 38GiB to GPU in 2.23557s (16.9979GiB/s) | 20260505 19:04:56.859 | INFO | Analyzing chunk frames = [5579264,5579776) | 20260505 19:04:56.859 | INFO | MemInfo: { | 20260505 19:04:56.859 | INFO | "gpu" : " 17 GB", | 20260505 19:04:56.859 | INFO | "huge" : "", | 20260505 19:04:56.859 | INFO | "huge_shared" : " 27 GB", | 20260505 19:04:56.859 | INFO | "malloc" : " 14 GB", | 20260505 19:04:56.859 | INFO | "pinned" : " 29 GB" | 20260505 19:04:56.859 | INFO | } | 20260505 19:04:56.859 | INFO | Perf report for the last 5.13698 seconds: | 20260505 19:04:56.859 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:04:56.859 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:56.860 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:56.860 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:56.860 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 19:04:56.860 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:56.860 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:56.860 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:04:56.860 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:56.860 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:04:56.860 | INFO | Uploaded 36GiB to GPU in 2.13942s (16.827GiB/s) | 20260505 19:04:56.860 | INFO | Downloaded 34GiB to GPU in 2.03581s (16.7009GiB/s) | 20260505 19:05:02.008 | INFO | Analyzing chunk frames = [5579776,5580288) | 20260505 19:05:02.008 | INFO | MemInfo: { | 20260505 19:05:02.008 | INFO | "gpu" : " 17 GB", | 20260505 19:05:02.008 | INFO | "huge" : "", | 20260505 19:05:02.008 | INFO | "huge_shared" : " 27 GB", | 20260505 19:05:02.008 | INFO | "malloc" : " 14 GB", | 20260505 19:05:02.008 | INFO | "pinned" : " 29 GB" | 20260505 19:05:02.008 | INFO | } | 20260505 19:05:02.009 | INFO | Perf report for the last 5.1492 seconds: | 20260505 19:05:02.009 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:05:02.009 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:02.009 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:02.009 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:02.009 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 19:05:02.009 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:02.009 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:02.009 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:05:02.009 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:02.009 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:02.009 | INFO | Uploaded 36GiB to GPU in 2.2146s (16.2558GiB/s) | 20260505 19:05:02.009 | INFO | Downloaded 34GiB to GPU in 2.07652s (16.3735GiB/s) | 20260505 19:05:07.150 | INFO | Analyzing chunk frames = [5580288,5580800) | 20260505 19:05:07.150 | INFO | MemInfo: { | 20260505 19:05:07.150 | INFO | "gpu" : " 17 GB", | 20260505 19:05:07.150 | INFO | "huge" : "", | 20260505 19:05:07.150 | INFO | "huge_shared" : " 27 GB", | 20260505 19:05:07.150 | INFO | "malloc" : " 14 GB", | 20260505 19:05:07.150 | INFO | "pinned" : " 29 GB" | 20260505 19:05:07.150 | INFO | } | 20260505 19:05:07.150 | INFO | Perf report for the last 5.14192 seconds: | 20260505 19:05:07.151 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:05:07.151 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:07.151 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:07.151 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:07.151 | INFO | ANALYSIS : 62.5%, 3.0, 97.1ms, 0 | 20260505 19:05:07.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:07.151 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:07.151 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260505 19:05:07.151 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:07.151 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:07.151 | INFO | Uploaded 36GiB to GPU in 2.21154s (16.2783GiB/s) | 20260505 19:05:07.151 | INFO | Downloaded 34GiB to GPU in 2.06597s (16.4571GiB/s) | 20260505 19:05:12.282 | INFO | Analyzing chunk frames = [5580800,5581312) | 20260505 19:05:12.282 | INFO | MemInfo: { | 20260505 19:05:12.282 | INFO | "gpu" : " 17 GB", | 20260505 19:05:12.282 | INFO | "huge" : "", | 20260505 19:05:12.282 | INFO | "huge_shared" : " 27 GB", | 20260505 19:05:12.282 | INFO | "malloc" : " 14 GB", | 20260505 19:05:12.282 | INFO | "pinned" : " 29 GB" | 20260505 19:05:12.282 | INFO | } | 20260505 19:05:12.282 | INFO | Perf report for the last 5.13122 seconds: | 20260505 19:05:12.282 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:05:12.282 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:12.282 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:12.282 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:12.282 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260505 19:05:12.282 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:12.282 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:12.282 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:05:12.282 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:12.282 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:12.282 | INFO | Uploaded 36GiB to GPU in 2.13368s (16.8723GiB/s) | 20260505 19:05:12.282 | INFO | Downloaded 34GiB to GPU in 2.06846s (16.4373GiB/s) | 20260505 19:05:17.411 | INFO | Analyzing chunk frames = [5581312,5581824) | 20260505 19:05:17.411 | INFO | MemInfo: { | 20260505 19:05:17.411 | INFO | "gpu" : " 17 GB", | 20260505 19:05:17.411 | INFO | "huge" : "", | 20260505 19:05:17.411 | INFO | "huge_shared" : " 27 GB", | 20260505 19:05:17.411 | INFO | "malloc" : " 14 GB", | 20260505 19:05:17.411 | INFO | "pinned" : " 29 GB" | 20260505 19:05:17.411 | INFO | } | 20260505 19:05:17.411 | INFO | Perf report for the last 5.12905 seconds: | 20260505 19:05:17.411 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:05:17.411 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:17.411 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:17.411 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:17.411 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260505 19:05:17.411 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:17.411 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:17.411 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 19:05:17.411 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:17.411 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:17.411 | INFO | Uploaded 36GiB to GPU in 2.13917s (16.829GiB/s) | 20260505 19:05:17.411 | INFO | Downloaded 34GiB to GPU in 2.05408s (16.5524GiB/s) | 20260505 19:05:22.563 | INFO | Analyzing chunk frames = [5581824,5582336) | 20260505 19:05:22.563 | INFO | MemInfo: { | 20260505 19:05:22.563 | INFO | "gpu" : " 17 GB", | 20260505 19:05:22.563 | INFO | "huge" : "", | 20260505 19:05:22.563 | INFO | "huge_shared" : " 27 GB", | 20260505 19:05:22.563 | INFO | "malloc" : " 14 GB", | 20260505 19:05:22.563 | INFO | "pinned" : " 29 GB" | 20260505 19:05:22.563 | INFO | } | 20260505 19:05:22.563 | INFO | Perf report for the last 5.15246 seconds: | 20260505 19:05:22.563 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:05:22.563 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:22.563 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:22.563 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:22.563 | INFO | ANALYSIS : 63.4%, 3.0, 99.2ms, 0 | 20260505 19:05:22.563 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:22.563 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:22.563 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 19:05:22.563 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:22.563 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:22.564 | INFO | Uploaded 36GiB to GPU in 2.14255s (16.8024GiB/s) | 20260505 19:05:22.564 | INFO | Downloaded 34GiB to GPU in 2.18369s (15.57GiB/s) | 20260505 19:05:27.705 | INFO | Analyzing chunk frames = [5582336,5582848) | 20260505 19:05:27.705 | INFO | MemInfo: { | 20260505 19:05:27.705 | INFO | "gpu" : " 17 GB", | 20260505 19:05:27.705 | INFO | "huge" : "", | 20260505 19:05:27.705 | INFO | "huge_shared" : " 27 GB", | 20260505 19:05:27.705 | INFO | "malloc" : " 14 GB", | 20260505 19:05:27.705 | INFO | "pinned" : " 29 GB" | 20260505 19:05:27.705 | INFO | } | 20260505 19:05:27.706 | INFO | Perf report for the last 5.14229 seconds: | 20260505 19:05:27.706 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:05:27.706 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:27.706 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:27.706 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:27.706 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 19:05:27.706 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:27.706 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:27.706 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:05:27.706 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:27.706 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:27.706 | INFO | Uploaded 36GiB to GPU in 2.20152s (16.3523GiB/s) | 20260505 19:05:27.706 | INFO | Downloaded 34GiB to GPU in 2.0433s (16.6397GiB/s) | 20260505 19:05:27.841 | INFO | Starting a round of Baz IO | 20260505 19:05:31.007 | INFO | Populating RTMetrics report | 20260505 19:05:31.008 | INFO | RTMetrics report populated | 20260505 19:05:31.008 | INFO | Opening tmp RTMetrics json file | 20260505 19:05:31.017 | INFO | Serializing RTMetrics to json | 20260505 19:05:31.019 | INFO | Serialization done, swapping tmp file now... | 20260505 19:05:31.020 | INFO | RTMetrics done | 20260505 19:05:31.118 | INFO | Finished a round of Baz IO: Wrote 2.066 GiB in 3.276 seconds | 20260505 19:05:31.118 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.82% / 0.00% / 78.18% | 20260505 19:05:31.118 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.222154% / 3.97364e-06% / 9.13938e-05% | 20260505 19:05:31.118 | INFO | Baz IO overall: | 20260505 19:05:31.118 | INFO | BytesWritten : 3276612049663 | 20260505 19:05:31.118 | INFO | NumEvents : 1551170712117 | 20260505 19:05:32.841 | INFO | Analyzing chunk frames = [5582848,5583360) | 20260505 19:05:32.841 | INFO | MemInfo: { | 20260505 19:05:32.841 | INFO | "gpu" : " 17 GB", | 20260505 19:05:32.841 | INFO | "huge" : "", | 20260505 19:05:32.841 | INFO | "huge_shared" : " 27 GB", | 20260505 19:05:32.841 | INFO | "malloc" : " 14 GB", | 20260505 19:05:32.841 | INFO | "pinned" : " 29 GB" | 20260505 19:05:32.841 | INFO | } | 20260505 19:05:32.842 | INFO | Perf report for the last 5.13595 seconds: | 20260505 19:05:32.842 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:05:32.842 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:32.842 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:32.842 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:32.842 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 19:05:32.842 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:32.842 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260505 19:05:32.842 | INFO | PRE_HQ : 60.4%, 1.8, 55.9ms, 0 | 20260505 19:05:32.842 | INFO | BAZWRITER : 63.3%, 3.8, 125.1ms, 0 | 20260505 19:05:32.842 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 19:05:32.842 | INFO | Uploaded 36GiB to GPU in 2.18041s (16.5107GiB/s) | 20260505 19:05:32.842 | INFO | Downloaded 38GiB to GPU in 2.25492s (16.8521GiB/s) | 20260505 19:05:37.975 | INFO | Analyzing chunk frames = [5583360,5583872) | 20260505 19:05:37.976 | INFO | MemInfo: { | 20260505 19:05:37.976 | INFO | "gpu" : " 17 GB", | 20260505 19:05:37.976 | INFO | "huge" : "", | 20260505 19:05:37.976 | INFO | "huge_shared" : " 27 GB", | 20260505 19:05:37.976 | INFO | "malloc" : " 14 GB", | 20260505 19:05:37.976 | INFO | "pinned" : " 29 GB" | 20260505 19:05:37.976 | INFO | } | 20260505 19:05:37.976 | INFO | Perf report for the last 5.13418 seconds: | 20260505 19:05:37.976 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:05:37.976 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:37.976 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:37.976 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:37.976 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 19:05:37.976 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:37.976 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:37.976 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:05:37.976 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:37.976 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:37.976 | INFO | Uploaded 36GiB to GPU in 2.15028s (16.742GiB/s) | 20260505 19:05:37.976 | INFO | Downloaded 34GiB to GPU in 2.05095s (16.5777GiB/s) | 20260505 19:05:43.114 | INFO | Analyzing chunk frames = [5583872,5584384) | 20260505 19:05:43.114 | INFO | MemInfo: { | 20260505 19:05:43.114 | INFO | "gpu" : " 17 GB", | 20260505 19:05:43.114 | INFO | "huge" : "", | 20260505 19:05:43.114 | INFO | "huge_shared" : " 27 GB", | 20260505 19:05:43.114 | INFO | "malloc" : " 14 GB", | 20260505 19:05:43.114 | INFO | "pinned" : " 29 GB" | 20260505 19:05:43.114 | INFO | } | 20260505 19:05:43.114 | INFO | Perf report for the last 5.13851 seconds: | 20260505 19:05:43.114 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:05:43.114 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:43.114 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:43.114 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:43.114 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260505 19:05:43.115 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:43.115 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:43.115 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:05:43.115 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:43.115 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:43.115 | INFO | Uploaded 36GiB to GPU in 2.2073s (16.3095GiB/s) | 20260505 19:05:43.115 | INFO | Downloaded 34GiB to GPU in 2.03844s (16.6794GiB/s) | 20260505 19:05:48.255 | INFO | Analyzing chunk frames = [5584384,5584896) | 20260505 19:05:48.255 | INFO | MemInfo: { | 20260505 19:05:48.255 | INFO | "gpu" : " 17 GB", | 20260505 19:05:48.255 | INFO | "huge" : "", | 20260505 19:05:48.255 | INFO | "huge_shared" : " 27 GB", | 20260505 19:05:48.255 | INFO | "malloc" : " 14 GB", | 20260505 19:05:48.255 | INFO | "pinned" : " 29 GB" | 20260505 19:05:48.255 | INFO | } | 20260505 19:05:48.255 | INFO | Perf report for the last 5.14092 seconds: | 20260505 19:05:48.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:05:48.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:48.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:48.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:48.255 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260505 19:05:48.255 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:48.255 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:48.255 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:05:48.255 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:48.255 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:48.256 | INFO | Uploaded 36GiB to GPU in 2.17943s (16.5181GiB/s) | 20260505 19:05:48.256 | INFO | Downloaded 34GiB to GPU in 2.0714s (16.414GiB/s) | 20260505 19:05:53.397 | INFO | Analyzing chunk frames = [5584896,5585408) | 20260505 19:05:53.397 | INFO | MemInfo: { | 20260505 19:05:53.397 | INFO | "gpu" : " 17 GB", | 20260505 19:05:53.397 | INFO | "huge" : "", | 20260505 19:05:53.397 | INFO | "huge_shared" : " 27 GB", | 20260505 19:05:53.397 | INFO | "malloc" : " 14 GB", | 20260505 19:05:53.397 | INFO | "pinned" : " 29 GB" | 20260505 19:05:53.397 | INFO | } | 20260505 19:05:53.398 | INFO | Perf report for the last 5.14266 seconds: | 20260505 19:05:53.398 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:05:53.398 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:53.398 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:53.398 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:53.398 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260505 19:05:53.398 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:53.398 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:53.398 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:05:53.398 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:53.398 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:53.398 | INFO | Uploaded 36GiB to GPU in 2.13517s (16.8605GiB/s) | 20260505 19:05:53.398 | INFO | Downloaded 34GiB to GPU in 2.05941s (16.5096GiB/s) | 20260505 19:05:58.532 | INFO | Analyzing chunk frames = [5585408,5585920) | 20260505 19:05:58.532 | INFO | MemInfo: { | 20260505 19:05:58.532 | INFO | "gpu" : " 17 GB", | 20260505 19:05:58.532 | INFO | "huge" : "", | 20260505 19:05:58.532 | INFO | "huge_shared" : " 27 GB", | 20260505 19:05:58.532 | INFO | "malloc" : " 14 GB", | 20260505 19:05:58.532 | INFO | "pinned" : " 29 GB" | 20260505 19:05:58.532 | INFO | } | 20260505 19:05:58.532 | INFO | Perf report for the last 5.13434 seconds: | 20260505 19:05:58.532 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:05:58.532 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:58.532 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:58.532 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:58.532 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260505 19:05:58.532 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:58.532 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:58.532 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 19:05:58.532 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:58.532 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:05:58.532 | INFO | Uploaded 36GiB to GPU in 2.15358s (16.7163GiB/s) | 20260505 19:05:58.533 | INFO | Downloaded 34GiB to GPU in 2.05424s (16.5512GiB/s) | 20260505 19:06:03.670 | INFO | Analyzing chunk frames = [5585920,5586432) | 20260505 19:06:03.670 | INFO | MemInfo: { | 20260505 19:06:03.670 | INFO | "gpu" : " 17 GB", | 20260505 19:06:03.670 | INFO | "huge" : "", | 20260505 19:06:03.670 | INFO | "huge_shared" : " 27 GB", | 20260505 19:06:03.670 | INFO | "malloc" : " 14 GB", | 20260505 19:06:03.671 | INFO | "pinned" : " 29 GB" | 20260505 19:06:03.671 | INFO | } | 20260505 19:06:03.671 | INFO | Perf report for the last 5.1384 seconds: | 20260505 19:06:03.671 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:06:03.671 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:03.671 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:03.671 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:03.671 | INFO | ANALYSIS : 63.4%, 3.0, 98.9ms, 0 | 20260505 19:06:03.671 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:03.671 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:03.671 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 19:06:03.671 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:03.671 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:03.671 | INFO | Uploaded 36GiB to GPU in 2.13434s (16.8671GiB/s) | 20260505 19:06:03.671 | INFO | Downloaded 34GiB to GPU in 2.14494s (15.8513GiB/s) | 20260505 19:06:08.817 | INFO | Analyzing chunk frames = [5586432,5586944) | 20260505 19:06:08.817 | INFO | MemInfo: { | 20260505 19:06:08.817 | INFO | "gpu" : " 17 GB", | 20260505 19:06:08.817 | INFO | "huge" : "", | 20260505 19:06:08.817 | INFO | "huge_shared" : " 27 GB", | 20260505 19:06:08.817 | INFO | "malloc" : " 14 GB", | 20260505 19:06:08.817 | INFO | "pinned" : " 29 GB" | 20260505 19:06:08.817 | INFO | } | 20260505 19:06:08.817 | INFO | Perf report for the last 5.14672 seconds: | 20260505 19:06:08.817 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:06:08.817 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:08.817 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:08.817 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:08.817 | INFO | ANALYSIS : 62.7%, 3.0, 98.1ms, 0 | 20260505 19:06:08.817 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:08.817 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:08.817 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:06:08.818 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:08.818 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:08.818 | INFO | Uploaded 36GiB to GPU in 2.19295s (16.4162GiB/s) | 20260505 19:06:08.818 | INFO | Downloaded 34GiB to GPU in 2.02977s (16.7507GiB/s) | 20260505 19:06:08.818 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 19:06:08.818 | INFO | Total mem usage: 15.3 GiB | 20260505 19:06:08.818 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 19:06:08.818 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.03%) | 20260505 19:06:08.818 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 19:06:08.818 | INFO | High Water Breakdown: | 20260505 19:06:08.818 | INFO | 61% : 8.66 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 19:06:08.818 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 19:06:08.818 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 19:06:08.818 | INFO | ------------------------------------------ | 20260505 19:06:08.818 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:06:08.818 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 19:06:08.818 | INFO | Total mem usage: 26.5 GiB | 20260505 19:06:08.818 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 19:06:08.818 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 19:06:08.818 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 19:06:08.818 | INFO | High Water Breakdown: | 20260505 19:06:08.818 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:06:08.818 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:06:08.818 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:06:08.818 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:06:08.818 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:06:08.818 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:06:08.818 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 19:06:08.818 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 19:06:08.818 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:06:08.818 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 19:06:08.818 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 19:06:08.818 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 19:06:08.818 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 19:06:08.818 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 19:06:08.818 | INFO | ------------------------------------------ | 20260505 19:06:08.818 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:06:08.818 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 19:06:08.818 | INFO | Total mem usage: 25.6 GiB | 20260505 19:06:08.818 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 19:06:08.818 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 19:06:08.818 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 19:06:08.818 | INFO | High Water Breakdown: | 20260505 19:06:08.818 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 19:06:08.818 | INFO | ------------------------------------------ | 20260505 19:06:08.818 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:06:08.818 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 19:06:08.818 | INFO | Total mem usage: 15.7 GiB | 20260505 19:06:08.819 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 19:06:08.819 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 19:06:08.819 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 19:06:08.819 | INFO | High Water Breakdown: | 20260505 19:06:08.819 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 19:06:08.819 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 19:06:08.819 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 19:06:08.819 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 19:06:08.819 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 19:06:08.819 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 19:06:08.819 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 19:06:08.819 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 19:06:08.819 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 19:06:08.819 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:06:08.819 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 19:06:08.819 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:06:08.819 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 19:06:08.819 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 19:06:08.819 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 19:06:08.819 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 19:06:08.819 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:06:08.819 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 19:06:08.819 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:06:08.819 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:06:08.819 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:06:08.819 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 19:06:08.819 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 19:06:08.819 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:06:08.819 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 19:06:08.819 | INFO | ------------------------------------------ | 20260505 19:06:08.819 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:06:08.819 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 19:06:08.819 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:06:08.819 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:06:08.819 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.49e+01, 1.34e+01 / 2.40e+00 | 20260505 19:06:08.819 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.03e-01, 1.16e-02 / 3.14e-03 | 20260505 19:06:08.819 | INFO | PerfReport: 11.75%, Baseline -- 3.70e+00 / 4.53e+00, 5.54e-01 / 1.61e-01 | 20260505 19:06:08.819 | INFO | PerfReport: 3.26%, Histogram -- 1.03e+00 / 1.51e+00, 1.85e-01 / 1.02e-01 | 20260505 19:06:08.819 | INFO | PerfReport: 15.96%, DME -- 4.02e+01 / 4.45e+01, 4.06e+00 / 2.24e+00 | 20260505 19:06:08.819 | INFO | PerfReport: 56.96%, FrameLabeling -- 1.79e+01 / 1.96e+01, 1.43e+00 / 1.29e+00 | 20260505 19:06:08.819 | INFO | PerfReport: 7.31%, PulseAccumulating -- 2.30e+00 / 2.87e+00, 4.42e-01 / 1.90e-01 | 20260505 19:06:08.819 | INFO | PerfReport: 3.30%, Metrics -- 1.04e+00 / 1.37e+00, 2.86e-01 / 1.29e-01 | 20260505 19:06:08.819 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.62e-01 / 4.34e-01, 6.34e-02 / 4.06e-02 | 20260505 19:06:08.819 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 19:06:08.819 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:06:08.819 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:06:08.819 | INFO | PerfReport: Full Execution -- 9.98e+01 / -nan, 2.55e+01 / 0.00e+00 | 20260505 19:06:08.819 | INFO | PerfReport: 22.62%, Upload -- 2.26e+01 / 0.00e+00, 7.93e+00 / 0.00e+00 | 20260505 19:06:08.819 | INFO | PerfReport: 21.84%, Download -- 2.18e+01 / 0.00e+00, 7.80e+00 / 0.00e+00 | 20260505 19:06:08.819 | INFO | PerfReport: 31.54%, Compute -- 3.15e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 19:06:08.819 | INFO | PerfReport: 2.40%, LockUpload -- 2.39e+00 / 0.00e+00, 8.04e+00 / 0.00e+00 | 20260505 19:06:08.819 | INFO | PerfReport: 1.35%, LockDownload -- 1.35e+00 / 0.00e+00, 2.82e+00 / 0.00e+00 | 20260505 19:06:08.819 | INFO | PerfReport: 20.24%, LockCompute -- 2.02e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 19:06:08.819 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.90e-02 / 0.00e+00 | 20260505 19:06:08.946 | INFO | Starting a round of Baz IO | 20260505 19:06:12.125 | INFO | Populating RTMetrics report | 20260505 19:06:12.125 | INFO | RTMetrics report populated | 20260505 19:06:12.125 | INFO | Opening tmp RTMetrics json file | 20260505 19:06:12.127 | INFO | Serializing RTMetrics to json | 20260505 19:06:12.130 | INFO | Serialization done, swapping tmp file now... | 20260505 19:06:12.130 | INFO | RTMetrics done | 20260505 19:06:12.217 | INFO | Finished a round of Baz IO: Wrote 2.065 GiB in 3.271 seconds | 20260505 19:06:12.217 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.82% / 0.00% / 78.18% | 20260505 19:06:12.217 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.221332% / 0% / 7.94729e-05% | 20260505 19:06:12.217 | INFO | Baz IO overall: | 20260505 19:06:12.217 | INFO | BytesWritten : 3278829507359 | 20260505 19:06:12.217 | INFO | NumEvents : 1552202129909 | 20260505 19:06:13.958 | INFO | Analyzing chunk frames = [5586944,5587456) | 20260505 19:06:13.958 | INFO | MemInfo: { | 20260505 19:06:13.958 | INFO | "gpu" : " 17 GB", | 20260505 19:06:13.958 | INFO | "huge" : "", | 20260505 19:06:13.958 | INFO | "huge_shared" : " 27 GB", | 20260505 19:06:13.958 | INFO | "malloc" : " 14 GB", | 20260505 19:06:13.958 | INFO | "pinned" : " 29 GB" | 20260505 19:06:13.958 | INFO | } | 20260505 19:06:13.958 | INFO | Perf report for the last 5.14035 seconds: | 20260505 19:06:13.958 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:06:13.958 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:13.958 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:13.958 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:13.958 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 19:06:13.958 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:13.958 | INFO | RT_METRICS : 44.3%, 1.0, 23.3ms, 0 | 20260505 19:06:13.958 | INFO | PRE_HQ : 60.3%, 1.8, 55.6ms, 0 | 20260505 19:06:13.958 | INFO | BAZWRITER : 63.2%, 3.8, 122.3ms, 0 | 20260505 19:06:13.958 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 19:06:13.958 | INFO | Uploaded 36GiB to GPU in 2.15179s (16.7303GiB/s) | 20260505 19:06:13.958 | INFO | Downloaded 38GiB to GPU in 2.23428s (17.0077GiB/s) | 20260505 19:06:19.095 | INFO | Analyzing chunk frames = [5587456,5587968) | 20260505 19:06:19.095 | INFO | MemInfo: { | 20260505 19:06:19.095 | INFO | "gpu" : " 17 GB", | 20260505 19:06:19.095 | INFO | "huge" : "", | 20260505 19:06:19.095 | INFO | "huge_shared" : " 27 GB", | 20260505 19:06:19.095 | INFO | "malloc" : " 14 GB", | 20260505 19:06:19.095 | INFO | "pinned" : " 29 GB" | 20260505 19:06:19.095 | INFO | } | 20260505 19:06:19.096 | INFO | Perf report for the last 5.13778 seconds: | 20260505 19:06:19.096 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:06:19.096 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:19.096 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:19.096 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:19.096 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260505 19:06:19.096 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:19.096 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:19.096 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:06:19.096 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:19.096 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:19.096 | INFO | Uploaded 36GiB to GPU in 2.13949s (16.8264GiB/s) | 20260505 19:06:19.096 | INFO | Downloaded 34GiB to GPU in 2.04427s (16.6318GiB/s) | 20260505 19:06:24.229 | INFO | Analyzing chunk frames = [5587968,5588480) | 20260505 19:06:24.229 | INFO | MemInfo: { | 20260505 19:06:24.229 | INFO | "gpu" : " 17 GB", | 20260505 19:06:24.229 | INFO | "huge" : "", | 20260505 19:06:24.229 | INFO | "huge_shared" : " 27 GB", | 20260505 19:06:24.229 | INFO | "malloc" : " 14 GB", | 20260505 19:06:24.230 | INFO | "pinned" : " 29 GB" | 20260505 19:06:24.230 | INFO | } | 20260505 19:06:24.230 | INFO | Perf report for the last 5.13405 seconds: | 20260505 19:06:24.230 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:06:24.230 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:24.230 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:24.230 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:24.230 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260505 19:06:24.230 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:24.230 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:24.230 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:06:24.230 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:24.230 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:24.230 | INFO | Uploaded 36GiB to GPU in 2.12573s (16.9354GiB/s) | 20260505 19:06:24.230 | INFO | Downloaded 34GiB to GPU in 2.02357s (16.802GiB/s) | 20260505 19:06:29.365 | INFO | Analyzing chunk frames = [5588480,5588992) | 20260505 19:06:29.365 | INFO | MemInfo: { | 20260505 19:06:29.365 | INFO | "gpu" : " 17 GB", | 20260505 19:06:29.365 | INFO | "huge" : "", | 20260505 19:06:29.365 | INFO | "huge_shared" : " 27 GB", | 20260505 19:06:29.365 | INFO | "malloc" : " 14 GB", | 20260505 19:06:29.365 | INFO | "pinned" : " 29 GB" | 20260505 19:06:29.365 | INFO | } | 20260505 19:06:29.365 | INFO | Perf report for the last 5.13516 seconds: | 20260505 19:06:29.365 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:06:29.365 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:29.365 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:29.365 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:29.365 | INFO | ANALYSIS : 62.7%, 3.0, 97.3ms, 0 | 20260505 19:06:29.365 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:29.365 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:29.365 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 19:06:29.365 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:29.365 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:29.365 | INFO | Uploaded 36GiB to GPU in 2.21952s (16.2198GiB/s) | 20260505 19:06:29.365 | INFO | Downloaded 34GiB to GPU in 2.12879s (15.9715GiB/s) | 20260505 19:06:34.514 | INFO | Analyzing chunk frames = [5588992,5589504) | 20260505 19:06:34.514 | INFO | MemInfo: { | 20260505 19:06:34.514 | INFO | "gpu" : " 17 GB", | 20260505 19:06:34.514 | INFO | "huge" : "", | 20260505 19:06:34.514 | INFO | "huge_shared" : " 27 GB", | 20260505 19:06:34.514 | INFO | "malloc" : " 14 GB", | 20260505 19:06:34.514 | INFO | "pinned" : " 29 GB" | 20260505 19:06:34.514 | INFO | } | 20260505 19:06:34.514 | INFO | Perf report for the last 5.14909 seconds: | 20260505 19:06:34.514 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:06:34.514 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:34.514 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:34.514 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:34.514 | INFO | ANALYSIS : 61.8%, 3.0, 96.7ms, 0 | 20260505 19:06:34.514 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:34.514 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:34.514 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:06:34.514 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:34.514 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:34.514 | INFO | Uploaded 36GiB to GPU in 2.13s (16.9014GiB/s) | 20260505 19:06:34.514 | INFO | Downloaded 34GiB to GPU in 2.05915s (16.5116GiB/s) | 20260505 19:06:39.657 | INFO | Analyzing chunk frames = [5589504,5590016) | 20260505 19:06:39.657 | INFO | MemInfo: { | 20260505 19:06:39.657 | INFO | "gpu" : " 17 GB", | 20260505 19:06:39.657 | INFO | "huge" : "", | 20260505 19:06:39.657 | INFO | "huge_shared" : " 27 GB", | 20260505 19:06:39.657 | INFO | "malloc" : " 14 GB", | 20260505 19:06:39.657 | INFO | "pinned" : " 29 GB" | 20260505 19:06:39.657 | INFO | } | 20260505 19:06:39.658 | INFO | Perf report for the last 5.1437 seconds: | 20260505 19:06:39.658 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:06:39.658 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:39.658 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:39.658 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:39.658 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 19:06:39.658 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:39.658 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:39.658 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:06:39.658 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:39.658 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:39.658 | INFO | Uploaded 36GiB to GPU in 2.13085s (16.8946GiB/s) | 20260505 19:06:39.658 | INFO | Downloaded 34GiB to GPU in 2.05325s (16.5591GiB/s) | 20260505 19:06:44.789 | INFO | Analyzing chunk frames = [5590016,5590528) | 20260505 19:06:44.789 | INFO | MemInfo: { | 20260505 19:06:44.789 | INFO | "gpu" : " 17 GB", | 20260505 19:06:44.789 | INFO | "huge" : "", | 20260505 19:06:44.789 | INFO | "huge_shared" : " 27 GB", | 20260505 19:06:44.789 | INFO | "malloc" : " 14 GB", | 20260505 19:06:44.789 | INFO | "pinned" : " 29 GB" | 20260505 19:06:44.790 | INFO | } | 20260505 19:06:44.790 | INFO | Perf report for the last 5.13205 seconds: | 20260505 19:06:44.790 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:06:44.790 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:44.790 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:44.790 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:44.790 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 19:06:44.790 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:44.790 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:44.790 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 19:06:44.790 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:44.790 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:44.790 | INFO | Uploaded 36GiB to GPU in 2.1773s (16.5342GiB/s) | 20260505 19:06:44.790 | INFO | Downloaded 34GiB to GPU in 2.12874s (15.9719GiB/s) | 20260505 19:06:49.926 | INFO | Analyzing chunk frames = [5590528,5591040) | 20260505 19:06:49.926 | INFO | MemInfo: { | 20260505 19:06:49.926 | INFO | "gpu" : " 17 GB", | 20260505 19:06:49.926 | INFO | "huge" : "", | 20260505 19:06:49.926 | INFO | "huge_shared" : " 27 GB", | 20260505 19:06:49.926 | INFO | "malloc" : " 14 GB", | 20260505 19:06:49.926 | INFO | "pinned" : " 29 GB" | 20260505 19:06:49.926 | INFO | } | 20260505 19:06:49.926 | INFO | Perf report for the last 5.13647 seconds: | 20260505 19:06:49.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:06:49.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:49.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:49.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:49.926 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 19:06:49.926 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:49.926 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:49.926 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 19:06:49.926 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:49.926 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:49.926 | INFO | Uploaded 36GiB to GPU in 2.19244s (16.4201GiB/s) | 20260505 19:06:49.926 | INFO | Downloaded 34GiB to GPU in 2.03054s (16.7443GiB/s) | 20260505 19:06:50.070 | INFO | Starting a round of Baz IO | 20260505 19:06:53.238 | INFO | Populating RTMetrics report | 20260505 19:06:53.239 | INFO | RTMetrics report populated | 20260505 19:06:53.239 | INFO | Opening tmp RTMetrics json file | 20260505 19:06:53.240 | INFO | Serializing RTMetrics to json | 20260505 19:06:53.242 | INFO | Serialization done, swapping tmp file now... | 20260505 19:06:53.242 | INFO | RTMetrics done | 20260505 19:06:53.330 | INFO | Finished a round of Baz IO: Wrote 2.065 GiB in 3.26 seconds | 20260505 19:06:53.330 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.82% / 0.00% / 78.18% | 20260505 19:06:53.330 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.221725% / 0% / 7.94729e-05% | 20260505 19:06:53.330 | INFO | Baz IO overall: | 20260505 19:06:53.330 | INFO | BytesWritten : 3281046541208 | 20260505 19:06:53.330 | INFO | NumEvents : 1553233518478 | 20260505 19:06:55.068 | INFO | Analyzing chunk frames = [5591040,5591552) | 20260505 19:06:55.068 | INFO | MemInfo: { | 20260505 19:06:55.068 | INFO | "gpu" : " 17 GB", | 20260505 19:06:55.068 | INFO | "huge" : "", | 20260505 19:06:55.068 | INFO | "huge_shared" : " 27 GB", | 20260505 19:06:55.068 | INFO | "malloc" : " 14 GB", | 20260505 19:06:55.068 | INFO | "pinned" : " 29 GB" | 20260505 19:06:55.068 | INFO | } | 20260505 19:06:55.068 | INFO | Perf report for the last 5.14178 seconds: | 20260505 19:06:55.068 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:06:55.068 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:55.068 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:55.068 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:55.068 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 19:06:55.068 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:06:55.068 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260505 19:06:55.068 | INFO | PRE_HQ : 60.3%, 1.8, 55.7ms, 0 | 20260505 19:06:55.068 | INFO | BAZWRITER : 63.1%, 3.7, 120.9ms, 0 | 20260505 19:06:55.068 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 19:06:55.068 | INFO | Uploaded 36GiB to GPU in 2.16228s (16.6491GiB/s) | 20260505 19:06:55.068 | INFO | Downloaded 38GiB to GPU in 2.24781s (16.9054GiB/s) | 20260505 19:07:00.206 | INFO | Analyzing chunk frames = [5591552,5592064) | 20260505 19:07:00.206 | INFO | MemInfo: { | 20260505 19:07:00.206 | INFO | "gpu" : " 17 GB", | 20260505 19:07:00.206 | INFO | "huge" : "", | 20260505 19:07:00.206 | INFO | "huge_shared" : " 27 GB", | 20260505 19:07:00.206 | INFO | "malloc" : " 14 GB", | 20260505 19:07:00.206 | INFO | "pinned" : " 29 GB" | 20260505 19:07:00.206 | INFO | } | 20260505 19:07:00.206 | INFO | Perf report for the last 5.13832 seconds: | 20260505 19:07:00.206 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:07:00.206 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:00.206 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:00.206 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:00.206 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 19:07:00.206 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:00.206 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:00.206 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:07:00.206 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:00.206 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:00.207 | INFO | Uploaded 36GiB to GPU in 2.13642s (16.8506GiB/s) | 20260505 19:07:00.207 | INFO | Downloaded 34GiB to GPU in 2.02913s (16.7559GiB/s) | 20260505 19:07:05.347 | INFO | Analyzing chunk frames = [5592064,5592576) | 20260505 19:07:05.347 | INFO | MemInfo: { | 20260505 19:07:05.347 | INFO | "gpu" : " 17 GB", | 20260505 19:07:05.347 | INFO | "huge" : "", | 20260505 19:07:05.347 | INFO | "huge_shared" : " 27 GB", | 20260505 19:07:05.347 | INFO | "malloc" : " 14 GB", | 20260505 19:07:05.347 | INFO | "pinned" : " 29 GB" | 20260505 19:07:05.347 | INFO | } | 20260505 19:07:05.348 | INFO | Perf report for the last 5.14135 seconds: | 20260505 19:07:05.348 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:07:05.348 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:05.348 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:05.348 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:05.348 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260505 19:07:05.348 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:05.348 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:05.348 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:07:05.348 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:05.348 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:05.348 | INFO | Uploaded 36GiB to GPU in 2.21778s (16.2324GiB/s) | 20260505 19:07:05.348 | INFO | Downloaded 34GiB to GPU in 2.0754s (16.3824GiB/s) | 20260505 19:07:10.484 | INFO | Analyzing chunk frames = [5592576,5593088) | 20260505 19:07:10.484 | INFO | MemInfo: { | 20260505 19:07:10.484 | INFO | "gpu" : " 17 GB", | 20260505 19:07:10.484 | INFO | "huge" : "", | 20260505 19:07:10.484 | INFO | "huge_shared" : " 27 GB", | 20260505 19:07:10.484 | INFO | "malloc" : " 14 GB", | 20260505 19:07:10.484 | INFO | "pinned" : " 29 GB" | 20260505 19:07:10.484 | INFO | } | 20260505 19:07:10.484 | INFO | Perf report for the last 5.13641 seconds: | 20260505 19:07:10.484 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:07:10.484 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:10.484 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:10.484 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:10.484 | INFO | ANALYSIS : 62.3%, 3.0, 96.7ms, 0 | 20260505 19:07:10.484 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:10.484 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:10.484 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 19:07:10.484 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:10.484 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:10.484 | INFO | Uploaded 36GiB to GPU in 2.18291s (16.4918GiB/s) | 20260505 19:07:10.484 | INFO | Downloaded 34GiB to GPU in 2.06612s (16.456GiB/s) | 20260505 19:07:15.621 | INFO | Analyzing chunk frames = [5593088,5593600) | 20260505 19:07:15.621 | INFO | MemInfo: { | 20260505 19:07:15.621 | INFO | "gpu" : " 17 GB", | 20260505 19:07:15.621 | INFO | "huge" : "", | 20260505 19:07:15.621 | INFO | "huge_shared" : " 27 GB", | 20260505 19:07:15.621 | INFO | "malloc" : " 14 GB", | 20260505 19:07:15.621 | INFO | "pinned" : " 29 GB" | 20260505 19:07:15.621 | INFO | } | 20260505 19:07:15.621 | INFO | Perf report for the last 5.13736 seconds: | 20260505 19:07:15.621 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:07:15.622 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:15.622 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:15.622 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:15.622 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 19:07:15.622 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:15.622 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:15.622 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:07:15.622 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:15.622 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:15.622 | INFO | Uploaded 36GiB to GPU in 2.14386s (16.7921GiB/s) | 20260505 19:07:15.622 | INFO | Downloaded 34GiB to GPU in 2.06415s (16.4717GiB/s) | 20260505 19:07:20.768 | INFO | Analyzing chunk frames = [5593600,5594112) | 20260505 19:07:20.768 | INFO | MemInfo: { | 20260505 19:07:20.768 | INFO | "gpu" : " 17 GB", | 20260505 19:07:20.768 | INFO | "huge" : "", | 20260505 19:07:20.768 | INFO | "huge_shared" : " 27 GB", | 20260505 19:07:20.768 | INFO | "malloc" : " 14 GB", | 20260505 19:07:20.768 | INFO | "pinned" : " 29 GB" | 20260505 19:07:20.768 | INFO | } | 20260505 19:07:20.768 | INFO | Perf report for the last 5.14712 seconds: | 20260505 19:07:20.768 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:07:20.768 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:20.768 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:20.768 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:20.768 | INFO | ANALYSIS : 62.3%, 3.0, 97.4ms, 0 | 20260505 19:07:20.769 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:20.769 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:20.769 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:07:20.769 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:20.769 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:20.769 | INFO | Uploaded 36GiB to GPU in 2.14424s (16.7892GiB/s) | 20260505 19:07:20.769 | INFO | Downloaded 34GiB to GPU in 2.04927s (16.5913GiB/s) | 20260505 19:07:25.906 | INFO | Analyzing chunk frames = [5594112,5594624) | 20260505 19:07:25.906 | INFO | MemInfo: { | 20260505 19:07:25.906 | INFO | "gpu" : " 17 GB", | 20260505 19:07:25.906 | INFO | "huge" : "", | 20260505 19:07:25.906 | INFO | "huge_shared" : " 27 GB", | 20260505 19:07:25.906 | INFO | "malloc" : " 14 GB", | 20260505 19:07:25.906 | INFO | "pinned" : " 29 GB" | 20260505 19:07:25.906 | INFO | } | 20260505 19:07:25.906 | INFO | Perf report for the last 5.13735 seconds: | 20260505 19:07:25.906 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:07:25.906 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:25.906 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:25.906 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:25.906 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 19:07:25.906 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:25.906 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:25.906 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 19:07:25.906 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:25.906 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:25.906 | INFO | Uploaded 36GiB to GPU in 2.13394s (16.8702GiB/s) | 20260505 19:07:25.906 | INFO | Downloaded 34GiB to GPU in 2.14198s (15.8731GiB/s) | 20260505 19:07:31.039 | INFO | Analyzing chunk frames = [5594624,5595136) | 20260505 19:07:31.039 | INFO | MemInfo: { | 20260505 19:07:31.039 | INFO | "gpu" : " 17 GB", | 20260505 19:07:31.039 | INFO | "huge" : "", | 20260505 19:07:31.039 | INFO | "huge_shared" : " 27 GB", | 20260505 19:07:31.039 | INFO | "malloc" : " 14 GB", | 20260505 19:07:31.039 | INFO | "pinned" : " 29 GB" | 20260505 19:07:31.039 | INFO | } | 20260505 19:07:31.039 | INFO | Perf report for the last 5.13349 seconds: | 20260505 19:07:31.039 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:07:31.039 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:31.039 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:31.039 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:31.039 | INFO | ANALYSIS : 63.1%, 3.0, 98.3ms, 0 | 20260505 19:07:31.039 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:31.039 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:31.039 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 19:07:31.039 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:31.039 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:31.040 | INFO | Uploaded 36GiB to GPU in 2.19894s (16.3715GiB/s) | 20260505 19:07:31.040 | INFO | Downloaded 34GiB to GPU in 2.03809s (16.6823GiB/s) | 20260505 19:07:31.040 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 19:07:31.040 | INFO | Total mem usage: 15.3 GiB | 20260505 19:07:31.040 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 19:07:31.040 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.06%) | 20260505 19:07:31.040 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 19:07:31.040 | INFO | High Water Breakdown: | 20260505 19:07:31.040 | INFO | 61% : 8.66 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 19:07:31.040 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 19:07:31.040 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 19:07:31.040 | INFO | ------------------------------------------ | 20260505 19:07:31.040 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:07:31.040 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 19:07:31.040 | INFO | Total mem usage: 26.5 GiB | 20260505 19:07:31.040 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 19:07:31.040 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 19:07:31.040 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 19:07:31.040 | INFO | High Water Breakdown: | 20260505 19:07:31.040 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:07:31.040 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:07:31.040 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:07:31.040 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:07:31.040 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:07:31.040 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:07:31.040 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 19:07:31.040 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 19:07:31.040 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:07:31.040 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 19:07:31.040 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 19:07:31.040 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 19:07:31.040 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 19:07:31.040 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 19:07:31.040 | INFO | ------------------------------------------ | 20260505 19:07:31.040 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:07:31.040 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 19:07:31.040 | INFO | Total mem usage: 25.6 GiB | 20260505 19:07:31.040 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 19:07:31.040 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 19:07:31.040 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 19:07:31.040 | INFO | High Water Breakdown: | 20260505 19:07:31.040 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 19:07:31.040 | INFO | ------------------------------------------ | 20260505 19:07:31.040 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:07:31.040 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 19:07:31.040 | INFO | Total mem usage: 15.7 GiB | 20260505 19:07:31.041 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 19:07:31.041 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 19:07:31.041 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 19:07:31.041 | INFO | High Water Breakdown: | 20260505 19:07:31.041 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 19:07:31.041 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 19:07:31.041 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 19:07:31.041 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 19:07:31.041 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 19:07:31.041 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 19:07:31.041 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 19:07:31.041 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 19:07:31.041 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 19:07:31.041 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:07:31.041 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 19:07:31.041 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:07:31.041 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 19:07:31.041 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 19:07:31.041 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 19:07:31.041 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 19:07:31.041 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:07:31.041 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 19:07:31.041 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:07:31.041 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:07:31.041 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:07:31.041 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 19:07:31.041 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 19:07:31.041 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:07:31.041 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 19:07:31.041 | INFO | ------------------------------------------ | 20260505 19:07:31.041 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:07:31.041 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 19:07:31.041 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:07:31.041 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:07:31.041 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.45e+01, 1.35e+01 / 1.92e+00 | 20260505 19:07:31.041 | INFO | PerfReport: 0.29%, AggregateStats -- 9.24e-02 / 1.04e-01, 1.17e-02 / 6.62e-03 | 20260505 19:07:31.041 | INFO | PerfReport: 11.78%, Baseline -- 3.71e+00 / 4.54e+00, 5.60e-01 / 1.31e-01 | 20260505 19:07:31.041 | INFO | PerfReport: 3.26%, Histogram -- 1.03e+00 / 1.52e+00, 1.86e-01 / 1.01e-01 | 20260505 19:07:31.041 | INFO | PerfReport: 15.95%, DME -- 4.02e+01 / 4.44e+01, 4.13e+00 / 2.04e+00 | 20260505 19:07:31.041 | INFO | PerfReport: 56.94%, FrameLabeling -- 1.79e+01 / 1.93e+01, 1.42e+00 / 1.09e+00 | 20260505 19:07:31.041 | INFO | PerfReport: 7.31%, PulseAccumulating -- 2.30e+00 / 2.88e+00, 4.41e-01 / 2.02e-01 | 20260505 19:07:31.041 | INFO | PerfReport: 3.31%, Metrics -- 1.04e+00 / 1.38e+00, 2.87e-01 / 1.33e-01 | 20260505 19:07:31.041 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.62e-01 / 4.24e-01, 6.35e-02 / 3.80e-02 | 20260505 19:07:31.041 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 19:07:31.041 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:07:31.041 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:07:31.041 | INFO | PerfReport: Full Execution -- 9.97e+01 / -nan, 2.56e+01 / 0.00e+00 | 20260505 19:07:31.041 | INFO | PerfReport: 22.58%, Upload -- 2.25e+01 / 0.00e+00, 7.89e+00 / 0.00e+00 | 20260505 19:07:31.041 | INFO | PerfReport: 21.82%, Download -- 2.18e+01 / 0.00e+00, 7.75e+00 / 0.00e+00 | 20260505 19:07:31.041 | INFO | PerfReport: 31.58%, Compute -- 3.15e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 19:07:31.041 | INFO | PerfReport: 2.36%, LockUpload -- 2.35e+00 / 0.00e+00, 7.95e+00 / 0.00e+00 | 20260505 19:07:31.041 | INFO | PerfReport: 1.33%, LockDownload -- 1.33e+00 / 0.00e+00, 2.78e+00 / 0.00e+00 | 20260505 19:07:31.041 | INFO | PerfReport: 20.32%, LockCompute -- 2.03e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 19:07:31.041 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.70e-02 / 0.00e+00 | 20260505 19:07:31.174 | INFO | Starting a round of Baz IO | 20260505 19:07:34.347 | INFO | Populating RTMetrics report | 20260505 19:07:34.347 | INFO | RTMetrics report populated | 20260505 19:07:34.347 | INFO | Opening tmp RTMetrics json file | 20260505 19:07:34.351 | INFO | Serializing RTMetrics to json | 20260505 19:07:34.352 | INFO | Serialization done, swapping tmp file now... | 20260505 19:07:34.352 | INFO | RTMetrics done | 20260505 19:07:34.437 | INFO | Finished a round of Baz IO: Wrote 2.064 GiB in 3.263 seconds | 20260505 19:07:34.438 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.82% / 0.00% / 78.18% | 20260505 19:07:34.438 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.221368% / 3.97364e-06% / 7.15256e-05% | 20260505 19:07:34.438 | INFO | Baz IO overall: | 20260505 19:07:34.438 | INFO | BytesWritten : 3283263065782 | 20260505 19:07:34.438 | INFO | NumEvents : 1554264407877 | 20260505 19:07:36.181 | INFO | Analyzing chunk frames = [5595136,5595648) | 20260505 19:07:36.181 | INFO | MemInfo: { | 20260505 19:07:36.181 | INFO | "gpu" : " 17 GB", | 20260505 19:07:36.181 | INFO | "huge" : "", | 20260505 19:07:36.181 | INFO | "huge_shared" : " 27 GB", | 20260505 19:07:36.181 | INFO | "malloc" : " 14 GB", | 20260505 19:07:36.181 | INFO | "pinned" : " 29 GB" | 20260505 19:07:36.181 | INFO | } | 20260505 19:07:36.181 | INFO | Perf report for the last 5.14174 seconds: | 20260505 19:07:36.181 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:07:36.181 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:36.181 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:36.181 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:36.181 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 19:07:36.181 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:36.181 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260505 19:07:36.181 | INFO | PRE_HQ : 60.2%, 1.8, 55.4ms, 0 | 20260505 19:07:36.181 | INFO | BAZWRITER : 62.9%, 3.7, 120.6ms, 0 | 20260505 19:07:36.181 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 19:07:36.182 | INFO | Uploaded 36GiB to GPU in 2.17427s (16.5573GiB/s) | 20260505 19:07:36.182 | INFO | Downloaded 38GiB to GPU in 2.25389s (16.8597GiB/s) | 20260505 19:07:41.323 | INFO | Analyzing chunk frames = [5595648,5596160) | 20260505 19:07:41.323 | INFO | MemInfo: { | 20260505 19:07:41.323 | INFO | "gpu" : " 17 GB", | 20260505 19:07:41.323 | INFO | "huge" : "", | 20260505 19:07:41.323 | INFO | "huge_shared" : " 27 GB", | 20260505 19:07:41.323 | INFO | "malloc" : " 14 GB", | 20260505 19:07:41.323 | INFO | "pinned" : " 29 GB" | 20260505 19:07:41.323 | INFO | } | 20260505 19:07:41.323 | INFO | Perf report for the last 5.14187 seconds: | 20260505 19:07:41.323 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:07:41.323 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:41.323 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:41.323 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:41.323 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 19:07:41.323 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:41.323 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:41.323 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:07:41.323 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:41.323 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:41.323 | INFO | Uploaded 36GiB to GPU in 2.12578s (16.935GiB/s) | 20260505 19:07:41.323 | INFO | Downloaded 34GiB to GPU in 2.02756s (16.7689GiB/s) | 20260505 19:07:46.470 | INFO | Analyzing chunk frames = [5596160,5596672) | 20260505 19:07:46.470 | INFO | MemInfo: { | 20260505 19:07:46.470 | INFO | "gpu" : " 17 GB", | 20260505 19:07:46.470 | INFO | "huge" : "", | 20260505 19:07:46.470 | INFO | "huge_shared" : " 27 GB", | 20260505 19:07:46.470 | INFO | "malloc" : " 14 GB", | 20260505 19:07:46.470 | INFO | "pinned" : " 29 GB" | 20260505 19:07:46.470 | INFO | } | 20260505 19:07:46.470 | INFO | Perf report for the last 5.14733 seconds: | 20260505 19:07:46.470 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:07:46.471 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:46.471 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:46.471 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:46.471 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260505 19:07:46.471 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:46.471 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:46.471 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260505 19:07:46.471 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:46.471 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:46.471 | INFO | Uploaded 36GiB to GPU in 2.12039s (16.978GiB/s) | 20260505 19:07:46.471 | INFO | Downloaded 34GiB to GPU in 2.02412s (16.7974GiB/s) | 20260505 19:07:51.603 | INFO | Analyzing chunk frames = [5596672,5597184) | 20260505 19:07:51.603 | INFO | MemInfo: { | 20260505 19:07:51.603 | INFO | "gpu" : " 17 GB", | 20260505 19:07:51.603 | INFO | "huge" : "", | 20260505 19:07:51.603 | INFO | "huge_shared" : " 27 GB", | 20260505 19:07:51.603 | INFO | "malloc" : " 14 GB", | 20260505 19:07:51.603 | INFO | "pinned" : " 29 GB" | 20260505 19:07:51.603 | INFO | } | 20260505 19:07:51.603 | INFO | Perf report for the last 5.13255 seconds: | 20260505 19:07:51.603 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:07:51.603 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:51.603 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:51.603 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:51.603 | INFO | ANALYSIS : 62.2%, 3.0, 96.5ms, 0 | 20260505 19:07:51.603 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:51.603 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:51.603 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:07:51.603 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:51.603 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:51.603 | INFO | Uploaded 36GiB to GPU in 2.13936s (16.8274GiB/s) | 20260505 19:07:51.603 | INFO | Downloaded 34GiB to GPU in 2.03341s (16.7207GiB/s) | 20260505 19:07:56.737 | INFO | Analyzing chunk frames = [5597184,5597696) | 20260505 19:07:56.737 | INFO | MemInfo: { | 20260505 19:07:56.737 | INFO | "gpu" : " 17 GB", | 20260505 19:07:56.737 | INFO | "huge" : "", | 20260505 19:07:56.737 | INFO | "huge_shared" : " 27 GB", | 20260505 19:07:56.737 | INFO | "malloc" : " 14 GB", | 20260505 19:07:56.737 | INFO | "pinned" : " 29 GB" | 20260505 19:07:56.737 | INFO | } | 20260505 19:07:56.737 | INFO | Perf report for the last 5.13418 seconds: | 20260505 19:07:56.737 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:07:56.737 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:56.737 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:56.737 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:56.737 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260505 19:07:56.737 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:56.737 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:56.737 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:07:56.737 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:56.737 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:07:56.737 | INFO | Uploaded 36GiB to GPU in 2.12169s (16.9676GiB/s) | 20260505 19:07:56.738 | INFO | Downloaded 34GiB to GPU in 2.06222s (16.4871GiB/s) | 20260505 19:08:01.875 | INFO | Analyzing chunk frames = [5597696,5598208) | 20260505 19:08:01.875 | INFO | MemInfo: { | 20260505 19:08:01.875 | INFO | "gpu" : " 17 GB", | 20260505 19:08:01.875 | INFO | "huge" : "", | 20260505 19:08:01.875 | INFO | "huge_shared" : " 27 GB", | 20260505 19:08:01.875 | INFO | "malloc" : " 14 GB", | 20260505 19:08:01.875 | INFO | "pinned" : " 29 GB" | 20260505 19:08:01.875 | INFO | } | 20260505 19:08:01.875 | INFO | Perf report for the last 5.13841 seconds: | 20260505 19:08:01.875 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:08:01.875 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:01.875 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:01.876 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:01.876 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 19:08:01.876 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:01.876 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:01.876 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:08:01.876 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:01.876 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:01.876 | INFO | Uploaded 36GiB to GPU in 2.13508s (16.8612GiB/s) | 20260505 19:08:01.876 | INFO | Downloaded 34GiB to GPU in 2.05003s (16.5851GiB/s) | 20260505 19:08:07.020 | INFO | Analyzing chunk frames = [5598208,5598720) | 20260505 19:08:07.020 | INFO | MemInfo: { | 20260505 19:08:07.020 | INFO | "gpu" : " 17 GB", | 20260505 19:08:07.020 | INFO | "huge" : "", | 20260505 19:08:07.020 | INFO | "huge_shared" : " 27 GB", | 20260505 19:08:07.020 | INFO | "malloc" : " 14 GB", | 20260505 19:08:07.020 | INFO | "pinned" : " 29 GB" | 20260505 19:08:07.020 | INFO | } | 20260505 19:08:07.020 | INFO | Perf report for the last 5.14464 seconds: | 20260505 19:08:07.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:08:07.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:07.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:07.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:07.020 | INFO | ANALYSIS : 63.2%, 3.0, 98.8ms, 0 | 20260505 19:08:07.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:07.020 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:07.020 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 19:08:07.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:07.020 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:07.020 | INFO | Uploaded 36GiB to GPU in 2.12891s (16.91GiB/s) | 20260505 19:08:07.020 | INFO | Downloaded 34GiB to GPU in 2.14s (15.8878GiB/s) | 20260505 19:08:12.157 | INFO | Analyzing chunk frames = [5598720,5599232) | 20260505 19:08:12.157 | INFO | MemInfo: { | 20260505 19:08:12.157 | INFO | "gpu" : " 17 GB", | 20260505 19:08:12.157 | INFO | "huge" : "", | 20260505 19:08:12.157 | INFO | "huge_shared" : " 27 GB", | 20260505 19:08:12.157 | INFO | "malloc" : " 14 GB", | 20260505 19:08:12.157 | INFO | "pinned" : " 29 GB" | 20260505 19:08:12.157 | INFO | } | 20260505 19:08:12.157 | INFO | Perf report for the last 5.13685 seconds: | 20260505 19:08:12.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:08:12.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:12.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:12.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:12.157 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 19:08:12.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:12.157 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:12.157 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 19:08:12.157 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:12.157 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:12.157 | INFO | Uploaded 36GiB to GPU in 2.20415s (16.3328GiB/s) | 20260505 19:08:12.157 | INFO | Downloaded 34GiB to GPU in 2.04168s (16.6529GiB/s) | 20260505 19:08:12.291 | INFO | Starting a round of Baz IO | 20260505 19:08:15.463 | INFO | Populating RTMetrics report | 20260505 19:08:15.463 | INFO | RTMetrics report populated | 20260505 19:08:15.463 | INFO | Opening tmp RTMetrics json file | 20260505 19:08:15.464 | INFO | Serializing RTMetrics to json | 20260505 19:08:15.465 | INFO | Serialization done, swapping tmp file now... | 20260505 19:08:15.466 | INFO | RTMetrics done | 20260505 19:08:15.553 | INFO | Finished a round of Baz IO: Wrote 2.062 GiB in 3.262 seconds | 20260505 19:08:15.553 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.82% / 0.00% / 78.18% | 20260505 19:08:15.553 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.219353% / 3.97364e-06% / 0.000115236% | 20260505 19:08:15.553 | INFO | Baz IO overall: | 20260505 19:08:15.553 | INFO | BytesWritten : 3285476667336 | 20260505 19:08:15.553 | INFO | NumEvents : 1555293078395 | 20260505 19:08:17.299 | INFO | Analyzing chunk frames = [5599232,5599744) | 20260505 19:08:17.299 | INFO | MemInfo: { | 20260505 19:08:17.299 | INFO | "gpu" : " 17 GB", | 20260505 19:08:17.299 | INFO | "huge" : "", | 20260505 19:08:17.299 | INFO | "huge_shared" : " 27 GB", | 20260505 19:08:17.299 | INFO | "malloc" : " 14 GB", | 20260505 19:08:17.299 | INFO | "pinned" : " 29 GB" | 20260505 19:08:17.299 | INFO | } | 20260505 19:08:17.299 | INFO | Perf report for the last 5.14237 seconds: | 20260505 19:08:17.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:08:17.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:17.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:17.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:17.299 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 19:08:17.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:17.299 | INFO | RT_METRICS : 44.7%, 1.0, 23.4ms, 0 | 20260505 19:08:17.300 | INFO | PRE_HQ : 60.4%, 1.8, 56.0ms, 0 | 20260505 19:08:17.300 | INFO | BAZWRITER : 63.0%, 3.7, 120.9ms, 0 | 20260505 19:08:17.300 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 19:08:17.300 | INFO | Uploaded 36GiB to GPU in 2.16577s (16.6222GiB/s) | 20260505 19:08:17.300 | INFO | Downloaded 38GiB to GPU in 2.2514s (16.8784GiB/s) | 20260505 19:08:22.432 | INFO | Analyzing chunk frames = [5599744,5600256) | 20260505 19:08:22.432 | INFO | MemInfo: { | 20260505 19:08:22.432 | INFO | "gpu" : " 17 GB", | 20260505 19:08:22.432 | INFO | "huge" : "", | 20260505 19:08:22.432 | INFO | "huge_shared" : " 27 GB", | 20260505 19:08:22.432 | INFO | "malloc" : " 14 GB", | 20260505 19:08:22.432 | INFO | "pinned" : " 29 GB" | 20260505 19:08:22.432 | INFO | } | 20260505 19:08:22.432 | INFO | Perf report for the last 5.13307 seconds: | 20260505 19:08:22.432 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:08:22.432 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:22.432 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:22.432 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:22.433 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260505 19:08:22.433 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:22.433 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:22.433 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:08:22.433 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:22.433 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:22.433 | INFO | Uploaded 36GiB to GPU in 2.12429s (16.9468GiB/s) | 20260505 19:08:22.433 | INFO | Downloaded 34GiB to GPU in 2.02614s (16.7807GiB/s) | 20260505 19:08:27.577 | INFO | Analyzing chunk frames = [5600256,5600768) | 20260505 19:08:27.577 | INFO | MemInfo: { | 20260505 19:08:27.577 | INFO | "gpu" : " 17 GB", | 20260505 19:08:27.577 | INFO | "huge" : "", | 20260505 19:08:27.577 | INFO | "huge_shared" : " 27 GB", | 20260505 19:08:27.577 | INFO | "malloc" : " 14 GB", | 20260505 19:08:27.577 | INFO | "pinned" : " 29 GB" | 20260505 19:08:27.577 | INFO | } | 20260505 19:08:27.577 | INFO | Perf report for the last 5.14498 seconds: | 20260505 19:08:27.578 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:08:27.578 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:27.578 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:27.578 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:27.578 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260505 19:08:27.578 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:27.578 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:27.578 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:08:27.578 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:27.578 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:27.578 | INFO | Uploaded 36GiB to GPU in 2.21245s (16.2715GiB/s) | 20260505 19:08:27.578 | INFO | Downloaded 34GiB to GPU in 2.0685s (16.437GiB/s) | 20260505 19:08:32.714 | INFO | Analyzing chunk frames = [5600768,5601280) | 20260505 19:08:32.714 | INFO | MemInfo: { | 20260505 19:08:32.714 | INFO | "gpu" : " 17 GB", | 20260505 19:08:32.714 | INFO | "huge" : "", | 20260505 19:08:32.714 | INFO | "huge_shared" : " 27 GB", | 20260505 19:08:32.714 | INFO | "malloc" : " 14 GB", | 20260505 19:08:32.714 | INFO | "pinned" : " 29 GB" | 20260505 19:08:32.714 | INFO | } | 20260505 19:08:32.714 | INFO | Perf report for the last 5.13666 seconds: | 20260505 19:08:32.714 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:08:32.714 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:32.714 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:32.714 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:32.714 | INFO | ANALYSIS : 62.4%, 3.0, 96.9ms, 0 | 20260505 19:08:32.714 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:32.714 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:32.714 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:08:32.714 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:32.714 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:32.715 | INFO | Uploaded 36GiB to GPU in 2.20279s (16.3429GiB/s) | 20260505 19:08:32.715 | INFO | Downloaded 34GiB to GPU in 2.04113s (16.6574GiB/s) | 20260505 19:08:37.851 | INFO | Analyzing chunk frames = [5601280,5601792) | 20260505 19:08:37.851 | INFO | MemInfo: { | 20260505 19:08:37.851 | INFO | "gpu" : " 17 GB", | 20260505 19:08:37.851 | INFO | "huge" : "", | 20260505 19:08:37.851 | INFO | "huge_shared" : " 27 GB", | 20260505 19:08:37.851 | INFO | "malloc" : " 14 GB", | 20260505 19:08:37.851 | INFO | "pinned" : " 29 GB" | 20260505 19:08:37.851 | INFO | } | 20260505 19:08:37.851 | INFO | Perf report for the last 5.13649 seconds: | 20260505 19:08:37.851 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:08:37.851 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:37.851 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:37.851 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:37.851 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260505 19:08:37.851 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:37.851 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:37.851 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:08:37.851 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:37.851 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:37.851 | INFO | Uploaded 36GiB to GPU in 2.13966s (16.8251GiB/s) | 20260505 19:08:37.851 | INFO | Downloaded 34GiB to GPU in 2.06112s (16.4959GiB/s) | 20260505 19:08:42.987 | INFO | Analyzing chunk frames = [5601792,5602304) | 20260505 19:08:42.987 | INFO | MemInfo: { | 20260505 19:08:42.987 | INFO | "gpu" : " 17 GB", | 20260505 19:08:42.987 | INFO | "huge" : "", | 20260505 19:08:42.987 | INFO | "huge_shared" : " 27 GB", | 20260505 19:08:42.987 | INFO | "malloc" : " 14 GB", | 20260505 19:08:42.987 | INFO | "pinned" : " 29 GB" | 20260505 19:08:42.987 | INFO | } | 20260505 19:08:42.987 | INFO | Perf report for the last 5.13683 seconds: | 20260505 19:08:42.987 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:08:42.987 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:42.987 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:42.987 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:42.987 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260505 19:08:42.987 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:42.987 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:42.987 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:08:42.988 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:42.988 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:42.988 | INFO | Uploaded 36GiB to GPU in 2.13947s (16.8266GiB/s) | 20260505 19:08:42.988 | INFO | Downloaded 34GiB to GPU in 2.05367s (16.5557GiB/s) | 20260505 19:08:48.135 | INFO | Analyzing chunk frames = [5602304,5602816) | 20260505 19:08:48.135 | INFO | MemInfo: { | 20260505 19:08:48.135 | INFO | "gpu" : " 17 GB", | 20260505 19:08:48.135 | INFO | "huge" : "", | 20260505 19:08:48.135 | INFO | "huge_shared" : " 27 GB", | 20260505 19:08:48.135 | INFO | "malloc" : " 14 GB", | 20260505 19:08:48.135 | INFO | "pinned" : " 29 GB" | 20260505 19:08:48.135 | INFO | } | 20260505 19:08:48.135 | INFO | Perf report for the last 5.14716 seconds: | 20260505 19:08:48.135 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:08:48.135 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:48.135 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:48.135 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:48.135 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 19:08:48.135 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:48.135 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:48.135 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 19:08:48.135 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:48.135 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:48.135 | INFO | Uploaded 36GiB to GPU in 2.15497s (16.7056GiB/s) | 20260505 19:08:48.135 | INFO | Downloaded 34GiB to GPU in 2.1776s (15.6135GiB/s) | 20260505 19:08:53.278 | INFO | Analyzing chunk frames = [5602816,5603328) | 20260505 19:08:53.278 | INFO | MemInfo: { | 20260505 19:08:53.278 | INFO | "gpu" : " 17 GB", | 20260505 19:08:53.278 | INFO | "huge" : "", | 20260505 19:08:53.278 | INFO | "huge_shared" : " 27 GB", | 20260505 19:08:53.278 | INFO | "malloc" : " 14 GB", | 20260505 19:08:53.278 | INFO | "pinned" : " 29 GB" | 20260505 19:08:53.278 | INFO | } | 20260505 19:08:53.278 | INFO | Perf report for the last 5.14382 seconds: | 20260505 19:08:53.279 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:08:53.279 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:53.279 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:53.279 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:53.279 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 19:08:53.279 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:53.279 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:53.279 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260505 19:08:53.279 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:53.279 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:53.279 | INFO | Uploaded 36GiB to GPU in 2.2056s (16.3221GiB/s) | 20260505 19:08:53.279 | INFO | Downloaded 34GiB to GPU in 2.04793s (16.6021GiB/s) | 20260505 19:08:53.279 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 19:08:53.279 | INFO | Total mem usage: 15.3 GiB | 20260505 19:08:53.279 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 19:08:53.279 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.14%) | 20260505 19:08:53.279 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 19:08:53.279 | INFO | High Water Breakdown: | 20260505 19:08:53.279 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 19:08:53.279 | INFO | 38.9% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 19:08:53.279 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 19:08:53.279 | INFO | ------------------------------------------ | 20260505 19:08:53.279 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:08:53.279 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 19:08:53.279 | INFO | Total mem usage: 26.5 GiB | 20260505 19:08:53.279 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 19:08:53.279 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 19:08:53.279 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 19:08:53.279 | INFO | High Water Breakdown: | 20260505 19:08:53.279 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:08:53.279 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:08:53.279 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:08:53.279 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:08:53.279 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:08:53.279 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:08:53.279 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 19:08:53.279 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 19:08:53.279 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:08:53.279 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 19:08:53.279 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 19:08:53.279 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 19:08:53.279 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 19:08:53.279 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 19:08:53.279 | INFO | ------------------------------------------ | 20260505 19:08:53.279 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:08:53.279 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 19:08:53.279 | INFO | Total mem usage: 25.6 GiB | 20260505 19:08:53.279 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 19:08:53.280 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 19:08:53.280 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 19:08:53.280 | INFO | High Water Breakdown: | 20260505 19:08:53.280 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 19:08:53.280 | INFO | ------------------------------------------ | 20260505 19:08:53.280 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:08:53.280 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 19:08:53.280 | INFO | Total mem usage: 15.7 GiB | 20260505 19:08:53.280 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 19:08:53.280 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 19:08:53.280 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 19:08:53.280 | INFO | High Water Breakdown: | 20260505 19:08:53.280 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 19:08:53.280 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 19:08:53.280 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 19:08:53.280 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 19:08:53.280 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 19:08:53.280 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 19:08:53.280 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 19:08:53.280 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 19:08:53.280 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 19:08:53.280 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:08:53.280 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 19:08:53.280 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:08:53.280 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 19:08:53.280 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 19:08:53.280 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 19:08:53.280 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 19:08:53.280 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:08:53.280 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 19:08:53.280 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:08:53.280 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:08:53.280 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:08:53.280 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 19:08:53.280 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 19:08:53.280 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:08:53.280 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 19:08:53.280 | INFO | ------------------------------------------ | 20260505 19:08:53.280 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:08:53.280 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 19:08:53.280 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:08:53.280 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:08:53.280 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.46e+01, 1.36e+01 / 2.18e+00 | 20260505 19:08:53.280 | INFO | PerfReport: 0.29%, AggregateStats -- 9.24e-02 / 1.03e-01, 1.07e-02 / 2.67e-03 | 20260505 19:08:53.280 | INFO | PerfReport: 11.77%, Baseline -- 3.71e+00 / 4.56e+00, 5.63e-01 / 1.05e-01 | 20260505 19:08:53.280 | INFO | PerfReport: 3.26%, Histogram -- 1.03e+00 / 1.53e+00, 1.88e-01 / 8.69e-02 | 20260505 19:08:53.280 | INFO | PerfReport: 15.95%, DME -- 4.02e+01 / 4.44e+01, 4.17e+00 / 2.23e+00 | 20260505 19:08:53.280 | INFO | PerfReport: 56.97%, FrameLabeling -- 1.79e+01 / 1.93e+01, 1.44e+00 / 9.01e-01 | 20260505 19:08:53.280 | INFO | PerfReport: 7.30%, PulseAccumulating -- 2.30e+00 / 2.89e+00, 4.43e-01 / 1.48e-01 | 20260505 19:08:53.280 | INFO | PerfReport: 3.30%, Metrics -- 1.04e+00 / 1.37e+00, 2.87e-01 / 1.68e-01 | 20260505 19:08:53.280 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.62e-01 / 4.30e-01, 6.37e-02 / 4.52e-02 | 20260505 19:08:53.281 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 19:08:53.281 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:08:53.281 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:08:53.281 | INFO | PerfReport: Full Execution -- 9.97e+01 / -nan, 2.56e+01 / 0.00e+00 | 20260505 19:08:53.281 | INFO | PerfReport: 22.53%, Upload -- 2.25e+01 / 0.00e+00, 7.82e+00 / 0.00e+00 | 20260505 19:08:53.281 | INFO | PerfReport: 21.79%, Download -- 2.17e+01 / 0.00e+00, 7.70e+00 / 0.00e+00 | 20260505 19:08:53.281 | INFO | PerfReport: 31.61%, Compute -- 3.15e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 19:08:53.281 | INFO | PerfReport: 2.35%, LockUpload -- 2.35e+00 / 0.00e+00, 8.12e+00 / 0.00e+00 | 20260505 19:08:53.281 | INFO | PerfReport: 1.37%, LockDownload -- 1.37e+00 / 0.00e+00, 2.93e+00 / 0.00e+00 | 20260505 19:08:53.281 | INFO | PerfReport: 20.34%, LockCompute -- 2.03e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260505 19:08:53.281 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 2.24e-02 / 0.00e+00 | 20260505 19:08:53.411 | INFO | Starting a round of Baz IO | 20260505 19:08:56.583 | INFO | Populating RTMetrics report | 20260505 19:08:56.583 | INFO | RTMetrics report populated | 20260505 19:08:56.583 | INFO | Opening tmp RTMetrics json file | 20260505 19:08:56.584 | INFO | Serializing RTMetrics to json | 20260505 19:08:56.585 | INFO | Serialization done, swapping tmp file now... | 20260505 19:08:56.585 | INFO | RTMetrics done | 20260505 19:08:56.675 | INFO | Finished a round of Baz IO: Wrote 2.062 GiB in 3.264 seconds | 20260505 19:08:56.675 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.81% / 0.00% / 78.18% | 20260505 19:08:56.675 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.219544% / 3.97364e-06% / 8.74201e-05% | 20260505 19:08:56.675 | INFO | Baz IO overall: | 20260505 19:08:56.675 | INFO | BytesWritten : 3287690458078 | 20260505 19:08:56.675 | INFO | NumEvents : 1556322103486 | 20260505 19:08:58.422 | INFO | Analyzing chunk frames = [5603328,5603840) | 20260505 19:08:58.423 | INFO | MemInfo: { | 20260505 19:08:58.423 | INFO | "gpu" : " 17 GB", | 20260505 19:08:58.423 | INFO | "huge" : "", | 20260505 19:08:58.423 | INFO | "huge_shared" : " 27 GB", | 20260505 19:08:58.423 | INFO | "malloc" : " 14 GB", | 20260505 19:08:58.423 | INFO | "pinned" : " 29 GB" | 20260505 19:08:58.423 | INFO | } | 20260505 19:08:58.423 | INFO | Perf report for the last 5.14409 seconds: | 20260505 19:08:58.423 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:08:58.423 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:58.423 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:58.423 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:58.423 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 19:08:58.423 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:08:58.423 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260505 19:08:58.423 | INFO | PRE_HQ : 60.0%, 1.8, 55.4ms, 0 | 20260505 19:08:58.423 | INFO | BAZWRITER : 63.0%, 3.7, 120.6ms, 0 | 20260505 19:08:58.423 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 19:08:58.423 | INFO | Uploaded 36GiB to GPU in 2.16308s (16.6429GiB/s) | 20260505 19:08:58.423 | INFO | Downloaded 38GiB to GPU in 2.24454s (16.93GiB/s) | 20260505 19:09:03.551 | INFO | Analyzing chunk frames = [5603840,5604352) | 20260505 19:09:03.551 | INFO | MemInfo: { | 20260505 19:09:03.551 | INFO | "gpu" : " 17 GB", | 20260505 19:09:03.551 | INFO | "huge" : "", | 20260505 19:09:03.551 | INFO | "huge_shared" : " 27 GB", | 20260505 19:09:03.551 | INFO | "malloc" : " 14 GB", | 20260505 19:09:03.551 | INFO | "pinned" : " 29 GB" | 20260505 19:09:03.551 | INFO | } | 20260505 19:09:03.551 | INFO | Perf report for the last 5.12884 seconds: | 20260505 19:09:03.552 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:09:03.552 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:03.552 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:03.552 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:03.552 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260505 19:09:03.552 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:03.552 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:03.552 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 19:09:03.552 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:03.552 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:03.552 | INFO | Uploaded 36GiB to GPU in 2.09041s (17.2215GiB/s) | 20260505 19:09:03.552 | INFO | Downloaded 34GiB to GPU in 2.02454s (16.7939GiB/s) | 20260505 19:09:08.681 | INFO | Analyzing chunk frames = [5604352,5604864) | 20260505 19:09:08.681 | INFO | MemInfo: { | 20260505 19:09:08.681 | INFO | "gpu" : " 17 GB", | 20260505 19:09:08.681 | INFO | "huge" : "", | 20260505 19:09:08.681 | INFO | "huge_shared" : " 27 GB", | 20260505 19:09:08.681 | INFO | "malloc" : " 14 GB", | 20260505 19:09:08.681 | INFO | "pinned" : " 29 GB" | 20260505 19:09:08.681 | INFO | } | 20260505 19:09:08.681 | INFO | Perf report for the last 5.12996 seconds: | 20260505 19:09:08.682 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:09:08.682 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:08.682 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:08.682 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:08.682 | INFO | ANALYSIS : 62.8%, 3.0, 97.6ms, 0 | 20260505 19:09:08.682 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:08.682 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:08.682 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:09:08.682 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:08.682 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:08.682 | INFO | Uploaded 36GiB to GPU in 2.21025s (16.2878GiB/s) | 20260505 19:09:08.682 | INFO | Downloaded 34GiB to GPU in 2.07332s (16.3988GiB/s) | 20260505 19:09:13.831 | INFO | Analyzing chunk frames = [5604864,5605376) | 20260505 19:09:13.831 | INFO | MemInfo: { | 20260505 19:09:13.831 | INFO | "gpu" : " 17 GB", | 20260505 19:09:13.831 | INFO | "huge" : "", | 20260505 19:09:13.831 | INFO | "huge_shared" : " 27 GB", | 20260505 19:09:13.831 | INFO | "malloc" : " 14 GB", | 20260505 19:09:13.831 | INFO | "pinned" : " 29 GB" | 20260505 19:09:13.831 | INFO | } | 20260505 19:09:13.831 | INFO | Perf report for the last 5.14955 seconds: | 20260505 19:09:13.831 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:09:13.831 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:13.831 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:13.831 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:13.831 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260505 19:09:13.831 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:13.831 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:13.831 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 19:09:13.831 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:13.831 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:13.831 | INFO | Uploaded 36GiB to GPU in 2.18716s (16.4597GiB/s) | 20260505 19:09:13.831 | INFO | Downloaded 34GiB to GPU in 2.14143s (15.8772GiB/s) | 20260505 19:09:18.979 | INFO | Analyzing chunk frames = [5605376,5605888) | 20260505 19:09:18.979 | INFO | MemInfo: { | 20260505 19:09:18.979 | INFO | "gpu" : " 17 GB", | 20260505 19:09:18.979 | INFO | "huge" : "", | 20260505 19:09:18.979 | INFO | "huge_shared" : " 27 GB", | 20260505 19:09:18.979 | INFO | "malloc" : " 14 GB", | 20260505 19:09:18.979 | INFO | "pinned" : " 29 GB" | 20260505 19:09:18.979 | INFO | } | 20260505 19:09:18.979 | INFO | Perf report for the last 5.14788 seconds: | 20260505 19:09:18.979 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:09:18.979 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:18.979 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:18.979 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:18.979 | INFO | ANALYSIS : 61.8%, 3.0, 96.7ms, 0 | 20260505 19:09:18.979 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:18.979 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:18.979 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:09:18.979 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:18.979 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:18.979 | INFO | Uploaded 36GiB to GPU in 2.13251s (16.8815GiB/s) | 20260505 19:09:18.979 | INFO | Downloaded 34GiB to GPU in 2.06318s (16.4794GiB/s) | 20260505 19:09:24.109 | INFO | Analyzing chunk frames = [5605888,5606400) | 20260505 19:09:24.109 | INFO | MemInfo: { | 20260505 19:09:24.109 | INFO | "gpu" : " 17 GB", | 20260505 19:09:24.109 | INFO | "huge" : "", | 20260505 19:09:24.109 | INFO | "huge_shared" : " 27 GB", | 20260505 19:09:24.109 | INFO | "malloc" : " 14 GB", | 20260505 19:09:24.109 | INFO | "pinned" : " 29 GB" | 20260505 19:09:24.109 | INFO | } | 20260505 19:09:24.109 | INFO | Perf report for the last 5.13014 seconds: | 20260505 19:09:24.109 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:09:24.109 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:24.109 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:24.109 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:24.109 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260505 19:09:24.109 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:24.109 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:24.109 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260505 19:09:24.109 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:24.109 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:24.109 | INFO | Uploaded 36GiB to GPU in 2.0838s (17.2761GiB/s) | 20260505 19:09:24.109 | INFO | Downloaded 34GiB to GPU in 2.03093s (16.7411GiB/s) | 20260505 19:09:29.244 | INFO | Analyzing chunk frames = [5606400,5606912) | 20260505 19:09:29.244 | INFO | MemInfo: { | 20260505 19:09:29.244 | INFO | "gpu" : " 17 GB", | 20260505 19:09:29.244 | INFO | "huge" : "", | 20260505 19:09:29.244 | INFO | "huge_shared" : " 27 GB", | 20260505 19:09:29.244 | INFO | "malloc" : " 14 GB", | 20260505 19:09:29.244 | INFO | "pinned" : " 29 GB" | 20260505 19:09:29.244 | INFO | } | 20260505 19:09:29.244 | INFO | Perf report for the last 5.13548 seconds: | 20260505 19:09:29.244 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:09:29.244 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:29.244 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:29.244 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:29.244 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 19:09:29.244 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:29.245 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:29.245 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 19:09:29.245 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:29.245 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:29.245 | INFO | Uploaded 36GiB to GPU in 2.11557s (17.0167GiB/s) | 20260505 19:09:29.245 | INFO | Downloaded 34GiB to GPU in 2.14864s (15.824GiB/s) | 20260505 19:09:34.385 | INFO | Analyzing chunk frames = [5606912,5607424) | 20260505 19:09:34.385 | INFO | MemInfo: { | 20260505 19:09:34.385 | INFO | "gpu" : " 17 GB", | 20260505 19:09:34.385 | INFO | "huge" : "", | 20260505 19:09:34.385 | INFO | "huge_shared" : " 27 GB", | 20260505 19:09:34.385 | INFO | "malloc" : " 14 GB", | 20260505 19:09:34.385 | INFO | "pinned" : " 29 GB" | 20260505 19:09:34.385 | INFO | } | 20260505 19:09:34.385 | INFO | Perf report for the last 5.14096 seconds: | 20260505 19:09:34.385 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:09:34.385 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:34.385 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:34.386 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:34.386 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 19:09:34.386 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:34.386 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:34.386 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:09:34.386 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:34.386 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:34.386 | INFO | Uploaded 36GiB to GPU in 2.18981s (16.4398GiB/s) | 20260505 19:09:34.386 | INFO | Downloaded 34GiB to GPU in 2.03473s (16.7098GiB/s) | 20260505 19:09:34.520 | INFO | Starting a round of Baz IO | 20260505 19:09:37.698 | INFO | Populating RTMetrics report | 20260505 19:09:37.698 | INFO | RTMetrics report populated | 20260505 19:09:37.699 | INFO | Opening tmp RTMetrics json file | 20260505 19:09:37.700 | INFO | Serializing RTMetrics to json | 20260505 19:09:37.701 | INFO | Serialization done, swapping tmp file now... | 20260505 19:09:37.701 | INFO | RTMetrics done | 20260505 19:09:37.788 | INFO | Finished a round of Baz IO: Wrote 2.063 GiB in 3.268 seconds | 20260505 19:09:37.788 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.81% / 0.00% / 78.19% | 20260505 19:09:37.788 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.220144% / 0% / 7.94729e-05% | 20260505 19:09:37.788 | INFO | Baz IO overall: | 20260505 19:09:37.788 | INFO | BytesWritten : 3289906007144 | 20260505 19:09:37.788 | INFO | NumEvents : 1557352804924 | 20260505 19:09:39.530 | INFO | Analyzing chunk frames = [5607424,5607936) | 20260505 19:09:39.530 | INFO | MemInfo: { | 20260505 19:09:39.530 | INFO | "gpu" : " 17 GB", | 20260505 19:09:39.530 | INFO | "huge" : "", | 20260505 19:09:39.530 | INFO | "huge_shared" : " 27 GB", | 20260505 19:09:39.530 | INFO | "malloc" : " 14 GB", | 20260505 19:09:39.530 | INFO | "pinned" : " 29 GB" | 20260505 19:09:39.530 | INFO | } | 20260505 19:09:39.530 | INFO | Perf report for the last 5.14446 seconds: | 20260505 19:09:39.530 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:09:39.530 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:39.530 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:39.530 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:39.530 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 19:09:39.530 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:39.530 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260505 19:09:39.530 | INFO | PRE_HQ : 60.3%, 1.8, 55.7ms, 0 | 20260505 19:09:39.530 | INFO | BAZWRITER : 63.1%, 3.7, 121.4ms, 0 | 20260505 19:09:39.530 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 19:09:39.530 | INFO | Uploaded 36GiB to GPU in 2.16326s (16.6415GiB/s) | 20260505 19:09:39.530 | INFO | Downloaded 38GiB to GPU in 2.24125s (16.9548GiB/s) | 20260505 19:09:44.666 | INFO | Analyzing chunk frames = [5607936,5608448) | 20260505 19:09:44.666 | INFO | MemInfo: { | 20260505 19:09:44.666 | INFO | "gpu" : " 17 GB", | 20260505 19:09:44.666 | INFO | "huge" : "", | 20260505 19:09:44.666 | INFO | "huge_shared" : " 27 GB", | 20260505 19:09:44.666 | INFO | "malloc" : " 14 GB", | 20260505 19:09:44.666 | INFO | "pinned" : " 29 GB" | 20260505 19:09:44.666 | INFO | } | 20260505 19:09:44.666 | INFO | Perf report for the last 5.13631 seconds: | 20260505 19:09:44.666 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:09:44.666 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:44.666 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:44.666 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:44.666 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 19:09:44.667 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:44.667 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:44.667 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:09:44.667 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:44.667 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:44.667 | INFO | Uploaded 36GiB to GPU in 2.09667s (17.1701GiB/s) | 20260505 19:09:44.667 | INFO | Downloaded 34GiB to GPU in 2.02661s (16.7768GiB/s) | 20260505 19:09:49.798 | INFO | Analyzing chunk frames = [5608448,5608960) | 20260505 19:09:49.798 | INFO | MemInfo: { | 20260505 19:09:49.798 | INFO | "gpu" : " 17 GB", | 20260505 19:09:49.798 | INFO | "huge" : "", | 20260505 19:09:49.798 | INFO | "huge_shared" : " 27 GB", | 20260505 19:09:49.798 | INFO | "malloc" : " 14 GB", | 20260505 19:09:49.798 | INFO | "pinned" : " 29 GB" | 20260505 19:09:49.798 | INFO | } | 20260505 19:09:49.798 | INFO | Perf report for the last 5.13212 seconds: | 20260505 19:09:49.799 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:09:49.799 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:49.799 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:49.799 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:49.799 | INFO | ANALYSIS : 62.4%, 3.0, 97.0ms, 0 | 20260505 19:09:49.799 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:49.799 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:49.799 | INFO | PRE_HQ : 27.5%, 1.0, 14.2ms, 0 | 20260505 19:09:49.799 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:49.799 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:49.799 | INFO | Uploaded 36GiB to GPU in 2.14001s (16.8224GiB/s) | 20260505 19:09:49.799 | INFO | Downloaded 34GiB to GPU in 2.03542s (16.7042GiB/s) | 20260505 19:09:54.938 | INFO | Analyzing chunk frames = [5608960,5609472) | 20260505 19:09:54.938 | INFO | MemInfo: { | 20260505 19:09:54.938 | INFO | "gpu" : " 17 GB", | 20260505 19:09:54.938 | INFO | "huge" : "", | 20260505 19:09:54.938 | INFO | "huge_shared" : " 27 GB", | 20260505 19:09:54.938 | INFO | "malloc" : " 14 GB", | 20260505 19:09:54.938 | INFO | "pinned" : " 29 GB" | 20260505 19:09:54.938 | INFO | } | 20260505 19:09:54.938 | INFO | Perf report for the last 5.13961 seconds: | 20260505 19:09:54.938 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:09:54.938 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:54.938 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:54.938 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:54.938 | INFO | ANALYSIS : 62.7%, 3.0, 97.4ms, 0 | 20260505 19:09:54.938 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:54.938 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:54.938 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 19:09:54.938 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:54.938 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:09:54.938 | INFO | Uploaded 36GiB to GPU in 2.21918s (16.2222GiB/s) | 20260505 19:09:54.938 | INFO | Downloaded 34GiB to GPU in 2.12394s (16.008GiB/s) | 20260505 19:10:00.089 | INFO | Analyzing chunk frames = [5609472,5609984) | 20260505 19:10:00.089 | INFO | MemInfo: { | 20260505 19:10:00.089 | INFO | "gpu" : " 17 GB", | 20260505 19:10:00.089 | INFO | "huge" : "", | 20260505 19:10:00.089 | INFO | "huge_shared" : " 27 GB", | 20260505 19:10:00.089 | INFO | "malloc" : " 14 GB", | 20260505 19:10:00.089 | INFO | "pinned" : " 29 GB" | 20260505 19:10:00.089 | INFO | } | 20260505 19:10:00.089 | INFO | Perf report for the last 5.15128 seconds: | 20260505 19:10:00.089 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:10:00.089 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:00.089 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:00.089 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:00.090 | INFO | ANALYSIS : 62.0%, 3.0, 97.1ms, 0 | 20260505 19:10:00.090 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:00.090 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:00.090 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:10:00.090 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:00.090 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:00.090 | INFO | Uploaded 36GiB to GPU in 2.14258s (16.8022GiB/s) | 20260505 19:10:00.090 | INFO | Downloaded 34GiB to GPU in 2.06305s (16.4805GiB/s) | 20260505 19:10:05.231 | INFO | Analyzing chunk frames = [5609984,5610496) | 20260505 19:10:05.231 | INFO | MemInfo: { | 20260505 19:10:05.231 | INFO | "gpu" : " 17 GB", | 20260505 19:10:05.231 | INFO | "huge" : "", | 20260505 19:10:05.231 | INFO | "huge_shared" : " 27 GB", | 20260505 19:10:05.231 | INFO | "malloc" : " 14 GB", | 20260505 19:10:05.231 | INFO | "pinned" : " 29 GB" | 20260505 19:10:05.231 | INFO | } | 20260505 19:10:05.231 | INFO | Perf report for the last 5.14134 seconds: | 20260505 19:10:05.231 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:10:05.231 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:05.231 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:05.231 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:05.231 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 19:10:05.231 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:05.231 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:05.231 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:10:05.231 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:05.231 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:05.231 | INFO | Uploaded 36GiB to GPU in 2.15798s (16.6823GiB/s) | 20260505 19:10:05.231 | INFO | Downloaded 34GiB to GPU in 2.06162s (16.4919GiB/s) | 20260505 19:10:10.364 | INFO | Analyzing chunk frames = [5610496,5611008) | 20260505 19:10:10.364 | INFO | MemInfo: { | 20260505 19:10:10.364 | INFO | "gpu" : " 17 GB", | 20260505 19:10:10.364 | INFO | "huge" : "", | 20260505 19:10:10.364 | INFO | "huge_shared" : " 27 GB", | 20260505 19:10:10.364 | INFO | "malloc" : " 14 GB", | 20260505 19:10:10.364 | INFO | "pinned" : " 29 GB" | 20260505 19:10:10.364 | INFO | } | 20260505 19:10:10.364 | INFO | Perf report for the last 5.13352 seconds: | 20260505 19:10:10.364 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:10:10.364 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:10.364 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:10.364 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:10.364 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 19:10:10.364 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:10.364 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:10.364 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 19:10:10.364 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:10.364 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:10.364 | INFO | Uploaded 36GiB to GPU in 2.11565s (17.016GiB/s) | 20260505 19:10:10.364 | INFO | Downloaded 34GiB to GPU in 2.12288s (16.016GiB/s) | 20260505 19:10:15.504 | INFO | Analyzing chunk frames = [5611008,5611520) | 20260505 19:10:15.504 | INFO | MemInfo: { | 20260505 19:10:15.504 | INFO | "gpu" : " 17 GB", | 20260505 19:10:15.504 | INFO | "huge" : "", | 20260505 19:10:15.504 | INFO | "huge_shared" : " 27 GB", | 20260505 19:10:15.504 | INFO | "malloc" : " 14 GB", | 20260505 19:10:15.504 | INFO | "pinned" : " 29 GB" | 20260505 19:10:15.504 | INFO | } | 20260505 19:10:15.504 | INFO | Perf report for the last 5.14007 seconds: | 20260505 19:10:15.504 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:10:15.504 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:15.504 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:15.504 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:15.505 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 19:10:15.505 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:15.505 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:15.505 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 19:10:15.505 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:15.505 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:15.505 | INFO | Uploaded 36GiB to GPU in 2.20361s (16.3368GiB/s) | 20260505 19:10:15.505 | INFO | Downloaded 34GiB to GPU in 2.04554s (16.6215GiB/s) | 20260505 19:10:15.505 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 19:10:15.505 | INFO | Total mem usage: 15.3 GiB | 20260505 19:10:15.505 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 19:10:15.505 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.23%) | 20260505 19:10:15.505 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 19:10:15.505 | INFO | High Water Breakdown: | 20260505 19:10:15.505 | INFO | 61% : 8.64 GiB ( 90.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 19:10:15.505 | INFO | 38.8% : 5.5 GiB ( 82.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 19:10:15.505 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 19:10:15.505 | INFO | ------------------------------------------ | 20260505 19:10:15.505 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:10:15.505 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 19:10:15.505 | INFO | Total mem usage: 26.5 GiB | 20260505 19:10:15.505 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 19:10:15.505 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 19:10:15.505 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 19:10:15.505 | INFO | High Water Breakdown: | 20260505 19:10:15.505 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:10:15.505 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:10:15.505 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:10:15.505 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:10:15.505 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:10:15.505 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:10:15.505 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 19:10:15.505 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 19:10:15.505 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:10:15.505 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 19:10:15.505 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 19:10:15.505 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 19:10:15.505 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 19:10:15.505 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 19:10:15.505 | INFO | ------------------------------------------ | 20260505 19:10:15.505 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:10:15.505 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 19:10:15.505 | INFO | Total mem usage: 25.6 GiB | 20260505 19:10:15.505 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 19:10:15.505 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 19:10:15.505 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 19:10:15.505 | INFO | High Water Breakdown: | 20260505 19:10:15.505 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 19:10:15.505 | INFO | ------------------------------------------ | 20260505 19:10:15.505 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:10:15.505 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 19:10:15.505 | INFO | Total mem usage: 15.7 GiB | 20260505 19:10:15.505 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 19:10:15.505 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 19:10:15.505 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 19:10:15.506 | INFO | High Water Breakdown: | 20260505 19:10:15.506 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 19:10:15.506 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 19:10:15.506 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 19:10:15.506 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 19:10:15.506 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 19:10:15.506 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 19:10:15.506 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 19:10:15.506 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 19:10:15.506 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 19:10:15.506 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:10:15.506 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 19:10:15.506 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:10:15.506 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 19:10:15.506 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 19:10:15.506 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 19:10:15.506 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 19:10:15.506 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:10:15.506 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 19:10:15.506 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:10:15.506 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:10:15.506 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:10:15.506 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 19:10:15.506 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 19:10:15.506 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:10:15.506 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 19:10:15.506 | INFO | ------------------------------------------ | 20260505 19:10:15.506 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:10:15.506 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 19:10:15.506 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:10:15.506 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:10:15.506 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.48e+01, 1.36e+01 / 1.75e+00 | 20260505 19:10:15.506 | INFO | PerfReport: 0.29%, AggregateStats -- 9.24e-02 / 1.03e-01, 1.08e-02 / 3.60e-03 | 20260505 19:10:15.506 | INFO | PerfReport: 11.77%, Baseline -- 3.71e+00 / 4.56e+00, 5.63e-01 / 1.18e-01 | 20260505 19:10:15.506 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.53e+00, 1.90e-01 / 8.14e-02 | 20260505 19:10:15.506 | INFO | PerfReport: 15.98%, DME -- 4.03e+01 / 4.46e+01, 4.08e+00 / 1.75e+00 | 20260505 19:10:15.506 | INFO | PerfReport: 56.93%, FrameLabeling -- 1.79e+01 / 1.94e+01, 1.43e+00 / 9.26e-01 | 20260505 19:10:15.506 | INFO | PerfReport: 7.29%, PulseAccumulating -- 2.30e+00 / 2.90e+00, 4.40e-01 / 9.76e-02 | 20260505 19:10:15.506 | INFO | PerfReport: 3.31%, Metrics -- 1.04e+00 / 1.40e+00, 2.87e-01 / 1.40e-01 | 20260505 19:10:15.506 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.63e-01 / 4.29e-01, 6.41e-02 / 3.43e-02 | 20260505 19:10:15.506 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 19:10:15.506 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:10:15.506 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:10:15.506 | INFO | PerfReport: Full Execution -- 9.96e+01 / -nan, 2.55e+01 / 0.00e+00 | 20260505 19:10:15.506 | INFO | PerfReport: 22.49%, Upload -- 2.24e+01 / 0.00e+00, 7.81e+00 / 0.00e+00 | 20260505 19:10:15.506 | INFO | PerfReport: 21.88%, Download -- 2.18e+01 / 0.00e+00, 7.77e+00 / 0.00e+00 | 20260505 19:10:15.506 | INFO | PerfReport: 31.63%, Compute -- 3.15e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 19:10:15.506 | INFO | PerfReport: 2.27%, LockUpload -- 2.26e+00 / 0.00e+00, 7.87e+00 / 0.00e+00 | 20260505 19:10:15.506 | INFO | PerfReport: 1.38%, LockDownload -- 1.38e+00 / 0.00e+00, 2.87e+00 / 0.00e+00 | 20260505 19:10:15.506 | INFO | PerfReport: 20.33%, LockCompute -- 2.03e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 19:10:15.506 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.69e-02 / 0.00e+00 | 20260505 19:10:15.638 | INFO | Starting a round of Baz IO | 20260505 19:10:18.810 | INFO | Populating RTMetrics report | 20260505 19:10:18.810 | INFO | RTMetrics report populated | 20260505 19:10:18.810 | INFO | Opening tmp RTMetrics json file | 20260505 19:10:18.813 | INFO | Serializing RTMetrics to json | 20260505 19:10:18.814 | INFO | Serialization done, swapping tmp file now... | 20260505 19:10:18.815 | INFO | RTMetrics done | 20260505 19:10:18.903 | INFO | Finished a round of Baz IO: Wrote 2.063 GiB in 3.264 seconds | 20260505 19:10:18.903 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.81% / 0.00% / 78.19% | 20260505 19:10:18.903 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.21923% / 0% / 7.15256e-05% | 20260505 19:10:18.903 | INFO | Baz IO overall: | 20260505 19:10:18.903 | INFO | BytesWritten : 3292120955263 | 20260505 19:10:18.903 | INFO | NumEvents : 1558382891048 | 20260505 19:10:20.639 | INFO | Analyzing chunk frames = [5611520,5612032) | 20260505 19:10:20.639 | INFO | MemInfo: { | 20260505 19:10:20.639 | INFO | "gpu" : " 17 GB", | 20260505 19:10:20.639 | INFO | "huge" : "", | 20260505 19:10:20.639 | INFO | "huge_shared" : " 27 GB", | 20260505 19:10:20.639 | INFO | "malloc" : " 14 GB", | 20260505 19:10:20.639 | INFO | "pinned" : " 29 GB" | 20260505 19:10:20.639 | INFO | } | 20260505 19:10:20.639 | INFO | Perf report for the last 5.13497 seconds: | 20260505 19:10:20.639 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:10:20.639 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:20.639 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:20.639 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:20.639 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 19:10:20.639 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:20.640 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260505 19:10:20.640 | INFO | PRE_HQ : 60.5%, 1.8, 55.8ms, 0 | 20260505 19:10:20.640 | INFO | BAZWRITER : 63.1%, 3.7, 121.0ms, 0 | 20260505 19:10:20.640 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 19:10:20.640 | INFO | Uploaded 36GiB to GPU in 2.16586s (16.6216GiB/s) | 20260505 19:10:20.640 | INFO | Downloaded 38GiB to GPU in 2.25181s (16.8753GiB/s) | 20260505 19:10:25.781 | INFO | Analyzing chunk frames = [5612032,5612544) | 20260505 19:10:25.781 | INFO | MemInfo: { | 20260505 19:10:25.781 | INFO | "gpu" : " 17 GB", | 20260505 19:10:25.781 | INFO | "huge" : "", | 20260505 19:10:25.781 | INFO | "huge_shared" : " 27 GB", | 20260505 19:10:25.781 | INFO | "malloc" : " 14 GB", | 20260505 19:10:25.781 | INFO | "pinned" : " 29 GB" | 20260505 19:10:25.781 | INFO | } | 20260505 19:10:25.781 | INFO | Perf report for the last 5.14221 seconds: | 20260505 19:10:25.782 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:10:25.782 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:25.782 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:25.782 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:25.782 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260505 19:10:25.782 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:25.782 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:25.782 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:10:25.782 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:25.782 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:25.782 | INFO | Uploaded 36GiB to GPU in 2.13347s (16.8739GiB/s) | 20260505 19:10:25.782 | INFO | Downloaded 34GiB to GPU in 2.02962s (16.7519GiB/s) | 20260505 19:10:30.931 | INFO | Analyzing chunk frames = [5612544,5613056) | 20260505 19:10:30.931 | INFO | MemInfo: { | 20260505 19:10:30.931 | INFO | "gpu" : " 17 GB", | 20260505 19:10:30.931 | INFO | "huge" : "", | 20260505 19:10:30.931 | INFO | "huge_shared" : " 27 GB", | 20260505 19:10:30.931 | INFO | "malloc" : " 14 GB", | 20260505 19:10:30.931 | INFO | "pinned" : " 29 GB" | 20260505 19:10:30.931 | INFO | } | 20260505 19:10:30.931 | INFO | Perf report for the last 5.14971 seconds: | 20260505 19:10:30.931 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:10:30.931 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:30.931 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:30.931 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:30.931 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260505 19:10:30.931 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:30.931 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:30.931 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 19:10:30.931 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:30.931 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:30.932 | INFO | Uploaded 36GiB to GPU in 2.1396s (16.8256GiB/s) | 20260505 19:10:30.932 | INFO | Downloaded 34GiB to GPU in 2.03327s (16.7218GiB/s) | 20260505 19:10:36.062 | INFO | Analyzing chunk frames = [5613056,5613568) | 20260505 19:10:36.062 | INFO | MemInfo: { | 20260505 19:10:36.062 | INFO | "gpu" : " 17 GB", | 20260505 19:10:36.062 | INFO | "huge" : "", | 20260505 19:10:36.062 | INFO | "huge_shared" : " 27 GB", | 20260505 19:10:36.062 | INFO | "malloc" : " 14 GB", | 20260505 19:10:36.062 | INFO | "pinned" : " 29 GB" | 20260505 19:10:36.062 | INFO | } | 20260505 19:10:36.062 | INFO | Perf report for the last 5.13075 seconds: | 20260505 19:10:36.062 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:10:36.062 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:36.062 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:36.062 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:36.062 | INFO | ANALYSIS : 62.6%, 3.0, 97.1ms, 0 | 20260505 19:10:36.062 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:36.062 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:36.062 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 19:10:36.062 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:36.062 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:36.062 | INFO | Uploaded 36GiB to GPU in 2.21273s (16.2695GiB/s) | 20260505 19:10:36.062 | INFO | Downloaded 34GiB to GPU in 2.05903s (16.5126GiB/s) | 20260505 19:10:41.192 | INFO | Analyzing chunk frames = [5613568,5614080) | 20260505 19:10:41.192 | INFO | MemInfo: { | 20260505 19:10:41.192 | INFO | "gpu" : " 17 GB", | 20260505 19:10:41.192 | INFO | "huge" : "", | 20260505 19:10:41.192 | INFO | "huge_shared" : " 27 GB", | 20260505 19:10:41.192 | INFO | "malloc" : " 14 GB", | 20260505 19:10:41.192 | INFO | "pinned" : " 29 GB" | 20260505 19:10:41.192 | INFO | } | 20260505 19:10:41.192 | INFO | Perf report for the last 5.12981 seconds: | 20260505 19:10:41.192 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:10:41.192 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:41.192 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:41.192 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:41.192 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260505 19:10:41.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:41.192 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:41.192 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 19:10:41.192 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:41.192 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:41.192 | INFO | Uploaded 36GiB to GPU in 2.14545s (16.7797GiB/s) | 20260505 19:10:41.192 | INFO | Downloaded 34GiB to GPU in 2.06415s (16.4717GiB/s) | 20260505 19:10:46.338 | INFO | Analyzing chunk frames = [5614080,5614592) | 20260505 19:10:46.338 | INFO | MemInfo: { | 20260505 19:10:46.338 | INFO | "gpu" : " 17 GB", | 20260505 19:10:46.338 | INFO | "huge" : "", | 20260505 19:10:46.338 | INFO | "huge_shared" : " 27 GB", | 20260505 19:10:46.338 | INFO | "malloc" : " 14 GB", | 20260505 19:10:46.338 | INFO | "pinned" : " 29 GB" | 20260505 19:10:46.338 | INFO | } | 20260505 19:10:46.338 | INFO | Perf report for the last 5.14667 seconds: | 20260505 19:10:46.338 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:10:46.338 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:46.338 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:46.339 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:46.339 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 19:10:46.339 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:46.339 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:46.339 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:10:46.339 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:46.339 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:46.339 | INFO | Uploaded 36GiB to GPU in 2.14132s (16.8121GiB/s) | 20260505 19:10:46.339 | INFO | Downloaded 34GiB to GPU in 2.04043s (16.6631GiB/s) | 20260505 19:10:51.479 | INFO | Analyzing chunk frames = [5614592,5615104) | 20260505 19:10:51.479 | INFO | MemInfo: { | 20260505 19:10:51.479 | INFO | "gpu" : " 17 GB", | 20260505 19:10:51.479 | INFO | "huge" : "", | 20260505 19:10:51.479 | INFO | "huge_shared" : " 27 GB", | 20260505 19:10:51.479 | INFO | "malloc" : " 14 GB", | 20260505 19:10:51.479 | INFO | "pinned" : " 29 GB" | 20260505 19:10:51.479 | INFO | } | 20260505 19:10:51.479 | INFO | Perf report for the last 5.14089 seconds: | 20260505 19:10:51.479 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:10:51.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:51.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:51.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:51.479 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 19:10:51.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:51.479 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:51.479 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 19:10:51.480 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:51.480 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:51.480 | INFO | Uploaded 36GiB to GPU in 2.11109s (17.0528GiB/s) | 20260505 19:10:51.480 | INFO | Downloaded 34GiB to GPU in 2.13903s (15.8951GiB/s) | 20260505 19:10:56.614 | INFO | Analyzing chunk frames = [5615104,5615616) | 20260505 19:10:56.614 | INFO | MemInfo: { | 20260505 19:10:56.614 | INFO | "gpu" : " 17 GB", | 20260505 19:10:56.614 | INFO | "huge" : "", | 20260505 19:10:56.614 | INFO | "huge_shared" : " 27 GB", | 20260505 19:10:56.614 | INFO | "malloc" : " 14 GB", | 20260505 19:10:56.614 | INFO | "pinned" : " 29 GB" | 20260505 19:10:56.614 | INFO | } | 20260505 19:10:56.615 | INFO | Perf report for the last 5.13528 seconds: | 20260505 19:10:56.615 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:10:56.615 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:56.615 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:56.615 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:56.615 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 19:10:56.615 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:56.615 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:56.615 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:10:56.615 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:56.615 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:10:56.615 | INFO | Uploaded 36GiB to GPU in 2.20639s (16.3162GiB/s) | 20260505 19:10:56.615 | INFO | Downloaded 34GiB to GPU in 2.04773s (16.6037GiB/s) | 20260505 19:10:56.748 | INFO | Starting a round of Baz IO | 20260505 19:10:59.920 | INFO | Populating RTMetrics report | 20260505 19:10:59.920 | INFO | RTMetrics report populated | 20260505 19:10:59.920 | INFO | Opening tmp RTMetrics json file | 20260505 19:10:59.922 | INFO | Serializing RTMetrics to json | 20260505 19:10:59.923 | INFO | Serialization done, swapping tmp file now... | 20260505 19:10:59.923 | INFO | RTMetrics done | 20260505 19:11:00.010 | INFO | Finished a round of Baz IO: Wrote 2.065 GiB in 3.262 seconds | 20260505 19:11:00.010 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.81% / 0.00% / 78.19% | 20260505 19:11:00.010 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.219874% / 0% / 7.15256e-05% | 20260505 19:11:00.010 | INFO | Baz IO overall: | 20260505 19:11:00.010 | INFO | BytesWritten : 3294338198947 | 20260505 19:11:00.010 | INFO | NumEvents : 1559414735171 | 20260505 19:11:01.749 | INFO | Analyzing chunk frames = [5615616,5616128) | 20260505 19:11:01.749 | INFO | MemInfo: { | 20260505 19:11:01.749 | INFO | "gpu" : " 17 GB", | 20260505 19:11:01.749 | INFO | "huge" : "", | 20260505 19:11:01.749 | INFO | "huge_shared" : " 27 GB", | 20260505 19:11:01.749 | INFO | "malloc" : " 14 GB", | 20260505 19:11:01.749 | INFO | "pinned" : " 29 GB" | 20260505 19:11:01.749 | INFO | } | 20260505 19:11:01.749 | INFO | Perf report for the last 5.13471 seconds: | 20260505 19:11:01.750 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:11:01.750 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:01.750 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:01.750 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:01.750 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 19:11:01.750 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:01.750 | INFO | RT_METRICS : 44.4%, 1.0, 23.2ms, 0 | 20260505 19:11:01.750 | INFO | PRE_HQ : 60.1%, 1.8, 55.7ms, 0 | 20260505 19:11:01.750 | INFO | BAZWRITER : 63.1%, 3.7, 121.1ms, 0 | 20260505 19:11:01.750 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 19:11:01.750 | INFO | Uploaded 36GiB to GPU in 2.15987s (16.6677GiB/s) | 20260505 19:11:01.750 | INFO | Downloaded 38GiB to GPU in 2.24435s (16.9314GiB/s) | 20260505 19:11:06.893 | INFO | Analyzing chunk frames = [5616128,5616640) | 20260505 19:11:06.893 | INFO | MemInfo: { | 20260505 19:11:06.893 | INFO | "gpu" : " 17 GB", | 20260505 19:11:06.893 | INFO | "huge" : "", | 20260505 19:11:06.893 | INFO | "huge_shared" : " 27 GB", | 20260505 19:11:06.893 | INFO | "malloc" : " 14 GB", | 20260505 19:11:06.893 | INFO | "pinned" : " 29 GB" | 20260505 19:11:06.893 | INFO | } | 20260505 19:11:06.893 | INFO | Perf report for the last 5.14406 seconds: | 20260505 19:11:06.893 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:11:06.893 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:06.893 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:06.893 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:06.893 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 19:11:06.893 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:06.893 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:06.893 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:11:06.893 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:06.893 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:06.894 | INFO | Uploaded 36GiB to GPU in 2.12223s (16.9633GiB/s) | 20260505 19:11:06.894 | INFO | Downloaded 34GiB to GPU in 2.02689s (16.7745GiB/s) | 20260505 19:11:12.038 | INFO | Analyzing chunk frames = [5616640,5617152) | 20260505 19:11:12.038 | INFO | MemInfo: { | 20260505 19:11:12.038 | INFO | "gpu" : " 17 GB", | 20260505 19:11:12.038 | INFO | "huge" : "", | 20260505 19:11:12.038 | INFO | "huge_shared" : " 27 GB", | 20260505 19:11:12.038 | INFO | "malloc" : " 14 GB", | 20260505 19:11:12.038 | INFO | "pinned" : " 29 GB" | 20260505 19:11:12.038 | INFO | } | 20260505 19:11:12.038 | INFO | Perf report for the last 5.14484 seconds: | 20260505 19:11:12.038 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:11:12.038 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:12.038 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:12.038 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:12.038 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 19:11:12.038 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:12.038 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:12.038 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:11:12.038 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:12.038 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:12.039 | INFO | Uploaded 36GiB to GPU in 2.21399s (16.2603GiB/s) | 20260505 19:11:12.039 | INFO | Downloaded 34GiB to GPU in 2.07468s (16.3881GiB/s) | 20260505 19:11:17.178 | INFO | Analyzing chunk frames = [5617152,5617664) | 20260505 19:11:17.178 | INFO | MemInfo: { | 20260505 19:11:17.178 | INFO | "gpu" : " 17 GB", | 20260505 19:11:17.178 | INFO | "huge" : "", | 20260505 19:11:17.178 | INFO | "huge_shared" : " 27 GB", | 20260505 19:11:17.178 | INFO | "malloc" : " 14 GB", | 20260505 19:11:17.178 | INFO | "pinned" : " 29 GB" | 20260505 19:11:17.178 | INFO | } | 20260505 19:11:17.178 | INFO | Perf report for the last 5.14005 seconds: | 20260505 19:11:17.178 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:11:17.178 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:17.178 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:17.178 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:17.178 | INFO | ANALYSIS : 62.4%, 3.0, 96.9ms, 0 | 20260505 19:11:17.178 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:17.178 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:17.178 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:11:17.178 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:17.178 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:17.179 | INFO | Uploaded 36GiB to GPU in 2.19699s (16.3861GiB/s) | 20260505 19:11:17.179 | INFO | Downloaded 34GiB to GPU in 2.06292s (16.4815GiB/s) | 20260505 19:11:22.311 | INFO | Analyzing chunk frames = [5617664,5618176) | 20260505 19:11:22.311 | INFO | MemInfo: { | 20260505 19:11:22.311 | INFO | "gpu" : " 17 GB", | 20260505 19:11:22.311 | INFO | "huge" : "", | 20260505 19:11:22.311 | INFO | "huge_shared" : " 27 GB", | 20260505 19:11:22.311 | INFO | "malloc" : " 14 GB", | 20260505 19:11:22.311 | INFO | "pinned" : " 29 GB" | 20260505 19:11:22.311 | INFO | } | 20260505 19:11:22.312 | INFO | Perf report for the last 5.13333 seconds: | 20260505 19:11:22.312 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:11:22.312 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:22.312 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:22.312 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:22.312 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260505 19:11:22.312 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:22.312 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:22.312 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:11:22.312 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:22.312 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:22.312 | INFO | Uploaded 36GiB to GPU in 2.1244s (16.946GiB/s) | 20260505 19:11:22.312 | INFO | Downloaded 34GiB to GPU in 2.05849s (16.517GiB/s) | 20260505 19:11:27.450 | INFO | Analyzing chunk frames = [5618176,5618688) | 20260505 19:11:27.450 | INFO | MemInfo: { | 20260505 19:11:27.450 | INFO | "gpu" : " 17 GB", | 20260505 19:11:27.450 | INFO | "huge" : "", | 20260505 19:11:27.450 | INFO | "huge_shared" : " 27 GB", | 20260505 19:11:27.450 | INFO | "malloc" : " 14 GB", | 20260505 19:11:27.450 | INFO | "pinned" : " 29 GB" | 20260505 19:11:27.450 | INFO | } | 20260505 19:11:27.450 | INFO | Perf report for the last 5.13826 seconds: | 20260505 19:11:27.450 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:11:27.450 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:27.450 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:27.450 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:27.450 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 19:11:27.450 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:27.450 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:27.450 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:11:27.450 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:27.450 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:27.450 | INFO | Uploaded 36GiB to GPU in 2.12842s (16.914GiB/s) | 20260505 19:11:27.450 | INFO | Downloaded 34GiB to GPU in 2.04937s (16.5904GiB/s) | 20260505 19:11:32.592 | INFO | Analyzing chunk frames = [5618688,5619200) | 20260505 19:11:32.592 | INFO | MemInfo: { | 20260505 19:11:32.592 | INFO | "gpu" : " 17 GB", | 20260505 19:11:32.592 | INFO | "huge" : "", | 20260505 19:11:32.592 | INFO | "huge_shared" : " 27 GB", | 20260505 19:11:32.592 | INFO | "malloc" : " 14 GB", | 20260505 19:11:32.592 | INFO | "pinned" : " 29 GB" | 20260505 19:11:32.592 | INFO | } | 20260505 19:11:32.592 | INFO | Perf report for the last 5.14252 seconds: | 20260505 19:11:32.592 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:11:32.592 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:32.592 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:32.592 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:32.593 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 19:11:32.593 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:32.593 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:32.593 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 19:11:32.593 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:32.593 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:32.593 | INFO | Uploaded 36GiB to GPU in 2.13849s (16.8343GiB/s) | 20260505 19:11:32.593 | INFO | Downloaded 34GiB to GPU in 2.15896s (15.7483GiB/s) | 20260505 19:11:37.736 | INFO | Analyzing chunk frames = [5619200,5619712) | 20260505 19:11:37.736 | INFO | MemInfo: { | 20260505 19:11:37.736 | INFO | "gpu" : " 17 GB", | 20260505 19:11:37.736 | INFO | "huge" : "", | 20260505 19:11:37.736 | INFO | "huge_shared" : " 27 GB", | 20260505 19:11:37.736 | INFO | "malloc" : " 14 GB", | 20260505 19:11:37.736 | INFO | "pinned" : " 29 GB" | 20260505 19:11:37.736 | INFO | } | 20260505 19:11:37.736 | INFO | Perf report for the last 5.14335 seconds: | 20260505 19:11:37.736 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:11:37.736 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:37.736 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:37.736 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:37.736 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 19:11:37.736 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:37.736 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:37.736 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:11:37.736 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:37.736 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:37.736 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 19:11:37.736 | INFO | Total mem usage: 15.3 GiB | 20260505 19:11:37.736 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 19:11:37.736 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.88%) | 20260505 19:11:37.736 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 19:11:37.736 | INFO | High Water Breakdown: | 20260505 19:11:37.736 | INFO | 60.9% : 8.57 GiB ( 90.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 19:11:37.736 | INFO | 38.9% : 5.48 GiB ( 82.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 19:11:37.736 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 19:11:37.736 | INFO | ------------------------------------------ | 20260505 19:11:37.736 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:11:37.736 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 19:11:37.736 | INFO | Total mem usage: 26.5 GiB | 20260505 19:11:37.736 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 19:11:37.736 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 19:11:37.736 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 19:11:37.736 | INFO | High Water Breakdown: | 20260505 19:11:37.736 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:11:37.736 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:11:37.736 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:11:37.736 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:11:37.736 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:11:37.736 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:11:37.736 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 19:11:37.736 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 19:11:37.736 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:11:37.736 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 19:11:37.736 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 19:11:37.736 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 19:11:37.736 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 19:11:37.736 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 19:11:37.736 | INFO | ------------------------------------------ | 20260505 19:11:37.736 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:11:37.736 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 19:11:37.736 | INFO | Total mem usage: 25.6 GiB | 20260505 19:11:37.736 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 19:11:37.736 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 19:11:37.736 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 19:11:37.736 | INFO | High Water Breakdown: | 20260505 19:11:37.736 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 19:11:37.736 | INFO | ------------------------------------------ | 20260505 19:11:37.736 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:11:37.736 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 19:11:37.736 | INFO | Total mem usage: 15.7 GiB | 20260505 19:11:37.736 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 19:11:37.736 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 19:11:37.736 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 19:11:37.736 | INFO | High Water Breakdown: | 20260505 19:11:37.736 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 19:11:37.736 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 19:11:37.736 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 19:11:37.736 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 19:11:37.736 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 19:11:37.736 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 19:11:37.736 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 19:11:37.736 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 19:11:37.736 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 19:11:37.736 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:11:37.736 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 19:11:37.736 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:11:37.736 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 19:11:37.736 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 19:11:37.736 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 19:11:37.736 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 19:11:37.736 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:11:37.736 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 19:11:37.736 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:11:37.736 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:11:37.736 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:11:37.736 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 19:11:37.736 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 19:11:37.736 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:11:37.736 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 19:11:37.736 | INFO | ------------------------------------------ | 20260505 19:11:37.736 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:11:37.736 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 19:11:37.736 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:11:37.736 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:11:37.736 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.46e+01, 1.35e+01 / 1.98e+00 | 20260505 19:11:37.736 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.02e-01, 1.04e-02 / 3.28e-03 | 20260505 19:11:37.736 | INFO | PerfReport: 11.77%, Baseline -- 3.71e+00 / 4.54e+00, 5.58e-01 / 1.41e-01 | 20260505 19:11:37.736 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.55e+00, 1.90e-01 / 6.72e-02 | 20260505 19:11:37.736 | INFO | PerfReport: 15.93%, DME -- 4.01e+01 / 4.43e+01, 3.92e+00 / 1.98e+00 | 20260505 19:11:37.736 | INFO | PerfReport: 56.96%, FrameLabeling -- 1.79e+01 / 1.94e+01, 1.42e+00 / 1.09e+00 | 20260505 19:11:37.736 | INFO | PerfReport: 7.31%, PulseAccumulating -- 2.30e+00 / 2.90e+00, 4.44e-01 / 1.69e-01 | 20260505 19:11:37.736 | INFO | PerfReport: 3.31%, Metrics -- 1.04e+00 / 1.39e+00, 2.88e-01 / 1.46e-01 | 20260505 19:11:37.736 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.63e-01 / 4.34e-01, 6.38e-02 / 4.27e-02 | 20260505 19:11:37.736 | INFO | Uploaded 36GiB to GPU in 2.19702s (16.3859GiB/s) | 20260505 19:11:37.736 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 19:11:37.737 | INFO | Downloaded 34GiB to GPU in 2.04381s (16.6356GiB/s) | 20260505 19:11:37.738 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:11:37.738 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:11:37.738 | INFO | PerfReport: Full Execution -- 9.97e+01 / -nan, 2.56e+01 / 0.00e+00 | 20260505 19:11:37.738 | INFO | PerfReport: 22.55%, Upload -- 2.25e+01 / 0.00e+00, 7.85e+00 / 0.00e+00 | 20260505 19:11:37.738 | INFO | PerfReport: 21.79%, Download -- 2.17e+01 / 0.00e+00, 7.73e+00 / 0.00e+00 | 20260505 19:11:37.738 | INFO | PerfReport: 31.58%, Compute -- 3.15e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 19:11:37.738 | INFO | PerfReport: 2.40%, LockUpload -- 2.39e+00 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260505 19:11:37.738 | INFO | PerfReport: 1.36%, LockDownload -- 1.36e+00 / 0.00e+00, 2.87e+00 / 0.00e+00 | 20260505 19:11:37.738 | INFO | PerfReport: 20.30%, LockCompute -- 2.03e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260505 19:11:37.738 | INFO | PerfReport: 0.01%, Misc -- 1.47e-02 / 0.00e+00, 2.87e-02 / 0.00e+00 | 20260505 19:11:37.868 | INFO | Starting a round of Baz IO | 20260505 19:11:41.038 | INFO | Populating RTMetrics report | 20260505 19:11:41.038 | INFO | RTMetrics report populated | 20260505 19:11:41.038 | INFO | Opening tmp RTMetrics json file | 20260505 19:11:41.039 | INFO | Serializing RTMetrics to json | 20260505 19:11:41.040 | INFO | Serialization done, swapping tmp file now... | 20260505 19:11:41.041 | INFO | RTMetrics done | 20260505 19:11:41.127 | INFO | Finished a round of Baz IO: Wrote 2.062 GiB in 3.258 seconds | 20260505 19:11:41.127 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.81% / 0.00% / 78.19% | 20260505 19:11:41.127 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.21861% / 3.97364e-06% / 5.96046e-05% | 20260505 19:11:41.127 | INFO | Baz IO overall: | 20260505 19:11:41.127 | INFO | BytesWritten : 3296551926783 | 20260505 19:11:41.127 | INFO | NumEvents : 1560444142907 | 20260505 19:11:42.876 | INFO | Analyzing chunk frames = [5619712,5620224) | 20260505 19:11:42.876 | INFO | MemInfo: { | 20260505 19:11:42.876 | INFO | "gpu" : " 17 GB", | 20260505 19:11:42.876 | INFO | "huge" : "", | 20260505 19:11:42.876 | INFO | "huge_shared" : " 27 GB", | 20260505 19:11:42.876 | INFO | "malloc" : " 14 GB", | 20260505 19:11:42.876 | INFO | "pinned" : " 29 GB" | 20260505 19:11:42.876 | INFO | } | 20260505 19:11:42.876 | INFO | Perf report for the last 5.14025 seconds: | 20260505 19:11:42.876 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:11:42.876 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:42.876 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:42.876 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:42.876 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 19:11:42.876 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:42.876 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260505 19:11:42.876 | INFO | PRE_HQ : 60.0%, 1.8, 55.4ms, 0 | 20260505 19:11:42.876 | INFO | BAZWRITER : 62.9%, 3.8, 122.4ms, 0 | 20260505 19:11:42.876 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 19:11:42.876 | INFO | Uploaded 36GiB to GPU in 2.15737s (16.687GiB/s) | 20260505 19:11:42.876 | INFO | Downloaded 38GiB to GPU in 2.25704s (16.8362GiB/s) | 20260505 19:11:48.009 | INFO | Analyzing chunk frames = [5620224,5620736) | 20260505 19:11:48.009 | INFO | MemInfo: { | 20260505 19:11:48.009 | INFO | "gpu" : " 17 GB", | 20260505 19:11:48.009 | INFO | "huge" : "", | 20260505 19:11:48.009 | INFO | "huge_shared" : " 27 GB", | 20260505 19:11:48.009 | INFO | "malloc" : " 14 GB", | 20260505 19:11:48.009 | INFO | "pinned" : " 29 GB" | 20260505 19:11:48.009 | INFO | } | 20260505 19:11:48.009 | INFO | Perf report for the last 5.13277 seconds: | 20260505 19:11:48.009 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:11:48.009 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:48.009 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:48.009 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:48.009 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 19:11:48.009 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:48.009 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:48.009 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:11:48.009 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:48.009 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:48.009 | INFO | Uploaded 36GiB to GPU in 2.15031s (16.7418GiB/s) | 20260505 19:11:48.009 | INFO | Downloaded 34GiB to GPU in 2.04977s (16.5873GiB/s) | 20260505 19:11:53.149 | INFO | Analyzing chunk frames = [5620736,5621248) | 20260505 19:11:53.149 | INFO | MemInfo: { | 20260505 19:11:53.149 | INFO | "gpu" : " 17 GB", | 20260505 19:11:53.149 | INFO | "huge" : "", | 20260505 19:11:53.149 | INFO | "huge_shared" : " 27 GB", | 20260505 19:11:53.149 | INFO | "malloc" : " 14 GB", | 20260505 19:11:53.149 | INFO | "pinned" : " 29 GB" | 20260505 19:11:53.149 | INFO | } | 20260505 19:11:53.149 | INFO | Perf report for the last 5.1405 seconds: | 20260505 19:11:53.150 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:11:53.150 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:53.150 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:53.150 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:53.150 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 19:11:53.150 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:53.150 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:53.150 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:11:53.150 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:53.150 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:53.150 | INFO | Uploaded 36GiB to GPU in 2.21002s (16.2894GiB/s) | 20260505 19:11:53.150 | INFO | Downloaded 34GiB to GPU in 2.06967s (16.4277GiB/s) | 20260505 19:11:58.292 | INFO | Analyzing chunk frames = [5621248,5621760) | 20260505 19:11:58.292 | INFO | MemInfo: { | 20260505 19:11:58.292 | INFO | "gpu" : " 17 GB", | 20260505 19:11:58.292 | INFO | "huge" : "", | 20260505 19:11:58.292 | INFO | "huge_shared" : " 27 GB", | 20260505 19:11:58.292 | INFO | "malloc" : " 14 GB", | 20260505 19:11:58.292 | INFO | "pinned" : " 29 GB" | 20260505 19:11:58.292 | INFO | } | 20260505 19:11:58.292 | INFO | Perf report for the last 5.14225 seconds: | 20260505 19:11:58.292 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:11:58.292 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:58.292 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:58.292 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:58.292 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260505 19:11:58.292 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:58.292 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:58.292 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:11:58.292 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:58.292 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:11:58.292 | INFO | Uploaded 36GiB to GPU in 2.19476s (16.4027GiB/s) | 20260505 19:11:58.292 | INFO | Downloaded 34GiB to GPU in 2.0602s (16.5033GiB/s) | 20260505 19:12:03.431 | INFO | Analyzing chunk frames = [5621760,5622272) | 20260505 19:12:03.431 | INFO | MemInfo: { | 20260505 19:12:03.431 | INFO | "gpu" : " 17 GB", | 20260505 19:12:03.431 | INFO | "huge" : "", | 20260505 19:12:03.431 | INFO | "huge_shared" : " 27 GB", | 20260505 19:12:03.431 | INFO | "malloc" : " 14 GB", | 20260505 19:12:03.431 | INFO | "pinned" : " 29 GB" | 20260505 19:12:03.431 | INFO | } | 20260505 19:12:03.431 | INFO | Perf report for the last 5.13906 seconds: | 20260505 19:12:03.431 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:12:03.431 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:03.431 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:03.431 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:03.431 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260505 19:12:03.431 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:03.431 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:03.431 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:12:03.431 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:03.431 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:03.431 | INFO | Uploaded 36GiB to GPU in 2.13119s (16.892GiB/s) | 20260505 19:12:03.431 | INFO | Downloaded 34GiB to GPU in 2.06228s (16.4866GiB/s) | 20260505 19:12:08.563 | INFO | Analyzing chunk frames = [5622272,5622784) | 20260505 19:12:08.563 | INFO | MemInfo: { | 20260505 19:12:08.563 | INFO | "gpu" : " 17 GB", | 20260505 19:12:08.563 | INFO | "huge" : "", | 20260505 19:12:08.563 | INFO | "huge_shared" : " 27 GB", | 20260505 19:12:08.563 | INFO | "malloc" : " 14 GB", | 20260505 19:12:08.563 | INFO | "pinned" : " 29 GB" | 20260505 19:12:08.563 | INFO | } | 20260505 19:12:08.563 | INFO | Perf report for the last 5.13241 seconds: | 20260505 19:12:08.563 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:12:08.563 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:08.563 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:08.563 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:08.563 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260505 19:12:08.563 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:08.563 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:08.563 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:12:08.563 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:08.563 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:08.563 | INFO | Uploaded 36GiB to GPU in 2.14084s (16.8158GiB/s) | 20260505 19:12:08.563 | INFO | Downloaded 34GiB to GPU in 2.05859s (16.5161GiB/s) | 20260505 19:12:13.701 | INFO | Analyzing chunk frames = [5622784,5623296) | 20260505 19:12:13.701 | INFO | MemInfo: { | 20260505 19:12:13.701 | INFO | "gpu" : " 17 GB", | 20260505 19:12:13.701 | INFO | "huge" : "", | 20260505 19:12:13.701 | INFO | "huge_shared" : " 27 GB", | 20260505 19:12:13.701 | INFO | "malloc" : " 14 GB", | 20260505 19:12:13.701 | INFO | "pinned" : " 29 GB" | 20260505 19:12:13.701 | INFO | } | 20260505 19:12:13.701 | INFO | Perf report for the last 5.13822 seconds: | 20260505 19:12:13.701 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:12:13.701 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:13.701 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:13.701 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:13.701 | INFO | ANALYSIS : 63.5%, 3.0, 99.1ms, 0 | 20260505 19:12:13.701 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:13.701 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:13.701 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 19:12:13.701 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:13.701 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:13.702 | INFO | Uploaded 36GiB to GPU in 2.15983s (16.668GiB/s) | 20260505 19:12:13.702 | INFO | Downloaded 34GiB to GPU in 2.19395s (15.4972GiB/s) | 20260505 19:12:18.848 | INFO | Analyzing chunk frames = [5623296,5623808) | 20260505 19:12:18.848 | INFO | MemInfo: { | 20260505 19:12:18.848 | INFO | "gpu" : " 17 GB", | 20260505 19:12:18.848 | INFO | "huge" : "", | 20260505 19:12:18.848 | INFO | "huge_shared" : " 27 GB", | 20260505 19:12:18.848 | INFO | "malloc" : " 14 GB", | 20260505 19:12:18.848 | INFO | "pinned" : " 29 GB" | 20260505 19:12:18.848 | INFO | } | 20260505 19:12:18.848 | INFO | Perf report for the last 5.14696 seconds: | 20260505 19:12:18.848 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:12:18.848 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:18.848 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:18.848 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:18.848 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 19:12:18.848 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:18.848 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:18.848 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260505 19:12:18.848 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:18.849 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:18.849 | INFO | Uploaded 36GiB to GPU in 2.19339s (16.4129GiB/s) | 20260505 19:12:18.849 | INFO | Downloaded 34GiB to GPU in 2.0341s (16.715GiB/s) | 20260505 19:12:18.982 | INFO | Starting a round of Baz IO | 20260505 19:12:22.157 | INFO | Populating RTMetrics report | 20260505 19:12:22.157 | INFO | RTMetrics report populated | 20260505 19:12:22.157 | INFO | Opening tmp RTMetrics json file | 20260505 19:12:22.161 | INFO | Serializing RTMetrics to json | 20260505 19:12:22.163 | INFO | Serialization done, swapping tmp file now... | 20260505 19:12:22.163 | INFO | RTMetrics done | 20260505 19:12:22.250 | INFO | Finished a round of Baz IO: Wrote 2.06 GiB in 3.267 seconds | 20260505 19:12:22.250 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.81% / 0.00% / 78.19% | 20260505 19:12:22.250 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.21805% / 0% / 9.93411e-05% | 20260505 19:12:22.250 | INFO | Baz IO overall: | 20260505 19:12:22.250 | INFO | BytesWritten : 3298764302852 | 20260505 19:12:22.250 | INFO | NumEvents : 1561472298388 | 20260505 19:12:23.993 | INFO | Analyzing chunk frames = [5623808,5624320) | 20260505 19:12:23.993 | INFO | MemInfo: { | 20260505 19:12:23.993 | INFO | "gpu" : " 17 GB", | 20260505 19:12:23.993 | INFO | "huge" : "", | 20260505 19:12:23.993 | INFO | "huge_shared" : " 27 GB", | 20260505 19:12:23.993 | INFO | "malloc" : " 14 GB", | 20260505 19:12:23.993 | INFO | "pinned" : " 29 GB" | 20260505 19:12:23.993 | INFO | } | 20260505 19:12:23.993 | INFO | Perf report for the last 5.14473 seconds: | 20260505 19:12:23.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:12:23.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:23.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:23.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:23.993 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 19:12:23.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:23.993 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260505 19:12:23.993 | INFO | PRE_HQ : 60.1%, 1.8, 55.5ms, 0 | 20260505 19:12:23.993 | INFO | BAZWRITER : 63.0%, 3.7, 121.2ms, 0 | 20260505 19:12:23.993 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 19:12:23.993 | INFO | Uploaded 36GiB to GPU in 2.1622s (16.6497GiB/s) | 20260505 19:12:23.993 | INFO | Downloaded 38GiB to GPU in 2.24627s (16.9169GiB/s) | 20260505 19:12:29.129 | INFO | Analyzing chunk frames = [5624320,5624832) | 20260505 19:12:29.129 | INFO | MemInfo: { | 20260505 19:12:29.129 | INFO | "gpu" : " 17 GB", | 20260505 19:12:29.129 | INFO | "huge" : "", | 20260505 19:12:29.129 | INFO | "huge_shared" : " 27 GB", | 20260505 19:12:29.129 | INFO | "malloc" : " 14 GB", | 20260505 19:12:29.129 | INFO | "pinned" : " 29 GB" | 20260505 19:12:29.129 | INFO | } | 20260505 19:12:29.129 | INFO | Perf report for the last 5.13617 seconds: | 20260505 19:12:29.129 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:12:29.129 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:29.129 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:29.129 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:29.129 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260505 19:12:29.129 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:29.129 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:29.129 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260505 19:12:29.129 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:29.129 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:29.130 | INFO | Uploaded 36GiB to GPU in 2.09771s (17.1615GiB/s) | 20260505 19:12:29.130 | INFO | Downloaded 34GiB to GPU in 2.0293s (16.7545GiB/s) | 20260505 19:12:34.260 | INFO | Analyzing chunk frames = [5624832,5625344) | 20260505 19:12:34.260 | INFO | MemInfo: { | 20260505 19:12:34.260 | INFO | "gpu" : " 17 GB", | 20260505 19:12:34.260 | INFO | "huge" : "", | 20260505 19:12:34.260 | INFO | "huge_shared" : " 27 GB", | 20260505 19:12:34.260 | INFO | "malloc" : " 14 GB", | 20260505 19:12:34.260 | INFO | "pinned" : " 29 GB" | 20260505 19:12:34.260 | INFO | } | 20260505 19:12:34.260 | INFO | Perf report for the last 5.13076 seconds: | 20260505 19:12:34.260 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:12:34.260 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:34.260 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:34.260 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:34.260 | INFO | ANALYSIS : 62.9%, 3.0, 97.7ms, 0 | 20260505 19:12:34.260 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:34.260 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:34.260 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:12:34.260 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:34.260 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:34.260 | INFO | Uploaded 36GiB to GPU in 2.21134s (16.2798GiB/s) | 20260505 19:12:34.260 | INFO | Downloaded 34GiB to GPU in 2.07331s (16.3989GiB/s) | 20260505 19:12:39.404 | INFO | Analyzing chunk frames = [5625344,5625856) | 20260505 19:12:39.404 | INFO | MemInfo: { | 20260505 19:12:39.404 | INFO | "gpu" : " 17 GB", | 20260505 19:12:39.404 | INFO | "huge" : "", | 20260505 19:12:39.404 | INFO | "huge_shared" : " 27 GB", | 20260505 19:12:39.404 | INFO | "malloc" : " 14 GB", | 20260505 19:12:39.405 | INFO | "pinned" : " 29 GB" | 20260505 19:12:39.405 | INFO | } | 20260505 19:12:39.405 | INFO | Perf report for the last 5.14453 seconds: | 20260505 19:12:39.405 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:12:39.405 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:39.405 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:39.405 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:39.405 | INFO | ANALYSIS : 62.6%, 3.0, 97.4ms, 0 | 20260505 19:12:39.405 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:39.405 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:39.405 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:12:39.405 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:39.405 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:39.405 | INFO | Uploaded 36GiB to GPU in 2.221s (16.2089GiB/s) | 20260505 19:12:39.405 | INFO | Downloaded 34GiB to GPU in 2.12169s (16.0249GiB/s) | 20260505 19:12:44.543 | INFO | Analyzing chunk frames = [5625856,5626368) | 20260505 19:12:44.543 | INFO | MemInfo: { | 20260505 19:12:44.543 | INFO | "gpu" : " 17 GB", | 20260505 19:12:44.543 | INFO | "huge" : "", | 20260505 19:12:44.543 | INFO | "huge_shared" : " 27 GB", | 20260505 19:12:44.543 | INFO | "malloc" : " 14 GB", | 20260505 19:12:44.543 | INFO | "pinned" : " 29 GB" | 20260505 19:12:44.543 | INFO | } | 20260505 19:12:44.543 | INFO | Perf report for the last 5.13815 seconds: | 20260505 19:12:44.543 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:12:44.543 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:44.543 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:44.543 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:44.543 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260505 19:12:44.543 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:44.543 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:44.543 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:12:44.543 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:44.543 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:44.543 | INFO | Uploaded 36GiB to GPU in 2.13886s (16.8314GiB/s) | 20260505 19:12:44.543 | INFO | Downloaded 34GiB to GPU in 2.06341s (16.4776GiB/s) | 20260505 19:12:49.686 | INFO | Analyzing chunk frames = [5626368,5626880) | 20260505 19:12:49.686 | INFO | MemInfo: { | 20260505 19:12:49.686 | INFO | "gpu" : " 17 GB", | 20260505 19:12:49.686 | INFO | "huge" : "", | 20260505 19:12:49.686 | INFO | "huge_shared" : " 27 GB", | 20260505 19:12:49.686 | INFO | "malloc" : " 14 GB", | 20260505 19:12:49.686 | INFO | "pinned" : " 29 GB" | 20260505 19:12:49.686 | INFO | } | 20260505 19:12:49.686 | INFO | Perf report for the last 5.14355 seconds: | 20260505 19:12:49.686 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:12:49.686 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:49.686 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:49.686 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:49.686 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 19:12:49.686 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:49.686 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:49.686 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:12:49.686 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:49.686 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:49.687 | INFO | Uploaded 36GiB to GPU in 2.16611s (16.6196GiB/s) | 20260505 19:12:49.687 | INFO | Downloaded 34GiB to GPU in 2.06637s (16.454GiB/s) | 20260505 19:12:54.821 | INFO | Analyzing chunk frames = [5626880,5627392) | 20260505 19:12:54.821 | INFO | MemInfo: { | 20260505 19:12:54.821 | INFO | "gpu" : " 17 GB", | 20260505 19:12:54.821 | INFO | "huge" : "", | 20260505 19:12:54.821 | INFO | "huge_shared" : " 27 GB", | 20260505 19:12:54.821 | INFO | "malloc" : " 14 GB", | 20260505 19:12:54.821 | INFO | "pinned" : " 29 GB" | 20260505 19:12:54.821 | INFO | } | 20260505 19:12:54.821 | INFO | Perf report for the last 5.13447 seconds: | 20260505 19:12:54.821 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:12:54.821 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:54.821 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:54.821 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:54.821 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 19:12:54.821 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:54.821 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:54.821 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 19:12:54.821 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:54.821 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:54.821 | INFO | Uploaded 36GiB to GPU in 2.12101s (16.973GiB/s) | 20260505 19:12:54.821 | INFO | Downloaded 34GiB to GPU in 2.1489s (15.8221GiB/s) | 20260505 19:12:59.955 | INFO | Analyzing chunk frames = [5627392,5627904) | 20260505 19:12:59.955 | INFO | MemInfo: { | 20260505 19:12:59.955 | INFO | "gpu" : " 17 GB", | 20260505 19:12:59.955 | INFO | "huge" : "", | 20260505 19:12:59.955 | INFO | "huge_shared" : " 27 GB", | 20260505 19:12:59.955 | INFO | "malloc" : " 14 GB", | 20260505 19:12:59.955 | INFO | "pinned" : " 29 GB" | 20260505 19:12:59.955 | INFO | } | 20260505 19:12:59.955 | INFO | Perf report for the last 5.13464 seconds: | 20260505 19:12:59.955 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:12:59.955 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:59.956 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:59.956 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:59.956 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 19:12:59.956 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:59.956 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:59.956 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 19:12:59.956 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:59.956 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:12:59.956 | INFO | Uploaded 36GiB to GPU in 2.19784s (16.3797GiB/s) | 20260505 19:12:59.956 | INFO | Downloaded 34GiB to GPU in 2.03757s (16.6866GiB/s) | 20260505 19:12:59.956 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 19:12:59.956 | INFO | Total mem usage: 15.3 GiB | 20260505 19:12:59.956 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 19:12:59.956 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.34%) | 20260505 19:12:59.956 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 19:12:59.956 | INFO | High Water Breakdown: | 20260505 19:12:59.956 | INFO | 60.9% : 8.62 GiB ( 90.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 19:12:59.956 | INFO | 38.9% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 19:12:59.956 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 19:12:59.956 | INFO | ------------------------------------------ | 20260505 19:12:59.956 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:12:59.956 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 19:12:59.956 | INFO | Total mem usage: 26.5 GiB | 20260505 19:12:59.956 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 19:12:59.956 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 19:12:59.956 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260505 19:12:59.956 | INFO | High Water Breakdown: | 20260505 19:12:59.956 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:12:59.956 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:12:59.956 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:12:59.956 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:12:59.956 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:12:59.956 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:12:59.956 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 19:12:59.956 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 19:12:59.956 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:12:59.956 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 19:12:59.956 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 19:12:59.956 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 19:12:59.956 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 19:12:59.956 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 19:12:59.956 | INFO | ------------------------------------------ | 20260505 19:12:59.956 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:12:59.956 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 19:12:59.956 | INFO | Total mem usage: 25.6 GiB | 20260505 19:12:59.957 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 19:12:59.957 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 19:12:59.957 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 19:12:59.957 | INFO | High Water Breakdown: | 20260505 19:12:59.957 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 19:12:59.957 | INFO | ------------------------------------------ | 20260505 19:12:59.957 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:12:59.957 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 19:12:59.957 | INFO | Total mem usage: 15.7 GiB | 20260505 19:12:59.957 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 19:12:59.957 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 19:12:59.957 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 19:12:59.957 | INFO | High Water Breakdown: | 20260505 19:12:59.957 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 19:12:59.957 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 19:12:59.957 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 19:12:59.957 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 19:12:59.957 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 19:12:59.957 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 19:12:59.957 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 19:12:59.957 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 19:12:59.957 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 19:12:59.957 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:12:59.957 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 19:12:59.957 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:12:59.957 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 19:12:59.957 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 19:12:59.957 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 19:12:59.957 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 19:12:59.957 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:12:59.957 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 19:12:59.957 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:12:59.957 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:12:59.957 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:12:59.957 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 19:12:59.957 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 19:12:59.957 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:12:59.957 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 19:12:59.957 | INFO | ------------------------------------------ | 20260505 19:12:59.957 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:12:59.957 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 19:12:59.957 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:12:59.957 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:12:59.957 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.45e+01, 1.35e+01 / 2.03e+00 | 20260505 19:12:59.957 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.03e-01, 1.03e-02 / 3.65e-03 | 20260505 19:12:59.957 | INFO | PerfReport: 11.75%, Baseline -- 3.70e+00 / 4.56e+00, 5.46e-01 / 8.66e-02 | 20260505 19:12:59.957 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.53e+00, 1.87e-01 / 8.17e-02 | 20260505 19:12:59.957 | INFO | PerfReport: 15.97%, DME -- 4.02e+01 / 4.44e+01, 4.09e+00 / 1.97e+00 | 20260505 19:12:59.957 | INFO | PerfReport: 56.95%, FrameLabeling -- 1.79e+01 / 1.92e+01, 1.39e+00 / 7.96e-01 | 20260505 19:12:59.957 | INFO | PerfReport: 7.32%, PulseAccumulating -- 2.30e+00 / 2.90e+00, 4.40e-01 / 1.53e-01 | 20260505 19:12:59.957 | INFO | PerfReport: 3.30%, Metrics -- 1.04e+00 / 1.38e+00, 2.84e-01 / 1.45e-01 | 20260505 19:12:59.957 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.62e-01 / 4.32e-01, 6.33e-02 / 3.45e-02 | 20260505 19:12:59.957 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 19:12:59.957 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:12:59.957 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:12:59.957 | INFO | PerfReport: Full Execution -- 9.97e+01 / -nan, 2.56e+01 / 0.00e+00 | 20260505 19:12:59.957 | INFO | PerfReport: 22.62%, Upload -- 2.26e+01 / 0.00e+00, 7.86e+00 / 0.00e+00 | 20260505 19:12:59.957 | INFO | PerfReport: 21.92%, Download -- 2.19e+01 / 0.00e+00, 7.78e+00 / 0.00e+00 | 20260505 19:12:59.957 | INFO | PerfReport: 31.54%, Compute -- 3.15e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 19:12:59.957 | INFO | PerfReport: 2.37%, LockUpload -- 2.37e+00 / 0.00e+00, 7.92e+00 / 0.00e+00 | 20260505 19:12:59.957 | INFO | PerfReport: 1.34%, LockDownload -- 1.34e+00 / 0.00e+00, 2.80e+00 / 0.00e+00 | 20260505 19:12:59.957 | INFO | PerfReport: 20.19%, LockCompute -- 2.01e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 19:12:59.957 | INFO | PerfReport: 0.02%, Misc -- 1.68e-02 / 0.00e+00, 6.69e-02 / 0.00e+00 | 20260505 19:13:00.090 | INFO | Starting a round of Baz IO | 20260505 19:13:03.265 | INFO | Populating RTMetrics report | 20260505 19:13:03.265 | INFO | RTMetrics report populated | 20260505 19:13:03.265 | INFO | Opening tmp RTMetrics json file | 20260505 19:13:03.273 | INFO | Serializing RTMetrics to json | 20260505 19:13:03.276 | INFO | Serialization done, swapping tmp file now... | 20260505 19:13:03.277 | INFO | RTMetrics done | 20260505 19:13:03.360 | INFO | Finished a round of Baz IO: Wrote 2.059 GiB in 3.269 seconds | 20260505 19:13:03.360 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.81% / 0.00% / 78.19% | 20260505 19:13:03.360 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.217255% / 0% / 6.75519e-05% | 20260505 19:13:03.360 | INFO | Baz IO overall: | 20260505 19:13:03.360 | INFO | BytesWritten : 3300975192273 | 20260505 19:13:03.360 | INFO | NumEvents : 1562499563972 | 20260505 19:13:05.109 | INFO | Analyzing chunk frames = [5627904,5628416) | 20260505 19:13:05.109 | INFO | MemInfo: { | 20260505 19:13:05.109 | INFO | "gpu" : " 17 GB", | 20260505 19:13:05.109 | INFO | "huge" : "", | 20260505 19:13:05.109 | INFO | "huge_shared" : " 27 GB", | 20260505 19:13:05.109 | INFO | "malloc" : " 14 GB", | 20260505 19:13:05.109 | INFO | "pinned" : " 29 GB" | 20260505 19:13:05.109 | INFO | } | 20260505 19:13:05.109 | INFO | Perf report for the last 5.1533 seconds: | 20260505 19:13:05.109 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:13:05.109 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:05.109 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:05.109 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:05.109 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 19:13:05.109 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:05.109 | INFO | RT_METRICS : 44.3%, 1.0, 23.3ms, 0 | 20260505 19:13:05.109 | INFO | PRE_HQ : 60.2%, 1.8, 55.6ms, 0 | 20260505 19:13:05.109 | INFO | BAZWRITER : 62.9%, 3.8, 123.3ms, 0 | 20260505 19:13:05.109 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 19:13:05.109 | INFO | Uploaded 36GiB to GPU in 2.16691s (16.6135GiB/s) | 20260505 19:13:05.109 | INFO | Downloaded 38GiB to GPU in 2.25091s (16.8821GiB/s) | 20260505 19:13:10.243 | INFO | Analyzing chunk frames = [5628416,5628928) | 20260505 19:13:10.243 | INFO | MemInfo: { | 20260505 19:13:10.243 | INFO | "gpu" : " 17 GB", | 20260505 19:13:10.244 | INFO | "huge" : "", | 20260505 19:13:10.244 | INFO | "huge_shared" : " 27 GB", | 20260505 19:13:10.244 | INFO | "malloc" : " 14 GB", | 20260505 19:13:10.244 | INFO | "pinned" : " 29 GB" | 20260505 19:13:10.244 | INFO | } | 20260505 19:13:10.244 | INFO | Perf report for the last 5.13482 seconds: | 20260505 19:13:10.244 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:13:10.244 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:10.244 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:10.244 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:10.244 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260505 19:13:10.244 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:10.244 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:10.244 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:13:10.244 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:10.244 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:10.244 | INFO | Uploaded 36GiB to GPU in 2.09825s (17.1571GiB/s) | 20260505 19:13:10.244 | INFO | Downloaded 34GiB to GPU in 2.02384s (16.7998GiB/s) | 20260505 19:13:15.377 | INFO | Analyzing chunk frames = [5628928,5629440) | 20260505 19:13:15.377 | INFO | MemInfo: { | 20260505 19:13:15.377 | INFO | "gpu" : " 17 GB", | 20260505 19:13:15.377 | INFO | "huge" : "", | 20260505 19:13:15.377 | INFO | "huge_shared" : " 27 GB", | 20260505 19:13:15.377 | INFO | "malloc" : " 14 GB", | 20260505 19:13:15.377 | INFO | "pinned" : " 29 GB" | 20260505 19:13:15.377 | INFO | } | 20260505 19:13:15.377 | INFO | Perf report for the last 5.13384 seconds: | 20260505 19:13:15.377 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:13:15.377 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:15.377 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:15.377 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:15.377 | INFO | ANALYSIS : 62.4%, 3.0, 97.0ms, 0 | 20260505 19:13:15.378 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:15.378 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:15.378 | INFO | PRE_HQ : 27.5%, 1.0, 14.2ms, 0 | 20260505 19:13:15.378 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:15.378 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:15.378 | INFO | Uploaded 36GiB to GPU in 2.13497s (16.862GiB/s) | 20260505 19:13:15.378 | INFO | Downloaded 34GiB to GPU in 2.04036s (16.6637GiB/s) | 20260505 19:13:20.510 | INFO | Analyzing chunk frames = [5629440,5629952) | 20260505 19:13:20.510 | INFO | MemInfo: { | 20260505 19:13:20.510 | INFO | "gpu" : " 17 GB", | 20260505 19:13:20.510 | INFO | "huge" : "", | 20260505 19:13:20.510 | INFO | "huge_shared" : " 27 GB", | 20260505 19:13:20.510 | INFO | "malloc" : " 14 GB", | 20260505 19:13:20.510 | INFO | "pinned" : " 29 GB" | 20260505 19:13:20.510 | INFO | } | 20260505 19:13:20.510 | INFO | Perf report for the last 5.1329 seconds: | 20260505 19:13:20.510 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:13:20.510 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:20.510 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:20.510 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:20.510 | INFO | ANALYSIS : 62.3%, 3.0, 96.7ms, 0 | 20260505 19:13:20.510 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:20.510 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:20.510 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 19:13:20.510 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:20.510 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:20.511 | INFO | Uploaded 36GiB to GPU in 2.17539s (16.5488GiB/s) | 20260505 19:13:20.511 | INFO | Downloaded 34GiB to GPU in 2.05739s (16.5258GiB/s) | 20260505 19:13:25.657 | INFO | Analyzing chunk frames = [5629952,5630464) | 20260505 19:13:25.657 | INFO | MemInfo: { | 20260505 19:13:25.657 | INFO | "gpu" : " 17 GB", | 20260505 19:13:25.657 | INFO | "huge" : "", | 20260505 19:13:25.657 | INFO | "huge_shared" : " 27 GB", | 20260505 19:13:25.657 | INFO | "malloc" : " 14 GB", | 20260505 19:13:25.657 | INFO | "pinned" : " 29 GB" | 20260505 19:13:25.657 | INFO | } | 20260505 19:13:25.657 | INFO | Perf report for the last 5.14634 seconds: | 20260505 19:13:25.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:13:25.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:25.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:25.657 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:25.657 | INFO | ANALYSIS : 62.0%, 3.0, 97.0ms, 0 | 20260505 19:13:25.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:25.657 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:25.657 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 19:13:25.657 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:25.657 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:25.657 | INFO | Uploaded 36GiB to GPU in 2.14744s (16.7642GiB/s) | 20260505 19:13:25.657 | INFO | Downloaded 34GiB to GPU in 2.06235s (16.4861GiB/s) | 20260505 19:13:30.800 | INFO | Analyzing chunk frames = [5630464,5630976) | 20260505 19:13:30.800 | INFO | MemInfo: { | 20260505 19:13:30.800 | INFO | "gpu" : " 17 GB", | 20260505 19:13:30.800 | INFO | "huge" : "", | 20260505 19:13:30.800 | INFO | "huge_shared" : " 27 GB", | 20260505 19:13:30.800 | INFO | "malloc" : " 14 GB", | 20260505 19:13:30.800 | INFO | "pinned" : " 29 GB" | 20260505 19:13:30.800 | INFO | } | 20260505 19:13:30.800 | INFO | Perf report for the last 5.1436 seconds: | 20260505 19:13:30.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:13:30.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:30.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:30.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:30.800 | INFO | ANALYSIS : 62.1%, 3.0, 97.1ms, 0 | 20260505 19:13:30.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:30.800 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:30.801 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:13:30.801 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:30.801 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:30.801 | INFO | Uploaded 36GiB to GPU in 2.14347s (16.7952GiB/s) | 20260505 19:13:30.801 | INFO | Downloaded 34GiB to GPU in 2.05588s (16.5379GiB/s) | 20260505 19:13:35.939 | INFO | Analyzing chunk frames = [5630976,5631488) | 20260505 19:13:35.939 | INFO | MemInfo: { | 20260505 19:13:35.939 | INFO | "gpu" : " 17 GB", | 20260505 19:13:35.939 | INFO | "huge" : "", | 20260505 19:13:35.939 | INFO | "huge_shared" : " 27 GB", | 20260505 19:13:35.939 | INFO | "malloc" : " 14 GB", | 20260505 19:13:35.939 | INFO | "pinned" : " 29 GB" | 20260505 19:13:35.939 | INFO | } | 20260505 19:13:35.939 | INFO | Perf report for the last 5.13861 seconds: | 20260505 19:13:35.939 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:13:35.939 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:35.939 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:35.939 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:35.939 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 19:13:35.939 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:35.939 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:35.939 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 19:13:35.939 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:35.939 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:35.939 | INFO | Uploaded 36GiB to GPU in 2.11873s (16.9913GiB/s) | 20260505 19:13:35.939 | INFO | Downloaded 34GiB to GPU in 2.12273s (16.0171GiB/s) | 20260505 19:13:41.073 | INFO | Analyzing chunk frames = [5631488,5632000) | 20260505 19:13:41.073 | INFO | MemInfo: { | 20260505 19:13:41.073 | INFO | "gpu" : " 17 GB", | 20260505 19:13:41.073 | INFO | "huge" : "", | 20260505 19:13:41.073 | INFO | "huge_shared" : " 27 GB", | 20260505 19:13:41.073 | INFO | "malloc" : " 14 GB", | 20260505 19:13:41.073 | INFO | "pinned" : " 29 GB" | 20260505 19:13:41.073 | INFO | } | 20260505 19:13:41.073 | INFO | Perf report for the last 5.13423 seconds: | 20260505 19:13:41.073 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:13:41.073 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:41.073 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:41.073 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:41.073 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 19:13:41.073 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:41.073 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:41.073 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260505 19:13:41.073 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:41.073 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:41.073 | INFO | Uploaded 36GiB to GPU in 2.1953s (16.3986GiB/s) | 20260505 19:13:41.073 | INFO | Downloaded 34GiB to GPU in 2.0313s (16.7381GiB/s) | 20260505 19:13:41.210 | INFO | Starting a round of Baz IO | 20260505 19:13:44.393 | INFO | Populating RTMetrics report | 20260505 19:13:44.394 | INFO | RTMetrics report populated | 20260505 19:13:44.394 | INFO | Opening tmp RTMetrics json file | 20260505 19:13:44.396 | INFO | Serializing RTMetrics to json | 20260505 19:13:44.398 | INFO | Serialization done, swapping tmp file now... | 20260505 19:13:44.398 | INFO | RTMetrics done | 20260505 19:13:44.486 | INFO | Finished a round of Baz IO: Wrote 2.059 GiB in 3.275 seconds | 20260505 19:13:44.486 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.80% / 0.00% / 78.19% | 20260505 19:13:44.486 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.216552% / 3.97364e-06% / 9.93411e-05% | 20260505 19:13:44.486 | INFO | Baz IO overall: | 20260505 19:13:44.486 | INFO | BytesWritten : 3303186397935 | 20260505 19:13:44.486 | INFO | NumEvents : 1563527060559 | 20260505 19:13:46.213 | INFO | Analyzing chunk frames = [5632000,5632512) | 20260505 19:13:46.213 | INFO | MemInfo: { | 20260505 19:13:46.213 | INFO | "gpu" : " 17 GB", | 20260505 19:13:46.213 | INFO | "huge" : "", | 20260505 19:13:46.213 | INFO | "huge_shared" : " 27 GB", | 20260505 19:13:46.213 | INFO | "malloc" : " 14 GB", | 20260505 19:13:46.213 | INFO | "pinned" : " 29 GB" | 20260505 19:13:46.213 | INFO | } | 20260505 19:13:46.213 | INFO | Perf report for the last 5.13979 seconds: | 20260505 19:13:46.213 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:13:46.213 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:46.213 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:46.213 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:46.213 | INFO | ANALYSIS : 63.5%, 3.0, 99.0ms, 0 | 20260505 19:13:46.213 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:46.213 | INFO | RT_METRICS : 44.3%, 1.0, 23.3ms, 0 | 20260505 19:13:46.213 | INFO | PRE_HQ : 60.3%, 1.8, 55.7ms, 0 | 20260505 19:13:46.213 | INFO | BAZWRITER : 63.0%, 3.8, 121.9ms, 0 | 20260505 19:13:46.213 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 19:13:46.213 | INFO | Uploaded 36GiB to GPU in 2.18187s (16.4996GiB/s) | 20260505 19:13:46.213 | INFO | Downloaded 38GiB to GPU in 2.25029s (16.8867GiB/s) | 20260505 19:13:51.357 | INFO | Analyzing chunk frames = [5632512,5633024) | 20260505 19:13:51.357 | INFO | MemInfo: { | 20260505 19:13:51.357 | INFO | "gpu" : " 17 GB", | 20260505 19:13:51.357 | INFO | "huge" : "", | 20260505 19:13:51.357 | INFO | "huge_shared" : " 27 GB", | 20260505 19:13:51.358 | INFO | "malloc" : " 14 GB", | 20260505 19:13:51.358 | INFO | "pinned" : " 29 GB" | 20260505 19:13:51.358 | INFO | } | 20260505 19:13:51.358 | INFO | Perf report for the last 5.14456 seconds: | 20260505 19:13:51.358 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:13:51.358 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:51.358 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:51.358 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:51.358 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260505 19:13:51.358 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:51.358 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:51.358 | INFO | PRE_HQ : 27.2%, 1.0, 14.2ms, 0 | 20260505 19:13:51.358 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:51.358 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:51.358 | INFO | Uploaded 36GiB to GPU in 2.14051s (16.8184GiB/s) | 20260505 19:13:51.358 | INFO | Downloaded 34GiB to GPU in 2.0373s (16.6888GiB/s) | 20260505 19:13:56.498 | INFO | Analyzing chunk frames = [5633024,5633536) | 20260505 19:13:56.498 | INFO | MemInfo: { | 20260505 19:13:56.498 | INFO | "gpu" : " 17 GB", | 20260505 19:13:56.498 | INFO | "huge" : "", | 20260505 19:13:56.498 | INFO | "huge_shared" : " 27 GB", | 20260505 19:13:56.498 | INFO | "malloc" : " 14 GB", | 20260505 19:13:56.498 | INFO | "pinned" : " 29 GB" | 20260505 19:13:56.498 | INFO | } | 20260505 19:13:56.498 | INFO | Perf report for the last 5.1403 seconds: | 20260505 19:13:56.498 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:13:56.498 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:56.498 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:56.498 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:56.498 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260505 19:13:56.498 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:56.498 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:56.498 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:13:56.498 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:56.498 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:13:56.498 | INFO | Uploaded 36GiB to GPU in 2.13341s (16.8744GiB/s) | 20260505 19:13:56.498 | INFO | Downloaded 34GiB to GPU in 2.02406s (16.7979GiB/s) | 20260505 19:14:01.636 | INFO | Analyzing chunk frames = [5633536,5634048) | 20260505 19:14:01.636 | INFO | MemInfo: { | 20260505 19:14:01.636 | INFO | "gpu" : " 17 GB", | 20260505 19:14:01.636 | INFO | "huge" : "", | 20260505 19:14:01.636 | INFO | "huge_shared" : " 27 GB", | 20260505 19:14:01.636 | INFO | "malloc" : " 14 GB", | 20260505 19:14:01.636 | INFO | "pinned" : " 29 GB" | 20260505 19:14:01.636 | INFO | } | 20260505 19:14:01.636 | INFO | Perf report for the last 5.13828 seconds: | 20260505 19:14:01.636 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:14:01.636 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:01.636 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:01.636 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:01.636 | INFO | ANALYSIS : 62.4%, 3.0, 96.9ms, 0 | 20260505 19:14:01.636 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:01.636 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:01.636 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 19:14:01.636 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:01.636 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:01.637 | INFO | Uploaded 36GiB to GPU in 2.19834s (16.376GiB/s) | 20260505 19:14:01.637 | INFO | Downloaded 34GiB to GPU in 2.04042s (16.6632GiB/s) | 20260505 19:14:06.773 | INFO | Analyzing chunk frames = [5634048,5634560) | 20260505 19:14:06.773 | INFO | MemInfo: { | 20260505 19:14:06.773 | INFO | "gpu" : " 17 GB", | 20260505 19:14:06.773 | INFO | "huge" : "", | 20260505 19:14:06.773 | INFO | "huge_shared" : " 27 GB", | 20260505 19:14:06.773 | INFO | "malloc" : " 14 GB", | 20260505 19:14:06.773 | INFO | "pinned" : " 29 GB" | 20260505 19:14:06.773 | INFO | } | 20260505 19:14:06.773 | INFO | Perf report for the last 5.13687 seconds: | 20260505 19:14:06.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:14:06.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:06.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:06.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:06.773 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 19:14:06.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:06.773 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:06.773 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260505 19:14:06.773 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:06.773 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:06.773 | INFO | Uploaded 36GiB to GPU in 2.14199s (16.8068GiB/s) | 20260505 19:14:06.774 | INFO | Downloaded 34GiB to GPU in 2.06013s (16.5038GiB/s) | 20260505 19:14:11.917 | INFO | Analyzing chunk frames = [5634560,5635072) | 20260505 19:14:11.917 | INFO | MemInfo: { | 20260505 19:14:11.917 | INFO | "gpu" : " 17 GB", | 20260505 19:14:11.917 | INFO | "huge" : "", | 20260505 19:14:11.917 | INFO | "huge_shared" : " 27 GB", | 20260505 19:14:11.917 | INFO | "malloc" : " 14 GB", | 20260505 19:14:11.917 | INFO | "pinned" : " 29 GB" | 20260505 19:14:11.917 | INFO | } | 20260505 19:14:11.918 | INFO | Perf report for the last 5.1445 seconds: | 20260505 19:14:11.918 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:14:11.918 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:11.918 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:11.918 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:11.918 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260505 19:14:11.918 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:11.918 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:11.918 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:14:11.918 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:11.918 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:11.918 | INFO | Uploaded 36GiB to GPU in 2.13701s (16.846GiB/s) | 20260505 19:14:11.918 | INFO | Downloaded 34GiB to GPU in 2.04815s (16.6003GiB/s) | 20260505 19:14:17.055 | INFO | Analyzing chunk frames = [5635072,5635584) | 20260505 19:14:17.055 | INFO | MemInfo: { | 20260505 19:14:17.055 | INFO | "gpu" : " 17 GB", | 20260505 19:14:17.055 | INFO | "huge" : "", | 20260505 19:14:17.055 | INFO | "huge_shared" : " 27 GB", | 20260505 19:14:17.055 | INFO | "malloc" : " 14 GB", | 20260505 19:14:17.055 | INFO | "pinned" : " 29 GB" | 20260505 19:14:17.055 | INFO | } | 20260505 19:14:17.055 | INFO | Perf report for the last 5.13725 seconds: | 20260505 19:14:17.055 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:14:17.055 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:17.055 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:17.055 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:17.055 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 19:14:17.055 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:17.055 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:17.055 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 19:14:17.055 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:17.055 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:17.055 | INFO | Uploaded 36GiB to GPU in 2.12681s (16.9267GiB/s) | 20260505 19:14:17.055 | INFO | Downloaded 34GiB to GPU in 2.12291s (16.0157GiB/s) | 20260505 19:14:22.191 | INFO | Analyzing chunk frames = [5635584,5636096) | 20260505 19:14:22.191 | INFO | MemInfo: { | 20260505 19:14:22.191 | INFO | "gpu" : " 17 GB", | 20260505 19:14:22.191 | INFO | "huge" : "", | 20260505 19:14:22.191 | INFO | "huge_shared" : " 27 GB", | 20260505 19:14:22.191 | INFO | "malloc" : " 14 GB", | 20260505 19:14:22.191 | INFO | "pinned" : " 29 GB" | 20260505 19:14:22.191 | INFO | } | 20260505 19:14:22.191 | INFO | Perf report for the last 5.13646 seconds: | 20260505 19:14:22.191 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:14:22.191 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:22.191 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:22.191 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:22.191 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 19:14:22.191 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:22.191 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:22.191 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 19:14:22.191 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:22.191 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:22.192 | INFO | Uploaded 36GiB to GPU in 2.20288s (16.3422GiB/s) | 20260505 19:14:22.192 | INFO | Downloaded 34GiB to GPU in 2.03786s (16.6842GiB/s) | 20260505 19:14:22.192 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 19:14:22.192 | INFO | Total mem usage: 15.3 GiB | 20260505 19:14:22.192 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 19:14:22.192 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.05%) | 20260505 19:14:22.192 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 19:14:22.192 | INFO | High Water Breakdown: | 20260505 19:14:22.192 | INFO | 61% : 8.66 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 19:14:22.192 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 19:14:22.192 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 19:14:22.192 | INFO | ------------------------------------------ | 20260505 19:14:22.192 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:14:22.192 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 19:14:22.192 | INFO | Total mem usage: 26.5 GiB | 20260505 19:14:22.192 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 19:14:22.192 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 19:14:22.192 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 19:14:22.192 | INFO | High Water Breakdown: | 20260505 19:14:22.192 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:14:22.192 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:14:22.192 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:14:22.192 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:14:22.192 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:14:22.192 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:14:22.192 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 19:14:22.192 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 19:14:22.192 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:14:22.192 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 19:14:22.192 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 19:14:22.192 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 19:14:22.192 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 19:14:22.192 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 19:14:22.192 | INFO | ------------------------------------------ | 20260505 19:14:22.192 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:14:22.192 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 19:14:22.192 | INFO | Total mem usage: 25.6 GiB | 20260505 19:14:22.192 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 19:14:22.192 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 19:14:22.192 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 19:14:22.192 | INFO | High Water Breakdown: | 20260505 19:14:22.192 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 19:14:22.192 | INFO | ------------------------------------------ | 20260505 19:14:22.192 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:14:22.192 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 19:14:22.192 | INFO | Total mem usage: 15.7 GiB | 20260505 19:14:22.192 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 19:14:22.192 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 19:14:22.192 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 19:14:22.192 | INFO | High Water Breakdown: | 20260505 19:14:22.192 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 19:14:22.192 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 19:14:22.192 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 19:14:22.192 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 19:14:22.192 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 19:14:22.192 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 19:14:22.192 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 19:14:22.192 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 19:14:22.192 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 19:14:22.192 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:14:22.192 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 19:14:22.192 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:14:22.192 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 19:14:22.192 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 19:14:22.192 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 19:14:22.192 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 19:14:22.192 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:14:22.192 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 19:14:22.192 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:14:22.192 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:14:22.192 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:14:22.192 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 19:14:22.192 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 19:14:22.192 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:14:22.192 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 19:14:22.192 | INFO | ------------------------------------------ | 20260505 19:14:22.192 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:14:22.192 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 19:14:22.192 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:14:22.192 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:14:22.192 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.43e+01, 1.34e+01 / 1.46e+00 | 20260505 19:14:22.192 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.02e-01, 1.03e-02 / 3.95e-03 | 20260505 19:14:22.192 | INFO | PerfReport: 11.81%, Baseline -- 3.72e+00 / 4.56e+00, 5.72e-01 / 1.18e-01 | 20260505 19:14:22.192 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.50e+00, 1.88e-01 / 9.69e-02 | 20260505 19:14:22.192 | INFO | PerfReport: 15.91%, DME -- 4.01e+01 / 4.42e+01, 3.99e+00 / 1.52e+00 | 20260505 19:14:22.192 | INFO | PerfReport: 57.02%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.54e+00 / 1.23e+00 | 20260505 19:14:22.192 | INFO | PerfReport: 7.26%, PulseAccumulating -- 2.29e+00 / 2.85e+00, 4.39e-01 / 2.10e-01 | 20260505 19:14:22.192 | INFO | PerfReport: 3.28%, Metrics -- 1.04e+00 / 1.35e+00, 2.82e-01 / 1.42e-01 | 20260505 19:14:22.192 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.61e-01 / 4.22e-01, 6.40e-02 / 3.18e-02 | 20260505 19:14:22.192 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 19:14:22.192 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:14:22.192 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:14:22.193 | INFO | PerfReport: Full Execution -- 9.96e+01 / -nan, 2.57e+01 / 0.00e+00 | 20260505 19:14:22.193 | INFO | PerfReport: 22.52%, Upload -- 2.24e+01 / 0.00e+00, 7.76e+00 / 0.00e+00 | 20260505 19:14:22.193 | INFO | PerfReport: 21.75%, Download -- 2.17e+01 / 0.00e+00, 7.71e+00 / 0.00e+00 | 20260505 19:14:22.193 | INFO | PerfReport: 31.68%, Compute -- 3.16e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 19:14:22.193 | INFO | PerfReport: 2.32%, LockUpload -- 2.31e+00 / 0.00e+00, 8.04e+00 / 0.00e+00 | 20260505 19:14:22.193 | INFO | PerfReport: 1.37%, LockDownload -- 1.36e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260505 19:14:22.193 | INFO | PerfReport: 20.35%, LockCompute -- 2.03e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 19:14:22.193 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.99e-02 / 0.00e+00 | 20260505 19:14:22.324 | INFO | Starting a round of Baz IO | 20260505 19:14:25.494 | INFO | Populating RTMetrics report | 20260505 19:14:25.494 | INFO | RTMetrics report populated | 20260505 19:14:25.494 | INFO | Opening tmp RTMetrics json file | 20260505 19:14:25.504 | INFO | Serializing RTMetrics to json | 20260505 19:14:25.505 | INFO | Serialization done, swapping tmp file now... | 20260505 19:14:25.505 | INFO | RTMetrics done | 20260505 19:14:25.586 | INFO | Finished a round of Baz IO: Wrote 2.063 GiB in 3.262 seconds | 20260505 19:14:25.586 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.80% / 0.00% / 78.20% | 20260505 19:14:25.586 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.218578% / 0% / 6.35783e-05% | 20260505 19:14:25.586 | INFO | Baz IO overall: | 20260505 19:14:25.586 | INFO | BytesWritten : 3305401611032 | 20260505 19:14:25.586 | INFO | NumEvents : 1564557790385 | 20260505 19:14:27.323 | INFO | Analyzing chunk frames = [5636096,5636608) | 20260505 19:14:27.323 | INFO | MemInfo: { | 20260505 19:14:27.323 | INFO | "gpu" : " 17 GB", | 20260505 19:14:27.323 | INFO | "huge" : "", | 20260505 19:14:27.323 | INFO | "huge_shared" : " 27 GB", | 20260505 19:14:27.323 | INFO | "malloc" : " 14 GB", | 20260505 19:14:27.323 | INFO | "pinned" : " 29 GB" | 20260505 19:14:27.323 | INFO | } | 20260505 19:14:27.323 | INFO | Perf report for the last 5.13163 seconds: | 20260505 19:14:27.323 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:14:27.323 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:27.323 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:27.323 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:27.323 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 19:14:27.323 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:27.323 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260505 19:14:27.323 | INFO | PRE_HQ : 60.3%, 1.8, 56.0ms, 0 | 20260505 19:14:27.323 | INFO | BAZWRITER : 63.1%, 3.8, 121.8ms, 0 | 20260505 19:14:27.323 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260505 19:14:27.323 | INFO | Uploaded 36GiB to GPU in 2.1584s (16.679GiB/s) | 20260505 19:14:27.323 | INFO | Downloaded 38GiB to GPU in 2.25224s (16.8721GiB/s) | 20260505 19:14:32.462 | INFO | Analyzing chunk frames = [5636608,5637120) | 20260505 19:14:32.462 | INFO | MemInfo: { | 20260505 19:14:32.462 | INFO | "gpu" : " 17 GB", | 20260505 19:14:32.462 | INFO | "huge" : "", | 20260505 19:14:32.462 | INFO | "huge_shared" : " 27 GB", | 20260505 19:14:32.462 | INFO | "malloc" : " 14 GB", | 20260505 19:14:32.462 | INFO | "pinned" : " 29 GB" | 20260505 19:14:32.462 | INFO | } | 20260505 19:14:32.462 | INFO | Perf report for the last 5.13916 seconds: | 20260505 19:14:32.462 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:14:32.462 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:32.462 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:32.462 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:32.462 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 19:14:32.462 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:32.462 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:32.462 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 19:14:32.462 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:32.462 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:32.462 | INFO | Uploaded 36GiB to GPU in 2.13313s (16.8766GiB/s) | 20260505 19:14:32.462 | INFO | Downloaded 34GiB to GPU in 2.03171s (16.7346GiB/s) | 20260505 19:14:37.611 | INFO | Analyzing chunk frames = [5637120,5637632) | 20260505 19:14:37.611 | INFO | MemInfo: { | 20260505 19:14:37.611 | INFO | "gpu" : " 17 GB", | 20260505 19:14:37.611 | INFO | "huge" : "", | 20260505 19:14:37.611 | INFO | "huge_shared" : " 27 GB", | 20260505 19:14:37.611 | INFO | "malloc" : " 14 GB", | 20260505 19:14:37.611 | INFO | "pinned" : " 29 GB" | 20260505 19:14:37.611 | INFO | } | 20260505 19:14:37.611 | INFO | Perf report for the last 5.14887 seconds: | 20260505 19:14:37.611 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:14:37.611 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:37.611 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:37.611 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:37.611 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260505 19:14:37.611 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:37.611 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:37.611 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 19:14:37.611 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:37.611 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:37.611 | INFO | Uploaded 36GiB to GPU in 2.21303s (16.2673GiB/s) | 20260505 19:14:37.611 | INFO | Downloaded 34GiB to GPU in 2.07888s (16.355GiB/s) | 20260505 19:14:42.756 | INFO | Analyzing chunk frames = [5637632,5638144) | 20260505 19:14:42.756 | INFO | MemInfo: { | 20260505 19:14:42.756 | INFO | "gpu" : " 17 GB", | 20260505 19:14:42.756 | INFO | "huge" : "", | 20260505 19:14:42.756 | INFO | "huge_shared" : " 27 GB", | 20260505 19:14:42.756 | INFO | "malloc" : " 14 GB", | 20260505 19:14:42.756 | INFO | "pinned" : " 29 GB" | 20260505 19:14:42.756 | INFO | } | 20260505 19:14:42.756 | INFO | Perf report for the last 5.14511 seconds: | 20260505 19:14:42.756 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:14:42.756 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:42.756 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:42.756 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:42.756 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260505 19:14:42.756 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:42.756 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:42.756 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 19:14:42.756 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:42.756 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:42.756 | INFO | Uploaded 36GiB to GPU in 2.20718s (16.3104GiB/s) | 20260505 19:14:42.756 | INFO | Downloaded 34GiB to GPU in 2.03869s (16.6774GiB/s) | 20260505 19:14:47.888 | INFO | Analyzing chunk frames = [5638144,5638656) | 20260505 19:14:47.888 | INFO | MemInfo: { | 20260505 19:14:47.888 | INFO | "gpu" : " 17 GB", | 20260505 19:14:47.888 | INFO | "huge" : "", | 20260505 19:14:47.888 | INFO | "huge_shared" : " 27 GB", | 20260505 19:14:47.888 | INFO | "malloc" : " 14 GB", | 20260505 19:14:47.888 | INFO | "pinned" : " 29 GB" | 20260505 19:14:47.888 | INFO | } | 20260505 19:14:47.888 | INFO | Perf report for the last 5.1322 seconds: | 20260505 19:14:47.888 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:14:47.888 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:47.888 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:47.888 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:47.888 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260505 19:14:47.888 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:47.888 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:47.888 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 19:14:47.888 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:47.888 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:47.889 | INFO | Uploaded 36GiB to GPU in 2.13492s (16.8625GiB/s) | 20260505 19:14:47.889 | INFO | Downloaded 34GiB to GPU in 2.06419s (16.4713GiB/s) | 20260505 19:14:53.021 | INFO | Analyzing chunk frames = [5638656,5639168) | 20260505 19:14:53.021 | INFO | MemInfo: { | 20260505 19:14:53.021 | INFO | "gpu" : " 17 GB", | 20260505 19:14:53.021 | INFO | "huge" : "", | 20260505 19:14:53.021 | INFO | "huge_shared" : " 27 GB", | 20260505 19:14:53.021 | INFO | "malloc" : " 14 GB", | 20260505 19:14:53.021 | INFO | "pinned" : " 29 GB" | 20260505 19:14:53.021 | INFO | } | 20260505 19:14:53.021 | INFO | Perf report for the last 5.13261 seconds: | 20260505 19:14:53.021 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:14:53.021 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:53.021 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:53.021 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:53.021 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 19:14:53.021 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:53.021 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:53.021 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 19:14:53.021 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:53.021 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:53.021 | INFO | Uploaded 36GiB to GPU in 2.1284s (16.9141GiB/s) | 20260505 19:14:53.021 | INFO | Downloaded 34GiB to GPU in 2.05322s (16.5594GiB/s) | 20260505 19:14:58.165 | INFO | Analyzing chunk frames = [5639168,5639680) | 20260505 19:14:58.165 | INFO | MemInfo: { | 20260505 19:14:58.165 | INFO | "gpu" : " 17 GB", | 20260505 19:14:58.165 | INFO | "huge" : "", | 20260505 19:14:58.165 | INFO | "huge_shared" : " 27 GB", | 20260505 19:14:58.165 | INFO | "malloc" : " 14 GB", | 20260505 19:14:58.165 | INFO | "pinned" : " 29 GB" | 20260505 19:14:58.165 | INFO | } | 20260505 19:14:58.165 | INFO | Perf report for the last 5.14444 seconds: | 20260505 19:14:58.165 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:14:58.165 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:58.165 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:58.165 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:58.165 | INFO | ANALYSIS : 63.3%, 3.0, 99.0ms, 0 | 20260505 19:14:58.165 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:58.165 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:58.165 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 19:14:58.166 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:58.166 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:14:58.166 | INFO | Uploaded 36GiB to GPU in 2.1448s (16.7848GiB/s) | 20260505 19:14:58.166 | INFO | Downloaded 34GiB to GPU in 2.16108s (15.7329GiB/s) | 20260505 19:15:03.305 | INFO | Analyzing chunk frames = [5639680,5640192) | 20260505 19:15:03.305 | INFO | MemInfo: { | 20260505 19:15:03.305 | INFO | "gpu" : " 17 GB", | 20260505 19:15:03.305 | INFO | "huge" : "", | 20260505 19:15:03.305 | INFO | "huge_shared" : " 27 GB", | 20260505 19:15:03.305 | INFO | "malloc" : " 14 GB", | 20260505 19:15:03.305 | INFO | "pinned" : " 29 GB" | 20260505 19:15:03.305 | INFO | } | 20260505 19:15:03.305 | INFO | Perf report for the last 5.13945 seconds: | 20260505 19:15:03.305 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:15:03.305 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:03.305 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:03.305 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:03.305 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 19:15:03.305 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:03.305 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:03.305 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 19:15:03.305 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:03.305 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:03.305 | INFO | Uploaded 36GiB to GPU in 2.20194s (16.3492GiB/s) | 20260505 19:15:03.305 | INFO | Downloaded 34GiB to GPU in 2.04224s (16.6484GiB/s) | 20260505 19:15:03.433 | INFO | Starting a round of Baz IO | 20260505 19:15:06.603 | INFO | Populating RTMetrics report | 20260505 19:15:06.603 | INFO | RTMetrics report populated | 20260505 19:15:06.603 | INFO | Opening tmp RTMetrics json file | 20260505 19:15:06.603 | INFO | Serializing RTMetrics to json | 20260505 19:15:06.605 | INFO | Serialization done, swapping tmp file now... | 20260505 19:15:06.605 | INFO | RTMetrics done | 20260505 19:15:06.697 | INFO | Finished a round of Baz IO: Wrote 2.061 GiB in 3.263 seconds | 20260505 19:15:06.697 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.80% / 0.00% / 78.20% | 20260505 19:15:06.697 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.216409% / 0% / 9.13938e-05% | 20260505 19:15:06.697 | INFO | Baz IO overall: | 20260505 19:15:06.697 | INFO | BytesWritten : 3307614428399 | 20260505 19:15:06.697 | INFO | NumEvents : 1565586999982 | 20260505 19:15:08.446 | INFO | Analyzing chunk frames = [5640192,5640704) | 20260505 19:15:08.446 | INFO | MemInfo: { | 20260505 19:15:08.446 | INFO | "gpu" : " 17 GB", | 20260505 19:15:08.446 | INFO | "huge" : "", | 20260505 19:15:08.446 | INFO | "huge_shared" : " 27 GB", | 20260505 19:15:08.446 | INFO | "malloc" : " 14 GB", | 20260505 19:15:08.446 | INFO | "pinned" : " 29 GB" | 20260505 19:15:08.447 | INFO | } | 20260505 19:15:08.447 | INFO | Perf report for the last 5.14167 seconds: | 20260505 19:15:08.447 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:15:08.447 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:08.447 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:08.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:08.447 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 19:15:08.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:08.447 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260505 19:15:08.447 | INFO | PRE_HQ : 60.3%, 1.8, 55.7ms, 0 | 20260505 19:15:08.447 | INFO | BAZWRITER : 63.0%, 3.8, 121.9ms, 0 | 20260505 19:15:08.447 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 19:15:08.447 | INFO | Uploaded 36GiB to GPU in 2.15171s (16.7309GiB/s) | 20260505 19:15:08.447 | INFO | Downloaded 38GiB to GPU in 2.23846s (16.976GiB/s) | 20260505 19:15:13.580 | INFO | Analyzing chunk frames = [5640704,5641216) | 20260505 19:15:13.580 | INFO | MemInfo: { | 20260505 19:15:13.580 | INFO | "gpu" : " 17 GB", | 20260505 19:15:13.580 | INFO | "huge" : "", | 20260505 19:15:13.580 | INFO | "huge_shared" : " 27 GB", | 20260505 19:15:13.580 | INFO | "malloc" : " 14 GB", | 20260505 19:15:13.580 | INFO | "pinned" : " 29 GB" | 20260505 19:15:13.580 | INFO | } | 20260505 19:15:13.580 | INFO | Perf report for the last 5.13348 seconds: | 20260505 19:15:13.580 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:15:13.580 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:13.580 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:13.580 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:13.580 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 19:15:13.580 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:13.580 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:13.580 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:15:13.580 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:13.580 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:13.580 | INFO | Uploaded 36GiB to GPU in 2.14901s (16.7519GiB/s) | 20260505 19:15:13.580 | INFO | Downloaded 34GiB to GPU in 2.04886s (16.5946GiB/s) | 20260505 19:15:18.719 | INFO | Analyzing chunk frames = [5641216,5641728) | 20260505 19:15:18.719 | INFO | MemInfo: { | 20260505 19:15:18.719 | INFO | "gpu" : " 17 GB", | 20260505 19:15:18.719 | INFO | "huge" : "", | 20260505 19:15:18.719 | INFO | "huge_shared" : " 27 GB", | 20260505 19:15:18.719 | INFO | "malloc" : " 14 GB", | 20260505 19:15:18.719 | INFO | "pinned" : " 29 GB" | 20260505 19:15:18.719 | INFO | } | 20260505 19:15:18.719 | INFO | Perf report for the last 5.13889 seconds: | 20260505 19:15:18.719 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:15:18.719 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:18.719 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:18.719 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:18.719 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 19:15:18.719 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:18.719 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:18.719 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 19:15:18.719 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:18.719 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:18.719 | INFO | Uploaded 36GiB to GPU in 2.20783s (16.3056GiB/s) | 20260505 19:15:18.719 | INFO | Downloaded 34GiB to GPU in 2.04519s (16.6244GiB/s) | 20260505 19:15:23.866 | INFO | Analyzing chunk frames = [5641728,5642240) | 20260505 19:15:23.866 | INFO | MemInfo: { | 20260505 19:15:23.866 | INFO | "gpu" : " 17 GB", | 20260505 19:15:23.866 | INFO | "huge" : "", | 20260505 19:15:23.866 | INFO | "huge_shared" : " 27 GB", | 20260505 19:15:23.866 | INFO | "malloc" : " 14 GB", | 20260505 19:15:23.866 | INFO | "pinned" : " 29 GB" | 20260505 19:15:23.866 | INFO | } | 20260505 19:15:23.866 | INFO | Perf report for the last 5.14726 seconds: | 20260505 19:15:23.866 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:15:23.866 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:23.866 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:23.866 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:23.866 | INFO | ANALYSIS : 62.6%, 3.0, 97.4ms, 0 | 20260505 19:15:23.866 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:23.866 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:23.866 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:15:23.866 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:23.866 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:23.866 | INFO | Uploaded 36GiB to GPU in 2.22221s (16.2001GiB/s) | 20260505 19:15:23.866 | INFO | Downloaded 34GiB to GPU in 2.12345s (16.0117GiB/s) | 20260505 19:15:29.006 | INFO | Analyzing chunk frames = [5642240,5642752) | 20260505 19:15:29.006 | INFO | MemInfo: { | 20260505 19:15:29.006 | INFO | "gpu" : " 17 GB", | 20260505 19:15:29.006 | INFO | "huge" : "", | 20260505 19:15:29.006 | INFO | "huge_shared" : " 27 GB", | 20260505 19:15:29.006 | INFO | "malloc" : " 14 GB", | 20260505 19:15:29.006 | INFO | "pinned" : " 29 GB" | 20260505 19:15:29.006 | INFO | } | 20260505 19:15:29.006 | INFO | Perf report for the last 5.1395 seconds: | 20260505 19:15:29.006 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:15:29.006 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:29.006 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:29.006 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:29.006 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 19:15:29.006 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:29.006 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:29.006 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 19:15:29.006 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:29.006 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:29.006 | INFO | Uploaded 36GiB to GPU in 2.12843s (16.9139GiB/s) | 20260505 19:15:29.006 | INFO | Downloaded 34GiB to GPU in 2.06133s (16.4942GiB/s) | 20260505 19:15:34.136 | INFO | Analyzing chunk frames = [5642752,5643264) | 20260505 19:15:34.136 | INFO | MemInfo: { | 20260505 19:15:34.136 | INFO | "gpu" : " 17 GB", | 20260505 19:15:34.136 | INFO | "huge" : "", | 20260505 19:15:34.136 | INFO | "huge_shared" : " 27 GB", | 20260505 19:15:34.136 | INFO | "malloc" : " 14 GB", | 20260505 19:15:34.136 | INFO | "pinned" : " 29 GB" | 20260505 19:15:34.136 | INFO | } | 20260505 19:15:34.136 | INFO | Perf report for the last 5.13075 seconds: | 20260505 19:15:34.136 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:15:34.136 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:34.136 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:34.137 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:34.137 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 19:15:34.137 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:34.137 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:34.137 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 19:15:34.137 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:34.137 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:34.137 | INFO | Uploaded 36GiB to GPU in 2.14562s (16.7783GiB/s) | 20260505 19:15:34.137 | INFO | Downloaded 34GiB to GPU in 2.0592s (16.5112GiB/s) | 20260505 19:15:39.272 | INFO | Analyzing chunk frames = [5643264,5643776) | 20260505 19:15:39.272 | INFO | MemInfo: { | 20260505 19:15:39.272 | INFO | "gpu" : " 17 GB", | 20260505 19:15:39.272 | INFO | "huge" : "", | 20260505 19:15:39.272 | INFO | "huge_shared" : " 27 GB", | 20260505 19:15:39.272 | INFO | "malloc" : " 14 GB", | 20260505 19:15:39.272 | INFO | "pinned" : " 29 GB" | 20260505 19:15:39.272 | INFO | } | 20260505 19:15:39.272 | INFO | Perf report for the last 5.13577 seconds: | 20260505 19:15:39.272 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:15:39.272 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:39.272 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:39.272 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:39.272 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260505 19:15:39.272 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:39.272 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:39.272 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 19:15:39.272 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:39.272 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:39.272 | INFO | Uploaded 36GiB to GPU in 2.17839s (16.526GiB/s) | 20260505 19:15:39.273 | INFO | Downloaded 34GiB to GPU in 2.1752s (15.6308GiB/s) | 20260505 19:15:44.419 | INFO | Analyzing chunk frames = [5643776,5644288) | 20260505 19:15:44.419 | INFO | MemInfo: { | 20260505 19:15:44.419 | INFO | "gpu" : " 17 GB", | 20260505 19:15:44.419 | INFO | "huge" : "", | 20260505 19:15:44.419 | INFO | "huge_shared" : " 27 GB", | 20260505 19:15:44.419 | INFO | "malloc" : " 14 GB", | 20260505 19:15:44.419 | INFO | "pinned" : " 29 GB" | 20260505 19:15:44.419 | INFO | } | 20260505 19:15:44.419 | INFO | Perf report for the last 5.14697 seconds: | 20260505 19:15:44.419 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:15:44.419 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:44.419 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:44.419 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:44.419 | INFO | ANALYSIS : 62.9%, 3.0, 98.4ms, 0 | 20260505 19:15:44.419 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:44.419 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:44.419 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:15:44.419 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:44.419 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:44.419 | INFO | Uploaded 36GiB to GPU in 2.20385s (16.335GiB/s) | 20260505 19:15:44.420 | INFO | Downloaded 34GiB to GPU in 2.0391s (16.6741GiB/s) | 20260505 19:15:44.420 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 19:15:44.420 | INFO | Total mem usage: 15.3 GiB | 20260505 19:15:44.420 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 19:15:44.420 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.05%) | 20260505 19:15:44.420 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 19:15:44.420 | INFO | High Water Breakdown: | 20260505 19:15:44.420 | INFO | 61% : 8.66 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 19:15:44.420 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 19:15:44.420 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 19:15:44.420 | INFO | ------------------------------------------ | 20260505 19:15:44.420 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:15:44.420 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 19:15:44.420 | INFO | Total mem usage: 26.5 GiB | 20260505 19:15:44.420 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 19:15:44.420 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 19:15:44.420 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 19:15:44.420 | INFO | High Water Breakdown: | 20260505 19:15:44.420 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:15:44.420 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:15:44.420 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:15:44.420 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:15:44.420 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:15:44.420 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:15:44.420 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 19:15:44.420 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 19:15:44.420 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:15:44.420 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 19:15:44.420 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 19:15:44.420 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 19:15:44.420 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 19:15:44.420 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 19:15:44.420 | INFO | ------------------------------------------ | 20260505 19:15:44.420 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:15:44.420 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 19:15:44.420 | INFO | Total mem usage: 25.6 GiB | 20260505 19:15:44.420 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 19:15:44.420 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 19:15:44.420 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 19:15:44.420 | INFO | High Water Breakdown: | 20260505 19:15:44.420 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 19:15:44.420 | INFO | ------------------------------------------ | 20260505 19:15:44.420 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:15:44.420 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 19:15:44.420 | INFO | Total mem usage: 15.7 GiB | 20260505 19:15:44.420 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 19:15:44.420 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 19:15:44.420 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 19:15:44.420 | INFO | High Water Breakdown: | 20260505 19:15:44.420 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 19:15:44.420 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 19:15:44.420 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 19:15:44.420 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 19:15:44.420 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 19:15:44.420 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 19:15:44.420 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 19:15:44.420 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 19:15:44.420 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 19:15:44.420 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:15:44.420 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 19:15:44.420 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:15:44.420 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 19:15:44.420 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 19:15:44.420 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 19:15:44.420 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 19:15:44.420 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:15:44.420 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 19:15:44.420 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:15:44.420 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:15:44.420 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:15:44.420 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 19:15:44.420 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 19:15:44.420 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:15:44.420 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 19:15:44.420 | INFO | ------------------------------------------ | 20260505 19:15:44.420 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:15:44.420 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 19:15:44.420 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:15:44.420 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:15:44.420 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.47e+01, 1.34e+01 / 2.24e+00 | 20260505 19:15:44.420 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.02e-01, 1.10e-02 / 3.85e-03 | 20260505 19:15:44.420 | INFO | PerfReport: 11.76%, Baseline -- 3.69e+00 / 4.53e+00, 5.46e-01 / 1.28e-01 | 20260505 19:15:44.420 | INFO | PerfReport: 3.24%, Histogram -- 1.02e+00 / 1.52e+00, 1.82e-01 / 9.29e-02 | 20260505 19:15:44.421 | INFO | PerfReport: 16.01%, DME -- 4.03e+01 / 4.46e+01, 4.24e+00 / 2.43e+00 | 20260505 19:15:44.421 | INFO | PerfReport: 56.89%, FrameLabeling -- 1.79e+01 / 1.93e+01, 1.33e+00 / 6.75e-01 | 20260505 19:15:44.421 | INFO | PerfReport: 7.34%, PulseAccumulating -- 2.31e+00 / 2.89e+00, 4.42e-01 / 1.24e-01 | 20260505 19:15:44.421 | INFO | PerfReport: 3.31%, Metrics -- 1.04e+00 / 1.37e+00, 2.85e-01 / 1.54e-01 | 20260505 19:15:44.421 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.61e-01 / 4.28e-01, 6.38e-02 / 4.39e-02 | 20260505 19:15:44.421 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 19:15:44.421 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:15:44.421 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:15:44.421 | INFO | PerfReport: Full Execution -- 9.98e+01 / -nan, 2.55e+01 / 0.00e+00 | 20260505 19:15:44.421 | INFO | PerfReport: 22.64%, Upload -- 2.26e+01 / 0.00e+00, 7.91e+00 / 0.00e+00 | 20260505 19:15:44.421 | INFO | PerfReport: 21.86%, Download -- 2.18e+01 / 0.00e+00, 7.73e+00 / 0.00e+00 | 20260505 19:15:44.421 | INFO | PerfReport: 31.49%, Compute -- 3.14e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 19:15:44.421 | INFO | PerfReport: 2.44%, LockUpload -- 2.44e+00 / 0.00e+00, 8.06e+00 / 0.00e+00 | 20260505 19:15:44.421 | INFO | PerfReport: 1.35%, LockDownload -- 1.34e+00 / 0.00e+00, 2.81e+00 / 0.00e+00 | 20260505 19:15:44.421 | INFO | PerfReport: 20.22%, LockCompute -- 2.02e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 19:15:44.421 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.89e-02 / 0.00e+00 | 20260505 19:15:44.550 | INFO | Starting a round of Baz IO | 20260505 19:15:47.723 | INFO | Populating RTMetrics report | 20260505 19:15:47.724 | INFO | RTMetrics report populated | 20260505 19:15:47.724 | INFO | Opening tmp RTMetrics json file | 20260505 19:15:47.727 | INFO | Serializing RTMetrics to json | 20260505 19:15:47.728 | INFO | Serialization done, swapping tmp file now... | 20260505 19:15:47.728 | INFO | RTMetrics done | 20260505 19:15:47.815 | INFO | Finished a round of Baz IO: Wrote 2.06 GiB in 3.264 seconds | 20260505 19:15:47.815 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.80% / 0.00% / 78.20% | 20260505 19:15:47.815 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.216583% / 3.97364e-06% / 7.94729e-05% | 20260505 19:15:47.815 | INFO | Baz IO overall: | 20260505 19:15:47.815 | INFO | BytesWritten : 3309826689455 | 20260505 19:15:47.815 | INFO | NumEvents : 1566615849600 | 20260505 19:15:49.563 | INFO | Analyzing chunk frames = [5644288,5644800) | 20260505 19:15:49.563 | INFO | MemInfo: { | 20260505 19:15:49.563 | INFO | "gpu" : " 17 GB", | 20260505 19:15:49.563 | INFO | "huge" : "", | 20260505 19:15:49.563 | INFO | "huge_shared" : " 27 GB", | 20260505 19:15:49.563 | INFO | "malloc" : " 14 GB", | 20260505 19:15:49.563 | INFO | "pinned" : " 29 GB" | 20260505 19:15:49.563 | INFO | } | 20260505 19:15:49.563 | INFO | Perf report for the last 5.14386 seconds: | 20260505 19:15:49.563 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:15:49.563 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:49.563 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:49.563 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:49.563 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 19:15:49.563 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:49.563 | INFO | RT_METRICS : 44.5%, 1.0, 23.4ms, 0 | 20260505 19:15:49.563 | INFO | PRE_HQ : 60.2%, 1.8, 55.7ms, 0 | 20260505 19:15:49.563 | INFO | BAZWRITER : 63.0%, 3.8, 122.0ms, 0 | 20260505 19:15:49.563 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 19:15:49.563 | INFO | Uploaded 36GiB to GPU in 2.16168s (16.6537GiB/s) | 20260505 19:15:49.563 | INFO | Downloaded 38GiB to GPU in 2.25085s (16.8825GiB/s) | 20260505 19:15:54.705 | INFO | Analyzing chunk frames = [5644800,5645312) | 20260505 19:15:54.705 | INFO | MemInfo: { | 20260505 19:15:54.705 | INFO | "gpu" : " 17 GB", | 20260505 19:15:54.705 | INFO | "huge" : "", | 20260505 19:15:54.705 | INFO | "huge_shared" : " 27 GB", | 20260505 19:15:54.705 | INFO | "malloc" : " 14 GB", | 20260505 19:15:54.705 | INFO | "pinned" : " 29 GB" | 20260505 19:15:54.705 | INFO | } | 20260505 19:15:54.705 | INFO | Perf report for the last 5.14201 seconds: | 20260505 19:15:54.705 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:15:54.705 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:54.705 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:54.705 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:54.705 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260505 19:15:54.705 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:54.705 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:54.705 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:15:54.705 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:54.705 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:54.705 | INFO | Uploaded 36GiB to GPU in 2.09986s (17.144GiB/s) | 20260505 19:15:54.705 | INFO | Downloaded 34GiB to GPU in 2.03054s (16.7443GiB/s) | 20260505 19:15:59.839 | INFO | Analyzing chunk frames = [5645312,5645824) | 20260505 19:15:59.839 | INFO | MemInfo: { | 20260505 19:15:59.839 | INFO | "gpu" : " 17 GB", | 20260505 19:15:59.839 | INFO | "huge" : "", | 20260505 19:15:59.839 | INFO | "huge_shared" : " 27 GB", | 20260505 19:15:59.839 | INFO | "malloc" : " 14 GB", | 20260505 19:15:59.839 | INFO | "pinned" : " 29 GB" | 20260505 19:15:59.839 | INFO | } | 20260505 19:15:59.839 | INFO | Perf report for the last 5.13428 seconds: | 20260505 19:15:59.839 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:15:59.839 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:59.839 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:59.839 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:59.840 | INFO | ANALYSIS : 62.2%, 3.0, 96.6ms, 0 | 20260505 19:15:59.840 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:59.840 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:59.840 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:15:59.840 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:59.840 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:15:59.840 | INFO | Uploaded 36GiB to GPU in 2.12194s (16.9656GiB/s) | 20260505 19:15:59.840 | INFO | Downloaded 34GiB to GPU in 2.01643s (16.8615GiB/s) | 20260505 19:16:04.973 | INFO | Analyzing chunk frames = [5645824,5646336) | 20260505 19:16:04.973 | INFO | MemInfo: { | 20260505 19:16:04.973 | INFO | "gpu" : " 17 GB", | 20260505 19:16:04.973 | INFO | "huge" : "", | 20260505 19:16:04.973 | INFO | "huge_shared" : " 27 GB", | 20260505 19:16:04.973 | INFO | "malloc" : " 14 GB", | 20260505 19:16:04.973 | INFO | "pinned" : " 29 GB" | 20260505 19:16:04.973 | INFO | } | 20260505 19:16:04.973 | INFO | Perf report for the last 5.13329 seconds: | 20260505 19:16:04.973 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:16:04.973 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:04.973 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:04.973 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:04.973 | INFO | ANALYSIS : 62.4%, 3.0, 96.8ms, 0 | 20260505 19:16:04.973 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:04.973 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:04.973 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 19:16:04.973 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:04.973 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:04.973 | INFO | Uploaded 36GiB to GPU in 2.16722s (16.6111GiB/s) | 20260505 19:16:04.973 | INFO | Downloaded 34GiB to GPU in 2.06529s (16.4626GiB/s) | 20260505 19:16:10.125 | INFO | Analyzing chunk frames = [5646336,5646848) | 20260505 19:16:10.125 | INFO | MemInfo: { | 20260505 19:16:10.125 | INFO | "gpu" : " 17 GB", | 20260505 19:16:10.125 | INFO | "huge" : "", | 20260505 19:16:10.125 | INFO | "huge_shared" : " 27 GB", | 20260505 19:16:10.125 | INFO | "malloc" : " 14 GB", | 20260505 19:16:10.125 | INFO | "pinned" : " 29 GB" | 20260505 19:16:10.125 | INFO | } | 20260505 19:16:10.125 | INFO | Perf report for the last 5.15244 seconds: | 20260505 19:16:10.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:16:10.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:10.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:10.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:10.125 | INFO | ANALYSIS : 62.0%, 3.0, 97.0ms, 0 | 20260505 19:16:10.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:10.125 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:10.125 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260505 19:16:10.125 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:10.125 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:10.126 | INFO | Uploaded 36GiB to GPU in 2.14809s (16.759GiB/s) | 20260505 19:16:10.126 | INFO | Downloaded 34GiB to GPU in 2.06186s (16.4899GiB/s) | 20260505 19:16:15.261 | INFO | Analyzing chunk frames = [5646848,5647360) | 20260505 19:16:15.261 | INFO | MemInfo: { | 20260505 19:16:15.261 | INFO | "gpu" : " 17 GB", | 20260505 19:16:15.261 | INFO | "huge" : "", | 20260505 19:16:15.261 | INFO | "huge_shared" : " 27 GB", | 20260505 19:16:15.261 | INFO | "malloc" : " 14 GB", | 20260505 19:16:15.262 | INFO | "pinned" : " 29 GB" | 20260505 19:16:15.262 | INFO | } | 20260505 19:16:15.262 | INFO | Perf report for the last 5.13637 seconds: | 20260505 19:16:15.262 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:16:15.262 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:15.262 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:15.262 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:15.262 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260505 19:16:15.262 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:15.262 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:15.262 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 19:16:15.262 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:15.262 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:15.262 | INFO | Uploaded 36GiB to GPU in 2.10501s (17.102GiB/s) | 20260505 19:16:15.262 | INFO | Downloaded 34GiB to GPU in 2.03406s (16.7153GiB/s) | 20260505 19:16:20.394 | INFO | Analyzing chunk frames = [5647360,5647872) | 20260505 19:16:20.394 | INFO | MemInfo: { | 20260505 19:16:20.394 | INFO | "gpu" : " 17 GB", | 20260505 19:16:20.394 | INFO | "huge" : "", | 20260505 19:16:20.394 | INFO | "huge_shared" : " 27 GB", | 20260505 19:16:20.394 | INFO | "malloc" : " 14 GB", | 20260505 19:16:20.394 | INFO | "pinned" : " 29 GB" | 20260505 19:16:20.394 | INFO | } | 20260505 19:16:20.394 | INFO | Perf report for the last 5.13257 seconds: | 20260505 19:16:20.394 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:16:20.394 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:20.394 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:20.394 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:20.394 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 19:16:20.394 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:20.394 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:20.394 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 19:16:20.394 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:20.394 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:20.394 | INFO | Uploaded 36GiB to GPU in 2.11268s (17.0399GiB/s) | 20260505 19:16:20.395 | INFO | Downloaded 34GiB to GPU in 2.1357s (15.9198GiB/s) | 20260505 19:16:25.530 | INFO | Analyzing chunk frames = [5647872,5648384) | 20260505 19:16:25.530 | INFO | MemInfo: { | 20260505 19:16:25.530 | INFO | "gpu" : " 17 GB", | 20260505 19:16:25.530 | INFO | "huge" : "", | 20260505 19:16:25.530 | INFO | "huge_shared" : " 27 GB", | 20260505 19:16:25.530 | INFO | "malloc" : " 14 GB", | 20260505 19:16:25.530 | INFO | "pinned" : " 29 GB" | 20260505 19:16:25.530 | INFO | } | 20260505 19:16:25.530 | INFO | Perf report for the last 5.13588 seconds: | 20260505 19:16:25.530 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:16:25.530 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:25.530 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:25.530 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:25.530 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 19:16:25.530 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:25.530 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:25.530 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 19:16:25.530 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:25.530 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:25.530 | INFO | Uploaded 36GiB to GPU in 2.19633s (16.391GiB/s) | 20260505 19:16:25.530 | INFO | Downloaded 34GiB to GPU in 2.02695s (16.774GiB/s) | 20260505 19:16:25.664 | INFO | Starting a round of Baz IO | 20260505 19:16:28.841 | INFO | Populating RTMetrics report | 20260505 19:16:28.842 | INFO | RTMetrics report populated | 20260505 19:16:28.842 | INFO | Opening tmp RTMetrics json file | 20260505 19:16:28.843 | INFO | Serializing RTMetrics to json | 20260505 19:16:28.844 | INFO | Serialization done, swapping tmp file now... | 20260505 19:16:28.844 | INFO | RTMetrics done | 20260505 19:16:28.937 | INFO | Finished a round of Baz IO: Wrote 2.057 GiB in 3.272 seconds | 20260505 19:16:28.937 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.80% / 0.00% / 78.20% | 20260505 19:16:28.937 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.215062% / 0% / 7.15256e-05% | 20260505 19:16:28.937 | INFO | Baz IO overall: | 20260505 19:16:28.937 | INFO | BytesWritten : 3312035740535 | 20260505 19:16:28.937 | INFO | NumEvents : 1567642618384 | 20260505 19:16:30.674 | INFO | Analyzing chunk frames = [5648384,5648896) | 20260505 19:16:30.674 | INFO | MemInfo: { | 20260505 19:16:30.674 | INFO | "gpu" : " 17 GB", | 20260505 19:16:30.674 | INFO | "huge" : "", | 20260505 19:16:30.674 | INFO | "huge_shared" : " 27 GB", | 20260505 19:16:30.674 | INFO | "malloc" : " 14 GB", | 20260505 19:16:30.674 | INFO | "pinned" : " 29 GB" | 20260505 19:16:30.674 | INFO | } | 20260505 19:16:30.674 | INFO | Perf report for the last 5.14391 seconds: | 20260505 19:16:30.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:16:30.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:30.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:30.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:30.674 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 19:16:30.674 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:30.674 | INFO | RT_METRICS : 44.1%, 1.0, 23.2ms, 0 | 20260505 19:16:30.674 | INFO | PRE_HQ : 60.3%, 1.8, 55.4ms, 0 | 20260505 19:16:30.674 | INFO | BAZWRITER : 63.2%, 3.7, 121.6ms, 0 | 20260505 19:16:30.674 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 19:16:30.674 | INFO | Uploaded 36GiB to GPU in 2.16929s (16.5953GiB/s) | 20260505 19:16:30.674 | INFO | Downloaded 38GiB to GPU in 2.24654s (16.9149GiB/s) | 20260505 19:16:35.817 | INFO | Analyzing chunk frames = [5648896,5649408) | 20260505 19:16:35.817 | INFO | MemInfo: { | 20260505 19:16:35.817 | INFO | "gpu" : " 17 GB", | 20260505 19:16:35.817 | INFO | "huge" : "", | 20260505 19:16:35.817 | INFO | "huge_shared" : " 27 GB", | 20260505 19:16:35.817 | INFO | "malloc" : " 14 GB", | 20260505 19:16:35.817 | INFO | "pinned" : " 29 GB" | 20260505 19:16:35.817 | INFO | } | 20260505 19:16:35.817 | INFO | Perf report for the last 5.14316 seconds: | 20260505 19:16:35.817 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:16:35.817 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:35.817 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:35.817 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:35.817 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260505 19:16:35.817 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:35.817 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:35.817 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 19:16:35.817 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:35.817 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:35.817 | INFO | Uploaded 36GiB to GPU in 2.15301s (16.7207GiB/s) | 20260505 19:16:35.817 | INFO | Downloaded 34GiB to GPU in 2.04914s (16.5924GiB/s) | 20260505 19:16:40.953 | INFO | Analyzing chunk frames = [5649408,5649920) | 20260505 19:16:40.953 | INFO | MemInfo: { | 20260505 19:16:40.953 | INFO | "gpu" : " 17 GB", | 20260505 19:16:40.953 | INFO | "huge" : "", | 20260505 19:16:40.953 | INFO | "huge_shared" : " 27 GB", | 20260505 19:16:40.953 | INFO | "malloc" : " 14 GB", | 20260505 19:16:40.953 | INFO | "pinned" : " 29 GB" | 20260505 19:16:40.953 | INFO | } | 20260505 19:16:40.953 | INFO | Perf report for the last 5.1358 seconds: | 20260505 19:16:40.953 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:16:40.953 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:40.953 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:40.953 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:40.953 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260505 19:16:40.953 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:40.953 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:40.953 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:16:40.953 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:40.953 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:40.953 | INFO | Uploaded 36GiB to GPU in 2.12813s (16.9163GiB/s) | 20260505 19:16:40.953 | INFO | Downloaded 34GiB to GPU in 2.01373s (16.8841GiB/s) | 20260505 19:16:46.087 | INFO | Analyzing chunk frames = [5649920,5650432) | 20260505 19:16:46.087 | INFO | MemInfo: { | 20260505 19:16:46.087 | INFO | "gpu" : " 17 GB", | 20260505 19:16:46.087 | INFO | "huge" : "", | 20260505 19:16:46.087 | INFO | "huge_shared" : " 27 GB", | 20260505 19:16:46.087 | INFO | "malloc" : " 14 GB", | 20260505 19:16:46.087 | INFO | "pinned" : " 29 GB" | 20260505 19:16:46.087 | INFO | } | 20260505 19:16:46.087 | INFO | Perf report for the last 5.13421 seconds: | 20260505 19:16:46.087 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:16:46.087 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:46.087 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:46.087 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:46.087 | INFO | ANALYSIS : 62.3%, 3.0, 96.7ms, 0 | 20260505 19:16:46.087 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:46.087 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:46.087 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 19:16:46.087 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:46.087 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:46.087 | INFO | Uploaded 36GiB to GPU in 2.18417s (16.4822GiB/s) | 20260505 19:16:46.087 | INFO | Downloaded 34GiB to GPU in 2.05151s (16.5732GiB/s) | 20260505 19:16:51.225 | INFO | Analyzing chunk frames = [5650432,5650944) | 20260505 19:16:51.225 | INFO | MemInfo: { | 20260505 19:16:51.225 | INFO | "gpu" : " 17 GB", | 20260505 19:16:51.225 | INFO | "huge" : "", | 20260505 19:16:51.225 | INFO | "huge_shared" : " 27 GB", | 20260505 19:16:51.225 | INFO | "malloc" : " 14 GB", | 20260505 19:16:51.225 | INFO | "pinned" : " 29 GB" | 20260505 19:16:51.225 | INFO | } | 20260505 19:16:51.225 | INFO | Perf report for the last 5.13788 seconds: | 20260505 19:16:51.225 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:16:51.225 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:51.225 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:51.225 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:51.225 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 19:16:51.225 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:51.225 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:51.225 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:16:51.225 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:51.225 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:51.225 | INFO | Uploaded 36GiB to GPU in 2.14949s (16.7481GiB/s) | 20260505 19:16:51.225 | INFO | Downloaded 34GiB to GPU in 2.05889s (16.5138GiB/s) | 20260505 19:16:56.372 | INFO | Analyzing chunk frames = [5650944,5651456) | 20260505 19:16:56.372 | INFO | MemInfo: { | 20260505 19:16:56.372 | INFO | "gpu" : " 17 GB", | 20260505 19:16:56.372 | INFO | "huge" : "", | 20260505 19:16:56.372 | INFO | "huge_shared" : " 27 GB", | 20260505 19:16:56.372 | INFO | "malloc" : " 14 GB", | 20260505 19:16:56.372 | INFO | "pinned" : " 29 GB" | 20260505 19:16:56.372 | INFO | } | 20260505 19:16:56.372 | INFO | Perf report for the last 5.14676 seconds: | 20260505 19:16:56.372 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:16:56.372 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:56.372 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:56.372 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:56.372 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260505 19:16:56.372 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:56.372 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:56.372 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:16:56.372 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:56.372 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:16:56.372 | INFO | Uploaded 36GiB to GPU in 2.14731s (16.7651GiB/s) | 20260505 19:16:56.372 | INFO | Downloaded 34GiB to GPU in 2.0465s (16.6138GiB/s) | 20260505 19:17:01.515 | INFO | Analyzing chunk frames = [5651456,5651968) | 20260505 19:17:01.515 | INFO | MemInfo: { | 20260505 19:17:01.515 | INFO | "gpu" : " 17 GB", | 20260505 19:17:01.515 | INFO | "huge" : "", | 20260505 19:17:01.515 | INFO | "huge_shared" : " 27 GB", | 20260505 19:17:01.515 | INFO | "malloc" : " 14 GB", | 20260505 19:17:01.515 | INFO | "pinned" : " 29 GB" | 20260505 19:17:01.515 | INFO | } | 20260505 19:17:01.515 | INFO | Perf report for the last 5.14336 seconds: | 20260505 19:17:01.515 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:17:01.515 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:01.515 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:01.515 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:01.515 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 19:17:01.515 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:01.515 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:01.515 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 19:17:01.515 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:01.515 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:01.515 | INFO | Uploaded 36GiB to GPU in 2.1064s (17.0908GiB/s) | 20260505 19:17:01.515 | INFO | Downloaded 34GiB to GPU in 2.1377s (15.9049GiB/s) | 20260505 19:17:06.648 | INFO | Analyzing chunk frames = [5651968,5652480) | 20260505 19:17:06.648 | INFO | MemInfo: { | 20260505 19:17:06.648 | INFO | "gpu" : " 17 GB", | 20260505 19:17:06.648 | INFO | "huge" : "", | 20260505 19:17:06.648 | INFO | "huge_shared" : " 27 GB", | 20260505 19:17:06.648 | INFO | "malloc" : " 14 GB", | 20260505 19:17:06.648 | INFO | "pinned" : " 29 GB" | 20260505 19:17:06.648 | INFO | } | 20260505 19:17:06.648 | INFO | Perf report for the last 5.13306 seconds: | 20260505 19:17:06.648 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:17:06.648 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:06.648 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:06.648 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:06.648 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260505 19:17:06.648 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:06.648 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:06.648 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 19:17:06.648 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:06.648 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:06.648 | INFO | Uploaded 36GiB to GPU in 2.11879s (16.9908GiB/s) | 20260505 19:17:06.648 | INFO | Downloaded 34GiB to GPU in 2.00795s (16.9327GiB/s) | 20260505 19:17:06.649 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 19:17:06.649 | INFO | Total mem usage: 15.3 GiB | 20260505 19:17:06.649 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 19:17:06.649 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.06%) | 20260505 19:17:06.649 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 19:17:06.649 | INFO | High Water Breakdown: | 20260505 19:17:06.649 | INFO | 61% : 8.66 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 19:17:06.649 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 19:17:06.649 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 19:17:06.649 | INFO | ------------------------------------------ | 20260505 19:17:06.649 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:17:06.649 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 19:17:06.649 | INFO | Total mem usage: 26.5 GiB | 20260505 19:17:06.649 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 19:17:06.649 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 19:17:06.649 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 19:17:06.649 | INFO | High Water Breakdown: | 20260505 19:17:06.649 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:17:06.649 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:17:06.649 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:17:06.649 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:17:06.649 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:17:06.649 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:17:06.649 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 19:17:06.649 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 19:17:06.649 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:17:06.649 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 19:17:06.649 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 19:17:06.649 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 19:17:06.649 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 19:17:06.649 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 19:17:06.649 | INFO | ------------------------------------------ | 20260505 19:17:06.649 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:17:06.649 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 19:17:06.649 | INFO | Total mem usage: 25.6 GiB | 20260505 19:17:06.649 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 19:17:06.649 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 19:17:06.649 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 19:17:06.649 | INFO | High Water Breakdown: | 20260505 19:17:06.649 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 19:17:06.649 | INFO | ------------------------------------------ | 20260505 19:17:06.649 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:17:06.649 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 19:17:06.649 | INFO | Total mem usage: 15.7 GiB | 20260505 19:17:06.649 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 19:17:06.649 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 19:17:06.649 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 19:17:06.649 | INFO | High Water Breakdown: | 20260505 19:17:06.649 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 19:17:06.649 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 19:17:06.649 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 19:17:06.649 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 19:17:06.649 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 19:17:06.649 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 19:17:06.649 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 19:17:06.649 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 19:17:06.649 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 19:17:06.649 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:17:06.649 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 19:17:06.649 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:17:06.649 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 19:17:06.649 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 19:17:06.649 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 19:17:06.649 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 19:17:06.649 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:17:06.649 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 19:17:06.649 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:17:06.649 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:17:06.649 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:17:06.649 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 19:17:06.650 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 19:17:06.650 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:17:06.650 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 19:17:06.650 | INFO | ------------------------------------------ | 20260505 19:17:06.650 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:17:06.650 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 19:17:06.650 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:17:06.650 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:17:06.650 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.48e+01, 1.37e+01 / 2.17e+00 | 20260505 19:17:06.650 | INFO | PerfReport: 0.29%, AggregateStats -- 9.24e-02 / 1.02e-01, 1.04e-02 / 3.69e-03 | 20260505 19:17:06.650 | INFO | PerfReport: 11.79%, Baseline -- 3.72e+00 / 4.56e+00, 5.61e-01 / 1.20e-01 | 20260505 19:17:06.650 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.52e+00, 1.91e-01 / 9.71e-02 | 20260505 19:17:06.650 | INFO | PerfReport: 15.97%, DME -- 4.03e+01 / 4.47e+01, 4.27e+00 / 2.29e+00 | 20260505 19:17:06.650 | INFO | PerfReport: 56.97%, FrameLabeling -- 1.80e+01 / 1.92e+01, 1.50e+00 / 9.23e-01 | 20260505 19:17:06.650 | INFO | PerfReport: 7.26%, PulseAccumulating -- 2.29e+00 / 2.91e+00, 4.31e-01 / 1.53e-01 | 20260505 19:17:06.650 | INFO | PerfReport: 3.29%, Metrics -- 1.04e+00 / 1.36e+00, 2.80e-01 / 1.19e-01 | 20260505 19:17:06.650 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.62e-01 / 4.27e-01, 6.36e-02 / 3.30e-02 | 20260505 19:17:06.650 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 19:17:06.650 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:17:06.650 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:17:06.650 | INFO | PerfReport: Full Execution -- 9.95e+01 / -nan, 2.54e+01 / 0.00e+00 | 20260505 19:17:06.650 | INFO | PerfReport: 22.42%, Upload -- 2.23e+01 / 0.00e+00, 7.53e+00 / 0.00e+00 | 20260505 19:17:06.650 | INFO | PerfReport: 21.74%, Download -- 2.16e+01 / 0.00e+00, 7.56e+00 / 0.00e+00 | 20260505 19:17:06.650 | INFO | PerfReport: 31.73%, Compute -- 3.16e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 19:17:06.650 | INFO | PerfReport: 2.29%, LockUpload -- 2.28e+00 / 0.00e+00, 7.97e+00 / 0.00e+00 | 20260505 19:17:06.650 | INFO | PerfReport: 1.35%, LockDownload -- 1.35e+00 / 0.00e+00, 2.93e+00 / 0.00e+00 | 20260505 19:17:06.650 | INFO | PerfReport: 20.46%, LockCompute -- 2.04e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 19:17:06.650 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.46e-02 / 0.00e+00 | 20260505 19:17:06.784 | INFO | Starting a round of Baz IO | 20260505 19:17:09.955 | INFO | Populating RTMetrics report | 20260505 19:17:09.955 | INFO | RTMetrics report populated | 20260505 19:17:09.955 | INFO | Opening tmp RTMetrics json file | 20260505 19:17:09.965 | INFO | Serializing RTMetrics to json | 20260505 19:17:09.966 | INFO | Serialization done, swapping tmp file now... | 20260505 19:17:09.967 | INFO | RTMetrics done | 20260505 19:17:10.046 | INFO | Finished a round of Baz IO: Wrote 2.057 GiB in 3.262 seconds | 20260505 19:17:10.046 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.80% / 0.00% / 78.20% | 20260505 19:17:10.046 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.21497% / 0% / 6.35783e-05% | 20260505 19:17:10.046 | INFO | Baz IO overall: | 20260505 19:17:10.046 | INFO | BytesWritten : 3314244773274 | 20260505 19:17:10.046 | INFO | NumEvents : 1568669546809 | 20260505 19:17:11.785 | INFO | Analyzing chunk frames = [5652480,5652992) | 20260505 19:17:11.785 | INFO | MemInfo: { | 20260505 19:17:11.785 | INFO | "gpu" : " 17 GB", | 20260505 19:17:11.785 | INFO | "huge" : "", | 20260505 19:17:11.785 | INFO | "huge_shared" : " 27 GB", | 20260505 19:17:11.785 | INFO | "malloc" : " 14 GB", | 20260505 19:17:11.785 | INFO | "pinned" : " 29 GB" | 20260505 19:17:11.785 | INFO | } | 20260505 19:17:11.785 | INFO | Perf report for the last 5.13655 seconds: | 20260505 19:17:11.785 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:17:11.785 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:11.785 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:11.785 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:11.785 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 19:17:11.785 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:11.785 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260505 19:17:11.785 | INFO | PRE_HQ : 60.4%, 1.8, 55.5ms, 0 | 20260505 19:17:11.785 | INFO | BAZWRITER : 63.0%, 3.8, 122.2ms, 0 | 20260505 19:17:11.785 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260505 19:17:11.785 | INFO | Uploaded 36GiB to GPU in 2.15814s (16.681GiB/s) | 20260505 19:17:11.785 | INFO | Downloaded 38GiB to GPU in 2.24086s (16.9578GiB/s) | 20260505 19:17:16.929 | INFO | Analyzing chunk frames = [5652992,5653504) | 20260505 19:17:16.929 | INFO | MemInfo: { | 20260505 19:17:16.929 | INFO | "gpu" : " 17 GB", | 20260505 19:17:16.929 | INFO | "huge" : "", | 20260505 19:17:16.929 | INFO | "huge_shared" : " 27 GB", | 20260505 19:17:16.929 | INFO | "malloc" : " 14 GB", | 20260505 19:17:16.929 | INFO | "pinned" : " 29 GB" | 20260505 19:17:16.929 | INFO | } | 20260505 19:17:16.929 | INFO | Perf report for the last 5.14408 seconds: | 20260505 19:17:16.929 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:17:16.929 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:16.929 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:16.929 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:16.929 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260505 19:17:16.929 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:16.929 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:16.929 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:17:16.929 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:16.929 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:16.929 | INFO | Uploaded 36GiB to GPU in 2.13349s (16.8737GiB/s) | 20260505 19:17:16.929 | INFO | Downloaded 34GiB to GPU in 2.03127s (16.7383GiB/s) | 20260505 19:17:22.069 | INFO | Analyzing chunk frames = [5653504,5654016) | 20260505 19:17:22.069 | INFO | MemInfo: { | 20260505 19:17:22.069 | INFO | "gpu" : " 17 GB", | 20260505 19:17:22.069 | INFO | "huge" : "", | 20260505 19:17:22.069 | INFO | "huge_shared" : " 27 GB", | 20260505 19:17:22.069 | INFO | "malloc" : " 14 GB", | 20260505 19:17:22.069 | INFO | "pinned" : " 29 GB" | 20260505 19:17:22.069 | INFO | } | 20260505 19:17:22.069 | INFO | Perf report for the last 5.14022 seconds: | 20260505 19:17:22.069 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:17:22.069 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:22.069 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:22.069 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:22.069 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260505 19:17:22.069 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:22.069 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:22.069 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:17:22.069 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:22.069 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:22.069 | INFO | Uploaded 36GiB to GPU in 2.13938s (16.8273GiB/s) | 20260505 19:17:22.069 | INFO | Downloaded 34GiB to GPU in 2.03419s (16.7142GiB/s) | 20260505 19:17:27.207 | INFO | Analyzing chunk frames = [5654016,5654528) | 20260505 19:17:27.207 | INFO | MemInfo: { | 20260505 19:17:27.207 | INFO | "gpu" : " 17 GB", | 20260505 19:17:27.207 | INFO | "huge" : "", | 20260505 19:17:27.207 | INFO | "huge_shared" : " 27 GB", | 20260505 19:17:27.207 | INFO | "malloc" : " 14 GB", | 20260505 19:17:27.207 | INFO | "pinned" : " 29 GB" | 20260505 19:17:27.207 | INFO | } | 20260505 19:17:27.207 | INFO | Perf report for the last 5.1378 seconds: | 20260505 19:17:27.207 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:17:27.207 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:27.207 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:27.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:27.207 | INFO | ANALYSIS : 62.4%, 3.0, 97.0ms, 0 | 20260505 19:17:27.207 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:27.207 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:27.207 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:17:27.207 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:27.207 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:27.207 | INFO | Uploaded 36GiB to GPU in 2.2022s (16.3473GiB/s) | 20260505 19:17:27.207 | INFO | Downloaded 34GiB to GPU in 2.04482s (16.6274GiB/s) | 20260505 19:17:32.342 | INFO | Analyzing chunk frames = [5654528,5655040) | 20260505 19:17:32.342 | INFO | MemInfo: { | 20260505 19:17:32.342 | INFO | "gpu" : " 17 GB", | 20260505 19:17:32.342 | INFO | "huge" : "", | 20260505 19:17:32.342 | INFO | "huge_shared" : " 27 GB", | 20260505 19:17:32.342 | INFO | "malloc" : " 14 GB", | 20260505 19:17:32.342 | INFO | "pinned" : " 29 GB" | 20260505 19:17:32.342 | INFO | } | 20260505 19:17:32.342 | INFO | Perf report for the last 5.13495 seconds: | 20260505 19:17:32.342 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:17:32.342 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:32.342 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:32.342 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:32.342 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260505 19:17:32.342 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:32.342 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:32.342 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 19:17:32.342 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:32.342 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:32.342 | INFO | Uploaded 36GiB to GPU in 2.14146s (16.811GiB/s) | 20260505 19:17:32.342 | INFO | Downloaded 34GiB to GPU in 2.05696s (16.5292GiB/s) | 20260505 19:17:37.484 | INFO | Analyzing chunk frames = [5655040,5655552) | 20260505 19:17:37.484 | INFO | MemInfo: { | 20260505 19:17:37.484 | INFO | "gpu" : " 17 GB", | 20260505 19:17:37.484 | INFO | "huge" : "", | 20260505 19:17:37.484 | INFO | "huge_shared" : " 27 GB", | 20260505 19:17:37.484 | INFO | "malloc" : " 14 GB", | 20260505 19:17:37.484 | INFO | "pinned" : " 29 GB" | 20260505 19:17:37.484 | INFO | } | 20260505 19:17:37.484 | INFO | Perf report for the last 5.14189 seconds: | 20260505 19:17:37.484 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:17:37.484 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:37.484 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:37.484 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:37.484 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 19:17:37.484 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:37.484 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:37.484 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:17:37.484 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:37.484 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:37.484 | INFO | Uploaded 36GiB to GPU in 2.14209s (16.806GiB/s) | 20260505 19:17:37.484 | INFO | Downloaded 34GiB to GPU in 2.04988s (16.5863GiB/s) | 20260505 19:17:42.631 | INFO | Analyzing chunk frames = [5655552,5656064) | 20260505 19:17:42.631 | INFO | MemInfo: { | 20260505 19:17:42.631 | INFO | "gpu" : " 17 GB", | 20260505 19:17:42.631 | INFO | "huge" : "", | 20260505 19:17:42.631 | INFO | "huge_shared" : " 27 GB", | 20260505 19:17:42.631 | INFO | "malloc" : " 14 GB", | 20260505 19:17:42.631 | INFO | "pinned" : " 29 GB" | 20260505 19:17:42.631 | INFO | } | 20260505 19:17:42.631 | INFO | Perf report for the last 5.14723 seconds: | 20260505 19:17:42.631 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:17:42.631 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:42.631 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:42.631 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:42.631 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 19:17:42.631 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:42.631 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:42.631 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 19:17:42.631 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:42.631 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:42.631 | INFO | Uploaded 36GiB to GPU in 2.12931s (16.9069GiB/s) | 20260505 19:17:42.631 | INFO | Downloaded 34GiB to GPU in 2.14934s (15.8188GiB/s) | 20260505 19:17:47.767 | INFO | Analyzing chunk frames = [5656064,5656576) | 20260505 19:17:47.767 | INFO | MemInfo: { | 20260505 19:17:47.767 | INFO | "gpu" : " 17 GB", | 20260505 19:17:47.767 | INFO | "huge" : "", | 20260505 19:17:47.767 | INFO | "huge_shared" : " 27 GB", | 20260505 19:17:47.767 | INFO | "malloc" : " 14 GB", | 20260505 19:17:47.767 | INFO | "pinned" : " 29 GB" | 20260505 19:17:47.767 | INFO | } | 20260505 19:17:47.767 | INFO | Perf report for the last 5.13585 seconds: | 20260505 19:17:47.767 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:17:47.767 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:47.767 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:47.767 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:47.767 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260505 19:17:47.767 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:47.767 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:47.767 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:17:47.767 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:47.767 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:47.767 | INFO | Uploaded 36GiB to GPU in 2.12697s (16.9255GiB/s) | 20260505 19:17:47.767 | INFO | Downloaded 34GiB to GPU in 2.00915s (16.9226GiB/s) | 20260505 19:17:47.899 | INFO | Starting a round of Baz IO | 20260505 19:17:51.069 | INFO | Populating RTMetrics report | 20260505 19:17:51.069 | INFO | RTMetrics report populated | 20260505 19:17:51.069 | INFO | Opening tmp RTMetrics json file | 20260505 19:17:51.070 | INFO | Serializing RTMetrics to json | 20260505 19:17:51.071 | INFO | Serialization done, swapping tmp file now... | 20260505 19:17:51.072 | INFO | RTMetrics done | 20260505 19:17:51.168 | INFO | Finished a round of Baz IO: Wrote 2.059 GiB in 3.269 seconds | 20260505 19:17:51.168 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.80% / 0.00% / 78.20% | 20260505 19:17:51.169 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.215216% / 1.19209e-05% / 6.35783e-05% | 20260505 19:17:51.169 | INFO | Baz IO overall: | 20260505 19:17:51.169 | INFO | BytesWritten : 3316455708424 | 20260505 19:17:51.169 | INFO | NumEvents : 1569698052012 | 20260505 19:17:52.900 | INFO | Analyzing chunk frames = [5656576,5657088) | 20260505 19:17:52.900 | INFO | MemInfo: { | 20260505 19:17:52.900 | INFO | "gpu" : " 17 GB", | 20260505 19:17:52.900 | INFO | "huge" : "", | 20260505 19:17:52.900 | INFO | "huge_shared" : " 27 GB", | 20260505 19:17:52.900 | INFO | "malloc" : " 14 GB", | 20260505 19:17:52.900 | INFO | "pinned" : " 29 GB" | 20260505 19:17:52.900 | INFO | } | 20260505 19:17:52.900 | INFO | Perf report for the last 5.1329 seconds: | 20260505 19:17:52.900 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:17:52.900 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:52.900 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:52.900 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:52.900 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 19:17:52.900 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:52.900 | INFO | RT_METRICS : 44.2%, 1.0, 23.1ms, 0 | 20260505 19:17:52.900 | INFO | PRE_HQ : 60.4%, 1.8, 55.6ms, 0 | 20260505 19:17:52.900 | INFO | BAZWRITER : 63.2%, 3.8, 122.4ms, 0 | 20260505 19:17:52.900 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 19:17:52.900 | INFO | Uploaded 36GiB to GPU in 2.15881s (16.6758GiB/s) | 20260505 19:17:52.900 | INFO | Downloaded 38GiB to GPU in 2.23789s (16.9803GiB/s) | 20260505 19:17:58.039 | INFO | Analyzing chunk frames = [5657088,5657600) | 20260505 19:17:58.039 | INFO | MemInfo: { | 20260505 19:17:58.039 | INFO | "gpu" : " 17 GB", | 20260505 19:17:58.039 | INFO | "huge" : "", | 20260505 19:17:58.039 | INFO | "huge_shared" : " 27 GB", | 20260505 19:17:58.039 | INFO | "malloc" : " 14 GB", | 20260505 19:17:58.039 | INFO | "pinned" : " 29 GB" | 20260505 19:17:58.039 | INFO | } | 20260505 19:17:58.039 | INFO | Perf report for the last 5.13911 seconds: | 20260505 19:17:58.039 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:17:58.039 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:58.039 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:58.039 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:58.039 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 19:17:58.039 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:58.039 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:58.039 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 19:17:58.039 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:58.039 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:17:58.039 | INFO | Uploaded 36GiB to GPU in 2.1239s (16.9499GiB/s) | 20260505 19:17:58.039 | INFO | Downloaded 34GiB to GPU in 2.02894s (16.7575GiB/s) | 20260505 19:18:03.182 | INFO | Analyzing chunk frames = [5657600,5658112) | 20260505 19:18:03.182 | INFO | MemInfo: { | 20260505 19:18:03.182 | INFO | "gpu" : " 17 GB", | 20260505 19:18:03.182 | INFO | "huge" : "", | 20260505 19:18:03.182 | INFO | "huge_shared" : " 27 GB", | 20260505 19:18:03.182 | INFO | "malloc" : " 14 GB", | 20260505 19:18:03.182 | INFO | "pinned" : " 29 GB" | 20260505 19:18:03.182 | INFO | } | 20260505 19:18:03.182 | INFO | Perf report for the last 5.14335 seconds: | 20260505 19:18:03.182 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:18:03.182 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:03.182 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:03.182 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:03.182 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 19:18:03.182 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:03.182 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:03.182 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260505 19:18:03.182 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:03.182 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:03.183 | INFO | Uploaded 36GiB to GPU in 2.20484s (16.3277GiB/s) | 20260505 19:18:03.183 | INFO | Downloaded 34GiB to GPU in 2.04947s (16.5897GiB/s) | 20260505 19:18:08.323 | INFO | Analyzing chunk frames = [5658112,5658624) | 20260505 19:18:08.323 | INFO | MemInfo: { | 20260505 19:18:08.323 | INFO | "gpu" : " 17 GB", | 20260505 19:18:08.323 | INFO | "huge" : "", | 20260505 19:18:08.323 | INFO | "huge_shared" : " 27 GB", | 20260505 19:18:08.323 | INFO | "malloc" : " 14 GB", | 20260505 19:18:08.323 | INFO | "pinned" : " 29 GB" | 20260505 19:18:08.323 | INFO | } | 20260505 19:18:08.323 | INFO | Perf report for the last 5.14068 seconds: | 20260505 19:18:08.323 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:18:08.323 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:08.323 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:08.323 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:08.323 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260505 19:18:08.323 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:08.323 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:08.323 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:18:08.323 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:08.323 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:08.323 | INFO | Uploaded 36GiB to GPU in 2.16215s (16.6501GiB/s) | 20260505 19:18:08.323 | INFO | Downloaded 34GiB to GPU in 2.05672s (16.5312GiB/s) | 20260505 19:18:13.462 | INFO | Analyzing chunk frames = [5658624,5659136) | 20260505 19:18:13.462 | INFO | MemInfo: { | 20260505 19:18:13.462 | INFO | "gpu" : " 17 GB", | 20260505 19:18:13.462 | INFO | "huge" : "", | 20260505 19:18:13.462 | INFO | "huge_shared" : " 27 GB", | 20260505 19:18:13.462 | INFO | "malloc" : " 14 GB", | 20260505 19:18:13.462 | INFO | "pinned" : " 29 GB" | 20260505 19:18:13.462 | INFO | } | 20260505 19:18:13.462 | INFO | Perf report for the last 5.13863 seconds: | 20260505 19:18:13.462 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:18:13.462 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:13.462 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:13.462 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:13.462 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260505 19:18:13.462 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:13.462 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:13.462 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:18:13.462 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:13.462 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:13.462 | INFO | Uploaded 36GiB to GPU in 2.12914s (16.9082GiB/s) | 20260505 19:18:13.462 | INFO | Downloaded 34GiB to GPU in 2.06132s (16.4943GiB/s) | 20260505 19:18:18.596 | INFO | Analyzing chunk frames = [5659136,5659648) | 20260505 19:18:18.596 | INFO | MemInfo: { | 20260505 19:18:18.596 | INFO | "gpu" : " 17 GB", | 20260505 19:18:18.596 | INFO | "huge" : "", | 20260505 19:18:18.596 | INFO | "huge_shared" : " 27 GB", | 20260505 19:18:18.596 | INFO | "malloc" : " 14 GB", | 20260505 19:18:18.597 | INFO | "pinned" : " 29 GB" | 20260505 19:18:18.597 | INFO | } | 20260505 19:18:18.597 | INFO | Perf report for the last 5.13488 seconds: | 20260505 19:18:18.597 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:18:18.597 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:18.597 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:18.597 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:18.597 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260505 19:18:18.597 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:18.597 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:18.597 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:18:18.597 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:18.597 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:18.597 | INFO | Uploaded 36GiB to GPU in 2.15295s (16.7212GiB/s) | 20260505 19:18:18.597 | INFO | Downloaded 34GiB to GPU in 2.06161s (16.492GiB/s) | 20260505 19:18:23.736 | INFO | Analyzing chunk frames = [5659648,5660160) | 20260505 19:18:23.736 | INFO | MemInfo: { | 20260505 19:18:23.736 | INFO | "gpu" : " 17 GB", | 20260505 19:18:23.736 | INFO | "huge" : "", | 20260505 19:18:23.736 | INFO | "huge_shared" : " 27 GB", | 20260505 19:18:23.736 | INFO | "malloc" : " 14 GB", | 20260505 19:18:23.736 | INFO | "pinned" : " 29 GB" | 20260505 19:18:23.736 | INFO | } | 20260505 19:18:23.736 | INFO | Perf report for the last 5.13923 seconds: | 20260505 19:18:23.736 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:18:23.736 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:23.736 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:23.736 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:23.736 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 19:18:23.736 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:23.736 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:23.736 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 19:18:23.736 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:23.736 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:23.736 | INFO | Uploaded 36GiB to GPU in 2.12105s (16.9727GiB/s) | 20260505 19:18:23.736 | INFO | Downloaded 34GiB to GPU in 2.14908s (15.8207GiB/s) | 20260505 19:18:28.881 | INFO | Analyzing chunk frames = [5660160,5660672) | 20260505 19:18:28.881 | INFO | MemInfo: { | 20260505 19:18:28.881 | INFO | "gpu" : " 17 GB", | 20260505 19:18:28.881 | INFO | "huge" : "", | 20260505 19:18:28.881 | INFO | "huge_shared" : " 27 GB", | 20260505 19:18:28.881 | INFO | "malloc" : " 14 GB", | 20260505 19:18:28.881 | INFO | "pinned" : " 29 GB" | 20260505 19:18:28.881 | INFO | } | 20260505 19:18:28.881 | INFO | Perf report for the last 5.14551 seconds: | 20260505 19:18:28.881 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:18:28.881 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:28.881 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:28.881 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:28.881 | INFO | ANALYSIS : 62.9%, 3.0, 98.4ms, 0 | 20260505 19:18:28.881 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:28.881 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:28.881 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260505 19:18:28.881 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:28.881 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:28.881 | INFO | Uploaded 36GiB to GPU in 2.20006s (16.3632GiB/s) | 20260505 19:18:28.881 | INFO | Downloaded 34GiB to GPU in 2.0352s (16.706GiB/s) | 20260505 19:18:28.882 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 19:18:28.882 | INFO | Total mem usage: 15.3 GiB | 20260505 19:18:28.882 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 19:18:28.882 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.05%) | 20260505 19:18:28.882 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 19:18:28.882 | INFO | High Water Breakdown: | 20260505 19:18:28.882 | INFO | 61% : 8.66 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 19:18:28.882 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 19:18:28.882 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 19:18:28.882 | INFO | ------------------------------------------ | 20260505 19:18:28.882 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:18:28.882 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 19:18:28.882 | INFO | Total mem usage: 26.5 GiB | 20260505 19:18:28.882 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 19:18:28.882 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 19:18:28.882 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 19:18:28.882 | INFO | High Water Breakdown: | 20260505 19:18:28.882 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:18:28.882 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:18:28.882 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:18:28.882 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:18:28.882 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:18:28.882 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:18:28.882 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 19:18:28.882 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 19:18:28.882 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:18:28.882 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 19:18:28.882 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 19:18:28.882 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 19:18:28.882 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 19:18:28.882 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 19:18:28.882 | INFO | ------------------------------------------ | 20260505 19:18:28.882 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:18:28.882 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 19:18:28.882 | INFO | Total mem usage: 25.6 GiB | 20260505 19:18:28.882 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 19:18:28.882 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 19:18:28.882 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 19:18:28.882 | INFO | High Water Breakdown: | 20260505 19:18:28.882 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 19:18:28.882 | INFO | ------------------------------------------ | 20260505 19:18:28.882 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:18:28.882 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 19:18:28.882 | INFO | Total mem usage: 15.7 GiB | 20260505 19:18:28.882 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 19:18:28.883 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 19:18:28.883 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 19:18:28.883 | INFO | High Water Breakdown: | 20260505 19:18:28.883 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 19:18:28.883 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 19:18:28.883 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 19:18:28.883 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 19:18:28.883 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 19:18:28.883 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 19:18:28.883 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 19:18:28.883 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 19:18:28.883 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 19:18:28.883 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:18:28.883 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 19:18:28.883 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:18:28.883 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 19:18:28.883 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 19:18:28.883 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 19:18:28.883 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 19:18:28.883 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:18:28.883 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 19:18:28.883 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:18:28.883 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:18:28.883 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:18:28.883 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 19:18:28.883 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 19:18:28.883 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:18:28.883 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 19:18:28.883 | INFO | ------------------------------------------ | 20260505 19:18:28.883 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:18:28.883 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 19:18:28.883 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:18:28.883 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:18:28.883 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.42e+01, 1.35e+01 / 1.86e+00 | 20260505 19:18:28.883 | INFO | PerfReport: 0.29%, AggregateStats -- 9.27e-02 / 1.03e-01, 1.04e-02 / 3.33e-03 | 20260505 19:18:28.883 | INFO | PerfReport: 11.81%, Baseline -- 3.72e+00 / 4.55e+00, 5.65e-01 / 1.11e-01 | 20260505 19:18:28.883 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.54e+00, 1.92e-01 / 9.02e-02 | 20260505 19:18:28.883 | INFO | PerfReport: 15.91%, DME -- 4.01e+01 / 4.42e+01, 4.07e+00 / 1.89e+00 | 20260505 19:18:28.883 | INFO | PerfReport: 56.97%, FrameLabeling -- 1.80e+01 / 1.93e+01, 1.48e+00 / 1.09e+00 | 20260505 19:18:28.883 | INFO | PerfReport: 7.28%, PulseAccumulating -- 2.30e+00 / 2.87e+00, 4.43e-01 / 1.91e-01 | 20260505 19:18:28.883 | INFO | PerfReport: 3.30%, Metrics -- 1.04e+00 / 1.36e+00, 2.89e-01 / 1.52e-01 | 20260505 19:18:28.883 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.62e-01 / 4.23e-01, 6.48e-02 / 4.09e-02 | 20260505 19:18:28.883 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 19:18:28.883 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:18:28.883 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:18:28.883 | INFO | PerfReport: Full Execution -- 9.96e+01 / -nan, 2.54e+01 / 0.00e+00 | 20260505 19:18:28.883 | INFO | PerfReport: 22.50%, Upload -- 2.24e+01 / 0.00e+00, 7.80e+00 / 0.00e+00 | 20260505 19:18:28.883 | INFO | PerfReport: 21.76%, Download -- 2.17e+01 / 0.00e+00, 7.70e+00 / 0.00e+00 | 20260505 19:18:28.883 | INFO | PerfReport: 31.65%, Compute -- 3.15e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 19:18:28.883 | INFO | PerfReport: 2.32%, LockUpload -- 2.31e+00 / 0.00e+00, 7.97e+00 / 0.00e+00 | 20260505 19:18:28.883 | INFO | PerfReport: 1.38%, LockDownload -- 1.38e+00 / 0.00e+00, 2.90e+00 / 0.00e+00 | 20260505 19:18:28.883 | INFO | PerfReport: 20.38%, LockCompute -- 2.03e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260505 19:18:28.883 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.71e-02 / 0.00e+00 | 20260505 19:18:29.013 | INFO | Starting a round of Baz IO | 20260505 19:18:32.188 | INFO | Populating RTMetrics report | 20260505 19:18:32.189 | INFO | RTMetrics report populated | 20260505 19:18:32.189 | INFO | Opening tmp RTMetrics json file | 20260505 19:18:32.191 | INFO | Serializing RTMetrics to json | 20260505 19:18:32.192 | INFO | Serialization done, swapping tmp file now... | 20260505 19:18:32.193 | INFO | RTMetrics done | 20260505 19:18:32.284 | INFO | Finished a round of Baz IO: Wrote 2.058 GiB in 3.27 seconds | 20260505 19:18:32.284 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.80% / 0.00% / 78.20% | 20260505 19:18:32.284 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.21406% / 0% / 7.54992e-05% | 20260505 19:18:32.284 | INFO | Baz IO overall: | 20260505 19:18:32.284 | INFO | BytesWritten : 3318665385680 | 20260505 19:18:32.284 | INFO | NumEvents : 1570725589382 | 20260505 19:18:34.024 | INFO | Analyzing chunk frames = [5660672,5661184) | 20260505 19:18:34.024 | INFO | MemInfo: { | 20260505 19:18:34.024 | INFO | "gpu" : " 17 GB", | 20260505 19:18:34.024 | INFO | "huge" : "", | 20260505 19:18:34.024 | INFO | "huge_shared" : " 27 GB", | 20260505 19:18:34.024 | INFO | "malloc" : " 14 GB", | 20260505 19:18:34.024 | INFO | "pinned" : " 29 GB" | 20260505 19:18:34.024 | INFO | } | 20260505 19:18:34.024 | INFO | Perf report for the last 5.14271 seconds: | 20260505 19:18:34.024 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:18:34.024 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:34.024 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:34.024 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:34.024 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 19:18:34.024 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:34.024 | INFO | RT_METRICS : 44.3%, 1.0, 23.3ms, 0 | 20260505 19:18:34.024 | INFO | PRE_HQ : 60.2%, 1.8, 55.6ms, 0 | 20260505 19:18:34.024 | INFO | BAZWRITER : 63.1%, 3.8, 123.1ms, 0 | 20260505 19:18:34.024 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 19:18:34.024 | INFO | Uploaded 36GiB to GPU in 2.16232s (16.6488GiB/s) | 20260505 19:18:34.024 | INFO | Downloaded 38GiB to GPU in 2.2501s (16.8881GiB/s) | 20260505 19:18:39.162 | INFO | Analyzing chunk frames = [5661184,5661696) | 20260505 19:18:39.162 | INFO | MemInfo: { | 20260505 19:18:39.162 | INFO | "gpu" : " 17 GB", | 20260505 19:18:39.162 | INFO | "huge" : "", | 20260505 19:18:39.162 | INFO | "huge_shared" : " 27 GB", | 20260505 19:18:39.162 | INFO | "malloc" : " 14 GB", | 20260505 19:18:39.162 | INFO | "pinned" : " 29 GB" | 20260505 19:18:39.162 | INFO | } | 20260505 19:18:39.162 | INFO | Perf report for the last 5.13815 seconds: | 20260505 19:18:39.162 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:18:39.162 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:39.162 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:39.162 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:39.162 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 19:18:39.162 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:39.162 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:39.162 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:18:39.162 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:39.162 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:39.162 | INFO | Uploaded 36GiB to GPU in 2.1279s (16.9181GiB/s) | 20260505 19:18:39.163 | INFO | Downloaded 34GiB to GPU in 2.03246s (16.7285GiB/s) | 20260505 19:18:44.293 | INFO | Analyzing chunk frames = [5661696,5662208) | 20260505 19:18:44.293 | INFO | MemInfo: { | 20260505 19:18:44.293 | INFO | "gpu" : " 17 GB", | 20260505 19:18:44.293 | INFO | "huge" : "", | 20260505 19:18:44.293 | INFO | "huge_shared" : " 27 GB", | 20260505 19:18:44.293 | INFO | "malloc" : " 14 GB", | 20260505 19:18:44.293 | INFO | "pinned" : " 29 GB" | 20260505 19:18:44.293 | INFO | } | 20260505 19:18:44.293 | INFO | Perf report for the last 5.13085 seconds: | 20260505 19:18:44.293 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:18:44.293 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:44.293 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:44.293 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:44.293 | INFO | ANALYSIS : 62.4%, 3.0, 96.9ms, 0 | 20260505 19:18:44.293 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:44.293 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:44.293 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:18:44.293 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:44.293 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:44.293 | INFO | Uploaded 36GiB to GPU in 2.13877s (16.8321GiB/s) | 20260505 19:18:44.293 | INFO | Downloaded 34GiB to GPU in 2.03379s (16.7176GiB/s) | 20260505 19:18:49.439 | INFO | Analyzing chunk frames = [5662208,5662720) | 20260505 19:18:49.439 | INFO | MemInfo: { | 20260505 19:18:49.439 | INFO | "gpu" : " 17 GB", | 20260505 19:18:49.439 | INFO | "huge" : "", | 20260505 19:18:49.439 | INFO | "huge_shared" : " 27 GB", | 20260505 19:18:49.439 | INFO | "malloc" : " 14 GB", | 20260505 19:18:49.439 | INFO | "pinned" : " 29 GB" | 20260505 19:18:49.439 | INFO | } | 20260505 19:18:49.439 | INFO | Perf report for the last 5.14614 seconds: | 20260505 19:18:49.439 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:18:49.439 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:49.439 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:49.439 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:49.439 | INFO | ANALYSIS : 62.6%, 3.0, 97.3ms, 0 | 20260505 19:18:49.439 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:49.439 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:49.439 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:18:49.439 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:49.439 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:49.440 | INFO | Uploaded 36GiB to GPU in 2.2216s (16.2046GiB/s) | 20260505 19:18:49.440 | INFO | Downloaded 34GiB to GPU in 2.12293s (16.0156GiB/s) | 20260505 19:18:54.577 | INFO | Analyzing chunk frames = [5662720,5663232) | 20260505 19:18:54.577 | INFO | MemInfo: { | 20260505 19:18:54.577 | INFO | "gpu" : " 17 GB", | 20260505 19:18:54.577 | INFO | "huge" : "", | 20260505 19:18:54.577 | INFO | "huge_shared" : " 27 GB", | 20260505 19:18:54.577 | INFO | "malloc" : " 14 GB", | 20260505 19:18:54.578 | INFO | "pinned" : " 29 GB" | 20260505 19:18:54.578 | INFO | } | 20260505 19:18:54.578 | INFO | Perf report for the last 5.13831 seconds: | 20260505 19:18:54.578 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:18:54.578 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:54.578 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:54.578 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:54.578 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 19:18:54.578 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:54.578 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:54.578 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:18:54.578 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:54.578 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:54.578 | INFO | Uploaded 36GiB to GPU in 2.13216s (16.8843GiB/s) | 20260505 19:18:54.578 | INFO | Downloaded 34GiB to GPU in 2.06007s (16.5043GiB/s) | 20260505 19:18:59.711 | INFO | Analyzing chunk frames = [5663232,5663744) | 20260505 19:18:59.711 | INFO | MemInfo: { | 20260505 19:18:59.711 | INFO | "gpu" : " 17 GB", | 20260505 19:18:59.711 | INFO | "huge" : "", | 20260505 19:18:59.711 | INFO | "huge_shared" : " 27 GB", | 20260505 19:18:59.711 | INFO | "malloc" : " 14 GB", | 20260505 19:18:59.711 | INFO | "pinned" : " 29 GB" | 20260505 19:18:59.711 | INFO | } | 20260505 19:18:59.711 | INFO | Perf report for the last 5.13366 seconds: | 20260505 19:18:59.711 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:18:59.711 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:59.711 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:59.711 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:59.711 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 19:18:59.711 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:59.711 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:59.711 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:18:59.711 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:59.711 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:18:59.712 | INFO | Uploaded 36GiB to GPU in 2.14443s (16.7877GiB/s) | 20260505 19:18:59.712 | INFO | Downloaded 34GiB to GPU in 2.05717s (16.5275GiB/s) | 20260505 19:19:04.853 | INFO | Analyzing chunk frames = [5663744,5664256) | 20260505 19:19:04.853 | INFO | MemInfo: { | 20260505 19:19:04.853 | INFO | "gpu" : " 17 GB", | 20260505 19:19:04.853 | INFO | "huge" : "", | 20260505 19:19:04.853 | INFO | "huge_shared" : " 27 GB", | 20260505 19:19:04.853 | INFO | "malloc" : " 14 GB", | 20260505 19:19:04.853 | INFO | "pinned" : " 29 GB" | 20260505 19:19:04.853 | INFO | } | 20260505 19:19:04.853 | INFO | Perf report for the last 5.14156 seconds: | 20260505 19:19:04.853 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:19:04.853 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:04.853 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:04.853 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:04.853 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 19:19:04.853 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:04.853 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:04.853 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 19:19:04.853 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:04.853 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:04.853 | INFO | Uploaded 36GiB to GPU in 2.12845s (16.9137GiB/s) | 20260505 19:19:04.853 | INFO | Downloaded 34GiB to GPU in 2.13584s (15.9188GiB/s) | 20260505 19:19:09.992 | INFO | Analyzing chunk frames = [5664256,5664768) | 20260505 19:19:09.992 | INFO | MemInfo: { | 20260505 19:19:09.992 | INFO | "gpu" : " 17 GB", | 20260505 19:19:09.992 | INFO | "huge" : "", | 20260505 19:19:09.992 | INFO | "huge_shared" : " 27 GB", | 20260505 19:19:09.992 | INFO | "malloc" : " 14 GB", | 20260505 19:19:09.992 | INFO | "pinned" : " 29 GB" | 20260505 19:19:09.992 | INFO | } | 20260505 19:19:09.992 | INFO | Perf report for the last 5.13933 seconds: | 20260505 19:19:09.992 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:19:09.992 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:09.992 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:09.992 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:09.992 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 19:19:09.992 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:09.992 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:09.992 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:19:09.992 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:09.992 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:09.992 | INFO | Uploaded 36GiB to GPU in 2.19315s (16.4147GiB/s) | 20260505 19:19:09.992 | INFO | Downloaded 34GiB to GPU in 2.03577s (16.7013GiB/s) | 20260505 19:19:10.126 | INFO | Starting a round of Baz IO | 20260505 19:19:13.300 | INFO | Populating RTMetrics report | 20260505 19:19:13.300 | INFO | RTMetrics report populated | 20260505 19:19:13.300 | INFO | Opening tmp RTMetrics json file | 20260505 19:19:13.302 | INFO | Serializing RTMetrics to json | 20260505 19:19:13.303 | INFO | Serialization done, swapping tmp file now... | 20260505 19:19:13.303 | INFO | RTMetrics done | 20260505 19:19:13.392 | INFO | Finished a round of Baz IO: Wrote 2.057 GiB in 3.266 seconds | 20260505 19:19:13.392 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.79% / 0.00% / 78.20% | 20260505 19:19:13.392 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.213655% / 3.97364e-06% / 6.75519e-05% | 20260505 19:19:13.392 | INFO | Baz IO overall: | 20260505 19:19:13.392 | INFO | BytesWritten : 3320873609477 | 20260505 19:19:13.392 | INFO | NumEvents : 1571752330182 | 20260505 19:19:15.137 | INFO | Analyzing chunk frames = [5664768,5665280) | 20260505 19:19:15.137 | INFO | MemInfo: { | 20260505 19:19:15.137 | INFO | "gpu" : " 17 GB", | 20260505 19:19:15.137 | INFO | "huge" : "", | 20260505 19:19:15.137 | INFO | "huge_shared" : " 27 GB", | 20260505 19:19:15.137 | INFO | "malloc" : " 14 GB", | 20260505 19:19:15.137 | INFO | "pinned" : " 29 GB" | 20260505 19:19:15.138 | INFO | } | 20260505 19:19:15.138 | INFO | Perf report for the last 5.14552 seconds: | 20260505 19:19:15.138 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:19:15.138 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:15.138 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:15.138 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:15.138 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 19:19:15.138 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:15.138 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260505 19:19:15.138 | INFO | PRE_HQ : 60.0%, 1.8, 55.1ms, 0 | 20260505 19:19:15.138 | INFO | BAZWRITER : 63.1%, 3.8, 122.6ms, 0 | 20260505 19:19:15.138 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 19:19:15.138 | INFO | Uploaded 36GiB to GPU in 2.16054s (16.6625GiB/s) | 20260505 19:19:15.138 | INFO | Downloaded 38GiB to GPU in 2.24749s (16.9078GiB/s) | 20260505 19:19:20.278 | INFO | Analyzing chunk frames = [5665280,5665792) | 20260505 19:19:20.279 | INFO | MemInfo: { | 20260505 19:19:20.279 | INFO | "gpu" : " 17 GB", | 20260505 19:19:20.279 | INFO | "huge" : "", | 20260505 19:19:20.279 | INFO | "huge_shared" : " 27 GB", | 20260505 19:19:20.279 | INFO | "malloc" : " 14 GB", | 20260505 19:19:20.279 | INFO | "pinned" : " 29 GB" | 20260505 19:19:20.279 | INFO | } | 20260505 19:19:20.279 | INFO | Perf report for the last 5.14105 seconds: | 20260505 19:19:20.279 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:19:20.279 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:20.279 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:20.279 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:20.279 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 19:19:20.279 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:20.279 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:20.279 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 19:19:20.279 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:20.279 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:20.279 | INFO | Uploaded 36GiB to GPU in 2.08444s (17.2709GiB/s) | 20260505 19:19:20.279 | INFO | Downloaded 34GiB to GPU in 2.02572s (16.7841GiB/s) | 20260505 19:19:25.407 | INFO | Analyzing chunk frames = [5665792,5666304) | 20260505 19:19:25.407 | INFO | MemInfo: { | 20260505 19:19:25.407 | INFO | "gpu" : " 17 GB", | 20260505 19:19:25.407 | INFO | "huge" : "", | 20260505 19:19:25.407 | INFO | "huge_shared" : " 27 GB", | 20260505 19:19:25.407 | INFO | "malloc" : " 14 GB", | 20260505 19:19:25.407 | INFO | "pinned" : " 29 GB" | 20260505 19:19:25.407 | INFO | } | 20260505 19:19:25.407 | INFO | Perf report for the last 5.12854 seconds: | 20260505 19:19:25.407 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:19:25.407 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:25.407 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:25.407 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:25.407 | INFO | ANALYSIS : 62.4%, 3.0, 96.9ms, 0 | 20260505 19:19:25.407 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:25.407 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:25.407 | INFO | PRE_HQ : 27.5%, 1.0, 14.2ms, 0 | 20260505 19:19:25.407 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:25.407 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:25.408 | INFO | Uploaded 36GiB to GPU in 2.13365s (16.8725GiB/s) | 20260505 19:19:25.408 | INFO | Downloaded 34GiB to GPU in 2.02858s (16.7605GiB/s) | 20260505 19:19:30.543 | INFO | Analyzing chunk frames = [5666304,5666816) | 20260505 19:19:30.543 | INFO | MemInfo: { | 20260505 19:19:30.543 | INFO | "gpu" : " 17 GB", | 20260505 19:19:30.543 | INFO | "huge" : "", | 20260505 19:19:30.543 | INFO | "huge_shared" : " 27 GB", | 20260505 19:19:30.543 | INFO | "malloc" : " 14 GB", | 20260505 19:19:30.543 | INFO | "pinned" : " 29 GB" | 20260505 19:19:30.543 | INFO | } | 20260505 19:19:30.543 | INFO | Perf report for the last 5.136 seconds: | 20260505 19:19:30.543 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:19:30.543 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:30.543 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:30.543 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:30.543 | INFO | ANALYSIS : 62.3%, 3.0, 96.7ms, 0 | 20260505 19:19:30.543 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:30.543 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:30.543 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:19:30.543 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:30.544 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:30.544 | INFO | Uploaded 36GiB to GPU in 2.17335s (16.5643GiB/s) | 20260505 19:19:30.544 | INFO | Downloaded 34GiB to GPU in 2.05808s (16.5203GiB/s) | 20260505 19:19:35.693 | INFO | Analyzing chunk frames = [5666816,5667328) | 20260505 19:19:35.693 | INFO | MemInfo: { | 20260505 19:19:35.693 | INFO | "gpu" : " 17 GB", | 20260505 19:19:35.693 | INFO | "huge" : "", | 20260505 19:19:35.693 | INFO | "huge_shared" : " 27 GB", | 20260505 19:19:35.693 | INFO | "malloc" : " 14 GB", | 20260505 19:19:35.693 | INFO | "pinned" : " 29 GB" | 20260505 19:19:35.693 | INFO | } | 20260505 19:19:35.693 | INFO | Perf report for the last 5.14951 seconds: | 20260505 19:19:35.693 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:19:35.693 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:35.693 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:35.693 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:35.693 | INFO | ANALYSIS : 61.9%, 3.0, 96.9ms, 0 | 20260505 19:19:35.693 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:35.693 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:35.693 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:19:35.693 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:35.693 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:35.693 | INFO | Uploaded 36GiB to GPU in 2.14561s (16.7785GiB/s) | 20260505 19:19:35.693 | INFO | Downloaded 34GiB to GPU in 2.06041s (16.5016GiB/s) | 20260505 19:19:40.833 | INFO | Analyzing chunk frames = [5667328,5667840) | 20260505 19:19:40.833 | INFO | MemInfo: { | 20260505 19:19:40.833 | INFO | "gpu" : " 17 GB", | 20260505 19:19:40.833 | INFO | "huge" : "", | 20260505 19:19:40.833 | INFO | "huge_shared" : " 27 GB", | 20260505 19:19:40.833 | INFO | "malloc" : " 14 GB", | 20260505 19:19:40.833 | INFO | "pinned" : " 29 GB" | 20260505 19:19:40.833 | INFO | } | 20260505 19:19:40.833 | INFO | Perf report for the last 5.14036 seconds: | 20260505 19:19:40.833 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:19:40.833 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:40.833 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:40.833 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:40.833 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 19:19:40.833 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:40.833 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:40.833 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:19:40.833 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:40.833 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:40.833 | INFO | Uploaded 36GiB to GPU in 2.13581s (16.8554GiB/s) | 20260505 19:19:40.833 | INFO | Downloaded 34GiB to GPU in 2.0569s (16.5297GiB/s) | 20260505 19:19:45.974 | INFO | Analyzing chunk frames = [5667840,5668352) | 20260505 19:19:45.974 | INFO | MemInfo: { | 20260505 19:19:45.974 | INFO | "gpu" : " 17 GB", | 20260505 19:19:45.974 | INFO | "huge" : "", | 20260505 19:19:45.974 | INFO | "huge_shared" : " 27 GB", | 20260505 19:19:45.974 | INFO | "malloc" : " 14 GB", | 20260505 19:19:45.974 | INFO | "pinned" : " 29 GB" | 20260505 19:19:45.974 | INFO | } | 20260505 19:19:45.974 | INFO | Perf report for the last 5.14084 seconds: | 20260505 19:19:45.974 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:19:45.974 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:45.974 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:45.974 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:45.974 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 19:19:45.974 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:45.974 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:45.974 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 19:19:45.974 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:45.974 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:45.975 | INFO | Uploaded 36GiB to GPU in 2.11278s (17.0392GiB/s) | 20260505 19:19:45.975 | INFO | Downloaded 34GiB to GPU in 2.16193s (15.7267GiB/s) | 20260505 19:19:51.110 | INFO | Analyzing chunk frames = [5668352,5668864) | 20260505 19:19:51.110 | INFO | MemInfo: { | 20260505 19:19:51.110 | INFO | "gpu" : " 17 GB", | 20260505 19:19:51.110 | INFO | "huge" : "", | 20260505 19:19:51.110 | INFO | "huge_shared" : " 27 GB", | 20260505 19:19:51.110 | INFO | "malloc" : " 14 GB", | 20260505 19:19:51.110 | INFO | "pinned" : " 29 GB" | 20260505 19:19:51.110 | INFO | } | 20260505 19:19:51.110 | INFO | Perf report for the last 5.13605 seconds: | 20260505 19:19:51.110 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:19:51.110 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:51.110 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:51.110 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:51.110 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 19:19:51.110 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:51.110 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:51.110 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:19:51.110 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:51.110 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:51.110 | INFO | Uploaded 36GiB to GPU in 2.13097s (16.8937GiB/s) | 20260505 19:19:51.110 | INFO | Downloaded 34GiB to GPU in 2.01375s (16.8839GiB/s) | 20260505 19:19:51.111 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 19:19:51.111 | INFO | Total mem usage: 15.3 GiB | 20260505 19:19:51.111 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 19:19:51.111 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.14%) | 20260505 19:19:51.111 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 19:19:51.111 | INFO | High Water Breakdown: | 20260505 19:19:51.111 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 19:19:51.111 | INFO | 38.9% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 19:19:51.111 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 19:19:51.111 | INFO | ------------------------------------------ | 20260505 19:19:51.111 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:19:51.111 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 19:19:51.111 | INFO | Total mem usage: 26.5 GiB | 20260505 19:19:51.111 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 19:19:51.111 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 19:19:51.111 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 19:19:51.111 | INFO | High Water Breakdown: | 20260505 19:19:51.111 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:19:51.111 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:19:51.111 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:19:51.111 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:19:51.111 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:19:51.111 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:19:51.111 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 19:19:51.111 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 19:19:51.111 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:19:51.111 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 19:19:51.111 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 19:19:51.111 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 19:19:51.111 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 19:19:51.111 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 19:19:51.111 | INFO | ------------------------------------------ | 20260505 19:19:51.111 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:19:51.111 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 19:19:51.111 | INFO | Total mem usage: 25.6 GiB | 20260505 19:19:51.111 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 19:19:51.111 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 19:19:51.111 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 19:19:51.111 | INFO | High Water Breakdown: | 20260505 19:19:51.111 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 19:19:51.111 | INFO | ------------------------------------------ | 20260505 19:19:51.111 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:19:51.111 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 19:19:51.111 | INFO | Total mem usage: 15.7 GiB | 20260505 19:19:51.111 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 19:19:51.111 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 19:19:51.111 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 19:19:51.111 | INFO | High Water Breakdown: | 20260505 19:19:51.111 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 19:19:51.111 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 19:19:51.111 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 19:19:51.111 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 19:19:51.111 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 19:19:51.111 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 19:19:51.111 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 19:19:51.111 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 19:19:51.111 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 19:19:51.111 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:19:51.111 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 19:19:51.111 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:19:51.111 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 19:19:51.111 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 19:19:51.111 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 19:19:51.111 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 19:19:51.111 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:19:51.111 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 19:19:51.111 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:19:51.112 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:19:51.112 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:19:51.112 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 19:19:51.112 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 19:19:51.112 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:19:51.112 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 19:19:51.112 | INFO | ------------------------------------------ | 20260505 19:19:51.112 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:19:51.112 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 19:19:51.112 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:19:51.112 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:19:51.112 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.48e+01, 1.36e+01 / 2.04e+00 | 20260505 19:19:51.112 | INFO | PerfReport: 0.29%, AggregateStats -- 9.27e-02 / 1.02e-01, 1.16e-02 / 4.78e-03 | 20260505 19:19:51.112 | INFO | PerfReport: 11.78%, Baseline -- 3.72e+00 / 4.50e+00, 5.67e-01 / 1.66e-01 | 20260505 19:19:51.112 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.50e+00, 1.91e-01 / 1.05e-01 | 20260505 19:19:51.112 | INFO | PerfReport: 15.95%, DME -- 4.03e+01 / 4.46e+01, 4.12e+00 / 2.03e+00 | 20260505 19:19:51.112 | INFO | PerfReport: 57.05%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.55e+00 / 1.21e+00 | 20260505 19:19:51.112 | INFO | PerfReport: 7.24%, PulseAccumulating -- 2.28e+00 / 2.89e+00, 4.37e-01 / 1.80e-01 | 20260505 19:19:51.112 | INFO | PerfReport: 3.27%, Metrics -- 1.03e+00 / 1.35e+00, 2.85e-01 / 1.79e-01 | 20260505 19:19:51.112 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.61e-01 / 4.20e-01, 6.45e-02 / 4.32e-02 | 20260505 19:19:51.112 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 19:19:51.112 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:19:51.112 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:19:51.112 | INFO | PerfReport: Full Execution -- 9.96e+01 / -nan, 2.54e+01 / 0.00e+00 | 20260505 19:19:51.112 | INFO | PerfReport: 22.45%, Upload -- 2.23e+01 / 0.00e+00, 7.73e+00 / 0.00e+00 | 20260505 19:19:51.112 | INFO | PerfReport: 21.83%, Download -- 2.17e+01 / 0.00e+00, 7.74e+00 / 0.00e+00 | 20260505 19:19:51.112 | INFO | PerfReport: 31.72%, Compute -- 3.16e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 19:19:51.112 | INFO | PerfReport: 2.22%, LockUpload -- 2.21e+00 / 0.00e+00, 7.77e+00 / 0.00e+00 | 20260505 19:19:51.112 | INFO | PerfReport: 1.40%, LockDownload -- 1.39e+00 / 0.00e+00, 2.95e+00 / 0.00e+00 | 20260505 19:19:51.112 | INFO | PerfReport: 20.37%, LockCompute -- 2.03e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 19:19:51.112 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.64e-02 / 0.00e+00 | 20260505 19:19:51.243 | INFO | Starting a round of Baz IO | 20260505 19:19:54.419 | INFO | Populating RTMetrics report | 20260505 19:19:54.419 | INFO | RTMetrics report populated | 20260505 19:19:54.419 | INFO | Opening tmp RTMetrics json file | 20260505 19:19:54.422 | INFO | Serializing RTMetrics to json | 20260505 19:19:54.423 | INFO | Serialization done, swapping tmp file now... | 20260505 19:19:54.423 | INFO | RTMetrics done | 20260505 19:19:54.512 | INFO | Finished a round of Baz IO: Wrote 2.055 GiB in 3.269 seconds | 20260505 19:19:54.512 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.79% / 0.00% / 78.21% | 20260505 19:19:54.512 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.213007% / 0% / 7.54992e-05% | 20260505 19:19:54.512 | INFO | Baz IO overall: | 20260505 19:19:54.512 | INFO | BytesWritten : 3323080622077 | 20260505 19:19:54.512 | INFO | NumEvents : 1572778267193 | 20260505 19:19:56.246 | INFO | Analyzing chunk frames = [5668864,5669376) | 20260505 19:19:56.246 | INFO | MemInfo: { | 20260505 19:19:56.246 | INFO | "gpu" : " 17 GB", | 20260505 19:19:56.246 | INFO | "huge" : "", | 20260505 19:19:56.246 | INFO | "huge_shared" : " 27 GB", | 20260505 19:19:56.246 | INFO | "malloc" : " 14 GB", | 20260505 19:19:56.246 | INFO | "pinned" : " 29 GB" | 20260505 19:19:56.246 | INFO | } | 20260505 19:19:56.246 | INFO | Perf report for the last 5.13616 seconds: | 20260505 19:19:56.246 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:19:56.246 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:56.246 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:56.246 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:56.246 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 19:19:56.246 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:19:56.246 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260505 19:19:56.247 | INFO | PRE_HQ : 60.3%, 1.8, 55.4ms, 0 | 20260505 19:19:56.247 | INFO | BAZWRITER : 63.2%, 3.8, 122.1ms, 0 | 20260505 19:19:56.247 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 19:19:56.247 | INFO | Uploaded 36GiB to GPU in 2.15722s (16.6881GiB/s) | 20260505 19:19:56.247 | INFO | Downloaded 38GiB to GPU in 2.23863s (16.9747GiB/s) | 20260505 19:20:01.392 | INFO | Analyzing chunk frames = [5669376,5669888) | 20260505 19:20:01.392 | INFO | MemInfo: { | 20260505 19:20:01.392 | INFO | "gpu" : " 17 GB", | 20260505 19:20:01.392 | INFO | "huge" : "", | 20260505 19:20:01.392 | INFO | "huge_shared" : " 27 GB", | 20260505 19:20:01.392 | INFO | "malloc" : " 14 GB", | 20260505 19:20:01.392 | INFO | "pinned" : " 29 GB" | 20260505 19:20:01.392 | INFO | } | 20260505 19:20:01.392 | INFO | Perf report for the last 5.14614 seconds: | 20260505 19:20:01.392 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:20:01.392 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:01.392 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:01.392 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:01.393 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260505 19:20:01.393 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:01.393 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:01.393 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:20:01.393 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:01.393 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:01.393 | INFO | Uploaded 36GiB to GPU in 2.13907s (16.8297GiB/s) | 20260505 19:20:01.393 | INFO | Downloaded 34GiB to GPU in 2.03139s (16.7373GiB/s) | 20260505 19:20:06.526 | INFO | Analyzing chunk frames = [5669888,5670400) | 20260505 19:20:06.527 | INFO | MemInfo: { | 20260505 19:20:06.527 | INFO | "gpu" : " 17 GB", | 20260505 19:20:06.527 | INFO | "huge" : "", | 20260505 19:20:06.527 | INFO | "huge_shared" : " 27 GB", | 20260505 19:20:06.527 | INFO | "malloc" : " 14 GB", | 20260505 19:20:06.527 | INFO | "pinned" : " 29 GB" | 20260505 19:20:06.527 | INFO | } | 20260505 19:20:06.527 | INFO | Perf report for the last 5.13413 seconds: | 20260505 19:20:06.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:20:06.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:06.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:06.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:06.527 | INFO | ANALYSIS : 62.2%, 3.0, 95.7ms, 0 | 20260505 19:20:06.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:06.527 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:06.527 | INFO | PRE_HQ : 27.3%, 1.0, 14.0ms, 0 | 20260505 19:20:06.527 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:06.527 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:06.527 | INFO | Uploaded 36GiB to GPU in 2.11873s (16.9913GiB/s) | 20260505 19:20:06.527 | INFO | Downloaded 34GiB to GPU in 2.01552s (16.8691GiB/s) | 20260505 19:20:11.662 | INFO | Analyzing chunk frames = [5670400,5670912) | 20260505 19:20:11.662 | INFO | MemInfo: { | 20260505 19:20:11.662 | INFO | "gpu" : " 17 GB", | 20260505 19:20:11.662 | INFO | "huge" : "", | 20260505 19:20:11.662 | INFO | "huge_shared" : " 27 GB", | 20260505 19:20:11.662 | INFO | "malloc" : " 14 GB", | 20260505 19:20:11.663 | INFO | "pinned" : " 29 GB" | 20260505 19:20:11.663 | INFO | } | 20260505 19:20:11.663 | INFO | Perf report for the last 5.13614 seconds: | 20260505 19:20:11.663 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:20:11.663 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:11.663 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:11.663 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:11.663 | INFO | ANALYSIS : 62.4%, 3.0, 96.9ms, 0 | 20260505 19:20:11.663 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:11.663 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:11.663 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:20:11.663 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:11.663 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:11.663 | INFO | Uploaded 36GiB to GPU in 2.19511s (16.4001GiB/s) | 20260505 19:20:11.663 | INFO | Downloaded 34GiB to GPU in 2.04404s (16.6337GiB/s) | 20260505 19:20:16.799 | INFO | Analyzing chunk frames = [5670912,5671424) | 20260505 19:20:16.799 | INFO | MemInfo: { | 20260505 19:20:16.799 | INFO | "gpu" : " 17 GB", | 20260505 19:20:16.799 | INFO | "huge" : "", | 20260505 19:20:16.799 | INFO | "huge_shared" : " 27 GB", | 20260505 19:20:16.799 | INFO | "malloc" : " 14 GB", | 20260505 19:20:16.800 | INFO | "pinned" : " 29 GB" | 20260505 19:20:16.800 | INFO | } | 20260505 19:20:16.800 | INFO | Perf report for the last 5.13688 seconds: | 20260505 19:20:16.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:20:16.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:16.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:16.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:16.800 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 19:20:16.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:16.800 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:16.800 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 19:20:16.800 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:16.800 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:16.800 | INFO | Uploaded 36GiB to GPU in 2.15631s (16.6952GiB/s) | 20260505 19:20:16.800 | INFO | Downloaded 34GiB to GPU in 2.06223s (16.487GiB/s) | 20260505 19:20:21.944 | INFO | Analyzing chunk frames = [5671424,5671936) | 20260505 19:20:21.944 | INFO | MemInfo: { | 20260505 19:20:21.944 | INFO | "gpu" : " 17 GB", | 20260505 19:20:21.944 | INFO | "huge" : "", | 20260505 19:20:21.944 | INFO | "huge_shared" : " 27 GB", | 20260505 19:20:21.944 | INFO | "malloc" : " 14 GB", | 20260505 19:20:21.944 | INFO | "pinned" : " 29 GB" | 20260505 19:20:21.944 | INFO | } | 20260505 19:20:21.944 | INFO | Perf report for the last 5.14421 seconds: | 20260505 19:20:21.944 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:20:21.944 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:21.944 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:21.944 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:21.944 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 19:20:21.944 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:21.944 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:21.944 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:20:21.944 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:21.944 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:21.944 | INFO | Uploaded 36GiB to GPU in 2.14756s (16.7632GiB/s) | 20260505 19:20:21.944 | INFO | Downloaded 34GiB to GPU in 2.04873s (16.5957GiB/s) | 20260505 19:20:27.091 | INFO | Analyzing chunk frames = [5671936,5672448) | 20260505 19:20:27.091 | INFO | MemInfo: { | 20260505 19:20:27.091 | INFO | "gpu" : " 17 GB", | 20260505 19:20:27.091 | INFO | "huge" : "", | 20260505 19:20:27.091 | INFO | "huge_shared" : " 27 GB", | 20260505 19:20:27.091 | INFO | "malloc" : " 14 GB", | 20260505 19:20:27.091 | INFO | "pinned" : " 29 GB" | 20260505 19:20:27.091 | INFO | } | 20260505 19:20:27.092 | INFO | Perf report for the last 5.14762 seconds: | 20260505 19:20:27.092 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:20:27.092 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:27.092 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:27.092 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:27.092 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 19:20:27.092 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:27.092 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:27.092 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 19:20:27.092 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:27.092 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:27.092 | INFO | Uploaded 36GiB to GPU in 2.10954s (17.0653GiB/s) | 20260505 19:20:27.092 | INFO | Downloaded 34GiB to GPU in 2.13483s (15.9263GiB/s) | 20260505 19:20:32.227 | INFO | Analyzing chunk frames = [5672448,5672960) | 20260505 19:20:32.227 | INFO | MemInfo: { | 20260505 19:20:32.227 | INFO | "gpu" : " 17 GB", | 20260505 19:20:32.227 | INFO | "huge" : "", | 20260505 19:20:32.227 | INFO | "huge_shared" : " 27 GB", | 20260505 19:20:32.227 | INFO | "malloc" : " 14 GB", | 20260505 19:20:32.227 | INFO | "pinned" : " 29 GB" | 20260505 19:20:32.227 | INFO | } | 20260505 19:20:32.227 | INFO | Perf report for the last 5.1356 seconds: | 20260505 19:20:32.227 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:20:32.227 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:32.227 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:32.227 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:32.227 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 19:20:32.227 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:32.227 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:32.227 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 19:20:32.227 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:32.227 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:32.227 | INFO | Uploaded 36GiB to GPU in 2.09692s (17.1681GiB/s) | 20260505 19:20:32.227 | INFO | Downloaded 34GiB to GPU in 1.9999s (17.0008GiB/s) | 20260505 19:20:32.360 | INFO | Starting a round of Baz IO | 20260505 19:20:35.532 | INFO | Populating RTMetrics report | 20260505 19:20:35.533 | INFO | RTMetrics report populated | 20260505 19:20:35.533 | INFO | Opening tmp RTMetrics json file | 20260505 19:20:35.534 | INFO | Serializing RTMetrics to json | 20260505 19:20:35.535 | INFO | Serialization done, swapping tmp file now... | 20260505 19:20:35.535 | INFO | RTMetrics done | 20260505 19:20:35.628 | INFO | Finished a round of Baz IO: Wrote 2.056 GiB in 3.267 seconds | 20260505 19:20:35.628 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.79% / 0.00% / 78.21% | 20260505 19:20:35.628 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.213305% / 0% / 8.74201e-05% | 20260505 19:20:35.628 | INFO | Baz IO overall: | 20260505 19:20:35.628 | INFO | BytesWritten : 3325288751049 | 20260505 19:20:35.628 | INFO | NumEvents : 1573805292836 | 20260505 19:20:37.354 | INFO | Analyzing chunk frames = [5672960,5673472) | 20260505 19:20:37.354 | INFO | MemInfo: { | 20260505 19:20:37.354 | INFO | "gpu" : " 17 GB", | 20260505 19:20:37.354 | INFO | "huge" : "", | 20260505 19:20:37.354 | INFO | "huge_shared" : " 27 GB", | 20260505 19:20:37.354 | INFO | "malloc" : " 14 GB", | 20260505 19:20:37.354 | INFO | "pinned" : " 29 GB" | 20260505 19:20:37.354 | INFO | } | 20260505 19:20:37.354 | INFO | Perf report for the last 5.12729 seconds: | 20260505 19:20:37.354 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:20:37.354 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:37.354 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:37.354 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:37.354 | INFO | ANALYSIS : 63.4%, 3.0, 98.6ms, 0 | 20260505 19:20:37.354 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:37.354 | INFO | RT_METRICS : 44.4%, 1.0, 23.2ms, 0 | 20260505 19:20:37.354 | INFO | PRE_HQ : 60.4%, 1.8, 55.4ms, 0 | 20260505 19:20:37.355 | INFO | BAZWRITER : 63.3%, 3.8, 122.2ms, 0 | 20260505 19:20:37.355 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 19:20:37.355 | INFO | Uploaded 36GiB to GPU in 2.16457s (16.6315GiB/s) | 20260505 19:20:37.355 | INFO | Downloaded 38GiB to GPU in 2.24435s (16.9314GiB/s) | 20260505 19:20:42.496 | INFO | Analyzing chunk frames = [5673472,5673984) | 20260505 19:20:42.496 | INFO | MemInfo: { | 20260505 19:20:42.496 | INFO | "gpu" : " 17 GB", | 20260505 19:20:42.496 | INFO | "huge" : "", | 20260505 19:20:42.496 | INFO | "huge_shared" : " 27 GB", | 20260505 19:20:42.496 | INFO | "malloc" : " 14 GB", | 20260505 19:20:42.496 | INFO | "pinned" : " 29 GB" | 20260505 19:20:42.496 | INFO | } | 20260505 19:20:42.496 | INFO | Perf report for the last 5.14162 seconds: | 20260505 19:20:42.496 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:20:42.496 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:42.496 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:42.496 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:42.496 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 19:20:42.496 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:42.496 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:42.496 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:20:42.496 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:42.496 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:42.496 | INFO | Uploaded 36GiB to GPU in 2.14693s (16.7681GiB/s) | 20260505 19:20:42.496 | INFO | Downloaded 34GiB to GPU in 2.03813s (16.682GiB/s) | 20260505 19:20:47.643 | INFO | Analyzing chunk frames = [5673984,5674496) | 20260505 19:20:47.643 | INFO | MemInfo: { | 20260505 19:20:47.643 | INFO | "gpu" : " 17 GB", | 20260505 19:20:47.643 | INFO | "huge" : "", | 20260505 19:20:47.643 | INFO | "huge_shared" : " 27 GB", | 20260505 19:20:47.643 | INFO | "malloc" : " 14 GB", | 20260505 19:20:47.643 | INFO | "pinned" : " 29 GB" | 20260505 19:20:47.643 | INFO | } | 20260505 19:20:47.643 | INFO | Perf report for the last 5.14697 seconds: | 20260505 19:20:47.643 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:20:47.643 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:47.643 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:47.643 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:47.643 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 19:20:47.643 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:47.643 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:47.643 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:20:47.643 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:47.643 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:47.643 | INFO | Uploaded 36GiB to GPU in 2.20968s (16.292GiB/s) | 20260505 19:20:47.643 | INFO | Downloaded 34GiB to GPU in 2.06615s (16.4557GiB/s) | 20260505 19:20:52.786 | INFO | Analyzing chunk frames = [5674496,5675008) | 20260505 19:20:52.786 | INFO | MemInfo: { | 20260505 19:20:52.786 | INFO | "gpu" : " 17 GB", | 20260505 19:20:52.786 | INFO | "huge" : "", | 20260505 19:20:52.786 | INFO | "huge_shared" : " 27 GB", | 20260505 19:20:52.786 | INFO | "malloc" : " 14 GB", | 20260505 19:20:52.786 | INFO | "pinned" : " 29 GB" | 20260505 19:20:52.786 | INFO | } | 20260505 19:20:52.786 | INFO | Perf report for the last 5.14295 seconds: | 20260505 19:20:52.786 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:20:52.786 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:52.786 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:52.786 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:52.786 | INFO | ANALYSIS : 62.5%, 3.0, 97.1ms, 0 | 20260505 19:20:52.786 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:52.786 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:52.786 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:20:52.786 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:52.786 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:52.786 | INFO | Uploaded 36GiB to GPU in 2.2084s (16.3014GiB/s) | 20260505 19:20:52.786 | INFO | Downloaded 34GiB to GPU in 2.0407s (16.6609GiB/s) | 20260505 19:20:57.918 | INFO | Analyzing chunk frames = [5675008,5675520) | 20260505 19:20:57.918 | INFO | MemInfo: { | 20260505 19:20:57.918 | INFO | "gpu" : " 17 GB", | 20260505 19:20:57.918 | INFO | "huge" : "", | 20260505 19:20:57.918 | INFO | "huge_shared" : " 27 GB", | 20260505 19:20:57.918 | INFO | "malloc" : " 14 GB", | 20260505 19:20:57.918 | INFO | "pinned" : " 29 GB" | 20260505 19:20:57.918 | INFO | } | 20260505 19:20:57.918 | INFO | Perf report for the last 5.13183 seconds: | 20260505 19:20:57.918 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:20:57.918 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:57.918 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:57.918 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:57.918 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260505 19:20:57.918 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:57.918 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:57.918 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:20:57.918 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:57.918 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:20:57.918 | INFO | Uploaded 36GiB to GPU in 2.13076s (16.8953GiB/s) | 20260505 19:20:57.918 | INFO | Downloaded 34GiB to GPU in 2.0702s (16.4236GiB/s) | 20260505 19:21:03.055 | INFO | Analyzing chunk frames = [5675520,5676032) | 20260505 19:21:03.055 | INFO | MemInfo: { | 20260505 19:21:03.055 | INFO | "gpu" : " 17 GB", | 20260505 19:21:03.055 | INFO | "huge" : "", | 20260505 19:21:03.055 | INFO | "huge_shared" : " 27 GB", | 20260505 19:21:03.055 | INFO | "malloc" : " 14 GB", | 20260505 19:21:03.055 | INFO | "pinned" : " 29 GB" | 20260505 19:21:03.055 | INFO | } | 20260505 19:21:03.055 | INFO | Perf report for the last 5.13724 seconds: | 20260505 19:21:03.055 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:21:03.055 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:03.055 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:03.055 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:03.055 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 19:21:03.055 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:03.055 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:03.055 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260505 19:21:03.055 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:03.055 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:03.056 | INFO | Uploaded 36GiB to GPU in 2.12745s (16.9216GiB/s) | 20260505 19:21:03.056 | INFO | Downloaded 34GiB to GPU in 2.04973s (16.5875GiB/s) | 20260505 19:21:08.204 | INFO | Analyzing chunk frames = [5676032,5676544) | 20260505 19:21:08.204 | INFO | MemInfo: { | 20260505 19:21:08.204 | INFO | "gpu" : " 17 GB", | 20260505 19:21:08.204 | INFO | "huge" : "", | 20260505 19:21:08.204 | INFO | "huge_shared" : " 27 GB", | 20260505 19:21:08.204 | INFO | "malloc" : " 14 GB", | 20260505 19:21:08.205 | INFO | "pinned" : " 29 GB" | 20260505 19:21:08.205 | INFO | } | 20260505 19:21:08.205 | INFO | Perf report for the last 5.14952 seconds: | 20260505 19:21:08.205 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:21:08.205 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:08.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:08.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:08.205 | INFO | ANALYSIS : 62.9%, 3.0, 98.5ms, 0 | 20260505 19:21:08.205 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:08.205 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:08.205 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 19:21:08.205 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:08.205 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:08.205 | INFO | Uploaded 36GiB to GPU in 2.12271s (16.9594GiB/s) | 20260505 19:21:08.205 | INFO | Downloaded 34GiB to GPU in 2.1492s (15.8198GiB/s) | 20260505 19:21:13.339 | INFO | Analyzing chunk frames = [5676544,5677056) | 20260505 19:21:13.339 | INFO | MemInfo: { | 20260505 19:21:13.339 | INFO | "gpu" : " 17 GB", | 20260505 19:21:13.339 | INFO | "huge" : "", | 20260505 19:21:13.339 | INFO | "huge_shared" : " 27 GB", | 20260505 19:21:13.339 | INFO | "malloc" : " 14 GB", | 20260505 19:21:13.339 | INFO | "pinned" : " 29 GB" | 20260505 19:21:13.339 | INFO | } | 20260505 19:21:13.339 | INFO | Perf report for the last 5.13445 seconds: | 20260505 19:21:13.339 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:21:13.339 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:13.339 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:13.339 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:13.339 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 19:21:13.339 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:13.339 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:13.339 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 19:21:13.339 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:13.339 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:13.339 | INFO | Uploaded 36GiB to GPU in 2.12224s (16.9632GiB/s) | 20260505 19:21:13.340 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 19:21:13.340 | INFO | Total mem usage: 15.3 GiB | 20260505 19:21:13.340 | INFO | Downloaded 34GiB to GPU in 2.00498s (16.9578GiB/s) | 20260505 19:21:13.340 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 19:21:13.340 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.05%) | 20260505 19:21:13.340 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 19:21:13.340 | INFO | High Water Breakdown: | 20260505 19:21:13.340 | INFO | 61% : 8.66 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 19:21:13.340 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 19:21:13.340 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 19:21:13.340 | INFO | ------------------------------------------ | 20260505 19:21:13.340 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:21:13.340 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 19:21:13.340 | INFO | Total mem usage: 26.5 GiB | 20260505 19:21:13.340 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 19:21:13.340 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 19:21:13.340 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 19:21:13.340 | INFO | High Water Breakdown: | 20260505 19:21:13.340 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:21:13.340 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:21:13.340 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:21:13.340 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:21:13.340 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:21:13.340 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:21:13.340 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 19:21:13.340 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 19:21:13.340 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:21:13.340 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 19:21:13.340 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 19:21:13.340 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 19:21:13.340 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 19:21:13.340 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 19:21:13.340 | INFO | ------------------------------------------ | 20260505 19:21:13.340 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:21:13.340 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 19:21:13.340 | INFO | Total mem usage: 25.6 GiB | 20260505 19:21:13.340 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 19:21:13.340 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 19:21:13.340 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 19:21:13.340 | INFO | High Water Breakdown: | 20260505 19:21:13.340 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 19:21:13.340 | INFO | ------------------------------------------ | 20260505 19:21:13.340 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:21:13.340 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 19:21:13.340 | INFO | Total mem usage: 15.7 GiB | 20260505 19:21:13.340 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 19:21:13.340 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 19:21:13.340 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 19:21:13.340 | INFO | High Water Breakdown: | 20260505 19:21:13.340 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 19:21:13.340 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 19:21:13.340 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 19:21:13.340 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 19:21:13.340 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 19:21:13.340 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 19:21:13.341 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 19:21:13.341 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 19:21:13.341 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 19:21:13.341 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:21:13.341 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 19:21:13.341 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:21:13.341 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 19:21:13.341 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 19:21:13.341 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 19:21:13.341 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 19:21:13.341 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:21:13.341 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 19:21:13.341 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:21:13.341 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:21:13.341 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:21:13.341 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 19:21:13.341 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 19:21:13.341 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:21:13.341 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 19:21:13.341 | INFO | ------------------------------------------ | 20260505 19:21:13.341 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:21:13.341 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 19:21:13.341 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:21:13.341 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:21:13.341 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.46e+01, 1.34e+01 / 1.98e+00 | 20260505 19:21:13.341 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.03e-01, 1.03e-02 / 4.22e-03 | 20260505 19:21:13.341 | INFO | PerfReport: 11.80%, Baseline -- 3.72e+00 / 4.54e+00, 5.65e-01 / 1.19e-01 | 20260505 19:21:13.341 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.54e+00, 1.90e-01 / 9.17e-02 | 20260505 19:21:13.341 | INFO | PerfReport: 15.96%, DME -- 4.02e+01 / 4.44e+01, 4.00e+00 / 2.06e+00 | 20260505 19:21:13.341 | INFO | PerfReport: 56.95%, FrameLabeling -- 1.79e+01 / 1.92e+01, 1.45e+00 / 1.11e+00 | 20260505 19:21:13.341 | INFO | PerfReport: 7.27%, PulseAccumulating -- 2.29e+00 / 2.89e+00, 4.38e-01 / 1.63e-01 | 20260505 19:21:13.341 | INFO | PerfReport: 3.30%, Metrics -- 1.04e+00 / 1.37e+00, 2.86e-01 / 1.35e-01 | 20260505 19:21:13.341 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.61e-01 / 4.25e-01, 6.44e-02 / 3.20e-02 | 20260505 19:21:13.341 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 19:21:13.341 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:21:13.341 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:21:13.341 | INFO | PerfReport: Full Execution -- 9.96e+01 / -nan, 2.55e+01 / 0.00e+00 | 20260505 19:21:13.341 | INFO | PerfReport: 22.45%, Upload -- 2.24e+01 / 0.00e+00, 7.70e+00 / 0.00e+00 | 20260505 19:21:13.341 | INFO | PerfReport: 21.72%, Download -- 2.16e+01 / 0.00e+00, 7.65e+00 / 0.00e+00 | 20260505 19:21:13.341 | INFO | PerfReport: 31.64%, Compute -- 3.15e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 19:21:13.341 | INFO | PerfReport: 2.36%, LockUpload -- 2.35e+00 / 0.00e+00, 8.18e+00 / 0.00e+00 | 20260505 19:21:13.341 | INFO | PerfReport: 1.39%, LockDownload -- 1.39e+00 / 0.00e+00, 2.95e+00 / 0.00e+00 | 20260505 19:21:13.341 | INFO | PerfReport: 20.42%, LockCompute -- 2.03e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 19:21:13.341 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.50e-02 / 0.00e+00 | 20260505 19:21:13.470 | INFO | Starting a round of Baz IO | 20260505 19:21:16.643 | INFO | Populating RTMetrics report | 20260505 19:21:16.643 | INFO | RTMetrics report populated | 20260505 19:21:16.643 | INFO | Opening tmp RTMetrics json file | 20260505 19:21:16.644 | INFO | Serializing RTMetrics to json | 20260505 19:21:16.645 | INFO | Serialization done, swapping tmp file now... | 20260505 19:21:16.645 | INFO | RTMetrics done | 20260505 19:21:16.731 | INFO | Finished a round of Baz IO: Wrote 2.057 GiB in 3.26 seconds | 20260505 19:21:16.731 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.79% / 0.00% / 78.21% | 20260505 19:21:16.731 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.212777% / 0% / 9.93411e-05% | 20260505 19:21:16.731 | INFO | Baz IO overall: | 20260505 19:21:16.731 | INFO | BytesWritten : 3327497056087 | 20260505 19:21:16.731 | INFO | NumEvents : 1574832298074 | 20260505 19:21:18.478 | INFO | Analyzing chunk frames = [5677056,5677568) | 20260505 19:21:18.478 | INFO | MemInfo: { | 20260505 19:21:18.478 | INFO | "gpu" : " 17 GB", | 20260505 19:21:18.478 | INFO | "huge" : "", | 20260505 19:21:18.478 | INFO | "huge_shared" : " 27 GB", | 20260505 19:21:18.478 | INFO | "malloc" : " 14 GB", | 20260505 19:21:18.478 | INFO | "pinned" : " 29 GB" | 20260505 19:21:18.478 | INFO | } | 20260505 19:21:18.478 | INFO | Perf report for the last 5.13926 seconds: | 20260505 19:21:18.478 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:21:18.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:18.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:18.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:18.479 | INFO | ANALYSIS : 63.1%, 3.0, 98.3ms, 0 | 20260505 19:21:18.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:18.479 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260505 19:21:18.479 | INFO | PRE_HQ : 60.3%, 1.8, 56.0ms, 0 | 20260505 19:21:18.479 | INFO | BAZWRITER : 62.9%, 3.8, 124.4ms, 0 | 20260505 19:21:18.479 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 19:21:18.479 | INFO | Uploaded 36GiB to GPU in 2.15827s (16.68GiB/s) | 20260505 19:21:18.479 | INFO | Downloaded 38GiB to GPU in 2.25187s (16.8749GiB/s) | 20260505 19:21:23.615 | INFO | Analyzing chunk frames = [5677568,5678080) | 20260505 19:21:23.615 | INFO | MemInfo: { | 20260505 19:21:23.615 | INFO | "gpu" : " 17 GB", | 20260505 19:21:23.615 | INFO | "huge" : "", | 20260505 19:21:23.615 | INFO | "huge_shared" : " 27 GB", | 20260505 19:21:23.615 | INFO | "malloc" : " 14 GB", | 20260505 19:21:23.615 | INFO | "pinned" : " 29 GB" | 20260505 19:21:23.615 | INFO | } | 20260505 19:21:23.616 | INFO | Perf report for the last 5.13716 seconds: | 20260505 19:21:23.616 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:21:23.616 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:23.616 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:23.616 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:23.616 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 19:21:23.616 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:23.616 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:23.616 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 19:21:23.616 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:23.616 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:23.616 | INFO | Uploaded 36GiB to GPU in 2.14973s (16.7463GiB/s) | 20260505 19:21:23.616 | INFO | Downloaded 34GiB to GPU in 2.04848s (16.5977GiB/s) | 20260505 19:21:28.755 | INFO | Analyzing chunk frames = [5678080,5678592) | 20260505 19:21:28.756 | INFO | MemInfo: { | 20260505 19:21:28.756 | INFO | "gpu" : " 17 GB", | 20260505 19:21:28.756 | INFO | "huge" : "", | 20260505 19:21:28.756 | INFO | "huge_shared" : " 27 GB", | 20260505 19:21:28.756 | INFO | "malloc" : " 14 GB", | 20260505 19:21:28.756 | INFO | "pinned" : " 29 GB" | 20260505 19:21:28.756 | INFO | } | 20260505 19:21:28.756 | INFO | Perf report for the last 5.14028 seconds: | 20260505 19:21:28.756 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:21:28.756 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:28.756 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:28.756 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:28.756 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260505 19:21:28.756 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:28.756 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:28.756 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 19:21:28.756 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:28.756 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:28.756 | INFO | Uploaded 36GiB to GPU in 2.14175s (16.8087GiB/s) | 20260505 19:21:28.756 | INFO | Downloaded 34GiB to GPU in 2.03966s (16.6694GiB/s) | 20260505 19:21:33.894 | INFO | Analyzing chunk frames = [5678592,5679104) | 20260505 19:21:33.894 | INFO | MemInfo: { | 20260505 19:21:33.894 | INFO | "gpu" : " 17 GB", | 20260505 19:21:33.894 | INFO | "huge" : "", | 20260505 19:21:33.894 | INFO | "huge_shared" : " 27 GB", | 20260505 19:21:33.894 | INFO | "malloc" : " 14 GB", | 20260505 19:21:33.894 | INFO | "pinned" : " 29 GB" | 20260505 19:21:33.894 | INFO | } | 20260505 19:21:33.894 | INFO | Perf report for the last 5.13859 seconds: | 20260505 19:21:33.894 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:21:33.894 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:33.894 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:33.894 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:33.894 | INFO | ANALYSIS : 62.3%, 3.0, 96.7ms, 0 | 20260505 19:21:33.894 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:33.894 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:33.895 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 19:21:33.895 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:33.895 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:33.895 | INFO | Uploaded 36GiB to GPU in 2.18134s (16.5036GiB/s) | 20260505 19:21:33.895 | INFO | Downloaded 34GiB to GPU in 2.05539s (16.5418GiB/s) | 20260505 19:21:39.037 | INFO | Analyzing chunk frames = [5679104,5679616) | 20260505 19:21:39.037 | INFO | MemInfo: { | 20260505 19:21:39.037 | INFO | "gpu" : " 17 GB", | 20260505 19:21:39.037 | INFO | "huge" : "", | 20260505 19:21:39.037 | INFO | "huge_shared" : " 27 GB", | 20260505 19:21:39.037 | INFO | "malloc" : " 14 GB", | 20260505 19:21:39.037 | INFO | "pinned" : " 29 GB" | 20260505 19:21:39.037 | INFO | } | 20260505 19:21:39.037 | INFO | Perf report for the last 5.14235 seconds: | 20260505 19:21:39.037 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:21:39.037 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:39.037 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:39.037 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:39.037 | INFO | ANALYSIS : 61.8%, 3.0, 96.6ms, 0 | 20260505 19:21:39.037 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:39.037 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:39.037 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 19:21:39.037 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:39.037 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:39.037 | INFO | Uploaded 36GiB to GPU in 2.13175s (16.8875GiB/s) | 20260505 19:21:39.037 | INFO | Downloaded 34GiB to GPU in 2.06243s (16.4854GiB/s) | 20260505 19:21:44.176 | INFO | Analyzing chunk frames = [5679616,5680128) | 20260505 19:21:44.176 | INFO | MemInfo: { | 20260505 19:21:44.176 | INFO | "gpu" : " 17 GB", | 20260505 19:21:44.176 | INFO | "huge" : "", | 20260505 19:21:44.176 | INFO | "huge_shared" : " 27 GB", | 20260505 19:21:44.176 | INFO | "malloc" : " 14 GB", | 20260505 19:21:44.176 | INFO | "pinned" : " 29 GB" | 20260505 19:21:44.176 | INFO | } | 20260505 19:21:44.176 | INFO | Perf report for the last 5.13965 seconds: | 20260505 19:21:44.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:21:44.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:44.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:44.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:44.176 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 19:21:44.177 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:44.177 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:44.177 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 19:21:44.177 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:44.177 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:44.177 | INFO | Uploaded 36GiB to GPU in 2.13134s (16.8908GiB/s) | 20260505 19:21:44.177 | INFO | Downloaded 34GiB to GPU in 2.04923s (16.5916GiB/s) | 20260505 19:21:49.305 | INFO | Analyzing chunk frames = [5680128,5680640) | 20260505 19:21:49.305 | INFO | MemInfo: { | 20260505 19:21:49.305 | INFO | "gpu" : " 17 GB", | 20260505 19:21:49.305 | INFO | "huge" : "", | 20260505 19:21:49.305 | INFO | "huge_shared" : " 27 GB", | 20260505 19:21:49.305 | INFO | "malloc" : " 14 GB", | 20260505 19:21:49.306 | INFO | "pinned" : " 29 GB" | 20260505 19:21:49.306 | INFO | } | 20260505 19:21:49.306 | INFO | Perf report for the last 5.12927 seconds: | 20260505 19:21:49.306 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:21:49.306 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:49.306 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:49.306 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:49.306 | INFO | ANALYSIS : 63.4%, 3.0, 98.8ms, 0 | 20260505 19:21:49.306 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:49.306 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:49.306 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 19:21:49.306 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:49.306 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:49.306 | INFO | Uploaded 36GiB to GPU in 2.13595s (16.8543GiB/s) | 20260505 19:21:49.306 | INFO | Downloaded 34GiB to GPU in 2.14295s (15.866GiB/s) | 20260505 19:21:54.454 | INFO | Analyzing chunk frames = [5680640,5681152) | 20260505 19:21:54.454 | INFO | MemInfo: { | 20260505 19:21:54.454 | INFO | "gpu" : " 17 GB", | 20260505 19:21:54.454 | INFO | "huge" : "", | 20260505 19:21:54.454 | INFO | "huge_shared" : " 27 GB", | 20260505 19:21:54.454 | INFO | "malloc" : " 14 GB", | 20260505 19:21:54.454 | INFO | "pinned" : " 29 GB" | 20260505 19:21:54.454 | INFO | } | 20260505 19:21:54.454 | INFO | Perf report for the last 5.14854 seconds: | 20260505 19:21:54.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:21:54.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:54.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:54.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:54.454 | INFO | ANALYSIS : 62.8%, 3.0, 98.3ms, 0 | 20260505 19:21:54.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:54.454 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:54.454 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 19:21:54.454 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:54.454 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:54.455 | INFO | Uploaded 36GiB to GPU in 2.18697s (16.4611GiB/s) | 20260505 19:21:54.455 | INFO | Downloaded 34GiB to GPU in 2.03125s (16.7384GiB/s) | 20260505 19:21:54.584 | INFO | Starting a round of Baz IO | 20260505 19:21:57.757 | INFO | Populating RTMetrics report | 20260505 19:21:57.757 | INFO | RTMetrics report populated | 20260505 19:21:57.757 | INFO | Opening tmp RTMetrics json file | 20260505 19:21:57.761 | INFO | Serializing RTMetrics to json | 20260505 19:21:57.764 | INFO | Serialization done, swapping tmp file now... | 20260505 19:21:57.766 | INFO | RTMetrics done | 20260505 19:21:57.850 | INFO | Finished a round of Baz IO: Wrote 2.056 GiB in 3.266 seconds | 20260505 19:21:57.850 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.79% / 0.00% / 78.21% | 20260505 19:21:57.850 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.212308% / 3.97364e-06% / 0.000107288% | 20260505 19:21:57.850 | INFO | Baz IO overall: | 20260505 19:21:57.850 | INFO | BytesWritten : 3329704183627 | 20260505 19:21:57.850 | INFO | NumEvents : 1575858542153 | 20260505 19:21:59.600 | INFO | Analyzing chunk frames = [5681152,5681664) | 20260505 19:21:59.600 | INFO | MemInfo: { | 20260505 19:21:59.600 | INFO | "gpu" : " 17 GB", | 20260505 19:21:59.600 | INFO | "huge" : "", | 20260505 19:21:59.600 | INFO | "huge_shared" : " 27 GB", | 20260505 19:21:59.600 | INFO | "malloc" : " 14 GB", | 20260505 19:21:59.600 | INFO | "pinned" : " 29 GB" | 20260505 19:21:59.600 | INFO | } | 20260505 19:21:59.601 | INFO | Perf report for the last 5.14625 seconds: | 20260505 19:21:59.601 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:21:59.601 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:59.601 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:59.601 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:59.601 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 19:21:59.601 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:21:59.601 | INFO | RT_METRICS : 44.3%, 1.0, 23.3ms, 0 | 20260505 19:21:59.601 | INFO | PRE_HQ : 60.4%, 1.8, 56.0ms, 0 | 20260505 19:21:59.601 | INFO | BAZWRITER : 63.0%, 3.8, 123.9ms, 0 | 20260505 19:21:59.601 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 19:21:59.601 | INFO | Uploaded 36GiB to GPU in 2.16611s (16.6196GiB/s) | 20260505 19:21:59.601 | INFO | Downloaded 38GiB to GPU in 2.24848s (16.9003GiB/s) | 20260505 19:22:04.734 | INFO | Analyzing chunk frames = [5681664,5682176) | 20260505 19:22:04.734 | INFO | MemInfo: { | 20260505 19:22:04.734 | INFO | "gpu" : " 17 GB", | 20260505 19:22:04.734 | INFO | "huge" : "", | 20260505 19:22:04.734 | INFO | "huge_shared" : " 27 GB", | 20260505 19:22:04.734 | INFO | "malloc" : " 14 GB", | 20260505 19:22:04.734 | INFO | "pinned" : " 29 GB" | 20260505 19:22:04.734 | INFO | } | 20260505 19:22:04.734 | INFO | Perf report for the last 5.13393 seconds: | 20260505 19:22:04.734 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:22:04.734 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:04.734 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:04.734 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:04.735 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260505 19:22:04.735 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:04.735 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:04.735 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:22:04.735 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:04.735 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:04.735 | INFO | Uploaded 36GiB to GPU in 2.09047s (17.221GiB/s) | 20260505 19:22:04.735 | INFO | Downloaded 34GiB to GPU in 2.0257s (16.7843GiB/s) | 20260505 19:22:09.863 | INFO | Analyzing chunk frames = [5682176,5682688) | 20260505 19:22:09.863 | INFO | MemInfo: { | 20260505 19:22:09.863 | INFO | "gpu" : " 17 GB", | 20260505 19:22:09.863 | INFO | "huge" : "", | 20260505 19:22:09.863 | INFO | "huge_shared" : " 27 GB", | 20260505 19:22:09.863 | INFO | "malloc" : " 14 GB", | 20260505 19:22:09.863 | INFO | "pinned" : " 29 GB" | 20260505 19:22:09.863 | INFO | } | 20260505 19:22:09.863 | INFO | Perf report for the last 5.12877 seconds: | 20260505 19:22:09.863 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:22:09.863 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:09.863 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:09.863 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:09.863 | INFO | ANALYSIS : 62.3%, 3.0, 96.7ms, 0 | 20260505 19:22:09.863 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:09.863 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:09.863 | INFO | PRE_HQ : 27.5%, 1.0, 14.2ms, 0 | 20260505 19:22:09.863 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:09.863 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:09.864 | INFO | Uploaded 36GiB to GPU in 2.12259s (16.9604GiB/s) | 20260505 19:22:09.864 | INFO | Downloaded 34GiB to GPU in 2.01761s (16.8516GiB/s) | 20260505 19:22:15.009 | INFO | Analyzing chunk frames = [5682688,5683200) | 20260505 19:22:15.009 | INFO | MemInfo: { | 20260505 19:22:15.009 | INFO | "gpu" : " 17 GB", | 20260505 19:22:15.009 | INFO | "huge" : "", | 20260505 19:22:15.009 | INFO | "huge_shared" : " 27 GB", | 20260505 19:22:15.009 | INFO | "malloc" : " 14 GB", | 20260505 19:22:15.009 | INFO | "pinned" : " 29 GB" | 20260505 19:22:15.009 | INFO | } | 20260505 19:22:15.009 | INFO | Perf report for the last 5.14574 seconds: | 20260505 19:22:15.009 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:22:15.009 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:15.009 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:15.009 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:15.009 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260505 19:22:15.009 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:15.009 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:15.009 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 19:22:15.009 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:15.009 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:15.009 | INFO | Uploaded 36GiB to GPU in 2.20183s (16.35GiB/s) | 20260505 19:22:15.009 | INFO | Downloaded 34GiB to GPU in 2.14204s (15.8727GiB/s) | 20260505 19:22:20.149 | INFO | Analyzing chunk frames = [5683200,5683712) | 20260505 19:22:20.149 | INFO | MemInfo: { | 20260505 19:22:20.149 | INFO | "gpu" : " 17 GB", | 20260505 19:22:20.149 | INFO | "huge" : "", | 20260505 19:22:20.149 | INFO | "huge_shared" : " 27 GB", | 20260505 19:22:20.149 | INFO | "malloc" : " 14 GB", | 20260505 19:22:20.149 | INFO | "pinned" : " 29 GB" | 20260505 19:22:20.149 | INFO | } | 20260505 19:22:20.150 | INFO | Perf report for the last 5.14051 seconds: | 20260505 19:22:20.150 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:22:20.150 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:20.150 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:20.150 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:20.150 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 19:22:20.150 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:20.150 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:20.150 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:22:20.150 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:20.150 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:20.150 | INFO | Uploaded 36GiB to GPU in 2.13101s (16.8934GiB/s) | 20260505 19:22:20.150 | INFO | Downloaded 34GiB to GPU in 2.05545s (16.5414GiB/s) | 20260505 19:22:25.291 | INFO | Analyzing chunk frames = [5683712,5684224) | 20260505 19:22:25.291 | INFO | MemInfo: { | 20260505 19:22:25.291 | INFO | "gpu" : " 17 GB", | 20260505 19:22:25.291 | INFO | "huge" : "", | 20260505 19:22:25.291 | INFO | "huge_shared" : " 27 GB", | 20260505 19:22:25.292 | INFO | "malloc" : " 14 GB", | 20260505 19:22:25.292 | INFO | "pinned" : " 29 GB" | 20260505 19:22:25.292 | INFO | } | 20260505 19:22:25.292 | INFO | Perf report for the last 5.1421 seconds: | 20260505 19:22:25.292 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:22:25.292 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:25.292 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:25.292 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:25.292 | INFO | ANALYSIS : 62.3%, 3.0, 97.4ms, 0 | 20260505 19:22:25.292 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:25.292 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:25.292 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 19:22:25.292 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:25.292 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:25.292 | INFO | Uploaded 36GiB to GPU in 2.15972s (16.6688GiB/s) | 20260505 19:22:25.292 | INFO | Downloaded 34GiB to GPU in 2.06352s (16.4767GiB/s) | 20260505 19:22:30.431 | INFO | Analyzing chunk frames = [5684224,5684736) | 20260505 19:22:30.431 | INFO | MemInfo: { | 20260505 19:22:30.431 | INFO | "gpu" : " 17 GB", | 20260505 19:22:30.431 | INFO | "huge" : "", | 20260505 19:22:30.431 | INFO | "huge_shared" : " 27 GB", | 20260505 19:22:30.431 | INFO | "malloc" : " 14 GB", | 20260505 19:22:30.431 | INFO | "pinned" : " 29 GB" | 20260505 19:22:30.431 | INFO | } | 20260505 19:22:30.431 | INFO | Perf report for the last 5.13969 seconds: | 20260505 19:22:30.431 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:22:30.431 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:30.431 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:30.431 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:30.431 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 19:22:30.431 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:30.431 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:30.431 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 19:22:30.431 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:30.431 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:30.432 | INFO | Uploaded 36GiB to GPU in 2.13481s (16.8634GiB/s) | 20260505 19:22:30.432 | INFO | Downloaded 34GiB to GPU in 2.13553s (15.9211GiB/s) | 20260505 19:22:35.565 | INFO | Analyzing chunk frames = [5684736,5685248) | 20260505 19:22:35.565 | INFO | MemInfo: { | 20260505 19:22:35.565 | INFO | "gpu" : " 17 GB", | 20260505 19:22:35.565 | INFO | "huge" : "", | 20260505 19:22:35.565 | INFO | "huge_shared" : " 27 GB", | 20260505 19:22:35.565 | INFO | "malloc" : " 14 GB", | 20260505 19:22:35.565 | INFO | "pinned" : " 29 GB" | 20260505 19:22:35.565 | INFO | } | 20260505 19:22:35.565 | INFO | Perf report for the last 5.13381 seconds: | 20260505 19:22:35.565 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:22:35.565 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:35.565 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:35.565 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:35.565 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 19:22:35.565 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:35.565 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:35.565 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 19:22:35.565 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:35.565 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:35.565 | INFO | Uploaded 36GiB to GPU in 2.1913s (16.4286GiB/s) | 20260505 19:22:35.566 | INFO | Downloaded 34GiB to GPU in 2.04554s (16.6215GiB/s) | 20260505 19:22:35.566 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 19:22:35.566 | INFO | Total mem usage: 15.3 GiB | 20260505 19:22:35.566 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 19:22:35.566 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.17%) | 20260505 19:22:35.566 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 19:22:35.566 | INFO | High Water Breakdown: | 20260505 19:22:35.566 | INFO | 61% : 8.64 GiB ( 90.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 19:22:35.566 | INFO | 38.9% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 19:22:35.566 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 19:22:35.566 | INFO | ------------------------------------------ | 20260505 19:22:35.566 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:22:35.566 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 19:22:35.566 | INFO | Total mem usage: 26.5 GiB | 20260505 19:22:35.566 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 19:22:35.566 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 19:22:35.566 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 19:22:35.566 | INFO | High Water Breakdown: | 20260505 19:22:35.566 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:22:35.566 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:22:35.566 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:22:35.566 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:22:35.566 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:22:35.566 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:22:35.566 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 19:22:35.566 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 19:22:35.566 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:22:35.566 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 19:22:35.566 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 19:22:35.566 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 19:22:35.566 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 19:22:35.566 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 19:22:35.566 | INFO | ------------------------------------------ | 20260505 19:22:35.566 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:22:35.566 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 19:22:35.566 | INFO | Total mem usage: 25.6 GiB | 20260505 19:22:35.566 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 19:22:35.566 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 19:22:35.566 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 19:22:35.566 | INFO | High Water Breakdown: | 20260505 19:22:35.566 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 19:22:35.566 | INFO | ------------------------------------------ | 20260505 19:22:35.566 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:22:35.566 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 19:22:35.566 | INFO | Total mem usage: 15.7 GiB | 20260505 19:22:35.566 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 19:22:35.566 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 19:22:35.566 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 19:22:35.566 | INFO | High Water Breakdown: | 20260505 19:22:35.566 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 19:22:35.566 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 19:22:35.566 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 19:22:35.566 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 19:22:35.566 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 19:22:35.566 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 19:22:35.566 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 19:22:35.566 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 19:22:35.566 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 19:22:35.566 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:22:35.566 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 19:22:35.566 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:22:35.566 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 19:22:35.567 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 19:22:35.567 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 19:22:35.567 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 19:22:35.567 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:22:35.567 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 19:22:35.567 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:22:35.567 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:22:35.567 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:22:35.567 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 19:22:35.567 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 19:22:35.567 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:22:35.567 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 19:22:35.567 | INFO | ------------------------------------------ | 20260505 19:22:35.567 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:22:35.567 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 19:22:35.567 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:22:35.567 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:22:35.567 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.49e+01, 1.37e+01 / 2.25e+00 | 20260505 19:22:35.567 | INFO | PerfReport: 0.29%, AggregateStats -- 9.27e-02 / 1.03e-01, 1.02e-02 / 3.23e-03 | 20260505 19:22:35.567 | INFO | PerfReport: 11.81%, Baseline -- 3.72e+00 / 4.54e+00, 5.68e-01 / 1.49e-01 | 20260505 19:22:35.567 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.55e+00, 1.93e-01 / 7.29e-02 | 20260505 19:22:35.567 | INFO | PerfReport: 15.98%, DME -- 4.03e+01 / 4.48e+01, 4.15e+00 / 2.26e+00 | 20260505 19:22:35.567 | INFO | PerfReport: 56.92%, FrameLabeling -- 1.79e+01 / 1.93e+01, 1.45e+00 / 1.18e+00 | 20260505 19:22:35.567 | INFO | PerfReport: 7.27%, PulseAccumulating -- 2.29e+00 / 2.87e+00, 4.38e-01 / 2.01e-01 | 20260505 19:22:35.567 | INFO | PerfReport: 3.30%, Metrics -- 1.04e+00 / 1.38e+00, 2.87e-01 / 1.39e-01 | 20260505 19:22:35.567 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.61e-01 / 4.22e-01, 6.38e-02 / 3.66e-02 | 20260505 19:22:35.567 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 19:22:35.567 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:22:35.567 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:22:35.567 | INFO | PerfReport: Full Execution -- 9.96e+01 / -nan, 2.54e+01 / 0.00e+00 | 20260505 19:22:35.567 | INFO | PerfReport: 22.50%, Upload -- 2.24e+01 / 0.00e+00, 7.74e+00 / 0.00e+00 | 20260505 19:22:35.567 | INFO | PerfReport: 21.85%, Download -- 2.18e+01 / 0.00e+00, 7.71e+00 / 0.00e+00 | 20260505 19:22:35.567 | INFO | PerfReport: 31.66%, Compute -- 3.15e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 19:22:35.567 | INFO | PerfReport: 2.22%, LockUpload -- 2.22e+00 / 0.00e+00, 7.82e+00 / 0.00e+00 | 20260505 19:22:35.567 | INFO | PerfReport: 1.41%, LockDownload -- 1.40e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260505 19:22:35.567 | INFO | PerfReport: 20.34%, LockCompute -- 2.03e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 19:22:35.567 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 2.41e-02 / 0.00e+00 | 20260505 19:22:35.700 | INFO | Starting a round of Baz IO | 20260505 19:22:38.872 | INFO | Populating RTMetrics report | 20260505 19:22:38.873 | INFO | RTMetrics report populated | 20260505 19:22:38.873 | INFO | Opening tmp RTMetrics json file | 20260505 19:22:38.874 | INFO | Serializing RTMetrics to json | 20260505 19:22:38.875 | INFO | Serialization done, swapping tmp file now... | 20260505 19:22:38.875 | INFO | RTMetrics done | 20260505 19:22:38.967 | INFO | Finished a round of Baz IO: Wrote 2.057 GiB in 3.266 seconds | 20260505 19:22:38.967 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.79% / 0.00% / 78.21% | 20260505 19:22:38.967 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.212046% / 3.97364e-06% / 0.000123183% | 20260505 19:22:38.967 | INFO | Baz IO overall: | 20260505 19:22:38.967 | INFO | BytesWritten : 3331912416918 | 20260505 19:22:38.967 | INFO | NumEvents : 1576885646777 | 20260505 19:22:40.717 | INFO | Analyzing chunk frames = [5685248,5685760) | 20260505 19:22:40.717 | INFO | MemInfo: { | 20260505 19:22:40.717 | INFO | "gpu" : " 17 GB", | 20260505 19:22:40.717 | INFO | "huge" : "", | 20260505 19:22:40.717 | INFO | "huge_shared" : " 27 GB", | 20260505 19:22:40.717 | INFO | "malloc" : " 14 GB", | 20260505 19:22:40.717 | INFO | "pinned" : " 29 GB" | 20260505 19:22:40.717 | INFO | } | 20260505 19:22:40.717 | INFO | Perf report for the last 5.15179 seconds: | 20260505 19:22:40.717 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:22:40.717 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:40.717 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:40.717 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:40.717 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 19:22:40.717 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:40.717 | INFO | RT_METRICS : 44.1%, 1.0, 23.2ms, 0 | 20260505 19:22:40.717 | INFO | PRE_HQ : 60.1%, 1.8, 55.5ms, 0 | 20260505 19:22:40.717 | INFO | BAZWRITER : 62.9%, 3.8, 122.4ms, 0 | 20260505 19:22:40.717 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 19:22:40.717 | INFO | Uploaded 36GiB to GPU in 2.16776s (16.607GiB/s) | 20260505 19:22:40.717 | INFO | Downloaded 38GiB to GPU in 2.2583s (16.8268GiB/s) | 20260505 19:22:45.846 | INFO | Analyzing chunk frames = [5685760,5686272) | 20260505 19:22:45.846 | INFO | MemInfo: { | 20260505 19:22:45.846 | INFO | "gpu" : " 17 GB", | 20260505 19:22:45.846 | INFO | "huge" : "", | 20260505 19:22:45.846 | INFO | "huge_shared" : " 27 GB", | 20260505 19:22:45.846 | INFO | "malloc" : " 14 GB", | 20260505 19:22:45.847 | INFO | "pinned" : " 29 GB" | 20260505 19:22:45.847 | INFO | } | 20260505 19:22:45.847 | INFO | Perf report for the last 5.12956 seconds: | 20260505 19:22:45.847 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:22:45.847 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:45.847 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:45.847 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:45.847 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260505 19:22:45.847 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:45.847 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:45.847 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:22:45.847 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:45.847 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:45.847 | INFO | Uploaded 36GiB to GPU in 2.0854s (17.2628GiB/s) | 20260505 19:22:45.847 | INFO | Downloaded 34GiB to GPU in 2.02823s (16.7634GiB/s) | 20260505 19:22:50.979 | INFO | Analyzing chunk frames = [5686272,5686784) | 20260505 19:22:50.979 | INFO | MemInfo: { | 20260505 19:22:50.979 | INFO | "gpu" : " 17 GB", | 20260505 19:22:50.979 | INFO | "huge" : "", | 20260505 19:22:50.980 | INFO | "huge_shared" : " 27 GB", | 20260505 19:22:50.980 | INFO | "malloc" : " 14 GB", | 20260505 19:22:50.980 | INFO | "pinned" : " 29 GB" | 20260505 19:22:50.980 | INFO | } | 20260505 19:22:50.980 | INFO | Perf report for the last 5.13321 seconds: | 20260505 19:22:50.980 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:22:50.980 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:50.980 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:50.980 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:50.980 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260505 19:22:50.980 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:50.980 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:50.980 | INFO | PRE_HQ : 27.5%, 1.0, 14.2ms, 0 | 20260505 19:22:50.980 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:50.980 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:50.980 | INFO | Uploaded 36GiB to GPU in 2.13154s (16.8892GiB/s) | 20260505 19:22:50.980 | INFO | Downloaded 34GiB to GPU in 2.02526s (16.788GiB/s) | 20260505 19:22:56.121 | INFO | Analyzing chunk frames = [5686784,5687296) | 20260505 19:22:56.121 | INFO | MemInfo: { | 20260505 19:22:56.121 | INFO | "gpu" : " 17 GB", | 20260505 19:22:56.121 | INFO | "huge" : "", | 20260505 19:22:56.122 | INFO | "huge_shared" : " 27 GB", | 20260505 19:22:56.122 | INFO | "malloc" : " 14 GB", | 20260505 19:22:56.122 | INFO | "pinned" : " 29 GB" | 20260505 19:22:56.122 | INFO | } | 20260505 19:22:56.122 | INFO | Perf report for the last 5.14198 seconds: | 20260505 19:22:56.122 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:22:56.122 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:56.122 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:56.122 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:56.122 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260505 19:22:56.122 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:56.122 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:56.122 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:22:56.122 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:56.122 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:22:56.122 | INFO | Uploaded 36GiB to GPU in 2.15911s (16.6735GiB/s) | 20260505 19:22:56.122 | INFO | Downloaded 34GiB to GPU in 2.0555s (16.541GiB/s) | 20260505 19:23:01.262 | INFO | Analyzing chunk frames = [5687296,5687808) | 20260505 19:23:01.262 | INFO | MemInfo: { | 20260505 19:23:01.262 | INFO | "gpu" : " 17 GB", | 20260505 19:23:01.262 | INFO | "huge" : "", | 20260505 19:23:01.262 | INFO | "huge_shared" : " 27 GB", | 20260505 19:23:01.262 | INFO | "malloc" : " 14 GB", | 20260505 19:23:01.262 | INFO | "pinned" : " 29 GB" | 20260505 19:23:01.262 | INFO | } | 20260505 19:23:01.262 | INFO | Perf report for the last 5.14023 seconds: | 20260505 19:23:01.262 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:23:01.262 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:01.262 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:01.262 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:01.262 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 19:23:01.262 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:01.262 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:01.262 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 19:23:01.262 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:01.262 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:01.262 | INFO | Uploaded 36GiB to GPU in 2.14545s (16.7797GiB/s) | 20260505 19:23:01.262 | INFO | Downloaded 34GiB to GPU in 2.05972s (16.5071GiB/s) | 20260505 19:23:06.407 | INFO | Analyzing chunk frames = [5687808,5688320) | 20260505 19:23:06.407 | INFO | MemInfo: { | 20260505 19:23:06.407 | INFO | "gpu" : " 17 GB", | 20260505 19:23:06.407 | INFO | "huge" : "", | 20260505 19:23:06.407 | INFO | "huge_shared" : " 27 GB", | 20260505 19:23:06.407 | INFO | "malloc" : " 14 GB", | 20260505 19:23:06.407 | INFO | "pinned" : " 29 GB" | 20260505 19:23:06.407 | INFO | } | 20260505 19:23:06.407 | INFO | Perf report for the last 5.14478 seconds: | 20260505 19:23:06.407 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:23:06.407 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:06.407 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:06.407 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:06.407 | INFO | ANALYSIS : 62.3%, 3.0, 97.4ms, 0 | 20260505 19:23:06.407 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:06.407 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:06.407 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:23:06.407 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:06.407 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:06.407 | INFO | Uploaded 36GiB to GPU in 2.16503s (16.6279GiB/s) | 20260505 19:23:06.407 | INFO | Downloaded 34GiB to GPU in 2.06851s (16.4369GiB/s) | 20260505 19:23:11.552 | INFO | Analyzing chunk frames = [5688320,5688832) | 20260505 19:23:11.552 | INFO | MemInfo: { | 20260505 19:23:11.552 | INFO | "gpu" : " 17 GB", | 20260505 19:23:11.552 | INFO | "huge" : "", | 20260505 19:23:11.552 | INFO | "huge_shared" : " 27 GB", | 20260505 19:23:11.552 | INFO | "malloc" : " 14 GB", | 20260505 19:23:11.552 | INFO | "pinned" : " 29 GB" | 20260505 19:23:11.552 | INFO | } | 20260505 19:23:11.552 | INFO | Perf report for the last 5.14496 seconds: | 20260505 19:23:11.552 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:23:11.552 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:11.552 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:11.552 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:11.552 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 19:23:11.552 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:11.552 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:11.552 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 19:23:11.552 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:11.552 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:11.552 | INFO | Uploaded 36GiB to GPU in 2.10451s (17.1061GiB/s) | 20260505 19:23:11.552 | INFO | Downloaded 34GiB to GPU in 2.13727s (15.9081GiB/s) | 20260505 19:23:16.678 | INFO | Analyzing chunk frames = [5688832,5689344) | 20260505 19:23:16.678 | INFO | MemInfo: { | 20260505 19:23:16.678 | INFO | "gpu" : " 17 GB", | 20260505 19:23:16.678 | INFO | "huge" : "", | 20260505 19:23:16.678 | INFO | "huge_shared" : " 27 GB", | 20260505 19:23:16.678 | INFO | "malloc" : " 14 GB", | 20260505 19:23:16.678 | INFO | "pinned" : " 29 GB" | 20260505 19:23:16.678 | INFO | } | 20260505 19:23:16.678 | INFO | Perf report for the last 5.12631 seconds: | 20260505 19:23:16.678 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:23:16.678 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:16.678 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:16.678 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:16.678 | INFO | ANALYSIS : 62.6%, 3.0, 97.4ms, 0 | 20260505 19:23:16.678 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:16.678 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:16.678 | INFO | PRE_HQ : 27.9%, 1.0, 14.4ms, 0 | 20260505 19:23:16.678 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:16.678 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:16.678 | INFO | Uploaded 36GiB to GPU in 2.11855s (16.9928GiB/s) | 20260505 19:23:16.678 | INFO | Downloaded 34GiB to GPU in 2.00963s (16.9185GiB/s) | 20260505 19:23:16.814 | INFO | Starting a round of Baz IO | 20260505 19:23:19.989 | INFO | Populating RTMetrics report | 20260505 19:23:19.989 | INFO | RTMetrics report populated | 20260505 19:23:19.989 | INFO | Opening tmp RTMetrics json file | 20260505 19:23:19.991 | INFO | Serializing RTMetrics to json | 20260505 19:23:19.994 | INFO | Serialization done, swapping tmp file now... | 20260505 19:23:19.994 | INFO | RTMetrics done | 20260505 19:23:20.082 | INFO | Finished a round of Baz IO: Wrote 2.054 GiB in 3.268 seconds | 20260505 19:23:20.083 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.79% / 0.00% / 78.21% | 20260505 19:23:20.083 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.210683% / 0% / 0.000119209% | 20260505 19:23:20.083 | INFO | Baz IO overall: | 20260505 19:23:20.083 | INFO | BytesWritten : 3334117739768 | 20260505 19:23:20.083 | INFO | NumEvents : 1577910766423 | 20260505 19:23:21.814 | INFO | Analyzing chunk frames = [5689344,5689856) | 20260505 19:23:21.814 | INFO | MemInfo: { | 20260505 19:23:21.814 | INFO | "gpu" : " 17 GB", | 20260505 19:23:21.814 | INFO | "huge" : "", | 20260505 19:23:21.814 | INFO | "huge_shared" : " 27 GB", | 20260505 19:23:21.814 | INFO | "malloc" : " 14 GB", | 20260505 19:23:21.814 | INFO | "pinned" : " 29 GB" | 20260505 19:23:21.814 | INFO | } | 20260505 19:23:21.814 | INFO | Perf report for the last 5.13587 seconds: | 20260505 19:23:21.814 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:23:21.814 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:21.814 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:21.814 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:21.814 | INFO | ANALYSIS : 63.4%, 3.0, 98.8ms, 0 | 20260505 19:23:21.814 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:21.814 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260505 19:23:21.814 | INFO | PRE_HQ : 60.4%, 1.8, 55.8ms, 0 | 20260505 19:23:21.814 | INFO | BAZWRITER : 63.1%, 3.8, 123.0ms, 0 | 20260505 19:23:21.814 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 19:23:21.814 | INFO | Uploaded 36GiB to GPU in 2.14813s (16.7588GiB/s) | 20260505 19:23:21.814 | INFO | Downloaded 38GiB to GPU in 2.2428s (16.9431GiB/s) | 20260505 19:23:26.964 | INFO | Analyzing chunk frames = [5689856,5690368) | 20260505 19:23:26.964 | INFO | MemInfo: { | 20260505 19:23:26.964 | INFO | "gpu" : " 17 GB", | 20260505 19:23:26.964 | INFO | "huge" : "", | 20260505 19:23:26.964 | INFO | "huge_shared" : " 27 GB", | 20260505 19:23:26.964 | INFO | "malloc" : " 14 GB", | 20260505 19:23:26.964 | INFO | "pinned" : " 29 GB" | 20260505 19:23:26.964 | INFO | } | 20260505 19:23:26.964 | INFO | Perf report for the last 5.15008 seconds: | 20260505 19:23:26.964 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:23:26.964 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:26.964 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:26.964 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:26.964 | INFO | ANALYSIS : 62.5%, 3.0, 97.8ms, 0 | 20260505 19:23:26.964 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:26.964 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:26.964 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:23:26.964 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:26.964 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:26.964 | INFO | Uploaded 36GiB to GPU in 2.13913s (16.8293GiB/s) | 20260505 19:23:26.964 | INFO | Downloaded 34GiB to GPU in 2.03412s (16.7148GiB/s) | 20260505 19:23:32.108 | INFO | Analyzing chunk frames = [5690368,5690880) | 20260505 19:23:32.108 | INFO | MemInfo: { | 20260505 19:23:32.108 | INFO | "gpu" : " 17 GB", | 20260505 19:23:32.108 | INFO | "huge" : "", | 20260505 19:23:32.108 | INFO | "huge_shared" : " 27 GB", | 20260505 19:23:32.108 | INFO | "malloc" : " 14 GB", | 20260505 19:23:32.108 | INFO | "pinned" : " 29 GB" | 20260505 19:23:32.108 | INFO | } | 20260505 19:23:32.108 | INFO | Perf report for the last 5.144 seconds: | 20260505 19:23:32.108 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:23:32.108 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:32.108 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:32.108 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:32.108 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260505 19:23:32.108 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:32.108 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:32.108 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 19:23:32.108 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:32.108 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:32.108 | INFO | Uploaded 36GiB to GPU in 2.12116s (16.9718GiB/s) | 20260505 19:23:32.108 | INFO | Downloaded 34GiB to GPU in 2.01065s (16.91GiB/s) | 20260505 19:23:37.241 | INFO | Analyzing chunk frames = [5690880,5691392) | 20260505 19:23:37.241 | INFO | MemInfo: { | 20260505 19:23:37.241 | INFO | "gpu" : " 17 GB", | 20260505 19:23:37.241 | INFO | "huge" : "", | 20260505 19:23:37.241 | INFO | "huge_shared" : " 27 GB", | 20260505 19:23:37.241 | INFO | "malloc" : " 14 GB", | 20260505 19:23:37.242 | INFO | "pinned" : " 29 GB" | 20260505 19:23:37.242 | INFO | } | 20260505 19:23:37.242 | INFO | Perf report for the last 5.13354 seconds: | 20260505 19:23:37.242 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:23:37.242 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:37.242 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:37.242 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:37.242 | INFO | ANALYSIS : 62.2%, 3.0, 96.5ms, 0 | 20260505 19:23:37.242 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:37.242 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:37.242 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:23:37.242 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:37.242 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:37.242 | INFO | Uploaded 36GiB to GPU in 2.1298s (16.903GiB/s) | 20260505 19:23:37.242 | INFO | Downloaded 34GiB to GPU in 2.04512s (16.625GiB/s) | 20260505 19:23:42.379 | INFO | Analyzing chunk frames = [5691392,5691904) | 20260505 19:23:42.379 | INFO | MemInfo: { | 20260505 19:23:42.379 | INFO | "gpu" : " 17 GB", | 20260505 19:23:42.379 | INFO | "huge" : "", | 20260505 19:23:42.379 | INFO | "huge_shared" : " 27 GB", | 20260505 19:23:42.379 | INFO | "malloc" : " 14 GB", | 20260505 19:23:42.379 | INFO | "pinned" : " 29 GB" | 20260505 19:23:42.379 | INFO | } | 20260505 19:23:42.379 | INFO | Perf report for the last 5.13735 seconds: | 20260505 19:23:42.379 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:23:42.379 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:42.379 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:42.379 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:42.379 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 19:23:42.379 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:42.379 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:42.379 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:23:42.379 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:42.379 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:42.379 | INFO | Uploaded 36GiB to GPU in 2.13162s (16.8885GiB/s) | 20260505 19:23:42.379 | INFO | Downloaded 34GiB to GPU in 2.062s (16.4888GiB/s) | 20260505 19:23:47.518 | INFO | Analyzing chunk frames = [5691904,5692416) | 20260505 19:23:47.518 | INFO | MemInfo: { | 20260505 19:23:47.518 | INFO | "gpu" : " 17 GB", | 20260505 19:23:47.518 | INFO | "huge" : "", | 20260505 19:23:47.518 | INFO | "huge_shared" : " 27 GB", | 20260505 19:23:47.518 | INFO | "malloc" : " 14 GB", | 20260505 19:23:47.518 | INFO | "pinned" : " 29 GB" | 20260505 19:23:47.518 | INFO | } | 20260505 19:23:47.518 | INFO | Perf report for the last 5.13884 seconds: | 20260505 19:23:47.518 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:23:47.518 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:47.518 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:47.518 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:47.518 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 19:23:47.518 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:47.518 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:47.518 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:23:47.518 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:47.518 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:47.518 | INFO | Uploaded 36GiB to GPU in 2.13989s (16.8233GiB/s) | 20260505 19:23:47.518 | INFO | Downloaded 34GiB to GPU in 2.05318s (16.5597GiB/s) | 20260505 19:23:52.657 | INFO | Analyzing chunk frames = [5692416,5692928) | 20260505 19:23:52.657 | INFO | MemInfo: { | 20260505 19:23:52.657 | INFO | "gpu" : " 17 GB", | 20260505 19:23:52.657 | INFO | "huge" : "", | 20260505 19:23:52.657 | INFO | "huge_shared" : " 27 GB", | 20260505 19:23:52.657 | INFO | "malloc" : " 14 GB", | 20260505 19:23:52.657 | INFO | "pinned" : " 29 GB" | 20260505 19:23:52.657 | INFO | } | 20260505 19:23:52.657 | INFO | Perf report for the last 5.13942 seconds: | 20260505 19:23:52.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:23:52.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:52.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:52.657 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:52.657 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 19:23:52.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:52.657 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:52.657 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 19:23:52.657 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:52.657 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:52.658 | INFO | Uploaded 36GiB to GPU in 2.12553s (16.937GiB/s) | 20260505 19:23:52.658 | INFO | Downloaded 34GiB to GPU in 2.1361s (15.9169GiB/s) | 20260505 19:23:57.800 | INFO | Analyzing chunk frames = [5692928,5693440) | 20260505 19:23:57.800 | INFO | MemInfo: { | 20260505 19:23:57.800 | INFO | "gpu" : " 17 GB", | 20260505 19:23:57.800 | INFO | "huge" : "", | 20260505 19:23:57.800 | INFO | "huge_shared" : " 27 GB", | 20260505 19:23:57.800 | INFO | "malloc" : " 14 GB", | 20260505 19:23:57.800 | INFO | "pinned" : " 29 GB" | 20260505 19:23:57.800 | INFO | } | 20260505 19:23:57.800 | INFO | Perf report for the last 5.14318 seconds: | 20260505 19:23:57.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:23:57.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:57.801 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:57.801 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:57.801 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260505 19:23:57.801 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:57.801 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:57.801 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 19:23:57.801 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:57.801 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:23:57.801 | INFO | Uploaded 36GiB to GPU in 2.11979s (16.9828GiB/s) | 20260505 19:23:57.801 | INFO | Downloaded 34GiB to GPU in 2.00915s (16.9226GiB/s) | 20260505 19:23:57.801 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 19:23:57.801 | INFO | Total mem usage: 15.3 GiB | 20260505 19:23:57.801 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 19:23:57.801 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.08%) | 20260505 19:23:57.801 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 19:23:57.801 | INFO | High Water Breakdown: | 20260505 19:23:57.801 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 19:23:57.801 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 19:23:57.801 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 19:23:57.801 | INFO | ------------------------------------------ | 20260505 19:23:57.801 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:23:57.801 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 19:23:57.801 | INFO | Total mem usage: 26.5 GiB | 20260505 19:23:57.801 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 19:23:57.801 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 19:23:57.801 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 19:23:57.801 | INFO | High Water Breakdown: | 20260505 19:23:57.801 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:23:57.801 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:23:57.801 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:23:57.801 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:23:57.801 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:23:57.801 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:23:57.801 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 19:23:57.801 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 19:23:57.801 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:23:57.801 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 19:23:57.801 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 19:23:57.801 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 19:23:57.801 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 19:23:57.801 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 19:23:57.801 | INFO | ------------------------------------------ | 20260505 19:23:57.801 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:23:57.801 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 19:23:57.802 | INFO | Total mem usage: 25.6 GiB | 20260505 19:23:57.802 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 19:23:57.802 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 19:23:57.802 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 19:23:57.802 | INFO | High Water Breakdown: | 20260505 19:23:57.802 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 19:23:57.802 | INFO | ------------------------------------------ | 20260505 19:23:57.802 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:23:57.802 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 19:23:57.802 | INFO | Total mem usage: 15.7 GiB | 20260505 19:23:57.802 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 19:23:57.802 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 19:23:57.802 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 19:23:57.802 | INFO | High Water Breakdown: | 20260505 19:23:57.802 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 19:23:57.802 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 19:23:57.802 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 19:23:57.802 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 19:23:57.802 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 19:23:57.802 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 19:23:57.802 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 19:23:57.802 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 19:23:57.802 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 19:23:57.802 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:23:57.802 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 19:23:57.802 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:23:57.802 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 19:23:57.802 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 19:23:57.802 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 19:23:57.802 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 19:23:57.802 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:23:57.802 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 19:23:57.802 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:23:57.802 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:23:57.802 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:23:57.802 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 19:23:57.802 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 19:23:57.802 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:23:57.802 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 19:23:57.802 | INFO | ------------------------------------------ | 20260505 19:23:57.802 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:23:57.802 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 19:23:57.802 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:23:57.802 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:23:57.802 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.50e+01, 1.37e+01 / 2.06e+00 | 20260505 19:23:57.802 | INFO | PerfReport: 0.29%, AggregateStats -- 9.24e-02 / 1.02e-01, 1.02e-02 / 4.95e-03 | 20260505 19:23:57.802 | INFO | PerfReport: 11.78%, Baseline -- 3.72e+00 / 4.51e+00, 5.75e-01 / 1.45e-01 | 20260505 19:23:57.802 | INFO | PerfReport: 3.29%, Histogram -- 1.04e+00 / 1.53e+00, 1.96e-01 / 9.39e-02 | 20260505 19:23:57.802 | INFO | PerfReport: 15.96%, DME -- 4.03e+01 / 4.48e+01, 4.08e+00 / 2.25e+00 | 20260505 19:23:57.802 | INFO | PerfReport: 57.03%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.55e+00 / 1.22e+00 | 20260505 19:23:57.802 | INFO | PerfReport: 7.22%, PulseAccumulating -- 2.28e+00 / 2.87e+00, 4.36e-01 / 1.92e-01 | 20260505 19:23:57.802 | INFO | PerfReport: 3.28%, Metrics -- 1.04e+00 / 1.38e+00, 2.88e-01 / 1.50e-01 | 20260505 19:23:57.802 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.61e-01 / 4.23e-01, 6.46e-02 / 3.48e-02 | 20260505 19:23:57.802 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 19:23:57.802 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:23:57.802 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:23:57.802 | INFO | PerfReport: Full Execution -- 9.95e+01 / -nan, 2.55e+01 / 0.00e+00 | 20260505 19:23:57.802 | INFO | PerfReport: 22.34%, Upload -- 2.22e+01 / 0.00e+00, 7.66e+00 / 0.00e+00 | 20260505 19:23:57.802 | INFO | PerfReport: 21.75%, Download -- 2.16e+01 / 0.00e+00, 7.70e+00 / 0.00e+00 | 20260505 19:23:57.802 | INFO | PerfReport: 31.76%, Compute -- 3.16e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 19:23:57.802 | INFO | PerfReport: 2.23%, LockUpload -- 2.22e+00 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260505 19:23:57.802 | INFO | PerfReport: 1.39%, LockDownload -- 1.38e+00 / 0.00e+00, 3.02e+00 / 0.00e+00 | 20260505 19:23:57.802 | INFO | PerfReport: 20.52%, LockCompute -- 2.04e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 19:23:57.802 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.76e-02 / 0.00e+00 | 20260505 19:23:57.930 | INFO | Starting a round of Baz IO | 20260505 19:24:01.101 | INFO | Populating RTMetrics report | 20260505 19:24:01.102 | INFO | RTMetrics report populated | 20260505 19:24:01.102 | INFO | Opening tmp RTMetrics json file | 20260505 19:24:01.104 | INFO | Serializing RTMetrics to json | 20260505 19:24:01.106 | INFO | Serialization done, swapping tmp file now... | 20260505 19:24:01.107 | INFO | RTMetrics done | 20260505 19:24:01.196 | INFO | Finished a round of Baz IO: Wrote 2.053 GiB in 3.265 seconds | 20260505 19:24:01.196 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.78% / 0.00% / 78.21% | 20260505 19:24:01.196 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.209578% / 0% / 7.15256e-05% | 20260505 19:24:01.196 | INFO | Baz IO overall: | 20260505 19:24:01.196 | INFO | BytesWritten : 3336322172775 | 20260505 19:24:01.196 | INFO | NumEvents : 1578935183272 | 20260505 19:24:02.930 | INFO | Analyzing chunk frames = [5693440,5693952) | 20260505 19:24:02.930 | INFO | MemInfo: { | 20260505 19:24:02.930 | INFO | "gpu" : " 17 GB", | 20260505 19:24:02.930 | INFO | "huge" : "", | 20260505 19:24:02.930 | INFO | "huge_shared" : " 27 GB", | 20260505 19:24:02.930 | INFO | "malloc" : " 14 GB", | 20260505 19:24:02.930 | INFO | "pinned" : " 29 GB" | 20260505 19:24:02.930 | INFO | } | 20260505 19:24:02.930 | INFO | Perf report for the last 5.12962 seconds: | 20260505 19:24:02.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:24:02.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:02.930 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:02.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:02.930 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 19:24:02.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:02.930 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260505 19:24:02.930 | INFO | PRE_HQ : 60.2%, 1.8, 55.7ms, 0 | 20260505 19:24:02.930 | INFO | BAZWRITER : 63.1%, 3.8, 122.9ms, 0 | 20260505 19:24:02.930 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 19:24:02.930 | INFO | Uploaded 36GiB to GPU in 2.17129s (16.58GiB/s) | 20260505 19:24:02.930 | INFO | Downloaded 38GiB to GPU in 2.24718s (16.9101GiB/s) | 20260505 19:24:08.068 | INFO | Analyzing chunk frames = [5693952,5694464) | 20260505 19:24:08.068 | INFO | MemInfo: { | 20260505 19:24:08.068 | INFO | "gpu" : " 17 GB", | 20260505 19:24:08.068 | INFO | "huge" : "", | 20260505 19:24:08.068 | INFO | "huge_shared" : " 27 GB", | 20260505 19:24:08.068 | INFO | "malloc" : " 14 GB", | 20260505 19:24:08.068 | INFO | "pinned" : " 29 GB" | 20260505 19:24:08.068 | INFO | } | 20260505 19:24:08.068 | INFO | Perf report for the last 5.13784 seconds: | 20260505 19:24:08.068 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:24:08.068 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:08.068 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:08.068 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:08.068 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260505 19:24:08.068 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:08.068 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:08.068 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260505 19:24:08.068 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:08.068 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:08.068 | INFO | Uploaded 36GiB to GPU in 2.13291s (16.8784GiB/s) | 20260505 19:24:08.068 | INFO | Downloaded 34GiB to GPU in 2.02807s (16.7647GiB/s) | 20260505 19:24:13.216 | INFO | Analyzing chunk frames = [5694464,5694976) | 20260505 19:24:13.216 | INFO | MemInfo: { | 20260505 19:24:13.216 | INFO | "gpu" : " 17 GB", | 20260505 19:24:13.216 | INFO | "huge" : "", | 20260505 19:24:13.216 | INFO | "huge_shared" : " 27 GB", | 20260505 19:24:13.216 | INFO | "malloc" : " 14 GB", | 20260505 19:24:13.216 | INFO | "pinned" : " 29 GB" | 20260505 19:24:13.216 | INFO | } | 20260505 19:24:13.216 | INFO | Perf report for the last 5.14825 seconds: | 20260505 19:24:13.216 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:24:13.216 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:13.216 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:13.216 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:13.216 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 19:24:13.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:13.216 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:13.216 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:24:13.216 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:13.216 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:13.216 | INFO | Uploaded 36GiB to GPU in 2.21214s (16.2739GiB/s) | 20260505 19:24:13.217 | INFO | Downloaded 34GiB to GPU in 2.07497s (16.3858GiB/s) | 20260505 19:24:18.359 | INFO | Analyzing chunk frames = [5694976,5695488) | 20260505 19:24:18.359 | INFO | MemInfo: { | 20260505 19:24:18.359 | INFO | "gpu" : " 17 GB", | 20260505 19:24:18.359 | INFO | "huge" : "", | 20260505 19:24:18.359 | INFO | "huge_shared" : " 27 GB", | 20260505 19:24:18.360 | INFO | "malloc" : " 14 GB", | 20260505 19:24:18.360 | INFO | "pinned" : " 29 GB" | 20260505 19:24:18.360 | INFO | } | 20260505 19:24:18.360 | INFO | Perf report for the last 5.14346 seconds: | 20260505 19:24:18.360 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:24:18.360 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:18.360 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:18.360 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:18.360 | INFO | ANALYSIS : 62.4%, 3.0, 97.0ms, 0 | 20260505 19:24:18.360 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:18.360 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:18.360 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 19:24:18.360 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:18.360 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:18.360 | INFO | Uploaded 36GiB to GPU in 2.20153s (16.3523GiB/s) | 20260505 19:24:18.360 | INFO | Downloaded 34GiB to GPU in 2.03741s (16.6879GiB/s) | 20260505 19:24:23.496 | INFO | Analyzing chunk frames = [5695488,5696000) | 20260505 19:24:23.496 | INFO | MemInfo: { | 20260505 19:24:23.496 | INFO | "gpu" : " 17 GB", | 20260505 19:24:23.496 | INFO | "huge" : "", | 20260505 19:24:23.496 | INFO | "huge_shared" : " 27 GB", | 20260505 19:24:23.496 | INFO | "malloc" : " 14 GB", | 20260505 19:24:23.496 | INFO | "pinned" : " 29 GB" | 20260505 19:24:23.496 | INFO | } | 20260505 19:24:23.496 | INFO | Perf report for the last 5.13614 seconds: | 20260505 19:24:23.496 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:24:23.496 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:23.496 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:23.496 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:23.496 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260505 19:24:23.496 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:23.496 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:23.496 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:24:23.496 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:23.496 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:23.496 | INFO | Uploaded 36GiB to GPU in 2.12372s (16.9514GiB/s) | 20260505 19:24:23.496 | INFO | Downloaded 34GiB to GPU in 2.06161s (16.492GiB/s) | 20260505 19:24:28.622 | INFO | Analyzing chunk frames = [5696000,5696512) | 20260505 19:24:28.622 | INFO | MemInfo: { | 20260505 19:24:28.622 | INFO | "gpu" : " 17 GB", | 20260505 19:24:28.622 | INFO | "huge" : "", | 20260505 19:24:28.622 | INFO | "huge_shared" : " 27 GB", | 20260505 19:24:28.622 | INFO | "malloc" : " 14 GB", | 20260505 19:24:28.622 | INFO | "pinned" : " 29 GB" | 20260505 19:24:28.622 | INFO | } | 20260505 19:24:28.622 | INFO | Perf report for the last 5.12603 seconds: | 20260505 19:24:28.622 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:24:28.622 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:28.622 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:28.622 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:28.622 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260505 19:24:28.622 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:28.622 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:28.622 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:24:28.622 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:28.622 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:28.622 | INFO | Uploaded 36GiB to GPU in 2.13986s (16.8235GiB/s) | 20260505 19:24:28.622 | INFO | Downloaded 34GiB to GPU in 2.05434s (16.5503GiB/s) | 20260505 19:24:33.771 | INFO | Analyzing chunk frames = [5696512,5697024) | 20260505 19:24:33.772 | INFO | MemInfo: { | 20260505 19:24:33.772 | INFO | "gpu" : " 17 GB", | 20260505 19:24:33.772 | INFO | "huge" : "", | 20260505 19:24:33.772 | INFO | "huge_shared" : " 27 GB", | 20260505 19:24:33.772 | INFO | "malloc" : " 14 GB", | 20260505 19:24:33.772 | INFO | "pinned" : " 29 GB" | 20260505 19:24:33.772 | INFO | } | 20260505 19:24:33.772 | INFO | Perf report for the last 5.14996 seconds: | 20260505 19:24:33.772 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:24:33.772 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:33.772 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:33.772 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:33.772 | INFO | ANALYSIS : 63.1%, 3.0, 98.8ms, 0 | 20260505 19:24:33.772 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:33.772 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:33.772 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 19:24:33.772 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:33.772 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:33.772 | INFO | Uploaded 36GiB to GPU in 2.13698s (16.8462GiB/s) | 20260505 19:24:33.772 | INFO | Downloaded 34GiB to GPU in 2.18057s (15.5922GiB/s) | 20260505 19:24:38.913 | INFO | Analyzing chunk frames = [5697024,5697536) | 20260505 19:24:38.913 | INFO | MemInfo: { | 20260505 19:24:38.913 | INFO | "gpu" : " 17 GB", | 20260505 19:24:38.913 | INFO | "huge" : "", | 20260505 19:24:38.913 | INFO | "huge_shared" : " 27 GB", | 20260505 19:24:38.913 | INFO | "malloc" : " 14 GB", | 20260505 19:24:38.913 | INFO | "pinned" : " 29 GB" | 20260505 19:24:38.913 | INFO | } | 20260505 19:24:38.913 | INFO | Perf report for the last 5.14095 seconds: | 20260505 19:24:38.913 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:24:38.913 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:38.913 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:38.913 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:38.913 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 19:24:38.913 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:38.913 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:38.913 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 19:24:38.913 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:38.913 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:38.913 | INFO | Uploaded 36GiB to GPU in 2.19737s (16.3832GiB/s) | 20260505 19:24:38.913 | INFO | Downloaded 34GiB to GPU in 2.03559s (16.7028GiB/s) | 20260505 19:24:39.047 | INFO | Starting a round of Baz IO | 20260505 19:24:42.211 | INFO | Populating RTMetrics report | 20260505 19:24:42.211 | INFO | RTMetrics report populated | 20260505 19:24:42.211 | INFO | Opening tmp RTMetrics json file | 20260505 19:24:42.212 | INFO | Serializing RTMetrics to json | 20260505 19:24:42.213 | INFO | Serialization done, swapping tmp file now... | 20260505 19:24:42.214 | INFO | RTMetrics done | 20260505 19:24:42.305 | INFO | Finished a round of Baz IO: Wrote 2.052 GiB in 3.257 seconds | 20260505 19:24:42.305 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.78% / 0.00% / 78.22% | 20260505 19:24:42.305 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.21023% / 3.97364e-06% / 7.54992e-05% | 20260505 19:24:42.305 | INFO | Baz IO overall: | 20260505 19:24:42.305 | INFO | BytesWritten : 3338525909617 | 20260505 19:24:42.305 | INFO | NumEvents : 1579959178934 | 20260505 19:24:44.051 | INFO | Analyzing chunk frames = [5697536,5698048) | 20260505 19:24:44.051 | INFO | MemInfo: { | 20260505 19:24:44.051 | INFO | "gpu" : " 17 GB", | 20260505 19:24:44.051 | INFO | "huge" : "", | 20260505 19:24:44.051 | INFO | "huge_shared" : " 27 GB", | 20260505 19:24:44.051 | INFO | "malloc" : " 14 GB", | 20260505 19:24:44.051 | INFO | "pinned" : " 29 GB" | 20260505 19:24:44.051 | INFO | } | 20260505 19:24:44.051 | INFO | Perf report for the last 5.1386 seconds: | 20260505 19:24:44.051 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:24:44.051 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:44.051 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:44.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:44.052 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 19:24:44.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:44.052 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260505 19:24:44.052 | INFO | PRE_HQ : 60.0%, 1.8, 55.6ms, 0 | 20260505 19:24:44.052 | INFO | BAZWRITER : 62.9%, 3.8, 122.1ms, 0 | 20260505 19:24:44.052 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 19:24:44.052 | INFO | Uploaded 36GiB to GPU in 2.16043s (16.6633GiB/s) | 20260505 19:24:44.052 | INFO | Downloaded 38GiB to GPU in 2.25726s (16.8346GiB/s) | 20260505 19:24:49.189 | INFO | Analyzing chunk frames = [5698048,5698560) | 20260505 19:24:49.189 | INFO | MemInfo: { | 20260505 19:24:49.189 | INFO | "gpu" : " 17 GB", | 20260505 19:24:49.189 | INFO | "huge" : "", | 20260505 19:24:49.189 | INFO | "huge_shared" : " 27 GB", | 20260505 19:24:49.189 | INFO | "malloc" : " 14 GB", | 20260505 19:24:49.189 | INFO | "pinned" : " 29 GB" | 20260505 19:24:49.189 | INFO | } | 20260505 19:24:49.190 | INFO | Perf report for the last 5.13806 seconds: | 20260505 19:24:49.190 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:24:49.190 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:49.190 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:49.190 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:49.190 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260505 19:24:49.190 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:49.190 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:49.190 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:24:49.190 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:49.190 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:49.190 | INFO | Uploaded 36GiB to GPU in 2.10772s (17.0801GiB/s) | 20260505 19:24:49.190 | INFO | Downloaded 34GiB to GPU in 2.03109s (16.7397GiB/s) | 20260505 19:24:54.328 | INFO | Analyzing chunk frames = [5698560,5699072) | 20260505 19:24:54.328 | INFO | MemInfo: { | 20260505 19:24:54.328 | INFO | "gpu" : " 17 GB", | 20260505 19:24:54.328 | INFO | "huge" : "", | 20260505 19:24:54.328 | INFO | "huge_shared" : " 27 GB", | 20260505 19:24:54.328 | INFO | "malloc" : " 14 GB", | 20260505 19:24:54.328 | INFO | "pinned" : " 29 GB" | 20260505 19:24:54.328 | INFO | } | 20260505 19:24:54.328 | INFO | Perf report for the last 5.13835 seconds: | 20260505 19:24:54.328 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:24:54.328 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:54.328 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:54.328 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:54.328 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260505 19:24:54.328 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:54.328 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:54.328 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:24:54.328 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:54.328 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:54.328 | INFO | Uploaded 36GiB to GPU in 2.13954s (16.8261GiB/s) | 20260505 19:24:54.328 | INFO | Downloaded 34GiB to GPU in 2.0395s (16.6707GiB/s) | 20260505 19:24:59.475 | INFO | Analyzing chunk frames = [5699072,5699584) | 20260505 19:24:59.475 | INFO | MemInfo: { | 20260505 19:24:59.475 | INFO | "gpu" : " 17 GB", | 20260505 19:24:59.475 | INFO | "huge" : "", | 20260505 19:24:59.475 | INFO | "huge_shared" : " 27 GB", | 20260505 19:24:59.475 | INFO | "malloc" : " 14 GB", | 20260505 19:24:59.475 | INFO | "pinned" : " 29 GB" | 20260505 19:24:59.475 | INFO | } | 20260505 19:24:59.475 | INFO | Perf report for the last 5.14746 seconds: | 20260505 19:24:59.475 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:24:59.475 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:59.475 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:59.475 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:59.475 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260505 19:24:59.475 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:59.475 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:59.475 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:24:59.475 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:59.475 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:24:59.476 | INFO | Uploaded 36GiB to GPU in 2.19943s (16.3679GiB/s) | 20260505 19:24:59.476 | INFO | Downloaded 34GiB to GPU in 2.07306s (16.4008GiB/s) | 20260505 19:25:04.610 | INFO | Analyzing chunk frames = [5699584,5700096) | 20260505 19:25:04.610 | INFO | MemInfo: { | 20260505 19:25:04.610 | INFO | "gpu" : " 17 GB", | 20260505 19:25:04.610 | INFO | "huge" : "", | 20260505 19:25:04.610 | INFO | "huge_shared" : " 27 GB", | 20260505 19:25:04.610 | INFO | "malloc" : " 14 GB", | 20260505 19:25:04.610 | INFO | "pinned" : " 29 GB" | 20260505 19:25:04.610 | INFO | } | 20260505 19:25:04.610 | INFO | Perf report for the last 5.13449 seconds: | 20260505 19:25:04.610 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:25:04.610 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:04.610 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:04.610 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:04.610 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260505 19:25:04.610 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:04.610 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:04.610 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:25:04.610 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:04.610 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:04.610 | INFO | Uploaded 36GiB to GPU in 2.13154s (16.8892GiB/s) | 20260505 19:25:04.610 | INFO | Downloaded 34GiB to GPU in 2.05964s (16.5077GiB/s) | 20260505 19:25:09.751 | INFO | Analyzing chunk frames = [5700096,5700608) | 20260505 19:25:09.751 | INFO | MemInfo: { | 20260505 19:25:09.751 | INFO | "gpu" : " 17 GB", | 20260505 19:25:09.751 | INFO | "huge" : "", | 20260505 19:25:09.751 | INFO | "huge_shared" : " 27 GB", | 20260505 19:25:09.752 | INFO | "malloc" : " 14 GB", | 20260505 19:25:09.752 | INFO | "pinned" : " 29 GB" | 20260505 19:25:09.752 | INFO | } | 20260505 19:25:09.752 | INFO | Perf report for the last 5.14185 seconds: | 20260505 19:25:09.752 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:25:09.752 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:09.752 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:09.752 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:09.752 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 19:25:09.752 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:09.752 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:09.752 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 19:25:09.752 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:09.752 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:09.752 | INFO | Uploaded 36GiB to GPU in 2.14458s (16.7865GiB/s) | 20260505 19:25:09.752 | INFO | Downloaded 34GiB to GPU in 2.05165s (16.572GiB/s) | 20260505 19:25:14.885 | INFO | Analyzing chunk frames = [5700608,5701120) | 20260505 19:25:14.885 | INFO | MemInfo: { | 20260505 19:25:14.885 | INFO | "gpu" : " 17 GB", | 20260505 19:25:14.885 | INFO | "huge" : "", | 20260505 19:25:14.885 | INFO | "huge_shared" : " 27 GB", | 20260505 19:25:14.885 | INFO | "malloc" : " 14 GB", | 20260505 19:25:14.885 | INFO | "pinned" : " 29 GB" | 20260505 19:25:14.885 | INFO | } | 20260505 19:25:14.885 | INFO | Perf report for the last 5.13309 seconds: | 20260505 19:25:14.885 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:25:14.885 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:14.885 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:14.885 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:14.885 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 19:25:14.885 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:14.885 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:14.885 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 19:25:14.885 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:14.885 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:14.885 | INFO | Uploaded 36GiB to GPU in 2.12175s (16.9672GiB/s) | 20260505 19:25:14.885 | INFO | Downloaded 34GiB to GPU in 2.13601s (15.9175GiB/s) | 20260505 19:25:20.025 | INFO | Analyzing chunk frames = [5701120,5701632) | 20260505 19:25:20.025 | INFO | MemInfo: { | 20260505 19:25:20.025 | INFO | "gpu" : " 17 GB", | 20260505 19:25:20.025 | INFO | "huge" : "", | 20260505 19:25:20.025 | INFO | "huge_shared" : " 27 GB", | 20260505 19:25:20.025 | INFO | "malloc" : " 14 GB", | 20260505 19:25:20.025 | INFO | "pinned" : " 29 GB" | 20260505 19:25:20.025 | INFO | } | 20260505 19:25:20.026 | INFO | Perf report for the last 5.14068 seconds: | 20260505 19:25:20.026 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:25:20.026 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:20.026 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:20.026 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:20.026 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 19:25:20.026 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:20.026 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:20.026 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:25:20.026 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:20.026 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:20.026 | INFO | Uploaded 36GiB to GPU in 2.19491s (16.4016GiB/s) | 20260505 19:25:20.026 | INFO | Downloaded 34GiB to GPU in 2.03134s (16.7377GiB/s) | 20260505 19:25:20.026 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 19:25:20.026 | INFO | Total mem usage: 15.3 GiB | 20260505 19:25:20.026 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 19:25:20.026 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.08%) | 20260505 19:25:20.026 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 19:25:20.026 | INFO | High Water Breakdown: | 20260505 19:25:20.026 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 19:25:20.026 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 19:25:20.026 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 19:25:20.026 | INFO | ------------------------------------------ | 20260505 19:25:20.026 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:25:20.026 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 19:25:20.026 | INFO | Total mem usage: 26.5 GiB | 20260505 19:25:20.026 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 19:25:20.026 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 19:25:20.026 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 19:25:20.026 | INFO | High Water Breakdown: | 20260505 19:25:20.026 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:25:20.026 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:25:20.026 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:25:20.026 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:25:20.026 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:25:20.026 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:25:20.026 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 19:25:20.026 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 19:25:20.026 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:25:20.026 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 19:25:20.026 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 19:25:20.027 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 19:25:20.027 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 19:25:20.027 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 19:25:20.027 | INFO | ------------------------------------------ | 20260505 19:25:20.027 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:25:20.027 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 19:25:20.027 | INFO | Total mem usage: 25.6 GiB | 20260505 19:25:20.027 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 19:25:20.027 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 19:25:20.027 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 19:25:20.027 | INFO | High Water Breakdown: | 20260505 19:25:20.027 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 19:25:20.027 | INFO | ------------------------------------------ | 20260505 19:25:20.027 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:25:20.027 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 19:25:20.027 | INFO | Total mem usage: 15.7 GiB | 20260505 19:25:20.027 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 19:25:20.027 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 19:25:20.027 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 19:25:20.027 | INFO | High Water Breakdown: | 20260505 19:25:20.027 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 19:25:20.027 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 19:25:20.027 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 19:25:20.027 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 19:25:20.027 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 19:25:20.027 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 19:25:20.027 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 19:25:20.027 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 19:25:20.027 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 19:25:20.027 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:25:20.027 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 19:25:20.027 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:25:20.027 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 19:25:20.027 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 19:25:20.027 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 19:25:20.027 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 19:25:20.027 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:25:20.027 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 19:25:20.027 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:25:20.027 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:25:20.027 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:25:20.027 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 19:25:20.027 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 19:25:20.027 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:25:20.027 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 19:25:20.027 | INFO | ------------------------------------------ | 20260505 19:25:20.027 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:25:20.027 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 19:25:20.027 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:25:20.027 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:25:20.027 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.47e+01, 1.34e+01 / 1.74e+00 | 20260505 19:25:20.027 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.03e-01, 1.15e-02 / 3.91e-03 | 20260505 19:25:20.027 | INFO | PerfReport: 11.78%, Baseline -- 3.71e+00 / 4.57e+00, 5.57e-01 / 1.34e-01 | 20260505 19:25:20.027 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.53e+00, 1.87e-01 / 9.10e-02 | 20260505 19:25:20.027 | INFO | PerfReport: 15.93%, DME -- 4.01e+01 / 4.43e+01, 4.12e+00 / 1.80e+00 | 20260505 19:25:20.027 | INFO | PerfReport: 57.00%, FrameLabeling -- 1.79e+01 / 1.95e+01, 1.44e+00 / 1.25e+00 | 20260505 19:25:20.027 | INFO | PerfReport: 7.29%, PulseAccumulating -- 2.30e+00 / 2.89e+00, 4.42e-01 / 1.76e-01 | 20260505 19:25:20.027 | INFO | PerfReport: 3.29%, Metrics -- 1.04e+00 / 1.39e+00, 2.84e-01 / 1.69e-01 | 20260505 19:25:20.027 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.61e-01 / 4.33e-01, 6.35e-02 / 3.90e-02 | 20260505 19:25:20.027 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 19:25:20.027 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:25:20.027 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:25:20.027 | INFO | PerfReport: Full Execution -- 9.96e+01 / -nan, 2.55e+01 / 0.00e+00 | 20260505 19:25:20.027 | INFO | PerfReport: 22.56%, Upload -- 2.25e+01 / 0.00e+00, 7.81e+00 / 0.00e+00 | 20260505 19:25:20.027 | INFO | PerfReport: 21.83%, Download -- 2.17e+01 / 0.00e+00, 7.75e+00 / 0.00e+00 | 20260505 19:25:20.027 | INFO | PerfReport: 31.60%, Compute -- 3.15e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 19:25:20.027 | INFO | PerfReport: 2.32%, LockUpload -- 2.31e+00 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260505 19:25:20.027 | INFO | PerfReport: 1.41%, LockDownload -- 1.40e+00 / 0.00e+00, 2.97e+00 / 0.00e+00 | 20260505 19:25:20.027 | INFO | PerfReport: 20.27%, LockCompute -- 2.02e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 19:25:20.027 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.81e-02 / 0.00e+00 | 20260505 19:25:20.155 | INFO | Starting a round of Baz IO | 20260505 19:25:23.331 | INFO | Populating RTMetrics report | 20260505 19:25:23.331 | INFO | RTMetrics report populated | 20260505 19:25:23.331 | INFO | Opening tmp RTMetrics json file | 20260505 19:25:23.332 | INFO | Serializing RTMetrics to json | 20260505 19:25:23.333 | INFO | Serialization done, swapping tmp file now... | 20260505 19:25:23.333 | INFO | RTMetrics done | 20260505 19:25:23.425 | INFO | Finished a round of Baz IO: Wrote 2.051 GiB in 3.269 seconds | 20260505 19:25:23.425 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.78% / 0.00% / 78.22% | 20260505 19:25:23.425 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.208735% / 3.97364e-06% / 8.74201e-05% | 20260505 19:25:23.425 | INFO | Baz IO overall: | 20260505 19:25:23.425 | INFO | BytesWritten : 3340727682375 | 20260505 19:25:23.425 | INFO | NumEvents : 1580981799628 | 20260505 19:25:25.167 | INFO | Analyzing chunk frames = [5701632,5702144) | 20260505 19:25:25.167 | INFO | MemInfo: { | 20260505 19:25:25.167 | INFO | "gpu" : " 17 GB", | 20260505 19:25:25.167 | INFO | "huge" : "", | 20260505 19:25:25.167 | INFO | "huge_shared" : " 27 GB", | 20260505 19:25:25.167 | INFO | "malloc" : " 14 GB", | 20260505 19:25:25.167 | INFO | "pinned" : " 29 GB" | 20260505 19:25:25.167 | INFO | } | 20260505 19:25:25.167 | INFO | Perf report for the last 5.14152 seconds: | 20260505 19:25:25.167 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:25:25.167 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:25.167 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:25.167 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:25.167 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 19:25:25.167 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:25.167 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260505 19:25:25.167 | INFO | PRE_HQ : 60.3%, 1.8, 55.5ms, 0 | 20260505 19:25:25.167 | INFO | BAZWRITER : 63.1%, 3.8, 122.5ms, 0 | 20260505 19:25:25.167 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 19:25:25.167 | INFO | Uploaded 36GiB to GPU in 2.15939s (16.6714GiB/s) | 20260505 19:25:25.167 | INFO | Downloaded 38GiB to GPU in 2.25401s (16.8588GiB/s) | 20260505 19:25:30.307 | INFO | Analyzing chunk frames = [5702144,5702656) | 20260505 19:25:30.307 | INFO | MemInfo: { | 20260505 19:25:30.307 | INFO | "gpu" : " 17 GB", | 20260505 19:25:30.307 | INFO | "huge" : "", | 20260505 19:25:30.307 | INFO | "huge_shared" : " 27 GB", | 20260505 19:25:30.307 | INFO | "malloc" : " 14 GB", | 20260505 19:25:30.307 | INFO | "pinned" : " 29 GB" | 20260505 19:25:30.307 | INFO | } | 20260505 19:25:30.307 | INFO | Perf report for the last 5.13995 seconds: | 20260505 19:25:30.307 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:25:30.307 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:30.307 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:30.307 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:30.307 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260505 19:25:30.307 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:30.307 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:30.307 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:25:30.307 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:30.307 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:30.307 | INFO | Uploaded 36GiB to GPU in 2.09328s (17.1979GiB/s) | 20260505 19:25:30.307 | INFO | Downloaded 34GiB to GPU in 2.02812s (16.7643GiB/s) | 20260505 19:25:35.441 | INFO | Analyzing chunk frames = [5702656,5703168) | 20260505 19:25:35.441 | INFO | MemInfo: { | 20260505 19:25:35.441 | INFO | "gpu" : " 17 GB", | 20260505 19:25:35.442 | INFO | "huge" : "", | 20260505 19:25:35.442 | INFO | "huge_shared" : " 27 GB", | 20260505 19:25:35.442 | INFO | "malloc" : " 14 GB", | 20260505 19:25:35.442 | INFO | "pinned" : " 29 GB" | 20260505 19:25:35.442 | INFO | } | 20260505 19:25:35.442 | INFO | Perf report for the last 5.13471 seconds: | 20260505 19:25:35.442 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:25:35.442 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:35.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:35.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:35.442 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260505 19:25:35.442 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:35.442 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:35.442 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:25:35.442 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:35.442 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:35.442 | INFO | Uploaded 36GiB to GPU in 2.13612s (16.853GiB/s) | 20260505 19:25:35.442 | INFO | Downloaded 34GiB to GPU in 2.03021s (16.7471GiB/s) | 20260505 19:25:40.577 | INFO | Analyzing chunk frames = [5703168,5703680) | 20260505 19:25:40.577 | INFO | MemInfo: { | 20260505 19:25:40.577 | INFO | "gpu" : " 17 GB", | 20260505 19:25:40.577 | INFO | "huge" : "", | 20260505 19:25:40.577 | INFO | "huge_shared" : " 27 GB", | 20260505 19:25:40.577 | INFO | "malloc" : " 14 GB", | 20260505 19:25:40.577 | INFO | "pinned" : " 29 GB" | 20260505 19:25:40.577 | INFO | } | 20260505 19:25:40.577 | INFO | Perf report for the last 5.13545 seconds: | 20260505 19:25:40.577 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:25:40.577 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:40.577 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:40.577 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:40.577 | INFO | ANALYSIS : 62.8%, 3.0, 97.5ms, 0 | 20260505 19:25:40.577 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:40.577 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:40.577 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 19:25:40.577 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:40.577 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:40.577 | INFO | Uploaded 36GiB to GPU in 2.22292s (16.1949GiB/s) | 20260505 19:25:40.578 | INFO | Downloaded 34GiB to GPU in 2.12154s (16.0261GiB/s) | 20260505 19:25:45.723 | INFO | Analyzing chunk frames = [5703680,5704192) | 20260505 19:25:45.723 | INFO | MemInfo: { | 20260505 19:25:45.723 | INFO | "gpu" : " 17 GB", | 20260505 19:25:45.723 | INFO | "huge" : "", | 20260505 19:25:45.723 | INFO | "huge_shared" : " 27 GB", | 20260505 19:25:45.723 | INFO | "malloc" : " 14 GB", | 20260505 19:25:45.723 | INFO | "pinned" : " 29 GB" | 20260505 19:25:45.723 | INFO | } | 20260505 19:25:45.724 | INFO | Perf report for the last 5.14648 seconds: | 20260505 19:25:45.724 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:25:45.724 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:45.724 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:45.724 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:45.724 | INFO | ANALYSIS : 62.0%, 3.0, 97.0ms, 0 | 20260505 19:25:45.724 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:45.724 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:45.724 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:25:45.724 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:45.724 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:45.724 | INFO | Uploaded 36GiB to GPU in 2.15761s (16.6851GiB/s) | 20260505 19:25:45.724 | INFO | Downloaded 34GiB to GPU in 2.06302s (16.4807GiB/s) | 20260505 19:25:50.867 | INFO | Analyzing chunk frames = [5704192,5704704) | 20260505 19:25:50.867 | INFO | MemInfo: { | 20260505 19:25:50.867 | INFO | "gpu" : " 17 GB", | 20260505 19:25:50.867 | INFO | "huge" : "", | 20260505 19:25:50.867 | INFO | "huge_shared" : " 27 GB", | 20260505 19:25:50.867 | INFO | "malloc" : " 14 GB", | 20260505 19:25:50.867 | INFO | "pinned" : " 29 GB" | 20260505 19:25:50.867 | INFO | } | 20260505 19:25:50.867 | INFO | Perf report for the last 5.14313 seconds: | 20260505 19:25:50.867 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:25:50.867 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:50.867 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:50.867 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:50.867 | INFO | ANALYSIS : 62.1%, 3.0, 97.1ms, 0 | 20260505 19:25:50.867 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:50.867 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:50.867 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:25:50.867 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:50.867 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:50.867 | INFO | Uploaded 36GiB to GPU in 2.14409s (16.7903GiB/s) | 20260505 19:25:50.867 | INFO | Downloaded 34GiB to GPU in 2.0539s (16.5539GiB/s) | 20260505 19:25:56.003 | INFO | Analyzing chunk frames = [5704704,5705216) | 20260505 19:25:56.003 | INFO | MemInfo: { | 20260505 19:25:56.003 | INFO | "gpu" : " 17 GB", | 20260505 19:25:56.003 | INFO | "huge" : "", | 20260505 19:25:56.003 | INFO | "huge_shared" : " 27 GB", | 20260505 19:25:56.003 | INFO | "malloc" : " 14 GB", | 20260505 19:25:56.003 | INFO | "pinned" : " 29 GB" | 20260505 19:25:56.003 | INFO | } | 20260505 19:25:56.003 | INFO | Perf report for the last 5.13601 seconds: | 20260505 19:25:56.003 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:25:56.003 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:56.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:56.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:56.003 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 19:25:56.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:56.003 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:56.003 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 19:25:56.003 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:56.003 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:25:56.003 | INFO | Uploaded 36GiB to GPU in 2.12481s (16.9427GiB/s) | 20260505 19:25:56.003 | INFO | Downloaded 34GiB to GPU in 2.15014s (15.8129GiB/s) | 20260505 19:26:01.136 | INFO | Analyzing chunk frames = [5705216,5705728) | 20260505 19:26:01.136 | INFO | MemInfo: { | 20260505 19:26:01.136 | INFO | "gpu" : " 17 GB", | 20260505 19:26:01.136 | INFO | "huge" : "", | 20260505 19:26:01.136 | INFO | "huge_shared" : " 27 GB", | 20260505 19:26:01.136 | INFO | "malloc" : " 14 GB", | 20260505 19:26:01.136 | INFO | "pinned" : " 29 GB" | 20260505 19:26:01.136 | INFO | } | 20260505 19:26:01.136 | INFO | Perf report for the last 5.13288 seconds: | 20260505 19:26:01.136 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:26:01.136 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:01.136 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:01.136 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:01.136 | INFO | ANALYSIS : 63.1%, 3.0, 98.3ms, 0 | 20260505 19:26:01.136 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:01.136 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:01.136 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 19:26:01.136 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:01.136 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:01.136 | INFO | Uploaded 36GiB to GPU in 2.20601s (16.3191GiB/s) | 20260505 19:26:01.136 | INFO | Downloaded 34GiB to GPU in 2.04735s (16.6069GiB/s) | 20260505 19:26:01.270 | INFO | Starting a round of Baz IO | 20260505 19:26:04.445 | INFO | Populating RTMetrics report | 20260505 19:26:04.446 | INFO | RTMetrics report populated | 20260505 19:26:04.446 | INFO | Opening tmp RTMetrics json file | 20260505 19:26:04.447 | INFO | Serializing RTMetrics to json | 20260505 19:26:04.448 | INFO | Serialization done, swapping tmp file now... | 20260505 19:26:04.449 | INFO | RTMetrics done | 20260505 19:26:04.540 | INFO | Finished a round of Baz IO: Wrote 2.052 GiB in 3.27 seconds | 20260505 19:26:04.540 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.78% / 0.00% / 78.22% | 20260505 19:26:04.540 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.209451% / 7.94729e-06% / 9.93411e-05% | 20260505 19:26:04.540 | INFO | Baz IO overall: | 20260505 19:26:04.540 | INFO | BytesWritten : 3342931037018 | 20260505 19:26:04.540 | INFO | NumEvents : 1582005949437 | 20260505 19:26:06.283 | INFO | Analyzing chunk frames = [5705728,5706240) | 20260505 19:26:06.283 | INFO | MemInfo: { | 20260505 19:26:06.283 | INFO | "gpu" : " 17 GB", | 20260505 19:26:06.283 | INFO | "huge" : "", | 20260505 19:26:06.283 | INFO | "huge_shared" : " 27 GB", | 20260505 19:26:06.283 | INFO | "malloc" : " 14 GB", | 20260505 19:26:06.283 | INFO | "pinned" : " 29 GB" | 20260505 19:26:06.283 | INFO | } | 20260505 19:26:06.283 | INFO | Perf report for the last 5.14772 seconds: | 20260505 19:26:06.284 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:26:06.284 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:06.284 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:06.284 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:06.284 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 19:26:06.284 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:06.284 | INFO | RT_METRICS : 44.3%, 1.0, 23.3ms, 0 | 20260505 19:26:06.284 | INFO | PRE_HQ : 60.4%, 1.8, 55.8ms, 0 | 20260505 19:26:06.284 | INFO | BAZWRITER : 63.0%, 3.8, 123.6ms, 0 | 20260505 19:26:06.284 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 19:26:06.284 | INFO | Uploaded 36GiB to GPU in 2.16417s (16.6345GiB/s) | 20260505 19:26:06.284 | INFO | Downloaded 38GiB to GPU in 2.24768s (16.9063GiB/s) | 20260505 19:26:11.421 | INFO | Analyzing chunk frames = [5706240,5706752) | 20260505 19:26:11.421 | INFO | MemInfo: { | 20260505 19:26:11.421 | INFO | "gpu" : " 17 GB", | 20260505 19:26:11.421 | INFO | "huge" : "", | 20260505 19:26:11.421 | INFO | "huge_shared" : " 27 GB", | 20260505 19:26:11.421 | INFO | "malloc" : " 14 GB", | 20260505 19:26:11.421 | INFO | "pinned" : " 29 GB" | 20260505 19:26:11.421 | INFO | } | 20260505 19:26:11.421 | INFO | Perf report for the last 5.13784 seconds: | 20260505 19:26:11.421 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:26:11.421 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:11.421 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:11.421 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:11.421 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 19:26:11.421 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:11.421 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:11.421 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:26:11.421 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:11.421 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:11.422 | INFO | Uploaded 36GiB to GPU in 2.14576s (16.7773GiB/s) | 20260505 19:26:11.422 | INFO | Downloaded 34GiB to GPU in 2.05039s (16.5822GiB/s) | 20260505 19:26:16.561 | INFO | Analyzing chunk frames = [5706752,5707264) | 20260505 19:26:16.561 | INFO | MemInfo: { | 20260505 19:26:16.561 | INFO | "gpu" : " 17 GB", | 20260505 19:26:16.561 | INFO | "huge" : "", | 20260505 19:26:16.561 | INFO | "huge_shared" : " 27 GB", | 20260505 19:26:16.561 | INFO | "malloc" : " 14 GB", | 20260505 19:26:16.561 | INFO | "pinned" : " 29 GB" | 20260505 19:26:16.561 | INFO | } | 20260505 19:26:16.561 | INFO | Perf report for the last 5.13994 seconds: | 20260505 19:26:16.561 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:26:16.561 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:16.561 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:16.561 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:16.561 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260505 19:26:16.561 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:16.561 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:16.561 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 19:26:16.561 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:16.561 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:16.562 | INFO | Uploaded 36GiB to GPU in 2.14045s (16.8189GiB/s) | 20260505 19:26:16.562 | INFO | Downloaded 34GiB to GPU in 2.03094s (16.741GiB/s) | 20260505 19:26:21.697 | INFO | Analyzing chunk frames = [5707264,5707776) | 20260505 19:26:21.697 | INFO | MemInfo: { | 20260505 19:26:21.697 | INFO | "gpu" : " 17 GB", | 20260505 19:26:21.697 | INFO | "huge" : "", | 20260505 19:26:21.697 | INFO | "huge_shared" : " 27 GB", | 20260505 19:26:21.697 | INFO | "malloc" : " 14 GB", | 20260505 19:26:21.697 | INFO | "pinned" : " 29 GB" | 20260505 19:26:21.697 | INFO | } | 20260505 19:26:21.697 | INFO | Perf report for the last 5.13601 seconds: | 20260505 19:26:21.697 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:26:21.697 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:21.697 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:21.697 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:21.697 | INFO | ANALYSIS : 62.4%, 3.0, 96.8ms, 0 | 20260505 19:26:21.697 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:21.697 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:21.697 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:26:21.697 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:21.697 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:21.698 | INFO | Uploaded 36GiB to GPU in 2.19068s (16.4332GiB/s) | 20260505 19:26:21.698 | INFO | Downloaded 34GiB to GPU in 2.05885s (16.514GiB/s) | 20260505 19:26:26.833 | INFO | Analyzing chunk frames = [5707776,5708288) | 20260505 19:26:26.833 | INFO | MemInfo: { | 20260505 19:26:26.833 | INFO | "gpu" : " 17 GB", | 20260505 19:26:26.833 | INFO | "huge" : "", | 20260505 19:26:26.833 | INFO | "huge_shared" : " 27 GB", | 20260505 19:26:26.833 | INFO | "malloc" : " 14 GB", | 20260505 19:26:26.833 | INFO | "pinned" : " 29 GB" | 20260505 19:26:26.833 | INFO | } | 20260505 19:26:26.833 | INFO | Perf report for the last 5.13562 seconds: | 20260505 19:26:26.833 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:26:26.833 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:26.833 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:26.833 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:26.833 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 19:26:26.833 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:26.833 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:26.833 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:26:26.833 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:26.833 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:26.833 | INFO | Uploaded 36GiB to GPU in 2.14606s (16.775GiB/s) | 20260505 19:26:26.833 | INFO | Downloaded 34GiB to GPU in 2.06482s (16.4663GiB/s) | 20260505 19:26:31.980 | INFO | Analyzing chunk frames = [5708288,5708800) | 20260505 19:26:31.980 | INFO | MemInfo: { | 20260505 19:26:31.980 | INFO | "gpu" : " 17 GB", | 20260505 19:26:31.980 | INFO | "huge" : "", | 20260505 19:26:31.980 | INFO | "huge_shared" : " 27 GB", | 20260505 19:26:31.980 | INFO | "malloc" : " 14 GB", | 20260505 19:26:31.980 | INFO | "pinned" : " 29 GB" | 20260505 19:26:31.980 | INFO | } | 20260505 19:26:31.980 | INFO | Perf report for the last 5.14736 seconds: | 20260505 19:26:31.980 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:26:31.980 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:31.980 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:31.980 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:31.980 | INFO | ANALYSIS : 62.2%, 3.0, 97.3ms, 0 | 20260505 19:26:31.980 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:31.980 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:31.980 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:26:31.980 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:31.980 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:31.981 | INFO | Uploaded 36GiB to GPU in 2.15614s (16.6965GiB/s) | 20260505 19:26:31.981 | INFO | Downloaded 34GiB to GPU in 2.06309s (16.4802GiB/s) | 20260505 19:26:37.123 | INFO | Analyzing chunk frames = [5708800,5709312) | 20260505 19:26:37.123 | INFO | MemInfo: { | 20260505 19:26:37.123 | INFO | "gpu" : " 17 GB", | 20260505 19:26:37.123 | INFO | "huge" : "", | 20260505 19:26:37.123 | INFO | "huge_shared" : " 27 GB", | 20260505 19:26:37.123 | INFO | "malloc" : " 14 GB", | 20260505 19:26:37.123 | INFO | "pinned" : " 29 GB" | 20260505 19:26:37.123 | INFO | } | 20260505 19:26:37.123 | INFO | Perf report for the last 5.14252 seconds: | 20260505 19:26:37.123 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:26:37.123 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:37.123 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:37.123 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:37.123 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 19:26:37.123 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:37.123 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:37.123 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 19:26:37.123 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:37.123 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:37.123 | INFO | Uploaded 36GiB to GPU in 2.11026s (17.0595GiB/s) | 20260505 19:26:37.123 | INFO | Downloaded 34GiB to GPU in 2.12158s (16.0258GiB/s) | 20260505 19:26:42.251 | INFO | Analyzing chunk frames = [5709312,5709824) | 20260505 19:26:42.251 | INFO | MemInfo: { | 20260505 19:26:42.251 | INFO | "gpu" : " 17 GB", | 20260505 19:26:42.251 | INFO | "huge" : "", | 20260505 19:26:42.251 | INFO | "huge_shared" : " 27 GB", | 20260505 19:26:42.251 | INFO | "malloc" : " 14 GB", | 20260505 19:26:42.251 | INFO | "pinned" : " 29 GB" | 20260505 19:26:42.251 | INFO | } | 20260505 19:26:42.251 | INFO | Perf report for the last 5.12832 seconds: | 20260505 19:26:42.251 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:26:42.251 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:42.251 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:42.251 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:42.251 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260505 19:26:42.251 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:42.251 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:42.251 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:26:42.251 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:42.251 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:42.251 | INFO | Uploaded 36GiB to GPU in 2.12018s (16.9797GiB/s) | 20260505 19:26:42.251 | INFO | Downloaded 34GiB to GPU in 2.01227s (16.8964GiB/s) | 20260505 19:26:42.252 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 19:26:42.252 | INFO | Total mem usage: 15.3 GiB | 20260505 19:26:42.252 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 19:26:42.252 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.07%) | 20260505 19:26:42.252 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 19:26:42.252 | INFO | High Water Breakdown: | 20260505 19:26:42.252 | INFO | 61% : 8.66 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 19:26:42.252 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 19:26:42.252 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 19:26:42.252 | INFO | ------------------------------------------ | 20260505 19:26:42.252 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:26:42.252 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 19:26:42.252 | INFO | Total mem usage: 26.5 GiB | 20260505 19:26:42.252 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 19:26:42.252 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 19:26:42.252 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 19:26:42.252 | INFO | High Water Breakdown: | 20260505 19:26:42.252 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:26:42.252 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:26:42.252 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:26:42.252 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:26:42.252 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:26:42.252 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:26:42.252 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 19:26:42.252 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 19:26:42.252 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:26:42.252 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 19:26:42.252 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 19:26:42.252 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 19:26:42.252 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 19:26:42.252 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 19:26:42.252 | INFO | ------------------------------------------ | 20260505 19:26:42.252 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:26:42.252 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 19:26:42.252 | INFO | Total mem usage: 25.6 GiB | 20260505 19:26:42.253 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 19:26:42.253 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 19:26:42.253 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 19:26:42.253 | INFO | High Water Breakdown: | 20260505 19:26:42.253 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 19:26:42.253 | INFO | ------------------------------------------ | 20260505 19:26:42.253 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:26:42.253 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 19:26:42.253 | INFO | Total mem usage: 15.7 GiB | 20260505 19:26:42.253 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 19:26:42.253 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 19:26:42.253 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 19:26:42.253 | INFO | High Water Breakdown: | 20260505 19:26:42.253 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 19:26:42.253 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 19:26:42.253 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 19:26:42.253 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 19:26:42.253 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 19:26:42.253 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 19:26:42.253 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 19:26:42.253 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 19:26:42.253 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 19:26:42.253 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:26:42.253 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 19:26:42.253 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:26:42.253 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 19:26:42.253 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 19:26:42.253 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 19:26:42.253 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 19:26:42.253 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:26:42.253 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 19:26:42.253 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:26:42.253 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:26:42.253 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:26:42.253 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 19:26:42.253 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 19:26:42.253 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:26:42.253 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 19:26:42.253 | INFO | ------------------------------------------ | 20260505 19:26:42.253 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:26:42.253 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 19:26:42.253 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:26:42.253 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:26:42.253 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.47e+01, 1.36e+01 / 1.71e+00 | 20260505 19:26:42.253 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.03e-01, 1.03e-02 / 4.41e-03 | 20260505 19:26:42.253 | INFO | PerfReport: 11.78%, Baseline -- 3.71e+00 / 4.53e+00, 5.66e-01 / 1.60e-01 | 20260505 19:26:42.253 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.52e+00, 1.89e-01 / 1.10e-01 | 20260505 19:26:42.253 | INFO | PerfReport: 15.95%, DME -- 4.02e+01 / 4.45e+01, 3.97e+00 / 1.87e+00 | 20260505 19:26:42.253 | INFO | PerfReport: 57.01%, FrameLabeling -- 1.80e+01 / 1.93e+01, 1.47e+00 / 1.03e+00 | 20260505 19:26:42.253 | INFO | PerfReport: 7.27%, PulseAccumulating -- 2.29e+00 / 2.88e+00, 4.39e-01 / 1.76e-01 | 20260505 19:26:42.253 | INFO | PerfReport: 3.28%, Metrics -- 1.04e+00 / 1.37e+00, 2.85e-01 / 1.51e-01 | 20260505 19:26:42.253 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.61e-01 / 4.32e-01, 6.46e-02 / 3.13e-02 | 20260505 19:26:42.253 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 19:26:42.253 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:26:42.253 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:26:42.253 | INFO | PerfReport: Full Execution -- 9.96e+01 / -nan, 2.54e+01 / 0.00e+00 | 20260505 19:26:42.253 | INFO | PerfReport: 22.50%, Upload -- 2.24e+01 / 0.00e+00, 7.79e+00 / 0.00e+00 | 20260505 19:26:42.253 | INFO | PerfReport: 21.84%, Download -- 2.17e+01 / 0.00e+00, 7.73e+00 / 0.00e+00 | 20260505 19:26:42.253 | INFO | PerfReport: 31.66%, Compute -- 3.15e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 19:26:42.253 | INFO | PerfReport: 2.34%, LockUpload -- 2.33e+00 / 0.00e+00, 7.96e+00 / 0.00e+00 | 20260505 19:26:42.253 | INFO | PerfReport: 1.35%, LockDownload -- 1.34e+00 / 0.00e+00, 2.90e+00 / 0.00e+00 | 20260505 19:26:42.253 | INFO | PerfReport: 20.30%, LockCompute -- 2.02e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 19:26:42.253 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.42e-02 / 0.00e+00 | 20260505 19:26:42.384 | INFO | Starting a round of Baz IO | 20260505 19:26:45.555 | INFO | Populating RTMetrics report | 20260505 19:26:45.556 | INFO | RTMetrics report populated | 20260505 19:26:45.556 | INFO | Opening tmp RTMetrics json file | 20260505 19:26:45.558 | INFO | Serializing RTMetrics to json | 20260505 19:26:45.559 | INFO | Serialization done, swapping tmp file now... | 20260505 19:26:45.560 | INFO | RTMetrics done | 20260505 19:26:45.647 | INFO | Finished a round of Baz IO: Wrote 2.051 GiB in 3.262 seconds | 20260505 19:26:45.647 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.78% / 0.00% / 78.22% | 20260505 19:26:45.647 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.209053% / 3.97364e-06% / 7.94729e-05% | 20260505 19:26:45.647 | INFO | Baz IO overall: | 20260505 19:26:45.647 | INFO | BytesWritten : 3345133292822 | 20260505 19:26:45.647 | INFO | NumEvents : 1583029060804 | 20260505 19:26:47.393 | INFO | Analyzing chunk frames = [5709824,5710336) | 20260505 19:26:47.393 | INFO | MemInfo: { | 20260505 19:26:47.393 | INFO | "gpu" : " 17 GB", | 20260505 19:26:47.393 | INFO | "huge" : "", | 20260505 19:26:47.393 | INFO | "huge_shared" : " 27 GB", | 20260505 19:26:47.393 | INFO | "malloc" : " 14 GB", | 20260505 19:26:47.393 | INFO | "pinned" : " 29 GB" | 20260505 19:26:47.393 | INFO | } | 20260505 19:26:47.393 | INFO | Perf report for the last 5.14164 seconds: | 20260505 19:26:47.393 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:26:47.393 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:47.393 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:47.393 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:47.393 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 19:26:47.393 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:47.393 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260505 19:26:47.393 | INFO | PRE_HQ : 60.0%, 1.8, 55.4ms, 0 | 20260505 19:26:47.393 | INFO | BAZWRITER : 62.9%, 3.8, 121.7ms, 0 | 20260505 19:26:47.393 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 19:26:47.393 | INFO | Uploaded 36GiB to GPU in 2.15774s (16.6841GiB/s) | 20260505 19:26:47.393 | INFO | Downloaded 38GiB to GPU in 2.24422s (16.9324GiB/s) | 20260505 19:26:52.535 | INFO | Analyzing chunk frames = [5710336,5710848) | 20260505 19:26:52.535 | INFO | MemInfo: { | 20260505 19:26:52.535 | INFO | "gpu" : " 17 GB", | 20260505 19:26:52.535 | INFO | "huge" : "", | 20260505 19:26:52.535 | INFO | "huge_shared" : " 27 GB", | 20260505 19:26:52.535 | INFO | "malloc" : " 14 GB", | 20260505 19:26:52.535 | INFO | "pinned" : " 29 GB" | 20260505 19:26:52.535 | INFO | } | 20260505 19:26:52.535 | INFO | Perf report for the last 5.14217 seconds: | 20260505 19:26:52.535 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:26:52.535 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:52.535 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:52.535 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:52.535 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260505 19:26:52.535 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:52.535 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:52.535 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 19:26:52.535 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:52.535 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:52.535 | INFO | Uploaded 36GiB to GPU in 2.13023s (16.8996GiB/s) | 20260505 19:26:52.535 | INFO | Downloaded 34GiB to GPU in 2.0262s (16.7802GiB/s) | 20260505 19:26:57.675 | INFO | Analyzing chunk frames = [5710848,5711360) | 20260505 19:26:57.675 | INFO | MemInfo: { | 20260505 19:26:57.675 | INFO | "gpu" : " 17 GB", | 20260505 19:26:57.675 | INFO | "huge" : "", | 20260505 19:26:57.675 | INFO | "huge_shared" : " 27 GB", | 20260505 19:26:57.675 | INFO | "malloc" : " 14 GB", | 20260505 19:26:57.675 | INFO | "pinned" : " 29 GB" | 20260505 19:26:57.675 | INFO | } | 20260505 19:26:57.675 | INFO | Perf report for the last 5.1401 seconds: | 20260505 19:26:57.675 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:26:57.675 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:57.675 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:57.675 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:57.675 | INFO | ANALYSIS : 62.4%, 3.0, 97.0ms, 0 | 20260505 19:26:57.675 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:57.675 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:57.675 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:26:57.675 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:57.675 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:26:57.675 | INFO | Uploaded 36GiB to GPU in 2.13908s (16.8297GiB/s) | 20260505 19:26:57.675 | INFO | Downloaded 34GiB to GPU in 2.04782s (16.603GiB/s) | 20260505 19:27:02.814 | INFO | Analyzing chunk frames = [5711360,5711872) | 20260505 19:27:02.814 | INFO | MemInfo: { | 20260505 19:27:02.814 | INFO | "gpu" : " 17 GB", | 20260505 19:27:02.814 | INFO | "huge" : "", | 20260505 19:27:02.814 | INFO | "huge_shared" : " 27 GB", | 20260505 19:27:02.814 | INFO | "malloc" : " 14 GB", | 20260505 19:27:02.814 | INFO | "pinned" : " 29 GB" | 20260505 19:27:02.814 | INFO | } | 20260505 19:27:02.814 | INFO | Perf report for the last 5.13894 seconds: | 20260505 19:27:02.814 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:27:02.814 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:02.814 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:02.814 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:02.814 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260505 19:27:02.814 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:02.814 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:02.814 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260505 19:27:02.814 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:02.814 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:02.814 | INFO | Uploaded 36GiB to GPU in 2.19055s (16.4342GiB/s) | 20260505 19:27:02.814 | INFO | Downloaded 34GiB to GPU in 2.06361s (16.476GiB/s) | 20260505 19:27:07.953 | INFO | Analyzing chunk frames = [5711872,5712384) | 20260505 19:27:07.953 | INFO | MemInfo: { | 20260505 19:27:07.953 | INFO | "gpu" : " 17 GB", | 20260505 19:27:07.953 | INFO | "huge" : "", | 20260505 19:27:07.953 | INFO | "huge_shared" : " 27 GB", | 20260505 19:27:07.953 | INFO | "malloc" : " 14 GB", | 20260505 19:27:07.953 | INFO | "pinned" : " 29 GB" | 20260505 19:27:07.953 | INFO | } | 20260505 19:27:07.953 | INFO | Perf report for the last 5.13923 seconds: | 20260505 19:27:07.953 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:27:07.953 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:07.953 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:07.953 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:07.953 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 19:27:07.953 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:07.953 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:07.953 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:27:07.953 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:07.953 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:07.954 | INFO | Uploaded 36GiB to GPU in 2.12701s (16.9251GiB/s) | 20260505 19:27:07.954 | INFO | Downloaded 34GiB to GPU in 2.0625s (16.4848GiB/s) | 20260505 19:27:13.088 | INFO | Analyzing chunk frames = [5712384,5712896) | 20260505 19:27:13.088 | INFO | MemInfo: { | 20260505 19:27:13.088 | INFO | "gpu" : " 17 GB", | 20260505 19:27:13.088 | INFO | "huge" : "", | 20260505 19:27:13.088 | INFO | "huge_shared" : " 27 GB", | 20260505 19:27:13.088 | INFO | "malloc" : " 14 GB", | 20260505 19:27:13.088 | INFO | "pinned" : " 29 GB" | 20260505 19:27:13.088 | INFO | } | 20260505 19:27:13.088 | INFO | Perf report for the last 5.13512 seconds: | 20260505 19:27:13.088 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:27:13.088 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:13.088 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:13.088 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:13.089 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 19:27:13.089 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:13.089 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:13.089 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:27:13.089 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:13.089 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:13.089 | INFO | Uploaded 36GiB to GPU in 2.13383s (16.871GiB/s) | 20260505 19:27:13.089 | INFO | Downloaded 34GiB to GPU in 2.05109s (16.5765GiB/s) | 20260505 19:27:18.232 | INFO | Analyzing chunk frames = [5712896,5713408) | 20260505 19:27:18.232 | INFO | MemInfo: { | 20260505 19:27:18.232 | INFO | "gpu" : " 17 GB", | 20260505 19:27:18.232 | INFO | "huge" : "", | 20260505 19:27:18.232 | INFO | "huge_shared" : " 27 GB", | 20260505 19:27:18.232 | INFO | "malloc" : " 14 GB", | 20260505 19:27:18.232 | INFO | "pinned" : " 29 GB" | 20260505 19:27:18.232 | INFO | } | 20260505 19:27:18.232 | INFO | Perf report for the last 5.14374 seconds: | 20260505 19:27:18.232 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:27:18.232 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:18.232 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:18.232 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:18.232 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 19:27:18.232 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:18.232 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:18.232 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 19:27:18.232 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:18.232 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:18.233 | INFO | Uploaded 36GiB to GPU in 2.12417s (16.9478GiB/s) | 20260505 19:27:18.233 | INFO | Downloaded 34GiB to GPU in 2.13845s (15.8993GiB/s) | 20260505 19:27:23.374 | INFO | Analyzing chunk frames = [5713408,5713920) | 20260505 19:27:23.374 | INFO | MemInfo: { | 20260505 19:27:23.374 | INFO | "gpu" : " 17 GB", | 20260505 19:27:23.374 | INFO | "huge" : "", | 20260505 19:27:23.374 | INFO | "huge_shared" : " 27 GB", | 20260505 19:27:23.374 | INFO | "malloc" : " 14 GB", | 20260505 19:27:23.374 | INFO | "pinned" : " 29 GB" | 20260505 19:27:23.374 | INFO | } | 20260505 19:27:23.374 | INFO | Perf report for the last 5.14223 seconds: | 20260505 19:27:23.374 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:27:23.374 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:23.374 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:23.375 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:23.375 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 19:27:23.375 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:23.375 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:23.375 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 19:27:23.375 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:23.375 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:23.375 | INFO | Uploaded 36GiB to GPU in 2.20848s (16.3008GiB/s) | 20260505 19:27:23.375 | INFO | Downloaded 34GiB to GPU in 2.04924s (16.5915GiB/s) | 20260505 19:27:23.506 | INFO | Starting a round of Baz IO | 20260505 19:27:26.677 | INFO | Populating RTMetrics report | 20260505 19:27:26.678 | INFO | RTMetrics report populated | 20260505 19:27:26.678 | INFO | Opening tmp RTMetrics json file | 20260505 19:27:26.679 | INFO | Serializing RTMetrics to json | 20260505 19:27:26.681 | INFO | Serialization done, swapping tmp file now... | 20260505 19:27:26.681 | INFO | RTMetrics done | 20260505 19:27:26.772 | INFO | Finished a round of Baz IO: Wrote 2.049 GiB in 3.265 seconds | 20260505 19:27:26.772 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.78% / 0.00% / 78.22% | 20260505 19:27:26.772 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.207977% / 3.97364e-06% / 9.93411e-05% | 20260505 19:27:26.772 | INFO | Baz IO overall: | 20260505 19:27:26.772 | INFO | BytesWritten : 3347333329937 | 20260505 19:27:26.772 | INFO | NumEvents : 1584050732643 | 20260505 19:27:28.510 | INFO | Analyzing chunk frames = [5713920,5714432) | 20260505 19:27:28.510 | INFO | MemInfo: { | 20260505 19:27:28.510 | INFO | "gpu" : " 17 GB", | 20260505 19:27:28.510 | INFO | "huge" : "", | 20260505 19:27:28.510 | INFO | "huge_shared" : " 27 GB", | 20260505 19:27:28.510 | INFO | "malloc" : " 14 GB", | 20260505 19:27:28.510 | INFO | "pinned" : " 29 GB" | 20260505 19:27:28.510 | INFO | } | 20260505 19:27:28.510 | INFO | Perf report for the last 5.13597 seconds: | 20260505 19:27:28.510 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:27:28.510 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:28.510 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:28.510 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:28.511 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 19:27:28.511 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:28.511 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260505 19:27:28.511 | INFO | PRE_HQ : 60.2%, 1.8, 55.7ms, 0 | 20260505 19:27:28.511 | INFO | BAZWRITER : 63.1%, 3.8, 122.8ms, 0 | 20260505 19:27:28.511 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 19:27:28.511 | INFO | Uploaded 36GiB to GPU in 2.15797s (16.6823GiB/s) | 20260505 19:27:28.511 | INFO | Downloaded 38GiB to GPU in 2.24046s (16.9608GiB/s) | 20260505 19:27:33.647 | INFO | Analyzing chunk frames = [5714432,5714944) | 20260505 19:27:33.647 | INFO | MemInfo: { | 20260505 19:27:33.647 | INFO | "gpu" : " 17 GB", | 20260505 19:27:33.647 | INFO | "huge" : "", | 20260505 19:27:33.647 | INFO | "huge_shared" : " 27 GB", | 20260505 19:27:33.647 | INFO | "malloc" : " 14 GB", | 20260505 19:27:33.647 | INFO | "pinned" : " 29 GB" | 20260505 19:27:33.647 | INFO | } | 20260505 19:27:33.647 | INFO | Perf report for the last 5.13687 seconds: | 20260505 19:27:33.647 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:27:33.647 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:33.647 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:33.647 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:33.647 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 19:27:33.647 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:33.647 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:33.647 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 19:27:33.647 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:33.647 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:33.648 | INFO | Uploaded 36GiB to GPU in 2.13385s (16.8709GiB/s) | 20260505 19:27:33.648 | INFO | Downloaded 34GiB to GPU in 2.02994s (16.7493GiB/s) | 20260505 19:27:38.795 | INFO | Analyzing chunk frames = [5714944,5715456) | 20260505 19:27:38.795 | INFO | MemInfo: { | 20260505 19:27:38.795 | INFO | "gpu" : " 17 GB", | 20260505 19:27:38.795 | INFO | "huge" : "", | 20260505 19:27:38.795 | INFO | "huge_shared" : " 27 GB", | 20260505 19:27:38.795 | INFO | "malloc" : " 14 GB", | 20260505 19:27:38.795 | INFO | "pinned" : " 29 GB" | 20260505 19:27:38.795 | INFO | } | 20260505 19:27:38.795 | INFO | Perf report for the last 5.14749 seconds: | 20260505 19:27:38.795 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:27:38.795 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:38.795 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:38.795 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:38.795 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 19:27:38.795 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:38.795 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:38.795 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 19:27:38.795 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:38.795 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:38.795 | INFO | Uploaded 36GiB to GPU in 2.21247s (16.2714GiB/s) | 20260505 19:27:38.795 | INFO | Downloaded 34GiB to GPU in 2.07508s (16.3849GiB/s) | 20260505 19:27:43.930 | INFO | Analyzing chunk frames = [5715456,5715968) | 20260505 19:27:43.930 | INFO | MemInfo: { | 20260505 19:27:43.930 | INFO | "gpu" : " 17 GB", | 20260505 19:27:43.930 | INFO | "huge" : "", | 20260505 19:27:43.930 | INFO | "huge_shared" : " 27 GB", | 20260505 19:27:43.930 | INFO | "malloc" : " 14 GB", | 20260505 19:27:43.930 | INFO | "pinned" : " 29 GB" | 20260505 19:27:43.930 | INFO | } | 20260505 19:27:43.930 | INFO | Perf report for the last 5.13544 seconds: | 20260505 19:27:43.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:27:43.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:43.930 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:43.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:43.930 | INFO | ANALYSIS : 62.2%, 3.0, 96.5ms, 0 | 20260505 19:27:43.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:43.930 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:43.930 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 19:27:43.930 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:43.930 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:43.931 | INFO | Uploaded 36GiB to GPU in 2.136s (16.8539GiB/s) | 20260505 19:27:43.931 | INFO | Downloaded 34GiB to GPU in 2.04227s (16.6481GiB/s) | 20260505 19:27:49.066 | INFO | Analyzing chunk frames = [5715968,5716480) | 20260505 19:27:49.066 | INFO | MemInfo: { | 20260505 19:27:49.066 | INFO | "gpu" : " 17 GB", | 20260505 19:27:49.066 | INFO | "huge" : "", | 20260505 19:27:49.066 | INFO | "huge_shared" : " 27 GB", | 20260505 19:27:49.066 | INFO | "malloc" : " 14 GB", | 20260505 19:27:49.066 | INFO | "pinned" : " 29 GB" | 20260505 19:27:49.066 | INFO | } | 20260505 19:27:49.066 | INFO | Perf report for the last 5.13613 seconds: | 20260505 19:27:49.066 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:27:49.066 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:49.066 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:49.066 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:49.066 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 19:27:49.067 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:49.067 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:49.067 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:27:49.067 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:49.067 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:49.067 | INFO | Uploaded 36GiB to GPU in 2.13063s (16.8964GiB/s) | 20260505 19:27:49.067 | INFO | Downloaded 34GiB to GPU in 2.06156s (16.4924GiB/s) | 20260505 19:27:54.199 | INFO | Analyzing chunk frames = [5716480,5716992) | 20260505 19:27:54.199 | INFO | MemInfo: { | 20260505 19:27:54.199 | INFO | "gpu" : " 17 GB", | 20260505 19:27:54.199 | INFO | "huge" : "", | 20260505 19:27:54.199 | INFO | "huge_shared" : " 27 GB", | 20260505 19:27:54.199 | INFO | "malloc" : " 14 GB", | 20260505 19:27:54.199 | INFO | "pinned" : " 29 GB" | 20260505 19:27:54.199 | INFO | } | 20260505 19:27:54.199 | INFO | Perf report for the last 5.13297 seconds: | 20260505 19:27:54.199 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:27:54.199 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:54.199 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:54.199 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:54.199 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260505 19:27:54.199 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:54.200 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:54.200 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 19:27:54.200 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:54.200 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:54.200 | INFO | Uploaded 36GiB to GPU in 2.16654s (16.6164GiB/s) | 20260505 19:27:54.200 | INFO | Downloaded 34GiB to GPU in 2.06369s (16.4754GiB/s) | 20260505 19:27:59.343 | INFO | Analyzing chunk frames = [5716992,5717504) | 20260505 19:27:59.343 | INFO | MemInfo: { | 20260505 19:27:59.343 | INFO | "gpu" : " 17 GB", | 20260505 19:27:59.343 | INFO | "huge" : "", | 20260505 19:27:59.343 | INFO | "huge_shared" : " 27 GB", | 20260505 19:27:59.343 | INFO | "malloc" : " 14 GB", | 20260505 19:27:59.343 | INFO | "pinned" : " 29 GB" | 20260505 19:27:59.343 | INFO | } | 20260505 19:27:59.343 | INFO | Perf report for the last 5.14365 seconds: | 20260505 19:27:59.343 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:27:59.343 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:59.343 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:59.343 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:59.343 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 19:27:59.343 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:59.343 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:59.343 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 19:27:59.343 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:59.343 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:27:59.343 | INFO | Uploaded 36GiB to GPU in 2.13869s (16.8327GiB/s) | 20260505 19:27:59.343 | INFO | Downloaded 34GiB to GPU in 2.15001s (15.8139GiB/s) | 20260505 19:28:04.489 | INFO | Analyzing chunk frames = [5717504,5718016) | 20260505 19:28:04.489 | INFO | MemInfo: { | 20260505 19:28:04.489 | INFO | "gpu" : " 17 GB", | 20260505 19:28:04.489 | INFO | "huge" : "", | 20260505 19:28:04.489 | INFO | "huge_shared" : " 27 GB", | 20260505 19:28:04.489 | INFO | "malloc" : " 14 GB", | 20260505 19:28:04.489 | INFO | "pinned" : " 29 GB" | 20260505 19:28:04.489 | INFO | } | 20260505 19:28:04.489 | INFO | Perf report for the last 5.14591 seconds: | 20260505 19:28:04.489 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:28:04.489 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:04.489 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:04.489 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:04.489 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 19:28:04.489 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:04.489 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:04.489 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 19:28:04.489 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:04.489 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:04.489 | INFO | Uploaded 36GiB to GPU in 2.19985s (16.3647GiB/s) | 20260505 19:28:04.489 | INFO | Downloaded 34GiB to GPU in 2.03742s (16.6878GiB/s) | 20260505 19:28:04.490 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 19:28:04.490 | INFO | Total mem usage: 15.3 GiB | 20260505 19:28:04.490 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 19:28:04.490 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.06%) | 20260505 19:28:04.490 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 19:28:04.490 | INFO | High Water Breakdown: | 20260505 19:28:04.490 | INFO | 61% : 8.66 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 19:28:04.490 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 19:28:04.490 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 19:28:04.490 | INFO | ------------------------------------------ | 20260505 19:28:04.490 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:28:04.490 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 19:28:04.490 | INFO | Total mem usage: 26.5 GiB | 20260505 19:28:04.490 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 19:28:04.490 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 19:28:04.490 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 19:28:04.490 | INFO | High Water Breakdown: | 20260505 19:28:04.490 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:28:04.490 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:28:04.490 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:28:04.490 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:28:04.490 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:28:04.490 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:28:04.490 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 19:28:04.490 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 19:28:04.490 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:28:04.490 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 19:28:04.490 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 19:28:04.490 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 19:28:04.490 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 19:28:04.490 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 19:28:04.490 | INFO | ------------------------------------------ | 20260505 19:28:04.490 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:28:04.490 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 19:28:04.490 | INFO | Total mem usage: 25.6 GiB | 20260505 19:28:04.490 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 19:28:04.490 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 19:28:04.490 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 19:28:04.490 | INFO | High Water Breakdown: | 20260505 19:28:04.490 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 19:28:04.490 | INFO | ------------------------------------------ | 20260505 19:28:04.490 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:28:04.490 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 19:28:04.490 | INFO | Total mem usage: 15.7 GiB | 20260505 19:28:04.490 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 19:28:04.490 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 19:28:04.490 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 19:28:04.490 | INFO | High Water Breakdown: | 20260505 19:28:04.490 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 19:28:04.490 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 19:28:04.490 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 19:28:04.490 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 19:28:04.490 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 19:28:04.490 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 19:28:04.490 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 19:28:04.490 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 19:28:04.490 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 19:28:04.490 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:28:04.490 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 19:28:04.490 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:28:04.491 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 19:28:04.491 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 19:28:04.491 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 19:28:04.491 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 19:28:04.491 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:28:04.491 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 19:28:04.491 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:28:04.491 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:28:04.491 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:28:04.491 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 19:28:04.491 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 19:28:04.491 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:28:04.491 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 19:28:04.491 | INFO | ------------------------------------------ | 20260505 19:28:04.491 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:28:04.491 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 19:28:04.491 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:28:04.491 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:28:04.491 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.46e+01, 1.35e+01 / 2.05e+00 | 20260505 19:28:04.491 | INFO | PerfReport: 0.29%, AggregateStats -- 9.24e-02 / 1.02e-01, 1.07e-02 / 2.77e-03 | 20260505 19:28:04.491 | INFO | PerfReport: 11.78%, Baseline -- 3.71e+00 / 4.55e+00, 5.59e-01 / 1.38e-01 | 20260505 19:28:04.491 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.55e+00, 1.92e-01 / 6.67e-02 | 20260505 19:28:04.491 | INFO | PerfReport: 15.95%, DME -- 4.02e+01 / 4.42e+01, 4.08e+00 / 2.14e+00 | 20260505 19:28:04.491 | INFO | PerfReport: 56.96%, FrameLabeling -- 1.79e+01 / 1.95e+01, 1.43e+00 / 1.22e+00 | 20260505 19:28:04.491 | INFO | PerfReport: 7.29%, PulseAccumulating -- 2.30e+00 / 2.89e+00, 4.36e-01 / 1.71e-01 | 20260505 19:28:04.491 | INFO | PerfReport: 3.30%, Metrics -- 1.04e+00 / 1.39e+00, 2.88e-01 / 1.41e-01 | 20260505 19:28:04.491 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.61e-01 / 4.31e-01, 6.38e-02 / 3.86e-02 | 20260505 19:28:04.491 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 19:28:04.491 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:28:04.491 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:28:04.491 | INFO | PerfReport: Full Execution -- 9.97e+01 / -nan, 2.56e+01 / 0.00e+00 | 20260505 19:28:04.491 | INFO | PerfReport: 22.53%, Upload -- 2.25e+01 / 0.00e+00, 7.75e+00 / 0.00e+00 | 20260505 19:28:04.491 | INFO | PerfReport: 21.81%, Download -- 2.17e+01 / 0.00e+00, 7.72e+00 / 0.00e+00 | 20260505 19:28:04.491 | INFO | PerfReport: 31.59%, Compute -- 3.15e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 19:28:04.491 | INFO | PerfReport: 2.35%, LockUpload -- 2.35e+00 / 0.00e+00, 8.04e+00 / 0.00e+00 | 20260505 19:28:04.491 | INFO | PerfReport: 1.36%, LockDownload -- 1.35e+00 / 0.00e+00, 2.86e+00 / 0.00e+00 | 20260505 19:28:04.491 | INFO | PerfReport: 20.35%, LockCompute -- 2.03e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260505 19:28:04.491 | INFO | PerfReport: 0.01%, Misc -- 1.27e-02 / 0.00e+00, 1.53e-02 / 0.00e+00 | 20260505 19:28:04.618 | INFO | Starting a round of Baz IO | 20260505 19:28:07.792 | INFO | Populating RTMetrics report | 20260505 19:28:07.792 | INFO | RTMetrics report populated | 20260505 19:28:07.792 | INFO | Opening tmp RTMetrics json file | 20260505 19:28:07.799 | INFO | Serializing RTMetrics to json | 20260505 19:28:07.800 | INFO | Serialization done, swapping tmp file now... | 20260505 19:28:07.801 | INFO | RTMetrics done | 20260505 19:28:07.896 | INFO | Finished a round of Baz IO: Wrote 2.051 GiB in 3.277 seconds | 20260505 19:28:07.896 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.78% / 0.00% / 78.22% | 20260505 19:28:07.896 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.207527% / 0% / 9.13938e-05% | 20260505 19:28:07.896 | INFO | Baz IO overall: | 20260505 19:28:07.896 | INFO | BytesWritten : 3349536079444 | 20260505 19:28:07.896 | INFO | NumEvents : 1585074655723 | 20260505 19:28:09.631 | INFO | Analyzing chunk frames = [5718016,5718528) | 20260505 19:28:09.631 | INFO | MemInfo: { | 20260505 19:28:09.631 | INFO | "gpu" : " 17 GB", | 20260505 19:28:09.631 | INFO | "huge" : "", | 20260505 19:28:09.631 | INFO | "huge_shared" : " 27 GB", | 20260505 19:28:09.631 | INFO | "malloc" : " 14 GB", | 20260505 19:28:09.631 | INFO | "pinned" : " 29 GB" | 20260505 19:28:09.631 | INFO | } | 20260505 19:28:09.632 | INFO | Perf report for the last 5.1425 seconds: | 20260505 19:28:09.632 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:28:09.632 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:09.632 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:09.632 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:09.632 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 19:28:09.632 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:09.632 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260505 19:28:09.632 | INFO | PRE_HQ : 60.2%, 1.8, 55.4ms, 0 | 20260505 19:28:09.632 | INFO | BAZWRITER : 63.3%, 3.8, 123.9ms, 0 | 20260505 19:28:09.632 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 19:28:09.632 | INFO | Uploaded 36GiB to GPU in 2.16421s (16.6343GiB/s) | 20260505 19:28:09.632 | INFO | Downloaded 38GiB to GPU in 2.25225s (16.872GiB/s) | 20260505 19:28:14.767 | INFO | Analyzing chunk frames = [5718528,5719040) | 20260505 19:28:14.767 | INFO | MemInfo: { | 20260505 19:28:14.767 | INFO | "gpu" : " 17 GB", | 20260505 19:28:14.767 | INFO | "huge" : "", | 20260505 19:28:14.767 | INFO | "huge_shared" : " 27 GB", | 20260505 19:28:14.767 | INFO | "malloc" : " 14 GB", | 20260505 19:28:14.768 | INFO | "pinned" : " 29 GB" | 20260505 19:28:14.768 | INFO | } | 20260505 19:28:14.768 | INFO | Perf report for the last 5.13615 seconds: | 20260505 19:28:14.768 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:28:14.768 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:14.768 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:14.768 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:14.768 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 19:28:14.768 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:14.768 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:14.768 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:28:14.768 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:14.768 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:14.768 | INFO | Uploaded 36GiB to GPU in 2.07695s (17.3331GiB/s) | 20260505 19:28:14.768 | INFO | Downloaded 34GiB to GPU in 2.02921s (16.7553GiB/s) | 20260505 19:28:19.897 | INFO | Analyzing chunk frames = [5719040,5719552) | 20260505 19:28:19.897 | INFO | MemInfo: { | 20260505 19:28:19.897 | INFO | "gpu" : " 17 GB", | 20260505 19:28:19.897 | INFO | "huge" : "", | 20260505 19:28:19.897 | INFO | "huge_shared" : " 27 GB", | 20260505 19:28:19.897 | INFO | "malloc" : " 14 GB", | 20260505 19:28:19.897 | INFO | "pinned" : " 29 GB" | 20260505 19:28:19.897 | INFO | } | 20260505 19:28:19.897 | INFO | Perf report for the last 5.12955 seconds: | 20260505 19:28:19.897 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:28:19.897 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:19.897 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:19.897 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:19.897 | INFO | ANALYSIS : 62.5%, 3.0, 97.0ms, 0 | 20260505 19:28:19.897 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:19.897 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:19.897 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:28:19.897 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:19.897 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:19.898 | INFO | Uploaded 36GiB to GPU in 2.13737s (16.8432GiB/s) | 20260505 19:28:19.898 | INFO | Downloaded 34GiB to GPU in 2.04117s (16.6571GiB/s) | 20260505 19:28:25.040 | INFO | Analyzing chunk frames = [5719552,5720064) | 20260505 19:28:25.040 | INFO | MemInfo: { | 20260505 19:28:25.040 | INFO | "gpu" : " 17 GB", | 20260505 19:28:25.040 | INFO | "huge" : "", | 20260505 19:28:25.040 | INFO | "huge_shared" : " 27 GB", | 20260505 19:28:25.040 | INFO | "malloc" : " 14 GB", | 20260505 19:28:25.040 | INFO | "pinned" : " 29 GB" | 20260505 19:28:25.040 | INFO | } | 20260505 19:28:25.040 | INFO | Perf report for the last 5.14278 seconds: | 20260505 19:28:25.040 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:28:25.040 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:25.040 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:25.040 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:25.040 | INFO | ANALYSIS : 62.4%, 3.0, 97.0ms, 0 | 20260505 19:28:25.040 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:25.040 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:25.040 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:28:25.040 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:25.040 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:25.040 | INFO | Uploaded 36GiB to GPU in 2.18982s (16.4397GiB/s) | 20260505 19:28:25.040 | INFO | Downloaded 34GiB to GPU in 2.13989s (15.8887GiB/s) | 20260505 19:28:30.181 | INFO | Analyzing chunk frames = [5720064,5720576) | 20260505 19:28:30.181 | INFO | MemInfo: { | 20260505 19:28:30.181 | INFO | "gpu" : " 17 GB", | 20260505 19:28:30.181 | INFO | "huge" : "", | 20260505 19:28:30.181 | INFO | "huge_shared" : " 27 GB", | 20260505 19:28:30.181 | INFO | "malloc" : " 14 GB", | 20260505 19:28:30.182 | INFO | "pinned" : " 29 GB" | 20260505 19:28:30.182 | INFO | } | 20260505 19:28:30.182 | INFO | Perf report for the last 5.14154 seconds: | 20260505 19:28:30.182 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:28:30.182 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:30.182 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:30.182 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:30.182 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 19:28:30.182 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:30.182 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:30.182 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:28:30.182 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:30.182 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:30.182 | INFO | Uploaded 36GiB to GPU in 2.1369s (16.8468GiB/s) | 20260505 19:28:30.182 | INFO | Downloaded 34GiB to GPU in 2.06406s (16.4724GiB/s) | 20260505 19:28:35.323 | INFO | Analyzing chunk frames = [5720576,5721088) | 20260505 19:28:35.323 | INFO | MemInfo: { | 20260505 19:28:35.323 | INFO | "gpu" : " 17 GB", | 20260505 19:28:35.323 | INFO | "huge" : "", | 20260505 19:28:35.323 | INFO | "huge_shared" : " 27 GB", | 20260505 19:28:35.323 | INFO | "malloc" : " 14 GB", | 20260505 19:28:35.323 | INFO | "pinned" : " 29 GB" | 20260505 19:28:35.323 | INFO | } | 20260505 19:28:35.323 | INFO | Perf report for the last 5.1417 seconds: | 20260505 19:28:35.323 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:28:35.323 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:35.323 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:35.323 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:35.323 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260505 19:28:35.323 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:35.323 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:35.323 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:28:35.323 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:35.323 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:35.324 | INFO | Uploaded 36GiB to GPU in 2.14918s (16.7506GiB/s) | 20260505 19:28:35.324 | INFO | Downloaded 34GiB to GPU in 2.0604s (16.5017GiB/s) | 20260505 19:28:40.458 | INFO | Analyzing chunk frames = [5721088,5721600) | 20260505 19:28:40.458 | INFO | MemInfo: { | 20260505 19:28:40.458 | INFO | "gpu" : " 17 GB", | 20260505 19:28:40.458 | INFO | "huge" : "", | 20260505 19:28:40.458 | INFO | "huge_shared" : " 27 GB", | 20260505 19:28:40.458 | INFO | "malloc" : " 14 GB", | 20260505 19:28:40.458 | INFO | "pinned" : " 29 GB" | 20260505 19:28:40.458 | INFO | } | 20260505 19:28:40.458 | INFO | Perf report for the last 5.13507 seconds: | 20260505 19:28:40.458 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:28:40.458 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:40.458 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:40.458 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:40.458 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 19:28:40.458 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:40.458 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:40.458 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 19:28:40.458 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:40.459 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:40.459 | INFO | Uploaded 36GiB to GPU in 2.10858s (17.0731GiB/s) | 20260505 19:28:40.459 | INFO | Downloaded 34GiB to GPU in 2.15366s (15.7871GiB/s) | 20260505 19:28:45.599 | INFO | Analyzing chunk frames = [5721600,5722112) | 20260505 19:28:45.599 | INFO | MemInfo: { | 20260505 19:28:45.599 | INFO | "gpu" : " 17 GB", | 20260505 19:28:45.599 | INFO | "huge" : "", | 20260505 19:28:45.599 | INFO | "huge_shared" : " 27 GB", | 20260505 19:28:45.599 | INFO | "malloc" : " 14 GB", | 20260505 19:28:45.599 | INFO | "pinned" : " 29 GB" | 20260505 19:28:45.599 | INFO | } | 20260505 19:28:45.599 | INFO | Perf report for the last 5.14103 seconds: | 20260505 19:28:45.599 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:28:45.599 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:45.599 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:45.599 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:45.599 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 19:28:45.599 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:45.600 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:45.600 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:28:45.600 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:45.600 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:45.600 | INFO | Uploaded 36GiB to GPU in 2.2046s (16.3295GiB/s) | 20260505 19:28:45.600 | INFO | Downloaded 34GiB to GPU in 2.0417s (16.6528GiB/s) | 20260505 19:28:45.748 | INFO | Starting a round of Baz IO | 20260505 19:28:48.906 | INFO | Populating RTMetrics report | 20260505 19:28:48.906 | INFO | RTMetrics report populated | 20260505 19:28:48.906 | INFO | Opening tmp RTMetrics json file | 20260505 19:28:48.933 | INFO | Serializing RTMetrics to json | 20260505 19:28:48.998 | INFO | Serialization done, swapping tmp file now... | 20260505 19:28:49.012 | INFO | RTMetrics done | 20260505 19:28:49.030 | INFO | Finished a round of Baz IO: Wrote 2.05 GiB in 3.282 seconds | 20260505 19:28:49.030 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.77% / 0.00% / 78.22% | 20260505 19:28:49.030 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.206423% / 3.97364e-06% / 7.94729e-05% | 20260505 19:28:49.030 | INFO | Baz IO overall: | 20260505 19:28:49.030 | INFO | BytesWritten : 3351737470537 | 20260505 19:28:49.030 | INFO | NumEvents : 1586097673586 | 20260505 19:28:50.747 | INFO | Analyzing chunk frames = [5722112,5722624) | 20260505 19:28:50.747 | INFO | MemInfo: { | 20260505 19:28:50.747 | INFO | "gpu" : " 17 GB", | 20260505 19:28:50.747 | INFO | "huge" : "", | 20260505 19:28:50.747 | INFO | "huge_shared" : " 27 GB", | 20260505 19:28:50.747 | INFO | "malloc" : " 14 GB", | 20260505 19:28:50.747 | INFO | "pinned" : " 29 GB" | 20260505 19:28:50.747 | INFO | } | 20260505 19:28:50.747 | INFO | Perf report for the last 5.14784 seconds: | 20260505 19:28:50.747 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:28:50.747 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:50.747 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:50.747 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:50.747 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 19:28:50.747 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:50.747 | INFO | RT_METRICS : 46.0%, 1.0, 24.2ms, 0 | 20260505 19:28:50.747 | INFO | PRE_HQ : 60.1%, 1.8, 55.4ms, 0 | 20260505 19:28:50.747 | INFO | BAZWRITER : 63.6%, 3.9, 127.4ms, 0 | 20260505 19:28:50.747 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260505 19:28:50.748 | INFO | Uploaded 36GiB to GPU in 2.16857s (16.6008GiB/s) | 20260505 19:28:50.748 | INFO | Downloaded 38GiB to GPU in 2.24521s (16.9249GiB/s) | 20260505 19:28:55.882 | INFO | Analyzing chunk frames = [5722624,5723136) | 20260505 19:28:55.882 | INFO | MemInfo: { | 20260505 19:28:55.882 | INFO | "gpu" : " 17 GB", | 20260505 19:28:55.882 | INFO | "huge" : "", | 20260505 19:28:55.882 | INFO | "huge_shared" : " 27 GB", | 20260505 19:28:55.882 | INFO | "malloc" : " 14 GB", | 20260505 19:28:55.882 | INFO | "pinned" : " 29 GB" | 20260505 19:28:55.882 | INFO | } | 20260505 19:28:55.882 | INFO | Perf report for the last 5.13493 seconds: | 20260505 19:28:55.882 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:28:55.882 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:55.882 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:55.882 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:55.882 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 19:28:55.882 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:55.882 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:55.882 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:28:55.882 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:55.882 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:28:55.882 | INFO | Uploaded 36GiB to GPU in 2.14757s (16.7631GiB/s) | 20260505 19:28:55.882 | INFO | Downloaded 34GiB to GPU in 2.04598s (16.618GiB/s) | 20260505 19:29:01.015 | INFO | Analyzing chunk frames = [5723136,5723648) | 20260505 19:29:01.015 | INFO | MemInfo: { | 20260505 19:29:01.015 | INFO | "gpu" : " 17 GB", | 20260505 19:29:01.015 | INFO | "huge" : "", | 20260505 19:29:01.015 | INFO | "huge_shared" : " 27 GB", | 20260505 19:29:01.015 | INFO | "malloc" : " 14 GB", | 20260505 19:29:01.015 | INFO | "pinned" : " 29 GB" | 20260505 19:29:01.015 | INFO | } | 20260505 19:29:01.015 | INFO | Perf report for the last 5.13307 seconds: | 20260505 19:29:01.015 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:29:01.015 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:01.015 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:01.015 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:01.015 | INFO | ANALYSIS : 62.2%, 3.0, 96.6ms, 0 | 20260505 19:29:01.015 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:01.015 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:01.015 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:29:01.015 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:01.015 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:01.015 | INFO | Uploaded 36GiB to GPU in 2.12097s (16.9734GiB/s) | 20260505 19:29:01.016 | INFO | Downloaded 34GiB to GPU in 2.02101s (16.8233GiB/s) | 20260505 19:29:06.150 | INFO | Analyzing chunk frames = [5723648,5724160) | 20260505 19:29:06.150 | INFO | MemInfo: { | 20260505 19:29:06.150 | INFO | "gpu" : " 17 GB", | 20260505 19:29:06.150 | INFO | "huge" : "", | 20260505 19:29:06.150 | INFO | "huge_shared" : " 27 GB", | 20260505 19:29:06.150 | INFO | "malloc" : " 14 GB", | 20260505 19:29:06.150 | INFO | "pinned" : " 29 GB" | 20260505 19:29:06.151 | INFO | } | 20260505 19:29:06.151 | INFO | Perf report for the last 5.13529 seconds: | 20260505 19:29:06.151 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:29:06.151 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:06.151 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:06.151 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:06.151 | INFO | ANALYSIS : 62.7%, 3.0, 97.4ms, 0 | 20260505 19:29:06.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:06.151 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:06.151 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 19:29:06.151 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:06.151 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:06.151 | INFO | Uploaded 36GiB to GPU in 2.22308s (16.1938GiB/s) | 20260505 19:29:06.151 | INFO | Downloaded 34GiB to GPU in 2.13411s (15.9317GiB/s) | 20260505 19:29:11.297 | INFO | Analyzing chunk frames = [5724160,5724672) | 20260505 19:29:11.297 | INFO | MemInfo: { | 20260505 19:29:11.297 | INFO | "gpu" : " 17 GB", | 20260505 19:29:11.297 | INFO | "huge" : "", | 20260505 19:29:11.297 | INFO | "huge_shared" : " 27 GB", | 20260505 19:29:11.297 | INFO | "malloc" : " 14 GB", | 20260505 19:29:11.297 | INFO | "pinned" : " 29 GB" | 20260505 19:29:11.297 | INFO | } | 20260505 19:29:11.297 | INFO | Perf report for the last 5.14688 seconds: | 20260505 19:29:11.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:29:11.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:11.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:11.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:11.298 | INFO | ANALYSIS : 61.9%, 3.0, 96.8ms, 0 | 20260505 19:29:11.298 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:11.298 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:11.298 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:29:11.298 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:11.298 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:11.298 | INFO | Uploaded 36GiB to GPU in 2.12474s (16.9432GiB/s) | 20260505 19:29:11.298 | INFO | Downloaded 34GiB to GPU in 2.05791s (16.5216GiB/s) | 20260505 19:29:16.439 | INFO | Analyzing chunk frames = [5724672,5725184) | 20260505 19:29:16.440 | INFO | MemInfo: { | 20260505 19:29:16.440 | INFO | "gpu" : " 17 GB", | 20260505 19:29:16.440 | INFO | "huge" : "", | 20260505 19:29:16.440 | INFO | "huge_shared" : " 27 GB", | 20260505 19:29:16.440 | INFO | "malloc" : " 14 GB", | 20260505 19:29:16.440 | INFO | "pinned" : " 29 GB" | 20260505 19:29:16.440 | INFO | } | 20260505 19:29:16.440 | INFO | Perf report for the last 5.14231 seconds: | 20260505 19:29:16.440 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:29:16.440 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:16.440 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:16.440 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:16.440 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260505 19:29:16.440 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:16.440 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:16.440 | INFO | PRE_HQ : 27.2%, 1.0, 14.2ms, 0 | 20260505 19:29:16.440 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:16.440 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:16.440 | INFO | Uploaded 36GiB to GPU in 2.14825s (16.7578GiB/s) | 20260505 19:29:16.440 | INFO | Downloaded 34GiB to GPU in 2.05969s (16.5073GiB/s) | 20260505 19:29:21.578 | INFO | Analyzing chunk frames = [5725184,5725696) | 20260505 19:29:21.578 | INFO | MemInfo: { | 20260505 19:29:21.578 | INFO | "gpu" : " 17 GB", | 20260505 19:29:21.578 | INFO | "huge" : "", | 20260505 19:29:21.578 | INFO | "huge_shared" : " 27 GB", | 20260505 19:29:21.578 | INFO | "malloc" : " 14 GB", | 20260505 19:29:21.578 | INFO | "pinned" : " 29 GB" | 20260505 19:29:21.578 | INFO | } | 20260505 19:29:21.578 | INFO | Perf report for the last 5.13788 seconds: | 20260505 19:29:21.578 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:29:21.578 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:21.578 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:21.578 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:21.578 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 19:29:21.578 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:21.578 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:21.578 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 19:29:21.578 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:21.578 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:21.578 | INFO | Uploaded 36GiB to GPU in 2.12284s (16.9584GiB/s) | 20260505 19:29:21.578 | INFO | Downloaded 34GiB to GPU in 2.13192s (15.948GiB/s) | 20260505 19:29:26.715 | INFO | Analyzing chunk frames = [5725696,5726208) | 20260505 19:29:26.715 | INFO | MemInfo: { | 20260505 19:29:26.715 | INFO | "gpu" : " 17 GB", | 20260505 19:29:26.715 | INFO | "huge" : "", | 20260505 19:29:26.715 | INFO | "huge_shared" : " 27 GB", | 20260505 19:29:26.715 | INFO | "malloc" : " 14 GB", | 20260505 19:29:26.715 | INFO | "pinned" : " 29 GB" | 20260505 19:29:26.715 | INFO | } | 20260505 19:29:26.715 | INFO | Perf report for the last 5.13722 seconds: | 20260505 19:29:26.715 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:29:26.715 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:26.715 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:26.715 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:26.715 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 19:29:26.715 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:26.715 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:26.715 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:29:26.715 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:26.715 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:26.715 | INFO | Uploaded 36GiB to GPU in 2.20343s (16.3382GiB/s) | 20260505 19:29:26.715 | INFO | Downloaded 34GiB to GPU in 2.04347s (16.6384GiB/s) | 20260505 19:29:26.715 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 19:29:26.715 | INFO | Total mem usage: 15.3 GiB | 20260505 19:29:26.715 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 19:29:26.715 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.08%) | 20260505 19:29:26.715 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 19:29:26.715 | INFO | High Water Breakdown: | 20260505 19:29:26.715 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 19:29:26.715 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 19:29:26.715 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 19:29:26.715 | INFO | ------------------------------------------ | 20260505 19:29:26.715 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:29:26.715 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 19:29:26.715 | INFO | Total mem usage: 26.5 GiB | 20260505 19:29:26.715 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 19:29:26.715 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 19:29:26.715 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 19:29:26.716 | INFO | High Water Breakdown: | 20260505 19:29:26.716 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:29:26.716 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:29:26.716 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:29:26.716 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:29:26.716 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:29:26.716 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:29:26.716 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 19:29:26.716 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 19:29:26.716 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:29:26.716 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 19:29:26.716 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 19:29:26.716 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 19:29:26.716 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 19:29:26.716 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 19:29:26.716 | INFO | ------------------------------------------ | 20260505 19:29:26.716 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:29:26.716 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 19:29:26.716 | INFO | Total mem usage: 25.6 GiB | 20260505 19:29:26.716 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 19:29:26.716 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 19:29:26.716 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 19:29:26.716 | INFO | High Water Breakdown: | 20260505 19:29:26.716 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 19:29:26.716 | INFO | ------------------------------------------ | 20260505 19:29:26.716 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:29:26.716 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 19:29:26.716 | INFO | Total mem usage: 15.7 GiB | 20260505 19:29:26.716 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 19:29:26.716 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 19:29:26.716 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 19:29:26.716 | INFO | High Water Breakdown: | 20260505 19:29:26.716 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 19:29:26.716 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 19:29:26.716 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 19:29:26.716 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 19:29:26.716 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 19:29:26.716 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 19:29:26.716 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 19:29:26.716 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 19:29:26.716 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 19:29:26.716 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:29:26.716 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 19:29:26.716 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:29:26.716 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 19:29:26.716 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 19:29:26.716 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 19:29:26.716 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 19:29:26.716 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:29:26.716 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 19:29:26.716 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:29:26.716 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:29:26.716 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:29:26.716 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 19:29:26.716 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 19:29:26.716 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:29:26.716 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 19:29:26.716 | INFO | ------------------------------------------ | 20260505 19:29:26.716 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:29:26.716 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 19:29:26.716 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:29:26.716 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:29:26.716 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.49e+01, 1.37e+01 / 1.95e+00 | 20260505 19:29:26.716 | INFO | PerfReport: 0.29%, AggregateStats -- 9.27e-02 / 1.02e-01, 1.07e-02 / 4.60e-03 | 20260505 19:29:26.716 | INFO | PerfReport: 11.79%, Baseline -- 3.71e+00 / 4.55e+00, 5.63e-01 / 1.34e-01 | 20260505 19:29:26.716 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.51e+00, 1.88e-01 / 9.54e-02 | 20260505 19:29:26.716 | INFO | PerfReport: 15.99%, DME -- 4.03e+01 / 4.48e+01, 4.25e+00 / 2.10e+00 | 20260505 19:29:26.716 | INFO | PerfReport: 56.92%, FrameLabeling -- 1.79e+01 / 1.93e+01, 1.40e+00 / 1.11e+00 | 20260505 19:29:26.716 | INFO | PerfReport: 7.28%, PulseAccumulating -- 2.29e+00 / 2.87e+00, 4.42e-01 / 1.86e-01 | 20260505 19:29:26.716 | INFO | PerfReport: 3.29%, Metrics -- 1.04e+00 / 1.38e+00, 2.88e-01 / 1.53e-01 | 20260505 19:29:26.716 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.61e-01 / 4.25e-01, 6.41e-02 / 3.36e-02 | 20260505 19:29:26.716 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 19:29:26.716 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:29:26.716 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:29:26.716 | INFO | PerfReport: Full Execution -- 9.96e+01 / -nan, 2.56e+01 / 0.00e+00 | 20260505 19:29:26.716 | INFO | PerfReport: 22.50%, Upload -- 2.24e+01 / 0.00e+00, 7.81e+00 / 0.00e+00 | 20260505 19:29:26.716 | INFO | PerfReport: 21.91%, Download -- 2.18e+01 / 0.00e+00, 7.81e+00 / 0.00e+00 | 20260505 19:29:26.716 | INFO | PerfReport: 31.62%, Compute -- 3.15e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 19:29:26.716 | INFO | PerfReport: 2.27%, LockUpload -- 2.26e+00 / 0.00e+00, 7.79e+00 / 0.00e+00 | 20260505 19:29:26.716 | INFO | PerfReport: 1.42%, LockDownload -- 1.41e+00 / 0.00e+00, 2.95e+00 / 0.00e+00 | 20260505 19:29:26.716 | INFO | PerfReport: 20.27%, LockCompute -- 2.02e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 19:29:26.716 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.62e-02 / 0.00e+00 | 20260505 19:29:26.851 | INFO | Starting a round of Baz IO | 20260505 19:29:30.028 | INFO | Populating RTMetrics report | 20260505 19:29:30.029 | INFO | RTMetrics report populated | 20260505 19:29:30.029 | INFO | Opening tmp RTMetrics json file | 20260505 19:29:30.031 | INFO | Serializing RTMetrics to json | 20260505 19:29:30.032 | INFO | Serialization done, swapping tmp file now... | 20260505 19:29:30.032 | INFO | RTMetrics done | 20260505 19:29:30.119 | INFO | Finished a round of Baz IO: Wrote 2.048 GiB in 3.268 seconds | 20260505 19:29:30.119 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.77% / 0.00% / 78.23% | 20260505 19:29:30.119 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.206335% / 3.97364e-06% / 9.93411e-05% | 20260505 19:29:30.119 | INFO | Baz IO overall: | 20260505 19:29:30.119 | INFO | BytesWritten : 3353936651733 | 20260505 19:29:30.119 | INFO | NumEvents : 1587119244333 | 20260505 19:29:31.851 | INFO | Analyzing chunk frames = [5726208,5726720) | 20260505 19:29:31.851 | INFO | MemInfo: { | 20260505 19:29:31.851 | INFO | "gpu" : " 17 GB", | 20260505 19:29:31.851 | INFO | "huge" : "", | 20260505 19:29:31.851 | INFO | "huge_shared" : " 27 GB", | 20260505 19:29:31.851 | INFO | "malloc" : " 14 GB", | 20260505 19:29:31.851 | INFO | "pinned" : " 29 GB" | 20260505 19:29:31.851 | INFO | } | 20260505 19:29:31.851 | INFO | Perf report for the last 5.13651 seconds: | 20260505 19:29:31.851 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:29:31.851 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:31.851 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:31.852 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:31.852 | INFO | ANALYSIS : 63.4%, 3.0, 98.8ms, 0 | 20260505 19:29:31.852 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:31.852 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260505 19:29:31.852 | INFO | PRE_HQ : 60.5%, 1.8, 56.0ms, 0 | 20260505 19:29:31.852 | INFO | BAZWRITER : 63.1%, 3.8, 122.0ms, 0 | 20260505 19:29:31.852 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 19:29:31.852 | INFO | Uploaded 36GiB to GPU in 2.15602s (16.6975GiB/s) | 20260505 19:29:31.852 | INFO | Downloaded 38GiB to GPU in 2.2485s (16.9002GiB/s) | 20260505 19:29:36.991 | INFO | Analyzing chunk frames = [5726720,5727232) | 20260505 19:29:36.991 | INFO | MemInfo: { | 20260505 19:29:36.991 | INFO | "gpu" : " 17 GB", | 20260505 19:29:36.991 | INFO | "huge" : "", | 20260505 19:29:36.991 | INFO | "huge_shared" : " 27 GB", | 20260505 19:29:36.991 | INFO | "malloc" : " 14 GB", | 20260505 19:29:36.991 | INFO | "pinned" : " 29 GB" | 20260505 19:29:36.991 | INFO | } | 20260505 19:29:36.991 | INFO | Perf report for the last 5.13949 seconds: | 20260505 19:29:36.991 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:29:36.991 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:36.991 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:36.991 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:36.991 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 19:29:36.991 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:36.991 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:36.991 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:29:36.991 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:36.991 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:36.991 | INFO | Uploaded 36GiB to GPU in 2.13086s (16.8946GiB/s) | 20260505 19:29:36.991 | INFO | Downloaded 34GiB to GPU in 2.02582s (16.7834GiB/s) | 20260505 19:29:42.134 | INFO | Analyzing chunk frames = [5727232,5727744) | 20260505 19:29:42.134 | INFO | MemInfo: { | 20260505 19:29:42.134 | INFO | "gpu" : " 17 GB", | 20260505 19:29:42.134 | INFO | "huge" : "", | 20260505 19:29:42.134 | INFO | "huge_shared" : " 27 GB", | 20260505 19:29:42.134 | INFO | "malloc" : " 14 GB", | 20260505 19:29:42.134 | INFO | "pinned" : " 29 GB" | 20260505 19:29:42.134 | INFO | } | 20260505 19:29:42.134 | INFO | Perf report for the last 5.14343 seconds: | 20260505 19:29:42.134 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:29:42.134 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:42.134 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:42.134 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:42.134 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260505 19:29:42.134 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:42.134 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:42.134 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 19:29:42.135 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:42.135 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:42.135 | INFO | Uploaded 36GiB to GPU in 2.13863s (16.8332GiB/s) | 20260505 19:29:42.135 | INFO | Downloaded 34GiB to GPU in 2.03082s (16.742GiB/s) | 20260505 19:29:47.267 | INFO | Analyzing chunk frames = [5727744,5728256) | 20260505 19:29:47.267 | INFO | MemInfo: { | 20260505 19:29:47.267 | INFO | "gpu" : " 17 GB", | 20260505 19:29:47.267 | INFO | "huge" : "", | 20260505 19:29:47.268 | INFO | "huge_shared" : " 27 GB", | 20260505 19:29:47.268 | INFO | "malloc" : " 14 GB", | 20260505 19:29:47.268 | INFO | "pinned" : " 29 GB" | 20260505 19:29:47.268 | INFO | } | 20260505 19:29:47.268 | INFO | Perf report for the last 5.13335 seconds: | 20260505 19:29:47.268 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:29:47.268 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:47.268 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:47.268 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:47.268 | INFO | ANALYSIS : 62.5%, 3.0, 97.0ms, 0 | 20260505 19:29:47.268 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:47.268 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:47.268 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:29:47.268 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:47.268 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:47.268 | INFO | Uploaded 36GiB to GPU in 2.21261s (16.2704GiB/s) | 20260505 19:29:47.268 | INFO | Downloaded 34GiB to GPU in 2.04845s (16.5979GiB/s) | 20260505 19:29:52.405 | INFO | Analyzing chunk frames = [5728256,5728768) | 20260505 19:29:52.405 | INFO | MemInfo: { | 20260505 19:29:52.405 | INFO | "gpu" : " 17 GB", | 20260505 19:29:52.405 | INFO | "huge" : "", | 20260505 19:29:52.405 | INFO | "huge_shared" : " 27 GB", | 20260505 19:29:52.405 | INFO | "malloc" : " 14 GB", | 20260505 19:29:52.405 | INFO | "pinned" : " 29 GB" | 20260505 19:29:52.405 | INFO | } | 20260505 19:29:52.406 | INFO | Perf report for the last 5.13774 seconds: | 20260505 19:29:52.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:29:52.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:52.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:52.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:52.406 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 19:29:52.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:52.406 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:52.406 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:29:52.406 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:52.406 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:52.406 | INFO | Uploaded 36GiB to GPU in 2.14645s (16.7718GiB/s) | 20260505 19:29:52.406 | INFO | Downloaded 34GiB to GPU in 2.06109s (16.4961GiB/s) | 20260505 19:29:57.553 | INFO | Analyzing chunk frames = [5728768,5729280) | 20260505 19:29:57.553 | INFO | MemInfo: { | 20260505 19:29:57.553 | INFO | "gpu" : " 17 GB", | 20260505 19:29:57.553 | INFO | "huge" : "", | 20260505 19:29:57.553 | INFO | "huge_shared" : " 27 GB", | 20260505 19:29:57.553 | INFO | "malloc" : " 14 GB", | 20260505 19:29:57.553 | INFO | "pinned" : " 29 GB" | 20260505 19:29:57.553 | INFO | } | 20260505 19:29:57.553 | INFO | Perf report for the last 5.14721 seconds: | 20260505 19:29:57.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:29:57.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:57.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:57.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:57.553 | INFO | ANALYSIS : 62.1%, 3.0, 97.1ms, 0 | 20260505 19:29:57.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:57.553 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:57.553 | INFO | PRE_HQ : 27.0%, 1.0, 14.1ms, 0 | 20260505 19:29:57.553 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:57.553 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:29:57.553 | INFO | Uploaded 36GiB to GPU in 2.13089s (16.8944GiB/s) | 20260505 19:29:57.553 | INFO | Downloaded 34GiB to GPU in 2.04431s (16.6316GiB/s) | 20260505 19:30:02.697 | INFO | Analyzing chunk frames = [5729280,5729792) | 20260505 19:30:02.697 | INFO | MemInfo: { | 20260505 19:30:02.697 | INFO | "gpu" : " 17 GB", | 20260505 19:30:02.697 | INFO | "huge" : "", | 20260505 19:30:02.697 | INFO | "huge_shared" : " 27 GB", | 20260505 19:30:02.697 | INFO | "malloc" : " 14 GB", | 20260505 19:30:02.697 | INFO | "pinned" : " 29 GB" | 20260505 19:30:02.697 | INFO | } | 20260505 19:30:02.697 | INFO | Perf report for the last 5.14438 seconds: | 20260505 19:30:02.697 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:30:02.697 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:02.697 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:02.697 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:02.697 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 19:30:02.697 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:02.697 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:02.697 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 19:30:02.697 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:02.697 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:02.697 | INFO | Uploaded 36GiB to GPU in 2.13113s (16.8925GiB/s) | 20260505 19:30:02.697 | INFO | Downloaded 34GiB to GPU in 2.13819s (15.9013GiB/s) | 20260505 19:30:07.829 | INFO | Analyzing chunk frames = [5729792,5730304) | 20260505 19:30:07.829 | INFO | MemInfo: { | 20260505 19:30:07.829 | INFO | "gpu" : " 17 GB", | 20260505 19:30:07.829 | INFO | "huge" : "", | 20260505 19:30:07.829 | INFO | "huge_shared" : " 27 GB", | 20260505 19:30:07.829 | INFO | "malloc" : " 14 GB", | 20260505 19:30:07.829 | INFO | "pinned" : " 29 GB" | 20260505 19:30:07.829 | INFO | } | 20260505 19:30:07.829 | INFO | Perf report for the last 5.13191 seconds: | 20260505 19:30:07.829 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:30:07.829 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:07.829 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:07.829 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:07.829 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260505 19:30:07.829 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:07.829 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:07.829 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 19:30:07.829 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:07.829 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:07.829 | INFO | Uploaded 36GiB to GPU in 2.10309s (17.1176GiB/s) | 20260505 19:30:07.829 | INFO | Downloaded 34GiB to GPU in 1.99194s (17.0688GiB/s) | 20260505 19:30:07.961 | INFO | Starting a round of Baz IO | 20260505 19:30:11.132 | INFO | Populating RTMetrics report | 20260505 19:30:11.133 | INFO | RTMetrics report populated | 20260505 19:30:11.133 | INFO | Opening tmp RTMetrics json file | 20260505 19:30:11.135 | INFO | Serializing RTMetrics to json | 20260505 19:30:11.138 | INFO | Serialization done, swapping tmp file now... | 20260505 19:30:11.140 | INFO | RTMetrics done | 20260505 19:30:11.223 | INFO | Finished a round of Baz IO: Wrote 2.049 GiB in 3.262 seconds | 20260505 19:30:11.223 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.77% / 0.00% / 78.23% | 20260505 19:30:11.223 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.206999% / 0% / 6.35783e-05% | 20260505 19:30:11.223 | INFO | Baz IO overall: | 20260505 19:30:11.224 | INFO | BytesWritten : 3356137047158 | 20260505 19:30:11.224 | INFO | NumEvents : 1588141936978 | 20260505 19:30:12.963 | INFO | Analyzing chunk frames = [5730304,5730816) | 20260505 19:30:12.963 | INFO | MemInfo: { | 20260505 19:30:12.963 | INFO | "gpu" : " 17 GB", | 20260505 19:30:12.963 | INFO | "huge" : "", | 20260505 19:30:12.963 | INFO | "huge_shared" : " 27 GB", | 20260505 19:30:12.963 | INFO | "malloc" : " 14 GB", | 20260505 19:30:12.963 | INFO | "pinned" : " 29 GB" | 20260505 19:30:12.963 | INFO | } | 20260505 19:30:12.964 | INFO | Perf report for the last 5.13445 seconds: | 20260505 19:30:12.964 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:30:12.964 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:12.964 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:12.964 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:12.964 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 19:30:12.964 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:12.964 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260505 19:30:12.964 | INFO | PRE_HQ : 60.1%, 1.8, 55.4ms, 0 | 20260505 19:30:12.964 | INFO | BAZWRITER : 63.1%, 3.8, 122.2ms, 0 | 20260505 19:30:12.964 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 19:30:12.964 | INFO | Uploaded 36GiB to GPU in 2.14936s (16.7491GiB/s) | 20260505 19:30:12.964 | INFO | Downloaded 38GiB to GPU in 2.2565s (16.8403GiB/s) | 20260505 19:30:18.104 | INFO | Analyzing chunk frames = [5730816,5731328) | 20260505 19:30:18.104 | INFO | MemInfo: { | 20260505 19:30:18.104 | INFO | "gpu" : " 17 GB", | 20260505 19:30:18.104 | INFO | "huge" : "", | 20260505 19:30:18.104 | INFO | "huge_shared" : " 27 GB", | 20260505 19:30:18.104 | INFO | "malloc" : " 14 GB", | 20260505 19:30:18.104 | INFO | "pinned" : " 29 GB" | 20260505 19:30:18.104 | INFO | } | 20260505 19:30:18.104 | INFO | Perf report for the last 5.14037 seconds: | 20260505 19:30:18.104 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:30:18.104 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:18.104 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:18.104 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:18.104 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 19:30:18.104 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:18.104 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:18.104 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:30:18.104 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:18.104 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:18.104 | INFO | Uploaded 36GiB to GPU in 2.13616s (16.8527GiB/s) | 20260505 19:30:18.104 | INFO | Downloaded 34GiB to GPU in 2.03443s (16.7123GiB/s) | 20260505 19:30:23.249 | INFO | Analyzing chunk frames = [5731328,5731840) | 20260505 19:30:23.249 | INFO | MemInfo: { | 20260505 19:30:23.249 | INFO | "gpu" : " 17 GB", | 20260505 19:30:23.249 | INFO | "huge" : "", | 20260505 19:30:23.249 | INFO | "huge_shared" : " 27 GB", | 20260505 19:30:23.249 | INFO | "malloc" : " 14 GB", | 20260505 19:30:23.249 | INFO | "pinned" : " 29 GB" | 20260505 19:30:23.249 | INFO | } | 20260505 19:30:23.249 | INFO | Perf report for the last 5.14511 seconds: | 20260505 19:30:23.249 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:30:23.249 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:23.249 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:23.249 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:23.249 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 19:30:23.249 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:23.249 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:23.249 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:30:23.249 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:23.249 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:23.249 | INFO | Uploaded 36GiB to GPU in 2.21379s (16.2617GiB/s) | 20260505 19:30:23.249 | INFO | Downloaded 34GiB to GPU in 2.0751s (16.3847GiB/s) | 20260505 19:30:28.395 | INFO | Analyzing chunk frames = [5731840,5732352) | 20260505 19:30:28.395 | INFO | MemInfo: { | 20260505 19:30:28.395 | INFO | "gpu" : " 17 GB", | 20260505 19:30:28.395 | INFO | "huge" : "", | 20260505 19:30:28.395 | INFO | "huge_shared" : " 27 GB", | 20260505 19:30:28.395 | INFO | "malloc" : " 14 GB", | 20260505 19:30:28.395 | INFO | "pinned" : " 29 GB" | 20260505 19:30:28.395 | INFO | } | 20260505 19:30:28.395 | INFO | Perf report for the last 5.14616 seconds: | 20260505 19:30:28.395 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:30:28.395 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:28.395 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:28.395 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:28.395 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260505 19:30:28.395 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:28.395 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:28.395 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:30:28.395 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:28.395 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:28.395 | INFO | Uploaded 36GiB to GPU in 2.21229s (16.2727GiB/s) | 20260505 19:30:28.395 | INFO | Downloaded 34GiB to GPU in 2.06075s (16.4989GiB/s) | 20260505 19:30:33.525 | INFO | Analyzing chunk frames = [5732352,5732864) | 20260505 19:30:33.525 | INFO | MemInfo: { | 20260505 19:30:33.525 | INFO | "gpu" : " 17 GB", | 20260505 19:30:33.525 | INFO | "huge" : "", | 20260505 19:30:33.525 | INFO | "huge_shared" : " 27 GB", | 20260505 19:30:33.525 | INFO | "malloc" : " 14 GB", | 20260505 19:30:33.525 | INFO | "pinned" : " 29 GB" | 20260505 19:30:33.525 | INFO | } | 20260505 19:30:33.525 | INFO | Perf report for the last 5.1296 seconds: | 20260505 19:30:33.525 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:30:33.525 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:33.525 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:33.525 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:33.525 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260505 19:30:33.525 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:33.525 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:33.525 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:30:33.525 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:33.525 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:33.525 | INFO | Uploaded 36GiB to GPU in 2.12506s (16.9407GiB/s) | 20260505 19:30:33.525 | INFO | Downloaded 34GiB to GPU in 2.06014s (16.5038GiB/s) | 20260505 19:30:38.661 | INFO | Analyzing chunk frames = [5732864,5733376) | 20260505 19:30:38.661 | INFO | MemInfo: { | 20260505 19:30:38.661 | INFO | "gpu" : " 17 GB", | 20260505 19:30:38.661 | INFO | "huge" : "", | 20260505 19:30:38.661 | INFO | "huge_shared" : " 27 GB", | 20260505 19:30:38.661 | INFO | "malloc" : " 14 GB", | 20260505 19:30:38.661 | INFO | "pinned" : " 29 GB" | 20260505 19:30:38.661 | INFO | } | 20260505 19:30:38.661 | INFO | Perf report for the last 5.13663 seconds: | 20260505 19:30:38.661 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:30:38.661 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:38.661 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:38.661 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:38.662 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260505 19:30:38.662 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:38.662 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:38.662 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260505 19:30:38.662 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:38.662 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:38.662 | INFO | Uploaded 36GiB to GPU in 2.13711s (16.8452GiB/s) | 20260505 19:30:38.662 | INFO | Downloaded 34GiB to GPU in 2.05423s (16.5512GiB/s) | 20260505 19:30:43.806 | INFO | Analyzing chunk frames = [5733376,5733888) | 20260505 19:30:43.806 | INFO | MemInfo: { | 20260505 19:30:43.806 | INFO | "gpu" : " 17 GB", | 20260505 19:30:43.806 | INFO | "huge" : "", | 20260505 19:30:43.806 | INFO | "huge_shared" : " 27 GB", | 20260505 19:30:43.806 | INFO | "malloc" : " 14 GB", | 20260505 19:30:43.806 | INFO | "pinned" : " 29 GB" | 20260505 19:30:43.806 | INFO | } | 20260505 19:30:43.806 | INFO | Perf report for the last 5.14441 seconds: | 20260505 19:30:43.806 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:30:43.806 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:43.806 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:43.806 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:43.806 | INFO | ANALYSIS : 63.4%, 3.0, 99.1ms, 0 | 20260505 19:30:43.806 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:43.806 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:43.806 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 19:30:43.806 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:43.806 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:43.806 | INFO | Uploaded 36GiB to GPU in 2.14171s (16.809GiB/s) | 20260505 19:30:43.806 | INFO | Downloaded 34GiB to GPU in 2.16877s (15.6771GiB/s) | 20260505 19:30:48.945 | INFO | Analyzing chunk frames = [5733888,5734400) | 20260505 19:30:48.945 | INFO | MemInfo: { | 20260505 19:30:48.945 | INFO | "gpu" : " 17 GB", | 20260505 19:30:48.945 | INFO | "huge" : "", | 20260505 19:30:48.945 | INFO | "huge_shared" : " 27 GB", | 20260505 19:30:48.945 | INFO | "malloc" : " 14 GB", | 20260505 19:30:48.945 | INFO | "pinned" : " 29 GB" | 20260505 19:30:48.945 | INFO | } | 20260505 19:30:48.945 | INFO | Perf report for the last 5.13939 seconds: | 20260505 19:30:48.945 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:30:48.945 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:48.945 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:48.945 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:48.945 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 19:30:48.945 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:48.945 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:48.945 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:30:48.945 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:48.945 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:48.946 | INFO | Uploaded 36GiB to GPU in 2.20281s (16.3428GiB/s) | 20260505 19:30:48.946 | INFO | Downloaded 34GiB to GPU in 2.04868s (16.5961GiB/s) | 20260505 19:30:48.946 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 19:30:48.946 | INFO | Total mem usage: 15.3 GiB | 20260505 19:30:48.946 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 19:30:48.946 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.06%) | 20260505 19:30:48.946 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 19:30:48.946 | INFO | High Water Breakdown: | 20260505 19:30:48.946 | INFO | 61% : 8.66 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 19:30:48.946 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 19:30:48.946 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 19:30:48.946 | INFO | ------------------------------------------ | 20260505 19:30:48.946 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:30:48.946 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 19:30:48.946 | INFO | Total mem usage: 26.5 GiB | 20260505 19:30:48.946 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 19:30:48.946 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 19:30:48.946 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 19:30:48.946 | INFO | High Water Breakdown: | 20260505 19:30:48.946 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:30:48.946 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:30:48.946 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:30:48.946 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:30:48.946 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:30:48.946 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:30:48.946 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 19:30:48.946 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 19:30:48.946 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:30:48.946 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 19:30:48.946 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 19:30:48.946 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 19:30:48.946 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 19:30:48.946 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 19:30:48.946 | INFO | ------------------------------------------ | 20260505 19:30:48.946 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:30:48.946 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 19:30:48.946 | INFO | Total mem usage: 25.6 GiB | 20260505 19:30:48.946 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 19:30:48.946 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 19:30:48.946 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 19:30:48.946 | INFO | High Water Breakdown: | 20260505 19:30:48.946 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 19:30:48.946 | INFO | ------------------------------------------ | 20260505 19:30:48.946 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:30:48.946 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 19:30:48.946 | INFO | Total mem usage: 15.7 GiB | 20260505 19:30:48.946 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 19:30:48.946 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 19:30:48.946 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 19:30:48.946 | INFO | High Water Breakdown: | 20260505 19:30:48.947 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 19:30:48.947 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 19:30:48.947 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 19:30:48.947 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 19:30:48.947 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 19:30:48.947 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 19:30:48.947 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 19:30:48.947 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 19:30:48.947 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 19:30:48.947 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:30:48.947 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 19:30:48.947 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:30:48.947 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 19:30:48.947 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 19:30:48.947 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 19:30:48.947 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 19:30:48.947 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:30:48.947 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 19:30:48.947 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:30:48.947 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:30:48.947 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:30:48.947 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 19:30:48.947 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 19:30:48.947 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:30:48.947 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 19:30:48.947 | INFO | ------------------------------------------ | 20260505 19:30:48.947 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:30:48.947 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 19:30:48.947 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:30:48.947 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:30:48.947 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.49e+01, 1.36e+01 / 2.16e+00 | 20260505 19:30:48.947 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.04e-01, 1.05e-02 / 2.97e-03 | 20260505 19:30:48.947 | INFO | PerfReport: 11.77%, Baseline -- 3.71e+00 / 4.53e+00, 5.67e-01 / 1.32e-01 | 20260505 19:30:48.947 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.52e+00, 1.89e-01 / 9.65e-02 | 20260505 19:30:48.947 | INFO | PerfReport: 16.00%, DME -- 4.03e+01 / 4.48e+01, 4.40e+00 / 2.23e+00 | 20260505 19:30:48.947 | INFO | PerfReport: 56.94%, FrameLabeling -- 1.79e+01 / 1.92e+01, 1.43e+00 / 8.36e-01 | 20260505 19:30:48.947 | INFO | PerfReport: 7.29%, PulseAccumulating -- 2.30e+00 / 2.91e+00, 4.41e-01 / 1.39e-01 | 20260505 19:30:48.947 | INFO | PerfReport: 3.29%, Metrics -- 1.04e+00 / 1.38e+00, 2.85e-01 / 1.27e-01 | 20260505 19:30:48.947 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.60e-01 / 4.31e-01, 6.38e-02 / 3.69e-02 | 20260505 19:30:48.947 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 19:30:48.947 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:30:48.947 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:30:48.947 | INFO | PerfReport: Full Execution -- 9.97e+01 / -nan, 2.58e+01 / 0.00e+00 | 20260505 19:30:48.947 | INFO | PerfReport: 22.51%, Upload -- 2.24e+01 / 0.00e+00, 7.81e+00 / 0.00e+00 | 20260505 19:30:48.947 | INFO | PerfReport: 21.78%, Download -- 2.17e+01 / 0.00e+00, 7.71e+00 / 0.00e+00 | 20260505 19:30:48.947 | INFO | PerfReport: 31.61%, Compute -- 3.15e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 19:30:48.947 | INFO | PerfReport: 2.37%, LockUpload -- 2.36e+00 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260505 19:30:48.947 | INFO | PerfReport: 1.35%, LockDownload -- 1.35e+00 / 0.00e+00, 2.90e+00 / 0.00e+00 | 20260505 19:30:48.947 | INFO | PerfReport: 20.36%, LockCompute -- 2.03e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 19:30:48.947 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.89e-02 / 0.00e+00 | 20260505 19:30:49.073 | INFO | Starting a round of Baz IO | 20260505 19:30:52.247 | INFO | Populating RTMetrics report | 20260505 19:30:52.247 | INFO | RTMetrics report populated | 20260505 19:30:52.247 | INFO | Opening tmp RTMetrics json file | 20260505 19:30:52.248 | INFO | Serializing RTMetrics to json | 20260505 19:30:52.250 | INFO | Serialization done, swapping tmp file now... | 20260505 19:30:52.250 | INFO | RTMetrics done | 20260505 19:30:52.339 | INFO | Finished a round of Baz IO: Wrote 2.048 GiB in 3.265 seconds | 20260505 19:30:52.339 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.77% / 0.00% / 78.23% | 20260505 19:30:52.339 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.205664% / 0% / 6.35783e-05% | 20260505 19:30:52.339 | INFO | Baz IO overall: | 20260505 19:30:52.339 | INFO | BytesWritten : 3358335651908 | 20260505 19:30:52.339 | INFO | NumEvents : 1589163200836 | 20260505 19:30:54.080 | INFO | Analyzing chunk frames = [5734400,5734912) | 20260505 19:30:54.080 | INFO | MemInfo: { | 20260505 19:30:54.080 | INFO | "gpu" : " 17 GB", | 20260505 19:30:54.080 | INFO | "huge" : "", | 20260505 19:30:54.080 | INFO | "huge_shared" : " 27 GB", | 20260505 19:30:54.080 | INFO | "malloc" : " 14 GB", | 20260505 19:30:54.080 | INFO | "pinned" : " 29 GB" | 20260505 19:30:54.080 | INFO | } | 20260505 19:30:54.080 | INFO | Perf report for the last 5.13504 seconds: | 20260505 19:30:54.080 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:30:54.080 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:54.080 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:54.080 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:54.080 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 19:30:54.080 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:54.080 | INFO | RT_METRICS : 44.2%, 1.0, 23.1ms, 0 | 20260505 19:30:54.080 | INFO | PRE_HQ : 60.3%, 1.8, 55.4ms, 0 | 20260505 19:30:54.080 | INFO | BAZWRITER : 63.1%, 3.7, 120.6ms, 0 | 20260505 19:30:54.081 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 19:30:54.081 | INFO | Uploaded 36GiB to GPU in 2.14979s (16.7458GiB/s) | 20260505 19:30:54.081 | INFO | Downloaded 38GiB to GPU in 2.23857s (16.9751GiB/s) | 20260505 19:30:59.219 | INFO | Analyzing chunk frames = [5734912,5735424) | 20260505 19:30:59.219 | INFO | MemInfo: { | 20260505 19:30:59.219 | INFO | "gpu" : " 17 GB", | 20260505 19:30:59.219 | INFO | "huge" : "", | 20260505 19:30:59.219 | INFO | "huge_shared" : " 27 GB", | 20260505 19:30:59.219 | INFO | "malloc" : " 14 GB", | 20260505 19:30:59.219 | INFO | "pinned" : " 29 GB" | 20260505 19:30:59.219 | INFO | } | 20260505 19:30:59.219 | INFO | Perf report for the last 5.13885 seconds: | 20260505 19:30:59.219 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:30:59.219 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:59.219 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:59.219 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:59.219 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 19:30:59.219 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:59.219 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:59.219 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:30:59.219 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:59.219 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:30:59.219 | INFO | Uploaded 36GiB to GPU in 2.12909s (16.9086GiB/s) | 20260505 19:30:59.220 | INFO | Downloaded 34GiB to GPU in 2.0209s (16.8242GiB/s) | 20260505 19:31:04.359 | INFO | Analyzing chunk frames = [5735424,5735936) | 20260505 19:31:04.359 | INFO | MemInfo: { | 20260505 19:31:04.359 | INFO | "gpu" : " 17 GB", | 20260505 19:31:04.359 | INFO | "huge" : "", | 20260505 19:31:04.359 | INFO | "huge_shared" : " 27 GB", | 20260505 19:31:04.359 | INFO | "malloc" : " 14 GB", | 20260505 19:31:04.359 | INFO | "pinned" : " 29 GB" | 20260505 19:31:04.359 | INFO | } | 20260505 19:31:04.359 | INFO | Perf report for the last 5.14027 seconds: | 20260505 19:31:04.359 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:31:04.359 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:04.359 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:04.359 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:04.360 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 19:31:04.360 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:04.360 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:04.360 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:31:04.360 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:04.360 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:04.360 | INFO | Uploaded 36GiB to GPU in 2.21062s (16.285GiB/s) | 20260505 19:31:04.360 | INFO | Downloaded 34GiB to GPU in 2.0637s (16.4753GiB/s) | 20260505 19:31:09.504 | INFO | Analyzing chunk frames = [5735936,5736448) | 20260505 19:31:09.504 | INFO | MemInfo: { | 20260505 19:31:09.504 | INFO | "gpu" : " 17 GB", | 20260505 19:31:09.504 | INFO | "huge" : "", | 20260505 19:31:09.504 | INFO | "huge_shared" : " 27 GB", | 20260505 19:31:09.504 | INFO | "malloc" : " 14 GB", | 20260505 19:31:09.504 | INFO | "pinned" : " 29 GB" | 20260505 19:31:09.504 | INFO | } | 20260505 19:31:09.504 | INFO | Perf report for the last 5.14503 seconds: | 20260505 19:31:09.504 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:31:09.504 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:09.504 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:09.504 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:09.504 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260505 19:31:09.505 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:09.505 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:09.505 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:31:09.505 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:09.505 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:09.505 | INFO | Uploaded 36GiB to GPU in 2.19796s (16.3788GiB/s) | 20260505 19:31:09.505 | INFO | Downloaded 34GiB to GPU in 2.05884s (16.5142GiB/s) | 20260505 19:31:14.641 | INFO | Analyzing chunk frames = [5736448,5736960) | 20260505 19:31:14.641 | INFO | MemInfo: { | 20260505 19:31:14.641 | INFO | "gpu" : " 17 GB", | 20260505 19:31:14.641 | INFO | "huge" : "", | 20260505 19:31:14.641 | INFO | "huge_shared" : " 27 GB", | 20260505 19:31:14.641 | INFO | "malloc" : " 14 GB", | 20260505 19:31:14.641 | INFO | "pinned" : " 29 GB" | 20260505 19:31:14.641 | INFO | } | 20260505 19:31:14.641 | INFO | Perf report for the last 5.13658 seconds: | 20260505 19:31:14.641 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:31:14.641 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:14.641 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:14.641 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:14.641 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260505 19:31:14.641 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:14.641 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:14.641 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 19:31:14.641 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:14.641 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:14.641 | INFO | Uploaded 36GiB to GPU in 2.12067s (16.9758GiB/s) | 20260505 19:31:14.641 | INFO | Downloaded 34GiB to GPU in 2.05805s (16.5205GiB/s) | 20260505 19:31:19.778 | INFO | Analyzing chunk frames = [5736960,5737472) | 20260505 19:31:19.778 | INFO | MemInfo: { | 20260505 19:31:19.778 | INFO | "gpu" : " 17 GB", | 20260505 19:31:19.778 | INFO | "huge" : "", | 20260505 19:31:19.778 | INFO | "huge_shared" : " 27 GB", | 20260505 19:31:19.778 | INFO | "malloc" : " 14 GB", | 20260505 19:31:19.778 | INFO | "pinned" : " 29 GB" | 20260505 19:31:19.778 | INFO | } | 20260505 19:31:19.778 | INFO | Perf report for the last 5.1367 seconds: | 20260505 19:31:19.778 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:31:19.778 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:19.778 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:19.778 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:19.778 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260505 19:31:19.778 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:19.778 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:19.778 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:31:19.778 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:19.778 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:19.778 | INFO | Uploaded 36GiB to GPU in 2.15578s (16.6993GiB/s) | 20260505 19:31:19.778 | INFO | Downloaded 34GiB to GPU in 2.05959s (16.5081GiB/s) | 20260505 19:31:24.912 | INFO | Analyzing chunk frames = [5737472,5737984) | 20260505 19:31:24.912 | INFO | MemInfo: { | 20260505 19:31:24.912 | INFO | "gpu" : " 17 GB", | 20260505 19:31:24.912 | INFO | "huge" : "", | 20260505 19:31:24.912 | INFO | "huge_shared" : " 27 GB", | 20260505 19:31:24.912 | INFO | "malloc" : " 14 GB", | 20260505 19:31:24.912 | INFO | "pinned" : " 29 GB" | 20260505 19:31:24.912 | INFO | } | 20260505 19:31:24.912 | INFO | Perf report for the last 5.13399 seconds: | 20260505 19:31:24.912 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:31:24.912 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:24.912 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:24.912 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:24.912 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 19:31:24.912 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:24.912 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:24.912 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 19:31:24.912 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:24.912 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:24.912 | INFO | Uploaded 36GiB to GPU in 2.117s (17.0052GiB/s) | 20260505 19:31:24.912 | INFO | Downloaded 34GiB to GPU in 2.17409s (15.6387GiB/s) | 20260505 19:31:30.058 | INFO | Analyzing chunk frames = [5737984,5738496) | 20260505 19:31:30.058 | INFO | MemInfo: { | 20260505 19:31:30.058 | INFO | "gpu" : " 17 GB", | 20260505 19:31:30.058 | INFO | "huge" : "", | 20260505 19:31:30.058 | INFO | "huge_shared" : " 27 GB", | 20260505 19:31:30.058 | INFO | "malloc" : " 14 GB", | 20260505 19:31:30.058 | INFO | "pinned" : " 29 GB" | 20260505 19:31:30.058 | INFO | } | 20260505 19:31:30.058 | INFO | Perf report for the last 5.14658 seconds: | 20260505 19:31:30.058 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:31:30.058 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:30.058 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:30.058 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:30.058 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 19:31:30.058 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:30.058 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:30.058 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:31:30.058 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:30.059 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:30.059 | INFO | Uploaded 36GiB to GPU in 2.19822s (16.3769GiB/s) | 20260505 19:31:30.059 | INFO | Downloaded 34GiB to GPU in 2.04189s (16.6512GiB/s) | 20260505 19:31:30.187 | INFO | Starting a round of Baz IO | 20260505 19:31:33.362 | INFO | Populating RTMetrics report | 20260505 19:31:33.362 | INFO | RTMetrics report populated | 20260505 19:31:33.362 | INFO | Opening tmp RTMetrics json file | 20260505 19:31:33.363 | INFO | Serializing RTMetrics to json | 20260505 19:31:33.364 | INFO | Serialization done, swapping tmp file now... | 20260505 19:31:33.364 | INFO | RTMetrics done | 20260505 19:31:33.450 | INFO | Finished a round of Baz IO: Wrote 2.048 GiB in 3.262 seconds | 20260505 19:31:33.450 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.77% / 0.00% / 78.23% | 20260505 19:31:33.450 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.205076% / 0% / 6.35783e-05% | 20260505 19:31:33.450 | INFO | Baz IO overall: | 20260505 19:31:33.450 | INFO | BytesWritten : 3360534843856 | 20260505 19:31:33.450 | INFO | NumEvents : 1590184694148 | 20260505 19:31:35.200 | INFO | Analyzing chunk frames = [5738496,5739008) | 20260505 19:31:35.200 | INFO | MemInfo: { | 20260505 19:31:35.200 | INFO | "gpu" : " 17 GB", | 20260505 19:31:35.200 | INFO | "huge" : "", | 20260505 19:31:35.200 | INFO | "huge_shared" : " 27 GB", | 20260505 19:31:35.201 | INFO | "malloc" : " 14 GB", | 20260505 19:31:35.201 | INFO | "pinned" : " 29 GB" | 20260505 19:31:35.201 | INFO | } | 20260505 19:31:35.201 | INFO | Perf report for the last 5.14217 seconds: | 20260505 19:31:35.201 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:31:35.201 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:35.201 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:35.201 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:35.201 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 19:31:35.201 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:35.201 | INFO | RT_METRICS : 44.1%, 1.0, 23.2ms, 0 | 20260505 19:31:35.201 | INFO | PRE_HQ : 60.1%, 1.8, 55.3ms, 0 | 20260505 19:31:35.201 | INFO | BAZWRITER : 62.9%, 3.7, 120.2ms, 0 | 20260505 19:31:35.201 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 19:31:35.201 | INFO | Uploaded 36GiB to GPU in 2.16397s (16.6361GiB/s) | 20260505 19:31:35.201 | INFO | Downloaded 38GiB to GPU in 2.24235s (16.9465GiB/s) | 20260505 19:31:40.341 | INFO | Analyzing chunk frames = [5739008,5739520) | 20260505 19:31:40.341 | INFO | MemInfo: { | 20260505 19:31:40.341 | INFO | "gpu" : " 17 GB", | 20260505 19:31:40.341 | INFO | "huge" : "", | 20260505 19:31:40.341 | INFO | "huge_shared" : " 27 GB", | 20260505 19:31:40.341 | INFO | "malloc" : " 14 GB", | 20260505 19:31:40.341 | INFO | "pinned" : " 29 GB" | 20260505 19:31:40.341 | INFO | } | 20260505 19:31:40.341 | INFO | Perf report for the last 5.14071 seconds: | 20260505 19:31:40.341 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:31:40.341 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:40.341 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:40.341 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:40.341 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260505 19:31:40.341 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:40.341 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:40.341 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:31:40.341 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:40.341 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:40.341 | INFO | Uploaded 36GiB to GPU in 2.15366s (16.7158GiB/s) | 20260505 19:31:40.342 | INFO | Downloaded 34GiB to GPU in 2.05119s (16.5757GiB/s) | 20260505 19:31:45.475 | INFO | Analyzing chunk frames = [5739520,5740032) | 20260505 19:31:45.475 | INFO | MemInfo: { | 20260505 19:31:45.475 | INFO | "gpu" : " 17 GB", | 20260505 19:31:45.475 | INFO | "huge" : "", | 20260505 19:31:45.475 | INFO | "huge_shared" : " 27 GB", | 20260505 19:31:45.475 | INFO | "malloc" : " 14 GB", | 20260505 19:31:45.475 | INFO | "pinned" : " 29 GB" | 20260505 19:31:45.475 | INFO | } | 20260505 19:31:45.475 | INFO | Perf report for the last 5.1336 seconds: | 20260505 19:31:45.475 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:31:45.475 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:45.475 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:45.475 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:45.475 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260505 19:31:45.475 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:45.475 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:45.475 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:31:45.475 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:45.475 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:45.475 | INFO | Uploaded 36GiB to GPU in 2.1271s (16.9244GiB/s) | 20260505 19:31:45.475 | INFO | Downloaded 34GiB to GPU in 2.02324s (16.8047GiB/s) | 20260505 19:31:50.612 | INFO | Analyzing chunk frames = [5740032,5740544) | 20260505 19:31:50.612 | INFO | MemInfo: { | 20260505 19:31:50.612 | INFO | "gpu" : " 17 GB", | 20260505 19:31:50.612 | INFO | "huge" : "", | 20260505 19:31:50.612 | INFO | "huge_shared" : " 27 GB", | 20260505 19:31:50.612 | INFO | "malloc" : " 14 GB", | 20260505 19:31:50.612 | INFO | "pinned" : " 29 GB" | 20260505 19:31:50.612 | INFO | } | 20260505 19:31:50.613 | INFO | Perf report for the last 5.13771 seconds: | 20260505 19:31:50.613 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:31:50.613 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:50.613 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:50.613 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:50.613 | INFO | ANALYSIS : 62.7%, 3.0, 97.3ms, 0 | 20260505 19:31:50.613 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:50.613 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:50.613 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:31:50.613 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:50.613 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:50.613 | INFO | Uploaded 36GiB to GPU in 2.22103s (16.2087GiB/s) | 20260505 19:31:50.613 | INFO | Downloaded 34GiB to GPU in 2.11901s (16.0453GiB/s) | 20260505 19:31:55.755 | INFO | Analyzing chunk frames = [5740544,5741056) | 20260505 19:31:55.755 | INFO | MemInfo: { | 20260505 19:31:55.755 | INFO | "gpu" : " 17 GB", | 20260505 19:31:55.755 | INFO | "huge" : "", | 20260505 19:31:55.755 | INFO | "huge_shared" : " 27 GB", | 20260505 19:31:55.755 | INFO | "malloc" : " 14 GB", | 20260505 19:31:55.755 | INFO | "pinned" : " 29 GB" | 20260505 19:31:55.755 | INFO | } | 20260505 19:31:55.755 | INFO | Perf report for the last 5.14247 seconds: | 20260505 19:31:55.755 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:31:55.755 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:55.755 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:55.755 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:55.755 | INFO | ANALYSIS : 61.8%, 3.0, 96.6ms, 0 | 20260505 19:31:55.755 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:55.755 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:55.755 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:31:55.755 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:55.755 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:31:55.755 | INFO | Uploaded 36GiB to GPU in 2.13682s (16.8474GiB/s) | 20260505 19:31:55.755 | INFO | Downloaded 34GiB to GPU in 2.05949s (16.509GiB/s) | 20260505 19:32:00.896 | INFO | Analyzing chunk frames = [5741056,5741568) | 20260505 19:32:00.896 | INFO | MemInfo: { | 20260505 19:32:00.896 | INFO | "gpu" : " 17 GB", | 20260505 19:32:00.896 | INFO | "huge" : "", | 20260505 19:32:00.896 | INFO | "huge_shared" : " 27 GB", | 20260505 19:32:00.896 | INFO | "malloc" : " 14 GB", | 20260505 19:32:00.897 | INFO | "pinned" : " 29 GB" | 20260505 19:32:00.897 | INFO | } | 20260505 19:32:00.897 | INFO | Perf report for the last 5.14146 seconds: | 20260505 19:32:00.897 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:32:00.897 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:00.897 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:00.897 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:00.897 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260505 19:32:00.897 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:00.897 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:00.897 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:32:00.897 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:00.897 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:00.897 | INFO | Uploaded 36GiB to GPU in 2.15533s (16.7028GiB/s) | 20260505 19:32:00.897 | INFO | Downloaded 34GiB to GPU in 2.0578s (16.5225GiB/s) | 20260505 19:32:06.032 | INFO | Analyzing chunk frames = [5741568,5742080) | 20260505 19:32:06.032 | INFO | MemInfo: { | 20260505 19:32:06.032 | INFO | "gpu" : " 17 GB", | 20260505 19:32:06.032 | INFO | "huge" : "", | 20260505 19:32:06.032 | INFO | "huge_shared" : " 27 GB", | 20260505 19:32:06.032 | INFO | "malloc" : " 14 GB", | 20260505 19:32:06.032 | INFO | "pinned" : " 29 GB" | 20260505 19:32:06.032 | INFO | } | 20260505 19:32:06.032 | INFO | Perf report for the last 5.13563 seconds: | 20260505 19:32:06.032 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:32:06.032 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:06.032 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:06.032 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:06.032 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 19:32:06.032 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:06.032 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:06.032 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 19:32:06.032 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:06.032 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:06.033 | INFO | Uploaded 36GiB to GPU in 2.12188s (16.9661GiB/s) | 20260505 19:32:06.033 | INFO | Downloaded 34GiB to GPU in 2.13185s (15.9486GiB/s) | 20260505 19:32:11.170 | INFO | Analyzing chunk frames = [5742080,5742592) | 20260505 19:32:11.170 | INFO | MemInfo: { | 20260505 19:32:11.170 | INFO | "gpu" : " 17 GB", | 20260505 19:32:11.170 | INFO | "huge" : "", | 20260505 19:32:11.170 | INFO | "huge_shared" : " 27 GB", | 20260505 19:32:11.170 | INFO | "malloc" : " 14 GB", | 20260505 19:32:11.170 | INFO | "pinned" : " 29 GB" | 20260505 19:32:11.170 | INFO | } | 20260505 19:32:11.170 | INFO | Perf report for the last 5.13789 seconds: | 20260505 19:32:11.170 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:32:11.170 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:11.170 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:11.170 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:11.170 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 19:32:11.170 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:11.170 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:11.170 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 19:32:11.170 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:11.170 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:11.170 | INFO | Uploaded 36GiB to GPU in 2.20168s (16.3512GiB/s) | 20260505 19:32:11.170 | INFO | Downloaded 34GiB to GPU in 2.04196s (16.6507GiB/s) | 20260505 19:32:11.171 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 19:32:11.171 | INFO | Total mem usage: 15.3 GiB | 20260505 19:32:11.171 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 19:32:11.171 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.05%) | 20260505 19:32:11.171 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 19:32:11.171 | INFO | High Water Breakdown: | 20260505 19:32:11.171 | INFO | 61% : 8.66 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 19:32:11.171 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 19:32:11.171 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 19:32:11.171 | INFO | ------------------------------------------ | 20260505 19:32:11.171 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:32:11.171 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 19:32:11.171 | INFO | Total mem usage: 26.5 GiB | 20260505 19:32:11.171 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 19:32:11.171 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 19:32:11.171 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 19:32:11.171 | INFO | High Water Breakdown: | 20260505 19:32:11.171 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:32:11.171 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:32:11.171 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:32:11.171 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:32:11.171 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:32:11.171 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:32:11.171 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 19:32:11.171 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 19:32:11.171 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:32:11.171 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 19:32:11.171 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 19:32:11.171 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 19:32:11.171 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 19:32:11.171 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 19:32:11.171 | INFO | ------------------------------------------ | 20260505 19:32:11.171 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:32:11.171 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 19:32:11.171 | INFO | Total mem usage: 25.6 GiB | 20260505 19:32:11.171 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 19:32:11.171 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 19:32:11.171 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 19:32:11.171 | INFO | High Water Breakdown: | 20260505 19:32:11.171 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 19:32:11.171 | INFO | ------------------------------------------ | 20260505 19:32:11.171 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:32:11.171 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 19:32:11.171 | INFO | Total mem usage: 15.7 GiB | 20260505 19:32:11.171 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 19:32:11.171 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 19:32:11.171 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 19:32:11.171 | INFO | High Water Breakdown: | 20260505 19:32:11.171 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 19:32:11.171 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 19:32:11.171 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 19:32:11.171 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 19:32:11.171 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 19:32:11.171 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 19:32:11.171 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 19:32:11.171 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 19:32:11.171 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 19:32:11.171 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:32:11.171 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 19:32:11.171 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:32:11.171 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 19:32:11.171 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 19:32:11.171 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 19:32:11.171 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 19:32:11.171 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:32:11.171 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 19:32:11.171 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:32:11.171 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:32:11.171 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:32:11.171 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 19:32:11.171 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 19:32:11.171 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:32:11.171 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 19:32:11.171 | INFO | ------------------------------------------ | 20260505 19:32:11.172 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:32:11.172 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 19:32:11.172 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:32:11.172 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:32:11.172 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.46e+01, 1.35e+01 / 2.02e+00 | 20260505 19:32:11.172 | INFO | PerfReport: 0.29%, AggregateStats -- 9.24e-02 / 1.02e-01, 1.06e-02 / 3.79e-03 | 20260505 19:32:11.172 | INFO | PerfReport: 11.80%, Baseline -- 3.71e+00 / 4.55e+00, 5.57e-01 / 9.60e-02 | 20260505 19:32:11.172 | INFO | PerfReport: 3.26%, Histogram -- 1.03e+00 / 1.54e+00, 1.87e-01 / 6.73e-02 | 20260505 19:32:11.172 | INFO | PerfReport: 15.98%, DME -- 4.02e+01 / 4.44e+01, 4.19e+00 / 2.01e+00 | 20260505 19:32:11.172 | INFO | PerfReport: 56.88%, FrameLabeling -- 1.79e+01 / 1.93e+01, 1.33e+00 / 9.77e-01 | 20260505 19:32:11.172 | INFO | PerfReport: 7.33%, PulseAccumulating -- 2.30e+00 / 2.89e+00, 4.44e-01 / 1.57e-01 | 20260505 19:32:11.172 | INFO | PerfReport: 3.30%, Metrics -- 1.04e+00 / 1.36e+00, 2.86e-01 / 1.82e-01 | 20260505 19:32:11.172 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.60e-01 / 4.27e-01, 6.36e-02 / 4.24e-02 | 20260505 19:32:11.172 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 19:32:11.172 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:32:11.172 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:32:11.172 | INFO | PerfReport: Full Execution -- 9.97e+01 / -nan, 2.57e+01 / 0.00e+00 | 20260505 19:32:11.172 | INFO | PerfReport: 22.58%, Upload -- 2.25e+01 / 0.00e+00, 7.85e+00 / 0.00e+00 | 20260505 19:32:11.172 | INFO | PerfReport: 21.85%, Download -- 2.18e+01 / 0.00e+00, 7.78e+00 / 0.00e+00 | 20260505 19:32:11.172 | INFO | PerfReport: 31.54%, Compute -- 3.14e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 19:32:11.172 | INFO | PerfReport: 2.36%, LockUpload -- 2.35e+00 / 0.00e+00, 7.95e+00 / 0.00e+00 | 20260505 19:32:11.172 | INFO | PerfReport: 1.39%, LockDownload -- 1.38e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260505 19:32:11.172 | INFO | PerfReport: 20.28%, LockCompute -- 2.02e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 19:32:11.172 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 2.31e-02 / 0.00e+00 | 20260505 19:32:11.304 | INFO | Starting a round of Baz IO | 20260505 19:32:14.478 | INFO | Populating RTMetrics report | 20260505 19:32:14.478 | INFO | RTMetrics report populated | 20260505 19:32:14.478 | INFO | Opening tmp RTMetrics json file | 20260505 19:32:14.480 | INFO | Serializing RTMetrics to json | 20260505 19:32:14.482 | INFO | Serialization done, swapping tmp file now... | 20260505 19:32:14.482 | INFO | RTMetrics done | 20260505 19:32:14.569 | INFO | Finished a round of Baz IO: Wrote 2.048 GiB in 3.265 seconds | 20260505 19:32:14.569 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.77% / 0.00% / 78.23% | 20260505 19:32:14.569 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.205433% / 0% / 7.54992e-05% | 20260505 19:32:14.569 | INFO | Baz IO overall: | 20260505 19:32:14.569 | INFO | BytesWritten : 3362734308259 | 20260505 19:32:14.569 | INFO | NumEvents : 1591206584687 | 20260505 19:32:16.315 | INFO | Analyzing chunk frames = [5742592,5743104) | 20260505 19:32:16.315 | INFO | MemInfo: { | 20260505 19:32:16.315 | INFO | "gpu" : " 17 GB", | 20260505 19:32:16.315 | INFO | "huge" : "", | 20260505 19:32:16.315 | INFO | "huge_shared" : " 27 GB", | 20260505 19:32:16.315 | INFO | "malloc" : " 14 GB", | 20260505 19:32:16.315 | INFO | "pinned" : " 29 GB" | 20260505 19:32:16.315 | INFO | } | 20260505 19:32:16.315 | INFO | Perf report for the last 5.14472 seconds: | 20260505 19:32:16.315 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:32:16.315 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:16.315 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:16.315 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:16.315 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 19:32:16.315 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:16.315 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260505 19:32:16.315 | INFO | PRE_HQ : 60.3%, 1.8, 55.4ms, 0 | 20260505 19:32:16.315 | INFO | BAZWRITER : 62.9%, 3.8, 121.9ms, 0 | 20260505 19:32:16.315 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 19:32:16.315 | INFO | Uploaded 36GiB to GPU in 2.16704s (16.6125GiB/s) | 20260505 19:32:16.315 | INFO | Downloaded 38GiB to GPU in 2.25091s (16.8821GiB/s) | 20260505 19:32:21.460 | INFO | Analyzing chunk frames = [5743104,5743616) | 20260505 19:32:21.460 | INFO | MemInfo: { | 20260505 19:32:21.460 | INFO | "gpu" : " 17 GB", | 20260505 19:32:21.460 | INFO | "huge" : "", | 20260505 19:32:21.460 | INFO | "huge_shared" : " 27 GB", | 20260505 19:32:21.460 | INFO | "malloc" : " 14 GB", | 20260505 19:32:21.460 | INFO | "pinned" : " 29 GB" | 20260505 19:32:21.460 | INFO | } | 20260505 19:32:21.460 | INFO | Perf report for the last 5.14535 seconds: | 20260505 19:32:21.460 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:32:21.460 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:21.460 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:21.460 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:21.460 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260505 19:32:21.460 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:21.460 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:21.460 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 19:32:21.460 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:21.460 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:21.461 | INFO | Uploaded 36GiB to GPU in 2.15206s (16.7282GiB/s) | 20260505 19:32:21.461 | INFO | Downloaded 34GiB to GPU in 2.04794s (16.6021GiB/s) | 20260505 19:32:26.589 | INFO | Analyzing chunk frames = [5743616,5744128) | 20260505 19:32:26.589 | INFO | MemInfo: { | 20260505 19:32:26.589 | INFO | "gpu" : " 17 GB", | 20260505 19:32:26.589 | INFO | "huge" : "", | 20260505 19:32:26.589 | INFO | "huge_shared" : " 27 GB", | 20260505 19:32:26.589 | INFO | "malloc" : " 14 GB", | 20260505 19:32:26.589 | INFO | "pinned" : " 29 GB" | 20260505 19:32:26.589 | INFO | } | 20260505 19:32:26.589 | INFO | Perf report for the last 5.12901 seconds: | 20260505 19:32:26.589 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:32:26.589 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:26.589 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:26.589 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:26.589 | INFO | ANALYSIS : 62.1%, 3.0, 96.5ms, 0 | 20260505 19:32:26.589 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:26.589 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:26.589 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260505 19:32:26.589 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:26.589 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:26.590 | INFO | Uploaded 36GiB to GPU in 2.13093s (16.894GiB/s) | 20260505 19:32:26.590 | INFO | Downloaded 34GiB to GPU in 2.02673s (16.7758GiB/s) | 20260505 19:32:31.726 | INFO | Analyzing chunk frames = [5744128,5744640) | 20260505 19:32:31.726 | INFO | MemInfo: { | 20260505 19:32:31.726 | INFO | "gpu" : " 17 GB", | 20260505 19:32:31.726 | INFO | "huge" : "", | 20260505 19:32:31.726 | INFO | "huge_shared" : " 27 GB", | 20260505 19:32:31.726 | INFO | "malloc" : " 14 GB", | 20260505 19:32:31.726 | INFO | "pinned" : " 29 GB" | 20260505 19:32:31.726 | INFO | } | 20260505 19:32:31.726 | INFO | Perf report for the last 5.13676 seconds: | 20260505 19:32:31.726 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:32:31.726 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:31.726 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:31.726 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:31.726 | INFO | ANALYSIS : 62.7%, 3.0, 97.4ms, 0 | 20260505 19:32:31.726 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:31.726 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:31.726 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 19:32:31.726 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:31.726 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:31.726 | INFO | Uploaded 36GiB to GPU in 2.22339s (16.1915GiB/s) | 20260505 19:32:31.726 | INFO | Downloaded 34GiB to GPU in 2.12504s (15.9997GiB/s) | 20260505 19:32:36.869 | INFO | Analyzing chunk frames = [5744640,5745152) | 20260505 19:32:36.869 | INFO | MemInfo: { | 20260505 19:32:36.869 | INFO | "gpu" : " 17 GB", | 20260505 19:32:36.869 | INFO | "huge" : "", | 20260505 19:32:36.869 | INFO | "huge_shared" : " 27 GB", | 20260505 19:32:36.869 | INFO | "malloc" : " 14 GB", | 20260505 19:32:36.869 | INFO | "pinned" : " 29 GB" | 20260505 19:32:36.869 | INFO | } | 20260505 19:32:36.869 | INFO | Perf report for the last 5.14316 seconds: | 20260505 19:32:36.869 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:32:36.869 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:36.869 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:36.869 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:36.869 | INFO | ANALYSIS : 61.9%, 3.0, 96.8ms, 0 | 20260505 19:32:36.869 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:36.869 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:36.869 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 19:32:36.869 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:36.869 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:36.869 | INFO | Uploaded 36GiB to GPU in 2.14388s (16.792GiB/s) | 20260505 19:32:36.869 | INFO | Downloaded 34GiB to GPU in 2.05839s (16.5178GiB/s) | 20260505 19:32:42.012 | INFO | Analyzing chunk frames = [5745152,5745664) | 20260505 19:32:42.012 | INFO | MemInfo: { | 20260505 19:32:42.012 | INFO | "gpu" : " 17 GB", | 20260505 19:32:42.012 | INFO | "huge" : "", | 20260505 19:32:42.012 | INFO | "huge_shared" : " 27 GB", | 20260505 19:32:42.012 | INFO | "malloc" : " 14 GB", | 20260505 19:32:42.012 | INFO | "pinned" : " 29 GB" | 20260505 19:32:42.012 | INFO | } | 20260505 19:32:42.012 | INFO | Perf report for the last 5.14277 seconds: | 20260505 19:32:42.012 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:32:42.012 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:42.012 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:42.012 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:42.012 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260505 19:32:42.012 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:42.012 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:42.012 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:32:42.012 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:42.012 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:42.012 | INFO | Uploaded 36GiB to GPU in 2.14064s (16.8174GiB/s) | 20260505 19:32:42.012 | INFO | Downloaded 34GiB to GPU in 2.05262s (16.5642GiB/s) | 20260505 19:32:47.159 | INFO | Analyzing chunk frames = [5745664,5746176) | 20260505 19:32:47.159 | INFO | MemInfo: { | 20260505 19:32:47.159 | INFO | "gpu" : " 17 GB", | 20260505 19:32:47.159 | INFO | "huge" : "", | 20260505 19:32:47.159 | INFO | "huge_shared" : " 27 GB", | 20260505 19:32:47.159 | INFO | "malloc" : " 14 GB", | 20260505 19:32:47.159 | INFO | "pinned" : " 29 GB" | 20260505 19:32:47.159 | INFO | } | 20260505 19:32:47.159 | INFO | Perf report for the last 5.14717 seconds: | 20260505 19:32:47.159 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:32:47.159 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:47.159 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:47.159 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:47.159 | INFO | ANALYSIS : 62.9%, 3.0, 98.4ms, 0 | 20260505 19:32:47.159 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:47.159 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:47.159 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 19:32:47.159 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:47.159 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:47.159 | INFO | Uploaded 36GiB to GPU in 2.11413s (17.0282GiB/s) | 20260505 19:32:47.159 | INFO | Downloaded 34GiB to GPU in 2.13863s (15.8981GiB/s) | 20260505 19:32:52.288 | INFO | Analyzing chunk frames = [5746176,5746688) | 20260505 19:32:52.288 | INFO | MemInfo: { | 20260505 19:32:52.288 | INFO | "gpu" : " 17 GB", | 20260505 19:32:52.288 | INFO | "huge" : "", | 20260505 19:32:52.288 | INFO | "huge_shared" : " 27 GB", | 20260505 19:32:52.288 | INFO | "malloc" : " 14 GB", | 20260505 19:32:52.288 | INFO | "pinned" : " 29 GB" | 20260505 19:32:52.288 | INFO | } | 20260505 19:32:52.288 | INFO | Perf report for the last 5.12871 seconds: | 20260505 19:32:52.288 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:32:52.288 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:52.288 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:52.288 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:52.288 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260505 19:32:52.288 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:52.288 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:52.288 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:32:52.288 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:52.288 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:52.288 | INFO | Uploaded 36GiB to GPU in 2.11213s (17.0444GiB/s) | 20260505 19:32:52.288 | INFO | Downloaded 34GiB to GPU in 1.99877s (17.0105GiB/s) | 20260505 19:32:52.424 | INFO | Starting a round of Baz IO | 20260505 19:32:55.597 | INFO | Populating RTMetrics report | 20260505 19:32:55.597 | INFO | RTMetrics report populated | 20260505 19:32:55.597 | INFO | Opening tmp RTMetrics json file | 20260505 19:32:55.598 | INFO | Serializing RTMetrics to json | 20260505 19:32:55.600 | INFO | Serialization done, swapping tmp file now... | 20260505 19:32:55.600 | INFO | RTMetrics done | 20260505 19:32:55.685 | INFO | Finished a round of Baz IO: Wrote 2.047 GiB in 3.261 seconds | 20260505 19:32:55.685 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.77% / 0.00% / 78.23% | 20260505 19:32:55.685 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.205056% / 3.97364e-06% / 0.000107288% | 20260505 19:32:55.685 | INFO | Baz IO overall: | 20260505 19:32:55.685 | INFO | BytesWritten : 3364931981865 | 20260505 19:32:55.686 | INFO | NumEvents : 1592227367060 | 20260505 19:32:57.420 | INFO | Analyzing chunk frames = [5746688,5747200) | 20260505 19:32:57.420 | INFO | MemInfo: { | 20260505 19:32:57.420 | INFO | "gpu" : " 17 GB", | 20260505 19:32:57.420 | INFO | "huge" : "", | 20260505 19:32:57.420 | INFO | "huge_shared" : " 27 GB", | 20260505 19:32:57.420 | INFO | "malloc" : " 14 GB", | 20260505 19:32:57.420 | INFO | "pinned" : " 29 GB" | 20260505 19:32:57.420 | INFO | } | 20260505 19:32:57.420 | INFO | Perf report for the last 5.13207 seconds: | 20260505 19:32:57.420 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:32:57.420 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:57.420 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:57.420 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:57.420 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260505 19:32:57.420 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:32:57.420 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260505 19:32:57.420 | INFO | PRE_HQ : 60.4%, 1.8, 55.4ms, 0 | 20260505 19:32:57.420 | INFO | BAZWRITER : 63.1%, 3.7, 120.3ms, 0 | 20260505 19:32:57.420 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 19:32:57.420 | INFO | Uploaded 36GiB to GPU in 2.16716s (16.6116GiB/s) | 20260505 19:32:57.420 | INFO | Downloaded 38GiB to GPU in 2.24715s (16.9103GiB/s) | 20260505 19:33:02.569 | INFO | Analyzing chunk frames = [5747200,5747712) | 20260505 19:33:02.569 | INFO | MemInfo: { | 20260505 19:33:02.569 | INFO | "gpu" : " 17 GB", | 20260505 19:33:02.569 | INFO | "huge" : "", | 20260505 19:33:02.569 | INFO | "huge_shared" : " 27 GB", | 20260505 19:33:02.569 | INFO | "malloc" : " 14 GB", | 20260505 19:33:02.569 | INFO | "pinned" : " 29 GB" | 20260505 19:33:02.569 | INFO | } | 20260505 19:33:02.569 | INFO | Perf report for the last 5.14944 seconds: | 20260505 19:33:02.570 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:33:02.570 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:02.570 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:02.570 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:02.570 | INFO | ANALYSIS : 62.5%, 3.0, 97.8ms, 0 | 20260505 19:33:02.570 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:02.570 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:02.570 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:33:02.570 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:02.570 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:02.570 | INFO | Uploaded 36GiB to GPU in 2.13302s (16.8775GiB/s) | 20260505 19:33:02.570 | INFO | Downloaded 34GiB to GPU in 2.02723s (16.7717GiB/s) | 20260505 19:33:07.705 | INFO | Analyzing chunk frames = [5747712,5748224) | 20260505 19:33:07.705 | INFO | MemInfo: { | 20260505 19:33:07.705 | INFO | "gpu" : " 17 GB", | 20260505 19:33:07.705 | INFO | "huge" : "", | 20260505 19:33:07.705 | INFO | "huge_shared" : " 27 GB", | 20260505 19:33:07.706 | INFO | "malloc" : " 14 GB", | 20260505 19:33:07.706 | INFO | "pinned" : " 29 GB" | 20260505 19:33:07.706 | INFO | } | 20260505 19:33:07.706 | INFO | Perf report for the last 5.13613 seconds: | 20260505 19:33:07.706 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:33:07.706 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:07.706 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:07.706 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:07.706 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260505 19:33:07.706 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:07.706 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:07.706 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:33:07.706 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:07.706 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:07.706 | INFO | Uploaded 36GiB to GPU in 2.1228s (16.9588GiB/s) | 20260505 19:33:07.706 | INFO | Downloaded 34GiB to GPU in 2.01576s (16.8671GiB/s) | 20260505 19:33:12.841 | INFO | Analyzing chunk frames = [5748224,5748736) | 20260505 19:33:12.841 | INFO | MemInfo: { | 20260505 19:33:12.841 | INFO | "gpu" : " 17 GB", | 20260505 19:33:12.841 | INFO | "huge" : "", | 20260505 19:33:12.841 | INFO | "huge_shared" : " 27 GB", | 20260505 19:33:12.841 | INFO | "malloc" : " 14 GB", | 20260505 19:33:12.841 | INFO | "pinned" : " 29 GB" | 20260505 19:33:12.841 | INFO | } | 20260505 19:33:12.841 | INFO | Perf report for the last 5.13549 seconds: | 20260505 19:33:12.841 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:33:12.841 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:12.841 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:12.841 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:12.841 | INFO | ANALYSIS : 62.3%, 3.0, 96.7ms, 0 | 20260505 19:33:12.841 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:12.841 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:12.841 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:33:12.841 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:12.841 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:12.841 | INFO | Uploaded 36GiB to GPU in 2.18238s (16.4957GiB/s) | 20260505 19:33:12.841 | INFO | Downloaded 34GiB to GPU in 2.06518s (16.4635GiB/s) | 20260505 19:33:17.980 | INFO | Analyzing chunk frames = [5748736,5749248) | 20260505 19:33:17.980 | INFO | MemInfo: { | 20260505 19:33:17.980 | INFO | "gpu" : " 17 GB", | 20260505 19:33:17.980 | INFO | "huge" : "", | 20260505 19:33:17.980 | INFO | "huge_shared" : " 27 GB", | 20260505 19:33:17.980 | INFO | "malloc" : " 14 GB", | 20260505 19:33:17.981 | INFO | "pinned" : " 29 GB" | 20260505 19:33:17.981 | INFO | } | 20260505 19:33:17.981 | INFO | Perf report for the last 5.13961 seconds: | 20260505 19:33:17.981 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:33:17.981 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:17.981 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:17.981 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:17.981 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 19:33:17.981 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:17.981 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:17.981 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 19:33:17.981 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:17.981 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:17.981 | INFO | Uploaded 36GiB to GPU in 2.14543s (16.7799GiB/s) | 20260505 19:33:17.981 | INFO | Downloaded 34GiB to GPU in 2.0595s (16.5089GiB/s) | 20260505 19:33:23.125 | INFO | Analyzing chunk frames = [5749248,5749760) | 20260505 19:33:23.125 | INFO | MemInfo: { | 20260505 19:33:23.125 | INFO | "gpu" : " 17 GB", | 20260505 19:33:23.125 | INFO | "huge" : "", | 20260505 19:33:23.125 | INFO | "huge_shared" : " 27 GB", | 20260505 19:33:23.125 | INFO | "malloc" : " 14 GB", | 20260505 19:33:23.125 | INFO | "pinned" : " 29 GB" | 20260505 19:33:23.125 | INFO | } | 20260505 19:33:23.125 | INFO | Perf report for the last 5.14411 seconds: | 20260505 19:33:23.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:33:23.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:23.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:23.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:23.125 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260505 19:33:23.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:23.125 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:23.125 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 19:33:23.125 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:23.125 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:23.125 | INFO | Uploaded 36GiB to GPU in 2.13807s (16.8376GiB/s) | 20260505 19:33:23.125 | INFO | Downloaded 34GiB to GPU in 2.05237s (16.5662GiB/s) | 20260505 19:33:28.266 | INFO | Analyzing chunk frames = [5749760,5750272) | 20260505 19:33:28.266 | INFO | MemInfo: { | 20260505 19:33:28.266 | INFO | "gpu" : " 17 GB", | 20260505 19:33:28.266 | INFO | "huge" : "", | 20260505 19:33:28.266 | INFO | "huge_shared" : " 27 GB", | 20260505 19:33:28.266 | INFO | "malloc" : " 14 GB", | 20260505 19:33:28.266 | INFO | "pinned" : " 29 GB" | 20260505 19:33:28.266 | INFO | } | 20260505 19:33:28.266 | INFO | Perf report for the last 5.14111 seconds: | 20260505 19:33:28.266 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:33:28.266 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:28.266 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:28.266 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:28.266 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 19:33:28.266 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:28.266 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:28.266 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 19:33:28.266 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:28.266 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:28.266 | INFO | Uploaded 36GiB to GPU in 2.11166s (17.0482GiB/s) | 20260505 19:33:28.266 | INFO | Downloaded 34GiB to GPU in 2.13571s (15.9198GiB/s) | 20260505 19:33:33.403 | INFO | Analyzing chunk frames = [5750272,5750784) | 20260505 19:33:33.403 | INFO | MemInfo: { | 20260505 19:33:33.403 | INFO | "gpu" : " 17 GB", | 20260505 19:33:33.403 | INFO | "huge" : "", | 20260505 19:33:33.403 | INFO | "huge_shared" : " 27 GB", | 20260505 19:33:33.403 | INFO | "malloc" : " 14 GB", | 20260505 19:33:33.403 | INFO | "pinned" : " 29 GB" | 20260505 19:33:33.403 | INFO | } | 20260505 19:33:33.403 | INFO | Perf report for the last 5.1373 seconds: | 20260505 19:33:33.403 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:33:33.403 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:33.403 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:33.403 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:33.403 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260505 19:33:33.403 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:33.403 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:33.403 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 19:33:33.403 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:33.403 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:33.404 | INFO | Uploaded 36GiB to GPU in 2.11822s (16.9954GiB/s) | 20260505 19:33:33.404 | INFO | Downloaded 34GiB to GPU in 2.00845s (16.9285GiB/s) | 20260505 19:33:33.404 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 19:33:33.404 | INFO | Total mem usage: 15.3 GiB | 20260505 19:33:33.404 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 19:33:33.404 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.19%) | 20260505 19:33:33.404 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 19:33:33.404 | INFO | High Water Breakdown: | 20260505 19:33:33.404 | INFO | 61% : 8.64 GiB ( 90.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 19:33:33.404 | INFO | 38.9% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 19:33:33.404 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 19:33:33.404 | INFO | ------------------------------------------ | 20260505 19:33:33.404 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:33:33.404 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 19:33:33.404 | INFO | Total mem usage: 26.5 GiB | 20260505 19:33:33.404 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 19:33:33.404 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 19:33:33.404 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 19:33:33.404 | INFO | High Water Breakdown: | 20260505 19:33:33.404 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:33:33.404 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:33:33.404 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:33:33.404 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:33:33.404 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:33:33.404 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:33:33.404 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 19:33:33.404 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 19:33:33.404 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:33:33.404 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 19:33:33.404 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 19:33:33.404 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 19:33:33.404 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 19:33:33.404 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 19:33:33.404 | INFO | ------------------------------------------ | 20260505 19:33:33.404 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:33:33.404 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 19:33:33.404 | INFO | Total mem usage: 25.6 GiB | 20260505 19:33:33.404 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 19:33:33.404 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 19:33:33.404 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 19:33:33.404 | INFO | High Water Breakdown: | 20260505 19:33:33.404 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 19:33:33.404 | INFO | ------------------------------------------ | 20260505 19:33:33.404 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:33:33.404 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 19:33:33.404 | INFO | Total mem usage: 15.7 GiB | 20260505 19:33:33.405 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 19:33:33.405 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 19:33:33.405 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 19:33:33.405 | INFO | High Water Breakdown: | 20260505 19:33:33.405 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 19:33:33.405 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 19:33:33.405 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 19:33:33.405 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 19:33:33.405 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 19:33:33.405 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 19:33:33.405 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 19:33:33.405 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 19:33:33.405 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 19:33:33.405 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:33:33.405 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 19:33:33.405 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:33:33.405 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 19:33:33.405 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 19:33:33.405 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 19:33:33.405 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 19:33:33.405 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:33:33.405 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 19:33:33.405 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:33:33.405 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:33:33.405 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:33:33.405 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 19:33:33.405 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 19:33:33.405 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:33:33.405 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 19:33:33.405 | INFO | ------------------------------------------ | 20260505 19:33:33.405 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:33:33.405 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 19:33:33.405 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:33:33.405 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:33:33.405 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.48e+01, 1.35e+01 / 1.71e+00 | 20260505 19:33:33.405 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.03e-01, 1.05e-02 / 4.03e-03 | 20260505 19:33:33.405 | INFO | PerfReport: 11.80%, Baseline -- 3.72e+00 / 4.55e+00, 5.72e-01 / 1.33e-01 | 20260505 19:33:33.405 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.53e+00, 1.93e-01 / 7.06e-02 | 20260505 19:33:33.405 | INFO | PerfReport: 16.00%, DME -- 4.03e+01 / 4.47e+01, 4.30e+00 / 1.83e+00 | 20260505 19:33:33.405 | INFO | PerfReport: 56.90%, FrameLabeling -- 1.79e+01 / 1.93e+01, 1.40e+00 / 9.40e-01 | 20260505 19:33:33.405 | INFO | PerfReport: 7.29%, PulseAccumulating -- 2.29e+00 / 2.89e+00, 4.41e-01 / 1.60e-01 | 20260505 19:33:33.405 | INFO | PerfReport: 3.29%, Metrics -- 1.04e+00 / 1.37e+00, 2.86e-01 / 1.47e-01 | 20260505 19:33:33.405 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.60e-01 / 4.21e-01, 6.34e-02 / 3.75e-02 | 20260505 19:33:33.405 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 19:33:33.405 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:33:33.405 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:33:33.405 | INFO | PerfReport: Full Execution -- 9.95e+01 / -nan, 2.55e+01 / 0.00e+00 | 20260505 19:33:33.405 | INFO | PerfReport: 22.44%, Upload -- 2.23e+01 / 0.00e+00, 7.74e+00 / 0.00e+00 | 20260505 19:33:33.405 | INFO | PerfReport: 21.79%, Download -- 2.17e+01 / 0.00e+00, 7.73e+00 / 0.00e+00 | 20260505 19:33:33.405 | INFO | PerfReport: 31.65%, Compute -- 3.15e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 19:33:33.405 | INFO | PerfReport: 2.30%, LockUpload -- 2.29e+00 / 0.00e+00, 8.00e+00 / 0.00e+00 | 20260505 19:33:33.405 | INFO | PerfReport: 1.38%, LockDownload -- 1.38e+00 / 0.00e+00, 2.97e+00 / 0.00e+00 | 20260505 19:33:33.405 | INFO | PerfReport: 20.41%, LockCompute -- 2.03e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260505 19:33:33.405 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.62e-02 / 0.00e+00 | 20260505 19:33:33.534 | INFO | Starting a round of Baz IO | 20260505 19:33:36.708 | INFO | Populating RTMetrics report | 20260505 19:33:36.708 | INFO | RTMetrics report populated | 20260505 19:33:36.708 | INFO | Opening tmp RTMetrics json file | 20260505 19:33:36.709 | INFO | Serializing RTMetrics to json | 20260505 19:33:36.710 | INFO | Serialization done, swapping tmp file now... | 20260505 19:33:36.710 | INFO | RTMetrics done | 20260505 19:33:36.808 | INFO | Finished a round of Baz IO: Wrote 2.046 GiB in 3.273 seconds | 20260505 19:33:36.808 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.76% / 0.00% / 78.23% | 20260505 19:33:36.808 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.203546% / 0% / 9.93411e-05% | 20260505 19:33:36.808 | INFO | Baz IO overall: | 20260505 19:33:36.808 | INFO | BytesWritten : 3367128829496 | 20260505 19:33:36.808 | INFO | NumEvents : 1593247325024 | 20260505 19:33:38.538 | INFO | Analyzing chunk frames = [5750784,5751296) | 20260505 19:33:38.538 | INFO | MemInfo: { | 20260505 19:33:38.538 | INFO | "gpu" : " 17 GB", | 20260505 19:33:38.538 | INFO | "huge" : "", | 20260505 19:33:38.538 | INFO | "huge_shared" : " 27 GB", | 20260505 19:33:38.538 | INFO | "malloc" : " 14 GB", | 20260505 19:33:38.538 | INFO | "pinned" : " 29 GB" | 20260505 19:33:38.538 | INFO | } | 20260505 19:33:38.538 | INFO | Perf report for the last 5.13479 seconds: | 20260505 19:33:38.538 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:33:38.538 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:38.538 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:38.538 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:38.538 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 19:33:38.538 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:38.538 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260505 19:33:38.538 | INFO | PRE_HQ : 60.1%, 1.8, 55.3ms, 0 | 20260505 19:33:38.538 | INFO | BAZWRITER : 63.2%, 3.7, 121.1ms, 0 | 20260505 19:33:38.538 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 19:33:38.538 | INFO | Uploaded 36GiB to GPU in 2.15884s (16.6756GiB/s) | 20260505 19:33:38.538 | INFO | Downloaded 38GiB to GPU in 2.24242s (16.946GiB/s) | 20260505 19:33:43.673 | INFO | Analyzing chunk frames = [5751296,5751808) | 20260505 19:33:43.673 | INFO | MemInfo: { | 20260505 19:33:43.673 | INFO | "gpu" : " 17 GB", | 20260505 19:33:43.673 | INFO | "huge" : "", | 20260505 19:33:43.673 | INFO | "huge_shared" : " 27 GB", | 20260505 19:33:43.673 | INFO | "malloc" : " 14 GB", | 20260505 19:33:43.673 | INFO | "pinned" : " 29 GB" | 20260505 19:33:43.673 | INFO | } | 20260505 19:33:43.673 | INFO | Perf report for the last 5.13543 seconds: | 20260505 19:33:43.673 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:33:43.673 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:43.673 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:43.673 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:43.674 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 19:33:43.674 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:43.674 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:43.674 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:33:43.674 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:43.674 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:43.674 | INFO | Uploaded 36GiB to GPU in 2.135s (16.8618GiB/s) | 20260505 19:33:43.674 | INFO | Downloaded 34GiB to GPU in 2.0278s (16.767GiB/s) | 20260505 19:33:48.818 | INFO | Analyzing chunk frames = [5751808,5752320) | 20260505 19:33:48.818 | INFO | MemInfo: { | 20260505 19:33:48.818 | INFO | "gpu" : " 17 GB", | 20260505 19:33:48.818 | INFO | "huge" : "", | 20260505 19:33:48.818 | INFO | "huge_shared" : " 27 GB", | 20260505 19:33:48.818 | INFO | "malloc" : " 14 GB", | 20260505 19:33:48.818 | INFO | "pinned" : " 29 GB" | 20260505 19:33:48.818 | INFO | } | 20260505 19:33:48.818 | INFO | Perf report for the last 5.1444 seconds: | 20260505 19:33:48.818 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:33:48.818 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:48.818 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:48.818 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:48.818 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 19:33:48.818 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:48.818 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:48.818 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:33:48.818 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:48.818 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:48.818 | INFO | Uploaded 36GiB to GPU in 2.21035s (16.287GiB/s) | 20260505 19:33:48.818 | INFO | Downloaded 34GiB to GPU in 2.04273s (16.6444GiB/s) | 20260505 19:33:53.968 | INFO | Analyzing chunk frames = [5752320,5752832) | 20260505 19:33:53.968 | INFO | MemInfo: { | 20260505 19:33:53.968 | INFO | "gpu" : " 17 GB", | 20260505 19:33:53.968 | INFO | "huge" : "", | 20260505 19:33:53.968 | INFO | "huge_shared" : " 27 GB", | 20260505 19:33:53.968 | INFO | "malloc" : " 14 GB", | 20260505 19:33:53.968 | INFO | "pinned" : " 29 GB" | 20260505 19:33:53.968 | INFO | } | 20260505 19:33:53.968 | INFO | Perf report for the last 5.15058 seconds: | 20260505 19:33:53.968 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:33:53.968 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:53.968 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:53.968 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:53.968 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260505 19:33:53.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:53.969 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:53.969 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260505 19:33:53.969 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:53.969 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:53.969 | INFO | Uploaded 36GiB to GPU in 2.19426s (16.4064GiB/s) | 20260505 19:33:53.969 | INFO | Downloaded 34GiB to GPU in 2.04164s (16.6533GiB/s) | 20260505 19:33:59.109 | INFO | Analyzing chunk frames = [5752832,5753344) | 20260505 19:33:59.109 | INFO | MemInfo: { | 20260505 19:33:59.109 | INFO | "gpu" : " 17 GB", | 20260505 19:33:59.109 | INFO | "huge" : "", | 20260505 19:33:59.109 | INFO | "huge_shared" : " 27 GB", | 20260505 19:33:59.109 | INFO | "malloc" : " 14 GB", | 20260505 19:33:59.109 | INFO | "pinned" : " 29 GB" | 20260505 19:33:59.109 | INFO | } | 20260505 19:33:59.109 | INFO | Perf report for the last 5.14072 seconds: | 20260505 19:33:59.109 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:33:59.109 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:59.109 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:59.109 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:59.109 | INFO | ANALYSIS : 61.6%, 3.0, 96.2ms, 0 | 20260505 19:33:59.109 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:59.109 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:59.109 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:33:59.109 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:59.109 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:33:59.109 | INFO | Uploaded 36GiB to GPU in 2.12323s (16.9553GiB/s) | 20260505 19:33:59.110 | INFO | Downloaded 34GiB to GPU in 2.05943s (16.5094GiB/s) | 20260505 19:34:04.233 | INFO | Analyzing chunk frames = [5753344,5753856) | 20260505 19:34:04.233 | INFO | MemInfo: { | 20260505 19:34:04.233 | INFO | "gpu" : " 17 GB", | 20260505 19:34:04.233 | INFO | "huge" : "", | 20260505 19:34:04.233 | INFO | "huge_shared" : " 27 GB", | 20260505 19:34:04.233 | INFO | "malloc" : " 14 GB", | 20260505 19:34:04.233 | INFO | "pinned" : " 29 GB" | 20260505 19:34:04.233 | INFO | } | 20260505 19:34:04.233 | INFO | Perf report for the last 5.12389 seconds: | 20260505 19:34:04.233 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:34:04.233 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:04.233 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:04.233 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:04.233 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260505 19:34:04.233 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:04.233 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:04.233 | INFO | PRE_HQ : 27.5%, 1.0, 14.2ms, 0 | 20260505 19:34:04.233 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:04.233 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:04.234 | INFO | Uploaded 36GiB to GPU in 2.09872s (17.1533GiB/s) | 20260505 19:34:04.234 | INFO | Downloaded 34GiB to GPU in 2.03421s (16.7141GiB/s) | 20260505 19:34:09.380 | INFO | Analyzing chunk frames = [5753856,5754368) | 20260505 19:34:09.380 | INFO | MemInfo: { | 20260505 19:34:09.380 | INFO | "gpu" : " 17 GB", | 20260505 19:34:09.380 | INFO | "huge" : "", | 20260505 19:34:09.380 | INFO | "huge_shared" : " 27 GB", | 20260505 19:34:09.380 | INFO | "malloc" : " 14 GB", | 20260505 19:34:09.380 | INFO | "pinned" : " 29 GB" | 20260505 19:34:09.380 | INFO | } | 20260505 19:34:09.380 | INFO | Perf report for the last 5.14682 seconds: | 20260505 19:34:09.380 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:34:09.380 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:09.380 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:09.380 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:09.380 | INFO | ANALYSIS : 63.1%, 3.0, 98.7ms, 0 | 20260505 19:34:09.380 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:09.380 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:09.380 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 19:34:09.380 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:09.380 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:09.380 | INFO | Uploaded 36GiB to GPU in 2.13382s (16.8711GiB/s) | 20260505 19:34:09.380 | INFO | Downloaded 34GiB to GPU in 2.15597s (15.7701GiB/s) | 20260505 19:34:14.526 | INFO | Analyzing chunk frames = [5754368,5754880) | 20260505 19:34:14.526 | INFO | MemInfo: { | 20260505 19:34:14.526 | INFO | "gpu" : " 17 GB", | 20260505 19:34:14.526 | INFO | "huge" : "", | 20260505 19:34:14.526 | INFO | "huge_shared" : " 27 GB", | 20260505 19:34:14.526 | INFO | "malloc" : " 14 GB", | 20260505 19:34:14.526 | INFO | "pinned" : " 29 GB" | 20260505 19:34:14.526 | INFO | } | 20260505 19:34:14.526 | INFO | Perf report for the last 5.1464 seconds: | 20260505 19:34:14.526 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:34:14.526 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:14.526 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:14.526 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:14.526 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 19:34:14.526 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:14.526 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:14.526 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 19:34:14.526 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:14.527 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:14.527 | INFO | Uploaded 36GiB to GPU in 2.20493s (16.3271GiB/s) | 20260505 19:34:14.527 | INFO | Downloaded 34GiB to GPU in 2.04625s (16.6158GiB/s) | 20260505 19:34:14.661 | INFO | Starting a round of Baz IO | 20260505 19:34:17.830 | INFO | Populating RTMetrics report | 20260505 19:34:17.831 | INFO | RTMetrics report populated | 20260505 19:34:17.831 | INFO | Opening tmp RTMetrics json file | 20260505 19:34:17.831 | INFO | Serializing RTMetrics to json | 20260505 19:34:17.832 | INFO | Serialization done, swapping tmp file now... | 20260505 19:34:17.833 | INFO | RTMetrics done | 20260505 19:34:17.922 | INFO | Finished a round of Baz IO: Wrote 2.046 GiB in 3.261 seconds | 20260505 19:34:17.922 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.76% / 0.00% / 78.24% | 20260505 19:34:17.922 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.20359% / 7.94729e-06% / 9.93411e-05% | 20260505 19:34:17.922 | INFO | Baz IO overall: | 20260505 19:34:17.922 | INFO | BytesWritten : 3369325556783 | 20260505 19:34:17.922 | INFO | NumEvents : 1594267305931 | 20260505 19:34:19.662 | INFO | Analyzing chunk frames = [5754880,5755392) | 20260505 19:34:19.662 | INFO | MemInfo: { | 20260505 19:34:19.662 | INFO | "gpu" : " 17 GB", | 20260505 19:34:19.662 | INFO | "huge" : "", | 20260505 19:34:19.662 | INFO | "huge_shared" : " 27 GB", | 20260505 19:34:19.662 | INFO | "malloc" : " 14 GB", | 20260505 19:34:19.662 | INFO | "pinned" : " 29 GB" | 20260505 19:34:19.662 | INFO | } | 20260505 19:34:19.662 | INFO | Perf report for the last 5.1355 seconds: | 20260505 19:34:19.662 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:34:19.662 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:19.662 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:19.662 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:19.662 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 19:34:19.662 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:19.662 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260505 19:34:19.662 | INFO | PRE_HQ : 60.4%, 1.8, 55.9ms, 0 | 20260505 19:34:19.662 | INFO | BAZWRITER : 63.1%, 3.7, 120.9ms, 0 | 20260505 19:34:19.662 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 19:34:19.662 | INFO | Uploaded 36GiB to GPU in 2.17019s (16.5884GiB/s) | 20260505 19:34:19.662 | INFO | Downloaded 38GiB to GPU in 2.24314s (16.9405GiB/s) | 20260505 19:34:24.793 | INFO | Analyzing chunk frames = [5755392,5755904) | 20260505 19:34:24.793 | INFO | MemInfo: { | 20260505 19:34:24.793 | INFO | "gpu" : " 17 GB", | 20260505 19:34:24.793 | INFO | "huge" : "", | 20260505 19:34:24.793 | INFO | "huge_shared" : " 27 GB", | 20260505 19:34:24.793 | INFO | "malloc" : " 14 GB", | 20260505 19:34:24.793 | INFO | "pinned" : " 29 GB" | 20260505 19:34:24.793 | INFO | } | 20260505 19:34:24.793 | INFO | Perf report for the last 5.1316 seconds: | 20260505 19:34:24.793 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:34:24.793 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:24.793 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:24.794 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:24.794 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260505 19:34:24.794 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:24.794 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:24.794 | INFO | PRE_HQ : 27.5%, 1.0, 14.2ms, 0 | 20260505 19:34:24.794 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:24.794 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:24.794 | INFO | Uploaded 36GiB to GPU in 2.09385s (17.1932GiB/s) | 20260505 19:34:24.794 | INFO | Downloaded 34GiB to GPU in 2.0282s (16.7637GiB/s) | 20260505 19:34:29.930 | INFO | Analyzing chunk frames = [5755904,5756416) | 20260505 19:34:29.930 | INFO | MemInfo: { | 20260505 19:34:29.930 | INFO | "gpu" : " 17 GB", | 20260505 19:34:29.930 | INFO | "huge" : "", | 20260505 19:34:29.930 | INFO | "huge_shared" : " 27 GB", | 20260505 19:34:29.930 | INFO | "malloc" : " 14 GB", | 20260505 19:34:29.930 | INFO | "pinned" : " 29 GB" | 20260505 19:34:29.930 | INFO | } | 20260505 19:34:29.930 | INFO | Perf report for the last 5.13698 seconds: | 20260505 19:34:29.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:34:29.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:29.930 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:29.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:29.931 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 19:34:29.931 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:29.931 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:29.931 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 19:34:29.931 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:29.931 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:29.931 | INFO | Uploaded 36GiB to GPU in 2.2071s (16.311GiB/s) | 20260505 19:34:29.931 | INFO | Downloaded 34GiB to GPU in 2.04546s (16.6222GiB/s) | 20260505 19:34:35.081 | INFO | Analyzing chunk frames = [5756416,5756928) | 20260505 19:34:35.081 | INFO | MemInfo: { | 20260505 19:34:35.081 | INFO | "gpu" : " 17 GB", | 20260505 19:34:35.081 | INFO | "huge" : "", | 20260505 19:34:35.081 | INFO | "huge_shared" : " 27 GB", | 20260505 19:34:35.081 | INFO | "malloc" : " 14 GB", | 20260505 19:34:35.081 | INFO | "pinned" : " 29 GB" | 20260505 19:34:35.081 | INFO | } | 20260505 19:34:35.081 | INFO | Perf report for the last 5.1503 seconds: | 20260505 19:34:35.081 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:34:35.081 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:35.081 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:35.081 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:35.081 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260505 19:34:35.081 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:35.081 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:35.081 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 19:34:35.081 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:35.081 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:35.081 | INFO | Uploaded 36GiB to GPU in 2.16393s (16.6364GiB/s) | 20260505 19:34:35.081 | INFO | Downloaded 34GiB to GPU in 2.05611s (16.5361GiB/s) | 20260505 19:34:40.216 | INFO | Analyzing chunk frames = [5756928,5757440) | 20260505 19:34:40.216 | INFO | MemInfo: { | 20260505 19:34:40.216 | INFO | "gpu" : " 17 GB", | 20260505 19:34:40.216 | INFO | "huge" : "", | 20260505 19:34:40.216 | INFO | "huge_shared" : " 27 GB", | 20260505 19:34:40.216 | INFO | "malloc" : " 14 GB", | 20260505 19:34:40.216 | INFO | "pinned" : " 29 GB" | 20260505 19:34:40.216 | INFO | } | 20260505 19:34:40.216 | INFO | Perf report for the last 5.13545 seconds: | 20260505 19:34:40.216 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:34:40.216 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:40.216 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:40.216 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:40.216 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 19:34:40.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:40.217 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:40.217 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:34:40.217 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:40.217 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:40.217 | INFO | Uploaded 36GiB to GPU in 2.13404s (16.8694GiB/s) | 20260505 19:34:40.217 | INFO | Downloaded 34GiB to GPU in 2.06097s (16.4971GiB/s) | 20260505 19:34:45.349 | INFO | Analyzing chunk frames = [5757440,5757952) | 20260505 19:34:45.349 | INFO | MemInfo: { | 20260505 19:34:45.349 | INFO | "gpu" : " 17 GB", | 20260505 19:34:45.349 | INFO | "huge" : "", | 20260505 19:34:45.349 | INFO | "huge_shared" : " 27 GB", | 20260505 19:34:45.349 | INFO | "malloc" : " 14 GB", | 20260505 19:34:45.349 | INFO | "pinned" : " 29 GB" | 20260505 19:34:45.349 | INFO | } | 20260505 19:34:45.349 | INFO | Perf report for the last 5.13259 seconds: | 20260505 19:34:45.349 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:34:45.349 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:45.349 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:45.349 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:45.349 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 19:34:45.349 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:45.349 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:45.349 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 19:34:45.349 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:45.349 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:45.349 | INFO | Uploaded 36GiB to GPU in 2.15292s (16.7215GiB/s) | 20260505 19:34:45.349 | INFO | Downloaded 34GiB to GPU in 2.06005s (16.5045GiB/s) | 20260505 19:34:50.487 | INFO | Analyzing chunk frames = [5757952,5758464) | 20260505 19:34:50.487 | INFO | MemInfo: { | 20260505 19:34:50.487 | INFO | "gpu" : " 17 GB", | 20260505 19:34:50.487 | INFO | "huge" : "", | 20260505 19:34:50.487 | INFO | "huge_shared" : " 27 GB", | 20260505 19:34:50.487 | INFO | "malloc" : " 14 GB", | 20260505 19:34:50.487 | INFO | "pinned" : " 29 GB" | 20260505 19:34:50.487 | INFO | } | 20260505 19:34:50.487 | INFO | Perf report for the last 5.13824 seconds: | 20260505 19:34:50.487 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:34:50.487 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:50.487 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:50.487 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:50.487 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 19:34:50.487 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:50.487 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:50.487 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 19:34:50.487 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:50.487 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:50.487 | INFO | Uploaded 36GiB to GPU in 2.15302s (16.7207GiB/s) | 20260505 19:34:50.487 | INFO | Downloaded 34GiB to GPU in 2.17532s (15.6299GiB/s) | 20260505 19:34:55.626 | INFO | Analyzing chunk frames = [5758464,5758976) | 20260505 19:34:55.626 | INFO | MemInfo: { | 20260505 19:34:55.626 | INFO | "gpu" : " 17 GB", | 20260505 19:34:55.626 | INFO | "huge" : "", | 20260505 19:34:55.626 | INFO | "huge_shared" : " 27 GB", | 20260505 19:34:55.627 | INFO | "malloc" : " 14 GB", | 20260505 19:34:55.627 | INFO | "pinned" : " 29 GB" | 20260505 19:34:55.627 | INFO | } | 20260505 19:34:55.627 | INFO | Perf report for the last 5.13945 seconds: | 20260505 19:34:55.627 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:34:55.627 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:55.627 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:55.627 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:55.627 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 19:34:55.627 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:55.627 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:55.627 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:34:55.627 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:55.627 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:34:55.627 | INFO | Uploaded 36GiB to GPU in 2.18277s (16.4928GiB/s) | 20260505 19:34:55.627 | INFO | Downloaded 34GiB to GPU in 2.02998s (16.7489GiB/s) | 20260505 19:34:55.627 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 19:34:55.627 | INFO | Total mem usage: 15.3 GiB | 20260505 19:34:55.627 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 19:34:55.627 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.08%) | 20260505 19:34:55.627 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 19:34:55.627 | INFO | High Water Breakdown: | 20260505 19:34:55.627 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 19:34:55.627 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 19:34:55.627 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 19:34:55.627 | INFO | ------------------------------------------ | 20260505 19:34:55.627 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:34:55.627 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 19:34:55.627 | INFO | Total mem usage: 26.5 GiB | 20260505 19:34:55.627 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 19:34:55.627 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 19:34:55.627 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 19:34:55.627 | INFO | High Water Breakdown: | 20260505 19:34:55.627 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:34:55.627 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:34:55.627 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:34:55.627 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:34:55.628 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:34:55.628 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:34:55.628 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 19:34:55.628 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 19:34:55.628 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:34:55.628 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 19:34:55.628 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 19:34:55.628 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 19:34:55.628 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 19:34:55.628 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 19:34:55.628 | INFO | ------------------------------------------ | 20260505 19:34:55.628 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:34:55.628 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 19:34:55.628 | INFO | Total mem usage: 25.6 GiB | 20260505 19:34:55.628 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 19:34:55.628 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 19:34:55.628 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 19:34:55.628 | INFO | High Water Breakdown: | 20260505 19:34:55.628 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 19:34:55.628 | INFO | ------------------------------------------ | 20260505 19:34:55.628 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:34:55.628 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 19:34:55.628 | INFO | Total mem usage: 15.7 GiB | 20260505 19:34:55.628 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 19:34:55.628 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 19:34:55.628 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 19:34:55.628 | INFO | High Water Breakdown: | 20260505 19:34:55.628 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 19:34:55.628 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 19:34:55.628 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 19:34:55.628 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 19:34:55.628 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 19:34:55.628 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 19:34:55.628 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 19:34:55.628 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 19:34:55.628 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 19:34:55.628 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:34:55.628 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 19:34:55.628 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:34:55.628 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 19:34:55.628 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 19:34:55.628 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 19:34:55.628 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 19:34:55.628 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:34:55.628 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 19:34:55.628 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:34:55.628 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:34:55.628 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:34:55.628 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 19:34:55.628 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 19:34:55.628 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:34:55.628 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 19:34:55.628 | INFO | ------------------------------------------ | 20260505 19:34:55.628 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:34:55.629 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 19:34:55.629 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:34:55.629 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:34:55.629 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.44e+01, 1.31e+01 / 1.68e+00 | 20260505 19:34:55.629 | INFO | PerfReport: 0.29%, AggregateStats -- 9.24e-02 / 1.04e-01, 1.02e-02 / 9.11e-03 | 20260505 19:34:55.629 | INFO | PerfReport: 11.79%, Baseline -- 3.71e+00 / 4.54e+00, 5.62e-01 / 1.35e-01 | 20260505 19:34:55.629 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.54e+00, 1.88e-01 / 8.65e-02 | 20260505 19:34:55.629 | INFO | PerfReport: 15.94%, DME -- 4.02e+01 / 4.42e+01, 4.16e+00 / 1.71e+00 | 20260505 19:34:55.629 | INFO | PerfReport: 57.05%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.53e+00 / 1.07e+00 | 20260505 19:34:55.629 | INFO | PerfReport: 7.25%, PulseAccumulating -- 2.28e+00 / 2.86e+00, 4.38e-01 / 1.90e-01 | 20260505 19:34:55.629 | INFO | PerfReport: 3.27%, Metrics -- 1.03e+00 / 1.35e+00, 2.81e-01 / 1.55e-01 | 20260505 19:34:55.629 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.60e-01 / 4.20e-01, 6.32e-02 / 4.18e-02 | 20260505 19:34:55.629 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 19:34:55.629 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:34:55.629 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:34:55.629 | INFO | PerfReport: Full Execution -- 9.96e+01 / -nan, 2.56e+01 / 0.00e+00 | 20260505 19:34:55.629 | INFO | PerfReport: 22.56%, Upload -- 2.25e+01 / 0.00e+00, 7.88e+00 / 0.00e+00 | 20260505 19:34:55.629 | INFO | PerfReport: 21.79%, Download -- 2.17e+01 / 0.00e+00, 7.69e+00 / 0.00e+00 | 20260505 19:34:55.629 | INFO | PerfReport: 31.63%, Compute -- 3.15e+01 / 0.00e+00, 1.31e+01 / 0.00e+00 | 20260505 19:34:55.629 | INFO | PerfReport: 2.35%, LockUpload -- 2.34e+00 / 0.00e+00, 7.90e+00 / 0.00e+00 | 20260505 19:34:55.629 | INFO | PerfReport: 1.34%, LockDownload -- 1.33e+00 / 0.00e+00, 2.83e+00 / 0.00e+00 | 20260505 19:34:55.629 | INFO | PerfReport: 20.32%, LockCompute -- 2.02e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 19:34:55.629 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.68e-02 / 0.00e+00 | 20260505 19:34:55.759 | INFO | Starting a round of Baz IO | 20260505 19:34:58.934 | INFO | Populating RTMetrics report | 20260505 19:34:58.935 | INFO | RTMetrics report populated | 20260505 19:34:58.935 | INFO | Opening tmp RTMetrics json file | 20260505 19:34:58.935 | INFO | Serializing RTMetrics to json | 20260505 19:34:58.936 | INFO | Serialization done, swapping tmp file now... | 20260505 19:34:58.937 | INFO | RTMetrics done | 20260505 19:34:59.035 | INFO | Finished a round of Baz IO: Wrote 2.044 GiB in 3.275 seconds | 20260505 19:34:59.035 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.76% / 0.00% / 78.24% | 20260505 19:34:59.035 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.20256% / 7.94729e-06% / 6.35783e-05% | 20260505 19:34:59.035 | INFO | Baz IO overall: | 20260505 19:34:59.035 | INFO | BytesWritten : 3371520535215 | 20260505 19:34:59.035 | INFO | NumEvents : 1595285817730 | 20260505 19:35:00.775 | INFO | Analyzing chunk frames = [5758976,5759488) | 20260505 19:35:00.775 | INFO | MemInfo: { | 20260505 19:35:00.775 | INFO | "gpu" : " 17 GB", | 20260505 19:35:00.775 | INFO | "huge" : "", | 20260505 19:35:00.775 | INFO | "huge_shared" : " 27 GB", | 20260505 19:35:00.775 | INFO | "malloc" : " 14 GB", | 20260505 19:35:00.775 | INFO | "pinned" : " 29 GB" | 20260505 19:35:00.775 | INFO | } | 20260505 19:35:00.775 | INFO | Perf report for the last 5.14809 seconds: | 20260505 19:35:00.775 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:35:00.775 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:00.775 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:00.775 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:00.775 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 19:35:00.775 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:00.775 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260505 19:35:00.775 | INFO | PRE_HQ : 60.3%, 1.8, 55.8ms, 0 | 20260505 19:35:00.775 | INFO | BAZWRITER : 63.1%, 3.7, 120.8ms, 0 | 20260505 19:35:00.775 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 19:35:00.775 | INFO | Uploaded 36GiB to GPU in 2.15814s (16.6811GiB/s) | 20260505 19:35:00.775 | INFO | Downloaded 38GiB to GPU in 2.24265s (16.9442GiB/s) | 20260505 19:35:05.918 | INFO | Analyzing chunk frames = [5759488,5760000) | 20260505 19:35:05.918 | INFO | MemInfo: { | 20260505 19:35:05.918 | INFO | "gpu" : " 17 GB", | 20260505 19:35:05.918 | INFO | "huge" : "", | 20260505 19:35:05.918 | INFO | "huge_shared" : " 27 GB", | 20260505 19:35:05.918 | INFO | "malloc" : " 14 GB", | 20260505 19:35:05.918 | INFO | "pinned" : " 29 GB" | 20260505 19:35:05.918 | INFO | } | 20260505 19:35:05.918 | INFO | Perf report for the last 5.14347 seconds: | 20260505 19:35:05.918 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:35:05.918 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:05.918 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:05.918 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:05.918 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260505 19:35:05.918 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:05.918 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:05.918 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:35:05.918 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:05.918 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:05.918 | INFO | Uploaded 36GiB to GPU in 2.13846s (16.8346GiB/s) | 20260505 19:35:05.918 | INFO | Downloaded 34GiB to GPU in 2.04252s (16.6461GiB/s) | 20260505 19:35:11.050 | INFO | Analyzing chunk frames = [5760000,5760512) | 20260505 19:35:11.050 | INFO | MemInfo: { | 20260505 19:35:11.050 | INFO | "gpu" : " 17 GB", | 20260505 19:35:11.050 | INFO | "huge" : "", | 20260505 19:35:11.050 | INFO | "huge_shared" : " 27 GB", | 20260505 19:35:11.050 | INFO | "malloc" : " 14 GB", | 20260505 19:35:11.050 | INFO | "pinned" : " 29 GB" | 20260505 19:35:11.050 | INFO | } | 20260505 19:35:11.050 | INFO | Perf report for the last 5.13186 seconds: | 20260505 19:35:11.050 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:35:11.050 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:11.050 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:11.050 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:11.050 | INFO | ANALYSIS : 62.2%, 3.0, 96.6ms, 0 | 20260505 19:35:11.050 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:11.050 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:11.050 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 19:35:11.050 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:11.050 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:11.050 | INFO | Uploaded 36GiB to GPU in 2.13436s (16.8669GiB/s) | 20260505 19:35:11.050 | INFO | Downloaded 34GiB to GPU in 2.01853s (16.8439GiB/s) | 20260505 19:35:16.184 | INFO | Analyzing chunk frames = [5760512,5761024) | 20260505 19:35:16.184 | INFO | MemInfo: { | 20260505 19:35:16.184 | INFO | "gpu" : " 17 GB", | 20260505 19:35:16.184 | INFO | "huge" : "", | 20260505 19:35:16.184 | INFO | "huge_shared" : " 27 GB", | 20260505 19:35:16.184 | INFO | "malloc" : " 14 GB", | 20260505 19:35:16.184 | INFO | "pinned" : " 29 GB" | 20260505 19:35:16.184 | INFO | } | 20260505 19:35:16.184 | INFO | Perf report for the last 5.1344 seconds: | 20260505 19:35:16.184 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:35:16.184 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:16.184 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:16.184 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:16.184 | INFO | ANALYSIS : 62.4%, 3.0, 96.8ms, 0 | 20260505 19:35:16.184 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:16.184 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:16.185 | INFO | PRE_HQ : 28.1%, 1.0, 14.5ms, 0 | 20260505 19:35:16.185 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:16.185 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:16.185 | INFO | Uploaded 36GiB to GPU in 2.17924s (16.5195GiB/s) | 20260505 19:35:16.185 | INFO | Downloaded 34GiB to GPU in 2.05414s (16.552GiB/s) | 20260505 19:35:21.333 | INFO | Analyzing chunk frames = [5761024,5761536) | 20260505 19:35:21.333 | INFO | MemInfo: { | 20260505 19:35:21.333 | INFO | "gpu" : " 17 GB", | 20260505 19:35:21.333 | INFO | "huge" : "", | 20260505 19:35:21.333 | INFO | "huge_shared" : " 27 GB", | 20260505 19:35:21.333 | INFO | "malloc" : " 14 GB", | 20260505 19:35:21.333 | INFO | "pinned" : " 29 GB" | 20260505 19:35:21.333 | INFO | } | 20260505 19:35:21.333 | INFO | Perf report for the last 5.14853 seconds: | 20260505 19:35:21.333 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:35:21.333 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:21.333 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:21.333 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:21.333 | INFO | ANALYSIS : 61.9%, 3.0, 96.8ms, 0 | 20260505 19:35:21.333 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:21.333 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:21.333 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 19:35:21.333 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:21.333 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:21.333 | INFO | Uploaded 36GiB to GPU in 2.1434s (16.7958GiB/s) | 20260505 19:35:21.333 | INFO | Downloaded 34GiB to GPU in 2.06176s (16.4908GiB/s) | 20260505 19:35:26.479 | INFO | Analyzing chunk frames = [5761536,5762048) | 20260505 19:35:26.479 | INFO | MemInfo: { | 20260505 19:35:26.479 | INFO | "gpu" : " 17 GB", | 20260505 19:35:26.479 | INFO | "huge" : "", | 20260505 19:35:26.479 | INFO | "huge_shared" : " 27 GB", | 20260505 19:35:26.479 | INFO | "malloc" : " 14 GB", | 20260505 19:35:26.479 | INFO | "pinned" : " 29 GB" | 20260505 19:35:26.479 | INFO | } | 20260505 19:35:26.479 | INFO | Perf report for the last 5.14586 seconds: | 20260505 19:35:26.479 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:35:26.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:26.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:26.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:26.479 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 19:35:26.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:26.479 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:26.479 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:35:26.479 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:26.479 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:26.479 | INFO | Uploaded 36GiB to GPU in 2.16505s (16.6278GiB/s) | 20260505 19:35:26.479 | INFO | Downloaded 34GiB to GPU in 2.06329s (16.4785GiB/s) | 20260505 19:35:31.609 | INFO | Analyzing chunk frames = [5762048,5762560) | 20260505 19:35:31.609 | INFO | MemInfo: { | 20260505 19:35:31.609 | INFO | "gpu" : " 17 GB", | 20260505 19:35:31.609 | INFO | "huge" : "", | 20260505 19:35:31.609 | INFO | "huge_shared" : " 27 GB", | 20260505 19:35:31.609 | INFO | "malloc" : " 14 GB", | 20260505 19:35:31.609 | INFO | "pinned" : " 29 GB" | 20260505 19:35:31.609 | INFO | } | 20260505 19:35:31.609 | INFO | Perf report for the last 5.13065 seconds: | 20260505 19:35:31.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:35:31.610 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:31.610 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:31.610 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:31.610 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 19:35:31.610 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:31.610 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:31.610 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 19:35:31.610 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:31.610 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:31.610 | INFO | Uploaded 36GiB to GPU in 2.13681s (16.8475GiB/s) | 20260505 19:35:31.610 | INFO | Downloaded 34GiB to GPU in 2.10444s (16.1563GiB/s) | 20260505 19:35:36.745 | INFO | Analyzing chunk frames = [5762560,5763072) | 20260505 19:35:36.745 | INFO | MemInfo: { | 20260505 19:35:36.745 | INFO | "gpu" : " 17 GB", | 20260505 19:35:36.745 | INFO | "huge" : "", | 20260505 19:35:36.745 | INFO | "huge_shared" : " 27 GB", | 20260505 19:35:36.745 | INFO | "malloc" : " 14 GB", | 20260505 19:35:36.745 | INFO | "pinned" : " 29 GB" | 20260505 19:35:36.745 | INFO | } | 20260505 19:35:36.745 | INFO | Perf report for the last 5.13539 seconds: | 20260505 19:35:36.745 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:35:36.745 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:36.745 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:36.745 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:36.745 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 19:35:36.745 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:36.745 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:36.745 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:35:36.745 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:36.745 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:36.745 | INFO | Uploaded 36GiB to GPU in 2.12445s (16.9456GiB/s) | 20260505 19:35:36.745 | INFO | Downloaded 34GiB to GPU in 2.00643s (16.9455GiB/s) | 20260505 19:35:36.879 | INFO | Starting a round of Baz IO | 20260505 19:35:40.048 | INFO | Populating RTMetrics report | 20260505 19:35:40.049 | INFO | RTMetrics report populated | 20260505 19:35:40.049 | INFO | Opening tmp RTMetrics json file | 20260505 19:35:40.049 | INFO | Serializing RTMetrics to json | 20260505 19:35:40.051 | INFO | Serialization done, swapping tmp file now... | 20260505 19:35:40.051 | INFO | RTMetrics done | 20260505 19:35:40.138 | INFO | Finished a round of Baz IO: Wrote 2.044 GiB in 3.258 seconds | 20260505 19:35:40.138 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.76% / 0.00% / 78.24% | 20260505 19:35:40.138 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.202326% / 0% / 6.75519e-05% | 20260505 19:35:40.138 | INFO | Baz IO overall: | 20260505 19:35:40.138 | INFO | BytesWritten : 3373715527089 | 20260505 19:35:40.138 | INFO | NumEvents : 1596304433698 | 20260505 19:35:41.891 | INFO | Analyzing chunk frames = [5763072,5763584) | 20260505 19:35:41.891 | INFO | MemInfo: { | 20260505 19:35:41.891 | INFO | "gpu" : " 17 GB", | 20260505 19:35:41.891 | INFO | "huge" : "", | 20260505 19:35:41.891 | INFO | "huge_shared" : " 27 GB", | 20260505 19:35:41.891 | INFO | "malloc" : " 14 GB", | 20260505 19:35:41.891 | INFO | "pinned" : " 29 GB" | 20260505 19:35:41.891 | INFO | } | 20260505 19:35:41.891 | INFO | Perf report for the last 5.14614 seconds: | 20260505 19:35:41.891 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:35:41.891 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:41.891 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:41.891 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:41.891 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 19:35:41.891 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:41.891 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260505 19:35:41.891 | INFO | PRE_HQ : 60.1%, 1.8, 55.7ms, 0 | 20260505 19:35:41.891 | INFO | BAZWRITER : 62.9%, 3.7, 120.6ms, 0 | 20260505 19:35:41.891 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 19:35:41.891 | INFO | Uploaded 36GiB to GPU in 2.15859s (16.6776GiB/s) | 20260505 19:35:41.891 | INFO | Downloaded 38GiB to GPU in 2.23904s (16.9716GiB/s) | 20260505 19:35:47.028 | INFO | Analyzing chunk frames = [5763584,5764096) | 20260505 19:35:47.028 | INFO | MemInfo: { | 20260505 19:35:47.028 | INFO | "gpu" : " 17 GB", | 20260505 19:35:47.028 | INFO | "huge" : "", | 20260505 19:35:47.028 | INFO | "huge_shared" : " 27 GB", | 20260505 19:35:47.028 | INFO | "malloc" : " 14 GB", | 20260505 19:35:47.028 | INFO | "pinned" : " 29 GB" | 20260505 19:35:47.028 | INFO | } | 20260505 19:35:47.028 | INFO | Perf report for the last 5.13671 seconds: | 20260505 19:35:47.028 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:35:47.028 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:47.028 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:47.028 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:47.028 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 19:35:47.028 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:47.028 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:47.028 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:35:47.028 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:47.028 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:47.028 | INFO | Uploaded 36GiB to GPU in 2.14717s (16.7663GiB/s) | 20260505 19:35:47.028 | INFO | Downloaded 34GiB to GPU in 2.05278s (16.5629GiB/s) | 20260505 19:35:52.164 | INFO | Analyzing chunk frames = [5764096,5764608) | 20260505 19:35:52.164 | INFO | MemInfo: { | 20260505 19:35:52.164 | INFO | "gpu" : " 17 GB", | 20260505 19:35:52.164 | INFO | "huge" : "", | 20260505 19:35:52.164 | INFO | "huge_shared" : " 27 GB", | 20260505 19:35:52.164 | INFO | "malloc" : " 14 GB", | 20260505 19:35:52.164 | INFO | "pinned" : " 29 GB" | 20260505 19:35:52.164 | INFO | } | 20260505 19:35:52.164 | INFO | Perf report for the last 5.13639 seconds: | 20260505 19:35:52.164 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:35:52.164 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:52.164 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:52.164 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:52.164 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260505 19:35:52.164 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:52.164 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:52.164 | INFO | PRE_HQ : 27.5%, 1.0, 14.2ms, 0 | 20260505 19:35:52.164 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:52.164 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:52.164 | INFO | Uploaded 36GiB to GPU in 2.1334s (16.8745GiB/s) | 20260505 19:35:52.164 | INFO | Downloaded 34GiB to GPU in 2.0164s (16.8617GiB/s) | 20260505 19:35:57.299 | INFO | Analyzing chunk frames = [5764608,5765120) | 20260505 19:35:57.299 | INFO | MemInfo: { | 20260505 19:35:57.299 | INFO | "gpu" : " 17 GB", | 20260505 19:35:57.299 | INFO | "huge" : "", | 20260505 19:35:57.299 | INFO | "huge_shared" : " 27 GB", | 20260505 19:35:57.299 | INFO | "malloc" : " 14 GB", | 20260505 19:35:57.299 | INFO | "pinned" : " 29 GB" | 20260505 19:35:57.299 | INFO | } | 20260505 19:35:57.299 | INFO | Perf report for the last 5.13495 seconds: | 20260505 19:35:57.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:35:57.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:57.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:57.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:57.299 | INFO | ANALYSIS : 62.3%, 3.0, 96.7ms, 0 | 20260505 19:35:57.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:57.299 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:57.299 | INFO | PRE_HQ : 27.7%, 1.0, 14.3ms, 0 | 20260505 19:35:57.299 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:57.299 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:35:57.299 | INFO | Uploaded 36GiB to GPU in 2.18212s (16.4977GiB/s) | 20260505 19:35:57.299 | INFO | Downloaded 34GiB to GPU in 2.07045s (16.4215GiB/s) | 20260505 19:36:02.440 | INFO | Analyzing chunk frames = [5765120,5765632) | 20260505 19:36:02.440 | INFO | MemInfo: { | 20260505 19:36:02.440 | INFO | "gpu" : " 17 GB", | 20260505 19:36:02.440 | INFO | "huge" : "", | 20260505 19:36:02.440 | INFO | "huge_shared" : " 27 GB", | 20260505 19:36:02.440 | INFO | "malloc" : " 14 GB", | 20260505 19:36:02.440 | INFO | "pinned" : " 29 GB" | 20260505 19:36:02.440 | INFO | } | 20260505 19:36:02.440 | INFO | Perf report for the last 5.14086 seconds: | 20260505 19:36:02.440 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:36:02.440 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:02.440 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:02.440 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:02.440 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260505 19:36:02.440 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:02.440 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:02.440 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 19:36:02.440 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:02.440 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:02.440 | INFO | Uploaded 36GiB to GPU in 2.13708s (16.8455GiB/s) | 20260505 19:36:02.440 | INFO | Downloaded 34GiB to GPU in 2.05942s (16.5095GiB/s) | 20260505 19:36:07.584 | INFO | Analyzing chunk frames = [5765632,5766144) | 20260505 19:36:07.584 | INFO | MemInfo: { | 20260505 19:36:07.584 | INFO | "gpu" : " 17 GB", | 20260505 19:36:07.584 | INFO | "huge" : "", | 20260505 19:36:07.584 | INFO | "huge_shared" : " 27 GB", | 20260505 19:36:07.584 | INFO | "malloc" : " 14 GB", | 20260505 19:36:07.584 | INFO | "pinned" : " 29 GB" | 20260505 19:36:07.584 | INFO | } | 20260505 19:36:07.584 | INFO | Perf report for the last 5.14416 seconds: | 20260505 19:36:07.584 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:36:07.584 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:07.584 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:07.584 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:07.584 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 19:36:07.584 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:07.584 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:07.584 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:36:07.584 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:07.584 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:07.584 | INFO | Uploaded 36GiB to GPU in 2.13758s (16.8415GiB/s) | 20260505 19:36:07.584 | INFO | Downloaded 34GiB to GPU in 2.05185s (16.5704GiB/s) | 20260505 19:36:12.731 | INFO | Analyzing chunk frames = [5766144,5766656) | 20260505 19:36:12.731 | INFO | MemInfo: { | 20260505 19:36:12.731 | INFO | "gpu" : " 17 GB", | 20260505 19:36:12.731 | INFO | "huge" : "", | 20260505 19:36:12.731 | INFO | "huge_shared" : " 27 GB", | 20260505 19:36:12.731 | INFO | "malloc" : " 14 GB", | 20260505 19:36:12.731 | INFO | "pinned" : " 29 GB" | 20260505 19:36:12.731 | INFO | } | 20260505 19:36:12.731 | INFO | Perf report for the last 5.14712 seconds: | 20260505 19:36:12.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:36:12.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:12.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:12.731 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:12.731 | INFO | ANALYSIS : 62.9%, 3.0, 98.4ms, 0 | 20260505 19:36:12.731 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:12.731 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:12.731 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 19:36:12.731 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:12.731 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:12.732 | INFO | Uploaded 36GiB to GPU in 2.11967s (16.9837GiB/s) | 20260505 19:36:12.732 | INFO | Downloaded 34GiB to GPU in 2.13227s (15.9454GiB/s) | 20260505 19:36:17.862 | INFO | Analyzing chunk frames = [5766656,5767168) | 20260505 19:36:17.862 | INFO | MemInfo: { | 20260505 19:36:17.862 | INFO | "gpu" : " 17 GB", | 20260505 19:36:17.862 | INFO | "huge" : "", | 20260505 19:36:17.862 | INFO | "huge_shared" : " 27 GB", | 20260505 19:36:17.862 | INFO | "malloc" : " 14 GB", | 20260505 19:36:17.862 | INFO | "pinned" : " 29 GB" | 20260505 19:36:17.862 | INFO | } | 20260505 19:36:17.862 | INFO | Perf report for the last 5.13115 seconds: | 20260505 19:36:17.862 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:36:17.862 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:17.862 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:17.862 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:17.863 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260505 19:36:17.863 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:17.863 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:17.863 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 19:36:17.863 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:17.863 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:17.863 | INFO | Uploaded 36GiB to GPU in 2.11865s (16.992GiB/s) | 20260505 19:36:17.863 | INFO | Downloaded 34GiB to GPU in 2.00687s (16.9418GiB/s) | 20260505 19:36:17.863 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 19:36:17.863 | INFO | Total mem usage: 15.3 GiB | 20260505 19:36:17.863 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 19:36:17.863 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.08%) | 20260505 19:36:17.863 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 19:36:17.863 | INFO | High Water Breakdown: | 20260505 19:36:17.863 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 19:36:17.863 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 19:36:17.863 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 19:36:17.863 | INFO | ------------------------------------------ | 20260505 19:36:17.863 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:36:17.863 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 19:36:17.863 | INFO | Total mem usage: 26.5 GiB | 20260505 19:36:17.863 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 19:36:17.863 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 19:36:17.863 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 19:36:17.863 | INFO | High Water Breakdown: | 20260505 19:36:17.863 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:36:17.863 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:36:17.863 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:36:17.863 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:36:17.863 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:36:17.863 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:36:17.863 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 19:36:17.863 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 19:36:17.863 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:36:17.863 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 19:36:17.863 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 19:36:17.863 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 19:36:17.863 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 19:36:17.863 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 19:36:17.863 | INFO | ------------------------------------------ | 20260505 19:36:17.863 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:36:17.863 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 19:36:17.863 | INFO | Total mem usage: 25.6 GiB | 20260505 19:36:17.863 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 19:36:17.863 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 19:36:17.863 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 19:36:17.863 | INFO | High Water Breakdown: | 20260505 19:36:17.863 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 19:36:17.863 | INFO | ------------------------------------------ | 20260505 19:36:17.863 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:36:17.863 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 19:36:17.864 | INFO | Total mem usage: 15.7 GiB | 20260505 19:36:17.864 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 19:36:17.864 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 19:36:17.864 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 19:36:17.864 | INFO | High Water Breakdown: | 20260505 19:36:17.864 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 19:36:17.864 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 19:36:17.864 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 19:36:17.864 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 19:36:17.864 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 19:36:17.864 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 19:36:17.864 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 19:36:17.864 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 19:36:17.864 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 19:36:17.864 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:36:17.864 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 19:36:17.864 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:36:17.864 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 19:36:17.864 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 19:36:17.864 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 19:36:17.864 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 19:36:17.864 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:36:17.864 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 19:36:17.864 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:36:17.864 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:36:17.864 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:36:17.864 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 19:36:17.864 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 19:36:17.864 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:36:17.864 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 19:36:17.864 | INFO | ------------------------------------------ | 20260505 19:36:17.864 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:36:17.864 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 19:36:17.864 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:36:17.864 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:36:17.864 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.45e+01, 1.35e+01 / 1.51e+00 | 20260505 19:36:17.864 | INFO | PerfReport: 0.29%, AggregateStats -- 9.27e-02 / 1.04e-01, 1.05e-02 / 4.82e-03 | 20260505 19:36:17.864 | INFO | PerfReport: 11.79%, Baseline -- 3.72e+00 / 4.50e+00, 5.72e-01 / 1.78e-01 | 20260505 19:36:17.864 | INFO | PerfReport: 3.28%, Histogram -- 1.04e+00 / 1.51e+00, 1.93e-01 / 1.04e-01 | 20260505 19:36:17.864 | INFO | PerfReport: 15.92%, DME -- 4.02e+01 / 4.42e+01, 4.06e+00 / 1.64e+00 | 20260505 19:36:17.864 | INFO | PerfReport: 57.09%, FrameLabeling -- 1.80e+01 / 1.96e+01, 1.57e+00 / 1.25e+00 | 20260505 19:36:17.864 | INFO | PerfReport: 7.22%, PulseAccumulating -- 2.28e+00 / 2.82e+00, 4.35e-01 / 2.32e-01 | 20260505 19:36:17.864 | INFO | PerfReport: 3.26%, Metrics -- 1.03e+00 / 1.35e+00, 2.80e-01 / 1.47e-01 | 20260505 19:36:17.864 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.60e-01 / 4.14e-01, 6.40e-02 / 4.03e-02 | 20260505 19:36:17.864 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 19:36:17.864 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:36:17.864 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:36:17.864 | INFO | PerfReport: Full Execution -- 9.95e+01 / -nan, 2.56e+01 / 0.00e+00 | 20260505 19:36:17.864 | INFO | PerfReport: 22.46%, Upload -- 2.23e+01 / 0.00e+00, 7.79e+00 / 0.00e+00 | 20260505 19:36:17.864 | INFO | PerfReport: 21.74%, Download -- 2.16e+01 / 0.00e+00, 7.73e+00 / 0.00e+00 | 20260505 19:36:17.864 | INFO | PerfReport: 31.72%, Compute -- 3.16e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 19:36:17.864 | INFO | PerfReport: 2.27%, LockUpload -- 2.25e+00 / 0.00e+00, 7.87e+00 / 0.00e+00 | 20260505 19:36:17.864 | INFO | PerfReport: 1.34%, LockDownload -- 1.33e+00 / 0.00e+00, 2.86e+00 / 0.00e+00 | 20260505 19:36:17.864 | INFO | PerfReport: 20.46%, LockCompute -- 2.04e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 19:36:17.864 | INFO | PerfReport: 0.01%, Misc -- 1.25e-02 / 0.00e+00, 1.23e-02 / 0.00e+00 | 20260505 19:36:18.010 | INFO | Starting a round of Baz IO | 20260505 19:36:21.168 | INFO | Populating RTMetrics report | 20260505 19:36:21.169 | INFO | RTMetrics report populated | 20260505 19:36:21.169 | INFO | Opening tmp RTMetrics json file | 20260505 19:36:21.169 | INFO | Serializing RTMetrics to json | 20260505 19:36:21.171 | INFO | Serialization done, swapping tmp file now... | 20260505 19:36:21.171 | INFO | RTMetrics done | 20260505 19:36:21.258 | INFO | Finished a round of Baz IO: Wrote 2.048 GiB in 3.248 seconds | 20260505 19:36:21.258 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.76% / 0.00% / 78.24% | 20260505 19:36:21.258 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.203705% / 0% / 7.94729e-05% | 20260505 19:36:21.258 | INFO | Baz IO overall: | 20260505 19:36:21.258 | INFO | BytesWritten : 3375914658075 | 20260505 19:36:21.258 | INFO | NumEvents : 1597326630317 | 20260505 19:36:22.994 | INFO | Analyzing chunk frames = [5767168,5767680) | 20260505 19:36:22.994 | INFO | MemInfo: { | 20260505 19:36:22.994 | INFO | "gpu" : " 17 GB", | 20260505 19:36:22.994 | INFO | "huge" : "", | 20260505 19:36:22.994 | INFO | "huge_shared" : " 27 GB", | 20260505 19:36:22.994 | INFO | "malloc" : " 14 GB", | 20260505 19:36:22.994 | INFO | "pinned" : " 29 GB" | 20260505 19:36:22.994 | INFO | } | 20260505 19:36:22.994 | INFO | Perf report for the last 5.13146 seconds: | 20260505 19:36:22.994 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:36:22.994 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:22.994 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:22.994 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:22.994 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 19:36:22.994 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:22.994 | INFO | RT_METRICS : 44.2%, 1.0, 23.1ms, 0 | 20260505 19:36:22.994 | INFO | PRE_HQ : 60.4%, 1.8, 55.6ms, 0 | 20260505 19:36:22.994 | INFO | BAZWRITER : 63.2%, 3.7, 120.2ms, 0 | 20260505 19:36:22.994 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 19:36:22.994 | INFO | Uploaded 36GiB to GPU in 2.16166s (16.6539GiB/s) | 20260505 19:36:22.994 | INFO | Downloaded 38GiB to GPU in 2.23691s (16.9877GiB/s) | 20260505 19:36:28.137 | INFO | Analyzing chunk frames = [5767680,5768192) | 20260505 19:36:28.137 | INFO | MemInfo: { | 20260505 19:36:28.137 | INFO | "gpu" : " 17 GB", | 20260505 19:36:28.137 | INFO | "huge" : "", | 20260505 19:36:28.137 | INFO | "huge_shared" : " 27 GB", | 20260505 19:36:28.137 | INFO | "malloc" : " 14 GB", | 20260505 19:36:28.138 | INFO | "pinned" : " 29 GB" | 20260505 19:36:28.138 | INFO | } | 20260505 19:36:28.138 | INFO | Perf report for the last 5.14362 seconds: | 20260505 19:36:28.138 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:36:28.138 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:28.138 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:28.138 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:28.138 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 19:36:28.138 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:28.138 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:28.138 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:36:28.138 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:28.138 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:28.138 | INFO | Uploaded 36GiB to GPU in 2.1376s (16.8413GiB/s) | 20260505 19:36:28.138 | INFO | Downloaded 34GiB to GPU in 2.03014s (16.7476GiB/s) | 20260505 19:36:33.278 | INFO | Analyzing chunk frames = [5768192,5768704) | 20260505 19:36:33.278 | INFO | MemInfo: { | 20260505 19:36:33.278 | INFO | "gpu" : " 17 GB", | 20260505 19:36:33.279 | INFO | "huge" : "", | 20260505 19:36:33.279 | INFO | "huge_shared" : " 27 GB", | 20260505 19:36:33.279 | INFO | "malloc" : " 14 GB", | 20260505 19:36:33.279 | INFO | "pinned" : " 29 GB" | 20260505 19:36:33.279 | INFO | } | 20260505 19:36:33.279 | INFO | Perf report for the last 5.14107 seconds: | 20260505 19:36:33.279 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:36:33.279 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:33.279 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:33.279 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:33.279 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260505 19:36:33.279 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:33.279 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:33.279 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 19:36:33.279 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:33.279 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:33.279 | INFO | Uploaded 36GiB to GPU in 2.13127s (16.8913GiB/s) | 20260505 19:36:33.279 | INFO | Downloaded 34GiB to GPU in 2.03117s (16.7391GiB/s) | 20260505 19:36:38.422 | INFO | Analyzing chunk frames = [5768704,5769216) | 20260505 19:36:38.422 | INFO | MemInfo: { | 20260505 19:36:38.422 | INFO | "gpu" : " 17 GB", | 20260505 19:36:38.422 | INFO | "huge" : "", | 20260505 19:36:38.422 | INFO | "huge_shared" : " 27 GB", | 20260505 19:36:38.422 | INFO | "malloc" : " 14 GB", | 20260505 19:36:38.423 | INFO | "pinned" : " 29 GB" | 20260505 19:36:38.423 | INFO | } | 20260505 19:36:38.423 | INFO | Perf report for the last 5.14403 seconds: | 20260505 19:36:38.423 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:36:38.423 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:38.423 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:38.423 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:38.423 | INFO | ANALYSIS : 62.4%, 3.0, 97.0ms, 0 | 20260505 19:36:38.423 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:38.423 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:38.423 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:36:38.423 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:38.423 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:38.423 | INFO | Uploaded 36GiB to GPU in 2.2093s (16.2948GiB/s) | 20260505 19:36:38.423 | INFO | Downloaded 34GiB to GPU in 2.04412s (16.6331GiB/s) | 20260505 19:36:43.554 | INFO | Analyzing chunk frames = [5769216,5769728) | 20260505 19:36:43.554 | INFO | MemInfo: { | 20260505 19:36:43.554 | INFO | "gpu" : " 17 GB", | 20260505 19:36:43.554 | INFO | "huge" : "", | 20260505 19:36:43.554 | INFO | "huge_shared" : " 27 GB", | 20260505 19:36:43.554 | INFO | "malloc" : " 14 GB", | 20260505 19:36:43.555 | INFO | "pinned" : " 29 GB" | 20260505 19:36:43.555 | INFO | } | 20260505 19:36:43.555 | INFO | Perf report for the last 5.13189 seconds: | 20260505 19:36:43.555 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:36:43.555 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:43.555 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:43.555 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:43.555 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 19:36:43.555 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:43.555 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:43.555 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 19:36:43.555 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:43.555 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:43.555 | INFO | Uploaded 36GiB to GPU in 2.1358s (16.8555GiB/s) | 20260505 19:36:43.555 | INFO | Downloaded 34GiB to GPU in 2.06275s (16.4829GiB/s) | 20260505 19:36:48.695 | INFO | Analyzing chunk frames = [5769728,5770240) | 20260505 19:36:48.695 | INFO | MemInfo: { | 20260505 19:36:48.695 | INFO | "gpu" : " 17 GB", | 20260505 19:36:48.695 | INFO | "huge" : "", | 20260505 19:36:48.695 | INFO | "huge_shared" : " 27 GB", | 20260505 19:36:48.695 | INFO | "malloc" : " 14 GB", | 20260505 19:36:48.695 | INFO | "pinned" : " 29 GB" | 20260505 19:36:48.695 | INFO | } | 20260505 19:36:48.695 | INFO | Perf report for the last 5.14025 seconds: | 20260505 19:36:48.695 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:36:48.695 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:48.695 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:48.695 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:48.695 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 19:36:48.695 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:48.695 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:48.695 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:36:48.695 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:48.695 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:48.695 | INFO | Uploaded 36GiB to GPU in 2.14498s (16.7833GiB/s) | 20260505 19:36:48.695 | INFO | Downloaded 34GiB to GPU in 2.05369s (16.5555GiB/s) | 20260505 19:36:53.846 | INFO | Analyzing chunk frames = [5770240,5770752) | 20260505 19:36:53.846 | INFO | MemInfo: { | 20260505 19:36:53.846 | INFO | "gpu" : " 17 GB", | 20260505 19:36:53.846 | INFO | "huge" : "", | 20260505 19:36:53.846 | INFO | "huge_shared" : " 27 GB", | 20260505 19:36:53.846 | INFO | "malloc" : " 14 GB", | 20260505 19:36:53.846 | INFO | "pinned" : " 29 GB" | 20260505 19:36:53.846 | INFO | } | 20260505 19:36:53.846 | INFO | Perf report for the last 5.15159 seconds: | 20260505 19:36:53.846 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:36:53.846 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:53.846 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:53.847 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:53.847 | INFO | ANALYSIS : 63.0%, 3.0, 98.6ms, 0 | 20260505 19:36:53.847 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:53.847 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:53.847 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 19:36:53.847 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:53.847 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:53.847 | INFO | Uploaded 36GiB to GPU in 2.12499s (16.9413GiB/s) | 20260505 19:36:53.847 | INFO | Downloaded 34GiB to GPU in 2.13695s (15.9106GiB/s) | 20260505 19:36:58.975 | INFO | Analyzing chunk frames = [5770752,5771264) | 20260505 19:36:58.975 | INFO | MemInfo: { | 20260505 19:36:58.975 | INFO | "gpu" : " 17 GB", | 20260505 19:36:58.975 | INFO | "huge" : "", | 20260505 19:36:58.975 | INFO | "huge_shared" : " 27 GB", | 20260505 19:36:58.975 | INFO | "malloc" : " 14 GB", | 20260505 19:36:58.975 | INFO | "pinned" : " 29 GB" | 20260505 19:36:58.975 | INFO | } | 20260505 19:36:58.975 | INFO | Perf report for the last 5.12831 seconds: | 20260505 19:36:58.975 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:36:58.975 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:58.975 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:58.975 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:58.975 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260505 19:36:58.975 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:58.975 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:58.975 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 19:36:58.975 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:58.975 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:36:58.975 | INFO | Uploaded 36GiB to GPU in 2.10033s (17.1401GiB/s) | 20260505 19:36:58.975 | INFO | Downloaded 34GiB to GPU in 2.00349s (16.9704GiB/s) | 20260505 19:36:59.107 | INFO | Starting a round of Baz IO | 20260505 19:37:02.281 | INFO | Populating RTMetrics report | 20260505 19:37:02.282 | INFO | RTMetrics report populated | 20260505 19:37:02.282 | INFO | Opening tmp RTMetrics json file | 20260505 19:37:02.283 | INFO | Serializing RTMetrics to json | 20260505 19:37:02.288 | INFO | Serialization done, swapping tmp file now... | 20260505 19:37:02.289 | INFO | RTMetrics done | 20260505 19:37:02.369 | INFO | Finished a round of Baz IO: Wrote 2.047 GiB in 3.262 seconds | 20260505 19:37:02.369 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.76% / 0.00% / 78.24% | 20260505 19:37:02.369 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.203045% / 0% / 9.93411e-05% | 20260505 19:37:02.369 | INFO | Baz IO overall: | 20260505 19:37:02.369 | INFO | BytesWritten : 3378112817481 | 20260505 19:37:02.369 | INFO | NumEvents : 1598348086666 | 20260505 19:37:04.109 | INFO | Analyzing chunk frames = [5771264,5771776) | 20260505 19:37:04.109 | INFO | MemInfo: { | 20260505 19:37:04.109 | INFO | "gpu" : " 17 GB", | 20260505 19:37:04.109 | INFO | "huge" : "", | 20260505 19:37:04.109 | INFO | "huge_shared" : " 27 GB", | 20260505 19:37:04.109 | INFO | "malloc" : " 14 GB", | 20260505 19:37:04.109 | INFO | "pinned" : " 29 GB" | 20260505 19:37:04.109 | INFO | } | 20260505 19:37:04.109 | INFO | Perf report for the last 5.1345 seconds: | 20260505 19:37:04.109 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:37:04.109 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:04.109 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:04.109 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:04.109 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 19:37:04.109 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:04.109 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260505 19:37:04.109 | INFO | PRE_HQ : 60.3%, 1.8, 55.4ms, 0 | 20260505 19:37:04.109 | INFO | BAZWRITER : 63.1%, 3.7, 119.5ms, 0 | 20260505 19:37:04.109 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 19:37:04.110 | INFO | Uploaded 36GiB to GPU in 2.16362s (16.6388GiB/s) | 20260505 19:37:04.110 | INFO | Downloaded 38GiB to GPU in 2.2496s (16.8919GiB/s) | 20260505 19:37:09.250 | INFO | Analyzing chunk frames = [5771776,5772288) | 20260505 19:37:09.250 | INFO | MemInfo: { | 20260505 19:37:09.250 | INFO | "gpu" : " 17 GB", | 20260505 19:37:09.250 | INFO | "huge" : "", | 20260505 19:37:09.250 | INFO | "huge_shared" : " 27 GB", | 20260505 19:37:09.250 | INFO | "malloc" : " 14 GB", | 20260505 19:37:09.250 | INFO | "pinned" : " 29 GB" | 20260505 19:37:09.250 | INFO | } | 20260505 19:37:09.250 | INFO | Perf report for the last 5.14116 seconds: | 20260505 19:37:09.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:37:09.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:09.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:09.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:09.250 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 19:37:09.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:09.251 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:09.251 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:37:09.251 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:09.251 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:09.251 | INFO | Uploaded 36GiB to GPU in 2.13443s (16.8663GiB/s) | 20260505 19:37:09.251 | INFO | Downloaded 34GiB to GPU in 2.03266s (16.7268GiB/s) | 20260505 19:37:14.392 | INFO | Analyzing chunk frames = [5772288,5772800) | 20260505 19:37:14.392 | INFO | MemInfo: { | 20260505 19:37:14.392 | INFO | "gpu" : " 17 GB", | 20260505 19:37:14.392 | INFO | "huge" : "", | 20260505 19:37:14.392 | INFO | "huge_shared" : " 27 GB", | 20260505 19:37:14.392 | INFO | "malloc" : " 14 GB", | 20260505 19:37:14.392 | INFO | "pinned" : " 29 GB" | 20260505 19:37:14.392 | INFO | } | 20260505 19:37:14.392 | INFO | Perf report for the last 5.1416 seconds: | 20260505 19:37:14.392 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:37:14.392 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:14.392 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:14.392 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:14.392 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 19:37:14.392 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:14.392 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:14.392 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:37:14.392 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:14.392 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:14.392 | INFO | Uploaded 36GiB to GPU in 2.21133s (16.2798GiB/s) | 20260505 19:37:14.392 | INFO | Downloaded 34GiB to GPU in 2.06727s (16.4468GiB/s) | 20260505 19:37:19.535 | INFO | Analyzing chunk frames = [5772800,5773312) | 20260505 19:37:19.535 | INFO | MemInfo: { | 20260505 19:37:19.535 | INFO | "gpu" : " 17 GB", | 20260505 19:37:19.535 | INFO | "huge" : "", | 20260505 19:37:19.535 | INFO | "huge_shared" : " 27 GB", | 20260505 19:37:19.535 | INFO | "malloc" : " 14 GB", | 20260505 19:37:19.535 | INFO | "pinned" : " 29 GB" | 20260505 19:37:19.535 | INFO | } | 20260505 19:37:19.535 | INFO | Perf report for the last 5.14274 seconds: | 20260505 19:37:19.535 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:37:19.535 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:19.535 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:19.535 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:19.535 | INFO | ANALYSIS : 62.4%, 3.0, 97.0ms, 0 | 20260505 19:37:19.535 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:19.535 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:19.535 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260505 19:37:19.535 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:19.535 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:19.535 | INFO | Uploaded 36GiB to GPU in 2.21098s (16.2824GiB/s) | 20260505 19:37:19.535 | INFO | Downloaded 34GiB to GPU in 2.04406s (16.6336GiB/s) | 20260505 19:37:24.680 | INFO | Analyzing chunk frames = [5773312,5773824) | 20260505 19:37:24.681 | INFO | MemInfo: { | 20260505 19:37:24.681 | INFO | "gpu" : " 17 GB", | 20260505 19:37:24.681 | INFO | "huge" : "", | 20260505 19:37:24.681 | INFO | "huge_shared" : " 27 GB", | 20260505 19:37:24.681 | INFO | "malloc" : " 14 GB", | 20260505 19:37:24.681 | INFO | "pinned" : " 29 GB" | 20260505 19:37:24.681 | INFO | } | 20260505 19:37:24.681 | INFO | Perf report for the last 5.14595 seconds: | 20260505 19:37:24.681 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:37:24.681 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:24.681 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:24.681 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:24.681 | INFO | ANALYSIS : 61.7%, 3.0, 96.5ms, 0 | 20260505 19:37:24.681 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:24.681 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:24.681 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:37:24.681 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:24.681 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:24.681 | INFO | Uploaded 36GiB to GPU in 2.12365s (16.9519GiB/s) | 20260505 19:37:24.681 | INFO | Downloaded 34GiB to GPU in 2.06147s (16.4931GiB/s) | 20260505 19:37:29.811 | INFO | Analyzing chunk frames = [5773824,5774336) | 20260505 19:37:29.811 | INFO | MemInfo: { | 20260505 19:37:29.811 | INFO | "gpu" : " 17 GB", | 20260505 19:37:29.811 | INFO | "huge" : "", | 20260505 19:37:29.811 | INFO | "huge_shared" : " 27 GB", | 20260505 19:37:29.811 | INFO | "malloc" : " 14 GB", | 20260505 19:37:29.811 | INFO | "pinned" : " 29 GB" | 20260505 19:37:29.811 | INFO | } | 20260505 19:37:29.811 | INFO | Perf report for the last 5.12981 seconds: | 20260505 19:37:29.811 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:37:29.811 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:29.811 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:29.811 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:29.811 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260505 19:37:29.811 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:29.811 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:29.811 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260505 19:37:29.811 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:29.811 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:29.811 | INFO | Uploaded 36GiB to GPU in 2.15856s (16.6778GiB/s) | 20260505 19:37:29.811 | INFO | Downloaded 34GiB to GPU in 2.06522s (16.4632GiB/s) | 20260505 19:37:34.944 | INFO | Analyzing chunk frames = [5774336,5774848) | 20260505 19:37:34.944 | INFO | MemInfo: { | 20260505 19:37:34.944 | INFO | "gpu" : " 17 GB", | 20260505 19:37:34.944 | INFO | "huge" : "", | 20260505 19:37:34.944 | INFO | "huge_shared" : " 27 GB", | 20260505 19:37:34.944 | INFO | "malloc" : " 14 GB", | 20260505 19:37:34.944 | INFO | "pinned" : " 29 GB" | 20260505 19:37:34.944 | INFO | } | 20260505 19:37:34.945 | INFO | Perf report for the last 5.1339 seconds: | 20260505 19:37:34.945 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:37:34.945 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:34.945 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:34.945 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:34.945 | INFO | ANALYSIS : 63.4%, 3.0, 98.8ms, 0 | 20260505 19:37:34.945 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:34.945 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:34.945 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 19:37:34.945 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:34.945 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:34.945 | INFO | Uploaded 36GiB to GPU in 2.13438s (16.8667GiB/s) | 20260505 19:37:34.945 | INFO | Downloaded 34GiB to GPU in 2.14289s (15.8664GiB/s) | 20260505 19:37:40.094 | INFO | Analyzing chunk frames = [5774848,5775360) | 20260505 19:37:40.094 | INFO | MemInfo: { | 20260505 19:37:40.094 | INFO | "gpu" : " 17 GB", | 20260505 19:37:40.094 | INFO | "huge" : "", | 20260505 19:37:40.094 | INFO | "huge_shared" : " 27 GB", | 20260505 19:37:40.094 | INFO | "malloc" : " 14 GB", | 20260505 19:37:40.094 | INFO | "pinned" : " 29 GB" | 20260505 19:37:40.094 | INFO | } | 20260505 19:37:40.094 | INFO | Perf report for the last 5.14966 seconds: | 20260505 19:37:40.094 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:37:40.094 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:40.094 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:40.094 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:40.094 | INFO | ANALYSIS : 62.7%, 3.0, 98.2ms, 0 | 20260505 19:37:40.094 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:40.094 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:40.094 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:37:40.094 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:40.094 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:40.094 | INFO | Uploaded 36GiB to GPU in 2.19702s (16.3858GiB/s) | 20260505 19:37:40.094 | INFO | Downloaded 34GiB to GPU in 2.03735s (16.6883GiB/s) | 20260505 19:37:40.095 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 19:37:40.095 | INFO | Total mem usage: 15.3 GiB | 20260505 19:37:40.095 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 19:37:40.095 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.28%) | 20260505 19:37:40.095 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 19:37:40.095 | INFO | High Water Breakdown: | 20260505 19:37:40.095 | INFO | 61% : 8.63 GiB ( 90.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 19:37:40.095 | INFO | 38.9% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 19:37:40.095 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 19:37:40.095 | INFO | ------------------------------------------ | 20260505 19:37:40.095 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:37:40.095 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 19:37:40.095 | INFO | Total mem usage: 26.5 GiB | 20260505 19:37:40.095 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 19:37:40.095 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 19:37:40.095 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 19:37:40.095 | INFO | High Water Breakdown: | 20260505 19:37:40.095 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:37:40.095 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:37:40.095 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:37:40.095 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:37:40.095 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:37:40.095 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:37:40.095 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 19:37:40.095 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 19:37:40.095 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:37:40.095 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 19:37:40.095 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 19:37:40.095 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 19:37:40.095 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 19:37:40.095 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 19:37:40.095 | INFO | ------------------------------------------ | 20260505 19:37:40.095 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:37:40.095 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 19:37:40.095 | INFO | Total mem usage: 25.6 GiB | 20260505 19:37:40.095 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 19:37:40.095 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 19:37:40.095 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 19:37:40.095 | INFO | High Water Breakdown: | 20260505 19:37:40.095 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 19:37:40.095 | INFO | ------------------------------------------ | 20260505 19:37:40.095 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:37:40.095 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 19:37:40.095 | INFO | Total mem usage: 15.7 GiB | 20260505 19:37:40.095 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 19:37:40.095 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 19:37:40.095 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 19:37:40.095 | INFO | High Water Breakdown: | 20260505 19:37:40.095 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 19:37:40.095 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 19:37:40.095 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 19:37:40.095 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 19:37:40.095 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 19:37:40.095 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 19:37:40.095 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 19:37:40.095 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 19:37:40.095 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 19:37:40.095 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:37:40.095 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 19:37:40.095 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:37:40.095 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 19:37:40.095 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 19:37:40.095 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 19:37:40.095 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 19:37:40.095 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:37:40.095 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 19:37:40.095 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:37:40.095 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:37:40.095 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:37:40.095 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 19:37:40.095 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 19:37:40.095 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:37:40.095 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 19:37:40.096 | INFO | ------------------------------------------ | 20260505 19:37:40.096 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:37:40.096 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 19:37:40.096 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:37:40.096 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:37:40.096 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.46e+01, 1.35e+01 / 2.33e+00 | 20260505 19:37:40.096 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.04e-01, 1.04e-02 / 7.50e-03 | 20260505 19:37:40.096 | INFO | PerfReport: 11.78%, Baseline -- 3.71e+00 / 4.54e+00, 5.61e-01 / 1.18e-01 | 20260505 19:37:40.096 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.53e+00, 1.88e-01 / 8.84e-02 | 20260505 19:37:40.096 | INFO | PerfReport: 15.97%, DME -- 4.02e+01 / 4.44e+01, 4.09e+00 / 2.38e+00 | 20260505 19:37:40.096 | INFO | PerfReport: 56.94%, FrameLabeling -- 1.79e+01 / 1.93e+01, 1.41e+00 / 1.07e+00 | 20260505 19:37:40.096 | INFO | PerfReport: 7.31%, PulseAccumulating -- 2.30e+00 / 2.91e+00, 4.44e-01 / 1.45e-01 | 20260505 19:37:40.096 | INFO | PerfReport: 3.29%, Metrics -- 1.04e+00 / 1.38e+00, 2.87e-01 / 1.58e-01 | 20260505 19:37:40.096 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.59e-01 / 4.33e-01, 6.35e-02 / 3.79e-02 | 20260505 19:37:40.096 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 19:37:40.096 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:37:40.096 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:37:40.096 | INFO | PerfReport: Full Execution -- 9.96e+01 / -nan, 2.57e+01 / 0.00e+00 | 20260505 19:37:40.096 | INFO | PerfReport: 22.53%, Upload -- 2.24e+01 / 0.00e+00, 7.88e+00 / 0.00e+00 | 20260505 19:37:40.096 | INFO | PerfReport: 21.76%, Download -- 2.17e+01 / 0.00e+00, 7.70e+00 / 0.00e+00 | 20260505 19:37:40.096 | INFO | PerfReport: 31.58%, Compute -- 3.15e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 19:37:40.096 | INFO | PerfReport: 2.41%, LockUpload -- 2.40e+00 / 0.00e+00, 8.14e+00 / 0.00e+00 | 20260505 19:37:40.096 | INFO | PerfReport: 1.33%, LockDownload -- 1.33e+00 / 0.00e+00, 2.82e+00 / 0.00e+00 | 20260505 19:37:40.096 | INFO | PerfReport: 20.38%, LockCompute -- 2.03e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 19:37:40.096 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.23e-02 / 0.00e+00 | 20260505 19:37:40.228 | INFO | Starting a round of Baz IO | 20260505 19:37:43.394 | INFO | Populating RTMetrics report | 20260505 19:37:43.395 | INFO | RTMetrics report populated | 20260505 19:37:43.395 | INFO | Opening tmp RTMetrics json file | 20260505 19:37:43.396 | INFO | Serializing RTMetrics to json | 20260505 19:37:43.397 | INFO | Serialization done, swapping tmp file now... | 20260505 19:37:43.397 | INFO | RTMetrics done | 20260505 19:37:43.485 | INFO | Finished a round of Baz IO: Wrote 2.045 GiB in 3.256 seconds | 20260505 19:37:43.485 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.76% / 0.00% / 78.24% | 20260505 19:37:43.485 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.202417% / 0% / 7.54992e-05% | 20260505 19:37:43.485 | INFO | Baz IO overall: | 20260505 19:37:43.485 | INFO | BytesWritten : 3380308739460 | 20260505 19:37:43.485 | INFO | NumEvents : 1599368035564 | 20260505 19:37:45.233 | INFO | Analyzing chunk frames = [5775360,5775872) | 20260505 19:37:45.233 | INFO | MemInfo: { | 20260505 19:37:45.233 | INFO | "gpu" : " 17 GB", | 20260505 19:37:45.233 | INFO | "huge" : "", | 20260505 19:37:45.233 | INFO | "huge_shared" : " 27 GB", | 20260505 19:37:45.233 | INFO | "malloc" : " 14 GB", | 20260505 19:37:45.233 | INFO | "pinned" : " 29 GB" | 20260505 19:37:45.233 | INFO | } | 20260505 19:37:45.233 | INFO | Perf report for the last 5.13908 seconds: | 20260505 19:37:45.233 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:37:45.233 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:45.233 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:45.233 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:45.233 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 19:37:45.233 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:45.233 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260505 19:37:45.233 | INFO | PRE_HQ : 60.0%, 1.8, 55.1ms, 0 | 20260505 19:37:45.233 | INFO | BAZWRITER : 62.9%, 3.7, 120.2ms, 0 | 20260505 19:37:45.234 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 19:37:45.234 | INFO | Uploaded 36GiB to GPU in 2.17386s (16.5604GiB/s) | 20260505 19:37:45.234 | INFO | Downloaded 38GiB to GPU in 2.24747s (16.9079GiB/s) | 20260505 19:37:50.372 | INFO | Analyzing chunk frames = [5775872,5776384) | 20260505 19:37:50.372 | INFO | MemInfo: { | 20260505 19:37:50.372 | INFO | "gpu" : " 17 GB", | 20260505 19:37:50.372 | INFO | "huge" : "", | 20260505 19:37:50.372 | INFO | "huge_shared" : " 27 GB", | 20260505 19:37:50.372 | INFO | "malloc" : " 14 GB", | 20260505 19:37:50.372 | INFO | "pinned" : " 29 GB" | 20260505 19:37:50.373 | INFO | } | 20260505 19:37:50.373 | INFO | Perf report for the last 5.13936 seconds: | 20260505 19:37:50.373 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:37:50.373 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:50.373 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:50.373 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:50.373 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 19:37:50.373 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:50.373 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:50.373 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:37:50.373 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:50.373 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:50.373 | INFO | Uploaded 36GiB to GPU in 2.10227s (17.1243GiB/s) | 20260505 19:37:50.373 | INFO | Downloaded 34GiB to GPU in 2.02607s (16.7813GiB/s) | 20260505 19:37:55.502 | INFO | Analyzing chunk frames = [5776384,5776896) | 20260505 19:37:55.502 | INFO | MemInfo: { | 20260505 19:37:55.502 | INFO | "gpu" : " 17 GB", | 20260505 19:37:55.502 | INFO | "huge" : "", | 20260505 19:37:55.502 | INFO | "huge_shared" : " 27 GB", | 20260505 19:37:55.502 | INFO | "malloc" : " 14 GB", | 20260505 19:37:55.502 | INFO | "pinned" : " 29 GB" | 20260505 19:37:55.502 | INFO | } | 20260505 19:37:55.503 | INFO | Perf report for the last 5.12986 seconds: | 20260505 19:37:55.503 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:37:55.503 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:55.503 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:55.503 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:55.503 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260505 19:37:55.503 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:55.503 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:55.503 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:37:55.503 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:55.503 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:37:55.503 | INFO | Uploaded 36GiB to GPU in 2.12627s (16.931GiB/s) | 20260505 19:37:55.503 | INFO | Downloaded 34GiB to GPU in 2.02294s (16.8073GiB/s) | 20260505 19:38:00.648 | INFO | Analyzing chunk frames = [5776896,5777408) | 20260505 19:38:00.648 | INFO | MemInfo: { | 20260505 19:38:00.648 | INFO | "gpu" : " 17 GB", | 20260505 19:38:00.648 | INFO | "huge" : "", | 20260505 19:38:00.648 | INFO | "huge_shared" : " 27 GB", | 20260505 19:38:00.648 | INFO | "malloc" : " 14 GB", | 20260505 19:38:00.648 | INFO | "pinned" : " 29 GB" | 20260505 19:38:00.648 | INFO | } | 20260505 19:38:00.648 | INFO | Perf report for the last 5.14588 seconds: | 20260505 19:38:00.648 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:38:00.648 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:00.648 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:00.648 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:00.648 | INFO | ANALYSIS : 62.6%, 3.0, 97.4ms, 0 | 20260505 19:38:00.648 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:00.649 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:00.649 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:38:00.649 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:00.649 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:00.649 | INFO | Uploaded 36GiB to GPU in 2.22254s (16.1977GiB/s) | 20260505 19:38:00.649 | INFO | Downloaded 34GiB to GPU in 2.12725s (15.983GiB/s) | 20260505 19:38:05.791 | INFO | Analyzing chunk frames = [5777408,5777920) | 20260505 19:38:05.791 | INFO | MemInfo: { | 20260505 19:38:05.791 | INFO | "gpu" : " 17 GB", | 20260505 19:38:05.791 | INFO | "huge" : "", | 20260505 19:38:05.791 | INFO | "huge_shared" : " 27 GB", | 20260505 19:38:05.791 | INFO | "malloc" : " 14 GB", | 20260505 19:38:05.791 | INFO | "pinned" : " 29 GB" | 20260505 19:38:05.791 | INFO | } | 20260505 19:38:05.791 | INFO | Perf report for the last 5.14272 seconds: | 20260505 19:38:05.791 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:38:05.791 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:05.791 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:05.791 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:05.791 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 19:38:05.791 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:05.791 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:05.791 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:38:05.791 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:05.791 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:05.792 | INFO | Uploaded 36GiB to GPU in 2.1371s (16.8453GiB/s) | 20260505 19:38:05.792 | INFO | Downloaded 34GiB to GPU in 2.06053s (16.5006GiB/s) | 20260505 19:38:10.928 | INFO | Analyzing chunk frames = [5777920,5778432) | 20260505 19:38:10.928 | INFO | MemInfo: { | 20260505 19:38:10.928 | INFO | "gpu" : " 17 GB", | 20260505 19:38:10.928 | INFO | "huge" : "", | 20260505 19:38:10.928 | INFO | "huge_shared" : " 27 GB", | 20260505 19:38:10.928 | INFO | "malloc" : " 14 GB", | 20260505 19:38:10.928 | INFO | "pinned" : " 29 GB" | 20260505 19:38:10.928 | INFO | } | 20260505 19:38:10.928 | INFO | Perf report for the last 5.13677 seconds: | 20260505 19:38:10.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:38:10.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:10.928 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:10.928 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:10.928 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 19:38:10.928 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:10.928 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:10.928 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:38:10.928 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:10.928 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:10.928 | INFO | Uploaded 36GiB to GPU in 2.14117s (16.8133GiB/s) | 20260505 19:38:10.928 | INFO | Downloaded 34GiB to GPU in 2.05873s (16.515GiB/s) | 20260505 19:38:16.065 | INFO | Analyzing chunk frames = [5778432,5778944) | 20260505 19:38:16.065 | INFO | MemInfo: { | 20260505 19:38:16.065 | INFO | "gpu" : " 17 GB", | 20260505 19:38:16.065 | INFO | "huge" : "", | 20260505 19:38:16.065 | INFO | "huge_shared" : " 27 GB", | 20260505 19:38:16.065 | INFO | "malloc" : " 14 GB", | 20260505 19:38:16.065 | INFO | "pinned" : " 29 GB" | 20260505 19:38:16.065 | INFO | } | 20260505 19:38:16.065 | INFO | Perf report for the last 5.13697 seconds: | 20260505 19:38:16.065 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:38:16.065 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:16.065 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:16.065 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:16.065 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 19:38:16.065 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:16.065 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:16.065 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 19:38:16.065 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:16.065 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:16.065 | INFO | Uploaded 36GiB to GPU in 2.11502s (17.0211GiB/s) | 20260505 19:38:16.065 | INFO | Downloaded 34GiB to GPU in 2.13444s (15.9292GiB/s) | 20260505 19:38:21.200 | INFO | Analyzing chunk frames = [5778944,5779456) | 20260505 19:38:21.200 | INFO | MemInfo: { | 20260505 19:38:21.200 | INFO | "gpu" : " 17 GB", | 20260505 19:38:21.200 | INFO | "huge" : "", | 20260505 19:38:21.200 | INFO | "huge_shared" : " 27 GB", | 20260505 19:38:21.200 | INFO | "malloc" : " 14 GB", | 20260505 19:38:21.200 | INFO | "pinned" : " 29 GB" | 20260505 19:38:21.200 | INFO | } | 20260505 19:38:21.200 | INFO | Perf report for the last 5.13514 seconds: | 20260505 19:38:21.200 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:38:21.200 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:21.200 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:21.200 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:21.200 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 19:38:21.200 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:21.200 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:21.200 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:38:21.200 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:21.200 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:21.201 | INFO | Uploaded 36GiB to GPU in 2.20119s (16.3548GiB/s) | 20260505 19:38:21.201 | INFO | Downloaded 34GiB to GPU in 2.04004s (16.6663GiB/s) | 20260505 19:38:21.336 | INFO | Starting a round of Baz IO | 20260505 19:38:24.505 | INFO | Populating RTMetrics report | 20260505 19:38:24.505 | INFO | RTMetrics report populated | 20260505 19:38:24.505 | INFO | Opening tmp RTMetrics json file | 20260505 19:38:24.506 | INFO | Serializing RTMetrics to json | 20260505 19:38:24.507 | INFO | Serialization done, swapping tmp file now... | 20260505 19:38:24.508 | INFO | RTMetrics done | 20260505 19:38:24.594 | INFO | Finished a round of Baz IO: Wrote 2.044 GiB in 3.257 seconds | 20260505 19:38:24.594 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.76% / 0.00% / 78.24% | 20260505 19:38:24.594 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.202469% / 3.97364e-06% / 9.53674e-05% | 20260505 19:38:24.594 | INFO | Baz IO overall: | 20260505 19:38:24.594 | INFO | BytesWritten : 3382503551179 | 20260505 19:38:24.594 | INFO | NumEvents : 1600387622704 | 20260505 19:38:26.348 | INFO | Analyzing chunk frames = [5779456,5779968) | 20260505 19:38:26.348 | INFO | MemInfo: { | 20260505 19:38:26.348 | INFO | "gpu" : " 17 GB", | 20260505 19:38:26.348 | INFO | "huge" : "", | 20260505 19:38:26.348 | INFO | "huge_shared" : " 27 GB", | 20260505 19:38:26.348 | INFO | "malloc" : " 14 GB", | 20260505 19:38:26.348 | INFO | "pinned" : " 29 GB" | 20260505 19:38:26.348 | INFO | } | 20260505 19:38:26.348 | INFO | Perf report for the last 5.14815 seconds: | 20260505 19:38:26.348 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:38:26.348 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:26.348 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:26.348 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:26.348 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 19:38:26.348 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:26.348 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260505 19:38:26.348 | INFO | PRE_HQ : 60.1%, 1.8, 55.3ms, 0 | 20260505 19:38:26.348 | INFO | BAZWRITER : 62.8%, 3.7, 118.7ms, 0 | 20260505 19:38:26.348 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 19:38:26.349 | INFO | Uploaded 36GiB to GPU in 2.15355s (16.7166GiB/s) | 20260505 19:38:26.349 | INFO | Downloaded 38GiB to GPU in 2.24493s (16.927GiB/s) | 20260505 19:38:31.490 | INFO | Analyzing chunk frames = [5779968,5780480) | 20260505 19:38:31.490 | INFO | MemInfo: { | 20260505 19:38:31.490 | INFO | "gpu" : " 17 GB", | 20260505 19:38:31.490 | INFO | "huge" : "", | 20260505 19:38:31.490 | INFO | "huge_shared" : " 27 GB", | 20260505 19:38:31.490 | INFO | "malloc" : " 14 GB", | 20260505 19:38:31.490 | INFO | "pinned" : " 29 GB" | 20260505 19:38:31.490 | INFO | } | 20260505 19:38:31.490 | INFO | Perf report for the last 5.14214 seconds: | 20260505 19:38:31.491 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:38:31.491 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:31.491 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:31.491 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:31.491 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 19:38:31.491 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:31.491 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:31.491 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:38:31.491 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:31.491 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:31.491 | INFO | Uploaded 36GiB to GPU in 2.08854s (17.2369GiB/s) | 20260505 19:38:31.491 | INFO | Downloaded 34GiB to GPU in 2.02537s (16.787GiB/s) | 20260505 19:38:36.620 | INFO | Analyzing chunk frames = [5780480,5780992) | 20260505 19:38:36.620 | INFO | MemInfo: { | 20260505 19:38:36.620 | INFO | "gpu" : " 17 GB", | 20260505 19:38:36.620 | INFO | "huge" : "", | 20260505 19:38:36.620 | INFO | "huge_shared" : " 27 GB", | 20260505 19:38:36.620 | INFO | "malloc" : " 14 GB", | 20260505 19:38:36.620 | INFO | "pinned" : " 29 GB" | 20260505 19:38:36.620 | INFO | } | 20260505 19:38:36.620 | INFO | Perf report for the last 5.12972 seconds: | 20260505 19:38:36.620 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:38:36.620 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:36.620 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:36.620 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:36.620 | INFO | ANALYSIS : 62.2%, 3.0, 96.6ms, 0 | 20260505 19:38:36.620 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:36.620 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:36.620 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260505 19:38:36.620 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:36.620 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:36.621 | INFO | Uploaded 36GiB to GPU in 2.13425s (16.8678GiB/s) | 20260505 19:38:36.621 | INFO | Downloaded 34GiB to GPU in 2.01752s (16.8524GiB/s) | 20260505 19:38:41.756 | INFO | Analyzing chunk frames = [5780992,5781504) | 20260505 19:38:41.756 | INFO | MemInfo: { | 20260505 19:38:41.756 | INFO | "gpu" : " 17 GB", | 20260505 19:38:41.756 | INFO | "huge" : "", | 20260505 19:38:41.756 | INFO | "huge_shared" : " 27 GB", | 20260505 19:38:41.756 | INFO | "malloc" : " 14 GB", | 20260505 19:38:41.756 | INFO | "pinned" : " 29 GB" | 20260505 19:38:41.756 | INFO | } | 20260505 19:38:41.756 | INFO | Perf report for the last 5.13615 seconds: | 20260505 19:38:41.756 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:38:41.756 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:41.756 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:41.756 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:41.756 | INFO | ANALYSIS : 62.4%, 3.0, 96.8ms, 0 | 20260505 19:38:41.756 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:41.756 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:41.756 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:38:41.756 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:41.756 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:41.757 | INFO | Uploaded 36GiB to GPU in 2.16898s (16.5977GiB/s) | 20260505 19:38:41.757 | INFO | Downloaded 34GiB to GPU in 2.06457s (16.4683GiB/s) | 20260505 19:38:46.902 | INFO | Analyzing chunk frames = [5781504,5782016) | 20260505 19:38:46.902 | INFO | MemInfo: { | 20260505 19:38:46.902 | INFO | "gpu" : " 17 GB", | 20260505 19:38:46.902 | INFO | "huge" : "", | 20260505 19:38:46.902 | INFO | "huge_shared" : " 27 GB", | 20260505 19:38:46.902 | INFO | "malloc" : " 14 GB", | 20260505 19:38:46.902 | INFO | "pinned" : " 29 GB" | 20260505 19:38:46.902 | INFO | } | 20260505 19:38:46.902 | INFO | Perf report for the last 5.14603 seconds: | 20260505 19:38:46.902 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:38:46.902 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:46.902 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:46.902 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:46.902 | INFO | ANALYSIS : 61.9%, 3.0, 96.8ms, 0 | 20260505 19:38:46.902 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:46.902 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:46.902 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:38:46.903 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:46.903 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:46.903 | INFO | Uploaded 36GiB to GPU in 2.13672s (16.8483GiB/s) | 20260505 19:38:46.903 | INFO | Downloaded 34GiB to GPU in 2.06534s (16.4622GiB/s) | 20260505 19:38:52.048 | INFO | Analyzing chunk frames = [5782016,5782528) | 20260505 19:38:52.048 | INFO | MemInfo: { | 20260505 19:38:52.048 | INFO | "gpu" : " 17 GB", | 20260505 19:38:52.048 | INFO | "huge" : "", | 20260505 19:38:52.048 | INFO | "huge_shared" : " 27 GB", | 20260505 19:38:52.048 | INFO | "malloc" : " 14 GB", | 20260505 19:38:52.048 | INFO | "pinned" : " 29 GB" | 20260505 19:38:52.048 | INFO | } | 20260505 19:38:52.048 | INFO | Perf report for the last 5.14597 seconds: | 20260505 19:38:52.048 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:38:52.049 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:52.049 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:52.049 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:52.049 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260505 19:38:52.049 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:52.049 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:52.049 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 19:38:52.049 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:52.049 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:52.049 | INFO | Uploaded 36GiB to GPU in 2.14397s (16.7913GiB/s) | 20260505 19:38:52.049 | INFO | Downloaded 34GiB to GPU in 2.05679s (16.5306GiB/s) | 20260505 19:38:57.187 | INFO | Analyzing chunk frames = [5782528,5783040) | 20260505 19:38:57.187 | INFO | MemInfo: { | 20260505 19:38:57.187 | INFO | "gpu" : " 17 GB", | 20260505 19:38:57.187 | INFO | "huge" : "", | 20260505 19:38:57.187 | INFO | "huge_shared" : " 27 GB", | 20260505 19:38:57.187 | INFO | "malloc" : " 14 GB", | 20260505 19:38:57.188 | INFO | "pinned" : " 29 GB" | 20260505 19:38:57.188 | INFO | } | 20260505 19:38:57.188 | INFO | Perf report for the last 5.1394 seconds: | 20260505 19:38:57.188 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:38:57.188 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:57.188 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:57.188 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:57.188 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 19:38:57.188 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:57.188 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:57.188 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 19:38:57.188 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:57.188 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:38:57.188 | INFO | Uploaded 36GiB to GPU in 2.14142s (16.8113GiB/s) | 20260505 19:38:57.188 | INFO | Downloaded 34GiB to GPU in 2.14581s (15.8448GiB/s) | 20260505 19:39:02.321 | INFO | Analyzing chunk frames = [5783040,5783552) | 20260505 19:39:02.321 | INFO | MemInfo: { | 20260505 19:39:02.321 | INFO | "gpu" : " 17 GB", | 20260505 19:39:02.321 | INFO | "huge" : "", | 20260505 19:39:02.321 | INFO | "huge_shared" : " 27 GB", | 20260505 19:39:02.321 | INFO | "malloc" : " 14 GB", | 20260505 19:39:02.321 | INFO | "pinned" : " 29 GB" | 20260505 19:39:02.321 | INFO | } | 20260505 19:39:02.321 | INFO | Perf report for the last 5.13331 seconds: | 20260505 19:39:02.321 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:39:02.321 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:02.321 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:02.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:02.321 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260505 19:39:02.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:02.321 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:02.321 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:39:02.321 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:02.321 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:02.321 | INFO | Uploaded 36GiB to GPU in 2.10175s (17.1286GiB/s) | 20260505 19:39:02.321 | INFO | Downloaded 34GiB to GPU in 2.00517s (16.9562GiB/s) | 20260505 19:39:02.321 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 19:39:02.321 | INFO | Total mem usage: 15.3 GiB | 20260505 19:39:02.321 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 19:39:02.321 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.16%) | 20260505 19:39:02.321 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 19:39:02.321 | INFO | High Water Breakdown: | 20260505 19:39:02.321 | INFO | 61% : 8.64 GiB ( 90.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 19:39:02.322 | INFO | 38.9% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 19:39:02.322 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 19:39:02.322 | INFO | ------------------------------------------ | 20260505 19:39:02.322 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:39:02.322 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 19:39:02.322 | INFO | Total mem usage: 26.5 GiB | 20260505 19:39:02.322 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 19:39:02.322 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 19:39:02.322 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 19:39:02.322 | INFO | High Water Breakdown: | 20260505 19:39:02.322 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:39:02.322 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:39:02.322 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:39:02.322 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:39:02.322 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:39:02.322 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:39:02.322 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 19:39:02.322 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 19:39:02.322 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:39:02.322 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 19:39:02.322 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 19:39:02.322 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 19:39:02.322 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 19:39:02.322 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 19:39:02.322 | INFO | ------------------------------------------ | 20260505 19:39:02.322 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:39:02.322 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 19:39:02.322 | INFO | Total mem usage: 25.6 GiB | 20260505 19:39:02.322 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 19:39:02.322 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 19:39:02.322 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 19:39:02.322 | INFO | High Water Breakdown: | 20260505 19:39:02.322 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 19:39:02.322 | INFO | ------------------------------------------ | 20260505 19:39:02.322 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:39:02.322 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 19:39:02.322 | INFO | Total mem usage: 15.7 GiB | 20260505 19:39:02.322 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 19:39:02.322 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 19:39:02.322 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 19:39:02.322 | INFO | High Water Breakdown: | 20260505 19:39:02.322 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 19:39:02.322 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 19:39:02.322 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 19:39:02.322 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 19:39:02.322 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 19:39:02.322 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 19:39:02.322 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 19:39:02.322 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 19:39:02.322 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 19:39:02.322 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:39:02.322 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 19:39:02.322 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:39:02.322 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 19:39:02.322 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 19:39:02.322 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 19:39:02.322 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 19:39:02.322 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:39:02.322 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 19:39:02.322 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:39:02.322 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:39:02.322 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:39:02.322 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 19:39:02.322 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 19:39:02.322 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:39:02.322 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 19:39:02.322 | INFO | ------------------------------------------ | 20260505 19:39:02.322 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:39:02.322 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 19:39:02.322 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:39:02.322 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:39:02.322 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.49e+01, 1.35e+01 / 1.94e+00 | 20260505 19:39:02.322 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.02e-01, 1.12e-02 / 3.61e-03 | 20260505 19:39:02.322 | INFO | PerfReport: 11.77%, Baseline -- 3.71e+00 / 4.53e+00, 5.60e-01 / 1.45e-01 | 20260505 19:39:02.322 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.53e+00, 1.93e-01 / 8.13e-02 | 20260505 19:39:02.323 | INFO | PerfReport: 16.00%, DME -- 4.03e+01 / 4.48e+01, 4.25e+00 / 2.05e+00 | 20260505 19:39:02.323 | INFO | PerfReport: 56.98%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.47e+00 / 1.10e+00 | 20260505 19:39:02.323 | INFO | PerfReport: 7.25%, PulseAccumulating -- 2.28e+00 / 2.88e+00, 4.36e-01 / 1.69e-01 | 20260505 19:39:02.323 | INFO | PerfReport: 3.28%, Metrics -- 1.03e+00 / 1.36e+00, 2.83e-01 / 1.28e-01 | 20260505 19:39:02.323 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.60e-01 / 4.24e-01, 6.35e-02 / 3.34e-02 | 20260505 19:39:02.323 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 19:39:02.323 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:39:02.323 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:39:02.323 | INFO | PerfReport: Full Execution -- 9.95e+01 / -nan, 2.55e+01 / 0.00e+00 | 20260505 19:39:02.323 | INFO | PerfReport: 22.44%, Upload -- 2.23e+01 / 0.00e+00, 7.75e+00 / 0.00e+00 | 20260505 19:39:02.323 | INFO | PerfReport: 21.82%, Download -- 2.17e+01 / 0.00e+00, 7.71e+00 / 0.00e+00 | 20260505 19:39:02.323 | INFO | PerfReport: 31.68%, Compute -- 3.15e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 19:39:02.323 | INFO | PerfReport: 2.28%, LockUpload -- 2.27e+00 / 0.00e+00, 7.82e+00 / 0.00e+00 | 20260505 19:39:02.323 | INFO | PerfReport: 1.36%, LockDownload -- 1.36e+00 / 0.00e+00, 2.87e+00 / 0.00e+00 | 20260505 19:39:02.323 | INFO | PerfReport: 20.39%, LockCompute -- 2.03e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 19:39:02.323 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.74e-02 / 0.00e+00 | 20260505 19:39:02.452 | INFO | Starting a round of Baz IO | 20260505 19:39:05.623 | INFO | Populating RTMetrics report | 20260505 19:39:05.624 | INFO | RTMetrics report populated | 20260505 19:39:05.624 | INFO | Opening tmp RTMetrics json file | 20260505 19:39:05.624 | INFO | Serializing RTMetrics to json | 20260505 19:39:05.625 | INFO | Serialization done, swapping tmp file now... | 20260505 19:39:05.626 | INFO | RTMetrics done | 20260505 19:39:05.709 | INFO | Finished a round of Baz IO: Wrote 2.043 GiB in 3.256 seconds | 20260505 19:39:05.709 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.75% / 0.00% / 78.24% | 20260505 19:39:05.709 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.201551% / 0% / 9.13938e-05% | 20260505 19:39:05.709 | INFO | Baz IO overall: | 20260505 19:39:05.709 | INFO | BytesWritten : 3384696853912 | 20260505 19:39:05.709 | INFO | NumEvents : 1601406401172 | 20260505 19:39:07.459 | INFO | Analyzing chunk frames = [5783552,5784064) | 20260505 19:39:07.459 | INFO | MemInfo: { | 20260505 19:39:07.459 | INFO | "gpu" : " 17 GB", | 20260505 19:39:07.459 | INFO | "huge" : "", | 20260505 19:39:07.459 | INFO | "huge_shared" : " 27 GB", | 20260505 19:39:07.459 | INFO | "malloc" : " 14 GB", | 20260505 19:39:07.459 | INFO | "pinned" : " 29 GB" | 20260505 19:39:07.459 | INFO | } | 20260505 19:39:07.459 | INFO | Perf report for the last 5.13758 seconds: | 20260505 19:39:07.459 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:39:07.459 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:07.459 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:07.459 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:07.459 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 19:39:07.459 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:07.459 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260505 19:39:07.459 | INFO | PRE_HQ : 60.2%, 1.8, 55.4ms, 0 | 20260505 19:39:07.459 | INFO | BAZWRITER : 63.0%, 3.7, 119.1ms, 0 | 20260505 19:39:07.459 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 19:39:07.459 | INFO | Uploaded 36GiB to GPU in 2.15161s (16.7316GiB/s) | 20260505 19:39:07.459 | INFO | Downloaded 38GiB to GPU in 2.24204s (16.9489GiB/s) | 20260505 19:39:12.601 | INFO | Analyzing chunk frames = [5784064,5784576) | 20260505 19:39:12.601 | INFO | MemInfo: { | 20260505 19:39:12.601 | INFO | "gpu" : " 17 GB", | 20260505 19:39:12.601 | INFO | "huge" : "", | 20260505 19:39:12.601 | INFO | "huge_shared" : " 27 GB", | 20260505 19:39:12.601 | INFO | "malloc" : " 14 GB", | 20260505 19:39:12.601 | INFO | "pinned" : " 29 GB" | 20260505 19:39:12.601 | INFO | } | 20260505 19:39:12.601 | INFO | Perf report for the last 5.14247 seconds: | 20260505 19:39:12.601 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:39:12.601 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:12.601 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:12.601 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:12.601 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260505 19:39:12.601 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:12.601 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:12.601 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:39:12.601 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:12.601 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:12.601 | INFO | Uploaded 36GiB to GPU in 2.12189s (16.966GiB/s) | 20260505 19:39:12.601 | INFO | Downloaded 34GiB to GPU in 2.02871s (16.7594GiB/s) | 20260505 19:39:17.744 | INFO | Analyzing chunk frames = [5784576,5785088) | 20260505 19:39:17.744 | INFO | MemInfo: { | 20260505 19:39:17.744 | INFO | "gpu" : " 17 GB", | 20260505 19:39:17.744 | INFO | "huge" : "", | 20260505 19:39:17.744 | INFO | "huge_shared" : " 27 GB", | 20260505 19:39:17.744 | INFO | "malloc" : " 14 GB", | 20260505 19:39:17.744 | INFO | "pinned" : " 29 GB" | 20260505 19:39:17.744 | INFO | } | 20260505 19:39:17.744 | INFO | Perf report for the last 5.14295 seconds: | 20260505 19:39:17.744 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:39:17.744 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:17.744 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:17.744 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:17.744 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260505 19:39:17.744 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:17.744 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:17.744 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260505 19:39:17.744 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:17.744 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:17.744 | INFO | Uploaded 36GiB to GPU in 2.12494s (16.9417GiB/s) | 20260505 19:39:17.744 | INFO | Downloaded 34GiB to GPU in 2.01852s (16.844GiB/s) | 20260505 19:39:22.883 | INFO | Analyzing chunk frames = [5785088,5785600) | 20260505 19:39:22.883 | INFO | MemInfo: { | 20260505 19:39:22.883 | INFO | "gpu" : " 17 GB", | 20260505 19:39:22.883 | INFO | "huge" : "", | 20260505 19:39:22.883 | INFO | "huge_shared" : " 27 GB", | 20260505 19:39:22.883 | INFO | "malloc" : " 14 GB", | 20260505 19:39:22.883 | INFO | "pinned" : " 29 GB" | 20260505 19:39:22.883 | INFO | } | 20260505 19:39:22.883 | INFO | Perf report for the last 5.13928 seconds: | 20260505 19:39:22.884 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:39:22.884 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:22.884 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:22.884 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:22.884 | INFO | ANALYSIS : 62.5%, 3.0, 97.0ms, 0 | 20260505 19:39:22.884 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:22.884 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:22.884 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:39:22.884 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:22.884 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:22.884 | INFO | Uploaded 36GiB to GPU in 2.20293s (16.3419GiB/s) | 20260505 19:39:22.884 | INFO | Downloaded 34GiB to GPU in 2.04088s (16.6594GiB/s) | 20260505 19:39:28.013 | INFO | Analyzing chunk frames = [5785600,5786112) | 20260505 19:39:28.013 | INFO | MemInfo: { | 20260505 19:39:28.013 | INFO | "gpu" : " 17 GB", | 20260505 19:39:28.013 | INFO | "huge" : "", | 20260505 19:39:28.013 | INFO | "huge_shared" : " 27 GB", | 20260505 19:39:28.013 | INFO | "malloc" : " 14 GB", | 20260505 19:39:28.013 | INFO | "pinned" : " 29 GB" | 20260505 19:39:28.013 | INFO | } | 20260505 19:39:28.013 | INFO | Perf report for the last 5.12998 seconds: | 20260505 19:39:28.013 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:39:28.013 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:28.013 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:28.013 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:28.013 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260505 19:39:28.013 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:28.013 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:28.013 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260505 19:39:28.013 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:28.013 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:28.014 | INFO | Uploaded 36GiB to GPU in 2.1317s (16.8879GiB/s) | 20260505 19:39:28.014 | INFO | Downloaded 34GiB to GPU in 2.06104s (16.4965GiB/s) | 20260505 19:39:33.157 | INFO | Analyzing chunk frames = [5786112,5786624) | 20260505 19:39:33.157 | INFO | MemInfo: { | 20260505 19:39:33.157 | INFO | "gpu" : " 17 GB", | 20260505 19:39:33.157 | INFO | "huge" : "", | 20260505 19:39:33.157 | INFO | "huge_shared" : " 27 GB", | 20260505 19:39:33.157 | INFO | "malloc" : " 14 GB", | 20260505 19:39:33.157 | INFO | "pinned" : " 29 GB" | 20260505 19:39:33.157 | INFO | } | 20260505 19:39:33.158 | INFO | Perf report for the last 5.14423 seconds: | 20260505 19:39:33.158 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:39:33.158 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:33.158 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:33.158 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:33.158 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260505 19:39:33.158 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:33.158 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:33.158 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:39:33.158 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:33.158 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:33.158 | INFO | Uploaded 36GiB to GPU in 2.14648s (16.7716GiB/s) | 20260505 19:39:33.158 | INFO | Downloaded 34GiB to GPU in 2.04937s (16.5904GiB/s) | 20260505 19:39:38.298 | INFO | Analyzing chunk frames = [5786624,5787136) | 20260505 19:39:38.298 | INFO | MemInfo: { | 20260505 19:39:38.298 | INFO | "gpu" : " 17 GB", | 20260505 19:39:38.298 | INFO | "huge" : "", | 20260505 19:39:38.298 | INFO | "huge_shared" : " 27 GB", | 20260505 19:39:38.298 | INFO | "malloc" : " 14 GB", | 20260505 19:39:38.298 | INFO | "pinned" : " 29 GB" | 20260505 19:39:38.298 | INFO | } | 20260505 19:39:38.298 | INFO | Perf report for the last 5.14072 seconds: | 20260505 19:39:38.298 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:39:38.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:38.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:38.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:38.299 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 19:39:38.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:38.299 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:38.299 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 19:39:38.299 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:38.299 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:38.299 | INFO | Uploaded 36GiB to GPU in 2.11407s (17.0287GiB/s) | 20260505 19:39:38.299 | INFO | Downloaded 34GiB to GPU in 2.13788s (15.9036GiB/s) | 20260505 19:39:43.436 | INFO | Analyzing chunk frames = [5787136,5787648) | 20260505 19:39:43.436 | INFO | MemInfo: { | 20260505 19:39:43.436 | INFO | "gpu" : " 17 GB", | 20260505 19:39:43.436 | INFO | "huge" : "", | 20260505 19:39:43.436 | INFO | "huge_shared" : " 27 GB", | 20260505 19:39:43.436 | INFO | "malloc" : " 14 GB", | 20260505 19:39:43.436 | INFO | "pinned" : " 29 GB" | 20260505 19:39:43.436 | INFO | } | 20260505 19:39:43.436 | INFO | Perf report for the last 5.13767 seconds: | 20260505 19:39:43.436 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:39:43.436 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:43.436 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:43.436 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:43.436 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 19:39:43.436 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:43.436 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:43.436 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 19:39:43.436 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:43.436 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:43.436 | INFO | Uploaded 36GiB to GPU in 2.10011s (17.142GiB/s) | 20260505 19:39:43.436 | INFO | Downloaded 34GiB to GPU in 1.9972s (17.0239GiB/s) | 20260505 19:39:43.570 | INFO | Starting a round of Baz IO | 20260505 19:39:46.738 | INFO | Populating RTMetrics report | 20260505 19:39:46.738 | INFO | RTMetrics report populated | 20260505 19:39:46.738 | INFO | Opening tmp RTMetrics json file | 20260505 19:39:46.739 | INFO | Serializing RTMetrics to json | 20260505 19:39:46.740 | INFO | Serialization done, swapping tmp file now... | 20260505 19:39:46.740 | INFO | RTMetrics done | 20260505 19:39:46.831 | INFO | Finished a round of Baz IO: Wrote 2.044 GiB in 3.261 seconds | 20260505 19:39:46.831 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.75% / 0.00% / 78.25% | 20260505 19:39:46.831 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.201142% / 3.97364e-06% / 7.94729e-05% | 20260505 19:39:46.831 | INFO | Baz IO overall: | 20260505 19:39:46.831 | INFO | BytesWritten : 3386891544694 | 20260505 19:39:46.831 | INFO | NumEvents : 1602426342021 | 20260505 19:39:48.572 | INFO | Analyzing chunk frames = [5787648,5788160) | 20260505 19:39:48.572 | INFO | MemInfo: { | 20260505 19:39:48.572 | INFO | "gpu" : " 17 GB", | 20260505 19:39:48.572 | INFO | "huge" : "", | 20260505 19:39:48.572 | INFO | "huge_shared" : " 27 GB", | 20260505 19:39:48.572 | INFO | "malloc" : " 14 GB", | 20260505 19:39:48.572 | INFO | "pinned" : " 29 GB" | 20260505 19:39:48.572 | INFO | } | 20260505 19:39:48.572 | INFO | Perf report for the last 5.13583 seconds: | 20260505 19:39:48.572 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:39:48.572 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:48.572 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:48.572 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:48.572 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 19:39:48.572 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:48.572 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260505 19:39:48.572 | INFO | PRE_HQ : 60.1%, 1.8, 55.4ms, 0 | 20260505 19:39:48.572 | INFO | BAZWRITER : 63.0%, 3.7, 120.2ms, 0 | 20260505 19:39:48.572 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 19:39:48.572 | INFO | Uploaded 36GiB to GPU in 2.17038s (16.587GiB/s) | 20260505 19:39:48.572 | INFO | Downloaded 38GiB to GPU in 2.24463s (16.9293GiB/s) | 20260505 19:39:53.708 | INFO | Analyzing chunk frames = [5788160,5788672) | 20260505 19:39:53.708 | INFO | MemInfo: { | 20260505 19:39:53.708 | INFO | "gpu" : " 17 GB", | 20260505 19:39:53.708 | INFO | "huge" : "", | 20260505 19:39:53.708 | INFO | "huge_shared" : " 27 GB", | 20260505 19:39:53.708 | INFO | "malloc" : " 14 GB", | 20260505 19:39:53.708 | INFO | "pinned" : " 29 GB" | 20260505 19:39:53.708 | INFO | } | 20260505 19:39:53.708 | INFO | Perf report for the last 5.13663 seconds: | 20260505 19:39:53.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:39:53.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:53.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:53.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:53.709 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 19:39:53.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:53.709 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:53.709 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260505 19:39:53.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:53.709 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:53.709 | INFO | Uploaded 36GiB to GPU in 2.14573s (16.7775GiB/s) | 20260505 19:39:53.709 | INFO | Downloaded 34GiB to GPU in 2.04884s (16.5948GiB/s) | 20260505 19:39:58.855 | INFO | Analyzing chunk frames = [5788672,5789184) | 20260505 19:39:58.855 | INFO | MemInfo: { | 20260505 19:39:58.855 | INFO | "gpu" : " 17 GB", | 20260505 19:39:58.855 | INFO | "huge" : "", | 20260505 19:39:58.855 | INFO | "huge_shared" : " 27 GB", | 20260505 19:39:58.855 | INFO | "malloc" : " 14 GB", | 20260505 19:39:58.855 | INFO | "pinned" : " 29 GB" | 20260505 19:39:58.855 | INFO | } | 20260505 19:39:58.855 | INFO | Perf report for the last 5.14698 seconds: | 20260505 19:39:58.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:39:58.855 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:58.856 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:58.856 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:58.856 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 19:39:58.856 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:58.856 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:58.856 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:39:58.856 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:58.856 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:39:58.856 | INFO | Uploaded 36GiB to GPU in 2.21127s (16.2802GiB/s) | 20260505 19:39:58.856 | INFO | Downloaded 34GiB to GPU in 2.06236s (16.486GiB/s) | 20260505 19:40:04.004 | INFO | Analyzing chunk frames = [5789184,5789696) | 20260505 19:40:04.004 | INFO | MemInfo: { | 20260505 19:40:04.004 | INFO | "gpu" : " 17 GB", | 20260505 19:40:04.004 | INFO | "huge" : "", | 20260505 19:40:04.004 | INFO | "huge_shared" : " 27 GB", | 20260505 19:40:04.004 | INFO | "malloc" : " 14 GB", | 20260505 19:40:04.004 | INFO | "pinned" : " 29 GB" | 20260505 19:40:04.004 | INFO | } | 20260505 19:40:04.004 | INFO | Perf report for the last 5.14865 seconds: | 20260505 19:40:04.004 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:40:04.004 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:04.004 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:04.004 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:04.004 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260505 19:40:04.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:04.004 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:04.004 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:40:04.004 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:04.004 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:04.005 | INFO | Uploaded 36GiB to GPU in 2.19362s (16.4112GiB/s) | 20260505 19:40:04.005 | INFO | Downloaded 34GiB to GPU in 2.03803s (16.6828GiB/s) | 20260505 19:40:09.134 | INFO | Analyzing chunk frames = [5789696,5790208) | 20260505 19:40:09.134 | INFO | MemInfo: { | 20260505 19:40:09.134 | INFO | "gpu" : " 17 GB", | 20260505 19:40:09.134 | INFO | "huge" : "", | 20260505 19:40:09.134 | INFO | "huge_shared" : " 27 GB", | 20260505 19:40:09.134 | INFO | "malloc" : " 14 GB", | 20260505 19:40:09.134 | INFO | "pinned" : " 29 GB" | 20260505 19:40:09.134 | INFO | } | 20260505 19:40:09.134 | INFO | Perf report for the last 5.12983 seconds: | 20260505 19:40:09.134 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:40:09.134 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:09.134 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:09.134 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:09.134 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260505 19:40:09.134 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:09.134 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:09.134 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260505 19:40:09.134 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:09.134 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:09.134 | INFO | Uploaded 36GiB to GPU in 2.12974s (16.9034GiB/s) | 20260505 19:40:09.134 | INFO | Downloaded 34GiB to GPU in 2.06315s (16.4797GiB/s) | 20260505 19:40:14.266 | INFO | Analyzing chunk frames = [5790208,5790720) | 20260505 19:40:14.266 | INFO | MemInfo: { | 20260505 19:40:14.266 | INFO | "gpu" : " 17 GB", | 20260505 19:40:14.266 | INFO | "huge" : "", | 20260505 19:40:14.266 | INFO | "huge_shared" : " 27 GB", | 20260505 19:40:14.266 | INFO | "malloc" : " 14 GB", | 20260505 19:40:14.266 | INFO | "pinned" : " 29 GB" | 20260505 19:40:14.266 | INFO | } | 20260505 19:40:14.266 | INFO | Perf report for the last 5.13186 seconds: | 20260505 19:40:14.266 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:40:14.266 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:14.266 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:14.266 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:14.266 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260505 19:40:14.266 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:14.266 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:14.266 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:40:14.266 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:14.266 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:14.266 | INFO | Uploaded 36GiB to GPU in 2.14449s (16.7872GiB/s) | 20260505 19:40:14.266 | INFO | Downloaded 34GiB to GPU in 2.05897s (16.5131GiB/s) | 20260505 19:40:19.409 | INFO | Analyzing chunk frames = [5790720,5791232) | 20260505 19:40:19.409 | INFO | MemInfo: { | 20260505 19:40:19.409 | INFO | "gpu" : " 17 GB", | 20260505 19:40:19.409 | INFO | "huge" : "", | 20260505 19:40:19.409 | INFO | "huge_shared" : " 27 GB", | 20260505 19:40:19.409 | INFO | "malloc" : " 14 GB", | 20260505 19:40:19.409 | INFO | "pinned" : " 29 GB" | 20260505 19:40:19.410 | INFO | } | 20260505 19:40:19.410 | INFO | Perf report for the last 5.14379 seconds: | 20260505 19:40:19.410 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:40:19.410 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:19.410 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:19.410 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:19.410 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 19:40:19.410 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:19.410 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:19.410 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 19:40:19.410 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:19.410 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:19.410 | INFO | Uploaded 36GiB to GPU in 2.1278s (16.9189GiB/s) | 20260505 19:40:19.410 | INFO | Downloaded 34GiB to GPU in 2.15138s (15.8038GiB/s) | 20260505 19:40:24.551 | INFO | Analyzing chunk frames = [5791232,5791744) | 20260505 19:40:24.551 | INFO | MemInfo: { | 20260505 19:40:24.551 | INFO | "gpu" : " 17 GB", | 20260505 19:40:24.551 | INFO | "huge" : "", | 20260505 19:40:24.551 | INFO | "huge_shared" : " 27 GB", | 20260505 19:40:24.551 | INFO | "malloc" : " 14 GB", | 20260505 19:40:24.551 | INFO | "pinned" : " 29 GB" | 20260505 19:40:24.551 | INFO | } | 20260505 19:40:24.551 | INFO | Perf report for the last 5.14174 seconds: | 20260505 19:40:24.551 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:40:24.551 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:24.551 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:24.551 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:24.551 | INFO | ANALYSIS : 62.5%, 3.0, 97.7ms, 0 | 20260505 19:40:24.552 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:24.552 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:24.552 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:40:24.552 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:24.552 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:24.552 | INFO | Uploaded 36GiB to GPU in 2.14313s (16.7978GiB/s) | 20260505 19:40:24.552 | INFO | Downloaded 34GiB to GPU in 2.02358s (16.8019GiB/s) | 20260505 19:40:24.552 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 19:40:24.552 | INFO | Total mem usage: 15.3 GiB | 20260505 19:40:24.552 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 19:40:24.552 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.75%) | 20260505 19:40:24.552 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 19:40:24.552 | INFO | High Water Breakdown: | 20260505 19:40:24.552 | INFO | 61% : 8.59 GiB ( 90.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 19:40:24.552 | INFO | 38.9% : 5.48 GiB ( 82.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 19:40:24.552 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 19:40:24.552 | INFO | ------------------------------------------ | 20260505 19:40:24.552 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:40:24.552 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 19:40:24.552 | INFO | Total mem usage: 26.5 GiB | 20260505 19:40:24.552 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 19:40:24.552 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 19:40:24.552 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 19:40:24.552 | INFO | High Water Breakdown: | 20260505 19:40:24.552 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:40:24.552 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:40:24.552 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:40:24.552 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:40:24.552 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:40:24.552 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:40:24.552 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 19:40:24.552 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 19:40:24.552 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:40:24.552 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 19:40:24.552 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 19:40:24.552 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 19:40:24.552 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 19:40:24.552 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 19:40:24.552 | INFO | ------------------------------------------ | 20260505 19:40:24.552 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:40:24.552 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 19:40:24.552 | INFO | Total mem usage: 25.6 GiB | 20260505 19:40:24.552 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 19:40:24.552 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 19:40:24.552 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 19:40:24.552 | INFO | High Water Breakdown: | 20260505 19:40:24.553 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 19:40:24.553 | INFO | ------------------------------------------ | 20260505 19:40:24.553 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:40:24.553 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 19:40:24.553 | INFO | Total mem usage: 15.7 GiB | 20260505 19:40:24.553 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 19:40:24.553 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 19:40:24.553 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 19:40:24.553 | INFO | High Water Breakdown: | 20260505 19:40:24.553 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 19:40:24.553 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 19:40:24.553 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 19:40:24.553 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 19:40:24.553 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 19:40:24.553 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 19:40:24.553 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 19:40:24.553 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 19:40:24.553 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 19:40:24.553 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:40:24.553 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 19:40:24.553 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:40:24.553 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 19:40:24.553 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 19:40:24.553 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 19:40:24.553 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 19:40:24.553 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:40:24.553 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 19:40:24.553 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:40:24.553 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:40:24.553 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:40:24.553 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 19:40:24.553 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 19:40:24.553 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:40:24.553 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 19:40:24.553 | INFO | ------------------------------------------ | 20260505 19:40:24.553 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:40:24.553 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 19:40:24.553 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:40:24.553 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:40:24.553 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.44e+01, 1.35e+01 / 1.92e+00 | 20260505 19:40:24.553 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.03e-01, 1.05e-02 / 4.31e-03 | 20260505 19:40:24.553 | INFO | PerfReport: 11.78%, Baseline -- 3.71e+00 / 4.52e+00, 5.65e-01 / 1.60e-01 | 20260505 19:40:24.553 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.54e+00, 1.92e-01 / 8.74e-02 | 20260505 19:40:24.553 | INFO | PerfReport: 15.94%, DME -- 4.01e+01 / 4.43e+01, 3.99e+00 / 1.92e+00 | 20260505 19:40:24.553 | INFO | PerfReport: 57.00%, FrameLabeling -- 1.79e+01 / 1.93e+01, 1.46e+00 / 1.10e+00 | 20260505 19:40:24.553 | INFO | PerfReport: 7.27%, PulseAccumulating -- 2.29e+00 / 2.87e+00, 4.39e-01 / 1.99e-01 | 20260505 19:40:24.553 | INFO | PerfReport: 3.29%, Metrics -- 1.03e+00 / 1.38e+00, 2.83e-01 / 1.57e-01 | 20260505 19:40:24.553 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.60e-01 / 4.30e-01, 6.36e-02 / 4.10e-02 | 20260505 19:40:24.553 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 19:40:24.553 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:40:24.553 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:40:24.553 | INFO | PerfReport: Full Execution -- 9.95e+01 / -nan, 2.55e+01 / 0.00e+00 | 20260505 19:40:24.553 | INFO | PerfReport: 22.48%, Upload -- 2.24e+01 / 0.00e+00, 7.81e+00 / 0.00e+00 | 20260505 19:40:24.553 | INFO | PerfReport: 21.76%, Download -- 2.17e+01 / 0.00e+00, 7.72e+00 / 0.00e+00 | 20260505 19:40:24.554 | INFO | PerfReport: 31.62%, Compute -- 3.15e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 19:40:24.554 | INFO | PerfReport: 2.34%, LockUpload -- 2.32e+00 / 0.00e+00, 8.07e+00 / 0.00e+00 | 20260505 19:40:24.554 | INFO | PerfReport: 1.39%, LockDownload -- 1.39e+00 / 0.00e+00, 2.95e+00 / 0.00e+00 | 20260505 19:40:24.554 | INFO | PerfReport: 20.40%, LockCompute -- 2.03e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 19:40:24.554 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.65e-02 / 0.00e+00 | 20260505 19:40:24.685 | INFO | Starting a round of Baz IO | 20260505 19:40:27.853 | INFO | Populating RTMetrics report | 20260505 19:40:27.854 | INFO | RTMetrics report populated | 20260505 19:40:27.854 | INFO | Opening tmp RTMetrics json file | 20260505 19:40:27.857 | INFO | Serializing RTMetrics to json | 20260505 19:40:27.859 | INFO | Serialization done, swapping tmp file now... | 20260505 19:40:27.861 | INFO | RTMetrics done | 20260505 19:40:27.941 | INFO | Finished a round of Baz IO: Wrote 2.042 GiB in 3.255 seconds | 20260505 19:40:27.941 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.75% / 0.00% / 78.25% | 20260505 19:40:27.941 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.200379% / 3.97364e-06% / 7.54992e-05% | 20260505 19:40:27.941 | INFO | Baz IO overall: | 20260505 19:40:27.941 | INFO | BytesWritten : 3389084291359 | 20260505 19:40:27.941 | INFO | NumEvents : 1603444789421 | 20260505 19:40:29.690 | INFO | Analyzing chunk frames = [5791744,5792256) | 20260505 19:40:29.690 | INFO | MemInfo: { | 20260505 19:40:29.690 | INFO | "gpu" : " 17 GB", | 20260505 19:40:29.690 | INFO | "huge" : "", | 20260505 19:40:29.690 | INFO | "huge_shared" : " 27 GB", | 20260505 19:40:29.690 | INFO | "malloc" : " 14 GB", | 20260505 19:40:29.690 | INFO | "pinned" : " 29 GB" | 20260505 19:40:29.690 | INFO | } | 20260505 19:40:29.690 | INFO | Perf report for the last 5.13819 seconds: | 20260505 19:40:29.690 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:40:29.690 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:29.690 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:29.690 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:29.690 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 19:40:29.690 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:29.690 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260505 19:40:29.690 | INFO | PRE_HQ : 60.1%, 1.8, 55.1ms, 0 | 20260505 19:40:29.690 | INFO | BAZWRITER : 62.8%, 3.7, 119.2ms, 0 | 20260505 19:40:29.690 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 19:40:29.690 | INFO | Uploaded 36GiB to GPU in 2.17219s (16.5732GiB/s) | 20260505 19:40:29.690 | INFO | Downloaded 38GiB to GPU in 2.244s (16.9341GiB/s) | 20260505 19:40:34.833 | INFO | Analyzing chunk frames = [5792256,5792768) | 20260505 19:40:34.833 | INFO | MemInfo: { | 20260505 19:40:34.833 | INFO | "gpu" : " 17 GB", | 20260505 19:40:34.833 | INFO | "huge" : "", | 20260505 19:40:34.833 | INFO | "huge_shared" : " 27 GB", | 20260505 19:40:34.833 | INFO | "malloc" : " 14 GB", | 20260505 19:40:34.833 | INFO | "pinned" : " 29 GB" | 20260505 19:40:34.833 | INFO | } | 20260505 19:40:34.833 | INFO | Perf report for the last 5.14374 seconds: | 20260505 19:40:34.833 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:40:34.833 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:34.833 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:34.833 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:34.833 | INFO | ANALYSIS : 62.5%, 3.0, 97.7ms, 0 | 20260505 19:40:34.833 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:34.833 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:34.833 | INFO | PRE_HQ : 27.0%, 1.0, 14.1ms, 0 | 20260505 19:40:34.833 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:34.834 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:34.834 | INFO | Uploaded 36GiB to GPU in 2.13549s (16.8579GiB/s) | 20260505 19:40:34.834 | INFO | Downloaded 34GiB to GPU in 2.03466s (16.7104GiB/s) | 20260505 19:40:39.964 | INFO | Analyzing chunk frames = [5792768,5793280) | 20260505 19:40:39.964 | INFO | MemInfo: { | 20260505 19:40:39.964 | INFO | "gpu" : " 17 GB", | 20260505 19:40:39.964 | INFO | "huge" : "", | 20260505 19:40:39.964 | INFO | "huge_shared" : " 27 GB", | 20260505 19:40:39.964 | INFO | "malloc" : " 14 GB", | 20260505 19:40:39.964 | INFO | "pinned" : " 29 GB" | 20260505 19:40:39.964 | INFO | } | 20260505 19:40:39.964 | INFO | Perf report for the last 5.1303 seconds: | 20260505 19:40:39.964 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:40:39.964 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:39.964 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:39.964 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:39.964 | INFO | ANALYSIS : 62.8%, 3.0, 97.6ms, 0 | 20260505 19:40:39.964 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:39.964 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:39.964 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:40:39.964 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:39.964 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:39.964 | INFO | Uploaded 36GiB to GPU in 2.20949s (16.2933GiB/s) | 20260505 19:40:39.964 | INFO | Downloaded 34GiB to GPU in 2.0736s (16.3966GiB/s) | 20260505 19:40:45.109 | INFO | Analyzing chunk frames = [5793280,5793792) | 20260505 19:40:45.109 | INFO | MemInfo: { | 20260505 19:40:45.109 | INFO | "gpu" : " 17 GB", | 20260505 19:40:45.109 | INFO | "huge" : "", | 20260505 19:40:45.109 | INFO | "huge_shared" : " 27 GB", | 20260505 19:40:45.109 | INFO | "malloc" : " 14 GB", | 20260505 19:40:45.109 | INFO | "pinned" : " 29 GB" | 20260505 19:40:45.109 | INFO | } | 20260505 19:40:45.109 | INFO | Perf report for the last 5.14554 seconds: | 20260505 19:40:45.109 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:40:45.109 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:45.109 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:45.109 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:45.109 | INFO | ANALYSIS : 62.2%, 3.0, 96.6ms, 0 | 20260505 19:40:45.109 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:45.109 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:45.109 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:40:45.109 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:45.109 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:45.109 | INFO | Uploaded 36GiB to GPU in 2.17532s (16.5493GiB/s) | 20260505 19:40:45.110 | INFO | Downloaded 34GiB to GPU in 2.07263s (16.4043GiB/s) | 20260505 19:40:50.251 | INFO | Analyzing chunk frames = [5793792,5794304) | 20260505 19:40:50.251 | INFO | MemInfo: { | 20260505 19:40:50.251 | INFO | "gpu" : " 17 GB", | 20260505 19:40:50.251 | INFO | "huge" : "", | 20260505 19:40:50.251 | INFO | "huge_shared" : " 27 GB", | 20260505 19:40:50.251 | INFO | "malloc" : " 14 GB", | 20260505 19:40:50.251 | INFO | "pinned" : " 29 GB" | 20260505 19:40:50.251 | INFO | } | 20260505 19:40:50.251 | INFO | Perf report for the last 5.14151 seconds: | 20260505 19:40:50.251 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:40:50.251 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:50.251 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:50.251 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:50.251 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260505 19:40:50.251 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:50.251 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:50.251 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:40:50.251 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:50.251 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:50.251 | INFO | Uploaded 36GiB to GPU in 2.11955s (16.9847GiB/s) | 20260505 19:40:50.251 | INFO | Downloaded 34GiB to GPU in 2.06042s (16.5015GiB/s) | 20260505 19:40:55.384 | INFO | Analyzing chunk frames = [5794304,5794816) | 20260505 19:40:55.384 | INFO | MemInfo: { | 20260505 19:40:55.384 | INFO | "gpu" : " 17 GB", | 20260505 19:40:55.384 | INFO | "huge" : "", | 20260505 19:40:55.384 | INFO | "huge_shared" : " 27 GB", | 20260505 19:40:55.384 | INFO | "malloc" : " 14 GB", | 20260505 19:40:55.384 | INFO | "pinned" : " 29 GB" | 20260505 19:40:55.384 | INFO | } | 20260505 19:40:55.384 | INFO | Perf report for the last 5.13333 seconds: | 20260505 19:40:55.384 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:40:55.384 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:55.384 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:55.384 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:55.384 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 19:40:55.384 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:55.384 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:55.384 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:40:55.384 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:55.384 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:40:55.384 | INFO | Uploaded 36GiB to GPU in 2.09423s (17.1901GiB/s) | 20260505 19:40:55.384 | INFO | Downloaded 34GiB to GPU in 2.03859s (16.6782GiB/s) | 20260505 19:41:00.516 | INFO | Analyzing chunk frames = [5794816,5795328) | 20260505 19:41:00.516 | INFO | MemInfo: { | 20260505 19:41:00.516 | INFO | "gpu" : " 17 GB", | 20260505 19:41:00.516 | INFO | "huge" : "", | 20260505 19:41:00.516 | INFO | "huge_shared" : " 27 GB", | 20260505 19:41:00.516 | INFO | "malloc" : " 14 GB", | 20260505 19:41:00.516 | INFO | "pinned" : " 29 GB" | 20260505 19:41:00.516 | INFO | } | 20260505 19:41:00.516 | INFO | Perf report for the last 5.13232 seconds: | 20260505 19:41:00.517 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:41:00.517 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:00.517 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:00.517 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:00.517 | INFO | ANALYSIS : 63.4%, 3.0, 98.8ms, 0 | 20260505 19:41:00.517 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:00.517 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:00.517 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 19:41:00.517 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:00.517 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:00.517 | INFO | Uploaded 36GiB to GPU in 2.14164s (16.8096GiB/s) | 20260505 19:41:00.517 | INFO | Downloaded 34GiB to GPU in 2.17234s (15.6513GiB/s) | 20260505 19:41:05.666 | INFO | Analyzing chunk frames = [5795328,5795840) | 20260505 19:41:05.666 | INFO | MemInfo: { | 20260505 19:41:05.666 | INFO | "gpu" : " 17 GB", | 20260505 19:41:05.666 | INFO | "huge" : "", | 20260505 19:41:05.666 | INFO | "huge_shared" : " 27 GB", | 20260505 19:41:05.666 | INFO | "malloc" : " 14 GB", | 20260505 19:41:05.666 | INFO | "pinned" : " 29 GB" | 20260505 19:41:05.666 | INFO | } | 20260505 19:41:05.666 | INFO | Perf report for the last 5.14932 seconds: | 20260505 19:41:05.666 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:41:05.666 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:05.666 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:05.666 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:05.666 | INFO | ANALYSIS : 62.7%, 3.0, 98.1ms, 0 | 20260505 19:41:05.666 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:05.666 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:05.666 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:41:05.666 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:05.666 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:05.666 | INFO | Uploaded 36GiB to GPU in 2.19489s (16.4017GiB/s) | 20260505 19:41:05.666 | INFO | Downloaded 34GiB to GPU in 2.03596s (16.6997GiB/s) | 20260505 19:41:05.798 | INFO | Starting a round of Baz IO | 20260505 19:41:08.964 | INFO | Populating RTMetrics report | 20260505 19:41:08.964 | INFO | RTMetrics report populated | 20260505 19:41:08.964 | INFO | Opening tmp RTMetrics json file | 20260505 19:41:08.966 | INFO | Serializing RTMetrics to json | 20260505 19:41:08.967 | INFO | Serialization done, swapping tmp file now... | 20260505 19:41:08.967 | INFO | RTMetrics done | 20260505 19:41:09.051 | INFO | Finished a round of Baz IO: Wrote 2.04 GiB in 3.253 seconds | 20260505 19:41:09.052 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.75% / 0.00% / 78.25% | 20260505 19:41:09.052 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.200025% / 0% / 7.94729e-05% | 20260505 19:41:09.052 | INFO | Baz IO overall: | 20260505 19:41:09.052 | INFO | BytesWritten : 3391275169927 | 20260505 19:41:09.052 | INFO | NumEvents : 1604462085450 | 20260505 19:41:10.809 | INFO | Analyzing chunk frames = [5795840,5796352) | 20260505 19:41:10.809 | INFO | MemInfo: { | 20260505 19:41:10.809 | INFO | "gpu" : " 17 GB", | 20260505 19:41:10.809 | INFO | "huge" : "", | 20260505 19:41:10.809 | INFO | "huge_shared" : " 27 GB", | 20260505 19:41:10.809 | INFO | "malloc" : " 14 GB", | 20260505 19:41:10.809 | INFO | "pinned" : " 29 GB" | 20260505 19:41:10.809 | INFO | } | 20260505 19:41:10.809 | INFO | Perf report for the last 5.14338 seconds: | 20260505 19:41:10.809 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:41:10.809 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:10.809 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:10.809 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:10.809 | INFO | ANALYSIS : 63.1%, 3.0, 98.2ms, 0 | 20260505 19:41:10.809 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:10.809 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260505 19:41:10.809 | INFO | PRE_HQ : 60.2%, 1.8, 55.4ms, 0 | 20260505 19:41:10.809 | INFO | BAZWRITER : 62.8%, 3.8, 122.2ms, 0 | 20260505 19:41:10.809 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 19:41:10.810 | INFO | Uploaded 36GiB to GPU in 2.16943s (16.5943GiB/s) | 20260505 19:41:10.810 | INFO | Downloaded 38GiB to GPU in 2.25726s (16.8346GiB/s) | 20260505 19:41:15.946 | INFO | Analyzing chunk frames = [5796352,5796864) | 20260505 19:41:15.946 | INFO | MemInfo: { | 20260505 19:41:15.946 | INFO | "gpu" : " 17 GB", | 20260505 19:41:15.946 | INFO | "huge" : "", | 20260505 19:41:15.946 | INFO | "huge_shared" : " 27 GB", | 20260505 19:41:15.946 | INFO | "malloc" : " 14 GB", | 20260505 19:41:15.946 | INFO | "pinned" : " 29 GB" | 20260505 19:41:15.946 | INFO | } | 20260505 19:41:15.946 | INFO | Perf report for the last 5.13733 seconds: | 20260505 19:41:15.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:41:15.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:15.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:15.947 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:15.947 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 19:41:15.947 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:15.947 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:15.947 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:41:15.947 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:15.947 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:15.947 | INFO | Uploaded 36GiB to GPU in 2.11016s (17.0603GiB/s) | 20260505 19:41:15.947 | INFO | Downloaded 34GiB to GPU in 2.03372s (16.7181GiB/s) | 20260505 19:41:21.087 | INFO | Analyzing chunk frames = [5796864,5797376) | 20260505 19:41:21.087 | INFO | MemInfo: { | 20260505 19:41:21.087 | INFO | "gpu" : " 17 GB", | 20260505 19:41:21.087 | INFO | "huge" : "", | 20260505 19:41:21.087 | INFO | "huge_shared" : " 27 GB", | 20260505 19:41:21.087 | INFO | "malloc" : " 14 GB", | 20260505 19:41:21.087 | INFO | "pinned" : " 29 GB" | 20260505 19:41:21.087 | INFO | } | 20260505 19:41:21.087 | INFO | Perf report for the last 5.14077 seconds: | 20260505 19:41:21.087 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:41:21.087 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:21.087 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:21.087 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:21.087 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260505 19:41:21.088 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:21.088 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:21.088 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:41:21.088 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:21.088 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:21.088 | INFO | Uploaded 36GiB to GPU in 2.14277s (16.8007GiB/s) | 20260505 19:41:21.088 | INFO | Downloaded 34GiB to GPU in 2.03829s (16.6807GiB/s) | 20260505 19:41:26.219 | INFO | Analyzing chunk frames = [5797376,5797888) | 20260505 19:41:26.219 | INFO | MemInfo: { | 20260505 19:41:26.219 | INFO | "gpu" : " 17 GB", | 20260505 19:41:26.219 | INFO | "huge" : "", | 20260505 19:41:26.219 | INFO | "huge_shared" : " 27 GB", | 20260505 19:41:26.219 | INFO | "malloc" : " 14 GB", | 20260505 19:41:26.219 | INFO | "pinned" : " 29 GB" | 20260505 19:41:26.219 | INFO | } | 20260505 19:41:26.219 | INFO | Perf report for the last 5.13202 seconds: | 20260505 19:41:26.219 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:41:26.219 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:26.219 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:26.220 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:26.220 | INFO | ANALYSIS : 62.4%, 3.0, 96.8ms, 0 | 20260505 19:41:26.220 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:26.220 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:26.220 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:41:26.220 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:26.220 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:26.220 | INFO | Uploaded 36GiB to GPU in 2.19632s (16.3911GiB/s) | 20260505 19:41:26.220 | INFO | Downloaded 34GiB to GPU in 2.05974s (16.507GiB/s) | 20260505 19:41:31.363 | INFO | Analyzing chunk frames = [5797888,5798400) | 20260505 19:41:31.363 | INFO | MemInfo: { | 20260505 19:41:31.363 | INFO | "gpu" : " 17 GB", | 20260505 19:41:31.363 | INFO | "huge" : "", | 20260505 19:41:31.363 | INFO | "huge_shared" : " 27 GB", | 20260505 19:41:31.363 | INFO | "malloc" : " 14 GB", | 20260505 19:41:31.363 | INFO | "pinned" : " 29 GB" | 20260505 19:41:31.363 | INFO | } | 20260505 19:41:31.363 | INFO | Perf report for the last 5.14386 seconds: | 20260505 19:41:31.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:41:31.363 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:31.363 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:31.363 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:31.363 | INFO | ANALYSIS : 61.8%, 3.0, 96.6ms, 0 | 20260505 19:41:31.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:31.363 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:31.363 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 19:41:31.363 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:31.363 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:31.363 | INFO | Uploaded 36GiB to GPU in 2.13193s (16.8861GiB/s) | 20260505 19:41:31.363 | INFO | Downloaded 34GiB to GPU in 2.06484s (16.4662GiB/s) | 20260505 19:41:36.504 | INFO | Analyzing chunk frames = [5798400,5798912) | 20260505 19:41:36.504 | INFO | MemInfo: { | 20260505 19:41:36.504 | INFO | "gpu" : " 17 GB", | 20260505 19:41:36.504 | INFO | "huge" : "", | 20260505 19:41:36.504 | INFO | "huge_shared" : " 27 GB", | 20260505 19:41:36.504 | INFO | "malloc" : " 14 GB", | 20260505 19:41:36.504 | INFO | "pinned" : " 29 GB" | 20260505 19:41:36.504 | INFO | } | 20260505 19:41:36.504 | INFO | Perf report for the last 5.141 seconds: | 20260505 19:41:36.504 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:41:36.504 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:36.504 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:36.504 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:36.504 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260505 19:41:36.504 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:36.504 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:36.504 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:41:36.504 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:36.504 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:36.504 | INFO | Uploaded 36GiB to GPU in 2.14147s (16.8109GiB/s) | 20260505 19:41:36.505 | INFO | Downloaded 34GiB to GPU in 2.05612s (16.536GiB/s) | 20260505 19:41:41.640 | INFO | Analyzing chunk frames = [5798912,5799424) | 20260505 19:41:41.640 | INFO | MemInfo: { | 20260505 19:41:41.640 | INFO | "gpu" : " 17 GB", | 20260505 19:41:41.640 | INFO | "huge" : "", | 20260505 19:41:41.640 | INFO | "huge_shared" : " 27 GB", | 20260505 19:41:41.640 | INFO | "malloc" : " 14 GB", | 20260505 19:41:41.640 | INFO | "pinned" : " 29 GB" | 20260505 19:41:41.640 | INFO | } | 20260505 19:41:41.641 | INFO | Perf report for the last 5.13652 seconds: | 20260505 19:41:41.641 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:41:41.641 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:41.641 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:41.641 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:41.641 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 19:41:41.641 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:41.641 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:41.641 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 19:41:41.641 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:41.641 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:41.641 | INFO | Uploaded 36GiB to GPU in 2.11418s (17.0279GiB/s) | 20260505 19:41:41.641 | INFO | Downloaded 34GiB to GPU in 2.1425s (15.8693GiB/s) | 20260505 19:41:46.774 | INFO | Analyzing chunk frames = [5799424,5799936) | 20260505 19:41:46.774 | INFO | MemInfo: { | 20260505 19:41:46.774 | INFO | "gpu" : " 17 GB", | 20260505 19:41:46.774 | INFO | "huge" : "", | 20260505 19:41:46.774 | INFO | "huge_shared" : " 27 GB", | 20260505 19:41:46.774 | INFO | "malloc" : " 14 GB", | 20260505 19:41:46.774 | INFO | "pinned" : " 29 GB" | 20260505 19:41:46.774 | INFO | } | 20260505 19:41:46.774 | INFO | Perf report for the last 5.13373 seconds: | 20260505 19:41:46.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:41:46.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:46.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:46.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:46.774 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 19:41:46.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:46.774 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:46.775 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 19:41:46.775 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:46.775 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:46.775 | INFO | Uploaded 36GiB to GPU in 2.20582s (16.3205GiB/s) | 20260505 19:41:46.775 | INFO | Downloaded 34GiB to GPU in 2.04828s (16.5993GiB/s) | 20260505 19:41:46.775 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 19:41:46.775 | INFO | Total mem usage: 15.3 GiB | 20260505 19:41:46.775 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 19:41:46.775 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.06%) | 20260505 19:41:46.775 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 19:41:46.775 | INFO | High Water Breakdown: | 20260505 19:41:46.775 | INFO | 61% : 8.66 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 19:41:46.775 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 19:41:46.775 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 19:41:46.775 | INFO | ------------------------------------------ | 20260505 19:41:46.775 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:41:46.775 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 19:41:46.775 | INFO | Total mem usage: 26.5 GiB | 20260505 19:41:46.775 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 19:41:46.775 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 19:41:46.775 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 19:41:46.775 | INFO | High Water Breakdown: | 20260505 19:41:46.775 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:41:46.775 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:41:46.775 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:41:46.775 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:41:46.775 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:41:46.775 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:41:46.775 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 19:41:46.775 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 19:41:46.775 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:41:46.775 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 19:41:46.775 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 19:41:46.775 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 19:41:46.775 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 19:41:46.775 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 19:41:46.775 | INFO | ------------------------------------------ | 20260505 19:41:46.775 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:41:46.775 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 19:41:46.775 | INFO | Total mem usage: 25.6 GiB | 20260505 19:41:46.775 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 19:41:46.775 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 19:41:46.775 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 19:41:46.775 | INFO | High Water Breakdown: | 20260505 19:41:46.775 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 19:41:46.775 | INFO | ------------------------------------------ | 20260505 19:41:46.775 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:41:46.776 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 19:41:46.776 | INFO | Total mem usage: 15.7 GiB | 20260505 19:41:46.776 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 19:41:46.776 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 19:41:46.776 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 19:41:46.776 | INFO | High Water Breakdown: | 20260505 19:41:46.776 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 19:41:46.776 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 19:41:46.776 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 19:41:46.776 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 19:41:46.776 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 19:41:46.776 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 19:41:46.776 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 19:41:46.776 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 19:41:46.776 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 19:41:46.776 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:41:46.776 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 19:41:46.776 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:41:46.776 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 19:41:46.776 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 19:41:46.776 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 19:41:46.776 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 19:41:46.776 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:41:46.776 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 19:41:46.776 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:41:46.776 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:41:46.776 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:41:46.776 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 19:41:46.776 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 19:41:46.776 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:41:46.776 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 19:41:46.776 | INFO | ------------------------------------------ | 20260505 19:41:46.776 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:41:46.776 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 19:41:46.776 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:41:46.776 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:41:46.776 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.47e+01, 1.35e+01 / 1.99e+00 | 20260505 19:41:46.776 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.04e-01, 1.03e-02 / 4.78e-03 | 20260505 19:41:46.776 | INFO | PerfReport: 11.77%, Baseline -- 3.71e+00 / 4.59e+00, 5.64e-01 / 1.41e-01 | 20260505 19:41:46.776 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.51e+00, 1.90e-01 / 9.92e-02 | 20260505 19:41:46.776 | INFO | PerfReport: 15.94%, DME -- 4.02e+01 / 4.44e+01, 4.14e+00 / 2.05e+00 | 20260505 19:41:46.776 | INFO | PerfReport: 57.07%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.51e+00 / 1.13e+00 | 20260505 19:41:46.776 | INFO | PerfReport: 7.24%, PulseAccumulating -- 2.28e+00 / 2.83e+00, 4.38e-01 / 2.30e-01 | 20260505 19:41:46.776 | INFO | PerfReport: 3.27%, Metrics -- 1.03e+00 / 1.36e+00, 2.83e-01 / 1.46e-01 | 20260505 19:41:46.776 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.59e-01 / 4.19e-01, 6.28e-02 / 3.59e-02 | 20260505 19:41:46.776 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 19:41:46.776 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:41:46.776 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:41:46.776 | INFO | PerfReport: Full Execution -- 9.95e+01 / -nan, 2.56e+01 / 0.00e+00 | 20260505 19:41:46.776 | INFO | PerfReport: 22.54%, Upload -- 2.24e+01 / 0.00e+00, 7.77e+00 / 0.00e+00 | 20260505 19:41:46.776 | INFO | PerfReport: 21.87%, Download -- 2.18e+01 / 0.00e+00, 7.79e+00 / 0.00e+00 | 20260505 19:41:46.776 | INFO | PerfReport: 31.66%, Compute -- 3.15e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 19:41:46.776 | INFO | PerfReport: 2.26%, LockUpload -- 2.24e+00 / 0.00e+00, 7.83e+00 / 0.00e+00 | 20260505 19:41:46.776 | INFO | PerfReport: 1.37%, LockDownload -- 1.36e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260505 19:41:46.776 | INFO | PerfReport: 20.29%, LockCompute -- 2.02e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 19:41:46.776 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.00e-02 / 0.00e+00 | 20260505 19:41:46.908 | INFO | Starting a round of Baz IO | 20260505 19:41:50.077 | INFO | Populating RTMetrics report | 20260505 19:41:50.077 | INFO | RTMetrics report populated | 20260505 19:41:50.077 | INFO | Opening tmp RTMetrics json file | 20260505 19:41:50.078 | INFO | Serializing RTMetrics to json | 20260505 19:41:50.079 | INFO | Serialization done, swapping tmp file now... | 20260505 19:41:50.080 | INFO | RTMetrics done | 20260505 19:41:50.169 | INFO | Finished a round of Baz IO: Wrote 2.041 GiB in 3.26 seconds | 20260505 19:41:50.169 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.75% / 0.00% / 78.25% | 20260505 19:41:50.169 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.200335% / 3.97364e-06% / 7.15256e-05% | 20260505 19:41:50.169 | INFO | Baz IO overall: | 20260505 19:41:50.169 | INFO | BytesWritten : 3393466313615 | 20260505 19:41:50.169 | INFO | NumEvents : 1605479645322 | 20260505 19:41:51.916 | INFO | Analyzing chunk frames = [5799936,5800448) | 20260505 19:41:51.916 | INFO | MemInfo: { | 20260505 19:41:51.916 | INFO | "gpu" : " 17 GB", | 20260505 19:41:51.916 | INFO | "huge" : "", | 20260505 19:41:51.916 | INFO | "huge_shared" : " 27 GB", | 20260505 19:41:51.916 | INFO | "malloc" : " 14 GB", | 20260505 19:41:51.916 | INFO | "pinned" : " 29 GB" | 20260505 19:41:51.916 | INFO | } | 20260505 19:41:51.916 | INFO | Perf report for the last 5.14185 seconds: | 20260505 19:41:51.916 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:41:51.916 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:51.916 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:51.917 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:51.917 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 19:41:51.917 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:51.917 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260505 19:41:51.917 | INFO | PRE_HQ : 60.0%, 1.8, 55.4ms, 0 | 20260505 19:41:51.917 | INFO | BAZWRITER : 62.8%, 3.7, 120.3ms, 0 | 20260505 19:41:51.917 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 19:41:51.917 | INFO | Uploaded 36GiB to GPU in 2.15577s (16.6994GiB/s) | 20260505 19:41:51.917 | INFO | Downloaded 38GiB to GPU in 2.24116s (16.9555GiB/s) | 20260505 19:41:57.061 | INFO | Analyzing chunk frames = [5800448,5800960) | 20260505 19:41:57.061 | INFO | MemInfo: { | 20260505 19:41:57.061 | INFO | "gpu" : " 17 GB", | 20260505 19:41:57.061 | INFO | "huge" : "", | 20260505 19:41:57.061 | INFO | "huge_shared" : " 27 GB", | 20260505 19:41:57.061 | INFO | "malloc" : " 14 GB", | 20260505 19:41:57.061 | INFO | "pinned" : " 29 GB" | 20260505 19:41:57.061 | INFO | } | 20260505 19:41:57.061 | INFO | Perf report for the last 5.14518 seconds: | 20260505 19:41:57.062 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:41:57.062 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:57.062 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:57.062 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:57.062 | INFO | ANALYSIS : 62.7%, 3.0, 98.0ms, 0 | 20260505 19:41:57.062 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:57.062 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:57.062 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 19:41:57.062 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:57.062 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:41:57.062 | INFO | Uploaded 36GiB to GPU in 2.15463s (16.7082GiB/s) | 20260505 19:41:57.062 | INFO | Downloaded 34GiB to GPU in 2.04939s (16.5903GiB/s) | 20260505 19:42:02.199 | INFO | Analyzing chunk frames = [5800960,5801472) | 20260505 19:42:02.199 | INFO | MemInfo: { | 20260505 19:42:02.199 | INFO | "gpu" : " 17 GB", | 20260505 19:42:02.199 | INFO | "huge" : "", | 20260505 19:42:02.199 | INFO | "huge_shared" : " 27 GB", | 20260505 19:42:02.199 | INFO | "malloc" : " 14 GB", | 20260505 19:42:02.199 | INFO | "pinned" : " 29 GB" | 20260505 19:42:02.199 | INFO | } | 20260505 19:42:02.199 | INFO | Perf report for the last 5.13763 seconds: | 20260505 19:42:02.199 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:42:02.199 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:02.199 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:02.199 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:02.199 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260505 19:42:02.199 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:02.199 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:02.199 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:42:02.199 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:02.199 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:02.199 | INFO | Uploaded 36GiB to GPU in 2.12605s (16.9328GiB/s) | 20260505 19:42:02.199 | INFO | Downloaded 34GiB to GPU in 2.01303s (16.89GiB/s) | 20260505 19:42:07.328 | INFO | Analyzing chunk frames = [5801472,5801984) | 20260505 19:42:07.328 | INFO | MemInfo: { | 20260505 19:42:07.328 | INFO | "gpu" : " 17 GB", | 20260505 19:42:07.328 | INFO | "huge" : "", | 20260505 19:42:07.328 | INFO | "huge_shared" : " 27 GB", | 20260505 19:42:07.328 | INFO | "malloc" : " 14 GB", | 20260505 19:42:07.328 | INFO | "pinned" : " 29 GB" | 20260505 19:42:07.328 | INFO | } | 20260505 19:42:07.328 | INFO | Perf report for the last 5.12875 seconds: | 20260505 19:42:07.328 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:42:07.328 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:07.328 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:07.328 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:07.328 | INFO | ANALYSIS : 62.5%, 3.0, 96.9ms, 0 | 20260505 19:42:07.328 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:07.328 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:07.328 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 19:42:07.328 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:07.328 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:07.328 | INFO | Uploaded 36GiB to GPU in 2.19873s (16.3731GiB/s) | 20260505 19:42:07.328 | INFO | Downloaded 34GiB to GPU in 2.04378s (16.6358GiB/s) | 20260505 19:42:12.471 | INFO | Analyzing chunk frames = [5801984,5802496) | 20260505 19:42:12.471 | INFO | MemInfo: { | 20260505 19:42:12.471 | INFO | "gpu" : " 17 GB", | 20260505 19:42:12.471 | INFO | "huge" : "", | 20260505 19:42:12.471 | INFO | "huge_shared" : " 27 GB", | 20260505 19:42:12.471 | INFO | "malloc" : " 14 GB", | 20260505 19:42:12.471 | INFO | "pinned" : " 29 GB" | 20260505 19:42:12.471 | INFO | } | 20260505 19:42:12.471 | INFO | Perf report for the last 5.14301 seconds: | 20260505 19:42:12.471 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:42:12.471 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:12.471 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:12.471 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:12.471 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260505 19:42:12.471 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:12.471 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:12.471 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:42:12.471 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:12.471 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:12.471 | INFO | Uploaded 36GiB to GPU in 2.14345s (16.7953GiB/s) | 20260505 19:42:12.471 | INFO | Downloaded 34GiB to GPU in 2.0643s (16.4704GiB/s) | 20260505 19:42:17.617 | INFO | Analyzing chunk frames = [5802496,5803008) | 20260505 19:42:17.617 | INFO | MemInfo: { | 20260505 19:42:17.617 | INFO | "gpu" : " 17 GB", | 20260505 19:42:17.617 | INFO | "huge" : "", | 20260505 19:42:17.617 | INFO | "huge_shared" : " 27 GB", | 20260505 19:42:17.617 | INFO | "malloc" : " 14 GB", | 20260505 19:42:17.617 | INFO | "pinned" : " 29 GB" | 20260505 19:42:17.617 | INFO | } | 20260505 19:42:17.617 | INFO | Perf report for the last 5.14604 seconds: | 20260505 19:42:17.617 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:42:17.617 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:17.617 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:17.617 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:17.617 | INFO | ANALYSIS : 62.3%, 3.0, 97.4ms, 0 | 20260505 19:42:17.617 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:17.617 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:17.617 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 19:42:17.617 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:17.617 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:17.617 | INFO | Uploaded 36GiB to GPU in 2.1611s (16.6581GiB/s) | 20260505 19:42:17.617 | INFO | Downloaded 34GiB to GPU in 2.0638s (16.4745GiB/s) | 20260505 19:42:22.761 | INFO | Analyzing chunk frames = [5803008,5803520) | 20260505 19:42:22.761 | INFO | MemInfo: { | 20260505 19:42:22.761 | INFO | "gpu" : " 17 GB", | 20260505 19:42:22.761 | INFO | "huge" : "", | 20260505 19:42:22.761 | INFO | "huge_shared" : " 27 GB", | 20260505 19:42:22.761 | INFO | "malloc" : " 14 GB", | 20260505 19:42:22.761 | INFO | "pinned" : " 29 GB" | 20260505 19:42:22.761 | INFO | } | 20260505 19:42:22.761 | INFO | Perf report for the last 5.14423 seconds: | 20260505 19:42:22.761 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:42:22.761 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:22.761 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:22.761 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:22.761 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 19:42:22.761 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:22.761 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:22.761 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 19:42:22.761 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:22.761 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:22.762 | INFO | Uploaded 36GiB to GPU in 2.12793s (16.9179GiB/s) | 20260505 19:42:22.762 | INFO | Downloaded 34GiB to GPU in 2.12136s (16.0275GiB/s) | 20260505 19:42:27.895 | INFO | Analyzing chunk frames = [5803520,5804032) | 20260505 19:42:27.895 | INFO | MemInfo: { | 20260505 19:42:27.895 | INFO | "gpu" : " 17 GB", | 20260505 19:42:27.895 | INFO | "huge" : "", | 20260505 19:42:27.895 | INFO | "huge_shared" : " 27 GB", | 20260505 19:42:27.895 | INFO | "malloc" : " 14 GB", | 20260505 19:42:27.895 | INFO | "pinned" : " 29 GB" | 20260505 19:42:27.895 | INFO | } | 20260505 19:42:27.895 | INFO | Perf report for the last 5.13408 seconds: | 20260505 19:42:27.895 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:42:27.895 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:27.895 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:27.895 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:27.895 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 19:42:27.895 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:27.895 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:27.895 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:42:27.895 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:27.896 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:27.896 | INFO | Uploaded 36GiB to GPU in 2.09742s (17.164GiB/s) | 20260505 19:42:27.896 | INFO | Downloaded 34GiB to GPU in 1.9995s (17.0043GiB/s) | 20260505 19:42:28.030 | INFO | Starting a round of Baz IO | 20260505 19:42:31.203 | INFO | Populating RTMetrics report | 20260505 19:42:31.204 | INFO | RTMetrics report populated | 20260505 19:42:31.204 | INFO | Opening tmp RTMetrics json file | 20260505 19:42:31.204 | INFO | Serializing RTMetrics to json | 20260505 19:42:31.205 | INFO | Serialization done, swapping tmp file now... | 20260505 19:42:31.206 | INFO | RTMetrics done | 20260505 19:42:31.292 | INFO | Finished a round of Baz IO: Wrote 2.041 GiB in 3.261 seconds | 20260505 19:42:31.292 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.75% / 0.00% / 78.25% | 20260505 19:42:31.292 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.199234% / 3.97364e-06% / 0.000103315% | 20260505 19:42:31.292 | INFO | Baz IO overall: | 20260505 19:42:31.292 | INFO | BytesWritten : 3395658113574 | 20260505 19:42:31.292 | INFO | NumEvents : 1606497469025 | 20260505 19:42:33.033 | INFO | Analyzing chunk frames = [5804032,5804544) | 20260505 19:42:33.033 | INFO | MemInfo: { | 20260505 19:42:33.033 | INFO | "gpu" : " 17 GB", | 20260505 19:42:33.033 | INFO | "huge" : "", | 20260505 19:42:33.033 | INFO | "huge_shared" : " 27 GB", | 20260505 19:42:33.033 | INFO | "malloc" : " 14 GB", | 20260505 19:42:33.033 | INFO | "pinned" : " 29 GB" | 20260505 19:42:33.033 | INFO | } | 20260505 19:42:33.033 | INFO | Perf report for the last 5.13733 seconds: | 20260505 19:42:33.033 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:42:33.033 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:33.033 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:33.033 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:33.033 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 19:42:33.033 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:33.033 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260505 19:42:33.033 | INFO | PRE_HQ : 60.3%, 1.8, 55.4ms, 0 | 20260505 19:42:33.033 | INFO | BAZWRITER : 63.0%, 3.7, 120.1ms, 0 | 20260505 19:42:33.033 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 19:42:33.033 | INFO | Uploaded 36GiB to GPU in 2.16984s (16.5911GiB/s) | 20260505 19:42:33.033 | INFO | Downloaded 38GiB to GPU in 2.25005s (16.8885GiB/s) | 20260505 19:42:38.175 | INFO | Analyzing chunk frames = [5804544,5805056) | 20260505 19:42:38.175 | INFO | MemInfo: { | 20260505 19:42:38.175 | INFO | "gpu" : " 17 GB", | 20260505 19:42:38.175 | INFO | "huge" : "", | 20260505 19:42:38.175 | INFO | "huge_shared" : " 27 GB", | 20260505 19:42:38.175 | INFO | "malloc" : " 14 GB", | 20260505 19:42:38.175 | INFO | "pinned" : " 29 GB" | 20260505 19:42:38.175 | INFO | } | 20260505 19:42:38.175 | INFO | Perf report for the last 5.14194 seconds: | 20260505 19:42:38.175 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:42:38.175 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:38.175 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:38.175 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:38.175 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260505 19:42:38.175 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:38.175 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:38.175 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:42:38.175 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:38.175 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:38.175 | INFO | Uploaded 36GiB to GPU in 2.13171s (16.8879GiB/s) | 20260505 19:42:38.175 | INFO | Downloaded 34GiB to GPU in 2.02802s (16.7651GiB/s) | 20260505 19:42:43.315 | INFO | Analyzing chunk frames = [5805056,5805568) | 20260505 19:42:43.315 | INFO | MemInfo: { | 20260505 19:42:43.315 | INFO | "gpu" : " 17 GB", | 20260505 19:42:43.315 | INFO | "huge" : "", | 20260505 19:42:43.315 | INFO | "huge_shared" : " 27 GB", | 20260505 19:42:43.315 | INFO | "malloc" : " 14 GB", | 20260505 19:42:43.315 | INFO | "pinned" : " 29 GB" | 20260505 19:42:43.315 | INFO | } | 20260505 19:42:43.315 | INFO | Perf report for the last 5.13999 seconds: | 20260505 19:42:43.315 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:42:43.315 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:43.315 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:43.315 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:43.315 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260505 19:42:43.315 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:43.315 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:43.315 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:42:43.315 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:43.315 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:43.315 | INFO | Uploaded 36GiB to GPU in 2.11915s (16.9879GiB/s) | 20260505 19:42:43.315 | INFO | Downloaded 34GiB to GPU in 2.02161s (16.8183GiB/s) | 20260505 19:42:48.449 | INFO | Analyzing chunk frames = [5805568,5806080) | 20260505 19:42:48.449 | INFO | MemInfo: { | 20260505 19:42:48.449 | INFO | "gpu" : " 17 GB", | 20260505 19:42:48.449 | INFO | "huge" : "", | 20260505 19:42:48.449 | INFO | "huge_shared" : " 27 GB", | 20260505 19:42:48.449 | INFO | "malloc" : " 14 GB", | 20260505 19:42:48.449 | INFO | "pinned" : " 29 GB" | 20260505 19:42:48.449 | INFO | } | 20260505 19:42:48.449 | INFO | Perf report for the last 5.13485 seconds: | 20260505 19:42:48.450 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:42:48.450 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:48.450 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:48.450 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:48.450 | INFO | ANALYSIS : 62.6%, 3.0, 97.2ms, 0 | 20260505 19:42:48.450 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:48.450 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:48.450 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:42:48.450 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:48.450 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:48.450 | INFO | Uploaded 36GiB to GPU in 2.21463s (16.2555GiB/s) | 20260505 19:42:48.450 | INFO | Downloaded 34GiB to GPU in 2.04661s (16.6128GiB/s) | 20260505 19:42:53.589 | INFO | Analyzing chunk frames = [5806080,5806592) | 20260505 19:42:53.589 | INFO | MemInfo: { | 20260505 19:42:53.589 | INFO | "gpu" : " 17 GB", | 20260505 19:42:53.589 | INFO | "huge" : "", | 20260505 19:42:53.589 | INFO | "huge_shared" : " 27 GB", | 20260505 19:42:53.589 | INFO | "malloc" : " 14 GB", | 20260505 19:42:53.589 | INFO | "pinned" : " 29 GB" | 20260505 19:42:53.589 | INFO | } | 20260505 19:42:53.589 | INFO | Perf report for the last 5.13933 seconds: | 20260505 19:42:53.589 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:42:53.589 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:53.589 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:53.589 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:53.589 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 19:42:53.589 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:53.589 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:53.589 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:42:53.589 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:53.589 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:53.589 | INFO | Uploaded 36GiB to GPU in 2.12718s (16.9238GiB/s) | 20260505 19:42:53.589 | INFO | Downloaded 34GiB to GPU in 2.06316s (16.4796GiB/s) | 20260505 19:42:58.728 | INFO | Analyzing chunk frames = [5806592,5807104) | 20260505 19:42:58.728 | INFO | MemInfo: { | 20260505 19:42:58.728 | INFO | "gpu" : " 17 GB", | 20260505 19:42:58.728 | INFO | "huge" : "", | 20260505 19:42:58.728 | INFO | "huge_shared" : " 27 GB", | 20260505 19:42:58.728 | INFO | "malloc" : " 14 GB", | 20260505 19:42:58.728 | INFO | "pinned" : " 29 GB" | 20260505 19:42:58.728 | INFO | } | 20260505 19:42:58.728 | INFO | Perf report for the last 5.1395 seconds: | 20260505 19:42:58.728 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:42:58.728 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:58.728 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:58.728 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:58.728 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 19:42:58.728 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:58.728 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:58.728 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:42:58.728 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:58.728 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:42:58.729 | INFO | Uploaded 36GiB to GPU in 2.16202s (16.6511GiB/s) | 20260505 19:42:58.729 | INFO | Downloaded 34GiB to GPU in 2.05828s (16.5187GiB/s) | 20260505 19:43:03.871 | INFO | Analyzing chunk frames = [5807104,5807616) | 20260505 19:43:03.871 | INFO | MemInfo: { | 20260505 19:43:03.871 | INFO | "gpu" : " 17 GB", | 20260505 19:43:03.871 | INFO | "huge" : "", | 20260505 19:43:03.871 | INFO | "huge_shared" : " 27 GB", | 20260505 19:43:03.871 | INFO | "malloc" : " 14 GB", | 20260505 19:43:03.871 | INFO | "pinned" : " 29 GB" | 20260505 19:43:03.871 | INFO | } | 20260505 19:43:03.871 | INFO | Perf report for the last 5.14248 seconds: | 20260505 19:43:03.871 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:43:03.871 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:03.871 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:03.871 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:03.871 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 19:43:03.871 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:03.871 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:03.871 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 19:43:03.871 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:03.871 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:03.871 | INFO | Uploaded 36GiB to GPU in 2.11742s (17.0018GiB/s) | 20260505 19:43:03.871 | INFO | Downloaded 34GiB to GPU in 2.13748s (15.9066GiB/s) | 20260505 19:43:09.009 | INFO | Analyzing chunk frames = [5807616,5808128) | 20260505 19:43:09.010 | INFO | MemInfo: { | 20260505 19:43:09.010 | INFO | "gpu" : " 17 GB", | 20260505 19:43:09.010 | INFO | "huge" : "", | 20260505 19:43:09.010 | INFO | "huge_shared" : " 27 GB", | 20260505 19:43:09.010 | INFO | "malloc" : " 14 GB", | 20260505 19:43:09.010 | INFO | "pinned" : " 29 GB" | 20260505 19:43:09.010 | INFO | } | 20260505 19:43:09.010 | INFO | Perf report for the last 5.13883 seconds: | 20260505 19:43:09.010 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:43:09.010 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:09.010 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:09.010 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:09.010 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 19:43:09.010 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:09.010 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:09.010 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 19:43:09.010 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:09.010 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:09.010 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 19:43:09.010 | INFO | Total mem usage: 15.3 GiB | 20260505 19:43:09.010 | INFO | Uploaded 36GiB to GPU in 2.20327s (16.3393GiB/s) | 20260505 19:43:09.010 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 19:43:09.010 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.89%) | 20260505 19:43:09.010 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 19:43:09.010 | INFO | Downloaded 34GiB to GPU in 2.04774s (16.6037GiB/s) | 20260505 19:43:09.010 | INFO | High Water Breakdown: | 20260505 19:43:09.010 | INFO | 60.9% : 8.57 GiB ( 90.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 19:43:09.010 | INFO | 39% : 5.48 GiB ( 82.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 19:43:09.010 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 19:43:09.010 | INFO | ------------------------------------------ | 20260505 19:43:09.010 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:43:09.010 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 19:43:09.010 | INFO | Total mem usage: 26.5 GiB | 20260505 19:43:09.010 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 19:43:09.010 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 19:43:09.010 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 19:43:09.010 | INFO | High Water Breakdown: | 20260505 19:43:09.010 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:43:09.010 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:43:09.010 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:43:09.010 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:43:09.010 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:43:09.010 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:43:09.010 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 19:43:09.010 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 19:43:09.010 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:43:09.010 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 19:43:09.010 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 19:43:09.010 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 19:43:09.010 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 19:43:09.010 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 19:43:09.010 | INFO | ------------------------------------------ | 20260505 19:43:09.010 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:43:09.010 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 19:43:09.010 | INFO | Total mem usage: 25.6 GiB | 20260505 19:43:09.010 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 19:43:09.010 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 19:43:09.010 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 19:43:09.010 | INFO | High Water Breakdown: | 20260505 19:43:09.010 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 19:43:09.010 | INFO | ------------------------------------------ | 20260505 19:43:09.010 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:43:09.010 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 19:43:09.010 | INFO | Total mem usage: 15.7 GiB | 20260505 19:43:09.010 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 19:43:09.010 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 19:43:09.010 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 19:43:09.010 | INFO | High Water Breakdown: | 20260505 19:43:09.010 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 19:43:09.010 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 19:43:09.010 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 19:43:09.010 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 19:43:09.010 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 19:43:09.010 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 19:43:09.010 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 19:43:09.010 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 19:43:09.010 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 19:43:09.011 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:43:09.011 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 19:43:09.011 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:43:09.011 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 19:43:09.011 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 19:43:09.011 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 19:43:09.011 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 19:43:09.011 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:43:09.011 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 19:43:09.011 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:43:09.011 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:43:09.011 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:43:09.011 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 19:43:09.011 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 19:43:09.011 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:43:09.011 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 19:43:09.011 | INFO | ------------------------------------------ | 20260505 19:43:09.011 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:43:09.011 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 19:43:09.011 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:43:09.011 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:43:09.011 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.45e+01, 1.35e+01 / 1.89e+00 | 20260505 19:43:09.011 | INFO | PerfReport: 0.29%, AggregateStats -- 9.24e-02 / 1.03e-01, 1.13e-02 / 3.35e-03 | 20260505 19:43:09.011 | INFO | PerfReport: 11.78%, Baseline -- 3.70e+00 / 4.58e+00, 5.56e-01 / 1.12e-01 | 20260505 19:43:09.011 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.54e+00, 1.90e-01 / 8.58e-02 | 20260505 19:43:09.011 | INFO | PerfReport: 15.99%, DME -- 4.02e+01 / 4.43e+01, 4.09e+00 / 1.88e+00 | 20260505 19:43:09.011 | INFO | PerfReport: 56.96%, FrameLabeling -- 1.79e+01 / 1.93e+01, 1.42e+00 / 9.56e-01 | 20260505 19:43:09.011 | INFO | PerfReport: 7.29%, PulseAccumulating -- 2.29e+00 / 2.88e+00, 4.41e-01 / 1.62e-01 | 20260505 19:43:09.011 | INFO | PerfReport: 3.28%, Metrics -- 1.03e+00 / 1.35e+00, 2.84e-01 / 1.33e-01 | 20260505 19:43:09.011 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.59e-01 / 4.23e-01, 6.28e-02 / 3.46e-02 | 20260505 19:43:09.011 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 19:43:09.011 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:43:09.011 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:43:09.011 | INFO | PerfReport: Full Execution -- 9.96e+01 / -nan, 2.55e+01 / 0.00e+00 | 20260505 19:43:09.011 | INFO | PerfReport: 22.50%, Upload -- 2.24e+01 / 0.00e+00, 7.83e+00 / 0.00e+00 | 20260505 19:43:09.011 | INFO | PerfReport: 21.74%, Download -- 2.16e+01 / 0.00e+00, 7.69e+00 / 0.00e+00 | 20260505 19:43:09.011 | INFO | PerfReport: 31.59%, Compute -- 3.15e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 19:43:09.011 | INFO | PerfReport: 2.38%, LockUpload -- 2.37e+00 / 0.00e+00, 8.17e+00 / 0.00e+00 | 20260505 19:43:09.011 | INFO | PerfReport: 1.36%, LockDownload -- 1.36e+00 / 0.00e+00, 2.89e+00 / 0.00e+00 | 20260505 19:43:09.011 | INFO | PerfReport: 20.41%, LockCompute -- 2.03e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 19:43:09.011 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.35e-02 / 0.00e+00 | 20260505 19:43:09.142 | INFO | Starting a round of Baz IO | 20260505 19:43:12.313 | INFO | Populating RTMetrics report | 20260505 19:43:12.313 | INFO | RTMetrics report populated | 20260505 19:43:12.313 | INFO | Opening tmp RTMetrics json file | 20260505 19:43:12.314 | INFO | Serializing RTMetrics to json | 20260505 19:43:12.315 | INFO | Serialization done, swapping tmp file now... | 20260505 19:43:12.316 | INFO | RTMetrics done | 20260505 19:43:12.404 | INFO | Finished a round of Baz IO: Wrote 2.04 GiB in 3.262 seconds | 20260505 19:43:12.404 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.75% / 0.00% / 78.25% | 20260505 19:43:12.404 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.198726% / 0% / 9.93411e-05% | 20260505 19:43:12.404 | INFO | Baz IO overall: | 20260505 19:43:12.404 | INFO | BytesWritten : 3397848332739 | 20260505 19:43:12.405 | INFO | NumEvents : 1607513714531 | 20260505 19:43:14.146 | INFO | Analyzing chunk frames = [5808128,5808640) | 20260505 19:43:14.146 | INFO | MemInfo: { | 20260505 19:43:14.146 | INFO | "gpu" : " 17 GB", | 20260505 19:43:14.146 | INFO | "huge" : "", | 20260505 19:43:14.146 | INFO | "huge_shared" : " 27 GB", | 20260505 19:43:14.146 | INFO | "malloc" : " 14 GB", | 20260505 19:43:14.146 | INFO | "pinned" : " 29 GB" | 20260505 19:43:14.146 | INFO | } | 20260505 19:43:14.146 | INFO | Perf report for the last 5.13621 seconds: | 20260505 19:43:14.146 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:43:14.146 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:14.146 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:14.146 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:14.146 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 19:43:14.146 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:14.146 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260505 19:43:14.146 | INFO | PRE_HQ : 60.2%, 1.8, 55.6ms, 0 | 20260505 19:43:14.146 | INFO | BAZWRITER : 63.0%, 3.7, 120.1ms, 0 | 20260505 19:43:14.146 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 19:43:14.146 | INFO | Uploaded 36GiB to GPU in 2.15603s (16.6974GiB/s) | 20260505 19:43:14.146 | INFO | Downloaded 38GiB to GPU in 2.25236s (16.8712GiB/s) | 20260505 19:43:19.276 | INFO | Analyzing chunk frames = [5808640,5809152) | 20260505 19:43:19.276 | INFO | MemInfo: { | 20260505 19:43:19.276 | INFO | "gpu" : " 17 GB", | 20260505 19:43:19.276 | INFO | "huge" : "", | 20260505 19:43:19.276 | INFO | "huge_shared" : " 27 GB", | 20260505 19:43:19.276 | INFO | "malloc" : " 14 GB", | 20260505 19:43:19.276 | INFO | "pinned" : " 29 GB" | 20260505 19:43:19.276 | INFO | } | 20260505 19:43:19.276 | INFO | Perf report for the last 5.13046 seconds: | 20260505 19:43:19.276 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:43:19.276 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:19.276 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:19.276 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:19.276 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 19:43:19.276 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:19.276 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:19.276 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260505 19:43:19.276 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:19.276 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:19.277 | INFO | Uploaded 36GiB to GPU in 2.13506s (16.8613GiB/s) | 20260505 19:43:19.277 | INFO | Downloaded 34GiB to GPU in 2.02974s (16.7509GiB/s) | 20260505 19:43:24.428 | INFO | Analyzing chunk frames = [5809152,5809664) | 20260505 19:43:24.428 | INFO | MemInfo: { | 20260505 19:43:24.428 | INFO | "gpu" : " 17 GB", | 20260505 19:43:24.428 | INFO | "huge" : "", | 20260505 19:43:24.428 | INFO | "huge_shared" : " 27 GB", | 20260505 19:43:24.428 | INFO | "malloc" : " 14 GB", | 20260505 19:43:24.428 | INFO | "pinned" : " 29 GB" | 20260505 19:43:24.428 | INFO | } | 20260505 19:43:24.428 | INFO | Perf report for the last 5.15188 seconds: | 20260505 19:43:24.428 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:43:24.428 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:24.428 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:24.428 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:24.428 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 19:43:24.428 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:24.428 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:24.428 | INFO | PRE_HQ : 27.2%, 1.0, 14.2ms, 0 | 20260505 19:43:24.428 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:24.428 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:24.429 | INFO | Uploaded 36GiB to GPU in 2.2096s (16.2925GiB/s) | 20260505 19:43:24.429 | INFO | Downloaded 34GiB to GPU in 2.047s (16.6097GiB/s) | 20260505 19:43:29.569 | INFO | Analyzing chunk frames = [5809664,5810176) | 20260505 19:43:29.569 | INFO | MemInfo: { | 20260505 19:43:29.569 | INFO | "gpu" : " 17 GB", | 20260505 19:43:29.569 | INFO | "huge" : "", | 20260505 19:43:29.569 | INFO | "huge_shared" : " 27 GB", | 20260505 19:43:29.569 | INFO | "malloc" : " 14 GB", | 20260505 19:43:29.569 | INFO | "pinned" : " 29 GB" | 20260505 19:43:29.569 | INFO | } | 20260505 19:43:29.569 | INFO | Perf report for the last 5.14123 seconds: | 20260505 19:43:29.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:43:29.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:29.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:29.570 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:29.570 | INFO | ANALYSIS : 62.5%, 3.0, 97.1ms, 0 | 20260505 19:43:29.570 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:29.570 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:29.570 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:43:29.570 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:29.570 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:29.570 | INFO | Uploaded 36GiB to GPU in 2.21213s (16.2739GiB/s) | 20260505 19:43:29.570 | INFO | Downloaded 34GiB to GPU in 2.06082s (16.4983GiB/s) | 20260505 19:43:34.705 | INFO | Analyzing chunk frames = [5810176,5810688) | 20260505 19:43:34.705 | INFO | MemInfo: { | 20260505 19:43:34.705 | INFO | "gpu" : " 17 GB", | 20260505 19:43:34.705 | INFO | "huge" : "", | 20260505 19:43:34.705 | INFO | "huge_shared" : " 27 GB", | 20260505 19:43:34.705 | INFO | "malloc" : " 14 GB", | 20260505 19:43:34.705 | INFO | "pinned" : " 29 GB" | 20260505 19:43:34.705 | INFO | } | 20260505 19:43:34.705 | INFO | Perf report for the last 5.13572 seconds: | 20260505 19:43:34.705 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:43:34.705 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:34.705 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:34.705 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:34.705 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260505 19:43:34.705 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:34.705 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:34.705 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:43:34.705 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:34.705 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:34.705 | INFO | Uploaded 36GiB to GPU in 2.13129s (16.8912GiB/s) | 20260505 19:43:34.705 | INFO | Downloaded 34GiB to GPU in 2.06303s (16.4806GiB/s) | 20260505 19:43:39.839 | INFO | Analyzing chunk frames = [5810688,5811200) | 20260505 19:43:39.839 | INFO | MemInfo: { | 20260505 19:43:39.839 | INFO | "gpu" : " 17 GB", | 20260505 19:43:39.839 | INFO | "huge" : "", | 20260505 19:43:39.839 | INFO | "huge_shared" : " 27 GB", | 20260505 19:43:39.839 | INFO | "malloc" : " 14 GB", | 20260505 19:43:39.839 | INFO | "pinned" : " 29 GB" | 20260505 19:43:39.839 | INFO | } | 20260505 19:43:39.839 | INFO | Perf report for the last 5.13417 seconds: | 20260505 19:43:39.839 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:43:39.839 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:39.839 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:39.839 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:39.839 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 19:43:39.839 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:39.839 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:39.839 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:43:39.839 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:39.839 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:39.840 | INFO | Uploaded 36GiB to GPU in 2.13204s (16.8853GiB/s) | 20260505 19:43:39.840 | INFO | Downloaded 34GiB to GPU in 2.05309s (16.5604GiB/s) | 20260505 19:43:44.984 | INFO | Analyzing chunk frames = [5811200,5811712) | 20260505 19:43:44.984 | INFO | MemInfo: { | 20260505 19:43:44.984 | INFO | "gpu" : " 17 GB", | 20260505 19:43:44.984 | INFO | "huge" : "", | 20260505 19:43:44.984 | INFO | "huge_shared" : " 27 GB", | 20260505 19:43:44.984 | INFO | "malloc" : " 14 GB", | 20260505 19:43:44.984 | INFO | "pinned" : " 29 GB" | 20260505 19:43:44.984 | INFO | } | 20260505 19:43:44.984 | INFO | Perf report for the last 5.14438 seconds: | 20260505 19:43:44.984 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:43:44.984 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:44.984 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:44.984 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:44.984 | INFO | ANALYSIS : 63.3%, 3.0, 99.0ms, 0 | 20260505 19:43:44.984 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:44.984 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:44.984 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 19:43:44.984 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:44.984 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:44.984 | INFO | Uploaded 36GiB to GPU in 2.1485s (16.7559GiB/s) | 20260505 19:43:44.984 | INFO | Downloaded 34GiB to GPU in 2.16459s (15.7073GiB/s) | 20260505 19:43:50.125 | INFO | Analyzing chunk frames = [5811712,5812224) | 20260505 19:43:50.125 | INFO | MemInfo: { | 20260505 19:43:50.125 | INFO | "gpu" : " 17 GB", | 20260505 19:43:50.125 | INFO | "huge" : "", | 20260505 19:43:50.125 | INFO | "huge_shared" : " 27 GB", | 20260505 19:43:50.125 | INFO | "malloc" : " 14 GB", | 20260505 19:43:50.125 | INFO | "pinned" : " 29 GB" | 20260505 19:43:50.125 | INFO | } | 20260505 19:43:50.125 | INFO | Perf report for the last 5.14121 seconds: | 20260505 19:43:50.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:43:50.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:50.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:50.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:50.125 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 19:43:50.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:50.125 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:50.125 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:43:50.125 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:50.125 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:50.125 | INFO | Uploaded 36GiB to GPU in 2.19618s (16.3921GiB/s) | 20260505 19:43:50.125 | INFO | Downloaded 34GiB to GPU in 2.03792s (16.6837GiB/s) | 20260505 19:43:50.257 | INFO | Starting a round of Baz IO | 20260505 19:43:53.429 | INFO | Populating RTMetrics report | 20260505 19:43:53.429 | INFO | RTMetrics report populated | 20260505 19:43:53.430 | INFO | Opening tmp RTMetrics json file | 20260505 19:43:53.431 | INFO | Serializing RTMetrics to json | 20260505 19:43:53.432 | INFO | Serialization done, swapping tmp file now... | 20260505 19:43:53.433 | INFO | RTMetrics done | 20260505 19:43:53.516 | INFO | Finished a round of Baz IO: Wrote 2.04 GiB in 3.259 seconds | 20260505 19:43:53.516 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.75% / 0.00% / 78.25% | 20260505 19:43:53.516 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.198778% / 0% / 7.54992e-05% | 20260505 19:43:53.516 | INFO | Baz IO overall: | 20260505 19:43:53.516 | INFO | BytesWritten : 3400038644881 | 20260505 19:43:53.516 | INFO | NumEvents : 1608530449023 | 20260505 19:43:55.266 | INFO | Analyzing chunk frames = [5812224,5812736) | 20260505 19:43:55.266 | INFO | MemInfo: { | 20260505 19:43:55.266 | INFO | "gpu" : " 17 GB", | 20260505 19:43:55.266 | INFO | "huge" : "", | 20260505 19:43:55.266 | INFO | "huge_shared" : " 27 GB", | 20260505 19:43:55.267 | INFO | "malloc" : " 14 GB", | 20260505 19:43:55.267 | INFO | "pinned" : " 29 GB" | 20260505 19:43:55.267 | INFO | } | 20260505 19:43:55.267 | INFO | Perf report for the last 5.14157 seconds: | 20260505 19:43:55.267 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:43:55.267 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:55.267 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:55.267 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:55.267 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 19:43:55.267 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:43:55.267 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260505 19:43:55.267 | INFO | PRE_HQ : 60.1%, 1.8, 55.2ms, 0 | 20260505 19:43:55.267 | INFO | BAZWRITER : 62.9%, 3.7, 120.4ms, 0 | 20260505 19:43:55.267 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 19:43:55.267 | INFO | Uploaded 36GiB to GPU in 2.16756s (16.6085GiB/s) | 20260505 19:43:55.267 | INFO | Downloaded 38GiB to GPU in 2.24726s (16.9095GiB/s) | 20260505 19:44:00.403 | INFO | Analyzing chunk frames = [5812736,5813248) | 20260505 19:44:00.403 | INFO | MemInfo: { | 20260505 19:44:00.403 | INFO | "gpu" : " 17 GB", | 20260505 19:44:00.403 | INFO | "huge" : "", | 20260505 19:44:00.403 | INFO | "huge_shared" : " 27 GB", | 20260505 19:44:00.403 | INFO | "malloc" : " 14 GB", | 20260505 19:44:00.403 | INFO | "pinned" : " 29 GB" | 20260505 19:44:00.403 | INFO | } | 20260505 19:44:00.403 | INFO | Perf report for the last 5.13641 seconds: | 20260505 19:44:00.403 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:44:00.403 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:00.403 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:00.403 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:00.403 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 19:44:00.403 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:00.403 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:00.403 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 19:44:00.403 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:00.403 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:00.403 | INFO | Uploaded 36GiB to GPU in 2.08834s (17.2386GiB/s) | 20260505 19:44:00.403 | INFO | Downloaded 34GiB to GPU in 2.02562s (16.7849GiB/s) | 20260505 19:44:05.539 | INFO | Analyzing chunk frames = [5813248,5813760) | 20260505 19:44:05.539 | INFO | MemInfo: { | 20260505 19:44:05.539 | INFO | "gpu" : " 17 GB", | 20260505 19:44:05.539 | INFO | "huge" : "", | 20260505 19:44:05.539 | INFO | "huge_shared" : " 27 GB", | 20260505 19:44:05.539 | INFO | "malloc" : " 14 GB", | 20260505 19:44:05.539 | INFO | "pinned" : " 29 GB" | 20260505 19:44:05.539 | INFO | } | 20260505 19:44:05.539 | INFO | Perf report for the last 5.13585 seconds: | 20260505 19:44:05.539 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:44:05.539 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:05.539 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:05.539 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:05.539 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260505 19:44:05.539 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:05.539 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:05.539 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260505 19:44:05.539 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:05.539 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:05.539 | INFO | Uploaded 36GiB to GPU in 2.12184s (16.9664GiB/s) | 20260505 19:44:05.539 | INFO | Downloaded 34GiB to GPU in 2.02072s (16.8257GiB/s) | 20260505 19:44:10.682 | INFO | Analyzing chunk frames = [5813760,5814272) | 20260505 19:44:10.682 | INFO | MemInfo: { | 20260505 19:44:10.682 | INFO | "gpu" : " 17 GB", | 20260505 19:44:10.682 | INFO | "huge" : "", | 20260505 19:44:10.682 | INFO | "huge_shared" : " 27 GB", | 20260505 19:44:10.682 | INFO | "malloc" : " 14 GB", | 20260505 19:44:10.682 | INFO | "pinned" : " 29 GB" | 20260505 19:44:10.682 | INFO | } | 20260505 19:44:10.682 | INFO | Perf report for the last 5.14325 seconds: | 20260505 19:44:10.682 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:44:10.682 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:10.682 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:10.682 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:10.682 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260505 19:44:10.682 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:10.682 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:10.682 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:44:10.682 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:10.682 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:10.682 | INFO | Uploaded 36GiB to GPU in 2.19274s (16.4178GiB/s) | 20260505 19:44:10.682 | INFO | Downloaded 34GiB to GPU in 2.0669s (16.4497GiB/s) | 20260505 19:44:15.823 | INFO | Analyzing chunk frames = [5814272,5814784) | 20260505 19:44:15.823 | INFO | MemInfo: { | 20260505 19:44:15.823 | INFO | "gpu" : " 17 GB", | 20260505 19:44:15.823 | INFO | "huge" : "", | 20260505 19:44:15.823 | INFO | "huge_shared" : " 27 GB", | 20260505 19:44:15.823 | INFO | "malloc" : " 14 GB", | 20260505 19:44:15.823 | INFO | "pinned" : " 29 GB" | 20260505 19:44:15.823 | INFO | } | 20260505 19:44:15.823 | INFO | Perf report for the last 5.14119 seconds: | 20260505 19:44:15.823 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:44:15.823 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:15.823 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:15.823 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:15.823 | INFO | ANALYSIS : 61.8%, 3.0, 96.6ms, 0 | 20260505 19:44:15.823 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:15.823 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:15.823 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:44:15.823 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:15.823 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:15.824 | INFO | Uploaded 36GiB to GPU in 2.12914s (16.9082GiB/s) | 20260505 19:44:15.824 | INFO | Downloaded 34GiB to GPU in 2.06207s (16.4883GiB/s) | 20260505 19:44:20.957 | INFO | Analyzing chunk frames = [5814784,5815296) | 20260505 19:44:20.957 | INFO | MemInfo: { | 20260505 19:44:20.957 | INFO | "gpu" : " 17 GB", | 20260505 19:44:20.957 | INFO | "huge" : "", | 20260505 19:44:20.957 | INFO | "huge_shared" : " 27 GB", | 20260505 19:44:20.957 | INFO | "malloc" : " 14 GB", | 20260505 19:44:20.957 | INFO | "pinned" : " 29 GB" | 20260505 19:44:20.957 | INFO | } | 20260505 19:44:20.957 | INFO | Perf report for the last 5.13371 seconds: | 20260505 19:44:20.957 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:44:20.957 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:20.957 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:20.957 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:20.957 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260505 19:44:20.957 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:20.957 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:20.957 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:44:20.957 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:20.957 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:20.957 | INFO | Uploaded 36GiB to GPU in 2.15638s (16.6947GiB/s) | 20260505 19:44:20.957 | INFO | Downloaded 34GiB to GPU in 2.0619s (16.4897GiB/s) | 20260505 19:44:26.094 | INFO | Analyzing chunk frames = [5815296,5815808) | 20260505 19:44:26.094 | INFO | MemInfo: { | 20260505 19:44:26.094 | INFO | "gpu" : " 17 GB", | 20260505 19:44:26.094 | INFO | "huge" : "", | 20260505 19:44:26.094 | INFO | "huge_shared" : " 27 GB", | 20260505 19:44:26.094 | INFO | "malloc" : " 14 GB", | 20260505 19:44:26.094 | INFO | "pinned" : " 29 GB" | 20260505 19:44:26.094 | INFO | } | 20260505 19:44:26.094 | INFO | Perf report for the last 5.13708 seconds: | 20260505 19:44:26.094 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:44:26.094 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:26.094 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:26.094 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:26.094 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 19:44:26.094 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:26.094 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:26.094 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 19:44:26.094 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:26.094 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:26.094 | INFO | Uploaded 36GiB to GPU in 2.13305s (16.8772GiB/s) | 20260505 19:44:26.094 | INFO | Downloaded 34GiB to GPU in 2.15291s (15.7926GiB/s) | 20260505 19:44:31.236 | INFO | Analyzing chunk frames = [5815808,5816320) | 20260505 19:44:31.236 | INFO | MemInfo: { | 20260505 19:44:31.236 | INFO | "gpu" : " 17 GB", | 20260505 19:44:31.236 | INFO | "huge" : "", | 20260505 19:44:31.236 | INFO | "huge_shared" : " 27 GB", | 20260505 19:44:31.236 | INFO | "malloc" : " 14 GB", | 20260505 19:44:31.236 | INFO | "pinned" : " 29 GB" | 20260505 19:44:31.236 | INFO | } | 20260505 19:44:31.236 | INFO | Perf report for the last 5.14183 seconds: | 20260505 19:44:31.236 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:44:31.236 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:31.236 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:31.236 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:31.236 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 19:44:31.236 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:31.236 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:31.236 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:44:31.236 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:31.236 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:31.236 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 19:44:31.236 | INFO | Total mem usage: 15.3 GiB | 20260505 19:44:31.236 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 19:44:31.236 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.91%) | 20260505 19:44:31.236 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 19:44:31.236 | INFO | High Water Breakdown: | 20260505 19:44:31.236 | INFO | 60.9% : 8.57 GiB ( 90.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 19:44:31.236 | INFO | 38.9% : 5.48 GiB ( 82.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 19:44:31.236 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 19:44:31.236 | INFO | ------------------------------------------ | 20260505 19:44:31.236 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:44:31.236 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 19:44:31.236 | INFO | Total mem usage: 26.5 GiB | 20260505 19:44:31.236 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 19:44:31.236 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 19:44:31.236 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 19:44:31.236 | INFO | High Water Breakdown: | 20260505 19:44:31.236 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:44:31.236 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:44:31.236 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:44:31.236 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:44:31.236 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:44:31.236 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:44:31.236 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 19:44:31.237 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 19:44:31.237 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:44:31.237 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 19:44:31.237 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 19:44:31.237 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 19:44:31.237 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 19:44:31.237 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 19:44:31.237 | INFO | ------------------------------------------ | 20260505 19:44:31.237 | INFO | Uploaded 36GiB to GPU in 2.20087s (16.3572GiB/s) | 20260505 19:44:31.237 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:44:31.237 | INFO | Downloaded 34GiB to GPU in 2.04086s (16.6596GiB/s) | 20260505 19:44:31.237 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 19:44:31.237 | INFO | Total mem usage: 25.6 GiB | 20260505 19:44:31.237 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 19:44:31.237 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 19:44:31.237 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 19:44:31.237 | INFO | High Water Breakdown: | 20260505 19:44:31.237 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 19:44:31.237 | INFO | ------------------------------------------ | 20260505 19:44:31.237 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:44:31.237 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 19:44:31.237 | INFO | Total mem usage: 15.7 GiB | 20260505 19:44:31.237 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 19:44:31.237 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 19:44:31.237 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 19:44:31.237 | INFO | High Water Breakdown: | 20260505 19:44:31.237 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 19:44:31.237 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 19:44:31.237 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 19:44:31.237 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 19:44:31.237 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 19:44:31.237 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 19:44:31.237 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 19:44:31.237 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 19:44:31.237 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 19:44:31.237 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:44:31.237 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 19:44:31.237 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:44:31.237 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 19:44:31.237 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 19:44:31.237 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 19:44:31.237 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 19:44:31.237 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:44:31.237 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 19:44:31.237 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:44:31.237 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:44:31.237 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:44:31.237 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 19:44:31.237 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 19:44:31.237 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:44:31.237 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 19:44:31.237 | INFO | ------------------------------------------ | 20260505 19:44:31.237 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:44:31.237 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 19:44:31.237 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:44:31.237 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:44:31.237 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.44e+01, 1.36e+01 / 1.90e+00 | 20260505 19:44:31.237 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.03e-01, 1.09e-02 / 2.31e-03 | 20260505 19:44:31.237 | INFO | PerfReport: 11.77%, Baseline -- 3.70e+00 / 4.54e+00, 5.57e-01 / 1.53e-01 | 20260505 19:44:31.237 | INFO | PerfReport: 3.26%, Histogram -- 1.03e+00 / 1.52e+00, 1.89e-01 / 1.05e-01 | 20260505 19:44:31.237 | INFO | PerfReport: 15.95%, DME -- 4.02e+01 / 4.41e+01, 3.98e+00 / 1.78e+00 | 20260505 19:44:31.237 | INFO | PerfReport: 57.01%, FrameLabeling -- 1.79e+01 / 1.94e+01, 1.46e+00 / 1.19e+00 | 20260505 19:44:31.237 | INFO | PerfReport: 7.29%, PulseAccumulating -- 2.29e+00 / 2.90e+00, 4.40e-01 / 1.38e-01 | 20260505 19:44:31.237 | INFO | PerfReport: 3.29%, Metrics -- 1.03e+00 / 1.37e+00, 2.83e-01 / 1.16e-01 | 20260505 19:44:31.237 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.59e-01 / 4.31e-01, 6.35e-02 / 3.93e-02 | 20260505 19:44:31.237 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 19:44:31.237 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:44:31.237 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:44:31.237 | INFO | PerfReport: Full Execution -- 9.97e+01 / -nan, 2.55e+01 / 0.00e+00 | 20260505 19:44:31.237 | INFO | PerfReport: 22.55%, Upload -- 2.25e+01 / 0.00e+00, 7.83e+00 / 0.00e+00 | 20260505 19:44:31.237 | INFO | PerfReport: 21.81%, Download -- 2.17e+01 / 0.00e+00, 7.75e+00 / 0.00e+00 | 20260505 19:44:31.237 | INFO | PerfReport: 31.59%, Compute -- 3.15e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 19:44:31.237 | INFO | PerfReport: 2.37%, LockUpload -- 2.36e+00 / 0.00e+00, 8.02e+00 / 0.00e+00 | 20260505 19:44:31.237 | INFO | PerfReport: 1.36%, LockDownload -- 1.36e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260505 19:44:31.237 | INFO | PerfReport: 20.31%, LockCompute -- 2.02e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260505 19:44:31.237 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.20e-02 / 0.00e+00 | 20260505 19:44:31.366 | INFO | Starting a round of Baz IO | 20260505 19:44:34.537 | INFO | Populating RTMetrics report | 20260505 19:44:34.537 | INFO | RTMetrics report populated | 20260505 19:44:34.537 | INFO | Opening tmp RTMetrics json file | 20260505 19:44:34.538 | INFO | Serializing RTMetrics to json | 20260505 19:44:34.540 | INFO | Serialization done, swapping tmp file now... | 20260505 19:44:34.540 | INFO | RTMetrics done | 20260505 19:44:34.623 | INFO | Finished a round of Baz IO: Wrote 2.04 GiB in 3.256 seconds | 20260505 19:44:34.623 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.74% / 0.00% / 78.25% | 20260505 19:44:34.623 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.198201% / 3.97364e-06% / 0.000103315% | 20260505 19:44:34.623 | INFO | Baz IO overall: | 20260505 19:44:34.623 | INFO | BytesWritten : 3402229520356 | 20260505 19:44:34.623 | INFO | NumEvents : 1609547957327 | 20260505 19:44:36.379 | INFO | Analyzing chunk frames = [5816320,5816832) | 20260505 19:44:36.379 | INFO | MemInfo: { | 20260505 19:44:36.379 | INFO | "gpu" : " 17 GB", | 20260505 19:44:36.379 | INFO | "huge" : "", | 20260505 19:44:36.379 | INFO | "huge_shared" : " 27 GB", | 20260505 19:44:36.379 | INFO | "malloc" : " 14 GB", | 20260505 19:44:36.379 | INFO | "pinned" : " 29 GB" | 20260505 19:44:36.379 | INFO | } | 20260505 19:44:36.379 | INFO | Perf report for the last 5.14292 seconds: | 20260505 19:44:36.379 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:44:36.379 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:36.379 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:36.379 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:36.379 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 19:44:36.379 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:36.379 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260505 19:44:36.379 | INFO | PRE_HQ : 60.3%, 1.8, 55.4ms, 0 | 20260505 19:44:36.379 | INFO | BAZWRITER : 62.9%, 3.7, 120.4ms, 0 | 20260505 19:44:36.379 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 19:44:36.379 | INFO | Uploaded 36GiB to GPU in 2.15874s (16.6764GiB/s) | 20260505 19:44:36.379 | INFO | Downloaded 38GiB to GPU in 2.24002s (16.9641GiB/s) | 20260505 19:44:41.519 | INFO | Analyzing chunk frames = [5816832,5817344) | 20260505 19:44:41.519 | INFO | MemInfo: { | 20260505 19:44:41.519 | INFO | "gpu" : " 17 GB", | 20260505 19:44:41.519 | INFO | "huge" : "", | 20260505 19:44:41.520 | INFO | "huge_shared" : " 27 GB", | 20260505 19:44:41.520 | INFO | "malloc" : " 14 GB", | 20260505 19:44:41.520 | INFO | "pinned" : " 29 GB" | 20260505 19:44:41.520 | INFO | } | 20260505 19:44:41.520 | INFO | Perf report for the last 5.14069 seconds: | 20260505 19:44:41.520 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:44:41.520 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:41.520 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:41.520 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:41.520 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 19:44:41.520 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:41.520 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:41.520 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 19:44:41.520 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:41.520 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:41.520 | INFO | Uploaded 36GiB to GPU in 2.08092s (17.3GiB/s) | 20260505 19:44:41.520 | INFO | Downloaded 34GiB to GPU in 2.02618s (16.7804GiB/s) | 20260505 19:44:46.652 | INFO | Analyzing chunk frames = [5817344,5817856) | 20260505 19:44:46.652 | INFO | MemInfo: { | 20260505 19:44:46.652 | INFO | "gpu" : " 17 GB", | 20260505 19:44:46.652 | INFO | "huge" : "", | 20260505 19:44:46.652 | INFO | "huge_shared" : " 27 GB", | 20260505 19:44:46.652 | INFO | "malloc" : " 14 GB", | 20260505 19:44:46.652 | INFO | "pinned" : " 29 GB" | 20260505 19:44:46.652 | INFO | } | 20260505 19:44:46.652 | INFO | Perf report for the last 5.13252 seconds: | 20260505 19:44:46.652 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:44:46.652 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:46.652 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:46.652 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:46.652 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260505 19:44:46.652 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:46.652 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:46.652 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:44:46.652 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:46.652 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:46.652 | INFO | Uploaded 36GiB to GPU in 2.12716s (16.924GiB/s) | 20260505 19:44:46.652 | INFO | Downloaded 34GiB to GPU in 2.01578s (16.8669GiB/s) | 20260505 19:44:51.787 | INFO | Analyzing chunk frames = [5817856,5818368) | 20260505 19:44:51.787 | INFO | MemInfo: { | 20260505 19:44:51.787 | INFO | "gpu" : " 17 GB", | 20260505 19:44:51.787 | INFO | "huge" : "", | 20260505 19:44:51.787 | INFO | "huge_shared" : " 27 GB", | 20260505 19:44:51.787 | INFO | "malloc" : " 14 GB", | 20260505 19:44:51.787 | INFO | "pinned" : " 29 GB" | 20260505 19:44:51.787 | INFO | } | 20260505 19:44:51.787 | INFO | Perf report for the last 5.13521 seconds: | 20260505 19:44:51.787 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:44:51.787 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:51.787 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:51.787 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:51.787 | INFO | ANALYSIS : 62.4%, 3.0, 96.8ms, 0 | 20260505 19:44:51.787 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:51.787 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:51.787 | INFO | PRE_HQ : 27.5%, 1.0, 14.2ms, 0 | 20260505 19:44:51.787 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:51.787 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:51.788 | INFO | Uploaded 36GiB to GPU in 2.18522s (16.4743GiB/s) | 20260505 19:44:51.788 | INFO | Downloaded 34GiB to GPU in 2.06638s (16.4539GiB/s) | 20260505 19:44:56.936 | INFO | Analyzing chunk frames = [5818368,5818880) | 20260505 19:44:56.936 | INFO | MemInfo: { | 20260505 19:44:56.936 | INFO | "gpu" : " 17 GB", | 20260505 19:44:56.936 | INFO | "huge" : "", | 20260505 19:44:56.936 | INFO | "huge_shared" : " 27 GB", | 20260505 19:44:56.936 | INFO | "malloc" : " 14 GB", | 20260505 19:44:56.936 | INFO | "pinned" : " 29 GB" | 20260505 19:44:56.936 | INFO | } | 20260505 19:44:56.936 | INFO | Perf report for the last 5.14889 seconds: | 20260505 19:44:56.936 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:44:56.936 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:56.936 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:56.936 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:56.936 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260505 19:44:56.936 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:56.936 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:56.936 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:44:56.936 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:56.936 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:44:56.937 | INFO | Uploaded 36GiB to GPU in 2.14611s (16.7746GiB/s) | 20260505 19:44:56.937 | INFO | Downloaded 34GiB to GPU in 2.06107s (16.4963GiB/s) | 20260505 19:45:02.078 | INFO | Analyzing chunk frames = [5818880,5819392) | 20260505 19:45:02.078 | INFO | MemInfo: { | 20260505 19:45:02.078 | INFO | "gpu" : " 17 GB", | 20260505 19:45:02.078 | INFO | "huge" : "", | 20260505 19:45:02.078 | INFO | "huge_shared" : " 27 GB", | 20260505 19:45:02.078 | INFO | "malloc" : " 14 GB", | 20260505 19:45:02.078 | INFO | "pinned" : " 29 GB" | 20260505 19:45:02.078 | INFO | } | 20260505 19:45:02.078 | INFO | Perf report for the last 5.14199 seconds: | 20260505 19:45:02.078 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:45:02.078 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:02.078 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:02.078 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:02.078 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 19:45:02.078 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:02.078 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:02.078 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:45:02.078 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:02.078 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:02.079 | INFO | Uploaded 36GiB to GPU in 2.14592s (16.776GiB/s) | 20260505 19:45:02.079 | INFO | Downloaded 34GiB to GPU in 2.0605s (16.5009GiB/s) | 20260505 19:45:07.218 | INFO | Analyzing chunk frames = [5819392,5819904) | 20260505 19:45:07.218 | INFO | MemInfo: { | 20260505 19:45:07.218 | INFO | "gpu" : " 17 GB", | 20260505 19:45:07.218 | INFO | "huge" : "", | 20260505 19:45:07.218 | INFO | "huge_shared" : " 27 GB", | 20260505 19:45:07.218 | INFO | "malloc" : " 14 GB", | 20260505 19:45:07.218 | INFO | "pinned" : " 29 GB" | 20260505 19:45:07.218 | INFO | } | 20260505 19:45:07.218 | INFO | Perf report for the last 5.13946 seconds: | 20260505 19:45:07.218 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:45:07.218 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:07.218 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:07.218 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:07.218 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 19:45:07.218 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:07.218 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:07.218 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 19:45:07.218 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:07.218 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:07.218 | INFO | Uploaded 36GiB to GPU in 2.12536s (16.9383GiB/s) | 20260505 19:45:07.218 | INFO | Downloaded 34GiB to GPU in 2.16404s (15.7113GiB/s) | 20260505 19:45:12.356 | INFO | Analyzing chunk frames = [5819904,5820416) | 20260505 19:45:12.356 | INFO | MemInfo: { | 20260505 19:45:12.356 | INFO | "gpu" : " 17 GB", | 20260505 19:45:12.356 | INFO | "huge" : "", | 20260505 19:45:12.356 | INFO | "huge_shared" : " 27 GB", | 20260505 19:45:12.356 | INFO | "malloc" : " 14 GB", | 20260505 19:45:12.356 | INFO | "pinned" : " 29 GB" | 20260505 19:45:12.356 | INFO | } | 20260505 19:45:12.356 | INFO | Perf report for the last 5.13846 seconds: | 20260505 19:45:12.356 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:45:12.356 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:12.356 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:12.356 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:12.356 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 19:45:12.356 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:12.356 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:12.356 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:45:12.356 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:12.356 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:12.356 | INFO | Uploaded 36GiB to GPU in 2.20723s (16.3101GiB/s) | 20260505 19:45:12.356 | INFO | Downloaded 34GiB to GPU in 2.04865s (16.5963GiB/s) | 20260505 19:45:12.485 | INFO | Starting a round of Baz IO | 20260505 19:45:15.658 | INFO | Populating RTMetrics report | 20260505 19:45:15.658 | INFO | RTMetrics report populated | 20260505 19:45:15.658 | INFO | Opening tmp RTMetrics json file | 20260505 19:45:15.662 | INFO | Serializing RTMetrics to json | 20260505 19:45:15.663 | INFO | Serialization done, swapping tmp file now... | 20260505 19:45:15.664 | INFO | RTMetrics done | 20260505 19:45:15.745 | INFO | Finished a round of Baz IO: Wrote 2.038 GiB in 3.26 seconds | 20260505 19:45:15.745 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.74% / 0.00% / 78.26% | 20260505 19:45:15.745 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.197554% / 0% / 6.75519e-05% | 20260505 19:45:15.745 | INFO | Baz IO overall: | 20260505 19:45:15.745 | INFO | BytesWritten : 3404417518264 | 20260505 19:45:15.745 | INFO | NumEvents : 1610563374372 | 20260505 19:45:17.493 | INFO | Analyzing chunk frames = [5820416,5820928) | 20260505 19:45:17.493 | INFO | MemInfo: { | 20260505 19:45:17.493 | INFO | "gpu" : " 17 GB", | 20260505 19:45:17.493 | INFO | "huge" : "", | 20260505 19:45:17.497 | INFO | "huge_shared" : " 27 GB", | 20260505 19:45:17.497 | INFO | "malloc" : " 14 GB", | 20260505 19:45:17.497 | INFO | "pinned" : " 29 GB" | 20260505 19:45:17.497 | INFO | } | 20260505 19:45:17.497 | INFO | Perf report for the last 5.1404 seconds: | 20260505 19:45:17.497 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:45:17.497 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:17.497 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:17.497 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:17.497 | INFO | ANALYSIS : 63.2%, 3.0, 98.4ms, 0 | 20260505 19:45:17.497 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:17.497 | INFO | RT_METRICS : 44.1%, 1.0, 23.2ms, 0 | 20260505 19:45:17.497 | INFO | PRE_HQ : 60.3%, 1.8, 55.8ms, 0 | 20260505 19:45:17.497 | INFO | BAZWRITER : 63.0%, 3.7, 120.8ms, 0 | 20260505 19:45:17.497 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 19:45:17.497 | INFO | Uploaded 36GiB to GPU in 2.16957s (16.5932GiB/s) | 20260505 19:45:17.497 | INFO | Downloaded 38GiB to GPU in 2.25893s (16.8221GiB/s) | 20260505 19:45:22.634 | INFO | Analyzing chunk frames = [5820928,5821440) | 20260505 19:45:22.634 | INFO | MemInfo: { | 20260505 19:45:22.634 | INFO | "gpu" : " 17 GB", | 20260505 19:45:22.634 | INFO | "huge" : "", | 20260505 19:45:22.634 | INFO | "huge_shared" : " 27 GB", | 20260505 19:45:22.634 | INFO | "malloc" : " 14 GB", | 20260505 19:45:22.634 | INFO | "pinned" : " 29 GB" | 20260505 19:45:22.634 | INFO | } | 20260505 19:45:22.634 | INFO | Perf report for the last 5.13721 seconds: | 20260505 19:45:22.634 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:45:22.634 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:22.634 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:22.634 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:22.634 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260505 19:45:22.634 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:22.634 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:22.634 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:45:22.634 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:22.634 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:22.634 | INFO | Uploaded 36GiB to GPU in 2.1038s (17.1119GiB/s) | 20260505 19:45:22.634 | INFO | Downloaded 34GiB to GPU in 2.02567s (16.7846GiB/s) | 20260505 19:45:27.771 | INFO | Analyzing chunk frames = [5821440,5821952) | 20260505 19:45:27.771 | INFO | MemInfo: { | 20260505 19:45:27.771 | INFO | "gpu" : " 17 GB", | 20260505 19:45:27.771 | INFO | "huge" : "", | 20260505 19:45:27.771 | INFO | "huge_shared" : " 27 GB", | 20260505 19:45:27.771 | INFO | "malloc" : " 14 GB", | 20260505 19:45:27.771 | INFO | "pinned" : " 29 GB" | 20260505 19:45:27.771 | INFO | } | 20260505 19:45:27.771 | INFO | Perf report for the last 5.13744 seconds: | 20260505 19:45:27.771 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:45:27.771 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:27.771 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:27.771 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:27.771 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260505 19:45:27.771 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:27.771 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:27.771 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:45:27.771 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:27.771 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:27.772 | INFO | Uploaded 36GiB to GPU in 2.13129s (16.8912GiB/s) | 20260505 19:45:27.772 | INFO | Downloaded 34GiB to GPU in 2.01463s (16.8765GiB/s) | 20260505 19:45:32.905 | INFO | Analyzing chunk frames = [5821952,5822464) | 20260505 19:45:32.905 | INFO | MemInfo: { | 20260505 19:45:32.905 | INFO | "gpu" : " 17 GB", | 20260505 19:45:32.905 | INFO | "huge" : "", | 20260505 19:45:32.905 | INFO | "huge_shared" : " 27 GB", | 20260505 19:45:32.905 | INFO | "malloc" : " 14 GB", | 20260505 19:45:32.905 | INFO | "pinned" : " 29 GB" | 20260505 19:45:32.905 | INFO | } | 20260505 19:45:32.906 | INFO | Perf report for the last 5.13421 seconds: | 20260505 19:45:32.906 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:45:32.906 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:32.906 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:32.906 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:32.906 | INFO | ANALYSIS : 62.4%, 3.0, 96.9ms, 0 | 20260505 19:45:32.906 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:32.906 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:32.906 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 19:45:32.906 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:32.906 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:32.906 | INFO | Uploaded 36GiB to GPU in 2.18352s (16.4872GiB/s) | 20260505 19:45:32.906 | INFO | Downloaded 34GiB to GPU in 2.06951s (16.429GiB/s) | 20260505 19:45:38.041 | INFO | Analyzing chunk frames = [5822464,5822976) | 20260505 19:45:38.041 | INFO | MemInfo: { | 20260505 19:45:38.041 | INFO | "gpu" : " 17 GB", | 20260505 19:45:38.041 | INFO | "huge" : "", | 20260505 19:45:38.041 | INFO | "huge_shared" : " 27 GB", | 20260505 19:45:38.041 | INFO | "malloc" : " 14 GB", | 20260505 19:45:38.041 | INFO | "pinned" : " 29 GB" | 20260505 19:45:38.041 | INFO | } | 20260505 19:45:38.041 | INFO | Perf report for the last 5.13561 seconds: | 20260505 19:45:38.041 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:45:38.041 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:38.041 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:38.041 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:38.041 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 19:45:38.041 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:38.041 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:38.041 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:45:38.041 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:38.041 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:38.042 | INFO | Uploaded 36GiB to GPU in 2.14268s (16.8014GiB/s) | 20260505 19:45:38.042 | INFO | Downloaded 34GiB to GPU in 2.06548s (16.4611GiB/s) | 20260505 19:45:43.193 | INFO | Analyzing chunk frames = [5822976,5823488) | 20260505 19:45:43.193 | INFO | MemInfo: { | 20260505 19:45:43.193 | INFO | "gpu" : " 17 GB", | 20260505 19:45:43.193 | INFO | "huge" : "", | 20260505 19:45:43.193 | INFO | "huge_shared" : " 27 GB", | 20260505 19:45:43.193 | INFO | "malloc" : " 14 GB", | 20260505 19:45:43.193 | INFO | "pinned" : " 29 GB" | 20260505 19:45:43.193 | INFO | } | 20260505 19:45:43.194 | INFO | Perf report for the last 5.15233 seconds: | 20260505 19:45:43.194 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:45:43.194 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:43.194 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:43.194 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:43.194 | INFO | ANALYSIS : 62.3%, 3.0, 97.5ms, 0 | 20260505 19:45:43.194 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:43.194 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:43.194 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:45:43.194 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:43.194 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:43.194 | INFO | Uploaded 36GiB to GPU in 2.14461s (16.7863GiB/s) | 20260505 19:45:43.194 | INFO | Downloaded 34GiB to GPU in 2.04966s (16.5882GiB/s) | 20260505 19:45:48.330 | INFO | Analyzing chunk frames = [5823488,5824000) | 20260505 19:45:48.330 | INFO | MemInfo: { | 20260505 19:45:48.330 | INFO | "gpu" : " 17 GB", | 20260505 19:45:48.330 | INFO | "huge" : "", | 20260505 19:45:48.330 | INFO | "huge_shared" : " 27 GB", | 20260505 19:45:48.330 | INFO | "malloc" : " 14 GB", | 20260505 19:45:48.330 | INFO | "pinned" : " 29 GB" | 20260505 19:45:48.330 | INFO | } | 20260505 19:45:48.330 | INFO | Perf report for the last 5.13694 seconds: | 20260505 19:45:48.330 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:45:48.330 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:48.330 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:48.330 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:48.330 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 19:45:48.330 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:48.331 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:48.331 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 19:45:48.331 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:48.331 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:48.331 | INFO | Uploaded 36GiB to GPU in 2.12967s (16.904GiB/s) | 20260505 19:45:48.331 | INFO | Downloaded 34GiB to GPU in 2.15688s (15.7635GiB/s) | 20260505 19:45:53.471 | INFO | Analyzing chunk frames = [5824000,5824512) | 20260505 19:45:53.471 | INFO | MemInfo: { | 20260505 19:45:53.471 | INFO | "gpu" : " 17 GB", | 20260505 19:45:53.471 | INFO | "huge" : "", | 20260505 19:45:53.471 | INFO | "huge_shared" : " 27 GB", | 20260505 19:45:53.471 | INFO | "malloc" : " 14 GB", | 20260505 19:45:53.471 | INFO | "pinned" : " 29 GB" | 20260505 19:45:53.471 | INFO | } | 20260505 19:45:53.471 | INFO | Perf report for the last 5.14048 seconds: | 20260505 19:45:53.471 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:45:53.471 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:53.471 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:53.471 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:53.471 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 19:45:53.471 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:53.471 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:53.471 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 19:45:53.471 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:53.471 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:53.471 | INFO | Uploaded 36GiB to GPU in 2.20456s (16.3298GiB/s) | 20260505 19:45:53.471 | INFO | Downloaded 34GiB to GPU in 2.0467s (16.6121GiB/s) | 20260505 19:45:53.471 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 19:45:53.471 | INFO | Total mem usage: 15.3 GiB | 20260505 19:45:53.471 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 19:45:53.471 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.09%) | 20260505 19:45:53.471 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 19:45:53.471 | INFO | High Water Breakdown: | 20260505 19:45:53.471 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 19:45:53.471 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 19:45:53.471 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 19:45:53.471 | INFO | ------------------------------------------ | 20260505 19:45:53.471 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:45:53.471 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 19:45:53.471 | INFO | Total mem usage: 26.5 GiB | 20260505 19:45:53.471 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 19:45:53.471 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 19:45:53.471 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 19:45:53.471 | INFO | High Water Breakdown: | 20260505 19:45:53.471 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:45:53.471 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:45:53.471 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:45:53.471 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:45:53.471 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:45:53.472 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:45:53.472 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 19:45:53.472 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 19:45:53.472 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:45:53.472 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 19:45:53.472 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 19:45:53.472 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 19:45:53.472 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 19:45:53.472 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 19:45:53.472 | INFO | ------------------------------------------ | 20260505 19:45:53.472 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:45:53.472 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 19:45:53.472 | INFO | Total mem usage: 25.6 GiB | 20260505 19:45:53.472 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 19:45:53.472 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 19:45:53.472 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 19:45:53.472 | INFO | High Water Breakdown: | 20260505 19:45:53.472 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 19:45:53.472 | INFO | ------------------------------------------ | 20260505 19:45:53.472 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:45:53.472 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 19:45:53.472 | INFO | Total mem usage: 15.7 GiB | 20260505 19:45:53.472 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 19:45:53.472 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 19:45:53.472 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 19:45:53.472 | INFO | High Water Breakdown: | 20260505 19:45:53.472 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 19:45:53.472 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 19:45:53.472 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 19:45:53.472 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 19:45:53.472 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 19:45:53.472 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 19:45:53.472 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 19:45:53.472 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 19:45:53.472 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 19:45:53.472 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:45:53.472 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 19:45:53.472 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:45:53.472 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 19:45:53.472 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 19:45:53.472 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 19:45:53.472 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 19:45:53.472 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:45:53.472 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 19:45:53.472 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:45:53.472 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:45:53.472 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:45:53.472 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 19:45:53.472 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 19:45:53.472 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:45:53.472 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 19:45:53.472 | INFO | ------------------------------------------ | 20260505 19:45:53.472 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:45:53.472 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 19:45:53.472 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:45:53.472 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:45:53.472 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.46e+01, 1.36e+01 / 1.76e+00 | 20260505 19:45:53.472 | INFO | PerfReport: 0.29%, AggregateStats -- 9.26e-02 / 1.03e-01, 1.06e-02 / 4.15e-03 | 20260505 19:45:53.472 | INFO | PerfReport: 11.79%, Baseline -- 3.71e+00 / 4.53e+00, 5.65e-01 / 1.70e-01 | 20260505 19:45:53.472 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.53e+00, 1.90e-01 / 8.55e-02 | 20260505 19:45:53.472 | INFO | PerfReport: 15.97%, DME -- 4.03e+01 / 4.44e+01, 4.09e+00 / 1.86e+00 | 20260505 19:45:53.472 | INFO | PerfReport: 57.04%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.50e+00 / 1.05e+00 | 20260505 19:45:53.472 | INFO | PerfReport: 7.22%, PulseAccumulating -- 2.28e+00 / 2.84e+00, 4.36e-01 / 2.10e-01 | 20260505 19:45:53.472 | INFO | PerfReport: 3.26%, Metrics -- 1.03e+00 / 1.34e+00, 2.80e-01 / 1.72e-01 | 20260505 19:45:53.472 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.59e-01 / 4.18e-01, 6.33e-02 / 3.79e-02 | 20260505 19:45:53.472 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 19:45:53.472 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:45:53.472 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:45:53.472 | INFO | PerfReport: Full Execution -- 9.96e+01 / -nan, 2.57e+01 / 0.00e+00 | 20260505 19:45:53.472 | INFO | PerfReport: 22.48%, Upload -- 2.24e+01 / 0.00e+00, 7.77e+00 / 0.00e+00 | 20260505 19:45:53.472 | INFO | PerfReport: 21.82%, Download -- 2.17e+01 / 0.00e+00, 7.77e+00 / 0.00e+00 | 20260505 19:45:53.472 | INFO | PerfReport: 31.65%, Compute -- 3.15e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 19:45:53.472 | INFO | PerfReport: 2.33%, LockUpload -- 2.32e+00 / 0.00e+00, 8.00e+00 / 0.00e+00 | 20260505 19:45:53.472 | INFO | PerfReport: 1.39%, LockDownload -- 1.39e+00 / 0.00e+00, 2.98e+00 / 0.00e+00 | 20260505 19:45:53.472 | INFO | PerfReport: 20.31%, LockCompute -- 2.02e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 19:45:53.472 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.72e-02 / 0.00e+00 | 20260505 19:45:53.600 | INFO | Starting a round of Baz IO | 20260505 19:45:56.768 | INFO | Populating RTMetrics report | 20260505 19:45:56.769 | INFO | RTMetrics report populated | 20260505 19:45:56.769 | INFO | Opening tmp RTMetrics json file | 20260505 19:45:56.769 | INFO | Serializing RTMetrics to json | 20260505 19:45:56.770 | INFO | Serialization done, swapping tmp file now... | 20260505 19:45:56.771 | INFO | RTMetrics done | 20260505 19:45:56.856 | INFO | Finished a round of Baz IO: Wrote 2.037 GiB in 3.255 seconds | 20260505 19:45:56.856 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.74% / 0.00% / 78.26% | 20260505 19:45:56.856 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.196719% / 0% / 7.15256e-05% | 20260505 19:45:56.856 | INFO | Baz IO overall: | 20260505 19:45:56.856 | INFO | BytesWritten : 3406604568117 | 20260505 19:45:56.856 | INFO | NumEvents : 1611578398575 | 20260505 19:45:58.599 | INFO | Analyzing chunk frames = [5824512,5825024) | 20260505 19:45:58.599 | INFO | MemInfo: { | 20260505 19:45:58.599 | INFO | "gpu" : " 17 GB", | 20260505 19:45:58.600 | INFO | "huge" : "", | 20260505 19:45:58.600 | INFO | "huge_shared" : " 27 GB", | 20260505 19:45:58.600 | INFO | "malloc" : " 14 GB", | 20260505 19:45:58.600 | INFO | "pinned" : " 29 GB" | 20260505 19:45:58.600 | INFO | } | 20260505 19:45:58.600 | INFO | Perf report for the last 5.12865 seconds: | 20260505 19:45:58.600 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:45:58.600 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:58.600 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:58.600 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:58.600 | INFO | ANALYSIS : 63.2%, 3.0, 98.3ms, 0 | 20260505 19:45:58.600 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:45:58.600 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260505 19:45:58.600 | INFO | PRE_HQ : 60.6%, 1.8, 55.7ms, 0 | 20260505 19:45:58.600 | INFO | BAZWRITER : 63.0%, 3.7, 119.5ms, 0 | 20260505 19:45:58.600 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 19:45:58.600 | INFO | Uploaded 36GiB to GPU in 2.15505s (16.705GiB/s) | 20260505 19:45:58.600 | INFO | Downloaded 38GiB to GPU in 2.24983s (16.8902GiB/s) | 20260505 19:46:03.748 | INFO | Analyzing chunk frames = [5825024,5825536) | 20260505 19:46:03.748 | INFO | MemInfo: { | 20260505 19:46:03.748 | INFO | "gpu" : " 17 GB", | 20260505 19:46:03.748 | INFO | "huge" : "", | 20260505 19:46:03.748 | INFO | "huge_shared" : " 27 GB", | 20260505 19:46:03.749 | INFO | "malloc" : " 14 GB", | 20260505 19:46:03.749 | INFO | "pinned" : " 29 GB" | 20260505 19:46:03.749 | INFO | } | 20260505 19:46:03.749 | INFO | Perf report for the last 5.14926 seconds: | 20260505 19:46:03.749 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:46:03.749 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:03.749 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:03.749 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:03.749 | INFO | ANALYSIS : 62.6%, 3.0, 97.9ms, 0 | 20260505 19:46:03.749 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:03.749 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:03.749 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 19:46:03.749 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:03.749 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:03.749 | INFO | Uploaded 36GiB to GPU in 2.14199s (16.8068GiB/s) | 20260505 19:46:03.749 | INFO | Downloaded 34GiB to GPU in 2.02521s (16.7884GiB/s) | 20260505 19:46:08.887 | INFO | Analyzing chunk frames = [5825536,5826048) | 20260505 19:46:08.887 | INFO | MemInfo: { | 20260505 19:46:08.887 | INFO | "gpu" : " 17 GB", | 20260505 19:46:08.887 | INFO | "huge" : "", | 20260505 19:46:08.887 | INFO | "huge_shared" : " 27 GB", | 20260505 19:46:08.887 | INFO | "malloc" : " 14 GB", | 20260505 19:46:08.887 | INFO | "pinned" : " 29 GB" | 20260505 19:46:08.887 | INFO | } | 20260505 19:46:08.887 | INFO | Perf report for the last 5.1381 seconds: | 20260505 19:46:08.887 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:46:08.887 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:08.887 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:08.887 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:08.887 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260505 19:46:08.887 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:08.887 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:08.887 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:46:08.887 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:08.887 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:08.887 | INFO | Uploaded 36GiB to GPU in 2.12839s (16.9142GiB/s) | 20260505 19:46:08.887 | INFO | Downloaded 34GiB to GPU in 2.00831s (16.9296GiB/s) | 20260505 19:46:14.032 | INFO | Analyzing chunk frames = [5826048,5826560) | 20260505 19:46:14.032 | INFO | MemInfo: { | 20260505 19:46:14.032 | INFO | "gpu" : " 17 GB", | 20260505 19:46:14.032 | INFO | "huge" : "", | 20260505 19:46:14.032 | INFO | "huge_shared" : " 27 GB", | 20260505 19:46:14.032 | INFO | "malloc" : " 14 GB", | 20260505 19:46:14.032 | INFO | "pinned" : " 29 GB" | 20260505 19:46:14.032 | INFO | } | 20260505 19:46:14.033 | INFO | Perf report for the last 5.14552 seconds: | 20260505 19:46:14.033 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:46:14.033 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:14.033 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:14.033 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:14.033 | INFO | ANALYSIS : 62.4%, 3.0, 97.0ms, 0 | 20260505 19:46:14.033 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:14.033 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:14.033 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 19:46:14.033 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:14.033 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:14.033 | INFO | Uploaded 36GiB to GPU in 2.21029s (16.2874GiB/s) | 20260505 19:46:14.033 | INFO | Downloaded 34GiB to GPU in 2.03331s (16.7215GiB/s) | 20260505 19:46:19.166 | INFO | Analyzing chunk frames = [5826560,5827072) | 20260505 19:46:19.166 | INFO | MemInfo: { | 20260505 19:46:19.166 | INFO | "gpu" : " 17 GB", | 20260505 19:46:19.166 | INFO | "huge" : "", | 20260505 19:46:19.166 | INFO | "huge_shared" : " 27 GB", | 20260505 19:46:19.166 | INFO | "malloc" : " 14 GB", | 20260505 19:46:19.166 | INFO | "pinned" : " 29 GB" | 20260505 19:46:19.166 | INFO | } | 20260505 19:46:19.166 | INFO | Perf report for the last 5.13375 seconds: | 20260505 19:46:19.166 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:46:19.166 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:19.166 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:19.166 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:19.166 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260505 19:46:19.166 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:19.166 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:19.166 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:46:19.166 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:19.166 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:19.167 | INFO | Uploaded 36GiB to GPU in 2.13329s (16.8753GiB/s) | 20260505 19:46:19.167 | INFO | Downloaded 34GiB to GPU in 2.06366s (16.4756GiB/s) | 20260505 19:46:24.302 | INFO | Analyzing chunk frames = [5827072,5827584) | 20260505 19:46:24.302 | INFO | MemInfo: { | 20260505 19:46:24.302 | INFO | "gpu" : " 17 GB", | 20260505 19:46:24.302 | INFO | "huge" : "", | 20260505 19:46:24.302 | INFO | "huge_shared" : " 27 GB", | 20260505 19:46:24.302 | INFO | "malloc" : " 14 GB", | 20260505 19:46:24.302 | INFO | "pinned" : " 29 GB" | 20260505 19:46:24.302 | INFO | } | 20260505 19:46:24.303 | INFO | Perf report for the last 5.13622 seconds: | 20260505 19:46:24.303 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:46:24.303 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:24.303 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:24.303 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:24.303 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 19:46:24.303 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:24.303 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:24.303 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:46:24.303 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:24.303 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:24.303 | INFO | Uploaded 36GiB to GPU in 2.14348s (16.7951GiB/s) | 20260505 19:46:24.303 | INFO | Downloaded 34GiB to GPU in 2.05653s (16.5327GiB/s) | 20260505 19:46:29.450 | INFO | Analyzing chunk frames = [5827584,5828096) | 20260505 19:46:29.450 | INFO | MemInfo: { | 20260505 19:46:29.450 | INFO | "gpu" : " 17 GB", | 20260505 19:46:29.450 | INFO | "huge" : "", | 20260505 19:46:29.450 | INFO | "huge_shared" : " 27 GB", | 20260505 19:46:29.450 | INFO | "malloc" : " 14 GB", | 20260505 19:46:29.450 | INFO | "pinned" : " 29 GB" | 20260505 19:46:29.450 | INFO | } | 20260505 19:46:29.450 | INFO | Perf report for the last 5.14787 seconds: | 20260505 19:46:29.450 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:46:29.450 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:29.450 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:29.450 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:29.450 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 19:46:29.450 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:29.450 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:29.450 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 19:46:29.450 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:29.451 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:29.451 | INFO | Uploaded 36GiB to GPU in 2.13711s (16.8451GiB/s) | 20260505 19:46:29.451 | INFO | Downloaded 34GiB to GPU in 2.13191s (15.9481GiB/s) | 20260505 19:46:34.588 | INFO | Analyzing chunk frames = [5828096,5828608) | 20260505 19:46:34.588 | INFO | MemInfo: { | 20260505 19:46:34.588 | INFO | "gpu" : " 17 GB", | 20260505 19:46:34.588 | INFO | "huge" : "", | 20260505 19:46:34.588 | INFO | "huge_shared" : " 27 GB", | 20260505 19:46:34.588 | INFO | "malloc" : " 14 GB", | 20260505 19:46:34.588 | INFO | "pinned" : " 29 GB" | 20260505 19:46:34.588 | INFO | } | 20260505 19:46:34.588 | INFO | Perf report for the last 5.13742 seconds: | 20260505 19:46:34.588 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:46:34.588 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:34.588 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:34.588 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:34.588 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260505 19:46:34.588 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:34.588 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:34.588 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:46:34.588 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:34.588 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:34.588 | INFO | Uploaded 36GiB to GPU in 2.11001s (17.0615GiB/s) | 20260505 19:46:34.588 | INFO | Downloaded 34GiB to GPU in 1.99356s (17.0549GiB/s) | 20260505 19:46:34.722 | INFO | Starting a round of Baz IO | 20260505 19:46:37.887 | INFO | Populating RTMetrics report | 20260505 19:46:37.888 | INFO | RTMetrics report populated | 20260505 19:46:37.888 | INFO | Opening tmp RTMetrics json file | 20260505 19:46:37.888 | INFO | Serializing RTMetrics to json | 20260505 19:46:37.890 | INFO | Serialization done, swapping tmp file now... | 20260505 19:46:37.890 | INFO | RTMetrics done | 20260505 19:46:37.978 | INFO | Finished a round of Baz IO: Wrote 2.036 GiB in 3.255 seconds | 20260505 19:46:37.978 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.74% / 0.00% / 78.26% | 20260505 19:46:37.978 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.195746% / 0% / 0.000103315% | 20260505 19:46:37.978 | INFO | Baz IO overall: | 20260505 19:46:37.978 | INFO | BytesWritten : 3408790678728 | 20260505 19:46:37.978 | INFO | NumEvents : 1612592727584 | 20260505 19:46:39.718 | INFO | Analyzing chunk frames = [5828608,5829120) | 20260505 19:46:39.718 | INFO | MemInfo: { | 20260505 19:46:39.718 | INFO | "gpu" : " 17 GB", | 20260505 19:46:39.718 | INFO | "huge" : "", | 20260505 19:46:39.718 | INFO | "huge_shared" : " 27 GB", | 20260505 19:46:39.718 | INFO | "malloc" : " 14 GB", | 20260505 19:46:39.718 | INFO | "pinned" : " 29 GB" | 20260505 19:46:39.718 | INFO | } | 20260505 19:46:39.718 | INFO | Perf report for the last 5.13001 seconds: | 20260505 19:46:39.718 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:46:39.718 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:39.718 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:39.718 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:39.718 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260505 19:46:39.718 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:39.718 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260505 19:46:39.718 | INFO | PRE_HQ : 60.2%, 1.8, 55.3ms, 0 | 20260505 19:46:39.718 | INFO | BAZWRITER : 63.0%, 3.7, 119.4ms, 0 | 20260505 19:46:39.718 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 19:46:39.718 | INFO | Uploaded 36GiB to GPU in 2.18054s (16.5097GiB/s) | 20260505 19:46:39.718 | INFO | Downloaded 38GiB to GPU in 2.25493s (16.852GiB/s) | 20260505 19:46:44.860 | INFO | Analyzing chunk frames = [5829120,5829632) | 20260505 19:46:44.860 | INFO | MemInfo: { | 20260505 19:46:44.860 | INFO | "gpu" : " 17 GB", | 20260505 19:46:44.861 | INFO | "huge" : "", | 20260505 19:46:44.861 | INFO | "huge_shared" : " 27 GB", | 20260505 19:46:44.861 | INFO | "malloc" : " 14 GB", | 20260505 19:46:44.861 | INFO | "pinned" : " 29 GB" | 20260505 19:46:44.861 | INFO | } | 20260505 19:46:44.861 | INFO | Perf report for the last 5.14276 seconds: | 20260505 19:46:44.861 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:46:44.861 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:44.861 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:44.861 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:44.861 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260505 19:46:44.861 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:44.861 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:44.861 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 19:46:44.861 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:44.861 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:44.861 | INFO | Uploaded 36GiB to GPU in 2.13975s (16.8244GiB/s) | 20260505 19:46:44.861 | INFO | Downloaded 34GiB to GPU in 2.04029s (16.6643GiB/s) | 20260505 19:46:49.999 | INFO | Analyzing chunk frames = [5829632,5830144) | 20260505 19:46:49.999 | INFO | MemInfo: { | 20260505 19:46:49.999 | INFO | "gpu" : " 17 GB", | 20260505 19:46:49.999 | INFO | "huge" : "", | 20260505 19:46:49.999 | INFO | "huge_shared" : " 27 GB", | 20260505 19:46:49.999 | INFO | "malloc" : " 14 GB", | 20260505 19:46:49.999 | INFO | "pinned" : " 29 GB" | 20260505 19:46:49.999 | INFO | } | 20260505 19:46:49.999 | INFO | Perf report for the last 5.13846 seconds: | 20260505 19:46:49.999 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:46:49.999 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:49.999 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:49.999 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:49.999 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 19:46:49.999 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:49.999 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:49.999 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:46:49.999 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:49.999 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:49.999 | INFO | Uploaded 36GiB to GPU in 2.21286s (16.2686GiB/s) | 20260505 19:46:49.999 | INFO | Downloaded 34GiB to GPU in 2.06684s (16.4503GiB/s) | 20260505 19:46:55.141 | INFO | Analyzing chunk frames = [5830144,5830656) | 20260505 19:46:55.141 | INFO | MemInfo: { | 20260505 19:46:55.141 | INFO | "gpu" : " 17 GB", | 20260505 19:46:55.141 | INFO | "huge" : "", | 20260505 19:46:55.141 | INFO | "huge_shared" : " 27 GB", | 20260505 19:46:55.141 | INFO | "malloc" : " 14 GB", | 20260505 19:46:55.141 | INFO | "pinned" : " 29 GB" | 20260505 19:46:55.141 | INFO | } | 20260505 19:46:55.141 | INFO | Perf report for the last 5.142 seconds: | 20260505 19:46:55.141 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:46:55.141 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:55.141 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:55.141 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:55.141 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260505 19:46:55.141 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:55.141 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:55.141 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:46:55.141 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:55.141 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:46:55.141 | INFO | Uploaded 36GiB to GPU in 2.19427s (16.4063GiB/s) | 20260505 19:46:55.142 | INFO | Downloaded 34GiB to GPU in 2.06998s (16.4253GiB/s) | 20260505 19:47:00.279 | INFO | Analyzing chunk frames = [5830656,5831168) | 20260505 19:47:00.279 | INFO | MemInfo: { | 20260505 19:47:00.279 | INFO | "gpu" : " 17 GB", | 20260505 19:47:00.279 | INFO | "huge" : "", | 20260505 19:47:00.279 | INFO | "huge_shared" : " 27 GB", | 20260505 19:47:00.279 | INFO | "malloc" : " 14 GB", | 20260505 19:47:00.279 | INFO | "pinned" : " 29 GB" | 20260505 19:47:00.279 | INFO | } | 20260505 19:47:00.280 | INFO | Perf report for the last 5.13846 seconds: | 20260505 19:47:00.280 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:47:00.280 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:00.280 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:00.280 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:00.280 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 19:47:00.280 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:00.280 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:00.280 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:47:00.280 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:00.280 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:00.280 | INFO | Uploaded 36GiB to GPU in 2.13497s (16.862GiB/s) | 20260505 19:47:00.280 | INFO | Downloaded 34GiB to GPU in 2.06018s (16.5034GiB/s) | 20260505 19:47:05.419 | INFO | Analyzing chunk frames = [5831168,5831680) | 20260505 19:47:05.419 | INFO | MemInfo: { | 20260505 19:47:05.419 | INFO | "gpu" : " 17 GB", | 20260505 19:47:05.419 | INFO | "huge" : "", | 20260505 19:47:05.419 | INFO | "huge_shared" : " 27 GB", | 20260505 19:47:05.419 | INFO | "malloc" : " 14 GB", | 20260505 19:47:05.419 | INFO | "pinned" : " 29 GB" | 20260505 19:47:05.419 | INFO | } | 20260505 19:47:05.419 | INFO | Perf report for the last 5.13968 seconds: | 20260505 19:47:05.419 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:47:05.419 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:05.419 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:05.419 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:05.419 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 19:47:05.419 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:05.419 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:05.419 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:47:05.419 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:05.419 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:05.420 | INFO | Uploaded 36GiB to GPU in 2.15947s (16.6707GiB/s) | 20260505 19:47:05.420 | INFO | Downloaded 34GiB to GPU in 2.06276s (16.4828GiB/s) | 20260505 19:47:10.551 | INFO | Analyzing chunk frames = [5831680,5832192) | 20260505 19:47:10.551 | INFO | MemInfo: { | 20260505 19:47:10.551 | INFO | "gpu" : " 17 GB", | 20260505 19:47:10.551 | INFO | "huge" : "", | 20260505 19:47:10.551 | INFO | "huge_shared" : " 27 GB", | 20260505 19:47:10.552 | INFO | "malloc" : " 14 GB", | 20260505 19:47:10.552 | INFO | "pinned" : " 29 GB" | 20260505 19:47:10.552 | INFO | } | 20260505 19:47:10.552 | INFO | Perf report for the last 5.13233 seconds: | 20260505 19:47:10.552 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:47:10.552 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:10.552 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:10.552 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:10.552 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 19:47:10.552 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:10.552 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:10.552 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 19:47:10.552 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:10.552 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:10.552 | INFO | Uploaded 36GiB to GPU in 2.12406s (16.9487GiB/s) | 20260505 19:47:10.552 | INFO | Downloaded 34GiB to GPU in 2.13448s (15.9289GiB/s) | 20260505 19:47:15.700 | INFO | Analyzing chunk frames = [5832192,5832704) | 20260505 19:47:15.700 | INFO | MemInfo: { | 20260505 19:47:15.700 | INFO | "gpu" : " 17 GB", | 20260505 19:47:15.700 | INFO | "huge" : "", | 20260505 19:47:15.700 | INFO | "huge_shared" : " 27 GB", | 20260505 19:47:15.700 | INFO | "malloc" : " 14 GB", | 20260505 19:47:15.700 | INFO | "pinned" : " 29 GB" | 20260505 19:47:15.700 | INFO | } | 20260505 19:47:15.700 | INFO | Perf report for the last 5.14876 seconds: | 20260505 19:47:15.700 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:47:15.700 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:15.700 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:15.700 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:15.700 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 19:47:15.700 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:15.700 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:15.700 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:47:15.700 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:15.700 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:15.701 | INFO | Uploaded 36GiB to GPU in 2.20424s (16.3322GiB/s) | 20260505 19:47:15.701 | INFO | Downloaded 34GiB to GPU in 2.04078s (16.6603GiB/s) | 20260505 19:47:15.701 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 19:47:15.701 | INFO | Total mem usage: 15.3 GiB | 20260505 19:47:15.701 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 19:47:15.701 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.92%) | 20260505 19:47:15.701 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 19:47:15.701 | INFO | High Water Breakdown: | 20260505 19:47:15.701 | INFO | 60.9% : 8.57 GiB ( 90.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 19:47:15.701 | INFO | 38.9% : 5.48 GiB ( 82.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 19:47:15.701 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 19:47:15.701 | INFO | ------------------------------------------ | 20260505 19:47:15.701 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:47:15.701 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 19:47:15.701 | INFO | Total mem usage: 26.5 GiB | 20260505 19:47:15.701 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 19:47:15.701 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 19:47:15.701 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 19:47:15.701 | INFO | High Water Breakdown: | 20260505 19:47:15.701 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:47:15.701 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:47:15.701 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:47:15.701 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:47:15.701 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:47:15.701 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:47:15.701 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 19:47:15.701 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 19:47:15.701 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:47:15.701 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 19:47:15.701 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 19:47:15.701 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 19:47:15.701 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 19:47:15.701 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 19:47:15.701 | INFO | ------------------------------------------ | 20260505 19:47:15.701 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:47:15.701 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 19:47:15.701 | INFO | Total mem usage: 25.6 GiB | 20260505 19:47:15.701 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 19:47:15.701 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 19:47:15.701 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 19:47:15.701 | INFO | High Water Breakdown: | 20260505 19:47:15.701 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 19:47:15.701 | INFO | ------------------------------------------ | 20260505 19:47:15.701 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:47:15.701 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 19:47:15.701 | INFO | Total mem usage: 15.7 GiB | 20260505 19:47:15.701 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 19:47:15.701 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 19:47:15.701 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 19:47:15.701 | INFO | High Water Breakdown: | 20260505 19:47:15.701 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 19:47:15.701 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 19:47:15.701 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 19:47:15.701 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 19:47:15.701 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 19:47:15.701 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 19:47:15.701 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 19:47:15.701 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 19:47:15.701 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 19:47:15.701 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:47:15.701 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 19:47:15.702 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:47:15.702 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 19:47:15.702 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 19:47:15.702 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 19:47:15.702 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 19:47:15.702 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:47:15.702 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 19:47:15.702 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:47:15.702 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:47:15.702 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:47:15.702 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 19:47:15.702 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 19:47:15.702 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:47:15.702 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 19:47:15.702 | INFO | ------------------------------------------ | 20260505 19:47:15.702 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:47:15.702 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 19:47:15.702 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:47:15.702 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:47:15.702 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.44e+01, 1.34e+01 / 2.10e+00 | 20260505 19:47:15.702 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.04e-01, 1.12e-02 / 3.27e-03 | 20260505 19:47:15.702 | INFO | PerfReport: 11.80%, Baseline -- 3.71e+00 / 4.57e+00, 5.62e-01 / 8.52e-02 | 20260505 19:47:15.702 | INFO | PerfReport: 3.26%, Histogram -- 1.03e+00 / 1.54e+00, 1.87e-01 / 6.85e-02 | 20260505 19:47:15.702 | INFO | PerfReport: 15.99%, DME -- 4.02e+01 / 4.43e+01, 4.15e+00 / 2.17e+00 | 20260505 19:47:15.702 | INFO | PerfReport: 56.94%, FrameLabeling -- 1.79e+01 / 1.92e+01, 1.41e+00 / 9.07e-01 | 20260505 19:47:15.702 | INFO | PerfReport: 7.29%, PulseAccumulating -- 2.29e+00 / 2.87e+00, 4.39e-01 / 2.03e-01 | 20260505 19:47:15.702 | INFO | PerfReport: 3.28%, Metrics -- 1.03e+00 / 1.35e+00, 2.83e-01 / 1.63e-01 | 20260505 19:47:15.702 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.58e-01 / 4.32e-01, 6.34e-02 / 3.94e-02 | 20260505 19:47:15.702 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 19:47:15.702 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:47:15.702 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:47:15.702 | INFO | PerfReport: Full Execution -- 9.96e+01 / -nan, 2.54e+01 / 0.00e+00 | 20260505 19:47:15.702 | INFO | PerfReport: 22.57%, Upload -- 2.25e+01 / 0.00e+00, 7.86e+00 / 0.00e+00 | 20260505 19:47:15.702 | INFO | PerfReport: 21.77%, Download -- 2.17e+01 / 0.00e+00, 7.71e+00 / 0.00e+00 | 20260505 19:47:15.702 | INFO | PerfReport: 31.57%, Compute -- 3.14e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 19:47:15.702 | INFO | PerfReport: 2.40%, LockUpload -- 2.39e+00 / 0.00e+00, 8.08e+00 / 0.00e+00 | 20260505 19:47:15.702 | INFO | PerfReport: 1.34%, LockDownload -- 1.33e+00 / 0.00e+00, 2.82e+00 / 0.00e+00 | 20260505 19:47:15.702 | INFO | PerfReport: 20.34%, LockCompute -- 2.02e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260505 19:47:15.702 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.21e-02 / 0.00e+00 | 20260505 19:47:15.834 | INFO | Starting a round of Baz IO | 20260505 19:47:19.001 | INFO | Populating RTMetrics report | 20260505 19:47:19.002 | INFO | RTMetrics report populated | 20260505 19:47:19.002 | INFO | Opening tmp RTMetrics json file | 20260505 19:47:19.003 | INFO | Serializing RTMetrics to json | 20260505 19:47:19.004 | INFO | Serialization done, swapping tmp file now... | 20260505 19:47:19.004 | INFO | RTMetrics done | 20260505 19:47:19.088 | INFO | Finished a round of Baz IO: Wrote 2.036 GiB in 3.254 seconds | 20260505 19:47:19.088 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.74% / 0.00% / 78.26% | 20260505 19:47:19.088 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.195626% / 7.94729e-06% / 0.000115236% | 20260505 19:47:19.088 | INFO | Baz IO overall: | 20260505 19:47:19.088 | INFO | BytesWritten : 3410976375475 | 20260505 19:47:19.088 | INFO | NumEvents : 1613607020826 | 20260505 19:47:20.839 | INFO | Analyzing chunk frames = [5832704,5833216) | 20260505 19:47:20.839 | INFO | MemInfo: { | 20260505 19:47:20.839 | INFO | "gpu" : " 17 GB", | 20260505 19:47:20.839 | INFO | "huge" : "", | 20260505 19:47:20.839 | INFO | "huge_shared" : " 27 GB", | 20260505 19:47:20.839 | INFO | "malloc" : " 14 GB", | 20260505 19:47:20.839 | INFO | "pinned" : " 29 GB" | 20260505 19:47:20.839 | INFO | } | 20260505 19:47:20.839 | INFO | Perf report for the last 5.13833 seconds: | 20260505 19:47:20.839 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:47:20.839 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:20.839 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:20.839 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:20.839 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 19:47:20.839 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:20.839 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260505 19:47:20.839 | INFO | PRE_HQ : 60.1%, 1.8, 55.2ms, 0 | 20260505 19:47:20.839 | INFO | BAZWRITER : 62.9%, 3.7, 120.3ms, 0 | 20260505 19:47:20.839 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 19:47:20.839 | INFO | Uploaded 36GiB to GPU in 2.18292s (16.4917GiB/s) | 20260505 19:47:20.839 | INFO | Downloaded 38GiB to GPU in 2.25935s (16.819GiB/s) | 20260505 19:47:25.980 | INFO | Analyzing chunk frames = [5833216,5833728) | 20260505 19:47:25.980 | INFO | MemInfo: { | 20260505 19:47:25.980 | INFO | "gpu" : " 17 GB", | 20260505 19:47:25.980 | INFO | "huge" : "", | 20260505 19:47:25.980 | INFO | "huge_shared" : " 27 GB", | 20260505 19:47:25.980 | INFO | "malloc" : " 14 GB", | 20260505 19:47:25.980 | INFO | "pinned" : " 29 GB" | 20260505 19:47:25.980 | INFO | } | 20260505 19:47:25.980 | INFO | Perf report for the last 5.14113 seconds: | 20260505 19:47:25.980 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:47:25.980 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:25.980 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:25.980 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:25.980 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260505 19:47:25.980 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:25.980 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:25.980 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 19:47:25.980 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:25.980 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:25.980 | INFO | Uploaded 36GiB to GPU in 2.14111s (16.8137GiB/s) | 20260505 19:47:25.980 | INFO | Downloaded 34GiB to GPU in 2.042s (16.6503GiB/s) | 20260505 19:47:31.113 | INFO | Analyzing chunk frames = [5833728,5834240) | 20260505 19:47:31.113 | INFO | MemInfo: { | 20260505 19:47:31.113 | INFO | "gpu" : " 17 GB", | 20260505 19:47:31.113 | INFO | "huge" : "", | 20260505 19:47:31.113 | INFO | "huge_shared" : " 27 GB", | 20260505 19:47:31.113 | INFO | "malloc" : " 14 GB", | 20260505 19:47:31.113 | INFO | "pinned" : " 29 GB" | 20260505 19:47:31.113 | INFO | } | 20260505 19:47:31.113 | INFO | Perf report for the last 5.13362 seconds: | 20260505 19:47:31.113 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:47:31.113 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:31.113 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:31.114 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:31.114 | INFO | ANALYSIS : 62.8%, 3.0, 97.6ms, 0 | 20260505 19:47:31.114 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:31.114 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:31.114 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:47:31.114 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:31.114 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:31.114 | INFO | Uploaded 36GiB to GPU in 2.21318s (16.2662GiB/s) | 20260505 19:47:31.114 | INFO | Downloaded 34GiB to GPU in 2.06637s (16.454GiB/s) | 20260505 19:47:36.256 | INFO | Analyzing chunk frames = [5834240,5834752) | 20260505 19:47:36.256 | INFO | MemInfo: { | 20260505 19:47:36.256 | INFO | "gpu" : " 17 GB", | 20260505 19:47:36.256 | INFO | "huge" : "", | 20260505 19:47:36.256 | INFO | "huge_shared" : " 27 GB", | 20260505 19:47:36.256 | INFO | "malloc" : " 14 GB", | 20260505 19:47:36.256 | INFO | "pinned" : " 29 GB" | 20260505 19:47:36.256 | INFO | } | 20260505 19:47:36.256 | INFO | Perf report for the last 5.14294 seconds: | 20260505 19:47:36.256 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:47:36.256 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:36.256 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:36.256 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:36.256 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260505 19:47:36.256 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:36.256 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:36.257 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:47:36.257 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:36.257 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:36.257 | INFO | Uploaded 36GiB to GPU in 2.18192s (16.4992GiB/s) | 20260505 19:47:36.257 | INFO | Downloaded 34GiB to GPU in 2.06882s (16.4345GiB/s) | 20260505 19:47:41.400 | INFO | Analyzing chunk frames = [5834752,5835264) | 20260505 19:47:41.400 | INFO | MemInfo: { | 20260505 19:47:41.400 | INFO | "gpu" : " 17 GB", | 20260505 19:47:41.400 | INFO | "huge" : "", | 20260505 19:47:41.400 | INFO | "huge_shared" : " 27 GB", | 20260505 19:47:41.400 | INFO | "malloc" : " 14 GB", | 20260505 19:47:41.400 | INFO | "pinned" : " 29 GB" | 20260505 19:47:41.400 | INFO | } | 20260505 19:47:41.400 | INFO | Perf report for the last 5.14368 seconds: | 20260505 19:47:41.400 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:47:41.400 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:41.400 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:41.400 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:41.400 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260505 19:47:41.400 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:41.400 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:41.400 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:47:41.400 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:41.400 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:41.400 | INFO | Uploaded 36GiB to GPU in 2.11812s (16.9962GiB/s) | 20260505 19:47:41.401 | INFO | Downloaded 34GiB to GPU in 2.06103s (16.4966GiB/s) | 20260505 19:47:46.536 | INFO | Analyzing chunk frames = [5835264,5835776) | 20260505 19:47:46.536 | INFO | MemInfo: { | 20260505 19:47:46.536 | INFO | "gpu" : " 17 GB", | 20260505 19:47:46.536 | INFO | "huge" : "", | 20260505 19:47:46.536 | INFO | "huge_shared" : " 27 GB", | 20260505 19:47:46.536 | INFO | "malloc" : " 14 GB", | 20260505 19:47:46.536 | INFO | "pinned" : " 29 GB" | 20260505 19:47:46.536 | INFO | } | 20260505 19:47:46.536 | INFO | Perf report for the last 5.13617 seconds: | 20260505 19:47:46.536 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:47:46.536 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:46.536 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:46.536 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:46.536 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 19:47:46.536 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:46.536 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:46.536 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:47:46.536 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:46.536 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:46.537 | INFO | Uploaded 36GiB to GPU in 2.08228s (17.2887GiB/s) | 20260505 19:47:46.537 | INFO | Downloaded 34GiB to GPU in 2.04157s (16.6538GiB/s) | 20260505 19:47:51.668 | INFO | Analyzing chunk frames = [5835776,5836288) | 20260505 19:47:51.668 | INFO | MemInfo: { | 20260505 19:47:51.668 | INFO | "gpu" : " 17 GB", | 20260505 19:47:51.668 | INFO | "huge" : "", | 20260505 19:47:51.668 | INFO | "huge_shared" : " 27 GB", | 20260505 19:47:51.668 | INFO | "malloc" : " 14 GB", | 20260505 19:47:51.668 | INFO | "pinned" : " 29 GB" | 20260505 19:47:51.668 | INFO | } | 20260505 19:47:51.668 | INFO | Perf report for the last 5.13166 seconds: | 20260505 19:47:51.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:47:51.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:51.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:51.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:51.668 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 19:47:51.668 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:51.668 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:51.668 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 19:47:51.668 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:51.668 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:51.668 | INFO | Uploaded 36GiB to GPU in 2.12388s (16.9501GiB/s) | 20260505 19:47:51.668 | INFO | Downloaded 34GiB to GPU in 2.14918s (15.82GiB/s) | 20260505 19:47:56.808 | INFO | Analyzing chunk frames = [5836288,5836800) | 20260505 19:47:56.808 | INFO | MemInfo: { | 20260505 19:47:56.808 | INFO | "gpu" : " 17 GB", | 20260505 19:47:56.808 | INFO | "huge" : "", | 20260505 19:47:56.808 | INFO | "huge_shared" : " 27 GB", | 20260505 19:47:56.808 | INFO | "malloc" : " 14 GB", | 20260505 19:47:56.808 | INFO | "pinned" : " 29 GB" | 20260505 19:47:56.808 | INFO | } | 20260505 19:47:56.808 | INFO | Perf report for the last 5.14049 seconds: | 20260505 19:47:56.808 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:47:56.808 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:56.808 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:56.809 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:56.809 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 19:47:56.809 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:56.809 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:56.809 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:47:56.809 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:56.809 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:47:56.809 | INFO | Uploaded 36GiB to GPU in 2.17521s (16.5501GiB/s) | 20260505 19:47:56.809 | INFO | Downloaded 34GiB to GPU in 2.0257s (16.7844GiB/s) | 20260505 19:47:56.945 | INFO | Starting a round of Baz IO | 20260505 19:48:00.116 | INFO | Populating RTMetrics report | 20260505 19:48:00.116 | INFO | RTMetrics report populated | 20260505 19:48:00.116 | INFO | Opening tmp RTMetrics json file | 20260505 19:48:00.117 | INFO | Serializing RTMetrics to json | 20260505 19:48:00.121 | INFO | Serialization done, swapping tmp file now... | 20260505 19:48:00.125 | INFO | RTMetrics done | 20260505 19:48:00.202 | INFO | Finished a round of Baz IO: Wrote 2.036 GiB in 3.257 seconds | 20260505 19:48:00.202 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.74% / 0.00% / 78.26% | 20260505 19:48:00.202 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.195372% / 0% / 9.53674e-05% | 20260505 19:48:00.202 | INFO | Baz IO overall: | 20260505 19:48:00.202 | INFO | BytesWritten : 3413162436845 | 20260505 19:48:00.202 | INFO | NumEvents : 1614621426131 | 20260505 19:48:01.950 | INFO | Analyzing chunk frames = [5836800,5837312) | 20260505 19:48:01.950 | INFO | MemInfo: { | 20260505 19:48:01.950 | INFO | "gpu" : " 17 GB", | 20260505 19:48:01.950 | INFO | "huge" : "", | 20260505 19:48:01.950 | INFO | "huge_shared" : " 27 GB", | 20260505 19:48:01.950 | INFO | "malloc" : " 14 GB", | 20260505 19:48:01.950 | INFO | "pinned" : " 29 GB" | 20260505 19:48:01.950 | INFO | } | 20260505 19:48:01.950 | INFO | Perf report for the last 5.1419 seconds: | 20260505 19:48:01.950 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:48:01.950 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:01.950 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:01.950 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:01.950 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 19:48:01.950 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:01.950 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260505 19:48:01.950 | INFO | PRE_HQ : 60.2%, 1.8, 55.6ms, 0 | 20260505 19:48:01.950 | INFO | BAZWRITER : 62.8%, 3.7, 119.2ms, 0 | 20260505 19:48:01.950 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 19:48:01.951 | INFO | Uploaded 36GiB to GPU in 2.14002s (16.8223GiB/s) | 20260505 19:48:01.951 | INFO | Downloaded 38GiB to GPU in 2.24788s (16.9048GiB/s) | 20260505 19:48:07.090 | INFO | Analyzing chunk frames = [5837312,5837824) | 20260505 19:48:07.090 | INFO | MemInfo: { | 20260505 19:48:07.090 | INFO | "gpu" : " 17 GB", | 20260505 19:48:07.090 | INFO | "huge" : "", | 20260505 19:48:07.091 | INFO | "huge_shared" : " 27 GB", | 20260505 19:48:07.091 | INFO | "malloc" : " 14 GB", | 20260505 19:48:07.091 | INFO | "pinned" : " 29 GB" | 20260505 19:48:07.091 | INFO | } | 20260505 19:48:07.091 | INFO | Perf report for the last 5.14019 seconds: | 20260505 19:48:07.091 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:48:07.091 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:07.091 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:07.091 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:07.091 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260505 19:48:07.091 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:07.091 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:07.091 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 19:48:07.091 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:07.091 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:07.091 | INFO | Uploaded 36GiB to GPU in 2.14544s (16.7798GiB/s) | 20260505 19:48:07.091 | INFO | Downloaded 34GiB to GPU in 2.05082s (16.5787GiB/s) | 20260505 19:48:12.230 | INFO | Analyzing chunk frames = [5837824,5838336) | 20260505 19:48:12.230 | INFO | MemInfo: { | 20260505 19:48:12.230 | INFO | "gpu" : " 17 GB", | 20260505 19:48:12.230 | INFO | "huge" : "", | 20260505 19:48:12.230 | INFO | "huge_shared" : " 27 GB", | 20260505 19:48:12.230 | INFO | "malloc" : " 14 GB", | 20260505 19:48:12.230 | INFO | "pinned" : " 29 GB" | 20260505 19:48:12.230 | INFO | } | 20260505 19:48:12.230 | INFO | Perf report for the last 5.13944 seconds: | 20260505 19:48:12.230 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:48:12.230 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:12.230 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:12.230 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:12.230 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260505 19:48:12.230 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:12.230 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:12.230 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:48:12.230 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:12.230 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:12.230 | INFO | Uploaded 36GiB to GPU in 2.13585s (16.8551GiB/s) | 20260505 19:48:12.230 | INFO | Downloaded 34GiB to GPU in 2.03878s (16.6766GiB/s) | 20260505 19:48:17.367 | INFO | Analyzing chunk frames = [5838336,5838848) | 20260505 19:48:17.367 | INFO | MemInfo: { | 20260505 19:48:17.367 | INFO | "gpu" : " 17 GB", | 20260505 19:48:17.367 | INFO | "huge" : "", | 20260505 19:48:17.367 | INFO | "huge_shared" : " 27 GB", | 20260505 19:48:17.367 | INFO | "malloc" : " 14 GB", | 20260505 19:48:17.367 | INFO | "pinned" : " 29 GB" | 20260505 19:48:17.367 | INFO | } | 20260505 19:48:17.367 | INFO | Perf report for the last 5.13677 seconds: | 20260505 19:48:17.367 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:48:17.367 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:17.367 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:17.367 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:17.367 | INFO | ANALYSIS : 62.4%, 3.0, 96.8ms, 0 | 20260505 19:48:17.367 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:17.367 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:17.367 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:48:17.367 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:17.367 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:17.367 | INFO | Uploaded 36GiB to GPU in 2.19797s (16.3787GiB/s) | 20260505 19:48:17.367 | INFO | Downloaded 34GiB to GPU in 2.0334s (16.7208GiB/s) | 20260505 19:48:22.508 | INFO | Analyzing chunk frames = [5838848,5839360) | 20260505 19:48:22.508 | INFO | MemInfo: { | 20260505 19:48:22.508 | INFO | "gpu" : " 17 GB", | 20260505 19:48:22.508 | INFO | "huge" : "", | 20260505 19:48:22.508 | INFO | "huge_shared" : " 27 GB", | 20260505 19:48:22.508 | INFO | "malloc" : " 14 GB", | 20260505 19:48:22.508 | INFO | "pinned" : " 29 GB" | 20260505 19:48:22.508 | INFO | } | 20260505 19:48:22.508 | INFO | Perf report for the last 5.14111 seconds: | 20260505 19:48:22.508 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:48:22.508 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:22.508 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:22.508 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:22.508 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 19:48:22.508 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:22.508 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:22.508 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:48:22.508 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:22.508 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:22.508 | INFO | Uploaded 36GiB to GPU in 2.11934s (16.9864GiB/s) | 20260505 19:48:22.508 | INFO | Downloaded 34GiB to GPU in 2.06217s (16.4875GiB/s) | 20260505 19:48:27.647 | INFO | Analyzing chunk frames = [5839360,5839872) | 20260505 19:48:27.647 | INFO | MemInfo: { | 20260505 19:48:27.647 | INFO | "gpu" : " 17 GB", | 20260505 19:48:27.647 | INFO | "huge" : "", | 20260505 19:48:27.647 | INFO | "huge_shared" : " 27 GB", | 20260505 19:48:27.647 | INFO | "malloc" : " 14 GB", | 20260505 19:48:27.647 | INFO | "pinned" : " 29 GB" | 20260505 19:48:27.647 | INFO | } | 20260505 19:48:27.647 | INFO | Perf report for the last 5.13888 seconds: | 20260505 19:48:27.647 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:48:27.647 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:27.647 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:27.647 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:27.647 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 19:48:27.647 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:27.647 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:27.647 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 19:48:27.647 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:27.647 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:27.647 | INFO | Uploaded 36GiB to GPU in 2.15593s (16.6981GiB/s) | 20260505 19:48:27.647 | INFO | Downloaded 34GiB to GPU in 2.06064s (16.4997GiB/s) | 20260505 19:48:32.784 | INFO | Analyzing chunk frames = [5839872,5840384) | 20260505 19:48:32.784 | INFO | MemInfo: { | 20260505 19:48:32.784 | INFO | "gpu" : " 17 GB", | 20260505 19:48:32.784 | INFO | "huge" : "", | 20260505 19:48:32.784 | INFO | "huge_shared" : " 27 GB", | 20260505 19:48:32.784 | INFO | "malloc" : " 14 GB", | 20260505 19:48:32.784 | INFO | "pinned" : " 29 GB" | 20260505 19:48:32.784 | INFO | } | 20260505 19:48:32.784 | INFO | Perf report for the last 5.13702 seconds: | 20260505 19:48:32.784 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:48:32.784 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:32.784 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:32.784 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:32.784 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 19:48:32.784 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:32.784 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:32.784 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 19:48:32.784 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:32.784 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:32.784 | INFO | Uploaded 36GiB to GPU in 2.12201s (16.9651GiB/s) | 20260505 19:48:32.784 | INFO | Downloaded 34GiB to GPU in 2.13758s (15.9059GiB/s) | 20260505 19:48:37.924 | INFO | Analyzing chunk frames = [5840384,5840896) | 20260505 19:48:37.924 | INFO | MemInfo: { | 20260505 19:48:37.924 | INFO | "gpu" : " 17 GB", | 20260505 19:48:37.924 | INFO | "huge" : "", | 20260505 19:48:37.924 | INFO | "huge_shared" : " 27 GB", | 20260505 19:48:37.924 | INFO | "malloc" : " 14 GB", | 20260505 19:48:37.925 | INFO | "pinned" : " 29 GB" | 20260505 19:48:37.925 | INFO | } | 20260505 19:48:37.925 | INFO | Perf report for the last 5.14071 seconds: | 20260505 19:48:37.925 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:48:37.925 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:37.925 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:37.925 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:37.925 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 19:48:37.925 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:37.925 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:37.925 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 19:48:37.925 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:37.925 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:37.925 | INFO | Uploaded 36GiB to GPU in 2.20293s (16.3419GiB/s) | 20260505 19:48:37.925 | INFO | Downloaded 34GiB to GPU in 2.03784s (16.6843GiB/s) | 20260505 19:48:37.925 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 19:48:37.925 | INFO | Total mem usage: 15.3 GiB | 20260505 19:48:37.925 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 19:48:37.925 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.38%) | 20260505 19:48:37.925 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 19:48:37.925 | INFO | High Water Breakdown: | 20260505 19:48:37.925 | INFO | 61% : 8.63 GiB ( 90.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 19:48:37.925 | INFO | 38.9% : 5.5 GiB ( 82.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 19:48:37.925 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 19:48:37.925 | INFO | ------------------------------------------ | 20260505 19:48:37.925 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:48:37.925 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 19:48:37.925 | INFO | Total mem usage: 26.5 GiB | 20260505 19:48:37.925 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 19:48:37.925 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 19:48:37.925 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 19:48:37.925 | INFO | High Water Breakdown: | 20260505 19:48:37.925 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:48:37.925 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:48:37.925 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:48:37.925 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:48:37.925 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:48:37.925 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:48:37.925 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 19:48:37.925 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 19:48:37.925 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:48:37.925 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 19:48:37.926 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 19:48:37.926 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 19:48:37.926 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 19:48:37.926 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 19:48:37.926 | INFO | ------------------------------------------ | 20260505 19:48:37.926 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:48:37.926 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 19:48:37.926 | INFO | Total mem usage: 25.6 GiB | 20260505 19:48:37.926 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 19:48:37.926 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 19:48:37.926 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 19:48:37.926 | INFO | High Water Breakdown: | 20260505 19:48:37.926 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 19:48:37.926 | INFO | ------------------------------------------ | 20260505 19:48:37.926 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:48:37.926 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 19:48:37.926 | INFO | Total mem usage: 15.7 GiB | 20260505 19:48:37.926 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 19:48:37.926 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 19:48:37.926 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 19:48:37.926 | INFO | High Water Breakdown: | 20260505 19:48:37.926 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 19:48:37.926 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 19:48:37.926 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 19:48:37.926 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 19:48:37.926 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 19:48:37.926 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 19:48:37.926 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 19:48:37.926 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 19:48:37.926 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 19:48:37.926 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:48:37.926 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 19:48:37.926 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:48:37.926 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 19:48:37.926 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 19:48:37.926 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 19:48:37.926 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 19:48:37.926 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:48:37.926 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 19:48:37.926 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:48:37.926 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:48:37.926 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:48:37.926 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 19:48:37.926 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 19:48:37.926 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:48:37.926 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 19:48:37.926 | INFO | ------------------------------------------ | 20260505 19:48:37.926 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:48:37.926 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 19:48:37.926 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:48:37.926 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:48:37.926 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.44e+01, 1.36e+01 / 1.93e+00 | 20260505 19:48:37.926 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.03e-01, 1.02e-02 / 4.59e-03 | 20260505 19:48:37.926 | INFO | PerfReport: 11.79%, Baseline -- 3.71e+00 / 4.53e+00, 5.63e-01 / 1.48e-01 | 20260505 19:48:37.926 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.53e+00, 1.92e-01 / 9.01e-02 | 20260505 19:48:37.926 | INFO | PerfReport: 15.97%, DME -- 4.02e+01 / 4.43e+01, 4.04e+00 / 1.93e+00 | 20260505 19:48:37.926 | INFO | PerfReport: 56.98%, FrameLabeling -- 1.79e+01 / 1.92e+01, 1.45e+00 / 9.52e-01 | 20260505 19:48:37.926 | INFO | PerfReport: 7.28%, PulseAccumulating -- 2.29e+00 / 2.87e+00, 4.38e-01 / 1.85e-01 | 20260505 19:48:37.926 | INFO | PerfReport: 3.28%, Metrics -- 1.03e+00 / 1.37e+00, 2.84e-01 / 1.18e-01 | 20260505 19:48:37.926 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.58e-01 / 4.26e-01, 6.34e-02 / 3.21e-02 | 20260505 19:48:37.926 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 19:48:37.926 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:48:37.926 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:48:37.926 | INFO | PerfReport: Full Execution -- 9.96e+01 / -nan, 2.56e+01 / 0.00e+00 | 20260505 19:48:37.926 | INFO | PerfReport: 22.52%, Upload -- 2.24e+01 / 0.00e+00, 7.74e+00 / 0.00e+00 | 20260505 19:48:37.926 | INFO | PerfReport: 21.83%, Download -- 2.17e+01 / 0.00e+00, 7.74e+00 / 0.00e+00 | 20260505 19:48:37.926 | INFO | PerfReport: 31.62%, Compute -- 3.15e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 19:48:37.926 | INFO | PerfReport: 2.28%, LockUpload -- 2.27e+00 / 0.00e+00, 7.95e+00 / 0.00e+00 | 20260505 19:48:37.926 | INFO | PerfReport: 1.42%, LockDownload -- 1.41e+00 / 0.00e+00, 2.99e+00 / 0.00e+00 | 20260505 19:48:37.926 | INFO | PerfReport: 20.32%, LockCompute -- 2.02e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 19:48:37.926 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.78e-02 / 0.00e+00 | 20260505 19:48:38.057 | INFO | Starting a round of Baz IO | 20260505 19:48:41.239 | INFO | Populating RTMetrics report | 20260505 19:48:41.239 | INFO | RTMetrics report populated | 20260505 19:48:41.239 | INFO | Opening tmp RTMetrics json file | 20260505 19:48:41.243 | INFO | Serializing RTMetrics to json | 20260505 19:48:41.245 | INFO | Serialization done, swapping tmp file now... | 20260505 19:48:41.245 | INFO | RTMetrics done | 20260505 19:48:41.321 | INFO | Finished a round of Baz IO: Wrote 2.036 GiB in 3.263 seconds | 20260505 19:48:41.321 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.74% / 0.00% / 78.26% | 20260505 19:48:41.321 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.195348% / 0% / 0.000107288% | 20260505 19:48:41.321 | INFO | Baz IO overall: | 20260505 19:48:41.321 | INFO | BytesWritten : 3415348619952 | 20260505 19:48:41.321 | INFO | NumEvents : 1615636005552 | 20260505 19:48:43.063 | INFO | Analyzing chunk frames = [5840896,5841408) | 20260505 19:48:43.063 | INFO | MemInfo: { | 20260505 19:48:43.063 | INFO | "gpu" : " 17 GB", | 20260505 19:48:43.063 | INFO | "huge" : "", | 20260505 19:48:43.063 | INFO | "huge_shared" : " 27 GB", | 20260505 19:48:43.063 | INFO | "malloc" : " 14 GB", | 20260505 19:48:43.063 | INFO | "pinned" : " 29 GB" | 20260505 19:48:43.063 | INFO | } | 20260505 19:48:43.063 | INFO | Perf report for the last 5.13869 seconds: | 20260505 19:48:43.063 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:48:43.063 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:43.063 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:43.063 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:43.063 | INFO | ANALYSIS : 63.4%, 3.0, 98.8ms, 0 | 20260505 19:48:43.063 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:43.063 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260505 19:48:43.063 | INFO | PRE_HQ : 60.3%, 1.8, 55.4ms, 0 | 20260505 19:48:43.063 | INFO | BAZWRITER : 63.0%, 3.7, 120.3ms, 0 | 20260505 19:48:43.063 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 19:48:43.064 | INFO | Uploaded 36GiB to GPU in 2.17068s (16.5847GiB/s) | 20260505 19:48:43.064 | INFO | Downloaded 38GiB to GPU in 2.24514s (16.9254GiB/s) | 20260505 19:48:48.211 | INFO | Analyzing chunk frames = [5841408,5841920) | 20260505 19:48:48.211 | INFO | MemInfo: { | 20260505 19:48:48.211 | INFO | "gpu" : " 17 GB", | 20260505 19:48:48.211 | INFO | "huge" : "", | 20260505 19:48:48.211 | INFO | "huge_shared" : " 27 GB", | 20260505 19:48:48.211 | INFO | "malloc" : " 14 GB", | 20260505 19:48:48.211 | INFO | "pinned" : " 29 GB" | 20260505 19:48:48.211 | INFO | } | 20260505 19:48:48.211 | INFO | Perf report for the last 5.14795 seconds: | 20260505 19:48:48.211 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:48:48.211 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:48.211 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:48.211 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:48.211 | INFO | ANALYSIS : 62.5%, 3.0, 97.8ms, 0 | 20260505 19:48:48.211 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:48.211 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:48.211 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:48:48.211 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:48.211 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:48.212 | INFO | Uploaded 36GiB to GPU in 2.14276s (16.8008GiB/s) | 20260505 19:48:48.212 | INFO | Downloaded 34GiB to GPU in 2.04599s (16.6179GiB/s) | 20260505 19:48:53.354 | INFO | Analyzing chunk frames = [5841920,5842432) | 20260505 19:48:53.354 | INFO | MemInfo: { | 20260505 19:48:53.354 | INFO | "gpu" : " 17 GB", | 20260505 19:48:53.354 | INFO | "huge" : "", | 20260505 19:48:53.354 | INFO | "huge_shared" : " 27 GB", | 20260505 19:48:53.354 | INFO | "malloc" : " 14 GB", | 20260505 19:48:53.354 | INFO | "pinned" : " 29 GB" | 20260505 19:48:53.354 | INFO | } | 20260505 19:48:53.354 | INFO | Perf report for the last 5.14275 seconds: | 20260505 19:48:53.354 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:48:53.354 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:53.354 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:53.354 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:53.354 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260505 19:48:53.354 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:53.354 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:53.354 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:48:53.354 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:53.354 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:53.354 | INFO | Uploaded 36GiB to GPU in 2.13355s (16.8733GiB/s) | 20260505 19:48:53.354 | INFO | Downloaded 34GiB to GPU in 2.02261s (16.8099GiB/s) | 20260505 19:48:58.483 | INFO | Analyzing chunk frames = [5842432,5842944) | 20260505 19:48:58.483 | INFO | MemInfo: { | 20260505 19:48:58.483 | INFO | "gpu" : " 17 GB", | 20260505 19:48:58.483 | INFO | "huge" : "", | 20260505 19:48:58.483 | INFO | "huge_shared" : " 27 GB", | 20260505 19:48:58.483 | INFO | "malloc" : " 14 GB", | 20260505 19:48:58.483 | INFO | "pinned" : " 29 GB" | 20260505 19:48:58.483 | INFO | } | 20260505 19:48:58.483 | INFO | Perf report for the last 5.12883 seconds: | 20260505 19:48:58.483 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:48:58.483 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:58.483 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:58.483 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:58.483 | INFO | ANALYSIS : 62.3%, 3.0, 96.6ms, 0 | 20260505 19:48:58.483 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:58.483 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:58.483 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260505 19:48:58.483 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:58.483 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:48:58.483 | INFO | Uploaded 36GiB to GPU in 2.1442s (16.7895GiB/s) | 20260505 19:48:58.483 | INFO | Downloaded 34GiB to GPU in 2.04946s (16.5898GiB/s) | 20260505 19:49:03.616 | INFO | Analyzing chunk frames = [5842944,5843456) | 20260505 19:49:03.616 | INFO | MemInfo: { | 20260505 19:49:03.616 | INFO | "gpu" : " 17 GB", | 20260505 19:49:03.616 | INFO | "huge" : "", | 20260505 19:49:03.616 | INFO | "huge_shared" : " 27 GB", | 20260505 19:49:03.616 | INFO | "malloc" : " 14 GB", | 20260505 19:49:03.616 | INFO | "pinned" : " 29 GB" | 20260505 19:49:03.616 | INFO | } | 20260505 19:49:03.616 | INFO | Perf report for the last 5.13341 seconds: | 20260505 19:49:03.616 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:49:03.616 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:03.616 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:03.616 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:03.616 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260505 19:49:03.616 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:03.616 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:03.616 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260505 19:49:03.616 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:03.616 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:03.617 | INFO | Uploaded 36GiB to GPU in 2.12668s (16.9278GiB/s) | 20260505 19:49:03.617 | INFO | Downloaded 34GiB to GPU in 2.0642s (16.4713GiB/s) | 20260505 19:49:08.764 | INFO | Analyzing chunk frames = [5843456,5843968) | 20260505 19:49:08.764 | INFO | MemInfo: { | 20260505 19:49:08.764 | INFO | "gpu" : " 17 GB", | 20260505 19:49:08.764 | INFO | "huge" : "", | 20260505 19:49:08.764 | INFO | "huge_shared" : " 27 GB", | 20260505 19:49:08.764 | INFO | "malloc" : " 14 GB", | 20260505 19:49:08.764 | INFO | "pinned" : " 29 GB" | 20260505 19:49:08.764 | INFO | } | 20260505 19:49:08.764 | INFO | Perf report for the last 5.14784 seconds: | 20260505 19:49:08.764 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:49:08.764 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:08.764 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:08.764 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:08.764 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260505 19:49:08.764 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:08.764 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:08.764 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 19:49:08.764 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:08.764 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:08.764 | INFO | Uploaded 36GiB to GPU in 2.14084s (16.8158GiB/s) | 20260505 19:49:08.764 | INFO | Downloaded 34GiB to GPU in 2.0416s (16.6536GiB/s) | 20260505 19:49:13.903 | INFO | Analyzing chunk frames = [5843968,5844480) | 20260505 19:49:13.903 | INFO | MemInfo: { | 20260505 19:49:13.903 | INFO | "gpu" : " 17 GB", | 20260505 19:49:13.903 | INFO | "huge" : "", | 20260505 19:49:13.903 | INFO | "huge_shared" : " 27 GB", | 20260505 19:49:13.903 | INFO | "malloc" : " 14 GB", | 20260505 19:49:13.903 | INFO | "pinned" : " 29 GB" | 20260505 19:49:13.903 | INFO | } | 20260505 19:49:13.903 | INFO | Perf report for the last 5.13923 seconds: | 20260505 19:49:13.903 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:49:13.903 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:13.903 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:13.903 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:13.903 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 19:49:13.903 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:13.903 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:13.903 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 19:49:13.903 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:13.903 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:13.904 | INFO | Uploaded 36GiB to GPU in 2.13289s (16.8785GiB/s) | 20260505 19:49:13.904 | INFO | Downloaded 34GiB to GPU in 2.15761s (15.7582GiB/s) | 20260505 19:49:19.045 | INFO | Analyzing chunk frames = [5844480,5844992) | 20260505 19:49:19.045 | INFO | MemInfo: { | 20260505 19:49:19.045 | INFO | "gpu" : " 17 GB", | 20260505 19:49:19.045 | INFO | "huge" : "", | 20260505 19:49:19.045 | INFO | "huge_shared" : " 27 GB", | 20260505 19:49:19.045 | INFO | "malloc" : " 14 GB", | 20260505 19:49:19.045 | INFO | "pinned" : " 29 GB" | 20260505 19:49:19.045 | INFO | } | 20260505 19:49:19.046 | INFO | Perf report for the last 5.14209 seconds: | 20260505 19:49:19.046 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:49:19.046 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:19.046 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:19.046 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:19.046 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260505 19:49:19.046 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:19.046 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:19.046 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:49:19.046 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:19.046 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:19.046 | INFO | Uploaded 36GiB to GPU in 2.1238s (16.9508GiB/s) | 20260505 19:49:19.046 | INFO | Downloaded 34GiB to GPU in 2.01406s (16.8814GiB/s) | 20260505 19:49:19.181 | INFO | Starting a round of Baz IO | 20260505 19:49:22.343 | INFO | Populating RTMetrics report | 20260505 19:49:22.343 | INFO | RTMetrics report populated | 20260505 19:49:22.343 | INFO | Opening tmp RTMetrics json file | 20260505 19:49:22.344 | INFO | Serializing RTMetrics to json | 20260505 19:49:22.345 | INFO | Serialization done, swapping tmp file now... | 20260505 19:49:22.345 | INFO | RTMetrics done | 20260505 19:49:22.431 | INFO | Finished a round of Baz IO: Wrote 2.034 GiB in 3.25 seconds | 20260505 19:49:22.431 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.74% / 0.00% / 78.26% | 20260505 19:49:22.431 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.195007% / 0% / 0.000115236% | 20260505 19:49:22.431 | INFO | Baz IO overall: | 20260505 19:49:22.431 | INFO | BytesWritten : 3417532881639 | 20260505 19:49:22.431 | INFO | NumEvents : 1616649309932 | 20260505 19:49:24.182 | INFO | Analyzing chunk frames = [5844992,5845504) | 20260505 19:49:24.182 | INFO | MemInfo: { | 20260505 19:49:24.182 | INFO | "gpu" : " 17 GB", | 20260505 19:49:24.182 | INFO | "huge" : "", | 20260505 19:49:24.183 | INFO | "huge_shared" : " 27 GB", | 20260505 19:49:24.183 | INFO | "malloc" : " 14 GB", | 20260505 19:49:24.183 | INFO | "pinned" : " 29 GB" | 20260505 19:49:24.183 | INFO | } | 20260505 19:49:24.183 | INFO | Perf report for the last 5.13708 seconds: | 20260505 19:49:24.183 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:49:24.183 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:24.183 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:24.183 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:24.183 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 19:49:24.183 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:24.183 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260505 19:49:24.183 | INFO | PRE_HQ : 59.7%, 1.8, 55.2ms, 0 | 20260505 19:49:24.183 | INFO | BAZWRITER : 62.8%, 3.7, 119.4ms, 0 | 20260505 19:49:24.183 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 19:49:24.183 | INFO | Uploaded 36GiB to GPU in 2.14141s (16.8113GiB/s) | 20260505 19:49:24.183 | INFO | Downloaded 38GiB to GPU in 2.24874s (16.8984GiB/s) | 20260505 19:49:29.318 | INFO | Analyzing chunk frames = [5845504,5846016) | 20260505 19:49:29.318 | INFO | MemInfo: { | 20260505 19:49:29.318 | INFO | "gpu" : " 17 GB", | 20260505 19:49:29.318 | INFO | "huge" : "", | 20260505 19:49:29.318 | INFO | "huge_shared" : " 27 GB", | 20260505 19:49:29.318 | INFO | "malloc" : " 14 GB", | 20260505 19:49:29.318 | INFO | "pinned" : " 29 GB" | 20260505 19:49:29.318 | INFO | } | 20260505 19:49:29.318 | INFO | Perf report for the last 5.13546 seconds: | 20260505 19:49:29.318 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:49:29.318 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:29.318 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:29.318 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:29.318 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 19:49:29.318 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:29.318 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:29.318 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 19:49:29.318 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:29.318 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:29.318 | INFO | Uploaded 36GiB to GPU in 2.15018s (16.7428GiB/s) | 20260505 19:49:29.318 | INFO | Downloaded 34GiB to GPU in 2.05021s (16.5837GiB/s) | 20260505 19:49:34.460 | INFO | Analyzing chunk frames = [5846016,5846528) | 20260505 19:49:34.460 | INFO | MemInfo: { | 20260505 19:49:34.460 | INFO | "gpu" : " 17 GB", | 20260505 19:49:34.460 | INFO | "huge" : "", | 20260505 19:49:34.460 | INFO | "huge_shared" : " 27 GB", | 20260505 19:49:34.460 | INFO | "malloc" : " 14 GB", | 20260505 19:49:34.460 | INFO | "pinned" : " 29 GB" | 20260505 19:49:34.460 | INFO | } | 20260505 19:49:34.460 | INFO | Perf report for the last 5.14206 seconds: | 20260505 19:49:34.460 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:49:34.460 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:34.460 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:34.460 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:34.460 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260505 19:49:34.460 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:34.460 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:34.460 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:49:34.460 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:34.460 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:34.460 | INFO | Uploaded 36GiB to GPU in 2.13742s (16.8427GiB/s) | 20260505 19:49:34.460 | INFO | Downloaded 34GiB to GPU in 2.03629s (16.697GiB/s) | 20260505 19:49:39.601 | INFO | Analyzing chunk frames = [5846528,5847040) | 20260505 19:49:39.601 | INFO | MemInfo: { | 20260505 19:49:39.601 | INFO | "gpu" : " 17 GB", | 20260505 19:49:39.601 | INFO | "huge" : "", | 20260505 19:49:39.601 | INFO | "huge_shared" : " 27 GB", | 20260505 19:49:39.601 | INFO | "malloc" : " 14 GB", | 20260505 19:49:39.601 | INFO | "pinned" : " 29 GB" | 20260505 19:49:39.601 | INFO | } | 20260505 19:49:39.601 | INFO | Perf report for the last 5.14134 seconds: | 20260505 19:49:39.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:49:39.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:39.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:39.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:39.602 | INFO | ANALYSIS : 62.4%, 3.0, 97.0ms, 0 | 20260505 19:49:39.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:39.602 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:39.602 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:49:39.602 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:39.602 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:39.602 | INFO | Uploaded 36GiB to GPU in 2.2084s (16.3014GiB/s) | 20260505 19:49:39.602 | INFO | Downloaded 34GiB to GPU in 2.04061s (16.6616GiB/s) | 20260505 19:49:44.737 | INFO | Analyzing chunk frames = [5847040,5847552) | 20260505 19:49:44.737 | INFO | MemInfo: { | 20260505 19:49:44.737 | INFO | "gpu" : " 17 GB", | 20260505 19:49:44.737 | INFO | "huge" : "", | 20260505 19:49:44.737 | INFO | "huge_shared" : " 27 GB", | 20260505 19:49:44.737 | INFO | "malloc" : " 14 GB", | 20260505 19:49:44.737 | INFO | "pinned" : " 29 GB" | 20260505 19:49:44.737 | INFO | } | 20260505 19:49:44.737 | INFO | Perf report for the last 5.13588 seconds: | 20260505 19:49:44.737 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:49:44.737 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:44.737 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:44.737 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:44.737 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260505 19:49:44.737 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:44.737 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:44.737 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:49:44.737 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:44.737 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:44.738 | INFO | Uploaded 36GiB to GPU in 2.13347s (16.8739GiB/s) | 20260505 19:49:44.738 | INFO | Downloaded 34GiB to GPU in 2.06107s (16.4963GiB/s) | 20260505 19:49:49.872 | INFO | Analyzing chunk frames = [5847552,5848064) | 20260505 19:49:49.872 | INFO | MemInfo: { | 20260505 19:49:49.872 | INFO | "gpu" : " 17 GB", | 20260505 19:49:49.872 | INFO | "huge" : "", | 20260505 19:49:49.872 | INFO | "huge_shared" : " 27 GB", | 20260505 19:49:49.872 | INFO | "malloc" : " 14 GB", | 20260505 19:49:49.872 | INFO | "pinned" : " 29 GB" | 20260505 19:49:49.872 | INFO | } | 20260505 19:49:49.872 | INFO | Perf report for the last 5.1351 seconds: | 20260505 19:49:49.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:49:49.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:49.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:49.873 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:49.873 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 19:49:49.873 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:49.873 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:49.873 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 19:49:49.873 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:49.873 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:49.873 | INFO | Uploaded 36GiB to GPU in 2.14168s (16.8092GiB/s) | 20260505 19:49:49.873 | INFO | Downloaded 34GiB to GPU in 2.05845s (16.5173GiB/s) | 20260505 19:49:55.018 | INFO | Analyzing chunk frames = [5848064,5848576) | 20260505 19:49:55.018 | INFO | MemInfo: { | 20260505 19:49:55.018 | INFO | "gpu" : " 17 GB", | 20260505 19:49:55.018 | INFO | "huge" : "", | 20260505 19:49:55.018 | INFO | "huge_shared" : " 27 GB", | 20260505 19:49:55.018 | INFO | "malloc" : " 14 GB", | 20260505 19:49:55.018 | INFO | "pinned" : " 29 GB" | 20260505 19:49:55.018 | INFO | } | 20260505 19:49:55.018 | INFO | Perf report for the last 5.14581 seconds: | 20260505 19:49:55.018 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:49:55.018 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:55.018 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:55.018 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:55.018 | INFO | ANALYSIS : 63.1%, 3.0, 98.7ms, 0 | 20260505 19:49:55.018 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:55.018 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:55.018 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 19:49:55.018 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:55.018 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:49:55.019 | INFO | Uploaded 36GiB to GPU in 2.1156s (17.0165GiB/s) | 20260505 19:49:55.019 | INFO | Downloaded 34GiB to GPU in 2.14033s (15.8854GiB/s) | 20260505 19:50:00.161 | INFO | Analyzing chunk frames = [5848576,5849088) | 20260505 19:50:00.161 | INFO | MemInfo: { | 20260505 19:50:00.161 | INFO | "gpu" : " 17 GB", | 20260505 19:50:00.161 | INFO | "huge" : "", | 20260505 19:50:00.161 | INFO | "huge_shared" : " 27 GB", | 20260505 19:50:00.161 | INFO | "malloc" : " 14 GB", | 20260505 19:50:00.161 | INFO | "pinned" : " 29 GB" | 20260505 19:50:00.161 | INFO | } | 20260505 19:50:00.161 | INFO | Perf report for the last 5.14256 seconds: | 20260505 19:50:00.161 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:50:00.161 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:00.161 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:00.161 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:00.161 | INFO | ANALYSIS : 62.3%, 3.0, 97.4ms, 0 | 20260505 19:50:00.161 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:00.161 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:00.161 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 19:50:00.161 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:00.161 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:00.161 | INFO | Uploaded 36GiB to GPU in 2.12461s (16.9442GiB/s) | 20260505 19:50:00.161 | INFO | Downloaded 34GiB to GPU in 2.01343s (16.8866GiB/s) | 20260505 19:50:00.161 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 19:50:00.161 | INFO | Total mem usage: 15.3 GiB | 20260505 19:50:00.161 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 19:50:00.161 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.11%) | 20260505 19:50:00.161 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 19:50:00.161 | INFO | High Water Breakdown: | 20260505 19:50:00.161 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 19:50:00.161 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 19:50:00.161 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 19:50:00.161 | INFO | ------------------------------------------ | 20260505 19:50:00.161 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:50:00.161 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 19:50:00.161 | INFO | Total mem usage: 26.5 GiB | 20260505 19:50:00.161 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 19:50:00.161 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 19:50:00.161 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 19:50:00.161 | INFO | High Water Breakdown: | 20260505 19:50:00.161 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:50:00.161 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:50:00.161 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:50:00.161 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:50:00.162 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:50:00.162 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:50:00.162 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 19:50:00.162 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 19:50:00.162 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:50:00.162 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 19:50:00.162 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 19:50:00.162 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 19:50:00.162 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 19:50:00.162 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 19:50:00.162 | INFO | ------------------------------------------ | 20260505 19:50:00.162 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:50:00.162 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 19:50:00.162 | INFO | Total mem usage: 25.6 GiB | 20260505 19:50:00.162 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 19:50:00.162 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 19:50:00.162 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 19:50:00.162 | INFO | High Water Breakdown: | 20260505 19:50:00.162 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 19:50:00.162 | INFO | ------------------------------------------ | 20260505 19:50:00.162 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:50:00.162 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 19:50:00.162 | INFO | Total mem usage: 15.7 GiB | 20260505 19:50:00.162 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 19:50:00.162 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 19:50:00.162 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 19:50:00.162 | INFO | High Water Breakdown: | 20260505 19:50:00.162 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 19:50:00.162 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 19:50:00.162 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 19:50:00.162 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 19:50:00.162 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 19:50:00.162 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 19:50:00.162 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 19:50:00.162 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 19:50:00.162 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 19:50:00.162 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:50:00.162 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 19:50:00.162 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:50:00.162 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 19:50:00.162 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 19:50:00.162 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 19:50:00.162 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 19:50:00.162 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:50:00.162 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 19:50:00.162 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:50:00.162 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:50:00.162 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:50:00.162 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 19:50:00.162 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 19:50:00.162 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:50:00.162 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 19:50:00.162 | INFO | ------------------------------------------ | 20260505 19:50:00.162 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:50:00.162 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 19:50:00.162 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:50:00.162 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:50:00.162 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.47e+01, 1.36e+01 / 2.23e+00 | 20260505 19:50:00.162 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.04e-01, 1.11e-02 / 4.85e-03 | 20260505 19:50:00.162 | INFO | PerfReport: 11.79%, Baseline -- 3.71e+00 / 4.53e+00, 5.63e-01 / 1.33e-01 | 20260505 19:50:00.162 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.52e+00, 1.91e-01 / 9.58e-02 | 20260505 19:50:00.162 | INFO | PerfReport: 15.97%, DME -- 4.02e+01 / 4.45e+01, 4.21e+00 / 2.30e+00 | 20260505 19:50:00.162 | INFO | PerfReport: 57.04%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.48e+00 / 1.17e+00 | 20260505 19:50:00.162 | INFO | PerfReport: 7.23%, PulseAccumulating -- 2.28e+00 / 2.85e+00, 4.35e-01 / 1.77e-01 | 20260505 19:50:00.162 | INFO | PerfReport: 3.26%, Metrics -- 1.03e+00 / 1.37e+00, 2.85e-01 / 1.52e-01 | 20260505 19:50:00.162 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.58e-01 / 4.29e-01, 6.40e-02 / 4.14e-02 | 20260505 19:50:00.162 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 19:50:00.162 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:50:00.162 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:50:00.162 | INFO | PerfReport: Full Execution -- 9.95e+01 / -nan, 2.54e+01 / 0.00e+00 | 20260505 19:50:00.162 | INFO | PerfReport: 22.43%, Upload -- 2.23e+01 / 0.00e+00, 7.64e+00 / 0.00e+00 | 20260505 19:50:00.162 | INFO | PerfReport: 21.79%, Download -- 2.17e+01 / 0.00e+00, 7.64e+00 / 0.00e+00 | 20260505 19:50:00.162 | INFO | PerfReport: 31.67%, Compute -- 3.15e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 19:50:00.162 | INFO | PerfReport: 2.29%, LockUpload -- 2.28e+00 / 0.00e+00, 8.09e+00 / 0.00e+00 | 20260505 19:50:00.162 | INFO | PerfReport: 1.38%, LockDownload -- 1.37e+00 / 0.00e+00, 2.96e+00 / 0.00e+00 | 20260505 19:50:00.162 | INFO | PerfReport: 20.43%, LockCompute -- 2.03e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 19:50:00.162 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.81e-02 / 0.00e+00 | 20260505 19:50:00.295 | INFO | Starting a round of Baz IO | 20260505 19:50:03.456 | INFO | Populating RTMetrics report | 20260505 19:50:03.456 | INFO | RTMetrics report populated | 20260505 19:50:03.456 | INFO | Opening tmp RTMetrics json file | 20260505 19:50:03.461 | INFO | Serializing RTMetrics to json | 20260505 19:50:03.462 | INFO | Serialization done, swapping tmp file now... | 20260505 19:50:03.463 | INFO | RTMetrics done | 20260505 19:50:03.545 | INFO | Finished a round of Baz IO: Wrote 2.035 GiB in 3.25 seconds | 20260505 19:50:03.545 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.73% / 0.00% / 78.26% | 20260505 19:50:03.545 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.195384% / 7.94729e-06% / 7.94729e-05% | 20260505 19:50:03.545 | INFO | Baz IO overall: | 20260505 19:50:03.545 | INFO | BytesWritten : 3419717777885 | 20260505 19:50:03.545 | INFO | NumEvents : 1617663198932 | 20260505 19:50:05.300 | INFO | Analyzing chunk frames = [5849088,5849600) | 20260505 19:50:05.300 | INFO | MemInfo: { | 20260505 19:50:05.300 | INFO | "gpu" : " 17 GB", | 20260505 19:50:05.300 | INFO | "huge" : "", | 20260505 19:50:05.300 | INFO | "huge_shared" : " 27 GB", | 20260505 19:50:05.300 | INFO | "malloc" : " 14 GB", | 20260505 19:50:05.300 | INFO | "pinned" : " 29 GB" | 20260505 19:50:05.300 | INFO | } | 20260505 19:50:05.301 | INFO | Perf report for the last 5.13959 seconds: | 20260505 19:50:05.301 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:50:05.301 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:05.301 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:05.301 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:05.301 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 19:50:05.301 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:05.301 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260505 19:50:05.301 | INFO | PRE_HQ : 60.0%, 1.8, 55.5ms, 0 | 20260505 19:50:05.301 | INFO | BAZWRITER : 62.8%, 3.7, 118.9ms, 0 | 20260505 19:50:05.301 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 19:50:05.301 | INFO | Uploaded 36GiB to GPU in 2.16205s (16.6509GiB/s) | 20260505 19:50:05.301 | INFO | Downloaded 38GiB to GPU in 2.25377s (16.8606GiB/s) | 20260505 19:50:10.429 | INFO | Analyzing chunk frames = [5849600,5850112) | 20260505 19:50:10.429 | INFO | MemInfo: { | 20260505 19:50:10.429 | INFO | "gpu" : " 17 GB", | 20260505 19:50:10.430 | INFO | "huge" : "", | 20260505 19:50:10.430 | INFO | "huge_shared" : " 27 GB", | 20260505 19:50:10.430 | INFO | "malloc" : " 14 GB", | 20260505 19:50:10.430 | INFO | "pinned" : " 29 GB" | 20260505 19:50:10.430 | INFO | } | 20260505 19:50:10.430 | INFO | Perf report for the last 5.12909 seconds: | 20260505 19:50:10.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:50:10.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:10.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:10.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:10.430 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260505 19:50:10.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:10.430 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:10.430 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260505 19:50:10.430 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:10.430 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:10.430 | INFO | Uploaded 36GiB to GPU in 2.08737s (17.2466GiB/s) | 20260505 19:50:10.430 | INFO | Downloaded 34GiB to GPU in 2.02601s (16.7817GiB/s) | 20260505 19:50:15.567 | INFO | Analyzing chunk frames = [5850112,5850624) | 20260505 19:50:15.567 | INFO | MemInfo: { | 20260505 19:50:15.567 | INFO | "gpu" : " 17 GB", | 20260505 19:50:15.567 | INFO | "huge" : "", | 20260505 19:50:15.567 | INFO | "huge_shared" : " 27 GB", | 20260505 19:50:15.567 | INFO | "malloc" : " 14 GB", | 20260505 19:50:15.567 | INFO | "pinned" : " 29 GB" | 20260505 19:50:15.567 | INFO | } | 20260505 19:50:15.567 | INFO | Perf report for the last 5.13768 seconds: | 20260505 19:50:15.567 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:50:15.567 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:15.567 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:15.567 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:15.567 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 19:50:15.567 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:15.567 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:15.567 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:50:15.567 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:15.567 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:15.568 | INFO | Uploaded 36GiB to GPU in 2.21106s (16.2818GiB/s) | 20260505 19:50:15.568 | INFO | Downloaded 34GiB to GPU in 2.06921s (16.4314GiB/s) | 20260505 19:50:20.718 | INFO | Analyzing chunk frames = [5850624,5851136) | 20260505 19:50:20.718 | INFO | MemInfo: { | 20260505 19:50:20.718 | INFO | "gpu" : " 17 GB", | 20260505 19:50:20.718 | INFO | "huge" : "", | 20260505 19:50:20.718 | INFO | "huge_shared" : " 27 GB", | 20260505 19:50:20.718 | INFO | "malloc" : " 14 GB", | 20260505 19:50:20.718 | INFO | "pinned" : " 29 GB" | 20260505 19:50:20.718 | INFO | } | 20260505 19:50:20.718 | INFO | Perf report for the last 5.15068 seconds: | 20260505 19:50:20.718 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:50:20.718 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:20.718 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:20.718 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:20.718 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260505 19:50:20.718 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:20.718 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:20.718 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:50:20.718 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:20.718 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:20.718 | INFO | Uploaded 36GiB to GPU in 2.17517s (16.5504GiB/s) | 20260505 19:50:20.718 | INFO | Downloaded 34GiB to GPU in 2.0582s (16.5193GiB/s) | 20260505 19:50:25.859 | INFO | Analyzing chunk frames = [5851136,5851648) | 20260505 19:50:25.859 | INFO | MemInfo: { | 20260505 19:50:25.859 | INFO | "gpu" : " 17 GB", | 20260505 19:50:25.859 | INFO | "huge" : "", | 20260505 19:50:25.859 | INFO | "huge_shared" : " 27 GB", | 20260505 19:50:25.859 | INFO | "malloc" : " 14 GB", | 20260505 19:50:25.859 | INFO | "pinned" : " 29 GB" | 20260505 19:50:25.859 | INFO | } | 20260505 19:50:25.859 | INFO | Perf report for the last 5.14143 seconds: | 20260505 19:50:25.859 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:50:25.859 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:25.859 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:25.859 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:25.859 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260505 19:50:25.860 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:25.860 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:25.860 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:50:25.860 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:25.860 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:25.860 | INFO | Uploaded 36GiB to GPU in 2.125s (16.9412GiB/s) | 20260505 19:50:25.860 | INFO | Downloaded 34GiB to GPU in 2.05962s (16.5079GiB/s) | 20260505 19:50:30.994 | INFO | Analyzing chunk frames = [5851648,5852160) | 20260505 19:50:30.994 | INFO | MemInfo: { | 20260505 19:50:30.994 | INFO | "gpu" : " 17 GB", | 20260505 19:50:30.994 | INFO | "huge" : "", | 20260505 19:50:30.994 | INFO | "huge_shared" : " 27 GB", | 20260505 19:50:30.994 | INFO | "malloc" : " 14 GB", | 20260505 19:50:30.994 | INFO | "pinned" : " 29 GB" | 20260505 19:50:30.994 | INFO | } | 20260505 19:50:30.994 | INFO | Perf report for the last 5.13496 seconds: | 20260505 19:50:30.994 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:50:30.994 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:30.994 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:30.994 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:30.994 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260505 19:50:30.994 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:30.994 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:30.994 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 19:50:30.994 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:30.994 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:30.995 | INFO | Uploaded 36GiB to GPU in 2.08567s (17.2606GiB/s) | 20260505 19:50:30.995 | INFO | Downloaded 34GiB to GPU in 2.03734s (16.6885GiB/s) | 20260505 19:50:36.129 | INFO | Analyzing chunk frames = [5852160,5852672) | 20260505 19:50:36.129 | INFO | MemInfo: { | 20260505 19:50:36.129 | INFO | "gpu" : " 17 GB", | 20260505 19:50:36.129 | INFO | "huge" : "", | 20260505 19:50:36.129 | INFO | "huge_shared" : " 27 GB", | 20260505 19:50:36.129 | INFO | "malloc" : " 14 GB", | 20260505 19:50:36.129 | INFO | "pinned" : " 29 GB" | 20260505 19:50:36.129 | INFO | } | 20260505 19:50:36.129 | INFO | Perf report for the last 5.13474 seconds: | 20260505 19:50:36.129 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:50:36.129 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:36.129 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:36.129 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:36.129 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 19:50:36.129 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:36.129 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:36.129 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 19:50:36.129 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:36.129 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:36.129 | INFO | Uploaded 36GiB to GPU in 2.12335s (16.9544GiB/s) | 20260505 19:50:36.129 | INFO | Downloaded 34GiB to GPU in 2.13102s (15.9548GiB/s) | 20260505 19:50:41.275 | INFO | Analyzing chunk frames = [5852672,5853184) | 20260505 19:50:41.275 | INFO | MemInfo: { | 20260505 19:50:41.275 | INFO | "gpu" : " 17 GB", | 20260505 19:50:41.275 | INFO | "huge" : "", | 20260505 19:50:41.275 | INFO | "huge_shared" : " 27 GB", | 20260505 19:50:41.275 | INFO | "malloc" : " 14 GB", | 20260505 19:50:41.275 | INFO | "pinned" : " 29 GB" | 20260505 19:50:41.275 | INFO | } | 20260505 19:50:41.275 | INFO | Perf report for the last 5.14609 seconds: | 20260505 19:50:41.275 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:50:41.275 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:41.275 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:41.275 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:41.275 | INFO | ANALYSIS : 62.7%, 3.0, 98.1ms, 0 | 20260505 19:50:41.275 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:41.275 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:41.275 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:50:41.275 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:41.275 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:41.276 | INFO | Uploaded 36GiB to GPU in 2.19441s (16.4053GiB/s) | 20260505 19:50:41.276 | INFO | Downloaded 34GiB to GPU in 2.03409s (16.7151GiB/s) | 20260505 19:50:41.408 | INFO | Starting a round of Baz IO | 20260505 19:50:44.576 | INFO | Populating RTMetrics report | 20260505 19:50:44.576 | INFO | RTMetrics report populated | 20260505 19:50:44.576 | INFO | Opening tmp RTMetrics json file | 20260505 19:50:44.577 | INFO | Serializing RTMetrics to json | 20260505 19:50:44.578 | INFO | Serialization done, swapping tmp file now... | 20260505 19:50:44.578 | INFO | RTMetrics done | 20260505 19:50:44.667 | INFO | Finished a round of Baz IO: Wrote 2.035 GiB in 3.258 seconds | 20260505 19:50:44.667 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.73% / 0.00% / 78.27% | 20260505 19:50:44.667 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.194625% / 0% / 9.93411e-05% | 20260505 19:50:44.667 | INFO | Baz IO overall: | 20260505 19:50:44.667 | INFO | BytesWritten : 3421902310347 | 20260505 19:50:44.667 | INFO | NumEvents : 1618677006132 | 20260505 19:50:46.407 | INFO | Analyzing chunk frames = [5853184,5853696) | 20260505 19:50:46.407 | INFO | MemInfo: { | 20260505 19:50:46.407 | INFO | "gpu" : " 17 GB", | 20260505 19:50:46.407 | INFO | "huge" : "", | 20260505 19:50:46.407 | INFO | "huge_shared" : " 27 GB", | 20260505 19:50:46.407 | INFO | "malloc" : " 14 GB", | 20260505 19:50:46.407 | INFO | "pinned" : " 29 GB" | 20260505 19:50:46.407 | INFO | } | 20260505 19:50:46.407 | INFO | Perf report for the last 5.13216 seconds: | 20260505 19:50:46.407 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:50:46.407 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:46.407 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:46.407 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:46.407 | INFO | ANALYSIS : 63.2%, 3.0, 98.4ms, 0 | 20260505 19:50:46.407 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:46.407 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260505 19:50:46.407 | INFO | PRE_HQ : 60.1%, 1.8, 55.6ms, 0 | 20260505 19:50:46.407 | INFO | BAZWRITER : 63.0%, 3.7, 119.6ms, 0 | 20260505 19:50:46.407 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 19:50:46.408 | INFO | Uploaded 36GiB to GPU in 2.1625s (16.6474GiB/s) | 20260505 19:50:46.408 | INFO | Downloaded 38GiB to GPU in 2.261s (16.8067GiB/s) | 20260505 19:50:51.549 | INFO | Analyzing chunk frames = [5853696,5854208) | 20260505 19:50:51.549 | INFO | MemInfo: { | 20260505 19:50:51.549 | INFO | "gpu" : " 17 GB", | 20260505 19:50:51.549 | INFO | "huge" : "", | 20260505 19:50:51.549 | INFO | "huge_shared" : " 27 GB", | 20260505 19:50:51.549 | INFO | "malloc" : " 14 GB", | 20260505 19:50:51.549 | INFO | "pinned" : " 29 GB" | 20260505 19:50:51.549 | INFO | } | 20260505 19:50:51.549 | INFO | Perf report for the last 5.1416 seconds: | 20260505 19:50:51.549 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:50:51.549 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:51.549 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:51.549 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:51.549 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 19:50:51.549 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:51.549 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:51.549 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:50:51.549 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:51.549 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:51.549 | INFO | Uploaded 36GiB to GPU in 2.15078s (16.7381GiB/s) | 20260505 19:50:51.549 | INFO | Downloaded 34GiB to GPU in 2.05461s (16.5481GiB/s) | 20260505 19:50:56.686 | INFO | Analyzing chunk frames = [5854208,5854720) | 20260505 19:50:56.686 | INFO | MemInfo: { | 20260505 19:50:56.686 | INFO | "gpu" : " 17 GB", | 20260505 19:50:56.686 | INFO | "huge" : "", | 20260505 19:50:56.686 | INFO | "huge_shared" : " 27 GB", | 20260505 19:50:56.686 | INFO | "malloc" : " 14 GB", | 20260505 19:50:56.686 | INFO | "pinned" : " 29 GB" | 20260505 19:50:56.686 | INFO | } | 20260505 19:50:56.686 | INFO | Perf report for the last 5.13711 seconds: | 20260505 19:50:56.686 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:50:56.686 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:56.686 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:56.686 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:56.686 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260505 19:50:56.686 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:56.686 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:56.686 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:50:56.686 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:56.686 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:50:56.686 | INFO | Uploaded 36GiB to GPU in 2.11599s (17.0133GiB/s) | 20260505 19:50:56.686 | INFO | Downloaded 34GiB to GPU in 2.0173s (16.8542GiB/s) | 20260505 19:51:01.823 | INFO | Analyzing chunk frames = [5854720,5855232) | 20260505 19:51:01.823 | INFO | MemInfo: { | 20260505 19:51:01.823 | INFO | "gpu" : " 17 GB", | 20260505 19:51:01.823 | INFO | "huge" : "", | 20260505 19:51:01.823 | INFO | "huge_shared" : " 27 GB", | 20260505 19:51:01.823 | INFO | "malloc" : " 14 GB", | 20260505 19:51:01.823 | INFO | "pinned" : " 29 GB" | 20260505 19:51:01.823 | INFO | } | 20260505 19:51:01.823 | INFO | Perf report for the last 5.13686 seconds: | 20260505 19:51:01.823 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:51:01.823 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:01.823 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:01.823 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:01.823 | INFO | ANALYSIS : 62.4%, 3.0, 96.9ms, 0 | 20260505 19:51:01.823 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:01.823 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:01.823 | INFO | PRE_HQ : 27.5%, 1.0, 14.2ms, 0 | 20260505 19:51:01.823 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:01.823 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:01.823 | INFO | Uploaded 36GiB to GPU in 2.20085s (16.3573GiB/s) | 20260505 19:51:01.823 | INFO | Downloaded 34GiB to GPU in 2.04905s (16.5931GiB/s) | 20260505 19:51:06.967 | INFO | Analyzing chunk frames = [5855232,5855744) | 20260505 19:51:06.967 | INFO | MemInfo: { | 20260505 19:51:06.967 | INFO | "gpu" : " 17 GB", | 20260505 19:51:06.967 | INFO | "huge" : "", | 20260505 19:51:06.967 | INFO | "huge_shared" : " 27 GB", | 20260505 19:51:06.967 | INFO | "malloc" : " 14 GB", | 20260505 19:51:06.967 | INFO | "pinned" : " 29 GB" | 20260505 19:51:06.967 | INFO | } | 20260505 19:51:06.967 | INFO | Perf report for the last 5.1442 seconds: | 20260505 19:51:06.967 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:51:06.967 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:06.967 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:06.967 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:06.967 | INFO | ANALYSIS : 61.8%, 3.0, 96.6ms, 0 | 20260505 19:51:06.967 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:06.967 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:06.967 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:51:06.967 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:06.967 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:06.968 | INFO | Uploaded 36GiB to GPU in 2.13424s (16.8678GiB/s) | 20260505 19:51:06.968 | INFO | Downloaded 34GiB to GPU in 2.06166s (16.4916GiB/s) | 20260505 19:51:12.114 | INFO | Analyzing chunk frames = [5855744,5856256) | 20260505 19:51:12.114 | INFO | MemInfo: { | 20260505 19:51:12.114 | INFO | "gpu" : " 17 GB", | 20260505 19:51:12.114 | INFO | "huge" : "", | 20260505 19:51:12.114 | INFO | "huge_shared" : " 27 GB", | 20260505 19:51:12.114 | INFO | "malloc" : " 14 GB", | 20260505 19:51:12.114 | INFO | "pinned" : " 29 GB" | 20260505 19:51:12.115 | INFO | } | 20260505 19:51:12.115 | INFO | Perf report for the last 5.1473 seconds: | 20260505 19:51:12.115 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:51:12.115 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:12.115 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:12.115 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:12.115 | INFO | ANALYSIS : 62.1%, 3.0, 97.2ms, 0 | 20260505 19:51:12.115 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:12.115 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:12.115 | INFO | PRE_HQ : 27.2%, 1.0, 14.2ms, 0 | 20260505 19:51:12.115 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:12.115 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:12.115 | INFO | Uploaded 36GiB to GPU in 2.15534s (16.7027GiB/s) | 20260505 19:51:12.115 | INFO | Downloaded 34GiB to GPU in 2.06102s (16.4966GiB/s) | 20260505 19:51:17.241 | INFO | Analyzing chunk frames = [5856256,5856768) | 20260505 19:51:17.241 | INFO | MemInfo: { | 20260505 19:51:17.241 | INFO | "gpu" : " 17 GB", | 20260505 19:51:17.241 | INFO | "huge" : "", | 20260505 19:51:17.241 | INFO | "huge_shared" : " 27 GB", | 20260505 19:51:17.241 | INFO | "malloc" : " 14 GB", | 20260505 19:51:17.241 | INFO | "pinned" : " 29 GB" | 20260505 19:51:17.241 | INFO | } | 20260505 19:51:17.241 | INFO | Perf report for the last 5.12634 seconds: | 20260505 19:51:17.241 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:51:17.241 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:17.241 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:17.241 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:17.241 | INFO | ANALYSIS : 63.1%, 3.0, 98.3ms, 0 | 20260505 19:51:17.241 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:17.241 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:17.241 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260505 19:51:17.241 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:17.241 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:17.241 | INFO | Uploaded 36GiB to GPU in 2.12916s (16.9081GiB/s) | 20260505 19:51:17.241 | INFO | Downloaded 34GiB to GPU in 2.15169s (15.8016GiB/s) | 20260505 19:51:22.378 | INFO | Analyzing chunk frames = [5856768,5857280) | 20260505 19:51:22.378 | INFO | MemInfo: { | 20260505 19:51:22.378 | INFO | "gpu" : " 17 GB", | 20260505 19:51:22.378 | INFO | "huge" : "", | 20260505 19:51:22.378 | INFO | "huge_shared" : " 27 GB", | 20260505 19:51:22.379 | INFO | "malloc" : " 14 GB", | 20260505 19:51:22.379 | INFO | "pinned" : " 29 GB" | 20260505 19:51:22.379 | INFO | } | 20260505 19:51:22.379 | INFO | Perf report for the last 5.13764 seconds: | 20260505 19:51:22.379 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:51:22.379 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:22.379 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:22.379 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:22.379 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 19:51:22.379 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:22.379 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:22.379 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:51:22.379 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:22.379 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:22.379 | INFO | Uploaded 36GiB to GPU in 2.19974s (16.3656GiB/s) | 20260505 19:51:22.379 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 19:51:22.379 | INFO | Total mem usage: 15.3 GiB | 20260505 19:51:22.379 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 19:51:22.379 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.48%) | 20260505 19:51:22.379 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 19:51:22.379 | INFO | Downloaded 34GiB to GPU in 2.03658s (16.6947GiB/s) | 20260505 19:51:22.379 | INFO | High Water Breakdown: | 20260505 19:51:22.379 | INFO | 60.9% : 8.6 GiB ( 90.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 19:51:22.379 | INFO | 39% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 19:51:22.379 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 19:51:22.379 | INFO | ------------------------------------------ | 20260505 19:51:22.379 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:51:22.379 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 19:51:22.379 | INFO | Total mem usage: 26.5 GiB | 20260505 19:51:22.379 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 19:51:22.379 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 19:51:22.379 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 19:51:22.379 | INFO | High Water Breakdown: | 20260505 19:51:22.379 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:51:22.379 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:51:22.379 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:51:22.379 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:51:22.379 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:51:22.379 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:51:22.379 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 19:51:22.379 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 19:51:22.379 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:51:22.379 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 19:51:22.379 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 19:51:22.379 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 19:51:22.379 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 19:51:22.379 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 19:51:22.379 | INFO | ------------------------------------------ | 20260505 19:51:22.379 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:51:22.379 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 19:51:22.379 | INFO | Total mem usage: 25.6 GiB | 20260505 19:51:22.379 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 19:51:22.379 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 19:51:22.379 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 19:51:22.379 | INFO | High Water Breakdown: | 20260505 19:51:22.379 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 19:51:22.379 | INFO | ------------------------------------------ | 20260505 19:51:22.379 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:51:22.379 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 19:51:22.379 | INFO | Total mem usage: 15.7 GiB | 20260505 19:51:22.379 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 19:51:22.379 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 19:51:22.379 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 19:51:22.379 | INFO | High Water Breakdown: | 20260505 19:51:22.379 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 19:51:22.379 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 19:51:22.379 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 19:51:22.379 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 19:51:22.379 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 19:51:22.379 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 19:51:22.379 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 19:51:22.379 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 19:51:22.379 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 19:51:22.379 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:51:22.379 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 19:51:22.379 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:51:22.379 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 19:51:22.379 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 19:51:22.379 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 19:51:22.379 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 19:51:22.379 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:51:22.379 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 19:51:22.379 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:51:22.379 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:51:22.379 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:51:22.379 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 19:51:22.380 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 19:51:22.380 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:51:22.380 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 19:51:22.380 | INFO | ------------------------------------------ | 20260505 19:51:22.380 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:51:22.380 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 19:51:22.380 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:51:22.380 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:51:22.380 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.43e+01, 1.35e+01 / 1.92e+00 | 20260505 19:51:22.380 | INFO | PerfReport: 0.29%, AggregateStats -- 9.25e-02 / 1.02e-01, 1.10e-02 / 4.21e-03 | 20260505 19:51:22.380 | INFO | PerfReport: 11.78%, Baseline -- 3.71e+00 / 4.53e+00, 5.56e-01 / 1.38e-01 | 20260505 19:51:22.380 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.51e+00, 1.88e-01 / 9.56e-02 | 20260505 19:51:22.380 | INFO | PerfReport: 15.95%, DME -- 4.02e+01 / 4.42e+01, 3.89e+00 / 1.96e+00 | 20260505 19:51:22.380 | INFO | PerfReport: 57.04%, FrameLabeling -- 1.79e+01 / 1.93e+01, 1.46e+00 / 9.92e-01 | 20260505 19:51:22.380 | INFO | PerfReport: 7.26%, PulseAccumulating -- 2.29e+00 / 2.86e+00, 4.35e-01 / 2.01e-01 | 20260505 19:51:22.380 | INFO | PerfReport: 3.27%, Metrics -- 1.03e+00 / 1.33e+00, 2.78e-01 / 1.03e-01 | 20260505 19:51:22.380 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.58e-01 / 4.20e-01, 6.29e-02 / 2.91e-02 | 20260505 19:51:22.380 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 19:51:22.380 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:51:22.380 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:51:22.380 | INFO | PerfReport: Full Execution -- 9.95e+01 / -nan, 2.54e+01 / 0.00e+00 | 20260505 19:51:22.380 | INFO | PerfReport: 22.52%, Upload -- 2.24e+01 / 0.00e+00, 7.74e+00 / 0.00e+00 | 20260505 19:51:22.380 | INFO | PerfReport: 21.83%, Download -- 2.17e+01 / 0.00e+00, 7.67e+00 / 0.00e+00 | 20260505 19:51:22.380 | INFO | PerfReport: 31.64%, Compute -- 3.15e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 19:51:22.380 | INFO | PerfReport: 2.30%, LockUpload -- 2.29e+00 / 0.00e+00, 7.89e+00 / 0.00e+00 | 20260505 19:51:22.380 | INFO | PerfReport: 1.37%, LockDownload -- 1.37e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260505 19:51:22.380 | INFO | PerfReport: 20.32%, LockCompute -- 2.02e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 19:51:22.380 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.75e-02 / 0.00e+00 | 20260505 19:51:22.514 | INFO | Starting a round of Baz IO | 20260505 19:51:25.686 | INFO | Populating RTMetrics report | 20260505 19:51:25.687 | INFO | RTMetrics report populated | 20260505 19:51:25.687 | INFO | Opening tmp RTMetrics json file | 20260505 19:51:25.689 | INFO | Serializing RTMetrics to json | 20260505 19:51:25.692 | INFO | Serialization done, swapping tmp file now... | 20260505 19:51:25.692 | INFO | RTMetrics done | 20260505 19:51:25.777 | INFO | Finished a round of Baz IO: Wrote 2.035 GiB in 3.262 seconds | 20260505 19:51:25.777 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.73% / 0.00% / 78.27% | 20260505 19:51:25.777 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.194585% / 3.97364e-06% / 9.13938e-05% | 20260505 19:51:25.777 | INFO | Baz IO overall: | 20260505 19:51:25.777 | INFO | BytesWritten : 3424087065970 | 20260505 19:51:25.777 | INFO | NumEvents : 1619690854060 | 20260505 19:51:27.527 | INFO | Analyzing chunk frames = [5857280,5857792) | 20260505 19:51:27.527 | INFO | MemInfo: { | 20260505 19:51:27.527 | INFO | "gpu" : " 17 GB", | 20260505 19:51:27.527 | INFO | "huge" : "", | 20260505 19:51:27.527 | INFO | "huge_shared" : " 27 GB", | 20260505 19:51:27.527 | INFO | "malloc" : " 14 GB", | 20260505 19:51:27.527 | INFO | "pinned" : " 29 GB" | 20260505 19:51:27.527 | INFO | } | 20260505 19:51:27.527 | INFO | Perf report for the last 5.14844 seconds: | 20260505 19:51:27.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:51:27.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:27.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:27.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:27.527 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 19:51:27.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:27.527 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260505 19:51:27.527 | INFO | PRE_HQ : 60.1%, 1.8, 55.2ms, 0 | 20260505 19:51:27.527 | INFO | BAZWRITER : 62.9%, 3.7, 120.3ms, 0 | 20260505 19:51:27.527 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 19:51:27.527 | INFO | Uploaded 36GiB to GPU in 2.16417s (16.6345GiB/s) | 20260505 19:51:27.527 | INFO | Downloaded 38GiB to GPU in 2.24758s (16.9071GiB/s) | 20260505 19:51:32.674 | INFO | Analyzing chunk frames = [5857792,5858304) | 20260505 19:51:32.674 | INFO | MemInfo: { | 20260505 19:51:32.674 | INFO | "gpu" : " 17 GB", | 20260505 19:51:32.674 | INFO | "huge" : "", | 20260505 19:51:32.674 | INFO | "huge_shared" : " 27 GB", | 20260505 19:51:32.674 | INFO | "malloc" : " 14 GB", | 20260505 19:51:32.674 | INFO | "pinned" : " 29 GB" | 20260505 19:51:32.674 | INFO | } | 20260505 19:51:32.674 | INFO | Perf report for the last 5.14712 seconds: | 20260505 19:51:32.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:51:32.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:32.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:32.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:32.674 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260505 19:51:32.674 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:32.674 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:32.674 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:51:32.674 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:32.674 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:32.674 | INFO | Uploaded 36GiB to GPU in 2.09679s (17.1691GiB/s) | 20260505 19:51:32.674 | INFO | Downloaded 34GiB to GPU in 2.03026s (16.7466GiB/s) | 20260505 19:51:37.806 | INFO | Analyzing chunk frames = [5858304,5858816) | 20260505 19:51:37.806 | INFO | MemInfo: { | 20260505 19:51:37.806 | INFO | "gpu" : " 17 GB", | 20260505 19:51:37.806 | INFO | "huge" : "", | 20260505 19:51:37.806 | INFO | "huge_shared" : " 27 GB", | 20260505 19:51:37.806 | INFO | "malloc" : " 14 GB", | 20260505 19:51:37.806 | INFO | "pinned" : " 29 GB" | 20260505 19:51:37.806 | INFO | } | 20260505 19:51:37.806 | INFO | Perf report for the last 5.13155 seconds: | 20260505 19:51:37.806 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:51:37.806 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:37.806 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:37.806 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:37.806 | INFO | ANALYSIS : 62.2%, 3.0, 96.6ms, 0 | 20260505 19:51:37.806 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:37.806 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:37.806 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260505 19:51:37.806 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:37.806 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:37.806 | INFO | Uploaded 36GiB to GPU in 2.14003s (16.8222GiB/s) | 20260505 19:51:37.806 | INFO | Downloaded 34GiB to GPU in 2.03475s (16.7096GiB/s) | 20260505 19:51:42.940 | INFO | Analyzing chunk frames = [5858816,5859328) | 20260505 19:51:42.940 | INFO | MemInfo: { | 20260505 19:51:42.940 | INFO | "gpu" : " 17 GB", | 20260505 19:51:42.940 | INFO | "huge" : "", | 20260505 19:51:42.940 | INFO | "huge_shared" : " 27 GB", | 20260505 19:51:42.940 | INFO | "malloc" : " 14 GB", | 20260505 19:51:42.940 | INFO | "pinned" : " 29 GB" | 20260505 19:51:42.940 | INFO | } | 20260505 19:51:42.940 | INFO | Perf report for the last 5.13472 seconds: | 20260505 19:51:42.940 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:51:42.940 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:42.940 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:42.940 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:42.940 | INFO | ANALYSIS : 62.5%, 3.0, 97.0ms, 0 | 20260505 19:51:42.941 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:42.941 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:42.941 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:51:42.941 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:42.941 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:42.941 | INFO | Uploaded 36GiB to GPU in 2.20961s (16.2925GiB/s) | 20260505 19:51:42.941 | INFO | Downloaded 34GiB to GPU in 2.04333s (16.6395GiB/s) | 20260505 19:51:48.079 | INFO | Analyzing chunk frames = [5859328,5859840) | 20260505 19:51:48.079 | INFO | MemInfo: { | 20260505 19:51:48.079 | INFO | "gpu" : " 17 GB", | 20260505 19:51:48.079 | INFO | "huge" : "", | 20260505 19:51:48.079 | INFO | "huge_shared" : " 27 GB", | 20260505 19:51:48.079 | INFO | "malloc" : " 14 GB", | 20260505 19:51:48.080 | INFO | "pinned" : " 29 GB" | 20260505 19:51:48.080 | INFO | } | 20260505 19:51:48.080 | INFO | Perf report for the last 5.1391 seconds: | 20260505 19:51:48.080 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:51:48.080 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:48.080 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:48.080 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:48.080 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 19:51:48.080 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:48.080 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:48.080 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:51:48.080 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:48.080 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:48.080 | INFO | Uploaded 36GiB to GPU in 2.14833s (16.7572GiB/s) | 20260505 19:51:48.080 | INFO | Downloaded 34GiB to GPU in 2.06356s (16.4763GiB/s) | 20260505 19:51:53.224 | INFO | Analyzing chunk frames = [5859840,5860352) | 20260505 19:51:53.224 | INFO | MemInfo: { | 20260505 19:51:53.224 | INFO | "gpu" : " 17 GB", | 20260505 19:51:53.224 | INFO | "huge" : "", | 20260505 19:51:53.224 | INFO | "huge_shared" : " 27 GB", | 20260505 19:51:53.224 | INFO | "malloc" : " 14 GB", | 20260505 19:51:53.224 | INFO | "pinned" : " 29 GB" | 20260505 19:51:53.224 | INFO | } | 20260505 19:51:53.224 | INFO | Perf report for the last 5.14447 seconds: | 20260505 19:51:53.224 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:51:53.224 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:53.224 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:53.224 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:53.224 | INFO | ANALYSIS : 62.1%, 3.0, 97.1ms, 0 | 20260505 19:51:53.224 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:53.224 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:53.224 | INFO | PRE_HQ : 27.0%, 1.0, 14.1ms, 0 | 20260505 19:51:53.224 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:53.224 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:53.224 | INFO | Uploaded 36GiB to GPU in 2.14019s (16.8209GiB/s) | 20260505 19:51:53.224 | INFO | Downloaded 34GiB to GPU in 2.05393s (16.5537GiB/s) | 20260505 19:51:58.361 | INFO | Analyzing chunk frames = [5860352,5860864) | 20260505 19:51:58.361 | INFO | MemInfo: { | 20260505 19:51:58.361 | INFO | "gpu" : " 17 GB", | 20260505 19:51:58.361 | INFO | "huge" : "", | 20260505 19:51:58.361 | INFO | "huge_shared" : " 27 GB", | 20260505 19:51:58.361 | INFO | "malloc" : " 14 GB", | 20260505 19:51:58.361 | INFO | "pinned" : " 29 GB" | 20260505 19:51:58.361 | INFO | } | 20260505 19:51:58.361 | INFO | Perf report for the last 5.13731 seconds: | 20260505 19:51:58.361 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:51:58.361 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:58.361 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:58.361 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:58.361 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 19:51:58.361 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:58.361 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:58.361 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 19:51:58.361 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:58.361 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:51:58.362 | INFO | Uploaded 36GiB to GPU in 2.13155s (16.8891GiB/s) | 20260505 19:51:58.362 | INFO | Downloaded 34GiB to GPU in 2.12652s (15.9885GiB/s) | 20260505 19:52:03.497 | INFO | Analyzing chunk frames = [5860864,5861376) | 20260505 19:52:03.497 | INFO | MemInfo: { | 20260505 19:52:03.497 | INFO | "gpu" : " 17 GB", | 20260505 19:52:03.497 | INFO | "huge" : "", | 20260505 19:52:03.497 | INFO | "huge_shared" : " 27 GB", | 20260505 19:52:03.497 | INFO | "malloc" : " 14 GB", | 20260505 19:52:03.497 | INFO | "pinned" : " 29 GB" | 20260505 19:52:03.497 | INFO | } | 20260505 19:52:03.497 | INFO | Perf report for the last 5.13535 seconds: | 20260505 19:52:03.497 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:52:03.497 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:03.497 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:03.497 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:03.497 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 19:52:03.497 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:03.497 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:03.497 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 19:52:03.497 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:03.497 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:03.497 | INFO | Uploaded 36GiB to GPU in 2.20602s (16.319GiB/s) | 20260505 19:52:03.497 | INFO | Downloaded 34GiB to GPU in 2.04378s (16.6359GiB/s) | 20260505 19:52:03.632 | INFO | Starting a round of Baz IO | 20260505 19:52:06.805 | INFO | Populating RTMetrics report | 20260505 19:52:06.805 | INFO | RTMetrics report populated | 20260505 19:52:06.805 | INFO | Opening tmp RTMetrics json file | 20260505 19:52:06.806 | INFO | Serializing RTMetrics to json | 20260505 19:52:06.808 | INFO | Serialization done, swapping tmp file now... | 20260505 19:52:06.808 | INFO | RTMetrics done | 20260505 19:52:06.892 | INFO | Finished a round of Baz IO: Wrote 2.035 GiB in 3.26 seconds | 20260505 19:52:06.892 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.73% / 0.00% / 78.27% | 20260505 19:52:06.892 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.194883% / 3.97364e-06% / 8.74201e-05% | 20260505 19:52:06.892 | INFO | Baz IO overall: | 20260505 19:52:06.892 | INFO | BytesWritten : 3426271819524 | 20260505 19:52:06.892 | INFO | NumEvents : 1620704744000 | 20260505 19:52:08.636 | INFO | Analyzing chunk frames = [5861376,5861888) | 20260505 19:52:08.636 | INFO | MemInfo: { | 20260505 19:52:08.636 | INFO | "gpu" : " 17 GB", | 20260505 19:52:08.636 | INFO | "huge" : "", | 20260505 19:52:08.636 | INFO | "huge_shared" : " 27 GB", | 20260505 19:52:08.636 | INFO | "malloc" : " 14 GB", | 20260505 19:52:08.636 | INFO | "pinned" : " 29 GB" | 20260505 19:52:08.636 | INFO | } | 20260505 19:52:08.636 | INFO | Perf report for the last 5.13937 seconds: | 20260505 19:52:08.636 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:52:08.636 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:08.636 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:08.636 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:08.636 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 19:52:08.636 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:08.636 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260505 19:52:08.636 | INFO | PRE_HQ : 60.2%, 1.8, 55.7ms, 0 | 20260505 19:52:08.636 | INFO | BAZWRITER : 63.0%, 3.7, 119.1ms, 0 | 20260505 19:52:08.636 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 19:52:08.636 | INFO | Uploaded 36GiB to GPU in 2.16165s (16.654GiB/s) | 20260505 19:52:08.637 | INFO | Downloaded 38GiB to GPU in 2.23406s (17.0094GiB/s) | 20260505 19:52:13.781 | INFO | Analyzing chunk frames = [5861888,5862400) | 20260505 19:52:13.781 | INFO | MemInfo: { | 20260505 19:52:13.781 | INFO | "gpu" : " 17 GB", | 20260505 19:52:13.781 | INFO | "huge" : "", | 20260505 19:52:13.781 | INFO | "huge_shared" : " 27 GB", | 20260505 19:52:13.781 | INFO | "malloc" : " 14 GB", | 20260505 19:52:13.781 | INFO | "pinned" : " 29 GB" | 20260505 19:52:13.781 | INFO | } | 20260505 19:52:13.781 | INFO | Perf report for the last 5.14502 seconds: | 20260505 19:52:13.781 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:52:13.781 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:13.781 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:13.781 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:13.781 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260505 19:52:13.781 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:13.781 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:13.781 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 19:52:13.781 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:13.781 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:13.781 | INFO | Uploaded 36GiB to GPU in 2.1355s (16.8579GiB/s) | 20260505 19:52:13.781 | INFO | Downloaded 34GiB to GPU in 2.04003s (16.6664GiB/s) | 20260505 19:52:18.920 | INFO | Analyzing chunk frames = [5862400,5862912) | 20260505 19:52:18.920 | INFO | MemInfo: { | 20260505 19:52:18.920 | INFO | "gpu" : " 17 GB", | 20260505 19:52:18.920 | INFO | "huge" : "", | 20260505 19:52:18.920 | INFO | "huge_shared" : " 27 GB", | 20260505 19:52:18.920 | INFO | "malloc" : " 14 GB", | 20260505 19:52:18.920 | INFO | "pinned" : " 29 GB" | 20260505 19:52:18.920 | INFO | } | 20260505 19:52:18.920 | INFO | Perf report for the last 5.13924 seconds: | 20260505 19:52:18.921 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:52:18.921 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:18.921 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:18.921 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:18.921 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260505 19:52:18.921 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:18.921 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:18.921 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:52:18.921 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:18.921 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:18.921 | INFO | Uploaded 36GiB to GPU in 2.12034s (16.9784GiB/s) | 20260505 19:52:18.921 | INFO | Downloaded 34GiB to GPU in 2.02152s (16.819GiB/s) | 20260505 19:52:24.054 | INFO | Analyzing chunk frames = [5862912,5863424) | 20260505 19:52:24.054 | INFO | MemInfo: { | 20260505 19:52:24.054 | INFO | "gpu" : " 17 GB", | 20260505 19:52:24.054 | INFO | "huge" : "", | 20260505 19:52:24.054 | INFO | "huge_shared" : " 27 GB", | 20260505 19:52:24.054 | INFO | "malloc" : " 14 GB", | 20260505 19:52:24.054 | INFO | "pinned" : " 29 GB" | 20260505 19:52:24.054 | INFO | } | 20260505 19:52:24.054 | INFO | Perf report for the last 5.13399 seconds: | 20260505 19:52:24.055 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:52:24.055 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:24.055 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:24.055 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:24.055 | INFO | ANALYSIS : 62.5%, 3.0, 97.0ms, 0 | 20260505 19:52:24.055 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:24.055 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:24.055 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:52:24.055 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:24.055 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:24.055 | INFO | Uploaded 36GiB to GPU in 2.19985s (16.3648GiB/s) | 20260505 19:52:24.055 | INFO | Downloaded 34GiB to GPU in 2.043s (16.6422GiB/s) | 20260505 19:52:29.189 | INFO | Analyzing chunk frames = [5863424,5863936) | 20260505 19:52:29.189 | INFO | MemInfo: { | 20260505 19:52:29.189 | INFO | "gpu" : " 17 GB", | 20260505 19:52:29.189 | INFO | "huge" : "", | 20260505 19:52:29.189 | INFO | "huge_shared" : " 27 GB", | 20260505 19:52:29.189 | INFO | "malloc" : " 14 GB", | 20260505 19:52:29.189 | INFO | "pinned" : " 29 GB" | 20260505 19:52:29.189 | INFO | } | 20260505 19:52:29.189 | INFO | Perf report for the last 5.13467 seconds: | 20260505 19:52:29.189 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:52:29.189 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:29.189 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:29.189 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:29.189 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 19:52:29.189 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:29.189 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:29.189 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:52:29.189 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:29.189 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:29.189 | INFO | Uploaded 36GiB to GPU in 2.12988s (16.9024GiB/s) | 20260505 19:52:29.189 | INFO | Downloaded 34GiB to GPU in 2.06358s (16.4762GiB/s) | 20260505 19:52:34.335 | INFO | Analyzing chunk frames = [5863936,5864448) | 20260505 19:52:34.335 | INFO | MemInfo: { | 20260505 19:52:34.335 | INFO | "gpu" : " 17 GB", | 20260505 19:52:34.335 | INFO | "huge" : "", | 20260505 19:52:34.335 | INFO | "huge_shared" : " 27 GB", | 20260505 19:52:34.335 | INFO | "malloc" : " 14 GB", | 20260505 19:52:34.335 | INFO | "pinned" : " 29 GB" | 20260505 19:52:34.335 | INFO | } | 20260505 19:52:34.335 | INFO | Perf report for the last 5.14597 seconds: | 20260505 19:52:34.335 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:52:34.335 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:34.335 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:34.335 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:34.335 | INFO | ANALYSIS : 62.1%, 3.0, 97.1ms, 0 | 20260505 19:52:34.335 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:34.335 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:34.335 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:52:34.335 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:34.335 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:34.336 | INFO | Uploaded 36GiB to GPU in 2.12318s (16.9557GiB/s) | 20260505 19:52:34.336 | INFO | Downloaded 34GiB to GPU in 2.04875s (16.5955GiB/s) | 20260505 19:52:39.481 | INFO | Analyzing chunk frames = [5864448,5864960) | 20260505 19:52:39.481 | INFO | MemInfo: { | 20260505 19:52:39.481 | INFO | "gpu" : " 17 GB", | 20260505 19:52:39.481 | INFO | "huge" : "", | 20260505 19:52:39.481 | INFO | "huge_shared" : " 27 GB", | 20260505 19:52:39.481 | INFO | "malloc" : " 14 GB", | 20260505 19:52:39.481 | INFO | "pinned" : " 29 GB" | 20260505 19:52:39.481 | INFO | } | 20260505 19:52:39.481 | INFO | Perf report for the last 5.14583 seconds: | 20260505 19:52:39.481 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:52:39.481 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:39.481 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:39.481 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:39.481 | INFO | ANALYSIS : 62.9%, 3.0, 97.3ms, 0 | 20260505 19:52:39.481 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:39.481 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:39.481 | INFO | PRE_HQ : 28.1%, 1.0, 14.5ms, 0 | 20260505 19:52:39.481 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:39.481 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:39.481 | INFO | Uploaded 36GiB to GPU in 2.10579s (17.0957GiB/s) | 20260505 19:52:39.481 | INFO | Downloaded 34GiB to GPU in 2.13933s (15.8928GiB/s) | 20260505 19:52:44.623 | INFO | Analyzing chunk frames = [5864960,5865472) | 20260505 19:52:44.623 | INFO | MemInfo: { | 20260505 19:52:44.623 | INFO | "gpu" : " 17 GB", | 20260505 19:52:44.623 | INFO | "huge" : "", | 20260505 19:52:44.623 | INFO | "huge_shared" : " 27 GB", | 20260505 19:52:44.623 | INFO | "malloc" : " 14 GB", | 20260505 19:52:44.623 | INFO | "pinned" : " 29 GB" | 20260505 19:52:44.623 | INFO | } | 20260505 19:52:44.623 | INFO | Perf report for the last 5.14209 seconds: | 20260505 19:52:44.623 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:52:44.623 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:44.623 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:44.623 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:44.623 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260505 19:52:44.623 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:44.623 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:44.623 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:52:44.623 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:44.623 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:44.623 | INFO | Uploaded 36GiB to GPU in 2.12535s (16.9384GiB/s) | 20260505 19:52:44.623 | INFO | Downloaded 34GiB to GPU in 2.01906s (16.8395GiB/s) | 20260505 19:52:44.624 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 19:52:44.624 | INFO | Total mem usage: 15.3 GiB | 20260505 19:52:44.624 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 19:52:44.624 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.1%) | 20260505 19:52:44.624 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 19:52:44.624 | INFO | High Water Breakdown: | 20260505 19:52:44.624 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 19:52:44.624 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 19:52:44.624 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 19:52:44.624 | INFO | ------------------------------------------ | 20260505 19:52:44.624 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:52:44.624 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 19:52:44.624 | INFO | Total mem usage: 26.5 GiB | 20260505 19:52:44.624 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 19:52:44.624 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 19:52:44.624 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 19:52:44.624 | INFO | High Water Breakdown: | 20260505 19:52:44.624 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:52:44.624 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:52:44.624 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:52:44.624 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:52:44.624 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:52:44.624 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:52:44.624 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 19:52:44.624 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 19:52:44.624 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:52:44.624 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 19:52:44.624 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 19:52:44.624 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 19:52:44.624 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 19:52:44.624 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 19:52:44.624 | INFO | ------------------------------------------ | 20260505 19:52:44.624 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:52:44.624 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 19:52:44.624 | INFO | Total mem usage: 25.6 GiB | 20260505 19:52:44.624 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 19:52:44.624 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 19:52:44.624 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 19:52:44.624 | INFO | High Water Breakdown: | 20260505 19:52:44.624 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 19:52:44.624 | INFO | ------------------------------------------ | 20260505 19:52:44.624 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:52:44.624 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 19:52:44.624 | INFO | Total mem usage: 15.7 GiB | 20260505 19:52:44.624 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 19:52:44.624 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 19:52:44.624 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 19:52:44.624 | INFO | High Water Breakdown: | 20260505 19:52:44.624 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 19:52:44.624 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 19:52:44.624 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 19:52:44.624 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 19:52:44.624 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 19:52:44.624 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 19:52:44.624 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 19:52:44.624 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 19:52:44.624 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 19:52:44.624 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:52:44.624 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 19:52:44.624 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:52:44.624 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 19:52:44.624 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 19:52:44.624 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 19:52:44.624 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 19:52:44.624 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:52:44.624 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 19:52:44.624 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:52:44.624 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:52:44.624 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:52:44.624 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 19:52:44.624 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 19:52:44.624 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:52:44.624 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 19:52:44.624 | INFO | ------------------------------------------ | 20260505 19:52:44.624 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:52:44.625 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 19:52:44.625 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:52:44.625 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:52:44.625 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.47e+01, 1.33e+01 / 1.98e+00 | 20260505 19:52:44.625 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.03e-01, 1.14e-02 / 2.91e-03 | 20260505 19:52:44.625 | INFO | PerfReport: 11.79%, Baseline -- 3.71e+00 / 4.54e+00, 5.66e-01 / 1.49e-01 | 20260505 19:52:44.625 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.53e+00, 1.89e-01 / 9.92e-02 | 20260505 19:52:44.625 | INFO | PerfReport: 15.97%, DME -- 4.02e+01 / 4.44e+01, 4.20e+00 / 2.04e+00 | 20260505 19:52:44.625 | INFO | PerfReport: 57.00%, FrameLabeling -- 1.79e+01 / 1.94e+01, 1.45e+00 / 1.07e+00 | 20260505 19:52:44.625 | INFO | PerfReport: 7.27%, PulseAccumulating -- 2.29e+00 / 2.85e+00, 4.40e-01 / 2.21e-01 | 20260505 19:52:44.625 | INFO | PerfReport: 3.28%, Metrics -- 1.03e+00 / 1.37e+00, 2.86e-01 / 1.60e-01 | 20260505 19:52:44.625 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.58e-01 / 4.20e-01, 6.29e-02 / 3.03e-02 | 20260505 19:52:44.625 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 19:52:44.625 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:52:44.625 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:52:44.625 | INFO | PerfReport: Full Execution -- 9.95e+01 / -nan, 2.56e+01 / 0.00e+00 | 20260505 19:52:44.625 | INFO | PerfReport: 22.47%, Upload -- 2.24e+01 / 0.00e+00, 7.77e+00 / 0.00e+00 | 20260505 19:52:44.625 | INFO | PerfReport: 21.76%, Download -- 2.17e+01 / 0.00e+00, 7.73e+00 / 0.00e+00 | 20260505 19:52:44.625 | INFO | PerfReport: 31.64%, Compute -- 3.15e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260505 19:52:44.625 | INFO | PerfReport: 2.36%, LockUpload -- 2.35e+00 / 0.00e+00, 8.16e+00 / 0.00e+00 | 20260505 19:52:44.625 | INFO | PerfReport: 1.36%, LockDownload -- 1.35e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260505 19:52:44.625 | INFO | PerfReport: 20.40%, LockCompute -- 2.03e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 19:52:44.625 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.95e-02 / 0.00e+00 | 20260505 19:52:44.756 | INFO | Starting a round of Baz IO | 20260505 19:52:47.922 | INFO | Populating RTMetrics report | 20260505 19:52:47.922 | INFO | RTMetrics report populated | 20260505 19:52:47.922 | INFO | Opening tmp RTMetrics json file | 20260505 19:52:47.923 | INFO | Serializing RTMetrics to json | 20260505 19:52:47.924 | INFO | Serialization done, swapping tmp file now... | 20260505 19:52:47.925 | INFO | RTMetrics done | 20260505 19:52:48.017 | INFO | Finished a round of Baz IO: Wrote 2.032 GiB in 3.26 seconds | 20260505 19:52:48.017 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.73% / 0.00% / 78.27% | 20260505 19:52:48.017 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.193401% / 7.94729e-06% / 8.74201e-05% | 20260505 19:52:48.017 | INFO | Baz IO overall: | 20260505 19:52:48.017 | INFO | BytesWritten : 3428453487164 | 20260505 19:52:48.017 | INFO | NumEvents : 1621716545614 | 20260505 19:52:49.755 | INFO | Analyzing chunk frames = [5865472,5865984) | 20260505 19:52:49.755 | INFO | MemInfo: { | 20260505 19:52:49.755 | INFO | "gpu" : " 17 GB", | 20260505 19:52:49.755 | INFO | "huge" : "", | 20260505 19:52:49.755 | INFO | "huge_shared" : " 27 GB", | 20260505 19:52:49.755 | INFO | "malloc" : " 14 GB", | 20260505 19:52:49.755 | INFO | "pinned" : " 29 GB" | 20260505 19:52:49.755 | INFO | } | 20260505 19:52:49.755 | INFO | Perf report for the last 5.13177 seconds: | 20260505 19:52:49.755 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:52:49.755 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:49.755 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:49.755 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:49.755 | INFO | ANALYSIS : 63.2%, 3.0, 98.4ms, 0 | 20260505 19:52:49.755 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:49.755 | INFO | RT_METRICS : 44.2%, 1.0, 23.1ms, 0 | 20260505 19:52:49.755 | INFO | PRE_HQ : 60.2%, 1.8, 55.5ms, 0 | 20260505 19:52:49.755 | INFO | BAZWRITER : 63.0%, 3.7, 120.0ms, 0 | 20260505 19:52:49.755 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 19:52:49.755 | INFO | Uploaded 36GiB to GPU in 2.16663s (16.6157GiB/s) | 20260505 19:52:49.755 | INFO | Downloaded 38GiB to GPU in 2.23319s (17.016GiB/s) | 20260505 19:52:54.885 | INFO | Analyzing chunk frames = [5865984,5866496) | 20260505 19:52:54.885 | INFO | MemInfo: { | 20260505 19:52:54.885 | INFO | "gpu" : " 17 GB", | 20260505 19:52:54.885 | INFO | "huge" : "", | 20260505 19:52:54.885 | INFO | "huge_shared" : " 27 GB", | 20260505 19:52:54.885 | INFO | "malloc" : " 14 GB", | 20260505 19:52:54.885 | INFO | "pinned" : " 29 GB" | 20260505 19:52:54.885 | INFO | } | 20260505 19:52:54.885 | INFO | Perf report for the last 5.13038 seconds: | 20260505 19:52:54.885 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:52:54.885 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:54.885 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:54.885 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:54.885 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 19:52:54.885 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:54.885 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:54.885 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:52:54.885 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:54.885 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:52:54.886 | INFO | Uploaded 36GiB to GPU in 2.14459s (16.7864GiB/s) | 20260505 19:52:54.886 | INFO | Downloaded 34GiB to GPU in 2.05261s (16.5643GiB/s) | 20260505 19:53:00.037 | INFO | Analyzing chunk frames = [5866496,5867008) | 20260505 19:53:00.037 | INFO | MemInfo: { | 20260505 19:53:00.037 | INFO | "gpu" : " 17 GB", | 20260505 19:53:00.037 | INFO | "huge" : "", | 20260505 19:53:00.037 | INFO | "huge_shared" : " 27 GB", | 20260505 19:53:00.037 | INFO | "malloc" : " 14 GB", | 20260505 19:53:00.037 | INFO | "pinned" : " 29 GB" | 20260505 19:53:00.037 | INFO | } | 20260505 19:53:00.037 | INFO | Perf report for the last 5.15212 seconds: | 20260505 19:53:00.037 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:53:00.037 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:00.037 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:00.037 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:00.037 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260505 19:53:00.037 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:00.037 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:00.037 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 19:53:00.037 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:00.037 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:00.038 | INFO | Uploaded 36GiB to GPU in 2.19789s (16.3794GiB/s) | 20260505 19:53:00.038 | INFO | Downloaded 34GiB to GPU in 2.03737s (16.6882GiB/s) | 20260505 19:53:05.181 | INFO | Analyzing chunk frames = [5867008,5867520) | 20260505 19:53:05.181 | INFO | MemInfo: { | 20260505 19:53:05.181 | INFO | "gpu" : " 17 GB", | 20260505 19:53:05.181 | INFO | "huge" : "", | 20260505 19:53:05.181 | INFO | "huge_shared" : " 27 GB", | 20260505 19:53:05.181 | INFO | "malloc" : " 14 GB", | 20260505 19:53:05.181 | INFO | "pinned" : " 29 GB" | 20260505 19:53:05.181 | INFO | } | 20260505 19:53:05.181 | INFO | Perf report for the last 5.14359 seconds: | 20260505 19:53:05.181 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:53:05.181 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:05.181 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:05.181 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:05.181 | INFO | ANALYSIS : 62.4%, 3.0, 97.0ms, 0 | 20260505 19:53:05.181 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:05.181 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:05.181 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:53:05.181 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:05.181 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:05.181 | INFO | Uploaded 36GiB to GPU in 2.20849s (16.3007GiB/s) | 20260505 19:53:05.181 | INFO | Downloaded 34GiB to GPU in 2.04135s (16.6556GiB/s) | 20260505 19:53:10.314 | INFO | Analyzing chunk frames = [5867520,5868032) | 20260505 19:53:10.314 | INFO | MemInfo: { | 20260505 19:53:10.314 | INFO | "gpu" : " 17 GB", | 20260505 19:53:10.314 | INFO | "huge" : "", | 20260505 19:53:10.314 | INFO | "huge_shared" : " 27 GB", | 20260505 19:53:10.314 | INFO | "malloc" : " 14 GB", | 20260505 19:53:10.314 | INFO | "pinned" : " 29 GB" | 20260505 19:53:10.314 | INFO | } | 20260505 19:53:10.314 | INFO | Perf report for the last 5.13323 seconds: | 20260505 19:53:10.314 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:53:10.314 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:10.314 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:10.314 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:10.314 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260505 19:53:10.314 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:10.314 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:10.314 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:53:10.314 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:10.314 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:10.314 | INFO | Uploaded 36GiB to GPU in 2.16431s (16.6335GiB/s) | 20260505 19:53:10.314 | INFO | Downloaded 34GiB to GPU in 2.07081s (16.4187GiB/s) | 20260505 19:53:15.451 | INFO | Analyzing chunk frames = [5868032,5868544) | 20260505 19:53:15.451 | INFO | MemInfo: { | 20260505 19:53:15.451 | INFO | "gpu" : " 17 GB", | 20260505 19:53:15.451 | INFO | "huge" : "", | 20260505 19:53:15.451 | INFO | "huge_shared" : " 27 GB", | 20260505 19:53:15.451 | INFO | "malloc" : " 14 GB", | 20260505 19:53:15.451 | INFO | "pinned" : " 29 GB" | 20260505 19:53:15.451 | INFO | } | 20260505 19:53:15.451 | INFO | Perf report for the last 5.13705 seconds: | 20260505 19:53:15.451 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:53:15.451 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:15.451 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:15.451 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:15.451 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 19:53:15.451 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:15.451 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:15.451 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 19:53:15.451 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:15.451 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:15.451 | INFO | Uploaded 36GiB to GPU in 2.14522s (16.7815GiB/s) | 20260505 19:53:15.452 | INFO | Downloaded 34GiB to GPU in 2.0588s (16.5145GiB/s) | 20260505 19:53:20.587 | INFO | Analyzing chunk frames = [5868544,5869056) | 20260505 19:53:20.587 | INFO | MemInfo: { | 20260505 19:53:20.587 | INFO | "gpu" : " 17 GB", | 20260505 19:53:20.587 | INFO | "huge" : "", | 20260505 19:53:20.587 | INFO | "huge_shared" : " 27 GB", | 20260505 19:53:20.587 | INFO | "malloc" : " 14 GB", | 20260505 19:53:20.587 | INFO | "pinned" : " 29 GB" | 20260505 19:53:20.588 | INFO | } | 20260505 19:53:20.588 | INFO | Perf report for the last 5.13633 seconds: | 20260505 19:53:20.588 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:53:20.588 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:20.588 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:20.588 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:20.588 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 19:53:20.588 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:20.588 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:20.588 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 19:53:20.588 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:20.588 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:20.588 | INFO | Uploaded 36GiB to GPU in 2.13069s (16.896GiB/s) | 20260505 19:53:20.588 | INFO | Downloaded 34GiB to GPU in 2.13333s (15.9375GiB/s) | 20260505 19:53:25.731 | INFO | Analyzing chunk frames = [5869056,5869568) | 20260505 19:53:25.731 | INFO | MemInfo: { | 20260505 19:53:25.731 | INFO | "gpu" : " 17 GB", | 20260505 19:53:25.731 | INFO | "huge" : "", | 20260505 19:53:25.731 | INFO | "huge_shared" : " 27 GB", | 20260505 19:53:25.731 | INFO | "malloc" : " 14 GB", | 20260505 19:53:25.731 | INFO | "pinned" : " 29 GB" | 20260505 19:53:25.731 | INFO | } | 20260505 19:53:25.731 | INFO | Perf report for the last 5.14324 seconds: | 20260505 19:53:25.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:53:25.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:25.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:25.731 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:25.731 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 19:53:25.731 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:25.731 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:25.731 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:53:25.731 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:25.731 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:25.731 | INFO | Uploaded 36GiB to GPU in 2.20468s (16.3289GiB/s) | 20260505 19:53:25.731 | INFO | Downloaded 34GiB to GPU in 2.0428s (16.6438GiB/s) | 20260505 19:53:25.865 | INFO | Starting a round of Baz IO | 20260505 19:53:29.029 | INFO | Populating RTMetrics report | 20260505 19:53:29.029 | INFO | RTMetrics report populated | 20260505 19:53:29.029 | INFO | Opening tmp RTMetrics json file | 20260505 19:53:29.030 | INFO | Serializing RTMetrics to json | 20260505 19:53:29.031 | INFO | Serialization done, swapping tmp file now... | 20260505 19:53:29.031 | INFO | RTMetrics done | 20260505 19:53:29.114 | INFO | Finished a round of Baz IO: Wrote 2.031 GiB in 3.249 seconds | 20260505 19:53:29.114 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.73% / 0.00% / 78.27% | 20260505 19:53:29.114 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.192253% / 7.94729e-06% / 8.34465e-05% | 20260505 19:53:29.114 | INFO | Baz IO overall: | 20260505 19:53:29.114 | INFO | BytesWritten : 3430634551507 | 20260505 19:53:29.114 | INFO | NumEvents : 1622727674421 | 20260505 19:53:30.869 | INFO | Analyzing chunk frames = [5869568,5870080) | 20260505 19:53:30.869 | INFO | MemInfo: { | 20260505 19:53:30.869 | INFO | "gpu" : " 17 GB", | 20260505 19:53:30.869 | INFO | "huge" : "", | 20260505 19:53:30.869 | INFO | "huge_shared" : " 27 GB", | 20260505 19:53:30.869 | INFO | "malloc" : " 14 GB", | 20260505 19:53:30.869 | INFO | "pinned" : " 29 GB" | 20260505 19:53:30.869 | INFO | } | 20260505 19:53:30.869 | INFO | Perf report for the last 5.1383 seconds: | 20260505 19:53:30.869 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:53:30.869 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:30.869 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:30.869 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:30.869 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 19:53:30.869 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:30.869 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260505 19:53:30.869 | INFO | PRE_HQ : 60.1%, 1.8, 55.3ms, 0 | 20260505 19:53:30.869 | INFO | BAZWRITER : 62.7%, 3.7, 119.0ms, 0 | 20260505 19:53:30.869 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 19:53:30.869 | INFO | Uploaded 36GiB to GPU in 2.16471s (16.6304GiB/s) | 20260505 19:53:30.869 | INFO | Downloaded 38GiB to GPU in 2.26098s (16.8069GiB/s) | 20260505 19:53:36.002 | INFO | Analyzing chunk frames = [5870080,5870592) | 20260505 19:53:36.002 | INFO | MemInfo: { | 20260505 19:53:36.002 | INFO | "gpu" : " 17 GB", | 20260505 19:53:36.002 | INFO | "huge" : "", | 20260505 19:53:36.002 | INFO | "huge_shared" : " 27 GB", | 20260505 19:53:36.002 | INFO | "malloc" : " 14 GB", | 20260505 19:53:36.002 | INFO | "pinned" : " 29 GB" | 20260505 19:53:36.002 | INFO | } | 20260505 19:53:36.003 | INFO | Perf report for the last 5.13341 seconds: | 20260505 19:53:36.003 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:53:36.003 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:36.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:36.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:36.003 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 19:53:36.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:36.003 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:36.003 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:53:36.003 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:36.003 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:36.003 | INFO | Uploaded 36GiB to GPU in 2.14834s (16.7571GiB/s) | 20260505 19:53:36.003 | INFO | Downloaded 34GiB to GPU in 2.05023s (16.5835GiB/s) | 20260505 19:53:41.143 | INFO | Analyzing chunk frames = [5870592,5871104) | 20260505 19:53:41.143 | INFO | MemInfo: { | 20260505 19:53:41.143 | INFO | "gpu" : " 17 GB", | 20260505 19:53:41.143 | INFO | "huge" : "", | 20260505 19:53:41.143 | INFO | "huge_shared" : " 27 GB", | 20260505 19:53:41.143 | INFO | "malloc" : " 14 GB", | 20260505 19:53:41.143 | INFO | "pinned" : " 29 GB" | 20260505 19:53:41.143 | INFO | } | 20260505 19:53:41.143 | INFO | Perf report for the last 5.1408 seconds: | 20260505 19:53:41.143 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:53:41.143 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:41.143 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:41.143 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:41.143 | INFO | ANALYSIS : 62.7%, 3.0, 97.5ms, 0 | 20260505 19:53:41.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:41.143 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:41.143 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:53:41.143 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:41.143 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:41.144 | INFO | Uploaded 36GiB to GPU in 2.20643s (16.316GiB/s) | 20260505 19:53:41.144 | INFO | Downloaded 34GiB to GPU in 2.03735s (16.6883GiB/s) | 20260505 19:53:46.292 | INFO | Analyzing chunk frames = [5871104,5871616) | 20260505 19:53:46.292 | INFO | MemInfo: { | 20260505 19:53:46.293 | INFO | "gpu" : " 17 GB", | 20260505 19:53:46.293 | INFO | "huge" : "", | 20260505 19:53:46.293 | INFO | "huge_shared" : " 27 GB", | 20260505 19:53:46.293 | INFO | "malloc" : " 14 GB", | 20260505 19:53:46.293 | INFO | "pinned" : " 29 GB" | 20260505 19:53:46.293 | INFO | } | 20260505 19:53:46.293 | INFO | Perf report for the last 5.14935 seconds: | 20260505 19:53:46.293 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:53:46.293 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:46.293 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:46.293 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:46.293 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260505 19:53:46.293 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:46.293 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:46.293 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:53:46.293 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:46.293 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:46.293 | INFO | Uploaded 36GiB to GPU in 2.18432s (16.4811GiB/s) | 20260505 19:53:46.293 | INFO | Downloaded 34GiB to GPU in 2.0739s (16.3943GiB/s) | 20260505 19:53:51.432 | INFO | Analyzing chunk frames = [5871616,5872128) | 20260505 19:53:51.432 | INFO | MemInfo: { | 20260505 19:53:51.432 | INFO | "gpu" : " 17 GB", | 20260505 19:53:51.432 | INFO | "huge" : "", | 20260505 19:53:51.432 | INFO | "huge_shared" : " 27 GB", | 20260505 19:53:51.432 | INFO | "malloc" : " 14 GB", | 20260505 19:53:51.432 | INFO | "pinned" : " 29 GB" | 20260505 19:53:51.432 | INFO | } | 20260505 19:53:51.432 | INFO | Perf report for the last 5.13963 seconds: | 20260505 19:53:51.432 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:53:51.432 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:51.432 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:51.432 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:51.432 | INFO | ANALYSIS : 61.6%, 3.0, 96.2ms, 0 | 20260505 19:53:51.432 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:51.432 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:51.432 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:53:51.432 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:51.432 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:51.433 | INFO | Uploaded 36GiB to GPU in 2.12819s (16.9158GiB/s) | 20260505 19:53:51.433 | INFO | Downloaded 34GiB to GPU in 2.06108s (16.4962GiB/s) | 20260505 19:53:56.569 | INFO | Analyzing chunk frames = [5872128,5872640) | 20260505 19:53:56.569 | INFO | MemInfo: { | 20260505 19:53:56.569 | INFO | "gpu" : " 17 GB", | 20260505 19:53:56.569 | INFO | "huge" : "", | 20260505 19:53:56.569 | INFO | "huge_shared" : " 27 GB", | 20260505 19:53:56.569 | INFO | "malloc" : " 14 GB", | 20260505 19:53:56.569 | INFO | "pinned" : " 29 GB" | 20260505 19:53:56.569 | INFO | } | 20260505 19:53:56.569 | INFO | Perf report for the last 5.13702 seconds: | 20260505 19:53:56.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:53:56.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:56.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:56.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:56.569 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 19:53:56.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:56.569 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:56.570 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 19:53:56.570 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:56.570 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:53:56.570 | INFO | Uploaded 36GiB to GPU in 2.16131s (16.6566GiB/s) | 20260505 19:53:56.570 | INFO | Downloaded 34GiB to GPU in 2.06735s (16.4462GiB/s) | 20260505 19:54:01.702 | INFO | Analyzing chunk frames = [5872640,5873152) | 20260505 19:54:01.702 | INFO | MemInfo: { | 20260505 19:54:01.702 | INFO | "gpu" : " 17 GB", | 20260505 19:54:01.702 | INFO | "huge" : "", | 20260505 19:54:01.702 | INFO | "huge_shared" : " 27 GB", | 20260505 19:54:01.702 | INFO | "malloc" : " 14 GB", | 20260505 19:54:01.702 | INFO | "pinned" : " 29 GB" | 20260505 19:54:01.702 | INFO | } | 20260505 19:54:01.702 | INFO | Perf report for the last 5.13233 seconds: | 20260505 19:54:01.702 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:54:01.702 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:01.702 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:01.702 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:01.702 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 19:54:01.702 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:01.702 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:01.702 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 19:54:01.702 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:01.702 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:01.702 | INFO | Uploaded 36GiB to GPU in 2.12385s (16.9503GiB/s) | 20260505 19:54:01.702 | INFO | Downloaded 34GiB to GPU in 2.15284s (15.7931GiB/s) | 20260505 19:54:06.840 | INFO | Analyzing chunk frames = [5873152,5873664) | 20260505 19:54:06.840 | INFO | MemInfo: { | 20260505 19:54:06.840 | INFO | "gpu" : " 17 GB", | 20260505 19:54:06.840 | INFO | "huge" : "", | 20260505 19:54:06.840 | INFO | "huge_shared" : " 27 GB", | 20260505 19:54:06.840 | INFO | "malloc" : " 14 GB", | 20260505 19:54:06.840 | INFO | "pinned" : " 29 GB" | 20260505 19:54:06.840 | INFO | } | 20260505 19:54:06.840 | INFO | Perf report for the last 5.1386 seconds: | 20260505 19:54:06.840 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:54:06.840 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:06.840 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:06.840 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:06.840 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260505 19:54:06.840 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:06.840 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:06.840 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:54:06.840 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:06.840 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:06.841 | INFO | Uploaded 36GiB to GPU in 2.19933s (16.3686GiB/s) | 20260505 19:54:06.841 | INFO | Downloaded 34GiB to GPU in 2.04108s (16.6578GiB/s) | 20260505 19:54:06.841 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 19:54:06.841 | INFO | Total mem usage: 15.3 GiB | 20260505 19:54:06.841 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 19:54:06.841 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.11%) | 20260505 19:54:06.841 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 19:54:06.841 | INFO | High Water Breakdown: | 20260505 19:54:06.841 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 19:54:06.841 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 19:54:06.841 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 19:54:06.841 | INFO | ------------------------------------------ | 20260505 19:54:06.841 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:54:06.841 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 19:54:06.841 | INFO | Total mem usage: 26.5 GiB | 20260505 19:54:06.841 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 19:54:06.841 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 19:54:06.841 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 19:54:06.841 | INFO | High Water Breakdown: | 20260505 19:54:06.841 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:54:06.841 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:54:06.841 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:54:06.841 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:54:06.841 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:54:06.841 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:54:06.841 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 19:54:06.841 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 19:54:06.841 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:54:06.841 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 19:54:06.841 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 19:54:06.841 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 19:54:06.841 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 19:54:06.841 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 19:54:06.841 | INFO | ------------------------------------------ | 20260505 19:54:06.841 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:54:06.841 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 19:54:06.841 | INFO | Total mem usage: 25.6 GiB | 20260505 19:54:06.841 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 19:54:06.841 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 19:54:06.841 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 19:54:06.841 | INFO | High Water Breakdown: | 20260505 19:54:06.841 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 19:54:06.841 | INFO | ------------------------------------------ | 20260505 19:54:06.841 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:54:06.841 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 19:54:06.841 | INFO | Total mem usage: 15.7 GiB | 20260505 19:54:06.841 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 19:54:06.841 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 19:54:06.841 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 19:54:06.841 | INFO | High Water Breakdown: | 20260505 19:54:06.841 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 19:54:06.841 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 19:54:06.841 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 19:54:06.841 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 19:54:06.841 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 19:54:06.841 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 19:54:06.841 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 19:54:06.841 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 19:54:06.841 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 19:54:06.841 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:54:06.841 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 19:54:06.841 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:54:06.841 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 19:54:06.841 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 19:54:06.841 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 19:54:06.841 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 19:54:06.841 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:54:06.841 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 19:54:06.841 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:54:06.841 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:54:06.841 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:54:06.841 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 19:54:06.841 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 19:54:06.841 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:54:06.841 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 19:54:06.842 | INFO | ------------------------------------------ | 20260505 19:54:06.842 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:54:06.842 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 19:54:06.842 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:54:06.842 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:54:06.842 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.44e+01, 1.35e+01 / 2.15e+00 | 20260505 19:54:06.842 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.04e-01, 1.12e-02 / 7.39e-03 | 20260505 19:54:06.842 | INFO | PerfReport: 11.78%, Baseline -- 3.70e+00 / 4.54e+00, 5.51e-01 / 1.55e-01 | 20260505 19:54:06.842 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.51e+00, 1.81e-01 / 1.04e-01 | 20260505 19:54:06.842 | INFO | PerfReport: 15.94%, DME -- 4.00e+01 / 4.41e+01, 4.01e+00 / 2.16e+00 | 20260505 19:54:06.842 | INFO | PerfReport: 57.01%, FrameLabeling -- 1.79e+01 / 1.95e+01, 1.39e+00 / 1.00e+00 | 20260505 19:54:06.842 | INFO | PerfReport: 7.31%, PulseAccumulating -- 2.30e+00 / 2.83e+00, 4.41e-01 / 2.44e-01 | 20260505 19:54:06.842 | INFO | PerfReport: 3.27%, Metrics -- 1.03e+00 / 1.33e+00, 2.81e-01 / 1.36e-01 | 20260505 19:54:06.842 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.58e-01 / 4.28e-01, 6.33e-02 / 3.56e-02 | 20260505 19:54:06.842 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 19:54:06.842 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:54:06.842 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:54:06.842 | INFO | PerfReport: Full Execution -- 9.96e+01 / -nan, 2.55e+01 / 0.00e+00 | 20260505 19:54:06.842 | INFO | PerfReport: 22.67%, Upload -- 2.26e+01 / 0.00e+00, 7.91e+00 / 0.00e+00 | 20260505 19:54:06.842 | INFO | PerfReport: 21.84%, Download -- 2.18e+01 / 0.00e+00, 7.77e+00 / 0.00e+00 | 20260505 19:54:06.842 | INFO | PerfReport: 31.53%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 19:54:06.842 | INFO | PerfReport: 2.42%, LockUpload -- 2.41e+00 / 0.00e+00, 8.05e+00 / 0.00e+00 | 20260505 19:54:06.842 | INFO | PerfReport: 1.35%, LockDownload -- 1.34e+00 / 0.00e+00, 2.79e+00 / 0.00e+00 | 20260505 19:54:06.842 | INFO | PerfReport: 20.18%, LockCompute -- 2.01e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 19:54:06.842 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.79e-02 / 0.00e+00 | 20260505 19:54:06.973 | INFO | Starting a round of Baz IO | 20260505 19:54:10.144 | INFO | Populating RTMetrics report | 20260505 19:54:10.145 | INFO | RTMetrics report populated | 20260505 19:54:10.145 | INFO | Opening tmp RTMetrics json file | 20260505 19:54:10.146 | INFO | Serializing RTMetrics to json | 20260505 19:54:10.147 | INFO | Serialization done, swapping tmp file now... | 20260505 19:54:10.147 | INFO | RTMetrics done | 20260505 19:54:10.234 | INFO | Finished a round of Baz IO: Wrote 2.031 GiB in 3.261 seconds | 20260505 19:54:10.235 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.73% / 0.00% / 78.27% | 20260505 19:54:10.235 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.191867% / 0% / 7.94729e-05% | 20260505 19:54:10.235 | INFO | Baz IO overall: | 20260505 19:54:10.235 | INFO | BytesWritten : 3432815440664 | 20260505 19:54:10.235 | INFO | NumEvents : 1623738932845 | 20260505 19:54:11.985 | INFO | Analyzing chunk frames = [5873664,5874176) | 20260505 19:54:11.985 | INFO | MemInfo: { | 20260505 19:54:11.985 | INFO | "gpu" : " 17 GB", | 20260505 19:54:11.985 | INFO | "huge" : "", | 20260505 19:54:11.985 | INFO | "huge_shared" : " 27 GB", | 20260505 19:54:11.985 | INFO | "malloc" : " 14 GB", | 20260505 19:54:11.985 | INFO | "pinned" : " 29 GB" | 20260505 19:54:11.985 | INFO | } | 20260505 19:54:11.985 | INFO | Perf report for the last 5.14486 seconds: | 20260505 19:54:11.985 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:54:11.985 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:11.985 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:11.985 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:11.985 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 19:54:11.985 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:11.985 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260505 19:54:11.985 | INFO | PRE_HQ : 60.1%, 1.8, 55.2ms, 0 | 20260505 19:54:11.985 | INFO | BAZWRITER : 62.9%, 3.7, 118.6ms, 0 | 20260505 19:54:11.985 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 19:54:11.985 | INFO | Uploaded 36GiB to GPU in 2.1554s (16.7023GiB/s) | 20260505 19:54:11.985 | INFO | Downloaded 38GiB to GPU in 2.24552s (16.9226GiB/s) | 20260505 19:54:17.132 | INFO | Analyzing chunk frames = [5874176,5874688) | 20260505 19:54:17.132 | INFO | MemInfo: { | 20260505 19:54:17.132 | INFO | "gpu" : " 17 GB", | 20260505 19:54:17.132 | INFO | "huge" : "", | 20260505 19:54:17.132 | INFO | "huge_shared" : " 27 GB", | 20260505 19:54:17.132 | INFO | "malloc" : " 14 GB", | 20260505 19:54:17.132 | INFO | "pinned" : " 29 GB" | 20260505 19:54:17.132 | INFO | } | 20260505 19:54:17.132 | INFO | Perf report for the last 5.14696 seconds: | 20260505 19:54:17.132 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:54:17.132 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:17.132 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:17.132 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:17.132 | INFO | ANALYSIS : 62.1%, 3.0, 97.1ms, 0 | 20260505 19:54:17.132 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:17.132 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:17.132 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:54:17.132 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:17.132 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:17.132 | INFO | Uploaded 36GiB to GPU in 2.09987s (17.1439GiB/s) | 20260505 19:54:17.132 | INFO | Downloaded 34GiB to GPU in 2.02733s (16.7709GiB/s) | 20260505 19:54:22.264 | INFO | Analyzing chunk frames = [5874688,5875200) | 20260505 19:54:22.264 | INFO | MemInfo: { | 20260505 19:54:22.264 | INFO | "gpu" : " 17 GB", | 20260505 19:54:22.264 | INFO | "huge" : "", | 20260505 19:54:22.264 | INFO | "huge_shared" : " 27 GB", | 20260505 19:54:22.264 | INFO | "malloc" : " 14 GB", | 20260505 19:54:22.264 | INFO | "pinned" : " 29 GB" | 20260505 19:54:22.264 | INFO | } | 20260505 19:54:22.264 | INFO | Perf report for the last 5.13208 seconds: | 20260505 19:54:22.264 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:54:22.264 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:22.264 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:22.264 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:22.264 | INFO | ANALYSIS : 62.2%, 3.0, 96.6ms, 0 | 20260505 19:54:22.264 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:22.264 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:22.264 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:54:22.264 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:22.264 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:22.265 | INFO | Uploaded 36GiB to GPU in 2.1286s (16.9125GiB/s) | 20260505 19:54:22.265 | INFO | Downloaded 34GiB to GPU in 2.0212s (16.8217GiB/s) | 20260505 19:54:27.399 | INFO | Analyzing chunk frames = [5875200,5875712) | 20260505 19:54:27.399 | INFO | MemInfo: { | 20260505 19:54:27.399 | INFO | "gpu" : " 17 GB", | 20260505 19:54:27.399 | INFO | "huge" : "", | 20260505 19:54:27.399 | INFO | "huge_shared" : " 27 GB", | 20260505 19:54:27.399 | INFO | "malloc" : " 14 GB", | 20260505 19:54:27.399 | INFO | "pinned" : " 29 GB" | 20260505 19:54:27.399 | INFO | } | 20260505 19:54:27.399 | INFO | Perf report for the last 5.13505 seconds: | 20260505 19:54:27.399 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:54:27.399 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:27.399 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:27.399 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:27.399 | INFO | ANALYSIS : 62.3%, 3.0, 96.6ms, 0 | 20260505 19:54:27.399 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:27.399 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:27.399 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 19:54:27.399 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:27.399 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:27.400 | INFO | Uploaded 36GiB to GPU in 2.1721s (16.5738GiB/s) | 20260505 19:54:27.400 | INFO | Downloaded 34GiB to GPU in 2.05766s (16.5236GiB/s) | 20260505 19:54:32.542 | INFO | Analyzing chunk frames = [5875712,5876224) | 20260505 19:54:32.542 | INFO | MemInfo: { | 20260505 19:54:32.542 | INFO | "gpu" : " 17 GB", | 20260505 19:54:32.542 | INFO | "huge" : "", | 20260505 19:54:32.542 | INFO | "huge_shared" : " 27 GB", | 20260505 19:54:32.542 | INFO | "malloc" : " 14 GB", | 20260505 19:54:32.542 | INFO | "pinned" : " 29 GB" | 20260505 19:54:32.542 | INFO | } | 20260505 19:54:32.542 | INFO | Perf report for the last 5.14268 seconds: | 20260505 19:54:32.542 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:54:32.542 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:32.542 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:32.542 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:32.542 | INFO | ANALYSIS : 61.8%, 3.0, 96.6ms, 0 | 20260505 19:54:32.542 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:32.542 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:32.542 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:54:32.542 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:32.542 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:32.542 | INFO | Uploaded 36GiB to GPU in 2.13274s (16.8797GiB/s) | 20260505 19:54:32.542 | INFO | Downloaded 34GiB to GPU in 2.06264s (16.4837GiB/s) | 20260505 19:54:37.681 | INFO | Analyzing chunk frames = [5876224,5876736) | 20260505 19:54:37.681 | INFO | MemInfo: { | 20260505 19:54:37.681 | INFO | "gpu" : " 17 GB", | 20260505 19:54:37.681 | INFO | "huge" : "", | 20260505 19:54:37.681 | INFO | "huge_shared" : " 27 GB", | 20260505 19:54:37.681 | INFO | "malloc" : " 14 GB", | 20260505 19:54:37.681 | INFO | "pinned" : " 29 GB" | 20260505 19:54:37.681 | INFO | } | 20260505 19:54:37.681 | INFO | Perf report for the last 5.13937 seconds: | 20260505 19:54:37.681 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:54:37.681 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:37.681 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:37.681 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:37.681 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 19:54:37.681 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:37.681 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:37.681 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 19:54:37.681 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:37.681 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:37.682 | INFO | Uploaded 36GiB to GPU in 2.14984s (16.7454GiB/s) | 20260505 19:54:37.682 | INFO | Downloaded 34GiB to GPU in 2.06004s (16.5045GiB/s) | 20260505 19:54:42.818 | INFO | Analyzing chunk frames = [5876736,5877248) | 20260505 19:54:42.818 | INFO | MemInfo: { | 20260505 19:54:42.818 | INFO | "gpu" : " 17 GB", | 20260505 19:54:42.818 | INFO | "huge" : "", | 20260505 19:54:42.818 | INFO | "huge_shared" : " 27 GB", | 20260505 19:54:42.818 | INFO | "malloc" : " 14 GB", | 20260505 19:54:42.818 | INFO | "pinned" : " 29 GB" | 20260505 19:54:42.818 | INFO | } | 20260505 19:54:42.818 | INFO | Perf report for the last 5.13625 seconds: | 20260505 19:54:42.818 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:54:42.818 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:42.818 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:42.818 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:42.818 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 19:54:42.818 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:42.818 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:42.818 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 19:54:42.818 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:42.818 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:42.818 | INFO | Uploaded 36GiB to GPU in 2.10721s (17.0842GiB/s) | 20260505 19:54:42.818 | INFO | Downloaded 34GiB to GPU in 2.14158s (15.8761GiB/s) | 20260505 19:54:47.954 | INFO | Analyzing chunk frames = [5877248,5877760) | 20260505 19:54:47.954 | INFO | MemInfo: { | 20260505 19:54:47.954 | INFO | "gpu" : " 17 GB", | 20260505 19:54:47.954 | INFO | "huge" : "", | 20260505 19:54:47.954 | INFO | "huge_shared" : " 27 GB", | 20260505 19:54:47.954 | INFO | "malloc" : " 14 GB", | 20260505 19:54:47.954 | INFO | "pinned" : " 29 GB" | 20260505 19:54:47.954 | INFO | } | 20260505 19:54:47.954 | INFO | Perf report for the last 5.13615 seconds: | 20260505 19:54:47.954 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:54:47.954 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:47.954 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:47.954 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:47.954 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 19:54:47.954 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:47.954 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:47.954 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 19:54:47.954 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:47.954 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:47.954 | INFO | Uploaded 36GiB to GPU in 2.20581s (16.3206GiB/s) | 20260505 19:54:47.954 | INFO | Downloaded 34GiB to GPU in 2.05183s (16.5706GiB/s) | 20260505 19:54:48.086 | INFO | Starting a round of Baz IO | 20260505 19:54:51.259 | INFO | Populating RTMetrics report | 20260505 19:54:51.259 | INFO | RTMetrics report populated | 20260505 19:54:51.259 | INFO | Opening tmp RTMetrics json file | 20260505 19:54:51.262 | INFO | Serializing RTMetrics to json | 20260505 19:54:51.265 | INFO | Serialization done, swapping tmp file now... | 20260505 19:54:51.265 | INFO | RTMetrics done | 20260505 19:54:51.346 | INFO | Finished a round of Baz IO: Wrote 2.032 GiB in 3.26 seconds | 20260505 19:54:51.346 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.73% / 0.00% / 78.27% | 20260505 19:54:51.346 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.191911% / 0% / 0.000103315% | 20260505 19:54:51.346 | INFO | Baz IO overall: | 20260505 19:54:51.346 | INFO | BytesWritten : 3434997448621 | 20260505 19:54:51.346 | INFO | NumEvents : 1624751092629 | 20260505 19:54:53.098 | INFO | Analyzing chunk frames = [5877760,5878272) | 20260505 19:54:53.098 | INFO | MemInfo: { | 20260505 19:54:53.098 | INFO | "gpu" : " 17 GB", | 20260505 19:54:53.098 | INFO | "huge" : "", | 20260505 19:54:53.098 | INFO | "huge_shared" : " 27 GB", | 20260505 19:54:53.098 | INFO | "malloc" : " 14 GB", | 20260505 19:54:53.098 | INFO | "pinned" : " 29 GB" | 20260505 19:54:53.098 | INFO | } | 20260505 19:54:53.098 | INFO | Perf report for the last 5.144 seconds: | 20260505 19:54:53.098 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:54:53.098 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:53.098 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:53.098 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:53.098 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 19:54:53.098 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:53.098 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260505 19:54:53.098 | INFO | PRE_HQ : 60.1%, 1.8, 55.2ms, 0 | 20260505 19:54:53.098 | INFO | BAZWRITER : 62.9%, 3.7, 118.7ms, 0 | 20260505 19:54:53.098 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 19:54:53.098 | INFO | Uploaded 36GiB to GPU in 2.1573s (16.6875GiB/s) | 20260505 19:54:53.098 | INFO | Downloaded 38GiB to GPU in 2.23851s (16.9755GiB/s) | 20260505 19:54:58.243 | INFO | Analyzing chunk frames = [5878272,5878784) | 20260505 19:54:58.243 | INFO | MemInfo: { | 20260505 19:54:58.243 | INFO | "gpu" : " 17 GB", | 20260505 19:54:58.243 | INFO | "huge" : "", | 20260505 19:54:58.243 | INFO | "huge_shared" : " 27 GB", | 20260505 19:54:58.243 | INFO | "malloc" : " 14 GB", | 20260505 19:54:58.243 | INFO | "pinned" : " 29 GB" | 20260505 19:54:58.243 | INFO | } | 20260505 19:54:58.243 | INFO | Perf report for the last 5.14527 seconds: | 20260505 19:54:58.243 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:54:58.243 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:58.243 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:58.243 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:58.243 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260505 19:54:58.243 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:58.243 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:58.243 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 19:54:58.243 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:58.243 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:54:58.243 | INFO | Uploaded 36GiB to GPU in 2.14516s (16.782GiB/s) | 20260505 19:54:58.243 | INFO | Downloaded 34GiB to GPU in 2.04955s (16.589GiB/s) | 20260505 19:55:03.382 | INFO | Analyzing chunk frames = [5878784,5879296) | 20260505 19:55:03.382 | INFO | MemInfo: { | 20260505 19:55:03.382 | INFO | "gpu" : " 17 GB", | 20260505 19:55:03.382 | INFO | "huge" : "", | 20260505 19:55:03.382 | INFO | "huge_shared" : " 27 GB", | 20260505 19:55:03.382 | INFO | "malloc" : " 14 GB", | 20260505 19:55:03.382 | INFO | "pinned" : " 29 GB" | 20260505 19:55:03.382 | INFO | } | 20260505 19:55:03.382 | INFO | Perf report for the last 5.13872 seconds: | 20260505 19:55:03.382 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:55:03.382 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:03.382 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:03.382 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:03.382 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260505 19:55:03.382 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:03.382 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:03.382 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:55:03.382 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:03.382 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:03.382 | INFO | Uploaded 36GiB to GPU in 2.13453s (16.8655GiB/s) | 20260505 19:55:03.382 | INFO | Downloaded 34GiB to GPU in 2.02221s (16.8133GiB/s) | 20260505 19:55:08.513 | INFO | Analyzing chunk frames = [5879296,5879808) | 20260505 19:55:08.513 | INFO | MemInfo: { | 20260505 19:55:08.513 | INFO | "gpu" : " 17 GB", | 20260505 19:55:08.513 | INFO | "huge" : "", | 20260505 19:55:08.513 | INFO | "huge_shared" : " 27 GB", | 20260505 19:55:08.513 | INFO | "malloc" : " 14 GB", | 20260505 19:55:08.513 | INFO | "pinned" : " 29 GB" | 20260505 19:55:08.513 | INFO | } | 20260505 19:55:08.513 | INFO | Perf report for the last 5.1316 seconds: | 20260505 19:55:08.513 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:55:08.513 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:08.513 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:08.514 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:08.514 | INFO | ANALYSIS : 62.5%, 3.0, 97.0ms, 0 | 20260505 19:55:08.514 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:08.514 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:08.514 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:55:08.514 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:08.514 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:08.514 | INFO | Uploaded 36GiB to GPU in 2.20874s (16.2989GiB/s) | 20260505 19:55:08.514 | INFO | Downloaded 34GiB to GPU in 2.04786s (16.6027GiB/s) | 20260505 19:55:13.651 | INFO | Analyzing chunk frames = [5879808,5880320) | 20260505 19:55:13.651 | INFO | MemInfo: { | 20260505 19:55:13.651 | INFO | "gpu" : " 17 GB", | 20260505 19:55:13.651 | INFO | "huge" : "", | 20260505 19:55:13.651 | INFO | "huge_shared" : " 27 GB", | 20260505 19:55:13.651 | INFO | "malloc" : " 14 GB", | 20260505 19:55:13.651 | INFO | "pinned" : " 29 GB" | 20260505 19:55:13.651 | INFO | } | 20260505 19:55:13.651 | INFO | Perf report for the last 5.13794 seconds: | 20260505 19:55:13.651 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:55:13.651 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:13.651 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:13.651 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:13.651 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260505 19:55:13.651 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:13.651 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:13.651 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 19:55:13.651 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:13.651 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:13.652 | INFO | Uploaded 36GiB to GPU in 2.13304s (16.8773GiB/s) | 20260505 19:55:13.652 | INFO | Downloaded 34GiB to GPU in 2.05938s (16.5098GiB/s) | 20260505 19:55:18.795 | INFO | Analyzing chunk frames = [5880320,5880832) | 20260505 19:55:18.795 | INFO | MemInfo: { | 20260505 19:55:18.795 | INFO | "gpu" : " 17 GB", | 20260505 19:55:18.795 | INFO | "huge" : "", | 20260505 19:55:18.795 | INFO | "huge_shared" : " 27 GB", | 20260505 19:55:18.795 | INFO | "malloc" : " 14 GB", | 20260505 19:55:18.795 | INFO | "pinned" : " 29 GB" | 20260505 19:55:18.795 | INFO | } | 20260505 19:55:18.795 | INFO | Perf report for the last 5.14409 seconds: | 20260505 19:55:18.795 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:55:18.795 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:18.796 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:18.796 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:18.796 | INFO | ANALYSIS : 62.1%, 3.0, 97.1ms, 0 | 20260505 19:55:18.796 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:18.796 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:18.796 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:55:18.796 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:18.796 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:18.796 | INFO | Uploaded 36GiB to GPU in 2.13576s (16.8558GiB/s) | 20260505 19:55:18.796 | INFO | Downloaded 34GiB to GPU in 2.05022s (16.5836GiB/s) | 20260505 19:55:23.936 | INFO | Analyzing chunk frames = [5880832,5881344) | 20260505 19:55:23.936 | INFO | MemInfo: { | 20260505 19:55:23.936 | INFO | "gpu" : " 17 GB", | 20260505 19:55:23.936 | INFO | "huge" : "", | 20260505 19:55:23.936 | INFO | "huge_shared" : " 27 GB", | 20260505 19:55:23.936 | INFO | "malloc" : " 14 GB", | 20260505 19:55:23.936 | INFO | "pinned" : " 29 GB" | 20260505 19:55:23.936 | INFO | } | 20260505 19:55:23.936 | INFO | Perf report for the last 5.14035 seconds: | 20260505 19:55:23.936 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:55:23.936 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:23.936 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:23.936 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:23.936 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 19:55:23.936 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:23.936 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:23.936 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 19:55:23.936 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:23.936 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:23.936 | INFO | Uploaded 36GiB to GPU in 2.1075s (17.0818GiB/s) | 20260505 19:55:23.936 | INFO | Downloaded 34GiB to GPU in 2.13544s (15.9218GiB/s) | 20260505 19:55:29.078 | INFO | Analyzing chunk frames = [5881344,5881856) | 20260505 19:55:29.078 | INFO | MemInfo: { | 20260505 19:55:29.078 | INFO | "gpu" : " 17 GB", | 20260505 19:55:29.078 | INFO | "huge" : "", | 20260505 19:55:29.078 | INFO | "huge_shared" : " 27 GB", | 20260505 19:55:29.078 | INFO | "malloc" : " 14 GB", | 20260505 19:55:29.078 | INFO | "pinned" : " 29 GB" | 20260505 19:55:29.078 | INFO | } | 20260505 19:55:29.078 | INFO | Perf report for the last 5.14225 seconds: | 20260505 19:55:29.078 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:55:29.078 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:29.078 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:29.078 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:29.078 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260505 19:55:29.078 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:29.078 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:29.078 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:55:29.078 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:29.078 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:29.078 | INFO | Uploaded 36GiB to GPU in 2.11887s (16.9902GiB/s) | 20260505 19:55:29.078 | INFO | Downloaded 34GiB to GPU in 2.01188s (16.8996GiB/s) | 20260505 19:55:29.078 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 19:55:29.078 | INFO | Total mem usage: 15.3 GiB | 20260505 19:55:29.078 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 19:55:29.078 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.3%) | 20260505 19:55:29.078 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 19:55:29.078 | INFO | High Water Breakdown: | 20260505 19:55:29.078 | INFO | 61.1% : 8.64 GiB ( 90.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 19:55:29.078 | INFO | 38.8% : 5.49 GiB ( 82.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 19:55:29.078 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 19:55:29.079 | INFO | ------------------------------------------ | 20260505 19:55:29.079 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:55:29.079 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 19:55:29.079 | INFO | Total mem usage: 26.5 GiB | 20260505 19:55:29.079 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 19:55:29.079 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 19:55:29.079 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 19:55:29.079 | INFO | High Water Breakdown: | 20260505 19:55:29.079 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:55:29.079 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:55:29.079 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:55:29.079 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:55:29.079 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:55:29.079 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:55:29.079 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 19:55:29.079 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 19:55:29.079 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:55:29.079 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 19:55:29.079 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 19:55:29.079 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 19:55:29.079 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 19:55:29.079 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 19:55:29.079 | INFO | ------------------------------------------ | 20260505 19:55:29.079 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:55:29.079 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 19:55:29.079 | INFO | Total mem usage: 25.6 GiB | 20260505 19:55:29.079 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 19:55:29.079 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 19:55:29.079 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 19:55:29.079 | INFO | High Water Breakdown: | 20260505 19:55:29.079 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 19:55:29.079 | INFO | ------------------------------------------ | 20260505 19:55:29.079 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:55:29.079 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 19:55:29.079 | INFO | Total mem usage: 15.7 GiB | 20260505 19:55:29.079 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 19:55:29.079 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 19:55:29.079 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 19:55:29.079 | INFO | High Water Breakdown: | 20260505 19:55:29.079 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 19:55:29.079 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 19:55:29.079 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 19:55:29.079 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 19:55:29.079 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 19:55:29.079 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 19:55:29.079 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 19:55:29.079 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 19:55:29.079 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 19:55:29.079 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:55:29.079 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 19:55:29.079 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:55:29.079 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 19:55:29.079 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 19:55:29.079 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 19:55:29.079 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 19:55:29.079 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:55:29.079 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 19:55:29.079 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:55:29.079 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:55:29.079 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:55:29.079 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 19:55:29.079 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 19:55:29.079 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:55:29.079 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 19:55:29.079 | INFO | ------------------------------------------ | 20260505 19:55:29.079 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:55:29.079 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 19:55:29.079 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:55:29.079 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:55:29.079 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.46e+01, 1.34e+01 / 1.60e+00 | 20260505 19:55:29.079 | INFO | PerfReport: 0.29%, AggregateStats -- 9.24e-02 / 1.03e-01, 1.09e-02 / 4.35e-03 | 20260505 19:55:29.079 | INFO | PerfReport: 11.79%, Baseline -- 3.71e+00 / 4.51e+00, 5.59e-01 / 1.92e-01 | 20260505 19:55:29.079 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.51e+00, 1.86e-01 / 1.04e-01 | 20260505 19:55:29.079 | INFO | PerfReport: 15.95%, DME -- 4.02e+01 / 4.45e+01, 3.96e+00 / 1.81e+00 | 20260505 19:55:29.079 | INFO | PerfReport: 57.02%, FrameLabeling -- 1.80e+01 / 1.93e+01, 1.44e+00 / 1.06e+00 | 20260505 19:55:29.079 | INFO | PerfReport: 7.26%, PulseAccumulating -- 2.29e+00 / 2.85e+00, 4.34e-01 / 2.13e-01 | 20260505 19:55:29.079 | INFO | PerfReport: 3.27%, Metrics -- 1.03e+00 / 1.36e+00, 2.82e-01 / 1.49e-01 | 20260505 19:55:29.079 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.58e-01 / 4.22e-01, 6.33e-02 / 3.71e-02 | 20260505 19:55:29.079 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 19:55:29.079 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:55:29.079 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:55:29.079 | INFO | PerfReport: Full Execution -- 9.94e+01 / -nan, 2.54e+01 / 0.00e+00 | 20260505 19:55:29.079 | INFO | PerfReport: 22.45%, Upload -- 2.23e+01 / 0.00e+00, 7.66e+00 / 0.00e+00 | 20260505 19:55:29.079 | INFO | PerfReport: 21.79%, Download -- 2.17e+01 / 0.00e+00, 7.61e+00 / 0.00e+00 | 20260505 19:55:29.079 | INFO | PerfReport: 31.66%, Compute -- 3.15e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 19:55:29.079 | INFO | PerfReport: 2.28%, LockUpload -- 2.27e+00 / 0.00e+00, 7.96e+00 / 0.00e+00 | 20260505 19:55:29.079 | INFO | PerfReport: 1.41%, LockDownload -- 1.40e+00 / 0.00e+00, 2.93e+00 / 0.00e+00 | 20260505 19:55:29.079 | INFO | PerfReport: 20.39%, LockCompute -- 2.03e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 19:55:29.079 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 1.97e-02 / 0.00e+00 | 20260505 19:55:29.210 | INFO | Starting a round of Baz IO | 20260505 19:55:32.378 | INFO | Populating RTMetrics report | 20260505 19:55:32.378 | INFO | RTMetrics report populated | 20260505 19:55:32.378 | INFO | Opening tmp RTMetrics json file | 20260505 19:55:32.379 | INFO | Serializing RTMetrics to json | 20260505 19:55:32.380 | INFO | Serialization done, swapping tmp file now... | 20260505 19:55:32.380 | INFO | RTMetrics done | 20260505 19:55:32.469 | INFO | Finished a round of Baz IO: Wrote 2.031 GiB in 3.258 seconds | 20260505 19:55:32.469 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.72% / 0.00% / 78.27% | 20260505 19:55:32.469 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.190826% / 3.97364e-06% / 6.35783e-05% | 20260505 19:55:32.469 | INFO | Baz IO overall: | 20260505 19:55:32.469 | INFO | BytesWritten : 3437178313579 | 20260505 19:55:32.469 | INFO | NumEvents : 1625762449904 | 20260505 19:55:34.213 | INFO | Analyzing chunk frames = [5881856,5882368) | 20260505 19:55:34.213 | INFO | MemInfo: { | 20260505 19:55:34.213 | INFO | "gpu" : " 17 GB", | 20260505 19:55:34.213 | INFO | "huge" : "", | 20260505 19:55:34.213 | INFO | "huge_shared" : " 27 GB", | 20260505 19:55:34.213 | INFO | "malloc" : " 14 GB", | 20260505 19:55:34.213 | INFO | "pinned" : " 29 GB" | 20260505 19:55:34.213 | INFO | } | 20260505 19:55:34.213 | INFO | Perf report for the last 5.13452 seconds: | 20260505 19:55:34.213 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:55:34.213 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:34.213 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:34.213 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:34.213 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 19:55:34.213 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:34.213 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260505 19:55:34.213 | INFO | PRE_HQ : 60.2%, 1.8, 55.1ms, 0 | 20260505 19:55:34.213 | INFO | BAZWRITER : 63.0%, 3.7, 119.2ms, 0 | 20260505 19:55:34.213 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 19:55:34.213 | INFO | Uploaded 36GiB to GPU in 2.14939s (16.7489GiB/s) | 20260505 19:55:34.213 | INFO | Downloaded 38GiB to GPU in 2.2483s (16.9017GiB/s) | 20260505 19:55:39.353 | INFO | Analyzing chunk frames = [5882368,5882880) | 20260505 19:55:39.353 | INFO | MemInfo: { | 20260505 19:55:39.353 | INFO | "gpu" : " 17 GB", | 20260505 19:55:39.353 | INFO | "huge" : "", | 20260505 19:55:39.353 | INFO | "huge_shared" : " 27 GB", | 20260505 19:55:39.353 | INFO | "malloc" : " 14 GB", | 20260505 19:55:39.353 | INFO | "pinned" : " 29 GB" | 20260505 19:55:39.353 | INFO | } | 20260505 19:55:39.353 | INFO | Perf report for the last 5.14032 seconds: | 20260505 19:55:39.353 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:55:39.353 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:39.353 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:39.353 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:39.353 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 19:55:39.353 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:39.353 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:39.353 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 19:55:39.353 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:39.353 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:39.353 | INFO | Uploaded 36GiB to GPU in 2.12895s (16.9097GiB/s) | 20260505 19:55:39.353 | INFO | Downloaded 34GiB to GPU in 2.02906s (16.7565GiB/s) | 20260505 19:55:44.494 | INFO | Analyzing chunk frames = [5882880,5883392) | 20260505 19:55:44.494 | INFO | MemInfo: { | 20260505 19:55:44.494 | INFO | "gpu" : " 17 GB", | 20260505 19:55:44.494 | INFO | "huge" : "", | 20260505 19:55:44.494 | INFO | "huge_shared" : " 27 GB", | 20260505 19:55:44.494 | INFO | "malloc" : " 14 GB", | 20260505 19:55:44.494 | INFO | "pinned" : " 29 GB" | 20260505 19:55:44.494 | INFO | } | 20260505 19:55:44.494 | INFO | Perf report for the last 5.14149 seconds: | 20260505 19:55:44.494 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:55:44.494 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:44.494 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:44.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:44.495 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260505 19:55:44.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:44.495 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:44.495 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:55:44.495 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:44.495 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:44.495 | INFO | Uploaded 36GiB to GPU in 2.11709s (17.0045GiB/s) | 20260505 19:55:44.495 | INFO | Downloaded 34GiB to GPU in 2.02594s (16.7823GiB/s) | 20260505 19:55:49.632 | INFO | Analyzing chunk frames = [5883392,5883904) | 20260505 19:55:49.632 | INFO | MemInfo: { | 20260505 19:55:49.632 | INFO | "gpu" : " 17 GB", | 20260505 19:55:49.632 | INFO | "huge" : "", | 20260505 19:55:49.632 | INFO | "huge_shared" : " 27 GB", | 20260505 19:55:49.632 | INFO | "malloc" : " 14 GB", | 20260505 19:55:49.632 | INFO | "pinned" : " 29 GB" | 20260505 19:55:49.632 | INFO | } | 20260505 19:55:49.632 | INFO | Perf report for the last 5.1377 seconds: | 20260505 19:55:49.632 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:55:49.632 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:49.632 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:49.632 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:49.632 | INFO | ANALYSIS : 62.5%, 3.0, 97.1ms, 0 | 20260505 19:55:49.632 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:49.632 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:49.632 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:55:49.632 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:49.632 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:49.632 | INFO | Uploaded 36GiB to GPU in 2.21004s (16.2893GiB/s) | 20260505 19:55:49.632 | INFO | Downloaded 34GiB to GPU in 2.0469s (16.6105GiB/s) | 20260505 19:55:54.766 | INFO | Analyzing chunk frames = [5883904,5884416) | 20260505 19:55:54.766 | INFO | MemInfo: { | 20260505 19:55:54.766 | INFO | "gpu" : " 17 GB", | 20260505 19:55:54.766 | INFO | "huge" : "", | 20260505 19:55:54.766 | INFO | "huge_shared" : " 27 GB", | 20260505 19:55:54.766 | INFO | "malloc" : " 14 GB", | 20260505 19:55:54.766 | INFO | "pinned" : " 29 GB" | 20260505 19:55:54.766 | INFO | } | 20260505 19:55:54.766 | INFO | Perf report for the last 5.13377 seconds: | 20260505 19:55:54.766 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:55:54.766 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:54.766 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:54.766 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:54.766 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260505 19:55:54.766 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:54.766 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:54.766 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260505 19:55:54.766 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:54.766 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:54.766 | INFO | Uploaded 36GiB to GPU in 2.12789s (16.9182GiB/s) | 20260505 19:55:54.766 | INFO | Downloaded 34GiB to GPU in 2.06129s (16.4945GiB/s) | 20260505 19:55:59.904 | INFO | Analyzing chunk frames = [5884416,5884928) | 20260505 19:55:59.904 | INFO | MemInfo: { | 20260505 19:55:59.904 | INFO | "gpu" : " 17 GB", | 20260505 19:55:59.904 | INFO | "huge" : "", | 20260505 19:55:59.904 | INFO | "huge_shared" : " 27 GB", | 20260505 19:55:59.904 | INFO | "malloc" : " 14 GB", | 20260505 19:55:59.904 | INFO | "pinned" : " 29 GB" | 20260505 19:55:59.904 | INFO | } | 20260505 19:55:59.904 | INFO | Perf report for the last 5.13817 seconds: | 20260505 19:55:59.904 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:55:59.904 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:59.904 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:59.904 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:59.904 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260505 19:55:59.904 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:59.904 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:59.904 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:55:59.904 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:59.904 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:55:59.904 | INFO | Uploaded 36GiB to GPU in 2.14196s (16.8071GiB/s) | 20260505 19:55:59.904 | INFO | Downloaded 34GiB to GPU in 2.04916s (16.5922GiB/s) | 20260505 19:56:05.054 | INFO | Analyzing chunk frames = [5884928,5885440) | 20260505 19:56:05.054 | INFO | MemInfo: { | 20260505 19:56:05.054 | INFO | "gpu" : " 17 GB", | 20260505 19:56:05.054 | INFO | "huge" : "", | 20260505 19:56:05.054 | INFO | "huge_shared" : " 27 GB", | 20260505 19:56:05.054 | INFO | "malloc" : " 14 GB", | 20260505 19:56:05.054 | INFO | "pinned" : " 29 GB" | 20260505 19:56:05.054 | INFO | } | 20260505 19:56:05.054 | INFO | Perf report for the last 5.14941 seconds: | 20260505 19:56:05.054 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:56:05.054 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:05.054 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:05.054 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:05.054 | INFO | ANALYSIS : 62.9%, 3.0, 98.5ms, 0 | 20260505 19:56:05.054 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:05.054 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:05.054 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 19:56:05.054 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:05.054 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:05.054 | INFO | Uploaded 36GiB to GPU in 2.10819s (17.0763GiB/s) | 20260505 19:56:05.054 | INFO | Downloaded 34GiB to GPU in 2.13935s (15.8926GiB/s) | 20260505 19:56:10.192 | INFO | Analyzing chunk frames = [5885440,5885952) | 20260505 19:56:10.192 | INFO | MemInfo: { | 20260505 19:56:10.192 | INFO | "gpu" : " 17 GB", | 20260505 19:56:10.192 | INFO | "huge" : "", | 20260505 19:56:10.192 | INFO | "huge_shared" : " 27 GB", | 20260505 19:56:10.192 | INFO | "malloc" : " 14 GB", | 20260505 19:56:10.192 | INFO | "pinned" : " 29 GB" | 20260505 19:56:10.192 | INFO | } | 20260505 19:56:10.192 | INFO | Perf report for the last 5.13853 seconds: | 20260505 19:56:10.192 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:56:10.192 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:10.192 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:10.192 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:10.192 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 19:56:10.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:10.192 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:10.192 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:56:10.192 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:10.192 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:10.192 | INFO | Uploaded 36GiB to GPU in 2.09642s (17.1721GiB/s) | 20260505 19:56:10.192 | INFO | Downloaded 34GiB to GPU in 1.99947s (17.0045GiB/s) | 20260505 19:56:10.325 | INFO | Starting a round of Baz IO | 20260505 19:56:13.493 | INFO | Populating RTMetrics report | 20260505 19:56:13.493 | INFO | RTMetrics report populated | 20260505 19:56:13.493 | INFO | Opening tmp RTMetrics json file | 20260505 19:56:13.495 | INFO | Serializing RTMetrics to json | 20260505 19:56:13.497 | INFO | Serialization done, swapping tmp file now... | 20260505 19:56:13.498 | INFO | RTMetrics done | 20260505 19:56:13.589 | INFO | Finished a round of Baz IO: Wrote 2.029 GiB in 3.263 seconds | 20260505 19:56:13.589 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.72% / 0.00% / 78.28% | 20260505 19:56:13.589 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.191001% / 0% / 0.000103315% | 20260505 19:56:13.589 | INFO | Baz IO overall: | 20260505 19:56:13.589 | INFO | BytesWritten : 3439356836147 | 20260505 19:56:13.589 | INFO | NumEvents : 1626772045921 | 20260505 19:56:15.325 | INFO | Analyzing chunk frames = [5885952,5886464) | 20260505 19:56:15.325 | INFO | MemInfo: { | 20260505 19:56:15.325 | INFO | "gpu" : " 17 GB", | 20260505 19:56:15.325 | INFO | "huge" : "", | 20260505 19:56:15.325 | INFO | "huge_shared" : " 27 GB", | 20260505 19:56:15.325 | INFO | "malloc" : " 14 GB", | 20260505 19:56:15.325 | INFO | "pinned" : " 29 GB" | 20260505 19:56:15.325 | INFO | } | 20260505 19:56:15.325 | INFO | Perf report for the last 5.1328 seconds: | 20260505 19:56:15.325 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:56:15.325 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:15.325 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:15.325 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:15.325 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 19:56:15.325 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:15.325 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260505 19:56:15.325 | INFO | PRE_HQ : 60.0%, 1.8, 55.3ms, 0 | 20260505 19:56:15.325 | INFO | BAZWRITER : 63.1%, 3.7, 119.3ms, 0 | 20260505 19:56:15.325 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 19:56:15.325 | INFO | Uploaded 36GiB to GPU in 2.16785s (16.6064GiB/s) | 20260505 19:56:15.325 | INFO | Downloaded 38GiB to GPU in 2.25929s (16.8195GiB/s) | 20260505 19:56:20.460 | INFO | Analyzing chunk frames = [5886464,5886976) | 20260505 19:56:20.460 | INFO | MemInfo: { | 20260505 19:56:20.460 | INFO | "gpu" : " 17 GB", | 20260505 19:56:20.460 | INFO | "huge" : "", | 20260505 19:56:20.460 | INFO | "huge_shared" : " 27 GB", | 20260505 19:56:20.460 | INFO | "malloc" : " 14 GB", | 20260505 19:56:20.460 | INFO | "pinned" : " 29 GB" | 20260505 19:56:20.460 | INFO | } | 20260505 19:56:20.460 | INFO | Perf report for the last 5.13516 seconds: | 20260505 19:56:20.460 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:56:20.460 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:20.460 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:20.460 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:20.460 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260505 19:56:20.460 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:20.460 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:20.460 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:56:20.460 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:20.460 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:20.460 | INFO | Uploaded 36GiB to GPU in 2.14489s (16.7841GiB/s) | 20260505 19:56:20.460 | INFO | Downloaded 34GiB to GPU in 2.04358s (16.6375GiB/s) | 20260505 19:56:25.605 | INFO | Analyzing chunk frames = [5886976,5887488) | 20260505 19:56:25.605 | INFO | MemInfo: { | 20260505 19:56:25.605 | INFO | "gpu" : " 17 GB", | 20260505 19:56:25.605 | INFO | "huge" : "", | 20260505 19:56:25.605 | INFO | "huge_shared" : " 27 GB", | 20260505 19:56:25.605 | INFO | "malloc" : " 14 GB", | 20260505 19:56:25.605 | INFO | "pinned" : " 29 GB" | 20260505 19:56:25.605 | INFO | } | 20260505 19:56:25.605 | INFO | Perf report for the last 5.14474 seconds: | 20260505 19:56:25.605 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:56:25.605 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:25.605 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:25.605 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:25.605 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 19:56:25.605 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:25.605 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:25.605 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:56:25.605 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:25.605 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:25.605 | INFO | Uploaded 36GiB to GPU in 2.21129s (16.2801GiB/s) | 20260505 19:56:25.605 | INFO | Downloaded 34GiB to GPU in 2.04536s (16.623GiB/s) | 20260505 19:56:30.746 | INFO | Analyzing chunk frames = [5887488,5888000) | 20260505 19:56:30.746 | INFO | MemInfo: { | 20260505 19:56:30.746 | INFO | "gpu" : " 17 GB", | 20260505 19:56:30.746 | INFO | "huge" : "", | 20260505 19:56:30.746 | INFO | "huge_shared" : " 27 GB", | 20260505 19:56:30.746 | INFO | "malloc" : " 14 GB", | 20260505 19:56:30.746 | INFO | "pinned" : " 29 GB" | 20260505 19:56:30.746 | INFO | } | 20260505 19:56:30.746 | INFO | Perf report for the last 5.14117 seconds: | 20260505 19:56:30.746 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:56:30.746 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:30.746 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:30.746 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:30.746 | INFO | ANALYSIS : 62.3%, 3.0, 96.7ms, 0 | 20260505 19:56:30.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:30.746 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:30.746 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 19:56:30.746 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:30.746 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:30.746 | INFO | Uploaded 36GiB to GPU in 2.17515s (16.5506GiB/s) | 20260505 19:56:30.746 | INFO | Downloaded 34GiB to GPU in 2.05683s (16.5303GiB/s) | 20260505 19:56:35.892 | INFO | Analyzing chunk frames = [5888000,5888512) | 20260505 19:56:35.892 | INFO | MemInfo: { | 20260505 19:56:35.892 | INFO | "gpu" : " 17 GB", | 20260505 19:56:35.892 | INFO | "huge" : "", | 20260505 19:56:35.892 | INFO | "huge_shared" : " 27 GB", | 20260505 19:56:35.892 | INFO | "malloc" : " 14 GB", | 20260505 19:56:35.892 | INFO | "pinned" : " 29 GB" | 20260505 19:56:35.892 | INFO | } | 20260505 19:56:35.892 | INFO | Perf report for the last 5.1457 seconds: | 20260505 19:56:35.892 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:56:35.892 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:35.892 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:35.892 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:35.892 | INFO | ANALYSIS : 61.7%, 3.0, 96.5ms, 0 | 20260505 19:56:35.892 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:35.892 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:35.892 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:56:35.892 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:35.892 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:35.892 | INFO | Uploaded 36GiB to GPU in 2.14915s (16.7508GiB/s) | 20260505 19:56:35.892 | INFO | Downloaded 34GiB to GPU in 2.06277s (16.4827GiB/s) | 20260505 19:56:41.029 | INFO | Analyzing chunk frames = [5888512,5889024) | 20260505 19:56:41.029 | INFO | MemInfo: { | 20260505 19:56:41.029 | INFO | "gpu" : " 17 GB", | 20260505 19:56:41.029 | INFO | "huge" : "", | 20260505 19:56:41.029 | INFO | "huge_shared" : " 27 GB", | 20260505 19:56:41.029 | INFO | "malloc" : " 14 GB", | 20260505 19:56:41.029 | INFO | "pinned" : " 29 GB" | 20260505 19:56:41.029 | INFO | } | 20260505 19:56:41.029 | INFO | Perf report for the last 5.13752 seconds: | 20260505 19:56:41.029 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:56:41.029 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:41.029 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:41.029 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:41.029 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260505 19:56:41.029 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:41.029 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:41.029 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 19:56:41.029 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:41.029 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:41.030 | INFO | Uploaded 36GiB to GPU in 2.0993s (17.1486GiB/s) | 20260505 19:56:41.030 | INFO | Downloaded 34GiB to GPU in 2.0356s (16.7027GiB/s) | 20260505 19:56:46.157 | INFO | Analyzing chunk frames = [5889024,5889536) | 20260505 19:56:46.157 | INFO | MemInfo: { | 20260505 19:56:46.157 | INFO | "gpu" : " 17 GB", | 20260505 19:56:46.157 | INFO | "huge" : "", | 20260505 19:56:46.157 | INFO | "huge_shared" : " 27 GB", | 20260505 19:56:46.157 | INFO | "malloc" : " 14 GB", | 20260505 19:56:46.157 | INFO | "pinned" : " 29 GB" | 20260505 19:56:46.157 | INFO | } | 20260505 19:56:46.157 | INFO | Perf report for the last 5.12802 seconds: | 20260505 19:56:46.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:56:46.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:46.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:46.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:46.157 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 19:56:46.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:46.157 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:46.157 | INFO | PRE_HQ : 28.7%, 1.0, 14.8ms, 0 | 20260505 19:56:46.157 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:46.157 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:46.158 | INFO | Uploaded 36GiB to GPU in 2.10907s (17.0691GiB/s) | 20260505 19:56:46.158 | INFO | Downloaded 34GiB to GPU in 2.1485s (15.825GiB/s) | 20260505 19:56:51.308 | INFO | Analyzing chunk frames = [5889536,5890048) | 20260505 19:56:51.308 | INFO | MemInfo: { | 20260505 19:56:51.308 | INFO | "gpu" : " 17 GB", | 20260505 19:56:51.308 | INFO | "huge" : "", | 20260505 19:56:51.308 | INFO | "huge_shared" : " 27 GB", | 20260505 19:56:51.308 | INFO | "malloc" : " 14 GB", | 20260505 19:56:51.308 | INFO | "pinned" : " 29 GB" | 20260505 19:56:51.308 | INFO | } | 20260505 19:56:51.308 | INFO | Perf report for the last 5.15076 seconds: | 20260505 19:56:51.308 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:56:51.308 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:51.308 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:51.308 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:51.308 | INFO | ANALYSIS : 62.7%, 3.0, 98.2ms, 0 | 20260505 19:56:51.308 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:51.308 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:51.308 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:56:51.308 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:51.308 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:51.308 | INFO | Uploaded 36GiB to GPU in 2.19529s (16.3987GiB/s) | 20260505 19:56:51.309 | INFO | Downloaded 34GiB to GPU in 2.03657s (16.6947GiB/s) | 20260505 19:56:51.309 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 19:56:51.309 | INFO | Total mem usage: 15.3 GiB | 20260505 19:56:51.309 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 19:56:51.309 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.48%) | 20260505 19:56:51.309 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 19:56:51.309 | INFO | High Water Breakdown: | 20260505 19:56:51.309 | INFO | 60.9% : 8.6 GiB ( 90.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 19:56:51.309 | INFO | 39% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 19:56:51.309 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 19:56:51.309 | INFO | ------------------------------------------ | 20260505 19:56:51.309 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:56:51.309 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 19:56:51.309 | INFO | Total mem usage: 26.5 GiB | 20260505 19:56:51.309 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 19:56:51.309 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 19:56:51.309 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 19:56:51.309 | INFO | High Water Breakdown: | 20260505 19:56:51.309 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:56:51.309 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:56:51.309 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:56:51.309 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:56:51.309 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:56:51.309 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:56:51.309 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 19:56:51.309 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 19:56:51.309 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:56:51.309 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 19:56:51.309 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 19:56:51.309 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 19:56:51.309 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 19:56:51.309 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 19:56:51.309 | INFO | ------------------------------------------ | 20260505 19:56:51.309 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:56:51.309 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 19:56:51.309 | INFO | Total mem usage: 25.6 GiB | 20260505 19:56:51.309 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 19:56:51.309 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 19:56:51.309 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 19:56:51.309 | INFO | High Water Breakdown: | 20260505 19:56:51.309 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 19:56:51.309 | INFO | ------------------------------------------ | 20260505 19:56:51.309 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:56:51.309 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 19:56:51.309 | INFO | Total mem usage: 15.7 GiB | 20260505 19:56:51.309 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 19:56:51.309 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 19:56:51.309 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 19:56:51.309 | INFO | High Water Breakdown: | 20260505 19:56:51.309 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 19:56:51.309 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 19:56:51.309 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 19:56:51.309 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 19:56:51.309 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 19:56:51.309 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 19:56:51.309 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 19:56:51.309 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 19:56:51.309 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 19:56:51.309 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:56:51.309 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 19:56:51.309 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:56:51.309 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 19:56:51.309 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 19:56:51.309 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 19:56:51.309 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 19:56:51.309 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:56:51.309 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 19:56:51.309 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:56:51.309 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:56:51.309 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:56:51.309 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 19:56:51.309 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 19:56:51.309 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:56:51.309 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 19:56:51.309 | INFO | ------------------------------------------ | 20260505 19:56:51.309 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:56:51.309 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 19:56:51.309 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:56:51.309 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:56:51.309 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.43e+01, 1.34e+01 / 2.11e+00 | 20260505 19:56:51.309 | INFO | PerfReport: 0.29%, AggregateStats -- 9.25e-02 / 1.02e-01, 1.15e-02 / 2.82e-03 | 20260505 19:56:51.309 | INFO | PerfReport: 11.81%, Baseline -- 3.71e+00 / 4.58e+00, 5.65e-01 / 8.48e-02 | 20260505 19:56:51.309 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.52e+00, 1.89e-01 / 7.97e-02 | 20260505 19:56:51.309 | INFO | PerfReport: 15.96%, DME -- 4.02e+01 / 4.43e+01, 4.07e+00 / 2.14e+00 | 20260505 19:56:51.309 | INFO | PerfReport: 56.98%, FrameLabeling -- 1.79e+01 / 1.91e+01, 1.41e+00 / 7.06e-01 | 20260505 19:56:51.309 | INFO | PerfReport: 7.29%, PulseAccumulating -- 2.29e+00 / 2.90e+00, 4.41e-01 / 1.11e-01 | 20260505 19:56:51.309 | INFO | PerfReport: 3.27%, Metrics -- 1.03e+00 / 1.37e+00, 2.80e-01 / 1.37e-01 | 20260505 19:56:51.309 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.57e-01 / 4.29e-01, 6.30e-02 / 3.91e-02 | 20260505 19:56:51.309 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 19:56:51.309 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:56:51.309 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:56:51.309 | INFO | PerfReport: Full Execution -- 9.95e+01 / -nan, 2.53e+01 / 0.00e+00 | 20260505 19:56:51.309 | INFO | PerfReport: 22.47%, Upload -- 2.24e+01 / 0.00e+00, 7.73e+00 / 0.00e+00 | 20260505 19:56:51.309 | INFO | PerfReport: 21.79%, Download -- 2.17e+01 / 0.00e+00, 7.71e+00 / 0.00e+00 | 20260505 19:56:51.309 | INFO | PerfReport: 31.63%, Compute -- 3.15e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 19:56:51.309 | INFO | PerfReport: 2.32%, LockUpload -- 2.31e+00 / 0.00e+00, 7.98e+00 / 0.00e+00 | 20260505 19:56:51.309 | INFO | PerfReport: 1.40%, LockDownload -- 1.39e+00 / 0.00e+00, 3.01e+00 / 0.00e+00 | 20260505 19:56:51.309 | INFO | PerfReport: 20.38%, LockCompute -- 2.03e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 19:56:51.309 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.75e-02 / 0.00e+00 | 20260505 19:56:51.443 | INFO | Starting a round of Baz IO | 20260505 19:56:54.605 | INFO | Populating RTMetrics report | 20260505 19:56:54.606 | INFO | RTMetrics report populated | 20260505 19:56:54.606 | INFO | Opening tmp RTMetrics json file | 20260505 19:56:54.612 | INFO | Serializing RTMetrics to json | 20260505 19:56:54.613 | INFO | Serialization done, swapping tmp file now... | 20260505 19:56:54.614 | INFO | RTMetrics done | 20260505 19:56:54.695 | INFO | Finished a round of Baz IO: Wrote 2.031 GiB in 3.251 seconds | 20260505 19:56:54.695 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.72% / 0.00% / 78.28% | 20260505 19:56:54.695 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.190254% / 3.97364e-06% / 0.000107288% | 20260505 19:56:54.695 | INFO | Baz IO overall: | 20260505 19:56:54.695 | INFO | BytesWritten : 3441537227137 | 20260505 19:56:54.695 | INFO | NumEvents : 1627783116069 | 20260505 19:56:56.453 | INFO | Analyzing chunk frames = [5890048,5890560) | 20260505 19:56:56.453 | INFO | MemInfo: { | 20260505 19:56:56.453 | INFO | "gpu" : " 17 GB", | 20260505 19:56:56.453 | INFO | "huge" : "", | 20260505 19:56:56.453 | INFO | "huge_shared" : " 27 GB", | 20260505 19:56:56.453 | INFO | "malloc" : " 14 GB", | 20260505 19:56:56.453 | INFO | "pinned" : " 29 GB" | 20260505 19:56:56.453 | INFO | } | 20260505 19:56:56.453 | INFO | Perf report for the last 5.14493 seconds: | 20260505 19:56:56.453 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:56:56.453 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:56.453 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:56.453 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:56.453 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 19:56:56.453 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:56:56.453 | INFO | RT_METRICS : 44.1%, 1.0, 23.2ms, 0 | 20260505 19:56:56.453 | INFO | PRE_HQ : 60.0%, 1.8, 55.7ms, 0 | 20260505 19:56:56.453 | INFO | BAZWRITER : 62.8%, 3.7, 118.6ms, 0 | 20260505 19:56:56.453 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 19:56:56.453 | INFO | Uploaded 36GiB to GPU in 2.17134s (16.5796GiB/s) | 20260505 19:56:56.453 | INFO | Downloaded 38GiB to GPU in 2.25222s (16.8723GiB/s) | 20260505 19:57:01.577 | INFO | Analyzing chunk frames = [5890560,5891072) | 20260505 19:57:01.577 | INFO | MemInfo: { | 20260505 19:57:01.577 | INFO | "gpu" : " 17 GB", | 20260505 19:57:01.577 | INFO | "huge" : "", | 20260505 19:57:01.577 | INFO | "huge_shared" : " 27 GB", | 20260505 19:57:01.577 | INFO | "malloc" : " 14 GB", | 20260505 19:57:01.577 | INFO | "pinned" : " 29 GB" | 20260505 19:57:01.577 | INFO | } | 20260505 19:57:01.577 | INFO | Perf report for the last 5.12399 seconds: | 20260505 19:57:01.577 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:57:01.577 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:01.577 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:01.577 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:01.577 | INFO | ANALYSIS : 62.5%, 3.0, 97.2ms, 0 | 20260505 19:57:01.577 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:01.577 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:01.577 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260505 19:57:01.577 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:01.577 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:01.577 | INFO | Uploaded 36GiB to GPU in 2.10217s (17.1251GiB/s) | 20260505 19:57:01.577 | INFO | Downloaded 34GiB to GPU in 2.02284s (16.808GiB/s) | 20260505 19:57:06.718 | INFO | Analyzing chunk frames = [5891072,5891584) | 20260505 19:57:06.718 | INFO | MemInfo: { | 20260505 19:57:06.718 | INFO | "gpu" : " 17 GB", | 20260505 19:57:06.718 | INFO | "huge" : "", | 20260505 19:57:06.718 | INFO | "huge_shared" : " 27 GB", | 20260505 19:57:06.718 | INFO | "malloc" : " 14 GB", | 20260505 19:57:06.718 | INFO | "pinned" : " 29 GB" | 20260505 19:57:06.718 | INFO | } | 20260505 19:57:06.718 | INFO | Perf report for the last 5.14112 seconds: | 20260505 19:57:06.718 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:57:06.718 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:06.718 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:06.718 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:06.718 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 19:57:06.718 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:06.718 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:06.718 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:57:06.718 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:06.718 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:06.718 | INFO | Uploaded 36GiB to GPU in 2.21262s (16.2703GiB/s) | 20260505 19:57:06.718 | INFO | Downloaded 34GiB to GPU in 2.06554s (16.4606GiB/s) | 20260505 19:57:11.859 | INFO | Analyzing chunk frames = [5891584,5892096) | 20260505 19:57:11.859 | INFO | MemInfo: { | 20260505 19:57:11.859 | INFO | "gpu" : " 17 GB", | 20260505 19:57:11.859 | INFO | "huge" : "", | 20260505 19:57:11.859 | INFO | "huge_shared" : " 27 GB", | 20260505 19:57:11.859 | INFO | "malloc" : " 14 GB", | 20260505 19:57:11.859 | INFO | "pinned" : " 29 GB" | 20260505 19:57:11.859 | INFO | } | 20260505 19:57:11.859 | INFO | Perf report for the last 5.14092 seconds: | 20260505 19:57:11.859 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:57:11.859 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:11.859 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:11.859 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:11.859 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260505 19:57:11.859 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:11.859 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:11.859 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:57:11.859 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:11.859 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:11.859 | INFO | Uploaded 36GiB to GPU in 2.18973s (16.4404GiB/s) | 20260505 19:57:11.859 | INFO | Downloaded 34GiB to GPU in 2.0573s (16.5265GiB/s) | 20260505 19:57:17.004 | INFO | Analyzing chunk frames = [5892096,5892608) | 20260505 19:57:17.004 | INFO | MemInfo: { | 20260505 19:57:17.004 | INFO | "gpu" : " 17 GB", | 20260505 19:57:17.004 | INFO | "huge" : "", | 20260505 19:57:17.004 | INFO | "huge_shared" : " 27 GB", | 20260505 19:57:17.004 | INFO | "malloc" : " 14 GB", | 20260505 19:57:17.004 | INFO | "pinned" : " 29 GB" | 20260505 19:57:17.004 | INFO | } | 20260505 19:57:17.004 | INFO | Perf report for the last 5.14474 seconds: | 20260505 19:57:17.004 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:57:17.004 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:17.004 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:17.004 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:17.004 | INFO | ANALYSIS : 61.6%, 3.0, 96.3ms, 0 | 20260505 19:57:17.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:17.004 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:17.004 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:57:17.004 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:17.004 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:17.004 | INFO | Uploaded 36GiB to GPU in 2.12771s (16.9196GiB/s) | 20260505 19:57:17.004 | INFO | Downloaded 34GiB to GPU in 2.06083s (16.4982GiB/s) | 20260505 19:57:22.138 | INFO | Analyzing chunk frames = [5892608,5893120) | 20260505 19:57:22.138 | INFO | MemInfo: { | 20260505 19:57:22.138 | INFO | "gpu" : " 17 GB", | 20260505 19:57:22.138 | INFO | "huge" : "", | 20260505 19:57:22.138 | INFO | "huge_shared" : " 27 GB", | 20260505 19:57:22.138 | INFO | "malloc" : " 14 GB", | 20260505 19:57:22.138 | INFO | "pinned" : " 29 GB" | 20260505 19:57:22.138 | INFO | } | 20260505 19:57:22.138 | INFO | Perf report for the last 5.13421 seconds: | 20260505 19:57:22.138 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:57:22.138 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:22.138 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:22.138 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:22.138 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 19:57:22.138 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:22.138 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:22.138 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 19:57:22.138 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:22.138 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:22.138 | INFO | Uploaded 36GiB to GPU in 2.08899s (17.2332GiB/s) | 20260505 19:57:22.138 | INFO | Downloaded 34GiB to GPU in 2.03944s (16.6713GiB/s) | 20260505 19:57:27.274 | INFO | Analyzing chunk frames = [5893120,5893632) | 20260505 19:57:27.274 | INFO | MemInfo: { | 20260505 19:57:27.274 | INFO | "gpu" : " 17 GB", | 20260505 19:57:27.274 | INFO | "huge" : "", | 20260505 19:57:27.274 | INFO | "huge_shared" : " 27 GB", | 20260505 19:57:27.274 | INFO | "malloc" : " 14 GB", | 20260505 19:57:27.274 | INFO | "pinned" : " 29 GB" | 20260505 19:57:27.274 | INFO | } | 20260505 19:57:27.274 | INFO | Perf report for the last 5.13595 seconds: | 20260505 19:57:27.274 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:57:27.274 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:27.274 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:27.274 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:27.274 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 19:57:27.274 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:27.274 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:27.274 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 19:57:27.274 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:27.274 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:27.274 | INFO | Uploaded 36GiB to GPU in 2.12538s (16.9382GiB/s) | 20260505 19:57:27.275 | INFO | Downloaded 34GiB to GPU in 2.14116s (15.8793GiB/s) | 20260505 19:57:32.412 | INFO | Analyzing chunk frames = [5893632,5894144) | 20260505 19:57:32.412 | INFO | MemInfo: { | 20260505 19:57:32.412 | INFO | "gpu" : " 17 GB", | 20260505 19:57:32.412 | INFO | "huge" : "", | 20260505 19:57:32.412 | INFO | "huge_shared" : " 27 GB", | 20260505 19:57:32.412 | INFO | "malloc" : " 14 GB", | 20260505 19:57:32.412 | INFO | "pinned" : " 29 GB" | 20260505 19:57:32.412 | INFO | } | 20260505 19:57:32.412 | INFO | Perf report for the last 5.13806 seconds: | 20260505 19:57:32.412 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:57:32.412 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:32.412 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:32.412 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:32.412 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 19:57:32.412 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:32.412 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:32.412 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 19:57:32.412 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:32.412 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:32.412 | INFO | Uploaded 36GiB to GPU in 2.16552s (16.6241GiB/s) | 20260505 19:57:32.412 | INFO | Downloaded 34GiB to GPU in 2.03071s (16.7429GiB/s) | 20260505 19:57:32.546 | INFO | Starting a round of Baz IO | 20260505 19:57:35.721 | INFO | Populating RTMetrics report | 20260505 19:57:35.721 | INFO | RTMetrics report populated | 20260505 19:57:35.721 | INFO | Opening tmp RTMetrics json file | 20260505 19:57:35.722 | INFO | Serializing RTMetrics to json | 20260505 19:57:35.723 | INFO | Serialization done, swapping tmp file now... | 20260505 19:57:35.723 | INFO | RTMetrics done | 20260505 19:57:35.808 | INFO | Finished a round of Baz IO: Wrote 2.028 GiB in 3.262 seconds | 20260505 19:57:35.808 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.72% / 0.00% / 78.28% | 20260505 19:57:35.808 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.190183% / 0% / 7.54992e-05% | 20260505 19:57:35.808 | INFO | Baz IO overall: | 20260505 19:57:35.809 | INFO | BytesWritten : 3443715042732 | 20260505 19:57:35.809 | INFO | NumEvents : 1628792446460 | 20260505 19:57:37.560 | INFO | Analyzing chunk frames = [5894144,5894656) | 20260505 19:57:37.560 | INFO | MemInfo: { | 20260505 19:57:37.560 | INFO | "gpu" : " 17 GB", | 20260505 19:57:37.560 | INFO | "huge" : "", | 20260505 19:57:37.560 | INFO | "huge_shared" : " 27 GB", | 20260505 19:57:37.560 | INFO | "malloc" : " 14 GB", | 20260505 19:57:37.560 | INFO | "pinned" : " 29 GB" | 20260505 19:57:37.560 | INFO | } | 20260505 19:57:37.560 | INFO | Perf report for the last 5.14807 seconds: | 20260505 19:57:37.560 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:57:37.560 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:37.560 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:37.560 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:37.560 | INFO | ANALYSIS : 63.2%, 3.0, 98.8ms, 0 | 20260505 19:57:37.560 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:37.560 | INFO | RT_METRICS : 43.9%, 1.0, 23.1ms, 0 | 20260505 19:57:37.560 | INFO | PRE_HQ : 60.1%, 1.8, 55.1ms, 0 | 20260505 19:57:37.560 | INFO | BAZWRITER : 62.9%, 3.6, 117.7ms, 0 | 20260505 19:57:37.560 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 19:57:37.560 | INFO | Uploaded 36GiB to GPU in 2.16644s (16.6171GiB/s) | 20260505 19:57:37.560 | INFO | Downloaded 38GiB to GPU in 2.24465s (16.9291GiB/s) | 20260505 19:57:42.703 | INFO | Analyzing chunk frames = [5894656,5895168) | 20260505 19:57:42.703 | INFO | MemInfo: { | 20260505 19:57:42.703 | INFO | "gpu" : " 17 GB", | 20260505 19:57:42.703 | INFO | "huge" : "", | 20260505 19:57:42.703 | INFO | "huge_shared" : " 27 GB", | 20260505 19:57:42.703 | INFO | "malloc" : " 14 GB", | 20260505 19:57:42.703 | INFO | "pinned" : " 29 GB" | 20260505 19:57:42.703 | INFO | } | 20260505 19:57:42.703 | INFO | Perf report for the last 5.14269 seconds: | 20260505 19:57:42.703 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:57:42.703 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:42.703 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:42.703 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:42.703 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260505 19:57:42.703 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:42.703 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:42.703 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:57:42.703 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:42.703 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:42.703 | INFO | Uploaded 36GiB to GPU in 2.0828s (17.2844GiB/s) | 20260505 19:57:42.703 | INFO | Downloaded 34GiB to GPU in 2.02252s (16.8107GiB/s) | 20260505 19:57:47.837 | INFO | Analyzing chunk frames = [5895168,5895680) | 20260505 19:57:47.837 | INFO | MemInfo: { | 20260505 19:57:47.837 | INFO | "gpu" : " 17 GB", | 20260505 19:57:47.837 | INFO | "huge" : "", | 20260505 19:57:47.838 | INFO | "huge_shared" : " 27 GB", | 20260505 19:57:47.838 | INFO | "malloc" : " 14 GB", | 20260505 19:57:47.838 | INFO | "pinned" : " 29 GB" | 20260505 19:57:47.838 | INFO | } | 20260505 19:57:47.838 | INFO | Perf report for the last 5.13469 seconds: | 20260505 19:57:47.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:57:47.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:47.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:47.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:47.838 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260505 19:57:47.838 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:47.838 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:47.838 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:57:47.838 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:47.838 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:47.838 | INFO | Uploaded 36GiB to GPU in 2.13229s (16.8832GiB/s) | 20260505 19:57:47.838 | INFO | Downloaded 34GiB to GPU in 2.02651s (16.7776GiB/s) | 20260505 19:57:52.976 | INFO | Analyzing chunk frames = [5895680,5896192) | 20260505 19:57:52.976 | INFO | MemInfo: { | 20260505 19:57:52.976 | INFO | "gpu" : " 17 GB", | 20260505 19:57:52.976 | INFO | "huge" : "", | 20260505 19:57:52.976 | INFO | "huge_shared" : " 27 GB", | 20260505 19:57:52.976 | INFO | "malloc" : " 14 GB", | 20260505 19:57:52.976 | INFO | "pinned" : " 29 GB" | 20260505 19:57:52.976 | INFO | } | 20260505 19:57:52.977 | INFO | Perf report for the last 5.13889 seconds: | 20260505 19:57:52.977 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:57:52.977 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:52.977 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:52.977 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:52.977 | INFO | ANALYSIS : 62.5%, 3.0, 97.0ms, 0 | 20260505 19:57:52.977 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:52.977 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:52.977 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:57:52.977 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:52.977 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:52.977 | INFO | Uploaded 36GiB to GPU in 2.2076s (16.3073GiB/s) | 20260505 19:57:52.977 | INFO | Downloaded 34GiB to GPU in 2.04336s (16.6393GiB/s) | 20260505 19:57:58.115 | INFO | Analyzing chunk frames = [5896192,5896704) | 20260505 19:57:58.115 | INFO | MemInfo: { | 20260505 19:57:58.115 | INFO | "gpu" : " 17 GB", | 20260505 19:57:58.115 | INFO | "huge" : "", | 20260505 19:57:58.115 | INFO | "huge_shared" : " 27 GB", | 20260505 19:57:58.115 | INFO | "malloc" : " 14 GB", | 20260505 19:57:58.115 | INFO | "pinned" : " 29 GB" | 20260505 19:57:58.115 | INFO | } | 20260505 19:57:58.115 | INFO | Perf report for the last 5.13835 seconds: | 20260505 19:57:58.115 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:57:58.115 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:58.115 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:58.115 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:58.115 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260505 19:57:58.115 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:58.115 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:58.115 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:57:58.115 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:58.115 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:57:58.115 | INFO | Uploaded 36GiB to GPU in 2.13835s (16.8354GiB/s) | 20260505 19:57:58.115 | INFO | Downloaded 34GiB to GPU in 2.06215s (16.4877GiB/s) | 20260505 19:58:03.257 | INFO | Analyzing chunk frames = [5896704,5897216) | 20260505 19:58:03.257 | INFO | MemInfo: { | 20260505 19:58:03.257 | INFO | "gpu" : " 17 GB", | 20260505 19:58:03.257 | INFO | "huge" : "", | 20260505 19:58:03.257 | INFO | "huge_shared" : " 27 GB", | 20260505 19:58:03.257 | INFO | "malloc" : " 14 GB", | 20260505 19:58:03.257 | INFO | "pinned" : " 29 GB" | 20260505 19:58:03.257 | INFO | } | 20260505 19:58:03.257 | INFO | Perf report for the last 5.14202 seconds: | 20260505 19:58:03.257 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:58:03.257 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:03.257 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:03.257 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:03.257 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260505 19:58:03.257 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:03.257 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:03.257 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:58:03.257 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:03.257 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:03.257 | INFO | Uploaded 36GiB to GPU in 2.15942s (16.6711GiB/s) | 20260505 19:58:03.257 | INFO | Downloaded 34GiB to GPU in 2.06383s (16.4742GiB/s) | 20260505 19:58:08.396 | INFO | Analyzing chunk frames = [5897216,5897728) | 20260505 19:58:08.396 | INFO | MemInfo: { | 20260505 19:58:08.396 | INFO | "gpu" : " 17 GB", | 20260505 19:58:08.396 | INFO | "huge" : "", | 20260505 19:58:08.396 | INFO | "huge_shared" : " 27 GB", | 20260505 19:58:08.396 | INFO | "malloc" : " 14 GB", | 20260505 19:58:08.396 | INFO | "pinned" : " 29 GB" | 20260505 19:58:08.396 | INFO | } | 20260505 19:58:08.397 | INFO | Perf report for the last 5.13972 seconds: | 20260505 19:58:08.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:58:08.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:08.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:08.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:08.397 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 19:58:08.397 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:08.397 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:08.397 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 19:58:08.397 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:08.397 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:08.397 | INFO | Uploaded 36GiB to GPU in 2.11403s (17.0291GiB/s) | 20260505 19:58:08.397 | INFO | Downloaded 34GiB to GPU in 2.13181s (15.9489GiB/s) | 20260505 19:58:13.527 | INFO | Analyzing chunk frames = [5897728,5898240) | 20260505 19:58:13.527 | INFO | MemInfo: { | 20260505 19:58:13.527 | INFO | "gpu" : " 17 GB", | 20260505 19:58:13.527 | INFO | "huge" : "", | 20260505 19:58:13.527 | INFO | "huge_shared" : " 27 GB", | 20260505 19:58:13.527 | INFO | "malloc" : " 14 GB", | 20260505 19:58:13.527 | INFO | "pinned" : " 29 GB" | 20260505 19:58:13.527 | INFO | } | 20260505 19:58:13.527 | INFO | Perf report for the last 5.1305 seconds: | 20260505 19:58:13.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:58:13.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:13.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:13.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:13.527 | INFO | ANALYSIS : 62.8%, 3.0, 97.8ms, 0 | 20260505 19:58:13.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:13.527 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:13.527 | INFO | PRE_HQ : 27.5%, 1.0, 14.2ms, 0 | 20260505 19:58:13.527 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:13.527 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:13.527 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 19:58:13.527 | INFO | Total mem usage: 15.3 GiB | 20260505 19:58:13.527 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 19:58:13.527 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.11%) | 20260505 19:58:13.527 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 19:58:13.527 | INFO | High Water Breakdown: | 20260505 19:58:13.527 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 19:58:13.527 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 19:58:13.527 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 19:58:13.527 | INFO | ------------------------------------------ | 20260505 19:58:13.527 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:58:13.527 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 19:58:13.527 | INFO | Total mem usage: 26.5 GiB | 20260505 19:58:13.527 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 19:58:13.527 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 19:58:13.527 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 19:58:13.527 | INFO | High Water Breakdown: | 20260505 19:58:13.527 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:58:13.527 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:58:13.527 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:58:13.527 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:58:13.527 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:58:13.527 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:58:13.527 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 19:58:13.527 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 19:58:13.527 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:58:13.527 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 19:58:13.527 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 19:58:13.527 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 19:58:13.527 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 19:58:13.527 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 19:58:13.527 | INFO | ------------------------------------------ | 20260505 19:58:13.527 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:58:13.527 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 19:58:13.527 | INFO | Total mem usage: 25.6 GiB | 20260505 19:58:13.527 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 19:58:13.527 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 19:58:13.527 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 19:58:13.527 | INFO | High Water Breakdown: | 20260505 19:58:13.527 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 19:58:13.527 | INFO | ------------------------------------------ | 20260505 19:58:13.527 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:58:13.527 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 19:58:13.527 | INFO | Total mem usage: 15.7 GiB | 20260505 19:58:13.527 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 19:58:13.527 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 19:58:13.527 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 19:58:13.527 | INFO | High Water Breakdown: | 20260505 19:58:13.528 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 19:58:13.528 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 19:58:13.528 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 19:58:13.528 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 19:58:13.528 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 19:58:13.528 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 19:58:13.528 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 19:58:13.528 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 19:58:13.528 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 19:58:13.528 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:58:13.528 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 19:58:13.528 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:58:13.528 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 19:58:13.528 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 19:58:13.528 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 19:58:13.528 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 19:58:13.528 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:58:13.528 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 19:58:13.528 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:58:13.528 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:58:13.528 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:58:13.528 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 19:58:13.528 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 19:58:13.528 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:58:13.528 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 19:58:13.528 | INFO | ------------------------------------------ | 20260505 19:58:13.528 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:58:13.528 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 19:58:13.528 | INFO | Uploaded 36GiB to GPU in 2.12967s (16.904GiB/s) | 20260505 19:58:13.529 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:58:13.529 | INFO | Downloaded 34GiB to GPU in 2.01347s (16.8863GiB/s) | 20260505 19:58:13.529 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:58:13.529 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.48e+01, 1.36e+01 / 2.00e+00 | 20260505 19:58:13.529 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.03e-01, 1.07e-02 / 3.40e-03 | 20260505 19:58:13.529 | INFO | PerfReport: 11.79%, Baseline -- 3.72e+00 / 4.58e+00, 5.72e-01 / 9.49e-02 | 20260505 19:58:13.529 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.52e+00, 1.91e-01 / 9.35e-02 | 20260505 19:58:13.529 | INFO | PerfReport: 15.98%, DME -- 4.03e+01 / 4.45e+01, 4.03e+00 / 2.11e+00 | 20260505 19:58:13.529 | INFO | PerfReport: 57.03%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.48e+00 / 1.13e+00 | 20260505 19:58:13.529 | INFO | PerfReport: 7.23%, PulseAccumulating -- 2.28e+00 / 2.86e+00, 4.37e-01 / 1.90e-01 | 20260505 19:58:13.529 | INFO | PerfReport: 3.25%, Metrics -- 1.03e+00 / 1.35e+00, 2.83e-01 / 1.37e-01 | 20260505 19:58:13.529 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.57e-01 / 4.25e-01, 6.36e-02 / 4.01e-02 | 20260505 19:58:13.530 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 19:58:13.530 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:58:13.530 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:58:13.530 | INFO | PerfReport: Full Execution -- 9.95e+01 / -nan, 2.54e+01 / 0.00e+00 | 20260505 19:58:13.530 | INFO | PerfReport: 22.46%, Upload -- 2.23e+01 / 0.00e+00, 7.73e+00 / 0.00e+00 | 20260505 19:58:13.530 | INFO | PerfReport: 21.78%, Download -- 2.17e+01 / 0.00e+00, 7.73e+00 / 0.00e+00 | 20260505 19:58:13.530 | INFO | PerfReport: 31.69%, Compute -- 3.15e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 19:58:13.530 | INFO | PerfReport: 2.27%, LockUpload -- 2.26e+00 / 0.00e+00, 7.85e+00 / 0.00e+00 | 20260505 19:58:13.530 | INFO | PerfReport: 1.34%, LockDownload -- 1.34e+00 / 0.00e+00, 2.87e+00 / 0.00e+00 | 20260505 19:58:13.530 | INFO | PerfReport: 20.43%, LockCompute -- 2.03e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 19:58:13.530 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.00e-02 / 0.00e+00 | 20260505 19:58:13.663 | INFO | Starting a round of Baz IO | 20260505 19:58:16.836 | INFO | Populating RTMetrics report | 20260505 19:58:16.837 | INFO | RTMetrics report populated | 20260505 19:58:16.837 | INFO | Opening tmp RTMetrics json file | 20260505 19:58:16.838 | INFO | Serializing RTMetrics to json | 20260505 19:58:16.840 | INFO | Serialization done, swapping tmp file now... | 20260505 19:58:16.840 | INFO | RTMetrics done | 20260505 19:58:16.922 | INFO | Finished a round of Baz IO: Wrote 2.028 GiB in 3.259 seconds | 20260505 19:58:16.922 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.72% / 0.00% / 78.28% | 20260505 19:58:16.922 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.189594% / 0% / 8.34465e-05% | 20260505 19:58:16.922 | INFO | Baz IO overall: | 20260505 19:58:16.922 | INFO | BytesWritten : 3445893105331 | 20260505 19:58:16.922 | INFO | NumEvents : 1629802153321 | 20260505 19:58:18.666 | INFO | Analyzing chunk frames = [5898240,5898752) | 20260505 19:58:18.666 | INFO | MemInfo: { | 20260505 19:58:18.666 | INFO | "gpu" : " 17 GB", | 20260505 19:58:18.666 | INFO | "huge" : "", | 20260505 19:58:18.666 | INFO | "huge_shared" : " 27 GB", | 20260505 19:58:18.666 | INFO | "malloc" : " 14 GB", | 20260505 19:58:18.666 | INFO | "pinned" : " 29 GB" | 20260505 19:58:18.666 | INFO | } | 20260505 19:58:18.666 | INFO | Perf report for the last 5.13886 seconds: | 20260505 19:58:18.666 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:58:18.666 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:18.666 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:18.666 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:18.666 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 19:58:18.666 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:18.666 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260505 19:58:18.666 | INFO | PRE_HQ : 60.2%, 1.8, 55.2ms, 0 | 20260505 19:58:18.666 | INFO | BAZWRITER : 62.9%, 3.7, 118.5ms, 0 | 20260505 19:58:18.666 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 19:58:18.667 | INFO | Uploaded 36GiB to GPU in 2.16436s (16.6331GiB/s) | 20260505 19:58:18.667 | INFO | Downloaded 38GiB to GPU in 2.24274s (16.9436GiB/s) | 20260505 19:58:23.816 | INFO | Analyzing chunk frames = [5898752,5899264) | 20260505 19:58:23.816 | INFO | MemInfo: { | 20260505 19:58:23.816 | INFO | "gpu" : " 17 GB", | 20260505 19:58:23.816 | INFO | "huge" : "", | 20260505 19:58:23.816 | INFO | "huge_shared" : " 27 GB", | 20260505 19:58:23.816 | INFO | "malloc" : " 14 GB", | 20260505 19:58:23.816 | INFO | "pinned" : " 29 GB" | 20260505 19:58:23.816 | INFO | } | 20260505 19:58:23.816 | INFO | Perf report for the last 5.15015 seconds: | 20260505 19:58:23.816 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:58:23.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:23.816 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:23.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:23.816 | INFO | ANALYSIS : 62.5%, 3.0, 97.8ms, 0 | 20260505 19:58:23.816 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:23.816 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:23.816 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 19:58:23.816 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:23.816 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:23.816 | INFO | Uploaded 36GiB to GPU in 2.13819s (16.8367GiB/s) | 20260505 19:58:23.816 | INFO | Downloaded 34GiB to GPU in 2.02964s (16.7517GiB/s) | 20260505 19:58:28.958 | INFO | Analyzing chunk frames = [5899264,5899776) | 20260505 19:58:28.958 | INFO | MemInfo: { | 20260505 19:58:28.958 | INFO | "gpu" : " 17 GB", | 20260505 19:58:28.958 | INFO | "huge" : "", | 20260505 19:58:28.958 | INFO | "huge_shared" : " 27 GB", | 20260505 19:58:28.958 | INFO | "malloc" : " 14 GB", | 20260505 19:58:28.958 | INFO | "pinned" : " 29 GB" | 20260505 19:58:28.958 | INFO | } | 20260505 19:58:28.958 | INFO | Perf report for the last 5.14152 seconds: | 20260505 19:58:28.958 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:58:28.958 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:28.958 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:28.958 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:28.958 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260505 19:58:28.958 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:28.958 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:28.958 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:58:28.958 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:28.958 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:28.958 | INFO | Uploaded 36GiB to GPU in 2.129s (16.9093GiB/s) | 20260505 19:58:28.958 | INFO | Downloaded 34GiB to GPU in 2.00899s (16.924GiB/s) | 20260505 19:58:34.089 | INFO | Analyzing chunk frames = [5899776,5900288) | 20260505 19:58:34.089 | INFO | MemInfo: { | 20260505 19:58:34.089 | INFO | "gpu" : " 17 GB", | 20260505 19:58:34.089 | INFO | "huge" : "", | 20260505 19:58:34.089 | INFO | "huge_shared" : " 27 GB", | 20260505 19:58:34.089 | INFO | "malloc" : " 14 GB", | 20260505 19:58:34.089 | INFO | "pinned" : " 29 GB" | 20260505 19:58:34.089 | INFO | } | 20260505 19:58:34.089 | INFO | Perf report for the last 5.13114 seconds: | 20260505 19:58:34.089 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:58:34.089 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:34.089 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:34.089 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:34.089 | INFO | ANALYSIS : 62.2%, 3.0, 96.5ms, 0 | 20260505 19:58:34.089 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:34.089 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:34.089 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:58:34.089 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:34.089 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:34.089 | INFO | Uploaded 36GiB to GPU in 2.14941s (16.7488GiB/s) | 20260505 19:58:34.089 | INFO | Downloaded 34GiB to GPU in 2.03298s (16.7242GiB/s) | 20260505 19:58:39.220 | INFO | Analyzing chunk frames = [5900288,5900800) | 20260505 19:58:39.220 | INFO | MemInfo: { | 20260505 19:58:39.220 | INFO | "gpu" : " 17 GB", | 20260505 19:58:39.220 | INFO | "huge" : "", | 20260505 19:58:39.220 | INFO | "huge_shared" : " 27 GB", | 20260505 19:58:39.220 | INFO | "malloc" : " 14 GB", | 20260505 19:58:39.220 | INFO | "pinned" : " 29 GB" | 20260505 19:58:39.220 | INFO | } | 20260505 19:58:39.220 | INFO | Perf report for the last 5.13123 seconds: | 20260505 19:58:39.220 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:58:39.220 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:39.220 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:39.220 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:39.220 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260505 19:58:39.220 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:39.220 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:39.220 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:58:39.220 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:39.220 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:39.220 | INFO | Uploaded 36GiB to GPU in 2.13871s (16.8326GiB/s) | 20260505 19:58:39.221 | INFO | Downloaded 34GiB to GPU in 2.06142s (16.4935GiB/s) | 20260505 19:58:44.366 | INFO | Analyzing chunk frames = [5900800,5901312) | 20260505 19:58:44.366 | INFO | MemInfo: { | 20260505 19:58:44.366 | INFO | "gpu" : " 17 GB", | 20260505 19:58:44.366 | INFO | "huge" : "", | 20260505 19:58:44.366 | INFO | "huge_shared" : " 27 GB", | 20260505 19:58:44.366 | INFO | "malloc" : " 14 GB", | 20260505 19:58:44.366 | INFO | "pinned" : " 29 GB" | 20260505 19:58:44.366 | INFO | } | 20260505 19:58:44.366 | INFO | Perf report for the last 5.14603 seconds: | 20260505 19:58:44.366 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:58:44.366 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:44.366 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:44.366 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:44.366 | INFO | ANALYSIS : 62.3%, 3.0, 97.5ms, 0 | 20260505 19:58:44.366 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:44.366 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:44.366 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:58:44.366 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:44.366 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:44.366 | INFO | Uploaded 36GiB to GPU in 2.15481s (16.7068GiB/s) | 20260505 19:58:44.366 | INFO | Downloaded 34GiB to GPU in 2.04594s (16.6182GiB/s) | 20260505 19:58:49.510 | INFO | Analyzing chunk frames = [5901312,5901824) | 20260505 19:58:49.510 | INFO | MemInfo: { | 20260505 19:58:49.510 | INFO | "gpu" : " 17 GB", | 20260505 19:58:49.510 | INFO | "huge" : "", | 20260505 19:58:49.510 | INFO | "huge_shared" : " 27 GB", | 20260505 19:58:49.510 | INFO | "malloc" : " 14 GB", | 20260505 19:58:49.510 | INFO | "pinned" : " 29 GB" | 20260505 19:58:49.510 | INFO | } | 20260505 19:58:49.511 | INFO | Perf report for the last 5.14433 seconds: | 20260505 19:58:49.511 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:58:49.511 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:49.511 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:49.511 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:49.511 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 19:58:49.511 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:49.511 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:49.511 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 19:58:49.511 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:49.511 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:49.511 | INFO | Uploaded 36GiB to GPU in 2.12471s (16.9435GiB/s) | 20260505 19:58:49.511 | INFO | Downloaded 34GiB to GPU in 2.13549s (15.9214GiB/s) | 20260505 19:58:54.658 | INFO | Analyzing chunk frames = [5901824,5902336) | 20260505 19:58:54.658 | INFO | MemInfo: { | 20260505 19:58:54.658 | INFO | "gpu" : " 17 GB", | 20260505 19:58:54.658 | INFO | "huge" : "", | 20260505 19:58:54.658 | INFO | "huge_shared" : " 27 GB", | 20260505 19:58:54.658 | INFO | "malloc" : " 14 GB", | 20260505 19:58:54.658 | INFO | "pinned" : " 29 GB" | 20260505 19:58:54.658 | INFO | } | 20260505 19:58:54.658 | INFO | Perf report for the last 5.14788 seconds: | 20260505 19:58:54.658 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:58:54.658 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:54.658 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:54.658 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:54.658 | INFO | ANALYSIS : 62.3%, 3.0, 97.4ms, 0 | 20260505 19:58:54.658 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:54.658 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:54.658 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:58:54.659 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:54.659 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:54.659 | INFO | Uploaded 36GiB to GPU in 2.11416s (17.0281GiB/s) | 20260505 19:58:54.659 | INFO | Downloaded 34GiB to GPU in 2.00899s (16.9239GiB/s) | 20260505 19:58:54.788 | INFO | Starting a round of Baz IO | 20260505 19:58:57.950 | INFO | Populating RTMetrics report | 20260505 19:58:57.950 | INFO | RTMetrics report populated | 20260505 19:58:57.950 | INFO | Opening tmp RTMetrics json file | 20260505 19:58:57.951 | INFO | Serializing RTMetrics to json | 20260505 19:58:57.952 | INFO | Serialization done, swapping tmp file now... | 20260505 19:58:57.952 | INFO | RTMetrics done | 20260505 19:58:58.040 | INFO | Finished a round of Baz IO: Wrote 2.029 GiB in 3.252 seconds | 20260505 19:58:58.040 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.72% / 0.00% / 78.28% | 20260505 19:58:58.040 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.189388% / 3.97364e-06% / 9.93411e-05% | 20260505 19:58:58.040 | INFO | Baz IO overall: | 20260505 19:58:58.040 | INFO | BytesWritten : 3448072018961 | 20260505 19:58:58.040 | INFO | NumEvents : 1630812256099 | 20260505 19:58:59.787 | INFO | Analyzing chunk frames = [5902336,5902848) | 20260505 19:58:59.787 | INFO | MemInfo: { | 20260505 19:58:59.787 | INFO | "gpu" : " 17 GB", | 20260505 19:58:59.787 | INFO | "huge" : "", | 20260505 19:58:59.787 | INFO | "huge_shared" : " 27 GB", | 20260505 19:58:59.787 | INFO | "malloc" : " 14 GB", | 20260505 19:58:59.787 | INFO | "pinned" : " 29 GB" | 20260505 19:58:59.787 | INFO | } | 20260505 19:58:59.787 | INFO | Perf report for the last 5.12886 seconds: | 20260505 19:58:59.787 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:58:59.787 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:59.787 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:59.787 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:59.787 | INFO | ANALYSIS : 63.1%, 3.0, 98.2ms, 0 | 20260505 19:58:59.787 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:58:59.787 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260505 19:58:59.787 | INFO | PRE_HQ : 60.2%, 1.8, 55.4ms, 0 | 20260505 19:58:59.787 | INFO | BAZWRITER : 62.8%, 3.7, 118.6ms, 0 | 20260505 19:58:59.787 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 19:58:59.788 | INFO | Uploaded 36GiB to GPU in 2.13458s (16.8651GiB/s) | 20260505 19:58:59.788 | INFO | Downloaded 38GiB to GPU in 2.22565s (17.0737GiB/s) | 20260505 19:59:04.922 | INFO | Analyzing chunk frames = [5902848,5903360) | 20260505 19:59:04.922 | INFO | MemInfo: { | 20260505 19:59:04.922 | INFO | "gpu" : " 17 GB", | 20260505 19:59:04.922 | INFO | "huge" : "", | 20260505 19:59:04.922 | INFO | "huge_shared" : " 27 GB", | 20260505 19:59:04.922 | INFO | "malloc" : " 14 GB", | 20260505 19:59:04.922 | INFO | "pinned" : " 29 GB" | 20260505 19:59:04.922 | INFO | } | 20260505 19:59:04.922 | INFO | Perf report for the last 5.13477 seconds: | 20260505 19:59:04.922 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:59:04.922 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:04.922 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:04.922 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:04.922 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 19:59:04.922 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:04.922 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:04.922 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 19:59:04.922 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:04.922 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:04.922 | INFO | Uploaded 36GiB to GPU in 2.13862s (16.8333GiB/s) | 20260505 19:59:04.922 | INFO | Downloaded 34GiB to GPU in 2.03759s (16.6864GiB/s) | 20260505 19:59:10.070 | INFO | Analyzing chunk frames = [5903360,5903872) | 20260505 19:59:10.070 | INFO | MemInfo: { | 20260505 19:59:10.070 | INFO | "gpu" : " 17 GB", | 20260505 19:59:10.070 | INFO | "huge" : "", | 20260505 19:59:10.070 | INFO | "huge_shared" : " 27 GB", | 20260505 19:59:10.070 | INFO | "malloc" : " 14 GB", | 20260505 19:59:10.070 | INFO | "pinned" : " 29 GB" | 20260505 19:59:10.070 | INFO | } | 20260505 19:59:10.070 | INFO | Perf report for the last 5.14835 seconds: | 20260505 19:59:10.070 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:59:10.070 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:10.070 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:10.070 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:10.071 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 19:59:10.071 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:10.071 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:10.071 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 19:59:10.071 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:10.071 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:10.071 | INFO | Uploaded 36GiB to GPU in 2.14671s (16.7698GiB/s) | 20260505 19:59:10.071 | INFO | Downloaded 34GiB to GPU in 2.04559s (16.6211GiB/s) | 20260505 19:59:15.208 | INFO | Analyzing chunk frames = [5903872,5904384) | 20260505 19:59:15.208 | INFO | MemInfo: { | 20260505 19:59:15.208 | INFO | "gpu" : " 17 GB", | 20260505 19:59:15.208 | INFO | "huge" : "", | 20260505 19:59:15.208 | INFO | "huge_shared" : " 27 GB", | 20260505 19:59:15.208 | INFO | "malloc" : " 14 GB", | 20260505 19:59:15.208 | INFO | "pinned" : " 29 GB" | 20260505 19:59:15.208 | INFO | } | 20260505 19:59:15.209 | INFO | Perf report for the last 5.13807 seconds: | 20260505 19:59:15.209 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:59:15.209 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:15.209 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:15.209 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:15.209 | INFO | ANALYSIS : 62.5%, 3.0, 97.1ms, 0 | 20260505 19:59:15.209 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:15.209 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:15.209 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:59:15.209 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:15.209 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:15.209 | INFO | Uploaded 36GiB to GPU in 2.21316s (16.2663GiB/s) | 20260505 19:59:15.209 | INFO | Downloaded 34GiB to GPU in 2.0582s (16.5193GiB/s) | 20260505 19:59:20.343 | INFO | Analyzing chunk frames = [5904384,5904896) | 20260505 19:59:20.343 | INFO | MemInfo: { | 20260505 19:59:20.343 | INFO | "gpu" : " 17 GB", | 20260505 19:59:20.343 | INFO | "huge" : "", | 20260505 19:59:20.343 | INFO | "huge_shared" : " 27 GB", | 20260505 19:59:20.343 | INFO | "malloc" : " 14 GB", | 20260505 19:59:20.343 | INFO | "pinned" : " 29 GB" | 20260505 19:59:20.343 | INFO | } | 20260505 19:59:20.343 | INFO | Perf report for the last 5.13483 seconds: | 20260505 19:59:20.343 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:59:20.343 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:20.343 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:20.343 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:20.343 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 19:59:20.343 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:20.343 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:20.343 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 19:59:20.343 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:20.343 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:20.344 | INFO | Uploaded 36GiB to GPU in 2.14084s (16.8158GiB/s) | 20260505 19:59:20.344 | INFO | Downloaded 34GiB to GPU in 2.06196s (16.4891GiB/s) | 20260505 19:59:25.480 | INFO | Analyzing chunk frames = [5904896,5905408) | 20260505 19:59:25.480 | INFO | MemInfo: { | 20260505 19:59:25.480 | INFO | "gpu" : " 17 GB", | 20260505 19:59:25.480 | INFO | "huge" : "", | 20260505 19:59:25.480 | INFO | "huge_shared" : " 27 GB", | 20260505 19:59:25.480 | INFO | "malloc" : " 14 GB", | 20260505 19:59:25.480 | INFO | "pinned" : " 29 GB" | 20260505 19:59:25.480 | INFO | } | 20260505 19:59:25.480 | INFO | Perf report for the last 5.13674 seconds: | 20260505 19:59:25.480 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:59:25.480 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:25.480 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:25.480 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:25.480 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 19:59:25.480 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:25.480 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:25.480 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 19:59:25.480 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:25.480 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:25.480 | INFO | Uploaded 36GiB to GPU in 2.15292s (16.7215GiB/s) | 20260505 19:59:25.480 | INFO | Downloaded 34GiB to GPU in 2.05705s (16.5285GiB/s) | 20260505 19:59:30.621 | INFO | Analyzing chunk frames = [5905408,5905920) | 20260505 19:59:30.622 | INFO | MemInfo: { | 20260505 19:59:30.622 | INFO | "gpu" : " 17 GB", | 20260505 19:59:30.622 | INFO | "huge" : "", | 20260505 19:59:30.622 | INFO | "huge_shared" : " 27 GB", | 20260505 19:59:30.622 | INFO | "malloc" : " 14 GB", | 20260505 19:59:30.622 | INFO | "pinned" : " 29 GB" | 20260505 19:59:30.622 | INFO | } | 20260505 19:59:30.622 | INFO | Perf report for the last 5.14153 seconds: | 20260505 19:59:30.622 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:59:30.622 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:30.622 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:30.622 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:30.622 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 19:59:30.622 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:30.622 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:30.622 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 19:59:30.622 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:30.622 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:30.622 | INFO | Uploaded 36GiB to GPU in 2.14184s (16.808GiB/s) | 20260505 19:59:30.622 | INFO | Downloaded 34GiB to GPU in 2.1475s (15.8323GiB/s) | 20260505 19:59:35.764 | INFO | Analyzing chunk frames = [5905920,5906432) | 20260505 19:59:35.764 | INFO | MemInfo: { | 20260505 19:59:35.764 | INFO | "gpu" : " 17 GB", | 20260505 19:59:35.764 | INFO | "huge" : "", | 20260505 19:59:35.764 | INFO | "huge_shared" : " 27 GB", | 20260505 19:59:35.764 | INFO | "malloc" : " 14 GB", | 20260505 19:59:35.765 | INFO | "pinned" : " 29 GB" | 20260505 19:59:35.765 | INFO | } | 20260505 19:59:35.765 | INFO | Perf report for the last 5.14291 seconds: | 20260505 19:59:35.765 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:59:35.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:35.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:35.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:35.765 | INFO | ANALYSIS : 62.4%, 3.0, 97.6ms, 0 | 20260505 19:59:35.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:35.765 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:35.765 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 19:59:35.765 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:35.765 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:35.765 | INFO | Uploaded 36GiB to GPU in 2.12035s (16.9783GiB/s) | 20260505 19:59:35.765 | INFO | Downloaded 34GiB to GPU in 2.00715s (16.9395GiB/s) | 20260505 19:59:35.765 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 19:59:35.765 | INFO | Total mem usage: 15.3 GiB | 20260505 19:59:35.765 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 19:59:35.765 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.11%) | 20260505 19:59:35.765 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 19:59:35.765 | INFO | High Water Breakdown: | 20260505 19:59:35.765 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 19:59:35.765 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 19:59:35.765 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 19:59:35.765 | INFO | ------------------------------------------ | 20260505 19:59:35.765 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:59:35.765 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 19:59:35.765 | INFO | Total mem usage: 26.5 GiB | 20260505 19:59:35.765 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 19:59:35.765 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 19:59:35.765 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 19:59:35.765 | INFO | High Water Breakdown: | 20260505 19:59:35.765 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:59:35.765 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:59:35.765 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:59:35.765 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:59:35.765 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:59:35.765 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:59:35.765 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 19:59:35.765 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 19:59:35.765 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:59:35.765 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 19:59:35.765 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 19:59:35.765 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 19:59:35.765 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 19:59:35.765 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 19:59:35.765 | INFO | ------------------------------------------ | 20260505 19:59:35.765 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:59:35.765 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 19:59:35.765 | INFO | Total mem usage: 25.6 GiB | 20260505 19:59:35.766 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 19:59:35.766 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 19:59:35.766 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 19:59:35.766 | INFO | High Water Breakdown: | 20260505 19:59:35.766 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 19:59:35.766 | INFO | ------------------------------------------ | 20260505 19:59:35.766 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:59:35.766 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 19:59:35.766 | INFO | Total mem usage: 15.7 GiB | 20260505 19:59:35.766 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 19:59:35.766 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 19:59:35.766 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 19:59:35.766 | INFO | High Water Breakdown: | 20260505 19:59:35.766 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 19:59:35.766 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 19:59:35.766 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 19:59:35.766 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 19:59:35.766 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 19:59:35.766 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 19:59:35.766 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 19:59:35.766 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 19:59:35.766 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 19:59:35.766 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 19:59:35.766 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 19:59:35.766 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 19:59:35.766 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 19:59:35.766 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 19:59:35.766 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 19:59:35.766 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 19:59:35.766 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 19:59:35.766 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 19:59:35.766 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 19:59:35.766 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 19:59:35.766 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 19:59:35.766 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 19:59:35.766 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 19:59:35.766 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 19:59:35.766 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 19:59:35.766 | INFO | ------------------------------------------ | 20260505 19:59:35.766 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 19:59:35.766 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 19:59:35.766 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:59:35.766 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:59:35.766 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.47e+01, 1.33e+01 / 1.74e+00 | 20260505 19:59:35.766 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.03e-01, 1.03e-02 / 5.66e-03 | 20260505 19:59:35.766 | INFO | PerfReport: 11.78%, Baseline -- 3.71e+00 / 4.53e+00, 5.64e-01 / 1.65e-01 | 20260505 19:59:35.766 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.51e+00, 1.90e-01 / 9.26e-02 | 20260505 19:59:35.766 | INFO | PerfReport: 15.95%, DME -- 4.02e+01 / 4.44e+01, 4.02e+00 / 1.88e+00 | 20260505 19:59:35.766 | INFO | PerfReport: 57.11%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.55e+00 / 1.08e+00 | 20260505 19:59:35.766 | INFO | PerfReport: 7.21%, PulseAccumulating -- 2.27e+00 / 2.85e+00, 4.31e-01 / 1.91e-01 | 20260505 19:59:35.766 | INFO | PerfReport: 3.26%, Metrics -- 1.03e+00 / 1.36e+00, 2.87e-01 / 1.28e-01 | 20260505 19:59:35.766 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.58e-01 / 4.24e-01, 6.43e-02 / 3.87e-02 | 20260505 19:59:35.766 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 19:59:35.766 | INFO | PerfReport: Gathered 1536 samples. | 20260505 19:59:35.766 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 19:59:35.766 | INFO | PerfReport: Full Execution -- 9.95e+01 / -nan, 2.53e+01 / 0.00e+00 | 20260505 19:59:35.766 | INFO | PerfReport: 22.45%, Upload -- 2.23e+01 / 0.00e+00, 7.73e+00 / 0.00e+00 | 20260505 19:59:35.766 | INFO | PerfReport: 21.73%, Download -- 2.16e+01 / 0.00e+00, 7.66e+00 / 0.00e+00 | 20260505 19:59:35.766 | INFO | PerfReport: 31.70%, Compute -- 3.15e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 19:59:35.766 | INFO | PerfReport: 2.32%, LockUpload -- 2.31e+00 / 0.00e+00, 8.09e+00 / 0.00e+00 | 20260505 19:59:35.766 | INFO | PerfReport: 1.33%, LockDownload -- 1.32e+00 / 0.00e+00, 2.87e+00 / 0.00e+00 | 20260505 19:59:35.766 | INFO | PerfReport: 20.46%, LockCompute -- 2.04e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260505 19:59:35.766 | INFO | PerfReport: 0.02%, Misc -- 1.65e-02 / 0.00e+00, 7.83e-02 / 0.00e+00 | 20260505 19:59:35.898 | INFO | Starting a round of Baz IO | 20260505 19:59:39.058 | INFO | Populating RTMetrics report | 20260505 19:59:39.059 | INFO | RTMetrics report populated | 20260505 19:59:39.059 | INFO | Opening tmp RTMetrics json file | 20260505 19:59:39.060 | INFO | Serializing RTMetrics to json | 20260505 19:59:39.061 | INFO | Serialization done, swapping tmp file now... | 20260505 19:59:39.061 | INFO | RTMetrics done | 20260505 19:59:39.149 | INFO | Finished a round of Baz IO: Wrote 2.028 GiB in 3.25 seconds | 20260505 19:59:39.149 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.72% / 0.00% / 78.28% | 20260505 19:59:39.149 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.189435% / 0% / 8.74201e-05% | 20260505 19:59:39.149 | INFO | Baz IO overall: | 20260505 19:59:39.149 | INFO | BytesWritten : 3450249615129 | 20260505 19:59:39.149 | INFO | NumEvents : 1631821753008 | 20260505 19:59:40.903 | INFO | Analyzing chunk frames = [5906432,5906944) | 20260505 19:59:40.903 | INFO | MemInfo: { | 20260505 19:59:40.903 | INFO | "gpu" : " 17 GB", | 20260505 19:59:40.903 | INFO | "huge" : "", | 20260505 19:59:40.903 | INFO | "huge_shared" : " 27 GB", | 20260505 19:59:40.903 | INFO | "malloc" : " 14 GB", | 20260505 19:59:40.903 | INFO | "pinned" : " 29 GB" | 20260505 19:59:40.903 | INFO | } | 20260505 19:59:40.903 | INFO | Perf report for the last 5.13799 seconds: | 20260505 19:59:40.903 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:59:40.903 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:40.903 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:40.903 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:40.903 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 19:59:40.903 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:40.903 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260505 19:59:40.903 | INFO | PRE_HQ : 60.1%, 1.8, 55.3ms, 0 | 20260505 19:59:40.903 | INFO | BAZWRITER : 62.8%, 3.7, 119.7ms, 0 | 20260505 19:59:40.903 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 19:59:40.903 | INFO | Uploaded 36GiB to GPU in 2.18284s (16.4923GiB/s) | 20260505 19:59:40.903 | INFO | Downloaded 38GiB to GPU in 2.24889s (16.8973GiB/s) | 20260505 19:59:46.034 | INFO | Analyzing chunk frames = [5906944,5907456) | 20260505 19:59:46.034 | INFO | MemInfo: { | 20260505 19:59:46.034 | INFO | "gpu" : " 17 GB", | 20260505 19:59:46.034 | INFO | "huge" : "", | 20260505 19:59:46.034 | INFO | "huge_shared" : " 27 GB", | 20260505 19:59:46.034 | INFO | "malloc" : " 14 GB", | 20260505 19:59:46.034 | INFO | "pinned" : " 29 GB" | 20260505 19:59:46.034 | INFO | } | 20260505 19:59:46.034 | INFO | Perf report for the last 5.13167 seconds: | 20260505 19:59:46.034 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:59:46.034 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:46.034 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:46.034 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:46.034 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260505 19:59:46.034 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:46.034 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:46.034 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260505 19:59:46.034 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:46.034 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:46.034 | INFO | Uploaded 36GiB to GPU in 2.10433s (17.1076GiB/s) | 20260505 19:59:46.034 | INFO | Downloaded 34GiB to GPU in 2.02446s (16.7946GiB/s) | 20260505 19:59:51.173 | INFO | Analyzing chunk frames = [5907456,5907968) | 20260505 19:59:51.173 | INFO | MemInfo: { | 20260505 19:59:51.173 | INFO | "gpu" : " 17 GB", | 20260505 19:59:51.173 | INFO | "huge" : "", | 20260505 19:59:51.173 | INFO | "huge_shared" : " 27 GB", | 20260505 19:59:51.173 | INFO | "malloc" : " 14 GB", | 20260505 19:59:51.173 | INFO | "pinned" : " 29 GB" | 20260505 19:59:51.173 | INFO | } | 20260505 19:59:51.173 | INFO | Perf report for the last 5.13874 seconds: | 20260505 19:59:51.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:59:51.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:51.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:51.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:51.173 | INFO | ANALYSIS : 62.7%, 3.0, 97.5ms, 0 | 20260505 19:59:51.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:51.173 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:51.173 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:59:51.173 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:51.173 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:51.173 | INFO | Uploaded 36GiB to GPU in 2.21131s (16.2799GiB/s) | 20260505 19:59:51.173 | INFO | Downloaded 34GiB to GPU in 2.06761s (16.4441GiB/s) | 20260505 19:59:56.322 | INFO | Analyzing chunk frames = [5907968,5908480) | 20260505 19:59:56.322 | INFO | MemInfo: { | 20260505 19:59:56.322 | INFO | "gpu" : " 17 GB", | 20260505 19:59:56.322 | INFO | "huge" : "", | 20260505 19:59:56.322 | INFO | "huge_shared" : " 27 GB", | 20260505 19:59:56.322 | INFO | "malloc" : " 14 GB", | 20260505 19:59:56.322 | INFO | "pinned" : " 29 GB" | 20260505 19:59:56.322 | INFO | } | 20260505 19:59:56.322 | INFO | Perf report for the last 5.14889 seconds: | 20260505 19:59:56.322 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 19:59:56.322 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:56.322 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:56.322 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:56.322 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260505 19:59:56.322 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:56.322 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:56.322 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 19:59:56.322 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:56.322 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 19:59:56.322 | INFO | Uploaded 36GiB to GPU in 2.18621s (16.4669GiB/s) | 20260505 19:59:56.322 | INFO | Downloaded 34GiB to GPU in 2.07254s (16.405GiB/s) | 20260505 20:00:01.463 | INFO | Analyzing chunk frames = [5908480,5908992) | 20260505 20:00:01.463 | INFO | MemInfo: { | 20260505 20:00:01.463 | INFO | "gpu" : " 17 GB", | 20260505 20:00:01.463 | INFO | "huge" : "", | 20260505 20:00:01.463 | INFO | "huge_shared" : " 27 GB", | 20260505 20:00:01.463 | INFO | "malloc" : " 14 GB", | 20260505 20:00:01.463 | INFO | "pinned" : " 29 GB" | 20260505 20:00:01.463 | INFO | } | 20260505 20:00:01.463 | INFO | Perf report for the last 5.14103 seconds: | 20260505 20:00:01.463 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:00:01.463 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:01.463 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:01.463 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:01.463 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260505 20:00:01.463 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:01.463 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:01.463 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:00:01.463 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:01.463 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:01.463 | INFO | Uploaded 36GiB to GPU in 2.13045s (16.8979GiB/s) | 20260505 20:00:01.463 | INFO | Downloaded 34GiB to GPU in 2.06186s (16.49GiB/s) | 20260505 20:00:06.606 | INFO | Analyzing chunk frames = [5908992,5909504) | 20260505 20:00:06.606 | INFO | MemInfo: { | 20260505 20:00:06.606 | INFO | "gpu" : " 17 GB", | 20260505 20:00:06.606 | INFO | "huge" : "", | 20260505 20:00:06.606 | INFO | "huge_shared" : " 27 GB", | 20260505 20:00:06.606 | INFO | "malloc" : " 14 GB", | 20260505 20:00:06.606 | INFO | "pinned" : " 29 GB" | 20260505 20:00:06.606 | INFO | } | 20260505 20:00:06.606 | INFO | Perf report for the last 5.14353 seconds: | 20260505 20:00:06.606 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:00:06.606 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:06.606 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:06.607 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:06.607 | INFO | ANALYSIS : 62.1%, 3.0, 97.1ms, 0 | 20260505 20:00:06.607 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:06.607 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:06.607 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:00:06.607 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:06.607 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:06.607 | INFO | Uploaded 36GiB to GPU in 2.14031s (16.82GiB/s) | 20260505 20:00:06.607 | INFO | Downloaded 34GiB to GPU in 2.05982s (16.5063GiB/s) | 20260505 20:00:11.734 | INFO | Analyzing chunk frames = [5909504,5910016) | 20260505 20:00:11.734 | INFO | MemInfo: { | 20260505 20:00:11.734 | INFO | "gpu" : " 17 GB", | 20260505 20:00:11.734 | INFO | "huge" : "", | 20260505 20:00:11.734 | INFO | "huge_shared" : " 27 GB", | 20260505 20:00:11.734 | INFO | "malloc" : " 14 GB", | 20260505 20:00:11.734 | INFO | "pinned" : " 29 GB" | 20260505 20:00:11.734 | INFO | } | 20260505 20:00:11.734 | INFO | Perf report for the last 5.12765 seconds: | 20260505 20:00:11.734 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:00:11.734 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:11.734 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:11.734 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:11.734 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 20:00:11.734 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:11.734 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:11.734 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 20:00:11.734 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:11.734 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:11.734 | INFO | Uploaded 36GiB to GPU in 2.12154s (16.9688GiB/s) | 20260505 20:00:11.734 | INFO | Downloaded 34GiB to GPU in 2.14901s (15.8212GiB/s) | 20260505 20:00:16.880 | INFO | Analyzing chunk frames = [5910016,5910528) | 20260505 20:00:16.880 | INFO | MemInfo: { | 20260505 20:00:16.880 | INFO | "gpu" : " 17 GB", | 20260505 20:00:16.880 | INFO | "huge" : "", | 20260505 20:00:16.880 | INFO | "huge_shared" : " 27 GB", | 20260505 20:00:16.880 | INFO | "malloc" : " 14 GB", | 20260505 20:00:16.880 | INFO | "pinned" : " 29 GB" | 20260505 20:00:16.880 | INFO | } | 20260505 20:00:16.880 | INFO | Perf report for the last 5.14561 seconds: | 20260505 20:00:16.880 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:00:16.880 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:16.880 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:16.880 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:16.880 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 20:00:16.880 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:16.880 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:16.880 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 20:00:16.880 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:16.880 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:16.880 | INFO | Uploaded 36GiB to GPU in 2.19845s (16.3752GiB/s) | 20260505 20:00:16.880 | INFO | Downloaded 34GiB to GPU in 2.03621s (16.6977GiB/s) | 20260505 20:00:17.013 | INFO | Starting a round of Baz IO | 20260505 20:00:20.179 | INFO | Populating RTMetrics report | 20260505 20:00:20.179 | INFO | RTMetrics report populated | 20260505 20:00:20.179 | INFO | Opening tmp RTMetrics json file | 20260505 20:00:20.181 | INFO | Serializing RTMetrics to json | 20260505 20:00:20.182 | INFO | Serialization done, swapping tmp file now... | 20260505 20:00:20.182 | INFO | RTMetrics done | 20260505 20:00:20.268 | INFO | Finished a round of Baz IO: Wrote 2.026 GiB in 3.255 seconds | 20260505 20:00:20.268 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.72% / 0.00% / 78.28% | 20260505 20:00:20.268 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.188371% / 7.94729e-06% / 6.75519e-05% | 20260505 20:00:20.268 | INFO | Baz IO overall: | 20260505 20:00:20.268 | INFO | BytesWritten : 3452425403061 | 20260505 20:00:20.268 | INFO | NumEvents : 1632830149977 | 20260505 20:00:22.016 | INFO | Analyzing chunk frames = [5910528,5911040) | 20260505 20:00:22.016 | INFO | MemInfo: { | 20260505 20:00:22.016 | INFO | "gpu" : " 17 GB", | 20260505 20:00:22.016 | INFO | "huge" : "", | 20260505 20:00:22.017 | INFO | "huge_shared" : " 27 GB", | 20260505 20:00:22.017 | INFO | "malloc" : " 14 GB", | 20260505 20:00:22.017 | INFO | "pinned" : " 29 GB" | 20260505 20:00:22.017 | INFO | } | 20260505 20:00:22.017 | INFO | Perf report for the last 5.13686 seconds: | 20260505 20:00:22.017 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:00:22.017 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:22.017 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:22.017 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:22.017 | INFO | ANALYSIS : 63.2%, 3.0, 98.4ms, 0 | 20260505 20:00:22.017 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:22.017 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260505 20:00:22.017 | INFO | PRE_HQ : 60.1%, 1.8, 55.4ms, 0 | 20260505 20:00:22.017 | INFO | BAZWRITER : 62.8%, 3.7, 118.6ms, 0 | 20260505 20:00:22.017 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 20:00:22.017 | INFO | Uploaded 36GiB to GPU in 2.16718s (16.6115GiB/s) | 20260505 20:00:22.017 | INFO | Downloaded 38GiB to GPU in 2.2505s (16.8851GiB/s) | 20260505 20:00:27.157 | INFO | Analyzing chunk frames = [5911040,5911552) | 20260505 20:00:27.157 | INFO | MemInfo: { | 20260505 20:00:27.157 | INFO | "gpu" : " 17 GB", | 20260505 20:00:27.157 | INFO | "huge" : "", | 20260505 20:00:27.157 | INFO | "huge_shared" : " 27 GB", | 20260505 20:00:27.157 | INFO | "malloc" : " 14 GB", | 20260505 20:00:27.157 | INFO | "pinned" : " 29 GB" | 20260505 20:00:27.157 | INFO | } | 20260505 20:00:27.158 | INFO | Perf report for the last 5.14088 seconds: | 20260505 20:00:27.158 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:00:27.158 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:27.158 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:27.158 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:27.158 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 20:00:27.158 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:27.158 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:27.158 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:00:27.158 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:27.158 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:27.158 | INFO | Uploaded 36GiB to GPU in 2.10718s (17.0845GiB/s) | 20260505 20:00:27.158 | INFO | Downloaded 34GiB to GPU in 2.03303s (16.7238GiB/s) | 20260505 20:00:32.297 | INFO | Analyzing chunk frames = [5911552,5912064) | 20260505 20:00:32.297 | INFO | MemInfo: { | 20260505 20:00:32.297 | INFO | "gpu" : " 17 GB", | 20260505 20:00:32.297 | INFO | "huge" : "", | 20260505 20:00:32.297 | INFO | "huge_shared" : " 27 GB", | 20260505 20:00:32.297 | INFO | "malloc" : " 14 GB", | 20260505 20:00:32.297 | INFO | "pinned" : " 29 GB" | 20260505 20:00:32.297 | INFO | } | 20260505 20:00:32.297 | INFO | Perf report for the last 5.13909 seconds: | 20260505 20:00:32.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:00:32.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:32.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:32.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:32.297 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260505 20:00:32.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:32.297 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:32.297 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:00:32.297 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:32.297 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:32.297 | INFO | Uploaded 36GiB to GPU in 2.12545s (16.9376GiB/s) | 20260505 20:00:32.297 | INFO | Downloaded 34GiB to GPU in 2.01706s (16.8562GiB/s) | 20260505 20:00:37.436 | INFO | Analyzing chunk frames = [5912064,5912576) | 20260505 20:00:37.436 | INFO | MemInfo: { | 20260505 20:00:37.436 | INFO | "gpu" : " 17 GB", | 20260505 20:00:37.436 | INFO | "huge" : "", | 20260505 20:00:37.436 | INFO | "huge_shared" : " 27 GB", | 20260505 20:00:37.436 | INFO | "malloc" : " 14 GB", | 20260505 20:00:37.436 | INFO | "pinned" : " 29 GB" | 20260505 20:00:37.436 | INFO | } | 20260505 20:00:37.436 | INFO | Perf report for the last 5.1391 seconds: | 20260505 20:00:37.436 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:00:37.436 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:37.436 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:37.436 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:37.436 | INFO | ANALYSIS : 62.2%, 3.0, 96.6ms, 0 | 20260505 20:00:37.436 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:37.436 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:37.436 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:00:37.436 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:37.436 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:37.436 | INFO | Uploaded 36GiB to GPU in 2.16872s (16.5997GiB/s) | 20260505 20:00:37.436 | INFO | Downloaded 34GiB to GPU in 2.06151s (16.4928GiB/s) | 20260505 20:00:42.579 | INFO | Analyzing chunk frames = [5912576,5913088) | 20260505 20:00:42.579 | INFO | MemInfo: { | 20260505 20:00:42.579 | INFO | "gpu" : " 17 GB", | 20260505 20:00:42.579 | INFO | "huge" : "", | 20260505 20:00:42.579 | INFO | "huge_shared" : " 27 GB", | 20260505 20:00:42.579 | INFO | "malloc" : " 14 GB", | 20260505 20:00:42.579 | INFO | "pinned" : " 29 GB" | 20260505 20:00:42.579 | INFO | } | 20260505 20:00:42.579 | INFO | Perf report for the last 5.1434 seconds: | 20260505 20:00:42.579 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:00:42.579 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:42.579 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:42.579 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:42.579 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260505 20:00:42.579 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:42.579 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:42.579 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 20:00:42.579 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:42.579 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:42.579 | INFO | Uploaded 36GiB to GPU in 2.1382s (16.8366GiB/s) | 20260505 20:00:42.579 | INFO | Downloaded 34GiB to GPU in 2.06172s (16.4911GiB/s) | 20260505 20:00:47.719 | INFO | Analyzing chunk frames = [5913088,5913600) | 20260505 20:00:47.719 | INFO | MemInfo: { | 20260505 20:00:47.719 | INFO | "gpu" : " 17 GB", | 20260505 20:00:47.719 | INFO | "huge" : "", | 20260505 20:00:47.719 | INFO | "huge_shared" : " 27 GB", | 20260505 20:00:47.719 | INFO | "malloc" : " 14 GB", | 20260505 20:00:47.719 | INFO | "pinned" : " 29 GB" | 20260505 20:00:47.719 | INFO | } | 20260505 20:00:47.719 | INFO | Perf report for the last 5.13965 seconds: | 20260505 20:00:47.719 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:00:47.719 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:47.719 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:47.719 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:47.719 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 20:00:47.719 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:47.719 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:47.719 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260505 20:00:47.719 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:47.719 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:47.719 | INFO | Uploaded 36GiB to GPU in 2.08806s (17.2409GiB/s) | 20260505 20:00:47.719 | INFO | Downloaded 34GiB to GPU in 2.03855s (16.6785GiB/s) | 20260505 20:00:52.849 | INFO | Analyzing chunk frames = [5913600,5914112) | 20260505 20:00:52.849 | INFO | MemInfo: { | 20260505 20:00:52.849 | INFO | "gpu" : " 17 GB", | 20260505 20:00:52.849 | INFO | "huge" : "", | 20260505 20:00:52.849 | INFO | "huge_shared" : " 27 GB", | 20260505 20:00:52.849 | INFO | "malloc" : " 14 GB", | 20260505 20:00:52.849 | INFO | "pinned" : " 29 GB" | 20260505 20:00:52.849 | INFO | } | 20260505 20:00:52.849 | INFO | Perf report for the last 5.13045 seconds: | 20260505 20:00:52.849 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:00:52.849 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:52.849 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:52.849 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:52.849 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 20:00:52.849 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:52.849 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:52.849 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 20:00:52.849 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:52.849 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:52.849 | INFO | Uploaded 36GiB to GPU in 2.16913s (16.5965GiB/s) | 20260505 20:00:52.850 | INFO | Downloaded 34GiB to GPU in 2.13047s (15.9589GiB/s) | 20260505 20:00:57.985 | INFO | Analyzing chunk frames = [5914112,5914624) | 20260505 20:00:57.985 | INFO | MemInfo: { | 20260505 20:00:57.985 | INFO | "gpu" : " 17 GB", | 20260505 20:00:57.985 | INFO | "huge" : "", | 20260505 20:00:57.985 | INFO | "huge_shared" : " 27 GB", | 20260505 20:00:57.985 | INFO | "malloc" : " 14 GB", | 20260505 20:00:57.985 | INFO | "pinned" : " 29 GB" | 20260505 20:00:57.985 | INFO | } | 20260505 20:00:57.985 | INFO | Perf report for the last 5.13584 seconds: | 20260505 20:00:57.985 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:00:57.985 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:57.985 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:57.985 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:57.985 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 20:00:57.985 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:57.985 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:57.985 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 20:00:57.985 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:57.985 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:00:57.985 | INFO | Uploaded 36GiB to GPU in 2.20903s (16.2968GiB/s) | 20260505 20:00:57.985 | INFO | Downloaded 34GiB to GPU in 2.04907s (16.5929GiB/s) | 20260505 20:00:57.985 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 20:00:57.985 | INFO | Total mem usage: 15.3 GiB | 20260505 20:00:57.985 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 20:00:57.985 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.2%) | 20260505 20:00:57.985 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 20:00:57.985 | INFO | High Water Breakdown: | 20260505 20:00:57.985 | INFO | 61% : 8.64 GiB ( 90.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 20:00:57.985 | INFO | 38.9% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 20:00:57.985 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 20:00:57.985 | INFO | ------------------------------------------ | 20260505 20:00:57.986 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:00:57.986 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 20:00:57.986 | INFO | Total mem usage: 26.5 GiB | 20260505 20:00:57.986 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 20:00:57.986 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 20:00:57.986 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 20:00:57.986 | INFO | High Water Breakdown: | 20260505 20:00:57.986 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:00:57.986 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:00:57.986 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:00:57.986 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:00:57.986 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:00:57.986 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:00:57.986 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 20:00:57.986 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 20:00:57.986 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:00:57.986 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 20:00:57.986 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 20:00:57.986 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 20:00:57.986 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 20:00:57.986 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 20:00:57.986 | INFO | ------------------------------------------ | 20260505 20:00:57.986 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:00:57.986 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 20:00:57.986 | INFO | Total mem usage: 25.6 GiB | 20260505 20:00:57.986 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 20:00:57.986 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 20:00:57.986 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 20:00:57.986 | INFO | High Water Breakdown: | 20260505 20:00:57.986 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 20:00:57.986 | INFO | ------------------------------------------ | 20260505 20:00:57.986 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:00:57.986 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 20:00:57.986 | INFO | Total mem usage: 15.7 GiB | 20260505 20:00:57.986 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 20:00:57.986 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 20:00:57.986 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 20:00:57.986 | INFO | High Water Breakdown: | 20260505 20:00:57.986 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 20:00:57.986 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 20:00:57.986 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 20:00:57.986 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 20:00:57.986 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 20:00:57.986 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 20:00:57.986 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 20:00:57.986 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 20:00:57.986 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 20:00:57.986 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:00:57.986 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 20:00:57.986 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:00:57.986 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 20:00:57.986 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 20:00:57.986 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 20:00:57.986 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 20:00:57.986 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:00:57.986 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 20:00:57.986 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:00:57.986 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:00:57.986 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:00:57.986 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 20:00:57.986 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 20:00:57.986 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:00:57.986 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 20:00:57.986 | INFO | ------------------------------------------ | 20260505 20:00:57.986 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:00:57.986 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 20:00:57.986 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:00:57.986 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:00:57.986 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.45e+01, 1.34e+01 / 1.79e+00 | 20260505 20:00:57.986 | INFO | PerfReport: 0.29%, AggregateStats -- 9.25e-02 / 1.03e-01, 1.04e-02 / 4.72e-03 | 20260505 20:00:57.986 | INFO | PerfReport: 11.78%, Baseline -- 3.71e+00 / 4.57e+00, 5.62e-01 / 1.32e-01 | 20260505 20:00:57.986 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.52e+00, 1.90e-01 / 1.04e-01 | 20260505 20:00:57.986 | INFO | PerfReport: 15.92%, DME -- 4.01e+01 / 4.41e+01, 3.97e+00 / 1.70e+00 | 20260505 20:00:57.986 | INFO | PerfReport: 57.10%, FrameLabeling -- 1.80e+01 / 1.96e+01, 1.49e+00 / 1.23e+00 | 20260505 20:00:57.986 | INFO | PerfReport: 7.23%, PulseAccumulating -- 2.28e+00 / 2.83e+00, 4.35e-01 / 2.04e-01 | 20260505 20:00:57.986 | INFO | PerfReport: 3.26%, Metrics -- 1.03e+00 / 1.35e+00, 2.80e-01 / 1.68e-01 | 20260505 20:00:57.986 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.57e-01 / 4.12e-01, 6.24e-02 / 2.97e-02 | 20260505 20:00:57.986 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 20:00:57.986 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:00:57.986 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:00:57.986 | INFO | PerfReport: Full Execution -- 9.95e+01 / -nan, 2.55e+01 / 0.00e+00 | 20260505 20:00:57.986 | INFO | PerfReport: 22.55%, Upload -- 2.24e+01 / 0.00e+00, 7.82e+00 / 0.00e+00 | 20260505 20:00:57.986 | INFO | PerfReport: 21.84%, Download -- 2.17e+01 / 0.00e+00, 7.73e+00 / 0.00e+00 | 20260505 20:00:57.986 | INFO | PerfReport: 31.66%, Compute -- 3.15e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 20:00:57.986 | INFO | PerfReport: 2.29%, LockUpload -- 2.28e+00 / 0.00e+00, 7.80e+00 / 0.00e+00 | 20260505 20:00:57.986 | INFO | PerfReport: 1.33%, LockDownload -- 1.32e+00 / 0.00e+00, 2.82e+00 / 0.00e+00 | 20260505 20:00:57.986 | INFO | PerfReport: 20.31%, LockCompute -- 2.02e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 20:00:57.986 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.86e-02 / 0.00e+00 | 20260505 20:00:58.120 | INFO | Starting a round of Baz IO | 20260505 20:01:01.297 | INFO | Populating RTMetrics report | 20260505 20:01:01.297 | INFO | RTMetrics report populated | 20260505 20:01:01.297 | INFO | Opening tmp RTMetrics json file | 20260505 20:01:01.298 | INFO | Serializing RTMetrics to json | 20260505 20:01:01.299 | INFO | Serialization done, swapping tmp file now... | 20260505 20:01:01.299 | INFO | RTMetrics done | 20260505 20:01:01.388 | INFO | Finished a round of Baz IO: Wrote 2.025 GiB in 3.267 seconds | 20260505 20:01:01.388 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.71% / 0.00% / 78.28% | 20260505 20:01:01.388 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.187174% / 0% / 7.54992e-05% | 20260505 20:01:01.388 | INFO | Baz IO overall: | 20260505 20:01:01.388 | INFO | BytesWritten : 3454599856594 | 20260505 20:01:01.388 | INFO | NumEvents : 1633838088516 | 20260505 20:01:03.131 | INFO | Analyzing chunk frames = [5914624,5915136) | 20260505 20:01:03.131 | INFO | MemInfo: { | 20260505 20:01:03.131 | INFO | "gpu" : " 17 GB", | 20260505 20:01:03.131 | INFO | "huge" : "", | 20260505 20:01:03.131 | INFO | "huge_shared" : " 27 GB", | 20260505 20:01:03.131 | INFO | "malloc" : " 14 GB", | 20260505 20:01:03.131 | INFO | "pinned" : " 29 GB" | 20260505 20:01:03.131 | INFO | } | 20260505 20:01:03.131 | INFO | Perf report for the last 5.14565 seconds: | 20260505 20:01:03.131 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:01:03.131 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:03.131 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:03.131 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:03.131 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260505 20:01:03.131 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:03.131 | INFO | RT_METRICS : 44.1%, 1.0, 23.2ms, 0 | 20260505 20:01:03.131 | INFO | PRE_HQ : 60.3%, 1.8, 55.9ms, 0 | 20260505 20:01:03.131 | INFO | BAZWRITER : 62.9%, 3.7, 120.6ms, 0 | 20260505 20:01:03.131 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 20:01:03.131 | INFO | Uploaded 36GiB to GPU in 2.17712s (16.5356GiB/s) | 20260505 20:01:03.131 | INFO | Downloaded 38GiB to GPU in 2.24871s (16.8986GiB/s) | 20260505 20:01:08.277 | INFO | Analyzing chunk frames = [5915136,5915648) | 20260505 20:01:08.277 | INFO | MemInfo: { | 20260505 20:01:08.277 | INFO | "gpu" : " 17 GB", | 20260505 20:01:08.277 | INFO | "huge" : "", | 20260505 20:01:08.277 | INFO | "huge_shared" : " 27 GB", | 20260505 20:01:08.277 | INFO | "malloc" : " 14 GB", | 20260505 20:01:08.277 | INFO | "pinned" : " 29 GB" | 20260505 20:01:08.277 | INFO | } | 20260505 20:01:08.277 | INFO | Perf report for the last 5.14648 seconds: | 20260505 20:01:08.277 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:01:08.277 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:08.277 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:08.277 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:08.277 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260505 20:01:08.277 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:08.277 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:08.277 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:01:08.277 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:08.277 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:08.277 | INFO | Uploaded 36GiB to GPU in 2.09416s (17.1907GiB/s) | 20260505 20:01:08.277 | INFO | Downloaded 34GiB to GPU in 2.02518s (16.7887GiB/s) | 20260505 20:01:13.414 | INFO | Analyzing chunk frames = [5915648,5916160) | 20260505 20:01:13.414 | INFO | MemInfo: { | 20260505 20:01:13.414 | INFO | "gpu" : " 17 GB", | 20260505 20:01:13.414 | INFO | "huge" : "", | 20260505 20:01:13.414 | INFO | "huge_shared" : " 27 GB", | 20260505 20:01:13.414 | INFO | "malloc" : " 14 GB", | 20260505 20:01:13.414 | INFO | "pinned" : " 29 GB" | 20260505 20:01:13.414 | INFO | } | 20260505 20:01:13.414 | INFO | Perf report for the last 5.13647 seconds: | 20260505 20:01:13.414 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:01:13.414 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:13.414 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:13.414 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:13.414 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260505 20:01:13.414 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:13.414 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:13.414 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:01:13.414 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:13.414 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:13.414 | INFO | Uploaded 36GiB to GPU in 2.13724s (16.8442GiB/s) | 20260505 20:01:13.414 | INFO | Downloaded 34GiB to GPU in 2.02324s (16.8047GiB/s) | 20260505 20:01:18.548 | INFO | Analyzing chunk frames = [5916160,5916672) | 20260505 20:01:18.548 | INFO | MemInfo: { | 20260505 20:01:18.548 | INFO | "gpu" : " 17 GB", | 20260505 20:01:18.548 | INFO | "huge" : "", | 20260505 20:01:18.548 | INFO | "huge_shared" : " 27 GB", | 20260505 20:01:18.548 | INFO | "malloc" : " 14 GB", | 20260505 20:01:18.548 | INFO | "pinned" : " 29 GB" | 20260505 20:01:18.548 | INFO | } | 20260505 20:01:18.548 | INFO | Perf report for the last 5.13413 seconds: | 20260505 20:01:18.548 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:01:18.548 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:18.548 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:18.548 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:18.548 | INFO | ANALYSIS : 62.5%, 3.0, 97.0ms, 0 | 20260505 20:01:18.548 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:18.548 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:18.548 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 20:01:18.548 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:18.548 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:18.548 | INFO | Uploaded 36GiB to GPU in 2.21081s (16.2836GiB/s) | 20260505 20:01:18.548 | INFO | Downloaded 34GiB to GPU in 2.04465s (16.6288GiB/s) | 20260505 20:01:23.686 | INFO | Analyzing chunk frames = [5916672,5917184) | 20260505 20:01:23.687 | INFO | MemInfo: { | 20260505 20:01:23.687 | INFO | "gpu" : " 17 GB", | 20260505 20:01:23.687 | INFO | "huge" : "", | 20260505 20:01:23.687 | INFO | "huge_shared" : " 27 GB", | 20260505 20:01:23.687 | INFO | "malloc" : " 14 GB", | 20260505 20:01:23.687 | INFO | "pinned" : " 29 GB" | 20260505 20:01:23.687 | INFO | } | 20260505 20:01:23.687 | INFO | Perf report for the last 5.13873 seconds: | 20260505 20:01:23.687 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:01:23.687 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:23.687 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:23.687 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:23.687 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260505 20:01:23.687 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:23.687 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:23.687 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 20:01:23.687 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:23.687 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:23.687 | INFO | Uploaded 36GiB to GPU in 2.13981s (16.8239GiB/s) | 20260505 20:01:23.687 | INFO | Downloaded 34GiB to GPU in 2.06189s (16.4897GiB/s) | 20260505 20:01:28.832 | INFO | Analyzing chunk frames = [5917184,5917696) | 20260505 20:01:28.832 | INFO | MemInfo: { | 20260505 20:01:28.832 | INFO | "gpu" : " 17 GB", | 20260505 20:01:28.832 | INFO | "huge" : "", | 20260505 20:01:28.832 | INFO | "huge_shared" : " 27 GB", | 20260505 20:01:28.832 | INFO | "malloc" : " 14 GB", | 20260505 20:01:28.832 | INFO | "pinned" : " 29 GB" | 20260505 20:01:28.832 | INFO | } | 20260505 20:01:28.832 | INFO | Perf report for the last 5.1457 seconds: | 20260505 20:01:28.832 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:01:28.832 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:28.832 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:28.832 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:28.832 | INFO | ANALYSIS : 62.0%, 3.0, 97.0ms, 0 | 20260505 20:01:28.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:28.832 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:28.832 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:01:28.832 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:28.832 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:28.833 | INFO | Uploaded 36GiB to GPU in 2.16421s (16.6342GiB/s) | 20260505 20:01:28.833 | INFO | Downloaded 34GiB to GPU in 2.05497s (16.5453GiB/s) | 20260505 20:01:33.967 | INFO | Analyzing chunk frames = [5917696,5918208) | 20260505 20:01:33.967 | INFO | MemInfo: { | 20260505 20:01:33.967 | INFO | "gpu" : " 17 GB", | 20260505 20:01:33.967 | INFO | "huge" : "", | 20260505 20:01:33.967 | INFO | "huge_shared" : " 27 GB", | 20260505 20:01:33.967 | INFO | "malloc" : " 14 GB", | 20260505 20:01:33.967 | INFO | "pinned" : " 29 GB" | 20260505 20:01:33.967 | INFO | } | 20260505 20:01:33.967 | INFO | Perf report for the last 5.13436 seconds: | 20260505 20:01:33.967 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:01:33.967 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:33.967 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:33.967 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:33.967 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 20:01:33.967 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:33.967 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:33.967 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 20:01:33.967 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:33.967 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:33.967 | INFO | Uploaded 36GiB to GPU in 2.14037s (16.8195GiB/s) | 20260505 20:01:33.967 | INFO | Downloaded 34GiB to GPU in 2.15316s (15.7907GiB/s) | 20260505 20:01:39.102 | INFO | Analyzing chunk frames = [5918208,5918720) | 20260505 20:01:39.102 | INFO | MemInfo: { | 20260505 20:01:39.102 | INFO | "gpu" : " 17 GB", | 20260505 20:01:39.102 | INFO | "huge" : "", | 20260505 20:01:39.102 | INFO | "huge_shared" : " 27 GB", | 20260505 20:01:39.102 | INFO | "malloc" : " 14 GB", | 20260505 20:01:39.102 | INFO | "pinned" : " 29 GB" | 20260505 20:01:39.102 | INFO | } | 20260505 20:01:39.102 | INFO | Perf report for the last 5.13566 seconds: | 20260505 20:01:39.102 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:01:39.102 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:39.102 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:39.102 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:39.102 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260505 20:01:39.102 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:39.102 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:39.102 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:01:39.103 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:39.103 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:39.103 | INFO | Uploaded 36GiB to GPU in 2.12542s (16.9378GiB/s) | 20260505 20:01:39.103 | INFO | Downloaded 34GiB to GPU in 2.01219s (16.897GiB/s) | 20260505 20:01:39.236 | INFO | Starting a round of Baz IO | 20260505 20:01:42.409 | INFO | Populating RTMetrics report | 20260505 20:01:42.410 | INFO | RTMetrics report populated | 20260505 20:01:42.410 | INFO | Opening tmp RTMetrics json file | 20260505 20:01:42.415 | INFO | Serializing RTMetrics to json | 20260505 20:01:42.417 | INFO | Serialization done, swapping tmp file now... | 20260505 20:01:42.417 | INFO | RTMetrics done | 20260505 20:01:42.502 | INFO | Finished a round of Baz IO: Wrote 2.027 GiB in 3.265 seconds | 20260505 20:01:42.502 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.71% / 0.00% / 78.29% | 20260505 20:01:42.502 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.187794% / 0% / 9.13938e-05% | 20260505 20:01:42.502 | INFO | Baz IO overall: | 20260505 20:01:42.502 | INFO | BytesWritten : 3456776061874 | 20260505 20:01:42.502 | INFO | NumEvents : 1634847643757 | 20260505 20:01:44.246 | INFO | Analyzing chunk frames = [5918720,5919232) | 20260505 20:01:44.246 | INFO | MemInfo: { | 20260505 20:01:44.246 | INFO | "gpu" : " 17 GB", | 20260505 20:01:44.246 | INFO | "huge" : "", | 20260505 20:01:44.246 | INFO | "huge_shared" : " 27 GB", | 20260505 20:01:44.246 | INFO | "malloc" : " 14 GB", | 20260505 20:01:44.246 | INFO | "pinned" : " 29 GB" | 20260505 20:01:44.246 | INFO | } | 20260505 20:01:44.246 | INFO | Perf report for the last 5.14359 seconds: | 20260505 20:01:44.246 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:01:44.246 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:44.246 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:44.246 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:44.246 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 20:01:44.246 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:44.246 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260505 20:01:44.246 | INFO | PRE_HQ : 60.3%, 1.8, 55.8ms, 0 | 20260505 20:01:44.246 | INFO | BAZWRITER : 62.9%, 3.7, 120.5ms, 0 | 20260505 20:01:44.246 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260505 20:01:44.246 | INFO | Uploaded 36GiB to GPU in 2.16003s (16.6664GiB/s) | 20260505 20:01:44.246 | INFO | Downloaded 38GiB to GPU in 2.24277s (16.9433GiB/s) | 20260505 20:01:49.391 | INFO | Analyzing chunk frames = [5919232,5919744) | 20260505 20:01:49.391 | INFO | MemInfo: { | 20260505 20:01:49.391 | INFO | "gpu" : " 17 GB", | 20260505 20:01:49.391 | INFO | "huge" : "", | 20260505 20:01:49.391 | INFO | "huge_shared" : " 27 GB", | 20260505 20:01:49.391 | INFO | "malloc" : " 14 GB", | 20260505 20:01:49.391 | INFO | "pinned" : " 29 GB" | 20260505 20:01:49.391 | INFO | } | 20260505 20:01:49.391 | INFO | Perf report for the last 5.14486 seconds: | 20260505 20:01:49.391 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:01:49.391 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:49.391 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:49.391 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:49.391 | INFO | ANALYSIS : 62.5%, 3.0, 97.7ms, 0 | 20260505 20:01:49.391 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:49.391 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:49.391 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 20:01:49.391 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:49.391 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:49.391 | INFO | Uploaded 36GiB to GPU in 2.13448s (16.866GiB/s) | 20260505 20:01:49.391 | INFO | Downloaded 34GiB to GPU in 2.03021s (16.7471GiB/s) | 20260505 20:01:54.526 | INFO | Analyzing chunk frames = [5919744,5920256) | 20260505 20:01:54.526 | INFO | MemInfo: { | 20260505 20:01:54.526 | INFO | "gpu" : " 17 GB", | 20260505 20:01:54.526 | INFO | "huge" : "", | 20260505 20:01:54.526 | INFO | "huge_shared" : " 27 GB", | 20260505 20:01:54.526 | INFO | "malloc" : " 14 GB", | 20260505 20:01:54.526 | INFO | "pinned" : " 29 GB" | 20260505 20:01:54.526 | INFO | } | 20260505 20:01:54.526 | INFO | Perf report for the last 5.13526 seconds: | 20260505 20:01:54.526 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:01:54.526 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:54.526 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:54.526 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:54.526 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260505 20:01:54.526 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:54.526 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:54.526 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 20:01:54.526 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:54.526 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:54.526 | INFO | Uploaded 36GiB to GPU in 2.12541s (16.9379GiB/s) | 20260505 20:01:54.526 | INFO | Downloaded 34GiB to GPU in 2.01233s (16.8959GiB/s) | 20260505 20:01:59.664 | INFO | Analyzing chunk frames = [5920256,5920768) | 20260505 20:01:59.664 | INFO | MemInfo: { | 20260505 20:01:59.664 | INFO | "gpu" : " 17 GB", | 20260505 20:01:59.664 | INFO | "huge" : "", | 20260505 20:01:59.664 | INFO | "huge_shared" : " 27 GB", | 20260505 20:01:59.664 | INFO | "malloc" : " 14 GB", | 20260505 20:01:59.664 | INFO | "pinned" : " 29 GB" | 20260505 20:01:59.664 | INFO | } | 20260505 20:01:59.664 | INFO | Perf report for the last 5.13818 seconds: | 20260505 20:01:59.664 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:01:59.664 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:59.664 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:59.664 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:59.664 | INFO | ANALYSIS : 62.1%, 3.0, 96.5ms, 0 | 20260505 20:01:59.664 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:59.664 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:59.664 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 20:01:59.664 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:59.665 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:01:59.665 | INFO | Uploaded 36GiB to GPU in 2.14421s (16.7894GiB/s) | 20260505 20:01:59.665 | INFO | Downloaded 34GiB to GPU in 2.04616s (16.6165GiB/s) | 20260505 20:02:04.801 | INFO | Analyzing chunk frames = [5920768,5921280) | 20260505 20:02:04.801 | INFO | MemInfo: { | 20260505 20:02:04.801 | INFO | "gpu" : " 17 GB", | 20260505 20:02:04.801 | INFO | "huge" : "", | 20260505 20:02:04.801 | INFO | "huge_shared" : " 27 GB", | 20260505 20:02:04.801 | INFO | "malloc" : " 14 GB", | 20260505 20:02:04.801 | INFO | "pinned" : " 29 GB" | 20260505 20:02:04.801 | INFO | } | 20260505 20:02:04.801 | INFO | Perf report for the last 5.13691 seconds: | 20260505 20:02:04.801 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:02:04.801 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:04.801 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:04.801 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:04.801 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260505 20:02:04.801 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:04.801 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:04.801 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:02:04.801 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:04.801 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:04.801 | INFO | Uploaded 36GiB to GPU in 2.13934s (16.8276GiB/s) | 20260505 20:02:04.801 | INFO | Downloaded 34GiB to GPU in 2.05965s (16.5076GiB/s) | 20260505 20:02:09.937 | INFO | Analyzing chunk frames = [5921280,5921792) | 20260505 20:02:09.937 | INFO | MemInfo: { | 20260505 20:02:09.937 | INFO | "gpu" : " 17 GB", | 20260505 20:02:09.937 | INFO | "huge" : "", | 20260505 20:02:09.937 | INFO | "huge_shared" : " 27 GB", | 20260505 20:02:09.937 | INFO | "malloc" : " 14 GB", | 20260505 20:02:09.937 | INFO | "pinned" : " 29 GB" | 20260505 20:02:09.937 | INFO | } | 20260505 20:02:09.937 | INFO | Perf report for the last 5.13593 seconds: | 20260505 20:02:09.937 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:02:09.937 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:09.937 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:09.937 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:09.937 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 20:02:09.937 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:09.937 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:09.937 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:02:09.937 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:09.937 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:09.937 | INFO | Uploaded 36GiB to GPU in 2.1398s (16.824GiB/s) | 20260505 20:02:09.937 | INFO | Downloaded 34GiB to GPU in 2.05302s (16.561GiB/s) | 20260505 20:02:15.084 | INFO | Analyzing chunk frames = [5921792,5922304) | 20260505 20:02:15.084 | INFO | MemInfo: { | 20260505 20:02:15.084 | INFO | "gpu" : " 17 GB", | 20260505 20:02:15.084 | INFO | "huge" : "", | 20260505 20:02:15.084 | INFO | "huge_shared" : " 27 GB", | 20260505 20:02:15.084 | INFO | "malloc" : " 14 GB", | 20260505 20:02:15.084 | INFO | "pinned" : " 29 GB" | 20260505 20:02:15.084 | INFO | } | 20260505 20:02:15.084 | INFO | Perf report for the last 5.14683 seconds: | 20260505 20:02:15.084 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:02:15.084 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:15.084 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:15.084 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:15.084 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 20:02:15.084 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:15.084 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:15.084 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 20:02:15.084 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:15.084 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:15.084 | INFO | Uploaded 36GiB to GPU in 2.11628s (17.011GiB/s) | 20260505 20:02:15.084 | INFO | Downloaded 34GiB to GPU in 2.14527s (15.8488GiB/s) | 20260505 20:02:20.234 | INFO | Analyzing chunk frames = [5922304,5922816) | 20260505 20:02:20.234 | INFO | MemInfo: { | 20260505 20:02:20.234 | INFO | "gpu" : " 17 GB", | 20260505 20:02:20.234 | INFO | "huge" : "", | 20260505 20:02:20.234 | INFO | "huge_shared" : " 27 GB", | 20260505 20:02:20.235 | INFO | "malloc" : " 14 GB", | 20260505 20:02:20.235 | INFO | "pinned" : " 29 GB" | 20260505 20:02:20.235 | INFO | } | 20260505 20:02:20.235 | INFO | Perf report for the last 5.15051 seconds: | 20260505 20:02:20.235 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:02:20.235 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:20.235 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:20.235 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:20.235 | INFO | ANALYSIS : 62.3%, 3.0, 97.5ms, 0 | 20260505 20:02:20.235 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:20.235 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:20.235 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 20:02:20.235 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:20.235 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:20.235 | INFO | Uploaded 36GiB to GPU in 2.12576s (16.9351GiB/s) | 20260505 20:02:20.235 | INFO | Downloaded 34GiB to GPU in 2.00994s (16.9159GiB/s) | 20260505 20:02:20.235 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 20:02:20.235 | INFO | Total mem usage: 15.3 GiB | 20260505 20:02:20.235 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 20:02:20.235 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.11%) | 20260505 20:02:20.235 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 20:02:20.235 | INFO | High Water Breakdown: | 20260505 20:02:20.235 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 20:02:20.235 | INFO | 38.8% : 5.51 GiB ( 82.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 20:02:20.235 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 20:02:20.235 | INFO | ------------------------------------------ | 20260505 20:02:20.235 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:02:20.235 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 20:02:20.235 | INFO | Total mem usage: 26.5 GiB | 20260505 20:02:20.235 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 20:02:20.235 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 20:02:20.235 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 20:02:20.235 | INFO | High Water Breakdown: | 20260505 20:02:20.235 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:02:20.235 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:02:20.235 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:02:20.235 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:02:20.235 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:02:20.235 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:02:20.235 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 20:02:20.235 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 20:02:20.235 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:02:20.235 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 20:02:20.235 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 20:02:20.235 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 20:02:20.235 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 20:02:20.235 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 20:02:20.235 | INFO | ------------------------------------------ | 20260505 20:02:20.235 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:02:20.235 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 20:02:20.235 | INFO | Total mem usage: 25.6 GiB | 20260505 20:02:20.236 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 20:02:20.236 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 20:02:20.236 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 20:02:20.236 | INFO | High Water Breakdown: | 20260505 20:02:20.236 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 20:02:20.236 | INFO | ------------------------------------------ | 20260505 20:02:20.236 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:02:20.236 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 20:02:20.236 | INFO | Total mem usage: 15.7 GiB | 20260505 20:02:20.236 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 20:02:20.236 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 20:02:20.236 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 20:02:20.236 | INFO | High Water Breakdown: | 20260505 20:02:20.236 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 20:02:20.236 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 20:02:20.236 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 20:02:20.236 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 20:02:20.236 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 20:02:20.236 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 20:02:20.236 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 20:02:20.236 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 20:02:20.236 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 20:02:20.236 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:02:20.236 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 20:02:20.236 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:02:20.236 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 20:02:20.236 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 20:02:20.236 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 20:02:20.236 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 20:02:20.236 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:02:20.236 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 20:02:20.236 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:02:20.236 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:02:20.236 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:02:20.236 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 20:02:20.236 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 20:02:20.236 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:02:20.236 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 20:02:20.236 | INFO | ------------------------------------------ | 20260505 20:02:20.236 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:02:20.236 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 20:02:20.236 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:02:20.236 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:02:20.236 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.47e+01, 1.36e+01 / 1.72e+00 | 20260505 20:02:20.236 | INFO | PerfReport: 0.29%, AggregateStats -- 9.27e-02 / 1.03e-01, 1.05e-02 / 4.54e-03 | 20260505 20:02:20.236 | INFO | PerfReport: 11.79%, Baseline -- 3.71e+00 / 4.54e+00, 5.63e-01 / 1.19e-01 | 20260505 20:02:20.236 | INFO | PerfReport: 3.29%, Histogram -- 1.04e+00 / 1.53e+00, 1.94e-01 / 9.07e-02 | 20260505 20:02:20.236 | INFO | PerfReport: 15.95%, DME -- 4.02e+01 / 4.44e+01, 4.03e+00 / 1.78e+00 | 20260505 20:02:20.236 | INFO | PerfReport: 57.03%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.48e+00 / 1.02e+00 | 20260505 20:02:20.236 | INFO | PerfReport: 7.24%, PulseAccumulating -- 2.28e+00 / 2.85e+00, 4.33e-01 / 2.08e-01 | 20260505 20:02:20.236 | INFO | PerfReport: 3.27%, Metrics -- 1.03e+00 / 1.36e+00, 2.85e-01 / 1.30e-01 | 20260505 20:02:20.236 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.58e-01 / 4.22e-01, 6.32e-02 / 4.02e-02 | 20260505 20:02:20.236 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 20:02:20.236 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:02:20.236 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:02:20.236 | INFO | PerfReport: Full Execution -- 9.94e+01 / -nan, 2.54e+01 / 0.00e+00 | 20260505 20:02:20.236 | INFO | PerfReport: 22.44%, Upload -- 2.23e+01 / 0.00e+00, 7.67e+00 / 0.00e+00 | 20260505 20:02:20.236 | INFO | PerfReport: 21.76%, Download -- 2.16e+01 / 0.00e+00, 7.69e+00 / 0.00e+00 | 20260505 20:02:20.236 | INFO | PerfReport: 31.69%, Compute -- 3.15e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 20:02:20.236 | INFO | PerfReport: 2.28%, LockUpload -- 2.27e+00 / 0.00e+00, 8.07e+00 / 0.00e+00 | 20260505 20:02:20.236 | INFO | PerfReport: 1.37%, LockDownload -- 1.37e+00 / 0.00e+00, 2.95e+00 / 0.00e+00 | 20260505 20:02:20.236 | INFO | PerfReport: 20.43%, LockCompute -- 2.03e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 20:02:20.236 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.37e-02 / 0.00e+00 | 20260505 20:02:20.367 | INFO | Starting a round of Baz IO | 20260505 20:02:23.525 | INFO | Populating RTMetrics report | 20260505 20:02:23.525 | INFO | RTMetrics report populated | 20260505 20:02:23.525 | INFO | Opening tmp RTMetrics json file | 20260505 20:02:23.526 | INFO | Serializing RTMetrics to json | 20260505 20:02:23.528 | INFO | Serialization done, swapping tmp file now... | 20260505 20:02:23.531 | INFO | RTMetrics done | 20260505 20:02:23.617 | INFO | Finished a round of Baz IO: Wrote 2.026 GiB in 3.249 seconds | 20260505 20:02:23.617 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.71% / 0.00% / 78.29% | 20260505 20:02:23.617 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.187365% / 0% / 6.35783e-05% | 20260505 20:02:23.617 | INFO | Baz IO overall: | 20260505 20:02:23.617 | INFO | BytesWritten : 3458951409930 | 20260505 20:02:23.617 | INFO | NumEvents : 1635856472012 | 20260505 20:02:25.361 | INFO | Analyzing chunk frames = [5922816,5923328) | 20260505 20:02:25.361 | INFO | MemInfo: { | 20260505 20:02:25.361 | INFO | "gpu" : " 17 GB", | 20260505 20:02:25.361 | INFO | "huge" : "", | 20260505 20:02:25.361 | INFO | "huge_shared" : " 27 GB", | 20260505 20:02:25.361 | INFO | "malloc" : " 14 GB", | 20260505 20:02:25.361 | INFO | "pinned" : " 29 GB" | 20260505 20:02:25.361 | INFO | } | 20260505 20:02:25.361 | INFO | Perf report for the last 5.1267 seconds: | 20260505 20:02:25.361 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:02:25.361 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:25.361 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:25.361 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:25.361 | INFO | ANALYSIS : 63.1%, 3.0, 98.2ms, 0 | 20260505 20:02:25.361 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:25.361 | INFO | RT_METRICS : 44.2%, 1.0, 23.1ms, 0 | 20260505 20:02:25.361 | INFO | PRE_HQ : 60.2%, 1.8, 55.7ms, 0 | 20260505 20:02:25.361 | INFO | BAZWRITER : 62.9%, 3.7, 119.5ms, 0 | 20260505 20:02:25.361 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 20:02:25.361 | INFO | Uploaded 36GiB to GPU in 2.17437s (16.5565GiB/s) | 20260505 20:02:25.362 | INFO | Downloaded 38GiB to GPU in 2.2494s (16.8934GiB/s) | 20260505 20:02:30.493 | INFO | Analyzing chunk frames = [5923328,5923840) | 20260505 20:02:30.494 | INFO | MemInfo: { | 20260505 20:02:30.494 | INFO | "gpu" : " 17 GB", | 20260505 20:02:30.494 | INFO | "huge" : "", | 20260505 20:02:30.494 | INFO | "huge_shared" : " 27 GB", | 20260505 20:02:30.494 | INFO | "malloc" : " 14 GB", | 20260505 20:02:30.494 | INFO | "pinned" : " 29 GB" | 20260505 20:02:30.494 | INFO | } | 20260505 20:02:30.494 | INFO | Perf report for the last 5.13232 seconds: | 20260505 20:02:30.494 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:02:30.494 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:30.494 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:30.494 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:30.494 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260505 20:02:30.494 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:30.494 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:30.494 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:02:30.494 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:30.494 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:30.494 | INFO | Uploaded 36GiB to GPU in 2.14956s (16.7476GiB/s) | 20260505 20:02:30.494 | INFO | Downloaded 34GiB to GPU in 2.04819s (16.6001GiB/s) | 20260505 20:02:35.643 | INFO | Analyzing chunk frames = [5923840,5924352) | 20260505 20:02:35.643 | INFO | MemInfo: { | 20260505 20:02:35.643 | INFO | "gpu" : " 17 GB", | 20260505 20:02:35.643 | INFO | "huge" : "", | 20260505 20:02:35.643 | INFO | "huge_shared" : " 27 GB", | 20260505 20:02:35.643 | INFO | "malloc" : " 14 GB", | 20260505 20:02:35.643 | INFO | "pinned" : " 29 GB" | 20260505 20:02:35.643 | INFO | } | 20260505 20:02:35.643 | INFO | Perf report for the last 5.14981 seconds: | 20260505 20:02:35.643 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:02:35.643 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:35.644 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:35.644 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:35.644 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 20:02:35.644 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:35.644 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:35.644 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:02:35.644 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:35.644 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:35.644 | INFO | Uploaded 36GiB to GPU in 2.2137s (16.2624GiB/s) | 20260505 20:02:35.644 | INFO | Downloaded 34GiB to GPU in 2.0685s (16.4371GiB/s) | 20260505 20:02:40.783 | INFO | Analyzing chunk frames = [5924352,5924864) | 20260505 20:02:40.783 | INFO | MemInfo: { | 20260505 20:02:40.783 | INFO | "gpu" : " 17 GB", | 20260505 20:02:40.783 | INFO | "huge" : "", | 20260505 20:02:40.783 | INFO | "huge_shared" : " 27 GB", | 20260505 20:02:40.783 | INFO | "malloc" : " 14 GB", | 20260505 20:02:40.783 | INFO | "pinned" : " 29 GB" | 20260505 20:02:40.783 | INFO | } | 20260505 20:02:40.783 | INFO | Perf report for the last 5.13967 seconds: | 20260505 20:02:40.783 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:02:40.783 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:40.783 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:40.783 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:40.783 | INFO | ANALYSIS : 62.5%, 3.0, 97.1ms, 0 | 20260505 20:02:40.783 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:40.783 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:40.783 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 20:02:40.783 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:40.783 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:40.783 | INFO | Uploaded 36GiB to GPU in 2.21251s (16.2711GiB/s) | 20260505 20:02:40.783 | INFO | Downloaded 34GiB to GPU in 2.06968s (16.4277GiB/s) | 20260505 20:02:45.917 | INFO | Analyzing chunk frames = [5924864,5925376) | 20260505 20:02:45.917 | INFO | MemInfo: { | 20260505 20:02:45.917 | INFO | "gpu" : " 17 GB", | 20260505 20:02:45.917 | INFO | "huge" : "", | 20260505 20:02:45.917 | INFO | "huge_shared" : " 27 GB", | 20260505 20:02:45.917 | INFO | "malloc" : " 14 GB", | 20260505 20:02:45.917 | INFO | "pinned" : " 29 GB" | 20260505 20:02:45.917 | INFO | } | 20260505 20:02:45.917 | INFO | Perf report for the last 5.13381 seconds: | 20260505 20:02:45.917 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:02:45.917 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:45.917 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:45.917 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:45.917 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260505 20:02:45.917 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:45.917 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:45.917 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 20:02:45.917 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:45.917 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:45.917 | INFO | Uploaded 36GiB to GPU in 2.13978s (16.8242GiB/s) | 20260505 20:02:45.917 | INFO | Downloaded 34GiB to GPU in 2.06244s (16.4853GiB/s) | 20260505 20:02:51.053 | INFO | Analyzing chunk frames = [5925376,5925888) | 20260505 20:02:51.053 | INFO | MemInfo: { | 20260505 20:02:51.053 | INFO | "gpu" : " 17 GB", | 20260505 20:02:51.053 | INFO | "huge" : "", | 20260505 20:02:51.053 | INFO | "huge_shared" : " 27 GB", | 20260505 20:02:51.053 | INFO | "malloc" : " 14 GB", | 20260505 20:02:51.053 | INFO | "pinned" : " 29 GB" | 20260505 20:02:51.053 | INFO | } | 20260505 20:02:51.053 | INFO | Perf report for the last 5.13602 seconds: | 20260505 20:02:51.053 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:02:51.053 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:51.053 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:51.053 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:51.053 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 20:02:51.053 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:51.053 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:51.053 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 20:02:51.053 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:51.053 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:51.053 | INFO | Uploaded 36GiB to GPU in 2.14856s (16.7554GiB/s) | 20260505 20:02:51.053 | INFO | Downloaded 34GiB to GPU in 2.05614s (16.5358GiB/s) | 20260505 20:02:56.192 | INFO | Analyzing chunk frames = [5925888,5926400) | 20260505 20:02:56.192 | INFO | MemInfo: { | 20260505 20:02:56.192 | INFO | "gpu" : " 17 GB", | 20260505 20:02:56.192 | INFO | "huge" : "", | 20260505 20:02:56.192 | INFO | "huge_shared" : " 27 GB", | 20260505 20:02:56.192 | INFO | "malloc" : " 14 GB", | 20260505 20:02:56.192 | INFO | "pinned" : " 29 GB" | 20260505 20:02:56.192 | INFO | } | 20260505 20:02:56.192 | INFO | Perf report for the last 5.13948 seconds: | 20260505 20:02:56.192 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:02:56.192 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:56.192 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:56.192 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:56.193 | INFO | ANALYSIS : 63.4%, 3.0, 99.0ms, 0 | 20260505 20:02:56.193 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:56.193 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:56.193 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 20:02:56.193 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:56.193 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:02:56.193 | INFO | Uploaded 36GiB to GPU in 2.1329s (16.8784GiB/s) | 20260505 20:02:56.193 | INFO | Downloaded 34GiB to GPU in 2.16072s (15.7355GiB/s) | 20260505 20:03:01.340 | INFO | Analyzing chunk frames = [5926400,5926912) | 20260505 20:03:01.340 | INFO | MemInfo: { | 20260505 20:03:01.340 | INFO | "gpu" : " 17 GB", | 20260505 20:03:01.340 | INFO | "huge" : "", | 20260505 20:03:01.340 | INFO | "huge_shared" : " 27 GB", | 20260505 20:03:01.340 | INFO | "malloc" : " 14 GB", | 20260505 20:03:01.340 | INFO | "pinned" : " 29 GB" | 20260505 20:03:01.340 | INFO | } | 20260505 20:03:01.340 | INFO | Perf report for the last 5.148 seconds: | 20260505 20:03:01.340 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:03:01.340 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:01.340 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:01.340 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:01.340 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 20:03:01.340 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:01.341 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:01.341 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260505 20:03:01.341 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:01.341 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:01.341 | INFO | Uploaded 36GiB to GPU in 2.2036s (16.3369GiB/s) | 20260505 20:03:01.341 | INFO | Downloaded 34GiB to GPU in 2.04161s (16.6535GiB/s) | 20260505 20:03:01.475 | INFO | Starting a round of Baz IO | 20260505 20:03:04.639 | INFO | Populating RTMetrics report | 20260505 20:03:04.639 | INFO | RTMetrics report populated | 20260505 20:03:04.639 | INFO | Opening tmp RTMetrics json file | 20260505 20:03:04.639 | INFO | Serializing RTMetrics to json | 20260505 20:03:04.641 | INFO | Serialization done, swapping tmp file now... | 20260505 20:03:04.641 | INFO | RTMetrics done | 20260505 20:03:04.727 | INFO | Finished a round of Baz IO: Wrote 2.025 GiB in 3.251 seconds | 20260505 20:03:04.727 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.71% / 0.00% / 78.29% | 20260505 20:03:04.727 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.186805% / 0% / 8.74201e-05% | 20260505 20:03:04.727 | INFO | Baz IO overall: | 20260505 20:03:04.727 | INFO | BytesWritten : 3461126219570 | 20260505 20:03:04.727 | INFO | NumEvents : 1636865132962 | 20260505 20:03:06.477 | INFO | Analyzing chunk frames = [5926912,5927424) | 20260505 20:03:06.477 | INFO | MemInfo: { | 20260505 20:03:06.477 | INFO | "gpu" : " 17 GB", | 20260505 20:03:06.477 | INFO | "huge" : "", | 20260505 20:03:06.477 | INFO | "huge_shared" : " 27 GB", | 20260505 20:03:06.477 | INFO | "malloc" : " 14 GB", | 20260505 20:03:06.477 | INFO | "pinned" : " 29 GB" | 20260505 20:03:06.477 | INFO | } | 20260505 20:03:06.477 | INFO | Perf report for the last 5.13663 seconds: | 20260505 20:03:06.477 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:03:06.477 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:06.477 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:06.477 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:06.477 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 20:03:06.477 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:06.477 | INFO | RT_METRICS : 44.2%, 1.0, 23.1ms, 0 | 20260505 20:03:06.477 | INFO | PRE_HQ : 60.1%, 1.8, 55.2ms, 0 | 20260505 20:03:06.477 | INFO | BAZWRITER : 62.8%, 3.7, 118.9ms, 0 | 20260505 20:03:06.477 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 20:03:06.477 | INFO | Uploaded 36GiB to GPU in 2.16891s (16.5982GiB/s) | 20260505 20:03:06.477 | INFO | Downloaded 38GiB to GPU in 2.25244s (16.8706GiB/s) | 20260505 20:03:11.609 | INFO | Analyzing chunk frames = [5927424,5927936) | 20260505 20:03:11.609 | INFO | MemInfo: { | 20260505 20:03:11.609 | INFO | "gpu" : " 17 GB", | 20260505 20:03:11.609 | INFO | "huge" : "", | 20260505 20:03:11.609 | INFO | "huge_shared" : " 27 GB", | 20260505 20:03:11.609 | INFO | "malloc" : " 14 GB", | 20260505 20:03:11.609 | INFO | "pinned" : " 29 GB" | 20260505 20:03:11.609 | INFO | } | 20260505 20:03:11.609 | INFO | Perf report for the last 5.1317 seconds: | 20260505 20:03:11.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:03:11.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:11.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:11.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:11.609 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260505 20:03:11.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:11.609 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:11.609 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 20:03:11.609 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:11.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:11.609 | INFO | Uploaded 36GiB to GPU in 2.15204s (16.7283GiB/s) | 20260505 20:03:11.609 | INFO | Downloaded 34GiB to GPU in 2.05087s (16.5783GiB/s) | 20260505 20:03:16.747 | INFO | Analyzing chunk frames = [5927936,5928448) | 20260505 20:03:16.747 | INFO | MemInfo: { | 20260505 20:03:16.747 | INFO | "gpu" : " 17 GB", | 20260505 20:03:16.747 | INFO | "huge" : "", | 20260505 20:03:16.747 | INFO | "huge_shared" : " 27 GB", | 20260505 20:03:16.747 | INFO | "malloc" : " 14 GB", | 20260505 20:03:16.747 | INFO | "pinned" : " 29 GB" | 20260505 20:03:16.747 | INFO | } | 20260505 20:03:16.747 | INFO | Perf report for the last 5.13778 seconds: | 20260505 20:03:16.747 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:03:16.747 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:16.747 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:16.747 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:16.747 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 20:03:16.747 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:16.747 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:16.747 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:03:16.747 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:16.747 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:16.747 | INFO | Uploaded 36GiB to GPU in 2.21481s (16.2542GiB/s) | 20260505 20:03:16.747 | INFO | Downloaded 34GiB to GPU in 2.06233s (16.4862GiB/s) | 20260505 20:03:21.892 | INFO | Analyzing chunk frames = [5928448,5928960) | 20260505 20:03:21.892 | INFO | MemInfo: { | 20260505 20:03:21.892 | INFO | "gpu" : " 17 GB", | 20260505 20:03:21.892 | INFO | "huge" : "", | 20260505 20:03:21.892 | INFO | "huge_shared" : " 27 GB", | 20260505 20:03:21.892 | INFO | "malloc" : " 14 GB", | 20260505 20:03:21.892 | INFO | "pinned" : " 29 GB" | 20260505 20:03:21.892 | INFO | } | 20260505 20:03:21.892 | INFO | Perf report for the last 5.14515 seconds: | 20260505 20:03:21.892 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:03:21.892 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:21.892 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:21.892 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:21.892 | INFO | ANALYSIS : 62.4%, 3.0, 97.0ms, 0 | 20260505 20:03:21.892 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:21.892 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:21.892 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:03:21.892 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:21.892 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:21.892 | INFO | Uploaded 36GiB to GPU in 2.19492s (16.4015GiB/s) | 20260505 20:03:21.892 | INFO | Downloaded 34GiB to GPU in 2.13829s (15.9006GiB/s) | 20260505 20:03:27.037 | INFO | Analyzing chunk frames = [5928960,5929472) | 20260505 20:03:27.037 | INFO | MemInfo: { | 20260505 20:03:27.037 | INFO | "gpu" : " 17 GB", | 20260505 20:03:27.038 | INFO | "huge" : "", | 20260505 20:03:27.038 | INFO | "huge_shared" : " 27 GB", | 20260505 20:03:27.038 | INFO | "malloc" : " 14 GB", | 20260505 20:03:27.038 | INFO | "pinned" : " 29 GB" | 20260505 20:03:27.038 | INFO | } | 20260505 20:03:27.038 | INFO | Perf report for the last 5.14585 seconds: | 20260505 20:03:27.038 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:03:27.038 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:27.038 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:27.038 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:27.038 | INFO | ANALYSIS : 61.6%, 3.0, 96.4ms, 0 | 20260505 20:03:27.038 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:27.038 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:27.038 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:03:27.038 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:27.038 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:27.038 | INFO | Uploaded 36GiB to GPU in 2.13567s (16.8565GiB/s) | 20260505 20:03:27.038 | INFO | Downloaded 34GiB to GPU in 2.06022s (16.5031GiB/s) | 20260505 20:03:32.176 | INFO | Analyzing chunk frames = [5929472,5929984) | 20260505 20:03:32.176 | INFO | MemInfo: { | 20260505 20:03:32.176 | INFO | "gpu" : " 17 GB", | 20260505 20:03:32.176 | INFO | "huge" : "", | 20260505 20:03:32.176 | INFO | "huge_shared" : " 27 GB", | 20260505 20:03:32.176 | INFO | "malloc" : " 14 GB", | 20260505 20:03:32.176 | INFO | "pinned" : " 29 GB" | 20260505 20:03:32.176 | INFO | } | 20260505 20:03:32.176 | INFO | Perf report for the last 5.13811 seconds: | 20260505 20:03:32.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:03:32.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:32.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:32.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:32.176 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 20:03:32.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:32.176 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:32.176 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:03:32.176 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:32.176 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:32.176 | INFO | Uploaded 36GiB to GPU in 2.08018s (17.3062GiB/s) | 20260505 20:03:32.176 | INFO | Downloaded 34GiB to GPU in 2.02668s (16.7762GiB/s) | 20260505 20:03:37.309 | INFO | Analyzing chunk frames = [5929984,5930496) | 20260505 20:03:37.309 | INFO | MemInfo: { | 20260505 20:03:37.309 | INFO | "gpu" : " 17 GB", | 20260505 20:03:37.309 | INFO | "huge" : "", | 20260505 20:03:37.309 | INFO | "huge_shared" : " 27 GB", | 20260505 20:03:37.309 | INFO | "malloc" : " 14 GB", | 20260505 20:03:37.309 | INFO | "pinned" : " 29 GB" | 20260505 20:03:37.309 | INFO | } | 20260505 20:03:37.309 | INFO | Perf report for the last 5.13363 seconds: | 20260505 20:03:37.309 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:03:37.309 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:37.309 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:37.309 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:37.309 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 20:03:37.309 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:37.309 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:37.309 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 20:03:37.310 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:37.310 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:37.310 | INFO | Uploaded 36GiB to GPU in 2.1332s (16.8761GiB/s) | 20260505 20:03:37.310 | INFO | Downloaded 34GiB to GPU in 2.13809s (15.902GiB/s) | 20260505 20:03:42.447 | INFO | Analyzing chunk frames = [5930496,5931008) | 20260505 20:03:42.447 | INFO | MemInfo: { | 20260505 20:03:42.447 | INFO | "gpu" : " 17 GB", | 20260505 20:03:42.447 | INFO | "huge" : "", | 20260505 20:03:42.447 | INFO | "huge_shared" : " 27 GB", | 20260505 20:03:42.447 | INFO | "malloc" : " 14 GB", | 20260505 20:03:42.447 | INFO | "pinned" : " 29 GB" | 20260505 20:03:42.447 | INFO | } | 20260505 20:03:42.447 | INFO | Perf report for the last 5.1378 seconds: | 20260505 20:03:42.447 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:03:42.447 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:42.447 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:42.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:42.447 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 20:03:42.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:42.447 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:42.447 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:03:42.447 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:42.447 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:42.447 | INFO | Uploaded 36GiB to GPU in 2.19921s (16.3695GiB/s) | 20260505 20:03:42.447 | INFO | Downloaded 34GiB to GPU in 2.03865s (16.6777GiB/s) | 20260505 20:03:42.448 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 20:03:42.448 | INFO | Total mem usage: 15.3 GiB | 20260505 20:03:42.448 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 20:03:42.448 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.85%) | 20260505 20:03:42.448 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 20:03:42.448 | INFO | High Water Breakdown: | 20260505 20:03:42.448 | INFO | 60.9% : 8.58 GiB ( 90.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 20:03:42.448 | INFO | 38.9% : 5.48 GiB ( 82.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 20:03:42.448 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 20:03:42.448 | INFO | ------------------------------------------ | 20260505 20:03:42.448 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:03:42.448 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 20:03:42.448 | INFO | Total mem usage: 26.5 GiB | 20260505 20:03:42.448 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 20:03:42.448 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 20:03:42.448 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 20:03:42.448 | INFO | High Water Breakdown: | 20260505 20:03:42.448 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:03:42.448 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:03:42.448 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:03:42.448 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:03:42.448 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:03:42.448 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:03:42.448 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 20:03:42.448 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 20:03:42.448 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:03:42.448 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 20:03:42.448 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 20:03:42.448 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 20:03:42.448 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 20:03:42.448 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 20:03:42.448 | INFO | ------------------------------------------ | 20260505 20:03:42.448 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:03:42.448 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 20:03:42.448 | INFO | Total mem usage: 25.6 GiB | 20260505 20:03:42.448 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 20:03:42.448 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 20:03:42.448 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 20:03:42.448 | INFO | High Water Breakdown: | 20260505 20:03:42.448 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 20:03:42.448 | INFO | ------------------------------------------ | 20260505 20:03:42.448 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:03:42.448 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 20:03:42.448 | INFO | Total mem usage: 15.7 GiB | 20260505 20:03:42.448 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 20:03:42.448 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 20:03:42.448 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 20:03:42.448 | INFO | High Water Breakdown: | 20260505 20:03:42.448 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 20:03:42.448 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 20:03:42.448 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 20:03:42.448 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 20:03:42.448 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 20:03:42.448 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 20:03:42.448 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 20:03:42.448 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 20:03:42.448 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 20:03:42.448 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:03:42.448 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 20:03:42.448 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:03:42.448 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 20:03:42.448 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 20:03:42.448 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 20:03:42.448 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 20:03:42.448 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:03:42.448 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 20:03:42.448 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:03:42.448 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:03:42.448 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:03:42.448 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 20:03:42.448 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 20:03:42.448 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:03:42.448 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 20:03:42.448 | INFO | ------------------------------------------ | 20260505 20:03:42.448 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:03:42.448 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 20:03:42.448 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:03:42.448 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:03:42.448 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.48e+01, 1.35e+01 / 2.07e+00 | 20260505 20:03:42.448 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.03e-01, 1.06e-02 / 2.63e-03 | 20260505 20:03:42.448 | INFO | PerfReport: 11.77%, Baseline -- 3.69e+00 / 4.55e+00, 5.53e-01 / 1.20e-01 | 20260505 20:03:42.448 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.52e+00, 1.85e-01 / 8.12e-02 | 20260505 20:03:42.448 | INFO | PerfReport: 16.03%, DME -- 4.02e+01 / 4.47e+01, 4.23e+00 / 2.23e+00 | 20260505 20:03:42.448 | INFO | PerfReport: 56.95%, FrameLabeling -- 1.79e+01 / 1.93e+01, 1.35e+00 / 1.04e+00 | 20260505 20:03:42.448 | INFO | PerfReport: 7.29%, PulseAccumulating -- 2.29e+00 / 2.84e+00, 4.38e-01 / 2.24e-01 | 20260505 20:03:42.448 | INFO | PerfReport: 3.27%, Metrics -- 1.03e+00 / 1.35e+00, 2.81e-01 / 1.35e-01 | 20260505 20:03:42.448 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.57e-01 / 4.31e-01, 6.33e-02 / 4.07e-02 | 20260505 20:03:42.448 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 20:03:42.448 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:03:42.448 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:03:42.448 | INFO | PerfReport: Full Execution -- 9.96e+01 / -nan, 2.56e+01 / 0.00e+00 | 20260505 20:03:42.448 | INFO | PerfReport: 22.65%, Upload -- 2.26e+01 / 0.00e+00, 7.93e+00 / 0.00e+00 | 20260505 20:03:42.448 | INFO | PerfReport: 21.91%, Download -- 2.18e+01 / 0.00e+00, 7.84e+00 / 0.00e+00 | 20260505 20:03:42.448 | INFO | PerfReport: 31.51%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 20:03:42.448 | INFO | PerfReport: 2.35%, LockUpload -- 2.34e+00 / 0.00e+00, 7.98e+00 / 0.00e+00 | 20260505 20:03:42.448 | INFO | PerfReport: 1.37%, LockDownload -- 1.36e+00 / 0.00e+00, 2.79e+00 / 0.00e+00 | 20260505 20:03:42.448 | INFO | PerfReport: 20.20%, LockCompute -- 2.01e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 20:03:42.448 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.51e-02 / 0.00e+00 | 20260505 20:03:42.580 | INFO | Starting a round of Baz IO | 20260505 20:03:45.754 | INFO | Populating RTMetrics report | 20260505 20:03:45.755 | INFO | RTMetrics report populated | 20260505 20:03:45.755 | INFO | Opening tmp RTMetrics json file | 20260505 20:03:45.756 | INFO | Serializing RTMetrics to json | 20260505 20:03:45.757 | INFO | Serialization done, swapping tmp file now... | 20260505 20:03:45.757 | INFO | RTMetrics done | 20260505 20:03:45.847 | INFO | Finished a round of Baz IO: Wrote 2.025 GiB in 3.267 seconds | 20260505 20:03:45.847 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.71% / 0.00% / 78.29% | 20260505 20:03:45.847 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.187139% / 7.94729e-06% / 7.54992e-05% | 20260505 20:03:45.847 | INFO | Baz IO overall: | 20260505 20:03:45.847 | INFO | BytesWritten : 3463300580982 | 20260505 20:03:45.847 | INFO | NumEvents : 1637873315705 | 20260505 20:03:47.594 | INFO | Analyzing chunk frames = [5931008,5931520) | 20260505 20:03:47.594 | INFO | MemInfo: { | 20260505 20:03:47.594 | INFO | "gpu" : " 17 GB", | 20260505 20:03:47.594 | INFO | "huge" : "", | 20260505 20:03:47.594 | INFO | "huge_shared" : " 27 GB", | 20260505 20:03:47.594 | INFO | "malloc" : " 14 GB", | 20260505 20:03:47.594 | INFO | "pinned" : " 29 GB" | 20260505 20:03:47.594 | INFO | } | 20260505 20:03:47.594 | INFO | Perf report for the last 5.14665 seconds: | 20260505 20:03:47.594 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:03:47.594 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:47.594 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:47.594 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:47.594 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 20:03:47.594 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:47.594 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260505 20:03:47.594 | INFO | PRE_HQ : 60.0%, 1.8, 55.6ms, 0 | 20260505 20:03:47.594 | INFO | BAZWRITER : 63.0%, 3.7, 119.6ms, 0 | 20260505 20:03:47.594 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 20:03:47.594 | INFO | Uploaded 36GiB to GPU in 2.17243s (16.5713GiB/s) | 20260505 20:03:47.594 | INFO | Downloaded 38GiB to GPU in 2.25979s (16.8157GiB/s) | 20260505 20:03:52.731 | INFO | Analyzing chunk frames = [5931520,5932032) | 20260505 20:03:52.731 | INFO | MemInfo: { | 20260505 20:03:52.731 | INFO | "gpu" : " 17 GB", | 20260505 20:03:52.731 | INFO | "huge" : "", | 20260505 20:03:52.731 | INFO | "huge_shared" : " 27 GB", | 20260505 20:03:52.731 | INFO | "malloc" : " 14 GB", | 20260505 20:03:52.731 | INFO | "pinned" : " 29 GB" | 20260505 20:03:52.731 | INFO | } | 20260505 20:03:52.731 | INFO | Perf report for the last 5.13754 seconds: | 20260505 20:03:52.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:03:52.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:52.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:52.732 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:52.732 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 20:03:52.732 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:52.732 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:52.732 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:03:52.732 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:52.732 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:52.732 | INFO | Uploaded 36GiB to GPU in 2.08186s (17.2922GiB/s) | 20260505 20:03:52.732 | INFO | Downloaded 34GiB to GPU in 2.02201s (16.815GiB/s) | 20260505 20:03:57.872 | INFO | Analyzing chunk frames = [5932032,5932544) | 20260505 20:03:57.872 | INFO | MemInfo: { | 20260505 20:03:57.872 | INFO | "gpu" : " 17 GB", | 20260505 20:03:57.872 | INFO | "huge" : "", | 20260505 20:03:57.872 | INFO | "huge_shared" : " 27 GB", | 20260505 20:03:57.872 | INFO | "malloc" : " 14 GB", | 20260505 20:03:57.872 | INFO | "pinned" : " 29 GB" | 20260505 20:03:57.872 | INFO | } | 20260505 20:03:57.872 | INFO | Perf report for the last 5.14094 seconds: | 20260505 20:03:57.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:03:57.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:57.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:57.872 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:57.872 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260505 20:03:57.872 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:57.872 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:57.872 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:03:57.872 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:57.872 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:03:57.873 | INFO | Uploaded 36GiB to GPU in 2.11863s (16.9921GiB/s) | 20260505 20:03:57.873 | INFO | Downloaded 34GiB to GPU in 2.01602s (16.8649GiB/s) | 20260505 20:04:03.007 | INFO | Analyzing chunk frames = [5932544,5933056) | 20260505 20:04:03.007 | INFO | MemInfo: { | 20260505 20:04:03.007 | INFO | "gpu" : " 17 GB", | 20260505 20:04:03.007 | INFO | "huge" : "", | 20260505 20:04:03.007 | INFO | "huge_shared" : " 27 GB", | 20260505 20:04:03.007 | INFO | "malloc" : " 14 GB", | 20260505 20:04:03.007 | INFO | "pinned" : " 29 GB" | 20260505 20:04:03.007 | INFO | } | 20260505 20:04:03.007 | INFO | Perf report for the last 5.13496 seconds: | 20260505 20:04:03.007 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:04:03.007 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:03.007 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:03.007 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:03.007 | INFO | ANALYSIS : 62.6%, 3.0, 97.1ms, 0 | 20260505 20:04:03.007 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:03.007 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:03.007 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:04:03.007 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:03.007 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:03.008 | INFO | Uploaded 36GiB to GPU in 2.21023s (16.2879GiB/s) | 20260505 20:04:03.008 | INFO | Downloaded 34GiB to GPU in 2.04377s (16.6359GiB/s) | 20260505 20:04:08.147 | INFO | Analyzing chunk frames = [5933056,5933568) | 20260505 20:04:08.147 | INFO | MemInfo: { | 20260505 20:04:08.147 | INFO | "gpu" : " 17 GB", | 20260505 20:04:08.147 | INFO | "huge" : "", | 20260505 20:04:08.148 | INFO | "huge_shared" : " 27 GB", | 20260505 20:04:08.148 | INFO | "malloc" : " 14 GB", | 20260505 20:04:08.148 | INFO | "pinned" : " 29 GB" | 20260505 20:04:08.148 | INFO | } | 20260505 20:04:08.148 | INFO | Perf report for the last 5.14021 seconds: | 20260505 20:04:08.148 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:04:08.148 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:08.148 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:08.148 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:08.148 | INFO | ANALYSIS : 61.8%, 3.0, 96.6ms, 0 | 20260505 20:04:08.148 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:08.148 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:08.148 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 20:04:08.148 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:08.148 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:08.148 | INFO | Uploaded 36GiB to GPU in 2.13613s (16.8529GiB/s) | 20260505 20:04:08.148 | INFO | Downloaded 34GiB to GPU in 2.06371s (16.4752GiB/s) | 20260505 20:04:13.289 | INFO | Analyzing chunk frames = [5933568,5934080) | 20260505 20:04:13.289 | INFO | MemInfo: { | 20260505 20:04:13.289 | INFO | "gpu" : " 17 GB", | 20260505 20:04:13.289 | INFO | "huge" : "", | 20260505 20:04:13.289 | INFO | "huge_shared" : " 27 GB", | 20260505 20:04:13.289 | INFO | "malloc" : " 14 GB", | 20260505 20:04:13.289 | INFO | "pinned" : " 29 GB" | 20260505 20:04:13.289 | INFO | } | 20260505 20:04:13.289 | INFO | Perf report for the last 5.14108 seconds: | 20260505 20:04:13.289 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:04:13.289 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:13.289 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:13.289 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:13.289 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 20:04:13.289 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:13.289 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:13.289 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 20:04:13.289 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:13.289 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:13.289 | INFO | Uploaded 36GiB to GPU in 2.13878s (16.832GiB/s) | 20260505 20:04:13.289 | INFO | Downloaded 34GiB to GPU in 2.05677s (16.5308GiB/s) | 20260505 20:04:18.430 | INFO | Analyzing chunk frames = [5934080,5934592) | 20260505 20:04:18.430 | INFO | MemInfo: { | 20260505 20:04:18.430 | INFO | "gpu" : " 17 GB", | 20260505 20:04:18.430 | INFO | "huge" : "", | 20260505 20:04:18.430 | INFO | "huge_shared" : " 27 GB", | 20260505 20:04:18.430 | INFO | "malloc" : " 14 GB", | 20260505 20:04:18.430 | INFO | "pinned" : " 29 GB" | 20260505 20:04:18.430 | INFO | } | 20260505 20:04:18.430 | INFO | Perf report for the last 5.14169 seconds: | 20260505 20:04:18.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:04:18.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:18.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:18.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:18.430 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 20:04:18.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:18.430 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:18.430 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 20:04:18.430 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:18.430 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:18.431 | INFO | Uploaded 36GiB to GPU in 2.11135s (17.0507GiB/s) | 20260505 20:04:18.431 | INFO | Downloaded 34GiB to GPU in 2.1362s (15.9161GiB/s) | 20260505 20:04:23.562 | INFO | Analyzing chunk frames = [5934592,5935104) | 20260505 20:04:23.562 | INFO | MemInfo: { | 20260505 20:04:23.562 | INFO | "gpu" : " 17 GB", | 20260505 20:04:23.563 | INFO | "huge" : "", | 20260505 20:04:23.563 | INFO | "huge_shared" : " 27 GB", | 20260505 20:04:23.563 | INFO | "malloc" : " 14 GB", | 20260505 20:04:23.563 | INFO | "pinned" : " 29 GB" | 20260505 20:04:23.563 | INFO | } | 20260505 20:04:23.563 | INFO | Perf report for the last 5.13226 seconds: | 20260505 20:04:23.563 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:04:23.563 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:23.563 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:23.563 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:23.563 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260505 20:04:23.563 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:23.563 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:23.563 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260505 20:04:23.563 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:23.563 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:23.563 | INFO | Uploaded 36GiB to GPU in 2.13073s (16.8957GiB/s) | 20260505 20:04:23.563 | INFO | Downloaded 34GiB to GPU in 2.00964s (16.9185GiB/s) | 20260505 20:04:23.697 | INFO | Starting a round of Baz IO | 20260505 20:04:26.873 | INFO | Populating RTMetrics report | 20260505 20:04:26.873 | INFO | RTMetrics report populated | 20260505 20:04:26.873 | INFO | Opening tmp RTMetrics json file | 20260505 20:04:26.874 | INFO | Serializing RTMetrics to json | 20260505 20:04:26.875 | INFO | Serialization done, swapping tmp file now... | 20260505 20:04:26.875 | INFO | RTMetrics done | 20260505 20:04:26.962 | INFO | Finished a round of Baz IO: Wrote 2.025 GiB in 3.265 seconds | 20260505 20:04:26.962 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.71% / 0.00% / 78.29% | 20260505 20:04:26.962 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.186761% / 0% / 8.74201e-05% | 20260505 20:04:26.962 | INFO | Baz IO overall: | 20260505 20:04:26.962 | INFO | BytesWritten : 3465474583988 | 20260505 20:04:26.962 | INFO | NumEvents : 1638881443152 | 20260505 20:04:28.702 | INFO | Analyzing chunk frames = [5935104,5935616) | 20260505 20:04:28.702 | INFO | MemInfo: { | 20260505 20:04:28.702 | INFO | "gpu" : " 17 GB", | 20260505 20:04:28.702 | INFO | "huge" : "", | 20260505 20:04:28.702 | INFO | "huge_shared" : " 27 GB", | 20260505 20:04:28.702 | INFO | "malloc" : " 14 GB", | 20260505 20:04:28.702 | INFO | "pinned" : " 29 GB" | 20260505 20:04:28.702 | INFO | } | 20260505 20:04:28.702 | INFO | Perf report for the last 5.13899 seconds: | 20260505 20:04:28.702 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:04:28.702 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:28.702 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:28.702 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:28.702 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 20:04:28.702 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:28.702 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260505 20:04:28.702 | INFO | PRE_HQ : 60.3%, 1.8, 55.4ms, 0 | 20260505 20:04:28.702 | INFO | BAZWRITER : 63.0%, 3.7, 119.9ms, 0 | 20260505 20:04:28.702 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 20:04:28.702 | INFO | Uploaded 36GiB to GPU in 2.16957s (16.5931GiB/s) | 20260505 20:04:28.702 | INFO | Downloaded 38GiB to GPU in 2.24891s (16.8971GiB/s) | 20260505 20:04:33.846 | INFO | Analyzing chunk frames = [5935616,5936128) | 20260505 20:04:33.846 | INFO | MemInfo: { | 20260505 20:04:33.846 | INFO | "gpu" : " 17 GB", | 20260505 20:04:33.846 | INFO | "huge" : "", | 20260505 20:04:33.846 | INFO | "huge_shared" : " 27 GB", | 20260505 20:04:33.846 | INFO | "malloc" : " 14 GB", | 20260505 20:04:33.846 | INFO | "pinned" : " 29 GB" | 20260505 20:04:33.846 | INFO | } | 20260505 20:04:33.846 | INFO | Perf report for the last 5.1443 seconds: | 20260505 20:04:33.846 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:04:33.846 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:33.846 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:33.846 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:33.846 | INFO | ANALYSIS : 62.5%, 3.0, 97.7ms, 0 | 20260505 20:04:33.846 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:33.846 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:33.846 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:04:33.846 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:33.846 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:33.846 | INFO | Uploaded 36GiB to GPU in 2.14493s (16.7838GiB/s) | 20260505 20:04:33.846 | INFO | Downloaded 34GiB to GPU in 2.04124s (16.6565GiB/s) | 20260505 20:04:38.990 | INFO | Analyzing chunk frames = [5936128,5936640) | 20260505 20:04:38.990 | INFO | MemInfo: { | 20260505 20:04:38.990 | INFO | "gpu" : " 17 GB", | 20260505 20:04:38.990 | INFO | "huge" : "", | 20260505 20:04:38.990 | INFO | "huge_shared" : " 27 GB", | 20260505 20:04:38.990 | INFO | "malloc" : " 14 GB", | 20260505 20:04:38.990 | INFO | "pinned" : " 29 GB" | 20260505 20:04:38.990 | INFO | } | 20260505 20:04:38.990 | INFO | Perf report for the last 5.14411 seconds: | 20260505 20:04:38.990 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:04:38.990 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:38.990 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:38.990 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:38.990 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260505 20:04:38.990 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:38.990 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:38.990 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:04:38.990 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:38.990 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:38.990 | INFO | Uploaded 36GiB to GPU in 2.13503s (16.8616GiB/s) | 20260505 20:04:38.990 | INFO | Downloaded 34GiB to GPU in 2.01688s (16.8577GiB/s) | 20260505 20:04:44.118 | INFO | Analyzing chunk frames = [5936640,5937152) | 20260505 20:04:44.118 | INFO | MemInfo: { | 20260505 20:04:44.118 | INFO | "gpu" : " 17 GB", | 20260505 20:04:44.118 | INFO | "huge" : "", | 20260505 20:04:44.118 | INFO | "huge_shared" : " 27 GB", | 20260505 20:04:44.118 | INFO | "malloc" : " 14 GB", | 20260505 20:04:44.118 | INFO | "pinned" : " 29 GB" | 20260505 20:04:44.118 | INFO | } | 20260505 20:04:44.118 | INFO | Perf report for the last 5.12818 seconds: | 20260505 20:04:44.118 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:04:44.118 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:44.118 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:44.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:44.118 | INFO | ANALYSIS : 62.2%, 3.0, 96.4ms, 0 | 20260505 20:04:44.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:44.118 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:44.118 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:04:44.118 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:44.118 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:44.119 | INFO | Uploaded 36GiB to GPU in 2.15069s (16.7388GiB/s) | 20260505 20:04:44.119 | INFO | Downloaded 34GiB to GPU in 2.04064s (16.6615GiB/s) | 20260505 20:04:49.255 | INFO | Analyzing chunk frames = [5937152,5937664) | 20260505 20:04:49.255 | INFO | MemInfo: { | 20260505 20:04:49.255 | INFO | "gpu" : " 17 GB", | 20260505 20:04:49.255 | INFO | "huge" : "", | 20260505 20:04:49.255 | INFO | "huge_shared" : " 27 GB", | 20260505 20:04:49.255 | INFO | "malloc" : " 14 GB", | 20260505 20:04:49.255 | INFO | "pinned" : " 29 GB" | 20260505 20:04:49.255 | INFO | } | 20260505 20:04:49.255 | INFO | Perf report for the last 5.13687 seconds: | 20260505 20:04:49.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:04:49.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:49.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:49.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:49.255 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260505 20:04:49.255 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:49.255 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:49.255 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:04:49.255 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:49.255 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:49.255 | INFO | Uploaded 36GiB to GPU in 2.13229s (16.8833GiB/s) | 20260505 20:04:49.255 | INFO | Downloaded 34GiB to GPU in 2.06004s (16.5046GiB/s) | 20260505 20:04:54.400 | INFO | Analyzing chunk frames = [5937664,5938176) | 20260505 20:04:54.400 | INFO | MemInfo: { | 20260505 20:04:54.400 | INFO | "gpu" : " 17 GB", | 20260505 20:04:54.400 | INFO | "huge" : "", | 20260505 20:04:54.400 | INFO | "huge_shared" : " 27 GB", | 20260505 20:04:54.400 | INFO | "malloc" : " 14 GB", | 20260505 20:04:54.400 | INFO | "pinned" : " 29 GB" | 20260505 20:04:54.400 | INFO | } | 20260505 20:04:54.400 | INFO | Perf report for the last 5.14484 seconds: | 20260505 20:04:54.400 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:04:54.400 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:54.400 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:54.400 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:54.400 | INFO | ANALYSIS : 62.3%, 3.0, 97.4ms, 0 | 20260505 20:04:54.400 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:54.400 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:54.400 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:04:54.400 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:54.400 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:54.400 | INFO | Uploaded 36GiB to GPU in 2.15023s (16.7424GiB/s) | 20260505 20:04:54.400 | INFO | Downloaded 34GiB to GPU in 2.05053s (16.5811GiB/s) | 20260505 20:04:59.545 | INFO | Analyzing chunk frames = [5938176,5938688) | 20260505 20:04:59.545 | INFO | MemInfo: { | 20260505 20:04:59.545 | INFO | "gpu" : " 17 GB", | 20260505 20:04:59.545 | INFO | "huge" : "", | 20260505 20:04:59.545 | INFO | "huge_shared" : " 27 GB", | 20260505 20:04:59.545 | INFO | "malloc" : " 14 GB", | 20260505 20:04:59.545 | INFO | "pinned" : " 29 GB" | 20260505 20:04:59.545 | INFO | } | 20260505 20:04:59.545 | INFO | Perf report for the last 5.14481 seconds: | 20260505 20:04:59.545 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:04:59.545 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:59.545 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:59.545 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:59.545 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 20:04:59.545 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:59.545 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:59.545 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 20:04:59.545 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:59.545 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:04:59.545 | INFO | Uploaded 36GiB to GPU in 2.12079s (16.9748GiB/s) | 20260505 20:04:59.545 | INFO | Downloaded 34GiB to GPU in 2.14422s (15.8566GiB/s) | 20260505 20:05:04.682 | INFO | Analyzing chunk frames = [5938688,5939200) | 20260505 20:05:04.682 | INFO | MemInfo: { | 20260505 20:05:04.682 | INFO | "gpu" : " 17 GB", | 20260505 20:05:04.682 | INFO | "huge" : "", | 20260505 20:05:04.682 | INFO | "huge_shared" : " 27 GB", | 20260505 20:05:04.683 | INFO | "malloc" : " 14 GB", | 20260505 20:05:04.683 | INFO | "pinned" : " 29 GB" | 20260505 20:05:04.683 | INFO | } | 20260505 20:05:04.683 | INFO | Perf report for the last 5.13771 seconds: | 20260505 20:05:04.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:05:04.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:04.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:04.683 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:04.683 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260505 20:05:04.683 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:04.683 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:04.683 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:05:04.683 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:04.683 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:04.683 | INFO | Uploaded 36GiB to GPU in 2.12134s (16.9704GiB/s) | 20260505 20:05:04.683 | INFO | Downloaded 34GiB to GPU in 2.01157s (16.9022GiB/s) | 20260505 20:05:04.683 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 20:05:04.683 | INFO | Total mem usage: 15.3 GiB | 20260505 20:05:04.683 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 20:05:04.683 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.1%) | 20260505 20:05:04.683 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 20:05:04.683 | INFO | High Water Breakdown: | 20260505 20:05:04.683 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 20:05:04.683 | INFO | 38.8% : 5.51 GiB ( 82.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 20:05:04.683 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 20:05:04.683 | INFO | ------------------------------------------ | 20260505 20:05:04.683 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:05:04.683 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 20:05:04.683 | INFO | Total mem usage: 26.5 GiB | 20260505 20:05:04.683 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 20:05:04.683 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 20:05:04.683 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 20:05:04.683 | INFO | High Water Breakdown: | 20260505 20:05:04.683 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:05:04.683 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:05:04.683 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:05:04.683 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:05:04.683 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:05:04.683 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:05:04.683 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 20:05:04.683 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 20:05:04.683 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:05:04.683 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 20:05:04.683 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 20:05:04.683 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 20:05:04.683 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 20:05:04.683 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 20:05:04.683 | INFO | ------------------------------------------ | 20260505 20:05:04.683 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:05:04.683 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 20:05:04.683 | INFO | Total mem usage: 25.6 GiB | 20260505 20:05:04.683 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 20:05:04.683 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 20:05:04.684 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 20:05:04.684 | INFO | High Water Breakdown: | 20260505 20:05:04.684 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 20:05:04.684 | INFO | ------------------------------------------ | 20260505 20:05:04.684 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:05:04.684 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 20:05:04.684 | INFO | Total mem usage: 15.7 GiB | 20260505 20:05:04.684 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 20:05:04.684 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 20:05:04.684 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 20:05:04.684 | INFO | High Water Breakdown: | 20260505 20:05:04.684 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 20:05:04.684 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 20:05:04.684 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 20:05:04.684 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 20:05:04.684 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 20:05:04.684 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 20:05:04.684 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 20:05:04.684 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 20:05:04.684 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 20:05:04.684 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:05:04.684 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 20:05:04.684 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:05:04.684 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 20:05:04.684 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 20:05:04.684 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 20:05:04.684 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 20:05:04.684 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:05:04.684 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 20:05:04.684 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:05:04.684 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:05:04.684 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:05:04.684 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 20:05:04.684 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 20:05:04.684 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:05:04.684 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 20:05:04.684 | INFO | ------------------------------------------ | 20260505 20:05:04.684 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:05:04.684 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 20:05:04.684 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:05:04.684 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:05:04.684 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.45e+01, 1.36e+01 / 1.44e+00 | 20260505 20:05:04.684 | INFO | PerfReport: 0.29%, AggregateStats -- 9.26e-02 / 1.04e-01, 1.17e-02 / 5.45e-03 | 20260505 20:05:04.684 | INFO | PerfReport: 11.80%, Baseline -- 3.72e+00 / 4.54e+00, 5.72e-01 / 1.19e-01 | 20260505 20:05:04.684 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.51e+00, 1.92e-01 / 9.97e-02 | 20260505 20:05:04.684 | INFO | PerfReport: 15.95%, DME -- 4.02e+01 / 4.43e+01, 3.90e+00 / 1.71e+00 | 20260505 20:05:04.684 | INFO | PerfReport: 57.07%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.51e+00 / 1.02e+00 | 20260505 20:05:04.684 | INFO | PerfReport: 7.22%, PulseAccumulating -- 2.27e+00 / 2.84e+00, 4.34e-01 / 1.80e-01 | 20260505 20:05:04.684 | INFO | PerfReport: 3.25%, Metrics -- 1.02e+00 / 1.36e+00, 2.81e-01 / 1.46e-01 | 20260505 20:05:04.684 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.56e-01 / 4.17e-01, 6.28e-02 / 3.40e-02 | 20260505 20:05:04.684 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 20:05:04.684 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:05:04.684 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:05:04.684 | INFO | PerfReport: Full Execution -- 9.94e+01 / -nan, 2.54e+01 / 0.00e+00 | 20260505 20:05:04.684 | INFO | PerfReport: 22.41%, Upload -- 2.23e+01 / 0.00e+00, 7.72e+00 / 0.00e+00 | 20260505 20:05:04.684 | INFO | PerfReport: 21.76%, Download -- 2.16e+01 / 0.00e+00, 7.69e+00 / 0.00e+00 | 20260505 20:05:04.684 | INFO | PerfReport: 31.71%, Compute -- 3.15e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 20:05:04.684 | INFO | PerfReport: 2.28%, LockUpload -- 2.27e+00 / 0.00e+00, 8.04e+00 / 0.00e+00 | 20260505 20:05:04.684 | INFO | PerfReport: 1.38%, LockDownload -- 1.37e+00 / 0.00e+00, 3.00e+00 / 0.00e+00 | 20260505 20:05:04.684 | INFO | PerfReport: 20.45%, LockCompute -- 2.03e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260505 20:05:04.684 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.88e-02 / 0.00e+00 | 20260505 20:05:04.813 | INFO | Starting a round of Baz IO | 20260505 20:05:07.983 | INFO | Populating RTMetrics report | 20260505 20:05:07.983 | INFO | RTMetrics report populated | 20260505 20:05:07.983 | INFO | Opening tmp RTMetrics json file | 20260505 20:05:07.984 | INFO | Serializing RTMetrics to json | 20260505 20:05:07.985 | INFO | Serialization done, swapping tmp file now... | 20260505 20:05:07.986 | INFO | RTMetrics done | 20260505 20:05:08.073 | INFO | Finished a round of Baz IO: Wrote 2.025 GiB in 3.26 seconds | 20260505 20:05:08.073 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.71% / 0.00% / 78.29% | 20260505 20:05:08.073 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.186221% / 0% / 6.35783e-05% | 20260505 20:05:08.073 | INFO | Baz IO overall: | 20260505 20:05:08.073 | INFO | BytesWritten : 3467649356162 | 20260505 20:05:08.073 | INFO | NumEvents : 1639889976568 | 20260505 20:05:09.818 | INFO | Analyzing chunk frames = [5939200,5939712) | 20260505 20:05:09.818 | INFO | MemInfo: { | 20260505 20:05:09.818 | INFO | "gpu" : " 17 GB", | 20260505 20:05:09.818 | INFO | "huge" : "", | 20260505 20:05:09.819 | INFO | "huge_shared" : " 27 GB", | 20260505 20:05:09.819 | INFO | "malloc" : " 14 GB", | 20260505 20:05:09.819 | INFO | "pinned" : " 29 GB" | 20260505 20:05:09.819 | INFO | } | 20260505 20:05:09.819 | INFO | Perf report for the last 5.13601 seconds: | 20260505 20:05:09.819 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:05:09.819 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:09.819 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:09.819 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:09.819 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 20:05:09.819 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:09.819 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260505 20:05:09.819 | INFO | PRE_HQ : 60.4%, 1.8, 55.3ms, 0 | 20260505 20:05:09.819 | INFO | BAZWRITER : 63.0%, 3.7, 119.5ms, 0 | 20260505 20:05:09.819 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 20:05:09.819 | INFO | Uploaded 36GiB to GPU in 2.15044s (16.7407GiB/s) | 20260505 20:05:09.819 | INFO | Downloaded 38GiB to GPU in 2.24442s (16.9308GiB/s) | 20260505 20:05:14.962 | INFO | Analyzing chunk frames = [5939712,5940224) | 20260505 20:05:14.962 | INFO | MemInfo: { | 20260505 20:05:14.962 | INFO | "gpu" : " 17 GB", | 20260505 20:05:14.962 | INFO | "huge" : "", | 20260505 20:05:14.962 | INFO | "huge_shared" : " 27 GB", | 20260505 20:05:14.962 | INFO | "malloc" : " 14 GB", | 20260505 20:05:14.962 | INFO | "pinned" : " 29 GB" | 20260505 20:05:14.962 | INFO | } | 20260505 20:05:14.962 | INFO | Perf report for the last 5.14388 seconds: | 20260505 20:05:14.963 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:05:14.963 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:14.963 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:14.963 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:14.963 | INFO | ANALYSIS : 62.5%, 3.0, 97.7ms, 0 | 20260505 20:05:14.963 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:14.963 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:14.963 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:05:14.963 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:14.963 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:14.963 | INFO | Uploaded 36GiB to GPU in 2.15222s (16.7269GiB/s) | 20260505 20:05:14.963 | INFO | Downloaded 34GiB to GPU in 2.05211s (16.5683GiB/s) | 20260505 20:05:20.101 | INFO | Analyzing chunk frames = [5940224,5940736) | 20260505 20:05:20.101 | INFO | MemInfo: { | 20260505 20:05:20.101 | INFO | "gpu" : " 17 GB", | 20260505 20:05:20.101 | INFO | "huge" : "", | 20260505 20:05:20.101 | INFO | "huge_shared" : " 27 GB", | 20260505 20:05:20.101 | INFO | "malloc" : " 14 GB", | 20260505 20:05:20.101 | INFO | "pinned" : " 29 GB" | 20260505 20:05:20.101 | INFO | } | 20260505 20:05:20.101 | INFO | Perf report for the last 5.13868 seconds: | 20260505 20:05:20.101 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:05:20.101 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:20.101 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:20.101 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:20.101 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260505 20:05:20.101 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:20.101 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:20.101 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:05:20.101 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:20.101 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:20.101 | INFO | Uploaded 36GiB to GPU in 2.1262s (16.9317GiB/s) | 20260505 20:05:20.102 | INFO | Downloaded 34GiB to GPU in 2.0173s (16.8542GiB/s) | 20260505 20:05:25.236 | INFO | Analyzing chunk frames = [5940736,5941248) | 20260505 20:05:25.236 | INFO | MemInfo: { | 20260505 20:05:25.236 | INFO | "gpu" : " 17 GB", | 20260505 20:05:25.236 | INFO | "huge" : "", | 20260505 20:05:25.236 | INFO | "huge_shared" : " 27 GB", | 20260505 20:05:25.236 | INFO | "malloc" : " 14 GB", | 20260505 20:05:25.236 | INFO | "pinned" : " 29 GB" | 20260505 20:05:25.236 | INFO | } | 20260505 20:05:25.236 | INFO | Perf report for the last 5.13519 seconds: | 20260505 20:05:25.236 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:05:25.236 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:25.236 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:25.236 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:25.236 | INFO | ANALYSIS : 62.2%, 3.0, 96.5ms, 0 | 20260505 20:05:25.236 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:25.236 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:25.237 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:05:25.237 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:25.237 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:25.237 | INFO | Uploaded 36GiB to GPU in 2.14394s (16.7915GiB/s) | 20260505 20:05:25.237 | INFO | Downloaded 34GiB to GPU in 2.04899s (16.5935GiB/s) | 20260505 20:05:30.380 | INFO | Analyzing chunk frames = [5941248,5941760) | 20260505 20:05:30.380 | INFO | MemInfo: { | 20260505 20:05:30.380 | INFO | "gpu" : " 17 GB", | 20260505 20:05:30.380 | INFO | "huge" : "", | 20260505 20:05:30.380 | INFO | "huge_shared" : " 27 GB", | 20260505 20:05:30.381 | INFO | "malloc" : " 14 GB", | 20260505 20:05:30.381 | INFO | "pinned" : " 29 GB" | 20260505 20:05:30.381 | INFO | } | 20260505 20:05:30.381 | INFO | Perf report for the last 5.14421 seconds: | 20260505 20:05:30.381 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:05:30.381 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:30.381 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:30.381 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:30.381 | INFO | ANALYSIS : 61.8%, 3.0, 96.6ms, 0 | 20260505 20:05:30.381 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:30.381 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:30.381 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:05:30.381 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:30.381 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:30.381 | INFO | Uploaded 36GiB to GPU in 2.13432s (16.8672GiB/s) | 20260505 20:05:30.381 | INFO | Downloaded 34GiB to GPU in 2.06274s (16.4829GiB/s) | 20260505 20:05:35.513 | INFO | Analyzing chunk frames = [5941760,5942272) | 20260505 20:05:35.513 | INFO | MemInfo: { | 20260505 20:05:35.513 | INFO | "gpu" : " 17 GB", | 20260505 20:05:35.513 | INFO | "huge" : "", | 20260505 20:05:35.513 | INFO | "huge_shared" : " 27 GB", | 20260505 20:05:35.513 | INFO | "malloc" : " 14 GB", | 20260505 20:05:35.513 | INFO | "pinned" : " 29 GB" | 20260505 20:05:35.513 | INFO | } | 20260505 20:05:35.513 | INFO | Perf report for the last 5.13226 seconds: | 20260505 20:05:35.513 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:05:35.513 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:35.513 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:35.513 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:35.513 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 20:05:35.513 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:35.513 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:35.513 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 20:05:35.513 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:35.513 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:35.513 | INFO | Uploaded 36GiB to GPU in 2.14776s (16.7616GiB/s) | 20260505 20:05:35.513 | INFO | Downloaded 34GiB to GPU in 2.05962s (16.5079GiB/s) | 20260505 20:05:40.660 | INFO | Analyzing chunk frames = [5942272,5942784) | 20260505 20:05:40.660 | INFO | MemInfo: { | 20260505 20:05:40.660 | INFO | "gpu" : " 17 GB", | 20260505 20:05:40.660 | INFO | "huge" : "", | 20260505 20:05:40.660 | INFO | "huge_shared" : " 27 GB", | 20260505 20:05:40.660 | INFO | "malloc" : " 14 GB", | 20260505 20:05:40.660 | INFO | "pinned" : " 29 GB" | 20260505 20:05:40.660 | INFO | } | 20260505 20:05:40.660 | INFO | Perf report for the last 5.14693 seconds: | 20260505 20:05:40.660 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:05:40.660 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:40.660 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:40.660 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:40.660 | INFO | ANALYSIS : 63.2%, 3.0, 98.9ms, 0 | 20260505 20:05:40.660 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:40.660 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:40.660 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 20:05:40.660 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:40.660 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:40.660 | INFO | Uploaded 36GiB to GPU in 2.16106s (16.6585GiB/s) | 20260505 20:05:40.660 | INFO | Downloaded 34GiB to GPU in 2.17926s (15.6016GiB/s) | 20260505 20:05:45.797 | INFO | Analyzing chunk frames = [5942784,5943296) | 20260505 20:05:45.797 | INFO | MemInfo: { | 20260505 20:05:45.797 | INFO | "gpu" : " 17 GB", | 20260505 20:05:45.797 | INFO | "huge" : "", | 20260505 20:05:45.797 | INFO | "huge_shared" : " 27 GB", | 20260505 20:05:45.797 | INFO | "malloc" : " 14 GB", | 20260505 20:05:45.797 | INFO | "pinned" : " 29 GB" | 20260505 20:05:45.797 | INFO | } | 20260505 20:05:45.797 | INFO | Perf report for the last 5.13713 seconds: | 20260505 20:05:45.797 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:05:45.797 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:45.797 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:45.797 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:45.797 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 20:05:45.797 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:45.797 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:45.797 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 20:05:45.797 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:45.797 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:45.797 | INFO | Uploaded 36GiB to GPU in 2.11358s (17.0327GiB/s) | 20260505 20:05:45.797 | INFO | Downloaded 34GiB to GPU in 2.00202s (16.9828GiB/s) | 20260505 20:05:45.931 | INFO | Starting a round of Baz IO | 20260505 20:05:49.094 | INFO | Populating RTMetrics report | 20260505 20:05:49.094 | INFO | RTMetrics report populated | 20260505 20:05:49.094 | INFO | Opening tmp RTMetrics json file | 20260505 20:05:49.097 | INFO | Serializing RTMetrics to json | 20260505 20:05:49.101 | INFO | Serialization done, swapping tmp file now... | 20260505 20:05:49.102 | INFO | RTMetrics done | 20260505 20:05:49.184 | INFO | Finished a round of Baz IO: Wrote 2.025 GiB in 3.252 seconds | 20260505 20:05:49.184 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.71% / 0.00% / 78.29% | 20260505 20:05:49.184 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.18603% / 0% / 0.00013113% | 20260505 20:05:49.184 | INFO | Baz IO overall: | 20260505 20:05:49.184 | INFO | BytesWritten : 3469823680101 | 20260505 20:05:49.184 | INFO | NumEvents : 1640898205817 | 20260505 20:05:50.936 | INFO | Analyzing chunk frames = [5943296,5943808) | 20260505 20:05:50.936 | INFO | MemInfo: { | 20260505 20:05:50.936 | INFO | "gpu" : " 17 GB", | 20260505 20:05:50.936 | INFO | "huge" : "", | 20260505 20:05:50.936 | INFO | "huge_shared" : " 27 GB", | 20260505 20:05:50.936 | INFO | "malloc" : " 14 GB", | 20260505 20:05:50.936 | INFO | "pinned" : " 29 GB" | 20260505 20:05:50.936 | INFO | } | 20260505 20:05:50.936 | INFO | Perf report for the last 5.1395 seconds: | 20260505 20:05:50.936 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:05:50.936 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:50.937 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:50.937 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:50.937 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 20:05:50.937 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:50.937 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260505 20:05:50.937 | INFO | PRE_HQ : 60.1%, 1.8, 55.5ms, 0 | 20260505 20:05:50.937 | INFO | BAZWRITER : 62.7%, 3.7, 119.2ms, 0 | 20260505 20:05:50.937 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 20:05:50.937 | INFO | Uploaded 36GiB to GPU in 2.17599s (16.5442GiB/s) | 20260505 20:05:50.937 | INFO | Downloaded 38GiB to GPU in 2.25613s (16.843GiB/s) | 20260505 20:05:56.070 | INFO | Analyzing chunk frames = [5943808,5944320) | 20260505 20:05:56.070 | INFO | MemInfo: { | 20260505 20:05:56.070 | INFO | "gpu" : " 17 GB", | 20260505 20:05:56.070 | INFO | "huge" : "", | 20260505 20:05:56.070 | INFO | "huge_shared" : " 27 GB", | 20260505 20:05:56.070 | INFO | "malloc" : " 14 GB", | 20260505 20:05:56.070 | INFO | "pinned" : " 29 GB" | 20260505 20:05:56.070 | INFO | } | 20260505 20:05:56.070 | INFO | Perf report for the last 5.13388 seconds: | 20260505 20:05:56.070 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:05:56.070 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:56.070 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:56.070 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:56.070 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 20:05:56.070 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:56.070 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:56.070 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:05:56.070 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:56.071 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:05:56.071 | INFO | Uploaded 36GiB to GPU in 2.10707s (17.0853GiB/s) | 20260505 20:05:56.071 | INFO | Downloaded 34GiB to GPU in 2.02759s (16.7686GiB/s) | 20260505 20:06:01.210 | INFO | Analyzing chunk frames = [5944320,5944832) | 20260505 20:06:01.210 | INFO | MemInfo: { | 20260505 20:06:01.210 | INFO | "gpu" : " 17 GB", | 20260505 20:06:01.210 | INFO | "huge" : "", | 20260505 20:06:01.210 | INFO | "huge_shared" : " 27 GB", | 20260505 20:06:01.210 | INFO | "malloc" : " 14 GB", | 20260505 20:06:01.210 | INFO | "pinned" : " 29 GB" | 20260505 20:06:01.210 | INFO | } | 20260505 20:06:01.210 | INFO | Perf report for the last 5.13999 seconds: | 20260505 20:06:01.210 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:06:01.210 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:01.210 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:01.210 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:01.210 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260505 20:06:01.210 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:01.210 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:01.210 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:06:01.210 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:01.211 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:01.211 | INFO | Uploaded 36GiB to GPU in 2.14798s (16.76GiB/s) | 20260505 20:06:01.211 | INFO | Downloaded 34GiB to GPU in 2.04426s (16.632GiB/s) | 20260505 20:06:06.351 | INFO | Analyzing chunk frames = [5944832,5945344) | 20260505 20:06:06.351 | INFO | MemInfo: { | 20260505 20:06:06.351 | INFO | "gpu" : " 17 GB", | 20260505 20:06:06.351 | INFO | "huge" : "", | 20260505 20:06:06.351 | INFO | "huge_shared" : " 27 GB", | 20260505 20:06:06.351 | INFO | "malloc" : " 14 GB", | 20260505 20:06:06.351 | INFO | "pinned" : " 29 GB" | 20260505 20:06:06.351 | INFO | } | 20260505 20:06:06.352 | INFO | Perf report for the last 5.14116 seconds: | 20260505 20:06:06.352 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:06:06.352 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:06.352 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:06.352 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:06.352 | INFO | ANALYSIS : 62.4%, 3.0, 97.0ms, 0 | 20260505 20:06:06.352 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:06.352 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:06.352 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:06:06.352 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:06.352 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:06.352 | INFO | Uploaded 36GiB to GPU in 2.20209s (16.3481GiB/s) | 20260505 20:06:06.352 | INFO | Downloaded 34GiB to GPU in 2.04321s (16.6405GiB/s) | 20260505 20:06:11.499 | INFO | Analyzing chunk frames = [5945344,5945856) | 20260505 20:06:11.499 | INFO | MemInfo: { | 20260505 20:06:11.499 | INFO | "gpu" : " 17 GB", | 20260505 20:06:11.499 | INFO | "huge" : "", | 20260505 20:06:11.499 | INFO | "huge_shared" : " 27 GB", | 20260505 20:06:11.499 | INFO | "malloc" : " 14 GB", | 20260505 20:06:11.499 | INFO | "pinned" : " 29 GB" | 20260505 20:06:11.499 | INFO | } | 20260505 20:06:11.499 | INFO | Perf report for the last 5.14766 seconds: | 20260505 20:06:11.499 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:06:11.499 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:11.499 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:11.499 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:11.499 | INFO | ANALYSIS : 61.6%, 3.0, 96.4ms, 0 | 20260505 20:06:11.499 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:11.499 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:11.499 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:06:11.499 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:11.499 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:11.500 | INFO | Uploaded 36GiB to GPU in 2.1259s (16.934GiB/s) | 20260505 20:06:11.500 | INFO | Downloaded 34GiB to GPU in 2.06275s (16.4828GiB/s) | 20260505 20:06:16.635 | INFO | Analyzing chunk frames = [5945856,5946368) | 20260505 20:06:16.635 | INFO | MemInfo: { | 20260505 20:06:16.635 | INFO | "gpu" : " 17 GB", | 20260505 20:06:16.635 | INFO | "huge" : "", | 20260505 20:06:16.635 | INFO | "huge_shared" : " 27 GB", | 20260505 20:06:16.635 | INFO | "malloc" : " 14 GB", | 20260505 20:06:16.635 | INFO | "pinned" : " 29 GB" | 20260505 20:06:16.635 | INFO | } | 20260505 20:06:16.635 | INFO | Perf report for the last 5.13556 seconds: | 20260505 20:06:16.635 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:06:16.635 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:16.635 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:16.635 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:16.635 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 20:06:16.635 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:16.635 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:16.635 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:06:16.635 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:16.635 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:16.635 | INFO | Uploaded 36GiB to GPU in 2.0927s (17.2027GiB/s) | 20260505 20:06:16.635 | INFO | Downloaded 34GiB to GPU in 2.03713s (16.6901GiB/s) | 20260505 20:06:21.765 | INFO | Analyzing chunk frames = [5946368,5946880) | 20260505 20:06:21.765 | INFO | MemInfo: { | 20260505 20:06:21.765 | INFO | "gpu" : " 17 GB", | 20260505 20:06:21.765 | INFO | "huge" : "", | 20260505 20:06:21.765 | INFO | "huge_shared" : " 27 GB", | 20260505 20:06:21.765 | INFO | "malloc" : " 14 GB", | 20260505 20:06:21.765 | INFO | "pinned" : " 29 GB" | 20260505 20:06:21.765 | INFO | } | 20260505 20:06:21.766 | INFO | Perf report for the last 5.13078 seconds: | 20260505 20:06:21.766 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:06:21.766 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:21.766 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:21.766 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:21.766 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 20:06:21.766 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:21.766 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:21.766 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 20:06:21.766 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:21.766 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:21.766 | INFO | Uploaded 36GiB to GPU in 2.12513s (16.9401GiB/s) | 20260505 20:06:21.766 | INFO | Downloaded 34GiB to GPU in 2.1506s (15.8095GiB/s) | 20260505 20:06:26.919 | INFO | Analyzing chunk frames = [5946880,5947392) | 20260505 20:06:26.919 | INFO | MemInfo: { | 20260505 20:06:26.919 | INFO | "gpu" : " 17 GB", | 20260505 20:06:26.919 | INFO | "huge" : "", | 20260505 20:06:26.919 | INFO | "huge_shared" : " 27 GB", | 20260505 20:06:26.919 | INFO | "malloc" : " 14 GB", | 20260505 20:06:26.919 | INFO | "pinned" : " 29 GB" | 20260505 20:06:26.919 | INFO | } | 20260505 20:06:26.919 | INFO | Perf report for the last 5.1535 seconds: | 20260505 20:06:26.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:06:26.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:26.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:26.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:26.919 | INFO | ANALYSIS : 62.7%, 3.0, 98.2ms, 0 | 20260505 20:06:26.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:26.919 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:26.919 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260505 20:06:26.919 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:26.919 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:26.919 | INFO | Uploaded 36GiB to GPU in 2.19539s (16.398GiB/s) | 20260505 20:06:26.919 | INFO | Downloaded 34GiB to GPU in 2.04072s (16.6608GiB/s) | 20260505 20:06:26.920 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 20:06:26.920 | INFO | Total mem usage: 15.3 GiB | 20260505 20:06:26.920 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 20:06:26.920 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.12%) | 20260505 20:06:26.920 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 20:06:26.920 | INFO | High Water Breakdown: | 20260505 20:06:26.920 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 20:06:26.920 | INFO | 38.8% : 5.51 GiB ( 82.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 20:06:26.920 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 20:06:26.920 | INFO | ------------------------------------------ | 20260505 20:06:26.920 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:06:26.920 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 20:06:26.920 | INFO | Total mem usage: 26.5 GiB | 20260505 20:06:26.920 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 20:06:26.920 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 20:06:26.920 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 20:06:26.920 | INFO | High Water Breakdown: | 20260505 20:06:26.920 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:06:26.920 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:06:26.920 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:06:26.920 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:06:26.920 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:06:26.920 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:06:26.920 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 20:06:26.920 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 20:06:26.920 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:06:26.920 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 20:06:26.920 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 20:06:26.920 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 20:06:26.920 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 20:06:26.920 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 20:06:26.920 | INFO | ------------------------------------------ | 20260505 20:06:26.920 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:06:26.920 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 20:06:26.920 | INFO | Total mem usage: 25.6 GiB | 20260505 20:06:26.920 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 20:06:26.920 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 20:06:26.920 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 20:06:26.920 | INFO | High Water Breakdown: | 20260505 20:06:26.920 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 20:06:26.920 | INFO | ------------------------------------------ | 20260505 20:06:26.920 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:06:26.920 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 20:06:26.920 | INFO | Total mem usage: 15.7 GiB | 20260505 20:06:26.920 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 20:06:26.920 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 20:06:26.920 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 20:06:26.920 | INFO | High Water Breakdown: | 20260505 20:06:26.920 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 20:06:26.920 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 20:06:26.920 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 20:06:26.920 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 20:06:26.920 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 20:06:26.920 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 20:06:26.920 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 20:06:26.920 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 20:06:26.920 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 20:06:26.920 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:06:26.920 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 20:06:26.920 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:06:26.920 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 20:06:26.920 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 20:06:26.920 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 20:06:26.920 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 20:06:26.920 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:06:26.920 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 20:06:26.920 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:06:26.920 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:06:26.920 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:06:26.920 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 20:06:26.920 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 20:06:26.920 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:06:26.920 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 20:06:26.920 | INFO | ------------------------------------------ | 20260505 20:06:26.920 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:06:26.920 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 20:06:26.920 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:06:26.920 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:06:26.920 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.45e+01, 1.34e+01 / 1.89e+00 | 20260505 20:06:26.920 | INFO | PerfReport: 0.29%, AggregateStats -- 9.25e-02 / 1.03e-01, 1.02e-02 / 4.60e-03 | 20260505 20:06:26.920 | INFO | PerfReport: 11.79%, Baseline -- 3.72e+00 / 4.53e+00, 5.66e-01 / 1.94e-01 | 20260505 20:06:26.921 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.51e+00, 1.90e-01 / 1.08e-01 | 20260505 20:06:26.921 | INFO | PerfReport: 15.95%, DME -- 4.02e+01 / 4.43e+01, 4.21e+00 / 2.04e+00 | 20260505 20:06:26.921 | INFO | PerfReport: 57.08%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.51e+00 / 1.09e+00 | 20260505 20:06:26.921 | INFO | PerfReport: 7.22%, PulseAccumulating -- 2.28e+00 / 2.86e+00, 4.34e-01 / 2.21e-01 | 20260505 20:06:26.921 | INFO | PerfReport: 3.25%, Metrics -- 1.02e+00 / 1.33e+00, 2.78e-01 / 1.11e-01 | 20260505 20:06:26.921 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.57e-01 / 4.18e-01, 6.27e-02 / 3.69e-02 | 20260505 20:06:26.921 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 20:06:26.921 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:06:26.921 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:06:26.921 | INFO | PerfReport: Full Execution -- 9.94e+01 / -nan, 2.55e+01 / 0.00e+00 | 20260505 20:06:26.921 | INFO | PerfReport: 22.47%, Upload -- 2.23e+01 / 0.00e+00, 7.72e+00 / 0.00e+00 | 20260505 20:06:26.921 | INFO | PerfReport: 21.83%, Download -- 2.17e+01 / 0.00e+00, 7.71e+00 / 0.00e+00 | 20260505 20:06:26.921 | INFO | PerfReport: 31.71%, Compute -- 3.15e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260505 20:06:26.921 | INFO | PerfReport: 2.23%, LockUpload -- 2.22e+00 / 0.00e+00, 7.94e+00 / 0.00e+00 | 20260505 20:06:26.921 | INFO | PerfReport: 1.36%, LockDownload -- 1.35e+00 / 0.00e+00, 2.90e+00 / 0.00e+00 | 20260505 20:06:26.921 | INFO | PerfReport: 20.39%, LockCompute -- 2.03e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260505 20:06:26.921 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.38e-02 / 0.00e+00 | 20260505 20:06:27.048 | INFO | Starting a round of Baz IO | 20260505 20:06:30.206 | INFO | Populating RTMetrics report | 20260505 20:06:30.206 | INFO | RTMetrics report populated | 20260505 20:06:30.206 | INFO | Opening tmp RTMetrics json file | 20260505 20:06:30.207 | INFO | Serializing RTMetrics to json | 20260505 20:06:30.209 | INFO | Serialization done, swapping tmp file now... | 20260505 20:06:30.209 | INFO | RTMetrics done | 20260505 20:06:30.293 | INFO | Finished a round of Baz IO: Wrote 2.025 GiB in 3.244 seconds | 20260505 20:06:30.293 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.70% / 0.00% / 78.29% | 20260505 20:06:30.293 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.185692% / 3.97364e-06% / 8.34465e-05% | 20260505 20:06:30.293 | INFO | Baz IO overall: | 20260505 20:06:30.293 | INFO | BytesWritten : 3471997672339 | 20260505 20:06:30.293 | INFO | NumEvents : 1641906563907 | 20260505 20:06:32.051 | INFO | Analyzing chunk frames = [5947392,5947904) | 20260505 20:06:32.051 | INFO | MemInfo: { | 20260505 20:06:32.051 | INFO | "gpu" : " 17 GB", | 20260505 20:06:32.051 | INFO | "huge" : "", | 20260505 20:06:32.051 | INFO | "huge_shared" : " 27 GB", | 20260505 20:06:32.051 | INFO | "malloc" : " 14 GB", | 20260505 20:06:32.051 | INFO | "pinned" : " 29 GB" | 20260505 20:06:32.051 | INFO | } | 20260505 20:06:32.052 | INFO | Perf report for the last 5.13242 seconds: | 20260505 20:06:32.052 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:06:32.052 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:32.052 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:32.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:32.052 | INFO | ANALYSIS : 63.0%, 3.0, 98.0ms, 0 | 20260505 20:06:32.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:32.052 | INFO | RT_METRICS : 44.4%, 1.0, 23.2ms, 0 | 20260505 20:06:32.052 | INFO | PRE_HQ : 60.5%, 1.8, 55.7ms, 0 | 20260505 20:06:32.052 | INFO | BAZWRITER : 62.6%, 3.7, 118.5ms, 0 | 20260505 20:06:32.052 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 20:06:32.052 | INFO | Uploaded 36GiB to GPU in 2.16565s (16.6232GiB/s) | 20260505 20:06:32.052 | INFO | Downloaded 38GiB to GPU in 2.24646s (16.9155GiB/s) | 20260505 20:06:37.186 | INFO | Analyzing chunk frames = [5947904,5948416) | 20260505 20:06:37.186 | INFO | MemInfo: { | 20260505 20:06:37.186 | INFO | "gpu" : " 17 GB", | 20260505 20:06:37.186 | INFO | "huge" : "", | 20260505 20:06:37.186 | INFO | "huge_shared" : " 27 GB", | 20260505 20:06:37.186 | INFO | "malloc" : " 14 GB", | 20260505 20:06:37.186 | INFO | "pinned" : " 29 GB" | 20260505 20:06:37.186 | INFO | } | 20260505 20:06:37.186 | INFO | Perf report for the last 5.13431 seconds: | 20260505 20:06:37.186 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:06:37.186 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:37.186 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:37.186 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:37.186 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 20:06:37.186 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:37.186 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:37.186 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 20:06:37.186 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:37.186 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:37.186 | INFO | Uploaded 36GiB to GPU in 2.0931s (17.1993GiB/s) | 20260505 20:06:37.186 | INFO | Downloaded 34GiB to GPU in 2.03155s (16.736GiB/s) | 20260505 20:06:42.330 | INFO | Analyzing chunk frames = [5948416,5948928) | 20260505 20:06:42.330 | INFO | MemInfo: { | 20260505 20:06:42.330 | INFO | "gpu" : " 17 GB", | 20260505 20:06:42.330 | INFO | "huge" : "", | 20260505 20:06:42.330 | INFO | "huge_shared" : " 27 GB", | 20260505 20:06:42.330 | INFO | "malloc" : " 14 GB", | 20260505 20:06:42.330 | INFO | "pinned" : " 29 GB" | 20260505 20:06:42.330 | INFO | } | 20260505 20:06:42.330 | INFO | Perf report for the last 5.14424 seconds: | 20260505 20:06:42.330 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:06:42.330 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:42.330 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:42.330 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:42.330 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260505 20:06:42.330 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:42.330 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:42.330 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:06:42.330 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:42.330 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:42.330 | INFO | Uploaded 36GiB to GPU in 2.13448s (16.866GiB/s) | 20260505 20:06:42.330 | INFO | Downloaded 34GiB to GPU in 2.04295s (16.6426GiB/s) | 20260505 20:06:47.465 | INFO | Analyzing chunk frames = [5948928,5949440) | 20260505 20:06:47.465 | INFO | MemInfo: { | 20260505 20:06:47.465 | INFO | "gpu" : " 17 GB", | 20260505 20:06:47.465 | INFO | "huge" : "", | 20260505 20:06:47.465 | INFO | "huge_shared" : " 27 GB", | 20260505 20:06:47.465 | INFO | "malloc" : " 14 GB", | 20260505 20:06:47.465 | INFO | "pinned" : " 29 GB" | 20260505 20:06:47.465 | INFO | } | 20260505 20:06:47.465 | INFO | Perf report for the last 5.13496 seconds: | 20260505 20:06:47.465 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:06:47.465 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:47.465 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:47.465 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:47.465 | INFO | ANALYSIS : 62.4%, 3.0, 96.9ms, 0 | 20260505 20:06:47.465 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:47.465 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:47.465 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 20:06:47.465 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:47.465 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:47.465 | INFO | Uploaded 36GiB to GPU in 2.20164s (16.3514GiB/s) | 20260505 20:06:47.465 | INFO | Downloaded 34GiB to GPU in 2.05045s (16.5817GiB/s) | 20260505 20:06:52.607 | INFO | Analyzing chunk frames = [5949440,5949952) | 20260505 20:06:52.607 | INFO | MemInfo: { | 20260505 20:06:52.607 | INFO | "gpu" : " 17 GB", | 20260505 20:06:52.607 | INFO | "huge" : "", | 20260505 20:06:52.607 | INFO | "huge_shared" : " 27 GB", | 20260505 20:06:52.607 | INFO | "malloc" : " 14 GB", | 20260505 20:06:52.607 | INFO | "pinned" : " 29 GB" | 20260505 20:06:52.607 | INFO | } | 20260505 20:06:52.607 | INFO | Perf report for the last 5.14201 seconds: | 20260505 20:06:52.607 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:06:52.607 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:52.607 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:52.607 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:52.607 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 20:06:52.607 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:52.607 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:52.607 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 20:06:52.607 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:52.607 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:52.607 | INFO | Uploaded 36GiB to GPU in 2.11937s (16.9862GiB/s) | 20260505 20:06:52.607 | INFO | Downloaded 34GiB to GPU in 2.06148s (16.493GiB/s) | 20260505 20:06:57.751 | INFO | Analyzing chunk frames = [5949952,5950464) | 20260505 20:06:57.751 | INFO | MemInfo: { | 20260505 20:06:57.751 | INFO | "gpu" : " 17 GB", | 20260505 20:06:57.751 | INFO | "huge" : "", | 20260505 20:06:57.751 | INFO | "huge_shared" : " 27 GB", | 20260505 20:06:57.751 | INFO | "malloc" : " 14 GB", | 20260505 20:06:57.751 | INFO | "pinned" : " 29 GB" | 20260505 20:06:57.751 | INFO | } | 20260505 20:06:57.751 | INFO | Perf report for the last 5.14371 seconds: | 20260505 20:06:57.751 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:06:57.751 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:57.751 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:57.751 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:57.751 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260505 20:06:57.751 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:57.751 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:57.751 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:06:57.751 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:57.751 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:06:57.751 | INFO | Uploaded 36GiB to GPU in 2.10203s (17.1263GiB/s) | 20260505 20:06:57.751 | INFO | Downloaded 34GiB to GPU in 2.03347s (16.7202GiB/s) | 20260505 20:07:02.881 | INFO | Analyzing chunk frames = [5950464,5950976) | 20260505 20:07:02.881 | INFO | MemInfo: { | 20260505 20:07:02.881 | INFO | "gpu" : " 17 GB", | 20260505 20:07:02.881 | INFO | "huge" : "", | 20260505 20:07:02.881 | INFO | "huge_shared" : " 27 GB", | 20260505 20:07:02.881 | INFO | "malloc" : " 14 GB", | 20260505 20:07:02.881 | INFO | "pinned" : " 29 GB" | 20260505 20:07:02.881 | INFO | } | 20260505 20:07:02.881 | INFO | Perf report for the last 5.13022 seconds: | 20260505 20:07:02.881 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:07:02.881 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:02.881 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:02.881 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:02.881 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 20:07:02.881 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:02.881 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:02.881 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 20:07:02.881 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:02.881 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:02.881 | INFO | Uploaded 36GiB to GPU in 2.11719s (17.0037GiB/s) | 20260505 20:07:02.881 | INFO | Downloaded 34GiB to GPU in 2.15422s (15.783GiB/s) | 20260505 20:07:08.018 | INFO | Analyzing chunk frames = [5950976,5951488) | 20260505 20:07:08.018 | INFO | MemInfo: { | 20260505 20:07:08.018 | INFO | "gpu" : " 17 GB", | 20260505 20:07:08.018 | INFO | "huge" : "", | 20260505 20:07:08.018 | INFO | "huge_shared" : " 27 GB", | 20260505 20:07:08.018 | INFO | "malloc" : " 14 GB", | 20260505 20:07:08.018 | INFO | "pinned" : " 29 GB" | 20260505 20:07:08.018 | INFO | } | 20260505 20:07:08.018 | INFO | Perf report for the last 5.13707 seconds: | 20260505 20:07:08.018 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:07:08.018 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:08.018 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:08.018 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:08.018 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 20:07:08.018 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:08.018 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:08.018 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 20:07:08.018 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:08.018 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:08.018 | INFO | Uploaded 36GiB to GPU in 2.19761s (16.3815GiB/s) | 20260505 20:07:08.018 | INFO | Downloaded 34GiB to GPU in 2.03887s (16.6759GiB/s) | 20260505 20:07:08.150 | INFO | Starting a round of Baz IO | 20260505 20:07:11.319 | INFO | Populating RTMetrics report | 20260505 20:07:11.320 | INFO | RTMetrics report populated | 20260505 20:07:11.320 | INFO | Opening tmp RTMetrics json file | 20260505 20:07:11.320 | INFO | Serializing RTMetrics to json | 20260505 20:07:11.321 | INFO | Serialization done, swapping tmp file now... | 20260505 20:07:11.322 | INFO | RTMetrics done | 20260505 20:07:11.412 | INFO | Finished a round of Baz IO: Wrote 2.024 GiB in 3.261 seconds | 20260505 20:07:11.412 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.70% / 0.00% / 78.30% | 20260505 20:07:11.412 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.18609% / 0% / 7.15256e-05% | 20260505 20:07:11.412 | INFO | Baz IO overall: | 20260505 20:07:11.412 | INFO | BytesWritten : 3474171230682 | 20260505 20:07:11.412 | INFO | NumEvents : 1642914261905 | 20260505 20:07:13.163 | INFO | Analyzing chunk frames = [5951488,5952000) | 20260505 20:07:13.163 | INFO | MemInfo: { | 20260505 20:07:13.163 | INFO | "gpu" : " 17 GB", | 20260505 20:07:13.163 | INFO | "huge" : "", | 20260505 20:07:13.163 | INFO | "huge_shared" : " 27 GB", | 20260505 20:07:13.163 | INFO | "malloc" : " 14 GB", | 20260505 20:07:13.163 | INFO | "pinned" : " 29 GB" | 20260505 20:07:13.163 | INFO | } | 20260505 20:07:13.163 | INFO | Perf report for the last 5.1452 seconds: | 20260505 20:07:13.163 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:07:13.163 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:13.163 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:13.163 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:13.163 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 20:07:13.163 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:13.163 | INFO | RT_METRICS : 44.1%, 1.0, 23.2ms, 0 | 20260505 20:07:13.163 | INFO | PRE_HQ : 60.3%, 1.8, 55.3ms, 0 | 20260505 20:07:13.163 | INFO | BAZWRITER : 63.0%, 3.7, 119.2ms, 0 | 20260505 20:07:13.163 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 20:07:13.164 | INFO | Uploaded 36GiB to GPU in 2.16024s (16.6648GiB/s) | 20260505 20:07:13.164 | INFO | Downloaded 38GiB to GPU in 2.25392s (16.8595GiB/s) | 20260505 20:07:18.307 | INFO | Analyzing chunk frames = [5952000,5952512) | 20260505 20:07:18.307 | INFO | MemInfo: { | 20260505 20:07:18.307 | INFO | "gpu" : " 17 GB", | 20260505 20:07:18.307 | INFO | "huge" : "", | 20260505 20:07:18.307 | INFO | "huge_shared" : " 27 GB", | 20260505 20:07:18.307 | INFO | "malloc" : " 14 GB", | 20260505 20:07:18.307 | INFO | "pinned" : " 29 GB" | 20260505 20:07:18.308 | INFO | } | 20260505 20:07:18.308 | INFO | Perf report for the last 5.14424 seconds: | 20260505 20:07:18.308 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:07:18.308 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:18.308 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:18.308 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:18.308 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 20:07:18.308 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:18.308 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:18.308 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:07:18.308 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:18.308 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:18.308 | INFO | Uploaded 36GiB to GPU in 2.07554s (17.3449GiB/s) | 20260505 20:07:18.308 | INFO | Downloaded 34GiB to GPU in 2.02446s (16.7946GiB/s) | 20260505 20:07:23.444 | INFO | Analyzing chunk frames = [5952512,5953024) | 20260505 20:07:23.444 | INFO | MemInfo: { | 20260505 20:07:23.444 | INFO | "gpu" : " 17 GB", | 20260505 20:07:23.444 | INFO | "huge" : "", | 20260505 20:07:23.445 | INFO | "huge_shared" : " 27 GB", | 20260505 20:07:23.445 | INFO | "malloc" : " 14 GB", | 20260505 20:07:23.445 | INFO | "pinned" : " 29 GB" | 20260505 20:07:23.445 | INFO | } | 20260505 20:07:23.445 | INFO | Perf report for the last 5.13704 seconds: | 20260505 20:07:23.445 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:07:23.445 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:23.445 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:23.445 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:23.445 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260505 20:07:23.445 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:23.445 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:23.445 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 20:07:23.445 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:23.445 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:23.445 | INFO | Uploaded 36GiB to GPU in 2.13021s (16.8997GiB/s) | 20260505 20:07:23.445 | INFO | Downloaded 34GiB to GPU in 2.02806s (16.7648GiB/s) | 20260505 20:07:28.578 | INFO | Analyzing chunk frames = [5953024,5953536) | 20260505 20:07:28.578 | INFO | MemInfo: { | 20260505 20:07:28.578 | INFO | "gpu" : " 17 GB", | 20260505 20:07:28.578 | INFO | "huge" : "", | 20260505 20:07:28.578 | INFO | "huge_shared" : " 27 GB", | 20260505 20:07:28.578 | INFO | "malloc" : " 14 GB", | 20260505 20:07:28.578 | INFO | "pinned" : " 29 GB" | 20260505 20:07:28.578 | INFO | } | 20260505 20:07:28.578 | INFO | Perf report for the last 5.13355 seconds: | 20260505 20:07:28.578 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:07:28.578 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:28.578 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:28.578 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:28.578 | INFO | ANALYSIS : 62.5%, 3.0, 97.0ms, 0 | 20260505 20:07:28.578 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:28.578 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:28.578 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 20:07:28.578 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:28.578 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:28.578 | INFO | Uploaded 36GiB to GPU in 2.20606s (16.3187GiB/s) | 20260505 20:07:28.578 | INFO | Downloaded 34GiB to GPU in 2.03903s (16.6746GiB/s) | 20260505 20:07:33.721 | INFO | Analyzing chunk frames = [5953536,5954048) | 20260505 20:07:33.721 | INFO | MemInfo: { | 20260505 20:07:33.721 | INFO | "gpu" : " 17 GB", | 20260505 20:07:33.721 | INFO | "huge" : "", | 20260505 20:07:33.721 | INFO | "huge_shared" : " 27 GB", | 20260505 20:07:33.721 | INFO | "malloc" : " 14 GB", | 20260505 20:07:33.721 | INFO | "pinned" : " 29 GB" | 20260505 20:07:33.721 | INFO | } | 20260505 20:07:33.721 | INFO | Perf report for the last 5.14306 seconds: | 20260505 20:07:33.721 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:07:33.721 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:33.721 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:33.721 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:33.721 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 20:07:33.721 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:33.721 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:33.721 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 20:07:33.721 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:33.721 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:33.721 | INFO | Uploaded 36GiB to GPU in 2.12717s (16.9239GiB/s) | 20260505 20:07:33.721 | INFO | Downloaded 34GiB to GPU in 2.0625s (16.4849GiB/s) | 20260505 20:07:38.859 | INFO | Analyzing chunk frames = [5954048,5954560) | 20260505 20:07:38.859 | INFO | MemInfo: { | 20260505 20:07:38.859 | INFO | "gpu" : " 17 GB", | 20260505 20:07:38.859 | INFO | "huge" : "", | 20260505 20:07:38.859 | INFO | "huge_shared" : " 27 GB", | 20260505 20:07:38.859 | INFO | "malloc" : " 14 GB", | 20260505 20:07:38.859 | INFO | "pinned" : " 29 GB" | 20260505 20:07:38.859 | INFO | } | 20260505 20:07:38.859 | INFO | Perf report for the last 5.13805 seconds: | 20260505 20:07:38.859 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:07:38.859 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:38.859 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:38.859 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:38.859 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 20:07:38.859 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:38.859 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:38.859 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:07:38.859 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:38.859 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:38.860 | INFO | Uploaded 36GiB to GPU in 2.14288s (16.7999GiB/s) | 20260505 20:07:38.860 | INFO | Downloaded 34GiB to GPU in 2.0587s (16.5153GiB/s) | 20260505 20:07:44.002 | INFO | Analyzing chunk frames = [5954560,5955072) | 20260505 20:07:44.002 | INFO | MemInfo: { | 20260505 20:07:44.002 | INFO | "gpu" : " 17 GB", | 20260505 20:07:44.002 | INFO | "huge" : "", | 20260505 20:07:44.002 | INFO | "huge_shared" : " 27 GB", | 20260505 20:07:44.002 | INFO | "malloc" : " 14 GB", | 20260505 20:07:44.002 | INFO | "pinned" : " 29 GB" | 20260505 20:07:44.002 | INFO | } | 20260505 20:07:44.002 | INFO | Perf report for the last 5.14285 seconds: | 20260505 20:07:44.002 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:07:44.002 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:44.002 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:44.002 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:44.002 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 20:07:44.002 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:44.002 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:44.002 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 20:07:44.002 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:44.002 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:44.002 | INFO | Uploaded 36GiB to GPU in 2.10579s (17.0958GiB/s) | 20260505 20:07:44.002 | INFO | Downloaded 34GiB to GPU in 2.14835s (15.8261GiB/s) | 20260505 20:07:49.135 | INFO | Analyzing chunk frames = [5955072,5955584) | 20260505 20:07:49.135 | INFO | MemInfo: { | 20260505 20:07:49.135 | INFO | "gpu" : " 17 GB", | 20260505 20:07:49.135 | INFO | "huge" : "", | 20260505 20:07:49.135 | INFO | "huge_shared" : " 27 GB", | 20260505 20:07:49.135 | INFO | "malloc" : " 14 GB", | 20260505 20:07:49.135 | INFO | "pinned" : " 29 GB" | 20260505 20:07:49.135 | INFO | } | 20260505 20:07:49.135 | INFO | Perf report for the last 5.13311 seconds: | 20260505 20:07:49.135 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:07:49.135 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:49.135 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:49.135 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:49.135 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260505 20:07:49.135 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:49.135 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:49.135 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 20:07:49.135 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:49.135 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:49.136 | INFO | Uploaded 36GiB to GPU in 2.09792s (17.1598GiB/s) | 20260505 20:07:49.136 | INFO | Downloaded 34GiB to GPU in 2.00228s (16.9806GiB/s) | 20260505 20:07:49.136 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 20:07:49.136 | INFO | Total mem usage: 15.3 GiB | 20260505 20:07:49.136 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 20:07:49.136 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.38%) | 20260505 20:07:49.136 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 20:07:49.136 | INFO | High Water Breakdown: | 20260505 20:07:49.136 | INFO | 61.1% : 8.64 GiB ( 90.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 20:07:49.136 | INFO | 38.8% : 5.48 GiB ( 82.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 20:07:49.136 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 20:07:49.136 | INFO | ------------------------------------------ | 20260505 20:07:49.136 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:07:49.136 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 20:07:49.136 | INFO | Total mem usage: 26.5 GiB | 20260505 20:07:49.136 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 20:07:49.136 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 20:07:49.136 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 20:07:49.136 | INFO | High Water Breakdown: | 20260505 20:07:49.136 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:07:49.136 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:07:49.136 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:07:49.136 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:07:49.136 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:07:49.136 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:07:49.136 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 20:07:49.136 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 20:07:49.136 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:07:49.136 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 20:07:49.136 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 20:07:49.136 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 20:07:49.136 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 20:07:49.136 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 20:07:49.136 | INFO | ------------------------------------------ | 20260505 20:07:49.136 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:07:49.136 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 20:07:49.136 | INFO | Total mem usage: 25.6 GiB | 20260505 20:07:49.136 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 20:07:49.136 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 20:07:49.136 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 20:07:49.136 | INFO | High Water Breakdown: | 20260505 20:07:49.136 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 20:07:49.136 | INFO | ------------------------------------------ | 20260505 20:07:49.136 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:07:49.136 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 20:07:49.136 | INFO | Total mem usage: 15.7 GiB | 20260505 20:07:49.136 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 20:07:49.136 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 20:07:49.136 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 20:07:49.136 | INFO | High Water Breakdown: | 20260505 20:07:49.136 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 20:07:49.136 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 20:07:49.136 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 20:07:49.136 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 20:07:49.136 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 20:07:49.136 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 20:07:49.136 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 20:07:49.136 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 20:07:49.136 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 20:07:49.136 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:07:49.136 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 20:07:49.136 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:07:49.137 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 20:07:49.137 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 20:07:49.137 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 20:07:49.137 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 20:07:49.137 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:07:49.137 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 20:07:49.137 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:07:49.137 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:07:49.137 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:07:49.137 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 20:07:49.137 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 20:07:49.137 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:07:49.137 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 20:07:49.137 | INFO | ------------------------------------------ | 20260505 20:07:49.137 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:07:49.137 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 20:07:49.137 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:07:49.137 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:07:49.137 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.44e+01, 1.35e+01 / 1.65e+00 | 20260505 20:07:49.137 | INFO | PerfReport: 0.29%, AggregateStats -- 9.26e-02 / 1.04e-01, 1.06e-02 / 4.40e-03 | 20260505 20:07:49.137 | INFO | PerfReport: 11.80%, Baseline -- 3.72e+00 / 4.58e+00, 5.74e-01 / 1.35e-01 | 20260505 20:07:49.137 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.54e+00, 1.92e-01 / 7.55e-02 | 20260505 20:07:49.137 | INFO | PerfReport: 15.94%, DME -- 4.02e+01 / 4.43e+01, 3.93e+00 / 1.66e+00 | 20260505 20:07:49.137 | INFO | PerfReport: 57.07%, FrameLabeling -- 1.80e+01 / 1.93e+01, 1.53e+00 / 1.03e+00 | 20260505 20:07:49.137 | INFO | PerfReport: 7.23%, PulseAccumulating -- 2.28e+00 / 2.85e+00, 4.37e-01 / 1.99e-01 | 20260505 20:07:49.137 | INFO | PerfReport: 3.26%, Metrics -- 1.03e+00 / 1.34e+00, 2.83e-01 / 1.28e-01 | 20260505 20:07:49.137 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.57e-01 / 4.15e-01, 6.27e-02 / 2.97e-02 | 20260505 20:07:49.137 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 20:07:49.137 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:07:49.137 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:07:49.137 | INFO | PerfReport: Full Execution -- 9.93e+01 / -nan, 2.53e+01 / 0.00e+00 | 20260505 20:07:49.137 | INFO | PerfReport: 22.40%, Upload -- 2.23e+01 / 0.00e+00, 7.67e+00 / 0.00e+00 | 20260505 20:07:49.137 | INFO | PerfReport: 21.82%, Download -- 2.17e+01 / 0.00e+00, 7.73e+00 / 0.00e+00 | 20260505 20:07:49.137 | INFO | PerfReport: 31.73%, Compute -- 3.15e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 20:07:49.137 | INFO | PerfReport: 2.23%, LockUpload -- 2.21e+00 / 0.00e+00, 7.93e+00 / 0.00e+00 | 20260505 20:07:49.137 | INFO | PerfReport: 1.41%, LockDownload -- 1.40e+00 / 0.00e+00, 3.02e+00 / 0.00e+00 | 20260505 20:07:49.137 | INFO | PerfReport: 20.39%, LockCompute -- 2.03e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 20:07:49.137 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.55e-02 / 0.00e+00 | 20260505 20:07:49.268 | INFO | Starting a round of Baz IO | 20260505 20:07:52.439 | INFO | Populating RTMetrics report | 20260505 20:07:52.440 | INFO | RTMetrics report populated | 20260505 20:07:52.440 | INFO | Opening tmp RTMetrics json file | 20260505 20:07:52.443 | INFO | Serializing RTMetrics to json | 20260505 20:07:52.444 | INFO | Serialization done, swapping tmp file now... | 20260505 20:07:52.445 | INFO | RTMetrics done | 20260505 20:07:52.529 | INFO | Finished a round of Baz IO: Wrote 2.021 GiB in 3.261 seconds | 20260505 20:07:52.529 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.70% / 0.00% / 78.30% | 20260505 20:07:52.529 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.184504% / 0% / 7.94729e-05% | 20260505 20:07:52.529 | INFO | Baz IO overall: | 20260505 20:07:52.529 | INFO | BytesWritten : 3476341535980 | 20260505 20:07:52.530 | INFO | NumEvents : 1643919565588 | 20260505 20:07:54.275 | INFO | Analyzing chunk frames = [5955584,5956096) | 20260505 20:07:54.275 | INFO | MemInfo: { | 20260505 20:07:54.275 | INFO | "gpu" : " 17 GB", | 20260505 20:07:54.275 | INFO | "huge" : "", | 20260505 20:07:54.275 | INFO | "huge_shared" : " 27 GB", | 20260505 20:07:54.275 | INFO | "malloc" : " 14 GB", | 20260505 20:07:54.275 | INFO | "pinned" : " 29 GB" | 20260505 20:07:54.275 | INFO | } | 20260505 20:07:54.275 | INFO | Perf report for the last 5.13973 seconds: | 20260505 20:07:54.275 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:07:54.275 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:54.275 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:54.275 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:54.275 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 20:07:54.275 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:54.275 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260505 20:07:54.275 | INFO | PRE_HQ : 60.1%, 1.8, 55.0ms, 0 | 20260505 20:07:54.275 | INFO | BAZWRITER : 63.0%, 3.7, 119.4ms, 0 | 20260505 20:07:54.275 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 20:07:54.275 | INFO | Uploaded 36GiB to GPU in 2.17059s (16.5853GiB/s) | 20260505 20:07:54.275 | INFO | Downloaded 38GiB to GPU in 2.2537s (16.8612GiB/s) | 20260505 20:07:59.417 | INFO | Analyzing chunk frames = [5956096,5956608) | 20260505 20:07:59.417 | INFO | MemInfo: { | 20260505 20:07:59.417 | INFO | "gpu" : " 17 GB", | 20260505 20:07:59.417 | INFO | "huge" : "", | 20260505 20:07:59.417 | INFO | "huge_shared" : " 27 GB", | 20260505 20:07:59.417 | INFO | "malloc" : " 14 GB", | 20260505 20:07:59.417 | INFO | "pinned" : " 29 GB" | 20260505 20:07:59.417 | INFO | } | 20260505 20:07:59.417 | INFO | Perf report for the last 5.14215 seconds: | 20260505 20:07:59.417 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:07:59.417 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:59.417 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:59.417 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:59.417 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260505 20:07:59.417 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:59.417 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:59.417 | INFO | PRE_HQ : 27.2%, 1.0, 14.2ms, 0 | 20260505 20:07:59.417 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:59.417 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:07:59.418 | INFO | Uploaded 36GiB to GPU in 2.15036s (16.7414GiB/s) | 20260505 20:07:59.418 | INFO | Downloaded 34GiB to GPU in 2.04863s (16.5964GiB/s) | 20260505 20:08:04.566 | INFO | Analyzing chunk frames = [5956608,5957120) | 20260505 20:08:04.566 | INFO | MemInfo: { | 20260505 20:08:04.566 | INFO | "gpu" : " 17 GB", | 20260505 20:08:04.566 | INFO | "huge" : "", | 20260505 20:08:04.566 | INFO | "huge_shared" : " 27 GB", | 20260505 20:08:04.566 | INFO | "malloc" : " 14 GB", | 20260505 20:08:04.566 | INFO | "pinned" : " 29 GB" | 20260505 20:08:04.566 | INFO | } | 20260505 20:08:04.566 | INFO | Perf report for the last 5.1488 seconds: | 20260505 20:08:04.566 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:08:04.566 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:04.566 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:04.566 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:04.566 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260505 20:08:04.566 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:04.566 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:04.566 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:08:04.566 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:04.566 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:04.566 | INFO | Uploaded 36GiB to GPU in 2.12939s (16.9062GiB/s) | 20260505 20:08:04.566 | INFO | Downloaded 34GiB to GPU in 2.01662s (16.8599GiB/s) | 20260505 20:08:09.698 | INFO | Analyzing chunk frames = [5957120,5957632) | 20260505 20:08:09.698 | INFO | MemInfo: { | 20260505 20:08:09.698 | INFO | "gpu" : " 17 GB", | 20260505 20:08:09.698 | INFO | "huge" : "", | 20260505 20:08:09.698 | INFO | "huge_shared" : " 27 GB", | 20260505 20:08:09.698 | INFO | "malloc" : " 14 GB", | 20260505 20:08:09.698 | INFO | "pinned" : " 29 GB" | 20260505 20:08:09.698 | INFO | } | 20260505 20:08:09.698 | INFO | Perf report for the last 5.13202 seconds: | 20260505 20:08:09.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:08:09.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:09.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:09.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:09.698 | INFO | ANALYSIS : 62.1%, 3.0, 96.3ms, 0 | 20260505 20:08:09.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:09.698 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:09.698 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:08:09.698 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:09.698 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:09.698 | INFO | Uploaded 36GiB to GPU in 2.13447s (16.866GiB/s) | 20260505 20:08:09.698 | INFO | Downloaded 34GiB to GPU in 2.03183s (16.7337GiB/s) | 20260505 20:08:14.825 | INFO | Analyzing chunk frames = [5957632,5958144) | 20260505 20:08:14.825 | INFO | MemInfo: { | 20260505 20:08:14.825 | INFO | "gpu" : " 17 GB", | 20260505 20:08:14.825 | INFO | "huge" : "", | 20260505 20:08:14.825 | INFO | "huge_shared" : " 27 GB", | 20260505 20:08:14.825 | INFO | "malloc" : " 14 GB", | 20260505 20:08:14.825 | INFO | "pinned" : " 29 GB" | 20260505 20:08:14.825 | INFO | } | 20260505 20:08:14.825 | INFO | Perf report for the last 5.12675 seconds: | 20260505 20:08:14.825 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:08:14.825 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:14.825 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:14.825 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:14.825 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260505 20:08:14.825 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:14.825 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:14.825 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:08:14.825 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:14.825 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:14.825 | INFO | Uploaded 36GiB to GPU in 2.13006s (16.9009GiB/s) | 20260505 20:08:14.825 | INFO | Downloaded 34GiB to GPU in 2.06218s (16.4874GiB/s) | 20260505 20:08:19.973 | INFO | Analyzing chunk frames = [5958144,5958656) | 20260505 20:08:19.973 | INFO | MemInfo: { | 20260505 20:08:19.973 | INFO | "gpu" : " 17 GB", | 20260505 20:08:19.973 | INFO | "huge" : "", | 20260505 20:08:19.973 | INFO | "huge_shared" : " 27 GB", | 20260505 20:08:19.973 | INFO | "malloc" : " 14 GB", | 20260505 20:08:19.973 | INFO | "pinned" : " 29 GB" | 20260505 20:08:19.973 | INFO | } | 20260505 20:08:19.973 | INFO | Perf report for the last 5.14831 seconds: | 20260505 20:08:19.973 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:08:19.973 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:19.973 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:19.973 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:19.973 | INFO | ANALYSIS : 62.3%, 3.0, 97.4ms, 0 | 20260505 20:08:19.973 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:19.973 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:19.973 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:08:19.973 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:19.973 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:19.973 | INFO | Uploaded 36GiB to GPU in 2.15125s (16.7345GiB/s) | 20260505 20:08:19.973 | INFO | Downloaded 34GiB to GPU in 2.04948s (16.5896GiB/s) | 20260505 20:08:25.116 | INFO | Analyzing chunk frames = [5958656,5959168) | 20260505 20:08:25.116 | INFO | MemInfo: { | 20260505 20:08:25.116 | INFO | "gpu" : " 17 GB", | 20260505 20:08:25.116 | INFO | "huge" : "", | 20260505 20:08:25.116 | INFO | "huge_shared" : " 27 GB", | 20260505 20:08:25.116 | INFO | "malloc" : " 14 GB", | 20260505 20:08:25.116 | INFO | "pinned" : " 29 GB" | 20260505 20:08:25.116 | INFO | } | 20260505 20:08:25.116 | INFO | Perf report for the last 5.14324 seconds: | 20260505 20:08:25.116 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:08:25.116 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:25.116 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:25.116 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:25.116 | INFO | ANALYSIS : 62.9%, 3.0, 98.4ms, 0 | 20260505 20:08:25.116 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:25.116 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:25.117 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 20:08:25.117 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:25.117 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:25.117 | INFO | Uploaded 36GiB to GPU in 2.10791s (17.0785GiB/s) | 20260505 20:08:25.117 | INFO | Downloaded 34GiB to GPU in 2.13884s (15.8965GiB/s) | 20260505 20:08:30.258 | INFO | Analyzing chunk frames = [5959168,5959680) | 20260505 20:08:30.258 | INFO | MemInfo: { | 20260505 20:08:30.258 | INFO | "gpu" : " 17 GB", | 20260505 20:08:30.258 | INFO | "huge" : "", | 20260505 20:08:30.258 | INFO | "huge_shared" : " 27 GB", | 20260505 20:08:30.258 | INFO | "malloc" : " 14 GB", | 20260505 20:08:30.258 | INFO | "pinned" : " 29 GB" | 20260505 20:08:30.258 | INFO | } | 20260505 20:08:30.259 | INFO | Perf report for the last 5.14216 seconds: | 20260505 20:08:30.259 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:08:30.259 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:30.259 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:30.259 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:30.259 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 20:08:30.259 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:30.259 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:30.259 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:08:30.259 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:30.259 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:30.259 | INFO | Uploaded 36GiB to GPU in 2.11248s (17.0416GiB/s) | 20260505 20:08:30.259 | INFO | Downloaded 34GiB to GPU in 2.00814s (16.9311GiB/s) | 20260505 20:08:30.395 | INFO | Starting a round of Baz IO | 20260505 20:08:33.549 | INFO | Populating RTMetrics report | 20260505 20:08:33.549 | INFO | RTMetrics report populated | 20260505 20:08:33.549 | INFO | Opening tmp RTMetrics json file | 20260505 20:08:33.549 | INFO | Serializing RTMetrics to json | 20260505 20:08:33.551 | INFO | Serialization done, swapping tmp file now... | 20260505 20:08:33.551 | INFO | RTMetrics done | 20260505 20:08:33.644 | INFO | Finished a round of Baz IO: Wrote 2.023 GiB in 3.248 seconds | 20260505 20:08:33.644 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.70% / 0.00% / 78.30% | 20260505 20:08:33.644 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.185156% / 0% / 0.000107288% | 20260505 20:08:33.644 | INFO | Baz IO overall: | 20260505 20:08:33.644 | INFO | BytesWritten : 3478513374082 | 20260505 20:08:33.644 | INFO | NumEvents : 1644926355593 | 20260505 20:08:35.390 | INFO | Analyzing chunk frames = [5959680,5960192) | 20260505 20:08:35.390 | INFO | MemInfo: { | 20260505 20:08:35.390 | INFO | "gpu" : " 17 GB", | 20260505 20:08:35.390 | INFO | "huge" : "", | 20260505 20:08:35.390 | INFO | "huge_shared" : " 27 GB", | 20260505 20:08:35.390 | INFO | "malloc" : " 14 GB", | 20260505 20:08:35.390 | INFO | "pinned" : " 29 GB" | 20260505 20:08:35.390 | INFO | } | 20260505 20:08:35.391 | INFO | Perf report for the last 5.13194 seconds: | 20260505 20:08:35.391 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:08:35.391 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:35.391 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:35.391 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:35.391 | INFO | ANALYSIS : 63.0%, 3.0, 98.1ms, 0 | 20260505 20:08:35.391 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:35.391 | INFO | RT_METRICS : 44.2%, 1.0, 23.1ms, 0 | 20260505 20:08:35.391 | INFO | PRE_HQ : 60.1%, 1.8, 55.6ms, 0 | 20260505 20:08:35.391 | INFO | BAZWRITER : 62.9%, 3.7, 119.6ms, 0 | 20260505 20:08:35.391 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 20:08:35.391 | INFO | Uploaded 36GiB to GPU in 2.17185s (16.5757GiB/s) | 20260505 20:08:35.391 | INFO | Downloaded 38GiB to GPU in 2.2526s (16.8694GiB/s) | 20260505 20:08:40.530 | INFO | Analyzing chunk frames = [5960192,5960704) | 20260505 20:08:40.530 | INFO | MemInfo: { | 20260505 20:08:40.530 | INFO | "gpu" : " 17 GB", | 20260505 20:08:40.530 | INFO | "huge" : "", | 20260505 20:08:40.530 | INFO | "huge_shared" : " 27 GB", | 20260505 20:08:40.530 | INFO | "malloc" : " 14 GB", | 20260505 20:08:40.530 | INFO | "pinned" : " 29 GB" | 20260505 20:08:40.530 | INFO | } | 20260505 20:08:40.530 | INFO | Perf report for the last 5.13945 seconds: | 20260505 20:08:40.530 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:08:40.530 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:40.530 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:40.530 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:40.531 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 20:08:40.531 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:40.531 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:40.531 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 20:08:40.531 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:40.531 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:40.531 | INFO | Uploaded 36GiB to GPU in 2.13789s (16.839GiB/s) | 20260505 20:08:40.531 | INFO | Downloaded 34GiB to GPU in 2.03396s (16.7161GiB/s) | 20260505 20:08:45.676 | INFO | Analyzing chunk frames = [5960704,5961216) | 20260505 20:08:45.676 | INFO | MemInfo: { | 20260505 20:08:45.676 | INFO | "gpu" : " 17 GB", | 20260505 20:08:45.676 | INFO | "huge" : "", | 20260505 20:08:45.676 | INFO | "huge_shared" : " 27 GB", | 20260505 20:08:45.676 | INFO | "malloc" : " 14 GB", | 20260505 20:08:45.676 | INFO | "pinned" : " 29 GB" | 20260505 20:08:45.676 | INFO | } | 20260505 20:08:45.676 | INFO | Perf report for the last 5.14613 seconds: | 20260505 20:08:45.676 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:08:45.676 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:45.676 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:45.676 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:45.676 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260505 20:08:45.676 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:45.676 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:45.676 | INFO | PRE_HQ : 27.2%, 1.0, 14.2ms, 0 | 20260505 20:08:45.676 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:45.676 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:45.676 | INFO | Uploaded 36GiB to GPU in 2.1407s (16.817GiB/s) | 20260505 20:08:45.676 | INFO | Downloaded 34GiB to GPU in 2.03867s (16.6775GiB/s) | 20260505 20:08:50.808 | INFO | Analyzing chunk frames = [5961216,5961728) | 20260505 20:08:50.808 | INFO | MemInfo: { | 20260505 20:08:50.808 | INFO | "gpu" : " 17 GB", | 20260505 20:08:50.808 | INFO | "huge" : "", | 20260505 20:08:50.809 | INFO | "huge_shared" : " 27 GB", | 20260505 20:08:50.809 | INFO | "malloc" : " 14 GB", | 20260505 20:08:50.809 | INFO | "pinned" : " 29 GB" | 20260505 20:08:50.809 | INFO | } | 20260505 20:08:50.809 | INFO | Perf report for the last 5.13251 seconds: | 20260505 20:08:50.809 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:08:50.809 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:50.809 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:50.809 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:50.809 | INFO | ANALYSIS : 62.5%, 3.0, 97.0ms, 0 | 20260505 20:08:50.809 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:50.809 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:50.809 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:08:50.809 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:50.809 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:50.809 | INFO | Uploaded 36GiB to GPU in 2.21201s (16.2748GiB/s) | 20260505 20:08:50.809 | INFO | Downloaded 34GiB to GPU in 2.06024s (16.5029GiB/s) | 20260505 20:08:55.950 | INFO | Analyzing chunk frames = [5961728,5962240) | 20260505 20:08:55.950 | INFO | MemInfo: { | 20260505 20:08:55.950 | INFO | "gpu" : " 17 GB", | 20260505 20:08:55.950 | INFO | "huge" : "", | 20260505 20:08:55.950 | INFO | "huge_shared" : " 27 GB", | 20260505 20:08:55.950 | INFO | "malloc" : " 14 GB", | 20260505 20:08:55.950 | INFO | "pinned" : " 29 GB" | 20260505 20:08:55.950 | INFO | } | 20260505 20:08:55.950 | INFO | Perf report for the last 5.14163 seconds: | 20260505 20:08:55.950 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:08:55.950 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:55.950 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:55.950 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:55.950 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 20:08:55.950 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:55.950 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:55.950 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 20:08:55.950 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:55.950 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:08:55.951 | INFO | Uploaded 36GiB to GPU in 2.13375s (16.8717GiB/s) | 20260505 20:08:55.951 | INFO | Downloaded 34GiB to GPU in 2.06213s (16.4878GiB/s) | 20260505 20:09:01.086 | INFO | Analyzing chunk frames = [5962240,5962752) | 20260505 20:09:01.086 | INFO | MemInfo: { | 20260505 20:09:01.086 | INFO | "gpu" : " 17 GB", | 20260505 20:09:01.086 | INFO | "huge" : "", | 20260505 20:09:01.086 | INFO | "huge_shared" : " 27 GB", | 20260505 20:09:01.086 | INFO | "malloc" : " 14 GB", | 20260505 20:09:01.086 | INFO | "pinned" : " 29 GB" | 20260505 20:09:01.086 | INFO | } | 20260505 20:09:01.086 | INFO | Perf report for the last 5.13594 seconds: | 20260505 20:09:01.086 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:09:01.086 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:01.086 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:01.086 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:01.086 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260505 20:09:01.086 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:01.086 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:01.086 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 20:09:01.086 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:01.086 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:01.087 | INFO | Uploaded 36GiB to GPU in 2.15036s (16.7414GiB/s) | 20260505 20:09:01.087 | INFO | Downloaded 34GiB to GPU in 2.05824s (16.519GiB/s) | 20260505 20:09:06.229 | INFO | Analyzing chunk frames = [5962752,5963264) | 20260505 20:09:06.229 | INFO | MemInfo: { | 20260505 20:09:06.229 | INFO | "gpu" : " 17 GB", | 20260505 20:09:06.229 | INFO | "huge" : "", | 20260505 20:09:06.229 | INFO | "huge_shared" : " 27 GB", | 20260505 20:09:06.229 | INFO | "malloc" : " 14 GB", | 20260505 20:09:06.229 | INFO | "pinned" : " 29 GB" | 20260505 20:09:06.229 | INFO | } | 20260505 20:09:06.229 | INFO | Perf report for the last 5.14263 seconds: | 20260505 20:09:06.229 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:09:06.229 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:06.229 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:06.229 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:06.229 | INFO | ANALYSIS : 63.1%, 3.0, 98.7ms, 0 | 20260505 20:09:06.229 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:06.229 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:06.229 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 20:09:06.229 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:06.229 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:06.229 | INFO | Uploaded 36GiB to GPU in 2.13443s (16.8663GiB/s) | 20260505 20:09:06.229 | INFO | Downloaded 34GiB to GPU in 2.13423s (15.9308GiB/s) | 20260505 20:09:11.369 | INFO | Analyzing chunk frames = [5963264,5963776) | 20260505 20:09:11.369 | INFO | MemInfo: { | 20260505 20:09:11.369 | INFO | "gpu" : " 17 GB", | 20260505 20:09:11.369 | INFO | "huge" : "", | 20260505 20:09:11.369 | INFO | "huge_shared" : " 27 GB", | 20260505 20:09:11.369 | INFO | "malloc" : " 14 GB", | 20260505 20:09:11.369 | INFO | "pinned" : " 29 GB" | 20260505 20:09:11.369 | INFO | } | 20260505 20:09:11.369 | INFO | Perf report for the last 5.13996 seconds: | 20260505 20:09:11.369 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:09:11.369 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:11.369 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:11.369 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:11.369 | INFO | ANALYSIS : 62.3%, 3.0, 95.5ms, 0 | 20260505 20:09:11.369 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:11.369 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:11.369 | INFO | PRE_HQ : 27.9%, 1.0, 14.3ms, 0 | 20260505 20:09:11.369 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:11.369 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:11.369 | INFO | Uploaded 36GiB to GPU in 2.12305s (16.9568GiB/s) | 20260505 20:09:11.369 | INFO | Downloaded 34GiB to GPU in 2.01757s (16.852GiB/s) | 20260505 20:09:11.369 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 20:09:11.369 | INFO | Total mem usage: 15.3 GiB | 20260505 20:09:11.369 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 20:09:11.369 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.12%) | 20260505 20:09:11.369 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 20:09:11.369 | INFO | High Water Breakdown: | 20260505 20:09:11.369 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 20:09:11.369 | INFO | 38.8% : 5.51 GiB ( 82.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 20:09:11.369 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 20:09:11.369 | INFO | ------------------------------------------ | 20260505 20:09:11.369 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:09:11.369 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 20:09:11.369 | INFO | Total mem usage: 26.5 GiB | 20260505 20:09:11.369 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 20:09:11.369 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 20:09:11.369 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 20:09:11.370 | INFO | High Water Breakdown: | 20260505 20:09:11.370 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:09:11.370 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:09:11.370 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:09:11.370 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:09:11.370 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:09:11.370 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:09:11.370 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 20:09:11.370 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 20:09:11.370 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:09:11.370 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 20:09:11.370 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 20:09:11.370 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 20:09:11.370 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 20:09:11.370 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 20:09:11.370 | INFO | ------------------------------------------ | 20260505 20:09:11.370 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:09:11.370 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 20:09:11.370 | INFO | Total mem usage: 25.6 GiB | 20260505 20:09:11.370 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 20:09:11.370 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 20:09:11.370 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 20:09:11.370 | INFO | High Water Breakdown: | 20260505 20:09:11.370 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 20:09:11.370 | INFO | ------------------------------------------ | 20260505 20:09:11.370 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:09:11.370 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 20:09:11.370 | INFO | Total mem usage: 15.7 GiB | 20260505 20:09:11.370 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 20:09:11.370 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 20:09:11.370 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 20:09:11.370 | INFO | High Water Breakdown: | 20260505 20:09:11.370 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 20:09:11.370 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 20:09:11.370 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 20:09:11.370 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 20:09:11.370 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 20:09:11.370 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 20:09:11.370 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 20:09:11.370 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 20:09:11.370 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 20:09:11.370 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:09:11.370 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 20:09:11.370 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:09:11.370 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 20:09:11.370 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 20:09:11.370 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 20:09:11.370 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 20:09:11.370 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:09:11.370 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 20:09:11.370 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:09:11.370 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:09:11.370 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:09:11.370 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 20:09:11.370 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 20:09:11.370 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:09:11.370 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 20:09:11.370 | INFO | ------------------------------------------ | 20260505 20:09:11.370 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:09:11.370 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 20:09:11.370 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:09:11.370 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:09:11.370 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.46e+01, 1.36e+01 / 2.09e+00 | 20260505 20:09:11.370 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.05e-01, 1.06e-02 / 8.48e-03 | 20260505 20:09:11.370 | INFO | PerfReport: 11.79%, Baseline -- 3.71e+00 / 4.53e+00, 5.69e-01 / 1.40e-01 | 20260505 20:09:11.370 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.53e+00, 1.93e-01 / 8.79e-02 | 20260505 20:09:11.370 | INFO | PerfReport: 15.96%, DME -- 4.02e+01 / 4.43e+01, 4.12e+00 / 2.15e+00 | 20260505 20:09:11.370 | INFO | PerfReport: 57.08%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.49e+00 / 1.08e+00 | 20260505 20:09:11.370 | INFO | PerfReport: 7.21%, PulseAccumulating -- 2.27e+00 / 2.84e+00, 4.35e-01 / 2.23e-01 | 20260505 20:09:11.370 | INFO | PerfReport: 3.25%, Metrics -- 1.02e+00 / 1.35e+00, 2.83e-01 / 1.47e-01 | 20260505 20:09:11.370 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.57e-01 / 4.22e-01, 6.38e-02 / 3.64e-02 | 20260505 20:09:11.370 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 20:09:11.370 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:09:11.370 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:09:11.370 | INFO | PerfReport: Full Execution -- 9.93e+01 / -nan, 2.54e+01 / 0.00e+00 | 20260505 20:09:11.370 | INFO | PerfReport: 22.47%, Upload -- 2.23e+01 / 0.00e+00, 7.74e+00 / 0.00e+00 | 20260505 20:09:11.370 | INFO | PerfReport: 21.80%, Download -- 2.17e+01 / 0.00e+00, 7.70e+00 / 0.00e+00 | 20260505 20:09:11.370 | INFO | PerfReport: 31.70%, Compute -- 3.15e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 20:09:11.370 | INFO | PerfReport: 2.29%, LockUpload -- 2.27e+00 / 0.00e+00, 8.07e+00 / 0.00e+00 | 20260505 20:09:11.370 | INFO | PerfReport: 1.36%, LockDownload -- 1.35e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260505 20:09:11.370 | INFO | PerfReport: 20.36%, LockCompute -- 2.02e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 20:09:11.370 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.70e-02 / 0.00e+00 | 20260505 20:09:11.503 | INFO | Starting a round of Baz IO | 20260505 20:09:14.670 | INFO | Populating RTMetrics report | 20260505 20:09:14.670 | INFO | RTMetrics report populated | 20260505 20:09:14.670 | INFO | Opening tmp RTMetrics json file | 20260505 20:09:14.673 | INFO | Serializing RTMetrics to json | 20260505 20:09:14.678 | INFO | Serialization done, swapping tmp file now... | 20260505 20:09:14.678 | INFO | RTMetrics done | 20260505 20:09:14.759 | INFO | Finished a round of Baz IO: Wrote 2.021 GiB in 3.255 seconds | 20260505 20:09:14.759 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.70% / 0.00% / 78.30% | 20260505 20:09:14.759 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.183972% / 0% / 0.000127157% | 20260505 20:09:14.759 | INFO | Baz IO overall: | 20260505 20:09:14.759 | INFO | BytesWritten : 3480683718582 | 20260505 20:09:14.759 | INFO | NumEvents : 1645932214878 | 20260505 20:09:16.509 | INFO | Analyzing chunk frames = [5963776,5964288) | 20260505 20:09:16.509 | INFO | MemInfo: { | 20260505 20:09:16.509 | INFO | "gpu" : " 17 GB", | 20260505 20:09:16.509 | INFO | "huge" : "", | 20260505 20:09:16.509 | INFO | "huge_shared" : " 27 GB", | 20260505 20:09:16.509 | INFO | "malloc" : " 14 GB", | 20260505 20:09:16.509 | INFO | "pinned" : " 29 GB" | 20260505 20:09:16.509 | INFO | } | 20260505 20:09:16.509 | INFO | Perf report for the last 5.14026 seconds: | 20260505 20:09:16.509 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:09:16.509 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:16.509 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:16.509 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:16.509 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 20:09:16.509 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:16.509 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260505 20:09:16.509 | INFO | PRE_HQ : 60.1%, 1.8, 55.8ms, 0 | 20260505 20:09:16.509 | INFO | BAZWRITER : 62.9%, 3.7, 121.0ms, 0 | 20260505 20:09:16.509 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 20:09:16.510 | INFO | Uploaded 36GiB to GPU in 2.16365s (16.6385GiB/s) | 20260505 20:09:16.510 | INFO | Downloaded 38GiB to GPU in 2.25284s (16.8676GiB/s) | 20260505 20:09:21.647 | INFO | Analyzing chunk frames = [5964288,5964800) | 20260505 20:09:21.647 | INFO | MemInfo: { | 20260505 20:09:21.647 | INFO | "gpu" : " 17 GB", | 20260505 20:09:21.647 | INFO | "huge" : "", | 20260505 20:09:21.647 | INFO | "huge_shared" : " 27 GB", | 20260505 20:09:21.647 | INFO | "malloc" : " 14 GB", | 20260505 20:09:21.647 | INFO | "pinned" : " 29 GB" | 20260505 20:09:21.647 | INFO | } | 20260505 20:09:21.647 | INFO | Perf report for the last 5.1381 seconds: | 20260505 20:09:21.647 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:09:21.647 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:21.647 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:21.647 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:21.647 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 20:09:21.647 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:21.647 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:21.647 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 20:09:21.647 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:21.647 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:21.648 | INFO | Uploaded 36GiB to GPU in 2.08609s (17.2571GiB/s) | 20260505 20:09:21.648 | INFO | Downloaded 34GiB to GPU in 2.0283s (16.7628GiB/s) | 20260505 20:09:26.778 | INFO | Analyzing chunk frames = [5964800,5965312) | 20260505 20:09:26.778 | INFO | MemInfo: { | 20260505 20:09:26.778 | INFO | "gpu" : " 17 GB", | 20260505 20:09:26.778 | INFO | "huge" : "", | 20260505 20:09:26.778 | INFO | "huge_shared" : " 27 GB", | 20260505 20:09:26.778 | INFO | "malloc" : " 14 GB", | 20260505 20:09:26.778 | INFO | "pinned" : " 29 GB" | 20260505 20:09:26.778 | INFO | } | 20260505 20:09:26.778 | INFO | Perf report for the last 5.13074 seconds: | 20260505 20:09:26.778 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:09:26.778 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:26.778 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:26.778 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:26.778 | INFO | ANALYSIS : 62.4%, 3.0, 96.9ms, 0 | 20260505 20:09:26.778 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:26.778 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:26.778 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260505 20:09:26.778 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:26.778 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:26.778 | INFO | Uploaded 36GiB to GPU in 2.1336s (16.8729GiB/s) | 20260505 20:09:26.778 | INFO | Downloaded 34GiB to GPU in 2.03096s (16.7409GiB/s) | 20260505 20:09:31.929 | INFO | Analyzing chunk frames = [5965312,5965824) | 20260505 20:09:31.929 | INFO | MemInfo: { | 20260505 20:09:31.929 | INFO | "gpu" : " 17 GB", | 20260505 20:09:31.929 | INFO | "huge" : "", | 20260505 20:09:31.929 | INFO | "huge_shared" : " 27 GB", | 20260505 20:09:31.929 | INFO | "malloc" : " 14 GB", | 20260505 20:09:31.929 | INFO | "pinned" : " 29 GB" | 20260505 20:09:31.929 | INFO | } | 20260505 20:09:31.929 | INFO | Perf report for the last 5.15109 seconds: | 20260505 20:09:31.929 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:09:31.929 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:31.929 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:31.929 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:31.929 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260505 20:09:31.929 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:31.929 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:31.929 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 20:09:31.929 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:31.929 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:31.929 | INFO | Uploaded 36GiB to GPU in 2.16211s (16.6504GiB/s) | 20260505 20:09:31.929 | INFO | Downloaded 34GiB to GPU in 2.05688s (16.5299GiB/s) | 20260505 20:09:37.070 | INFO | Analyzing chunk frames = [5965824,5966336) | 20260505 20:09:37.070 | INFO | MemInfo: { | 20260505 20:09:37.070 | INFO | "gpu" : " 17 GB", | 20260505 20:09:37.070 | INFO | "huge" : "", | 20260505 20:09:37.070 | INFO | "huge_shared" : " 27 GB", | 20260505 20:09:37.070 | INFO | "malloc" : " 14 GB", | 20260505 20:09:37.070 | INFO | "pinned" : " 29 GB" | 20260505 20:09:37.070 | INFO | } | 20260505 20:09:37.070 | INFO | Perf report for the last 5.14117 seconds: | 20260505 20:09:37.070 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:09:37.070 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:37.070 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:37.070 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:37.070 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260505 20:09:37.070 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:37.070 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:37.070 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:09:37.070 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:37.070 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:37.071 | INFO | Uploaded 36GiB to GPU in 2.13073s (16.8956GiB/s) | 20260505 20:09:37.071 | INFO | Downloaded 34GiB to GPU in 2.0598s (16.5064GiB/s) | 20260505 20:09:42.206 | INFO | Analyzing chunk frames = [5966336,5966848) | 20260505 20:09:42.206 | INFO | MemInfo: { | 20260505 20:09:42.206 | INFO | "gpu" : " 17 GB", | 20260505 20:09:42.206 | INFO | "huge" : "", | 20260505 20:09:42.206 | INFO | "huge_shared" : " 27 GB", | 20260505 20:09:42.206 | INFO | "malloc" : " 14 GB", | 20260505 20:09:42.206 | INFO | "pinned" : " 29 GB" | 20260505 20:09:42.206 | INFO | } | 20260505 20:09:42.206 | INFO | Perf report for the last 5.13615 seconds: | 20260505 20:09:42.206 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:09:42.206 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:42.206 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:42.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:42.207 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 20:09:42.207 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:42.207 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:42.207 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:09:42.207 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:42.207 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:42.207 | INFO | Uploaded 36GiB to GPU in 2.14487s (16.7842GiB/s) | 20260505 20:09:42.207 | INFO | Downloaded 34GiB to GPU in 2.05984s (16.5062GiB/s) | 20260505 20:09:47.341 | INFO | Analyzing chunk frames = [5966848,5967360) | 20260505 20:09:47.341 | INFO | MemInfo: { | 20260505 20:09:47.341 | INFO | "gpu" : " 17 GB", | 20260505 20:09:47.341 | INFO | "huge" : "", | 20260505 20:09:47.341 | INFO | "huge_shared" : " 27 GB", | 20260505 20:09:47.341 | INFO | "malloc" : " 14 GB", | 20260505 20:09:47.341 | INFO | "pinned" : " 29 GB" | 20260505 20:09:47.341 | INFO | } | 20260505 20:09:47.342 | INFO | Perf report for the last 5.13509 seconds: | 20260505 20:09:47.342 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:09:47.342 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:47.342 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:47.342 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:47.342 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 20:09:47.342 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:47.342 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:47.342 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 20:09:47.342 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:47.342 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:47.342 | INFO | Uploaded 36GiB to GPU in 2.12508s (16.9406GiB/s) | 20260505 20:09:47.342 | INFO | Downloaded 34GiB to GPU in 2.143s (15.8656GiB/s) | 20260505 20:09:52.482 | INFO | Analyzing chunk frames = [5967360,5967872) | 20260505 20:09:52.482 | INFO | MemInfo: { | 20260505 20:09:52.482 | INFO | "gpu" : " 17 GB", | 20260505 20:09:52.482 | INFO | "huge" : "", | 20260505 20:09:52.482 | INFO | "huge_shared" : " 27 GB", | 20260505 20:09:52.482 | INFO | "malloc" : " 14 GB", | 20260505 20:09:52.482 | INFO | "pinned" : " 29 GB" | 20260505 20:09:52.482 | INFO | } | 20260505 20:09:52.482 | INFO | Perf report for the last 5.14079 seconds: | 20260505 20:09:52.482 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:09:52.483 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:52.483 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:52.483 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:52.483 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 20:09:52.483 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:52.483 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:52.483 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:09:52.483 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:52.483 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:52.483 | INFO | Uploaded 36GiB to GPU in 2.18976s (16.4401GiB/s) | 20260505 20:09:52.483 | INFO | Downloaded 34GiB to GPU in 2.02782s (16.7668GiB/s) | 20260505 20:09:52.612 | INFO | Starting a round of Baz IO | 20260505 20:09:55.784 | INFO | Populating RTMetrics report | 20260505 20:09:55.784 | INFO | RTMetrics report populated | 20260505 20:09:55.784 | INFO | Opening tmp RTMetrics json file | 20260505 20:09:55.785 | INFO | Serializing RTMetrics to json | 20260505 20:09:55.786 | INFO | Serialization done, swapping tmp file now... | 20260505 20:09:55.787 | INFO | RTMetrics done | 20260505 20:09:55.871 | INFO | Finished a round of Baz IO: Wrote 2.022 GiB in 3.258 seconds | 20260505 20:09:55.871 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.70% / 0.00% / 78.30% | 20260505 20:09:55.871 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.18301% / 3.97364e-06% / 7.94729e-05% | 20260505 20:09:55.871 | INFO | Baz IO overall: | 20260505 20:09:55.871 | INFO | BytesWritten : 3482855073616 | 20260505 20:09:55.871 | INFO | NumEvents : 1646938782452 | 20260505 20:09:57.622 | INFO | Analyzing chunk frames = [5967872,5968384) | 20260505 20:09:57.622 | INFO | MemInfo: { | 20260505 20:09:57.622 | INFO | "gpu" : " 17 GB", | 20260505 20:09:57.622 | INFO | "huge" : "", | 20260505 20:09:57.622 | INFO | "huge_shared" : " 27 GB", | 20260505 20:09:57.622 | INFO | "malloc" : " 14 GB", | 20260505 20:09:57.622 | INFO | "pinned" : " 29 GB" | 20260505 20:09:57.622 | INFO | } | 20260505 20:09:57.622 | INFO | Perf report for the last 5.14004 seconds: | 20260505 20:09:57.622 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:09:57.622 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:57.622 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:57.622 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:57.622 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 20:09:57.623 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:09:57.623 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260505 20:09:57.623 | INFO | PRE_HQ : 60.2%, 1.8, 55.2ms, 0 | 20260505 20:09:57.623 | INFO | BAZWRITER : 62.9%, 3.7, 120.1ms, 0 | 20260505 20:09:57.623 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 20:09:57.623 | INFO | Uploaded 36GiB to GPU in 2.15446s (16.7095GiB/s) | 20260505 20:09:57.623 | INFO | Downloaded 38GiB to GPU in 2.25281s (16.8678GiB/s) | 20260505 20:10:02.762 | INFO | Analyzing chunk frames = [5968384,5968896) | 20260505 20:10:02.762 | INFO | MemInfo: { | 20260505 20:10:02.762 | INFO | "gpu" : " 17 GB", | 20260505 20:10:02.762 | INFO | "huge" : "", | 20260505 20:10:02.762 | INFO | "huge_shared" : " 27 GB", | 20260505 20:10:02.762 | INFO | "malloc" : " 14 GB", | 20260505 20:10:02.762 | INFO | "pinned" : " 29 GB" | 20260505 20:10:02.762 | INFO | } | 20260505 20:10:02.762 | INFO | Perf report for the last 5.13997 seconds: | 20260505 20:10:02.762 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:10:02.762 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:02.762 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:02.762 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:02.762 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 20:10:02.762 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:02.762 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:02.763 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:10:02.763 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:02.763 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:02.763 | INFO | Uploaded 36GiB to GPU in 2.09627s (17.1734GiB/s) | 20260505 20:10:02.763 | INFO | Downloaded 34GiB to GPU in 2.02364s (16.8014GiB/s) | 20260505 20:10:07.900 | INFO | Analyzing chunk frames = [5968896,5969408) | 20260505 20:10:07.900 | INFO | MemInfo: { | 20260505 20:10:07.900 | INFO | "gpu" : " 17 GB", | 20260505 20:10:07.900 | INFO | "huge" : "", | 20260505 20:10:07.900 | INFO | "huge_shared" : " 27 GB", | 20260505 20:10:07.901 | INFO | "malloc" : " 14 GB", | 20260505 20:10:07.901 | INFO | "pinned" : " 29 GB" | 20260505 20:10:07.901 | INFO | } | 20260505 20:10:07.901 | INFO | Perf report for the last 5.13827 seconds: | 20260505 20:10:07.901 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:10:07.901 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:07.901 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:07.901 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:07.901 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260505 20:10:07.901 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:07.901 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:07.901 | INFO | PRE_HQ : 27.5%, 1.0, 14.2ms, 0 | 20260505 20:10:07.901 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:07.901 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:07.901 | INFO | Uploaded 36GiB to GPU in 2.11986s (16.9823GiB/s) | 20260505 20:10:07.901 | INFO | Downloaded 34GiB to GPU in 2.0113s (16.9045GiB/s) | 20260505 20:10:13.038 | INFO | Analyzing chunk frames = [5969408,5969920) | 20260505 20:10:13.039 | INFO | MemInfo: { | 20260505 20:10:13.039 | INFO | "gpu" : " 17 GB", | 20260505 20:10:13.039 | INFO | "huge" : "", | 20260505 20:10:13.039 | INFO | "huge_shared" : " 27 GB", | 20260505 20:10:13.039 | INFO | "malloc" : " 14 GB", | 20260505 20:10:13.039 | INFO | "pinned" : " 29 GB" | 20260505 20:10:13.039 | INFO | } | 20260505 20:10:13.039 | INFO | Perf report for the last 5.13811 seconds: | 20260505 20:10:13.039 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:10:13.039 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:13.039 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:13.039 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:13.039 | INFO | ANALYSIS : 62.3%, 3.0, 96.7ms, 0 | 20260505 20:10:13.039 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:13.039 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:13.039 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:10:13.039 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:13.039 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:13.039 | INFO | Uploaded 36GiB to GPU in 2.19554s (16.3969GiB/s) | 20260505 20:10:13.039 | INFO | Downloaded 34GiB to GPU in 2.06863s (16.436GiB/s) | 20260505 20:10:18.184 | INFO | Analyzing chunk frames = [5969920,5970432) | 20260505 20:10:18.184 | INFO | MemInfo: { | 20260505 20:10:18.184 | INFO | "gpu" : " 17 GB", | 20260505 20:10:18.184 | INFO | "huge" : "", | 20260505 20:10:18.184 | INFO | "huge_shared" : " 27 GB", | 20260505 20:10:18.184 | INFO | "malloc" : " 14 GB", | 20260505 20:10:18.184 | INFO | "pinned" : " 29 GB" | 20260505 20:10:18.184 | INFO | } | 20260505 20:10:18.184 | INFO | Perf report for the last 5.14505 seconds: | 20260505 20:10:18.184 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:10:18.184 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:18.184 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:18.184 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:18.184 | INFO | ANALYSIS : 61.6%, 3.0, 96.3ms, 0 | 20260505 20:10:18.184 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:18.184 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:18.184 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 20:10:18.184 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:18.184 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:18.184 | INFO | Uploaded 36GiB to GPU in 2.13147s (16.8897GiB/s) | 20260505 20:10:18.184 | INFO | Downloaded 34GiB to GPU in 2.05911s (16.512GiB/s) | 20260505 20:10:23.328 | INFO | Analyzing chunk frames = [5970432,5970944) | 20260505 20:10:23.328 | INFO | MemInfo: { | 20260505 20:10:23.328 | INFO | "gpu" : " 17 GB", | 20260505 20:10:23.328 | INFO | "huge" : "", | 20260505 20:10:23.328 | INFO | "huge_shared" : " 27 GB", | 20260505 20:10:23.328 | INFO | "malloc" : " 14 GB", | 20260505 20:10:23.328 | INFO | "pinned" : " 29 GB" | 20260505 20:10:23.328 | INFO | } | 20260505 20:10:23.328 | INFO | Perf report for the last 5.144 seconds: | 20260505 20:10:23.328 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:10:23.328 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:23.328 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:23.328 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:23.328 | INFO | ANALYSIS : 61.8%, 3.0, 96.6ms, 0 | 20260505 20:10:23.328 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:23.328 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:23.328 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:10:23.328 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:23.328 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:23.328 | INFO | Uploaded 36GiB to GPU in 2.09279s (17.2019GiB/s) | 20260505 20:10:23.328 | INFO | Downloaded 34GiB to GPU in 2.03611s (16.6985GiB/s) | 20260505 20:10:28.457 | INFO | Analyzing chunk frames = [5970944,5971456) | 20260505 20:10:28.457 | INFO | MemInfo: { | 20260505 20:10:28.457 | INFO | "gpu" : " 17 GB", | 20260505 20:10:28.457 | INFO | "huge" : "", | 20260505 20:10:28.457 | INFO | "huge_shared" : " 27 GB", | 20260505 20:10:28.457 | INFO | "malloc" : " 14 GB", | 20260505 20:10:28.457 | INFO | "pinned" : " 29 GB" | 20260505 20:10:28.457 | INFO | } | 20260505 20:10:28.458 | INFO | Perf report for the last 5.12964 seconds: | 20260505 20:10:28.458 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:10:28.458 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:28.458 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:28.458 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:28.458 | INFO | ANALYSIS : 62.8%, 3.0, 97.8ms, 0 | 20260505 20:10:28.458 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:28.458 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:28.458 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 20:10:28.458 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:28.458 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:28.458 | INFO | Uploaded 36GiB to GPU in 2.1192s (16.9876GiB/s) | 20260505 20:10:28.458 | INFO | Downloaded 34GiB to GPU in 2.08272s (16.3248GiB/s) | 20260505 20:10:33.594 | INFO | Analyzing chunk frames = [5971456,5971968) | 20260505 20:10:33.594 | INFO | MemInfo: { | 20260505 20:10:33.594 | INFO | "gpu" : " 17 GB", | 20260505 20:10:33.594 | INFO | "huge" : "", | 20260505 20:10:33.594 | INFO | "huge_shared" : " 27 GB", | 20260505 20:10:33.594 | INFO | "malloc" : " 14 GB", | 20260505 20:10:33.594 | INFO | "pinned" : " 29 GB" | 20260505 20:10:33.594 | INFO | } | 20260505 20:10:33.594 | INFO | Perf report for the last 5.13675 seconds: | 20260505 20:10:33.594 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:10:33.594 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:33.594 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:33.594 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:33.594 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 20:10:33.594 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:33.594 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:33.594 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 20:10:33.594 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:33.594 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:33.595 | INFO | Uploaded 36GiB to GPU in 2.20632s (16.3167GiB/s) | 20260505 20:10:33.595 | INFO | Downloaded 34GiB to GPU in 2.03813s (16.682GiB/s) | 20260505 20:10:33.595 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 20:10:33.595 | INFO | Total mem usage: 15.3 GiB | 20260505 20:10:33.595 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 20:10:33.595 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.28%) | 20260505 20:10:33.595 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 20:10:33.595 | INFO | High Water Breakdown: | 20260505 20:10:33.595 | INFO | 61% : 8.64 GiB ( 90.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 20:10:33.595 | INFO | 38.8% : 5.5 GiB ( 82.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 20:10:33.595 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 20:10:33.595 | INFO | ------------------------------------------ | 20260505 20:10:33.595 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:10:33.595 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 20:10:33.595 | INFO | Total mem usage: 26.5 GiB | 20260505 20:10:33.595 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 20:10:33.595 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 20:10:33.595 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 20:10:33.595 | INFO | High Water Breakdown: | 20260505 20:10:33.595 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:10:33.595 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:10:33.595 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:10:33.595 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:10:33.595 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:10:33.595 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:10:33.595 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 20:10:33.595 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 20:10:33.595 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:10:33.595 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 20:10:33.595 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 20:10:33.595 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 20:10:33.595 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 20:10:33.595 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 20:10:33.595 | INFO | ------------------------------------------ | 20260505 20:10:33.595 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:10:33.595 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 20:10:33.595 | INFO | Total mem usage: 25.6 GiB | 20260505 20:10:33.595 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 20:10:33.595 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 20:10:33.595 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 20:10:33.595 | INFO | High Water Breakdown: | 20260505 20:10:33.595 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 20:10:33.595 | INFO | ------------------------------------------ | 20260505 20:10:33.595 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:10:33.595 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 20:10:33.595 | INFO | Total mem usage: 15.7 GiB | 20260505 20:10:33.595 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 20:10:33.596 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 20:10:33.596 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 20:10:33.596 | INFO | High Water Breakdown: | 20260505 20:10:33.596 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 20:10:33.596 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 20:10:33.596 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 20:10:33.596 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 20:10:33.596 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 20:10:33.596 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 20:10:33.596 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 20:10:33.596 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 20:10:33.596 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 20:10:33.596 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:10:33.596 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 20:10:33.596 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:10:33.596 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 20:10:33.596 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 20:10:33.596 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 20:10:33.596 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 20:10:33.596 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:10:33.596 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 20:10:33.596 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:10:33.596 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:10:33.596 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:10:33.596 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 20:10:33.596 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 20:10:33.596 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:10:33.596 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 20:10:33.596 | INFO | ------------------------------------------ | 20260505 20:10:33.596 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:10:33.596 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 20:10:33.596 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:10:33.596 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:10:33.596 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.45e+01, 1.33e+01 / 1.85e+00 | 20260505 20:10:33.596 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 1.05e-01, 1.06e-02 / 6.33e-03 | 20260505 20:10:33.596 | INFO | PerfReport: 11.81%, Baseline -- 3.72e+00 / 4.51e+00, 5.74e-01 / 1.65e-01 | 20260505 20:10:33.596 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.52e+00, 1.91e-01 / 9.62e-02 | 20260505 20:10:33.596 | INFO | PerfReport: 15.94%, DME -- 4.02e+01 / 4.43e+01, 4.02e+00 / 1.91e+00 | 20260505 20:10:33.596 | INFO | PerfReport: 57.10%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.56e+00 / 1.08e+00 | 20260505 20:10:33.596 | INFO | PerfReport: 7.20%, PulseAccumulating -- 2.27e+00 / 2.84e+00, 4.36e-01 / 1.96e-01 | 20260505 20:10:33.596 | INFO | PerfReport: 3.25%, Metrics -- 1.02e+00 / 1.37e+00, 2.81e-01 / 1.31e-01 | 20260505 20:10:33.596 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.57e-01 / 4.19e-01, 6.33e-02 / 3.25e-02 | 20260505 20:10:33.596 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 20:10:33.596 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:10:33.596 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:10:33.596 | INFO | PerfReport: Full Execution -- 9.93e+01 / -nan, 2.55e+01 / 0.00e+00 | 20260505 20:10:33.597 | INFO | PerfReport: 22.45%, Upload -- 2.23e+01 / 0.00e+00, 7.77e+00 / 0.00e+00 | 20260505 20:10:33.597 | INFO | PerfReport: 21.78%, Download -- 2.16e+01 / 0.00e+00, 7.73e+00 / 0.00e+00 | 20260505 20:10:33.597 | INFO | PerfReport: 31.74%, Compute -- 3.15e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260505 20:10:33.597 | INFO | PerfReport: 2.21%, LockUpload -- 2.20e+00 / 0.00e+00, 7.75e+00 / 0.00e+00 | 20260505 20:10:33.597 | INFO | PerfReport: 1.36%, LockDownload -- 1.35e+00 / 0.00e+00, 2.92e+00 / 0.00e+00 | 20260505 20:10:33.597 | INFO | PerfReport: 20.45%, LockCompute -- 2.03e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 20:10:33.597 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 2.04e-02 / 0.00e+00 | 20260505 20:10:33.729 | INFO | Starting a round of Baz IO | 20260505 20:10:36.899 | INFO | Populating RTMetrics report | 20260505 20:10:36.900 | INFO | RTMetrics report populated | 20260505 20:10:36.900 | INFO | Opening tmp RTMetrics json file | 20260505 20:10:36.900 | INFO | Serializing RTMetrics to json | 20260505 20:10:36.902 | INFO | Serialization done, swapping tmp file now... | 20260505 20:10:36.902 | INFO | RTMetrics done | 20260505 20:10:36.990 | INFO | Finished a round of Baz IO: Wrote 2.023 GiB in 3.261 seconds | 20260505 20:10:36.990 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.70% / 0.00% / 78.30% | 20260505 20:10:36.990 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.184222% / 3.97364e-06% / 8.34465e-05% | 20260505 20:10:36.990 | INFO | Baz IO overall: | 20260505 20:10:36.991 | INFO | BytesWritten : 3485026942005 | 20260505 20:10:36.991 | INFO | NumEvents : 1647945893129 | 20260505 20:10:38.739 | INFO | Analyzing chunk frames = [5971968,5972480) | 20260505 20:10:38.739 | INFO | MemInfo: { | 20260505 20:10:38.739 | INFO | "gpu" : " 17 GB", | 20260505 20:10:38.739 | INFO | "huge" : "", | 20260505 20:10:38.739 | INFO | "huge_shared" : " 27 GB", | 20260505 20:10:38.739 | INFO | "malloc" : " 14 GB", | 20260505 20:10:38.739 | INFO | "pinned" : " 29 GB" | 20260505 20:10:38.739 | INFO | } | 20260505 20:10:38.739 | INFO | Perf report for the last 5.14498 seconds: | 20260505 20:10:38.739 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:10:38.739 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:38.739 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:38.739 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:38.739 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 20:10:38.739 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:38.739 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260505 20:10:38.739 | INFO | PRE_HQ : 60.0%, 1.8, 55.1ms, 0 | 20260505 20:10:38.739 | INFO | BAZWRITER : 63.0%, 3.7, 120.1ms, 0 | 20260505 20:10:38.739 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 20:10:38.740 | INFO | Uploaded 36GiB to GPU in 2.16358s (16.6391GiB/s) | 20260505 20:10:38.740 | INFO | Downloaded 38GiB to GPU in 2.24882s (16.8977GiB/s) | 20260505 20:10:43.878 | INFO | Analyzing chunk frames = [5972480,5972992) | 20260505 20:10:43.878 | INFO | MemInfo: { | 20260505 20:10:43.878 | INFO | "gpu" : " 17 GB", | 20260505 20:10:43.878 | INFO | "huge" : "", | 20260505 20:10:43.878 | INFO | "huge_shared" : " 27 GB", | 20260505 20:10:43.879 | INFO | "malloc" : " 14 GB", | 20260505 20:10:43.879 | INFO | "pinned" : " 29 GB" | 20260505 20:10:43.879 | INFO | } | 20260505 20:10:43.879 | INFO | Perf report for the last 5.13937 seconds: | 20260505 20:10:43.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:10:43.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:43.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:43.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:43.879 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 20:10:43.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:43.879 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:43.879 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 20:10:43.879 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:43.879 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:43.879 | INFO | Uploaded 36GiB to GPU in 2.10127s (17.1325GiB/s) | 20260505 20:10:43.879 | INFO | Downloaded 34GiB to GPU in 2.02635s (16.779GiB/s) | 20260505 20:10:49.018 | INFO | Analyzing chunk frames = [5972992,5973504) | 20260505 20:10:49.018 | INFO | MemInfo: { | 20260505 20:10:49.018 | INFO | "gpu" : " 17 GB", | 20260505 20:10:49.018 | INFO | "huge" : "", | 20260505 20:10:49.018 | INFO | "huge_shared" : " 27 GB", | 20260505 20:10:49.018 | INFO | "malloc" : " 14 GB", | 20260505 20:10:49.018 | INFO | "pinned" : " 29 GB" | 20260505 20:10:49.018 | INFO | } | 20260505 20:10:49.018 | INFO | Perf report for the last 5.1393 seconds: | 20260505 20:10:49.018 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:10:49.018 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:49.018 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:49.018 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:49.018 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260505 20:10:49.018 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:49.018 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:49.018 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:10:49.018 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:49.018 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:49.018 | INFO | Uploaded 36GiB to GPU in 2.13032s (16.8989GiB/s) | 20260505 20:10:49.018 | INFO | Downloaded 34GiB to GPU in 2.00924s (16.9218GiB/s) | 20260505 20:10:54.156 | INFO | Analyzing chunk frames = [5973504,5974016) | 20260505 20:10:54.156 | INFO | MemInfo: { | 20260505 20:10:54.156 | INFO | "gpu" : " 17 GB", | 20260505 20:10:54.156 | INFO | "huge" : "", | 20260505 20:10:54.156 | INFO | "huge_shared" : " 27 GB", | 20260505 20:10:54.156 | INFO | "malloc" : " 14 GB", | 20260505 20:10:54.156 | INFO | "pinned" : " 29 GB" | 20260505 20:10:54.156 | INFO | } | 20260505 20:10:54.156 | INFO | Perf report for the last 5.13811 seconds: | 20260505 20:10:54.156 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:10:54.156 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:54.156 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:54.156 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:54.156 | INFO | ANALYSIS : 62.5%, 3.0, 97.1ms, 0 | 20260505 20:10:54.156 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:54.156 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:54.156 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:10:54.156 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:54.156 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:54.156 | INFO | Uploaded 36GiB to GPU in 2.21138s (16.2794GiB/s) | 20260505 20:10:54.156 | INFO | Downloaded 34GiB to GPU in 2.04502s (16.6257GiB/s) | 20260505 20:10:59.291 | INFO | Analyzing chunk frames = [5974016,5974528) | 20260505 20:10:59.291 | INFO | MemInfo: { | 20260505 20:10:59.291 | INFO | "gpu" : " 17 GB", | 20260505 20:10:59.291 | INFO | "huge" : "", | 20260505 20:10:59.292 | INFO | "huge_shared" : " 27 GB", | 20260505 20:10:59.292 | INFO | "malloc" : " 14 GB", | 20260505 20:10:59.292 | INFO | "pinned" : " 29 GB" | 20260505 20:10:59.292 | INFO | } | 20260505 20:10:59.292 | INFO | Perf report for the last 5.13563 seconds: | 20260505 20:10:59.292 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:10:59.292 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:59.292 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:59.292 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:59.292 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260505 20:10:59.292 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:59.292 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:59.292 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:10:59.292 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:59.292 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:10:59.292 | INFO | Uploaded 36GiB to GPU in 2.13519s (16.8604GiB/s) | 20260505 20:10:59.292 | INFO | Downloaded 34GiB to GPU in 2.0603s (16.5024GiB/s) | 20260505 20:11:04.437 | INFO | Analyzing chunk frames = [5974528,5975040) | 20260505 20:11:04.437 | INFO | MemInfo: { | 20260505 20:11:04.437 | INFO | "gpu" : " 17 GB", | 20260505 20:11:04.437 | INFO | "huge" : "", | 20260505 20:11:04.437 | INFO | "huge_shared" : " 27 GB", | 20260505 20:11:04.437 | INFO | "malloc" : " 14 GB", | 20260505 20:11:04.437 | INFO | "pinned" : " 29 GB" | 20260505 20:11:04.437 | INFO | } | 20260505 20:11:04.437 | INFO | Perf report for the last 5.14556 seconds: | 20260505 20:11:04.437 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:11:04.437 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:04.437 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:04.437 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:04.437 | INFO | ANALYSIS : 62.2%, 3.0, 97.3ms, 0 | 20260505 20:11:04.437 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:04.437 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:04.437 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:11:04.437 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:04.437 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:04.438 | INFO | Uploaded 36GiB to GPU in 2.15703s (16.6896GiB/s) | 20260505 20:11:04.438 | INFO | Downloaded 34GiB to GPU in 2.05821s (16.5192GiB/s) | 20260505 20:11:09.574 | INFO | Analyzing chunk frames = [5975040,5975552) | 20260505 20:11:09.574 | INFO | MemInfo: { | 20260505 20:11:09.574 | INFO | "gpu" : " 17 GB", | 20260505 20:11:09.574 | INFO | "huge" : "", | 20260505 20:11:09.574 | INFO | "huge_shared" : " 27 GB", | 20260505 20:11:09.574 | INFO | "malloc" : " 14 GB", | 20260505 20:11:09.574 | INFO | "pinned" : " 29 GB" | 20260505 20:11:09.574 | INFO | } | 20260505 20:11:09.574 | INFO | Perf report for the last 5.13691 seconds: | 20260505 20:11:09.574 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:11:09.574 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:09.574 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:09.574 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:09.574 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 20:11:09.574 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:09.574 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:09.574 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 20:11:09.574 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:09.574 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:09.574 | INFO | Uploaded 36GiB to GPU in 2.12646s (16.9295GiB/s) | 20260505 20:11:09.575 | INFO | Downloaded 34GiB to GPU in 2.14165s (15.8756GiB/s) | 20260505 20:11:14.708 | INFO | Analyzing chunk frames = [5975552,5976064) | 20260505 20:11:14.708 | INFO | MemInfo: { | 20260505 20:11:14.708 | INFO | "gpu" : " 17 GB", | 20260505 20:11:14.708 | INFO | "huge" : "", | 20260505 20:11:14.708 | INFO | "huge_shared" : " 27 GB", | 20260505 20:11:14.708 | INFO | "malloc" : " 14 GB", | 20260505 20:11:14.708 | INFO | "pinned" : " 29 GB" | 20260505 20:11:14.708 | INFO | } | 20260505 20:11:14.708 | INFO | Perf report for the last 5.13425 seconds: | 20260505 20:11:14.708 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:11:14.708 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:14.708 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:14.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:14.709 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 20:11:14.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:14.709 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:14.709 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 20:11:14.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:14.709 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:14.709 | INFO | Uploaded 36GiB to GPU in 2.12758s (16.9206GiB/s) | 20260505 20:11:14.709 | INFO | Downloaded 34GiB to GPU in 2.02435s (16.7956GiB/s) | 20260505 20:11:14.843 | INFO | Starting a round of Baz IO | 20260505 20:11:18.013 | INFO | Populating RTMetrics report | 20260505 20:11:18.013 | INFO | RTMetrics report populated | 20260505 20:11:18.013 | INFO | Opening tmp RTMetrics json file | 20260505 20:11:18.014 | INFO | Serializing RTMetrics to json | 20260505 20:11:18.018 | INFO | Serialization done, swapping tmp file now... | 20260505 20:11:18.018 | INFO | RTMetrics done | 20260505 20:11:18.102 | INFO | Finished a round of Baz IO: Wrote 2.023 GiB in 3.258 seconds | 20260505 20:11:18.102 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.70% / 0.00% / 78.30% | 20260505 20:11:18.102 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.184059% / 3.97364e-06% / 8.34465e-05% | 20260505 20:11:18.102 | INFO | Baz IO overall: | 20260505 20:11:18.102 | INFO | BytesWritten : 3487198774806 | 20260505 20:11:18.102 | INFO | NumEvents : 1648953027113 | 20260505 20:11:19.850 | INFO | Analyzing chunk frames = [5976064,5976576) | 20260505 20:11:19.850 | INFO | MemInfo: { | 20260505 20:11:19.850 | INFO | "gpu" : " 17 GB", | 20260505 20:11:19.850 | INFO | "huge" : "", | 20260505 20:11:19.850 | INFO | "huge_shared" : " 27 GB", | 20260505 20:11:19.850 | INFO | "malloc" : " 14 GB", | 20260505 20:11:19.850 | INFO | "pinned" : " 29 GB" | 20260505 20:11:19.850 | INFO | } | 20260505 20:11:19.850 | INFO | Perf report for the last 5.14174 seconds: | 20260505 20:11:19.850 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:11:19.850 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:19.850 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:19.850 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:19.850 | INFO | ANALYSIS : 63.2%, 3.0, 97.6ms, 0 | 20260505 20:11:19.850 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:19.850 | INFO | RT_METRICS : 44.1%, 1.0, 22.9ms, 0 | 20260505 20:11:19.850 | INFO | PRE_HQ : 60.3%, 1.8, 55.0ms, 0 | 20260505 20:11:19.850 | INFO | BAZWRITER : 62.9%, 3.7, 117.7ms, 0 | 20260505 20:11:19.850 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 20:11:19.850 | INFO | Uploaded 36GiB to GPU in 2.16083s (16.6603GiB/s) | 20260505 20:11:19.851 | INFO | Downloaded 38GiB to GPU in 2.24627s (16.9169GiB/s) | 20260505 20:11:24.992 | INFO | Analyzing chunk frames = [5976576,5977088) | 20260505 20:11:24.992 | INFO | MemInfo: { | 20260505 20:11:24.992 | INFO | "gpu" : " 17 GB", | 20260505 20:11:24.992 | INFO | "huge" : "", | 20260505 20:11:24.992 | INFO | "huge_shared" : " 27 GB", | 20260505 20:11:24.992 | INFO | "malloc" : " 14 GB", | 20260505 20:11:24.992 | INFO | "pinned" : " 29 GB" | 20260505 20:11:24.992 | INFO | } | 20260505 20:11:24.992 | INFO | Perf report for the last 5.14211 seconds: | 20260505 20:11:24.992 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:11:24.992 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:24.992 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:24.992 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:24.992 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260505 20:11:24.992 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:24.992 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:24.992 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:11:24.993 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:24.993 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:24.993 | INFO | Uploaded 36GiB to GPU in 2.15408s (16.7125GiB/s) | 20260505 20:11:24.993 | INFO | Downloaded 34GiB to GPU in 2.04905s (16.593GiB/s) | 20260505 20:11:30.136 | INFO | Analyzing chunk frames = [5977088,5977600) | 20260505 20:11:30.136 | INFO | MemInfo: { | 20260505 20:11:30.136 | INFO | "gpu" : " 17 GB", | 20260505 20:11:30.136 | INFO | "huge" : "", | 20260505 20:11:30.136 | INFO | "huge_shared" : " 27 GB", | 20260505 20:11:30.136 | INFO | "malloc" : " 14 GB", | 20260505 20:11:30.136 | INFO | "pinned" : " 29 GB" | 20260505 20:11:30.136 | INFO | } | 20260505 20:11:30.136 | INFO | Perf report for the last 5.14353 seconds: | 20260505 20:11:30.136 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:11:30.136 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:30.136 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:30.136 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:30.136 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260505 20:11:30.136 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:30.136 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:30.136 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:11:30.136 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:30.136 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:30.136 | INFO | Uploaded 36GiB to GPU in 2.13515s (16.8607GiB/s) | 20260505 20:11:30.136 | INFO | Downloaded 34GiB to GPU in 2.0198s (16.8333GiB/s) | 20260505 20:11:35.273 | INFO | Analyzing chunk frames = [5977600,5978112) | 20260505 20:11:35.273 | INFO | MemInfo: { | 20260505 20:11:35.273 | INFO | "gpu" : " 17 GB", | 20260505 20:11:35.273 | INFO | "huge" : "", | 20260505 20:11:35.273 | INFO | "huge_shared" : " 27 GB", | 20260505 20:11:35.273 | INFO | "malloc" : " 14 GB", | 20260505 20:11:35.273 | INFO | "pinned" : " 29 GB" | 20260505 20:11:35.273 | INFO | } | 20260505 20:11:35.273 | INFO | Perf report for the last 5.13747 seconds: | 20260505 20:11:35.273 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:11:35.273 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:35.273 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:35.273 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:35.273 | INFO | ANALYSIS : 62.0%, 3.0, 96.3ms, 0 | 20260505 20:11:35.273 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:35.273 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:35.273 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:11:35.273 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:35.274 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:35.274 | INFO | Uploaded 36GiB to GPU in 2.13764s (16.841GiB/s) | 20260505 20:11:35.274 | INFO | Downloaded 34GiB to GPU in 2.03708s (16.6906GiB/s) | 20260505 20:11:40.402 | INFO | Analyzing chunk frames = [5978112,5978624) | 20260505 20:11:40.402 | INFO | MemInfo: { | 20260505 20:11:40.402 | INFO | "gpu" : " 17 GB", | 20260505 20:11:40.402 | INFO | "huge" : "", | 20260505 20:11:40.402 | INFO | "huge_shared" : " 27 GB", | 20260505 20:11:40.402 | INFO | "malloc" : " 14 GB", | 20260505 20:11:40.402 | INFO | "pinned" : " 29 GB" | 20260505 20:11:40.402 | INFO | } | 20260505 20:11:40.402 | INFO | Perf report for the last 5.1288 seconds: | 20260505 20:11:40.402 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:11:40.402 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:40.402 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:40.402 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:40.402 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260505 20:11:40.402 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:40.402 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:40.402 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:11:40.402 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:40.402 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:40.402 | INFO | Uploaded 36GiB to GPU in 2.13782s (16.8396GiB/s) | 20260505 20:11:40.402 | INFO | Downloaded 34GiB to GPU in 2.06048s (16.501GiB/s) | 20260505 20:11:45.544 | INFO | Analyzing chunk frames = [5978624,5979136) | 20260505 20:11:45.544 | INFO | MemInfo: { | 20260505 20:11:45.544 | INFO | "gpu" : " 17 GB", | 20260505 20:11:45.544 | INFO | "huge" : "", | 20260505 20:11:45.544 | INFO | "huge_shared" : " 27 GB", | 20260505 20:11:45.544 | INFO | "malloc" : " 14 GB", | 20260505 20:11:45.544 | INFO | "pinned" : " 29 GB" | 20260505 20:11:45.544 | INFO | } | 20260505 20:11:45.544 | INFO | Perf report for the last 5.14179 seconds: | 20260505 20:11:45.544 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:11:45.544 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:45.544 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:45.544 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:45.544 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 20:11:45.544 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:45.544 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:45.544 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 20:11:45.544 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:45.544 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:45.544 | INFO | Uploaded 36GiB to GPU in 2.14746s (16.764GiB/s) | 20260505 20:11:45.544 | INFO | Downloaded 34GiB to GPU in 2.05044s (16.5818GiB/s) | 20260505 20:11:50.692 | INFO | Analyzing chunk frames = [5979136,5979648) | 20260505 20:11:50.692 | INFO | MemInfo: { | 20260505 20:11:50.692 | INFO | "gpu" : " 17 GB", | 20260505 20:11:50.692 | INFO | "huge" : "", | 20260505 20:11:50.692 | INFO | "huge_shared" : " 27 GB", | 20260505 20:11:50.692 | INFO | "malloc" : " 14 GB", | 20260505 20:11:50.692 | INFO | "pinned" : " 29 GB" | 20260505 20:11:50.692 | INFO | } | 20260505 20:11:50.692 | INFO | Perf report for the last 5.14792 seconds: | 20260505 20:11:50.692 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:11:50.692 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:50.692 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:50.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:50.692 | INFO | ANALYSIS : 63.1%, 3.0, 98.8ms, 0 | 20260505 20:11:50.692 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:50.692 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:50.692 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 20:11:50.692 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:50.692 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:50.692 | INFO | Uploaded 36GiB to GPU in 2.12848s (16.9135GiB/s) | 20260505 20:11:50.692 | INFO | Downloaded 34GiB to GPU in 2.14307s (15.8651GiB/s) | 20260505 20:11:55.829 | INFO | Analyzing chunk frames = [5979648,5980160) | 20260505 20:11:55.829 | INFO | MemInfo: { | 20260505 20:11:55.829 | INFO | "gpu" : " 17 GB", | 20260505 20:11:55.829 | INFO | "huge" : "", | 20260505 20:11:55.829 | INFO | "huge_shared" : " 27 GB", | 20260505 20:11:55.829 | INFO | "malloc" : " 14 GB", | 20260505 20:11:55.829 | INFO | "pinned" : " 29 GB" | 20260505 20:11:55.829 | INFO | } | 20260505 20:11:55.829 | INFO | Perf report for the last 5.13753 seconds: | 20260505 20:11:55.829 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:11:55.829 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:55.829 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:55.830 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:55.830 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260505 20:11:55.830 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:55.830 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:55.830 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:11:55.830 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:55.830 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:11:55.830 | INFO | Uploaded 36GiB to GPU in 2.12326s (16.955GiB/s) | 20260505 20:11:55.830 | INFO | Downloaded 34GiB to GPU in 2.01334s (16.8874GiB/s) | 20260505 20:11:55.830 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 20:11:55.830 | INFO | Total mem usage: 15.3 GiB | 20260505 20:11:55.830 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 20:11:55.830 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.1%) | 20260505 20:11:55.830 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 20:11:55.830 | INFO | High Water Breakdown: | 20260505 20:11:55.830 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 20:11:55.830 | INFO | 38.8% : 5.51 GiB ( 82.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 20:11:55.830 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 20:11:55.830 | INFO | ------------------------------------------ | 20260505 20:11:55.830 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:11:55.830 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 20:11:55.830 | INFO | Total mem usage: 26.5 GiB | 20260505 20:11:55.830 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 20:11:55.830 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 20:11:55.830 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 20:11:55.830 | INFO | High Water Breakdown: | 20260505 20:11:55.830 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:11:55.830 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:11:55.830 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:11:55.830 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:11:55.830 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:11:55.830 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:11:55.830 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 20:11:55.830 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 20:11:55.830 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:11:55.830 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 20:11:55.831 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 20:11:55.831 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 20:11:55.831 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 20:11:55.831 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 20:11:55.831 | INFO | ------------------------------------------ | 20260505 20:11:55.831 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:11:55.831 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 20:11:55.831 | INFO | Total mem usage: 25.6 GiB | 20260505 20:11:55.831 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 20:11:55.831 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 20:11:55.831 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 20:11:55.831 | INFO | High Water Breakdown: | 20260505 20:11:55.831 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 20:11:55.831 | INFO | ------------------------------------------ | 20260505 20:11:55.831 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:11:55.831 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 20:11:55.831 | INFO | Total mem usage: 15.7 GiB | 20260505 20:11:55.831 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 20:11:55.831 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 20:11:55.831 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 20:11:55.831 | INFO | High Water Breakdown: | 20260505 20:11:55.831 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 20:11:55.831 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 20:11:55.831 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 20:11:55.831 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 20:11:55.831 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 20:11:55.831 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 20:11:55.831 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 20:11:55.831 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 20:11:55.831 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 20:11:55.831 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:11:55.831 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 20:11:55.831 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:11:55.831 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 20:11:55.831 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 20:11:55.831 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 20:11:55.831 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 20:11:55.831 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:11:55.831 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 20:11:55.831 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:11:55.831 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:11:55.831 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:11:55.831 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 20:11:55.831 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 20:11:55.831 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:11:55.831 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 20:11:55.831 | INFO | ------------------------------------------ | 20260505 20:11:55.831 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:11:55.831 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 20:11:55.831 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:11:55.831 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:11:55.831 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.48e+01, 1.34e+01 / 2.11e+00 | 20260505 20:11:55.831 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.04e-01, 1.03e-02 / 4.53e-03 | 20260505 20:11:55.832 | INFO | PerfReport: 11.79%, Baseline -- 3.71e+00 / 4.56e+00, 5.71e-01 / 1.33e-01 | 20260505 20:11:55.832 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.52e+00, 1.92e-01 / 8.95e-02 | 20260505 20:11:55.832 | INFO | PerfReport: 15.97%, DME -- 4.02e+01 / 4.45e+01, 4.21e+00 / 2.25e+00 | 20260505 20:11:55.832 | INFO | PerfReport: 57.07%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.50e+00 / 1.06e+00 | 20260505 20:11:55.832 | INFO | PerfReport: 7.22%, PulseAccumulating -- 2.27e+00 / 2.85e+00, 4.39e-01 / 1.57e-01 | 20260505 20:11:55.832 | INFO | PerfReport: 3.25%, Metrics -- 1.02e+00 / 1.35e+00, 2.83e-01 / 1.55e-01 | 20260505 20:11:55.832 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.56e-01 / 4.17e-01, 6.33e-02 / 3.55e-02 | 20260505 20:11:55.832 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 20:11:55.832 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:11:55.832 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:11:55.832 | INFO | PerfReport: Full Execution -- 9.94e+01 / -nan, 2.54e+01 / 0.00e+00 | 20260505 20:11:55.832 | INFO | PerfReport: 22.45%, Upload -- 2.23e+01 / 0.00e+00, 7.75e+00 / 0.00e+00 | 20260505 20:11:55.832 | INFO | PerfReport: 21.77%, Download -- 2.16e+01 / 0.00e+00, 7.70e+00 / 0.00e+00 | 20260505 20:11:55.832 | INFO | PerfReport: 31.69%, Compute -- 3.15e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 20:11:55.832 | INFO | PerfReport: 2.30%, LockUpload -- 2.29e+00 / 0.00e+00, 8.12e+00 / 0.00e+00 | 20260505 20:11:55.832 | INFO | PerfReport: 1.35%, LockDownload -- 1.34e+00 / 0.00e+00, 2.93e+00 / 0.00e+00 | 20260505 20:11:55.832 | INFO | PerfReport: 20.43%, LockCompute -- 2.03e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 20:11:55.832 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.65e-02 / 0.00e+00 | 20260505 20:11:55.963 | INFO | Starting a round of Baz IO | 20260505 20:11:59.130 | INFO | Populating RTMetrics report | 20260505 20:11:59.131 | INFO | RTMetrics report populated | 20260505 20:11:59.131 | INFO | Opening tmp RTMetrics json file | 20260505 20:11:59.132 | INFO | Serializing RTMetrics to json | 20260505 20:11:59.133 | INFO | Serialization done, swapping tmp file now... | 20260505 20:11:59.133 | INFO | RTMetrics done | 20260505 20:11:59.216 | INFO | Finished a round of Baz IO: Wrote 2.019 GiB in 3.252 seconds | 20260505 20:11:59.216 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.69% / 0.00% / 78.30% | 20260505 20:11:59.216 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.182398% / 0% / 6.35783e-05% | 20260505 20:11:59.216 | INFO | Baz IO overall: | 20260505 20:11:59.216 | INFO | BytesWritten : 3489366696545 | 20260505 20:11:59.216 | INFO | NumEvents : 1649957624807 | 20260505 20:12:00.965 | INFO | Analyzing chunk frames = [5980160,5980672) | 20260505 20:12:00.965 | INFO | MemInfo: { | 20260505 20:12:00.965 | INFO | "gpu" : " 17 GB", | 20260505 20:12:00.965 | INFO | "huge" : "", | 20260505 20:12:00.965 | INFO | "huge_shared" : " 27 GB", | 20260505 20:12:00.965 | INFO | "malloc" : " 14 GB", | 20260505 20:12:00.965 | INFO | "pinned" : " 29 GB" | 20260505 20:12:00.965 | INFO | } | 20260505 20:12:00.965 | INFO | Perf report for the last 5.13559 seconds: | 20260505 20:12:00.965 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:12:00.965 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:00.965 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:00.965 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:00.965 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 20:12:00.965 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:00.965 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260505 20:12:00.965 | INFO | PRE_HQ : 60.1%, 1.8, 55.3ms, 0 | 20260505 20:12:00.965 | INFO | BAZWRITER : 62.9%, 3.7, 118.0ms, 0 | 20260505 20:12:00.965 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 20:12:00.965 | INFO | Uploaded 36GiB to GPU in 2.17495s (16.5521GiB/s) | 20260505 20:12:00.965 | INFO | Downloaded 38GiB to GPU in 2.25968s (16.8166GiB/s) | 20260505 20:12:06.105 | INFO | Analyzing chunk frames = [5980672,5981184) | 20260505 20:12:06.105 | INFO | MemInfo: { | 20260505 20:12:06.105 | INFO | "gpu" : " 17 GB", | 20260505 20:12:06.105 | INFO | "huge" : "", | 20260505 20:12:06.105 | INFO | "huge_shared" : " 27 GB", | 20260505 20:12:06.105 | INFO | "malloc" : " 14 GB", | 20260505 20:12:06.105 | INFO | "pinned" : " 29 GB" | 20260505 20:12:06.105 | INFO | } | 20260505 20:12:06.105 | INFO | Perf report for the last 5.13985 seconds: | 20260505 20:12:06.105 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:12:06.105 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:06.105 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:06.105 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:06.105 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260505 20:12:06.105 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:06.105 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:06.105 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:12:06.105 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:06.105 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:06.105 | INFO | Uploaded 36GiB to GPU in 2.14252s (16.8026GiB/s) | 20260505 20:12:06.105 | INFO | Downloaded 34GiB to GPU in 2.04971s (16.5877GiB/s) | 20260505 20:12:11.245 | INFO | Analyzing chunk frames = [5981184,5981696) | 20260505 20:12:11.245 | INFO | MemInfo: { | 20260505 20:12:11.245 | INFO | "gpu" : " 17 GB", | 20260505 20:12:11.245 | INFO | "huge" : "", | 20260505 20:12:11.245 | INFO | "huge_shared" : " 27 GB", | 20260505 20:12:11.245 | INFO | "malloc" : " 14 GB", | 20260505 20:12:11.245 | INFO | "pinned" : " 29 GB" | 20260505 20:12:11.245 | INFO | } | 20260505 20:12:11.245 | INFO | Perf report for the last 5.13989 seconds: | 20260505 20:12:11.245 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:12:11.245 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:11.245 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:11.245 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:11.245 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260505 20:12:11.245 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:11.245 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:11.245 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:12:11.245 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:11.245 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:11.245 | INFO | Uploaded 36GiB to GPU in 2.11497s (17.0216GiB/s) | 20260505 20:12:11.245 | INFO | Downloaded 34GiB to GPU in 2.01555s (16.8688GiB/s) | 20260505 20:12:16.385 | INFO | Analyzing chunk frames = [5981696,5982208) | 20260505 20:12:16.385 | INFO | MemInfo: { | 20260505 20:12:16.385 | INFO | "gpu" : " 17 GB", | 20260505 20:12:16.385 | INFO | "huge" : "", | 20260505 20:12:16.385 | INFO | "huge_shared" : " 27 GB", | 20260505 20:12:16.385 | INFO | "malloc" : " 14 GB", | 20260505 20:12:16.385 | INFO | "pinned" : " 29 GB" | 20260505 20:12:16.385 | INFO | } | 20260505 20:12:16.385 | INFO | Perf report for the last 5.14 seconds: | 20260505 20:12:16.385 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:12:16.385 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:16.385 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:16.385 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:16.385 | INFO | ANALYSIS : 62.4%, 3.0, 97.0ms, 0 | 20260505 20:12:16.385 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:16.385 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:16.385 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 20:12:16.385 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:16.385 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:16.385 | INFO | Uploaded 36GiB to GPU in 2.21221s (16.2733GiB/s) | 20260505 20:12:16.385 | INFO | Downloaded 34GiB to GPU in 2.07689s (16.3707GiB/s) | 20260505 20:12:21.526 | INFO | Analyzing chunk frames = [5982208,5982720) | 20260505 20:12:21.526 | INFO | MemInfo: { | 20260505 20:12:21.526 | INFO | "gpu" : " 17 GB", | 20260505 20:12:21.526 | INFO | "huge" : "", | 20260505 20:12:21.526 | INFO | "huge_shared" : " 27 GB", | 20260505 20:12:21.526 | INFO | "malloc" : " 14 GB", | 20260505 20:12:21.526 | INFO | "pinned" : " 29 GB" | 20260505 20:12:21.526 | INFO | } | 20260505 20:12:21.526 | INFO | Perf report for the last 5.14095 seconds: | 20260505 20:12:21.526 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:12:21.526 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:21.526 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:21.526 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:21.526 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 20:12:21.526 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:21.526 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:21.526 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:12:21.526 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:21.526 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:21.526 | INFO | Uploaded 36GiB to GPU in 2.12868s (16.9119GiB/s) | 20260505 20:12:21.526 | INFO | Downloaded 34GiB to GPU in 2.06423s (16.471GiB/s) | 20260505 20:12:26.659 | INFO | Analyzing chunk frames = [5982720,5983232) | 20260505 20:12:26.659 | INFO | MemInfo: { | 20260505 20:12:26.659 | INFO | "gpu" : " 17 GB", | 20260505 20:12:26.659 | INFO | "huge" : "", | 20260505 20:12:26.659 | INFO | "huge_shared" : " 27 GB", | 20260505 20:12:26.659 | INFO | "malloc" : " 14 GB", | 20260505 20:12:26.659 | INFO | "pinned" : " 29 GB" | 20260505 20:12:26.659 | INFO | } | 20260505 20:12:26.659 | INFO | Perf report for the last 5.13343 seconds: | 20260505 20:12:26.659 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:12:26.659 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:26.659 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:26.659 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:26.659 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 20:12:26.659 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:26.659 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:26.659 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 20:12:26.659 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:26.659 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:26.660 | INFO | Uploaded 36GiB to GPU in 2.15158s (16.7319GiB/s) | 20260505 20:12:26.660 | INFO | Downloaded 34GiB to GPU in 2.06197s (16.4891GiB/s) | 20260505 20:12:31.798 | INFO | Analyzing chunk frames = [5983232,5983744) | 20260505 20:12:31.798 | INFO | MemInfo: { | 20260505 20:12:31.798 | INFO | "gpu" : " 17 GB", | 20260505 20:12:31.798 | INFO | "huge" : "", | 20260505 20:12:31.798 | INFO | "huge_shared" : " 27 GB", | 20260505 20:12:31.798 | INFO | "malloc" : " 14 GB", | 20260505 20:12:31.798 | INFO | "pinned" : " 29 GB" | 20260505 20:12:31.798 | INFO | } | 20260505 20:12:31.798 | INFO | Perf report for the last 5.13892 seconds: | 20260505 20:12:31.798 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:12:31.798 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:31.798 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:31.798 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:31.798 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 20:12:31.798 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:31.798 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:31.798 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 20:12:31.798 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:31.798 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:31.798 | INFO | Uploaded 36GiB to GPU in 2.11695s (17.0056GiB/s) | 20260505 20:12:31.798 | INFO | Downloaded 34GiB to GPU in 2.13499s (15.9252GiB/s) | 20260505 20:12:36.950 | INFO | Analyzing chunk frames = [5983744,5984256) | 20260505 20:12:36.950 | INFO | MemInfo: { | 20260505 20:12:36.950 | INFO | "gpu" : " 17 GB", | 20260505 20:12:36.950 | INFO | "huge" : "", | 20260505 20:12:36.950 | INFO | "huge_shared" : " 27 GB", | 20260505 20:12:36.950 | INFO | "malloc" : " 14 GB", | 20260505 20:12:36.950 | INFO | "pinned" : " 29 GB" | 20260505 20:12:36.950 | INFO | } | 20260505 20:12:36.950 | INFO | Perf report for the last 5.1522 seconds: | 20260505 20:12:36.950 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:12:36.950 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:36.950 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:36.950 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:36.950 | INFO | ANALYSIS : 62.8%, 3.0, 98.3ms, 0 | 20260505 20:12:36.951 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:36.951 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:36.951 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 20:12:36.951 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:36.951 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:36.951 | INFO | Uploaded 36GiB to GPU in 2.20447s (16.3305GiB/s) | 20260505 20:12:36.951 | INFO | Downloaded 34GiB to GPU in 2.04582s (16.6192GiB/s) | 20260505 20:12:37.078 | INFO | Starting a round of Baz IO | 20260505 20:12:40.236 | INFO | Populating RTMetrics report | 20260505 20:12:40.237 | INFO | RTMetrics report populated | 20260505 20:12:40.237 | INFO | Opening tmp RTMetrics json file | 20260505 20:12:40.237 | INFO | Serializing RTMetrics to json | 20260505 20:12:40.238 | INFO | Serialization done, swapping tmp file now... | 20260505 20:12:40.239 | INFO | RTMetrics done | 20260505 20:12:40.325 | INFO | Finished a round of Baz IO: Wrote 2.021 GiB in 3.246 seconds | 20260505 20:12:40.325 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.69% / 0.00% / 78.31% | 20260505 20:12:40.325 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.183062% / 0% / 9.13938e-05% | 20260505 20:12:40.325 | INFO | Baz IO overall: | 20260505 20:12:40.325 | INFO | BytesWritten : 3491536973829 | 20260505 20:12:40.325 | INFO | NumEvents : 1650964063858 | 20260505 20:12:42.091 | INFO | Analyzing chunk frames = [5984256,5984768) | 20260505 20:12:42.091 | INFO | MemInfo: { | 20260505 20:12:42.091 | INFO | "gpu" : " 17 GB", | 20260505 20:12:42.091 | INFO | "huge" : "", | 20260505 20:12:42.091 | INFO | "huge_shared" : " 27 GB", | 20260505 20:12:42.091 | INFO | "malloc" : " 14 GB", | 20260505 20:12:42.091 | INFO | "pinned" : " 29 GB" | 20260505 20:12:42.091 | INFO | } | 20260505 20:12:42.091 | INFO | Perf report for the last 5.14095 seconds: | 20260505 20:12:42.091 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:12:42.091 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:42.091 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:42.091 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:42.091 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260505 20:12:42.091 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:42.091 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260505 20:12:42.091 | INFO | PRE_HQ : 60.2%, 1.8, 55.1ms, 0 | 20260505 20:12:42.091 | INFO | BAZWRITER : 62.4%, 3.7, 118.5ms, 0 | 20260505 20:12:42.091 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 20:12:42.092 | INFO | Uploaded 36GiB to GPU in 2.17462s (16.5546GiB/s) | 20260505 20:12:42.092 | INFO | Downloaded 38GiB to GPU in 2.24011s (16.9635GiB/s) | 20260505 20:12:47.215 | INFO | Analyzing chunk frames = [5984768,5985280) | 20260505 20:12:47.215 | INFO | MemInfo: { | 20260505 20:12:47.215 | INFO | "gpu" : " 17 GB", | 20260505 20:12:47.215 | INFO | "huge" : "", | 20260505 20:12:47.215 | INFO | "huge_shared" : " 27 GB", | 20260505 20:12:47.215 | INFO | "malloc" : " 14 GB", | 20260505 20:12:47.215 | INFO | "pinned" : " 29 GB" | 20260505 20:12:47.215 | INFO | } | 20260505 20:12:47.215 | INFO | Perf report for the last 5.12347 seconds: | 20260505 20:12:47.215 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:12:47.215 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:47.215 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:47.215 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:47.215 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260505 20:12:47.215 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:47.215 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:47.215 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260505 20:12:47.215 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:47.215 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:47.215 | INFO | Uploaded 36GiB to GPU in 2.08355s (17.2782GiB/s) | 20260505 20:12:47.215 | INFO | Downloaded 34GiB to GPU in 2.02516s (16.7888GiB/s) | 20260505 20:12:52.350 | INFO | Analyzing chunk frames = [5985280,5985792) | 20260505 20:12:52.350 | INFO | MemInfo: { | 20260505 20:12:52.350 | INFO | "gpu" : " 17 GB", | 20260505 20:12:52.350 | INFO | "huge" : "", | 20260505 20:12:52.350 | INFO | "huge_shared" : " 27 GB", | 20260505 20:12:52.351 | INFO | "malloc" : " 14 GB", | 20260505 20:12:52.351 | INFO | "pinned" : " 29 GB" | 20260505 20:12:52.351 | INFO | } | 20260505 20:12:52.351 | INFO | Perf report for the last 5.13578 seconds: | 20260505 20:12:52.351 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:12:52.351 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:52.351 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:52.351 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:52.351 | INFO | ANALYSIS : 62.5%, 3.0, 97.1ms, 0 | 20260505 20:12:52.351 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:52.351 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:52.351 | INFO | PRE_HQ : 27.5%, 1.0, 14.2ms, 0 | 20260505 20:12:52.351 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:52.351 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:52.351 | INFO | Uploaded 36GiB to GPU in 2.14921s (16.7504GiB/s) | 20260505 20:12:52.351 | INFO | Downloaded 34GiB to GPU in 2.05348s (16.5573GiB/s) | 20260505 20:12:57.501 | INFO | Analyzing chunk frames = [5985792,5986304) | 20260505 20:12:57.501 | INFO | MemInfo: { | 20260505 20:12:57.501 | INFO | "gpu" : " 17 GB", | 20260505 20:12:57.501 | INFO | "huge" : "", | 20260505 20:12:57.501 | INFO | "huge_shared" : " 27 GB", | 20260505 20:12:57.501 | INFO | "malloc" : " 14 GB", | 20260505 20:12:57.501 | INFO | "pinned" : " 29 GB" | 20260505 20:12:57.501 | INFO | } | 20260505 20:12:57.501 | INFO | Perf report for the last 5.1507 seconds: | 20260505 20:12:57.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:12:57.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:57.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:57.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:57.501 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260505 20:12:57.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:57.501 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:57.501 | INFO | PRE_HQ : 27.2%, 1.0, 14.2ms, 0 | 20260505 20:12:57.502 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:57.502 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:12:57.502 | INFO | Uploaded 36GiB to GPU in 2.2233s (16.1922GiB/s) | 20260505 20:12:57.502 | INFO | Downloaded 34GiB to GPU in 2.11789s (16.0537GiB/s) | 20260505 20:13:02.643 | INFO | Analyzing chunk frames = [5986304,5986816) | 20260505 20:13:02.643 | INFO | MemInfo: { | 20260505 20:13:02.643 | INFO | "gpu" : " 17 GB", | 20260505 20:13:02.643 | INFO | "huge" : "", | 20260505 20:13:02.643 | INFO | "huge_shared" : " 27 GB", | 20260505 20:13:02.643 | INFO | "malloc" : " 14 GB", | 20260505 20:13:02.643 | INFO | "pinned" : " 29 GB" | 20260505 20:13:02.644 | INFO | } | 20260505 20:13:02.644 | INFO | Perf report for the last 5.14235 seconds: | 20260505 20:13:02.644 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:13:02.644 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:02.644 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:02.644 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:02.644 | INFO | ANALYSIS : 61.7%, 3.0, 96.5ms, 0 | 20260505 20:13:02.644 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:02.644 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:02.644 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:13:02.644 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:02.644 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:02.644 | INFO | Uploaded 36GiB to GPU in 2.12889s (16.9102GiB/s) | 20260505 20:13:02.644 | INFO | Downloaded 34GiB to GPU in 2.06042s (16.5015GiB/s) | 20260505 20:13:07.780 | INFO | Analyzing chunk frames = [5986816,5987328) | 20260505 20:13:07.780 | INFO | MemInfo: { | 20260505 20:13:07.780 | INFO | "gpu" : " 17 GB", | 20260505 20:13:07.780 | INFO | "huge" : "", | 20260505 20:13:07.780 | INFO | "huge_shared" : " 27 GB", | 20260505 20:13:07.780 | INFO | "malloc" : " 14 GB", | 20260505 20:13:07.780 | INFO | "pinned" : " 29 GB" | 20260505 20:13:07.780 | INFO | } | 20260505 20:13:07.780 | INFO | Perf report for the last 5.1365 seconds: | 20260505 20:13:07.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:13:07.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:07.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:07.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:07.780 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 20:13:07.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:07.780 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:07.780 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:13:07.780 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:07.780 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:07.781 | INFO | Uploaded 36GiB to GPU in 2.08388s (17.2755GiB/s) | 20260505 20:13:07.781 | INFO | Downloaded 34GiB to GPU in 2.033s (16.724GiB/s) | 20260505 20:13:12.916 | INFO | Analyzing chunk frames = [5987328,5987840) | 20260505 20:13:12.916 | INFO | MemInfo: { | 20260505 20:13:12.916 | INFO | "gpu" : " 17 GB", | 20260505 20:13:12.916 | INFO | "huge" : "", | 20260505 20:13:12.916 | INFO | "huge_shared" : " 27 GB", | 20260505 20:13:12.916 | INFO | "malloc" : " 14 GB", | 20260505 20:13:12.916 | INFO | "pinned" : " 29 GB" | 20260505 20:13:12.916 | INFO | } | 20260505 20:13:12.916 | INFO | Perf report for the last 5.13557 seconds: | 20260505 20:13:12.916 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:13:12.916 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:12.916 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:12.916 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:12.916 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 20:13:12.916 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:12.916 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:12.916 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 20:13:12.916 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:12.916 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:12.916 | INFO | Uploaded 36GiB to GPU in 2.13117s (16.8921GiB/s) | 20260505 20:13:12.916 | INFO | Downloaded 34GiB to GPU in 2.14071s (15.8826GiB/s) | 20260505 20:13:18.056 | INFO | Analyzing chunk frames = [5987840,5988352) | 20260505 20:13:18.056 | INFO | MemInfo: { | 20260505 20:13:18.056 | INFO | "gpu" : " 17 GB", | 20260505 20:13:18.056 | INFO | "huge" : "", | 20260505 20:13:18.056 | INFO | "huge_shared" : " 27 GB", | 20260505 20:13:18.056 | INFO | "malloc" : " 14 GB", | 20260505 20:13:18.057 | INFO | "pinned" : " 29 GB" | 20260505 20:13:18.057 | INFO | } | 20260505 20:13:18.057 | INFO | Perf report for the last 5.1409 seconds: | 20260505 20:13:18.057 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:13:18.057 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:18.057 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:18.057 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:18.057 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 20:13:18.057 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:18.057 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:18.057 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 20:13:18.057 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:18.057 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:18.057 | INFO | Uploaded 36GiB to GPU in 2.20318s (16.34GiB/s) | 20260505 20:13:18.057 | INFO | Downloaded 34GiB to GPU in 2.03629s (16.697GiB/s) | 20260505 20:13:18.057 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 20:13:18.057 | INFO | Total mem usage: 15.3 GiB | 20260505 20:13:18.057 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 20:13:18.057 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.11%) | 20260505 20:13:18.057 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 20:13:18.057 | INFO | High Water Breakdown: | 20260505 20:13:18.057 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 20:13:18.057 | INFO | 38.8% : 5.5 GiB ( 82.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 20:13:18.057 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 20:13:18.057 | INFO | ------------------------------------------ | 20260505 20:13:18.057 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:13:18.057 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 20:13:18.057 | INFO | Total mem usage: 26.5 GiB | 20260505 20:13:18.057 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 20:13:18.057 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 20:13:18.057 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 20:13:18.057 | INFO | High Water Breakdown: | 20260505 20:13:18.057 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:13:18.057 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:13:18.057 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:13:18.057 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:13:18.057 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:13:18.057 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:13:18.058 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 20:13:18.058 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 20:13:18.058 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:13:18.058 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 20:13:18.058 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 20:13:18.058 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 20:13:18.058 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 20:13:18.058 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 20:13:18.058 | INFO | ------------------------------------------ | 20260505 20:13:18.058 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:13:18.058 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 20:13:18.058 | INFO | Total mem usage: 25.6 GiB | 20260505 20:13:18.058 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 20:13:18.058 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 20:13:18.058 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 20:13:18.058 | INFO | High Water Breakdown: | 20260505 20:13:18.058 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 20:13:18.058 | INFO | ------------------------------------------ | 20260505 20:13:18.058 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:13:18.058 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 20:13:18.058 | INFO | Total mem usage: 15.7 GiB | 20260505 20:13:18.058 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 20:13:18.058 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 20:13:18.058 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 20:13:18.058 | INFO | High Water Breakdown: | 20260505 20:13:18.058 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 20:13:18.058 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 20:13:18.058 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 20:13:18.058 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 20:13:18.058 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 20:13:18.058 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 20:13:18.058 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 20:13:18.058 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 20:13:18.058 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 20:13:18.058 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:13:18.058 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 20:13:18.058 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:13:18.058 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 20:13:18.058 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 20:13:18.058 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 20:13:18.058 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 20:13:18.058 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:13:18.058 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 20:13:18.058 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:13:18.058 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:13:18.058 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:13:18.058 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 20:13:18.058 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 20:13:18.058 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:13:18.058 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 20:13:18.058 | INFO | ------------------------------------------ | 20260505 20:13:18.058 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:13:18.058 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 20:13:18.058 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:13:18.058 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:13:18.058 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.47e+01, 1.35e+01 / 2.00e+00 | 20260505 20:13:18.058 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.04e-01, 1.10e-02 / 5.11e-03 | 20260505 20:13:18.058 | INFO | PerfReport: 11.77%, Baseline -- 3.70e+00 / 4.56e+00, 5.55e-01 / 1.26e-01 | 20260505 20:13:18.058 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.51e+00, 1.84e-01 / 9.43e-02 | 20260505 20:13:18.058 | INFO | PerfReport: 16.01%, DME -- 4.02e+01 / 4.45e+01, 4.23e+00 / 1.97e+00 | 20260505 20:13:18.058 | INFO | PerfReport: 57.00%, FrameLabeling -- 1.79e+01 / 1.94e+01, 1.38e+00 / 1.01e+00 | 20260505 20:13:18.058 | INFO | PerfReport: 7.28%, PulseAccumulating -- 2.29e+00 / 2.84e+00, 4.38e-01 / 1.96e-01 | 20260505 20:13:18.058 | INFO | PerfReport: 3.25%, Metrics -- 1.02e+00 / 1.33e+00, 2.78e-01 / 1.19e-01 | 20260505 20:13:18.058 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.56e-01 / 4.19e-01, 6.25e-02 / 3.51e-02 | 20260505 20:13:18.058 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 20:13:18.058 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:13:18.058 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:13:18.058 | INFO | PerfReport: Full Execution -- 9.95e+01 / -nan, 2.54e+01 / 0.00e+00 | 20260505 20:13:18.058 | INFO | PerfReport: 22.53%, Upload -- 2.24e+01 / 0.00e+00, 7.80e+00 / 0.00e+00 | 20260505 20:13:18.058 | INFO | PerfReport: 21.88%, Download -- 2.18e+01 / 0.00e+00, 7.71e+00 / 0.00e+00 | 20260505 20:13:18.058 | INFO | PerfReport: 31.59%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 20:13:18.058 | INFO | PerfReport: 2.34%, LockUpload -- 2.33e+00 / 0.00e+00, 7.96e+00 / 0.00e+00 | 20260505 20:13:18.058 | INFO | PerfReport: 1.36%, LockDownload -- 1.35e+00 / 0.00e+00, 2.84e+00 / 0.00e+00 | 20260505 20:13:18.058 | INFO | PerfReport: 20.29%, LockCompute -- 2.02e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 20:13:18.058 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.78e-02 / 0.00e+00 | 20260505 20:13:18.191 | INFO | Starting a round of Baz IO | 20260505 20:13:21.351 | INFO | Populating RTMetrics report | 20260505 20:13:21.351 | INFO | RTMetrics report populated | 20260505 20:13:21.351 | INFO | Opening tmp RTMetrics json file | 20260505 20:13:21.353 | INFO | Serializing RTMetrics to json | 20260505 20:13:21.354 | INFO | Serialization done, swapping tmp file now... | 20260505 20:13:21.354 | INFO | RTMetrics done | 20260505 20:13:21.437 | INFO | Finished a round of Baz IO: Wrote 2.021 GiB in 3.246 seconds | 20260505 20:13:21.437 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.69% / 0.00% / 78.31% | 20260505 20:13:21.437 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.182227% / 0% / 8.34465e-05% | 20260505 20:13:21.437 | INFO | Baz IO overall: | 20260505 20:13:21.437 | INFO | BytesWritten : 3493706577552 | 20260505 20:13:21.437 | INFO | NumEvents : 1651970214803 | 20260505 20:13:23.198 | INFO | Analyzing chunk frames = [5988352,5988864) | 20260505 20:13:23.198 | INFO | MemInfo: { | 20260505 20:13:23.198 | INFO | "gpu" : " 17 GB", | 20260505 20:13:23.198 | INFO | "huge" : "", | 20260505 20:13:23.198 | INFO | "huge_shared" : " 27 GB", | 20260505 20:13:23.198 | INFO | "malloc" : " 14 GB", | 20260505 20:13:23.198 | INFO | "pinned" : " 29 GB" | 20260505 20:13:23.198 | INFO | } | 20260505 20:13:23.198 | INFO | Perf report for the last 5.14086 seconds: | 20260505 20:13:23.198 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:13:23.198 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:23.198 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:23.198 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:23.198 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 20:13:23.198 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:23.198 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260505 20:13:23.198 | INFO | PRE_HQ : 60.0%, 1.8, 55.2ms, 0 | 20260505 20:13:23.198 | INFO | BAZWRITER : 62.7%, 3.7, 119.8ms, 0 | 20260505 20:13:23.198 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 20:13:23.198 | INFO | Uploaded 36GiB to GPU in 2.1673s (16.6105GiB/s) | 20260505 20:13:23.198 | INFO | Downloaded 38GiB to GPU in 2.25456s (16.8547GiB/s) | 20260505 20:13:28.334 | INFO | Analyzing chunk frames = [5988864,5989376) | 20260505 20:13:28.334 | INFO | MemInfo: { | 20260505 20:13:28.334 | INFO | "gpu" : " 17 GB", | 20260505 20:13:28.334 | INFO | "huge" : "", | 20260505 20:13:28.334 | INFO | "huge_shared" : " 27 GB", | 20260505 20:13:28.334 | INFO | "malloc" : " 14 GB", | 20260505 20:13:28.334 | INFO | "pinned" : " 29 GB" | 20260505 20:13:28.334 | INFO | } | 20260505 20:13:28.334 | INFO | Perf report for the last 5.13622 seconds: | 20260505 20:13:28.334 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:13:28.334 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:28.334 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:28.334 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:28.334 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 20:13:28.334 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:28.334 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:28.334 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:13:28.334 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:28.334 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:28.334 | INFO | Uploaded 36GiB to GPU in 2.08006s (17.3072GiB/s) | 20260505 20:13:28.334 | INFO | Downloaded 34GiB to GPU in 2.02225s (16.813GiB/s) | 20260505 20:13:33.470 | INFO | Analyzing chunk frames = [5989376,5989888) | 20260505 20:13:33.470 | INFO | MemInfo: { | 20260505 20:13:33.470 | INFO | "gpu" : " 17 GB", | 20260505 20:13:33.470 | INFO | "huge" : "", | 20260505 20:13:33.470 | INFO | "huge_shared" : " 27 GB", | 20260505 20:13:33.470 | INFO | "malloc" : " 14 GB", | 20260505 20:13:33.470 | INFO | "pinned" : " 29 GB" | 20260505 20:13:33.470 | INFO | } | 20260505 20:13:33.470 | INFO | Perf report for the last 5.13632 seconds: | 20260505 20:13:33.470 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:13:33.470 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:33.470 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:33.470 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:33.470 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260505 20:13:33.470 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:33.470 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:33.470 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 20:13:33.470 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:33.470 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:33.470 | INFO | Uploaded 36GiB to GPU in 2.11945s (16.9856GiB/s) | 20260505 20:13:33.470 | INFO | Downloaded 34GiB to GPU in 2.01996s (16.832GiB/s) | 20260505 20:13:38.611 | INFO | Analyzing chunk frames = [5989888,5990400) | 20260505 20:13:38.611 | INFO | MemInfo: { | 20260505 20:13:38.611 | INFO | "gpu" : " 17 GB", | 20260505 20:13:38.611 | INFO | "huge" : "", | 20260505 20:13:38.611 | INFO | "huge_shared" : " 27 GB", | 20260505 20:13:38.611 | INFO | "malloc" : " 14 GB", | 20260505 20:13:38.611 | INFO | "pinned" : " 29 GB" | 20260505 20:13:38.611 | INFO | } | 20260505 20:13:38.611 | INFO | Perf report for the last 5.14092 seconds: | 20260505 20:13:38.611 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:13:38.611 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:38.611 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:38.611 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:38.611 | INFO | ANALYSIS : 62.3%, 3.0, 96.7ms, 0 | 20260505 20:13:38.611 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:38.611 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:38.611 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:13:38.611 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:38.611 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:38.611 | INFO | Uploaded 36GiB to GPU in 2.18261s (16.494GiB/s) | 20260505 20:13:38.611 | INFO | Downloaded 34GiB to GPU in 2.0729s (16.4021GiB/s) | 20260505 20:13:43.752 | INFO | Analyzing chunk frames = [5990400,5990912) | 20260505 20:13:43.752 | INFO | MemInfo: { | 20260505 20:13:43.752 | INFO | "gpu" : " 17 GB", | 20260505 20:13:43.752 | INFO | "huge" : "", | 20260505 20:13:43.752 | INFO | "huge_shared" : " 27 GB", | 20260505 20:13:43.752 | INFO | "malloc" : " 14 GB", | 20260505 20:13:43.752 | INFO | "pinned" : " 29 GB" | 20260505 20:13:43.752 | INFO | } | 20260505 20:13:43.753 | INFO | Perf report for the last 5.14143 seconds: | 20260505 20:13:43.753 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:13:43.753 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:43.753 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:43.753 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:43.753 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 20:13:43.753 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:43.753 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:43.753 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:13:43.753 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:43.753 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:43.753 | INFO | Uploaded 36GiB to GPU in 2.13579s (16.8556GiB/s) | 20260505 20:13:43.753 | INFO | Downloaded 34GiB to GPU in 2.06348s (16.4771GiB/s) | 20260505 20:13:48.901 | INFO | Analyzing chunk frames = [5990912,5991424) | 20260505 20:13:48.901 | INFO | MemInfo: { | 20260505 20:13:48.901 | INFO | "gpu" : " 17 GB", | 20260505 20:13:48.901 | INFO | "huge" : "", | 20260505 20:13:48.901 | INFO | "huge_shared" : " 27 GB", | 20260505 20:13:48.901 | INFO | "malloc" : " 14 GB", | 20260505 20:13:48.901 | INFO | "pinned" : " 29 GB" | 20260505 20:13:48.901 | INFO | } | 20260505 20:13:48.901 | INFO | Perf report for the last 5.14846 seconds: | 20260505 20:13:48.901 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:13:48.901 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:48.901 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:48.901 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:48.901 | INFO | ANALYSIS : 62.1%, 3.0, 97.2ms, 0 | 20260505 20:13:48.901 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:48.901 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:48.901 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:13:48.901 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:48.901 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:48.901 | INFO | Uploaded 36GiB to GPU in 2.14838s (16.7568GiB/s) | 20260505 20:13:48.901 | INFO | Downloaded 34GiB to GPU in 2.06125s (16.4948GiB/s) | 20260505 20:13:54.034 | INFO | Analyzing chunk frames = [5991424,5991936) | 20260505 20:13:54.034 | INFO | MemInfo: { | 20260505 20:13:54.034 | INFO | "gpu" : " 17 GB", | 20260505 20:13:54.034 | INFO | "huge" : "", | 20260505 20:13:54.034 | INFO | "huge_shared" : " 27 GB", | 20260505 20:13:54.034 | INFO | "malloc" : " 14 GB", | 20260505 20:13:54.034 | INFO | "pinned" : " 29 GB" | 20260505 20:13:54.034 | INFO | } | 20260505 20:13:54.034 | INFO | Perf report for the last 5.13325 seconds: | 20260505 20:13:54.034 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:13:54.034 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:54.034 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:54.034 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:54.034 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 20:13:54.034 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:54.034 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:54.034 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 20:13:54.034 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:54.035 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:54.035 | INFO | Uploaded 36GiB to GPU in 2.14562s (16.7784GiB/s) | 20260505 20:13:54.035 | INFO | Downloaded 34GiB to GPU in 2.14621s (15.8419GiB/s) | 20260505 20:13:59.165 | INFO | Analyzing chunk frames = [5991936,5992448) | 20260505 20:13:59.165 | INFO | MemInfo: { | 20260505 20:13:59.165 | INFO | "gpu" : " 17 GB", | 20260505 20:13:59.165 | INFO | "huge" : "", | 20260505 20:13:59.165 | INFO | "huge_shared" : " 27 GB", | 20260505 20:13:59.165 | INFO | "malloc" : " 14 GB", | 20260505 20:13:59.165 | INFO | "pinned" : " 29 GB" | 20260505 20:13:59.165 | INFO | } | 20260505 20:13:59.165 | INFO | Perf report for the last 5.13076 seconds: | 20260505 20:13:59.165 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:13:59.165 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:59.165 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:59.165 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:59.165 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 20:13:59.165 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:59.165 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:59.165 | INFO | PRE_HQ : 27.5%, 1.0, 14.2ms, 0 | 20260505 20:13:59.165 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:59.165 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:13:59.165 | INFO | Uploaded 36GiB to GPU in 2.12807s (16.9167GiB/s) | 20260505 20:13:59.165 | INFO | Downloaded 34GiB to GPU in 2.01471s (16.8759GiB/s) | 20260505 20:13:59.299 | INFO | Starting a round of Baz IO | 20260505 20:14:02.469 | INFO | Populating RTMetrics report | 20260505 20:14:02.469 | INFO | RTMetrics report populated | 20260505 20:14:02.469 | INFO | Opening tmp RTMetrics json file | 20260505 20:14:02.470 | INFO | Serializing RTMetrics to json | 20260505 20:14:02.471 | INFO | Serialization done, swapping tmp file now... | 20260505 20:14:02.471 | INFO | RTMetrics done | 20260505 20:14:02.556 | INFO | Finished a round of Baz IO: Wrote 2.018 GiB in 3.256 seconds | 20260505 20:14:02.556 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.69% / 0.00% / 78.31% | 20260505 20:14:02.556 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.180781% / 3.97364e-06% / 9.13938e-05% | 20260505 20:14:02.556 | INFO | Baz IO overall: | 20260505 20:14:02.556 | INFO | BytesWritten : 3495873855741 | 20260505 20:14:02.556 | INFO | NumEvents : 1652974364098 | 20260505 20:14:04.307 | INFO | Analyzing chunk frames = [5992448,5992960) | 20260505 20:14:04.307 | INFO | MemInfo: { | 20260505 20:14:04.307 | INFO | "gpu" : " 17 GB", | 20260505 20:14:04.307 | INFO | "huge" : "", | 20260505 20:14:04.307 | INFO | "huge_shared" : " 27 GB", | 20260505 20:14:04.307 | INFO | "malloc" : " 14 GB", | 20260505 20:14:04.307 | INFO | "pinned" : " 29 GB" | 20260505 20:14:04.307 | INFO | } | 20260505 20:14:04.307 | INFO | Perf report for the last 5.14194 seconds: | 20260505 20:14:04.307 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:14:04.307 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:04.307 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:04.307 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:04.307 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 20:14:04.307 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:04.307 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260505 20:14:04.307 | INFO | PRE_HQ : 60.5%, 1.8, 55.8ms, 0 | 20260505 20:14:04.307 | INFO | BAZWRITER : 62.8%, 3.7, 118.9ms, 0 | 20260505 20:14:04.307 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 20:14:04.307 | INFO | Uploaded 36GiB to GPU in 2.15794s (16.6826GiB/s) | 20260505 20:14:04.307 | INFO | Downloaded 38GiB to GPU in 2.23619s (16.9932GiB/s) | 20260505 20:14:09.454 | INFO | Analyzing chunk frames = [5992960,5993472) | 20260505 20:14:09.454 | INFO | MemInfo: { | 20260505 20:14:09.454 | INFO | "gpu" : " 17 GB", | 20260505 20:14:09.454 | INFO | "huge" : "", | 20260505 20:14:09.454 | INFO | "huge_shared" : " 27 GB", | 20260505 20:14:09.454 | INFO | "malloc" : " 14 GB", | 20260505 20:14:09.454 | INFO | "pinned" : " 29 GB" | 20260505 20:14:09.454 | INFO | } | 20260505 20:14:09.454 | INFO | Perf report for the last 5.14734 seconds: | 20260505 20:14:09.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:14:09.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:09.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:09.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:09.454 | INFO | ANALYSIS : 62.5%, 3.0, 97.8ms, 0 | 20260505 20:14:09.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:09.454 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:09.454 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 20:14:09.454 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:09.454 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:09.455 | INFO | Uploaded 36GiB to GPU in 2.14812s (16.7588GiB/s) | 20260505 20:14:09.455 | INFO | Downloaded 34GiB to GPU in 2.04673s (16.6119GiB/s) | 20260505 20:14:14.599 | INFO | Analyzing chunk frames = [5993472,5993984) | 20260505 20:14:14.599 | INFO | MemInfo: { | 20260505 20:14:14.599 | INFO | "gpu" : " 17 GB", | 20260505 20:14:14.599 | INFO | "huge" : "", | 20260505 20:14:14.599 | INFO | "huge_shared" : " 27 GB", | 20260505 20:14:14.599 | INFO | "malloc" : " 14 GB", | 20260505 20:14:14.599 | INFO | "pinned" : " 29 GB" | 20260505 20:14:14.599 | INFO | } | 20260505 20:14:14.599 | INFO | Perf report for the last 5.14506 seconds: | 20260505 20:14:14.599 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:14:14.599 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:14.599 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:14.600 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:14.600 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260505 20:14:14.600 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:14.600 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:14.600 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:14:14.600 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:14.600 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:14.600 | INFO | Uploaded 36GiB to GPU in 2.13568s (16.8565GiB/s) | 20260505 20:14:14.600 | INFO | Downloaded 34GiB to GPU in 2.02551s (16.7859GiB/s) | 20260505 20:14:19.735 | INFO | Analyzing chunk frames = [5993984,5994496) | 20260505 20:14:19.735 | INFO | MemInfo: { | 20260505 20:14:19.735 | INFO | "gpu" : " 17 GB", | 20260505 20:14:19.735 | INFO | "huge" : "", | 20260505 20:14:19.735 | INFO | "huge_shared" : " 27 GB", | 20260505 20:14:19.735 | INFO | "malloc" : " 14 GB", | 20260505 20:14:19.735 | INFO | "pinned" : " 29 GB" | 20260505 20:14:19.735 | INFO | } | 20260505 20:14:19.735 | INFO | Perf report for the last 5.13603 seconds: | 20260505 20:14:19.735 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:14:19.736 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:19.736 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:19.736 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:19.736 | INFO | ANALYSIS : 62.5%, 3.0, 97.1ms, 0 | 20260505 20:14:19.736 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:19.736 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:19.736 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:14:19.736 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:19.736 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:19.736 | INFO | Uploaded 36GiB to GPU in 2.21482s (16.2541GiB/s) | 20260505 20:14:19.736 | INFO | Downloaded 34GiB to GPU in 2.07663s (16.3727GiB/s) | 20260505 20:14:24.866 | INFO | Analyzing chunk frames = [5994496,5995008) | 20260505 20:14:24.866 | INFO | MemInfo: { | 20260505 20:14:24.866 | INFO | "gpu" : " 17 GB", | 20260505 20:14:24.866 | INFO | "huge" : "", | 20260505 20:14:24.866 | INFO | "huge_shared" : " 27 GB", | 20260505 20:14:24.866 | INFO | "malloc" : " 14 GB", | 20260505 20:14:24.866 | INFO | "pinned" : " 29 GB" | 20260505 20:14:24.866 | INFO | } | 20260505 20:14:24.866 | INFO | Perf report for the last 5.13083 seconds: | 20260505 20:14:24.866 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:14:24.866 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:24.866 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:24.866 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:24.866 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260505 20:14:24.866 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:24.866 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:24.866 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 20:14:24.866 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:24.866 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:24.867 | INFO | Uploaded 36GiB to GPU in 2.13451s (16.8657GiB/s) | 20260505 20:14:24.867 | INFO | Downloaded 34GiB to GPU in 2.06133s (16.4942GiB/s) | 20260505 20:14:30.007 | INFO | Analyzing chunk frames = [5995008,5995520) | 20260505 20:14:30.007 | INFO | MemInfo: { | 20260505 20:14:30.007 | INFO | "gpu" : " 17 GB", | 20260505 20:14:30.007 | INFO | "huge" : "", | 20260505 20:14:30.007 | INFO | "huge_shared" : " 27 GB", | 20260505 20:14:30.008 | INFO | "malloc" : " 14 GB", | 20260505 20:14:30.008 | INFO | "pinned" : " 29 GB" | 20260505 20:14:30.008 | INFO | } | 20260505 20:14:30.008 | INFO | Perf report for the last 5.14134 seconds: | 20260505 20:14:30.008 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:14:30.008 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:30.008 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:30.008 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:30.008 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260505 20:14:30.008 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:30.008 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:30.008 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 20:14:30.008 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:30.008 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:30.008 | INFO | Uploaded 36GiB to GPU in 2.14089s (16.8154GiB/s) | 20260505 20:14:30.008 | INFO | Downloaded 34GiB to GPU in 2.05525s (16.543GiB/s) | 20260505 20:14:35.146 | INFO | Analyzing chunk frames = [5995520,5996032) | 20260505 20:14:35.146 | INFO | MemInfo: { | 20260505 20:14:35.146 | INFO | "gpu" : " 17 GB", | 20260505 20:14:35.146 | INFO | "huge" : "", | 20260505 20:14:35.146 | INFO | "huge_shared" : " 27 GB", | 20260505 20:14:35.146 | INFO | "malloc" : " 14 GB", | 20260505 20:14:35.146 | INFO | "pinned" : " 29 GB" | 20260505 20:14:35.146 | INFO | } | 20260505 20:14:35.146 | INFO | Perf report for the last 5.13857 seconds: | 20260505 20:14:35.146 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:14:35.146 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:35.146 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:35.146 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:35.146 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 20:14:35.146 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:35.146 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:35.146 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 20:14:35.146 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:35.146 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:35.147 | INFO | Uploaded 36GiB to GPU in 2.11924s (16.9872GiB/s) | 20260505 20:14:35.147 | INFO | Downloaded 34GiB to GPU in 2.15419s (15.7832GiB/s) | 20260505 20:14:40.286 | INFO | Analyzing chunk frames = [5996032,5996544) | 20260505 20:14:40.286 | INFO | MemInfo: { | 20260505 20:14:40.286 | INFO | "gpu" : " 17 GB", | 20260505 20:14:40.286 | INFO | "huge" : "", | 20260505 20:14:40.286 | INFO | "huge_shared" : " 27 GB", | 20260505 20:14:40.286 | INFO | "malloc" : " 14 GB", | 20260505 20:14:40.286 | INFO | "pinned" : " 29 GB" | 20260505 20:14:40.286 | INFO | } | 20260505 20:14:40.287 | INFO | Perf report for the last 5.14021 seconds: | 20260505 20:14:40.287 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:14:40.287 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:40.287 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:40.287 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:40.287 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 20:14:40.287 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:40.287 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:40.287 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:14:40.287 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:40.287 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:40.287 | INFO | Uploaded 36GiB to GPU in 2.09867s (17.1537GiB/s) | 20260505 20:14:40.287 | INFO | Downloaded 34GiB to GPU in 1.99485s (17.0439GiB/s) | 20260505 20:14:40.287 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 20:14:40.287 | INFO | Total mem usage: 15.3 GiB | 20260505 20:14:40.287 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 20:14:40.287 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.29%) | 20260505 20:14:40.287 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 20:14:40.287 | INFO | High Water Breakdown: | 20260505 20:14:40.287 | INFO | 61% : 8.64 GiB ( 90.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 20:14:40.287 | INFO | 38.8% : 5.5 GiB ( 82.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 20:14:40.287 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 20:14:40.287 | INFO | ------------------------------------------ | 20260505 20:14:40.287 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:14:40.287 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 20:14:40.287 | INFO | Total mem usage: 26.5 GiB | 20260505 20:14:40.287 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 20:14:40.287 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 20:14:40.287 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 20:14:40.287 | INFO | High Water Breakdown: | 20260505 20:14:40.287 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:14:40.287 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:14:40.287 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:14:40.287 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:14:40.287 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:14:40.287 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:14:40.287 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 20:14:40.287 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 20:14:40.287 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:14:40.287 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 20:14:40.287 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 20:14:40.287 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 20:14:40.287 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 20:14:40.287 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 20:14:40.287 | INFO | ------------------------------------------ | 20260505 20:14:40.287 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:14:40.287 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 20:14:40.287 | INFO | Total mem usage: 25.6 GiB | 20260505 20:14:40.287 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 20:14:40.287 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 20:14:40.287 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 20:14:40.287 | INFO | High Water Breakdown: | 20260505 20:14:40.287 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 20:14:40.287 | INFO | ------------------------------------------ | 20260505 20:14:40.287 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:14:40.287 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 20:14:40.287 | INFO | Total mem usage: 15.7 GiB | 20260505 20:14:40.287 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 20:14:40.287 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 20:14:40.287 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 20:14:40.287 | INFO | High Water Breakdown: | 20260505 20:14:40.287 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 20:14:40.287 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 20:14:40.287 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 20:14:40.287 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 20:14:40.287 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 20:14:40.287 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 20:14:40.287 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 20:14:40.287 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 20:14:40.287 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 20:14:40.287 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:14:40.287 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 20:14:40.287 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:14:40.287 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 20:14:40.287 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 20:14:40.287 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 20:14:40.287 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 20:14:40.288 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:14:40.288 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 20:14:40.288 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:14:40.288 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:14:40.288 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:14:40.288 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 20:14:40.288 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 20:14:40.288 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:14:40.288 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 20:14:40.288 | INFO | ------------------------------------------ | 20260505 20:14:40.288 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:14:40.288 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 20:14:40.288 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:14:40.288 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:14:40.288 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.47e+01, 1.35e+01 / 2.23e+00 | 20260505 20:14:40.288 | INFO | PerfReport: 0.29%, AggregateStats -- 9.24e-02 / 1.03e-01, 1.08e-02 / 3.42e-03 | 20260505 20:14:40.288 | INFO | PerfReport: 11.79%, Baseline -- 3.71e+00 / 4.56e+00, 5.69e-01 / 1.43e-01 | 20260505 20:14:40.288 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.52e+00, 1.90e-01 / 9.44e-02 | 20260505 20:14:40.288 | INFO | PerfReport: 15.99%, DME -- 4.03e+01 / 4.46e+01, 4.33e+00 / 2.40e+00 | 20260505 20:14:40.288 | INFO | PerfReport: 57.02%, FrameLabeling -- 1.79e+01 / 1.93e+01, 1.45e+00 / 9.08e-01 | 20260505 20:14:40.288 | INFO | PerfReport: 7.25%, PulseAccumulating -- 2.28e+00 / 2.86e+00, 4.41e-01 / 2.01e-01 | 20260505 20:14:40.288 | INFO | PerfReport: 3.25%, Metrics -- 1.02e+00 / 1.34e+00, 2.83e-01 / 1.47e-01 | 20260505 20:14:40.288 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.57e-01 / 4.24e-01, 6.37e-02 / 4.32e-02 | 20260505 20:14:40.288 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 20:14:40.288 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:14:40.288 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:14:40.288 | INFO | PerfReport: Full Execution -- 9.94e+01 / -nan, 2.53e+01 / 0.00e+00 | 20260505 20:14:40.288 | INFO | PerfReport: 22.44%, Upload -- 2.23e+01 / 0.00e+00, 7.74e+00 / 0.00e+00 | 20260505 20:14:40.288 | INFO | PerfReport: 21.82%, Download -- 2.17e+01 / 0.00e+00, 7.76e+00 / 0.00e+00 | 20260505 20:14:40.288 | INFO | PerfReport: 31.68%, Compute -- 3.15e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 20:14:40.288 | INFO | PerfReport: 2.26%, LockUpload -- 2.24e+00 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260505 20:14:40.288 | INFO | PerfReport: 1.40%, LockDownload -- 1.39e+00 / 0.00e+00, 2.98e+00 / 0.00e+00 | 20260505 20:14:40.288 | INFO | PerfReport: 20.38%, LockCompute -- 2.03e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 20:14:40.288 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.84e-02 / 0.00e+00 | 20260505 20:14:40.419 | INFO | Starting a round of Baz IO | 20260505 20:14:43.585 | INFO | Populating RTMetrics report | 20260505 20:14:43.585 | INFO | RTMetrics report populated | 20260505 20:14:43.585 | INFO | Opening tmp RTMetrics json file | 20260505 20:14:43.586 | INFO | Serializing RTMetrics to json | 20260505 20:14:43.588 | INFO | Serialization done, swapping tmp file now... | 20260505 20:14:43.588 | INFO | RTMetrics done | 20260505 20:14:43.672 | INFO | Finished a round of Baz IO: Wrote 2.019 GiB in 3.253 seconds | 20260505 20:14:43.672 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.69% / 0.00% / 78.31% | 20260505 20:14:43.672 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.181063% / 3.97364e-06% / 9.53674e-05% | 20260505 20:14:43.673 | INFO | Baz IO overall: | 20260505 20:14:43.673 | INFO | BytesWritten : 3498041378732 | 20260505 20:14:43.673 | INFO | NumEvents : 1653978795278 | 20260505 20:14:45.422 | INFO | Analyzing chunk frames = [5996544,5997056) | 20260505 20:14:45.422 | INFO | MemInfo: { | 20260505 20:14:45.422 | INFO | "gpu" : " 17 GB", | 20260505 20:14:45.422 | INFO | "huge" : "", | 20260505 20:14:45.422 | INFO | "huge_shared" : " 27 GB", | 20260505 20:14:45.422 | INFO | "malloc" : " 14 GB", | 20260505 20:14:45.422 | INFO | "pinned" : " 29 GB" | 20260505 20:14:45.422 | INFO | } | 20260505 20:14:45.422 | INFO | Perf report for the last 5.13532 seconds: | 20260505 20:14:45.422 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:14:45.422 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:45.422 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:45.422 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:45.422 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 20:14:45.422 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:45.422 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260505 20:14:45.422 | INFO | PRE_HQ : 60.0%, 1.8, 55.1ms, 0 | 20260505 20:14:45.422 | INFO | BAZWRITER : 62.9%, 3.7, 118.6ms, 0 | 20260505 20:14:45.422 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 20:14:45.422 | INFO | Uploaded 36GiB to GPU in 2.15886s (16.6755GiB/s) | 20260505 20:14:45.422 | INFO | Downloaded 38GiB to GPU in 2.24789s (16.9048GiB/s) | 20260505 20:14:50.563 | INFO | Analyzing chunk frames = [5997056,5997568) | 20260505 20:14:50.563 | INFO | MemInfo: { | 20260505 20:14:50.563 | INFO | "gpu" : " 17 GB", | 20260505 20:14:50.563 | INFO | "huge" : "", | 20260505 20:14:50.563 | INFO | "huge_shared" : " 27 GB", | 20260505 20:14:50.563 | INFO | "malloc" : " 14 GB", | 20260505 20:14:50.563 | INFO | "pinned" : " 29 GB" | 20260505 20:14:50.563 | INFO | } | 20260505 20:14:50.563 | INFO | Perf report for the last 5.14144 seconds: | 20260505 20:14:50.563 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:14:50.563 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:50.563 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:50.563 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:50.563 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 20:14:50.563 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:50.563 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:50.563 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:14:50.563 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:50.563 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:50.564 | INFO | Uploaded 36GiB to GPU in 2.15226s (16.7266GiB/s) | 20260505 20:14:50.564 | INFO | Downloaded 34GiB to GPU in 2.05119s (16.5758GiB/s) | 20260505 20:14:55.710 | INFO | Analyzing chunk frames = [5997568,5998080) | 20260505 20:14:55.710 | INFO | MemInfo: { | 20260505 20:14:55.710 | INFO | "gpu" : " 17 GB", | 20260505 20:14:55.710 | INFO | "huge" : "", | 20260505 20:14:55.710 | INFO | "huge_shared" : " 27 GB", | 20260505 20:14:55.710 | INFO | "malloc" : " 14 GB", | 20260505 20:14:55.710 | INFO | "pinned" : " 29 GB" | 20260505 20:14:55.710 | INFO | } | 20260505 20:14:55.710 | INFO | Perf report for the last 5.14644 seconds: | 20260505 20:14:55.710 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:14:55.710 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:55.710 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:55.710 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:55.710 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260505 20:14:55.710 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:55.710 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:55.710 | INFO | PRE_HQ : 27.2%, 1.0, 14.2ms, 0 | 20260505 20:14:55.710 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:55.710 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:14:55.710 | INFO | Uploaded 36GiB to GPU in 2.14106s (16.8141GiB/s) | 20260505 20:14:55.711 | INFO | Downloaded 34GiB to GPU in 2.03597s (16.6996GiB/s) | 20260505 20:15:00.850 | INFO | Analyzing chunk frames = [5998080,5998592) | 20260505 20:15:00.850 | INFO | MemInfo: { | 20260505 20:15:00.850 | INFO | "gpu" : " 17 GB", | 20260505 20:15:00.850 | INFO | "huge" : "", | 20260505 20:15:00.850 | INFO | "huge_shared" : " 27 GB", | 20260505 20:15:00.850 | INFO | "malloc" : " 14 GB", | 20260505 20:15:00.850 | INFO | "pinned" : " 29 GB" | 20260505 20:15:00.850 | INFO | } | 20260505 20:15:00.850 | INFO | Perf report for the last 5.14034 seconds: | 20260505 20:15:00.850 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:15:00.850 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:00.850 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:00.850 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:00.850 | INFO | ANALYSIS : 62.1%, 3.0, 96.5ms, 0 | 20260505 20:15:00.850 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:00.850 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:00.850 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:15:00.850 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:00.850 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:00.850 | INFO | Uploaded 36GiB to GPU in 2.14401s (16.791GiB/s) | 20260505 20:15:00.850 | INFO | Downloaded 34GiB to GPU in 2.04862s (16.5965GiB/s) | 20260505 20:15:05.983 | INFO | Analyzing chunk frames = [5998592,5999104) | 20260505 20:15:05.983 | INFO | MemInfo: { | 20260505 20:15:05.983 | INFO | "gpu" : " 17 GB", | 20260505 20:15:05.983 | INFO | "huge" : "", | 20260505 20:15:05.983 | INFO | "huge_shared" : " 27 GB", | 20260505 20:15:05.983 | INFO | "malloc" : " 14 GB", | 20260505 20:15:05.983 | INFO | "pinned" : " 29 GB" | 20260505 20:15:05.983 | INFO | } | 20260505 20:15:05.983 | INFO | Perf report for the last 5.1326 seconds: | 20260505 20:15:05.983 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:15:05.983 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:05.983 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:05.983 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:05.983 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260505 20:15:05.983 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:05.983 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:05.983 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260505 20:15:05.983 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:05.983 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:05.983 | INFO | Uploaded 36GiB to GPU in 2.14242s (16.8034GiB/s) | 20260505 20:15:05.983 | INFO | Downloaded 34GiB to GPU in 2.06079s (16.4985GiB/s) | 20260505 20:15:11.115 | INFO | Analyzing chunk frames = [5999104,5999616) | 20260505 20:15:11.115 | INFO | MemInfo: { | 20260505 20:15:11.115 | INFO | "gpu" : " 17 GB", | 20260505 20:15:11.115 | INFO | "huge" : "", | 20260505 20:15:11.115 | INFO | "huge_shared" : " 27 GB", | 20260505 20:15:11.115 | INFO | "malloc" : " 14 GB", | 20260505 20:15:11.115 | INFO | "pinned" : " 29 GB" | 20260505 20:15:11.115 | INFO | } | 20260505 20:15:11.115 | INFO | Perf report for the last 5.13271 seconds: | 20260505 20:15:11.115 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:15:11.115 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:11.115 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:11.115 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:11.115 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 20:15:11.115 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:11.116 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:11.116 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 20:15:11.116 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:11.116 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:11.116 | INFO | Uploaded 36GiB to GPU in 2.1457s (16.7777GiB/s) | 20260505 20:15:11.116 | INFO | Downloaded 34GiB to GPU in 2.06s (16.5048GiB/s) | 20260505 20:15:16.263 | INFO | Analyzing chunk frames = [5999616,6000128) | 20260505 20:15:16.263 | INFO | MemInfo: { | 20260505 20:15:16.263 | INFO | "gpu" : " 17 GB", | 20260505 20:15:16.263 | INFO | "huge" : "", | 20260505 20:15:16.263 | INFO | "huge_shared" : " 27 GB", | 20260505 20:15:16.263 | INFO | "malloc" : " 14 GB", | 20260505 20:15:16.263 | INFO | "pinned" : " 29 GB" | 20260505 20:15:16.263 | INFO | } | 20260505 20:15:16.263 | INFO | Perf report for the last 5.14761 seconds: | 20260505 20:15:16.263 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:15:16.263 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:16.263 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:16.263 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:16.263 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 20:15:16.263 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:16.263 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:16.263 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260505 20:15:16.263 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:16.263 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:16.263 | INFO | Uploaded 36GiB to GPU in 2.12872s (16.9116GiB/s) | 20260505 20:15:16.263 | INFO | Downloaded 34GiB to GPU in 2.14396s (15.8585GiB/s) | 20260505 20:15:21.403 | INFO | Analyzing chunk frames = [6000128,6000640) | 20260505 20:15:21.403 | INFO | MemInfo: { | 20260505 20:15:21.403 | INFO | "gpu" : " 17 GB", | 20260505 20:15:21.403 | INFO | "huge" : "", | 20260505 20:15:21.403 | INFO | "huge_shared" : " 27 GB", | 20260505 20:15:21.403 | INFO | "malloc" : " 14 GB", | 20260505 20:15:21.403 | INFO | "pinned" : " 29 GB" | 20260505 20:15:21.403 | INFO | } | 20260505 20:15:21.403 | INFO | Perf report for the last 5.1404 seconds: | 20260505 20:15:21.403 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:15:21.403 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:21.403 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:21.404 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:21.404 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260505 20:15:21.404 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:21.404 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:21.404 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 20:15:21.404 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:21.404 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:21.404 | INFO | Uploaded 36GiB to GPU in 2.11352s (17.0332GiB/s) | 20260505 20:15:21.404 | INFO | Downloaded 34GiB to GPU in 2.00569s (16.9518GiB/s) | 20260505 20:15:21.538 | INFO | Starting a round of Baz IO | 20260505 20:15:24.706 | INFO | Populating RTMetrics report | 20260505 20:15:24.706 | INFO | RTMetrics report populated | 20260505 20:15:24.706 | INFO | Opening tmp RTMetrics json file | 20260505 20:15:24.707 | INFO | Serializing RTMetrics to json | 20260505 20:15:24.712 | INFO | Serialization done, swapping tmp file now... | 20260505 20:15:24.714 | INFO | RTMetrics done | 20260505 20:15:24.789 | INFO | Finished a round of Baz IO: Wrote 2.016 GiB in 3.251 seconds | 20260505 20:15:24.789 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.69% / 0.00% / 78.31% | 20260505 20:15:24.789 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.180944% / 3.97364e-06% / 8.74201e-05% | 20260505 20:15:24.789 | INFO | Baz IO overall: | 20260505 20:15:24.789 | INFO | BytesWritten : 3500206291003 | 20260505 20:15:24.789 | INFO | NumEvents : 1654981493865 | 20260505 20:15:26.542 | INFO | Analyzing chunk frames = [6000640,6001152) | 20260505 20:15:26.542 | INFO | MemInfo: { | 20260505 20:15:26.542 | INFO | "gpu" : " 17 GB", | 20260505 20:15:26.542 | INFO | "huge" : "", | 20260505 20:15:26.542 | INFO | "huge_shared" : " 27 GB", | 20260505 20:15:26.542 | INFO | "malloc" : " 14 GB", | 20260505 20:15:26.542 | INFO | "pinned" : " 29 GB" | 20260505 20:15:26.542 | INFO | } | 20260505 20:15:26.542 | INFO | Perf report for the last 5.13838 seconds: | 20260505 20:15:26.542 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:15:26.542 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:26.542 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:26.542 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:26.542 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 20:15:26.542 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:26.542 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260505 20:15:26.542 | INFO | PRE_HQ : 60.2%, 1.8, 55.3ms, 0 | 20260505 20:15:26.542 | INFO | BAZWRITER : 62.8%, 3.7, 118.5ms, 0 | 20260505 20:15:26.542 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 20:15:26.542 | INFO | Uploaded 36GiB to GPU in 2.17132s (16.5798GiB/s) | 20260505 20:15:26.542 | INFO | Downloaded 38GiB to GPU in 2.24513s (16.9255GiB/s) | 20260505 20:15:31.681 | INFO | Analyzing chunk frames = [6001152,6001664) | 20260505 20:15:31.681 | INFO | MemInfo: { | 20260505 20:15:31.681 | INFO | "gpu" : " 17 GB", | 20260505 20:15:31.681 | INFO | "huge" : "", | 20260505 20:15:31.681 | INFO | "huge_shared" : " 27 GB", | 20260505 20:15:31.681 | INFO | "malloc" : " 14 GB", | 20260505 20:15:31.681 | INFO | "pinned" : " 29 GB" | 20260505 20:15:31.681 | INFO | } | 20260505 20:15:31.681 | INFO | Perf report for the last 5.13962 seconds: | 20260505 20:15:31.681 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:15:31.681 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:31.681 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:31.681 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:31.682 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 20:15:31.682 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:31.682 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:31.682 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:15:31.682 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:31.682 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:31.682 | INFO | Uploaded 36GiB to GPU in 2.09066s (17.2194GiB/s) | 20260505 20:15:31.682 | INFO | Downloaded 34GiB to GPU in 2.02502s (16.79GiB/s) | 20260505 20:15:36.815 | INFO | Analyzing chunk frames = [6001664,6002176) | 20260505 20:15:36.815 | INFO | MemInfo: { | 20260505 20:15:36.815 | INFO | "gpu" : " 17 GB", | 20260505 20:15:36.815 | INFO | "huge" : "", | 20260505 20:15:36.815 | INFO | "huge_shared" : " 27 GB", | 20260505 20:15:36.815 | INFO | "malloc" : " 14 GB", | 20260505 20:15:36.815 | INFO | "pinned" : " 29 GB" | 20260505 20:15:36.815 | INFO | } | 20260505 20:15:36.815 | INFO | Perf report for the last 5.13381 seconds: | 20260505 20:15:36.815 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:15:36.815 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:36.815 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:36.815 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:36.815 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260505 20:15:36.815 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:36.815 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:36.815 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:15:36.815 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:36.815 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:36.816 | INFO | Uploaded 36GiB to GPU in 2.13281s (16.8792GiB/s) | 20260505 20:15:36.816 | INFO | Downloaded 34GiB to GPU in 2.02836s (16.7623GiB/s) | 20260505 20:15:41.963 | INFO | Analyzing chunk frames = [6002176,6002688) | 20260505 20:15:41.963 | INFO | MemInfo: { | 20260505 20:15:41.963 | INFO | "gpu" : " 17 GB", | 20260505 20:15:41.963 | INFO | "huge" : "", | 20260505 20:15:41.963 | INFO | "huge_shared" : " 27 GB", | 20260505 20:15:41.963 | INFO | "malloc" : " 14 GB", | 20260505 20:15:41.963 | INFO | "pinned" : " 29 GB" | 20260505 20:15:41.963 | INFO | } | 20260505 20:15:41.963 | INFO | Perf report for the last 5.1475 seconds: | 20260505 20:15:41.963 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:15:41.963 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:41.963 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:41.963 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:41.963 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260505 20:15:41.963 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:41.963 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:41.963 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:15:41.963 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:41.963 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:41.963 | INFO | Uploaded 36GiB to GPU in 2.20447s (16.3305GiB/s) | 20260505 20:15:41.963 | INFO | Downloaded 34GiB to GPU in 2.04451s (16.6299GiB/s) | 20260505 20:15:47.100 | INFO | Analyzing chunk frames = [6002688,6003200) | 20260505 20:15:47.100 | INFO | MemInfo: { | 20260505 20:15:47.100 | INFO | "gpu" : " 17 GB", | 20260505 20:15:47.100 | INFO | "huge" : "", | 20260505 20:15:47.100 | INFO | "huge_shared" : " 27 GB", | 20260505 20:15:47.101 | INFO | "malloc" : " 14 GB", | 20260505 20:15:47.101 | INFO | "pinned" : " 29 GB" | 20260505 20:15:47.101 | INFO | } | 20260505 20:15:47.101 | INFO | Perf report for the last 5.13785 seconds: | 20260505 20:15:47.101 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:15:47.101 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:47.101 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:47.101 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:47.101 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260505 20:15:47.101 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:47.101 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:47.101 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:15:47.101 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:47.101 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:47.101 | INFO | Uploaded 36GiB to GPU in 2.13046s (16.8977GiB/s) | 20260505 20:15:47.101 | INFO | Downloaded 34GiB to GPU in 2.06242s (16.4855GiB/s) | 20260505 20:15:52.231 | INFO | Analyzing chunk frames = [6003200,6003712) | 20260505 20:15:52.231 | INFO | MemInfo: { | 20260505 20:15:52.231 | INFO | "gpu" : " 17 GB", | 20260505 20:15:52.231 | INFO | "huge" : "", | 20260505 20:15:52.231 | INFO | "huge_shared" : " 27 GB", | 20260505 20:15:52.231 | INFO | "malloc" : " 14 GB", | 20260505 20:15:52.231 | INFO | "pinned" : " 29 GB" | 20260505 20:15:52.231 | INFO | } | 20260505 20:15:52.231 | INFO | Perf report for the last 5.13071 seconds: | 20260505 20:15:52.231 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:15:52.231 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:52.231 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:52.231 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:52.231 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 20:15:52.231 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:52.231 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:52.231 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:15:52.232 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:52.232 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:52.232 | INFO | Uploaded 36GiB to GPU in 2.13671s (16.8483GiB/s) | 20260505 20:15:52.232 | INFO | Downloaded 34GiB to GPU in 2.05654s (16.5326GiB/s) | 20260505 20:15:57.371 | INFO | Analyzing chunk frames = [6003712,6004224) | 20260505 20:15:57.371 | INFO | MemInfo: { | 20260505 20:15:57.371 | INFO | "gpu" : " 17 GB", | 20260505 20:15:57.371 | INFO | "huge" : "", | 20260505 20:15:57.371 | INFO | "huge_shared" : " 27 GB", | 20260505 20:15:57.371 | INFO | "malloc" : " 14 GB", | 20260505 20:15:57.371 | INFO | "pinned" : " 29 GB" | 20260505 20:15:57.371 | INFO | } | 20260505 20:15:57.371 | INFO | Perf report for the last 5.14002 seconds: | 20260505 20:15:57.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:15:57.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:57.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:57.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:57.371 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 20:15:57.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:57.372 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:57.372 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260505 20:15:57.372 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:57.372 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:15:57.372 | INFO | Uploaded 36GiB to GPU in 2.11335s (17.0346GiB/s) | 20260505 20:15:57.372 | INFO | Downloaded 34GiB to GPU in 2.13339s (15.9371GiB/s) | 20260505 20:16:02.518 | INFO | Analyzing chunk frames = [6004224,6004736) | 20260505 20:16:02.518 | INFO | MemInfo: { | 20260505 20:16:02.518 | INFO | "gpu" : " 17 GB", | 20260505 20:16:02.518 | INFO | "huge" : "", | 20260505 20:16:02.518 | INFO | "huge_shared" : " 27 GB", | 20260505 20:16:02.518 | INFO | "malloc" : " 14 GB", | 20260505 20:16:02.518 | INFO | "pinned" : " 29 GB" | 20260505 20:16:02.518 | INFO | } | 20260505 20:16:02.518 | INFO | Perf report for the last 5.14654 seconds: | 20260505 20:16:02.518 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:16:02.518 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:02.518 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:02.518 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:02.518 | INFO | ANALYSIS : 62.8%, 3.0, 98.3ms, 0 | 20260505 20:16:02.518 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:02.518 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:02.518 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260505 20:16:02.518 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:02.518 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:02.518 | INFO | Uploaded 36GiB to GPU in 2.20274s (16.3433GiB/s) | 20260505 20:16:02.518 | INFO | Downloaded 34GiB to GPU in 2.03618s (16.6979GiB/s) | 20260505 20:16:02.518 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 20:16:02.518 | INFO | Total mem usage: 15.3 GiB | 20260505 20:16:02.518 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 20:16:02.518 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.73%) | 20260505 20:16:02.518 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 20:16:02.518 | INFO | High Water Breakdown: | 20260505 20:16:02.518 | INFO | 60.9% : 8.58 GiB ( 90.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 20:16:02.518 | INFO | 39% : 5.5 GiB ( 82.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 20:16:02.518 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 20:16:02.518 | INFO | ------------------------------------------ | 20260505 20:16:02.518 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:16:02.519 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 20:16:02.519 | INFO | Total mem usage: 26.5 GiB | 20260505 20:16:02.519 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 20:16:02.519 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 20:16:02.519 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 20:16:02.519 | INFO | High Water Breakdown: | 20260505 20:16:02.519 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:16:02.519 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:16:02.519 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:16:02.519 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:16:02.519 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:16:02.519 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:16:02.519 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 20:16:02.519 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 20:16:02.519 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:16:02.519 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 20:16:02.519 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 20:16:02.519 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 20:16:02.519 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 20:16:02.519 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 20:16:02.519 | INFO | ------------------------------------------ | 20260505 20:16:02.519 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:16:02.519 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 20:16:02.519 | INFO | Total mem usage: 25.6 GiB | 20260505 20:16:02.519 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 20:16:02.519 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 20:16:02.519 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 20:16:02.519 | INFO | High Water Breakdown: | 20260505 20:16:02.519 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 20:16:02.519 | INFO | ------------------------------------------ | 20260505 20:16:02.519 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:16:02.519 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 20:16:02.519 | INFO | Total mem usage: 15.7 GiB | 20260505 20:16:02.519 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 20:16:02.519 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 20:16:02.519 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 20:16:02.519 | INFO | High Water Breakdown: | 20260505 20:16:02.519 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 20:16:02.519 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 20:16:02.519 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 20:16:02.519 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 20:16:02.519 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 20:16:02.519 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 20:16:02.519 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 20:16:02.519 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 20:16:02.519 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 20:16:02.519 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:16:02.519 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 20:16:02.519 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:16:02.519 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 20:16:02.519 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 20:16:02.519 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 20:16:02.519 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 20:16:02.519 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:16:02.519 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 20:16:02.519 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:16:02.519 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:16:02.519 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:16:02.519 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 20:16:02.519 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 20:16:02.519 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:16:02.519 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 20:16:02.519 | INFO | ------------------------------------------ | 20260505 20:16:02.519 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:16:02.519 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 20:16:02.519 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:16:02.519 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:16:02.519 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.47e+01, 1.34e+01 / 2.12e+00 | 20260505 20:16:02.519 | INFO | PerfReport: 0.29%, AggregateStats -- 9.25e-02 / 1.04e-01, 1.06e-02 / 5.67e-03 | 20260505 20:16:02.519 | INFO | PerfReport: 11.79%, Baseline -- 3.71e+00 / 4.54e+00, 5.63e-01 / 1.29e-01 | 20260505 20:16:02.519 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.51e+00, 1.90e-01 / 9.64e-02 | 20260505 20:16:02.519 | INFO | PerfReport: 15.96%, DME -- 4.02e+01 / 4.44e+01, 4.23e+00 / 2.18e+00 | 20260505 20:16:02.519 | INFO | PerfReport: 57.10%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.51e+00 / 1.18e+00 | 20260505 20:16:02.519 | INFO | PerfReport: 7.20%, PulseAccumulating -- 2.27e+00 / 2.85e+00, 4.30e-01 / 1.92e-01 | 20260505 20:16:02.519 | INFO | PerfReport: 3.24%, Metrics -- 1.02e+00 / 1.36e+00, 2.83e-01 / 1.13e-01 | 20260505 20:16:02.519 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.56e-01 / 4.25e-01, 6.35e-02 / 3.63e-02 | 20260505 20:16:02.519 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 20:16:02.519 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:16:02.519 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:16:02.519 | INFO | PerfReport: Full Execution -- 9.94e+01 / -nan, 2.52e+01 / 0.00e+00 | 20260505 20:16:02.519 | INFO | PerfReport: 22.47%, Upload -- 2.23e+01 / 0.00e+00, 7.69e+00 / 0.00e+00 | 20260505 20:16:02.519 | INFO | PerfReport: 21.80%, Download -- 2.17e+01 / 0.00e+00, 7.68e+00 / 0.00e+00 | 20260505 20:16:02.519 | INFO | PerfReport: 31.68%, Compute -- 3.15e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 20:16:02.520 | INFO | PerfReport: 2.30%, LockUpload -- 2.29e+00 / 0.00e+00, 7.98e+00 / 0.00e+00 | 20260505 20:16:02.520 | INFO | PerfReport: 1.35%, LockDownload -- 1.34e+00 / 0.00e+00, 2.89e+00 / 0.00e+00 | 20260505 20:16:02.520 | INFO | PerfReport: 20.39%, LockCompute -- 2.03e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 20:16:02.520 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.39e-02 / 0.00e+00 | 20260505 20:16:02.647 | INFO | Starting a round of Baz IO | 20260505 20:16:05.815 | INFO | Populating RTMetrics report | 20260505 20:16:05.815 | INFO | RTMetrics report populated | 20260505 20:16:05.818 | INFO | Opening tmp RTMetrics json file | 20260505 20:16:05.818 | INFO | Serializing RTMetrics to json | 20260505 20:16:05.820 | INFO | Serialization done, swapping tmp file now... | 20260505 20:16:05.820 | INFO | RTMetrics done | 20260505 20:16:05.901 | INFO | Finished a round of Baz IO: Wrote 2.016 GiB in 3.254 seconds | 20260505 20:16:05.901 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.69% / 0.00% / 78.31% | 20260505 20:16:05.901 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.180097% / 0% / 8.34465e-05% | 20260505 20:16:05.901 | INFO | Baz IO overall: | 20260505 20:16:05.901 | INFO | BytesWritten : 3502371061163 | 20260505 20:16:05.901 | INFO | NumEvents : 1655983994619 | 20260505 20:16:07.660 | INFO | Analyzing chunk frames = [6004736,6005248) | 20260505 20:16:07.660 | INFO | MemInfo: { | 20260505 20:16:07.660 | INFO | "gpu" : " 17 GB", | 20260505 20:16:07.660 | INFO | "huge" : "", | 20260505 20:16:07.660 | INFO | "huge_shared" : " 27 GB", | 20260505 20:16:07.660 | INFO | "malloc" : " 14 GB", | 20260505 20:16:07.660 | INFO | "pinned" : " 29 GB" | 20260505 20:16:07.660 | INFO | } | 20260505 20:16:07.660 | INFO | Perf report for the last 5.14205 seconds: | 20260505 20:16:07.660 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:16:07.660 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:07.660 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:07.660 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:07.660 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 20:16:07.660 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:07.660 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260505 20:16:07.660 | INFO | PRE_HQ : 60.3%, 1.8, 55.2ms, 0 | 20260505 20:16:07.660 | INFO | BAZWRITER : 62.7%, 3.7, 118.1ms, 0 | 20260505 20:16:07.660 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 20:16:07.660 | INFO | Uploaded 36GiB to GPU in 2.15098s (16.7366GiB/s) | 20260505 20:16:07.660 | INFO | Downloaded 38GiB to GPU in 2.24371s (16.9362GiB/s) | 20260505 20:16:12.795 | INFO | Analyzing chunk frames = [6005248,6005760) | 20260505 20:16:12.795 | INFO | MemInfo: { | 20260505 20:16:12.795 | INFO | "gpu" : " 17 GB", | 20260505 20:16:12.795 | INFO | "huge" : "", | 20260505 20:16:12.795 | INFO | "huge_shared" : " 27 GB", | 20260505 20:16:12.795 | INFO | "malloc" : " 14 GB", | 20260505 20:16:12.795 | INFO | "pinned" : " 29 GB" | 20260505 20:16:12.795 | INFO | } | 20260505 20:16:12.795 | INFO | Perf report for the last 5.13502 seconds: | 20260505 20:16:12.795 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:16:12.795 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:12.795 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:12.795 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:12.795 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 20:16:12.795 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:12.795 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:12.795 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 20:16:12.795 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:12.795 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:12.795 | INFO | Uploaded 36GiB to GPU in 2.08406s (17.274GiB/s) | 20260505 20:16:12.795 | INFO | Downloaded 34GiB to GPU in 2.02359s (16.8018GiB/s) | 20260505 20:16:17.930 | INFO | Analyzing chunk frames = [6005760,6006272) | 20260505 20:16:17.930 | INFO | MemInfo: { | 20260505 20:16:17.930 | INFO | "gpu" : " 17 GB", | 20260505 20:16:17.930 | INFO | "huge" : "", | 20260505 20:16:17.930 | INFO | "huge_shared" : " 27 GB", | 20260505 20:16:17.930 | INFO | "malloc" : " 14 GB", | 20260505 20:16:17.930 | INFO | "pinned" : " 29 GB" | 20260505 20:16:17.930 | INFO | } | 20260505 20:16:17.930 | INFO | Perf report for the last 5.13479 seconds: | 20260505 20:16:17.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:16:17.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:17.930 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:17.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:17.930 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260505 20:16:17.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:17.930 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:17.930 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 20:16:17.930 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:17.930 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:17.930 | INFO | Uploaded 36GiB to GPU in 2.13972s (16.8246GiB/s) | 20260505 20:16:17.930 | INFO | Downloaded 34GiB to GPU in 2.03406s (16.7153GiB/s) | 20260505 20:16:23.072 | INFO | Analyzing chunk frames = [6006272,6006784) | 20260505 20:16:23.073 | INFO | MemInfo: { | 20260505 20:16:23.073 | INFO | "gpu" : " 17 GB", | 20260505 20:16:23.073 | INFO | "huge" : "", | 20260505 20:16:23.073 | INFO | "huge_shared" : " 27 GB", | 20260505 20:16:23.073 | INFO | "malloc" : " 14 GB", | 20260505 20:16:23.073 | INFO | "pinned" : " 29 GB" | 20260505 20:16:23.073 | INFO | } | 20260505 20:16:23.073 | INFO | Perf report for the last 5.14282 seconds: | 20260505 20:16:23.073 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:16:23.073 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:23.073 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:23.073 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:23.073 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260505 20:16:23.073 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:23.073 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:23.073 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:16:23.073 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:23.073 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:23.073 | INFO | Uploaded 36GiB to GPU in 2.18761s (16.4563GiB/s) | 20260505 20:16:23.073 | INFO | Downloaded 34GiB to GPU in 2.0678s (16.4426GiB/s) | 20260505 20:16:28.211 | INFO | Analyzing chunk frames = [6006784,6007296) | 20260505 20:16:28.211 | INFO | MemInfo: { | 20260505 20:16:28.211 | INFO | "gpu" : " 17 GB", | 20260505 20:16:28.211 | INFO | "huge" : "", | 20260505 20:16:28.211 | INFO | "huge_shared" : " 27 GB", | 20260505 20:16:28.211 | INFO | "malloc" : " 14 GB", | 20260505 20:16:28.211 | INFO | "pinned" : " 29 GB" | 20260505 20:16:28.211 | INFO | } | 20260505 20:16:28.211 | INFO | Perf report for the last 5.13866 seconds: | 20260505 20:16:28.211 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:16:28.211 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:28.211 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:28.211 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:28.211 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 20:16:28.211 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:28.211 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:28.211 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:16:28.211 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:28.211 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:28.212 | INFO | Uploaded 36GiB to GPU in 2.13315s (16.8765GiB/s) | 20260505 20:16:28.212 | INFO | Downloaded 34GiB to GPU in 2.06055s (16.5005GiB/s) | 20260505 20:16:33.357 | INFO | Analyzing chunk frames = [6007296,6007808) | 20260505 20:16:33.357 | INFO | MemInfo: { | 20260505 20:16:33.357 | INFO | "gpu" : " 17 GB", | 20260505 20:16:33.357 | INFO | "huge" : "", | 20260505 20:16:33.357 | INFO | "huge_shared" : " 27 GB", | 20260505 20:16:33.357 | INFO | "malloc" : " 14 GB", | 20260505 20:16:33.357 | INFO | "pinned" : " 29 GB" | 20260505 20:16:33.357 | INFO | } | 20260505 20:16:33.357 | INFO | Perf report for the last 5.14572 seconds: | 20260505 20:16:33.357 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:16:33.357 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:33.357 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:33.357 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:33.357 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260505 20:16:33.357 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:33.357 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:33.357 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260505 20:16:33.357 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:33.357 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:33.357 | INFO | Uploaded 36GiB to GPU in 2.16076s (16.6608GiB/s) | 20260505 20:16:33.357 | INFO | Downloaded 34GiB to GPU in 2.06592s (16.4576GiB/s) | 20260505 20:16:38.493 | INFO | Analyzing chunk frames = [6007808,6008320) | 20260505 20:16:38.493 | INFO | MemInfo: { | 20260505 20:16:38.493 | INFO | "gpu" : " 17 GB", | 20260505 20:16:38.493 | INFO | "huge" : "", | 20260505 20:16:38.494 | INFO | "huge_shared" : " 27 GB", | 20260505 20:16:38.494 | INFO | "malloc" : " 14 GB", | 20260505 20:16:38.494 | INFO | "pinned" : " 29 GB" | 20260505 20:16:38.494 | INFO | } | 20260505 20:16:38.494 | INFO | Perf report for the last 5.13661 seconds: | 20260505 20:16:38.494 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:16:38.494 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:38.494 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:38.494 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:38.494 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 20:16:38.494 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:38.494 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:38.494 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 20:16:38.494 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:38.494 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:38.494 | INFO | Uploaded 36GiB to GPU in 2.13101s (16.8934GiB/s) | 20260505 20:16:38.494 | INFO | Downloaded 34GiB to GPU in 2.14043s (15.8847GiB/s) | 20260505 20:16:43.627 | INFO | Analyzing chunk frames = [6008320,6008832) | 20260505 20:16:43.627 | INFO | MemInfo: { | 20260505 20:16:43.627 | INFO | "gpu" : " 17 GB", | 20260505 20:16:43.627 | INFO | "huge" : "", | 20260505 20:16:43.627 | INFO | "huge_shared" : " 27 GB", | 20260505 20:16:43.627 | INFO | "malloc" : " 14 GB", | 20260505 20:16:43.627 | INFO | "pinned" : " 29 GB" | 20260505 20:16:43.627 | INFO | } | 20260505 20:16:43.627 | INFO | Perf report for the last 5.13322 seconds: | 20260505 20:16:43.627 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:16:43.627 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:43.627 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:43.627 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:43.627 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260505 20:16:43.627 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:43.627 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:43.627 | INFO | PRE_HQ : 27.7%, 1.0, 14.3ms, 0 | 20260505 20:16:43.627 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:43.627 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:43.627 | INFO | Uploaded 36GiB to GPU in 2.13655s (16.8496GiB/s) | 20260505 20:16:43.627 | INFO | Downloaded 34GiB to GPU in 2.02259s (16.8102GiB/s) | 20260505 20:16:43.760 | INFO | Starting a round of Baz IO | 20260505 20:16:46.929 | INFO | Populating RTMetrics report | 20260505 20:16:46.929 | INFO | RTMetrics report populated | 20260505 20:16:46.929 | INFO | Opening tmp RTMetrics json file | 20260505 20:16:46.930 | INFO | Serializing RTMetrics to json | 20260505 20:16:46.932 | INFO | Serialization done, swapping tmp file now... | 20260505 20:16:46.932 | INFO | RTMetrics done | 20260505 20:16:47.016 | INFO | Finished a round of Baz IO: Wrote 2.02 GiB in 3.255 seconds | 20260505 20:16:47.016 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.69% / 0.00% / 78.31% | 20260505 20:16:47.016 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.180336% / 0% / 5.96046e-05% | 20260505 20:16:47.016 | INFO | Baz IO overall: | 20260505 20:16:47.016 | INFO | BytesWritten : 3504540308867 | 20260505 20:16:47.016 | INFO | NumEvents : 1656990453538 | 20260505 20:16:48.773 | INFO | Analyzing chunk frames = [6008832,6009344) | 20260505 20:16:48.773 | INFO | MemInfo: { | 20260505 20:16:48.773 | INFO | "gpu" : " 17 GB", | 20260505 20:16:48.773 | INFO | "huge" : "", | 20260505 20:16:48.773 | INFO | "huge_shared" : " 27 GB", | 20260505 20:16:48.773 | INFO | "malloc" : " 14 GB", | 20260505 20:16:48.773 | INFO | "pinned" : " 29 GB" | 20260505 20:16:48.773 | INFO | } | 20260505 20:16:48.773 | INFO | Perf report for the last 5.14579 seconds: | 20260505 20:16:48.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:16:48.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:48.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:48.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:48.773 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 20:16:48.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:48.773 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260505 20:16:48.773 | INFO | PRE_HQ : 60.2%, 1.8, 55.4ms, 0 | 20260505 20:16:48.773 | INFO | BAZWRITER : 62.7%, 3.7, 119.7ms, 0 | 20260505 20:16:48.773 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 20:16:48.773 | INFO | Uploaded 36GiB to GPU in 2.15459s (16.7085GiB/s) | 20260505 20:16:48.773 | INFO | Downloaded 38GiB to GPU in 2.24644s (16.9156GiB/s) | 20260505 20:16:53.914 | INFO | Analyzing chunk frames = [6009344,6009856) | 20260505 20:16:53.914 | INFO | MemInfo: { | 20260505 20:16:53.914 | INFO | "gpu" : " 17 GB", | 20260505 20:16:53.914 | INFO | "huge" : "", | 20260505 20:16:53.914 | INFO | "huge_shared" : " 27 GB", | 20260505 20:16:53.914 | INFO | "malloc" : " 14 GB", | 20260505 20:16:53.914 | INFO | "pinned" : " 29 GB" | 20260505 20:16:53.914 | INFO | } | 20260505 20:16:53.914 | INFO | Perf report for the last 5.14154 seconds: | 20260505 20:16:53.914 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:16:53.914 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:53.914 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:53.914 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:53.914 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 20:16:53.914 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:53.914 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:53.914 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 20:16:53.914 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:53.914 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:53.915 | INFO | Uploaded 36GiB to GPU in 2.07299s (17.3662GiB/s) | 20260505 20:16:53.915 | INFO | Downloaded 34GiB to GPU in 2.02113s (16.8223GiB/s) | 20260505 20:16:59.048 | INFO | Analyzing chunk frames = [6009856,6010368) | 20260505 20:16:59.048 | INFO | MemInfo: { | 20260505 20:16:59.048 | INFO | "gpu" : " 17 GB", | 20260505 20:16:59.048 | INFO | "huge" : "", | 20260505 20:16:59.048 | INFO | "huge_shared" : " 27 GB", | 20260505 20:16:59.048 | INFO | "malloc" : " 14 GB", | 20260505 20:16:59.048 | INFO | "pinned" : " 29 GB" | 20260505 20:16:59.048 | INFO | } | 20260505 20:16:59.048 | INFO | Perf report for the last 5.13404 seconds: | 20260505 20:16:59.048 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:16:59.048 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:59.048 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:59.048 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:59.048 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260505 20:16:59.048 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:59.049 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:59.049 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:16:59.049 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:59.049 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:16:59.049 | INFO | Uploaded 36GiB to GPU in 2.1247s (16.9435GiB/s) | 20260505 20:16:59.049 | INFO | Downloaded 34GiB to GPU in 2.01739s (16.8534GiB/s) | 20260505 20:17:04.178 | INFO | Analyzing chunk frames = [6010368,6010880) | 20260505 20:17:04.178 | INFO | MemInfo: { | 20260505 20:17:04.178 | INFO | "gpu" : " 17 GB", | 20260505 20:17:04.178 | INFO | "huge" : "", | 20260505 20:17:04.178 | INFO | "huge_shared" : " 27 GB", | 20260505 20:17:04.178 | INFO | "malloc" : " 14 GB", | 20260505 20:17:04.178 | INFO | "pinned" : " 29 GB" | 20260505 20:17:04.178 | INFO | } | 20260505 20:17:04.178 | INFO | Perf report for the last 5.12947 seconds: | 20260505 20:17:04.178 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:17:04.178 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:04.178 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:04.178 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:04.178 | INFO | ANALYSIS : 62.6%, 3.0, 97.0ms, 0 | 20260505 20:17:04.178 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:04.178 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:04.178 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 20:17:04.178 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:04.178 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:04.178 | INFO | Uploaded 36GiB to GPU in 2.20539s (16.3237GiB/s) | 20260505 20:17:04.178 | INFO | Downloaded 34GiB to GPU in 2.04024s (16.6647GiB/s) | 20260505 20:17:09.324 | INFO | Analyzing chunk frames = [6010880,6011392) | 20260505 20:17:09.324 | INFO | MemInfo: { | 20260505 20:17:09.324 | INFO | "gpu" : " 17 GB", | 20260505 20:17:09.324 | INFO | "huge" : "", | 20260505 20:17:09.324 | INFO | "huge_shared" : " 27 GB", | 20260505 20:17:09.324 | INFO | "malloc" : " 14 GB", | 20260505 20:17:09.324 | INFO | "pinned" : " 29 GB" | 20260505 20:17:09.324 | INFO | } | 20260505 20:17:09.325 | INFO | Perf report for the last 5.14667 seconds: | 20260505 20:17:09.325 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:17:09.325 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:09.325 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:09.325 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:09.325 | INFO | ANALYSIS : 61.9%, 3.0, 96.8ms, 0 | 20260505 20:17:09.325 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:09.325 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:09.325 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 20:17:09.325 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:09.325 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:09.325 | INFO | Uploaded 36GiB to GPU in 2.14155s (16.8103GiB/s) | 20260505 20:17:09.325 | INFO | Downloaded 34GiB to GPU in 2.06635s (16.4542GiB/s) | 20260505 20:17:14.470 | INFO | Analyzing chunk frames = [6011392,6011904) | 20260505 20:17:14.470 | INFO | MemInfo: { | 20260505 20:17:14.470 | INFO | "gpu" : " 17 GB", | 20260505 20:17:14.470 | INFO | "huge" : "", | 20260505 20:17:14.470 | INFO | "huge_shared" : " 27 GB", | 20260505 20:17:14.470 | INFO | "malloc" : " 14 GB", | 20260505 20:17:14.470 | INFO | "pinned" : " 29 GB" | 20260505 20:17:14.470 | INFO | } | 20260505 20:17:14.470 | INFO | Perf report for the last 5.14561 seconds: | 20260505 20:17:14.470 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:17:14.470 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:14.470 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:14.470 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:14.470 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260505 20:17:14.470 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:14.470 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:14.470 | INFO | PRE_HQ : 27.2%, 1.0, 14.2ms, 0 | 20260505 20:17:14.470 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:14.470 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:14.470 | INFO | Uploaded 36GiB to GPU in 2.14711s (16.7668GiB/s) | 20260505 20:17:14.470 | INFO | Downloaded 34GiB to GPU in 2.05817s (16.5195GiB/s) | 20260505 20:17:19.610 | INFO | Analyzing chunk frames = [6011904,6012416) | 20260505 20:17:19.610 | INFO | MemInfo: { | 20260505 20:17:19.610 | INFO | "gpu" : " 17 GB", | 20260505 20:17:19.610 | INFO | "huge" : "", | 20260505 20:17:19.610 | INFO | "huge_shared" : " 27 GB", | 20260505 20:17:19.610 | INFO | "malloc" : " 14 GB", | 20260505 20:17:19.610 | INFO | "pinned" : " 29 GB" | 20260505 20:17:19.610 | INFO | } | 20260505 20:17:19.610 | INFO | Perf report for the last 5.1402 seconds: | 20260505 20:17:19.610 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:17:19.610 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:19.610 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:19.610 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:19.610 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 20:17:19.610 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:19.611 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:19.611 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 20:17:19.611 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:19.611 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:19.611 | INFO | Uploaded 36GiB to GPU in 2.16171s (16.6535GiB/s) | 20260505 20:17:19.611 | INFO | Downloaded 34GiB to GPU in 2.1263s (15.9903GiB/s) | 20260505 20:17:24.741 | INFO | Analyzing chunk frames = [6012416,6012928) | 20260505 20:17:24.741 | INFO | MemInfo: { | 20260505 20:17:24.741 | INFO | "gpu" : " 17 GB", | 20260505 20:17:24.741 | INFO | "huge" : "", | 20260505 20:17:24.742 | INFO | "huge_shared" : " 27 GB", | 20260505 20:17:24.742 | INFO | "malloc" : " 14 GB", | 20260505 20:17:24.742 | INFO | "pinned" : " 29 GB" | 20260505 20:17:24.742 | INFO | } | 20260505 20:17:24.742 | INFO | Perf report for the last 5.1313 seconds: | 20260505 20:17:24.742 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:17:24.742 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:24.742 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:24.742 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:24.742 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 20:17:24.742 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:24.742 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:24.742 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:17:24.742 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:24.742 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:24.742 | INFO | Uploaded 36GiB to GPU in 2.13375s (16.8717GiB/s) | 20260505 20:17:24.742 | INFO | Downloaded 34GiB to GPU in 2.00648s (16.9451GiB/s) | 20260505 20:17:24.742 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 20:17:24.742 | INFO | Total mem usage: 15.3 GiB | 20260505 20:17:24.742 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 20:17:24.742 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.22%) | 20260505 20:17:24.742 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 20:17:24.742 | INFO | High Water Breakdown: | 20260505 20:17:24.742 | INFO | 61% : 8.64 GiB ( 90.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 20:17:24.742 | INFO | 38.9% : 5.5 GiB ( 82.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 20:17:24.742 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 20:17:24.742 | INFO | ------------------------------------------ | 20260505 20:17:24.742 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:17:24.742 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 20:17:24.742 | INFO | Total mem usage: 26.5 GiB | 20260505 20:17:24.743 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 20:17:24.743 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 20:17:24.743 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 20:17:24.743 | INFO | High Water Breakdown: | 20260505 20:17:24.743 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:17:24.743 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:17:24.743 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:17:24.743 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:17:24.743 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:17:24.743 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:17:24.743 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 20:17:24.743 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 20:17:24.743 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:17:24.743 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 20:17:24.743 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 20:17:24.743 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 20:17:24.743 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 20:17:24.743 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 20:17:24.743 | INFO | ------------------------------------------ | 20260505 20:17:24.743 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:17:24.743 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 20:17:24.743 | INFO | Total mem usage: 25.6 GiB | 20260505 20:17:24.743 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 20:17:24.743 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 20:17:24.743 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 20:17:24.743 | INFO | High Water Breakdown: | 20260505 20:17:24.743 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 20:17:24.743 | INFO | ------------------------------------------ | 20260505 20:17:24.743 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:17:24.743 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 20:17:24.743 | INFO | Total mem usage: 15.7 GiB | 20260505 20:17:24.743 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 20:17:24.743 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 20:17:24.743 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 20:17:24.743 | INFO | High Water Breakdown: | 20260505 20:17:24.743 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 20:17:24.743 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 20:17:24.743 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 20:17:24.743 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 20:17:24.743 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 20:17:24.743 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 20:17:24.743 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 20:17:24.743 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 20:17:24.743 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 20:17:24.743 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:17:24.743 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 20:17:24.743 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:17:24.743 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 20:17:24.743 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 20:17:24.743 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 20:17:24.743 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 20:17:24.743 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:17:24.743 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 20:17:24.743 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:17:24.743 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:17:24.743 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:17:24.743 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 20:17:24.743 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 20:17:24.743 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:17:24.743 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 20:17:24.743 | INFO | ------------------------------------------ | 20260505 20:17:24.743 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:17:24.743 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 20:17:24.743 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:17:24.743 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:17:24.743 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.45e+01, 1.35e+01 / 1.70e+00 | 20260505 20:17:24.743 | INFO | PerfReport: 0.29%, AggregateStats -- 9.25e-02 / 1.04e-01, 1.02e-02 / 5.63e-03 | 20260505 20:17:24.743 | INFO | PerfReport: 11.82%, Baseline -- 3.72e+00 / 4.58e+00, 5.74e-01 / 8.97e-02 | 20260505 20:17:24.743 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.51e+00, 1.90e-01 / 9.69e-02 | 20260505 20:17:24.743 | INFO | PerfReport: 15.95%, DME -- 4.02e+01 / 4.43e+01, 4.00e+00 / 1.82e+00 | 20260505 20:17:24.743 | INFO | PerfReport: 57.03%, FrameLabeling -- 1.80e+01 / 1.93e+01, 1.49e+00 / 9.97e-01 | 20260505 20:17:24.743 | INFO | PerfReport: 7.24%, PulseAccumulating -- 2.28e+00 / 2.88e+00, 4.44e-01 / 1.50e-01 | 20260505 20:17:24.743 | INFO | PerfReport: 3.25%, Metrics -- 1.02e+00 / 1.36e+00, 2.83e-01 / 1.24e-01 | 20260505 20:17:24.743 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.56e-01 / 4.17e-01, 6.29e-02 / 3.77e-02 | 20260505 20:17:24.743 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 20:17:24.743 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:17:24.743 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:17:24.743 | INFO | PerfReport: Full Execution -- 9.94e+01 / -nan, 2.53e+01 / 0.00e+00 | 20260505 20:17:24.743 | INFO | PerfReport: 22.45%, Upload -- 2.23e+01 / 0.00e+00, 7.68e+00 / 0.00e+00 | 20260505 20:17:24.743 | INFO | PerfReport: 21.78%, Download -- 2.16e+01 / 0.00e+00, 7.70e+00 / 0.00e+00 | 20260505 20:17:24.743 | INFO | PerfReport: 31.69%, Compute -- 3.15e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 20:17:24.743 | INFO | PerfReport: 2.27%, LockUpload -- 2.25e+00 / 0.00e+00, 7.87e+00 / 0.00e+00 | 20260505 20:17:24.743 | INFO | PerfReport: 1.36%, LockDownload -- 1.35e+00 / 0.00e+00, 2.90e+00 / 0.00e+00 | 20260505 20:17:24.743 | INFO | PerfReport: 20.44%, LockCompute -- 2.03e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 20:17:24.743 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.62e-02 / 0.00e+00 | 20260505 20:17:24.875 | INFO | Starting a round of Baz IO | 20260505 20:17:28.049 | INFO | Populating RTMetrics report | 20260505 20:17:28.049 | INFO | RTMetrics report populated | 20260505 20:17:28.049 | INFO | Opening tmp RTMetrics json file | 20260505 20:17:28.050 | INFO | Serializing RTMetrics to json | 20260505 20:17:28.051 | INFO | Serialization done, swapping tmp file now... | 20260505 20:17:28.051 | INFO | RTMetrics done | 20260505 20:17:28.137 | INFO | Finished a round of Baz IO: Wrote 2.018 GiB in 3.261 seconds | 20260505 20:17:28.137 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.69% / 0.00% / 78.31% | 20260505 20:17:28.137 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.179207% / 1.19209e-05% / 8.74201e-05% | 20260505 20:17:28.137 | INFO | Baz IO overall: | 20260505 20:17:28.137 | INFO | BytesWritten : 3506706656005 | 20260505 20:17:28.137 | INFO | NumEvents : 1657994214521 | 20260505 20:17:29.878 | INFO | Analyzing chunk frames = [6012928,6013440) | 20260505 20:17:29.878 | INFO | MemInfo: { | 20260505 20:17:29.878 | INFO | "gpu" : " 17 GB", | 20260505 20:17:29.878 | INFO | "huge" : "", | 20260505 20:17:29.878 | INFO | "huge_shared" : " 27 GB", | 20260505 20:17:29.878 | INFO | "malloc" : " 14 GB", | 20260505 20:17:29.878 | INFO | "pinned" : " 29 GB" | 20260505 20:17:29.878 | INFO | } | 20260505 20:17:29.878 | INFO | Perf report for the last 5.13652 seconds: | 20260505 20:17:29.878 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:17:29.878 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:29.878 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:29.878 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:29.878 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 20:17:29.878 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:29.878 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260505 20:17:29.878 | INFO | PRE_HQ : 60.3%, 1.8, 55.2ms, 0 | 20260505 20:17:29.878 | INFO | BAZWRITER : 63.0%, 3.7, 118.6ms, 0 | 20260505 20:17:29.878 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 20:17:29.879 | INFO | Uploaded 36GiB to GPU in 2.17393s (16.5598GiB/s) | 20260505 20:17:29.879 | INFO | Downloaded 38GiB to GPU in 2.24762s (16.9068GiB/s) | 20260505 20:17:35.029 | INFO | Analyzing chunk frames = [6013440,6013952) | 20260505 20:17:35.029 | INFO | MemInfo: { | 20260505 20:17:35.029 | INFO | "gpu" : " 17 GB", | 20260505 20:17:35.029 | INFO | "huge" : "", | 20260505 20:17:35.029 | INFO | "huge_shared" : " 27 GB", | 20260505 20:17:35.029 | INFO | "malloc" : " 14 GB", | 20260505 20:17:35.029 | INFO | "pinned" : " 29 GB" | 20260505 20:17:35.029 | INFO | } | 20260505 20:17:35.029 | INFO | Perf report for the last 5.15081 seconds: | 20260505 20:17:35.029 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:17:35.029 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:35.029 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:35.029 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:35.029 | INFO | ANALYSIS : 62.4%, 3.0, 97.7ms, 0 | 20260505 20:17:35.029 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:35.029 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:35.029 | INFO | PRE_HQ : 27.0%, 1.0, 14.1ms, 0 | 20260505 20:17:35.029 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:35.029 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:35.029 | INFO | Uploaded 36GiB to GPU in 2.1401s (16.8216GiB/s) | 20260505 20:17:35.029 | INFO | Downloaded 34GiB to GPU in 2.04079s (16.6603GiB/s) | 20260505 20:17:40.165 | INFO | Analyzing chunk frames = [6013952,6014464) | 20260505 20:17:40.165 | INFO | MemInfo: { | 20260505 20:17:40.165 | INFO | "gpu" : " 17 GB", | 20260505 20:17:40.165 | INFO | "huge" : "", | 20260505 20:17:40.165 | INFO | "huge_shared" : " 27 GB", | 20260505 20:17:40.165 | INFO | "malloc" : " 14 GB", | 20260505 20:17:40.165 | INFO | "pinned" : " 29 GB" | 20260505 20:17:40.165 | INFO | } | 20260505 20:17:40.165 | INFO | Perf report for the last 5.13608 seconds: | 20260505 20:17:40.165 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:17:40.165 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:40.165 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:40.165 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:40.165 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260505 20:17:40.165 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:40.165 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:40.165 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:17:40.165 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:40.165 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:40.165 | INFO | Uploaded 36GiB to GPU in 2.13355s (16.8733GiB/s) | 20260505 20:17:40.166 | INFO | Downloaded 34GiB to GPU in 2.02373s (16.8007GiB/s) | 20260505 20:17:45.307 | INFO | Analyzing chunk frames = [6014464,6014976) | 20260505 20:17:45.307 | INFO | MemInfo: { | 20260505 20:17:45.307 | INFO | "gpu" : " 17 GB", | 20260505 20:17:45.307 | INFO | "huge" : "", | 20260505 20:17:45.307 | INFO | "huge_shared" : " 27 GB", | 20260505 20:17:45.307 | INFO | "malloc" : " 14 GB", | 20260505 20:17:45.307 | INFO | "pinned" : " 29 GB" | 20260505 20:17:45.307 | INFO | } | 20260505 20:17:45.308 | INFO | Perf report for the last 5.14238 seconds: | 20260505 20:17:45.308 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:17:45.308 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:45.308 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:45.308 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:45.308 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260505 20:17:45.308 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:45.308 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:45.308 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:17:45.308 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:45.308 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:45.308 | INFO | Uploaded 36GiB to GPU in 2.21585s (16.2466GiB/s) | 20260505 20:17:45.308 | INFO | Downloaded 34GiB to GPU in 2.06189s (16.4897GiB/s) | 20260505 20:17:50.439 | INFO | Analyzing chunk frames = [6014976,6015488) | 20260505 20:17:50.439 | INFO | MemInfo: { | 20260505 20:17:50.439 | INFO | "gpu" : " 17 GB", | 20260505 20:17:50.439 | INFO | "huge" : "", | 20260505 20:17:50.439 | INFO | "huge_shared" : " 27 GB", | 20260505 20:17:50.439 | INFO | "malloc" : " 14 GB", | 20260505 20:17:50.440 | INFO | "pinned" : " 29 GB" | 20260505 20:17:50.440 | INFO | } | 20260505 20:17:50.440 | INFO | Perf report for the last 5.13207 seconds: | 20260505 20:17:50.440 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:17:50.440 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:50.440 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:50.440 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:50.440 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260505 20:17:50.440 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:50.440 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:50.440 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 20:17:50.440 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:50.440 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:50.440 | INFO | Uploaded 36GiB to GPU in 2.13583s (16.8553GiB/s) | 20260505 20:17:50.440 | INFO | Downloaded 34GiB to GPU in 2.06311s (16.48GiB/s) | 20260505 20:17:55.581 | INFO | Analyzing chunk frames = [6015488,6016000) | 20260505 20:17:55.581 | INFO | MemInfo: { | 20260505 20:17:55.581 | INFO | "gpu" : " 17 GB", | 20260505 20:17:55.581 | INFO | "huge" : "", | 20260505 20:17:55.581 | INFO | "huge_shared" : " 27 GB", | 20260505 20:17:55.581 | INFO | "malloc" : " 14 GB", | 20260505 20:17:55.581 | INFO | "pinned" : " 29 GB" | 20260505 20:17:55.581 | INFO | } | 20260505 20:17:55.581 | INFO | Perf report for the last 5.1417 seconds: | 20260505 20:17:55.581 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:17:55.581 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:55.581 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:55.581 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:55.581 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 20:17:55.581 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:55.581 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:55.581 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 20:17:55.581 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:55.581 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:17:55.582 | INFO | Uploaded 36GiB to GPU in 2.15629s (16.6953GiB/s) | 20260505 20:17:55.582 | INFO | Downloaded 34GiB to GPU in 2.05773s (16.523GiB/s) | 20260505 20:18:00.722 | INFO | Analyzing chunk frames = [6016000,6016512) | 20260505 20:18:00.723 | INFO | MemInfo: { | 20260505 20:18:00.723 | INFO | "gpu" : " 17 GB", | 20260505 20:18:00.723 | INFO | "huge" : "", | 20260505 20:18:00.723 | INFO | "huge_shared" : " 27 GB", | 20260505 20:18:00.723 | INFO | "malloc" : " 14 GB", | 20260505 20:18:00.723 | INFO | "pinned" : " 29 GB" | 20260505 20:18:00.723 | INFO | } | 20260505 20:18:00.723 | INFO | Perf report for the last 5.14144 seconds: | 20260505 20:18:00.723 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:18:00.723 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:00.723 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:00.723 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:00.723 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 20:18:00.723 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:00.723 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:00.723 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 20:18:00.723 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:00.723 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:00.723 | INFO | Uploaded 36GiB to GPU in 2.11285s (17.0386GiB/s) | 20260505 20:18:00.723 | INFO | Downloaded 34GiB to GPU in 2.14127s (15.8784GiB/s) | 20260505 20:18:05.863 | INFO | Analyzing chunk frames = [6016512,6017024) | 20260505 20:18:05.863 | INFO | MemInfo: { | 20260505 20:18:05.863 | INFO | "gpu" : " 17 GB", | 20260505 20:18:05.863 | INFO | "huge" : "", | 20260505 20:18:05.863 | INFO | "huge_shared" : " 27 GB", | 20260505 20:18:05.863 | INFO | "malloc" : " 14 GB", | 20260505 20:18:05.863 | INFO | "pinned" : " 29 GB" | 20260505 20:18:05.863 | INFO | } | 20260505 20:18:05.863 | INFO | Perf report for the last 5.14006 seconds: | 20260505 20:18:05.863 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:18:05.863 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:05.863 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:05.863 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:05.863 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260505 20:18:05.863 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:05.863 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:05.863 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:18:05.863 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:05.863 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:05.863 | INFO | Uploaded 36GiB to GPU in 2.12167s (16.9678GiB/s) | 20260505 20:18:05.863 | INFO | Downloaded 34GiB to GPU in 2.01393s (16.8824GiB/s) | 20260505 20:18:05.992 | INFO | Starting a round of Baz IO | 20260505 20:18:09.163 | INFO | Populating RTMetrics report | 20260505 20:18:09.163 | INFO | RTMetrics report populated | 20260505 20:18:09.163 | INFO | Opening tmp RTMetrics json file | 20260505 20:18:09.164 | INFO | Serializing RTMetrics to json | 20260505 20:18:09.166 | INFO | Serialization done, swapping tmp file now... | 20260505 20:18:09.166 | INFO | RTMetrics done | 20260505 20:18:09.251 | INFO | Finished a round of Baz IO: Wrote 2.015 GiB in 3.258 seconds | 20260505 20:18:09.251 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.68% / 0.00% / 78.31% | 20260505 20:18:09.251 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.178738% / 3.97364e-06% / 9.93411e-05% | 20260505 20:18:09.251 | INFO | Baz IO overall: | 20260505 20:18:09.251 | INFO | BytesWritten : 3508870566597 | 20260505 20:18:09.251 | INFO | NumEvents : 1658996355082 | 20260505 20:18:10.995 | INFO | Analyzing chunk frames = [6017024,6017536) | 20260505 20:18:10.995 | INFO | MemInfo: { | 20260505 20:18:10.995 | INFO | "gpu" : " 17 GB", | 20260505 20:18:10.995 | INFO | "huge" : "", | 20260505 20:18:10.995 | INFO | "huge_shared" : " 27 GB", | 20260505 20:18:10.995 | INFO | "malloc" : " 14 GB", | 20260505 20:18:10.995 | INFO | "pinned" : " 29 GB" | 20260505 20:18:10.995 | INFO | } | 20260505 20:18:10.995 | INFO | Perf report for the last 5.13213 seconds: | 20260505 20:18:10.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:18:10.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:10.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:10.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:10.995 | INFO | ANALYSIS : 63.2%, 3.0, 98.4ms, 0 | 20260505 20:18:10.995 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:10.995 | INFO | RT_METRICS : 44.2%, 1.0, 23.1ms, 0 | 20260505 20:18:10.995 | INFO | PRE_HQ : 60.3%, 1.8, 55.5ms, 0 | 20260505 20:18:10.995 | INFO | BAZWRITER : 62.9%, 3.7, 119.4ms, 0 | 20260505 20:18:10.995 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 20:18:10.995 | INFO | Uploaded 36GiB to GPU in 2.16283s (16.6449GiB/s) | 20260505 20:18:10.995 | INFO | Downloaded 38GiB to GPU in 2.25502s (16.8513GiB/s) | 20260505 20:18:16.130 | INFO | Analyzing chunk frames = [6017536,6018048) | 20260505 20:18:16.130 | INFO | MemInfo: { | 20260505 20:18:16.130 | INFO | "gpu" : " 17 GB", | 20260505 20:18:16.130 | INFO | "huge" : "", | 20260505 20:18:16.130 | INFO | "huge_shared" : " 27 GB", | 20260505 20:18:16.130 | INFO | "malloc" : " 14 GB", | 20260505 20:18:16.130 | INFO | "pinned" : " 29 GB" | 20260505 20:18:16.130 | INFO | } | 20260505 20:18:16.130 | INFO | Perf report for the last 5.13542 seconds: | 20260505 20:18:16.130 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:18:16.130 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:16.130 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:16.130 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:16.130 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 20:18:16.130 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:16.131 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:16.131 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:18:16.131 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:16.131 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:16.131 | INFO | Uploaded 36GiB to GPU in 2.14016s (16.8212GiB/s) | 20260505 20:18:16.131 | INFO | Downloaded 34GiB to GPU in 2.0374s (16.6879GiB/s) | 20260505 20:18:21.282 | INFO | Analyzing chunk frames = [6018048,6018560) | 20260505 20:18:21.282 | INFO | MemInfo: { | 20260505 20:18:21.282 | INFO | "gpu" : " 17 GB", | 20260505 20:18:21.282 | INFO | "huge" : "", | 20260505 20:18:21.282 | INFO | "huge_shared" : " 27 GB", | 20260505 20:18:21.283 | INFO | "malloc" : " 14 GB", | 20260505 20:18:21.283 | INFO | "pinned" : " 29 GB" | 20260505 20:18:21.283 | INFO | } | 20260505 20:18:21.283 | INFO | Perf report for the last 5.15216 seconds: | 20260505 20:18:21.283 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:18:21.283 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:21.283 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:21.283 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:21.283 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 20:18:21.283 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:21.283 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:21.283 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:18:21.283 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:21.283 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:21.283 | INFO | Uploaded 36GiB to GPU in 2.14669s (16.77GiB/s) | 20260505 20:18:21.283 | INFO | Downloaded 34GiB to GPU in 2.04456s (16.6295GiB/s) | 20260505 20:18:26.422 | INFO | Analyzing chunk frames = [6018560,6019072) | 20260505 20:18:26.422 | INFO | MemInfo: { | 20260505 20:18:26.422 | INFO | "gpu" : " 17 GB", | 20260505 20:18:26.422 | INFO | "huge" : "", | 20260505 20:18:26.422 | INFO | "huge_shared" : " 27 GB", | 20260505 20:18:26.422 | INFO | "malloc" : " 14 GB", | 20260505 20:18:26.422 | INFO | "pinned" : " 29 GB" | 20260505 20:18:26.422 | INFO | } | 20260505 20:18:26.422 | INFO | Perf report for the last 5.13958 seconds: | 20260505 20:18:26.422 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:18:26.422 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:26.422 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:26.422 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:26.422 | INFO | ANALYSIS : 62.0%, 3.0, 96.3ms, 0 | 20260505 20:18:26.422 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:26.422 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:26.422 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 20:18:26.422 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:26.422 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:26.422 | INFO | Uploaded 36GiB to GPU in 2.14449s (16.7872GiB/s) | 20260505 20:18:26.423 | INFO | Downloaded 34GiB to GPU in 2.02658s (16.777GiB/s) | 20260505 20:18:31.560 | INFO | Analyzing chunk frames = [6019072,6019584) | 20260505 20:18:31.560 | INFO | MemInfo: { | 20260505 20:18:31.560 | INFO | "gpu" : " 17 GB", | 20260505 20:18:31.560 | INFO | "huge" : "", | 20260505 20:18:31.561 | INFO | "huge_shared" : " 27 GB", | 20260505 20:18:31.561 | INFO | "malloc" : " 14 GB", | 20260505 20:18:31.561 | INFO | "pinned" : " 29 GB" | 20260505 20:18:31.561 | INFO | } | 20260505 20:18:31.561 | INFO | Perf report for the last 5.13852 seconds: | 20260505 20:18:31.561 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:18:31.561 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:31.561 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:31.561 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:31.561 | INFO | ANALYSIS : 61.6%, 3.0, 96.2ms, 0 | 20260505 20:18:31.561 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:31.561 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:31.561 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 20:18:31.561 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:31.561 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:31.561 | INFO | Uploaded 36GiB to GPU in 2.13367s (16.8723GiB/s) | 20260505 20:18:31.561 | INFO | Downloaded 34GiB to GPU in 2.05917s (16.5115GiB/s) | 20260505 20:18:36.689 | INFO | Analyzing chunk frames = [6019584,6020096) | 20260505 20:18:36.689 | INFO | MemInfo: { | 20260505 20:18:36.689 | INFO | "gpu" : " 17 GB", | 20260505 20:18:36.689 | INFO | "huge" : "", | 20260505 20:18:36.689 | INFO | "huge_shared" : " 27 GB", | 20260505 20:18:36.690 | INFO | "malloc" : " 14 GB", | 20260505 20:18:36.690 | INFO | "pinned" : " 29 GB" | 20260505 20:18:36.690 | INFO | } | 20260505 20:18:36.690 | INFO | Perf report for the last 5.12904 seconds: | 20260505 20:18:36.690 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:18:36.690 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:36.690 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:36.690 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:36.690 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260505 20:18:36.690 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:36.690 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:36.690 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 20:18:36.690 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:36.690 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:36.690 | INFO | Uploaded 36GiB to GPU in 2.09943s (17.1475GiB/s) | 20260505 20:18:36.690 | INFO | Downloaded 34GiB to GPU in 2.03776s (16.685GiB/s) | 20260505 20:18:41.833 | INFO | Analyzing chunk frames = [6020096,6020608) | 20260505 20:18:41.833 | INFO | MemInfo: { | 20260505 20:18:41.833 | INFO | "gpu" : " 17 GB", | 20260505 20:18:41.834 | INFO | "huge" : "", | 20260505 20:18:41.834 | INFO | "huge_shared" : " 27 GB", | 20260505 20:18:41.834 | INFO | "malloc" : " 14 GB", | 20260505 20:18:41.834 | INFO | "pinned" : " 29 GB" | 20260505 20:18:41.834 | INFO | } | 20260505 20:18:41.834 | INFO | Perf report for the last 5.14401 seconds: | 20260505 20:18:41.834 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:18:41.834 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:41.834 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:41.834 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:41.834 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 20:18:41.834 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:41.834 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:41.834 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 20:18:41.834 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:41.834 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:41.834 | INFO | Uploaded 36GiB to GPU in 2.16016s (16.6654GiB/s) | 20260505 20:18:41.834 | INFO | Downloaded 34GiB to GPU in 2.17828s (15.6087GiB/s) | 20260505 20:18:46.979 | INFO | Analyzing chunk frames = [6020608,6021120) | 20260505 20:18:46.979 | INFO | MemInfo: { | 20260505 20:18:46.979 | INFO | "gpu" : " 17 GB", | 20260505 20:18:46.979 | INFO | "huge" : "", | 20260505 20:18:46.979 | INFO | "huge_shared" : " 27 GB", | 20260505 20:18:46.979 | INFO | "malloc" : " 14 GB", | 20260505 20:18:46.979 | INFO | "pinned" : " 29 GB" | 20260505 20:18:46.979 | INFO | } | 20260505 20:18:46.979 | INFO | Perf report for the last 5.14563 seconds: | 20260505 20:18:46.979 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:18:46.979 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:46.979 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:46.979 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:46.979 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260505 20:18:46.979 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:46.980 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:46.980 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 20:18:46.980 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:46.980 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:46.980 | INFO | Uploaded 36GiB to GPU in 2.12471s (16.9435GiB/s) | 20260505 20:18:46.980 | INFO | Downloaded 34GiB to GPU in 2.01595s (16.8655GiB/s) | 20260505 20:18:46.980 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 20:18:46.980 | INFO | Total mem usage: 15.3 GiB | 20260505 20:18:46.980 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 20:18:46.980 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.12%) | 20260505 20:18:46.980 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 20:18:46.980 | INFO | High Water Breakdown: | 20260505 20:18:46.980 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 20:18:46.980 | INFO | 38.8% : 5.5 GiB ( 82.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 20:18:46.980 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 20:18:46.980 | INFO | ------------------------------------------ | 20260505 20:18:46.980 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:18:46.980 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 20:18:46.980 | INFO | Total mem usage: 26.5 GiB | 20260505 20:18:46.980 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 20:18:46.980 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 20:18:46.980 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 20:18:46.980 | INFO | High Water Breakdown: | 20260505 20:18:46.980 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:18:46.980 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:18:46.980 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:18:46.980 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:18:46.980 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:18:46.980 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:18:46.980 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 20:18:46.980 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 20:18:46.980 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:18:46.980 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 20:18:46.980 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 20:18:46.980 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 20:18:46.980 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 20:18:46.980 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 20:18:46.980 | INFO | ------------------------------------------ | 20260505 20:18:46.980 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:18:46.980 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 20:18:46.980 | INFO | Total mem usage: 25.6 GiB | 20260505 20:18:46.981 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 20:18:46.981 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 20:18:46.981 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 20:18:46.981 | INFO | High Water Breakdown: | 20260505 20:18:46.981 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 20:18:46.981 | INFO | ------------------------------------------ | 20260505 20:18:46.981 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:18:46.981 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 20:18:46.981 | INFO | Total mem usage: 15.7 GiB | 20260505 20:18:46.981 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 20:18:46.981 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 20:18:46.981 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 20:18:46.981 | INFO | High Water Breakdown: | 20260505 20:18:46.981 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 20:18:46.981 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 20:18:46.981 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 20:18:46.981 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 20:18:46.981 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 20:18:46.981 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 20:18:46.981 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 20:18:46.981 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 20:18:46.981 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 20:18:46.981 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:18:46.981 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 20:18:46.981 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:18:46.981 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 20:18:46.981 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 20:18:46.981 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 20:18:46.981 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 20:18:46.981 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:18:46.981 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 20:18:46.981 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:18:46.981 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:18:46.981 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:18:46.981 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 20:18:46.981 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 20:18:46.981 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:18:46.981 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 20:18:46.981 | INFO | ------------------------------------------ | 20260505 20:18:46.981 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:18:46.981 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 20:18:46.981 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:18:46.981 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:18:46.981 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.46e+01, 1.35e+01 / 1.66e+00 | 20260505 20:18:46.981 | INFO | PerfReport: 0.29%, AggregateStats -- 9.25e-02 / 1.03e-01, 1.11e-02 / 4.38e-03 | 20260505 20:18:46.981 | INFO | PerfReport: 11.78%, Baseline -- 3.71e+00 / 4.55e+00, 5.67e-01 / 1.26e-01 | 20260505 20:18:46.981 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.51e+00, 1.90e-01 / 9.71e-02 | 20260505 20:18:46.981 | INFO | PerfReport: 15.94%, DME -- 4.02e+01 / 4.44e+01, 4.18e+00 / 1.76e+00 | 20260505 20:18:46.981 | INFO | PerfReport: 57.10%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.51e+00 / 1.05e+00 | 20260505 20:18:46.981 | INFO | PerfReport: 7.22%, PulseAccumulating -- 2.27e+00 / 2.87e+00, 4.38e-01 / 1.97e-01 | 20260505 20:18:46.981 | INFO | PerfReport: 3.25%, Metrics -- 1.02e+00 / 1.36e+00, 2.84e-01 / 1.44e-01 | 20260505 20:18:46.981 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.56e-01 / 4.22e-01, 6.43e-02 / 3.14e-02 | 20260505 20:18:46.981 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 20:18:46.981 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:18:46.981 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:18:46.981 | INFO | PerfReport: Full Execution -- 9.94e+01 / -nan, 2.53e+01 / 0.00e+00 | 20260505 20:18:46.981 | INFO | PerfReport: 22.47%, Upload -- 2.23e+01 / 0.00e+00, 7.76e+00 / 0.00e+00 | 20260505 20:18:46.981 | INFO | PerfReport: 21.82%, Download -- 2.17e+01 / 0.00e+00, 7.71e+00 / 0.00e+00 | 20260505 20:18:46.981 | INFO | PerfReport: 31.70%, Compute -- 3.15e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 20:18:46.981 | INFO | PerfReport: 2.28%, LockUpload -- 2.26e+00 / 0.00e+00, 7.97e+00 / 0.00e+00 | 20260505 20:18:46.981 | INFO | PerfReport: 1.34%, LockDownload -- 1.33e+00 / 0.00e+00, 2.90e+00 / 0.00e+00 | 20260505 20:18:46.981 | INFO | PerfReport: 20.38%, LockCompute -- 2.03e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 20:18:46.981 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.05e-02 / 0.00e+00 | 20260505 20:18:47.110 | INFO | Starting a round of Baz IO | 20260505 20:18:50.272 | INFO | Populating RTMetrics report | 20260505 20:18:50.272 | INFO | RTMetrics report populated | 20260505 20:18:50.272 | INFO | Opening tmp RTMetrics json file | 20260505 20:18:50.273 | INFO | Serializing RTMetrics to json | 20260505 20:18:50.277 | INFO | Serialization done, swapping tmp file now... | 20260505 20:18:50.279 | INFO | RTMetrics done | 20260505 20:18:50.361 | INFO | Finished a round of Baz IO: Wrote 2.017 GiB in 3.251 seconds | 20260505 20:18:50.361 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.68% / 0.00% / 78.32% | 20260505 20:18:50.361 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.179426% / 0% / 0.000103315% | 20260505 20:18:50.361 | INFO | Baz IO overall: | 20260505 20:18:50.361 | INFO | BytesWritten : 3511036112541 | 20260505 20:18:50.361 | INFO | NumEvents : 1659999767383 | 20260505 20:18:52.117 | INFO | Analyzing chunk frames = [6021120,6021632) | 20260505 20:18:52.117 | INFO | MemInfo: { | 20260505 20:18:52.117 | INFO | "gpu" : " 17 GB", | 20260505 20:18:52.117 | INFO | "huge" : "", | 20260505 20:18:52.117 | INFO | "huge_shared" : " 27 GB", | 20260505 20:18:52.117 | INFO | "malloc" : " 14 GB", | 20260505 20:18:52.117 | INFO | "pinned" : " 29 GB" | 20260505 20:18:52.117 | INFO | } | 20260505 20:18:52.117 | INFO | Perf report for the last 5.13786 seconds: | 20260505 20:18:52.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:18:52.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:52.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:52.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:52.117 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 20:18:52.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:52.117 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260505 20:18:52.118 | INFO | PRE_HQ : 60.3%, 1.8, 55.4ms, 0 | 20260505 20:18:52.118 | INFO | BAZWRITER : 62.7%, 3.7, 119.6ms, 0 | 20260505 20:18:52.118 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 20:18:52.118 | INFO | Uploaded 36GiB to GPU in 2.16882s (16.5989GiB/s) | 20260505 20:18:52.118 | INFO | Downloaded 38GiB to GPU in 2.24425s (16.9321GiB/s) | 20260505 20:18:57.253 | INFO | Analyzing chunk frames = [6021632,6022144) | 20260505 20:18:57.253 | INFO | MemInfo: { | 20260505 20:18:57.253 | INFO | "gpu" : " 17 GB", | 20260505 20:18:57.253 | INFO | "huge" : "", | 20260505 20:18:57.253 | INFO | "huge_shared" : " 27 GB", | 20260505 20:18:57.254 | INFO | "malloc" : " 14 GB", | 20260505 20:18:57.254 | INFO | "pinned" : " 29 GB" | 20260505 20:18:57.254 | INFO | } | 20260505 20:18:57.254 | INFO | Perf report for the last 5.13634 seconds: | 20260505 20:18:57.254 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:18:57.254 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:57.254 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:57.254 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:57.254 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260505 20:18:57.254 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:57.254 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:57.254 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:18:57.254 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:57.254 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:18:57.254 | INFO | Uploaded 36GiB to GPU in 2.08245s (17.2873GiB/s) | 20260505 20:18:57.254 | INFO | Downloaded 34GiB to GPU in 2.02829s (16.7629GiB/s) | 20260505 20:19:02.385 | INFO | Analyzing chunk frames = [6022144,6022656) | 20260505 20:19:02.385 | INFO | MemInfo: { | 20260505 20:19:02.385 | INFO | "gpu" : " 17 GB", | 20260505 20:19:02.385 | INFO | "huge" : "", | 20260505 20:19:02.385 | INFO | "huge_shared" : " 27 GB", | 20260505 20:19:02.385 | INFO | "malloc" : " 14 GB", | 20260505 20:19:02.385 | INFO | "pinned" : " 29 GB" | 20260505 20:19:02.385 | INFO | } | 20260505 20:19:02.386 | INFO | Perf report for the last 5.13189 seconds: | 20260505 20:19:02.386 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:19:02.386 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:02.386 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:02.386 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:02.386 | INFO | ANALYSIS : 62.2%, 3.0, 96.6ms, 0 | 20260505 20:19:02.386 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:02.386 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:02.386 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:19:02.386 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:02.386 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:02.386 | INFO | Uploaded 36GiB to GPU in 2.13297s (16.8779GiB/s) | 20260505 20:19:02.386 | INFO | Downloaded 34GiB to GPU in 2.01261s (16.8935GiB/s) | 20260505 20:19:07.534 | INFO | Analyzing chunk frames = [6022656,6023168) | 20260505 20:19:07.534 | INFO | MemInfo: { | 20260505 20:19:07.534 | INFO | "gpu" : " 17 GB", | 20260505 20:19:07.534 | INFO | "huge" : "", | 20260505 20:19:07.534 | INFO | "huge_shared" : " 27 GB", | 20260505 20:19:07.534 | INFO | "malloc" : " 14 GB", | 20260505 20:19:07.534 | INFO | "pinned" : " 29 GB" | 20260505 20:19:07.534 | INFO | } | 20260505 20:19:07.534 | INFO | Perf report for the last 5.14886 seconds: | 20260505 20:19:07.534 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:19:07.534 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:07.534 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:07.534 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:07.535 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260505 20:19:07.535 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:07.535 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:07.535 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 20:19:07.535 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:07.535 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:07.535 | INFO | Uploaded 36GiB to GPU in 2.18243s (16.4954GiB/s) | 20260505 20:19:07.535 | INFO | Downloaded 34GiB to GPU in 2.06824s (16.4391GiB/s) | 20260505 20:19:12.675 | INFO | Analyzing chunk frames = [6023168,6023680) | 20260505 20:19:12.675 | INFO | MemInfo: { | 20260505 20:19:12.675 | INFO | "gpu" : " 17 GB", | 20260505 20:19:12.675 | INFO | "huge" : "", | 20260505 20:19:12.675 | INFO | "huge_shared" : " 27 GB", | 20260505 20:19:12.675 | INFO | "malloc" : " 14 GB", | 20260505 20:19:12.675 | INFO | "pinned" : " 29 GB" | 20260505 20:19:12.675 | INFO | } | 20260505 20:19:12.675 | INFO | Perf report for the last 5.14035 seconds: | 20260505 20:19:12.675 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:19:12.675 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:12.675 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:12.675 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:12.675 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 20:19:12.675 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:12.675 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:12.675 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 20:19:12.675 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:12.675 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:12.675 | INFO | Uploaded 36GiB to GPU in 2.13601s (16.8539GiB/s) | 20260505 20:19:12.675 | INFO | Downloaded 34GiB to GPU in 2.06086s (16.498GiB/s) | 20260505 20:19:17.807 | INFO | Analyzing chunk frames = [6023680,6024192) | 20260505 20:19:17.807 | INFO | MemInfo: { | 20260505 20:19:17.807 | INFO | "gpu" : " 17 GB", | 20260505 20:19:17.807 | INFO | "huge" : "", | 20260505 20:19:17.807 | INFO | "huge_shared" : " 27 GB", | 20260505 20:19:17.807 | INFO | "malloc" : " 14 GB", | 20260505 20:19:17.807 | INFO | "pinned" : " 29 GB" | 20260505 20:19:17.807 | INFO | } | 20260505 20:19:17.807 | INFO | Perf report for the last 5.13256 seconds: | 20260505 20:19:17.807 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:19:17.807 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:17.807 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:17.807 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:17.807 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260505 20:19:17.807 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:17.807 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:17.807 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:19:17.807 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:17.807 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:17.808 | INFO | Uploaded 36GiB to GPU in 2.08922s (17.2314GiB/s) | 20260505 20:19:17.808 | INFO | Downloaded 34GiB to GPU in 2.03172s (16.7346GiB/s) | 20260505 20:19:22.948 | INFO | Analyzing chunk frames = [6024192,6024704) | 20260505 20:19:22.948 | INFO | MemInfo: { | 20260505 20:19:22.948 | INFO | "gpu" : " 17 GB", | 20260505 20:19:22.948 | INFO | "huge" : "", | 20260505 20:19:22.948 | INFO | "huge_shared" : " 27 GB", | 20260505 20:19:22.948 | INFO | "malloc" : " 14 GB", | 20260505 20:19:22.948 | INFO | "pinned" : " 29 GB" | 20260505 20:19:22.948 | INFO | } | 20260505 20:19:22.948 | INFO | Perf report for the last 5.1405 seconds: | 20260505 20:19:22.948 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:19:22.948 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:22.948 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:22.948 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:22.948 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 20:19:22.948 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:22.948 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:22.948 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 20:19:22.948 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:22.948 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:22.948 | INFO | Uploaded 36GiB to GPU in 2.12651s (16.9292GiB/s) | 20260505 20:19:22.948 | INFO | Downloaded 34GiB to GPU in 2.14043s (15.8846GiB/s) | 20260505 20:19:28.088 | INFO | Analyzing chunk frames = [6024704,6025216) | 20260505 20:19:28.088 | INFO | MemInfo: { | 20260505 20:19:28.088 | INFO | "gpu" : " 17 GB", | 20260505 20:19:28.088 | INFO | "huge" : "", | 20260505 20:19:28.088 | INFO | "huge_shared" : " 27 GB", | 20260505 20:19:28.088 | INFO | "malloc" : " 14 GB", | 20260505 20:19:28.088 | INFO | "pinned" : " 29 GB" | 20260505 20:19:28.088 | INFO | } | 20260505 20:19:28.088 | INFO | Perf report for the last 5.14054 seconds: | 20260505 20:19:28.088 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:19:28.088 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:28.088 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:28.088 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:28.088 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 20:19:28.088 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:28.088 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:28.089 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:19:28.089 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:28.089 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:28.089 | INFO | Uploaded 36GiB to GPU in 2.20997s (16.2898GiB/s) | 20260505 20:19:28.089 | INFO | Downloaded 34GiB to GPU in 2.04864s (16.5964GiB/s) | 20260505 20:19:28.217 | INFO | Starting a round of Baz IO | 20260505 20:19:31.389 | INFO | Populating RTMetrics report | 20260505 20:19:31.390 | INFO | RTMetrics report populated | 20260505 20:19:31.390 | INFO | Opening tmp RTMetrics json file | 20260505 20:19:31.390 | INFO | Serializing RTMetrics to json | 20260505 20:19:31.393 | INFO | Serialization done, swapping tmp file now... | 20260505 20:19:31.397 | INFO | RTMetrics done | 20260505 20:19:31.484 | INFO | Finished a round of Baz IO: Wrote 2.016 GiB in 3.267 seconds | 20260505 20:19:31.484 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.68% / 0.00% / 78.32% | 20260505 20:19:31.484 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.178778% / 0% / 8.74201e-05% | 20260505 20:19:31.484 | INFO | Baz IO overall: | 20260505 20:19:31.484 | INFO | BytesWritten : 3513200783755 | 20260505 20:19:31.484 | INFO | NumEvents : 1661002276264 | 20260505 20:19:33.229 | INFO | Analyzing chunk frames = [6025216,6025728) | 20260505 20:19:33.229 | INFO | MemInfo: { | 20260505 20:19:33.229 | INFO | "gpu" : " 17 GB", | 20260505 20:19:33.229 | INFO | "huge" : "", | 20260505 20:19:33.229 | INFO | "huge_shared" : " 27 GB", | 20260505 20:19:33.229 | INFO | "malloc" : " 14 GB", | 20260505 20:19:33.229 | INFO | "pinned" : " 29 GB" | 20260505 20:19:33.229 | INFO | } | 20260505 20:19:33.229 | INFO | Perf report for the last 5.14066 seconds: | 20260505 20:19:33.229 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:19:33.229 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:33.229 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:33.229 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:33.229 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 20:19:33.229 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:33.229 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260505 20:19:33.229 | INFO | PRE_HQ : 60.0%, 1.8, 56.0ms, 0 | 20260505 20:19:33.229 | INFO | BAZWRITER : 63.0%, 3.7, 119.9ms, 0 | 20260505 20:19:33.229 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 20:19:33.229 | INFO | Uploaded 36GiB to GPU in 2.13305s (16.8772GiB/s) | 20260505 20:19:33.229 | INFO | Downloaded 38GiB to GPU in 2.24583s (16.9203GiB/s) | 20260505 20:19:38.368 | INFO | Analyzing chunk frames = [6025728,6026240) | 20260505 20:19:38.368 | INFO | MemInfo: { | 20260505 20:19:38.368 | INFO | "gpu" : " 17 GB", | 20260505 20:19:38.368 | INFO | "huge" : "", | 20260505 20:19:38.368 | INFO | "huge_shared" : " 27 GB", | 20260505 20:19:38.368 | INFO | "malloc" : " 14 GB", | 20260505 20:19:38.368 | INFO | "pinned" : " 29 GB" | 20260505 20:19:38.368 | INFO | } | 20260505 20:19:38.368 | INFO | Perf report for the last 5.13931 seconds: | 20260505 20:19:38.369 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:19:38.369 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:38.369 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:38.369 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:38.369 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 20:19:38.369 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:38.369 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:38.369 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:19:38.369 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:38.369 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:38.369 | INFO | Uploaded 36GiB to GPU in 2.07522s (17.3476GiB/s) | 20260505 20:19:38.369 | INFO | Downloaded 34GiB to GPU in 2.02184s (16.8164GiB/s) | 20260505 20:19:43.501 | INFO | Analyzing chunk frames = [6026240,6026752) | 20260505 20:19:43.501 | INFO | MemInfo: { | 20260505 20:19:43.501 | INFO | "gpu" : " 17 GB", | 20260505 20:19:43.501 | INFO | "huge" : "", | 20260505 20:19:43.501 | INFO | "huge_shared" : " 27 GB", | 20260505 20:19:43.501 | INFO | "malloc" : " 14 GB", | 20260505 20:19:43.501 | INFO | "pinned" : " 29 GB" | 20260505 20:19:43.501 | INFO | } | 20260505 20:19:43.501 | INFO | Perf report for the last 5.13271 seconds: | 20260505 20:19:43.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:19:43.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:43.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:43.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:43.501 | INFO | ANALYSIS : 62.1%, 3.0, 96.5ms, 0 | 20260505 20:19:43.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:43.501 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:43.501 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:19:43.501 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:43.501 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:43.501 | INFO | Uploaded 36GiB to GPU in 2.11807s (16.9966GiB/s) | 20260505 20:19:43.501 | INFO | Downloaded 34GiB to GPU in 2.02058s (16.8269GiB/s) | 20260505 20:19:48.638 | INFO | Analyzing chunk frames = [6026752,6027264) | 20260505 20:19:48.638 | INFO | MemInfo: { | 20260505 20:19:48.638 | INFO | "gpu" : " 17 GB", | 20260505 20:19:48.638 | INFO | "huge" : "", | 20260505 20:19:48.638 | INFO | "huge_shared" : " 27 GB", | 20260505 20:19:48.638 | INFO | "malloc" : " 14 GB", | 20260505 20:19:48.638 | INFO | "pinned" : " 29 GB" | 20260505 20:19:48.638 | INFO | } | 20260505 20:19:48.638 | INFO | Perf report for the last 5.13718 seconds: | 20260505 20:19:48.638 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:19:48.638 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:48.638 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:48.639 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:48.639 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260505 20:19:48.639 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:48.639 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:48.639 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 20:19:48.639 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:48.639 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:48.639 | INFO | Uploaded 36GiB to GPU in 2.19866s (16.3736GiB/s) | 20260505 20:19:48.639 | INFO | Downloaded 34GiB to GPU in 2.06933s (16.4305GiB/s) | 20260505 20:19:53.789 | INFO | Analyzing chunk frames = [6027264,6027776) | 20260505 20:19:53.789 | INFO | MemInfo: { | 20260505 20:19:53.789 | INFO | "gpu" : " 17 GB", | 20260505 20:19:53.789 | INFO | "huge" : "", | 20260505 20:19:53.789 | INFO | "huge_shared" : " 27 GB", | 20260505 20:19:53.789 | INFO | "malloc" : " 14 GB", | 20260505 20:19:53.789 | INFO | "pinned" : " 29 GB" | 20260505 20:19:53.789 | INFO | } | 20260505 20:19:53.790 | INFO | Perf report for the last 5.15119 seconds: | 20260505 20:19:53.790 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:19:53.790 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:53.790 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:53.790 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:53.790 | INFO | ANALYSIS : 61.9%, 3.0, 96.8ms, 0 | 20260505 20:19:53.790 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:53.790 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:53.790 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:19:53.790 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:53.790 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:53.790 | INFO | Uploaded 36GiB to GPU in 2.14203s (16.8065GiB/s) | 20260505 20:19:53.790 | INFO | Downloaded 34GiB to GPU in 2.06274s (16.4829GiB/s) | 20260505 20:19:58.939 | INFO | Analyzing chunk frames = [6027776,6028288) | 20260505 20:19:58.939 | INFO | MemInfo: { | 20260505 20:19:58.939 | INFO | "gpu" : " 17 GB", | 20260505 20:19:58.939 | INFO | "huge" : "", | 20260505 20:19:58.939 | INFO | "huge_shared" : " 27 GB", | 20260505 20:19:58.939 | INFO | "malloc" : " 14 GB", | 20260505 20:19:58.939 | INFO | "pinned" : " 29 GB" | 20260505 20:19:58.939 | INFO | } | 20260505 20:19:58.939 | INFO | Perf report for the last 5.14949 seconds: | 20260505 20:19:58.939 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:19:58.939 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:58.939 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:58.939 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:58.939 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260505 20:19:58.939 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:58.939 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:58.939 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:19:58.939 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:58.939 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:19:58.939 | INFO | Uploaded 36GiB to GPU in 2.08473s (17.2684GiB/s) | 20260505 20:19:58.939 | INFO | Downloaded 34GiB to GPU in 2.03477s (16.7095GiB/s) | 20260505 20:20:04.069 | INFO | Analyzing chunk frames = [6028288,6028800) | 20260505 20:20:04.069 | INFO | MemInfo: { | 20260505 20:20:04.069 | INFO | "gpu" : " 17 GB", | 20260505 20:20:04.069 | INFO | "huge" : "", | 20260505 20:20:04.069 | INFO | "huge_shared" : " 27 GB", | 20260505 20:20:04.069 | INFO | "malloc" : " 14 GB", | 20260505 20:20:04.069 | INFO | "pinned" : " 29 GB" | 20260505 20:20:04.069 | INFO | } | 20260505 20:20:04.069 | INFO | Perf report for the last 5.13031 seconds: | 20260505 20:20:04.069 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:20:04.069 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:04.069 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:04.069 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:04.069 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260505 20:20:04.069 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:04.069 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:04.069 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 20:20:04.069 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:04.069 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:04.070 | INFO | Uploaded 36GiB to GPU in 2.10236s (17.1236GiB/s) | 20260505 20:20:04.070 | INFO | Downloaded 34GiB to GPU in 2.08024s (16.3443GiB/s) | 20260505 20:20:09.205 | INFO | Analyzing chunk frames = [6028800,6029312) | 20260505 20:20:09.205 | INFO | MemInfo: { | 20260505 20:20:09.205 | INFO | "gpu" : " 17 GB", | 20260505 20:20:09.205 | INFO | "huge" : "", | 20260505 20:20:09.205 | INFO | "huge_shared" : " 27 GB", | 20260505 20:20:09.205 | INFO | "malloc" : " 14 GB", | 20260505 20:20:09.205 | INFO | "pinned" : " 29 GB" | 20260505 20:20:09.205 | INFO | } | 20260505 20:20:09.205 | INFO | Perf report for the last 5.13596 seconds: | 20260505 20:20:09.205 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:20:09.205 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:09.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:09.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:09.205 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260505 20:20:09.205 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:09.205 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:09.205 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260505 20:20:09.205 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:09.205 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:09.206 | INFO | Uploaded 36GiB to GPU in 2.11903s (16.9889GiB/s) | 20260505 20:20:09.206 | INFO | Downloaded 34GiB to GPU in 2.01368s (16.8845GiB/s) | 20260505 20:20:09.206 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 20:20:09.206 | INFO | Total mem usage: 15.3 GiB | 20260505 20:20:09.206 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 20:20:09.206 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.3%) | 20260505 20:20:09.206 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 20:20:09.206 | INFO | High Water Breakdown: | 20260505 20:20:09.206 | INFO | 61% : 8.64 GiB ( 90.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 20:20:09.206 | INFO | 38.8% : 5.49 GiB ( 82.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 20:20:09.206 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 20:20:09.206 | INFO | ------------------------------------------ | 20260505 20:20:09.206 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:20:09.206 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 20:20:09.206 | INFO | Total mem usage: 26.5 GiB | 20260505 20:20:09.206 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 20:20:09.206 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 20:20:09.206 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 20:20:09.206 | INFO | High Water Breakdown: | 20260505 20:20:09.206 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:20:09.206 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:20:09.206 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:20:09.206 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:20:09.206 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:20:09.206 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:20:09.206 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 20:20:09.206 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 20:20:09.206 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:20:09.206 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 20:20:09.206 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 20:20:09.206 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 20:20:09.206 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 20:20:09.206 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 20:20:09.206 | INFO | ------------------------------------------ | 20260505 20:20:09.206 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:20:09.206 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 20:20:09.206 | INFO | Total mem usage: 25.6 GiB | 20260505 20:20:09.206 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 20:20:09.206 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 20:20:09.206 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 20:20:09.206 | INFO | High Water Breakdown: | 20260505 20:20:09.206 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 20:20:09.206 | INFO | ------------------------------------------ | 20260505 20:20:09.206 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:20:09.206 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 20:20:09.206 | INFO | Total mem usage: 15.7 GiB | 20260505 20:20:09.206 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 20:20:09.206 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 20:20:09.206 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 20:20:09.206 | INFO | High Water Breakdown: | 20260505 20:20:09.206 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 20:20:09.206 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 20:20:09.206 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 20:20:09.206 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 20:20:09.206 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 20:20:09.206 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 20:20:09.206 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 20:20:09.206 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 20:20:09.206 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 20:20:09.206 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:20:09.206 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 20:20:09.206 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:20:09.206 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 20:20:09.206 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 20:20:09.206 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 20:20:09.206 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 20:20:09.207 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:20:09.207 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 20:20:09.207 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:20:09.207 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:20:09.207 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:20:09.207 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 20:20:09.207 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 20:20:09.207 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:20:09.207 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 20:20:09.207 | INFO | ------------------------------------------ | 20260505 20:20:09.207 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:20:09.207 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 20:20:09.207 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:20:09.207 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:20:09.207 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.45e+01, 1.35e+01 / 1.57e+00 | 20260505 20:20:09.207 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.04e-01, 1.09e-02 / 3.07e-03 | 20260505 20:20:09.207 | INFO | PerfReport: 11.81%, Baseline -- 3.72e+00 / 4.52e+00, 5.69e-01 / 1.37e-01 | 20260505 20:20:09.207 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.53e+00, 1.92e-01 / 8.81e-02 | 20260505 20:20:09.207 | INFO | PerfReport: 15.95%, DME -- 4.02e+01 / 4.44e+01, 3.98e+00 / 1.70e+00 | 20260505 20:20:09.207 | INFO | PerfReport: 57.09%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.52e+00 / 9.71e-01 | 20260505 20:20:09.207 | INFO | PerfReport: 7.20%, PulseAccumulating -- 2.27e+00 / 2.85e+00, 4.34e-01 / 1.68e-01 | 20260505 20:20:09.207 | INFO | PerfReport: 3.24%, Metrics -- 1.02e+00 / 1.35e+00, 2.76e-01 / 1.19e-01 | 20260505 20:20:09.207 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.56e-01 / 4.23e-01, 6.18e-02 / 3.76e-02 | 20260505 20:20:09.207 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 20:20:09.207 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:20:09.207 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:20:09.207 | INFO | PerfReport: Full Execution -- 9.92e+01 / -nan, 2.53e+01 / 0.00e+00 | 20260505 20:20:09.207 | INFO | PerfReport: 22.37%, Upload -- 2.22e+01 / 0.00e+00, 7.64e+00 / 0.00e+00 | 20260505 20:20:09.207 | INFO | PerfReport: 21.77%, Download -- 2.16e+01 / 0.00e+00, 7.65e+00 / 0.00e+00 | 20260505 20:20:09.207 | INFO | PerfReport: 31.76%, Compute -- 3.15e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 20:20:09.207 | INFO | PerfReport: 2.22%, LockUpload -- 2.20e+00 / 0.00e+00, 7.85e+00 / 0.00e+00 | 20260505 20:20:09.207 | INFO | PerfReport: 1.31%, LockDownload -- 1.30e+00 / 0.00e+00, 2.85e+00 / 0.00e+00 | 20260505 20:20:09.207 | INFO | PerfReport: 20.55%, LockCompute -- 2.04e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 20:20:09.207 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.94e-02 / 0.00e+00 | 20260505 20:20:09.338 | INFO | Starting a round of Baz IO | 20260505 20:20:12.501 | INFO | Populating RTMetrics report | 20260505 20:20:12.501 | INFO | RTMetrics report populated | 20260505 20:20:12.501 | INFO | Opening tmp RTMetrics json file | 20260505 20:20:12.502 | INFO | Serializing RTMetrics to json | 20260505 20:20:12.508 | INFO | Serialization done, swapping tmp file now... | 20260505 20:20:12.510 | INFO | RTMetrics done | 20260505 20:20:12.588 | INFO | Finished a round of Baz IO: Wrote 2.015 GiB in 3.25 seconds | 20260505 20:20:12.588 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.68% / 0.00% / 78.32% | 20260505 20:20:12.588 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.177936% / 3.97364e-06% / 7.94729e-05% | 20260505 20:20:12.588 | INFO | Baz IO overall: | 20260505 20:20:12.589 | INFO | BytesWritten : 3515363982978 | 20260505 20:20:12.589 | INFO | NumEvents : 1662004066603 | 20260505 20:20:14.343 | INFO | Analyzing chunk frames = [6029312,6029824) | 20260505 20:20:14.343 | INFO | MemInfo: { | 20260505 20:20:14.343 | INFO | "gpu" : " 17 GB", | 20260505 20:20:14.343 | INFO | "huge" : "", | 20260505 20:20:14.343 | INFO | "huge_shared" : " 27 GB", | 20260505 20:20:14.343 | INFO | "malloc" : " 14 GB", | 20260505 20:20:14.343 | INFO | "pinned" : " 29 GB" | 20260505 20:20:14.343 | INFO | } | 20260505 20:20:14.343 | INFO | Perf report for the last 5.13763 seconds: | 20260505 20:20:14.343 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:20:14.343 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:14.343 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:14.343 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:14.343 | INFO | ANALYSIS : 63.1%, 3.0, 98.3ms, 0 | 20260505 20:20:14.343 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:14.343 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260505 20:20:14.343 | INFO | PRE_HQ : 59.9%, 1.8, 55.3ms, 0 | 20260505 20:20:14.343 | INFO | BAZWRITER : 62.7%, 3.6, 117.4ms, 0 | 20260505 20:20:14.343 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 20:20:14.343 | INFO | Uploaded 36GiB to GPU in 2.17302s (16.5668GiB/s) | 20260505 20:20:14.343 | INFO | Downloaded 38GiB to GPU in 2.25016s (16.8877GiB/s) | 20260505 20:20:19.481 | INFO | Analyzing chunk frames = [6029824,6030336) | 20260505 20:20:19.481 | INFO | MemInfo: { | 20260505 20:20:19.481 | INFO | "gpu" : " 17 GB", | 20260505 20:20:19.481 | INFO | "huge" : "", | 20260505 20:20:19.481 | INFO | "huge_shared" : " 27 GB", | 20260505 20:20:19.481 | INFO | "malloc" : " 14 GB", | 20260505 20:20:19.481 | INFO | "pinned" : " 29 GB" | 20260505 20:20:19.481 | INFO | } | 20260505 20:20:19.481 | INFO | Perf report for the last 5.13786 seconds: | 20260505 20:20:19.481 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:20:19.481 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:19.481 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:19.481 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:19.481 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 20:20:19.481 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:19.481 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:19.481 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:20:19.481 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:19.481 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:19.481 | INFO | Uploaded 36GiB to GPU in 2.08919s (17.2315GiB/s) | 20260505 20:20:19.481 | INFO | Downloaded 34GiB to GPU in 2.03038s (16.7456GiB/s) | 20260505 20:20:24.622 | INFO | Analyzing chunk frames = [6030336,6030848) | 20260505 20:20:24.622 | INFO | MemInfo: { | 20260505 20:20:24.622 | INFO | "gpu" : " 17 GB", | 20260505 20:20:24.622 | INFO | "huge" : "", | 20260505 20:20:24.622 | INFO | "huge_shared" : " 27 GB", | 20260505 20:20:24.622 | INFO | "malloc" : " 14 GB", | 20260505 20:20:24.622 | INFO | "pinned" : " 29 GB" | 20260505 20:20:24.622 | INFO | } | 20260505 20:20:24.622 | INFO | Perf report for the last 5.14155 seconds: | 20260505 20:20:24.622 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:20:24.622 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:24.622 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:24.622 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:24.622 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260505 20:20:24.622 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:24.622 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:24.622 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:20:24.622 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:24.623 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:24.623 | INFO | Uploaded 36GiB to GPU in 2.12488s (16.9421GiB/s) | 20260505 20:20:24.623 | INFO | Downloaded 34GiB to GPU in 2.02226s (16.8129GiB/s) | 20260505 20:20:29.760 | INFO | Analyzing chunk frames = [6030848,6031360) | 20260505 20:20:29.760 | INFO | MemInfo: { | 20260505 20:20:29.760 | INFO | "gpu" : " 17 GB", | 20260505 20:20:29.760 | INFO | "huge" : "", | 20260505 20:20:29.760 | INFO | "huge_shared" : " 27 GB", | 20260505 20:20:29.760 | INFO | "malloc" : " 14 GB", | 20260505 20:20:29.760 | INFO | "pinned" : " 29 GB" | 20260505 20:20:29.760 | INFO | } | 20260505 20:20:29.760 | INFO | Perf report for the last 5.13754 seconds: | 20260505 20:20:29.760 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:20:29.760 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:29.760 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:29.760 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:29.760 | INFO | ANALYSIS : 62.4%, 3.0, 97.0ms, 0 | 20260505 20:20:29.760 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:29.760 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:29.760 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:20:29.760 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:29.760 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:29.760 | INFO | Uploaded 36GiB to GPU in 2.20407s (16.3334GiB/s) | 20260505 20:20:29.760 | INFO | Downloaded 34GiB to GPU in 2.03788s (16.684GiB/s) | 20260505 20:20:34.896 | INFO | Analyzing chunk frames = [6031360,6031872) | 20260505 20:20:34.896 | INFO | MemInfo: { | 20260505 20:20:34.896 | INFO | "gpu" : " 17 GB", | 20260505 20:20:34.896 | INFO | "huge" : "", | 20260505 20:20:34.896 | INFO | "huge_shared" : " 27 GB", | 20260505 20:20:34.896 | INFO | "malloc" : " 14 GB", | 20260505 20:20:34.896 | INFO | "pinned" : " 29 GB" | 20260505 20:20:34.896 | INFO | } | 20260505 20:20:34.896 | INFO | Perf report for the last 5.13621 seconds: | 20260505 20:20:34.896 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:20:34.896 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:34.896 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:34.896 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:34.896 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 20:20:34.896 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:34.896 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:34.896 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:20:34.896 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:34.896 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:34.896 | INFO | Uploaded 36GiB to GPU in 2.13647s (16.8502GiB/s) | 20260505 20:20:34.897 | INFO | Downloaded 34GiB to GPU in 2.06165s (16.4917GiB/s) | 20260505 20:20:40.049 | INFO | Analyzing chunk frames = [6031872,6032384) | 20260505 20:20:40.049 | INFO | MemInfo: { | 20260505 20:20:40.049 | INFO | "gpu" : " 17 GB", | 20260505 20:20:40.049 | INFO | "huge" : "", | 20260505 20:20:40.049 | INFO | "huge_shared" : " 27 GB", | 20260505 20:20:40.049 | INFO | "malloc" : " 14 GB", | 20260505 20:20:40.049 | INFO | "pinned" : " 29 GB" | 20260505 20:20:40.049 | INFO | } | 20260505 20:20:40.049 | INFO | Perf report for the last 5.15297 seconds: | 20260505 20:20:40.049 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:20:40.049 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:40.049 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:40.049 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:40.049 | INFO | ANALYSIS : 62.1%, 3.0, 97.2ms, 0 | 20260505 20:20:40.049 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:40.049 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:40.049 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:20:40.049 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:40.049 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:40.049 | INFO | Uploaded 36GiB to GPU in 2.15306s (16.7204GiB/s) | 20260505 20:20:40.050 | INFO | Downloaded 34GiB to GPU in 2.05667s (16.5316GiB/s) | 20260505 20:20:45.184 | INFO | Analyzing chunk frames = [6032384,6032896) | 20260505 20:20:45.184 | INFO | MemInfo: { | 20260505 20:20:45.184 | INFO | "gpu" : " 17 GB", | 20260505 20:20:45.184 | INFO | "huge" : "", | 20260505 20:20:45.184 | INFO | "huge_shared" : " 27 GB", | 20260505 20:20:45.184 | INFO | "malloc" : " 14 GB", | 20260505 20:20:45.184 | INFO | "pinned" : " 29 GB" | 20260505 20:20:45.184 | INFO | } | 20260505 20:20:45.185 | INFO | Perf report for the last 5.13537 seconds: | 20260505 20:20:45.185 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:20:45.185 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:45.185 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:45.185 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:45.185 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 20:20:45.185 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:45.185 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:45.185 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 20:20:45.185 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:45.185 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:45.185 | INFO | Uploaded 36GiB to GPU in 2.16081s (16.6604GiB/s) | 20260505 20:20:45.185 | INFO | Downloaded 34GiB to GPU in 2.07821s (16.3603GiB/s) | 20260505 20:20:50.317 | INFO | Analyzing chunk frames = [6032896,6033408) | 20260505 20:20:50.317 | INFO | MemInfo: { | 20260505 20:20:50.317 | INFO | "gpu" : " 17 GB", | 20260505 20:20:50.317 | INFO | "huge" : "", | 20260505 20:20:50.317 | INFO | "huge_shared" : " 27 GB", | 20260505 20:20:50.317 | INFO | "malloc" : " 14 GB", | 20260505 20:20:50.317 | INFO | "pinned" : " 29 GB" | 20260505 20:20:50.317 | INFO | } | 20260505 20:20:50.317 | INFO | Perf report for the last 5.13226 seconds: | 20260505 20:20:50.317 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:20:50.317 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:50.317 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:50.317 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:50.317 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260505 20:20:50.317 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:50.317 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:50.317 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 20:20:50.317 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:50.317 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:50.317 | INFO | Uploaded 36GiB to GPU in 2.10617s (17.0926GiB/s) | 20260505 20:20:50.317 | INFO | Downloaded 34GiB to GPU in 2.00873s (16.9261GiB/s) | 20260505 20:20:50.445 | INFO | Starting a round of Baz IO | 20260505 20:20:53.614 | INFO | Populating RTMetrics report | 20260505 20:20:53.614 | INFO | RTMetrics report populated | 20260505 20:20:53.614 | INFO | Opening tmp RTMetrics json file | 20260505 20:20:53.615 | INFO | Serializing RTMetrics to json | 20260505 20:20:53.616 | INFO | Serialization done, swapping tmp file now... | 20260505 20:20:53.617 | INFO | RTMetrics done | 20260505 20:20:53.701 | INFO | Finished a round of Baz IO: Wrote 2.014 GiB in 3.255 seconds | 20260505 20:20:53.701 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.68% / 0.00% / 78.32% | 20260505 20:20:53.701 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.177328% / 3.97364e-06% / 7.15256e-05% | 20260505 20:20:53.701 | INFO | Baz IO overall: | 20260505 20:20:53.701 | INFO | BytesWritten : 3517526201948 | 20260505 20:20:53.701 | INFO | NumEvents : 1663005412528 | 20260505 20:20:55.446 | INFO | Analyzing chunk frames = [6033408,6033920) | 20260505 20:20:55.446 | INFO | MemInfo: { | 20260505 20:20:55.446 | INFO | "gpu" : " 17 GB", | 20260505 20:20:55.446 | INFO | "huge" : "", | 20260505 20:20:55.446 | INFO | "huge_shared" : " 27 GB", | 20260505 20:20:55.446 | INFO | "malloc" : " 14 GB", | 20260505 20:20:55.446 | INFO | "pinned" : " 29 GB" | 20260505 20:20:55.446 | INFO | } | 20260505 20:20:55.447 | INFO | Perf report for the last 5.12972 seconds: | 20260505 20:20:55.447 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:20:55.447 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:55.447 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:55.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:55.447 | INFO | ANALYSIS : 63.2%, 3.0, 98.4ms, 0 | 20260505 20:20:55.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:20:55.447 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260505 20:20:55.447 | INFO | PRE_HQ : 60.2%, 1.8, 55.3ms, 0 | 20260505 20:20:55.447 | INFO | BAZWRITER : 63.0%, 3.7, 118.1ms, 0 | 20260505 20:20:55.447 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 20:20:55.447 | INFO | Uploaded 36GiB to GPU in 2.15296s (16.7212GiB/s) | 20260505 20:20:55.447 | INFO | Downloaded 38GiB to GPU in 2.2395s (16.968GiB/s) | 20260505 20:21:00.598 | INFO | Analyzing chunk frames = [6033920,6034432) | 20260505 20:21:00.598 | INFO | MemInfo: { | 20260505 20:21:00.598 | INFO | "gpu" : " 17 GB", | 20260505 20:21:00.598 | INFO | "huge" : "", | 20260505 20:21:00.598 | INFO | "huge_shared" : " 27 GB", | 20260505 20:21:00.598 | INFO | "malloc" : " 14 GB", | 20260505 20:21:00.599 | INFO | "pinned" : " 29 GB" | 20260505 20:21:00.599 | INFO | } | 20260505 20:21:00.599 | INFO | Perf report for the last 5.15205 seconds: | 20260505 20:21:00.599 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:21:00.599 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:00.599 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:00.599 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:00.599 | INFO | ANALYSIS : 62.5%, 3.0, 97.8ms, 0 | 20260505 20:21:00.599 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:00.599 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:00.599 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:21:00.599 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:00.599 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:00.599 | INFO | Uploaded 36GiB to GPU in 2.14282s (16.8003GiB/s) | 20260505 20:21:00.599 | INFO | Downloaded 34GiB to GPU in 2.02817s (16.7639GiB/s) | 20260505 20:21:05.737 | INFO | Analyzing chunk frames = [6034432,6034944) | 20260505 20:21:05.737 | INFO | MemInfo: { | 20260505 20:21:05.737 | INFO | "gpu" : " 17 GB", | 20260505 20:21:05.737 | INFO | "huge" : "", | 20260505 20:21:05.737 | INFO | "huge_shared" : " 27 GB", | 20260505 20:21:05.737 | INFO | "malloc" : " 14 GB", | 20260505 20:21:05.737 | INFO | "pinned" : " 29 GB" | 20260505 20:21:05.737 | INFO | } | 20260505 20:21:05.737 | INFO | Perf report for the last 5.13875 seconds: | 20260505 20:21:05.737 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:21:05.737 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:05.737 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:05.737 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:05.737 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260505 20:21:05.737 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:05.737 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:05.737 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 20:21:05.737 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:05.737 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:05.738 | INFO | Uploaded 36GiB to GPU in 2.12405s (16.9488GiB/s) | 20260505 20:21:05.738 | INFO | Downloaded 34GiB to GPU in 2.01364s (16.8848GiB/s) | 20260505 20:21:10.884 | INFO | Analyzing chunk frames = [6034944,6035456) | 20260505 20:21:10.884 | INFO | MemInfo: { | 20260505 20:21:10.884 | INFO | "gpu" : " 17 GB", | 20260505 20:21:10.884 | INFO | "huge" : "", | 20260505 20:21:10.884 | INFO | "huge_shared" : " 27 GB", | 20260505 20:21:10.884 | INFO | "malloc" : " 14 GB", | 20260505 20:21:10.884 | INFO | "pinned" : " 29 GB" | 20260505 20:21:10.884 | INFO | } | 20260505 20:21:10.884 | INFO | Perf report for the last 5.14657 seconds: | 20260505 20:21:10.884 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:21:10.884 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:10.884 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:10.884 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:10.884 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260505 20:21:10.884 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:10.884 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:10.884 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:21:10.884 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:10.884 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:10.884 | INFO | Uploaded 36GiB to GPU in 2.21578s (16.2471GiB/s) | 20260505 20:21:10.884 | INFO | Downloaded 34GiB to GPU in 2.06283s (16.4822GiB/s) | 20260505 20:21:16.018 | INFO | Analyzing chunk frames = [6035456,6035968) | 20260505 20:21:16.018 | INFO | MemInfo: { | 20260505 20:21:16.018 | INFO | "gpu" : " 17 GB", | 20260505 20:21:16.018 | INFO | "huge" : "", | 20260505 20:21:16.018 | INFO | "huge_shared" : " 27 GB", | 20260505 20:21:16.018 | INFO | "malloc" : " 14 GB", | 20260505 20:21:16.018 | INFO | "pinned" : " 29 GB" | 20260505 20:21:16.018 | INFO | } | 20260505 20:21:16.018 | INFO | Perf report for the last 5.13414 seconds: | 20260505 20:21:16.018 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:21:16.018 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:16.018 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:16.018 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:16.018 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260505 20:21:16.018 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:16.018 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:16.018 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:21:16.018 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:16.018 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:16.018 | INFO | Uploaded 36GiB to GPU in 2.1348s (16.8634GiB/s) | 20260505 20:21:16.018 | INFO | Downloaded 34GiB to GPU in 2.06267s (16.4835GiB/s) | 20260505 20:21:21.153 | INFO | Analyzing chunk frames = [6035968,6036480) | 20260505 20:21:21.153 | INFO | MemInfo: { | 20260505 20:21:21.153 | INFO | "gpu" : " 17 GB", | 20260505 20:21:21.153 | INFO | "huge" : "", | 20260505 20:21:21.153 | INFO | "huge_shared" : " 27 GB", | 20260505 20:21:21.153 | INFO | "malloc" : " 14 GB", | 20260505 20:21:21.153 | INFO | "pinned" : " 29 GB" | 20260505 20:21:21.153 | INFO | } | 20260505 20:21:21.153 | INFO | Perf report for the last 5.13486 seconds: | 20260505 20:21:21.153 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:21:21.153 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:21.153 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:21.153 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:21.153 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 20:21:21.153 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:21.153 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:21.153 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:21:21.153 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:21.153 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:21.153 | INFO | Uploaded 36GiB to GPU in 2.14897s (16.7522GiB/s) | 20260505 20:21:21.153 | INFO | Downloaded 34GiB to GPU in 2.0573s (16.5265GiB/s) | 20260505 20:21:26.295 | INFO | Analyzing chunk frames = [6036480,6036992) | 20260505 20:21:26.295 | INFO | MemInfo: { | 20260505 20:21:26.295 | INFO | "gpu" : " 17 GB", | 20260505 20:21:26.295 | INFO | "huge" : "", | 20260505 20:21:26.295 | INFO | "huge_shared" : " 27 GB", | 20260505 20:21:26.295 | INFO | "malloc" : " 14 GB", | 20260505 20:21:26.295 | INFO | "pinned" : " 29 GB" | 20260505 20:21:26.295 | INFO | } | 20260505 20:21:26.295 | INFO | Perf report for the last 5.1422 seconds: | 20260505 20:21:26.295 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:21:26.295 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:26.295 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:26.295 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:26.295 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 20:21:26.295 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:26.295 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:26.295 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 20:21:26.295 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:26.295 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:26.295 | INFO | Uploaded 36GiB to GPU in 2.14519s (16.7818GiB/s) | 20260505 20:21:26.296 | INFO | Downloaded 34GiB to GPU in 2.14553s (15.8469GiB/s) | 20260505 20:21:31.432 | INFO | Analyzing chunk frames = [6036992,6037504) | 20260505 20:21:31.432 | INFO | MemInfo: { | 20260505 20:21:31.432 | INFO | "gpu" : " 17 GB", | 20260505 20:21:31.432 | INFO | "huge" : "", | 20260505 20:21:31.432 | INFO | "huge_shared" : " 27 GB", | 20260505 20:21:31.433 | INFO | "malloc" : " 14 GB", | 20260505 20:21:31.433 | INFO | "pinned" : " 29 GB" | 20260505 20:21:31.433 | INFO | } | 20260505 20:21:31.433 | INFO | Perf report for the last 5.13752 seconds: | 20260505 20:21:31.433 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:21:31.433 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:31.433 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:31.433 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:31.433 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 20:21:31.433 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:31.433 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:31.433 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:21:31.433 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:31.433 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:31.433 | INFO | Uploaded 36GiB to GPU in 2.10327s (17.1162GiB/s) | 20260505 20:21:31.433 | INFO | Downloaded 34GiB to GPU in 2.0017s (16.9856GiB/s) | 20260505 20:21:31.433 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 20:21:31.433 | INFO | Total mem usage: 15.3 GiB | 20260505 20:21:31.433 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 20:21:31.433 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.14%) | 20260505 20:21:31.433 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 20:21:31.433 | INFO | High Water Breakdown: | 20260505 20:21:31.433 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 20:21:31.433 | INFO | 38.8% : 5.5 GiB ( 82.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 20:21:31.433 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 20:21:31.433 | INFO | ------------------------------------------ | 20260505 20:21:31.433 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:21:31.433 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 20:21:31.433 | INFO | Total mem usage: 26.5 GiB | 20260505 20:21:31.433 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 20:21:31.433 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 20:21:31.433 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 20:21:31.433 | INFO | High Water Breakdown: | 20260505 20:21:31.433 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:21:31.433 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:21:31.433 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:21:31.433 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:21:31.433 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:21:31.433 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:21:31.433 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 20:21:31.433 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 20:21:31.433 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:21:31.433 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 20:21:31.433 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 20:21:31.433 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 20:21:31.434 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 20:21:31.434 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 20:21:31.434 | INFO | ------------------------------------------ | 20260505 20:21:31.434 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:21:31.434 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 20:21:31.434 | INFO | Total mem usage: 25.6 GiB | 20260505 20:21:31.434 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 20:21:31.434 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 20:21:31.434 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 20:21:31.434 | INFO | High Water Breakdown: | 20260505 20:21:31.434 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 20:21:31.434 | INFO | ------------------------------------------ | 20260505 20:21:31.434 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:21:31.434 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 20:21:31.434 | INFO | Total mem usage: 15.7 GiB | 20260505 20:21:31.434 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 20:21:31.434 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 20:21:31.434 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 20:21:31.434 | INFO | High Water Breakdown: | 20260505 20:21:31.434 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 20:21:31.434 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 20:21:31.434 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 20:21:31.434 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 20:21:31.434 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 20:21:31.434 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 20:21:31.434 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 20:21:31.434 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 20:21:31.434 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 20:21:31.434 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:21:31.434 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 20:21:31.434 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:21:31.434 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 20:21:31.434 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 20:21:31.434 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 20:21:31.434 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 20:21:31.434 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:21:31.434 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 20:21:31.434 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:21:31.434 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:21:31.434 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:21:31.434 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 20:21:31.434 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 20:21:31.434 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:21:31.434 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 20:21:31.434 | INFO | ------------------------------------------ | 20260505 20:21:31.434 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:21:31.434 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 20:21:31.434 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:21:31.434 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:21:31.434 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.44e+01, 1.35e+01 / 1.42e+00 | 20260505 20:21:31.434 | INFO | PerfReport: 0.29%, AggregateStats -- 9.25e-02 / 1.04e-01, 1.06e-02 / 5.30e-03 | 20260505 20:21:31.434 | INFO | PerfReport: 11.77%, Baseline -- 3.70e+00 / 4.50e+00, 5.57e-01 / 1.68e-01 | 20260505 20:21:31.434 | INFO | PerfReport: 3.26%, Histogram -- 1.03e+00 / 1.53e+00, 1.89e-01 / 8.88e-02 | 20260505 20:21:31.434 | INFO | PerfReport: 15.94%, DME -- 4.01e+01 / 4.41e+01, 4.01e+00 / 1.44e+00 | 20260505 20:21:31.434 | INFO | PerfReport: 57.11%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.50e+00 / 1.13e+00 | 20260505 20:21:31.434 | INFO | PerfReport: 7.23%, PulseAccumulating -- 2.28e+00 / 2.85e+00, 4.33e-01 / 1.53e-01 | 20260505 20:21:31.434 | INFO | PerfReport: 3.25%, Metrics -- 1.02e+00 / 1.34e+00, 2.81e-01 / 1.50e-01 | 20260505 20:21:31.434 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.55e-01 / 4.15e-01, 6.24e-02 / 3.54e-02 | 20260505 20:21:31.434 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 20:21:31.434 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:21:31.434 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:21:31.434 | INFO | PerfReport: Full Execution -- 9.94e+01 / -nan, 2.55e+01 / 0.00e+00 | 20260505 20:21:31.434 | INFO | PerfReport: 22.49%, Upload -- 2.23e+01 / 0.00e+00, 7.73e+00 / 0.00e+00 | 20260505 20:21:31.434 | INFO | PerfReport: 21.73%, Download -- 2.16e+01 / 0.00e+00, 7.65e+00 / 0.00e+00 | 20260505 20:21:31.434 | INFO | PerfReport: 31.67%, Compute -- 3.15e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 20:21:31.434 | INFO | PerfReport: 2.33%, LockUpload -- 2.31e+00 / 0.00e+00, 8.08e+00 / 0.00e+00 | 20260505 20:21:31.434 | INFO | PerfReport: 1.30%, LockDownload -- 1.29e+00 / 0.00e+00, 2.79e+00 / 0.00e+00 | 20260505 20:21:31.434 | INFO | PerfReport: 20.47%, LockCompute -- 2.03e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 20:21:31.434 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.40e-02 / 0.00e+00 | 20260505 20:21:31.562 | INFO | Starting a round of Baz IO | 20260505 20:21:34.731 | INFO | Populating RTMetrics report | 20260505 20:21:34.731 | INFO | RTMetrics report populated | 20260505 20:21:34.731 | INFO | Opening tmp RTMetrics json file | 20260505 20:21:34.732 | INFO | Serializing RTMetrics to json | 20260505 20:21:34.735 | INFO | Serialization done, swapping tmp file now... | 20260505 20:21:34.735 | INFO | RTMetrics done | 20260505 20:21:34.819 | INFO | Finished a round of Baz IO: Wrote 2.013 GiB in 3.257 seconds | 20260505 20:21:34.819 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.68% / 0.00% / 78.32% | 20260505 20:21:34.819 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.176593% / 3.97364e-06% / 9.13938e-05% | 20260505 20:21:34.819 | INFO | Baz IO overall: | 20260505 20:21:34.819 | INFO | BytesWritten : 3519687977471 | 20260505 20:21:34.819 | INFO | NumEvents : 1664006313150 | 20260505 20:21:36.571 | INFO | Analyzing chunk frames = [6037504,6038016) | 20260505 20:21:36.571 | INFO | MemInfo: { | 20260505 20:21:36.571 | INFO | "gpu" : " 17 GB", | 20260505 20:21:36.571 | INFO | "huge" : "", | 20260505 20:21:36.571 | INFO | "huge_shared" : " 27 GB", | 20260505 20:21:36.571 | INFO | "malloc" : " 14 GB", | 20260505 20:21:36.571 | INFO | "pinned" : " 29 GB" | 20260505 20:21:36.571 | INFO | } | 20260505 20:21:36.571 | INFO | Perf report for the last 5.13869 seconds: | 20260505 20:21:36.572 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:21:36.572 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:36.572 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:36.572 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:36.572 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 20:21:36.572 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:36.572 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260505 20:21:36.572 | INFO | PRE_HQ : 60.3%, 1.8, 55.3ms, 0 | 20260505 20:21:36.572 | INFO | BAZWRITER : 62.9%, 3.7, 119.2ms, 0 | 20260505 20:21:36.572 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 20:21:36.572 | INFO | Uploaded 36GiB to GPU in 2.16363s (16.6387GiB/s) | 20260505 20:21:36.572 | INFO | Downloaded 38GiB to GPU in 2.25221s (16.8723GiB/s) | 20260505 20:21:41.709 | INFO | Analyzing chunk frames = [6038016,6038528) | 20260505 20:21:41.709 | INFO | MemInfo: { | 20260505 20:21:41.709 | INFO | "gpu" : " 17 GB", | 20260505 20:21:41.709 | INFO | "huge" : "", | 20260505 20:21:41.709 | INFO | "huge_shared" : " 27 GB", | 20260505 20:21:41.709 | INFO | "malloc" : " 14 GB", | 20260505 20:21:41.709 | INFO | "pinned" : " 29 GB" | 20260505 20:21:41.709 | INFO | } | 20260505 20:21:41.709 | INFO | Perf report for the last 5.13758 seconds: | 20260505 20:21:41.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:21:41.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:41.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:41.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:41.709 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 20:21:41.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:41.709 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:41.709 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 20:21:41.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:41.709 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:41.709 | INFO | Uploaded 36GiB to GPU in 2.09345s (17.1965GiB/s) | 20260505 20:21:41.709 | INFO | Downloaded 34GiB to GPU in 2.03116s (16.7392GiB/s) | 20260505 20:21:46.853 | INFO | Analyzing chunk frames = [6038528,6039040) | 20260505 20:21:46.853 | INFO | MemInfo: { | 20260505 20:21:46.853 | INFO | "gpu" : " 17 GB", | 20260505 20:21:46.853 | INFO | "huge" : "", | 20260505 20:21:46.853 | INFO | "huge_shared" : " 27 GB", | 20260505 20:21:46.853 | INFO | "malloc" : " 14 GB", | 20260505 20:21:46.853 | INFO | "pinned" : " 29 GB" | 20260505 20:21:46.853 | INFO | } | 20260505 20:21:46.853 | INFO | Perf report for the last 5.14431 seconds: | 20260505 20:21:46.853 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:21:46.853 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:46.853 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:46.853 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:46.853 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260505 20:21:46.853 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:46.853 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:46.853 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:21:46.853 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:46.853 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:46.854 | INFO | Uploaded 36GiB to GPU in 2.14066s (16.8172GiB/s) | 20260505 20:21:46.854 | INFO | Downloaded 34GiB to GPU in 2.0376s (16.6863GiB/s) | 20260505 20:21:51.995 | INFO | Analyzing chunk frames = [6039040,6039552) | 20260505 20:21:51.995 | INFO | MemInfo: { | 20260505 20:21:51.995 | INFO | "gpu" : " 17 GB", | 20260505 20:21:51.995 | INFO | "huge" : "", | 20260505 20:21:51.995 | INFO | "huge_shared" : " 27 GB", | 20260505 20:21:51.995 | INFO | "malloc" : " 14 GB", | 20260505 20:21:51.995 | INFO | "pinned" : " 29 GB" | 20260505 20:21:51.995 | INFO | } | 20260505 20:21:51.996 | INFO | Perf report for the last 5.14219 seconds: | 20260505 20:21:51.996 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:21:51.996 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:51.996 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:51.996 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:51.996 | INFO | ANALYSIS : 62.4%, 3.0, 97.0ms, 0 | 20260505 20:21:51.996 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:51.996 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:51.996 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:21:51.996 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:51.996 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:51.996 | INFO | Uploaded 36GiB to GPU in 2.21227s (16.2729GiB/s) | 20260505 20:21:51.996 | INFO | Downloaded 34GiB to GPU in 2.06391s (16.4736GiB/s) | 20260505 20:21:57.129 | INFO | Analyzing chunk frames = [6039552,6040064) | 20260505 20:21:57.129 | INFO | MemInfo: { | 20260505 20:21:57.129 | INFO | "gpu" : " 17 GB", | 20260505 20:21:57.129 | INFO | "huge" : "", | 20260505 20:21:57.129 | INFO | "huge_shared" : " 27 GB", | 20260505 20:21:57.129 | INFO | "malloc" : " 14 GB", | 20260505 20:21:57.129 | INFO | "pinned" : " 29 GB" | 20260505 20:21:57.129 | INFO | } | 20260505 20:21:57.129 | INFO | Perf report for the last 5.13361 seconds: | 20260505 20:21:57.129 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:21:57.129 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:57.129 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:57.129 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:57.129 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260505 20:21:57.129 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:57.129 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:57.129 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:21:57.129 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:57.129 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:21:57.129 | INFO | Uploaded 36GiB to GPU in 2.12279s (16.9589GiB/s) | 20260505 20:21:57.129 | INFO | Downloaded 34GiB to GPU in 2.06014s (16.5037GiB/s) | 20260505 20:22:02.264 | INFO | Analyzing chunk frames = [6040064,6040576) | 20260505 20:22:02.264 | INFO | MemInfo: { | 20260505 20:22:02.264 | INFO | "gpu" : " 17 GB", | 20260505 20:22:02.264 | INFO | "huge" : "", | 20260505 20:22:02.264 | INFO | "huge_shared" : " 27 GB", | 20260505 20:22:02.264 | INFO | "malloc" : " 14 GB", | 20260505 20:22:02.264 | INFO | "pinned" : " 29 GB" | 20260505 20:22:02.264 | INFO | } | 20260505 20:22:02.264 | INFO | Perf report for the last 5.13524 seconds: | 20260505 20:22:02.265 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:22:02.265 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:02.265 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:02.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:02.265 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 20:22:02.265 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:02.265 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:02.265 | INFO | PRE_HQ : 27.5%, 1.0, 14.2ms, 0 | 20260505 20:22:02.265 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:02.265 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:02.265 | INFO | Uploaded 36GiB to GPU in 2.15131s (16.734GiB/s) | 20260505 20:22:02.265 | INFO | Downloaded 34GiB to GPU in 2.063s (16.4809GiB/s) | 20260505 20:22:07.401 | INFO | Analyzing chunk frames = [6040576,6041088) | 20260505 20:22:07.401 | INFO | MemInfo: { | 20260505 20:22:07.401 | INFO | "gpu" : " 17 GB", | 20260505 20:22:07.401 | INFO | "huge" : "", | 20260505 20:22:07.401 | INFO | "huge_shared" : " 27 GB", | 20260505 20:22:07.401 | INFO | "malloc" : " 14 GB", | 20260505 20:22:07.401 | INFO | "pinned" : " 29 GB" | 20260505 20:22:07.401 | INFO | } | 20260505 20:22:07.401 | INFO | Perf report for the last 5.13686 seconds: | 20260505 20:22:07.401 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:22:07.401 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:07.401 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:07.401 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:07.401 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 20:22:07.401 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:07.401 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:07.401 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 20:22:07.401 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:07.401 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:07.402 | INFO | Uploaded 36GiB to GPU in 2.11151s (17.0494GiB/s) | 20260505 20:22:07.402 | INFO | Downloaded 34GiB to GPU in 2.13368s (15.9349GiB/s) | 20260505 20:22:12.548 | INFO | Analyzing chunk frames = [6041088,6041600) | 20260505 20:22:12.548 | INFO | MemInfo: { | 20260505 20:22:12.548 | INFO | "gpu" : " 17 GB", | 20260505 20:22:12.548 | INFO | "huge" : "", | 20260505 20:22:12.548 | INFO | "huge_shared" : " 27 GB", | 20260505 20:22:12.548 | INFO | "malloc" : " 14 GB", | 20260505 20:22:12.548 | INFO | "pinned" : " 29 GB" | 20260505 20:22:12.548 | INFO | } | 20260505 20:22:12.548 | INFO | Perf report for the last 5.14647 seconds: | 20260505 20:22:12.548 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:22:12.548 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:12.548 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:12.548 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:12.548 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 20:22:12.548 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:12.548 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:12.548 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 20:22:12.548 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:12.548 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:12.548 | INFO | Uploaded 36GiB to GPU in 2.19983s (16.3649GiB/s) | 20260505 20:22:12.548 | INFO | Downloaded 34GiB to GPU in 2.0357s (16.7019GiB/s) | 20260505 20:22:12.679 | INFO | Starting a round of Baz IO | 20260505 20:22:15.839 | INFO | Populating RTMetrics report | 20260505 20:22:15.839 | INFO | RTMetrics report populated | 20260505 20:22:15.839 | INFO | Opening tmp RTMetrics json file | 20260505 20:22:15.840 | INFO | Serializing RTMetrics to json | 20260505 20:22:15.841 | INFO | Serialization done, swapping tmp file now... | 20260505 20:22:15.842 | INFO | RTMetrics done | 20260505 20:22:15.924 | INFO | Finished a round of Baz IO: Wrote 2.012 GiB in 3.244 seconds | 20260505 20:22:15.924 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.68% / 0.00% / 78.32% | 20260505 20:22:15.924 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.177093% / 0% / 0.000103315% | 20260505 20:22:15.924 | INFO | Baz IO overall: | 20260505 20:22:15.924 | INFO | BytesWritten : 3521848267398 | 20260505 20:22:15.924 | INFO | NumEvents : 1665006846297 | 20260505 20:22:17.698 | INFO | Analyzing chunk frames = [6041600,6042112) | 20260505 20:22:17.698 | INFO | MemInfo: { | 20260505 20:22:17.698 | INFO | "gpu" : " 17 GB", | 20260505 20:22:17.698 | INFO | "huge" : "", | 20260505 20:22:17.698 | INFO | "huge_shared" : " 27 GB", | 20260505 20:22:17.698 | INFO | "malloc" : " 14 GB", | 20260505 20:22:17.698 | INFO | "pinned" : " 29 GB" | 20260505 20:22:17.698 | INFO | } | 20260505 20:22:17.698 | INFO | Perf report for the last 5.15051 seconds: | 20260505 20:22:17.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:22:17.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:17.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:17.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:17.698 | INFO | ANALYSIS : 62.8%, 3.0, 98.3ms, 0 | 20260505 20:22:17.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:17.698 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260505 20:22:17.698 | INFO | PRE_HQ : 60.0%, 1.8, 55.6ms, 0 | 20260505 20:22:17.698 | INFO | BAZWRITER : 62.5%, 3.7, 118.4ms, 0 | 20260505 20:22:17.698 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 20:22:17.699 | INFO | Uploaded 36GiB to GPU in 2.17079s (16.5838GiB/s) | 20260505 20:22:17.699 | INFO | Downloaded 38GiB to GPU in 2.24925s (16.8945GiB/s) | 20260505 20:22:22.832 | INFO | Analyzing chunk frames = [6042112,6042624) | 20260505 20:22:22.832 | INFO | MemInfo: { | 20260505 20:22:22.832 | INFO | "gpu" : " 17 GB", | 20260505 20:22:22.832 | INFO | "huge" : "", | 20260505 20:22:22.832 | INFO | "huge_shared" : " 27 GB", | 20260505 20:22:22.832 | INFO | "malloc" : " 14 GB", | 20260505 20:22:22.832 | INFO | "pinned" : " 29 GB" | 20260505 20:22:22.832 | INFO | } | 20260505 20:22:22.832 | INFO | Perf report for the last 5.13351 seconds: | 20260505 20:22:22.832 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:22:22.832 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:22.832 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:22.832 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:22.832 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260505 20:22:22.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:22.832 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:22.832 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:22:22.832 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:22.832 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:22.832 | INFO | Uploaded 36GiB to GPU in 2.08838s (17.2382GiB/s) | 20260505 20:22:22.832 | INFO | Downloaded 34GiB to GPU in 2.02492s (16.7908GiB/s) | 20260505 20:22:27.963 | INFO | Analyzing chunk frames = [6042624,6043136) | 20260505 20:22:27.963 | INFO | MemInfo: { | 20260505 20:22:27.963 | INFO | "gpu" : " 17 GB", | 20260505 20:22:27.963 | INFO | "huge" : "", | 20260505 20:22:27.963 | INFO | "huge_shared" : " 27 GB", | 20260505 20:22:27.963 | INFO | "malloc" : " 14 GB", | 20260505 20:22:27.963 | INFO | "pinned" : " 29 GB" | 20260505 20:22:27.963 | INFO | } | 20260505 20:22:27.963 | INFO | Perf report for the last 5.13141 seconds: | 20260505 20:22:27.963 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:22:27.963 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:27.963 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:27.963 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:27.963 | INFO | ANALYSIS : 62.3%, 3.0, 96.7ms, 0 | 20260505 20:22:27.963 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:27.963 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:27.963 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:22:27.963 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:27.963 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:27.964 | INFO | Uploaded 36GiB to GPU in 2.1242s (16.9476GiB/s) | 20260505 20:22:27.964 | INFO | Downloaded 34GiB to GPU in 2.01577s (16.867GiB/s) | 20260505 20:22:33.106 | INFO | Analyzing chunk frames = [6043136,6043648) | 20260505 20:22:33.106 | INFO | MemInfo: { | 20260505 20:22:33.106 | INFO | "gpu" : " 17 GB", | 20260505 20:22:33.106 | INFO | "huge" : "", | 20260505 20:22:33.106 | INFO | "huge_shared" : " 27 GB", | 20260505 20:22:33.106 | INFO | "malloc" : " 14 GB", | 20260505 20:22:33.106 | INFO | "pinned" : " 29 GB" | 20260505 20:22:33.106 | INFO | } | 20260505 20:22:33.106 | INFO | Perf report for the last 5.1427 seconds: | 20260505 20:22:33.106 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:22:33.106 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:33.106 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:33.106 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:33.106 | INFO | ANALYSIS : 62.2%, 3.0, 96.6ms, 0 | 20260505 20:22:33.106 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:33.106 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:33.106 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:22:33.106 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:33.106 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:33.106 | INFO | Uploaded 36GiB to GPU in 2.18359s (16.4866GiB/s) | 20260505 20:22:33.106 | INFO | Downloaded 34GiB to GPU in 2.06757s (16.4444GiB/s) | 20260505 20:22:38.249 | INFO | Analyzing chunk frames = [6043648,6044160) | 20260505 20:22:38.249 | INFO | MemInfo: { | 20260505 20:22:38.249 | INFO | "gpu" : " 17 GB", | 20260505 20:22:38.249 | INFO | "huge" : "", | 20260505 20:22:38.250 | INFO | "huge_shared" : " 27 GB", | 20260505 20:22:38.250 | INFO | "malloc" : " 14 GB", | 20260505 20:22:38.250 | INFO | "pinned" : " 29 GB" | 20260505 20:22:38.250 | INFO | } | 20260505 20:22:38.250 | INFO | Perf report for the last 5.14364 seconds: | 20260505 20:22:38.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:22:38.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:38.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:38.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:38.250 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260505 20:22:38.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:38.250 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:38.250 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:22:38.250 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:38.250 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:38.250 | INFO | Uploaded 36GiB to GPU in 2.13426s (16.8677GiB/s) | 20260505 20:22:38.250 | INFO | Downloaded 34GiB to GPU in 2.06366s (16.4756GiB/s) | 20260505 20:22:43.383 | INFO | Analyzing chunk frames = [6044160,6044672) | 20260505 20:22:43.383 | INFO | MemInfo: { | 20260505 20:22:43.383 | INFO | "gpu" : " 17 GB", | 20260505 20:22:43.383 | INFO | "huge" : "", | 20260505 20:22:43.383 | INFO | "huge_shared" : " 27 GB", | 20260505 20:22:43.383 | INFO | "malloc" : " 14 GB", | 20260505 20:22:43.383 | INFO | "pinned" : " 29 GB" | 20260505 20:22:43.383 | INFO | } | 20260505 20:22:43.384 | INFO | Perf report for the last 5.13393 seconds: | 20260505 20:22:43.384 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:22:43.384 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:43.384 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:43.384 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:43.384 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260505 20:22:43.384 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:43.384 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:43.384 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 20:22:43.384 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:43.384 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:43.384 | INFO | Uploaded 36GiB to GPU in 2.07893s (17.3166GiB/s) | 20260505 20:22:43.384 | INFO | Downloaded 34GiB to GPU in 2.0385s (16.6789GiB/s) | 20260505 20:22:48.522 | INFO | Analyzing chunk frames = [6044672,6045184) | 20260505 20:22:48.522 | INFO | MemInfo: { | 20260505 20:22:48.522 | INFO | "gpu" : " 17 GB", | 20260505 20:22:48.522 | INFO | "huge" : "", | 20260505 20:22:48.522 | INFO | "huge_shared" : " 27 GB", | 20260505 20:22:48.522 | INFO | "malloc" : " 14 GB", | 20260505 20:22:48.522 | INFO | "pinned" : " 29 GB" | 20260505 20:22:48.522 | INFO | } | 20260505 20:22:48.522 | INFO | Perf report for the last 5.13853 seconds: | 20260505 20:22:48.522 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:22:48.522 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:48.522 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:48.522 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:48.522 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 20:22:48.522 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:48.522 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:48.522 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 20:22:48.522 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:48.522 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:48.522 | INFO | Uploaded 36GiB to GPU in 2.11694s (17.0057GiB/s) | 20260505 20:22:48.522 | INFO | Downloaded 34GiB to GPU in 2.14467s (15.8533GiB/s) | 20260505 20:22:53.659 | INFO | Analyzing chunk frames = [6045184,6045696) | 20260505 20:22:53.659 | INFO | MemInfo: { | 20260505 20:22:53.659 | INFO | "gpu" : " 17 GB", | 20260505 20:22:53.659 | INFO | "huge" : "", | 20260505 20:22:53.659 | INFO | "huge_shared" : " 27 GB", | 20260505 20:22:53.659 | INFO | "malloc" : " 14 GB", | 20260505 20:22:53.659 | INFO | "pinned" : " 29 GB" | 20260505 20:22:53.659 | INFO | } | 20260505 20:22:53.659 | INFO | Perf report for the last 5.13726 seconds: | 20260505 20:22:53.659 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:22:53.659 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:53.659 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:53.659 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:53.659 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260505 20:22:53.659 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:53.659 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:53.659 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:22:53.659 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:53.660 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:53.660 | INFO | Uploaded 36GiB to GPU in 2.12956s (16.9049GiB/s) | 20260505 20:22:53.660 | INFO | Downloaded 34GiB to GPU in 2.01067s (16.9098GiB/s) | 20260505 20:22:53.660 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 20:22:53.660 | INFO | Total mem usage: 15.3 GiB | 20260505 20:22:53.660 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 20:22:53.660 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.15%) | 20260505 20:22:53.660 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 20:22:53.660 | INFO | High Water Breakdown: | 20260505 20:22:53.660 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 20:22:53.660 | INFO | 38.8% : 5.5 GiB ( 82.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 20:22:53.660 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 20:22:53.660 | INFO | ------------------------------------------ | 20260505 20:22:53.660 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:22:53.660 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 20:22:53.660 | INFO | Total mem usage: 26.5 GiB | 20260505 20:22:53.660 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 20:22:53.660 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 20:22:53.660 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 20:22:53.660 | INFO | High Water Breakdown: | 20260505 20:22:53.660 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:22:53.660 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:22:53.660 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:22:53.660 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:22:53.660 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:22:53.660 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:22:53.660 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 20:22:53.660 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 20:22:53.660 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:22:53.660 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 20:22:53.660 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 20:22:53.660 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 20:22:53.660 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 20:22:53.660 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 20:22:53.660 | INFO | ------------------------------------------ | 20260505 20:22:53.660 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:22:53.660 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 20:22:53.660 | INFO | Total mem usage: 25.6 GiB | 20260505 20:22:53.660 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 20:22:53.660 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 20:22:53.660 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 20:22:53.660 | INFO | High Water Breakdown: | 20260505 20:22:53.660 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 20:22:53.660 | INFO | ------------------------------------------ | 20260505 20:22:53.660 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:22:53.660 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 20:22:53.660 | INFO | Total mem usage: 15.7 GiB | 20260505 20:22:53.661 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 20:22:53.661 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 20:22:53.661 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 20:22:53.661 | INFO | High Water Breakdown: | 20260505 20:22:53.661 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 20:22:53.661 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 20:22:53.661 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 20:22:53.661 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 20:22:53.661 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 20:22:53.661 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 20:22:53.661 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 20:22:53.661 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 20:22:53.661 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 20:22:53.661 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:22:53.661 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 20:22:53.661 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:22:53.661 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 20:22:53.661 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 20:22:53.661 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 20:22:53.661 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 20:22:53.661 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:22:53.661 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 20:22:53.661 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:22:53.661 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:22:53.661 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:22:53.661 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 20:22:53.661 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 20:22:53.661 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:22:53.661 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 20:22:53.661 | INFO | ------------------------------------------ | 20260505 20:22:53.661 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:22:53.661 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 20:22:53.661 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:22:53.661 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:22:53.661 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.46e+01, 1.30e+01 / 2.05e+00 | 20260505 20:22:53.661 | INFO | PerfReport: 0.29%, AggregateStats -- 9.26e-02 / 1.04e-01, 1.07e-02 / 6.86e-03 | 20260505 20:22:53.661 | INFO | PerfReport: 11.79%, Baseline -- 3.71e+00 / 4.54e+00, 5.68e-01 / 1.35e-01 | 20260505 20:22:53.661 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.52e+00, 1.89e-01 / 8.88e-02 | 20260505 20:22:53.661 | INFO | PerfReport: 15.94%, DME -- 4.02e+01 / 4.44e+01, 4.10e+00 / 2.15e+00 | 20260505 20:22:53.661 | INFO | PerfReport: 57.11%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.52e+00 / 1.15e+00 | 20260505 20:22:53.661 | INFO | PerfReport: 7.22%, PulseAccumulating -- 2.27e+00 / 2.83e+00, 4.38e-01 / 2.26e-01 | 20260505 20:22:53.661 | INFO | PerfReport: 3.24%, Metrics -- 1.02e+00 / 1.32e+00, 2.81e-01 / 1.45e-01 | 20260505 20:22:53.661 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.56e-01 / 4.20e-01, 6.28e-02 / 3.85e-02 | 20260505 20:22:53.661 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 20:22:53.661 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:22:53.661 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:22:53.661 | INFO | PerfReport: Full Execution -- 9.93e+01 / -nan, 2.51e+01 / 0.00e+00 | 20260505 20:22:53.661 | INFO | PerfReport: 22.44%, Upload -- 2.23e+01 / 0.00e+00, 7.70e+00 / 0.00e+00 | 20260505 20:22:53.661 | INFO | PerfReport: 21.83%, Download -- 2.17e+01 / 0.00e+00, 7.74e+00 / 0.00e+00 | 20260505 20:22:53.661 | INFO | PerfReport: 31.72%, Compute -- 3.15e+01 / 0.00e+00, 1.30e+01 / 0.00e+00 | 20260505 20:22:53.661 | INFO | PerfReport: 2.23%, LockUpload -- 2.21e+00 / 0.00e+00, 7.84e+00 / 0.00e+00 | 20260505 20:22:53.661 | INFO | PerfReport: 1.40%, LockDownload -- 1.39e+00 / 0.00e+00, 2.99e+00 / 0.00e+00 | 20260505 20:22:53.661 | INFO | PerfReport: 20.36%, LockCompute -- 2.02e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 20:22:53.661 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.84e-02 / 0.00e+00 | 20260505 20:22:53.792 | INFO | Starting a round of Baz IO | 20260505 20:22:56.962 | INFO | Populating RTMetrics report | 20260505 20:22:56.962 | INFO | RTMetrics report populated | 20260505 20:22:56.962 | INFO | Opening tmp RTMetrics json file | 20260505 20:22:56.963 | INFO | Serializing RTMetrics to json | 20260505 20:22:56.964 | INFO | Serialization done, swapping tmp file now... | 20260505 20:22:56.964 | INFO | RTMetrics done | 20260505 20:22:57.046 | INFO | Finished a round of Baz IO: Wrote 2.014 GiB in 3.254 seconds | 20260505 20:22:57.046 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.68% / 0.00% / 78.32% | 20260505 20:22:57.046 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.176771% / 3.97364e-06% / 9.93411e-05% | 20260505 20:22:57.046 | INFO | Baz IO overall: | 20260505 20:22:57.046 | INFO | BytesWritten : 3524010330947 | 20260505 20:22:57.046 | INFO | NumEvents : 1666008914475 | 20260505 20:22:58.807 | INFO | Analyzing chunk frames = [6045696,6046208) | 20260505 20:22:58.807 | INFO | MemInfo: { | 20260505 20:22:58.807 | INFO | "gpu" : " 17 GB", | 20260505 20:22:58.807 | INFO | "huge" : "", | 20260505 20:22:58.807 | INFO | "huge_shared" : " 27 GB", | 20260505 20:22:58.807 | INFO | "malloc" : " 14 GB", | 20260505 20:22:58.807 | INFO | "pinned" : " 29 GB" | 20260505 20:22:58.807 | INFO | } | 20260505 20:22:58.807 | INFO | Perf report for the last 5.1477 seconds: | 20260505 20:22:58.807 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:22:58.807 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:58.807 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:58.807 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:58.807 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 20:22:58.807 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:22:58.807 | INFO | RT_METRICS : 43.9%, 1.0, 23.1ms, 0 | 20260505 20:22:58.807 | INFO | PRE_HQ : 60.1%, 1.8, 55.0ms, 0 | 20260505 20:22:58.807 | INFO | BAZWRITER : 62.7%, 3.7, 118.7ms, 0 | 20260505 20:22:58.807 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 20:22:58.808 | INFO | Uploaded 36GiB to GPU in 2.17104s (16.5819GiB/s) | 20260505 20:22:58.808 | INFO | Downloaded 38GiB to GPU in 2.25458s (16.8546GiB/s) | 20260505 20:23:03.945 | INFO | Analyzing chunk frames = [6046208,6046720) | 20260505 20:23:03.945 | INFO | MemInfo: { | 20260505 20:23:03.945 | INFO | "gpu" : " 17 GB", | 20260505 20:23:03.945 | INFO | "huge" : "", | 20260505 20:23:03.945 | INFO | "huge_shared" : " 27 GB", | 20260505 20:23:03.945 | INFO | "malloc" : " 14 GB", | 20260505 20:23:03.945 | INFO | "pinned" : " 29 GB" | 20260505 20:23:03.945 | INFO | } | 20260505 20:23:03.945 | INFO | Perf report for the last 5.13794 seconds: | 20260505 20:23:03.945 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:23:03.945 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:03.945 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:03.945 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:03.945 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 20:23:03.945 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:03.945 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:03.945 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:23:03.945 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:03.945 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:03.945 | INFO | Uploaded 36GiB to GPU in 2.0869s (17.2505GiB/s) | 20260505 20:23:03.945 | INFO | Downloaded 34GiB to GPU in 2.02504s (16.7898GiB/s) | 20260505 20:23:09.075 | INFO | Analyzing chunk frames = [6046720,6047232) | 20260505 20:23:09.075 | INFO | MemInfo: { | 20260505 20:23:09.075 | INFO | "gpu" : " 17 GB", | 20260505 20:23:09.075 | INFO | "huge" : "", | 20260505 20:23:09.075 | INFO | "huge_shared" : " 27 GB", | 20260505 20:23:09.075 | INFO | "malloc" : " 14 GB", | 20260505 20:23:09.075 | INFO | "pinned" : " 29 GB" | 20260505 20:23:09.075 | INFO | } | 20260505 20:23:09.075 | INFO | Perf report for the last 5.13042 seconds: | 20260505 20:23:09.075 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:23:09.075 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:09.075 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:09.075 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:09.076 | INFO | ANALYSIS : 62.2%, 3.0, 96.6ms, 0 | 20260505 20:23:09.076 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:09.076 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:09.076 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:23:09.076 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:09.076 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:09.076 | INFO | Uploaded 36GiB to GPU in 2.13834s (16.8355GiB/s) | 20260505 20:23:09.076 | INFO | Downloaded 34GiB to GPU in 2.02002s (16.8315GiB/s) | 20260505 20:23:14.212 | INFO | Analyzing chunk frames = [6047232,6047744) | 20260505 20:23:14.212 | INFO | MemInfo: { | 20260505 20:23:14.212 | INFO | "gpu" : " 17 GB", | 20260505 20:23:14.212 | INFO | "huge" : "", | 20260505 20:23:14.212 | INFO | "huge_shared" : " 27 GB", | 20260505 20:23:14.212 | INFO | "malloc" : " 14 GB", | 20260505 20:23:14.212 | INFO | "pinned" : " 29 GB" | 20260505 20:23:14.212 | INFO | } | 20260505 20:23:14.212 | INFO | Perf report for the last 5.13702 seconds: | 20260505 20:23:14.212 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:23:14.212 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:14.212 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:14.212 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:14.213 | INFO | ANALYSIS : 62.3%, 3.0, 96.7ms, 0 | 20260505 20:23:14.213 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:14.213 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:14.213 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:23:14.213 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:14.213 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:14.213 | INFO | Uploaded 36GiB to GPU in 2.1662s (16.619GiB/s) | 20260505 20:23:14.213 | INFO | Downloaded 34GiB to GPU in 2.0582s (16.5193GiB/s) | 20260505 20:23:19.355 | INFO | Analyzing chunk frames = [6047744,6048256) | 20260505 20:23:19.355 | INFO | MemInfo: { | 20260505 20:23:19.355 | INFO | "gpu" : " 17 GB", | 20260505 20:23:19.355 | INFO | "huge" : "", | 20260505 20:23:19.355 | INFO | "huge_shared" : " 27 GB", | 20260505 20:23:19.355 | INFO | "malloc" : " 14 GB", | 20260505 20:23:19.355 | INFO | "pinned" : " 29 GB" | 20260505 20:23:19.355 | INFO | } | 20260505 20:23:19.356 | INFO | Perf report for the last 5.14303 seconds: | 20260505 20:23:19.356 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:23:19.356 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:19.356 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:19.356 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:19.356 | INFO | ANALYSIS : 61.9%, 3.0, 96.8ms, 0 | 20260505 20:23:19.356 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:19.356 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:19.356 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:23:19.356 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:19.356 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:19.356 | INFO | Uploaded 36GiB to GPU in 2.1432s (16.7973GiB/s) | 20260505 20:23:19.356 | INFO | Downloaded 34GiB to GPU in 2.06743s (16.4455GiB/s) | 20260505 20:23:24.499 | INFO | Analyzing chunk frames = [6048256,6048768) | 20260505 20:23:24.499 | INFO | MemInfo: { | 20260505 20:23:24.499 | INFO | "gpu" : " 17 GB", | 20260505 20:23:24.499 | INFO | "huge" : "", | 20260505 20:23:24.499 | INFO | "huge_shared" : " 27 GB", | 20260505 20:23:24.499 | INFO | "malloc" : " 14 GB", | 20260505 20:23:24.499 | INFO | "pinned" : " 29 GB" | 20260505 20:23:24.499 | INFO | } | 20260505 20:23:24.499 | INFO | Perf report for the last 5.14361 seconds: | 20260505 20:23:24.499 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:23:24.499 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:24.499 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:24.499 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:24.499 | INFO | ANALYSIS : 62.1%, 3.0, 97.1ms, 0 | 20260505 20:23:24.499 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:24.499 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:24.499 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:23:24.499 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:24.499 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:24.499 | INFO | Uploaded 36GiB to GPU in 2.14681s (16.7691GiB/s) | 20260505 20:23:24.500 | INFO | Downloaded 34GiB to GPU in 2.05591s (16.5377GiB/s) | 20260505 20:23:29.641 | INFO | Analyzing chunk frames = [6048768,6049280) | 20260505 20:23:29.641 | INFO | MemInfo: { | 20260505 20:23:29.641 | INFO | "gpu" : " 17 GB", | 20260505 20:23:29.641 | INFO | "huge" : "", | 20260505 20:23:29.641 | INFO | "huge_shared" : " 27 GB", | 20260505 20:23:29.641 | INFO | "malloc" : " 14 GB", | 20260505 20:23:29.641 | INFO | "pinned" : " 29 GB" | 20260505 20:23:29.641 | INFO | } | 20260505 20:23:29.642 | INFO | Perf report for the last 5.14236 seconds: | 20260505 20:23:29.642 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:23:29.642 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:29.642 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:29.642 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:29.642 | INFO | ANALYSIS : 62.7%, 3.0, 98.0ms, 0 | 20260505 20:23:29.642 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:29.642 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:29.642 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 20:23:29.642 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:29.642 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:29.642 | INFO | Uploaded 36GiB to GPU in 2.15884s (16.6756GiB/s) | 20260505 20:23:29.642 | INFO | Downloaded 34GiB to GPU in 2.07447s (16.3897GiB/s) | 20260505 20:23:34.776 | INFO | Analyzing chunk frames = [6049280,6049792) | 20260505 20:23:34.776 | INFO | MemInfo: { | 20260505 20:23:34.776 | INFO | "gpu" : " 17 GB", | 20260505 20:23:34.776 | INFO | "huge" : "", | 20260505 20:23:34.776 | INFO | "huge_shared" : " 27 GB", | 20260505 20:23:34.776 | INFO | "malloc" : " 14 GB", | 20260505 20:23:34.776 | INFO | "pinned" : " 29 GB" | 20260505 20:23:34.776 | INFO | } | 20260505 20:23:34.776 | INFO | Perf report for the last 5.13441 seconds: | 20260505 20:23:34.776 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:23:34.776 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:34.776 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:34.776 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:34.776 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 20:23:34.776 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:34.776 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:34.776 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 20:23:34.776 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:34.776 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:34.776 | INFO | Uploaded 36GiB to GPU in 2.10231s (17.124GiB/s) | 20260505 20:23:34.776 | INFO | Downloaded 34GiB to GPU in 1.99861s (17.0118GiB/s) | 20260505 20:23:34.905 | INFO | Starting a round of Baz IO | 20260505 20:23:38.072 | INFO | Populating RTMetrics report | 20260505 20:23:38.072 | INFO | RTMetrics report populated | 20260505 20:23:38.072 | INFO | Opening tmp RTMetrics json file | 20260505 20:23:38.073 | INFO | Serializing RTMetrics to json | 20260505 20:23:38.074 | INFO | Serialization done, swapping tmp file now... | 20260505 20:23:38.074 | INFO | RTMetrics done | 20260505 20:23:38.156 | INFO | Finished a round of Baz IO: Wrote 2.012 GiB in 3.251 seconds | 20260505 20:23:38.156 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.67% / 0.00% / 78.32% | 20260505 20:23:38.156 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.176307% / 0% / 7.94729e-05% | 20260505 20:23:38.156 | INFO | Baz IO overall: | 20260505 20:23:38.156 | INFO | BytesWritten : 3526171200945 | 20260505 20:23:38.156 | INFO | NumEvents : 1667009932059 | 20260505 20:23:39.914 | INFO | Analyzing chunk frames = [6049792,6050304) | 20260505 20:23:39.914 | INFO | MemInfo: { | 20260505 20:23:39.914 | INFO | "gpu" : " 17 GB", | 20260505 20:23:39.914 | INFO | "huge" : "", | 20260505 20:23:39.914 | INFO | "huge_shared" : " 27 GB", | 20260505 20:23:39.914 | INFO | "malloc" : " 14 GB", | 20260505 20:23:39.914 | INFO | "pinned" : " 29 GB" | 20260505 20:23:39.914 | INFO | } | 20260505 20:23:39.914 | INFO | Perf report for the last 5.13785 seconds: | 20260505 20:23:39.914 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:23:39.914 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:39.914 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:39.914 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:39.914 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 20:23:39.914 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:39.914 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260505 20:23:39.914 | INFO | PRE_HQ : 60.3%, 1.8, 55.5ms, 0 | 20260505 20:23:39.914 | INFO | BAZWRITER : 62.7%, 3.7, 118.0ms, 0 | 20260505 20:23:39.914 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 20:23:39.914 | INFO | Uploaded 36GiB to GPU in 2.16296s (16.6438GiB/s) | 20260505 20:23:39.914 | INFO | Downloaded 38GiB to GPU in 2.24735s (16.9088GiB/s) | 20260505 20:23:45.059 | INFO | Analyzing chunk frames = [6050304,6050816) | 20260505 20:23:45.059 | INFO | MemInfo: { | 20260505 20:23:45.059 | INFO | "gpu" : " 17 GB", | 20260505 20:23:45.059 | INFO | "huge" : "", | 20260505 20:23:45.060 | INFO | "huge_shared" : " 27 GB", | 20260505 20:23:45.060 | INFO | "malloc" : " 14 GB", | 20260505 20:23:45.060 | INFO | "pinned" : " 29 GB" | 20260505 20:23:45.060 | INFO | } | 20260505 20:23:45.060 | INFO | Perf report for the last 5.14585 seconds: | 20260505 20:23:45.060 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:23:45.060 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:45.060 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:45.060 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:45.060 | INFO | ANALYSIS : 62.5%, 3.0, 97.7ms, 0 | 20260505 20:23:45.060 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:45.060 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:45.060 | INFO | PRE_HQ : 27.0%, 1.0, 14.1ms, 0 | 20260505 20:23:45.060 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:45.060 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:45.060 | INFO | Uploaded 36GiB to GPU in 2.14392s (16.7917GiB/s) | 20260505 20:23:45.060 | INFO | Downloaded 34GiB to GPU in 2.04737s (16.6067GiB/s) | 20260505 20:23:50.204 | INFO | Analyzing chunk frames = [6050816,6051328) | 20260505 20:23:50.204 | INFO | MemInfo: { | 20260505 20:23:50.204 | INFO | "gpu" : " 17 GB", | 20260505 20:23:50.204 | INFO | "huge" : "", | 20260505 20:23:50.204 | INFO | "huge_shared" : " 27 GB", | 20260505 20:23:50.204 | INFO | "malloc" : " 14 GB", | 20260505 20:23:50.204 | INFO | "pinned" : " 29 GB" | 20260505 20:23:50.204 | INFO | } | 20260505 20:23:50.204 | INFO | Perf report for the last 5.14451 seconds: | 20260505 20:23:50.204 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:23:50.204 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:50.204 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:50.204 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:50.204 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260505 20:23:50.204 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:50.204 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:50.204 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:23:50.204 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:50.204 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:50.204 | INFO | Uploaded 36GiB to GPU in 2.13123s (16.8917GiB/s) | 20260505 20:23:50.205 | INFO | Downloaded 34GiB to GPU in 2.02285s (16.808GiB/s) | 20260505 20:23:55.335 | INFO | Analyzing chunk frames = [6051328,6051840) | 20260505 20:23:55.335 | INFO | MemInfo: { | 20260505 20:23:55.335 | INFO | "gpu" : " 17 GB", | 20260505 20:23:55.335 | INFO | "huge" : "", | 20260505 20:23:55.335 | INFO | "huge_shared" : " 27 GB", | 20260505 20:23:55.335 | INFO | "malloc" : " 14 GB", | 20260505 20:23:55.336 | INFO | "pinned" : " 29 GB" | 20260505 20:23:55.336 | INFO | } | 20260505 20:23:55.336 | INFO | Perf report for the last 5.13142 seconds: | 20260505 20:23:55.336 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:23:55.336 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:55.336 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:55.336 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:55.336 | INFO | ANALYSIS : 62.1%, 3.0, 96.3ms, 0 | 20260505 20:23:55.336 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:55.336 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:55.336 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:23:55.336 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:55.336 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:23:55.336 | INFO | Uploaded 36GiB to GPU in 2.13536s (16.859GiB/s) | 20260505 20:23:55.336 | INFO | Downloaded 34GiB to GPU in 2.03853s (16.6787GiB/s) | 20260505 20:24:00.469 | INFO | Analyzing chunk frames = [6051840,6052352) | 20260505 20:24:00.469 | INFO | MemInfo: { | 20260505 20:24:00.469 | INFO | "gpu" : " 17 GB", | 20260505 20:24:00.469 | INFO | "huge" : "", | 20260505 20:24:00.469 | INFO | "huge_shared" : " 27 GB", | 20260505 20:24:00.469 | INFO | "malloc" : " 14 GB", | 20260505 20:24:00.469 | INFO | "pinned" : " 29 GB" | 20260505 20:24:00.469 | INFO | } | 20260505 20:24:00.469 | INFO | Perf report for the last 5.13331 seconds: | 20260505 20:24:00.469 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:24:00.469 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:00.469 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:00.469 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:00.469 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260505 20:24:00.469 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:00.469 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:00.469 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 20:24:00.469 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:00.469 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:00.469 | INFO | Uploaded 36GiB to GPU in 2.13202s (16.8854GiB/s) | 20260505 20:24:00.469 | INFO | Downloaded 34GiB to GPU in 2.06461s (16.468GiB/s) | 20260505 20:24:05.616 | INFO | Analyzing chunk frames = [6052352,6052864) | 20260505 20:24:05.616 | INFO | MemInfo: { | 20260505 20:24:05.616 | INFO | "gpu" : " 17 GB", | 20260505 20:24:05.616 | INFO | "huge" : "", | 20260505 20:24:05.616 | INFO | "huge_shared" : " 27 GB", | 20260505 20:24:05.616 | INFO | "malloc" : " 14 GB", | 20260505 20:24:05.617 | INFO | "pinned" : " 29 GB" | 20260505 20:24:05.617 | INFO | } | 20260505 20:24:05.617 | INFO | Perf report for the last 5.14773 seconds: | 20260505 20:24:05.617 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:24:05.617 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:05.617 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:05.617 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:05.617 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 20:24:05.617 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:05.617 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:05.617 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:24:05.617 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:05.617 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:05.617 | INFO | Uploaded 36GiB to GPU in 2.14587s (16.7764GiB/s) | 20260505 20:24:05.617 | INFO | Downloaded 34GiB to GPU in 2.05437s (16.5501GiB/s) | 20260505 20:24:10.755 | INFO | Analyzing chunk frames = [6052864,6053376) | 20260505 20:24:10.755 | INFO | MemInfo: { | 20260505 20:24:10.755 | INFO | "gpu" : " 17 GB", | 20260505 20:24:10.755 | INFO | "huge" : "", | 20260505 20:24:10.755 | INFO | "huge_shared" : " 27 GB", | 20260505 20:24:10.755 | INFO | "malloc" : " 14 GB", | 20260505 20:24:10.755 | INFO | "pinned" : " 29 GB" | 20260505 20:24:10.755 | INFO | } | 20260505 20:24:10.755 | INFO | Perf report for the last 5.13814 seconds: | 20260505 20:24:10.755 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:24:10.755 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:10.755 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:10.755 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:10.755 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 20:24:10.755 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:10.755 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:10.755 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 20:24:10.755 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:10.755 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:10.755 | INFO | Uploaded 36GiB to GPU in 2.13559s (16.8571GiB/s) | 20260505 20:24:10.755 | INFO | Downloaded 34GiB to GPU in 2.14068s (15.8828GiB/s) | 20260505 20:24:15.891 | INFO | Analyzing chunk frames = [6053376,6053888) | 20260505 20:24:15.891 | INFO | MemInfo: { | 20260505 20:24:15.891 | INFO | "gpu" : " 17 GB", | 20260505 20:24:15.891 | INFO | "huge" : "", | 20260505 20:24:15.891 | INFO | "huge_shared" : " 27 GB", | 20260505 20:24:15.891 | INFO | "malloc" : " 14 GB", | 20260505 20:24:15.891 | INFO | "pinned" : " 29 GB" | 20260505 20:24:15.891 | INFO | } | 20260505 20:24:15.891 | INFO | Perf report for the last 5.13653 seconds: | 20260505 20:24:15.891 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:24:15.891 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:15.891 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:15.891 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:15.891 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 20:24:15.891 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:15.892 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:15.892 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 20:24:15.892 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:15.892 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:15.892 | INFO | Uploaded 36GiB to GPU in 2.1087s (17.0722GiB/s) | 20260505 20:24:15.892 | INFO | Downloaded 34GiB to GPU in 1.99891s (17.0092GiB/s) | 20260505 20:24:15.892 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 20:24:15.892 | INFO | Total mem usage: 15.3 GiB | 20260505 20:24:15.892 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 20:24:15.892 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.15%) | 20260505 20:24:15.892 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 20:24:15.892 | INFO | High Water Breakdown: | 20260505 20:24:15.892 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 20:24:15.892 | INFO | 38.8% : 5.5 GiB ( 82.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 20:24:15.892 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 20:24:15.892 | INFO | ------------------------------------------ | 20260505 20:24:15.892 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:24:15.892 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 20:24:15.892 | INFO | Total mem usage: 26.5 GiB | 20260505 20:24:15.892 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 20:24:15.892 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 20:24:15.892 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 20:24:15.892 | INFO | High Water Breakdown: | 20260505 20:24:15.892 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:24:15.892 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:24:15.892 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:24:15.892 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:24:15.892 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:24:15.892 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:24:15.892 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 20:24:15.892 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 20:24:15.892 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:24:15.892 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 20:24:15.892 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 20:24:15.893 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 20:24:15.893 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 20:24:15.893 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 20:24:15.893 | INFO | ------------------------------------------ | 20260505 20:24:15.893 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:24:15.893 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 20:24:15.893 | INFO | Total mem usage: 25.6 GiB | 20260505 20:24:15.893 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 20:24:15.893 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 20:24:15.893 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 20:24:15.893 | INFO | High Water Breakdown: | 20260505 20:24:15.893 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 20:24:15.893 | INFO | ------------------------------------------ | 20260505 20:24:15.893 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:24:15.893 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 20:24:15.893 | INFO | Total mem usage: 15.7 GiB | 20260505 20:24:15.893 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 20:24:15.893 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 20:24:15.893 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 20:24:15.893 | INFO | High Water Breakdown: | 20260505 20:24:15.893 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 20:24:15.893 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 20:24:15.893 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 20:24:15.893 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 20:24:15.893 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 20:24:15.893 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 20:24:15.893 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 20:24:15.893 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 20:24:15.893 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 20:24:15.893 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:24:15.893 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 20:24:15.893 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:24:15.893 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 20:24:15.893 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 20:24:15.893 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 20:24:15.893 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 20:24:15.893 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:24:15.893 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 20:24:15.893 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:24:15.893 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:24:15.893 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:24:15.893 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 20:24:15.893 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 20:24:15.893 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:24:15.893 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 20:24:15.893 | INFO | ------------------------------------------ | 20260505 20:24:15.893 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:24:15.893 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 20:24:15.893 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:24:15.893 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:24:15.893 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.43e+01, 1.36e+01 / 1.60e+00 | 20260505 20:24:15.893 | INFO | PerfReport: 0.29%, AggregateStats -- 9.25e-02 / 1.03e-01, 1.04e-02 / 3.78e-03 | 20260505 20:24:15.893 | INFO | PerfReport: 11.80%, Baseline -- 3.71e+00 / 4.53e+00, 5.63e-01 / 1.57e-01 | 20260505 20:24:15.893 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.53e+00, 1.89e-01 / 8.86e-02 | 20260505 20:24:15.893 | INFO | PerfReport: 15.95%, DME -- 4.02e+01 / 4.43e+01, 4.07e+00 / 1.64e+00 | 20260505 20:24:15.893 | INFO | PerfReport: 57.07%, FrameLabeling -- 1.80e+01 / 1.93e+01, 1.50e+00 / 1.25e+00 | 20260505 20:24:15.893 | INFO | PerfReport: 7.23%, PulseAccumulating -- 2.27e+00 / 2.84e+00, 4.36e-01 / 2.08e-01 | 20260505 20:24:15.893 | INFO | PerfReport: 3.26%, Metrics -- 1.02e+00 / 1.35e+00, 2.85e-01 / 1.46e-01 | 20260505 20:24:15.893 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.56e-01 / 4.21e-01, 6.28e-02 / 4.29e-02 | 20260505 20:24:15.893 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 20:24:15.893 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:24:15.893 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:24:15.893 | INFO | PerfReport: Full Execution -- 9.92e+01 / -nan, 2.53e+01 / 0.00e+00 | 20260505 20:24:15.893 | INFO | PerfReport: 22.44%, Upload -- 2.23e+01 / 0.00e+00, 7.64e+00 / 0.00e+00 | 20260505 20:24:15.893 | INFO | PerfReport: 21.76%, Download -- 2.16e+01 / 0.00e+00, 7.61e+00 / 0.00e+00 | 20260505 20:24:15.894 | INFO | PerfReport: 31.73%, Compute -- 3.15e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 20:24:15.894 | INFO | PerfReport: 2.21%, LockUpload -- 2.20e+00 / 0.00e+00, 7.89e+00 / 0.00e+00 | 20260505 20:24:15.894 | INFO | PerfReport: 1.37%, LockDownload -- 1.36e+00 / 0.00e+00, 2.90e+00 / 0.00e+00 | 20260505 20:24:15.894 | INFO | PerfReport: 20.47%, LockCompute -- 2.03e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 20:24:15.894 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.87e-02 / 0.00e+00 | 20260505 20:24:16.020 | INFO | Starting a round of Baz IO | 20260505 20:24:19.189 | INFO | Populating RTMetrics report | 20260505 20:24:19.189 | INFO | RTMetrics report populated | 20260505 20:24:19.189 | INFO | Opening tmp RTMetrics json file | 20260505 20:24:19.190 | INFO | Serializing RTMetrics to json | 20260505 20:24:19.191 | INFO | Serialization done, swapping tmp file now... | 20260505 20:24:19.191 | INFO | RTMetrics done | 20260505 20:24:19.283 | INFO | Finished a round of Baz IO: Wrote 2.013 GiB in 3.262 seconds | 20260505 20:24:19.283 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.67% / 0.00% / 78.33% | 20260505 20:24:19.283 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.177352% / 0% / 0.000123183% | 20260505 20:24:19.283 | INFO | Baz IO overall: | 20260505 20:24:19.283 | INFO | BytesWritten : 3528332985838 | 20260505 20:24:19.283 | INFO | NumEvents : 1668011885693 | 20260505 20:24:21.029 | INFO | Analyzing chunk frames = [6053888,6054400) | 20260505 20:24:21.029 | INFO | MemInfo: { | 20260505 20:24:21.029 | INFO | "gpu" : " 17 GB", | 20260505 20:24:21.029 | INFO | "huge" : "", | 20260505 20:24:21.029 | INFO | "huge_shared" : " 27 GB", | 20260505 20:24:21.029 | INFO | "malloc" : " 14 GB", | 20260505 20:24:21.029 | INFO | "pinned" : " 29 GB" | 20260505 20:24:21.029 | INFO | } | 20260505 20:24:21.029 | INFO | Perf report for the last 5.13766 seconds: | 20260505 20:24:21.029 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:24:21.029 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:21.029 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:21.029 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:21.029 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 20:24:21.029 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:21.029 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260505 20:24:21.029 | INFO | PRE_HQ : 60.2%, 1.8, 55.4ms, 0 | 20260505 20:24:21.029 | INFO | BAZWRITER : 63.0%, 3.7, 118.2ms, 0 | 20260505 20:24:21.029 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 20:24:21.029 | INFO | Uploaded 36GiB to GPU in 2.15423s (16.7113GiB/s) | 20260505 20:24:21.029 | INFO | Downloaded 38GiB to GPU in 2.24634s (16.9164GiB/s) | 20260505 20:24:26.167 | INFO | Analyzing chunk frames = [6054400,6054912) | 20260505 20:24:26.167 | INFO | MemInfo: { | 20260505 20:24:26.167 | INFO | "gpu" : " 17 GB", | 20260505 20:24:26.167 | INFO | "huge" : "", | 20260505 20:24:26.167 | INFO | "huge_shared" : " 27 GB", | 20260505 20:24:26.168 | INFO | "malloc" : " 14 GB", | 20260505 20:24:26.168 | INFO | "pinned" : " 29 GB" | 20260505 20:24:26.168 | INFO | } | 20260505 20:24:26.168 | INFO | Perf report for the last 5.13861 seconds: | 20260505 20:24:26.168 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:24:26.168 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:26.168 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:26.168 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:26.168 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 20:24:26.168 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:26.168 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:26.168 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:24:26.168 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:26.168 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:26.168 | INFO | Uploaded 36GiB to GPU in 2.13387s (16.8708GiB/s) | 20260505 20:24:26.168 | INFO | Downloaded 34GiB to GPU in 2.02894s (16.7576GiB/s) | 20260505 20:24:31.310 | INFO | Analyzing chunk frames = [6054912,6055424) | 20260505 20:24:31.310 | INFO | MemInfo: { | 20260505 20:24:31.310 | INFO | "gpu" : " 17 GB", | 20260505 20:24:31.310 | INFO | "huge" : "", | 20260505 20:24:31.310 | INFO | "huge_shared" : " 27 GB", | 20260505 20:24:31.310 | INFO | "malloc" : " 14 GB", | 20260505 20:24:31.310 | INFO | "pinned" : " 29 GB" | 20260505 20:24:31.310 | INFO | } | 20260505 20:24:31.310 | INFO | Perf report for the last 5.14213 seconds: | 20260505 20:24:31.310 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:24:31.310 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:31.310 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:31.310 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:31.310 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260505 20:24:31.310 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:31.310 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:31.310 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:24:31.310 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:31.310 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:31.310 | INFO | Uploaded 36GiB to GPU in 2.1387s (16.8327GiB/s) | 20260505 20:24:31.310 | INFO | Downloaded 34GiB to GPU in 2.02706s (16.773GiB/s) | 20260505 20:24:36.451 | INFO | Analyzing chunk frames = [6055424,6055936) | 20260505 20:24:36.451 | INFO | MemInfo: { | 20260505 20:24:36.451 | INFO | "gpu" : " 17 GB", | 20260505 20:24:36.451 | INFO | "huge" : "", | 20260505 20:24:36.451 | INFO | "huge_shared" : " 27 GB", | 20260505 20:24:36.451 | INFO | "malloc" : " 14 GB", | 20260505 20:24:36.451 | INFO | "pinned" : " 29 GB" | 20260505 20:24:36.451 | INFO | } | 20260505 20:24:36.451 | INFO | Perf report for the last 5.14111 seconds: | 20260505 20:24:36.451 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:24:36.451 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:36.451 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:36.451 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:36.451 | INFO | ANALYSIS : 62.5%, 3.0, 97.0ms, 0 | 20260505 20:24:36.451 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:36.451 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:36.451 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:24:36.451 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:36.451 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:36.451 | INFO | Uploaded 36GiB to GPU in 2.2151s (16.2521GiB/s) | 20260505 20:24:36.451 | INFO | Downloaded 34GiB to GPU in 2.05851s (16.5168GiB/s) | 20260505 20:24:41.583 | INFO | Analyzing chunk frames = [6055936,6056448) | 20260505 20:24:41.583 | INFO | MemInfo: { | 20260505 20:24:41.583 | INFO | "gpu" : " 17 GB", | 20260505 20:24:41.583 | INFO | "huge" : "", | 20260505 20:24:41.584 | INFO | "huge_shared" : " 27 GB", | 20260505 20:24:41.584 | INFO | "malloc" : " 14 GB", | 20260505 20:24:41.584 | INFO | "pinned" : " 29 GB" | 20260505 20:24:41.584 | INFO | } | 20260505 20:24:41.584 | INFO | Perf report for the last 5.13269 seconds: | 20260505 20:24:41.584 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:24:41.584 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:41.584 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:41.584 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:41.584 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260505 20:24:41.584 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:41.584 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:41.584 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 20:24:41.584 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:41.584 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:41.584 | INFO | Uploaded 36GiB to GPU in 2.13334s (16.8749GiB/s) | 20260505 20:24:41.584 | INFO | Downloaded 34GiB to GPU in 2.06117s (16.4954GiB/s) | 20260505 20:24:46.722 | INFO | Analyzing chunk frames = [6056448,6056960) | 20260505 20:24:46.722 | INFO | MemInfo: { | 20260505 20:24:46.722 | INFO | "gpu" : " 17 GB", | 20260505 20:24:46.722 | INFO | "huge" : "", | 20260505 20:24:46.723 | INFO | "huge_shared" : " 27 GB", | 20260505 20:24:46.723 | INFO | "malloc" : " 14 GB", | 20260505 20:24:46.723 | INFO | "pinned" : " 29 GB" | 20260505 20:24:46.723 | INFO | } | 20260505 20:24:46.723 | INFO | Perf report for the last 5.13896 seconds: | 20260505 20:24:46.723 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:24:46.723 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:46.723 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:46.723 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:46.723 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 20:24:46.723 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:46.723 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:46.723 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 20:24:46.723 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:46.723 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:46.723 | INFO | Uploaded 36GiB to GPU in 2.16185s (16.6524GiB/s) | 20260505 20:24:46.723 | INFO | Downloaded 34GiB to GPU in 2.0603s (16.5024GiB/s) | 20260505 20:24:51.870 | INFO | Analyzing chunk frames = [6056960,6057472) | 20260505 20:24:51.870 | INFO | MemInfo: { | 20260505 20:24:51.870 | INFO | "gpu" : " 17 GB", | 20260505 20:24:51.870 | INFO | "huge" : "", | 20260505 20:24:51.870 | INFO | "huge_shared" : " 27 GB", | 20260505 20:24:51.870 | INFO | "malloc" : " 14 GB", | 20260505 20:24:51.870 | INFO | "pinned" : " 29 GB" | 20260505 20:24:51.870 | INFO | } | 20260505 20:24:51.870 | INFO | Perf report for the last 5.14754 seconds: | 20260505 20:24:51.870 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:24:51.870 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:51.870 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:51.870 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:51.870 | INFO | ANALYSIS : 62.9%, 3.0, 98.4ms, 0 | 20260505 20:24:51.870 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:51.870 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:51.870 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 20:24:51.870 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:51.870 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:51.870 | INFO | Uploaded 36GiB to GPU in 2.12243s (16.9617GiB/s) | 20260505 20:24:51.870 | INFO | Downloaded 34GiB to GPU in 2.13566s (15.9201GiB/s) | 20260505 20:24:57.008 | INFO | Analyzing chunk frames = [6057472,6057984) | 20260505 20:24:57.008 | INFO | MemInfo: { | 20260505 20:24:57.008 | INFO | "gpu" : " 17 GB", | 20260505 20:24:57.008 | INFO | "huge" : "", | 20260505 20:24:57.008 | INFO | "huge_shared" : " 27 GB", | 20260505 20:24:57.008 | INFO | "malloc" : " 14 GB", | 20260505 20:24:57.008 | INFO | "pinned" : " 29 GB" | 20260505 20:24:57.008 | INFO | } | 20260505 20:24:57.008 | INFO | Perf report for the last 5.13779 seconds: | 20260505 20:24:57.008 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:24:57.008 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:57.008 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:57.008 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:57.008 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 20:24:57.008 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:57.008 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:57.008 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 20:24:57.008 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:57.008 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:24:57.008 | INFO | Uploaded 36GiB to GPU in 2.09713s (17.1663GiB/s) | 20260505 20:24:57.008 | INFO | Downloaded 34GiB to GPU in 1.99719s (17.0239GiB/s) | 20260505 20:24:57.140 | INFO | Starting a round of Baz IO | 20260505 20:25:00.305 | INFO | Populating RTMetrics report | 20260505 20:25:00.305 | INFO | RTMetrics report populated | 20260505 20:25:00.305 | INFO | Opening tmp RTMetrics json file | 20260505 20:25:00.306 | INFO | Serializing RTMetrics to json | 20260505 20:25:00.307 | INFO | Serialization done, swapping tmp file now... | 20260505 20:25:00.307 | INFO | RTMetrics done | 20260505 20:25:00.393 | INFO | Finished a round of Baz IO: Wrote 2.014 GiB in 3.253 seconds | 20260505 20:25:00.393 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.67% / 0.00% / 78.33% | 20260505 20:25:00.393 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.176827% / 0% / 0.000107288% | 20260505 20:25:00.393 | INFO | Baz IO overall: | 20260505 20:25:00.393 | INFO | BytesWritten : 3530495005857 | 20260505 20:25:00.393 | INFO | NumEvents : 1669013932932 | 20260505 20:25:02.151 | INFO | Analyzing chunk frames = [6057984,6058496) | 20260505 20:25:02.151 | INFO | MemInfo: { | 20260505 20:25:02.151 | INFO | "gpu" : " 17 GB", | 20260505 20:25:02.151 | INFO | "huge" : "", | 20260505 20:25:02.151 | INFO | "huge_shared" : " 27 GB", | 20260505 20:25:02.151 | INFO | "malloc" : " 14 GB", | 20260505 20:25:02.151 | INFO | "pinned" : " 29 GB" | 20260505 20:25:02.151 | INFO | } | 20260505 20:25:02.151 | INFO | Perf report for the last 5.14309 seconds: | 20260505 20:25:02.151 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:25:02.151 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:02.151 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:02.151 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:02.151 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 20:25:02.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:02.151 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260505 20:25:02.151 | INFO | PRE_HQ : 60.0%, 1.8, 55.5ms, 0 | 20260505 20:25:02.151 | INFO | BAZWRITER : 62.8%, 3.7, 119.9ms, 0 | 20260505 20:25:02.151 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 20:25:02.151 | INFO | Uploaded 36GiB to GPU in 2.17333s (16.5644GiB/s) | 20260505 20:25:02.151 | INFO | Downloaded 38GiB to GPU in 2.25869s (16.8239GiB/s) | 20260505 20:25:07.288 | INFO | Analyzing chunk frames = [6058496,6059008) | 20260505 20:25:07.288 | INFO | MemInfo: { | 20260505 20:25:07.288 | INFO | "gpu" : " 17 GB", | 20260505 20:25:07.288 | INFO | "huge" : "", | 20260505 20:25:07.288 | INFO | "huge_shared" : " 27 GB", | 20260505 20:25:07.288 | INFO | "malloc" : " 14 GB", | 20260505 20:25:07.288 | INFO | "pinned" : " 29 GB" | 20260505 20:25:07.288 | INFO | } | 20260505 20:25:07.288 | INFO | Perf report for the last 5.13731 seconds: | 20260505 20:25:07.288 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:25:07.288 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:07.288 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:07.288 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:07.288 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 20:25:07.289 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:07.289 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:07.289 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 20:25:07.289 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:07.289 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:07.289 | INFO | Uploaded 36GiB to GPU in 2.07525s (17.3473GiB/s) | 20260505 20:25:07.289 | INFO | Downloaded 34GiB to GPU in 2.02511s (16.7893GiB/s) | 20260505 20:25:12.422 | INFO | Analyzing chunk frames = [6059008,6059520) | 20260505 20:25:12.422 | INFO | MemInfo: { | 20260505 20:25:12.422 | INFO | "gpu" : " 17 GB", | 20260505 20:25:12.422 | INFO | "huge" : "", | 20260505 20:25:12.422 | INFO | "huge_shared" : " 27 GB", | 20260505 20:25:12.422 | INFO | "malloc" : " 14 GB", | 20260505 20:25:12.422 | INFO | "pinned" : " 29 GB" | 20260505 20:25:12.422 | INFO | } | 20260505 20:25:12.422 | INFO | Perf report for the last 5.13373 seconds: | 20260505 20:25:12.422 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:25:12.422 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:12.422 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:12.422 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:12.422 | INFO | ANALYSIS : 62.1%, 3.0, 96.5ms, 0 | 20260505 20:25:12.422 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:12.422 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:12.422 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:25:12.422 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:12.422 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:12.422 | INFO | Uploaded 36GiB to GPU in 2.13731s (16.8436GiB/s) | 20260505 20:25:12.422 | INFO | Downloaded 34GiB to GPU in 2.02098s (16.8235GiB/s) | 20260505 20:25:17.566 | INFO | Analyzing chunk frames = [6059520,6060032) | 20260505 20:25:17.566 | INFO | MemInfo: { | 20260505 20:25:17.566 | INFO | "gpu" : " 17 GB", | 20260505 20:25:17.566 | INFO | "huge" : "", | 20260505 20:25:17.566 | INFO | "huge_shared" : " 27 GB", | 20260505 20:25:17.566 | INFO | "malloc" : " 14 GB", | 20260505 20:25:17.566 | INFO | "pinned" : " 29 GB" | 20260505 20:25:17.567 | INFO | } | 20260505 20:25:17.567 | INFO | Perf report for the last 5.14457 seconds: | 20260505 20:25:17.567 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:25:17.567 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:17.567 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:17.567 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:17.567 | INFO | ANALYSIS : 62.4%, 3.0, 97.0ms, 0 | 20260505 20:25:17.567 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:17.567 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:17.567 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:25:17.567 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:17.567 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:17.567 | INFO | Uploaded 36GiB to GPU in 2.21368s (16.2625GiB/s) | 20260505 20:25:17.567 | INFO | Downloaded 34GiB to GPU in 2.06517s (16.4635GiB/s) | 20260505 20:25:22.707 | INFO | Analyzing chunk frames = [6060032,6060544) | 20260505 20:25:22.707 | INFO | MemInfo: { | 20260505 20:25:22.707 | INFO | "gpu" : " 17 GB", | 20260505 20:25:22.707 | INFO | "huge" : "", | 20260505 20:25:22.707 | INFO | "huge_shared" : " 27 GB", | 20260505 20:25:22.707 | INFO | "malloc" : " 14 GB", | 20260505 20:25:22.707 | INFO | "pinned" : " 29 GB" | 20260505 20:25:22.707 | INFO | } | 20260505 20:25:22.707 | INFO | Perf report for the last 5.14067 seconds: | 20260505 20:25:22.707 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:25:22.707 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:22.707 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:22.707 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:22.707 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260505 20:25:22.707 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:22.708 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:22.708 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:25:22.708 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:22.708 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:22.708 | INFO | Uploaded 36GiB to GPU in 2.13561s (16.857GiB/s) | 20260505 20:25:22.708 | INFO | Downloaded 34GiB to GPU in 2.06146s (16.4932GiB/s) | 20260505 20:25:27.837 | INFO | Analyzing chunk frames = [6060544,6061056) | 20260505 20:25:27.837 | INFO | MemInfo: { | 20260505 20:25:27.837 | INFO | "gpu" : " 17 GB", | 20260505 20:25:27.837 | INFO | "huge" : "", | 20260505 20:25:27.837 | INFO | "huge_shared" : " 27 GB", | 20260505 20:25:27.837 | INFO | "malloc" : " 14 GB", | 20260505 20:25:27.837 | INFO | "pinned" : " 29 GB" | 20260505 20:25:27.837 | INFO | } | 20260505 20:25:27.837 | INFO | Perf report for the last 5.12999 seconds: | 20260505 20:25:27.837 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:25:27.837 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:27.837 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:27.837 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:27.838 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260505 20:25:27.838 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:27.838 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:27.838 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:25:27.838 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:27.838 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:27.838 | INFO | Uploaded 36GiB to GPU in 2.07703s (17.3324GiB/s) | 20260505 20:25:27.838 | INFO | Downloaded 34GiB to GPU in 2.03368s (16.7184GiB/s) | 20260505 20:25:32.980 | INFO | Analyzing chunk frames = [6061056,6061568) | 20260505 20:25:32.980 | INFO | MemInfo: { | 20260505 20:25:32.980 | INFO | "gpu" : " 17 GB", | 20260505 20:25:32.980 | INFO | "huge" : "", | 20260505 20:25:32.980 | INFO | "huge_shared" : " 27 GB", | 20260505 20:25:32.980 | INFO | "malloc" : " 14 GB", | 20260505 20:25:32.980 | INFO | "pinned" : " 29 GB" | 20260505 20:25:32.980 | INFO | } | 20260505 20:25:32.980 | INFO | Perf report for the last 5.14276 seconds: | 20260505 20:25:32.980 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:25:32.980 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:32.980 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:32.980 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:32.980 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 20:25:32.980 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:32.980 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:32.980 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 20:25:32.980 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:32.980 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:32.980 | INFO | Uploaded 36GiB to GPU in 2.11284s (17.0387GiB/s) | 20260505 20:25:32.981 | INFO | Downloaded 34GiB to GPU in 2.15089s (15.8074GiB/s) | 20260505 20:25:38.122 | INFO | Analyzing chunk frames = [6061568,6062080) | 20260505 20:25:38.122 | INFO | MemInfo: { | 20260505 20:25:38.122 | INFO | "gpu" : " 17 GB", | 20260505 20:25:38.122 | INFO | "huge" : "", | 20260505 20:25:38.122 | INFO | "huge_shared" : " 27 GB", | 20260505 20:25:38.123 | INFO | "malloc" : " 14 GB", | 20260505 20:25:38.123 | INFO | "pinned" : " 29 GB" | 20260505 20:25:38.123 | INFO | } | 20260505 20:25:38.123 | INFO | Perf report for the last 5.1424 seconds: | 20260505 20:25:38.123 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:25:38.123 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:38.123 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:38.123 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:38.123 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 20:25:38.123 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:38.123 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:38.123 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:25:38.123 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:38.123 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:38.123 | INFO | Uploaded 36GiB to GPU in 2.20845s (16.3011GiB/s) | 20260505 20:25:38.123 | INFO | Downloaded 34GiB to GPU in 2.05229s (16.5668GiB/s) | 20260505 20:25:38.123 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 20:25:38.123 | INFO | Total mem usage: 15.3 GiB | 20260505 20:25:38.123 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 20:25:38.123 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.14%) | 20260505 20:25:38.123 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 20:25:38.123 | INFO | High Water Breakdown: | 20260505 20:25:38.123 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 20:25:38.123 | INFO | 38.8% : 5.5 GiB ( 82.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 20:25:38.123 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 20:25:38.123 | INFO | ------------------------------------------ | 20260505 20:25:38.123 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:25:38.123 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 20:25:38.123 | INFO | Total mem usage: 26.5 GiB | 20260505 20:25:38.123 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 20:25:38.123 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 20:25:38.123 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 20:25:38.123 | INFO | High Water Breakdown: | 20260505 20:25:38.123 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:25:38.123 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:25:38.123 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:25:38.123 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:25:38.123 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:25:38.123 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:25:38.123 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 20:25:38.123 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 20:25:38.124 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:25:38.124 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 20:25:38.124 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 20:25:38.124 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 20:25:38.124 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 20:25:38.124 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 20:25:38.124 | INFO | ------------------------------------------ | 20260505 20:25:38.124 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:25:38.124 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 20:25:38.124 | INFO | Total mem usage: 25.6 GiB | 20260505 20:25:38.124 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 20:25:38.124 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 20:25:38.124 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 20:25:38.124 | INFO | High Water Breakdown: | 20260505 20:25:38.124 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 20:25:38.124 | INFO | ------------------------------------------ | 20260505 20:25:38.124 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:25:38.124 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 20:25:38.124 | INFO | Total mem usage: 15.7 GiB | 20260505 20:25:38.124 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 20:25:38.124 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 20:25:38.124 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 20:25:38.124 | INFO | High Water Breakdown: | 20260505 20:25:38.124 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 20:25:38.124 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 20:25:38.124 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 20:25:38.124 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 20:25:38.124 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 20:25:38.124 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 20:25:38.124 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 20:25:38.124 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 20:25:38.124 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 20:25:38.124 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:25:38.124 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 20:25:38.124 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:25:38.124 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 20:25:38.124 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 20:25:38.124 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 20:25:38.124 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 20:25:38.124 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:25:38.124 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 20:25:38.124 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:25:38.124 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:25:38.124 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:25:38.124 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 20:25:38.124 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 20:25:38.124 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:25:38.124 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 20:25:38.124 | INFO | ------------------------------------------ | 20260505 20:25:38.124 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:25:38.124 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 20:25:38.124 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:25:38.124 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:25:38.124 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.44e+01, 1.33e+01 / 1.87e+00 | 20260505 20:25:38.124 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.04e-01, 1.04e-02 / 5.04e-03 | 20260505 20:25:38.124 | INFO | PerfReport: 11.78%, Baseline -- 3.70e+00 / 4.56e+00, 5.63e-01 / 9.25e-02 | 20260505 20:25:38.124 | INFO | PerfReport: 3.26%, Histogram -- 1.03e+00 / 1.54e+00, 1.88e-01 / 8.04e-02 | 20260505 20:25:38.124 | INFO | PerfReport: 15.96%, DME -- 4.01e+01 / 4.42e+01, 4.12e+00 / 1.90e+00 | 20260505 20:25:38.124 | INFO | PerfReport: 57.05%, FrameLabeling -- 1.79e+01 / 1.94e+01, 1.43e+00 / 9.13e-01 | 20260505 20:25:38.124 | INFO | PerfReport: 7.27%, PulseAccumulating -- 2.28e+00 / 2.87e+00, 4.39e-01 / 1.58e-01 | 20260505 20:25:38.124 | INFO | PerfReport: 3.25%, Metrics -- 1.02e+00 / 1.36e+00, 2.80e-01 / 1.39e-01 | 20260505 20:25:38.124 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.55e-01 / 4.28e-01, 6.31e-02 / 3.89e-02 | 20260505 20:25:38.125 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 20:25:38.125 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:25:38.125 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:25:38.125 | INFO | PerfReport: Full Execution -- 9.93e+01 / -nan, 2.53e+01 / 0.00e+00 | 20260505 20:25:38.125 | INFO | PerfReport: 22.47%, Upload -- 2.23e+01 / 0.00e+00, 7.79e+00 / 0.00e+00 | 20260505 20:25:38.125 | INFO | PerfReport: 21.81%, Download -- 2.17e+01 / 0.00e+00, 7.71e+00 / 0.00e+00 | 20260505 20:25:38.125 | INFO | PerfReport: 31.64%, Compute -- 3.14e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260505 20:25:38.125 | INFO | PerfReport: 2.35%, LockUpload -- 2.33e+00 / 0.00e+00, 8.07e+00 / 0.00e+00 | 20260505 20:25:38.125 | INFO | PerfReport: 1.35%, LockDownload -- 1.34e+00 / 0.00e+00, 2.92e+00 / 0.00e+00 | 20260505 20:25:38.125 | INFO | PerfReport: 20.37%, LockCompute -- 2.02e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 20:25:38.125 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.71e-02 / 0.00e+00 | 20260505 20:25:38.256 | INFO | Starting a round of Baz IO | 20260505 20:25:41.434 | INFO | Populating RTMetrics report | 20260505 20:25:41.434 | INFO | RTMetrics report populated | 20260505 20:25:41.434 | INFO | Opening tmp RTMetrics json file | 20260505 20:25:41.438 | INFO | Serializing RTMetrics to json | 20260505 20:25:41.440 | INFO | Serialization done, swapping tmp file now... | 20260505 20:25:41.441 | INFO | RTMetrics done | 20260505 20:25:41.523 | INFO | Finished a round of Baz IO: Wrote 2.011 GiB in 3.266 seconds | 20260505 20:25:41.523 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.67% / 0.00% / 78.33% | 20260505 20:25:41.523 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.175869% / 3.97364e-06% / 7.94729e-05% | 20260505 20:25:41.523 | INFO | Baz IO overall: | 20260505 20:25:41.523 | INFO | BytesWritten : 3532654244337 | 20260505 20:25:41.523 | INFO | NumEvents : 1670013885610 | 20260505 20:25:43.266 | INFO | Analyzing chunk frames = [6062080,6062592) | 20260505 20:25:43.266 | INFO | MemInfo: { | 20260505 20:25:43.266 | INFO | "gpu" : " 17 GB", | 20260505 20:25:43.266 | INFO | "huge" : "", | 20260505 20:25:43.266 | INFO | "huge_shared" : " 27 GB", | 20260505 20:25:43.266 | INFO | "malloc" : " 14 GB", | 20260505 20:25:43.266 | INFO | "pinned" : " 29 GB" | 20260505 20:25:43.266 | INFO | } | 20260505 20:25:43.266 | INFO | Perf report for the last 5.14318 seconds: | 20260505 20:25:43.266 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:25:43.266 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:43.266 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:43.266 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:43.266 | INFO | ANALYSIS : 63.3%, 3.0, 98.9ms, 0 | 20260505 20:25:43.266 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:43.266 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260505 20:25:43.266 | INFO | PRE_HQ : 59.8%, 1.8, 55.4ms, 0 | 20260505 20:25:43.266 | INFO | BAZWRITER : 63.1%, 3.7, 119.1ms, 0 | 20260505 20:25:43.266 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 20:25:43.266 | INFO | Uploaded 36GiB to GPU in 2.16479s (16.6298GiB/s) | 20260505 20:25:43.266 | INFO | Downloaded 38GiB to GPU in 2.24552s (16.9226GiB/s) | 20260505 20:25:48.403 | INFO | Analyzing chunk frames = [6062592,6063104) | 20260505 20:25:48.403 | INFO | MemInfo: { | 20260505 20:25:48.403 | INFO | "gpu" : " 17 GB", | 20260505 20:25:48.403 | INFO | "huge" : "", | 20260505 20:25:48.403 | INFO | "huge_shared" : " 27 GB", | 20260505 20:25:48.403 | INFO | "malloc" : " 14 GB", | 20260505 20:25:48.403 | INFO | "pinned" : " 29 GB" | 20260505 20:25:48.403 | INFO | } | 20260505 20:25:48.404 | INFO | Perf report for the last 5.1377 seconds: | 20260505 20:25:48.404 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:25:48.404 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:48.404 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:48.404 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:48.404 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 20:25:48.404 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:48.404 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:48.404 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:25:48.404 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:48.404 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:48.404 | INFO | Uploaded 36GiB to GPU in 2.08799s (17.2415GiB/s) | 20260505 20:25:48.404 | INFO | Downloaded 34GiB to GPU in 2.02655s (16.7773GiB/s) | 20260505 20:25:53.535 | INFO | Analyzing chunk frames = [6063104,6063616) | 20260505 20:25:53.535 | INFO | MemInfo: { | 20260505 20:25:53.535 | INFO | "gpu" : " 17 GB", | 20260505 20:25:53.535 | INFO | "huge" : "", | 20260505 20:25:53.535 | INFO | "huge_shared" : " 27 GB", | 20260505 20:25:53.535 | INFO | "malloc" : " 14 GB", | 20260505 20:25:53.535 | INFO | "pinned" : " 29 GB" | 20260505 20:25:53.535 | INFO | } | 20260505 20:25:53.536 | INFO | Perf report for the last 5.13216 seconds: | 20260505 20:25:53.536 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:25:53.536 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:53.536 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:53.536 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:53.536 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260505 20:25:53.536 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:53.536 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:53.536 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:25:53.536 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:53.536 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:53.536 | INFO | Uploaded 36GiB to GPU in 2.13842s (16.8349GiB/s) | 20260505 20:25:53.536 | INFO | Downloaded 34GiB to GPU in 2.0208s (16.825GiB/s) | 20260505 20:25:58.676 | INFO | Analyzing chunk frames = [6063616,6064128) | 20260505 20:25:58.676 | INFO | MemInfo: { | 20260505 20:25:58.676 | INFO | "gpu" : " 17 GB", | 20260505 20:25:58.676 | INFO | "huge" : "", | 20260505 20:25:58.676 | INFO | "huge_shared" : " 27 GB", | 20260505 20:25:58.676 | INFO | "malloc" : " 14 GB", | 20260505 20:25:58.676 | INFO | "pinned" : " 29 GB" | 20260505 20:25:58.676 | INFO | } | 20260505 20:25:58.676 | INFO | Perf report for the last 5.14064 seconds: | 20260505 20:25:58.676 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:25:58.676 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:58.676 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:58.676 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:58.676 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260505 20:25:58.676 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:58.676 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:58.676 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 20:25:58.676 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:58.676 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:25:58.677 | INFO | Uploaded 36GiB to GPU in 2.19731s (16.3837GiB/s) | 20260505 20:25:58.677 | INFO | Downloaded 34GiB to GPU in 2.04976s (16.5873GiB/s) | 20260505 20:26:03.819 | INFO | Analyzing chunk frames = [6064128,6064640) | 20260505 20:26:03.819 | INFO | MemInfo: { | 20260505 20:26:03.819 | INFO | "gpu" : " 17 GB", | 20260505 20:26:03.819 | INFO | "huge" : "", | 20260505 20:26:03.819 | INFO | "huge_shared" : " 27 GB", | 20260505 20:26:03.819 | INFO | "malloc" : " 14 GB", | 20260505 20:26:03.819 | INFO | "pinned" : " 29 GB" | 20260505 20:26:03.819 | INFO | } | 20260505 20:26:03.819 | INFO | Perf report for the last 5.14263 seconds: | 20260505 20:26:03.819 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:26:03.819 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:03.819 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:03.819 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:03.819 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260505 20:26:03.819 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:03.819 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:03.819 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:26:03.819 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:03.819 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:03.819 | INFO | Uploaded 36GiB to GPU in 2.13685s (16.8473GiB/s) | 20260505 20:26:03.819 | INFO | Downloaded 34GiB to GPU in 2.06261s (16.484GiB/s) | 20260505 20:26:08.961 | INFO | Analyzing chunk frames = [6064640,6065152) | 20260505 20:26:08.961 | INFO | MemInfo: { | 20260505 20:26:08.961 | INFO | "gpu" : " 17 GB", | 20260505 20:26:08.961 | INFO | "huge" : "", | 20260505 20:26:08.961 | INFO | "huge_shared" : " 27 GB", | 20260505 20:26:08.962 | INFO | "malloc" : " 14 GB", | 20260505 20:26:08.962 | INFO | "pinned" : " 29 GB" | 20260505 20:26:08.962 | INFO | } | 20260505 20:26:08.962 | INFO | Perf report for the last 5.14274 seconds: | 20260505 20:26:08.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:26:08.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:08.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:08.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:08.962 | INFO | ANALYSIS : 62.1%, 3.0, 97.1ms, 0 | 20260505 20:26:08.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:08.962 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:08.962 | INFO | PRE_HQ : 27.0%, 1.0, 14.1ms, 0 | 20260505 20:26:08.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:08.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:08.962 | INFO | Uploaded 36GiB to GPU in 2.1557s (16.6999GiB/s) | 20260505 20:26:08.962 | INFO | Downloaded 34GiB to GPU in 2.06241s (16.4856GiB/s) | 20260505 20:26:14.098 | INFO | Analyzing chunk frames = [6065152,6065664) | 20260505 20:26:14.098 | INFO | MemInfo: { | 20260505 20:26:14.098 | INFO | "gpu" : " 17 GB", | 20260505 20:26:14.098 | INFO | "huge" : "", | 20260505 20:26:14.098 | INFO | "huge_shared" : " 27 GB", | 20260505 20:26:14.098 | INFO | "malloc" : " 14 GB", | 20260505 20:26:14.098 | INFO | "pinned" : " 29 GB" | 20260505 20:26:14.098 | INFO | } | 20260505 20:26:14.098 | INFO | Perf report for the last 5.13669 seconds: | 20260505 20:26:14.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:26:14.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:14.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:14.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:14.099 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 20:26:14.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:14.099 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:14.099 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 20:26:14.099 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:14.099 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:14.099 | INFO | Uploaded 36GiB to GPU in 2.12773s (16.9194GiB/s) | 20260505 20:26:14.099 | INFO | Downloaded 34GiB to GPU in 2.14493s (15.8514GiB/s) | 20260505 20:26:19.239 | INFO | Analyzing chunk frames = [6065664,6066176) | 20260505 20:26:19.239 | INFO | MemInfo: { | 20260505 20:26:19.239 | INFO | "gpu" : " 17 GB", | 20260505 20:26:19.239 | INFO | "huge" : "", | 20260505 20:26:19.239 | INFO | "huge_shared" : " 27 GB", | 20260505 20:26:19.239 | INFO | "malloc" : " 14 GB", | 20260505 20:26:19.239 | INFO | "pinned" : " 29 GB" | 20260505 20:26:19.239 | INFO | } | 20260505 20:26:19.239 | INFO | Perf report for the last 5.14098 seconds: | 20260505 20:26:19.239 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:26:19.239 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:19.239 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:19.239 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:19.240 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260505 20:26:19.240 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:19.240 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:19.240 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:26:19.240 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:19.240 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:19.240 | INFO | Uploaded 36GiB to GPU in 2.12591s (16.9339GiB/s) | 20260505 20:26:19.240 | INFO | Downloaded 34GiB to GPU in 2.01294s (16.8908GiB/s) | 20260505 20:26:19.376 | INFO | Starting a round of Baz IO | 20260505 20:26:22.533 | INFO | Populating RTMetrics report | 20260505 20:26:22.533 | INFO | RTMetrics report populated | 20260505 20:26:22.533 | INFO | Opening tmp RTMetrics json file | 20260505 20:26:22.535 | INFO | Serializing RTMetrics to json | 20260505 20:26:22.538 | INFO | Serialization done, swapping tmp file now... | 20260505 20:26:22.538 | INFO | RTMetrics done | 20260505 20:26:22.620 | INFO | Finished a round of Baz IO: Wrote 2.012 GiB in 3.244 seconds | 20260505 20:26:22.621 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.67% / 0.00% / 78.33% | 20260505 20:26:22.621 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.175699% / 0% / 9.13938e-05% | 20260505 20:26:22.621 | INFO | Baz IO overall: | 20260505 20:26:22.621 | INFO | BytesWritten : 3534814475190 | 20260505 20:26:22.621 | INFO | NumEvents : 1671015071173 | 20260505 20:26:24.380 | INFO | Analyzing chunk frames = [6066176,6066688) | 20260505 20:26:24.380 | INFO | MemInfo: { | 20260505 20:26:24.380 | INFO | "gpu" : " 17 GB", | 20260505 20:26:24.380 | INFO | "huge" : "", | 20260505 20:26:24.380 | INFO | "huge_shared" : " 27 GB", | 20260505 20:26:24.380 | INFO | "malloc" : " 14 GB", | 20260505 20:26:24.380 | INFO | "pinned" : " 29 GB" | 20260505 20:26:24.380 | INFO | } | 20260505 20:26:24.380 | INFO | Perf report for the last 5.14106 seconds: | 20260505 20:26:24.381 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:26:24.381 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:24.381 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:24.381 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:24.381 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 20:26:24.381 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:24.381 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260505 20:26:24.381 | INFO | PRE_HQ : 60.3%, 1.8, 55.1ms, 0 | 20260505 20:26:24.381 | INFO | BAZWRITER : 62.7%, 3.7, 118.0ms, 0 | 20260505 20:26:24.381 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 20:26:24.381 | INFO | Uploaded 36GiB to GPU in 2.17342s (16.5638GiB/s) | 20260505 20:26:24.381 | INFO | Downloaded 38GiB to GPU in 2.25455s (16.8548GiB/s) | 20260505 20:26:29.518 | INFO | Analyzing chunk frames = [6066688,6067200) | 20260505 20:26:29.518 | INFO | MemInfo: { | 20260505 20:26:29.518 | INFO | "gpu" : " 17 GB", | 20260505 20:26:29.518 | INFO | "huge" : "", | 20260505 20:26:29.518 | INFO | "huge_shared" : " 27 GB", | 20260505 20:26:29.518 | INFO | "malloc" : " 14 GB", | 20260505 20:26:29.518 | INFO | "pinned" : " 29 GB" | 20260505 20:26:29.518 | INFO | } | 20260505 20:26:29.518 | INFO | Perf report for the last 5.13738 seconds: | 20260505 20:26:29.518 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:26:29.518 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:29.518 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:29.518 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:29.518 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260505 20:26:29.518 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:29.518 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:29.518 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 20:26:29.518 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:29.518 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:29.518 | INFO | Uploaded 36GiB to GPU in 2.08764s (17.2443GiB/s) | 20260505 20:26:29.518 | INFO | Downloaded 34GiB to GPU in 2.02718s (16.772GiB/s) | 20260505 20:26:34.652 | INFO | Analyzing chunk frames = [6067200,6067712) | 20260505 20:26:34.652 | INFO | MemInfo: { | 20260505 20:26:34.652 | INFO | "gpu" : " 17 GB", | 20260505 20:26:34.652 | INFO | "huge" : "", | 20260505 20:26:34.652 | INFO | "huge_shared" : " 27 GB", | 20260505 20:26:34.652 | INFO | "malloc" : " 14 GB", | 20260505 20:26:34.652 | INFO | "pinned" : " 29 GB" | 20260505 20:26:34.652 | INFO | } | 20260505 20:26:34.652 | INFO | Perf report for the last 5.13445 seconds: | 20260505 20:26:34.652 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:26:34.652 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:34.652 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:34.652 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:34.652 | INFO | ANALYSIS : 62.1%, 3.0, 96.5ms, 0 | 20260505 20:26:34.652 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:34.652 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:34.652 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:26:34.652 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:34.652 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:34.653 | INFO | Uploaded 36GiB to GPU in 2.12444s (16.9456GiB/s) | 20260505 20:26:34.653 | INFO | Downloaded 34GiB to GPU in 2.0161s (16.8643GiB/s) | 20260505 20:26:39.788 | INFO | Analyzing chunk frames = [6067712,6068224) | 20260505 20:26:39.788 | INFO | MemInfo: { | 20260505 20:26:39.788 | INFO | "gpu" : " 17 GB", | 20260505 20:26:39.788 | INFO | "huge" : "", | 20260505 20:26:39.788 | INFO | "huge_shared" : " 27 GB", | 20260505 20:26:39.788 | INFO | "malloc" : " 14 GB", | 20260505 20:26:39.788 | INFO | "pinned" : " 29 GB" | 20260505 20:26:39.788 | INFO | } | 20260505 20:26:39.788 | INFO | Perf report for the last 5.13586 seconds: | 20260505 20:26:39.788 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:26:39.788 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:39.788 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:39.788 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:39.788 | INFO | ANALYSIS : 62.4%, 3.0, 96.8ms, 0 | 20260505 20:26:39.788 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:39.788 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:39.788 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:26:39.788 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:39.788 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:39.788 | INFO | Uploaded 36GiB to GPU in 2.20202s (16.3486GiB/s) | 20260505 20:26:39.789 | INFO | Downloaded 34GiB to GPU in 2.04694s (16.6101GiB/s) | 20260505 20:26:44.929 | INFO | Analyzing chunk frames = [6068224,6068736) | 20260505 20:26:44.929 | INFO | MemInfo: { | 20260505 20:26:44.929 | INFO | "gpu" : " 17 GB", | 20260505 20:26:44.930 | INFO | "huge" : "", | 20260505 20:26:44.930 | INFO | "huge_shared" : " 27 GB", | 20260505 20:26:44.930 | INFO | "malloc" : " 14 GB", | 20260505 20:26:44.930 | INFO | "pinned" : " 29 GB" | 20260505 20:26:44.930 | INFO | } | 20260505 20:26:44.930 | INFO | Perf report for the last 5.14151 seconds: | 20260505 20:26:44.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:26:44.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:44.930 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:44.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:44.930 | INFO | ANALYSIS : 61.9%, 3.0, 95.8ms, 0 | 20260505 20:26:44.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:44.930 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:44.930 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260505 20:26:44.930 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:44.930 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:44.930 | INFO | Uploaded 36GiB to GPU in 2.14399s (16.7911GiB/s) | 20260505 20:26:44.930 | INFO | Downloaded 34GiB to GPU in 2.06575s (16.4589GiB/s) | 20260505 20:26:50.074 | INFO | Analyzing chunk frames = [6068736,6069248) | 20260505 20:26:50.074 | INFO | MemInfo: { | 20260505 20:26:50.074 | INFO | "gpu" : " 17 GB", | 20260505 20:26:50.074 | INFO | "huge" : "", | 20260505 20:26:50.074 | INFO | "huge_shared" : " 27 GB", | 20260505 20:26:50.074 | INFO | "malloc" : " 14 GB", | 20260505 20:26:50.074 | INFO | "pinned" : " 29 GB" | 20260505 20:26:50.074 | INFO | } | 20260505 20:26:50.074 | INFO | Perf report for the last 5.14462 seconds: | 20260505 20:26:50.074 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:26:50.074 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:50.074 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:50.074 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:50.074 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260505 20:26:50.074 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:50.074 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:50.074 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:26:50.075 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:50.075 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:50.075 | INFO | Uploaded 36GiB to GPU in 2.15273s (16.7229GiB/s) | 20260505 20:26:50.075 | INFO | Downloaded 34GiB to GPU in 2.06032s (16.5023GiB/s) | 20260505 20:26:55.221 | INFO | Analyzing chunk frames = [6069248,6069760) | 20260505 20:26:55.221 | INFO | MemInfo: { | 20260505 20:26:55.221 | INFO | "gpu" : " 17 GB", | 20260505 20:26:55.221 | INFO | "huge" : "", | 20260505 20:26:55.221 | INFO | "huge_shared" : " 27 GB", | 20260505 20:26:55.221 | INFO | "malloc" : " 14 GB", | 20260505 20:26:55.221 | INFO | "pinned" : " 29 GB" | 20260505 20:26:55.221 | INFO | } | 20260505 20:26:55.221 | INFO | Perf report for the last 5.14656 seconds: | 20260505 20:26:55.221 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:26:55.221 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:55.221 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:55.221 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:55.221 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 20:26:55.221 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:55.221 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:55.221 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 20:26:55.221 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:55.221 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:26:55.221 | INFO | Uploaded 36GiB to GPU in 2.14993s (16.7447GiB/s) | 20260505 20:26:55.221 | INFO | Downloaded 34GiB to GPU in 2.10832s (16.1266GiB/s) | 20260505 20:27:00.352 | INFO | Analyzing chunk frames = [6069760,6070272) | 20260505 20:27:00.352 | INFO | MemInfo: { | 20260505 20:27:00.352 | INFO | "gpu" : " 17 GB", | 20260505 20:27:00.352 | INFO | "huge" : "", | 20260505 20:27:00.352 | INFO | "huge_shared" : " 27 GB", | 20260505 20:27:00.352 | INFO | "malloc" : " 14 GB", | 20260505 20:27:00.352 | INFO | "pinned" : " 29 GB" | 20260505 20:27:00.352 | INFO | } | 20260505 20:27:00.352 | INFO | Perf report for the last 5.13081 seconds: | 20260505 20:27:00.352 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:27:00.352 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:00.352 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:00.352 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:00.352 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260505 20:27:00.352 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:00.352 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:00.352 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 20:27:00.352 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:00.352 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:00.352 | INFO | Uploaded 36GiB to GPU in 2.11201s (17.0453GiB/s) | 20260505 20:27:00.352 | INFO | Downloaded 34GiB to GPU in 2.00818s (16.9308GiB/s) | 20260505 20:27:00.352 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 20:27:00.352 | INFO | Total mem usage: 15.3 GiB | 20260505 20:27:00.352 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 20:27:00.352 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.43%) | 20260505 20:27:00.352 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 20:27:00.352 | INFO | High Water Breakdown: | 20260505 20:27:00.352 | INFO | 61.1% : 8.63 GiB ( 90.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 20:27:00.352 | INFO | 38.8% : 5.48 GiB ( 82.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 20:27:00.352 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 20:27:00.352 | INFO | ------------------------------------------ | 20260505 20:27:00.352 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:27:00.352 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 20:27:00.352 | INFO | Total mem usage: 26.5 GiB | 20260505 20:27:00.352 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 20:27:00.352 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 20:27:00.352 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 20:27:00.352 | INFO | High Water Breakdown: | 20260505 20:27:00.352 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:27:00.352 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:27:00.352 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:27:00.352 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:27:00.352 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:27:00.352 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:27:00.352 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 20:27:00.352 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 20:27:00.353 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:27:00.353 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 20:27:00.353 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 20:27:00.353 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 20:27:00.353 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 20:27:00.353 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 20:27:00.353 | INFO | ------------------------------------------ | 20260505 20:27:00.353 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:27:00.353 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 20:27:00.353 | INFO | Total mem usage: 25.6 GiB | 20260505 20:27:00.353 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 20:27:00.353 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 20:27:00.353 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 20:27:00.353 | INFO | High Water Breakdown: | 20260505 20:27:00.353 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 20:27:00.353 | INFO | ------------------------------------------ | 20260505 20:27:00.353 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:27:00.353 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 20:27:00.353 | INFO | Total mem usage: 15.7 GiB | 20260505 20:27:00.353 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 20:27:00.353 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 20:27:00.353 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 20:27:00.353 | INFO | High Water Breakdown: | 20260505 20:27:00.353 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 20:27:00.353 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 20:27:00.353 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 20:27:00.353 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 20:27:00.353 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 20:27:00.353 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 20:27:00.353 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 20:27:00.353 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 20:27:00.353 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 20:27:00.353 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:27:00.353 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 20:27:00.353 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:27:00.353 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 20:27:00.353 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 20:27:00.353 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 20:27:00.353 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 20:27:00.353 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:27:00.353 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 20:27:00.353 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:27:00.353 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:27:00.353 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:27:00.353 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 20:27:00.353 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 20:27:00.353 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:27:00.353 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 20:27:00.353 | INFO | ------------------------------------------ | 20260505 20:27:00.353 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:27:00.353 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 20:27:00.353 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:27:00.353 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:27:00.353 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.41e+01, 1.36e+01 / 1.28e+00 | 20260505 20:27:00.353 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.05e-01, 1.01e-02 / 6.74e-03 | 20260505 20:27:00.353 | INFO | PerfReport: 11.81%, Baseline -- 3.71e+00 / 4.55e+00, 5.56e-01 / 1.48e-01 | 20260505 20:27:00.353 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.52e+00, 1.86e-01 / 1.03e-01 | 20260505 20:27:00.353 | INFO | PerfReport: 15.93%, DME -- 4.01e+01 / 4.40e+01, 3.94e+00 / 1.41e+00 | 20260505 20:27:00.353 | INFO | PerfReport: 57.05%, FrameLabeling -- 1.79e+01 / 1.94e+01, 1.44e+00 / 1.16e+00 | 20260505 20:27:00.353 | INFO | PerfReport: 7.26%, PulseAccumulating -- 2.28e+00 / 2.84e+00, 4.35e-01 / 2.18e-01 | 20260505 20:27:00.353 | INFO | PerfReport: 3.25%, Metrics -- 1.02e+00 / 1.34e+00, 2.78e-01 / 1.52e-01 | 20260505 20:27:00.353 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.55e-01 / 4.14e-01, 6.19e-02 / 3.68e-02 | 20260505 20:27:00.353 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 20:27:00.353 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:27:00.353 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:27:00.353 | INFO | PerfReport: Full Execution -- 9.93e+01 / -nan, 2.51e+01 / 0.00e+00 | 20260505 20:27:00.353 | INFO | PerfReport: 22.48%, Upload -- 2.23e+01 / 0.00e+00, 7.67e+00 / 0.00e+00 | 20260505 20:27:00.353 | INFO | PerfReport: 21.78%, Download -- 2.16e+01 / 0.00e+00, 7.62e+00 / 0.00e+00 | 20260505 20:27:00.353 | INFO | PerfReport: 31.66%, Compute -- 3.14e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 20:27:00.353 | INFO | PerfReport: 2.28%, LockUpload -- 2.26e+00 / 0.00e+00, 7.97e+00 / 0.00e+00 | 20260505 20:27:00.353 | INFO | PerfReport: 1.38%, LockDownload -- 1.37e+00 / 0.00e+00, 2.90e+00 / 0.00e+00 | 20260505 20:27:00.353 | INFO | PerfReport: 20.41%, LockCompute -- 2.03e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 20:27:00.353 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 2.59e-02 / 0.00e+00 | 20260505 20:27:00.481 | INFO | Starting a round of Baz IO | 20260505 20:27:03.647 | INFO | Populating RTMetrics report | 20260505 20:27:03.647 | INFO | RTMetrics report populated | 20260505 20:27:03.647 | INFO | Opening tmp RTMetrics json file | 20260505 20:27:03.648 | INFO | Serializing RTMetrics to json | 20260505 20:27:03.649 | INFO | Serialization done, swapping tmp file now... | 20260505 20:27:03.651 | INFO | RTMetrics done | 20260505 20:27:03.737 | INFO | Finished a round of Baz IO: Wrote 2.01 GiB in 3.256 seconds | 20260505 20:27:03.737 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.67% / 0.00% / 78.33% | 20260505 20:27:03.737 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.174236% / 3.97364e-06% / 9.93411e-05% | 20260505 20:27:03.738 | INFO | Baz IO overall: | 20260505 20:27:03.738 | INFO | BytesWritten : 3536972538364 | 20260505 20:27:03.738 | INFO | NumEvents : 1672014479350 | 20260505 20:27:05.483 | INFO | Analyzing chunk frames = [6070272,6070784) | 20260505 20:27:05.483 | INFO | MemInfo: { | 20260505 20:27:05.483 | INFO | "gpu" : " 17 GB", | 20260505 20:27:05.483 | INFO | "huge" : "", | 20260505 20:27:05.483 | INFO | "huge_shared" : " 27 GB", | 20260505 20:27:05.483 | INFO | "malloc" : " 14 GB", | 20260505 20:27:05.483 | INFO | "pinned" : " 29 GB" | 20260505 20:27:05.483 | INFO | } | 20260505 20:27:05.483 | INFO | Perf report for the last 5.13104 seconds: | 20260505 20:27:05.483 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:27:05.483 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:05.483 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:05.483 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:05.483 | INFO | ANALYSIS : 63.2%, 3.0, 98.3ms, 0 | 20260505 20:27:05.483 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:05.483 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260505 20:27:05.483 | INFO | PRE_HQ : 60.1%, 1.8, 55.0ms, 0 | 20260505 20:27:05.483 | INFO | BAZWRITER : 62.9%, 3.7, 118.6ms, 0 | 20260505 20:27:05.483 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 20:27:05.483 | INFO | Uploaded 36GiB to GPU in 2.15788s (16.6831GiB/s) | 20260505 20:27:05.483 | INFO | Downloaded 38GiB to GPU in 2.24156s (16.9524GiB/s) | 20260505 20:27:10.627 | INFO | Analyzing chunk frames = [6070784,6071296) | 20260505 20:27:10.627 | INFO | MemInfo: { | 20260505 20:27:10.627 | INFO | "gpu" : " 17 GB", | 20260505 20:27:10.627 | INFO | "huge" : "", | 20260505 20:27:10.627 | INFO | "huge_shared" : " 27 GB", | 20260505 20:27:10.627 | INFO | "malloc" : " 14 GB", | 20260505 20:27:10.627 | INFO | "pinned" : " 29 GB" | 20260505 20:27:10.627 | INFO | } | 20260505 20:27:10.628 | INFO | Perf report for the last 5.14471 seconds: | 20260505 20:27:10.628 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:27:10.628 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:10.628 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:10.628 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:10.628 | INFO | ANALYSIS : 62.5%, 3.0, 97.7ms, 0 | 20260505 20:27:10.628 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:10.628 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:10.628 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 20:27:10.628 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:10.628 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:10.628 | INFO | Uploaded 36GiB to GPU in 2.13881s (16.8318GiB/s) | 20260505 20:27:10.628 | INFO | Downloaded 34GiB to GPU in 2.03373s (16.7181GiB/s) | 20260505 20:27:15.769 | INFO | Analyzing chunk frames = [6071296,6071808) | 20260505 20:27:15.769 | INFO | MemInfo: { | 20260505 20:27:15.769 | INFO | "gpu" : " 17 GB", | 20260505 20:27:15.769 | INFO | "huge" : "", | 20260505 20:27:15.769 | INFO | "huge_shared" : " 27 GB", | 20260505 20:27:15.769 | INFO | "malloc" : " 14 GB", | 20260505 20:27:15.769 | INFO | "pinned" : " 29 GB" | 20260505 20:27:15.769 | INFO | } | 20260505 20:27:15.769 | INFO | Perf report for the last 5.14128 seconds: | 20260505 20:27:15.769 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:27:15.769 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:15.769 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:15.769 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:15.769 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260505 20:27:15.769 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:15.769 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:15.769 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:27:15.769 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:15.769 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:15.769 | INFO | Uploaded 36GiB to GPU in 2.13998s (16.8226GiB/s) | 20260505 20:27:15.769 | INFO | Downloaded 34GiB to GPU in 2.03298s (16.7242GiB/s) | 20260505 20:27:20.908 | INFO | Analyzing chunk frames = [6071808,6072320) | 20260505 20:27:20.908 | INFO | MemInfo: { | 20260505 20:27:20.908 | INFO | "gpu" : " 17 GB", | 20260505 20:27:20.908 | INFO | "huge" : "", | 20260505 20:27:20.908 | INFO | "huge_shared" : " 27 GB", | 20260505 20:27:20.908 | INFO | "malloc" : " 14 GB", | 20260505 20:27:20.908 | INFO | "pinned" : " 29 GB" | 20260505 20:27:20.908 | INFO | } | 20260505 20:27:20.908 | INFO | Perf report for the last 5.13908 seconds: | 20260505 20:27:20.908 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:27:20.908 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:20.908 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:20.908 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:20.908 | INFO | ANALYSIS : 62.4%, 3.0, 97.0ms, 0 | 20260505 20:27:20.908 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:20.908 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:20.908 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:27:20.908 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:20.908 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:20.908 | INFO | Uploaded 36GiB to GPU in 2.2072s (16.3103GiB/s) | 20260505 20:27:20.908 | INFO | Downloaded 34GiB to GPU in 2.03845s (16.6794GiB/s) | 20260505 20:27:26.049 | INFO | Analyzing chunk frames = [6072320,6072832) | 20260505 20:27:26.049 | INFO | MemInfo: { | 20260505 20:27:26.049 | INFO | "gpu" : " 17 GB", | 20260505 20:27:26.049 | INFO | "huge" : "", | 20260505 20:27:26.049 | INFO | "huge_shared" : " 27 GB", | 20260505 20:27:26.049 | INFO | "malloc" : " 14 GB", | 20260505 20:27:26.049 | INFO | "pinned" : " 29 GB" | 20260505 20:27:26.049 | INFO | } | 20260505 20:27:26.049 | INFO | Perf report for the last 5.14123 seconds: | 20260505 20:27:26.049 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:27:26.049 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:26.049 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:26.049 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:26.049 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260505 20:27:26.049 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:26.049 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:26.049 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 20:27:26.049 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:26.049 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:26.049 | INFO | Uploaded 36GiB to GPU in 2.12851s (16.9132GiB/s) | 20260505 20:27:26.050 | INFO | Downloaded 34GiB to GPU in 2.06029s (16.5025GiB/s) | 20260505 20:27:31.190 | INFO | Analyzing chunk frames = [6072832,6073344) | 20260505 20:27:31.190 | INFO | MemInfo: { | 20260505 20:27:31.190 | INFO | "gpu" : " 17 GB", | 20260505 20:27:31.190 | INFO | "huge" : "", | 20260505 20:27:31.190 | INFO | "huge_shared" : " 27 GB", | 20260505 20:27:31.190 | INFO | "malloc" : " 14 GB", | 20260505 20:27:31.191 | INFO | "pinned" : " 29 GB" | 20260505 20:27:31.191 | INFO | } | 20260505 20:27:31.191 | INFO | Perf report for the last 5.14142 seconds: | 20260505 20:27:31.191 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:27:31.191 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:31.191 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:31.191 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:31.191 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 20:27:31.191 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:31.191 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:31.191 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 20:27:31.191 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:31.191 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:31.191 | INFO | Uploaded 36GiB to GPU in 2.14308s (16.7983GiB/s) | 20260505 20:27:31.191 | INFO | Downloaded 34GiB to GPU in 2.0568s (16.5306GiB/s) | 20260505 20:27:36.330 | INFO | Analyzing chunk frames = [6073344,6073856) | 20260505 20:27:36.330 | INFO | MemInfo: { | 20260505 20:27:36.330 | INFO | "gpu" : " 17 GB", | 20260505 20:27:36.330 | INFO | "huge" : "", | 20260505 20:27:36.330 | INFO | "huge_shared" : " 27 GB", | 20260505 20:27:36.331 | INFO | "malloc" : " 14 GB", | 20260505 20:27:36.331 | INFO | "pinned" : " 29 GB" | 20260505 20:27:36.331 | INFO | } | 20260505 20:27:36.331 | INFO | Perf report for the last 5.14005 seconds: | 20260505 20:27:36.331 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:27:36.331 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:36.331 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:36.331 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:36.331 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 20:27:36.331 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:36.331 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:36.331 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 20:27:36.331 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:36.331 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:36.331 | INFO | Uploaded 36GiB to GPU in 2.13725s (16.844GiB/s) | 20260505 20:27:36.331 | INFO | Downloaded 34GiB to GPU in 2.15432s (15.7822GiB/s) | 20260505 20:27:41.468 | INFO | Analyzing chunk frames = [6073856,6074368) | 20260505 20:27:41.468 | INFO | MemInfo: { | 20260505 20:27:41.468 | INFO | "gpu" : " 17 GB", | 20260505 20:27:41.468 | INFO | "huge" : "", | 20260505 20:27:41.468 | INFO | "huge_shared" : " 27 GB", | 20260505 20:27:41.469 | INFO | "malloc" : " 14 GB", | 20260505 20:27:41.469 | INFO | "pinned" : " 29 GB" | 20260505 20:27:41.469 | INFO | } | 20260505 20:27:41.469 | INFO | Perf report for the last 5.13789 seconds: | 20260505 20:27:41.469 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:27:41.469 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:41.469 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:41.469 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:41.469 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 20:27:41.469 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:41.469 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:41.469 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:27:41.469 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:41.469 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:41.469 | INFO | Uploaded 36GiB to GPU in 2.11125s (17.0515GiB/s) | 20260505 20:27:41.469 | INFO | Downloaded 34GiB to GPU in 1.98883s (17.0955GiB/s) | 20260505 20:27:41.602 | INFO | Starting a round of Baz IO | 20260505 20:27:44.763 | INFO | Populating RTMetrics report | 20260505 20:27:44.764 | INFO | RTMetrics report populated | 20260505 20:27:44.764 | INFO | Opening tmp RTMetrics json file | 20260505 20:27:44.765 | INFO | Serializing RTMetrics to json | 20260505 20:27:44.766 | INFO | Serialization done, swapping tmp file now... | 20260505 20:27:44.767 | INFO | RTMetrics done | 20260505 20:27:44.852 | INFO | Finished a round of Baz IO: Wrote 2.007 GiB in 3.25 seconds | 20260505 20:27:44.852 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.67% / 0.00% / 78.33% | 20260505 20:27:44.852 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.173477% / 3.97364e-06% / 9.93411e-05% | 20260505 20:27:44.852 | INFO | Baz IO overall: | 20260505 20:27:44.852 | INFO | BytesWritten : 3539127499944 | 20260505 20:27:44.852 | INFO | NumEvents : 1673011465777 | 20260505 20:27:46.603 | INFO | Analyzing chunk frames = [6074368,6074880) | 20260505 20:27:46.603 | INFO | MemInfo: { | 20260505 20:27:46.603 | INFO | "gpu" : " 17 GB", | 20260505 20:27:46.603 | INFO | "huge" : "", | 20260505 20:27:46.603 | INFO | "huge_shared" : " 27 GB", | 20260505 20:27:46.603 | INFO | "malloc" : " 14 GB", | 20260505 20:27:46.603 | INFO | "pinned" : " 29 GB" | 20260505 20:27:46.603 | INFO | } | 20260505 20:27:46.603 | INFO | Perf report for the last 5.13451 seconds: | 20260505 20:27:46.603 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:27:46.603 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:46.603 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:46.603 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:46.603 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 20:27:46.603 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:46.603 | INFO | RT_METRICS : 44.0%, 1.0, 23.0ms, 0 | 20260505 20:27:46.603 | INFO | PRE_HQ : 59.9%, 1.8, 55.0ms, 0 | 20260505 20:27:46.603 | INFO | BAZWRITER : 62.8%, 3.7, 118.1ms, 0 | 20260505 20:27:46.603 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 20:27:46.603 | INFO | Uploaded 36GiB to GPU in 2.16207s (16.6507GiB/s) | 20260505 20:27:46.603 | INFO | Downloaded 38GiB to GPU in 2.24023s (16.9626GiB/s) | 20260505 20:27:51.736 | INFO | Analyzing chunk frames = [6074880,6075392) | 20260505 20:27:51.736 | INFO | MemInfo: { | 20260505 20:27:51.736 | INFO | "gpu" : " 17 GB", | 20260505 20:27:51.736 | INFO | "huge" : "", | 20260505 20:27:51.736 | INFO | "huge_shared" : " 27 GB", | 20260505 20:27:51.736 | INFO | "malloc" : " 14 GB", | 20260505 20:27:51.737 | INFO | "pinned" : " 29 GB" | 20260505 20:27:51.737 | INFO | } | 20260505 20:27:51.737 | INFO | Perf report for the last 5.13355 seconds: | 20260505 20:27:51.737 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:27:51.737 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:51.737 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:51.737 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:51.737 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260505 20:27:51.737 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:51.737 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:51.737 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 20:27:51.737 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:51.737 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:51.737 | INFO | Uploaded 36GiB to GPU in 2.1454s (16.7801GiB/s) | 20260505 20:27:51.737 | INFO | Downloaded 34GiB to GPU in 2.04744s (16.6061GiB/s) | 20260505 20:27:56.888 | INFO | Analyzing chunk frames = [6075392,6075904) | 20260505 20:27:56.888 | INFO | MemInfo: { | 20260505 20:27:56.888 | INFO | "gpu" : " 17 GB", | 20260505 20:27:56.888 | INFO | "huge" : "", | 20260505 20:27:56.888 | INFO | "huge_shared" : " 27 GB", | 20260505 20:27:56.888 | INFO | "malloc" : " 14 GB", | 20260505 20:27:56.888 | INFO | "pinned" : " 29 GB" | 20260505 20:27:56.888 | INFO | } | 20260505 20:27:56.888 | INFO | Perf report for the last 5.15155 seconds: | 20260505 20:27:56.888 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:27:56.888 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:56.888 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:56.888 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:56.888 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 20:27:56.888 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:56.888 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:56.888 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:27:56.888 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:56.888 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:27:56.889 | INFO | Uploaded 36GiB to GPU in 2.13501s (16.8618GiB/s) | 20260505 20:27:56.889 | INFO | Downloaded 34GiB to GPU in 2.04128s (16.6562GiB/s) | 20260505 20:28:02.032 | INFO | Analyzing chunk frames = [6075904,6076416) | 20260505 20:28:02.032 | INFO | MemInfo: { | 20260505 20:28:02.032 | INFO | "gpu" : " 17 GB", | 20260505 20:28:02.032 | INFO | "huge" : "", | 20260505 20:28:02.032 | INFO | "huge_shared" : " 27 GB", | 20260505 20:28:02.032 | INFO | "malloc" : " 14 GB", | 20260505 20:28:02.032 | INFO | "pinned" : " 29 GB" | 20260505 20:28:02.032 | INFO | } | 20260505 20:28:02.032 | INFO | Perf report for the last 5.14412 seconds: | 20260505 20:28:02.032 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:28:02.032 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:02.032 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:02.032 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:02.032 | INFO | ANALYSIS : 62.4%, 3.0, 97.0ms, 0 | 20260505 20:28:02.032 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:02.032 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:02.032 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:28:02.032 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:02.033 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:02.033 | INFO | Uploaded 36GiB to GPU in 2.21409s (16.2595GiB/s) | 20260505 20:28:02.033 | INFO | Downloaded 34GiB to GPU in 2.06529s (16.4626GiB/s) | 20260505 20:28:07.166 | INFO | Analyzing chunk frames = [6076416,6076928) | 20260505 20:28:07.166 | INFO | MemInfo: { | 20260505 20:28:07.166 | INFO | "gpu" : " 17 GB", | 20260505 20:28:07.166 | INFO | "huge" : "", | 20260505 20:28:07.166 | INFO | "huge_shared" : " 27 GB", | 20260505 20:28:07.166 | INFO | "malloc" : " 14 GB", | 20260505 20:28:07.166 | INFO | "pinned" : " 29 GB" | 20260505 20:28:07.166 | INFO | } | 20260505 20:28:07.166 | INFO | Perf report for the last 5.1335 seconds: | 20260505 20:28:07.166 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:28:07.166 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:07.166 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:07.166 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:07.166 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260505 20:28:07.166 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:07.166 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:07.166 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:28:07.166 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:07.166 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:07.166 | INFO | Uploaded 36GiB to GPU in 2.14184s (16.808GiB/s) | 20260505 20:28:07.166 | INFO | Downloaded 34GiB to GPU in 2.06255s (16.4845GiB/s) | 20260505 20:28:12.299 | INFO | Analyzing chunk frames = [6076928,6077440) | 20260505 20:28:12.299 | INFO | MemInfo: { | 20260505 20:28:12.299 | INFO | "gpu" : " 17 GB", | 20260505 20:28:12.299 | INFO | "huge" : "", | 20260505 20:28:12.299 | INFO | "huge_shared" : " 27 GB", | 20260505 20:28:12.300 | INFO | "malloc" : " 14 GB", | 20260505 20:28:12.300 | INFO | "pinned" : " 29 GB" | 20260505 20:28:12.300 | INFO | } | 20260505 20:28:12.300 | INFO | Perf report for the last 5.13377 seconds: | 20260505 20:28:12.300 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:28:12.300 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:12.300 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:12.300 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:12.300 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 20:28:12.300 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:12.300 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:12.300 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:28:12.300 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:12.300 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:12.300 | INFO | Uploaded 36GiB to GPU in 2.16099s (16.659GiB/s) | 20260505 20:28:12.300 | INFO | Downloaded 34GiB to GPU in 2.0649s (16.4657GiB/s) | 20260505 20:28:17.437 | INFO | Analyzing chunk frames = [6077440,6077952) | 20260505 20:28:17.437 | INFO | MemInfo: { | 20260505 20:28:17.437 | INFO | "gpu" : " 17 GB", | 20260505 20:28:17.437 | INFO | "huge" : "", | 20260505 20:28:17.437 | INFO | "huge_shared" : " 27 GB", | 20260505 20:28:17.437 | INFO | "malloc" : " 14 GB", | 20260505 20:28:17.437 | INFO | "pinned" : " 29 GB" | 20260505 20:28:17.437 | INFO | } | 20260505 20:28:17.437 | INFO | Perf report for the last 5.13747 seconds: | 20260505 20:28:17.437 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:28:17.437 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:17.437 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:17.437 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:17.437 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 20:28:17.437 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:17.437 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:17.437 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 20:28:17.437 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:17.437 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:17.438 | INFO | Uploaded 36GiB to GPU in 2.11716s (17.0039GiB/s) | 20260505 20:28:17.438 | INFO | Downloaded 34GiB to GPU in 2.1476s (15.8316GiB/s) | 20260505 20:28:22.581 | INFO | Analyzing chunk frames = [6077952,6078464) | 20260505 20:28:22.581 | INFO | MemInfo: { | 20260505 20:28:22.581 | INFO | "gpu" : " 17 GB", | 20260505 20:28:22.581 | INFO | "huge" : "", | 20260505 20:28:22.581 | INFO | "huge_shared" : " 27 GB", | 20260505 20:28:22.581 | INFO | "malloc" : " 14 GB", | 20260505 20:28:22.581 | INFO | "pinned" : " 29 GB" | 20260505 20:28:22.581 | INFO | } | 20260505 20:28:22.581 | INFO | Perf report for the last 5.14378 seconds: | 20260505 20:28:22.581 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:28:22.581 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:22.581 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:22.581 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:22.581 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 20:28:22.581 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:22.581 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:22.581 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 20:28:22.581 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:22.581 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:22.581 | INFO | Uploaded 36GiB to GPU in 2.20646s (16.3157GiB/s) | 20260505 20:28:22.581 | INFO | Downloaded 34GiB to GPU in 2.04668s (16.6123GiB/s) | 20260505 20:28:22.582 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 20:28:22.582 | INFO | Total mem usage: 15.3 GiB | 20260505 20:28:22.582 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 20:28:22.582 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.15%) | 20260505 20:28:22.582 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 20:28:22.582 | INFO | High Water Breakdown: | 20260505 20:28:22.582 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 20:28:22.582 | INFO | 38.8% : 5.5 GiB ( 82.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 20:28:22.582 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 20:28:22.582 | INFO | ------------------------------------------ | 20260505 20:28:22.582 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:28:22.582 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 20:28:22.582 | INFO | Total mem usage: 26.5 GiB | 20260505 20:28:22.582 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 20:28:22.582 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 20:28:22.582 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 20:28:22.582 | INFO | High Water Breakdown: | 20260505 20:28:22.582 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:28:22.582 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:28:22.582 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:28:22.582 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:28:22.582 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:28:22.582 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:28:22.582 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 20:28:22.582 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 20:28:22.582 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:28:22.582 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 20:28:22.582 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 20:28:22.582 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 20:28:22.582 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 20:28:22.582 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 20:28:22.582 | INFO | ------------------------------------------ | 20260505 20:28:22.582 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:28:22.582 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 20:28:22.582 | INFO | Total mem usage: 25.6 GiB | 20260505 20:28:22.582 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 20:28:22.582 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 20:28:22.582 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 20:28:22.582 | INFO | High Water Breakdown: | 20260505 20:28:22.582 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 20:28:22.582 | INFO | ------------------------------------------ | 20260505 20:28:22.582 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:28:22.582 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 20:28:22.582 | INFO | Total mem usage: 15.7 GiB | 20260505 20:28:22.582 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 20:28:22.582 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 20:28:22.582 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 20:28:22.582 | INFO | High Water Breakdown: | 20260505 20:28:22.582 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 20:28:22.583 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 20:28:22.583 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 20:28:22.583 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 20:28:22.583 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 20:28:22.583 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 20:28:22.583 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 20:28:22.583 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 20:28:22.583 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 20:28:22.583 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:28:22.583 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 20:28:22.583 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:28:22.583 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 20:28:22.583 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 20:28:22.583 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 20:28:22.583 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 20:28:22.583 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:28:22.583 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 20:28:22.583 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:28:22.583 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:28:22.583 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:28:22.583 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 20:28:22.583 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 20:28:22.583 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:28:22.583 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 20:28:22.583 | INFO | ------------------------------------------ | 20260505 20:28:22.583 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:28:22.583 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 20:28:22.583 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:28:22.583 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:28:22.583 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.43e+01, 1.34e+01 / 1.88e+00 | 20260505 20:28:22.583 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.04e-01, 1.13e-02 / 5.69e-03 | 20260505 20:28:22.583 | INFO | PerfReport: 11.78%, Baseline -- 3.70e+00 / 4.55e+00, 5.53e-01 / 1.70e-01 | 20260505 20:28:22.583 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.51e+00, 1.85e-01 / 9.02e-02 | 20260505 20:28:22.583 | INFO | PerfReport: 15.93%, DME -- 4.01e+01 / 4.40e+01, 3.94e+00 / 1.86e+00 | 20260505 20:28:22.583 | INFO | PerfReport: 57.11%, FrameLabeling -- 1.79e+01 / 1.95e+01, 1.46e+00 / 1.05e+00 | 20260505 20:28:22.583 | INFO | PerfReport: 7.25%, PulseAccumulating -- 2.28e+00 / 2.82e+00, 4.36e-01 / 2.27e-01 | 20260505 20:28:22.583 | INFO | PerfReport: 3.24%, Metrics -- 1.02e+00 / 1.36e+00, 2.80e-01 / 1.64e-01 | 20260505 20:28:22.583 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.55e-01 / 4.24e-01, 6.22e-02 / 3.46e-02 | 20260505 20:28:22.583 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 20:28:22.583 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:28:22.583 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:28:22.583 | INFO | PerfReport: Full Execution -- 9.94e+01 / -nan, 2.53e+01 / 0.00e+00 | 20260505 20:28:22.583 | INFO | PerfReport: 22.55%, Upload -- 2.24e+01 / 0.00e+00, 7.74e+00 / 0.00e+00 | 20260505 20:28:22.583 | INFO | PerfReport: 21.82%, Download -- 2.17e+01 / 0.00e+00, 7.69e+00 / 0.00e+00 | 20260505 20:28:22.583 | INFO | PerfReport: 31.60%, Compute -- 3.14e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 20:28:22.583 | INFO | PerfReport: 2.35%, LockUpload -- 2.34e+00 / 0.00e+00, 8.04e+00 / 0.00e+00 | 20260505 20:28:22.583 | INFO | PerfReport: 1.37%, LockDownload -- 1.36e+00 / 0.00e+00, 2.89e+00 / 0.00e+00 | 20260505 20:28:22.583 | INFO | PerfReport: 20.29%, LockCompute -- 2.02e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 20:28:22.583 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.99e-02 / 0.00e+00 | 20260505 20:28:22.716 | INFO | Starting a round of Baz IO | 20260505 20:28:25.880 | INFO | Populating RTMetrics report | 20260505 20:28:25.880 | INFO | RTMetrics report populated | 20260505 20:28:25.880 | INFO | Opening tmp RTMetrics json file | 20260505 20:28:25.883 | INFO | Serializing RTMetrics to json | 20260505 20:28:25.884 | INFO | Serialization done, swapping tmp file now... | 20260505 20:28:25.884 | INFO | RTMetrics done | 20260505 20:28:25.972 | INFO | Finished a round of Baz IO: Wrote 2.007 GiB in 3.255 seconds | 20260505 20:28:25.972 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.67% / 0.00% / 78.33% | 20260505 20:28:25.972 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.173207% / 0% / 9.93411e-05% | 20260505 20:28:25.972 | INFO | Baz IO overall: | 20260505 20:28:25.972 | INFO | BytesWritten : 3541282769632 | 20260505 20:28:25.972 | INFO | NumEvents : 1674008757264 | 20260505 20:28:27.723 | INFO | Analyzing chunk frames = [6078464,6078976) | 20260505 20:28:27.723 | INFO | MemInfo: { | 20260505 20:28:27.723 | INFO | "gpu" : " 17 GB", | 20260505 20:28:27.723 | INFO | "huge" : "", | 20260505 20:28:27.723 | INFO | "huge_shared" : " 27 GB", | 20260505 20:28:27.723 | INFO | "malloc" : " 14 GB", | 20260505 20:28:27.723 | INFO | "pinned" : " 29 GB" | 20260505 20:28:27.723 | INFO | } | 20260505 20:28:27.723 | INFO | Perf report for the last 5.14225 seconds: | 20260505 20:28:27.723 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:28:27.723 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:27.723 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:27.723 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:27.723 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 20:28:27.723 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:27.723 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260505 20:28:27.723 | INFO | PRE_HQ : 60.2%, 1.8, 55.5ms, 0 | 20260505 20:28:27.723 | INFO | BAZWRITER : 62.8%, 3.7, 119.0ms, 0 | 20260505 20:28:27.723 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 20:28:27.724 | INFO | Uploaded 36GiB to GPU in 2.17475s (16.5537GiB/s) | 20260505 20:28:27.724 | INFO | Downloaded 38GiB to GPU in 2.25211s (16.8731GiB/s) | 20260505 20:28:32.857 | INFO | Analyzing chunk frames = [6078976,6079488) | 20260505 20:28:32.857 | INFO | MemInfo: { | 20260505 20:28:32.857 | INFO | "gpu" : " 17 GB", | 20260505 20:28:32.857 | INFO | "huge" : "", | 20260505 20:28:32.857 | INFO | "huge_shared" : " 27 GB", | 20260505 20:28:32.857 | INFO | "malloc" : " 14 GB", | 20260505 20:28:32.858 | INFO | "pinned" : " 29 GB" | 20260505 20:28:32.858 | INFO | } | 20260505 20:28:32.858 | INFO | Perf report for the last 5.13447 seconds: | 20260505 20:28:32.858 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:28:32.858 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:32.858 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:32.858 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:32.858 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260505 20:28:32.858 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:32.858 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:32.858 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:28:32.858 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:32.858 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:32.858 | INFO | Uploaded 36GiB to GPU in 2.08595s (17.2583GiB/s) | 20260505 20:28:32.858 | INFO | Downloaded 34GiB to GPU in 2.02894s (16.7575GiB/s) | 20260505 20:28:37.994 | INFO | Analyzing chunk frames = [6079488,6080000) | 20260505 20:28:37.994 | INFO | MemInfo: { | 20260505 20:28:37.994 | INFO | "gpu" : " 17 GB", | 20260505 20:28:37.994 | INFO | "huge" : "", | 20260505 20:28:37.994 | INFO | "huge_shared" : " 27 GB", | 20260505 20:28:37.994 | INFO | "malloc" : " 14 GB", | 20260505 20:28:37.994 | INFO | "pinned" : " 29 GB" | 20260505 20:28:37.994 | INFO | } | 20260505 20:28:37.994 | INFO | Perf report for the last 5.13624 seconds: | 20260505 20:28:37.994 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:28:37.994 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:37.994 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:37.994 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:37.994 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260505 20:28:37.994 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:37.994 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:37.994 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:28:37.994 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:37.994 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:37.994 | INFO | Uploaded 36GiB to GPU in 2.13989s (16.8233GiB/s) | 20260505 20:28:37.994 | INFO | Downloaded 34GiB to GPU in 2.03878s (16.6767GiB/s) | 20260505 20:28:43.143 | INFO | Analyzing chunk frames = [6080000,6080512) | 20260505 20:28:43.143 | INFO | MemInfo: { | 20260505 20:28:43.143 | INFO | "gpu" : " 17 GB", | 20260505 20:28:43.144 | INFO | "huge" : "", | 20260505 20:28:43.144 | INFO | "huge_shared" : " 27 GB", | 20260505 20:28:43.144 | INFO | "malloc" : " 14 GB", | 20260505 20:28:43.144 | INFO | "pinned" : " 29 GB" | 20260505 20:28:43.144 | INFO | } | 20260505 20:28:43.144 | INFO | Perf report for the last 5.14976 seconds: | 20260505 20:28:43.144 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:28:43.144 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:43.144 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:43.144 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:43.144 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260505 20:28:43.144 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:43.144 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:43.144 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 20:28:43.144 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:43.144 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:43.144 | INFO | Uploaded 36GiB to GPU in 2.19104s (16.4306GiB/s) | 20260505 20:28:43.144 | INFO | Downloaded 34GiB to GPU in 2.06196s (16.4892GiB/s) | 20260505 20:28:48.283 | INFO | Analyzing chunk frames = [6080512,6081024) | 20260505 20:28:48.283 | INFO | MemInfo: { | 20260505 20:28:48.283 | INFO | "gpu" : " 17 GB", | 20260505 20:28:48.283 | INFO | "huge" : "", | 20260505 20:28:48.283 | INFO | "huge_shared" : " 27 GB", | 20260505 20:28:48.283 | INFO | "malloc" : " 14 GB", | 20260505 20:28:48.283 | INFO | "pinned" : " 29 GB" | 20260505 20:28:48.283 | INFO | } | 20260505 20:28:48.283 | INFO | Perf report for the last 5.13912 seconds: | 20260505 20:28:48.283 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:28:48.283 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:48.283 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:48.283 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:48.283 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260505 20:28:48.283 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:48.283 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:48.283 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 20:28:48.283 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:48.283 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:48.283 | INFO | Uploaded 36GiB to GPU in 2.1483s (16.7574GiB/s) | 20260505 20:28:48.283 | INFO | Downloaded 34GiB to GPU in 2.06431s (16.4704GiB/s) | 20260505 20:28:53.421 | INFO | Analyzing chunk frames = [6081024,6081536) | 20260505 20:28:53.421 | INFO | MemInfo: { | 20260505 20:28:53.421 | INFO | "gpu" : " 17 GB", | 20260505 20:28:53.421 | INFO | "huge" : "", | 20260505 20:28:53.421 | INFO | "huge_shared" : " 27 GB", | 20260505 20:28:53.421 | INFO | "malloc" : " 14 GB", | 20260505 20:28:53.421 | INFO | "pinned" : " 29 GB" | 20260505 20:28:53.421 | INFO | } | 20260505 20:28:53.421 | INFO | Perf report for the last 5.1381 seconds: | 20260505 20:28:53.421 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:28:53.421 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:53.421 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:53.421 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:53.421 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260505 20:28:53.421 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:53.421 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:53.421 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:28:53.421 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:53.421 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:53.421 | INFO | Uploaded 36GiB to GPU in 2.07544s (17.3457GiB/s) | 20260505 20:28:53.421 | INFO | Downloaded 34GiB to GPU in 2.04212s (16.6493GiB/s) | 20260505 20:28:58.551 | INFO | Analyzing chunk frames = [6081536,6082048) | 20260505 20:28:58.551 | INFO | MemInfo: { | 20260505 20:28:58.551 | INFO | "gpu" : " 17 GB", | 20260505 20:28:58.551 | INFO | "huge" : "", | 20260505 20:28:58.551 | INFO | "huge_shared" : " 27 GB", | 20260505 20:28:58.551 | INFO | "malloc" : " 14 GB", | 20260505 20:28:58.551 | INFO | "pinned" : " 29 GB" | 20260505 20:28:58.551 | INFO | } | 20260505 20:28:58.551 | INFO | Perf report for the last 5.1299 seconds: | 20260505 20:28:58.551 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:28:58.551 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:58.551 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:58.551 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:58.551 | INFO | ANALYSIS : 63.1%, 3.0, 98.3ms, 0 | 20260505 20:28:58.551 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:58.551 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:58.551 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 20:28:58.551 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:58.551 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:28:58.551 | INFO | Uploaded 36GiB to GPU in 2.11995s (16.9815GiB/s) | 20260505 20:28:58.551 | INFO | Downloaded 34GiB to GPU in 2.13688s (15.9111GiB/s) | 20260505 20:29:03.695 | INFO | Analyzing chunk frames = [6082048,6082560) | 20260505 20:29:03.695 | INFO | MemInfo: { | 20260505 20:29:03.695 | INFO | "gpu" : " 17 GB", | 20260505 20:29:03.695 | INFO | "huge" : "", | 20260505 20:29:03.695 | INFO | "huge_shared" : " 27 GB", | 20260505 20:29:03.695 | INFO | "malloc" : " 14 GB", | 20260505 20:29:03.695 | INFO | "pinned" : " 29 GB" | 20260505 20:29:03.695 | INFO | } | 20260505 20:29:03.695 | INFO | Perf report for the last 5.14407 seconds: | 20260505 20:29:03.695 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:29:03.695 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:03.695 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:03.695 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:03.695 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 20:29:03.695 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:03.695 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:03.695 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 20:29:03.695 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:03.695 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:03.695 | INFO | Uploaded 36GiB to GPU in 2.19812s (16.3776GiB/s) | 20260505 20:29:03.695 | INFO | Downloaded 34GiB to GPU in 2.03546s (16.7038GiB/s) | 20260505 20:29:03.824 | INFO | Starting a round of Baz IO | 20260505 20:29:06.997 | INFO | Populating RTMetrics report | 20260505 20:29:06.997 | INFO | RTMetrics report populated | 20260505 20:29:06.997 | INFO | Opening tmp RTMetrics json file | 20260505 20:29:06.998 | INFO | Serializing RTMetrics to json | 20260505 20:29:06.999 | INFO | Serialization done, swapping tmp file now... | 20260505 20:29:07.000 | INFO | RTMetrics done | 20260505 20:29:07.088 | INFO | Finished a round of Baz IO: Wrote 2.008 GiB in 3.264 seconds | 20260505 20:29:07.088 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.67% / 0.00% / 78.33% | 20260505 20:29:07.088 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.173167% / 0% / 9.53674e-05% | 20260505 20:29:07.088 | INFO | Baz IO overall: | 20260505 20:29:07.088 | INFO | BytesWritten : 3543438873993 | 20260505 20:29:07.088 | INFO | NumEvents : 1675006878303 | 20260505 20:29:08.836 | INFO | Analyzing chunk frames = [6082560,6083072) | 20260505 20:29:08.836 | INFO | MemInfo: { | 20260505 20:29:08.836 | INFO | "gpu" : " 17 GB", | 20260505 20:29:08.836 | INFO | "huge" : "", | 20260505 20:29:08.836 | INFO | "huge_shared" : " 27 GB", | 20260505 20:29:08.836 | INFO | "malloc" : " 14 GB", | 20260505 20:29:08.836 | INFO | "pinned" : " 29 GB" | 20260505 20:29:08.836 | INFO | } | 20260505 20:29:08.836 | INFO | Perf report for the last 5.14098 seconds: | 20260505 20:29:08.836 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:29:08.836 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:08.836 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:08.836 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:08.836 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 20:29:08.836 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:08.836 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260505 20:29:08.836 | INFO | PRE_HQ : 60.4%, 1.8, 55.5ms, 0 | 20260505 20:29:08.836 | INFO | BAZWRITER : 63.0%, 3.7, 119.6ms, 0 | 20260505 20:29:08.836 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 20:29:08.836 | INFO | Uploaded 36GiB to GPU in 2.16531s (16.6258GiB/s) | 20260505 20:29:08.836 | INFO | Downloaded 38GiB to GPU in 2.24739s (16.9085GiB/s) | 20260505 20:29:13.980 | INFO | Analyzing chunk frames = [6083072,6083584) | 20260505 20:29:13.980 | INFO | MemInfo: { | 20260505 20:29:13.980 | INFO | "gpu" : " 17 GB", | 20260505 20:29:13.980 | INFO | "huge" : "", | 20260505 20:29:13.980 | INFO | "huge_shared" : " 27 GB", | 20260505 20:29:13.980 | INFO | "malloc" : " 14 GB", | 20260505 20:29:13.980 | INFO | "pinned" : " 29 GB" | 20260505 20:29:13.980 | INFO | } | 20260505 20:29:13.980 | INFO | Perf report for the last 5.14441 seconds: | 20260505 20:29:13.980 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:29:13.980 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:13.980 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:13.980 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:13.980 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260505 20:29:13.980 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:13.980 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:13.980 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:29:13.980 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:13.981 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:13.981 | INFO | Uploaded 36GiB to GPU in 2.09064s (17.2196GiB/s) | 20260505 20:29:13.981 | INFO | Downloaded 34GiB to GPU in 2.0239s (16.7993GiB/s) | 20260505 20:29:19.109 | INFO | Analyzing chunk frames = [6083584,6084096) | 20260505 20:29:19.109 | INFO | MemInfo: { | 20260505 20:29:19.109 | INFO | "gpu" : " 17 GB", | 20260505 20:29:19.109 | INFO | "huge" : "", | 20260505 20:29:19.109 | INFO | "huge_shared" : " 27 GB", | 20260505 20:29:19.109 | INFO | "malloc" : " 14 GB", | 20260505 20:29:19.109 | INFO | "pinned" : " 29 GB" | 20260505 20:29:19.109 | INFO | } | 20260505 20:29:19.110 | INFO | Perf report for the last 5.12918 seconds: | 20260505 20:29:19.110 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:29:19.110 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:19.110 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:19.110 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:19.110 | INFO | ANALYSIS : 62.2%, 3.0, 96.5ms, 0 | 20260505 20:29:19.110 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:19.110 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:19.110 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260505 20:29:19.110 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:19.110 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:19.110 | INFO | Uploaded 36GiB to GPU in 2.13125s (16.8915GiB/s) | 20260505 20:29:19.110 | INFO | Downloaded 34GiB to GPU in 2.02416s (16.7971GiB/s) | 20260505 20:29:24.249 | INFO | Analyzing chunk frames = [6084096,6084608) | 20260505 20:29:24.249 | INFO | MemInfo: { | 20260505 20:29:24.249 | INFO | "gpu" : " 17 GB", | 20260505 20:29:24.249 | INFO | "huge" : "", | 20260505 20:29:24.249 | INFO | "huge_shared" : " 27 GB", | 20260505 20:29:24.249 | INFO | "malloc" : " 14 GB", | 20260505 20:29:24.249 | INFO | "pinned" : " 29 GB" | 20260505 20:29:24.249 | INFO | } | 20260505 20:29:24.249 | INFO | Perf report for the last 5.13952 seconds: | 20260505 20:29:24.249 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:29:24.249 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:24.249 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:24.249 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:24.249 | INFO | ANALYSIS : 62.3%, 3.0, 95.7ms, 0 | 20260505 20:29:24.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:24.250 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:24.250 | INFO | PRE_HQ : 27.3%, 1.0, 14.0ms, 0 | 20260505 20:29:24.250 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:24.250 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:24.250 | INFO | Uploaded 36GiB to GPU in 2.17895s (16.5217GiB/s) | 20260505 20:29:24.250 | INFO | Downloaded 34GiB to GPU in 2.0607s (16.4993GiB/s) | 20260505 20:29:29.395 | INFO | Analyzing chunk frames = [6084608,6085120) | 20260505 20:29:29.395 | INFO | MemInfo: { | 20260505 20:29:29.395 | INFO | "gpu" : " 17 GB", | 20260505 20:29:29.395 | INFO | "huge" : "", | 20260505 20:29:29.395 | INFO | "huge_shared" : " 27 GB", | 20260505 20:29:29.395 | INFO | "malloc" : " 14 GB", | 20260505 20:29:29.395 | INFO | "pinned" : " 29 GB" | 20260505 20:29:29.395 | INFO | } | 20260505 20:29:29.396 | INFO | Perf report for the last 5.1464 seconds: | 20260505 20:29:29.396 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:29:29.396 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:29.396 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:29.396 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:29.396 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260505 20:29:29.396 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:29.396 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:29.396 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:29:29.396 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:29.396 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:29.396 | INFO | Uploaded 36GiB to GPU in 2.13646s (16.8503GiB/s) | 20260505 20:29:29.396 | INFO | Downloaded 34GiB to GPU in 2.06388s (16.4738GiB/s) | 20260505 20:29:34.533 | INFO | Analyzing chunk frames = [6085120,6085632) | 20260505 20:29:34.533 | INFO | MemInfo: { | 20260505 20:29:34.533 | INFO | "gpu" : " 17 GB", | 20260505 20:29:34.533 | INFO | "huge" : "", | 20260505 20:29:34.533 | INFO | "huge_shared" : " 27 GB", | 20260505 20:29:34.533 | INFO | "malloc" : " 14 GB", | 20260505 20:29:34.533 | INFO | "pinned" : " 29 GB" | 20260505 20:29:34.533 | INFO | } | 20260505 20:29:34.533 | INFO | Perf report for the last 5.13744 seconds: | 20260505 20:29:34.533 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:29:34.533 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:34.533 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:34.533 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:34.533 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260505 20:29:34.533 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:34.533 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:34.533 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 20:29:34.533 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:34.533 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:34.533 | INFO | Uploaded 36GiB to GPU in 2.09012s (17.2239GiB/s) | 20260505 20:29:34.533 | INFO | Downloaded 34GiB to GPU in 2.03655s (16.6949GiB/s) | 20260505 20:29:39.674 | INFO | Analyzing chunk frames = [6085632,6086144) | 20260505 20:29:39.674 | INFO | MemInfo: { | 20260505 20:29:39.674 | INFO | "gpu" : " 17 GB", | 20260505 20:29:39.674 | INFO | "huge" : "", | 20260505 20:29:39.674 | INFO | "huge_shared" : " 27 GB", | 20260505 20:29:39.674 | INFO | "malloc" : " 14 GB", | 20260505 20:29:39.674 | INFO | "pinned" : " 29 GB" | 20260505 20:29:39.674 | INFO | } | 20260505 20:29:39.674 | INFO | Perf report for the last 5.14134 seconds: | 20260505 20:29:39.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:29:39.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:39.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:39.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:39.674 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 20:29:39.674 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:39.674 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:39.674 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 20:29:39.674 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:39.675 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:39.675 | INFO | Uploaded 36GiB to GPU in 2.14898s (16.7521GiB/s) | 20260505 20:29:39.675 | INFO | Downloaded 34GiB to GPU in 2.1308s (15.9564GiB/s) | 20260505 20:29:44.807 | INFO | Analyzing chunk frames = [6086144,6086656) | 20260505 20:29:44.807 | INFO | MemInfo: { | 20260505 20:29:44.807 | INFO | "gpu" : " 17 GB", | 20260505 20:29:44.807 | INFO | "huge" : "", | 20260505 20:29:44.808 | INFO | "huge_shared" : " 27 GB", | 20260505 20:29:44.808 | INFO | "malloc" : " 14 GB", | 20260505 20:29:44.808 | INFO | "pinned" : " 29 GB" | 20260505 20:29:44.808 | INFO | } | 20260505 20:29:44.808 | INFO | Perf report for the last 5.13335 seconds: | 20260505 20:29:44.808 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:29:44.808 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:44.808 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:44.808 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:44.808 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 20:29:44.808 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:44.808 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:44.808 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 20:29:44.808 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:44.808 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:44.808 | INFO | Uploaded 36GiB to GPU in 2.20888s (16.2979GiB/s) | 20260505 20:29:44.808 | INFO | Downloaded 34GiB to GPU in 2.05427s (16.5509GiB/s) | 20260505 20:29:44.808 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 20:29:44.808 | INFO | Total mem usage: 15.3 GiB | 20260505 20:29:44.808 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 20:29:44.808 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.14%) | 20260505 20:29:44.808 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 20:29:44.808 | INFO | High Water Breakdown: | 20260505 20:29:44.808 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 20:29:44.808 | INFO | 38.8% : 5.5 GiB ( 82.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 20:29:44.808 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 20:29:44.808 | INFO | ------------------------------------------ | 20260505 20:29:44.808 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:29:44.808 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 20:29:44.808 | INFO | Total mem usage: 26.5 GiB | 20260505 20:29:44.808 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 20:29:44.808 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 20:29:44.808 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 20:29:44.808 | INFO | High Water Breakdown: | 20260505 20:29:44.808 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:29:44.808 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:29:44.808 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:29:44.808 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:29:44.808 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:29:44.808 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:29:44.808 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 20:29:44.808 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 20:29:44.808 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:29:44.808 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 20:29:44.808 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 20:29:44.808 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 20:29:44.808 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 20:29:44.808 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 20:29:44.808 | INFO | ------------------------------------------ | 20260505 20:29:44.808 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:29:44.808 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 20:29:44.808 | INFO | Total mem usage: 25.6 GiB | 20260505 20:29:44.808 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 20:29:44.808 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 20:29:44.808 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 20:29:44.808 | INFO | High Water Breakdown: | 20260505 20:29:44.808 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 20:29:44.808 | INFO | ------------------------------------------ | 20260505 20:29:44.809 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:29:44.809 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 20:29:44.809 | INFO | Total mem usage: 15.7 GiB | 20260505 20:29:44.809 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 20:29:44.809 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 20:29:44.809 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 20:29:44.809 | INFO | High Water Breakdown: | 20260505 20:29:44.809 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 20:29:44.809 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 20:29:44.809 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 20:29:44.809 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 20:29:44.809 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 20:29:44.809 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 20:29:44.809 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 20:29:44.809 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 20:29:44.809 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 20:29:44.809 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:29:44.809 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 20:29:44.809 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:29:44.809 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 20:29:44.809 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 20:29:44.809 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 20:29:44.809 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 20:29:44.809 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:29:44.809 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 20:29:44.809 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:29:44.809 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:29:44.809 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:29:44.809 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 20:29:44.809 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 20:29:44.809 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:29:44.809 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 20:29:44.809 | INFO | ------------------------------------------ | 20260505 20:29:44.809 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:29:44.809 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 20:29:44.809 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:29:44.809 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:29:44.809 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.46e+01, 1.34e+01 / 1.91e+00 | 20260505 20:29:44.809 | INFO | PerfReport: 0.29%, AggregateStats -- 9.27e-02 / 1.03e-01, 1.08e-02 / 3.05e-03 | 20260505 20:29:44.809 | INFO | PerfReport: 11.79%, Baseline -- 3.71e+00 / 4.54e+00, 5.69e-01 / 1.56e-01 | 20260505 20:29:44.809 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.51e+00, 1.91e-01 / 1.11e-01 | 20260505 20:29:44.809 | INFO | PerfReport: 15.96%, DME -- 4.02e+01 / 4.43e+01, 4.10e+00 / 2.00e+00 | 20260505 20:29:44.809 | INFO | PerfReport: 57.09%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.49e+00 / 1.07e+00 | 20260505 20:29:44.809 | INFO | PerfReport: 7.22%, PulseAccumulating -- 2.27e+00 / 2.82e+00, 4.38e-01 / 2.37e-01 | 20260505 20:29:44.809 | INFO | PerfReport: 3.25%, Metrics -- 1.02e+00 / 1.36e+00, 2.84e-01 / 1.66e-01 | 20260505 20:29:44.809 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.55e-01 / 4.15e-01, 6.22e-02 / 3.23e-02 | 20260505 20:29:44.809 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 20:29:44.809 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:29:44.809 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:29:44.809 | INFO | PerfReport: Full Execution -- 9.93e+01 / -nan, 2.55e+01 / 0.00e+00 | 20260505 20:29:44.809 | INFO | PerfReport: 22.48%, Upload -- 2.23e+01 / 0.00e+00, 7.75e+00 / 0.00e+00 | 20260505 20:29:44.809 | INFO | PerfReport: 21.84%, Download -- 2.17e+01 / 0.00e+00, 7.74e+00 / 0.00e+00 | 20260505 20:29:44.809 | INFO | PerfReport: 31.72%, Compute -- 3.15e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 20:29:44.809 | INFO | PerfReport: 2.21%, LockUpload -- 2.20e+00 / 0.00e+00, 7.76e+00 / 0.00e+00 | 20260505 20:29:44.809 | INFO | PerfReport: 1.34%, LockDownload -- 1.33e+00 / 0.00e+00, 2.85e+00 / 0.00e+00 | 20260505 20:29:44.809 | INFO | PerfReport: 20.39%, LockCompute -- 2.02e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260505 20:29:44.809 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 1.92e-02 / 0.00e+00 | 20260505 20:29:44.941 | INFO | Starting a round of Baz IO | 20260505 20:29:48.115 | INFO | Populating RTMetrics report | 20260505 20:29:48.115 | INFO | RTMetrics report populated | 20260505 20:29:48.115 | INFO | Opening tmp RTMetrics json file | 20260505 20:29:48.116 | INFO | Serializing RTMetrics to json | 20260505 20:29:48.117 | INFO | Serialization done, swapping tmp file now... | 20260505 20:29:48.118 | INFO | RTMetrics done | 20260505 20:29:48.206 | INFO | Finished a round of Baz IO: Wrote 2.008 GiB in 3.265 seconds | 20260505 20:29:48.206 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.66% / 0.00% / 78.33% | 20260505 20:29:48.206 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.17362% / 0% / 0.000119209% | 20260505 20:29:48.206 | INFO | Baz IO overall: | 20260505 20:29:48.206 | INFO | BytesWritten : 3545594520082 | 20260505 20:29:48.206 | INFO | NumEvents : 1676004912392 | 20260505 20:29:49.948 | INFO | Analyzing chunk frames = [6086656,6087168) | 20260505 20:29:49.948 | INFO | MemInfo: { | 20260505 20:29:49.948 | INFO | "gpu" : " 17 GB", | 20260505 20:29:49.948 | INFO | "huge" : "", | 20260505 20:29:49.948 | INFO | "huge_shared" : " 27 GB", | 20260505 20:29:49.948 | INFO | "malloc" : " 14 GB", | 20260505 20:29:49.948 | INFO | "pinned" : " 29 GB" | 20260505 20:29:49.948 | INFO | } | 20260505 20:29:49.948 | INFO | Perf report for the last 5.14081 seconds: | 20260505 20:29:49.948 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:29:49.948 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:49.949 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:49.949 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:49.949 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 20:29:49.949 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:49.949 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260505 20:29:49.949 | INFO | PRE_HQ : 60.3%, 1.8, 55.4ms, 0 | 20260505 20:29:49.949 | INFO | BAZWRITER : 63.1%, 3.7, 119.8ms, 0 | 20260505 20:29:49.949 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 20:29:49.949 | INFO | Uploaded 36GiB to GPU in 2.16445s (16.6324GiB/s) | 20260505 20:29:49.949 | INFO | Downloaded 38GiB to GPU in 2.23905s (16.9715GiB/s) | 20260505 20:29:55.091 | INFO | Analyzing chunk frames = [6087168,6087680) | 20260505 20:29:55.091 | INFO | MemInfo: { | 20260505 20:29:55.091 | INFO | "gpu" : " 17 GB", | 20260505 20:29:55.091 | INFO | "huge" : "", | 20260505 20:29:55.091 | INFO | "huge_shared" : " 27 GB", | 20260505 20:29:55.091 | INFO | "malloc" : " 14 GB", | 20260505 20:29:55.091 | INFO | "pinned" : " 29 GB" | 20260505 20:29:55.091 | INFO | } | 20260505 20:29:55.092 | INFO | Perf report for the last 5.14305 seconds: | 20260505 20:29:55.092 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:29:55.092 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:55.092 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:55.092 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:55.092 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 20:29:55.092 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:55.092 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:55.092 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:29:55.092 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:55.092 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:29:55.092 | INFO | Uploaded 36GiB to GPU in 2.09033s (17.2222GiB/s) | 20260505 20:29:55.092 | INFO | Downloaded 34GiB to GPU in 2.02491s (16.7909GiB/s) | 20260505 20:30:00.236 | INFO | Analyzing chunk frames = [6087680,6088192) | 20260505 20:30:00.236 | INFO | MemInfo: { | 20260505 20:30:00.236 | INFO | "gpu" : " 17 GB", | 20260505 20:30:00.236 | INFO | "huge" : "", | 20260505 20:30:00.236 | INFO | "huge_shared" : " 27 GB", | 20260505 20:30:00.236 | INFO | "malloc" : " 14 GB", | 20260505 20:30:00.236 | INFO | "pinned" : " 29 GB" | 20260505 20:30:00.236 | INFO | } | 20260505 20:30:00.236 | INFO | Perf report for the last 5.14432 seconds: | 20260505 20:30:00.236 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:30:00.236 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:00.236 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:00.236 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:00.236 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260505 20:30:00.236 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:00.236 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:00.236 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:30:00.236 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:00.236 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:00.236 | INFO | Uploaded 36GiB to GPU in 2.12518s (16.9397GiB/s) | 20260505 20:30:00.236 | INFO | Downloaded 34GiB to GPU in 2.01427s (16.8796GiB/s) | 20260505 20:30:05.363 | INFO | Analyzing chunk frames = [6088192,6088704) | 20260505 20:30:05.363 | INFO | MemInfo: { | 20260505 20:30:05.363 | INFO | "gpu" : " 17 GB", | 20260505 20:30:05.363 | INFO | "huge" : "", | 20260505 20:30:05.363 | INFO | "huge_shared" : " 27 GB", | 20260505 20:30:05.363 | INFO | "malloc" : " 14 GB", | 20260505 20:30:05.363 | INFO | "pinned" : " 29 GB" | 20260505 20:30:05.363 | INFO | } | 20260505 20:30:05.363 | INFO | Perf report for the last 5.12733 seconds: | 20260505 20:30:05.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:30:05.363 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:05.363 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:05.363 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:05.363 | INFO | ANALYSIS : 62.1%, 3.0, 96.3ms, 0 | 20260505 20:30:05.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:05.363 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:05.363 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:30:05.363 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:05.363 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:05.364 | INFO | Uploaded 36GiB to GPU in 2.14938s (16.749GiB/s) | 20260505 20:30:05.364 | INFO | Downloaded 34GiB to GPU in 2.02615s (16.7806GiB/s) | 20260505 20:30:10.501 | INFO | Analyzing chunk frames = [6088704,6089216) | 20260505 20:30:10.501 | INFO | MemInfo: { | 20260505 20:30:10.501 | INFO | "gpu" : " 17 GB", | 20260505 20:30:10.501 | INFO | "huge" : "", | 20260505 20:30:10.501 | INFO | "huge_shared" : " 27 GB", | 20260505 20:30:10.501 | INFO | "malloc" : " 14 GB", | 20260505 20:30:10.501 | INFO | "pinned" : " 29 GB" | 20260505 20:30:10.501 | INFO | } | 20260505 20:30:10.501 | INFO | Perf report for the last 5.13804 seconds: | 20260505 20:30:10.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:30:10.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:10.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:10.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:10.501 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260505 20:30:10.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:10.501 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:10.501 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 20:30:10.501 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:10.501 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:10.502 | INFO | Uploaded 36GiB to GPU in 2.13824s (16.8362GiB/s) | 20260505 20:30:10.502 | INFO | Downloaded 34GiB to GPU in 2.06231s (16.4863GiB/s) | 20260505 20:30:15.651 | INFO | Analyzing chunk frames = [6089216,6089728) | 20260505 20:30:15.651 | INFO | MemInfo: { | 20260505 20:30:15.651 | INFO | "gpu" : " 17 GB", | 20260505 20:30:15.651 | INFO | "huge" : "", | 20260505 20:30:15.651 | INFO | "huge_shared" : " 27 GB", | 20260505 20:30:15.651 | INFO | "malloc" : " 14 GB", | 20260505 20:30:15.651 | INFO | "pinned" : " 29 GB" | 20260505 20:30:15.651 | INFO | } | 20260505 20:30:15.651 | INFO | Perf report for the last 5.14983 seconds: | 20260505 20:30:15.651 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:30:15.651 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:15.651 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:15.651 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:15.651 | INFO | ANALYSIS : 62.1%, 3.0, 97.1ms, 0 | 20260505 20:30:15.651 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:15.651 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:15.651 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260505 20:30:15.651 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:15.651 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:15.651 | INFO | Uploaded 36GiB to GPU in 2.14102s (16.8145GiB/s) | 20260505 20:30:15.651 | INFO | Downloaded 34GiB to GPU in 2.05551s (16.5409GiB/s) | 20260505 20:30:20.795 | INFO | Analyzing chunk frames = [6089728,6090240) | 20260505 20:30:20.795 | INFO | MemInfo: { | 20260505 20:30:20.795 | INFO | "gpu" : " 17 GB", | 20260505 20:30:20.795 | INFO | "huge" : "", | 20260505 20:30:20.795 | INFO | "huge_shared" : " 27 GB", | 20260505 20:30:20.795 | INFO | "malloc" : " 14 GB", | 20260505 20:30:20.795 | INFO | "pinned" : " 29 GB" | 20260505 20:30:20.795 | INFO | } | 20260505 20:30:20.795 | INFO | Perf report for the last 5.1443 seconds: | 20260505 20:30:20.795 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:30:20.795 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:20.795 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:20.795 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:20.796 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 20:30:20.796 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:20.796 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:20.796 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 20:30:20.796 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:20.796 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:20.796 | INFO | Uploaded 36GiB to GPU in 2.16127s (16.6568GiB/s) | 20260505 20:30:20.796 | INFO | Downloaded 34GiB to GPU in 2.12896s (15.9702GiB/s) | 20260505 20:30:25.930 | INFO | Analyzing chunk frames = [6090240,6090752) | 20260505 20:30:25.930 | INFO | MemInfo: { | 20260505 20:30:25.930 | INFO | "gpu" : " 17 GB", | 20260505 20:30:25.930 | INFO | "huge" : "", | 20260505 20:30:25.930 | INFO | "huge_shared" : " 27 GB", | 20260505 20:30:25.930 | INFO | "malloc" : " 14 GB", | 20260505 20:30:25.930 | INFO | "pinned" : " 29 GB" | 20260505 20:30:25.930 | INFO | } | 20260505 20:30:25.930 | INFO | Perf report for the last 5.13437 seconds: | 20260505 20:30:25.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:30:25.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:25.930 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:25.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:25.930 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 20:30:25.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:25.930 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:25.930 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:30:25.930 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:25.930 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:25.930 | INFO | Uploaded 36GiB to GPU in 2.11243s (17.042GiB/s) | 20260505 20:30:25.930 | INFO | Downloaded 34GiB to GPU in 1.99816s (17.0156GiB/s) | 20260505 20:30:26.066 | INFO | Starting a round of Baz IO | 20260505 20:30:29.219 | INFO | Populating RTMetrics report | 20260505 20:30:29.219 | INFO | RTMetrics report populated | 20260505 20:30:29.219 | INFO | Opening tmp RTMetrics json file | 20260505 20:30:29.221 | INFO | Serializing RTMetrics to json | 20260505 20:30:29.222 | INFO | Serialization done, swapping tmp file now... | 20260505 20:30:29.222 | INFO | RTMetrics done | 20260505 20:30:29.303 | INFO | Finished a round of Baz IO: Wrote 2.007 GiB in 3.237 seconds | 20260505 20:30:29.303 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.66% / 0.00% / 78.34% | 20260505 20:30:29.303 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.17271% / 3.97364e-06% / 8.74201e-05% | 20260505 20:30:29.303 | INFO | Baz IO overall: | 20260505 20:30:29.303 | INFO | BytesWritten : 3547750021644 | 20260505 20:30:29.303 | INFO | NumEvents : 1677002500619 | 20260505 20:30:31.066 | INFO | Analyzing chunk frames = [6090752,6091264) | 20260505 20:30:31.066 | INFO | MemInfo: { | 20260505 20:30:31.066 | INFO | "gpu" : " 17 GB", | 20260505 20:30:31.066 | INFO | "huge" : "", | 20260505 20:30:31.066 | INFO | "huge_shared" : " 27 GB", | 20260505 20:30:31.066 | INFO | "malloc" : " 14 GB", | 20260505 20:30:31.066 | INFO | "pinned" : " 29 GB" | 20260505 20:30:31.066 | INFO | } | 20260505 20:30:31.067 | INFO | Perf report for the last 5.13669 seconds: | 20260505 20:30:31.067 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:30:31.067 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:31.067 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:31.067 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:31.067 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 20:30:31.067 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:31.067 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260505 20:30:31.067 | INFO | PRE_HQ : 60.0%, 1.8, 55.0ms, 0 | 20260505 20:30:31.067 | INFO | BAZWRITER : 62.6%, 3.7, 118.8ms, 0 | 20260505 20:30:31.067 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 20:30:31.067 | INFO | Uploaded 36GiB to GPU in 2.17433s (16.5568GiB/s) | 20260505 20:30:31.067 | INFO | Downloaded 38GiB to GPU in 2.2505s (16.8851GiB/s) | 20260505 20:30:36.205 | INFO | Analyzing chunk frames = [6091264,6091776) | 20260505 20:30:36.205 | INFO | MemInfo: { | 20260505 20:30:36.205 | INFO | "gpu" : " 17 GB", | 20260505 20:30:36.205 | INFO | "huge" : "", | 20260505 20:30:36.205 | INFO | "huge_shared" : " 27 GB", | 20260505 20:30:36.205 | INFO | "malloc" : " 14 GB", | 20260505 20:30:36.205 | INFO | "pinned" : " 29 GB" | 20260505 20:30:36.205 | INFO | } | 20260505 20:30:36.205 | INFO | Perf report for the last 5.1389 seconds: | 20260505 20:30:36.205 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:30:36.205 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:36.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:36.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:36.205 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 20:30:36.206 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:36.206 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:36.206 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 20:30:36.206 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:36.206 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:36.206 | INFO | Uploaded 36GiB to GPU in 2.09947s (17.1472GiB/s) | 20260505 20:30:36.206 | INFO | Downloaded 34GiB to GPU in 2.02685s (16.7748GiB/s) | 20260505 20:30:41.343 | INFO | Analyzing chunk frames = [6091776,6092288) | 20260505 20:30:41.343 | INFO | MemInfo: { | 20260505 20:30:41.343 | INFO | "gpu" : " 17 GB", | 20260505 20:30:41.343 | INFO | "huge" : "", | 20260505 20:30:41.343 | INFO | "huge_shared" : " 27 GB", | 20260505 20:30:41.343 | INFO | "malloc" : " 14 GB", | 20260505 20:30:41.343 | INFO | "pinned" : " 29 GB" | 20260505 20:30:41.343 | INFO | } | 20260505 20:30:41.343 | INFO | Perf report for the last 5.1376 seconds: | 20260505 20:30:41.343 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:30:41.343 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:41.343 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:41.343 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:41.343 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260505 20:30:41.343 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:41.343 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:41.343 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 20:30:41.343 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:41.343 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:41.343 | INFO | Uploaded 36GiB to GPU in 2.13115s (16.8923GiB/s) | 20260505 20:30:41.343 | INFO | Downloaded 34GiB to GPU in 2.01344s (16.8865GiB/s) | 20260505 20:30:46.482 | INFO | Analyzing chunk frames = [6092288,6092800) | 20260505 20:30:46.482 | INFO | MemInfo: { | 20260505 20:30:46.482 | INFO | "gpu" : " 17 GB", | 20260505 20:30:46.482 | INFO | "huge" : "", | 20260505 20:30:46.482 | INFO | "huge_shared" : " 27 GB", | 20260505 20:30:46.482 | INFO | "malloc" : " 14 GB", | 20260505 20:30:46.482 | INFO | "pinned" : " 29 GB" | 20260505 20:30:46.482 | INFO | } | 20260505 20:30:46.483 | INFO | Perf report for the last 5.13944 seconds: | 20260505 20:30:46.483 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:30:46.483 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:46.483 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:46.483 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:46.483 | INFO | ANALYSIS : 62.1%, 3.0, 96.5ms, 0 | 20260505 20:30:46.483 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:46.483 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:46.483 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:30:46.483 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:46.483 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:46.483 | INFO | Uploaded 36GiB to GPU in 2.14113s (16.8136GiB/s) | 20260505 20:30:46.483 | INFO | Downloaded 34GiB to GPU in 2.04184s (16.6517GiB/s) | 20260505 20:30:51.619 | INFO | Analyzing chunk frames = [6092800,6093312) | 20260505 20:30:51.619 | INFO | MemInfo: { | 20260505 20:30:51.619 | INFO | "gpu" : " 17 GB", | 20260505 20:30:51.619 | INFO | "huge" : "", | 20260505 20:30:51.619 | INFO | "huge_shared" : " 27 GB", | 20260505 20:30:51.619 | INFO | "malloc" : " 14 GB", | 20260505 20:30:51.619 | INFO | "pinned" : " 29 GB" | 20260505 20:30:51.619 | INFO | } | 20260505 20:30:51.619 | INFO | Perf report for the last 5.13627 seconds: | 20260505 20:30:51.619 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:30:51.619 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:51.619 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:51.619 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:51.619 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260505 20:30:51.619 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:51.619 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:51.619 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:30:51.619 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:51.619 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:51.619 | INFO | Uploaded 36GiB to GPU in 2.13145s (16.8899GiB/s) | 20260505 20:30:51.619 | INFO | Downloaded 34GiB to GPU in 2.06s (16.5048GiB/s) | 20260505 20:30:56.757 | INFO | Analyzing chunk frames = [6093312,6093824) | 20260505 20:30:56.757 | INFO | MemInfo: { | 20260505 20:30:56.757 | INFO | "gpu" : " 17 GB", | 20260505 20:30:56.757 | INFO | "huge" : "", | 20260505 20:30:56.757 | INFO | "huge_shared" : " 27 GB", | 20260505 20:30:56.757 | INFO | "malloc" : " 14 GB", | 20260505 20:30:56.757 | INFO | "pinned" : " 29 GB" | 20260505 20:30:56.757 | INFO | } | 20260505 20:30:56.757 | INFO | Perf report for the last 5.13843 seconds: | 20260505 20:30:56.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:30:56.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:56.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:56.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:56.757 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 20:30:56.757 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:56.757 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:56.757 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:30:56.757 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:56.757 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:30:56.758 | INFO | Uploaded 36GiB to GPU in 2.15039s (16.7411GiB/s) | 20260505 20:30:56.758 | INFO | Downloaded 34GiB to GPU in 2.05997s (16.5051GiB/s) | 20260505 20:31:01.901 | INFO | Analyzing chunk frames = [6093824,6094336) | 20260505 20:31:01.901 | INFO | MemInfo: { | 20260505 20:31:01.901 | INFO | "gpu" : " 17 GB", | 20260505 20:31:01.901 | INFO | "huge" : "", | 20260505 20:31:01.901 | INFO | "huge_shared" : " 27 GB", | 20260505 20:31:01.901 | INFO | "malloc" : " 14 GB", | 20260505 20:31:01.901 | INFO | "pinned" : " 29 GB" | 20260505 20:31:01.901 | INFO | } | 20260505 20:31:01.901 | INFO | Perf report for the last 5.14408 seconds: | 20260505 20:31:01.901 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:31:01.901 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:01.901 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:01.901 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:01.901 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 20:31:01.901 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:01.901 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:01.901 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 20:31:01.901 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:01.901 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:01.902 | INFO | Uploaded 36GiB to GPU in 2.12462s (16.9442GiB/s) | 20260505 20:31:01.902 | INFO | Downloaded 34GiB to GPU in 2.13875s (15.8971GiB/s) | 20260505 20:31:07.047 | INFO | Analyzing chunk frames = [6094336,6094848) | 20260505 20:31:07.047 | INFO | MemInfo: { | 20260505 20:31:07.047 | INFO | "gpu" : " 17 GB", | 20260505 20:31:07.047 | INFO | "huge" : "", | 20260505 20:31:07.047 | INFO | "huge_shared" : " 27 GB", | 20260505 20:31:07.047 | INFO | "malloc" : " 14 GB", | 20260505 20:31:07.047 | INFO | "pinned" : " 29 GB" | 20260505 20:31:07.047 | INFO | } | 20260505 20:31:07.047 | INFO | Perf report for the last 5.14611 seconds: | 20260505 20:31:07.047 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:31:07.047 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:07.047 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:07.047 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:07.048 | INFO | ANALYSIS : 62.3%, 3.0, 97.4ms, 0 | 20260505 20:31:07.048 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:07.048 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:07.048 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:31:07.048 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:07.048 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:07.048 | INFO | Uploaded 36GiB to GPU in 2.12836s (16.9144GiB/s) | 20260505 20:31:07.048 | INFO | Downloaded 34GiB to GPU in 2.01352s (16.8858GiB/s) | 20260505 20:31:07.048 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 20:31:07.048 | INFO | Total mem usage: 15.3 GiB | 20260505 20:31:07.048 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 20:31:07.048 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.14%) | 20260505 20:31:07.048 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 20:31:07.048 | INFO | High Water Breakdown: | 20260505 20:31:07.048 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 20:31:07.048 | INFO | 38.8% : 5.5 GiB ( 82.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 20:31:07.048 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 20:31:07.048 | INFO | ------------------------------------------ | 20260505 20:31:07.048 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:31:07.048 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 20:31:07.048 | INFO | Total mem usage: 26.5 GiB | 20260505 20:31:07.048 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 20:31:07.048 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 20:31:07.048 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 20:31:07.048 | INFO | High Water Breakdown: | 20260505 20:31:07.048 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:31:07.048 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:31:07.048 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:31:07.048 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:31:07.048 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:31:07.048 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:31:07.048 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 20:31:07.048 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 20:31:07.048 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:31:07.048 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 20:31:07.048 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 20:31:07.048 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 20:31:07.048 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 20:31:07.048 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 20:31:07.048 | INFO | ------------------------------------------ | 20260505 20:31:07.048 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:31:07.048 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 20:31:07.048 | INFO | Total mem usage: 25.6 GiB | 20260505 20:31:07.048 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 20:31:07.048 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 20:31:07.048 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 20:31:07.048 | INFO | High Water Breakdown: | 20260505 20:31:07.048 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 20:31:07.048 | INFO | ------------------------------------------ | 20260505 20:31:07.048 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:31:07.048 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 20:31:07.048 | INFO | Total mem usage: 15.7 GiB | 20260505 20:31:07.048 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 20:31:07.048 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 20:31:07.048 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 20:31:07.048 | INFO | High Water Breakdown: | 20260505 20:31:07.048 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 20:31:07.048 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 20:31:07.048 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 20:31:07.048 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 20:31:07.048 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 20:31:07.048 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 20:31:07.048 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 20:31:07.048 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 20:31:07.048 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 20:31:07.048 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:31:07.048 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 20:31:07.048 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:31:07.048 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 20:31:07.048 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 20:31:07.048 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 20:31:07.048 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 20:31:07.048 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:31:07.048 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 20:31:07.049 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:31:07.049 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:31:07.049 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:31:07.049 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 20:31:07.049 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 20:31:07.049 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:31:07.049 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 20:31:07.049 | INFO | ------------------------------------------ | 20260505 20:31:07.049 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:31:07.049 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 20:31:07.049 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:31:07.049 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:31:07.049 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.47e+01, 1.36e+01 / 1.91e+00 | 20260505 20:31:07.049 | INFO | PerfReport: 0.29%, AggregateStats -- 9.25e-02 / 1.03e-01, 1.02e-02 / 5.39e-03 | 20260505 20:31:07.049 | INFO | PerfReport: 11.81%, Baseline -- 3.72e+00 / 4.51e+00, 5.78e-01 / 1.82e-01 | 20260505 20:31:07.049 | INFO | PerfReport: 3.29%, Histogram -- 1.04e+00 / 1.51e+00, 1.94e-01 / 9.90e-02 | 20260505 20:31:07.049 | INFO | PerfReport: 15.96%, DME -- 4.02e+01 / 4.44e+01, 4.02e+00 / 1.97e+00 | 20260505 20:31:07.049 | INFO | PerfReport: 57.12%, FrameLabeling -- 1.80e+01 / 1.96e+01, 1.56e+00 / 1.35e+00 | 20260505 20:31:07.049 | INFO | PerfReport: 7.18%, PulseAccumulating -- 2.26e+00 / 2.82e+00, 4.33e-01 / 2.20e-01 | 20260505 20:31:07.049 | INFO | PerfReport: 3.23%, Metrics -- 1.02e+00 / 1.33e+00, 2.81e-01 / 1.68e-01 | 20260505 20:31:07.049 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.54e-01 / 4.10e-01, 6.25e-02 / 3.64e-02 | 20260505 20:31:07.049 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 20:31:07.049 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:31:07.049 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:31:07.049 | INFO | PerfReport: Full Execution -- 9.92e+01 / -nan, 2.54e+01 / 0.00e+00 | 20260505 20:31:07.049 | INFO | PerfReport: 22.41%, Upload -- 2.22e+01 / 0.00e+00, 7.70e+00 / 0.00e+00 | 20260505 20:31:07.049 | INFO | PerfReport: 21.75%, Download -- 2.16e+01 / 0.00e+00, 7.66e+00 / 0.00e+00 | 20260505 20:31:07.049 | INFO | PerfReport: 31.77%, Compute -- 3.15e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 20:31:07.049 | INFO | PerfReport: 2.23%, LockUpload -- 2.21e+00 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260505 20:31:07.049 | INFO | PerfReport: 1.34%, LockDownload -- 1.33e+00 / 0.00e+00, 2.89e+00 / 0.00e+00 | 20260505 20:31:07.049 | INFO | PerfReport: 20.49%, LockCompute -- 2.03e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 20:31:07.049 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.83e-02 / 0.00e+00 | 20260505 20:31:07.178 | INFO | Starting a round of Baz IO | 20260505 20:31:10.336 | INFO | Populating RTMetrics report | 20260505 20:31:10.336 | INFO | RTMetrics report populated | 20260505 20:31:10.336 | INFO | Opening tmp RTMetrics json file | 20260505 20:31:10.338 | INFO | Serializing RTMetrics to json | 20260505 20:31:10.339 | INFO | Serialization done, swapping tmp file now... | 20260505 20:31:10.339 | INFO | RTMetrics done | 20260505 20:31:10.423 | INFO | Finished a round of Baz IO: Wrote 2.005 GiB in 3.245 seconds | 20260505 20:31:10.423 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.66% / 0.00% / 78.34% | 20260505 20:31:10.423 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.172937% / 0% / 7.15256e-05% | 20260505 20:31:10.423 | INFO | Baz IO overall: | 20260505 20:31:10.423 | INFO | BytesWritten : 3549903291838 | 20260505 20:31:10.423 | INFO | NumEvents : 1677998837153 | 20260505 20:31:12.176 | INFO | Analyzing chunk frames = [6094848,6095360) | 20260505 20:31:12.176 | INFO | MemInfo: { | 20260505 20:31:12.176 | INFO | "gpu" : " 17 GB", | 20260505 20:31:12.176 | INFO | "huge" : "", | 20260505 20:31:12.176 | INFO | "huge_shared" : " 27 GB", | 20260505 20:31:12.176 | INFO | "malloc" : " 14 GB", | 20260505 20:31:12.176 | INFO | "pinned" : " 29 GB" | 20260505 20:31:12.176 | INFO | } | 20260505 20:31:12.176 | INFO | Perf report for the last 5.12883 seconds: | 20260505 20:31:12.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:31:12.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:12.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:12.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:12.176 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 20:31:12.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:12.176 | INFO | RT_METRICS : 44.0%, 1.0, 23.0ms, 0 | 20260505 20:31:12.176 | INFO | PRE_HQ : 60.2%, 1.8, 54.9ms, 0 | 20260505 20:31:12.176 | INFO | BAZWRITER : 62.8%, 3.7, 118.5ms, 0 | 20260505 20:31:12.177 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 20:31:12.177 | INFO | Uploaded 36GiB to GPU in 2.16601s (16.6204GiB/s) | 20260505 20:31:12.177 | INFO | Downloaded 38GiB to GPU in 2.2455s (16.9228GiB/s) | 20260505 20:31:17.311 | INFO | Analyzing chunk frames = [6095360,6095872) | 20260505 20:31:17.311 | INFO | MemInfo: { | 20260505 20:31:17.311 | INFO | "gpu" : " 17 GB", | 20260505 20:31:17.311 | INFO | "huge" : "", | 20260505 20:31:17.311 | INFO | "huge_shared" : " 27 GB", | 20260505 20:31:17.311 | INFO | "malloc" : " 14 GB", | 20260505 20:31:17.311 | INFO | "pinned" : " 29 GB" | 20260505 20:31:17.312 | INFO | } | 20260505 20:31:17.312 | INFO | Perf report for the last 5.13528 seconds: | 20260505 20:31:17.312 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:31:17.312 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:17.312 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:17.312 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:17.312 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 20:31:17.312 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:17.312 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:17.312 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:31:17.312 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:17.312 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:17.312 | INFO | Uploaded 36GiB to GPU in 2.14059s (16.8178GiB/s) | 20260505 20:31:17.312 | INFO | Downloaded 34GiB to GPU in 2.04486s (16.627GiB/s) | 20260505 20:31:22.454 | INFO | Analyzing chunk frames = [6095872,6096384) | 20260505 20:31:22.454 | INFO | MemInfo: { | 20260505 20:31:22.454 | INFO | "gpu" : " 17 GB", | 20260505 20:31:22.454 | INFO | "huge" : "", | 20260505 20:31:22.454 | INFO | "huge_shared" : " 27 GB", | 20260505 20:31:22.454 | INFO | "malloc" : " 14 GB", | 20260505 20:31:22.455 | INFO | "pinned" : " 29 GB" | 20260505 20:31:22.455 | INFO | } | 20260505 20:31:22.455 | INFO | Perf report for the last 5.14303 seconds: | 20260505 20:31:22.455 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:31:22.455 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:22.455 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:22.455 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:22.455 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260505 20:31:22.455 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:22.455 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:22.455 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:31:22.455 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:22.455 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:22.455 | INFO | Uploaded 36GiB to GPU in 2.14105s (16.8142GiB/s) | 20260505 20:31:22.455 | INFO | Downloaded 34GiB to GPU in 2.05079s (16.579GiB/s) | 20260505 20:31:27.602 | INFO | Analyzing chunk frames = [6096384,6096896) | 20260505 20:31:27.602 | INFO | MemInfo: { | 20260505 20:31:27.602 | INFO | "gpu" : " 17 GB", | 20260505 20:31:27.602 | INFO | "huge" : "", | 20260505 20:31:27.602 | INFO | "huge_shared" : " 27 GB", | 20260505 20:31:27.602 | INFO | "malloc" : " 14 GB", | 20260505 20:31:27.602 | INFO | "pinned" : " 29 GB" | 20260505 20:31:27.602 | INFO | } | 20260505 20:31:27.602 | INFO | Perf report for the last 5.1472 seconds: | 20260505 20:31:27.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:31:27.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:27.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:27.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:27.602 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260505 20:31:27.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:27.602 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:27.602 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:31:27.602 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:27.602 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:27.602 | INFO | Uploaded 36GiB to GPU in 2.20831s (16.3021GiB/s) | 20260505 20:31:27.602 | INFO | Downloaded 34GiB to GPU in 2.04094s (16.659GiB/s) | 20260505 20:31:32.740 | INFO | Analyzing chunk frames = [6096896,6097408) | 20260505 20:31:32.740 | INFO | MemInfo: { | 20260505 20:31:32.740 | INFO | "gpu" : " 17 GB", | 20260505 20:31:32.740 | INFO | "huge" : "", | 20260505 20:31:32.741 | INFO | "huge_shared" : " 27 GB", | 20260505 20:31:32.741 | INFO | "malloc" : " 14 GB", | 20260505 20:31:32.741 | INFO | "pinned" : " 29 GB" | 20260505 20:31:32.741 | INFO | } | 20260505 20:31:32.741 | INFO | Perf report for the last 5.13882 seconds: | 20260505 20:31:32.741 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:31:32.741 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:32.741 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:32.741 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:32.741 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260505 20:31:32.741 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:32.741 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:32.741 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:31:32.741 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:32.741 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:32.741 | INFO | Uploaded 36GiB to GPU in 2.15034s (16.7416GiB/s) | 20260505 20:31:32.741 | INFO | Downloaded 34GiB to GPU in 2.06808s (16.4404GiB/s) | 20260505 20:31:37.874 | INFO | Analyzing chunk frames = [6097408,6097920) | 20260505 20:31:37.874 | INFO | MemInfo: { | 20260505 20:31:37.874 | INFO | "gpu" : " 17 GB", | 20260505 20:31:37.874 | INFO | "huge" : "", | 20260505 20:31:37.874 | INFO | "huge_shared" : " 27 GB", | 20260505 20:31:37.874 | INFO | "malloc" : " 14 GB", | 20260505 20:31:37.874 | INFO | "pinned" : " 29 GB" | 20260505 20:31:37.874 | INFO | } | 20260505 20:31:37.874 | INFO | Perf report for the last 5.13363 seconds: | 20260505 20:31:37.874 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:31:37.874 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:37.874 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:37.874 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:37.874 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 20:31:37.874 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:37.874 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:37.874 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 20:31:37.874 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:37.874 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:37.875 | INFO | Uploaded 36GiB to GPU in 2.08182s (17.2925GiB/s) | 20260505 20:31:37.875 | INFO | Downloaded 34GiB to GPU in 2.02783s (16.7667GiB/s) | 20260505 20:31:43.017 | INFO | Analyzing chunk frames = [6097920,6098432) | 20260505 20:31:43.017 | INFO | MemInfo: { | 20260505 20:31:43.017 | INFO | "gpu" : " 17 GB", | 20260505 20:31:43.017 | INFO | "huge" : "", | 20260505 20:31:43.017 | INFO | "huge_shared" : " 27 GB", | 20260505 20:31:43.017 | INFO | "malloc" : " 14 GB", | 20260505 20:31:43.017 | INFO | "pinned" : " 29 GB" | 20260505 20:31:43.017 | INFO | } | 20260505 20:31:43.017 | INFO | Perf report for the last 5.14284 seconds: | 20260505 20:31:43.017 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:31:43.017 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:43.017 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:43.017 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:43.017 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 20:31:43.017 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:43.017 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:43.017 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 20:31:43.017 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:43.017 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:43.017 | INFO | Uploaded 36GiB to GPU in 2.12284s (16.9584GiB/s) | 20260505 20:31:43.018 | INFO | Downloaded 34GiB to GPU in 2.12133s (16.0277GiB/s) | 20260505 20:31:48.160 | INFO | Analyzing chunk frames = [6098432,6098944) | 20260505 20:31:48.160 | INFO | MemInfo: { | 20260505 20:31:48.160 | INFO | "gpu" : " 17 GB", | 20260505 20:31:48.160 | INFO | "huge" : "", | 20260505 20:31:48.160 | INFO | "huge_shared" : " 27 GB", | 20260505 20:31:48.160 | INFO | "malloc" : " 14 GB", | 20260505 20:31:48.160 | INFO | "pinned" : " 29 GB" | 20260505 20:31:48.160 | INFO | } | 20260505 20:31:48.160 | INFO | Perf report for the last 5.14296 seconds: | 20260505 20:31:48.160 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:31:48.160 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:48.160 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:48.160 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:48.160 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 20:31:48.160 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:48.160 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:48.160 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:31:48.160 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:48.160 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:48.160 | INFO | Uploaded 36GiB to GPU in 2.20608s (16.3185GiB/s) | 20260505 20:31:48.161 | INFO | Downloaded 34GiB to GPU in 2.04862s (16.5966GiB/s) | 20260505 20:31:48.294 | INFO | Starting a round of Baz IO | 20260505 20:31:51.450 | INFO | Populating RTMetrics report | 20260505 20:31:51.450 | INFO | RTMetrics report populated | 20260505 20:31:51.450 | INFO | Opening tmp RTMetrics json file | 20260505 20:31:51.451 | INFO | Serializing RTMetrics to json | 20260505 20:31:51.452 | INFO | Serialization done, swapping tmp file now... | 20260505 20:31:51.452 | INFO | RTMetrics done | 20260505 20:31:51.541 | INFO | Finished a round of Baz IO: Wrote 2.005 GiB in 3.247 seconds | 20260505 20:31:51.541 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.66% / 0.00% / 78.34% | 20260505 20:31:51.541 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.171328% / 0% / 8.74201e-05% | 20260505 20:31:51.541 | INFO | Baz IO overall: | 20260505 20:31:51.541 | INFO | BytesWritten : 3552056292766 | 20260505 20:31:51.541 | INFO | NumEvents : 1678995155932 | 20260505 20:31:53.292 | INFO | Analyzing chunk frames = [6098944,6099456) | 20260505 20:31:53.292 | INFO | MemInfo: { | 20260505 20:31:53.292 | INFO | "gpu" : " 17 GB", | 20260505 20:31:53.292 | INFO | "huge" : "", | 20260505 20:31:53.292 | INFO | "huge_shared" : " 27 GB", | 20260505 20:31:53.292 | INFO | "malloc" : " 14 GB", | 20260505 20:31:53.292 | INFO | "pinned" : " 29 GB" | 20260505 20:31:53.292 | INFO | } | 20260505 20:31:53.292 | INFO | Perf report for the last 5.13209 seconds: | 20260505 20:31:53.292 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:31:53.292 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:53.292 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:53.292 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:53.292 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 20:31:53.292 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:53.292 | INFO | RT_METRICS : 44.0%, 1.0, 23.0ms, 0 | 20260505 20:31:53.292 | INFO | PRE_HQ : 60.2%, 1.8, 54.9ms, 0 | 20260505 20:31:53.292 | INFO | BAZWRITER : 62.8%, 3.7, 118.4ms, 0 | 20260505 20:31:53.292 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 20:31:53.293 | INFO | Uploaded 36GiB to GPU in 2.18173s (16.5007GiB/s) | 20260505 20:31:53.293 | INFO | Downloaded 38GiB to GPU in 2.25225s (16.872GiB/s) | 20260505 20:31:58.431 | INFO | Analyzing chunk frames = [6099456,6099968) | 20260505 20:31:58.431 | INFO | MemInfo: { | 20260505 20:31:58.431 | INFO | "gpu" : " 17 GB", | 20260505 20:31:58.431 | INFO | "huge" : "", | 20260505 20:31:58.431 | INFO | "huge_shared" : " 27 GB", | 20260505 20:31:58.431 | INFO | "malloc" : " 14 GB", | 20260505 20:31:58.431 | INFO | "pinned" : " 29 GB" | 20260505 20:31:58.431 | INFO | } | 20260505 20:31:58.431 | INFO | Perf report for the last 5.13863 seconds: | 20260505 20:31:58.431 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:31:58.431 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:58.431 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:58.431 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:58.431 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 20:31:58.431 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:58.431 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:58.431 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 20:31:58.431 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:58.431 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:31:58.431 | INFO | Uploaded 36GiB to GPU in 2.09666s (17.1701GiB/s) | 20260505 20:31:58.431 | INFO | Downloaded 34GiB to GPU in 2.03378s (16.7176GiB/s) | 20260505 20:32:03.569 | INFO | Analyzing chunk frames = [6099968,6100480) | 20260505 20:32:03.569 | INFO | MemInfo: { | 20260505 20:32:03.569 | INFO | "gpu" : " 17 GB", | 20260505 20:32:03.569 | INFO | "huge" : "", | 20260505 20:32:03.569 | INFO | "huge_shared" : " 27 GB", | 20260505 20:32:03.569 | INFO | "malloc" : " 14 GB", | 20260505 20:32:03.569 | INFO | "pinned" : " 29 GB" | 20260505 20:32:03.569 | INFO | } | 20260505 20:32:03.569 | INFO | Perf report for the last 5.13829 seconds: | 20260505 20:32:03.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:32:03.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:03.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:03.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:03.569 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260505 20:32:03.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:03.569 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:03.569 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:32:03.569 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:03.569 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:03.569 | INFO | Uploaded 36GiB to GPU in 2.13289s (16.8785GiB/s) | 20260505 20:32:03.569 | INFO | Downloaded 34GiB to GPU in 2.02495s (16.7906GiB/s) | 20260505 20:32:08.711 | INFO | Analyzing chunk frames = [6100480,6100992) | 20260505 20:32:08.711 | INFO | MemInfo: { | 20260505 20:32:08.711 | INFO | "gpu" : " 17 GB", | 20260505 20:32:08.711 | INFO | "huge" : "", | 20260505 20:32:08.711 | INFO | "huge_shared" : " 27 GB", | 20260505 20:32:08.711 | INFO | "malloc" : " 14 GB", | 20260505 20:32:08.711 | INFO | "pinned" : " 29 GB" | 20260505 20:32:08.711 | INFO | } | 20260505 20:32:08.711 | INFO | Perf report for the last 5.14169 seconds: | 20260505 20:32:08.711 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:32:08.711 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:08.711 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:08.711 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:08.711 | INFO | ANALYSIS : 62.4%, 3.0, 96.9ms, 0 | 20260505 20:32:08.711 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:08.711 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:08.711 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:32:08.711 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:08.711 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:08.711 | INFO | Uploaded 36GiB to GPU in 2.20383s (16.3352GiB/s) | 20260505 20:32:08.711 | INFO | Downloaded 34GiB to GPU in 2.02957s (16.7523GiB/s) | 20260505 20:32:13.854 | INFO | Analyzing chunk frames = [6100992,6101504) | 20260505 20:32:13.854 | INFO | MemInfo: { | 20260505 20:32:13.854 | INFO | "gpu" : " 17 GB", | 20260505 20:32:13.854 | INFO | "huge" : "", | 20260505 20:32:13.854 | INFO | "huge_shared" : " 27 GB", | 20260505 20:32:13.854 | INFO | "malloc" : " 14 GB", | 20260505 20:32:13.854 | INFO | "pinned" : " 29 GB" | 20260505 20:32:13.854 | INFO | } | 20260505 20:32:13.854 | INFO | Perf report for the last 5.14324 seconds: | 20260505 20:32:13.854 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:32:13.854 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:13.854 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:13.854 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:13.854 | INFO | ANALYSIS : 61.6%, 3.0, 96.3ms, 0 | 20260505 20:32:13.854 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:13.854 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:13.854 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:32:13.854 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:13.854 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:13.854 | INFO | Uploaded 36GiB to GPU in 2.13098s (16.8936GiB/s) | 20260505 20:32:13.854 | INFO | Downloaded 34GiB to GPU in 2.05789s (16.5218GiB/s) | 20260505 20:32:18.991 | INFO | Analyzing chunk frames = [6101504,6102016) | 20260505 20:32:18.991 | INFO | MemInfo: { | 20260505 20:32:18.991 | INFO | "gpu" : " 17 GB", | 20260505 20:32:18.991 | INFO | "huge" : "", | 20260505 20:32:18.991 | INFO | "huge_shared" : " 27 GB", | 20260505 20:32:18.991 | INFO | "malloc" : " 14 GB", | 20260505 20:32:18.991 | INFO | "pinned" : " 29 GB" | 20260505 20:32:18.991 | INFO | } | 20260505 20:32:18.992 | INFO | Perf report for the last 5.1374 seconds: | 20260505 20:32:18.992 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:32:18.992 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:18.992 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:18.992 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:18.992 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260505 20:32:18.992 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:18.992 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:18.992 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:32:18.992 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:18.992 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:18.992 | INFO | Uploaded 36GiB to GPU in 2.09251s (17.2043GiB/s) | 20260505 20:32:18.992 | INFO | Downloaded 34GiB to GPU in 2.03699s (16.6913GiB/s) | 20260505 20:32:24.124 | INFO | Analyzing chunk frames = [6102016,6102528) | 20260505 20:32:24.124 | INFO | MemInfo: { | 20260505 20:32:24.124 | INFO | "gpu" : " 17 GB", | 20260505 20:32:24.124 | INFO | "huge" : "", | 20260505 20:32:24.124 | INFO | "huge_shared" : " 27 GB", | 20260505 20:32:24.124 | INFO | "malloc" : " 14 GB", | 20260505 20:32:24.124 | INFO | "pinned" : " 29 GB" | 20260505 20:32:24.124 | INFO | } | 20260505 20:32:24.124 | INFO | Perf report for the last 5.13268 seconds: | 20260505 20:32:24.124 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:32:24.124 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:24.124 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:24.124 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:24.124 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 20:32:24.124 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:24.124 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:24.124 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 20:32:24.124 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:24.124 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:24.125 | INFO | Uploaded 36GiB to GPU in 2.12511s (16.9403GiB/s) | 20260505 20:32:24.125 | INFO | Downloaded 34GiB to GPU in 2.14857s (15.8245GiB/s) | 20260505 20:32:29.261 | INFO | Analyzing chunk frames = [6102528,6103040) | 20260505 20:32:29.261 | INFO | MemInfo: { | 20260505 20:32:29.261 | INFO | "gpu" : " 17 GB", | 20260505 20:32:29.261 | INFO | "huge" : "", | 20260505 20:32:29.261 | INFO | "huge_shared" : " 27 GB", | 20260505 20:32:29.262 | INFO | "malloc" : " 14 GB", | 20260505 20:32:29.262 | INFO | "pinned" : " 29 GB" | 20260505 20:32:29.262 | INFO | } | 20260505 20:32:29.262 | INFO | Perf report for the last 5.13816 seconds: | 20260505 20:32:29.262 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:32:29.262 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:29.262 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:29.262 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:29.262 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 20:32:29.262 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:29.262 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:29.262 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 20:32:29.262 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:29.262 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:29.263 | INFO | Uploaded 36GiB to GPU in 2.19692s (16.3866GiB/s) | 20260505 20:32:29.263 | INFO | Downloaded 34GiB to GPU in 2.03522s (16.7058GiB/s) | 20260505 20:32:29.263 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 20:32:29.263 | INFO | Total mem usage: 15.3 GiB | 20260505 20:32:29.263 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 20:32:29.263 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.79%) | 20260505 20:32:29.263 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 20:32:29.263 | INFO | High Water Breakdown: | 20260505 20:32:29.263 | INFO | 61% : 8.59 GiB ( 90.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 20:32:29.263 | INFO | 38.9% : 5.48 GiB ( 82.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 20:32:29.263 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 20:32:29.263 | INFO | ------------------------------------------ | 20260505 20:32:29.263 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:32:29.263 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 20:32:29.263 | INFO | Total mem usage: 26.5 GiB | 20260505 20:32:29.263 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 20:32:29.263 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 20:32:29.263 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 20:32:29.263 | INFO | High Water Breakdown: | 20260505 20:32:29.263 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:32:29.263 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:32:29.263 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:32:29.263 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:32:29.263 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:32:29.263 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:32:29.263 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 20:32:29.263 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 20:32:29.263 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:32:29.263 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 20:32:29.263 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 20:32:29.263 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 20:32:29.263 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 20:32:29.263 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 20:32:29.263 | INFO | ------------------------------------------ | 20260505 20:32:29.263 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:32:29.263 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 20:32:29.263 | INFO | Total mem usage: 25.6 GiB | 20260505 20:32:29.263 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 20:32:29.263 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 20:32:29.263 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 20:32:29.263 | INFO | High Water Breakdown: | 20260505 20:32:29.263 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 20:32:29.263 | INFO | ------------------------------------------ | 20260505 20:32:29.263 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:32:29.263 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 20:32:29.263 | INFO | Total mem usage: 15.7 GiB | 20260505 20:32:29.263 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 20:32:29.263 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 20:32:29.263 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 20:32:29.263 | INFO | High Water Breakdown: | 20260505 20:32:29.263 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 20:32:29.263 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 20:32:29.263 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 20:32:29.263 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 20:32:29.264 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 20:32:29.264 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 20:32:29.264 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 20:32:29.264 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 20:32:29.264 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 20:32:29.264 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:32:29.264 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 20:32:29.264 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:32:29.264 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 20:32:29.264 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 20:32:29.264 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 20:32:29.264 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 20:32:29.264 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:32:29.264 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 20:32:29.264 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:32:29.264 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:32:29.264 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:32:29.264 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 20:32:29.264 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 20:32:29.264 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:32:29.264 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 20:32:29.264 | INFO | ------------------------------------------ | 20260505 20:32:29.264 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:32:29.264 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 20:32:29.264 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:32:29.264 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:32:29.264 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.43e+01, 1.35e+01 / 1.78e+00 | 20260505 20:32:29.264 | INFO | PerfReport: 0.29%, AggregateStats -- 9.25e-02 / 1.03e-01, 1.05e-02 / 3.91e-03 | 20260505 20:32:29.264 | INFO | PerfReport: 11.79%, Baseline -- 3.71e+00 / 4.56e+00, 5.59e-01 / 1.35e-01 | 20260505 20:32:29.264 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.51e+00, 1.86e-01 / 1.04e-01 | 20260505 20:32:29.264 | INFO | PerfReport: 15.95%, DME -- 4.01e+01 / 4.41e+01, 3.86e+00 / 1.70e+00 | 20260505 20:32:29.264 | INFO | PerfReport: 57.11%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.47e+00 / 1.20e+00 | 20260505 20:32:29.264 | INFO | PerfReport: 7.23%, PulseAccumulating -- 2.27e+00 / 2.81e+00, 4.36e-01 / 2.60e-01 | 20260505 20:32:29.264 | INFO | PerfReport: 3.23%, Metrics -- 1.02e+00 / 1.33e+00, 2.78e-01 / 1.61e-01 | 20260505 20:32:29.264 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.55e-01 / 4.16e-01, 6.27e-02 / 3.54e-02 | 20260505 20:32:29.264 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 20:32:29.264 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:32:29.264 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:32:29.264 | INFO | PerfReport: Full Execution -- 9.94e+01 / -nan, 2.54e+01 / 0.00e+00 | 20260505 20:32:29.264 | INFO | PerfReport: 22.53%, Upload -- 2.24e+01 / 0.00e+00, 7.73e+00 / 0.00e+00 | 20260505 20:32:29.264 | INFO | PerfReport: 21.80%, Download -- 2.17e+01 / 0.00e+00, 7.65e+00 / 0.00e+00 | 20260505 20:32:29.264 | INFO | PerfReport: 31.66%, Compute -- 3.15e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 20:32:29.264 | INFO | PerfReport: 2.33%, LockUpload -- 2.31e+00 / 0.00e+00, 7.98e+00 / 0.00e+00 | 20260505 20:32:29.264 | INFO | PerfReport: 1.33%, LockDownload -- 1.32e+00 / 0.00e+00, 2.84e+00 / 0.00e+00 | 20260505 20:32:29.264 | INFO | PerfReport: 20.34%, LockCompute -- 2.02e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 20:32:29.264 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.55e-02 / 0.00e+00 | 20260505 20:32:29.394 | INFO | Starting a round of Baz IO | 20260505 20:32:32.566 | INFO | Populating RTMetrics report | 20260505 20:32:32.566 | INFO | RTMetrics report populated | 20260505 20:32:32.566 | INFO | Opening tmp RTMetrics json file | 20260505 20:32:32.567 | INFO | Serializing RTMetrics to json | 20260505 20:32:32.568 | INFO | Serialization done, swapping tmp file now... | 20260505 20:32:32.568 | INFO | RTMetrics done | 20260505 20:32:32.656 | INFO | Finished a round of Baz IO: Wrote 2.005 GiB in 3.262 seconds | 20260505 20:32:32.657 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.66% / 0.00% / 78.34% | 20260505 20:32:32.657 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.171415% / 7.94729e-06% / 6.75519e-05% | 20260505 20:32:32.657 | INFO | Baz IO overall: | 20260505 20:32:32.657 | INFO | BytesWritten : 3554209579834 | 20260505 20:32:32.657 | INFO | NumEvents : 1679991594918 | 20260505 20:32:34.409 | INFO | Analyzing chunk frames = [6103040,6103552) | 20260505 20:32:34.409 | INFO | MemInfo: { | 20260505 20:32:34.409 | INFO | "gpu" : " 17 GB", | 20260505 20:32:34.409 | INFO | "huge" : "", | 20260505 20:32:34.409 | INFO | "huge_shared" : " 27 GB", | 20260505 20:32:34.409 | INFO | "malloc" : " 14 GB", | 20260505 20:32:34.409 | INFO | "pinned" : " 29 GB" | 20260505 20:32:34.409 | INFO | } | 20260505 20:32:34.410 | INFO | Perf report for the last 5.14711 seconds: | 20260505 20:32:34.410 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:32:34.410 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:34.410 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:34.410 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:34.410 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 20:32:34.410 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:34.410 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260505 20:32:34.410 | INFO | PRE_HQ : 59.8%, 1.8, 55.0ms, 0 | 20260505 20:32:34.410 | INFO | BAZWRITER : 62.9%, 3.7, 118.3ms, 0 | 20260505 20:32:34.410 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 20:32:34.410 | INFO | Uploaded 36GiB to GPU in 2.14542s (16.78GiB/s) | 20260505 20:32:34.410 | INFO | Downloaded 38GiB to GPU in 2.24479s (16.9281GiB/s) | 20260505 20:32:39.560 | INFO | Analyzing chunk frames = [6103552,6104064) | 20260505 20:32:39.560 | INFO | MemInfo: { | 20260505 20:32:39.560 | INFO | "gpu" : " 17 GB", | 20260505 20:32:39.560 | INFO | "huge" : "", | 20260505 20:32:39.560 | INFO | "huge_shared" : " 27 GB", | 20260505 20:32:39.560 | INFO | "malloc" : " 14 GB", | 20260505 20:32:39.560 | INFO | "pinned" : " 29 GB" | 20260505 20:32:39.560 | INFO | } | 20260505 20:32:39.560 | INFO | Perf report for the last 5.1506 seconds: | 20260505 20:32:39.560 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:32:39.560 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:39.560 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:39.560 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:39.560 | INFO | ANALYSIS : 61.9%, 3.0, 96.9ms, 0 | 20260505 20:32:39.560 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:39.560 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:39.560 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:32:39.560 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:39.560 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:39.560 | INFO | Uploaded 36GiB to GPU in 2.09037s (17.2218GiB/s) | 20260505 20:32:39.560 | INFO | Downloaded 34GiB to GPU in 2.02985s (16.75GiB/s) | 20260505 20:32:44.684 | INFO | Analyzing chunk frames = [6104064,6104576) | 20260505 20:32:44.684 | INFO | MemInfo: { | 20260505 20:32:44.684 | INFO | "gpu" : " 17 GB", | 20260505 20:32:44.684 | INFO | "huge" : "", | 20260505 20:32:44.684 | INFO | "huge_shared" : " 27 GB", | 20260505 20:32:44.684 | INFO | "malloc" : " 14 GB", | 20260505 20:32:44.684 | INFO | "pinned" : " 29 GB" | 20260505 20:32:44.684 | INFO | } | 20260505 20:32:44.684 | INFO | Perf report for the last 5.12422 seconds: | 20260505 20:32:44.684 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:32:44.684 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:44.684 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:44.684 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:44.684 | INFO | ANALYSIS : 62.1%, 3.0, 96.4ms, 0 | 20260505 20:32:44.684 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:44.684 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:44.684 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:32:44.685 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:44.685 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:44.685 | INFO | Uploaded 36GiB to GPU in 2.15031s (16.7418GiB/s) | 20260505 20:32:44.685 | INFO | Downloaded 34GiB to GPU in 2.05351s (16.557GiB/s) | 20260505 20:32:49.821 | INFO | Analyzing chunk frames = [6104576,6105088) | 20260505 20:32:49.821 | INFO | MemInfo: { | 20260505 20:32:49.821 | INFO | "gpu" : " 17 GB", | 20260505 20:32:49.821 | INFO | "huge" : "", | 20260505 20:32:49.821 | INFO | "huge_shared" : " 27 GB", | 20260505 20:32:49.821 | INFO | "malloc" : " 14 GB", | 20260505 20:32:49.821 | INFO | "pinned" : " 29 GB" | 20260505 20:32:49.821 | INFO | } | 20260505 20:32:49.821 | INFO | Perf report for the last 5.13702 seconds: | 20260505 20:32:49.821 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:32:49.821 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:49.821 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:49.821 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:49.821 | INFO | ANALYSIS : 62.4%, 3.0, 96.9ms, 0 | 20260505 20:32:49.821 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:49.821 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:49.821 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260505 20:32:49.821 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:49.821 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:49.822 | INFO | Uploaded 36GiB to GPU in 2.2067s (16.314GiB/s) | 20260505 20:32:49.822 | INFO | Downloaded 34GiB to GPU in 2.04377s (16.6359GiB/s) | 20260505 20:32:54.969 | INFO | Analyzing chunk frames = [6105088,6105600) | 20260505 20:32:54.969 | INFO | MemInfo: { | 20260505 20:32:54.969 | INFO | "gpu" : " 17 GB", | 20260505 20:32:54.969 | INFO | "huge" : "", | 20260505 20:32:54.969 | INFO | "huge_shared" : " 27 GB", | 20260505 20:32:54.969 | INFO | "malloc" : " 14 GB", | 20260505 20:32:54.969 | INFO | "pinned" : " 29 GB" | 20260505 20:32:54.969 | INFO | } | 20260505 20:32:54.969 | INFO | Perf report for the last 5.14751 seconds: | 20260505 20:32:54.969 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:32:54.969 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:54.969 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:54.969 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:54.969 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260505 20:32:54.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:54.969 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:54.969 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:32:54.969 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:54.969 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:32:54.969 | INFO | Uploaded 36GiB to GPU in 2.14115s (16.8134GiB/s) | 20260505 20:32:54.969 | INFO | Downloaded 34GiB to GPU in 2.06356s (16.4764GiB/s) | 20260505 20:33:00.104 | INFO | Analyzing chunk frames = [6105600,6106112) | 20260505 20:33:00.104 | INFO | MemInfo: { | 20260505 20:33:00.104 | INFO | "gpu" : " 17 GB", | 20260505 20:33:00.105 | INFO | "huge" : "", | 20260505 20:33:00.105 | INFO | "huge_shared" : " 27 GB", | 20260505 20:33:00.105 | INFO | "malloc" : " 14 GB", | 20260505 20:33:00.105 | INFO | "pinned" : " 29 GB" | 20260505 20:33:00.105 | INFO | } | 20260505 20:33:00.105 | INFO | Perf report for the last 5.1358 seconds: | 20260505 20:33:00.105 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:33:00.105 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:00.105 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:00.105 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:00.105 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 20:33:00.105 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:00.105 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:00.105 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:33:00.105 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:00.105 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:00.105 | INFO | Uploaded 36GiB to GPU in 2.15733s (16.6873GiB/s) | 20260505 20:33:00.105 | INFO | Downloaded 34GiB to GPU in 2.06407s (16.4723GiB/s) | 20260505 20:33:05.242 | INFO | Analyzing chunk frames = [6106112,6106624) | 20260505 20:33:05.242 | INFO | MemInfo: { | 20260505 20:33:05.242 | INFO | "gpu" : " 17 GB", | 20260505 20:33:05.242 | INFO | "huge" : "", | 20260505 20:33:05.242 | INFO | "huge_shared" : " 27 GB", | 20260505 20:33:05.242 | INFO | "malloc" : " 14 GB", | 20260505 20:33:05.242 | INFO | "pinned" : " 29 GB" | 20260505 20:33:05.242 | INFO | } | 20260505 20:33:05.242 | INFO | Perf report for the last 5.13763 seconds: | 20260505 20:33:05.242 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:33:05.242 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:05.242 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:05.242 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:05.242 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 20:33:05.242 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:05.242 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:05.242 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 20:33:05.243 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:05.243 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:05.243 | INFO | Uploaded 36GiB to GPU in 2.1378s (16.8397GiB/s) | 20260505 20:33:05.243 | INFO | Downloaded 34GiB to GPU in 2.15059s (15.8096GiB/s) | 20260505 20:33:10.381 | INFO | Analyzing chunk frames = [6106624,6107136) | 20260505 20:33:10.381 | INFO | MemInfo: { | 20260505 20:33:10.381 | INFO | "gpu" : " 17 GB", | 20260505 20:33:10.381 | INFO | "huge" : "", | 20260505 20:33:10.381 | INFO | "huge_shared" : " 27 GB", | 20260505 20:33:10.382 | INFO | "malloc" : " 14 GB", | 20260505 20:33:10.382 | INFO | "pinned" : " 29 GB" | 20260505 20:33:10.382 | INFO | } | 20260505 20:33:10.382 | INFO | Perf report for the last 5.13919 seconds: | 20260505 20:33:10.382 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:33:10.382 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:10.382 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:10.382 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:10.382 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 20:33:10.382 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:10.382 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:10.382 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:33:10.382 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:10.382 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:10.382 | INFO | Uploaded 36GiB to GPU in 2.20418s (16.3326GiB/s) | 20260505 20:33:10.382 | INFO | Downloaded 34GiB to GPU in 2.04667s (16.6124GiB/s) | 20260505 20:33:10.516 | INFO | Starting a round of Baz IO | 20260505 20:33:13.685 | INFO | Populating RTMetrics report | 20260505 20:33:13.685 | INFO | RTMetrics report populated | 20260505 20:33:13.685 | INFO | Opening tmp RTMetrics json file | 20260505 20:33:13.687 | INFO | Serializing RTMetrics to json | 20260505 20:33:13.689 | INFO | Serialization done, swapping tmp file now... | 20260505 20:33:13.689 | INFO | RTMetrics done | 20260505 20:33:13.768 | INFO | Finished a round of Baz IO: Wrote 2.006 GiB in 3.251 seconds | 20260505 20:33:13.768 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.66% / 0.00% / 78.34% | 20260505 20:33:13.768 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.172027% / 0% / 7.94729e-05% | 20260505 20:33:13.768 | INFO | Baz IO overall: | 20260505 20:33:13.768 | INFO | BytesWritten : 3556363475834 | 20260505 20:33:13.768 | INFO | NumEvents : 1680988945464 | 20260505 20:33:15.520 | INFO | Analyzing chunk frames = [6107136,6107648) | 20260505 20:33:15.520 | INFO | MemInfo: { | 20260505 20:33:15.520 | INFO | "gpu" : " 17 GB", | 20260505 20:33:15.520 | INFO | "huge" : "", | 20260505 20:33:15.520 | INFO | "huge_shared" : " 27 GB", | 20260505 20:33:15.520 | INFO | "malloc" : " 14 GB", | 20260505 20:33:15.520 | INFO | "pinned" : " 29 GB" | 20260505 20:33:15.520 | INFO | } | 20260505 20:33:15.521 | INFO | Perf report for the last 5.13894 seconds: | 20260505 20:33:15.521 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:33:15.521 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:15.521 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:15.521 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:15.521 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 20:33:15.521 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:15.521 | INFO | RT_METRICS : 44.0%, 1.0, 23.0ms, 0 | 20260505 20:33:15.521 | INFO | PRE_HQ : 60.1%, 1.8, 55.0ms, 0 | 20260505 20:33:15.521 | INFO | BAZWRITER : 62.8%, 3.7, 118.9ms, 0 | 20260505 20:33:15.521 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 20:33:15.521 | INFO | Uploaded 36GiB to GPU in 2.1634s (16.6405GiB/s) | 20260505 20:33:15.521 | INFO | Downloaded 38GiB to GPU in 2.24052s (16.9604GiB/s) | 20260505 20:33:20.665 | INFO | Analyzing chunk frames = [6107648,6108160) | 20260505 20:33:20.665 | INFO | MemInfo: { | 20260505 20:33:20.665 | INFO | "gpu" : " 17 GB", | 20260505 20:33:20.665 | INFO | "huge" : "", | 20260505 20:33:20.665 | INFO | "huge_shared" : " 27 GB", | 20260505 20:33:20.665 | INFO | "malloc" : " 14 GB", | 20260505 20:33:20.665 | INFO | "pinned" : " 29 GB" | 20260505 20:33:20.665 | INFO | } | 20260505 20:33:20.665 | INFO | Perf report for the last 5.14418 seconds: | 20260505 20:33:20.665 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:33:20.665 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:20.665 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:20.665 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:20.665 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260505 20:33:20.665 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:20.665 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:20.665 | INFO | PRE_HQ : 27.0%, 1.0, 14.1ms, 0 | 20260505 20:33:20.665 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:20.665 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:20.665 | INFO | Uploaded 36GiB to GPU in 2.09995s (17.1433GiB/s) | 20260505 20:33:20.665 | INFO | Downloaded 34GiB to GPU in 2.02482s (16.7917GiB/s) | 20260505 20:33:25.805 | INFO | Analyzing chunk frames = [6108160,6108672) | 20260505 20:33:25.805 | INFO | MemInfo: { | 20260505 20:33:25.805 | INFO | "gpu" : " 17 GB", | 20260505 20:33:25.805 | INFO | "huge" : "", | 20260505 20:33:25.805 | INFO | "huge_shared" : " 27 GB", | 20260505 20:33:25.805 | INFO | "malloc" : " 14 GB", | 20260505 20:33:25.805 | INFO | "pinned" : " 29 GB" | 20260505 20:33:25.805 | INFO | } | 20260505 20:33:25.805 | INFO | Perf report for the last 5.14068 seconds: | 20260505 20:33:25.805 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:33:25.805 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:25.805 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:25.805 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:25.805 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260505 20:33:25.805 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:25.805 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:25.806 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:33:25.806 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:25.806 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:25.806 | INFO | Uploaded 36GiB to GPU in 2.12634s (16.9305GiB/s) | 20260505 20:33:25.806 | INFO | Downloaded 34GiB to GPU in 2.01079s (16.9088GiB/s) | 20260505 20:33:30.938 | INFO | Analyzing chunk frames = [6108672,6109184) | 20260505 20:33:30.938 | INFO | MemInfo: { | 20260505 20:33:30.938 | INFO | "gpu" : " 17 GB", | 20260505 20:33:30.938 | INFO | "huge" : "", | 20260505 20:33:30.938 | INFO | "huge_shared" : " 27 GB", | 20260505 20:33:30.938 | INFO | "malloc" : " 14 GB", | 20260505 20:33:30.938 | INFO | "pinned" : " 29 GB" | 20260505 20:33:30.938 | INFO | } | 20260505 20:33:30.939 | INFO | Perf report for the last 5.1332 seconds: | 20260505 20:33:30.939 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:33:30.939 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:30.939 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:30.939 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:30.939 | INFO | ANALYSIS : 62.5%, 3.0, 97.0ms, 0 | 20260505 20:33:30.939 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:30.939 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:30.939 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 20:33:30.939 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:30.939 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:30.939 | INFO | Uploaded 36GiB to GPU in 2.21679s (16.2397GiB/s) | 20260505 20:33:30.939 | INFO | Downloaded 34GiB to GPU in 2.0617s (16.4912GiB/s) | 20260505 20:33:36.072 | INFO | Analyzing chunk frames = [6109184,6109696) | 20260505 20:33:36.072 | INFO | MemInfo: { | 20260505 20:33:36.072 | INFO | "gpu" : " 17 GB", | 20260505 20:33:36.072 | INFO | "huge" : "", | 20260505 20:33:36.072 | INFO | "huge_shared" : " 27 GB", | 20260505 20:33:36.072 | INFO | "malloc" : " 14 GB", | 20260505 20:33:36.072 | INFO | "pinned" : " 29 GB" | 20260505 20:33:36.072 | INFO | } | 20260505 20:33:36.073 | INFO | Perf report for the last 5.13389 seconds: | 20260505 20:33:36.073 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:33:36.073 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:36.073 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:36.073 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:36.073 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260505 20:33:36.073 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:36.073 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:36.073 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 20:33:36.073 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:36.073 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:36.073 | INFO | Uploaded 36GiB to GPU in 2.14807s (16.7592GiB/s) | 20260505 20:33:36.073 | INFO | Downloaded 34GiB to GPU in 2.06697s (16.4492GiB/s) | 20260505 20:33:41.224 | INFO | Analyzing chunk frames = [6109696,6110208) | 20260505 20:33:41.224 | INFO | MemInfo: { | 20260505 20:33:41.224 | INFO | "gpu" : " 17 GB", | 20260505 20:33:41.224 | INFO | "huge" : "", | 20260505 20:33:41.224 | INFO | "huge_shared" : " 27 GB", | 20260505 20:33:41.224 | INFO | "malloc" : " 14 GB", | 20260505 20:33:41.224 | INFO | "pinned" : " 29 GB" | 20260505 20:33:41.224 | INFO | } | 20260505 20:33:41.224 | INFO | Perf report for the last 5.15119 seconds: | 20260505 20:33:41.224 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:33:41.224 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:41.224 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:41.224 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:41.224 | INFO | ANALYSIS : 62.1%, 3.0, 97.2ms, 0 | 20260505 20:33:41.224 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:41.224 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:41.224 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260505 20:33:41.224 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:41.224 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:41.224 | INFO | Uploaded 36GiB to GPU in 2.15032s (16.7417GiB/s) | 20260505 20:33:41.224 | INFO | Downloaded 34GiB to GPU in 2.05271s (16.5635GiB/s) | 20260505 20:33:46.367 | INFO | Analyzing chunk frames = [6110208,6110720) | 20260505 20:33:46.367 | INFO | MemInfo: { | 20260505 20:33:46.367 | INFO | "gpu" : " 17 GB", | 20260505 20:33:46.367 | INFO | "huge" : "", | 20260505 20:33:46.367 | INFO | "huge_shared" : " 27 GB", | 20260505 20:33:46.367 | INFO | "malloc" : " 14 GB", | 20260505 20:33:46.367 | INFO | "pinned" : " 29 GB" | 20260505 20:33:46.367 | INFO | } | 20260505 20:33:46.367 | INFO | Perf report for the last 5.1436 seconds: | 20260505 20:33:46.367 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:33:46.367 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:46.367 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:46.367 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:46.367 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 20:33:46.367 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:46.367 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:46.367 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 20:33:46.368 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:46.368 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:46.368 | INFO | Uploaded 36GiB to GPU in 2.14447s (16.7874GiB/s) | 20260505 20:33:46.368 | INFO | Downloaded 34GiB to GPU in 2.10898s (16.1215GiB/s) | 20260505 20:33:51.506 | INFO | Analyzing chunk frames = [6110720,6111232) | 20260505 20:33:51.506 | INFO | MemInfo: { | 20260505 20:33:51.506 | INFO | "gpu" : " 17 GB", | 20260505 20:33:51.506 | INFO | "huge" : "", | 20260505 20:33:51.506 | INFO | "huge_shared" : " 27 GB", | 20260505 20:33:51.506 | INFO | "malloc" : " 14 GB", | 20260505 20:33:51.506 | INFO | "pinned" : " 29 GB" | 20260505 20:33:51.506 | INFO | } | 20260505 20:33:51.506 | INFO | Perf report for the last 5.13901 seconds: | 20260505 20:33:51.506 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:33:51.506 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:51.506 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:51.506 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:51.506 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 20:33:51.507 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:51.507 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:51.507 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 20:33:51.507 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:51.507 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:51.507 | INFO | Uploaded 36GiB to GPU in 2.11603s (17.013GiB/s) | 20260505 20:33:51.507 | INFO | Downloaded 34GiB to GPU in 2.00327s (16.9722GiB/s) | 20260505 20:33:51.507 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 20:33:51.507 | INFO | Total mem usage: 15.3 GiB | 20260505 20:33:51.507 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 20:33:51.507 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.15%) | 20260505 20:33:51.507 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 20:33:51.507 | INFO | High Water Breakdown: | 20260505 20:33:51.507 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 20:33:51.507 | INFO | 38.8% : 5.5 GiB ( 82.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 20:33:51.507 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 20:33:51.507 | INFO | ------------------------------------------ | 20260505 20:33:51.507 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:33:51.507 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 20:33:51.507 | INFO | Total mem usage: 26.5 GiB | 20260505 20:33:51.507 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 20:33:51.507 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 20:33:51.507 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 20:33:51.507 | INFO | High Water Breakdown: | 20260505 20:33:51.507 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:33:51.507 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:33:51.507 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:33:51.507 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:33:51.507 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:33:51.507 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:33:51.507 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 20:33:51.507 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 20:33:51.507 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:33:51.507 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 20:33:51.507 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 20:33:51.507 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 20:33:51.507 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 20:33:51.507 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 20:33:51.507 | INFO | ------------------------------------------ | 20260505 20:33:51.507 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:33:51.507 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 20:33:51.507 | INFO | Total mem usage: 25.6 GiB | 20260505 20:33:51.507 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 20:33:51.507 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 20:33:51.507 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 20:33:51.507 | INFO | High Water Breakdown: | 20260505 20:33:51.507 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 20:33:51.507 | INFO | ------------------------------------------ | 20260505 20:33:51.507 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:33:51.507 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 20:33:51.507 | INFO | Total mem usage: 15.7 GiB | 20260505 20:33:51.507 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 20:33:51.507 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 20:33:51.507 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 20:33:51.507 | INFO | High Water Breakdown: | 20260505 20:33:51.507 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 20:33:51.507 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 20:33:51.507 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 20:33:51.507 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 20:33:51.507 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 20:33:51.507 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 20:33:51.507 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 20:33:51.507 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 20:33:51.507 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 20:33:51.507 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:33:51.507 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 20:33:51.507 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:33:51.507 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 20:33:51.507 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 20:33:51.507 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 20:33:51.507 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 20:33:51.507 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:33:51.507 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 20:33:51.507 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:33:51.507 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:33:51.507 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:33:51.507 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 20:33:51.508 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 20:33:51.508 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:33:51.508 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 20:33:51.508 | INFO | ------------------------------------------ | 20260505 20:33:51.508 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:33:51.508 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 20:33:51.508 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:33:51.508 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:33:51.508 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.41e+01, 1.34e+01 / 1.27e+00 | 20260505 20:33:51.508 | INFO | PerfReport: 0.29%, AggregateStats -- 9.25e-02 / 1.03e-01, 2.02e-02 / 4.01e-03 | 20260505 20:33:51.508 | INFO | PerfReport: 11.80%, Baseline -- 3.71e+00 / 4.55e+00, 5.60e-01 / 1.37e-01 | 20260505 20:33:51.508 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.55e+00, 1.86e-01 / 6.98e-02 | 20260505 20:33:51.508 | INFO | PerfReport: 15.94%, DME -- 4.01e+01 / 4.40e+01, 3.86e+00 / 1.22e+00 | 20260505 20:33:51.508 | INFO | PerfReport: 57.07%, FrameLabeling -- 1.79e+01 / 1.93e+01, 1.42e+00 / 1.02e+00 | 20260505 20:33:51.508 | INFO | PerfReport: 7.26%, PulseAccumulating -- 2.28e+00 / 2.86e+00, 4.34e-01 / 1.85e-01 | 20260505 20:33:51.508 | INFO | PerfReport: 3.25%, Metrics -- 1.02e+00 / 1.37e+00, 2.80e-01 / 1.59e-01 | 20260505 20:33:51.508 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.54e-01 / 4.20e-01, 6.20e-02 / 3.29e-02 | 20260505 20:33:51.508 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 20:33:51.508 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:33:51.508 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:33:51.508 | INFO | PerfReport: Full Execution -- 9.94e+01 / -nan, 2.54e+01 / 0.00e+00 | 20260505 20:33:51.508 | INFO | PerfReport: 22.54%, Upload -- 2.24e+01 / 0.00e+00, 7.76e+00 / 0.00e+00 | 20260505 20:33:51.508 | INFO | PerfReport: 21.80%, Download -- 2.17e+01 / 0.00e+00, 7.65e+00 / 0.00e+00 | 20260505 20:33:51.508 | INFO | PerfReport: 31.62%, Compute -- 3.14e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 20:33:51.508 | INFO | PerfReport: 2.37%, LockUpload -- 2.36e+00 / 0.00e+00, 8.15e+00 / 0.00e+00 | 20260505 20:33:51.508 | INFO | PerfReport: 1.32%, LockDownload -- 1.31e+00 / 0.00e+00, 2.81e+00 / 0.00e+00 | 20260505 20:33:51.508 | INFO | PerfReport: 20.34%, LockCompute -- 2.02e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 20:33:51.508 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.74e-02 / 0.00e+00 | 20260505 20:33:51.637 | INFO | Starting a round of Baz IO | 20260505 20:33:54.796 | INFO | Populating RTMetrics report | 20260505 20:33:54.797 | INFO | RTMetrics report populated | 20260505 20:33:54.797 | INFO | Opening tmp RTMetrics json file | 20260505 20:33:54.799 | INFO | Serializing RTMetrics to json | 20260505 20:33:54.801 | INFO | Serialization done, swapping tmp file now... | 20260505 20:33:54.804 | INFO | RTMetrics done | 20260505 20:33:54.886 | INFO | Finished a round of Baz IO: Wrote 2.006 GiB in 3.249 seconds | 20260505 20:33:54.886 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.66% / 0.00% / 78.34% | 20260505 20:33:54.886 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.171789% / 0% / 5.5631e-05% | 20260505 20:33:54.886 | INFO | Baz IO overall: | 20260505 20:33:54.886 | INFO | BytesWritten : 3558517401882 | 20260505 20:33:54.886 | INFO | NumEvents : 1681986105285 | 20260505 20:33:56.635 | INFO | Analyzing chunk frames = [6111232,6111744) | 20260505 20:33:56.635 | INFO | MemInfo: { | 20260505 20:33:56.635 | INFO | "gpu" : " 17 GB", | 20260505 20:33:56.635 | INFO | "huge" : "", | 20260505 20:33:56.635 | INFO | "huge_shared" : " 27 GB", | 20260505 20:33:56.635 | INFO | "malloc" : " 14 GB", | 20260505 20:33:56.635 | INFO | "pinned" : " 29 GB" | 20260505 20:33:56.635 | INFO | } | 20260505 20:33:56.635 | INFO | Perf report for the last 5.12835 seconds: | 20260505 20:33:56.635 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:33:56.635 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:56.635 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:56.635 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:56.635 | INFO | ANALYSIS : 63.1%, 3.0, 98.2ms, 0 | 20260505 20:33:56.635 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:33:56.635 | INFO | RT_METRICS : 44.2%, 1.0, 23.1ms, 0 | 20260505 20:33:56.635 | INFO | PRE_HQ : 60.4%, 1.8, 55.6ms, 0 | 20260505 20:33:56.635 | INFO | BAZWRITER : 62.8%, 3.7, 118.7ms, 0 | 20260505 20:33:56.635 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 20:33:56.635 | INFO | Uploaded 36GiB to GPU in 2.17585s (16.5453GiB/s) | 20260505 20:33:56.635 | INFO | Downloaded 38GiB to GPU in 2.2377s (16.9817GiB/s) | 20260505 20:34:01.774 | INFO | Analyzing chunk frames = [6111744,6112256) | 20260505 20:34:01.774 | INFO | MemInfo: { | 20260505 20:34:01.774 | INFO | "gpu" : " 17 GB", | 20260505 20:34:01.774 | INFO | "huge" : "", | 20260505 20:34:01.774 | INFO | "huge_shared" : " 27 GB", | 20260505 20:34:01.774 | INFO | "malloc" : " 14 GB", | 20260505 20:34:01.774 | INFO | "pinned" : " 29 GB" | 20260505 20:34:01.774 | INFO | } | 20260505 20:34:01.774 | INFO | Perf report for the last 5.1392 seconds: | 20260505 20:34:01.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:34:01.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:01.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:01.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:01.774 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260505 20:34:01.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:01.774 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:01.774 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:34:01.774 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:01.774 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:01.774 | INFO | Uploaded 36GiB to GPU in 2.13563s (16.8569GiB/s) | 20260505 20:34:01.774 | INFO | Downloaded 34GiB to GPU in 2.03104s (16.7402GiB/s) | 20260505 20:34:06.921 | INFO | Analyzing chunk frames = [6112256,6112768) | 20260505 20:34:06.921 | INFO | MemInfo: { | 20260505 20:34:06.921 | INFO | "gpu" : " 17 GB", | 20260505 20:34:06.921 | INFO | "huge" : "", | 20260505 20:34:06.921 | INFO | "huge_shared" : " 27 GB", | 20260505 20:34:06.921 | INFO | "malloc" : " 14 GB", | 20260505 20:34:06.921 | INFO | "pinned" : " 29 GB" | 20260505 20:34:06.921 | INFO | } | 20260505 20:34:06.921 | INFO | Perf report for the last 5.14689 seconds: | 20260505 20:34:06.921 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:34:06.921 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:06.921 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:06.921 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:06.921 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260505 20:34:06.921 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:06.921 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:06.921 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260505 20:34:06.921 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:06.921 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:06.921 | INFO | Uploaded 36GiB to GPU in 2.11792s (16.9978GiB/s) | 20260505 20:34:06.921 | INFO | Downloaded 34GiB to GPU in 2.0248s (16.7918GiB/s) | 20260505 20:34:12.059 | INFO | Analyzing chunk frames = [6112768,6113280) | 20260505 20:34:12.059 | INFO | MemInfo: { | 20260505 20:34:12.059 | INFO | "gpu" : " 17 GB", | 20260505 20:34:12.059 | INFO | "huge" : "", | 20260505 20:34:12.059 | INFO | "huge_shared" : " 27 GB", | 20260505 20:34:12.060 | INFO | "malloc" : " 14 GB", | 20260505 20:34:12.060 | INFO | "pinned" : " 29 GB" | 20260505 20:34:12.060 | INFO | } | 20260505 20:34:12.060 | INFO | Perf report for the last 5.13889 seconds: | 20260505 20:34:12.060 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:34:12.060 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:12.060 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:12.060 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:12.060 | INFO | ANALYSIS : 62.5%, 3.0, 97.0ms, 0 | 20260505 20:34:12.060 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:12.060 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:12.060 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:34:12.060 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:12.060 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:12.060 | INFO | Uploaded 36GiB to GPU in 2.21315s (16.2664GiB/s) | 20260505 20:34:12.060 | INFO | Downloaded 34GiB to GPU in 2.07126s (16.4151GiB/s) | 20260505 20:34:17.195 | INFO | Analyzing chunk frames = [6113280,6113792) | 20260505 20:34:17.195 | INFO | MemInfo: { | 20260505 20:34:17.195 | INFO | "gpu" : " 17 GB", | 20260505 20:34:17.195 | INFO | "huge" : "", | 20260505 20:34:17.195 | INFO | "huge_shared" : " 27 GB", | 20260505 20:34:17.195 | INFO | "malloc" : " 14 GB", | 20260505 20:34:17.195 | INFO | "pinned" : " 29 GB" | 20260505 20:34:17.195 | INFO | } | 20260505 20:34:17.195 | INFO | Perf report for the last 5.13509 seconds: | 20260505 20:34:17.195 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:34:17.195 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:17.195 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:17.195 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:17.195 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260505 20:34:17.195 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:17.195 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:17.195 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 20:34:17.195 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:17.195 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:17.195 | INFO | Uploaded 36GiB to GPU in 2.12605s (16.9328GiB/s) | 20260505 20:34:17.195 | INFO | Downloaded 34GiB to GPU in 2.06173s (16.491GiB/s) | 20260505 20:34:22.332 | INFO | Analyzing chunk frames = [6113792,6114304) | 20260505 20:34:22.332 | INFO | MemInfo: { | 20260505 20:34:22.332 | INFO | "gpu" : " 17 GB", | 20260505 20:34:22.332 | INFO | "huge" : "", | 20260505 20:34:22.332 | INFO | "huge_shared" : " 27 GB", | 20260505 20:34:22.332 | INFO | "malloc" : " 14 GB", | 20260505 20:34:22.332 | INFO | "pinned" : " 29 GB" | 20260505 20:34:22.332 | INFO | } | 20260505 20:34:22.332 | INFO | Perf report for the last 5.13698 seconds: | 20260505 20:34:22.332 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:34:22.332 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:22.332 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:22.332 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:22.332 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 20:34:22.332 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:22.332 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:22.332 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:34:22.332 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:22.332 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:22.332 | INFO | Uploaded 36GiB to GPU in 2.15409s (16.7124GiB/s) | 20260505 20:34:22.332 | INFO | Downloaded 34GiB to GPU in 2.05933s (16.5102GiB/s) | 20260505 20:34:27.470 | INFO | Analyzing chunk frames = [6114304,6114816) | 20260505 20:34:27.470 | INFO | MemInfo: { | 20260505 20:34:27.470 | INFO | "gpu" : " 17 GB", | 20260505 20:34:27.470 | INFO | "huge" : "", | 20260505 20:34:27.470 | INFO | "huge_shared" : " 27 GB", | 20260505 20:34:27.470 | INFO | "malloc" : " 14 GB", | 20260505 20:34:27.470 | INFO | "pinned" : " 29 GB" | 20260505 20:34:27.470 | INFO | } | 20260505 20:34:27.470 | INFO | Perf report for the last 5.13837 seconds: | 20260505 20:34:27.470 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:34:27.470 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:27.470 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:27.470 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:27.470 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 20:34:27.470 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:27.470 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:27.470 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 20:34:27.470 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:27.470 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:27.471 | INFO | Uploaded 36GiB to GPU in 2.1203s (16.9787GiB/s) | 20260505 20:34:27.471 | INFO | Downloaded 34GiB to GPU in 2.13707s (15.9096GiB/s) | 20260505 20:34:32.612 | INFO | Analyzing chunk frames = [6114816,6115328) | 20260505 20:34:32.613 | INFO | MemInfo: { | 20260505 20:34:32.613 | INFO | "gpu" : " 17 GB", | 20260505 20:34:32.613 | INFO | "huge" : "", | 20260505 20:34:32.613 | INFO | "huge_shared" : " 27 GB", | 20260505 20:34:32.613 | INFO | "malloc" : " 14 GB", | 20260505 20:34:32.613 | INFO | "pinned" : " 29 GB" | 20260505 20:34:32.613 | INFO | } | 20260505 20:34:32.613 | INFO | Perf report for the last 5.14244 seconds: | 20260505 20:34:32.613 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:34:32.613 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:32.613 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:32.613 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:32.613 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260505 20:34:32.613 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:32.613 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:32.613 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:34:32.613 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:32.613 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:32.613 | INFO | Uploaded 36GiB to GPU in 2.12369s (16.9516GiB/s) | 20260505 20:34:32.613 | INFO | Downloaded 34GiB to GPU in 2.0139s (16.8827GiB/s) | 20260505 20:34:32.747 | INFO | Starting a round of Baz IO | 20260505 20:34:35.905 | INFO | Populating RTMetrics report | 20260505 20:34:35.905 | INFO | RTMetrics report populated | 20260505 20:34:35.905 | INFO | Opening tmp RTMetrics json file | 20260505 20:34:35.906 | INFO | Serializing RTMetrics to json | 20260505 20:34:35.907 | INFO | Serialization done, swapping tmp file now... | 20260505 20:34:35.908 | INFO | RTMetrics done | 20260505 20:34:35.991 | INFO | Finished a round of Baz IO: Wrote 2.004 GiB in 3.243 seconds | 20260505 20:34:35.991 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.66% / 0.00% / 78.34% | 20260505 20:34:35.991 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.170807% / 0% / 7.54992e-05% | 20260505 20:34:35.991 | INFO | Baz IO overall: | 20260505 20:34:35.991 | INFO | BytesWritten : 3560669469104 | 20260505 20:34:35.991 | INFO | NumEvents : 1682981935646 | 20260505 20:34:37.752 | INFO | Analyzing chunk frames = [6115328,6115840) | 20260505 20:34:37.752 | INFO | MemInfo: { | 20260505 20:34:37.752 | INFO | "gpu" : " 17 GB", | 20260505 20:34:37.752 | INFO | "huge" : "", | 20260505 20:34:37.752 | INFO | "huge_shared" : " 27 GB", | 20260505 20:34:37.752 | INFO | "malloc" : " 14 GB", | 20260505 20:34:37.752 | INFO | "pinned" : " 29 GB" | 20260505 20:34:37.752 | INFO | } | 20260505 20:34:37.752 | INFO | Perf report for the last 5.13936 seconds: | 20260505 20:34:37.752 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:34:37.752 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:37.752 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:37.752 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:37.752 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 20:34:37.752 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:37.752 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260505 20:34:37.752 | INFO | PRE_HQ : 60.1%, 1.8, 55.0ms, 0 | 20260505 20:34:37.752 | INFO | BAZWRITER : 62.7%, 3.7, 119.3ms, 0 | 20260505 20:34:37.752 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 20:34:37.752 | INFO | Uploaded 36GiB to GPU in 2.13752s (16.842GiB/s) | 20260505 20:34:37.752 | INFO | Downloaded 38GiB to GPU in 2.2466s (16.9145GiB/s) | 20260505 20:34:42.886 | INFO | Analyzing chunk frames = [6115840,6116352) | 20260505 20:34:42.886 | INFO | MemInfo: { | 20260505 20:34:42.886 | INFO | "gpu" : " 17 GB", | 20260505 20:34:42.886 | INFO | "huge" : "", | 20260505 20:34:42.886 | INFO | "huge_shared" : " 27 GB", | 20260505 20:34:42.886 | INFO | "malloc" : " 14 GB", | 20260505 20:34:42.886 | INFO | "pinned" : " 29 GB" | 20260505 20:34:42.886 | INFO | } | 20260505 20:34:42.886 | INFO | Perf report for the last 5.13426 seconds: | 20260505 20:34:42.886 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:34:42.886 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:42.886 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:42.886 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:42.886 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 20:34:42.886 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:42.886 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:42.886 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 20:34:42.886 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:42.886 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:42.887 | INFO | Uploaded 36GiB to GPU in 2.14515s (16.782GiB/s) | 20260505 20:34:42.887 | INFO | Downloaded 34GiB to GPU in 2.04711s (16.6088GiB/s) | 20260505 20:34:48.025 | INFO | Analyzing chunk frames = [6116352,6116864) | 20260505 20:34:48.025 | INFO | MemInfo: { | 20260505 20:34:48.025 | INFO | "gpu" : " 17 GB", | 20260505 20:34:48.025 | INFO | "huge" : "", | 20260505 20:34:48.025 | INFO | "huge_shared" : " 27 GB", | 20260505 20:34:48.025 | INFO | "malloc" : " 14 GB", | 20260505 20:34:48.025 | INFO | "pinned" : " 29 GB" | 20260505 20:34:48.025 | INFO | } | 20260505 20:34:48.025 | INFO | Perf report for the last 5.13889 seconds: | 20260505 20:34:48.025 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:34:48.025 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:48.025 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:48.025 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:48.025 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260505 20:34:48.025 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:48.025 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:48.025 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:34:48.025 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:48.025 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:48.026 | INFO | Uploaded 36GiB to GPU in 2.14387s (16.792GiB/s) | 20260505 20:34:48.026 | INFO | Downloaded 34GiB to GPU in 2.0357s (16.7019GiB/s) | 20260505 20:34:53.176 | INFO | Analyzing chunk frames = [6116864,6117376) | 20260505 20:34:53.176 | INFO | MemInfo: { | 20260505 20:34:53.176 | INFO | "gpu" : " 17 GB", | 20260505 20:34:53.176 | INFO | "huge" : "", | 20260505 20:34:53.176 | INFO | "huge_shared" : " 27 GB", | 20260505 20:34:53.176 | INFO | "malloc" : " 14 GB", | 20260505 20:34:53.176 | INFO | "pinned" : " 29 GB" | 20260505 20:34:53.176 | INFO | } | 20260505 20:34:53.176 | INFO | Perf report for the last 5.15081 seconds: | 20260505 20:34:53.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:34:53.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:53.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:53.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:53.176 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260505 20:34:53.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:53.176 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:53.176 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:34:53.176 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:53.176 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:53.176 | INFO | Uploaded 36GiB to GPU in 2.16383s (16.6371GiB/s) | 20260505 20:34:53.176 | INFO | Downloaded 34GiB to GPU in 2.05909s (16.5121GiB/s) | 20260505 20:34:58.319 | INFO | Analyzing chunk frames = [6117376,6117888) | 20260505 20:34:58.319 | INFO | MemInfo: { | 20260505 20:34:58.319 | INFO | "gpu" : " 17 GB", | 20260505 20:34:58.319 | INFO | "huge" : "", | 20260505 20:34:58.319 | INFO | "huge_shared" : " 27 GB", | 20260505 20:34:58.319 | INFO | "malloc" : " 14 GB", | 20260505 20:34:58.319 | INFO | "pinned" : " 29 GB" | 20260505 20:34:58.319 | INFO | } | 20260505 20:34:58.319 | INFO | Perf report for the last 5.14274 seconds: | 20260505 20:34:58.319 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:34:58.319 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:58.319 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:58.319 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:58.319 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260505 20:34:58.319 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:58.319 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:58.319 | INFO | PRE_HQ : 27.2%, 1.0, 14.2ms, 0 | 20260505 20:34:58.319 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:58.319 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:34:58.319 | INFO | Uploaded 36GiB to GPU in 2.14529s (16.7809GiB/s) | 20260505 20:34:58.319 | INFO | Downloaded 34GiB to GPU in 2.06675s (16.451GiB/s) | 20260505 20:35:03.448 | INFO | Analyzing chunk frames = [6117888,6118400) | 20260505 20:35:03.448 | INFO | MemInfo: { | 20260505 20:35:03.448 | INFO | "gpu" : " 17 GB", | 20260505 20:35:03.448 | INFO | "huge" : "", | 20260505 20:35:03.448 | INFO | "huge_shared" : " 27 GB", | 20260505 20:35:03.448 | INFO | "malloc" : " 14 GB", | 20260505 20:35:03.448 | INFO | "pinned" : " 29 GB" | 20260505 20:35:03.448 | INFO | } | 20260505 20:35:03.448 | INFO | Perf report for the last 5.129 seconds: | 20260505 20:35:03.448 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:35:03.448 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:03.448 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:03.448 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:03.448 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260505 20:35:03.448 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:03.448 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:03.448 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 20:35:03.448 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:03.448 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:03.448 | INFO | Uploaded 36GiB to GPU in 2.07818s (17.3229GiB/s) | 20260505 20:35:03.448 | INFO | Downloaded 34GiB to GPU in 2.02532s (16.7875GiB/s) | 20260505 20:35:08.584 | INFO | Analyzing chunk frames = [6118400,6118912) | 20260505 20:35:08.584 | INFO | MemInfo: { | 20260505 20:35:08.584 | INFO | "gpu" : " 17 GB", | 20260505 20:35:08.584 | INFO | "huge" : "", | 20260505 20:35:08.584 | INFO | "huge_shared" : " 27 GB", | 20260505 20:35:08.584 | INFO | "malloc" : " 14 GB", | 20260505 20:35:08.584 | INFO | "pinned" : " 29 GB" | 20260505 20:35:08.584 | INFO | } | 20260505 20:35:08.584 | INFO | Perf report for the last 5.13651 seconds: | 20260505 20:35:08.584 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:35:08.584 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:08.584 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:08.584 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:08.584 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 20:35:08.584 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:08.584 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:08.584 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 20:35:08.584 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:08.584 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:08.585 | INFO | Uploaded 36GiB to GPU in 2.14259s (16.8021GiB/s) | 20260505 20:35:08.585 | INFO | Downloaded 34GiB to GPU in 2.14257s (15.8688GiB/s) | 20260505 20:35:13.727 | INFO | Analyzing chunk frames = [6118912,6119424) | 20260505 20:35:13.727 | INFO | MemInfo: { | 20260505 20:35:13.727 | INFO | "gpu" : " 17 GB", | 20260505 20:35:13.727 | INFO | "huge" : "", | 20260505 20:35:13.727 | INFO | "huge_shared" : " 27 GB", | 20260505 20:35:13.728 | INFO | "malloc" : " 14 GB", | 20260505 20:35:13.728 | INFO | "pinned" : " 29 GB" | 20260505 20:35:13.728 | INFO | } | 20260505 20:35:13.728 | INFO | Perf report for the last 5.14333 seconds: | 20260505 20:35:13.728 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:35:13.728 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:13.728 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:13.728 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:13.728 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 20:35:13.728 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:13.728 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:13.728 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:35:13.728 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:13.728 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:13.728 | INFO | Uploaded 36GiB to GPU in 2.20344s (16.3381GiB/s) | 20260505 20:35:13.728 | INFO | Downloaded 34GiB to GPU in 2.04598s (16.618GiB/s) | 20260505 20:35:13.728 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 20:35:13.728 | INFO | Total mem usage: 15.3 GiB | 20260505 20:35:13.728 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 20:35:13.728 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.15%) | 20260505 20:35:13.728 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 20:35:13.728 | INFO | High Water Breakdown: | 20260505 20:35:13.728 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 20:35:13.728 | INFO | 38.8% : 5.5 GiB ( 82.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 20:35:13.728 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 20:35:13.728 | INFO | ------------------------------------------ | 20260505 20:35:13.728 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:35:13.728 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 20:35:13.728 | INFO | Total mem usage: 26.5 GiB | 20260505 20:35:13.728 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 20:35:13.729 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 20:35:13.729 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 20:35:13.729 | INFO | High Water Breakdown: | 20260505 20:35:13.729 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:35:13.729 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:35:13.729 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:35:13.729 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:35:13.729 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:35:13.729 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:35:13.729 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 20:35:13.729 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 20:35:13.729 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:35:13.729 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 20:35:13.729 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 20:35:13.729 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 20:35:13.729 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 20:35:13.729 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 20:35:13.729 | INFO | ------------------------------------------ | 20260505 20:35:13.729 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:35:13.729 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 20:35:13.729 | INFO | Total mem usage: 25.6 GiB | 20260505 20:35:13.729 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 20:35:13.729 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 20:35:13.729 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 20:35:13.729 | INFO | High Water Breakdown: | 20260505 20:35:13.729 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 20:35:13.729 | INFO | ------------------------------------------ | 20260505 20:35:13.729 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:35:13.729 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 20:35:13.729 | INFO | Total mem usage: 15.7 GiB | 20260505 20:35:13.729 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 20:35:13.729 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 20:35:13.729 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 20:35:13.729 | INFO | High Water Breakdown: | 20260505 20:35:13.729 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 20:35:13.729 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 20:35:13.729 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 20:35:13.729 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 20:35:13.729 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 20:35:13.729 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 20:35:13.729 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 20:35:13.729 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 20:35:13.729 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 20:35:13.729 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:35:13.729 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 20:35:13.729 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:35:13.729 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 20:35:13.729 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 20:35:13.729 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 20:35:13.729 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 20:35:13.729 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:35:13.729 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 20:35:13.729 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:35:13.729 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:35:13.729 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:35:13.729 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 20:35:13.729 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 20:35:13.729 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:35:13.729 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 20:35:13.729 | INFO | ------------------------------------------ | 20260505 20:35:13.729 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:35:13.729 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 20:35:13.729 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:35:13.729 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:35:13.729 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.47e+01, 1.36e+01 / 2.11e+00 | 20260505 20:35:13.729 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.03e-01, 1.02e-02 / 5.61e-03 | 20260505 20:35:13.729 | INFO | PerfReport: 11.78%, Baseline -- 3.70e+00 / 4.55e+00, 5.57e-01 / 1.79e-01 | 20260505 20:35:13.729 | INFO | PerfReport: 3.26%, Histogram -- 1.03e+00 / 1.50e+00, 1.86e-01 / 1.15e-01 | 20260505 20:35:13.729 | INFO | PerfReport: 15.98%, DME -- 4.02e+01 / 4.46e+01, 4.19e+00 / 2.15e+00 | 20260505 20:35:13.729 | INFO | PerfReport: 57.09%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.46e+00 / 1.07e+00 | 20260505 20:35:13.729 | INFO | PerfReport: 7.23%, PulseAccumulating -- 2.27e+00 / 2.82e+00, 4.37e-01 / 2.39e-01 | 20260505 20:35:13.729 | INFO | PerfReport: 3.23%, Metrics -- 1.02e+00 / 1.34e+00, 2.80e-01 / 1.41e-01 | 20260505 20:35:13.729 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.54e-01 / 4.21e-01, 6.23e-02 / 2.94e-02 | 20260505 20:35:13.729 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 20:35:13.729 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:35:13.729 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:35:13.729 | INFO | PerfReport: Full Execution -- 9.94e+01 / -nan, 2.54e+01 / 0.00e+00 | 20260505 20:35:13.729 | INFO | PerfReport: 22.49%, Upload -- 2.23e+01 / 0.00e+00, 7.73e+00 / 0.00e+00 | 20260505 20:35:13.729 | INFO | PerfReport: 21.82%, Download -- 2.17e+01 / 0.00e+00, 7.69e+00 / 0.00e+00 | 20260505 20:35:13.729 | INFO | PerfReport: 31.65%, Compute -- 3.15e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 20:35:13.729 | INFO | PerfReport: 2.30%, LockUpload -- 2.28e+00 / 0.00e+00, 7.94e+00 / 0.00e+00 | 20260505 20:35:13.729 | INFO | PerfReport: 1.37%, LockDownload -- 1.36e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260505 20:35:13.729 | INFO | PerfReport: 20.36%, LockCompute -- 2.02e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260505 20:35:13.729 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.55e-02 / 0.00e+00 | 20260505 20:35:13.868 | INFO | Starting a round of Baz IO | 20260505 20:35:17.024 | INFO | Populating RTMetrics report | 20260505 20:35:17.024 | INFO | RTMetrics report populated | 20260505 20:35:17.024 | INFO | Opening tmp RTMetrics json file | 20260505 20:35:17.025 | INFO | Serializing RTMetrics to json | 20260505 20:35:17.030 | INFO | Serialization done, swapping tmp file now... | 20260505 20:35:17.033 | INFO | RTMetrics done | 20260505 20:35:17.107 | INFO | Finished a round of Baz IO: Wrote 2.006 GiB in 3.239 seconds | 20260505 20:35:17.107 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.66% / 0.00% / 78.34% | 20260505 20:35:17.107 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.170823% / 3.97364e-06% / 8.34465e-05% | 20260505 20:35:17.107 | INFO | Baz IO overall: | 20260505 20:35:17.107 | INFO | BytesWritten : 3562822902870 | 20260505 20:35:17.107 | INFO | NumEvents : 1683979145398 | 20260505 20:35:18.872 | INFO | Analyzing chunk frames = [6119424,6119936) | 20260505 20:35:18.872 | INFO | MemInfo: { | 20260505 20:35:18.872 | INFO | "gpu" : " 17 GB", | 20260505 20:35:18.872 | INFO | "huge" : "", | 20260505 20:35:18.872 | INFO | "huge_shared" : " 27 GB", | 20260505 20:35:18.872 | INFO | "malloc" : " 14 GB", | 20260505 20:35:18.872 | INFO | "pinned" : " 29 GB" | 20260505 20:35:18.872 | INFO | } | 20260505 20:35:18.872 | INFO | Perf report for the last 5.14396 seconds: | 20260505 20:35:18.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:35:18.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:18.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:18.872 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:18.872 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 20:35:18.872 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:18.872 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260505 20:35:18.872 | INFO | PRE_HQ : 60.1%, 1.8, 55.1ms, 0 | 20260505 20:35:18.872 | INFO | BAZWRITER : 62.5%, 3.7, 118.9ms, 0 | 20260505 20:35:18.872 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 20:35:18.872 | INFO | Uploaded 36GiB to GPU in 2.17663s (16.5393GiB/s) | 20260505 20:35:18.872 | INFO | Downloaded 38GiB to GPU in 2.25558s (16.8471GiB/s) | 20260505 20:35:24.012 | INFO | Analyzing chunk frames = [6119936,6120448) | 20260505 20:35:24.012 | INFO | MemInfo: { | 20260505 20:35:24.012 | INFO | "gpu" : " 17 GB", | 20260505 20:35:24.012 | INFO | "huge" : "", | 20260505 20:35:24.012 | INFO | "huge_shared" : " 27 GB", | 20260505 20:35:24.012 | INFO | "malloc" : " 14 GB", | 20260505 20:35:24.012 | INFO | "pinned" : " 29 GB" | 20260505 20:35:24.012 | INFO | } | 20260505 20:35:24.012 | INFO | Perf report for the last 5.14075 seconds: | 20260505 20:35:24.012 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:35:24.012 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:24.012 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:24.012 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:24.012 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 20:35:24.013 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:24.013 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:24.013 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 20:35:24.013 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:24.013 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:24.013 | INFO | Uploaded 36GiB to GPU in 2.08053s (17.3033GiB/s) | 20260505 20:35:24.013 | INFO | Downloaded 34GiB to GPU in 2.02365s (16.8013GiB/s) | 20260505 20:35:29.148 | INFO | Analyzing chunk frames = [6120448,6120960) | 20260505 20:35:29.148 | INFO | MemInfo: { | 20260505 20:35:29.148 | INFO | "gpu" : " 17 GB", | 20260505 20:35:29.148 | INFO | "huge" : "", | 20260505 20:35:29.148 | INFO | "huge_shared" : " 27 GB", | 20260505 20:35:29.148 | INFO | "malloc" : " 14 GB", | 20260505 20:35:29.148 | INFO | "pinned" : " 29 GB" | 20260505 20:35:29.148 | INFO | } | 20260505 20:35:29.148 | INFO | Perf report for the last 5.13539 seconds: | 20260505 20:35:29.148 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:35:29.148 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:29.148 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:29.148 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:29.148 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260505 20:35:29.148 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:29.148 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:29.148 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:35:29.148 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:29.148 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:29.148 | INFO | Uploaded 36GiB to GPU in 2.1336s (16.8729GiB/s) | 20260505 20:35:29.148 | INFO | Downloaded 34GiB to GPU in 2.01979s (16.8334GiB/s) | 20260505 20:35:34.284 | INFO | Analyzing chunk frames = [6120960,6121472) | 20260505 20:35:34.284 | INFO | MemInfo: { | 20260505 20:35:34.284 | INFO | "gpu" : " 17 GB", | 20260505 20:35:34.284 | INFO | "huge" : "", | 20260505 20:35:34.284 | INFO | "huge_shared" : " 27 GB", | 20260505 20:35:34.284 | INFO | "malloc" : " 14 GB", | 20260505 20:35:34.284 | INFO | "pinned" : " 29 GB" | 20260505 20:35:34.284 | INFO | } | 20260505 20:35:34.284 | INFO | Perf report for the last 5.13636 seconds: | 20260505 20:35:34.284 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:35:34.284 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:34.284 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:34.284 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:34.284 | INFO | ANALYSIS : 62.5%, 3.0, 97.0ms, 0 | 20260505 20:35:34.284 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:34.284 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:34.284 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:35:34.284 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:34.284 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:34.284 | INFO | Uploaded 36GiB to GPU in 2.20732s (16.3094GiB/s) | 20260505 20:35:34.284 | INFO | Downloaded 34GiB to GPU in 2.03642s (16.696GiB/s) | 20260505 20:35:39.423 | INFO | Analyzing chunk frames = [6121472,6121984) | 20260505 20:35:39.424 | INFO | MemInfo: { | 20260505 20:35:39.424 | INFO | "gpu" : " 17 GB", | 20260505 20:35:39.424 | INFO | "huge" : "", | 20260505 20:35:39.424 | INFO | "huge_shared" : " 27 GB", | 20260505 20:35:39.424 | INFO | "malloc" : " 14 GB", | 20260505 20:35:39.424 | INFO | "pinned" : " 29 GB" | 20260505 20:35:39.424 | INFO | } | 20260505 20:35:39.424 | INFO | Perf report for the last 5.13956 seconds: | 20260505 20:35:39.424 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:35:39.424 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:39.424 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:39.424 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:39.424 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260505 20:35:39.424 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:39.424 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:39.424 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:35:39.424 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:39.424 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:39.424 | INFO | Uploaded 36GiB to GPU in 2.13815s (16.837GiB/s) | 20260505 20:35:39.424 | INFO | Downloaded 34GiB to GPU in 2.06192s (16.4895GiB/s) | 20260505 20:35:44.566 | INFO | Analyzing chunk frames = [6121984,6122496) | 20260505 20:35:44.566 | INFO | MemInfo: { | 20260505 20:35:44.566 | INFO | "gpu" : " 17 GB", | 20260505 20:35:44.566 | INFO | "huge" : "", | 20260505 20:35:44.566 | INFO | "huge_shared" : " 27 GB", | 20260505 20:35:44.566 | INFO | "malloc" : " 14 GB", | 20260505 20:35:44.566 | INFO | "pinned" : " 29 GB" | 20260505 20:35:44.566 | INFO | } | 20260505 20:35:44.566 | INFO | Perf report for the last 5.14247 seconds: | 20260505 20:35:44.566 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:35:44.566 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:44.566 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:44.566 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:44.566 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260505 20:35:44.566 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:44.566 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:44.566 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 20:35:44.566 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:44.566 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:44.567 | INFO | Uploaded 36GiB to GPU in 2.08014s (17.3065GiB/s) | 20260505 20:35:44.567 | INFO | Downloaded 34GiB to GPU in 2.0367s (16.6936GiB/s) | 20260505 20:35:49.700 | INFO | Analyzing chunk frames = [6122496,6123008) | 20260505 20:35:49.700 | INFO | MemInfo: { | 20260505 20:35:49.700 | INFO | "gpu" : " 17 GB", | 20260505 20:35:49.700 | INFO | "huge" : "", | 20260505 20:35:49.700 | INFO | "huge_shared" : " 27 GB", | 20260505 20:35:49.700 | INFO | "malloc" : " 14 GB", | 20260505 20:35:49.700 | INFO | "pinned" : " 29 GB" | 20260505 20:35:49.700 | INFO | } | 20260505 20:35:49.700 | INFO | Perf report for the last 5.13365 seconds: | 20260505 20:35:49.700 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:35:49.700 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:49.700 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:49.700 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:49.700 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 20:35:49.700 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:49.700 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:49.700 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 20:35:49.700 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:49.700 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:49.700 | INFO | Uploaded 36GiB to GPU in 2.14523s (16.7814GiB/s) | 20260505 20:35:49.700 | INFO | Downloaded 34GiB to GPU in 2.14302s (15.8655GiB/s) | 20260505 20:35:54.835 | INFO | Analyzing chunk frames = [6123008,6123520) | 20260505 20:35:54.835 | INFO | MemInfo: { | 20260505 20:35:54.835 | INFO | "gpu" : " 17 GB", | 20260505 20:35:54.835 | INFO | "huge" : "", | 20260505 20:35:54.835 | INFO | "huge_shared" : " 27 GB", | 20260505 20:35:54.835 | INFO | "malloc" : " 14 GB", | 20260505 20:35:54.835 | INFO | "pinned" : " 29 GB" | 20260505 20:35:54.835 | INFO | } | 20260505 20:35:54.835 | INFO | Perf report for the last 5.1349 seconds: | 20260505 20:35:54.835 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:35:54.835 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:54.835 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:54.835 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:54.835 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260505 20:35:54.835 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:54.835 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:54.835 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:35:54.835 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:54.835 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:54.835 | INFO | Uploaded 36GiB to GPU in 2.10797s (17.078GiB/s) | 20260505 20:35:54.835 | INFO | Downloaded 34GiB to GPU in 2.01096s (16.9074GiB/s) | 20260505 20:35:54.967 | INFO | Starting a round of Baz IO | 20260505 20:35:58.138 | INFO | Populating RTMetrics report | 20260505 20:35:58.138 | INFO | RTMetrics report populated | 20260505 20:35:58.138 | INFO | Opening tmp RTMetrics json file | 20260505 20:35:58.140 | INFO | Serializing RTMetrics to json | 20260505 20:35:58.144 | INFO | Serialization done, swapping tmp file now... | 20260505 20:35:58.146 | INFO | RTMetrics done | 20260505 20:35:58.224 | INFO | Finished a round of Baz IO: Wrote 2.006 GiB in 3.256 seconds | 20260505 20:35:58.224 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.65% / 0.00% / 78.34% | 20260505 20:35:58.224 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.171053% / 3.97364e-06% / 0.000123183% | 20260505 20:35:58.224 | INFO | Baz IO overall: | 20260505 20:35:58.224 | INFO | BytesWritten : 3564976722629 | 20260505 20:35:58.224 | INFO | NumEvents : 1684976976943 | 20260505 20:35:59.983 | INFO | Analyzing chunk frames = [6123520,6124032) | 20260505 20:35:59.983 | INFO | MemInfo: { | 20260505 20:35:59.983 | INFO | "gpu" : " 17 GB", | 20260505 20:35:59.983 | INFO | "huge" : "", | 20260505 20:35:59.983 | INFO | "huge_shared" : " 27 GB", | 20260505 20:35:59.983 | INFO | "malloc" : " 14 GB", | 20260505 20:35:59.983 | INFO | "pinned" : " 29 GB" | 20260505 20:35:59.983 | INFO | } | 20260505 20:35:59.983 | INFO | Perf report for the last 5.14787 seconds: | 20260505 20:35:59.983 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:35:59.983 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:59.983 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:59.983 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:59.983 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 20:35:59.983 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:35:59.983 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260505 20:35:59.983 | INFO | PRE_HQ : 60.1%, 1.8, 55.3ms, 0 | 20260505 20:35:59.983 | INFO | BAZWRITER : 62.8%, 3.7, 119.0ms, 0 | 20260505 20:35:59.983 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 20:35:59.983 | INFO | Uploaded 36GiB to GPU in 2.16457s (16.6315GiB/s) | 20260505 20:35:59.983 | INFO | Downloaded 38GiB to GPU in 2.24107s (16.9562GiB/s) | 20260505 20:36:05.130 | INFO | Analyzing chunk frames = [6124032,6124544) | 20260505 20:36:05.130 | INFO | MemInfo: { | 20260505 20:36:05.130 | INFO | "gpu" : " 17 GB", | 20260505 20:36:05.130 | INFO | "huge" : "", | 20260505 20:36:05.130 | INFO | "huge_shared" : " 27 GB", | 20260505 20:36:05.130 | INFO | "malloc" : " 14 GB", | 20260505 20:36:05.130 | INFO | "pinned" : " 29 GB" | 20260505 20:36:05.130 | INFO | } | 20260505 20:36:05.130 | INFO | Perf report for the last 5.14742 seconds: | 20260505 20:36:05.130 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:36:05.130 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:05.130 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:05.130 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:05.130 | INFO | ANALYSIS : 62.1%, 3.0, 97.1ms, 0 | 20260505 20:36:05.130 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:05.130 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:05.130 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:36:05.130 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:05.130 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:05.130 | INFO | Uploaded 36GiB to GPU in 2.10271s (17.1207GiB/s) | 20260505 20:36:05.131 | INFO | Downloaded 34GiB to GPU in 2.03065s (16.7434GiB/s) | 20260505 20:36:10.265 | INFO | Analyzing chunk frames = [6124544,6125056) | 20260505 20:36:10.265 | INFO | MemInfo: { | 20260505 20:36:10.265 | INFO | "gpu" : " 17 GB", | 20260505 20:36:10.265 | INFO | "huge" : "", | 20260505 20:36:10.265 | INFO | "huge_shared" : " 27 GB", | 20260505 20:36:10.265 | INFO | "malloc" : " 14 GB", | 20260505 20:36:10.265 | INFO | "pinned" : " 29 GB" | 20260505 20:36:10.265 | INFO | } | 20260505 20:36:10.265 | INFO | Perf report for the last 5.13471 seconds: | 20260505 20:36:10.265 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:36:10.265 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:10.265 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:10.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:10.265 | INFO | ANALYSIS : 62.1%, 3.0, 96.5ms, 0 | 20260505 20:36:10.265 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:10.265 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:10.265 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:36:10.265 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:10.265 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:10.265 | INFO | Uploaded 36GiB to GPU in 2.1383s (16.8358GiB/s) | 20260505 20:36:10.265 | INFO | Downloaded 34GiB to GPU in 2.0251s (16.7893GiB/s) | 20260505 20:36:15.396 | INFO | Analyzing chunk frames = [6125056,6125568) | 20260505 20:36:15.396 | INFO | MemInfo: { | 20260505 20:36:15.396 | INFO | "gpu" : " 17 GB", | 20260505 20:36:15.396 | INFO | "huge" : "", | 20260505 20:36:15.396 | INFO | "huge_shared" : " 27 GB", | 20260505 20:36:15.396 | INFO | "malloc" : " 14 GB", | 20260505 20:36:15.396 | INFO | "pinned" : " 29 GB" | 20260505 20:36:15.396 | INFO | } | 20260505 20:36:15.396 | INFO | Perf report for the last 5.13145 seconds: | 20260505 20:36:15.396 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:36:15.396 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:15.396 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:15.396 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:15.396 | INFO | ANALYSIS : 62.2%, 3.0, 96.5ms, 0 | 20260505 20:36:15.396 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:15.396 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:15.396 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260505 20:36:15.396 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:15.396 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:15.397 | INFO | Uploaded 36GiB to GPU in 2.13952s (16.8262GiB/s) | 20260505 20:36:15.397 | INFO | Downloaded 34GiB to GPU in 2.04523s (16.624GiB/s) | 20260505 20:36:20.537 | INFO | Analyzing chunk frames = [6125568,6126080) | 20260505 20:36:20.537 | INFO | MemInfo: { | 20260505 20:36:20.537 | INFO | "gpu" : " 17 GB", | 20260505 20:36:20.537 | INFO | "huge" : "", | 20260505 20:36:20.537 | INFO | "huge_shared" : " 27 GB", | 20260505 20:36:20.537 | INFO | "malloc" : " 14 GB", | 20260505 20:36:20.537 | INFO | "pinned" : " 29 GB" | 20260505 20:36:20.537 | INFO | } | 20260505 20:36:20.537 | INFO | Perf report for the last 5.1404 seconds: | 20260505 20:36:20.537 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:36:20.537 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:20.537 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:20.537 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:20.537 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 20:36:20.537 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:20.537 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:20.537 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:36:20.537 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:20.537 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:20.537 | INFO | Uploaded 36GiB to GPU in 2.13986s (16.8235GiB/s) | 20260505 20:36:20.537 | INFO | Downloaded 34GiB to GPU in 2.06224s (16.4869GiB/s) | 20260505 20:36:25.677 | INFO | Analyzing chunk frames = [6126080,6126592) | 20260505 20:36:25.677 | INFO | MemInfo: { | 20260505 20:36:25.677 | INFO | "gpu" : " 17 GB", | 20260505 20:36:25.677 | INFO | "huge" : "", | 20260505 20:36:25.677 | INFO | "huge_shared" : " 27 GB", | 20260505 20:36:25.677 | INFO | "malloc" : " 14 GB", | 20260505 20:36:25.677 | INFO | "pinned" : " 29 GB" | 20260505 20:36:25.677 | INFO | } | 20260505 20:36:25.677 | INFO | Perf report for the last 5.14025 seconds: | 20260505 20:36:25.677 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:36:25.677 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:25.677 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:25.677 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:25.677 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 20:36:25.677 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:25.677 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:25.677 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:36:25.677 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:25.677 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:25.677 | INFO | Uploaded 36GiB to GPU in 2.16336s (16.6408GiB/s) | 20260505 20:36:25.677 | INFO | Downloaded 34GiB to GPU in 2.06577s (16.4588GiB/s) | 20260505 20:36:30.824 | INFO | Analyzing chunk frames = [6126592,6127104) | 20260505 20:36:30.824 | INFO | MemInfo: { | 20260505 20:36:30.824 | INFO | "gpu" : " 17 GB", | 20260505 20:36:30.824 | INFO | "huge" : "", | 20260505 20:36:30.824 | INFO | "huge_shared" : " 27 GB", | 20260505 20:36:30.825 | INFO | "malloc" : " 14 GB", | 20260505 20:36:30.825 | INFO | "pinned" : " 29 GB" | 20260505 20:36:30.825 | INFO | } | 20260505 20:36:30.825 | INFO | Perf report for the last 5.14763 seconds: | 20260505 20:36:30.825 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:36:30.825 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:30.825 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:30.825 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:30.825 | INFO | ANALYSIS : 62.9%, 3.0, 98.4ms, 0 | 20260505 20:36:30.825 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:30.825 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:30.825 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 20:36:30.825 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:30.825 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:30.825 | INFO | Uploaded 36GiB to GPU in 2.12694s (16.9257GiB/s) | 20260505 20:36:30.825 | INFO | Downloaded 34GiB to GPU in 2.14009s (15.8872GiB/s) | 20260505 20:36:35.964 | INFO | Analyzing chunk frames = [6127104,6127616) | 20260505 20:36:35.964 | INFO | MemInfo: { | 20260505 20:36:35.964 | INFO | "gpu" : " 17 GB", | 20260505 20:36:35.964 | INFO | "huge" : "", | 20260505 20:36:35.964 | INFO | "huge_shared" : " 27 GB", | 20260505 20:36:35.964 | INFO | "malloc" : " 14 GB", | 20260505 20:36:35.964 | INFO | "pinned" : " 29 GB" | 20260505 20:36:35.964 | INFO | } | 20260505 20:36:35.965 | INFO | Perf report for the last 5.13992 seconds: | 20260505 20:36:35.965 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:36:35.965 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:35.965 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:35.965 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:35.965 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 20:36:35.965 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:35.965 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:35.965 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:36:35.965 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:35.965 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:35.965 | INFO | Uploaded 36GiB to GPU in 2.11021s (17.0599GiB/s) | 20260505 20:36:35.965 | INFO | Downloaded 34GiB to GPU in 2.01075s (16.9091GiB/s) | 20260505 20:36:35.965 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 20:36:35.965 | INFO | Total mem usage: 15.3 GiB | 20260505 20:36:35.965 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 20:36:35.965 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.12%) | 20260505 20:36:35.965 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 20:36:35.965 | INFO | High Water Breakdown: | 20260505 20:36:35.965 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 20:36:35.965 | INFO | 38.8% : 5.5 GiB ( 82.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 20:36:35.965 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 20:36:35.965 | INFO | ------------------------------------------ | 20260505 20:36:35.965 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:36:35.965 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 20:36:35.965 | INFO | Total mem usage: 26.5 GiB | 20260505 20:36:35.965 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 20:36:35.965 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 20:36:35.965 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 20:36:35.965 | INFO | High Water Breakdown: | 20260505 20:36:35.965 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:36:35.965 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:36:35.965 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:36:35.965 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:36:35.965 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:36:35.965 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:36:35.965 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 20:36:35.965 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 20:36:35.965 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:36:35.965 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 20:36:35.965 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 20:36:35.965 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 20:36:35.966 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 20:36:35.966 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 20:36:35.966 | INFO | ------------------------------------------ | 20260505 20:36:35.966 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:36:35.966 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 20:36:35.966 | INFO | Total mem usage: 25.6 GiB | 20260505 20:36:35.966 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 20:36:35.966 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 20:36:35.966 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 20:36:35.966 | INFO | High Water Breakdown: | 20260505 20:36:35.966 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 20:36:35.966 | INFO | ------------------------------------------ | 20260505 20:36:35.966 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:36:35.966 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 20:36:35.966 | INFO | Total mem usage: 15.7 GiB | 20260505 20:36:35.966 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 20:36:35.966 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 20:36:35.966 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 20:36:35.966 | INFO | High Water Breakdown: | 20260505 20:36:35.966 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 20:36:35.966 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 20:36:35.966 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 20:36:35.966 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 20:36:35.966 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 20:36:35.966 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 20:36:35.966 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 20:36:35.966 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 20:36:35.966 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 20:36:35.966 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:36:35.966 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 20:36:35.966 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:36:35.966 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 20:36:35.966 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 20:36:35.966 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 20:36:35.966 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 20:36:35.966 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:36:35.966 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 20:36:35.966 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:36:35.966 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:36:35.966 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:36:35.966 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 20:36:35.966 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 20:36:35.966 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:36:35.966 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 20:36:35.966 | INFO | ------------------------------------------ | 20260505 20:36:35.966 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:36:35.966 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 20:36:35.966 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:36:35.966 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:36:35.966 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.46e+01, 1.35e+01 / 1.87e+00 | 20260505 20:36:35.966 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.04e-01, 1.04e-02 / 3.68e-03 | 20260505 20:36:35.966 | INFO | PerfReport: 11.79%, Baseline -- 3.71e+00 / 4.55e+00, 5.65e-01 / 1.07e-01 | 20260505 20:36:35.966 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.53e+00, 1.92e-01 / 8.74e-02 | 20260505 20:36:35.966 | INFO | PerfReport: 15.99%, DME -- 4.02e+01 / 4.44e+01, 4.08e+00 / 1.89e+00 | 20260505 20:36:35.966 | INFO | PerfReport: 57.06%, FrameLabeling -- 1.79e+01 / 1.94e+01, 1.47e+00 / 1.15e+00 | 20260505 20:36:35.966 | INFO | PerfReport: 7.22%, PulseAccumulating -- 2.27e+00 / 2.85e+00, 4.34e-01 / 2.13e-01 | 20260505 20:36:35.966 | INFO | PerfReport: 3.24%, Metrics -- 1.02e+00 / 1.34e+00, 2.81e-01 / 1.34e-01 | 20260505 20:36:35.966 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.54e-01 / 4.17e-01, 6.19e-02 / 3.38e-02 | 20260505 20:36:35.966 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 20:36:35.966 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:36:35.966 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:36:35.966 | INFO | PerfReport: Full Execution -- 9.92e+01 / -nan, 2.51e+01 / 0.00e+00 | 20260505 20:36:35.966 | INFO | PerfReport: 22.41%, Upload -- 2.22e+01 / 0.00e+00, 7.70e+00 / 0.00e+00 | 20260505 20:36:35.966 | INFO | PerfReport: 21.79%, Download -- 2.16e+01 / 0.00e+00, 7.70e+00 / 0.00e+00 | 20260505 20:36:35.966 | INFO | PerfReport: 31.70%, Compute -- 3.15e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 20:36:35.966 | INFO | PerfReport: 2.26%, LockUpload -- 2.25e+00 / 0.00e+00, 8.03e+00 / 0.00e+00 | 20260505 20:36:35.966 | INFO | PerfReport: 1.35%, LockDownload -- 1.34e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260505 20:36:35.966 | INFO | PerfReport: 20.48%, LockCompute -- 2.03e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 20:36:35.966 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.37e-02 / 0.00e+00 | 20260505 20:36:36.097 | INFO | Starting a round of Baz IO | 20260505 20:36:39.265 | INFO | Populating RTMetrics report | 20260505 20:36:39.266 | INFO | RTMetrics report populated | 20260505 20:36:39.266 | INFO | Opening tmp RTMetrics json file | 20260505 20:36:39.267 | INFO | Serializing RTMetrics to json | 20260505 20:36:39.268 | INFO | Serialization done, swapping tmp file now... | 20260505 20:36:39.269 | INFO | RTMetrics done | 20260505 20:36:39.352 | INFO | Finished a round of Baz IO: Wrote 2.007 GiB in 3.254 seconds | 20260505 20:36:39.352 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.65% / 0.00% / 78.35% | 20260505 20:36:39.352 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.170573% / 3.97364e-06% / 7.94729e-05% | 20260505 20:36:39.352 | INFO | Baz IO overall: | 20260505 20:36:39.352 | INFO | BytesWritten : 3567131606564 | 20260505 20:36:39.352 | INFO | NumEvents : 1685975038871 | 20260505 20:36:41.094 | INFO | Analyzing chunk frames = [6127616,6128128) | 20260505 20:36:41.094 | INFO | MemInfo: { | 20260505 20:36:41.094 | INFO | "gpu" : " 17 GB", | 20260505 20:36:41.094 | INFO | "huge" : "", | 20260505 20:36:41.094 | INFO | "huge_shared" : " 27 GB", | 20260505 20:36:41.094 | INFO | "malloc" : " 14 GB", | 20260505 20:36:41.094 | INFO | "pinned" : " 29 GB" | 20260505 20:36:41.094 | INFO | } | 20260505 20:36:41.094 | INFO | Perf report for the last 5.1298 seconds: | 20260505 20:36:41.095 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:36:41.095 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:41.095 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:41.095 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:41.095 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 20:36:41.095 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:41.095 | INFO | RT_METRICS : 44.0%, 1.0, 23.0ms, 0 | 20260505 20:36:41.095 | INFO | PRE_HQ : 60.3%, 1.8, 55.1ms, 0 | 20260505 20:36:41.095 | INFO | BAZWRITER : 63.0%, 3.7, 119.9ms, 0 | 20260505 20:36:41.095 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 20:36:41.095 | INFO | Uploaded 36GiB to GPU in 2.16272s (16.6457GiB/s) | 20260505 20:36:41.095 | INFO | Downloaded 38GiB to GPU in 2.2508s (16.8829GiB/s) | 20260505 20:36:46.239 | INFO | Analyzing chunk frames = [6128128,6128640) | 20260505 20:36:46.239 | INFO | MemInfo: { | 20260505 20:36:46.239 | INFO | "gpu" : " 17 GB", | 20260505 20:36:46.239 | INFO | "huge" : "", | 20260505 20:36:46.239 | INFO | "huge_shared" : " 27 GB", | 20260505 20:36:46.239 | INFO | "malloc" : " 14 GB", | 20260505 20:36:46.239 | INFO | "pinned" : " 29 GB" | 20260505 20:36:46.239 | INFO | } | 20260505 20:36:46.239 | INFO | Perf report for the last 5.14458 seconds: | 20260505 20:36:46.239 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:36:46.239 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:46.239 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:46.239 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:46.239 | INFO | ANALYSIS : 62.5%, 3.0, 97.7ms, 0 | 20260505 20:36:46.239 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:46.239 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:46.239 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 20:36:46.239 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:46.239 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:46.239 | INFO | Uploaded 36GiB to GPU in 2.15932s (16.6719GiB/s) | 20260505 20:36:46.239 | INFO | Downloaded 34GiB to GPU in 2.05787s (16.5219GiB/s) | 20260505 20:36:51.378 | INFO | Analyzing chunk frames = [6128640,6129152) | 20260505 20:36:51.378 | INFO | MemInfo: { | 20260505 20:36:51.378 | INFO | "gpu" : " 17 GB", | 20260505 20:36:51.378 | INFO | "huge" : "", | 20260505 20:36:51.378 | INFO | "huge_shared" : " 27 GB", | 20260505 20:36:51.378 | INFO | "malloc" : " 14 GB", | 20260505 20:36:51.378 | INFO | "pinned" : " 29 GB" | 20260505 20:36:51.378 | INFO | } | 20260505 20:36:51.379 | INFO | Perf report for the last 5.13952 seconds: | 20260505 20:36:51.379 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:36:51.379 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:51.379 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:51.379 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:51.379 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260505 20:36:51.379 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:51.379 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:51.379 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:36:51.379 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:51.379 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:51.379 | INFO | Uploaded 36GiB to GPU in 2.12799s (16.9174GiB/s) | 20260505 20:36:51.379 | INFO | Downloaded 34GiB to GPU in 2.02435s (16.7955GiB/s) | 20260505 20:36:56.512 | INFO | Analyzing chunk frames = [6129152,6129664) | 20260505 20:36:56.512 | INFO | MemInfo: { | 20260505 20:36:56.512 | INFO | "gpu" : " 17 GB", | 20260505 20:36:56.512 | INFO | "huge" : "", | 20260505 20:36:56.512 | INFO | "huge_shared" : " 27 GB", | 20260505 20:36:56.512 | INFO | "malloc" : " 14 GB", | 20260505 20:36:56.512 | INFO | "pinned" : " 29 GB" | 20260505 20:36:56.512 | INFO | } | 20260505 20:36:56.512 | INFO | Perf report for the last 5.13372 seconds: | 20260505 20:36:56.512 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:36:56.512 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:56.512 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:56.512 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:56.512 | INFO | ANALYSIS : 62.1%, 3.0, 96.4ms, 0 | 20260505 20:36:56.512 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:56.512 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:56.512 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:36:56.512 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:56.512 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:36:56.513 | INFO | Uploaded 36GiB to GPU in 2.15372s (16.7153GiB/s) | 20260505 20:36:56.513 | INFO | Downloaded 34GiB to GPU in 2.04388s (16.635GiB/s) | 20260505 20:37:01.649 | INFO | Analyzing chunk frames = [6129664,6130176) | 20260505 20:37:01.649 | INFO | MemInfo: { | 20260505 20:37:01.649 | INFO | "gpu" : " 17 GB", | 20260505 20:37:01.649 | INFO | "huge" : "", | 20260505 20:37:01.649 | INFO | "huge_shared" : " 27 GB", | 20260505 20:37:01.649 | INFO | "malloc" : " 14 GB", | 20260505 20:37:01.649 | INFO | "pinned" : " 29 GB" | 20260505 20:37:01.649 | INFO | } | 20260505 20:37:01.649 | INFO | Perf report for the last 5.1369 seconds: | 20260505 20:37:01.649 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:37:01.649 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:01.649 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:01.649 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:01.649 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 20:37:01.649 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:01.649 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:01.649 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:37:01.649 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:01.649 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:01.650 | INFO | Uploaded 36GiB to GPU in 2.14123s (16.8127GiB/s) | 20260505 20:37:01.650 | INFO | Downloaded 34GiB to GPU in 2.06276s (16.4828GiB/s) | 20260505 20:37:06.791 | INFO | Analyzing chunk frames = [6130176,6130688) | 20260505 20:37:06.791 | INFO | MemInfo: { | 20260505 20:37:06.791 | INFO | "gpu" : " 17 GB", | 20260505 20:37:06.791 | INFO | "huge" : "", | 20260505 20:37:06.791 | INFO | "huge_shared" : " 27 GB", | 20260505 20:37:06.791 | INFO | "malloc" : " 14 GB", | 20260505 20:37:06.791 | INFO | "pinned" : " 29 GB" | 20260505 20:37:06.791 | INFO | } | 20260505 20:37:06.791 | INFO | Perf report for the last 5.14197 seconds: | 20260505 20:37:06.791 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:37:06.791 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:06.791 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:06.791 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:06.791 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260505 20:37:06.791 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:06.791 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:06.791 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:37:06.791 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:06.791 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:06.791 | INFO | Uploaded 36GiB to GPU in 2.14565s (16.7782GiB/s) | 20260505 20:37:06.791 | INFO | Downloaded 34GiB to GPU in 2.05832s (16.5184GiB/s) | 20260505 20:37:11.936 | INFO | Analyzing chunk frames = [6130688,6131200) | 20260505 20:37:11.936 | INFO | MemInfo: { | 20260505 20:37:11.936 | INFO | "gpu" : " 17 GB", | 20260505 20:37:11.936 | INFO | "huge" : "", | 20260505 20:37:11.936 | INFO | "huge_shared" : " 27 GB", | 20260505 20:37:11.936 | INFO | "malloc" : " 14 GB", | 20260505 20:37:11.936 | INFO | "pinned" : " 29 GB" | 20260505 20:37:11.936 | INFO | } | 20260505 20:37:11.936 | INFO | Perf report for the last 5.14501 seconds: | 20260505 20:37:11.936 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:37:11.936 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:11.936 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:11.936 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:11.936 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 20:37:11.936 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:11.936 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:11.936 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 20:37:11.936 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:11.936 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:11.936 | INFO | Uploaded 36GiB to GPU in 2.11428s (17.0271GiB/s) | 20260505 20:37:11.937 | INFO | Downloaded 34GiB to GPU in 2.13795s (15.9031GiB/s) | 20260505 20:37:17.076 | INFO | Analyzing chunk frames = [6131200,6131712) | 20260505 20:37:17.076 | INFO | MemInfo: { | 20260505 20:37:17.076 | INFO | "gpu" : " 17 GB", | 20260505 20:37:17.076 | INFO | "huge" : "", | 20260505 20:37:17.076 | INFO | "huge_shared" : " 27 GB", | 20260505 20:37:17.076 | INFO | "malloc" : " 14 GB", | 20260505 20:37:17.076 | INFO | "pinned" : " 29 GB" | 20260505 20:37:17.076 | INFO | } | 20260505 20:37:17.076 | INFO | Perf report for the last 5.14003 seconds: | 20260505 20:37:17.076 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:37:17.076 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:17.076 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:17.076 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:17.076 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 20:37:17.076 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:17.076 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:17.076 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:37:17.076 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:17.076 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:17.076 | INFO | Uploaded 36GiB to GPU in 2.11379s (17.0311GiB/s) | 20260505 20:37:17.076 | INFO | Downloaded 34GiB to GPU in 1.99885s (17.0098GiB/s) | 20260505 20:37:17.211 | INFO | Starting a round of Baz IO | 20260505 20:37:20.365 | INFO | Populating RTMetrics report | 20260505 20:37:20.366 | INFO | RTMetrics report populated | 20260505 20:37:20.366 | INFO | Opening tmp RTMetrics json file | 20260505 20:37:20.366 | INFO | Serializing RTMetrics to json | 20260505 20:37:20.368 | INFO | Serialization done, swapping tmp file now... | 20260505 20:37:20.368 | INFO | RTMetrics done | 20260505 20:37:20.454 | INFO | Finished a round of Baz IO: Wrote 2.005 GiB in 3.242 seconds | 20260505 20:37:20.454 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.65% / 0.00% / 78.35% | 20260505 20:37:20.454 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.170358% / 0% / 0.000147025% | 20260505 20:37:20.454 | INFO | Baz IO overall: | 20260505 20:37:20.454 | INFO | BytesWritten : 3569284890563 | 20260505 20:37:20.454 | INFO | NumEvents : 1686972043433 | 20260505 20:37:22.210 | INFO | Analyzing chunk frames = [6131712,6132224) | 20260505 20:37:22.210 | INFO | MemInfo: { | 20260505 20:37:22.210 | INFO | "gpu" : " 17 GB", | 20260505 20:37:22.210 | INFO | "huge" : "", | 20260505 20:37:22.210 | INFO | "huge_shared" : " 27 GB", | 20260505 20:37:22.210 | INFO | "malloc" : " 14 GB", | 20260505 20:37:22.210 | INFO | "pinned" : " 29 GB" | 20260505 20:37:22.210 | INFO | } | 20260505 20:37:22.210 | INFO | Perf report for the last 5.13373 seconds: | 20260505 20:37:22.210 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:37:22.210 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:22.210 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:22.210 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:22.210 | INFO | ANALYSIS : 63.0%, 3.0, 98.1ms, 0 | 20260505 20:37:22.210 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:22.210 | INFO | RT_METRICS : 44.0%, 1.0, 23.0ms, 0 | 20260505 20:37:22.210 | INFO | PRE_HQ : 60.0%, 1.8, 55.1ms, 0 | 20260505 20:37:22.210 | INFO | BAZWRITER : 62.7%, 3.7, 119.6ms, 0 | 20260505 20:37:22.210 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 20:37:22.210 | INFO | Uploaded 36GiB to GPU in 2.15606s (16.6971GiB/s) | 20260505 20:37:22.210 | INFO | Downloaded 38GiB to GPU in 2.24699s (16.9115GiB/s) | 20260505 20:37:27.344 | INFO | Analyzing chunk frames = [6132224,6132736) | 20260505 20:37:27.344 | INFO | MemInfo: { | 20260505 20:37:27.344 | INFO | "gpu" : " 17 GB", | 20260505 20:37:27.344 | INFO | "huge" : "", | 20260505 20:37:27.344 | INFO | "huge_shared" : " 27 GB", | 20260505 20:37:27.344 | INFO | "malloc" : " 14 GB", | 20260505 20:37:27.344 | INFO | "pinned" : " 29 GB" | 20260505 20:37:27.344 | INFO | } | 20260505 20:37:27.344 | INFO | Perf report for the last 5.13438 seconds: | 20260505 20:37:27.344 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:37:27.344 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:27.344 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:27.344 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:27.344 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 20:37:27.344 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:27.344 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:27.344 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 20:37:27.344 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:27.344 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:27.345 | INFO | Uploaded 36GiB to GPU in 2.14408s (16.7904GiB/s) | 20260505 20:37:27.345 | INFO | Downloaded 34GiB to GPU in 2.04915s (16.5922GiB/s) | 20260505 20:37:32.491 | INFO | Analyzing chunk frames = [6132736,6133248) | 20260505 20:37:32.491 | INFO | MemInfo: { | 20260505 20:37:32.491 | INFO | "gpu" : " 17 GB", | 20260505 20:37:32.491 | INFO | "huge" : "", | 20260505 20:37:32.491 | INFO | "huge_shared" : " 27 GB", | 20260505 20:37:32.491 | INFO | "malloc" : " 14 GB", | 20260505 20:37:32.491 | INFO | "pinned" : " 29 GB" | 20260505 20:37:32.491 | INFO | } | 20260505 20:37:32.491 | INFO | Perf report for the last 5.14647 seconds: | 20260505 20:37:32.491 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:37:32.491 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:32.491 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:32.491 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:32.491 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260505 20:37:32.491 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:32.491 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:32.491 | INFO | PRE_HQ : 27.2%, 1.0, 14.2ms, 0 | 20260505 20:37:32.491 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:32.491 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:32.491 | INFO | Uploaded 36GiB to GPU in 2.14223s (16.8049GiB/s) | 20260505 20:37:32.491 | INFO | Downloaded 34GiB to GPU in 2.03679s (16.6929GiB/s) | 20260505 20:37:37.630 | INFO | Analyzing chunk frames = [6133248,6133760) | 20260505 20:37:37.630 | INFO | MemInfo: { | 20260505 20:37:37.630 | INFO | "gpu" : " 17 GB", | 20260505 20:37:37.630 | INFO | "huge" : "", | 20260505 20:37:37.630 | INFO | "huge_shared" : " 27 GB", | 20260505 20:37:37.630 | INFO | "malloc" : " 14 GB", | 20260505 20:37:37.630 | INFO | "pinned" : " 29 GB" | 20260505 20:37:37.630 | INFO | } | 20260505 20:37:37.630 | INFO | Perf report for the last 5.13953 seconds: | 20260505 20:37:37.630 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:37:37.630 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:37.630 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:37.630 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:37.630 | INFO | ANALYSIS : 62.4%, 3.0, 97.0ms, 0 | 20260505 20:37:37.630 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:37.630 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:37.630 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:37:37.630 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:37.631 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:37.631 | INFO | Uploaded 36GiB to GPU in 2.20613s (16.3182GiB/s) | 20260505 20:37:37.631 | INFO | Downloaded 34GiB to GPU in 2.03749s (16.6872GiB/s) | 20260505 20:37:42.771 | INFO | Analyzing chunk frames = [6133760,6134272) | 20260505 20:37:42.771 | INFO | MemInfo: { | 20260505 20:37:42.771 | INFO | "gpu" : " 17 GB", | 20260505 20:37:42.771 | INFO | "huge" : "", | 20260505 20:37:42.771 | INFO | "huge_shared" : " 27 GB", | 20260505 20:37:42.771 | INFO | "malloc" : " 14 GB", | 20260505 20:37:42.771 | INFO | "pinned" : " 29 GB" | 20260505 20:37:42.771 | INFO | } | 20260505 20:37:42.771 | INFO | Perf report for the last 5.14055 seconds: | 20260505 20:37:42.771 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:37:42.771 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:42.771 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:42.771 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:42.771 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 20:37:42.771 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:42.771 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:42.771 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:37:42.771 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:42.771 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:42.771 | INFO | Uploaded 36GiB to GPU in 2.1414s (16.8114GiB/s) | 20260505 20:37:42.771 | INFO | Downloaded 34GiB to GPU in 2.06565s (16.4597GiB/s) | 20260505 20:37:47.911 | INFO | Analyzing chunk frames = [6134272,6134784) | 20260505 20:37:47.911 | INFO | MemInfo: { | 20260505 20:37:47.911 | INFO | "gpu" : " 17 GB", | 20260505 20:37:47.911 | INFO | "huge" : "", | 20260505 20:37:47.911 | INFO | "huge_shared" : " 27 GB", | 20260505 20:37:47.911 | INFO | "malloc" : " 14 GB", | 20260505 20:37:47.911 | INFO | "pinned" : " 29 GB" | 20260505 20:37:47.911 | INFO | } | 20260505 20:37:47.911 | INFO | Perf report for the last 5.13984 seconds: | 20260505 20:37:47.911 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:37:47.911 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:47.911 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:47.911 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:47.911 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 20:37:47.911 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:47.911 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:47.911 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:37:47.911 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:47.911 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:47.911 | INFO | Uploaded 36GiB to GPU in 2.14998s (16.7443GiB/s) | 20260505 20:37:47.911 | INFO | Downloaded 34GiB to GPU in 2.06094s (16.4973GiB/s) | 20260505 20:37:53.044 | INFO | Analyzing chunk frames = [6134784,6135296) | 20260505 20:37:53.044 | INFO | MemInfo: { | 20260505 20:37:53.044 | INFO | "gpu" : " 17 GB", | 20260505 20:37:53.044 | INFO | "huge" : "", | 20260505 20:37:53.044 | INFO | "huge_shared" : " 27 GB", | 20260505 20:37:53.044 | INFO | "malloc" : " 14 GB", | 20260505 20:37:53.044 | INFO | "pinned" : " 29 GB" | 20260505 20:37:53.044 | INFO | } | 20260505 20:37:53.044 | INFO | Perf report for the last 5.13335 seconds: | 20260505 20:37:53.044 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:37:53.044 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:53.044 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:53.044 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:53.044 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 20:37:53.044 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:53.044 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:53.044 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 20:37:53.044 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:53.044 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:53.044 | INFO | Uploaded 36GiB to GPU in 2.12206s (16.9647GiB/s) | 20260505 20:37:53.044 | INFO | Downloaded 34GiB to GPU in 2.13704s (15.9099GiB/s) | 20260505 20:37:58.188 | INFO | Analyzing chunk frames = [6135296,6135808) | 20260505 20:37:58.188 | INFO | MemInfo: { | 20260505 20:37:58.188 | INFO | "gpu" : " 17 GB", | 20260505 20:37:58.188 | INFO | "huge" : "", | 20260505 20:37:58.188 | INFO | "huge_shared" : " 27 GB", | 20260505 20:37:58.188 | INFO | "malloc" : " 14 GB", | 20260505 20:37:58.188 | INFO | "pinned" : " 29 GB" | 20260505 20:37:58.188 | INFO | } | 20260505 20:37:58.188 | INFO | Perf report for the last 5.14401 seconds: | 20260505 20:37:58.188 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:37:58.188 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:58.188 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:58.188 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:58.188 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 20:37:58.188 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:58.188 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:58.188 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:37:58.188 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:58.188 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:37:58.188 | INFO | Uploaded 36GiB to GPU in 2.20644s (16.3159GiB/s) | 20260505 20:37:58.188 | INFO | Downloaded 34GiB to GPU in 2.04778s (16.6034GiB/s) | 20260505 20:37:58.189 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 20:37:58.189 | INFO | Total mem usage: 15.3 GiB | 20260505 20:37:58.189 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 20:37:58.189 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.3%) | 20260505 20:37:58.189 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 20:37:58.189 | INFO | High Water Breakdown: | 20260505 20:37:58.189 | INFO | 61% : 8.64 GiB ( 90.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 20:37:58.189 | INFO | 38.8% : 5.49 GiB ( 82.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 20:37:58.189 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 20:37:58.189 | INFO | ------------------------------------------ | 20260505 20:37:58.189 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:37:58.189 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 20:37:58.189 | INFO | Total mem usage: 26.5 GiB | 20260505 20:37:58.189 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 20:37:58.189 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 20:37:58.189 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 20:37:58.189 | INFO | High Water Breakdown: | 20260505 20:37:58.189 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:37:58.189 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:37:58.189 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:37:58.189 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:37:58.189 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:37:58.189 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:37:58.189 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 20:37:58.189 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 20:37:58.189 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:37:58.189 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 20:37:58.189 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 20:37:58.189 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 20:37:58.189 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 20:37:58.189 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 20:37:58.189 | INFO | ------------------------------------------ | 20260505 20:37:58.189 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:37:58.189 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 20:37:58.189 | INFO | Total mem usage: 25.6 GiB | 20260505 20:37:58.189 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 20:37:58.189 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 20:37:58.189 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 20:37:58.189 | INFO | High Water Breakdown: | 20260505 20:37:58.189 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 20:37:58.189 | INFO | ------------------------------------------ | 20260505 20:37:58.189 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:37:58.189 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 20:37:58.189 | INFO | Total mem usage: 15.7 GiB | 20260505 20:37:58.189 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 20:37:58.189 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 20:37:58.189 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 20:37:58.189 | INFO | High Water Breakdown: | 20260505 20:37:58.189 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 20:37:58.189 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 20:37:58.189 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 20:37:58.189 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 20:37:58.189 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 20:37:58.189 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 20:37:58.189 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 20:37:58.189 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 20:37:58.189 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 20:37:58.189 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:37:58.189 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 20:37:58.189 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:37:58.189 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 20:37:58.189 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 20:37:58.189 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 20:37:58.189 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 20:37:58.189 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:37:58.189 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 20:37:58.189 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:37:58.189 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:37:58.189 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:37:58.189 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 20:37:58.189 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 20:37:58.189 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:37:58.189 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 20:37:58.189 | INFO | ------------------------------------------ | 20260505 20:37:58.189 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:37:58.189 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 20:37:58.189 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:37:58.189 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:37:58.189 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.45e+01, 1.35e+01 / 1.80e+00 | 20260505 20:37:58.189 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.02e-01, 1.04e-02 / 3.84e-03 | 20260505 20:37:58.189 | INFO | PerfReport: 11.79%, Baseline -- 3.70e+00 / 4.53e+00, 5.61e-01 / 1.65e-01 | 20260505 20:37:58.189 | INFO | PerfReport: 3.26%, Histogram -- 1.03e+00 / 1.50e+00, 1.87e-01 / 1.10e-01 | 20260505 20:37:58.189 | INFO | PerfReport: 15.97%, DME -- 4.02e+01 / 4.43e+01, 4.09e+00 / 1.89e+00 | 20260505 20:37:58.189 | INFO | PerfReport: 57.08%, FrameLabeling -- 1.79e+01 / 1.94e+01, 1.46e+00 / 9.80e-01 | 20260505 20:37:58.189 | INFO | PerfReport: 7.25%, PulseAccumulating -- 2.28e+00 / 2.85e+00, 4.38e-01 / 2.24e-01 | 20260505 20:37:58.189 | INFO | PerfReport: 3.23%, Metrics -- 1.02e+00 / 1.33e+00, 2.77e-01 / 1.27e-01 | 20260505 20:37:58.189 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.54e-01 / 4.21e-01, 6.23e-02 / 3.81e-02 | 20260505 20:37:58.189 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 20:37:58.189 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:37:58.189 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:37:58.189 | INFO | PerfReport: Full Execution -- 9.94e+01 / -nan, 2.54e+01 / 0.00e+00 | 20260505 20:37:58.189 | INFO | PerfReport: 22.52%, Upload -- 2.24e+01 / 0.00e+00, 7.82e+00 / 0.00e+00 | 20260505 20:37:58.189 | INFO | PerfReport: 21.82%, Download -- 2.17e+01 / 0.00e+00, 7.77e+00 / 0.00e+00 | 20260505 20:37:58.189 | INFO | PerfReport: 31.63%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 20:37:58.189 | INFO | PerfReport: 2.32%, LockUpload -- 2.30e+00 / 0.00e+00, 8.07e+00 / 0.00e+00 | 20260505 20:37:58.189 | INFO | PerfReport: 1.37%, LockDownload -- 1.36e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260505 20:37:58.189 | INFO | PerfReport: 20.33%, LockCompute -- 2.02e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 20:37:58.189 | INFO | PerfReport: 0.01%, Misc -- 1.26e-02 / 0.00e+00, 1.25e-02 / 0.00e+00 | 20260505 20:37:58.322 | INFO | Starting a round of Baz IO | 20260505 20:38:01.484 | INFO | Populating RTMetrics report | 20260505 20:38:01.484 | INFO | RTMetrics report populated | 20260505 20:38:01.484 | INFO | Opening tmp RTMetrics json file | 20260505 20:38:01.485 | INFO | Serializing RTMetrics to json | 20260505 20:38:01.486 | INFO | Serialization done, swapping tmp file now... | 20260505 20:38:01.487 | INFO | RTMetrics done | 20260505 20:38:01.576 | INFO | Finished a round of Baz IO: Wrote 2.006 GiB in 3.253 seconds | 20260505 20:38:01.576 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.65% / 0.00% / 78.35% | 20260505 20:38:01.576 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.169905% / 0% / 0.000115236% | 20260505 20:38:01.576 | INFO | Baz IO overall: | 20260505 20:38:01.576 | INFO | BytesWritten : 3571438281039 | 20260505 20:38:01.576 | INFO | NumEvents : 1687969272030 | 20260505 20:38:03.338 | INFO | Analyzing chunk frames = [6135808,6136320) | 20260505 20:38:03.338 | INFO | MemInfo: { | 20260505 20:38:03.338 | INFO | "gpu" : " 17 GB", | 20260505 20:38:03.338 | INFO | "huge" : "", | 20260505 20:38:03.338 | INFO | "huge_shared" : " 27 GB", | 20260505 20:38:03.338 | INFO | "malloc" : " 14 GB", | 20260505 20:38:03.338 | INFO | "pinned" : " 29 GB" | 20260505 20:38:03.338 | INFO | } | 20260505 20:38:03.338 | INFO | Perf report for the last 5.14948 seconds: | 20260505 20:38:03.338 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:38:03.338 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:03.338 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:03.338 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:03.338 | INFO | ANALYSIS : 62.9%, 3.0, 98.4ms, 0 | 20260505 20:38:03.338 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:03.338 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260505 20:38:03.338 | INFO | PRE_HQ : 59.9%, 1.8, 55.2ms, 0 | 20260505 20:38:03.338 | INFO | BAZWRITER : 62.7%, 3.7, 119.7ms, 0 | 20260505 20:38:03.338 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 20:38:03.338 | INFO | Uploaded 36GiB to GPU in 2.17621s (16.5425GiB/s) | 20260505 20:38:03.338 | INFO | Downloaded 38GiB to GPU in 2.25751s (16.8327GiB/s) | 20260505 20:38:08.460 | INFO | Analyzing chunk frames = [6136320,6136832) | 20260505 20:38:08.460 | INFO | MemInfo: { | 20260505 20:38:08.460 | INFO | "gpu" : " 17 GB", | 20260505 20:38:08.460 | INFO | "huge" : "", | 20260505 20:38:08.460 | INFO | "huge_shared" : " 27 GB", | 20260505 20:38:08.460 | INFO | "malloc" : " 14 GB", | 20260505 20:38:08.460 | INFO | "pinned" : " 29 GB" | 20260505 20:38:08.460 | INFO | } | 20260505 20:38:08.461 | INFO | Perf report for the last 5.12284 seconds: | 20260505 20:38:08.461 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:38:08.461 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:08.461 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:08.461 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:08.461 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260505 20:38:08.461 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:08.461 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:08.461 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 20:38:08.461 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:08.461 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:08.461 | INFO | Uploaded 36GiB to GPU in 2.07799s (17.3244GiB/s) | 20260505 20:38:08.461 | INFO | Downloaded 34GiB to GPU in 2.02458s (16.7936GiB/s) | 20260505 20:38:13.597 | INFO | Analyzing chunk frames = [6136832,6137344) | 20260505 20:38:13.597 | INFO | MemInfo: { | 20260505 20:38:13.597 | INFO | "gpu" : " 17 GB", | 20260505 20:38:13.597 | INFO | "huge" : "", | 20260505 20:38:13.597 | INFO | "huge_shared" : " 27 GB", | 20260505 20:38:13.597 | INFO | "malloc" : " 14 GB", | 20260505 20:38:13.597 | INFO | "pinned" : " 29 GB" | 20260505 20:38:13.598 | INFO | } | 20260505 20:38:13.598 | INFO | Perf report for the last 5.13705 seconds: | 20260505 20:38:13.598 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:38:13.598 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:13.598 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:13.598 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:13.598 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260505 20:38:13.598 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:13.598 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:13.598 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:38:13.598 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:13.598 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:13.598 | INFO | Uploaded 36GiB to GPU in 2.14617s (16.7741GiB/s) | 20260505 20:38:13.598 | INFO | Downloaded 34GiB to GPU in 2.04658s (16.6131GiB/s) | 20260505 20:38:18.747 | INFO | Analyzing chunk frames = [6137344,6137856) | 20260505 20:38:18.747 | INFO | MemInfo: { | 20260505 20:38:18.747 | INFO | "gpu" : " 17 GB", | 20260505 20:38:18.747 | INFO | "huge" : "", | 20260505 20:38:18.747 | INFO | "huge_shared" : " 27 GB", | 20260505 20:38:18.747 | INFO | "malloc" : " 14 GB", | 20260505 20:38:18.747 | INFO | "pinned" : " 29 GB" | 20260505 20:38:18.747 | INFO | } | 20260505 20:38:18.747 | INFO | Perf report for the last 5.14976 seconds: | 20260505 20:38:18.747 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:38:18.747 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:18.747 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:18.747 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:18.747 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260505 20:38:18.747 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:18.747 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:18.747 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:38:18.747 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:18.748 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:18.748 | INFO | Uploaded 36GiB to GPU in 2.16246s (16.6477GiB/s) | 20260505 20:38:18.748 | INFO | Downloaded 34GiB to GPU in 2.0666s (16.4521GiB/s) | 20260505 20:38:23.890 | INFO | Analyzing chunk frames = [6137856,6138368) | 20260505 20:38:23.890 | INFO | MemInfo: { | 20260505 20:38:23.890 | INFO | "gpu" : " 17 GB", | 20260505 20:38:23.890 | INFO | "huge" : "", | 20260505 20:38:23.890 | INFO | "huge_shared" : " 27 GB", | 20260505 20:38:23.890 | INFO | "malloc" : " 14 GB", | 20260505 20:38:23.890 | INFO | "pinned" : " 29 GB" | 20260505 20:38:23.890 | INFO | } | 20260505 20:38:23.890 | INFO | Perf report for the last 5.1427 seconds: | 20260505 20:38:23.890 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:38:23.890 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:23.890 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:23.890 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:23.890 | INFO | ANALYSIS : 61.3%, 3.0, 95.8ms, 0 | 20260505 20:38:23.890 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:23.890 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:23.890 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:38:23.890 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:23.890 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:23.890 | INFO | Uploaded 36GiB to GPU in 2.09013s (17.2238GiB/s) | 20260505 20:38:23.890 | INFO | Downloaded 34GiB to GPU in 2.0445s (16.63GiB/s) | 20260505 20:38:29.021 | INFO | Analyzing chunk frames = [6138368,6138880) | 20260505 20:38:29.021 | INFO | MemInfo: { | 20260505 20:38:29.021 | INFO | "gpu" : " 17 GB", | 20260505 20:38:29.021 | INFO | "huge" : "", | 20260505 20:38:29.021 | INFO | "huge_shared" : " 27 GB", | 20260505 20:38:29.021 | INFO | "malloc" : " 14 GB", | 20260505 20:38:29.021 | INFO | "pinned" : " 29 GB" | 20260505 20:38:29.021 | INFO | } | 20260505 20:38:29.021 | INFO | Perf report for the last 5.13097 seconds: | 20260505 20:38:29.021 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:38:29.021 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:29.021 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:29.021 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:29.021 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260505 20:38:29.021 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:29.021 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:29.021 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:38:29.021 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:29.021 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:29.021 | INFO | Uploaded 36GiB to GPU in 2.08303s (17.2825GiB/s) | 20260505 20:38:29.021 | INFO | Downloaded 34GiB to GPU in 2.039s (16.6748GiB/s) | 20260505 20:38:34.155 | INFO | Analyzing chunk frames = [6138880,6139392) | 20260505 20:38:34.155 | INFO | MemInfo: { | 20260505 20:38:34.155 | INFO | "gpu" : " 17 GB", | 20260505 20:38:34.155 | INFO | "huge" : "", | 20260505 20:38:34.155 | INFO | "huge_shared" : " 27 GB", | 20260505 20:38:34.155 | INFO | "malloc" : " 14 GB", | 20260505 20:38:34.155 | INFO | "pinned" : " 29 GB" | 20260505 20:38:34.155 | INFO | } | 20260505 20:38:34.155 | INFO | Perf report for the last 5.13379 seconds: | 20260505 20:38:34.155 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:38:34.155 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:34.155 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:34.155 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:34.155 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260505 20:38:34.155 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:34.155 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:34.155 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 20:38:34.155 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:34.155 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:34.155 | INFO | Uploaded 36GiB to GPU in 2.1066s (17.0891GiB/s) | 20260505 20:38:34.155 | INFO | Downloaded 34GiB to GPU in 2.15089s (15.8074GiB/s) | 20260505 20:38:39.296 | INFO | Analyzing chunk frames = [6139392,6139904) | 20260505 20:38:39.296 | INFO | MemInfo: { | 20260505 20:38:39.296 | INFO | "gpu" : " 17 GB", | 20260505 20:38:39.296 | INFO | "huge" : "", | 20260505 20:38:39.296 | INFO | "huge_shared" : " 27 GB", | 20260505 20:38:39.296 | INFO | "malloc" : " 14 GB", | 20260505 20:38:39.296 | INFO | "pinned" : " 29 GB" | 20260505 20:38:39.296 | INFO | } | 20260505 20:38:39.296 | INFO | Perf report for the last 5.14113 seconds: | 20260505 20:38:39.296 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:38:39.296 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:39.296 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:39.296 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:39.296 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 20:38:39.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:39.296 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:39.296 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:38:39.296 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:39.296 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:39.296 | INFO | Uploaded 36GiB to GPU in 2.20475s (16.3284GiB/s) | 20260505 20:38:39.296 | INFO | Downloaded 34GiB to GPU in 2.03639s (16.6962GiB/s) | 20260505 20:38:39.428 | INFO | Starting a round of Baz IO | 20260505 20:38:42.592 | INFO | Populating RTMetrics report | 20260505 20:38:42.593 | INFO | RTMetrics report populated | 20260505 20:38:42.593 | INFO | Opening tmp RTMetrics json file | 20260505 20:38:42.596 | INFO | Serializing RTMetrics to json | 20260505 20:38:42.598 | INFO | Serialization done, swapping tmp file now... | 20260505 20:38:42.598 | INFO | RTMetrics done | 20260505 20:38:42.681 | INFO | Finished a round of Baz IO: Wrote 2.005 GiB in 3.252 seconds | 20260505 20:38:42.681 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.65% / 0.00% / 78.35% | 20260505 20:38:42.681 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.170382% / 0% / 7.94729e-05% | 20260505 20:38:42.681 | INFO | Baz IO overall: | 20260505 20:38:42.681 | INFO | BytesWritten : 3573591579429 | 20260505 20:38:42.681 | INFO | NumEvents : 1688966225007 | 20260505 20:38:44.441 | INFO | Analyzing chunk frames = [6139904,6140416) | 20260505 20:38:44.441 | INFO | MemInfo: { | 20260505 20:38:44.441 | INFO | "gpu" : " 17 GB", | 20260505 20:38:44.441 | INFO | "huge" : "", | 20260505 20:38:44.442 | INFO | "huge_shared" : " 27 GB", | 20260505 20:38:44.442 | INFO | "malloc" : " 14 GB", | 20260505 20:38:44.442 | INFO | "pinned" : " 29 GB" | 20260505 20:38:44.442 | INFO | } | 20260505 20:38:44.442 | INFO | Perf report for the last 5.14572 seconds: | 20260505 20:38:44.442 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:38:44.442 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:44.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:44.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:44.442 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 20:38:44.442 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:44.442 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260505 20:38:44.442 | INFO | PRE_HQ : 60.2%, 1.8, 55.4ms, 0 | 20260505 20:38:44.442 | INFO | BAZWRITER : 62.8%, 3.7, 119.6ms, 0 | 20260505 20:38:44.442 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 20:38:44.442 | INFO | Uploaded 36GiB to GPU in 2.1529s (16.7216GiB/s) | 20260505 20:38:44.442 | INFO | Downloaded 38GiB to GPU in 2.25485s (16.8526GiB/s) | 20260505 20:38:49.585 | INFO | Analyzing chunk frames = [6140416,6140928) | 20260505 20:38:49.585 | INFO | MemInfo: { | 20260505 20:38:49.585 | INFO | "gpu" : " 17 GB", | 20260505 20:38:49.585 | INFO | "huge" : "", | 20260505 20:38:49.585 | INFO | "huge_shared" : " 27 GB", | 20260505 20:38:49.585 | INFO | "malloc" : " 14 GB", | 20260505 20:38:49.585 | INFO | "pinned" : " 29 GB" | 20260505 20:38:49.585 | INFO | } | 20260505 20:38:49.585 | INFO | Perf report for the last 5.1434 seconds: | 20260505 20:38:49.585 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:38:49.585 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:49.585 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:49.585 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:49.585 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 20:38:49.585 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:49.585 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:49.585 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:38:49.585 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:49.585 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:49.585 | INFO | Uploaded 36GiB to GPU in 2.08601s (17.2578GiB/s) | 20260505 20:38:49.585 | INFO | Downloaded 34GiB to GPU in 2.02495s (16.7905GiB/s) | 20260505 20:38:54.723 | INFO | Analyzing chunk frames = [6140928,6141440) | 20260505 20:38:54.723 | INFO | MemInfo: { | 20260505 20:38:54.723 | INFO | "gpu" : " 17 GB", | 20260505 20:38:54.723 | INFO | "huge" : "", | 20260505 20:38:54.723 | INFO | "huge_shared" : " 27 GB", | 20260505 20:38:54.723 | INFO | "malloc" : " 14 GB", | 20260505 20:38:54.723 | INFO | "pinned" : " 29 GB" | 20260505 20:38:54.723 | INFO | } | 20260505 20:38:54.723 | INFO | Perf report for the last 5.13762 seconds: | 20260505 20:38:54.723 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:38:54.723 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:54.723 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:54.723 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:54.723 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260505 20:38:54.723 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:54.723 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:54.723 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:38:54.723 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:54.723 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:54.723 | INFO | Uploaded 36GiB to GPU in 2.13291s (16.8783GiB/s) | 20260505 20:38:54.723 | INFO | Downloaded 34GiB to GPU in 2.02739s (16.7703GiB/s) | 20260505 20:38:59.857 | INFO | Analyzing chunk frames = [6141440,6141952) | 20260505 20:38:59.857 | INFO | MemInfo: { | 20260505 20:38:59.857 | INFO | "gpu" : " 17 GB", | 20260505 20:38:59.857 | INFO | "huge" : "", | 20260505 20:38:59.858 | INFO | "huge_shared" : " 27 GB", | 20260505 20:38:59.858 | INFO | "malloc" : " 14 GB", | 20260505 20:38:59.858 | INFO | "pinned" : " 29 GB" | 20260505 20:38:59.858 | INFO | } | 20260505 20:38:59.858 | INFO | Perf report for the last 5.13493 seconds: | 20260505 20:38:59.858 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:38:59.858 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:59.858 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:59.858 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:59.858 | INFO | ANALYSIS : 62.5%, 3.0, 97.0ms, 0 | 20260505 20:38:59.858 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:59.858 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:59.858 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:38:59.858 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:59.858 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:38:59.858 | INFO | Uploaded 36GiB to GPU in 2.2134s (16.2645GiB/s) | 20260505 20:38:59.858 | INFO | Downloaded 34GiB to GPU in 2.06167s (16.4915GiB/s) | 20260505 20:39:04.997 | INFO | Analyzing chunk frames = [6141952,6142464) | 20260505 20:39:04.997 | INFO | MemInfo: { | 20260505 20:39:04.997 | INFO | "gpu" : " 17 GB", | 20260505 20:39:04.997 | INFO | "huge" : "", | 20260505 20:39:04.997 | INFO | "huge_shared" : " 27 GB", | 20260505 20:39:04.997 | INFO | "malloc" : " 14 GB", | 20260505 20:39:04.997 | INFO | "pinned" : " 29 GB" | 20260505 20:39:04.997 | INFO | } | 20260505 20:39:04.998 | INFO | Perf report for the last 5.13982 seconds: | 20260505 20:39:04.998 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:39:04.998 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:04.998 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:04.998 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:04.998 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 20:39:04.998 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:04.998 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:04.998 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:39:04.998 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:04.998 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:04.998 | INFO | Uploaded 36GiB to GPU in 2.13154s (16.8892GiB/s) | 20260505 20:39:04.998 | INFO | Downloaded 34GiB to GPU in 2.06042s (16.5015GiB/s) | 20260505 20:39:10.142 | INFO | Analyzing chunk frames = [6142464,6142976) | 20260505 20:39:10.142 | INFO | MemInfo: { | 20260505 20:39:10.142 | INFO | "gpu" : " 17 GB", | 20260505 20:39:10.142 | INFO | "huge" : "", | 20260505 20:39:10.142 | INFO | "huge_shared" : " 27 GB", | 20260505 20:39:10.142 | INFO | "malloc" : " 14 GB", | 20260505 20:39:10.142 | INFO | "pinned" : " 29 GB" | 20260505 20:39:10.142 | INFO | } | 20260505 20:39:10.142 | INFO | Perf report for the last 5.1445 seconds: | 20260505 20:39:10.142 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:39:10.142 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:10.142 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:10.142 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:10.142 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260505 20:39:10.142 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:10.142 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:10.142 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:39:10.142 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:10.142 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:10.142 | INFO | Uploaded 36GiB to GPU in 2.08977s (17.2268GiB/s) | 20260505 20:39:10.142 | INFO | Downloaded 34GiB to GPU in 2.02707s (16.773GiB/s) | 20260505 20:39:15.284 | INFO | Analyzing chunk frames = [6142976,6143488) | 20260505 20:39:15.284 | INFO | MemInfo: { | 20260505 20:39:15.284 | INFO | "gpu" : " 17 GB", | 20260505 20:39:15.284 | INFO | "huge" : "", | 20260505 20:39:15.284 | INFO | "huge_shared" : " 27 GB", | 20260505 20:39:15.284 | INFO | "malloc" : " 14 GB", | 20260505 20:39:15.284 | INFO | "pinned" : " 29 GB" | 20260505 20:39:15.284 | INFO | } | 20260505 20:39:15.284 | INFO | Perf report for the last 5.14229 seconds: | 20260505 20:39:15.284 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:39:15.284 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:15.284 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:15.284 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:15.284 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 20:39:15.284 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:15.284 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:15.284 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 20:39:15.284 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:15.284 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:15.285 | INFO | Uploaded 36GiB to GPU in 2.13013s (16.9003GiB/s) | 20260505 20:39:15.285 | INFO | Downloaded 34GiB to GPU in 2.14238s (15.8702GiB/s) | 20260505 20:39:20.412 | INFO | Analyzing chunk frames = [6143488,6144000) | 20260505 20:39:20.412 | INFO | MemInfo: { | 20260505 20:39:20.412 | INFO | "gpu" : " 17 GB", | 20260505 20:39:20.412 | INFO | "huge" : "", | 20260505 20:39:20.412 | INFO | "huge_shared" : " 27 GB", | 20260505 20:39:20.412 | INFO | "malloc" : " 14 GB", | 20260505 20:39:20.412 | INFO | "pinned" : " 29 GB" | 20260505 20:39:20.412 | INFO | } | 20260505 20:39:20.412 | INFO | Perf report for the last 5.1278 seconds: | 20260505 20:39:20.412 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:39:20.412 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:20.412 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:20.412 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:20.412 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260505 20:39:20.412 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:20.412 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:20.412 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 20:39:20.412 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:20.412 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:20.412 | INFO | Uploaded 36GiB to GPU in 2.11293s (17.038GiB/s) | 20260505 20:39:20.412 | INFO | Downloaded 34GiB to GPU in 2.01808s (16.8477GiB/s) | 20260505 20:39:20.413 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 20:39:20.413 | INFO | Total mem usage: 15.3 GiB | 20260505 20:39:20.413 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 20:39:20.413 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.14%) | 20260505 20:39:20.413 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 20:39:20.413 | INFO | High Water Breakdown: | 20260505 20:39:20.413 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 20:39:20.413 | INFO | 38.8% : 5.5 GiB ( 82.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 20:39:20.413 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 20:39:20.413 | INFO | ------------------------------------------ | 20260505 20:39:20.413 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:39:20.413 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 20:39:20.413 | INFO | Total mem usage: 26.5 GiB | 20260505 20:39:20.413 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 20:39:20.413 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 20:39:20.413 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 20:39:20.413 | INFO | High Water Breakdown: | 20260505 20:39:20.413 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:39:20.413 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:39:20.413 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:39:20.413 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:39:20.413 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:39:20.413 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:39:20.413 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 20:39:20.413 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 20:39:20.413 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:39:20.413 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 20:39:20.413 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 20:39:20.413 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 20:39:20.413 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 20:39:20.413 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 20:39:20.413 | INFO | ------------------------------------------ | 20260505 20:39:20.413 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:39:20.413 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 20:39:20.413 | INFO | Total mem usage: 25.6 GiB | 20260505 20:39:20.413 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 20:39:20.413 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 20:39:20.413 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 20:39:20.413 | INFO | High Water Breakdown: | 20260505 20:39:20.413 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 20:39:20.413 | INFO | ------------------------------------------ | 20260505 20:39:20.413 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:39:20.413 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 20:39:20.413 | INFO | Total mem usage: 15.7 GiB | 20260505 20:39:20.413 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 20:39:20.413 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 20:39:20.413 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 20:39:20.413 | INFO | High Water Breakdown: | 20260505 20:39:20.413 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 20:39:20.413 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 20:39:20.413 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 20:39:20.413 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 20:39:20.413 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 20:39:20.413 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 20:39:20.413 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 20:39:20.413 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 20:39:20.413 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 20:39:20.413 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:39:20.413 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 20:39:20.413 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:39:20.413 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 20:39:20.413 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 20:39:20.413 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 20:39:20.413 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 20:39:20.413 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:39:20.413 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 20:39:20.413 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:39:20.413 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:39:20.413 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:39:20.413 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 20:39:20.413 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 20:39:20.413 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:39:20.413 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 20:39:20.413 | INFO | ------------------------------------------ | 20260505 20:39:20.413 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:39:20.414 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 20:39:20.414 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:39:20.414 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:39:20.414 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.45e+01, 1.36e+01 / 1.97e+00 | 20260505 20:39:20.414 | INFO | PerfReport: 0.29%, AggregateStats -- 9.26e-02 / 1.03e-01, 1.05e-02 / 3.75e-03 | 20260505 20:39:20.414 | INFO | PerfReport: 11.79%, Baseline -- 3.71e+00 / 4.55e+00, 5.71e-01 / 1.46e-01 | 20260505 20:39:20.414 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.53e+00, 1.92e-01 / 8.89e-02 | 20260505 20:39:20.414 | INFO | PerfReport: 15.97%, DME -- 4.02e+01 / 4.44e+01, 3.98e+00 / 1.98e+00 | 20260505 20:39:20.414 | INFO | PerfReport: 57.09%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.50e+00 / 1.13e+00 | 20260505 20:39:20.414 | INFO | PerfReport: 7.21%, PulseAccumulating -- 2.27e+00 / 2.84e+00, 4.35e-01 / 2.11e-01 | 20260505 20:39:20.414 | INFO | PerfReport: 3.23%, Metrics -- 1.02e+00 / 1.35e+00, 2.83e-01 / 1.23e-01 | 20260505 20:39:20.414 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.54e-01 / 4.18e-01, 6.27e-02 / 2.99e-02 | 20260505 20:39:20.414 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 20:39:20.414 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:39:20.414 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:39:20.414 | INFO | PerfReport: Full Execution -- 9.92e+01 / -nan, 2.52e+01 / 0.00e+00 | 20260505 20:39:20.414 | INFO | PerfReport: 22.38%, Upload -- 2.22e+01 / 0.00e+00, 7.66e+00 / 0.00e+00 | 20260505 20:39:20.414 | INFO | PerfReport: 21.85%, Download -- 2.17e+01 / 0.00e+00, 7.72e+00 / 0.00e+00 | 20260505 20:39:20.414 | INFO | PerfReport: 31.76%, Compute -- 3.15e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 20:39:20.414 | INFO | PerfReport: 2.19%, LockUpload -- 2.18e+00 / 0.00e+00, 7.85e+00 / 0.00e+00 | 20260505 20:39:20.414 | INFO | PerfReport: 1.36%, LockDownload -- 1.35e+00 / 0.00e+00, 2.95e+00 / 0.00e+00 | 20260505 20:39:20.414 | INFO | PerfReport: 20.45%, LockCompute -- 2.03e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 20:39:20.414 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.83e-02 / 0.00e+00 | 20260505 20:39:20.545 | INFO | Starting a round of Baz IO | 20260505 20:39:23.714 | INFO | Populating RTMetrics report | 20260505 20:39:23.714 | INFO | RTMetrics report populated | 20260505 20:39:23.714 | INFO | Opening tmp RTMetrics json file | 20260505 20:39:23.715 | INFO | Serializing RTMetrics to json | 20260505 20:39:23.716 | INFO | Serialization done, swapping tmp file now... | 20260505 20:39:23.716 | INFO | RTMetrics done | 20260505 20:39:23.803 | INFO | Finished a round of Baz IO: Wrote 2.003 GiB in 3.258 seconds | 20260505 20:39:23.803 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.65% / 0.00% / 78.35% | 20260505 20:39:23.803 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.169643% / 0% / 8.74201e-05% | 20260505 20:39:23.803 | INFO | Baz IO overall: | 20260505 20:39:23.803 | INFO | BytesWritten : 3575742467060 | 20260505 20:39:23.803 | INFO | NumEvents : 1689961573686 | 20260505 20:39:25.554 | INFO | Analyzing chunk frames = [6144000,6144512) | 20260505 20:39:25.554 | INFO | MemInfo: { | 20260505 20:39:25.554 | INFO | "gpu" : " 17 GB", | 20260505 20:39:25.554 | INFO | "huge" : "", | 20260505 20:39:25.554 | INFO | "huge_shared" : " 27 GB", | 20260505 20:39:25.554 | INFO | "malloc" : " 14 GB", | 20260505 20:39:25.554 | INFO | "pinned" : " 29 GB" | 20260505 20:39:25.554 | INFO | } | 20260505 20:39:25.555 | INFO | Perf report for the last 5.14239 seconds: | 20260505 20:39:25.555 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:39:25.555 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:25.555 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:25.555 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:25.555 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 20:39:25.555 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:25.555 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260505 20:39:25.555 | INFO | PRE_HQ : 60.2%, 1.8, 55.2ms, 0 | 20260505 20:39:25.555 | INFO | BAZWRITER : 62.9%, 3.7, 118.3ms, 0 | 20260505 20:39:25.555 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 20:39:25.555 | INFO | Uploaded 36GiB to GPU in 2.1698s (16.5914GiB/s) | 20260505 20:39:25.555 | INFO | Downloaded 38GiB to GPU in 2.25295s (16.8668GiB/s) | 20260505 20:39:30.697 | INFO | Analyzing chunk frames = [6144512,6145024) | 20260505 20:39:30.697 | INFO | MemInfo: { | 20260505 20:39:30.697 | INFO | "gpu" : " 17 GB", | 20260505 20:39:30.697 | INFO | "huge" : "", | 20260505 20:39:30.697 | INFO | "huge_shared" : " 27 GB", | 20260505 20:39:30.697 | INFO | "malloc" : " 14 GB", | 20260505 20:39:30.697 | INFO | "pinned" : " 29 GB" | 20260505 20:39:30.697 | INFO | } | 20260505 20:39:30.697 | INFO | Perf report for the last 5.14274 seconds: | 20260505 20:39:30.697 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:39:30.697 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:30.697 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:30.697 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:30.697 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 20:39:30.697 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:30.697 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:30.697 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:39:30.697 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:30.697 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:30.698 | INFO | Uploaded 36GiB to GPU in 2.0764s (17.3377GiB/s) | 20260505 20:39:30.698 | INFO | Downloaded 34GiB to GPU in 2.02369s (16.801GiB/s) | 20260505 20:39:35.837 | INFO | Analyzing chunk frames = [6145024,6145536) | 20260505 20:39:35.837 | INFO | MemInfo: { | 20260505 20:39:35.837 | INFO | "gpu" : " 17 GB", | 20260505 20:39:35.837 | INFO | "huge" : "", | 20260505 20:39:35.837 | INFO | "huge_shared" : " 27 GB", | 20260505 20:39:35.837 | INFO | "malloc" : " 14 GB", | 20260505 20:39:35.837 | INFO | "pinned" : " 29 GB" | 20260505 20:39:35.837 | INFO | } | 20260505 20:39:35.837 | INFO | Perf report for the last 5.14005 seconds: | 20260505 20:39:35.837 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:39:35.837 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:35.837 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:35.837 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:35.837 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260505 20:39:35.837 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:35.837 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:35.837 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 20:39:35.838 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:35.838 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:35.838 | INFO | Uploaded 36GiB to GPU in 2.13365s (16.8725GiB/s) | 20260505 20:39:35.838 | INFO | Downloaded 34GiB to GPU in 2.02411s (16.7975GiB/s) | 20260505 20:39:40.966 | INFO | Analyzing chunk frames = [6145536,6146048) | 20260505 20:39:40.966 | INFO | MemInfo: { | 20260505 20:39:40.966 | INFO | "gpu" : " 17 GB", | 20260505 20:39:40.966 | INFO | "huge" : "", | 20260505 20:39:40.967 | INFO | "huge_shared" : " 27 GB", | 20260505 20:39:40.967 | INFO | "malloc" : " 14 GB", | 20260505 20:39:40.967 | INFO | "pinned" : " 29 GB" | 20260505 20:39:40.967 | INFO | } | 20260505 20:39:40.967 | INFO | Perf report for the last 5.12939 seconds: | 20260505 20:39:40.967 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:39:40.967 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:40.967 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:40.967 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:40.967 | INFO | ANALYSIS : 62.2%, 3.0, 96.4ms, 0 | 20260505 20:39:40.967 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:40.967 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:40.967 | INFO | PRE_HQ : 27.5%, 1.0, 14.2ms, 0 | 20260505 20:39:40.967 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:40.967 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:40.967 | INFO | Uploaded 36GiB to GPU in 2.14524s (16.7813GiB/s) | 20260505 20:39:40.967 | INFO | Downloaded 34GiB to GPU in 2.03981s (16.6682GiB/s) | 20260505 20:39:46.106 | INFO | Analyzing chunk frames = [6146048,6146560) | 20260505 20:39:46.106 | INFO | MemInfo: { | 20260505 20:39:46.106 | INFO | "gpu" : " 17 GB", | 20260505 20:39:46.106 | INFO | "huge" : "", | 20260505 20:39:46.106 | INFO | "huge_shared" : " 27 GB", | 20260505 20:39:46.106 | INFO | "malloc" : " 14 GB", | 20260505 20:39:46.106 | INFO | "pinned" : " 29 GB" | 20260505 20:39:46.106 | INFO | } | 20260505 20:39:46.106 | INFO | Perf report for the last 5.13915 seconds: | 20260505 20:39:46.106 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:39:46.106 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:46.106 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:46.106 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:46.106 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 20:39:46.106 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:46.106 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:46.106 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:39:46.106 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:46.106 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:46.106 | INFO | Uploaded 36GiB to GPU in 2.1396s (16.8256GiB/s) | 20260505 20:39:46.106 | INFO | Downloaded 34GiB to GPU in 2.06591s (16.4577GiB/s) | 20260505 20:39:51.256 | INFO | Analyzing chunk frames = [6146560,6147072) | 20260505 20:39:51.256 | INFO | MemInfo: { | 20260505 20:39:51.256 | INFO | "gpu" : " 17 GB", | 20260505 20:39:51.256 | INFO | "huge" : "", | 20260505 20:39:51.256 | INFO | "huge_shared" : " 27 GB", | 20260505 20:39:51.256 | INFO | "malloc" : " 14 GB", | 20260505 20:39:51.256 | INFO | "pinned" : " 29 GB" | 20260505 20:39:51.256 | INFO | } | 20260505 20:39:51.256 | INFO | Perf report for the last 5.14992 seconds: | 20260505 20:39:51.256 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:39:51.256 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:51.256 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:51.256 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:51.256 | INFO | ANALYSIS : 62.0%, 3.0, 97.0ms, 0 | 20260505 20:39:51.256 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:51.256 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:51.256 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:39:51.256 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:51.256 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:51.256 | INFO | Uploaded 36GiB to GPU in 2.13469s (16.8643GiB/s) | 20260505 20:39:51.256 | INFO | Downloaded 34GiB to GPU in 2.04993s (16.5859GiB/s) | 20260505 20:39:56.397 | INFO | Analyzing chunk frames = [6147072,6147584) | 20260505 20:39:56.397 | INFO | MemInfo: { | 20260505 20:39:56.397 | INFO | "gpu" : " 17 GB", | 20260505 20:39:56.397 | INFO | "huge" : "", | 20260505 20:39:56.397 | INFO | "huge_shared" : " 27 GB", | 20260505 20:39:56.397 | INFO | "malloc" : " 14 GB", | 20260505 20:39:56.397 | INFO | "pinned" : " 29 GB" | 20260505 20:39:56.397 | INFO | } | 20260505 20:39:56.397 | INFO | Perf report for the last 5.14142 seconds: | 20260505 20:39:56.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:39:56.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:56.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:56.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:56.397 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 20:39:56.397 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:56.397 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:56.397 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 20:39:56.397 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:56.397 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:39:56.398 | INFO | Uploaded 36GiB to GPU in 2.1278s (16.9189GiB/s) | 20260505 20:39:56.398 | INFO | Downloaded 34GiB to GPU in 2.14238s (15.8702GiB/s) | 20260505 20:40:01.536 | INFO | Analyzing chunk frames = [6147584,6148096) | 20260505 20:40:01.536 | INFO | MemInfo: { | 20260505 20:40:01.536 | INFO | "gpu" : " 17 GB", | 20260505 20:40:01.536 | INFO | "huge" : "", | 20260505 20:40:01.536 | INFO | "huge_shared" : " 27 GB", | 20260505 20:40:01.536 | INFO | "malloc" : " 14 GB", | 20260505 20:40:01.536 | INFO | "pinned" : " 29 GB" | 20260505 20:40:01.536 | INFO | } | 20260505 20:40:01.536 | INFO | Perf report for the last 5.13907 seconds: | 20260505 20:40:01.536 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:40:01.536 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:01.536 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:01.536 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:01.536 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 20:40:01.536 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:01.537 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:01.537 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 20:40:01.537 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:01.537 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:01.537 | INFO | Uploaded 36GiB to GPU in 2.10803s (17.0775GiB/s) | 20260505 20:40:01.537 | INFO | Downloaded 34GiB to GPU in 2.00179s (16.9848GiB/s) | 20260505 20:40:01.675 | INFO | Starting a round of Baz IO | 20260505 20:40:04.827 | INFO | Populating RTMetrics report | 20260505 20:40:04.827 | INFO | RTMetrics report populated | 20260505 20:40:04.827 | INFO | Opening tmp RTMetrics json file | 20260505 20:40:04.828 | INFO | Serializing RTMetrics to json | 20260505 20:40:04.829 | INFO | Serialization done, swapping tmp file now... | 20260505 20:40:04.829 | INFO | RTMetrics done | 20260505 20:40:04.915 | INFO | Finished a round of Baz IO: Wrote 2.002 GiB in 3.24 seconds | 20260505 20:40:04.915 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.65% / 0.00% / 78.35% | 20260505 20:40:04.915 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.169309% / 0% / 9.93411e-05% | 20260505 20:40:04.915 | INFO | Baz IO overall: | 20260505 20:40:04.915 | INFO | BytesWritten : 3577891875225 | 20260505 20:40:04.915 | INFO | NumEvents : 1690955879306 | 20260505 20:40:06.667 | INFO | Analyzing chunk frames = [6148096,6148608) | 20260505 20:40:06.667 | INFO | MemInfo: { | 20260505 20:40:06.668 | INFO | "gpu" : " 17 GB", | 20260505 20:40:06.668 | INFO | "huge" : "", | 20260505 20:40:06.668 | INFO | "huge_shared" : " 27 GB", | 20260505 20:40:06.668 | INFO | "malloc" : " 14 GB", | 20260505 20:40:06.668 | INFO | "pinned" : " 29 GB" | 20260505 20:40:06.668 | INFO | } | 20260505 20:40:06.668 | INFO | Perf report for the last 5.13134 seconds: | 20260505 20:40:06.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:40:06.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:06.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:06.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:06.668 | INFO | ANALYSIS : 63.1%, 3.0, 98.2ms, 0 | 20260505 20:40:06.668 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:06.668 | INFO | RT_METRICS : 44.0%, 1.0, 23.0ms, 0 | 20260505 20:40:06.668 | INFO | PRE_HQ : 60.3%, 1.8, 55.4ms, 0 | 20260505 20:40:06.668 | INFO | BAZWRITER : 62.8%, 3.7, 119.5ms, 0 | 20260505 20:40:06.668 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 20:40:06.668 | INFO | Uploaded 36GiB to GPU in 2.17133s (16.5797GiB/s) | 20260505 20:40:06.668 | INFO | Downloaded 38GiB to GPU in 2.25572s (16.846GiB/s) | 20260505 20:40:11.808 | INFO | Analyzing chunk frames = [6148608,6149120) | 20260505 20:40:11.808 | INFO | MemInfo: { | 20260505 20:40:11.808 | INFO | "gpu" : " 17 GB", | 20260505 20:40:11.808 | INFO | "huge" : "", | 20260505 20:40:11.808 | INFO | "huge_shared" : " 27 GB", | 20260505 20:40:11.809 | INFO | "malloc" : " 14 GB", | 20260505 20:40:11.809 | INFO | "pinned" : " 29 GB" | 20260505 20:40:11.809 | INFO | } | 20260505 20:40:11.809 | INFO | Perf report for the last 5.14083 seconds: | 20260505 20:40:11.809 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:40:11.809 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:11.809 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:11.809 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:11.809 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260505 20:40:11.809 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:11.809 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:11.809 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:40:11.809 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:11.809 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:11.809 | INFO | Uploaded 36GiB to GPU in 2.15299s (16.7209GiB/s) | 20260505 20:40:11.809 | INFO | Downloaded 34GiB to GPU in 2.05302s (16.561GiB/s) | 20260505 20:40:16.949 | INFO | Analyzing chunk frames = [6149120,6149632) | 20260505 20:40:16.949 | INFO | MemInfo: { | 20260505 20:40:16.949 | INFO | "gpu" : " 17 GB", | 20260505 20:40:16.949 | INFO | "huge" : "", | 20260505 20:40:16.949 | INFO | "huge_shared" : " 27 GB", | 20260505 20:40:16.949 | INFO | "malloc" : " 14 GB", | 20260505 20:40:16.949 | INFO | "pinned" : " 29 GB" | 20260505 20:40:16.949 | INFO | } | 20260505 20:40:16.949 | INFO | Perf report for the last 5.14071 seconds: | 20260505 20:40:16.949 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:40:16.949 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:16.949 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:16.949 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:16.949 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260505 20:40:16.949 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:16.949 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:16.949 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:40:16.949 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:16.949 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:16.950 | INFO | Uploaded 36GiB to GPU in 2.12407s (16.9486GiB/s) | 20260505 20:40:16.950 | INFO | Downloaded 34GiB to GPU in 2.02822s (16.7634GiB/s) | 20260505 20:40:22.089 | INFO | Analyzing chunk frames = [6149632,6150144) | 20260505 20:40:22.089 | INFO | MemInfo: { | 20260505 20:40:22.089 | INFO | "gpu" : " 17 GB", | 20260505 20:40:22.089 | INFO | "huge" : "", | 20260505 20:40:22.089 | INFO | "huge_shared" : " 27 GB", | 20260505 20:40:22.089 | INFO | "malloc" : " 14 GB", | 20260505 20:40:22.089 | INFO | "pinned" : " 29 GB" | 20260505 20:40:22.089 | INFO | } | 20260505 20:40:22.089 | INFO | Perf report for the last 5.14012 seconds: | 20260505 20:40:22.089 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:40:22.089 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:22.089 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:22.089 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:22.089 | INFO | ANALYSIS : 62.4%, 3.0, 96.9ms, 0 | 20260505 20:40:22.090 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:22.090 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:22.090 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260505 20:40:22.090 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:22.090 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:22.090 | INFO | Uploaded 36GiB to GPU in 2.21166s (16.2773GiB/s) | 20260505 20:40:22.090 | INFO | Downloaded 34GiB to GPU in 2.07118s (16.4158GiB/s) | 20260505 20:40:27.226 | INFO | Analyzing chunk frames = [6150144,6150656) | 20260505 20:40:27.226 | INFO | MemInfo: { | 20260505 20:40:27.226 | INFO | "gpu" : " 17 GB", | 20260505 20:40:27.226 | INFO | "huge" : "", | 20260505 20:40:27.226 | INFO | "huge_shared" : " 27 GB", | 20260505 20:40:27.226 | INFO | "malloc" : " 14 GB", | 20260505 20:40:27.226 | INFO | "pinned" : " 29 GB" | 20260505 20:40:27.226 | INFO | } | 20260505 20:40:27.226 | INFO | Perf report for the last 5.13677 seconds: | 20260505 20:40:27.226 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:40:27.226 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:27.226 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:27.226 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:27.226 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260505 20:40:27.226 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:27.226 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:27.226 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:40:27.226 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:27.226 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:27.226 | INFO | Uploaded 36GiB to GPU in 2.12102s (16.973GiB/s) | 20260505 20:40:27.227 | INFO | Downloaded 34GiB to GPU in 2.06148s (16.493GiB/s) | 20260505 20:40:32.363 | INFO | Analyzing chunk frames = [6150656,6151168) | 20260505 20:40:32.363 | INFO | MemInfo: { | 20260505 20:40:32.363 | INFO | "gpu" : " 17 GB", | 20260505 20:40:32.363 | INFO | "huge" : "", | 20260505 20:40:32.363 | INFO | "huge_shared" : " 27 GB", | 20260505 20:40:32.363 | INFO | "malloc" : " 14 GB", | 20260505 20:40:32.363 | INFO | "pinned" : " 29 GB" | 20260505 20:40:32.363 | INFO | } | 20260505 20:40:32.363 | INFO | Perf report for the last 5.13681 seconds: | 20260505 20:40:32.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:40:32.363 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:32.363 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:32.363 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:32.363 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 20:40:32.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:32.363 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:32.363 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 20:40:32.363 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:32.363 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:32.363 | INFO | Uploaded 36GiB to GPU in 2.13345s (16.8741GiB/s) | 20260505 20:40:32.363 | INFO | Downloaded 34GiB to GPU in 2.05916s (16.5116GiB/s) | 20260505 20:40:37.513 | INFO | Analyzing chunk frames = [6151168,6151680) | 20260505 20:40:37.513 | INFO | MemInfo: { | 20260505 20:40:37.513 | INFO | "gpu" : " 17 GB", | 20260505 20:40:37.513 | INFO | "huge" : "", | 20260505 20:40:37.513 | INFO | "huge_shared" : " 27 GB", | 20260505 20:40:37.513 | INFO | "malloc" : " 14 GB", | 20260505 20:40:37.513 | INFO | "pinned" : " 29 GB" | 20260505 20:40:37.513 | INFO | } | 20260505 20:40:37.513 | INFO | Perf report for the last 5.15043 seconds: | 20260505 20:40:37.513 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:40:37.513 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:37.514 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:37.514 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:37.514 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 20:40:37.514 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:37.514 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:37.514 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 20:40:37.514 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:37.514 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:37.514 | INFO | Uploaded 36GiB to GPU in 2.11828s (16.9949GiB/s) | 20260505 20:40:37.514 | INFO | Downloaded 34GiB to GPU in 2.14922s (15.8197GiB/s) | 20260505 20:40:42.652 | INFO | Analyzing chunk frames = [6151680,6152192) | 20260505 20:40:42.652 | INFO | MemInfo: { | 20260505 20:40:42.652 | INFO | "gpu" : " 17 GB", | 20260505 20:40:42.652 | INFO | "huge" : "", | 20260505 20:40:42.653 | INFO | "huge_shared" : " 27 GB", | 20260505 20:40:42.653 | INFO | "malloc" : " 14 GB", | 20260505 20:40:42.653 | INFO | "pinned" : " 29 GB" | 20260505 20:40:42.653 | INFO | } | 20260505 20:40:42.653 | INFO | Perf report for the last 5.13928 seconds: | 20260505 20:40:42.653 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:40:42.653 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:42.653 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:42.653 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:42.653 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 20:40:42.653 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:42.653 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:42.653 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 20:40:42.653 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:42.653 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:42.653 | INFO | Uploaded 36GiB to GPU in 2.11889s (16.99GiB/s) | 20260505 20:40:42.653 | INFO | Downloaded 34GiB to GPU in 2.00548s (16.9535GiB/s) | 20260505 20:40:42.653 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 20:40:42.653 | INFO | Total mem usage: 15.3 GiB | 20260505 20:40:42.653 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 20:40:42.653 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.15%) | 20260505 20:40:42.653 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 20:40:42.653 | INFO | High Water Breakdown: | 20260505 20:40:42.653 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 20:40:42.653 | INFO | 38.8% : 5.5 GiB ( 82.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 20:40:42.653 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 20:40:42.653 | INFO | ------------------------------------------ | 20260505 20:40:42.654 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:40:42.654 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 20:40:42.654 | INFO | Total mem usage: 26.5 GiB | 20260505 20:40:42.654 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 20:40:42.654 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 20:40:42.654 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 20:40:42.654 | INFO | High Water Breakdown: | 20260505 20:40:42.654 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:40:42.654 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:40:42.654 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:40:42.654 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:40:42.654 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:40:42.654 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:40:42.654 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 20:40:42.654 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 20:40:42.654 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:40:42.654 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 20:40:42.654 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 20:40:42.654 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 20:40:42.654 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 20:40:42.654 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 20:40:42.654 | INFO | ------------------------------------------ | 20260505 20:40:42.654 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:40:42.654 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 20:40:42.654 | INFO | Total mem usage: 25.6 GiB | 20260505 20:40:42.654 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 20:40:42.654 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 20:40:42.654 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 20:40:42.654 | INFO | High Water Breakdown: | 20260505 20:40:42.654 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 20:40:42.654 | INFO | ------------------------------------------ | 20260505 20:40:42.654 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:40:42.654 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 20:40:42.654 | INFO | Total mem usage: 15.7 GiB | 20260505 20:40:42.654 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 20:40:42.654 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 20:40:42.654 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 20:40:42.654 | INFO | High Water Breakdown: | 20260505 20:40:42.654 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 20:40:42.654 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 20:40:42.654 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 20:40:42.654 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 20:40:42.654 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 20:40:42.654 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 20:40:42.654 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 20:40:42.654 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 20:40:42.654 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 20:40:42.654 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:40:42.654 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 20:40:42.654 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:40:42.654 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 20:40:42.654 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 20:40:42.654 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 20:40:42.654 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 20:40:42.654 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:40:42.654 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 20:40:42.654 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:40:42.654 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:40:42.654 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:40:42.654 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 20:40:42.654 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 20:40:42.654 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:40:42.654 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 20:40:42.654 | INFO | ------------------------------------------ | 20260505 20:40:42.654 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:40:42.654 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 20:40:42.654 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:40:42.654 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:40:42.654 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.45e+01, 1.34e+01 / 1.86e+00 | 20260505 20:40:42.654 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.02e-01, 1.03e-02 / 2.65e-03 | 20260505 20:40:42.654 | INFO | PerfReport: 11.81%, Baseline -- 3.72e+00 / 4.57e+00, 5.72e-01 / 1.43e-01 | 20260505 20:40:42.654 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.52e+00, 1.90e-01 / 8.72e-02 | 20260505 20:40:42.654 | INFO | PerfReport: 15.97%, DME -- 4.02e+01 / 4.44e+01, 4.18e+00 / 1.98e+00 | 20260505 20:40:42.654 | INFO | PerfReport: 57.07%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.46e+00 / 1.04e+00 | 20260505 20:40:42.654 | INFO | PerfReport: 7.23%, PulseAccumulating -- 2.27e+00 / 2.81e+00, 4.35e-01 / 2.25e-01 | 20260505 20:40:42.654 | INFO | PerfReport: 3.23%, Metrics -- 1.02e+00 / 1.36e+00, 2.80e-01 / 1.65e-01 | 20260505 20:40:42.655 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.53e-01 / 4.14e-01, 6.23e-02 / 3.37e-02 | 20260505 20:40:42.655 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 20:40:42.655 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:40:42.655 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:40:42.655 | INFO | PerfReport: Full Execution -- 9.93e+01 / -nan, 2.52e+01 / 0.00e+00 | 20260505 20:40:42.655 | INFO | PerfReport: 22.42%, Upload -- 2.23e+01 / 0.00e+00, 7.67e+00 / 0.00e+00 | 20260505 20:40:42.655 | INFO | PerfReport: 21.83%, Download -- 2.17e+01 / 0.00e+00, 7.72e+00 / 0.00e+00 | 20260505 20:40:42.655 | INFO | PerfReport: 31.70%, Compute -- 3.15e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 20:40:42.655 | INFO | PerfReport: 2.21%, LockUpload -- 2.20e+00 / 0.00e+00, 8.09e+00 / 0.00e+00 | 20260505 20:40:42.655 | INFO | PerfReport: 1.44%, LockDownload -- 1.43e+00 / 0.00e+00, 3.05e+00 / 0.00e+00 | 20260505 20:40:42.655 | INFO | PerfReport: 20.39%, LockCompute -- 2.02e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 20:40:42.655 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.44e-02 / 0.00e+00 | 20260505 20:40:42.781 | INFO | Starting a round of Baz IO | 20260505 20:40:45.938 | INFO | Populating RTMetrics report | 20260505 20:40:45.939 | INFO | RTMetrics report populated | 20260505 20:40:45.939 | INFO | Opening tmp RTMetrics json file | 20260505 20:40:45.940 | INFO | Serializing RTMetrics to json | 20260505 20:40:45.943 | INFO | Serialization done, swapping tmp file now... | 20260505 20:40:45.946 | INFO | RTMetrics done | 20260505 20:40:46.030 | INFO | Finished a round of Baz IO: Wrote 2.003 GiB in 3.248 seconds | 20260505 20:40:46.030 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.65% / 0.00% / 78.35% | 20260505 20:40:46.030 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.168355% / 3.97364e-06% / 9.53674e-05% | 20260505 20:40:46.030 | INFO | Baz IO overall: | 20260505 20:40:46.030 | INFO | BytesWritten : 3580042321152 | 20260505 20:40:46.030 | INFO | NumEvents : 1691951290118 | 20260505 20:40:47.783 | INFO | Analyzing chunk frames = [6152192,6152704) | 20260505 20:40:47.783 | INFO | MemInfo: { | 20260505 20:40:47.783 | INFO | "gpu" : " 17 GB", | 20260505 20:40:47.783 | INFO | "huge" : "", | 20260505 20:40:47.783 | INFO | "huge_shared" : " 27 GB", | 20260505 20:40:47.783 | INFO | "malloc" : " 14 GB", | 20260505 20:40:47.783 | INFO | "pinned" : " 29 GB" | 20260505 20:40:47.783 | INFO | } | 20260505 20:40:47.783 | INFO | Perf report for the last 5.13046 seconds: | 20260505 20:40:47.783 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:40:47.783 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:47.783 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:47.783 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:47.783 | INFO | ANALYSIS : 63.0%, 3.0, 98.0ms, 0 | 20260505 20:40:47.783 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:47.783 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260505 20:40:47.783 | INFO | PRE_HQ : 60.3%, 1.8, 55.3ms, 0 | 20260505 20:40:47.783 | INFO | BAZWRITER : 62.6%, 3.7, 119.2ms, 0 | 20260505 20:40:47.783 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 20:40:47.784 | INFO | Uploaded 36GiB to GPU in 2.17462s (16.5546GiB/s) | 20260505 20:40:47.784 | INFO | Downloaded 38GiB to GPU in 2.24207s (16.9486GiB/s) | 20260505 20:40:52.917 | INFO | Analyzing chunk frames = [6152704,6153216) | 20260505 20:40:52.917 | INFO | MemInfo: { | 20260505 20:40:52.917 | INFO | "gpu" : " 17 GB", | 20260505 20:40:52.917 | INFO | "huge" : "", | 20260505 20:40:52.917 | INFO | "huge_shared" : " 27 GB", | 20260505 20:40:52.917 | INFO | "malloc" : " 14 GB", | 20260505 20:40:52.917 | INFO | "pinned" : " 29 GB" | 20260505 20:40:52.917 | INFO | } | 20260505 20:40:52.917 | INFO | Perf report for the last 5.13417 seconds: | 20260505 20:40:52.917 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:40:52.917 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:52.917 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:52.917 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:52.917 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 20:40:52.917 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:52.917 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:52.918 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 20:40:52.918 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:52.918 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:52.918 | INFO | Uploaded 36GiB to GPU in 2.09089s (17.2176GiB/s) | 20260505 20:40:52.918 | INFO | Downloaded 34GiB to GPU in 2.02511s (16.7892GiB/s) | 20260505 20:40:58.061 | INFO | Analyzing chunk frames = [6153216,6153728) | 20260505 20:40:58.061 | INFO | MemInfo: { | 20260505 20:40:58.061 | INFO | "gpu" : " 17 GB", | 20260505 20:40:58.061 | INFO | "huge" : "", | 20260505 20:40:58.061 | INFO | "huge_shared" : " 27 GB", | 20260505 20:40:58.061 | INFO | "malloc" : " 14 GB", | 20260505 20:40:58.061 | INFO | "pinned" : " 29 GB" | 20260505 20:40:58.061 | INFO | } | 20260505 20:40:58.061 | INFO | Perf report for the last 5.14361 seconds: | 20260505 20:40:58.061 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:40:58.061 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:58.061 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:58.061 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:58.061 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260505 20:40:58.061 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:58.061 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:58.061 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:40:58.061 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:58.061 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:40:58.061 | INFO | Uploaded 36GiB to GPU in 2.13157s (16.889GiB/s) | 20260505 20:40:58.061 | INFO | Downloaded 34GiB to GPU in 2.03226s (16.7302GiB/s) | 20260505 20:41:03.204 | INFO | Analyzing chunk frames = [6153728,6154240) | 20260505 20:41:03.204 | INFO | MemInfo: { | 20260505 20:41:03.204 | INFO | "gpu" : " 17 GB", | 20260505 20:41:03.204 | INFO | "huge" : "", | 20260505 20:41:03.204 | INFO | "huge_shared" : " 27 GB", | 20260505 20:41:03.204 | INFO | "malloc" : " 14 GB", | 20260505 20:41:03.204 | INFO | "pinned" : " 29 GB" | 20260505 20:41:03.204 | INFO | } | 20260505 20:41:03.204 | INFO | Perf report for the last 5.14345 seconds: | 20260505 20:41:03.204 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:41:03.204 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:03.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:03.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:03.205 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260505 20:41:03.205 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:03.205 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:03.205 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260505 20:41:03.205 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:03.205 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:03.205 | INFO | Uploaded 36GiB to GPU in 2.20475s (16.3284GiB/s) | 20260505 20:41:03.205 | INFO | Downloaded 34GiB to GPU in 2.04717s (16.6083GiB/s) | 20260505 20:41:08.349 | INFO | Analyzing chunk frames = [6154240,6154752) | 20260505 20:41:08.349 | INFO | MemInfo: { | 20260505 20:41:08.349 | INFO | "gpu" : " 17 GB", | 20260505 20:41:08.349 | INFO | "huge" : "", | 20260505 20:41:08.349 | INFO | "huge_shared" : " 27 GB", | 20260505 20:41:08.349 | INFO | "malloc" : " 14 GB", | 20260505 20:41:08.349 | INFO | "pinned" : " 29 GB" | 20260505 20:41:08.349 | INFO | } | 20260505 20:41:08.349 | INFO | Perf report for the last 5.1445 seconds: | 20260505 20:41:08.349 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:41:08.349 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:08.349 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:08.349 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:08.349 | INFO | ANALYSIS : 61.5%, 3.0, 96.1ms, 0 | 20260505 20:41:08.349 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:08.349 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:08.349 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 20:41:08.349 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:08.349 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:08.349 | INFO | Uploaded 36GiB to GPU in 2.12729s (16.9229GiB/s) | 20260505 20:41:08.349 | INFO | Downloaded 34GiB to GPU in 2.06162s (16.4919GiB/s) | 20260505 20:41:13.485 | INFO | Analyzing chunk frames = [6154752,6155264) | 20260505 20:41:13.485 | INFO | MemInfo: { | 20260505 20:41:13.485 | INFO | "gpu" : " 17 GB", | 20260505 20:41:13.485 | INFO | "huge" : "", | 20260505 20:41:13.485 | INFO | "huge_shared" : " 27 GB", | 20260505 20:41:13.485 | INFO | "malloc" : " 14 GB", | 20260505 20:41:13.485 | INFO | "pinned" : " 29 GB" | 20260505 20:41:13.485 | INFO | } | 20260505 20:41:13.485 | INFO | Perf report for the last 5.13601 seconds: | 20260505 20:41:13.485 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:41:13.485 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:13.485 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:13.485 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:13.485 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 20:41:13.485 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:13.485 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:13.485 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:41:13.485 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:13.485 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:13.485 | INFO | Uploaded 36GiB to GPU in 2.09658s (17.1708GiB/s) | 20260505 20:41:13.485 | INFO | Downloaded 34GiB to GPU in 2.03632s (16.6968GiB/s) | 20260505 20:41:18.619 | INFO | Analyzing chunk frames = [6155264,6155776) | 20260505 20:41:18.619 | INFO | MemInfo: { | 20260505 20:41:18.619 | INFO | "gpu" : " 17 GB", | 20260505 20:41:18.619 | INFO | "huge" : "", | 20260505 20:41:18.619 | INFO | "huge_shared" : " 27 GB", | 20260505 20:41:18.619 | INFO | "malloc" : " 14 GB", | 20260505 20:41:18.619 | INFO | "pinned" : " 29 GB" | 20260505 20:41:18.619 | INFO | } | 20260505 20:41:18.619 | INFO | Perf report for the last 5.13434 seconds: | 20260505 20:41:18.619 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:41:18.619 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:18.619 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:18.619 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:18.619 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 20:41:18.619 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:18.619 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:18.619 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260505 20:41:18.620 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:18.620 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:18.620 | INFO | Uploaded 36GiB to GPU in 2.11568s (17.0158GiB/s) | 20260505 20:41:18.620 | INFO | Downloaded 34GiB to GPU in 2.13591s (15.9183GiB/s) | 20260505 20:41:23.765 | INFO | Analyzing chunk frames = [6155776,6156288) | 20260505 20:41:23.765 | INFO | MemInfo: { | 20260505 20:41:23.765 | INFO | "gpu" : " 17 GB", | 20260505 20:41:23.765 | INFO | "huge" : "", | 20260505 20:41:23.765 | INFO | "huge_shared" : " 27 GB", | 20260505 20:41:23.765 | INFO | "malloc" : " 14 GB", | 20260505 20:41:23.765 | INFO | "pinned" : " 29 GB" | 20260505 20:41:23.765 | INFO | } | 20260505 20:41:23.765 | INFO | Perf report for the last 5.1459 seconds: | 20260505 20:41:23.765 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:41:23.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:23.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:23.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:23.765 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 20:41:23.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:23.765 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:23.765 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 20:41:23.765 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:23.765 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:23.766 | INFO | Uploaded 36GiB to GPU in 2.20489s (16.3273GiB/s) | 20260505 20:41:23.766 | INFO | Downloaded 34GiB to GPU in 2.04584s (16.6191GiB/s) | 20260505 20:41:23.895 | INFO | Starting a round of Baz IO | 20260505 20:41:27.045 | INFO | Populating RTMetrics report | 20260505 20:41:27.045 | INFO | RTMetrics report populated | 20260505 20:41:27.045 | INFO | Opening tmp RTMetrics json file | 20260505 20:41:27.046 | INFO | Serializing RTMetrics to json | 20260505 20:41:27.047 | INFO | Serialization done, swapping tmp file now... | 20260505 20:41:27.048 | INFO | RTMetrics done | 20260505 20:41:27.129 | INFO | Finished a round of Baz IO: Wrote 2.002 GiB in 3.234 seconds | 20260505 20:41:27.129 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.65% / 0.00% / 78.35% | 20260505 20:41:27.129 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.168526% / 3.97364e-06% / 0.000135104% | 20260505 20:41:27.130 | INFO | Baz IO overall: | 20260505 20:41:27.130 | INFO | BytesWritten : 3582192183416 | 20260505 20:41:27.130 | INFO | NumEvents : 1692946358129 | 20260505 20:41:28.904 | INFO | Analyzing chunk frames = [6156288,6156800) | 20260505 20:41:28.904 | INFO | MemInfo: { | 20260505 20:41:28.904 | INFO | "gpu" : " 17 GB", | 20260505 20:41:28.904 | INFO | "huge" : "", | 20260505 20:41:28.904 | INFO | "huge_shared" : " 27 GB", | 20260505 20:41:28.904 | INFO | "malloc" : " 14 GB", | 20260505 20:41:28.904 | INFO | "pinned" : " 29 GB" | 20260505 20:41:28.904 | INFO | } | 20260505 20:41:28.905 | INFO | Perf report for the last 5.13927 seconds: | 20260505 20:41:28.905 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:41:28.905 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:28.905 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:28.905 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:28.905 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260505 20:41:28.905 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:28.905 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260505 20:41:28.905 | INFO | PRE_HQ : 60.1%, 1.8, 55.0ms, 0 | 20260505 20:41:28.905 | INFO | BAZWRITER : 62.4%, 3.7, 118.5ms, 0 | 20260505 20:41:28.905 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 20:41:28.905 | INFO | Uploaded 36GiB to GPU in 2.1458s (16.777GiB/s) | 20260505 20:41:28.905 | INFO | Downloaded 38GiB to GPU in 2.24385s (16.9351GiB/s) | 20260505 20:41:34.038 | INFO | Analyzing chunk frames = [6156800,6157312) | 20260505 20:41:34.038 | INFO | MemInfo: { | 20260505 20:41:34.038 | INFO | "gpu" : " 17 GB", | 20260505 20:41:34.038 | INFO | "huge" : "", | 20260505 20:41:34.038 | INFO | "huge_shared" : " 27 GB", | 20260505 20:41:34.038 | INFO | "malloc" : " 14 GB", | 20260505 20:41:34.038 | INFO | "pinned" : " 29 GB" | 20260505 20:41:34.038 | INFO | } | 20260505 20:41:34.039 | INFO | Perf report for the last 5.134 seconds: | 20260505 20:41:34.039 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:41:34.039 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:34.039 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:34.039 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:34.039 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 20:41:34.039 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:34.039 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:34.039 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 20:41:34.039 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:34.039 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:34.039 | INFO | Uploaded 36GiB to GPU in 2.08342s (17.2793GiB/s) | 20260505 20:41:34.039 | INFO | Downloaded 34GiB to GPU in 2.02515s (16.7888GiB/s) | 20260505 20:41:39.178 | INFO | Analyzing chunk frames = [6157312,6157824) | 20260505 20:41:39.178 | INFO | MemInfo: { | 20260505 20:41:39.178 | INFO | "gpu" : " 17 GB", | 20260505 20:41:39.178 | INFO | "huge" : "", | 20260505 20:41:39.178 | INFO | "huge_shared" : " 27 GB", | 20260505 20:41:39.178 | INFO | "malloc" : " 14 GB", | 20260505 20:41:39.178 | INFO | "pinned" : " 29 GB" | 20260505 20:41:39.178 | INFO | } | 20260505 20:41:39.178 | INFO | Perf report for the last 5.1395 seconds: | 20260505 20:41:39.178 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:41:39.178 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:39.178 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:39.178 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:39.178 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260505 20:41:39.178 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:39.178 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:39.178 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 20:41:39.178 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:39.178 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:39.178 | INFO | Uploaded 36GiB to GPU in 2.12178s (16.9669GiB/s) | 20260505 20:41:39.178 | INFO | Downloaded 34GiB to GPU in 2.01506s (16.8729GiB/s) | 20260505 20:41:44.320 | INFO | Analyzing chunk frames = [6157824,6158336) | 20260505 20:41:44.320 | INFO | MemInfo: { | 20260505 20:41:44.320 | INFO | "gpu" : " 17 GB", | 20260505 20:41:44.320 | INFO | "huge" : "", | 20260505 20:41:44.320 | INFO | "huge_shared" : " 27 GB", | 20260505 20:41:44.320 | INFO | "malloc" : " 14 GB", | 20260505 20:41:44.320 | INFO | "pinned" : " 29 GB" | 20260505 20:41:44.320 | INFO | } | 20260505 20:41:44.320 | INFO | Perf report for the last 5.14239 seconds: | 20260505 20:41:44.321 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:41:44.321 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:44.321 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:44.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:44.321 | INFO | ANALYSIS : 62.4%, 3.0, 97.0ms, 0 | 20260505 20:41:44.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:44.321 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:44.321 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:41:44.321 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:44.321 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:44.321 | INFO | Uploaded 36GiB to GPU in 2.21224s (16.2731GiB/s) | 20260505 20:41:44.321 | INFO | Downloaded 34GiB to GPU in 2.06227s (16.4866GiB/s) | 20260505 20:41:49.460 | INFO | Analyzing chunk frames = [6158336,6158848) | 20260505 20:41:49.460 | INFO | MemInfo: { | 20260505 20:41:49.460 | INFO | "gpu" : " 17 GB", | 20260505 20:41:49.460 | INFO | "huge" : "", | 20260505 20:41:49.460 | INFO | "huge_shared" : " 27 GB", | 20260505 20:41:49.460 | INFO | "malloc" : " 14 GB", | 20260505 20:41:49.460 | INFO | "pinned" : " 29 GB" | 20260505 20:41:49.460 | INFO | } | 20260505 20:41:49.461 | INFO | Perf report for the last 5.14005 seconds: | 20260505 20:41:49.461 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:41:49.461 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:49.461 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:49.461 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:49.461 | INFO | ANALYSIS : 61.6%, 3.0, 96.2ms, 0 | 20260505 20:41:49.461 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:49.461 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:49.461 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 20:41:49.461 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:49.461 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:49.461 | INFO | Uploaded 36GiB to GPU in 2.12778s (16.9191GiB/s) | 20260505 20:41:49.461 | INFO | Downloaded 34GiB to GPU in 2.06174s (16.4909GiB/s) | 20260505 20:41:54.595 | INFO | Analyzing chunk frames = [6158848,6159360) | 20260505 20:41:54.595 | INFO | MemInfo: { | 20260505 20:41:54.595 | INFO | "gpu" : " 17 GB", | 20260505 20:41:54.595 | INFO | "huge" : "", | 20260505 20:41:54.595 | INFO | "huge_shared" : " 27 GB", | 20260505 20:41:54.595 | INFO | "malloc" : " 14 GB", | 20260505 20:41:54.595 | INFO | "pinned" : " 29 GB" | 20260505 20:41:54.595 | INFO | } | 20260505 20:41:54.595 | INFO | Perf report for the last 5.13491 seconds: | 20260505 20:41:54.595 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:41:54.595 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:54.595 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:54.595 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:54.596 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260505 20:41:54.596 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:54.596 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:54.596 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:41:54.596 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:54.596 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:54.596 | INFO | Uploaded 36GiB to GPU in 2.08043s (17.3041GiB/s) | 20260505 20:41:54.596 | INFO | Downloaded 34GiB to GPU in 2.03698s (16.6914GiB/s) | 20260505 20:41:59.730 | INFO | Analyzing chunk frames = [6159360,6159872) | 20260505 20:41:59.730 | INFO | MemInfo: { | 20260505 20:41:59.730 | INFO | "gpu" : " 17 GB", | 20260505 20:41:59.730 | INFO | "huge" : "", | 20260505 20:41:59.730 | INFO | "huge_shared" : " 27 GB", | 20260505 20:41:59.730 | INFO | "malloc" : " 14 GB", | 20260505 20:41:59.730 | INFO | "pinned" : " 29 GB" | 20260505 20:41:59.730 | INFO | } | 20260505 20:41:59.730 | INFO | Perf report for the last 5.13468 seconds: | 20260505 20:41:59.730 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:41:59.730 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:59.730 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:59.730 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:59.730 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 20:41:59.730 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:59.730 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:59.730 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 20:41:59.730 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:59.730 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:41:59.730 | INFO | Uploaded 36GiB to GPU in 2.13154s (16.8892GiB/s) | 20260505 20:41:59.731 | INFO | Downloaded 34GiB to GPU in 2.15107s (15.8061GiB/s) | 20260505 20:42:04.870 | INFO | Analyzing chunk frames = [6159872,6160384) | 20260505 20:42:04.870 | INFO | MemInfo: { | 20260505 20:42:04.870 | INFO | "gpu" : " 17 GB", | 20260505 20:42:04.870 | INFO | "huge" : "", | 20260505 20:42:04.870 | INFO | "huge_shared" : " 27 GB", | 20260505 20:42:04.870 | INFO | "malloc" : " 14 GB", | 20260505 20:42:04.870 | INFO | "pinned" : " 29 GB" | 20260505 20:42:04.870 | INFO | } | 20260505 20:42:04.870 | INFO | Perf report for the last 5.13957 seconds: | 20260505 20:42:04.870 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:42:04.870 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:04.870 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:04.870 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:04.870 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 20:42:04.870 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:04.870 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:04.870 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:42:04.870 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:04.870 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:04.870 | INFO | Uploaded 36GiB to GPU in 2.2017s (16.351GiB/s) | 20260505 20:42:04.870 | INFO | Downloaded 34GiB to GPU in 2.04292s (16.6429GiB/s) | 20260505 20:42:04.870 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 20:42:04.870 | INFO | Total mem usage: 15.3 GiB | 20260505 20:42:04.870 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 20:42:04.870 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.15%) | 20260505 20:42:04.870 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 20:42:04.870 | INFO | High Water Breakdown: | 20260505 20:42:04.870 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 20:42:04.870 | INFO | 38.8% : 5.5 GiB ( 82.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 20:42:04.870 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 20:42:04.870 | INFO | ------------------------------------------ | 20260505 20:42:04.870 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:42:04.870 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 20:42:04.870 | INFO | Total mem usage: 26.5 GiB | 20260505 20:42:04.870 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 20:42:04.870 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 20:42:04.870 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 20:42:04.870 | INFO | High Water Breakdown: | 20260505 20:42:04.870 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:42:04.870 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:42:04.870 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:42:04.870 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:42:04.870 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:42:04.870 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:42:04.870 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 20:42:04.870 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 20:42:04.871 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:42:04.871 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 20:42:04.871 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 20:42:04.871 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 20:42:04.871 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 20:42:04.871 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 20:42:04.871 | INFO | ------------------------------------------ | 20260505 20:42:04.871 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:42:04.871 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 20:42:04.871 | INFO | Total mem usage: 25.6 GiB | 20260505 20:42:04.871 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 20:42:04.871 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 20:42:04.871 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 20:42:04.871 | INFO | High Water Breakdown: | 20260505 20:42:04.871 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 20:42:04.871 | INFO | ------------------------------------------ | 20260505 20:42:04.871 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:42:04.871 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 20:42:04.871 | INFO | Total mem usage: 15.7 GiB | 20260505 20:42:04.871 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 20:42:04.871 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 20:42:04.871 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 20:42:04.871 | INFO | High Water Breakdown: | 20260505 20:42:04.871 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 20:42:04.871 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 20:42:04.871 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 20:42:04.871 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 20:42:04.871 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 20:42:04.871 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 20:42:04.871 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 20:42:04.871 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 20:42:04.871 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 20:42:04.871 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:42:04.871 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 20:42:04.871 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:42:04.871 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 20:42:04.871 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 20:42:04.871 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 20:42:04.871 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 20:42:04.871 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:42:04.871 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 20:42:04.871 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:42:04.871 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:42:04.871 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:42:04.871 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 20:42:04.871 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 20:42:04.871 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:42:04.871 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 20:42:04.871 | INFO | ------------------------------------------ | 20260505 20:42:04.871 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:42:04.871 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 20:42:04.871 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:42:04.871 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:42:04.871 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.46e+01, 1.35e+01 / 1.80e+00 | 20260505 20:42:04.871 | INFO | PerfReport: 0.29%, AggregateStats -- 9.25e-02 / 1.04e-01, 1.12e-02 / 5.87e-03 | 20260505 20:42:04.871 | INFO | PerfReport: 11.79%, Baseline -- 3.71e+00 / 4.56e+00, 5.63e-01 / 1.32e-01 | 20260505 20:42:04.871 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.52e+00, 1.87e-01 / 9.02e-02 | 20260505 20:42:04.871 | INFO | PerfReport: 15.96%, DME -- 4.02e+01 / 4.44e+01, 4.05e+00 / 1.84e+00 | 20260505 20:42:04.871 | INFO | PerfReport: 57.12%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.48e+00 / 1.22e+00 | 20260505 20:42:04.871 | INFO | PerfReport: 7.22%, PulseAccumulating -- 2.27e+00 / 2.82e+00, 4.35e-01 / 2.64e-01 | 20260505 20:42:04.871 | INFO | PerfReport: 3.23%, Metrics -- 1.02e+00 / 1.32e+00, 2.74e-01 / 1.30e-01 | 20260505 20:42:04.871 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.53e-01 / 4.08e-01, 6.15e-02 / 2.77e-02 | 20260505 20:42:04.871 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 20:42:04.871 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:42:04.871 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:42:04.871 | INFO | PerfReport: Full Execution -- 9.92e+01 / -nan, 2.53e+01 / 0.00e+00 | 20260505 20:42:04.871 | INFO | PerfReport: 22.47%, Upload -- 2.23e+01 / 0.00e+00, 7.71e+00 / 0.00e+00 | 20260505 20:42:04.871 | INFO | PerfReport: 21.82%, Download -- 2.17e+01 / 0.00e+00, 7.70e+00 / 0.00e+00 | 20260505 20:42:04.871 | INFO | PerfReport: 31.69%, Compute -- 3.15e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 20:42:04.871 | INFO | PerfReport: 2.26%, LockUpload -- 2.25e+00 / 0.00e+00, 7.95e+00 / 0.00e+00 | 20260505 20:42:04.871 | INFO | PerfReport: 1.37%, LockDownload -- 1.36e+00 / 0.00e+00, 2.90e+00 / 0.00e+00 | 20260505 20:42:04.871 | INFO | PerfReport: 20.36%, LockCompute -- 2.02e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 20:42:04.871 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 2.09e-02 / 0.00e+00 | 20260505 20:42:04.999 | INFO | Starting a round of Baz IO | 20260505 20:42:08.166 | INFO | Populating RTMetrics report | 20260505 20:42:08.167 | INFO | RTMetrics report populated | 20260505 20:42:08.167 | INFO | Opening tmp RTMetrics json file | 20260505 20:42:08.168 | INFO | Serializing RTMetrics to json | 20260505 20:42:08.169 | INFO | Serialization done, swapping tmp file now... | 20260505 20:42:08.169 | INFO | RTMetrics done | 20260505 20:42:08.253 | INFO | Finished a round of Baz IO: Wrote 2.004 GiB in 3.254 seconds | 20260505 20:42:08.253 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.64% / 0.00% / 78.35% | 20260505 20:42:08.253 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.168761% / 0% / 5.16574e-05% | 20260505 20:42:08.253 | INFO | Baz IO overall: | 20260505 20:42:08.253 | INFO | BytesWritten : 3584343627293 | 20260505 20:42:08.253 | INFO | NumEvents : 1693942594314 | 20260505 20:42:10.016 | INFO | Analyzing chunk frames = [6160384,6160896) | 20260505 20:42:10.016 | INFO | MemInfo: { | 20260505 20:42:10.016 | INFO | "gpu" : " 17 GB", | 20260505 20:42:10.016 | INFO | "huge" : "", | 20260505 20:42:10.016 | INFO | "huge_shared" : " 27 GB", | 20260505 20:42:10.016 | INFO | "malloc" : " 14 GB", | 20260505 20:42:10.016 | INFO | "pinned" : " 29 GB" | 20260505 20:42:10.016 | INFO | } | 20260505 20:42:10.016 | INFO | Perf report for the last 5.14642 seconds: | 20260505 20:42:10.016 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:42:10.016 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:10.016 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:10.016 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:10.016 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 20:42:10.016 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:10.016 | INFO | RT_METRICS : 43.9%, 1.0, 23.1ms, 0 | 20260505 20:42:10.016 | INFO | PRE_HQ : 59.9%, 1.8, 55.0ms, 0 | 20260505 20:42:10.016 | INFO | BAZWRITER : 62.8%, 3.7, 118.0ms, 0 | 20260505 20:42:10.016 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 20:42:10.016 | INFO | Uploaded 36GiB to GPU in 2.1514s (16.7333GiB/s) | 20260505 20:42:10.016 | INFO | Downloaded 38GiB to GPU in 2.2507s (16.8836GiB/s) | 20260505 20:42:15.158 | INFO | Analyzing chunk frames = [6160896,6161408) | 20260505 20:42:15.158 | INFO | MemInfo: { | 20260505 20:42:15.158 | INFO | "gpu" : " 17 GB", | 20260505 20:42:15.158 | INFO | "huge" : "", | 20260505 20:42:15.158 | INFO | "huge_shared" : " 27 GB", | 20260505 20:42:15.158 | INFO | "malloc" : " 14 GB", | 20260505 20:42:15.158 | INFO | "pinned" : " 29 GB" | 20260505 20:42:15.158 | INFO | } | 20260505 20:42:15.158 | INFO | Perf report for the last 5.14206 seconds: | 20260505 20:42:15.158 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:42:15.158 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:15.158 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:15.158 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:15.158 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 20:42:15.158 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:15.158 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:15.158 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 20:42:15.158 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:15.158 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:15.159 | INFO | Uploaded 36GiB to GPU in 2.0865s (17.2538GiB/s) | 20260505 20:42:15.159 | INFO | Downloaded 34GiB to GPU in 2.02306s (16.8062GiB/s) | 20260505 20:42:20.299 | INFO | Analyzing chunk frames = [6161408,6161920) | 20260505 20:42:20.299 | INFO | MemInfo: { | 20260505 20:42:20.299 | INFO | "gpu" : " 17 GB", | 20260505 20:42:20.299 | INFO | "huge" : "", | 20260505 20:42:20.299 | INFO | "huge_shared" : " 27 GB", | 20260505 20:42:20.299 | INFO | "malloc" : " 14 GB", | 20260505 20:42:20.299 | INFO | "pinned" : " 29 GB" | 20260505 20:42:20.300 | INFO | } | 20260505 20:42:20.300 | INFO | Perf report for the last 5.14131 seconds: | 20260505 20:42:20.300 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:42:20.300 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:20.300 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:20.300 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:20.300 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260505 20:42:20.300 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:20.300 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:20.300 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:42:20.300 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:20.300 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:20.300 | INFO | Uploaded 36GiB to GPU in 2.14004s (16.8221GiB/s) | 20260505 20:42:20.300 | INFO | Downloaded 34GiB to GPU in 2.03971s (16.669GiB/s) | 20260505 20:42:25.427 | INFO | Analyzing chunk frames = [6161920,6162432) | 20260505 20:42:25.427 | INFO | MemInfo: { | 20260505 20:42:25.427 | INFO | "gpu" : " 17 GB", | 20260505 20:42:25.427 | INFO | "huge" : "", | 20260505 20:42:25.427 | INFO | "huge_shared" : " 27 GB", | 20260505 20:42:25.428 | INFO | "malloc" : " 14 GB", | 20260505 20:42:25.428 | INFO | "pinned" : " 29 GB" | 20260505 20:42:25.428 | INFO | } | 20260505 20:42:25.428 | INFO | Perf report for the last 5.12801 seconds: | 20260505 20:42:25.428 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:42:25.428 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:25.428 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:25.428 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:25.428 | INFO | ANALYSIS : 62.6%, 3.0, 97.0ms, 0 | 20260505 20:42:25.428 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:25.428 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:25.428 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:42:25.428 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:25.428 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:25.428 | INFO | Uploaded 36GiB to GPU in 2.21267s (16.2699GiB/s) | 20260505 20:42:25.428 | INFO | Downloaded 34GiB to GPU in 2.04506s (16.6254GiB/s) | 20260505 20:42:30.572 | INFO | Analyzing chunk frames = [6162432,6162944) | 20260505 20:42:30.572 | INFO | MemInfo: { | 20260505 20:42:30.572 | INFO | "gpu" : " 17 GB", | 20260505 20:42:30.572 | INFO | "huge" : "", | 20260505 20:42:30.572 | INFO | "huge_shared" : " 27 GB", | 20260505 20:42:30.572 | INFO | "malloc" : " 14 GB", | 20260505 20:42:30.572 | INFO | "pinned" : " 29 GB" | 20260505 20:42:30.572 | INFO | } | 20260505 20:42:30.572 | INFO | Perf report for the last 5.14468 seconds: | 20260505 20:42:30.572 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:42:30.572 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:30.572 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:30.572 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:30.572 | INFO | ANALYSIS : 61.8%, 3.0, 96.6ms, 0 | 20260505 20:42:30.572 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:30.572 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:30.572 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:42:30.572 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:30.572 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:30.573 | INFO | Uploaded 36GiB to GPU in 2.14328s (16.7967GiB/s) | 20260505 20:42:30.573 | INFO | Downloaded 34GiB to GPU in 2.06489s (16.4657GiB/s) | 20260505 20:42:35.713 | INFO | Analyzing chunk frames = [6162944,6163456) | 20260505 20:42:35.713 | INFO | MemInfo: { | 20260505 20:42:35.713 | INFO | "gpu" : " 17 GB", | 20260505 20:42:35.713 | INFO | "huge" : "", | 20260505 20:42:35.713 | INFO | "huge_shared" : " 27 GB", | 20260505 20:42:35.713 | INFO | "malloc" : " 14 GB", | 20260505 20:42:35.713 | INFO | "pinned" : " 29 GB" | 20260505 20:42:35.713 | INFO | } | 20260505 20:42:35.713 | INFO | Perf report for the last 5.14034 seconds: | 20260505 20:42:35.713 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:42:35.713 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:35.713 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:35.713 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:35.713 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 20:42:35.713 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:35.713 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:35.713 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 20:42:35.713 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:35.713 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:35.713 | INFO | Uploaded 36GiB to GPU in 2.08583s (17.2593GiB/s) | 20260505 20:42:35.713 | INFO | Downloaded 34GiB to GPU in 2.03598s (16.6996GiB/s) | 20260505 20:42:40.851 | INFO | Analyzing chunk frames = [6163456,6163968) | 20260505 20:42:40.851 | INFO | MemInfo: { | 20260505 20:42:40.851 | INFO | "gpu" : " 17 GB", | 20260505 20:42:40.851 | INFO | "huge" : "", | 20260505 20:42:40.851 | INFO | "huge_shared" : " 27 GB", | 20260505 20:42:40.851 | INFO | "malloc" : " 14 GB", | 20260505 20:42:40.851 | INFO | "pinned" : " 29 GB" | 20260505 20:42:40.851 | INFO | } | 20260505 20:42:40.851 | INFO | Perf report for the last 5.13828 seconds: | 20260505 20:42:40.851 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:42:40.851 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:40.851 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:40.851 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:40.851 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 20:42:40.851 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:40.851 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:40.851 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 20:42:40.851 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:40.851 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:40.851 | INFO | Uploaded 36GiB to GPU in 2.16316s (16.6424GiB/s) | 20260505 20:42:40.851 | INFO | Downloaded 34GiB to GPU in 2.1231s (16.0143GiB/s) | 20260505 20:42:45.987 | INFO | Analyzing chunk frames = [6163968,6164480) | 20260505 20:42:45.987 | INFO | MemInfo: { | 20260505 20:42:45.987 | INFO | "gpu" : " 17 GB", | 20260505 20:42:45.987 | INFO | "huge" : "", | 20260505 20:42:45.987 | INFO | "huge_shared" : " 27 GB", | 20260505 20:42:45.987 | INFO | "malloc" : " 14 GB", | 20260505 20:42:45.987 | INFO | "pinned" : " 29 GB" | 20260505 20:42:45.987 | INFO | } | 20260505 20:42:45.987 | INFO | Perf report for the last 5.13654 seconds: | 20260505 20:42:45.987 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:42:45.987 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:45.987 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:45.988 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:45.988 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 20:42:45.988 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:45.988 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:45.988 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:42:45.988 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:45.988 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:45.988 | INFO | Uploaded 36GiB to GPU in 2.12157s (16.9685GiB/s) | 20260505 20:42:45.988 | INFO | Downloaded 34GiB to GPU in 2.00605s (16.9487GiB/s) | 20260505 20:42:46.117 | INFO | Starting a round of Baz IO | 20260505 20:42:49.293 | INFO | Populating RTMetrics report | 20260505 20:42:49.293 | INFO | RTMetrics report populated | 20260505 20:42:49.293 | INFO | Opening tmp RTMetrics json file | 20260505 20:42:49.294 | INFO | Serializing RTMetrics to json | 20260505 20:42:49.295 | INFO | Serialization done, swapping tmp file now... | 20260505 20:42:49.296 | INFO | RTMetrics done | 20260505 20:42:49.386 | INFO | Finished a round of Baz IO: Wrote 2.004 GiB in 3.269 seconds | 20260505 20:42:49.386 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.64% / 0.00% / 78.36% | 20260505 20:42:49.386 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.167318% / 0% / 7.94729e-05% | 20260505 20:42:49.386 | INFO | Baz IO overall: | 20260505 20:42:49.386 | INFO | BytesWritten : 3586495370965 | 20260505 20:42:49.386 | INFO | NumEvents : 1694937468793 | 20260505 20:42:51.129 | INFO | Analyzing chunk frames = [6164480,6164992) | 20260505 20:42:51.129 | INFO | MemInfo: { | 20260505 20:42:51.129 | INFO | "gpu" : " 17 GB", | 20260505 20:42:51.129 | INFO | "huge" : "", | 20260505 20:42:51.129 | INFO | "huge_shared" : " 27 GB", | 20260505 20:42:51.129 | INFO | "malloc" : " 14 GB", | 20260505 20:42:51.129 | INFO | "pinned" : " 29 GB" | 20260505 20:42:51.129 | INFO | } | 20260505 20:42:51.129 | INFO | Perf report for the last 5.14199 seconds: | 20260505 20:42:51.129 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:42:51.129 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:51.129 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:51.130 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:51.130 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 20:42:51.130 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:51.130 | INFO | RT_METRICS : 44.1%, 1.0, 23.2ms, 0 | 20260505 20:42:51.130 | INFO | PRE_HQ : 60.3%, 1.8, 55.6ms, 0 | 20260505 20:42:51.130 | INFO | BAZWRITER : 63.1%, 3.7, 119.6ms, 0 | 20260505 20:42:51.130 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 20:42:51.130 | INFO | Uploaded 36GiB to GPU in 2.17063s (16.585GiB/s) | 20260505 20:42:51.130 | INFO | Downloaded 38GiB to GPU in 2.24678s (16.9131GiB/s) | 20260505 20:42:56.275 | INFO | Analyzing chunk frames = [6164992,6165504) | 20260505 20:42:56.275 | INFO | MemInfo: { | 20260505 20:42:56.275 | INFO | "gpu" : " 17 GB", | 20260505 20:42:56.275 | INFO | "huge" : "", | 20260505 20:42:56.275 | INFO | "huge_shared" : " 27 GB", | 20260505 20:42:56.275 | INFO | "malloc" : " 14 GB", | 20260505 20:42:56.275 | INFO | "pinned" : " 29 GB" | 20260505 20:42:56.275 | INFO | } | 20260505 20:42:56.275 | INFO | Perf report for the last 5.14584 seconds: | 20260505 20:42:56.275 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:42:56.275 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:56.275 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:56.275 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:56.275 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 20:42:56.275 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:56.275 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:56.275 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 20:42:56.275 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:56.275 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:42:56.276 | INFO | Uploaded 36GiB to GPU in 2.09125s (17.2146GiB/s) | 20260505 20:42:56.276 | INFO | Downloaded 34GiB to GPU in 2.0257s (16.7843GiB/s) | 20260505 20:43:01.409 | INFO | Analyzing chunk frames = [6165504,6166016) | 20260505 20:43:01.409 | INFO | MemInfo: { | 20260505 20:43:01.409 | INFO | "gpu" : " 17 GB", | 20260505 20:43:01.409 | INFO | "huge" : "", | 20260505 20:43:01.409 | INFO | "huge_shared" : " 27 GB", | 20260505 20:43:01.409 | INFO | "malloc" : " 14 GB", | 20260505 20:43:01.409 | INFO | "pinned" : " 29 GB" | 20260505 20:43:01.409 | INFO | } | 20260505 20:43:01.409 | INFO | Perf report for the last 5.13381 seconds: | 20260505 20:43:01.409 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:43:01.409 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:01.409 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:01.409 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:01.409 | INFO | ANALYSIS : 62.1%, 3.0, 96.5ms, 0 | 20260505 20:43:01.409 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:01.409 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:01.409 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:43:01.409 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:01.409 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:01.409 | INFO | Uploaded 36GiB to GPU in 2.13331s (16.8752GiB/s) | 20260505 20:43:01.409 | INFO | Downloaded 34GiB to GPU in 2.02079s (16.8251GiB/s) | 20260505 20:43:06.546 | INFO | Analyzing chunk frames = [6166016,6166528) | 20260505 20:43:06.546 | INFO | MemInfo: { | 20260505 20:43:06.546 | INFO | "gpu" : " 17 GB", | 20260505 20:43:06.546 | INFO | "huge" : "", | 20260505 20:43:06.546 | INFO | "huge_shared" : " 27 GB", | 20260505 20:43:06.546 | INFO | "malloc" : " 14 GB", | 20260505 20:43:06.546 | INFO | "pinned" : " 29 GB" | 20260505 20:43:06.546 | INFO | } | 20260505 20:43:06.546 | INFO | Perf report for the last 5.13713 seconds: | 20260505 20:43:06.546 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:43:06.546 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:06.546 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:06.546 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:06.546 | INFO | ANALYSIS : 62.5%, 3.0, 97.0ms, 0 | 20260505 20:43:06.546 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:06.546 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:06.546 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:43:06.546 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:06.546 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:06.547 | INFO | Uploaded 36GiB to GPU in 2.21433s (16.2577GiB/s) | 20260505 20:43:06.547 | INFO | Downloaded 34GiB to GPU in 2.06788s (16.4419GiB/s) | 20260505 20:43:11.682 | INFO | Analyzing chunk frames = [6166528,6167040) | 20260505 20:43:11.682 | INFO | MemInfo: { | 20260505 20:43:11.682 | INFO | "gpu" : " 17 GB", | 20260505 20:43:11.682 | INFO | "huge" : "", | 20260505 20:43:11.683 | INFO | "huge_shared" : " 27 GB", | 20260505 20:43:11.683 | INFO | "malloc" : " 14 GB", | 20260505 20:43:11.683 | INFO | "pinned" : " 29 GB" | 20260505 20:43:11.683 | INFO | } | 20260505 20:43:11.683 | INFO | Perf report for the last 5.13636 seconds: | 20260505 20:43:11.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:43:11.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:11.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:11.683 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:11.683 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 20:43:11.683 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:11.683 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:11.683 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 20:43:11.683 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:11.683 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:11.683 | INFO | Uploaded 36GiB to GPU in 2.14681s (16.769GiB/s) | 20260505 20:43:11.683 | INFO | Downloaded 34GiB to GPU in 2.06321s (16.4792GiB/s) | 20260505 20:43:16.822 | INFO | Analyzing chunk frames = [6167040,6167552) | 20260505 20:43:16.822 | INFO | MemInfo: { | 20260505 20:43:16.822 | INFO | "gpu" : " 17 GB", | 20260505 20:43:16.822 | INFO | "huge" : "", | 20260505 20:43:16.822 | INFO | "huge_shared" : " 27 GB", | 20260505 20:43:16.822 | INFO | "malloc" : " 14 GB", | 20260505 20:43:16.822 | INFO | "pinned" : " 29 GB" | 20260505 20:43:16.822 | INFO | } | 20260505 20:43:16.822 | INFO | Perf report for the last 5.1397 seconds: | 20260505 20:43:16.822 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:43:16.822 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:16.822 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:16.822 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:16.822 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260505 20:43:16.822 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:16.822 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:16.822 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:43:16.823 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:16.823 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:16.823 | INFO | Uploaded 36GiB to GPU in 2.16187s (16.6522GiB/s) | 20260505 20:43:16.823 | INFO | Downloaded 34GiB to GPU in 2.06598s (16.4571GiB/s) | 20260505 20:43:21.965 | INFO | Analyzing chunk frames = [6167552,6168064) | 20260505 20:43:21.965 | INFO | MemInfo: { | 20260505 20:43:21.965 | INFO | "gpu" : " 17 GB", | 20260505 20:43:21.965 | INFO | "huge" : "", | 20260505 20:43:21.965 | INFO | "huge_shared" : " 27 GB", | 20260505 20:43:21.965 | INFO | "malloc" : " 14 GB", | 20260505 20:43:21.965 | INFO | "pinned" : " 29 GB" | 20260505 20:43:21.965 | INFO | } | 20260505 20:43:21.965 | INFO | Perf report for the last 5.14302 seconds: | 20260505 20:43:21.965 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:43:21.965 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:21.965 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:21.965 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:21.965 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 20:43:21.965 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:21.965 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:21.966 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 20:43:21.966 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:21.966 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:21.966 | INFO | Uploaded 36GiB to GPU in 2.11324s (17.0355GiB/s) | 20260505 20:43:21.966 | INFO | Downloaded 34GiB to GPU in 2.14891s (15.8219GiB/s) | 20260505 20:43:27.111 | INFO | Analyzing chunk frames = [6168064,6168576) | 20260505 20:43:27.111 | INFO | MemInfo: { | 20260505 20:43:27.111 | INFO | "gpu" : " 17 GB", | 20260505 20:43:27.111 | INFO | "huge" : "", | 20260505 20:43:27.111 | INFO | "huge_shared" : " 27 GB", | 20260505 20:43:27.112 | INFO | "malloc" : " 14 GB", | 20260505 20:43:27.112 | INFO | "pinned" : " 29 GB" | 20260505 20:43:27.112 | INFO | } | 20260505 20:43:27.112 | INFO | Perf report for the last 5.14621 seconds: | 20260505 20:43:27.112 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:43:27.112 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:27.112 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:27.112 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:27.112 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260505 20:43:27.112 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:27.112 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:27.112 | INFO | PRE_HQ : 27.2%, 1.0, 14.2ms, 0 | 20260505 20:43:27.112 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:27.112 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:27.112 | INFO | Uploaded 36GiB to GPU in 2.10548s (17.0983GiB/s) | 20260505 20:43:27.112 | INFO | Downloaded 34GiB to GPU in 2.0087s (16.9264GiB/s) | 20260505 20:43:27.112 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 20:43:27.112 | INFO | Total mem usage: 15.3 GiB | 20260505 20:43:27.112 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 20:43:27.112 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.7%) | 20260505 20:43:27.112 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 20:43:27.112 | INFO | High Water Breakdown: | 20260505 20:43:27.112 | INFO | 60.9% : 8.59 GiB ( 90.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 20:43:27.112 | INFO | 39% : 5.49 GiB ( 82.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 20:43:27.112 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 20:43:27.112 | INFO | ------------------------------------------ | 20260505 20:43:27.112 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:43:27.112 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 20:43:27.112 | INFO | Total mem usage: 26.5 GiB | 20260505 20:43:27.112 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 20:43:27.112 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 20:43:27.112 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 20:43:27.112 | INFO | High Water Breakdown: | 20260505 20:43:27.112 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:43:27.112 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:43:27.112 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:43:27.112 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:43:27.112 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:43:27.112 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:43:27.112 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 20:43:27.112 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 20:43:27.112 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:43:27.112 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 20:43:27.112 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 20:43:27.112 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 20:43:27.112 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 20:43:27.112 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 20:43:27.112 | INFO | ------------------------------------------ | 20260505 20:43:27.112 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:43:27.112 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 20:43:27.112 | INFO | Total mem usage: 25.6 GiB | 20260505 20:43:27.113 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 20:43:27.113 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 20:43:27.113 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 20:43:27.113 | INFO | High Water Breakdown: | 20260505 20:43:27.113 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 20:43:27.113 | INFO | ------------------------------------------ | 20260505 20:43:27.113 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:43:27.113 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 20:43:27.113 | INFO | Total mem usage: 15.7 GiB | 20260505 20:43:27.113 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 20:43:27.113 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 20:43:27.113 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 20:43:27.113 | INFO | High Water Breakdown: | 20260505 20:43:27.113 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 20:43:27.113 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 20:43:27.113 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 20:43:27.113 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 20:43:27.113 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 20:43:27.113 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 20:43:27.113 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 20:43:27.113 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 20:43:27.113 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 20:43:27.113 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:43:27.113 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 20:43:27.113 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:43:27.113 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 20:43:27.113 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 20:43:27.113 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 20:43:27.113 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 20:43:27.113 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:43:27.113 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 20:43:27.113 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:43:27.113 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:43:27.113 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:43:27.113 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 20:43:27.113 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 20:43:27.113 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:43:27.113 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 20:43:27.113 | INFO | ------------------------------------------ | 20260505 20:43:27.113 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:43:27.113 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 20:43:27.113 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:43:27.113 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:43:27.113 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.42e+01, 1.36e+01 / 1.52e+00 | 20260505 20:43:27.113 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.02e-01, 1.06e-02 / 3.66e-03 | 20260505 20:43:27.113 | INFO | PerfReport: 11.77%, Baseline -- 3.70e+00 / 4.58e+00, 5.64e-01 / 1.16e-01 | 20260505 20:43:27.113 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.52e+00, 1.91e-01 / 9.62e-02 | 20260505 20:43:27.113 | INFO | PerfReport: 15.93%, DME -- 4.01e+01 / 4.40e+01, 3.88e+00 / 1.59e+00 | 20260505 20:43:27.113 | INFO | PerfReport: 57.13%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.48e+00 / 9.78e-01 | 20260505 20:43:27.113 | INFO | PerfReport: 7.24%, PulseAccumulating -- 2.28e+00 / 2.85e+00, 4.35e-01 / 1.77e-01 | 20260505 20:43:27.113 | INFO | PerfReport: 3.24%, Metrics -- 1.02e+00 / 1.34e+00, 2.77e-01 / 1.41e-01 | 20260505 20:43:27.113 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.54e-01 / 4.13e-01, 6.19e-02 / 3.53e-02 | 20260505 20:43:27.113 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 20:43:27.113 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:43:27.113 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:43:27.113 | INFO | PerfReport: Full Execution -- 9.92e+01 / -nan, 2.53e+01 / 0.00e+00 | 20260505 20:43:27.113 | INFO | PerfReport: 22.46%, Upload -- 2.23e+01 / 0.00e+00, 7.70e+00 / 0.00e+00 | 20260505 20:43:27.113 | INFO | PerfReport: 21.80%, Download -- 2.16e+01 / 0.00e+00, 7.72e+00 / 0.00e+00 | 20260505 20:43:27.113 | INFO | PerfReport: 31.68%, Compute -- 3.14e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 20:43:27.113 | INFO | PerfReport: 2.30%, LockUpload -- 2.29e+00 / 0.00e+00, 8.02e+00 / 0.00e+00 | 20260505 20:43:27.113 | INFO | PerfReport: 1.33%, LockDownload -- 1.32e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260505 20:43:27.113 | INFO | PerfReport: 20.41%, LockCompute -- 2.03e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 20:43:27.113 | INFO | PerfReport: 0.01%, Misc -- 1.25e-02 / 0.00e+00, 1.35e-02 / 0.00e+00 | 20260505 20:43:27.241 | INFO | Starting a round of Baz IO | 20260505 20:43:30.398 | INFO | Populating RTMetrics report | 20260505 20:43:30.398 | INFO | RTMetrics report populated | 20260505 20:43:30.398 | INFO | Opening tmp RTMetrics json file | 20260505 20:43:30.399 | INFO | Serializing RTMetrics to json | 20260505 20:43:30.401 | INFO | Serialization done, swapping tmp file now... | 20260505 20:43:30.401 | INFO | RTMetrics done | 20260505 20:43:30.491 | INFO | Finished a round of Baz IO: Wrote 2.005 GiB in 3.249 seconds | 20260505 20:43:30.491 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.64% / 0.00% / 78.36% | 20260505 20:43:30.491 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.167028% / 3.97364e-06% / 7.15256e-05% | 20260505 20:43:30.491 | INFO | Baz IO overall: | 20260505 20:43:30.491 | INFO | BytesWritten : 3588648491493 | 20260505 20:43:30.491 | INFO | NumEvents : 1695933559837 | 20260505 20:43:32.245 | INFO | Analyzing chunk frames = [6168576,6169088) | 20260505 20:43:32.245 | INFO | MemInfo: { | 20260505 20:43:32.245 | INFO | "gpu" : " 17 GB", | 20260505 20:43:32.245 | INFO | "huge" : "", | 20260505 20:43:32.245 | INFO | "huge_shared" : " 27 GB", | 20260505 20:43:32.245 | INFO | "malloc" : " 14 GB", | 20260505 20:43:32.245 | INFO | "pinned" : " 29 GB" | 20260505 20:43:32.245 | INFO | } | 20260505 20:43:32.245 | INFO | Perf report for the last 5.13355 seconds: | 20260505 20:43:32.245 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:43:32.245 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:32.245 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:32.245 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:32.245 | INFO | ANALYSIS : 63.0%, 3.0, 98.0ms, 0 | 20260505 20:43:32.245 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:32.245 | INFO | RT_METRICS : 44.0%, 1.0, 23.0ms, 0 | 20260505 20:43:32.245 | INFO | PRE_HQ : 60.1%, 1.8, 55.1ms, 0 | 20260505 20:43:32.245 | INFO | BAZWRITER : 62.7%, 3.7, 120.6ms, 0 | 20260505 20:43:32.245 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 20:43:32.246 | INFO | Uploaded 36GiB to GPU in 2.11487s (17.0224GiB/s) | 20260505 20:43:32.246 | INFO | Downloaded 38GiB to GPU in 2.23645s (16.9912GiB/s) | 20260505 20:43:37.378 | INFO | Analyzing chunk frames = [6169088,6169600) | 20260505 20:43:37.378 | INFO | MemInfo: { | 20260505 20:43:37.378 | INFO | "gpu" : " 17 GB", | 20260505 20:43:37.378 | INFO | "huge" : "", | 20260505 20:43:37.378 | INFO | "huge_shared" : " 27 GB", | 20260505 20:43:37.378 | INFO | "malloc" : " 14 GB", | 20260505 20:43:37.378 | INFO | "pinned" : " 29 GB" | 20260505 20:43:37.378 | INFO | } | 20260505 20:43:37.378 | INFO | Perf report for the last 5.13308 seconds: | 20260505 20:43:37.378 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:43:37.378 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:37.378 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:37.378 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:37.378 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 20:43:37.378 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:37.378 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:37.378 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 20:43:37.378 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:37.378 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:37.378 | INFO | Uploaded 36GiB to GPU in 2.15868s (16.6768GiB/s) | 20260505 20:43:37.379 | INFO | Downloaded 34GiB to GPU in 2.05081s (16.5788GiB/s) | 20260505 20:43:42.524 | INFO | Analyzing chunk frames = [6169600,6170112) | 20260505 20:43:42.524 | INFO | MemInfo: { | 20260505 20:43:42.524 | INFO | "gpu" : " 17 GB", | 20260505 20:43:42.524 | INFO | "huge" : "", | 20260505 20:43:42.524 | INFO | "huge_shared" : " 27 GB", | 20260505 20:43:42.524 | INFO | "malloc" : " 14 GB", | 20260505 20:43:42.524 | INFO | "pinned" : " 29 GB" | 20260505 20:43:42.524 | INFO | } | 20260505 20:43:42.524 | INFO | Perf report for the last 5.1461 seconds: | 20260505 20:43:42.524 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:43:42.524 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:42.524 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:42.524 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:42.524 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260505 20:43:42.524 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:42.524 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:42.525 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:43:42.525 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:42.525 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:42.525 | INFO | Uploaded 36GiB to GPU in 2.14529s (16.781GiB/s) | 20260505 20:43:42.525 | INFO | Downloaded 34GiB to GPU in 2.04387s (16.6351GiB/s) | 20260505 20:43:47.665 | INFO | Analyzing chunk frames = [6170112,6170624) | 20260505 20:43:47.665 | INFO | MemInfo: { | 20260505 20:43:47.665 | INFO | "gpu" : " 17 GB", | 20260505 20:43:47.665 | INFO | "huge" : "", | 20260505 20:43:47.665 | INFO | "huge_shared" : " 27 GB", | 20260505 20:43:47.665 | INFO | "malloc" : " 14 GB", | 20260505 20:43:47.665 | INFO | "pinned" : " 29 GB" | 20260505 20:43:47.665 | INFO | } | 20260505 20:43:47.665 | INFO | Perf report for the last 5.14044 seconds: | 20260505 20:43:47.665 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:43:47.665 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:47.665 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:47.665 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:47.665 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260505 20:43:47.665 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:47.665 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:47.665 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:43:47.665 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:47.665 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:47.665 | INFO | Uploaded 36GiB to GPU in 2.14059s (16.8178GiB/s) | 20260505 20:43:47.665 | INFO | Downloaded 34GiB to GPU in 2.04145s (16.6549GiB/s) | 20260505 20:43:52.800 | INFO | Analyzing chunk frames = [6170624,6171136) | 20260505 20:43:52.800 | INFO | MemInfo: { | 20260505 20:43:52.800 | INFO | "gpu" : " 17 GB", | 20260505 20:43:52.800 | INFO | "huge" : "", | 20260505 20:43:52.800 | INFO | "huge_shared" : " 27 GB", | 20260505 20:43:52.800 | INFO | "malloc" : " 14 GB", | 20260505 20:43:52.800 | INFO | "pinned" : " 29 GB" | 20260505 20:43:52.800 | INFO | } | 20260505 20:43:52.800 | INFO | Perf report for the last 5.13537 seconds: | 20260505 20:43:52.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:43:52.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:52.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:52.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:52.800 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260505 20:43:52.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:52.800 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:52.800 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:43:52.800 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:52.800 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:52.800 | INFO | Uploaded 36GiB to GPU in 2.12844s (16.9138GiB/s) | 20260505 20:43:52.801 | INFO | Downloaded 34GiB to GPU in 2.06108s (16.4962GiB/s) | 20260505 20:43:57.938 | INFO | Analyzing chunk frames = [6171136,6171648) | 20260505 20:43:57.938 | INFO | MemInfo: { | 20260505 20:43:57.938 | INFO | "gpu" : " 17 GB", | 20260505 20:43:57.938 | INFO | "huge" : "", | 20260505 20:43:57.938 | INFO | "huge_shared" : " 27 GB", | 20260505 20:43:57.938 | INFO | "malloc" : " 14 GB", | 20260505 20:43:57.938 | INFO | "pinned" : " 29 GB" | 20260505 20:43:57.938 | INFO | } | 20260505 20:43:57.938 | INFO | Perf report for the last 5.13818 seconds: | 20260505 20:43:57.938 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:43:57.938 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:57.938 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:57.938 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:57.938 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260505 20:43:57.939 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:57.939 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:57.939 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:43:57.939 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:57.939 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:43:57.939 | INFO | Uploaded 36GiB to GPU in 2.15981s (16.6681GiB/s) | 20260505 20:43:57.939 | INFO | Downloaded 34GiB to GPU in 2.06284s (16.4821GiB/s) | 20260505 20:44:03.078 | INFO | Analyzing chunk frames = [6171648,6172160) | 20260505 20:44:03.078 | INFO | MemInfo: { | 20260505 20:44:03.078 | INFO | "gpu" : " 17 GB", | 20260505 20:44:03.078 | INFO | "huge" : "", | 20260505 20:44:03.078 | INFO | "huge_shared" : " 27 GB", | 20260505 20:44:03.078 | INFO | "malloc" : " 14 GB", | 20260505 20:44:03.078 | INFO | "pinned" : " 29 GB" | 20260505 20:44:03.078 | INFO | } | 20260505 20:44:03.078 | INFO | Perf report for the last 5.14004 seconds: | 20260505 20:44:03.078 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:44:03.078 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:03.078 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:03.079 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:03.079 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 20:44:03.079 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:03.079 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:03.079 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 20:44:03.079 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:03.079 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:03.079 | INFO | Uploaded 36GiB to GPU in 2.11786s (16.9983GiB/s) | 20260505 20:44:03.079 | INFO | Downloaded 34GiB to GPU in 2.13965s (15.8904GiB/s) | 20260505 20:44:08.222 | INFO | Analyzing chunk frames = [6172160,6172672) | 20260505 20:44:08.222 | INFO | MemInfo: { | 20260505 20:44:08.222 | INFO | "gpu" : " 17 GB", | 20260505 20:44:08.222 | INFO | "huge" : "", | 20260505 20:44:08.222 | INFO | "huge_shared" : " 27 GB", | 20260505 20:44:08.222 | INFO | "malloc" : " 14 GB", | 20260505 20:44:08.222 | INFO | "pinned" : " 29 GB" | 20260505 20:44:08.222 | INFO | } | 20260505 20:44:08.222 | INFO | Perf report for the last 5.14368 seconds: | 20260505 20:44:08.222 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:44:08.222 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:08.222 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:08.222 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:08.222 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260505 20:44:08.222 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:08.222 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:08.222 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:44:08.222 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:08.222 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:08.222 | INFO | Uploaded 36GiB to GPU in 2.12166s (16.9678GiB/s) | 20260505 20:44:08.222 | INFO | Downloaded 34GiB to GPU in 2.0147s (16.876GiB/s) | 20260505 20:44:08.354 | INFO | Starting a round of Baz IO | 20260505 20:44:11.511 | INFO | Populating RTMetrics report | 20260505 20:44:11.512 | INFO | RTMetrics report populated | 20260505 20:44:11.512 | INFO | Opening tmp RTMetrics json file | 20260505 20:44:11.512 | INFO | Serializing RTMetrics to json | 20260505 20:44:11.516 | INFO | Serialization done, swapping tmp file now... | 20260505 20:44:11.517 | INFO | RTMetrics done | 20260505 20:44:11.596 | INFO | Finished a round of Baz IO: Wrote 2.004 GiB in 3.241 seconds | 20260505 20:44:11.596 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.64% / 0.00% / 78.36% | 20260505 20:44:11.596 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.167501% / 3.97364e-06% / 7.54992e-05% | 20260505 20:44:11.596 | INFO | Baz IO overall: | 20260505 20:44:11.596 | INFO | BytesWritten : 3590799848376 | 20260505 20:44:11.596 | INFO | NumEvents : 1696929639912 | 20260505 20:44:13.360 | INFO | Analyzing chunk frames = [6172672,6173184) | 20260505 20:44:13.360 | INFO | MemInfo: { | 20260505 20:44:13.360 | INFO | "gpu" : " 17 GB", | 20260505 20:44:13.360 | INFO | "huge" : "", | 20260505 20:44:13.360 | INFO | "huge_shared" : " 27 GB", | 20260505 20:44:13.360 | INFO | "malloc" : " 14 GB", | 20260505 20:44:13.360 | INFO | "pinned" : " 29 GB" | 20260505 20:44:13.360 | INFO | } | 20260505 20:44:13.360 | INFO | Perf report for the last 5.13769 seconds: | 20260505 20:44:13.360 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:44:13.360 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:13.360 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:13.360 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:13.360 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 20:44:13.360 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:13.360 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260505 20:44:13.360 | INFO | PRE_HQ : 60.2%, 1.8, 55.0ms, 0 | 20260505 20:44:13.360 | INFO | BAZWRITER : 62.6%, 3.7, 119.9ms, 0 | 20260505 20:44:13.360 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 20:44:13.360 | INFO | Uploaded 36GiB to GPU in 2.18285s (16.4922GiB/s) | 20260505 20:44:13.360 | INFO | Downloaded 38GiB to GPU in 2.25418s (16.8576GiB/s) | 20260505 20:44:18.492 | INFO | Analyzing chunk frames = [6173184,6173696) | 20260505 20:44:18.492 | INFO | MemInfo: { | 20260505 20:44:18.492 | INFO | "gpu" : " 17 GB", | 20260505 20:44:18.492 | INFO | "huge" : "", | 20260505 20:44:18.492 | INFO | "huge_shared" : " 27 GB", | 20260505 20:44:18.492 | INFO | "malloc" : " 14 GB", | 20260505 20:44:18.493 | INFO | "pinned" : " 29 GB" | 20260505 20:44:18.493 | INFO | } | 20260505 20:44:18.493 | INFO | Perf report for the last 5.13269 seconds: | 20260505 20:44:18.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:44:18.493 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:18.493 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:18.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:18.493 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260505 20:44:18.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:18.493 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:18.493 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260505 20:44:18.493 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:18.493 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:18.493 | INFO | Uploaded 36GiB to GPU in 2.0785s (17.3202GiB/s) | 20260505 20:44:18.493 | INFO | Downloaded 34GiB to GPU in 2.02302s (16.8065GiB/s) | 20260505 20:44:23.631 | INFO | Analyzing chunk frames = [6173696,6174208) | 20260505 20:44:23.631 | INFO | MemInfo: { | 20260505 20:44:23.631 | INFO | "gpu" : " 17 GB", | 20260505 20:44:23.631 | INFO | "huge" : "", | 20260505 20:44:23.631 | INFO | "huge_shared" : " 27 GB", | 20260505 20:44:23.631 | INFO | "malloc" : " 14 GB", | 20260505 20:44:23.631 | INFO | "pinned" : " 29 GB" | 20260505 20:44:23.631 | INFO | } | 20260505 20:44:23.631 | INFO | Perf report for the last 5.13855 seconds: | 20260505 20:44:23.631 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:44:23.631 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:23.631 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:23.631 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:23.631 | INFO | ANALYSIS : 62.7%, 3.0, 97.5ms, 0 | 20260505 20:44:23.631 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:23.631 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:23.631 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:44:23.631 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:23.631 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:23.631 | INFO | Uploaded 36GiB to GPU in 2.21173s (16.2768GiB/s) | 20260505 20:44:23.631 | INFO | Downloaded 34GiB to GPU in 2.0698s (16.4267GiB/s) | 20260505 20:44:28.775 | INFO | Analyzing chunk frames = [6174208,6174720) | 20260505 20:44:28.775 | INFO | MemInfo: { | 20260505 20:44:28.775 | INFO | "gpu" : " 17 GB", | 20260505 20:44:28.775 | INFO | "huge" : "", | 20260505 20:44:28.775 | INFO | "huge_shared" : " 27 GB", | 20260505 20:44:28.775 | INFO | "malloc" : " 14 GB", | 20260505 20:44:28.775 | INFO | "pinned" : " 29 GB" | 20260505 20:44:28.775 | INFO | } | 20260505 20:44:28.775 | INFO | Perf report for the last 5.14388 seconds: | 20260505 20:44:28.775 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:44:28.775 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:28.775 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:28.775 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:28.775 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260505 20:44:28.775 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:28.775 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:28.775 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:44:28.775 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:28.775 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:28.775 | INFO | Uploaded 36GiB to GPU in 2.18565s (16.471GiB/s) | 20260505 20:44:28.775 | INFO | Downloaded 34GiB to GPU in 2.07515s (16.3844GiB/s) | 20260505 20:44:33.923 | INFO | Analyzing chunk frames = [6174720,6175232) | 20260505 20:44:33.923 | INFO | MemInfo: { | 20260505 20:44:33.923 | INFO | "gpu" : " 17 GB", | 20260505 20:44:33.923 | INFO | "huge" : "", | 20260505 20:44:33.923 | INFO | "huge_shared" : " 27 GB", | 20260505 20:44:33.923 | INFO | "malloc" : " 14 GB", | 20260505 20:44:33.923 | INFO | "pinned" : " 29 GB" | 20260505 20:44:33.923 | INFO | } | 20260505 20:44:33.923 | INFO | Perf report for the last 5.14796 seconds: | 20260505 20:44:33.923 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:44:33.923 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:33.923 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:33.923 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:33.923 | INFO | ANALYSIS : 61.5%, 3.0, 96.2ms, 0 | 20260505 20:44:33.923 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:33.923 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:33.923 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:44:33.923 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:33.923 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:33.923 | INFO | Uploaded 36GiB to GPU in 2.12919s (16.9079GiB/s) | 20260505 20:44:33.923 | INFO | Downloaded 34GiB to GPU in 2.06493s (16.4654GiB/s) | 20260505 20:44:39.057 | INFO | Analyzing chunk frames = [6175232,6175744) | 20260505 20:44:39.057 | INFO | MemInfo: { | 20260505 20:44:39.057 | INFO | "gpu" : " 17 GB", | 20260505 20:44:39.057 | INFO | "huge" : "", | 20260505 20:44:39.057 | INFO | "huge_shared" : " 27 GB", | 20260505 20:44:39.057 | INFO | "malloc" : " 14 GB", | 20260505 20:44:39.057 | INFO | "pinned" : " 29 GB" | 20260505 20:44:39.057 | INFO | } | 20260505 20:44:39.057 | INFO | Perf report for the last 5.13403 seconds: | 20260505 20:44:39.057 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:44:39.057 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:39.057 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:39.057 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:39.057 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260505 20:44:39.057 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:39.057 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:39.057 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:44:39.057 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:39.057 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:39.057 | INFO | Uploaded 36GiB to GPU in 2.07907s (17.3154GiB/s) | 20260505 20:44:39.057 | INFO | Downloaded 34GiB to GPU in 2.04156s (16.6539GiB/s) | 20260505 20:44:44.192 | INFO | Analyzing chunk frames = [6175744,6176256) | 20260505 20:44:44.192 | INFO | MemInfo: { | 20260505 20:44:44.192 | INFO | "gpu" : " 17 GB", | 20260505 20:44:44.192 | INFO | "huge" : "", | 20260505 20:44:44.192 | INFO | "huge_shared" : " 27 GB", | 20260505 20:44:44.192 | INFO | "malloc" : " 14 GB", | 20260505 20:44:44.192 | INFO | "pinned" : " 29 GB" | 20260505 20:44:44.192 | INFO | } | 20260505 20:44:44.192 | INFO | Perf report for the last 5.13493 seconds: | 20260505 20:44:44.192 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:44:44.192 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:44.192 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:44.192 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:44.192 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 20:44:44.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:44.192 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:44.192 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 20:44:44.192 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:44.192 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:44.192 | INFO | Uploaded 36GiB to GPU in 2.14669s (16.77GiB/s) | 20260505 20:44:44.192 | INFO | Downloaded 34GiB to GPU in 2.14247s (15.8695GiB/s) | 20260505 20:44:49.335 | INFO | Analyzing chunk frames = [6176256,6176768) | 20260505 20:44:49.335 | INFO | MemInfo: { | 20260505 20:44:49.335 | INFO | "gpu" : " 17 GB", | 20260505 20:44:49.335 | INFO | "huge" : "", | 20260505 20:44:49.335 | INFO | "huge_shared" : " 27 GB", | 20260505 20:44:49.335 | INFO | "malloc" : " 14 GB", | 20260505 20:44:49.335 | INFO | "pinned" : " 29 GB" | 20260505 20:44:49.335 | INFO | } | 20260505 20:44:49.335 | INFO | Perf report for the last 5.1429 seconds: | 20260505 20:44:49.335 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:44:49.335 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:49.335 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:49.335 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:49.335 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 20:44:49.335 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:49.335 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:49.335 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 20:44:49.335 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:49.335 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:49.335 | INFO | Uploaded 36GiB to GPU in 2.20088s (16.3571GiB/s) | 20260505 20:44:49.335 | INFO | Downloaded 34GiB to GPU in 2.04602s (16.6177GiB/s) | 20260505 20:44:49.335 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 20:44:49.335 | INFO | Total mem usage: 15.3 GiB | 20260505 20:44:49.335 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 20:44:49.335 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.11%) | 20260505 20:44:49.335 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 20:44:49.335 | INFO | High Water Breakdown: | 20260505 20:44:49.335 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 20:44:49.335 | INFO | 38.8% : 5.5 GiB ( 82.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 20:44:49.335 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 20:44:49.335 | INFO | ------------------------------------------ | 20260505 20:44:49.335 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:44:49.335 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 20:44:49.335 | INFO | Total mem usage: 26.5 GiB | 20260505 20:44:49.335 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 20:44:49.336 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 20:44:49.336 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 20:44:49.336 | INFO | High Water Breakdown: | 20260505 20:44:49.336 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:44:49.336 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:44:49.336 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:44:49.336 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:44:49.336 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:44:49.336 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:44:49.336 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 20:44:49.336 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 20:44:49.336 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:44:49.336 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 20:44:49.336 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 20:44:49.336 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 20:44:49.336 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 20:44:49.336 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 20:44:49.336 | INFO | ------------------------------------------ | 20260505 20:44:49.336 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:44:49.336 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 20:44:49.336 | INFO | Total mem usage: 25.6 GiB | 20260505 20:44:49.336 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 20:44:49.336 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 20:44:49.336 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 20:44:49.336 | INFO | High Water Breakdown: | 20260505 20:44:49.336 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 20:44:49.336 | INFO | ------------------------------------------ | 20260505 20:44:49.336 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:44:49.336 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 20:44:49.336 | INFO | Total mem usage: 15.7 GiB | 20260505 20:44:49.336 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 20:44:49.336 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 20:44:49.336 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 20:44:49.336 | INFO | High Water Breakdown: | 20260505 20:44:49.336 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 20:44:49.336 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 20:44:49.336 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 20:44:49.336 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 20:44:49.336 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 20:44:49.336 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 20:44:49.336 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 20:44:49.336 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 20:44:49.336 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 20:44:49.336 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:44:49.336 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 20:44:49.336 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:44:49.336 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 20:44:49.336 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 20:44:49.336 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 20:44:49.336 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 20:44:49.336 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:44:49.336 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 20:44:49.336 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:44:49.336 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:44:49.336 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:44:49.336 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 20:44:49.336 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 20:44:49.336 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:44:49.336 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 20:44:49.336 | INFO | ------------------------------------------ | 20260505 20:44:49.336 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:44:49.336 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 20:44:49.336 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:44:49.336 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:44:49.336 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.46e+01, 1.35e+01 / 2.18e+00 | 20260505 20:44:49.336 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.04e-01, 1.07e-02 / 4.99e-03 | 20260505 20:44:49.336 | INFO | PerfReport: 11.76%, Baseline -- 3.70e+00 / 4.51e+00, 5.62e-01 / 1.78e-01 | 20260505 20:44:49.336 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.49e+00, 1.86e-01 / 1.08e-01 | 20260505 20:44:49.336 | INFO | PerfReport: 15.95%, DME -- 4.02e+01 / 4.45e+01, 4.26e+00 / 2.23e+00 | 20260505 20:44:49.336 | INFO | PerfReport: 57.20%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.55e+00 / 1.12e+00 | 20260505 20:44:49.336 | INFO | PerfReport: 7.19%, PulseAccumulating -- 2.26e+00 / 2.79e+00, 4.31e-01 / 2.58e-01 | 20260505 20:44:49.336 | INFO | PerfReport: 3.22%, Metrics -- 1.01e+00 / 1.31e+00, 2.77e-01 / 1.30e-01 | 20260505 20:44:49.336 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.54e-01 / 4.16e-01, 6.35e-02 / 3.69e-02 | 20260505 20:44:49.337 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 20:44:49.337 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:44:49.337 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:44:49.337 | INFO | PerfReport: Full Execution -- 9.93e+01 / -nan, 2.54e+01 / 0.00e+00 | 20260505 20:44:49.337 | INFO | PerfReport: 22.49%, Upload -- 2.23e+01 / 0.00e+00, 7.73e+00 / 0.00e+00 | 20260505 20:44:49.337 | INFO | PerfReport: 21.87%, Download -- 2.17e+01 / 0.00e+00, 7.71e+00 / 0.00e+00 | 20260505 20:44:49.337 | INFO | PerfReport: 31.71%, Compute -- 3.15e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 20:44:49.337 | INFO | PerfReport: 2.25%, LockUpload -- 2.23e+00 / 0.00e+00, 7.79e+00 / 0.00e+00 | 20260505 20:44:49.337 | INFO | PerfReport: 1.35%, LockDownload -- 1.35e+00 / 0.00e+00, 2.90e+00 / 0.00e+00 | 20260505 20:44:49.337 | INFO | PerfReport: 20.31%, LockCompute -- 2.02e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 20:44:49.337 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.41e-02 / 0.00e+00 | 20260505 20:44:49.469 | INFO | Starting a round of Baz IO | 20260505 20:44:52.635 | INFO | Populating RTMetrics report | 20260505 20:44:52.635 | INFO | RTMetrics report populated | 20260505 20:44:52.635 | INFO | Opening tmp RTMetrics json file | 20260505 20:44:52.636 | INFO | Serializing RTMetrics to json | 20260505 20:44:52.638 | INFO | Serialization done, swapping tmp file now... | 20260505 20:44:52.640 | INFO | RTMetrics done | 20260505 20:44:52.719 | INFO | Finished a round of Baz IO: Wrote 2.001 GiB in 3.249 seconds | 20260505 20:44:52.719 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.64% / 0.00% / 78.36% | 20260505 20:44:52.719 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.167139% / 0% / 9.13938e-05% | 20260505 20:44:52.719 | INFO | Baz IO overall: | 20260505 20:44:52.719 | INFO | BytesWritten : 3592948338862 | 20260505 20:44:52.719 | INFO | NumEvents : 1697924026249 | 20260505 20:44:54.475 | INFO | Analyzing chunk frames = [6176768,6177280) | 20260505 20:44:54.475 | INFO | MemInfo: { | 20260505 20:44:54.475 | INFO | "gpu" : " 17 GB", | 20260505 20:44:54.475 | INFO | "huge" : "", | 20260505 20:44:54.475 | INFO | "huge_shared" : " 27 GB", | 20260505 20:44:54.475 | INFO | "malloc" : " 14 GB", | 20260505 20:44:54.475 | INFO | "pinned" : " 29 GB" | 20260505 20:44:54.475 | INFO | } | 20260505 20:44:54.475 | INFO | Perf report for the last 5.14027 seconds: | 20260505 20:44:54.475 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:44:54.475 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:54.475 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:54.475 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:54.475 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 20:44:54.475 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:54.475 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260505 20:44:54.475 | INFO | PRE_HQ : 59.9%, 1.8, 55.1ms, 0 | 20260505 20:44:54.475 | INFO | BAZWRITER : 62.7%, 3.7, 118.4ms, 0 | 20260505 20:44:54.475 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 20:44:54.476 | INFO | Uploaded 36GiB to GPU in 2.18034s (16.5112GiB/s) | 20260505 20:44:54.476 | INFO | Downloaded 38GiB to GPU in 2.2637s (16.7866GiB/s) | 20260505 20:44:59.613 | INFO | Analyzing chunk frames = [6177280,6177792) | 20260505 20:44:59.613 | INFO | MemInfo: { | 20260505 20:44:59.613 | INFO | "gpu" : " 17 GB", | 20260505 20:44:59.613 | INFO | "huge" : "", | 20260505 20:44:59.614 | INFO | "huge_shared" : " 27 GB", | 20260505 20:44:59.614 | INFO | "malloc" : " 14 GB", | 20260505 20:44:59.614 | INFO | "pinned" : " 29 GB" | 20260505 20:44:59.614 | INFO | } | 20260505 20:44:59.614 | INFO | Perf report for the last 5.13851 seconds: | 20260505 20:44:59.614 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:44:59.614 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:59.614 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:59.614 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:59.614 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 20:44:59.614 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:59.614 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:59.614 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 20:44:59.614 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:59.614 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:44:59.614 | INFO | Uploaded 36GiB to GPU in 2.10077s (17.1366GiB/s) | 20260505 20:44:59.614 | INFO | Downloaded 34GiB to GPU in 2.02786s (16.7664GiB/s) | 20260505 20:45:04.749 | INFO | Analyzing chunk frames = [6177792,6178304) | 20260505 20:45:04.749 | INFO | MemInfo: { | 20260505 20:45:04.749 | INFO | "gpu" : " 17 GB", | 20260505 20:45:04.749 | INFO | "huge" : "", | 20260505 20:45:04.749 | INFO | "huge_shared" : " 27 GB", | 20260505 20:45:04.750 | INFO | "malloc" : " 14 GB", | 20260505 20:45:04.750 | INFO | "pinned" : " 29 GB" | 20260505 20:45:04.750 | INFO | } | 20260505 20:45:04.750 | INFO | Perf report for the last 5.13591 seconds: | 20260505 20:45:04.750 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:45:04.750 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:04.750 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:04.750 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:04.750 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260505 20:45:04.750 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:04.750 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:04.750 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:45:04.750 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:04.750 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:04.750 | INFO | Uploaded 36GiB to GPU in 2.13561s (16.857GiB/s) | 20260505 20:45:04.750 | INFO | Downloaded 34GiB to GPU in 2.01875s (16.8421GiB/s) | 20260505 20:45:09.889 | INFO | Analyzing chunk frames = [6178304,6178816) | 20260505 20:45:09.889 | INFO | MemInfo: { | 20260505 20:45:09.889 | INFO | "gpu" : " 17 GB", | 20260505 20:45:09.889 | INFO | "huge" : "", | 20260505 20:45:09.889 | INFO | "huge_shared" : " 27 GB", | 20260505 20:45:09.889 | INFO | "malloc" : " 14 GB", | 20260505 20:45:09.889 | INFO | "pinned" : " 29 GB" | 20260505 20:45:09.889 | INFO | } | 20260505 20:45:09.890 | INFO | Perf report for the last 5.1399 seconds: | 20260505 20:45:09.890 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:45:09.890 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:09.890 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:09.890 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:09.890 | INFO | ANALYSIS : 62.4%, 3.0, 97.0ms, 0 | 20260505 20:45:09.890 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:09.890 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:09.890 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:45:09.890 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:09.890 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:09.890 | INFO | Uploaded 36GiB to GPU in 2.20497s (16.3268GiB/s) | 20260505 20:45:09.890 | INFO | Downloaded 34GiB to GPU in 2.04027s (16.6644GiB/s) | 20260505 20:45:15.034 | INFO | Analyzing chunk frames = [6178816,6179328) | 20260505 20:45:15.034 | INFO | MemInfo: { | 20260505 20:45:15.034 | INFO | "gpu" : " 17 GB", | 20260505 20:45:15.034 | INFO | "huge" : "", | 20260505 20:45:15.034 | INFO | "huge_shared" : " 27 GB", | 20260505 20:45:15.034 | INFO | "malloc" : " 14 GB", | 20260505 20:45:15.034 | INFO | "pinned" : " 29 GB" | 20260505 20:45:15.034 | INFO | } | 20260505 20:45:15.035 | INFO | Perf report for the last 5.14501 seconds: | 20260505 20:45:15.035 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:45:15.035 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:15.035 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:15.035 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:15.035 | INFO | ANALYSIS : 61.6%, 3.0, 96.2ms, 0 | 20260505 20:45:15.035 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:15.035 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:15.035 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 20:45:15.035 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:15.035 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:15.035 | INFO | Uploaded 36GiB to GPU in 2.13059s (16.8967GiB/s) | 20260505 20:45:15.035 | INFO | Downloaded 34GiB to GPU in 2.06085s (16.4981GiB/s) | 20260505 20:45:20.173 | INFO | Analyzing chunk frames = [6179328,6179840) | 20260505 20:45:20.173 | INFO | MemInfo: { | 20260505 20:45:20.173 | INFO | "gpu" : " 17 GB", | 20260505 20:45:20.173 | INFO | "huge" : "", | 20260505 20:45:20.173 | INFO | "huge_shared" : " 27 GB", | 20260505 20:45:20.173 | INFO | "malloc" : " 14 GB", | 20260505 20:45:20.173 | INFO | "pinned" : " 29 GB" | 20260505 20:45:20.173 | INFO | } | 20260505 20:45:20.173 | INFO | Perf report for the last 5.13858 seconds: | 20260505 20:45:20.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:45:20.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:20.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:20.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:20.173 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260505 20:45:20.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:20.173 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:20.173 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:45:20.173 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:20.173 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:20.173 | INFO | Uploaded 36GiB to GPU in 2.07188s (17.3755GiB/s) | 20260505 20:45:20.173 | INFO | Downloaded 34GiB to GPU in 2.03016s (16.7475GiB/s) | 20260505 20:45:25.307 | INFO | Analyzing chunk frames = [6179840,6180352) | 20260505 20:45:25.307 | INFO | MemInfo: { | 20260505 20:45:25.307 | INFO | "gpu" : " 17 GB", | 20260505 20:45:25.307 | INFO | "huge" : "", | 20260505 20:45:25.307 | INFO | "huge_shared" : " 27 GB", | 20260505 20:45:25.307 | INFO | "malloc" : " 14 GB", | 20260505 20:45:25.307 | INFO | "pinned" : " 29 GB" | 20260505 20:45:25.307 | INFO | } | 20260505 20:45:25.307 | INFO | Perf report for the last 5.13433 seconds: | 20260505 20:45:25.307 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:45:25.307 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:25.307 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:25.308 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:25.308 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 20:45:25.308 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:25.308 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:25.308 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 20:45:25.308 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:25.308 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:25.308 | INFO | Uploaded 36GiB to GPU in 2.13192s (16.8862GiB/s) | 20260505 20:45:25.308 | INFO | Downloaded 34GiB to GPU in 2.12788s (15.9784GiB/s) | 20260505 20:45:30.442 | INFO | Analyzing chunk frames = [6180352,6180864) | 20260505 20:45:30.442 | INFO | MemInfo: { | 20260505 20:45:30.442 | INFO | "gpu" : " 17 GB", | 20260505 20:45:30.442 | INFO | "huge" : "", | 20260505 20:45:30.442 | INFO | "huge_shared" : " 27 GB", | 20260505 20:45:30.442 | INFO | "malloc" : " 14 GB", | 20260505 20:45:30.442 | INFO | "pinned" : " 29 GB" | 20260505 20:45:30.442 | INFO | } | 20260505 20:45:30.442 | INFO | Perf report for the last 5.13463 seconds: | 20260505 20:45:30.442 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:45:30.442 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:30.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:30.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:30.442 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 20:45:30.442 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:30.442 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:30.442 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:45:30.442 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:30.442 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:30.442 | INFO | Uploaded 36GiB to GPU in 2.20493s (16.327GiB/s) | 20260505 20:45:30.442 | INFO | Downloaded 34GiB to GPU in 2.04901s (16.5934GiB/s) | 20260505 20:45:30.576 | INFO | Starting a round of Baz IO | 20260505 20:45:33.743 | INFO | Populating RTMetrics report | 20260505 20:45:33.743 | INFO | RTMetrics report populated | 20260505 20:45:33.743 | INFO | Opening tmp RTMetrics json file | 20260505 20:45:33.744 | INFO | Serializing RTMetrics to json | 20260505 20:45:33.747 | INFO | Serialization done, swapping tmp file now... | 20260505 20:45:33.750 | INFO | RTMetrics done | 20260505 20:45:33.829 | INFO | Finished a round of Baz IO: Wrote 1.999 GiB in 3.253 seconds | 20260505 20:45:33.830 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.64% / 0.00% / 78.36% | 20260505 20:45:33.830 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.166738% / 0% / 0.000123183% | 20260505 20:45:33.830 | INFO | Baz IO overall: | 20260505 20:45:33.830 | INFO | BytesWritten : 3595094571232 | 20260505 20:45:33.830 | INFO | NumEvents : 1698917050714 | 20260505 20:45:35.586 | INFO | Analyzing chunk frames = [6180864,6181376) | 20260505 20:45:35.587 | INFO | MemInfo: { | 20260505 20:45:35.587 | INFO | "gpu" : " 17 GB", | 20260505 20:45:35.587 | INFO | "huge" : "", | 20260505 20:45:35.587 | INFO | "huge_shared" : " 27 GB", | 20260505 20:45:35.587 | INFO | "malloc" : " 14 GB", | 20260505 20:45:35.587 | INFO | "pinned" : " 29 GB" | 20260505 20:45:35.587 | INFO | } | 20260505 20:45:35.587 | INFO | Perf report for the last 5.14467 seconds: | 20260505 20:45:35.587 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:45:35.587 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:35.587 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:35.587 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:35.587 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 20:45:35.587 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:35.587 | INFO | RT_METRICS : 43.9%, 1.0, 23.1ms, 0 | 20260505 20:45:35.587 | INFO | PRE_HQ : 59.9%, 1.8, 54.8ms, 0 | 20260505 20:45:35.587 | INFO | BAZWRITER : 62.6%, 3.7, 117.9ms, 0 | 20260505 20:45:35.587 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 20:45:35.587 | INFO | Uploaded 36GiB to GPU in 2.14973s (16.7463GiB/s) | 20260505 20:45:35.587 | INFO | Downloaded 38GiB to GPU in 2.25044s (16.8856GiB/s) | 20260505 20:45:40.727 | INFO | Analyzing chunk frames = [6181376,6181888) | 20260505 20:45:40.727 | INFO | MemInfo: { | 20260505 20:45:40.727 | INFO | "gpu" : " 17 GB", | 20260505 20:45:40.727 | INFO | "huge" : "", | 20260505 20:45:40.727 | INFO | "huge_shared" : " 27 GB", | 20260505 20:45:40.727 | INFO | "malloc" : " 14 GB", | 20260505 20:45:40.727 | INFO | "pinned" : " 29 GB" | 20260505 20:45:40.727 | INFO | } | 20260505 20:45:40.727 | INFO | Perf report for the last 5.13994 seconds: | 20260505 20:45:40.727 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:45:40.727 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:40.727 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:40.727 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:40.727 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260505 20:45:40.727 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:40.727 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:40.727 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 20:45:40.727 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:40.727 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:40.727 | INFO | Uploaded 36GiB to GPU in 2.10594s (17.0945GiB/s) | 20260505 20:45:40.727 | INFO | Downloaded 34GiB to GPU in 2.02909s (16.7563GiB/s) | 20260505 20:45:45.871 | INFO | Analyzing chunk frames = [6181888,6182400) | 20260505 20:45:45.871 | INFO | MemInfo: { | 20260505 20:45:45.871 | INFO | "gpu" : " 17 GB", | 20260505 20:45:45.871 | INFO | "huge" : "", | 20260505 20:45:45.871 | INFO | "huge_shared" : " 27 GB", | 20260505 20:45:45.871 | INFO | "malloc" : " 14 GB", | 20260505 20:45:45.871 | INFO | "pinned" : " 29 GB" | 20260505 20:45:45.871 | INFO | } | 20260505 20:45:45.871 | INFO | Perf report for the last 5.14474 seconds: | 20260505 20:45:45.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:45:45.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:45.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:45.872 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:45.872 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260505 20:45:45.872 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:45.872 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:45.872 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:45:45.872 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:45.872 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:45.872 | INFO | Uploaded 36GiB to GPU in 2.1285s (16.9133GiB/s) | 20260505 20:45:45.872 | INFO | Downloaded 34GiB to GPU in 2.02693s (16.7742GiB/s) | 20260505 20:45:51.003 | INFO | Analyzing chunk frames = [6182400,6182912) | 20260505 20:45:51.003 | INFO | MemInfo: { | 20260505 20:45:51.003 | INFO | "gpu" : " 17 GB", | 20260505 20:45:51.003 | INFO | "huge" : "", | 20260505 20:45:51.003 | INFO | "huge_shared" : " 27 GB", | 20260505 20:45:51.003 | INFO | "malloc" : " 14 GB", | 20260505 20:45:51.003 | INFO | "pinned" : " 29 GB" | 20260505 20:45:51.003 | INFO | } | 20260505 20:45:51.003 | INFO | Perf report for the last 5.13162 seconds: | 20260505 20:45:51.003 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:45:51.003 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:51.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:51.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:51.003 | INFO | ANALYSIS : 62.1%, 3.0, 96.3ms, 0 | 20260505 20:45:51.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:51.003 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:51.003 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:45:51.003 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:51.003 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:51.003 | INFO | Uploaded 36GiB to GPU in 2.13527s (16.8597GiB/s) | 20260505 20:45:51.003 | INFO | Downloaded 34GiB to GPU in 2.03488s (16.7086GiB/s) | 20260505 20:45:56.141 | INFO | Analyzing chunk frames = [6182912,6183424) | 20260505 20:45:56.141 | INFO | MemInfo: { | 20260505 20:45:56.141 | INFO | "gpu" : " 17 GB", | 20260505 20:45:56.141 | INFO | "huge" : "", | 20260505 20:45:56.141 | INFO | "huge_shared" : " 27 GB", | 20260505 20:45:56.141 | INFO | "malloc" : " 14 GB", | 20260505 20:45:56.142 | INFO | "pinned" : " 29 GB" | 20260505 20:45:56.142 | INFO | } | 20260505 20:45:56.142 | INFO | Perf report for the last 5.13852 seconds: | 20260505 20:45:56.142 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:45:56.142 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:56.142 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:56.142 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:56.142 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260505 20:45:56.142 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:56.142 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:56.142 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:45:56.142 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:56.142 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:45:56.142 | INFO | Uploaded 36GiB to GPU in 2.13698s (16.8462GiB/s) | 20260505 20:45:56.142 | INFO | Downloaded 34GiB to GPU in 2.06424s (16.4709GiB/s) | 20260505 20:46:01.287 | INFO | Analyzing chunk frames = [6183424,6183936) | 20260505 20:46:01.287 | INFO | MemInfo: { | 20260505 20:46:01.287 | INFO | "gpu" : " 17 GB", | 20260505 20:46:01.287 | INFO | "huge" : "", | 20260505 20:46:01.287 | INFO | "huge_shared" : " 27 GB", | 20260505 20:46:01.287 | INFO | "malloc" : " 14 GB", | 20260505 20:46:01.287 | INFO | "pinned" : " 29 GB" | 20260505 20:46:01.287 | INFO | } | 20260505 20:46:01.287 | INFO | Perf report for the last 5.14537 seconds: | 20260505 20:46:01.287 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:46:01.287 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:01.287 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:01.287 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:01.287 | INFO | ANALYSIS : 62.1%, 3.0, 97.1ms, 0 | 20260505 20:46:01.287 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:01.287 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:01.287 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:46:01.287 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:01.287 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:01.287 | INFO | Uploaded 36GiB to GPU in 2.15431s (16.7106GiB/s) | 20260505 20:46:01.287 | INFO | Downloaded 34GiB to GPU in 2.06178s (16.4906GiB/s) | 20260505 20:46:06.431 | INFO | Analyzing chunk frames = [6183936,6184448) | 20260505 20:46:06.431 | INFO | MemInfo: { | 20260505 20:46:06.431 | INFO | "gpu" : " 17 GB", | 20260505 20:46:06.431 | INFO | "huge" : "", | 20260505 20:46:06.431 | INFO | "huge_shared" : " 27 GB", | 20260505 20:46:06.431 | INFO | "malloc" : " 14 GB", | 20260505 20:46:06.431 | INFO | "pinned" : " 29 GB" | 20260505 20:46:06.431 | INFO | } | 20260505 20:46:06.432 | INFO | Perf report for the last 5.1445 seconds: | 20260505 20:46:06.432 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:46:06.432 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:06.432 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:06.432 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:06.432 | INFO | ANALYSIS : 62.7%, 3.0, 98.1ms, 0 | 20260505 20:46:06.432 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:06.432 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:06.432 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 20:46:06.432 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:06.432 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:06.432 | INFO | Uploaded 36GiB to GPU in 2.09803s (17.159GiB/s) | 20260505 20:46:06.432 | INFO | Downloaded 34GiB to GPU in 2.12197s (16.0229GiB/s) | 20260505 20:46:11.565 | INFO | Analyzing chunk frames = [6184448,6184960) | 20260505 20:46:11.565 | INFO | MemInfo: { | 20260505 20:46:11.565 | INFO | "gpu" : " 17 GB", | 20260505 20:46:11.565 | INFO | "huge" : "", | 20260505 20:46:11.565 | INFO | "huge_shared" : " 27 GB", | 20260505 20:46:11.565 | INFO | "malloc" : " 14 GB", | 20260505 20:46:11.565 | INFO | "pinned" : " 29 GB" | 20260505 20:46:11.565 | INFO | } | 20260505 20:46:11.565 | INFO | Perf report for the last 5.13391 seconds: | 20260505 20:46:11.565 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:46:11.565 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:11.565 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:11.565 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:11.565 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260505 20:46:11.565 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:11.565 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:11.565 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:46:11.565 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:11.565 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:11.566 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 20:46:11.566 | INFO | Total mem usage: 15.3 GiB | 20260505 20:46:11.566 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 20:46:11.566 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.14%) | 20260505 20:46:11.566 | INFO | Uploaded 36GiB to GPU in 2.12012s (16.9802GiB/s) | 20260505 20:46:11.566 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 20:46:11.566 | INFO | High Water Breakdown: | 20260505 20:46:11.566 | INFO | Downloaded 34GiB to GPU in 2.00816s (16.9309GiB/s) | 20260505 20:46:11.566 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 20:46:11.566 | INFO | 38.8% : 5.5 GiB ( 82.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 20:46:11.566 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 20:46:11.566 | INFO | ------------------------------------------ | 20260505 20:46:11.566 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:46:11.566 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 20:46:11.566 | INFO | Total mem usage: 26.5 GiB | 20260505 20:46:11.566 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 20:46:11.566 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 20:46:11.566 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 20:46:11.566 | INFO | High Water Breakdown: | 20260505 20:46:11.566 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:46:11.566 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:46:11.566 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:46:11.566 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:46:11.566 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:46:11.566 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:46:11.566 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 20:46:11.566 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 20:46:11.566 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:46:11.566 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 20:46:11.566 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 20:46:11.566 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 20:46:11.566 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 20:46:11.566 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 20:46:11.566 | INFO | ------------------------------------------ | 20260505 20:46:11.566 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:46:11.566 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 20:46:11.566 | INFO | Total mem usage: 25.6 GiB | 20260505 20:46:11.566 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 20:46:11.566 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 20:46:11.566 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 20:46:11.566 | INFO | High Water Breakdown: | 20260505 20:46:11.566 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 20:46:11.566 | INFO | ------------------------------------------ | 20260505 20:46:11.566 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:46:11.566 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 20:46:11.566 | INFO | Total mem usage: 15.7 GiB | 20260505 20:46:11.567 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 20:46:11.567 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 20:46:11.567 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 20:46:11.567 | INFO | High Water Breakdown: | 20260505 20:46:11.567 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 20:46:11.567 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 20:46:11.567 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 20:46:11.567 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 20:46:11.567 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 20:46:11.567 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 20:46:11.567 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 20:46:11.567 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 20:46:11.567 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 20:46:11.567 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:46:11.567 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 20:46:11.567 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:46:11.567 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 20:46:11.567 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 20:46:11.567 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 20:46:11.567 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 20:46:11.567 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:46:11.567 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 20:46:11.567 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:46:11.567 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:46:11.567 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:46:11.567 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 20:46:11.567 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 20:46:11.567 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:46:11.567 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 20:46:11.567 | INFO | ------------------------------------------ | 20260505 20:46:11.567 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:46:11.567 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 20:46:11.567 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:46:11.567 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:46:11.567 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.44e+01, 1.33e+01 / 1.52e+00 | 20260505 20:46:11.567 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.04e-01, 1.03e-02 / 4.33e-03 | 20260505 20:46:11.567 | INFO | PerfReport: 11.78%, Baseline -- 3.71e+00 / 4.57e+00, 5.65e-01 / 1.27e-01 | 20260505 20:46:11.567 | INFO | PerfReport: 3.26%, Histogram -- 1.03e+00 / 1.50e+00, 1.87e-01 / 1.05e-01 | 20260505 20:46:11.567 | INFO | PerfReport: 15.96%, DME -- 4.02e+01 / 4.42e+01, 4.00e+00 / 1.70e+00 | 20260505 20:46:11.567 | INFO | PerfReport: 57.14%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.53e+00 / 1.08e+00 | 20260505 20:46:11.567 | INFO | PerfReport: 7.21%, PulseAccumulating -- 2.27e+00 / 2.82e+00, 4.37e-01 / 2.42e-01 | 20260505 20:46:11.567 | INFO | PerfReport: 3.23%, Metrics -- 1.02e+00 / 1.35e+00, 2.77e-01 / 1.69e-01 | 20260505 20:46:11.567 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.54e-01 / 4.18e-01, 6.28e-02 / 3.60e-02 | 20260505 20:46:11.567 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 20:46:11.567 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:46:11.567 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:46:11.567 | INFO | PerfReport: Full Execution -- 9.93e+01 / -nan, 2.54e+01 / 0.00e+00 | 20260505 20:46:11.567 | INFO | PerfReport: 22.43%, Upload -- 2.23e+01 / 0.00e+00, 7.76e+00 / 0.00e+00 | 20260505 20:46:11.567 | INFO | PerfReport: 21.79%, Download -- 2.16e+01 / 0.00e+00, 7.69e+00 / 0.00e+00 | 20260505 20:46:11.567 | INFO | PerfReport: 31.71%, Compute -- 3.15e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260505 20:46:11.567 | INFO | PerfReport: 2.28%, LockUpload -- 2.26e+00 / 0.00e+00, 7.98e+00 / 0.00e+00 | 20260505 20:46:11.567 | INFO | PerfReport: 1.35%, LockDownload -- 1.34e+00 / 0.00e+00, 2.92e+00 / 0.00e+00 | 20260505 20:46:11.567 | INFO | PerfReport: 20.44%, LockCompute -- 2.03e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 20:46:11.567 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.63e-02 / 0.00e+00 | 20260505 20:46:11.698 | INFO | Starting a round of Baz IO | 20260505 20:46:14.856 | INFO | Populating RTMetrics report | 20260505 20:46:14.857 | INFO | RTMetrics report populated | 20260505 20:46:14.857 | INFO | Opening tmp RTMetrics json file | 20260505 20:46:14.858 | INFO | Serializing RTMetrics to json | 20260505 20:46:14.862 | INFO | Serialization done, swapping tmp file now... | 20260505 20:46:14.866 | INFO | RTMetrics done | 20260505 20:46:14.940 | INFO | Finished a round of Baz IO: Wrote 2 GiB in 3.241 seconds | 20260505 20:46:14.940 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.64% / 0.00% / 78.36% | 20260505 20:46:14.940 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.166305% / 0% / 9.53674e-05% | 20260505 20:46:14.940 | INFO | Baz IO overall: | 20260505 20:46:14.940 | INFO | BytesWritten : 3597242298227 | 20260505 20:46:14.940 | INFO | NumEvents : 1699911249810 | 20260505 20:46:16.704 | INFO | Analyzing chunk frames = [6184960,6185472) | 20260505 20:46:16.704 | INFO | MemInfo: { | 20260505 20:46:16.704 | INFO | "gpu" : " 17 GB", | 20260505 20:46:16.704 | INFO | "huge" : "", | 20260505 20:46:16.704 | INFO | "huge_shared" : " 27 GB", | 20260505 20:46:16.704 | INFO | "malloc" : " 14 GB", | 20260505 20:46:16.704 | INFO | "pinned" : " 29 GB" | 20260505 20:46:16.704 | INFO | } | 20260505 20:46:16.704 | INFO | Perf report for the last 5.13842 seconds: | 20260505 20:46:16.704 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:46:16.704 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:16.704 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:16.704 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:16.704 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 20:46:16.704 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:16.704 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260505 20:46:16.704 | INFO | PRE_HQ : 60.1%, 1.8, 55.1ms, 0 | 20260505 20:46:16.704 | INFO | BAZWRITER : 62.6%, 3.7, 118.5ms, 0 | 20260505 20:46:16.704 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 20:46:16.704 | INFO | Uploaded 36GiB to GPU in 2.17161s (16.5775GiB/s) | 20260505 20:46:16.704 | INFO | Downloaded 38GiB to GPU in 2.24773s (16.9059GiB/s) | 20260505 20:46:21.849 | INFO | Analyzing chunk frames = [6185472,6185984) | 20260505 20:46:21.849 | INFO | MemInfo: { | 20260505 20:46:21.849 | INFO | "gpu" : " 17 GB", | 20260505 20:46:21.849 | INFO | "huge" : "", | 20260505 20:46:21.849 | INFO | "huge_shared" : " 27 GB", | 20260505 20:46:21.849 | INFO | "malloc" : " 14 GB", | 20260505 20:46:21.849 | INFO | "pinned" : " 29 GB" | 20260505 20:46:21.849 | INFO | } | 20260505 20:46:21.849 | INFO | Perf report for the last 5.14535 seconds: | 20260505 20:46:21.849 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:46:21.849 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:21.849 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:21.849 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:21.849 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 20:46:21.849 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:21.849 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:21.849 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 20:46:21.849 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:21.849 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:21.849 | INFO | Uploaded 36GiB to GPU in 2.09114s (17.2155GiB/s) | 20260505 20:46:21.850 | INFO | Downloaded 34GiB to GPU in 2.0252s (16.7885GiB/s) | 20260505 20:46:26.985 | INFO | Analyzing chunk frames = [6185984,6186496) | 20260505 20:46:26.985 | INFO | MemInfo: { | 20260505 20:46:26.985 | INFO | "gpu" : " 17 GB", | 20260505 20:46:26.985 | INFO | "huge" : "", | 20260505 20:46:26.985 | INFO | "huge_shared" : " 27 GB", | 20260505 20:46:26.985 | INFO | "malloc" : " 14 GB", | 20260505 20:46:26.985 | INFO | "pinned" : " 29 GB" | 20260505 20:46:26.985 | INFO | } | 20260505 20:46:26.985 | INFO | Perf report for the last 5.13595 seconds: | 20260505 20:46:26.985 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:46:26.985 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:26.985 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:26.985 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:26.985 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260505 20:46:26.985 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:26.985 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:26.985 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:46:26.985 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:26.985 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:26.985 | INFO | Uploaded 36GiB to GPU in 2.16374s (16.6379GiB/s) | 20260505 20:46:26.985 | INFO | Downloaded 34GiB to GPU in 2.04974s (16.5875GiB/s) | 20260505 20:46:32.117 | INFO | Analyzing chunk frames = [6186496,6187008) | 20260505 20:46:32.117 | INFO | MemInfo: { | 20260505 20:46:32.117 | INFO | "gpu" : " 17 GB", | 20260505 20:46:32.117 | INFO | "huge" : "", | 20260505 20:46:32.117 | INFO | "huge_shared" : " 27 GB", | 20260505 20:46:32.117 | INFO | "malloc" : " 14 GB", | 20260505 20:46:32.117 | INFO | "pinned" : " 29 GB" | 20260505 20:46:32.117 | INFO | } | 20260505 20:46:32.117 | INFO | Perf report for the last 5.13212 seconds: | 20260505 20:46:32.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:46:32.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:32.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:32.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:32.117 | INFO | ANALYSIS : 61.9%, 3.0, 96.0ms, 0 | 20260505 20:46:32.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:32.117 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:32.118 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:46:32.118 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:32.118 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:32.118 | INFO | Uploaded 36GiB to GPU in 2.11583s (17.0146GiB/s) | 20260505 20:46:32.118 | INFO | Downloaded 34GiB to GPU in 2.0085s (16.9281GiB/s) | 20260505 20:46:37.260 | INFO | Analyzing chunk frames = [6187008,6187520) | 20260505 20:46:37.260 | INFO | MemInfo: { | 20260505 20:46:37.260 | INFO | "gpu" : " 17 GB", | 20260505 20:46:37.260 | INFO | "huge" : "", | 20260505 20:46:37.260 | INFO | "huge_shared" : " 27 GB", | 20260505 20:46:37.260 | INFO | "malloc" : " 14 GB", | 20260505 20:46:37.260 | INFO | "pinned" : " 29 GB" | 20260505 20:46:37.261 | INFO | } | 20260505 20:46:37.261 | INFO | Perf report for the last 5.14321 seconds: | 20260505 20:46:37.261 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:46:37.261 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:37.261 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:37.261 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:37.261 | INFO | ANALYSIS : 61.8%, 3.0, 96.6ms, 0 | 20260505 20:46:37.261 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:37.261 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:37.261 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:46:37.261 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:37.261 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:37.261 | INFO | Uploaded 36GiB to GPU in 2.14162s (16.8097GiB/s) | 20260505 20:46:37.261 | INFO | Downloaded 34GiB to GPU in 2.06241s (16.4856GiB/s) | 20260505 20:46:42.396 | INFO | Analyzing chunk frames = [6187520,6188032) | 20260505 20:46:42.396 | INFO | MemInfo: { | 20260505 20:46:42.396 | INFO | "gpu" : " 17 GB", | 20260505 20:46:42.396 | INFO | "huge" : "", | 20260505 20:46:42.396 | INFO | "huge_shared" : " 27 GB", | 20260505 20:46:42.396 | INFO | "malloc" : " 14 GB", | 20260505 20:46:42.397 | INFO | "pinned" : " 29 GB" | 20260505 20:46:42.397 | INFO | } | 20260505 20:46:42.397 | INFO | Perf report for the last 5.13612 seconds: | 20260505 20:46:42.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:46:42.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:42.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:42.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:42.397 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 20:46:42.397 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:42.397 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:42.397 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 20:46:42.397 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:42.397 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:42.397 | INFO | Uploaded 36GiB to GPU in 2.15535s (16.7026GiB/s) | 20260505 20:46:42.397 | INFO | Downloaded 34GiB to GPU in 2.06118s (16.4954GiB/s) | 20260505 20:46:47.540 | INFO | Analyzing chunk frames = [6188032,6188544) | 20260505 20:46:47.540 | INFO | MemInfo: { | 20260505 20:46:47.540 | INFO | "gpu" : " 17 GB", | 20260505 20:46:47.540 | INFO | "huge" : "", | 20260505 20:46:47.540 | INFO | "huge_shared" : " 27 GB", | 20260505 20:46:47.540 | INFO | "malloc" : " 14 GB", | 20260505 20:46:47.540 | INFO | "pinned" : " 29 GB" | 20260505 20:46:47.540 | INFO | } | 20260505 20:46:47.540 | INFO | Perf report for the last 5.14354 seconds: | 20260505 20:46:47.540 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:46:47.540 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:47.540 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:47.540 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:47.540 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 20:46:47.540 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:47.540 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:47.540 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 20:46:47.540 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:47.540 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:47.541 | INFO | Uploaded 36GiB to GPU in 2.14449s (16.7872GiB/s) | 20260505 20:46:47.541 | INFO | Downloaded 34GiB to GPU in 2.16509s (15.7038GiB/s) | 20260505 20:46:52.683 | INFO | Analyzing chunk frames = [6188544,6189056) | 20260505 20:46:52.683 | INFO | MemInfo: { | 20260505 20:46:52.683 | INFO | "gpu" : " 17 GB", | 20260505 20:46:52.683 | INFO | "huge" : "", | 20260505 20:46:52.683 | INFO | "huge_shared" : " 27 GB", | 20260505 20:46:52.683 | INFO | "malloc" : " 14 GB", | 20260505 20:46:52.683 | INFO | "pinned" : " 29 GB" | 20260505 20:46:52.683 | INFO | } | 20260505 20:46:52.683 | INFO | Perf report for the last 5.14297 seconds: | 20260505 20:46:52.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:46:52.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:52.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:52.683 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:52.683 | INFO | ANALYSIS : 62.1%, 3.0, 97.1ms, 0 | 20260505 20:46:52.683 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:52.683 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:52.683 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:46:52.683 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:52.683 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:52.684 | INFO | Uploaded 36GiB to GPU in 2.11286s (17.0385GiB/s) | 20260505 20:46:52.684 | INFO | Downloaded 34GiB to GPU in 2.00313s (16.9734GiB/s) | 20260505 20:46:52.813 | INFO | Starting a round of Baz IO | 20260505 20:46:55.970 | INFO | Populating RTMetrics report | 20260505 20:46:55.970 | INFO | RTMetrics report populated | 20260505 20:46:55.970 | INFO | Opening tmp RTMetrics json file | 20260505 20:46:55.971 | INFO | Serializing RTMetrics to json | 20260505 20:46:55.972 | INFO | Serialization done, swapping tmp file now... | 20260505 20:46:55.973 | INFO | RTMetrics done | 20260505 20:46:56.060 | INFO | Finished a round of Baz IO: Wrote 1.999 GiB in 3.246 seconds | 20260505 20:46:56.060 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.64% / 0.00% / 78.36% | 20260505 20:46:56.060 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.166122% / 0% / 0.000119209% | 20260505 20:46:56.060 | INFO | Baz IO overall: | 20260505 20:46:56.060 | INFO | BytesWritten : 3599388277217 | 20260505 20:46:56.060 | INFO | NumEvents : 1700904547340 | 20260505 20:46:57.819 | INFO | Analyzing chunk frames = [6189056,6189568) | 20260505 20:46:57.819 | INFO | MemInfo: { | 20260505 20:46:57.819 | INFO | "gpu" : " 17 GB", | 20260505 20:46:57.819 | INFO | "huge" : "", | 20260505 20:46:57.819 | INFO | "huge_shared" : " 27 GB", | 20260505 20:46:57.819 | INFO | "malloc" : " 14 GB", | 20260505 20:46:57.819 | INFO | "pinned" : " 29 GB" | 20260505 20:46:57.819 | INFO | } | 20260505 20:46:57.819 | INFO | Perf report for the last 5.13566 seconds: | 20260505 20:46:57.819 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:46:57.819 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:57.819 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:57.819 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:57.819 | INFO | ANALYSIS : 63.0%, 3.0, 98.1ms, 0 | 20260505 20:46:57.819 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:46:57.819 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260505 20:46:57.819 | INFO | PRE_HQ : 60.1%, 1.8, 54.9ms, 0 | 20260505 20:46:57.819 | INFO | BAZWRITER : 62.6%, 3.7, 118.0ms, 0 | 20260505 20:46:57.819 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 20:46:57.819 | INFO | Uploaded 36GiB to GPU in 2.15192s (16.7292GiB/s) | 20260505 20:46:57.819 | INFO | Downloaded 38GiB to GPU in 2.23602s (16.9945GiB/s) | 20260505 20:47:02.949 | INFO | Analyzing chunk frames = [6189568,6190080) | 20260505 20:47:02.949 | INFO | MemInfo: { | 20260505 20:47:02.949 | INFO | "gpu" : " 17 GB", | 20260505 20:47:02.949 | INFO | "huge" : "", | 20260505 20:47:02.949 | INFO | "huge_shared" : " 27 GB", | 20260505 20:47:02.949 | INFO | "malloc" : " 14 GB", | 20260505 20:47:02.949 | INFO | "pinned" : " 29 GB" | 20260505 20:47:02.949 | INFO | } | 20260505 20:47:02.949 | INFO | Perf report for the last 5.12997 seconds: | 20260505 20:47:02.949 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:47:02.949 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:02.949 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:02.949 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:02.949 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260505 20:47:02.949 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:02.949 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:02.949 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:47:02.949 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:02.949 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:02.949 | INFO | Uploaded 36GiB to GPU in 2.11344s (17.0338GiB/s) | 20260505 20:47:02.949 | INFO | Downloaded 34GiB to GPU in 2.03273s (16.7262GiB/s) | 20260505 20:47:08.097 | INFO | Analyzing chunk frames = [6190080,6190592) | 20260505 20:47:08.097 | INFO | MemInfo: { | 20260505 20:47:08.097 | INFO | "gpu" : " 17 GB", | 20260505 20:47:08.097 | INFO | "huge" : "", | 20260505 20:47:08.098 | INFO | "huge_shared" : " 27 GB", | 20260505 20:47:08.098 | INFO | "malloc" : " 14 GB", | 20260505 20:47:08.098 | INFO | "pinned" : " 29 GB" | 20260505 20:47:08.098 | INFO | } | 20260505 20:47:08.098 | INFO | Perf report for the last 5.14885 seconds: | 20260505 20:47:08.098 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:47:08.098 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:08.098 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:08.098 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:08.098 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260505 20:47:08.098 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:08.098 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:08.098 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:47:08.098 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:08.098 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:08.098 | INFO | Uploaded 36GiB to GPU in 2.1385s (16.8342GiB/s) | 20260505 20:47:08.098 | INFO | Downloaded 34GiB to GPU in 2.03754s (16.6868GiB/s) | 20260505 20:47:13.241 | INFO | Analyzing chunk frames = [6190592,6191104) | 20260505 20:47:13.241 | INFO | MemInfo: { | 20260505 20:47:13.241 | INFO | "gpu" : " 17 GB", | 20260505 20:47:13.241 | INFO | "huge" : "", | 20260505 20:47:13.241 | INFO | "huge_shared" : " 27 GB", | 20260505 20:47:13.241 | INFO | "malloc" : " 14 GB", | 20260505 20:47:13.241 | INFO | "pinned" : " 29 GB" | 20260505 20:47:13.241 | INFO | } | 20260505 20:47:13.241 | INFO | Perf report for the last 5.14342 seconds: | 20260505 20:47:13.241 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:47:13.241 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:13.241 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:13.241 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:13.241 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260505 20:47:13.241 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:13.241 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:13.241 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:47:13.241 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:13.241 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:13.241 | INFO | Uploaded 36GiB to GPU in 2.14379s (16.7927GiB/s) | 20260505 20:47:13.241 | INFO | Downloaded 34GiB to GPU in 2.04223s (16.6484GiB/s) | 20260505 20:47:18.378 | INFO | Analyzing chunk frames = [6191104,6191616) | 20260505 20:47:18.378 | INFO | MemInfo: { | 20260505 20:47:18.378 | INFO | "gpu" : " 17 GB", | 20260505 20:47:18.378 | INFO | "huge" : "", | 20260505 20:47:18.378 | INFO | "huge_shared" : " 27 GB", | 20260505 20:47:18.378 | INFO | "malloc" : " 14 GB", | 20260505 20:47:18.378 | INFO | "pinned" : " 29 GB" | 20260505 20:47:18.378 | INFO | } | 20260505 20:47:18.378 | INFO | Perf report for the last 5.13709 seconds: | 20260505 20:47:18.378 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:47:18.378 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:18.378 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:18.378 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:18.378 | INFO | ANALYSIS : 61.6%, 3.0, 96.2ms, 0 | 20260505 20:47:18.378 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:18.378 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:18.378 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:47:18.378 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:18.378 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:18.379 | INFO | Uploaded 36GiB to GPU in 2.13352s (16.8735GiB/s) | 20260505 20:47:18.379 | INFO | Downloaded 34GiB to GPU in 2.05928s (16.5106GiB/s) | 20260505 20:47:23.520 | INFO | Analyzing chunk frames = [6191616,6192128) | 20260505 20:47:23.520 | INFO | MemInfo: { | 20260505 20:47:23.520 | INFO | "gpu" : " 17 GB", | 20260505 20:47:23.520 | INFO | "huge" : "", | 20260505 20:47:23.520 | INFO | "huge_shared" : " 27 GB", | 20260505 20:47:23.520 | INFO | "malloc" : " 14 GB", | 20260505 20:47:23.520 | INFO | "pinned" : " 29 GB" | 20260505 20:47:23.520 | INFO | } | 20260505 20:47:23.520 | INFO | Perf report for the last 5.14156 seconds: | 20260505 20:47:23.520 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:47:23.520 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:23.520 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:23.520 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:23.520 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 20:47:23.520 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:23.520 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:23.520 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:47:23.520 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:23.520 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:23.520 | INFO | Uploaded 36GiB to GPU in 2.09264s (17.2031GiB/s) | 20260505 20:47:23.520 | INFO | Downloaded 34GiB to GPU in 2.03855s (16.6785GiB/s) | 20260505 20:47:28.652 | INFO | Analyzing chunk frames = [6192128,6192640) | 20260505 20:47:28.652 | INFO | MemInfo: { | 20260505 20:47:28.652 | INFO | "gpu" : " 17 GB", | 20260505 20:47:28.652 | INFO | "huge" : "", | 20260505 20:47:28.652 | INFO | "huge_shared" : " 27 GB", | 20260505 20:47:28.652 | INFO | "malloc" : " 14 GB", | 20260505 20:47:28.652 | INFO | "pinned" : " 29 GB" | 20260505 20:47:28.652 | INFO | } | 20260505 20:47:28.652 | INFO | Perf report for the last 5.13245 seconds: | 20260505 20:47:28.652 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:47:28.652 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:28.652 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:28.652 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:28.652 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 20:47:28.652 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:28.652 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:28.652 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 20:47:28.652 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:28.652 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:28.653 | INFO | Uploaded 36GiB to GPU in 2.1147s (17.0237GiB/s) | 20260505 20:47:28.653 | INFO | Downloaded 34GiB to GPU in 2.13939s (15.8924GiB/s) | 20260505 20:47:33.795 | INFO | Analyzing chunk frames = [6192640,6193152) | 20260505 20:47:33.795 | INFO | MemInfo: { | 20260505 20:47:33.795 | INFO | "gpu" : " 17 GB", | 20260505 20:47:33.795 | INFO | "huge" : "", | 20260505 20:47:33.795 | INFO | "huge_shared" : " 27 GB", | 20260505 20:47:33.795 | INFO | "malloc" : " 14 GB", | 20260505 20:47:33.795 | INFO | "pinned" : " 29 GB" | 20260505 20:47:33.795 | INFO | } | 20260505 20:47:33.795 | INFO | Perf report for the last 5.14281 seconds: | 20260505 20:47:33.795 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:47:33.795 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:33.795 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:33.795 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:33.795 | INFO | ANALYSIS : 62.3%, 3.0, 97.4ms, 0 | 20260505 20:47:33.795 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:33.795 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:33.795 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 20:47:33.795 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:33.795 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:33.795 | INFO | Uploaded 36GiB to GPU in 2.12774s (16.9194GiB/s) | 20260505 20:47:33.796 | INFO | Downloaded 34GiB to GPU in 2.01291s (16.891GiB/s) | 20260505 20:47:33.796 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 20:47:33.796 | INFO | Total mem usage: 15.3 GiB | 20260505 20:47:33.796 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 20:47:33.796 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.15%) | 20260505 20:47:33.796 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 20:47:33.796 | INFO | High Water Breakdown: | 20260505 20:47:33.796 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 20:47:33.796 | INFO | 38.8% : 5.5 GiB ( 82.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 20:47:33.796 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 20:47:33.796 | INFO | ------------------------------------------ | 20260505 20:47:33.796 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:47:33.796 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 20:47:33.796 | INFO | Total mem usage: 26.5 GiB | 20260505 20:47:33.796 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 20:47:33.796 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 20:47:33.796 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 20:47:33.796 | INFO | High Water Breakdown: | 20260505 20:47:33.796 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:47:33.796 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:47:33.796 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:47:33.796 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:47:33.796 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:47:33.796 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:47:33.796 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 20:47:33.796 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 20:47:33.796 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:47:33.796 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 20:47:33.796 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 20:47:33.796 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 20:47:33.796 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 20:47:33.796 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 20:47:33.796 | INFO | ------------------------------------------ | 20260505 20:47:33.796 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:47:33.796 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 20:47:33.796 | INFO | Total mem usage: 25.6 GiB | 20260505 20:47:33.796 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 20:47:33.796 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 20:47:33.796 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 20:47:33.796 | INFO | High Water Breakdown: | 20260505 20:47:33.796 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 20:47:33.796 | INFO | ------------------------------------------ | 20260505 20:47:33.796 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:47:33.796 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 20:47:33.796 | INFO | Total mem usage: 15.7 GiB | 20260505 20:47:33.796 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 20:47:33.796 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 20:47:33.796 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 20:47:33.796 | INFO | High Water Breakdown: | 20260505 20:47:33.796 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 20:47:33.796 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 20:47:33.796 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 20:47:33.796 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 20:47:33.796 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 20:47:33.796 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 20:47:33.796 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 20:47:33.796 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 20:47:33.796 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 20:47:33.796 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:47:33.796 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 20:47:33.796 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:47:33.796 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 20:47:33.796 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 20:47:33.796 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 20:47:33.796 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 20:47:33.796 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:47:33.796 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 20:47:33.796 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:47:33.796 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:47:33.796 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:47:33.796 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 20:47:33.796 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 20:47:33.796 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:47:33.797 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 20:47:33.797 | INFO | ------------------------------------------ | 20260505 20:47:33.797 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:47:33.797 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 20:47:33.797 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:47:33.797 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:47:33.797 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.47e+01, 1.30e+01 / 2.05e+00 | 20260505 20:47:33.797 | INFO | PerfReport: 0.29%, AggregateStats -- 9.27e-02 / 1.04e-01, 1.17e-02 / 4.57e-03 | 20260505 20:47:33.797 | INFO | PerfReport: 11.79%, Baseline -- 3.72e+00 / 4.52e+00, 5.68e-01 / 1.67e-01 | 20260505 20:47:33.797 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.51e+00, 1.87e-01 / 1.07e-01 | 20260505 20:47:33.797 | INFO | PerfReport: 15.93%, DME -- 4.02e+01 / 4.45e+01, 4.17e+00 / 2.09e+00 | 20260505 20:47:33.797 | INFO | PerfReport: 57.24%, FrameLabeling -- 1.81e+01 / 1.96e+01, 1.63e+00 / 1.19e+00 | 20260505 20:47:33.797 | INFO | PerfReport: 7.14%, PulseAccumulating -- 2.25e+00 / 2.79e+00, 4.26e-01 / 2.39e-01 | 20260505 20:47:33.797 | INFO | PerfReport: 3.20%, Metrics -- 1.01e+00 / 1.30e+00, 2.75e-01 / 1.30e-01 | 20260505 20:47:33.797 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.53e-01 / 4.08e-01, 6.27e-02 / 3.33e-02 | 20260505 20:47:33.797 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 20:47:33.797 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:47:33.797 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:47:33.797 | INFO | PerfReport: Full Execution -- 9.91e+01 / -nan, 2.50e+01 / 0.00e+00 | 20260505 20:47:33.797 | INFO | PerfReport: 22.40%, Upload -- 2.22e+01 / 0.00e+00, 7.58e+00 / 0.00e+00 | 20260505 20:47:33.797 | INFO | PerfReport: 21.82%, Download -- 2.16e+01 / 0.00e+00, 7.58e+00 / 0.00e+00 | 20260505 20:47:33.797 | INFO | PerfReport: 31.83%, Compute -- 3.16e+01 / 0.00e+00, 1.31e+01 / 0.00e+00 | 20260505 20:47:33.797 | INFO | PerfReport: 2.17%, LockUpload -- 2.15e+00 / 0.00e+00, 7.86e+00 / 0.00e+00 | 20260505 20:47:33.797 | INFO | PerfReport: 1.37%, LockDownload -- 1.35e+00 / 0.00e+00, 2.97e+00 / 0.00e+00 | 20260505 20:47:33.797 | INFO | PerfReport: 20.41%, LockCompute -- 2.02e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260505 20:47:33.797 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.92e-02 / 0.00e+00 | 20260505 20:47:33.932 | INFO | Starting a round of Baz IO | 20260505 20:47:37.088 | INFO | Populating RTMetrics report | 20260505 20:47:37.088 | INFO | RTMetrics report populated | 20260505 20:47:37.089 | INFO | Opening tmp RTMetrics json file | 20260505 20:47:37.089 | INFO | Serializing RTMetrics to json | 20260505 20:47:37.093 | INFO | Serialization done, swapping tmp file now... | 20260505 20:47:37.097 | INFO | RTMetrics done | 20260505 20:47:37.178 | INFO | Finished a round of Baz IO: Wrote 1.998 GiB in 3.245 seconds | 20260505 20:47:37.178 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.64% / 0.00% / 78.36% | 20260505 20:47:37.178 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.165323% / 7.94729e-06% / 8.74201e-05% | 20260505 20:47:37.178 | INFO | Baz IO overall: | 20260505 20:47:37.178 | INFO | BytesWritten : 3601533303283 | 20260505 20:47:37.178 | INFO | NumEvents : 1701897025305 | 20260505 20:47:38.935 | INFO | Analyzing chunk frames = [6193152,6193664) | 20260505 20:47:38.935 | INFO | MemInfo: { | 20260505 20:47:38.935 | INFO | "gpu" : " 17 GB", | 20260505 20:47:38.935 | INFO | "huge" : "", | 20260505 20:47:38.935 | INFO | "huge_shared" : " 27 GB", | 20260505 20:47:38.935 | INFO | "malloc" : " 14 GB", | 20260505 20:47:38.935 | INFO | "pinned" : " 29 GB" | 20260505 20:47:38.935 | INFO | } | 20260505 20:47:38.935 | INFO | Perf report for the last 5.14007 seconds: | 20260505 20:47:38.935 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:47:38.935 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:38.935 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:38.935 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:38.935 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 20:47:38.935 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:38.935 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260505 20:47:38.935 | INFO | PRE_HQ : 60.0%, 1.8, 55.0ms, 0 | 20260505 20:47:38.935 | INFO | BAZWRITER : 62.7%, 3.7, 119.5ms, 0 | 20260505 20:47:38.935 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 20:47:38.936 | INFO | Uploaded 36GiB to GPU in 2.1764s (16.5411GiB/s) | 20260505 20:47:38.936 | INFO | Downloaded 38GiB to GPU in 2.25434s (16.8564GiB/s) | 20260505 20:47:44.067 | INFO | Analyzing chunk frames = [6193664,6194176) | 20260505 20:47:44.067 | INFO | MemInfo: { | 20260505 20:47:44.067 | INFO | "gpu" : " 17 GB", | 20260505 20:47:44.067 | INFO | "huge" : "", | 20260505 20:47:44.067 | INFO | "huge_shared" : " 27 GB", | 20260505 20:47:44.067 | INFO | "malloc" : " 14 GB", | 20260505 20:47:44.067 | INFO | "pinned" : " 29 GB" | 20260505 20:47:44.067 | INFO | } | 20260505 20:47:44.067 | INFO | Perf report for the last 5.13209 seconds: | 20260505 20:47:44.067 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:47:44.067 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:44.067 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:44.067 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:44.067 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 20:47:44.067 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:44.067 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:44.067 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 20:47:44.067 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:44.067 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:44.068 | INFO | Uploaded 36GiB to GPU in 2.09701s (17.1673GiB/s) | 20260505 20:47:44.068 | INFO | Downloaded 34GiB to GPU in 2.02824s (16.7633GiB/s) | 20260505 20:47:49.208 | INFO | Analyzing chunk frames = [6194176,6194688) | 20260505 20:47:49.208 | INFO | MemInfo: { | 20260505 20:47:49.208 | INFO | "gpu" : " 17 GB", | 20260505 20:47:49.208 | INFO | "huge" : "", | 20260505 20:47:49.208 | INFO | "huge_shared" : " 27 GB", | 20260505 20:47:49.208 | INFO | "malloc" : " 14 GB", | 20260505 20:47:49.208 | INFO | "pinned" : " 29 GB" | 20260505 20:47:49.208 | INFO | } | 20260505 20:47:49.208 | INFO | Perf report for the last 5.14119 seconds: | 20260505 20:47:49.208 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:47:49.208 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:49.209 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:49.209 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:49.209 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260505 20:47:49.209 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:49.209 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:49.209 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 20:47:49.209 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:49.209 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:49.209 | INFO | Uploaded 36GiB to GPU in 2.13611s (16.853GiB/s) | 20260505 20:47:49.209 | INFO | Downloaded 34GiB to GPU in 2.03678s (16.6931GiB/s) | 20260505 20:47:54.348 | INFO | Analyzing chunk frames = [6194688,6195200) | 20260505 20:47:54.348 | INFO | MemInfo: { | 20260505 20:47:54.348 | INFO | "gpu" : " 17 GB", | 20260505 20:47:54.348 | INFO | "huge" : "", | 20260505 20:47:54.348 | INFO | "huge_shared" : " 27 GB", | 20260505 20:47:54.348 | INFO | "malloc" : " 14 GB", | 20260505 20:47:54.348 | INFO | "pinned" : " 29 GB" | 20260505 20:47:54.348 | INFO | } | 20260505 20:47:54.348 | INFO | Perf report for the last 5.13958 seconds: | 20260505 20:47:54.348 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:47:54.348 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:54.348 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:54.348 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:54.348 | INFO | ANALYSIS : 62.4%, 3.0, 97.0ms, 0 | 20260505 20:47:54.348 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:54.348 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:54.348 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:47:54.348 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:54.348 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:54.348 | INFO | Uploaded 36GiB to GPU in 2.21205s (16.2745GiB/s) | 20260505 20:47:54.348 | INFO | Downloaded 34GiB to GPU in 2.04518s (16.6244GiB/s) | 20260505 20:47:59.492 | INFO | Analyzing chunk frames = [6195200,6195712) | 20260505 20:47:59.492 | INFO | MemInfo: { | 20260505 20:47:59.492 | INFO | "gpu" : " 17 GB", | 20260505 20:47:59.492 | INFO | "huge" : "", | 20260505 20:47:59.493 | INFO | "huge_shared" : " 27 GB", | 20260505 20:47:59.493 | INFO | "malloc" : " 14 GB", | 20260505 20:47:59.493 | INFO | "pinned" : " 29 GB" | 20260505 20:47:59.493 | INFO | } | 20260505 20:47:59.493 | INFO | Perf report for the last 5.14461 seconds: | 20260505 20:47:59.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:47:59.493 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:59.493 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:59.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:59.493 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260505 20:47:59.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:59.493 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:59.493 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:47:59.493 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:59.493 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:47:59.493 | INFO | Uploaded 36GiB to GPU in 2.13375s (16.8717GiB/s) | 20260505 20:47:59.493 | INFO | Downloaded 34GiB to GPU in 2.06217s (16.4875GiB/s) | 20260505 20:48:04.631 | INFO | Analyzing chunk frames = [6195712,6196224) | 20260505 20:48:04.631 | INFO | MemInfo: { | 20260505 20:48:04.631 | INFO | "gpu" : " 17 GB", | 20260505 20:48:04.631 | INFO | "huge" : "", | 20260505 20:48:04.631 | INFO | "huge_shared" : " 27 GB", | 20260505 20:48:04.631 | INFO | "malloc" : " 14 GB", | 20260505 20:48:04.631 | INFO | "pinned" : " 29 GB" | 20260505 20:48:04.631 | INFO | } | 20260505 20:48:04.631 | INFO | Perf report for the last 5.13843 seconds: | 20260505 20:48:04.631 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:48:04.631 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:04.631 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:04.631 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:04.631 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 20:48:04.631 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:04.631 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:04.631 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 20:48:04.631 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:04.631 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:04.631 | INFO | Uploaded 36GiB to GPU in 2.09437s (17.189GiB/s) | 20260505 20:48:04.631 | INFO | Downloaded 34GiB to GPU in 2.03573s (16.7016GiB/s) | 20260505 20:48:09.762 | INFO | Analyzing chunk frames = [6196224,6196736) | 20260505 20:48:09.762 | INFO | MemInfo: { | 20260505 20:48:09.762 | INFO | "gpu" : " 17 GB", | 20260505 20:48:09.762 | INFO | "huge" : "", | 20260505 20:48:09.762 | INFO | "huge_shared" : " 27 GB", | 20260505 20:48:09.762 | INFO | "malloc" : " 14 GB", | 20260505 20:48:09.762 | INFO | "pinned" : " 29 GB" | 20260505 20:48:09.762 | INFO | } | 20260505 20:48:09.762 | INFO | Perf report for the last 5.13126 seconds: | 20260505 20:48:09.762 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:48:09.762 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:09.762 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:09.762 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:09.763 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 20:48:09.763 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:09.763 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:09.763 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 20:48:09.763 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:09.763 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:09.763 | INFO | Uploaded 36GiB to GPU in 2.11509s (17.0205GiB/s) | 20260505 20:48:09.763 | INFO | Downloaded 34GiB to GPU in 2.15005s (15.8136GiB/s) | 20260505 20:48:14.902 | INFO | Analyzing chunk frames = [6196736,6197248) | 20260505 20:48:14.903 | INFO | MemInfo: { | 20260505 20:48:14.903 | INFO | "gpu" : " 17 GB", | 20260505 20:48:14.903 | INFO | "huge" : "", | 20260505 20:48:14.903 | INFO | "huge_shared" : " 27 GB", | 20260505 20:48:14.903 | INFO | "malloc" : " 14 GB", | 20260505 20:48:14.903 | INFO | "pinned" : " 29 GB" | 20260505 20:48:14.903 | INFO | } | 20260505 20:48:14.903 | INFO | Perf report for the last 5.14028 seconds: | 20260505 20:48:14.903 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:48:14.903 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:14.903 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:14.903 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:14.903 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 20:48:14.903 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:14.903 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:14.903 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:48:14.903 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:14.903 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:14.903 | INFO | Uploaded 36GiB to GPU in 2.19679s (16.3876GiB/s) | 20260505 20:48:14.903 | INFO | Downloaded 34GiB to GPU in 2.03755s (16.6867GiB/s) | 20260505 20:48:15.031 | INFO | Starting a round of Baz IO | 20260505 20:48:18.196 | INFO | Populating RTMetrics report | 20260505 20:48:18.197 | INFO | RTMetrics report populated | 20260505 20:48:18.197 | INFO | Opening tmp RTMetrics json file | 20260505 20:48:18.197 | INFO | Serializing RTMetrics to json | 20260505 20:48:18.198 | INFO | Serialization done, swapping tmp file now... | 20260505 20:48:18.199 | INFO | RTMetrics done | 20260505 20:48:18.288 | INFO | Finished a round of Baz IO: Wrote 1.999 GiB in 3.256 seconds | 20260505 20:48:18.288 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.63% / 0.00% / 78.36% | 20260505 20:48:18.288 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.165776% / 0% / 9.93411e-05% | 20260505 20:48:18.288 | INFO | Baz IO overall: | 20260505 20:48:18.288 | INFO | BytesWritten : 3603679229897 | 20260505 20:48:18.288 | INFO | NumEvents : 1702890373550 | 20260505 20:48:20.048 | INFO | Analyzing chunk frames = [6197248,6197760) | 20260505 20:48:20.048 | INFO | MemInfo: { | 20260505 20:48:20.048 | INFO | "gpu" : " 17 GB", | 20260505 20:48:20.048 | INFO | "huge" : "", | 20260505 20:48:20.048 | INFO | "huge_shared" : " 27 GB", | 20260505 20:48:20.048 | INFO | "malloc" : " 14 GB", | 20260505 20:48:20.048 | INFO | "pinned" : " 29 GB" | 20260505 20:48:20.048 | INFO | } | 20260505 20:48:20.048 | INFO | Perf report for the last 5.14524 seconds: | 20260505 20:48:20.048 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:48:20.048 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:20.048 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:20.048 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:20.048 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 20:48:20.048 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:20.048 | INFO | RT_METRICS : 43.9%, 1.0, 23.1ms, 0 | 20260505 20:48:20.048 | INFO | PRE_HQ : 60.0%, 1.8, 55.4ms, 0 | 20260505 20:48:20.048 | INFO | BAZWRITER : 62.7%, 3.7, 120.0ms, 0 | 20260505 20:48:20.048 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 20:48:20.048 | INFO | Uploaded 36GiB to GPU in 2.16668s (16.6152GiB/s) | 20260505 20:48:20.048 | INFO | Downloaded 38GiB to GPU in 2.25166s (16.8765GiB/s) | 20260505 20:48:25.192 | INFO | Analyzing chunk frames = [6197760,6198272) | 20260505 20:48:25.192 | INFO | MemInfo: { | 20260505 20:48:25.192 | INFO | "gpu" : " 17 GB", | 20260505 20:48:25.192 | INFO | "huge" : "", | 20260505 20:48:25.192 | INFO | "huge_shared" : " 27 GB", | 20260505 20:48:25.192 | INFO | "malloc" : " 14 GB", | 20260505 20:48:25.192 | INFO | "pinned" : " 29 GB" | 20260505 20:48:25.192 | INFO | } | 20260505 20:48:25.192 | INFO | Perf report for the last 5.1443 seconds: | 20260505 20:48:25.192 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:48:25.192 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:25.192 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:25.192 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:25.192 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260505 20:48:25.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:25.192 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:25.192 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:48:25.192 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:25.192 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:25.193 | INFO | Uploaded 36GiB to GPU in 2.08649s (17.2539GiB/s) | 20260505 20:48:25.193 | INFO | Downloaded 34GiB to GPU in 2.02655s (16.7773GiB/s) | 20260505 20:48:30.327 | INFO | Analyzing chunk frames = [6198272,6198784) | 20260505 20:48:30.327 | INFO | MemInfo: { | 20260505 20:48:30.327 | INFO | "gpu" : " 17 GB", | 20260505 20:48:30.327 | INFO | "huge" : "", | 20260505 20:48:30.327 | INFO | "huge_shared" : " 27 GB", | 20260505 20:48:30.327 | INFO | "malloc" : " 14 GB", | 20260505 20:48:30.327 | INFO | "pinned" : " 29 GB" | 20260505 20:48:30.327 | INFO | } | 20260505 20:48:30.327 | INFO | Perf report for the last 5.13492 seconds: | 20260505 20:48:30.327 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:48:30.327 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:30.327 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:30.327 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:30.327 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260505 20:48:30.327 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:30.327 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:30.327 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:48:30.327 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:30.327 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:30.327 | INFO | Uploaded 36GiB to GPU in 2.14865s (16.7547GiB/s) | 20260505 20:48:30.327 | INFO | Downloaded 34GiB to GPU in 2.04835s (16.5987GiB/s) | 20260505 20:48:35.462 | INFO | Analyzing chunk frames = [6198784,6199296) | 20260505 20:48:35.462 | INFO | MemInfo: { | 20260505 20:48:35.462 | INFO | "gpu" : " 17 GB", | 20260505 20:48:35.462 | INFO | "huge" : "", | 20260505 20:48:35.462 | INFO | "huge_shared" : " 27 GB", | 20260505 20:48:35.462 | INFO | "malloc" : " 14 GB", | 20260505 20:48:35.462 | INFO | "pinned" : " 29 GB" | 20260505 20:48:35.462 | INFO | } | 20260505 20:48:35.462 | INFO | Perf report for the last 5.13499 seconds: | 20260505 20:48:35.462 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:48:35.462 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:35.462 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:35.462 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:35.462 | INFO | ANALYSIS : 62.4%, 3.0, 96.8ms, 0 | 20260505 20:48:35.462 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:35.462 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:35.462 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 20:48:35.462 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:35.462 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:35.463 | INFO | Uploaded 36GiB to GPU in 2.19779s (16.3801GiB/s) | 20260505 20:48:35.463 | INFO | Downloaded 34GiB to GPU in 2.06168s (16.4914GiB/s) | 20260505 20:48:40.605 | INFO | Analyzing chunk frames = [6199296,6199808) | 20260505 20:48:40.605 | INFO | MemInfo: { | 20260505 20:48:40.605 | INFO | "gpu" : " 17 GB", | 20260505 20:48:40.605 | INFO | "huge" : "", | 20260505 20:48:40.605 | INFO | "huge_shared" : " 27 GB", | 20260505 20:48:40.605 | INFO | "malloc" : " 14 GB", | 20260505 20:48:40.605 | INFO | "pinned" : " 29 GB" | 20260505 20:48:40.605 | INFO | } | 20260505 20:48:40.605 | INFO | Perf report for the last 5.14298 seconds: | 20260505 20:48:40.605 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:48:40.605 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:40.605 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:40.605 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:40.605 | INFO | ANALYSIS : 61.7%, 3.0, 96.5ms, 0 | 20260505 20:48:40.605 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:40.605 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:40.605 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:48:40.605 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:40.605 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:40.606 | INFO | Uploaded 36GiB to GPU in 2.13622s (16.8522GiB/s) | 20260505 20:48:40.606 | INFO | Downloaded 34GiB to GPU in 2.06634s (16.4542GiB/s) | 20260505 20:48:45.743 | INFO | Analyzing chunk frames = [6199808,6200320) | 20260505 20:48:45.743 | INFO | MemInfo: { | 20260505 20:48:45.743 | INFO | "gpu" : " 17 GB", | 20260505 20:48:45.743 | INFO | "huge" : "", | 20260505 20:48:45.744 | INFO | "huge_shared" : " 27 GB", | 20260505 20:48:45.744 | INFO | "malloc" : " 14 GB", | 20260505 20:48:45.744 | INFO | "pinned" : " 29 GB" | 20260505 20:48:45.744 | INFO | } | 20260505 20:48:45.744 | INFO | Perf report for the last 5.13849 seconds: | 20260505 20:48:45.744 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:48:45.744 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:45.744 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:45.744 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:45.744 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 20:48:45.744 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:45.744 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:45.744 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:48:45.744 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:45.744 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:45.744 | INFO | Uploaded 36GiB to GPU in 2.13931s (16.8279GiB/s) | 20260505 20:48:45.744 | INFO | Downloaded 34GiB to GPU in 2.05946s (16.5092GiB/s) | 20260505 20:48:50.887 | INFO | Analyzing chunk frames = [6200320,6200832) | 20260505 20:48:50.887 | INFO | MemInfo: { | 20260505 20:48:50.887 | INFO | "gpu" : " 17 GB", | 20260505 20:48:50.887 | INFO | "huge" : "", | 20260505 20:48:50.887 | INFO | "huge_shared" : " 27 GB", | 20260505 20:48:50.887 | INFO | "malloc" : " 14 GB", | 20260505 20:48:50.887 | INFO | "pinned" : " 29 GB" | 20260505 20:48:50.887 | INFO | } | 20260505 20:48:50.887 | INFO | Perf report for the last 5.14327 seconds: | 20260505 20:48:50.887 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:48:50.887 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:50.887 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:50.887 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:50.887 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 20:48:50.887 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:50.887 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:50.887 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260505 20:48:50.887 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:50.887 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:50.887 | INFO | Uploaded 36GiB to GPU in 2.16077s (16.6607GiB/s) | 20260505 20:48:50.887 | INFO | Downloaded 34GiB to GPU in 2.12439s (16.0046GiB/s) | 20260505 20:48:56.023 | INFO | Analyzing chunk frames = [6200832,6201344) | 20260505 20:48:56.023 | INFO | MemInfo: { | 20260505 20:48:56.023 | INFO | "gpu" : " 17 GB", | 20260505 20:48:56.023 | INFO | "huge" : "", | 20260505 20:48:56.023 | INFO | "huge_shared" : " 27 GB", | 20260505 20:48:56.023 | INFO | "malloc" : " 14 GB", | 20260505 20:48:56.023 | INFO | "pinned" : " 29 GB" | 20260505 20:48:56.023 | INFO | } | 20260505 20:48:56.023 | INFO | Perf report for the last 5.13615 seconds: | 20260505 20:48:56.023 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:48:56.023 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:56.023 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:56.023 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:56.023 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 20:48:56.023 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:56.023 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:56.023 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:48:56.023 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:56.023 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:48:56.023 | INFO | Uploaded 36GiB to GPU in 2.10597s (17.0943GiB/s) | 20260505 20:48:56.024 | INFO | Downloaded 34GiB to GPU in 2.00337s (16.9714GiB/s) | 20260505 20:48:56.024 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 20:48:56.024 | INFO | Total mem usage: 15.3 GiB | 20260505 20:48:56.024 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 20:48:56.024 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.82%) | 20260505 20:48:56.024 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 20:48:56.024 | INFO | High Water Breakdown: | 20260505 20:48:56.024 | INFO | 60.9% : 8.57 GiB ( 90.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 20:48:56.024 | INFO | 39% : 5.49 GiB ( 82.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 20:48:56.024 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 20:48:56.024 | INFO | ------------------------------------------ | 20260505 20:48:56.024 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:48:56.024 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 20:48:56.024 | INFO | Total mem usage: 26.5 GiB | 20260505 20:48:56.024 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 20:48:56.024 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 20:48:56.024 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 20:48:56.024 | INFO | High Water Breakdown: | 20260505 20:48:56.024 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:48:56.024 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:48:56.024 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:48:56.024 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:48:56.024 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:48:56.024 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:48:56.024 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 20:48:56.024 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 20:48:56.024 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:48:56.024 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 20:48:56.024 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 20:48:56.024 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 20:48:56.024 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 20:48:56.024 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 20:48:56.024 | INFO | ------------------------------------------ | 20260505 20:48:56.024 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:48:56.024 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 20:48:56.024 | INFO | Total mem usage: 25.6 GiB | 20260505 20:48:56.024 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 20:48:56.024 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 20:48:56.024 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 20:48:56.024 | INFO | High Water Breakdown: | 20260505 20:48:56.024 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 20:48:56.024 | INFO | ------------------------------------------ | 20260505 20:48:56.024 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:48:56.024 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 20:48:56.024 | INFO | Total mem usage: 15.7 GiB | 20260505 20:48:56.024 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 20:48:56.024 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 20:48:56.024 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 20:48:56.024 | INFO | High Water Breakdown: | 20260505 20:48:56.024 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 20:48:56.024 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 20:48:56.024 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 20:48:56.024 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 20:48:56.024 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 20:48:56.024 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 20:48:56.024 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 20:48:56.024 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 20:48:56.024 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 20:48:56.024 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:48:56.024 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 20:48:56.024 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:48:56.024 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 20:48:56.024 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 20:48:56.024 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 20:48:56.024 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 20:48:56.024 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:48:56.024 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 20:48:56.024 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:48:56.024 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:48:56.024 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:48:56.024 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 20:48:56.024 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 20:48:56.024 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:48:56.024 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 20:48:56.024 | INFO | ------------------------------------------ | 20260505 20:48:56.024 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:48:56.024 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 20:48:56.024 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:48:56.024 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:48:56.024 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.42e+01, 1.33e+01 / 1.62e+00 | 20260505 20:48:56.024 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.03e-01, 1.09e-02 / 3.37e-03 | 20260505 20:48:56.024 | INFO | PerfReport: 11.81%, Baseline -- 3.71e+00 / 4.57e+00, 5.71e-01 / 1.66e-01 | 20260505 20:48:56.024 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.51e+00, 1.90e-01 / 1.16e-01 | 20260505 20:48:56.024 | INFO | PerfReport: 15.94%, DME -- 4.01e+01 / 4.40e+01, 3.92e+00 / 1.61e+00 | 20260505 20:48:56.024 | INFO | PerfReport: 57.08%, FrameLabeling -- 1.79e+01 / 1.94e+01, 1.47e+00 / 1.01e+00 | 20260505 20:48:56.024 | INFO | PerfReport: 7.23%, PulseAccumulating -- 2.27e+00 / 2.83e+00, 4.35e-01 / 1.81e-01 | 20260505 20:48:56.024 | INFO | PerfReport: 3.24%, Metrics -- 1.02e+00 / 1.33e+00, 2.79e-01 / 1.46e-01 | 20260505 20:48:56.024 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.54e-01 / 4.12e-01, 6.22e-02 / 3.82e-02 | 20260505 20:48:56.025 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 20:48:56.025 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:48:56.025 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:48:56.025 | INFO | PerfReport: Full Execution -- 9.92e+01 / -nan, 2.50e+01 / 0.00e+00 | 20260505 20:48:56.025 | INFO | PerfReport: 22.51%, Upload -- 2.23e+01 / 0.00e+00, 7.70e+00 / 0.00e+00 | 20260505 20:48:56.025 | INFO | PerfReport: 21.84%, Download -- 2.17e+01 / 0.00e+00, 7.71e+00 / 0.00e+00 | 20260505 20:48:56.025 | INFO | PerfReport: 31.69%, Compute -- 3.14e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260505 20:48:56.025 | INFO | PerfReport: 2.25%, LockUpload -- 2.23e+00 / 0.00e+00, 7.89e+00 / 0.00e+00 | 20260505 20:48:56.025 | INFO | PerfReport: 1.38%, LockDownload -- 1.37e+00 / 0.00e+00, 2.94e+00 / 0.00e+00 | 20260505 20:48:56.025 | INFO | PerfReport: 20.32%, LockCompute -- 2.02e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 20:48:56.025 | INFO | PerfReport: 0.01%, Misc -- 1.27e-02 / 0.00e+00, 1.52e-02 / 0.00e+00 | 20260505 20:48:56.157 | INFO | Starting a round of Baz IO | 20260505 20:48:59.321 | INFO | Populating RTMetrics report | 20260505 20:48:59.321 | INFO | RTMetrics report populated | 20260505 20:48:59.321 | INFO | Opening tmp RTMetrics json file | 20260505 20:48:59.323 | INFO | Serializing RTMetrics to json | 20260505 20:48:59.327 | INFO | Serialization done, swapping tmp file now... | 20260505 20:48:59.328 | INFO | RTMetrics done | 20260505 20:48:59.404 | INFO | Finished a round of Baz IO: Wrote 1.999 GiB in 3.247 seconds | 20260505 20:48:59.405 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.63% / 0.00% / 78.37% | 20260505 20:48:59.405 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.164525% / 0% / 7.94729e-05% | 20260505 20:48:59.405 | INFO | Baz IO overall: | 20260505 20:48:59.405 | INFO | BytesWritten : 3605825520181 | 20260505 20:48:59.405 | INFO | NumEvents : 1703883790567 | 20260505 20:49:01.163 | INFO | Analyzing chunk frames = [6201344,6201856) | 20260505 20:49:01.163 | INFO | MemInfo: { | 20260505 20:49:01.163 | INFO | "gpu" : " 17 GB", | 20260505 20:49:01.163 | INFO | "huge" : "", | 20260505 20:49:01.163 | INFO | "huge_shared" : " 27 GB", | 20260505 20:49:01.163 | INFO | "malloc" : " 14 GB", | 20260505 20:49:01.163 | INFO | "pinned" : " 29 GB" | 20260505 20:49:01.163 | INFO | } | 20260505 20:49:01.163 | INFO | Perf report for the last 5.13993 seconds: | 20260505 20:49:01.163 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:49:01.163 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:01.163 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:01.163 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:01.163 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 20:49:01.163 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:01.163 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260505 20:49:01.163 | INFO | PRE_HQ : 59.7%, 1.8, 55.1ms, 0 | 20260505 20:49:01.163 | INFO | BAZWRITER : 62.6%, 3.6, 117.4ms, 0 | 20260505 20:49:01.163 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 20:49:01.163 | INFO | Uploaded 36GiB to GPU in 2.16298s (16.6437GiB/s) | 20260505 20:49:01.164 | INFO | Downloaded 38GiB to GPU in 2.25995s (16.8145GiB/s) | 20260505 20:49:06.299 | INFO | Analyzing chunk frames = [6201856,6202368) | 20260505 20:49:06.299 | INFO | MemInfo: { | 20260505 20:49:06.299 | INFO | "gpu" : " 17 GB", | 20260505 20:49:06.299 | INFO | "huge" : "", | 20260505 20:49:06.299 | INFO | "huge_shared" : " 27 GB", | 20260505 20:49:06.299 | INFO | "malloc" : " 14 GB", | 20260505 20:49:06.299 | INFO | "pinned" : " 29 GB" | 20260505 20:49:06.299 | INFO | } | 20260505 20:49:06.299 | INFO | Perf report for the last 5.1361 seconds: | 20260505 20:49:06.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:49:06.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:06.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:06.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:06.299 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 20:49:06.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:06.299 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:06.299 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 20:49:06.299 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:06.299 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:06.300 | INFO | Uploaded 36GiB to GPU in 2.07855s (17.3198GiB/s) | 20260505 20:49:06.300 | INFO | Downloaded 34GiB to GPU in 2.02702s (16.7734GiB/s) | 20260505 20:49:11.441 | INFO | Analyzing chunk frames = [6202368,6202880) | 20260505 20:49:11.441 | INFO | MemInfo: { | 20260505 20:49:11.441 | INFO | "gpu" : " 17 GB", | 20260505 20:49:11.441 | INFO | "huge" : "", | 20260505 20:49:11.441 | INFO | "huge_shared" : " 27 GB", | 20260505 20:49:11.441 | INFO | "malloc" : " 14 GB", | 20260505 20:49:11.441 | INFO | "pinned" : " 29 GB" | 20260505 20:49:11.441 | INFO | } | 20260505 20:49:11.441 | INFO | Perf report for the last 5.14222 seconds: | 20260505 20:49:11.441 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:49:11.441 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:11.441 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:11.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:11.442 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260505 20:49:11.442 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:11.442 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:11.442 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:49:11.442 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:11.442 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:11.442 | INFO | Uploaded 36GiB to GPU in 2.13459s (16.8651GiB/s) | 20260505 20:49:11.442 | INFO | Downloaded 34GiB to GPU in 2.02356s (16.8021GiB/s) | 20260505 20:49:16.575 | INFO | Analyzing chunk frames = [6202880,6203392) | 20260505 20:49:16.575 | INFO | MemInfo: { | 20260505 20:49:16.575 | INFO | "gpu" : " 17 GB", | 20260505 20:49:16.575 | INFO | "huge" : "", | 20260505 20:49:16.575 | INFO | "huge_shared" : " 27 GB", | 20260505 20:49:16.575 | INFO | "malloc" : " 14 GB", | 20260505 20:49:16.575 | INFO | "pinned" : " 29 GB" | 20260505 20:49:16.575 | INFO | } | 20260505 20:49:16.575 | INFO | Perf report for the last 5.13391 seconds: | 20260505 20:49:16.575 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:49:16.575 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:16.575 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:16.575 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:16.575 | INFO | ANALYSIS : 62.5%, 3.0, 97.0ms, 0 | 20260505 20:49:16.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:16.575 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:16.575 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:49:16.576 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:16.576 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:16.576 | INFO | Uploaded 36GiB to GPU in 2.20518s (16.3252GiB/s) | 20260505 20:49:16.576 | INFO | Downloaded 34GiB to GPU in 2.04215s (16.6491GiB/s) | 20260505 20:49:21.713 | INFO | Analyzing chunk frames = [6203392,6203904) | 20260505 20:49:21.713 | INFO | MemInfo: { | 20260505 20:49:21.714 | INFO | "gpu" : " 17 GB", | 20260505 20:49:21.714 | INFO | "huge" : "", | 20260505 20:49:21.714 | INFO | "huge_shared" : " 27 GB", | 20260505 20:49:21.714 | INFO | "malloc" : " 14 GB", | 20260505 20:49:21.714 | INFO | "pinned" : " 29 GB" | 20260505 20:49:21.714 | INFO | } | 20260505 20:49:21.714 | INFO | Perf report for the last 5.13838 seconds: | 20260505 20:49:21.714 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:49:21.714 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:21.714 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:21.714 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:21.714 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 20:49:21.714 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:21.714 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:21.714 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:49:21.714 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:21.714 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:21.714 | INFO | Uploaded 36GiB to GPU in 2.14272s (16.8011GiB/s) | 20260505 20:49:21.714 | INFO | Downloaded 34GiB to GPU in 2.06406s (16.4724GiB/s) | 20260505 20:49:26.861 | INFO | Analyzing chunk frames = [6203904,6204416) | 20260505 20:49:26.861 | INFO | MemInfo: { | 20260505 20:49:26.861 | INFO | "gpu" : " 17 GB", | 20260505 20:49:26.861 | INFO | "huge" : "", | 20260505 20:49:26.861 | INFO | "huge_shared" : " 27 GB", | 20260505 20:49:26.861 | INFO | "malloc" : " 14 GB", | 20260505 20:49:26.861 | INFO | "pinned" : " 29 GB" | 20260505 20:49:26.861 | INFO | } | 20260505 20:49:26.861 | INFO | Perf report for the last 5.14731 seconds: | 20260505 20:49:26.861 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:49:26.861 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:26.861 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:26.861 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:26.861 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260505 20:49:26.861 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:26.861 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:26.861 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:49:26.861 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:26.861 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:26.861 | INFO | Uploaded 36GiB to GPU in 2.15436s (16.7103GiB/s) | 20260505 20:49:26.861 | INFO | Downloaded 34GiB to GPU in 2.06084s (16.4981GiB/s) | 20260505 20:49:32.006 | INFO | Analyzing chunk frames = [6204416,6204928) | 20260505 20:49:32.006 | INFO | MemInfo: { | 20260505 20:49:32.006 | INFO | "gpu" : " 17 GB", | 20260505 20:49:32.006 | INFO | "huge" : "", | 20260505 20:49:32.006 | INFO | "huge_shared" : " 27 GB", | 20260505 20:49:32.006 | INFO | "malloc" : " 14 GB", | 20260505 20:49:32.006 | INFO | "pinned" : " 29 GB" | 20260505 20:49:32.007 | INFO | } | 20260505 20:49:32.007 | INFO | Perf report for the last 5.14545 seconds: | 20260505 20:49:32.007 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:49:32.007 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:32.007 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:32.007 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:32.007 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 20:49:32.007 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:32.007 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:32.007 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 20:49:32.007 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:32.007 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:32.007 | INFO | Uploaded 36GiB to GPU in 2.16166s (16.6539GiB/s) | 20260505 20:49:32.007 | INFO | Downloaded 34GiB to GPU in 2.12424s (16.0057GiB/s) | 20260505 20:49:37.136 | INFO | Analyzing chunk frames = [6204928,6205440) | 20260505 20:49:37.136 | INFO | MemInfo: { | 20260505 20:49:37.136 | INFO | "gpu" : " 17 GB", | 20260505 20:49:37.136 | INFO | "huge" : "", | 20260505 20:49:37.136 | INFO | "huge_shared" : " 27 GB", | 20260505 20:49:37.136 | INFO | "malloc" : " 14 GB", | 20260505 20:49:37.136 | INFO | "pinned" : " 29 GB" | 20260505 20:49:37.136 | INFO | } | 20260505 20:49:37.136 | INFO | Perf report for the last 5.12977 seconds: | 20260505 20:49:37.136 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:49:37.136 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:37.136 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:37.136 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:37.136 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260505 20:49:37.136 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:37.136 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:37.136 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:49:37.136 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:37.137 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:37.137 | INFO | Uploaded 36GiB to GPU in 2.11271s (17.0397GiB/s) | 20260505 20:49:37.137 | INFO | Downloaded 34GiB to GPU in 2.0041s (16.9652GiB/s) | 20260505 20:49:37.270 | INFO | Starting a round of Baz IO | 20260505 20:49:40.437 | INFO | Populating RTMetrics report | 20260505 20:49:40.437 | INFO | RTMetrics report populated | 20260505 20:49:40.437 | INFO | Opening tmp RTMetrics json file | 20260505 20:49:40.438 | INFO | Serializing RTMetrics to json | 20260505 20:49:40.439 | INFO | Serialization done, swapping tmp file now... | 20260505 20:49:40.441 | INFO | RTMetrics done | 20260505 20:49:40.527 | INFO | Finished a round of Baz IO: Wrote 1.995 GiB in 3.257 seconds | 20260505 20:49:40.528 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.63% / 0.00% / 78.37% | 20260505 20:49:40.528 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.163945% / 3.97364e-06% / 6.75519e-05% | 20260505 20:49:40.528 | INFO | Baz IO overall: | 20260505 20:49:40.528 | INFO | BytesWritten : 3607967515240 | 20260505 20:49:40.528 | INFO | NumEvents : 1704874278816 | 20260505 20:49:42.268 | INFO | Analyzing chunk frames = [6205440,6205952) | 20260505 20:49:42.268 | INFO | MemInfo: { | 20260505 20:49:42.268 | INFO | "gpu" : " 17 GB", | 20260505 20:49:42.268 | INFO | "huge" : "", | 20260505 20:49:42.268 | INFO | "huge_shared" : " 27 GB", | 20260505 20:49:42.268 | INFO | "malloc" : " 14 GB", | 20260505 20:49:42.268 | INFO | "pinned" : " 29 GB" | 20260505 20:49:42.268 | INFO | } | 20260505 20:49:42.268 | INFO | Perf report for the last 5.13166 seconds: | 20260505 20:49:42.268 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:49:42.268 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:42.268 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:42.268 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:42.268 | INFO | ANALYSIS : 63.3%, 3.0, 98.5ms, 0 | 20260505 20:49:42.268 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:42.268 | INFO | RT_METRICS : 44.0%, 1.0, 23.0ms, 0 | 20260505 20:49:42.268 | INFO | PRE_HQ : 60.2%, 1.8, 55.2ms, 0 | 20260505 20:49:42.268 | INFO | BAZWRITER : 63.0%, 3.7, 119.2ms, 0 | 20260505 20:49:42.268 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 20:49:42.268 | INFO | Uploaded 36GiB to GPU in 2.16638s (16.6176GiB/s) | 20260505 20:49:42.268 | INFO | Downloaded 38GiB to GPU in 2.25793s (16.8296GiB/s) | 20260505 20:49:47.413 | INFO | Analyzing chunk frames = [6205952,6206464) | 20260505 20:49:47.413 | INFO | MemInfo: { | 20260505 20:49:47.413 | INFO | "gpu" : " 17 GB", | 20260505 20:49:47.413 | INFO | "huge" : "", | 20260505 20:49:47.413 | INFO | "huge_shared" : " 27 GB", | 20260505 20:49:47.413 | INFO | "malloc" : " 14 GB", | 20260505 20:49:47.413 | INFO | "pinned" : " 29 GB" | 20260505 20:49:47.413 | INFO | } | 20260505 20:49:47.414 | INFO | Perf report for the last 5.14546 seconds: | 20260505 20:49:47.414 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:49:47.414 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:47.414 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:47.414 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:47.414 | INFO | ANALYSIS : 62.4%, 3.0, 97.6ms, 0 | 20260505 20:49:47.414 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:47.414 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:47.414 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:49:47.414 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:47.414 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:47.414 | INFO | Uploaded 36GiB to GPU in 2.13753s (16.8419GiB/s) | 20260505 20:49:47.414 | INFO | Downloaded 34GiB to GPU in 2.03957s (16.6702GiB/s) | 20260505 20:49:52.556 | INFO | Analyzing chunk frames = [6206464,6206976) | 20260505 20:49:52.556 | INFO | MemInfo: { | 20260505 20:49:52.556 | INFO | "gpu" : " 17 GB", | 20260505 20:49:52.556 | INFO | "huge" : "", | 20260505 20:49:52.556 | INFO | "huge_shared" : " 27 GB", | 20260505 20:49:52.556 | INFO | "malloc" : " 14 GB", | 20260505 20:49:52.556 | INFO | "pinned" : " 29 GB" | 20260505 20:49:52.556 | INFO | } | 20260505 20:49:52.556 | INFO | Perf report for the last 5.14262 seconds: | 20260505 20:49:52.556 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:49:52.556 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:52.556 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:52.556 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:52.556 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260505 20:49:52.556 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:52.556 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:52.556 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:49:52.556 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:52.556 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:52.556 | INFO | Uploaded 36GiB to GPU in 2.12652s (16.9291GiB/s) | 20260505 20:49:52.556 | INFO | Downloaded 34GiB to GPU in 2.02529s (16.7877GiB/s) | 20260505 20:49:57.701 | INFO | Analyzing chunk frames = [6206976,6207488) | 20260505 20:49:57.701 | INFO | MemInfo: { | 20260505 20:49:57.701 | INFO | "gpu" : " 17 GB", | 20260505 20:49:57.701 | INFO | "huge" : "", | 20260505 20:49:57.701 | INFO | "huge_shared" : " 27 GB", | 20260505 20:49:57.701 | INFO | "malloc" : " 14 GB", | 20260505 20:49:57.701 | INFO | "pinned" : " 29 GB" | 20260505 20:49:57.701 | INFO | } | 20260505 20:49:57.701 | INFO | Perf report for the last 5.14517 seconds: | 20260505 20:49:57.701 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:49:57.701 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:57.701 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:57.701 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:57.701 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260505 20:49:57.701 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:57.701 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:57.701 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:49:57.701 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:57.701 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:49:57.702 | INFO | Uploaded 36GiB to GPU in 2.14318s (16.7975GiB/s) | 20260505 20:49:57.702 | INFO | Downloaded 34GiB to GPU in 2.04444s (16.6305GiB/s) | 20260505 20:50:02.835 | INFO | Analyzing chunk frames = [6207488,6208000) | 20260505 20:50:02.835 | INFO | MemInfo: { | 20260505 20:50:02.835 | INFO | "gpu" : " 17 GB", | 20260505 20:50:02.835 | INFO | "huge" : "", | 20260505 20:50:02.835 | INFO | "huge_shared" : " 27 GB", | 20260505 20:50:02.835 | INFO | "malloc" : " 14 GB", | 20260505 20:50:02.835 | INFO | "pinned" : " 29 GB" | 20260505 20:50:02.835 | INFO | } | 20260505 20:50:02.835 | INFO | Perf report for the last 5.13405 seconds: | 20260505 20:50:02.835 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:50:02.835 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:02.835 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:02.835 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:02.835 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260505 20:50:02.835 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:02.835 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:02.835 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 20:50:02.835 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:02.835 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:02.836 | INFO | Uploaded 36GiB to GPU in 2.12749s (16.9213GiB/s) | 20260505 20:50:02.836 | INFO | Downloaded 34GiB to GPU in 2.06215s (16.4876GiB/s) | 20260505 20:50:07.971 | INFO | Analyzing chunk frames = [6208000,6208512) | 20260505 20:50:07.971 | INFO | MemInfo: { | 20260505 20:50:07.971 | INFO | "gpu" : " 17 GB", | 20260505 20:50:07.971 | INFO | "huge" : "", | 20260505 20:50:07.971 | INFO | "huge_shared" : " 27 GB", | 20260505 20:50:07.971 | INFO | "malloc" : " 14 GB", | 20260505 20:50:07.971 | INFO | "pinned" : " 29 GB" | 20260505 20:50:07.971 | INFO | } | 20260505 20:50:07.971 | INFO | Perf report for the last 5.1354 seconds: | 20260505 20:50:07.971 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:50:07.971 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:07.971 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:07.971 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:07.971 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260505 20:50:07.971 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:07.971 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:07.971 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 20:50:07.971 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:07.971 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:07.971 | INFO | Uploaded 36GiB to GPU in 2.15074s (16.7384GiB/s) | 20260505 20:50:07.971 | INFO | Downloaded 34GiB to GPU in 2.05961s (16.508GiB/s) | 20260505 20:50:13.118 | INFO | Analyzing chunk frames = [6208512,6209024) | 20260505 20:50:13.118 | INFO | MemInfo: { | 20260505 20:50:13.118 | INFO | "gpu" : " 17 GB", | 20260505 20:50:13.118 | INFO | "huge" : "", | 20260505 20:50:13.118 | INFO | "huge_shared" : " 27 GB", | 20260505 20:50:13.118 | INFO | "malloc" : " 14 GB", | 20260505 20:50:13.118 | INFO | "pinned" : " 29 GB" | 20260505 20:50:13.118 | INFO | } | 20260505 20:50:13.118 | INFO | Perf report for the last 5.14718 seconds: | 20260505 20:50:13.118 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:50:13.118 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:13.118 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:13.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:13.118 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 20:50:13.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:13.118 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:13.118 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 20:50:13.118 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:13.118 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:13.118 | INFO | Uploaded 36GiB to GPU in 2.11392s (17.03GiB/s) | 20260505 20:50:13.118 | INFO | Downloaded 34GiB to GPU in 2.13728s (15.9081GiB/s) | 20260505 20:50:18.251 | INFO | Analyzing chunk frames = [6209024,6209536) | 20260505 20:50:18.251 | INFO | MemInfo: { | 20260505 20:50:18.251 | INFO | "gpu" : " 17 GB", | 20260505 20:50:18.251 | INFO | "huge" : "", | 20260505 20:50:18.251 | INFO | "huge_shared" : " 27 GB", | 20260505 20:50:18.251 | INFO | "malloc" : " 14 GB", | 20260505 20:50:18.251 | INFO | "pinned" : " 29 GB" | 20260505 20:50:18.251 | INFO | } | 20260505 20:50:18.251 | INFO | Perf report for the last 5.13323 seconds: | 20260505 20:50:18.251 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:50:18.251 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:18.251 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:18.251 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:18.251 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 20:50:18.251 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:18.251 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:18.251 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:50:18.251 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:18.251 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:18.252 | INFO | Uploaded 36GiB to GPU in 2.11123s (17.0517GiB/s) | 20260505 20:50:18.252 | INFO | Downloaded 34GiB to GPU in 2.00406s (16.9656GiB/s) | 20260505 20:50:18.252 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 20:50:18.252 | INFO | Total mem usage: 15.3 GiB | 20260505 20:50:18.252 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 20:50:18.252 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -7.44%) | 20260505 20:50:18.252 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 20:50:18.252 | INFO | High Water Breakdown: | 20260505 20:50:18.252 | INFO | 61.1% : 8.72 GiB ( 91.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 20:50:18.252 | INFO | 38.8% : 5.53 GiB ( 83.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 20:50:18.252 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 20:50:18.252 | INFO | ------------------------------------------ | 20260505 20:50:18.252 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:50:18.252 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 20:50:18.252 | INFO | Total mem usage: 26.5 GiB | 20260505 20:50:18.252 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 20:50:18.252 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 20:50:18.252 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 20:50:18.252 | INFO | High Water Breakdown: | 20260505 20:50:18.252 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:50:18.252 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:50:18.252 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:50:18.252 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:50:18.252 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:50:18.252 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:50:18.252 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 20:50:18.252 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 20:50:18.252 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:50:18.252 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 20:50:18.252 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 20:50:18.252 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 20:50:18.252 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 20:50:18.252 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 20:50:18.252 | INFO | ------------------------------------------ | 20260505 20:50:18.252 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:50:18.252 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 20:50:18.252 | INFO | Total mem usage: 25.6 GiB | 20260505 20:50:18.252 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 20:50:18.252 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 20:50:18.252 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 20:50:18.252 | INFO | High Water Breakdown: | 20260505 20:50:18.252 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 20:50:18.252 | INFO | ------------------------------------------ | 20260505 20:50:18.252 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:50:18.252 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 20:50:18.252 | INFO | Total mem usage: 15.7 GiB | 20260505 20:50:18.252 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 20:50:18.252 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 20:50:18.252 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 20:50:18.252 | INFO | High Water Breakdown: | 20260505 20:50:18.252 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 20:50:18.252 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 20:50:18.252 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 20:50:18.252 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 20:50:18.252 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 20:50:18.252 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 20:50:18.252 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 20:50:18.252 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 20:50:18.252 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 20:50:18.252 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:50:18.252 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 20:50:18.252 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:50:18.252 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 20:50:18.252 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 20:50:18.252 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 20:50:18.252 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 20:50:18.252 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:50:18.253 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 20:50:18.253 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:50:18.253 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:50:18.253 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:50:18.253 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 20:50:18.253 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 20:50:18.253 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:50:18.253 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 20:50:18.253 | INFO | ------------------------------------------ | 20260505 20:50:18.253 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:50:18.253 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 20:50:18.253 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:50:18.253 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:50:18.253 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.43e+01, 1.35e+01 / 1.88e+00 | 20260505 20:50:18.253 | INFO | PerfReport: 0.29%, AggregateStats -- 9.24e-02 / 1.03e-01, 1.08e-02 / 4.22e-03 | 20260505 20:50:18.253 | INFO | PerfReport: 11.80%, Baseline -- 3.71e+00 / 4.55e+00, 5.62e-01 / 1.53e-01 | 20260505 20:50:18.253 | INFO | PerfReport: 3.26%, Histogram -- 1.03e+00 / 1.52e+00, 1.86e-01 / 1.05e-01 | 20260505 20:50:18.253 | INFO | PerfReport: 15.97%, DME -- 4.01e+01 / 4.43e+01, 4.12e+00 / 2.02e+00 | 20260505 20:50:18.253 | INFO | PerfReport: 57.05%, FrameLabeling -- 1.79e+01 / 1.93e+01, 1.42e+00 / 9.74e-01 | 20260505 20:50:18.253 | INFO | PerfReport: 7.26%, PulseAccumulating -- 2.28e+00 / 2.87e+00, 4.36e-01 / 1.75e-01 | 20260505 20:50:18.253 | INFO | PerfReport: 3.23%, Metrics -- 1.02e+00 / 1.32e+00, 2.75e-01 / 1.09e-01 | 20260505 20:50:18.253 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.53e-01 / 4.20e-01, 6.19e-02 / 3.62e-02 | 20260505 20:50:18.253 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 20:50:18.253 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:50:18.253 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:50:18.253 | INFO | PerfReport: Full Execution -- 9.93e+01 / -nan, 2.54e+01 / 0.00e+00 | 20260505 20:50:18.253 | INFO | PerfReport: 22.45%, Upload -- 2.23e+01 / 0.00e+00, 7.69e+00 / 0.00e+00 | 20260505 20:50:18.253 | INFO | PerfReport: 21.80%, Download -- 2.16e+01 / 0.00e+00, 7.69e+00 / 0.00e+00 | 20260505 20:50:18.253 | INFO | PerfReport: 31.65%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 20:50:18.253 | INFO | PerfReport: 2.28%, LockUpload -- 2.26e+00 / 0.00e+00, 8.09e+00 / 0.00e+00 | 20260505 20:50:18.253 | INFO | PerfReport: 1.40%, LockDownload -- 1.39e+00 / 0.00e+00, 2.95e+00 / 0.00e+00 | 20260505 20:50:18.253 | INFO | PerfReport: 20.41%, LockCompute -- 2.03e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 20:50:18.253 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.70e-02 / 0.00e+00 | 20260505 20:50:18.387 | INFO | Starting a round of Baz IO | 20260505 20:50:21.544 | INFO | Populating RTMetrics report | 20260505 20:50:21.544 | INFO | RTMetrics report populated | 20260505 20:50:21.544 | INFO | Opening tmp RTMetrics json file | 20260505 20:50:21.545 | INFO | Serializing RTMetrics to json | 20260505 20:50:21.546 | INFO | Serialization done, swapping tmp file now... | 20260505 20:50:21.547 | INFO | RTMetrics done | 20260505 20:50:21.630 | INFO | Finished a round of Baz IO: Wrote 1.995 GiB in 3.243 seconds | 20260505 20:50:21.630 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.63% / 0.00% / 78.37% | 20260505 20:50:21.630 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.164191% / 0% / 0.000119209% | 20260505 20:50:21.630 | INFO | Baz IO overall: | 20260505 20:50:21.630 | INFO | BytesWritten : 3610110080027 | 20260505 20:50:21.630 | INFO | NumEvents : 1705865222902 | 20260505 20:50:23.390 | INFO | Analyzing chunk frames = [6209536,6210048) | 20260505 20:50:23.390 | INFO | MemInfo: { | 20260505 20:50:23.390 | INFO | "gpu" : " 17 GB", | 20260505 20:50:23.390 | INFO | "huge" : "", | 20260505 20:50:23.390 | INFO | "huge_shared" : " 27 GB", | 20260505 20:50:23.390 | INFO | "malloc" : " 14 GB", | 20260505 20:50:23.390 | INFO | "pinned" : " 29 GB" | 20260505 20:50:23.390 | INFO | } | 20260505 20:50:23.390 | INFO | Perf report for the last 5.13869 seconds: | 20260505 20:50:23.390 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:50:23.390 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:23.390 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:23.390 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:23.390 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 20:50:23.390 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:23.390 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260505 20:50:23.390 | INFO | PRE_HQ : 59.7%, 1.8, 54.8ms, 0 | 20260505 20:50:23.390 | INFO | BAZWRITER : 62.7%, 3.7, 119.7ms, 0 | 20260505 20:50:23.390 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 20:50:23.390 | INFO | Uploaded 36GiB to GPU in 2.15374s (16.7151GiB/s) | 20260505 20:50:23.390 | INFO | Downloaded 38GiB to GPU in 2.24492s (16.9271GiB/s) | 20260505 20:50:28.530 | INFO | Analyzing chunk frames = [6210048,6210560) | 20260505 20:50:28.530 | INFO | MemInfo: { | 20260505 20:50:28.530 | INFO | "gpu" : " 17 GB", | 20260505 20:50:28.530 | INFO | "huge" : "", | 20260505 20:50:28.530 | INFO | "huge_shared" : " 27 GB", | 20260505 20:50:28.530 | INFO | "malloc" : " 14 GB", | 20260505 20:50:28.530 | INFO | "pinned" : " 29 GB" | 20260505 20:50:28.530 | INFO | } | 20260505 20:50:28.530 | INFO | Perf report for the last 5.13995 seconds: | 20260505 20:50:28.530 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:50:28.530 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:28.530 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:28.530 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:28.530 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 20:50:28.530 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:28.530 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:28.530 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 20:50:28.530 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:28.530 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:28.530 | INFO | Uploaded 36GiB to GPU in 2.10174s (17.1287GiB/s) | 20260505 20:50:28.530 | INFO | Downloaded 34GiB to GPU in 2.02932s (16.7544GiB/s) | 20260505 20:50:33.668 | INFO | Analyzing chunk frames = [6210560,6211072) | 20260505 20:50:33.668 | INFO | MemInfo: { | 20260505 20:50:33.668 | INFO | "gpu" : " 17 GB", | 20260505 20:50:33.668 | INFO | "huge" : "", | 20260505 20:50:33.668 | INFO | "huge_shared" : " 27 GB", | 20260505 20:50:33.668 | INFO | "malloc" : " 14 GB", | 20260505 20:50:33.668 | INFO | "pinned" : " 29 GB" | 20260505 20:50:33.668 | INFO | } | 20260505 20:50:33.668 | INFO | Perf report for the last 5.13813 seconds: | 20260505 20:50:33.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:50:33.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:33.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:33.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:33.668 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260505 20:50:33.668 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:33.668 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:33.668 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:50:33.668 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:33.668 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:33.668 | INFO | Uploaded 36GiB to GPU in 2.12906s (16.9089GiB/s) | 20260505 20:50:33.668 | INFO | Downloaded 34GiB to GPU in 2.01841s (16.8449GiB/s) | 20260505 20:50:38.812 | INFO | Analyzing chunk frames = [6211072,6211584) | 20260505 20:50:38.812 | INFO | MemInfo: { | 20260505 20:50:38.812 | INFO | "gpu" : " 17 GB", | 20260505 20:50:38.812 | INFO | "huge" : "", | 20260505 20:50:38.812 | INFO | "huge_shared" : " 27 GB", | 20260505 20:50:38.812 | INFO | "malloc" : " 14 GB", | 20260505 20:50:38.812 | INFO | "pinned" : " 29 GB" | 20260505 20:50:38.812 | INFO | } | 20260505 20:50:38.812 | INFO | Perf report for the last 5.14374 seconds: | 20260505 20:50:38.812 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:50:38.812 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:38.812 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:38.812 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:38.812 | INFO | ANALYSIS : 62.4%, 3.0, 97.0ms, 0 | 20260505 20:50:38.812 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:38.812 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:38.812 | INFO | PRE_HQ : 27.2%, 1.0, 14.2ms, 0 | 20260505 20:50:38.812 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:38.812 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:38.812 | INFO | Uploaded 36GiB to GPU in 2.21191s (16.2755GiB/s) | 20260505 20:50:38.812 | INFO | Downloaded 34GiB to GPU in 2.04726s (16.6076GiB/s) | 20260505 20:50:43.952 | INFO | Analyzing chunk frames = [6211584,6212096) | 20260505 20:50:43.952 | INFO | MemInfo: { | 20260505 20:50:43.952 | INFO | "gpu" : " 17 GB", | 20260505 20:50:43.953 | INFO | "huge" : "", | 20260505 20:50:43.953 | INFO | "huge_shared" : " 27 GB", | 20260505 20:50:43.953 | INFO | "malloc" : " 14 GB", | 20260505 20:50:43.953 | INFO | "pinned" : " 29 GB" | 20260505 20:50:43.953 | INFO | } | 20260505 20:50:43.953 | INFO | Perf report for the last 5.14082 seconds: | 20260505 20:50:43.953 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:50:43.953 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:43.953 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:43.953 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:43.953 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260505 20:50:43.953 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:43.953 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:43.953 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:50:43.953 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:43.953 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:43.953 | INFO | Uploaded 36GiB to GPU in 2.14637s (16.7725GiB/s) | 20260505 20:50:43.953 | INFO | Downloaded 34GiB to GPU in 2.0629s (16.4817GiB/s) | 20260505 20:50:49.081 | INFO | Analyzing chunk frames = [6212096,6212608) | 20260505 20:50:49.081 | INFO | MemInfo: { | 20260505 20:50:49.081 | INFO | "gpu" : " 17 GB", | 20260505 20:50:49.081 | INFO | "huge" : "", | 20260505 20:50:49.081 | INFO | "huge_shared" : " 27 GB", | 20260505 20:50:49.081 | INFO | "malloc" : " 14 GB", | 20260505 20:50:49.081 | INFO | "pinned" : " 29 GB" | 20260505 20:50:49.081 | INFO | } | 20260505 20:50:49.081 | INFO | Perf report for the last 5.128 seconds: | 20260505 20:50:49.081 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:50:49.081 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:49.081 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:49.081 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:49.081 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260505 20:50:49.081 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:49.081 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:49.081 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:50:49.081 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:49.081 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:49.081 | INFO | Uploaded 36GiB to GPU in 2.0877s (17.2438GiB/s) | 20260505 20:50:49.081 | INFO | Downloaded 34GiB to GPU in 2.03059s (16.7439GiB/s) | 20260505 20:50:54.220 | INFO | Analyzing chunk frames = [6212608,6213120) | 20260505 20:50:54.220 | INFO | MemInfo: { | 20260505 20:50:54.220 | INFO | "gpu" : " 17 GB", | 20260505 20:50:54.220 | INFO | "huge" : "", | 20260505 20:50:54.220 | INFO | "huge_shared" : " 27 GB", | 20260505 20:50:54.220 | INFO | "malloc" : " 14 GB", | 20260505 20:50:54.220 | INFO | "pinned" : " 29 GB" | 20260505 20:50:54.220 | INFO | } | 20260505 20:50:54.220 | INFO | Perf report for the last 5.13959 seconds: | 20260505 20:50:54.220 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:50:54.220 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:54.220 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:54.220 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:54.220 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 20:50:54.220 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:54.220 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:54.220 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 20:50:54.220 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:54.220 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:54.221 | INFO | Uploaded 36GiB to GPU in 2.12596s (16.9335GiB/s) | 20260505 20:50:54.221 | INFO | Downloaded 34GiB to GPU in 2.13802s (15.9026GiB/s) | 20260505 20:50:59.367 | INFO | Analyzing chunk frames = [6213120,6213632) | 20260505 20:50:59.367 | INFO | MemInfo: { | 20260505 20:50:59.367 | INFO | "gpu" : " 17 GB", | 20260505 20:50:59.367 | INFO | "huge" : "", | 20260505 20:50:59.367 | INFO | "huge_shared" : " 27 GB", | 20260505 20:50:59.367 | INFO | "malloc" : " 14 GB", | 20260505 20:50:59.367 | INFO | "pinned" : " 29 GB" | 20260505 20:50:59.368 | INFO | } | 20260505 20:50:59.368 | INFO | Perf report for the last 5.14739 seconds: | 20260505 20:50:59.368 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:50:59.368 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:59.368 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:59.368 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:59.368 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 20:50:59.368 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:59.368 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:59.368 | INFO | PRE_HQ : 27.2%, 1.0, 14.2ms, 0 | 20260505 20:50:59.368 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:59.368 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:50:59.368 | INFO | Uploaded 36GiB to GPU in 2.20393s (16.3344GiB/s) | 20260505 20:50:59.368 | INFO | Downloaded 34GiB to GPU in 2.03303s (16.7238GiB/s) | 20260505 20:50:59.502 | INFO | Starting a round of Baz IO | 20260505 20:51:02.661 | INFO | Populating RTMetrics report | 20260505 20:51:02.661 | INFO | RTMetrics report populated | 20260505 20:51:02.661 | INFO | Opening tmp RTMetrics json file | 20260505 20:51:02.662 | INFO | Serializing RTMetrics to json | 20260505 20:51:02.663 | INFO | Serialization done, swapping tmp file now... | 20260505 20:51:02.663 | INFO | RTMetrics done | 20260505 20:51:02.747 | INFO | Finished a round of Baz IO: Wrote 1.996 GiB in 3.244 seconds | 20260505 20:51:02.747 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.63% / 0.00% / 78.37% | 20260505 20:51:02.747 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.163269% / 0% / 0.000103315% | 20260505 20:51:02.747 | INFO | Baz IO overall: | 20260505 20:51:02.747 | INFO | BytesWritten : 3612252981863 | 20260505 20:51:02.747 | INFO | NumEvents : 1706856779241 | 20260505 20:51:04.512 | INFO | Analyzing chunk frames = [6213632,6214144) | 20260505 20:51:04.512 | INFO | MemInfo: { | 20260505 20:51:04.512 | INFO | "gpu" : " 17 GB", | 20260505 20:51:04.512 | INFO | "huge" : "", | 20260505 20:51:04.512 | INFO | "huge_shared" : " 27 GB", | 20260505 20:51:04.512 | INFO | "malloc" : " 14 GB", | 20260505 20:51:04.512 | INFO | "pinned" : " 29 GB" | 20260505 20:51:04.512 | INFO | } | 20260505 20:51:04.512 | INFO | Perf report for the last 5.14422 seconds: | 20260505 20:51:04.512 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:51:04.512 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:04.512 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:04.512 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:04.512 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 20:51:04.512 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:04.512 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260505 20:51:04.512 | INFO | PRE_HQ : 59.9%, 1.8, 55.0ms, 0 | 20260505 20:51:04.512 | INFO | BAZWRITER : 62.6%, 3.7, 119.3ms, 0 | 20260505 20:51:04.512 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 20:51:04.512 | INFO | Uploaded 36GiB to GPU in 2.16682s (16.6142GiB/s) | 20260505 20:51:04.512 | INFO | Downloaded 38GiB to GPU in 2.24256s (16.9449GiB/s) | 20260505 20:51:09.645 | INFO | Analyzing chunk frames = [6214144,6214656) | 20260505 20:51:09.645 | INFO | MemInfo: { | 20260505 20:51:09.645 | INFO | "gpu" : " 17 GB", | 20260505 20:51:09.645 | INFO | "huge" : "", | 20260505 20:51:09.645 | INFO | "huge_shared" : " 27 GB", | 20260505 20:51:09.645 | INFO | "malloc" : " 14 GB", | 20260505 20:51:09.645 | INFO | "pinned" : " 29 GB" | 20260505 20:51:09.645 | INFO | } | 20260505 20:51:09.645 | INFO | Perf report for the last 5.13314 seconds: | 20260505 20:51:09.645 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:51:09.645 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:09.645 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:09.645 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:09.645 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 20:51:09.645 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:09.645 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:09.645 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 20:51:09.645 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:09.645 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:09.645 | INFO | Uploaded 36GiB to GPU in 2.09216s (17.2071GiB/s) | 20260505 20:51:09.645 | INFO | Downloaded 34GiB to GPU in 2.02646s (16.7781GiB/s) | 20260505 20:51:14.781 | INFO | Analyzing chunk frames = [6214656,6215168) | 20260505 20:51:14.781 | INFO | MemInfo: { | 20260505 20:51:14.781 | INFO | "gpu" : " 17 GB", | 20260505 20:51:14.781 | INFO | "huge" : "", | 20260505 20:51:14.782 | INFO | "huge_shared" : " 27 GB", | 20260505 20:51:14.782 | INFO | "malloc" : " 14 GB", | 20260505 20:51:14.782 | INFO | "pinned" : " 29 GB" | 20260505 20:51:14.782 | INFO | } | 20260505 20:51:14.782 | INFO | Perf report for the last 5.1366 seconds: | 20260505 20:51:14.782 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:51:14.782 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:14.782 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:14.782 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:14.782 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260505 20:51:14.782 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:14.782 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:14.782 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 20:51:14.782 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:14.782 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:14.782 | INFO | Uploaded 36GiB to GPU in 2.13231s (16.8831GiB/s) | 20260505 20:51:14.782 | INFO | Downloaded 34GiB to GPU in 2.01441s (16.8784GiB/s) | 20260505 20:51:19.924 | INFO | Analyzing chunk frames = [6215168,6215680) | 20260505 20:51:19.924 | INFO | MemInfo: { | 20260505 20:51:19.924 | INFO | "gpu" : " 17 GB", | 20260505 20:51:19.924 | INFO | "huge" : "", | 20260505 20:51:19.924 | INFO | "huge_shared" : " 27 GB", | 20260505 20:51:19.924 | INFO | "malloc" : " 14 GB", | 20260505 20:51:19.924 | INFO | "pinned" : " 29 GB" | 20260505 20:51:19.924 | INFO | } | 20260505 20:51:19.924 | INFO | Perf report for the last 5.14215 seconds: | 20260505 20:51:19.924 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:51:19.924 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:19.924 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:19.924 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:19.924 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260505 20:51:19.924 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:19.924 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:19.924 | INFO | PRE_HQ : 27.2%, 1.0, 14.2ms, 0 | 20260505 20:51:19.924 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:19.924 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:19.924 | INFO | Uploaded 36GiB to GPU in 2.19802s (16.3784GiB/s) | 20260505 20:51:19.924 | INFO | Downloaded 34GiB to GPU in 2.06609s (16.4562GiB/s) | 20260505 20:51:25.062 | INFO | Analyzing chunk frames = [6215680,6216192) | 20260505 20:51:25.062 | INFO | MemInfo: { | 20260505 20:51:25.062 | INFO | "gpu" : " 17 GB", | 20260505 20:51:25.062 | INFO | "huge" : "", | 20260505 20:51:25.062 | INFO | "huge_shared" : " 27 GB", | 20260505 20:51:25.062 | INFO | "malloc" : " 14 GB", | 20260505 20:51:25.062 | INFO | "pinned" : " 29 GB" | 20260505 20:51:25.062 | INFO | } | 20260505 20:51:25.062 | INFO | Perf report for the last 5.13815 seconds: | 20260505 20:51:25.062 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:51:25.062 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:25.062 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:25.062 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:25.062 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260505 20:51:25.062 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:25.062 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:25.062 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:51:25.062 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:25.062 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:25.062 | INFO | Uploaded 36GiB to GPU in 2.13629s (16.8516GiB/s) | 20260505 20:51:25.062 | INFO | Downloaded 34GiB to GPU in 2.06011s (16.504GiB/s) | 20260505 20:51:30.211 | INFO | Analyzing chunk frames = [6216192,6216704) | 20260505 20:51:30.211 | INFO | MemInfo: { | 20260505 20:51:30.211 | INFO | "gpu" : " 17 GB", | 20260505 20:51:30.211 | INFO | "huge" : "", | 20260505 20:51:30.211 | INFO | "huge_shared" : " 27 GB", | 20260505 20:51:30.211 | INFO | "malloc" : " 14 GB", | 20260505 20:51:30.211 | INFO | "pinned" : " 29 GB" | 20260505 20:51:30.211 | INFO | } | 20260505 20:51:30.211 | INFO | Perf report for the last 5.14887 seconds: | 20260505 20:51:30.211 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:51:30.211 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:30.211 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:30.211 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:30.211 | INFO | ANALYSIS : 61.7%, 3.0, 96.5ms, 0 | 20260505 20:51:30.211 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:30.211 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:30.211 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 20:51:30.211 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:30.211 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:30.211 | INFO | Uploaded 36GiB to GPU in 2.09899s (17.1511GiB/s) | 20260505 20:51:30.211 | INFO | Downloaded 34GiB to GPU in 2.03853s (16.6787GiB/s) | 20260505 20:51:35.342 | INFO | Analyzing chunk frames = [6216704,6217216) | 20260505 20:51:35.342 | INFO | MemInfo: { | 20260505 20:51:35.342 | INFO | "gpu" : " 17 GB", | 20260505 20:51:35.342 | INFO | "huge" : "", | 20260505 20:51:35.342 | INFO | "huge_shared" : " 27 GB", | 20260505 20:51:35.342 | INFO | "malloc" : " 14 GB", | 20260505 20:51:35.342 | INFO | "pinned" : " 29 GB" | 20260505 20:51:35.342 | INFO | } | 20260505 20:51:35.342 | INFO | Perf report for the last 5.13132 seconds: | 20260505 20:51:35.342 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:51:35.342 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:35.342 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:35.342 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:35.342 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 20:51:35.342 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:35.342 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:35.342 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 20:51:35.342 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:35.342 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:35.343 | INFO | Uploaded 36GiB to GPU in 2.05448s (17.5227GiB/s) | 20260505 20:51:35.343 | INFO | Downloaded 34GiB to GPU in 2.04172s (16.6526GiB/s) | 20260505 20:51:40.479 | INFO | Analyzing chunk frames = [6217216,6217728) | 20260505 20:51:40.479 | INFO | MemInfo: { | 20260505 20:51:40.479 | INFO | "gpu" : " 17 GB", | 20260505 20:51:40.479 | INFO | "huge" : "", | 20260505 20:51:40.479 | INFO | "huge_shared" : " 27 GB", | 20260505 20:51:40.479 | INFO | "malloc" : " 14 GB", | 20260505 20:51:40.479 | INFO | "pinned" : " 29 GB" | 20260505 20:51:40.479 | INFO | } | 20260505 20:51:40.479 | INFO | Perf report for the last 5.13697 seconds: | 20260505 20:51:40.479 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:51:40.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:40.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:40.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:40.479 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260505 20:51:40.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:40.479 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:40.479 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:51:40.479 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:40.479 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:40.479 | INFO | Uploaded 36GiB to GPU in 2.12448s (16.9454GiB/s) | 20260505 20:51:40.479 | INFO | Downloaded 34GiB to GPU in 2.01614s (16.8639GiB/s) | 20260505 20:51:40.480 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 20:51:40.480 | INFO | Total mem usage: 15.3 GiB | 20260505 20:51:40.480 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 20:51:40.480 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.26%) | 20260505 20:51:40.480 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 20:51:40.480 | INFO | High Water Breakdown: | 20260505 20:51:40.480 | INFO | 61% : 8.63 GiB ( 90.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 20:51:40.480 | INFO | 38.9% : 5.5 GiB ( 82.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 20:51:40.480 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 20:51:40.480 | INFO | ------------------------------------------ | 20260505 20:51:40.480 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:51:40.480 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 20:51:40.480 | INFO | Total mem usage: 26.5 GiB | 20260505 20:51:40.480 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 20:51:40.480 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 20:51:40.480 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 20:51:40.480 | INFO | High Water Breakdown: | 20260505 20:51:40.480 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:51:40.480 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:51:40.480 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:51:40.480 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:51:40.480 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:51:40.480 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:51:40.480 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 20:51:40.480 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 20:51:40.480 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:51:40.480 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 20:51:40.480 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 20:51:40.480 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 20:51:40.480 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 20:51:40.480 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 20:51:40.480 | INFO | ------------------------------------------ | 20260505 20:51:40.480 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:51:40.480 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 20:51:40.480 | INFO | Total mem usage: 25.6 GiB | 20260505 20:51:40.480 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 20:51:40.480 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 20:51:40.480 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 20:51:40.480 | INFO | High Water Breakdown: | 20260505 20:51:40.480 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 20:51:40.480 | INFO | ------------------------------------------ | 20260505 20:51:40.480 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:51:40.480 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 20:51:40.480 | INFO | Total mem usage: 15.7 GiB | 20260505 20:51:40.480 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 20:51:40.480 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 20:51:40.480 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 20:51:40.480 | INFO | High Water Breakdown: | 20260505 20:51:40.480 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 20:51:40.480 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 20:51:40.480 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 20:51:40.480 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 20:51:40.480 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 20:51:40.480 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 20:51:40.480 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 20:51:40.480 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 20:51:40.480 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 20:51:40.480 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:51:40.480 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 20:51:40.480 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:51:40.480 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 20:51:40.480 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 20:51:40.480 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 20:51:40.480 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 20:51:40.480 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:51:40.480 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 20:51:40.480 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:51:40.480 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:51:40.480 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:51:40.480 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 20:51:40.480 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 20:51:40.480 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:51:40.480 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 20:51:40.480 | INFO | ------------------------------------------ | 20260505 20:51:40.480 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:51:40.480 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 20:51:40.480 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:51:40.480 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:51:40.480 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.46e+01, 1.34e+01 / 1.74e+00 | 20260505 20:51:40.480 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.04e-01, 1.03e-02 / 3.61e-03 | 20260505 20:51:40.480 | INFO | PerfReport: 11.80%, Baseline -- 3.72e+00 / 4.53e+00, 5.66e-01 / 1.94e-01 | 20260505 20:51:40.480 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.51e+00, 1.92e-01 / 1.11e-01 | 20260505 20:51:40.480 | INFO | PerfReport: 15.98%, DME -- 4.03e+01 / 4.44e+01, 4.05e+00 / 1.77e+00 | 20260505 20:51:40.480 | INFO | PerfReport: 57.11%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.52e+00 / 1.07e+00 | 20260505 20:51:40.480 | INFO | PerfReport: 7.19%, PulseAccumulating -- 2.27e+00 / 2.83e+00, 4.31e-01 / 2.19e-01 | 20260505 20:51:40.480 | INFO | PerfReport: 3.22%, Metrics -- 1.01e+00 / 1.34e+00, 2.76e-01 / 1.54e-01 | 20260505 20:51:40.480 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.53e-01 / 4.17e-01, 6.24e-02 / 3.75e-02 | 20260505 20:51:40.480 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 20:51:40.480 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:51:40.480 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:51:40.480 | INFO | PerfReport: Full Execution -- 9.92e+01 / -nan, 2.53e+01 / 0.00e+00 | 20260505 20:51:40.480 | INFO | PerfReport: 22.42%, Upload -- 2.22e+01 / 0.00e+00, 7.60e+00 / 0.00e+00 | 20260505 20:51:40.481 | INFO | PerfReport: 21.73%, Download -- 2.16e+01 / 0.00e+00, 7.63e+00 / 0.00e+00 | 20260505 20:51:40.481 | INFO | PerfReport: 31.76%, Compute -- 3.15e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 20:51:40.481 | INFO | PerfReport: 2.27%, LockUpload -- 2.25e+00 / 0.00e+00, 7.92e+00 / 0.00e+00 | 20260505 20:51:40.481 | INFO | PerfReport: 1.28%, LockDownload -- 1.27e+00 / 0.00e+00, 2.80e+00 / 0.00e+00 | 20260505 20:51:40.481 | INFO | PerfReport: 20.53%, LockCompute -- 2.04e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 20:51:40.481 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.96e-02 / 0.00e+00 | 20260505 20:51:40.612 | INFO | Starting a round of Baz IO | 20260505 20:51:43.773 | INFO | Populating RTMetrics report | 20260505 20:51:43.774 | INFO | RTMetrics report populated | 20260505 20:51:43.774 | INFO | Opening tmp RTMetrics json file | 20260505 20:51:43.816 | INFO | Serializing RTMetrics to json | 20260505 20:51:43.857 | INFO | Serialization done, swapping tmp file now... | 20260505 20:51:43.857 | INFO | RTMetrics done | 20260505 20:51:43.870 | INFO | Finished a round of Baz IO: Wrote 1.995 GiB in 3.257 seconds | 20260505 20:51:43.870 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.63% / 0.00% / 78.37% | 20260505 20:51:43.870 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.163372% / 3.97364e-06% / 8.34465e-05% | 20260505 20:51:43.870 | INFO | Baz IO overall: | 20260505 20:51:43.870 | INFO | BytesWritten : 3614394809664 | 20260505 20:51:43.870 | INFO | NumEvents : 1707847614802 | 20260505 20:51:45.623 | INFO | Analyzing chunk frames = [6217728,6218240) | 20260505 20:51:45.623 | INFO | MemInfo: { | 20260505 20:51:45.623 | INFO | "gpu" : " 17 GB", | 20260505 20:51:45.623 | INFO | "huge" : "", | 20260505 20:51:45.623 | INFO | "huge_shared" : " 27 GB", | 20260505 20:51:45.623 | INFO | "malloc" : " 14 GB", | 20260505 20:51:45.623 | INFO | "pinned" : " 29 GB" | 20260505 20:51:45.623 | INFO | } | 20260505 20:51:45.624 | INFO | Perf report for the last 5.14443 seconds: | 20260505 20:51:45.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:51:45.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:45.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:45.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:45.624 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 20:51:45.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:45.624 | INFO | RT_METRICS : 45.5%, 1.0, 23.9ms, 0 | 20260505 20:51:45.624 | INFO | PRE_HQ : 59.7%, 1.8, 54.9ms, 0 | 20260505 20:51:45.624 | INFO | BAZWRITER : 62.8%, 3.7, 120.1ms, 0 | 20260505 20:51:45.624 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260505 20:51:45.624 | INFO | Uploaded 36GiB to GPU in 2.15599s (16.6976GiB/s) | 20260505 20:51:45.624 | INFO | Downloaded 38GiB to GPU in 2.256s (16.844GiB/s) | 20260505 20:51:50.761 | INFO | Analyzing chunk frames = [6218240,6218752) | 20260505 20:51:50.761 | INFO | MemInfo: { | 20260505 20:51:50.761 | INFO | "gpu" : " 17 GB", | 20260505 20:51:50.761 | INFO | "huge" : "", | 20260505 20:51:50.761 | INFO | "huge_shared" : " 27 GB", | 20260505 20:51:50.761 | INFO | "malloc" : " 14 GB", | 20260505 20:51:50.761 | INFO | "pinned" : " 29 GB" | 20260505 20:51:50.761 | INFO | } | 20260505 20:51:50.761 | INFO | Perf report for the last 5.13758 seconds: | 20260505 20:51:50.761 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:51:50.761 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:50.761 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:50.761 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:50.761 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 20:51:50.761 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:50.761 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:50.761 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 20:51:50.761 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:50.761 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:50.761 | INFO | Uploaded 36GiB to GPU in 2.09127s (17.2144GiB/s) | 20260505 20:51:50.762 | INFO | Downloaded 34GiB to GPU in 2.02489s (16.791GiB/s) | 20260505 20:51:55.900 | INFO | Analyzing chunk frames = [6218752,6219264) | 20260505 20:51:55.900 | INFO | MemInfo: { | 20260505 20:51:55.900 | INFO | "gpu" : " 17 GB", | 20260505 20:51:55.900 | INFO | "huge" : "", | 20260505 20:51:55.900 | INFO | "huge_shared" : " 27 GB", | 20260505 20:51:55.900 | INFO | "malloc" : " 14 GB", | 20260505 20:51:55.900 | INFO | "pinned" : " 29 GB" | 20260505 20:51:55.900 | INFO | } | 20260505 20:51:55.900 | INFO | Perf report for the last 5.13899 seconds: | 20260505 20:51:55.900 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:51:55.900 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:55.900 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:55.900 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:55.900 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260505 20:51:55.900 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:55.900 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:55.900 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 20:51:55.900 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:55.900 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:51:55.901 | INFO | Uploaded 36GiB to GPU in 2.13559s (16.8572GiB/s) | 20260505 20:51:55.901 | INFO | Downloaded 34GiB to GPU in 2.02431s (16.7959GiB/s) | 20260505 20:52:01.036 | INFO | Analyzing chunk frames = [6219264,6219776) | 20260505 20:52:01.036 | INFO | MemInfo: { | 20260505 20:52:01.036 | INFO | "gpu" : " 17 GB", | 20260505 20:52:01.036 | INFO | "huge" : "", | 20260505 20:52:01.036 | INFO | "huge_shared" : " 27 GB", | 20260505 20:52:01.036 | INFO | "malloc" : " 14 GB", | 20260505 20:52:01.036 | INFO | "pinned" : " 29 GB" | 20260505 20:52:01.036 | INFO | } | 20260505 20:52:01.036 | INFO | Perf report for the last 5.13629 seconds: | 20260505 20:52:01.036 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:52:01.036 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:01.037 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:01.037 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:01.037 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260505 20:52:01.037 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:01.037 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:01.037 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:52:01.037 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:01.037 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:01.037 | INFO | Uploaded 36GiB to GPU in 2.19253s (16.4194GiB/s) | 20260505 20:52:01.037 | INFO | Downloaded 34GiB to GPU in 2.03963s (16.6697GiB/s) | 20260505 20:52:06.178 | INFO | Analyzing chunk frames = [6219776,6220288) | 20260505 20:52:06.178 | INFO | MemInfo: { | 20260505 20:52:06.178 | INFO | "gpu" : " 17 GB", | 20260505 20:52:06.178 | INFO | "huge" : "", | 20260505 20:52:06.178 | INFO | "huge_shared" : " 27 GB", | 20260505 20:52:06.178 | INFO | "malloc" : " 14 GB", | 20260505 20:52:06.178 | INFO | "pinned" : " 29 GB" | 20260505 20:52:06.178 | INFO | } | 20260505 20:52:06.178 | INFO | Perf report for the last 5.14146 seconds: | 20260505 20:52:06.178 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:52:06.178 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:06.178 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:06.178 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:06.178 | INFO | ANALYSIS : 61.8%, 3.0, 96.6ms, 0 | 20260505 20:52:06.178 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:06.178 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:06.178 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 20:52:06.178 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:06.178 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:06.178 | INFO | Uploaded 36GiB to GPU in 2.14012s (16.8215GiB/s) | 20260505 20:52:06.178 | INFO | Downloaded 34GiB to GPU in 2.06436s (16.47GiB/s) | 20260505 20:52:11.322 | INFO | Analyzing chunk frames = [6220288,6220800) | 20260505 20:52:11.322 | INFO | MemInfo: { | 20260505 20:52:11.322 | INFO | "gpu" : " 17 GB", | 20260505 20:52:11.322 | INFO | "huge" : "", | 20260505 20:52:11.322 | INFO | "huge_shared" : " 27 GB", | 20260505 20:52:11.322 | INFO | "malloc" : " 14 GB", | 20260505 20:52:11.322 | INFO | "pinned" : " 29 GB" | 20260505 20:52:11.322 | INFO | } | 20260505 20:52:11.322 | INFO | Perf report for the last 5.14455 seconds: | 20260505 20:52:11.323 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:52:11.323 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:11.323 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:11.323 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:11.323 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260505 20:52:11.323 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:11.323 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:11.323 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 20:52:11.323 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:11.323 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:11.323 | INFO | Uploaded 36GiB to GPU in 2.1584s (16.679GiB/s) | 20260505 20:52:11.323 | INFO | Downloaded 34GiB to GPU in 2.06381s (16.4744GiB/s) | 20260505 20:52:16.457 | INFO | Analyzing chunk frames = [6220800,6221312) | 20260505 20:52:16.457 | INFO | MemInfo: { | 20260505 20:52:16.457 | INFO | "gpu" : " 17 GB", | 20260505 20:52:16.457 | INFO | "huge" : "", | 20260505 20:52:16.457 | INFO | "huge_shared" : " 27 GB", | 20260505 20:52:16.458 | INFO | "malloc" : " 14 GB", | 20260505 20:52:16.458 | INFO | "pinned" : " 29 GB" | 20260505 20:52:16.458 | INFO | } | 20260505 20:52:16.458 | INFO | Perf report for the last 5.13512 seconds: | 20260505 20:52:16.458 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:52:16.458 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:16.458 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:16.458 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:16.458 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 20:52:16.458 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:16.458 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:16.458 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 20:52:16.458 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:16.458 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:16.458 | INFO | Uploaded 36GiB to GPU in 2.1427s (16.8012GiB/s) | 20260505 20:52:16.458 | INFO | Downloaded 34GiB to GPU in 2.12838s (15.9746GiB/s) | 20260505 20:52:21.593 | INFO | Analyzing chunk frames = [6221312,6221824) | 20260505 20:52:21.593 | INFO | MemInfo: { | 20260505 20:52:21.593 | INFO | "gpu" : " 17 GB", | 20260505 20:52:21.593 | INFO | "huge" : "", | 20260505 20:52:21.593 | INFO | "huge_shared" : " 27 GB", | 20260505 20:52:21.593 | INFO | "malloc" : " 14 GB", | 20260505 20:52:21.593 | INFO | "pinned" : " 29 GB" | 20260505 20:52:21.593 | INFO | } | 20260505 20:52:21.593 | INFO | Perf report for the last 5.13548 seconds: | 20260505 20:52:21.593 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:52:21.593 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:21.593 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:21.593 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:21.593 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260505 20:52:21.593 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:21.593 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:21.593 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 20:52:21.593 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:21.593 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:21.593 | INFO | Uploaded 36GiB to GPU in 2.11372s (17.0316GiB/s) | 20260505 20:52:21.593 | INFO | Downloaded 34GiB to GPU in 2.00432s (16.9634GiB/s) | 20260505 20:52:21.732 | INFO | Starting a round of Baz IO | 20260505 20:52:24.892 | INFO | Populating RTMetrics report | 20260505 20:52:24.893 | INFO | RTMetrics report populated | 20260505 20:52:24.893 | INFO | Opening tmp RTMetrics json file | 20260505 20:52:24.893 | INFO | Serializing RTMetrics to json | 20260505 20:52:24.894 | INFO | Serialization done, swapping tmp file now... | 20260505 20:52:24.895 | INFO | RTMetrics done | 20260505 20:52:24.981 | INFO | Finished a round of Baz IO: Wrote 1.994 GiB in 3.249 seconds | 20260505 20:52:24.981 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.63% / 0.00% / 78.37% | 20260505 20:52:24.981 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.163579% / 0% / 0.000107288% | 20260505 20:52:24.981 | INFO | Baz IO overall: | 20260505 20:52:24.981 | INFO | BytesWritten : 3616535466561 | 20260505 20:52:24.981 | INFO | NumEvents : 1708837804654 | 20260505 20:52:26.727 | INFO | Analyzing chunk frames = [6221824,6222336) | 20260505 20:52:26.727 | INFO | MemInfo: { | 20260505 20:52:26.727 | INFO | "gpu" : " 17 GB", | 20260505 20:52:26.727 | INFO | "huge" : "", | 20260505 20:52:26.728 | INFO | "huge_shared" : " 27 GB", | 20260505 20:52:26.728 | INFO | "malloc" : " 14 GB", | 20260505 20:52:26.728 | INFO | "pinned" : " 29 GB" | 20260505 20:52:26.728 | INFO | } | 20260505 20:52:26.728 | INFO | Perf report for the last 5.13455 seconds: | 20260505 20:52:26.728 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:52:26.728 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:26.728 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:26.728 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:26.728 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 20:52:26.728 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:26.728 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260505 20:52:26.728 | INFO | PRE_HQ : 60.0%, 1.8, 54.9ms, 0 | 20260505 20:52:26.728 | INFO | BAZWRITER : 62.8%, 3.7, 120.3ms, 0 | 20260505 20:52:26.728 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 20:52:26.728 | INFO | Uploaded 36GiB to GPU in 2.16456s (16.6316GiB/s) | 20260505 20:52:26.728 | INFO | Downloaded 38GiB to GPU in 2.24802s (16.9037GiB/s) | 20260505 20:52:31.877 | INFO | Analyzing chunk frames = [6222336,6222848) | 20260505 20:52:31.877 | INFO | MemInfo: { | 20260505 20:52:31.877 | INFO | "gpu" : " 17 GB", | 20260505 20:52:31.877 | INFO | "huge" : "", | 20260505 20:52:31.877 | INFO | "huge_shared" : " 27 GB", | 20260505 20:52:31.877 | INFO | "malloc" : " 14 GB", | 20260505 20:52:31.877 | INFO | "pinned" : " 29 GB" | 20260505 20:52:31.877 | INFO | } | 20260505 20:52:31.877 | INFO | Perf report for the last 5.14921 seconds: | 20260505 20:52:31.877 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:52:31.877 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:31.877 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:31.877 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:31.877 | INFO | ANALYSIS : 62.5%, 3.0, 97.8ms, 0 | 20260505 20:52:31.877 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:31.877 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:31.877 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 20:52:31.877 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:31.877 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:31.877 | INFO | Uploaded 36GiB to GPU in 2.14064s (16.8174GiB/s) | 20260505 20:52:31.877 | INFO | Downloaded 34GiB to GPU in 2.03515s (16.7064GiB/s) | 20260505 20:52:37.020 | INFO | Analyzing chunk frames = [6222848,6223360) | 20260505 20:52:37.020 | INFO | MemInfo: { | 20260505 20:52:37.020 | INFO | "gpu" : " 17 GB", | 20260505 20:52:37.020 | INFO | "huge" : "", | 20260505 20:52:37.020 | INFO | "huge_shared" : " 27 GB", | 20260505 20:52:37.020 | INFO | "malloc" : " 14 GB", | 20260505 20:52:37.020 | INFO | "pinned" : " 29 GB" | 20260505 20:52:37.020 | INFO | } | 20260505 20:52:37.021 | INFO | Perf report for the last 5.14359 seconds: | 20260505 20:52:37.021 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:52:37.021 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:37.021 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:37.021 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:37.021 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260505 20:52:37.021 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:37.021 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:37.021 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:52:37.021 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:37.021 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:37.021 | INFO | Uploaded 36GiB to GPU in 2.12927s (16.9072GiB/s) | 20260505 20:52:37.021 | INFO | Downloaded 34GiB to GPU in 2.02395s (16.7988GiB/s) | 20260505 20:52:42.160 | INFO | Analyzing chunk frames = [6223360,6223872) | 20260505 20:52:42.160 | INFO | MemInfo: { | 20260505 20:52:42.160 | INFO | "gpu" : " 17 GB", | 20260505 20:52:42.160 | INFO | "huge" : "", | 20260505 20:52:42.160 | INFO | "huge_shared" : " 27 GB", | 20260505 20:52:42.160 | INFO | "malloc" : " 14 GB", | 20260505 20:52:42.160 | INFO | "pinned" : " 29 GB" | 20260505 20:52:42.160 | INFO | } | 20260505 20:52:42.161 | INFO | Perf report for the last 5.13997 seconds: | 20260505 20:52:42.161 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:52:42.161 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:42.161 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:42.161 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:42.161 | INFO | ANALYSIS : 62.1%, 3.0, 96.5ms, 0 | 20260505 20:52:42.161 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:42.161 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:42.161 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:52:42.161 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:42.161 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:42.161 | INFO | Uploaded 36GiB to GPU in 2.13977s (16.8242GiB/s) | 20260505 20:52:42.161 | INFO | Downloaded 34GiB to GPU in 2.04705s (16.6093GiB/s) | 20260505 20:52:47.292 | INFO | Analyzing chunk frames = [6223872,6224384) | 20260505 20:52:47.292 | INFO | MemInfo: { | 20260505 20:52:47.292 | INFO | "gpu" : " 17 GB", | 20260505 20:52:47.292 | INFO | "huge" : "", | 20260505 20:52:47.292 | INFO | "huge_shared" : " 27 GB", | 20260505 20:52:47.292 | INFO | "malloc" : " 14 GB", | 20260505 20:52:47.292 | INFO | "pinned" : " 29 GB" | 20260505 20:52:47.292 | INFO | } | 20260505 20:52:47.292 | INFO | Perf report for the last 5.13163 seconds: | 20260505 20:52:47.292 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:52:47.292 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:47.292 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:47.292 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:47.292 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260505 20:52:47.292 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:47.292 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:47.292 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:52:47.292 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:47.292 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:47.292 | INFO | Uploaded 36GiB to GPU in 2.13402s (16.8696GiB/s) | 20260505 20:52:47.293 | INFO | Downloaded 34GiB to GPU in 2.06288s (16.4818GiB/s) | 20260505 20:52:52.430 | INFO | Analyzing chunk frames = [6224384,6224896) | 20260505 20:52:52.430 | INFO | MemInfo: { | 20260505 20:52:52.430 | INFO | "gpu" : " 17 GB", | 20260505 20:52:52.430 | INFO | "huge" : "", | 20260505 20:52:52.430 | INFO | "huge_shared" : " 27 GB", | 20260505 20:52:52.430 | INFO | "malloc" : " 14 GB", | 20260505 20:52:52.430 | INFO | "pinned" : " 29 GB" | 20260505 20:52:52.430 | INFO | } | 20260505 20:52:52.431 | INFO | Perf report for the last 5.13841 seconds: | 20260505 20:52:52.431 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:52:52.431 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:52.431 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:52.431 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:52.431 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 20:52:52.431 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:52.431 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:52.431 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:52:52.431 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:52.431 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:52.431 | INFO | Uploaded 36GiB to GPU in 2.15878s (16.6761GiB/s) | 20260505 20:52:52.431 | INFO | Downloaded 34GiB to GPU in 2.05991s (16.5056GiB/s) | 20260505 20:52:57.570 | INFO | Analyzing chunk frames = [6224896,6225408) | 20260505 20:52:57.570 | INFO | MemInfo: { | 20260505 20:52:57.570 | INFO | "gpu" : " 17 GB", | 20260505 20:52:57.570 | INFO | "huge" : "", | 20260505 20:52:57.571 | INFO | "huge_shared" : " 27 GB", | 20260505 20:52:57.571 | INFO | "malloc" : " 14 GB", | 20260505 20:52:57.571 | INFO | "pinned" : " 29 GB" | 20260505 20:52:57.571 | INFO | } | 20260505 20:52:57.571 | INFO | Perf report for the last 5.14008 seconds: | 20260505 20:52:57.571 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:52:57.571 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:57.571 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:57.571 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:57.571 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 20:52:57.571 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:57.571 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:57.571 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 20:52:57.571 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:57.571 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:52:57.571 | INFO | Uploaded 36GiB to GPU in 2.1281s (16.9165GiB/s) | 20260505 20:52:57.571 | INFO | Downloaded 34GiB to GPU in 2.152s (15.7993GiB/s) | 20260505 20:53:02.708 | INFO | Analyzing chunk frames = [6225408,6225920) | 20260505 20:53:02.708 | INFO | MemInfo: { | 20260505 20:53:02.708 | INFO | "gpu" : " 17 GB", | 20260505 20:53:02.708 | INFO | "huge" : "", | 20260505 20:53:02.708 | INFO | "huge_shared" : " 27 GB", | 20260505 20:53:02.708 | INFO | "malloc" : " 14 GB", | 20260505 20:53:02.708 | INFO | "pinned" : " 29 GB" | 20260505 20:53:02.709 | INFO | } | 20260505 20:53:02.709 | INFO | Perf report for the last 5.13792 seconds: | 20260505 20:53:02.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:53:02.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:02.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:02.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:02.709 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 20:53:02.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:02.709 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:02.709 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:53:02.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:02.709 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:02.709 | INFO | Uploaded 36GiB to GPU in 2.10893s (17.0703GiB/s) | 20260505 20:53:02.709 | INFO | Downloaded 34GiB to GPU in 1.99925s (17.0064GiB/s) | 20260505 20:53:02.709 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 20:53:02.709 | INFO | Total mem usage: 15.3 GiB | 20260505 20:53:02.709 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 20:53:02.709 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.15%) | 20260505 20:53:02.709 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 20:53:02.709 | INFO | High Water Breakdown: | 20260505 20:53:02.709 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 20:53:02.709 | INFO | 38.8% : 5.5 GiB ( 82.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 20:53:02.709 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 20:53:02.709 | INFO | ------------------------------------------ | 20260505 20:53:02.709 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:53:02.709 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 20:53:02.709 | INFO | Total mem usage: 26.5 GiB | 20260505 20:53:02.709 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 20:53:02.709 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 20:53:02.710 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 20:53:02.710 | INFO | High Water Breakdown: | 20260505 20:53:02.710 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:53:02.710 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:53:02.710 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:53:02.710 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:53:02.710 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:53:02.710 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:53:02.710 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 20:53:02.710 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 20:53:02.710 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:53:02.710 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 20:53:02.710 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 20:53:02.710 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 20:53:02.710 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 20:53:02.710 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 20:53:02.710 | INFO | ------------------------------------------ | 20260505 20:53:02.710 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:53:02.710 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 20:53:02.710 | INFO | Total mem usage: 25.6 GiB | 20260505 20:53:02.710 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 20:53:02.710 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 20:53:02.710 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 20:53:02.710 | INFO | High Water Breakdown: | 20260505 20:53:02.710 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 20:53:02.710 | INFO | ------------------------------------------ | 20260505 20:53:02.710 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:53:02.710 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 20:53:02.710 | INFO | Total mem usage: 15.7 GiB | 20260505 20:53:02.710 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 20:53:02.710 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 20:53:02.710 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 20:53:02.710 | INFO | High Water Breakdown: | 20260505 20:53:02.710 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 20:53:02.710 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 20:53:02.710 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 20:53:02.710 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 20:53:02.710 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 20:53:02.710 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 20:53:02.710 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 20:53:02.710 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 20:53:02.710 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 20:53:02.710 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:53:02.710 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 20:53:02.710 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:53:02.710 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 20:53:02.710 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 20:53:02.710 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 20:53:02.710 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 20:53:02.710 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:53:02.710 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 20:53:02.710 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:53:02.710 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:53:02.710 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:53:02.710 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 20:53:02.710 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 20:53:02.710 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:53:02.710 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 20:53:02.710 | INFO | ------------------------------------------ | 20260505 20:53:02.710 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:53:02.710 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 20:53:02.710 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:53:02.710 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:53:02.710 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.42e+01, 1.35e+01 / 1.56e+00 | 20260505 20:53:02.710 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.03e-01, 1.14e-02 / 4.13e-03 | 20260505 20:53:02.710 | INFO | PerfReport: 11.82%, Baseline -- 3.71e+00 / 4.56e+00, 5.69e-01 / 1.77e-01 | 20260505 20:53:02.710 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.54e+00, 1.90e-01 / 7.71e-02 | 20260505 20:53:02.710 | INFO | PerfReport: 15.94%, DME -- 4.01e+01 / 4.41e+01, 4.00e+00 / 1.64e+00 | 20260505 20:53:02.710 | INFO | PerfReport: 57.05%, FrameLabeling -- 1.79e+01 / 1.94e+01, 1.44e+00 / 9.50e-01 | 20260505 20:53:02.710 | INFO | PerfReport: 7.25%, PulseAccumulating -- 2.28e+00 / 2.82e+00, 4.35e-01 / 1.92e-01 | 20260505 20:53:02.710 | INFO | PerfReport: 3.24%, Metrics -- 1.02e+00 / 1.35e+00, 2.79e-01 / 1.22e-01 | 20260505 20:53:02.710 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.53e-01 / 4.19e-01, 6.26e-02 / 3.36e-02 | 20260505 20:53:02.710 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 20:53:02.710 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:53:02.710 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:53:02.710 | INFO | PerfReport: Full Execution -- 9.92e+01 / -nan, 2.52e+01 / 0.00e+00 | 20260505 20:53:02.710 | INFO | PerfReport: 22.46%, Upload -- 2.23e+01 / 0.00e+00, 7.70e+00 / 0.00e+00 | 20260505 20:53:02.710 | INFO | PerfReport: 21.81%, Download -- 2.16e+01 / 0.00e+00, 7.65e+00 / 0.00e+00 | 20260505 20:53:02.710 | INFO | PerfReport: 31.67%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 20:53:02.710 | INFO | PerfReport: 2.28%, LockUpload -- 2.27e+00 / 0.00e+00, 8.05e+00 / 0.00e+00 | 20260505 20:53:02.710 | INFO | PerfReport: 1.37%, LockDownload -- 1.36e+00 / 0.00e+00, 2.90e+00 / 0.00e+00 | 20260505 20:53:02.710 | INFO | PerfReport: 20.39%, LockCompute -- 2.02e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 20:53:02.710 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.88e-02 / 0.00e+00 | 20260505 20:53:02.840 | INFO | Starting a round of Baz IO | 20260505 20:53:06.011 | INFO | Populating RTMetrics report | 20260505 20:53:06.011 | INFO | RTMetrics report populated | 20260505 20:53:06.011 | INFO | Opening tmp RTMetrics json file | 20260505 20:53:06.012 | INFO | Serializing RTMetrics to json | 20260505 20:53:06.013 | INFO | Serialization done, swapping tmp file now... | 20260505 20:53:06.013 | INFO | RTMetrics done | 20260505 20:53:06.097 | INFO | Finished a round of Baz IO: Wrote 1.994 GiB in 3.257 seconds | 20260505 20:53:06.097 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.63% / 0.00% / 78.37% | 20260505 20:53:06.097 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.162776% / 0% / 8.34465e-05% | 20260505 20:53:06.097 | INFO | Baz IO overall: | 20260505 20:53:06.097 | INFO | BytesWritten : 3618676847973 | 20260505 20:53:06.097 | INFO | NumEvents : 1709828482984 | 20260505 20:53:07.845 | INFO | Analyzing chunk frames = [6225920,6226432) | 20260505 20:53:07.845 | INFO | MemInfo: { | 20260505 20:53:07.845 | INFO | "gpu" : " 17 GB", | 20260505 20:53:07.845 | INFO | "huge" : "", | 20260505 20:53:07.845 | INFO | "huge_shared" : " 27 GB", | 20260505 20:53:07.845 | INFO | "malloc" : " 14 GB", | 20260505 20:53:07.845 | INFO | "pinned" : " 29 GB" | 20260505 20:53:07.845 | INFO | } | 20260505 20:53:07.845 | INFO | Perf report for the last 5.13661 seconds: | 20260505 20:53:07.845 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:53:07.845 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:07.845 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:07.845 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:07.845 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 20:53:07.845 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:07.845 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260505 20:53:07.845 | INFO | PRE_HQ : 60.1%, 1.8, 54.9ms, 0 | 20260505 20:53:07.845 | INFO | BAZWRITER : 62.9%, 3.7, 120.0ms, 0 | 20260505 20:53:07.845 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 20:53:07.846 | INFO | Uploaded 36GiB to GPU in 2.17207s (16.574GiB/s) | 20260505 20:53:07.846 | INFO | Downloaded 38GiB to GPU in 2.25786s (16.8301GiB/s) | 20260505 20:53:12.987 | INFO | Analyzing chunk frames = [6226432,6226944) | 20260505 20:53:12.987 | INFO | MemInfo: { | 20260505 20:53:12.987 | INFO | "gpu" : " 17 GB", | 20260505 20:53:12.987 | INFO | "huge" : "", | 20260505 20:53:12.987 | INFO | "huge_shared" : " 27 GB", | 20260505 20:53:12.987 | INFO | "malloc" : " 14 GB", | 20260505 20:53:12.987 | INFO | "pinned" : " 29 GB" | 20260505 20:53:12.987 | INFO | } | 20260505 20:53:12.988 | INFO | Perf report for the last 5.14233 seconds: | 20260505 20:53:12.988 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:53:12.988 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:12.988 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:12.988 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:12.988 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260505 20:53:12.988 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:12.988 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:12.988 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 20:53:12.988 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:12.988 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:12.988 | INFO | Uploaded 36GiB to GPU in 2.14913s (16.7509GiB/s) | 20260505 20:53:12.988 | INFO | Downloaded 34GiB to GPU in 2.04725s (16.6077GiB/s) | 20260505 20:53:18.136 | INFO | Analyzing chunk frames = [6226944,6227456) | 20260505 20:53:18.136 | INFO | MemInfo: { | 20260505 20:53:18.136 | INFO | "gpu" : " 17 GB", | 20260505 20:53:18.136 | INFO | "huge" : "", | 20260505 20:53:18.136 | INFO | "huge_shared" : " 27 GB", | 20260505 20:53:18.136 | INFO | "malloc" : " 14 GB", | 20260505 20:53:18.136 | INFO | "pinned" : " 29 GB" | 20260505 20:53:18.136 | INFO | } | 20260505 20:53:18.136 | INFO | Perf report for the last 5.1487 seconds: | 20260505 20:53:18.136 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:53:18.136 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:18.136 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:18.136 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:18.136 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260505 20:53:18.136 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:18.136 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:18.136 | INFO | PRE_HQ : 27.2%, 1.0, 14.2ms, 0 | 20260505 20:53:18.136 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:18.136 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:18.137 | INFO | Uploaded 36GiB to GPU in 2.12138s (16.9701GiB/s) | 20260505 20:53:18.137 | INFO | Downloaded 34GiB to GPU in 2.02204s (16.8147GiB/s) | 20260505 20:53:23.274 | INFO | Analyzing chunk frames = [6227456,6227968) | 20260505 20:53:23.274 | INFO | MemInfo: { | 20260505 20:53:23.274 | INFO | "gpu" : " 17 GB", | 20260505 20:53:23.274 | INFO | "huge" : "", | 20260505 20:53:23.274 | INFO | "huge_shared" : " 27 GB", | 20260505 20:53:23.274 | INFO | "malloc" : " 14 GB", | 20260505 20:53:23.274 | INFO | "pinned" : " 29 GB" | 20260505 20:53:23.274 | INFO | } | 20260505 20:53:23.274 | INFO | Perf report for the last 5.13805 seconds: | 20260505 20:53:23.274 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:53:23.274 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:23.274 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:23.274 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:23.274 | INFO | ANALYSIS : 61.9%, 3.0, 96.1ms, 0 | 20260505 20:53:23.274 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:23.274 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:23.274 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 20:53:23.274 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:23.274 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:23.275 | INFO | Uploaded 36GiB to GPU in 2.13368s (16.8723GiB/s) | 20260505 20:53:23.275 | INFO | Downloaded 34GiB to GPU in 2.02247s (16.8111GiB/s) | 20260505 20:53:28.403 | INFO | Analyzing chunk frames = [6227968,6228480) | 20260505 20:53:28.403 | INFO | MemInfo: { | 20260505 20:53:28.403 | INFO | "gpu" : " 17 GB", | 20260505 20:53:28.403 | INFO | "huge" : "", | 20260505 20:53:28.403 | INFO | "huge_shared" : " 27 GB", | 20260505 20:53:28.403 | INFO | "malloc" : " 14 GB", | 20260505 20:53:28.403 | INFO | "pinned" : " 29 GB" | 20260505 20:53:28.403 | INFO | } | 20260505 20:53:28.403 | INFO | Perf report for the last 5.12865 seconds: | 20260505 20:53:28.403 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:53:28.403 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:28.403 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:28.403 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:28.403 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260505 20:53:28.403 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:28.403 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:28.403 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:53:28.403 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:28.403 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:28.403 | INFO | Uploaded 36GiB to GPU in 2.13838s (16.8352GiB/s) | 20260505 20:53:28.403 | INFO | Downloaded 34GiB to GPU in 2.06118s (16.4954GiB/s) | 20260505 20:53:33.540 | INFO | Analyzing chunk frames = [6228480,6228992) | 20260505 20:53:33.540 | INFO | MemInfo: { | 20260505 20:53:33.540 | INFO | "gpu" : " 17 GB", | 20260505 20:53:33.540 | INFO | "huge" : "", | 20260505 20:53:33.540 | INFO | "huge_shared" : " 27 GB", | 20260505 20:53:33.540 | INFO | "malloc" : " 14 GB", | 20260505 20:53:33.540 | INFO | "pinned" : " 29 GB" | 20260505 20:53:33.540 | INFO | } | 20260505 20:53:33.540 | INFO | Perf report for the last 5.13689 seconds: | 20260505 20:53:33.540 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:53:33.540 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:33.540 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:33.540 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:33.540 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 20:53:33.540 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:33.540 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:33.540 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:53:33.540 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:33.540 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:33.540 | INFO | Uploaded 36GiB to GPU in 2.16022s (16.665GiB/s) | 20260505 20:53:33.540 | INFO | Downloaded 34GiB to GPU in 2.06246s (16.4852GiB/s) | 20260505 20:53:38.685 | INFO | Analyzing chunk frames = [6228992,6229504) | 20260505 20:53:38.685 | INFO | MemInfo: { | 20260505 20:53:38.685 | INFO | "gpu" : " 17 GB", | 20260505 20:53:38.685 | INFO | "huge" : "", | 20260505 20:53:38.685 | INFO | "huge_shared" : " 27 GB", | 20260505 20:53:38.685 | INFO | "malloc" : " 14 GB", | 20260505 20:53:38.685 | INFO | "pinned" : " 29 GB" | 20260505 20:53:38.685 | INFO | } | 20260505 20:53:38.685 | INFO | Perf report for the last 5.14508 seconds: | 20260505 20:53:38.685 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:53:38.685 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:38.685 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:38.685 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:38.685 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 20:53:38.685 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:38.685 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:38.685 | INFO | PRE_HQ : 28.3%, 1.0, 14.8ms, 0 | 20260505 20:53:38.685 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:38.685 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:38.685 | INFO | Uploaded 36GiB to GPU in 2.12216s (16.9638GiB/s) | 20260505 20:53:38.685 | INFO | Downloaded 34GiB to GPU in 2.13373s (15.9346GiB/s) | 20260505 20:53:43.824 | INFO | Analyzing chunk frames = [6229504,6230016) | 20260505 20:53:43.824 | INFO | MemInfo: { | 20260505 20:53:43.824 | INFO | "gpu" : " 17 GB", | 20260505 20:53:43.824 | INFO | "huge" : "", | 20260505 20:53:43.824 | INFO | "huge_shared" : " 27 GB", | 20260505 20:53:43.824 | INFO | "malloc" : " 14 GB", | 20260505 20:53:43.824 | INFO | "pinned" : " 29 GB" | 20260505 20:53:43.824 | INFO | } | 20260505 20:53:43.824 | INFO | Perf report for the last 5.13926 seconds: | 20260505 20:53:43.824 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:53:43.824 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:43.824 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:43.824 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:43.824 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 20:53:43.824 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:43.824 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:43.824 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 20:53:43.824 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:43.824 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:43.825 | INFO | Uploaded 36GiB to GPU in 2.09949s (17.147GiB/s) | 20260505 20:53:43.825 | INFO | Downloaded 34GiB to GPU in 2.00976s (16.9174GiB/s) | 20260505 20:53:43.958 | INFO | Starting a round of Baz IO | 20260505 20:53:47.115 | INFO | Populating RTMetrics report | 20260505 20:53:47.115 | INFO | RTMetrics report populated | 20260505 20:53:47.115 | INFO | Opening tmp RTMetrics json file | 20260505 20:53:47.118 | INFO | Serializing RTMetrics to json | 20260505 20:53:47.121 | INFO | Serialization done, swapping tmp file now... | 20260505 20:53:47.122 | INFO | RTMetrics done | 20260505 20:53:47.196 | INFO | Finished a round of Baz IO: Wrote 1.994 GiB in 3.238 seconds | 20260505 20:53:47.196 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.63% / 0.00% / 78.37% | 20260505 20:53:47.196 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.161016% / 0% / 5.5631e-05% | 20260505 20:53:47.196 | INFO | Baz IO overall: | 20260505 20:53:47.196 | INFO | BytesWritten : 3620817701076 | 20260505 20:53:47.196 | INFO | NumEvents : 1710818843529 | 20260505 20:53:48.970 | INFO | Analyzing chunk frames = [6230016,6230528) | 20260505 20:53:48.970 | INFO | MemInfo: { | 20260505 20:53:48.970 | INFO | "gpu" : " 17 GB", | 20260505 20:53:48.971 | INFO | "huge" : "", | 20260505 20:53:48.971 | INFO | "huge_shared" : " 27 GB", | 20260505 20:53:48.971 | INFO | "malloc" : " 14 GB", | 20260505 20:53:48.971 | INFO | "pinned" : " 29 GB" | 20260505 20:53:48.971 | INFO | } | 20260505 20:53:48.971 | INFO | Perf report for the last 5.14641 seconds: | 20260505 20:53:48.971 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:53:48.971 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:48.971 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:48.971 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:48.971 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 20:53:48.971 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:48.971 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260505 20:53:48.971 | INFO | PRE_HQ : 60.0%, 1.8, 55.2ms, 0 | 20260505 20:53:48.971 | INFO | BAZWRITER : 62.4%, 3.7, 118.3ms, 0 | 20260505 20:53:48.971 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 20:53:48.971 | INFO | Uploaded 36GiB to GPU in 2.17431s (16.557GiB/s) | 20260505 20:53:48.971 | INFO | Downloaded 38GiB to GPU in 2.24785s (16.9051GiB/s) | 20260505 20:53:54.109 | INFO | Analyzing chunk frames = [6230528,6231040) | 20260505 20:53:54.109 | INFO | MemInfo: { | 20260505 20:53:54.109 | INFO | "gpu" : " 17 GB", | 20260505 20:53:54.109 | INFO | "huge" : "", | 20260505 20:53:54.109 | INFO | "huge_shared" : " 27 GB", | 20260505 20:53:54.109 | INFO | "malloc" : " 14 GB", | 20260505 20:53:54.109 | INFO | "pinned" : " 29 GB" | 20260505 20:53:54.109 | INFO | } | 20260505 20:53:54.109 | INFO | Perf report for the last 5.13837 seconds: | 20260505 20:53:54.109 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:53:54.109 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:54.109 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:54.109 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:54.109 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 20:53:54.109 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:54.109 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:54.109 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 20:53:54.109 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:54.109 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:54.109 | INFO | Uploaded 36GiB to GPU in 2.09025s (17.2229GiB/s) | 20260505 20:53:54.109 | INFO | Downloaded 34GiB to GPU in 2.02736s (16.7706GiB/s) | 20260505 20:53:59.243 | INFO | Analyzing chunk frames = [6231040,6231552) | 20260505 20:53:59.243 | INFO | MemInfo: { | 20260505 20:53:59.243 | INFO | "gpu" : " 17 GB", | 20260505 20:53:59.243 | INFO | "huge" : "", | 20260505 20:53:59.243 | INFO | "huge_shared" : " 27 GB", | 20260505 20:53:59.243 | INFO | "malloc" : " 14 GB", | 20260505 20:53:59.243 | INFO | "pinned" : " 29 GB" | 20260505 20:53:59.243 | INFO | } | 20260505 20:53:59.243 | INFO | Perf report for the last 5.13426 seconds: | 20260505 20:53:59.243 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:53:59.243 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:59.243 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:59.243 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:59.243 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260505 20:53:59.243 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:59.243 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:59.243 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 20:53:59.243 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:59.243 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:53:59.244 | INFO | Uploaded 36GiB to GPU in 2.14912s (16.751GiB/s) | 20260505 20:53:59.244 | INFO | Downloaded 34GiB to GPU in 2.03274s (16.7262GiB/s) | 20260505 20:54:04.386 | INFO | Analyzing chunk frames = [6231552,6232064) | 20260505 20:54:04.386 | INFO | MemInfo: { | 20260505 20:54:04.386 | INFO | "gpu" : " 17 GB", | 20260505 20:54:04.387 | INFO | "huge" : "", | 20260505 20:54:04.387 | INFO | "huge_shared" : " 27 GB", | 20260505 20:54:04.387 | INFO | "malloc" : " 14 GB", | 20260505 20:54:04.387 | INFO | "pinned" : " 29 GB" | 20260505 20:54:04.387 | INFO | } | 20260505 20:54:04.387 | INFO | Perf report for the last 5.1433 seconds: | 20260505 20:54:04.387 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:54:04.387 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:04.387 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:04.387 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:04.387 | INFO | ANALYSIS : 62.4%, 3.0, 96.9ms, 0 | 20260505 20:54:04.387 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:04.387 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:04.387 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:54:04.387 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:04.387 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:04.387 | INFO | Uploaded 36GiB to GPU in 2.21206s (16.2745GiB/s) | 20260505 20:54:04.387 | INFO | Downloaded 34GiB to GPU in 2.05097s (16.5775GiB/s) | 20260505 20:54:09.524 | INFO | Analyzing chunk frames = [6232064,6232576) | 20260505 20:54:09.524 | INFO | MemInfo: { | 20260505 20:54:09.524 | INFO | "gpu" : " 17 GB", | 20260505 20:54:09.524 | INFO | "huge" : "", | 20260505 20:54:09.524 | INFO | "huge_shared" : " 27 GB", | 20260505 20:54:09.524 | INFO | "malloc" : " 14 GB", | 20260505 20:54:09.524 | INFO | "pinned" : " 29 GB" | 20260505 20:54:09.524 | INFO | } | 20260505 20:54:09.524 | INFO | Perf report for the last 5.13748 seconds: | 20260505 20:54:09.524 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:54:09.524 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:09.524 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:09.524 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:09.524 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260505 20:54:09.524 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:09.524 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:09.524 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 20:54:09.524 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:09.524 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:09.524 | INFO | Uploaded 36GiB to GPU in 2.13902s (16.8302GiB/s) | 20260505 20:54:09.524 | INFO | Downloaded 34GiB to GPU in 2.06308s (16.4802GiB/s) | 20260505 20:54:14.660 | INFO | Analyzing chunk frames = [6232576,6233088) | 20260505 20:54:14.660 | INFO | MemInfo: { | 20260505 20:54:14.660 | INFO | "gpu" : " 17 GB", | 20260505 20:54:14.660 | INFO | "huge" : "", | 20260505 20:54:14.660 | INFO | "huge_shared" : " 27 GB", | 20260505 20:54:14.660 | INFO | "malloc" : " 14 GB", | 20260505 20:54:14.660 | INFO | "pinned" : " 29 GB" | 20260505 20:54:14.660 | INFO | } | 20260505 20:54:14.660 | INFO | Perf report for the last 5.13586 seconds: | 20260505 20:54:14.660 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:54:14.660 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:14.660 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:14.660 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:14.660 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 20:54:14.660 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:14.660 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:14.660 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 20:54:14.660 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:14.660 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:14.660 | INFO | Uploaded 36GiB to GPU in 2.15871s (16.6766GiB/s) | 20260505 20:54:14.660 | INFO | Downloaded 34GiB to GPU in 2.06402s (16.4727GiB/s) | 20260505 20:54:19.797 | INFO | Analyzing chunk frames = [6233088,6233600) | 20260505 20:54:19.797 | INFO | MemInfo: { | 20260505 20:54:19.797 | INFO | "gpu" : " 17 GB", | 20260505 20:54:19.797 | INFO | "huge" : "", | 20260505 20:54:19.797 | INFO | "huge_shared" : " 27 GB", | 20260505 20:54:19.797 | INFO | "malloc" : " 14 GB", | 20260505 20:54:19.798 | INFO | "pinned" : " 29 GB" | 20260505 20:54:19.798 | INFO | } | 20260505 20:54:19.798 | INFO | Perf report for the last 5.13766 seconds: | 20260505 20:54:19.798 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:54:19.798 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:19.798 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:19.798 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:19.798 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 20:54:19.798 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:19.798 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:19.798 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 20:54:19.798 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:19.798 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:19.798 | INFO | Uploaded 36GiB to GPU in 2.12063s (16.9761GiB/s) | 20260505 20:54:19.798 | INFO | Downloaded 34GiB to GPU in 2.15076s (15.8084GiB/s) | 20260505 20:54:24.943 | INFO | Analyzing chunk frames = [6233600,6234112) | 20260505 20:54:24.943 | INFO | MemInfo: { | 20260505 20:54:24.943 | INFO | "gpu" : " 17 GB", | 20260505 20:54:24.943 | INFO | "huge" : "", | 20260505 20:54:24.943 | INFO | "huge_shared" : " 27 GB", | 20260505 20:54:24.943 | INFO | "malloc" : " 14 GB", | 20260505 20:54:24.943 | INFO | "pinned" : " 29 GB" | 20260505 20:54:24.943 | INFO | } | 20260505 20:54:24.943 | INFO | Perf report for the last 5.14508 seconds: | 20260505 20:54:24.943 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:54:24.943 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:24.943 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:24.943 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:24.943 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 20:54:24.943 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:24.943 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:24.943 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:54:24.943 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:24.943 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:24.943 | INFO | Uploaded 36GiB to GPU in 2.20839s (16.3015GiB/s) | 20260505 20:54:24.943 | INFO | Downloaded 34GiB to GPU in 2.0481s (16.6007GiB/s) | 20260505 20:54:24.943 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 20:54:24.943 | INFO | Total mem usage: 15.3 GiB | 20260505 20:54:24.943 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 20:54:24.943 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.33%) | 20260505 20:54:24.943 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 20:54:24.943 | INFO | High Water Breakdown: | 20260505 20:54:24.943 | INFO | 61% : 8.64 GiB ( 90.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 20:54:24.943 | INFO | 38.8% : 5.49 GiB ( 82.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 20:54:24.943 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 20:54:24.943 | INFO | ------------------------------------------ | 20260505 20:54:24.943 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:54:24.943 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 20:54:24.943 | INFO | Total mem usage: 26.5 GiB | 20260505 20:54:24.944 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 20:54:24.944 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 20:54:24.944 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 20:54:24.944 | INFO | High Water Breakdown: | 20260505 20:54:24.944 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:54:24.944 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:54:24.944 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:54:24.944 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:54:24.944 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:54:24.944 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:54:24.944 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 20:54:24.944 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 20:54:24.944 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:54:24.944 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 20:54:24.944 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 20:54:24.944 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 20:54:24.944 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 20:54:24.944 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 20:54:24.944 | INFO | ------------------------------------------ | 20260505 20:54:24.944 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:54:24.944 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 20:54:24.944 | INFO | Total mem usage: 25.6 GiB | 20260505 20:54:24.944 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 20:54:24.944 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 20:54:24.944 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 20:54:24.944 | INFO | High Water Breakdown: | 20260505 20:54:24.944 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 20:54:24.944 | INFO | ------------------------------------------ | 20260505 20:54:24.944 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:54:24.944 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 20:54:24.944 | INFO | Total mem usage: 15.7 GiB | 20260505 20:54:24.944 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 20:54:24.944 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 20:54:24.944 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 20:54:24.944 | INFO | High Water Breakdown: | 20260505 20:54:24.944 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 20:54:24.944 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 20:54:24.944 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 20:54:24.944 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 20:54:24.944 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 20:54:24.944 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 20:54:24.944 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 20:54:24.944 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 20:54:24.944 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 20:54:24.944 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:54:24.944 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 20:54:24.944 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:54:24.944 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 20:54:24.944 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 20:54:24.944 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 20:54:24.944 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 20:54:24.944 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:54:24.944 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 20:54:24.944 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:54:24.944 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:54:24.944 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:54:24.944 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 20:54:24.944 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 20:54:24.944 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:54:24.944 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 20:54:24.944 | INFO | ------------------------------------------ | 20260505 20:54:24.944 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:54:24.944 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 20:54:24.944 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:54:24.944 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:54:24.944 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.44e+01, 1.36e+01 / 1.94e+00 | 20260505 20:54:24.944 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.04e-01, 1.02e-02 / 4.54e-03 | 20260505 20:54:24.944 | INFO | PerfReport: 11.78%, Baseline -- 3.70e+00 / 4.53e+00, 5.57e-01 / 1.50e-01 | 20260505 20:54:24.944 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.53e+00, 1.90e-01 / 8.03e-02 | 20260505 20:54:24.944 | INFO | PerfReport: 15.96%, DME -- 4.01e+01 / 4.42e+01, 4.09e+00 / 1.98e+00 | 20260505 20:54:24.944 | INFO | PerfReport: 57.12%, FrameLabeling -- 1.79e+01 / 1.95e+01, 1.44e+00 / 1.03e+00 | 20260505 20:54:24.944 | INFO | PerfReport: 7.23%, PulseAccumulating -- 2.27e+00 / 2.83e+00, 4.36e-01 / 2.20e-01 | 20260505 20:54:24.944 | INFO | PerfReport: 3.23%, Metrics -- 1.01e+00 / 1.33e+00, 2.79e-01 / 1.21e-01 | 20260505 20:54:24.944 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.52e-01 / 4.16e-01, 6.19e-02 / 3.04e-02 | 20260505 20:54:24.944 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 20:54:24.944 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:54:24.944 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:54:24.944 | INFO | PerfReport: Full Execution -- 9.93e+01 / -nan, 2.53e+01 / 0.00e+00 | 20260505 20:54:24.944 | INFO | PerfReport: 22.52%, Upload -- 2.24e+01 / 0.00e+00, 7.75e+00 / 0.00e+00 | 20260505 20:54:24.944 | INFO | PerfReport: 21.83%, Download -- 2.17e+01 / 0.00e+00, 7.68e+00 / 0.00e+00 | 20260505 20:54:24.944 | INFO | PerfReport: 31.63%, Compute -- 3.14e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 20:54:24.944 | INFO | PerfReport: 2.35%, LockUpload -- 2.34e+00 / 0.00e+00, 8.06e+00 / 0.00e+00 | 20260505 20:54:24.944 | INFO | PerfReport: 1.37%, LockDownload -- 1.36e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260505 20:54:24.944 | INFO | PerfReport: 20.29%, LockCompute -- 2.02e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 20:54:24.944 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.56e-02 / 0.00e+00 | 20260505 20:54:25.093 | INFO | Starting a round of Baz IO | 20260505 20:54:28.235 | INFO | Populating RTMetrics report | 20260505 20:54:28.235 | INFO | RTMetrics report populated | 20260505 20:54:28.235 | INFO | Opening tmp RTMetrics json file | 20260505 20:54:28.236 | INFO | Serializing RTMetrics to json | 20260505 20:54:28.237 | INFO | Serialization done, swapping tmp file now... | 20260505 20:54:28.237 | INFO | RTMetrics done | 20260505 20:54:28.327 | INFO | Finished a round of Baz IO: Wrote 1.992 GiB in 3.234 seconds | 20260505 20:54:28.327 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.62% / 0.00% / 78.37% | 20260505 20:54:28.327 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.161064% / 0% / 7.94729e-05% | 20260505 20:54:28.327 | INFO | Baz IO overall: | 20260505 20:54:28.327 | INFO | BytesWritten : 3622956818732 | 20260505 20:54:28.327 | INFO | NumEvents : 1711808144129 | 20260505 20:54:30.080 | INFO | Analyzing chunk frames = [6234112,6234624) | 20260505 20:54:30.080 | INFO | MemInfo: { | 20260505 20:54:30.080 | INFO | "gpu" : " 17 GB", | 20260505 20:54:30.080 | INFO | "huge" : "", | 20260505 20:54:30.080 | INFO | "huge_shared" : " 27 GB", | 20260505 20:54:30.080 | INFO | "malloc" : " 14 GB", | 20260505 20:54:30.080 | INFO | "pinned" : " 29 GB" | 20260505 20:54:30.080 | INFO | } | 20260505 20:54:30.080 | INFO | Perf report for the last 5.13757 seconds: | 20260505 20:54:30.080 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:54:30.080 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:30.080 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:30.080 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:30.081 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 20:54:30.081 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:30.081 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260505 20:54:30.081 | INFO | PRE_HQ : 59.8%, 1.8, 54.7ms, 0 | 20260505 20:54:30.081 | INFO | BAZWRITER : 62.6%, 3.7, 118.7ms, 0 | 20260505 20:54:30.081 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 20:54:30.081 | INFO | Uploaded 36GiB to GPU in 2.17633s (16.5416GiB/s) | 20260505 20:54:30.081 | INFO | Downloaded 38GiB to GPU in 2.24576s (16.9208GiB/s) | 20260505 20:54:35.221 | INFO | Analyzing chunk frames = [6234624,6235136) | 20260505 20:54:35.221 | INFO | MemInfo: { | 20260505 20:54:35.221 | INFO | "gpu" : " 17 GB", | 20260505 20:54:35.221 | INFO | "huge" : "", | 20260505 20:54:35.221 | INFO | "huge_shared" : " 27 GB", | 20260505 20:54:35.221 | INFO | "malloc" : " 14 GB", | 20260505 20:54:35.221 | INFO | "pinned" : " 29 GB" | 20260505 20:54:35.221 | INFO | } | 20260505 20:54:35.221 | INFO | Perf report for the last 5.14093 seconds: | 20260505 20:54:35.221 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:54:35.221 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:35.221 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:35.221 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:35.221 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 20:54:35.221 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:35.221 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:35.221 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 20:54:35.221 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:35.221 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:35.222 | INFO | Uploaded 36GiB to GPU in 2.0771s (17.3319GiB/s) | 20260505 20:54:35.222 | INFO | Downloaded 34GiB to GPU in 2.02271s (16.8092GiB/s) | 20260505 20:54:40.356 | INFO | Analyzing chunk frames = [6235136,6235648) | 20260505 20:54:40.356 | INFO | MemInfo: { | 20260505 20:54:40.356 | INFO | "gpu" : " 17 GB", | 20260505 20:54:40.356 | INFO | "huge" : "", | 20260505 20:54:40.356 | INFO | "huge_shared" : " 27 GB", | 20260505 20:54:40.357 | INFO | "malloc" : " 14 GB", | 20260505 20:54:40.357 | INFO | "pinned" : " 29 GB" | 20260505 20:54:40.357 | INFO | } | 20260505 20:54:40.357 | INFO | Perf report for the last 5.1353 seconds: | 20260505 20:54:40.357 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:54:40.357 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:40.357 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:40.357 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:40.357 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260505 20:54:40.357 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:40.357 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:40.357 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:54:40.357 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:40.357 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:40.357 | INFO | Uploaded 36GiB to GPU in 2.13397s (16.87GiB/s) | 20260505 20:54:40.357 | INFO | Downloaded 34GiB to GPU in 2.02328s (16.8044GiB/s) | 20260505 20:54:45.492 | INFO | Analyzing chunk frames = [6235648,6236160) | 20260505 20:54:45.492 | INFO | MemInfo: { | 20260505 20:54:45.492 | INFO | "gpu" : " 17 GB", | 20260505 20:54:45.492 | INFO | "huge" : "", | 20260505 20:54:45.492 | INFO | "huge_shared" : " 27 GB", | 20260505 20:54:45.492 | INFO | "malloc" : " 14 GB", | 20260505 20:54:45.492 | INFO | "pinned" : " 29 GB" | 20260505 20:54:45.492 | INFO | } | 20260505 20:54:45.492 | INFO | Perf report for the last 5.13514 seconds: | 20260505 20:54:45.492 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:54:45.492 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:45.492 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:45.492 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:45.492 | INFO | ANALYSIS : 62.3%, 3.0, 96.7ms, 0 | 20260505 20:54:45.492 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:45.492 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:45.492 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:54:45.492 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:45.492 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:45.492 | INFO | Uploaded 36GiB to GPU in 2.19344s (16.4126GiB/s) | 20260505 20:54:45.492 | INFO | Downloaded 34GiB to GPU in 2.06669s (16.4514GiB/s) | 20260505 20:54:50.635 | INFO | Analyzing chunk frames = [6236160,6236672) | 20260505 20:54:50.635 | INFO | MemInfo: { | 20260505 20:54:50.635 | INFO | "gpu" : " 17 GB", | 20260505 20:54:50.636 | INFO | "huge" : "", | 20260505 20:54:50.636 | INFO | "huge_shared" : " 27 GB", | 20260505 20:54:50.636 | INFO | "malloc" : " 14 GB", | 20260505 20:54:50.636 | INFO | "pinned" : " 29 GB" | 20260505 20:54:50.636 | INFO | } | 20260505 20:54:50.636 | INFO | Perf report for the last 5.14391 seconds: | 20260505 20:54:50.636 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:54:50.636 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:50.636 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:50.636 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:50.636 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 20:54:50.636 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:50.636 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:50.636 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:54:50.636 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:50.636 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:50.636 | INFO | Uploaded 36GiB to GPU in 2.13056s (16.897GiB/s) | 20260505 20:54:50.636 | INFO | Downloaded 34GiB to GPU in 2.06533s (16.4623GiB/s) | 20260505 20:54:55.784 | INFO | Analyzing chunk frames = [6236672,6237184) | 20260505 20:54:55.784 | INFO | MemInfo: { | 20260505 20:54:55.784 | INFO | "gpu" : " 17 GB", | 20260505 20:54:55.784 | INFO | "huge" : "", | 20260505 20:54:55.784 | INFO | "huge_shared" : " 27 GB", | 20260505 20:54:55.784 | INFO | "malloc" : " 14 GB", | 20260505 20:54:55.784 | INFO | "pinned" : " 29 GB" | 20260505 20:54:55.784 | INFO | } | 20260505 20:54:55.784 | INFO | Perf report for the last 5.14856 seconds: | 20260505 20:54:55.784 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:54:55.784 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:55.784 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:55.784 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:55.784 | INFO | ANALYSIS : 61.7%, 3.0, 96.6ms, 0 | 20260505 20:54:55.784 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:55.784 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:55.784 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 20:54:55.784 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:55.784 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:54:55.785 | INFO | Uploaded 36GiB to GPU in 2.09345s (17.1965GiB/s) | 20260505 20:54:55.785 | INFO | Downloaded 34GiB to GPU in 2.03309s (16.7233GiB/s) | 20260505 20:55:00.915 | INFO | Analyzing chunk frames = [6237184,6237696) | 20260505 20:55:00.915 | INFO | MemInfo: { | 20260505 20:55:00.915 | INFO | "gpu" : " 17 GB", | 20260505 20:55:00.915 | INFO | "huge" : "", | 20260505 20:55:00.915 | INFO | "huge_shared" : " 27 GB", | 20260505 20:55:00.915 | INFO | "malloc" : " 14 GB", | 20260505 20:55:00.915 | INFO | "pinned" : " 29 GB" | 20260505 20:55:00.915 | INFO | } | 20260505 20:55:00.915 | INFO | Perf report for the last 5.13079 seconds: | 20260505 20:55:00.915 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:55:00.915 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:00.915 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:00.915 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:00.915 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260505 20:55:00.915 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:00.915 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:00.915 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 20:55:00.915 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:00.915 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:00.915 | INFO | Uploaded 36GiB to GPU in 2.10515s (17.1009GiB/s) | 20260505 20:55:00.915 | INFO | Downloaded 34GiB to GPU in 2.0875s (16.2874GiB/s) | 20260505 20:55:06.050 | INFO | Analyzing chunk frames = [6237696,6238208) | 20260505 20:55:06.050 | INFO | MemInfo: { | 20260505 20:55:06.050 | INFO | "gpu" : " 17 GB", | 20260505 20:55:06.050 | INFO | "huge" : "", | 20260505 20:55:06.050 | INFO | "huge_shared" : " 27 GB", | 20260505 20:55:06.050 | INFO | "malloc" : " 14 GB", | 20260505 20:55:06.050 | INFO | "pinned" : " 29 GB" | 20260505 20:55:06.050 | INFO | } | 20260505 20:55:06.050 | INFO | Perf report for the last 5.13511 seconds: | 20260505 20:55:06.050 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:55:06.050 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:06.050 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:06.050 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:06.050 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260505 20:55:06.050 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:06.050 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:06.050 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:55:06.050 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:06.050 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:06.050 | INFO | Uploaded 36GiB to GPU in 2.11358s (17.0327GiB/s) | 20260505 20:55:06.050 | INFO | Downloaded 34GiB to GPU in 2.01871s (16.8424GiB/s) | 20260505 20:55:06.180 | INFO | Starting a round of Baz IO | 20260505 20:55:09.350 | INFO | Populating RTMetrics report | 20260505 20:55:09.350 | INFO | RTMetrics report populated | 20260505 20:55:09.350 | INFO | Opening tmp RTMetrics json file | 20260505 20:55:09.352 | INFO | Serializing RTMetrics to json | 20260505 20:55:09.353 | INFO | Serialization done, swapping tmp file now... | 20260505 20:55:09.354 | INFO | RTMetrics done | 20260505 20:55:09.441 | INFO | Finished a round of Baz IO: Wrote 1.993 GiB in 3.261 seconds | 20260505 20:55:09.441 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.62% / 0.00% / 78.38% | 20260505 20:55:09.441 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.16191% / 0% / 7.15256e-05% | 20260505 20:55:09.441 | INFO | Baz IO overall: | 20260505 20:55:09.441 | INFO | BytesWritten : 3625096860281 | 20260505 20:55:09.441 | INFO | NumEvents : 1712798356040 | 20260505 20:55:11.196 | INFO | Analyzing chunk frames = [6238208,6238720) | 20260505 20:55:11.196 | INFO | MemInfo: { | 20260505 20:55:11.196 | INFO | "gpu" : " 17 GB", | 20260505 20:55:11.196 | INFO | "huge" : "", | 20260505 20:55:11.196 | INFO | "huge_shared" : " 27 GB", | 20260505 20:55:11.196 | INFO | "malloc" : " 14 GB", | 20260505 20:55:11.196 | INFO | "pinned" : " 29 GB" | 20260505 20:55:11.196 | INFO | } | 20260505 20:55:11.196 | INFO | Perf report for the last 5.14561 seconds: | 20260505 20:55:11.196 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:55:11.196 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:11.196 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:11.196 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:11.196 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 20:55:11.196 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:11.196 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260505 20:55:11.196 | INFO | PRE_HQ : 59.8%, 1.8, 54.9ms, 0 | 20260505 20:55:11.196 | INFO | BAZWRITER : 62.8%, 3.7, 119.0ms, 0 | 20260505 20:55:11.196 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 20:55:11.196 | INFO | Uploaded 36GiB to GPU in 2.16697s (16.6131GiB/s) | 20260505 20:55:11.196 | INFO | Downloaded 38GiB to GPU in 2.24812s (16.903GiB/s) | 20260505 20:55:16.334 | INFO | Analyzing chunk frames = [6238720,6239232) | 20260505 20:55:16.334 | INFO | MemInfo: { | 20260505 20:55:16.334 | INFO | "gpu" : " 17 GB", | 20260505 20:55:16.334 | INFO | "huge" : "", | 20260505 20:55:16.334 | INFO | "huge_shared" : " 27 GB", | 20260505 20:55:16.334 | INFO | "malloc" : " 14 GB", | 20260505 20:55:16.334 | INFO | "pinned" : " 29 GB" | 20260505 20:55:16.334 | INFO | } | 20260505 20:55:16.334 | INFO | Perf report for the last 5.13794 seconds: | 20260505 20:55:16.334 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:55:16.334 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:16.334 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:16.334 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:16.334 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 20:55:16.334 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:16.334 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:16.334 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 20:55:16.334 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:16.334 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:16.334 | INFO | Uploaded 36GiB to GPU in 2.07446s (17.3539GiB/s) | 20260505 20:55:16.334 | INFO | Downloaded 34GiB to GPU in 2.01599s (16.8652GiB/s) | 20260505 20:55:21.475 | INFO | Analyzing chunk frames = [6239232,6239744) | 20260505 20:55:21.475 | INFO | MemInfo: { | 20260505 20:55:21.475 | INFO | "gpu" : " 17 GB", | 20260505 20:55:21.475 | INFO | "huge" : "", | 20260505 20:55:21.475 | INFO | "huge_shared" : " 27 GB", | 20260505 20:55:21.475 | INFO | "malloc" : " 14 GB", | 20260505 20:55:21.475 | INFO | "pinned" : " 29 GB" | 20260505 20:55:21.475 | INFO | } | 20260505 20:55:21.475 | INFO | Perf report for the last 5.14169 seconds: | 20260505 20:55:21.475 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:55:21.475 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:21.475 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:21.476 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:21.476 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260505 20:55:21.476 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:21.476 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:21.476 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:55:21.476 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:21.476 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:21.476 | INFO | Uploaded 36GiB to GPU in 2.13501s (16.8618GiB/s) | 20260505 20:55:21.476 | INFO | Downloaded 34GiB to GPU in 2.02633s (16.7791GiB/s) | 20260505 20:55:26.611 | INFO | Analyzing chunk frames = [6239744,6240256) | 20260505 20:55:26.611 | INFO | MemInfo: { | 20260505 20:55:26.611 | INFO | "gpu" : " 17 GB", | 20260505 20:55:26.611 | INFO | "huge" : "", | 20260505 20:55:26.611 | INFO | "huge_shared" : " 27 GB", | 20260505 20:55:26.611 | INFO | "malloc" : " 14 GB", | 20260505 20:55:26.611 | INFO | "pinned" : " 29 GB" | 20260505 20:55:26.611 | INFO | } | 20260505 20:55:26.611 | INFO | Perf report for the last 5.13532 seconds: | 20260505 20:55:26.611 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:55:26.611 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:26.611 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:26.611 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:26.611 | INFO | ANALYSIS : 62.4%, 3.0, 96.9ms, 0 | 20260505 20:55:26.611 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:26.611 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:26.611 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:55:26.611 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:26.611 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:26.611 | INFO | Uploaded 36GiB to GPU in 2.20746s (16.3083GiB/s) | 20260505 20:55:26.611 | INFO | Downloaded 34GiB to GPU in 2.07576s (16.3796GiB/s) | 20260505 20:55:31.747 | INFO | Analyzing chunk frames = [6240256,6240768) | 20260505 20:55:31.747 | INFO | MemInfo: { | 20260505 20:55:31.747 | INFO | "gpu" : " 17 GB", | 20260505 20:55:31.747 | INFO | "huge" : "", | 20260505 20:55:31.748 | INFO | "huge_shared" : " 27 GB", | 20260505 20:55:31.748 | INFO | "malloc" : " 14 GB", | 20260505 20:55:31.748 | INFO | "pinned" : " 29 GB" | 20260505 20:55:31.748 | INFO | } | 20260505 20:55:31.748 | INFO | Perf report for the last 5.13681 seconds: | 20260505 20:55:31.748 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:55:31.748 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:31.748 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:31.748 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:31.748 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260505 20:55:31.748 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:31.748 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:31.748 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:55:31.748 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:31.748 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:31.748 | INFO | Uploaded 36GiB to GPU in 2.12951s (16.9053GiB/s) | 20260505 20:55:31.748 | INFO | Downloaded 34GiB to GPU in 2.05867s (16.5155GiB/s) | 20260505 20:55:36.897 | INFO | Analyzing chunk frames = [6240768,6241280) | 20260505 20:55:36.897 | INFO | MemInfo: { | 20260505 20:55:36.897 | INFO | "gpu" : " 17 GB", | 20260505 20:55:36.897 | INFO | "huge" : "", | 20260505 20:55:36.897 | INFO | "huge_shared" : " 27 GB", | 20260505 20:55:36.897 | INFO | "malloc" : " 14 GB", | 20260505 20:55:36.897 | INFO | "pinned" : " 29 GB" | 20260505 20:55:36.897 | INFO | } | 20260505 20:55:36.897 | INFO | Perf report for the last 5.14939 seconds: | 20260505 20:55:36.897 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:55:36.897 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:36.897 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:36.897 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:36.897 | INFO | ANALYSIS : 62.0%, 3.0, 97.0ms, 0 | 20260505 20:55:36.897 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:36.897 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:36.897 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 20:55:36.897 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:36.897 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:36.897 | INFO | Uploaded 36GiB to GPU in 2.1437s (16.7934GiB/s) | 20260505 20:55:36.897 | INFO | Downloaded 34GiB to GPU in 2.0596s (16.508GiB/s) | 20260505 20:55:42.033 | INFO | Analyzing chunk frames = [6241280,6241792) | 20260505 20:55:42.033 | INFO | MemInfo: { | 20260505 20:55:42.033 | INFO | "gpu" : " 17 GB", | 20260505 20:55:42.033 | INFO | "huge" : "", | 20260505 20:55:42.033 | INFO | "huge_shared" : " 27 GB", | 20260505 20:55:42.033 | INFO | "malloc" : " 14 GB", | 20260505 20:55:42.033 | INFO | "pinned" : " 29 GB" | 20260505 20:55:42.033 | INFO | } | 20260505 20:55:42.033 | INFO | Perf report for the last 5.13599 seconds: | 20260505 20:55:42.033 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:55:42.033 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:42.033 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:42.033 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:42.033 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260505 20:55:42.033 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:42.033 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:42.033 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 20:55:42.033 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:42.033 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:42.033 | INFO | Uploaded 36GiB to GPU in 2.13119s (16.8919GiB/s) | 20260505 20:55:42.033 | INFO | Downloaded 34GiB to GPU in 2.09924s (16.1963GiB/s) | 20260505 20:55:47.169 | INFO | Analyzing chunk frames = [6241792,6242304) | 20260505 20:55:47.169 | INFO | MemInfo: { | 20260505 20:55:47.169 | INFO | "gpu" : " 17 GB", | 20260505 20:55:47.169 | INFO | "huge" : "", | 20260505 20:55:47.169 | INFO | "huge_shared" : " 27 GB", | 20260505 20:55:47.169 | INFO | "malloc" : " 14 GB", | 20260505 20:55:47.169 | INFO | "pinned" : " 29 GB" | 20260505 20:55:47.169 | INFO | } | 20260505 20:55:47.169 | INFO | Perf report for the last 5.13587 seconds: | 20260505 20:55:47.169 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:55:47.169 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:47.169 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:47.169 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:47.169 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 20:55:47.169 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:47.169 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:47.169 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:55:47.169 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:47.169 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:47.169 | INFO | Uploaded 36GiB to GPU in 2.10305s (17.118GiB/s) | 20260505 20:55:47.169 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 20:55:47.169 | INFO | Total mem usage: 15.3 GiB | 20260505 20:55:47.169 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 20:55:47.169 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.32%) | 20260505 20:55:47.169 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 20:55:47.169 | INFO | Downloaded 34GiB to GPU in 1.9962s (17.0324GiB/s) | 20260505 20:55:47.169 | INFO | High Water Breakdown: | 20260505 20:55:47.169 | INFO | 61% : 8.64 GiB ( 90.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 20:55:47.170 | INFO | 38.8% : 5.49 GiB ( 82.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 20:55:47.170 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 20:55:47.170 | INFO | ------------------------------------------ | 20260505 20:55:47.170 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:55:47.170 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 20:55:47.170 | INFO | Total mem usage: 26.5 GiB | 20260505 20:55:47.170 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 20:55:47.170 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 20:55:47.170 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 20:55:47.170 | INFO | High Water Breakdown: | 20260505 20:55:47.170 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:55:47.170 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:55:47.170 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:55:47.170 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:55:47.170 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:55:47.170 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:55:47.170 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 20:55:47.170 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 20:55:47.170 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:55:47.170 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 20:55:47.170 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 20:55:47.170 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 20:55:47.170 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 20:55:47.170 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 20:55:47.170 | INFO | ------------------------------------------ | 20260505 20:55:47.170 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:55:47.170 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 20:55:47.170 | INFO | Total mem usage: 25.6 GiB | 20260505 20:55:47.170 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 20:55:47.170 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 20:55:47.170 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 20:55:47.170 | INFO | High Water Breakdown: | 20260505 20:55:47.170 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 20:55:47.170 | INFO | ------------------------------------------ | 20260505 20:55:47.170 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:55:47.170 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 20:55:47.170 | INFO | Total mem usage: 15.7 GiB | 20260505 20:55:47.170 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 20:55:47.170 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 20:55:47.170 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 20:55:47.170 | INFO | High Water Breakdown: | 20260505 20:55:47.170 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 20:55:47.170 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 20:55:47.170 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 20:55:47.170 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 20:55:47.170 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 20:55:47.170 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 20:55:47.170 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 20:55:47.170 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 20:55:47.170 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 20:55:47.170 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:55:47.170 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 20:55:47.170 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:55:47.170 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 20:55:47.170 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 20:55:47.170 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 20:55:47.170 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 20:55:47.170 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:55:47.170 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 20:55:47.170 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:55:47.170 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:55:47.170 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:55:47.170 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 20:55:47.170 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 20:55:47.170 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:55:47.170 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 20:55:47.170 | INFO | ------------------------------------------ | 20260505 20:55:47.170 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:55:47.170 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 20:55:47.170 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:55:47.170 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:55:47.170 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.41e+01, 1.34e+01 / 8.65e-01 | 20260505 20:55:47.170 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.03e-01, 1.04e-02 / 3.21e-03 | 20260505 20:55:47.170 | INFO | PerfReport: 11.79%, Baseline -- 3.71e+00 / 4.53e+00, 5.64e-01 / 1.55e-01 | 20260505 20:55:47.170 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.51e+00, 1.89e-01 / 8.01e-02 | 20260505 20:55:47.170 | INFO | PerfReport: 15.93%, DME -- 4.01e+01 / 4.39e+01, 3.76e+00 / 9.20e-01 | 20260505 20:55:47.170 | INFO | PerfReport: 57.15%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.53e+00 / 1.10e+00 | 20260505 20:55:47.170 | INFO | PerfReport: 7.21%, PulseAccumulating -- 2.27e+00 / 2.80e+00, 4.32e-01 / 1.98e-01 | 20260505 20:55:47.170 | INFO | PerfReport: 3.23%, Metrics -- 1.01e+00 / 1.35e+00, 2.79e-01 / 1.77e-01 | 20260505 20:55:47.170 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.52e-01 / 4.13e-01, 6.16e-02 / 3.78e-02 | 20260505 20:55:47.170 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 20:55:47.170 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:55:47.170 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:55:47.170 | INFO | PerfReport: Full Execution -- 9.91e+01 / -nan, 2.51e+01 / 0.00e+00 | 20260505 20:55:47.170 | INFO | PerfReport: 22.41%, Upload -- 2.22e+01 / 0.00e+00, 7.65e+00 / 0.00e+00 | 20260505 20:55:47.170 | INFO | PerfReport: 21.77%, Download -- 2.16e+01 / 0.00e+00, 7.66e+00 / 0.00e+00 | 20260505 20:55:47.170 | INFO | PerfReport: 31.75%, Compute -- 3.15e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 20:55:47.170 | INFO | PerfReport: 2.21%, LockUpload -- 2.19e+00 / 0.00e+00, 7.85e+00 / 0.00e+00 | 20260505 20:55:47.170 | INFO | PerfReport: 1.31%, LockDownload -- 1.30e+00 / 0.00e+00, 2.83e+00 / 0.00e+00 | 20260505 20:55:47.170 | INFO | PerfReport: 20.53%, LockCompute -- 2.03e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 20:55:47.170 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.66e-02 / 0.00e+00 | 20260505 20:55:47.299 | INFO | Starting a round of Baz IO | 20260505 20:55:50.473 | INFO | Populating RTMetrics report | 20260505 20:55:50.474 | INFO | RTMetrics report populated | 20260505 20:55:50.474 | INFO | Opening tmp RTMetrics json file | 20260505 20:55:50.474 | INFO | Serializing RTMetrics to json | 20260505 20:55:50.476 | INFO | Serialization done, swapping tmp file now... | 20260505 20:55:50.476 | INFO | RTMetrics done | 20260505 20:55:50.561 | INFO | Finished a round of Baz IO: Wrote 1.992 GiB in 3.262 seconds | 20260505 20:55:50.562 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.62% / 0.00% / 78.38% | 20260505 20:55:50.562 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.161016% / 3.97364e-06% / 7.15256e-05% | 20260505 20:55:50.562 | INFO | Baz IO overall: | 20260505 20:55:50.562 | INFO | BytesWritten : 3627236010088 | 20260505 20:55:50.562 | INFO | NumEvents : 1713787867955 | 20260505 20:55:52.306 | INFO | Analyzing chunk frames = [6242304,6242816) | 20260505 20:55:52.306 | INFO | MemInfo: { | 20260505 20:55:52.306 | INFO | "gpu" : " 17 GB", | 20260505 20:55:52.306 | INFO | "huge" : "", | 20260505 20:55:52.306 | INFO | "huge_shared" : " 27 GB", | 20260505 20:55:52.306 | INFO | "malloc" : " 14 GB", | 20260505 20:55:52.306 | INFO | "pinned" : " 29 GB" | 20260505 20:55:52.307 | INFO | } | 20260505 20:55:52.307 | INFO | Perf report for the last 5.13765 seconds: | 20260505 20:55:52.307 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:55:52.307 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:52.307 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:52.307 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:52.307 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 20:55:52.307 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:52.307 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260505 20:55:52.307 | INFO | PRE_HQ : 59.9%, 1.8, 54.8ms, 0 | 20260505 20:55:52.307 | INFO | BAZWRITER : 63.0%, 3.7, 120.1ms, 0 | 20260505 20:55:52.307 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 20:55:52.307 | INFO | Uploaded 36GiB to GPU in 2.17193s (16.5751GiB/s) | 20260505 20:55:52.307 | INFO | Downloaded 38GiB to GPU in 2.23895s (16.9722GiB/s) | 20260505 20:55:57.445 | INFO | Analyzing chunk frames = [6242816,6243328) | 20260505 20:55:57.445 | INFO | MemInfo: { | 20260505 20:55:57.445 | INFO | "gpu" : " 17 GB", | 20260505 20:55:57.445 | INFO | "huge" : "", | 20260505 20:55:57.445 | INFO | "huge_shared" : " 27 GB", | 20260505 20:55:57.445 | INFO | "malloc" : " 14 GB", | 20260505 20:55:57.445 | INFO | "pinned" : " 29 GB" | 20260505 20:55:57.446 | INFO | } | 20260505 20:55:57.446 | INFO | Perf report for the last 5.139 seconds: | 20260505 20:55:57.446 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:55:57.446 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:57.446 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:57.446 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:57.446 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 20:55:57.446 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:57.446 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:57.446 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 20:55:57.446 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:57.446 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:55:57.446 | INFO | Uploaded 36GiB to GPU in 2.14291s (16.7996GiB/s) | 20260505 20:55:57.446 | INFO | Downloaded 34GiB to GPU in 2.04295s (16.6426GiB/s) | 20260505 20:56:02.589 | INFO | Analyzing chunk frames = [6243328,6243840) | 20260505 20:56:02.589 | INFO | MemInfo: { | 20260505 20:56:02.589 | INFO | "gpu" : " 17 GB", | 20260505 20:56:02.589 | INFO | "huge" : "", | 20260505 20:56:02.589 | INFO | "huge_shared" : " 27 GB", | 20260505 20:56:02.589 | INFO | "malloc" : " 14 GB", | 20260505 20:56:02.589 | INFO | "pinned" : " 29 GB" | 20260505 20:56:02.589 | INFO | } | 20260505 20:56:02.589 | INFO | Perf report for the last 5.1433 seconds: | 20260505 20:56:02.589 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:56:02.589 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:02.589 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:02.589 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:02.589 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260505 20:56:02.589 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:02.589 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:02.589 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 20:56:02.589 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:02.589 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:02.589 | INFO | Uploaded 36GiB to GPU in 2.12825s (16.9153GiB/s) | 20260505 20:56:02.589 | INFO | Downloaded 34GiB to GPU in 2.02433s (16.7957GiB/s) | 20260505 20:56:07.728 | INFO | Analyzing chunk frames = [6243840,6244352) | 20260505 20:56:07.728 | INFO | MemInfo: { | 20260505 20:56:07.728 | INFO | "gpu" : " 17 GB", | 20260505 20:56:07.729 | INFO | "huge" : "", | 20260505 20:56:07.729 | INFO | "huge_shared" : " 27 GB", | 20260505 20:56:07.729 | INFO | "malloc" : " 14 GB", | 20260505 20:56:07.729 | INFO | "pinned" : " 29 GB" | 20260505 20:56:07.729 | INFO | } | 20260505 20:56:07.729 | INFO | Perf report for the last 5.13986 seconds: | 20260505 20:56:07.729 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:56:07.729 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:07.729 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:07.729 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:07.729 | INFO | ANALYSIS : 62.0%, 3.0, 96.3ms, 0 | 20260505 20:56:07.729 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:07.729 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:07.729 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:56:07.729 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:07.729 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:07.729 | INFO | Uploaded 36GiB to GPU in 2.13666s (16.8487GiB/s) | 20260505 20:56:07.729 | INFO | Downloaded 34GiB to GPU in 2.03999s (16.6668GiB/s) | 20260505 20:56:12.862 | INFO | Analyzing chunk frames = [6244352,6244864) | 20260505 20:56:12.862 | INFO | MemInfo: { | 20260505 20:56:12.862 | INFO | "gpu" : " 17 GB", | 20260505 20:56:12.862 | INFO | "huge" : "", | 20260505 20:56:12.862 | INFO | "huge_shared" : " 27 GB", | 20260505 20:56:12.862 | INFO | "malloc" : " 14 GB", | 20260505 20:56:12.862 | INFO | "pinned" : " 29 GB" | 20260505 20:56:12.862 | INFO | } | 20260505 20:56:12.862 | INFO | Perf report for the last 5.13355 seconds: | 20260505 20:56:12.862 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:56:12.862 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:12.862 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:12.862 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:12.862 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260505 20:56:12.862 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:12.862 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:12.862 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260505 20:56:12.862 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:12.862 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:12.863 | INFO | Uploaded 36GiB to GPU in 2.13548s (16.8581GiB/s) | 20260505 20:56:12.863 | INFO | Downloaded 34GiB to GPU in 2.05966s (16.5076GiB/s) | 20260505 20:56:18.001 | INFO | Analyzing chunk frames = [6244864,6245376) | 20260505 20:56:18.001 | INFO | MemInfo: { | 20260505 20:56:18.001 | INFO | "gpu" : " 17 GB", | 20260505 20:56:18.001 | INFO | "huge" : "", | 20260505 20:56:18.001 | INFO | "huge_shared" : " 27 GB", | 20260505 20:56:18.002 | INFO | "malloc" : " 14 GB", | 20260505 20:56:18.002 | INFO | "pinned" : " 29 GB" | 20260505 20:56:18.002 | INFO | } | 20260505 20:56:18.002 | INFO | Perf report for the last 5.1393 seconds: | 20260505 20:56:18.002 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:56:18.002 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:18.002 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:18.002 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:18.002 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 20:56:18.002 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:18.002 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:18.002 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 20:56:18.002 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:18.002 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:18.002 | INFO | Uploaded 36GiB to GPU in 2.14642s (16.7721GiB/s) | 20260505 20:56:18.002 | INFO | Downloaded 34GiB to GPU in 2.06159s (16.4921GiB/s) | 20260505 20:56:23.149 | INFO | Analyzing chunk frames = [6245376,6245888) | 20260505 20:56:23.149 | INFO | MemInfo: { | 20260505 20:56:23.149 | INFO | "gpu" : " 17 GB", | 20260505 20:56:23.149 | INFO | "huge" : "", | 20260505 20:56:23.149 | INFO | "huge_shared" : " 27 GB", | 20260505 20:56:23.150 | INFO | "malloc" : " 14 GB", | 20260505 20:56:23.150 | INFO | "pinned" : " 29 GB" | 20260505 20:56:23.150 | INFO | } | 20260505 20:56:23.150 | INFO | Perf report for the last 5.148 seconds: | 20260505 20:56:23.150 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:56:23.150 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:23.150 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:23.150 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:23.150 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 20:56:23.150 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:23.150 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:23.150 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 20:56:23.150 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:23.150 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:23.150 | INFO | Uploaded 36GiB to GPU in 2.12791s (16.918GiB/s) | 20260505 20:56:23.151 | INFO | Downloaded 34GiB to GPU in 2.13859s (15.8984GiB/s) | 20260505 20:56:28.288 | INFO | Analyzing chunk frames = [6245888,6246400) | 20260505 20:56:28.288 | INFO | MemInfo: { | 20260505 20:56:28.288 | INFO | "gpu" : " 17 GB", | 20260505 20:56:28.288 | INFO | "huge" : "", | 20260505 20:56:28.288 | INFO | "huge_shared" : " 27 GB", | 20260505 20:56:28.288 | INFO | "malloc" : " 14 GB", | 20260505 20:56:28.288 | INFO | "pinned" : " 29 GB" | 20260505 20:56:28.288 | INFO | } | 20260505 20:56:28.288 | INFO | Perf report for the last 5.13854 seconds: | 20260505 20:56:28.288 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:56:28.288 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:28.288 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:28.288 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:28.288 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 20:56:28.288 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:28.288 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:28.288 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:56:28.288 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:28.288 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:28.288 | INFO | Uploaded 36GiB to GPU in 2.10312s (17.1174GiB/s) | 20260505 20:56:28.289 | INFO | Downloaded 34GiB to GPU in 2.00603s (16.9489GiB/s) | 20260505 20:56:28.425 | INFO | Starting a round of Baz IO | 20260505 20:56:31.581 | INFO | Populating RTMetrics report | 20260505 20:56:31.581 | INFO | RTMetrics report populated | 20260505 20:56:31.581 | INFO | Opening tmp RTMetrics json file | 20260505 20:56:31.585 | INFO | Serializing RTMetrics to json | 20260505 20:56:31.588 | INFO | Serialization done, swapping tmp file now... | 20260505 20:56:31.588 | INFO | RTMetrics done | 20260505 20:56:31.671 | INFO | Finished a round of Baz IO: Wrote 1.99 GiB in 3.246 seconds | 20260505 20:56:31.671 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.62% / 0.00% / 78.38% | 20260505 20:56:31.671 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.160845% / 1.19209e-05% / 8.34465e-05% | 20260505 20:56:31.671 | INFO | Baz IO overall: | 20260505 20:56:31.671 | INFO | BytesWritten : 3629373286538 | 20260505 20:56:31.671 | INFO | NumEvents : 1714775947512 | 20260505 20:56:33.425 | INFO | Analyzing chunk frames = [6246400,6246912) | 20260505 20:56:33.425 | INFO | MemInfo: { | 20260505 20:56:33.425 | INFO | "gpu" : " 17 GB", | 20260505 20:56:33.425 | INFO | "huge" : "", | 20260505 20:56:33.425 | INFO | "huge_shared" : " 27 GB", | 20260505 20:56:33.425 | INFO | "malloc" : " 14 GB", | 20260505 20:56:33.425 | INFO | "pinned" : " 29 GB" | 20260505 20:56:33.425 | INFO | } | 20260505 20:56:33.425 | INFO | Perf report for the last 5.13667 seconds: | 20260505 20:56:33.425 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:56:33.425 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:33.425 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:33.425 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:33.425 | INFO | ANALYSIS : 63.1%, 3.0, 98.2ms, 0 | 20260505 20:56:33.425 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:33.425 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260505 20:56:33.425 | INFO | PRE_HQ : 60.0%, 1.8, 54.9ms, 0 | 20260505 20:56:33.425 | INFO | BAZWRITER : 62.7%, 3.7, 120.1ms, 0 | 20260505 20:56:33.425 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 20:56:33.425 | INFO | Uploaded 36GiB to GPU in 2.17408s (16.5588GiB/s) | 20260505 20:56:33.425 | INFO | Downloaded 38GiB to GPU in 2.24965s (16.8915GiB/s) | 20260505 20:56:38.566 | INFO | Analyzing chunk frames = [6246912,6247424) | 20260505 20:56:38.566 | INFO | MemInfo: { | 20260505 20:56:38.566 | INFO | "gpu" : " 17 GB", | 20260505 20:56:38.566 | INFO | "huge" : "", | 20260505 20:56:38.566 | INFO | "huge_shared" : " 27 GB", | 20260505 20:56:38.566 | INFO | "malloc" : " 14 GB", | 20260505 20:56:38.566 | INFO | "pinned" : " 29 GB" | 20260505 20:56:38.566 | INFO | } | 20260505 20:56:38.566 | INFO | Perf report for the last 5.14126 seconds: | 20260505 20:56:38.566 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:56:38.566 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:38.566 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:38.566 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:38.566 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 20:56:38.566 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:38.566 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:38.566 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 20:56:38.566 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:38.566 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:38.566 | INFO | Uploaded 36GiB to GPU in 2.07999s (17.3078GiB/s) | 20260505 20:56:38.566 | INFO | Downloaded 34GiB to GPU in 2.01615s (16.8638GiB/s) | 20260505 20:56:43.703 | INFO | Analyzing chunk frames = [6247424,6247936) | 20260505 20:56:43.703 | INFO | MemInfo: { | 20260505 20:56:43.703 | INFO | "gpu" : " 17 GB", | 20260505 20:56:43.703 | INFO | "huge" : "", | 20260505 20:56:43.703 | INFO | "huge_shared" : " 27 GB", | 20260505 20:56:43.703 | INFO | "malloc" : " 14 GB", | 20260505 20:56:43.703 | INFO | "pinned" : " 29 GB" | 20260505 20:56:43.703 | INFO | } | 20260505 20:56:43.703 | INFO | Perf report for the last 5.13704 seconds: | 20260505 20:56:43.703 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:56:43.703 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:43.703 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:43.703 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:43.703 | INFO | ANALYSIS : 62.1%, 3.0, 96.5ms, 0 | 20260505 20:56:43.703 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:43.703 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:43.703 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 20:56:43.703 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:43.703 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:43.703 | INFO | Uploaded 36GiB to GPU in 2.1315s (16.8895GiB/s) | 20260505 20:56:43.704 | INFO | Downloaded 34GiB to GPU in 2.01885s (16.8413GiB/s) | 20260505 20:56:48.842 | INFO | Analyzing chunk frames = [6247936,6248448) | 20260505 20:56:48.842 | INFO | MemInfo: { | 20260505 20:56:48.842 | INFO | "gpu" : " 17 GB", | 20260505 20:56:48.842 | INFO | "huge" : "", | 20260505 20:56:48.842 | INFO | "huge_shared" : " 27 GB", | 20260505 20:56:48.842 | INFO | "malloc" : " 14 GB", | 20260505 20:56:48.842 | INFO | "pinned" : " 29 GB" | 20260505 20:56:48.842 | INFO | } | 20260505 20:56:48.843 | INFO | Perf report for the last 5.13933 seconds: | 20260505 20:56:48.843 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:56:48.843 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:48.843 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:48.843 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:48.843 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260505 20:56:48.843 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:48.843 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:48.843 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:56:48.843 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:48.843 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:48.843 | INFO | Uploaded 36GiB to GPU in 2.21139s (16.2794GiB/s) | 20260505 20:56:48.843 | INFO | Downloaded 34GiB to GPU in 2.07298s (16.4015GiB/s) | 20260505 20:56:53.978 | INFO | Analyzing chunk frames = [6248448,6248960) | 20260505 20:56:53.978 | INFO | MemInfo: { | 20260505 20:56:53.978 | INFO | "gpu" : " 17 GB", | 20260505 20:56:53.978 | INFO | "huge" : "", | 20260505 20:56:53.978 | INFO | "huge_shared" : " 27 GB", | 20260505 20:56:53.978 | INFO | "malloc" : " 14 GB", | 20260505 20:56:53.978 | INFO | "pinned" : " 29 GB" | 20260505 20:56:53.978 | INFO | } | 20260505 20:56:53.978 | INFO | Perf report for the last 5.13527 seconds: | 20260505 20:56:53.978 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:56:53.978 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:53.978 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:53.978 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:53.978 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260505 20:56:53.978 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:53.978 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:53.978 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 20:56:53.978 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:53.978 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:53.978 | INFO | Uploaded 36GiB to GPU in 2.14099s (16.8146GiB/s) | 20260505 20:56:53.978 | INFO | Downloaded 34GiB to GPU in 2.0558s (16.5386GiB/s) | 20260505 20:56:59.115 | INFO | Analyzing chunk frames = [6248960,6249472) | 20260505 20:56:59.115 | INFO | MemInfo: { | 20260505 20:56:59.115 | INFO | "gpu" : " 17 GB", | 20260505 20:56:59.115 | INFO | "huge" : "", | 20260505 20:56:59.115 | INFO | "huge_shared" : " 27 GB", | 20260505 20:56:59.115 | INFO | "malloc" : " 14 GB", | 20260505 20:56:59.115 | INFO | "pinned" : " 29 GB" | 20260505 20:56:59.115 | INFO | } | 20260505 20:56:59.115 | INFO | Perf report for the last 5.13717 seconds: | 20260505 20:56:59.115 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:56:59.115 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:59.115 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:59.115 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:59.115 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 20:56:59.115 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:59.115 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:59.115 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 20:56:59.115 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:59.115 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:56:59.115 | INFO | Uploaded 36GiB to GPU in 2.15552s (16.7013GiB/s) | 20260505 20:56:59.115 | INFO | Downloaded 34GiB to GPU in 2.06329s (16.4785GiB/s) | 20260505 20:57:04.254 | INFO | Analyzing chunk frames = [6249472,6249984) | 20260505 20:57:04.254 | INFO | MemInfo: { | 20260505 20:57:04.254 | INFO | "gpu" : " 17 GB", | 20260505 20:57:04.254 | INFO | "huge" : "", | 20260505 20:57:04.254 | INFO | "huge_shared" : " 27 GB", | 20260505 20:57:04.254 | INFO | "malloc" : " 14 GB", | 20260505 20:57:04.254 | INFO | "pinned" : " 29 GB" | 20260505 20:57:04.254 | INFO | } | 20260505 20:57:04.255 | INFO | Perf report for the last 5.13952 seconds: | 20260505 20:57:04.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:57:04.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:04.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:04.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:04.255 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 20:57:04.255 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:04.255 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:04.255 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 20:57:04.255 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:04.255 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:04.255 | INFO | Uploaded 36GiB to GPU in 2.12757s (16.9208GiB/s) | 20260505 20:57:04.255 | INFO | Downloaded 34GiB to GPU in 2.13629s (15.9155GiB/s) | 20260505 20:57:09.401 | INFO | Analyzing chunk frames = [6249984,6250496) | 20260505 20:57:09.401 | INFO | MemInfo: { | 20260505 20:57:09.401 | INFO | "gpu" : " 17 GB", | 20260505 20:57:09.401 | INFO | "huge" : "", | 20260505 20:57:09.401 | INFO | "huge_shared" : " 27 GB", | 20260505 20:57:09.401 | INFO | "malloc" : " 14 GB", | 20260505 20:57:09.401 | INFO | "pinned" : " 29 GB" | 20260505 20:57:09.401 | INFO | } | 20260505 20:57:09.401 | INFO | Perf report for the last 5.1469 seconds: | 20260505 20:57:09.401 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:57:09.401 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:09.401 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:09.401 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:09.402 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 20:57:09.402 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:09.402 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:09.402 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:57:09.402 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:09.402 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:09.402 | INFO | Uploaded 36GiB to GPU in 2.20424s (16.3322GiB/s) | 20260505 20:57:09.402 | INFO | Downloaded 34GiB to GPU in 2.04804s (16.6012GiB/s) | 20260505 20:57:09.402 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 20:57:09.402 | INFO | Total mem usage: 15.3 GiB | 20260505 20:57:09.402 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 20:57:09.402 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.13%) | 20260505 20:57:09.402 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 20:57:09.402 | INFO | High Water Breakdown: | 20260505 20:57:09.402 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 20:57:09.402 | INFO | 38.8% : 5.5 GiB ( 82.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 20:57:09.402 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 20:57:09.402 | INFO | ------------------------------------------ | 20260505 20:57:09.402 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:57:09.402 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 20:57:09.402 | INFO | Total mem usage: 26.5 GiB | 20260505 20:57:09.402 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 20:57:09.402 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 20:57:09.402 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 20:57:09.402 | INFO | High Water Breakdown: | 20260505 20:57:09.402 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:57:09.402 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:57:09.402 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:57:09.402 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:57:09.402 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:57:09.402 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:57:09.402 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 20:57:09.402 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 20:57:09.402 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:57:09.402 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 20:57:09.402 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 20:57:09.402 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 20:57:09.402 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 20:57:09.402 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 20:57:09.402 | INFO | ------------------------------------------ | 20260505 20:57:09.402 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:57:09.402 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 20:57:09.402 | INFO | Total mem usage: 25.6 GiB | 20260505 20:57:09.402 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 20:57:09.402 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 20:57:09.402 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 20:57:09.402 | INFO | High Water Breakdown: | 20260505 20:57:09.402 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 20:57:09.402 | INFO | ------------------------------------------ | 20260505 20:57:09.402 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:57:09.402 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 20:57:09.402 | INFO | Total mem usage: 15.7 GiB | 20260505 20:57:09.402 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 20:57:09.402 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 20:57:09.402 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 20:57:09.402 | INFO | High Water Breakdown: | 20260505 20:57:09.402 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 20:57:09.402 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 20:57:09.402 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 20:57:09.402 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 20:57:09.402 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 20:57:09.403 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 20:57:09.403 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 20:57:09.403 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 20:57:09.403 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 20:57:09.403 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:57:09.403 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 20:57:09.403 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:57:09.403 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 20:57:09.403 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 20:57:09.403 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 20:57:09.403 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 20:57:09.403 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:57:09.403 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 20:57:09.403 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:57:09.403 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:57:09.403 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:57:09.403 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 20:57:09.403 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 20:57:09.403 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:57:09.403 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 20:57:09.403 | INFO | ------------------------------------------ | 20260505 20:57:09.403 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:57:09.403 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 20:57:09.403 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:57:09.403 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:57:09.403 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.47e+01, 1.36e+01 / 2.13e+00 | 20260505 20:57:09.403 | INFO | PerfReport: 0.29%, AggregateStats -- 9.26e-02 / 1.04e-01, 1.07e-02 / 6.51e-03 | 20260505 20:57:09.403 | INFO | PerfReport: 11.79%, Baseline -- 3.70e+00 / 4.51e+00, 5.62e-01 / 1.90e-01 | 20260505 20:57:09.403 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.52e+00, 1.90e-01 / 8.78e-02 | 20260505 20:57:09.403 | INFO | PerfReport: 15.98%, DME -- 4.02e+01 / 4.45e+01, 4.14e+00 / 2.22e+00 | 20260505 20:57:09.403 | INFO | PerfReport: 57.08%, FrameLabeling -- 1.79e+01 / 1.94e+01, 1.46e+00 / 1.05e+00 | 20260505 20:57:09.403 | INFO | PerfReport: 7.23%, PulseAccumulating -- 2.27e+00 / 2.84e+00, 4.36e-01 / 2.12e-01 | 20260505 20:57:09.403 | INFO | PerfReport: 3.22%, Metrics -- 1.01e+00 / 1.34e+00, 2.75e-01 / 1.43e-01 | 20260505 20:57:09.403 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.52e-01 / 4.20e-01, 6.17e-02 / 3.06e-02 | 20260505 20:57:09.403 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 20:57:09.403 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:57:09.403 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:57:09.403 | INFO | PerfReport: Full Execution -- 9.93e+01 / -nan, 2.54e+01 / 0.00e+00 | 20260505 20:57:09.403 | INFO | PerfReport: 22.50%, Upload -- 2.23e+01 / 0.00e+00, 7.77e+00 / 0.00e+00 | 20260505 20:57:09.403 | INFO | PerfReport: 21.82%, Download -- 2.17e+01 / 0.00e+00, 7.70e+00 / 0.00e+00 | 20260505 20:57:09.403 | INFO | PerfReport: 31.65%, Compute -- 3.14e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 20:57:09.403 | INFO | PerfReport: 2.29%, LockUpload -- 2.27e+00 / 0.00e+00, 8.07e+00 / 0.00e+00 | 20260505 20:57:09.403 | INFO | PerfReport: 1.36%, LockDownload -- 1.35e+00 / 0.00e+00, 2.86e+00 / 0.00e+00 | 20260505 20:57:09.403 | INFO | PerfReport: 20.37%, LockCompute -- 2.02e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 20:57:09.403 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.24e-02 / 0.00e+00 | 20260505 20:57:09.536 | INFO | Starting a round of Baz IO | 20260505 20:57:12.693 | INFO | Populating RTMetrics report | 20260505 20:57:12.693 | INFO | RTMetrics report populated | 20260505 20:57:12.693 | INFO | Opening tmp RTMetrics json file | 20260505 20:57:12.695 | INFO | Serializing RTMetrics to json | 20260505 20:57:12.696 | INFO | Serialization done, swapping tmp file now... | 20260505 20:57:12.697 | INFO | RTMetrics done | 20260505 20:57:12.782 | INFO | Finished a round of Baz IO: Wrote 1.991 GiB in 3.246 seconds | 20260505 20:57:12.782 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.62% / 0.00% / 78.38% | 20260505 20:57:12.782 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.160503% / 0% / 6.75519e-05% | 20260505 20:57:12.782 | INFO | Baz IO overall: | 20260505 20:57:12.782 | INFO | BytesWritten : 3631511152996 | 20260505 20:57:12.782 | INFO | NumEvents : 1715764786151 | 20260505 20:57:14.547 | INFO | Analyzing chunk frames = [6250496,6251008) | 20260505 20:57:14.547 | INFO | MemInfo: { | 20260505 20:57:14.547 | INFO | "gpu" : " 17 GB", | 20260505 20:57:14.547 | INFO | "huge" : "", | 20260505 20:57:14.547 | INFO | "huge_shared" : " 27 GB", | 20260505 20:57:14.547 | INFO | "malloc" : " 14 GB", | 20260505 20:57:14.547 | INFO | "pinned" : " 29 GB" | 20260505 20:57:14.547 | INFO | } | 20260505 20:57:14.547 | INFO | Perf report for the last 5.14599 seconds: | 20260505 20:57:14.547 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:57:14.548 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:14.548 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:14.548 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:14.548 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 20:57:14.548 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:14.548 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260505 20:57:14.548 | INFO | PRE_HQ : 60.0%, 1.8, 55.3ms, 0 | 20260505 20:57:14.548 | INFO | BAZWRITER : 62.6%, 3.7, 119.7ms, 0 | 20260505 20:57:14.548 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 20:57:14.548 | INFO | Uploaded 36GiB to GPU in 2.16635s (16.6178GiB/s) | 20260505 20:57:14.548 | INFO | Downloaded 38GiB to GPU in 2.24706s (16.911GiB/s) | 20260505 20:57:19.677 | INFO | Analyzing chunk frames = [6251008,6251520) | 20260505 20:57:19.677 | INFO | MemInfo: { | 20260505 20:57:19.677 | INFO | "gpu" : " 17 GB", | 20260505 20:57:19.677 | INFO | "huge" : "", | 20260505 20:57:19.677 | INFO | "huge_shared" : " 27 GB", | 20260505 20:57:19.677 | INFO | "malloc" : " 14 GB", | 20260505 20:57:19.677 | INFO | "pinned" : " 29 GB" | 20260505 20:57:19.677 | INFO | } | 20260505 20:57:19.678 | INFO | Perf report for the last 5.13005 seconds: | 20260505 20:57:19.678 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:57:19.678 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:19.678 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:19.678 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:19.678 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260505 20:57:19.678 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:19.678 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:19.678 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:57:19.678 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:19.678 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:19.678 | INFO | Uploaded 36GiB to GPU in 2.07542s (17.3459GiB/s) | 20260505 20:57:19.678 | INFO | Downloaded 34GiB to GPU in 2.02659s (16.7769GiB/s) | 20260505 20:57:24.809 | INFO | Analyzing chunk frames = [6251520,6252032) | 20260505 20:57:24.809 | INFO | MemInfo: { | 20260505 20:57:24.809 | INFO | "gpu" : " 17 GB", | 20260505 20:57:24.809 | INFO | "huge" : "", | 20260505 20:57:24.809 | INFO | "huge_shared" : " 27 GB", | 20260505 20:57:24.809 | INFO | "malloc" : " 14 GB", | 20260505 20:57:24.809 | INFO | "pinned" : " 29 GB" | 20260505 20:57:24.809 | INFO | } | 20260505 20:57:24.809 | INFO | Perf report for the last 5.13124 seconds: | 20260505 20:57:24.809 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:57:24.809 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:24.809 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:24.809 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:24.809 | INFO | ANALYSIS : 62.2%, 3.0, 96.6ms, 0 | 20260505 20:57:24.809 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:24.809 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:24.809 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 20:57:24.809 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:24.809 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:24.809 | INFO | Uploaded 36GiB to GPU in 2.11948s (16.9853GiB/s) | 20260505 20:57:24.809 | INFO | Downloaded 34GiB to GPU in 2.02215s (16.8138GiB/s) | 20260505 20:57:29.956 | INFO | Analyzing chunk frames = [6252032,6252544) | 20260505 20:57:29.956 | INFO | MemInfo: { | 20260505 20:57:29.956 | INFO | "gpu" : " 17 GB", | 20260505 20:57:29.956 | INFO | "huge" : "", | 20260505 20:57:29.956 | INFO | "huge_shared" : " 27 GB", | 20260505 20:57:29.956 | INFO | "malloc" : " 14 GB", | 20260505 20:57:29.956 | INFO | "pinned" : " 29 GB" | 20260505 20:57:29.956 | INFO | } | 20260505 20:57:29.956 | INFO | Perf report for the last 5.14705 seconds: | 20260505 20:57:29.956 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:57:29.956 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:29.956 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:29.956 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:29.956 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260505 20:57:29.956 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:29.956 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:29.956 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:57:29.956 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:29.956 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:29.956 | INFO | Uploaded 36GiB to GPU in 2.20329s (16.3392GiB/s) | 20260505 20:57:29.956 | INFO | Downloaded 34GiB to GPU in 2.06035s (16.5021GiB/s) | 20260505 20:57:35.098 | INFO | Analyzing chunk frames = [6252544,6253056) | 20260505 20:57:35.098 | INFO | MemInfo: { | 20260505 20:57:35.098 | INFO | "gpu" : " 17 GB", | 20260505 20:57:35.098 | INFO | "huge" : "", | 20260505 20:57:35.098 | INFO | "huge_shared" : " 27 GB", | 20260505 20:57:35.098 | INFO | "malloc" : " 14 GB", | 20260505 20:57:35.098 | INFO | "pinned" : " 29 GB" | 20260505 20:57:35.098 | INFO | } | 20260505 20:57:35.098 | INFO | Perf report for the last 5.14198 seconds: | 20260505 20:57:35.098 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:57:35.098 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:35.098 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:35.098 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:35.098 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260505 20:57:35.098 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:35.098 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:35.098 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 20:57:35.098 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:35.098 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:35.098 | INFO | Uploaded 36GiB to GPU in 2.13447s (16.866GiB/s) | 20260505 20:57:35.098 | INFO | Downloaded 34GiB to GPU in 2.06493s (16.4655GiB/s) | 20260505 20:57:40.237 | INFO | Analyzing chunk frames = [6253056,6253568) | 20260505 20:57:40.237 | INFO | MemInfo: { | 20260505 20:57:40.237 | INFO | "gpu" : " 17 GB", | 20260505 20:57:40.237 | INFO | "huge" : "", | 20260505 20:57:40.237 | INFO | "huge_shared" : " 27 GB", | 20260505 20:57:40.237 | INFO | "malloc" : " 14 GB", | 20260505 20:57:40.237 | INFO | "pinned" : " 29 GB" | 20260505 20:57:40.237 | INFO | } | 20260505 20:57:40.237 | INFO | Perf report for the last 5.13956 seconds: | 20260505 20:57:40.237 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:57:40.237 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:40.237 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:40.237 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:40.237 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260505 20:57:40.237 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:40.237 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:40.237 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:57:40.237 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:40.238 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:40.238 | INFO | Uploaded 36GiB to GPU in 2.08218s (17.2896GiB/s) | 20260505 20:57:40.238 | INFO | Downloaded 34GiB to GPU in 2.03974s (16.6688GiB/s) | 20260505 20:57:45.376 | INFO | Analyzing chunk frames = [6253568,6254080) | 20260505 20:57:45.376 | INFO | MemInfo: { | 20260505 20:57:45.376 | INFO | "gpu" : " 17 GB", | 20260505 20:57:45.376 | INFO | "huge" : "", | 20260505 20:57:45.376 | INFO | "huge_shared" : " 27 GB", | 20260505 20:57:45.376 | INFO | "malloc" : " 14 GB", | 20260505 20:57:45.376 | INFO | "pinned" : " 29 GB" | 20260505 20:57:45.376 | INFO | } | 20260505 20:57:45.376 | INFO | Perf report for the last 5.13856 seconds: | 20260505 20:57:45.376 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:57:45.376 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:45.376 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:45.376 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:45.376 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 20:57:45.376 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:45.376 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:45.376 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 20:57:45.376 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:45.376 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:45.376 | INFO | Uploaded 36GiB to GPU in 2.14173s (16.8088GiB/s) | 20260505 20:57:45.376 | INFO | Downloaded 34GiB to GPU in 2.16465s (15.7069GiB/s) | 20260505 20:57:50.516 | INFO | Analyzing chunk frames = [6254080,6254592) | 20260505 20:57:50.516 | INFO | MemInfo: { | 20260505 20:57:50.516 | INFO | "gpu" : " 17 GB", | 20260505 20:57:50.516 | INFO | "huge" : "", | 20260505 20:57:50.516 | INFO | "huge_shared" : " 27 GB", | 20260505 20:57:50.516 | INFO | "malloc" : " 14 GB", | 20260505 20:57:50.516 | INFO | "pinned" : " 29 GB" | 20260505 20:57:50.516 | INFO | } | 20260505 20:57:50.516 | INFO | Perf report for the last 5.14025 seconds: | 20260505 20:57:50.516 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:57:50.516 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:50.516 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:50.516 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:50.516 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260505 20:57:50.516 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:50.516 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:50.516 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:57:50.516 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:50.516 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:50.516 | INFO | Uploaded 36GiB to GPU in 2.11869s (16.9916GiB/s) | 20260505 20:57:50.516 | INFO | Downloaded 34GiB to GPU in 2.01404s (16.8815GiB/s) | 20260505 20:57:50.651 | INFO | Starting a round of Baz IO | 20260505 20:57:53.815 | INFO | Populating RTMetrics report | 20260505 20:57:53.815 | INFO | RTMetrics report populated | 20260505 20:57:53.815 | INFO | Opening tmp RTMetrics json file | 20260505 20:57:53.819 | INFO | Serializing RTMetrics to json | 20260505 20:57:53.822 | INFO | Serialization done, swapping tmp file now... | 20260505 20:57:53.822 | INFO | RTMetrics done | 20260505 20:57:53.905 | INFO | Finished a round of Baz IO: Wrote 1.99 GiB in 3.254 seconds | 20260505 20:57:53.906 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.62% / 0.00% / 78.38% | 20260505 20:57:53.906 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.160162% / 0% / 5.96046e-05% | 20260505 20:57:53.906 | INFO | Baz IO overall: | 20260505 20:57:53.906 | INFO | BytesWritten : 3633648131482 | 20260505 20:57:53.906 | INFO | NumEvents : 1716752845408 | 20260505 20:57:55.656 | INFO | Analyzing chunk frames = [6254592,6255104) | 20260505 20:57:55.656 | INFO | MemInfo: { | 20260505 20:57:55.656 | INFO | "gpu" : " 17 GB", | 20260505 20:57:55.656 | INFO | "huge" : "", | 20260505 20:57:55.656 | INFO | "huge_shared" : " 27 GB", | 20260505 20:57:55.656 | INFO | "malloc" : " 14 GB", | 20260505 20:57:55.656 | INFO | "pinned" : " 29 GB" | 20260505 20:57:55.656 | INFO | } | 20260505 20:57:55.656 | INFO | Perf report for the last 5.1395 seconds: | 20260505 20:57:55.656 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:57:55.656 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:55.656 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:55.656 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:55.656 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 20:57:55.656 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:57:55.656 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260505 20:57:55.656 | INFO | PRE_HQ : 60.2%, 1.8, 55.5ms, 0 | 20260505 20:57:55.656 | INFO | BAZWRITER : 62.8%, 3.7, 119.9ms, 0 | 20260505 20:57:55.656 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 20:57:55.656 | INFO | Uploaded 36GiB to GPU in 2.1747s (16.554GiB/s) | 20260505 20:57:55.656 | INFO | Downloaded 38GiB to GPU in 2.24873s (16.8984GiB/s) | 20260505 20:58:00.791 | INFO | Analyzing chunk frames = [6255104,6255616) | 20260505 20:58:00.791 | INFO | MemInfo: { | 20260505 20:58:00.791 | INFO | "gpu" : " 17 GB", | 20260505 20:58:00.791 | INFO | "huge" : "", | 20260505 20:58:00.791 | INFO | "huge_shared" : " 27 GB", | 20260505 20:58:00.791 | INFO | "malloc" : " 14 GB", | 20260505 20:58:00.791 | INFO | "pinned" : " 29 GB" | 20260505 20:58:00.791 | INFO | } | 20260505 20:58:00.791 | INFO | Perf report for the last 5.13564 seconds: | 20260505 20:58:00.791 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:58:00.791 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:00.791 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:00.791 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:00.791 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 20:58:00.791 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:00.791 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:00.792 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 20:58:00.792 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:00.792 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:00.792 | INFO | Uploaded 36GiB to GPU in 2.0872s (17.248GiB/s) | 20260505 20:58:00.792 | INFO | Downloaded 34GiB to GPU in 2.02619s (16.7803GiB/s) | 20260505 20:58:05.929 | INFO | Analyzing chunk frames = [6255616,6256128) | 20260505 20:58:05.929 | INFO | MemInfo: { | 20260505 20:58:05.929 | INFO | "gpu" : " 17 GB", | 20260505 20:58:05.929 | INFO | "huge" : "", | 20260505 20:58:05.929 | INFO | "huge_shared" : " 27 GB", | 20260505 20:58:05.929 | INFO | "malloc" : " 14 GB", | 20260505 20:58:05.929 | INFO | "pinned" : " 29 GB" | 20260505 20:58:05.929 | INFO | } | 20260505 20:58:05.929 | INFO | Perf report for the last 5.13782 seconds: | 20260505 20:58:05.929 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:58:05.929 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:05.929 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:05.929 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:05.929 | INFO | ANALYSIS : 62.1%, 3.0, 96.5ms, 0 | 20260505 20:58:05.929 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:05.929 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:05.929 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:58:05.929 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:05.929 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:05.930 | INFO | Uploaded 36GiB to GPU in 2.12719s (16.9238GiB/s) | 20260505 20:58:05.930 | INFO | Downloaded 34GiB to GPU in 2.01803s (16.8481GiB/s) | 20260505 20:58:11.065 | INFO | Analyzing chunk frames = [6256128,6256640) | 20260505 20:58:11.065 | INFO | MemInfo: { | 20260505 20:58:11.065 | INFO | "gpu" : " 17 GB", | 20260505 20:58:11.065 | INFO | "huge" : "", | 20260505 20:58:11.065 | INFO | "huge_shared" : " 27 GB", | 20260505 20:58:11.065 | INFO | "malloc" : " 14 GB", | 20260505 20:58:11.065 | INFO | "pinned" : " 29 GB" | 20260505 20:58:11.065 | INFO | } | 20260505 20:58:11.065 | INFO | Perf report for the last 5.13618 seconds: | 20260505 20:58:11.065 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:58:11.065 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:11.065 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:11.065 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:11.065 | INFO | ANALYSIS : 62.3%, 3.0, 96.7ms, 0 | 20260505 20:58:11.065 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:11.065 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:11.065 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:58:11.066 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:11.066 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:11.066 | INFO | Uploaded 36GiB to GPU in 2.20717s (16.3105GiB/s) | 20260505 20:58:11.066 | INFO | Downloaded 34GiB to GPU in 2.04064s (16.6614GiB/s) | 20260505 20:58:16.212 | INFO | Analyzing chunk frames = [6256640,6257152) | 20260505 20:58:16.212 | INFO | MemInfo: { | 20260505 20:58:16.212 | INFO | "gpu" : " 17 GB", | 20260505 20:58:16.212 | INFO | "huge" : "", | 20260505 20:58:16.212 | INFO | "huge_shared" : " 27 GB", | 20260505 20:58:16.212 | INFO | "malloc" : " 14 GB", | 20260505 20:58:16.213 | INFO | "pinned" : " 29 GB" | 20260505 20:58:16.213 | INFO | } | 20260505 20:58:16.213 | INFO | Perf report for the last 5.14724 seconds: | 20260505 20:58:16.213 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:58:16.213 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:16.213 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:16.213 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:16.213 | INFO | ANALYSIS : 61.6%, 3.0, 96.4ms, 0 | 20260505 20:58:16.213 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:16.213 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:16.213 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260505 20:58:16.213 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:16.213 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:16.213 | INFO | Uploaded 36GiB to GPU in 2.14191s (16.8074GiB/s) | 20260505 20:58:16.213 | INFO | Downloaded 34GiB to GPU in 2.06221s (16.4872GiB/s) | 20260505 20:58:21.355 | INFO | Analyzing chunk frames = [6257152,6257664) | 20260505 20:58:21.355 | INFO | MemInfo: { | 20260505 20:58:21.355 | INFO | "gpu" : " 17 GB", | 20260505 20:58:21.355 | INFO | "huge" : "", | 20260505 20:58:21.355 | INFO | "huge_shared" : " 27 GB", | 20260505 20:58:21.355 | INFO | "malloc" : " 14 GB", | 20260505 20:58:21.355 | INFO | "pinned" : " 29 GB" | 20260505 20:58:21.355 | INFO | } | 20260505 20:58:21.355 | INFO | Perf report for the last 5.14261 seconds: | 20260505 20:58:21.355 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:58:21.355 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:21.355 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:21.355 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:21.355 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260505 20:58:21.355 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:21.355 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:21.355 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:58:21.355 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:21.355 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:21.356 | INFO | Uploaded 36GiB to GPU in 2.07787s (17.3254GiB/s) | 20260505 20:58:21.356 | INFO | Downloaded 34GiB to GPU in 2.03268s (16.7267GiB/s) | 20260505 20:58:26.493 | INFO | Analyzing chunk frames = [6257664,6258176) | 20260505 20:58:26.493 | INFO | MemInfo: { | 20260505 20:58:26.493 | INFO | "gpu" : " 17 GB", | 20260505 20:58:26.493 | INFO | "huge" : "", | 20260505 20:58:26.493 | INFO | "huge_shared" : " 27 GB", | 20260505 20:58:26.493 | INFO | "malloc" : " 14 GB", | 20260505 20:58:26.493 | INFO | "pinned" : " 29 GB" | 20260505 20:58:26.493 | INFO | } | 20260505 20:58:26.493 | INFO | Perf report for the last 5.13755 seconds: | 20260505 20:58:26.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:58:26.493 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:26.493 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:26.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:26.493 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 20:58:26.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:26.493 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:26.493 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 20:58:26.493 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:26.493 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:26.493 | INFO | Uploaded 36GiB to GPU in 2.15891s (16.6751GiB/s) | 20260505 20:58:26.493 | INFO | Downloaded 34GiB to GPU in 2.12325s (16.0132GiB/s) | 20260505 20:58:31.625 | INFO | Analyzing chunk frames = [6258176,6258688) | 20260505 20:58:31.625 | INFO | MemInfo: { | 20260505 20:58:31.625 | INFO | "gpu" : " 17 GB", | 20260505 20:58:31.625 | INFO | "huge" : "", | 20260505 20:58:31.625 | INFO | "huge_shared" : " 27 GB", | 20260505 20:58:31.625 | INFO | "malloc" : " 14 GB", | 20260505 20:58:31.625 | INFO | "pinned" : " 29 GB" | 20260505 20:58:31.625 | INFO | } | 20260505 20:58:31.625 | INFO | Perf report for the last 5.13236 seconds: | 20260505 20:58:31.625 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:58:31.625 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:31.625 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:31.625 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:31.625 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260505 20:58:31.625 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:31.625 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:31.625 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:58:31.625 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:31.625 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:31.626 | INFO | Uploaded 36GiB to GPU in 2.10243s (17.1231GiB/s) | 20260505 20:58:31.626 | INFO | Downloaded 34GiB to GPU in 1.99866s (17.0114GiB/s) | 20260505 20:58:31.626 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 20:58:31.626 | INFO | Total mem usage: 15.3 GiB | 20260505 20:58:31.626 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 20:58:31.626 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.14%) | 20260505 20:58:31.626 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 20:58:31.626 | INFO | High Water Breakdown: | 20260505 20:58:31.626 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 20:58:31.626 | INFO | 38.8% : 5.5 GiB ( 82.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 20:58:31.626 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 20:58:31.626 | INFO | ------------------------------------------ | 20260505 20:58:31.626 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:58:31.626 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 20:58:31.626 | INFO | Total mem usage: 26.5 GiB | 20260505 20:58:31.626 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 20:58:31.626 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 20:58:31.626 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 20:58:31.626 | INFO | High Water Breakdown: | 20260505 20:58:31.626 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:58:31.626 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:58:31.626 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:58:31.626 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:58:31.626 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:58:31.626 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:58:31.626 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 20:58:31.626 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 20:58:31.626 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:58:31.626 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 20:58:31.626 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 20:58:31.626 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 20:58:31.626 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 20:58:31.626 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 20:58:31.626 | INFO | ------------------------------------------ | 20260505 20:58:31.626 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:58:31.626 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 20:58:31.626 | INFO | Total mem usage: 25.6 GiB | 20260505 20:58:31.626 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 20:58:31.626 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 20:58:31.626 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 20:58:31.626 | INFO | High Water Breakdown: | 20260505 20:58:31.626 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 20:58:31.626 | INFO | ------------------------------------------ | 20260505 20:58:31.626 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:58:31.626 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 20:58:31.626 | INFO | Total mem usage: 15.7 GiB | 20260505 20:58:31.626 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 20:58:31.626 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 20:58:31.626 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 20:58:31.626 | INFO | High Water Breakdown: | 20260505 20:58:31.626 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 20:58:31.626 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 20:58:31.626 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 20:58:31.626 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 20:58:31.626 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 20:58:31.626 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 20:58:31.627 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 20:58:31.627 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 20:58:31.627 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 20:58:31.627 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:58:31.627 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 20:58:31.627 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:58:31.627 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 20:58:31.627 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 20:58:31.627 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 20:58:31.627 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 20:58:31.627 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:58:31.627 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 20:58:31.627 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:58:31.627 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:58:31.627 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:58:31.627 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 20:58:31.627 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 20:58:31.627 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:58:31.627 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 20:58:31.627 | INFO | ------------------------------------------ | 20260505 20:58:31.627 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:58:31.627 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 20:58:31.627 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:58:31.627 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:58:31.627 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.43e+01, 1.35e+01 / 1.23e+00 | 20260505 20:58:31.627 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 1.03e-01, 1.09e-02 / 4.15e-03 | 20260505 20:58:31.627 | INFO | PerfReport: 11.81%, Baseline -- 3.72e+00 / 4.58e+00, 5.72e-01 / 6.64e-02 | 20260505 20:58:31.627 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.54e+00, 1.91e-01 / 5.61e-02 | 20260505 20:58:31.627 | INFO | PerfReport: 15.93%, DME -- 4.01e+01 / 4.41e+01, 4.03e+00 / 1.24e+00 | 20260505 20:58:31.627 | INFO | PerfReport: 57.12%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.50e+00 / 8.07e-01 | 20260505 20:58:31.627 | INFO | PerfReport: 7.20%, PulseAccumulating -- 2.27e+00 / 2.83e+00, 4.35e-01 / 1.40e-01 | 20260505 20:58:31.627 | INFO | PerfReport: 3.23%, Metrics -- 1.02e+00 / 1.33e+00, 2.76e-01 / 1.38e-01 | 20260505 20:58:31.627 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.53e-01 / 4.16e-01, 6.23e-02 / 4.09e-02 | 20260505 20:58:31.627 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 20:58:31.627 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:58:31.627 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:58:31.627 | INFO | PerfReport: Full Execution -- 9.91e+01 / -nan, 2.53e+01 / 0.00e+00 | 20260505 20:58:31.627 | INFO | PerfReport: 22.41%, Upload -- 2.22e+01 / 0.00e+00, 7.65e+00 / 0.00e+00 | 20260505 20:58:31.627 | INFO | PerfReport: 21.80%, Download -- 2.16e+01 / 0.00e+00, 7.72e+00 / 0.00e+00 | 20260505 20:58:31.627 | INFO | PerfReport: 31.74%, Compute -- 3.15e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 20:58:31.627 | INFO | PerfReport: 2.19%, LockUpload -- 2.17e+00 / 0.00e+00, 7.84e+00 / 0.00e+00 | 20260505 20:58:31.627 | INFO | PerfReport: 1.39%, LockDownload -- 1.38e+00 / 0.00e+00, 2.93e+00 / 0.00e+00 | 20260505 20:58:31.627 | INFO | PerfReport: 20.45%, LockCompute -- 2.03e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 20:58:31.627 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.84e-02 / 0.00e+00 | 20260505 20:58:31.754 | INFO | Starting a round of Baz IO | 20260505 20:58:34.926 | INFO | Populating RTMetrics report | 20260505 20:58:34.926 | INFO | RTMetrics report populated | 20260505 20:58:34.926 | INFO | Opening tmp RTMetrics json file | 20260505 20:58:34.927 | INFO | Serializing RTMetrics to json | 20260505 20:58:34.931 | INFO | Serialization done, swapping tmp file now... | 20260505 20:58:34.931 | INFO | RTMetrics done | 20260505 20:58:35.013 | INFO | Finished a round of Baz IO: Wrote 1.989 GiB in 3.259 seconds | 20260505 20:58:35.013 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.62% / 0.00% / 78.38% | 20260505 20:58:35.013 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.159494% / 3.97364e-06% / 6.35783e-05% | 20260505 20:58:35.013 | INFO | Baz IO overall: | 20260505 20:58:35.013 | INFO | BytesWritten : 3635784245240 | 20260505 20:58:35.013 | INFO | NumEvents : 1717740423963 | 20260505 20:58:36.761 | INFO | Analyzing chunk frames = [6258688,6259200) | 20260505 20:58:36.761 | INFO | MemInfo: { | 20260505 20:58:36.761 | INFO | "gpu" : " 17 GB", | 20260505 20:58:36.761 | INFO | "huge" : "", | 20260505 20:58:36.761 | INFO | "huge_shared" : " 27 GB", | 20260505 20:58:36.761 | INFO | "malloc" : " 14 GB", | 20260505 20:58:36.761 | INFO | "pinned" : " 29 GB" | 20260505 20:58:36.761 | INFO | } | 20260505 20:58:36.761 | INFO | Perf report for the last 5.13623 seconds: | 20260505 20:58:36.762 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:58:36.762 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:36.762 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:36.762 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:36.762 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 20:58:36.762 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:36.762 | INFO | RT_METRICS : 44.0%, 1.0, 23.0ms, 0 | 20260505 20:58:36.762 | INFO | PRE_HQ : 60.1%, 1.8, 55.0ms, 0 | 20260505 20:58:36.762 | INFO | BAZWRITER : 63.0%, 3.7, 119.5ms, 0 | 20260505 20:58:36.762 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 20:58:36.762 | INFO | Uploaded 36GiB to GPU in 2.16493s (16.6287GiB/s) | 20260505 20:58:36.762 | INFO | Downloaded 38GiB to GPU in 2.24324s (16.9398GiB/s) | 20260505 20:58:41.907 | INFO | Analyzing chunk frames = [6259200,6259712) | 20260505 20:58:41.907 | INFO | MemInfo: { | 20260505 20:58:41.907 | INFO | "gpu" : " 17 GB", | 20260505 20:58:41.907 | INFO | "huge" : "", | 20260505 20:58:41.907 | INFO | "huge_shared" : " 27 GB", | 20260505 20:58:41.907 | INFO | "malloc" : " 14 GB", | 20260505 20:58:41.907 | INFO | "pinned" : " 29 GB" | 20260505 20:58:41.907 | INFO | } | 20260505 20:58:41.907 | INFO | Perf report for the last 5.14576 seconds: | 20260505 20:58:41.907 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:58:41.907 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:41.907 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:41.907 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:41.907 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 20:58:41.907 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:41.907 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:41.907 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 20:58:41.907 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:41.907 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:41.908 | INFO | Uploaded 36GiB to GPU in 2.09952s (17.1468GiB/s) | 20260505 20:58:41.908 | INFO | Downloaded 34GiB to GPU in 2.02706s (16.773GiB/s) | 20260505 20:58:47.048 | INFO | Analyzing chunk frames = [6259712,6260224) | 20260505 20:58:47.048 | INFO | MemInfo: { | 20260505 20:58:47.048 | INFO | "gpu" : " 17 GB", | 20260505 20:58:47.048 | INFO | "huge" : "", | 20260505 20:58:47.048 | INFO | "huge_shared" : " 27 GB", | 20260505 20:58:47.049 | INFO | "malloc" : " 14 GB", | 20260505 20:58:47.049 | INFO | "pinned" : " 29 GB" | 20260505 20:58:47.049 | INFO | } | 20260505 20:58:47.049 | INFO | Perf report for the last 5.14147 seconds: | 20260505 20:58:47.049 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:58:47.049 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:47.049 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:47.049 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:47.049 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260505 20:58:47.049 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:47.049 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:47.049 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:58:47.049 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:47.049 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:47.049 | INFO | Uploaded 36GiB to GPU in 2.13578s (16.8557GiB/s) | 20260505 20:58:47.049 | INFO | Downloaded 34GiB to GPU in 2.01452s (16.8775GiB/s) | 20260505 20:58:52.184 | INFO | Analyzing chunk frames = [6260224,6260736) | 20260505 20:58:52.184 | INFO | MemInfo: { | 20260505 20:58:52.184 | INFO | "gpu" : " 17 GB", | 20260505 20:58:52.184 | INFO | "huge" : "", | 20260505 20:58:52.184 | INFO | "huge_shared" : " 27 GB", | 20260505 20:58:52.184 | INFO | "malloc" : " 14 GB", | 20260505 20:58:52.184 | INFO | "pinned" : " 29 GB" | 20260505 20:58:52.184 | INFO | } | 20260505 20:58:52.184 | INFO | Perf report for the last 5.13572 seconds: | 20260505 20:58:52.184 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:58:52.184 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:52.185 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:52.185 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:52.185 | INFO | ANALYSIS : 62.0%, 3.0, 96.3ms, 0 | 20260505 20:58:52.185 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:52.185 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:52.185 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 20:58:52.185 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:52.185 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:52.185 | INFO | Uploaded 36GiB to GPU in 2.13641s (16.8507GiB/s) | 20260505 20:58:52.185 | INFO | Downloaded 34GiB to GPU in 2.04023s (16.6648GiB/s) | 20260505 20:58:57.321 | INFO | Analyzing chunk frames = [6260736,6261248) | 20260505 20:58:57.321 | INFO | MemInfo: { | 20260505 20:58:57.321 | INFO | "gpu" : " 17 GB", | 20260505 20:58:57.321 | INFO | "huge" : "", | 20260505 20:58:57.321 | INFO | "huge_shared" : " 27 GB", | 20260505 20:58:57.321 | INFO | "malloc" : " 14 GB", | 20260505 20:58:57.321 | INFO | "pinned" : " 29 GB" | 20260505 20:58:57.321 | INFO | } | 20260505 20:58:57.321 | INFO | Perf report for the last 5.13656 seconds: | 20260505 20:58:57.321 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:58:57.321 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:57.321 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:57.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:57.321 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260505 20:58:57.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:57.321 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:57.321 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 20:58:57.321 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:57.321 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:58:57.321 | INFO | Uploaded 36GiB to GPU in 2.13779s (16.8398GiB/s) | 20260505 20:58:57.321 | INFO | Downloaded 34GiB to GPU in 2.06032s (16.5023GiB/s) | 20260505 20:59:02.468 | INFO | Analyzing chunk frames = [6261248,6261760) | 20260505 20:59:02.468 | INFO | MemInfo: { | 20260505 20:59:02.468 | INFO | "gpu" : " 17 GB", | 20260505 20:59:02.468 | INFO | "huge" : "", | 20260505 20:59:02.468 | INFO | "huge_shared" : " 27 GB", | 20260505 20:59:02.468 | INFO | "malloc" : " 14 GB", | 20260505 20:59:02.468 | INFO | "pinned" : " 29 GB" | 20260505 20:59:02.468 | INFO | } | 20260505 20:59:02.468 | INFO | Perf report for the last 5.14691 seconds: | 20260505 20:59:02.468 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:59:02.468 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:02.468 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:02.468 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:02.468 | INFO | ANALYSIS : 62.1%, 3.0, 97.1ms, 0 | 20260505 20:59:02.468 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:02.468 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:02.468 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 20:59:02.468 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:02.468 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:02.468 | INFO | Uploaded 36GiB to GPU in 2.15467s (16.7079GiB/s) | 20260505 20:59:02.468 | INFO | Downloaded 34GiB to GPU in 2.06146s (16.4932GiB/s) | 20260505 20:59:07.607 | INFO | Analyzing chunk frames = [6261760,6262272) | 20260505 20:59:07.607 | INFO | MemInfo: { | 20260505 20:59:07.607 | INFO | "gpu" : " 17 GB", | 20260505 20:59:07.607 | INFO | "huge" : "", | 20260505 20:59:07.607 | INFO | "huge_shared" : " 27 GB", | 20260505 20:59:07.607 | INFO | "malloc" : " 14 GB", | 20260505 20:59:07.607 | INFO | "pinned" : " 29 GB" | 20260505 20:59:07.607 | INFO | } | 20260505 20:59:07.608 | INFO | Perf report for the last 5.13957 seconds: | 20260505 20:59:07.608 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:59:07.608 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:07.608 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:07.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:07.608 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 20:59:07.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:07.608 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:07.608 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 20:59:07.608 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:07.608 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:07.608 | INFO | Uploaded 36GiB to GPU in 2.14948s (16.7483GiB/s) | 20260505 20:59:07.608 | INFO | Downloaded 34GiB to GPU in 2.10823s (16.1273GiB/s) | 20260505 20:59:12.742 | INFO | Analyzing chunk frames = [6262272,6262784) | 20260505 20:59:12.742 | INFO | MemInfo: { | 20260505 20:59:12.742 | INFO | "gpu" : " 17 GB", | 20260505 20:59:12.742 | INFO | "huge" : "", | 20260505 20:59:12.742 | INFO | "huge_shared" : " 27 GB", | 20260505 20:59:12.742 | INFO | "malloc" : " 14 GB", | 20260505 20:59:12.742 | INFO | "pinned" : " 29 GB" | 20260505 20:59:12.742 | INFO | } | 20260505 20:59:12.742 | INFO | Perf report for the last 5.13462 seconds: | 20260505 20:59:12.742 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:59:12.742 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:12.742 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:12.742 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:12.742 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260505 20:59:12.742 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:12.742 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:12.742 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:59:12.742 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:12.742 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:12.742 | INFO | Uploaded 36GiB to GPU in 2.09611s (17.1747GiB/s) | 20260505 20:59:12.742 | INFO | Downloaded 34GiB to GPU in 1.98627s (17.1175GiB/s) | 20260505 20:59:12.877 | INFO | Starting a round of Baz IO | 20260505 20:59:16.032 | INFO | Populating RTMetrics report | 20260505 20:59:16.032 | INFO | RTMetrics report populated | 20260505 20:59:16.032 | INFO | Opening tmp RTMetrics json file | 20260505 20:59:16.033 | INFO | Serializing RTMetrics to json | 20260505 20:59:16.034 | INFO | Serialization done, swapping tmp file now... | 20260505 20:59:16.034 | INFO | RTMetrics done | 20260505 20:59:16.118 | INFO | Finished a round of Baz IO: Wrote 1.988 GiB in 3.24 seconds | 20260505 20:59:16.118 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.62% / 0.00% / 78.38% | 20260505 20:59:16.118 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.15986% / 3.97364e-06% / 8.74201e-05% | 20260505 20:59:16.118 | INFO | Baz IO overall: | 20260505 20:59:16.118 | INFO | BytesWritten : 3637918599903 | 20260505 20:59:16.118 | INFO | NumEvents : 1718726789680 | 20260505 20:59:17.881 | INFO | Analyzing chunk frames = [6262784,6263296) | 20260505 20:59:17.881 | INFO | MemInfo: { | 20260505 20:59:17.881 | INFO | "gpu" : " 17 GB", | 20260505 20:59:17.881 | INFO | "huge" : "", | 20260505 20:59:17.881 | INFO | "huge_shared" : " 27 GB", | 20260505 20:59:17.881 | INFO | "malloc" : " 14 GB", | 20260505 20:59:17.881 | INFO | "pinned" : " 29 GB" | 20260505 20:59:17.881 | INFO | } | 20260505 20:59:17.881 | INFO | Perf report for the last 5.1387 seconds: | 20260505 20:59:17.881 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:59:17.881 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:17.881 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:17.881 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:17.881 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 20:59:17.881 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:17.881 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260505 20:59:17.881 | INFO | PRE_HQ : 60.0%, 1.8, 55.2ms, 0 | 20260505 20:59:17.881 | INFO | BAZWRITER : 62.7%, 3.7, 119.1ms, 0 | 20260505 20:59:17.881 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 20:59:17.881 | INFO | Uploaded 36GiB to GPU in 2.17487s (16.5527GiB/s) | 20260505 20:59:17.881 | INFO | Downloaded 38GiB to GPU in 2.25833s (16.8266GiB/s) | 20260505 20:59:23.019 | INFO | Analyzing chunk frames = [6263296,6263808) | 20260505 20:59:23.019 | INFO | MemInfo: { | 20260505 20:59:23.019 | INFO | "gpu" : " 17 GB", | 20260505 20:59:23.019 | INFO | "huge" : "", | 20260505 20:59:23.019 | INFO | "huge_shared" : " 27 GB", | 20260505 20:59:23.019 | INFO | "malloc" : " 14 GB", | 20260505 20:59:23.019 | INFO | "pinned" : " 29 GB" | 20260505 20:59:23.019 | INFO | } | 20260505 20:59:23.019 | INFO | Perf report for the last 5.13795 seconds: | 20260505 20:59:23.019 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:59:23.019 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:23.019 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:23.019 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:23.019 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 20:59:23.019 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:23.019 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:23.019 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 20:59:23.019 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:23.019 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:23.019 | INFO | Uploaded 36GiB to GPU in 2.15501s (16.7052GiB/s) | 20260505 20:59:23.019 | INFO | Downloaded 34GiB to GPU in 2.05065s (16.5801GiB/s) | 20260505 20:59:28.162 | INFO | Analyzing chunk frames = [6263808,6264320) | 20260505 20:59:28.162 | INFO | MemInfo: { | 20260505 20:59:28.162 | INFO | "gpu" : " 17 GB", | 20260505 20:59:28.162 | INFO | "huge" : "", | 20260505 20:59:28.162 | INFO | "huge_shared" : " 27 GB", | 20260505 20:59:28.162 | INFO | "malloc" : " 14 GB", | 20260505 20:59:28.162 | INFO | "pinned" : " 29 GB" | 20260505 20:59:28.162 | INFO | } | 20260505 20:59:28.162 | INFO | Perf report for the last 5.14352 seconds: | 20260505 20:59:28.162 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:59:28.162 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:28.162 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:28.162 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:28.162 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260505 20:59:28.162 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:28.163 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:28.163 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 20:59:28.163 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:28.163 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:28.163 | INFO | Uploaded 36GiB to GPU in 2.14445s (16.7875GiB/s) | 20260505 20:59:28.163 | INFO | Downloaded 34GiB to GPU in 2.02465s (16.793GiB/s) | 20260505 20:59:33.305 | INFO | Analyzing chunk frames = [6264320,6264832) | 20260505 20:59:33.305 | INFO | MemInfo: { | 20260505 20:59:33.305 | INFO | "gpu" : " 17 GB", | 20260505 20:59:33.305 | INFO | "huge" : "", | 20260505 20:59:33.305 | INFO | "huge_shared" : " 27 GB", | 20260505 20:59:33.305 | INFO | "malloc" : " 14 GB", | 20260505 20:59:33.305 | INFO | "pinned" : " 29 GB" | 20260505 20:59:33.305 | INFO | } | 20260505 20:59:33.305 | INFO | Perf report for the last 5.14271 seconds: | 20260505 20:59:33.305 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:59:33.305 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:33.305 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:33.305 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:33.305 | INFO | ANALYSIS : 62.1%, 3.0, 96.5ms, 0 | 20260505 20:59:33.305 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:33.305 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:33.305 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 20:59:33.305 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:33.305 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:33.305 | INFO | Uploaded 36GiB to GPU in 2.13963s (16.8253GiB/s) | 20260505 20:59:33.306 | INFO | Downloaded 34GiB to GPU in 2.04926s (16.5914GiB/s) | 20260505 20:59:38.434 | INFO | Analyzing chunk frames = [6264832,6265344) | 20260505 20:59:38.434 | INFO | MemInfo: { | 20260505 20:59:38.434 | INFO | "gpu" : " 17 GB", | 20260505 20:59:38.434 | INFO | "huge" : "", | 20260505 20:59:38.434 | INFO | "huge_shared" : " 27 GB", | 20260505 20:59:38.435 | INFO | "malloc" : " 14 GB", | 20260505 20:59:38.435 | INFO | "pinned" : " 29 GB" | 20260505 20:59:38.435 | INFO | } | 20260505 20:59:38.435 | INFO | Perf report for the last 5.1296 seconds: | 20260505 20:59:38.435 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:59:38.435 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:38.435 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:38.435 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:38.435 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260505 20:59:38.435 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:38.435 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:38.435 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 20:59:38.435 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:38.435 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:38.435 | INFO | Uploaded 36GiB to GPU in 2.1378s (16.8398GiB/s) | 20260505 20:59:38.435 | INFO | Downloaded 34GiB to GPU in 2.05694s (16.5294GiB/s) | 20260505 20:59:43.573 | INFO | Analyzing chunk frames = [6265344,6265856) | 20260505 20:59:43.573 | INFO | MemInfo: { | 20260505 20:59:43.573 | INFO | "gpu" : " 17 GB", | 20260505 20:59:43.573 | INFO | "huge" : "", | 20260505 20:59:43.573 | INFO | "huge_shared" : " 27 GB", | 20260505 20:59:43.573 | INFO | "malloc" : " 14 GB", | 20260505 20:59:43.573 | INFO | "pinned" : " 29 GB" | 20260505 20:59:43.573 | INFO | } | 20260505 20:59:43.573 | INFO | Perf report for the last 5.13829 seconds: | 20260505 20:59:43.573 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:59:43.573 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:43.573 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:43.573 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:43.573 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 20:59:43.573 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:43.573 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:43.573 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 20:59:43.573 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:43.573 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:43.573 | INFO | Uploaded 36GiB to GPU in 2.15968s (16.6692GiB/s) | 20260505 20:59:43.573 | INFO | Downloaded 34GiB to GPU in 2.06343s (16.4774GiB/s) | 20260505 20:59:48.718 | INFO | Analyzing chunk frames = [6265856,6266368) | 20260505 20:59:48.718 | INFO | MemInfo: { | 20260505 20:59:48.718 | INFO | "gpu" : " 17 GB", | 20260505 20:59:48.718 | INFO | "huge" : "", | 20260505 20:59:48.718 | INFO | "huge_shared" : " 27 GB", | 20260505 20:59:48.718 | INFO | "malloc" : " 14 GB", | 20260505 20:59:48.718 | INFO | "pinned" : " 29 GB" | 20260505 20:59:48.718 | INFO | } | 20260505 20:59:48.718 | INFO | Perf report for the last 5.14522 seconds: | 20260505 20:59:48.718 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:59:48.718 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:48.718 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:48.718 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:48.718 | INFO | ANALYSIS : 62.9%, 3.0, 98.4ms, 0 | 20260505 20:59:48.718 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:48.718 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:48.718 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260505 20:59:48.718 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:48.718 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:48.719 | INFO | Uploaded 36GiB to GPU in 2.13049s (16.8975GiB/s) | 20260505 20:59:48.719 | INFO | Downloaded 34GiB to GPU in 2.1532s (15.7905GiB/s) | 20260505 20:59:53.856 | INFO | Analyzing chunk frames = [6266368,6266880) | 20260505 20:59:53.856 | INFO | MemInfo: { | 20260505 20:59:53.856 | INFO | "gpu" : " 17 GB", | 20260505 20:59:53.856 | INFO | "huge" : "", | 20260505 20:59:53.856 | INFO | "huge_shared" : " 27 GB", | 20260505 20:59:53.856 | INFO | "malloc" : " 14 GB", | 20260505 20:59:53.856 | INFO | "pinned" : " 29 GB" | 20260505 20:59:53.857 | INFO | } | 20260505 20:59:53.857 | INFO | Perf report for the last 5.13841 seconds: | 20260505 20:59:53.857 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:59:53.857 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:53.857 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:53.857 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:53.857 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 20:59:53.857 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:53.857 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:53.857 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 20:59:53.857 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:53.857 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:53.857 | INFO | Uploaded 36GiB to GPU in 2.10933s (17.067GiB/s) | 20260505 20:59:53.857 | INFO | Downloaded 34GiB to GPU in 2.01503s (16.8732GiB/s) | 20260505 20:59:53.857 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 20:59:53.857 | INFO | Total mem usage: 15.3 GiB | 20260505 20:59:53.857 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 20:59:53.857 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.97%) | 20260505 20:59:53.857 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 20:59:53.857 | INFO | High Water Breakdown: | 20260505 20:59:53.857 | INFO | 60.9% : 8.57 GiB ( 90% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 20:59:53.857 | INFO | 38.9% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 20:59:53.857 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 20:59:53.857 | INFO | ------------------------------------------ | 20260505 20:59:53.857 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:59:53.857 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 20:59:53.857 | INFO | Total mem usage: 26.5 GiB | 20260505 20:59:53.857 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 20:59:53.857 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 20:59:53.857 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 20:59:53.857 | INFO | High Water Breakdown: | 20260505 20:59:53.857 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:59:53.857 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:59:53.857 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:59:53.857 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:59:53.857 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:59:53.857 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:59:53.857 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 20:59:53.857 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 20:59:53.857 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:59:53.857 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 20:59:53.857 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 20:59:53.857 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 20:59:53.857 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 20:59:53.858 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 20:59:53.858 | INFO | ------------------------------------------ | 20260505 20:59:53.858 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:59:53.858 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 20:59:53.858 | INFO | Total mem usage: 25.6 GiB | 20260505 20:59:53.858 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 20:59:53.858 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 20:59:53.858 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 20:59:53.858 | INFO | High Water Breakdown: | 20260505 20:59:53.858 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 20:59:53.858 | INFO | ------------------------------------------ | 20260505 20:59:53.858 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:59:53.858 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 20:59:53.858 | INFO | Total mem usage: 15.7 GiB | 20260505 20:59:53.858 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 20:59:53.858 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 20:59:53.858 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 20:59:53.858 | INFO | High Water Breakdown: | 20260505 20:59:53.858 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 20:59:53.858 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 20:59:53.858 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 20:59:53.858 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 20:59:53.858 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 20:59:53.858 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 20:59:53.858 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 20:59:53.858 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 20:59:53.858 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 20:59:53.858 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 20:59:53.858 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 20:59:53.858 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 20:59:53.858 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 20:59:53.858 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 20:59:53.858 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 20:59:53.858 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 20:59:53.858 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 20:59:53.858 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 20:59:53.858 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 20:59:53.858 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 20:59:53.858 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 20:59:53.858 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 20:59:53.858 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 20:59:53.858 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 20:59:53.858 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 20:59:53.858 | INFO | ------------------------------------------ | 20260505 20:59:53.858 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 20:59:53.858 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 20:59:53.858 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:59:53.858 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:59:53.858 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.43e+01, 1.35e+01 / 1.47e+00 | 20260505 20:59:53.858 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.04e-01, 1.05e-02 / 6.13e-03 | 20260505 20:59:53.858 | INFO | PerfReport: 11.77%, Baseline -- 3.70e+00 / 4.56e+00, 5.63e-01 / 1.28e-01 | 20260505 20:59:53.858 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.50e+00, 1.87e-01 / 1.03e-01 | 20260505 20:59:53.858 | INFO | PerfReport: 15.90%, DME -- 4.00e+01 / 4.40e+01, 4.01e+00 / 1.45e+00 | 20260505 20:59:53.858 | INFO | PerfReport: 57.23%, FrameLabeling -- 1.80e+01 / 1.96e+01, 1.57e+00 / 1.13e+00 | 20260505 20:59:53.858 | INFO | PerfReport: 7.20%, PulseAccumulating -- 2.26e+00 / 2.80e+00, 4.32e-01 / 1.85e-01 | 20260505 20:59:53.858 | INFO | PerfReport: 3.21%, Metrics -- 1.01e+00 / 1.34e+00, 2.79e-01 / 1.59e-01 | 20260505 20:59:53.858 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.52e-01 / 4.12e-01, 6.14e-02 / 3.87e-02 | 20260505 20:59:53.858 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 20:59:53.858 | INFO | PerfReport: Gathered 1536 samples. | 20260505 20:59:53.858 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 20:59:53.858 | INFO | PerfReport: Full Execution -- 9.92e+01 / -nan, 2.54e+01 / 0.00e+00 | 20260505 20:59:53.858 | INFO | PerfReport: 22.47%, Upload -- 2.23e+01 / 0.00e+00, 7.76e+00 / 0.00e+00 | 20260505 20:59:53.858 | INFO | PerfReport: 21.80%, Download -- 2.16e+01 / 0.00e+00, 7.71e+00 / 0.00e+00 | 20260505 20:59:53.858 | INFO | PerfReport: 31.73%, Compute -- 3.15e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 20:59:53.858 | INFO | PerfReport: 2.24%, LockUpload -- 2.22e+00 / 0.00e+00, 7.98e+00 / 0.00e+00 | 20260505 20:59:53.858 | INFO | PerfReport: 1.33%, LockDownload -- 1.32e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260505 20:59:53.858 | INFO | PerfReport: 20.42%, LockCompute -- 2.03e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 20:59:53.858 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.58e-02 / 0.00e+00 | 20260505 20:59:53.990 | INFO | Starting a round of Baz IO | 20260505 20:59:57.153 | INFO | Populating RTMetrics report | 20260505 20:59:57.154 | INFO | RTMetrics report populated | 20260505 20:59:57.154 | INFO | Opening tmp RTMetrics json file | 20260505 20:59:57.155 | INFO | Serializing RTMetrics to json | 20260505 20:59:57.156 | INFO | Serialization done, swapping tmp file now... | 20260505 20:59:57.156 | INFO | RTMetrics done | 20260505 20:59:57.244 | INFO | Finished a round of Baz IO: Wrote 1.988 GiB in 3.254 seconds | 20260505 20:59:57.244 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.62% / 0.00% / 78.38% | 20260505 20:59:57.244 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.159653% / 0% / 7.15256e-05% | 20260505 20:59:57.244 | INFO | Baz IO overall: | 20260505 20:59:57.244 | INFO | BytesWritten : 3640053672136 | 20260505 20:59:57.244 | INFO | NumEvents : 1719713578906 | 20260505 20:59:58.996 | INFO | Analyzing chunk frames = [6266880,6267392) | 20260505 20:59:58.996 | INFO | MemInfo: { | 20260505 20:59:58.996 | INFO | "gpu" : " 17 GB", | 20260505 20:59:58.996 | INFO | "huge" : "", | 20260505 20:59:58.996 | INFO | "huge_shared" : " 27 GB", | 20260505 20:59:58.996 | INFO | "malloc" : " 14 GB", | 20260505 20:59:58.996 | INFO | "pinned" : " 29 GB" | 20260505 20:59:58.996 | INFO | } | 20260505 20:59:58.996 | INFO | Perf report for the last 5.13904 seconds: | 20260505 20:59:58.996 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 20:59:58.996 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:58.996 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:58.996 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:58.996 | INFO | ANALYSIS : 63.1%, 3.0, 98.3ms, 0 | 20260505 20:59:58.996 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 20:59:58.996 | INFO | RT_METRICS : 44.0%, 1.0, 23.0ms, 0 | 20260505 20:59:58.996 | INFO | PRE_HQ : 60.0%, 1.8, 55.2ms, 0 | 20260505 20:59:58.996 | INFO | BAZWRITER : 62.8%, 3.7, 119.2ms, 0 | 20260505 20:59:58.996 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 20:59:58.996 | INFO | Uploaded 36GiB to GPU in 2.16836s (16.6024GiB/s) | 20260505 20:59:58.996 | INFO | Downloaded 38GiB to GPU in 2.25378s (16.8606GiB/s) | 20260505 21:00:04.135 | INFO | Analyzing chunk frames = [6267392,6267904) | 20260505 21:00:04.135 | INFO | MemInfo: { | 20260505 21:00:04.135 | INFO | "gpu" : " 17 GB", | 20260505 21:00:04.135 | INFO | "huge" : "", | 20260505 21:00:04.135 | INFO | "huge_shared" : " 27 GB", | 20260505 21:00:04.135 | INFO | "malloc" : " 14 GB", | 20260505 21:00:04.135 | INFO | "pinned" : " 29 GB" | 20260505 21:00:04.135 | INFO | } | 20260505 21:00:04.136 | INFO | Perf report for the last 5.1398 seconds: | 20260505 21:00:04.136 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:00:04.136 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:04.136 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:04.136 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:04.136 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 21:00:04.136 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:04.136 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:04.136 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 21:00:04.136 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:04.136 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:04.136 | INFO | Uploaded 36GiB to GPU in 2.10634s (17.0913GiB/s) | 20260505 21:00:04.136 | INFO | Downloaded 34GiB to GPU in 2.0251s (16.7893GiB/s) | 20260505 21:00:09.274 | INFO | Analyzing chunk frames = [6267904,6268416) | 20260505 21:00:09.274 | INFO | MemInfo: { | 20260505 21:00:09.274 | INFO | "gpu" : " 17 GB", | 20260505 21:00:09.274 | INFO | "huge" : "", | 20260505 21:00:09.274 | INFO | "huge_shared" : " 27 GB", | 20260505 21:00:09.274 | INFO | "malloc" : " 14 GB", | 20260505 21:00:09.275 | INFO | "pinned" : " 29 GB" | 20260505 21:00:09.275 | INFO | } | 20260505 21:00:09.275 | INFO | Perf report for the last 5.13902 seconds: | 20260505 21:00:09.275 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:00:09.275 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:09.275 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:09.275 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:09.275 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260505 21:00:09.275 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:09.275 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:09.275 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:00:09.275 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:09.275 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:09.275 | INFO | Uploaded 36GiB to GPU in 2.13537s (16.8589GiB/s) | 20260505 21:00:09.275 | INFO | Downloaded 34GiB to GPU in 2.0286s (16.7603GiB/s) | 20260505 21:00:14.417 | INFO | Analyzing chunk frames = [6268416,6268928) | 20260505 21:00:14.417 | INFO | MemInfo: { | 20260505 21:00:14.417 | INFO | "gpu" : " 17 GB", | 20260505 21:00:14.417 | INFO | "huge" : "", | 20260505 21:00:14.417 | INFO | "huge_shared" : " 27 GB", | 20260505 21:00:14.417 | INFO | "malloc" : " 14 GB", | 20260505 21:00:14.417 | INFO | "pinned" : " 29 GB" | 20260505 21:00:14.417 | INFO | } | 20260505 21:00:14.418 | INFO | Perf report for the last 5.14298 seconds: | 20260505 21:00:14.418 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:00:14.418 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:14.418 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:14.418 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:14.418 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260505 21:00:14.418 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:14.418 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:14.418 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 21:00:14.418 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:14.418 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:14.418 | INFO | Uploaded 36GiB to GPU in 2.20713s (16.3108GiB/s) | 20260505 21:00:14.418 | INFO | Downloaded 34GiB to GPU in 2.04457s (16.6294GiB/s) | 20260505 21:00:19.561 | INFO | Analyzing chunk frames = [6268928,6269440) | 20260505 21:00:19.561 | INFO | MemInfo: { | 20260505 21:00:19.561 | INFO | "gpu" : " 17 GB", | 20260505 21:00:19.561 | INFO | "huge" : "", | 20260505 21:00:19.561 | INFO | "huge_shared" : " 27 GB", | 20260505 21:00:19.561 | INFO | "malloc" : " 14 GB", | 20260505 21:00:19.561 | INFO | "pinned" : " 29 GB" | 20260505 21:00:19.561 | INFO | } | 20260505 21:00:19.561 | INFO | Perf report for the last 5.14334 seconds: | 20260505 21:00:19.561 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:00:19.561 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:19.561 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:19.561 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:19.561 | INFO | ANALYSIS : 61.6%, 3.0, 96.2ms, 0 | 20260505 21:00:19.561 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:19.561 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:19.561 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 21:00:19.561 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:19.561 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:19.561 | INFO | Uploaded 36GiB to GPU in 2.13632s (16.8514GiB/s) | 20260505 21:00:19.561 | INFO | Downloaded 34GiB to GPU in 2.0632s (16.4792GiB/s) | 20260505 21:00:24.693 | INFO | Analyzing chunk frames = [6269440,6269952) | 20260505 21:00:24.693 | INFO | MemInfo: { | 20260505 21:00:24.693 | INFO | "gpu" : " 17 GB", | 20260505 21:00:24.693 | INFO | "huge" : "", | 20260505 21:00:24.693 | INFO | "huge_shared" : " 27 GB", | 20260505 21:00:24.693 | INFO | "malloc" : " 14 GB", | 20260505 21:00:24.693 | INFO | "pinned" : " 29 GB" | 20260505 21:00:24.693 | INFO | } | 20260505 21:00:24.693 | INFO | Perf report for the last 5.13249 seconds: | 20260505 21:00:24.693 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:00:24.693 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:24.693 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:24.693 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:24.693 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260505 21:00:24.693 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:24.693 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:24.694 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:00:24.694 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:24.694 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:24.694 | INFO | Uploaded 36GiB to GPU in 2.08958s (17.2284GiB/s) | 20260505 21:00:24.694 | INFO | Downloaded 34GiB to GPU in 2.04018s (16.6652GiB/s) | 20260505 21:00:29.832 | INFO | Analyzing chunk frames = [6269952,6270464) | 20260505 21:00:29.832 | INFO | MemInfo: { | 20260505 21:00:29.832 | INFO | "gpu" : " 17 GB", | 20260505 21:00:29.832 | INFO | "huge" : "", | 20260505 21:00:29.832 | INFO | "huge_shared" : " 27 GB", | 20260505 21:00:29.832 | INFO | "malloc" : " 14 GB", | 20260505 21:00:29.832 | INFO | "pinned" : " 29 GB" | 20260505 21:00:29.832 | INFO | } | 20260505 21:00:29.832 | INFO | Perf report for the last 5.13884 seconds: | 20260505 21:00:29.832 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:00:29.832 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:29.832 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:29.832 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:29.832 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 21:00:29.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:29.832 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:29.832 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 21:00:29.832 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:29.832 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:29.833 | INFO | Uploaded 36GiB to GPU in 2.13902s (16.8301GiB/s) | 20260505 21:00:29.833 | INFO | Downloaded 34GiB to GPU in 2.15263s (15.7946GiB/s) | 20260505 21:00:34.973 | INFO | Analyzing chunk frames = [6270464,6270976) | 20260505 21:00:34.973 | INFO | MemInfo: { | 20260505 21:00:34.973 | INFO | "gpu" : " 17 GB", | 20260505 21:00:34.973 | INFO | "huge" : "", | 20260505 21:00:34.973 | INFO | "huge_shared" : " 27 GB", | 20260505 21:00:34.973 | INFO | "malloc" : " 14 GB", | 20260505 21:00:34.973 | INFO | "pinned" : " 29 GB" | 20260505 21:00:34.973 | INFO | } | 20260505 21:00:34.974 | INFO | Perf report for the last 5.14123 seconds: | 20260505 21:00:34.974 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:00:34.974 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:34.974 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:34.974 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:34.974 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260505 21:00:34.974 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:34.974 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:34.974 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 21:00:34.974 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:34.974 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:34.974 | INFO | Uploaded 36GiB to GPU in 2.12466s (16.9439GiB/s) | 20260505 21:00:34.974 | INFO | Downloaded 34GiB to GPU in 2.01499s (16.8736GiB/s) | 20260505 21:00:35.105 | INFO | Starting a round of Baz IO | 20260505 21:00:38.268 | INFO | Populating RTMetrics report | 20260505 21:00:38.268 | INFO | RTMetrics report populated | 20260505 21:00:38.268 | INFO | Opening tmp RTMetrics json file | 20260505 21:00:38.269 | INFO | Serializing RTMetrics to json | 20260505 21:00:38.270 | INFO | Serialization done, swapping tmp file now... | 20260505 21:00:38.270 | INFO | RTMetrics done | 20260505 21:00:38.353 | INFO | Finished a round of Baz IO: Wrote 1.987 GiB in 3.248 seconds | 20260505 21:00:38.354 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.61% / 0.00% / 78.38% | 20260505 21:00:38.354 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.158262% / 0% / 7.54992e-05% | 20260505 21:00:38.354 | INFO | Baz IO overall: | 20260505 21:00:38.354 | INFO | BytesWritten : 3642187688372 | 20260505 21:00:38.354 | INFO | NumEvents : 1720699627338 | 20260505 21:00:40.115 | INFO | Analyzing chunk frames = [6270976,6271488) | 20260505 21:00:40.115 | INFO | MemInfo: { | 20260505 21:00:40.115 | INFO | "gpu" : " 17 GB", | 20260505 21:00:40.115 | INFO | "huge" : "", | 20260505 21:00:40.115 | INFO | "huge_shared" : " 27 GB", | 20260505 21:00:40.115 | INFO | "malloc" : " 14 GB", | 20260505 21:00:40.115 | INFO | "pinned" : " 29 GB" | 20260505 21:00:40.115 | INFO | } | 20260505 21:00:40.115 | INFO | Perf report for the last 5.14134 seconds: | 20260505 21:00:40.115 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:00:40.115 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:40.115 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:40.115 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:40.115 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 21:00:40.115 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:40.115 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260505 21:00:40.115 | INFO | PRE_HQ : 60.4%, 1.8, 55.3ms, 0 | 20260505 21:00:40.115 | INFO | BAZWRITER : 62.7%, 3.7, 120.2ms, 0 | 20260505 21:00:40.115 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 21:00:40.115 | INFO | Uploaded 36GiB to GPU in 2.17795s (16.5293GiB/s) | 20260505 21:00:40.115 | INFO | Downloaded 38GiB to GPU in 2.23901s (16.9718GiB/s) | 20260505 21:00:45.250 | INFO | Analyzing chunk frames = [6271488,6272000) | 20260505 21:00:45.250 | INFO | MemInfo: { | 20260505 21:00:45.250 | INFO | "gpu" : " 17 GB", | 20260505 21:00:45.250 | INFO | "huge" : "", | 20260505 21:00:45.250 | INFO | "huge_shared" : " 27 GB", | 20260505 21:00:45.250 | INFO | "malloc" : " 14 GB", | 20260505 21:00:45.250 | INFO | "pinned" : " 29 GB" | 20260505 21:00:45.250 | INFO | } | 20260505 21:00:45.250 | INFO | Perf report for the last 5.13541 seconds: | 20260505 21:00:45.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:00:45.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:45.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:45.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:45.250 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260505 21:00:45.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:45.250 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:45.250 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260505 21:00:45.250 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:45.250 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:45.251 | INFO | Uploaded 36GiB to GPU in 2.07683s (17.3341GiB/s) | 20260505 21:00:45.251 | INFO | Downloaded 34GiB to GPU in 2.02162s (16.8182GiB/s) | 20260505 21:00:50.383 | INFO | Analyzing chunk frames = [6272000,6272512) | 20260505 21:00:50.383 | INFO | MemInfo: { | 20260505 21:00:50.383 | INFO | "gpu" : " 17 GB", | 20260505 21:00:50.383 | INFO | "huge" : "", | 20260505 21:00:50.383 | INFO | "huge_shared" : " 27 GB", | 20260505 21:00:50.383 | INFO | "malloc" : " 14 GB", | 20260505 21:00:50.383 | INFO | "pinned" : " 29 GB" | 20260505 21:00:50.383 | INFO | } | 20260505 21:00:50.383 | INFO | Perf report for the last 5.13296 seconds: | 20260505 21:00:50.383 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:00:50.383 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:50.383 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:50.383 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:50.383 | INFO | ANALYSIS : 62.1%, 3.0, 96.5ms, 0 | 20260505 21:00:50.383 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:50.383 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:50.383 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:00:50.383 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:50.383 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:50.384 | INFO | Uploaded 36GiB to GPU in 2.13341s (16.8744GiB/s) | 20260505 21:00:50.384 | INFO | Downloaded 34GiB to GPU in 2.01442s (16.8783GiB/s) | 20260505 21:00:55.528 | INFO | Analyzing chunk frames = [6272512,6273024) | 20260505 21:00:55.528 | INFO | MemInfo: { | 20260505 21:00:55.528 | INFO | "gpu" : " 17 GB", | 20260505 21:00:55.528 | INFO | "huge" : "", | 20260505 21:00:55.528 | INFO | "huge_shared" : " 27 GB", | 20260505 21:00:55.528 | INFO | "malloc" : " 14 GB", | 20260505 21:00:55.528 | INFO | "pinned" : " 29 GB" | 20260505 21:00:55.528 | INFO | } | 20260505 21:00:55.528 | INFO | Perf report for the last 5.14515 seconds: | 20260505 21:00:55.528 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:00:55.528 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:55.528 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:55.528 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:55.529 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260505 21:00:55.529 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:55.529 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:55.529 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 21:00:55.529 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:55.529 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:00:55.529 | INFO | Uploaded 36GiB to GPU in 2.1885s (16.4496GiB/s) | 20260505 21:00:55.529 | INFO | Downloaded 34GiB to GPU in 2.06023s (16.503GiB/s) | 20260505 21:01:00.671 | INFO | Analyzing chunk frames = [6273024,6273536) | 20260505 21:01:00.671 | INFO | MemInfo: { | 20260505 21:01:00.671 | INFO | "gpu" : " 17 GB", | 20260505 21:01:00.671 | INFO | "huge" : "", | 20260505 21:01:00.671 | INFO | "huge_shared" : " 27 GB", | 20260505 21:01:00.671 | INFO | "malloc" : " 14 GB", | 20260505 21:01:00.671 | INFO | "pinned" : " 29 GB" | 20260505 21:01:00.671 | INFO | } | 20260505 21:01:00.672 | INFO | Perf report for the last 5.14328 seconds: | 20260505 21:01:00.672 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:01:00.672 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:00.672 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:00.672 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:00.672 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 21:01:00.672 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:00.672 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:00.672 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 21:01:00.672 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:00.672 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:00.672 | INFO | Uploaded 36GiB to GPU in 2.1334s (16.8745GiB/s) | 20260505 21:01:00.672 | INFO | Downloaded 34GiB to GPU in 2.05945s (16.5092GiB/s) | 20260505 21:01:05.816 | INFO | Analyzing chunk frames = [6273536,6274048) | 20260505 21:01:05.816 | INFO | MemInfo: { | 20260505 21:01:05.816 | INFO | "gpu" : " 17 GB", | 20260505 21:01:05.816 | INFO | "huge" : "", | 20260505 21:01:05.816 | INFO | "huge_shared" : " 27 GB", | 20260505 21:01:05.816 | INFO | "malloc" : " 14 GB", | 20260505 21:01:05.816 | INFO | "pinned" : " 29 GB" | 20260505 21:01:05.816 | INFO | } | 20260505 21:01:05.816 | INFO | Perf report for the last 5.14436 seconds: | 20260505 21:01:05.816 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:01:05.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:05.816 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:05.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:05.816 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 21:01:05.816 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:05.816 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:05.816 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 21:01:05.816 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:05.816 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:05.817 | INFO | Uploaded 36GiB to GPU in 2.09692s (17.168GiB/s) | 20260505 21:01:05.817 | INFO | Downloaded 34GiB to GPU in 2.03866s (16.6777GiB/s) | 20260505 21:01:10.946 | INFO | Analyzing chunk frames = [6274048,6274560) | 20260505 21:01:10.946 | INFO | MemInfo: { | 20260505 21:01:10.946 | INFO | "gpu" : " 17 GB", | 20260505 21:01:10.946 | INFO | "huge" : "", | 20260505 21:01:10.946 | INFO | "huge_shared" : " 27 GB", | 20260505 21:01:10.946 | INFO | "malloc" : " 14 GB", | 20260505 21:01:10.946 | INFO | "pinned" : " 29 GB" | 20260505 21:01:10.946 | INFO | } | 20260505 21:01:10.946 | INFO | Perf report for the last 5.13024 seconds: | 20260505 21:01:10.946 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:01:10.946 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:10.946 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:10.946 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:10.946 | INFO | ANALYSIS : 62.9%, 3.0, 98.0ms, 0 | 20260505 21:01:10.946 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:10.946 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:10.946 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 21:01:10.946 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:10.946 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:10.947 | INFO | Uploaded 36GiB to GPU in 2.16364s (16.6386GiB/s) | 20260505 21:01:10.947 | INFO | Downloaded 34GiB to GPU in 2.12899s (15.97GiB/s) | 20260505 21:01:16.081 | INFO | Analyzing chunk frames = [6274560,6275072) | 20260505 21:01:16.081 | INFO | MemInfo: { | 20260505 21:01:16.081 | INFO | "gpu" : " 17 GB", | 20260505 21:01:16.081 | INFO | "huge" : "", | 20260505 21:01:16.081 | INFO | "huge_shared" : " 27 GB", | 20260505 21:01:16.081 | INFO | "malloc" : " 14 GB", | 20260505 21:01:16.081 | INFO | "pinned" : " 29 GB" | 20260505 21:01:16.081 | INFO | } | 20260505 21:01:16.081 | INFO | Perf report for the last 5.13488 seconds: | 20260505 21:01:16.081 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:01:16.081 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:16.081 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:16.081 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:16.081 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260505 21:01:16.081 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:16.081 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:16.081 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 21:01:16.081 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:16.081 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:16.082 | INFO | Uploaded 36GiB to GPU in 2.12573s (16.9354GiB/s) | 20260505 21:01:16.082 | INFO | Downloaded 34GiB to GPU in 2.01844s (16.8447GiB/s) | 20260505 21:01:16.082 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 21:01:16.082 | INFO | Total mem usage: 15.3 GiB | 20260505 21:01:16.082 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 21:01:16.082 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.96%) | 20260505 21:01:16.082 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 21:01:16.082 | INFO | High Water Breakdown: | 20260505 21:01:16.082 | INFO | 60.9% : 8.57 GiB ( 90% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 21:01:16.082 | INFO | 38.9% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 21:01:16.082 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 21:01:16.082 | INFO | ------------------------------------------ | 20260505 21:01:16.082 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:01:16.082 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 21:01:16.082 | INFO | Total mem usage: 26.5 GiB | 20260505 21:01:16.082 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 21:01:16.082 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 21:01:16.082 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 21:01:16.082 | INFO | High Water Breakdown: | 20260505 21:01:16.082 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:01:16.082 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:01:16.082 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:01:16.082 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:01:16.082 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:01:16.082 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:01:16.082 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 21:01:16.082 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 21:01:16.082 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:01:16.082 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 21:01:16.082 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 21:01:16.082 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 21:01:16.082 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 21:01:16.082 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 21:01:16.082 | INFO | ------------------------------------------ | 20260505 21:01:16.082 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:01:16.082 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 21:01:16.082 | INFO | Total mem usage: 25.6 GiB | 20260505 21:01:16.082 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 21:01:16.082 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 21:01:16.082 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 21:01:16.082 | INFO | High Water Breakdown: | 20260505 21:01:16.082 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 21:01:16.082 | INFO | ------------------------------------------ | 20260505 21:01:16.082 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:01:16.082 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 21:01:16.082 | INFO | Total mem usage: 15.7 GiB | 20260505 21:01:16.082 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 21:01:16.082 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 21:01:16.082 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 21:01:16.082 | INFO | High Water Breakdown: | 20260505 21:01:16.082 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 21:01:16.082 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 21:01:16.082 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 21:01:16.082 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 21:01:16.083 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 21:01:16.083 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 21:01:16.083 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 21:01:16.083 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 21:01:16.083 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 21:01:16.083 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:01:16.083 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 21:01:16.083 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:01:16.083 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 21:01:16.083 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 21:01:16.083 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 21:01:16.083 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 21:01:16.083 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:01:16.083 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 21:01:16.083 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:01:16.083 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:01:16.083 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:01:16.083 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 21:01:16.083 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 21:01:16.083 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:01:16.083 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 21:01:16.083 | INFO | ------------------------------------------ | 20260505 21:01:16.083 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:01:16.083 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 21:01:16.083 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:01:16.083 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:01:16.083 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.45e+01, 1.35e+01 / 1.47e+00 | 20260505 21:01:16.083 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 1.04e-01, 1.15e-02 / 7.03e-03 | 20260505 21:01:16.083 | INFO | PerfReport: 11.81%, Baseline -- 3.72e+00 / 4.53e+00, 5.69e-01 / 1.42e-01 | 20260505 21:01:16.083 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.52e+00, 1.90e-01 / 9.37e-02 | 20260505 21:01:16.083 | INFO | PerfReport: 15.94%, DME -- 4.01e+01 / 4.42e+01, 3.94e+00 / 1.63e+00 | 20260505 21:01:16.083 | INFO | PerfReport: 57.15%, FrameLabeling -- 1.80e+01 / 1.96e+01, 1.52e+00 / 1.07e+00 | 20260505 21:01:16.083 | INFO | PerfReport: 7.18%, PulseAccumulating -- 2.26e+00 / 2.80e+00, 4.28e-01 / 2.11e-01 | 20260505 21:01:16.083 | INFO | PerfReport: 3.22%, Metrics -- 1.01e+00 / 1.31e+00, 2.76e-01 / 1.32e-01 | 20260505 21:01:16.083 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.52e-01 / 4.11e-01, 6.19e-02 / 3.57e-02 | 20260505 21:01:16.083 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 21:01:16.083 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:01:16.083 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:01:16.083 | INFO | PerfReport: Full Execution -- 9.91e+01 / -nan, 2.48e+01 / 0.00e+00 | 20260505 21:01:16.083 | INFO | PerfReport: 22.47%, Upload -- 2.23e+01 / 0.00e+00, 7.60e+00 / 0.00e+00 | 20260505 21:01:16.083 | INFO | PerfReport: 21.81%, Download -- 2.16e+01 / 0.00e+00, 7.60e+00 / 0.00e+00 | 20260505 21:01:16.083 | INFO | PerfReport: 31.76%, Compute -- 3.15e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 21:01:16.083 | INFO | PerfReport: 2.20%, LockUpload -- 2.18e+00 / 0.00e+00, 7.82e+00 / 0.00e+00 | 20260505 21:01:16.083 | INFO | PerfReport: 1.33%, LockDownload -- 1.32e+00 / 0.00e+00, 2.86e+00 / 0.00e+00 | 20260505 21:01:16.083 | INFO | PerfReport: 20.41%, LockCompute -- 2.02e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 21:01:16.083 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.51e-02 / 0.00e+00 | 20260505 21:01:16.213 | INFO | Starting a round of Baz IO | 20260505 21:01:19.385 | INFO | Populating RTMetrics report | 20260505 21:01:19.385 | INFO | RTMetrics report populated | 20260505 21:01:19.385 | INFO | Opening tmp RTMetrics json file | 20260505 21:01:19.387 | INFO | Serializing RTMetrics to json | 20260505 21:01:19.389 | INFO | Serialization done, swapping tmp file now... | 20260505 21:01:19.389 | INFO | RTMetrics done | 20260505 21:01:19.475 | INFO | Finished a round of Baz IO: Wrote 1.987 GiB in 3.262 seconds | 20260505 21:01:19.475 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.61% / 0.00% / 78.39% | 20260505 21:01:19.475 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.157511% / 0% / 0.000103315% | 20260505 21:01:19.475 | INFO | Baz IO overall: | 20260505 21:01:19.475 | INFO | BytesWritten : 3644321167417 | 20260505 21:01:19.475 | INFO | NumEvents : 1721685296479 | 20260505 21:01:21.230 | INFO | Analyzing chunk frames = [6275072,6275584) | 20260505 21:01:21.230 | INFO | MemInfo: { | 20260505 21:01:21.230 | INFO | "gpu" : " 17 GB", | 20260505 21:01:21.230 | INFO | "huge" : "", | 20260505 21:01:21.230 | INFO | "huge_shared" : " 27 GB", | 20260505 21:01:21.230 | INFO | "malloc" : " 14 GB", | 20260505 21:01:21.230 | INFO | "pinned" : " 29 GB" | 20260505 21:01:21.230 | INFO | } | 20260505 21:01:21.230 | INFO | Perf report for the last 5.14904 seconds: | 20260505 21:01:21.230 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:01:21.230 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:21.230 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:21.230 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:21.230 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 21:01:21.230 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:21.230 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260505 21:01:21.230 | INFO | PRE_HQ : 60.1%, 1.8, 55.3ms, 0 | 20260505 21:01:21.230 | INFO | BAZWRITER : 62.7%, 3.7, 118.8ms, 0 | 20260505 21:01:21.230 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 21:01:21.231 | INFO | Uploaded 36GiB to GPU in 2.16472s (16.6303GiB/s) | 20260505 21:01:21.231 | INFO | Downloaded 38GiB to GPU in 2.24196s (16.9495GiB/s) | 20260505 21:01:26.369 | INFO | Analyzing chunk frames = [6275584,6276096) | 20260505 21:01:26.369 | INFO | MemInfo: { | 20260505 21:01:26.369 | INFO | "gpu" : " 17 GB", | 20260505 21:01:26.369 | INFO | "huge" : "", | 20260505 21:01:26.369 | INFO | "huge_shared" : " 27 GB", | 20260505 21:01:26.369 | INFO | "malloc" : " 14 GB", | 20260505 21:01:26.369 | INFO | "pinned" : " 29 GB" | 20260505 21:01:26.369 | INFO | } | 20260505 21:01:26.369 | INFO | Perf report for the last 5.13901 seconds: | 20260505 21:01:26.369 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:01:26.369 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:26.369 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:26.369 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:26.369 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 21:01:26.369 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:26.369 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:26.369 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:01:26.369 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:26.369 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:26.370 | INFO | Uploaded 36GiB to GPU in 2.09279s (17.202GiB/s) | 20260505 21:01:26.370 | INFO | Downloaded 34GiB to GPU in 2.03249s (16.7283GiB/s) | 20260505 21:01:31.508 | INFO | Analyzing chunk frames = [6276096,6276608) | 20260505 21:01:31.508 | INFO | MemInfo: { | 20260505 21:01:31.508 | INFO | "gpu" : " 17 GB", | 20260505 21:01:31.508 | INFO | "huge" : "", | 20260505 21:01:31.508 | INFO | "huge_shared" : " 27 GB", | 20260505 21:01:31.508 | INFO | "malloc" : " 14 GB", | 20260505 21:01:31.508 | INFO | "pinned" : " 29 GB" | 20260505 21:01:31.508 | INFO | } | 20260505 21:01:31.508 | INFO | Perf report for the last 5.13908 seconds: | 20260505 21:01:31.508 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:01:31.508 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:31.508 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:31.508 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:31.508 | INFO | ANALYSIS : 62.1%, 3.0, 96.5ms, 0 | 20260505 21:01:31.508 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:31.508 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:31.509 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 21:01:31.509 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:31.509 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:31.509 | INFO | Uploaded 36GiB to GPU in 2.14066s (16.8173GiB/s) | 20260505 21:01:31.509 | INFO | Downloaded 34GiB to GPU in 2.02175s (16.8171GiB/s) | 20260505 21:01:36.647 | INFO | Analyzing chunk frames = [6276608,6277120) | 20260505 21:01:36.647 | INFO | MemInfo: { | 20260505 21:01:36.647 | INFO | "gpu" : " 17 GB", | 20260505 21:01:36.647 | INFO | "huge" : "", | 20260505 21:01:36.647 | INFO | "huge_shared" : " 27 GB", | 20260505 21:01:36.647 | INFO | "malloc" : " 14 GB", | 20260505 21:01:36.647 | INFO | "pinned" : " 29 GB" | 20260505 21:01:36.647 | INFO | } | 20260505 21:01:36.647 | INFO | Perf report for the last 5.13878 seconds: | 20260505 21:01:36.647 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:01:36.647 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:36.647 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:36.647 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:36.647 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260505 21:01:36.647 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:36.647 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:36.647 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260505 21:01:36.647 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:36.647 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:36.647 | INFO | Uploaded 36GiB to GPU in 2.20249s (16.3452GiB/s) | 20260505 21:01:36.647 | INFO | Downloaded 34GiB to GPU in 2.02829s (16.7629GiB/s) | 20260505 21:01:41.787 | INFO | Analyzing chunk frames = [6277120,6277632) | 20260505 21:01:41.787 | INFO | MemInfo: { | 20260505 21:01:41.787 | INFO | "gpu" : " 17 GB", | 20260505 21:01:41.787 | INFO | "huge" : "", | 20260505 21:01:41.787 | INFO | "huge_shared" : " 27 GB", | 20260505 21:01:41.787 | INFO | "malloc" : " 14 GB", | 20260505 21:01:41.787 | INFO | "pinned" : " 29 GB" | 20260505 21:01:41.787 | INFO | } | 20260505 21:01:41.787 | INFO | Perf report for the last 5.13991 seconds: | 20260505 21:01:41.787 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:01:41.787 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:41.787 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:41.787 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:41.787 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 21:01:41.787 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:41.787 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:41.787 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 21:01:41.787 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:41.787 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:41.787 | INFO | Uploaded 36GiB to GPU in 2.14975s (16.7462GiB/s) | 20260505 21:01:41.787 | INFO | Downloaded 34GiB to GPU in 2.06349s (16.4769GiB/s) | 20260505 21:01:46.926 | INFO | Analyzing chunk frames = [6277632,6278144) | 20260505 21:01:46.926 | INFO | MemInfo: { | 20260505 21:01:46.926 | INFO | "gpu" : " 17 GB", | 20260505 21:01:46.926 | INFO | "huge" : "", | 20260505 21:01:46.926 | INFO | "huge_shared" : " 27 GB", | 20260505 21:01:46.926 | INFO | "malloc" : " 14 GB", | 20260505 21:01:46.926 | INFO | "pinned" : " 29 GB" | 20260505 21:01:46.926 | INFO | } | 20260505 21:01:46.927 | INFO | Perf report for the last 5.13959 seconds: | 20260505 21:01:46.927 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:01:46.927 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:46.927 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:46.927 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:46.927 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260505 21:01:46.927 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:46.927 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:46.927 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:01:46.927 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:46.927 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:46.927 | INFO | Uploaded 36GiB to GPU in 2.08634s (17.2551GiB/s) | 20260505 21:01:46.927 | INFO | Downloaded 34GiB to GPU in 2.0411s (16.6577GiB/s) | 20260505 21:01:52.063 | INFO | Analyzing chunk frames = [6278144,6278656) | 20260505 21:01:52.063 | INFO | MemInfo: { | 20260505 21:01:52.063 | INFO | "gpu" : " 17 GB", | 20260505 21:01:52.063 | INFO | "huge" : "", | 20260505 21:01:52.063 | INFO | "huge_shared" : " 27 GB", | 20260505 21:01:52.063 | INFO | "malloc" : " 14 GB", | 20260505 21:01:52.064 | INFO | "pinned" : " 29 GB" | 20260505 21:01:52.064 | INFO | } | 20260505 21:01:52.064 | INFO | Perf report for the last 5.13688 seconds: | 20260505 21:01:52.064 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:01:52.064 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:52.064 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:52.064 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:52.064 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 21:01:52.064 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:52.064 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:52.064 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 21:01:52.064 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:52.064 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:52.064 | INFO | Uploaded 36GiB to GPU in 2.076s (17.3411GiB/s) | 20260505 21:01:52.064 | INFO | Downloaded 34GiB to GPU in 2.06616s (16.4557GiB/s) | 20260505 21:01:57.201 | INFO | Analyzing chunk frames = [6278656,6279168) | 20260505 21:01:57.201 | INFO | MemInfo: { | 20260505 21:01:57.201 | INFO | "gpu" : " 17 GB", | 20260505 21:01:57.201 | INFO | "huge" : "", | 20260505 21:01:57.201 | INFO | "huge_shared" : " 27 GB", | 20260505 21:01:57.201 | INFO | "malloc" : " 14 GB", | 20260505 21:01:57.201 | INFO | "pinned" : " 29 GB" | 20260505 21:01:57.201 | INFO | } | 20260505 21:01:57.201 | INFO | Perf report for the last 5.13707 seconds: | 20260505 21:01:57.201 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:01:57.201 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:57.201 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:57.201 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:57.201 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 21:01:57.201 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:57.201 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:57.201 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 21:01:57.201 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:57.201 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:01:57.201 | INFO | Uploaded 36GiB to GPU in 2.09824s (17.1572GiB/s) | 20260505 21:01:57.201 | INFO | Downloaded 34GiB to GPU in 2.00514s (16.9564GiB/s) | 20260505 21:01:57.335 | INFO | Starting a round of Baz IO | 20260505 21:02:00.490 | INFO | Populating RTMetrics report | 20260505 21:02:00.490 | INFO | RTMetrics report populated | 20260505 21:02:00.490 | INFO | Opening tmp RTMetrics json file | 20260505 21:02:00.491 | INFO | Serializing RTMetrics to json | 20260505 21:02:00.493 | INFO | Serialization done, swapping tmp file now... | 20260505 21:02:00.493 | INFO | RTMetrics done | 20260505 21:02:00.583 | INFO | Finished a round of Baz IO: Wrote 1.987 GiB in 3.247 seconds | 20260505 21:02:00.583 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.61% / 0.00% / 78.39% | 20260505 21:02:00.583 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.158179% / 7.94729e-06% / 0.00013113% | 20260505 21:02:00.583 | INFO | Baz IO overall: | 20260505 21:02:00.583 | INFO | BytesWritten : 3646454267747 | 20260505 21:02:00.583 | INFO | NumEvents : 1722670786101 | 20260505 21:02:02.334 | INFO | Analyzing chunk frames = [6279168,6279680) | 20260505 21:02:02.334 | INFO | MemInfo: { | 20260505 21:02:02.334 | INFO | "gpu" : " 17 GB", | 20260505 21:02:02.334 | INFO | "huge" : "", | 20260505 21:02:02.334 | INFO | "huge_shared" : " 27 GB", | 20260505 21:02:02.334 | INFO | "malloc" : " 14 GB", | 20260505 21:02:02.334 | INFO | "pinned" : " 29 GB" | 20260505 21:02:02.334 | INFO | } | 20260505 21:02:02.334 | INFO | Perf report for the last 5.13351 seconds: | 20260505 21:02:02.334 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:02:02.334 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:02.334 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:02.334 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:02.334 | INFO | ANALYSIS : 63.0%, 3.0, 98.1ms, 0 | 20260505 21:02:02.334 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:02.334 | INFO | RT_METRICS : 44.0%, 1.0, 23.0ms, 0 | 20260505 21:02:02.334 | INFO | PRE_HQ : 60.2%, 1.8, 55.5ms, 0 | 20260505 21:02:02.334 | INFO | BAZWRITER : 62.8%, 3.7, 120.0ms, 0 | 20260505 21:02:02.335 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 21:02:02.335 | INFO | Uploaded 36GiB to GPU in 2.17356s (16.5627GiB/s) | 20260505 21:02:02.335 | INFO | Downloaded 38GiB to GPU in 2.24465s (16.9291GiB/s) | 20260505 21:02:07.486 | INFO | Analyzing chunk frames = [6279680,6280192) | 20260505 21:02:07.487 | INFO | MemInfo: { | 20260505 21:02:07.487 | INFO | "gpu" : " 17 GB", | 20260505 21:02:07.487 | INFO | "huge" : "", | 20260505 21:02:07.487 | INFO | "huge_shared" : " 27 GB", | 20260505 21:02:07.487 | INFO | "malloc" : " 14 GB", | 20260505 21:02:07.487 | INFO | "pinned" : " 29 GB" | 20260505 21:02:07.487 | INFO | } | 20260505 21:02:07.487 | INFO | Perf report for the last 5.15236 seconds: | 20260505 21:02:07.487 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:02:07.487 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:07.487 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:07.487 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:07.487 | INFO | ANALYSIS : 62.1%, 3.0, 97.2ms, 0 | 20260505 21:02:07.487 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:07.487 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:07.487 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:02:07.487 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:07.487 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:07.487 | INFO | Uploaded 36GiB to GPU in 2.10191s (17.1272GiB/s) | 20260505 21:02:07.487 | INFO | Downloaded 34GiB to GPU in 2.02642s (16.7784GiB/s) | 20260505 21:02:12.629 | INFO | Analyzing chunk frames = [6280192,6280704) | 20260505 21:02:12.629 | INFO | MemInfo: { | 20260505 21:02:12.629 | INFO | "gpu" : " 17 GB", | 20260505 21:02:12.629 | INFO | "huge" : "", | 20260505 21:02:12.629 | INFO | "huge_shared" : " 27 GB", | 20260505 21:02:12.629 | INFO | "malloc" : " 14 GB", | 20260505 21:02:12.629 | INFO | "pinned" : " 29 GB" | 20260505 21:02:12.629 | INFO | } | 20260505 21:02:12.630 | INFO | Perf report for the last 5.14292 seconds: | 20260505 21:02:12.630 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:02:12.630 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:12.630 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:12.630 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:12.630 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260505 21:02:12.630 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:12.630 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:12.630 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:02:12.630 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:12.630 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:12.630 | INFO | Uploaded 36GiB to GPU in 2.13691s (16.8468GiB/s) | 20260505 21:02:12.630 | INFO | Downloaded 34GiB to GPU in 2.01758s (16.8518GiB/s) | 20260505 21:02:17.760 | INFO | Analyzing chunk frames = [6280704,6281216) | 20260505 21:02:17.760 | INFO | MemInfo: { | 20260505 21:02:17.760 | INFO | "gpu" : " 17 GB", | 20260505 21:02:17.760 | INFO | "huge" : "", | 20260505 21:02:17.760 | INFO | "huge_shared" : " 27 GB", | 20260505 21:02:17.760 | INFO | "malloc" : " 14 GB", | 20260505 21:02:17.760 | INFO | "pinned" : " 29 GB" | 20260505 21:02:17.760 | INFO | } | 20260505 21:02:17.761 | INFO | Perf report for the last 5.13098 seconds: | 20260505 21:02:17.761 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:02:17.761 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:17.761 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:17.761 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:17.761 | INFO | ANALYSIS : 62.0%, 3.0, 96.2ms, 0 | 20260505 21:02:17.761 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:17.761 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:17.761 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260505 21:02:17.761 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:17.761 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:17.761 | INFO | Uploaded 36GiB to GPU in 2.1321s (16.8847GiB/s) | 20260505 21:02:17.761 | INFO | Downloaded 34GiB to GPU in 2.03677s (16.6931GiB/s) | 20260505 21:02:22.894 | INFO | Analyzing chunk frames = [6281216,6281728) | 20260505 21:02:22.894 | INFO | MemInfo: { | 20260505 21:02:22.894 | INFO | "gpu" : " 17 GB", | 20260505 21:02:22.894 | INFO | "huge" : "", | 20260505 21:02:22.894 | INFO | "huge_shared" : " 27 GB", | 20260505 21:02:22.894 | INFO | "malloc" : " 14 GB", | 20260505 21:02:22.894 | INFO | "pinned" : " 29 GB" | 20260505 21:02:22.894 | INFO | } | 20260505 21:02:22.894 | INFO | Perf report for the last 5.13388 seconds: | 20260505 21:02:22.894 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:02:22.894 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:22.894 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:22.894 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:22.894 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260505 21:02:22.894 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:22.894 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:22.894 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:02:22.894 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:22.894 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:22.895 | INFO | Uploaded 36GiB to GPU in 2.13516s (16.8606GiB/s) | 20260505 21:02:22.895 | INFO | Downloaded 34GiB to GPU in 2.06297s (16.4811GiB/s) | 20260505 21:02:28.035 | INFO | Analyzing chunk frames = [6281728,6282240) | 20260505 21:02:28.035 | INFO | MemInfo: { | 20260505 21:02:28.035 | INFO | "gpu" : " 17 GB", | 20260505 21:02:28.035 | INFO | "huge" : "", | 20260505 21:02:28.035 | INFO | "huge_shared" : " 27 GB", | 20260505 21:02:28.035 | INFO | "malloc" : " 14 GB", | 20260505 21:02:28.035 | INFO | "pinned" : " 29 GB" | 20260505 21:02:28.035 | INFO | } | 20260505 21:02:28.035 | INFO | Perf report for the last 5.14091 seconds: | 20260505 21:02:28.035 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:02:28.035 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:28.035 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:28.035 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:28.035 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 21:02:28.035 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:28.035 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:28.035 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:02:28.035 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:28.035 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:28.036 | INFO | Uploaded 36GiB to GPU in 2.1384s (16.835GiB/s) | 20260505 21:02:28.036 | INFO | Downloaded 34GiB to GPU in 2.0502s (16.5838GiB/s) | 20260505 21:02:33.177 | INFO | Analyzing chunk frames = [6282240,6282752) | 20260505 21:02:33.177 | INFO | MemInfo: { | 20260505 21:02:33.177 | INFO | "gpu" : " 17 GB", | 20260505 21:02:33.177 | INFO | "huge" : "", | 20260505 21:02:33.177 | INFO | "huge_shared" : " 27 GB", | 20260505 21:02:33.177 | INFO | "malloc" : " 14 GB", | 20260505 21:02:33.177 | INFO | "pinned" : " 29 GB" | 20260505 21:02:33.177 | INFO | } | 20260505 21:02:33.177 | INFO | Perf report for the last 5.14171 seconds: | 20260505 21:02:33.177 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:02:33.177 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:33.177 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:33.177 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:33.177 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 21:02:33.177 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:33.177 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:33.177 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260505 21:02:33.177 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:33.177 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:33.177 | INFO | Uploaded 36GiB to GPU in 2.11816s (16.9959GiB/s) | 20260505 21:02:33.177 | INFO | Downloaded 34GiB to GPU in 2.13807s (15.9022GiB/s) | 20260505 21:02:38.319 | INFO | Analyzing chunk frames = [6282752,6283264) | 20260505 21:02:38.319 | INFO | MemInfo: { | 20260505 21:02:38.319 | INFO | "gpu" : " 17 GB", | 20260505 21:02:38.319 | INFO | "huge" : "", | 20260505 21:02:38.319 | INFO | "huge_shared" : " 27 GB", | 20260505 21:02:38.319 | INFO | "malloc" : " 14 GB", | 20260505 21:02:38.319 | INFO | "pinned" : " 29 GB" | 20260505 21:02:38.319 | INFO | } | 20260505 21:02:38.319 | INFO | Perf report for the last 5.14199 seconds: | 20260505 21:02:38.319 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:02:38.319 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:38.319 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:38.319 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:38.319 | INFO | ANALYSIS : 62.1%, 3.0, 97.1ms, 0 | 20260505 21:02:38.319 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:38.319 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:38.319 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:02:38.319 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:38.319 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:38.319 | INFO | Uploaded 36GiB to GPU in 2.11881s (16.9907GiB/s) | 20260505 21:02:38.319 | INFO | Downloaded 34GiB to GPU in 1.99889s (17.0094GiB/s) | 20260505 21:02:38.320 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 21:02:38.320 | INFO | Total mem usage: 15.3 GiB | 20260505 21:02:38.320 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 21:02:38.320 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.14%) | 20260505 21:02:38.320 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 21:02:38.320 | INFO | High Water Breakdown: | 20260505 21:02:38.320 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 21:02:38.320 | INFO | 38.8% : 5.5 GiB ( 82.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 21:02:38.320 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 21:02:38.320 | INFO | ------------------------------------------ | 20260505 21:02:38.320 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:02:38.320 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 21:02:38.320 | INFO | Total mem usage: 26.5 GiB | 20260505 21:02:38.320 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 21:02:38.320 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 21:02:38.320 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 21:02:38.320 | INFO | High Water Breakdown: | 20260505 21:02:38.320 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:02:38.320 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:02:38.320 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:02:38.320 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:02:38.320 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:02:38.320 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:02:38.320 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 21:02:38.320 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 21:02:38.320 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:02:38.320 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 21:02:38.320 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 21:02:38.320 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 21:02:38.320 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 21:02:38.320 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 21:02:38.320 | INFO | ------------------------------------------ | 20260505 21:02:38.320 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:02:38.320 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 21:02:38.320 | INFO | Total mem usage: 25.6 GiB | 20260505 21:02:38.320 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 21:02:38.320 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 21:02:38.320 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 21:02:38.320 | INFO | High Water Breakdown: | 20260505 21:02:38.320 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 21:02:38.320 | INFO | ------------------------------------------ | 20260505 21:02:38.320 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:02:38.320 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 21:02:38.320 | INFO | Total mem usage: 15.7 GiB | 20260505 21:02:38.320 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 21:02:38.320 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 21:02:38.320 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 21:02:38.320 | INFO | High Water Breakdown: | 20260505 21:02:38.320 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 21:02:38.320 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 21:02:38.320 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 21:02:38.320 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 21:02:38.320 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 21:02:38.320 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 21:02:38.320 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 21:02:38.320 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 21:02:38.320 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 21:02:38.320 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:02:38.320 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 21:02:38.320 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:02:38.320 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 21:02:38.320 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 21:02:38.320 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 21:02:38.320 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 21:02:38.320 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:02:38.320 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 21:02:38.321 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:02:38.321 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:02:38.321 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:02:38.321 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 21:02:38.321 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 21:02:38.321 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:02:38.321 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 21:02:38.321 | INFO | ------------------------------------------ | 20260505 21:02:38.321 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:02:38.321 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 21:02:38.321 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:02:38.321 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:02:38.321 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.44e+01, 1.34e+01 / 1.15e+00 | 20260505 21:02:38.321 | INFO | PerfReport: 0.29%, AggregateStats -- 9.24e-02 / 1.02e-01, 1.04e-02 / 4.55e-03 | 20260505 21:02:38.321 | INFO | PerfReport: 11.81%, Baseline -- 3.72e+00 / 4.52e+00, 5.72e-01 / 1.88e-01 | 20260505 21:02:38.321 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.53e+00, 1.91e-01 / 9.68e-02 | 20260505 21:02:38.321 | INFO | PerfReport: 15.97%, DME -- 4.02e+01 / 4.44e+01, 4.10e+00 / 1.26e+00 | 20260505 21:02:38.321 | INFO | PerfReport: 57.11%, FrameLabeling -- 1.80e+01 / 1.93e+01, 1.52e+00 / 1.13e+00 | 20260505 21:02:38.321 | INFO | PerfReport: 7.20%, PulseAccumulating -- 2.27e+00 / 2.82e+00, 4.31e-01 / 2.23e-01 | 20260505 21:02:38.321 | INFO | PerfReport: 3.21%, Metrics -- 1.01e+00 / 1.31e+00, 2.74e-01 / 1.27e-01 | 20260505 21:02:38.321 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.52e-01 / 4.09e-01, 6.17e-02 / 3.92e-02 | 20260505 21:02:38.321 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 21:02:38.321 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:02:38.321 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:02:38.321 | INFO | PerfReport: Full Execution -- 9.91e+01 / -nan, 2.52e+01 / 0.00e+00 | 20260505 21:02:38.321 | INFO | PerfReport: 22.39%, Upload -- 2.22e+01 / 0.00e+00, 7.65e+00 / 0.00e+00 | 20260505 21:02:38.321 | INFO | PerfReport: 21.74%, Download -- 2.15e+01 / 0.00e+00, 7.63e+00 / 0.00e+00 | 20260505 21:02:38.321 | INFO | PerfReport: 31.77%, Compute -- 3.15e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 21:02:38.321 | INFO | PerfReport: 2.23%, LockUpload -- 2.21e+00 / 0.00e+00, 7.97e+00 / 0.00e+00 | 20260505 21:02:38.321 | INFO | PerfReport: 1.32%, LockDownload -- 1.30e+00 / 0.00e+00, 2.85e+00 / 0.00e+00 | 20260505 21:02:38.321 | INFO | PerfReport: 20.55%, LockCompute -- 2.04e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 21:02:38.321 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.70e-02 / 0.00e+00 | 20260505 21:02:38.450 | INFO | Starting a round of Baz IO | 20260505 21:02:41.605 | INFO | Populating RTMetrics report | 20260505 21:02:41.605 | INFO | RTMetrics report populated | 20260505 21:02:41.605 | INFO | Opening tmp RTMetrics json file | 20260505 21:02:41.606 | INFO | Serializing RTMetrics to json | 20260505 21:02:41.607 | INFO | Serialization done, swapping tmp file now... | 20260505 21:02:41.608 | INFO | RTMetrics done | 20260505 21:02:41.689 | INFO | Finished a round of Baz IO: Wrote 1.987 GiB in 3.238 seconds | 20260505 21:02:41.689 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.61% / 0.00% / 78.39% | 20260505 21:02:41.689 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.157436% / 0% / 0.000123183% | 20260505 21:02:41.689 | INFO | Baz IO overall: | 20260505 21:02:41.689 | INFO | BytesWritten : 3648588224483 | 20260505 21:02:41.689 | INFO | NumEvents : 1723657226092 | 20260505 21:02:43.460 | INFO | Analyzing chunk frames = [6283264,6283776) | 20260505 21:02:43.460 | INFO | MemInfo: { | 20260505 21:02:43.460 | INFO | "gpu" : " 17 GB", | 20260505 21:02:43.460 | INFO | "huge" : "", | 20260505 21:02:43.460 | INFO | "huge_shared" : " 27 GB", | 20260505 21:02:43.460 | INFO | "malloc" : " 14 GB", | 20260505 21:02:43.460 | INFO | "pinned" : " 29 GB" | 20260505 21:02:43.460 | INFO | } | 20260505 21:02:43.460 | INFO | Perf report for the last 5.14094 seconds: | 20260505 21:02:43.460 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:02:43.460 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:43.460 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:43.460 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:43.460 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 21:02:43.460 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:43.460 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260505 21:02:43.460 | INFO | PRE_HQ : 59.9%, 1.8, 54.8ms, 0 | 20260505 21:02:43.460 | INFO | BAZWRITER : 62.5%, 3.7, 118.4ms, 0 | 20260505 21:02:43.460 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 21:02:43.460 | INFO | Uploaded 36GiB to GPU in 2.16876s (16.5993GiB/s) | 20260505 21:02:43.461 | INFO | Downloaded 38GiB to GPU in 2.24122s (16.9551GiB/s) | 20260505 21:02:48.593 | INFO | Analyzing chunk frames = [6283776,6284288) | 20260505 21:02:48.593 | INFO | MemInfo: { | 20260505 21:02:48.593 | INFO | "gpu" : " 17 GB", | 20260505 21:02:48.593 | INFO | "huge" : "", | 20260505 21:02:48.593 | INFO | "huge_shared" : " 27 GB", | 20260505 21:02:48.593 | INFO | "malloc" : " 14 GB", | 20260505 21:02:48.593 | INFO | "pinned" : " 29 GB" | 20260505 21:02:48.593 | INFO | } | 20260505 21:02:48.593 | INFO | Perf report for the last 5.13301 seconds: | 20260505 21:02:48.593 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:02:48.593 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:48.593 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:48.593 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:48.593 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 21:02:48.593 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:48.593 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:48.593 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:02:48.593 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:48.593 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:48.594 | INFO | Uploaded 36GiB to GPU in 2.08913s (17.232GiB/s) | 20260505 21:02:48.594 | INFO | Downloaded 34GiB to GPU in 2.02112s (16.8224GiB/s) | 20260505 21:02:53.740 | INFO | Analyzing chunk frames = [6284288,6284800) | 20260505 21:02:53.740 | INFO | MemInfo: { | 20260505 21:02:53.740 | INFO | "gpu" : " 17 GB", | 20260505 21:02:53.740 | INFO | "huge" : "", | 20260505 21:02:53.740 | INFO | "huge_shared" : " 27 GB", | 20260505 21:02:53.740 | INFO | "malloc" : " 14 GB", | 20260505 21:02:53.740 | INFO | "pinned" : " 29 GB" | 20260505 21:02:53.740 | INFO | } | 20260505 21:02:53.740 | INFO | Perf report for the last 5.14715 seconds: | 20260505 21:02:53.740 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:02:53.740 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:53.740 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:53.740 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:53.740 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260505 21:02:53.740 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:53.740 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:53.740 | INFO | PRE_HQ : 27.0%, 1.0, 14.1ms, 0 | 20260505 21:02:53.740 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:53.740 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:53.741 | INFO | Uploaded 36GiB to GPU in 2.1462s (16.7738GiB/s) | 20260505 21:02:53.741 | INFO | Downloaded 34GiB to GPU in 2.02056s (16.827GiB/s) | 20260505 21:02:58.879 | INFO | Analyzing chunk frames = [6284800,6285312) | 20260505 21:02:58.879 | INFO | MemInfo: { | 20260505 21:02:58.879 | INFO | "gpu" : " 17 GB", | 20260505 21:02:58.879 | INFO | "huge" : "", | 20260505 21:02:58.879 | INFO | "huge_shared" : " 27 GB", | 20260505 21:02:58.879 | INFO | "malloc" : " 14 GB", | 20260505 21:02:58.879 | INFO | "pinned" : " 29 GB" | 20260505 21:02:58.879 | INFO | } | 20260505 21:02:58.879 | INFO | Perf report for the last 5.1388 seconds: | 20260505 21:02:58.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:02:58.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:58.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:58.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:58.879 | INFO | ANALYSIS : 61.9%, 3.0, 96.1ms, 0 | 20260505 21:02:58.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:58.879 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:58.879 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:02:58.879 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:58.879 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:02:58.879 | INFO | Uploaded 36GiB to GPU in 2.13675s (16.848GiB/s) | 20260505 21:02:58.879 | INFO | Downloaded 34GiB to GPU in 2.0192s (16.8383GiB/s) | 20260505 21:03:04.013 | INFO | Analyzing chunk frames = [6285312,6285824) | 20260505 21:03:04.013 | INFO | MemInfo: { | 20260505 21:03:04.013 | INFO | "gpu" : " 17 GB", | 20260505 21:03:04.013 | INFO | "huge" : "", | 20260505 21:03:04.013 | INFO | "huge_shared" : " 27 GB", | 20260505 21:03:04.013 | INFO | "malloc" : " 14 GB", | 20260505 21:03:04.013 | INFO | "pinned" : " 29 GB" | 20260505 21:03:04.013 | INFO | } | 20260505 21:03:04.013 | INFO | Perf report for the last 5.13427 seconds: | 20260505 21:03:04.013 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:03:04.013 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:04.013 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:04.013 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:04.013 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260505 21:03:04.013 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:04.013 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:04.013 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:03:04.013 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:04.013 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:04.014 | INFO | Uploaded 36GiB to GPU in 2.15824s (16.6803GiB/s) | 20260505 21:03:04.014 | INFO | Downloaded 34GiB to GPU in 2.0659s (16.4577GiB/s) | 20260505 21:03:09.150 | INFO | Analyzing chunk frames = [6285824,6286336) | 20260505 21:03:09.150 | INFO | MemInfo: { | 20260505 21:03:09.150 | INFO | "gpu" : " 17 GB", | 20260505 21:03:09.150 | INFO | "huge" : "", | 20260505 21:03:09.150 | INFO | "huge_shared" : " 27 GB", | 20260505 21:03:09.150 | INFO | "malloc" : " 14 GB", | 20260505 21:03:09.150 | INFO | "pinned" : " 29 GB" | 20260505 21:03:09.150 | INFO | } | 20260505 21:03:09.150 | INFO | Perf report for the last 5.13662 seconds: | 20260505 21:03:09.150 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:03:09.150 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:09.150 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:09.150 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:09.150 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 21:03:09.150 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:09.150 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:09.150 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 21:03:09.150 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:09.150 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:09.150 | INFO | Uploaded 36GiB to GPU in 2.15763s (16.685GiB/s) | 20260505 21:03:09.150 | INFO | Downloaded 34GiB to GPU in 2.06337s (16.4779GiB/s) | 20260505 21:03:14.288 | INFO | Analyzing chunk frames = [6286336,6286848) | 20260505 21:03:14.288 | INFO | MemInfo: { | 20260505 21:03:14.288 | INFO | "gpu" : " 17 GB", | 20260505 21:03:14.288 | INFO | "huge" : "", | 20260505 21:03:14.288 | INFO | "huge_shared" : " 27 GB", | 20260505 21:03:14.288 | INFO | "malloc" : " 14 GB", | 20260505 21:03:14.288 | INFO | "pinned" : " 29 GB" | 20260505 21:03:14.288 | INFO | } | 20260505 21:03:14.289 | INFO | Perf report for the last 5.13857 seconds: | 20260505 21:03:14.289 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:03:14.289 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:14.289 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:14.289 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:14.289 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 21:03:14.289 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:14.289 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:14.289 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 21:03:14.289 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:14.289 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:14.289 | INFO | Uploaded 36GiB to GPU in 2.14467s (16.7858GiB/s) | 20260505 21:03:14.289 | INFO | Downloaded 34GiB to GPU in 2.14343s (15.8624GiB/s) | 20260505 21:03:19.435 | INFO | Analyzing chunk frames = [6286848,6287360) | 20260505 21:03:19.435 | INFO | MemInfo: { | 20260505 21:03:19.435 | INFO | "gpu" : " 17 GB", | 20260505 21:03:19.435 | INFO | "huge" : "", | 20260505 21:03:19.435 | INFO | "huge_shared" : " 27 GB", | 20260505 21:03:19.435 | INFO | "malloc" : " 14 GB", | 20260505 21:03:19.435 | INFO | "pinned" : " 29 GB" | 20260505 21:03:19.435 | INFO | } | 20260505 21:03:19.435 | INFO | Perf report for the last 5.14636 seconds: | 20260505 21:03:19.435 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:03:19.435 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:19.435 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:19.435 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:19.435 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260505 21:03:19.435 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:19.435 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:19.435 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:03:19.435 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:19.435 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:19.435 | INFO | Uploaded 36GiB to GPU in 2.11866s (16.9918GiB/s) | 20260505 21:03:19.435 | INFO | Downloaded 34GiB to GPU in 2.00143s (16.9879GiB/s) | 20260505 21:03:19.563 | INFO | Starting a round of Baz IO | 20260505 21:03:22.719 | INFO | Populating RTMetrics report | 20260505 21:03:22.719 | INFO | RTMetrics report populated | 20260505 21:03:22.719 | INFO | Opening tmp RTMetrics json file | 20260505 21:03:22.720 | INFO | Serializing RTMetrics to json | 20260505 21:03:22.721 | INFO | Serialization done, swapping tmp file now... | 20260505 21:03:22.721 | INFO | RTMetrics done | 20260505 21:03:22.803 | INFO | Finished a round of Baz IO: Wrote 1.985 GiB in 3.24 seconds | 20260505 21:03:22.803 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.61% / 0.00% / 78.39% | 20260505 21:03:22.803 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.156999% / 3.97364e-06% / 0.000111262% | 20260505 21:03:22.804 | INFO | Baz IO overall: | 20260505 21:03:22.804 | INFO | BytesWritten : 3650719459176 | 20260505 21:03:22.804 | INFO | NumEvents : 1724641242546 | 20260505 21:03:24.574 | INFO | Analyzing chunk frames = [6287360,6287872) | 20260505 21:03:24.574 | INFO | MemInfo: { | 20260505 21:03:24.574 | INFO | "gpu" : " 17 GB", | 20260505 21:03:24.574 | INFO | "huge" : "", | 20260505 21:03:24.574 | INFO | "huge_shared" : " 27 GB", | 20260505 21:03:24.574 | INFO | "malloc" : " 14 GB", | 20260505 21:03:24.574 | INFO | "pinned" : " 29 GB" | 20260505 21:03:24.574 | INFO | } | 20260505 21:03:24.574 | INFO | Perf report for the last 5.13906 seconds: | 20260505 21:03:24.574 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:03:24.574 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:24.574 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:24.574 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:24.574 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 21:03:24.574 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:24.574 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260505 21:03:24.574 | INFO | PRE_HQ : 60.0%, 1.8, 54.8ms, 0 | 20260505 21:03:24.574 | INFO | BAZWRITER : 62.4%, 3.7, 118.2ms, 0 | 20260505 21:03:24.574 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 21:03:24.574 | INFO | Uploaded 36GiB to GPU in 2.14504s (16.7829GiB/s) | 20260505 21:03:24.574 | INFO | Downloaded 38GiB to GPU in 2.237s (16.9871GiB/s) | 20260505 21:03:29.707 | INFO | Analyzing chunk frames = [6287872,6288384) | 20260505 21:03:29.707 | INFO | MemInfo: { | 20260505 21:03:29.707 | INFO | "gpu" : " 17 GB", | 20260505 21:03:29.707 | INFO | "huge" : "", | 20260505 21:03:29.707 | INFO | "huge_shared" : " 27 GB", | 20260505 21:03:29.707 | INFO | "malloc" : " 14 GB", | 20260505 21:03:29.707 | INFO | "pinned" : " 29 GB" | 20260505 21:03:29.707 | INFO | } | 20260505 21:03:29.707 | INFO | Perf report for the last 5.1327 seconds: | 20260505 21:03:29.707 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:03:29.707 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:29.707 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:29.707 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:29.707 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260505 21:03:29.707 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:29.707 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:29.707 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 21:03:29.707 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:29.707 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:29.707 | INFO | Uploaded 36GiB to GPU in 2.08132s (17.2967GiB/s) | 20260505 21:03:29.707 | INFO | Downloaded 34GiB to GPU in 2.02265s (16.8096GiB/s) | 20260505 21:03:34.843 | INFO | Analyzing chunk frames = [6288384,6288896) | 20260505 21:03:34.843 | INFO | MemInfo: { | 20260505 21:03:34.843 | INFO | "gpu" : " 17 GB", | 20260505 21:03:34.843 | INFO | "huge" : "", | 20260505 21:03:34.843 | INFO | "huge_shared" : " 27 GB", | 20260505 21:03:34.843 | INFO | "malloc" : " 14 GB", | 20260505 21:03:34.843 | INFO | "pinned" : " 29 GB" | 20260505 21:03:34.843 | INFO | } | 20260505 21:03:34.843 | INFO | Perf report for the last 5.13601 seconds: | 20260505 21:03:34.843 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:03:34.843 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:34.843 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:34.843 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:34.843 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260505 21:03:34.843 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:34.843 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:34.843 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 21:03:34.843 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:34.843 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:34.843 | INFO | Uploaded 36GiB to GPU in 2.13289s (16.8785GiB/s) | 20260505 21:03:34.843 | INFO | Downloaded 34GiB to GPU in 2.04463s (16.6289GiB/s) | 20260505 21:03:39.990 | INFO | Analyzing chunk frames = [6288896,6289408) | 20260505 21:03:39.990 | INFO | MemInfo: { | 20260505 21:03:39.990 | INFO | "gpu" : " 17 GB", | 20260505 21:03:39.990 | INFO | "huge" : "", | 20260505 21:03:39.990 | INFO | "huge_shared" : " 27 GB", | 20260505 21:03:39.990 | INFO | "malloc" : " 14 GB", | 20260505 21:03:39.990 | INFO | "pinned" : " 29 GB" | 20260505 21:03:39.990 | INFO | } | 20260505 21:03:39.990 | INFO | Perf report for the last 5.14732 seconds: | 20260505 21:03:39.990 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:03:39.990 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:39.990 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:39.990 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:39.990 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260505 21:03:39.990 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:39.990 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:39.990 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:03:39.990 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:39.990 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:39.990 | INFO | Uploaded 36GiB to GPU in 2.19626s (16.3915GiB/s) | 20260505 21:03:39.990 | INFO | Downloaded 34GiB to GPU in 2.05197s (16.5695GiB/s) | 20260505 21:03:45.129 | INFO | Analyzing chunk frames = [6289408,6289920) | 20260505 21:03:45.129 | INFO | MemInfo: { | 20260505 21:03:45.129 | INFO | "gpu" : " 17 GB", | 20260505 21:03:45.129 | INFO | "huge" : "", | 20260505 21:03:45.129 | INFO | "huge_shared" : " 27 GB", | 20260505 21:03:45.129 | INFO | "malloc" : " 14 GB", | 20260505 21:03:45.129 | INFO | "pinned" : " 29 GB" | 20260505 21:03:45.129 | INFO | } | 20260505 21:03:45.129 | INFO | Perf report for the last 5.13936 seconds: | 20260505 21:03:45.129 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:03:45.129 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:45.129 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:45.129 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:45.129 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260505 21:03:45.129 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:45.129 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:45.129 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:03:45.129 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:45.129 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:45.130 | INFO | Uploaded 36GiB to GPU in 2.1305s (16.8974GiB/s) | 20260505 21:03:45.130 | INFO | Downloaded 34GiB to GPU in 2.06274s (16.4829GiB/s) | 20260505 21:03:50.269 | INFO | Analyzing chunk frames = [6289920,6290432) | 20260505 21:03:50.269 | INFO | MemInfo: { | 20260505 21:03:50.269 | INFO | "gpu" : " 17 GB", | 20260505 21:03:50.269 | INFO | "huge" : "", | 20260505 21:03:50.269 | INFO | "huge_shared" : " 27 GB", | 20260505 21:03:50.269 | INFO | "malloc" : " 14 GB", | 20260505 21:03:50.269 | INFO | "pinned" : " 29 GB" | 20260505 21:03:50.269 | INFO | } | 20260505 21:03:50.269 | INFO | Perf report for the last 5.13976 seconds: | 20260505 21:03:50.269 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:03:50.269 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:50.269 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:50.269 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:50.269 | INFO | ANALYSIS : 61.6%, 3.0, 96.2ms, 0 | 20260505 21:03:50.269 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:50.269 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:50.269 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 21:03:50.269 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:50.269 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:50.269 | INFO | Uploaded 36GiB to GPU in 2.07046s (17.3874GiB/s) | 20260505 21:03:50.269 | INFO | Downloaded 34GiB to GPU in 2.03405s (16.7154GiB/s) | 20260505 21:03:55.406 | INFO | Analyzing chunk frames = [6290432,6290944) | 20260505 21:03:55.406 | INFO | MemInfo: { | 20260505 21:03:55.406 | INFO | "gpu" : " 17 GB", | 20260505 21:03:55.406 | INFO | "huge" : "", | 20260505 21:03:55.406 | INFO | "huge_shared" : " 27 GB", | 20260505 21:03:55.406 | INFO | "malloc" : " 14 GB", | 20260505 21:03:55.406 | INFO | "pinned" : " 29 GB" | 20260505 21:03:55.406 | INFO | } | 20260505 21:03:55.406 | INFO | Perf report for the last 5.13696 seconds: | 20260505 21:03:55.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:03:55.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:55.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:55.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:55.406 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 21:03:55.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:55.406 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:55.406 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 21:03:55.406 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:55.406 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:03:55.406 | INFO | Uploaded 36GiB to GPU in 2.1617s (16.6536GiB/s) | 20260505 21:03:55.406 | INFO | Downloaded 34GiB to GPU in 2.11808s (16.0523GiB/s) | 20260505 21:04:00.547 | INFO | Analyzing chunk frames = [6290944,6291456) | 20260505 21:04:00.547 | INFO | MemInfo: { | 20260505 21:04:00.547 | INFO | "gpu" : " 17 GB", | 20260505 21:04:00.547 | INFO | "huge" : "", | 20260505 21:04:00.547 | INFO | "huge_shared" : " 27 GB", | 20260505 21:04:00.547 | INFO | "malloc" : " 14 GB", | 20260505 21:04:00.547 | INFO | "pinned" : " 29 GB" | 20260505 21:04:00.547 | INFO | } | 20260505 21:04:00.547 | INFO | Perf report for the last 5.14131 seconds: | 20260505 21:04:00.547 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:04:00.547 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:00.547 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:00.547 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:00.548 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 21:04:00.548 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:00.548 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:00.548 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 21:04:00.548 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:00.548 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:00.548 | INFO | Uploaded 36GiB to GPU in 2.11461s (17.0244GiB/s) | 20260505 21:04:00.548 | INFO | Downloaded 34GiB to GPU in 2.01469s (16.8761GiB/s) | 20260505 21:04:00.548 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 21:04:00.548 | INFO | Total mem usage: 15.3 GiB | 20260505 21:04:00.548 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 21:04:00.548 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.81%) | 20260505 21:04:00.548 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 21:04:00.548 | INFO | High Water Breakdown: | 20260505 21:04:00.548 | INFO | 60.9% : 8.57 GiB ( 90.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 21:04:00.548 | INFO | 39% : 5.49 GiB ( 82.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 21:04:00.548 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 21:04:00.548 | INFO | ------------------------------------------ | 20260505 21:04:00.548 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:04:00.548 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 21:04:00.548 | INFO | Total mem usage: 26.5 GiB | 20260505 21:04:00.548 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 21:04:00.548 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 21:04:00.548 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 21:04:00.548 | INFO | High Water Breakdown: | 20260505 21:04:00.548 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:04:00.548 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:04:00.548 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:04:00.548 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:04:00.548 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:04:00.548 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:04:00.548 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 21:04:00.548 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 21:04:00.548 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:04:00.548 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 21:04:00.548 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 21:04:00.548 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 21:04:00.548 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 21:04:00.548 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 21:04:00.548 | INFO | ------------------------------------------ | 20260505 21:04:00.548 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:04:00.548 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 21:04:00.548 | INFO | Total mem usage: 25.6 GiB | 20260505 21:04:00.548 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 21:04:00.548 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 21:04:00.548 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 21:04:00.548 | INFO | High Water Breakdown: | 20260505 21:04:00.548 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 21:04:00.548 | INFO | ------------------------------------------ | 20260505 21:04:00.548 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:04:00.548 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 21:04:00.548 | INFO | Total mem usage: 15.7 GiB | 20260505 21:04:00.548 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 21:04:00.548 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 21:04:00.548 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 21:04:00.548 | INFO | High Water Breakdown: | 20260505 21:04:00.548 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 21:04:00.548 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 21:04:00.549 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 21:04:00.549 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 21:04:00.549 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 21:04:00.549 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 21:04:00.549 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 21:04:00.549 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 21:04:00.549 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 21:04:00.549 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:04:00.549 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 21:04:00.549 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:04:00.549 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 21:04:00.549 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 21:04:00.549 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 21:04:00.549 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 21:04:00.549 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:04:00.549 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 21:04:00.549 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:04:00.549 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:04:00.549 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:04:00.549 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 21:04:00.549 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 21:04:00.549 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:04:00.549 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 21:04:00.549 | INFO | ------------------------------------------ | 20260505 21:04:00.549 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:04:00.549 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 21:04:00.549 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:04:00.549 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:04:00.549 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.44e+01, 1.35e+01 / 1.39e+00 | 20260505 21:04:00.549 | INFO | PerfReport: 0.29%, AggregateStats -- 9.24e-02 / 1.05e-01, 1.11e-02 / 8.86e-03 | 20260505 21:04:00.549 | INFO | PerfReport: 11.80%, Baseline -- 3.72e+00 / 4.52e+00, 5.75e-01 / 1.61e-01 | 20260505 21:04:00.549 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.51e+00, 1.92e-01 / 1.11e-01 | 20260505 21:04:00.549 | INFO | PerfReport: 15.93%, DME -- 4.01e+01 / 4.42e+01, 4.02e+00 / 1.56e+00 | 20260505 21:04:00.549 | INFO | PerfReport: 57.22%, FrameLabeling -- 1.80e+01 / 1.96e+01, 1.59e+00 / 1.30e+00 | 20260505 21:04:00.549 | INFO | PerfReport: 7.15%, PulseAccumulating -- 2.25e+00 / 2.77e+00, 4.32e-01 / 2.69e-01 | 20260505 21:04:00.549 | INFO | PerfReport: 3.21%, Metrics -- 1.01e+00 / 1.29e+00, 2.77e-01 / 1.23e-01 | 20260505 21:04:00.549 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.51e-01 / 4.06e-01, 6.16e-02 / 3.49e-02 | 20260505 21:04:00.549 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 21:04:00.549 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:04:00.549 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:04:00.549 | INFO | PerfReport: Full Execution -- 9.91e+01 / -nan, 2.53e+01 / 0.00e+00 | 20260505 21:04:00.549 | INFO | PerfReport: 22.44%, Upload -- 2.22e+01 / 0.00e+00, 7.70e+00 / 0.00e+00 | 20260505 21:04:00.549 | INFO | PerfReport: 21.79%, Download -- 2.16e+01 / 0.00e+00, 7.70e+00 / 0.00e+00 | 20260505 21:04:00.549 | INFO | PerfReport: 31.79%, Compute -- 3.15e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 21:04:00.549 | INFO | PerfReport: 2.20%, LockUpload -- 2.18e+00 / 0.00e+00, 7.86e+00 / 0.00e+00 | 20260505 21:04:00.549 | INFO | PerfReport: 1.35%, LockDownload -- 1.33e+00 / 0.00e+00, 2.89e+00 / 0.00e+00 | 20260505 21:04:00.549 | INFO | PerfReport: 20.42%, LockCompute -- 2.02e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 21:04:00.549 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.84e-02 / 0.00e+00 | 20260505 21:04:00.682 | INFO | Starting a round of Baz IO | 20260505 21:04:03.836 | INFO | Populating RTMetrics report | 20260505 21:04:03.836 | INFO | RTMetrics report populated | 20260505 21:04:03.836 | INFO | Opening tmp RTMetrics json file | 20260505 21:04:03.837 | INFO | Serializing RTMetrics to json | 20260505 21:04:03.839 | INFO | Serialization done, swapping tmp file now... | 20260505 21:04:03.839 | INFO | RTMetrics done | 20260505 21:04:03.926 | INFO | Finished a round of Baz IO: Wrote 1.983 GiB in 3.244 seconds | 20260505 21:04:03.926 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.61% / 0.00% / 78.39% | 20260505 21:04:03.926 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.157007% / 7.94729e-06% / 0.000119209% | 20260505 21:04:03.926 | INFO | Baz IO overall: | 20260505 21:04:03.926 | INFO | BytesWritten : 3652849131054 | 20260505 21:04:03.926 | INFO | NumEvents : 1725624387974 | 20260505 21:04:05.684 | INFO | Analyzing chunk frames = [6291456,6291968) | 20260505 21:04:05.684 | INFO | MemInfo: { | 20260505 21:04:05.684 | INFO | "gpu" : " 17 GB", | 20260505 21:04:05.684 | INFO | "huge" : "", | 20260505 21:04:05.684 | INFO | "huge_shared" : " 27 GB", | 20260505 21:04:05.684 | INFO | "malloc" : " 14 GB", | 20260505 21:04:05.684 | INFO | "pinned" : " 29 GB" | 20260505 21:04:05.684 | INFO | } | 20260505 21:04:05.685 | INFO | Perf report for the last 5.13706 seconds: | 20260505 21:04:05.685 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:04:05.685 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:05.685 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:05.685 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:05.685 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 21:04:05.685 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:05.685 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260505 21:04:05.685 | INFO | PRE_HQ : 60.0%, 1.8, 54.9ms, 0 | 20260505 21:04:05.685 | INFO | BAZWRITER : 62.7%, 3.7, 118.3ms, 0 | 20260505 21:04:05.685 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 21:04:05.685 | INFO | Uploaded 36GiB to GPU in 2.16892s (16.5981GiB/s) | 20260505 21:04:05.685 | INFO | Downloaded 38GiB to GPU in 2.23596s (16.9949GiB/s) | 20260505 21:04:10.828 | INFO | Analyzing chunk frames = [6291968,6292480) | 20260505 21:04:10.828 | INFO | MemInfo: { | 20260505 21:04:10.828 | INFO | "gpu" : " 17 GB", | 20260505 21:04:10.828 | INFO | "huge" : "", | 20260505 21:04:10.828 | INFO | "huge_shared" : " 27 GB", | 20260505 21:04:10.828 | INFO | "malloc" : " 14 GB", | 20260505 21:04:10.828 | INFO | "pinned" : " 29 GB" | 20260505 21:04:10.828 | INFO | } | 20260505 21:04:10.828 | INFO | Perf report for the last 5.14337 seconds: | 20260505 21:04:10.828 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:04:10.828 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:10.828 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:10.828 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:10.828 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260505 21:04:10.828 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:10.828 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:10.828 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 21:04:10.828 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:10.828 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:10.828 | INFO | Uploaded 36GiB to GPU in 2.07043s (17.3877GiB/s) | 20260505 21:04:10.828 | INFO | Downloaded 34GiB to GPU in 2.02057s (16.8269GiB/s) | 20260505 21:04:15.962 | INFO | Analyzing chunk frames = [6292480,6292992) | 20260505 21:04:15.962 | INFO | MemInfo: { | 20260505 21:04:15.962 | INFO | "gpu" : " 17 GB", | 20260505 21:04:15.962 | INFO | "huge" : "", | 20260505 21:04:15.962 | INFO | "huge_shared" : " 27 GB", | 20260505 21:04:15.962 | INFO | "malloc" : " 14 GB", | 20260505 21:04:15.962 | INFO | "pinned" : " 29 GB" | 20260505 21:04:15.962 | INFO | } | 20260505 21:04:15.962 | INFO | Perf report for the last 5.13416 seconds: | 20260505 21:04:15.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:04:15.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:15.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:15.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:15.962 | INFO | ANALYSIS : 62.1%, 3.0, 96.5ms, 0 | 20260505 21:04:15.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:15.962 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:15.962 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:04:15.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:15.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:15.962 | INFO | Uploaded 36GiB to GPU in 2.13186s (16.8867GiB/s) | 20260505 21:04:15.962 | INFO | Downloaded 34GiB to GPU in 2.0242s (16.7968GiB/s) | 20260505 21:04:21.099 | INFO | Analyzing chunk frames = [6292992,6293504) | 20260505 21:04:21.099 | INFO | MemInfo: { | 20260505 21:04:21.099 | INFO | "gpu" : " 17 GB", | 20260505 21:04:21.099 | INFO | "huge" : "", | 20260505 21:04:21.099 | INFO | "huge_shared" : " 27 GB", | 20260505 21:04:21.099 | INFO | "malloc" : " 14 GB", | 20260505 21:04:21.099 | INFO | "pinned" : " 29 GB" | 20260505 21:04:21.099 | INFO | } | 20260505 21:04:21.099 | INFO | Perf report for the last 5.13717 seconds: | 20260505 21:04:21.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:04:21.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:21.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:21.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:21.099 | INFO | ANALYSIS : 62.3%, 3.0, 96.7ms, 0 | 20260505 21:04:21.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:21.099 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:21.099 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:04:21.099 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:21.099 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:21.100 | INFO | Uploaded 36GiB to GPU in 2.1986s (16.3741GiB/s) | 20260505 21:04:21.100 | INFO | Downloaded 34GiB to GPU in 2.0645s (16.4688GiB/s) | 20260505 21:04:26.247 | INFO | Analyzing chunk frames = [6293504,6294016) | 20260505 21:04:26.247 | INFO | MemInfo: { | 20260505 21:04:26.247 | INFO | "gpu" : " 17 GB", | 20260505 21:04:26.247 | INFO | "huge" : "", | 20260505 21:04:26.247 | INFO | "huge_shared" : " 27 GB", | 20260505 21:04:26.247 | INFO | "malloc" : " 14 GB", | 20260505 21:04:26.247 | INFO | "pinned" : " 29 GB" | 20260505 21:04:26.248 | INFO | } | 20260505 21:04:26.248 | INFO | Perf report for the last 5.14842 seconds: | 20260505 21:04:26.248 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:04:26.248 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:26.248 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:26.248 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:26.248 | INFO | ANALYSIS : 61.5%, 3.0, 96.2ms, 0 | 20260505 21:04:26.248 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:26.248 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:26.248 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 21:04:26.248 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:26.248 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:26.248 | INFO | Uploaded 36GiB to GPU in 2.1292s (16.9078GiB/s) | 20260505 21:04:26.248 | INFO | Downloaded 34GiB to GPU in 2.0664s (16.4537GiB/s) | 20260505 21:04:31.384 | INFO | Analyzing chunk frames = [6294016,6294528) | 20260505 21:04:31.384 | INFO | MemInfo: { | 20260505 21:04:31.384 | INFO | "gpu" : " 17 GB", | 20260505 21:04:31.384 | INFO | "huge" : "", | 20260505 21:04:31.384 | INFO | "huge_shared" : " 27 GB", | 20260505 21:04:31.384 | INFO | "malloc" : " 14 GB", | 20260505 21:04:31.384 | INFO | "pinned" : " 29 GB" | 20260505 21:04:31.384 | INFO | } | 20260505 21:04:31.384 | INFO | Perf report for the last 5.13596 seconds: | 20260505 21:04:31.384 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:04:31.384 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:31.384 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:31.384 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:31.384 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260505 21:04:31.384 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:31.384 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:31.384 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:04:31.384 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:31.384 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:31.384 | INFO | Uploaded 36GiB to GPU in 2.07356s (17.3615GiB/s) | 20260505 21:04:31.384 | INFO | Downloaded 34GiB to GPU in 2.03453s (16.7115GiB/s) | 20260505 21:04:36.520 | INFO | Analyzing chunk frames = [6294528,6295040) | 20260505 21:04:36.520 | INFO | MemInfo: { | 20260505 21:04:36.520 | INFO | "gpu" : " 17 GB", | 20260505 21:04:36.520 | INFO | "huge" : "", | 20260505 21:04:36.520 | INFO | "huge_shared" : " 27 GB", | 20260505 21:04:36.520 | INFO | "malloc" : " 14 GB", | 20260505 21:04:36.520 | INFO | "pinned" : " 29 GB" | 20260505 21:04:36.520 | INFO | } | 20260505 21:04:36.520 | INFO | Perf report for the last 5.13613 seconds: | 20260505 21:04:36.520 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:04:36.520 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:36.520 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:36.520 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:36.520 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 21:04:36.520 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:36.520 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:36.520 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 21:04:36.520 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:36.520 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:36.520 | INFO | Uploaded 36GiB to GPU in 2.18425s (16.4816GiB/s) | 20260505 21:04:36.520 | INFO | Downloaded 34GiB to GPU in 2.12105s (16.0298GiB/s) | 20260505 21:04:41.660 | INFO | Analyzing chunk frames = [6295040,6295552) | 20260505 21:04:41.660 | INFO | MemInfo: { | 20260505 21:04:41.660 | INFO | "gpu" : " 17 GB", | 20260505 21:04:41.660 | INFO | "huge" : "", | 20260505 21:04:41.660 | INFO | "huge_shared" : " 27 GB", | 20260505 21:04:41.660 | INFO | "malloc" : " 14 GB", | 20260505 21:04:41.660 | INFO | "pinned" : " 29 GB" | 20260505 21:04:41.660 | INFO | } | 20260505 21:04:41.660 | INFO | Perf report for the last 5.14034 seconds: | 20260505 21:04:41.660 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:04:41.660 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:41.660 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:41.660 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:41.660 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 21:04:41.660 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:41.660 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:41.660 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 21:04:41.660 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:41.660 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:41.660 | INFO | Uploaded 36GiB to GPU in 2.11192s (17.0461GiB/s) | 20260505 21:04:41.660 | INFO | Downloaded 34GiB to GPU in 2.02167s (16.8178GiB/s) | 20260505 21:04:41.790 | INFO | Starting a round of Baz IO | 20260505 21:04:44.961 | INFO | Populating RTMetrics report | 20260505 21:04:44.961 | INFO | RTMetrics report populated | 20260505 21:04:44.961 | INFO | Opening tmp RTMetrics json file | 20260505 21:04:44.967 | INFO | Serializing RTMetrics to json | 20260505 21:04:44.978 | INFO | Serialization done, swapping tmp file now... | 20260505 21:04:44.983 | INFO | RTMetrics done | 20260505 21:04:45.045 | INFO | Finished a round of Baz IO: Wrote 1.986 GiB in 3.255 seconds | 20260505 21:04:45.046 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.61% / 0.00% / 78.39% | 20260505 21:04:45.046 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.15643% / 0% / 0.000103315% | 20260505 21:04:45.046 | INFO | Baz IO overall: | 20260505 21:04:45.046 | INFO | BytesWritten : 3654982093037 | 20260505 21:04:45.046 | INFO | NumEvents : 1726610247448 | 20260505 21:04:46.799 | INFO | Analyzing chunk frames = [6295552,6296064) | 20260505 21:04:46.799 | INFO | MemInfo: { | 20260505 21:04:46.799 | INFO | "gpu" : " 17 GB", | 20260505 21:04:46.799 | INFO | "huge" : "", | 20260505 21:04:46.799 | INFO | "huge_shared" : " 27 GB", | 20260505 21:04:46.799 | INFO | "malloc" : " 14 GB", | 20260505 21:04:46.799 | INFO | "pinned" : " 29 GB" | 20260505 21:04:46.799 | INFO | } | 20260505 21:04:46.799 | INFO | Perf report for the last 5.13857 seconds: | 20260505 21:04:46.799 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:04:46.799 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:46.799 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:46.799 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:46.799 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 21:04:46.799 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:46.799 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260505 21:04:46.799 | INFO | PRE_HQ : 60.5%, 1.8, 55.5ms, 0 | 20260505 21:04:46.799 | INFO | BAZWRITER : 62.9%, 3.7, 118.3ms, 0 | 20260505 21:04:46.799 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260505 21:04:46.799 | INFO | Uploaded 36GiB to GPU in 2.16327s (16.6415GiB/s) | 20260505 21:04:46.799 | INFO | Downloaded 38GiB to GPU in 2.24787s (16.9049GiB/s) | 20260505 21:04:51.941 | INFO | Analyzing chunk frames = [6296064,6296576) | 20260505 21:04:51.941 | INFO | MemInfo: { | 20260505 21:04:51.941 | INFO | "gpu" : " 17 GB", | 20260505 21:04:51.941 | INFO | "huge" : "", | 20260505 21:04:51.941 | INFO | "huge_shared" : " 27 GB", | 20260505 21:04:51.941 | INFO | "malloc" : " 14 GB", | 20260505 21:04:51.941 | INFO | "pinned" : " 29 GB" | 20260505 21:04:51.941 | INFO | } | 20260505 21:04:51.942 | INFO | Perf report for the last 5.14296 seconds: | 20260505 21:04:51.942 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:04:51.942 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:51.942 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:51.942 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:51.942 | INFO | ANALYSIS : 62.1%, 3.0, 97.1ms, 0 | 20260505 21:04:51.942 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:51.942 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:51.942 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:04:51.942 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:51.942 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:51.942 | INFO | Uploaded 36GiB to GPU in 2.09737s (17.1644GiB/s) | 20260505 21:04:51.942 | INFO | Downloaded 34GiB to GPU in 2.01914s (16.8389GiB/s) | 20260505 21:04:57.079 | INFO | Analyzing chunk frames = [6296576,6297088) | 20260505 21:04:57.079 | INFO | MemInfo: { | 20260505 21:04:57.079 | INFO | "gpu" : " 17 GB", | 20260505 21:04:57.079 | INFO | "huge" : "", | 20260505 21:04:57.079 | INFO | "huge_shared" : " 27 GB", | 20260505 21:04:57.079 | INFO | "malloc" : " 14 GB", | 20260505 21:04:57.079 | INFO | "pinned" : " 29 GB" | 20260505 21:04:57.079 | INFO | } | 20260505 21:04:57.079 | INFO | Perf report for the last 5.13767 seconds: | 20260505 21:04:57.079 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:04:57.079 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:57.079 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:57.079 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:57.079 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260505 21:04:57.079 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:57.079 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:57.080 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:04:57.080 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:57.080 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:04:57.080 | INFO | Uploaded 36GiB to GPU in 2.13275s (16.8796GiB/s) | 20260505 21:04:57.080 | INFO | Downloaded 34GiB to GPU in 2.01551s (16.8692GiB/s) | 20260505 21:05:02.218 | INFO | Analyzing chunk frames = [6297088,6297600) | 20260505 21:05:02.218 | INFO | MemInfo: { | 20260505 21:05:02.218 | INFO | "gpu" : " 17 GB", | 20260505 21:05:02.218 | INFO | "huge" : "", | 20260505 21:05:02.218 | INFO | "huge_shared" : " 27 GB", | 20260505 21:05:02.218 | INFO | "malloc" : " 14 GB", | 20260505 21:05:02.218 | INFO | "pinned" : " 29 GB" | 20260505 21:05:02.218 | INFO | } | 20260505 21:05:02.218 | INFO | Perf report for the last 5.13895 seconds: | 20260505 21:05:02.219 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:05:02.219 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:02.219 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:02.219 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:02.219 | INFO | ANALYSIS : 62.4%, 3.0, 96.9ms, 0 | 20260505 21:05:02.219 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:02.219 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:02.219 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:05:02.219 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:02.219 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:02.219 | INFO | Uploaded 36GiB to GPU in 2.20993s (16.2901GiB/s) | 20260505 21:05:02.219 | INFO | Downloaded 34GiB to GPU in 2.07054s (16.4208GiB/s) | 20260505 21:05:07.354 | INFO | Analyzing chunk frames = [6297600,6298112) | 20260505 21:05:07.354 | INFO | MemInfo: { | 20260505 21:05:07.354 | INFO | "gpu" : " 17 GB", | 20260505 21:05:07.354 | INFO | "huge" : "", | 20260505 21:05:07.354 | INFO | "huge_shared" : " 27 GB", | 20260505 21:05:07.354 | INFO | "malloc" : " 14 GB", | 20260505 21:05:07.354 | INFO | "pinned" : " 29 GB" | 20260505 21:05:07.354 | INFO | } | 20260505 21:05:07.354 | INFO | Perf report for the last 5.13609 seconds: | 20260505 21:05:07.355 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:05:07.355 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:07.355 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:07.355 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:07.355 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260505 21:05:07.355 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:07.355 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:07.355 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:05:07.355 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:07.355 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:07.355 | INFO | Uploaded 36GiB to GPU in 2.13513s (16.8608GiB/s) | 20260505 21:05:07.355 | INFO | Downloaded 34GiB to GPU in 2.05932s (16.5103GiB/s) | 20260505 21:05:12.500 | INFO | Analyzing chunk frames = [6298112,6298624) | 20260505 21:05:12.501 | INFO | MemInfo: { | 20260505 21:05:12.501 | INFO | "gpu" : " 17 GB", | 20260505 21:05:12.501 | INFO | "huge" : "", | 20260505 21:05:12.501 | INFO | "huge_shared" : " 27 GB", | 20260505 21:05:12.501 | INFO | "malloc" : " 14 GB", | 20260505 21:05:12.501 | INFO | "pinned" : " 29 GB" | 20260505 21:05:12.501 | INFO | } | 20260505 21:05:12.501 | INFO | Perf report for the last 5.14628 seconds: | 20260505 21:05:12.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:05:12.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:12.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:12.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:12.501 | INFO | ANALYSIS : 62.0%, 3.0, 97.0ms, 0 | 20260505 21:05:12.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:12.501 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:12.501 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:05:12.501 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:12.501 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:12.501 | INFO | Uploaded 36GiB to GPU in 2.14875s (16.754GiB/s) | 20260505 21:05:12.501 | INFO | Downloaded 34GiB to GPU in 2.05992s (16.5055GiB/s) | 20260505 21:05:17.639 | INFO | Analyzing chunk frames = [6298624,6299136) | 20260505 21:05:17.639 | INFO | MemInfo: { | 20260505 21:05:17.639 | INFO | "gpu" : " 17 GB", | 20260505 21:05:17.639 | INFO | "huge" : "", | 20260505 21:05:17.639 | INFO | "huge_shared" : " 27 GB", | 20260505 21:05:17.640 | INFO | "malloc" : " 14 GB", | 20260505 21:05:17.640 | INFO | "pinned" : " 29 GB" | 20260505 21:05:17.640 | INFO | } | 20260505 21:05:17.640 | INFO | Perf report for the last 5.139 seconds: | 20260505 21:05:17.640 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:05:17.640 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:17.640 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:17.640 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:17.640 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260505 21:05:17.640 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:17.640 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:17.640 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 21:05:17.640 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:17.640 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:17.640 | INFO | Uploaded 36GiB to GPU in 2.0493s (17.567GiB/s) | 20260505 21:05:17.640 | INFO | Downloaded 34GiB to GPU in 2.05985s (16.506GiB/s) | 20260505 21:05:22.779 | INFO | Analyzing chunk frames = [6299136,6299648) | 20260505 21:05:22.779 | INFO | MemInfo: { | 20260505 21:05:22.779 | INFO | "gpu" : " 17 GB", | 20260505 21:05:22.779 | INFO | "huge" : "", | 20260505 21:05:22.779 | INFO | "huge_shared" : " 27 GB", | 20260505 21:05:22.779 | INFO | "malloc" : " 14 GB", | 20260505 21:05:22.779 | INFO | "pinned" : " 29 GB" | 20260505 21:05:22.779 | INFO | } | 20260505 21:05:22.779 | INFO | Perf report for the last 5.1394 seconds: | 20260505 21:05:22.779 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:05:22.779 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:22.779 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:22.779 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:22.779 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 21:05:22.779 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:22.779 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:22.779 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 21:05:22.779 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:22.779 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:22.779 | INFO | Uploaded 36GiB to GPU in 2.12104s (16.9728GiB/s) | 20260505 21:05:22.780 | INFO | Downloaded 34GiB to GPU in 2.01369s (16.8844GiB/s) | 20260505 21:05:22.780 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 21:05:22.780 | INFO | Total mem usage: 15.3 GiB | 20260505 21:05:22.780 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 21:05:22.780 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.16%) | 20260505 21:05:22.780 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 21:05:22.780 | INFO | High Water Breakdown: | 20260505 21:05:22.780 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 21:05:22.780 | INFO | 38.8% : 5.5 GiB ( 82.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 21:05:22.780 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 21:05:22.780 | INFO | ------------------------------------------ | 20260505 21:05:22.780 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:05:22.780 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 21:05:22.780 | INFO | Total mem usage: 26.5 GiB | 20260505 21:05:22.780 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 21:05:22.780 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 21:05:22.780 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 21:05:22.780 | INFO | High Water Breakdown: | 20260505 21:05:22.780 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:05:22.780 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:05:22.780 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:05:22.780 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:05:22.780 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:05:22.780 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:05:22.780 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 21:05:22.780 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 21:05:22.780 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:05:22.780 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 21:05:22.780 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 21:05:22.780 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 21:05:22.780 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 21:05:22.780 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 21:05:22.780 | INFO | ------------------------------------------ | 20260505 21:05:22.780 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:05:22.780 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 21:05:22.780 | INFO | Total mem usage: 25.6 GiB | 20260505 21:05:22.780 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 21:05:22.780 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 21:05:22.780 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 21:05:22.780 | INFO | High Water Breakdown: | 20260505 21:05:22.780 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 21:05:22.780 | INFO | ------------------------------------------ | 20260505 21:05:22.780 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:05:22.780 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 21:05:22.780 | INFO | Total mem usage: 15.7 GiB | 20260505 21:05:22.780 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 21:05:22.780 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 21:05:22.780 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 21:05:22.780 | INFO | High Water Breakdown: | 20260505 21:05:22.780 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 21:05:22.780 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 21:05:22.780 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 21:05:22.780 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 21:05:22.780 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 21:05:22.780 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 21:05:22.780 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 21:05:22.780 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 21:05:22.780 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 21:05:22.780 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:05:22.780 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 21:05:22.780 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:05:22.780 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 21:05:22.780 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 21:05:22.780 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 21:05:22.780 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 21:05:22.780 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:05:22.780 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 21:05:22.780 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:05:22.780 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:05:22.780 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:05:22.781 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 21:05:22.781 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 21:05:22.781 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:05:22.781 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 21:05:22.781 | INFO | ------------------------------------------ | 20260505 21:05:22.781 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:05:22.781 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 21:05:22.781 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:05:22.781 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:05:22.781 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.43e+01, 1.33e+01 / 9.65e-01 | 20260505 21:05:22.781 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.02e-01, 1.01e-02 / 4.37e-03 | 20260505 21:05:22.781 | INFO | PerfReport: 11.79%, Baseline -- 3.71e+00 / 4.49e+00, 5.65e-01 / 2.05e-01 | 20260505 21:05:22.781 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.51e+00, 1.89e-01 / 1.05e-01 | 20260505 21:05:22.781 | INFO | PerfReport: 15.94%, DME -- 4.01e+01 / 4.41e+01, 3.83e+00 / 9.98e-01 | 20260505 21:05:22.781 | INFO | PerfReport: 57.18%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.54e+00 / 1.26e+00 | 20260505 21:05:22.781 | INFO | PerfReport: 7.19%, PulseAccumulating -- 2.26e+00 / 2.80e+00, 4.31e-01 / 2.36e-01 | 20260505 21:05:22.781 | INFO | PerfReport: 3.22%, Metrics -- 1.01e+00 / 1.33e+00, 2.77e-01 / 1.51e-01 | 20260505 21:05:22.781 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.52e-01 / 4.08e-01, 6.13e-02 / 3.67e-02 | 20260505 21:05:22.781 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 21:05:22.781 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:05:22.781 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:05:22.781 | INFO | PerfReport: Full Execution -- 9.91e+01 / -nan, 2.52e+01 / 0.00e+00 | 20260505 21:05:22.781 | INFO | PerfReport: 22.42%, Upload -- 2.22e+01 / 0.00e+00, 7.70e+00 / 0.00e+00 | 20260505 21:05:22.781 | INFO | PerfReport: 21.76%, Download -- 2.16e+01 / 0.00e+00, 7.63e+00 / 0.00e+00 | 20260505 21:05:22.781 | INFO | PerfReport: 31.74%, Compute -- 3.15e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260505 21:05:22.781 | INFO | PerfReport: 2.24%, LockUpload -- 2.22e+00 / 0.00e+00, 7.85e+00 / 0.00e+00 | 20260505 21:05:22.781 | INFO | PerfReport: 1.28%, LockDownload -- 1.27e+00 / 0.00e+00, 2.79e+00 / 0.00e+00 | 20260505 21:05:22.781 | INFO | PerfReport: 20.54%, LockCompute -- 2.04e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 21:05:22.781 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.77e-02 / 0.00e+00 | 20260505 21:05:22.912 | INFO | Starting a round of Baz IO | 20260505 21:05:26.078 | INFO | Populating RTMetrics report | 20260505 21:05:26.078 | INFO | RTMetrics report populated | 20260505 21:05:26.078 | INFO | Opening tmp RTMetrics json file | 20260505 21:05:26.079 | INFO | Serializing RTMetrics to json | 20260505 21:05:26.080 | INFO | Serialization done, swapping tmp file now... | 20260505 21:05:26.080 | INFO | RTMetrics done | 20260505 21:05:26.167 | INFO | Finished a round of Baz IO: Wrote 1.987 GiB in 3.255 seconds | 20260505 21:05:26.167 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.61% / 0.00% / 78.39% | 20260505 21:05:26.167 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.15618% / 3.97364e-06% / 8.34465e-05% | 20260505 21:05:26.167 | INFO | Baz IO overall: | 20260505 21:05:26.167 | INFO | BytesWritten : 3657115241893 | 20260505 21:05:26.167 | INFO | NumEvents : 1727596512881 | 20260505 21:05:27.917 | INFO | Analyzing chunk frames = [6299648,6300160) | 20260505 21:05:27.917 | INFO | MemInfo: { | 20260505 21:05:27.917 | INFO | "gpu" : " 17 GB", | 20260505 21:05:27.917 | INFO | "huge" : "", | 20260505 21:05:27.917 | INFO | "huge_shared" : " 27 GB", | 20260505 21:05:27.917 | INFO | "malloc" : " 14 GB", | 20260505 21:05:27.917 | INFO | "pinned" : " 29 GB" | 20260505 21:05:27.917 | INFO | } | 20260505 21:05:27.917 | INFO | Perf report for the last 5.13747 seconds: | 20260505 21:05:27.917 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:05:27.917 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:27.917 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:27.917 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:27.917 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 21:05:27.917 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:27.917 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260505 21:05:27.917 | INFO | PRE_HQ : 60.1%, 1.8, 55.2ms, 0 | 20260505 21:05:27.917 | INFO | BAZWRITER : 62.8%, 3.7, 117.8ms, 0 | 20260505 21:05:27.917 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 21:05:27.917 | INFO | Uploaded 36GiB to GPU in 2.18024s (16.5119GiB/s) | 20260505 21:05:27.917 | INFO | Downloaded 38GiB to GPU in 2.24851s (16.9GiB/s) | 20260505 21:05:33.054 | INFO | Analyzing chunk frames = [6300160,6300672) | 20260505 21:05:33.054 | INFO | MemInfo: { | 20260505 21:05:33.054 | INFO | "gpu" : " 17 GB", | 20260505 21:05:33.054 | INFO | "huge" : "", | 20260505 21:05:33.054 | INFO | "huge_shared" : " 27 GB", | 20260505 21:05:33.054 | INFO | "malloc" : " 14 GB", | 20260505 21:05:33.054 | INFO | "pinned" : " 29 GB" | 20260505 21:05:33.054 | INFO | } | 20260505 21:05:33.054 | INFO | Perf report for the last 5.13735 seconds: | 20260505 21:05:33.054 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:05:33.054 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:33.054 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:33.054 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:33.054 | INFO | ANALYSIS : 62.3%, 3.0, 96.3ms, 0 | 20260505 21:05:33.054 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:33.054 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:33.054 | INFO | PRE_HQ : 27.2%, 1.0, 14.0ms, 0 | 20260505 21:05:33.054 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:33.054 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:33.054 | INFO | Uploaded 36GiB to GPU in 2.09656s (17.171GiB/s) | 20260505 21:05:33.055 | INFO | Downloaded 34GiB to GPU in 2.02375s (16.8005GiB/s) | 20260505 21:05:38.198 | INFO | Analyzing chunk frames = [6300672,6301184) | 20260505 21:05:38.198 | INFO | MemInfo: { | 20260505 21:05:38.198 | INFO | "gpu" : " 17 GB", | 20260505 21:05:38.198 | INFO | "huge" : "", | 20260505 21:05:38.198 | INFO | "huge_shared" : " 27 GB", | 20260505 21:05:38.198 | INFO | "malloc" : " 14 GB", | 20260505 21:05:38.198 | INFO | "pinned" : " 29 GB" | 20260505 21:05:38.198 | INFO | } | 20260505 21:05:38.198 | INFO | Perf report for the last 5.14426 seconds: | 20260505 21:05:38.198 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:05:38.198 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:38.198 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:38.198 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:38.198 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260505 21:05:38.198 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:38.198 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:38.198 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:05:38.198 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:38.198 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:38.199 | INFO | Uploaded 36GiB to GPU in 2.12508s (16.9405GiB/s) | 20260505 21:05:38.199 | INFO | Downloaded 34GiB to GPU in 2.01465s (16.8763GiB/s) | 20260505 21:05:43.338 | INFO | Analyzing chunk frames = [6301184,6301696) | 20260505 21:05:43.338 | INFO | MemInfo: { | 20260505 21:05:43.338 | INFO | "gpu" : " 17 GB", | 20260505 21:05:43.338 | INFO | "huge" : "", | 20260505 21:05:43.338 | INFO | "huge_shared" : " 27 GB", | 20260505 21:05:43.338 | INFO | "malloc" : " 14 GB", | 20260505 21:05:43.338 | INFO | "pinned" : " 29 GB" | 20260505 21:05:43.338 | INFO | } | 20260505 21:05:43.338 | INFO | Perf report for the last 5.13983 seconds: | 20260505 21:05:43.338 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:05:43.338 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:43.338 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:43.338 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:43.338 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260505 21:05:43.338 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:43.338 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:43.338 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 21:05:43.338 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:43.338 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:43.338 | INFO | Uploaded 36GiB to GPU in 2.1386s (16.8335GiB/s) | 20260505 21:05:43.338 | INFO | Downloaded 34GiB to GPU in 2.01268s (16.8929GiB/s) | 20260505 21:05:48.467 | INFO | Analyzing chunk frames = [6301696,6302208) | 20260505 21:05:48.467 | INFO | MemInfo: { | 20260505 21:05:48.467 | INFO | "gpu" : " 17 GB", | 20260505 21:05:48.467 | INFO | "huge" : "", | 20260505 21:05:48.467 | INFO | "huge_shared" : " 27 GB", | 20260505 21:05:48.467 | INFO | "malloc" : " 14 GB", | 20260505 21:05:48.467 | INFO | "pinned" : " 29 GB" | 20260505 21:05:48.467 | INFO | } | 20260505 21:05:48.467 | INFO | Perf report for the last 5.12884 seconds: | 20260505 21:05:48.467 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:05:48.467 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:48.467 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:48.467 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:48.467 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260505 21:05:48.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:48.467 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:48.467 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 21:05:48.467 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:48.467 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:48.467 | INFO | Uploaded 36GiB to GPU in 2.09145s (17.2129GiB/s) | 20260505 21:05:48.467 | INFO | Downloaded 34GiB to GPU in 2.04433s (16.6314GiB/s) | 20260505 21:05:53.608 | INFO | Analyzing chunk frames = [6302208,6302720) | 20260505 21:05:53.608 | INFO | MemInfo: { | 20260505 21:05:53.608 | INFO | "gpu" : " 17 GB", | 20260505 21:05:53.608 | INFO | "huge" : "", | 20260505 21:05:53.608 | INFO | "huge_shared" : " 27 GB", | 20260505 21:05:53.608 | INFO | "malloc" : " 14 GB", | 20260505 21:05:53.608 | INFO | "pinned" : " 29 GB" | 20260505 21:05:53.608 | INFO | } | 20260505 21:05:53.608 | INFO | Perf report for the last 5.14153 seconds: | 20260505 21:05:53.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:05:53.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:53.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:53.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:53.609 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 21:05:53.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:53.609 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:53.609 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:05:53.609 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:53.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:53.609 | INFO | Uploaded 36GiB to GPU in 2.14591s (16.7761GiB/s) | 20260505 21:05:53.609 | INFO | Downloaded 34GiB to GPU in 2.05845s (16.5173GiB/s) | 20260505 21:05:58.752 | INFO | Analyzing chunk frames = [6302720,6303232) | 20260505 21:05:58.752 | INFO | MemInfo: { | 20260505 21:05:58.752 | INFO | "gpu" : " 17 GB", | 20260505 21:05:58.752 | INFO | "huge" : "", | 20260505 21:05:58.752 | INFO | "huge_shared" : " 27 GB", | 20260505 21:05:58.752 | INFO | "malloc" : " 14 GB", | 20260505 21:05:58.752 | INFO | "pinned" : " 29 GB" | 20260505 21:05:58.752 | INFO | } | 20260505 21:05:58.752 | INFO | Perf report for the last 5.14385 seconds: | 20260505 21:05:58.752 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:05:58.752 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:58.752 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:58.752 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:58.752 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 21:05:58.752 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:58.752 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:58.752 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 21:05:58.752 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:58.752 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:05:58.753 | INFO | Uploaded 36GiB to GPU in 2.1206s (16.9763GiB/s) | 20260505 21:05:58.753 | INFO | Downloaded 34GiB to GPU in 2.13616s (15.9164GiB/s) | 20260505 21:06:03.893 | INFO | Analyzing chunk frames = [6303232,6303744) | 20260505 21:06:03.893 | INFO | MemInfo: { | 20260505 21:06:03.893 | INFO | "gpu" : " 17 GB", | 20260505 21:06:03.893 | INFO | "huge" : "", | 20260505 21:06:03.893 | INFO | "huge_shared" : " 27 GB", | 20260505 21:06:03.893 | INFO | "malloc" : " 14 GB", | 20260505 21:06:03.893 | INFO | "pinned" : " 29 GB" | 20260505 21:06:03.893 | INFO | } | 20260505 21:06:03.893 | INFO | Perf report for the last 5.14057 seconds: | 20260505 21:06:03.893 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:06:03.893 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:03.893 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:03.893 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:03.893 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 21:06:03.893 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:03.893 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:03.893 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 21:06:03.893 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:03.893 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:03.893 | INFO | Uploaded 36GiB to GPU in 2.09823s (17.1573GiB/s) | 20260505 21:06:03.893 | INFO | Downloaded 34GiB to GPU in 2.00334s (16.9716GiB/s) | 20260505 21:06:04.027 | INFO | Starting a round of Baz IO | 20260505 21:06:07.180 | INFO | Populating RTMetrics report | 20260505 21:06:07.180 | INFO | RTMetrics report populated | 20260505 21:06:07.180 | INFO | Opening tmp RTMetrics json file | 20260505 21:06:07.181 | INFO | Serializing RTMetrics to json | 20260505 21:06:07.182 | INFO | Serialization done, swapping tmp file now... | 20260505 21:06:07.182 | INFO | RTMetrics done | 20260505 21:06:07.266 | INFO | Finished a round of Baz IO: Wrote 1.983 GiB in 3.238 seconds | 20260505 21:06:07.266 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.61% / 0.00% / 78.39% | 20260505 21:06:07.266 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.15614% / 0% / 8.74201e-05% | 20260505 21:06:07.266 | INFO | Baz IO overall: | 20260505 21:06:07.266 | INFO | BytesWritten : 3659244015863 | 20260505 21:06:07.266 | INFO | NumEvents : 1728580133238 | 20260505 21:06:09.029 | INFO | Analyzing chunk frames = [6303744,6304256) | 20260505 21:06:09.029 | INFO | MemInfo: { | 20260505 21:06:09.029 | INFO | "gpu" : " 17 GB", | 20260505 21:06:09.029 | INFO | "huge" : "", | 20260505 21:06:09.029 | INFO | "huge_shared" : " 27 GB", | 20260505 21:06:09.029 | INFO | "malloc" : " 14 GB", | 20260505 21:06:09.029 | INFO | "pinned" : " 29 GB" | 20260505 21:06:09.029 | INFO | } | 20260505 21:06:09.029 | INFO | Perf report for the last 5.13594 seconds: | 20260505 21:06:09.029 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:06:09.029 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:09.029 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:09.029 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:09.029 | INFO | ANALYSIS : 62.9%, 3.0, 98.0ms, 0 | 20260505 21:06:09.029 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:09.029 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260505 21:06:09.029 | INFO | PRE_HQ : 60.1%, 1.8, 55.1ms, 0 | 20260505 21:06:09.029 | INFO | BAZWRITER : 62.6%, 3.7, 119.2ms, 0 | 20260505 21:06:09.029 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 21:06:09.029 | INFO | Uploaded 36GiB to GPU in 2.16863s (16.6003GiB/s) | 20260505 21:06:09.029 | INFO | Downloaded 38GiB to GPU in 2.24562s (16.9218GiB/s) | 20260505 21:06:14.164 | INFO | Analyzing chunk frames = [6304256,6304768) | 20260505 21:06:14.164 | INFO | MemInfo: { | 20260505 21:06:14.164 | INFO | "gpu" : " 17 GB", | 20260505 21:06:14.164 | INFO | "huge" : "", | 20260505 21:06:14.164 | INFO | "huge_shared" : " 27 GB", | 20260505 21:06:14.164 | INFO | "malloc" : " 14 GB", | 20260505 21:06:14.165 | INFO | "pinned" : " 29 GB" | 20260505 21:06:14.165 | INFO | } | 20260505 21:06:14.165 | INFO | Perf report for the last 5.13575 seconds: | 20260505 21:06:14.165 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:06:14.165 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:14.165 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:14.165 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:14.165 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 21:06:14.165 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:14.165 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:14.165 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 21:06:14.165 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:14.165 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:14.165 | INFO | Uploaded 36GiB to GPU in 2.16155s (16.6547GiB/s) | 20260505 21:06:14.165 | INFO | Downloaded 34GiB to GPU in 2.05201s (16.5691GiB/s) | 20260505 21:06:19.310 | INFO | Analyzing chunk frames = [6304768,6305280) | 20260505 21:06:19.310 | INFO | MemInfo: { | 20260505 21:06:19.310 | INFO | "gpu" : " 17 GB", | 20260505 21:06:19.310 | INFO | "huge" : "", | 20260505 21:06:19.310 | INFO | "huge_shared" : " 27 GB", | 20260505 21:06:19.310 | INFO | "malloc" : " 14 GB", | 20260505 21:06:19.310 | INFO | "pinned" : " 29 GB" | 20260505 21:06:19.310 | INFO | } | 20260505 21:06:19.310 | INFO | Perf report for the last 5.14511 seconds: | 20260505 21:06:19.310 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:06:19.310 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:19.310 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:19.310 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:19.310 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260505 21:06:19.310 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:19.310 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:19.310 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 21:06:19.310 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:19.310 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:19.310 | INFO | Uploaded 36GiB to GPU in 2.12602s (16.933GiB/s) | 20260505 21:06:19.310 | INFO | Downloaded 34GiB to GPU in 2.01543s (16.8698GiB/s) | 20260505 21:06:24.447 | INFO | Analyzing chunk frames = [6305280,6305792) | 20260505 21:06:24.447 | INFO | MemInfo: { | 20260505 21:06:24.447 | INFO | "gpu" : " 17 GB", | 20260505 21:06:24.447 | INFO | "huge" : "", | 20260505 21:06:24.447 | INFO | "huge_shared" : " 27 GB", | 20260505 21:06:24.447 | INFO | "malloc" : " 14 GB", | 20260505 21:06:24.447 | INFO | "pinned" : " 29 GB" | 20260505 21:06:24.447 | INFO | } | 20260505 21:06:24.447 | INFO | Perf report for the last 5.13704 seconds: | 20260505 21:06:24.447 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:06:24.447 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:24.447 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:24.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:24.447 | INFO | ANALYSIS : 62.0%, 3.0, 96.3ms, 0 | 20260505 21:06:24.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:24.447 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:24.447 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:06:24.447 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:24.447 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:24.448 | INFO | Uploaded 36GiB to GPU in 2.14105s (16.8142GiB/s) | 20260505 21:06:24.448 | INFO | Downloaded 34GiB to GPU in 2.03667s (16.6939GiB/s) | 20260505 21:06:29.587 | INFO | Analyzing chunk frames = [6305792,6306304) | 20260505 21:06:29.587 | INFO | MemInfo: { | 20260505 21:06:29.587 | INFO | "gpu" : " 17 GB", | 20260505 21:06:29.587 | INFO | "huge" : "", | 20260505 21:06:29.587 | INFO | "huge_shared" : " 27 GB", | 20260505 21:06:29.587 | INFO | "malloc" : " 14 GB", | 20260505 21:06:29.587 | INFO | "pinned" : " 29 GB" | 20260505 21:06:29.587 | INFO | } | 20260505 21:06:29.587 | INFO | Perf report for the last 5.14044 seconds: | 20260505 21:06:29.587 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:06:29.587 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:29.587 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:29.587 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:29.587 | INFO | ANALYSIS : 61.5%, 3.0, 96.1ms, 0 | 20260505 21:06:29.587 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:29.587 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:29.587 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 21:06:29.587 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:29.587 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:29.588 | INFO | Uploaded 36GiB to GPU in 2.13127s (16.8914GiB/s) | 20260505 21:06:29.588 | INFO | Downloaded 34GiB to GPU in 2.06154s (16.4925GiB/s) | 20260505 21:06:34.727 | INFO | Analyzing chunk frames = [6306304,6306816) | 20260505 21:06:34.727 | INFO | MemInfo: { | 20260505 21:06:34.727 | INFO | "gpu" : " 17 GB", | 20260505 21:06:34.727 | INFO | "huge" : "", | 20260505 21:06:34.727 | INFO | "huge_shared" : " 27 GB", | 20260505 21:06:34.727 | INFO | "malloc" : " 14 GB", | 20260505 21:06:34.727 | INFO | "pinned" : " 29 GB" | 20260505 21:06:34.727 | INFO | } | 20260505 21:06:34.727 | INFO | Perf report for the last 5.13952 seconds: | 20260505 21:06:34.727 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:06:34.727 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:34.727 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:34.727 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:34.727 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 21:06:34.727 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:34.727 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:34.727 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 21:06:34.727 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:34.727 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:34.727 | INFO | Uploaded 36GiB to GPU in 2.15851s (16.6782GiB/s) | 20260505 21:06:34.727 | INFO | Downloaded 34GiB to GPU in 2.06303s (16.4806GiB/s) | 20260505 21:06:39.865 | INFO | Analyzing chunk frames = [6306816,6307328) | 20260505 21:06:39.865 | INFO | MemInfo: { | 20260505 21:06:39.865 | INFO | "gpu" : " 17 GB", | 20260505 21:06:39.865 | INFO | "huge" : "", | 20260505 21:06:39.865 | INFO | "huge_shared" : " 27 GB", | 20260505 21:06:39.865 | INFO | "malloc" : " 14 GB", | 20260505 21:06:39.865 | INFO | "pinned" : " 29 GB" | 20260505 21:06:39.865 | INFO | } | 20260505 21:06:39.865 | INFO | Perf report for the last 5.13835 seconds: | 20260505 21:06:39.865 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:06:39.865 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:39.865 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:39.865 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:39.865 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 21:06:39.865 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:39.865 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:39.865 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 21:06:39.865 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:39.865 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:39.865 | INFO | Uploaded 36GiB to GPU in 2.11595s (17.0137GiB/s) | 20260505 21:06:39.865 | INFO | Downloaded 34GiB to GPU in 2.14447s (15.8547GiB/s) | 20260505 21:06:45.009 | INFO | Analyzing chunk frames = [6307328,6307840) | 20260505 21:06:45.009 | INFO | MemInfo: { | 20260505 21:06:45.009 | INFO | "gpu" : " 17 GB", | 20260505 21:06:45.009 | INFO | "huge" : "", | 20260505 21:06:45.009 | INFO | "huge_shared" : " 27 GB", | 20260505 21:06:45.009 | INFO | "malloc" : " 14 GB", | 20260505 21:06:45.009 | INFO | "pinned" : " 29 GB" | 20260505 21:06:45.009 | INFO | } | 20260505 21:06:45.009 | INFO | Perf report for the last 5.14423 seconds: | 20260505 21:06:45.009 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:06:45.009 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:45.009 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:45.009 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:45.009 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260505 21:06:45.010 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:45.010 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:45.010 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 21:06:45.010 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:45.010 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:45.010 | INFO | Uploaded 36GiB to GPU in 2.12397s (16.9494GiB/s) | 20260505 21:06:45.010 | INFO | Downloaded 34GiB to GPU in 2.01684s (16.858GiB/s) | 20260505 21:06:45.010 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 21:06:45.010 | INFO | Total mem usage: 15.3 GiB | 20260505 21:06:45.010 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 21:06:45.010 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.7%) | 20260505 21:06:45.010 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 21:06:45.010 | INFO | High Water Breakdown: | 20260505 21:06:45.010 | INFO | 60.9% : 8.59 GiB ( 90.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 21:06:45.010 | INFO | 39% : 5.49 GiB ( 82.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 21:06:45.010 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 21:06:45.010 | INFO | ------------------------------------------ | 20260505 21:06:45.010 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:06:45.010 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 21:06:45.010 | INFO | Total mem usage: 26.5 GiB | 20260505 21:06:45.010 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 21:06:45.010 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 21:06:45.010 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 21:06:45.010 | INFO | High Water Breakdown: | 20260505 21:06:45.010 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:06:45.010 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:06:45.010 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:06:45.010 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:06:45.010 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:06:45.010 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:06:45.010 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 21:06:45.010 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 21:06:45.010 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:06:45.010 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 21:06:45.010 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 21:06:45.010 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 21:06:45.010 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 21:06:45.010 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 21:06:45.010 | INFO | ------------------------------------------ | 20260505 21:06:45.010 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:06:45.010 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 21:06:45.010 | INFO | Total mem usage: 25.6 GiB | 20260505 21:06:45.010 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 21:06:45.010 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 21:06:45.010 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 21:06:45.010 | INFO | High Water Breakdown: | 20260505 21:06:45.010 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 21:06:45.010 | INFO | ------------------------------------------ | 20260505 21:06:45.010 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:06:45.011 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 21:06:45.011 | INFO | Total mem usage: 15.7 GiB | 20260505 21:06:45.011 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 21:06:45.011 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 21:06:45.011 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 21:06:45.011 | INFO | High Water Breakdown: | 20260505 21:06:45.011 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 21:06:45.011 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 21:06:45.011 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 21:06:45.011 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 21:06:45.011 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 21:06:45.011 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 21:06:45.011 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 21:06:45.011 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 21:06:45.011 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 21:06:45.011 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:06:45.011 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 21:06:45.011 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:06:45.011 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 21:06:45.011 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 21:06:45.011 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 21:06:45.011 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 21:06:45.011 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:06:45.011 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 21:06:45.011 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:06:45.011 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:06:45.011 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:06:45.011 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 21:06:45.011 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 21:06:45.011 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:06:45.011 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 21:06:45.011 | INFO | ------------------------------------------ | 20260505 21:06:45.011 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:06:45.011 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 21:06:45.011 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:06:45.011 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:06:45.011 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.45e+01, 1.33e+01 / 1.49e+00 | 20260505 21:06:45.011 | INFO | PerfReport: 0.29%, AggregateStats -- 9.25e-02 / 1.04e-01, 1.04e-02 / 6.49e-03 | 20260505 21:06:45.011 | INFO | PerfReport: 11.78%, Baseline -- 3.71e+00 / 4.52e+00, 5.62e-01 / 1.41e-01 | 20260505 21:06:45.011 | INFO | PerfReport: 3.29%, Histogram -- 1.04e+00 / 1.51e+00, 1.90e-01 / 9.61e-02 | 20260505 21:06:45.011 | INFO | PerfReport: 15.96%, DME -- 4.02e+01 / 4.44e+01, 4.16e+00 / 1.68e+00 | 20260505 21:06:45.011 | INFO | PerfReport: 57.19%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.55e+00 / 1.10e+00 | 20260505 21:06:45.011 | INFO | PerfReport: 7.17%, PulseAccumulating -- 2.26e+00 / 2.83e+00, 4.31e-01 / 2.15e-01 | 20260505 21:06:45.011 | INFO | PerfReport: 3.20%, Metrics -- 1.01e+00 / 1.31e+00, 2.71e-01 / 1.29e-01 | 20260505 21:06:45.011 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.51e-01 / 4.15e-01, 6.23e-02 / 4.20e-02 | 20260505 21:06:45.011 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 21:06:45.011 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:06:45.011 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:06:45.011 | INFO | PerfReport: Full Execution -- 9.91e+01 / -nan, 2.51e+01 / 0.00e+00 | 20260505 21:06:45.011 | INFO | PerfReport: 22.41%, Upload -- 2.22e+01 / 0.00e+00, 7.63e+00 / 0.00e+00 | 20260505 21:06:45.011 | INFO | PerfReport: 21.79%, Download -- 2.16e+01 / 0.00e+00, 7.63e+00 / 0.00e+00 | 20260505 21:06:45.011 | INFO | PerfReport: 31.77%, Compute -- 3.15e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260505 21:06:45.011 | INFO | PerfReport: 2.23%, LockUpload -- 2.21e+00 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260505 21:06:45.011 | INFO | PerfReport: 1.37%, LockDownload -- 1.36e+00 / 0.00e+00, 2.95e+00 / 0.00e+00 | 20260505 21:06:45.011 | INFO | PerfReport: 20.42%, LockCompute -- 2.02e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260505 21:06:45.011 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.29e-02 / 0.00e+00 | 20260505 21:06:45.138 | INFO | Starting a round of Baz IO | 20260505 21:06:48.286 | INFO | Populating RTMetrics report | 20260505 21:06:48.286 | INFO | RTMetrics report populated | 20260505 21:06:48.286 | INFO | Opening tmp RTMetrics json file | 20260505 21:06:48.287 | INFO | Serializing RTMetrics to json | 20260505 21:06:48.289 | INFO | Serialization done, swapping tmp file now... | 20260505 21:06:48.289 | INFO | RTMetrics done | 20260505 21:06:48.370 | INFO | Finished a round of Baz IO: Wrote 1.982 GiB in 3.231 seconds | 20260505 21:06:48.370 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.61% / 0.00% / 78.39% | 20260505 21:06:48.370 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.155751% / 3.97364e-06% / 7.15256e-05% | 20260505 21:06:48.370 | INFO | Baz IO overall: | 20260505 21:06:48.370 | INFO | BytesWritten : 3661372228454 | 20260505 21:06:48.370 | INFO | NumEvents : 1729563488848 | 20260505 21:06:50.154 | INFO | Analyzing chunk frames = [6307840,6308352) | 20260505 21:06:50.154 | INFO | MemInfo: { | 20260505 21:06:50.154 | INFO | "gpu" : " 17 GB", | 20260505 21:06:50.154 | INFO | "huge" : "", | 20260505 21:06:50.154 | INFO | "huge_shared" : " 27 GB", | 20260505 21:06:50.154 | INFO | "malloc" : " 14 GB", | 20260505 21:06:50.154 | INFO | "pinned" : " 29 GB" | 20260505 21:06:50.154 | INFO | } | 20260505 21:06:50.154 | INFO | Perf report for the last 5.145 seconds: | 20260505 21:06:50.154 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:06:50.154 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:50.154 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:50.155 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:50.155 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260505 21:06:50.155 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:50.155 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260505 21:06:50.155 | INFO | PRE_HQ : 59.9%, 1.8, 55.2ms, 0 | 20260505 21:06:50.155 | INFO | BAZWRITER : 62.1%, 3.7, 117.5ms, 0 | 20260505 21:06:50.155 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 21:06:50.155 | INFO | Uploaded 36GiB to GPU in 2.12289s (16.958GiB/s) | 20260505 21:06:50.155 | INFO | Downloaded 38GiB to GPU in 2.22923s (17.0463GiB/s) | 20260505 21:06:55.284 | INFO | Analyzing chunk frames = [6308352,6308864) | 20260505 21:06:55.284 | INFO | MemInfo: { | 20260505 21:06:55.284 | INFO | "gpu" : " 17 GB", | 20260505 21:06:55.284 | INFO | "huge" : "", | 20260505 21:06:55.284 | INFO | "huge_shared" : " 27 GB", | 20260505 21:06:55.284 | INFO | "malloc" : " 14 GB", | 20260505 21:06:55.284 | INFO | "pinned" : " 29 GB" | 20260505 21:06:55.284 | INFO | } | 20260505 21:06:55.284 | INFO | Perf report for the last 5.12954 seconds: | 20260505 21:06:55.284 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:06:55.284 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:55.284 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:55.284 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:55.284 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260505 21:06:55.284 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:55.284 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:55.284 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 21:06:55.284 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:55.284 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:06:55.284 | INFO | Uploaded 36GiB to GPU in 2.08867s (17.2358GiB/s) | 20260505 21:06:55.284 | INFO | Downloaded 34GiB to GPU in 2.0244s (16.7951GiB/s) | 20260505 21:07:00.414 | INFO | Analyzing chunk frames = [6308864,6309376) | 20260505 21:07:00.414 | INFO | MemInfo: { | 20260505 21:07:00.414 | INFO | "gpu" : " 17 GB", | 20260505 21:07:00.414 | INFO | "huge" : "", | 20260505 21:07:00.414 | INFO | "huge_shared" : " 27 GB", | 20260505 21:07:00.414 | INFO | "malloc" : " 14 GB", | 20260505 21:07:00.414 | INFO | "pinned" : " 29 GB" | 20260505 21:07:00.414 | INFO | } | 20260505 21:07:00.414 | INFO | Perf report for the last 5.12964 seconds: | 20260505 21:07:00.414 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:07:00.414 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:00.414 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:00.414 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:00.414 | INFO | ANALYSIS : 62.2%, 3.0, 96.5ms, 0 | 20260505 21:07:00.414 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:00.414 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:00.414 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:07:00.414 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:00.414 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:00.414 | INFO | Uploaded 36GiB to GPU in 2.12854s (16.913GiB/s) | 20260505 21:07:00.414 | INFO | Downloaded 34GiB to GPU in 2.01608s (16.8644GiB/s) | 20260505 21:07:05.561 | INFO | Analyzing chunk frames = [6309376,6309888) | 20260505 21:07:05.561 | INFO | MemInfo: { | 20260505 21:07:05.561 | INFO | "gpu" : " 17 GB", | 20260505 21:07:05.561 | INFO | "huge" : "", | 20260505 21:07:05.561 | INFO | "huge_shared" : " 27 GB", | 20260505 21:07:05.561 | INFO | "malloc" : " 14 GB", | 20260505 21:07:05.561 | INFO | "pinned" : " 29 GB" | 20260505 21:07:05.561 | INFO | } | 20260505 21:07:05.561 | INFO | Perf report for the last 5.1475 seconds: | 20260505 21:07:05.561 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:07:05.561 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:05.561 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:05.561 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:05.561 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260505 21:07:05.561 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:05.561 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:05.561 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:07:05.561 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:05.561 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:05.562 | INFO | Uploaded 36GiB to GPU in 2.16976s (16.5917GiB/s) | 20260505 21:07:05.562 | INFO | Downloaded 34GiB to GPU in 2.05313s (16.5601GiB/s) | 20260505 21:07:10.707 | INFO | Analyzing chunk frames = [6309888,6310400) | 20260505 21:07:10.707 | INFO | MemInfo: { | 20260505 21:07:10.707 | INFO | "gpu" : " 17 GB", | 20260505 21:07:10.707 | INFO | "huge" : "", | 20260505 21:07:10.707 | INFO | "huge_shared" : " 27 GB", | 20260505 21:07:10.707 | INFO | "malloc" : " 14 GB", | 20260505 21:07:10.707 | INFO | "pinned" : " 29 GB" | 20260505 21:07:10.707 | INFO | } | 20260505 21:07:10.707 | INFO | Perf report for the last 5.14612 seconds: | 20260505 21:07:10.707 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:07:10.707 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:10.707 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:10.707 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:10.707 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260505 21:07:10.707 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:10.707 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:10.708 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:07:10.708 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:10.708 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:10.708 | INFO | Uploaded 36GiB to GPU in 2.13162s (16.8886GiB/s) | 20260505 21:07:10.708 | INFO | Downloaded 34GiB to GPU in 2.06393s (16.4734GiB/s) | 20260505 21:07:15.845 | INFO | Analyzing chunk frames = [6310400,6310912) | 20260505 21:07:15.845 | INFO | MemInfo: { | 20260505 21:07:15.845 | INFO | "gpu" : " 17 GB", | 20260505 21:07:15.845 | INFO | "huge" : "", | 20260505 21:07:15.845 | INFO | "huge_shared" : " 27 GB", | 20260505 21:07:15.845 | INFO | "malloc" : " 14 GB", | 20260505 21:07:15.845 | INFO | "pinned" : " 29 GB" | 20260505 21:07:15.845 | INFO | } | 20260505 21:07:15.846 | INFO | Perf report for the last 5.13819 seconds: | 20260505 21:07:15.846 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:07:15.846 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:15.846 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:15.846 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:15.846 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260505 21:07:15.846 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:15.846 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:15.846 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:07:15.846 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:15.846 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:15.846 | INFO | Uploaded 36GiB to GPU in 2.08167s (17.2938GiB/s) | 20260505 21:07:15.846 | INFO | Downloaded 34GiB to GPU in 2.04317s (16.6408GiB/s) | 20260505 21:07:20.983 | INFO | Analyzing chunk frames = [6310912,6311424) | 20260505 21:07:20.983 | INFO | MemInfo: { | 20260505 21:07:20.983 | INFO | "gpu" : " 17 GB", | 20260505 21:07:20.983 | INFO | "huge" : "", | 20260505 21:07:20.983 | INFO | "huge_shared" : " 27 GB", | 20260505 21:07:20.983 | INFO | "malloc" : " 14 GB", | 20260505 21:07:20.983 | INFO | "pinned" : " 29 GB" | 20260505 21:07:20.983 | INFO | } | 20260505 21:07:20.983 | INFO | Perf report for the last 5.13748 seconds: | 20260505 21:07:20.983 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:07:20.983 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:20.983 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:20.983 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:20.983 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 21:07:20.983 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:20.983 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:20.983 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 21:07:20.983 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:20.983 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:20.983 | INFO | Uploaded 36GiB to GPU in 2.13462s (16.8648GiB/s) | 20260505 21:07:20.983 | INFO | Downloaded 34GiB to GPU in 2.10335s (16.1647GiB/s) | 20260505 21:07:26.117 | INFO | Analyzing chunk frames = [6311424,6311936) | 20260505 21:07:26.117 | INFO | MemInfo: { | 20260505 21:07:26.117 | INFO | "gpu" : " 17 GB", | 20260505 21:07:26.117 | INFO | "huge" : "", | 20260505 21:07:26.117 | INFO | "huge_shared" : " 27 GB", | 20260505 21:07:26.117 | INFO | "malloc" : " 14 GB", | 20260505 21:07:26.117 | INFO | "pinned" : " 29 GB" | 20260505 21:07:26.117 | INFO | } | 20260505 21:07:26.117 | INFO | Perf report for the last 5.13384 seconds: | 20260505 21:07:26.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:07:26.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:26.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:26.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:26.117 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260505 21:07:26.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:26.117 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:26.117 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260505 21:07:26.117 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:26.117 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:26.117 | INFO | Uploaded 36GiB to GPU in 2.1162s (17.0116GiB/s) | 20260505 21:07:26.117 | INFO | Downloaded 34GiB to GPU in 2.01514s (16.8723GiB/s) | 20260505 21:07:26.244 | INFO | Starting a round of Baz IO | 20260505 21:07:29.416 | INFO | Populating RTMetrics report | 20260505 21:07:29.417 | INFO | RTMetrics report populated | 20260505 21:07:29.417 | INFO | Opening tmp RTMetrics json file | 20260505 21:07:29.417 | INFO | Serializing RTMetrics to json | 20260505 21:07:29.418 | INFO | Serialization done, swapping tmp file now... | 20260505 21:07:29.419 | INFO | RTMetrics done | 20260505 21:07:29.504 | INFO | Finished a round of Baz IO: Wrote 1.984 GiB in 3.259 seconds | 20260505 21:07:29.505 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.60% / 0.00% / 78.39% | 20260505 21:07:29.505 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.155425% / 0% / 7.94729e-05% | 20260505 21:07:29.505 | INFO | Baz IO overall: | 20260505 21:07:29.505 | INFO | BytesWritten : 3663502194316 | 20260505 21:07:29.505 | INFO | NumEvents : 1730548313717 | 20260505 21:07:31.261 | INFO | Analyzing chunk frames = [6311936,6312448) | 20260505 21:07:31.261 | INFO | MemInfo: { | 20260505 21:07:31.261 | INFO | "gpu" : " 17 GB", | 20260505 21:07:31.261 | INFO | "huge" : "", | 20260505 21:07:31.261 | INFO | "huge_shared" : " 27 GB", | 20260505 21:07:31.261 | INFO | "malloc" : " 14 GB", | 20260505 21:07:31.261 | INFO | "pinned" : " 29 GB" | 20260505 21:07:31.261 | INFO | } | 20260505 21:07:31.261 | INFO | Perf report for the last 5.14447 seconds: | 20260505 21:07:31.261 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:07:31.261 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:31.261 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:31.261 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:31.261 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 21:07:31.261 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:31.261 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260505 21:07:31.261 | INFO | PRE_HQ : 60.2%, 1.8, 55.2ms, 0 | 20260505 21:07:31.261 | INFO | BAZWRITER : 62.8%, 3.7, 118.4ms, 0 | 20260505 21:07:31.261 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 21:07:31.262 | INFO | Uploaded 36GiB to GPU in 2.1602s (16.6652GiB/s) | 20260505 21:07:31.262 | INFO | Downloaded 38GiB to GPU in 2.25019s (16.8875GiB/s) | 20260505 21:07:36.399 | INFO | Analyzing chunk frames = [6312448,6312960) | 20260505 21:07:36.399 | INFO | MemInfo: { | 20260505 21:07:36.399 | INFO | "gpu" : " 17 GB", | 20260505 21:07:36.399 | INFO | "huge" : "", | 20260505 21:07:36.399 | INFO | "huge_shared" : " 27 GB", | 20260505 21:07:36.399 | INFO | "malloc" : " 14 GB", | 20260505 21:07:36.399 | INFO | "pinned" : " 29 GB" | 20260505 21:07:36.399 | INFO | } | 20260505 21:07:36.399 | INFO | Perf report for the last 5.1379 seconds: | 20260505 21:07:36.399 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:07:36.399 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:36.399 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:36.399 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:36.399 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 21:07:36.399 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:36.399 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:36.399 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:07:36.399 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:36.399 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:36.400 | INFO | Uploaded 36GiB to GPU in 2.08224s (17.2891GiB/s) | 20260505 21:07:36.400 | INFO | Downloaded 34GiB to GPU in 2.02552s (16.7858GiB/s) | 20260505 21:07:41.536 | INFO | Analyzing chunk frames = [6312960,6313472) | 20260505 21:07:41.536 | INFO | MemInfo: { | 20260505 21:07:41.536 | INFO | "gpu" : " 17 GB", | 20260505 21:07:41.536 | INFO | "huge" : "", | 20260505 21:07:41.536 | INFO | "huge_shared" : " 27 GB", | 20260505 21:07:41.536 | INFO | "malloc" : " 14 GB", | 20260505 21:07:41.536 | INFO | "pinned" : " 29 GB" | 20260505 21:07:41.536 | INFO | } | 20260505 21:07:41.536 | INFO | Perf report for the last 5.13722 seconds: | 20260505 21:07:41.536 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:07:41.536 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:41.536 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:41.536 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:41.537 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260505 21:07:41.537 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:41.537 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:41.537 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:07:41.537 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:41.537 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:41.537 | INFO | Uploaded 36GiB to GPU in 2.1273s (16.9229GiB/s) | 20260505 21:07:41.537 | INFO | Downloaded 34GiB to GPU in 2.01837s (16.8453GiB/s) | 20260505 21:07:46.674 | INFO | Analyzing chunk frames = [6313472,6313984) | 20260505 21:07:46.674 | INFO | MemInfo: { | 20260505 21:07:46.674 | INFO | "gpu" : " 17 GB", | 20260505 21:07:46.674 | INFO | "huge" : "", | 20260505 21:07:46.674 | INFO | "huge_shared" : " 27 GB", | 20260505 21:07:46.674 | INFO | "malloc" : " 14 GB", | 20260505 21:07:46.674 | INFO | "pinned" : " 29 GB" | 20260505 21:07:46.674 | INFO | } | 20260505 21:07:46.674 | INFO | Perf report for the last 5.13763 seconds: | 20260505 21:07:46.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:07:46.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:46.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:46.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:46.674 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260505 21:07:46.674 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:46.674 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:46.674 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260505 21:07:46.674 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:46.674 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:46.674 | INFO | Uploaded 36GiB to GPU in 2.2013s (16.354GiB/s) | 20260505 21:07:46.674 | INFO | Downloaded 34GiB to GPU in 2.04452s (16.6298GiB/s) | 20260505 21:07:51.819 | INFO | Analyzing chunk frames = [6313984,6314496) | 20260505 21:07:51.819 | INFO | MemInfo: { | 20260505 21:07:51.819 | INFO | "gpu" : " 17 GB", | 20260505 21:07:51.819 | INFO | "huge" : "", | 20260505 21:07:51.819 | INFO | "huge_shared" : " 27 GB", | 20260505 21:07:51.819 | INFO | "malloc" : " 14 GB", | 20260505 21:07:51.819 | INFO | "pinned" : " 29 GB" | 20260505 21:07:51.819 | INFO | } | 20260505 21:07:51.819 | INFO | Perf report for the last 5.1446 seconds: | 20260505 21:07:51.819 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:07:51.819 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:51.819 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:51.819 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:51.819 | INFO | ANALYSIS : 61.8%, 3.0, 96.6ms, 0 | 20260505 21:07:51.819 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:51.819 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:51.819 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 21:07:51.819 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:51.819 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:51.819 | INFO | Uploaded 36GiB to GPU in 2.13183s (16.8869GiB/s) | 20260505 21:07:51.819 | INFO | Downloaded 34GiB to GPU in 2.06478s (16.4666GiB/s) | 20260505 21:07:56.962 | INFO | Analyzing chunk frames = [6314496,6315008) | 20260505 21:07:56.962 | INFO | MemInfo: { | 20260505 21:07:56.962 | INFO | "gpu" : " 17 GB", | 20260505 21:07:56.962 | INFO | "huge" : "", | 20260505 21:07:56.962 | INFO | "huge_shared" : " 27 GB", | 20260505 21:07:56.962 | INFO | "malloc" : " 14 GB", | 20260505 21:07:56.962 | INFO | "pinned" : " 29 GB" | 20260505 21:07:56.962 | INFO | } | 20260505 21:07:56.962 | INFO | Perf report for the last 5.14311 seconds: | 20260505 21:07:56.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:07:56.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:56.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:56.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:56.962 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260505 21:07:56.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:56.962 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:56.962 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:07:56.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:56.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:07:56.962 | INFO | Uploaded 36GiB to GPU in 2.08211s (17.2901GiB/s) | 20260505 21:07:56.962 | INFO | Downloaded 34GiB to GPU in 2.03406s (16.7154GiB/s) | 20260505 21:08:02.097 | INFO | Analyzing chunk frames = [6315008,6315520) | 20260505 21:08:02.097 | INFO | MemInfo: { | 20260505 21:08:02.097 | INFO | "gpu" : " 17 GB", | 20260505 21:08:02.097 | INFO | "huge" : "", | 20260505 21:08:02.097 | INFO | "huge_shared" : " 27 GB", | 20260505 21:08:02.097 | INFO | "malloc" : " 14 GB", | 20260505 21:08:02.097 | INFO | "pinned" : " 29 GB" | 20260505 21:08:02.097 | INFO | } | 20260505 21:08:02.097 | INFO | Perf report for the last 5.13546 seconds: | 20260505 21:08:02.097 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:08:02.097 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:02.097 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:02.097 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:02.097 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 21:08:02.097 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:02.097 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:02.097 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 21:08:02.097 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:02.097 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:02.098 | INFO | Uploaded 36GiB to GPU in 2.13126s (16.8914GiB/s) | 20260505 21:08:02.098 | INFO | Downloaded 34GiB to GPU in 2.10421s (16.1581GiB/s) | 20260505 21:08:07.229 | INFO | Analyzing chunk frames = [6315520,6316032) | 20260505 21:08:07.229 | INFO | MemInfo: { | 20260505 21:08:07.229 | INFO | "gpu" : " 17 GB", | 20260505 21:08:07.229 | INFO | "huge" : "", | 20260505 21:08:07.229 | INFO | "huge_shared" : " 27 GB", | 20260505 21:08:07.229 | INFO | "malloc" : " 14 GB", | 20260505 21:08:07.230 | INFO | "pinned" : " 29 GB" | 20260505 21:08:07.230 | INFO | } | 20260505 21:08:07.230 | INFO | Perf report for the last 5.13222 seconds: | 20260505 21:08:07.230 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:08:07.230 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:07.230 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:07.230 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:07.230 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 21:08:07.230 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:07.230 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:07.230 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:08:07.230 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:07.230 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:07.230 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 21:08:07.230 | INFO | Total mem usage: 15.3 GiB | 20260505 21:08:07.230 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 21:08:07.230 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.16%) | 20260505 21:08:07.230 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 21:08:07.230 | INFO | Uploaded 36GiB to GPU in 2.10632s (17.0914GiB/s) | 20260505 21:08:07.230 | INFO | High Water Breakdown: | 20260505 21:08:07.230 | INFO | Downloaded 34GiB to GPU in 2.00416s (16.9647GiB/s) | 20260505 21:08:07.230 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 21:08:07.230 | INFO | 38.8% : 5.5 GiB ( 82.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 21:08:07.230 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 21:08:07.230 | INFO | ------------------------------------------ | 20260505 21:08:07.230 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:08:07.230 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 21:08:07.230 | INFO | Total mem usage: 26.5 GiB | 20260505 21:08:07.230 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 21:08:07.230 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 21:08:07.230 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 21:08:07.230 | INFO | High Water Breakdown: | 20260505 21:08:07.230 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:08:07.230 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:08:07.230 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:08:07.230 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:08:07.230 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:08:07.230 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:08:07.230 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 21:08:07.230 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 21:08:07.230 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:08:07.230 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 21:08:07.230 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 21:08:07.230 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 21:08:07.230 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 21:08:07.230 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 21:08:07.230 | INFO | ------------------------------------------ | 20260505 21:08:07.230 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:08:07.230 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 21:08:07.230 | INFO | Total mem usage: 25.6 GiB | 20260505 21:08:07.230 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 21:08:07.230 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 21:08:07.230 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 21:08:07.230 | INFO | High Water Breakdown: | 20260505 21:08:07.230 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 21:08:07.230 | INFO | ------------------------------------------ | 20260505 21:08:07.230 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:08:07.230 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 21:08:07.230 | INFO | Total mem usage: 15.7 GiB | 20260505 21:08:07.230 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 21:08:07.230 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 21:08:07.230 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 21:08:07.230 | INFO | High Water Breakdown: | 20260505 21:08:07.230 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 21:08:07.230 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 21:08:07.230 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 21:08:07.230 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 21:08:07.230 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 21:08:07.230 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 21:08:07.230 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 21:08:07.230 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 21:08:07.231 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 21:08:07.231 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:08:07.231 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 21:08:07.231 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:08:07.231 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 21:08:07.231 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 21:08:07.231 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 21:08:07.231 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 21:08:07.231 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:08:07.231 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 21:08:07.231 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:08:07.231 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:08:07.231 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:08:07.231 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 21:08:07.231 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 21:08:07.231 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:08:07.231 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 21:08:07.231 | INFO | ------------------------------------------ | 20260505 21:08:07.231 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:08:07.231 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 21:08:07.231 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:08:07.231 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:08:07.231 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.40e+01, 1.34e+01 / 7.89e-01 | 20260505 21:08:07.231 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.02e-01, 1.03e-02 / 3.89e-03 | 20260505 21:08:07.231 | INFO | PerfReport: 11.80%, Baseline -- 3.71e+00 / 4.53e+00, 5.63e-01 / 2.09e-01 | 20260505 21:08:07.231 | INFO | PerfReport: 3.29%, Histogram -- 1.03e+00 / 1.52e+00, 1.92e-01 / 1.15e-01 | 20260505 21:08:07.231 | INFO | PerfReport: 15.96%, DME -- 4.01e+01 / 4.40e+01, 3.77e+00 / 8.58e-01 | 20260505 21:08:07.231 | INFO | PerfReport: 57.12%, FrameLabeling -- 1.80e+01 / 1.93e+01, 1.50e+00 / 1.12e+00 | 20260505 21:08:07.231 | INFO | PerfReport: 7.21%, PulseAccumulating -- 2.27e+00 / 2.81e+00, 4.33e-01 / 2.40e-01 | 20260505 21:08:07.231 | INFO | PerfReport: 3.22%, Metrics -- 1.01e+00 / 1.32e+00, 2.78e-01 / 1.48e-01 | 20260505 21:08:07.231 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.51e-01 / 4.05e-01, 6.11e-02 / 4.16e-02 | 20260505 21:08:07.231 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 21:08:07.231 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:08:07.231 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:08:07.231 | INFO | PerfReport: Full Execution -- 9.90e+01 / -nan, 2.50e+01 / 0.00e+00 | 20260505 21:08:07.231 | INFO | PerfReport: 22.35%, Upload -- 2.21e+01 / 0.00e+00, 7.60e+00 / 0.00e+00 | 20260505 21:08:07.231 | INFO | PerfReport: 21.75%, Download -- 2.15e+01 / 0.00e+00, 7.58e+00 / 0.00e+00 | 20260505 21:08:07.231 | INFO | PerfReport: 31.75%, Compute -- 3.15e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 21:08:07.231 | INFO | PerfReport: 2.21%, LockUpload -- 2.19e+00 / 0.00e+00, 7.82e+00 / 0.00e+00 | 20260505 21:08:07.231 | INFO | PerfReport: 1.37%, LockDownload -- 1.36e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260505 21:08:07.231 | INFO | PerfReport: 20.55%, LockCompute -- 2.04e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 21:08:07.231 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.80e-02 / 0.00e+00 | 20260505 21:08:07.362 | INFO | Starting a round of Baz IO | 20260505 21:08:10.529 | INFO | Populating RTMetrics report | 20260505 21:08:10.530 | INFO | RTMetrics report populated | 20260505 21:08:10.530 | INFO | Opening tmp RTMetrics json file | 20260505 21:08:10.531 | INFO | Serializing RTMetrics to json | 20260505 21:08:10.532 | INFO | Serialization done, swapping tmp file now... | 20260505 21:08:10.533 | INFO | RTMetrics done | 20260505 21:08:10.613 | INFO | Finished a round of Baz IO: Wrote 1.983 GiB in 3.251 seconds | 20260505 21:08:10.613 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.60% / 0.00% / 78.40% | 20260505 21:08:10.613 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.155807% / 3.97364e-06% / 0.000111262% | 20260505 21:08:10.613 | INFO | Baz IO overall: | 20260505 21:08:10.613 | INFO | BytesWritten : 3665631504302 | 20260505 21:08:10.613 | INFO | NumEvents : 1731532699857 | 20260505 21:08:12.367 | INFO | Analyzing chunk frames = [6316032,6316544) | 20260505 21:08:12.367 | INFO | MemInfo: { | 20260505 21:08:12.367 | INFO | "gpu" : " 17 GB", | 20260505 21:08:12.367 | INFO | "huge" : "", | 20260505 21:08:12.367 | INFO | "huge_shared" : " 27 GB", | 20260505 21:08:12.367 | INFO | "malloc" : " 14 GB", | 20260505 21:08:12.367 | INFO | "pinned" : " 29 GB" | 20260505 21:08:12.367 | INFO | } | 20260505 21:08:12.368 | INFO | Perf report for the last 5.13796 seconds: | 20260505 21:08:12.368 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:08:12.368 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:12.368 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:12.368 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:12.368 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 21:08:12.368 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:12.368 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260505 21:08:12.368 | INFO | PRE_HQ : 60.1%, 1.8, 55.0ms, 0 | 20260505 21:08:12.368 | INFO | BAZWRITER : 62.8%, 3.6, 117.5ms, 0 | 20260505 21:08:12.368 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 21:08:12.368 | INFO | Uploaded 36GiB to GPU in 2.15852s (16.6781GiB/s) | 20260505 21:08:12.368 | INFO | Downloaded 38GiB to GPU in 2.23841s (16.9763GiB/s) | 20260505 21:08:17.517 | INFO | Analyzing chunk frames = [6316544,6317056) | 20260505 21:08:17.517 | INFO | MemInfo: { | 20260505 21:08:17.517 | INFO | "gpu" : " 17 GB", | 20260505 21:08:17.517 | INFO | "huge" : "", | 20260505 21:08:17.517 | INFO | "huge_shared" : " 27 GB", | 20260505 21:08:17.517 | INFO | "malloc" : " 14 GB", | 20260505 21:08:17.517 | INFO | "pinned" : " 29 GB" | 20260505 21:08:17.517 | INFO | } | 20260505 21:08:17.517 | INFO | Perf report for the last 5.14969 seconds: | 20260505 21:08:17.517 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:08:17.517 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:17.517 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:17.517 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:17.517 | INFO | ANALYSIS : 62.1%, 3.0, 97.1ms, 0 | 20260505 21:08:17.517 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:17.517 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:17.517 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:08:17.517 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:17.517 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:17.517 | INFO | Uploaded 36GiB to GPU in 2.10087s (17.1358GiB/s) | 20260505 21:08:17.517 | INFO | Downloaded 34GiB to GPU in 2.02615s (16.7806GiB/s) | 20260505 21:08:22.658 | INFO | Analyzing chunk frames = [6317056,6317568) | 20260505 21:08:22.658 | INFO | MemInfo: { | 20260505 21:08:22.658 | INFO | "gpu" : " 17 GB", | 20260505 21:08:22.658 | INFO | "huge" : "", | 20260505 21:08:22.658 | INFO | "huge_shared" : " 27 GB", | 20260505 21:08:22.658 | INFO | "malloc" : " 14 GB", | 20260505 21:08:22.658 | INFO | "pinned" : " 29 GB" | 20260505 21:08:22.658 | INFO | } | 20260505 21:08:22.658 | INFO | Perf report for the last 5.14106 seconds: | 20260505 21:08:22.658 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:08:22.658 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:22.658 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:22.658 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:22.658 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260505 21:08:22.658 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:22.658 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:22.658 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:08:22.658 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:22.658 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:22.659 | INFO | Uploaded 36GiB to GPU in 2.13831s (16.8357GiB/s) | 20260505 21:08:22.659 | INFO | Downloaded 34GiB to GPU in 2.03036s (16.7458GiB/s) | 20260505 21:08:27.793 | INFO | Analyzing chunk frames = [6317568,6318080) | 20260505 21:08:27.793 | INFO | MemInfo: { | 20260505 21:08:27.793 | INFO | "gpu" : " 17 GB", | 20260505 21:08:27.793 | INFO | "huge" : "", | 20260505 21:08:27.793 | INFO | "huge_shared" : " 27 GB", | 20260505 21:08:27.793 | INFO | "malloc" : " 14 GB", | 20260505 21:08:27.793 | INFO | "pinned" : " 29 GB" | 20260505 21:08:27.793 | INFO | } | 20260505 21:08:27.793 | INFO | Perf report for the last 5.13463 seconds: | 20260505 21:08:27.793 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:08:27.793 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:27.793 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:27.793 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:27.793 | INFO | ANALYSIS : 61.8%, 3.0, 95.9ms, 0 | 20260505 21:08:27.793 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:27.793 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:27.793 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:08:27.793 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:27.793 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:27.793 | INFO | Uploaded 36GiB to GPU in 2.13124s (16.8916GiB/s) | 20260505 21:08:27.793 | INFO | Downloaded 34GiB to GPU in 2.01356s (16.8855GiB/s) | 20260505 21:08:32.931 | INFO | Analyzing chunk frames = [6318080,6318592) | 20260505 21:08:32.931 | INFO | MemInfo: { | 20260505 21:08:32.931 | INFO | "gpu" : " 17 GB", | 20260505 21:08:32.931 | INFO | "huge" : "", | 20260505 21:08:32.931 | INFO | "huge_shared" : " 27 GB", | 20260505 21:08:32.931 | INFO | "malloc" : " 14 GB", | 20260505 21:08:32.931 | INFO | "pinned" : " 29 GB" | 20260505 21:08:32.931 | INFO | } | 20260505 21:08:32.931 | INFO | Perf report for the last 5.13847 seconds: | 20260505 21:08:32.931 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:08:32.931 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:32.931 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:32.931 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:32.932 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 21:08:32.932 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:32.932 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:32.932 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 21:08:32.932 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:32.932 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:32.932 | INFO | Uploaded 36GiB to GPU in 2.13043s (16.898GiB/s) | 20260505 21:08:32.932 | INFO | Downloaded 34GiB to GPU in 2.06542s (16.4615GiB/s) | 20260505 21:08:38.076 | INFO | Analyzing chunk frames = [6318592,6319104) | 20260505 21:08:38.076 | INFO | MemInfo: { | 20260505 21:08:38.076 | INFO | "gpu" : " 17 GB", | 20260505 21:08:38.076 | INFO | "huge" : "", | 20260505 21:08:38.076 | INFO | "huge_shared" : " 27 GB", | 20260505 21:08:38.076 | INFO | "malloc" : " 14 GB", | 20260505 21:08:38.076 | INFO | "pinned" : " 29 GB" | 20260505 21:08:38.076 | INFO | } | 20260505 21:08:38.076 | INFO | Perf report for the last 5.14493 seconds: | 20260505 21:08:38.076 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:08:38.076 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:38.076 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:38.076 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:38.076 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260505 21:08:38.076 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:38.076 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:38.076 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 21:08:38.076 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:38.077 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:38.077 | INFO | Uploaded 36GiB to GPU in 2.13854s (16.8339GiB/s) | 20260505 21:08:38.077 | INFO | Downloaded 34GiB to GPU in 2.06073s (16.499GiB/s) | 20260505 21:08:43.210 | INFO | Analyzing chunk frames = [6319104,6319616) | 20260505 21:08:43.210 | INFO | MemInfo: { | 20260505 21:08:43.210 | INFO | "gpu" : " 17 GB", | 20260505 21:08:43.210 | INFO | "huge" : "", | 20260505 21:08:43.210 | INFO | "huge_shared" : " 27 GB", | 20260505 21:08:43.210 | INFO | "malloc" : " 14 GB", | 20260505 21:08:43.210 | INFO | "pinned" : " 29 GB" | 20260505 21:08:43.210 | INFO | } | 20260505 21:08:43.210 | INFO | Perf report for the last 5.13343 seconds: | 20260505 21:08:43.210 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:08:43.210 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:43.210 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:43.210 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:43.210 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 21:08:43.210 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:43.210 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:43.210 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 21:08:43.210 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:43.210 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:43.210 | INFO | Uploaded 36GiB to GPU in 2.11712s (17.0043GiB/s) | 20260505 21:08:43.210 | INFO | Downloaded 34GiB to GPU in 2.0886s (16.2789GiB/s) | 20260505 21:08:48.348 | INFO | Analyzing chunk frames = [6319616,6320128) | 20260505 21:08:48.348 | INFO | MemInfo: { | 20260505 21:08:48.348 | INFO | "gpu" : " 17 GB", | 20260505 21:08:48.348 | INFO | "huge" : "", | 20260505 21:08:48.348 | INFO | "huge_shared" : " 27 GB", | 20260505 21:08:48.348 | INFO | "malloc" : " 14 GB", | 20260505 21:08:48.348 | INFO | "pinned" : " 29 GB" | 20260505 21:08:48.348 | INFO | } | 20260505 21:08:48.348 | INFO | Perf report for the last 5.13801 seconds: | 20260505 21:08:48.348 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:08:48.348 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:48.348 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:48.348 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:48.348 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 21:08:48.348 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:48.348 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:48.348 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 21:08:48.348 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:48.348 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:48.348 | INFO | Uploaded 36GiB to GPU in 2.11289s (17.0383GiB/s) | 20260505 21:08:48.348 | INFO | Downloaded 34GiB to GPU in 2.01166s (16.9015GiB/s) | 20260505 21:08:48.483 | INFO | Starting a round of Baz IO | 20260505 21:08:51.635 | INFO | Populating RTMetrics report | 20260505 21:08:51.635 | INFO | RTMetrics report populated | 20260505 21:08:51.635 | INFO | Opening tmp RTMetrics json file | 20260505 21:08:51.636 | INFO | Serializing RTMetrics to json | 20260505 21:08:51.638 | INFO | Serialization done, swapping tmp file now... | 20260505 21:08:51.638 | INFO | RTMetrics done | 20260505 21:08:51.721 | INFO | Finished a round of Baz IO: Wrote 1.983 GiB in 3.238 seconds | 20260505 21:08:51.721 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.60% / 0.00% / 78.40% | 20260505 21:08:51.721 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.154908% / 0% / 6.75519e-05% | 20260505 21:08:51.721 | INFO | Baz IO overall: | 20260505 21:08:51.722 | INFO | BytesWritten : 3667760376219 | 20260505 21:08:51.722 | INFO | NumEvents : 1732516694621 | 20260505 21:08:53.486 | INFO | Analyzing chunk frames = [6320128,6320640) | 20260505 21:08:53.486 | INFO | MemInfo: { | 20260505 21:08:53.486 | INFO | "gpu" : " 17 GB", | 20260505 21:08:53.486 | INFO | "huge" : "", | 20260505 21:08:53.487 | INFO | "huge_shared" : " 27 GB", | 20260505 21:08:53.487 | INFO | "malloc" : " 14 GB", | 20260505 21:08:53.487 | INFO | "pinned" : " 29 GB" | 20260505 21:08:53.487 | INFO | } | 20260505 21:08:53.487 | INFO | Perf report for the last 5.13873 seconds: | 20260505 21:08:53.487 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:08:53.487 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:53.487 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:53.487 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:53.487 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 21:08:53.487 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:53.487 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260505 21:08:53.487 | INFO | PRE_HQ : 60.0%, 1.8, 54.8ms, 0 | 20260505 21:08:53.487 | INFO | BAZWRITER : 62.5%, 3.7, 118.1ms, 0 | 20260505 21:08:53.487 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 21:08:53.487 | INFO | Uploaded 36GiB to GPU in 2.16497s (16.6284GiB/s) | 20260505 21:08:53.487 | INFO | Downloaded 38GiB to GPU in 2.2442s (16.9325GiB/s) | 20260505 21:08:58.623 | INFO | Analyzing chunk frames = [6320640,6321152) | 20260505 21:08:58.623 | INFO | MemInfo: { | 20260505 21:08:58.623 | INFO | "gpu" : " 17 GB", | 20260505 21:08:58.623 | INFO | "huge" : "", | 20260505 21:08:58.623 | INFO | "huge_shared" : " 27 GB", | 20260505 21:08:58.623 | INFO | "malloc" : " 14 GB", | 20260505 21:08:58.623 | INFO | "pinned" : " 29 GB" | 20260505 21:08:58.623 | INFO | } | 20260505 21:08:58.624 | INFO | Perf report for the last 5.13691 seconds: | 20260505 21:08:58.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:08:58.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:58.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:58.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:58.624 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 21:08:58.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:58.624 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:58.624 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 21:08:58.624 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:58.624 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:08:58.624 | INFO | Uploaded 36GiB to GPU in 2.07859s (17.3194GiB/s) | 20260505 21:08:58.624 | INFO | Downloaded 34GiB to GPU in 2.02229s (16.8126GiB/s) | 20260505 21:09:03.772 | INFO | Analyzing chunk frames = [6321152,6321664) | 20260505 21:09:03.772 | INFO | MemInfo: { | 20260505 21:09:03.772 | INFO | "gpu" : " 17 GB", | 20260505 21:09:03.772 | INFO | "huge" : "", | 20260505 21:09:03.772 | INFO | "huge_shared" : " 27 GB", | 20260505 21:09:03.772 | INFO | "malloc" : " 14 GB", | 20260505 21:09:03.772 | INFO | "pinned" : " 29 GB" | 20260505 21:09:03.772 | INFO | } | 20260505 21:09:03.772 | INFO | Perf report for the last 5.14891 seconds: | 20260505 21:09:03.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:09:03.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:03.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:03.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:03.773 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 21:09:03.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:03.773 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:03.773 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:09:03.773 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:03.773 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:03.773 | INFO | Uploaded 36GiB to GPU in 2.13837s (16.8352GiB/s) | 20260505 21:09:03.773 | INFO | Downloaded 34GiB to GPU in 2.02395s (16.7988GiB/s) | 20260505 21:09:08.914 | INFO | Analyzing chunk frames = [6321664,6322176) | 20260505 21:09:08.914 | INFO | MemInfo: { | 20260505 21:09:08.914 | INFO | "gpu" : " 17 GB", | 20260505 21:09:08.914 | INFO | "huge" : "", | 20260505 21:09:08.914 | INFO | "huge_shared" : " 27 GB", | 20260505 21:09:08.914 | INFO | "malloc" : " 14 GB", | 20260505 21:09:08.914 | INFO | "pinned" : " 29 GB" | 20260505 21:09:08.914 | INFO | } | 20260505 21:09:08.914 | INFO | Perf report for the last 5.14183 seconds: | 20260505 21:09:08.914 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:09:08.914 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:08.914 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:08.914 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:08.914 | INFO | ANALYSIS : 61.9%, 3.0, 96.2ms, 0 | 20260505 21:09:08.914 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:08.914 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:08.914 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:09:08.914 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:08.914 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:08.915 | INFO | Uploaded 36GiB to GPU in 2.13249s (16.8817GiB/s) | 20260505 21:09:08.915 | INFO | Downloaded 34GiB to GPU in 2.02496s (16.7905GiB/s) | 20260505 21:09:14.043 | INFO | Analyzing chunk frames = [6322176,6322688) | 20260505 21:09:14.043 | INFO | MemInfo: { | 20260505 21:09:14.043 | INFO | "gpu" : " 17 GB", | 20260505 21:09:14.043 | INFO | "huge" : "", | 20260505 21:09:14.043 | INFO | "huge_shared" : " 27 GB", | 20260505 21:09:14.043 | INFO | "malloc" : " 14 GB", | 20260505 21:09:14.043 | INFO | "pinned" : " 29 GB" | 20260505 21:09:14.043 | INFO | } | 20260505 21:09:14.043 | INFO | Perf report for the last 5.12876 seconds: | 20260505 21:09:14.043 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:09:14.043 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:14.043 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:14.043 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:14.043 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260505 21:09:14.043 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:14.043 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:14.043 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:09:14.043 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:14.043 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:14.043 | INFO | Uploaded 36GiB to GPU in 2.13588s (16.8549GiB/s) | 20260505 21:09:14.043 | INFO | Downloaded 34GiB to GPU in 2.0602s (16.5032GiB/s) | 20260505 21:09:19.179 | INFO | Analyzing chunk frames = [6322688,6323200) | 20260505 21:09:19.179 | INFO | MemInfo: { | 20260505 21:09:19.179 | INFO | "gpu" : " 17 GB", | 20260505 21:09:19.179 | INFO | "huge" : "", | 20260505 21:09:19.179 | INFO | "huge_shared" : " 27 GB", | 20260505 21:09:19.179 | INFO | "malloc" : " 14 GB", | 20260505 21:09:19.179 | INFO | "pinned" : " 29 GB" | 20260505 21:09:19.179 | INFO | } | 20260505 21:09:19.179 | INFO | Perf report for the last 5.13586 seconds: | 20260505 21:09:19.179 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:09:19.179 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:19.179 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:19.179 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:19.179 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 21:09:19.179 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:19.179 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:19.179 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 21:09:19.179 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:19.179 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:19.179 | INFO | Uploaded 36GiB to GPU in 2.15626s (16.6956GiB/s) | 20260505 21:09:19.179 | INFO | Downloaded 34GiB to GPU in 2.06307s (16.4803GiB/s) | 20260505 21:09:24.322 | INFO | Analyzing chunk frames = [6323200,6323712) | 20260505 21:09:24.322 | INFO | MemInfo: { | 20260505 21:09:24.322 | INFO | "gpu" : " 17 GB", | 20260505 21:09:24.322 | INFO | "huge" : "", | 20260505 21:09:24.322 | INFO | "huge_shared" : " 27 GB", | 20260505 21:09:24.322 | INFO | "malloc" : " 14 GB", | 20260505 21:09:24.322 | INFO | "pinned" : " 29 GB" | 20260505 21:09:24.322 | INFO | } | 20260505 21:09:24.323 | INFO | Perf report for the last 5.14353 seconds: | 20260505 21:09:24.323 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:09:24.323 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:24.323 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:24.323 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:24.323 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 21:09:24.323 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:24.323 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:24.323 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 21:09:24.323 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:24.323 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:24.323 | INFO | Uploaded 36GiB to GPU in 2.11761s (17.0003GiB/s) | 20260505 21:09:24.323 | INFO | Downloaded 34GiB to GPU in 2.13675s (15.912GiB/s) | 20260505 21:09:29.471 | INFO | Analyzing chunk frames = [6323712,6324224) | 20260505 21:09:29.471 | INFO | MemInfo: { | 20260505 21:09:29.471 | INFO | "gpu" : " 17 GB", | 20260505 21:09:29.471 | INFO | "huge" : "", | 20260505 21:09:29.471 | INFO | "huge_shared" : " 27 GB", | 20260505 21:09:29.471 | INFO | "malloc" : " 14 GB", | 20260505 21:09:29.471 | INFO | "pinned" : " 29 GB" | 20260505 21:09:29.471 | INFO | } | 20260505 21:09:29.471 | INFO | Perf report for the last 5.14837 seconds: | 20260505 21:09:29.471 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:09:29.471 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:29.471 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:29.471 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:29.471 | INFO | ANALYSIS : 62.1%, 3.0, 97.2ms, 0 | 20260505 21:09:29.471 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:29.471 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:29.471 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:09:29.471 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:29.471 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:29.471 | INFO | Uploaded 36GiB to GPU in 2.10169s (17.129GiB/s) | 20260505 21:09:29.471 | INFO | Downloaded 34GiB to GPU in 2.01499s (16.8735GiB/s) | 20260505 21:09:29.471 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 21:09:29.471 | INFO | Total mem usage: 15.3 GiB | 20260505 21:09:29.471 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 21:09:29.471 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.45%) | 20260505 21:09:29.471 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 21:09:29.471 | INFO | High Water Breakdown: | 20260505 21:09:29.471 | INFO | 61.1% : 8.63 GiB ( 90.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 21:09:29.471 | INFO | 38.8% : 5.48 GiB ( 82.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 21:09:29.471 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 21:09:29.471 | INFO | ------------------------------------------ | 20260505 21:09:29.471 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:09:29.472 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 21:09:29.472 | INFO | Total mem usage: 26.5 GiB | 20260505 21:09:29.472 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 21:09:29.472 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 21:09:29.472 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 21:09:29.472 | INFO | High Water Breakdown: | 20260505 21:09:29.472 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:09:29.472 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:09:29.472 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:09:29.472 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:09:29.472 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:09:29.472 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:09:29.472 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 21:09:29.472 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 21:09:29.472 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:09:29.472 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 21:09:29.472 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 21:09:29.472 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 21:09:29.472 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 21:09:29.472 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 21:09:29.472 | INFO | ------------------------------------------ | 20260505 21:09:29.472 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:09:29.472 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 21:09:29.472 | INFO | Total mem usage: 25.6 GiB | 20260505 21:09:29.472 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 21:09:29.472 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 21:09:29.472 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 21:09:29.472 | INFO | High Water Breakdown: | 20260505 21:09:29.472 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 21:09:29.472 | INFO | ------------------------------------------ | 20260505 21:09:29.472 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:09:29.472 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 21:09:29.472 | INFO | Total mem usage: 15.7 GiB | 20260505 21:09:29.472 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 21:09:29.472 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 21:09:29.472 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 21:09:29.472 | INFO | High Water Breakdown: | 20260505 21:09:29.472 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 21:09:29.472 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 21:09:29.472 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 21:09:29.472 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 21:09:29.472 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 21:09:29.472 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 21:09:29.472 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 21:09:29.472 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 21:09:29.472 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 21:09:29.472 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:09:29.472 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 21:09:29.472 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:09:29.472 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 21:09:29.472 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 21:09:29.472 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 21:09:29.472 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 21:09:29.472 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:09:29.472 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 21:09:29.472 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:09:29.472 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:09:29.472 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:09:29.472 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 21:09:29.472 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 21:09:29.472 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:09:29.472 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 21:09:29.472 | INFO | ------------------------------------------ | 20260505 21:09:29.472 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:09:29.472 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 21:09:29.472 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:09:29.472 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:09:29.472 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.44e+01, 1.34e+01 / 1.61e+00 | 20260505 21:09:29.472 | INFO | PerfReport: 0.29%, AggregateStats -- 9.24e-02 / 1.04e-01, 1.04e-02 / 5.12e-03 | 20260505 21:09:29.472 | INFO | PerfReport: 11.80%, Baseline -- 3.71e+00 / 4.56e+00, 5.70e-01 / 1.66e-01 | 20260505 21:09:29.472 | INFO | PerfReport: 3.29%, Histogram -- 1.04e+00 / 1.52e+00, 1.92e-01 / 1.00e-01 | 20260505 21:09:29.472 | INFO | PerfReport: 15.93%, DME -- 4.01e+01 / 4.42e+01, 3.95e+00 / 1.71e+00 | 20260505 21:09:29.472 | INFO | PerfReport: 57.15%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.52e+00 / 1.09e+00 | 20260505 21:09:29.472 | INFO | PerfReport: 7.19%, PulseAccumulating -- 2.26e+00 / 2.82e+00, 4.32e-01 / 2.25e-01 | 20260505 21:09:29.472 | INFO | PerfReport: 3.21%, Metrics -- 1.01e+00 / 1.32e+00, 2.78e-01 / 1.36e-01 | 20260505 21:09:29.472 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.52e-01 / 4.09e-01, 6.17e-02 / 3.77e-02 | 20260505 21:09:29.472 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 21:09:29.472 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:09:29.472 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:09:29.472 | INFO | PerfReport: Full Execution -- 9.91e+01 / -nan, 2.51e+01 / 0.00e+00 | 20260505 21:09:29.472 | INFO | PerfReport: 22.38%, Upload -- 2.22e+01 / 0.00e+00, 7.63e+00 / 0.00e+00 | 20260505 21:09:29.472 | INFO | PerfReport: 21.77%, Download -- 2.16e+01 / 0.00e+00, 7.63e+00 / 0.00e+00 | 20260505 21:09:29.472 | INFO | PerfReport: 31.76%, Compute -- 3.15e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 21:09:29.472 | INFO | PerfReport: 2.22%, LockUpload -- 2.20e+00 / 0.00e+00, 7.94e+00 / 0.00e+00 | 20260505 21:09:29.472 | INFO | PerfReport: 1.34%, LockDownload -- 1.33e+00 / 0.00e+00, 2.90e+00 / 0.00e+00 | 20260505 21:09:29.472 | INFO | PerfReport: 20.50%, LockCompute -- 2.03e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 21:09:29.472 | INFO | PerfReport: 0.02%, Misc -- 1.49e-02 / 0.00e+00, 3.38e-02 / 0.00e+00 | 20260505 21:09:29.591 | INFO | Starting a round of Baz IO | 20260505 21:09:32.746 | INFO | Populating RTMetrics report | 20260505 21:09:32.746 | INFO | RTMetrics report populated | 20260505 21:09:32.746 | INFO | Opening tmp RTMetrics json file | 20260505 21:09:32.747 | INFO | Serializing RTMetrics to json | 20260505 21:09:32.748 | INFO | Serialization done, swapping tmp file now... | 20260505 21:09:32.748 | INFO | RTMetrics done | 20260505 21:09:32.829 | INFO | Finished a round of Baz IO: Wrote 1.981 GiB in 3.238 seconds | 20260505 21:09:32.829 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.60% / 0.00% / 78.40% | 20260505 21:09:32.830 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.154654% / 0% / 8.74201e-05% | 20260505 21:09:32.830 | INFO | Baz IO overall: | 20260505 21:09:32.830 | INFO | BytesWritten : 3669887415338 | 20260505 21:09:32.830 | INFO | NumEvents : 1733499382999 | 20260505 21:09:34.604 | INFO | Analyzing chunk frames = [6324224,6324736) | 20260505 21:09:34.604 | INFO | MemInfo: { | 20260505 21:09:34.604 | INFO | "gpu" : " 17 GB", | 20260505 21:09:34.604 | INFO | "huge" : "", | 20260505 21:09:34.604 | INFO | "huge_shared" : " 27 GB", | 20260505 21:09:34.604 | INFO | "malloc" : " 14 GB", | 20260505 21:09:34.604 | INFO | "pinned" : " 29 GB" | 20260505 21:09:34.605 | INFO | } | 20260505 21:09:34.605 | INFO | Perf report for the last 5.13373 seconds: | 20260505 21:09:34.605 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:09:34.605 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:34.605 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:34.605 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:34.605 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260505 21:09:34.605 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:34.605 | INFO | RT_METRICS : 43.8%, 1.0, 22.9ms, 0 | 20260505 21:09:34.605 | INFO | PRE_HQ : 60.0%, 1.8, 54.9ms, 0 | 20260505 21:09:34.605 | INFO | BAZWRITER : 62.3%, 3.7, 117.4ms, 0 | 20260505 21:09:34.605 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 21:09:34.605 | INFO | Uploaded 36GiB to GPU in 2.12833s (16.9146GiB/s) | 20260505 21:09:34.605 | INFO | Downloaded 38GiB to GPU in 2.23614s (16.9936GiB/s) | 20260505 21:09:39.743 | INFO | Analyzing chunk frames = [6324736,6325248) | 20260505 21:09:39.743 | INFO | MemInfo: { | 20260505 21:09:39.743 | INFO | "gpu" : " 17 GB", | 20260505 21:09:39.743 | INFO | "huge" : "", | 20260505 21:09:39.743 | INFO | "huge_shared" : " 27 GB", | 20260505 21:09:39.743 | INFO | "malloc" : " 14 GB", | 20260505 21:09:39.743 | INFO | "pinned" : " 29 GB" | 20260505 21:09:39.743 | INFO | } | 20260505 21:09:39.743 | INFO | Perf report for the last 5.13842 seconds: | 20260505 21:09:39.743 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:09:39.743 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:39.743 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:39.743 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:39.743 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260505 21:09:39.743 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:39.743 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:39.743 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:09:39.743 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:39.743 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:39.743 | INFO | Uploaded 36GiB to GPU in 2.07628s (17.3387GiB/s) | 20260505 21:09:39.743 | INFO | Downloaded 34GiB to GPU in 2.02526s (16.788GiB/s) | 20260505 21:09:44.882 | INFO | Analyzing chunk frames = [6325248,6325760) | 20260505 21:09:44.882 | INFO | MemInfo: { | 20260505 21:09:44.882 | INFO | "gpu" : " 17 GB", | 20260505 21:09:44.882 | INFO | "huge" : "", | 20260505 21:09:44.882 | INFO | "huge_shared" : " 27 GB", | 20260505 21:09:44.882 | INFO | "malloc" : " 14 GB", | 20260505 21:09:44.882 | INFO | "pinned" : " 29 GB" | 20260505 21:09:44.882 | INFO | } | 20260505 21:09:44.882 | INFO | Perf report for the last 5.1393 seconds: | 20260505 21:09:44.882 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:09:44.882 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:44.882 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:44.882 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:44.882 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260505 21:09:44.882 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:44.882 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:44.882 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:09:44.882 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:44.883 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:44.883 | INFO | Uploaded 36GiB to GPU in 2.13315s (16.8765GiB/s) | 20260505 21:09:44.883 | INFO | Downloaded 34GiB to GPU in 2.02047s (16.8277GiB/s) | 20260505 21:09:50.022 | INFO | Analyzing chunk frames = [6325760,6326272) | 20260505 21:09:50.022 | INFO | MemInfo: { | 20260505 21:09:50.022 | INFO | "gpu" : " 17 GB", | 20260505 21:09:50.022 | INFO | "huge" : "", | 20260505 21:09:50.022 | INFO | "huge_shared" : " 27 GB", | 20260505 21:09:50.023 | INFO | "malloc" : " 14 GB", | 20260505 21:09:50.023 | INFO | "pinned" : " 29 GB" | 20260505 21:09:50.023 | INFO | } | 20260505 21:09:50.023 | INFO | Perf report for the last 5.14033 seconds: | 20260505 21:09:50.023 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:09:50.023 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:50.023 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:50.023 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:50.023 | INFO | ANALYSIS : 62.4%, 3.0, 96.9ms, 0 | 20260505 21:09:50.023 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:50.023 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:50.023 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:09:50.023 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:50.023 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:50.023 | INFO | Uploaded 36GiB to GPU in 2.20619s (16.3177GiB/s) | 20260505 21:09:50.023 | INFO | Downloaded 34GiB to GPU in 2.04928s (16.5912GiB/s) | 20260505 21:09:55.162 | INFO | Analyzing chunk frames = [6326272,6326784) | 20260505 21:09:55.162 | INFO | MemInfo: { | 20260505 21:09:55.162 | INFO | "gpu" : " 17 GB", | 20260505 21:09:55.162 | INFO | "huge" : "", | 20260505 21:09:55.162 | INFO | "huge_shared" : " 27 GB", | 20260505 21:09:55.162 | INFO | "malloc" : " 14 GB", | 20260505 21:09:55.162 | INFO | "pinned" : " 29 GB" | 20260505 21:09:55.162 | INFO | } | 20260505 21:09:55.162 | INFO | Perf report for the last 5.13959 seconds: | 20260505 21:09:55.162 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:09:55.162 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:55.162 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:55.162 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:55.162 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260505 21:09:55.162 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:55.162 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:55.162 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:09:55.162 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:55.162 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:09:55.163 | INFO | Uploaded 36GiB to GPU in 2.12823s (16.9155GiB/s) | 20260505 21:09:55.163 | INFO | Downloaded 34GiB to GPU in 2.06078s (16.4986GiB/s) | 20260505 21:10:00.296 | INFO | Analyzing chunk frames = [6326784,6327296) | 20260505 21:10:00.296 | INFO | MemInfo: { | 20260505 21:10:00.296 | INFO | "gpu" : " 17 GB", | 20260505 21:10:00.296 | INFO | "huge" : "", | 20260505 21:10:00.296 | INFO | "huge_shared" : " 27 GB", | 20260505 21:10:00.296 | INFO | "malloc" : " 14 GB", | 20260505 21:10:00.296 | INFO | "pinned" : " 29 GB" | 20260505 21:10:00.296 | INFO | } | 20260505 21:10:00.296 | INFO | Perf report for the last 5.13409 seconds: | 20260505 21:10:00.296 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:10:00.296 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:00.296 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:00.296 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:00.296 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260505 21:10:00.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:00.296 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:00.296 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:10:00.296 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:00.297 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:00.297 | INFO | Uploaded 36GiB to GPU in 2.08878s (17.235GiB/s) | 20260505 21:10:00.297 | INFO | Downloaded 34GiB to GPU in 2.03449s (16.7118GiB/s) | 20260505 21:10:05.433 | INFO | Analyzing chunk frames = [6327296,6327808) | 20260505 21:10:05.433 | INFO | MemInfo: { | 20260505 21:10:05.433 | INFO | "gpu" : " 17 GB", | 20260505 21:10:05.433 | INFO | "huge" : "", | 20260505 21:10:05.433 | INFO | "huge_shared" : " 27 GB", | 20260505 21:10:05.433 | INFO | "malloc" : " 14 GB", | 20260505 21:10:05.433 | INFO | "pinned" : " 29 GB" | 20260505 21:10:05.433 | INFO | } | 20260505 21:10:05.433 | INFO | Perf report for the last 5.13704 seconds: | 20260505 21:10:05.434 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:10:05.434 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:05.434 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:05.434 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:05.434 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 21:10:05.434 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:05.434 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:05.434 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 21:10:05.434 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:05.434 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:05.434 | INFO | Uploaded 36GiB to GPU in 2.13035s (16.8986GiB/s) | 20260505 21:10:05.434 | INFO | Downloaded 34GiB to GPU in 2.15085s (15.8077GiB/s) | 20260505 21:10:10.580 | INFO | Analyzing chunk frames = [6327808,6328320) | 20260505 21:10:10.580 | INFO | MemInfo: { | 20260505 21:10:10.580 | INFO | "gpu" : " 17 GB", | 20260505 21:10:10.580 | INFO | "huge" : "", | 20260505 21:10:10.580 | INFO | "huge_shared" : " 27 GB", | 20260505 21:10:10.580 | INFO | "malloc" : " 14 GB", | 20260505 21:10:10.580 | INFO | "pinned" : " 29 GB" | 20260505 21:10:10.580 | INFO | } | 20260505 21:10:10.581 | INFO | Perf report for the last 5.14701 seconds: | 20260505 21:10:10.581 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:10:10.581 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:10.581 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:10.581 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:10.581 | INFO | ANALYSIS : 62.3%, 3.0, 97.4ms, 0 | 20260505 21:10:10.581 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:10.581 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:10.581 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:10:10.581 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:10.581 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:10.581 | INFO | Uploaded 36GiB to GPU in 2.11614s (17.0121GiB/s) | 20260505 21:10:10.581 | INFO | Downloaded 34GiB to GPU in 2.01722s (16.8549GiB/s) | 20260505 21:10:10.712 | INFO | Starting a round of Baz IO | 20260505 21:10:13.871 | INFO | Populating RTMetrics report | 20260505 21:10:13.871 | INFO | RTMetrics report populated | 20260505 21:10:13.871 | INFO | Opening tmp RTMetrics json file | 20260505 21:10:13.872 | INFO | Serializing RTMetrics to json | 20260505 21:10:13.873 | INFO | Serialization done, swapping tmp file now... | 20260505 21:10:13.874 | INFO | RTMetrics done | 20260505 21:10:13.962 | INFO | Finished a round of Baz IO: Wrote 1.98 GiB in 3.249 seconds | 20260505 21:10:13.962 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.60% / 0.00% / 78.40% | 20260505 21:10:13.962 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.153704% / 3.97364e-06% / 9.53674e-05% | 20260505 21:10:13.962 | INFO | Baz IO overall: | 20260505 21:10:13.962 | INFO | BytesWritten : 3672013417630 | 20260505 21:10:13.962 | INFO | NumEvents : 1734481112223 | 20260505 21:10:15.725 | INFO | Analyzing chunk frames = [6328320,6328832) | 20260505 21:10:15.725 | INFO | MemInfo: { | 20260505 21:10:15.725 | INFO | "gpu" : " 17 GB", | 20260505 21:10:15.725 | INFO | "huge" : "", | 20260505 21:10:15.725 | INFO | "huge_shared" : " 27 GB", | 20260505 21:10:15.725 | INFO | "malloc" : " 14 GB", | 20260505 21:10:15.725 | INFO | "pinned" : " 29 GB" | 20260505 21:10:15.725 | INFO | } | 20260505 21:10:15.725 | INFO | Perf report for the last 5.14438 seconds: | 20260505 21:10:15.725 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:10:15.725 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:15.725 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:15.725 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:15.725 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 21:10:15.725 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:15.725 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260505 21:10:15.725 | INFO | PRE_HQ : 60.1%, 1.8, 55.0ms, 0 | 20260505 21:10:15.725 | INFO | BAZWRITER : 62.7%, 3.6, 117.7ms, 0 | 20260505 21:10:15.725 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 21:10:15.725 | INFO | Uploaded 36GiB to GPU in 2.17886s (16.5224GiB/s) | 20260505 21:10:15.725 | INFO | Downloaded 38GiB to GPU in 2.25087s (16.8824GiB/s) | 20260505 21:10:20.856 | INFO | Analyzing chunk frames = [6328832,6329344) | 20260505 21:10:20.856 | INFO | MemInfo: { | 20260505 21:10:20.856 | INFO | "gpu" : " 17 GB", | 20260505 21:10:20.856 | INFO | "huge" : "", | 20260505 21:10:20.856 | INFO | "huge_shared" : " 27 GB", | 20260505 21:10:20.856 | INFO | "malloc" : " 14 GB", | 20260505 21:10:20.856 | INFO | "pinned" : " 29 GB" | 20260505 21:10:20.856 | INFO | } | 20260505 21:10:20.856 | INFO | Perf report for the last 5.13152 seconds: | 20260505 21:10:20.856 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:10:20.856 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:20.856 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:20.856 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:20.857 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260505 21:10:20.857 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:20.857 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:20.857 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 21:10:20.857 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:20.857 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:20.857 | INFO | Uploaded 36GiB to GPU in 2.07571s (17.3435GiB/s) | 20260505 21:10:20.857 | INFO | Downloaded 34GiB to GPU in 2.02755s (16.769GiB/s) | 20260505 21:10:25.991 | INFO | Analyzing chunk frames = [6329344,6329856) | 20260505 21:10:25.991 | INFO | MemInfo: { | 20260505 21:10:25.991 | INFO | "gpu" : " 17 GB", | 20260505 21:10:25.991 | INFO | "huge" : "", | 20260505 21:10:25.991 | INFO | "huge_shared" : " 27 GB", | 20260505 21:10:25.991 | INFO | "malloc" : " 14 GB", | 20260505 21:10:25.992 | INFO | "pinned" : " 29 GB" | 20260505 21:10:25.992 | INFO | } | 20260505 21:10:25.992 | INFO | Perf report for the last 5.1352 seconds: | 20260505 21:10:25.992 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:10:25.992 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:25.992 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:25.992 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:25.992 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260505 21:10:25.992 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:25.992 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:25.992 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:10:25.992 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:25.992 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:25.992 | INFO | Uploaded 36GiB to GPU in 2.13758s (16.8415GiB/s) | 20260505 21:10:25.992 | INFO | Downloaded 34GiB to GPU in 2.02223s (16.8131GiB/s) | 20260505 21:10:31.138 | INFO | Analyzing chunk frames = [6329856,6330368) | 20260505 21:10:31.138 | INFO | MemInfo: { | 20260505 21:10:31.138 | INFO | "gpu" : " 17 GB", | 20260505 21:10:31.138 | INFO | "huge" : "", | 20260505 21:10:31.138 | INFO | "huge_shared" : " 27 GB", | 20260505 21:10:31.138 | INFO | "malloc" : " 14 GB", | 20260505 21:10:31.138 | INFO | "pinned" : " 29 GB" | 20260505 21:10:31.138 | INFO | } | 20260505 21:10:31.138 | INFO | Perf report for the last 5.14667 seconds: | 20260505 21:10:31.138 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:10:31.138 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:31.138 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:31.138 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:31.138 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260505 21:10:31.138 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:31.138 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:31.138 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:10:31.138 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:31.138 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:31.139 | INFO | Uploaded 36GiB to GPU in 2.19509s (16.4003GiB/s) | 20260505 21:10:31.139 | INFO | Downloaded 34GiB to GPU in 2.04672s (16.612GiB/s) | 20260505 21:10:36.278 | INFO | Analyzing chunk frames = [6330368,6330880) | 20260505 21:10:36.278 | INFO | MemInfo: { | 20260505 21:10:36.278 | INFO | "gpu" : " 17 GB", | 20260505 21:10:36.278 | INFO | "huge" : "", | 20260505 21:10:36.278 | INFO | "huge_shared" : " 27 GB", | 20260505 21:10:36.278 | INFO | "malloc" : " 14 GB", | 20260505 21:10:36.278 | INFO | "pinned" : " 29 GB" | 20260505 21:10:36.278 | INFO | } | 20260505 21:10:36.278 | INFO | Perf report for the last 5.13951 seconds: | 20260505 21:10:36.278 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:10:36.278 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:36.278 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:36.278 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:36.278 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260505 21:10:36.278 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:36.278 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:36.278 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 21:10:36.278 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:36.278 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:36.278 | INFO | Uploaded 36GiB to GPU in 2.12867s (16.912GiB/s) | 20260505 21:10:36.278 | INFO | Downloaded 34GiB to GPU in 2.06206s (16.4883GiB/s) | 20260505 21:10:41.420 | INFO | Analyzing chunk frames = [6330880,6331392) | 20260505 21:10:41.420 | INFO | MemInfo: { | 20260505 21:10:41.420 | INFO | "gpu" : " 17 GB", | 20260505 21:10:41.420 | INFO | "huge" : "", | 20260505 21:10:41.420 | INFO | "huge_shared" : " 27 GB", | 20260505 21:10:41.420 | INFO | "malloc" : " 14 GB", | 20260505 21:10:41.420 | INFO | "pinned" : " 29 GB" | 20260505 21:10:41.420 | INFO | } | 20260505 21:10:41.420 | INFO | Perf report for the last 5.14206 seconds: | 20260505 21:10:41.420 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:10:41.420 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:41.420 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:41.420 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:41.420 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260505 21:10:41.420 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:41.420 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:41.420 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 21:10:41.420 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:41.420 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:41.420 | INFO | Uploaded 36GiB to GPU in 2.07822s (17.3225GiB/s) | 20260505 21:10:41.420 | INFO | Downloaded 34GiB to GPU in 2.03521s (16.7059GiB/s) | 20260505 21:10:46.553 | INFO | Analyzing chunk frames = [6331392,6331904) | 20260505 21:10:46.553 | INFO | MemInfo: { | 20260505 21:10:46.553 | INFO | "gpu" : " 17 GB", | 20260505 21:10:46.553 | INFO | "huge" : "", | 20260505 21:10:46.553 | INFO | "huge_shared" : " 27 GB", | 20260505 21:10:46.553 | INFO | "malloc" : " 14 GB", | 20260505 21:10:46.553 | INFO | "pinned" : " 29 GB" | 20260505 21:10:46.553 | INFO | } | 20260505 21:10:46.553 | INFO | Perf report for the last 5.1331 seconds: | 20260505 21:10:46.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:10:46.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:46.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:46.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:46.553 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 21:10:46.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:46.553 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:46.553 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 21:10:46.553 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:46.553 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:46.553 | INFO | Uploaded 36GiB to GPU in 2.14163s (16.8096GiB/s) | 20260505 21:10:46.553 | INFO | Downloaded 34GiB to GPU in 2.08314s (16.3215GiB/s) | 20260505 21:10:51.689 | INFO | Analyzing chunk frames = [6331904,6332416) | 20260505 21:10:51.689 | INFO | MemInfo: { | 20260505 21:10:51.689 | INFO | "gpu" : " 17 GB", | 20260505 21:10:51.689 | INFO | "huge" : "", | 20260505 21:10:51.689 | INFO | "huge_shared" : " 27 GB", | 20260505 21:10:51.689 | INFO | "malloc" : " 14 GB", | 20260505 21:10:51.689 | INFO | "pinned" : " 29 GB" | 20260505 21:10:51.689 | INFO | } | 20260505 21:10:51.689 | INFO | Perf report for the last 5.13634 seconds: | 20260505 21:10:51.689 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:10:51.689 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:51.689 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:51.689 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:51.689 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260505 21:10:51.689 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:51.689 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:51.690 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:10:51.690 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:51.690 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:51.690 | INFO | Uploaded 36GiB to GPU in 2.11454s (17.025GiB/s) | 20260505 21:10:51.690 | INFO | Downloaded 34GiB to GPU in 2.01765s (16.8513GiB/s) | 20260505 21:10:51.690 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 21:10:51.690 | INFO | Total mem usage: 15.3 GiB | 20260505 21:10:51.690 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 21:10:51.690 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.8%) | 20260505 21:10:51.690 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 21:10:51.690 | INFO | High Water Breakdown: | 20260505 21:10:51.690 | INFO | 61% : 8.59 GiB ( 90.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 21:10:51.690 | INFO | 38.9% : 5.48 GiB ( 82.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 21:10:51.690 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 21:10:51.690 | INFO | ------------------------------------------ | 20260505 21:10:51.690 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:10:51.690 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 21:10:51.690 | INFO | Total mem usage: 26.5 GiB | 20260505 21:10:51.690 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 21:10:51.690 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 21:10:51.690 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 21:10:51.690 | INFO | High Water Breakdown: | 20260505 21:10:51.690 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:10:51.690 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:10:51.690 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:10:51.690 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:10:51.690 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:10:51.690 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:10:51.690 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 21:10:51.690 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 21:10:51.690 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:10:51.690 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 21:10:51.690 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 21:10:51.690 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 21:10:51.690 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 21:10:51.690 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 21:10:51.690 | INFO | ------------------------------------------ | 20260505 21:10:51.690 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:10:51.690 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 21:10:51.690 | INFO | Total mem usage: 25.6 GiB | 20260505 21:10:51.690 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 21:10:51.690 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 21:10:51.690 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 21:10:51.690 | INFO | High Water Breakdown: | 20260505 21:10:51.690 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 21:10:51.690 | INFO | ------------------------------------------ | 20260505 21:10:51.690 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:10:51.690 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 21:10:51.690 | INFO | Total mem usage: 15.7 GiB | 20260505 21:10:51.691 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 21:10:51.691 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 21:10:51.691 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 21:10:51.691 | INFO | High Water Breakdown: | 20260505 21:10:51.691 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 21:10:51.691 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 21:10:51.691 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 21:10:51.691 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 21:10:51.691 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 21:10:51.691 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 21:10:51.691 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 21:10:51.691 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 21:10:51.691 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 21:10:51.691 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:10:51.691 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 21:10:51.691 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:10:51.691 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 21:10:51.691 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 21:10:51.691 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 21:10:51.691 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 21:10:51.691 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:10:51.691 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 21:10:51.691 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:10:51.691 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:10:51.691 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:10:51.691 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 21:10:51.691 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 21:10:51.691 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:10:51.691 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 21:10:51.691 | INFO | ------------------------------------------ | 20260505 21:10:51.691 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:10:51.691 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 21:10:51.691 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:10:51.691 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:10:51.691 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.42e+01, 1.32e+01 / 1.29e+00 | 20260505 21:10:51.691 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.04e-01, 1.05e-02 / 5.93e-03 | 20260505 21:10:51.691 | INFO | PerfReport: 11.80%, Baseline -- 3.71e+00 / 4.52e+00, 5.67e-01 / 2.08e-01 | 20260505 21:10:51.691 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.52e+00, 1.89e-01 / 1.03e-01 | 20260505 21:10:51.691 | INFO | PerfReport: 15.92%, DME -- 4.01e+01 / 4.40e+01, 3.92e+00 / 1.36e+00 | 20260505 21:10:51.691 | INFO | PerfReport: 57.20%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.54e+00 / 1.18e+00 | 20260505 21:10:51.691 | INFO | PerfReport: 7.17%, PulseAccumulating -- 2.26e+00 / 2.79e+00, 4.28e-01 / 2.30e-01 | 20260505 21:10:51.691 | INFO | PerfReport: 3.20%, Metrics -- 1.01e+00 / 1.31e+00, 2.74e-01 / 1.57e-01 | 20260505 21:10:51.691 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.51e-01 / 4.10e-01, 6.12e-02 / 3.77e-02 | 20260505 21:10:51.691 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 21:10:51.691 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:10:51.691 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:10:51.691 | INFO | PerfReport: Full Execution -- 9.90e+01 / -nan, 2.49e+01 / 0.00e+00 | 20260505 21:10:51.691 | INFO | PerfReport: 22.39%, Upload -- 2.22e+01 / 0.00e+00, 7.60e+00 / 0.00e+00 | 20260505 21:10:51.691 | INFO | PerfReport: 21.79%, Download -- 2.16e+01 / 0.00e+00, 7.64e+00 / 0.00e+00 | 20260505 21:10:51.691 | INFO | PerfReport: 31.78%, Compute -- 3.15e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260505 21:10:51.691 | INFO | PerfReport: 2.20%, LockUpload -- 2.18e+00 / 0.00e+00, 7.93e+00 / 0.00e+00 | 20260505 21:10:51.691 | INFO | PerfReport: 1.33%, LockDownload -- 1.32e+00 / 0.00e+00, 2.89e+00 / 0.00e+00 | 20260505 21:10:51.691 | INFO | PerfReport: 20.49%, LockCompute -- 2.03e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 21:10:51.691 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.57e-02 / 0.00e+00 | 20260505 21:10:51.819 | INFO | Starting a round of Baz IO | 20260505 21:10:54.984 | INFO | Populating RTMetrics report | 20260505 21:10:54.985 | INFO | RTMetrics report populated | 20260505 21:10:54.985 | INFO | Opening tmp RTMetrics json file | 20260505 21:10:54.986 | INFO | Serializing RTMetrics to json | 20260505 21:10:54.987 | INFO | Serialization done, swapping tmp file now... | 20260505 21:10:54.988 | INFO | RTMetrics done | 20260505 21:10:55.074 | INFO | Finished a round of Baz IO: Wrote 1.98 GiB in 3.254 seconds | 20260505 21:10:55.074 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.60% / 0.00% / 78.40% | 20260505 21:10:55.074 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.154273% / 0% / 7.94729e-05% | 20260505 21:10:55.074 | INFO | Baz IO overall: | 20260505 21:10:55.074 | INFO | BytesWritten : 3674139940855 | 20260505 21:10:55.074 | INFO | NumEvents : 1735463381442 | 20260505 21:10:56.832 | INFO | Analyzing chunk frames = [6332416,6332928) | 20260505 21:10:56.832 | INFO | MemInfo: { | 20260505 21:10:56.832 | INFO | "gpu" : " 17 GB", | 20260505 21:10:56.832 | INFO | "huge" : "", | 20260505 21:10:56.832 | INFO | "huge_shared" : " 27 GB", | 20260505 21:10:56.832 | INFO | "malloc" : " 14 GB", | 20260505 21:10:56.832 | INFO | "pinned" : " 29 GB" | 20260505 21:10:56.832 | INFO | } | 20260505 21:10:56.832 | INFO | Perf report for the last 5.14226 seconds: | 20260505 21:10:56.832 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:10:56.832 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:56.832 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:56.832 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:56.832 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 21:10:56.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:10:56.832 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260505 21:10:56.832 | INFO | PRE_HQ : 59.9%, 1.8, 55.1ms, 0 | 20260505 21:10:56.832 | INFO | BAZWRITER : 62.8%, 3.7, 117.9ms, 0 | 20260505 21:10:56.832 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 21:10:56.832 | INFO | Uploaded 36GiB to GPU in 2.15414s (16.712GiB/s) | 20260505 21:10:56.832 | INFO | Downloaded 38GiB to GPU in 2.24244s (16.9458GiB/s) | 20260505 21:11:01.972 | INFO | Analyzing chunk frames = [6332928,6333440) | 20260505 21:11:01.972 | INFO | MemInfo: { | 20260505 21:11:01.972 | INFO | "gpu" : " 17 GB", | 20260505 21:11:01.972 | INFO | "huge" : "", | 20260505 21:11:01.972 | INFO | "huge_shared" : " 27 GB", | 20260505 21:11:01.972 | INFO | "malloc" : " 14 GB", | 20260505 21:11:01.972 | INFO | "pinned" : " 29 GB" | 20260505 21:11:01.972 | INFO | } | 20260505 21:11:01.972 | INFO | Perf report for the last 5.14034 seconds: | 20260505 21:11:01.972 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:11:01.972 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:01.972 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:01.972 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:01.972 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 21:11:01.972 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:01.972 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:01.972 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:11:01.972 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:01.972 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:01.972 | INFO | Uploaded 36GiB to GPU in 2.09173s (17.2106GiB/s) | 20260505 21:11:01.972 | INFO | Downloaded 34GiB to GPU in 2.02096s (16.8237GiB/s) | 20260505 21:11:07.111 | INFO | Analyzing chunk frames = [6333440,6333952) | 20260505 21:11:07.111 | INFO | MemInfo: { | 20260505 21:11:07.111 | INFO | "gpu" : " 17 GB", | 20260505 21:11:07.111 | INFO | "huge" : "", | 20260505 21:11:07.111 | INFO | "huge_shared" : " 27 GB", | 20260505 21:11:07.111 | INFO | "malloc" : " 14 GB", | 20260505 21:11:07.111 | INFO | "pinned" : " 29 GB" | 20260505 21:11:07.111 | INFO | } | 20260505 21:11:07.112 | INFO | Perf report for the last 5.13951 seconds: | 20260505 21:11:07.112 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:11:07.112 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:07.112 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:07.112 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:07.112 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260505 21:11:07.112 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:07.112 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:07.112 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:11:07.112 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:07.112 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:07.112 | INFO | Uploaded 36GiB to GPU in 2.12603s (16.933GiB/s) | 20260505 21:11:07.112 | INFO | Downloaded 34GiB to GPU in 2.01689s (16.8577GiB/s) | 20260505 21:11:12.246 | INFO | Analyzing chunk frames = [6333952,6334464) | 20260505 21:11:12.246 | INFO | MemInfo: { | 20260505 21:11:12.246 | INFO | "gpu" : " 17 GB", | 20260505 21:11:12.246 | INFO | "huge" : "", | 20260505 21:11:12.246 | INFO | "huge_shared" : " 27 GB", | 20260505 21:11:12.246 | INFO | "malloc" : " 14 GB", | 20260505 21:11:12.246 | INFO | "pinned" : " 29 GB" | 20260505 21:11:12.246 | INFO | } | 20260505 21:11:12.246 | INFO | Perf report for the last 5.13457 seconds: | 20260505 21:11:12.246 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:11:12.246 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:12.246 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:12.246 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:12.246 | INFO | ANALYSIS : 62.1%, 3.0, 96.4ms, 0 | 20260505 21:11:12.246 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:12.246 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:12.246 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:11:12.246 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:12.246 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:12.246 | INFO | Uploaded 36GiB to GPU in 2.13717s (16.8447GiB/s) | 20260505 21:11:12.246 | INFO | Downloaded 34GiB to GPU in 2.04421s (16.6323GiB/s) | 20260505 21:11:17.385 | INFO | Analyzing chunk frames = [6334464,6334976) | 20260505 21:11:17.385 | INFO | MemInfo: { | 20260505 21:11:17.385 | INFO | "gpu" : " 17 GB", | 20260505 21:11:17.385 | INFO | "huge" : "", | 20260505 21:11:17.385 | INFO | "huge_shared" : " 27 GB", | 20260505 21:11:17.386 | INFO | "malloc" : " 14 GB", | 20260505 21:11:17.386 | INFO | "pinned" : " 29 GB" | 20260505 21:11:17.386 | INFO | } | 20260505 21:11:17.386 | INFO | Perf report for the last 5.13951 seconds: | 20260505 21:11:17.386 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:11:17.386 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:17.386 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:17.386 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:17.386 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 21:11:17.386 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:17.386 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:17.386 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 21:11:17.386 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:17.386 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:17.386 | INFO | Uploaded 36GiB to GPU in 2.13531s (16.8594GiB/s) | 20260505 21:11:17.386 | INFO | Downloaded 34GiB to GPU in 2.06077s (16.4987GiB/s) | 20260505 21:11:22.528 | INFO | Analyzing chunk frames = [6334976,6335488) | 20260505 21:11:22.528 | INFO | MemInfo: { | 20260505 21:11:22.528 | INFO | "gpu" : " 17 GB", | 20260505 21:11:22.528 | INFO | "huge" : "", | 20260505 21:11:22.528 | INFO | "huge_shared" : " 27 GB", | 20260505 21:11:22.528 | INFO | "malloc" : " 14 GB", | 20260505 21:11:22.528 | INFO | "pinned" : " 29 GB" | 20260505 21:11:22.528 | INFO | } | 20260505 21:11:22.528 | INFO | Perf report for the last 5.14252 seconds: | 20260505 21:11:22.528 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:11:22.528 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:22.528 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:22.528 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:22.528 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260505 21:11:22.528 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:22.528 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:22.528 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 21:11:22.528 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:22.528 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:22.529 | INFO | Uploaded 36GiB to GPU in 2.08763s (17.2444GiB/s) | 20260505 21:11:22.529 | INFO | Downloaded 34GiB to GPU in 2.03426s (16.7137GiB/s) | 20260505 21:11:27.674 | INFO | Analyzing chunk frames = [6335488,6336000) | 20260505 21:11:27.674 | INFO | MemInfo: { | 20260505 21:11:27.674 | INFO | "gpu" : " 17 GB", | 20260505 21:11:27.674 | INFO | "huge" : "", | 20260505 21:11:27.674 | INFO | "huge_shared" : " 27 GB", | 20260505 21:11:27.674 | INFO | "malloc" : " 14 GB", | 20260505 21:11:27.674 | INFO | "pinned" : " 29 GB" | 20260505 21:11:27.674 | INFO | } | 20260505 21:11:27.674 | INFO | Perf report for the last 5.14579 seconds: | 20260505 21:11:27.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:11:27.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:27.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:27.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:27.674 | INFO | ANALYSIS : 62.9%, 3.0, 98.4ms, 0 | 20260505 21:11:27.674 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:27.674 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:27.674 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 21:11:27.674 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:27.674 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:27.674 | INFO | Uploaded 36GiB to GPU in 2.1287s (16.9117GiB/s) | 20260505 21:11:27.674 | INFO | Downloaded 34GiB to GPU in 2.14279s (15.8672GiB/s) | 20260505 21:11:32.803 | INFO | Analyzing chunk frames = [6336000,6336512) | 20260505 21:11:32.803 | INFO | MemInfo: { | 20260505 21:11:32.803 | INFO | "gpu" : " 17 GB", | 20260505 21:11:32.803 | INFO | "huge" : "", | 20260505 21:11:32.803 | INFO | "huge_shared" : " 27 GB", | 20260505 21:11:32.803 | INFO | "malloc" : " 14 GB", | 20260505 21:11:32.803 | INFO | "pinned" : " 29 GB" | 20260505 21:11:32.803 | INFO | } | 20260505 21:11:32.803 | INFO | Perf report for the last 5.12872 seconds: | 20260505 21:11:32.803 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:11:32.803 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:32.803 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:32.803 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:32.803 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 21:11:32.803 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:32.803 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:32.803 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260505 21:11:32.803 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:32.803 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:32.803 | INFO | Uploaded 36GiB to GPU in 2.12175s (16.9671GiB/s) | 20260505 21:11:32.803 | INFO | Downloaded 34GiB to GPU in 2.00381s (16.9676GiB/s) | 20260505 21:11:32.935 | INFO | Starting a round of Baz IO | 20260505 21:11:36.109 | INFO | Populating RTMetrics report | 20260505 21:11:36.110 | INFO | RTMetrics report populated | 20260505 21:11:36.110 | INFO | Opening tmp RTMetrics json file | 20260505 21:11:36.111 | INFO | Serializing RTMetrics to json | 20260505 21:11:36.115 | INFO | Serialization done, swapping tmp file now... | 20260505 21:11:36.116 | INFO | RTMetrics done | 20260505 21:11:36.192 | INFO | Finished a round of Baz IO: Wrote 1.981 GiB in 3.256 seconds | 20260505 21:11:36.192 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.60% / 0.00% / 78.40% | 20260505 21:11:36.192 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.15444% / 3.97364e-06% / 8.74201e-05% | 20260505 21:11:36.192 | INFO | Baz IO overall: | 20260505 21:11:36.192 | INFO | BytesWritten : 3676267100731 | 20260505 21:11:36.192 | INFO | NumEvents : 1736446387668 | 20260505 21:11:37.942 | INFO | Analyzing chunk frames = [6336512,6337024) | 20260505 21:11:37.942 | INFO | MemInfo: { | 20260505 21:11:37.942 | INFO | "gpu" : " 17 GB", | 20260505 21:11:37.942 | INFO | "huge" : "", | 20260505 21:11:37.942 | INFO | "huge_shared" : " 27 GB", | 20260505 21:11:37.942 | INFO | "malloc" : " 14 GB", | 20260505 21:11:37.942 | INFO | "pinned" : " 29 GB" | 20260505 21:11:37.942 | INFO | } | 20260505 21:11:37.942 | INFO | Perf report for the last 5.13959 seconds: | 20260505 21:11:37.942 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:11:37.942 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:37.942 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:37.942 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:37.942 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 21:11:37.942 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:37.942 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260505 21:11:37.942 | INFO | PRE_HQ : 60.1%, 1.8, 55.1ms, 0 | 20260505 21:11:37.942 | INFO | BAZWRITER : 62.9%, 3.7, 118.0ms, 0 | 20260505 21:11:37.942 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 21:11:37.943 | INFO | Uploaded 36GiB to GPU in 2.16729s (16.6106GiB/s) | 20260505 21:11:37.943 | INFO | Downloaded 38GiB to GPU in 2.24579s (16.9205GiB/s) | 20260505 21:11:43.091 | INFO | Analyzing chunk frames = [6337024,6337536) | 20260505 21:11:43.091 | INFO | MemInfo: { | 20260505 21:11:43.091 | INFO | "gpu" : " 17 GB", | 20260505 21:11:43.091 | INFO | "huge" : "", | 20260505 21:11:43.091 | INFO | "huge_shared" : " 27 GB", | 20260505 21:11:43.091 | INFO | "malloc" : " 14 GB", | 20260505 21:11:43.092 | INFO | "pinned" : " 29 GB" | 20260505 21:11:43.092 | INFO | } | 20260505 21:11:43.092 | INFO | Perf report for the last 5.14929 seconds: | 20260505 21:11:43.092 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:11:43.092 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:43.092 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:43.092 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:43.092 | INFO | ANALYSIS : 62.5%, 3.0, 97.7ms, 0 | 20260505 21:11:43.092 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:43.092 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:43.092 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 21:11:43.092 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:43.092 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:43.092 | INFO | Uploaded 36GiB to GPU in 2.15204s (16.7284GiB/s) | 20260505 21:11:43.092 | INFO | Downloaded 34GiB to GPU in 2.05331s (16.5586GiB/s) | 20260505 21:11:48.233 | INFO | Analyzing chunk frames = [6337536,6338048) | 20260505 21:11:48.234 | INFO | MemInfo: { | 20260505 21:11:48.234 | INFO | "gpu" : " 17 GB", | 20260505 21:11:48.234 | INFO | "huge" : "", | 20260505 21:11:48.234 | INFO | "huge_shared" : " 27 GB", | 20260505 21:11:48.234 | INFO | "malloc" : " 14 GB", | 20260505 21:11:48.234 | INFO | "pinned" : " 29 GB" | 20260505 21:11:48.234 | INFO | } | 20260505 21:11:48.234 | INFO | Perf report for the last 5.14214 seconds: | 20260505 21:11:48.234 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:11:48.234 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:48.234 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:48.234 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:48.234 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260505 21:11:48.234 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:48.234 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:48.234 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:11:48.234 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:48.234 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:48.234 | INFO | Uploaded 36GiB to GPU in 2.14851s (16.7558GiB/s) | 20260505 21:11:48.234 | INFO | Downloaded 34GiB to GPU in 2.03985s (16.6679GiB/s) | 20260505 21:11:53.366 | INFO | Analyzing chunk frames = [6338048,6338560) | 20260505 21:11:53.366 | INFO | MemInfo: { | 20260505 21:11:53.366 | INFO | "gpu" : " 17 GB", | 20260505 21:11:53.366 | INFO | "huge" : "", | 20260505 21:11:53.366 | INFO | "huge_shared" : " 27 GB", | 20260505 21:11:53.366 | INFO | "malloc" : " 14 GB", | 20260505 21:11:53.367 | INFO | "pinned" : " 29 GB" | 20260505 21:11:53.367 | INFO | } | 20260505 21:11:53.367 | INFO | Perf report for the last 5.13293 seconds: | 20260505 21:11:53.367 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:11:53.367 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:53.367 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:53.367 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:53.367 | INFO | ANALYSIS : 61.8%, 3.0, 95.9ms, 0 | 20260505 21:11:53.367 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:53.367 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:53.367 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 21:11:53.367 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:53.367 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:53.367 | INFO | Uploaded 36GiB to GPU in 2.11494s (17.0217GiB/s) | 20260505 21:11:53.367 | INFO | Downloaded 34GiB to GPU in 2.00957s (16.9191GiB/s) | 20260505 21:11:58.500 | INFO | Analyzing chunk frames = [6338560,6339072) | 20260505 21:11:58.500 | INFO | MemInfo: { | 20260505 21:11:58.500 | INFO | "gpu" : " 17 GB", | 20260505 21:11:58.500 | INFO | "huge" : "", | 20260505 21:11:58.500 | INFO | "huge_shared" : " 27 GB", | 20260505 21:11:58.500 | INFO | "malloc" : " 14 GB", | 20260505 21:11:58.500 | INFO | "pinned" : " 29 GB" | 20260505 21:11:58.500 | INFO | } | 20260505 21:11:58.500 | INFO | Perf report for the last 5.13355 seconds: | 20260505 21:11:58.500 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:11:58.500 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:58.500 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:58.500 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:58.500 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260505 21:11:58.500 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:58.500 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:58.500 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:11:58.500 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:58.500 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:11:58.501 | INFO | Uploaded 36GiB to GPU in 2.13619s (16.8525GiB/s) | 20260505 21:11:58.501 | INFO | Downloaded 34GiB to GPU in 2.06379s (16.4745GiB/s) | 20260505 21:12:03.642 | INFO | Analyzing chunk frames = [6339072,6339584) | 20260505 21:12:03.642 | INFO | MemInfo: { | 20260505 21:12:03.642 | INFO | "gpu" : " 17 GB", | 20260505 21:12:03.642 | INFO | "huge" : "", | 20260505 21:12:03.642 | INFO | "huge_shared" : " 27 GB", | 20260505 21:12:03.642 | INFO | "malloc" : " 14 GB", | 20260505 21:12:03.642 | INFO | "pinned" : " 29 GB" | 20260505 21:12:03.642 | INFO | } | 20260505 21:12:03.642 | INFO | Perf report for the last 5.14189 seconds: | 20260505 21:12:03.642 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:12:03.642 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:03.642 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:03.642 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:03.642 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 21:12:03.642 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:03.642 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:03.642 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 21:12:03.642 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:03.642 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:03.642 | INFO | Uploaded 36GiB to GPU in 2.15155s (16.7321GiB/s) | 20260505 21:12:03.643 | INFO | Downloaded 34GiB to GPU in 2.05973s (16.507GiB/s) | 20260505 21:12:08.785 | INFO | Analyzing chunk frames = [6339584,6340096) | 20260505 21:12:08.785 | INFO | MemInfo: { | 20260505 21:12:08.785 | INFO | "gpu" : " 17 GB", | 20260505 21:12:08.785 | INFO | "huge" : "", | 20260505 21:12:08.785 | INFO | "huge_shared" : " 27 GB", | 20260505 21:12:08.785 | INFO | "malloc" : " 14 GB", | 20260505 21:12:08.785 | INFO | "pinned" : " 29 GB" | 20260505 21:12:08.785 | INFO | } | 20260505 21:12:08.785 | INFO | Perf report for the last 5.14296 seconds: | 20260505 21:12:08.785 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:12:08.785 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:08.785 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:08.785 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:08.785 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 21:12:08.785 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:08.785 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:08.785 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 21:12:08.785 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:08.785 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:08.785 | INFO | Uploaded 36GiB to GPU in 2.1368s (16.8476GiB/s) | 20260505 21:12:08.785 | INFO | Downloaded 34GiB to GPU in 2.15351s (15.7882GiB/s) | 20260505 21:12:13.929 | INFO | Analyzing chunk frames = [6340096,6340608) | 20260505 21:12:13.929 | INFO | MemInfo: { | 20260505 21:12:13.929 | INFO | "gpu" : " 17 GB", | 20260505 21:12:13.929 | INFO | "huge" : "", | 20260505 21:12:13.929 | INFO | "huge_shared" : " 27 GB", | 20260505 21:12:13.929 | INFO | "malloc" : " 14 GB", | 20260505 21:12:13.930 | INFO | "pinned" : " 29 GB" | 20260505 21:12:13.930 | INFO | } | 20260505 21:12:13.930 | INFO | Perf report for the last 5.14444 seconds: | 20260505 21:12:13.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:12:13.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:13.930 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:13.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:13.930 | INFO | ANALYSIS : 62.1%, 3.0, 97.1ms, 0 | 20260505 21:12:13.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:13.930 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:13.930 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:12:13.930 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:13.930 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:13.930 | INFO | Uploaded 36GiB to GPU in 2.10827s (17.0756GiB/s) | 20260505 21:12:13.930 | INFO | Downloaded 34GiB to GPU in 1.99935s (17.0056GiB/s) | 20260505 21:12:13.930 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 21:12:13.930 | INFO | Total mem usage: 15.3 GiB | 20260505 21:12:13.930 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 21:12:13.930 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.97%) | 20260505 21:12:13.930 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 21:12:13.930 | INFO | High Water Breakdown: | 20260505 21:12:13.930 | INFO | 60.9% : 8.57 GiB ( 90% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 21:12:13.930 | INFO | 38.9% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 21:12:13.930 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 21:12:13.930 | INFO | ------------------------------------------ | 20260505 21:12:13.930 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:12:13.930 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 21:12:13.930 | INFO | Total mem usage: 26.5 GiB | 20260505 21:12:13.930 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 21:12:13.930 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 21:12:13.930 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 21:12:13.930 | INFO | High Water Breakdown: | 20260505 21:12:13.930 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:12:13.930 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:12:13.930 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:12:13.930 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:12:13.930 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:12:13.930 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:12:13.930 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 21:12:13.930 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 21:12:13.930 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:12:13.930 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 21:12:13.930 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 21:12:13.930 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 21:12:13.930 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 21:12:13.930 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 21:12:13.930 | INFO | ------------------------------------------ | 20260505 21:12:13.930 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:12:13.930 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 21:12:13.930 | INFO | Total mem usage: 25.6 GiB | 20260505 21:12:13.930 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 21:12:13.930 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 21:12:13.930 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 21:12:13.930 | INFO | High Water Breakdown: | 20260505 21:12:13.930 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 21:12:13.930 | INFO | ------------------------------------------ | 20260505 21:12:13.931 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:12:13.931 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 21:12:13.931 | INFO | Total mem usage: 15.7 GiB | 20260505 21:12:13.931 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 21:12:13.931 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 21:12:13.931 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 21:12:13.931 | INFO | High Water Breakdown: | 20260505 21:12:13.931 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 21:12:13.931 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 21:12:13.931 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 21:12:13.931 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 21:12:13.931 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 21:12:13.931 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 21:12:13.931 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 21:12:13.931 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 21:12:13.931 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 21:12:13.931 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:12:13.931 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 21:12:13.931 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:12:13.931 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 21:12:13.931 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 21:12:13.931 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 21:12:13.931 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 21:12:13.931 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:12:13.931 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 21:12:13.931 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:12:13.931 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:12:13.931 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:12:13.931 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 21:12:13.931 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 21:12:13.931 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:12:13.931 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 21:12:13.931 | INFO | ------------------------------------------ | 20260505 21:12:13.931 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:12:13.931 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 21:12:13.931 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:12:13.931 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:12:13.931 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.46e+01, 1.34e+01 / 2.10e+00 | 20260505 21:12:13.931 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.04e-01, 1.06e-02 / 5.70e-03 | 20260505 21:12:13.931 | INFO | PerfReport: 11.80%, Baseline -- 3.71e+00 / 4.57e+00, 5.71e-01 / 1.06e-01 | 20260505 21:12:13.931 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.53e+00, 1.91e-01 / 8.55e-02 | 20260505 21:12:13.931 | INFO | PerfReport: 15.99%, DME -- 4.02e+01 / 4.44e+01, 4.11e+00 / 2.01e+00 | 20260505 21:12:13.931 | INFO | PerfReport: 57.08%, FrameLabeling -- 1.79e+01 / 1.93e+01, 1.44e+00 / 1.06e+00 | 20260505 21:12:13.931 | INFO | PerfReport: 7.20%, PulseAccumulating -- 2.26e+00 / 2.86e+00, 4.33e-01 / 1.51e-01 | 20260505 21:12:13.931 | INFO | PerfReport: 3.22%, Metrics -- 1.01e+00 / 1.35e+00, 2.77e-01 / 1.38e-01 | 20260505 21:12:13.931 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.52e-01 / 4.15e-01, 6.17e-02 / 3.93e-02 | 20260505 21:12:13.931 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 21:12:13.931 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:12:13.931 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:12:13.931 | INFO | PerfReport: Full Execution -- 9.91e+01 / -nan, 2.53e+01 / 0.00e+00 | 20260505 21:12:13.931 | INFO | PerfReport: 22.39%, Upload -- 2.22e+01 / 0.00e+00, 7.65e+00 / 0.00e+00 | 20260505 21:12:13.931 | INFO | PerfReport: 21.80%, Download -- 2.16e+01 / 0.00e+00, 7.67e+00 / 0.00e+00 | 20260505 21:12:13.931 | INFO | PerfReport: 31.70%, Compute -- 3.14e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 21:12:13.931 | INFO | PerfReport: 2.24%, LockUpload -- 2.22e+00 / 0.00e+00, 8.05e+00 / 0.00e+00 | 20260505 21:12:13.931 | INFO | PerfReport: 1.38%, LockDownload -- 1.37e+00 / 0.00e+00, 2.95e+00 / 0.00e+00 | 20260505 21:12:13.931 | INFO | PerfReport: 20.47%, LockCompute -- 2.03e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 21:12:13.931 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.15e-02 / 0.00e+00 | 20260505 21:12:14.061 | INFO | Starting a round of Baz IO | 20260505 21:12:17.220 | INFO | Populating RTMetrics report | 20260505 21:12:17.220 | INFO | RTMetrics report populated | 20260505 21:12:17.220 | INFO | Opening tmp RTMetrics json file | 20260505 21:12:17.222 | INFO | Serializing RTMetrics to json | 20260505 21:12:17.226 | INFO | Serialization done, swapping tmp file now... | 20260505 21:12:17.229 | INFO | RTMetrics done | 20260505 21:12:17.305 | INFO | Finished a round of Baz IO: Wrote 1.98 GiB in 3.244 seconds | 20260505 21:12:17.305 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.60% / 0.00% / 78.40% | 20260505 21:12:17.305 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.153943% / 3.97364e-06% / 0.000107288% | 20260505 21:12:17.305 | INFO | Baz IO overall: | 20260505 21:12:17.305 | INFO | BytesWritten : 3678393644784 | 20260505 21:12:17.305 | INFO | NumEvents : 1737428732007 | 20260505 21:12:19.064 | INFO | Analyzing chunk frames = [6340608,6341120) | 20260505 21:12:19.064 | INFO | MemInfo: { | 20260505 21:12:19.064 | INFO | "gpu" : " 17 GB", | 20260505 21:12:19.064 | INFO | "huge" : "", | 20260505 21:12:19.064 | INFO | "huge_shared" : " 27 GB", | 20260505 21:12:19.064 | INFO | "malloc" : " 14 GB", | 20260505 21:12:19.065 | INFO | "pinned" : " 29 GB" | 20260505 21:12:19.065 | INFO | } | 20260505 21:12:19.065 | INFO | Perf report for the last 5.13497 seconds: | 20260505 21:12:19.065 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:12:19.065 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:19.065 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:19.065 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:19.065 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 21:12:19.065 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:19.065 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260505 21:12:19.065 | INFO | PRE_HQ : 60.1%, 1.8, 54.8ms, 0 | 20260505 21:12:19.065 | INFO | BAZWRITER : 62.6%, 3.7, 117.6ms, 0 | 20260505 21:12:19.065 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 21:12:19.065 | INFO | Uploaded 36GiB to GPU in 2.1725s (16.5707GiB/s) | 20260505 21:12:19.065 | INFO | Downloaded 38GiB to GPU in 2.2455s (16.9227GiB/s) | 20260505 21:12:24.201 | INFO | Analyzing chunk frames = [6341120,6341632) | 20260505 21:12:24.201 | INFO | MemInfo: { | 20260505 21:12:24.201 | INFO | "gpu" : " 17 GB", | 20260505 21:12:24.201 | INFO | "huge" : "", | 20260505 21:12:24.201 | INFO | "huge_shared" : " 27 GB", | 20260505 21:12:24.201 | INFO | "malloc" : " 14 GB", | 20260505 21:12:24.201 | INFO | "pinned" : " 29 GB" | 20260505 21:12:24.201 | INFO | } | 20260505 21:12:24.201 | INFO | Perf report for the last 5.1361 seconds: | 20260505 21:12:24.201 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:12:24.201 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:24.201 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:24.201 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:24.201 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 21:12:24.201 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:24.201 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:24.201 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 21:12:24.201 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:24.201 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:24.201 | INFO | Uploaded 36GiB to GPU in 2.15941s (16.6712GiB/s) | 20260505 21:12:24.201 | INFO | Downloaded 34GiB to GPU in 2.04851s (16.5974GiB/s) | 20260505 21:12:29.340 | INFO | Analyzing chunk frames = [6341632,6342144) | 20260505 21:12:29.340 | INFO | MemInfo: { | 20260505 21:12:29.340 | INFO | "gpu" : " 17 GB", | 20260505 21:12:29.340 | INFO | "huge" : "", | 20260505 21:12:29.340 | INFO | "huge_shared" : " 27 GB", | 20260505 21:12:29.340 | INFO | "malloc" : " 14 GB", | 20260505 21:12:29.340 | INFO | "pinned" : " 29 GB" | 20260505 21:12:29.340 | INFO | } | 20260505 21:12:29.340 | INFO | Perf report for the last 5.13905 seconds: | 20260505 21:12:29.340 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:12:29.340 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:29.340 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:29.340 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:29.340 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260505 21:12:29.340 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:29.340 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:29.340 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:12:29.340 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:29.340 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:29.340 | INFO | Uploaded 36GiB to GPU in 2.12036s (16.9782GiB/s) | 20260505 21:12:29.340 | INFO | Downloaded 34GiB to GPU in 2.01366s (16.8847GiB/s) | 20260505 21:12:34.481 | INFO | Analyzing chunk frames = [6342144,6342656) | 20260505 21:12:34.481 | INFO | MemInfo: { | 20260505 21:12:34.481 | INFO | "gpu" : " 17 GB", | 20260505 21:12:34.481 | INFO | "huge" : "", | 20260505 21:12:34.481 | INFO | "huge_shared" : " 27 GB", | 20260505 21:12:34.481 | INFO | "malloc" : " 14 GB", | 20260505 21:12:34.481 | INFO | "pinned" : " 29 GB" | 20260505 21:12:34.481 | INFO | } | 20260505 21:12:34.481 | INFO | Perf report for the last 5.14104 seconds: | 20260505 21:12:34.481 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:12:34.481 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:34.481 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:34.481 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:34.481 | INFO | ANALYSIS : 61.9%, 3.0, 96.2ms, 0 | 20260505 21:12:34.481 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:34.481 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:34.481 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:12:34.481 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:34.481 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:34.481 | INFO | Uploaded 36GiB to GPU in 2.1416s (16.8098GiB/s) | 20260505 21:12:34.481 | INFO | Downloaded 34GiB to GPU in 2.03429s (16.7135GiB/s) | 20260505 21:12:39.615 | INFO | Analyzing chunk frames = [6342656,6343168) | 20260505 21:12:39.615 | INFO | MemInfo: { | 20260505 21:12:39.615 | INFO | "gpu" : " 17 GB", | 20260505 21:12:39.615 | INFO | "huge" : "", | 20260505 21:12:39.615 | INFO | "huge_shared" : " 27 GB", | 20260505 21:12:39.615 | INFO | "malloc" : " 14 GB", | 20260505 21:12:39.615 | INFO | "pinned" : " 29 GB" | 20260505 21:12:39.615 | INFO | } | 20260505 21:12:39.615 | INFO | Perf report for the last 5.13413 seconds: | 20260505 21:12:39.615 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:12:39.615 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:39.615 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:39.615 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:39.615 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260505 21:12:39.615 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:39.615 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:39.615 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:12:39.615 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:39.615 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:39.615 | INFO | Uploaded 36GiB to GPU in 2.13959s (16.8256GiB/s) | 20260505 21:12:39.616 | INFO | Downloaded 34GiB to GPU in 2.06215s (16.4877GiB/s) | 20260505 21:12:44.755 | INFO | Analyzing chunk frames = [6343168,6343680) | 20260505 21:12:44.755 | INFO | MemInfo: { | 20260505 21:12:44.755 | INFO | "gpu" : " 17 GB", | 20260505 21:12:44.755 | INFO | "huge" : "", | 20260505 21:12:44.755 | INFO | "huge_shared" : " 27 GB", | 20260505 21:12:44.755 | INFO | "malloc" : " 14 GB", | 20260505 21:12:44.755 | INFO | "pinned" : " 29 GB" | 20260505 21:12:44.755 | INFO | } | 20260505 21:12:44.755 | INFO | Perf report for the last 5.14046 seconds: | 20260505 21:12:44.756 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:12:44.756 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:44.756 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:44.756 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:44.756 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 21:12:44.756 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:44.756 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:44.756 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 21:12:44.756 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:44.756 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:44.756 | INFO | Uploaded 36GiB to GPU in 2.15444s (16.7097GiB/s) | 20260505 21:12:44.756 | INFO | Downloaded 34GiB to GPU in 2.05872s (16.5151GiB/s) | 20260505 21:12:49.895 | INFO | Analyzing chunk frames = [6343680,6344192) | 20260505 21:12:49.895 | INFO | MemInfo: { | 20260505 21:12:49.895 | INFO | "gpu" : " 17 GB", | 20260505 21:12:49.895 | INFO | "huge" : "", | 20260505 21:12:49.895 | INFO | "huge_shared" : " 27 GB", | 20260505 21:12:49.895 | INFO | "malloc" : " 14 GB", | 20260505 21:12:49.895 | INFO | "pinned" : " 29 GB" | 20260505 21:12:49.895 | INFO | } | 20260505 21:12:49.895 | INFO | Perf report for the last 5.13958 seconds: | 20260505 21:12:49.895 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:12:49.895 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:49.895 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:49.895 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:49.895 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 21:12:49.895 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:49.895 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:49.895 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 21:12:49.895 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:49.895 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:49.895 | INFO | Uploaded 36GiB to GPU in 2.14382s (16.7925GiB/s) | 20260505 21:12:49.895 | INFO | Downloaded 34GiB to GPU in 2.15211s (15.7985GiB/s) | 20260505 21:12:55.045 | INFO | Analyzing chunk frames = [6344192,6344704) | 20260505 21:12:55.045 | INFO | MemInfo: { | 20260505 21:12:55.045 | INFO | "gpu" : " 17 GB", | 20260505 21:12:55.045 | INFO | "huge" : "", | 20260505 21:12:55.045 | INFO | "huge_shared" : " 27 GB", | 20260505 21:12:55.045 | INFO | "malloc" : " 14 GB", | 20260505 21:12:55.045 | INFO | "pinned" : " 29 GB" | 20260505 21:12:55.045 | INFO | } | 20260505 21:12:55.045 | INFO | Perf report for the last 5.14979 seconds: | 20260505 21:12:55.045 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:12:55.045 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:55.045 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:55.045 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:55.045 | INFO | ANALYSIS : 62.7%, 3.0, 98.1ms, 0 | 20260505 21:12:55.045 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:55.045 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:55.045 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:12:55.045 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:55.045 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:12:55.045 | INFO | Uploaded 36GiB to GPU in 2.20457s (16.3297GiB/s) | 20260505 21:12:55.045 | INFO | Downloaded 34GiB to GPU in 2.04568s (16.6203GiB/s) | 20260505 21:12:55.175 | INFO | Starting a round of Baz IO | 20260505 21:12:58.327 | INFO | Populating RTMetrics report | 20260505 21:12:58.327 | INFO | RTMetrics report populated | 20260505 21:12:58.327 | INFO | Opening tmp RTMetrics json file | 20260505 21:12:58.328 | INFO | Serializing RTMetrics to json | 20260505 21:12:58.329 | INFO | Serialization done, swapping tmp file now... | 20260505 21:12:58.329 | INFO | RTMetrics done | 20260505 21:12:58.417 | INFO | Finished a round of Baz IO: Wrote 1.979 GiB in 3.241 seconds | 20260505 21:12:58.417 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.60% / 0.00% / 78.40% | 20260505 21:12:58.417 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.153597% / 3.97364e-06% / 8.34465e-05% | 20260505 21:12:58.417 | INFO | Baz IO overall: | 20260505 21:12:58.417 | INFO | BytesWritten : 3680518785767 | 20260505 21:12:58.417 | INFO | NumEvents : 1738410141882 | 20260505 21:13:00.181 | INFO | Analyzing chunk frames = [6344704,6345216) | 20260505 21:13:00.181 | INFO | MemInfo: { | 20260505 21:13:00.181 | INFO | "gpu" : " 17 GB", | 20260505 21:13:00.181 | INFO | "huge" : "", | 20260505 21:13:00.181 | INFO | "huge_shared" : " 27 GB", | 20260505 21:13:00.181 | INFO | "malloc" : " 14 GB", | 20260505 21:13:00.181 | INFO | "pinned" : " 29 GB" | 20260505 21:13:00.182 | INFO | } | 20260505 21:13:00.182 | INFO | Perf report for the last 5.13667 seconds: | 20260505 21:13:00.182 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:13:00.182 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:00.182 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:00.182 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:00.182 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260505 21:13:00.182 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:00.182 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260505 21:13:00.182 | INFO | PRE_HQ : 59.9%, 1.8, 54.6ms, 0 | 20260505 21:13:00.182 | INFO | BAZWRITER : 62.4%, 3.7, 118.1ms, 0 | 20260505 21:13:00.182 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 21:13:00.182 | INFO | Uploaded 36GiB to GPU in 2.16053s (16.6626GiB/s) | 20260505 21:13:00.182 | INFO | Downloaded 38GiB to GPU in 2.24347s (16.9381GiB/s) | 20260505 21:13:05.313 | INFO | Analyzing chunk frames = [6345216,6345728) | 20260505 21:13:05.313 | INFO | MemInfo: { | 20260505 21:13:05.313 | INFO | "gpu" : " 17 GB", | 20260505 21:13:05.313 | INFO | "huge" : "", | 20260505 21:13:05.313 | INFO | "huge_shared" : " 27 GB", | 20260505 21:13:05.313 | INFO | "malloc" : " 14 GB", | 20260505 21:13:05.313 | INFO | "pinned" : " 29 GB" | 20260505 21:13:05.313 | INFO | } | 20260505 21:13:05.313 | INFO | Perf report for the last 5.13178 seconds: | 20260505 21:13:05.314 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:13:05.314 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:05.314 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:05.314 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:05.314 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260505 21:13:05.314 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:05.314 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:05.314 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:13:05.314 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:05.314 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:05.314 | INFO | Uploaded 36GiB to GPU in 2.08554s (17.2617GiB/s) | 20260505 21:13:05.314 | INFO | Downloaded 34GiB to GPU in 2.02631s (16.7793GiB/s) | 20260505 21:13:10.450 | INFO | Analyzing chunk frames = [6345728,6346240) | 20260505 21:13:10.450 | INFO | MemInfo: { | 20260505 21:13:10.450 | INFO | "gpu" : " 17 GB", | 20260505 21:13:10.450 | INFO | "huge" : "", | 20260505 21:13:10.450 | INFO | "huge_shared" : " 27 GB", | 20260505 21:13:10.450 | INFO | "malloc" : " 14 GB", | 20260505 21:13:10.450 | INFO | "pinned" : " 29 GB" | 20260505 21:13:10.450 | INFO | } | 20260505 21:13:10.450 | INFO | Perf report for the last 5.13687 seconds: | 20260505 21:13:10.450 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:13:10.450 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:10.450 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:10.450 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:10.450 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260505 21:13:10.450 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:10.450 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:10.450 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:13:10.450 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:10.450 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:10.450 | INFO | Uploaded 36GiB to GPU in 2.12699s (16.9253GiB/s) | 20260505 21:13:10.450 | INFO | Downloaded 34GiB to GPU in 2.02344s (16.8031GiB/s) | 20260505 21:13:15.594 | INFO | Analyzing chunk frames = [6346240,6346752) | 20260505 21:13:15.594 | INFO | MemInfo: { | 20260505 21:13:15.594 | INFO | "gpu" : " 17 GB", | 20260505 21:13:15.594 | INFO | "huge" : "", | 20260505 21:13:15.594 | INFO | "huge_shared" : " 27 GB", | 20260505 21:13:15.594 | INFO | "malloc" : " 14 GB", | 20260505 21:13:15.594 | INFO | "pinned" : " 29 GB" | 20260505 21:13:15.594 | INFO | } | 20260505 21:13:15.594 | INFO | Perf report for the last 5.14337 seconds: | 20260505 21:13:15.594 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:13:15.594 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:15.594 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:15.594 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:15.594 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260505 21:13:15.594 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:15.594 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:15.594 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:13:15.594 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:15.594 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:15.594 | INFO | Uploaded 36GiB to GPU in 2.19895s (16.3714GiB/s) | 20260505 21:13:15.594 | INFO | Downloaded 34GiB to GPU in 2.03713s (16.6901GiB/s) | 20260505 21:13:20.738 | INFO | Analyzing chunk frames = [6346752,6347264) | 20260505 21:13:20.738 | INFO | MemInfo: { | 20260505 21:13:20.738 | INFO | "gpu" : " 17 GB", | 20260505 21:13:20.738 | INFO | "huge" : "", | 20260505 21:13:20.738 | INFO | "huge_shared" : " 27 GB", | 20260505 21:13:20.738 | INFO | "malloc" : " 14 GB", | 20260505 21:13:20.738 | INFO | "pinned" : " 29 GB" | 20260505 21:13:20.738 | INFO | } | 20260505 21:13:20.738 | INFO | Perf report for the last 5.14441 seconds: | 20260505 21:13:20.738 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:13:20.738 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:20.738 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:20.738 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:20.738 | INFO | ANALYSIS : 61.4%, 3.0, 95.9ms, 0 | 20260505 21:13:20.738 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:20.738 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:20.738 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 21:13:20.738 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:20.738 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:20.739 | INFO | Uploaded 36GiB to GPU in 2.13117s (16.8921GiB/s) | 20260505 21:13:20.739 | INFO | Downloaded 34GiB to GPU in 2.06412s (16.4719GiB/s) | 20260505 21:13:25.873 | INFO | Analyzing chunk frames = [6347264,6347776) | 20260505 21:13:25.873 | INFO | MemInfo: { | 20260505 21:13:25.873 | INFO | "gpu" : " 17 GB", | 20260505 21:13:25.873 | INFO | "huge" : "", | 20260505 21:13:25.873 | INFO | "huge_shared" : " 27 GB", | 20260505 21:13:25.873 | INFO | "malloc" : " 14 GB", | 20260505 21:13:25.873 | INFO | "pinned" : " 29 GB" | 20260505 21:13:25.874 | INFO | } | 20260505 21:13:25.874 | INFO | Perf report for the last 5.13557 seconds: | 20260505 21:13:25.874 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:13:25.874 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:25.874 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:25.874 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:25.874 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260505 21:13:25.874 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:25.874 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:25.874 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 21:13:25.874 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:25.874 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:25.874 | INFO | Uploaded 36GiB to GPU in 2.07676s (17.3347GiB/s) | 20260505 21:13:25.874 | INFO | Downloaded 34GiB to GPU in 2.03475s (16.7097GiB/s) | 20260505 21:13:31.012 | INFO | Analyzing chunk frames = [6347776,6348288) | 20260505 21:13:31.012 | INFO | MemInfo: { | 20260505 21:13:31.012 | INFO | "gpu" : " 17 GB", | 20260505 21:13:31.012 | INFO | "huge" : "", | 20260505 21:13:31.012 | INFO | "huge_shared" : " 27 GB", | 20260505 21:13:31.012 | INFO | "malloc" : " 14 GB", | 20260505 21:13:31.012 | INFO | "pinned" : " 29 GB" | 20260505 21:13:31.013 | INFO | } | 20260505 21:13:31.013 | INFO | Perf report for the last 5.13909 seconds: | 20260505 21:13:31.013 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:13:31.013 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:31.013 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:31.013 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:31.013 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 21:13:31.013 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:31.013 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:31.013 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 21:13:31.013 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:31.013 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:31.013 | INFO | Uploaded 36GiB to GPU in 2.12927s (16.9072GiB/s) | 20260505 21:13:31.013 | INFO | Downloaded 34GiB to GPU in 2.15619s (15.7685GiB/s) | 20260505 21:13:36.153 | INFO | Analyzing chunk frames = [6348288,6348800) | 20260505 21:13:36.153 | INFO | MemInfo: { | 20260505 21:13:36.153 | INFO | "gpu" : " 17 GB", | 20260505 21:13:36.153 | INFO | "huge" : "", | 20260505 21:13:36.153 | INFO | "huge_shared" : " 27 GB", | 20260505 21:13:36.153 | INFO | "malloc" : " 14 GB", | 20260505 21:13:36.153 | INFO | "pinned" : " 29 GB" | 20260505 21:13:36.154 | INFO | } | 20260505 21:13:36.154 | INFO | Perf report for the last 5.14083 seconds: | 20260505 21:13:36.154 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:13:36.154 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:36.154 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:36.154 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:36.154 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260505 21:13:36.154 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:36.154 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:36.154 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:13:36.154 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:36.154 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:36.154 | INFO | Uploaded 36GiB to GPU in 2.11524s (17.0194GiB/s) | 20260505 21:13:36.154 | INFO | Downloaded 34GiB to GPU in 2.01416s (16.8805GiB/s) | 20260505 21:13:36.154 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 21:13:36.154 | INFO | Total mem usage: 15.3 GiB | 20260505 21:13:36.154 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 21:13:36.154 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.86%) | 20260505 21:13:36.154 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 21:13:36.154 | INFO | High Water Breakdown: | 20260505 21:13:36.154 | INFO | 60.9% : 8.57 GiB ( 90% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 21:13:36.154 | INFO | 39% : 5.49 GiB ( 82.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 21:13:36.154 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 21:13:36.154 | INFO | ------------------------------------------ | 20260505 21:13:36.154 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:13:36.154 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 21:13:36.154 | INFO | Total mem usage: 26.5 GiB | 20260505 21:13:36.154 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 21:13:36.154 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 21:13:36.154 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 21:13:36.154 | INFO | High Water Breakdown: | 20260505 21:13:36.154 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:13:36.154 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:13:36.154 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:13:36.154 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:13:36.154 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:13:36.154 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:13:36.154 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 21:13:36.154 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 21:13:36.154 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:13:36.154 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 21:13:36.154 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 21:13:36.154 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 21:13:36.154 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 21:13:36.154 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 21:13:36.154 | INFO | ------------------------------------------ | 20260505 21:13:36.154 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:13:36.154 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 21:13:36.154 | INFO | Total mem usage: 25.6 GiB | 20260505 21:13:36.154 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 21:13:36.154 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 21:13:36.154 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 21:13:36.155 | INFO | High Water Breakdown: | 20260505 21:13:36.155 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 21:13:36.155 | INFO | ------------------------------------------ | 20260505 21:13:36.155 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:13:36.155 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 21:13:36.155 | INFO | Total mem usage: 15.7 GiB | 20260505 21:13:36.155 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 21:13:36.155 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 21:13:36.155 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 21:13:36.155 | INFO | High Water Breakdown: | 20260505 21:13:36.155 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 21:13:36.155 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 21:13:36.155 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 21:13:36.155 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 21:13:36.155 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 21:13:36.155 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 21:13:36.155 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 21:13:36.155 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 21:13:36.155 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 21:13:36.155 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:13:36.155 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 21:13:36.155 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:13:36.155 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 21:13:36.155 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 21:13:36.155 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 21:13:36.155 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 21:13:36.155 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:13:36.155 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 21:13:36.155 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:13:36.155 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:13:36.155 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:13:36.155 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 21:13:36.155 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 21:13:36.155 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:13:36.155 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 21:13:36.155 | INFO | ------------------------------------------ | 20260505 21:13:36.155 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:13:36.155 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 21:13:36.155 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:13:36.155 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:13:36.155 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.44e+01, 1.33e+01 / 1.46e+00 | 20260505 21:13:36.155 | INFO | PerfReport: 0.29%, AggregateStats -- 9.25e-02 / 1.04e-01, 1.05e-02 / 4.16e-03 | 20260505 21:13:36.155 | INFO | PerfReport: 11.83%, Baseline -- 3.72e+00 / 4.55e+00, 5.70e-01 / 1.51e-01 | 20260505 21:13:36.155 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.51e+00, 1.85e-01 / 8.80e-02 | 20260505 21:13:36.155 | INFO | PerfReport: 15.94%, DME -- 4.01e+01 / 4.43e+01, 4.01e+00 / 1.58e+00 | 20260505 21:13:36.155 | INFO | PerfReport: 57.13%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.49e+00 / 1.08e+00 | 20260505 21:13:36.155 | INFO | PerfReport: 7.20%, PulseAccumulating -- 2.26e+00 / 2.78e+00, 4.32e-01 / 2.64e-01 | 20260505 21:13:36.155 | INFO | PerfReport: 3.21%, Metrics -- 1.01e+00 / 1.31e+00, 2.75e-01 / 1.45e-01 | 20260505 21:13:36.155 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.51e-01 / 4.09e-01, 6.07e-02 / 3.40e-02 | 20260505 21:13:36.155 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 21:13:36.155 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:13:36.155 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:13:36.155 | INFO | PerfReport: Full Execution -- 9.91e+01 / -nan, 2.51e+01 / 0.00e+00 | 20260505 21:13:36.155 | INFO | PerfReport: 22.50%, Upload -- 2.23e+01 / 0.00e+00, 7.69e+00 / 0.00e+00 | 20260505 21:13:36.155 | INFO | PerfReport: 21.84%, Download -- 2.17e+01 / 0.00e+00, 7.65e+00 / 0.00e+00 | 20260505 21:13:36.155 | INFO | PerfReport: 31.72%, Compute -- 3.14e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260505 21:13:36.155 | INFO | PerfReport: 2.22%, LockUpload -- 2.20e+00 / 0.00e+00, 7.91e+00 / 0.00e+00 | 20260505 21:13:36.155 | INFO | PerfReport: 1.40%, LockDownload -- 1.39e+00 / 0.00e+00, 2.95e+00 / 0.00e+00 | 20260505 21:13:36.155 | INFO | PerfReport: 20.31%, LockCompute -- 2.01e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 21:13:36.155 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.72e-02 / 0.00e+00 | 20260505 21:13:36.286 | INFO | Starting a round of Baz IO | 20260505 21:13:39.443 | INFO | Populating RTMetrics report | 20260505 21:13:39.444 | INFO | RTMetrics report populated | 20260505 21:13:39.444 | INFO | Opening tmp RTMetrics json file | 20260505 21:13:39.444 | INFO | Serializing RTMetrics to json | 20260505 21:13:39.446 | INFO | Serialization done, swapping tmp file now... | 20260505 21:13:39.446 | INFO | RTMetrics done | 20260505 21:13:39.528 | INFO | Finished a round of Baz IO: Wrote 1.978 GiB in 3.242 seconds | 20260505 21:13:39.528 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.60% / 0.00% / 78.40% | 20260505 21:13:39.528 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.152942% / 0% / 0.000103315% | 20260505 21:13:39.528 | INFO | Baz IO overall: | 20260505 21:13:39.528 | INFO | BytesWritten : 3682643086037 | 20260505 21:13:39.528 | INFO | NumEvents : 1739390890886 | 20260505 21:13:41.289 | INFO | Analyzing chunk frames = [6348800,6349312) | 20260505 21:13:41.289 | INFO | MemInfo: { | 20260505 21:13:41.289 | INFO | "gpu" : " 17 GB", | 20260505 21:13:41.289 | INFO | "huge" : "", | 20260505 21:13:41.289 | INFO | "huge_shared" : " 27 GB", | 20260505 21:13:41.289 | INFO | "malloc" : " 14 GB", | 20260505 21:13:41.289 | INFO | "pinned" : " 29 GB" | 20260505 21:13:41.289 | INFO | } | 20260505 21:13:41.289 | INFO | Perf report for the last 5.1354 seconds: | 20260505 21:13:41.289 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:13:41.289 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:41.289 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:41.289 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:41.289 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 21:13:41.289 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:41.289 | INFO | RT_METRICS : 43.8%, 1.0, 22.9ms, 0 | 20260505 21:13:41.289 | INFO | PRE_HQ : 60.2%, 1.8, 55.2ms, 0 | 20260505 21:13:41.289 | INFO | BAZWRITER : 62.6%, 3.7, 118.1ms, 0 | 20260505 21:13:41.289 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 21:13:41.289 | INFO | Uploaded 36GiB to GPU in 2.17574s (16.5461GiB/s) | 20260505 21:13:41.290 | INFO | Downloaded 38GiB to GPU in 2.23953s (16.9678GiB/s) | 20260505 21:13:46.430 | INFO | Analyzing chunk frames = [6349312,6349824) | 20260505 21:13:46.430 | INFO | MemInfo: { | 20260505 21:13:46.430 | INFO | "gpu" : " 17 GB", | 20260505 21:13:46.430 | INFO | "huge" : "", | 20260505 21:13:46.430 | INFO | "huge_shared" : " 27 GB", | 20260505 21:13:46.430 | INFO | "malloc" : " 14 GB", | 20260505 21:13:46.430 | INFO | "pinned" : " 29 GB" | 20260505 21:13:46.430 | INFO | } | 20260505 21:13:46.430 | INFO | Perf report for the last 5.14146 seconds: | 20260505 21:13:46.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:13:46.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:46.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:46.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:46.431 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 21:13:46.431 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:46.431 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:46.431 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:13:46.431 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:46.431 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:46.431 | INFO | Uploaded 36GiB to GPU in 2.07553s (17.345GiB/s) | 20260505 21:13:46.431 | INFO | Downloaded 34GiB to GPU in 2.02193s (16.8156GiB/s) | 20260505 21:13:51.569 | INFO | Analyzing chunk frames = [6349824,6350336) | 20260505 21:13:51.569 | INFO | MemInfo: { | 20260505 21:13:51.569 | INFO | "gpu" : " 17 GB", | 20260505 21:13:51.569 | INFO | "huge" : "", | 20260505 21:13:51.569 | INFO | "huge_shared" : " 27 GB", | 20260505 21:13:51.569 | INFO | "malloc" : " 14 GB", | 20260505 21:13:51.569 | INFO | "pinned" : " 29 GB" | 20260505 21:13:51.569 | INFO | } | 20260505 21:13:51.569 | INFO | Perf report for the last 5.13838 seconds: | 20260505 21:13:51.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:13:51.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:51.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:51.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:51.569 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260505 21:13:51.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:51.569 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:51.569 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 21:13:51.569 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:51.569 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:51.569 | INFO | Uploaded 36GiB to GPU in 2.14113s (16.8136GiB/s) | 20260505 21:13:51.569 | INFO | Downloaded 34GiB to GPU in 2.01768s (16.851GiB/s) | 20260505 21:13:56.702 | INFO | Analyzing chunk frames = [6350336,6350848) | 20260505 21:13:56.702 | INFO | MemInfo: { | 20260505 21:13:56.702 | INFO | "gpu" : " 17 GB", | 20260505 21:13:56.702 | INFO | "huge" : "", | 20260505 21:13:56.702 | INFO | "huge_shared" : " 27 GB", | 20260505 21:13:56.702 | INFO | "malloc" : " 14 GB", | 20260505 21:13:56.702 | INFO | "pinned" : " 29 GB" | 20260505 21:13:56.702 | INFO | } | 20260505 21:13:56.703 | INFO | Perf report for the last 5.1337 seconds: | 20260505 21:13:56.703 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:13:56.703 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:56.703 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:56.703 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:56.703 | INFO | ANALYSIS : 62.5%, 3.0, 97.0ms, 0 | 20260505 21:13:56.703 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:56.703 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:56.703 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:13:56.703 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:56.703 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:13:56.703 | INFO | Uploaded 36GiB to GPU in 2.20835s (16.3018GiB/s) | 20260505 21:13:56.703 | INFO | Downloaded 34GiB to GPU in 2.04469s (16.6284GiB/s) | 20260505 21:14:01.851 | INFO | Analyzing chunk frames = [6350848,6351360) | 20260505 21:14:01.851 | INFO | MemInfo: { | 20260505 21:14:01.851 | INFO | "gpu" : " 17 GB", | 20260505 21:14:01.851 | INFO | "huge" : "", | 20260505 21:14:01.851 | INFO | "huge_shared" : " 27 GB", | 20260505 21:14:01.851 | INFO | "malloc" : " 14 GB", | 20260505 21:14:01.851 | INFO | "pinned" : " 29 GB" | 20260505 21:14:01.851 | INFO | } | 20260505 21:14:01.851 | INFO | Perf report for the last 5.14822 seconds: | 20260505 21:14:01.851 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:14:01.851 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:01.851 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:01.851 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:01.851 | INFO | ANALYSIS : 61.6%, 3.0, 96.4ms, 0 | 20260505 21:14:01.851 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:01.851 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:01.851 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:14:01.851 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:01.851 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:01.851 | INFO | Uploaded 36GiB to GPU in 2.13937s (16.8274GiB/s) | 20260505 21:14:01.851 | INFO | Downloaded 34GiB to GPU in 2.0602s (16.5033GiB/s) | 20260505 21:14:06.993 | INFO | Analyzing chunk frames = [6351360,6351872) | 20260505 21:14:06.993 | INFO | MemInfo: { | 20260505 21:14:06.993 | INFO | "gpu" : " 17 GB", | 20260505 21:14:06.993 | INFO | "huge" : "", | 20260505 21:14:06.993 | INFO | "huge_shared" : " 27 GB", | 20260505 21:14:06.993 | INFO | "malloc" : " 14 GB", | 20260505 21:14:06.993 | INFO | "pinned" : " 29 GB" | 20260505 21:14:06.993 | INFO | } | 20260505 21:14:06.993 | INFO | Perf report for the last 5.14199 seconds: | 20260505 21:14:06.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:14:06.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:06.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:06.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:06.993 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260505 21:14:06.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:06.993 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:06.993 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:14:06.993 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:06.993 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:06.993 | INFO | Uploaded 36GiB to GPU in 2.08011s (17.3068GiB/s) | 20260505 21:14:06.993 | INFO | Downloaded 34GiB to GPU in 2.03476s (16.7096GiB/s) | 20260505 21:14:12.128 | INFO | Analyzing chunk frames = [6351872,6352384) | 20260505 21:14:12.129 | INFO | MemInfo: { | 20260505 21:14:12.129 | INFO | "gpu" : " 17 GB", | 20260505 21:14:12.129 | INFO | "huge" : "", | 20260505 21:14:12.129 | INFO | "huge_shared" : " 27 GB", | 20260505 21:14:12.129 | INFO | "malloc" : " 14 GB", | 20260505 21:14:12.129 | INFO | "pinned" : " 29 GB" | 20260505 21:14:12.129 | INFO | } | 20260505 21:14:12.129 | INFO | Perf report for the last 5.13578 seconds: | 20260505 21:14:12.129 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:14:12.129 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:12.129 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:12.129 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:12.129 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260505 21:14:12.129 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:12.129 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:12.129 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 21:14:12.129 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:12.129 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:12.129 | INFO | Uploaded 36GiB to GPU in 2.14502s (16.7831GiB/s) | 20260505 21:14:12.129 | INFO | Downloaded 34GiB to GPU in 2.09264s (16.2474GiB/s) | 20260505 21:14:17.261 | INFO | Analyzing chunk frames = [6352384,6352896) | 20260505 21:14:17.261 | INFO | MemInfo: { | 20260505 21:14:17.261 | INFO | "gpu" : " 17 GB", | 20260505 21:14:17.261 | INFO | "huge" : "", | 20260505 21:14:17.261 | INFO | "huge_shared" : " 27 GB", | 20260505 21:14:17.261 | INFO | "malloc" : " 14 GB", | 20260505 21:14:17.261 | INFO | "pinned" : " 29 GB" | 20260505 21:14:17.261 | INFO | } | 20260505 21:14:17.261 | INFO | Perf report for the last 5.13264 seconds: | 20260505 21:14:17.261 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:14:17.261 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:17.261 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:17.261 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:17.261 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260505 21:14:17.261 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:17.261 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:17.261 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 21:14:17.261 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:17.261 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:17.262 | INFO | Uploaded 36GiB to GPU in 2.10967s (17.0643GiB/s) | 20260505 21:14:17.262 | INFO | Downloaded 34GiB to GPU in 2.01439s (16.8785GiB/s) | 20260505 21:14:17.392 | INFO | Starting a round of Baz IO | 20260505 21:14:20.566 | INFO | Populating RTMetrics report | 20260505 21:14:20.566 | INFO | RTMetrics report populated | 20260505 21:14:20.566 | INFO | Opening tmp RTMetrics json file | 20260505 21:14:20.569 | INFO | Serializing RTMetrics to json | 20260505 21:14:20.571 | INFO | Serialization done, swapping tmp file now... | 20260505 21:14:20.571 | INFO | RTMetrics done | 20260505 21:14:20.658 | INFO | Finished a round of Baz IO: Wrote 1.979 GiB in 3.265 seconds | 20260505 21:14:20.658 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.59% / 0.00% / 78.40% | 20260505 21:14:20.658 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.153299% / 0% / 9.13938e-05% | 20260505 21:14:20.658 | INFO | Baz IO overall: | 20260505 21:14:20.658 | INFO | BytesWritten : 3684767550472 | 20260505 21:14:20.658 | INFO | NumEvents : 1740372159198 | 20260505 21:14:22.408 | INFO | Analyzing chunk frames = [6352896,6353408) | 20260505 21:14:22.408 | INFO | MemInfo: { | 20260505 21:14:22.408 | INFO | "gpu" : " 17 GB", | 20260505 21:14:22.408 | INFO | "huge" : "", | 20260505 21:14:22.408 | INFO | "huge_shared" : " 27 GB", | 20260505 21:14:22.408 | INFO | "malloc" : " 14 GB", | 20260505 21:14:22.408 | INFO | "pinned" : " 29 GB" | 20260505 21:14:22.408 | INFO | } | 20260505 21:14:22.408 | INFO | Perf report for the last 5.14647 seconds: | 20260505 21:14:22.408 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:14:22.408 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:22.408 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:22.408 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:22.408 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 21:14:22.408 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:22.408 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260505 21:14:22.408 | INFO | PRE_HQ : 60.1%, 1.8, 55.5ms, 0 | 20260505 21:14:22.408 | INFO | BAZWRITER : 63.0%, 3.7, 118.9ms, 0 | 20260505 21:14:22.408 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 21:14:22.408 | INFO | Uploaded 36GiB to GPU in 2.16595s (16.6209GiB/s) | 20260505 21:14:22.408 | INFO | Downloaded 38GiB to GPU in 2.24688s (16.9124GiB/s) | 20260505 21:14:27.549 | INFO | Analyzing chunk frames = [6353408,6353920) | 20260505 21:14:27.549 | INFO | MemInfo: { | 20260505 21:14:27.549 | INFO | "gpu" : " 17 GB", | 20260505 21:14:27.549 | INFO | "huge" : "", | 20260505 21:14:27.549 | INFO | "huge_shared" : " 27 GB", | 20260505 21:14:27.549 | INFO | "malloc" : " 14 GB", | 20260505 21:14:27.549 | INFO | "pinned" : " 29 GB" | 20260505 21:14:27.549 | INFO | } | 20260505 21:14:27.549 | INFO | Perf report for the last 5.1417 seconds: | 20260505 21:14:27.549 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:14:27.549 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:27.549 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:27.549 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:27.549 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260505 21:14:27.550 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:27.550 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:27.550 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:14:27.550 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:27.550 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:27.550 | INFO | Uploaded 36GiB to GPU in 2.07528s (17.347GiB/s) | 20260505 21:14:27.550 | INFO | Downloaded 34GiB to GPU in 2.02138s (16.8202GiB/s) | 20260505 21:14:32.687 | INFO | Analyzing chunk frames = [6353920,6354432) | 20260505 21:14:32.687 | INFO | MemInfo: { | 20260505 21:14:32.687 | INFO | "gpu" : " 17 GB", | 20260505 21:14:32.687 | INFO | "huge" : "", | 20260505 21:14:32.687 | INFO | "huge_shared" : " 27 GB", | 20260505 21:14:32.687 | INFO | "malloc" : " 14 GB", | 20260505 21:14:32.687 | INFO | "pinned" : " 29 GB" | 20260505 21:14:32.687 | INFO | } | 20260505 21:14:32.688 | INFO | Perf report for the last 5.1381 seconds: | 20260505 21:14:32.688 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:14:32.688 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:32.688 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:32.688 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:32.688 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260505 21:14:32.688 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:32.688 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:32.688 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 21:14:32.688 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:32.688 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:32.688 | INFO | Uploaded 36GiB to GPU in 2.14381s (16.7926GiB/s) | 20260505 21:14:32.688 | INFO | Downloaded 34GiB to GPU in 2.04421s (16.6323GiB/s) | 20260505 21:14:37.823 | INFO | Analyzing chunk frames = [6354432,6354944) | 20260505 21:14:37.823 | INFO | MemInfo: { | 20260505 21:14:37.823 | INFO | "gpu" : " 17 GB", | 20260505 21:14:37.823 | INFO | "huge" : "", | 20260505 21:14:37.823 | INFO | "huge_shared" : " 27 GB", | 20260505 21:14:37.823 | INFO | "malloc" : " 14 GB", | 20260505 21:14:37.823 | INFO | "pinned" : " 29 GB" | 20260505 21:14:37.823 | INFO | } | 20260505 21:14:37.823 | INFO | Perf report for the last 5.13539 seconds: | 20260505 21:14:37.823 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:14:37.823 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:37.823 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:37.823 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:37.823 | INFO | ANALYSIS : 61.8%, 3.0, 96.0ms, 0 | 20260505 21:14:37.823 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:37.823 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:37.823 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 21:14:37.823 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:37.823 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:37.823 | INFO | Uploaded 36GiB to GPU in 2.12869s (16.9118GiB/s) | 20260505 21:14:37.823 | INFO | Downloaded 34GiB to GPU in 2.02109s (16.8226GiB/s) | 20260505 21:14:42.959 | INFO | Analyzing chunk frames = [6354944,6355456) | 20260505 21:14:42.959 | INFO | MemInfo: { | 20260505 21:14:42.959 | INFO | "gpu" : " 17 GB", | 20260505 21:14:42.959 | INFO | "huge" : "", | 20260505 21:14:42.959 | INFO | "huge_shared" : " 27 GB", | 20260505 21:14:42.959 | INFO | "malloc" : " 14 GB", | 20260505 21:14:42.959 | INFO | "pinned" : " 29 GB" | 20260505 21:14:42.959 | INFO | } | 20260505 21:14:42.959 | INFO | Perf report for the last 5.13647 seconds: | 20260505 21:14:42.959 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:14:42.959 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:42.959 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:42.959 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:42.960 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260505 21:14:42.960 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:42.960 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:42.960 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 21:14:42.960 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:42.960 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:42.960 | INFO | Uploaded 36GiB to GPU in 2.12183s (16.9665GiB/s) | 20260505 21:14:42.960 | INFO | Downloaded 34GiB to GPU in 2.05885s (16.5141GiB/s) | 20260505 21:14:48.105 | INFO | Analyzing chunk frames = [6355456,6355968) | 20260505 21:14:48.105 | INFO | MemInfo: { | 20260505 21:14:48.105 | INFO | "gpu" : " 17 GB", | 20260505 21:14:48.105 | INFO | "huge" : "", | 20260505 21:14:48.105 | INFO | "huge_shared" : " 27 GB", | 20260505 21:14:48.105 | INFO | "malloc" : " 14 GB", | 20260505 21:14:48.105 | INFO | "pinned" : " 29 GB" | 20260505 21:14:48.105 | INFO | } | 20260505 21:14:48.105 | INFO | Perf report for the last 5.14527 seconds: | 20260505 21:14:48.105 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:14:48.105 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:48.105 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:48.105 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:48.105 | INFO | ANALYSIS : 61.8%, 3.0, 96.6ms, 0 | 20260505 21:14:48.105 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:48.105 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:48.105 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 21:14:48.105 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:48.105 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:48.105 | INFO | Uploaded 36GiB to GPU in 2.1376s (16.8413GiB/s) | 20260505 21:14:48.105 | INFO | Downloaded 34GiB to GPU in 2.05598s (16.5372GiB/s) | 20260505 21:14:53.248 | INFO | Analyzing chunk frames = [6355968,6356480) | 20260505 21:14:53.248 | INFO | MemInfo: { | 20260505 21:14:53.248 | INFO | "gpu" : " 17 GB", | 20260505 21:14:53.248 | INFO | "huge" : "", | 20260505 21:14:53.248 | INFO | "huge_shared" : " 27 GB", | 20260505 21:14:53.248 | INFO | "malloc" : " 14 GB", | 20260505 21:14:53.248 | INFO | "pinned" : " 29 GB" | 20260505 21:14:53.248 | INFO | } | 20260505 21:14:53.248 | INFO | Perf report for the last 5.14351 seconds: | 20260505 21:14:53.248 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:14:53.248 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:53.248 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:53.248 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:53.248 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 21:14:53.248 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:53.248 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:53.248 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 21:14:53.248 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:53.248 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:53.248 | INFO | Uploaded 36GiB to GPU in 2.15401s (16.713GiB/s) | 20260505 21:14:53.248 | INFO | Downloaded 34GiB to GPU in 2.12419s (16.0061GiB/s) | 20260505 21:14:58.380 | INFO | Analyzing chunk frames = [6356480,6356992) | 20260505 21:14:58.380 | INFO | MemInfo: { | 20260505 21:14:58.380 | INFO | "gpu" : " 17 GB", | 20260505 21:14:58.380 | INFO | "huge" : "", | 20260505 21:14:58.380 | INFO | "huge_shared" : " 27 GB", | 20260505 21:14:58.380 | INFO | "malloc" : " 14 GB", | 20260505 21:14:58.380 | INFO | "pinned" : " 29 GB" | 20260505 21:14:58.380 | INFO | } | 20260505 21:14:58.381 | INFO | Perf report for the last 5.1323 seconds: | 20260505 21:14:58.381 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:14:58.381 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:58.381 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:58.381 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:58.381 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 21:14:58.381 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:58.381 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:58.381 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 21:14:58.381 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:58.381 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:14:58.381 | INFO | Uploaded 36GiB to GPU in 2.10627s (17.0918GiB/s) | 20260505 21:14:58.381 | INFO | Downloaded 34GiB to GPU in 2.00373s (16.9684GiB/s) | 20260505 21:14:58.381 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 21:14:58.381 | INFO | Total mem usage: 15.3 GiB | 20260505 21:14:58.381 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 21:14:58.381 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.23%) | 20260505 21:14:58.381 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 21:14:58.381 | INFO | High Water Breakdown: | 20260505 21:14:58.381 | INFO | 61% : 8.64 GiB ( 90.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 21:14:58.381 | INFO | 38.9% : 5.5 GiB ( 82.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 21:14:58.381 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 21:14:58.381 | INFO | ------------------------------------------ | 20260505 21:14:58.381 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:14:58.381 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 21:14:58.381 | INFO | Total mem usage: 26.5 GiB | 20260505 21:14:58.381 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 21:14:58.381 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 21:14:58.381 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 21:14:58.381 | INFO | High Water Breakdown: | 20260505 21:14:58.381 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:14:58.381 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:14:58.381 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:14:58.381 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:14:58.381 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:14:58.381 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:14:58.381 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 21:14:58.381 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 21:14:58.381 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:14:58.381 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 21:14:58.381 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 21:14:58.381 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 21:14:58.381 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 21:14:58.381 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 21:14:58.381 | INFO | ------------------------------------------ | 20260505 21:14:58.381 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:14:58.381 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 21:14:58.381 | INFO | Total mem usage: 25.6 GiB | 20260505 21:14:58.381 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 21:14:58.381 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 21:14:58.381 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 21:14:58.381 | INFO | High Water Breakdown: | 20260505 21:14:58.381 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 21:14:58.381 | INFO | ------------------------------------------ | 20260505 21:14:58.381 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:14:58.381 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 21:14:58.381 | INFO | Total mem usage: 15.7 GiB | 20260505 21:14:58.381 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 21:14:58.381 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 21:14:58.381 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 21:14:58.381 | INFO | High Water Breakdown: | 20260505 21:14:58.381 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 21:14:58.381 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 21:14:58.381 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 21:14:58.381 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 21:14:58.381 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 21:14:58.381 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 21:14:58.381 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 21:14:58.381 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 21:14:58.381 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 21:14:58.381 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:14:58.381 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 21:14:58.381 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:14:58.381 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 21:14:58.381 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 21:14:58.381 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 21:14:58.381 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 21:14:58.381 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:14:58.381 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 21:14:58.381 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:14:58.381 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:14:58.381 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:14:58.382 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 21:14:58.382 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 21:14:58.382 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:14:58.382 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 21:14:58.382 | INFO | ------------------------------------------ | 20260505 21:14:58.382 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:14:58.382 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 21:14:58.382 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:14:58.382 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:14:58.382 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.42e+01, 1.35e+01 / 1.27e+00 | 20260505 21:14:58.382 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.04e-01, 1.06e-02 / 4.23e-03 | 20260505 21:14:58.382 | INFO | PerfReport: 11.82%, Baseline -- 3.72e+00 / 4.54e+00, 5.76e-01 / 1.45e-01 | 20260505 21:14:58.382 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.53e+00, 1.89e-01 / 7.95e-02 | 20260505 21:14:58.382 | INFO | PerfReport: 15.94%, DME -- 4.01e+01 / 4.40e+01, 3.77e+00 / 1.28e+00 | 20260505 21:14:58.382 | INFO | PerfReport: 57.17%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.53e+00 / 9.38e-01 | 20260505 21:14:58.382 | INFO | PerfReport: 7.19%, PulseAccumulating -- 2.26e+00 / 2.79e+00, 4.35e-01 / 2.11e-01 | 20260505 21:14:58.382 | INFO | PerfReport: 3.21%, Metrics -- 1.01e+00 / 1.32e+00, 2.74e-01 / 1.29e-01 | 20260505 21:14:58.382 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.51e-01 / 4.09e-01, 6.13e-02 / 4.28e-02 | 20260505 21:14:58.382 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 21:14:58.382 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:14:58.382 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:14:58.382 | INFO | PerfReport: Full Execution -- 9.91e+01 / -nan, 2.52e+01 / 0.00e+00 | 20260505 21:14:58.382 | INFO | PerfReport: 22.42%, Upload -- 2.22e+01 / 0.00e+00, 7.63e+00 / 0.00e+00 | 20260505 21:14:58.382 | INFO | PerfReport: 21.76%, Download -- 2.16e+01 / 0.00e+00, 7.62e+00 / 0.00e+00 | 20260505 21:14:58.382 | INFO | PerfReport: 31.77%, Compute -- 3.15e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 21:14:58.382 | INFO | PerfReport: 2.20%, LockUpload -- 2.18e+00 / 0.00e+00, 7.91e+00 / 0.00e+00 | 20260505 21:14:58.382 | INFO | PerfReport: 1.33%, LockDownload -- 1.32e+00 / 0.00e+00, 2.85e+00 / 0.00e+00 | 20260505 21:14:58.382 | INFO | PerfReport: 20.51%, LockCompute -- 2.03e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260505 21:14:58.382 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.11e-02 / 0.00e+00 | 20260505 21:14:58.509 | INFO | Starting a round of Baz IO | 20260505 21:15:01.683 | INFO | Populating RTMetrics report | 20260505 21:15:01.684 | INFO | RTMetrics report populated | 20260505 21:15:01.684 | INFO | Opening tmp RTMetrics json file | 20260505 21:15:01.688 | INFO | Serializing RTMetrics to json | 20260505 21:15:01.691 | INFO | Serialization done, swapping tmp file now... | 20260505 21:15:01.693 | INFO | RTMetrics done | 20260505 21:15:01.771 | INFO | Finished a round of Baz IO: Wrote 1.978 GiB in 3.261 seconds | 20260505 21:15:01.771 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.59% / 0.00% / 78.41% | 20260505 21:15:01.771 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.152886% / 0% / 8.34465e-05% | 20260505 21:15:01.771 | INFO | Baz IO overall: | 20260505 21:15:01.771 | INFO | BytesWritten : 3686890981272 | 20260505 21:15:01.771 | INFO | NumEvents : 1741352679723 | 20260505 21:15:03.521 | INFO | Analyzing chunk frames = [6356992,6357504) | 20260505 21:15:03.521 | INFO | MemInfo: { | 20260505 21:15:03.521 | INFO | "gpu" : " 17 GB", | 20260505 21:15:03.521 | INFO | "huge" : "", | 20260505 21:15:03.521 | INFO | "huge_shared" : " 27 GB", | 20260505 21:15:03.521 | INFO | "malloc" : " 14 GB", | 20260505 21:15:03.521 | INFO | "pinned" : " 29 GB" | 20260505 21:15:03.521 | INFO | } | 20260505 21:15:03.521 | INFO | Perf report for the last 5.14023 seconds: | 20260505 21:15:03.521 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:15:03.521 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:03.521 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:03.521 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:03.521 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 21:15:03.521 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:03.521 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260505 21:15:03.521 | INFO | PRE_HQ : 60.4%, 1.8, 55.5ms, 0 | 20260505 21:15:03.521 | INFO | BAZWRITER : 62.8%, 3.6, 117.7ms, 0 | 20260505 21:15:03.521 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 21:15:03.521 | INFO | Uploaded 36GiB to GPU in 2.16428s (16.6337GiB/s) | 20260505 21:15:03.521 | INFO | Downloaded 38GiB to GPU in 2.23337s (17.0146GiB/s) | 20260505 21:15:08.657 | INFO | Analyzing chunk frames = [6357504,6358016) | 20260505 21:15:08.657 | INFO | MemInfo: { | 20260505 21:15:08.657 | INFO | "gpu" : " 17 GB", | 20260505 21:15:08.657 | INFO | "huge" : "", | 20260505 21:15:08.657 | INFO | "huge_shared" : " 27 GB", | 20260505 21:15:08.657 | INFO | "malloc" : " 14 GB", | 20260505 21:15:08.657 | INFO | "pinned" : " 29 GB" | 20260505 21:15:08.657 | INFO | } | 20260505 21:15:08.657 | INFO | Perf report for the last 5.13635 seconds: | 20260505 21:15:08.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:15:08.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:08.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:08.657 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:08.657 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 21:15:08.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:08.657 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:08.657 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:15:08.657 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:08.657 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:08.657 | INFO | Uploaded 36GiB to GPU in 2.09668s (17.17GiB/s) | 20260505 21:15:08.657 | INFO | Downloaded 34GiB to GPU in 2.02627s (16.7796GiB/s) | 20260505 21:15:13.800 | INFO | Analyzing chunk frames = [6358016,6358528) | 20260505 21:15:13.800 | INFO | MemInfo: { | 20260505 21:15:13.800 | INFO | "gpu" : " 17 GB", | 20260505 21:15:13.800 | INFO | "huge" : "", | 20260505 21:15:13.801 | INFO | "huge_shared" : " 27 GB", | 20260505 21:15:13.801 | INFO | "malloc" : " 14 GB", | 20260505 21:15:13.801 | INFO | "pinned" : " 29 GB" | 20260505 21:15:13.801 | INFO | } | 20260505 21:15:13.801 | INFO | Perf report for the last 5.14347 seconds: | 20260505 21:15:13.801 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:15:13.801 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:13.801 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:13.801 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:13.801 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260505 21:15:13.801 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:13.801 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:13.801 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:15:13.801 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:13.801 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:13.801 | INFO | Uploaded 36GiB to GPU in 2.13638s (16.8509GiB/s) | 20260505 21:15:13.801 | INFO | Downloaded 34GiB to GPU in 2.0241s (16.7976GiB/s) | 20260505 21:15:18.938 | INFO | Analyzing chunk frames = [6358528,6359040) | 20260505 21:15:18.938 | INFO | MemInfo: { | 20260505 21:15:18.938 | INFO | "gpu" : " 17 GB", | 20260505 21:15:18.938 | INFO | "huge" : "", | 20260505 21:15:18.938 | INFO | "huge_shared" : " 27 GB", | 20260505 21:15:18.939 | INFO | "malloc" : " 14 GB", | 20260505 21:15:18.939 | INFO | "pinned" : " 29 GB" | 20260505 21:15:18.939 | INFO | } | 20260505 21:15:18.939 | INFO | Perf report for the last 5.13794 seconds: | 20260505 21:15:18.939 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:15:18.939 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:18.939 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:18.939 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:18.939 | INFO | ANALYSIS : 61.9%, 3.0, 96.1ms, 0 | 20260505 21:15:18.939 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:18.939 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:18.939 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 21:15:18.939 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:18.939 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:18.939 | INFO | Uploaded 36GiB to GPU in 2.13226s (16.8835GiB/s) | 20260505 21:15:18.939 | INFO | Downloaded 34GiB to GPU in 2.02639s (16.7786GiB/s) | 20260505 21:15:24.076 | INFO | Analyzing chunk frames = [6359040,6359552) | 20260505 21:15:24.076 | INFO | MemInfo: { | 20260505 21:15:24.076 | INFO | "gpu" : " 17 GB", | 20260505 21:15:24.076 | INFO | "huge" : "", | 20260505 21:15:24.076 | INFO | "huge_shared" : " 27 GB", | 20260505 21:15:24.076 | INFO | "malloc" : " 14 GB", | 20260505 21:15:24.076 | INFO | "pinned" : " 29 GB" | 20260505 21:15:24.076 | INFO | } | 20260505 21:15:24.076 | INFO | Perf report for the last 5.13718 seconds: | 20260505 21:15:24.076 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:15:24.076 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:24.076 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:24.076 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:24.076 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260505 21:15:24.076 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:24.076 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:24.076 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 21:15:24.076 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:24.076 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:24.076 | INFO | Uploaded 36GiB to GPU in 2.13095s (16.8939GiB/s) | 20260505 21:15:24.076 | INFO | Downloaded 34GiB to GPU in 2.07429s (16.3912GiB/s) | 20260505 21:15:29.211 | INFO | Analyzing chunk frames = [6359552,6360064) | 20260505 21:15:29.211 | INFO | MemInfo: { | 20260505 21:15:29.211 | INFO | "gpu" : " 17 GB", | 20260505 21:15:29.211 | INFO | "huge" : "", | 20260505 21:15:29.211 | INFO | "huge_shared" : " 27 GB", | 20260505 21:15:29.211 | INFO | "malloc" : " 14 GB", | 20260505 21:15:29.211 | INFO | "pinned" : " 29 GB" | 20260505 21:15:29.211 | INFO | } | 20260505 21:15:29.211 | INFO | Perf report for the last 5.13532 seconds: | 20260505 21:15:29.211 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:15:29.211 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:29.211 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:29.211 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:29.211 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 21:15:29.211 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:29.211 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:29.211 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:15:29.211 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:29.211 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:29.211 | INFO | Uploaded 36GiB to GPU in 2.15822s (16.6804GiB/s) | 20260505 21:15:29.211 | INFO | Downloaded 34GiB to GPU in 2.05898s (16.513GiB/s) | 20260505 21:15:34.362 | INFO | Analyzing chunk frames = [6360064,6360576) | 20260505 21:15:34.362 | INFO | MemInfo: { | 20260505 21:15:34.362 | INFO | "gpu" : " 17 GB", | 20260505 21:15:34.362 | INFO | "huge" : "", | 20260505 21:15:34.362 | INFO | "huge_shared" : " 27 GB", | 20260505 21:15:34.362 | INFO | "malloc" : " 14 GB", | 20260505 21:15:34.362 | INFO | "pinned" : " 29 GB" | 20260505 21:15:34.362 | INFO | } | 20260505 21:15:34.362 | INFO | Perf report for the last 5.15072 seconds: | 20260505 21:15:34.362 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:15:34.362 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:34.362 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:34.362 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:34.362 | INFO | ANALYSIS : 63.0%, 3.0, 98.6ms, 0 | 20260505 21:15:34.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:34.362 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:34.362 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 21:15:34.362 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:34.362 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:34.362 | INFO | Uploaded 36GiB to GPU in 2.13679s (16.8477GiB/s) | 20260505 21:15:34.362 | INFO | Downloaded 34GiB to GPU in 2.16866s (15.6779GiB/s) | 20260505 21:15:39.504 | INFO | Analyzing chunk frames = [6360576,6361088) | 20260505 21:15:39.504 | INFO | MemInfo: { | 20260505 21:15:39.504 | INFO | "gpu" : " 17 GB", | 20260505 21:15:39.504 | INFO | "huge" : "", | 20260505 21:15:39.504 | INFO | "huge_shared" : " 27 GB", | 20260505 21:15:39.504 | INFO | "malloc" : " 14 GB", | 20260505 21:15:39.504 | INFO | "pinned" : " 29 GB" | 20260505 21:15:39.504 | INFO | } | 20260505 21:15:39.504 | INFO | Perf report for the last 5.14266 seconds: | 20260505 21:15:39.504 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:15:39.504 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:39.504 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:39.504 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:39.505 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260505 21:15:39.505 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:39.505 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:39.505 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 21:15:39.505 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:39.505 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:39.505 | INFO | Uploaded 36GiB to GPU in 2.10944s (17.0661GiB/s) | 20260505 21:15:39.505 | INFO | Downloaded 34GiB to GPU in 2.01601s (16.865GiB/s) | 20260505 21:15:39.625 | INFO | Starting a round of Baz IO | 20260505 21:15:42.787 | INFO | Populating RTMetrics report | 20260505 21:15:42.787 | INFO | RTMetrics report populated | 20260505 21:15:42.787 | INFO | Opening tmp RTMetrics json file | 20260505 21:15:42.788 | INFO | Serializing RTMetrics to json | 20260505 21:15:42.793 | INFO | Serialization done, swapping tmp file now... | 20260505 21:15:42.795 | INFO | RTMetrics done | 20260505 21:15:42.875 | INFO | Finished a round of Baz IO: Wrote 1.977 GiB in 3.249 seconds | 20260505 21:15:42.875 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.59% / 0.00% / 78.41% | 20260505 21:15:42.875 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.152385% / 0% / 0.000107288% | 20260505 21:15:42.875 | INFO | Baz IO overall: | 20260505 21:15:42.875 | INFO | BytesWritten : 3689014114796 | 20260505 21:15:42.875 | INFO | NumEvents : 1742332976909 | 20260505 21:15:44.636 | INFO | Analyzing chunk frames = [6361088,6361600) | 20260505 21:15:44.636 | INFO | MemInfo: { | 20260505 21:15:44.636 | INFO | "gpu" : " 17 GB", | 20260505 21:15:44.636 | INFO | "huge" : "", | 20260505 21:15:44.637 | INFO | "huge_shared" : " 27 GB", | 20260505 21:15:44.637 | INFO | "malloc" : " 14 GB", | 20260505 21:15:44.637 | INFO | "pinned" : " 29 GB" | 20260505 21:15:44.637 | INFO | } | 20260505 21:15:44.637 | INFO | Perf report for the last 5.1321 seconds: | 20260505 21:15:44.637 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:15:44.637 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:44.637 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:44.637 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:44.637 | INFO | ANALYSIS : 62.9%, 3.0, 98.0ms, 0 | 20260505 21:15:44.637 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:44.637 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260505 21:15:44.637 | INFO | PRE_HQ : 60.3%, 1.8, 54.9ms, 0 | 20260505 21:15:44.637 | INFO | BAZWRITER : 62.5%, 3.7, 117.7ms, 0 | 20260505 21:15:44.637 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 21:15:44.637 | INFO | Uploaded 36GiB to GPU in 2.16079s (16.6605GiB/s) | 20260505 21:15:44.637 | INFO | Downloaded 38GiB to GPU in 2.2299s (17.0411GiB/s) | 20260505 21:15:49.772 | INFO | Analyzing chunk frames = [6361600,6362112) | 20260505 21:15:49.772 | INFO | MemInfo: { | 20260505 21:15:49.772 | INFO | "gpu" : " 17 GB", | 20260505 21:15:49.772 | INFO | "huge" : "", | 20260505 21:15:49.772 | INFO | "huge_shared" : " 27 GB", | 20260505 21:15:49.772 | INFO | "malloc" : " 14 GB", | 20260505 21:15:49.772 | INFO | "pinned" : " 29 GB" | 20260505 21:15:49.772 | INFO | } | 20260505 21:15:49.772 | INFO | Perf report for the last 5.13588 seconds: | 20260505 21:15:49.772 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:15:49.772 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:49.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:49.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:49.773 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 21:15:49.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:49.773 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:49.773 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:15:49.773 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:49.773 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:49.773 | INFO | Uploaded 36GiB to GPU in 2.09567s (17.1782GiB/s) | 20260505 21:15:49.773 | INFO | Downloaded 34GiB to GPU in 2.02229s (16.8126GiB/s) | 20260505 21:15:54.913 | INFO | Analyzing chunk frames = [6362112,6362624) | 20260505 21:15:54.913 | INFO | MemInfo: { | 20260505 21:15:54.913 | INFO | "gpu" : " 17 GB", | 20260505 21:15:54.913 | INFO | "huge" : "", | 20260505 21:15:54.913 | INFO | "huge_shared" : " 27 GB", | 20260505 21:15:54.913 | INFO | "malloc" : " 14 GB", | 20260505 21:15:54.913 | INFO | "pinned" : " 29 GB" | 20260505 21:15:54.913 | INFO | } | 20260505 21:15:54.913 | INFO | Perf report for the last 5.14088 seconds: | 20260505 21:15:54.913 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:15:54.913 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:54.913 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:54.913 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:54.913 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260505 21:15:54.913 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:54.913 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:54.913 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:15:54.913 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:54.913 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:15:54.914 | INFO | Uploaded 36GiB to GPU in 2.13731s (16.8436GiB/s) | 20260505 21:15:54.914 | INFO | Downloaded 34GiB to GPU in 2.02052s (16.8274GiB/s) | 20260505 21:16:00.053 | INFO | Analyzing chunk frames = [6362624,6363136) | 20260505 21:16:00.053 | INFO | MemInfo: { | 20260505 21:16:00.053 | INFO | "gpu" : " 17 GB", | 20260505 21:16:00.053 | INFO | "huge" : "", | 20260505 21:16:00.053 | INFO | "huge_shared" : " 27 GB", | 20260505 21:16:00.053 | INFO | "malloc" : " 14 GB", | 20260505 21:16:00.053 | INFO | "pinned" : " 29 GB" | 20260505 21:16:00.053 | INFO | } | 20260505 21:16:00.053 | INFO | Perf report for the last 5.14006 seconds: | 20260505 21:16:00.053 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:16:00.053 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:00.053 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:00.053 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:00.053 | INFO | ANALYSIS : 62.0%, 3.0, 96.3ms, 0 | 20260505 21:16:00.054 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:00.054 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:00.054 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:16:00.054 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:00.054 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:00.054 | INFO | Uploaded 36GiB to GPU in 2.14589s (16.7762GiB/s) | 20260505 21:16:00.054 | INFO | Downloaded 34GiB to GPU in 2.04167s (16.653GiB/s) | 20260505 21:16:05.198 | INFO | Analyzing chunk frames = [6363136,6363648) | 20260505 21:16:05.198 | INFO | MemInfo: { | 20260505 21:16:05.198 | INFO | "gpu" : " 17 GB", | 20260505 21:16:05.198 | INFO | "huge" : "", | 20260505 21:16:05.198 | INFO | "huge_shared" : " 27 GB", | 20260505 21:16:05.198 | INFO | "malloc" : " 14 GB", | 20260505 21:16:05.198 | INFO | "pinned" : " 29 GB" | 20260505 21:16:05.198 | INFO | } | 20260505 21:16:05.198 | INFO | Perf report for the last 5.14454 seconds: | 20260505 21:16:05.198 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:16:05.198 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:05.198 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:05.198 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:05.198 | INFO | ANALYSIS : 61.6%, 3.0, 96.3ms, 0 | 20260505 21:16:05.198 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:05.198 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:05.198 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:16:05.198 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:05.198 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:05.198 | INFO | Uploaded 36GiB to GPU in 2.13451s (16.8657GiB/s) | 20260505 21:16:05.198 | INFO | Downloaded 34GiB to GPU in 2.06185s (16.4901GiB/s) | 20260505 21:16:10.335 | INFO | Analyzing chunk frames = [6363648,6364160) | 20260505 21:16:10.335 | INFO | MemInfo: { | 20260505 21:16:10.335 | INFO | "gpu" : " 17 GB", | 20260505 21:16:10.335 | INFO | "huge" : "", | 20260505 21:16:10.335 | INFO | "huge_shared" : " 27 GB", | 20260505 21:16:10.335 | INFO | "malloc" : " 14 GB", | 20260505 21:16:10.335 | INFO | "pinned" : " 29 GB" | 20260505 21:16:10.336 | INFO | } | 20260505 21:16:10.336 | INFO | Perf report for the last 5.13754 seconds: | 20260505 21:16:10.336 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:16:10.336 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:10.336 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:10.336 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:10.336 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260505 21:16:10.336 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:10.336 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:10.336 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 21:16:10.336 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:10.336 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:10.336 | INFO | Uploaded 36GiB to GPU in 2.07876s (17.318GiB/s) | 20260505 21:16:10.336 | INFO | Downloaded 34GiB to GPU in 2.02666s (16.7764GiB/s) | 20260505 21:16:15.469 | INFO | Analyzing chunk frames = [6364160,6364672) | 20260505 21:16:15.469 | INFO | MemInfo: { | 20260505 21:16:15.469 | INFO | "gpu" : " 17 GB", | 20260505 21:16:15.469 | INFO | "huge" : "", | 20260505 21:16:15.469 | INFO | "huge_shared" : " 27 GB", | 20260505 21:16:15.469 | INFO | "malloc" : " 14 GB", | 20260505 21:16:15.469 | INFO | "pinned" : " 29 GB" | 20260505 21:16:15.469 | INFO | } | 20260505 21:16:15.469 | INFO | Perf report for the last 5.13311 seconds: | 20260505 21:16:15.469 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:16:15.469 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:15.469 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:15.469 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:15.469 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 21:16:15.469 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:15.469 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:15.469 | INFO | PRE_HQ : 27.9%, 1.0, 14.4ms, 0 | 20260505 21:16:15.469 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:15.469 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:15.469 | INFO | Uploaded 36GiB to GPU in 2.13659s (16.8493GiB/s) | 20260505 21:16:15.469 | INFO | Downloaded 34GiB to GPU in 2.14368s (15.8605GiB/s) | 20260505 21:16:20.612 | INFO | Analyzing chunk frames = [6364672,6365184) | 20260505 21:16:20.612 | INFO | MemInfo: { | 20260505 21:16:20.612 | INFO | "gpu" : " 17 GB", | 20260505 21:16:20.612 | INFO | "huge" : "", | 20260505 21:16:20.612 | INFO | "huge_shared" : " 27 GB", | 20260505 21:16:20.612 | INFO | "malloc" : " 14 GB", | 20260505 21:16:20.612 | INFO | "pinned" : " 29 GB" | 20260505 21:16:20.612 | INFO | } | 20260505 21:16:20.612 | INFO | Perf report for the last 5.14366 seconds: | 20260505 21:16:20.612 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:16:20.612 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:20.612 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:20.612 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:20.612 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260505 21:16:20.612 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:20.612 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:20.612 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:16:20.612 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:20.612 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:20.613 | INFO | Uploaded 36GiB to GPU in 2.12513s (16.9401GiB/s) | 20260505 21:16:20.613 | INFO | Downloaded 34GiB to GPU in 2.01124s (16.905GiB/s) | 20260505 21:16:20.613 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 21:16:20.613 | INFO | Total mem usage: 15.3 GiB | 20260505 21:16:20.613 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 21:16:20.613 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.45%) | 20260505 21:16:20.613 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 21:16:20.613 | INFO | High Water Breakdown: | 20260505 21:16:20.613 | INFO | 61.1% : 8.63 GiB ( 90.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 21:16:20.613 | INFO | 38.8% : 5.48 GiB ( 82.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 21:16:20.613 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 21:16:20.613 | INFO | ------------------------------------------ | 20260505 21:16:20.613 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:16:20.613 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 21:16:20.613 | INFO | Total mem usage: 26.5 GiB | 20260505 21:16:20.613 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 21:16:20.613 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 21:16:20.613 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 21:16:20.613 | INFO | High Water Breakdown: | 20260505 21:16:20.613 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:16:20.613 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:16:20.613 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:16:20.613 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:16:20.613 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:16:20.613 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:16:20.613 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 21:16:20.613 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 21:16:20.613 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:16:20.613 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 21:16:20.613 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 21:16:20.613 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 21:16:20.613 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 21:16:20.613 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 21:16:20.613 | INFO | ------------------------------------------ | 20260505 21:16:20.613 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:16:20.613 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 21:16:20.613 | INFO | Total mem usage: 25.6 GiB | 20260505 21:16:20.613 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 21:16:20.613 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 21:16:20.613 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 21:16:20.613 | INFO | High Water Breakdown: | 20260505 21:16:20.613 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 21:16:20.613 | INFO | ------------------------------------------ | 20260505 21:16:20.613 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:16:20.613 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 21:16:20.613 | INFO | Total mem usage: 15.7 GiB | 20260505 21:16:20.613 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 21:16:20.613 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 21:16:20.613 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 21:16:20.613 | INFO | High Water Breakdown: | 20260505 21:16:20.613 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 21:16:20.613 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 21:16:20.613 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 21:16:20.613 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 21:16:20.613 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 21:16:20.613 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 21:16:20.613 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 21:16:20.613 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 21:16:20.613 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 21:16:20.613 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:16:20.613 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 21:16:20.613 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:16:20.613 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 21:16:20.613 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 21:16:20.613 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 21:16:20.613 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 21:16:20.613 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:16:20.613 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 21:16:20.613 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:16:20.613 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:16:20.613 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:16:20.613 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 21:16:20.613 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 21:16:20.613 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:16:20.613 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 21:16:20.613 | INFO | ------------------------------------------ | 20260505 21:16:20.613 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:16:20.613 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 21:16:20.613 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:16:20.613 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:16:20.613 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.46e+01, 1.35e+01 / 1.91e+00 | 20260505 21:16:20.613 | INFO | PerfReport: 0.29%, AggregateStats -- 9.24e-02 / 1.04e-01, 1.05e-02 / 6.74e-03 | 20260505 21:16:20.613 | INFO | PerfReport: 11.81%, Baseline -- 3.72e+00 / 4.49e+00, 5.72e-01 / 2.03e-01 | 20260505 21:16:20.613 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.50e+00, 1.90e-01 / 1.08e-01 | 20260505 21:16:20.613 | INFO | PerfReport: 15.95%, DME -- 4.02e+01 / 4.45e+01, 4.16e+00 / 2.11e+00 | 20260505 21:16:20.613 | INFO | PerfReport: 57.27%, FrameLabeling -- 1.81e+01 / 1.95e+01, 1.62e+00 / 1.23e+00 | 20260505 21:16:20.613 | INFO | PerfReport: 7.11%, PulseAccumulating -- 2.24e+00 / 2.76e+00, 4.26e-01 / 2.64e-01 | 20260505 21:16:20.613 | INFO | PerfReport: 3.17%, Metrics -- 1.00e+00 / 1.30e+00, 2.72e-01 / 1.53e-01 | 20260505 21:16:20.613 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.51e-01 / 4.10e-01, 6.32e-02 / 4.19e-02 | 20260505 21:16:20.614 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 21:16:20.614 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:16:20.614 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:16:20.614 | INFO | PerfReport: Full Execution -- 9.91e+01 / -nan, 2.50e+01 / 0.00e+00 | 20260505 21:16:20.614 | INFO | PerfReport: 22.38%, Upload -- 2.22e+01 / 0.00e+00, 7.62e+00 / 0.00e+00 | 20260505 21:16:20.614 | INFO | PerfReport: 21.80%, Download -- 2.16e+01 / 0.00e+00, 7.67e+00 / 0.00e+00 | 20260505 21:16:20.614 | INFO | PerfReport: 31.82%, Compute -- 3.15e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 21:16:20.614 | INFO | PerfReport: 2.19%, LockUpload -- 2.17e+00 / 0.00e+00, 7.96e+00 / 0.00e+00 | 20260505 21:16:20.614 | INFO | PerfReport: 1.35%, LockDownload -- 1.34e+00 / 0.00e+00, 2.92e+00 / 0.00e+00 | 20260505 21:16:20.614 | INFO | PerfReport: 20.45%, LockCompute -- 2.03e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 21:16:20.614 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.56e-02 / 0.00e+00 | 20260505 21:16:20.740 | INFO | Starting a round of Baz IO | 20260505 21:16:23.890 | INFO | Populating RTMetrics report | 20260505 21:16:23.890 | INFO | RTMetrics report populated | 20260505 21:16:23.890 | INFO | Opening tmp RTMetrics json file | 20260505 21:16:23.892 | INFO | Serializing RTMetrics to json | 20260505 21:16:23.897 | INFO | Serialization done, swapping tmp file now... | 20260505 21:16:23.897 | INFO | RTMetrics done | 20260505 21:16:23.976 | INFO | Finished a round of Baz IO: Wrote 1.977 GiB in 3.236 seconds | 20260505 21:16:23.976 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.59% / 0.00% / 78.41% | 20260505 21:16:23.976 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.151638% / 0% / 6.75519e-05% | 20260505 21:16:23.976 | INFO | Baz IO overall: | 20260505 21:16:23.976 | INFO | BytesWritten : 3691136916874 | 20260505 21:16:23.976 | INFO | NumEvents : 1743313404739 | 20260505 21:16:25.751 | INFO | Analyzing chunk frames = [6365184,6365696) | 20260505 21:16:25.751 | INFO | MemInfo: { | 20260505 21:16:25.751 | INFO | "gpu" : " 17 GB", | 20260505 21:16:25.751 | INFO | "huge" : "", | 20260505 21:16:25.751 | INFO | "huge_shared" : " 27 GB", | 20260505 21:16:25.751 | INFO | "malloc" : " 14 GB", | 20260505 21:16:25.751 | INFO | "pinned" : " 29 GB" | 20260505 21:16:25.751 | INFO | } | 20260505 21:16:25.751 | INFO | Perf report for the last 5.13883 seconds: | 20260505 21:16:25.751 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:16:25.751 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:25.751 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:25.751 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:25.751 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 21:16:25.751 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:25.751 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260505 21:16:25.751 | INFO | PRE_HQ : 59.8%, 1.8, 54.9ms, 0 | 20260505 21:16:25.751 | INFO | BAZWRITER : 62.3%, 3.7, 118.4ms, 0 | 20260505 21:16:25.751 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 21:16:25.751 | INFO | Uploaded 36GiB to GPU in 2.1313s (16.8911GiB/s) | 20260505 21:16:25.751 | INFO | Downloaded 38GiB to GPU in 2.22312s (17.0931GiB/s) | 20260505 21:16:30.887 | INFO | Analyzing chunk frames = [6365696,6366208) | 20260505 21:16:30.887 | INFO | MemInfo: { | 20260505 21:16:30.887 | INFO | "gpu" : " 17 GB", | 20260505 21:16:30.887 | INFO | "huge" : "", | 20260505 21:16:30.887 | INFO | "huge_shared" : " 27 GB", | 20260505 21:16:30.887 | INFO | "malloc" : " 14 GB", | 20260505 21:16:30.887 | INFO | "pinned" : " 29 GB" | 20260505 21:16:30.887 | INFO | } | 20260505 21:16:30.887 | INFO | Perf report for the last 5.13578 seconds: | 20260505 21:16:30.887 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:16:30.887 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:30.887 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:30.887 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:30.887 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260505 21:16:30.887 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:30.887 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:30.887 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 21:16:30.887 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:30.887 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:30.887 | INFO | Uploaded 36GiB to GPU in 2.08942s (17.2296GiB/s) | 20260505 21:16:30.887 | INFO | Downloaded 34GiB to GPU in 2.01931s (16.8374GiB/s) | 20260505 21:16:36.025 | INFO | Analyzing chunk frames = [6366208,6366720) | 20260505 21:16:36.025 | INFO | MemInfo: { | 20260505 21:16:36.025 | INFO | "gpu" : " 17 GB", | 20260505 21:16:36.025 | INFO | "huge" : "", | 20260505 21:16:36.025 | INFO | "huge_shared" : " 27 GB", | 20260505 21:16:36.025 | INFO | "malloc" : " 14 GB", | 20260505 21:16:36.025 | INFO | "pinned" : " 29 GB" | 20260505 21:16:36.025 | INFO | } | 20260505 21:16:36.025 | INFO | Perf report for the last 5.13826 seconds: | 20260505 21:16:36.025 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:16:36.025 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:36.025 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:36.025 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:36.025 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260505 21:16:36.025 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:36.025 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:36.025 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:16:36.025 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:36.025 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:36.025 | INFO | Uploaded 36GiB to GPU in 2.12953s (16.9051GiB/s) | 20260505 21:16:36.026 | INFO | Downloaded 34GiB to GPU in 2.0127s (16.8927GiB/s) | 20260505 21:16:41.166 | INFO | Analyzing chunk frames = [6366720,6367232) | 20260505 21:16:41.166 | INFO | MemInfo: { | 20260505 21:16:41.166 | INFO | "gpu" : " 17 GB", | 20260505 21:16:41.166 | INFO | "huge" : "", | 20260505 21:16:41.166 | INFO | "huge_shared" : " 27 GB", | 20260505 21:16:41.166 | INFO | "malloc" : " 14 GB", | 20260505 21:16:41.166 | INFO | "pinned" : " 29 GB" | 20260505 21:16:41.166 | INFO | } | 20260505 21:16:41.166 | INFO | Perf report for the last 5.14106 seconds: | 20260505 21:16:41.166 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:16:41.166 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:41.166 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:41.166 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:41.166 | INFO | ANALYSIS : 62.4%, 3.0, 97.0ms, 0 | 20260505 21:16:41.166 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:41.166 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:41.166 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:16:41.166 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:41.166 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:41.166 | INFO | Uploaded 36GiB to GPU in 2.20946s (16.2936GiB/s) | 20260505 21:16:41.167 | INFO | Downloaded 34GiB to GPU in 2.03557s (16.7029GiB/s) | 20260505 21:16:46.313 | INFO | Analyzing chunk frames = [6367232,6367744) | 20260505 21:16:46.313 | INFO | MemInfo: { | 20260505 21:16:46.313 | INFO | "gpu" : " 17 GB", | 20260505 21:16:46.313 | INFO | "huge" : "", | 20260505 21:16:46.313 | INFO | "huge_shared" : " 27 GB", | 20260505 21:16:46.313 | INFO | "malloc" : " 14 GB", | 20260505 21:16:46.313 | INFO | "pinned" : " 29 GB" | 20260505 21:16:46.313 | INFO | } | 20260505 21:16:46.313 | INFO | Perf report for the last 5.14646 seconds: | 20260505 21:16:46.313 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:16:46.313 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:46.313 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:46.313 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:46.313 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260505 21:16:46.313 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:46.313 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:46.313 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 21:16:46.313 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:46.313 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:46.313 | INFO | Uploaded 36GiB to GPU in 2.14214s (16.8056GiB/s) | 20260505 21:16:46.313 | INFO | Downloaded 34GiB to GPU in 2.06414s (16.4718GiB/s) | 20260505 21:16:51.450 | INFO | Analyzing chunk frames = [6367744,6368256) | 20260505 21:16:51.450 | INFO | MemInfo: { | 20260505 21:16:51.450 | INFO | "gpu" : " 17 GB", | 20260505 21:16:51.450 | INFO | "huge" : "", | 20260505 21:16:51.450 | INFO | "huge_shared" : " 27 GB", | 20260505 21:16:51.450 | INFO | "malloc" : " 14 GB", | 20260505 21:16:51.450 | INFO | "pinned" : " 29 GB" | 20260505 21:16:51.450 | INFO | } | 20260505 21:16:51.451 | INFO | Perf report for the last 5.13769 seconds: | 20260505 21:16:51.451 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:16:51.451 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:51.451 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:51.451 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:51.451 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 21:16:51.451 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:51.451 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:51.451 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:16:51.451 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:51.451 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:51.451 | INFO | Uploaded 36GiB to GPU in 2.10294s (17.1189GiB/s) | 20260505 21:16:51.451 | INFO | Downloaded 34GiB to GPU in 2.04084s (16.6598GiB/s) | 20260505 21:16:56.585 | INFO | Analyzing chunk frames = [6368256,6368768) | 20260505 21:16:56.585 | INFO | MemInfo: { | 20260505 21:16:56.585 | INFO | "gpu" : " 17 GB", | 20260505 21:16:56.585 | INFO | "huge" : "", | 20260505 21:16:56.585 | INFO | "huge_shared" : " 27 GB", | 20260505 21:16:56.585 | INFO | "malloc" : " 14 GB", | 20260505 21:16:56.586 | INFO | "pinned" : " 29 GB" | 20260505 21:16:56.586 | INFO | } | 20260505 21:16:56.586 | INFO | Perf report for the last 5.13504 seconds: | 20260505 21:16:56.586 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:16:56.586 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:56.586 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:56.586 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:56.586 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260505 21:16:56.586 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:56.586 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:56.586 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 21:16:56.586 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:56.586 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:16:56.586 | INFO | Uploaded 36GiB to GPU in 2.15723s (16.6881GiB/s) | 20260505 21:16:56.586 | INFO | Downloaded 34GiB to GPU in 2.12394s (16.0079GiB/s) | 20260505 21:17:01.722 | INFO | Analyzing chunk frames = [6368768,6369280) | 20260505 21:17:01.722 | INFO | MemInfo: { | 20260505 21:17:01.722 | INFO | "gpu" : " 17 GB", | 20260505 21:17:01.722 | INFO | "huge" : "", | 20260505 21:17:01.722 | INFO | "huge_shared" : " 27 GB", | 20260505 21:17:01.722 | INFO | "malloc" : " 14 GB", | 20260505 21:17:01.722 | INFO | "pinned" : " 29 GB" | 20260505 21:17:01.722 | INFO | } | 20260505 21:17:01.722 | INFO | Perf report for the last 5.13663 seconds: | 20260505 21:17:01.722 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:17:01.722 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:01.722 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:01.722 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:01.722 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260505 21:17:01.722 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:01.722 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:01.722 | INFO | PRE_HQ : 27.5%, 1.0, 14.2ms, 0 | 20260505 21:17:01.722 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:01.722 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:01.722 | INFO | Uploaded 36GiB to GPU in 2.12024s (16.9792GiB/s) | 20260505 21:17:01.722 | INFO | Downloaded 34GiB to GPU in 2.01745s (16.853GiB/s) | 20260505 21:17:01.855 | INFO | Starting a round of Baz IO | 20260505 21:17:05.024 | INFO | Populating RTMetrics report | 20260505 21:17:05.024 | INFO | RTMetrics report populated | 20260505 21:17:05.024 | INFO | Opening tmp RTMetrics json file | 20260505 21:17:05.025 | INFO | Serializing RTMetrics to json | 20260505 21:17:05.027 | INFO | Serialization done, swapping tmp file now... | 20260505 21:17:05.027 | INFO | RTMetrics done | 20260505 21:17:05.115 | INFO | Finished a round of Baz IO: Wrote 1.976 GiB in 3.259 seconds | 20260505 21:17:05.115 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.59% / 0.00% / 78.41% | 20260505 21:17:05.115 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.151924% / 0% / 0.000119209% | 20260505 21:17:05.115 | INFO | Baz IO overall: | 20260505 21:17:05.115 | INFO | BytesWritten : 3693258846803 | 20260505 21:17:05.115 | INFO | NumEvents : 1744293158746 | 20260505 21:17:06.866 | INFO | Analyzing chunk frames = [6369280,6369792) | 20260505 21:17:06.866 | INFO | MemInfo: { | 20260505 21:17:06.866 | INFO | "gpu" : " 17 GB", | 20260505 21:17:06.866 | INFO | "huge" : "", | 20260505 21:17:06.866 | INFO | "huge_shared" : " 27 GB", | 20260505 21:17:06.866 | INFO | "malloc" : " 14 GB", | 20260505 21:17:06.866 | INFO | "pinned" : " 29 GB" | 20260505 21:17:06.866 | INFO | } | 20260505 21:17:06.866 | INFO | Perf report for the last 5.14386 seconds: | 20260505 21:17:06.866 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:17:06.866 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:06.866 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:06.866 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:06.866 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 21:17:06.866 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:06.866 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260505 21:17:06.866 | INFO | PRE_HQ : 60.0%, 1.8, 54.9ms, 0 | 20260505 21:17:06.866 | INFO | BAZWRITER : 62.8%, 3.7, 118.8ms, 0 | 20260505 21:17:06.866 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 21:17:06.866 | INFO | Uploaded 36GiB to GPU in 2.18096s (16.5065GiB/s) | 20260505 21:17:06.866 | INFO | Downloaded 38GiB to GPU in 2.25771s (16.8312GiB/s) | 20260505 21:17:12.009 | INFO | Analyzing chunk frames = [6369792,6370304) | 20260505 21:17:12.009 | INFO | MemInfo: { | 20260505 21:17:12.009 | INFO | "gpu" : " 17 GB", | 20260505 21:17:12.009 | INFO | "huge" : "", | 20260505 21:17:12.009 | INFO | "huge_shared" : " 27 GB", | 20260505 21:17:12.009 | INFO | "malloc" : " 14 GB", | 20260505 21:17:12.009 | INFO | "pinned" : " 29 GB" | 20260505 21:17:12.009 | INFO | } | 20260505 21:17:12.009 | INFO | Perf report for the last 5.14274 seconds: | 20260505 21:17:12.009 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:17:12.009 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:12.009 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:12.009 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:12.009 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 21:17:12.009 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:12.009 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:12.009 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:17:12.009 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:12.009 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:12.009 | INFO | Uploaded 36GiB to GPU in 2.07456s (17.353GiB/s) | 20260505 21:17:12.009 | INFO | Downloaded 34GiB to GPU in 2.02552s (16.7858GiB/s) | 20260505 21:17:17.151 | INFO | Analyzing chunk frames = [6370304,6370816) | 20260505 21:17:17.151 | INFO | MemInfo: { | 20260505 21:17:17.151 | INFO | "gpu" : " 17 GB", | 20260505 21:17:17.151 | INFO | "huge" : "", | 20260505 21:17:17.151 | INFO | "huge_shared" : " 27 GB", | 20260505 21:17:17.151 | INFO | "malloc" : " 14 GB", | 20260505 21:17:17.151 | INFO | "pinned" : " 29 GB" | 20260505 21:17:17.151 | INFO | } | 20260505 21:17:17.151 | INFO | Perf report for the last 5.14232 seconds: | 20260505 21:17:17.151 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:17:17.151 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:17.151 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:17.151 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:17.151 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260505 21:17:17.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:17.151 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:17.151 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 21:17:17.151 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:17.151 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:17.152 | INFO | Uploaded 36GiB to GPU in 2.12455s (16.9448GiB/s) | 20260505 21:17:17.152 | INFO | Downloaded 34GiB to GPU in 2.0177s (16.8509GiB/s) | 20260505 21:17:22.279 | INFO | Analyzing chunk frames = [6370816,6371328) | 20260505 21:17:22.279 | INFO | MemInfo: { | 20260505 21:17:22.279 | INFO | "gpu" : " 17 GB", | 20260505 21:17:22.279 | INFO | "huge" : "", | 20260505 21:17:22.279 | INFO | "huge_shared" : " 27 GB", | 20260505 21:17:22.279 | INFO | "malloc" : " 14 GB", | 20260505 21:17:22.279 | INFO | "pinned" : " 29 GB" | 20260505 21:17:22.279 | INFO | } | 20260505 21:17:22.279 | INFO | Perf report for the last 5.12756 seconds: | 20260505 21:17:22.279 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:17:22.279 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:22.279 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:22.279 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:22.279 | INFO | ANALYSIS : 62.1%, 3.0, 96.3ms, 0 | 20260505 21:17:22.279 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:22.279 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:22.279 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:17:22.279 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:22.279 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:22.279 | INFO | Uploaded 36GiB to GPU in 2.14058s (16.8179GiB/s) | 20260505 21:17:22.279 | INFO | Downloaded 34GiB to GPU in 2.03809s (16.6823GiB/s) | 20260505 21:17:27.427 | INFO | Analyzing chunk frames = [6371328,6371840) | 20260505 21:17:27.427 | INFO | MemInfo: { | 20260505 21:17:27.427 | INFO | "gpu" : " 17 GB", | 20260505 21:17:27.427 | INFO | "huge" : "", | 20260505 21:17:27.427 | INFO | "huge_shared" : " 27 GB", | 20260505 21:17:27.427 | INFO | "malloc" : " 14 GB", | 20260505 21:17:27.427 | INFO | "pinned" : " 29 GB" | 20260505 21:17:27.427 | INFO | } | 20260505 21:17:27.427 | INFO | Perf report for the last 5.14795 seconds: | 20260505 21:17:27.427 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:17:27.427 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:27.427 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:27.427 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:27.427 | INFO | ANALYSIS : 61.6%, 3.0, 96.4ms, 0 | 20260505 21:17:27.427 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:27.427 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:27.427 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:17:27.427 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:27.427 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:27.427 | INFO | Uploaded 36GiB to GPU in 2.13111s (16.8926GiB/s) | 20260505 21:17:27.427 | INFO | Downloaded 34GiB to GPU in 2.0636s (16.4761GiB/s) | 20260505 21:17:32.563 | INFO | Analyzing chunk frames = [6371840,6372352) | 20260505 21:17:32.563 | INFO | MemInfo: { | 20260505 21:17:32.563 | INFO | "gpu" : " 17 GB", | 20260505 21:17:32.563 | INFO | "huge" : "", | 20260505 21:17:32.563 | INFO | "huge_shared" : " 27 GB", | 20260505 21:17:32.563 | INFO | "malloc" : " 14 GB", | 20260505 21:17:32.563 | INFO | "pinned" : " 29 GB" | 20260505 21:17:32.563 | INFO | } | 20260505 21:17:32.563 | INFO | Perf report for the last 5.13606 seconds: | 20260505 21:17:32.563 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:17:32.563 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:32.563 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:32.563 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:32.563 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260505 21:17:32.563 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:32.563 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:32.563 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:17:32.563 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:32.563 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:32.563 | INFO | Uploaded 36GiB to GPU in 2.07287s (17.3672GiB/s) | 20260505 21:17:32.563 | INFO | Downloaded 34GiB to GPU in 2.03089s (16.7414GiB/s) | 20260505 21:17:37.702 | INFO | Analyzing chunk frames = [6372352,6372864) | 20260505 21:17:37.702 | INFO | MemInfo: { | 20260505 21:17:37.702 | INFO | "gpu" : " 17 GB", | 20260505 21:17:37.702 | INFO | "huge" : "", | 20260505 21:17:37.702 | INFO | "huge_shared" : " 27 GB", | 20260505 21:17:37.702 | INFO | "malloc" : " 14 GB", | 20260505 21:17:37.702 | INFO | "pinned" : " 29 GB" | 20260505 21:17:37.702 | INFO | } | 20260505 21:17:37.702 | INFO | Perf report for the last 5.13909 seconds: | 20260505 21:17:37.702 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:17:37.702 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:37.702 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:37.702 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:37.702 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260505 21:17:37.702 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:37.702 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:37.702 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 21:17:37.702 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:37.702 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:37.702 | INFO | Uploaded 36GiB to GPU in 2.13951s (16.8263GiB/s) | 20260505 21:17:37.702 | INFO | Downloaded 34GiB to GPU in 2.11141s (16.1029GiB/s) | 20260505 21:17:42.837 | INFO | Analyzing chunk frames = [6372864,6373376) | 20260505 21:17:42.837 | INFO | MemInfo: { | 20260505 21:17:42.837 | INFO | "gpu" : " 17 GB", | 20260505 21:17:42.837 | INFO | "huge" : "", | 20260505 21:17:42.837 | INFO | "huge_shared" : " 27 GB", | 20260505 21:17:42.837 | INFO | "malloc" : " 14 GB", | 20260505 21:17:42.837 | INFO | "pinned" : " 29 GB" | 20260505 21:17:42.837 | INFO | } | 20260505 21:17:42.837 | INFO | Perf report for the last 5.13482 seconds: | 20260505 21:17:42.837 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:17:42.837 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:42.837 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:42.837 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:42.837 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 21:17:42.837 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:42.837 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:42.837 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 21:17:42.837 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:42.837 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:42.837 | INFO | Uploaded 36GiB to GPU in 2.09294s (17.2006GiB/s) | 20260505 21:17:42.837 | INFO | Downloaded 34GiB to GPU in 2.0001s (16.9992GiB/s) | 20260505 21:17:42.837 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 21:17:42.837 | INFO | Total mem usage: 15.3 GiB | 20260505 21:17:42.837 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 21:17:42.837 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.15%) | 20260505 21:17:42.837 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 21:17:42.837 | INFO | High Water Breakdown: | 20260505 21:17:42.837 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 21:17:42.837 | INFO | 38.8% : 5.5 GiB ( 82.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 21:17:42.837 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 21:17:42.837 | INFO | ------------------------------------------ | 20260505 21:17:42.837 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:17:42.837 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 21:17:42.837 | INFO | Total mem usage: 26.5 GiB | 20260505 21:17:42.837 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 21:17:42.837 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 21:17:42.837 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 21:17:42.837 | INFO | High Water Breakdown: | 20260505 21:17:42.837 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:17:42.837 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:17:42.837 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:17:42.837 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:17:42.837 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:17:42.837 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:17:42.837 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 21:17:42.837 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 21:17:42.837 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:17:42.837 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 21:17:42.837 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 21:17:42.837 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 21:17:42.837 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 21:17:42.837 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 21:17:42.837 | INFO | ------------------------------------------ | 20260505 21:17:42.837 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:17:42.837 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 21:17:42.837 | INFO | Total mem usage: 25.6 GiB | 20260505 21:17:42.837 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 21:17:42.837 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 21:17:42.837 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 21:17:42.837 | INFO | High Water Breakdown: | 20260505 21:17:42.837 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 21:17:42.837 | INFO | ------------------------------------------ | 20260505 21:17:42.837 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:17:42.837 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 21:17:42.837 | INFO | Total mem usage: 15.7 GiB | 20260505 21:17:42.837 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 21:17:42.837 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 21:17:42.837 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 21:17:42.837 | INFO | High Water Breakdown: | 20260505 21:17:42.837 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 21:17:42.837 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 21:17:42.837 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 21:17:42.837 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 21:17:42.837 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 21:17:42.837 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 21:17:42.838 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 21:17:42.838 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 21:17:42.838 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 21:17:42.838 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:17:42.838 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 21:17:42.838 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:17:42.838 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 21:17:42.838 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 21:17:42.838 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 21:17:42.838 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 21:17:42.838 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:17:42.838 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 21:17:42.838 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:17:42.838 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:17:42.838 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:17:42.838 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 21:17:42.838 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 21:17:42.838 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:17:42.838 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 21:17:42.838 | INFO | ------------------------------------------ | 20260505 21:17:42.838 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:17:42.838 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 21:17:42.838 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:17:42.838 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:17:42.838 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.41e+01, 1.29e+01 / 1.29e+00 | 20260505 21:17:42.838 | INFO | PerfReport: 0.29%, AggregateStats -- 9.24e-02 / 1.05e-01, 1.14e-02 / 5.58e-03 | 20260505 21:17:42.838 | INFO | PerfReport: 11.81%, Baseline -- 3.72e+00 / 4.54e+00, 5.77e-01 / 1.62e-01 | 20260505 21:17:42.838 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.52e+00, 1.89e-01 / 9.86e-02 | 20260505 21:17:42.838 | INFO | PerfReport: 15.95%, DME -- 4.02e+01 / 4.41e+01, 3.85e+00 / 1.40e+00 | 20260505 21:17:42.838 | INFO | PerfReport: 57.17%, FrameLabeling -- 1.80e+01 / 1.93e+01, 1.54e+00 / 9.12e-01 | 20260505 21:17:42.838 | INFO | PerfReport: 7.19%, PulseAccumulating -- 2.26e+00 / 2.83e+00, 4.31e-01 / 1.60e-01 | 20260505 21:17:42.838 | INFO | PerfReport: 3.20%, Metrics -- 1.01e+00 / 1.31e+00, 2.75e-01 / 1.08e-01 | 20260505 21:17:42.838 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.50e-01 / 4.10e-01, 6.16e-02 / 4.17e-02 | 20260505 21:17:42.838 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 21:17:42.838 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:17:42.838 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:17:42.838 | INFO | PerfReport: Full Execution -- 9.90e+01 / -nan, 2.52e+01 / 0.00e+00 | 20260505 21:17:42.838 | INFO | PerfReport: 22.38%, Upload -- 2.22e+01 / 0.00e+00, 7.65e+00 / 0.00e+00 | 20260505 21:17:42.838 | INFO | PerfReport: 21.75%, Download -- 2.15e+01 / 0.00e+00, 7.62e+00 / 0.00e+00 | 20260505 21:17:42.838 | INFO | PerfReport: 31.78%, Compute -- 3.15e+01 / 0.00e+00, 1.29e+01 / 0.00e+00 | 20260505 21:17:42.838 | INFO | PerfReport: 2.22%, LockUpload -- 2.19e+00 / 0.00e+00, 7.90e+00 / 0.00e+00 | 20260505 21:17:42.838 | INFO | PerfReport: 1.33%, LockDownload -- 1.32e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260505 21:17:42.838 | INFO | PerfReport: 20.52%, LockCompute -- 2.03e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260505 21:17:42.838 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.75e-02 / 0.00e+00 | 20260505 21:17:42.965 | INFO | Starting a round of Baz IO | 20260505 21:17:46.139 | INFO | Populating RTMetrics report | 20260505 21:17:46.139 | INFO | RTMetrics report populated | 20260505 21:17:46.139 | INFO | Opening tmp RTMetrics json file | 20260505 21:17:46.142 | INFO | Serializing RTMetrics to json | 20260505 21:17:46.144 | INFO | Serialization done, swapping tmp file now... | 20260505 21:17:46.144 | INFO | RTMetrics done | 20260505 21:17:46.229 | INFO | Finished a round of Baz IO: Wrote 1.975 GiB in 3.263 seconds | 20260505 21:17:46.229 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.59% / 0.00% / 78.41% | 20260505 21:17:46.229 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.151368% / 3.97364e-06% / 0.000103315% | 20260505 21:17:46.229 | INFO | Baz IO overall: | 20260505 21:17:46.229 | INFO | BytesWritten : 3695379598087 | 20260505 21:17:46.229 | INFO | NumEvents : 1745272433325 | 20260505 21:17:47.975 | INFO | Analyzing chunk frames = [6373376,6373888) | 20260505 21:17:47.975 | INFO | MemInfo: { | 20260505 21:17:47.975 | INFO | "gpu" : " 17 GB", | 20260505 21:17:47.975 | INFO | "huge" : "", | 20260505 21:17:47.975 | INFO | "huge_shared" : " 27 GB", | 20260505 21:17:47.975 | INFO | "malloc" : " 14 GB", | 20260505 21:17:47.975 | INFO | "pinned" : " 29 GB" | 20260505 21:17:47.975 | INFO | } | 20260505 21:17:47.975 | INFO | Perf report for the last 5.13873 seconds: | 20260505 21:17:47.975 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:17:47.975 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:47.975 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:47.975 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:47.975 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 21:17:47.975 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:47.976 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260505 21:17:47.976 | INFO | PRE_HQ : 60.2%, 1.8, 55.3ms, 0 | 20260505 21:17:47.976 | INFO | BAZWRITER : 63.0%, 3.7, 119.6ms, 0 | 20260505 21:17:47.976 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 21:17:47.976 | INFO | Uploaded 36GiB to GPU in 2.16408s (16.6353GiB/s) | 20260505 21:17:47.976 | INFO | Downloaded 38GiB to GPU in 2.24092s (16.9573GiB/s) | 20260505 21:17:53.123 | INFO | Analyzing chunk frames = [6373888,6374400) | 20260505 21:17:53.123 | INFO | MemInfo: { | 20260505 21:17:53.123 | INFO | "gpu" : " 17 GB", | 20260505 21:17:53.123 | INFO | "huge" : "", | 20260505 21:17:53.123 | INFO | "huge_shared" : " 27 GB", | 20260505 21:17:53.123 | INFO | "malloc" : " 14 GB", | 20260505 21:17:53.123 | INFO | "pinned" : " 29 GB" | 20260505 21:17:53.123 | INFO | } | 20260505 21:17:53.123 | INFO | Perf report for the last 5.14783 seconds: | 20260505 21:17:53.123 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:17:53.123 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:53.123 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:53.123 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:53.123 | INFO | ANALYSIS : 61.9%, 3.0, 96.8ms, 0 | 20260505 21:17:53.123 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:53.123 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:53.123 | INFO | PRE_HQ : 26.8%, 1.0, 14.0ms, 0 | 20260505 21:17:53.123 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:53.123 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:53.124 | INFO | Uploaded 36GiB to GPU in 2.07874s (17.3182GiB/s) | 20260505 21:17:53.124 | INFO | Downloaded 34GiB to GPU in 2.01831s (16.8458GiB/s) | 20260505 21:17:58.264 | INFO | Analyzing chunk frames = [6374400,6374912) | 20260505 21:17:58.264 | INFO | MemInfo: { | 20260505 21:17:58.264 | INFO | "gpu" : " 17 GB", | 20260505 21:17:58.264 | INFO | "huge" : "", | 20260505 21:17:58.264 | INFO | "huge_shared" : " 27 GB", | 20260505 21:17:58.264 | INFO | "malloc" : " 14 GB", | 20260505 21:17:58.264 | INFO | "pinned" : " 29 GB" | 20260505 21:17:58.264 | INFO | } | 20260505 21:17:58.264 | INFO | Perf report for the last 5.14036 seconds: | 20260505 21:17:58.264 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:17:58.264 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:58.264 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:58.264 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:58.264 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260505 21:17:58.264 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:58.264 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:58.264 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:17:58.264 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:58.264 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:17:58.264 | INFO | Uploaded 36GiB to GPU in 2.14223s (16.8049GiB/s) | 20260505 21:17:58.264 | INFO | Downloaded 34GiB to GPU in 2.03026s (16.7467GiB/s) | 20260505 21:18:03.399 | INFO | Analyzing chunk frames = [6374912,6375424) | 20260505 21:18:03.399 | INFO | MemInfo: { | 20260505 21:18:03.399 | INFO | "gpu" : " 17 GB", | 20260505 21:18:03.399 | INFO | "huge" : "", | 20260505 21:18:03.399 | INFO | "huge_shared" : " 27 GB", | 20260505 21:18:03.399 | INFO | "malloc" : " 14 GB", | 20260505 21:18:03.399 | INFO | "pinned" : " 29 GB" | 20260505 21:18:03.399 | INFO | } | 20260505 21:18:03.399 | INFO | Perf report for the last 5.1358 seconds: | 20260505 21:18:03.399 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:18:03.399 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:03.399 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:03.399 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:03.400 | INFO | ANALYSIS : 61.9%, 3.0, 96.0ms, 0 | 20260505 21:18:03.400 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:03.400 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:03.400 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 21:18:03.400 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:03.400 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:03.400 | INFO | Uploaded 36GiB to GPU in 2.11992s (16.9818GiB/s) | 20260505 21:18:03.400 | INFO | Downloaded 34GiB to GPU in 2.01815s (16.8471GiB/s) | 20260505 21:18:08.531 | INFO | Analyzing chunk frames = [6375424,6375936) | 20260505 21:18:08.531 | INFO | MemInfo: { | 20260505 21:18:08.531 | INFO | "gpu" : " 17 GB", | 20260505 21:18:08.531 | INFO | "huge" : "", | 20260505 21:18:08.531 | INFO | "huge_shared" : " 27 GB", | 20260505 21:18:08.531 | INFO | "malloc" : " 14 GB", | 20260505 21:18:08.531 | INFO | "pinned" : " 29 GB" | 20260505 21:18:08.531 | INFO | } | 20260505 21:18:08.531 | INFO | Perf report for the last 5.13176 seconds: | 20260505 21:18:08.531 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:18:08.531 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:08.531 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:08.531 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:08.531 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260505 21:18:08.531 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:08.531 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:08.531 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 21:18:08.531 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:08.531 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:08.532 | INFO | Uploaded 36GiB to GPU in 2.13587s (16.855GiB/s) | 20260505 21:18:08.532 | INFO | Downloaded 34GiB to GPU in 2.05708s (16.5283GiB/s) | 20260505 21:18:13.677 | INFO | Analyzing chunk frames = [6375936,6376448) | 20260505 21:18:13.678 | INFO | MemInfo: { | 20260505 21:18:13.678 | INFO | "gpu" : " 17 GB", | 20260505 21:18:13.678 | INFO | "huge" : "", | 20260505 21:18:13.678 | INFO | "huge_shared" : " 27 GB", | 20260505 21:18:13.678 | INFO | "malloc" : " 14 GB", | 20260505 21:18:13.678 | INFO | "pinned" : " 29 GB" | 20260505 21:18:13.678 | INFO | } | 20260505 21:18:13.678 | INFO | Perf report for the last 5.14631 seconds: | 20260505 21:18:13.678 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:18:13.678 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:13.678 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:13.678 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:13.678 | INFO | ANALYSIS : 62.0%, 3.0, 97.0ms, 0 | 20260505 21:18:13.678 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:13.678 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:13.678 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:18:13.678 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:13.678 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:13.678 | INFO | Uploaded 36GiB to GPU in 2.15798s (16.6823GiB/s) | 20260505 21:18:13.678 | INFO | Downloaded 34GiB to GPU in 2.0648s (16.4665GiB/s) | 20260505 21:18:18.816 | INFO | Analyzing chunk frames = [6376448,6376960) | 20260505 21:18:18.816 | INFO | MemInfo: { | 20260505 21:18:18.816 | INFO | "gpu" : " 17 GB", | 20260505 21:18:18.816 | INFO | "huge" : "", | 20260505 21:18:18.816 | INFO | "huge_shared" : " 27 GB", | 20260505 21:18:18.816 | INFO | "malloc" : " 14 GB", | 20260505 21:18:18.816 | INFO | "pinned" : " 29 GB" | 20260505 21:18:18.816 | INFO | } | 20260505 21:18:18.816 | INFO | Perf report for the last 5.13828 seconds: | 20260505 21:18:18.816 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:18:18.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:18.816 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:18.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:18.816 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 21:18:18.816 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:18.816 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:18.816 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 21:18:18.816 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:18.816 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:18.816 | INFO | Uploaded 36GiB to GPU in 2.15882s (16.6758GiB/s) | 20260505 21:18:18.816 | INFO | Downloaded 34GiB to GPU in 2.119s (16.0453GiB/s) | 20260505 21:18:23.955 | INFO | Analyzing chunk frames = [6376960,6377472) | 20260505 21:18:23.955 | INFO | MemInfo: { | 20260505 21:18:23.955 | INFO | "gpu" : " 17 GB", | 20260505 21:18:23.955 | INFO | "huge" : "", | 20260505 21:18:23.955 | INFO | "huge_shared" : " 27 GB", | 20260505 21:18:23.955 | INFO | "malloc" : " 14 GB", | 20260505 21:18:23.955 | INFO | "pinned" : " 29 GB" | 20260505 21:18:23.955 | INFO | } | 20260505 21:18:23.955 | INFO | Perf report for the last 5.1391 seconds: | 20260505 21:18:23.955 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:18:23.955 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:23.955 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:23.955 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:23.955 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 21:18:23.955 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:23.955 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:23.955 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 21:18:23.955 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:23.955 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:23.955 | INFO | Uploaded 36GiB to GPU in 2.11574s (17.0154GiB/s) | 20260505 21:18:23.955 | INFO | Downloaded 34GiB to GPU in 2.00429s (16.9636GiB/s) | 20260505 21:18:24.089 | INFO | Starting a round of Baz IO | 20260505 21:18:27.244 | INFO | Populating RTMetrics report | 20260505 21:18:27.244 | INFO | RTMetrics report populated | 20260505 21:18:27.244 | INFO | Opening tmp RTMetrics json file | 20260505 21:18:27.245 | INFO | Serializing RTMetrics to json | 20260505 21:18:27.246 | INFO | Serialization done, swapping tmp file now... | 20260505 21:18:27.246 | INFO | RTMetrics done | 20260505 21:18:27.330 | INFO | Finished a round of Baz IO: Wrote 1.976 GiB in 3.24 seconds | 20260505 21:18:27.330 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.59% / 0.00% / 78.41% | 20260505 21:18:27.330 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.151193% / 3.97364e-06% / 7.54992e-05% | 20260505 21:18:27.330 | INFO | Baz IO overall: | 20260505 21:18:27.330 | INFO | BytesWritten : 3697500797961 | 20260505 21:18:27.330 | INFO | NumEvents : 1746251918080 | 20260505 21:18:29.096 | INFO | Analyzing chunk frames = [6377472,6377984) | 20260505 21:18:29.096 | INFO | MemInfo: { | 20260505 21:18:29.096 | INFO | "gpu" : " 17 GB", | 20260505 21:18:29.096 | INFO | "huge" : "", | 20260505 21:18:29.096 | INFO | "huge_shared" : " 27 GB", | 20260505 21:18:29.096 | INFO | "malloc" : " 14 GB", | 20260505 21:18:29.096 | INFO | "pinned" : " 29 GB" | 20260505 21:18:29.096 | INFO | } | 20260505 21:18:29.096 | INFO | Perf report for the last 5.14114 seconds: | 20260505 21:18:29.096 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:18:29.096 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:29.096 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:29.096 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:29.096 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 21:18:29.096 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:29.096 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260505 21:18:29.096 | INFO | PRE_HQ : 59.9%, 1.8, 54.8ms, 0 | 20260505 21:18:29.096 | INFO | BAZWRITER : 62.6%, 3.7, 118.1ms, 0 | 20260505 21:18:29.096 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 21:18:29.096 | INFO | Uploaded 36GiB to GPU in 2.17389s (16.5602GiB/s) | 20260505 21:18:29.097 | INFO | Downloaded 38GiB to GPU in 2.25052s (16.885GiB/s) | 20260505 21:18:34.231 | INFO | Analyzing chunk frames = [6377984,6378496) | 20260505 21:18:34.231 | INFO | MemInfo: { | 20260505 21:18:34.231 | INFO | "gpu" : " 17 GB", | 20260505 21:18:34.231 | INFO | "huge" : "", | 20260505 21:18:34.231 | INFO | "huge_shared" : " 27 GB", | 20260505 21:18:34.231 | INFO | "malloc" : " 14 GB", | 20260505 21:18:34.231 | INFO | "pinned" : " 29 GB" | 20260505 21:18:34.231 | INFO | } | 20260505 21:18:34.231 | INFO | Perf report for the last 5.13478 seconds: | 20260505 21:18:34.231 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:18:34.231 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:34.231 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:34.231 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:34.231 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260505 21:18:34.231 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:34.231 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:34.231 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:18:34.231 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:34.231 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:34.231 | INFO | Uploaded 36GiB to GPU in 2.09332s (17.1976GiB/s) | 20260505 21:18:34.231 | INFO | Downloaded 34GiB to GPU in 2.0287s (16.7595GiB/s) | 20260505 21:18:39.380 | INFO | Analyzing chunk frames = [6378496,6379008) | 20260505 21:18:39.380 | INFO | MemInfo: { | 20260505 21:18:39.380 | INFO | "gpu" : " 17 GB", | 20260505 21:18:39.380 | INFO | "huge" : "", | 20260505 21:18:39.380 | INFO | "huge_shared" : " 27 GB", | 20260505 21:18:39.380 | INFO | "malloc" : " 14 GB", | 20260505 21:18:39.380 | INFO | "pinned" : " 29 GB" | 20260505 21:18:39.380 | INFO | } | 20260505 21:18:39.380 | INFO | Perf report for the last 5.14897 seconds: | 20260505 21:18:39.380 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:18:39.380 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:39.380 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:39.380 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:39.380 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260505 21:18:39.380 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:39.380 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:39.380 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:18:39.380 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:39.380 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:39.381 | INFO | Uploaded 36GiB to GPU in 2.12953s (16.9051GiB/s) | 20260505 21:18:39.381 | INFO | Downloaded 34GiB to GPU in 2.00995s (16.9158GiB/s) | 20260505 21:18:44.514 | INFO | Analyzing chunk frames = [6379008,6379520) | 20260505 21:18:44.514 | INFO | MemInfo: { | 20260505 21:18:44.514 | INFO | "gpu" : " 17 GB", | 20260505 21:18:44.514 | INFO | "huge" : "", | 20260505 21:18:44.514 | INFO | "huge_shared" : " 27 GB", | 20260505 21:18:44.514 | INFO | "malloc" : " 14 GB", | 20260505 21:18:44.514 | INFO | "pinned" : " 29 GB" | 20260505 21:18:44.514 | INFO | } | 20260505 21:18:44.514 | INFO | Perf report for the last 5.13455 seconds: | 20260505 21:18:44.514 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:18:44.514 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:44.514 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:44.514 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:44.514 | INFO | ANALYSIS : 62.0%, 3.0, 96.2ms, 0 | 20260505 21:18:44.515 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:44.515 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:44.515 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:18:44.515 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:44.515 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:44.515 | INFO | Uploaded 36GiB to GPU in 2.13222s (16.8838GiB/s) | 20260505 21:18:44.515 | INFO | Downloaded 34GiB to GPU in 2.04205s (16.65GiB/s) | 20260505 21:18:49.647 | INFO | Analyzing chunk frames = [6379520,6380032) | 20260505 21:18:49.647 | INFO | MemInfo: { | 20260505 21:18:49.647 | INFO | "gpu" : " 17 GB", | 20260505 21:18:49.647 | INFO | "huge" : "", | 20260505 21:18:49.647 | INFO | "huge_shared" : " 27 GB", | 20260505 21:18:49.647 | INFO | "malloc" : " 14 GB", | 20260505 21:18:49.647 | INFO | "pinned" : " 29 GB" | 20260505 21:18:49.647 | INFO | } | 20260505 21:18:49.647 | INFO | Perf report for the last 5.13295 seconds: | 20260505 21:18:49.648 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:18:49.648 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:49.648 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:49.648 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:49.648 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260505 21:18:49.648 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:49.648 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:49.648 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:18:49.648 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:49.648 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:49.648 | INFO | Uploaded 36GiB to GPU in 2.12366s (16.9518GiB/s) | 20260505 21:18:49.648 | INFO | Downloaded 34GiB to GPU in 2.0625s (16.4849GiB/s) | 20260505 21:18:54.784 | INFO | Analyzing chunk frames = [6380032,6380544) | 20260505 21:18:54.784 | INFO | MemInfo: { | 20260505 21:18:54.784 | INFO | "gpu" : " 17 GB", | 20260505 21:18:54.784 | INFO | "huge" : "", | 20260505 21:18:54.784 | INFO | "huge_shared" : " 27 GB", | 20260505 21:18:54.784 | INFO | "malloc" : " 14 GB", | 20260505 21:18:54.784 | INFO | "pinned" : " 29 GB" | 20260505 21:18:54.784 | INFO | } | 20260505 21:18:54.784 | INFO | Perf report for the last 5.13652 seconds: | 20260505 21:18:54.784 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:18:54.784 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:54.784 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:54.784 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:54.784 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 21:18:54.784 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:54.784 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:54.784 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 21:18:54.784 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:54.784 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:54.784 | INFO | Uploaded 36GiB to GPU in 2.15927s (16.6723GiB/s) | 20260505 21:18:54.784 | INFO | Downloaded 34GiB to GPU in 2.06304s (16.4805GiB/s) | 20260505 21:18:59.928 | INFO | Analyzing chunk frames = [6380544,6381056) | 20260505 21:18:59.928 | INFO | MemInfo: { | 20260505 21:18:59.928 | INFO | "gpu" : " 17 GB", | 20260505 21:18:59.928 | INFO | "huge" : "", | 20260505 21:18:59.928 | INFO | "huge_shared" : " 27 GB", | 20260505 21:18:59.928 | INFO | "malloc" : " 14 GB", | 20260505 21:18:59.928 | INFO | "pinned" : " 29 GB" | 20260505 21:18:59.928 | INFO | } | 20260505 21:18:59.928 | INFO | Perf report for the last 5.14378 seconds: | 20260505 21:18:59.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:18:59.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:59.928 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:59.928 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:59.928 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 21:18:59.928 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:59.928 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:59.928 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260505 21:18:59.928 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:59.928 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:18:59.928 | INFO | Uploaded 36GiB to GPU in 2.14068s (16.8171GiB/s) | 20260505 21:18:59.928 | INFO | Downloaded 34GiB to GPU in 2.16644s (15.6939GiB/s) | 20260505 21:19:05.072 | INFO | Analyzing chunk frames = [6381056,6381568) | 20260505 21:19:05.072 | INFO | MemInfo: { | 20260505 21:19:05.072 | INFO | "gpu" : " 17 GB", | 20260505 21:19:05.072 | INFO | "huge" : "", | 20260505 21:19:05.072 | INFO | "huge_shared" : " 27 GB", | 20260505 21:19:05.072 | INFO | "malloc" : " 14 GB", | 20260505 21:19:05.072 | INFO | "pinned" : " 29 GB" | 20260505 21:19:05.072 | INFO | } | 20260505 21:19:05.072 | INFO | Perf report for the last 5.14423 seconds: | 20260505 21:19:05.072 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:19:05.072 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:05.072 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:05.072 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:05.072 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 21:19:05.072 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:05.072 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:05.072 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260505 21:19:05.072 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:05.072 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:05.072 | INFO | Uploaded 36GiB to GPU in 2.09645s (17.1719GiB/s) | 20260505 21:19:05.072 | INFO | Downloaded 34GiB to GPU in 2.00515s (16.9563GiB/s) | 20260505 21:19:05.072 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 21:19:05.072 | INFO | Total mem usage: 15.3 GiB | 20260505 21:19:05.072 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 21:19:05.072 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.72%) | 20260505 21:19:05.072 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 21:19:05.072 | INFO | High Water Breakdown: | 20260505 21:19:05.072 | INFO | 60.9% : 8.59 GiB ( 90.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 21:19:05.072 | INFO | 38.9% : 5.49 GiB ( 82.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 21:19:05.072 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 21:19:05.072 | INFO | ------------------------------------------ | 20260505 21:19:05.072 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:19:05.072 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 21:19:05.072 | INFO | Total mem usage: 26.5 GiB | 20260505 21:19:05.072 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 21:19:05.072 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 21:19:05.072 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 21:19:05.073 | INFO | High Water Breakdown: | 20260505 21:19:05.073 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:19:05.073 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:19:05.073 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:19:05.073 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:19:05.073 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:19:05.073 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:19:05.073 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 21:19:05.073 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 21:19:05.073 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:19:05.073 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 21:19:05.073 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 21:19:05.073 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 21:19:05.073 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 21:19:05.073 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 21:19:05.073 | INFO | ------------------------------------------ | 20260505 21:19:05.073 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:19:05.073 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 21:19:05.073 | INFO | Total mem usage: 25.6 GiB | 20260505 21:19:05.073 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 21:19:05.073 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 21:19:05.073 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 21:19:05.073 | INFO | High Water Breakdown: | 20260505 21:19:05.073 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 21:19:05.073 | INFO | ------------------------------------------ | 20260505 21:19:05.073 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:19:05.073 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 21:19:05.073 | INFO | Total mem usage: 15.7 GiB | 20260505 21:19:05.073 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 21:19:05.073 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 21:19:05.073 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 21:19:05.073 | INFO | High Water Breakdown: | 20260505 21:19:05.073 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 21:19:05.073 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 21:19:05.073 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 21:19:05.073 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 21:19:05.073 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 21:19:05.073 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 21:19:05.073 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 21:19:05.073 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 21:19:05.073 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 21:19:05.073 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:19:05.073 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 21:19:05.073 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:19:05.073 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 21:19:05.073 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 21:19:05.073 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 21:19:05.073 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 21:19:05.073 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:19:05.073 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 21:19:05.073 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:19:05.073 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:19:05.073 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:19:05.073 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 21:19:05.073 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 21:19:05.073 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:19:05.073 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 21:19:05.073 | INFO | ------------------------------------------ | 20260505 21:19:05.073 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:19:05.073 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 21:19:05.073 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:19:05.073 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:19:05.073 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.42e+01, 1.35e+01 / 1.35e+00 | 20260505 21:19:05.074 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.02e-01, 1.15e-02 / 3.49e-03 | 20260505 21:19:05.074 | INFO | PerfReport: 11.80%, Baseline -- 3.71e+00 / 4.55e+00, 5.72e-01 / 1.41e-01 | 20260505 21:19:05.074 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.52e+00, 1.91e-01 / 8.83e-02 | 20260505 21:19:05.074 | INFO | PerfReport: 15.93%, DME -- 4.01e+01 / 4.41e+01, 3.95e+00 / 1.35e+00 | 20260505 21:19:05.074 | INFO | PerfReport: 57.21%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.57e+00 / 1.15e+00 | 20260505 21:19:05.074 | INFO | PerfReport: 7.18%, PulseAccumulating -- 2.26e+00 / 2.81e+00, 4.33e-01 / 2.00e-01 | 20260505 21:19:05.074 | INFO | PerfReport: 3.20%, Metrics -- 1.01e+00 / 1.31e+00, 2.74e-01 / 1.43e-01 | 20260505 21:19:05.074 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.50e-01 / 4.08e-01, 6.16e-02 / 4.02e-02 | 20260505 21:19:05.074 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 21:19:05.074 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:19:05.074 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:19:05.074 | INFO | PerfReport: Full Execution -- 9.91e+01 / -nan, 2.52e+01 / 0.00e+00 | 20260505 21:19:05.074 | INFO | PerfReport: 22.42%, Upload -- 2.22e+01 / 0.00e+00, 7.70e+00 / 0.00e+00 | 20260505 21:19:05.074 | INFO | PerfReport: 21.81%, Download -- 2.16e+01 / 0.00e+00, 7.67e+00 / 0.00e+00 | 20260505 21:19:05.074 | INFO | PerfReport: 31.77%, Compute -- 3.15e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 21:19:05.074 | INFO | PerfReport: 2.20%, LockUpload -- 2.18e+00 / 0.00e+00, 7.93e+00 / 0.00e+00 | 20260505 21:19:05.074 | INFO | PerfReport: 1.36%, LockDownload -- 1.35e+00 / 0.00e+00, 2.94e+00 / 0.00e+00 | 20260505 21:19:05.074 | INFO | PerfReport: 20.42%, LockCompute -- 2.02e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 21:19:05.074 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.70e-02 / 0.00e+00 | 20260505 21:19:05.196 | INFO | Starting a round of Baz IO | 20260505 21:19:08.353 | INFO | Populating RTMetrics report | 20260505 21:19:08.353 | INFO | RTMetrics report populated | 20260505 21:19:08.353 | INFO | Opening tmp RTMetrics json file | 20260505 21:19:08.354 | INFO | Serializing RTMetrics to json | 20260505 21:19:08.355 | INFO | Serialization done, swapping tmp file now... | 20260505 21:19:08.356 | INFO | RTMetrics done | 20260505 21:19:08.444 | INFO | Finished a round of Baz IO: Wrote 1.975 GiB in 3.248 seconds | 20260505 21:19:08.444 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.59% / 0.00% / 78.41% | 20260505 21:19:08.444 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.1507% / 0% / 7.94729e-05% | 20260505 21:19:08.444 | INFO | Baz IO overall: | 20260505 21:19:08.445 | INFO | BytesWritten : 3699621060557 | 20260505 21:19:08.445 | INFO | NumEvents : 1747230967500 | 20260505 21:19:10.213 | INFO | Analyzing chunk frames = [6381568,6382080) | 20260505 21:19:10.213 | INFO | MemInfo: { | 20260505 21:19:10.213 | INFO | "gpu" : " 17 GB", | 20260505 21:19:10.213 | INFO | "huge" : "", | 20260505 21:19:10.213 | INFO | "huge_shared" : " 27 GB", | 20260505 21:19:10.213 | INFO | "malloc" : " 14 GB", | 20260505 21:19:10.213 | INFO | "pinned" : " 29 GB" | 20260505 21:19:10.213 | INFO | } | 20260505 21:19:10.213 | INFO | Perf report for the last 5.14086 seconds: | 20260505 21:19:10.213 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:19:10.213 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:10.213 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:10.213 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:10.213 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 21:19:10.213 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:10.213 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260505 21:19:10.213 | INFO | PRE_HQ : 59.9%, 1.8, 54.9ms, 0 | 20260505 21:19:10.213 | INFO | BAZWRITER : 62.3%, 3.7, 119.7ms, 0 | 20260505 21:19:10.213 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 21:19:10.213 | INFO | Uploaded 36GiB to GPU in 2.15893s (16.6749GiB/s) | 20260505 21:19:10.213 | INFO | Downloaded 38GiB to GPU in 2.22354s (17.0899GiB/s) | 20260505 21:19:15.349 | INFO | Analyzing chunk frames = [6382080,6382592) | 20260505 21:19:15.349 | INFO | MemInfo: { | 20260505 21:19:15.349 | INFO | "gpu" : " 17 GB", | 20260505 21:19:15.349 | INFO | "huge" : "", | 20260505 21:19:15.349 | INFO | "huge_shared" : " 27 GB", | 20260505 21:19:15.349 | INFO | "malloc" : " 14 GB", | 20260505 21:19:15.349 | INFO | "pinned" : " 29 GB" | 20260505 21:19:15.349 | INFO | } | 20260505 21:19:15.349 | INFO | Perf report for the last 5.13612 seconds: | 20260505 21:19:15.349 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:19:15.349 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:15.349 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:15.349 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:15.349 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 21:19:15.349 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:15.349 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:15.349 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 21:19:15.349 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:15.349 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:15.349 | INFO | Uploaded 36GiB to GPU in 2.0799s (17.3085GiB/s) | 20260505 21:19:15.349 | INFO | Downloaded 34GiB to GPU in 2.01576s (16.8671GiB/s) | 20260505 21:19:20.488 | INFO | Analyzing chunk frames = [6382592,6383104) | 20260505 21:19:20.488 | INFO | MemInfo: { | 20260505 21:19:20.488 | INFO | "gpu" : " 17 GB", | 20260505 21:19:20.488 | INFO | "huge" : "", | 20260505 21:19:20.488 | INFO | "huge_shared" : " 27 GB", | 20260505 21:19:20.488 | INFO | "malloc" : " 14 GB", | 20260505 21:19:20.488 | INFO | "pinned" : " 29 GB" | 20260505 21:19:20.488 | INFO | } | 20260505 21:19:20.488 | INFO | Perf report for the last 5.13858 seconds: | 20260505 21:19:20.488 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:19:20.488 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:20.488 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:20.488 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:20.488 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260505 21:19:20.488 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:20.488 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:20.488 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:19:20.488 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:20.488 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:20.488 | INFO | Uploaded 36GiB to GPU in 2.12637s (16.9302GiB/s) | 20260505 21:19:20.488 | INFO | Downloaded 34GiB to GPU in 2.01659s (16.8601GiB/s) | 20260505 21:19:25.628 | INFO | Analyzing chunk frames = [6383104,6383616) | 20260505 21:19:25.628 | INFO | MemInfo: { | 20260505 21:19:25.628 | INFO | "gpu" : " 17 GB", | 20260505 21:19:25.628 | INFO | "huge" : "", | 20260505 21:19:25.628 | INFO | "huge_shared" : " 27 GB", | 20260505 21:19:25.628 | INFO | "malloc" : " 14 GB", | 20260505 21:19:25.628 | INFO | "pinned" : " 29 GB" | 20260505 21:19:25.628 | INFO | } | 20260505 21:19:25.628 | INFO | Perf report for the last 5.14017 seconds: | 20260505 21:19:25.628 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:19:25.628 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:25.628 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:25.628 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:25.628 | INFO | ANALYSIS : 62.3%, 3.0, 95.9ms, 0 | 20260505 21:19:25.628 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:25.628 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:25.628 | INFO | PRE_HQ : 27.5%, 1.0, 14.1ms, 0 | 20260505 21:19:25.628 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:25.628 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:25.628 | INFO | Uploaded 36GiB to GPU in 2.20663s (16.3145GiB/s) | 20260505 21:19:25.628 | INFO | Downloaded 34GiB to GPU in 2.05083s (16.5787GiB/s) | 20260505 21:19:30.773 | INFO | Analyzing chunk frames = [6383616,6384128) | 20260505 21:19:30.773 | INFO | MemInfo: { | 20260505 21:19:30.773 | INFO | "gpu" : " 17 GB", | 20260505 21:19:30.773 | INFO | "huge" : "", | 20260505 21:19:30.773 | INFO | "huge_shared" : " 27 GB", | 20260505 21:19:30.773 | INFO | "malloc" : " 14 GB", | 20260505 21:19:30.773 | INFO | "pinned" : " 29 GB" | 20260505 21:19:30.773 | INFO | } | 20260505 21:19:30.773 | INFO | Perf report for the last 5.14534 seconds: | 20260505 21:19:30.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:19:30.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:30.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:30.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:30.773 | INFO | ANALYSIS : 61.5%, 3.0, 96.2ms, 0 | 20260505 21:19:30.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:30.773 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:30.773 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 21:19:30.773 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:30.773 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:30.773 | INFO | Uploaded 36GiB to GPU in 2.15758s (16.6853GiB/s) | 20260505 21:19:30.773 | INFO | Downloaded 34GiB to GPU in 2.06712s (16.448GiB/s) | 20260505 21:19:35.906 | INFO | Analyzing chunk frames = [6384128,6384640) | 20260505 21:19:35.906 | INFO | MemInfo: { | 20260505 21:19:35.906 | INFO | "gpu" : " 17 GB", | 20260505 21:19:35.906 | INFO | "huge" : "", | 20260505 21:19:35.906 | INFO | "huge_shared" : " 27 GB", | 20260505 21:19:35.906 | INFO | "malloc" : " 14 GB", | 20260505 21:19:35.907 | INFO | "pinned" : " 29 GB" | 20260505 21:19:35.907 | INFO | } | 20260505 21:19:35.907 | INFO | Perf report for the last 5.13353 seconds: | 20260505 21:19:35.907 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:19:35.907 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:35.907 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:35.907 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:35.907 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260505 21:19:35.907 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:35.907 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:35.907 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 21:19:35.907 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:35.907 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:35.907 | INFO | Uploaded 36GiB to GPU in 2.07325s (17.364GiB/s) | 20260505 21:19:35.907 | INFO | Downloaded 34GiB to GPU in 2.03452s (16.7115GiB/s) | 20260505 21:19:41.042 | INFO | Analyzing chunk frames = [6384640,6385152) | 20260505 21:19:41.042 | INFO | MemInfo: { | 20260505 21:19:41.042 | INFO | "gpu" : " 17 GB", | 20260505 21:19:41.042 | INFO | "huge" : "", | 20260505 21:19:41.042 | INFO | "huge_shared" : " 27 GB", | 20260505 21:19:41.042 | INFO | "malloc" : " 14 GB", | 20260505 21:19:41.042 | INFO | "pinned" : " 29 GB" | 20260505 21:19:41.042 | INFO | } | 20260505 21:19:41.043 | INFO | Perf report for the last 5.13583 seconds: | 20260505 21:19:41.043 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:19:41.043 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:41.043 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:41.043 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:41.043 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 21:19:41.043 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:41.043 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:41.043 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 21:19:41.043 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:41.043 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:41.043 | INFO | Uploaded 36GiB to GPU in 2.16659s (16.616GiB/s) | 20260505 21:19:41.043 | INFO | Downloaded 34GiB to GPU in 2.13332s (15.9376GiB/s) | 20260505 21:19:46.186 | INFO | Analyzing chunk frames = [6385152,6385664) | 20260505 21:19:46.186 | INFO | MemInfo: { | 20260505 21:19:46.186 | INFO | "gpu" : " 17 GB", | 20260505 21:19:46.186 | INFO | "huge" : "", | 20260505 21:19:46.187 | INFO | "huge_shared" : " 27 GB", | 20260505 21:19:46.187 | INFO | "malloc" : " 14 GB", | 20260505 21:19:46.187 | INFO | "pinned" : " 29 GB" | 20260505 21:19:46.187 | INFO | } | 20260505 21:19:46.187 | INFO | Perf report for the last 5.14406 seconds: | 20260505 21:19:46.187 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:19:46.187 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:46.187 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:46.187 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:46.187 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260505 21:19:46.187 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:46.187 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:46.187 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 21:19:46.187 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:46.187 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:46.187 | INFO | Uploaded 36GiB to GPU in 2.12667s (16.9279GiB/s) | 20260505 21:19:46.187 | INFO | Downloaded 34GiB to GPU in 2.01545s (16.8697GiB/s) | 20260505 21:19:46.316 | INFO | Starting a round of Baz IO | 20260505 21:19:49.465 | INFO | Populating RTMetrics report | 20260505 21:19:49.465 | INFO | RTMetrics report populated | 20260505 21:19:49.465 | INFO | Opening tmp RTMetrics json file | 20260505 21:19:49.469 | INFO | Serializing RTMetrics to json | 20260505 21:19:49.471 | INFO | Serialization done, swapping tmp file now... | 20260505 21:19:49.473 | INFO | RTMetrics done | 20260505 21:19:49.553 | INFO | Finished a round of Baz IO: Wrote 1.973 GiB in 3.236 seconds | 20260505 21:19:49.553 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.59% / 0.00% / 78.41% | 20260505 21:19:49.553 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.150414% / 0% / 9.93411e-05% | 20260505 21:19:49.553 | INFO | Baz IO overall: | 20260505 21:19:49.553 | INFO | BytesWritten : 3701739479821 | 20260505 21:19:49.553 | INFO | NumEvents : 1748208477617 | 20260505 21:19:51.327 | INFO | Analyzing chunk frames = [6385664,6386176) | 20260505 21:19:51.328 | INFO | MemInfo: { | 20260505 21:19:51.328 | INFO | "gpu" : " 17 GB", | 20260505 21:19:51.328 | INFO | "huge" : "", | 20260505 21:19:51.328 | INFO | "huge_shared" : " 27 GB", | 20260505 21:19:51.328 | INFO | "malloc" : " 14 GB", | 20260505 21:19:51.328 | INFO | "pinned" : " 29 GB" | 20260505 21:19:51.328 | INFO | } | 20260505 21:19:51.328 | INFO | Perf report for the last 5.14102 seconds: | 20260505 21:19:51.328 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:19:51.328 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:51.328 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:51.328 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:51.328 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 21:19:51.328 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:51.328 | INFO | RT_METRICS : 44.1%, 1.0, 23.2ms, 0 | 20260505 21:19:51.328 | INFO | PRE_HQ : 60.0%, 1.8, 55.2ms, 0 | 20260505 21:19:51.328 | INFO | BAZWRITER : 62.3%, 3.7, 118.8ms, 0 | 20260505 21:19:51.328 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 21:19:51.328 | INFO | Uploaded 36GiB to GPU in 2.15229s (16.7263GiB/s) | 20260505 21:19:51.328 | INFO | Downloaded 38GiB to GPU in 2.24152s (16.9528GiB/s) | 20260505 21:19:56.462 | INFO | Analyzing chunk frames = [6386176,6386688) | 20260505 21:19:56.462 | INFO | MemInfo: { | 20260505 21:19:56.462 | INFO | "gpu" : " 17 GB", | 20260505 21:19:56.462 | INFO | "huge" : "", | 20260505 21:19:56.462 | INFO | "huge_shared" : " 27 GB", | 20260505 21:19:56.462 | INFO | "malloc" : " 14 GB", | 20260505 21:19:56.462 | INFO | "pinned" : " 29 GB" | 20260505 21:19:56.462 | INFO | } | 20260505 21:19:56.462 | INFO | Perf report for the last 5.13404 seconds: | 20260505 21:19:56.462 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:19:56.462 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:56.462 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:56.462 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:56.462 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260505 21:19:56.462 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:56.462 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:56.462 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:19:56.462 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:56.462 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:19:56.462 | INFO | Uploaded 36GiB to GPU in 2.08974s (17.2271GiB/s) | 20260505 21:19:56.462 | INFO | Downloaded 34GiB to GPU in 2.02517s (16.7887GiB/s) | 20260505 21:20:01.595 | INFO | Analyzing chunk frames = [6386688,6387200) | 20260505 21:20:01.595 | INFO | MemInfo: { | 20260505 21:20:01.595 | INFO | "gpu" : " 17 GB", | 20260505 21:20:01.595 | INFO | "huge" : "", | 20260505 21:20:01.595 | INFO | "huge_shared" : " 27 GB", | 20260505 21:20:01.595 | INFO | "malloc" : " 14 GB", | 20260505 21:20:01.595 | INFO | "pinned" : " 29 GB" | 20260505 21:20:01.595 | INFO | } | 20260505 21:20:01.595 | INFO | Perf report for the last 5.13374 seconds: | 20260505 21:20:01.595 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:20:01.595 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:01.595 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:01.595 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:01.595 | INFO | ANALYSIS : 62.1%, 3.0, 96.5ms, 0 | 20260505 21:20:01.595 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:01.596 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:01.596 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:20:01.596 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:01.596 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:01.596 | INFO | Uploaded 36GiB to GPU in 2.13852s (16.8341GiB/s) | 20260505 21:20:01.596 | INFO | Downloaded 34GiB to GPU in 2.01332s (16.8876GiB/s) | 20260505 21:20:06.737 | INFO | Analyzing chunk frames = [6387200,6387712) | 20260505 21:20:06.737 | INFO | MemInfo: { | 20260505 21:20:06.737 | INFO | "gpu" : " 17 GB", | 20260505 21:20:06.737 | INFO | "huge" : "", | 20260505 21:20:06.737 | INFO | "huge_shared" : " 27 GB", | 20260505 21:20:06.737 | INFO | "malloc" : " 14 GB", | 20260505 21:20:06.737 | INFO | "pinned" : " 29 GB" | 20260505 21:20:06.737 | INFO | } | 20260505 21:20:06.737 | INFO | Perf report for the last 5.14162 seconds: | 20260505 21:20:06.737 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:20:06.737 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:06.737 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:06.737 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:06.737 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260505 21:20:06.737 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:06.737 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:06.737 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 21:20:06.737 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:06.737 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:06.737 | INFO | Uploaded 36GiB to GPU in 2.19425s (16.4065GiB/s) | 20260505 21:20:06.737 | INFO | Downloaded 34GiB to GPU in 2.06356s (16.4764GiB/s) | 20260505 21:20:11.881 | INFO | Analyzing chunk frames = [6387712,6388224) | 20260505 21:20:11.881 | INFO | MemInfo: { | 20260505 21:20:11.881 | INFO | "gpu" : " 17 GB", | 20260505 21:20:11.881 | INFO | "huge" : "", | 20260505 21:20:11.881 | INFO | "huge_shared" : " 27 GB", | 20260505 21:20:11.881 | INFO | "malloc" : " 14 GB", | 20260505 21:20:11.881 | INFO | "pinned" : " 29 GB" | 20260505 21:20:11.881 | INFO | } | 20260505 21:20:11.881 | INFO | Perf report for the last 5.14424 seconds: | 20260505 21:20:11.881 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:20:11.881 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:11.881 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:11.881 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:11.881 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260505 21:20:11.881 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:11.881 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:11.881 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:20:11.881 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:11.881 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:11.882 | INFO | Uploaded 36GiB to GPU in 2.14079s (16.8162GiB/s) | 20260505 21:20:11.882 | INFO | Downloaded 34GiB to GPU in 2.06582s (16.4583GiB/s) | 20260505 21:20:17.025 | INFO | Analyzing chunk frames = [6388224,6388736) | 20260505 21:20:17.025 | INFO | MemInfo: { | 20260505 21:20:17.025 | INFO | "gpu" : " 17 GB", | 20260505 21:20:17.025 | INFO | "huge" : "", | 20260505 21:20:17.025 | INFO | "huge_shared" : " 27 GB", | 20260505 21:20:17.025 | INFO | "malloc" : " 14 GB", | 20260505 21:20:17.025 | INFO | "pinned" : " 29 GB" | 20260505 21:20:17.025 | INFO | } | 20260505 21:20:17.025 | INFO | Perf report for the last 5.14349 seconds: | 20260505 21:20:17.025 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:20:17.025 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:17.025 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:17.025 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:17.025 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 21:20:17.025 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:17.025 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:17.025 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 21:20:17.025 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:17.025 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:17.025 | INFO | Uploaded 36GiB to GPU in 2.09601s (17.1755GiB/s) | 20260505 21:20:17.025 | INFO | Downloaded 34GiB to GPU in 2.03622s (16.6976GiB/s) | 20260505 21:20:22.158 | INFO | Analyzing chunk frames = [6388736,6389248) | 20260505 21:20:22.158 | INFO | MemInfo: { | 20260505 21:20:22.158 | INFO | "gpu" : " 17 GB", | 20260505 21:20:22.158 | INFO | "huge" : "", | 20260505 21:20:22.158 | INFO | "huge_shared" : " 27 GB", | 20260505 21:20:22.158 | INFO | "malloc" : " 14 GB", | 20260505 21:20:22.158 | INFO | "pinned" : " 29 GB" | 20260505 21:20:22.158 | INFO | } | 20260505 21:20:22.158 | INFO | Perf report for the last 5.13335 seconds: | 20260505 21:20:22.158 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:20:22.158 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:22.158 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:22.158 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:22.158 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260505 21:20:22.158 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:22.158 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:22.158 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 21:20:22.158 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:22.158 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:22.158 | INFO | Uploaded 36GiB to GPU in 2.03961s (17.6504GiB/s) | 20260505 21:20:22.159 | INFO | Downloaded 34GiB to GPU in 2.04049s (16.6627GiB/s) | 20260505 21:20:27.295 | INFO | Analyzing chunk frames = [6389248,6389760) | 20260505 21:20:27.295 | INFO | MemInfo: { | 20260505 21:20:27.295 | INFO | "gpu" : " 17 GB", | 20260505 21:20:27.295 | INFO | "huge" : "", | 20260505 21:20:27.296 | INFO | "huge_shared" : " 27 GB", | 20260505 21:20:27.296 | INFO | "malloc" : " 14 GB", | 20260505 21:20:27.296 | INFO | "pinned" : " 29 GB" | 20260505 21:20:27.296 | INFO | } | 20260505 21:20:27.296 | INFO | Perf report for the last 5.13761 seconds: | 20260505 21:20:27.296 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:20:27.296 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:27.296 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:27.296 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:27.296 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260505 21:20:27.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:27.296 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:27.296 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:20:27.296 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:27.296 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:27.296 | INFO | Uploaded 36GiB to GPU in 2.12228s (16.9629GiB/s) | 20260505 21:20:27.296 | INFO | Downloaded 34GiB to GPU in 2.01165s (16.9015GiB/s) | 20260505 21:20:27.296 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 21:20:27.296 | INFO | Total mem usage: 15.3 GiB | 20260505 21:20:27.296 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 21:20:27.296 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.16%) | 20260505 21:20:27.296 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 21:20:27.296 | INFO | High Water Breakdown: | 20260505 21:20:27.296 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 21:20:27.296 | INFO | 38.8% : 5.5 GiB ( 82.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 21:20:27.297 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 21:20:27.297 | INFO | ------------------------------------------ | 20260505 21:20:27.297 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:20:27.297 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 21:20:27.297 | INFO | Total mem usage: 26.5 GiB | 20260505 21:20:27.297 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 21:20:27.297 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 21:20:27.297 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 21:20:27.297 | INFO | High Water Breakdown: | 20260505 21:20:27.297 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:20:27.297 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:20:27.297 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:20:27.297 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:20:27.297 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:20:27.297 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:20:27.297 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 21:20:27.297 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 21:20:27.297 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:20:27.297 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 21:20:27.297 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 21:20:27.297 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 21:20:27.297 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 21:20:27.297 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 21:20:27.297 | INFO | ------------------------------------------ | 20260505 21:20:27.297 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:20:27.297 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 21:20:27.297 | INFO | Total mem usage: 25.6 GiB | 20260505 21:20:27.297 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 21:20:27.297 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 21:20:27.297 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 21:20:27.297 | INFO | High Water Breakdown: | 20260505 21:20:27.297 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 21:20:27.297 | INFO | ------------------------------------------ | 20260505 21:20:27.297 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:20:27.297 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 21:20:27.297 | INFO | Total mem usage: 15.7 GiB | 20260505 21:20:27.297 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 21:20:27.297 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 21:20:27.297 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 21:20:27.297 | INFO | High Water Breakdown: | 20260505 21:20:27.297 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 21:20:27.297 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 21:20:27.297 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 21:20:27.297 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 21:20:27.297 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 21:20:27.297 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 21:20:27.297 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 21:20:27.297 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 21:20:27.297 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 21:20:27.297 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:20:27.297 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 21:20:27.297 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:20:27.297 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 21:20:27.297 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 21:20:27.297 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 21:20:27.297 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 21:20:27.297 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:20:27.297 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 21:20:27.297 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:20:27.297 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:20:27.297 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:20:27.297 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 21:20:27.297 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 21:20:27.297 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:20:27.297 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 21:20:27.297 | INFO | ------------------------------------------ | 20260505 21:20:27.297 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:20:27.297 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 21:20:27.297 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:20:27.297 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:20:27.297 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.43e+01, 1.36e+01 / 1.24e+00 | 20260505 21:20:27.297 | INFO | PerfReport: 0.29%, AggregateStats -- 9.24e-02 / 1.04e-01, 1.04e-02 / 4.72e-03 | 20260505 21:20:27.298 | INFO | PerfReport: 11.80%, Baseline -- 3.71e+00 / 4.53e+00, 5.66e-01 / 1.55e-01 | 20260505 21:20:27.298 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.51e+00, 1.90e-01 / 9.74e-02 | 20260505 21:20:27.298 | INFO | PerfReport: 15.98%, DME -- 4.02e+01 / 4.42e+01, 3.87e+00 / 1.34e+00 | 20260505 21:20:27.298 | INFO | PerfReport: 57.13%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.49e+00 / 1.16e+00 | 20260505 21:20:27.298 | INFO | PerfReport: 7.20%, PulseAccumulating -- 2.26e+00 / 2.79e+00, 4.38e-01 / 2.28e-01 | 20260505 21:20:27.298 | INFO | PerfReport: 3.20%, Metrics -- 1.01e+00 / 1.30e+00, 2.70e-01 / 1.26e-01 | 20260505 21:20:27.298 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.50e-01 / 4.06e-01, 6.10e-02 / 3.47e-02 | 20260505 21:20:27.298 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 21:20:27.298 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:20:27.298 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:20:27.298 | INFO | PerfReport: Full Execution -- 9.90e+01 / -nan, 2.51e+01 / 0.00e+00 | 20260505 21:20:27.298 | INFO | PerfReport: 22.41%, Upload -- 2.22e+01 / 0.00e+00, 7.63e+00 / 0.00e+00 | 20260505 21:20:27.298 | INFO | PerfReport: 21.74%, Download -- 2.15e+01 / 0.00e+00, 7.57e+00 / 0.00e+00 | 20260505 21:20:27.298 | INFO | PerfReport: 31.77%, Compute -- 3.15e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 21:20:27.298 | INFO | PerfReport: 2.23%, LockUpload -- 2.21e+00 / 0.00e+00, 7.86e+00 / 0.00e+00 | 20260505 21:20:27.298 | INFO | PerfReport: 1.30%, LockDownload -- 1.29e+00 / 0.00e+00, 2.86e+00 / 0.00e+00 | 20260505 21:20:27.298 | INFO | PerfReport: 20.53%, LockCompute -- 2.03e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 21:20:27.298 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.75e-02 / 0.00e+00 | 20260505 21:20:27.425 | INFO | Starting a round of Baz IO | 20260505 21:20:30.594 | INFO | Populating RTMetrics report | 20260505 21:20:30.594 | INFO | RTMetrics report populated | 20260505 21:20:30.594 | INFO | Opening tmp RTMetrics json file | 20260505 21:20:30.597 | INFO | Serializing RTMetrics to json | 20260505 21:20:30.601 | INFO | Serialization done, swapping tmp file now... | 20260505 21:20:30.601 | INFO | RTMetrics done | 20260505 21:20:30.679 | INFO | Finished a round of Baz IO: Wrote 1.974 GiB in 3.253 seconds | 20260505 21:20:30.679 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.59% / 0.00% / 78.41% | 20260505 21:20:30.679 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.150446% / 3.97364e-06% / 9.13938e-05% | 20260505 21:20:30.679 | INFO | Baz IO overall: | 20260505 21:20:30.679 | INFO | BytesWritten : 3703859335023 | 20260505 21:20:30.679 | INFO | NumEvents : 1749187108039 | 20260505 21:20:32.441 | INFO | Analyzing chunk frames = [6389760,6390272) | 20260505 21:20:32.441 | INFO | MemInfo: { | 20260505 21:20:32.441 | INFO | "gpu" : " 17 GB", | 20260505 21:20:32.441 | INFO | "huge" : "", | 20260505 21:20:32.441 | INFO | "huge_shared" : " 27 GB", | 20260505 21:20:32.441 | INFO | "malloc" : " 14 GB", | 20260505 21:20:32.441 | INFO | "pinned" : " 29 GB" | 20260505 21:20:32.441 | INFO | } | 20260505 21:20:32.441 | INFO | Perf report for the last 5.14566 seconds: | 20260505 21:20:32.442 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:20:32.442 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:32.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:32.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:32.442 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 21:20:32.442 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:32.442 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260505 21:20:32.442 | INFO | PRE_HQ : 60.2%, 1.8, 55.5ms, 0 | 20260505 21:20:32.442 | INFO | BAZWRITER : 62.7%, 3.7, 119.1ms, 0 | 20260505 21:20:32.442 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 21:20:32.442 | INFO | Uploaded 36GiB to GPU in 2.16159s (16.6544GiB/s) | 20260505 21:20:32.442 | INFO | Downloaded 38GiB to GPU in 2.24274s (16.9435GiB/s) | 20260505 21:20:37.578 | INFO | Analyzing chunk frames = [6390272,6390784) | 20260505 21:20:37.578 | INFO | MemInfo: { | 20260505 21:20:37.578 | INFO | "gpu" : " 17 GB", | 20260505 21:20:37.578 | INFO | "huge" : "", | 20260505 21:20:37.578 | INFO | "huge_shared" : " 27 GB", | 20260505 21:20:37.578 | INFO | "malloc" : " 14 GB", | 20260505 21:20:37.578 | INFO | "pinned" : " 29 GB" | 20260505 21:20:37.578 | INFO | } | 20260505 21:20:37.578 | INFO | Perf report for the last 5.13699 seconds: | 20260505 21:20:37.578 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:20:37.578 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:37.578 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:37.578 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:37.579 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 21:20:37.579 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:37.579 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:37.579 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:20:37.579 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:37.579 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:37.579 | INFO | Uploaded 36GiB to GPU in 2.07519s (17.3478GiB/s) | 20260505 21:20:37.579 | INFO | Downloaded 34GiB to GPU in 2.019s (16.84GiB/s) | 20260505 21:20:42.721 | INFO | Analyzing chunk frames = [6390784,6391296) | 20260505 21:20:42.721 | INFO | MemInfo: { | 20260505 21:20:42.721 | INFO | "gpu" : " 17 GB", | 20260505 21:20:42.721 | INFO | "huge" : "", | 20260505 21:20:42.721 | INFO | "huge_shared" : " 27 GB", | 20260505 21:20:42.721 | INFO | "malloc" : " 14 GB", | 20260505 21:20:42.721 | INFO | "pinned" : " 29 GB" | 20260505 21:20:42.721 | INFO | } | 20260505 21:20:42.721 | INFO | Perf report for the last 5.14284 seconds: | 20260505 21:20:42.721 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:20:42.721 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:42.721 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:42.721 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:42.721 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260505 21:20:42.721 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:42.721 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:42.721 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:20:42.721 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:42.721 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:42.722 | INFO | Uploaded 36GiB to GPU in 2.13398s (16.8699GiB/s) | 20260505 21:20:42.722 | INFO | Downloaded 34GiB to GPU in 2.01913s (16.839GiB/s) | 20260505 21:20:47.856 | INFO | Analyzing chunk frames = [6391296,6391808) | 20260505 21:20:47.856 | INFO | MemInfo: { | 20260505 21:20:47.856 | INFO | "gpu" : " 17 GB", | 20260505 21:20:47.856 | INFO | "huge" : "", | 20260505 21:20:47.856 | INFO | "huge_shared" : " 27 GB", | 20260505 21:20:47.856 | INFO | "malloc" : " 14 GB", | 20260505 21:20:47.856 | INFO | "pinned" : " 29 GB" | 20260505 21:20:47.856 | INFO | } | 20260505 21:20:47.856 | INFO | Perf report for the last 5.13487 seconds: | 20260505 21:20:47.856 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:20:47.856 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:47.856 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:47.856 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:47.856 | INFO | ANALYSIS : 62.0%, 3.0, 96.2ms, 0 | 20260505 21:20:47.856 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:47.856 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:47.856 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:20:47.856 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:47.856 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:47.856 | INFO | Uploaded 36GiB to GPU in 2.14611s (16.7745GiB/s) | 20260505 21:20:47.857 | INFO | Downloaded 34GiB to GPU in 2.03207s (16.7317GiB/s) | 20260505 21:20:52.992 | INFO | Analyzing chunk frames = [6391808,6392320) | 20260505 21:20:52.992 | INFO | MemInfo: { | 20260505 21:20:52.992 | INFO | "gpu" : " 17 GB", | 20260505 21:20:52.992 | INFO | "huge" : "", | 20260505 21:20:52.992 | INFO | "huge_shared" : " 27 GB", | 20260505 21:20:52.992 | INFO | "malloc" : " 14 GB", | 20260505 21:20:52.992 | INFO | "pinned" : " 29 GB" | 20260505 21:20:52.992 | INFO | } | 20260505 21:20:52.992 | INFO | Perf report for the last 5.13605 seconds: | 20260505 21:20:52.992 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:20:52.992 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:52.992 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:52.992 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:52.992 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260505 21:20:52.992 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:52.992 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:52.992 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:20:52.992 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:52.992 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:52.992 | INFO | Uploaded 36GiB to GPU in 2.13626s (16.8519GiB/s) | 20260505 21:20:52.992 | INFO | Downloaded 34GiB to GPU in 2.05611s (16.5361GiB/s) | 20260505 21:20:58.138 | INFO | Analyzing chunk frames = [6392320,6392832) | 20260505 21:20:58.138 | INFO | MemInfo: { | 20260505 21:20:58.138 | INFO | "gpu" : " 17 GB", | 20260505 21:20:58.138 | INFO | "huge" : "", | 20260505 21:20:58.138 | INFO | "huge_shared" : " 27 GB", | 20260505 21:20:58.138 | INFO | "malloc" : " 14 GB", | 20260505 21:20:58.138 | INFO | "pinned" : " 29 GB" | 20260505 21:20:58.138 | INFO | } | 20260505 21:20:58.138 | INFO | Perf report for the last 5.14583 seconds: | 20260505 21:20:58.138 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:20:58.138 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:58.138 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:58.138 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:58.138 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260505 21:20:58.138 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:58.138 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:58.138 | INFO | PRE_HQ : 27.0%, 1.0, 14.1ms, 0 | 20260505 21:20:58.138 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:58.138 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:20:58.138 | INFO | Uploaded 36GiB to GPU in 2.16713s (16.6118GiB/s) | 20260505 21:20:58.138 | INFO | Downloaded 34GiB to GPU in 2.06642s (16.4535GiB/s) | 20260505 21:21:03.276 | INFO | Analyzing chunk frames = [6392832,6393344) | 20260505 21:21:03.276 | INFO | MemInfo: { | 20260505 21:21:03.276 | INFO | "gpu" : " 17 GB", | 20260505 21:21:03.276 | INFO | "huge" : "", | 20260505 21:21:03.276 | INFO | "huge_shared" : " 27 GB", | 20260505 21:21:03.276 | INFO | "malloc" : " 14 GB", | 20260505 21:21:03.276 | INFO | "pinned" : " 29 GB" | 20260505 21:21:03.276 | INFO | } | 20260505 21:21:03.276 | INFO | Perf report for the last 5.13836 seconds: | 20260505 21:21:03.276 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:21:03.276 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:03.276 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:03.276 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:03.276 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 21:21:03.276 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:03.276 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:03.276 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 21:21:03.276 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:03.276 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:03.277 | INFO | Uploaded 36GiB to GPU in 2.14801s (16.7597GiB/s) | 20260505 21:21:03.277 | INFO | Downloaded 34GiB to GPU in 2.10688s (16.1376GiB/s) | 20260505 21:21:08.408 | INFO | Analyzing chunk frames = [6393344,6393856) | 20260505 21:21:08.408 | INFO | MemInfo: { | 20260505 21:21:08.408 | INFO | "gpu" : " 17 GB", | 20260505 21:21:08.408 | INFO | "huge" : "", | 20260505 21:21:08.408 | INFO | "huge_shared" : " 27 GB", | 20260505 21:21:08.408 | INFO | "malloc" : " 14 GB", | 20260505 21:21:08.408 | INFO | "pinned" : " 29 GB" | 20260505 21:21:08.408 | INFO | } | 20260505 21:21:08.408 | INFO | Perf report for the last 5.13163 seconds: | 20260505 21:21:08.408 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:21:08.408 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:08.408 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:08.408 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:08.408 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 21:21:08.408 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:08.408 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:08.408 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:21:08.408 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:08.408 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:08.408 | INFO | Uploaded 36GiB to GPU in 2.11366s (17.0321GiB/s) | 20260505 21:21:08.408 | INFO | Downloaded 34GiB to GPU in 1.99693s (17.0262GiB/s) | 20260505 21:21:08.541 | INFO | Starting a round of Baz IO | 20260505 21:21:11.715 | INFO | Populating RTMetrics report | 20260505 21:21:11.716 | INFO | RTMetrics report populated | 20260505 21:21:11.716 | INFO | Opening tmp RTMetrics json file | 20260505 21:21:11.716 | INFO | Serializing RTMetrics to json | 20260505 21:21:11.717 | INFO | Serialization done, swapping tmp file now... | 20260505 21:21:11.718 | INFO | RTMetrics done | 20260505 21:21:11.805 | INFO | Finished a round of Baz IO: Wrote 1.975 GiB in 3.263 seconds | 20260505 21:21:11.805 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.58% / 0.00% / 78.41% | 20260505 21:21:11.805 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.150375% / 0% / 9.13938e-05% | 20260505 21:21:11.805 | INFO | Baz IO overall: | 20260505 21:21:11.805 | INFO | BytesWritten : 3705980211232 | 20260505 21:21:11.805 | INFO | NumEvents : 1750166787269 | 20260505 21:21:13.548 | INFO | Analyzing chunk frames = [6393856,6394368) | 20260505 21:21:13.548 | INFO | MemInfo: { | 20260505 21:21:13.548 | INFO | "gpu" : " 17 GB", | 20260505 21:21:13.548 | INFO | "huge" : "", | 20260505 21:21:13.548 | INFO | "huge_shared" : " 27 GB", | 20260505 21:21:13.548 | INFO | "malloc" : " 14 GB", | 20260505 21:21:13.548 | INFO | "pinned" : " 29 GB" | 20260505 21:21:13.548 | INFO | } | 20260505 21:21:13.548 | INFO | Perf report for the last 5.14031 seconds: | 20260505 21:21:13.548 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:21:13.548 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:13.548 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:13.548 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:13.549 | INFO | ANALYSIS : 63.3%, 3.0, 98.7ms, 0 | 20260505 21:21:13.549 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:13.549 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260505 21:21:13.549 | INFO | PRE_HQ : 60.4%, 1.8, 55.1ms, 0 | 20260505 21:21:13.549 | INFO | BAZWRITER : 63.0%, 3.6, 118.2ms, 0 | 20260505 21:21:13.549 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 21:21:13.549 | INFO | Uploaded 36GiB to GPU in 2.1808s (16.5077GiB/s) | 20260505 21:21:13.549 | INFO | Downloaded 38GiB to GPU in 2.2589s (16.8224GiB/s) | 20260505 21:21:18.693 | INFO | Analyzing chunk frames = [6394368,6394880) | 20260505 21:21:18.693 | INFO | MemInfo: { | 20260505 21:21:18.693 | INFO | "gpu" : " 17 GB", | 20260505 21:21:18.693 | INFO | "huge" : "", | 20260505 21:21:18.693 | INFO | "huge_shared" : " 27 GB", | 20260505 21:21:18.693 | INFO | "malloc" : " 14 GB", | 20260505 21:21:18.693 | INFO | "pinned" : " 29 GB" | 20260505 21:21:18.693 | INFO | } | 20260505 21:21:18.693 | INFO | Perf report for the last 5.1442 seconds: | 20260505 21:21:18.693 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:21:18.693 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:18.693 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:18.693 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:18.693 | INFO | ANALYSIS : 62.5%, 3.0, 97.7ms, 0 | 20260505 21:21:18.693 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:18.693 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:18.693 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 21:21:18.693 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:18.693 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:18.693 | INFO | Uploaded 36GiB to GPU in 2.15564s (16.7004GiB/s) | 20260505 21:21:18.693 | INFO | Downloaded 34GiB to GPU in 2.0512s (16.5756GiB/s) | 20260505 21:21:23.837 | INFO | Analyzing chunk frames = [6394880,6395392) | 20260505 21:21:23.837 | INFO | MemInfo: { | 20260505 21:21:23.837 | INFO | "gpu" : " 17 GB", | 20260505 21:21:23.837 | INFO | "huge" : "", | 20260505 21:21:23.837 | INFO | "huge_shared" : " 27 GB", | 20260505 21:21:23.837 | INFO | "malloc" : " 14 GB", | 20260505 21:21:23.837 | INFO | "pinned" : " 29 GB" | 20260505 21:21:23.837 | INFO | } | 20260505 21:21:23.837 | INFO | Perf report for the last 5.14473 seconds: | 20260505 21:21:23.837 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:21:23.837 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:23.837 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:23.837 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:23.837 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260505 21:21:23.837 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:23.837 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:23.837 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:21:23.838 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:23.838 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:23.838 | INFO | Uploaded 36GiB to GPU in 2.1434s (16.7958GiB/s) | 20260505 21:21:23.838 | INFO | Downloaded 34GiB to GPU in 2.02135s (16.8204GiB/s) | 20260505 21:21:28.973 | INFO | Analyzing chunk frames = [6395392,6395904) | 20260505 21:21:28.973 | INFO | MemInfo: { | 20260505 21:21:28.973 | INFO | "gpu" : " 17 GB", | 20260505 21:21:28.973 | INFO | "huge" : "", | 20260505 21:21:28.973 | INFO | "huge_shared" : " 27 GB", | 20260505 21:21:28.973 | INFO | "malloc" : " 14 GB", | 20260505 21:21:28.973 | INFO | "pinned" : " 29 GB" | 20260505 21:21:28.973 | INFO | } | 20260505 21:21:28.973 | INFO | Perf report for the last 5.13542 seconds: | 20260505 21:21:28.973 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:21:28.973 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:28.973 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:28.973 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:28.973 | INFO | ANALYSIS : 61.8%, 3.0, 96.0ms, 0 | 20260505 21:21:28.973 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:28.973 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:28.973 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:21:28.973 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:28.973 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:28.973 | INFO | Uploaded 36GiB to GPU in 2.13261s (16.8807GiB/s) | 20260505 21:21:28.973 | INFO | Downloaded 34GiB to GPU in 2.02384s (16.7997GiB/s) | 20260505 21:21:34.107 | INFO | Analyzing chunk frames = [6395904,6396416) | 20260505 21:21:34.107 | INFO | MemInfo: { | 20260505 21:21:34.107 | INFO | "gpu" : " 17 GB", | 20260505 21:21:34.107 | INFO | "huge" : "", | 20260505 21:21:34.107 | INFO | "huge_shared" : " 27 GB", | 20260505 21:21:34.107 | INFO | "malloc" : " 14 GB", | 20260505 21:21:34.107 | INFO | "pinned" : " 29 GB" | 20260505 21:21:34.107 | INFO | } | 20260505 21:21:34.107 | INFO | Perf report for the last 5.13381 seconds: | 20260505 21:21:34.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:21:34.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:34.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:34.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:34.107 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260505 21:21:34.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:34.107 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:34.107 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 21:21:34.107 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:34.107 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:34.107 | INFO | Uploaded 36GiB to GPU in 2.14007s (16.8219GiB/s) | 20260505 21:21:34.107 | INFO | Downloaded 34GiB to GPU in 2.05869s (16.5153GiB/s) | 20260505 21:21:39.246 | INFO | Analyzing chunk frames = [6396416,6396928) | 20260505 21:21:39.246 | INFO | MemInfo: { | 20260505 21:21:39.246 | INFO | "gpu" : " 17 GB", | 20260505 21:21:39.246 | INFO | "huge" : "", | 20260505 21:21:39.246 | INFO | "huge_shared" : " 27 GB", | 20260505 21:21:39.246 | INFO | "malloc" : " 14 GB", | 20260505 21:21:39.247 | INFO | "pinned" : " 29 GB" | 20260505 21:21:39.247 | INFO | } | 20260505 21:21:39.247 | INFO | Perf report for the last 5.14001 seconds: | 20260505 21:21:39.247 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:21:39.247 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:39.247 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:39.247 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:39.247 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 21:21:39.247 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:39.247 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:39.247 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:21:39.247 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:39.247 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:39.247 | INFO | Uploaded 36GiB to GPU in 2.15837s (16.6793GiB/s) | 20260505 21:21:39.247 | INFO | Downloaded 34GiB to GPU in 2.06385s (16.4741GiB/s) | 20260505 21:21:44.394 | INFO | Analyzing chunk frames = [6396928,6397440) | 20260505 21:21:44.394 | INFO | MemInfo: { | 20260505 21:21:44.394 | INFO | "gpu" : " 17 GB", | 20260505 21:21:44.394 | INFO | "huge" : "", | 20260505 21:21:44.394 | INFO | "huge_shared" : " 27 GB", | 20260505 21:21:44.394 | INFO | "malloc" : " 14 GB", | 20260505 21:21:44.394 | INFO | "pinned" : " 29 GB" | 20260505 21:21:44.395 | INFO | } | 20260505 21:21:44.395 | INFO | Perf report for the last 5.14793 seconds: | 20260505 21:21:44.395 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:21:44.395 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:44.395 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:44.395 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:44.395 | INFO | ANALYSIS : 62.9%, 3.0, 98.4ms, 0 | 20260505 21:21:44.395 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:44.395 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:44.395 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 21:21:44.395 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:44.395 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:44.395 | INFO | Uploaded 36GiB to GPU in 2.13858s (16.8336GiB/s) | 20260505 21:21:44.395 | INFO | Downloaded 34GiB to GPU in 2.14109s (15.8798GiB/s) | 20260505 21:21:49.537 | INFO | Analyzing chunk frames = [6397440,6397952) | 20260505 21:21:49.537 | INFO | MemInfo: { | 20260505 21:21:49.537 | INFO | "gpu" : " 17 GB", | 20260505 21:21:49.537 | INFO | "huge" : "", | 20260505 21:21:49.537 | INFO | "huge_shared" : " 27 GB", | 20260505 21:21:49.537 | INFO | "malloc" : " 14 GB", | 20260505 21:21:49.537 | INFO | "pinned" : " 29 GB" | 20260505 21:21:49.537 | INFO | } | 20260505 21:21:49.537 | INFO | Perf report for the last 5.14203 seconds: | 20260505 21:21:49.537 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:21:49.537 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:49.537 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:49.537 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:49.537 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 21:21:49.537 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:49.537 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:49.537 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:21:49.537 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:49.537 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:49.537 | INFO | Uploaded 36GiB to GPU in 2.11016s (17.0603GiB/s) | 20260505 21:21:49.537 | INFO | Downloaded 34GiB to GPU in 2.00299s (16.9747GiB/s) | 20260505 21:21:49.537 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 21:21:49.537 | INFO | Total mem usage: 15.3 GiB | 20260505 21:21:49.537 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 21:21:49.537 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.15%) | 20260505 21:21:49.537 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 21:21:49.537 | INFO | High Water Breakdown: | 20260505 21:21:49.537 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 21:21:49.537 | INFO | 38.8% : 5.5 GiB ( 82.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 21:21:49.537 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 21:21:49.537 | INFO | ------------------------------------------ | 20260505 21:21:49.537 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:21:49.537 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 21:21:49.537 | INFO | Total mem usage: 26.5 GiB | 20260505 21:21:49.537 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 21:21:49.537 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 21:21:49.537 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 21:21:49.537 | INFO | High Water Breakdown: | 20260505 21:21:49.538 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:21:49.538 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:21:49.538 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:21:49.538 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:21:49.538 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:21:49.538 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:21:49.538 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 21:21:49.538 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 21:21:49.538 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:21:49.538 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 21:21:49.538 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 21:21:49.538 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 21:21:49.538 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 21:21:49.538 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 21:21:49.538 | INFO | ------------------------------------------ | 20260505 21:21:49.538 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:21:49.538 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 21:21:49.538 | INFO | Total mem usage: 25.6 GiB | 20260505 21:21:49.538 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 21:21:49.538 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 21:21:49.538 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 21:21:49.538 | INFO | High Water Breakdown: | 20260505 21:21:49.538 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 21:21:49.538 | INFO | ------------------------------------------ | 20260505 21:21:49.538 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:21:49.538 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 21:21:49.538 | INFO | Total mem usage: 15.7 GiB | 20260505 21:21:49.538 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 21:21:49.538 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 21:21:49.538 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 21:21:49.538 | INFO | High Water Breakdown: | 20260505 21:21:49.538 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 21:21:49.538 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 21:21:49.538 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 21:21:49.538 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 21:21:49.538 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 21:21:49.538 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 21:21:49.538 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 21:21:49.538 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 21:21:49.538 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 21:21:49.538 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:21:49.538 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 21:21:49.538 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:21:49.538 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 21:21:49.538 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 21:21:49.538 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 21:21:49.538 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 21:21:49.538 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:21:49.538 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 21:21:49.538 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:21:49.538 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:21:49.538 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:21:49.538 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 21:21:49.538 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 21:21:49.538 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:21:49.538 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 21:21:49.538 | INFO | ------------------------------------------ | 20260505 21:21:49.538 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:21:49.538 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 21:21:49.538 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:21:49.538 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:21:49.538 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.42e+01, 1.32e+01 / 1.87e+00 | 20260505 21:21:49.538 | INFO | PerfReport: 0.29%, AggregateStats -- 9.24e-02 / 1.03e-01, 1.05e-02 / 4.32e-03 | 20260505 21:21:49.538 | INFO | PerfReport: 11.82%, Baseline -- 3.71e+00 / 4.56e+00, 5.69e-01 / 1.24e-01 | 20260505 21:21:49.538 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.52e+00, 1.88e-01 / 1.08e-01 | 20260505 21:21:49.538 | INFO | PerfReport: 15.97%, DME -- 4.01e+01 / 4.41e+01, 3.99e+00 / 1.85e+00 | 20260505 21:21:49.538 | INFO | PerfReport: 57.09%, FrameLabeling -- 1.79e+01 / 1.93e+01, 1.43e+00 / 1.04e+00 | 20260505 21:21:49.538 | INFO | PerfReport: 7.23%, PulseAccumulating -- 2.27e+00 / 2.85e+00, 4.37e-01 / 1.85e-01 | 20260505 21:21:49.538 | INFO | PerfReport: 3.21%, Metrics -- 1.01e+00 / 1.34e+00, 2.77e-01 / 1.46e-01 | 20260505 21:21:49.538 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.50e-01 / 4.12e-01, 6.19e-02 / 3.34e-02 | 20260505 21:21:49.539 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 21:21:49.539 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:21:49.539 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:21:49.539 | INFO | PerfReport: Full Execution -- 9.91e+01 / -nan, 2.54e+01 / 0.00e+00 | 20260505 21:21:49.539 | INFO | PerfReport: 22.49%, Upload -- 2.23e+01 / 0.00e+00, 7.75e+00 / 0.00e+00 | 20260505 21:21:49.539 | INFO | PerfReport: 21.78%, Download -- 2.16e+01 / 0.00e+00, 7.68e+00 / 0.00e+00 | 20260505 21:21:49.539 | INFO | PerfReport: 31.69%, Compute -- 3.14e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260505 21:21:49.539 | INFO | PerfReport: 2.28%, LockUpload -- 2.26e+00 / 0.00e+00, 8.00e+00 / 0.00e+00 | 20260505 21:21:49.539 | INFO | PerfReport: 1.33%, LockDownload -- 1.32e+00 / 0.00e+00, 2.85e+00 / 0.00e+00 | 20260505 21:21:49.539 | INFO | PerfReport: 20.42%, LockCompute -- 2.02e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 21:21:49.539 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.36e-02 / 0.00e+00 | 20260505 21:21:49.664 | INFO | Starting a round of Baz IO | 20260505 21:21:52.818 | INFO | Populating RTMetrics report | 20260505 21:21:52.819 | INFO | RTMetrics report populated | 20260505 21:21:52.819 | INFO | Opening tmp RTMetrics json file | 20260505 21:21:52.819 | INFO | Serializing RTMetrics to json | 20260505 21:21:52.821 | INFO | Serialization done, swapping tmp file now... | 20260505 21:21:52.821 | INFO | RTMetrics done | 20260505 21:21:52.899 | INFO | Finished a round of Baz IO: Wrote 1.973 GiB in 3.235 seconds | 20260505 21:21:52.899 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.58% / 0.00% / 78.42% | 20260505 21:21:52.899 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.149616% / 0% / 0.000107288% | 20260505 21:21:52.899 | INFO | Baz IO overall: | 20260505 21:21:52.899 | INFO | BytesWritten : 3708098515894 | 20260505 21:21:52.899 | INFO | NumEvents : 1751144793095 | 20260505 21:21:54.668 | INFO | Analyzing chunk frames = [6397952,6398464) | 20260505 21:21:54.668 | INFO | MemInfo: { | 20260505 21:21:54.668 | INFO | "gpu" : " 17 GB", | 20260505 21:21:54.668 | INFO | "huge" : "", | 20260505 21:21:54.668 | INFO | "huge_shared" : " 27 GB", | 20260505 21:21:54.668 | INFO | "malloc" : " 14 GB", | 20260505 21:21:54.668 | INFO | "pinned" : " 29 GB" | 20260505 21:21:54.668 | INFO | } | 20260505 21:21:54.668 | INFO | Perf report for the last 5.13103 seconds: | 20260505 21:21:54.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:21:54.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:54.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:54.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:54.668 | INFO | ANALYSIS : 62.9%, 3.0, 97.9ms, 0 | 20260505 21:21:54.668 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:54.668 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260505 21:21:54.668 | INFO | PRE_HQ : 60.1%, 1.8, 54.8ms, 0 | 20260505 21:21:54.668 | INFO | BAZWRITER : 62.3%, 3.7, 116.8ms, 0 | 20260505 21:21:54.668 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 21:21:54.668 | INFO | Uploaded 36GiB to GPU in 2.16286s (16.6446GiB/s) | 20260505 21:21:54.668 | INFO | Downloaded 38GiB to GPU in 2.23091s (17.0334GiB/s) | 20260505 21:21:59.803 | INFO | Analyzing chunk frames = [6398464,6398976) | 20260505 21:21:59.803 | INFO | MemInfo: { | 20260505 21:21:59.803 | INFO | "gpu" : " 17 GB", | 20260505 21:21:59.803 | INFO | "huge" : "", | 20260505 21:21:59.803 | INFO | "huge_shared" : " 27 GB", | 20260505 21:21:59.803 | INFO | "malloc" : " 14 GB", | 20260505 21:21:59.803 | INFO | "pinned" : " 29 GB" | 20260505 21:21:59.803 | INFO | } | 20260505 21:21:59.803 | INFO | Perf report for the last 5.13564 seconds: | 20260505 21:21:59.803 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:21:59.803 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:59.803 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:59.803 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:59.803 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260505 21:21:59.803 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:59.803 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:59.804 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 21:21:59.804 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:59.804 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:21:59.804 | INFO | Uploaded 36GiB to GPU in 2.08498s (17.2664GiB/s) | 20260505 21:21:59.804 | INFO | Downloaded 34GiB to GPU in 2.02803s (16.765GiB/s) | 20260505 21:22:04.947 | INFO | Analyzing chunk frames = [6398976,6399488) | 20260505 21:22:04.947 | INFO | MemInfo: { | 20260505 21:22:04.947 | INFO | "gpu" : " 17 GB", | 20260505 21:22:04.947 | INFO | "huge" : "", | 20260505 21:22:04.947 | INFO | "huge_shared" : " 27 GB", | 20260505 21:22:04.947 | INFO | "malloc" : " 14 GB", | 20260505 21:22:04.947 | INFO | "pinned" : " 29 GB" | 20260505 21:22:04.947 | INFO | } | 20260505 21:22:04.947 | INFO | Perf report for the last 5.14347 seconds: | 20260505 21:22:04.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:22:04.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:04.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:04.947 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:04.947 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260505 21:22:04.947 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:04.947 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:04.947 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 21:22:04.947 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:04.947 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:04.947 | INFO | Uploaded 36GiB to GPU in 2.14442s (16.7878GiB/s) | 20260505 21:22:04.947 | INFO | Downloaded 34GiB to GPU in 2.02136s (16.8204GiB/s) | 20260505 21:22:10.088 | INFO | Analyzing chunk frames = [6399488,6400000) | 20260505 21:22:10.088 | INFO | MemInfo: { | 20260505 21:22:10.088 | INFO | "gpu" : " 17 GB", | 20260505 21:22:10.088 | INFO | "huge" : "", | 20260505 21:22:10.088 | INFO | "huge_shared" : " 27 GB", | 20260505 21:22:10.088 | INFO | "malloc" : " 14 GB", | 20260505 21:22:10.088 | INFO | "pinned" : " 29 GB" | 20260505 21:22:10.088 | INFO | } | 20260505 21:22:10.088 | INFO | Perf report for the last 5.14105 seconds: | 20260505 21:22:10.088 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:22:10.088 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:10.088 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:10.088 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:10.088 | INFO | ANALYSIS : 62.0%, 3.0, 96.3ms, 0 | 20260505 21:22:10.088 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:10.088 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:10.088 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:22:10.088 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:10.088 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:10.088 | INFO | Uploaded 36GiB to GPU in 2.14544s (16.7798GiB/s) | 20260505 21:22:10.088 | INFO | Downloaded 34GiB to GPU in 2.04087s (16.6596GiB/s) | 20260505 21:22:15.222 | INFO | Analyzing chunk frames = [6400000,6400512) | 20260505 21:22:15.222 | INFO | MemInfo: { | 20260505 21:22:15.222 | INFO | "gpu" : " 17 GB", | 20260505 21:22:15.222 | INFO | "huge" : "", | 20260505 21:22:15.222 | INFO | "huge_shared" : " 27 GB", | 20260505 21:22:15.222 | INFO | "malloc" : " 14 GB", | 20260505 21:22:15.222 | INFO | "pinned" : " 29 GB" | 20260505 21:22:15.222 | INFO | } | 20260505 21:22:15.222 | INFO | Perf report for the last 5.1344 seconds: | 20260505 21:22:15.222 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:22:15.222 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:15.222 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:15.222 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:15.222 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260505 21:22:15.222 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:15.222 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:15.222 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:22:15.222 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:15.222 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:15.223 | INFO | Uploaded 36GiB to GPU in 2.12692s (16.9259GiB/s) | 20260505 21:22:15.223 | INFO | Downloaded 34GiB to GPU in 2.06075s (16.4989GiB/s) | 20260505 21:22:20.358 | INFO | Analyzing chunk frames = [6400512,6401024) | 20260505 21:22:20.358 | INFO | MemInfo: { | 20260505 21:22:20.358 | INFO | "gpu" : " 17 GB", | 20260505 21:22:20.358 | INFO | "huge" : "", | 20260505 21:22:20.358 | INFO | "huge_shared" : " 27 GB", | 20260505 21:22:20.358 | INFO | "malloc" : " 14 GB", | 20260505 21:22:20.358 | INFO | "pinned" : " 29 GB" | 20260505 21:22:20.358 | INFO | } | 20260505 21:22:20.358 | INFO | Perf report for the last 5.13559 seconds: | 20260505 21:22:20.358 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:22:20.358 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:20.358 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:20.358 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:20.358 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260505 21:22:20.358 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:20.358 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:20.358 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:22:20.358 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:20.358 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:20.358 | INFO | Uploaded 36GiB to GPU in 2.08853s (17.237GiB/s) | 20260505 21:22:20.358 | INFO | Downloaded 34GiB to GPU in 2.03131s (16.738GiB/s) | 20260505 21:22:25.499 | INFO | Analyzing chunk frames = [6401024,6401536) | 20260505 21:22:25.499 | INFO | MemInfo: { | 20260505 21:22:25.499 | INFO | "gpu" : " 17 GB", | 20260505 21:22:25.499 | INFO | "huge" : "", | 20260505 21:22:25.499 | INFO | "huge_shared" : " 27 GB", | 20260505 21:22:25.499 | INFO | "malloc" : " 14 GB", | 20260505 21:22:25.499 | INFO | "pinned" : " 29 GB" | 20260505 21:22:25.500 | INFO | } | 20260505 21:22:25.500 | INFO | Perf report for the last 5.14181 seconds: | 20260505 21:22:25.500 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:22:25.500 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:25.500 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:25.500 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:25.500 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 21:22:25.500 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:25.500 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:25.500 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 21:22:25.500 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:25.500 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:25.500 | INFO | Uploaded 36GiB to GPU in 2.12399s (16.9492GiB/s) | 20260505 21:22:25.500 | INFO | Downloaded 34GiB to GPU in 2.14461s (15.8537GiB/s) | 20260505 21:22:30.643 | INFO | Analyzing chunk frames = [6401536,6402048) | 20260505 21:22:30.643 | INFO | MemInfo: { | 20260505 21:22:30.643 | INFO | "gpu" : " 17 GB", | 20260505 21:22:30.643 | INFO | "huge" : "", | 20260505 21:22:30.643 | INFO | "huge_shared" : " 27 GB", | 20260505 21:22:30.643 | INFO | "malloc" : " 14 GB", | 20260505 21:22:30.643 | INFO | "pinned" : " 29 GB" | 20260505 21:22:30.643 | INFO | } | 20260505 21:22:30.643 | INFO | Perf report for the last 5.14345 seconds: | 20260505 21:22:30.643 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:22:30.643 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:30.643 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:30.643 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:30.643 | INFO | ANALYSIS : 62.7%, 3.0, 98.0ms, 0 | 20260505 21:22:30.643 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:30.643 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:30.643 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:22:30.643 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:30.643 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:30.643 | INFO | Uploaded 36GiB to GPU in 2.1968s (16.3875GiB/s) | 20260505 21:22:30.644 | INFO | Downloaded 34GiB to GPU in 2.04805s (16.6012GiB/s) | 20260505 21:22:30.773 | INFO | Starting a round of Baz IO | 20260505 21:22:33.926 | INFO | Populating RTMetrics report | 20260505 21:22:33.927 | INFO | RTMetrics report populated | 20260505 21:22:33.927 | INFO | Opening tmp RTMetrics json file | 20260505 21:22:33.927 | INFO | Serializing RTMetrics to json | 20260505 21:22:33.932 | INFO | Serialization done, swapping tmp file now... | 20260505 21:22:33.934 | INFO | RTMetrics done | 20260505 21:22:34.012 | INFO | Finished a round of Baz IO: Wrote 1.974 GiB in 3.239 seconds | 20260505 21:22:34.012 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.58% / 0.00% / 78.42% | 20260505 21:22:34.012 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.149981% / 3.97364e-06% / 0.000115236% | 20260505 21:22:34.012 | INFO | Baz IO overall: | 20260505 21:22:34.012 | INFO | BytesWritten : 3710218318752 | 20260505 21:22:34.012 | INFO | NumEvents : 1752123577342 | 20260505 21:22:35.787 | INFO | Analyzing chunk frames = [6402048,6402560) | 20260505 21:22:35.787 | INFO | MemInfo: { | 20260505 21:22:35.787 | INFO | "gpu" : " 17 GB", | 20260505 21:22:35.787 | INFO | "huge" : "", | 20260505 21:22:35.787 | INFO | "huge_shared" : " 27 GB", | 20260505 21:22:35.787 | INFO | "malloc" : " 14 GB", | 20260505 21:22:35.787 | INFO | "pinned" : " 29 GB" | 20260505 21:22:35.787 | INFO | } | 20260505 21:22:35.787 | INFO | Perf report for the last 5.14357 seconds: | 20260505 21:22:35.787 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:22:35.787 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:35.787 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:35.787 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:35.787 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 21:22:35.787 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:35.787 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260505 21:22:35.787 | INFO | PRE_HQ : 59.9%, 1.8, 54.9ms, 0 | 20260505 21:22:35.787 | INFO | BAZWRITER : 62.3%, 3.7, 117.8ms, 0 | 20260505 21:22:35.787 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 21:22:35.787 | INFO | Uploaded 36GiB to GPU in 2.14937s (16.7491GiB/s) | 20260505 21:22:35.787 | INFO | Downloaded 38GiB to GPU in 2.2417s (16.9514GiB/s) | 20260505 21:22:40.920 | INFO | Analyzing chunk frames = [6402560,6403072) | 20260505 21:22:40.920 | INFO | MemInfo: { | 20260505 21:22:40.920 | INFO | "gpu" : " 17 GB", | 20260505 21:22:40.920 | INFO | "huge" : "", | 20260505 21:22:40.920 | INFO | "huge_shared" : " 27 GB", | 20260505 21:22:40.920 | INFO | "malloc" : " 14 GB", | 20260505 21:22:40.920 | INFO | "pinned" : " 29 GB" | 20260505 21:22:40.920 | INFO | } | 20260505 21:22:40.920 | INFO | Perf report for the last 5.13312 seconds: | 20260505 21:22:40.920 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:22:40.920 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:40.920 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:40.920 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:40.920 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260505 21:22:40.920 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:40.920 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:40.920 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 21:22:40.920 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:40.920 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:40.920 | INFO | Uploaded 36GiB to GPU in 2.08088s (17.3004GiB/s) | 20260505 21:22:40.920 | INFO | Downloaded 34GiB to GPU in 2.02021s (16.8299GiB/s) | 20260505 21:22:46.055 | INFO | Analyzing chunk frames = [6403072,6403584) | 20260505 21:22:46.055 | INFO | MemInfo: { | 20260505 21:22:46.055 | INFO | "gpu" : " 17 GB", | 20260505 21:22:46.055 | INFO | "huge" : "", | 20260505 21:22:46.055 | INFO | "huge_shared" : " 27 GB", | 20260505 21:22:46.055 | INFO | "malloc" : " 14 GB", | 20260505 21:22:46.055 | INFO | "pinned" : " 29 GB" | 20260505 21:22:46.055 | INFO | } | 20260505 21:22:46.055 | INFO | Perf report for the last 5.13506 seconds: | 20260505 21:22:46.055 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:22:46.055 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:46.055 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:46.055 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:46.055 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260505 21:22:46.055 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:46.055 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:46.055 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:22:46.055 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:46.055 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:46.055 | INFO | Uploaded 36GiB to GPU in 2.12028s (16.9789GiB/s) | 20260505 21:22:46.055 | INFO | Downloaded 34GiB to GPU in 2.02558s (16.7853GiB/s) | 20260505 21:22:51.204 | INFO | Analyzing chunk frames = [6403584,6404096) | 20260505 21:22:51.204 | INFO | MemInfo: { | 20260505 21:22:51.204 | INFO | "gpu" : " 17 GB", | 20260505 21:22:51.204 | INFO | "huge" : "", | 20260505 21:22:51.204 | INFO | "huge_shared" : " 27 GB", | 20260505 21:22:51.204 | INFO | "malloc" : " 14 GB", | 20260505 21:22:51.204 | INFO | "pinned" : " 29 GB" | 20260505 21:22:51.204 | INFO | } | 20260505 21:22:51.204 | INFO | Perf report for the last 5.14909 seconds: | 20260505 21:22:51.204 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:22:51.204 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:51.204 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:51.204 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:51.204 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260505 21:22:51.204 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:51.204 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:51.204 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 21:22:51.204 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:51.204 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:51.204 | INFO | Uploaded 36GiB to GPU in 2.20171s (16.351GiB/s) | 20260505 21:22:51.204 | INFO | Downloaded 34GiB to GPU in 2.04646s (16.6141GiB/s) | 20260505 21:22:56.339 | INFO | Analyzing chunk frames = [6404096,6404608) | 20260505 21:22:56.339 | INFO | MemInfo: { | 20260505 21:22:56.339 | INFO | "gpu" : " 17 GB", | 20260505 21:22:56.339 | INFO | "huge" : "", | 20260505 21:22:56.339 | INFO | "huge_shared" : " 27 GB", | 20260505 21:22:56.339 | INFO | "malloc" : " 14 GB", | 20260505 21:22:56.339 | INFO | "pinned" : " 29 GB" | 20260505 21:22:56.339 | INFO | } | 20260505 21:22:56.339 | INFO | Perf report for the last 5.13518 seconds: | 20260505 21:22:56.339 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:22:56.339 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:56.339 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:56.339 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:56.339 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260505 21:22:56.339 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:56.339 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:56.339 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:22:56.339 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:56.339 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:22:56.340 | INFO | Uploaded 36GiB to GPU in 2.16022s (16.665GiB/s) | 20260505 21:22:56.340 | INFO | Downloaded 34GiB to GPU in 2.06829s (16.4387GiB/s) | 20260505 21:23:01.484 | INFO | Analyzing chunk frames = [6404608,6405120) | 20260505 21:23:01.484 | INFO | MemInfo: { | 20260505 21:23:01.484 | INFO | "gpu" : " 17 GB", | 20260505 21:23:01.484 | INFO | "huge" : "", | 20260505 21:23:01.484 | INFO | "huge_shared" : " 27 GB", | 20260505 21:23:01.484 | INFO | "malloc" : " 14 GB", | 20260505 21:23:01.484 | INFO | "pinned" : " 29 GB" | 20260505 21:23:01.484 | INFO | } | 20260505 21:23:01.484 | INFO | Perf report for the last 5.1449 seconds: | 20260505 21:23:01.484 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:23:01.484 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:01.484 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:01.484 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:01.484 | INFO | ANALYSIS : 61.7%, 3.0, 96.5ms, 0 | 20260505 21:23:01.484 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:01.484 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:01.484 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:23:01.484 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:01.484 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:01.485 | INFO | Uploaded 36GiB to GPU in 2.08211s (17.2902GiB/s) | 20260505 21:23:01.485 | INFO | Downloaded 34GiB to GPU in 2.03396s (16.7162GiB/s) | 20260505 21:23:06.622 | INFO | Analyzing chunk frames = [6405120,6405632) | 20260505 21:23:06.622 | INFO | MemInfo: { | 20260505 21:23:06.622 | INFO | "gpu" : " 17 GB", | 20260505 21:23:06.622 | INFO | "huge" : "", | 20260505 21:23:06.622 | INFO | "huge_shared" : " 27 GB", | 20260505 21:23:06.622 | INFO | "malloc" : " 14 GB", | 20260505 21:23:06.622 | INFO | "pinned" : " 29 GB" | 20260505 21:23:06.622 | INFO | } | 20260505 21:23:06.622 | INFO | Perf report for the last 5.13806 seconds: | 20260505 21:23:06.622 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:23:06.622 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:06.622 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:06.622 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:06.622 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 21:23:06.622 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:06.622 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:06.622 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 21:23:06.622 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:06.622 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:06.623 | INFO | Uploaded 36GiB to GPU in 2.14918s (16.7506GiB/s) | 20260505 21:23:06.623 | INFO | Downloaded 34GiB to GPU in 2.13109s (15.9543GiB/s) | 20260505 21:23:11.759 | INFO | Analyzing chunk frames = [6405632,6406144) | 20260505 21:23:11.759 | INFO | MemInfo: { | 20260505 21:23:11.759 | INFO | "gpu" : " 17 GB", | 20260505 21:23:11.759 | INFO | "huge" : "", | 20260505 21:23:11.759 | INFO | "huge_shared" : " 27 GB", | 20260505 21:23:11.759 | INFO | "malloc" : " 14 GB", | 20260505 21:23:11.759 | INFO | "pinned" : " 29 GB" | 20260505 21:23:11.759 | INFO | } | 20260505 21:23:11.759 | INFO | Perf report for the last 5.13714 seconds: | 20260505 21:23:11.759 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:23:11.759 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:11.759 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:11.759 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:11.759 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 21:23:11.759 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:11.759 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:11.759 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:23:11.759 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:11.759 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:11.760 | INFO | Uploaded 36GiB to GPU in 2.10308s (17.1177GiB/s) | 20260505 21:23:11.760 | INFO | Downloaded 34GiB to GPU in 2.00408s (16.9654GiB/s) | 20260505 21:23:11.760 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 21:23:11.760 | INFO | Total mem usage: 15.3 GiB | 20260505 21:23:11.760 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 21:23:11.760 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.16%) | 20260505 21:23:11.760 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 21:23:11.760 | INFO | High Water Breakdown: | 20260505 21:23:11.760 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 21:23:11.760 | INFO | 38.8% : 5.5 GiB ( 82.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 21:23:11.760 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 21:23:11.760 | INFO | ------------------------------------------ | 20260505 21:23:11.760 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:23:11.760 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 21:23:11.760 | INFO | Total mem usage: 26.5 GiB | 20260505 21:23:11.760 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 21:23:11.760 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 21:23:11.760 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 21:23:11.760 | INFO | High Water Breakdown: | 20260505 21:23:11.760 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:23:11.760 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:23:11.760 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:23:11.760 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:23:11.760 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:23:11.760 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:23:11.760 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 21:23:11.760 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 21:23:11.760 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:23:11.760 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 21:23:11.760 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 21:23:11.760 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 21:23:11.760 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 21:23:11.760 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 21:23:11.760 | INFO | ------------------------------------------ | 20260505 21:23:11.760 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:23:11.760 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 21:23:11.760 | INFO | Total mem usage: 25.6 GiB | 20260505 21:23:11.760 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 21:23:11.760 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 21:23:11.760 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 21:23:11.760 | INFO | High Water Breakdown: | 20260505 21:23:11.760 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 21:23:11.760 | INFO | ------------------------------------------ | 20260505 21:23:11.760 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:23:11.760 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 21:23:11.760 | INFO | Total mem usage: 15.7 GiB | 20260505 21:23:11.760 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 21:23:11.760 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 21:23:11.760 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 21:23:11.760 | INFO | High Water Breakdown: | 20260505 21:23:11.760 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 21:23:11.760 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 21:23:11.760 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 21:23:11.760 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 21:23:11.760 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 21:23:11.760 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 21:23:11.760 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 21:23:11.761 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 21:23:11.761 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 21:23:11.761 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:23:11.761 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 21:23:11.761 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:23:11.761 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 21:23:11.761 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 21:23:11.761 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 21:23:11.761 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 21:23:11.761 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:23:11.761 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 21:23:11.761 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:23:11.761 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:23:11.761 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:23:11.761 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 21:23:11.761 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 21:23:11.761 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:23:11.761 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 21:23:11.761 | INFO | ------------------------------------------ | 20260505 21:23:11.761 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:23:11.761 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 21:23:11.761 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:23:11.761 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:23:11.761 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.44e+01, 1.35e+01 / 1.71e+00 | 20260505 21:23:11.761 | INFO | PerfReport: 0.29%, AggregateStats -- 9.25e-02 / 1.04e-01, 1.05e-02 / 1.05e-02 | 20260505 21:23:11.761 | INFO | PerfReport: 11.81%, Baseline -- 3.71e+00 / 4.56e+00, 5.65e-01 / 1.66e-01 | 20260505 21:23:11.761 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.50e+00, 1.88e-01 / 1.14e-01 | 20260505 21:23:11.761 | INFO | PerfReport: 15.96%, DME -- 4.02e+01 / 4.43e+01, 4.08e+00 / 1.77e+00 | 20260505 21:23:11.761 | INFO | PerfReport: 57.17%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.52e+00 / 1.14e+00 | 20260505 21:23:11.761 | INFO | PerfReport: 7.19%, PulseAccumulating -- 2.26e+00 / 2.80e+00, 4.35e-01 / 2.42e-01 | 20260505 21:23:11.761 | INFO | PerfReport: 3.19%, Metrics -- 1.00e+00 / 1.30e+00, 2.73e-01 / 9.89e-02 | 20260505 21:23:11.761 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.50e-01 / 4.09e-01, 6.18e-02 / 3.64e-02 | 20260505 21:23:11.761 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 21:23:11.761 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:23:11.761 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:23:11.761 | INFO | PerfReport: Full Execution -- 9.91e+01 / -nan, 2.52e+01 / 0.00e+00 | 20260505 21:23:11.761 | INFO | PerfReport: 22.43%, Upload -- 2.22e+01 / 0.00e+00, 7.62e+00 / 0.00e+00 | 20260505 21:23:11.761 | INFO | PerfReport: 21.81%, Download -- 2.16e+01 / 0.00e+00, 7.62e+00 / 0.00e+00 | 20260505 21:23:11.761 | INFO | PerfReport: 31.77%, Compute -- 3.15e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 21:23:11.761 | INFO | PerfReport: 2.18%, LockUpload -- 2.16e+00 / 0.00e+00, 7.87e+00 / 0.00e+00 | 20260505 21:23:11.761 | INFO | PerfReport: 1.37%, LockDownload -- 1.36e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260505 21:23:11.761 | INFO | PerfReport: 20.43%, LockCompute -- 2.02e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 21:23:11.761 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.70e-02 / 0.00e+00 | 20260505 21:23:11.889 | INFO | Starting a round of Baz IO | 20260505 21:23:15.044 | INFO | Populating RTMetrics report | 20260505 21:23:15.044 | INFO | RTMetrics report populated | 20260505 21:23:15.044 | INFO | Opening tmp RTMetrics json file | 20260505 21:23:15.045 | INFO | Serializing RTMetrics to json | 20260505 21:23:15.046 | INFO | Serialization done, swapping tmp file now... | 20260505 21:23:15.046 | INFO | RTMetrics done | 20260505 21:23:15.125 | INFO | Finished a round of Baz IO: Wrote 1.974 GiB in 3.236 seconds | 20260505 21:23:15.125 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.58% / 0.00% / 78.42% | 20260505 21:23:15.125 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.150017% / 0% / 0.000103315% | 20260505 21:23:15.125 | INFO | Baz IO overall: | 20260505 21:23:15.125 | INFO | BytesWritten : 3712337720568 | 20260505 21:23:15.125 | INFO | NumEvents : 1753102131219 | 20260505 21:23:16.900 | INFO | Analyzing chunk frames = [6406144,6406656) | 20260505 21:23:16.900 | INFO | MemInfo: { | 20260505 21:23:16.900 | INFO | "gpu" : " 17 GB", | 20260505 21:23:16.900 | INFO | "huge" : "", | 20260505 21:23:16.900 | INFO | "huge_shared" : " 27 GB", | 20260505 21:23:16.900 | INFO | "malloc" : " 14 GB", | 20260505 21:23:16.900 | INFO | "pinned" : " 29 GB" | 20260505 21:23:16.900 | INFO | } | 20260505 21:23:16.900 | INFO | Perf report for the last 5.14048 seconds: | 20260505 21:23:16.900 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:23:16.900 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:16.900 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:16.900 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:16.900 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 21:23:16.900 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:16.900 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260505 21:23:16.900 | INFO | PRE_HQ : 60.1%, 1.8, 55.2ms, 0 | 20260505 21:23:16.900 | INFO | BAZWRITER : 62.3%, 3.6, 116.7ms, 0 | 20260505 21:23:16.900 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 21:23:16.900 | INFO | Uploaded 36GiB to GPU in 2.1671s (16.6121GiB/s) | 20260505 21:23:16.900 | INFO | Downloaded 38GiB to GPU in 2.24361s (16.937GiB/s) | 20260505 21:23:22.040 | INFO | Analyzing chunk frames = [6406656,6407168) | 20260505 21:23:22.040 | INFO | MemInfo: { | 20260505 21:23:22.040 | INFO | "gpu" : " 17 GB", | 20260505 21:23:22.040 | INFO | "huge" : "", | 20260505 21:23:22.040 | INFO | "huge_shared" : " 27 GB", | 20260505 21:23:22.040 | INFO | "malloc" : " 14 GB", | 20260505 21:23:22.040 | INFO | "pinned" : " 29 GB" | 20260505 21:23:22.040 | INFO | } | 20260505 21:23:22.040 | INFO | Perf report for the last 5.13989 seconds: | 20260505 21:23:22.040 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:23:22.040 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:22.040 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:22.040 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:22.040 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 21:23:22.040 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:22.040 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:22.040 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 21:23:22.040 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:22.040 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:22.040 | INFO | Uploaded 36GiB to GPU in 2.07758s (17.3279GiB/s) | 20260505 21:23:22.040 | INFO | Downloaded 34GiB to GPU in 2.02159s (16.8184GiB/s) | 20260505 21:23:27.175 | INFO | Analyzing chunk frames = [6407168,6407680) | 20260505 21:23:27.175 | INFO | MemInfo: { | 20260505 21:23:27.175 | INFO | "gpu" : " 17 GB", | 20260505 21:23:27.175 | INFO | "huge" : "", | 20260505 21:23:27.175 | INFO | "huge_shared" : " 27 GB", | 20260505 21:23:27.175 | INFO | "malloc" : " 14 GB", | 20260505 21:23:27.175 | INFO | "pinned" : " 29 GB" | 20260505 21:23:27.175 | INFO | } | 20260505 21:23:27.175 | INFO | Perf report for the last 5.13535 seconds: | 20260505 21:23:27.175 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:23:27.175 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:27.175 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:27.175 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:27.175 | INFO | ANALYSIS : 62.1%, 3.0, 96.5ms, 0 | 20260505 21:23:27.175 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:27.175 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:27.175 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:23:27.175 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:27.175 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:27.176 | INFO | Uploaded 36GiB to GPU in 2.15s (16.7442GiB/s) | 20260505 21:23:27.176 | INFO | Downloaded 34GiB to GPU in 2.05231s (16.5667GiB/s) | 20260505 21:23:32.310 | INFO | Analyzing chunk frames = [6407680,6408192) | 20260505 21:23:32.310 | INFO | MemInfo: { | 20260505 21:23:32.310 | INFO | "gpu" : " 17 GB", | 20260505 21:23:32.310 | INFO | "huge" : "", | 20260505 21:23:32.310 | INFO | "huge_shared" : " 27 GB", | 20260505 21:23:32.310 | INFO | "malloc" : " 14 GB", | 20260505 21:23:32.310 | INFO | "pinned" : " 29 GB" | 20260505 21:23:32.310 | INFO | } | 20260505 21:23:32.310 | INFO | Perf report for the last 5.1347 seconds: | 20260505 21:23:32.310 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:23:32.310 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:32.310 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:32.310 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:32.310 | INFO | ANALYSIS : 62.4%, 3.0, 96.8ms, 0 | 20260505 21:23:32.310 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:32.310 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:32.310 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 21:23:32.310 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:32.310 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:32.310 | INFO | Uploaded 36GiB to GPU in 2.20573s (16.3211GiB/s) | 20260505 21:23:32.310 | INFO | Downloaded 34GiB to GPU in 2.0606s (16.5001GiB/s) | 20260505 21:23:37.455 | INFO | Analyzing chunk frames = [6408192,6408704) | 20260505 21:23:37.455 | INFO | MemInfo: { | 20260505 21:23:37.455 | INFO | "gpu" : " 17 GB", | 20260505 21:23:37.455 | INFO | "huge" : "", | 20260505 21:23:37.455 | INFO | "huge_shared" : " 27 GB", | 20260505 21:23:37.455 | INFO | "malloc" : " 14 GB", | 20260505 21:23:37.455 | INFO | "pinned" : " 29 GB" | 20260505 21:23:37.455 | INFO | } | 20260505 21:23:37.455 | INFO | Perf report for the last 5.14491 seconds: | 20260505 21:23:37.455 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:23:37.455 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:37.455 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:37.455 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:37.455 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 21:23:37.455 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:37.455 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:37.455 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:23:37.455 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:37.455 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:37.455 | INFO | Uploaded 36GiB to GPU in 2.14374s (16.793GiB/s) | 20260505 21:23:37.455 | INFO | Downloaded 34GiB to GPU in 2.06578s (16.4586GiB/s) | 20260505 21:23:42.595 | INFO | Analyzing chunk frames = [6408704,6409216) | 20260505 21:23:42.595 | INFO | MemInfo: { | 20260505 21:23:42.595 | INFO | "gpu" : " 17 GB", | 20260505 21:23:42.595 | INFO | "huge" : "", | 20260505 21:23:42.595 | INFO | "huge_shared" : " 27 GB", | 20260505 21:23:42.595 | INFO | "malloc" : " 14 GB", | 20260505 21:23:42.595 | INFO | "pinned" : " 29 GB" | 20260505 21:23:42.595 | INFO | } | 20260505 21:23:42.595 | INFO | Perf report for the last 5.14016 seconds: | 20260505 21:23:42.595 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:23:42.595 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:42.595 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:42.595 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:42.595 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 21:23:42.595 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:42.595 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:42.595 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:23:42.595 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:42.595 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:42.595 | INFO | Uploaded 36GiB to GPU in 2.16113s (16.6579GiB/s) | 20260505 21:23:42.595 | INFO | Downloaded 34GiB to GPU in 2.0697s (16.4275GiB/s) | 20260505 21:23:47.737 | INFO | Analyzing chunk frames = [6409216,6409728) | 20260505 21:23:47.737 | INFO | MemInfo: { | 20260505 21:23:47.737 | INFO | "gpu" : " 17 GB", | 20260505 21:23:47.737 | INFO | "huge" : "", | 20260505 21:23:47.737 | INFO | "huge_shared" : " 27 GB", | 20260505 21:23:47.737 | INFO | "malloc" : " 14 GB", | 20260505 21:23:47.737 | INFO | "pinned" : " 29 GB" | 20260505 21:23:47.737 | INFO | } | 20260505 21:23:47.737 | INFO | Perf report for the last 5.14247 seconds: | 20260505 21:23:47.737 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:23:47.737 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:47.737 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:47.738 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:47.738 | INFO | ANALYSIS : 62.7%, 3.0, 98.0ms, 0 | 20260505 21:23:47.738 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:47.738 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:47.738 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 21:23:47.738 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:47.738 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:47.738 | INFO | Uploaded 36GiB to GPU in 2.13285s (16.8788GiB/s) | 20260505 21:23:47.738 | INFO | Downloaded 34GiB to GPU in 2.09748s (16.2099GiB/s) | 20260505 21:23:52.868 | INFO | Analyzing chunk frames = [6409728,6410240) | 20260505 21:23:52.868 | INFO | MemInfo: { | 20260505 21:23:52.868 | INFO | "gpu" : " 17 GB", | 20260505 21:23:52.868 | INFO | "huge" : "", | 20260505 21:23:52.868 | INFO | "huge_shared" : " 27 GB", | 20260505 21:23:52.868 | INFO | "malloc" : " 14 GB", | 20260505 21:23:52.868 | INFO | "pinned" : " 29 GB" | 20260505 21:23:52.868 | INFO | } | 20260505 21:23:52.868 | INFO | Perf report for the last 5.1309 seconds: | 20260505 21:23:52.868 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:23:52.868 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:52.868 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:52.868 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:52.868 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260505 21:23:52.868 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:52.868 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:52.868 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 21:23:52.868 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:52.868 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:52.869 | INFO | Uploaded 36GiB to GPU in 2.10505s (17.1017GiB/s) | 20260505 21:23:52.869 | INFO | Downloaded 34GiB to GPU in 2.00298s (16.9747GiB/s) | 20260505 21:23:52.999 | INFO | Starting a round of Baz IO | 20260505 21:23:56.168 | INFO | Populating RTMetrics report | 20260505 21:23:56.169 | INFO | RTMetrics report populated | 20260505 21:23:56.169 | INFO | Opening tmp RTMetrics json file | 20260505 21:23:56.170 | INFO | Serializing RTMetrics to json | 20260505 21:23:56.173 | INFO | Serialization done, swapping tmp file now... | 20260505 21:23:56.174 | INFO | RTMetrics done | 20260505 21:23:56.257 | INFO | Finished a round of Baz IO: Wrote 1.971 GiB in 3.258 seconds | 20260505 21:23:56.257 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.58% / 0.00% / 78.42% | 20260505 21:23:56.257 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.148904% / 0% / 7.94729e-05% | 20260505 21:23:56.257 | INFO | Baz IO overall: | 20260505 21:23:56.257 | INFO | BytesWritten : 3714454175839 | 20260505 21:23:56.257 | INFO | NumEvents : 1754078739040 | 20260505 21:23:58.011 | INFO | Analyzing chunk frames = [6410240,6410752) | 20260505 21:23:58.011 | INFO | MemInfo: { | 20260505 21:23:58.011 | INFO | "gpu" : " 17 GB", | 20260505 21:23:58.011 | INFO | "huge" : "", | 20260505 21:23:58.011 | INFO | "huge_shared" : " 27 GB", | 20260505 21:23:58.011 | INFO | "malloc" : " 14 GB", | 20260505 21:23:58.011 | INFO | "pinned" : " 29 GB" | 20260505 21:23:58.011 | INFO | } | 20260505 21:23:58.011 | INFO | Perf report for the last 5.14257 seconds: | 20260505 21:23:58.011 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:23:58.011 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:58.011 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:58.011 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:58.011 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 21:23:58.011 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:23:58.011 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260505 21:23:58.011 | INFO | PRE_HQ : 60.1%, 1.8, 55.4ms, 0 | 20260505 21:23:58.011 | INFO | BAZWRITER : 62.8%, 3.6, 116.7ms, 0 | 20260505 21:23:58.011 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 21:23:58.011 | INFO | Uploaded 36GiB to GPU in 2.16399s (16.6359GiB/s) | 20260505 21:23:58.011 | INFO | Downloaded 38GiB to GPU in 2.25288s (16.8673GiB/s) | 20260505 21:24:03.153 | INFO | Analyzing chunk frames = [6410752,6411264) | 20260505 21:24:03.153 | INFO | MemInfo: { | 20260505 21:24:03.153 | INFO | "gpu" : " 17 GB", | 20260505 21:24:03.153 | INFO | "huge" : "", | 20260505 21:24:03.153 | INFO | "huge_shared" : " 27 GB", | 20260505 21:24:03.153 | INFO | "malloc" : " 14 GB", | 20260505 21:24:03.153 | INFO | "pinned" : " 29 GB" | 20260505 21:24:03.153 | INFO | } | 20260505 21:24:03.153 | INFO | Perf report for the last 5.14186 seconds: | 20260505 21:24:03.153 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:24:03.153 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:03.153 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:03.153 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:03.153 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 21:24:03.153 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:03.153 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:03.153 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:24:03.153 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:03.153 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:03.153 | INFO | Uploaded 36GiB to GPU in 2.06963s (17.3945GiB/s) | 20260505 21:24:03.153 | INFO | Downloaded 34GiB to GPU in 2.02149s (16.8192GiB/s) | 20260505 21:24:08.290 | INFO | Analyzing chunk frames = [6411264,6411776) | 20260505 21:24:08.290 | INFO | MemInfo: { | 20260505 21:24:08.290 | INFO | "gpu" : " 17 GB", | 20260505 21:24:08.290 | INFO | "huge" : "", | 20260505 21:24:08.291 | INFO | "huge_shared" : " 27 GB", | 20260505 21:24:08.291 | INFO | "malloc" : " 14 GB", | 20260505 21:24:08.291 | INFO | "pinned" : " 29 GB" | 20260505 21:24:08.291 | INFO | } | 20260505 21:24:08.291 | INFO | Perf report for the last 5.13779 seconds: | 20260505 21:24:08.291 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:24:08.291 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:08.291 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:08.291 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:08.291 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260505 21:24:08.291 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:08.291 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:08.291 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:24:08.291 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:08.291 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:08.291 | INFO | Uploaded 36GiB to GPU in 2.14087s (16.8156GiB/s) | 20260505 21:24:08.291 | INFO | Downloaded 34GiB to GPU in 2.02306s (16.8063GiB/s) | 20260505 21:24:13.428 | INFO | Analyzing chunk frames = [6411776,6412288) | 20260505 21:24:13.428 | INFO | MemInfo: { | 20260505 21:24:13.428 | INFO | "gpu" : " 17 GB", | 20260505 21:24:13.428 | INFO | "huge" : "", | 20260505 21:24:13.428 | INFO | "huge_shared" : " 27 GB", | 20260505 21:24:13.428 | INFO | "malloc" : " 14 GB", | 20260505 21:24:13.428 | INFO | "pinned" : " 29 GB" | 20260505 21:24:13.428 | INFO | } | 20260505 21:24:13.428 | INFO | Perf report for the last 5.13709 seconds: | 20260505 21:24:13.428 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:24:13.428 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:13.428 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:13.428 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:13.428 | INFO | ANALYSIS : 62.4%, 3.0, 96.8ms, 0 | 20260505 21:24:13.428 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:13.428 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:13.428 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:24:13.428 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:13.428 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:13.428 | INFO | Uploaded 36GiB to GPU in 2.2093s (16.2948GiB/s) | 20260505 21:24:13.428 | INFO | Downloaded 34GiB to GPU in 2.04902s (16.5933GiB/s) | 20260505 21:24:18.568 | INFO | Analyzing chunk frames = [6412288,6412800) | 20260505 21:24:18.568 | INFO | MemInfo: { | 20260505 21:24:18.568 | INFO | "gpu" : " 17 GB", | 20260505 21:24:18.568 | INFO | "huge" : "", | 20260505 21:24:18.568 | INFO | "huge_shared" : " 27 GB", | 20260505 21:24:18.568 | INFO | "malloc" : " 14 GB", | 20260505 21:24:18.568 | INFO | "pinned" : " 29 GB" | 20260505 21:24:18.568 | INFO | } | 20260505 21:24:18.568 | INFO | Perf report for the last 5.14035 seconds: | 20260505 21:24:18.568 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:24:18.568 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:18.568 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:18.568 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:18.568 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260505 21:24:18.568 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:18.568 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:18.568 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 21:24:18.568 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:18.568 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:18.568 | INFO | Uploaded 36GiB to GPU in 2.13823s (16.8363GiB/s) | 20260505 21:24:18.568 | INFO | Downloaded 34GiB to GPU in 2.06374s (16.4749GiB/s) | 20260505 21:24:23.717 | INFO | Analyzing chunk frames = [6412800,6413312) | 20260505 21:24:23.717 | INFO | MemInfo: { | 20260505 21:24:23.717 | INFO | "gpu" : " 17 GB", | 20260505 21:24:23.717 | INFO | "huge" : "", | 20260505 21:24:23.717 | INFO | "huge_shared" : " 27 GB", | 20260505 21:24:23.717 | INFO | "malloc" : " 14 GB", | 20260505 21:24:23.717 | INFO | "pinned" : " 29 GB" | 20260505 21:24:23.717 | INFO | } | 20260505 21:24:23.717 | INFO | Perf report for the last 5.14895 seconds: | 20260505 21:24:23.717 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:24:23.717 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:23.717 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:23.717 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:23.717 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260505 21:24:23.717 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:23.717 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:23.717 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 21:24:23.717 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:23.717 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:23.717 | INFO | Uploaded 36GiB to GPU in 2.09099s (17.2167GiB/s) | 20260505 21:24:23.717 | INFO | Downloaded 34GiB to GPU in 2.03647s (16.6955GiB/s) | 20260505 21:24:28.848 | INFO | Analyzing chunk frames = [6413312,6413824) | 20260505 21:24:28.848 | INFO | MemInfo: { | 20260505 21:24:28.848 | INFO | "gpu" : " 17 GB", | 20260505 21:24:28.848 | INFO | "huge" : "", | 20260505 21:24:28.848 | INFO | "huge_shared" : " 27 GB", | 20260505 21:24:28.848 | INFO | "malloc" : " 14 GB", | 20260505 21:24:28.848 | INFO | "pinned" : " 29 GB" | 20260505 21:24:28.848 | INFO | } | 20260505 21:24:28.848 | INFO | Perf report for the last 5.13116 seconds: | 20260505 21:24:28.848 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:24:28.848 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:28.848 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:28.848 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:28.848 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260505 21:24:28.848 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:28.848 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:28.848 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 21:24:28.848 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:28.848 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:28.849 | INFO | Uploaded 36GiB to GPU in 2.02562s (17.7723GiB/s) | 20260505 21:24:28.849 | INFO | Downloaded 34GiB to GPU in 2.03677s (16.6931GiB/s) | 20260505 21:24:33.985 | INFO | Analyzing chunk frames = [6413824,6414336) | 20260505 21:24:33.985 | INFO | MemInfo: { | 20260505 21:24:33.985 | INFO | "gpu" : " 17 GB", | 20260505 21:24:33.985 | INFO | "huge" : "", | 20260505 21:24:33.985 | INFO | "huge_shared" : " 27 GB", | 20260505 21:24:33.985 | INFO | "malloc" : " 14 GB", | 20260505 21:24:33.985 | INFO | "pinned" : " 29 GB" | 20260505 21:24:33.985 | INFO | } | 20260505 21:24:33.985 | INFO | Perf report for the last 5.13692 seconds: | 20260505 21:24:33.985 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:24:33.985 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:33.985 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:33.985 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:33.985 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 21:24:33.985 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:33.985 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:33.985 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:24:33.985 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:33.985 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:33.985 | INFO | Uploaded 36GiB to GPU in 2.12122s (16.9714GiB/s) | 20260505 21:24:33.985 | INFO | Downloaded 34GiB to GPU in 1.99877s (17.0105GiB/s) | 20260505 21:24:33.986 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 21:24:33.986 | INFO | Total mem usage: 15.3 GiB | 20260505 21:24:33.986 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 21:24:33.986 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.17%) | 20260505 21:24:33.986 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 21:24:33.986 | INFO | High Water Breakdown: | 20260505 21:24:33.986 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 21:24:33.986 | INFO | 38.8% : 5.5 GiB ( 82.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 21:24:33.986 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 21:24:33.986 | INFO | ------------------------------------------ | 20260505 21:24:33.986 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:24:33.986 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 21:24:33.986 | INFO | Total mem usage: 26.5 GiB | 20260505 21:24:33.986 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 21:24:33.986 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 21:24:33.986 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 21:24:33.986 | INFO | High Water Breakdown: | 20260505 21:24:33.986 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:24:33.986 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:24:33.986 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:24:33.986 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:24:33.986 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:24:33.986 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:24:33.986 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 21:24:33.986 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 21:24:33.986 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:24:33.986 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 21:24:33.986 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 21:24:33.986 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 21:24:33.986 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 21:24:33.986 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 21:24:33.986 | INFO | ------------------------------------------ | 20260505 21:24:33.986 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:24:33.986 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 21:24:33.986 | INFO | Total mem usage: 25.6 GiB | 20260505 21:24:33.986 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 21:24:33.986 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 21:24:33.986 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 21:24:33.986 | INFO | High Water Breakdown: | 20260505 21:24:33.986 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 21:24:33.986 | INFO | ------------------------------------------ | 20260505 21:24:33.986 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:24:33.986 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 21:24:33.986 | INFO | Total mem usage: 15.7 GiB | 20260505 21:24:33.986 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 21:24:33.987 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 21:24:33.987 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 21:24:33.987 | INFO | High Water Breakdown: | 20260505 21:24:33.987 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 21:24:33.987 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 21:24:33.987 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 21:24:33.987 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 21:24:33.987 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 21:24:33.987 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 21:24:33.987 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 21:24:33.987 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 21:24:33.987 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 21:24:33.987 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:24:33.987 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 21:24:33.987 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:24:33.987 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 21:24:33.987 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 21:24:33.987 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 21:24:33.987 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 21:24:33.987 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:24:33.987 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 21:24:33.987 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:24:33.987 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:24:33.987 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:24:33.987 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 21:24:33.987 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 21:24:33.987 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:24:33.987 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 21:24:33.987 | INFO | ------------------------------------------ | 20260505 21:24:33.987 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:24:33.987 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 21:24:33.987 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:24:33.987 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:24:33.987 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.39e+01, 1.35e+01 / 8.33e-01 | 20260505 21:24:33.987 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.03e-01, 1.01e-02 / 4.31e-03 | 20260505 21:24:33.987 | INFO | PerfReport: 11.78%, Baseline -- 3.70e+00 / 4.52e+00, 5.63e-01 / 2.12e-01 | 20260505 21:24:33.987 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.52e+00, 1.88e-01 / 1.05e-01 | 20260505 21:24:33.987 | INFO | PerfReport: 15.96%, DME -- 4.01e+01 / 4.40e+01, 3.75e+00 / 8.75e-01 | 20260505 21:24:33.987 | INFO | PerfReport: 57.14%, FrameLabeling -- 1.79e+01 / 1.93e+01, 1.47e+00 / 9.08e-01 | 20260505 21:24:33.987 | INFO | PerfReport: 7.23%, PulseAccumulating -- 2.27e+00 / 2.82e+00, 4.37e-01 / 2.06e-01 | 20260505 21:24:33.987 | INFO | PerfReport: 3.21%, Metrics -- 1.01e+00 / 1.30e+00, 2.72e-01 / 1.35e-01 | 20260505 21:24:33.987 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.50e-01 / 4.11e-01, 6.11e-02 / 4.04e-02 | 20260505 21:24:33.987 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 21:24:33.987 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:24:33.987 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:24:33.987 | INFO | PerfReport: Full Execution -- 9.91e+01 / -nan, 2.52e+01 / 0.00e+00 | 20260505 21:24:33.987 | INFO | PerfReport: 22.41%, Upload -- 2.22e+01 / 0.00e+00, 7.70e+00 / 0.00e+00 | 20260505 21:24:33.987 | INFO | PerfReport: 21.75%, Download -- 2.15e+01 / 0.00e+00, 7.66e+00 / 0.00e+00 | 20260505 21:24:33.987 | INFO | PerfReport: 31.70%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 21:24:33.987 | INFO | PerfReport: 2.29%, LockUpload -- 2.27e+00 / 0.00e+00, 8.02e+00 / 0.00e+00 | 20260505 21:24:33.987 | INFO | PerfReport: 1.30%, LockDownload -- 1.29e+00 / 0.00e+00, 2.83e+00 / 0.00e+00 | 20260505 21:24:33.987 | INFO | PerfReport: 20.53%, LockCompute -- 2.03e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 21:24:33.987 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.84e-02 / 0.00e+00 | 20260505 21:24:34.120 | INFO | Starting a round of Baz IO | 20260505 21:24:37.283 | INFO | Populating RTMetrics report | 20260505 21:24:37.283 | INFO | RTMetrics report populated | 20260505 21:24:37.283 | INFO | Opening tmp RTMetrics json file | 20260505 21:24:37.286 | INFO | Serializing RTMetrics to json | 20260505 21:24:37.290 | INFO | Serialization done, swapping tmp file now... | 20260505 21:24:37.291 | INFO | RTMetrics done | 20260505 21:24:37.371 | INFO | Finished a round of Baz IO: Wrote 1.974 GiB in 3.25 seconds | 20260505 21:24:37.371 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.58% / 0.00% / 78.42% | 20260505 21:24:37.371 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.148594% / 0% / 7.94729e-05% | 20260505 21:24:37.371 | INFO | Baz IO overall: | 20260505 21:24:37.371 | INFO | BytesWritten : 3716574003176 | 20260505 21:24:37.371 | INFO | NumEvents : 1755057571326 | 20260505 21:24:39.125 | INFO | Analyzing chunk frames = [6414336,6414848) | 20260505 21:24:39.125 | INFO | MemInfo: { | 20260505 21:24:39.125 | INFO | "gpu" : " 17 GB", | 20260505 21:24:39.125 | INFO | "huge" : "", | 20260505 21:24:39.125 | INFO | "huge_shared" : " 27 GB", | 20260505 21:24:39.125 | INFO | "malloc" : " 14 GB", | 20260505 21:24:39.125 | INFO | "pinned" : " 29 GB" | 20260505 21:24:39.125 | INFO | } | 20260505 21:24:39.125 | INFO | Perf report for the last 5.14029 seconds: | 20260505 21:24:39.126 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:24:39.126 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:39.126 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:39.126 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:39.126 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 21:24:39.126 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:39.126 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260505 21:24:39.126 | INFO | PRE_HQ : 59.9%, 1.8, 54.7ms, 0 | 20260505 21:24:39.126 | INFO | BAZWRITER : 62.7%, 3.6, 117.2ms, 0 | 20260505 21:24:39.126 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 21:24:39.126 | INFO | Uploaded 36GiB to GPU in 2.18863s (16.4486GiB/s) | 20260505 21:24:39.126 | INFO | Downloaded 38GiB to GPU in 2.2596s (16.8171GiB/s) | 20260505 21:24:44.265 | INFO | Analyzing chunk frames = [6414848,6415360) | 20260505 21:24:44.265 | INFO | MemInfo: { | 20260505 21:24:44.265 | INFO | "gpu" : " 17 GB", | 20260505 21:24:44.265 | INFO | "huge" : "", | 20260505 21:24:44.265 | INFO | "huge_shared" : " 27 GB", | 20260505 21:24:44.265 | INFO | "malloc" : " 14 GB", | 20260505 21:24:44.265 | INFO | "pinned" : " 29 GB" | 20260505 21:24:44.265 | INFO | } | 20260505 21:24:44.265 | INFO | Perf report for the last 5.13968 seconds: | 20260505 21:24:44.265 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:24:44.265 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:44.265 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:44.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:44.265 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 21:24:44.265 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:44.265 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:44.265 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 21:24:44.265 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:44.265 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:44.265 | INFO | Uploaded 36GiB to GPU in 2.08198s (17.2912GiB/s) | 20260505 21:24:44.265 | INFO | Downloaded 34GiB to GPU in 2.02497s (16.7904GiB/s) | 20260505 21:24:49.402 | INFO | Analyzing chunk frames = [6415360,6415872) | 20260505 21:24:49.402 | INFO | MemInfo: { | 20260505 21:24:49.402 | INFO | "gpu" : " 17 GB", | 20260505 21:24:49.402 | INFO | "huge" : "", | 20260505 21:24:49.403 | INFO | "huge_shared" : " 27 GB", | 20260505 21:24:49.403 | INFO | "malloc" : " 14 GB", | 20260505 21:24:49.403 | INFO | "pinned" : " 29 GB" | 20260505 21:24:49.403 | INFO | } | 20260505 21:24:49.403 | INFO | Perf report for the last 5.13756 seconds: | 20260505 21:24:49.403 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:24:49.403 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:49.403 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:49.403 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:49.403 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260505 21:24:49.403 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:49.403 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:49.403 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 21:24:49.403 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:49.403 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:49.403 | INFO | Uploaded 36GiB to GPU in 2.13044s (16.8979GiB/s) | 20260505 21:24:49.403 | INFO | Downloaded 34GiB to GPU in 2.01585s (16.8663GiB/s) | 20260505 21:24:54.548 | INFO | Analyzing chunk frames = [6415872,6416384) | 20260505 21:24:54.548 | INFO | MemInfo: { | 20260505 21:24:54.548 | INFO | "gpu" : " 17 GB", | 20260505 21:24:54.548 | INFO | "huge" : "", | 20260505 21:24:54.548 | INFO | "huge_shared" : " 27 GB", | 20260505 21:24:54.548 | INFO | "malloc" : " 14 GB", | 20260505 21:24:54.548 | INFO | "pinned" : " 29 GB" | 20260505 21:24:54.549 | INFO | } | 20260505 21:24:54.549 | INFO | Perf report for the last 5.14599 seconds: | 20260505 21:24:54.549 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:24:54.549 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:54.549 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:54.549 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:54.549 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260505 21:24:54.549 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:54.549 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:54.549 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:24:54.549 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:54.549 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:54.549 | INFO | Uploaded 36GiB to GPU in 2.14777s (16.7616GiB/s) | 20260505 21:24:54.549 | INFO | Downloaded 34GiB to GPU in 2.04421s (16.6323GiB/s) | 20260505 21:24:59.688 | INFO | Analyzing chunk frames = [6416384,6416896) | 20260505 21:24:59.688 | INFO | MemInfo: { | 20260505 21:24:59.688 | INFO | "gpu" : " 17 GB", | 20260505 21:24:59.688 | INFO | "huge" : "", | 20260505 21:24:59.688 | INFO | "huge_shared" : " 27 GB", | 20260505 21:24:59.688 | INFO | "malloc" : " 14 GB", | 20260505 21:24:59.688 | INFO | "pinned" : " 29 GB" | 20260505 21:24:59.688 | INFO | } | 20260505 21:24:59.688 | INFO | Perf report for the last 5.13948 seconds: | 20260505 21:24:59.688 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:24:59.688 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:59.688 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:59.688 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:59.688 | INFO | ANALYSIS : 61.6%, 3.0, 96.2ms, 0 | 20260505 21:24:59.688 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:59.688 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:59.688 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:24:59.688 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:59.688 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:24:59.688 | INFO | Uploaded 36GiB to GPU in 2.1579s (16.6829GiB/s) | 20260505 21:24:59.689 | INFO | Downloaded 34GiB to GPU in 2.0579s (16.5217GiB/s) | 20260505 21:25:04.818 | INFO | Analyzing chunk frames = [6416896,6417408) | 20260505 21:25:04.819 | INFO | MemInfo: { | 20260505 21:25:04.819 | INFO | "gpu" : " 17 GB", | 20260505 21:25:04.819 | INFO | "huge" : "", | 20260505 21:25:04.819 | INFO | "huge_shared" : " 27 GB", | 20260505 21:25:04.819 | INFO | "malloc" : " 14 GB", | 20260505 21:25:04.819 | INFO | "pinned" : " 29 GB" | 20260505 21:25:04.819 | INFO | } | 20260505 21:25:04.819 | INFO | Perf report for the last 5.13058 seconds: | 20260505 21:25:04.819 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:25:04.819 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:04.819 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:04.819 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:04.819 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260505 21:25:04.819 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:04.819 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:04.819 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:25:04.819 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:04.819 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:04.819 | INFO | Uploaded 36GiB to GPU in 2.07475s (17.3515GiB/s) | 20260505 21:25:04.819 | INFO | Downloaded 34GiB to GPU in 2.03522s (16.7058GiB/s) | 20260505 21:25:09.968 | INFO | Analyzing chunk frames = [6417408,6417920) | 20260505 21:25:09.968 | INFO | MemInfo: { | 20260505 21:25:09.968 | INFO | "gpu" : " 17 GB", | 20260505 21:25:09.968 | INFO | "huge" : "", | 20260505 21:25:09.968 | INFO | "huge_shared" : " 27 GB", | 20260505 21:25:09.968 | INFO | "malloc" : " 14 GB", | 20260505 21:25:09.968 | INFO | "pinned" : " 29 GB" | 20260505 21:25:09.968 | INFO | } | 20260505 21:25:09.968 | INFO | Perf report for the last 5.14917 seconds: | 20260505 21:25:09.968 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:25:09.968 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:09.968 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:09.968 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:09.968 | INFO | ANALYSIS : 62.9%, 3.0, 98.5ms, 0 | 20260505 21:25:09.968 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:09.968 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:09.968 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 21:25:09.968 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:09.968 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:09.968 | INFO | Uploaded 36GiB to GPU in 2.13256s (16.8811GiB/s) | 20260505 21:25:09.968 | INFO | Downloaded 34GiB to GPU in 2.14098s (15.8806GiB/s) | 20260505 21:25:15.101 | INFO | Analyzing chunk frames = [6417920,6418432) | 20260505 21:25:15.101 | INFO | MemInfo: { | 20260505 21:25:15.101 | INFO | "gpu" : " 17 GB", | 20260505 21:25:15.101 | INFO | "huge" : "", | 20260505 21:25:15.101 | INFO | "huge_shared" : " 27 GB", | 20260505 21:25:15.101 | INFO | "malloc" : " 14 GB", | 20260505 21:25:15.101 | INFO | "pinned" : " 29 GB" | 20260505 21:25:15.101 | INFO | } | 20260505 21:25:15.101 | INFO | Perf report for the last 5.13342 seconds: | 20260505 21:25:15.101 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:25:15.101 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:15.101 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:15.101 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:15.101 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 21:25:15.101 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:15.101 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:15.102 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:25:15.102 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:15.102 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:15.102 | INFO | Uploaded 36GiB to GPU in 2.11346s (17.0337GiB/s) | 20260505 21:25:15.102 | INFO | Downloaded 34GiB to GPU in 2.00837s (16.9291GiB/s) | 20260505 21:25:15.235 | INFO | Starting a round of Baz IO | 20260505 21:25:18.401 | INFO | Populating RTMetrics report | 20260505 21:25:18.401 | INFO | RTMetrics report populated | 20260505 21:25:18.401 | INFO | Opening tmp RTMetrics json file | 20260505 21:25:18.402 | INFO | Serializing RTMetrics to json | 20260505 21:25:18.405 | INFO | Serialization done, swapping tmp file now... | 20260505 21:25:18.407 | INFO | RTMetrics done | 20260505 21:25:18.487 | INFO | Finished a round of Baz IO: Wrote 1.973 GiB in 3.252 seconds | 20260505 21:25:18.487 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.58% / 0.00% / 78.42% | 20260505 21:25:18.487 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.148781% / 0% / 7.94729e-05% | 20260505 21:25:18.487 | INFO | Baz IO overall: | 20260505 21:25:18.487 | INFO | BytesWritten : 3718693005020 | 20260505 21:25:18.487 | INFO | NumEvents : 1756035948714 | 20260505 21:25:20.240 | INFO | Analyzing chunk frames = [6418432,6418944) | 20260505 21:25:20.240 | INFO | MemInfo: { | 20260505 21:25:20.240 | INFO | "gpu" : " 17 GB", | 20260505 21:25:20.240 | INFO | "huge" : "", | 20260505 21:25:20.240 | INFO | "huge_shared" : " 27 GB", | 20260505 21:25:20.240 | INFO | "malloc" : " 14 GB", | 20260505 21:25:20.240 | INFO | "pinned" : " 29 GB" | 20260505 21:25:20.240 | INFO | } | 20260505 21:25:20.240 | INFO | Perf report for the last 5.13844 seconds: | 20260505 21:25:20.240 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:25:20.240 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:20.240 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:20.240 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:20.240 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 21:25:20.240 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:20.240 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260505 21:25:20.240 | INFO | PRE_HQ : 60.2%, 1.8, 54.9ms, 0 | 20260505 21:25:20.240 | INFO | BAZWRITER : 62.8%, 3.6, 117.6ms, 0 | 20260505 21:25:20.240 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 21:25:20.240 | INFO | Uploaded 36GiB to GPU in 2.17784s (16.5302GiB/s) | 20260505 21:25:20.240 | INFO | Downloaded 38GiB to GPU in 2.25905s (16.8213GiB/s) | 20260505 21:25:25.381 | INFO | Analyzing chunk frames = [6418944,6419456) | 20260505 21:25:25.381 | INFO | MemInfo: { | 20260505 21:25:25.381 | INFO | "gpu" : " 17 GB", | 20260505 21:25:25.381 | INFO | "huge" : "", | 20260505 21:25:25.381 | INFO | "huge_shared" : " 27 GB", | 20260505 21:25:25.381 | INFO | "malloc" : " 14 GB", | 20260505 21:25:25.381 | INFO | "pinned" : " 29 GB" | 20260505 21:25:25.381 | INFO | } | 20260505 21:25:25.381 | INFO | Perf report for the last 5.14151 seconds: | 20260505 21:25:25.381 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:25:25.381 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:25.381 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:25.381 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:25.381 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 21:25:25.381 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:25.381 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:25.381 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 21:25:25.382 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:25.382 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:25.382 | INFO | Uploaded 36GiB to GPU in 2.06769s (17.4107GiB/s) | 20260505 21:25:25.382 | INFO | Downloaded 34GiB to GPU in 2.02291s (16.8075GiB/s) | 20260505 21:25:30.522 | INFO | Analyzing chunk frames = [6419456,6419968) | 20260505 21:25:30.522 | INFO | MemInfo: { | 20260505 21:25:30.522 | INFO | "gpu" : " 17 GB", | 20260505 21:25:30.522 | INFO | "huge" : "", | 20260505 21:25:30.522 | INFO | "huge_shared" : " 27 GB", | 20260505 21:25:30.522 | INFO | "malloc" : " 14 GB", | 20260505 21:25:30.522 | INFO | "pinned" : " 29 GB" | 20260505 21:25:30.522 | INFO | } | 20260505 21:25:30.522 | INFO | Perf report for the last 5.14086 seconds: | 20260505 21:25:30.522 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:25:30.522 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:30.522 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:30.522 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:30.522 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260505 21:25:30.522 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:30.522 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:30.522 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:25:30.522 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:30.522 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:30.522 | INFO | Uploaded 36GiB to GPU in 2.13546s (16.8582GiB/s) | 20260505 21:25:30.523 | INFO | Downloaded 34GiB to GPU in 2.02439s (16.7952GiB/s) | 20260505 21:25:35.664 | INFO | Analyzing chunk frames = [6419968,6420480) | 20260505 21:25:35.664 | INFO | MemInfo: { | 20260505 21:25:35.664 | INFO | "gpu" : " 17 GB", | 20260505 21:25:35.664 | INFO | "huge" : "", | 20260505 21:25:35.664 | INFO | "huge_shared" : " 27 GB", | 20260505 21:25:35.664 | INFO | "malloc" : " 14 GB", | 20260505 21:25:35.664 | INFO | "pinned" : " 29 GB" | 20260505 21:25:35.664 | INFO | } | 20260505 21:25:35.664 | INFO | Perf report for the last 5.14213 seconds: | 20260505 21:25:35.664 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:25:35.664 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:35.664 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:35.664 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:35.664 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260505 21:25:35.665 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:35.665 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:35.665 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:25:35.665 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:35.665 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:35.665 | INFO | Uploaded 36GiB to GPU in 2.14404s (16.7907GiB/s) | 20260505 21:25:35.665 | INFO | Downloaded 34GiB to GPU in 2.0329s (16.7248GiB/s) | 20260505 21:25:40.801 | INFO | Analyzing chunk frames = [6420480,6420992) | 20260505 21:25:40.801 | INFO | MemInfo: { | 20260505 21:25:40.801 | INFO | "gpu" : " 17 GB", | 20260505 21:25:40.802 | INFO | "huge" : "", | 20260505 21:25:40.802 | INFO | "huge_shared" : " 27 GB", | 20260505 21:25:40.802 | INFO | "malloc" : " 14 GB", | 20260505 21:25:40.802 | INFO | "pinned" : " 29 GB" | 20260505 21:25:40.802 | INFO | } | 20260505 21:25:40.802 | INFO | Perf report for the last 5.13729 seconds: | 20260505 21:25:40.802 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:25:40.802 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:40.802 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:40.802 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:40.802 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260505 21:25:40.802 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:40.802 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:40.802 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:25:40.802 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:40.802 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:40.802 | INFO | Uploaded 36GiB to GPU in 2.08657s (17.2532GiB/s) | 20260505 21:25:40.802 | INFO | Downloaded 34GiB to GPU in 2.03754s (16.6868GiB/s) | 20260505 21:25:45.934 | INFO | Analyzing chunk frames = [6420992,6421504) | 20260505 21:25:45.934 | INFO | MemInfo: { | 20260505 21:25:45.934 | INFO | "gpu" : " 17 GB", | 20260505 21:25:45.934 | INFO | "huge" : "", | 20260505 21:25:45.934 | INFO | "huge_shared" : " 27 GB", | 20260505 21:25:45.934 | INFO | "malloc" : " 14 GB", | 20260505 21:25:45.934 | INFO | "pinned" : " 29 GB" | 20260505 21:25:45.934 | INFO | } | 20260505 21:25:45.934 | INFO | Perf report for the last 5.1324 seconds: | 20260505 21:25:45.934 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:25:45.934 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:45.934 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:45.934 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:45.934 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260505 21:25:45.934 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:45.934 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:45.934 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 21:25:45.934 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:45.934 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:45.934 | INFO | Uploaded 36GiB to GPU in 2.07609s (17.3402GiB/s) | 20260505 21:25:45.934 | INFO | Downloaded 34GiB to GPU in 2.03537s (16.7046GiB/s) | 20260505 21:25:51.072 | INFO | Analyzing chunk frames = [6421504,6422016) | 20260505 21:25:51.072 | INFO | MemInfo: { | 20260505 21:25:51.072 | INFO | "gpu" : " 17 GB", | 20260505 21:25:51.072 | INFO | "huge" : "", | 20260505 21:25:51.072 | INFO | "huge_shared" : " 27 GB", | 20260505 21:25:51.072 | INFO | "malloc" : " 14 GB", | 20260505 21:25:51.072 | INFO | "pinned" : " 29 GB" | 20260505 21:25:51.072 | INFO | } | 20260505 21:25:51.072 | INFO | Perf report for the last 5.13828 seconds: | 20260505 21:25:51.072 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:25:51.072 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:51.072 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:51.072 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:51.072 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 21:25:51.072 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:51.073 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:51.073 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 21:25:51.073 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:51.073 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:51.073 | INFO | Uploaded 36GiB to GPU in 2.13553s (16.8576GiB/s) | 20260505 21:25:51.073 | INFO | Downloaded 34GiB to GPU in 2.12682s (15.9863GiB/s) | 20260505 21:25:56.216 | INFO | Analyzing chunk frames = [6422016,6422528) | 20260505 21:25:56.216 | INFO | MemInfo: { | 20260505 21:25:56.216 | INFO | "gpu" : " 17 GB", | 20260505 21:25:56.216 | INFO | "huge" : "", | 20260505 21:25:56.216 | INFO | "huge_shared" : " 27 GB", | 20260505 21:25:56.217 | INFO | "malloc" : " 14 GB", | 20260505 21:25:56.217 | INFO | "pinned" : " 29 GB" | 20260505 21:25:56.217 | INFO | } | 20260505 21:25:56.217 | INFO | Perf report for the last 5.14418 seconds: | 20260505 21:25:56.217 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:25:56.217 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:56.217 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:56.217 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:56.217 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 21:25:56.217 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:56.217 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:56.217 | INFO | PRE_HQ : 27.2%, 1.0, 14.2ms, 0 | 20260505 21:25:56.217 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:56.217 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:25:56.217 | INFO | Uploaded 36GiB to GPU in 2.20452s (16.3301GiB/s) | 20260505 21:25:56.217 | INFO | Downloaded 34GiB to GPU in 2.0445s (16.63GiB/s) | 20260505 21:25:56.217 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 21:25:56.217 | INFO | Total mem usage: 15.3 GiB | 20260505 21:25:56.217 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 21:25:56.217 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.16%) | 20260505 21:25:56.217 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 21:25:56.217 | INFO | High Water Breakdown: | 20260505 21:25:56.217 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 21:25:56.217 | INFO | 38.8% : 5.5 GiB ( 82.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 21:25:56.217 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 21:25:56.217 | INFO | ------------------------------------------ | 20260505 21:25:56.217 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:25:56.217 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 21:25:56.217 | INFO | Total mem usage: 26.5 GiB | 20260505 21:25:56.217 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 21:25:56.217 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 21:25:56.217 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 21:25:56.217 | INFO | High Water Breakdown: | 20260505 21:25:56.217 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:25:56.217 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:25:56.217 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:25:56.217 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:25:56.217 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:25:56.217 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:25:56.217 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 21:25:56.217 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 21:25:56.217 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:25:56.217 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 21:25:56.217 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 21:25:56.217 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 21:25:56.217 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 21:25:56.218 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 21:25:56.218 | INFO | ------------------------------------------ | 20260505 21:25:56.218 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:25:56.218 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 21:25:56.218 | INFO | Total mem usage: 25.6 GiB | 20260505 21:25:56.218 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 21:25:56.218 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 21:25:56.218 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 21:25:56.218 | INFO | High Water Breakdown: | 20260505 21:25:56.218 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 21:25:56.218 | INFO | ------------------------------------------ | 20260505 21:25:56.218 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:25:56.218 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 21:25:56.218 | INFO | Total mem usage: 15.7 GiB | 20260505 21:25:56.218 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 21:25:56.218 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 21:25:56.218 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 21:25:56.218 | INFO | High Water Breakdown: | 20260505 21:25:56.218 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 21:25:56.218 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 21:25:56.218 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 21:25:56.218 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 21:25:56.218 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 21:25:56.218 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 21:25:56.218 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 21:25:56.218 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 21:25:56.218 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 21:25:56.218 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:25:56.218 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 21:25:56.218 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:25:56.218 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 21:25:56.218 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 21:25:56.218 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 21:25:56.218 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 21:25:56.218 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:25:56.218 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 21:25:56.218 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:25:56.218 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:25:56.218 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:25:56.218 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 21:25:56.218 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 21:25:56.218 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:25:56.218 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 21:25:56.218 | INFO | ------------------------------------------ | 20260505 21:25:56.218 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:25:56.218 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 21:25:56.218 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:25:56.218 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:25:56.218 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.46e+01, 1.37e+01 / 1.86e+00 | 20260505 21:25:56.218 | INFO | PerfReport: 0.29%, AggregateStats -- 9.24e-02 / 1.03e-01, 1.06e-02 / 2.96e-03 | 20260505 21:25:56.218 | INFO | PerfReport: 11.80%, Baseline -- 3.72e+00 / 4.55e+00, 5.77e-01 / 1.31e-01 | 20260505 21:25:56.218 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.52e+00, 1.91e-01 / 8.43e-02 | 20260505 21:25:56.218 | INFO | PerfReport: 15.98%, DME -- 4.03e+01 / 4.44e+01, 4.06e+00 / 1.81e+00 | 20260505 21:25:56.218 | INFO | PerfReport: 57.21%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.58e+00 / 1.08e+00 | 20260505 21:25:56.218 | INFO | PerfReport: 7.14%, PulseAccumulating -- 2.25e+00 / 2.82e+00, 4.29e-01 / 1.81e-01 | 20260505 21:25:56.218 | INFO | PerfReport: 3.18%, Metrics -- 1.00e+00 / 1.32e+00, 2.75e-01 / 1.08e-01 | 20260505 21:25:56.218 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.50e-01 / 4.12e-01, 6.17e-02 / 2.72e-02 | 20260505 21:25:56.218 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 21:25:56.218 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:25:56.218 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:25:56.218 | INFO | PerfReport: Full Execution -- 9.91e+01 / -nan, 2.52e+01 / 0.00e+00 | 20260505 21:25:56.218 | INFO | PerfReport: 22.38%, Upload -- 2.22e+01 / 0.00e+00, 7.66e+00 / 0.00e+00 | 20260505 21:25:56.218 | INFO | PerfReport: 21.80%, Download -- 2.16e+01 / 0.00e+00, 7.69e+00 / 0.00e+00 | 20260505 21:25:56.218 | INFO | PerfReport: 31.80%, Compute -- 3.15e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260505 21:25:56.218 | INFO | PerfReport: 2.21%, LockUpload -- 2.19e+00 / 0.00e+00, 7.96e+00 / 0.00e+00 | 20260505 21:25:56.218 | INFO | PerfReport: 1.31%, LockDownload -- 1.30e+00 / 0.00e+00, 2.90e+00 / 0.00e+00 | 20260505 21:25:56.218 | INFO | PerfReport: 20.48%, LockCompute -- 2.03e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 21:25:56.218 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.23e-02 / 0.00e+00 | 20260505 21:25:56.350 | INFO | Starting a round of Baz IO | 20260505 21:25:59.510 | INFO | Populating RTMetrics report | 20260505 21:25:59.510 | INFO | RTMetrics report populated | 20260505 21:25:59.510 | INFO | Opening tmp RTMetrics json file | 20260505 21:25:59.511 | INFO | Serializing RTMetrics to json | 20260505 21:25:59.514 | INFO | Serialization done, swapping tmp file now... | 20260505 21:25:59.518 | INFO | RTMetrics done | 20260505 21:25:59.595 | INFO | Finished a round of Baz IO: Wrote 1.973 GiB in 3.244 seconds | 20260505 21:25:59.595 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.58% / 0.00% / 78.42% | 20260505 21:25:59.595 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.147796% / 3.97364e-06% / 6.75519e-05% | 20260505 21:25:59.595 | INFO | Baz IO overall: | 20260505 21:25:59.595 | INFO | BytesWritten : 3720811790870 | 20260505 21:25:59.595 | INFO | NumEvents : 1757014410472 | 20260505 21:26:01.361 | INFO | Analyzing chunk frames = [6422528,6423040) | 20260505 21:26:01.361 | INFO | MemInfo: { | 20260505 21:26:01.361 | INFO | "gpu" : " 17 GB", | 20260505 21:26:01.361 | INFO | "huge" : "", | 20260505 21:26:01.361 | INFO | "huge_shared" : " 27 GB", | 20260505 21:26:01.361 | INFO | "malloc" : " 14 GB", | 20260505 21:26:01.361 | INFO | "pinned" : " 29 GB" | 20260505 21:26:01.361 | INFO | } | 20260505 21:26:01.361 | INFO | Perf report for the last 5.14469 seconds: | 20260505 21:26:01.361 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:26:01.361 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:01.361 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:01.361 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:01.361 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 21:26:01.361 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:01.361 | INFO | RT_METRICS : 43.9%, 1.0, 23.1ms, 0 | 20260505 21:26:01.361 | INFO | PRE_HQ : 60.0%, 1.8, 55.1ms, 0 | 20260505 21:26:01.361 | INFO | BAZWRITER : 62.6%, 3.7, 118.2ms, 0 | 20260505 21:26:01.362 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 21:26:01.362 | INFO | Uploaded 36GiB to GPU in 2.14398s (16.7912GiB/s) | 20260505 21:26:01.362 | INFO | Downloaded 38GiB to GPU in 2.24031s (16.9619GiB/s) | 20260505 21:26:06.497 | INFO | Analyzing chunk frames = [6423040,6423552) | 20260505 21:26:06.497 | INFO | MemInfo: { | 20260505 21:26:06.497 | INFO | "gpu" : " 17 GB", | 20260505 21:26:06.497 | INFO | "huge" : "", | 20260505 21:26:06.497 | INFO | "huge_shared" : " 27 GB", | 20260505 21:26:06.497 | INFO | "malloc" : " 14 GB", | 20260505 21:26:06.497 | INFO | "pinned" : " 29 GB" | 20260505 21:26:06.497 | INFO | } | 20260505 21:26:06.497 | INFO | Perf report for the last 5.13609 seconds: | 20260505 21:26:06.497 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:26:06.497 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:06.497 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:06.497 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:06.497 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 21:26:06.497 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:06.498 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:06.498 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 21:26:06.498 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:06.498 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:06.498 | INFO | Uploaded 36GiB to GPU in 2.08385s (17.2757GiB/s) | 20260505 21:26:06.498 | INFO | Downloaded 34GiB to GPU in 2.02417s (16.797GiB/s) | 20260505 21:26:11.632 | INFO | Analyzing chunk frames = [6423552,6424064) | 20260505 21:26:11.632 | INFO | MemInfo: { | 20260505 21:26:11.632 | INFO | "gpu" : " 17 GB", | 20260505 21:26:11.632 | INFO | "huge" : "", | 20260505 21:26:11.632 | INFO | "huge_shared" : " 27 GB", | 20260505 21:26:11.632 | INFO | "malloc" : " 14 GB", | 20260505 21:26:11.632 | INFO | "pinned" : " 29 GB" | 20260505 21:26:11.632 | INFO | } | 20260505 21:26:11.632 | INFO | Perf report for the last 5.13442 seconds: | 20260505 21:26:11.632 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:26:11.632 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:11.632 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:11.632 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:11.632 | INFO | ANALYSIS : 62.2%, 3.0, 96.6ms, 0 | 20260505 21:26:11.632 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:11.632 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:11.632 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 21:26:11.632 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:11.632 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:11.632 | INFO | Uploaded 36GiB to GPU in 2.14405s (16.7907GiB/s) | 20260505 21:26:11.632 | INFO | Downloaded 34GiB to GPU in 2.0208s (16.825GiB/s) | 20260505 21:26:16.773 | INFO | Analyzing chunk frames = [6424064,6424576) | 20260505 21:26:16.773 | INFO | MemInfo: { | 20260505 21:26:16.773 | INFO | "gpu" : " 17 GB", | 20260505 21:26:16.773 | INFO | "huge" : "", | 20260505 21:26:16.773 | INFO | "huge_shared" : " 27 GB", | 20260505 21:26:16.773 | INFO | "malloc" : " 14 GB", | 20260505 21:26:16.773 | INFO | "pinned" : " 29 GB" | 20260505 21:26:16.773 | INFO | } | 20260505 21:26:16.773 | INFO | Perf report for the last 5.14112 seconds: | 20260505 21:26:16.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:26:16.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:16.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:16.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:16.773 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260505 21:26:16.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:16.773 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:16.773 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:26:16.773 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:16.773 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:16.773 | INFO | Uploaded 36GiB to GPU in 2.21185s (16.276GiB/s) | 20260505 21:26:16.773 | INFO | Downloaded 34GiB to GPU in 2.0493s (16.591GiB/s) | 20260505 21:26:21.913 | INFO | Analyzing chunk frames = [6424576,6425088) | 20260505 21:26:21.913 | INFO | MemInfo: { | 20260505 21:26:21.913 | INFO | "gpu" : " 17 GB", | 20260505 21:26:21.913 | INFO | "huge" : "", | 20260505 21:26:21.913 | INFO | "huge_shared" : " 27 GB", | 20260505 21:26:21.913 | INFO | "malloc" : " 14 GB", | 20260505 21:26:21.913 | INFO | "pinned" : " 29 GB" | 20260505 21:26:21.913 | INFO | } | 20260505 21:26:21.913 | INFO | Perf report for the last 5.14001 seconds: | 20260505 21:26:21.913 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:26:21.913 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:21.913 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:21.913 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:21.913 | INFO | ANALYSIS : 61.6%, 3.0, 96.2ms, 0 | 20260505 21:26:21.913 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:21.913 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:21.913 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:26:21.913 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:21.913 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:21.913 | INFO | Uploaded 36GiB to GPU in 2.13002s (16.9012GiB/s) | 20260505 21:26:21.913 | INFO | Downloaded 34GiB to GPU in 2.05845s (16.5173GiB/s) | 20260505 21:26:27.052 | INFO | Analyzing chunk frames = [6425088,6425600) | 20260505 21:26:27.052 | INFO | MemInfo: { | 20260505 21:26:27.052 | INFO | "gpu" : " 17 GB", | 20260505 21:26:27.052 | INFO | "huge" : "", | 20260505 21:26:27.052 | INFO | "huge_shared" : " 27 GB", | 20260505 21:26:27.052 | INFO | "malloc" : " 14 GB", | 20260505 21:26:27.052 | INFO | "pinned" : " 29 GB" | 20260505 21:26:27.052 | INFO | } | 20260505 21:26:27.052 | INFO | Perf report for the last 5.13883 seconds: | 20260505 21:26:27.052 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:26:27.052 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:27.052 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:27.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:27.052 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260505 21:26:27.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:27.052 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:27.052 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:26:27.052 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:27.052 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:27.052 | INFO | Uploaded 36GiB to GPU in 2.09523s (17.1819GiB/s) | 20260505 21:26:27.052 | INFO | Downloaded 34GiB to GPU in 2.03443s (16.7123GiB/s) | 20260505 21:26:32.186 | INFO | Analyzing chunk frames = [6425600,6426112) | 20260505 21:26:32.186 | INFO | MemInfo: { | 20260505 21:26:32.186 | INFO | "gpu" : " 17 GB", | 20260505 21:26:32.186 | INFO | "huge" : "", | 20260505 21:26:32.186 | INFO | "huge_shared" : " 27 GB", | 20260505 21:26:32.186 | INFO | "malloc" : " 14 GB", | 20260505 21:26:32.186 | INFO | "pinned" : " 29 GB" | 20260505 21:26:32.186 | INFO | } | 20260505 21:26:32.186 | INFO | Perf report for the last 5.13428 seconds: | 20260505 21:26:32.186 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:26:32.186 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:32.186 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:32.186 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:32.186 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 21:26:32.186 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:32.186 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:32.186 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 21:26:32.186 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:32.186 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:32.186 | INFO | Uploaded 36GiB to GPU in 2.15339s (16.7179GiB/s) | 20260505 21:26:32.186 | INFO | Downloaded 34GiB to GPU in 2.13123s (15.9532GiB/s) | 20260505 21:26:37.331 | INFO | Analyzing chunk frames = [6426112,6426624) | 20260505 21:26:37.331 | INFO | MemInfo: { | 20260505 21:26:37.331 | INFO | "gpu" : " 17 GB", | 20260505 21:26:37.331 | INFO | "huge" : "", | 20260505 21:26:37.331 | INFO | "huge_shared" : " 27 GB", | 20260505 21:26:37.331 | INFO | "malloc" : " 14 GB", | 20260505 21:26:37.331 | INFO | "pinned" : " 29 GB" | 20260505 21:26:37.331 | INFO | } | 20260505 21:26:37.331 | INFO | Perf report for the last 5.14494 seconds: | 20260505 21:26:37.331 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:26:37.331 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:37.331 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:37.331 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:37.331 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 21:26:37.331 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:37.331 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:37.331 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:26:37.331 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:37.331 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:37.331 | INFO | Uploaded 36GiB to GPU in 2.20303s (16.3412GiB/s) | 20260505 21:26:37.331 | INFO | Downloaded 34GiB to GPU in 2.04131s (16.656GiB/s) | 20260505 21:26:37.465 | INFO | Starting a round of Baz IO | 20260505 21:26:40.624 | INFO | Populating RTMetrics report | 20260505 21:26:40.624 | INFO | RTMetrics report populated | 20260505 21:26:40.624 | INFO | Opening tmp RTMetrics json file | 20260505 21:26:40.626 | INFO | Serializing RTMetrics to json | 20260505 21:26:40.629 | INFO | Serialization done, swapping tmp file now... | 20260505 21:26:40.630 | INFO | RTMetrics done | 20260505 21:26:40.706 | INFO | Finished a round of Baz IO: Wrote 1.972 GiB in 3.24 seconds | 20260505 21:26:40.706 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.58% / 0.00% / 78.42% | 20260505 21:26:40.706 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.148793% / 0% / 8.34465e-05% | 20260505 21:26:40.706 | INFO | Baz IO overall: | 20260505 21:26:40.706 | INFO | BytesWritten : 3722929342431 | 20260505 21:26:40.706 | INFO | NumEvents : 1757991977331 | 20260505 21:26:42.476 | INFO | Analyzing chunk frames = [6426624,6427136) | 20260505 21:26:42.476 | INFO | MemInfo: { | 20260505 21:26:42.476 | INFO | "gpu" : " 17 GB", | 20260505 21:26:42.476 | INFO | "huge" : "", | 20260505 21:26:42.476 | INFO | "huge_shared" : " 27 GB", | 20260505 21:26:42.476 | INFO | "malloc" : " 14 GB", | 20260505 21:26:42.476 | INFO | "pinned" : " 29 GB" | 20260505 21:26:42.476 | INFO | } | 20260505 21:26:42.476 | INFO | Perf report for the last 5.14491 seconds: | 20260505 21:26:42.476 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:26:42.476 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:42.476 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:42.476 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:42.476 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 21:26:42.476 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:42.476 | INFO | RT_METRICS : 43.7%, 1.0, 23.0ms, 0 | 20260505 21:26:42.476 | INFO | PRE_HQ : 60.0%, 1.8, 54.5ms, 0 | 20260505 21:26:42.476 | INFO | BAZWRITER : 62.5%, 3.6, 115.4ms, 0 | 20260505 21:26:42.476 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 21:26:42.476 | INFO | Uploaded 36GiB to GPU in 2.17812s (16.5281GiB/s) | 20260505 21:26:42.476 | INFO | Downloaded 38GiB to GPU in 2.25271s (16.8686GiB/s) | 20260505 21:26:47.617 | INFO | Analyzing chunk frames = [6427136,6427648) | 20260505 21:26:47.617 | INFO | MemInfo: { | 20260505 21:26:47.617 | INFO | "gpu" : " 17 GB", | 20260505 21:26:47.617 | INFO | "huge" : "", | 20260505 21:26:47.617 | INFO | "huge_shared" : " 27 GB", | 20260505 21:26:47.617 | INFO | "malloc" : " 14 GB", | 20260505 21:26:47.617 | INFO | "pinned" : " 29 GB" | 20260505 21:26:47.617 | INFO | } | 20260505 21:26:47.617 | INFO | Perf report for the last 5.14096 seconds: | 20260505 21:26:47.617 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:26:47.617 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:47.617 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:47.617 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:47.617 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 21:26:47.617 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:47.617 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:47.617 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 21:26:47.617 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:47.617 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:47.617 | INFO | Uploaded 36GiB to GPU in 2.0885s (17.2372GiB/s) | 20260505 21:26:47.617 | INFO | Downloaded 34GiB to GPU in 2.02749s (16.7695GiB/s) | 20260505 21:26:52.750 | INFO | Analyzing chunk frames = [6427648,6428160) | 20260505 21:26:52.750 | INFO | MemInfo: { | 20260505 21:26:52.750 | INFO | "gpu" : " 17 GB", | 20260505 21:26:52.750 | INFO | "huge" : "", | 20260505 21:26:52.750 | INFO | "huge_shared" : " 27 GB", | 20260505 21:26:52.750 | INFO | "malloc" : " 14 GB", | 20260505 21:26:52.750 | INFO | "pinned" : " 29 GB" | 20260505 21:26:52.750 | INFO | } | 20260505 21:26:52.750 | INFO | Perf report for the last 5.13335 seconds: | 20260505 21:26:52.750 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:26:52.750 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:52.750 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:52.750 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:52.750 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260505 21:26:52.750 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:52.750 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:52.750 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 21:26:52.750 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:52.750 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:52.751 | INFO | Uploaded 36GiB to GPU in 2.08454s (17.27GiB/s) | 20260505 21:26:52.751 | INFO | Downloaded 34GiB to GPU in 2.02612s (16.7808GiB/s) | 20260505 21:26:57.883 | INFO | Analyzing chunk frames = [6428160,6428672) | 20260505 21:26:57.883 | INFO | MemInfo: { | 20260505 21:26:57.883 | INFO | "gpu" : " 17 GB", | 20260505 21:26:57.883 | INFO | "huge" : "", | 20260505 21:26:57.883 | INFO | "huge_shared" : " 27 GB", | 20260505 21:26:57.883 | INFO | "malloc" : " 14 GB", | 20260505 21:26:57.883 | INFO | "pinned" : " 29 GB" | 20260505 21:26:57.883 | INFO | } | 20260505 21:26:57.883 | INFO | Perf report for the last 5.13264 seconds: | 20260505 21:26:57.883 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:26:57.883 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:57.883 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:57.883 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:57.883 | INFO | ANALYSIS : 62.2%, 3.0, 96.5ms, 0 | 20260505 21:26:57.883 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:57.883 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:57.883 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:26:57.883 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:57.883 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:26:57.883 | INFO | Uploaded 36GiB to GPU in 2.15142s (16.7331GiB/s) | 20260505 21:26:57.883 | INFO | Downloaded 34GiB to GPU in 2.05322s (16.5594GiB/s) | 20260505 21:27:03.032 | INFO | Analyzing chunk frames = [6428672,6429184) | 20260505 21:27:03.032 | INFO | MemInfo: { | 20260505 21:27:03.032 | INFO | "gpu" : " 17 GB", | 20260505 21:27:03.032 | INFO | "huge" : "", | 20260505 21:27:03.032 | INFO | "huge_shared" : " 27 GB", | 20260505 21:27:03.032 | INFO | "malloc" : " 14 GB", | 20260505 21:27:03.032 | INFO | "pinned" : " 29 GB" | 20260505 21:27:03.032 | INFO | } | 20260505 21:27:03.032 | INFO | Perf report for the last 5.14898 seconds: | 20260505 21:27:03.032 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:27:03.032 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:03.032 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:03.032 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:03.032 | INFO | ANALYSIS : 61.6%, 3.0, 96.3ms, 0 | 20260505 21:27:03.032 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:03.032 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:03.032 | INFO | PRE_HQ : 27.0%, 1.0, 14.1ms, 0 | 20260505 21:27:03.032 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:03.032 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:03.032 | INFO | Uploaded 36GiB to GPU in 2.13214s (16.8844GiB/s) | 20260505 21:27:03.032 | INFO | Downloaded 34GiB to GPU in 2.05814s (16.5198GiB/s) | 20260505 21:27:08.178 | INFO | Analyzing chunk frames = [6429184,6429696) | 20260505 21:27:08.178 | INFO | MemInfo: { | 20260505 21:27:08.178 | INFO | "gpu" : " 17 GB", | 20260505 21:27:08.178 | INFO | "huge" : "", | 20260505 21:27:08.178 | INFO | "huge_shared" : " 27 GB", | 20260505 21:27:08.178 | INFO | "malloc" : " 14 GB", | 20260505 21:27:08.178 | INFO | "pinned" : " 29 GB" | 20260505 21:27:08.178 | INFO | } | 20260505 21:27:08.178 | INFO | Perf report for the last 5.1464 seconds: | 20260505 21:27:08.178 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:27:08.178 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:08.178 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:08.178 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:08.178 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260505 21:27:08.178 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:08.178 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:08.178 | INFO | PRE_HQ : 26.8%, 1.0, 14.0ms, 0 | 20260505 21:27:08.179 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:08.179 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:08.179 | INFO | Uploaded 36GiB to GPU in 2.08116s (17.2981GiB/s) | 20260505 21:27:08.179 | INFO | Downloaded 34GiB to GPU in 2.03215s (16.731GiB/s) | 20260505 21:27:13.310 | INFO | Analyzing chunk frames = [6429696,6430208) | 20260505 21:27:13.310 | INFO | MemInfo: { | 20260505 21:27:13.310 | INFO | "gpu" : " 17 GB", | 20260505 21:27:13.310 | INFO | "huge" : "", | 20260505 21:27:13.310 | INFO | "huge_shared" : " 27 GB", | 20260505 21:27:13.310 | INFO | "malloc" : " 14 GB", | 20260505 21:27:13.311 | INFO | "pinned" : " 29 GB" | 20260505 21:27:13.311 | INFO | } | 20260505 21:27:13.311 | INFO | Perf report for the last 5.13219 seconds: | 20260505 21:27:13.311 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:27:13.311 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:13.311 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:13.311 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:13.311 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 21:27:13.311 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:13.311 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:13.311 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 21:27:13.311 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:13.311 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:13.311 | INFO | Uploaded 36GiB to GPU in 2.11688s (17.0062GiB/s) | 20260505 21:27:13.311 | INFO | Downloaded 34GiB to GPU in 2.08752s (16.2872GiB/s) | 20260505 21:27:18.447 | INFO | Analyzing chunk frames = [6430208,6430720) | 20260505 21:27:18.447 | INFO | MemInfo: { | 20260505 21:27:18.447 | INFO | "gpu" : " 17 GB", | 20260505 21:27:18.447 | INFO | "huge" : "", | 20260505 21:27:18.447 | INFO | "huge_shared" : " 27 GB", | 20260505 21:27:18.447 | INFO | "malloc" : " 14 GB", | 20260505 21:27:18.447 | INFO | "pinned" : " 29 GB" | 20260505 21:27:18.447 | INFO | } | 20260505 21:27:18.448 | INFO | Perf report for the last 5.13695 seconds: | 20260505 21:27:18.448 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:27:18.448 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:18.448 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:18.448 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:18.448 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260505 21:27:18.448 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:18.448 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:18.448 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260505 21:27:18.448 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:18.448 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:18.448 | INFO | Uploaded 36GiB to GPU in 2.11386s (17.0305GiB/s) | 20260505 21:27:18.448 | INFO | Downloaded 34GiB to GPU in 2.01519s (16.8719GiB/s) | 20260505 21:27:18.448 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 21:27:18.448 | INFO | Total mem usage: 15.3 GiB | 20260505 21:27:18.448 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 21:27:18.448 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.16%) | 20260505 21:27:18.448 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 21:27:18.448 | INFO | High Water Breakdown: | 20260505 21:27:18.448 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 21:27:18.448 | INFO | 38.8% : 5.5 GiB ( 82.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 21:27:18.448 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 21:27:18.448 | INFO | ------------------------------------------ | 20260505 21:27:18.448 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:27:18.448 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 21:27:18.448 | INFO | Total mem usage: 26.5 GiB | 20260505 21:27:18.448 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 21:27:18.448 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 21:27:18.448 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 21:27:18.448 | INFO | High Water Breakdown: | 20260505 21:27:18.448 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:27:18.448 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:27:18.448 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:27:18.448 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:27:18.448 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:27:18.449 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:27:18.449 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 21:27:18.449 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 21:27:18.449 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:27:18.449 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 21:27:18.449 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 21:27:18.449 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 21:27:18.449 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 21:27:18.449 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 21:27:18.449 | INFO | ------------------------------------------ | 20260505 21:27:18.449 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:27:18.449 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 21:27:18.449 | INFO | Total mem usage: 25.6 GiB | 20260505 21:27:18.449 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 21:27:18.449 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 21:27:18.449 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 21:27:18.449 | INFO | High Water Breakdown: | 20260505 21:27:18.449 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 21:27:18.449 | INFO | ------------------------------------------ | 20260505 21:27:18.449 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:27:18.449 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 21:27:18.449 | INFO | Total mem usage: 15.7 GiB | 20260505 21:27:18.449 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 21:27:18.449 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 21:27:18.449 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 21:27:18.449 | INFO | High Water Breakdown: | 20260505 21:27:18.449 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 21:27:18.449 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 21:27:18.449 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 21:27:18.449 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 21:27:18.449 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 21:27:18.449 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 21:27:18.449 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 21:27:18.449 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 21:27:18.449 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 21:27:18.449 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:27:18.449 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 21:27:18.449 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:27:18.449 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 21:27:18.449 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 21:27:18.449 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 21:27:18.449 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 21:27:18.449 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:27:18.449 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 21:27:18.449 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:27:18.449 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:27:18.449 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:27:18.449 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 21:27:18.449 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 21:27:18.449 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:27:18.449 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 21:27:18.449 | INFO | ------------------------------------------ | 20260505 21:27:18.449 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:27:18.449 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 21:27:18.449 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:27:18.449 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:27:18.449 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.42e+01, 1.33e+01 / 1.07e+00 | 20260505 21:27:18.449 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.03e-01, 1.04e-02 / 4.33e-03 | 20260505 21:27:18.449 | INFO | PerfReport: 11.79%, Baseline -- 3.71e+00 / 4.53e+00, 5.64e-01 / 1.85e-01 | 20260505 21:27:18.449 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.52e+00, 1.90e-01 / 9.59e-02 | 20260505 21:27:18.449 | INFO | PerfReport: 15.94%, DME -- 4.01e+01 / 4.40e+01, 3.93e+00 / 1.23e+00 | 20260505 21:27:18.449 | INFO | PerfReport: 57.22%, FrameLabeling -- 1.80e+01 / 1.96e+01, 1.57e+00 / 1.29e+00 | 20260505 21:27:18.449 | INFO | PerfReport: 7.17%, PulseAccumulating -- 2.25e+00 / 2.77e+00, 4.27e-01 / 2.75e-01 | 20260505 21:27:18.449 | INFO | PerfReport: 3.20%, Metrics -- 1.01e+00 / 1.32e+00, 2.76e-01 / 1.30e-01 | 20260505 21:27:18.449 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.50e-01 / 4.04e-01, 6.12e-02 / 3.31e-02 | 20260505 21:27:18.449 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 21:27:18.449 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:27:18.449 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:27:18.449 | INFO | PerfReport: Full Execution -- 9.91e+01 / -nan, 2.52e+01 / 0.00e+00 | 20260505 21:27:18.449 | INFO | PerfReport: 22.42%, Upload -- 2.22e+01 / 0.00e+00, 7.67e+00 / 0.00e+00 | 20260505 21:27:18.449 | INFO | PerfReport: 21.79%, Download -- 2.16e+01 / 0.00e+00, 7.65e+00 / 0.00e+00 | 20260505 21:27:18.449 | INFO | PerfReport: 31.76%, Compute -- 3.15e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260505 21:27:18.449 | INFO | PerfReport: 2.23%, LockUpload -- 2.21e+00 / 0.00e+00, 7.94e+00 / 0.00e+00 | 20260505 21:27:18.449 | INFO | PerfReport: 1.31%, LockDownload -- 1.30e+00 / 0.00e+00, 2.84e+00 / 0.00e+00 | 20260505 21:27:18.449 | INFO | PerfReport: 20.49%, LockCompute -- 2.03e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260505 21:27:18.449 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.00e-02 / 0.00e+00 | 20260505 21:27:18.582 | INFO | Starting a round of Baz IO | 20260505 21:27:21.751 | INFO | Populating RTMetrics report | 20260505 21:27:21.752 | INFO | RTMetrics report populated | 20260505 21:27:21.752 | INFO | Opening tmp RTMetrics json file | 20260505 21:27:21.753 | INFO | Serializing RTMetrics to json | 20260505 21:27:21.754 | INFO | Serialization done, swapping tmp file now... | 20260505 21:27:21.754 | INFO | RTMetrics done | 20260505 21:27:21.835 | INFO | Finished a round of Baz IO: Wrote 1.971 GiB in 3.252 seconds | 20260505 21:27:21.835 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.58% / 0.00% / 78.42% | 20260505 21:27:21.835 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.148221% / 3.97364e-06% / 7.94729e-05% | 20260505 21:27:21.835 | INFO | Baz IO overall: | 20260505 21:27:21.835 | INFO | BytesWritten : 3725046024129 | 20260505 21:27:21.835 | INFO | NumEvents : 1758968959084 | 20260505 21:27:23.581 | INFO | Analyzing chunk frames = [6430720,6431232) | 20260505 21:27:23.581 | INFO | MemInfo: { | 20260505 21:27:23.581 | INFO | "gpu" : " 17 GB", | 20260505 21:27:23.581 | INFO | "huge" : "", | 20260505 21:27:23.581 | INFO | "huge_shared" : " 27 GB", | 20260505 21:27:23.581 | INFO | "malloc" : " 14 GB", | 20260505 21:27:23.581 | INFO | "pinned" : " 29 GB" | 20260505 21:27:23.581 | INFO | } | 20260505 21:27:23.581 | INFO | Perf report for the last 5.13349 seconds: | 20260505 21:27:23.581 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:27:23.581 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:23.581 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:23.581 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:23.581 | INFO | ANALYSIS : 63.3%, 3.0, 98.6ms, 0 | 20260505 21:27:23.581 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:23.581 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260505 21:27:23.581 | INFO | PRE_HQ : 59.9%, 1.8, 54.6ms, 0 | 20260505 21:27:23.581 | INFO | BAZWRITER : 62.8%, 3.6, 116.9ms, 0 | 20260505 21:27:23.581 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 21:27:23.581 | INFO | Uploaded 36GiB to GPU in 2.19556s (16.3967GiB/s) | 20260505 21:27:23.581 | INFO | Downloaded 38GiB to GPU in 2.26545s (16.7737GiB/s) | 20260505 21:27:28.726 | INFO | Analyzing chunk frames = [6431232,6431744) | 20260505 21:27:28.726 | INFO | MemInfo: { | 20260505 21:27:28.726 | INFO | "gpu" : " 17 GB", | 20260505 21:27:28.726 | INFO | "huge" : "", | 20260505 21:27:28.726 | INFO | "huge_shared" : " 27 GB", | 20260505 21:27:28.726 | INFO | "malloc" : " 14 GB", | 20260505 21:27:28.726 | INFO | "pinned" : " 29 GB" | 20260505 21:27:28.726 | INFO | } | 20260505 21:27:28.726 | INFO | Perf report for the last 5.14516 seconds: | 20260505 21:27:28.726 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:27:28.726 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:28.726 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:28.726 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:28.726 | INFO | ANALYSIS : 62.0%, 3.0, 97.0ms, 0 | 20260505 21:27:28.726 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:28.726 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:28.726 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 21:27:28.726 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:28.726 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:28.727 | INFO | Uploaded 36GiB to GPU in 2.06974s (17.3935GiB/s) | 20260505 21:27:28.727 | INFO | Downloaded 34GiB to GPU in 2.0206s (16.8267GiB/s) | 20260505 21:27:33.866 | INFO | Analyzing chunk frames = [6431744,6432256) | 20260505 21:27:33.866 | INFO | MemInfo: { | 20260505 21:27:33.866 | INFO | "gpu" : " 17 GB", | 20260505 21:27:33.866 | INFO | "huge" : "", | 20260505 21:27:33.866 | INFO | "huge_shared" : " 27 GB", | 20260505 21:27:33.866 | INFO | "malloc" : " 14 GB", | 20260505 21:27:33.866 | INFO | "pinned" : " 29 GB" | 20260505 21:27:33.866 | INFO | } | 20260505 21:27:33.866 | INFO | Perf report for the last 5.13981 seconds: | 20260505 21:27:33.866 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:27:33.866 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:33.866 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:33.866 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:33.866 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260505 21:27:33.866 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:33.866 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:33.866 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:27:33.866 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:33.866 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:33.866 | INFO | Uploaded 36GiB to GPU in 2.14059s (16.8178GiB/s) | 20260505 21:27:33.866 | INFO | Downloaded 34GiB to GPU in 2.03925s (16.6728GiB/s) | 20260505 21:27:39.004 | INFO | Analyzing chunk frames = [6432256,6432768) | 20260505 21:27:39.004 | INFO | MemInfo: { | 20260505 21:27:39.004 | INFO | "gpu" : " 17 GB", | 20260505 21:27:39.004 | INFO | "huge" : "", | 20260505 21:27:39.004 | INFO | "huge_shared" : " 27 GB", | 20260505 21:27:39.004 | INFO | "malloc" : " 14 GB", | 20260505 21:27:39.004 | INFO | "pinned" : " 29 GB" | 20260505 21:27:39.004 | INFO | } | 20260505 21:27:39.004 | INFO | Perf report for the last 5.13794 seconds: | 20260505 21:27:39.004 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:27:39.004 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:39.004 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:39.004 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:39.004 | INFO | ANALYSIS : 62.4%, 3.0, 96.8ms, 0 | 20260505 21:27:39.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:39.004 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:39.004 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:27:39.004 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:39.004 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:39.004 | INFO | Uploaded 36GiB to GPU in 2.20568s (16.3215GiB/s) | 20260505 21:27:39.004 | INFO | Downloaded 34GiB to GPU in 2.07712s (16.3688GiB/s) | 20260505 21:27:44.139 | INFO | Analyzing chunk frames = [6432768,6433280) | 20260505 21:27:44.139 | INFO | MemInfo: { | 20260505 21:27:44.139 | INFO | "gpu" : " 17 GB", | 20260505 21:27:44.139 | INFO | "huge" : "", | 20260505 21:27:44.139 | INFO | "huge_shared" : " 27 GB", | 20260505 21:27:44.139 | INFO | "malloc" : " 14 GB", | 20260505 21:27:44.139 | INFO | "pinned" : " 29 GB" | 20260505 21:27:44.139 | INFO | } | 20260505 21:27:44.139 | INFO | Perf report for the last 5.13484 seconds: | 20260505 21:27:44.139 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:27:44.139 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:44.139 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:44.139 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:44.139 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260505 21:27:44.139 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:44.139 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:44.139 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:27:44.139 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:44.139 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:44.139 | INFO | Uploaded 36GiB to GPU in 2.12437s (16.9462GiB/s) | 20260505 21:27:44.139 | INFO | Downloaded 34GiB to GPU in 2.06175s (16.4909GiB/s) | 20260505 21:27:49.284 | INFO | Analyzing chunk frames = [6433280,6433792) | 20260505 21:27:49.284 | INFO | MemInfo: { | 20260505 21:27:49.284 | INFO | "gpu" : " 17 GB", | 20260505 21:27:49.284 | INFO | "huge" : "", | 20260505 21:27:49.284 | INFO | "huge_shared" : " 27 GB", | 20260505 21:27:49.284 | INFO | "malloc" : " 14 GB", | 20260505 21:27:49.284 | INFO | "pinned" : " 29 GB" | 20260505 21:27:49.284 | INFO | } | 20260505 21:27:49.284 | INFO | Perf report for the last 5.14518 seconds: | 20260505 21:27:49.284 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:27:49.284 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:49.284 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:49.284 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:49.284 | INFO | ANALYSIS : 62.1%, 3.0, 97.1ms, 0 | 20260505 21:27:49.284 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:49.284 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:49.284 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 21:27:49.284 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:49.284 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:49.284 | INFO | Uploaded 36GiB to GPU in 2.136s (16.8539GiB/s) | 20260505 21:27:49.284 | INFO | Downloaded 34GiB to GPU in 2.05563s (16.54GiB/s) | 20260505 21:27:54.429 | INFO | Analyzing chunk frames = [6433792,6434304) | 20260505 21:27:54.429 | INFO | MemInfo: { | 20260505 21:27:54.429 | INFO | "gpu" : " 17 GB", | 20260505 21:27:54.429 | INFO | "huge" : "", | 20260505 21:27:54.429 | INFO | "huge_shared" : " 27 GB", | 20260505 21:27:54.429 | INFO | "malloc" : " 14 GB", | 20260505 21:27:54.429 | INFO | "pinned" : " 29 GB" | 20260505 21:27:54.429 | INFO | } | 20260505 21:27:54.429 | INFO | Perf report for the last 5.14517 seconds: | 20260505 21:27:54.429 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:27:54.429 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:54.429 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:54.429 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:54.429 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 21:27:54.429 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:54.429 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:54.429 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 21:27:54.429 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:54.429 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:54.430 | INFO | Uploaded 36GiB to GPU in 2.12411s (16.9483GiB/s) | 20260505 21:27:54.430 | INFO | Downloaded 34GiB to GPU in 2.14239s (15.8701GiB/s) | 20260505 21:27:59.567 | INFO | Analyzing chunk frames = [6434304,6434816) | 20260505 21:27:59.567 | INFO | MemInfo: { | 20260505 21:27:59.567 | INFO | "gpu" : " 17 GB", | 20260505 21:27:59.567 | INFO | "huge" : "", | 20260505 21:27:59.567 | INFO | "huge_shared" : " 27 GB", | 20260505 21:27:59.567 | INFO | "malloc" : " 14 GB", | 20260505 21:27:59.567 | INFO | "pinned" : " 29 GB" | 20260505 21:27:59.567 | INFO | } | 20260505 21:27:59.567 | INFO | Perf report for the last 5.13761 seconds: | 20260505 21:27:59.567 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:27:59.567 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:59.567 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:59.567 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:59.567 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 21:27:59.567 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:59.567 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:59.567 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 21:27:59.567 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:59.567 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:27:59.567 | INFO | Uploaded 36GiB to GPU in 2.10901s (17.0696GiB/s) | 20260505 21:27:59.567 | INFO | Downloaded 34GiB to GPU in 2.00489s (16.9586GiB/s) | 20260505 21:27:59.694 | INFO | Starting a round of Baz IO | 20260505 21:28:02.843 | INFO | Populating RTMetrics report | 20260505 21:28:02.843 | INFO | RTMetrics report populated | 20260505 21:28:02.843 | INFO | Opening tmp RTMetrics json file | 20260505 21:28:02.844 | INFO | Serializing RTMetrics to json | 20260505 21:28:02.846 | INFO | Serialization done, swapping tmp file now... | 20260505 21:28:02.846 | INFO | RTMetrics done | 20260505 21:28:02.941 | INFO | Finished a round of Baz IO: Wrote 1.97 GiB in 3.246 seconds | 20260505 21:28:02.941 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.57% / 0.00% / 78.42% | 20260505 21:28:02.941 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.146945% / 0% / 7.94729e-05% | 20260505 21:28:02.941 | INFO | Baz IO overall: | 20260505 21:28:02.941 | INFO | BytesWritten : 3727161272134 | 20260505 21:28:02.941 | INFO | NumEvents : 1759945139233 | 20260505 21:28:04.699 | INFO | Analyzing chunk frames = [6434816,6435328) | 20260505 21:28:04.699 | INFO | MemInfo: { | 20260505 21:28:04.699 | INFO | "gpu" : " 17 GB", | 20260505 21:28:04.699 | INFO | "huge" : "", | 20260505 21:28:04.699 | INFO | "huge_shared" : " 27 GB", | 20260505 21:28:04.699 | INFO | "malloc" : " 14 GB", | 20260505 21:28:04.699 | INFO | "pinned" : " 29 GB" | 20260505 21:28:04.699 | INFO | } | 20260505 21:28:04.699 | INFO | Perf report for the last 5.1323 seconds: | 20260505 21:28:04.699 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:28:04.699 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:04.699 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:04.699 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:04.699 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260505 21:28:04.699 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:04.699 | INFO | RT_METRICS : 43.8%, 1.0, 22.9ms, 0 | 20260505 21:28:04.699 | INFO | PRE_HQ : 60.0%, 1.8, 54.7ms, 0 | 20260505 21:28:04.699 | INFO | BAZWRITER : 62.5%, 3.6, 116.9ms, 0 | 20260505 21:28:04.699 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 21:28:04.699 | INFO | Uploaded 36GiB to GPU in 2.16593s (16.621GiB/s) | 20260505 21:28:04.700 | INFO | Downloaded 38GiB to GPU in 2.2377s (16.9817GiB/s) | 20260505 21:28:09.834 | INFO | Analyzing chunk frames = [6435328,6435840) | 20260505 21:28:09.834 | INFO | MemInfo: { | 20260505 21:28:09.834 | INFO | "gpu" : " 17 GB", | 20260505 21:28:09.834 | INFO | "huge" : "", | 20260505 21:28:09.834 | INFO | "huge_shared" : " 27 GB", | 20260505 21:28:09.835 | INFO | "malloc" : " 14 GB", | 20260505 21:28:09.835 | INFO | "pinned" : " 29 GB" | 20260505 21:28:09.835 | INFO | } | 20260505 21:28:09.835 | INFO | Perf report for the last 5.13549 seconds: | 20260505 21:28:09.835 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:28:09.835 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:09.835 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:09.835 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:09.835 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 21:28:09.835 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:09.835 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:09.835 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 21:28:09.835 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:09.835 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:09.835 | INFO | Uploaded 36GiB to GPU in 2.0948s (17.1854GiB/s) | 20260505 21:28:09.835 | INFO | Downloaded 34GiB to GPU in 2.0239s (16.7993GiB/s) | 20260505 21:28:14.983 | INFO | Analyzing chunk frames = [6435840,6436352) | 20260505 21:28:14.983 | INFO | MemInfo: { | 20260505 21:28:14.983 | INFO | "gpu" : " 17 GB", | 20260505 21:28:14.983 | INFO | "huge" : "", | 20260505 21:28:14.983 | INFO | "huge_shared" : " 27 GB", | 20260505 21:28:14.983 | INFO | "malloc" : " 14 GB", | 20260505 21:28:14.983 | INFO | "pinned" : " 29 GB" | 20260505 21:28:14.983 | INFO | } | 20260505 21:28:14.983 | INFO | Perf report for the last 5.14868 seconds: | 20260505 21:28:14.983 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:28:14.983 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:14.983 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:14.983 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:14.983 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260505 21:28:14.983 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:14.983 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:14.983 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:28:14.984 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:14.984 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:14.984 | INFO | Uploaded 36GiB to GPU in 2.13844s (16.8347GiB/s) | 20260505 21:28:14.984 | INFO | Downloaded 34GiB to GPU in 2.03139s (16.7373GiB/s) | 20260505 21:28:20.136 | INFO | Analyzing chunk frames = [6436352,6436864) | 20260505 21:28:20.136 | INFO | MemInfo: { | 20260505 21:28:20.136 | INFO | "gpu" : " 17 GB", | 20260505 21:28:20.136 | INFO | "huge" : "", | 20260505 21:28:20.136 | INFO | "huge_shared" : " 27 GB", | 20260505 21:28:20.136 | INFO | "malloc" : " 14 GB", | 20260505 21:28:20.136 | INFO | "pinned" : " 29 GB" | 20260505 21:28:20.136 | INFO | } | 20260505 21:28:20.136 | INFO | Perf report for the last 5.15241 seconds: | 20260505 21:28:20.136 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:28:20.136 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:20.136 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:20.136 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:20.136 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260505 21:28:20.136 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:20.136 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:20.136 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:28:20.136 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:20.136 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:20.136 | INFO | Uploaded 36GiB to GPU in 2.14173s (16.8089GiB/s) | 20260505 21:28:20.136 | INFO | Downloaded 34GiB to GPU in 2.02462s (16.7933GiB/s) | 20260505 21:28:25.259 | INFO | Analyzing chunk frames = [6436864,6437376) | 20260505 21:28:25.259 | INFO | MemInfo: { | 20260505 21:28:25.259 | INFO | "gpu" : " 17 GB", | 20260505 21:28:25.259 | INFO | "huge" : "", | 20260505 21:28:25.259 | INFO | "huge_shared" : " 27 GB", | 20260505 21:28:25.259 | INFO | "malloc" : " 14 GB", | 20260505 21:28:25.259 | INFO | "pinned" : " 29 GB" | 20260505 21:28:25.259 | INFO | } | 20260505 21:28:25.259 | INFO | Perf report for the last 5.12346 seconds: | 20260505 21:28:25.259 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:28:25.259 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:25.259 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:25.259 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:25.259 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260505 21:28:25.259 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:25.259 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:25.259 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:28:25.259 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:25.259 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:25.260 | INFO | Uploaded 36GiB to GPU in 2.10031s (17.1403GiB/s) | 20260505 21:28:25.260 | INFO | Downloaded 34GiB to GPU in 2.04174s (16.6525GiB/s) | 20260505 21:28:30.394 | INFO | Analyzing chunk frames = [6437376,6437888) | 20260505 21:28:30.394 | INFO | MemInfo: { | 20260505 21:28:30.394 | INFO | "gpu" : " 17 GB", | 20260505 21:28:30.394 | INFO | "huge" : "", | 20260505 21:28:30.394 | INFO | "huge_shared" : " 27 GB", | 20260505 21:28:30.394 | INFO | "malloc" : " 14 GB", | 20260505 21:28:30.394 | INFO | "pinned" : " 29 GB" | 20260505 21:28:30.394 | INFO | } | 20260505 21:28:30.394 | INFO | Perf report for the last 5.13478 seconds: | 20260505 21:28:30.394 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:28:30.394 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:30.394 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:30.394 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:30.394 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260505 21:28:30.394 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:30.394 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:30.394 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:28:30.394 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:30.394 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:30.394 | INFO | Uploaded 36GiB to GPU in 2.16651s (16.6166GiB/s) | 20260505 21:28:30.394 | INFO | Downloaded 34GiB to GPU in 2.07107s (16.4166GiB/s) | 20260505 21:28:35.540 | INFO | Analyzing chunk frames = [6437888,6438400) | 20260505 21:28:35.540 | INFO | MemInfo: { | 20260505 21:28:35.540 | INFO | "gpu" : " 17 GB", | 20260505 21:28:35.540 | INFO | "huge" : "", | 20260505 21:28:35.540 | INFO | "huge_shared" : " 27 GB", | 20260505 21:28:35.540 | INFO | "malloc" : " 14 GB", | 20260505 21:28:35.540 | INFO | "pinned" : " 29 GB" | 20260505 21:28:35.540 | INFO | } | 20260505 21:28:35.540 | INFO | Perf report for the last 5.14615 seconds: | 20260505 21:28:35.540 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:28:35.540 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:35.540 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:35.540 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:35.540 | INFO | ANALYSIS : 62.9%, 3.0, 98.4ms, 0 | 20260505 21:28:35.540 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:35.540 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:35.540 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 21:28:35.540 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:35.540 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:35.540 | INFO | Uploaded 36GiB to GPU in 2.12748s (16.9214GiB/s) | 20260505 21:28:35.541 | INFO | Downloaded 34GiB to GPU in 2.13187s (15.9484GiB/s) | 20260505 21:28:40.676 | INFO | Analyzing chunk frames = [6438400,6438912) | 20260505 21:28:40.676 | INFO | MemInfo: { | 20260505 21:28:40.676 | INFO | "gpu" : " 17 GB", | 20260505 21:28:40.676 | INFO | "huge" : "", | 20260505 21:28:40.676 | INFO | "huge_shared" : " 27 GB", | 20260505 21:28:40.676 | INFO | "malloc" : " 14 GB", | 20260505 21:28:40.676 | INFO | "pinned" : " 29 GB" | 20260505 21:28:40.676 | INFO | } | 20260505 21:28:40.676 | INFO | Perf report for the last 5.13605 seconds: | 20260505 21:28:40.676 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:28:40.676 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:40.676 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:40.676 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:40.676 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 21:28:40.676 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:40.676 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:40.676 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:28:40.676 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:40.676 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:40.677 | INFO | Uploaded 36GiB to GPU in 2.1164s (17.01GiB/s) | 20260505 21:28:40.677 | INFO | Downloaded 34GiB to GPU in 1.99773s (17.0193GiB/s) | 20260505 21:28:40.677 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 21:28:40.677 | INFO | Total mem usage: 15.3 GiB | 20260505 21:28:40.677 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 21:28:40.677 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.15%) | 20260505 21:28:40.677 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 21:28:40.677 | INFO | High Water Breakdown: | 20260505 21:28:40.677 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 21:28:40.677 | INFO | 38.8% : 5.5 GiB ( 82.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 21:28:40.677 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 21:28:40.677 | INFO | ------------------------------------------ | 20260505 21:28:40.677 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:28:40.677 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 21:28:40.677 | INFO | Total mem usage: 26.5 GiB | 20260505 21:28:40.677 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 21:28:40.677 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 21:28:40.677 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 21:28:40.677 | INFO | High Water Breakdown: | 20260505 21:28:40.677 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:28:40.677 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:28:40.677 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:28:40.677 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:28:40.677 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:28:40.677 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:28:40.677 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 21:28:40.677 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 21:28:40.677 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:28:40.677 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 21:28:40.677 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 21:28:40.677 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 21:28:40.677 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 21:28:40.677 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 21:28:40.677 | INFO | ------------------------------------------ | 20260505 21:28:40.677 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:28:40.677 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 21:28:40.677 | INFO | Total mem usage: 25.6 GiB | 20260505 21:28:40.677 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 21:28:40.677 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 21:28:40.677 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 21:28:40.677 | INFO | High Water Breakdown: | 20260505 21:28:40.677 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 21:28:40.677 | INFO | ------------------------------------------ | 20260505 21:28:40.677 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:28:40.677 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 21:28:40.677 | INFO | Total mem usage: 15.7 GiB | 20260505 21:28:40.677 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 21:28:40.677 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 21:28:40.677 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 21:28:40.677 | INFO | High Water Breakdown: | 20260505 21:28:40.677 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 21:28:40.677 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 21:28:40.677 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 21:28:40.677 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 21:28:40.677 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 21:28:40.677 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 21:28:40.677 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 21:28:40.677 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 21:28:40.677 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 21:28:40.677 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:28:40.677 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 21:28:40.677 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:28:40.677 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 21:28:40.677 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 21:28:40.677 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 21:28:40.677 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 21:28:40.677 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:28:40.677 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 21:28:40.677 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:28:40.677 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:28:40.677 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:28:40.677 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 21:28:40.677 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 21:28:40.677 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:28:40.677 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 21:28:40.677 | INFO | ------------------------------------------ | 20260505 21:28:40.677 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:28:40.677 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 21:28:40.677 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:28:40.677 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:28:40.677 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.44e+01, 1.34e+01 / 1.95e+00 | 20260505 21:28:40.677 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.03e-01, 1.07e-02 / 5.59e-03 | 20260505 21:28:40.677 | INFO | PerfReport: 11.79%, Baseline -- 3.71e+00 / 4.53e+00, 5.67e-01 / 1.48e-01 | 20260505 21:28:40.678 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.49e+00, 1.87e-01 / 1.17e-01 | 20260505 21:28:40.678 | INFO | PerfReport: 15.99%, DME -- 4.02e+01 / 4.44e+01, 4.17e+00 / 1.98e+00 | 20260505 21:28:40.678 | INFO | PerfReport: 57.18%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.47e+00 / 1.06e+00 | 20260505 21:28:40.678 | INFO | PerfReport: 7.18%, PulseAccumulating -- 2.26e+00 / 2.80e+00, 4.34e-01 / 2.55e-01 | 20260505 21:28:40.678 | INFO | PerfReport: 3.18%, Metrics -- 1.00e+00 / 1.30e+00, 2.72e-01 / 1.17e-01 | 20260505 21:28:40.678 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.49e-01 / 4.07e-01, 6.08e-02 / 3.03e-02 | 20260505 21:28:40.678 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 21:28:40.678 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:28:40.678 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:28:40.678 | INFO | PerfReport: Full Execution -- 9.91e+01 / -nan, 2.53e+01 / 0.00e+00 | 20260505 21:28:40.678 | INFO | PerfReport: 22.44%, Upload -- 2.22e+01 / 0.00e+00, 7.69e+00 / 0.00e+00 | 20260505 21:28:40.678 | INFO | PerfReport: 21.83%, Download -- 2.16e+01 / 0.00e+00, 7.73e+00 / 0.00e+00 | 20260505 21:28:40.678 | INFO | PerfReport: 31.72%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 21:28:40.678 | INFO | PerfReport: 2.24%, LockUpload -- 2.22e+00 / 0.00e+00, 8.02e+00 / 0.00e+00 | 20260505 21:28:40.678 | INFO | PerfReport: 1.36%, LockDownload -- 1.35e+00 / 0.00e+00, 2.90e+00 / 0.00e+00 | 20260505 21:28:40.678 | INFO | PerfReport: 20.39%, LockCompute -- 2.02e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260505 21:28:40.678 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.61e-02 / 0.00e+00 | 20260505 21:28:40.808 | INFO | Starting a round of Baz IO | 20260505 21:28:43.975 | INFO | Populating RTMetrics report | 20260505 21:28:43.975 | INFO | RTMetrics report populated | 20260505 21:28:43.975 | INFO | Opening tmp RTMetrics json file | 20260505 21:28:43.978 | INFO | Serializing RTMetrics to json | 20260505 21:28:43.979 | INFO | Serialization done, swapping tmp file now... | 20260505 21:28:43.980 | INFO | RTMetrics done | 20260505 21:28:44.065 | INFO | Finished a round of Baz IO: Wrote 1.969 GiB in 3.257 seconds | 20260505 21:28:44.065 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.57% / 0.00% / 78.43% | 20260505 21:28:44.065 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.147084% / 7.94729e-06% / 0.000107288% | 20260505 21:28:44.065 | INFO | Baz IO overall: | 20260505 21:28:44.065 | INFO | BytesWritten : 3729275844297 | 20260505 21:28:44.065 | INFO | NumEvents : 1760921135183 | 20260505 21:28:45.814 | INFO | Analyzing chunk frames = [6438912,6439424) | 20260505 21:28:45.814 | INFO | MemInfo: { | 20260505 21:28:45.814 | INFO | "gpu" : " 17 GB", | 20260505 21:28:45.814 | INFO | "huge" : "", | 20260505 21:28:45.814 | INFO | "huge_shared" : " 27 GB", | 20260505 21:28:45.814 | INFO | "malloc" : " 14 GB", | 20260505 21:28:45.814 | INFO | "pinned" : " 29 GB" | 20260505 21:28:45.814 | INFO | } | 20260505 21:28:45.814 | INFO | Perf report for the last 5.13734 seconds: | 20260505 21:28:45.814 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:28:45.814 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:45.814 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:45.814 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:45.814 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 21:28:45.814 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:45.814 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260505 21:28:45.814 | INFO | PRE_HQ : 60.0%, 1.8, 55.0ms, 0 | 20260505 21:28:45.814 | INFO | BAZWRITER : 62.9%, 3.6, 116.9ms, 0 | 20260505 21:28:45.814 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 21:28:45.814 | INFO | Uploaded 36GiB to GPU in 2.17381s (16.5608GiB/s) | 20260505 21:28:45.814 | INFO | Downloaded 38GiB to GPU in 2.24353s (16.9376GiB/s) | 20260505 21:28:50.954 | INFO | Analyzing chunk frames = [6439424,6439936) | 20260505 21:28:50.954 | INFO | MemInfo: { | 20260505 21:28:50.954 | INFO | "gpu" : " 17 GB", | 20260505 21:28:50.954 | INFO | "huge" : "", | 20260505 21:28:50.954 | INFO | "huge_shared" : " 27 GB", | 20260505 21:28:50.954 | INFO | "malloc" : " 14 GB", | 20260505 21:28:50.954 | INFO | "pinned" : " 29 GB" | 20260505 21:28:50.954 | INFO | } | 20260505 21:28:50.954 | INFO | Perf report for the last 5.14076 seconds: | 20260505 21:28:50.954 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:28:50.954 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:50.954 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:50.954 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:50.954 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 21:28:50.954 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:50.955 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:50.955 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 21:28:50.955 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:50.955 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:50.955 | INFO | Uploaded 36GiB to GPU in 2.07991s (17.3084GiB/s) | 20260505 21:28:50.955 | INFO | Downloaded 34GiB to GPU in 2.02115s (16.8221GiB/s) | 20260505 21:28:56.092 | INFO | Analyzing chunk frames = [6439936,6440448) | 20260505 21:28:56.092 | INFO | MemInfo: { | 20260505 21:28:56.092 | INFO | "gpu" : " 17 GB", | 20260505 21:28:56.092 | INFO | "huge" : "", | 20260505 21:28:56.092 | INFO | "huge_shared" : " 27 GB", | 20260505 21:28:56.092 | INFO | "malloc" : " 14 GB", | 20260505 21:28:56.093 | INFO | "pinned" : " 29 GB" | 20260505 21:28:56.093 | INFO | } | 20260505 21:28:56.093 | INFO | Perf report for the last 5.13814 seconds: | 20260505 21:28:56.093 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:28:56.093 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:56.093 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:56.093 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:56.093 | INFO | ANALYSIS : 62.1%, 3.0, 96.5ms, 0 | 20260505 21:28:56.093 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:56.093 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:56.093 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 21:28:56.093 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:56.093 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:28:56.093 | INFO | Uploaded 36GiB to GPU in 2.13403s (16.8695GiB/s) | 20260505 21:28:56.093 | INFO | Downloaded 34GiB to GPU in 2.01848s (16.8443GiB/s) | 20260505 21:29:01.235 | INFO | Analyzing chunk frames = [6440448,6440960) | 20260505 21:29:01.235 | INFO | MemInfo: { | 20260505 21:29:01.235 | INFO | "gpu" : " 17 GB", | 20260505 21:29:01.235 | INFO | "huge" : "", | 20260505 21:29:01.235 | INFO | "huge_shared" : " 27 GB", | 20260505 21:29:01.235 | INFO | "malloc" : " 14 GB", | 20260505 21:29:01.235 | INFO | "pinned" : " 29 GB" | 20260505 21:29:01.235 | INFO | } | 20260505 21:29:01.235 | INFO | Perf report for the last 5.14272 seconds: | 20260505 21:29:01.235 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:29:01.235 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:01.235 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:01.235 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:01.235 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260505 21:29:01.235 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:01.235 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:01.235 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 21:29:01.235 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:01.235 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:01.236 | INFO | Uploaded 36GiB to GPU in 2.20474s (16.3284GiB/s) | 20260505 21:29:01.236 | INFO | Downloaded 34GiB to GPU in 2.04728s (16.6074GiB/s) | 20260505 21:29:06.378 | INFO | Analyzing chunk frames = [6440960,6441472) | 20260505 21:29:06.378 | INFO | MemInfo: { | 20260505 21:29:06.378 | INFO | "gpu" : " 17 GB", | 20260505 21:29:06.378 | INFO | "huge" : "", | 20260505 21:29:06.379 | INFO | "huge_shared" : " 27 GB", | 20260505 21:29:06.379 | INFO | "malloc" : " 14 GB", | 20260505 21:29:06.379 | INFO | "pinned" : " 29 GB" | 20260505 21:29:06.379 | INFO | } | 20260505 21:29:06.379 | INFO | Perf report for the last 5.14334 seconds: | 20260505 21:29:06.379 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:29:06.379 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:06.379 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:06.379 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:06.379 | INFO | ANALYSIS : 61.6%, 3.0, 96.2ms, 0 | 20260505 21:29:06.379 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:06.379 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:06.379 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:29:06.379 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:06.379 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:06.379 | INFO | Uploaded 36GiB to GPU in 2.13238s (16.8825GiB/s) | 20260505 21:29:06.379 | INFO | Downloaded 34GiB to GPU in 2.06032s (16.5023GiB/s) | 20260505 21:29:11.513 | INFO | Analyzing chunk frames = [6441472,6441984) | 20260505 21:29:11.513 | INFO | MemInfo: { | 20260505 21:29:11.513 | INFO | "gpu" : " 17 GB", | 20260505 21:29:11.513 | INFO | "huge" : "", | 20260505 21:29:11.513 | INFO | "huge_shared" : " 27 GB", | 20260505 21:29:11.513 | INFO | "malloc" : " 14 GB", | 20260505 21:29:11.513 | INFO | "pinned" : " 29 GB" | 20260505 21:29:11.513 | INFO | } | 20260505 21:29:11.513 | INFO | Perf report for the last 5.13458 seconds: | 20260505 21:29:11.513 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:29:11.513 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:11.513 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:11.513 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:11.513 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260505 21:29:11.513 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:11.513 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:11.513 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 21:29:11.513 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:11.513 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:11.513 | INFO | Uploaded 36GiB to GPU in 2.07719s (17.3311GiB/s) | 20260505 21:29:11.514 | INFO | Downloaded 34GiB to GPU in 2.0385s (16.6789GiB/s) | 20260505 21:29:16.649 | INFO | Analyzing chunk frames = [6441984,6442496) | 20260505 21:29:16.649 | INFO | MemInfo: { | 20260505 21:29:16.649 | INFO | "gpu" : " 17 GB", | 20260505 21:29:16.649 | INFO | "huge" : "", | 20260505 21:29:16.649 | INFO | "huge_shared" : " 27 GB", | 20260505 21:29:16.649 | INFO | "malloc" : " 14 GB", | 20260505 21:29:16.649 | INFO | "pinned" : " 29 GB" | 20260505 21:29:16.649 | INFO | } | 20260505 21:29:16.649 | INFO | Perf report for the last 5.13623 seconds: | 20260505 21:29:16.649 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:29:16.649 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:16.649 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:16.650 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:16.650 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 21:29:16.650 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:16.650 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:16.650 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 21:29:16.650 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:16.650 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:16.650 | INFO | Uploaded 36GiB to GPU in 2.14466s (16.7859GiB/s) | 20260505 21:29:16.650 | INFO | Downloaded 34GiB to GPU in 2.14629s (15.8413GiB/s) | 20260505 21:29:21.790 | INFO | Analyzing chunk frames = [6442496,6443008) | 20260505 21:29:21.790 | INFO | MemInfo: { | 20260505 21:29:21.790 | INFO | "gpu" : " 17 GB", | 20260505 21:29:21.790 | INFO | "huge" : "", | 20260505 21:29:21.790 | INFO | "huge_shared" : " 27 GB", | 20260505 21:29:21.790 | INFO | "malloc" : " 14 GB", | 20260505 21:29:21.790 | INFO | "pinned" : " 29 GB" | 20260505 21:29:21.790 | INFO | } | 20260505 21:29:21.790 | INFO | Perf report for the last 5.1406 seconds: | 20260505 21:29:21.790 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:29:21.790 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:21.790 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:21.790 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:21.790 | INFO | ANALYSIS : 62.3%, 3.0, 97.4ms, 0 | 20260505 21:29:21.790 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:21.790 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:21.790 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 21:29:21.790 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:21.790 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:21.790 | INFO | Uploaded 36GiB to GPU in 2.11996s (16.9814GiB/s) | 20260505 21:29:21.790 | INFO | Downloaded 34GiB to GPU in 2.01403s (16.8816GiB/s) | 20260505 21:29:21.925 | INFO | Starting a round of Baz IO | 20260505 21:29:25.090 | INFO | Populating RTMetrics report | 20260505 21:29:25.090 | INFO | RTMetrics report populated | 20260505 21:29:25.090 | INFO | Opening tmp RTMetrics json file | 20260505 21:29:25.091 | INFO | Serializing RTMetrics to json | 20260505 21:29:25.092 | INFO | Serialization done, swapping tmp file now... | 20260505 21:29:25.093 | INFO | RTMetrics done | 20260505 21:29:25.177 | INFO | Finished a round of Baz IO: Wrote 1.969 GiB in 3.251 seconds | 20260505 21:29:25.177 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.57% / 0.00% / 78.43% | 20260505 21:29:25.177 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.146715% / 3.97364e-06% / 9.93411e-05% | 20260505 21:29:25.177 | INFO | Baz IO overall: | 20260505 21:29:25.177 | INFO | BytesWritten : 3731389740972 | 20260505 21:29:25.177 | INFO | NumEvents : 1761896926069 | 20260505 21:29:26.933 | INFO | Analyzing chunk frames = [6443008,6443520) | 20260505 21:29:26.933 | INFO | MemInfo: { | 20260505 21:29:26.933 | INFO | "gpu" : " 17 GB", | 20260505 21:29:26.933 | INFO | "huge" : "", | 20260505 21:29:26.933 | INFO | "huge_shared" : " 27 GB", | 20260505 21:29:26.933 | INFO | "malloc" : " 14 GB", | 20260505 21:29:26.933 | INFO | "pinned" : " 29 GB" | 20260505 21:29:26.933 | INFO | } | 20260505 21:29:26.933 | INFO | Perf report for the last 5.1429 seconds: | 20260505 21:29:26.933 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:29:26.933 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:26.933 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:26.933 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:26.933 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 21:29:26.933 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:26.933 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260505 21:29:26.933 | INFO | PRE_HQ : 59.9%, 1.8, 54.8ms, 0 | 20260505 21:29:26.933 | INFO | BAZWRITER : 62.7%, 3.6, 117.1ms, 0 | 20260505 21:29:26.933 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 21:29:26.933 | INFO | Uploaded 36GiB to GPU in 2.1724s (16.5715GiB/s) | 20260505 21:29:26.933 | INFO | Downloaded 38GiB to GPU in 2.24501s (16.9264GiB/s) | 20260505 21:29:32.077 | INFO | Analyzing chunk frames = [6443520,6444032) | 20260505 21:29:32.077 | INFO | MemInfo: { | 20260505 21:29:32.077 | INFO | "gpu" : " 17 GB", | 20260505 21:29:32.078 | INFO | "huge" : "", | 20260505 21:29:32.078 | INFO | "huge_shared" : " 27 GB", | 20260505 21:29:32.078 | INFO | "malloc" : " 14 GB", | 20260505 21:29:32.078 | INFO | "pinned" : " 29 GB" | 20260505 21:29:32.078 | INFO | } | 20260505 21:29:32.078 | INFO | Perf report for the last 5.14473 seconds: | 20260505 21:29:32.078 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:29:32.078 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:32.078 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:32.078 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:32.078 | INFO | ANALYSIS : 61.9%, 3.0, 96.8ms, 0 | 20260505 21:29:32.078 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:32.078 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:32.078 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 21:29:32.078 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:32.078 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:32.078 | INFO | Uploaded 36GiB to GPU in 2.06111s (17.4663GiB/s) | 20260505 21:29:32.078 | INFO | Downloaded 34GiB to GPU in 2.01596s (16.8654GiB/s) | 20260505 21:29:37.206 | INFO | Analyzing chunk frames = [6444032,6444544) | 20260505 21:29:37.206 | INFO | MemInfo: { | 20260505 21:29:37.206 | INFO | "gpu" : " 17 GB", | 20260505 21:29:37.206 | INFO | "huge" : "", | 20260505 21:29:37.206 | INFO | "huge_shared" : " 27 GB", | 20260505 21:29:37.206 | INFO | "malloc" : " 14 GB", | 20260505 21:29:37.207 | INFO | "pinned" : " 29 GB" | 20260505 21:29:37.207 | INFO | } | 20260505 21:29:37.207 | INFO | Perf report for the last 5.12887 seconds: | 20260505 21:29:37.207 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:29:37.207 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:37.207 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:37.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:37.207 | INFO | ANALYSIS : 62.1%, 3.0, 96.5ms, 0 | 20260505 21:29:37.207 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:37.207 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:37.207 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:29:37.207 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:37.207 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:37.207 | INFO | Uploaded 36GiB to GPU in 2.13983s (16.8238GiB/s) | 20260505 21:29:37.207 | INFO | Downloaded 34GiB to GPU in 2.02853s (16.7609GiB/s) | 20260505 21:29:42.344 | INFO | Analyzing chunk frames = [6444544,6445056) | 20260505 21:29:42.344 | INFO | MemInfo: { | 20260505 21:29:42.344 | INFO | "gpu" : " 17 GB", | 20260505 21:29:42.344 | INFO | "huge" : "", | 20260505 21:29:42.344 | INFO | "huge_shared" : " 27 GB", | 20260505 21:29:42.344 | INFO | "malloc" : " 14 GB", | 20260505 21:29:42.344 | INFO | "pinned" : " 29 GB" | 20260505 21:29:42.344 | INFO | } | 20260505 21:29:42.344 | INFO | Perf report for the last 5.13758 seconds: | 20260505 21:29:42.344 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:29:42.344 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:42.344 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:42.344 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:42.344 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260505 21:29:42.344 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:42.344 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:42.344 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:29:42.344 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:42.344 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:42.344 | INFO | Uploaded 36GiB to GPU in 2.20724s (16.31GiB/s) | 20260505 21:29:42.345 | INFO | Downloaded 34GiB to GPU in 2.06719s (16.4475GiB/s) | 20260505 21:29:47.491 | INFO | Analyzing chunk frames = [6445056,6445568) | 20260505 21:29:47.491 | INFO | MemInfo: { | 20260505 21:29:47.491 | INFO | "gpu" : " 17 GB", | 20260505 21:29:47.491 | INFO | "huge" : "", | 20260505 21:29:47.492 | INFO | "huge_shared" : " 27 GB", | 20260505 21:29:47.492 | INFO | "malloc" : " 14 GB", | 20260505 21:29:47.492 | INFO | "pinned" : " 29 GB" | 20260505 21:29:47.492 | INFO | } | 20260505 21:29:47.492 | INFO | Perf report for the last 5.14744 seconds: | 20260505 21:29:47.492 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:29:47.492 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:47.492 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:47.492 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:47.492 | INFO | ANALYSIS : 61.5%, 3.0, 96.2ms, 0 | 20260505 21:29:47.492 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:47.492 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:47.492 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:29:47.492 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:47.492 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:47.492 | INFO | Uploaded 36GiB to GPU in 2.13454s (16.8654GiB/s) | 20260505 21:29:47.492 | INFO | Downloaded 34GiB to GPU in 2.06399s (16.4729GiB/s) | 20260505 21:29:52.628 | INFO | Analyzing chunk frames = [6445568,6446080) | 20260505 21:29:52.628 | INFO | MemInfo: { | 20260505 21:29:52.628 | INFO | "gpu" : " 17 GB", | 20260505 21:29:52.628 | INFO | "huge" : "", | 20260505 21:29:52.628 | INFO | "huge_shared" : " 27 GB", | 20260505 21:29:52.628 | INFO | "malloc" : " 14 GB", | 20260505 21:29:52.628 | INFO | "pinned" : " 29 GB" | 20260505 21:29:52.629 | INFO | } | 20260505 21:29:52.629 | INFO | Perf report for the last 5.13696 seconds: | 20260505 21:29:52.629 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:29:52.629 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:52.629 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:52.629 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:52.629 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260505 21:29:52.629 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:52.629 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:52.629 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:29:52.629 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:52.629 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:52.629 | INFO | Uploaded 36GiB to GPU in 2.07986s (17.3088GiB/s) | 20260505 21:29:52.629 | INFO | Downloaded 34GiB to GPU in 2.03592s (16.7001GiB/s) | 20260505 21:29:57.763 | INFO | Analyzing chunk frames = [6446080,6446592) | 20260505 21:29:57.763 | INFO | MemInfo: { | 20260505 21:29:57.763 | INFO | "gpu" : " 17 GB", | 20260505 21:29:57.763 | INFO | "huge" : "", | 20260505 21:29:57.763 | INFO | "huge_shared" : " 27 GB", | 20260505 21:29:57.763 | INFO | "malloc" : " 14 GB", | 20260505 21:29:57.763 | INFO | "pinned" : " 29 GB" | 20260505 21:29:57.763 | INFO | } | 20260505 21:29:57.763 | INFO | Perf report for the last 5.13478 seconds: | 20260505 21:29:57.763 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:29:57.763 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:57.763 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:57.763 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:57.764 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 21:29:57.764 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:57.764 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:57.764 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 21:29:57.764 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:57.764 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:29:57.764 | INFO | Uploaded 36GiB to GPU in 2.15709s (16.6892GiB/s) | 20260505 21:29:57.764 | INFO | Downloaded 34GiB to GPU in 2.12447s (16.004GiB/s) | 20260505 21:30:02.903 | INFO | Analyzing chunk frames = [6446592,6447104) | 20260505 21:30:02.904 | INFO | MemInfo: { | 20260505 21:30:02.904 | INFO | "gpu" : " 17 GB", | 20260505 21:30:02.904 | INFO | "huge" : "", | 20260505 21:30:02.904 | INFO | "huge_shared" : " 27 GB", | 20260505 21:30:02.904 | INFO | "malloc" : " 14 GB", | 20260505 21:30:02.904 | INFO | "pinned" : " 29 GB" | 20260505 21:30:02.904 | INFO | } | 20260505 21:30:02.904 | INFO | Perf report for the last 5.14032 seconds: | 20260505 21:30:02.904 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:30:02.904 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:02.904 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:02.904 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:02.904 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 21:30:02.904 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:02.904 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:02.904 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:30:02.904 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:02.904 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:02.904 | INFO | Uploaded 36GiB to GPU in 2.20414s (16.3329GiB/s) | 20260505 21:30:02.904 | INFO | Downloaded 34GiB to GPU in 2.04857s (16.5969GiB/s) | 20260505 21:30:02.904 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 21:30:02.904 | INFO | Total mem usage: 15.3 GiB | 20260505 21:30:02.904 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 21:30:02.904 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.17%) | 20260505 21:30:02.904 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 21:30:02.904 | INFO | High Water Breakdown: | 20260505 21:30:02.904 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 21:30:02.904 | INFO | 38.8% : 5.5 GiB ( 82.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 21:30:02.904 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 21:30:02.904 | INFO | ------------------------------------------ | 20260505 21:30:02.904 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:30:02.904 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 21:30:02.904 | INFO | Total mem usage: 26.5 GiB | 20260505 21:30:02.904 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 21:30:02.905 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 21:30:02.905 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 21:30:02.905 | INFO | High Water Breakdown: | 20260505 21:30:02.905 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:30:02.905 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:30:02.905 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:30:02.905 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:30:02.905 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:30:02.905 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:30:02.905 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 21:30:02.905 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 21:30:02.905 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:30:02.905 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 21:30:02.905 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 21:30:02.905 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 21:30:02.905 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 21:30:02.905 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 21:30:02.905 | INFO | ------------------------------------------ | 20260505 21:30:02.905 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:30:02.905 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 21:30:02.905 | INFO | Total mem usage: 25.6 GiB | 20260505 21:30:02.905 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 21:30:02.905 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 21:30:02.905 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 21:30:02.905 | INFO | High Water Breakdown: | 20260505 21:30:02.905 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 21:30:02.905 | INFO | ------------------------------------------ | 20260505 21:30:02.905 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:30:02.905 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 21:30:02.905 | INFO | Total mem usage: 15.7 GiB | 20260505 21:30:02.905 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 21:30:02.905 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 21:30:02.905 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 21:30:02.905 | INFO | High Water Breakdown: | 20260505 21:30:02.905 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 21:30:02.905 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 21:30:02.905 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 21:30:02.905 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 21:30:02.905 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 21:30:02.905 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 21:30:02.905 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 21:30:02.905 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 21:30:02.905 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 21:30:02.905 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:30:02.905 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 21:30:02.905 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:30:02.905 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 21:30:02.905 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 21:30:02.905 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 21:30:02.905 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 21:30:02.905 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:30:02.905 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 21:30:02.905 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:30:02.905 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:30:02.905 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:30:02.905 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 21:30:02.905 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 21:30:02.905 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:30:02.905 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 21:30:02.905 | INFO | ------------------------------------------ | 20260505 21:30:02.905 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:30:02.905 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 21:30:02.905 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:30:02.905 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:30:02.905 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.43e+01, 1.31e+01 / 1.34e+00 | 20260505 21:30:02.905 | INFO | PerfReport: 0.29%, AggregateStats -- 9.24e-02 / 1.05e-01, 1.17e-02 / 1.02e-02 | 20260505 21:30:02.905 | INFO | PerfReport: 11.79%, Baseline -- 3.70e+00 / 4.57e+00, 5.59e-01 / 1.30e-01 | 20260505 21:30:02.905 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.50e+00, 1.84e-01 / 1.17e-01 | 20260505 21:30:02.905 | INFO | PerfReport: 15.95%, DME -- 4.01e+01 / 4.41e+01, 4.01e+00 / 1.17e+00 | 20260505 21:30:02.905 | INFO | PerfReport: 57.18%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.49e+00 / 1.13e+00 | 20260505 21:30:02.905 | INFO | PerfReport: 7.20%, PulseAccumulating -- 2.26e+00 / 2.80e+00, 4.31e-01 / 2.37e-01 | 20260505 21:30:02.905 | INFO | PerfReport: 3.20%, Metrics -- 1.01e+00 / 1.32e+00, 2.72e-01 / 1.48e-01 | 20260505 21:30:02.905 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.50e-01 / 4.11e-01, 6.09e-02 / 3.80e-02 | 20260505 21:30:02.905 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 21:30:02.905 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:30:02.905 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:30:02.905 | INFO | PerfReport: Full Execution -- 9.92e+01 / -nan, 2.52e+01 / 0.00e+00 | 20260505 21:30:02.905 | INFO | PerfReport: 22.47%, Upload -- 2.23e+01 / 0.00e+00, 7.74e+00 / 0.00e+00 | 20260505 21:30:02.905 | INFO | PerfReport: 21.81%, Download -- 2.16e+01 / 0.00e+00, 7.64e+00 / 0.00e+00 | 20260505 21:30:02.905 | INFO | PerfReport: 31.68%, Compute -- 3.14e+01 / 0.00e+00, 1.31e+01 / 0.00e+00 | 20260505 21:30:02.905 | INFO | PerfReport: 2.27%, LockUpload -- 2.25e+00 / 0.00e+00, 7.97e+00 / 0.00e+00 | 20260505 21:30:02.905 | INFO | PerfReport: 1.33%, LockDownload -- 1.32e+00 / 0.00e+00, 2.79e+00 / 0.00e+00 | 20260505 21:30:02.905 | INFO | PerfReport: 20.43%, LockCompute -- 2.03e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 21:30:02.905 | INFO | PerfReport: 0.01%, Misc -- 1.26e-02 / 0.00e+00, 1.23e-02 / 0.00e+00 | 20260505 21:30:03.040 | INFO | Starting a round of Baz IO | 20260505 21:30:06.190 | INFO | Populating RTMetrics report | 20260505 21:30:06.190 | INFO | RTMetrics report populated | 20260505 21:30:06.190 | INFO | Opening tmp RTMetrics json file | 20260505 21:30:06.191 | INFO | Serializing RTMetrics to json | 20260505 21:30:06.192 | INFO | Serialization done, swapping tmp file now... | 20260505 21:30:06.192 | INFO | RTMetrics done | 20260505 21:30:06.271 | INFO | Finished a round of Baz IO: Wrote 1.968 GiB in 3.231 seconds | 20260505 21:30:06.271 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.57% / 0.00% / 78.43% | 20260505 21:30:06.271 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.146572% / 3.97364e-06% / 8.74201e-05% | 20260505 21:30:06.271 | INFO | Baz IO overall: | 20260505 21:30:06.271 | INFO | BytesWritten : 3733502848605 | 20260505 21:30:06.271 | INFO | NumEvents : 1762872238644 | 20260505 21:30:08.044 | INFO | Analyzing chunk frames = [6447104,6447616) | 20260505 21:30:08.044 | INFO | MemInfo: { | 20260505 21:30:08.044 | INFO | "gpu" : " 17 GB", | 20260505 21:30:08.044 | INFO | "huge" : "", | 20260505 21:30:08.044 | INFO | "huge_shared" : " 27 GB", | 20260505 21:30:08.044 | INFO | "malloc" : " 14 GB", | 20260505 21:30:08.044 | INFO | "pinned" : " 29 GB" | 20260505 21:30:08.044 | INFO | } | 20260505 21:30:08.044 | INFO | Perf report for the last 5.14028 seconds: | 20260505 21:30:08.044 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:30:08.044 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:08.044 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:08.044 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:08.044 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260505 21:30:08.044 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:08.044 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260505 21:30:08.044 | INFO | PRE_HQ : 59.8%, 1.8, 54.6ms, 0 | 20260505 21:30:08.044 | INFO | BAZWRITER : 62.4%, 3.6, 116.3ms, 0 | 20260505 21:30:08.044 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 21:30:08.044 | INFO | Uploaded 36GiB to GPU in 2.1754s (16.5487GiB/s) | 20260505 21:30:08.044 | INFO | Downloaded 38GiB to GPU in 2.25521s (16.8499GiB/s) | 20260505 21:30:13.185 | INFO | Analyzing chunk frames = [6447616,6448128) | 20260505 21:30:13.185 | INFO | MemInfo: { | 20260505 21:30:13.185 | INFO | "gpu" : " 17 GB", | 20260505 21:30:13.185 | INFO | "huge" : "", | 20260505 21:30:13.185 | INFO | "huge_shared" : " 27 GB", | 20260505 21:30:13.185 | INFO | "malloc" : " 14 GB", | 20260505 21:30:13.185 | INFO | "pinned" : " 29 GB" | 20260505 21:30:13.185 | INFO | } | 20260505 21:30:13.185 | INFO | Perf report for the last 5.14141 seconds: | 20260505 21:30:13.185 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:30:13.185 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:13.186 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:13.186 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:13.186 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 21:30:13.186 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:13.186 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:13.186 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 21:30:13.186 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:13.186 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:13.186 | INFO | Uploaded 36GiB to GPU in 2.08006s (17.3072GiB/s) | 20260505 21:30:13.186 | INFO | Downloaded 34GiB to GPU in 2.0222s (16.8134GiB/s) | 20260505 21:30:18.325 | INFO | Analyzing chunk frames = [6448128,6448640) | 20260505 21:30:18.326 | INFO | MemInfo: { | 20260505 21:30:18.326 | INFO | "gpu" : " 17 GB", | 20260505 21:30:18.326 | INFO | "huge" : "", | 20260505 21:30:18.326 | INFO | "huge_shared" : " 27 GB", | 20260505 21:30:18.326 | INFO | "malloc" : " 14 GB", | 20260505 21:30:18.326 | INFO | "pinned" : " 29 GB" | 20260505 21:30:18.326 | INFO | } | 20260505 21:30:18.326 | INFO | Perf report for the last 5.14024 seconds: | 20260505 21:30:18.326 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:30:18.326 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:18.326 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:18.326 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:18.326 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260505 21:30:18.326 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:18.326 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:18.326 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 21:30:18.326 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:18.326 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:18.326 | INFO | Uploaded 36GiB to GPU in 2.14192s (16.8074GiB/s) | 20260505 21:30:18.326 | INFO | Downloaded 34GiB to GPU in 2.0237s (16.8009GiB/s) | 20260505 21:30:23.461 | INFO | Analyzing chunk frames = [6448640,6449152) | 20260505 21:30:23.461 | INFO | MemInfo: { | 20260505 21:30:23.461 | INFO | "gpu" : " 17 GB", | 20260505 21:30:23.461 | INFO | "huge" : "", | 20260505 21:30:23.461 | INFO | "huge_shared" : " 27 GB", | 20260505 21:30:23.461 | INFO | "malloc" : " 14 GB", | 20260505 21:30:23.461 | INFO | "pinned" : " 29 GB" | 20260505 21:30:23.461 | INFO | } | 20260505 21:30:23.461 | INFO | Perf report for the last 5.1351 seconds: | 20260505 21:30:23.461 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:30:23.461 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:23.461 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:23.461 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:23.461 | INFO | ANALYSIS : 62.0%, 3.0, 96.3ms, 0 | 20260505 21:30:23.461 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:23.461 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:23.461 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:30:23.461 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:23.461 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:23.461 | INFO | Uploaded 36GiB to GPU in 2.13889s (16.8311GiB/s) | 20260505 21:30:23.461 | INFO | Downloaded 34GiB to GPU in 2.04485s (16.6271GiB/s) | 20260505 21:30:28.597 | INFO | Analyzing chunk frames = [6449152,6449664) | 20260505 21:30:28.597 | INFO | MemInfo: { | 20260505 21:30:28.597 | INFO | "gpu" : " 17 GB", | 20260505 21:30:28.597 | INFO | "huge" : "", | 20260505 21:30:28.597 | INFO | "huge_shared" : " 27 GB", | 20260505 21:30:28.597 | INFO | "malloc" : " 14 GB", | 20260505 21:30:28.597 | INFO | "pinned" : " 29 GB" | 20260505 21:30:28.597 | INFO | } | 20260505 21:30:28.598 | INFO | Perf report for the last 5.13668 seconds: | 20260505 21:30:28.598 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:30:28.598 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:28.598 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:28.598 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:28.598 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260505 21:30:28.598 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:28.598 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:28.598 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:30:28.598 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:28.598 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:28.598 | INFO | Uploaded 36GiB to GPU in 2.12988s (16.9024GiB/s) | 20260505 21:30:28.598 | INFO | Downloaded 34GiB to GPU in 2.06042s (16.5015GiB/s) | 20260505 21:30:33.738 | INFO | Analyzing chunk frames = [6449664,6450176) | 20260505 21:30:33.738 | INFO | MemInfo: { | 20260505 21:30:33.738 | INFO | "gpu" : " 17 GB", | 20260505 21:30:33.738 | INFO | "huge" : "", | 20260505 21:30:33.738 | INFO | "huge_shared" : " 27 GB", | 20260505 21:30:33.738 | INFO | "malloc" : " 14 GB", | 20260505 21:30:33.739 | INFO | "pinned" : " 29 GB" | 20260505 21:30:33.739 | INFO | } | 20260505 21:30:33.739 | INFO | Perf report for the last 5.14112 seconds: | 20260505 21:30:33.739 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:30:33.739 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:33.739 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:33.739 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:33.739 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260505 21:30:33.739 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:33.739 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:33.739 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:30:33.739 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:33.739 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:33.739 | INFO | Uploaded 36GiB to GPU in 2.16739s (16.6098GiB/s) | 20260505 21:30:33.739 | INFO | Downloaded 34GiB to GPU in 2.06998s (16.4253GiB/s) | 20260505 21:30:38.883 | INFO | Analyzing chunk frames = [6450176,6450688) | 20260505 21:30:38.883 | INFO | MemInfo: { | 20260505 21:30:38.883 | INFO | "gpu" : " 17 GB", | 20260505 21:30:38.883 | INFO | "huge" : "", | 20260505 21:30:38.883 | INFO | "huge_shared" : " 27 GB", | 20260505 21:30:38.883 | INFO | "malloc" : " 14 GB", | 20260505 21:30:38.883 | INFO | "pinned" : " 29 GB" | 20260505 21:30:38.883 | INFO | } | 20260505 21:30:38.883 | INFO | Perf report for the last 5.14478 seconds: | 20260505 21:30:38.883 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:30:38.883 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:38.883 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:38.884 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:38.884 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 21:30:38.884 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:38.884 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:38.884 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 21:30:38.884 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:38.884 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:38.884 | INFO | Uploaded 36GiB to GPU in 2.15008s (16.7436GiB/s) | 20260505 21:30:38.884 | INFO | Downloaded 34GiB to GPU in 2.12864s (15.9727GiB/s) | 20260505 21:30:44.025 | INFO | Analyzing chunk frames = [6450688,6451200) | 20260505 21:30:44.025 | INFO | MemInfo: { | 20260505 21:30:44.025 | INFO | "gpu" : " 17 GB", | 20260505 21:30:44.025 | INFO | "huge" : "", | 20260505 21:30:44.025 | INFO | "huge_shared" : " 27 GB", | 20260505 21:30:44.025 | INFO | "malloc" : " 14 GB", | 20260505 21:30:44.025 | INFO | "pinned" : " 29 GB" | 20260505 21:30:44.025 | INFO | } | 20260505 21:30:44.025 | INFO | Perf report for the last 5.14136 seconds: | 20260505 21:30:44.025 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:30:44.025 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:44.025 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:44.025 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:44.025 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 21:30:44.025 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:44.025 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:44.025 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:30:44.025 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:44.025 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:44.025 | INFO | Uploaded 36GiB to GPU in 2.12459s (16.9445GiB/s) | 20260505 21:30:44.025 | INFO | Downloaded 34GiB to GPU in 2.00127s (16.9892GiB/s) | 20260505 21:30:44.157 | INFO | Starting a round of Baz IO | 20260505 21:30:47.308 | INFO | Populating RTMetrics report | 20260505 21:30:47.308 | INFO | RTMetrics report populated | 20260505 21:30:47.308 | INFO | Opening tmp RTMetrics json file | 20260505 21:30:47.309 | INFO | Serializing RTMetrics to json | 20260505 21:30:47.310 | INFO | Serialization done, swapping tmp file now... | 20260505 21:30:47.310 | INFO | RTMetrics done | 20260505 21:30:47.393 | INFO | Finished a round of Baz IO: Wrote 1.968 GiB in 3.236 seconds | 20260505 21:30:47.393 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.57% / 0.00% / 78.43% | 20260505 21:30:47.393 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.146321% / 0% / 9.13938e-05% | 20260505 21:30:47.393 | INFO | Baz IO overall: | 20260505 21:30:47.393 | INFO | BytesWritten : 3735616470813 | 20260505 21:30:47.393 | INFO | NumEvents : 1763848196899 | 20260505 21:30:49.157 | INFO | Analyzing chunk frames = [6451200,6451712) | 20260505 21:30:49.157 | INFO | MemInfo: { | 20260505 21:30:49.157 | INFO | "gpu" : " 17 GB", | 20260505 21:30:49.157 | INFO | "huge" : "", | 20260505 21:30:49.157 | INFO | "huge_shared" : " 27 GB", | 20260505 21:30:49.157 | INFO | "malloc" : " 14 GB", | 20260505 21:30:49.157 | INFO | "pinned" : " 29 GB" | 20260505 21:30:49.157 | INFO | } | 20260505 21:30:49.157 | INFO | Perf report for the last 5.13253 seconds: | 20260505 21:30:49.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:30:49.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:49.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:49.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:49.157 | INFO | ANALYSIS : 62.9%, 3.0, 98.0ms, 0 | 20260505 21:30:49.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:49.157 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260505 21:30:49.157 | INFO | PRE_HQ : 60.0%, 1.8, 54.8ms, 0 | 20260505 21:30:49.158 | INFO | BAZWRITER : 62.5%, 3.6, 116.0ms, 0 | 20260505 21:30:49.158 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 21:30:49.158 | INFO | Uploaded 36GiB to GPU in 2.17878s (16.523GiB/s) | 20260505 21:30:49.158 | INFO | Downloaded 38GiB to GPU in 2.24023s (16.9625GiB/s) | 20260505 21:30:54.302 | INFO | Analyzing chunk frames = [6451712,6452224) | 20260505 21:30:54.302 | INFO | MemInfo: { | 20260505 21:30:54.302 | INFO | "gpu" : " 17 GB", | 20260505 21:30:54.302 | INFO | "huge" : "", | 20260505 21:30:54.302 | INFO | "huge_shared" : " 27 GB", | 20260505 21:30:54.302 | INFO | "malloc" : " 14 GB", | 20260505 21:30:54.302 | INFO | "pinned" : " 29 GB" | 20260505 21:30:54.302 | INFO | } | 20260505 21:30:54.302 | INFO | Perf report for the last 5.14493 seconds: | 20260505 21:30:54.302 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:30:54.302 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:54.302 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:54.302 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:54.302 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260505 21:30:54.302 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:54.302 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:54.302 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 21:30:54.302 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:54.303 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:54.303 | INFO | Uploaded 36GiB to GPU in 2.07305s (17.3657GiB/s) | 20260505 21:30:54.303 | INFO | Downloaded 34GiB to GPU in 2.01833s (16.8456GiB/s) | 20260505 21:30:59.442 | INFO | Analyzing chunk frames = [6452224,6452736) | 20260505 21:30:59.442 | INFO | MemInfo: { | 20260505 21:30:59.442 | INFO | "gpu" : " 17 GB", | 20260505 21:30:59.442 | INFO | "huge" : "", | 20260505 21:30:59.442 | INFO | "huge_shared" : " 27 GB", | 20260505 21:30:59.442 | INFO | "malloc" : " 14 GB", | 20260505 21:30:59.442 | INFO | "pinned" : " 29 GB" | 20260505 21:30:59.442 | INFO | } | 20260505 21:30:59.442 | INFO | Perf report for the last 5.13949 seconds: | 20260505 21:30:59.442 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:30:59.442 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:59.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:59.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:59.442 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260505 21:30:59.442 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:59.442 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:59.442 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:30:59.442 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:59.442 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:30:59.442 | INFO | Uploaded 36GiB to GPU in 2.13829s (16.8359GiB/s) | 20260505 21:30:59.442 | INFO | Downloaded 34GiB to GPU in 2.01896s (16.8404GiB/s) | 20260505 21:31:04.577 | INFO | Analyzing chunk frames = [6452736,6453248) | 20260505 21:31:04.578 | INFO | MemInfo: { | 20260505 21:31:04.578 | INFO | "gpu" : " 17 GB", | 20260505 21:31:04.578 | INFO | "huge" : "", | 20260505 21:31:04.578 | INFO | "huge_shared" : " 27 GB", | 20260505 21:31:04.578 | INFO | "malloc" : " 14 GB", | 20260505 21:31:04.578 | INFO | "pinned" : " 29 GB" | 20260505 21:31:04.578 | INFO | } | 20260505 21:31:04.578 | INFO | Perf report for the last 5.13594 seconds: | 20260505 21:31:04.578 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:31:04.578 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:04.578 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:04.578 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:04.578 | INFO | ANALYSIS : 62.0%, 3.0, 96.3ms, 0 | 20260505 21:31:04.578 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:04.578 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:04.578 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:31:04.578 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:04.578 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:04.578 | INFO | Uploaded 36GiB to GPU in 2.14461s (16.7863GiB/s) | 20260505 21:31:04.578 | INFO | Downloaded 34GiB to GPU in 2.04303s (16.642GiB/s) | 20260505 21:31:09.713 | INFO | Analyzing chunk frames = [6453248,6453760) | 20260505 21:31:09.713 | INFO | MemInfo: { | 20260505 21:31:09.713 | INFO | "gpu" : " 17 GB", | 20260505 21:31:09.713 | INFO | "huge" : "", | 20260505 21:31:09.713 | INFO | "huge_shared" : " 27 GB", | 20260505 21:31:09.713 | INFO | "malloc" : " 14 GB", | 20260505 21:31:09.713 | INFO | "pinned" : " 29 GB" | 20260505 21:31:09.713 | INFO | } | 20260505 21:31:09.713 | INFO | Perf report for the last 5.13566 seconds: | 20260505 21:31:09.713 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:31:09.713 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:09.714 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:09.714 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:09.714 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260505 21:31:09.714 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:09.714 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:09.714 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:31:09.714 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:09.714 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:09.714 | INFO | Uploaded 36GiB to GPU in 2.13684s (16.8473GiB/s) | 20260505 21:31:09.714 | INFO | Downloaded 34GiB to GPU in 2.05951s (16.5088GiB/s) | 20260505 21:31:14.852 | INFO | Analyzing chunk frames = [6453760,6454272) | 20260505 21:31:14.852 | INFO | MemInfo: { | 20260505 21:31:14.852 | INFO | "gpu" : " 17 GB", | 20260505 21:31:14.852 | INFO | "huge" : "", | 20260505 21:31:14.852 | INFO | "huge_shared" : " 27 GB", | 20260505 21:31:14.852 | INFO | "malloc" : " 14 GB", | 20260505 21:31:14.852 | INFO | "pinned" : " 29 GB" | 20260505 21:31:14.852 | INFO | } | 20260505 21:31:14.852 | INFO | Perf report for the last 5.13834 seconds: | 20260505 21:31:14.852 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:31:14.852 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:14.852 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:14.852 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:14.852 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 21:31:14.852 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:14.852 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:14.852 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:31:14.852 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:14.852 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:14.852 | INFO | Uploaded 36GiB to GPU in 2.15766s (16.6847GiB/s) | 20260505 21:31:14.852 | INFO | Downloaded 34GiB to GPU in 2.06371s (16.4752GiB/s) | 20260505 21:31:19.999 | INFO | Analyzing chunk frames = [6454272,6454784) | 20260505 21:31:19.999 | INFO | MemInfo: { | 20260505 21:31:19.999 | INFO | "gpu" : " 17 GB", | 20260505 21:31:19.999 | INFO | "huge" : "", | 20260505 21:31:19.999 | INFO | "huge_shared" : " 27 GB", | 20260505 21:31:19.999 | INFO | "malloc" : " 14 GB", | 20260505 21:31:19.999 | INFO | "pinned" : " 29 GB" | 20260505 21:31:19.999 | INFO | } | 20260505 21:31:19.999 | INFO | Perf report for the last 5.14733 seconds: | 20260505 21:31:19.999 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:31:19.999 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:19.999 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:19.999 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:19.999 | INFO | ANALYSIS : 62.7%, 3.0, 98.1ms, 0 | 20260505 21:31:19.999 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:19.999 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:19.999 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 21:31:19.999 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:19.999 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:19.999 | INFO | Uploaded 36GiB to GPU in 2.13342s (16.8743GiB/s) | 20260505 21:31:19.999 | INFO | Downloaded 34GiB to GPU in 2.1506s (15.8095GiB/s) | 20260505 21:31:25.138 | INFO | Analyzing chunk frames = [6454784,6455296) | 20260505 21:31:25.138 | INFO | MemInfo: { | 20260505 21:31:25.138 | INFO | "gpu" : " 17 GB", | 20260505 21:31:25.138 | INFO | "huge" : "", | 20260505 21:31:25.138 | INFO | "huge_shared" : " 27 GB", | 20260505 21:31:25.138 | INFO | "malloc" : " 14 GB", | 20260505 21:31:25.138 | INFO | "pinned" : " 29 GB" | 20260505 21:31:25.138 | INFO | } | 20260505 21:31:25.139 | INFO | Perf report for the last 5.1394 seconds: | 20260505 21:31:25.139 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:31:25.139 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:25.139 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:25.139 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:25.139 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 21:31:25.139 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:25.139 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:25.139 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 21:31:25.139 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:25.139 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:25.139 | INFO | Uploaded 36GiB to GPU in 2.11591s (17.014GiB/s) | 20260505 21:31:25.139 | INFO | Downloaded 34GiB to GPU in 1.99725s (17.0234GiB/s) | 20260505 21:31:25.139 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 21:31:25.139 | INFO | Total mem usage: 15.3 GiB | 20260505 21:31:25.139 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 21:31:25.139 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.79%) | 20260505 21:31:25.139 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 21:31:25.139 | INFO | High Water Breakdown: | 20260505 21:31:25.139 | INFO | 61% : 8.59 GiB ( 90.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 21:31:25.139 | INFO | 38.9% : 5.48 GiB ( 82.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 21:31:25.139 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 21:31:25.139 | INFO | ------------------------------------------ | 20260505 21:31:25.139 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:31:25.139 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 21:31:25.139 | INFO | Total mem usage: 26.5 GiB | 20260505 21:31:25.139 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 21:31:25.139 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 21:31:25.139 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 21:31:25.139 | INFO | High Water Breakdown: | 20260505 21:31:25.139 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:31:25.139 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:31:25.139 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:31:25.139 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:31:25.139 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:31:25.139 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:31:25.140 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 21:31:25.140 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 21:31:25.140 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:31:25.140 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 21:31:25.140 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 21:31:25.140 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 21:31:25.140 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 21:31:25.140 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 21:31:25.140 | INFO | ------------------------------------------ | 20260505 21:31:25.140 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:31:25.140 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 21:31:25.140 | INFO | Total mem usage: 25.6 GiB | 20260505 21:31:25.140 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 21:31:25.140 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 21:31:25.140 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 21:31:25.140 | INFO | High Water Breakdown: | 20260505 21:31:25.140 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 21:31:25.140 | INFO | ------------------------------------------ | 20260505 21:31:25.140 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:31:25.140 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 21:31:25.140 | INFO | Total mem usage: 15.7 GiB | 20260505 21:31:25.140 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 21:31:25.140 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 21:31:25.140 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 21:31:25.140 | INFO | High Water Breakdown: | 20260505 21:31:25.140 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 21:31:25.140 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 21:31:25.140 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 21:31:25.140 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 21:31:25.140 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 21:31:25.140 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 21:31:25.140 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 21:31:25.140 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 21:31:25.140 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 21:31:25.140 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:31:25.140 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 21:31:25.140 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:31:25.140 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 21:31:25.140 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 21:31:25.140 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 21:31:25.140 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 21:31:25.140 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:31:25.140 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 21:31:25.140 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:31:25.140 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:31:25.140 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:31:25.140 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 21:31:25.140 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 21:31:25.140 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:31:25.140 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 21:31:25.140 | INFO | ------------------------------------------ | 20260505 21:31:25.140 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:31:25.140 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 21:31:25.140 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:31:25.140 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:31:25.140 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.42e+01, 1.35e+01 / 1.66e+00 | 20260505 21:31:25.140 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.03e-01, 1.07e-02 / 3.08e-03 | 20260505 21:31:25.140 | INFO | PerfReport: 11.81%, Baseline -- 3.71e+00 / 4.57e+00, 5.69e-01 / 1.50e-01 | 20260505 21:31:25.140 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.52e+00, 1.87e-01 / 1.09e-01 | 20260505 21:31:25.140 | INFO | PerfReport: 15.93%, DME -- 4.01e+01 / 4.41e+01, 3.86e+00 / 1.65e+00 | 20260505 21:31:25.140 | INFO | PerfReport: 57.20%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.52e+00 / 1.24e+00 | 20260505 21:31:25.140 | INFO | PerfReport: 7.19%, PulseAccumulating -- 2.26e+00 / 2.80e+00, 4.31e-01 / 2.46e-01 | 20260505 21:31:25.140 | INFO | PerfReport: 3.19%, Metrics -- 1.00e+00 / 1.30e+00, 2.75e-01 / 1.43e-01 | 20260505 21:31:25.140 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.49e-01 / 4.06e-01, 6.08e-02 / 4.01e-02 | 20260505 21:31:25.140 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 21:31:25.140 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:31:25.140 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:31:25.140 | INFO | PerfReport: Full Execution -- 9.90e+01 / -nan, 2.52e+01 / 0.00e+00 | 20260505 21:31:25.140 | INFO | PerfReport: 22.47%, Upload -- 2.23e+01 / 0.00e+00, 7.70e+00 / 0.00e+00 | 20260505 21:31:25.140 | INFO | PerfReport: 21.82%, Download -- 2.16e+01 / 0.00e+00, 7.67e+00 / 0.00e+00 | 20260505 21:31:25.140 | INFO | PerfReport: 31.74%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 21:31:25.140 | INFO | PerfReport: 2.21%, LockUpload -- 2.19e+00 / 0.00e+00, 7.92e+00 / 0.00e+00 | 20260505 21:31:25.141 | INFO | PerfReport: 1.35%, LockDownload -- 1.34e+00 / 0.00e+00, 2.87e+00 / 0.00e+00 | 20260505 21:31:25.141 | INFO | PerfReport: 20.39%, LockCompute -- 2.02e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 21:31:25.141 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 1.81e-02 / 0.00e+00 | 20260505 21:31:25.267 | INFO | Starting a round of Baz IO | 20260505 21:31:28.418 | INFO | Populating RTMetrics report | 20260505 21:31:28.419 | INFO | RTMetrics report populated | 20260505 21:31:28.419 | INFO | Opening tmp RTMetrics json file | 20260505 21:31:28.419 | INFO | Serializing RTMetrics to json | 20260505 21:31:28.421 | INFO | Serialization done, swapping tmp file now... | 20260505 21:31:28.421 | INFO | RTMetrics done | 20260505 21:31:28.502 | INFO | Finished a round of Baz IO: Wrote 1.967 GiB in 3.235 seconds | 20260505 21:31:28.502 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.57% / 0.00% / 78.43% | 20260505 21:31:28.502 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.14561% / 3.97364e-06% / 7.54992e-05% | 20260505 21:31:28.503 | INFO | Baz IO overall: | 20260505 21:31:28.503 | INFO | BytesWritten : 3737728028759 | 20260505 21:31:28.503 | INFO | NumEvents : 1764822293656 | 20260505 21:31:30.270 | INFO | Analyzing chunk frames = [6455296,6455808) | 20260505 21:31:30.270 | INFO | MemInfo: { | 20260505 21:31:30.270 | INFO | "gpu" : " 17 GB", | 20260505 21:31:30.270 | INFO | "huge" : "", | 20260505 21:31:30.270 | INFO | "huge_shared" : " 27 GB", | 20260505 21:31:30.270 | INFO | "malloc" : " 14 GB", | 20260505 21:31:30.270 | INFO | "pinned" : " 29 GB" | 20260505 21:31:30.270 | INFO | } | 20260505 21:31:30.271 | INFO | Perf report for the last 5.13197 seconds: | 20260505 21:31:30.271 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:31:30.271 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:30.271 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:30.271 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:30.271 | INFO | ANALYSIS : 62.9%, 3.0, 97.9ms, 0 | 20260505 21:31:30.271 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:30.271 | INFO | RT_METRICS : 43.8%, 1.0, 22.9ms, 0 | 20260505 21:31:30.271 | INFO | PRE_HQ : 60.0%, 1.8, 54.7ms, 0 | 20260505 21:31:30.271 | INFO | BAZWRITER : 62.3%, 3.6, 115.9ms, 0 | 20260505 21:31:30.271 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 21:31:30.271 | INFO | Uploaded 36GiB to GPU in 2.16648s (16.6168GiB/s) | 20260505 21:31:30.271 | INFO | Downloaded 38GiB to GPU in 2.24016s (16.9631GiB/s) | 20260505 21:31:35.406 | INFO | Analyzing chunk frames = [6455808,6456320) | 20260505 21:31:35.406 | INFO | MemInfo: { | 20260505 21:31:35.406 | INFO | "gpu" : " 17 GB", | 20260505 21:31:35.406 | INFO | "huge" : "", | 20260505 21:31:35.407 | INFO | "huge_shared" : " 27 GB", | 20260505 21:31:35.407 | INFO | "malloc" : " 14 GB", | 20260505 21:31:35.407 | INFO | "pinned" : " 29 GB" | 20260505 21:31:35.407 | INFO | } | 20260505 21:31:35.407 | INFO | Perf report for the last 5.1361 seconds: | 20260505 21:31:35.407 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:31:35.407 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:35.407 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:35.407 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:35.407 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 21:31:35.407 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:35.407 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:35.407 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:31:35.407 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:35.407 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:35.407 | INFO | Uploaded 36GiB to GPU in 2.11327s (17.0352GiB/s) | 20260505 21:31:35.407 | INFO | Downloaded 34GiB to GPU in 2.02745s (16.7699GiB/s) | 20260505 21:31:40.554 | INFO | Analyzing chunk frames = [6456320,6456832) | 20260505 21:31:40.554 | INFO | MemInfo: { | 20260505 21:31:40.554 | INFO | "gpu" : " 17 GB", | 20260505 21:31:40.554 | INFO | "huge" : "", | 20260505 21:31:40.554 | INFO | "huge_shared" : " 27 GB", | 20260505 21:31:40.554 | INFO | "malloc" : " 14 GB", | 20260505 21:31:40.554 | INFO | "pinned" : " 29 GB" | 20260505 21:31:40.554 | INFO | } | 20260505 21:31:40.554 | INFO | Perf report for the last 5.14748 seconds: | 20260505 21:31:40.554 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:31:40.554 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:40.554 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:40.554 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:40.554 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260505 21:31:40.554 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:40.554 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:40.554 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 21:31:40.554 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:40.554 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:40.554 | INFO | Uploaded 36GiB to GPU in 2.13874s (16.8324GiB/s) | 20260505 21:31:40.554 | INFO | Downloaded 34GiB to GPU in 2.03746s (16.6874GiB/s) | 20260505 21:31:45.698 | INFO | Analyzing chunk frames = [6456832,6457344) | 20260505 21:31:45.698 | INFO | MemInfo: { | 20260505 21:31:45.698 | INFO | "gpu" : " 17 GB", | 20260505 21:31:45.698 | INFO | "huge" : "", | 20260505 21:31:45.698 | INFO | "huge_shared" : " 27 GB", | 20260505 21:31:45.698 | INFO | "malloc" : " 14 GB", | 20260505 21:31:45.698 | INFO | "pinned" : " 29 GB" | 20260505 21:31:45.698 | INFO | } | 20260505 21:31:45.698 | INFO | Perf report for the last 5.14424 seconds: | 20260505 21:31:45.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:31:45.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:45.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:45.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:45.698 | INFO | ANALYSIS : 61.9%, 3.0, 96.2ms, 0 | 20260505 21:31:45.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:45.699 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:45.699 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 21:31:45.699 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:45.699 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:45.699 | INFO | Uploaded 36GiB to GPU in 2.13682s (16.8474GiB/s) | 20260505 21:31:45.699 | INFO | Downloaded 34GiB to GPU in 2.04348s (16.6383GiB/s) | 20260505 21:31:50.833 | INFO | Analyzing chunk frames = [6457344,6457856) | 20260505 21:31:50.833 | INFO | MemInfo: { | 20260505 21:31:50.833 | INFO | "gpu" : " 17 GB", | 20260505 21:31:50.833 | INFO | "huge" : "", | 20260505 21:31:50.833 | INFO | "huge_shared" : " 27 GB", | 20260505 21:31:50.833 | INFO | "malloc" : " 14 GB", | 20260505 21:31:50.833 | INFO | "pinned" : " 29 GB" | 20260505 21:31:50.833 | INFO | } | 20260505 21:31:50.833 | INFO | Perf report for the last 5.1346 seconds: | 20260505 21:31:50.833 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:31:50.833 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:50.833 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:50.833 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:50.833 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260505 21:31:50.833 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:50.833 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:50.833 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:31:50.833 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:50.833 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:50.833 | INFO | Uploaded 36GiB to GPU in 2.10768s (17.0804GiB/s) | 20260505 21:31:50.833 | INFO | Downloaded 34GiB to GPU in 2.04081s (16.66GiB/s) | 20260505 21:31:55.973 | INFO | Analyzing chunk frames = [6457856,6458368) | 20260505 21:31:55.973 | INFO | MemInfo: { | 20260505 21:31:55.973 | INFO | "gpu" : " 17 GB", | 20260505 21:31:55.973 | INFO | "huge" : "", | 20260505 21:31:55.973 | INFO | "huge_shared" : " 27 GB", | 20260505 21:31:55.973 | INFO | "malloc" : " 14 GB", | 20260505 21:31:55.973 | INFO | "pinned" : " 29 GB" | 20260505 21:31:55.973 | INFO | } | 20260505 21:31:55.973 | INFO | Perf report for the last 5.14028 seconds: | 20260505 21:31:55.973 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:31:55.973 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:55.973 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:55.973 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:55.973 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260505 21:31:55.973 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:55.973 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:55.973 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 21:31:55.973 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:55.974 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:31:55.974 | INFO | Uploaded 36GiB to GPU in 2.08153s (17.2949GiB/s) | 20260505 21:31:55.974 | INFO | Downloaded 34GiB to GPU in 2.03331s (16.7215GiB/s) | 20260505 21:32:01.109 | INFO | Analyzing chunk frames = [6458368,6458880) | 20260505 21:32:01.109 | INFO | MemInfo: { | 20260505 21:32:01.109 | INFO | "gpu" : " 17 GB", | 20260505 21:32:01.109 | INFO | "huge" : "", | 20260505 21:32:01.109 | INFO | "huge_shared" : " 27 GB", | 20260505 21:32:01.109 | INFO | "malloc" : " 14 GB", | 20260505 21:32:01.109 | INFO | "pinned" : " 29 GB" | 20260505 21:32:01.109 | INFO | } | 20260505 21:32:01.109 | INFO | Perf report for the last 5.13566 seconds: | 20260505 21:32:01.109 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:32:01.109 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:01.109 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:01.109 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:01.109 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 21:32:01.109 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:01.109 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:01.109 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 21:32:01.109 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:01.109 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:01.109 | INFO | Uploaded 36GiB to GPU in 2.13238s (16.8825GiB/s) | 20260505 21:32:01.109 | INFO | Downloaded 34GiB to GPU in 2.15016s (15.8128GiB/s) | 20260505 21:32:06.252 | INFO | Analyzing chunk frames = [6458880,6459392) | 20260505 21:32:06.252 | INFO | MemInfo: { | 20260505 21:32:06.252 | INFO | "gpu" : " 17 GB", | 20260505 21:32:06.252 | INFO | "huge" : "", | 20260505 21:32:06.252 | INFO | "huge_shared" : " 27 GB", | 20260505 21:32:06.252 | INFO | "malloc" : " 14 GB", | 20260505 21:32:06.252 | INFO | "pinned" : " 29 GB" | 20260505 21:32:06.252 | INFO | } | 20260505 21:32:06.253 | INFO | Perf report for the last 5.14353 seconds: | 20260505 21:32:06.253 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:32:06.253 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:06.253 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:06.253 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:06.253 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260505 21:32:06.253 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:06.253 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:06.253 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 21:32:06.253 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:06.253 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:06.253 | INFO | Uploaded 36GiB to GPU in 2.10846s (17.074GiB/s) | 20260505 21:32:06.253 | INFO | Downloaded 34GiB to GPU in 2.00296s (16.9749GiB/s) | 20260505 21:32:06.388 | INFO | Starting a round of Baz IO | 20260505 21:32:09.543 | INFO | Populating RTMetrics report | 20260505 21:32:09.543 | INFO | RTMetrics report populated | 20260505 21:32:09.543 | INFO | Opening tmp RTMetrics json file | 20260505 21:32:09.544 | INFO | Serializing RTMetrics to json | 20260505 21:32:09.547 | INFO | Serialization done, swapping tmp file now... | 20260505 21:32:09.549 | INFO | RTMetrics done | 20260505 21:32:09.632 | INFO | Finished a round of Baz IO: Wrote 1.967 GiB in 3.243 seconds | 20260505 21:32:09.632 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.57% / 0.00% / 78.43% | 20260505 21:32:09.632 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.145562% / 0% / 4.37101e-05% | 20260505 21:32:09.632 | INFO | Baz IO overall: | 20260505 21:32:09.632 | INFO | BytesWritten : 3739840170428 | 20260505 21:32:09.632 | INFO | NumEvents : 1765797073398 | 20260505 21:32:11.391 | INFO | Analyzing chunk frames = [6459392,6459904) | 20260505 21:32:11.391 | INFO | MemInfo: { | 20260505 21:32:11.391 | INFO | "gpu" : " 17 GB", | 20260505 21:32:11.391 | INFO | "huge" : "", | 20260505 21:32:11.391 | INFO | "huge_shared" : " 27 GB", | 20260505 21:32:11.391 | INFO | "malloc" : " 14 GB", | 20260505 21:32:11.391 | INFO | "pinned" : " 29 GB" | 20260505 21:32:11.391 | INFO | } | 20260505 21:32:11.391 | INFO | Perf report for the last 5.13881 seconds: | 20260505 21:32:11.391 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:32:11.391 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:11.391 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:11.391 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:11.391 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 21:32:11.391 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:11.391 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260505 21:32:11.391 | INFO | PRE_HQ : 60.0%, 1.8, 55.4ms, 0 | 20260505 21:32:11.391 | INFO | BAZWRITER : 62.6%, 3.6, 116.3ms, 0 | 20260505 21:32:11.392 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 21:32:11.392 | INFO | Uploaded 36GiB to GPU in 2.16747s (16.6092GiB/s) | 20260505 21:32:11.392 | INFO | Downloaded 38GiB to GPU in 2.24799s (16.904GiB/s) | 20260505 21:32:16.529 | INFO | Analyzing chunk frames = [6459904,6460416) | 20260505 21:32:16.529 | INFO | MemInfo: { | 20260505 21:32:16.529 | INFO | "gpu" : " 17 GB", | 20260505 21:32:16.529 | INFO | "huge" : "", | 20260505 21:32:16.529 | INFO | "huge_shared" : " 27 GB", | 20260505 21:32:16.529 | INFO | "malloc" : " 14 GB", | 20260505 21:32:16.529 | INFO | "pinned" : " 29 GB" | 20260505 21:32:16.530 | INFO | } | 20260505 21:32:16.530 | INFO | Perf report for the last 5.13826 seconds: | 20260505 21:32:16.530 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:32:16.530 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:16.530 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:16.530 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:16.530 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260505 21:32:16.530 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:16.530 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:16.530 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 21:32:16.530 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:16.530 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:16.530 | INFO | Uploaded 36GiB to GPU in 2.08014s (17.3065GiB/s) | 20260505 21:32:16.530 | INFO | Downloaded 34GiB to GPU in 2.02308s (16.8061GiB/s) | 20260505 21:32:21.661 | INFO | Analyzing chunk frames = [6460416,6460928) | 20260505 21:32:21.661 | INFO | MemInfo: { | 20260505 21:32:21.661 | INFO | "gpu" : " 17 GB", | 20260505 21:32:21.662 | INFO | "huge" : "", | 20260505 21:32:21.662 | INFO | "huge_shared" : " 27 GB", | 20260505 21:32:21.662 | INFO | "malloc" : " 14 GB", | 20260505 21:32:21.662 | INFO | "pinned" : " 29 GB" | 20260505 21:32:21.662 | INFO | } | 20260505 21:32:21.662 | INFO | Perf report for the last 5.13205 seconds: | 20260505 21:32:21.662 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:32:21.662 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:21.662 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:21.662 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:21.662 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260505 21:32:21.662 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:21.662 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:21.662 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:32:21.662 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:21.662 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:21.662 | INFO | Uploaded 36GiB to GPU in 2.12711s (16.9244GiB/s) | 20260505 21:32:21.662 | INFO | Downloaded 34GiB to GPU in 2.01623s (16.8632GiB/s) | 20260505 21:32:26.808 | INFO | Analyzing chunk frames = [6460928,6461440) | 20260505 21:32:26.808 | INFO | MemInfo: { | 20260505 21:32:26.808 | INFO | "gpu" : " 17 GB", | 20260505 21:32:26.808 | INFO | "huge" : "", | 20260505 21:32:26.808 | INFO | "huge_shared" : " 27 GB", | 20260505 21:32:26.808 | INFO | "malloc" : " 14 GB", | 20260505 21:32:26.808 | INFO | "pinned" : " 29 GB" | 20260505 21:32:26.808 | INFO | } | 20260505 21:32:26.808 | INFO | Perf report for the last 5.14643 seconds: | 20260505 21:32:26.808 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:32:26.808 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:26.808 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:26.808 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:26.808 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260505 21:32:26.808 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:26.808 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:26.808 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:32:26.808 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:26.808 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:26.808 | INFO | Uploaded 36GiB to GPU in 2.20624s (16.3174GiB/s) | 20260505 21:32:26.808 | INFO | Downloaded 34GiB to GPU in 2.0494s (16.5903GiB/s) | 20260505 21:32:31.953 | INFO | Analyzing chunk frames = [6461440,6461952) | 20260505 21:32:31.953 | INFO | MemInfo: { | 20260505 21:32:31.953 | INFO | "gpu" : " 17 GB", | 20260505 21:32:31.953 | INFO | "huge" : "", | 20260505 21:32:31.953 | INFO | "huge_shared" : " 27 GB", | 20260505 21:32:31.953 | INFO | "malloc" : " 14 GB", | 20260505 21:32:31.953 | INFO | "pinned" : " 29 GB" | 20260505 21:32:31.953 | INFO | } | 20260505 21:32:31.953 | INFO | Perf report for the last 5.14504 seconds: | 20260505 21:32:31.953 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:32:31.953 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:31.953 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:31.953 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:31.953 | INFO | ANALYSIS : 61.6%, 3.0, 96.3ms, 0 | 20260505 21:32:31.953 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:31.953 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:31.953 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:32:31.953 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:31.953 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:31.953 | INFO | Uploaded 36GiB to GPU in 2.15746s (16.6863GiB/s) | 20260505 21:32:31.953 | INFO | Downloaded 34GiB to GPU in 2.06799s (16.4411GiB/s) | 20260505 21:32:37.086 | INFO | Analyzing chunk frames = [6461952,6462464) | 20260505 21:32:37.086 | INFO | MemInfo: { | 20260505 21:32:37.086 | INFO | "gpu" : " 17 GB", | 20260505 21:32:37.086 | INFO | "huge" : "", | 20260505 21:32:37.086 | INFO | "huge_shared" : " 27 GB", | 20260505 21:32:37.086 | INFO | "malloc" : " 14 GB", | 20260505 21:32:37.086 | INFO | "pinned" : " 29 GB" | 20260505 21:32:37.086 | INFO | } | 20260505 21:32:37.086 | INFO | Perf report for the last 5.13281 seconds: | 20260505 21:32:37.086 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:32:37.086 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:37.086 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:37.086 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:37.086 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260505 21:32:37.086 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:37.086 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:37.086 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:32:37.086 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:37.086 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:37.086 | INFO | Uploaded 36GiB to GPU in 2.06315s (17.4491GiB/s) | 20260505 21:32:37.086 | INFO | Downloaded 34GiB to GPU in 2.03105s (16.7401GiB/s) | 20260505 21:32:42.222 | INFO | Analyzing chunk frames = [6462464,6462976) | 20260505 21:32:42.222 | INFO | MemInfo: { | 20260505 21:32:42.222 | INFO | "gpu" : " 17 GB", | 20260505 21:32:42.223 | INFO | "huge" : "", | 20260505 21:32:42.223 | INFO | "huge_shared" : " 27 GB", | 20260505 21:32:42.223 | INFO | "malloc" : " 14 GB", | 20260505 21:32:42.223 | INFO | "pinned" : " 29 GB" | 20260505 21:32:42.223 | INFO | } | 20260505 21:32:42.223 | INFO | Perf report for the last 5.13674 seconds: | 20260505 21:32:42.223 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:32:42.223 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:42.223 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:42.223 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:42.223 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260505 21:32:42.223 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:42.223 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:42.223 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 21:32:42.223 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:42.223 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:42.223 | INFO | Uploaded 36GiB to GPU in 2.13285s (16.8789GiB/s) | 20260505 21:32:42.223 | INFO | Downloaded 34GiB to GPU in 2.1399s (15.8886GiB/s) | 20260505 21:32:47.362 | INFO | Analyzing chunk frames = [6462976,6463488) | 20260505 21:32:47.362 | INFO | MemInfo: { | 20260505 21:32:47.362 | INFO | "gpu" : " 17 GB", | 20260505 21:32:47.362 | INFO | "huge" : "", | 20260505 21:32:47.362 | INFO | "huge_shared" : " 27 GB", | 20260505 21:32:47.362 | INFO | "malloc" : " 14 GB", | 20260505 21:32:47.362 | INFO | "pinned" : " 29 GB" | 20260505 21:32:47.362 | INFO | } | 20260505 21:32:47.362 | INFO | Perf report for the last 5.13913 seconds: | 20260505 21:32:47.362 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:32:47.362 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:47.362 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:47.362 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:47.362 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 21:32:47.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:47.362 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:47.362 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 21:32:47.362 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:47.362 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:47.362 | INFO | Uploaded 36GiB to GPU in 2.20361s (16.3369GiB/s) | 20260505 21:32:47.362 | INFO | Downloaded 34GiB to GPU in 2.05216s (16.5679GiB/s) | 20260505 21:32:47.362 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 21:32:47.362 | INFO | Total mem usage: 15.3 GiB | 20260505 21:32:47.362 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 21:32:47.362 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.96%) | 20260505 21:32:47.362 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 21:32:47.362 | INFO | High Water Breakdown: | 20260505 21:32:47.362 | INFO | 60.9% : 8.57 GiB ( 90% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 21:32:47.362 | INFO | 39% : 5.48 GiB ( 82.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 21:32:47.362 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 21:32:47.362 | INFO | ------------------------------------------ | 20260505 21:32:47.362 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:32:47.362 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 21:32:47.362 | INFO | Total mem usage: 26.5 GiB | 20260505 21:32:47.362 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 21:32:47.362 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 21:32:47.362 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 21:32:47.363 | INFO | High Water Breakdown: | 20260505 21:32:47.363 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:32:47.363 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:32:47.363 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:32:47.363 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:32:47.363 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:32:47.363 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:32:47.363 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 21:32:47.363 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 21:32:47.363 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:32:47.363 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 21:32:47.363 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 21:32:47.363 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 21:32:47.363 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 21:32:47.363 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 21:32:47.363 | INFO | ------------------------------------------ | 20260505 21:32:47.363 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:32:47.363 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 21:32:47.363 | INFO | Total mem usage: 25.6 GiB | 20260505 21:32:47.363 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 21:32:47.363 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 21:32:47.363 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 21:32:47.363 | INFO | High Water Breakdown: | 20260505 21:32:47.363 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 21:32:47.363 | INFO | ------------------------------------------ | 20260505 21:32:47.363 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:32:47.363 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 21:32:47.363 | INFO | Total mem usage: 15.7 GiB | 20260505 21:32:47.363 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 21:32:47.363 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 21:32:47.363 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 21:32:47.363 | INFO | High Water Breakdown: | 20260505 21:32:47.363 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 21:32:47.363 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 21:32:47.363 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 21:32:47.363 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 21:32:47.363 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 21:32:47.363 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 21:32:47.363 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 21:32:47.363 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 21:32:47.363 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 21:32:47.363 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:32:47.363 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 21:32:47.363 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:32:47.363 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 21:32:47.363 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 21:32:47.363 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 21:32:47.363 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 21:32:47.363 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:32:47.363 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 21:32:47.363 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:32:47.363 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:32:47.363 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:32:47.363 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 21:32:47.363 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 21:32:47.363 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:32:47.363 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 21:32:47.363 | INFO | ------------------------------------------ | 20260505 21:32:47.363 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:32:47.363 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 21:32:47.363 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:32:47.363 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:32:47.363 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.45e+01, 1.33e+01 / 1.78e+00 | 20260505 21:32:47.363 | INFO | PerfReport: 0.29%, AggregateStats -- 9.26e-02 / 1.03e-01, 1.11e-02 / 4.37e-03 | 20260505 21:32:47.363 | INFO | PerfReport: 11.80%, Baseline -- 3.71e+00 / 4.54e+00, 5.65e-01 / 1.53e-01 | 20260505 21:32:47.363 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.51e+00, 1.90e-01 / 1.03e-01 | 20260505 21:32:47.363 | INFO | PerfReport: 15.97%, DME -- 4.02e+01 / 4.43e+01, 4.05e+00 / 1.92e+00 | 20260505 21:32:47.363 | INFO | PerfReport: 57.19%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.52e+00 / 1.01e+00 | 20260505 21:32:47.363 | INFO | PerfReport: 7.17%, PulseAccumulating -- 2.26e+00 / 2.80e+00, 4.32e-01 / 2.18e-01 | 20260505 21:32:47.363 | INFO | PerfReport: 3.19%, Metrics -- 1.00e+00 / 1.29e+00, 2.72e-01 / 1.30e-01 | 20260505 21:32:47.363 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.49e-01 / 4.04e-01, 6.07e-02 / 3.10e-02 | 20260505 21:32:47.363 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 21:32:47.363 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:32:47.363 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:32:47.363 | INFO | PerfReport: Full Execution -- 9.90e+01 / -nan, 2.50e+01 / 0.00e+00 | 20260505 21:32:47.363 | INFO | PerfReport: 22.44%, Upload -- 2.22e+01 / 0.00e+00, 7.67e+00 / 0.00e+00 | 20260505 21:32:47.363 | INFO | PerfReport: 21.84%, Download -- 2.16e+01 / 0.00e+00, 7.65e+00 / 0.00e+00 | 20260505 21:32:47.363 | INFO | PerfReport: 31.79%, Compute -- 3.15e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260505 21:32:47.363 | INFO | PerfReport: 2.19%, LockUpload -- 2.17e+00 / 0.00e+00, 7.89e+00 / 0.00e+00 | 20260505 21:32:47.363 | INFO | PerfReport: 1.33%, LockDownload -- 1.32e+00 / 0.00e+00, 2.89e+00 / 0.00e+00 | 20260505 21:32:47.363 | INFO | PerfReport: 20.40%, LockCompute -- 2.02e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 21:32:47.363 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.22e-02 / 0.00e+00 | 20260505 21:32:47.495 | INFO | Starting a round of Baz IO | 20260505 21:32:50.667 | INFO | Populating RTMetrics report | 20260505 21:32:50.667 | INFO | RTMetrics report populated | 20260505 21:32:50.667 | INFO | Opening tmp RTMetrics json file | 20260505 21:32:50.671 | INFO | Serializing RTMetrics to json | 20260505 21:32:50.674 | INFO | Serialization done, swapping tmp file now... | 20260505 21:32:50.676 | INFO | RTMetrics done | 20260505 21:32:50.748 | INFO | Finished a round of Baz IO: Wrote 1.968 GiB in 3.252 seconds | 20260505 21:32:50.748 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.57% / 0.00% / 78.43% | 20260505 21:32:50.748 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.145348% / 0% / 9.93411e-05% | 20260505 21:32:50.748 | INFO | Baz IO overall: | 20260505 21:32:50.748 | INFO | BytesWritten : 3741953352896 | 20260505 21:32:50.748 | INFO | NumEvents : 1766772702291 | 20260505 21:32:52.505 | INFO | Analyzing chunk frames = [6463488,6464000) | 20260505 21:32:52.505 | INFO | MemInfo: { | 20260505 21:32:52.505 | INFO | "gpu" : " 17 GB", | 20260505 21:32:52.505 | INFO | "huge" : "", | 20260505 21:32:52.505 | INFO | "huge_shared" : " 27 GB", | 20260505 21:32:52.505 | INFO | "malloc" : " 14 GB", | 20260505 21:32:52.505 | INFO | "pinned" : " 29 GB" | 20260505 21:32:52.505 | INFO | } | 20260505 21:32:52.505 | INFO | Perf report for the last 5.14331 seconds: | 20260505 21:32:52.505 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:32:52.505 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:52.505 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:52.505 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:52.505 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260505 21:32:52.505 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:52.505 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260505 21:32:52.505 | INFO | PRE_HQ : 60.1%, 1.8, 55.0ms, 0 | 20260505 21:32:52.505 | INFO | BAZWRITER : 62.9%, 3.6, 116.6ms, 0 | 20260505 21:32:52.505 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 21:32:52.506 | INFO | Uploaded 36GiB to GPU in 2.17655s (16.5399GiB/s) | 20260505 21:32:52.506 | INFO | Downloaded 38GiB to GPU in 2.26117s (16.8055GiB/s) | 20260505 21:32:57.650 | INFO | Analyzing chunk frames = [6464000,6464512) | 20260505 21:32:57.650 | INFO | MemInfo: { | 20260505 21:32:57.650 | INFO | "gpu" : " 17 GB", | 20260505 21:32:57.650 | INFO | "huge" : "", | 20260505 21:32:57.650 | INFO | "huge_shared" : " 27 GB", | 20260505 21:32:57.650 | INFO | "malloc" : " 14 GB", | 20260505 21:32:57.651 | INFO | "pinned" : " 29 GB" | 20260505 21:32:57.651 | INFO | } | 20260505 21:32:57.651 | INFO | Perf report for the last 5.14579 seconds: | 20260505 21:32:57.651 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:32:57.651 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:57.651 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:57.651 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:57.651 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260505 21:32:57.651 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:57.651 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:57.651 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 21:32:57.651 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:57.651 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:32:57.651 | INFO | Uploaded 36GiB to GPU in 2.0774s (17.3294GiB/s) | 20260505 21:32:57.651 | INFO | Downloaded 34GiB to GPU in 2.02249s (16.811GiB/s) | 20260505 21:33:02.784 | INFO | Analyzing chunk frames = [6464512,6465024) | 20260505 21:33:02.784 | INFO | MemInfo: { | 20260505 21:33:02.784 | INFO | "gpu" : " 17 GB", | 20260505 21:33:02.784 | INFO | "huge" : "", | 20260505 21:33:02.785 | INFO | "huge_shared" : " 27 GB", | 20260505 21:33:02.785 | INFO | "malloc" : " 14 GB", | 20260505 21:33:02.785 | INFO | "pinned" : " 29 GB" | 20260505 21:33:02.785 | INFO | } | 20260505 21:33:02.785 | INFO | Perf report for the last 5.13361 seconds: | 20260505 21:33:02.785 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:33:02.785 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:02.785 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:02.785 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:02.785 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260505 21:33:02.785 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:02.785 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:02.785 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260505 21:33:02.785 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:02.785 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:02.785 | INFO | Uploaded 36GiB to GPU in 2.10424s (17.1083GiB/s) | 20260505 21:33:02.785 | INFO | Downloaded 34GiB to GPU in 2.02537s (16.7871GiB/s) | 20260505 21:33:07.920 | INFO | Analyzing chunk frames = [6465024,6465536) | 20260505 21:33:07.920 | INFO | MemInfo: { | 20260505 21:33:07.920 | INFO | "gpu" : " 17 GB", | 20260505 21:33:07.920 | INFO | "huge" : "", | 20260505 21:33:07.920 | INFO | "huge_shared" : " 27 GB", | 20260505 21:33:07.920 | INFO | "malloc" : " 14 GB", | 20260505 21:33:07.920 | INFO | "pinned" : " 29 GB" | 20260505 21:33:07.920 | INFO | } | 20260505 21:33:07.920 | INFO | Perf report for the last 5.13518 seconds: | 20260505 21:33:07.920 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:33:07.920 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:07.920 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:07.920 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:07.920 | INFO | ANALYSIS : 62.3%, 3.0, 96.7ms, 0 | 20260505 21:33:07.920 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:07.920 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:07.920 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:33:07.920 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:07.920 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:07.920 | INFO | Uploaded 36GiB to GPU in 2.20802s (16.3042GiB/s) | 20260505 21:33:07.920 | INFO | Downloaded 34GiB to GPU in 2.06334s (16.4781GiB/s) | 20260505 21:33:13.061 | INFO | Analyzing chunk frames = [6465536,6466048) | 20260505 21:33:13.061 | INFO | MemInfo: { | 20260505 21:33:13.061 | INFO | "gpu" : " 17 GB", | 20260505 21:33:13.061 | INFO | "huge" : "", | 20260505 21:33:13.061 | INFO | "huge_shared" : " 27 GB", | 20260505 21:33:13.061 | INFO | "malloc" : " 14 GB", | 20260505 21:33:13.061 | INFO | "pinned" : " 29 GB" | 20260505 21:33:13.061 | INFO | } | 20260505 21:33:13.061 | INFO | Perf report for the last 5.14135 seconds: | 20260505 21:33:13.061 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:33:13.061 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:13.061 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:13.061 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:13.061 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260505 21:33:13.061 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:13.061 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:13.061 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:33:13.061 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:13.061 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:13.061 | INFO | Uploaded 36GiB to GPU in 2.14064s (16.8174GiB/s) | 20260505 21:33:13.062 | INFO | Downloaded 34GiB to GPU in 2.06694s (16.4495GiB/s) | 20260505 21:33:18.202 | INFO | Analyzing chunk frames = [6466048,6466560) | 20260505 21:33:18.202 | INFO | MemInfo: { | 20260505 21:33:18.202 | INFO | "gpu" : " 17 GB", | 20260505 21:33:18.202 | INFO | "huge" : "", | 20260505 21:33:18.202 | INFO | "huge_shared" : " 27 GB", | 20260505 21:33:18.202 | INFO | "malloc" : " 14 GB", | 20260505 21:33:18.202 | INFO | "pinned" : " 29 GB" | 20260505 21:33:18.202 | INFO | } | 20260505 21:33:18.202 | INFO | Perf report for the last 5.14086 seconds: | 20260505 21:33:18.202 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:33:18.202 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:18.202 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:18.202 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:18.202 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260505 21:33:18.202 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:18.202 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:18.202 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 21:33:18.202 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:18.202 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:18.202 | INFO | Uploaded 36GiB to GPU in 2.09059s (17.2201GiB/s) | 20260505 21:33:18.202 | INFO | Downloaded 34GiB to GPU in 2.04156s (16.6539GiB/s) | 20260505 21:33:23.344 | INFO | Analyzing chunk frames = [6466560,6467072) | 20260505 21:33:23.344 | INFO | MemInfo: { | 20260505 21:33:23.344 | INFO | "gpu" : " 17 GB", | 20260505 21:33:23.344 | INFO | "huge" : "", | 20260505 21:33:23.344 | INFO | "huge_shared" : " 27 GB", | 20260505 21:33:23.344 | INFO | "malloc" : " 14 GB", | 20260505 21:33:23.344 | INFO | "pinned" : " 29 GB" | 20260505 21:33:23.344 | INFO | } | 20260505 21:33:23.344 | INFO | Perf report for the last 5.14211 seconds: | 20260505 21:33:23.344 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:33:23.344 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:23.344 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:23.344 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:23.344 | INFO | ANALYSIS : 62.7%, 3.0, 98.0ms, 0 | 20260505 21:33:23.344 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:23.344 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:23.344 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 21:33:23.344 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:23.344 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:23.344 | INFO | Uploaded 36GiB to GPU in 2.14996s (16.7445GiB/s) | 20260505 21:33:23.345 | INFO | Downloaded 34GiB to GPU in 2.12931s (15.9676GiB/s) | 20260505 21:33:28.473 | INFO | Analyzing chunk frames = [6467072,6467584) | 20260505 21:33:28.473 | INFO | MemInfo: { | 20260505 21:33:28.473 | INFO | "gpu" : " 17 GB", | 20260505 21:33:28.473 | INFO | "huge" : "", | 20260505 21:33:28.473 | INFO | "huge_shared" : " 27 GB", | 20260505 21:33:28.473 | INFO | "malloc" : " 14 GB", | 20260505 21:33:28.473 | INFO | "pinned" : " 29 GB" | 20260505 21:33:28.474 | INFO | } | 20260505 21:33:28.474 | INFO | Perf report for the last 5.12936 seconds: | 20260505 21:33:28.474 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:33:28.474 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:28.474 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:28.474 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:28.474 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260505 21:33:28.474 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:28.474 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:28.474 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:33:28.474 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:28.474 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:28.474 | INFO | Uploaded 36GiB to GPU in 2.11915s (16.9879GiB/s) | 20260505 21:33:28.474 | INFO | Downloaded 34GiB to GPU in 2.01734s (16.8539GiB/s) | 20260505 21:33:28.603 | INFO | Starting a round of Baz IO | 20260505 21:33:31.772 | INFO | Populating RTMetrics report | 20260505 21:33:31.772 | INFO | RTMetrics report populated | 20260505 21:33:31.772 | INFO | Opening tmp RTMetrics json file | 20260505 21:33:31.773 | INFO | Serializing RTMetrics to json | 20260505 21:33:31.774 | INFO | Serialization done, swapping tmp file now... | 20260505 21:33:31.775 | INFO | RTMetrics done | 20260505 21:33:31.858 | INFO | Finished a round of Baz IO: Wrote 1.965 GiB in 3.254 seconds | 20260505 21:33:31.858 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.57% / 0.00% / 78.43% | 20260505 21:33:31.858 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.144839% / 0% / 0.000115236% | 20260505 21:33:31.858 | INFO | Baz IO overall: | 20260505 21:33:31.858 | INFO | BytesWritten : 3744063376111 | 20260505 21:33:31.858 | INFO | NumEvents : 1767746071612 | 20260505 21:33:33.616 | INFO | Analyzing chunk frames = [6467584,6468096) | 20260505 21:33:33.616 | INFO | MemInfo: { | 20260505 21:33:33.616 | INFO | "gpu" : " 17 GB", | 20260505 21:33:33.616 | INFO | "huge" : "", | 20260505 21:33:33.616 | INFO | "huge_shared" : " 27 GB", | 20260505 21:33:33.616 | INFO | "malloc" : " 14 GB", | 20260505 21:33:33.616 | INFO | "pinned" : " 29 GB" | 20260505 21:33:33.616 | INFO | } | 20260505 21:33:33.616 | INFO | Perf report for the last 5.14277 seconds: | 20260505 21:33:33.616 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:33:33.616 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:33.616 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:33.616 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:33.616 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 21:33:33.616 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:33.616 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260505 21:33:33.616 | INFO | PRE_HQ : 60.2%, 1.8, 55.4ms, 0 | 20260505 21:33:33.616 | INFO | BAZWRITER : 62.8%, 3.6, 116.5ms, 0 | 20260505 21:33:33.616 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 21:33:33.617 | INFO | Uploaded 36GiB to GPU in 2.15812s (16.6812GiB/s) | 20260505 21:33:33.617 | INFO | Downloaded 38GiB to GPU in 2.23968s (16.9667GiB/s) | 20260505 21:33:38.759 | INFO | Analyzing chunk frames = [6468096,6468608) | 20260505 21:33:38.759 | INFO | MemInfo: { | 20260505 21:33:38.759 | INFO | "gpu" : " 17 GB", | 20260505 21:33:38.759 | INFO | "huge" : "", | 20260505 21:33:38.759 | INFO | "huge_shared" : " 27 GB", | 20260505 21:33:38.759 | INFO | "malloc" : " 14 GB", | 20260505 21:33:38.759 | INFO | "pinned" : " 29 GB" | 20260505 21:33:38.759 | INFO | } | 20260505 21:33:38.759 | INFO | Perf report for the last 5.14281 seconds: | 20260505 21:33:38.759 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:33:38.759 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:38.759 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:38.759 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:38.759 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 21:33:38.759 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:38.759 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:38.759 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 21:33:38.759 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:38.759 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:38.759 | INFO | Uploaded 36GiB to GPU in 2.08834s (17.2386GiB/s) | 20260505 21:33:38.759 | INFO | Downloaded 34GiB to GPU in 2.02015s (16.8304GiB/s) | 20260505 21:33:43.900 | INFO | Analyzing chunk frames = [6468608,6469120) | 20260505 21:33:43.900 | INFO | MemInfo: { | 20260505 21:33:43.900 | INFO | "gpu" : " 17 GB", | 20260505 21:33:43.900 | INFO | "huge" : "", | 20260505 21:33:43.900 | INFO | "huge_shared" : " 27 GB", | 20260505 21:33:43.900 | INFO | "malloc" : " 14 GB", | 20260505 21:33:43.900 | INFO | "pinned" : " 29 GB" | 20260505 21:33:43.900 | INFO | } | 20260505 21:33:43.900 | INFO | Perf report for the last 5.14102 seconds: | 20260505 21:33:43.900 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:33:43.900 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:43.900 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:43.900 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:43.900 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260505 21:33:43.900 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:43.900 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:43.900 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 21:33:43.900 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:43.900 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:43.900 | INFO | Uploaded 36GiB to GPU in 2.15651s (16.6937GiB/s) | 20260505 21:33:43.900 | INFO | Downloaded 34GiB to GPU in 2.05001s (16.5853GiB/s) | 20260505 21:33:49.033 | INFO | Analyzing chunk frames = [6469120,6469632) | 20260505 21:33:49.033 | INFO | MemInfo: { | 20260505 21:33:49.033 | INFO | "gpu" : " 17 GB", | 20260505 21:33:49.033 | INFO | "huge" : "", | 20260505 21:33:49.033 | INFO | "huge_shared" : " 27 GB", | 20260505 21:33:49.033 | INFO | "malloc" : " 14 GB", | 20260505 21:33:49.033 | INFO | "pinned" : " 29 GB" | 20260505 21:33:49.034 | INFO | } | 20260505 21:33:49.034 | INFO | Perf report for the last 5.13338 seconds: | 20260505 21:33:49.034 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:33:49.034 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:49.034 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:49.034 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:49.034 | INFO | ANALYSIS : 61.9%, 3.0, 96.0ms, 0 | 20260505 21:33:49.034 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:49.034 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:49.034 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260505 21:33:49.034 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:49.034 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:49.034 | INFO | Uploaded 36GiB to GPU in 2.12044s (16.9776GiB/s) | 20260505 21:33:49.034 | INFO | Downloaded 34GiB to GPU in 2.00942s (16.9203GiB/s) | 20260505 21:33:54.169 | INFO | Analyzing chunk frames = [6469632,6470144) | 20260505 21:33:54.169 | INFO | MemInfo: { | 20260505 21:33:54.169 | INFO | "gpu" : " 17 GB", | 20260505 21:33:54.169 | INFO | "huge" : "", | 20260505 21:33:54.169 | INFO | "huge_shared" : " 27 GB", | 20260505 21:33:54.169 | INFO | "malloc" : " 14 GB", | 20260505 21:33:54.169 | INFO | "pinned" : " 29 GB" | 20260505 21:33:54.169 | INFO | } | 20260505 21:33:54.170 | INFO | Perf report for the last 5.13592 seconds: | 20260505 21:33:54.170 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:33:54.170 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:54.170 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:54.170 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:54.170 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260505 21:33:54.170 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:54.170 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:54.170 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:33:54.170 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:54.170 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:54.170 | INFO | Uploaded 36GiB to GPU in 2.1345s (16.8658GiB/s) | 20260505 21:33:54.170 | INFO | Downloaded 34GiB to GPU in 2.06149s (16.4929GiB/s) | 20260505 21:33:59.315 | INFO | Analyzing chunk frames = [6470144,6470656) | 20260505 21:33:59.315 | INFO | MemInfo: { | 20260505 21:33:59.315 | INFO | "gpu" : " 17 GB", | 20260505 21:33:59.315 | INFO | "huge" : "", | 20260505 21:33:59.316 | INFO | "huge_shared" : " 27 GB", | 20260505 21:33:59.316 | INFO | "malloc" : " 14 GB", | 20260505 21:33:59.316 | INFO | "pinned" : " 29 GB" | 20260505 21:33:59.316 | INFO | } | 20260505 21:33:59.316 | INFO | Perf report for the last 5.14611 seconds: | 20260505 21:33:59.316 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:33:59.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:59.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:59.316 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:59.316 | INFO | ANALYSIS : 62.1%, 3.0, 97.1ms, 0 | 20260505 21:33:59.316 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:59.316 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:59.316 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 21:33:59.316 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:59.316 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:33:59.316 | INFO | Uploaded 36GiB to GPU in 2.15016s (16.7429GiB/s) | 20260505 21:33:59.316 | INFO | Downloaded 34GiB to GPU in 2.05198s (16.5693GiB/s) | 20260505 21:34:04.458 | INFO | Analyzing chunk frames = [6470656,6471168) | 20260505 21:34:04.458 | INFO | MemInfo: { | 20260505 21:34:04.458 | INFO | "gpu" : " 17 GB", | 20260505 21:34:04.458 | INFO | "huge" : "", | 20260505 21:34:04.458 | INFO | "huge_shared" : " 27 GB", | 20260505 21:34:04.458 | INFO | "malloc" : " 14 GB", | 20260505 21:34:04.458 | INFO | "pinned" : " 29 GB" | 20260505 21:34:04.458 | INFO | } | 20260505 21:34:04.458 | INFO | Perf report for the last 5.1427 seconds: | 20260505 21:34:04.458 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:34:04.458 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:04.458 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:04.458 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:04.458 | INFO | ANALYSIS : 62.7%, 3.0, 98.0ms, 0 | 20260505 21:34:04.458 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:04.458 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:04.458 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260505 21:34:04.458 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:04.458 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:04.459 | INFO | Uploaded 36GiB to GPU in 2.15961s (16.6696GiB/s) | 20260505 21:34:04.459 | INFO | Downloaded 34GiB to GPU in 2.12325s (16.0132GiB/s) | 20260505 21:34:09.603 | INFO | Analyzing chunk frames = [6471168,6471680) | 20260505 21:34:09.603 | INFO | MemInfo: { | 20260505 21:34:09.603 | INFO | "gpu" : " 17 GB", | 20260505 21:34:09.603 | INFO | "huge" : "", | 20260505 21:34:09.603 | INFO | "huge_shared" : " 27 GB", | 20260505 21:34:09.603 | INFO | "malloc" : " 14 GB", | 20260505 21:34:09.603 | INFO | "pinned" : " 29 GB" | 20260505 21:34:09.603 | INFO | } | 20260505 21:34:09.603 | INFO | Perf report for the last 5.14494 seconds: | 20260505 21:34:09.603 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:34:09.603 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:09.603 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:09.603 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:09.603 | INFO | ANALYSIS : 62.1%, 3.0, 97.1ms, 0 | 20260505 21:34:09.603 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:09.603 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:09.603 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 21:34:09.603 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:09.603 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:09.604 | INFO | Uploaded 36GiB to GPU in 2.12124s (16.9712GiB/s) | 20260505 21:34:09.604 | INFO | Downloaded 34GiB to GPU in 2.00489s (16.9585GiB/s) | 20260505 21:34:09.604 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 21:34:09.604 | INFO | Total mem usage: 15.3 GiB | 20260505 21:34:09.604 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 21:34:09.604 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.98%) | 20260505 21:34:09.604 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 21:34:09.604 | INFO | High Water Breakdown: | 20260505 21:34:09.604 | INFO | 60.9% : 8.57 GiB ( 90% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 21:34:09.604 | INFO | 38.9% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 21:34:09.604 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 21:34:09.604 | INFO | ------------------------------------------ | 20260505 21:34:09.604 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:34:09.604 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 21:34:09.604 | INFO | Total mem usage: 26.5 GiB | 20260505 21:34:09.604 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 21:34:09.604 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 21:34:09.604 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 21:34:09.604 | INFO | High Water Breakdown: | 20260505 21:34:09.604 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:34:09.604 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:34:09.604 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:34:09.604 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:34:09.604 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:34:09.604 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:34:09.604 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 21:34:09.604 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 21:34:09.604 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:34:09.604 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 21:34:09.604 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 21:34:09.604 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 21:34:09.604 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 21:34:09.604 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 21:34:09.604 | INFO | ------------------------------------------ | 20260505 21:34:09.604 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:34:09.604 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 21:34:09.604 | INFO | Total mem usage: 25.6 GiB | 20260505 21:34:09.604 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 21:34:09.604 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 21:34:09.604 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 21:34:09.604 | INFO | High Water Breakdown: | 20260505 21:34:09.604 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 21:34:09.604 | INFO | ------------------------------------------ | 20260505 21:34:09.604 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:34:09.605 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 21:34:09.605 | INFO | Total mem usage: 15.7 GiB | 20260505 21:34:09.605 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 21:34:09.605 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 21:34:09.605 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 21:34:09.605 | INFO | High Water Breakdown: | 20260505 21:34:09.605 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 21:34:09.605 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 21:34:09.605 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 21:34:09.605 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 21:34:09.605 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 21:34:09.605 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 21:34:09.605 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 21:34:09.605 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 21:34:09.605 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 21:34:09.605 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:34:09.605 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 21:34:09.605 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:34:09.605 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 21:34:09.605 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 21:34:09.605 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 21:34:09.605 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 21:34:09.605 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:34:09.605 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 21:34:09.605 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:34:09.605 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:34:09.605 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:34:09.605 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 21:34:09.605 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 21:34:09.605 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:34:09.605 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 21:34:09.605 | INFO | ------------------------------------------ | 20260505 21:34:09.605 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:34:09.605 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 21:34:09.605 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:34:09.605 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:34:09.605 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.44e+01, 1.31e+01 / 1.67e+00 | 20260505 21:34:09.605 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.04e-01, 1.04e-02 / 7.28e-03 | 20260505 21:34:09.605 | INFO | PerfReport: 11.78%, Baseline -- 3.71e+00 / 4.54e+00, 5.68e-01 / 1.55e-01 | 20260505 21:34:09.605 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.51e+00, 1.89e-01 / 1.22e-01 | 20260505 21:34:09.605 | INFO | PerfReport: 15.94%, DME -- 4.01e+01 / 4.42e+01, 3.91e+00 / 1.77e+00 | 20260505 21:34:09.605 | INFO | PerfReport: 57.23%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.53e+00 / 1.19e+00 | 20260505 21:34:09.605 | INFO | PerfReport: 7.17%, PulseAccumulating -- 2.25e+00 / 2.82e+00, 4.31e-01 / 2.24e-01 | 20260505 21:34:09.605 | INFO | PerfReport: 3.19%, Metrics -- 1.00e+00 / 1.31e+00, 2.76e-01 / 1.71e-01 | 20260505 21:34:09.605 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.49e-01 / 4.08e-01, 6.19e-02 / 3.95e-02 | 20260505 21:34:09.605 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 21:34:09.605 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:34:09.605 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:34:09.605 | INFO | PerfReport: Full Execution -- 9.91e+01 / -nan, 2.53e+01 / 0.00e+00 | 20260505 21:34:09.605 | INFO | PerfReport: 22.45%, Upload -- 2.22e+01 / 0.00e+00, 7.68e+00 / 0.00e+00 | 20260505 21:34:09.605 | INFO | PerfReport: 21.81%, Download -- 2.16e+01 / 0.00e+00, 7.65e+00 / 0.00e+00 | 20260505 21:34:09.605 | INFO | PerfReport: 31.75%, Compute -- 3.14e+01 / 0.00e+00, 1.31e+01 / 0.00e+00 | 20260505 21:34:09.605 | INFO | PerfReport: 2.21%, LockUpload -- 2.19e+00 / 0.00e+00, 7.92e+00 / 0.00e+00 | 20260505 21:34:09.605 | INFO | PerfReport: 1.33%, LockDownload -- 1.32e+00 / 0.00e+00, 2.86e+00 / 0.00e+00 | 20260505 21:34:09.605 | INFO | PerfReport: 20.43%, LockCompute -- 2.02e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260505 21:34:09.605 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.79e-02 / 0.00e+00 | 20260505 21:34:09.725 | INFO | Starting a round of Baz IO | 20260505 21:34:12.881 | INFO | Populating RTMetrics report | 20260505 21:34:12.881 | INFO | RTMetrics report populated | 20260505 21:34:12.881 | INFO | Opening tmp RTMetrics json file | 20260505 21:34:12.883 | INFO | Serializing RTMetrics to json | 20260505 21:34:12.884 | INFO | Serialization done, swapping tmp file now... | 20260505 21:34:12.884 | INFO | RTMetrics done | 20260505 21:34:12.961 | INFO | Finished a round of Baz IO: Wrote 1.965 GiB in 3.235 seconds | 20260505 21:34:12.961 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.57% / 0.00% / 78.43% | 20260505 21:34:12.961 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.145141% / 3.97364e-06% / 9.53674e-05% | 20260505 21:34:12.961 | INFO | Baz IO overall: | 20260505 21:34:12.961 | INFO | BytesWritten : 3746173115234 | 20260505 21:34:12.961 | INFO | NumEvents : 1768719365738 | 20260505 21:34:14.732 | INFO | Analyzing chunk frames = [6471680,6472192) | 20260505 21:34:14.732 | INFO | MemInfo: { | 20260505 21:34:14.732 | INFO | "gpu" : " 17 GB", | 20260505 21:34:14.732 | INFO | "huge" : "", | 20260505 21:34:14.732 | INFO | "huge_shared" : " 27 GB", | 20260505 21:34:14.732 | INFO | "malloc" : " 14 GB", | 20260505 21:34:14.732 | INFO | "pinned" : " 29 GB" | 20260505 21:34:14.732 | INFO | } | 20260505 21:34:14.732 | INFO | Perf report for the last 5.12864 seconds: | 20260505 21:34:14.732 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:34:14.732 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:14.732 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:14.732 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:14.732 | INFO | ANALYSIS : 62.9%, 3.0, 97.9ms, 0 | 20260505 21:34:14.732 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:14.732 | INFO | RT_METRICS : 43.8%, 1.0, 22.9ms, 0 | 20260505 21:34:14.732 | INFO | PRE_HQ : 60.1%, 1.8, 54.6ms, 0 | 20260505 21:34:14.732 | INFO | BAZWRITER : 62.3%, 3.6, 115.7ms, 0 | 20260505 21:34:14.732 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 21:34:14.732 | INFO | Uploaded 36GiB to GPU in 2.13381s (16.8712GiB/s) | 20260505 21:34:14.732 | INFO | Downloaded 38GiB to GPU in 2.22868s (17.0505GiB/s) | 20260505 21:34:19.870 | INFO | Analyzing chunk frames = [6472192,6472704) | 20260505 21:34:19.870 | INFO | MemInfo: { | 20260505 21:34:19.870 | INFO | "gpu" : " 17 GB", | 20260505 21:34:19.870 | INFO | "huge" : "", | 20260505 21:34:19.871 | INFO | "huge_shared" : " 27 GB", | 20260505 21:34:19.871 | INFO | "malloc" : " 14 GB", | 20260505 21:34:19.871 | INFO | "pinned" : " 29 GB" | 20260505 21:34:19.871 | INFO | } | 20260505 21:34:19.871 | INFO | Perf report for the last 5.13871 seconds: | 20260505 21:34:19.871 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:34:19.871 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:19.871 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:19.871 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:19.871 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260505 21:34:19.871 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:19.871 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:19.871 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 21:34:19.871 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:19.871 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:19.871 | INFO | Uploaded 36GiB to GPU in 2.11008s (17.061GiB/s) | 20260505 21:34:19.871 | INFO | Downloaded 34GiB to GPU in 2.02447s (16.7945GiB/s) | 20260505 21:34:25.014 | INFO | Analyzing chunk frames = [6472704,6473216) | 20260505 21:34:25.014 | INFO | MemInfo: { | 20260505 21:34:25.014 | INFO | "gpu" : " 17 GB", | 20260505 21:34:25.014 | INFO | "huge" : "", | 20260505 21:34:25.014 | INFO | "huge_shared" : " 27 GB", | 20260505 21:34:25.014 | INFO | "malloc" : " 14 GB", | 20260505 21:34:25.014 | INFO | "pinned" : " 29 GB" | 20260505 21:34:25.014 | INFO | } | 20260505 21:34:25.014 | INFO | Perf report for the last 5.14345 seconds: | 20260505 21:34:25.014 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:34:25.014 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:25.014 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:25.014 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:25.014 | INFO | ANALYSIS : 62.0%, 3.0, 95.5ms, 0 | 20260505 21:34:25.014 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:25.014 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:25.014 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 21:34:25.014 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:25.014 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:25.014 | INFO | Uploaded 36GiB to GPU in 2.14476s (16.7851GiB/s) | 20260505 21:34:25.015 | INFO | Downloaded 34GiB to GPU in 2.02597s (16.7821GiB/s) | 20260505 21:34:30.154 | INFO | Analyzing chunk frames = [6473216,6473728) | 20260505 21:34:30.154 | INFO | MemInfo: { | 20260505 21:34:30.154 | INFO | "gpu" : " 17 GB", | 20260505 21:34:30.154 | INFO | "huge" : "", | 20260505 21:34:30.154 | INFO | "huge_shared" : " 27 GB", | 20260505 21:34:30.154 | INFO | "malloc" : " 14 GB", | 20260505 21:34:30.154 | INFO | "pinned" : " 29 GB" | 20260505 21:34:30.155 | INFO | } | 20260505 21:34:30.155 | INFO | Perf report for the last 5.14039 seconds: | 20260505 21:34:30.155 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:34:30.155 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:30.155 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:30.155 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:30.155 | INFO | ANALYSIS : 62.0%, 3.0, 96.3ms, 0 | 20260505 21:34:30.155 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:30.155 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:30.155 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 21:34:30.155 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:30.155 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:30.155 | INFO | Uploaded 36GiB to GPU in 2.14175s (16.8087GiB/s) | 20260505 21:34:30.155 | INFO | Downloaded 34GiB to GPU in 2.04026s (16.6645GiB/s) | 20260505 21:34:35.294 | INFO | Analyzing chunk frames = [6473728,6474240) | 20260505 21:34:35.294 | INFO | MemInfo: { | 20260505 21:34:35.294 | INFO | "gpu" : " 17 GB", | 20260505 21:34:35.294 | INFO | "huge" : "", | 20260505 21:34:35.294 | INFO | "huge_shared" : " 27 GB", | 20260505 21:34:35.294 | INFO | "malloc" : " 14 GB", | 20260505 21:34:35.294 | INFO | "pinned" : " 29 GB" | 20260505 21:34:35.294 | INFO | } | 20260505 21:34:35.294 | INFO | Perf report for the last 5.13981 seconds: | 20260505 21:34:35.294 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:34:35.294 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:35.294 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:35.294 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:35.294 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260505 21:34:35.294 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:35.294 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:35.294 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:34:35.294 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:35.294 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:35.295 | INFO | Uploaded 36GiB to GPU in 2.16611s (16.6197GiB/s) | 20260505 21:34:35.295 | INFO | Downloaded 34GiB to GPU in 2.07209s (16.4085GiB/s) | 20260505 21:34:40.431 | INFO | Analyzing chunk frames = [6474240,6474752) | 20260505 21:34:40.431 | INFO | MemInfo: { | 20260505 21:34:40.431 | INFO | "gpu" : " 17 GB", | 20260505 21:34:40.431 | INFO | "huge" : "", | 20260505 21:34:40.431 | INFO | "huge_shared" : " 27 GB", | 20260505 21:34:40.431 | INFO | "malloc" : " 14 GB", | 20260505 21:34:40.431 | INFO | "pinned" : " 29 GB" | 20260505 21:34:40.431 | INFO | } | 20260505 21:34:40.431 | INFO | Perf report for the last 5.13702 seconds: | 20260505 21:34:40.431 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:34:40.431 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:40.431 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:40.431 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:40.432 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260505 21:34:40.432 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:40.432 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:40.432 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 21:34:40.432 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:40.432 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:40.432 | INFO | Uploaded 36GiB to GPU in 2.07567s (17.3438GiB/s) | 20260505 21:34:40.432 | INFO | Downloaded 34GiB to GPU in 2.02892s (16.7577GiB/s) | 20260505 21:34:45.573 | INFO | Analyzing chunk frames = [6474752,6475264) | 20260505 21:34:45.573 | INFO | MemInfo: { | 20260505 21:34:45.573 | INFO | "gpu" : " 17 GB", | 20260505 21:34:45.573 | INFO | "huge" : "", | 20260505 21:34:45.573 | INFO | "huge_shared" : " 27 GB", | 20260505 21:34:45.573 | INFO | "malloc" : " 14 GB", | 20260505 21:34:45.573 | INFO | "pinned" : " 29 GB" | 20260505 21:34:45.573 | INFO | } | 20260505 21:34:45.573 | INFO | Perf report for the last 5.14196 seconds: | 20260505 21:34:45.573 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:34:45.573 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:45.573 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:45.573 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:45.573 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 21:34:45.573 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:45.573 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:45.573 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 21:34:45.573 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:45.574 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:45.574 | INFO | Uploaded 36GiB to GPU in 2.15559s (16.7008GiB/s) | 20260505 21:34:45.574 | INFO | Downloaded 34GiB to GPU in 2.15277s (15.7936GiB/s) | 20260505 21:34:50.715 | INFO | Analyzing chunk frames = [6475264,6475776) | 20260505 21:34:50.715 | INFO | MemInfo: { | 20260505 21:34:50.715 | INFO | "gpu" : " 17 GB", | 20260505 21:34:50.715 | INFO | "huge" : "", | 20260505 21:34:50.715 | INFO | "huge_shared" : " 27 GB", | 20260505 21:34:50.715 | INFO | "malloc" : " 14 GB", | 20260505 21:34:50.715 | INFO | "pinned" : " 29 GB" | 20260505 21:34:50.715 | INFO | } | 20260505 21:34:50.715 | INFO | Perf report for the last 5.14158 seconds: | 20260505 21:34:50.715 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:34:50.715 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:50.715 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:50.715 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:50.715 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 21:34:50.715 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:50.715 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:50.715 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 21:34:50.715 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:50.715 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:50.715 | INFO | Uploaded 36GiB to GPU in 2.12683s (16.9266GiB/s) | 20260505 21:34:50.715 | INFO | Downloaded 34GiB to GPU in 2.00921s (16.9221GiB/s) | 20260505 21:34:50.839 | INFO | Starting a round of Baz IO | 20260505 21:34:53.989 | INFO | Populating RTMetrics report | 20260505 21:34:53.989 | INFO | RTMetrics report populated | 20260505 21:34:53.989 | INFO | Opening tmp RTMetrics json file | 20260505 21:34:53.991 | INFO | Serializing RTMetrics to json | 20260505 21:34:53.994 | INFO | Serialization done, swapping tmp file now... | 20260505 21:34:53.995 | INFO | RTMetrics done | 20260505 21:34:54.074 | INFO | Finished a round of Baz IO: Wrote 1.966 GiB in 3.234 seconds | 20260505 21:34:54.074 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.56% / 0.00% / 78.43% | 20260505 21:34:54.074 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.145566% / 0% / 0.000107288% | 20260505 21:34:54.074 | INFO | Baz IO overall: | 20260505 21:34:54.074 | INFO | BytesWritten : 3748284087508 | 20260505 21:34:54.074 | INFO | NumEvents : 1769693512396 | 20260505 21:34:55.846 | INFO | Analyzing chunk frames = [6475776,6476288) | 20260505 21:34:55.846 | INFO | MemInfo: { | 20260505 21:34:55.846 | INFO | "gpu" : " 17 GB", | 20260505 21:34:55.846 | INFO | "huge" : "", | 20260505 21:34:55.846 | INFO | "huge_shared" : " 27 GB", | 20260505 21:34:55.846 | INFO | "malloc" : " 14 GB", | 20260505 21:34:55.846 | INFO | "pinned" : " 29 GB" | 20260505 21:34:55.846 | INFO | } | 20260505 21:34:55.847 | INFO | Perf report for the last 5.13155 seconds: | 20260505 21:34:55.847 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:34:55.847 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:55.847 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:55.847 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:55.847 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260505 21:34:55.847 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:34:55.847 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260505 21:34:55.847 | INFO | PRE_HQ : 59.9%, 1.8, 55.1ms, 0 | 20260505 21:34:55.847 | INFO | BAZWRITER : 62.2%, 3.7, 116.7ms, 0 | 20260505 21:34:55.847 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 21:34:55.847 | INFO | Uploaded 36GiB to GPU in 2.1158s (17.0149GiB/s) | 20260505 21:34:55.847 | INFO | Downloaded 38GiB to GPU in 2.22489s (17.0795GiB/s) | 20260505 21:35:00.979 | INFO | Analyzing chunk frames = [6476288,6476800) | 20260505 21:35:00.979 | INFO | MemInfo: { | 20260505 21:35:00.979 | INFO | "gpu" : " 17 GB", | 20260505 21:35:00.979 | INFO | "huge" : "", | 20260505 21:35:00.980 | INFO | "huge_shared" : " 27 GB", | 20260505 21:35:00.980 | INFO | "malloc" : " 14 GB", | 20260505 21:35:00.980 | INFO | "pinned" : " 29 GB" | 20260505 21:35:00.980 | INFO | } | 20260505 21:35:00.980 | INFO | Perf report for the last 5.1331 seconds: | 20260505 21:35:00.980 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:35:00.980 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:00.980 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:00.980 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:00.980 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260505 21:35:00.980 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:00.980 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:00.980 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 21:35:00.980 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:00.980 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:00.980 | INFO | Uploaded 36GiB to GPU in 2.08079s (17.3011GiB/s) | 20260505 21:35:00.980 | INFO | Downloaded 34GiB to GPU in 2.02812s (16.7643GiB/s) | 20260505 21:35:06.124 | INFO | Analyzing chunk frames = [6476800,6477312) | 20260505 21:35:06.124 | INFO | MemInfo: { | 20260505 21:35:06.124 | INFO | "gpu" : " 17 GB", | 20260505 21:35:06.124 | INFO | "huge" : "", | 20260505 21:35:06.124 | INFO | "huge_shared" : " 27 GB", | 20260505 21:35:06.124 | INFO | "malloc" : " 14 GB", | 20260505 21:35:06.124 | INFO | "pinned" : " 29 GB" | 20260505 21:35:06.124 | INFO | } | 20260505 21:35:06.124 | INFO | Perf report for the last 5.14436 seconds: | 20260505 21:35:06.124 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:35:06.124 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:06.124 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:06.124 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:06.124 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260505 21:35:06.124 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:06.124 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:06.124 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:35:06.124 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:06.124 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:06.124 | INFO | Uploaded 36GiB to GPU in 2.13109s (16.8928GiB/s) | 20260505 21:35:06.124 | INFO | Downloaded 34GiB to GPU in 2.02535s (16.7872GiB/s) | 20260505 21:35:11.268 | INFO | Analyzing chunk frames = [6477312,6477824) | 20260505 21:35:11.268 | INFO | MemInfo: { | 20260505 21:35:11.268 | INFO | "gpu" : " 17 GB", | 20260505 21:35:11.268 | INFO | "huge" : "", | 20260505 21:35:11.269 | INFO | "huge_shared" : " 27 GB", | 20260505 21:35:11.269 | INFO | "malloc" : " 14 GB", | 20260505 21:35:11.269 | INFO | "pinned" : " 29 GB" | 20260505 21:35:11.269 | INFO | } | 20260505 21:35:11.269 | INFO | Perf report for the last 5.14471 seconds: | 20260505 21:35:11.269 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:35:11.269 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:11.269 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:11.269 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:11.269 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260505 21:35:11.269 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:11.269 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:11.269 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:35:11.269 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:11.269 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:11.269 | INFO | Uploaded 36GiB to GPU in 2.20896s (16.2973GiB/s) | 20260505 21:35:11.269 | INFO | Downloaded 34GiB to GPU in 2.06463s (16.4678GiB/s) | 20260505 21:35:16.413 | INFO | Analyzing chunk frames = [6477824,6478336) | 20260505 21:35:16.413 | INFO | MemInfo: { | 20260505 21:35:16.413 | INFO | "gpu" : " 17 GB", | 20260505 21:35:16.413 | INFO | "huge" : "", | 20260505 21:35:16.413 | INFO | "huge_shared" : " 27 GB", | 20260505 21:35:16.413 | INFO | "malloc" : " 14 GB", | 20260505 21:35:16.413 | INFO | "pinned" : " 29 GB" | 20260505 21:35:16.413 | INFO | } | 20260505 21:35:16.413 | INFO | Perf report for the last 5.14463 seconds: | 20260505 21:35:16.413 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:35:16.413 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:16.413 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:16.413 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:16.413 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260505 21:35:16.413 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:16.413 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:16.413 | INFO | PRE_HQ : 27.2%, 1.0, 14.2ms, 0 | 20260505 21:35:16.413 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:16.414 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:16.414 | INFO | Uploaded 36GiB to GPU in 2.08797s (17.2416GiB/s) | 20260505 21:35:16.414 | INFO | Downloaded 34GiB to GPU in 2.04228s (16.6481GiB/s) | 20260505 21:35:21.550 | INFO | Analyzing chunk frames = [6478336,6478848) | 20260505 21:35:21.551 | INFO | MemInfo: { | 20260505 21:35:21.551 | INFO | "gpu" : " 17 GB", | 20260505 21:35:21.551 | INFO | "huge" : "", | 20260505 21:35:21.551 | INFO | "huge_shared" : " 27 GB", | 20260505 21:35:21.551 | INFO | "malloc" : " 14 GB", | 20260505 21:35:21.551 | INFO | "pinned" : " 29 GB" | 20260505 21:35:21.551 | INFO | } | 20260505 21:35:21.551 | INFO | Perf report for the last 5.13739 seconds: | 20260505 21:35:21.551 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:35:21.551 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:21.551 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:21.551 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:21.551 | INFO | ANALYSIS : 61.6%, 3.0, 96.2ms, 0 | 20260505 21:35:21.551 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:21.551 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:21.551 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:35:21.551 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:21.551 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:21.551 | INFO | Uploaded 36GiB to GPU in 2.08707s (17.2491GiB/s) | 20260505 21:35:21.551 | INFO | Downloaded 34GiB to GPU in 2.03971s (16.669GiB/s) | 20260505 21:35:26.681 | INFO | Analyzing chunk frames = [6478848,6479360) | 20260505 21:35:26.681 | INFO | MemInfo: { | 20260505 21:35:26.681 | INFO | "gpu" : " 17 GB", | 20260505 21:35:26.681 | INFO | "huge" : "", | 20260505 21:35:26.681 | INFO | "huge_shared" : " 27 GB", | 20260505 21:35:26.681 | INFO | "malloc" : " 14 GB", | 20260505 21:35:26.681 | INFO | "pinned" : " 29 GB" | 20260505 21:35:26.681 | INFO | } | 20260505 21:35:26.681 | INFO | Perf report for the last 5.13007 seconds: | 20260505 21:35:26.681 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:35:26.681 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:26.681 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:26.681 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:26.681 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 21:35:26.681 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:26.681 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:26.681 | INFO | PRE_HQ : 27.7%, 1.0, 14.3ms, 0 | 20260505 21:35:26.681 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:26.681 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:26.681 | INFO | Uploaded 36GiB to GPU in 2.16866s (16.6001GiB/s) | 20260505 21:35:26.681 | INFO | Downloaded 34GiB to GPU in 2.10892s (16.122GiB/s) | 20260505 21:35:31.825 | INFO | Analyzing chunk frames = [6479360,6479872) | 20260505 21:35:31.825 | INFO | MemInfo: { | 20260505 21:35:31.825 | INFO | "gpu" : " 17 GB", | 20260505 21:35:31.826 | INFO | "huge" : "", | 20260505 21:35:31.826 | INFO | "huge_shared" : " 27 GB", | 20260505 21:35:31.826 | INFO | "malloc" : " 14 GB", | 20260505 21:35:31.826 | INFO | "pinned" : " 29 GB" | 20260505 21:35:31.826 | INFO | } | 20260505 21:35:31.826 | INFO | Perf report for the last 5.14479 seconds: | 20260505 21:35:31.826 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:35:31.826 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:31.826 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:31.826 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:31.826 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 21:35:31.826 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:31.826 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:31.826 | INFO | PRE_HQ : 27.0%, 1.0, 14.1ms, 0 | 20260505 21:35:31.826 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:31.826 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:31.826 | INFO | Uploaded 36GiB to GPU in 2.10198s (17.1267GiB/s) | 20260505 21:35:31.826 | INFO | Downloaded 34GiB to GPU in 2.0006s (16.9949GiB/s) | 20260505 21:35:31.826 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 21:35:31.826 | INFO | Total mem usage: 15.3 GiB | 20260505 21:35:31.826 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 21:35:31.826 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.95%) | 20260505 21:35:31.826 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 21:35:31.826 | INFO | High Water Breakdown: | 20260505 21:35:31.826 | INFO | 60.9% : 8.57 GiB ( 90.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 21:35:31.826 | INFO | 38.9% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 21:35:31.826 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 21:35:31.826 | INFO | ------------------------------------------ | 20260505 21:35:31.826 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:35:31.826 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 21:35:31.826 | INFO | Total mem usage: 26.5 GiB | 20260505 21:35:31.826 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 21:35:31.826 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 21:35:31.826 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 21:35:31.826 | INFO | High Water Breakdown: | 20260505 21:35:31.826 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:35:31.826 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:35:31.826 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:35:31.826 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:35:31.826 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:35:31.826 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:35:31.826 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 21:35:31.826 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 21:35:31.826 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:35:31.827 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 21:35:31.827 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 21:35:31.827 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 21:35:31.827 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 21:35:31.827 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 21:35:31.827 | INFO | ------------------------------------------ | 20260505 21:35:31.827 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:35:31.827 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 21:35:31.827 | INFO | Total mem usage: 25.6 GiB | 20260505 21:35:31.827 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 21:35:31.827 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 21:35:31.827 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 21:35:31.827 | INFO | High Water Breakdown: | 20260505 21:35:31.827 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 21:35:31.827 | INFO | ------------------------------------------ | 20260505 21:35:31.827 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:35:31.827 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 21:35:31.827 | INFO | Total mem usage: 15.7 GiB | 20260505 21:35:31.827 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 21:35:31.827 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 21:35:31.827 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 21:35:31.827 | INFO | High Water Breakdown: | 20260505 21:35:31.827 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 21:35:31.827 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 21:35:31.827 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 21:35:31.827 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 21:35:31.827 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 21:35:31.827 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 21:35:31.827 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 21:35:31.827 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 21:35:31.827 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 21:35:31.827 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:35:31.827 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 21:35:31.827 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:35:31.827 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 21:35:31.827 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 21:35:31.827 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 21:35:31.827 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 21:35:31.827 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:35:31.827 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 21:35:31.827 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:35:31.827 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:35:31.827 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:35:31.827 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 21:35:31.827 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 21:35:31.827 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:35:31.827 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 21:35:31.827 | INFO | ------------------------------------------ | 20260505 21:35:31.827 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:35:31.827 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 21:35:31.827 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:35:31.827 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:35:31.827 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.42e+01, 1.34e+01 / 1.18e+00 | 20260505 21:35:31.827 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.04e-01, 1.04e-02 / 5.75e-03 | 20260505 21:35:31.827 | INFO | PerfReport: 11.77%, Baseline -- 3.70e+00 / 4.55e+00, 5.63e-01 / 1.86e-01 | 20260505 21:35:31.827 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.51e+00, 1.90e-01 / 1.09e-01 | 20260505 21:35:31.827 | INFO | PerfReport: 15.95%, DME -- 4.01e+01 / 4.41e+01, 3.84e+00 / 1.33e+00 | 20260505 21:35:31.827 | INFO | PerfReport: 57.26%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.55e+00 / 1.03e+00 | 20260505 21:35:31.827 | INFO | PerfReport: 7.16%, PulseAccumulating -- 2.25e+00 / 2.81e+00, 4.30e-01 / 1.85e-01 | 20260505 21:35:31.827 | INFO | PerfReport: 3.19%, Metrics -- 1.00e+00 / 1.31e+00, 2.77e-01 / 1.37e-01 | 20260505 21:35:31.827 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.49e-01 / 4.07e-01, 6.09e-02 / 3.82e-02 | 20260505 21:35:31.827 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 21:35:31.827 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:35:31.827 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:35:31.827 | INFO | PerfReport: Full Execution -- 9.89e+01 / -nan, 2.51e+01 / 0.00e+00 | 20260505 21:35:31.827 | INFO | PerfReport: 22.40%, Upload -- 2.22e+01 / 0.00e+00, 7.60e+00 / 0.00e+00 | 20260505 21:35:31.827 | INFO | PerfReport: 21.79%, Download -- 2.16e+01 / 0.00e+00, 7.65e+00 / 0.00e+00 | 20260505 21:35:31.827 | INFO | PerfReport: 31.79%, Compute -- 3.15e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 21:35:31.827 | INFO | PerfReport: 2.23%, LockUpload -- 2.20e+00 / 0.00e+00, 7.93e+00 / 0.00e+00 | 20260505 21:35:31.827 | INFO | PerfReport: 1.31%, LockDownload -- 1.30e+00 / 0.00e+00, 2.86e+00 / 0.00e+00 | 20260505 21:35:31.827 | INFO | PerfReport: 20.47%, LockCompute -- 2.03e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260505 21:35:31.827 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.75e-02 / 0.00e+00 | 20260505 21:35:31.956 | INFO | Starting a round of Baz IO | 20260505 21:35:35.105 | INFO | Populating RTMetrics report | 20260505 21:35:35.105 | INFO | RTMetrics report populated | 20260505 21:35:35.105 | INFO | Opening tmp RTMetrics json file | 20260505 21:35:35.106 | INFO | Serializing RTMetrics to json | 20260505 21:35:35.109 | INFO | Serialization done, swapping tmp file now... | 20260505 21:35:35.111 | INFO | RTMetrics done | 20260505 21:35:35.191 | INFO | Finished a round of Baz IO: Wrote 1.964 GiB in 3.235 seconds | 20260505 21:35:35.191 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.56% / 0.00% / 78.44% | 20260505 21:35:35.191 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.144629% / 0% / 8.34465e-05% | 20260505 21:35:35.191 | INFO | Baz IO overall: | 20260505 21:35:35.191 | INFO | BytesWritten : 3750392973521 | 20260505 21:35:35.191 | INFO | NumEvents : 1770666411810 | 20260505 21:35:36.964 | INFO | Analyzing chunk frames = [6479872,6480384) | 20260505 21:35:36.964 | INFO | MemInfo: { | 20260505 21:35:36.964 | INFO | "gpu" : " 17 GB", | 20260505 21:35:36.964 | INFO | "huge" : "", | 20260505 21:35:36.964 | INFO | "huge_shared" : " 27 GB", | 20260505 21:35:36.964 | INFO | "malloc" : " 14 GB", | 20260505 21:35:36.964 | INFO | "pinned" : " 29 GB" | 20260505 21:35:36.964 | INFO | } | 20260505 21:35:36.964 | INFO | Perf report for the last 5.13832 seconds: | 20260505 21:35:36.964 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:35:36.964 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:36.964 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:36.964 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:36.964 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 21:35:36.964 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:36.964 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260505 21:35:36.964 | INFO | PRE_HQ : 60.3%, 1.8, 55.2ms, 0 | 20260505 21:35:36.964 | INFO | BAZWRITER : 62.2%, 3.6, 115.9ms, 0 | 20260505 21:35:36.964 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 21:35:36.964 | INFO | Uploaded 36GiB to GPU in 2.15299s (16.721GiB/s) | 20260505 21:35:36.964 | INFO | Downloaded 38GiB to GPU in 2.23769s (16.9818GiB/s) | 20260505 21:35:42.101 | INFO | Analyzing chunk frames = [6480384,6480896) | 20260505 21:35:42.101 | INFO | MemInfo: { | 20260505 21:35:42.101 | INFO | "gpu" : " 17 GB", | 20260505 21:35:42.101 | INFO | "huge" : "", | 20260505 21:35:42.101 | INFO | "huge_shared" : " 27 GB", | 20260505 21:35:42.101 | INFO | "malloc" : " 14 GB", | 20260505 21:35:42.101 | INFO | "pinned" : " 29 GB" | 20260505 21:35:42.101 | INFO | } | 20260505 21:35:42.101 | INFO | Perf report for the last 5.13681 seconds: | 20260505 21:35:42.101 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:35:42.101 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:42.101 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:42.101 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:42.101 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 21:35:42.101 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:42.101 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:42.101 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 21:35:42.101 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:42.101 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:42.101 | INFO | Uploaded 36GiB to GPU in 2.07337s (17.363GiB/s) | 20260505 21:35:42.101 | INFO | Downloaded 34GiB to GPU in 2.01883s (16.8415GiB/s) | 20260505 21:35:47.244 | INFO | Analyzing chunk frames = [6480896,6481408) | 20260505 21:35:47.244 | INFO | MemInfo: { | 20260505 21:35:47.244 | INFO | "gpu" : " 17 GB", | 20260505 21:35:47.244 | INFO | "huge" : "", | 20260505 21:35:47.244 | INFO | "huge_shared" : " 27 GB", | 20260505 21:35:47.244 | INFO | "malloc" : " 14 GB", | 20260505 21:35:47.244 | INFO | "pinned" : " 29 GB" | 20260505 21:35:47.244 | INFO | } | 20260505 21:35:47.244 | INFO | Perf report for the last 5.14323 seconds: | 20260505 21:35:47.244 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:35:47.244 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:47.244 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:47.244 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:47.244 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260505 21:35:47.244 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:47.244 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:47.244 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:35:47.244 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:47.244 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:47.244 | INFO | Uploaded 36GiB to GPU in 2.14249s (16.8029GiB/s) | 20260505 21:35:47.244 | INFO | Downloaded 34GiB to GPU in 2.01765s (16.8513GiB/s) | 20260505 21:35:52.385 | INFO | Analyzing chunk frames = [6481408,6481920) | 20260505 21:35:52.385 | INFO | MemInfo: { | 20260505 21:35:52.385 | INFO | "gpu" : " 17 GB", | 20260505 21:35:52.386 | INFO | "huge" : "", | 20260505 21:35:52.386 | INFO | "huge_shared" : " 27 GB", | 20260505 21:35:52.386 | INFO | "malloc" : " 14 GB", | 20260505 21:35:52.386 | INFO | "pinned" : " 29 GB" | 20260505 21:35:52.386 | INFO | } | 20260505 21:35:52.386 | INFO | Perf report for the last 5.14166 seconds: | 20260505 21:35:52.386 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:35:52.386 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:52.386 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:52.386 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:52.386 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260505 21:35:52.386 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:52.386 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:52.386 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:35:52.386 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:52.386 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:52.386 | INFO | Uploaded 36GiB to GPU in 2.20683s (16.313GiB/s) | 20260505 21:35:52.386 | INFO | Downloaded 34GiB to GPU in 2.07527s (16.3834GiB/s) | 20260505 21:35:57.524 | INFO | Analyzing chunk frames = [6481920,6482432) | 20260505 21:35:57.524 | INFO | MemInfo: { | 20260505 21:35:57.524 | INFO | "gpu" : " 17 GB", | 20260505 21:35:57.524 | INFO | "huge" : "", | 20260505 21:35:57.524 | INFO | "huge_shared" : " 27 GB", | 20260505 21:35:57.524 | INFO | "malloc" : " 14 GB", | 20260505 21:35:57.524 | INFO | "pinned" : " 29 GB" | 20260505 21:35:57.524 | INFO | } | 20260505 21:35:57.524 | INFO | Perf report for the last 5.13829 seconds: | 20260505 21:35:57.524 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:35:57.524 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:57.524 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:57.524 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:57.524 | INFO | ANALYSIS : 61.6%, 3.0, 96.2ms, 0 | 20260505 21:35:57.524 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:57.524 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:57.524 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 21:35:57.524 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:57.524 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:35:57.524 | INFO | Uploaded 36GiB to GPU in 2.12851s (16.9133GiB/s) | 20260505 21:35:57.524 | INFO | Downloaded 34GiB to GPU in 2.0636s (16.4761GiB/s) | 20260505 21:36:02.662 | INFO | Analyzing chunk frames = [6482432,6482944) | 20260505 21:36:02.662 | INFO | MemInfo: { | 20260505 21:36:02.662 | INFO | "gpu" : " 17 GB", | 20260505 21:36:02.662 | INFO | "huge" : "", | 20260505 21:36:02.662 | INFO | "huge_shared" : " 27 GB", | 20260505 21:36:02.663 | INFO | "malloc" : " 14 GB", | 20260505 21:36:02.663 | INFO | "pinned" : " 29 GB" | 20260505 21:36:02.663 | INFO | } | 20260505 21:36:02.663 | INFO | Perf report for the last 5.13864 seconds: | 20260505 21:36:02.663 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:36:02.663 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:02.663 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:02.663 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:02.663 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260505 21:36:02.663 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:02.663 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:02.663 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 21:36:02.663 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:02.663 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:02.663 | INFO | Uploaded 36GiB to GPU in 2.07137s (17.3798GiB/s) | 20260505 21:36:02.663 | INFO | Downloaded 34GiB to GPU in 2.0341s (16.715GiB/s) | 20260505 21:36:07.791 | INFO | Analyzing chunk frames = [6482944,6483456) | 20260505 21:36:07.791 | INFO | MemInfo: { | 20260505 21:36:07.791 | INFO | "gpu" : " 17 GB", | 20260505 21:36:07.791 | INFO | "huge" : "", | 20260505 21:36:07.791 | INFO | "huge_shared" : " 27 GB", | 20260505 21:36:07.791 | INFO | "malloc" : " 14 GB", | 20260505 21:36:07.791 | INFO | "pinned" : " 29 GB" | 20260505 21:36:07.791 | INFO | } | 20260505 21:36:07.791 | INFO | Perf report for the last 5.12816 seconds: | 20260505 21:36:07.791 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:36:07.791 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:07.791 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:07.791 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:07.791 | INFO | ANALYSIS : 62.9%, 3.0, 98.0ms, 0 | 20260505 21:36:07.791 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:07.791 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:07.791 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 21:36:07.791 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:07.791 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:07.791 | INFO | Uploaded 36GiB to GPU in 2.13709s (16.8454GiB/s) | 20260505 21:36:07.791 | INFO | Downloaded 34GiB to GPU in 2.13066s (15.9575GiB/s) | 20260505 21:36:12.931 | INFO | Analyzing chunk frames = [6483456,6483968) | 20260505 21:36:12.931 | INFO | MemInfo: { | 20260505 21:36:12.931 | INFO | "gpu" : " 17 GB", | 20260505 21:36:12.931 | INFO | "huge" : "", | 20260505 21:36:12.931 | INFO | "huge_shared" : " 27 GB", | 20260505 21:36:12.931 | INFO | "malloc" : " 14 GB", | 20260505 21:36:12.931 | INFO | "pinned" : " 29 GB" | 20260505 21:36:12.931 | INFO | } | 20260505 21:36:12.931 | INFO | Perf report for the last 5.14052 seconds: | 20260505 21:36:12.931 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:36:12.931 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:12.931 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:12.931 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:12.931 | INFO | ANALYSIS : 62.3%, 3.0, 97.4ms, 0 | 20260505 21:36:12.931 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:12.931 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:12.932 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:36:12.932 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:12.932 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:12.932 | INFO | Uploaded 36GiB to GPU in 2.1198s (16.9828GiB/s) | 20260505 21:36:12.932 | INFO | Downloaded 34GiB to GPU in 2.01973s (16.8339GiB/s) | 20260505 21:36:13.060 | INFO | Starting a round of Baz IO | 20260505 21:36:16.232 | INFO | Populating RTMetrics report | 20260505 21:36:16.232 | INFO | RTMetrics report populated | 20260505 21:36:16.232 | INFO | Opening tmp RTMetrics json file | 20260505 21:36:16.233 | INFO | Serializing RTMetrics to json | 20260505 21:36:16.234 | INFO | Serialization done, swapping tmp file now... | 20260505 21:36:16.235 | INFO | RTMetrics done | 20260505 21:36:16.315 | INFO | Finished a round of Baz IO: Wrote 1.963 GiB in 3.254 seconds | 20260505 21:36:16.315 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.56% / 0.00% / 78.44% | 20260505 21:36:16.315 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.143981% / 0% / 0.000127157% | 20260505 21:36:16.315 | INFO | Baz IO overall: | 20260505 21:36:16.315 | INFO | BytesWritten : 3752500874083 | 20260505 21:36:16.315 | INFO | NumEvents : 1771638314804 | 20260505 21:36:18.077 | INFO | Analyzing chunk frames = [6483968,6484480) | 20260505 21:36:18.077 | INFO | MemInfo: { | 20260505 21:36:18.077 | INFO | "gpu" : " 17 GB", | 20260505 21:36:18.077 | INFO | "huge" : "", | 20260505 21:36:18.077 | INFO | "huge_shared" : " 27 GB", | 20260505 21:36:18.077 | INFO | "malloc" : " 14 GB", | 20260505 21:36:18.077 | INFO | "pinned" : " 29 GB" | 20260505 21:36:18.077 | INFO | } | 20260505 21:36:18.077 | INFO | Perf report for the last 5.1456 seconds: | 20260505 21:36:18.077 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:36:18.077 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:18.077 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:18.077 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:18.077 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 21:36:18.077 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:18.077 | INFO | RT_METRICS : 43.9%, 1.0, 23.1ms, 0 | 20260505 21:36:18.077 | INFO | PRE_HQ : 60.1%, 1.8, 54.8ms, 0 | 20260505 21:36:18.077 | INFO | BAZWRITER : 62.8%, 3.6, 116.6ms, 0 | 20260505 21:36:18.077 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 21:36:18.077 | INFO | Uploaded 36GiB to GPU in 2.16541s (16.625GiB/s) | 20260505 21:36:18.077 | INFO | Downloaded 38GiB to GPU in 2.24896s (16.8967GiB/s) | 20260505 21:36:23.222 | INFO | Analyzing chunk frames = [6484480,6484992) | 20260505 21:36:23.222 | INFO | MemInfo: { | 20260505 21:36:23.222 | INFO | "gpu" : " 17 GB", | 20260505 21:36:23.222 | INFO | "huge" : "", | 20260505 21:36:23.222 | INFO | "huge_shared" : " 27 GB", | 20260505 21:36:23.222 | INFO | "malloc" : " 14 GB", | 20260505 21:36:23.222 | INFO | "pinned" : " 29 GB" | 20260505 21:36:23.222 | INFO | } | 20260505 21:36:23.222 | INFO | Perf report for the last 5.14547 seconds: | 20260505 21:36:23.222 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:36:23.222 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:23.222 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:23.223 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:23.223 | INFO | ANALYSIS : 61.8%, 3.0, 96.7ms, 0 | 20260505 21:36:23.223 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:23.223 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:23.223 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 21:36:23.223 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:23.223 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:23.223 | INFO | Uploaded 36GiB to GPU in 2.07358s (17.3613GiB/s) | 20260505 21:36:23.223 | INFO | Downloaded 34GiB to GPU in 2.0223s (16.8126GiB/s) | 20260505 21:36:28.362 | INFO | Analyzing chunk frames = [6484992,6485504) | 20260505 21:36:28.362 | INFO | MemInfo: { | 20260505 21:36:28.362 | INFO | "gpu" : " 17 GB", | 20260505 21:36:28.362 | INFO | "huge" : "", | 20260505 21:36:28.362 | INFO | "huge_shared" : " 27 GB", | 20260505 21:36:28.362 | INFO | "malloc" : " 14 GB", | 20260505 21:36:28.362 | INFO | "pinned" : " 29 GB" | 20260505 21:36:28.362 | INFO | } | 20260505 21:36:28.362 | INFO | Perf report for the last 5.13976 seconds: | 20260505 21:36:28.362 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:36:28.362 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:28.362 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:28.362 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:28.362 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260505 21:36:28.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:28.362 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:28.362 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 21:36:28.362 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:28.362 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:28.362 | INFO | Uploaded 36GiB to GPU in 2.14055s (16.8181GiB/s) | 20260505 21:36:28.363 | INFO | Downloaded 34GiB to GPU in 2.03369s (16.7184GiB/s) | 20260505 21:36:33.495 | INFO | Analyzing chunk frames = [6485504,6486016) | 20260505 21:36:33.495 | INFO | MemInfo: { | 20260505 21:36:33.495 | INFO | "gpu" : " 17 GB", | 20260505 21:36:33.495 | INFO | "huge" : "", | 20260505 21:36:33.495 | INFO | "huge_shared" : " 27 GB", | 20260505 21:36:33.495 | INFO | "malloc" : " 14 GB", | 20260505 21:36:33.495 | INFO | "pinned" : " 29 GB" | 20260505 21:36:33.495 | INFO | } | 20260505 21:36:33.495 | INFO | Perf report for the last 5.13281 seconds: | 20260505 21:36:33.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:36:33.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:33.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:33.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:33.495 | INFO | ANALYSIS : 62.4%, 3.0, 96.8ms, 0 | 20260505 21:36:33.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:33.495 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:33.495 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:36:33.495 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:33.495 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:33.495 | INFO | Uploaded 36GiB to GPU in 2.20805s (16.304GiB/s) | 20260505 21:36:33.495 | INFO | Downloaded 34GiB to GPU in 2.07853s (16.3577GiB/s) | 20260505 21:36:38.636 | INFO | Analyzing chunk frames = [6486016,6486528) | 20260505 21:36:38.636 | INFO | MemInfo: { | 20260505 21:36:38.636 | INFO | "gpu" : " 17 GB", | 20260505 21:36:38.636 | INFO | "huge" : "", | 20260505 21:36:38.636 | INFO | "huge_shared" : " 27 GB", | 20260505 21:36:38.636 | INFO | "malloc" : " 14 GB", | 20260505 21:36:38.636 | INFO | "pinned" : " 29 GB" | 20260505 21:36:38.636 | INFO | } | 20260505 21:36:38.636 | INFO | Perf report for the last 5.1405 seconds: | 20260505 21:36:38.636 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:36:38.636 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:38.636 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:38.636 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:38.636 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260505 21:36:38.636 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:38.636 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:38.636 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 21:36:38.636 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:38.636 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:38.636 | INFO | Uploaded 36GiB to GPU in 2.13979s (16.8241GiB/s) | 20260505 21:36:38.636 | INFO | Downloaded 34GiB to GPU in 2.06545s (16.4613GiB/s) | 20260505 21:36:43.775 | INFO | Analyzing chunk frames = [6486528,6487040) | 20260505 21:36:43.775 | INFO | MemInfo: { | 20260505 21:36:43.775 | INFO | "gpu" : " 17 GB", | 20260505 21:36:43.775 | INFO | "huge" : "", | 20260505 21:36:43.775 | INFO | "huge_shared" : " 27 GB", | 20260505 21:36:43.775 | INFO | "malloc" : " 14 GB", | 20260505 21:36:43.775 | INFO | "pinned" : " 29 GB" | 20260505 21:36:43.775 | INFO | } | 20260505 21:36:43.775 | INFO | Perf report for the last 5.13947 seconds: | 20260505 21:36:43.775 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:36:43.775 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:43.775 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:43.775 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:43.775 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260505 21:36:43.775 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:43.775 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:43.775 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 21:36:43.775 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:43.775 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:43.775 | INFO | Uploaded 36GiB to GPU in 2.08123s (17.2975GiB/s) | 20260505 21:36:43.775 | INFO | Downloaded 34GiB to GPU in 2.03412s (16.7149GiB/s) | 20260505 21:36:48.915 | INFO | Analyzing chunk frames = [6487040,6487552) | 20260505 21:36:48.915 | INFO | MemInfo: { | 20260505 21:36:48.915 | INFO | "gpu" : " 17 GB", | 20260505 21:36:48.915 | INFO | "huge" : "", | 20260505 21:36:48.915 | INFO | "huge_shared" : " 27 GB", | 20260505 21:36:48.915 | INFO | "malloc" : " 14 GB", | 20260505 21:36:48.915 | INFO | "pinned" : " 29 GB" | 20260505 21:36:48.915 | INFO | } | 20260505 21:36:48.915 | INFO | Perf report for the last 5.14004 seconds: | 20260505 21:36:48.915 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:36:48.915 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:48.915 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:48.915 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:48.915 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260505 21:36:48.915 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:48.915 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:48.915 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 21:36:48.915 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:48.915 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:48.915 | INFO | Uploaded 36GiB to GPU in 2.1387s (16.8326GiB/s) | 20260505 21:36:48.915 | INFO | Downloaded 34GiB to GPU in 2.07798s (16.3621GiB/s) | 20260505 21:36:54.049 | INFO | Analyzing chunk frames = [6487552,6488064) | 20260505 21:36:54.049 | INFO | MemInfo: { | 20260505 21:36:54.049 | INFO | "gpu" : " 17 GB", | 20260505 21:36:54.049 | INFO | "huge" : "", | 20260505 21:36:54.049 | INFO | "huge_shared" : " 27 GB", | 20260505 21:36:54.049 | INFO | "malloc" : " 14 GB", | 20260505 21:36:54.049 | INFO | "pinned" : " 29 GB" | 20260505 21:36:54.049 | INFO | } | 20260505 21:36:54.049 | INFO | Perf report for the last 5.134 seconds: | 20260505 21:36:54.049 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:36:54.049 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:54.049 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:54.049 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:54.049 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260505 21:36:54.049 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:54.049 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:54.049 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:36:54.049 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:54.049 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:54.049 | INFO | Uploaded 36GiB to GPU in 2.11778s (16.999GiB/s) | 20260505 21:36:54.049 | INFO | Downloaded 34GiB to GPU in 2.01987s (16.8327GiB/s) | 20260505 21:36:54.050 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 21:36:54.050 | INFO | Total mem usage: 15.3 GiB | 20260505 21:36:54.050 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 21:36:54.050 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -9.01%) | 20260505 21:36:54.050 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 21:36:54.050 | INFO | High Water Breakdown: | 20260505 21:36:54.050 | INFO | 60.9% : 8.56 GiB ( 90% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 21:36:54.050 | INFO | 39% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 21:36:54.050 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 21:36:54.050 | INFO | ------------------------------------------ | 20260505 21:36:54.050 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:36:54.050 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 21:36:54.050 | INFO | Total mem usage: 26.5 GiB | 20260505 21:36:54.050 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 21:36:54.050 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 21:36:54.050 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 21:36:54.050 | INFO | High Water Breakdown: | 20260505 21:36:54.050 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:36:54.050 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:36:54.050 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:36:54.050 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:36:54.050 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:36:54.050 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:36:54.050 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 21:36:54.050 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 21:36:54.050 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:36:54.050 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 21:36:54.050 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 21:36:54.050 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 21:36:54.050 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 21:36:54.050 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 21:36:54.050 | INFO | ------------------------------------------ | 20260505 21:36:54.050 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:36:54.050 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 21:36:54.050 | INFO | Total mem usage: 25.6 GiB | 20260505 21:36:54.050 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 21:36:54.050 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 21:36:54.050 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 21:36:54.050 | INFO | High Water Breakdown: | 20260505 21:36:54.050 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 21:36:54.050 | INFO | ------------------------------------------ | 20260505 21:36:54.050 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:36:54.050 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 21:36:54.050 | INFO | Total mem usage: 15.7 GiB | 20260505 21:36:54.050 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 21:36:54.050 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 21:36:54.050 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 21:36:54.050 | INFO | High Water Breakdown: | 20260505 21:36:54.050 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 21:36:54.050 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 21:36:54.050 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 21:36:54.050 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 21:36:54.050 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 21:36:54.050 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 21:36:54.050 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 21:36:54.050 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 21:36:54.050 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 21:36:54.050 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:36:54.050 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 21:36:54.050 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:36:54.050 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 21:36:54.050 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 21:36:54.050 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 21:36:54.050 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 21:36:54.050 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:36:54.050 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 21:36:54.050 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:36:54.050 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:36:54.050 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:36:54.050 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 21:36:54.050 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 21:36:54.050 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:36:54.051 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 21:36:54.051 | INFO | ------------------------------------------ | 20260505 21:36:54.051 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:36:54.051 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 21:36:54.051 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:36:54.051 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:36:54.051 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.41e+01, 1.36e+01 / 1.24e+00 | 20260505 21:36:54.051 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.04e-01, 1.05e-02 / 4.86e-03 | 20260505 21:36:54.051 | INFO | PerfReport: 11.78%, Baseline -- 3.70e+00 / 4.55e+00, 5.61e-01 / 1.76e-01 | 20260505 21:36:54.051 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.52e+00, 1.87e-01 / 9.90e-02 | 20260505 21:36:54.051 | INFO | PerfReport: 15.95%, DME -- 4.01e+01 / 4.39e+01, 3.89e+00 / 1.30e+00 | 20260505 21:36:54.051 | INFO | PerfReport: 57.21%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.50e+00 / 1.03e+00 | 20260505 21:36:54.051 | INFO | PerfReport: 7.19%, PulseAccumulating -- 2.26e+00 / 2.80e+00, 4.32e-01 / 2.31e-01 | 20260505 21:36:54.051 | INFO | PerfReport: 3.19%, Metrics -- 1.00e+00 / 1.33e+00, 2.71e-01 / 1.38e-01 | 20260505 21:36:54.051 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.49e-01 / 4.15e-01, 6.12e-02 / 3.92e-02 | 20260505 21:36:54.051 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 21:36:54.051 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:36:54.051 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:36:54.051 | INFO | PerfReport: Full Execution -- 9.90e+01 / -nan, 2.49e+01 / 0.00e+00 | 20260505 21:36:54.051 | INFO | PerfReport: 22.42%, Upload -- 2.22e+01 / 0.00e+00, 7.69e+00 / 0.00e+00 | 20260505 21:36:54.051 | INFO | PerfReport: 21.82%, Download -- 2.16e+01 / 0.00e+00, 7.65e+00 / 0.00e+00 | 20260505 21:36:54.051 | INFO | PerfReport: 31.71%, Compute -- 3.14e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 21:36:54.051 | INFO | PerfReport: 2.23%, LockUpload -- 2.20e+00 / 0.00e+00, 7.92e+00 / 0.00e+00 | 20260505 21:36:54.051 | INFO | PerfReport: 1.32%, LockDownload -- 1.31e+00 / 0.00e+00, 2.85e+00 / 0.00e+00 | 20260505 21:36:54.051 | INFO | PerfReport: 20.48%, LockCompute -- 2.03e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 21:36:54.051 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 2.40e-02 / 0.00e+00 | 20260505 21:36:54.180 | INFO | Starting a round of Baz IO | 20260505 21:36:57.342 | INFO | Populating RTMetrics report | 20260505 21:36:57.342 | INFO | RTMetrics report populated | 20260505 21:36:57.342 | INFO | Opening tmp RTMetrics json file | 20260505 21:36:57.343 | INFO | Serializing RTMetrics to json | 20260505 21:36:57.344 | INFO | Serialization done, swapping tmp file now... | 20260505 21:36:57.345 | INFO | RTMetrics done | 20260505 21:36:57.431 | INFO | Finished a round of Baz IO: Wrote 1.964 GiB in 3.25 seconds | 20260505 21:36:57.431 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.56% / 0.00% / 78.44% | 20260505 21:36:57.431 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.14348% / 3.97364e-06% / 9.13938e-05% | 20260505 21:36:57.431 | INFO | Baz IO overall: | 20260505 21:36:57.431 | INFO | BytesWritten : 3754609428360 | 20260505 21:36:57.431 | INFO | NumEvents : 1772611023913 | 20260505 21:36:59.190 | INFO | Analyzing chunk frames = [6488064,6488576) | 20260505 21:36:59.190 | INFO | MemInfo: { | 20260505 21:36:59.190 | INFO | "gpu" : " 17 GB", | 20260505 21:36:59.190 | INFO | "huge" : "", | 20260505 21:36:59.190 | INFO | "huge_shared" : " 27 GB", | 20260505 21:36:59.190 | INFO | "malloc" : " 14 GB", | 20260505 21:36:59.190 | INFO | "pinned" : " 29 GB" | 20260505 21:36:59.190 | INFO | } | 20260505 21:36:59.190 | INFO | Perf report for the last 5.14064 seconds: | 20260505 21:36:59.190 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:36:59.190 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:59.190 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:59.190 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:59.190 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 21:36:59.190 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:36:59.190 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260505 21:36:59.190 | INFO | PRE_HQ : 59.9%, 1.8, 54.7ms, 0 | 20260505 21:36:59.190 | INFO | BAZWRITER : 62.7%, 3.6, 116.9ms, 0 | 20260505 21:36:59.190 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 21:36:59.190 | INFO | Uploaded 36GiB to GPU in 2.14466s (16.7859GiB/s) | 20260505 21:36:59.190 | INFO | Downloaded 38GiB to GPU in 2.24201s (16.9491GiB/s) | 20260505 21:37:04.340 | INFO | Analyzing chunk frames = [6488576,6489088) | 20260505 21:37:04.340 | INFO | MemInfo: { | 20260505 21:37:04.340 | INFO | "gpu" : " 17 GB", | 20260505 21:37:04.340 | INFO | "huge" : "", | 20260505 21:37:04.340 | INFO | "huge_shared" : " 27 GB", | 20260505 21:37:04.340 | INFO | "malloc" : " 14 GB", | 20260505 21:37:04.340 | INFO | "pinned" : " 29 GB" | 20260505 21:37:04.340 | INFO | } | 20260505 21:37:04.340 | INFO | Perf report for the last 5.15067 seconds: | 20260505 21:37:04.340 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:37:04.340 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:04.340 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:04.340 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:04.341 | INFO | ANALYSIS : 62.1%, 3.0, 97.1ms, 0 | 20260505 21:37:04.341 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:04.341 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:04.341 | INFO | PRE_HQ : 26.8%, 1.0, 14.0ms, 0 | 20260505 21:37:04.341 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:04.341 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:04.341 | INFO | Uploaded 36GiB to GPU in 2.10544s (17.0985GiB/s) | 20260505 21:37:04.341 | INFO | Downloaded 34GiB to GPU in 2.02599s (16.7819GiB/s) | 20260505 21:37:09.474 | INFO | Analyzing chunk frames = [6489088,6489600) | 20260505 21:37:09.474 | INFO | MemInfo: { | 20260505 21:37:09.474 | INFO | "gpu" : " 17 GB", | 20260505 21:37:09.474 | INFO | "huge" : "", | 20260505 21:37:09.474 | INFO | "huge_shared" : " 27 GB", | 20260505 21:37:09.475 | INFO | "malloc" : " 14 GB", | 20260505 21:37:09.475 | INFO | "pinned" : " 29 GB" | 20260505 21:37:09.475 | INFO | } | 20260505 21:37:09.475 | INFO | Perf report for the last 5.13424 seconds: | 20260505 21:37:09.475 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:37:09.475 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:09.475 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:09.475 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:09.475 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260505 21:37:09.475 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:09.475 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:09.475 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:37:09.475 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:09.475 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:09.475 | INFO | Uploaded 36GiB to GPU in 2.15631s (16.6952GiB/s) | 20260505 21:37:09.475 | INFO | Downloaded 34GiB to GPU in 2.05735s (16.5261GiB/s) | 20260505 21:37:14.609 | INFO | Analyzing chunk frames = [6489600,6490112) | 20260505 21:37:14.609 | INFO | MemInfo: { | 20260505 21:37:14.609 | INFO | "gpu" : " 17 GB", | 20260505 21:37:14.609 | INFO | "huge" : "", | 20260505 21:37:14.609 | INFO | "huge_shared" : " 27 GB", | 20260505 21:37:14.610 | INFO | "malloc" : " 14 GB", | 20260505 21:37:14.610 | INFO | "pinned" : " 29 GB" | 20260505 21:37:14.610 | INFO | } | 20260505 21:37:14.610 | INFO | Perf report for the last 5.13487 seconds: | 20260505 21:37:14.610 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:37:14.610 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:14.610 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:14.610 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:14.610 | INFO | ANALYSIS : 61.9%, 3.0, 96.0ms, 0 | 20260505 21:37:14.610 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:14.610 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:14.610 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:37:14.610 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:14.610 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:14.610 | INFO | Uploaded 36GiB to GPU in 2.129s (16.9094GiB/s) | 20260505 21:37:14.610 | INFO | Downloaded 34GiB to GPU in 2.02203s (16.8148GiB/s) | 20260505 21:37:19.742 | INFO | Analyzing chunk frames = [6490112,6490624) | 20260505 21:37:19.742 | INFO | MemInfo: { | 20260505 21:37:19.742 | INFO | "gpu" : " 17 GB", | 20260505 21:37:19.742 | INFO | "huge" : "", | 20260505 21:37:19.742 | INFO | "huge_shared" : " 27 GB", | 20260505 21:37:19.742 | INFO | "malloc" : " 14 GB", | 20260505 21:37:19.742 | INFO | "pinned" : " 29 GB" | 20260505 21:37:19.742 | INFO | } | 20260505 21:37:19.742 | INFO | Perf report for the last 5.13277 seconds: | 20260505 21:37:19.742 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:37:19.742 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:19.742 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:19.742 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:19.742 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260505 21:37:19.742 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:19.742 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:19.742 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:37:19.743 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:19.743 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:19.743 | INFO | Uploaded 36GiB to GPU in 2.12717s (16.9239GiB/s) | 20260505 21:37:19.743 | INFO | Downloaded 34GiB to GPU in 2.06243s (16.4854GiB/s) | 20260505 21:37:24.888 | INFO | Analyzing chunk frames = [6490624,6491136) | 20260505 21:37:24.888 | INFO | MemInfo: { | 20260505 21:37:24.888 | INFO | "gpu" : " 17 GB", | 20260505 21:37:24.888 | INFO | "huge" : "", | 20260505 21:37:24.888 | INFO | "huge_shared" : " 27 GB", | 20260505 21:37:24.888 | INFO | "malloc" : " 14 GB", | 20260505 21:37:24.888 | INFO | "pinned" : " 29 GB" | 20260505 21:37:24.888 | INFO | } | 20260505 21:37:24.888 | INFO | Perf report for the last 5.14542 seconds: | 20260505 21:37:24.888 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:37:24.888 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:24.888 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:24.888 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:24.888 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260505 21:37:24.888 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:24.888 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:24.888 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 21:37:24.888 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:24.888 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:24.888 | INFO | Uploaded 36GiB to GPU in 2.16475s (16.6301GiB/s) | 20260505 21:37:24.888 | INFO | Downloaded 34GiB to GPU in 2.06965s (16.4279GiB/s) | 20260505 21:37:30.030 | INFO | Analyzing chunk frames = [6491136,6491648) | 20260505 21:37:30.030 | INFO | MemInfo: { | 20260505 21:37:30.030 | INFO | "gpu" : " 17 GB", | 20260505 21:37:30.030 | INFO | "huge" : "", | 20260505 21:37:30.030 | INFO | "huge_shared" : " 27 GB", | 20260505 21:37:30.030 | INFO | "malloc" : " 14 GB", | 20260505 21:37:30.030 | INFO | "pinned" : " 29 GB" | 20260505 21:37:30.030 | INFO | } | 20260505 21:37:30.030 | INFO | Perf report for the last 5.14212 seconds: | 20260505 21:37:30.030 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:37:30.030 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:30.030 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:30.030 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:30.030 | INFO | ANALYSIS : 62.7%, 3.0, 98.0ms, 0 | 20260505 21:37:30.030 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:30.030 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:30.030 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 21:37:30.030 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:30.030 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:30.030 | INFO | Uploaded 36GiB to GPU in 2.13684s (16.8473GiB/s) | 20260505 21:37:30.030 | INFO | Downloaded 34GiB to GPU in 2.10804s (16.1287GiB/s) | 20260505 21:37:35.170 | INFO | Analyzing chunk frames = [6491648,6492160) | 20260505 21:37:35.170 | INFO | MemInfo: { | 20260505 21:37:35.170 | INFO | "gpu" : " 17 GB", | 20260505 21:37:35.170 | INFO | "huge" : "", | 20260505 21:37:35.170 | INFO | "huge_shared" : " 27 GB", | 20260505 21:37:35.171 | INFO | "malloc" : " 14 GB", | 20260505 21:37:35.171 | INFO | "pinned" : " 29 GB" | 20260505 21:37:35.171 | INFO | } | 20260505 21:37:35.171 | INFO | Perf report for the last 5.14071 seconds: | 20260505 21:37:35.171 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:37:35.171 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:35.171 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:35.171 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:35.171 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260505 21:37:35.171 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:35.171 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:35.171 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:37:35.171 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:35.171 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:35.171 | INFO | Uploaded 36GiB to GPU in 2.11831s (16.9947GiB/s) | 20260505 21:37:35.171 | INFO | Downloaded 34GiB to GPU in 2.011s (16.907GiB/s) | 20260505 21:37:35.300 | INFO | Starting a round of Baz IO | 20260505 21:37:38.451 | INFO | Populating RTMetrics report | 20260505 21:37:38.451 | INFO | RTMetrics report populated | 20260505 21:37:38.452 | INFO | Opening tmp RTMetrics json file | 20260505 21:37:38.452 | INFO | Serializing RTMetrics to json | 20260505 21:37:38.454 | INFO | Serialization done, swapping tmp file now... | 20260505 21:37:38.454 | INFO | RTMetrics done | 20260505 21:37:38.532 | INFO | Finished a round of Baz IO: Wrote 1.966 GiB in 3.231 seconds | 20260505 21:37:38.532 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.56% / 0.00% / 78.44% | 20260505 21:37:38.532 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.14441% / 0% / 8.74201e-05% | 20260505 21:37:38.532 | INFO | Baz IO overall: | 20260505 21:37:38.532 | INFO | BytesWritten : 3756720160453 | 20260505 21:37:38.532 | INFO | NumEvents : 1773585648457 | 20260505 21:37:40.308 | INFO | Analyzing chunk frames = [6492160,6492672) | 20260505 21:37:40.308 | INFO | MemInfo: { | 20260505 21:37:40.308 | INFO | "gpu" : " 17 GB", | 20260505 21:37:40.308 | INFO | "huge" : "", | 20260505 21:37:40.308 | INFO | "huge_shared" : " 27 GB", | 20260505 21:37:40.308 | INFO | "malloc" : " 14 GB", | 20260505 21:37:40.308 | INFO | "pinned" : " 29 GB" | 20260505 21:37:40.308 | INFO | } | 20260505 21:37:40.308 | INFO | Perf report for the last 5.13733 seconds: | 20260505 21:37:40.308 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:37:40.308 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:40.308 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:40.308 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:40.308 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 21:37:40.308 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:40.308 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260505 21:37:40.308 | INFO | PRE_HQ : 59.6%, 1.8, 54.6ms, 0 | 20260505 21:37:40.308 | INFO | BAZWRITER : 62.2%, 3.6, 116.2ms, 0 | 20260505 21:37:40.308 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 21:37:40.308 | INFO | Uploaded 36GiB to GPU in 2.12558s (16.9365GiB/s) | 20260505 21:37:40.308 | INFO | Downloaded 38GiB to GPU in 2.2255s (17.0748GiB/s) | 20260505 21:37:45.440 | INFO | Analyzing chunk frames = [6492672,6493184) | 20260505 21:37:45.440 | INFO | MemInfo: { | 20260505 21:37:45.440 | INFO | "gpu" : " 17 GB", | 20260505 21:37:45.440 | INFO | "huge" : "", | 20260505 21:37:45.440 | INFO | "huge_shared" : " 27 GB", | 20260505 21:37:45.440 | INFO | "malloc" : " 14 GB", | 20260505 21:37:45.440 | INFO | "pinned" : " 29 GB" | 20260505 21:37:45.440 | INFO | } | 20260505 21:37:45.440 | INFO | Perf report for the last 5.13166 seconds: | 20260505 21:37:45.440 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:37:45.440 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:45.440 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:45.440 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:45.440 | INFO | ANALYSIS : 62.4%, 3.0, 96.2ms, 0 | 20260505 21:37:45.440 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:45.440 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:45.440 | INFO | PRE_HQ : 26.9%, 1.0, 13.8ms, 0 | 20260505 21:37:45.440 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:45.440 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:45.440 | INFO | Uploaded 36GiB to GPU in 2.10396s (17.1106GiB/s) | 20260505 21:37:45.440 | INFO | Downloaded 34GiB to GPU in 2.02793s (16.7659GiB/s) | 20260505 21:37:50.593 | INFO | Analyzing chunk frames = [6493184,6493696) | 20260505 21:37:50.593 | INFO | MemInfo: { | 20260505 21:37:50.593 | INFO | "gpu" : " 17 GB", | 20260505 21:37:50.593 | INFO | "huge" : "", | 20260505 21:37:50.593 | INFO | "huge_shared" : " 27 GB", | 20260505 21:37:50.593 | INFO | "malloc" : " 14 GB", | 20260505 21:37:50.593 | INFO | "pinned" : " 29 GB" | 20260505 21:37:50.593 | INFO | } | 20260505 21:37:50.593 | INFO | Perf report for the last 5.15353 seconds: | 20260505 21:37:50.593 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:37:50.593 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:50.593 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:50.593 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:50.593 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 21:37:50.593 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:50.593 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:50.593 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 21:37:50.593 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:50.593 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:50.593 | INFO | Uploaded 36GiB to GPU in 2.12641s (16.9299GiB/s) | 20260505 21:37:50.593 | INFO | Downloaded 34GiB to GPU in 2.02647s (16.7779GiB/s) | 20260505 21:37:55.727 | INFO | Analyzing chunk frames = [6493696,6494208) | 20260505 21:37:55.727 | INFO | MemInfo: { | 20260505 21:37:55.727 | INFO | "gpu" : " 17 GB", | 20260505 21:37:55.727 | INFO | "huge" : "", | 20260505 21:37:55.727 | INFO | "huge_shared" : " 27 GB", | 20260505 21:37:55.727 | INFO | "malloc" : " 14 GB", | 20260505 21:37:55.727 | INFO | "pinned" : " 29 GB" | 20260505 21:37:55.727 | INFO | } | 20260505 21:37:55.727 | INFO | Perf report for the last 5.13389 seconds: | 20260505 21:37:55.727 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:37:55.727 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:55.727 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:55.727 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:55.727 | INFO | ANALYSIS : 61.8%, 3.0, 96.0ms, 0 | 20260505 21:37:55.727 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:55.727 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:55.727 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 21:37:55.727 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:55.727 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:37:55.727 | INFO | Uploaded 36GiB to GPU in 2.13382s (16.8711GiB/s) | 20260505 21:37:55.727 | INFO | Downloaded 34GiB to GPU in 2.01244s (16.895GiB/s) | 20260505 21:38:00.865 | INFO | Analyzing chunk frames = [6494208,6494720) | 20260505 21:38:00.865 | INFO | MemInfo: { | 20260505 21:38:00.865 | INFO | "gpu" : " 17 GB", | 20260505 21:38:00.865 | INFO | "huge" : "", | 20260505 21:38:00.865 | INFO | "huge_shared" : " 27 GB", | 20260505 21:38:00.865 | INFO | "malloc" : " 14 GB", | 20260505 21:38:00.865 | INFO | "pinned" : " 29 GB" | 20260505 21:38:00.865 | INFO | } | 20260505 21:38:00.865 | INFO | Perf report for the last 5.13795 seconds: | 20260505 21:38:00.865 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:38:00.865 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:00.865 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:00.865 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:00.865 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260505 21:38:00.865 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:00.865 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:00.865 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:38:00.865 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:00.865 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:00.865 | INFO | Uploaded 36GiB to GPU in 2.15821s (16.6805GiB/s) | 20260505 21:38:00.865 | INFO | Downloaded 34GiB to GPU in 2.06963s (16.428GiB/s) | 20260505 21:38:06.004 | INFO | Analyzing chunk frames = [6494720,6495232) | 20260505 21:38:06.004 | INFO | MemInfo: { | 20260505 21:38:06.004 | INFO | "gpu" : " 17 GB", | 20260505 21:38:06.004 | INFO | "huge" : "", | 20260505 21:38:06.004 | INFO | "huge_shared" : " 27 GB", | 20260505 21:38:06.004 | INFO | "malloc" : " 14 GB", | 20260505 21:38:06.004 | INFO | "pinned" : " 29 GB" | 20260505 21:38:06.004 | INFO | } | 20260505 21:38:06.004 | INFO | Perf report for the last 5.1388 seconds: | 20260505 21:38:06.004 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:38:06.004 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:06.004 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:06.004 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:06.004 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 21:38:06.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:06.004 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:06.004 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:38:06.004 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:06.004 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:06.004 | INFO | Uploaded 36GiB to GPU in 2.15683s (16.6911GiB/s) | 20260505 21:38:06.004 | INFO | Downloaded 34GiB to GPU in 2.06777s (16.4428GiB/s) | 20260505 21:38:11.143 | INFO | Analyzing chunk frames = [6495232,6495744) | 20260505 21:38:11.143 | INFO | MemInfo: { | 20260505 21:38:11.143 | INFO | "gpu" : " 17 GB", | 20260505 21:38:11.143 | INFO | "huge" : "", | 20260505 21:38:11.143 | INFO | "huge_shared" : " 27 GB", | 20260505 21:38:11.143 | INFO | "malloc" : " 14 GB", | 20260505 21:38:11.143 | INFO | "pinned" : " 29 GB" | 20260505 21:38:11.143 | INFO | } | 20260505 21:38:11.143 | INFO | Perf report for the last 5.1391 seconds: | 20260505 21:38:11.143 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:38:11.143 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:11.143 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:11.143 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:11.143 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 21:38:11.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:11.143 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:11.143 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 21:38:11.143 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:11.143 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:11.143 | INFO | Uploaded 36GiB to GPU in 2.16308s (16.6429GiB/s) | 20260505 21:38:11.143 | INFO | Downloaded 34GiB to GPU in 2.12693s (15.9855GiB/s) | 20260505 21:38:16.285 | INFO | Analyzing chunk frames = [6495744,6496256) | 20260505 21:38:16.285 | INFO | MemInfo: { | 20260505 21:38:16.285 | INFO | "gpu" : " 17 GB", | 20260505 21:38:16.285 | INFO | "huge" : "", | 20260505 21:38:16.286 | INFO | "huge_shared" : " 27 GB", | 20260505 21:38:16.286 | INFO | "malloc" : " 14 GB", | 20260505 21:38:16.286 | INFO | "pinned" : " 29 GB" | 20260505 21:38:16.286 | INFO | } | 20260505 21:38:16.286 | INFO | Perf report for the last 5.14266 seconds: | 20260505 21:38:16.286 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:38:16.286 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:16.286 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:16.286 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:16.286 | INFO | ANALYSIS : 62.1%, 3.0, 97.1ms, 0 | 20260505 21:38:16.286 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:16.286 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:16.286 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:38:16.286 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:16.286 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:16.286 | INFO | Uploaded 36GiB to GPU in 2.1061s (17.0932GiB/s) | 20260505 21:38:16.286 | INFO | Downloaded 34GiB to GPU in 2.00513s (16.9565GiB/s) | 20260505 21:38:16.286 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 21:38:16.286 | INFO | Total mem usage: 15.3 GiB | 20260505 21:38:16.286 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 21:38:16.286 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.18%) | 20260505 21:38:16.286 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 21:38:16.286 | INFO | High Water Breakdown: | 20260505 21:38:16.286 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 21:38:16.286 | INFO | 38.8% : 5.5 GiB ( 82.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 21:38:16.286 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 21:38:16.286 | INFO | ------------------------------------------ | 20260505 21:38:16.286 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:38:16.286 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 21:38:16.286 | INFO | Total mem usage: 26.5 GiB | 20260505 21:38:16.286 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 21:38:16.286 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 21:38:16.286 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 21:38:16.286 | INFO | High Water Breakdown: | 20260505 21:38:16.286 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:38:16.286 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:38:16.286 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:38:16.286 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:38:16.286 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:38:16.287 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:38:16.287 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 21:38:16.287 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 21:38:16.287 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:38:16.287 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 21:38:16.287 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 21:38:16.287 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 21:38:16.287 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 21:38:16.287 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 21:38:16.287 | INFO | ------------------------------------------ | 20260505 21:38:16.287 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:38:16.287 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 21:38:16.287 | INFO | Total mem usage: 25.6 GiB | 20260505 21:38:16.287 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 21:38:16.287 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 21:38:16.287 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 21:38:16.287 | INFO | High Water Breakdown: | 20260505 21:38:16.287 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 21:38:16.287 | INFO | ------------------------------------------ | 20260505 21:38:16.287 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:38:16.287 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 21:38:16.287 | INFO | Total mem usage: 15.7 GiB | 20260505 21:38:16.287 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 21:38:16.287 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 21:38:16.287 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 21:38:16.287 | INFO | High Water Breakdown: | 20260505 21:38:16.287 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 21:38:16.287 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 21:38:16.287 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 21:38:16.287 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 21:38:16.287 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 21:38:16.287 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 21:38:16.287 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 21:38:16.287 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 21:38:16.287 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 21:38:16.287 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:38:16.287 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 21:38:16.287 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:38:16.287 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 21:38:16.287 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 21:38:16.287 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 21:38:16.287 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 21:38:16.287 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:38:16.287 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 21:38:16.287 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:38:16.287 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:38:16.287 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:38:16.287 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 21:38:16.287 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 21:38:16.287 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:38:16.287 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 21:38:16.287 | INFO | ------------------------------------------ | 20260505 21:38:16.287 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:38:16.287 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 21:38:16.287 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:38:16.287 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:38:16.287 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.42e+01, 1.35e+01 / 1.06e+00 | 20260505 21:38:16.287 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.04e-01, 1.12e-02 / 7.41e-03 | 20260505 21:38:16.287 | INFO | PerfReport: 11.79%, Baseline -- 3.71e+00 / 4.54e+00, 5.67e-01 / 2.25e-01 | 20260505 21:38:16.287 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.51e+00, 1.89e-01 / 1.08e-01 | 20260505 21:38:16.287 | INFO | PerfReport: 15.92%, DME -- 4.01e+01 / 4.40e+01, 3.96e+00 / 1.15e+00 | 20260505 21:38:16.287 | INFO | PerfReport: 57.25%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.56e+00 / 1.06e+00 | 20260505 21:38:16.287 | INFO | PerfReport: 7.17%, PulseAccumulating -- 2.26e+00 / 2.79e+00, 4.32e-01 / 2.30e-01 | 20260505 21:38:16.287 | INFO | PerfReport: 3.19%, Metrics -- 1.00e+00 / 1.31e+00, 2.73e-01 / 1.39e-01 | 20260505 21:38:16.287 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.49e-01 / 4.10e-01, 6.09e-02 / 3.97e-02 | 20260505 21:38:16.287 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 21:38:16.287 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:38:16.287 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:38:16.287 | INFO | PerfReport: Full Execution -- 9.91e+01 / -nan, 2.54e+01 / 0.00e+00 | 20260505 21:38:16.287 | INFO | PerfReport: 22.45%, Upload -- 2.22e+01 / 0.00e+00, 7.70e+00 / 0.00e+00 | 20260505 21:38:16.287 | INFO | PerfReport: 21.80%, Download -- 2.16e+01 / 0.00e+00, 7.68e+00 / 0.00e+00 | 20260505 21:38:16.287 | INFO | PerfReport: 31.75%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 21:38:16.287 | INFO | PerfReport: 2.25%, LockUpload -- 2.23e+00 / 0.00e+00, 7.98e+00 / 0.00e+00 | 20260505 21:38:16.287 | INFO | PerfReport: 1.33%, LockDownload -- 1.32e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260505 21:38:16.287 | INFO | PerfReport: 20.41%, LockCompute -- 2.02e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 21:38:16.287 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 2.01e-02 / 0.00e+00 | 20260505 21:38:16.411 | INFO | Starting a round of Baz IO | 20260505 21:38:19.557 | INFO | Populating RTMetrics report | 20260505 21:38:19.557 | INFO | RTMetrics report populated | 20260505 21:38:19.557 | INFO | Opening tmp RTMetrics json file | 20260505 21:38:19.558 | INFO | Serializing RTMetrics to json | 20260505 21:38:19.559 | INFO | Serialization done, swapping tmp file now... | 20260505 21:38:19.559 | INFO | RTMetrics done | 20260505 21:38:19.644 | INFO | Finished a round of Baz IO: Wrote 1.966 GiB in 3.233 seconds | 20260505 21:38:19.644 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.56% / 0.00% / 78.44% | 20260505 21:38:19.644 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.144482% / 0% / 8.74201e-05% | 20260505 21:38:19.644 | INFO | Baz IO overall: | 20260505 21:38:19.644 | INFO | BytesWritten : 3758831015810 | 20260505 21:38:19.644 | INFO | NumEvents : 1774560206148 | 20260505 21:38:21.425 | INFO | Analyzing chunk frames = [6496256,6496768) | 20260505 21:38:21.425 | INFO | MemInfo: { | 20260505 21:38:21.425 | INFO | "gpu" : " 17 GB", | 20260505 21:38:21.425 | INFO | "huge" : "", | 20260505 21:38:21.425 | INFO | "huge_shared" : " 27 GB", | 20260505 21:38:21.425 | INFO | "malloc" : " 14 GB", | 20260505 21:38:21.425 | INFO | "pinned" : " 29 GB" | 20260505 21:38:21.425 | INFO | } | 20260505 21:38:21.425 | INFO | Perf report for the last 5.13903 seconds: | 20260505 21:38:21.425 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:38:21.425 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:21.425 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:21.425 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:21.425 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 21:38:21.425 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:21.425 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260505 21:38:21.425 | INFO | PRE_HQ : 59.9%, 1.8, 54.8ms, 0 | 20260505 21:38:21.425 | INFO | BAZWRITER : 62.2%, 3.6, 116.4ms, 0 | 20260505 21:38:21.425 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 21:38:21.425 | INFO | Uploaded 36GiB to GPU in 2.16316s (16.6423GiB/s) | 20260505 21:38:21.425 | INFO | Downloaded 38GiB to GPU in 2.23601s (16.9946GiB/s) | 20260505 21:38:26.560 | INFO | Analyzing chunk frames = [6496768,6497280) | 20260505 21:38:26.560 | INFO | MemInfo: { | 20260505 21:38:26.560 | INFO | "gpu" : " 17 GB", | 20260505 21:38:26.560 | INFO | "huge" : "", | 20260505 21:38:26.560 | INFO | "huge_shared" : " 27 GB", | 20260505 21:38:26.560 | INFO | "malloc" : " 14 GB", | 20260505 21:38:26.560 | INFO | "pinned" : " 29 GB" | 20260505 21:38:26.560 | INFO | } | 20260505 21:38:26.560 | INFO | Perf report for the last 5.13497 seconds: | 20260505 21:38:26.560 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:38:26.560 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:26.560 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:26.560 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:26.560 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260505 21:38:26.560 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:26.560 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:26.560 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 21:38:26.560 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:26.560 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:26.560 | INFO | Uploaded 36GiB to GPU in 2.0824s (17.2878GiB/s) | 20260505 21:38:26.560 | INFO | Downloaded 34GiB to GPU in 2.0253s (16.7876GiB/s) | 20260505 21:38:31.697 | INFO | Analyzing chunk frames = [6497280,6497792) | 20260505 21:38:31.697 | INFO | MemInfo: { | 20260505 21:38:31.697 | INFO | "gpu" : " 17 GB", | 20260505 21:38:31.697 | INFO | "huge" : "", | 20260505 21:38:31.697 | INFO | "huge_shared" : " 27 GB", | 20260505 21:38:31.697 | INFO | "malloc" : " 14 GB", | 20260505 21:38:31.697 | INFO | "pinned" : " 29 GB" | 20260505 21:38:31.697 | INFO | } | 20260505 21:38:31.697 | INFO | Perf report for the last 5.13745 seconds: | 20260505 21:38:31.697 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:38:31.697 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:31.697 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:31.697 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:31.697 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260505 21:38:31.697 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:31.697 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:31.697 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:38:31.697 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:31.697 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:31.697 | INFO | Uploaded 36GiB to GPU in 2.12696s (16.9256GiB/s) | 20260505 21:38:31.697 | INFO | Downloaded 34GiB to GPU in 2.0252s (16.7884GiB/s) | 20260505 21:38:36.839 | INFO | Analyzing chunk frames = [6497792,6498304) | 20260505 21:38:36.839 | INFO | MemInfo: { | 20260505 21:38:36.839 | INFO | "gpu" : " 17 GB", | 20260505 21:38:36.839 | INFO | "huge" : "", | 20260505 21:38:36.839 | INFO | "huge_shared" : " 27 GB", | 20260505 21:38:36.839 | INFO | "malloc" : " 14 GB", | 20260505 21:38:36.839 | INFO | "pinned" : " 29 GB" | 20260505 21:38:36.839 | INFO | } | 20260505 21:38:36.839 | INFO | Perf report for the last 5.14228 seconds: | 20260505 21:38:36.839 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:38:36.840 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:36.840 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:36.840 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:36.840 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260505 21:38:36.840 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:36.840 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:36.840 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 21:38:36.840 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:36.840 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:36.840 | INFO | Uploaded 36GiB to GPU in 2.21012s (16.2887GiB/s) | 20260505 21:38:36.840 | INFO | Downloaded 34GiB to GPU in 2.05198s (16.5694GiB/s) | 20260505 21:38:41.986 | INFO | Analyzing chunk frames = [6498304,6498816) | 20260505 21:38:41.986 | INFO | MemInfo: { | 20260505 21:38:41.986 | INFO | "gpu" : " 17 GB", | 20260505 21:38:41.986 | INFO | "huge" : "", | 20260505 21:38:41.986 | INFO | "huge_shared" : " 27 GB", | 20260505 21:38:41.986 | INFO | "malloc" : " 14 GB", | 20260505 21:38:41.986 | INFO | "pinned" : " 29 GB" | 20260505 21:38:41.986 | INFO | } | 20260505 21:38:41.986 | INFO | Perf report for the last 5.14621 seconds: | 20260505 21:38:41.986 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:38:41.986 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:41.986 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:41.986 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:41.986 | INFO | ANALYSIS : 61.5%, 3.0, 96.2ms, 0 | 20260505 21:38:41.986 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:41.986 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:41.986 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260505 21:38:41.986 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:41.986 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:41.986 | INFO | Uploaded 36GiB to GPU in 2.13936s (16.8275GiB/s) | 20260505 21:38:41.986 | INFO | Downloaded 34GiB to GPU in 2.06265s (16.4837GiB/s) | 20260505 21:38:47.119 | INFO | Analyzing chunk frames = [6498816,6499328) | 20260505 21:38:47.119 | INFO | MemInfo: { | 20260505 21:38:47.119 | INFO | "gpu" : " 17 GB", | 20260505 21:38:47.119 | INFO | "huge" : "", | 20260505 21:38:47.119 | INFO | "huge_shared" : " 27 GB", | 20260505 21:38:47.119 | INFO | "malloc" : " 14 GB", | 20260505 21:38:47.119 | INFO | "pinned" : " 29 GB" | 20260505 21:38:47.119 | INFO | } | 20260505 21:38:47.119 | INFO | Perf report for the last 5.13335 seconds: | 20260505 21:38:47.119 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:38:47.119 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:47.119 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:47.119 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:47.119 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260505 21:38:47.119 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:47.119 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:47.119 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 21:38:47.119 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:47.119 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:47.119 | INFO | Uploaded 36GiB to GPU in 2.07902s (17.3158GiB/s) | 20260505 21:38:47.119 | INFO | Downloaded 34GiB to GPU in 2.03964s (16.6696GiB/s) | 20260505 21:38:52.255 | INFO | Analyzing chunk frames = [6499328,6499840) | 20260505 21:38:52.255 | INFO | MemInfo: { | 20260505 21:38:52.255 | INFO | "gpu" : " 17 GB", | 20260505 21:38:52.255 | INFO | "huge" : "", | 20260505 21:38:52.255 | INFO | "huge_shared" : " 27 GB", | 20260505 21:38:52.255 | INFO | "malloc" : " 14 GB", | 20260505 21:38:52.255 | INFO | "pinned" : " 29 GB" | 20260505 21:38:52.255 | INFO | } | 20260505 21:38:52.255 | INFO | Perf report for the last 5.13566 seconds: | 20260505 21:38:52.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:38:52.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:52.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:52.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:52.255 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 21:38:52.255 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:52.255 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:52.255 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260505 21:38:52.255 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:52.255 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:52.255 | INFO | Uploaded 36GiB to GPU in 2.15332s (16.7184GiB/s) | 20260505 21:38:52.255 | INFO | Downloaded 34GiB to GPU in 2.11473s (16.0777GiB/s) | 20260505 21:38:57.397 | INFO | Analyzing chunk frames = [6499840,6500352) | 20260505 21:38:57.397 | INFO | MemInfo: { | 20260505 21:38:57.397 | INFO | "gpu" : " 17 GB", | 20260505 21:38:57.397 | INFO | "huge" : "", | 20260505 21:38:57.397 | INFO | "huge_shared" : " 27 GB", | 20260505 21:38:57.397 | INFO | "malloc" : " 14 GB", | 20260505 21:38:57.397 | INFO | "pinned" : " 29 GB" | 20260505 21:38:57.397 | INFO | } | 20260505 21:38:57.397 | INFO | Perf report for the last 5.14242 seconds: | 20260505 21:38:57.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:38:57.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:57.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:57.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:57.397 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 21:38:57.397 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:57.397 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:57.397 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 21:38:57.397 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:57.397 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:38:57.397 | INFO | Uploaded 36GiB to GPU in 2.11285s (17.0386GiB/s) | 20260505 21:38:57.397 | INFO | Downloaded 34GiB to GPU in 2.00982s (16.917GiB/s) | 20260505 21:38:57.534 | INFO | Starting a round of Baz IO | 20260505 21:39:00.690 | INFO | Populating RTMetrics report | 20260505 21:39:00.691 | INFO | RTMetrics report populated | 20260505 21:39:00.691 | INFO | Opening tmp RTMetrics json file | 20260505 21:39:00.691 | INFO | Serializing RTMetrics to json | 20260505 21:39:00.692 | INFO | Serialization done, swapping tmp file now... | 20260505 21:39:00.693 | INFO | RTMetrics done | 20260505 21:39:00.779 | INFO | Finished a round of Baz IO: Wrote 1.963 GiB in 3.245 seconds | 20260505 21:39:00.780 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.56% / 0.00% / 78.44% | 20260505 21:39:00.780 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.143222% / 0% / 7.15256e-05% | 20260505 21:39:00.780 | INFO | Baz IO overall: | 20260505 21:39:00.780 | INFO | BytesWritten : 3760938642216 | 20260505 21:39:00.780 | INFO | NumEvents : 1775532383113 | 20260505 21:39:02.537 | INFO | Analyzing chunk frames = [6500352,6500864) | 20260505 21:39:02.537 | INFO | MemInfo: { | 20260505 21:39:02.537 | INFO | "gpu" : " 17 GB", | 20260505 21:39:02.537 | INFO | "huge" : "", | 20260505 21:39:02.537 | INFO | "huge_shared" : " 27 GB", | 20260505 21:39:02.537 | INFO | "malloc" : " 14 GB", | 20260505 21:39:02.538 | INFO | "pinned" : " 29 GB" | 20260505 21:39:02.538 | INFO | } | 20260505 21:39:02.538 | INFO | Perf report for the last 5.14039 seconds: | 20260505 21:39:02.538 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:39:02.538 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:02.538 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:02.538 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:02.538 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 21:39:02.538 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:02.538 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260505 21:39:02.538 | INFO | PRE_HQ : 59.9%, 1.8, 55.8ms, 0 | 20260505 21:39:02.538 | INFO | BAZWRITER : 62.7%, 3.7, 117.7ms, 0 | 20260505 21:39:02.538 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 21:39:02.538 | INFO | Uploaded 36GiB to GPU in 2.16229s (16.649GiB/s) | 20260505 21:39:02.538 | INFO | Downloaded 38GiB to GPU in 2.24602s (16.9188GiB/s) | 20260505 21:39:07.679 | INFO | Analyzing chunk frames = [6500864,6501376) | 20260505 21:39:07.679 | INFO | MemInfo: { | 20260505 21:39:07.679 | INFO | "gpu" : " 17 GB", | 20260505 21:39:07.679 | INFO | "huge" : "", | 20260505 21:39:07.679 | INFO | "huge_shared" : " 27 GB", | 20260505 21:39:07.679 | INFO | "malloc" : " 14 GB", | 20260505 21:39:07.679 | INFO | "pinned" : " 29 GB" | 20260505 21:39:07.679 | INFO | } | 20260505 21:39:07.679 | INFO | Perf report for the last 5.14162 seconds: | 20260505 21:39:07.679 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:39:07.679 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:07.679 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:07.679 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:07.679 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 21:39:07.679 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:07.679 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:07.679 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 21:39:07.679 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:07.679 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:07.679 | INFO | Uploaded 36GiB to GPU in 2.09027s (17.2227GiB/s) | 20260505 21:39:07.680 | INFO | Downloaded 34GiB to GPU in 2.02585s (16.7831GiB/s) | 20260505 21:39:12.812 | INFO | Analyzing chunk frames = [6501376,6501888) | 20260505 21:39:12.812 | INFO | MemInfo: { | 20260505 21:39:12.812 | INFO | "gpu" : " 17 GB", | 20260505 21:39:12.812 | INFO | "huge" : "", | 20260505 21:39:12.812 | INFO | "huge_shared" : " 27 GB", | 20260505 21:39:12.812 | INFO | "malloc" : " 14 GB", | 20260505 21:39:12.812 | INFO | "pinned" : " 29 GB" | 20260505 21:39:12.812 | INFO | } | 20260505 21:39:12.812 | INFO | Perf report for the last 5.13278 seconds: | 20260505 21:39:12.812 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:39:12.812 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:12.812 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:12.812 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:12.812 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260505 21:39:12.812 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:12.812 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:12.812 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:39:12.812 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:12.812 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:12.812 | INFO | Uploaded 36GiB to GPU in 2.12649s (16.9293GiB/s) | 20260505 21:39:12.812 | INFO | Downloaded 34GiB to GPU in 2.01135s (16.9041GiB/s) | 20260505 21:39:17.951 | INFO | Analyzing chunk frames = [6501888,6502400) | 20260505 21:39:17.951 | INFO | MemInfo: { | 20260505 21:39:17.951 | INFO | "gpu" : " 17 GB", | 20260505 21:39:17.951 | INFO | "huge" : "", | 20260505 21:39:17.951 | INFO | "huge_shared" : " 27 GB", | 20260505 21:39:17.951 | INFO | "malloc" : " 14 GB", | 20260505 21:39:17.951 | INFO | "pinned" : " 29 GB" | 20260505 21:39:17.951 | INFO | } | 20260505 21:39:17.951 | INFO | Perf report for the last 5.1389 seconds: | 20260505 21:39:17.951 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:39:17.951 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:17.951 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:17.951 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:17.951 | INFO | ANALYSIS : 62.3%, 3.0, 96.0ms, 0 | 20260505 21:39:17.951 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:17.951 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:17.951 | INFO | PRE_HQ : 27.6%, 1.0, 14.2ms, 0 | 20260505 21:39:17.951 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:17.951 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:17.951 | INFO | Uploaded 36GiB to GPU in 2.21049s (16.286GiB/s) | 20260505 21:39:17.951 | INFO | Downloaded 34GiB to GPU in 2.06542s (16.4615GiB/s) | 20260505 21:39:23.094 | INFO | Analyzing chunk frames = [6502400,6502912) | 20260505 21:39:23.094 | INFO | MemInfo: { | 20260505 21:39:23.094 | INFO | "gpu" : " 17 GB", | 20260505 21:39:23.094 | INFO | "huge" : "", | 20260505 21:39:23.094 | INFO | "huge_shared" : " 27 GB", | 20260505 21:39:23.094 | INFO | "malloc" : " 14 GB", | 20260505 21:39:23.094 | INFO | "pinned" : " 29 GB" | 20260505 21:39:23.094 | INFO | } | 20260505 21:39:23.094 | INFO | Perf report for the last 5.14342 seconds: | 20260505 21:39:23.094 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:39:23.094 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:23.094 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:23.094 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:23.094 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260505 21:39:23.094 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:23.094 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:23.094 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:39:23.094 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:23.094 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:23.095 | INFO | Uploaded 36GiB to GPU in 2.13775s (16.8401GiB/s) | 20260505 21:39:23.095 | INFO | Downloaded 34GiB to GPU in 2.06463s (16.4678GiB/s) | 20260505 21:39:28.241 | INFO | Analyzing chunk frames = [6502912,6503424) | 20260505 21:39:28.241 | INFO | MemInfo: { | 20260505 21:39:28.241 | INFO | "gpu" : " 17 GB", | 20260505 21:39:28.241 | INFO | "huge" : "", | 20260505 21:39:28.241 | INFO | "huge_shared" : " 27 GB", | 20260505 21:39:28.241 | INFO | "malloc" : " 14 GB", | 20260505 21:39:28.241 | INFO | "pinned" : " 29 GB" | 20260505 21:39:28.241 | INFO | } | 20260505 21:39:28.241 | INFO | Perf report for the last 5.14635 seconds: | 20260505 21:39:28.241 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:39:28.241 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:28.241 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:28.241 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:28.241 | INFO | ANALYSIS : 61.6%, 3.0, 96.3ms, 0 | 20260505 21:39:28.241 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:28.241 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:28.241 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:39:28.241 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:28.241 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:28.241 | INFO | Uploaded 36GiB to GPU in 2.08411s (17.2735GiB/s) | 20260505 21:39:28.241 | INFO | Downloaded 34GiB to GPU in 2.0353s (16.7051GiB/s) | 20260505 21:39:33.371 | INFO | Analyzing chunk frames = [6503424,6503936) | 20260505 21:39:33.371 | INFO | MemInfo: { | 20260505 21:39:33.371 | INFO | "gpu" : " 17 GB", | 20260505 21:39:33.371 | INFO | "huge" : "", | 20260505 21:39:33.371 | INFO | "huge_shared" : " 27 GB", | 20260505 21:39:33.371 | INFO | "malloc" : " 14 GB", | 20260505 21:39:33.371 | INFO | "pinned" : " 29 GB" | 20260505 21:39:33.371 | INFO | } | 20260505 21:39:33.371 | INFO | Perf report for the last 5.1301 seconds: | 20260505 21:39:33.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:39:33.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:33.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:33.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:33.371 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260505 21:39:33.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:33.371 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:33.371 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 21:39:33.371 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:33.371 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:33.371 | INFO | Uploaded 36GiB to GPU in 2.12674s (16.9273GiB/s) | 20260505 21:39:33.371 | INFO | Downloaded 34GiB to GPU in 2.10816s (16.1278GiB/s) | 20260505 21:39:38.505 | INFO | Analyzing chunk frames = [6503936,6504448) | 20260505 21:39:38.505 | INFO | MemInfo: { | 20260505 21:39:38.505 | INFO | "gpu" : " 17 GB", | 20260505 21:39:38.505 | INFO | "huge" : "", | 20260505 21:39:38.505 | INFO | "huge_shared" : " 27 GB", | 20260505 21:39:38.505 | INFO | "malloc" : " 14 GB", | 20260505 21:39:38.505 | INFO | "pinned" : " 29 GB" | 20260505 21:39:38.505 | INFO | } | 20260505 21:39:38.505 | INFO | Perf report for the last 5.13436 seconds: | 20260505 21:39:38.505 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:39:38.505 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:38.505 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:38.505 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:38.505 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260505 21:39:38.505 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:38.505 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:38.505 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:39:38.505 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:38.505 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:38.505 | INFO | Uploaded 36GiB to GPU in 2.12441s (16.9458GiB/s) | 20260505 21:39:38.505 | INFO | Downloaded 34GiB to GPU in 2.01567s (16.8679GiB/s) | 20260505 21:39:38.506 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 21:39:38.506 | INFO | Total mem usage: 15.3 GiB | 20260505 21:39:38.506 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 21:39:38.506 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.97%) | 20260505 21:39:38.506 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 21:39:38.506 | INFO | High Water Breakdown: | 20260505 21:39:38.506 | INFO | 60.9% : 8.57 GiB ( 90% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 21:39:38.506 | INFO | 38.9% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 21:39:38.506 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 21:39:38.506 | INFO | ------------------------------------------ | 20260505 21:39:38.506 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:39:38.506 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 21:39:38.506 | INFO | Total mem usage: 26.5 GiB | 20260505 21:39:38.506 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 21:39:38.506 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 21:39:38.506 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 21:39:38.506 | INFO | High Water Breakdown: | 20260505 21:39:38.506 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:39:38.506 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:39:38.506 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:39:38.506 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:39:38.506 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:39:38.506 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:39:38.506 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 21:39:38.506 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 21:39:38.506 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:39:38.506 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 21:39:38.506 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 21:39:38.506 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 21:39:38.506 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 21:39:38.506 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 21:39:38.506 | INFO | ------------------------------------------ | 20260505 21:39:38.506 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:39:38.506 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 21:39:38.506 | INFO | Total mem usage: 25.6 GiB | 20260505 21:39:38.506 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 21:39:38.506 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 21:39:38.506 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 21:39:38.506 | INFO | High Water Breakdown: | 20260505 21:39:38.506 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 21:39:38.506 | INFO | ------------------------------------------ | 20260505 21:39:38.506 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:39:38.506 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 21:39:38.506 | INFO | Total mem usage: 15.7 GiB | 20260505 21:39:38.506 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 21:39:38.506 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 21:39:38.506 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 21:39:38.506 | INFO | High Water Breakdown: | 20260505 21:39:38.506 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 21:39:38.506 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 21:39:38.506 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 21:39:38.506 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 21:39:38.506 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 21:39:38.506 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 21:39:38.506 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 21:39:38.506 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 21:39:38.506 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 21:39:38.506 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:39:38.506 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 21:39:38.506 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:39:38.506 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 21:39:38.506 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 21:39:38.506 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 21:39:38.506 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 21:39:38.506 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:39:38.506 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 21:39:38.506 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:39:38.506 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:39:38.506 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:39:38.506 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 21:39:38.506 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 21:39:38.506 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:39:38.506 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 21:39:38.506 | INFO | ------------------------------------------ | 20260505 21:39:38.506 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:39:38.506 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 21:39:38.506 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:39:38.506 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:39:38.507 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.41e+01, 1.35e+01 / 7.65e-01 | 20260505 21:39:38.507 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.04e-01, 1.05e-02 / 5.38e-03 | 20260505 21:39:38.507 | INFO | PerfReport: 11.78%, Baseline -- 3.70e+00 / 4.49e+00, 5.66e-01 / 2.17e-01 | 20260505 21:39:38.507 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.52e+00, 1.89e-01 / 9.64e-02 | 20260505 21:39:38.507 | INFO | PerfReport: 15.92%, DME -- 4.00e+01 / 4.39e+01, 3.76e+00 / 6.68e-01 | 20260505 21:39:38.507 | INFO | PerfReport: 57.24%, FrameLabeling -- 1.80e+01 / 1.96e+01, 1.53e+00 / 1.28e+00 | 20260505 21:39:38.507 | INFO | PerfReport: 7.19%, PulseAccumulating -- 2.26e+00 / 2.77e+00, 4.32e-01 / 2.44e-01 | 20260505 21:39:38.507 | INFO | PerfReport: 3.20%, Metrics -- 1.01e+00 / 1.30e+00, 2.73e-01 / 1.53e-01 | 20260505 21:39:38.507 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.49e-01 / 4.02e-01, 6.00e-02 / 3.54e-02 | 20260505 21:39:38.507 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 21:39:38.507 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:39:38.507 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:39:38.507 | INFO | PerfReport: Full Execution -- 9.90e+01 / -nan, 2.50e+01 / 0.00e+00 | 20260505 21:39:38.507 | INFO | PerfReport: 22.44%, Upload -- 2.22e+01 / 0.00e+00, 7.69e+00 / 0.00e+00 | 20260505 21:39:38.507 | INFO | PerfReport: 21.79%, Download -- 2.16e+01 / 0.00e+00, 7.66e+00 / 0.00e+00 | 20260505 21:39:38.507 | INFO | PerfReport: 31.74%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 21:39:38.507 | INFO | PerfReport: 2.24%, LockUpload -- 2.22e+00 / 0.00e+00, 7.87e+00 / 0.00e+00 | 20260505 21:39:38.507 | INFO | PerfReport: 1.31%, LockDownload -- 1.29e+00 / 0.00e+00, 2.80e+00 / 0.00e+00 | 20260505 21:39:38.507 | INFO | PerfReport: 20.46%, LockCompute -- 2.03e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 21:39:38.507 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 2.16e-02 / 0.00e+00 | 20260505 21:39:38.641 | INFO | Starting a round of Baz IO | 20260505 21:39:41.806 | INFO | Populating RTMetrics report | 20260505 21:39:41.807 | INFO | RTMetrics report populated | 20260505 21:39:41.807 | INFO | Opening tmp RTMetrics json file | 20260505 21:39:41.810 | INFO | Serializing RTMetrics to json | 20260505 21:39:41.812 | INFO | Serialization done, swapping tmp file now... | 20260505 21:39:41.812 | INFO | RTMetrics done | 20260505 21:39:41.890 | INFO | Finished a round of Baz IO: Wrote 1.964 GiB in 3.249 seconds | 20260505 21:39:41.890 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.56% / 0.00% / 78.44% | 20260505 21:39:41.890 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.143234% / 0% / 8.34465e-05% | 20260505 21:39:41.890 | INFO | Baz IO overall: | 20260505 21:39:41.890 | INFO | BytesWritten : 3763047221532 | 20260505 21:39:41.890 | INFO | NumEvents : 1776505600698 | 20260505 21:39:43.648 | INFO | Analyzing chunk frames = [6504448,6504960) | 20260505 21:39:43.648 | INFO | MemInfo: { | 20260505 21:39:43.648 | INFO | "gpu" : " 17 GB", | 20260505 21:39:43.648 | INFO | "huge" : "", | 20260505 21:39:43.648 | INFO | "huge_shared" : " 27 GB", | 20260505 21:39:43.648 | INFO | "malloc" : " 14 GB", | 20260505 21:39:43.648 | INFO | "pinned" : " 29 GB" | 20260505 21:39:43.648 | INFO | } | 20260505 21:39:43.649 | INFO | Perf report for the last 5.14333 seconds: | 20260505 21:39:43.649 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:39:43.649 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:43.649 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:43.649 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:43.649 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 21:39:43.649 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:43.649 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260505 21:39:43.649 | INFO | PRE_HQ : 60.0%, 1.8, 54.9ms, 0 | 20260505 21:39:43.649 | INFO | BAZWRITER : 62.8%, 3.6, 116.8ms, 0 | 20260505 21:39:43.649 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 21:39:43.649 | INFO | Uploaded 36GiB to GPU in 2.16505s (16.6278GiB/s) | 20260505 21:39:43.649 | INFO | Downloaded 38GiB to GPU in 2.24549s (16.9228GiB/s) | 20260505 21:39:48.788 | INFO | Analyzing chunk frames = [6504960,6505472) | 20260505 21:39:48.788 | INFO | MemInfo: { | 20260505 21:39:48.788 | INFO | "gpu" : " 17 GB", | 20260505 21:39:48.788 | INFO | "huge" : "", | 20260505 21:39:48.788 | INFO | "huge_shared" : " 27 GB", | 20260505 21:39:48.788 | INFO | "malloc" : " 14 GB", | 20260505 21:39:48.788 | INFO | "pinned" : " 29 GB" | 20260505 21:39:48.788 | INFO | } | 20260505 21:39:48.788 | INFO | Perf report for the last 5.13993 seconds: | 20260505 21:39:48.788 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:39:48.788 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:48.788 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:48.788 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:48.788 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 21:39:48.789 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:48.789 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:48.789 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 21:39:48.789 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:48.789 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:48.789 | INFO | Uploaded 36GiB to GPU in 2.05951s (17.4799GiB/s) | 20260505 21:39:48.789 | INFO | Downloaded 34GiB to GPU in 2.01662s (16.8599GiB/s) | 20260505 21:39:53.934 | INFO | Analyzing chunk frames = [6505472,6505984) | 20260505 21:39:53.934 | INFO | MemInfo: { | 20260505 21:39:53.934 | INFO | "gpu" : " 17 GB", | 20260505 21:39:53.934 | INFO | "huge" : "", | 20260505 21:39:53.934 | INFO | "huge_shared" : " 27 GB", | 20260505 21:39:53.934 | INFO | "malloc" : " 14 GB", | 20260505 21:39:53.934 | INFO | "pinned" : " 29 GB" | 20260505 21:39:53.934 | INFO | } | 20260505 21:39:53.934 | INFO | Perf report for the last 5.14568 seconds: | 20260505 21:39:53.934 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:39:53.934 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:53.934 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:53.934 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:53.934 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260505 21:39:53.934 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:53.934 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:53.934 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:39:53.934 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:53.934 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:53.935 | INFO | Uploaded 36GiB to GPU in 2.12107s (16.9726GiB/s) | 20260505 21:39:53.935 | INFO | Downloaded 34GiB to GPU in 2.02133s (16.8206GiB/s) | 20260505 21:39:59.071 | INFO | Analyzing chunk frames = [6505984,6506496) | 20260505 21:39:59.071 | INFO | MemInfo: { | 20260505 21:39:59.071 | INFO | "gpu" : " 17 GB", | 20260505 21:39:59.071 | INFO | "huge" : "", | 20260505 21:39:59.071 | INFO | "huge_shared" : " 27 GB", | 20260505 21:39:59.071 | INFO | "malloc" : " 14 GB", | 20260505 21:39:59.071 | INFO | "pinned" : " 29 GB" | 20260505 21:39:59.072 | INFO | } | 20260505 21:39:59.072 | INFO | Perf report for the last 5.13738 seconds: | 20260505 21:39:59.072 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:39:59.072 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:59.072 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:59.072 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:59.072 | INFO | ANALYSIS : 61.8%, 3.0, 96.0ms, 0 | 20260505 21:39:59.072 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:59.072 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:59.072 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:39:59.072 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:59.072 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:39:59.072 | INFO | Uploaded 36GiB to GPU in 2.12477s (16.943GiB/s) | 20260505 21:39:59.072 | INFO | Downloaded 34GiB to GPU in 2.0224s (16.8117GiB/s) | 20260505 21:40:04.207 | INFO | Analyzing chunk frames = [6506496,6507008) | 20260505 21:40:04.207 | INFO | MemInfo: { | 20260505 21:40:04.207 | INFO | "gpu" : " 17 GB", | 20260505 21:40:04.207 | INFO | "huge" : "", | 20260505 21:40:04.207 | INFO | "huge_shared" : " 27 GB", | 20260505 21:40:04.207 | INFO | "malloc" : " 14 GB", | 20260505 21:40:04.207 | INFO | "pinned" : " 29 GB" | 20260505 21:40:04.207 | INFO | } | 20260505 21:40:04.207 | INFO | Perf report for the last 5.13534 seconds: | 20260505 21:40:04.207 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:40:04.207 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:04.207 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:04.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:04.207 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260505 21:40:04.207 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:04.207 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:04.207 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:40:04.207 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:04.207 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:04.207 | INFO | Uploaded 36GiB to GPU in 2.13479s (16.8635GiB/s) | 20260505 21:40:04.207 | INFO | Downloaded 34GiB to GPU in 2.05938s (16.5098GiB/s) | 20260505 21:40:09.349 | INFO | Analyzing chunk frames = [6507008,6507520) | 20260505 21:40:09.349 | INFO | MemInfo: { | 20260505 21:40:09.349 | INFO | "gpu" : " 17 GB", | 20260505 21:40:09.349 | INFO | "huge" : "", | 20260505 21:40:09.349 | INFO | "huge_shared" : " 27 GB", | 20260505 21:40:09.349 | INFO | "malloc" : " 14 GB", | 20260505 21:40:09.349 | INFO | "pinned" : " 29 GB" | 20260505 21:40:09.349 | INFO | } | 20260505 21:40:09.349 | INFO | Perf report for the last 5.14186 seconds: | 20260505 21:40:09.349 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:40:09.349 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:09.349 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:09.349 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:09.349 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 21:40:09.349 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:09.349 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:09.349 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:40:09.349 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:09.349 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:09.349 | INFO | Uploaded 36GiB to GPU in 2.09204s (17.2081GiB/s) | 20260505 21:40:09.349 | INFO | Downloaded 34GiB to GPU in 2.03391s (16.7165GiB/s) | 20260505 21:40:14.490 | INFO | Analyzing chunk frames = [6507520,6508032) | 20260505 21:40:14.490 | INFO | MemInfo: { | 20260505 21:40:14.490 | INFO | "gpu" : " 17 GB", | 20260505 21:40:14.490 | INFO | "huge" : "", | 20260505 21:40:14.490 | INFO | "huge_shared" : " 27 GB", | 20260505 21:40:14.490 | INFO | "malloc" : " 14 GB", | 20260505 21:40:14.490 | INFO | "pinned" : " 29 GB" | 20260505 21:40:14.490 | INFO | } | 20260505 21:40:14.490 | INFO | Perf report for the last 5.1415 seconds: | 20260505 21:40:14.490 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:40:14.490 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:14.490 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:14.490 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:14.490 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 21:40:14.491 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:14.491 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:14.491 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 21:40:14.491 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:14.491 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:14.491 | INFO | Uploaded 36GiB to GPU in 2.14776s (16.7616GiB/s) | 20260505 21:40:14.491 | INFO | Downloaded 34GiB to GPU in 2.11495s (16.0761GiB/s) | 20260505 21:40:19.628 | INFO | Analyzing chunk frames = [6508032,6508544) | 20260505 21:40:19.628 | INFO | MemInfo: { | 20260505 21:40:19.628 | INFO | "gpu" : " 17 GB", | 20260505 21:40:19.628 | INFO | "huge" : "", | 20260505 21:40:19.628 | INFO | "huge_shared" : " 27 GB", | 20260505 21:40:19.628 | INFO | "malloc" : " 14 GB", | 20260505 21:40:19.628 | INFO | "pinned" : " 29 GB" | 20260505 21:40:19.628 | INFO | } | 20260505 21:40:19.628 | INFO | Perf report for the last 5.13763 seconds: | 20260505 21:40:19.628 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:40:19.628 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:19.628 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:19.628 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:19.628 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 21:40:19.628 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:19.628 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:19.628 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:40:19.628 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:19.628 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:19.628 | INFO | Uploaded 36GiB to GPU in 2.11663s (17.0082GiB/s) | 20260505 21:40:19.628 | INFO | Downloaded 34GiB to GPU in 2.00924s (16.9218GiB/s) | 20260505 21:40:19.762 | INFO | Starting a round of Baz IO | 20260505 21:40:22.915 | INFO | Populating RTMetrics report | 20260505 21:40:22.915 | INFO | RTMetrics report populated | 20260505 21:40:22.915 | INFO | Opening tmp RTMetrics json file | 20260505 21:40:22.916 | INFO | Serializing RTMetrics to json | 20260505 21:40:22.917 | INFO | Serialization done, swapping tmp file now... | 20260505 21:40:22.919 | INFO | RTMetrics done | 20260505 21:40:22.997 | INFO | Finished a round of Baz IO: Wrote 1.962 GiB in 3.234 seconds | 20260505 21:40:22.997 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.56% / 0.00% / 78.44% | 20260505 21:40:22.997 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.142594% / 3.97364e-06% / 0.000103315% | 20260505 21:40:22.997 | INFO | Baz IO overall: | 20260505 21:40:22.997 | INFO | BytesWritten : 3765153409761 | 20260505 21:40:22.997 | INFO | NumEvents : 1777477093008 | 20260505 21:40:24.760 | INFO | Analyzing chunk frames = [6508544,6509056) | 20260505 21:40:24.760 | INFO | MemInfo: { | 20260505 21:40:24.760 | INFO | "gpu" : " 17 GB", | 20260505 21:40:24.760 | INFO | "huge" : "", | 20260505 21:40:24.760 | INFO | "huge_shared" : " 27 GB", | 20260505 21:40:24.760 | INFO | "malloc" : " 14 GB", | 20260505 21:40:24.760 | INFO | "pinned" : " 29 GB" | 20260505 21:40:24.760 | INFO | } | 20260505 21:40:24.760 | INFO | Perf report for the last 5.1321 seconds: | 20260505 21:40:24.760 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:40:24.760 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:24.760 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:24.760 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:24.760 | INFO | ANALYSIS : 63.0%, 3.0, 98.1ms, 0 | 20260505 21:40:24.760 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:24.760 | INFO | RT_METRICS : 43.8%, 1.0, 22.9ms, 0 | 20260505 21:40:24.760 | INFO | PRE_HQ : 59.7%, 1.8, 54.6ms, 0 | 20260505 21:40:24.760 | INFO | BAZWRITER : 62.6%, 3.6, 116.9ms, 0 | 20260505 21:40:24.760 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 21:40:24.760 | INFO | Uploaded 36GiB to GPU in 2.14545s (16.7797GiB/s) | 20260505 21:40:24.760 | INFO | Downloaded 38GiB to GPU in 2.24622s (16.9173GiB/s) | 20260505 21:40:29.904 | INFO | Analyzing chunk frames = [6509056,6509568) | 20260505 21:40:29.904 | INFO | MemInfo: { | 20260505 21:40:29.904 | INFO | "gpu" : " 17 GB", | 20260505 21:40:29.904 | INFO | "huge" : "", | 20260505 21:40:29.904 | INFO | "huge_shared" : " 27 GB", | 20260505 21:40:29.904 | INFO | "malloc" : " 14 GB", | 20260505 21:40:29.904 | INFO | "pinned" : " 29 GB" | 20260505 21:40:29.905 | INFO | } | 20260505 21:40:29.905 | INFO | Perf report for the last 5.14449 seconds: | 20260505 21:40:29.905 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:40:29.905 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:29.905 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:29.905 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:29.905 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260505 21:40:29.905 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:29.905 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:29.905 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 21:40:29.905 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:29.905 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:29.905 | INFO | Uploaded 36GiB to GPU in 2.10889s (17.0706GiB/s) | 20260505 21:40:29.905 | INFO | Downloaded 34GiB to GPU in 2.02434s (16.7956GiB/s) | 20260505 21:40:35.048 | INFO | Analyzing chunk frames = [6509568,6510080) | 20260505 21:40:35.048 | INFO | MemInfo: { | 20260505 21:40:35.048 | INFO | "gpu" : " 17 GB", | 20260505 21:40:35.048 | INFO | "huge" : "", | 20260505 21:40:35.048 | INFO | "huge_shared" : " 27 GB", | 20260505 21:40:35.048 | INFO | "malloc" : " 14 GB", | 20260505 21:40:35.048 | INFO | "pinned" : " 29 GB" | 20260505 21:40:35.048 | INFO | } | 20260505 21:40:35.048 | INFO | Perf report for the last 5.14318 seconds: | 20260505 21:40:35.048 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:40:35.048 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:35.048 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:35.048 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:35.048 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260505 21:40:35.048 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:35.048 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:35.048 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 21:40:35.048 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:35.048 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:35.048 | INFO | Uploaded 36GiB to GPU in 2.13171s (16.8878GiB/s) | 20260505 21:40:35.048 | INFO | Downloaded 34GiB to GPU in 2.01743s (16.8531GiB/s) | 20260505 21:40:40.186 | INFO | Analyzing chunk frames = [6510080,6510592) | 20260505 21:40:40.186 | INFO | MemInfo: { | 20260505 21:40:40.186 | INFO | "gpu" : " 17 GB", | 20260505 21:40:40.186 | INFO | "huge" : "", | 20260505 21:40:40.186 | INFO | "huge_shared" : " 27 GB", | 20260505 21:40:40.186 | INFO | "malloc" : " 14 GB", | 20260505 21:40:40.186 | INFO | "pinned" : " 29 GB" | 20260505 21:40:40.186 | INFO | } | 20260505 21:40:40.186 | INFO | Perf report for the last 5.13795 seconds: | 20260505 21:40:40.186 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:40:40.186 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:40.186 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:40.186 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:40.186 | INFO | ANALYSIS : 61.9%, 3.0, 96.1ms, 0 | 20260505 21:40:40.186 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:40.186 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:40.186 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:40:40.186 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:40.186 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:40.186 | INFO | Uploaded 36GiB to GPU in 2.12161s (16.9682GiB/s) | 20260505 21:40:40.186 | INFO | Downloaded 34GiB to GPU in 2.0205s (16.8275GiB/s) | 20260505 21:40:45.320 | INFO | Analyzing chunk frames = [6510592,6511104) | 20260505 21:40:45.320 | INFO | MemInfo: { | 20260505 21:40:45.320 | INFO | "gpu" : " 17 GB", | 20260505 21:40:45.320 | INFO | "huge" : "", | 20260505 21:40:45.320 | INFO | "huge_shared" : " 27 GB", | 20260505 21:40:45.320 | INFO | "malloc" : " 14 GB", | 20260505 21:40:45.320 | INFO | "pinned" : " 29 GB" | 20260505 21:40:45.320 | INFO | } | 20260505 21:40:45.320 | INFO | Perf report for the last 5.13469 seconds: | 20260505 21:40:45.320 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:40:45.320 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:45.320 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:45.320 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:45.320 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260505 21:40:45.320 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:45.320 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:45.321 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 21:40:45.321 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:45.321 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:45.321 | INFO | Uploaded 36GiB to GPU in 2.13914s (16.8292GiB/s) | 20260505 21:40:45.321 | INFO | Downloaded 34GiB to GPU in 2.05973s (16.507GiB/s) | 20260505 21:40:50.457 | INFO | Analyzing chunk frames = [6511104,6511616) | 20260505 21:40:50.457 | INFO | MemInfo: { | 20260505 21:40:50.457 | INFO | "gpu" : " 17 GB", | 20260505 21:40:50.457 | INFO | "huge" : "", | 20260505 21:40:50.457 | INFO | "huge_shared" : " 27 GB", | 20260505 21:40:50.457 | INFO | "malloc" : " 14 GB", | 20260505 21:40:50.457 | INFO | "pinned" : " 29 GB" | 20260505 21:40:50.457 | INFO | } | 20260505 21:40:50.457 | INFO | Perf report for the last 5.13702 seconds: | 20260505 21:40:50.457 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:40:50.457 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:50.457 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:50.457 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:50.457 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 21:40:50.457 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:50.457 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:50.457 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 21:40:50.457 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:50.458 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:50.458 | INFO | Uploaded 36GiB to GPU in 2.14455s (16.7867GiB/s) | 20260505 21:40:50.458 | INFO | Downloaded 34GiB to GPU in 2.05824s (16.519GiB/s) | 20260505 21:40:55.602 | INFO | Analyzing chunk frames = [6511616,6512128) | 20260505 21:40:55.602 | INFO | MemInfo: { | 20260505 21:40:55.602 | INFO | "gpu" : " 17 GB", | 20260505 21:40:55.602 | INFO | "huge" : "", | 20260505 21:40:55.602 | INFO | "huge_shared" : " 27 GB", | 20260505 21:40:55.602 | INFO | "malloc" : " 14 GB", | 20260505 21:40:55.602 | INFO | "pinned" : " 29 GB" | 20260505 21:40:55.602 | INFO | } | 20260505 21:40:55.602 | INFO | Perf report for the last 5.14428 seconds: | 20260505 21:40:55.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:40:55.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:55.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:55.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:55.602 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 21:40:55.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:55.602 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:55.602 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 21:40:55.602 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:55.602 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:40:55.602 | INFO | Uploaded 36GiB to GPU in 2.12172s (16.9674GiB/s) | 20260505 21:40:55.602 | INFO | Downloaded 34GiB to GPU in 2.15194s (15.7997GiB/s) | 20260505 21:41:00.744 | INFO | Analyzing chunk frames = [6512128,6512640) | 20260505 21:41:00.744 | INFO | MemInfo: { | 20260505 21:41:00.744 | INFO | "gpu" : " 17 GB", | 20260505 21:41:00.744 | INFO | "huge" : "", | 20260505 21:41:00.744 | INFO | "huge_shared" : " 27 GB", | 20260505 21:41:00.744 | INFO | "malloc" : " 14 GB", | 20260505 21:41:00.744 | INFO | "pinned" : " 29 GB" | 20260505 21:41:00.745 | INFO | } | 20260505 21:41:00.745 | INFO | Perf report for the last 5.14283 seconds: | 20260505 21:41:00.745 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:41:00.745 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:00.745 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:00.745 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:00.745 | INFO | ANALYSIS : 62.1%, 3.0, 97.1ms, 0 | 20260505 21:41:00.745 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:00.745 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:00.745 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:41:00.745 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:00.745 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:00.745 | INFO | Uploaded 36GiB to GPU in 2.11903s (16.9889GiB/s) | 20260505 21:41:00.745 | INFO | Downloaded 34GiB to GPU in 2.00292s (16.9752GiB/s) | 20260505 21:41:00.745 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 21:41:00.745 | INFO | Total mem usage: 15.3 GiB | 20260505 21:41:00.745 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 21:41:00.745 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.95%) | 20260505 21:41:00.745 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 21:41:00.745 | INFO | High Water Breakdown: | 20260505 21:41:00.745 | INFO | 60.9% : 8.57 GiB ( 90% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 21:41:00.745 | INFO | 39% : 5.48 GiB ( 82.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 21:41:00.745 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 21:41:00.745 | INFO | ------------------------------------------ | 20260505 21:41:00.745 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:41:00.745 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 21:41:00.745 | INFO | Total mem usage: 26.5 GiB | 20260505 21:41:00.745 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 21:41:00.745 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 21:41:00.745 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 21:41:00.745 | INFO | High Water Breakdown: | 20260505 21:41:00.745 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:41:00.745 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:41:00.745 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:41:00.745 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:41:00.745 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:41:00.745 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:41:00.745 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 21:41:00.745 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 21:41:00.745 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:41:00.745 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 21:41:00.745 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 21:41:00.745 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 21:41:00.745 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 21:41:00.745 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 21:41:00.745 | INFO | ------------------------------------------ | 20260505 21:41:00.745 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:41:00.745 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 21:41:00.745 | INFO | Total mem usage: 25.6 GiB | 20260505 21:41:00.745 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 21:41:00.745 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 21:41:00.745 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 21:41:00.745 | INFO | High Water Breakdown: | 20260505 21:41:00.745 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 21:41:00.745 | INFO | ------------------------------------------ | 20260505 21:41:00.745 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:41:00.745 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 21:41:00.745 | INFO | Total mem usage: 15.7 GiB | 20260505 21:41:00.745 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 21:41:00.745 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 21:41:00.745 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 21:41:00.745 | INFO | High Water Breakdown: | 20260505 21:41:00.745 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 21:41:00.745 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 21:41:00.745 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 21:41:00.745 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 21:41:00.745 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 21:41:00.745 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 21:41:00.745 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 21:41:00.745 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 21:41:00.746 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 21:41:00.746 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:41:00.746 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 21:41:00.746 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:41:00.746 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 21:41:00.746 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 21:41:00.746 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 21:41:00.746 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 21:41:00.746 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:41:00.746 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 21:41:00.746 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:41:00.746 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:41:00.746 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:41:00.746 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 21:41:00.746 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 21:41:00.746 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:41:00.746 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 21:41:00.746 | INFO | ------------------------------------------ | 20260505 21:41:00.746 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:41:00.746 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 21:41:00.746 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:41:00.746 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:41:00.746 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.44e+01, 1.35e+01 / 1.43e+00 | 20260505 21:41:00.746 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.04e-01, 1.04e-02 / 3.50e-03 | 20260505 21:41:00.746 | INFO | PerfReport: 11.83%, Baseline -- 3.72e+00 / 4.58e+00, 5.77e-01 / 1.46e-01 | 20260505 21:41:00.746 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.55e+00, 1.93e-01 / 7.47e-02 | 20260505 21:41:00.746 | INFO | PerfReport: 15.95%, DME -- 4.02e+01 / 4.41e+01, 3.93e+00 / 1.45e+00 | 20260505 21:41:00.746 | INFO | PerfReport: 57.19%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.55e+00 / 1.08e+00 | 20260505 21:41:00.746 | INFO | PerfReport: 7.15%, PulseAccumulating -- 2.25e+00 / 2.82e+00, 4.27e-01 / 2.15e-01 | 20260505 21:41:00.746 | INFO | PerfReport: 3.19%, Metrics -- 1.00e+00 / 1.34e+00, 2.74e-01 / 1.56e-01 | 20260505 21:41:00.746 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.49e-01 / 4.12e-01, 6.09e-02 / 3.57e-02 | 20260505 21:41:00.746 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 21:41:00.746 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:41:00.746 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:41:00.746 | INFO | PerfReport: Full Execution -- 9.90e+01 / -nan, 2.51e+01 / 0.00e+00 | 20260505 21:41:00.746 | INFO | PerfReport: 22.35%, Upload -- 2.21e+01 / 0.00e+00, 7.56e+00 / 0.00e+00 | 20260505 21:41:00.746 | INFO | PerfReport: 21.77%, Download -- 2.16e+01 / 0.00e+00, 7.60e+00 / 0.00e+00 | 20260505 21:41:00.746 | INFO | PerfReport: 31.79%, Compute -- 3.15e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 21:41:00.746 | INFO | PerfReport: 2.18%, LockUpload -- 2.16e+00 / 0.00e+00, 7.90e+00 / 0.00e+00 | 20260505 21:41:00.746 | INFO | PerfReport: 1.37%, LockDownload -- 1.36e+00 / 0.00e+00, 2.97e+00 / 0.00e+00 | 20260505 21:41:00.746 | INFO | PerfReport: 20.52%, LockCompute -- 2.03e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 21:41:00.746 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.96e-02 / 0.00e+00 | 20260505 21:41:00.869 | INFO | Starting a round of Baz IO | 20260505 21:41:04.023 | INFO | Populating RTMetrics report | 20260505 21:41:04.023 | INFO | RTMetrics report populated | 20260505 21:41:04.023 | INFO | Opening tmp RTMetrics json file | 20260505 21:41:04.024 | INFO | Serializing RTMetrics to json | 20260505 21:41:04.026 | INFO | Serialization done, swapping tmp file now... | 20260505 21:41:04.026 | INFO | RTMetrics done | 20260505 21:41:04.108 | INFO | Finished a round of Baz IO: Wrote 1.963 GiB in 3.238 seconds | 20260505 21:41:04.108 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.56% / 0.00% / 78.44% | 20260505 21:41:04.108 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.142817% / 7.94729e-06% / 8.74201e-05% | 20260505 21:41:04.108 | INFO | Baz IO overall: | 20260505 21:41:04.108 | INFO | BytesWritten : 3767260915986 | 20260505 21:41:04.108 | INFO | NumEvents : 1778449598230 | 20260505 21:41:05.882 | INFO | Analyzing chunk frames = [6512640,6513152) | 20260505 21:41:05.882 | INFO | MemInfo: { | 20260505 21:41:05.882 | INFO | "gpu" : " 17 GB", | 20260505 21:41:05.882 | INFO | "huge" : "", | 20260505 21:41:05.882 | INFO | "huge_shared" : " 27 GB", | 20260505 21:41:05.882 | INFO | "malloc" : " 14 GB", | 20260505 21:41:05.882 | INFO | "pinned" : " 29 GB" | 20260505 21:41:05.882 | INFO | } | 20260505 21:41:05.883 | INFO | Perf report for the last 5.13794 seconds: | 20260505 21:41:05.883 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:41:05.883 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:05.883 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:05.883 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:05.883 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260505 21:41:05.883 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:05.883 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260505 21:41:05.883 | INFO | PRE_HQ : 60.0%, 1.8, 54.8ms, 0 | 20260505 21:41:05.883 | INFO | BAZWRITER : 62.2%, 3.6, 115.9ms, 0 | 20260505 21:41:05.883 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 21:41:05.883 | INFO | Uploaded 36GiB to GPU in 2.17112s (16.5813GiB/s) | 20260505 21:41:05.883 | INFO | Downloaded 38GiB to GPU in 2.24209s (16.9485GiB/s) | 20260505 21:41:11.014 | INFO | Analyzing chunk frames = [6513152,6513664) | 20260505 21:41:11.014 | INFO | MemInfo: { | 20260505 21:41:11.014 | INFO | "gpu" : " 17 GB", | 20260505 21:41:11.014 | INFO | "huge" : "", | 20260505 21:41:11.014 | INFO | "huge_shared" : " 27 GB", | 20260505 21:41:11.014 | INFO | "malloc" : " 14 GB", | 20260505 21:41:11.014 | INFO | "pinned" : " 29 GB" | 20260505 21:41:11.014 | INFO | } | 20260505 21:41:11.014 | INFO | Perf report for the last 5.13158 seconds: | 20260505 21:41:11.014 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:41:11.014 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:11.014 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:11.014 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:11.014 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260505 21:41:11.014 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:11.014 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:11.014 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 21:41:11.014 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:11.014 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:11.014 | INFO | Uploaded 36GiB to GPU in 2.08776s (17.2433GiB/s) | 20260505 21:41:11.014 | INFO | Downloaded 34GiB to GPU in 2.02722s (16.7718GiB/s) | 20260505 21:41:16.165 | INFO | Analyzing chunk frames = [6513664,6514176) | 20260505 21:41:16.165 | INFO | MemInfo: { | 20260505 21:41:16.165 | INFO | "gpu" : " 17 GB", | 20260505 21:41:16.165 | INFO | "huge" : "", | 20260505 21:41:16.165 | INFO | "huge_shared" : " 27 GB", | 20260505 21:41:16.165 | INFO | "malloc" : " 14 GB", | 20260505 21:41:16.165 | INFO | "pinned" : " 29 GB" | 20260505 21:41:16.165 | INFO | } | 20260505 21:41:16.165 | INFO | Perf report for the last 5.15077 seconds: | 20260505 21:41:16.165 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:41:16.165 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:16.165 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:16.165 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:16.165 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260505 21:41:16.165 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:16.165 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:16.165 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:41:16.165 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:16.165 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:16.165 | INFO | Uploaded 36GiB to GPU in 2.13135s (16.8907GiB/s) | 20260505 21:41:16.165 | INFO | Downloaded 34GiB to GPU in 2.01838s (16.8452GiB/s) | 20260505 21:41:21.301 | INFO | Analyzing chunk frames = [6514176,6514688) | 20260505 21:41:21.301 | INFO | MemInfo: { | 20260505 21:41:21.301 | INFO | "gpu" : " 17 GB", | 20260505 21:41:21.301 | INFO | "huge" : "", | 20260505 21:41:21.301 | INFO | "huge_shared" : " 27 GB", | 20260505 21:41:21.301 | INFO | "malloc" : " 14 GB", | 20260505 21:41:21.301 | INFO | "pinned" : " 29 GB" | 20260505 21:41:21.301 | INFO | } | 20260505 21:41:21.301 | INFO | Perf report for the last 5.13617 seconds: | 20260505 21:41:21.301 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:41:21.301 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:21.301 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:21.301 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:21.301 | INFO | ANALYSIS : 61.8%, 3.0, 95.9ms, 0 | 20260505 21:41:21.301 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:21.301 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:21.301 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:41:21.301 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:21.301 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:21.301 | INFO | Uploaded 36GiB to GPU in 2.12794s (16.9178GiB/s) | 20260505 21:41:21.301 | INFO | Downloaded 34GiB to GPU in 2.01803s (16.8481GiB/s) | 20260505 21:41:26.436 | INFO | Analyzing chunk frames = [6514688,6515200) | 20260505 21:41:26.436 | INFO | MemInfo: { | 20260505 21:41:26.436 | INFO | "gpu" : " 17 GB", | 20260505 21:41:26.436 | INFO | "huge" : "", | 20260505 21:41:26.436 | INFO | "huge_shared" : " 27 GB", | 20260505 21:41:26.436 | INFO | "malloc" : " 14 GB", | 20260505 21:41:26.436 | INFO | "pinned" : " 29 GB" | 20260505 21:41:26.436 | INFO | } | 20260505 21:41:26.436 | INFO | Perf report for the last 5.13485 seconds: | 20260505 21:41:26.436 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:41:26.436 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:26.436 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:26.436 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:26.436 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260505 21:41:26.436 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:26.436 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:26.436 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 21:41:26.436 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:26.436 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:26.436 | INFO | Uploaded 36GiB to GPU in 2.15177s (16.7304GiB/s) | 20260505 21:41:26.436 | INFO | Downloaded 34GiB to GPU in 2.06612s (16.456GiB/s) | 20260505 21:41:31.575 | INFO | Analyzing chunk frames = [6515200,6515712) | 20260505 21:41:31.575 | INFO | MemInfo: { | 20260505 21:41:31.575 | INFO | "gpu" : " 17 GB", | 20260505 21:41:31.575 | INFO | "huge" : "", | 20260505 21:41:31.575 | INFO | "huge_shared" : " 27 GB", | 20260505 21:41:31.575 | INFO | "malloc" : " 14 GB", | 20260505 21:41:31.575 | INFO | "pinned" : " 29 GB" | 20260505 21:41:31.575 | INFO | } | 20260505 21:41:31.575 | INFO | Perf report for the last 5.13893 seconds: | 20260505 21:41:31.575 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:41:31.575 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:31.575 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:31.575 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:31.575 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260505 21:41:31.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:31.575 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:31.575 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:41:31.575 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:31.575 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:31.575 | INFO | Uploaded 36GiB to GPU in 2.08762s (17.2446GiB/s) | 20260505 21:41:31.575 | INFO | Downloaded 34GiB to GPU in 2.03363s (16.7189GiB/s) | 20260505 21:41:36.713 | INFO | Analyzing chunk frames = [6515712,6516224) | 20260505 21:41:36.713 | INFO | MemInfo: { | 20260505 21:41:36.713 | INFO | "gpu" : " 17 GB", | 20260505 21:41:36.713 | INFO | "huge" : "", | 20260505 21:41:36.713 | INFO | "huge_shared" : " 27 GB", | 20260505 21:41:36.713 | INFO | "malloc" : " 14 GB", | 20260505 21:41:36.713 | INFO | "pinned" : " 29 GB" | 20260505 21:41:36.714 | INFO | } | 20260505 21:41:36.714 | INFO | Perf report for the last 5.13877 seconds: | 20260505 21:41:36.714 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:41:36.714 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:36.714 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:36.714 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:36.714 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 21:41:36.714 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:36.714 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:36.714 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 21:41:36.714 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:36.714 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:36.714 | INFO | Uploaded 36GiB to GPU in 2.15253s (16.7245GiB/s) | 20260505 21:41:36.714 | INFO | Downloaded 34GiB to GPU in 2.1621s (15.7254GiB/s) | 20260505 21:41:41.855 | INFO | Analyzing chunk frames = [6516224,6516736) | 20260505 21:41:41.855 | INFO | MemInfo: { | 20260505 21:41:41.855 | INFO | "gpu" : " 17 GB", | 20260505 21:41:41.855 | INFO | "huge" : "", | 20260505 21:41:41.855 | INFO | "huge_shared" : " 27 GB", | 20260505 21:41:41.855 | INFO | "malloc" : " 14 GB", | 20260505 21:41:41.855 | INFO | "pinned" : " 29 GB" | 20260505 21:41:41.856 | INFO | } | 20260505 21:41:41.856 | INFO | Perf report for the last 5.14195 seconds: | 20260505 21:41:41.856 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:41:41.856 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:41.856 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:41.856 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:41.856 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 21:41:41.856 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:41.856 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:41.856 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 21:41:41.856 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:41.856 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:41.856 | INFO | Uploaded 36GiB to GPU in 2.10672s (17.0882GiB/s) | 20260505 21:41:41.856 | INFO | Downloaded 34GiB to GPU in 2.0065s (16.945GiB/s) | 20260505 21:41:41.981 | INFO | Starting a round of Baz IO | 20260505 21:41:45.132 | INFO | Populating RTMetrics report | 20260505 21:41:45.132 | INFO | RTMetrics report populated | 20260505 21:41:45.132 | INFO | Opening tmp RTMetrics json file | 20260505 21:41:45.132 | INFO | Serializing RTMetrics to json | 20260505 21:41:45.134 | INFO | Serialization done, swapping tmp file now... | 20260505 21:41:45.136 | INFO | RTMetrics done | 20260505 21:41:45.214 | INFO | Finished a round of Baz IO: Wrote 1.961 GiB in 3.232 seconds | 20260505 21:41:45.214 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.56% / 0.00% / 78.44% | 20260505 21:41:45.214 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.142201% / 0% / 0.000115236% | 20260505 21:41:45.214 | INFO | Baz IO overall: | 20260505 21:41:45.214 | INFO | BytesWritten : 3769366974779 | 20260505 21:41:45.214 | INFO | NumEvents : 1779421149994 | 20260505 21:41:46.996 | INFO | Analyzing chunk frames = [6516736,6517248) | 20260505 21:41:46.996 | INFO | MemInfo: { | 20260505 21:41:46.996 | INFO | "gpu" : " 17 GB", | 20260505 21:41:46.996 | INFO | "huge" : "", | 20260505 21:41:46.996 | INFO | "huge_shared" : " 27 GB", | 20260505 21:41:46.997 | INFO | "malloc" : " 14 GB", | 20260505 21:41:46.997 | INFO | "pinned" : " 29 GB" | 20260505 21:41:46.997 | INFO | } | 20260505 21:41:46.997 | INFO | Perf report for the last 5.14092 seconds: | 20260505 21:41:46.997 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:41:46.997 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:46.997 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:46.997 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:46.997 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 21:41:46.997 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:46.997 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260505 21:41:46.997 | INFO | PRE_HQ : 60.0%, 1.8, 54.9ms, 0 | 20260505 21:41:46.997 | INFO | BAZWRITER : 62.2%, 3.7, 116.9ms, 0 | 20260505 21:41:46.997 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 21:41:46.997 | INFO | Uploaded 36GiB to GPU in 2.16641s (16.6174GiB/s) | 20260505 21:41:46.997 | INFO | Downloaded 38GiB to GPU in 2.23569s (16.997GiB/s) | 20260505 21:41:52.136 | INFO | Analyzing chunk frames = [6517248,6517760) | 20260505 21:41:52.136 | INFO | MemInfo: { | 20260505 21:41:52.136 | INFO | "gpu" : " 17 GB", | 20260505 21:41:52.136 | INFO | "huge" : "", | 20260505 21:41:52.136 | INFO | "huge_shared" : " 27 GB", | 20260505 21:41:52.136 | INFO | "malloc" : " 14 GB", | 20260505 21:41:52.136 | INFO | "pinned" : " 29 GB" | 20260505 21:41:52.136 | INFO | } | 20260505 21:41:52.136 | INFO | Perf report for the last 5.13959 seconds: | 20260505 21:41:52.136 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:41:52.136 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:52.136 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:52.136 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:52.136 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 21:41:52.136 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:52.136 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:52.136 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 21:41:52.136 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:52.136 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:52.136 | INFO | Uploaded 36GiB to GPU in 2.08178s (17.2929GiB/s) | 20260505 21:41:52.137 | INFO | Downloaded 34GiB to GPU in 2.02249s (16.8109GiB/s) | 20260505 21:41:57.269 | INFO | Analyzing chunk frames = [6517760,6518272) | 20260505 21:41:57.269 | INFO | MemInfo: { | 20260505 21:41:57.269 | INFO | "gpu" : " 17 GB", | 20260505 21:41:57.269 | INFO | "huge" : "", | 20260505 21:41:57.269 | INFO | "huge_shared" : " 27 GB", | 20260505 21:41:57.269 | INFO | "malloc" : " 14 GB", | 20260505 21:41:57.269 | INFO | "pinned" : " 29 GB" | 20260505 21:41:57.269 | INFO | } | 20260505 21:41:57.269 | INFO | Perf report for the last 5.13276 seconds: | 20260505 21:41:57.269 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:41:57.269 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:57.269 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:57.269 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:57.269 | INFO | ANALYSIS : 62.1%, 3.0, 96.5ms, 0 | 20260505 21:41:57.269 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:57.269 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:57.269 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 21:41:57.269 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:57.269 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:41:57.269 | INFO | Uploaded 36GiB to GPU in 2.1336s (16.8729GiB/s) | 20260505 21:41:57.269 | INFO | Downloaded 34GiB to GPU in 2.01653s (16.8607GiB/s) | 20260505 21:42:02.411 | INFO | Analyzing chunk frames = [6518272,6518784) | 20260505 21:42:02.411 | INFO | MemInfo: { | 20260505 21:42:02.411 | INFO | "gpu" : " 17 GB", | 20260505 21:42:02.411 | INFO | "huge" : "", | 20260505 21:42:02.411 | INFO | "huge_shared" : " 27 GB", | 20260505 21:42:02.411 | INFO | "malloc" : " 14 GB", | 20260505 21:42:02.411 | INFO | "pinned" : " 29 GB" | 20260505 21:42:02.411 | INFO | } | 20260505 21:42:02.411 | INFO | Perf report for the last 5.14197 seconds: | 20260505 21:42:02.411 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:42:02.411 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:02.411 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:02.411 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:02.411 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260505 21:42:02.411 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:02.411 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:02.411 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:42:02.411 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:02.411 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:02.411 | INFO | Uploaded 36GiB to GPU in 2.2019s (16.3495GiB/s) | 20260505 21:42:02.411 | INFO | Downloaded 34GiB to GPU in 2.0452s (16.6243GiB/s) | 20260505 21:42:07.553 | INFO | Analyzing chunk frames = [6518784,6519296) | 20260505 21:42:07.553 | INFO | MemInfo: { | 20260505 21:42:07.553 | INFO | "gpu" : " 17 GB", | 20260505 21:42:07.553 | INFO | "huge" : "", | 20260505 21:42:07.553 | INFO | "huge_shared" : " 27 GB", | 20260505 21:42:07.553 | INFO | "malloc" : " 14 GB", | 20260505 21:42:07.553 | INFO | "pinned" : " 29 GB" | 20260505 21:42:07.553 | INFO | } | 20260505 21:42:07.553 | INFO | Perf report for the last 5.14207 seconds: | 20260505 21:42:07.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:42:07.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:07.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:07.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:07.553 | INFO | ANALYSIS : 61.6%, 3.0, 96.3ms, 0 | 20260505 21:42:07.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:07.553 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:07.553 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:42:07.553 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:07.553 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:07.553 | INFO | Uploaded 36GiB to GPU in 2.13792s (16.8388GiB/s) | 20260505 21:42:07.553 | INFO | Downloaded 34GiB to GPU in 2.06342s (16.4775GiB/s) | 20260505 21:42:12.693 | INFO | Analyzing chunk frames = [6519296,6519808) | 20260505 21:42:12.693 | INFO | MemInfo: { | 20260505 21:42:12.693 | INFO | "gpu" : " 17 GB", | 20260505 21:42:12.693 | INFO | "huge" : "", | 20260505 21:42:12.693 | INFO | "huge_shared" : " 27 GB", | 20260505 21:42:12.693 | INFO | "malloc" : " 14 GB", | 20260505 21:42:12.693 | INFO | "pinned" : " 29 GB" | 20260505 21:42:12.693 | INFO | } | 20260505 21:42:12.693 | INFO | Perf report for the last 5.14044 seconds: | 20260505 21:42:12.693 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:42:12.693 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:12.693 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:12.694 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:12.694 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260505 21:42:12.694 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:12.694 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:12.694 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:42:12.694 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:12.694 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:12.694 | INFO | Uploaded 36GiB to GPU in 2.08846s (17.2376GiB/s) | 20260505 21:42:12.694 | INFO | Downloaded 34GiB to GPU in 2.04037s (16.6637GiB/s) | 20260505 21:42:17.828 | INFO | Analyzing chunk frames = [6519808,6520320) | 20260505 21:42:17.828 | INFO | MemInfo: { | 20260505 21:42:17.828 | INFO | "gpu" : " 17 GB", | 20260505 21:42:17.828 | INFO | "huge" : "", | 20260505 21:42:17.828 | INFO | "huge_shared" : " 27 GB", | 20260505 21:42:17.828 | INFO | "malloc" : " 14 GB", | 20260505 21:42:17.828 | INFO | "pinned" : " 29 GB" | 20260505 21:42:17.828 | INFO | } | 20260505 21:42:17.828 | INFO | Perf report for the last 5.13481 seconds: | 20260505 21:42:17.828 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:42:17.828 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:17.828 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:17.828 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:17.828 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 21:42:17.828 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:17.828 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:17.828 | INFO | PRE_HQ : 27.9%, 1.0, 14.4ms, 0 | 20260505 21:42:17.828 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:17.828 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:17.829 | INFO | Uploaded 36GiB to GPU in 2.15228s (16.7264GiB/s) | 20260505 21:42:17.829 | INFO | Downloaded 34GiB to GPU in 2.1596s (15.7436GiB/s) | 20260505 21:42:22.967 | INFO | Analyzing chunk frames = [6520320,6520832) | 20260505 21:42:22.967 | INFO | MemInfo: { | 20260505 21:42:22.967 | INFO | "gpu" : " 17 GB", | 20260505 21:42:22.967 | INFO | "huge" : "", | 20260505 21:42:22.967 | INFO | "huge_shared" : " 27 GB", | 20260505 21:42:22.967 | INFO | "malloc" : " 14 GB", | 20260505 21:42:22.967 | INFO | "pinned" : " 29 GB" | 20260505 21:42:22.967 | INFO | } | 20260505 21:42:22.967 | INFO | Perf report for the last 5.13896 seconds: | 20260505 21:42:22.967 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:42:22.967 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:22.967 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:22.967 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:22.967 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 21:42:22.967 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:22.967 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:22.968 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 21:42:22.968 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:22.968 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:22.968 | INFO | Uploaded 36GiB to GPU in 2.10001s (17.1428GiB/s) | 20260505 21:42:22.968 | INFO | Downloaded 34GiB to GPU in 2.00497s (16.9578GiB/s) | 20260505 21:42:22.968 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 21:42:22.968 | INFO | Total mem usage: 15.3 GiB | 20260505 21:42:22.968 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 21:42:22.968 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.99%) | 20260505 21:42:22.968 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 21:42:22.968 | INFO | High Water Breakdown: | 20260505 21:42:22.968 | INFO | 60.9% : 8.57 GiB ( 90% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 21:42:22.968 | INFO | 38.9% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 21:42:22.968 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 21:42:22.968 | INFO | ------------------------------------------ | 20260505 21:42:22.968 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:42:22.968 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 21:42:22.968 | INFO | Total mem usage: 26.5 GiB | 20260505 21:42:22.968 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 21:42:22.968 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 21:42:22.968 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 21:42:22.968 | INFO | High Water Breakdown: | 20260505 21:42:22.968 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:42:22.968 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:42:22.968 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:42:22.968 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:42:22.968 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:42:22.968 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:42:22.968 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 21:42:22.968 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 21:42:22.968 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:42:22.968 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 21:42:22.968 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 21:42:22.968 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 21:42:22.968 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 21:42:22.968 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 21:42:22.968 | INFO | ------------------------------------------ | 20260505 21:42:22.968 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:42:22.968 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 21:42:22.968 | INFO | Total mem usage: 25.6 GiB | 20260505 21:42:22.968 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 21:42:22.968 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 21:42:22.968 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 21:42:22.968 | INFO | High Water Breakdown: | 20260505 21:42:22.968 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 21:42:22.968 | INFO | ------------------------------------------ | 20260505 21:42:22.968 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:42:22.968 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 21:42:22.968 | INFO | Total mem usage: 15.7 GiB | 20260505 21:42:22.969 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 21:42:22.969 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 21:42:22.969 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 21:42:22.969 | INFO | High Water Breakdown: | 20260505 21:42:22.969 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 21:42:22.969 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 21:42:22.969 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 21:42:22.969 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 21:42:22.969 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 21:42:22.969 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 21:42:22.969 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 21:42:22.969 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 21:42:22.969 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 21:42:22.969 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:42:22.969 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 21:42:22.969 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:42:22.969 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 21:42:22.969 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 21:42:22.969 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 21:42:22.969 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 21:42:22.969 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:42:22.969 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 21:42:22.969 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:42:22.969 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:42:22.969 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:42:22.969 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 21:42:22.969 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 21:42:22.969 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:42:22.969 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 21:42:22.969 | INFO | ------------------------------------------ | 20260505 21:42:22.969 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:42:22.969 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 21:42:22.969 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:42:22.969 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:42:22.969 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.43e+01, 1.35e+01 / 1.68e+00 | 20260505 21:42:22.969 | INFO | PerfReport: 0.29%, AggregateStats -- 9.24e-02 / 1.04e-01, 1.09e-02 / 3.46e-03 | 20260505 21:42:22.969 | INFO | PerfReport: 11.81%, Baseline -- 3.71e+00 / 4.55e+00, 5.69e-01 / 1.59e-01 | 20260505 21:42:22.969 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.54e+00, 1.91e-01 / 8.23e-02 | 20260505 21:42:22.969 | INFO | PerfReport: 15.98%, DME -- 4.01e+01 / 4.43e+01, 3.98e+00 / 1.74e+00 | 20260505 21:42:22.969 | INFO | PerfReport: 57.10%, FrameLabeling -- 1.79e+01 / 1.94e+01, 1.45e+00 / 1.06e+00 | 20260505 21:42:22.969 | INFO | PerfReport: 7.21%, PulseAccumulating -- 2.26e+00 / 2.80e+00, 4.35e-01 / 2.24e-01 | 20260505 21:42:22.969 | INFO | PerfReport: 3.20%, Metrics -- 1.01e+00 / 1.30e+00, 2.71e-01 / 1.38e-01 | 20260505 21:42:22.969 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.49e-01 / 4.07e-01, 6.10e-02 / 3.85e-02 | 20260505 21:42:22.969 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 21:42:22.969 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:42:22.969 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:42:22.969 | INFO | PerfReport: Full Execution -- 9.90e+01 / -nan, 2.50e+01 / 0.00e+00 | 20260505 21:42:22.969 | INFO | PerfReport: 22.42%, Upload -- 2.22e+01 / 0.00e+00, 7.66e+00 / 0.00e+00 | 20260505 21:42:22.969 | INFO | PerfReport: 21.82%, Download -- 2.16e+01 / 0.00e+00, 7.66e+00 / 0.00e+00 | 20260505 21:42:22.969 | INFO | PerfReport: 31.74%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 21:42:22.969 | INFO | PerfReport: 2.21%, LockUpload -- 2.19e+00 / 0.00e+00, 8.00e+00 / 0.00e+00 | 20260505 21:42:22.970 | INFO | PerfReport: 1.38%, LockDownload -- 1.37e+00 / 0.00e+00, 2.90e+00 / 0.00e+00 | 20260505 21:42:22.970 | INFO | PerfReport: 20.42%, LockCompute -- 2.02e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 21:42:22.970 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.76e-02 / 0.00e+00 | 20260505 21:42:23.104 | INFO | Starting a round of Baz IO | 20260505 21:42:26.261 | INFO | Populating RTMetrics report | 20260505 21:42:26.261 | INFO | RTMetrics report populated | 20260505 21:42:26.261 | INFO | Opening tmp RTMetrics json file | 20260505 21:42:26.262 | INFO | Serializing RTMetrics to json | 20260505 21:42:26.263 | INFO | Serialization done, swapping tmp file now... | 20260505 21:42:26.264 | INFO | RTMetrics done | 20260505 21:42:26.343 | INFO | Finished a round of Baz IO: Wrote 1.962 GiB in 3.238 seconds | 20260505 21:42:26.343 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.55% / 0.00% / 78.44% | 20260505 21:42:26.343 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.142018% / 7.94729e-06% / 9.93411e-05% | 20260505 21:42:26.343 | INFO | Baz IO overall: | 20260505 21:42:26.343 | INFO | BytesWritten : 3771473379172 | 20260505 21:42:26.343 | INFO | NumEvents : 1780392633265 | 20260505 21:42:28.115 | INFO | Analyzing chunk frames = [6520832,6521344) | 20260505 21:42:28.115 | INFO | MemInfo: { | 20260505 21:42:28.115 | INFO | "gpu" : " 17 GB", | 20260505 21:42:28.115 | INFO | "huge" : "", | 20260505 21:42:28.115 | INFO | "huge_shared" : " 27 GB", | 20260505 21:42:28.115 | INFO | "malloc" : " 14 GB", | 20260505 21:42:28.115 | INFO | "pinned" : " 29 GB" | 20260505 21:42:28.115 | INFO | } | 20260505 21:42:28.115 | INFO | Perf report for the last 5.14801 seconds: | 20260505 21:42:28.115 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:42:28.115 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:28.115 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:28.115 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:28.115 | INFO | ANALYSIS : 62.9%, 3.0, 98.4ms, 0 | 20260505 21:42:28.115 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:28.115 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260505 21:42:28.115 | INFO | PRE_HQ : 59.8%, 1.8, 54.6ms, 0 | 20260505 21:42:28.115 | INFO | BAZWRITER : 62.5%, 3.6, 117.0ms, 0 | 20260505 21:42:28.115 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 21:42:28.116 | INFO | Uploaded 36GiB to GPU in 2.17378s (16.561GiB/s) | 20260505 21:42:28.116 | INFO | Downloaded 38GiB to GPU in 2.247s (16.9114GiB/s) | 20260505 21:42:33.251 | INFO | Analyzing chunk frames = [6521344,6521856) | 20260505 21:42:33.251 | INFO | MemInfo: { | 20260505 21:42:33.251 | INFO | "gpu" : " 17 GB", | 20260505 21:42:33.251 | INFO | "huge" : "", | 20260505 21:42:33.251 | INFO | "huge_shared" : " 27 GB", | 20260505 21:42:33.251 | INFO | "malloc" : " 14 GB", | 20260505 21:42:33.251 | INFO | "pinned" : " 29 GB" | 20260505 21:42:33.251 | INFO | } | 20260505 21:42:33.251 | INFO | Perf report for the last 5.13591 seconds: | 20260505 21:42:33.251 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:42:33.251 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:33.251 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:33.251 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:33.251 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 21:42:33.251 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:33.251 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:33.251 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:42:33.251 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:33.251 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:33.252 | INFO | Uploaded 36GiB to GPU in 2.08705s (17.2492GiB/s) | 20260505 21:42:33.252 | INFO | Downloaded 34GiB to GPU in 2.02901s (16.757GiB/s) | 20260505 21:42:38.387 | INFO | Analyzing chunk frames = [6521856,6522368) | 20260505 21:42:38.387 | INFO | MemInfo: { | 20260505 21:42:38.387 | INFO | "gpu" : " 17 GB", | 20260505 21:42:38.387 | INFO | "huge" : "", | 20260505 21:42:38.387 | INFO | "huge_shared" : " 27 GB", | 20260505 21:42:38.387 | INFO | "malloc" : " 14 GB", | 20260505 21:42:38.387 | INFO | "pinned" : " 29 GB" | 20260505 21:42:38.387 | INFO | } | 20260505 21:42:38.387 | INFO | Perf report for the last 5.13626 seconds: | 20260505 21:42:38.387 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:42:38.387 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:38.387 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:38.387 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:38.388 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260505 21:42:38.388 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:38.388 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:38.388 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:42:38.388 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:38.388 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:38.388 | INFO | Uploaded 36GiB to GPU in 2.13535s (16.859GiB/s) | 20260505 21:42:38.388 | INFO | Downloaded 34GiB to GPU in 2.02493s (16.7907GiB/s) | 20260505 21:42:43.525 | INFO | Analyzing chunk frames = [6522368,6522880) | 20260505 21:42:43.525 | INFO | MemInfo: { | 20260505 21:42:43.525 | INFO | "gpu" : " 17 GB", | 20260505 21:42:43.525 | INFO | "huge" : "", | 20260505 21:42:43.525 | INFO | "huge_shared" : " 27 GB", | 20260505 21:42:43.525 | INFO | "malloc" : " 14 GB", | 20260505 21:42:43.525 | INFO | "pinned" : " 29 GB" | 20260505 21:42:43.525 | INFO | } | 20260505 21:42:43.526 | INFO | Perf report for the last 5.13808 seconds: | 20260505 21:42:43.526 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:42:43.526 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:43.526 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:43.526 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:43.526 | INFO | ANALYSIS : 62.4%, 3.0, 96.8ms, 0 | 20260505 21:42:43.526 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:43.526 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:43.526 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:42:43.526 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:43.526 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:43.526 | INFO | Uploaded 36GiB to GPU in 2.20943s (16.2938GiB/s) | 20260505 21:42:43.526 | INFO | Downloaded 34GiB to GPU in 2.05515s (16.5438GiB/s) | 20260505 21:42:48.672 | INFO | Analyzing chunk frames = [6522880,6523392) | 20260505 21:42:48.672 | INFO | MemInfo: { | 20260505 21:42:48.672 | INFO | "gpu" : " 17 GB", | 20260505 21:42:48.672 | INFO | "huge" : "", | 20260505 21:42:48.672 | INFO | "huge_shared" : " 27 GB", | 20260505 21:42:48.672 | INFO | "malloc" : " 14 GB", | 20260505 21:42:48.672 | INFO | "pinned" : " 29 GB" | 20260505 21:42:48.672 | INFO | } | 20260505 21:42:48.672 | INFO | Perf report for the last 5.1463 seconds: | 20260505 21:42:48.672 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:42:48.672 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:48.672 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:48.672 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:48.672 | INFO | ANALYSIS : 61.6%, 3.0, 96.3ms, 0 | 20260505 21:42:48.672 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:48.672 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:48.672 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 21:42:48.672 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:48.672 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:48.672 | INFO | Uploaded 36GiB to GPU in 2.12882s (16.9108GiB/s) | 20260505 21:42:48.672 | INFO | Downloaded 34GiB to GPU in 2.06352s (16.4767GiB/s) | 20260505 21:42:53.812 | INFO | Analyzing chunk frames = [6523392,6523904) | 20260505 21:42:53.812 | INFO | MemInfo: { | 20260505 21:42:53.812 | INFO | "gpu" : " 17 GB", | 20260505 21:42:53.812 | INFO | "huge" : "", | 20260505 21:42:53.812 | INFO | "huge_shared" : " 27 GB", | 20260505 21:42:53.812 | INFO | "malloc" : " 14 GB", | 20260505 21:42:53.812 | INFO | "pinned" : " 29 GB" | 20260505 21:42:53.812 | INFO | } | 20260505 21:42:53.812 | INFO | Perf report for the last 5.14056 seconds: | 20260505 21:42:53.813 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:42:53.813 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:53.813 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:53.813 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:53.813 | INFO | ANALYSIS : 61.5%, 3.0, 96.1ms, 0 | 20260505 21:42:53.813 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:53.813 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:53.813 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 21:42:53.813 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:53.813 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:53.813 | INFO | Uploaded 36GiB to GPU in 2.07719s (17.3311GiB/s) | 20260505 21:42:53.813 | INFO | Downloaded 34GiB to GPU in 2.03969s (16.6692GiB/s) | 20260505 21:42:58.945 | INFO | Analyzing chunk frames = [6523904,6524416) | 20260505 21:42:58.945 | INFO | MemInfo: { | 20260505 21:42:58.945 | INFO | "gpu" : " 17 GB", | 20260505 21:42:58.945 | INFO | "huge" : "", | 20260505 21:42:58.945 | INFO | "huge_shared" : " 27 GB", | 20260505 21:42:58.945 | INFO | "malloc" : " 14 GB", | 20260505 21:42:58.945 | INFO | "pinned" : " 29 GB" | 20260505 21:42:58.945 | INFO | } | 20260505 21:42:58.945 | INFO | Perf report for the last 5.13293 seconds: | 20260505 21:42:58.945 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:42:58.945 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:58.945 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:58.945 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:58.945 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260505 21:42:58.945 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:58.945 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:58.945 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 21:42:58.945 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:58.945 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:42:58.946 | INFO | Uploaded 36GiB to GPU in 2.09981s (17.1444GiB/s) | 20260505 21:42:58.946 | INFO | Downloaded 34GiB to GPU in 2.06706s (16.4485GiB/s) | 20260505 21:43:04.079 | INFO | Analyzing chunk frames = [6524416,6524928) | 20260505 21:43:04.079 | INFO | MemInfo: { | 20260505 21:43:04.079 | INFO | "gpu" : " 17 GB", | 20260505 21:43:04.079 | INFO | "huge" : "", | 20260505 21:43:04.079 | INFO | "huge_shared" : " 27 GB", | 20260505 21:43:04.079 | INFO | "malloc" : " 14 GB", | 20260505 21:43:04.079 | INFO | "pinned" : " 29 GB" | 20260505 21:43:04.079 | INFO | } | 20260505 21:43:04.079 | INFO | Perf report for the last 5.13324 seconds: | 20260505 21:43:04.079 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:43:04.079 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:04.079 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:04.079 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:04.079 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260505 21:43:04.079 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:04.079 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:04.079 | INFO | PRE_HQ : 27.5%, 1.0, 14.2ms, 0 | 20260505 21:43:04.079 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:04.079 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:04.079 | INFO | Uploaded 36GiB to GPU in 2.11509s (17.0206GiB/s) | 20260505 21:43:04.079 | INFO | Downloaded 34GiB to GPU in 2.01183s (16.9001GiB/s) | 20260505 21:43:04.207 | INFO | Starting a round of Baz IO | 20260505 21:43:07.377 | INFO | Populating RTMetrics report | 20260505 21:43:07.377 | INFO | RTMetrics report populated | 20260505 21:43:07.377 | INFO | Opening tmp RTMetrics json file | 20260505 21:43:07.378 | INFO | Serializing RTMetrics to json | 20260505 21:43:07.379 | INFO | Serialization done, swapping tmp file now... | 20260505 21:43:07.379 | INFO | RTMetrics done | 20260505 21:43:07.458 | INFO | Finished a round of Baz IO: Wrote 1.958 GiB in 3.25 seconds | 20260505 21:43:07.458 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.55% / 0.00% / 78.45% | 20260505 21:43:07.458 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.139046% / 3.97364e-06% / 6.75519e-05% | 20260505 21:43:07.458 | INFO | Baz IO overall: | 20260505 21:43:07.458 | INFO | BytesWritten : 3773575290653 | 20260505 21:43:07.459 | INFO | NumEvents : 1781361457018 | 20260505 21:43:09.221 | INFO | Analyzing chunk frames = [6524928,6525440) | 20260505 21:43:09.221 | INFO | MemInfo: { | 20260505 21:43:09.221 | INFO | "gpu" : " 17 GB", | 20260505 21:43:09.221 | INFO | "huge" : "", | 20260505 21:43:09.221 | INFO | "huge_shared" : " 27 GB", | 20260505 21:43:09.221 | INFO | "malloc" : " 14 GB", | 20260505 21:43:09.221 | INFO | "pinned" : " 29 GB" | 20260505 21:43:09.221 | INFO | } | 20260505 21:43:09.221 | INFO | Perf report for the last 5.14259 seconds: | 20260505 21:43:09.221 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:43:09.221 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:09.221 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:09.221 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:09.221 | INFO | ANALYSIS : 63.1%, 3.0, 98.3ms, 0 | 20260505 21:43:09.221 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:09.221 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260505 21:43:09.221 | INFO | PRE_HQ : 60.4%, 1.8, 55.4ms, 0 | 20260505 21:43:09.221 | INFO | BAZWRITER : 62.7%, 3.6, 116.7ms, 0 | 20260505 21:43:09.221 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 21:43:09.222 | INFO | Uploaded 36GiB to GPU in 2.16048s (16.663GiB/s) | 20260505 21:43:09.222 | INFO | Downloaded 38GiB to GPU in 2.24199s (16.9492GiB/s) | 20260505 21:43:14.370 | INFO | Analyzing chunk frames = [6525440,6525952) | 20260505 21:43:14.370 | INFO | MemInfo: { | 20260505 21:43:14.370 | INFO | "gpu" : " 17 GB", | 20260505 21:43:14.370 | INFO | "huge" : "", | 20260505 21:43:14.370 | INFO | "huge_shared" : " 27 GB", | 20260505 21:43:14.370 | INFO | "malloc" : " 14 GB", | 20260505 21:43:14.370 | INFO | "pinned" : " 29 GB" | 20260505 21:43:14.370 | INFO | } | 20260505 21:43:14.370 | INFO | Perf report for the last 5.14926 seconds: | 20260505 21:43:14.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:43:14.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:14.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:14.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:14.371 | INFO | ANALYSIS : 62.1%, 3.0, 97.1ms, 0 | 20260505 21:43:14.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:14.371 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:14.371 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 21:43:14.371 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:14.371 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:14.371 | INFO | Uploaded 36GiB to GPU in 2.09354s (17.1958GiB/s) | 20260505 21:43:14.371 | INFO | Downloaded 34GiB to GPU in 2.0239s (16.7992GiB/s) | 20260505 21:43:19.509 | INFO | Analyzing chunk frames = [6525952,6526464) | 20260505 21:43:19.509 | INFO | MemInfo: { | 20260505 21:43:19.509 | INFO | "gpu" : " 17 GB", | 20260505 21:43:19.509 | INFO | "huge" : "", | 20260505 21:43:19.509 | INFO | "huge_shared" : " 27 GB", | 20260505 21:43:19.509 | INFO | "malloc" : " 14 GB", | 20260505 21:43:19.509 | INFO | "pinned" : " 29 GB" | 20260505 21:43:19.509 | INFO | } | 20260505 21:43:19.509 | INFO | Perf report for the last 5.13862 seconds: | 20260505 21:43:19.509 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:43:19.509 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:19.509 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:19.509 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:19.509 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260505 21:43:19.509 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:19.509 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:19.509 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:43:19.509 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:19.509 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:19.509 | INFO | Uploaded 36GiB to GPU in 2.15084s (16.7376GiB/s) | 20260505 21:43:19.509 | INFO | Downloaded 34GiB to GPU in 2.05199s (16.5693GiB/s) | 20260505 21:43:24.645 | INFO | Analyzing chunk frames = [6526464,6526976) | 20260505 21:43:24.645 | INFO | MemInfo: { | 20260505 21:43:24.645 | INFO | "gpu" : " 17 GB", | 20260505 21:43:24.645 | INFO | "huge" : "", | 20260505 21:43:24.645 | INFO | "huge_shared" : " 27 GB", | 20260505 21:43:24.645 | INFO | "malloc" : " 14 GB", | 20260505 21:43:24.645 | INFO | "pinned" : " 29 GB" | 20260505 21:43:24.645 | INFO | } | 20260505 21:43:24.645 | INFO | Perf report for the last 5.13627 seconds: | 20260505 21:43:24.645 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:43:24.645 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:24.645 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:24.645 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:24.645 | INFO | ANALYSIS : 62.0%, 3.0, 96.2ms, 0 | 20260505 21:43:24.645 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:24.645 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:24.645 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 21:43:24.645 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:24.646 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:24.646 | INFO | Uploaded 36GiB to GPU in 2.13209s (16.8848GiB/s) | 20260505 21:43:24.646 | INFO | Downloaded 34GiB to GPU in 2.03407s (16.7152GiB/s) | 20260505 21:43:29.777 | INFO | Analyzing chunk frames = [6526976,6527488) | 20260505 21:43:29.777 | INFO | MemInfo: { | 20260505 21:43:29.777 | INFO | "gpu" : " 17 GB", | 20260505 21:43:29.777 | INFO | "huge" : "", | 20260505 21:43:29.777 | INFO | "huge_shared" : " 27 GB", | 20260505 21:43:29.777 | INFO | "malloc" : " 14 GB", | 20260505 21:43:29.777 | INFO | "pinned" : " 29 GB" | 20260505 21:43:29.777 | INFO | } | 20260505 21:43:29.777 | INFO | Perf report for the last 5.13142 seconds: | 20260505 21:43:29.777 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:43:29.777 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:29.777 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:29.777 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:29.777 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260505 21:43:29.777 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:29.777 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:29.777 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:43:29.777 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:29.777 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:29.777 | INFO | Uploaded 36GiB to GPU in 2.13666s (16.8487GiB/s) | 20260505 21:43:29.777 | INFO | Downloaded 34GiB to GPU in 2.06383s (16.4742GiB/s) | 20260505 21:43:34.922 | INFO | Analyzing chunk frames = [6527488,6528000) | 20260505 21:43:34.922 | INFO | MemInfo: { | 20260505 21:43:34.922 | INFO | "gpu" : " 17 GB", | 20260505 21:43:34.922 | INFO | "huge" : "", | 20260505 21:43:34.922 | INFO | "huge_shared" : " 27 GB", | 20260505 21:43:34.922 | INFO | "malloc" : " 14 GB", | 20260505 21:43:34.923 | INFO | "pinned" : " 29 GB" | 20260505 21:43:34.923 | INFO | } | 20260505 21:43:34.923 | INFO | Perf report for the last 5.14584 seconds: | 20260505 21:43:34.923 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:43:34.923 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:34.923 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:34.923 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:34.923 | INFO | ANALYSIS : 62.0%, 3.0, 97.0ms, 0 | 20260505 21:43:34.923 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:34.923 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:34.923 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:43:34.923 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:34.923 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:34.923 | INFO | Uploaded 36GiB to GPU in 2.15899s (16.6745GiB/s) | 20260505 21:43:34.923 | INFO | Downloaded 34GiB to GPU in 2.06481s (16.4664GiB/s) | 20260505 21:43:40.060 | INFO | Analyzing chunk frames = [6528000,6528512) | 20260505 21:43:40.060 | INFO | MemInfo: { | 20260505 21:43:40.060 | INFO | "gpu" : " 17 GB", | 20260505 21:43:40.060 | INFO | "huge" : "", | 20260505 21:43:40.060 | INFO | "huge_shared" : " 27 GB", | 20260505 21:43:40.060 | INFO | "malloc" : " 14 GB", | 20260505 21:43:40.060 | INFO | "pinned" : " 29 GB" | 20260505 21:43:40.060 | INFO | } | 20260505 21:43:40.060 | INFO | Perf report for the last 5.13721 seconds: | 20260505 21:43:40.060 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:43:40.060 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:40.060 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:40.060 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:40.060 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260505 21:43:40.060 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:40.060 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:40.060 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 21:43:40.060 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:40.060 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:40.060 | INFO | Uploaded 36GiB to GPU in 2.03266s (17.7108GiB/s) | 20260505 21:43:40.060 | INFO | Downloaded 34GiB to GPU in 2.04046s (16.6629GiB/s) | 20260505 21:43:45.199 | INFO | Analyzing chunk frames = [6528512,6529024) | 20260505 21:43:45.199 | INFO | MemInfo: { | 20260505 21:43:45.199 | INFO | "gpu" : " 17 GB", | 20260505 21:43:45.199 | INFO | "huge" : "", | 20260505 21:43:45.199 | INFO | "huge_shared" : " 27 GB", | 20260505 21:43:45.199 | INFO | "malloc" : " 14 GB", | 20260505 21:43:45.199 | INFO | "pinned" : " 29 GB" | 20260505 21:43:45.200 | INFO | } | 20260505 21:43:45.200 | INFO | Perf report for the last 5.13963 seconds: | 20260505 21:43:45.200 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:43:45.200 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:45.200 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:45.200 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:45.200 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 21:43:45.200 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:45.200 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:45.200 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:43:45.200 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:45.200 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:45.200 | INFO | Uploaded 36GiB to GPU in 2.10865s (17.0725GiB/s) | 20260505 21:43:45.200 | INFO | Downloaded 34GiB to GPU in 1.99385s (17.0524GiB/s) | 20260505 21:43:45.200 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 21:43:45.200 | INFO | Total mem usage: 15.3 GiB | 20260505 21:43:45.200 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 21:43:45.200 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.88%) | 20260505 21:43:45.200 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 21:43:45.200 | INFO | High Water Breakdown: | 20260505 21:43:45.200 | INFO | 60.8% : 8.56 GiB ( 90% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 21:43:45.200 | INFO | 39% : 5.49 GiB ( 82.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 21:43:45.200 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 21:43:45.200 | INFO | ------------------------------------------ | 20260505 21:43:45.200 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:43:45.200 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 21:43:45.200 | INFO | Total mem usage: 26.5 GiB | 20260505 21:43:45.200 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 21:43:45.200 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 21:43:45.200 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 21:43:45.200 | INFO | High Water Breakdown: | 20260505 21:43:45.200 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:43:45.200 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:43:45.200 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:43:45.200 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:43:45.200 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:43:45.200 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:43:45.200 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 21:43:45.200 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 21:43:45.200 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:43:45.200 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 21:43:45.200 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 21:43:45.200 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 21:43:45.200 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 21:43:45.200 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 21:43:45.200 | INFO | ------------------------------------------ | 20260505 21:43:45.200 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:43:45.200 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 21:43:45.200 | INFO | Total mem usage: 25.6 GiB | 20260505 21:43:45.201 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 21:43:45.201 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 21:43:45.201 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 21:43:45.201 | INFO | High Water Breakdown: | 20260505 21:43:45.201 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 21:43:45.201 | INFO | ------------------------------------------ | 20260505 21:43:45.201 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:43:45.201 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 21:43:45.201 | INFO | Total mem usage: 15.7 GiB | 20260505 21:43:45.201 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 21:43:45.201 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 21:43:45.201 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 21:43:45.201 | INFO | High Water Breakdown: | 20260505 21:43:45.201 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 21:43:45.201 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 21:43:45.201 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 21:43:45.201 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 21:43:45.201 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 21:43:45.201 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 21:43:45.201 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 21:43:45.201 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 21:43:45.201 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 21:43:45.201 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:43:45.201 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 21:43:45.201 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:43:45.201 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 21:43:45.201 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 21:43:45.201 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 21:43:45.201 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 21:43:45.201 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:43:45.201 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 21:43:45.201 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:43:45.201 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:43:45.201 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:43:45.201 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 21:43:45.201 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 21:43:45.201 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:43:45.201 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 21:43:45.201 | INFO | ------------------------------------------ | 20260505 21:43:45.201 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:43:45.201 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 21:43:45.201 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:43:45.201 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:43:45.201 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.42e+01, 1.35e+01 / 9.32e-01 | 20260505 21:43:45.201 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.04e-01, 1.03e-02 / 5.25e-03 | 20260505 21:43:45.201 | INFO | PerfReport: 11.80%, Baseline -- 3.71e+00 / 4.50e+00, 5.64e-01 / 2.04e-01 | 20260505 21:43:45.201 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.53e+00, 1.90e-01 / 8.64e-02 | 20260505 21:43:45.201 | INFO | PerfReport: 16.00%, DME -- 4.02e+01 / 4.42e+01, 3.85e+00 / 9.74e-01 | 20260505 21:43:45.201 | INFO | PerfReport: 57.15%, FrameLabeling -- 1.80e+01 / 1.93e+01, 1.49e+00 / 1.08e+00 | 20260505 21:43:45.201 | INFO | PerfReport: 7.19%, PulseAccumulating -- 2.26e+00 / 2.83e+00, 4.32e-01 / 1.91e-01 | 20260505 21:43:45.201 | INFO | PerfReport: 3.18%, Metrics -- 9.99e-01 / 1.31e+00, 2.69e-01 / 1.32e-01 | 20260505 21:43:45.201 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.47e-01 / 4.06e-01, 5.99e-02 / 2.68e-02 | 20260505 21:43:45.201 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 21:43:45.201 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:43:45.201 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:43:45.201 | INFO | PerfReport: Full Execution -- 9.90e+01 / -nan, 2.50e+01 / 0.00e+00 | 20260505 21:43:45.201 | INFO | PerfReport: 22.37%, Upload -- 2.21e+01 / 0.00e+00, 7.61e+00 / 0.00e+00 | 20260505 21:43:45.201 | INFO | PerfReport: 21.74%, Download -- 2.15e+01 / 0.00e+00, 7.57e+00 / 0.00e+00 | 20260505 21:43:45.201 | INFO | PerfReport: 31.75%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 21:43:45.201 | INFO | PerfReport: 2.23%, LockUpload -- 2.21e+00 / 0.00e+00, 7.91e+00 / 0.00e+00 | 20260505 21:43:45.201 | INFO | PerfReport: 1.31%, LockDownload -- 1.29e+00 / 0.00e+00, 2.86e+00 / 0.00e+00 | 20260505 21:43:45.201 | INFO | PerfReport: 20.58%, LockCompute -- 2.04e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 21:43:45.201 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.28e-02 / 0.00e+00 | 20260505 21:43:45.331 | INFO | Starting a round of Baz IO | 20260505 21:43:48.487 | INFO | Populating RTMetrics report | 20260505 21:43:48.487 | INFO | RTMetrics report populated | 20260505 21:43:48.487 | INFO | Opening tmp RTMetrics json file | 20260505 21:43:48.488 | INFO | Serializing RTMetrics to json | 20260505 21:43:48.490 | INFO | Serialization done, swapping tmp file now... | 20260505 21:43:48.491 | INFO | RTMetrics done | 20260505 21:43:48.569 | INFO | Finished a round of Baz IO: Wrote 1.958 GiB in 3.237 seconds | 20260505 21:43:48.569 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.55% / 0.00% / 78.45% | 20260505 21:43:48.569 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.139018% / 3.97364e-06% / 9.53674e-05% | 20260505 21:43:48.569 | INFO | Baz IO overall: | 20260505 21:43:48.569 | INFO | BytesWritten : 3775677296228 | 20260505 21:43:48.569 | INFO | NumEvents : 1782330363686 | 20260505 21:43:50.340 | INFO | Analyzing chunk frames = [6529024,6529536) | 20260505 21:43:50.340 | INFO | MemInfo: { | 20260505 21:43:50.340 | INFO | "gpu" : " 17 GB", | 20260505 21:43:50.340 | INFO | "huge" : "", | 20260505 21:43:50.340 | INFO | "huge_shared" : " 27 GB", | 20260505 21:43:50.340 | INFO | "malloc" : " 14 GB", | 20260505 21:43:50.340 | INFO | "pinned" : " 29 GB" | 20260505 21:43:50.340 | INFO | } | 20260505 21:43:50.340 | INFO | Perf report for the last 5.14023 seconds: | 20260505 21:43:50.340 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:43:50.340 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:50.340 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:50.340 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:50.340 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 21:43:50.340 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:50.340 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260505 21:43:50.340 | INFO | PRE_HQ : 59.7%, 1.8, 54.7ms, 0 | 20260505 21:43:50.340 | INFO | BAZWRITER : 62.4%, 3.6, 115.8ms, 0 | 20260505 21:43:50.340 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 21:43:50.340 | INFO | Uploaded 36GiB to GPU in 2.15739s (16.6868GiB/s) | 20260505 21:43:50.340 | INFO | Downloaded 38GiB to GPU in 2.23171s (17.0273GiB/s) | 20260505 21:43:55.481 | INFO | Analyzing chunk frames = [6529536,6530048) | 20260505 21:43:55.481 | INFO | MemInfo: { | 20260505 21:43:55.481 | INFO | "gpu" : " 17 GB", | 20260505 21:43:55.481 | INFO | "huge" : "", | 20260505 21:43:55.481 | INFO | "huge_shared" : " 27 GB", | 20260505 21:43:55.481 | INFO | "malloc" : " 14 GB", | 20260505 21:43:55.481 | INFO | "pinned" : " 29 GB" | 20260505 21:43:55.481 | INFO | } | 20260505 21:43:55.481 | INFO | Perf report for the last 5.1416 seconds: | 20260505 21:43:55.481 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:43:55.481 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:55.481 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:55.481 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:55.482 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 21:43:55.482 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:55.482 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:55.482 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 21:43:55.482 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:55.482 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:43:55.482 | INFO | Uploaded 36GiB to GPU in 2.15538s (16.7024GiB/s) | 20260505 21:43:55.482 | INFO | Downloaded 34GiB to GPU in 2.05141s (16.574GiB/s) | 20260505 21:44:00.622 | INFO | Analyzing chunk frames = [6530048,6530560) | 20260505 21:44:00.622 | INFO | MemInfo: { | 20260505 21:44:00.622 | INFO | "gpu" : " 17 GB", | 20260505 21:44:00.622 | INFO | "huge" : "", | 20260505 21:44:00.622 | INFO | "huge_shared" : " 27 GB", | 20260505 21:44:00.622 | INFO | "malloc" : " 14 GB", | 20260505 21:44:00.622 | INFO | "pinned" : " 29 GB" | 20260505 21:44:00.622 | INFO | } | 20260505 21:44:00.622 | INFO | Perf report for the last 5.14027 seconds: | 20260505 21:44:00.622 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:44:00.622 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:00.622 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:00.622 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:00.622 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260505 21:44:00.622 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:00.622 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:00.622 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:44:00.622 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:00.622 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:00.622 | INFO | Uploaded 36GiB to GPU in 2.14056s (16.818GiB/s) | 20260505 21:44:00.622 | INFO | Downloaded 34GiB to GPU in 2.02447s (16.7945GiB/s) | 20260505 21:44:05.762 | INFO | Analyzing chunk frames = [6530560,6531072) | 20260505 21:44:05.762 | INFO | MemInfo: { | 20260505 21:44:05.762 | INFO | "gpu" : " 17 GB", | 20260505 21:44:05.762 | INFO | "huge" : "", | 20260505 21:44:05.762 | INFO | "huge_shared" : " 27 GB", | 20260505 21:44:05.762 | INFO | "malloc" : " 14 GB", | 20260505 21:44:05.762 | INFO | "pinned" : " 29 GB" | 20260505 21:44:05.762 | INFO | } | 20260505 21:44:05.762 | INFO | Perf report for the last 5.14013 seconds: | 20260505 21:44:05.762 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:44:05.762 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:05.762 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:05.762 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:05.762 | INFO | ANALYSIS : 61.9%, 3.0, 96.2ms, 0 | 20260505 21:44:05.762 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:05.762 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:05.762 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 21:44:05.762 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:05.762 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:05.762 | INFO | Uploaded 36GiB to GPU in 2.13423s (16.8679GiB/s) | 20260505 21:44:05.762 | INFO | Downloaded 34GiB to GPU in 2.02592s (16.7825GiB/s) | 20260505 21:44:10.898 | INFO | Analyzing chunk frames = [6531072,6531584) | 20260505 21:44:10.898 | INFO | MemInfo: { | 20260505 21:44:10.898 | INFO | "gpu" : " 17 GB", | 20260505 21:44:10.898 | INFO | "huge" : "", | 20260505 21:44:10.898 | INFO | "huge_shared" : " 27 GB", | 20260505 21:44:10.898 | INFO | "malloc" : " 14 GB", | 20260505 21:44:10.898 | INFO | "pinned" : " 29 GB" | 20260505 21:44:10.898 | INFO | } | 20260505 21:44:10.898 | INFO | Perf report for the last 5.1366 seconds: | 20260505 21:44:10.898 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:44:10.898 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:10.898 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:10.899 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:10.899 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260505 21:44:10.899 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:10.899 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:10.899 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 21:44:10.899 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:10.899 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:10.899 | INFO | Uploaded 36GiB to GPU in 2.09321s (17.1985GiB/s) | 20260505 21:44:10.899 | INFO | Downloaded 34GiB to GPU in 2.03935s (16.672GiB/s) | 20260505 21:44:16.030 | INFO | Analyzing chunk frames = [6531584,6532096) | 20260505 21:44:16.030 | INFO | MemInfo: { | 20260505 21:44:16.030 | INFO | "gpu" : " 17 GB", | 20260505 21:44:16.030 | INFO | "huge" : "", | 20260505 21:44:16.030 | INFO | "huge_shared" : " 27 GB", | 20260505 21:44:16.030 | INFO | "malloc" : " 14 GB", | 20260505 21:44:16.030 | INFO | "pinned" : " 29 GB" | 20260505 21:44:16.030 | INFO | } | 20260505 21:44:16.030 | INFO | Perf report for the last 5.13195 seconds: | 20260505 21:44:16.030 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:44:16.030 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:16.030 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:16.030 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:16.030 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260505 21:44:16.031 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:16.031 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:16.031 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 21:44:16.031 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:16.031 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:16.031 | INFO | Uploaded 36GiB to GPU in 2.07373s (17.36GiB/s) | 20260505 21:44:16.031 | INFO | Downloaded 34GiB to GPU in 2.0316s (16.7356GiB/s) | 20260505 21:44:21.176 | INFO | Analyzing chunk frames = [6532096,6532608) | 20260505 21:44:21.176 | INFO | MemInfo: { | 20260505 21:44:21.176 | INFO | "gpu" : " 17 GB", | 20260505 21:44:21.176 | INFO | "huge" : "", | 20260505 21:44:21.176 | INFO | "huge_shared" : " 27 GB", | 20260505 21:44:21.176 | INFO | "malloc" : " 14 GB", | 20260505 21:44:21.176 | INFO | "pinned" : " 29 GB" | 20260505 21:44:21.176 | INFO | } | 20260505 21:44:21.177 | INFO | Perf report for the last 5.14627 seconds: | 20260505 21:44:21.177 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:44:21.177 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:21.177 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:21.177 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:21.177 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 21:44:21.177 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:21.177 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:21.177 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 21:44:21.177 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:21.177 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:21.177 | INFO | Uploaded 36GiB to GPU in 2.12152s (16.9689GiB/s) | 20260505 21:44:21.177 | INFO | Downloaded 34GiB to GPU in 2.12391s (16.0082GiB/s) | 20260505 21:44:26.322 | INFO | Analyzing chunk frames = [6532608,6533120) | 20260505 21:44:26.322 | INFO | MemInfo: { | 20260505 21:44:26.322 | INFO | "gpu" : " 17 GB", | 20260505 21:44:26.322 | INFO | "huge" : "", | 20260505 21:44:26.322 | INFO | "huge_shared" : " 27 GB", | 20260505 21:44:26.322 | INFO | "malloc" : " 14 GB", | 20260505 21:44:26.322 | INFO | "pinned" : " 29 GB" | 20260505 21:44:26.322 | INFO | } | 20260505 21:44:26.322 | INFO | Perf report for the last 5.14566 seconds: | 20260505 21:44:26.322 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:44:26.322 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:26.322 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:26.322 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:26.322 | INFO | ANALYSIS : 62.0%, 3.0, 97.0ms, 0 | 20260505 21:44:26.322 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:26.322 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:26.322 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:44:26.322 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:26.322 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:26.323 | INFO | Uploaded 36GiB to GPU in 2.10666s (17.0887GiB/s) | 20260505 21:44:26.323 | INFO | Downloaded 34GiB to GPU in 2.01019s (16.9139GiB/s) | 20260505 21:44:26.448 | INFO | Starting a round of Baz IO | 20260505 21:44:29.596 | INFO | Populating RTMetrics report | 20260505 21:44:29.596 | INFO | RTMetrics report populated | 20260505 21:44:29.596 | INFO | Opening tmp RTMetrics json file | 20260505 21:44:29.597 | INFO | Serializing RTMetrics to json | 20260505 21:44:29.598 | INFO | Serialization done, swapping tmp file now... | 20260505 21:44:29.598 | INFO | RTMetrics done | 20260505 21:44:29.678 | INFO | Finished a round of Baz IO: Wrote 1.956 GiB in 3.23 seconds | 20260505 21:44:29.678 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.55% / 0.00% / 78.45% | 20260505 21:44:29.678 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.138692% / 0% / 8.34465e-05% | 20260505 21:44:29.678 | INFO | Baz IO overall: | 20260505 21:44:29.679 | INFO | BytesWritten : 3777777571371 | 20260505 21:44:29.679 | INFO | NumEvents : 1783298369469 | 20260505 21:44:31.459 | INFO | Analyzing chunk frames = [6533120,6533632) | 20260505 21:44:31.459 | INFO | MemInfo: { | 20260505 21:44:31.459 | INFO | "gpu" : " 17 GB", | 20260505 21:44:31.459 | INFO | "huge" : "", | 20260505 21:44:31.459 | INFO | "huge_shared" : " 27 GB", | 20260505 21:44:31.459 | INFO | "malloc" : " 14 GB", | 20260505 21:44:31.459 | INFO | "pinned" : " 29 GB" | 20260505 21:44:31.459 | INFO | } | 20260505 21:44:31.459 | INFO | Perf report for the last 5.13679 seconds: | 20260505 21:44:31.459 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:44:31.459 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:31.459 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:31.459 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:31.459 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260505 21:44:31.459 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:31.459 | INFO | RT_METRICS : 43.8%, 1.0, 22.9ms, 0 | 20260505 21:44:31.459 | INFO | PRE_HQ : 60.0%, 1.8, 54.9ms, 0 | 20260505 21:44:31.459 | INFO | BAZWRITER : 62.1%, 3.6, 115.8ms, 0 | 20260505 21:44:31.459 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 21:44:31.459 | INFO | Uploaded 36GiB to GPU in 2.16921s (16.5959GiB/s) | 20260505 21:44:31.459 | INFO | Downloaded 38GiB to GPU in 2.23618s (16.9933GiB/s) | 20260505 21:44:36.588 | INFO | Analyzing chunk frames = [6533632,6534144) | 20260505 21:44:36.588 | INFO | MemInfo: { | 20260505 21:44:36.588 | INFO | "gpu" : " 17 GB", | 20260505 21:44:36.588 | INFO | "huge" : "", | 20260505 21:44:36.588 | INFO | "huge_shared" : " 27 GB", | 20260505 21:44:36.588 | INFO | "malloc" : " 14 GB", | 20260505 21:44:36.588 | INFO | "pinned" : " 29 GB" | 20260505 21:44:36.588 | INFO | } | 20260505 21:44:36.588 | INFO | Perf report for the last 5.12881 seconds: | 20260505 21:44:36.588 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:44:36.588 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:36.588 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:36.588 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:36.588 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260505 21:44:36.588 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:36.588 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:36.588 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 21:44:36.588 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:36.588 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:36.588 | INFO | Uploaded 36GiB to GPU in 2.07973s (17.31GiB/s) | 20260505 21:44:36.588 | INFO | Downloaded 34GiB to GPU in 2.02391s (16.7991GiB/s) | 20260505 21:44:41.729 | INFO | Analyzing chunk frames = [6534144,6534656) | 20260505 21:44:41.729 | INFO | MemInfo: { | 20260505 21:44:41.729 | INFO | "gpu" : " 17 GB", | 20260505 21:44:41.729 | INFO | "huge" : "", | 20260505 21:44:41.729 | INFO | "huge_shared" : " 27 GB", | 20260505 21:44:41.729 | INFO | "malloc" : " 14 GB", | 20260505 21:44:41.729 | INFO | "pinned" : " 29 GB" | 20260505 21:44:41.729 | INFO | } | 20260505 21:44:41.729 | INFO | Perf report for the last 5.14119 seconds: | 20260505 21:44:41.729 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:44:41.729 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:41.729 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:41.729 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:41.729 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260505 21:44:41.729 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:41.729 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:41.729 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:44:41.729 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:41.729 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:41.729 | INFO | Uploaded 36GiB to GPU in 2.12499s (16.9413GiB/s) | 20260505 21:44:41.730 | INFO | Downloaded 34GiB to GPU in 2.02619s (16.7803GiB/s) | 20260505 21:44:46.870 | INFO | Analyzing chunk frames = [6534656,6535168) | 20260505 21:44:46.870 | INFO | MemInfo: { | 20260505 21:44:46.870 | INFO | "gpu" : " 17 GB", | 20260505 21:44:46.870 | INFO | "huge" : "", | 20260505 21:44:46.870 | INFO | "huge_shared" : " 27 GB", | 20260505 21:44:46.870 | INFO | "malloc" : " 14 GB", | 20260505 21:44:46.870 | INFO | "pinned" : " 29 GB" | 20260505 21:44:46.870 | INFO | } | 20260505 21:44:46.870 | INFO | Perf report for the last 5.14099 seconds: | 20260505 21:44:46.870 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:44:46.870 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:46.870 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:46.870 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:46.870 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260505 21:44:46.870 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:46.870 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:46.870 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:44:46.870 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:46.870 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:46.871 | INFO | Uploaded 36GiB to GPU in 2.20638s (16.3163GiB/s) | 20260505 21:44:46.871 | INFO | Downloaded 34GiB to GPU in 2.07452s (16.3894GiB/s) | 20260505 21:44:52.015 | INFO | Analyzing chunk frames = [6535168,6535680) | 20260505 21:44:52.015 | INFO | MemInfo: { | 20260505 21:44:52.015 | INFO | "gpu" : " 17 GB", | 20260505 21:44:52.015 | INFO | "huge" : "", | 20260505 21:44:52.015 | INFO | "huge_shared" : " 27 GB", | 20260505 21:44:52.015 | INFO | "malloc" : " 14 GB", | 20260505 21:44:52.015 | INFO | "pinned" : " 29 GB" | 20260505 21:44:52.015 | INFO | } | 20260505 21:44:52.015 | INFO | Perf report for the last 5.14488 seconds: | 20260505 21:44:52.015 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:44:52.015 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:52.015 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:52.015 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:52.015 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260505 21:44:52.015 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:52.015 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:52.015 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:44:52.015 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:52.015 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:52.015 | INFO | Uploaded 36GiB to GPU in 2.08999s (17.225GiB/s) | 20260505 21:44:52.015 | INFO | Downloaded 34GiB to GPU in 2.05042s (16.582GiB/s) | 20260505 21:44:57.149 | INFO | Analyzing chunk frames = [6535680,6536192) | 20260505 21:44:57.149 | INFO | MemInfo: { | 20260505 21:44:57.149 | INFO | "gpu" : " 17 GB", | 20260505 21:44:57.149 | INFO | "huge" : "", | 20260505 21:44:57.149 | INFO | "huge_shared" : " 27 GB", | 20260505 21:44:57.149 | INFO | "malloc" : " 14 GB", | 20260505 21:44:57.149 | INFO | "pinned" : " 29 GB" | 20260505 21:44:57.149 | INFO | } | 20260505 21:44:57.149 | INFO | Perf report for the last 5.13421 seconds: | 20260505 21:44:57.149 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:44:57.149 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:57.149 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:57.149 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:57.149 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260505 21:44:57.149 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:57.149 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:57.149 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:44:57.149 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:57.150 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:44:57.150 | INFO | Uploaded 36GiB to GPU in 2.07638s (17.3379GiB/s) | 20260505 21:44:57.150 | INFO | Downloaded 34GiB to GPU in 2.03935s (16.672GiB/s) | 20260505 21:45:02.289 | INFO | Analyzing chunk frames = [6536192,6536704) | 20260505 21:45:02.289 | INFO | MemInfo: { | 20260505 21:45:02.289 | INFO | "gpu" : " 17 GB", | 20260505 21:45:02.289 | INFO | "huge" : "", | 20260505 21:45:02.289 | INFO | "huge_shared" : " 27 GB", | 20260505 21:45:02.289 | INFO | "malloc" : " 14 GB", | 20260505 21:45:02.289 | INFO | "pinned" : " 29 GB" | 20260505 21:45:02.289 | INFO | } | 20260505 21:45:02.290 | INFO | Perf report for the last 5.14021 seconds: | 20260505 21:45:02.290 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:45:02.290 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:02.290 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:02.290 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:02.290 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 21:45:02.290 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:02.290 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:02.290 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 21:45:02.290 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:02.290 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:02.290 | INFO | Uploaded 36GiB to GPU in 2.1512s (16.7349GiB/s) | 20260505 21:45:02.290 | INFO | Downloaded 34GiB to GPU in 2.12826s (15.9755GiB/s) | 20260505 21:45:07.435 | INFO | Analyzing chunk frames = [6536704,6537216) | 20260505 21:45:07.435 | INFO | MemInfo: { | 20260505 21:45:07.435 | INFO | "gpu" : " 17 GB", | 20260505 21:45:07.435 | INFO | "huge" : "", | 20260505 21:45:07.435 | INFO | "huge_shared" : " 27 GB", | 20260505 21:45:07.435 | INFO | "malloc" : " 14 GB", | 20260505 21:45:07.435 | INFO | "pinned" : " 29 GB" | 20260505 21:45:07.435 | INFO | } | 20260505 21:45:07.435 | INFO | Perf report for the last 5.1457 seconds: | 20260505 21:45:07.435 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:45:07.435 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:07.435 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:07.435 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:07.435 | INFO | ANALYSIS : 61.9%, 3.0, 96.8ms, 0 | 20260505 21:45:07.435 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:07.435 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:07.435 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:45:07.435 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:07.435 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:07.436 | INFO | Uploaded 36GiB to GPU in 2.08135s (17.2964GiB/s) | 20260505 21:45:07.436 | INFO | Downloaded 34GiB to GPU in 2.00595s (16.9496GiB/s) | 20260505 21:45:07.436 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 21:45:07.436 | INFO | Total mem usage: 15.3 GiB | 20260505 21:45:07.436 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 21:45:07.436 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.81%) | 20260505 21:45:07.436 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 21:45:07.436 | INFO | High Water Breakdown: | 20260505 21:45:07.436 | INFO | 61% : 8.59 GiB ( 90.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 21:45:07.436 | INFO | 38.9% : 5.48 GiB ( 82.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 21:45:07.436 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 21:45:07.436 | INFO | ------------------------------------------ | 20260505 21:45:07.436 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:45:07.436 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 21:45:07.436 | INFO | Total mem usage: 26.5 GiB | 20260505 21:45:07.436 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 21:45:07.436 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 21:45:07.436 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 21:45:07.436 | INFO | High Water Breakdown: | 20260505 21:45:07.436 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:45:07.436 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:45:07.436 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:45:07.436 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:45:07.436 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:45:07.436 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:45:07.436 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 21:45:07.436 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 21:45:07.436 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:45:07.436 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 21:45:07.436 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 21:45:07.436 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 21:45:07.436 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 21:45:07.436 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 21:45:07.436 | INFO | ------------------------------------------ | 20260505 21:45:07.436 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:45:07.436 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 21:45:07.436 | INFO | Total mem usage: 25.6 GiB | 20260505 21:45:07.436 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 21:45:07.436 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 21:45:07.436 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 21:45:07.436 | INFO | High Water Breakdown: | 20260505 21:45:07.436 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 21:45:07.436 | INFO | ------------------------------------------ | 20260505 21:45:07.436 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:45:07.436 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 21:45:07.436 | INFO | Total mem usage: 15.7 GiB | 20260505 21:45:07.436 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 21:45:07.436 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 21:45:07.436 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 21:45:07.436 | INFO | High Water Breakdown: | 20260505 21:45:07.436 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 21:45:07.436 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 21:45:07.436 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 21:45:07.436 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 21:45:07.437 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 21:45:07.437 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 21:45:07.437 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 21:45:07.437 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 21:45:07.437 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 21:45:07.437 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:45:07.437 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 21:45:07.437 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:45:07.437 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 21:45:07.437 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 21:45:07.437 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 21:45:07.437 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 21:45:07.437 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:45:07.437 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 21:45:07.437 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:45:07.437 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:45:07.437 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:45:07.437 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 21:45:07.437 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 21:45:07.437 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:45:07.437 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 21:45:07.437 | INFO | ------------------------------------------ | 20260505 21:45:07.437 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:45:07.437 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 21:45:07.437 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:45:07.437 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:45:07.437 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.43e+01, 1.34e+01 / 1.69e+00 | 20260505 21:45:07.437 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.04e-01, 1.11e-02 / 5.71e-03 | 20260505 21:45:07.437 | INFO | PerfReport: 11.80%, Baseline -- 3.71e+00 / 4.57e+00, 5.71e-01 / 8.78e-02 | 20260505 21:45:07.437 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.51e+00, 1.88e-01 / 9.23e-02 | 20260505 21:45:07.437 | INFO | PerfReport: 16.01%, DME -- 4.03e+01 / 4.43e+01, 3.94e+00 / 1.69e+00 | 20260505 21:45:07.437 | INFO | PerfReport: 57.18%, FrameLabeling -- 1.80e+01 / 1.93e+01, 1.51e+00 / 1.04e+00 | 20260505 21:45:07.437 | INFO | PerfReport: 7.16%, PulseAccumulating -- 2.25e+00 / 2.80e+00, 4.31e-01 / 2.24e-01 | 20260505 21:45:07.437 | INFO | PerfReport: 3.18%, Metrics -- 1.00e+00 / 1.30e+00, 2.72e-01 / 1.53e-01 | 20260505 21:45:07.437 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.48e-01 / 4.07e-01, 5.94e-02 / 3.66e-02 | 20260505 21:45:07.437 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 21:45:07.437 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:45:07.437 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:45:07.437 | INFO | PerfReport: Full Execution -- 9.89e+01 / -nan, 2.49e+01 / 0.00e+00 | 20260505 21:45:07.437 | INFO | PerfReport: 22.36%, Upload -- 2.21e+01 / 0.00e+00, 7.58e+00 / 0.00e+00 | 20260505 21:45:07.437 | INFO | PerfReport: 21.81%, Download -- 2.16e+01 / 0.00e+00, 7.63e+00 / 0.00e+00 | 20260505 21:45:07.437 | INFO | PerfReport: 31.81%, Compute -- 3.15e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 21:45:07.437 | INFO | PerfReport: 2.18%, LockUpload -- 2.15e+00 / 0.00e+00, 7.93e+00 / 0.00e+00 | 20260505 21:45:07.437 | INFO | PerfReport: 1.31%, LockDownload -- 1.30e+00 / 0.00e+00, 2.87e+00 / 0.00e+00 | 20260505 21:45:07.437 | INFO | PerfReport: 20.52%, LockCompute -- 2.03e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 21:45:07.437 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 2.03e-02 / 0.00e+00 | 20260505 21:45:07.562 | INFO | Starting a round of Baz IO | 20260505 21:45:10.714 | INFO | Populating RTMetrics report | 20260505 21:45:10.714 | INFO | RTMetrics report populated | 20260505 21:45:10.714 | INFO | Opening tmp RTMetrics json file | 20260505 21:45:10.715 | INFO | Serializing RTMetrics to json | 20260505 21:45:10.716 | INFO | Serialization done, swapping tmp file now... | 20260505 21:45:10.717 | INFO | RTMetrics done | 20260505 21:45:10.799 | INFO | Finished a round of Baz IO: Wrote 1.955 GiB in 3.237 seconds | 20260505 21:45:10.799 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.55% / 0.00% / 78.45% | 20260505 21:45:10.799 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.138887% / 3.97364e-06% / 7.94729e-05% | 20260505 21:45:10.799 | INFO | Baz IO overall: | 20260505 21:45:10.799 | INFO | BytesWritten : 3779877084761 | 20260505 21:45:10.799 | INFO | NumEvents : 1784266146286 | 20260505 21:45:12.570 | INFO | Analyzing chunk frames = [6537216,6537728) | 20260505 21:45:12.570 | INFO | MemInfo: { | 20260505 21:45:12.570 | INFO | "gpu" : " 17 GB", | 20260505 21:45:12.570 | INFO | "huge" : "", | 20260505 21:45:12.570 | INFO | "huge_shared" : " 27 GB", | 20260505 21:45:12.570 | INFO | "malloc" : " 14 GB", | 20260505 21:45:12.570 | INFO | "pinned" : " 29 GB" | 20260505 21:45:12.570 | INFO | } | 20260505 21:45:12.570 | INFO | Perf report for the last 5.13446 seconds: | 20260505 21:45:12.570 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:45:12.570 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:12.570 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:12.570 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:12.570 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260505 21:45:12.570 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:12.570 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260505 21:45:12.570 | INFO | PRE_HQ : 60.0%, 1.8, 54.6ms, 0 | 20260505 21:45:12.570 | INFO | BAZWRITER : 62.3%, 3.6, 116.3ms, 0 | 20260505 21:45:12.570 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 21:45:12.570 | INFO | Uploaded 36GiB to GPU in 2.16456s (16.6315GiB/s) | 20260505 21:45:12.570 | INFO | Downloaded 38GiB to GPU in 2.23665s (16.9897GiB/s) | 20260505 21:45:17.709 | INFO | Analyzing chunk frames = [6537728,6538240) | 20260505 21:45:17.709 | INFO | MemInfo: { | 20260505 21:45:17.709 | INFO | "gpu" : " 17 GB", | 20260505 21:45:17.709 | INFO | "huge" : "", | 20260505 21:45:17.709 | INFO | "huge_shared" : " 27 GB", | 20260505 21:45:17.709 | INFO | "malloc" : " 14 GB", | 20260505 21:45:17.709 | INFO | "pinned" : " 29 GB" | 20260505 21:45:17.709 | INFO | } | 20260505 21:45:17.709 | INFO | Perf report for the last 5.1391 seconds: | 20260505 21:45:17.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:45:17.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:17.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:17.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:17.709 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 21:45:17.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:17.709 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:17.709 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:45:17.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:17.709 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:17.709 | INFO | Uploaded 36GiB to GPU in 2.08696s (17.25GiB/s) | 20260505 21:45:17.709 | INFO | Downloaded 34GiB to GPU in 2.02844s (16.7616GiB/s) | 20260505 21:45:22.850 | INFO | Analyzing chunk frames = [6538240,6538752) | 20260505 21:45:22.850 | INFO | MemInfo: { | 20260505 21:45:22.850 | INFO | "gpu" : " 17 GB", | 20260505 21:45:22.850 | INFO | "huge" : "", | 20260505 21:45:22.850 | INFO | "huge_shared" : " 27 GB", | 20260505 21:45:22.850 | INFO | "malloc" : " 14 GB", | 20260505 21:45:22.850 | INFO | "pinned" : " 29 GB" | 20260505 21:45:22.850 | INFO | } | 20260505 21:45:22.850 | INFO | Perf report for the last 5.14095 seconds: | 20260505 21:45:22.850 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:45:22.850 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:22.850 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:22.850 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:22.850 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260505 21:45:22.850 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:22.850 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:22.850 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:45:22.850 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:22.850 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:22.850 | INFO | Uploaded 36GiB to GPU in 2.1353s (16.8595GiB/s) | 20260505 21:45:22.850 | INFO | Downloaded 34GiB to GPU in 2.02301s (16.8066GiB/s) | 20260505 21:45:27.986 | INFO | Analyzing chunk frames = [6538752,6539264) | 20260505 21:45:27.986 | INFO | MemInfo: { | 20260505 21:45:27.986 | INFO | "gpu" : " 17 GB", | 20260505 21:45:27.986 | INFO | "huge" : "", | 20260505 21:45:27.986 | INFO | "huge_shared" : " 27 GB", | 20260505 21:45:27.986 | INFO | "malloc" : " 14 GB", | 20260505 21:45:27.986 | INFO | "pinned" : " 29 GB" | 20260505 21:45:27.986 | INFO | } | 20260505 21:45:27.986 | INFO | Perf report for the last 5.13649 seconds: | 20260505 21:45:27.986 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:45:27.986 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:27.986 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:27.986 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:27.986 | INFO | ANALYSIS : 61.9%, 3.0, 96.1ms, 0 | 20260505 21:45:27.986 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:27.986 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:27.986 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:45:27.986 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:27.986 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:27.987 | INFO | Uploaded 36GiB to GPU in 2.14485s (16.7844GiB/s) | 20260505 21:45:27.987 | INFO | Downloaded 34GiB to GPU in 2.03877s (16.6767GiB/s) | 20260505 21:45:33.126 | INFO | Analyzing chunk frames = [6539264,6539776) | 20260505 21:45:33.126 | INFO | MemInfo: { | 20260505 21:45:33.126 | INFO | "gpu" : " 17 GB", | 20260505 21:45:33.126 | INFO | "huge" : "", | 20260505 21:45:33.126 | INFO | "huge_shared" : " 27 GB", | 20260505 21:45:33.126 | INFO | "malloc" : " 14 GB", | 20260505 21:45:33.126 | INFO | "pinned" : " 29 GB" | 20260505 21:45:33.126 | INFO | } | 20260505 21:45:33.126 | INFO | Perf report for the last 5.13987 seconds: | 20260505 21:45:33.126 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:45:33.126 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:33.126 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:33.126 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:33.126 | INFO | ANALYSIS : 61.6%, 3.0, 96.2ms, 0 | 20260505 21:45:33.126 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:33.126 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:33.126 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:45:33.126 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:33.126 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:33.126 | INFO | Uploaded 36GiB to GPU in 2.13856s (16.8337GiB/s) | 20260505 21:45:33.126 | INFO | Downloaded 34GiB to GPU in 2.06258s (16.4842GiB/s) | 20260505 21:45:38.268 | INFO | Analyzing chunk frames = [6539776,6540288) | 20260505 21:45:38.268 | INFO | MemInfo: { | 20260505 21:45:38.268 | INFO | "gpu" : " 17 GB", | 20260505 21:45:38.268 | INFO | "huge" : "", | 20260505 21:45:38.268 | INFO | "huge_shared" : " 27 GB", | 20260505 21:45:38.268 | INFO | "malloc" : " 14 GB", | 20260505 21:45:38.268 | INFO | "pinned" : " 29 GB" | 20260505 21:45:38.268 | INFO | } | 20260505 21:45:38.268 | INFO | Perf report for the last 5.1422 seconds: | 20260505 21:45:38.268 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:45:38.268 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:38.268 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:38.268 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:38.268 | INFO | ANALYSIS : 61.6%, 3.0, 96.2ms, 0 | 20260505 21:45:38.269 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:38.269 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:38.269 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 21:45:38.269 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:38.269 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:38.269 | INFO | Uploaded 36GiB to GPU in 2.07457s (17.353GiB/s) | 20260505 21:45:38.269 | INFO | Downloaded 34GiB to GPU in 2.03251s (16.7281GiB/s) | 20260505 21:45:43.399 | INFO | Analyzing chunk frames = [6540288,6540800) | 20260505 21:45:43.399 | INFO | MemInfo: { | 20260505 21:45:43.399 | INFO | "gpu" : " 17 GB", | 20260505 21:45:43.399 | INFO | "huge" : "", | 20260505 21:45:43.399 | INFO | "huge_shared" : " 27 GB", | 20260505 21:45:43.399 | INFO | "malloc" : " 14 GB", | 20260505 21:45:43.400 | INFO | "pinned" : " 29 GB" | 20260505 21:45:43.400 | INFO | } | 20260505 21:45:43.400 | INFO | Perf report for the last 5.13132 seconds: | 20260505 21:45:43.400 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:45:43.400 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:43.400 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:43.400 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:43.400 | INFO | ANALYSIS : 62.9%, 3.0, 98.0ms, 0 | 20260505 21:45:43.400 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:43.400 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:43.400 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 21:45:43.400 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:43.400 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:43.400 | INFO | Uploaded 36GiB to GPU in 2.14035s (16.8197GiB/s) | 20260505 21:45:43.400 | INFO | Downloaded 34GiB to GPU in 2.10814s (16.128GiB/s) | 20260505 21:45:48.538 | INFO | Analyzing chunk frames = [6540800,6541312) | 20260505 21:45:48.538 | INFO | MemInfo: { | 20260505 21:45:48.538 | INFO | "gpu" : " 17 GB", | 20260505 21:45:48.538 | INFO | "huge" : "", | 20260505 21:45:48.538 | INFO | "huge_shared" : " 27 GB", | 20260505 21:45:48.538 | INFO | "malloc" : " 14 GB", | 20260505 21:45:48.539 | INFO | "pinned" : " 29 GB" | 20260505 21:45:48.539 | INFO | } | 20260505 21:45:48.539 | INFO | Perf report for the last 5.13898 seconds: | 20260505 21:45:48.539 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:45:48.539 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:48.539 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:48.539 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:48.539 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260505 21:45:48.539 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:48.539 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:48.539 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:45:48.539 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:48.539 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:48.539 | INFO | Uploaded 36GiB to GPU in 2.13592s (16.8545GiB/s) | 20260505 21:45:48.539 | INFO | Downloaded 34GiB to GPU in 2.00886s (16.925GiB/s) | 20260505 21:45:48.668 | INFO | Starting a round of Baz IO | 20260505 21:45:51.837 | INFO | Populating RTMetrics report | 20260505 21:45:51.837 | INFO | RTMetrics report populated | 20260505 21:45:51.837 | INFO | Opening tmp RTMetrics json file | 20260505 21:45:51.840 | INFO | Serializing RTMetrics to json | 20260505 21:45:51.842 | INFO | Serialization done, swapping tmp file now... | 20260505 21:45:51.843 | INFO | RTMetrics done | 20260505 21:45:51.924 | INFO | Finished a round of Baz IO: Wrote 1.957 GiB in 3.256 seconds | 20260505 21:45:51.924 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.55% / 0.00% / 78.45% | 20260505 21:45:51.924 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.138859% / 0% / 9.53674e-05% | 20260505 21:45:51.924 | INFO | Baz IO overall: | 20260505 21:45:51.924 | INFO | BytesWritten : 3781978812658 | 20260505 21:45:51.924 | INFO | NumEvents : 1785235433533 | 20260505 21:45:53.686 | INFO | Analyzing chunk frames = [6541312,6541824) | 20260505 21:45:53.686 | INFO | MemInfo: { | 20260505 21:45:53.686 | INFO | "gpu" : " 17 GB", | 20260505 21:45:53.686 | INFO | "huge" : "", | 20260505 21:45:53.686 | INFO | "huge_shared" : " 27 GB", | 20260505 21:45:53.686 | INFO | "malloc" : " 14 GB", | 20260505 21:45:53.686 | INFO | "pinned" : " 29 GB" | 20260505 21:45:53.686 | INFO | } | 20260505 21:45:53.686 | INFO | Perf report for the last 5.14747 seconds: | 20260505 21:45:53.686 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:45:53.686 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:53.686 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:53.686 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:53.686 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 21:45:53.686 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:53.686 | INFO | RT_METRICS : 43.9%, 1.0, 23.1ms, 0 | 20260505 21:45:53.686 | INFO | PRE_HQ : 59.8%, 1.8, 54.6ms, 0 | 20260505 21:45:53.686 | INFO | BAZWRITER : 62.7%, 3.6, 116.8ms, 0 | 20260505 21:45:53.686 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 21:45:53.686 | INFO | Uploaded 36GiB to GPU in 2.15693s (16.6904GiB/s) | 20260505 21:45:53.686 | INFO | Downloaded 38GiB to GPU in 2.2389s (16.9726GiB/s) | 20260505 21:45:58.828 | INFO | Analyzing chunk frames = [6541824,6542336) | 20260505 21:45:58.828 | INFO | MemInfo: { | 20260505 21:45:58.828 | INFO | "gpu" : " 17 GB", | 20260505 21:45:58.828 | INFO | "huge" : "", | 20260505 21:45:58.828 | INFO | "huge_shared" : " 27 GB", | 20260505 21:45:58.828 | INFO | "malloc" : " 14 GB", | 20260505 21:45:58.828 | INFO | "pinned" : " 29 GB" | 20260505 21:45:58.828 | INFO | } | 20260505 21:45:58.828 | INFO | Perf report for the last 5.14155 seconds: | 20260505 21:45:58.828 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:45:58.828 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:58.828 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:58.828 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:58.828 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260505 21:45:58.828 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:58.828 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:58.828 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:45:58.828 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:58.828 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:45:58.828 | INFO | Uploaded 36GiB to GPU in 2.08673s (17.2519GiB/s) | 20260505 21:45:58.828 | INFO | Downloaded 34GiB to GPU in 2.02598s (16.782GiB/s) | 20260505 21:46:03.963 | INFO | Analyzing chunk frames = [6542336,6542848) | 20260505 21:46:03.963 | INFO | MemInfo: { | 20260505 21:46:03.963 | INFO | "gpu" : " 17 GB", | 20260505 21:46:03.963 | INFO | "huge" : "", | 20260505 21:46:03.963 | INFO | "huge_shared" : " 27 GB", | 20260505 21:46:03.963 | INFO | "malloc" : " 14 GB", | 20260505 21:46:03.963 | INFO | "pinned" : " 29 GB" | 20260505 21:46:03.963 | INFO | } | 20260505 21:46:03.963 | INFO | Perf report for the last 5.13531 seconds: | 20260505 21:46:03.963 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:46:03.963 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:03.963 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:03.963 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:03.963 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260505 21:46:03.963 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:03.963 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:03.963 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 21:46:03.963 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:03.963 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:03.964 | INFO | Uploaded 36GiB to GPU in 2.15004s (16.7438GiB/s) | 20260505 21:46:03.964 | INFO | Downloaded 34GiB to GPU in 2.02152s (16.819GiB/s) | 20260505 21:46:09.095 | INFO | Analyzing chunk frames = [6542848,6543360) | 20260505 21:46:09.095 | INFO | MemInfo: { | 20260505 21:46:09.095 | INFO | "gpu" : " 17 GB", | 20260505 21:46:09.095 | INFO | "huge" : "", | 20260505 21:46:09.095 | INFO | "huge_shared" : " 27 GB", | 20260505 21:46:09.095 | INFO | "malloc" : " 14 GB", | 20260505 21:46:09.095 | INFO | "pinned" : " 29 GB" | 20260505 21:46:09.096 | INFO | } | 20260505 21:46:09.096 | INFO | Perf report for the last 5.13257 seconds: | 20260505 21:46:09.096 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:46:09.096 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:09.096 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:09.096 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:09.096 | INFO | ANALYSIS : 62.4%, 3.0, 96.8ms, 0 | 20260505 21:46:09.096 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:09.096 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:09.096 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 21:46:09.096 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:09.096 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:09.096 | INFO | Uploaded 36GiB to GPU in 2.20893s (16.2975GiB/s) | 20260505 21:46:09.096 | INFO | Downloaded 34GiB to GPU in 2.0697s (16.4275GiB/s) | 20260505 21:46:14.238 | INFO | Analyzing chunk frames = [6543360,6543872) | 20260505 21:46:14.238 | INFO | MemInfo: { | 20260505 21:46:14.238 | INFO | "gpu" : " 17 GB", | 20260505 21:46:14.238 | INFO | "huge" : "", | 20260505 21:46:14.238 | INFO | "huge_shared" : " 27 GB", | 20260505 21:46:14.238 | INFO | "malloc" : " 14 GB", | 20260505 21:46:14.238 | INFO | "pinned" : " 29 GB" | 20260505 21:46:14.238 | INFO | } | 20260505 21:46:14.238 | INFO | Perf report for the last 5.14236 seconds: | 20260505 21:46:14.238 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:46:14.238 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:14.238 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:14.238 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:14.238 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260505 21:46:14.238 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:14.238 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:14.238 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:46:14.238 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:14.238 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:14.238 | INFO | Uploaded 36GiB to GPU in 2.13063s (16.8964GiB/s) | 20260505 21:46:14.238 | INFO | Downloaded 34GiB to GPU in 2.06118s (16.4954GiB/s) | 20260505 21:46:19.382 | INFO | Analyzing chunk frames = [6543872,6544384) | 20260505 21:46:19.382 | INFO | MemInfo: { | 20260505 21:46:19.382 | INFO | "gpu" : " 17 GB", | 20260505 21:46:19.382 | INFO | "huge" : "", | 20260505 21:46:19.382 | INFO | "huge_shared" : " 27 GB", | 20260505 21:46:19.382 | INFO | "malloc" : " 14 GB", | 20260505 21:46:19.383 | INFO | "pinned" : " 29 GB" | 20260505 21:46:19.383 | INFO | } | 20260505 21:46:19.383 | INFO | Perf report for the last 5.14464 seconds: | 20260505 21:46:19.383 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:46:19.383 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:19.383 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:19.383 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:19.383 | INFO | ANALYSIS : 61.6%, 3.0, 96.3ms, 0 | 20260505 21:46:19.383 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:19.383 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:19.383 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 21:46:19.383 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:19.383 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:19.383 | INFO | Uploaded 36GiB to GPU in 2.09056s (17.2203GiB/s) | 20260505 21:46:19.383 | INFO | Downloaded 34GiB to GPU in 2.04228s (16.6481GiB/s) | 20260505 21:46:24.520 | INFO | Analyzing chunk frames = [6544384,6544896) | 20260505 21:46:24.520 | INFO | MemInfo: { | 20260505 21:46:24.520 | INFO | "gpu" : " 17 GB", | 20260505 21:46:24.520 | INFO | "huge" : "", | 20260505 21:46:24.520 | INFO | "huge_shared" : " 27 GB", | 20260505 21:46:24.520 | INFO | "malloc" : " 14 GB", | 20260505 21:46:24.520 | INFO | "pinned" : " 29 GB" | 20260505 21:46:24.520 | INFO | } | 20260505 21:46:24.520 | INFO | Perf report for the last 5.13716 seconds: | 20260505 21:46:24.520 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:46:24.520 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:24.520 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:24.520 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:24.520 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260505 21:46:24.520 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:24.520 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:24.520 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 21:46:24.520 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:24.520 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:24.520 | INFO | Uploaded 36GiB to GPU in 2.02862s (17.746GiB/s) | 20260505 21:46:24.520 | INFO | Downloaded 34GiB to GPU in 2.03849s (16.679GiB/s) | 20260505 21:46:29.656 | INFO | Analyzing chunk frames = [6544896,6545408) | 20260505 21:46:29.656 | INFO | MemInfo: { | 20260505 21:46:29.656 | INFO | "gpu" : " 17 GB", | 20260505 21:46:29.656 | INFO | "huge" : "", | 20260505 21:46:29.656 | INFO | "huge_shared" : " 27 GB", | 20260505 21:46:29.656 | INFO | "malloc" : " 14 GB", | 20260505 21:46:29.656 | INFO | "pinned" : " 29 GB" | 20260505 21:46:29.656 | INFO | } | 20260505 21:46:29.656 | INFO | Perf report for the last 5.13618 seconds: | 20260505 21:46:29.656 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:46:29.656 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:29.656 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:29.656 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:29.656 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 21:46:29.656 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:29.656 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:29.656 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:46:29.656 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:29.656 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:29.656 | INFO | Uploaded 36GiB to GPU in 2.11248s (17.0415GiB/s) | 20260505 21:46:29.656 | INFO | Downloaded 34GiB to GPU in 1.99927s (17.0062GiB/s) | 20260505 21:46:29.657 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 21:46:29.657 | INFO | Total mem usage: 15.3 GiB | 20260505 21:46:29.657 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 21:46:29.657 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.81%) | 20260505 21:46:29.657 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 21:46:29.657 | INFO | High Water Breakdown: | 20260505 21:46:29.657 | INFO | 61% : 8.59 GiB ( 90.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 21:46:29.657 | INFO | 38.9% : 5.48 GiB ( 82.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 21:46:29.657 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 21:46:29.657 | INFO | ------------------------------------------ | 20260505 21:46:29.657 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:46:29.657 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 21:46:29.657 | INFO | Total mem usage: 26.5 GiB | 20260505 21:46:29.657 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 21:46:29.657 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 21:46:29.657 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 21:46:29.657 | INFO | High Water Breakdown: | 20260505 21:46:29.657 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:46:29.657 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:46:29.657 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:46:29.657 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:46:29.657 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:46:29.657 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:46:29.657 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 21:46:29.657 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 21:46:29.657 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:46:29.657 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 21:46:29.657 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 21:46:29.657 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 21:46:29.657 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 21:46:29.657 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 21:46:29.657 | INFO | ------------------------------------------ | 20260505 21:46:29.657 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:46:29.657 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 21:46:29.657 | INFO | Total mem usage: 25.6 GiB | 20260505 21:46:29.657 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 21:46:29.657 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 21:46:29.657 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 21:46:29.657 | INFO | High Water Breakdown: | 20260505 21:46:29.657 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 21:46:29.657 | INFO | ------------------------------------------ | 20260505 21:46:29.657 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:46:29.657 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 21:46:29.657 | INFO | Total mem usage: 15.7 GiB | 20260505 21:46:29.657 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 21:46:29.657 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 21:46:29.657 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 21:46:29.657 | INFO | High Water Breakdown: | 20260505 21:46:29.657 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 21:46:29.657 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 21:46:29.657 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 21:46:29.657 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 21:46:29.657 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 21:46:29.657 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 21:46:29.657 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 21:46:29.657 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 21:46:29.657 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 21:46:29.657 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:46:29.657 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 21:46:29.657 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:46:29.657 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 21:46:29.657 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 21:46:29.657 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 21:46:29.657 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 21:46:29.657 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:46:29.657 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 21:46:29.657 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:46:29.657 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:46:29.657 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:46:29.657 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 21:46:29.657 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 21:46:29.657 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:46:29.657 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 21:46:29.657 | INFO | ------------------------------------------ | 20260505 21:46:29.657 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:46:29.657 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 21:46:29.657 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:46:29.657 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:46:29.658 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.42e+01, 1.34e+01 / 8.74e-01 | 20260505 21:46:29.658 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.04e-01, 1.06e-02 / 6.60e-03 | 20260505 21:46:29.658 | INFO | PerfReport: 11.79%, Baseline -- 3.70e+00 / 4.51e+00, 5.60e-01 / 1.71e-01 | 20260505 21:46:29.658 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.53e+00, 1.88e-01 / 9.53e-02 | 20260505 21:46:29.658 | INFO | PerfReport: 15.97%, DME -- 4.01e+01 / 4.42e+01, 3.91e+00 / 9.98e-01 | 20260505 21:46:29.658 | INFO | PerfReport: 57.17%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.50e+00 / 1.13e+00 | 20260505 21:46:29.658 | INFO | PerfReport: 7.20%, PulseAccumulating -- 2.26e+00 / 2.82e+00, 4.29e-01 / 2.27e-01 | 20260505 21:46:29.658 | INFO | PerfReport: 3.19%, Metrics -- 1.00e+00 / 1.30e+00, 2.70e-01 / 1.24e-01 | 20260505 21:46:29.658 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.48e-01 / 4.08e-01, 5.95e-02 / 3.90e-02 | 20260505 21:46:29.658 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 21:46:29.658 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:46:29.658 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:46:29.658 | INFO | PerfReport: Full Execution -- 9.89e+01 / -nan, 2.48e+01 / 0.00e+00 | 20260505 21:46:29.658 | INFO | PerfReport: 22.37%, Upload -- 2.21e+01 / 0.00e+00, 7.55e+00 / 0.00e+00 | 20260505 21:46:29.658 | INFO | PerfReport: 21.74%, Download -- 2.15e+01 / 0.00e+00, 7.54e+00 / 0.00e+00 | 20260505 21:46:29.658 | INFO | PerfReport: 31.75%, Compute -- 3.14e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 21:46:29.658 | INFO | PerfReport: 2.22%, LockUpload -- 2.20e+00 / 0.00e+00, 7.90e+00 / 0.00e+00 | 20260505 21:46:29.658 | INFO | PerfReport: 1.31%, LockDownload -- 1.29e+00 / 0.00e+00, 2.84e+00 / 0.00e+00 | 20260505 21:46:29.658 | INFO | PerfReport: 20.60%, LockCompute -- 2.04e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 21:46:29.658 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.50e-02 / 0.00e+00 | 20260505 21:46:29.788 | INFO | Starting a round of Baz IO | 20260505 21:46:32.954 | INFO | Populating RTMetrics report | 20260505 21:46:32.954 | INFO | RTMetrics report populated | 20260505 21:46:32.954 | INFO | Opening tmp RTMetrics json file | 20260505 21:46:32.955 | INFO | Serializing RTMetrics to json | 20260505 21:46:32.956 | INFO | Serialization done, swapping tmp file now... | 20260505 21:46:32.956 | INFO | RTMetrics done | 20260505 21:46:33.039 | INFO | Finished a round of Baz IO: Wrote 1.956 GiB in 3.251 seconds | 20260505 21:46:33.039 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.55% / 0.00% / 78.45% | 20260505 21:46:33.039 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.138358% / 3.97364e-06% / 8.34465e-05% | 20260505 21:46:33.039 | INFO | Baz IO overall: | 20260505 21:46:33.039 | INFO | BytesWritten : 3784079348135 | 20260505 21:46:33.039 | INFO | NumEvents : 1786204194477 | 20260505 21:46:34.800 | INFO | Analyzing chunk frames = [6545408,6545920) | 20260505 21:46:34.800 | INFO | MemInfo: { | 20260505 21:46:34.800 | INFO | "gpu" : " 17 GB", | 20260505 21:46:34.800 | INFO | "huge" : "", | 20260505 21:46:34.800 | INFO | "huge_shared" : " 27 GB", | 20260505 21:46:34.800 | INFO | "malloc" : " 14 GB", | 20260505 21:46:34.800 | INFO | "pinned" : " 29 GB" | 20260505 21:46:34.800 | INFO | } | 20260505 21:46:34.800 | INFO | Perf report for the last 5.1437 seconds: | 20260505 21:46:34.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:46:34.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:34.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:34.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:34.800 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 21:46:34.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:34.800 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260505 21:46:34.800 | INFO | PRE_HQ : 59.8%, 1.8, 54.7ms, 0 | 20260505 21:46:34.800 | INFO | BAZWRITER : 62.7%, 3.6, 116.8ms, 0 | 20260505 21:46:34.800 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 21:46:34.800 | INFO | Uploaded 36GiB to GPU in 2.1465s (16.7715GiB/s) | 20260505 21:46:34.800 | INFO | Downloaded 38GiB to GPU in 2.23573s (16.9967GiB/s) | 20260505 21:46:39.942 | INFO | Analyzing chunk frames = [6545920,6546432) | 20260505 21:46:39.942 | INFO | MemInfo: { | 20260505 21:46:39.942 | INFO | "gpu" : " 17 GB", | 20260505 21:46:39.942 | INFO | "huge" : "", | 20260505 21:46:39.942 | INFO | "huge_shared" : " 27 GB", | 20260505 21:46:39.942 | INFO | "malloc" : " 14 GB", | 20260505 21:46:39.942 | INFO | "pinned" : " 29 GB" | 20260505 21:46:39.942 | INFO | } | 20260505 21:46:39.942 | INFO | Perf report for the last 5.14246 seconds: | 20260505 21:46:39.942 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:46:39.942 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:39.942 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:39.942 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:39.942 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260505 21:46:39.942 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:39.942 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:39.942 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 21:46:39.942 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:39.942 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:39.943 | INFO | Uploaded 36GiB to GPU in 2.07139s (17.3796GiB/s) | 20260505 21:46:39.943 | INFO | Downloaded 34GiB to GPU in 2.02173s (16.8173GiB/s) | 20260505 21:46:45.080 | INFO | Analyzing chunk frames = [6546432,6546944) | 20260505 21:46:45.080 | INFO | MemInfo: { | 20260505 21:46:45.080 | INFO | "gpu" : " 17 GB", | 20260505 21:46:45.080 | INFO | "huge" : "", | 20260505 21:46:45.080 | INFO | "huge_shared" : " 27 GB", | 20260505 21:46:45.081 | INFO | "malloc" : " 14 GB", | 20260505 21:46:45.081 | INFO | "pinned" : " 29 GB" | 20260505 21:46:45.081 | INFO | } | 20260505 21:46:45.081 | INFO | Perf report for the last 5.13843 seconds: | 20260505 21:46:45.081 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:46:45.081 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:45.081 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:45.081 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:45.081 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260505 21:46:45.081 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:45.081 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:45.081 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 21:46:45.081 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:45.081 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:45.081 | INFO | Uploaded 36GiB to GPU in 2.15634s (16.695GiB/s) | 20260505 21:46:45.081 | INFO | Downloaded 34GiB to GPU in 2.05501s (16.545GiB/s) | 20260505 21:46:50.215 | INFO | Analyzing chunk frames = [6546944,6547456) | 20260505 21:46:50.215 | INFO | MemInfo: { | 20260505 21:46:50.215 | INFO | "gpu" : " 17 GB", | 20260505 21:46:50.215 | INFO | "huge" : "", | 20260505 21:46:50.215 | INFO | "huge_shared" : " 27 GB", | 20260505 21:46:50.215 | INFO | "malloc" : " 14 GB", | 20260505 21:46:50.215 | INFO | "pinned" : " 29 GB" | 20260505 21:46:50.215 | INFO | } | 20260505 21:46:50.215 | INFO | Perf report for the last 5.13464 seconds: | 20260505 21:46:50.215 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:46:50.215 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:50.215 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:50.215 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:50.215 | INFO | ANALYSIS : 61.8%, 3.0, 95.9ms, 0 | 20260505 21:46:50.215 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:50.215 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:50.215 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:46:50.215 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:50.216 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:50.216 | INFO | Uploaded 36GiB to GPU in 2.12892s (16.91GiB/s) | 20260505 21:46:50.216 | INFO | Downloaded 34GiB to GPU in 2.01202s (16.8985GiB/s) | 20260505 21:46:55.350 | INFO | Analyzing chunk frames = [6547456,6547968) | 20260505 21:46:55.350 | INFO | MemInfo: { | 20260505 21:46:55.350 | INFO | "gpu" : " 17 GB", | 20260505 21:46:55.350 | INFO | "huge" : "", | 20260505 21:46:55.350 | INFO | "huge_shared" : " 27 GB", | 20260505 21:46:55.350 | INFO | "malloc" : " 14 GB", | 20260505 21:46:55.350 | INFO | "pinned" : " 29 GB" | 20260505 21:46:55.350 | INFO | } | 20260505 21:46:55.350 | INFO | Perf report for the last 5.13503 seconds: | 20260505 21:46:55.350 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:46:55.350 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:55.350 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:55.350 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:55.350 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260505 21:46:55.350 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:55.350 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:55.351 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 21:46:55.351 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:55.351 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:46:55.351 | INFO | Uploaded 36GiB to GPU in 2.13584s (16.8552GiB/s) | 20260505 21:46:55.351 | INFO | Downloaded 34GiB to GPU in 2.06612s (16.456GiB/s) | 20260505 21:47:00.493 | INFO | Analyzing chunk frames = [6547968,6548480) | 20260505 21:47:00.493 | INFO | MemInfo: { | 20260505 21:47:00.493 | INFO | "gpu" : " 17 GB", | 20260505 21:47:00.493 | INFO | "huge" : "", | 20260505 21:47:00.493 | INFO | "huge_shared" : " 27 GB", | 20260505 21:47:00.493 | INFO | "malloc" : " 14 GB", | 20260505 21:47:00.493 | INFO | "pinned" : " 29 GB" | 20260505 21:47:00.493 | INFO | } | 20260505 21:47:00.493 | INFO | Perf report for the last 5.14267 seconds: | 20260505 21:47:00.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:47:00.493 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:00.493 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:00.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:00.493 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 21:47:00.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:00.493 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:00.493 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 21:47:00.493 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:00.493 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:00.494 | INFO | Uploaded 36GiB to GPU in 2.09639s (17.1724GiB/s) | 20260505 21:47:00.494 | INFO | Downloaded 34GiB to GPU in 2.03269s (16.7266GiB/s) | 20260505 21:47:05.638 | INFO | Analyzing chunk frames = [6548480,6548992) | 20260505 21:47:05.638 | INFO | MemInfo: { | 20260505 21:47:05.638 | INFO | "gpu" : " 17 GB", | 20260505 21:47:05.638 | INFO | "huge" : "", | 20260505 21:47:05.638 | INFO | "huge_shared" : " 27 GB", | 20260505 21:47:05.638 | INFO | "malloc" : " 14 GB", | 20260505 21:47:05.638 | INFO | "pinned" : " 29 GB" | 20260505 21:47:05.638 | INFO | } | 20260505 21:47:05.638 | INFO | Perf report for the last 5.1448 seconds: | 20260505 21:47:05.638 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:47:05.638 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:05.638 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:05.638 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:05.638 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 21:47:05.638 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:05.638 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:05.638 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 21:47:05.638 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:05.638 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:05.638 | INFO | Uploaded 36GiB to GPU in 2.13887s (16.8313GiB/s) | 20260505 21:47:05.638 | INFO | Downloaded 34GiB to GPU in 2.10012s (16.1895GiB/s) | 20260505 21:47:10.781 | INFO | Analyzing chunk frames = [6548992,6549504) | 20260505 21:47:10.781 | INFO | MemInfo: { | 20260505 21:47:10.781 | INFO | "gpu" : " 17 GB", | 20260505 21:47:10.781 | INFO | "huge" : "", | 20260505 21:47:10.781 | INFO | "huge_shared" : " 27 GB", | 20260505 21:47:10.781 | INFO | "malloc" : " 14 GB", | 20260505 21:47:10.781 | INFO | "pinned" : " 29 GB" | 20260505 21:47:10.781 | INFO | } | 20260505 21:47:10.781 | INFO | Perf report for the last 5.14302 seconds: | 20260505 21:47:10.781 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:47:10.781 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:10.781 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:10.781 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:10.781 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 21:47:10.781 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:10.781 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:10.781 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 21:47:10.781 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:10.781 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:10.781 | INFO | Uploaded 36GiB to GPU in 2.11154s (17.0492GiB/s) | 20260505 21:47:10.781 | INFO | Downloaded 34GiB to GPU in 2.00482s (16.9591GiB/s) | 20260505 21:47:10.903 | INFO | Starting a round of Baz IO | 20260505 21:47:14.058 | INFO | Populating RTMetrics report | 20260505 21:47:14.058 | INFO | RTMetrics report populated | 20260505 21:47:14.058 | INFO | Opening tmp RTMetrics json file | 20260505 21:47:14.059 | INFO | Serializing RTMetrics to json | 20260505 21:47:14.060 | INFO | Serialization done, swapping tmp file now... | 20260505 21:47:14.061 | INFO | RTMetrics done | 20260505 21:47:14.143 | INFO | Finished a round of Baz IO: Wrote 1.956 GiB in 3.239 seconds | 20260505 21:47:14.143 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.55% / 0.00% / 78.45% | 20260505 21:47:14.143 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.138712% / 0% / 9.13938e-05% | 20260505 21:47:14.143 | INFO | Baz IO overall: | 20260505 21:47:14.143 | INFO | BytesWritten : 3786179130465 | 20260505 21:47:14.143 | INFO | NumEvents : 1787172238359 | 20260505 21:47:15.914 | INFO | Analyzing chunk frames = [6549504,6550016) | 20260505 21:47:15.914 | INFO | MemInfo: { | 20260505 21:47:15.914 | INFO | "gpu" : " 17 GB", | 20260505 21:47:15.914 | INFO | "huge" : "", | 20260505 21:47:15.914 | INFO | "huge_shared" : " 27 GB", | 20260505 21:47:15.914 | INFO | "malloc" : " 14 GB", | 20260505 21:47:15.914 | INFO | "pinned" : " 29 GB" | 20260505 21:47:15.914 | INFO | } | 20260505 21:47:15.914 | INFO | Perf report for the last 5.13291 seconds: | 20260505 21:47:15.914 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:47:15.914 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:15.914 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:15.914 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:15.914 | INFO | ANALYSIS : 62.8%, 3.0, 97.8ms, 0 | 20260505 21:47:15.914 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:15.914 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260505 21:47:15.914 | INFO | PRE_HQ : 60.1%, 1.8, 54.8ms, 0 | 20260505 21:47:15.914 | INFO | BAZWRITER : 62.3%, 3.6, 116.5ms, 0 | 20260505 21:47:15.914 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 21:47:15.914 | INFO | Uploaded 36GiB to GPU in 2.16314s (16.6425GiB/s) | 20260505 21:47:15.914 | INFO | Downloaded 38GiB to GPU in 2.22791s (17.0563GiB/s) | 20260505 21:47:21.050 | INFO | Analyzing chunk frames = [6550016,6550528) | 20260505 21:47:21.050 | INFO | MemInfo: { | 20260505 21:47:21.050 | INFO | "gpu" : " 17 GB", | 20260505 21:47:21.050 | INFO | "huge" : "", | 20260505 21:47:21.050 | INFO | "huge_shared" : " 27 GB", | 20260505 21:47:21.050 | INFO | "malloc" : " 14 GB", | 20260505 21:47:21.050 | INFO | "pinned" : " 29 GB" | 20260505 21:47:21.050 | INFO | } | 20260505 21:47:21.050 | INFO | Perf report for the last 5.13628 seconds: | 20260505 21:47:21.050 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:47:21.050 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:21.050 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:21.050 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:21.050 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 21:47:21.050 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:21.050 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:21.050 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260505 21:47:21.050 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:21.050 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:21.050 | INFO | Uploaded 36GiB to GPU in 2.08811s (17.2404GiB/s) | 20260505 21:47:21.051 | INFO | Downloaded 34GiB to GPU in 2.01933s (16.8373GiB/s) | 20260505 21:47:26.195 | INFO | Analyzing chunk frames = [6550528,6551040) | 20260505 21:47:26.195 | INFO | MemInfo: { | 20260505 21:47:26.195 | INFO | "gpu" : " 17 GB", | 20260505 21:47:26.195 | INFO | "huge" : "", | 20260505 21:47:26.195 | INFO | "huge_shared" : " 27 GB", | 20260505 21:47:26.195 | INFO | "malloc" : " 14 GB", | 20260505 21:47:26.195 | INFO | "pinned" : " 29 GB" | 20260505 21:47:26.195 | INFO | } | 20260505 21:47:26.195 | INFO | Perf report for the last 5.14467 seconds: | 20260505 21:47:26.195 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:47:26.195 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:26.195 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:26.195 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:26.195 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260505 21:47:26.195 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:26.195 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:26.195 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 21:47:26.195 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:26.195 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:26.195 | INFO | Uploaded 36GiB to GPU in 2.13396s (16.87GiB/s) | 20260505 21:47:26.195 | INFO | Downloaded 34GiB to GPU in 2.0249s (16.7909GiB/s) | 20260505 21:47:31.332 | INFO | Analyzing chunk frames = [6551040,6551552) | 20260505 21:47:31.332 | INFO | MemInfo: { | 20260505 21:47:31.332 | INFO | "gpu" : " 17 GB", | 20260505 21:47:31.332 | INFO | "huge" : "", | 20260505 21:47:31.332 | INFO | "huge_shared" : " 27 GB", | 20260505 21:47:31.332 | INFO | "malloc" : " 14 GB", | 20260505 21:47:31.332 | INFO | "pinned" : " 29 GB" | 20260505 21:47:31.332 | INFO | } | 20260505 21:47:31.332 | INFO | Perf report for the last 5.13722 seconds: | 20260505 21:47:31.332 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:47:31.332 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:31.332 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:31.332 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:31.332 | INFO | ANALYSIS : 61.8%, 3.0, 95.9ms, 0 | 20260505 21:47:31.332 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:31.332 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:31.332 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:47:31.332 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:31.332 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:31.332 | INFO | Uploaded 36GiB to GPU in 2.11658s (17.0086GiB/s) | 20260505 21:47:31.332 | INFO | Downloaded 34GiB to GPU in 2.01884s (16.8413GiB/s) | 20260505 21:47:36.471 | INFO | Analyzing chunk frames = [6551552,6552064) | 20260505 21:47:36.471 | INFO | MemInfo: { | 20260505 21:47:36.471 | INFO | "gpu" : " 17 GB", | 20260505 21:47:36.471 | INFO | "huge" : "", | 20260505 21:47:36.471 | INFO | "huge_shared" : " 27 GB", | 20260505 21:47:36.471 | INFO | "malloc" : " 14 GB", | 20260505 21:47:36.471 | INFO | "pinned" : " 29 GB" | 20260505 21:47:36.471 | INFO | } | 20260505 21:47:36.471 | INFO | Perf report for the last 5.13921 seconds: | 20260505 21:47:36.471 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:47:36.471 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:36.471 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:36.471 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:36.471 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260505 21:47:36.471 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:36.471 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:36.471 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 21:47:36.471 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:36.471 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:36.472 | INFO | Uploaded 36GiB to GPU in 2.09171s (17.2108GiB/s) | 20260505 21:47:36.472 | INFO | Downloaded 34GiB to GPU in 2.05335s (16.5583GiB/s) | 20260505 21:47:41.611 | INFO | Analyzing chunk frames = [6552064,6552576) | 20260505 21:47:41.611 | INFO | MemInfo: { | 20260505 21:47:41.611 | INFO | "gpu" : " 17 GB", | 20260505 21:47:41.611 | INFO | "huge" : "", | 20260505 21:47:41.611 | INFO | "huge_shared" : " 27 GB", | 20260505 21:47:41.611 | INFO | "malloc" : " 14 GB", | 20260505 21:47:41.611 | INFO | "pinned" : " 29 GB" | 20260505 21:47:41.611 | INFO | } | 20260505 21:47:41.611 | INFO | Perf report for the last 5.13933 seconds: | 20260505 21:47:41.611 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:47:41.611 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:41.611 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:41.611 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:41.611 | INFO | ANALYSIS : 61.6%, 3.0, 96.2ms, 0 | 20260505 21:47:41.611 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:41.611 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:41.611 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 21:47:41.611 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:41.611 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:41.611 | INFO | Uploaded 36GiB to GPU in 2.08124s (17.2974GiB/s) | 20260505 21:47:41.611 | INFO | Downloaded 34GiB to GPU in 2.03957s (16.6702GiB/s) | 20260505 21:47:46.752 | INFO | Analyzing chunk frames = [6552576,6553088) | 20260505 21:47:46.752 | INFO | MemInfo: { | 20260505 21:47:46.752 | INFO | "gpu" : " 17 GB", | 20260505 21:47:46.752 | INFO | "huge" : "", | 20260505 21:47:46.752 | INFO | "huge_shared" : " 27 GB", | 20260505 21:47:46.752 | INFO | "malloc" : " 14 GB", | 20260505 21:47:46.752 | INFO | "pinned" : " 29 GB" | 20260505 21:47:46.752 | INFO | } | 20260505 21:47:46.752 | INFO | Perf report for the last 5.14142 seconds: | 20260505 21:47:46.752 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:47:46.752 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:46.752 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:46.752 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:46.752 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 21:47:46.752 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:46.752 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:46.752 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260505 21:47:46.752 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:46.752 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:46.752 | INFO | Uploaded 36GiB to GPU in 2.16688s (16.6138GiB/s) | 20260505 21:47:46.752 | INFO | Downloaded 34GiB to GPU in 2.11649s (16.0643GiB/s) | 20260505 21:47:51.893 | INFO | Analyzing chunk frames = [6553088,6553600) | 20260505 21:47:51.893 | INFO | MemInfo: { | 20260505 21:47:51.893 | INFO | "gpu" : " 17 GB", | 20260505 21:47:51.893 | INFO | "huge" : "", | 20260505 21:47:51.893 | INFO | "huge_shared" : " 27 GB", | 20260505 21:47:51.893 | INFO | "malloc" : " 14 GB", | 20260505 21:47:51.893 | INFO | "pinned" : " 29 GB" | 20260505 21:47:51.893 | INFO | } | 20260505 21:47:51.893 | INFO | Perf report for the last 5.14103 seconds: | 20260505 21:47:51.893 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:47:51.893 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:51.893 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:51.893 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:51.893 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 21:47:51.893 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:51.893 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:51.893 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 21:47:51.893 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:51.893 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:51.893 | INFO | Uploaded 36GiB to GPU in 2.10596s (17.0943GiB/s) | 20260505 21:47:51.893 | INFO | Downloaded 34GiB to GPU in 2.00433s (16.9633GiB/s) | 20260505 21:47:51.894 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 21:47:51.894 | INFO | Total mem usage: 15.3 GiB | 20260505 21:47:51.894 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 21:47:51.894 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.18%) | 20260505 21:47:51.894 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 21:47:51.894 | INFO | High Water Breakdown: | 20260505 21:47:51.894 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 21:47:51.894 | INFO | 38.8% : 5.5 GiB ( 82.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 21:47:51.894 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 21:47:51.894 | INFO | ------------------------------------------ | 20260505 21:47:51.894 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:47:51.894 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 21:47:51.894 | INFO | Total mem usage: 26.5 GiB | 20260505 21:47:51.894 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 21:47:51.894 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 21:47:51.894 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 21:47:51.894 | INFO | High Water Breakdown: | 20260505 21:47:51.894 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:47:51.894 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:47:51.894 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:47:51.894 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:47:51.894 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:47:51.894 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:47:51.894 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 21:47:51.894 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 21:47:51.894 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:47:51.894 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 21:47:51.894 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 21:47:51.894 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 21:47:51.894 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 21:47:51.894 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 21:47:51.894 | INFO | ------------------------------------------ | 20260505 21:47:51.894 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:47:51.894 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 21:47:51.894 | INFO | Total mem usage: 25.6 GiB | 20260505 21:47:51.894 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 21:47:51.894 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 21:47:51.894 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 21:47:51.894 | INFO | High Water Breakdown: | 20260505 21:47:51.894 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 21:47:51.894 | INFO | ------------------------------------------ | 20260505 21:47:51.894 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:47:51.894 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 21:47:51.894 | INFO | Total mem usage: 15.7 GiB | 20260505 21:47:51.894 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 21:47:51.894 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 21:47:51.894 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 21:47:51.894 | INFO | High Water Breakdown: | 20260505 21:47:51.894 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 21:47:51.894 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 21:47:51.894 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 21:47:51.894 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 21:47:51.894 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 21:47:51.894 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 21:47:51.894 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 21:47:51.894 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 21:47:51.894 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 21:47:51.895 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:47:51.895 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 21:47:51.895 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:47:51.895 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 21:47:51.895 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 21:47:51.895 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 21:47:51.895 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 21:47:51.895 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:47:51.895 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 21:47:51.895 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:47:51.895 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:47:51.895 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:47:51.895 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 21:47:51.895 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 21:47:51.895 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:47:51.895 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 21:47:51.895 | INFO | ------------------------------------------ | 20260505 21:47:51.895 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:47:51.895 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 21:47:51.895 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:47:51.895 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:47:51.895 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.41e+01, 1.36e+01 / 7.90e-01 | 20260505 21:47:51.895 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.03e-01, 1.02e-02 / 3.70e-03 | 20260505 21:47:51.895 | INFO | PerfReport: 11.78%, Baseline -- 3.71e+00 / 4.55e+00, 5.71e-01 / 1.49e-01 | 20260505 21:47:51.895 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.52e+00, 1.92e-01 / 8.32e-02 | 20260505 21:47:51.895 | INFO | PerfReport: 15.94%, DME -- 4.01e+01 / 4.39e+01, 3.80e+00 / 9.31e-01 | 20260505 21:47:51.895 | INFO | PerfReport: 57.30%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.58e+00 / 9.87e-01 | 20260505 21:47:51.895 | INFO | PerfReport: 7.14%, PulseAccumulating -- 2.25e+00 / 2.78e+00, 4.30e-01 / 1.80e-01 | 20260505 21:47:51.895 | INFO | PerfReport: 3.17%, Metrics -- 9.97e-01 / 1.30e+00, 2.73e-01 / 1.25e-01 | 20260505 21:47:51.895 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.47e-01 / 4.04e-01, 5.93e-02 / 3.72e-02 | 20260505 21:47:51.895 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 21:47:51.895 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:47:51.895 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:47:51.895 | INFO | PerfReport: Full Execution -- 9.89e+01 / -nan, 2.52e+01 / 0.00e+00 | 20260505 21:47:51.895 | INFO | PerfReport: 22.34%, Upload -- 2.21e+01 / 0.00e+00, 7.64e+00 / 0.00e+00 | 20260505 21:47:51.895 | INFO | PerfReport: 21.75%, Download -- 2.15e+01 / 0.00e+00, 7.58e+00 / 0.00e+00 | 20260505 21:47:51.895 | INFO | PerfReport: 31.82%, Compute -- 3.15e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 21:47:51.895 | INFO | PerfReport: 2.22%, LockUpload -- 2.19e+00 / 0.00e+00, 7.97e+00 / 0.00e+00 | 20260505 21:47:51.895 | INFO | PerfReport: 1.27%, LockDownload -- 1.26e+00 / 0.00e+00, 2.81e+00 / 0.00e+00 | 20260505 21:47:51.895 | INFO | PerfReport: 20.58%, LockCompute -- 2.03e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260505 21:47:51.895 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.93e-02 / 0.00e+00 | 20260505 21:47:52.016 | INFO | Starting a round of Baz IO | 20260505 21:47:55.168 | INFO | Populating RTMetrics report | 20260505 21:47:55.168 | INFO | RTMetrics report populated | 20260505 21:47:55.168 | INFO | Opening tmp RTMetrics json file | 20260505 21:47:55.170 | INFO | Serializing RTMetrics to json | 20260505 21:47:55.171 | INFO | Serialization done, swapping tmp file now... | 20260505 21:47:55.171 | INFO | RTMetrics done | 20260505 21:47:55.248 | INFO | Finished a round of Baz IO: Wrote 1.954 GiB in 3.231 seconds | 20260505 21:47:55.248 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.55% / 0.00% / 78.45% | 20260505 21:47:55.248 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.137754% / 3.97364e-06% / 9.53674e-05% | 20260505 21:47:55.248 | INFO | Baz IO overall: | 20260505 21:47:55.248 | INFO | BytesWritten : 3788277690426 | 20260505 21:47:55.248 | INFO | NumEvents : 1788139427206 | 20260505 21:47:57.029 | INFO | Analyzing chunk frames = [6553600,6554112) | 20260505 21:47:57.029 | INFO | MemInfo: { | 20260505 21:47:57.029 | INFO | "gpu" : " 17 GB", | 20260505 21:47:57.029 | INFO | "huge" : "", | 20260505 21:47:57.029 | INFO | "huge_shared" : " 27 GB", | 20260505 21:47:57.029 | INFO | "malloc" : " 14 GB", | 20260505 21:47:57.029 | INFO | "pinned" : " 29 GB" | 20260505 21:47:57.029 | INFO | } | 20260505 21:47:57.029 | INFO | Perf report for the last 5.13589 seconds: | 20260505 21:47:57.029 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:47:57.029 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:57.029 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:57.029 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:57.029 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 21:47:57.029 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:47:57.029 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260505 21:47:57.029 | INFO | PRE_HQ : 59.8%, 1.8, 54.5ms, 0 | 20260505 21:47:57.029 | INFO | BAZWRITER : 62.1%, 3.6, 115.5ms, 0 | 20260505 21:47:57.029 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 21:47:57.029 | INFO | Uploaded 36GiB to GPU in 2.12597s (16.9335GiB/s) | 20260505 21:47:57.029 | INFO | Downloaded 38GiB to GPU in 2.22831s (17.0533GiB/s) | 20260505 21:48:02.162 | INFO | Analyzing chunk frames = [6554112,6554624) | 20260505 21:48:02.162 | INFO | MemInfo: { | 20260505 21:48:02.162 | INFO | "gpu" : " 17 GB", | 20260505 21:48:02.162 | INFO | "huge" : "", | 20260505 21:48:02.162 | INFO | "huge_shared" : " 27 GB", | 20260505 21:48:02.162 | INFO | "malloc" : " 14 GB", | 20260505 21:48:02.162 | INFO | "pinned" : " 29 GB" | 20260505 21:48:02.162 | INFO | } | 20260505 21:48:02.162 | INFO | Perf report for the last 5.13331 seconds: | 20260505 21:48:02.162 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:48:02.162 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:02.162 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:02.162 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:02.162 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260505 21:48:02.162 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:02.162 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:02.162 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 21:48:02.162 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:02.162 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:02.163 | INFO | Uploaded 36GiB to GPU in 2.07725s (17.3306GiB/s) | 20260505 21:48:02.163 | INFO | Downloaded 34GiB to GPU in 2.02481s (16.7917GiB/s) | 20260505 21:48:07.301 | INFO | Analyzing chunk frames = [6554624,6555136) | 20260505 21:48:07.301 | INFO | MemInfo: { | 20260505 21:48:07.301 | INFO | "gpu" : " 17 GB", | 20260505 21:48:07.301 | INFO | "huge" : "", | 20260505 21:48:07.301 | INFO | "huge_shared" : " 27 GB", | 20260505 21:48:07.301 | INFO | "malloc" : " 14 GB", | 20260505 21:48:07.301 | INFO | "pinned" : " 29 GB" | 20260505 21:48:07.301 | INFO | } | 20260505 21:48:07.301 | INFO | Perf report for the last 5.1388 seconds: | 20260505 21:48:07.301 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:48:07.301 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:07.301 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:07.301 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:07.301 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260505 21:48:07.301 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:07.301 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:07.301 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 21:48:07.301 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:07.301 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:07.301 | INFO | Uploaded 36GiB to GPU in 2.12623s (16.9314GiB/s) | 20260505 21:48:07.302 | INFO | Downloaded 34GiB to GPU in 2.02675s (16.7756GiB/s) | 20260505 21:48:12.446 | INFO | Analyzing chunk frames = [6555136,6555648) | 20260505 21:48:12.446 | INFO | MemInfo: { | 20260505 21:48:12.446 | INFO | "gpu" : " 17 GB", | 20260505 21:48:12.446 | INFO | "huge" : "", | 20260505 21:48:12.446 | INFO | "huge_shared" : " 27 GB", | 20260505 21:48:12.446 | INFO | "malloc" : " 14 GB", | 20260505 21:48:12.446 | INFO | "pinned" : " 29 GB" | 20260505 21:48:12.446 | INFO | } | 20260505 21:48:12.446 | INFO | Perf report for the last 5.14522 seconds: | 20260505 21:48:12.446 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:48:12.446 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:12.446 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:12.446 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:12.446 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260505 21:48:12.446 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:12.446 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:12.447 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 21:48:12.447 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:12.447 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:12.447 | INFO | Uploaded 36GiB to GPU in 2.20522s (16.3249GiB/s) | 20260505 21:48:12.447 | INFO | Downloaded 34GiB to GPU in 2.06286s (16.482GiB/s) | 20260505 21:48:17.593 | INFO | Analyzing chunk frames = [6555648,6556160) | 20260505 21:48:17.593 | INFO | MemInfo: { | 20260505 21:48:17.593 | INFO | "gpu" : " 17 GB", | 20260505 21:48:17.593 | INFO | "huge" : "", | 20260505 21:48:17.593 | INFO | "huge_shared" : " 27 GB", | 20260505 21:48:17.593 | INFO | "malloc" : " 14 GB", | 20260505 21:48:17.593 | INFO | "pinned" : " 29 GB" | 20260505 21:48:17.593 | INFO | } | 20260505 21:48:17.593 | INFO | Perf report for the last 5.14681 seconds: | 20260505 21:48:17.593 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:48:17.593 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:17.593 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:17.593 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:17.593 | INFO | ANALYSIS : 61.3%, 3.0, 95.8ms, 0 | 20260505 21:48:17.593 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:17.593 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:17.593 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 21:48:17.593 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:17.593 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:17.593 | INFO | Uploaded 36GiB to GPU in 2.09415s (17.1907GiB/s) | 20260505 21:48:17.593 | INFO | Downloaded 34GiB to GPU in 2.04959s (16.5887GiB/s) | 20260505 21:48:22.724 | INFO | Analyzing chunk frames = [6556160,6556672) | 20260505 21:48:22.724 | INFO | MemInfo: { | 20260505 21:48:22.724 | INFO | "gpu" : " 17 GB", | 20260505 21:48:22.724 | INFO | "huge" : "", | 20260505 21:48:22.724 | INFO | "huge_shared" : " 27 GB", | 20260505 21:48:22.724 | INFO | "malloc" : " 14 GB", | 20260505 21:48:22.724 | INFO | "pinned" : " 29 GB" | 20260505 21:48:22.724 | INFO | } | 20260505 21:48:22.724 | INFO | Perf report for the last 5.13126 seconds: | 20260505 21:48:22.724 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:48:22.724 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:22.725 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:22.725 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:22.725 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260505 21:48:22.725 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:22.725 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:22.725 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:48:22.725 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:22.725 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:22.725 | INFO | Uploaded 36GiB to GPU in 2.07967s (17.3105GiB/s) | 20260505 21:48:22.725 | INFO | Downloaded 34GiB to GPU in 2.03989s (16.6676GiB/s) | 20260505 21:48:27.859 | INFO | Analyzing chunk frames = [6556672,6557184) | 20260505 21:48:27.859 | INFO | MemInfo: { | 20260505 21:48:27.859 | INFO | "gpu" : " 17 GB", | 20260505 21:48:27.859 | INFO | "huge" : "", | 20260505 21:48:27.859 | INFO | "huge_shared" : " 27 GB", | 20260505 21:48:27.859 | INFO | "malloc" : " 14 GB", | 20260505 21:48:27.859 | INFO | "pinned" : " 29 GB" | 20260505 21:48:27.859 | INFO | } | 20260505 21:48:27.860 | INFO | Perf report for the last 5.13518 seconds: | 20260505 21:48:27.860 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:48:27.860 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:27.860 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:27.860 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:27.860 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 21:48:27.860 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:27.860 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:27.860 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 21:48:27.860 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:27.860 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:27.860 | INFO | Uploaded 36GiB to GPU in 2.12599s (16.9333GiB/s) | 20260505 21:48:27.860 | INFO | Downloaded 34GiB to GPU in 2.11009s (16.113GiB/s) | 20260505 21:48:33.003 | INFO | Analyzing chunk frames = [6557184,6557696) | 20260505 21:48:33.003 | INFO | MemInfo: { | 20260505 21:48:33.003 | INFO | "gpu" : " 17 GB", | 20260505 21:48:33.003 | INFO | "huge" : "", | 20260505 21:48:33.003 | INFO | "huge_shared" : " 27 GB", | 20260505 21:48:33.003 | INFO | "malloc" : " 14 GB", | 20260505 21:48:33.004 | INFO | "pinned" : " 29 GB" | 20260505 21:48:33.004 | INFO | } | 20260505 21:48:33.004 | INFO | Perf report for the last 5.1439 seconds: | 20260505 21:48:33.004 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:48:33.004 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:33.004 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:33.004 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:33.004 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 21:48:33.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:33.004 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:33.004 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:48:33.004 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:33.004 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:33.004 | INFO | Uploaded 36GiB to GPU in 2.20219s (16.3473GiB/s) | 20260505 21:48:33.004 | INFO | Downloaded 34GiB to GPU in 2.04896s (16.5938GiB/s) | 20260505 21:48:33.137 | INFO | Starting a round of Baz IO | 20260505 21:48:36.290 | INFO | Populating RTMetrics report | 20260505 21:48:36.290 | INFO | RTMetrics report populated | 20260505 21:48:36.290 | INFO | Opening tmp RTMetrics json file | 20260505 21:48:36.291 | INFO | Serializing RTMetrics to json | 20260505 21:48:36.292 | INFO | Serialization done, swapping tmp file now... | 20260505 21:48:36.293 | INFO | RTMetrics done | 20260505 21:48:36.370 | INFO | Finished a round of Baz IO: Wrote 1.953 GiB in 3.233 seconds | 20260505 21:48:36.370 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.55% / 0.00% / 78.45% | 20260505 21:48:36.370 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.137051% / 3.97364e-06% / 0.000119209% | 20260505 21:48:36.370 | INFO | Baz IO overall: | 20260505 21:48:36.370 | INFO | BytesWritten : 3790374279553 | 20260505 21:48:36.370 | INFO | NumEvents : 1789105465609 | 20260505 21:48:38.142 | INFO | Analyzing chunk frames = [6557696,6558208) | 20260505 21:48:38.142 | INFO | MemInfo: { | 20260505 21:48:38.142 | INFO | "gpu" : " 17 GB", | 20260505 21:48:38.142 | INFO | "huge" : "", | 20260505 21:48:38.142 | INFO | "huge_shared" : " 27 GB", | 20260505 21:48:38.142 | INFO | "malloc" : " 14 GB", | 20260505 21:48:38.142 | INFO | "pinned" : " 29 GB" | 20260505 21:48:38.142 | INFO | } | 20260505 21:48:38.142 | INFO | Perf report for the last 5.13874 seconds: | 20260505 21:48:38.142 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:48:38.142 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:38.142 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:38.142 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:38.142 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260505 21:48:38.142 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:38.142 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260505 21:48:38.142 | INFO | PRE_HQ : 59.7%, 1.8, 54.6ms, 0 | 20260505 21:48:38.142 | INFO | BAZWRITER : 62.4%, 3.6, 115.7ms, 0 | 20260505 21:48:38.142 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 21:48:38.143 | INFO | Uploaded 36GiB to GPU in 2.15172s (16.7308GiB/s) | 20260505 21:48:38.143 | INFO | Downloaded 38GiB to GPU in 2.24412s (16.9332GiB/s) | 20260505 21:48:43.282 | INFO | Analyzing chunk frames = [6558208,6558720) | 20260505 21:48:43.282 | INFO | MemInfo: { | 20260505 21:48:43.282 | INFO | "gpu" : " 17 GB", | 20260505 21:48:43.282 | INFO | "huge" : "", | 20260505 21:48:43.282 | INFO | "huge_shared" : " 27 GB", | 20260505 21:48:43.282 | INFO | "malloc" : " 14 GB", | 20260505 21:48:43.282 | INFO | "pinned" : " 29 GB" | 20260505 21:48:43.282 | INFO | } | 20260505 21:48:43.282 | INFO | Perf report for the last 5.13971 seconds: | 20260505 21:48:43.282 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:48:43.282 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:43.282 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:43.282 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:43.282 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 21:48:43.282 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:43.282 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:43.282 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 21:48:43.282 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:43.282 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:43.282 | INFO | Uploaded 36GiB to GPU in 2.07965s (17.3106GiB/s) | 20260505 21:48:43.282 | INFO | Downloaded 34GiB to GPU in 2.02428s (16.7961GiB/s) | 20260505 21:48:48.419 | INFO | Analyzing chunk frames = [6558720,6559232) | 20260505 21:48:48.419 | INFO | MemInfo: { | 20260505 21:48:48.419 | INFO | "gpu" : " 17 GB", | 20260505 21:48:48.419 | INFO | "huge" : "", | 20260505 21:48:48.419 | INFO | "huge_shared" : " 27 GB", | 20260505 21:48:48.419 | INFO | "malloc" : " 14 GB", | 20260505 21:48:48.420 | INFO | "pinned" : " 29 GB" | 20260505 21:48:48.420 | INFO | } | 20260505 21:48:48.420 | INFO | Perf report for the last 5.13756 seconds: | 20260505 21:48:48.420 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:48:48.420 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:48.420 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:48.420 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:48.420 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260505 21:48:48.420 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:48.420 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:48.420 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 21:48:48.420 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:48.420 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:48.420 | INFO | Uploaded 36GiB to GPU in 2.13075s (16.8955GiB/s) | 20260505 21:48:48.420 | INFO | Downloaded 34GiB to GPU in 2.01724s (16.8547GiB/s) | 20260505 21:48:53.557 | INFO | Analyzing chunk frames = [6559232,6559744) | 20260505 21:48:53.557 | INFO | MemInfo: { | 20260505 21:48:53.557 | INFO | "gpu" : " 17 GB", | 20260505 21:48:53.557 | INFO | "huge" : "", | 20260505 21:48:53.557 | INFO | "huge_shared" : " 27 GB", | 20260505 21:48:53.557 | INFO | "malloc" : " 14 GB", | 20260505 21:48:53.557 | INFO | "pinned" : " 29 GB" | 20260505 21:48:53.557 | INFO | } | 20260505 21:48:53.558 | INFO | Perf report for the last 5.13787 seconds: | 20260505 21:48:53.558 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:48:53.558 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:53.558 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:53.558 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:53.558 | INFO | ANALYSIS : 61.9%, 3.0, 96.1ms, 0 | 20260505 21:48:53.558 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:53.558 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:53.558 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:48:53.558 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:53.558 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:53.558 | INFO | Uploaded 36GiB to GPU in 2.13439s (16.8666GiB/s) | 20260505 21:48:53.558 | INFO | Downloaded 34GiB to GPU in 2.04311s (16.6413GiB/s) | 20260505 21:48:58.705 | INFO | Analyzing chunk frames = [6559744,6560256) | 20260505 21:48:58.705 | INFO | MemInfo: { | 20260505 21:48:58.705 | INFO | "gpu" : " 17 GB", | 20260505 21:48:58.705 | INFO | "huge" : "", | 20260505 21:48:58.705 | INFO | "huge_shared" : " 27 GB", | 20260505 21:48:58.705 | INFO | "malloc" : " 14 GB", | 20260505 21:48:58.705 | INFO | "pinned" : " 29 GB" | 20260505 21:48:58.705 | INFO | } | 20260505 21:48:58.705 | INFO | Perf report for the last 5.1473 seconds: | 20260505 21:48:58.705 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:48:58.705 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:58.705 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:58.705 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:58.705 | INFO | ANALYSIS : 61.5%, 3.0, 96.2ms, 0 | 20260505 21:48:58.705 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:58.705 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:58.705 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 21:48:58.705 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:58.705 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:48:58.705 | INFO | Uploaded 36GiB to GPU in 2.13899s (16.8303GiB/s) | 20260505 21:48:58.705 | INFO | Downloaded 34GiB to GPU in 2.06489s (16.4658GiB/s) | 20260505 21:49:03.845 | INFO | Analyzing chunk frames = [6560256,6560768) | 20260505 21:49:03.845 | INFO | MemInfo: { | 20260505 21:49:03.845 | INFO | "gpu" : " 17 GB", | 20260505 21:49:03.845 | INFO | "huge" : "", | 20260505 21:49:03.845 | INFO | "huge_shared" : " 27 GB", | 20260505 21:49:03.845 | INFO | "malloc" : " 14 GB", | 20260505 21:49:03.845 | INFO | "pinned" : " 29 GB" | 20260505 21:49:03.845 | INFO | } | 20260505 21:49:03.845 | INFO | Perf report for the last 5.14003 seconds: | 20260505 21:49:03.845 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:49:03.845 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:03.845 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:03.845 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:03.845 | INFO | ANALYSIS : 61.4%, 3.0, 95.9ms, 0 | 20260505 21:49:03.845 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:03.845 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:03.845 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 21:49:03.845 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:03.845 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:03.845 | INFO | Uploaded 36GiB to GPU in 2.07689s (17.3336GiB/s) | 20260505 21:49:03.845 | INFO | Downloaded 34GiB to GPU in 2.03735s (16.6883GiB/s) | 20260505 21:49:08.978 | INFO | Analyzing chunk frames = [6560768,6561280) | 20260505 21:49:08.978 | INFO | MemInfo: { | 20260505 21:49:08.978 | INFO | "gpu" : " 17 GB", | 20260505 21:49:08.978 | INFO | "huge" : "", | 20260505 21:49:08.978 | INFO | "huge_shared" : " 27 GB", | 20260505 21:49:08.978 | INFO | "malloc" : " 14 GB", | 20260505 21:49:08.978 | INFO | "pinned" : " 29 GB" | 20260505 21:49:08.978 | INFO | } | 20260505 21:49:08.978 | INFO | Perf report for the last 5.13299 seconds: | 20260505 21:49:08.978 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:49:08.978 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:08.978 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:08.978 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:08.978 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260505 21:49:08.978 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:08.978 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:08.978 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 21:49:08.978 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:08.978 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:08.978 | INFO | Uploaded 36GiB to GPU in 2.10997s (17.0619GiB/s) | 20260505 21:49:08.978 | INFO | Downloaded 34GiB to GPU in 2.06888s (16.434GiB/s) | 20260505 21:49:14.117 | INFO | Analyzing chunk frames = [6561280,6561792) | 20260505 21:49:14.117 | INFO | MemInfo: { | 20260505 21:49:14.117 | INFO | "gpu" : " 17 GB", | 20260505 21:49:14.117 | INFO | "huge" : "", | 20260505 21:49:14.117 | INFO | "huge_shared" : " 27 GB", | 20260505 21:49:14.117 | INFO | "malloc" : " 14 GB", | 20260505 21:49:14.117 | INFO | "pinned" : " 29 GB" | 20260505 21:49:14.117 | INFO | } | 20260505 21:49:14.117 | INFO | Perf report for the last 5.13899 seconds: | 20260505 21:49:14.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:49:14.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:14.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:14.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:14.117 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 21:49:14.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:14.117 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:14.117 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 21:49:14.117 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:14.117 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:14.117 | INFO | Uploaded 36GiB to GPU in 2.0966s (17.1706GiB/s) | 20260505 21:49:14.117 | INFO | Downloaded 34GiB to GPU in 2.0032s (16.9729GiB/s) | 20260505 21:49:14.117 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 21:49:14.117 | INFO | Total mem usage: 15.3 GiB | 20260505 21:49:14.117 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 21:49:14.117 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.98%) | 20260505 21:49:14.117 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 21:49:14.117 | INFO | High Water Breakdown: | 20260505 21:49:14.117 | INFO | 60.9% : 8.57 GiB ( 90% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 21:49:14.117 | INFO | 38.9% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 21:49:14.117 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 21:49:14.118 | INFO | ------------------------------------------ | 20260505 21:49:14.118 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:49:14.118 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 21:49:14.118 | INFO | Total mem usage: 26.5 GiB | 20260505 21:49:14.118 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 21:49:14.118 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 21:49:14.118 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 21:49:14.118 | INFO | High Water Breakdown: | 20260505 21:49:14.118 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:49:14.118 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:49:14.118 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:49:14.118 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:49:14.118 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:49:14.118 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:49:14.118 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 21:49:14.118 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 21:49:14.118 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:49:14.118 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 21:49:14.118 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 21:49:14.118 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 21:49:14.118 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 21:49:14.118 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 21:49:14.118 | INFO | ------------------------------------------ | 20260505 21:49:14.118 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:49:14.118 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 21:49:14.118 | INFO | Total mem usage: 25.6 GiB | 20260505 21:49:14.118 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 21:49:14.118 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 21:49:14.118 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 21:49:14.118 | INFO | High Water Breakdown: | 20260505 21:49:14.118 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 21:49:14.118 | INFO | ------------------------------------------ | 20260505 21:49:14.118 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:49:14.118 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 21:49:14.118 | INFO | Total mem usage: 15.7 GiB | 20260505 21:49:14.118 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 21:49:14.118 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 21:49:14.118 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 21:49:14.118 | INFO | High Water Breakdown: | 20260505 21:49:14.118 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 21:49:14.118 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 21:49:14.118 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 21:49:14.118 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 21:49:14.118 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 21:49:14.118 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 21:49:14.118 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 21:49:14.118 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 21:49:14.118 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 21:49:14.118 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:49:14.118 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 21:49:14.118 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:49:14.118 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 21:49:14.118 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 21:49:14.118 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 21:49:14.118 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 21:49:14.118 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:49:14.118 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 21:49:14.118 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:49:14.118 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:49:14.118 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:49:14.118 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 21:49:14.118 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 21:49:14.118 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:49:14.118 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 21:49:14.118 | INFO | ------------------------------------------ | 20260505 21:49:14.118 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:49:14.118 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 21:49:14.118 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:49:14.118 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:49:14.118 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.39e+01, 1.33e+01 / 9.68e-01 | 20260505 21:49:14.118 | INFO | PerfReport: 0.29%, AggregateStats -- 9.17e-02 / 1.04e-01, 1.02e-02 / 5.02e-03 | 20260505 21:49:14.118 | INFO | PerfReport: 11.78%, Baseline -- 3.70e+00 / 4.54e+00, 5.69e-01 / 1.67e-01 | 20260505 21:49:14.118 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.50e+00, 1.87e-01 / 1.14e-01 | 20260505 21:49:14.118 | INFO | PerfReport: 15.97%, DME -- 4.01e+01 / 4.39e+01, 3.83e+00 / 1.15e+00 | 20260505 21:49:14.118 | INFO | PerfReport: 57.25%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.55e+00 / 1.11e+00 | 20260505 21:49:14.118 | INFO | PerfReport: 7.16%, PulseAccumulating -- 2.25e+00 / 2.80e+00, 4.36e-01 / 2.19e-01 | 20260505 21:49:14.118 | INFO | PerfReport: 3.18%, Metrics -- 9.98e-01 / 1.29e+00, 2.74e-01 / 1.58e-01 | 20260505 21:49:14.118 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.47e-01 / 4.10e-01, 5.99e-02 / 3.79e-02 | 20260505 21:49:14.118 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 21:49:14.118 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:49:14.118 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:49:14.118 | INFO | PerfReport: Full Execution -- 9.89e+01 / -nan, 2.51e+01 / 0.00e+00 | 20260505 21:49:14.118 | INFO | PerfReport: 22.36%, Upload -- 2.21e+01 / 0.00e+00, 7.63e+00 / 0.00e+00 | 20260505 21:49:14.118 | INFO | PerfReport: 21.79%, Download -- 2.15e+01 / 0.00e+00, 7.68e+00 / 0.00e+00 | 20260505 21:49:14.118 | INFO | PerfReport: 31.77%, Compute -- 3.14e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260505 21:49:14.119 | INFO | PerfReport: 2.20%, LockUpload -- 2.18e+00 / 0.00e+00, 7.87e+00 / 0.00e+00 | 20260505 21:49:14.119 | INFO | PerfReport: 1.34%, LockDownload -- 1.32e+00 / 0.00e+00, 2.93e+00 / 0.00e+00 | 20260505 21:49:14.119 | INFO | PerfReport: 20.52%, LockCompute -- 2.03e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 21:49:14.119 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.33e-02 / 0.00e+00 | 20260505 21:49:14.254 | INFO | Starting a round of Baz IO | 20260505 21:49:17.407 | INFO | Populating RTMetrics report | 20260505 21:49:17.407 | INFO | RTMetrics report populated | 20260505 21:49:17.407 | INFO | Opening tmp RTMetrics json file | 20260505 21:49:17.410 | INFO | Serializing RTMetrics to json | 20260505 21:49:17.414 | INFO | Serialization done, swapping tmp file now... | 20260505 21:49:17.414 | INFO | RTMetrics done | 20260505 21:49:17.490 | INFO | Finished a round of Baz IO: Wrote 1.952 GiB in 3.235 seconds | 20260505 21:49:17.490 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.55% / 0.00% / 78.45% | 20260505 21:49:17.490 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.137575% / 0% / 0.000107288% | 20260505 21:49:17.490 | INFO | Baz IO overall: | 20260505 21:49:17.490 | INFO | BytesWritten : 3792469824481 | 20260505 21:49:17.490 | INFO | NumEvents : 1790070872093 | 20260505 21:49:19.257 | INFO | Analyzing chunk frames = [6561792,6562304) | 20260505 21:49:19.257 | INFO | MemInfo: { | 20260505 21:49:19.257 | INFO | "gpu" : " 17 GB", | 20260505 21:49:19.257 | INFO | "huge" : "", | 20260505 21:49:19.257 | INFO | "huge_shared" : " 27 GB", | 20260505 21:49:19.257 | INFO | "malloc" : " 14 GB", | 20260505 21:49:19.257 | INFO | "pinned" : " 29 GB" | 20260505 21:49:19.257 | INFO | } | 20260505 21:49:19.257 | INFO | Perf report for the last 5.14028 seconds: | 20260505 21:49:19.257 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:49:19.257 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:19.257 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:19.257 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:19.257 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 21:49:19.257 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:19.257 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260505 21:49:19.257 | INFO | PRE_HQ : 59.6%, 1.8, 54.6ms, 0 | 20260505 21:49:19.257 | INFO | BAZWRITER : 62.5%, 3.6, 115.9ms, 0 | 20260505 21:49:19.257 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 21:49:19.257 | INFO | Uploaded 36GiB to GPU in 2.14674s (16.7696GiB/s) | 20260505 21:49:19.258 | INFO | Downloaded 38GiB to GPU in 2.23616s (16.9934GiB/s) | 20260505 21:49:24.400 | INFO | Analyzing chunk frames = [6562304,6562816) | 20260505 21:49:24.400 | INFO | MemInfo: { | 20260505 21:49:24.400 | INFO | "gpu" : " 17 GB", | 20260505 21:49:24.400 | INFO | "huge" : "", | 20260505 21:49:24.400 | INFO | "huge_shared" : " 27 GB", | 20260505 21:49:24.400 | INFO | "malloc" : " 14 GB", | 20260505 21:49:24.400 | INFO | "pinned" : " 29 GB" | 20260505 21:49:24.400 | INFO | } | 20260505 21:49:24.400 | INFO | Perf report for the last 5.14332 seconds: | 20260505 21:49:24.400 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:49:24.400 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:24.400 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:24.400 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:24.401 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 21:49:24.401 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:24.401 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:24.401 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 21:49:24.401 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:24.401 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:24.401 | INFO | Uploaded 36GiB to GPU in 2.07933s (17.3133GiB/s) | 20260505 21:49:24.401 | INFO | Downloaded 34GiB to GPU in 2.03155s (16.736GiB/s) | 20260505 21:49:29.540 | INFO | Analyzing chunk frames = [6562816,6563328) | 20260505 21:49:29.540 | INFO | MemInfo: { | 20260505 21:49:29.540 | INFO | "gpu" : " 17 GB", | 20260505 21:49:29.540 | INFO | "huge" : "", | 20260505 21:49:29.540 | INFO | "huge_shared" : " 27 GB", | 20260505 21:49:29.540 | INFO | "malloc" : " 14 GB", | 20260505 21:49:29.540 | INFO | "pinned" : " 29 GB" | 20260505 21:49:29.540 | INFO | } | 20260505 21:49:29.540 | INFO | Perf report for the last 5.13964 seconds: | 20260505 21:49:29.540 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:49:29.540 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:29.540 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:29.540 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:29.540 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260505 21:49:29.540 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:29.540 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:29.540 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 21:49:29.540 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:29.540 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:29.540 | INFO | Uploaded 36GiB to GPU in 2.10752s (17.0817GiB/s) | 20260505 21:49:29.540 | INFO | Downloaded 34GiB to GPU in 2.02692s (16.7742GiB/s) | 20260505 21:49:34.669 | INFO | Analyzing chunk frames = [6563328,6563840) | 20260505 21:49:34.669 | INFO | MemInfo: { | 20260505 21:49:34.669 | INFO | "gpu" : " 17 GB", | 20260505 21:49:34.669 | INFO | "huge" : "", | 20260505 21:49:34.669 | INFO | "huge_shared" : " 27 GB", | 20260505 21:49:34.669 | INFO | "malloc" : " 14 GB", | 20260505 21:49:34.669 | INFO | "pinned" : " 29 GB" | 20260505 21:49:34.669 | INFO | } | 20260505 21:49:34.670 | INFO | Perf report for the last 5.12937 seconds: | 20260505 21:49:34.670 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:49:34.670 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:34.670 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:34.670 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:34.670 | INFO | ANALYSIS : 61.9%, 3.0, 96.0ms, 0 | 20260505 21:49:34.670 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:34.670 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:34.670 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 21:49:34.670 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:34.670 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:34.670 | INFO | Uploaded 36GiB to GPU in 2.1247s (16.9436GiB/s) | 20260505 21:49:34.670 | INFO | Downloaded 34GiB to GPU in 2.02548s (16.7862GiB/s) | 20260505 21:49:39.809 | INFO | Analyzing chunk frames = [6563840,6564352) | 20260505 21:49:39.809 | INFO | MemInfo: { | 20260505 21:49:39.809 | INFO | "gpu" : " 17 GB", | 20260505 21:49:39.809 | INFO | "huge" : "", | 20260505 21:49:39.809 | INFO | "huge_shared" : " 27 GB", | 20260505 21:49:39.809 | INFO | "malloc" : " 14 GB", | 20260505 21:49:39.809 | INFO | "pinned" : " 29 GB" | 20260505 21:49:39.809 | INFO | } | 20260505 21:49:39.809 | INFO | Perf report for the last 5.13973 seconds: | 20260505 21:49:39.809 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:49:39.809 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:39.809 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:39.809 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:39.809 | INFO | ANALYSIS : 61.6%, 3.0, 96.2ms, 0 | 20260505 21:49:39.809 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:39.809 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:39.809 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 21:49:39.809 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:39.809 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:39.810 | INFO | Uploaded 36GiB to GPU in 2.13237s (16.8826GiB/s) | 20260505 21:49:39.810 | INFO | Downloaded 34GiB to GPU in 2.06324s (16.4789GiB/s) | 20260505 21:49:44.953 | INFO | Analyzing chunk frames = [6564352,6564864) | 20260505 21:49:44.953 | INFO | MemInfo: { | 20260505 21:49:44.953 | INFO | "gpu" : " 17 GB", | 20260505 21:49:44.953 | INFO | "huge" : "", | 20260505 21:49:44.953 | INFO | "huge_shared" : " 27 GB", | 20260505 21:49:44.953 | INFO | "malloc" : " 14 GB", | 20260505 21:49:44.953 | INFO | "pinned" : " 29 GB" | 20260505 21:49:44.953 | INFO | } | 20260505 21:49:44.953 | INFO | Perf report for the last 5.14411 seconds: | 20260505 21:49:44.953 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:49:44.953 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:44.954 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:44.954 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:44.954 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 21:49:44.954 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:44.954 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:44.954 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 21:49:44.954 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:44.954 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:44.954 | INFO | Uploaded 36GiB to GPU in 2.15569s (16.7GiB/s) | 20260505 21:49:44.954 | INFO | Downloaded 34GiB to GPU in 2.06333s (16.4782GiB/s) | 20260505 21:49:50.095 | INFO | Analyzing chunk frames = [6564864,6565376) | 20260505 21:49:50.095 | INFO | MemInfo: { | 20260505 21:49:50.095 | INFO | "gpu" : " 17 GB", | 20260505 21:49:50.095 | INFO | "huge" : "", | 20260505 21:49:50.095 | INFO | "huge_shared" : " 27 GB", | 20260505 21:49:50.095 | INFO | "malloc" : " 14 GB", | 20260505 21:49:50.095 | INFO | "pinned" : " 29 GB" | 20260505 21:49:50.095 | INFO | } | 20260505 21:49:50.095 | INFO | Perf report for the last 5.14178 seconds: | 20260505 21:49:50.095 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:49:50.095 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:50.095 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:50.095 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:50.095 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 21:49:50.095 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:50.095 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:50.095 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 21:49:50.095 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:50.095 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:50.095 | INFO | Uploaded 36GiB to GPU in 2.13937s (16.8274GiB/s) | 20260505 21:49:50.096 | INFO | Downloaded 34GiB to GPU in 2.10644s (16.141GiB/s) | 20260505 21:49:55.231 | INFO | Analyzing chunk frames = [6565376,6565888) | 20260505 21:49:55.231 | INFO | MemInfo: { | 20260505 21:49:55.231 | INFO | "gpu" : " 17 GB", | 20260505 21:49:55.231 | INFO | "huge" : "", | 20260505 21:49:55.231 | INFO | "huge_shared" : " 27 GB", | 20260505 21:49:55.231 | INFO | "malloc" : " 14 GB", | 20260505 21:49:55.231 | INFO | "pinned" : " 29 GB" | 20260505 21:49:55.231 | INFO | } | 20260505 21:49:55.231 | INFO | Perf report for the last 5.13619 seconds: | 20260505 21:49:55.231 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:49:55.231 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:55.231 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:55.231 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:55.231 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 21:49:55.231 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:55.232 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:55.232 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:49:55.232 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:55.232 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:49:55.232 | INFO | Uploaded 36GiB to GPU in 2.10988s (17.0626GiB/s) | 20260505 21:49:55.232 | INFO | Downloaded 34GiB to GPU in 1.99965s (17.003GiB/s) | 20260505 21:49:55.366 | INFO | Starting a round of Baz IO | 20260505 21:49:58.520 | INFO | Populating RTMetrics report | 20260505 21:49:58.520 | INFO | RTMetrics report populated | 20260505 21:49:58.520 | INFO | Opening tmp RTMetrics json file | 20260505 21:49:58.521 | INFO | Serializing RTMetrics to json | 20260505 21:49:58.522 | INFO | Serialization done, swapping tmp file now... | 20260505 21:49:58.523 | INFO | RTMetrics done | 20260505 21:49:58.600 | INFO | Finished a round of Baz IO: Wrote 1.953 GiB in 3.234 seconds | 20260505 21:49:58.600 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.54% / 0.00% / 78.45% | 20260505 21:49:58.600 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.13779% / 0% / 0.000127157% | 20260505 21:49:58.600 | INFO | Baz IO overall: | 20260505 21:49:58.600 | INFO | BytesWritten : 3794567254936 | 20260505 21:49:58.600 | INFO | NumEvents : 1791038139451 | 20260505 21:50:00.369 | INFO | Analyzing chunk frames = [6565888,6566400) | 20260505 21:50:00.369 | INFO | MemInfo: { | 20260505 21:50:00.369 | INFO | "gpu" : " 17 GB", | 20260505 21:50:00.369 | INFO | "huge" : "", | 20260505 21:50:00.369 | INFO | "huge_shared" : " 27 GB", | 20260505 21:50:00.369 | INFO | "malloc" : " 14 GB", | 20260505 21:50:00.369 | INFO | "pinned" : " 29 GB" | 20260505 21:50:00.369 | INFO | } | 20260505 21:50:00.369 | INFO | Perf report for the last 5.13749 seconds: | 20260505 21:50:00.369 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:50:00.369 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:00.369 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:00.369 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:00.369 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260505 21:50:00.369 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:00.369 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260505 21:50:00.369 | INFO | PRE_HQ : 59.7%, 1.8, 54.9ms, 0 | 20260505 21:50:00.369 | INFO | BAZWRITER : 62.5%, 3.6, 115.7ms, 0 | 20260505 21:50:00.369 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 21:50:00.369 | INFO | Uploaded 36GiB to GPU in 2.16157s (16.6546GiB/s) | 20260505 21:50:00.369 | INFO | Downloaded 38GiB to GPU in 2.23934s (16.9693GiB/s) | 20260505 21:50:05.509 | INFO | Analyzing chunk frames = [6566400,6566912) | 20260505 21:50:05.509 | INFO | MemInfo: { | 20260505 21:50:05.509 | INFO | "gpu" : " 17 GB", | 20260505 21:50:05.509 | INFO | "huge" : "", | 20260505 21:50:05.509 | INFO | "huge_shared" : " 27 GB", | 20260505 21:50:05.509 | INFO | "malloc" : " 14 GB", | 20260505 21:50:05.509 | INFO | "pinned" : " 29 GB" | 20260505 21:50:05.509 | INFO | } | 20260505 21:50:05.510 | INFO | Perf report for the last 5.14064 seconds: | 20260505 21:50:05.510 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:50:05.510 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:05.510 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:05.510 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:05.510 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 21:50:05.510 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:05.510 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:05.510 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 21:50:05.510 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:05.510 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:05.510 | INFO | Uploaded 36GiB to GPU in 2.08663s (17.2527GiB/s) | 20260505 21:50:05.510 | INFO | Downloaded 34GiB to GPU in 2.01988s (16.8327GiB/s) | 20260505 21:50:10.651 | INFO | Analyzing chunk frames = [6566912,6567424) | 20260505 21:50:10.651 | INFO | MemInfo: { | 20260505 21:50:10.651 | INFO | "gpu" : " 17 GB", | 20260505 21:50:10.651 | INFO | "huge" : "", | 20260505 21:50:10.651 | INFO | "huge_shared" : " 27 GB", | 20260505 21:50:10.652 | INFO | "malloc" : " 14 GB", | 20260505 21:50:10.652 | INFO | "pinned" : " 29 GB" | 20260505 21:50:10.652 | INFO | } | 20260505 21:50:10.652 | INFO | Perf report for the last 5.14209 seconds: | 20260505 21:50:10.652 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:50:10.652 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:10.652 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:10.652 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:10.652 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260505 21:50:10.652 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:10.652 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:10.652 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:50:10.652 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:10.652 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:10.652 | INFO | Uploaded 36GiB to GPU in 2.13702s (16.8459GiB/s) | 20260505 21:50:10.652 | INFO | Downloaded 34GiB to GPU in 2.02891s (16.7578GiB/s) | 20260505 21:50:15.787 | INFO | Analyzing chunk frames = [6567424,6567936) | 20260505 21:50:15.787 | INFO | MemInfo: { | 20260505 21:50:15.787 | INFO | "gpu" : " 17 GB", | 20260505 21:50:15.787 | INFO | "huge" : "", | 20260505 21:50:15.787 | INFO | "huge_shared" : " 27 GB", | 20260505 21:50:15.787 | INFO | "malloc" : " 14 GB", | 20260505 21:50:15.787 | INFO | "pinned" : " 29 GB" | 20260505 21:50:15.787 | INFO | } | 20260505 21:50:15.787 | INFO | Perf report for the last 5.13572 seconds: | 20260505 21:50:15.787 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:50:15.787 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:15.787 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:15.787 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:15.787 | INFO | ANALYSIS : 61.9%, 3.0, 96.0ms, 0 | 20260505 21:50:15.787 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:15.787 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:15.788 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 21:50:15.788 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:15.788 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:15.788 | INFO | Uploaded 36GiB to GPU in 2.13706s (16.8456GiB/s) | 20260505 21:50:15.788 | INFO | Downloaded 34GiB to GPU in 2.02878s (16.7588GiB/s) | 20260505 21:50:20.930 | INFO | Analyzing chunk frames = [6567936,6568448) | 20260505 21:50:20.930 | INFO | MemInfo: { | 20260505 21:50:20.930 | INFO | "gpu" : " 17 GB", | 20260505 21:50:20.930 | INFO | "huge" : "", | 20260505 21:50:20.930 | INFO | "huge_shared" : " 27 GB", | 20260505 21:50:20.930 | INFO | "malloc" : " 14 GB", | 20260505 21:50:20.930 | INFO | "pinned" : " 29 GB" | 20260505 21:50:20.930 | INFO | } | 20260505 21:50:20.930 | INFO | Perf report for the last 5.14246 seconds: | 20260505 21:50:20.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:50:20.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:20.930 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:20.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:20.930 | INFO | ANALYSIS : 61.7%, 3.0, 96.5ms, 0 | 20260505 21:50:20.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:20.930 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:20.930 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 21:50:20.930 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:20.930 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:20.930 | INFO | Uploaded 36GiB to GPU in 2.14131s (16.8121GiB/s) | 20260505 21:50:20.930 | INFO | Downloaded 34GiB to GPU in 2.06378s (16.4747GiB/s) | 20260505 21:50:26.067 | INFO | Analyzing chunk frames = [6568448,6568960) | 20260505 21:50:26.067 | INFO | MemInfo: { | 20260505 21:50:26.067 | INFO | "gpu" : " 17 GB", | 20260505 21:50:26.067 | INFO | "huge" : "", | 20260505 21:50:26.067 | INFO | "huge_shared" : " 27 GB", | 20260505 21:50:26.067 | INFO | "malloc" : " 14 GB", | 20260505 21:50:26.067 | INFO | "pinned" : " 29 GB" | 20260505 21:50:26.067 | INFO | } | 20260505 21:50:26.067 | INFO | Perf report for the last 5.13752 seconds: | 20260505 21:50:26.067 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:50:26.067 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:26.067 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:26.067 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:26.067 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 21:50:26.067 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:26.067 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:26.068 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:50:26.068 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:26.068 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:26.068 | INFO | Uploaded 36GiB to GPU in 2.15658s (16.6931GiB/s) | 20260505 21:50:26.068 | INFO | Downloaded 34GiB to GPU in 2.06174s (16.491GiB/s) | 20260505 21:50:31.216 | INFO | Analyzing chunk frames = [6568960,6569472) | 20260505 21:50:31.216 | INFO | MemInfo: { | 20260505 21:50:31.216 | INFO | "gpu" : " 17 GB", | 20260505 21:50:31.216 | INFO | "huge" : "", | 20260505 21:50:31.216 | INFO | "huge_shared" : " 27 GB", | 20260505 21:50:31.216 | INFO | "malloc" : " 14 GB", | 20260505 21:50:31.216 | INFO | "pinned" : " 29 GB" | 20260505 21:50:31.216 | INFO | } | 20260505 21:50:31.216 | INFO | Perf report for the last 5.14867 seconds: | 20260505 21:50:31.216 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:50:31.216 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:31.216 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:31.216 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:31.216 | INFO | ANALYSIS : 62.5%, 3.0, 97.8ms, 0 | 20260505 21:50:31.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:31.216 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:31.216 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 21:50:31.216 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:31.216 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:31.216 | INFO | Uploaded 36GiB to GPU in 2.1212s (16.9715GiB/s) | 20260505 21:50:31.217 | INFO | Downloaded 34GiB to GPU in 2.10282s (16.1688GiB/s) | 20260505 21:50:36.350 | INFO | Analyzing chunk frames = [6569472,6569984) | 20260505 21:50:36.350 | INFO | MemInfo: { | 20260505 21:50:36.350 | INFO | "gpu" : " 17 GB", | 20260505 21:50:36.350 | INFO | "huge" : "", | 20260505 21:50:36.350 | INFO | "huge_shared" : " 27 GB", | 20260505 21:50:36.350 | INFO | "malloc" : " 14 GB", | 20260505 21:50:36.350 | INFO | "pinned" : " 29 GB" | 20260505 21:50:36.350 | INFO | } | 20260505 21:50:36.350 | INFO | Perf report for the last 5.13401 seconds: | 20260505 21:50:36.350 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:50:36.350 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:36.350 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:36.350 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:36.350 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 21:50:36.350 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:36.350 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:36.350 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 21:50:36.350 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:36.350 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:36.351 | INFO | Uploaded 36GiB to GPU in 2.12468s (16.9437GiB/s) | 20260505 21:50:36.351 | INFO | Downloaded 34GiB to GPU in 2.01334s (16.8873GiB/s) | 20260505 21:50:36.351 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 21:50:36.351 | INFO | Total mem usage: 15.3 GiB | 20260505 21:50:36.351 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 21:50:36.351 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.18%) | 20260505 21:50:36.351 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 21:50:36.351 | INFO | High Water Breakdown: | 20260505 21:50:36.351 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 21:50:36.351 | INFO | 38.8% : 5.5 GiB ( 82.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 21:50:36.351 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 21:50:36.351 | INFO | ------------------------------------------ | 20260505 21:50:36.351 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:50:36.351 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 21:50:36.351 | INFO | Total mem usage: 26.5 GiB | 20260505 21:50:36.351 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 21:50:36.351 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 21:50:36.351 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 21:50:36.351 | INFO | High Water Breakdown: | 20260505 21:50:36.351 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:50:36.351 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:50:36.351 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:50:36.351 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:50:36.351 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:50:36.351 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:50:36.351 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 21:50:36.351 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 21:50:36.351 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:50:36.351 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 21:50:36.351 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 21:50:36.351 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 21:50:36.351 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 21:50:36.351 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 21:50:36.351 | INFO | ------------------------------------------ | 20260505 21:50:36.351 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:50:36.351 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 21:50:36.351 | INFO | Total mem usage: 25.6 GiB | 20260505 21:50:36.351 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 21:50:36.351 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 21:50:36.351 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 21:50:36.351 | INFO | High Water Breakdown: | 20260505 21:50:36.351 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 21:50:36.351 | INFO | ------------------------------------------ | 20260505 21:50:36.351 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:50:36.351 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 21:50:36.351 | INFO | Total mem usage: 15.7 GiB | 20260505 21:50:36.351 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 21:50:36.352 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 21:50:36.352 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 21:50:36.352 | INFO | High Water Breakdown: | 20260505 21:50:36.352 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 21:50:36.352 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 21:50:36.352 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 21:50:36.352 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 21:50:36.352 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 21:50:36.352 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 21:50:36.352 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 21:50:36.352 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 21:50:36.352 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 21:50:36.352 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:50:36.352 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 21:50:36.352 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:50:36.352 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 21:50:36.352 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 21:50:36.352 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 21:50:36.352 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 21:50:36.352 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:50:36.352 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 21:50:36.352 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:50:36.352 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:50:36.352 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:50:36.352 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 21:50:36.352 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 21:50:36.352 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:50:36.352 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 21:50:36.352 | INFO | ------------------------------------------ | 20260505 21:50:36.352 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:50:36.352 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 21:50:36.352 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:50:36.352 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:50:36.352 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.39e+01, 1.35e+01 / 7.40e-01 | 20260505 21:50:36.352 | INFO | PerfReport: 0.29%, AggregateStats -- 9.25e-02 / 1.04e-01, 1.04e-02 / 4.12e-03 | 20260505 21:50:36.352 | INFO | PerfReport: 11.82%, Baseline -- 3.71e+00 / 4.54e+00, 5.68e-01 / 1.15e-01 | 20260505 21:50:36.352 | INFO | PerfReport: 3.29%, Histogram -- 1.03e+00 / 1.53e+00, 1.91e-01 / 7.03e-02 | 20260505 21:50:36.352 | INFO | PerfReport: 15.91%, DME -- 4.00e+01 / 4.38e+01, 3.88e+00 / 7.54e-01 | 20260505 21:50:36.352 | INFO | PerfReport: 57.24%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.53e+00 / 1.06e+00 | 20260505 21:50:36.352 | INFO | PerfReport: 7.16%, PulseAccumulating -- 2.25e+00 / 2.79e+00, 4.29e-01 / 2.15e-01 | 20260505 21:50:36.352 | INFO | PerfReport: 3.18%, Metrics -- 1.00e+00 / 1.31e+00, 2.72e-01 / 1.15e-01 | 20260505 21:50:36.352 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.47e-01 / 4.06e-01, 5.99e-02 / 3.66e-02 | 20260505 21:50:36.352 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 21:50:36.352 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:50:36.352 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:50:36.352 | INFO | PerfReport: Full Execution -- 9.89e+01 / -nan, 2.52e+01 / 0.00e+00 | 20260505 21:50:36.352 | INFO | PerfReport: 22.41%, Upload -- 2.22e+01 / 0.00e+00, 7.62e+00 / 0.00e+00 | 20260505 21:50:36.352 | INFO | PerfReport: 21.79%, Download -- 2.16e+01 / 0.00e+00, 7.66e+00 / 0.00e+00 | 20260505 21:50:36.352 | INFO | PerfReport: 31.77%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 21:50:36.352 | INFO | PerfReport: 2.22%, LockUpload -- 2.19e+00 / 0.00e+00, 7.97e+00 / 0.00e+00 | 20260505 21:50:36.352 | INFO | PerfReport: 1.35%, LockDownload -- 1.33e+00 / 0.00e+00, 2.89e+00 / 0.00e+00 | 20260505 21:50:36.352 | INFO | PerfReport: 20.45%, LockCompute -- 2.02e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 21:50:36.352 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.66e-02 / 0.00e+00 | 20260505 21:50:36.473 | INFO | Starting a round of Baz IO | 20260505 21:50:39.626 | INFO | Populating RTMetrics report | 20260505 21:50:39.626 | INFO | RTMetrics report populated | 20260505 21:50:39.626 | INFO | Opening tmp RTMetrics json file | 20260505 21:50:39.629 | INFO | Serializing RTMetrics to json | 20260505 21:50:39.630 | INFO | Serialization done, swapping tmp file now... | 20260505 21:50:39.631 | INFO | RTMetrics done | 20260505 21:50:39.709 | INFO | Finished a round of Baz IO: Wrote 1.954 GiB in 3.236 seconds | 20260505 21:50:39.709 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.54% / 0.00% / 78.46% | 20260505 21:50:39.709 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.137432% / 0% / 0.00013113% | 20260505 21:50:39.709 | INFO | Baz IO overall: | 20260505 21:50:39.709 | INFO | BytesWritten : 3796665474646 | 20260505 21:50:39.709 | INFO | NumEvents : 1792005994603 | 20260505 21:50:41.484 | INFO | Analyzing chunk frames = [6569984,6570496) | 20260505 21:50:41.484 | INFO | MemInfo: { | 20260505 21:50:41.484 | INFO | "gpu" : " 17 GB", | 20260505 21:50:41.484 | INFO | "huge" : "", | 20260505 21:50:41.484 | INFO | "huge_shared" : " 27 GB", | 20260505 21:50:41.484 | INFO | "malloc" : " 14 GB", | 20260505 21:50:41.484 | INFO | "pinned" : " 29 GB" | 20260505 21:50:41.484 | INFO | } | 20260505 21:50:41.484 | INFO | Perf report for the last 5.13376 seconds: | 20260505 21:50:41.484 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:50:41.484 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:41.484 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:41.484 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:41.484 | INFO | ANALYSIS : 62.8%, 3.0, 97.8ms, 0 | 20260505 21:50:41.484 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:41.484 | INFO | RT_METRICS : 43.8%, 1.0, 22.9ms, 0 | 20260505 21:50:41.484 | INFO | PRE_HQ : 60.0%, 1.8, 54.9ms, 0 | 20260505 21:50:41.484 | INFO | BAZWRITER : 62.2%, 3.6, 116.2ms, 0 | 20260505 21:50:41.484 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 21:50:41.484 | INFO | Uploaded 36GiB to GPU in 2.15554s (16.7011GiB/s) | 20260505 21:50:41.484 | INFO | Downloaded 38GiB to GPU in 2.23546s (16.9988GiB/s) | 20260505 21:50:46.620 | INFO | Analyzing chunk frames = [6570496,6571008) | 20260505 21:50:46.620 | INFO | MemInfo: { | 20260505 21:50:46.620 | INFO | "gpu" : " 17 GB", | 20260505 21:50:46.620 | INFO | "huge" : "", | 20260505 21:50:46.620 | INFO | "huge_shared" : " 27 GB", | 20260505 21:50:46.620 | INFO | "malloc" : " 14 GB", | 20260505 21:50:46.620 | INFO | "pinned" : " 29 GB" | 20260505 21:50:46.620 | INFO | } | 20260505 21:50:46.620 | INFO | Perf report for the last 5.13653 seconds: | 20260505 21:50:46.620 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:50:46.620 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:46.620 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:46.620 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:46.620 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 21:50:46.620 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:46.621 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:46.621 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 21:50:46.621 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:46.621 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:46.621 | INFO | Uploaded 36GiB to GPU in 2.09934s (17.1482GiB/s) | 20260505 21:50:46.621 | INFO | Downloaded 34GiB to GPU in 2.02677s (16.7755GiB/s) | 20260505 21:50:51.767 | INFO | Analyzing chunk frames = [6571008,6571520) | 20260505 21:50:51.767 | INFO | MemInfo: { | 20260505 21:50:51.767 | INFO | "gpu" : " 17 GB", | 20260505 21:50:51.767 | INFO | "huge" : "", | 20260505 21:50:51.767 | INFO | "huge_shared" : " 27 GB", | 20260505 21:50:51.767 | INFO | "malloc" : " 14 GB", | 20260505 21:50:51.767 | INFO | "pinned" : " 29 GB" | 20260505 21:50:51.767 | INFO | } | 20260505 21:50:51.768 | INFO | Perf report for the last 5.14708 seconds: | 20260505 21:50:51.768 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:50:51.768 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:51.768 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:51.768 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:51.768 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260505 21:50:51.768 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:51.768 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:51.768 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 21:50:51.768 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:51.768 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:51.768 | INFO | Uploaded 36GiB to GPU in 2.13372s (16.8719GiB/s) | 20260505 21:50:51.768 | INFO | Downloaded 34GiB to GPU in 2.01794s (16.8489GiB/s) | 20260505 21:50:56.909 | INFO | Analyzing chunk frames = [6571520,6572032) | 20260505 21:50:56.909 | INFO | MemInfo: { | 20260505 21:50:56.909 | INFO | "gpu" : " 17 GB", | 20260505 21:50:56.909 | INFO | "huge" : "", | 20260505 21:50:56.909 | INFO | "huge_shared" : " 27 GB", | 20260505 21:50:56.909 | INFO | "malloc" : " 14 GB", | 20260505 21:50:56.909 | INFO | "pinned" : " 29 GB" | 20260505 21:50:56.909 | INFO | } | 20260505 21:50:56.910 | INFO | Perf report for the last 5.14198 seconds: | 20260505 21:50:56.910 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:50:56.910 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:56.910 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:56.910 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:56.910 | INFO | ANALYSIS : 61.9%, 3.0, 96.2ms, 0 | 20260505 21:50:56.910 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:56.910 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:56.910 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 21:50:56.910 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:56.910 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:50:56.910 | INFO | Uploaded 36GiB to GPU in 2.14634s (16.7727GiB/s) | 20260505 21:50:56.910 | INFO | Downloaded 34GiB to GPU in 2.03537s (16.7046GiB/s) | 20260505 21:51:02.048 | INFO | Analyzing chunk frames = [6572032,6572544) | 20260505 21:51:02.048 | INFO | MemInfo: { | 20260505 21:51:02.048 | INFO | "gpu" : " 17 GB", | 20260505 21:51:02.048 | INFO | "huge" : "", | 20260505 21:51:02.048 | INFO | "huge_shared" : " 27 GB", | 20260505 21:51:02.048 | INFO | "malloc" : " 14 GB", | 20260505 21:51:02.048 | INFO | "pinned" : " 29 GB" | 20260505 21:51:02.048 | INFO | } | 20260505 21:51:02.048 | INFO | Perf report for the last 5.13845 seconds: | 20260505 21:51:02.048 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:51:02.048 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:02.048 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:02.048 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:02.048 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260505 21:51:02.048 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:02.048 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:02.048 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:51:02.048 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:02.048 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:02.048 | INFO | Uploaded 36GiB to GPU in 2.101s (17.1347GiB/s) | 20260505 21:51:02.048 | INFO | Downloaded 34GiB to GPU in 2.0443s (16.6316GiB/s) | 20260505 21:51:07.187 | INFO | Analyzing chunk frames = [6572544,6573056) | 20260505 21:51:07.187 | INFO | MemInfo: { | 20260505 21:51:07.187 | INFO | "gpu" : " 17 GB", | 20260505 21:51:07.187 | INFO | "huge" : "", | 20260505 21:51:07.187 | INFO | "huge_shared" : " 27 GB", | 20260505 21:51:07.187 | INFO | "malloc" : " 14 GB", | 20260505 21:51:07.187 | INFO | "pinned" : " 29 GB" | 20260505 21:51:07.187 | INFO | } | 20260505 21:51:07.187 | INFO | Perf report for the last 5.13868 seconds: | 20260505 21:51:07.187 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:51:07.187 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:07.187 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:07.187 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:07.187 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260505 21:51:07.187 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:07.187 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:07.187 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:51:07.187 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:07.187 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:07.187 | INFO | Uploaded 36GiB to GPU in 2.09202s (17.2083GiB/s) | 20260505 21:51:07.187 | INFO | Downloaded 34GiB to GPU in 2.04052s (16.6624GiB/s) | 20260505 21:51:12.320 | INFO | Analyzing chunk frames = [6573056,6573568) | 20260505 21:51:12.320 | INFO | MemInfo: { | 20260505 21:51:12.320 | INFO | "gpu" : " 17 GB", | 20260505 21:51:12.320 | INFO | "huge" : "", | 20260505 21:51:12.320 | INFO | "huge_shared" : " 27 GB", | 20260505 21:51:12.320 | INFO | "malloc" : " 14 GB", | 20260505 21:51:12.320 | INFO | "pinned" : " 29 GB" | 20260505 21:51:12.320 | INFO | } | 20260505 21:51:12.320 | INFO | Perf report for the last 5.13338 seconds: | 20260505 21:51:12.320 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:51:12.320 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:12.320 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:12.320 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:12.320 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 21:51:12.320 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:12.320 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:12.320 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 21:51:12.320 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:12.320 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:12.320 | INFO | Uploaded 36GiB to GPU in 2.14039s (16.8194GiB/s) | 20260505 21:51:12.320 | INFO | Downloaded 34GiB to GPU in 2.13142s (15.9518GiB/s) | 20260505 21:51:17.463 | INFO | Analyzing chunk frames = [6573568,6574080) | 20260505 21:51:17.463 | INFO | MemInfo: { | 20260505 21:51:17.463 | INFO | "gpu" : " 17 GB", | 20260505 21:51:17.463 | INFO | "huge" : "", | 20260505 21:51:17.463 | INFO | "huge_shared" : " 27 GB", | 20260505 21:51:17.463 | INFO | "malloc" : " 14 GB", | 20260505 21:51:17.463 | INFO | "pinned" : " 29 GB" | 20260505 21:51:17.463 | INFO | } | 20260505 21:51:17.463 | INFO | Perf report for the last 5.14285 seconds: | 20260505 21:51:17.463 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:51:17.463 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:17.463 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:17.463 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:17.463 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 21:51:17.463 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:17.463 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:17.463 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:51:17.463 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:17.463 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:17.463 | INFO | Uploaded 36GiB to GPU in 2.10945s (17.066GiB/s) | 20260505 21:51:17.463 | INFO | Downloaded 34GiB to GPU in 2.01124s (16.905GiB/s) | 20260505 21:51:17.589 | INFO | Starting a round of Baz IO | 20260505 21:51:20.739 | INFO | Populating RTMetrics report | 20260505 21:51:20.740 | INFO | RTMetrics report populated | 20260505 21:51:20.740 | INFO | Opening tmp RTMetrics json file | 20260505 21:51:20.740 | INFO | Serializing RTMetrics to json | 20260505 21:51:20.743 | INFO | Serialization done, swapping tmp file now... | 20260505 21:51:20.745 | INFO | RTMetrics done | 20260505 21:51:20.823 | INFO | Finished a round of Baz IO: Wrote 1.952 GiB in 3.233 seconds | 20260505 21:51:20.823 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.54% / 0.00% / 78.46% | 20260505 21:51:20.823 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.137536% / 7.94729e-06% / 0.000111262% | 20260505 21:51:20.823 | INFO | Baz IO overall: | 20260505 21:51:20.823 | INFO | BytesWritten : 3798761804444 | 20260505 21:51:20.823 | INFO | NumEvents : 1792972696965 | 20260505 21:51:22.602 | INFO | Analyzing chunk frames = [6574080,6574592) | 20260505 21:51:22.602 | INFO | MemInfo: { | 20260505 21:51:22.602 | INFO | "gpu" : " 17 GB", | 20260505 21:51:22.602 | INFO | "huge" : "", | 20260505 21:51:22.602 | INFO | "huge_shared" : " 27 GB", | 20260505 21:51:22.602 | INFO | "malloc" : " 14 GB", | 20260505 21:51:22.602 | INFO | "pinned" : " 29 GB" | 20260505 21:51:22.602 | INFO | } | 20260505 21:51:22.602 | INFO | Perf report for the last 5.13913 seconds: | 20260505 21:51:22.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:51:22.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:22.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:22.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:22.602 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 21:51:22.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:22.602 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260505 21:51:22.602 | INFO | PRE_HQ : 60.0%, 1.8, 54.7ms, 0 | 20260505 21:51:22.602 | INFO | BAZWRITER : 62.1%, 3.6, 116.0ms, 0 | 20260505 21:51:22.602 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 21:51:22.603 | INFO | Uploaded 36GiB to GPU in 2.16553s (16.6241GiB/s) | 20260505 21:51:22.603 | INFO | Downloaded 38GiB to GPU in 2.23969s (16.9666GiB/s) | 20260505 21:51:27.736 | INFO | Analyzing chunk frames = [6574592,6575104) | 20260505 21:51:27.736 | INFO | MemInfo: { | 20260505 21:51:27.736 | INFO | "gpu" : " 17 GB", | 20260505 21:51:27.736 | INFO | "huge" : "", | 20260505 21:51:27.736 | INFO | "huge_shared" : " 27 GB", | 20260505 21:51:27.736 | INFO | "malloc" : " 14 GB", | 20260505 21:51:27.736 | INFO | "pinned" : " 29 GB" | 20260505 21:51:27.737 | INFO | } | 20260505 21:51:27.737 | INFO | Perf report for the last 5.13459 seconds: | 20260505 21:51:27.737 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:51:27.737 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:27.737 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:27.737 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:27.737 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 21:51:27.737 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:27.737 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:27.737 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 21:51:27.737 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:27.737 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:27.737 | INFO | Uploaded 36GiB to GPU in 2.08701s (17.2495GiB/s) | 20260505 21:51:27.737 | INFO | Downloaded 34GiB to GPU in 2.02822s (16.7634GiB/s) | 20260505 21:51:32.876 | INFO | Analyzing chunk frames = [6575104,6575616) | 20260505 21:51:32.876 | INFO | MemInfo: { | 20260505 21:51:32.876 | INFO | "gpu" : " 17 GB", | 20260505 21:51:32.876 | INFO | "huge" : "", | 20260505 21:51:32.876 | INFO | "huge_shared" : " 27 GB", | 20260505 21:51:32.876 | INFO | "malloc" : " 14 GB", | 20260505 21:51:32.876 | INFO | "pinned" : " 29 GB" | 20260505 21:51:32.876 | INFO | } | 20260505 21:51:32.876 | INFO | Perf report for the last 5.13938 seconds: | 20260505 21:51:32.876 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:51:32.876 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:32.876 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:32.876 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:32.876 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260505 21:51:32.876 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:32.876 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:32.876 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:51:32.876 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:32.876 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:32.876 | INFO | Uploaded 36GiB to GPU in 2.13541s (16.8586GiB/s) | 20260505 21:51:32.876 | INFO | Downloaded 34GiB to GPU in 2.01951s (16.8357GiB/s) | 20260505 21:51:38.023 | INFO | Analyzing chunk frames = [6575616,6576128) | 20260505 21:51:38.023 | INFO | MemInfo: { | 20260505 21:51:38.023 | INFO | "gpu" : " 17 GB", | 20260505 21:51:38.023 | INFO | "huge" : "", | 20260505 21:51:38.023 | INFO | "huge_shared" : " 27 GB", | 20260505 21:51:38.023 | INFO | "malloc" : " 14 GB", | 20260505 21:51:38.023 | INFO | "pinned" : " 29 GB" | 20260505 21:51:38.023 | INFO | } | 20260505 21:51:38.023 | INFO | Perf report for the last 5.14708 seconds: | 20260505 21:51:38.023 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:51:38.023 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:38.023 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:38.023 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:38.023 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260505 21:51:38.023 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:38.023 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:38.023 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 21:51:38.023 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:38.023 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:38.023 | INFO | Uploaded 36GiB to GPU in 2.2114s (16.2793GiB/s) | 20260505 21:51:38.024 | INFO | Downloaded 34GiB to GPU in 2.05005s (16.585GiB/s) | 20260505 21:51:43.169 | INFO | Analyzing chunk frames = [6576128,6576640) | 20260505 21:51:43.169 | INFO | MemInfo: { | 20260505 21:51:43.169 | INFO | "gpu" : " 17 GB", | 20260505 21:51:43.169 | INFO | "huge" : "", | 20260505 21:51:43.169 | INFO | "huge_shared" : " 27 GB", | 20260505 21:51:43.169 | INFO | "malloc" : " 14 GB", | 20260505 21:51:43.169 | INFO | "pinned" : " 29 GB" | 20260505 21:51:43.169 | INFO | } | 20260505 21:51:43.169 | INFO | Perf report for the last 5.14604 seconds: | 20260505 21:51:43.169 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:51:43.169 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:43.169 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:43.169 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:43.169 | INFO | ANALYSIS : 61.6%, 3.0, 96.2ms, 0 | 20260505 21:51:43.169 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:43.169 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:43.169 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:51:43.169 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:43.169 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:43.169 | INFO | Uploaded 36GiB to GPU in 2.16561s (16.6235GiB/s) | 20260505 21:51:43.169 | INFO | Downloaded 34GiB to GPU in 2.07399s (16.3935GiB/s) | 20260505 21:51:48.302 | INFO | Analyzing chunk frames = [6576640,6577152) | 20260505 21:51:48.302 | INFO | MemInfo: { | 20260505 21:51:48.302 | INFO | "gpu" : " 17 GB", | 20260505 21:51:48.302 | INFO | "huge" : "", | 20260505 21:51:48.302 | INFO | "huge_shared" : " 27 GB", | 20260505 21:51:48.302 | INFO | "malloc" : " 14 GB", | 20260505 21:51:48.302 | INFO | "pinned" : " 29 GB" | 20260505 21:51:48.302 | INFO | } | 20260505 21:51:48.302 | INFO | Perf report for the last 5.13302 seconds: | 20260505 21:51:48.302 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:51:48.302 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:48.302 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:48.302 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:48.302 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260505 21:51:48.302 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:48.302 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:48.302 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:51:48.302 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:48.302 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:48.302 | INFO | Uploaded 36GiB to GPU in 2.06682s (17.4181GiB/s) | 20260505 21:51:48.302 | INFO | Downloaded 34GiB to GPU in 2.03695s (16.6916GiB/s) | 20260505 21:51:53.433 | INFO | Analyzing chunk frames = [6577152,6577664) | 20260505 21:51:53.433 | INFO | MemInfo: { | 20260505 21:51:53.433 | INFO | "gpu" : " 17 GB", | 20260505 21:51:53.433 | INFO | "huge" : "", | 20260505 21:51:53.433 | INFO | "huge_shared" : " 27 GB", | 20260505 21:51:53.433 | INFO | "malloc" : " 14 GB", | 20260505 21:51:53.433 | INFO | "pinned" : " 29 GB" | 20260505 21:51:53.433 | INFO | } | 20260505 21:51:53.433 | INFO | Perf report for the last 5.13049 seconds: | 20260505 21:51:53.433 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:51:53.433 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:53.433 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:53.433 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:53.433 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260505 21:51:53.433 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:53.433 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:53.433 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 21:51:53.433 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:53.433 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:53.433 | INFO | Uploaded 36GiB to GPU in 2.01648s (17.8529GiB/s) | 20260505 21:51:53.433 | INFO | Downloaded 34GiB to GPU in 2.03459s (16.711GiB/s) | 20260505 21:51:58.571 | INFO | Analyzing chunk frames = [6577664,6578176) | 20260505 21:51:58.571 | INFO | MemInfo: { | 20260505 21:51:58.571 | INFO | "gpu" : " 17 GB", | 20260505 21:51:58.571 | INFO | "huge" : "", | 20260505 21:51:58.571 | INFO | "huge_shared" : " 27 GB", | 20260505 21:51:58.571 | INFO | "malloc" : " 14 GB", | 20260505 21:51:58.571 | INFO | "pinned" : " 29 GB" | 20260505 21:51:58.571 | INFO | } | 20260505 21:51:58.571 | INFO | Perf report for the last 5.13864 seconds: | 20260505 21:51:58.571 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:51:58.571 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:58.571 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:58.571 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:58.571 | INFO | ANALYSIS : 62.7%, 3.0, 97.0ms, 0 | 20260505 21:51:58.571 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:58.571 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:58.571 | INFO | PRE_HQ : 27.0%, 1.0, 13.9ms, 0 | 20260505 21:51:58.572 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:58.572 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:51:58.572 | INFO | Uploaded 36GiB to GPU in 2.19762s (16.3814GiB/s) | 20260505 21:51:58.572 | INFO | Downloaded 34GiB to GPU in 2.04611s (16.6169GiB/s) | 20260505 21:51:58.572 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 21:51:58.572 | INFO | Total mem usage: 15.3 GiB | 20260505 21:51:58.572 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 21:51:58.572 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.46%) | 20260505 21:51:58.572 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 21:51:58.572 | INFO | High Water Breakdown: | 20260505 21:51:58.572 | INFO | 61.1% : 8.63 GiB ( 90.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 21:51:58.572 | INFO | 38.8% : 5.48 GiB ( 82.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 21:51:58.572 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 21:51:58.572 | INFO | ------------------------------------------ | 20260505 21:51:58.572 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:51:58.572 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 21:51:58.572 | INFO | Total mem usage: 26.5 GiB | 20260505 21:51:58.572 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 21:51:58.572 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 21:51:58.572 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 21:51:58.572 | INFO | High Water Breakdown: | 20260505 21:51:58.572 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:51:58.572 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:51:58.572 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:51:58.572 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:51:58.572 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:51:58.572 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:51:58.572 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 21:51:58.572 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 21:51:58.572 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:51:58.572 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 21:51:58.572 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 21:51:58.572 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 21:51:58.572 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 21:51:58.572 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 21:51:58.572 | INFO | ------------------------------------------ | 20260505 21:51:58.572 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:51:58.572 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 21:51:58.572 | INFO | Total mem usage: 25.6 GiB | 20260505 21:51:58.572 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 21:51:58.572 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 21:51:58.572 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 21:51:58.572 | INFO | High Water Breakdown: | 20260505 21:51:58.572 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 21:51:58.572 | INFO | ------------------------------------------ | 20260505 21:51:58.572 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:51:58.572 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 21:51:58.572 | INFO | Total mem usage: 15.7 GiB | 20260505 21:51:58.572 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 21:51:58.572 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 21:51:58.572 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 21:51:58.572 | INFO | High Water Breakdown: | 20260505 21:51:58.572 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 21:51:58.572 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 21:51:58.572 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 21:51:58.572 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 21:51:58.572 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 21:51:58.572 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 21:51:58.573 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 21:51:58.573 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 21:51:58.573 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 21:51:58.573 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:51:58.573 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 21:51:58.573 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:51:58.573 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 21:51:58.573 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 21:51:58.573 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 21:51:58.573 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 21:51:58.573 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:51:58.573 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 21:51:58.573 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:51:58.573 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:51:58.573 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:51:58.573 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 21:51:58.573 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 21:51:58.573 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:51:58.573 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 21:51:58.573 | INFO | ------------------------------------------ | 20260505 21:51:58.573 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:51:58.573 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 21:51:58.573 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:51:58.573 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:51:58.573 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.42e+01, 1.35e+01 / 1.16e+00 | 20260505 21:51:58.573 | INFO | PerfReport: 0.29%, AggregateStats -- 9.24e-02 / 1.04e-01, 1.23e-02 / 6.57e-03 | 20260505 21:51:58.573 | INFO | PerfReport: 11.81%, Baseline -- 3.71e+00 / 4.52e+00, 5.69e-01 / 1.67e-01 | 20260505 21:51:58.573 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.54e+00, 1.91e-01 / 8.32e-02 | 20260505 21:51:58.573 | INFO | PerfReport: 15.97%, DME -- 4.01e+01 / 4.41e+01, 3.89e+00 / 1.23e+00 | 20260505 21:51:58.573 | INFO | PerfReport: 57.19%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.48e+00 / 1.10e+00 | 20260505 21:51:58.573 | INFO | PerfReport: 7.18%, PulseAccumulating -- 2.25e+00 / 2.79e+00, 4.32e-01 / 2.31e-01 | 20260505 21:51:58.573 | INFO | PerfReport: 3.18%, Metrics -- 9.98e-01 / 1.28e+00, 2.69e-01 / 1.14e-01 | 20260505 21:51:58.573 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.47e-01 / 4.03e-01, 5.95e-02 / 3.01e-02 | 20260505 21:51:58.573 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 21:51:58.573 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:51:58.573 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:51:58.573 | INFO | PerfReport: Full Execution -- 9.88e+01 / -nan, 2.50e+01 / 0.00e+00 | 20260505 21:51:58.573 | INFO | PerfReport: 22.41%, Upload -- 2.22e+01 / 0.00e+00, 7.63e+00 / 0.00e+00 | 20260505 21:51:58.573 | INFO | PerfReport: 21.79%, Download -- 2.15e+01 / 0.00e+00, 7.58e+00 / 0.00e+00 | 20260505 21:51:58.573 | INFO | PerfReport: 31.77%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 21:51:58.573 | INFO | PerfReport: 2.24%, LockUpload -- 2.21e+00 / 0.00e+00, 7.97e+00 / 0.00e+00 | 20260505 21:51:58.573 | INFO | PerfReport: 1.29%, LockDownload -- 1.27e+00 / 0.00e+00, 2.80e+00 / 0.00e+00 | 20260505 21:51:58.573 | INFO | PerfReport: 20.49%, LockCompute -- 2.03e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 21:51:58.573 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.92e-02 / 0.00e+00 | 20260505 21:51:58.706 | INFO | Starting a round of Baz IO | 20260505 21:52:01.870 | INFO | Populating RTMetrics report | 20260505 21:52:01.871 | INFO | RTMetrics report populated | 20260505 21:52:01.871 | INFO | Opening tmp RTMetrics json file | 20260505 21:52:01.872 | INFO | Serializing RTMetrics to json | 20260505 21:52:01.873 | INFO | Serialization done, swapping tmp file now... | 20260505 21:52:01.874 | INFO | RTMetrics done | 20260505 21:52:01.955 | INFO | Finished a round of Baz IO: Wrote 1.952 GiB in 3.249 seconds | 20260505 21:52:01.955 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.54% / 0.00% / 78.46% | 20260505 21:52:01.955 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.136971% / 3.97364e-06% / 0.000123183% | 20260505 21:52:01.955 | INFO | Baz IO overall: | 20260505 21:52:01.955 | INFO | BytesWritten : 3800857719826 | 20260505 21:52:01.955 | INFO | NumEvents : 1793939080806 | 20260505 21:52:03.717 | INFO | Analyzing chunk frames = [6578176,6578688) | 20260505 21:52:03.717 | INFO | MemInfo: { | 20260505 21:52:03.717 | INFO | "gpu" : " 17 GB", | 20260505 21:52:03.717 | INFO | "huge" : "", | 20260505 21:52:03.717 | INFO | "huge_shared" : " 27 GB", | 20260505 21:52:03.717 | INFO | "malloc" : " 14 GB", | 20260505 21:52:03.717 | INFO | "pinned" : " 29 GB" | 20260505 21:52:03.717 | INFO | } | 20260505 21:52:03.717 | INFO | Perf report for the last 5.14589 seconds: | 20260505 21:52:03.717 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:52:03.717 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:03.717 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:03.717 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:03.717 | INFO | ANALYSIS : 63.1%, 3.0, 98.6ms, 0 | 20260505 21:52:03.717 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:03.717 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260505 21:52:03.717 | INFO | PRE_HQ : 59.7%, 1.8, 55.0ms, 0 | 20260505 21:52:03.717 | INFO | BAZWRITER : 62.6%, 3.6, 116.6ms, 0 | 20260505 21:52:03.717 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 21:52:03.718 | INFO | Uploaded 36GiB to GPU in 2.17652s (16.5402GiB/s) | 20260505 21:52:03.718 | INFO | Downloaded 38GiB to GPU in 2.24931s (16.8941GiB/s) | 20260505 21:52:08.860 | INFO | Analyzing chunk frames = [6578688,6579200) | 20260505 21:52:08.860 | INFO | MemInfo: { | 20260505 21:52:08.860 | INFO | "gpu" : " 17 GB", | 20260505 21:52:08.860 | INFO | "huge" : "", | 20260505 21:52:08.860 | INFO | "huge_shared" : " 27 GB", | 20260505 21:52:08.860 | INFO | "malloc" : " 14 GB", | 20260505 21:52:08.860 | INFO | "pinned" : " 29 GB" | 20260505 21:52:08.860 | INFO | } | 20260505 21:52:08.860 | INFO | Perf report for the last 5.14293 seconds: | 20260505 21:52:08.860 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:52:08.860 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:08.860 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:08.860 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:08.860 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260505 21:52:08.860 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:08.860 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:08.860 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 21:52:08.860 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:08.861 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:08.861 | INFO | Uploaded 36GiB to GPU in 2.0815s (17.2952GiB/s) | 20260505 21:52:08.861 | INFO | Downloaded 34GiB to GPU in 2.02657s (16.7771GiB/s) | 20260505 21:52:13.995 | INFO | Analyzing chunk frames = [6579200,6579712) | 20260505 21:52:13.995 | INFO | MemInfo: { | 20260505 21:52:13.995 | INFO | "gpu" : " 17 GB", | 20260505 21:52:13.995 | INFO | "huge" : "", | 20260505 21:52:13.995 | INFO | "huge_shared" : " 27 GB", | 20260505 21:52:13.995 | INFO | "malloc" : " 14 GB", | 20260505 21:52:13.995 | INFO | "pinned" : " 29 GB" | 20260505 21:52:13.995 | INFO | } | 20260505 21:52:13.995 | INFO | Perf report for the last 5.13491 seconds: | 20260505 21:52:13.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:52:13.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:13.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:13.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:13.995 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260505 21:52:13.995 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:13.995 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:13.995 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:52:13.995 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:13.995 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:13.995 | INFO | Uploaded 36GiB to GPU in 2.15683s (16.6912GiB/s) | 20260505 21:52:13.995 | INFO | Downloaded 34GiB to GPU in 2.05091s (16.578GiB/s) | 20260505 21:52:19.134 | INFO | Analyzing chunk frames = [6579712,6580224) | 20260505 21:52:19.134 | INFO | MemInfo: { | 20260505 21:52:19.134 | INFO | "gpu" : " 17 GB", | 20260505 21:52:19.134 | INFO | "huge" : "", | 20260505 21:52:19.134 | INFO | "huge_shared" : " 27 GB", | 20260505 21:52:19.134 | INFO | "malloc" : " 14 GB", | 20260505 21:52:19.134 | INFO | "pinned" : " 29 GB" | 20260505 21:52:19.134 | INFO | } | 20260505 21:52:19.134 | INFO | Perf report for the last 5.13894 seconds: | 20260505 21:52:19.134 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:52:19.134 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:19.134 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:19.134 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:19.134 | INFO | ANALYSIS : 62.0%, 3.0, 96.3ms, 0 | 20260505 21:52:19.134 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:19.134 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:19.134 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:52:19.134 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:19.134 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:19.134 | INFO | Uploaded 36GiB to GPU in 2.13831s (16.8357GiB/s) | 20260505 21:52:19.135 | INFO | Downloaded 34GiB to GPU in 2.05494s (16.5455GiB/s) | 20260505 21:52:24.276 | INFO | Analyzing chunk frames = [6580224,6580736) | 20260505 21:52:24.276 | INFO | MemInfo: { | 20260505 21:52:24.276 | INFO | "gpu" : " 17 GB", | 20260505 21:52:24.276 | INFO | "huge" : "", | 20260505 21:52:24.276 | INFO | "huge_shared" : " 27 GB", | 20260505 21:52:24.276 | INFO | "malloc" : " 14 GB", | 20260505 21:52:24.276 | INFO | "pinned" : " 29 GB" | 20260505 21:52:24.276 | INFO | } | 20260505 21:52:24.276 | INFO | Perf report for the last 5.14224 seconds: | 20260505 21:52:24.276 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:52:24.276 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:24.276 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:24.276 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:24.276 | INFO | ANALYSIS : 61.6%, 3.0, 96.2ms, 0 | 20260505 21:52:24.276 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:24.276 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:24.276 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 21:52:24.276 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:24.276 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:24.277 | INFO | Uploaded 36GiB to GPU in 2.1318s (16.8872GiB/s) | 20260505 21:52:24.277 | INFO | Downloaded 34GiB to GPU in 2.05981s (16.5064GiB/s) | 20260505 21:52:29.411 | INFO | Analyzing chunk frames = [6580736,6581248) | 20260505 21:52:29.411 | INFO | MemInfo: { | 20260505 21:52:29.411 | INFO | "gpu" : " 17 GB", | 20260505 21:52:29.411 | INFO | "huge" : "", | 20260505 21:52:29.411 | INFO | "huge_shared" : " 27 GB", | 20260505 21:52:29.411 | INFO | "malloc" : " 14 GB", | 20260505 21:52:29.411 | INFO | "pinned" : " 29 GB" | 20260505 21:52:29.411 | INFO | } | 20260505 21:52:29.411 | INFO | Perf report for the last 5.13496 seconds: | 20260505 21:52:29.412 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:52:29.412 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:29.412 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:29.412 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:29.412 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260505 21:52:29.412 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:29.412 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:29.412 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260505 21:52:29.412 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:29.412 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:29.412 | INFO | Uploaded 36GiB to GPU in 2.08148s (17.2954GiB/s) | 20260505 21:52:29.412 | INFO | Downloaded 34GiB to GPU in 2.03215s (16.731GiB/s) | 20260505 21:52:34.550 | INFO | Analyzing chunk frames = [6581248,6581760) | 20260505 21:52:34.550 | INFO | MemInfo: { | 20260505 21:52:34.550 | INFO | "gpu" : " 17 GB", | 20260505 21:52:34.550 | INFO | "huge" : "", | 20260505 21:52:34.550 | INFO | "huge_shared" : " 27 GB", | 20260505 21:52:34.550 | INFO | "malloc" : " 14 GB", | 20260505 21:52:34.550 | INFO | "pinned" : " 29 GB" | 20260505 21:52:34.550 | INFO | } | 20260505 21:52:34.550 | INFO | Perf report for the last 5.13898 seconds: | 20260505 21:52:34.550 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:52:34.550 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:34.550 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:34.550 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:34.550 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 21:52:34.550 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:34.550 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:34.550 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 21:52:34.550 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:34.550 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:34.551 | INFO | Uploaded 36GiB to GPU in 2.16104s (16.6586GiB/s) | 20260505 21:52:34.551 | INFO | Downloaded 34GiB to GPU in 2.13273s (15.942GiB/s) | 20260505 21:52:39.686 | INFO | Analyzing chunk frames = [6581760,6582272) | 20260505 21:52:39.686 | INFO | MemInfo: { | 20260505 21:52:39.686 | INFO | "gpu" : " 17 GB", | 20260505 21:52:39.686 | INFO | "huge" : "", | 20260505 21:52:39.686 | INFO | "huge_shared" : " 27 GB", | 20260505 21:52:39.686 | INFO | "malloc" : " 14 GB", | 20260505 21:52:39.686 | INFO | "pinned" : " 29 GB" | 20260505 21:52:39.686 | INFO | } | 20260505 21:52:39.686 | INFO | Perf report for the last 5.13619 seconds: | 20260505 21:52:39.687 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:52:39.687 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:39.687 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:39.687 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:39.687 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 21:52:39.687 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:39.687 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:39.687 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 21:52:39.687 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:39.687 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:39.687 | INFO | Uploaded 36GiB to GPU in 2.1071s (17.0851GiB/s) | 20260505 21:52:39.687 | INFO | Downloaded 34GiB to GPU in 2.00018s (16.9985GiB/s) | 20260505 21:52:39.819 | INFO | Starting a round of Baz IO | 20260505 21:52:42.986 | INFO | Populating RTMetrics report | 20260505 21:52:42.987 | INFO | RTMetrics report populated | 20260505 21:52:42.987 | INFO | Opening tmp RTMetrics json file | 20260505 21:52:42.989 | INFO | Serializing RTMetrics to json | 20260505 21:52:42.991 | INFO | Serialization done, swapping tmp file now... | 20260505 21:52:42.991 | INFO | RTMetrics done | 20260505 21:52:43.067 | INFO | Finished a round of Baz IO: Wrote 1.95 GiB in 3.248 seconds | 20260505 21:52:43.067 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.54% / 0.00% / 78.46% | 20260505 21:52:43.067 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.136395% / 0% / 9.93411e-05% | 20260505 21:52:43.067 | INFO | Baz IO overall: | 20260505 21:52:43.067 | INFO | BytesWritten : 3802951683254 | 20260505 21:52:43.067 | INFO | NumEvents : 1794903994602 | 20260505 21:52:44.826 | INFO | Analyzing chunk frames = [6582272,6582784) | 20260505 21:52:44.826 | INFO | MemInfo: { | 20260505 21:52:44.826 | INFO | "gpu" : " 17 GB", | 20260505 21:52:44.826 | INFO | "huge" : "", | 20260505 21:52:44.826 | INFO | "huge_shared" : " 27 GB", | 20260505 21:52:44.826 | INFO | "malloc" : " 14 GB", | 20260505 21:52:44.826 | INFO | "pinned" : " 29 GB" | 20260505 21:52:44.826 | INFO | } | 20260505 21:52:44.826 | INFO | Perf report for the last 5.13991 seconds: | 20260505 21:52:44.827 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:52:44.827 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:44.827 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:44.827 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:44.827 | INFO | ANALYSIS : 63.2%, 3.0, 98.5ms, 0 | 20260505 21:52:44.827 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:44.827 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260505 21:52:44.827 | INFO | PRE_HQ : 59.7%, 1.8, 54.4ms, 0 | 20260505 21:52:44.827 | INFO | BAZWRITER : 62.7%, 3.6, 116.7ms, 0 | 20260505 21:52:44.827 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 21:52:44.827 | INFO | Uploaded 36GiB to GPU in 2.15813s (16.6811GiB/s) | 20260505 21:52:44.827 | INFO | Downloaded 38GiB to GPU in 2.24449s (16.9304GiB/s) | 20260505 21:52:49.973 | INFO | Analyzing chunk frames = [6582784,6583296) | 20260505 21:52:49.973 | INFO | MemInfo: { | 20260505 21:52:49.973 | INFO | "gpu" : " 17 GB", | 20260505 21:52:49.973 | INFO | "huge" : "", | 20260505 21:52:49.973 | INFO | "huge_shared" : " 27 GB", | 20260505 21:52:49.973 | INFO | "malloc" : " 14 GB", | 20260505 21:52:49.973 | INFO | "pinned" : " 29 GB" | 20260505 21:52:49.973 | INFO | } | 20260505 21:52:49.973 | INFO | Perf report for the last 5.1463 seconds: | 20260505 21:52:49.973 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:52:49.973 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:49.973 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:49.973 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:49.973 | INFO | ANALYSIS : 61.8%, 3.0, 96.7ms, 0 | 20260505 21:52:49.973 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:49.973 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:49.973 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 21:52:49.973 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:49.973 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:49.973 | INFO | Uploaded 36GiB to GPU in 2.07947s (17.3121GiB/s) | 20260505 21:52:49.973 | INFO | Downloaded 34GiB to GPU in 2.02457s (16.7937GiB/s) | 20260505 21:52:55.118 | INFO | Analyzing chunk frames = [6583296,6583808) | 20260505 21:52:55.118 | INFO | MemInfo: { | 20260505 21:52:55.118 | INFO | "gpu" : " 17 GB", | 20260505 21:52:55.118 | INFO | "huge" : "", | 20260505 21:52:55.118 | INFO | "huge_shared" : " 27 GB", | 20260505 21:52:55.118 | INFO | "malloc" : " 14 GB", | 20260505 21:52:55.118 | INFO | "pinned" : " 29 GB" | 20260505 21:52:55.118 | INFO | } | 20260505 21:52:55.119 | INFO | Perf report for the last 5.146 seconds: | 20260505 21:52:55.119 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:52:55.119 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:55.119 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:55.119 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:55.119 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260505 21:52:55.119 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:55.119 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:55.119 | INFO | PRE_HQ : 26.8%, 1.0, 14.0ms, 0 | 20260505 21:52:55.119 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:55.119 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:52:55.119 | INFO | Uploaded 36GiB to GPU in 2.14576s (16.7773GiB/s) | 20260505 21:52:55.119 | INFO | Downloaded 34GiB to GPU in 2.03737s (16.6882GiB/s) | 20260505 21:53:00.250 | INFO | Analyzing chunk frames = [6583808,6584320) | 20260505 21:53:00.250 | INFO | MemInfo: { | 20260505 21:53:00.250 | INFO | "gpu" : " 17 GB", | 20260505 21:53:00.250 | INFO | "huge" : "", | 20260505 21:53:00.250 | INFO | "huge_shared" : " 27 GB", | 20260505 21:53:00.250 | INFO | "malloc" : " 14 GB", | 20260505 21:53:00.250 | INFO | "pinned" : " 29 GB" | 20260505 21:53:00.250 | INFO | } | 20260505 21:53:00.250 | INFO | Perf report for the last 5.13137 seconds: | 20260505 21:53:00.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:53:00.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:00.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:00.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:00.250 | INFO | ANALYSIS : 61.8%, 3.0, 95.8ms, 0 | 20260505 21:53:00.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:00.250 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:00.250 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:53:00.250 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:00.250 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:00.251 | INFO | Uploaded 36GiB to GPU in 2.12281s (16.9586GiB/s) | 20260505 21:53:00.251 | INFO | Downloaded 34GiB to GPU in 2.0041s (16.9653GiB/s) | 20260505 21:53:05.382 | INFO | Analyzing chunk frames = [6584320,6584832) | 20260505 21:53:05.382 | INFO | MemInfo: { | 20260505 21:53:05.382 | INFO | "gpu" : " 17 GB", | 20260505 21:53:05.382 | INFO | "huge" : "", | 20260505 21:53:05.382 | INFO | "huge_shared" : " 27 GB", | 20260505 21:53:05.382 | INFO | "malloc" : " 14 GB", | 20260505 21:53:05.382 | INFO | "pinned" : " 29 GB" | 20260505 21:53:05.382 | INFO | } | 20260505 21:53:05.382 | INFO | Perf report for the last 5.13184 seconds: | 20260505 21:53:05.382 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:53:05.382 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:05.382 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:05.382 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:05.382 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260505 21:53:05.382 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:05.382 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:05.382 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 21:53:05.382 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:05.382 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:05.382 | INFO | Uploaded 36GiB to GPU in 2.13532s (16.8593GiB/s) | 20260505 21:53:05.382 | INFO | Downloaded 34GiB to GPU in 2.05868s (16.5154GiB/s) | 20260505 21:53:10.527 | INFO | Analyzing chunk frames = [6584832,6585344) | 20260505 21:53:10.527 | INFO | MemInfo: { | 20260505 21:53:10.527 | INFO | "gpu" : " 17 GB", | 20260505 21:53:10.527 | INFO | "huge" : "", | 20260505 21:53:10.527 | INFO | "huge_shared" : " 27 GB", | 20260505 21:53:10.528 | INFO | "malloc" : " 14 GB", | 20260505 21:53:10.528 | INFO | "pinned" : " 29 GB" | 20260505 21:53:10.528 | INFO | } | 20260505 21:53:10.528 | INFO | Perf report for the last 5.14555 seconds: | 20260505 21:53:10.528 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:53:10.528 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:10.528 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:10.528 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:10.528 | INFO | ANALYSIS : 61.9%, 3.0, 96.8ms, 0 | 20260505 21:53:10.528 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:10.528 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:10.528 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 21:53:10.528 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:10.528 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:10.528 | INFO | Uploaded 36GiB to GPU in 2.15183s (16.73GiB/s) | 20260505 21:53:10.528 | INFO | Downloaded 34GiB to GPU in 2.05815s (16.5197GiB/s) | 20260505 21:53:15.666 | INFO | Analyzing chunk frames = [6585344,6585856) | 20260505 21:53:15.666 | INFO | MemInfo: { | 20260505 21:53:15.666 | INFO | "gpu" : " 17 GB", | 20260505 21:53:15.666 | INFO | "huge" : "", | 20260505 21:53:15.666 | INFO | "huge_shared" : " 27 GB", | 20260505 21:53:15.666 | INFO | "malloc" : " 14 GB", | 20260505 21:53:15.666 | INFO | "pinned" : " 29 GB" | 20260505 21:53:15.666 | INFO | } | 20260505 21:53:15.666 | INFO | Perf report for the last 5.13886 seconds: | 20260505 21:53:15.666 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:53:15.666 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:15.666 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:15.667 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:15.667 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 21:53:15.667 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:15.667 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:15.667 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 21:53:15.667 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:15.667 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:15.667 | INFO | Uploaded 36GiB to GPU in 2.15869s (16.6768GiB/s) | 20260505 21:53:15.667 | INFO | Downloaded 34GiB to GPU in 2.12336s (16.0124GiB/s) | 20260505 21:53:20.811 | INFO | Analyzing chunk frames = [6585856,6586368) | 20260505 21:53:20.811 | INFO | MemInfo: { | 20260505 21:53:20.811 | INFO | "gpu" : " 17 GB", | 20260505 21:53:20.811 | INFO | "huge" : "", | 20260505 21:53:20.811 | INFO | "huge_shared" : " 27 GB", | 20260505 21:53:20.811 | INFO | "malloc" : " 14 GB", | 20260505 21:53:20.812 | INFO | "pinned" : " 29 GB" | 20260505 21:53:20.812 | INFO | } | 20260505 21:53:20.812 | INFO | Perf report for the last 5.14526 seconds: | 20260505 21:53:20.812 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:53:20.812 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:20.812 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:20.812 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:20.812 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260505 21:53:20.812 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:20.812 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:20.812 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:53:20.812 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:20.812 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:20.812 | INFO | Uploaded 36GiB to GPU in 2.1055s (17.0981GiB/s) | 20260505 21:53:20.812 | INFO | Downloaded 34GiB to GPU in 1.99742s (17.0219GiB/s) | 20260505 21:53:20.812 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 21:53:20.812 | INFO | Total mem usage: 15.3 GiB | 20260505 21:53:20.812 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 21:53:20.812 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.86%) | 20260505 21:53:20.812 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 21:53:20.812 | INFO | High Water Breakdown: | 20260505 21:53:20.812 | INFO | 60.9% : 8.57 GiB ( 90% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 21:53:20.812 | INFO | 39% : 5.49 GiB ( 82.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 21:53:20.812 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 21:53:20.812 | INFO | ------------------------------------------ | 20260505 21:53:20.812 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:53:20.812 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 21:53:20.812 | INFO | Total mem usage: 26.5 GiB | 20260505 21:53:20.812 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 21:53:20.812 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 21:53:20.812 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 21:53:20.812 | INFO | High Water Breakdown: | 20260505 21:53:20.812 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:53:20.812 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:53:20.812 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:53:20.812 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:53:20.812 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:53:20.812 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:53:20.812 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 21:53:20.812 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 21:53:20.812 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:53:20.812 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 21:53:20.812 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 21:53:20.812 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 21:53:20.812 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 21:53:20.812 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 21:53:20.812 | INFO | ------------------------------------------ | 20260505 21:53:20.813 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:53:20.813 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 21:53:20.813 | INFO | Total mem usage: 25.6 GiB | 20260505 21:53:20.813 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 21:53:20.813 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 21:53:20.813 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 21:53:20.813 | INFO | High Water Breakdown: | 20260505 21:53:20.813 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 21:53:20.813 | INFO | ------------------------------------------ | 20260505 21:53:20.813 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:53:20.813 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 21:53:20.813 | INFO | Total mem usage: 15.7 GiB | 20260505 21:53:20.813 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 21:53:20.813 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 21:53:20.813 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 21:53:20.813 | INFO | High Water Breakdown: | 20260505 21:53:20.813 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 21:53:20.813 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 21:53:20.813 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 21:53:20.813 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 21:53:20.813 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 21:53:20.813 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 21:53:20.813 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 21:53:20.813 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 21:53:20.813 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 21:53:20.813 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:53:20.813 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 21:53:20.813 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:53:20.813 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 21:53:20.813 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 21:53:20.813 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 21:53:20.813 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 21:53:20.813 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:53:20.813 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 21:53:20.813 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:53:20.813 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:53:20.813 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:53:20.813 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 21:53:20.813 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 21:53:20.813 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:53:20.813 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 21:53:20.813 | INFO | ------------------------------------------ | 20260505 21:53:20.813 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:53:20.813 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 21:53:20.813 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:53:20.813 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:53:20.813 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.40e+01, 1.33e+01 / 1.33e+00 | 20260505 21:53:20.813 | INFO | PerfReport: 0.29%, AggregateStats -- 9.24e-02 / 1.04e-01, 1.07e-02 / 6.83e-03 | 20260505 21:53:20.813 | INFO | PerfReport: 11.81%, Baseline -- 3.71e+00 / 4.56e+00, 5.72e-01 / 1.51e-01 | 20260505 21:53:20.813 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.52e+00, 1.89e-01 / 9.49e-02 | 20260505 21:53:20.813 | INFO | PerfReport: 15.95%, DME -- 4.01e+01 / 4.39e+01, 3.90e+00 / 1.31e+00 | 20260505 21:53:20.813 | INFO | PerfReport: 57.20%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.50e+00 / 1.05e+00 | 20260505 21:53:20.813 | INFO | PerfReport: 7.18%, PulseAccumulating -- 2.25e+00 / 2.81e+00, 4.30e-01 / 1.93e-01 | 20260505 21:53:20.813 | INFO | PerfReport: 3.18%, Metrics -- 9.99e-01 / 1.31e+00, 2.73e-01 / 1.51e-01 | 20260505 21:53:20.813 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.47e-01 / 4.05e-01, 5.98e-02 / 3.69e-02 | 20260505 21:53:20.813 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 21:53:20.813 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:53:20.813 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:53:20.813 | INFO | PerfReport: Full Execution -- 9.89e+01 / -nan, 2.51e+01 / 0.00e+00 | 20260505 21:53:20.813 | INFO | PerfReport: 22.43%, Upload -- 2.22e+01 / 0.00e+00, 7.67e+00 / 0.00e+00 | 20260505 21:53:20.813 | INFO | PerfReport: 21.82%, Download -- 2.16e+01 / 0.00e+00, 7.70e+00 / 0.00e+00 | 20260505 21:53:20.813 | INFO | PerfReport: 31.73%, Compute -- 3.14e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260505 21:53:20.813 | INFO | PerfReport: 2.20%, LockUpload -- 2.18e+00 / 0.00e+00, 7.91e+00 / 0.00e+00 | 20260505 21:53:20.813 | INFO | PerfReport: 1.35%, LockDownload -- 1.34e+00 / 0.00e+00, 2.89e+00 / 0.00e+00 | 20260505 21:53:20.813 | INFO | PerfReport: 20.45%, LockCompute -- 2.02e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 21:53:20.813 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.68e-02 / 0.00e+00 | 20260505 21:53:20.937 | INFO | Starting a round of Baz IO | 20260505 21:53:24.089 | INFO | Populating RTMetrics report | 20260505 21:53:24.089 | INFO | RTMetrics report populated | 20260505 21:53:24.089 | INFO | Opening tmp RTMetrics json file | 20260505 21:53:24.091 | INFO | Serializing RTMetrics to json | 20260505 21:53:24.095 | INFO | Serialization done, swapping tmp file now... | 20260505 21:53:24.095 | INFO | RTMetrics done | 20260505 21:53:24.177 | INFO | Finished a round of Baz IO: Wrote 1.951 GiB in 3.239 seconds | 20260505 21:53:24.177 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.54% / 0.00% / 78.46% | 20260505 21:53:24.177 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.135922% / 3.97364e-06% / 9.53674e-05% | 20260505 21:53:24.177 | INFO | Baz IO overall: | 20260505 21:53:24.177 | INFO | BytesWritten : 3805046158672 | 20260505 21:53:24.177 | INFO | NumEvents : 1795869231829 | 20260505 21:53:25.946 | INFO | Analyzing chunk frames = [6586368,6586880) | 20260505 21:53:25.946 | INFO | MemInfo: { | 20260505 21:53:25.946 | INFO | "gpu" : " 17 GB", | 20260505 21:53:25.946 | INFO | "huge" : "", | 20260505 21:53:25.946 | INFO | "huge_shared" : " 27 GB", | 20260505 21:53:25.946 | INFO | "malloc" : " 14 GB", | 20260505 21:53:25.946 | INFO | "pinned" : " 29 GB" | 20260505 21:53:25.946 | INFO | } | 20260505 21:53:25.946 | INFO | Perf report for the last 5.13454 seconds: | 20260505 21:53:25.946 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:53:25.946 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:25.946 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:25.946 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:25.946 | INFO | ANALYSIS : 62.8%, 3.0, 97.8ms, 0 | 20260505 21:53:25.946 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:25.946 | INFO | RT_METRICS : 43.8%, 1.0, 22.9ms, 0 | 20260505 21:53:25.946 | INFO | PRE_HQ : 60.0%, 1.8, 54.5ms, 0 | 20260505 21:53:25.946 | INFO | BAZWRITER : 62.3%, 3.6, 116.3ms, 0 | 20260505 21:53:25.946 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 21:53:25.947 | INFO | Uploaded 36GiB to GPU in 2.16834s (16.6026GiB/s) | 20260505 21:53:25.947 | INFO | Downloaded 38GiB to GPU in 2.2433s (16.9393GiB/s) | 20260505 21:53:31.081 | INFO | Analyzing chunk frames = [6586880,6587392) | 20260505 21:53:31.081 | INFO | MemInfo: { | 20260505 21:53:31.081 | INFO | "gpu" : " 17 GB", | 20260505 21:53:31.081 | INFO | "huge" : "", | 20260505 21:53:31.081 | INFO | "huge_shared" : " 27 GB", | 20260505 21:53:31.081 | INFO | "malloc" : " 14 GB", | 20260505 21:53:31.081 | INFO | "pinned" : " 29 GB" | 20260505 21:53:31.081 | INFO | } | 20260505 21:53:31.081 | INFO | Perf report for the last 5.13496 seconds: | 20260505 21:53:31.081 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:53:31.081 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:31.081 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:31.081 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:31.081 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260505 21:53:31.081 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:31.081 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:31.081 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 21:53:31.081 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:31.081 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:31.082 | INFO | Uploaded 36GiB to GPU in 2.09017s (17.2235GiB/s) | 20260505 21:53:31.082 | INFO | Downloaded 34GiB to GPU in 2.02736s (16.7705GiB/s) | 20260505 21:53:36.224 | INFO | Analyzing chunk frames = [6587392,6587904) | 20260505 21:53:36.224 | INFO | MemInfo: { | 20260505 21:53:36.224 | INFO | "gpu" : " 17 GB", | 20260505 21:53:36.224 | INFO | "huge" : "", | 20260505 21:53:36.224 | INFO | "huge_shared" : " 27 GB", | 20260505 21:53:36.224 | INFO | "malloc" : " 14 GB", | 20260505 21:53:36.224 | INFO | "pinned" : " 29 GB" | 20260505 21:53:36.224 | INFO | } | 20260505 21:53:36.224 | INFO | Perf report for the last 5.14295 seconds: | 20260505 21:53:36.224 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:53:36.224 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:36.224 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:36.224 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:36.224 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260505 21:53:36.224 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:36.224 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:36.224 | INFO | PRE_HQ : 27.0%, 1.0, 14.1ms, 0 | 20260505 21:53:36.224 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:36.224 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:36.224 | INFO | Uploaded 36GiB to GPU in 2.12706s (16.9248GiB/s) | 20260505 21:53:36.225 | INFO | Downloaded 34GiB to GPU in 2.01826s (16.8462GiB/s) | 20260505 21:53:41.366 | INFO | Analyzing chunk frames = [6587904,6588416) | 20260505 21:53:41.366 | INFO | MemInfo: { | 20260505 21:53:41.366 | INFO | "gpu" : " 17 GB", | 20260505 21:53:41.366 | INFO | "huge" : "", | 20260505 21:53:41.366 | INFO | "huge_shared" : " 27 GB", | 20260505 21:53:41.366 | INFO | "malloc" : " 14 GB", | 20260505 21:53:41.366 | INFO | "pinned" : " 29 GB" | 20260505 21:53:41.367 | INFO | } | 20260505 21:53:41.367 | INFO | Perf report for the last 5.14248 seconds: | 20260505 21:53:41.367 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:53:41.367 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:41.367 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:41.367 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:41.367 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260505 21:53:41.367 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:41.367 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:41.367 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 21:53:41.367 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:41.367 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:41.367 | INFO | Uploaded 36GiB to GPU in 2.13896s (16.8306GiB/s) | 20260505 21:53:41.367 | INFO | Downloaded 34GiB to GPU in 2.02775s (16.7673GiB/s) | 20260505 21:53:46.497 | INFO | Analyzing chunk frames = [6588416,6588928) | 20260505 21:53:46.497 | INFO | MemInfo: { | 20260505 21:53:46.497 | INFO | "gpu" : " 17 GB", | 20260505 21:53:46.497 | INFO | "huge" : "", | 20260505 21:53:46.497 | INFO | "huge_shared" : " 27 GB", | 20260505 21:53:46.497 | INFO | "malloc" : " 14 GB", | 20260505 21:53:46.497 | INFO | "pinned" : " 29 GB" | 20260505 21:53:46.497 | INFO | } | 20260505 21:53:46.497 | INFO | Perf report for the last 5.13052 seconds: | 20260505 21:53:46.497 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:53:46.497 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:46.497 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:46.497 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:46.497 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260505 21:53:46.497 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:46.497 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:46.497 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:53:46.497 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:46.497 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:46.498 | INFO | Uploaded 36GiB to GPU in 2.08538s (17.263GiB/s) | 20260505 21:53:46.498 | INFO | Downloaded 34GiB to GPU in 2.03935s (16.672GiB/s) | 20260505 21:53:51.636 | INFO | Analyzing chunk frames = [6588928,6589440) | 20260505 21:53:51.636 | INFO | MemInfo: { | 20260505 21:53:51.636 | INFO | "gpu" : " 17 GB", | 20260505 21:53:51.636 | INFO | "huge" : "", | 20260505 21:53:51.636 | INFO | "huge_shared" : " 27 GB", | 20260505 21:53:51.636 | INFO | "malloc" : " 14 GB", | 20260505 21:53:51.636 | INFO | "pinned" : " 29 GB" | 20260505 21:53:51.636 | INFO | } | 20260505 21:53:51.636 | INFO | Perf report for the last 5.13893 seconds: | 20260505 21:53:51.636 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:53:51.636 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:51.636 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:51.636 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:51.636 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 21:53:51.636 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:51.636 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:51.636 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 21:53:51.636 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:51.636 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:51.636 | INFO | Uploaded 36GiB to GPU in 2.16228s (16.6491GiB/s) | 20260505 21:53:51.637 | INFO | Downloaded 34GiB to GPU in 2.06834s (16.4383GiB/s) | 20260505 21:53:56.783 | INFO | Analyzing chunk frames = [6589440,6589952) | 20260505 21:53:56.783 | INFO | MemInfo: { | 20260505 21:53:56.783 | INFO | "gpu" : " 17 GB", | 20260505 21:53:56.783 | INFO | "huge" : "", | 20260505 21:53:56.783 | INFO | "huge_shared" : " 27 GB", | 20260505 21:53:56.783 | INFO | "malloc" : " 14 GB", | 20260505 21:53:56.783 | INFO | "pinned" : " 29 GB" | 20260505 21:53:56.783 | INFO | } | 20260505 21:53:56.784 | INFO | Perf report for the last 5.14737 seconds: | 20260505 21:53:56.784 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:53:56.784 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:56.784 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:56.784 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:56.784 | INFO | ANALYSIS : 62.9%, 3.0, 98.4ms, 0 | 20260505 21:53:56.784 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:56.784 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:56.784 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 21:53:56.784 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:56.784 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:53:56.784 | INFO | Uploaded 36GiB to GPU in 2.12939s (16.9063GiB/s) | 20260505 21:53:56.784 | INFO | Downloaded 34GiB to GPU in 2.13716s (15.9089GiB/s) | 20260505 21:54:01.925 | INFO | Analyzing chunk frames = [6589952,6590464) | 20260505 21:54:01.925 | INFO | MemInfo: { | 20260505 21:54:01.925 | INFO | "gpu" : " 17 GB", | 20260505 21:54:01.925 | INFO | "huge" : "", | 20260505 21:54:01.925 | INFO | "huge_shared" : " 27 GB", | 20260505 21:54:01.925 | INFO | "malloc" : " 14 GB", | 20260505 21:54:01.925 | INFO | "pinned" : " 29 GB" | 20260505 21:54:01.925 | INFO | } | 20260505 21:54:01.925 | INFO | Perf report for the last 5.14149 seconds: | 20260505 21:54:01.925 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:54:01.925 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:01.925 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:01.925 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:01.925 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 21:54:01.925 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:01.925 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:01.925 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:54:01.925 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:01.925 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:01.925 | INFO | Uploaded 36GiB to GPU in 2.109s (17.0697GiB/s) | 20260505 21:54:01.925 | INFO | Downloaded 34GiB to GPU in 1.99954s (17.0039GiB/s) | 20260505 21:54:02.049 | INFO | Starting a round of Baz IO | 20260505 21:54:05.200 | INFO | Populating RTMetrics report | 20260505 21:54:05.201 | INFO | RTMetrics report populated | 20260505 21:54:05.201 | INFO | Opening tmp RTMetrics json file | 20260505 21:54:05.203 | INFO | Serializing RTMetrics to json | 20260505 21:54:05.204 | INFO | Serialization done, swapping tmp file now... | 20260505 21:54:05.205 | INFO | RTMetrics done | 20260505 21:54:05.280 | INFO | Finished a round of Baz IO: Wrote 1.953 GiB in 3.231 seconds | 20260505 21:54:05.280 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.54% / 0.00% / 78.46% | 20260505 21:54:05.280 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.136685% / 3.97364e-06% / 7.94729e-05% | 20260505 21:54:05.280 | INFO | Baz IO overall: | 20260505 21:54:05.280 | INFO | BytesWritten : 3807143320355 | 20260505 21:54:05.280 | INFO | NumEvents : 1796836560707 | 20260505 21:54:07.061 | INFO | Analyzing chunk frames = [6590464,6590976) | 20260505 21:54:07.061 | INFO | MemInfo: { | 20260505 21:54:07.061 | INFO | "gpu" : " 17 GB", | 20260505 21:54:07.061 | INFO | "huge" : "", | 20260505 21:54:07.061 | INFO | "huge_shared" : " 27 GB", | 20260505 21:54:07.061 | INFO | "malloc" : " 14 GB", | 20260505 21:54:07.061 | INFO | "pinned" : " 29 GB" | 20260505 21:54:07.061 | INFO | } | 20260505 21:54:07.061 | INFO | Perf report for the last 5.13631 seconds: | 20260505 21:54:07.061 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:54:07.061 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:07.061 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:07.061 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:07.061 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 21:54:07.061 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:07.061 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260505 21:54:07.061 | INFO | PRE_HQ : 60.1%, 1.8, 54.6ms, 0 | 20260505 21:54:07.061 | INFO | BAZWRITER : 62.2%, 3.6, 115.4ms, 0 | 20260505 21:54:07.061 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 21:54:07.062 | INFO | Uploaded 36GiB to GPU in 2.16615s (16.6193GiB/s) | 20260505 21:54:07.062 | INFO | Downloaded 38GiB to GPU in 2.23657s (16.9903GiB/s) | 20260505 21:54:12.198 | INFO | Analyzing chunk frames = [6590976,6591488) | 20260505 21:54:12.198 | INFO | MemInfo: { | 20260505 21:54:12.198 | INFO | "gpu" : " 17 GB", | 20260505 21:54:12.198 | INFO | "huge" : "", | 20260505 21:54:12.199 | INFO | "huge_shared" : " 27 GB", | 20260505 21:54:12.199 | INFO | "malloc" : " 14 GB", | 20260505 21:54:12.199 | INFO | "pinned" : " 29 GB" | 20260505 21:54:12.199 | INFO | } | 20260505 21:54:12.199 | INFO | Perf report for the last 5.13724 seconds: | 20260505 21:54:12.199 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:54:12.199 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:12.199 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:12.199 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:12.199 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 21:54:12.199 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:12.199 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:12.199 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 21:54:12.199 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:12.199 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:12.199 | INFO | Uploaded 36GiB to GPU in 2.08159s (17.2945GiB/s) | 20260505 21:54:12.199 | INFO | Downloaded 34GiB to GPU in 2.0245s (16.7942GiB/s) | 20260505 21:54:17.338 | INFO | Analyzing chunk frames = [6591488,6592000) | 20260505 21:54:17.338 | INFO | MemInfo: { | 20260505 21:54:17.338 | INFO | "gpu" : " 17 GB", | 20260505 21:54:17.338 | INFO | "huge" : "", | 20260505 21:54:17.338 | INFO | "huge_shared" : " 27 GB", | 20260505 21:54:17.338 | INFO | "malloc" : " 14 GB", | 20260505 21:54:17.338 | INFO | "pinned" : " 29 GB" | 20260505 21:54:17.338 | INFO | } | 20260505 21:54:17.338 | INFO | Perf report for the last 5.13952 seconds: | 20260505 21:54:17.338 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:54:17.338 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:17.338 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:17.338 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:17.338 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260505 21:54:17.338 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:17.338 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:17.338 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:54:17.338 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:17.338 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:17.338 | INFO | Uploaded 36GiB to GPU in 2.13336s (16.8747GiB/s) | 20260505 21:54:17.339 | INFO | Downloaded 34GiB to GPU in 2.01919s (16.8384GiB/s) | 20260505 21:54:22.480 | INFO | Analyzing chunk frames = [6592000,6592512) | 20260505 21:54:22.480 | INFO | MemInfo: { | 20260505 21:54:22.480 | INFO | "gpu" : " 17 GB", | 20260505 21:54:22.480 | INFO | "huge" : "", | 20260505 21:54:22.480 | INFO | "huge_shared" : " 27 GB", | 20260505 21:54:22.480 | INFO | "malloc" : " 14 GB", | 20260505 21:54:22.480 | INFO | "pinned" : " 29 GB" | 20260505 21:54:22.480 | INFO | } | 20260505 21:54:22.480 | INFO | Perf report for the last 5.14195 seconds: | 20260505 21:54:22.480 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:54:22.480 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:22.480 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:22.480 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:22.480 | INFO | ANALYSIS : 62.0%, 3.0, 96.3ms, 0 | 20260505 21:54:22.480 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:22.480 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:22.480 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:54:22.480 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:22.480 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:22.480 | INFO | Uploaded 36GiB to GPU in 2.13487s (16.8629GiB/s) | 20260505 21:54:22.480 | INFO | Downloaded 34GiB to GPU in 2.04387s (16.6351GiB/s) | 20260505 21:54:27.624 | INFO | Analyzing chunk frames = [6592512,6593024) | 20260505 21:54:27.624 | INFO | MemInfo: { | 20260505 21:54:27.624 | INFO | "gpu" : " 17 GB", | 20260505 21:54:27.624 | INFO | "huge" : "", | 20260505 21:54:27.624 | INFO | "huge_shared" : " 27 GB", | 20260505 21:54:27.624 | INFO | "malloc" : " 14 GB", | 20260505 21:54:27.624 | INFO | "pinned" : " 29 GB" | 20260505 21:54:27.624 | INFO | } | 20260505 21:54:27.624 | INFO | Perf report for the last 5.14426 seconds: | 20260505 21:54:27.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:54:27.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:27.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:27.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:27.624 | INFO | ANALYSIS : 61.1%, 3.0, 95.5ms, 0 | 20260505 21:54:27.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:27.625 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:27.625 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 21:54:27.625 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:27.625 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:27.625 | INFO | Uploaded 36GiB to GPU in 2.093s (17.2002GiB/s) | 20260505 21:54:27.625 | INFO | Downloaded 34GiB to GPU in 2.04619s (16.6162GiB/s) | 20260505 21:54:32.760 | INFO | Analyzing chunk frames = [6593024,6593536) | 20260505 21:54:32.760 | INFO | MemInfo: { | 20260505 21:54:32.760 | INFO | "gpu" : " 17 GB", | 20260505 21:54:32.760 | INFO | "huge" : "", | 20260505 21:54:32.760 | INFO | "huge_shared" : " 27 GB", | 20260505 21:54:32.760 | INFO | "malloc" : " 14 GB", | 20260505 21:54:32.760 | INFO | "pinned" : " 29 GB" | 20260505 21:54:32.760 | INFO | } | 20260505 21:54:32.760 | INFO | Perf report for the last 5.13591 seconds: | 20260505 21:54:32.760 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:54:32.760 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:32.760 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:32.760 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:32.760 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260505 21:54:32.760 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:32.760 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:32.760 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 21:54:32.760 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:32.760 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:32.761 | INFO | Uploaded 36GiB to GPU in 2.08016s (17.3063GiB/s) | 20260505 21:54:32.761 | INFO | Downloaded 34GiB to GPU in 2.04096s (16.6588GiB/s) | 20260505 21:54:37.895 | INFO | Analyzing chunk frames = [6593536,6594048) | 20260505 21:54:37.895 | INFO | MemInfo: { | 20260505 21:54:37.895 | INFO | "gpu" : " 17 GB", | 20260505 21:54:37.895 | INFO | "huge" : "", | 20260505 21:54:37.895 | INFO | "huge_shared" : " 27 GB", | 20260505 21:54:37.895 | INFO | "malloc" : " 14 GB", | 20260505 21:54:37.895 | INFO | "pinned" : " 29 GB" | 20260505 21:54:37.895 | INFO | } | 20260505 21:54:37.896 | INFO | Perf report for the last 5.1354 seconds: | 20260505 21:54:37.896 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:54:37.896 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:37.896 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:37.896 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:37.896 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 21:54:37.896 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:37.896 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:37.896 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 21:54:37.896 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:37.896 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:37.896 | INFO | Uploaded 36GiB to GPU in 2.11353s (17.0331GiB/s) | 20260505 21:54:37.896 | INFO | Downloaded 34GiB to GPU in 2.08849s (16.2797GiB/s) | 20260505 21:54:43.041 | INFO | Analyzing chunk frames = [6594048,6594560) | 20260505 21:54:43.041 | INFO | MemInfo: { | 20260505 21:54:43.041 | INFO | "gpu" : " 17 GB", | 20260505 21:54:43.041 | INFO | "huge" : "", | 20260505 21:54:43.041 | INFO | "huge_shared" : " 27 GB", | 20260505 21:54:43.041 | INFO | "malloc" : " 14 GB", | 20260505 21:54:43.041 | INFO | "pinned" : " 29 GB" | 20260505 21:54:43.041 | INFO | } | 20260505 21:54:43.041 | INFO | Perf report for the last 5.14532 seconds: | 20260505 21:54:43.041 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:54:43.041 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:43.041 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:43.041 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:43.041 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 21:54:43.041 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:43.041 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:43.041 | INFO | PRE_HQ : 26.8%, 1.0, 14.0ms, 0 | 20260505 21:54:43.041 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:43.041 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:43.041 | INFO | Uploaded 36GiB to GPU in 2.10345s (17.1147GiB/s) | 20260505 21:54:43.041 | INFO | Downloaded 34GiB to GPU in 2.00313s (16.9734GiB/s) | 20260505 21:54:43.041 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 21:54:43.041 | INFO | Total mem usage: 15.3 GiB | 20260505 21:54:43.041 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 21:54:43.041 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.46%) | 20260505 21:54:43.041 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 21:54:43.041 | INFO | High Water Breakdown: | 20260505 21:54:43.041 | INFO | 61.1% : 8.63 GiB ( 90.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 21:54:43.041 | INFO | 38.8% : 5.48 GiB ( 82.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 21:54:43.042 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 21:54:43.042 | INFO | ------------------------------------------ | 20260505 21:54:43.042 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:54:43.042 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 21:54:43.042 | INFO | Total mem usage: 26.5 GiB | 20260505 21:54:43.042 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 21:54:43.042 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 21:54:43.042 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260505 21:54:43.042 | INFO | High Water Breakdown: | 20260505 21:54:43.042 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:54:43.042 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:54:43.042 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:54:43.042 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:54:43.042 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:54:43.042 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:54:43.042 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 21:54:43.042 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 21:54:43.042 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:54:43.042 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 21:54:43.042 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 21:54:43.042 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 21:54:43.042 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 21:54:43.042 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 21:54:43.042 | INFO | ------------------------------------------ | 20260505 21:54:43.042 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:54:43.042 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 21:54:43.042 | INFO | Total mem usage: 25.6 GiB | 20260505 21:54:43.042 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 21:54:43.042 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 21:54:43.042 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 21:54:43.042 | INFO | High Water Breakdown: | 20260505 21:54:43.042 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 21:54:43.042 | INFO | ------------------------------------------ | 20260505 21:54:43.042 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:54:43.042 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 21:54:43.042 | INFO | Total mem usage: 15.7 GiB | 20260505 21:54:43.042 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 21:54:43.042 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 21:54:43.042 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 21:54:43.042 | INFO | High Water Breakdown: | 20260505 21:54:43.042 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 21:54:43.042 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 21:54:43.042 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 21:54:43.042 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 21:54:43.042 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 21:54:43.042 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 21:54:43.042 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 21:54:43.042 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 21:54:43.042 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 21:54:43.042 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:54:43.042 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 21:54:43.042 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:54:43.042 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 21:54:43.042 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 21:54:43.042 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 21:54:43.042 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 21:54:43.042 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:54:43.042 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 21:54:43.042 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:54:43.042 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:54:43.042 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:54:43.042 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 21:54:43.042 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 21:54:43.042 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:54:43.042 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 21:54:43.042 | INFO | ------------------------------------------ | 20260505 21:54:43.042 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:54:43.042 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 21:54:43.042 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:54:43.042 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:54:43.042 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.44e+01, 1.35e+01 / 1.78e+00 | 20260505 21:54:43.042 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.03e-01, 1.04e-02 / 5.90e-03 | 20260505 21:54:43.042 | INFO | PerfReport: 11.81%, Baseline -- 3.72e+00 / 4.53e+00, 5.73e-01 / 1.63e-01 | 20260505 21:54:43.042 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.53e+00, 1.90e-01 / 1.01e-01 | 20260505 21:54:43.042 | INFO | PerfReport: 15.99%, DME -- 4.02e+01 / 4.44e+01, 4.09e+00 / 1.86e+00 | 20260505 21:54:43.042 | INFO | PerfReport: 57.19%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.51e+00 / 1.19e+00 | 20260505 21:54:43.042 | INFO | PerfReport: 7.16%, PulseAccumulating -- 2.25e+00 / 2.79e+00, 4.31e-01 / 2.54e-01 | 20260505 21:54:43.042 | INFO | PerfReport: 3.17%, Metrics -- 9.97e-01 / 1.29e+00, 2.71e-01 / 1.32e-01 | 20260505 21:54:43.042 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.47e-01 / 3.99e-01, 5.95e-02 / 3.51e-02 | 20260505 21:54:43.043 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 21:54:43.043 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:54:43.043 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:54:43.043 | INFO | PerfReport: Full Execution -- 9.88e+01 / -nan, 2.49e+01 / 0.00e+00 | 20260505 21:54:43.043 | INFO | PerfReport: 22.35%, Upload -- 2.21e+01 / 0.00e+00, 7.55e+00 / 0.00e+00 | 20260505 21:54:43.043 | INFO | PerfReport: 21.79%, Download -- 2.15e+01 / 0.00e+00, 7.62e+00 / 0.00e+00 | 20260505 21:54:43.043 | INFO | PerfReport: 31.83%, Compute -- 3.15e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 21:54:43.043 | INFO | PerfReport: 2.15%, LockUpload -- 2.13e+00 / 0.00e+00, 7.95e+00 / 0.00e+00 | 20260505 21:54:43.043 | INFO | PerfReport: 1.34%, LockDownload -- 1.33e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260505 21:54:43.043 | INFO | PerfReport: 20.52%, LockCompute -- 2.03e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 21:54:43.043 | INFO | PerfReport: 0.02%, Misc -- 1.50e-02 / 0.00e+00, 3.18e-02 / 0.00e+00 | 20260505 21:54:43.163 | INFO | Starting a round of Baz IO | 20260505 21:54:46.316 | INFO | Populating RTMetrics report | 20260505 21:54:46.317 | INFO | RTMetrics report populated | 20260505 21:54:46.317 | INFO | Opening tmp RTMetrics json file | 20260505 21:54:46.318 | INFO | Serializing RTMetrics to json | 20260505 21:54:46.319 | INFO | Serialization done, swapping tmp file now... | 20260505 21:54:46.319 | INFO | RTMetrics done | 20260505 21:54:46.397 | INFO | Finished a round of Baz IO: Wrote 1.952 GiB in 3.234 seconds | 20260505 21:54:46.397 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.54% / 0.00% / 78.46% | 20260505 21:54:46.398 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.13603% / 0% / 9.93411e-05% | 20260505 21:54:46.398 | INFO | Baz IO overall: | 20260505 21:54:46.398 | INFO | BytesWritten : 3809239495303 | 20260505 21:54:46.398 | INFO | NumEvents : 1797802814825 | 20260505 21:54:48.177 | INFO | Analyzing chunk frames = [6594560,6595072) | 20260505 21:54:48.177 | INFO | MemInfo: { | 20260505 21:54:48.177 | INFO | "gpu" : " 17 GB", | 20260505 21:54:48.177 | INFO | "huge" : "", | 20260505 21:54:48.177 | INFO | "huge_shared" : " 27 GB", | 20260505 21:54:48.177 | INFO | "malloc" : " 14 GB", | 20260505 21:54:48.177 | INFO | "pinned" : " 29 GB" | 20260505 21:54:48.177 | INFO | } | 20260505 21:54:48.177 | INFO | Perf report for the last 5.13565 seconds: | 20260505 21:54:48.177 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:54:48.177 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:48.177 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:48.177 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:48.177 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 21:54:48.177 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:48.177 | INFO | RT_METRICS : 43.8%, 1.0, 22.9ms, 0 | 20260505 21:54:48.177 | INFO | PRE_HQ : 60.2%, 1.8, 54.9ms, 0 | 20260505 21:54:48.177 | INFO | BAZWRITER : 62.2%, 3.6, 116.1ms, 0 | 20260505 21:54:48.177 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 21:54:48.177 | INFO | Uploaded 36GiB to GPU in 2.16385s (16.637GiB/s) | 20260505 21:54:48.177 | INFO | Downloaded 38GiB to GPU in 2.22821s (17.054GiB/s) | 20260505 21:54:53.311 | INFO | Analyzing chunk frames = [6595072,6595584) | 20260505 21:54:53.311 | INFO | MemInfo: { | 20260505 21:54:53.311 | INFO | "gpu" : " 17 GB", | 20260505 21:54:53.311 | INFO | "huge" : "", | 20260505 21:54:53.311 | INFO | "huge_shared" : " 27 GB", | 20260505 21:54:53.311 | INFO | "malloc" : " 14 GB", | 20260505 21:54:53.311 | INFO | "pinned" : " 29 GB" | 20260505 21:54:53.311 | INFO | } | 20260505 21:54:53.311 | INFO | Perf report for the last 5.13454 seconds: | 20260505 21:54:53.311 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:54:53.311 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:53.311 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:53.311 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:53.311 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 21:54:53.311 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:53.311 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:53.311 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260505 21:54:53.311 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:53.311 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:53.312 | INFO | Uploaded 36GiB to GPU in 2.07921s (17.3143GiB/s) | 20260505 21:54:53.312 | INFO | Downloaded 34GiB to GPU in 2.02337s (16.8037GiB/s) | 20260505 21:54:58.448 | INFO | Analyzing chunk frames = [6595584,6596096) | 20260505 21:54:58.448 | INFO | MemInfo: { | 20260505 21:54:58.448 | INFO | "gpu" : " 17 GB", | 20260505 21:54:58.448 | INFO | "huge" : "", | 20260505 21:54:58.448 | INFO | "huge_shared" : " 27 GB", | 20260505 21:54:58.448 | INFO | "malloc" : " 14 GB", | 20260505 21:54:58.448 | INFO | "pinned" : " 29 GB" | 20260505 21:54:58.448 | INFO | } | 20260505 21:54:58.448 | INFO | Perf report for the last 5.13719 seconds: | 20260505 21:54:58.448 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:54:58.448 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:58.448 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:58.448 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:58.449 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260505 21:54:58.449 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:58.449 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:58.449 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:54:58.449 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:58.449 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:54:58.449 | INFO | Uploaded 36GiB to GPU in 2.14338s (16.7959GiB/s) | 20260505 21:54:58.449 | INFO | Downloaded 34GiB to GPU in 2.01923s (16.8381GiB/s) | 20260505 21:55:03.589 | INFO | Analyzing chunk frames = [6596096,6596608) | 20260505 21:55:03.589 | INFO | MemInfo: { | 20260505 21:55:03.589 | INFO | "gpu" : " 17 GB", | 20260505 21:55:03.589 | INFO | "huge" : "", | 20260505 21:55:03.589 | INFO | "huge_shared" : " 27 GB", | 20260505 21:55:03.589 | INFO | "malloc" : " 14 GB", | 20260505 21:55:03.589 | INFO | "pinned" : " 29 GB" | 20260505 21:55:03.589 | INFO | } | 20260505 21:55:03.589 | INFO | Perf report for the last 5.14073 seconds: | 20260505 21:55:03.589 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:55:03.589 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:03.589 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:03.589 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:03.589 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260505 21:55:03.589 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:03.589 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:03.589 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:55:03.589 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:03.589 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:03.589 | INFO | Uploaded 36GiB to GPU in 2.2102s (16.2881GiB/s) | 20260505 21:55:03.590 | INFO | Downloaded 34GiB to GPU in 2.05266s (16.5639GiB/s) | 20260505 21:55:08.732 | INFO | Analyzing chunk frames = [6596608,6597120) | 20260505 21:55:08.732 | INFO | MemInfo: { | 20260505 21:55:08.732 | INFO | "gpu" : " 17 GB", | 20260505 21:55:08.732 | INFO | "huge" : "", | 20260505 21:55:08.732 | INFO | "huge_shared" : " 27 GB", | 20260505 21:55:08.732 | INFO | "malloc" : " 14 GB", | 20260505 21:55:08.732 | INFO | "pinned" : " 29 GB" | 20260505 21:55:08.732 | INFO | } | 20260505 21:55:08.732 | INFO | Perf report for the last 5.14302 seconds: | 20260505 21:55:08.732 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:55:08.732 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:08.732 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:08.732 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:08.732 | INFO | ANALYSIS : 61.5%, 3.0, 96.1ms, 0 | 20260505 21:55:08.732 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:08.732 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:08.732 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 21:55:08.732 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:08.732 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:08.733 | INFO | Uploaded 36GiB to GPU in 2.13205s (16.8852GiB/s) | 20260505 21:55:08.733 | INFO | Downloaded 34GiB to GPU in 2.05975s (16.5069GiB/s) | 20260505 21:55:13.877 | INFO | Analyzing chunk frames = [6597120,6597632) | 20260505 21:55:13.877 | INFO | MemInfo: { | 20260505 21:55:13.877 | INFO | "gpu" : " 17 GB", | 20260505 21:55:13.877 | INFO | "huge" : "", | 20260505 21:55:13.877 | INFO | "huge_shared" : " 27 GB", | 20260505 21:55:13.877 | INFO | "malloc" : " 14 GB", | 20260505 21:55:13.877 | INFO | "pinned" : " 29 GB" | 20260505 21:55:13.877 | INFO | } | 20260505 21:55:13.877 | INFO | Perf report for the last 5.14461 seconds: | 20260505 21:55:13.877 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:55:13.877 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:13.877 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:13.877 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:13.877 | INFO | ANALYSIS : 61.6%, 3.0, 96.3ms, 0 | 20260505 21:55:13.877 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:13.877 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:13.877 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 21:55:13.877 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:13.877 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:13.877 | INFO | Uploaded 36GiB to GPU in 2.07969s (17.3103GiB/s) | 20260505 21:55:13.877 | INFO | Downloaded 34GiB to GPU in 2.0332s (16.7224GiB/s) | 20260505 21:55:19.014 | INFO | Analyzing chunk frames = [6597632,6598144) | 20260505 21:55:19.014 | INFO | MemInfo: { | 20260505 21:55:19.014 | INFO | "gpu" : " 17 GB", | 20260505 21:55:19.014 | INFO | "huge" : "", | 20260505 21:55:19.014 | INFO | "huge_shared" : " 27 GB", | 20260505 21:55:19.014 | INFO | "malloc" : " 14 GB", | 20260505 21:55:19.014 | INFO | "pinned" : " 29 GB" | 20260505 21:55:19.014 | INFO | } | 20260505 21:55:19.014 | INFO | Perf report for the last 5.13698 seconds: | 20260505 21:55:19.014 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:55:19.014 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:19.014 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:19.014 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:19.014 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260505 21:55:19.014 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:19.014 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:19.014 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 21:55:19.014 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:19.014 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:19.014 | INFO | Uploaded 36GiB to GPU in 2.1037s (17.1127GiB/s) | 20260505 21:55:19.014 | INFO | Downloaded 34GiB to GPU in 2.084s (16.3148GiB/s) | 20260505 21:55:24.145 | INFO | Analyzing chunk frames = [6598144,6598656) | 20260505 21:55:24.145 | INFO | MemInfo: { | 20260505 21:55:24.145 | INFO | "gpu" : " 17 GB", | 20260505 21:55:24.145 | INFO | "huge" : "", | 20260505 21:55:24.145 | INFO | "huge_shared" : " 27 GB", | 20260505 21:55:24.145 | INFO | "malloc" : " 14 GB", | 20260505 21:55:24.146 | INFO | "pinned" : " 29 GB" | 20260505 21:55:24.146 | INFO | } | 20260505 21:55:24.146 | INFO | Perf report for the last 5.13185 seconds: | 20260505 21:55:24.146 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:55:24.146 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:24.146 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:24.146 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:24.146 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260505 21:55:24.146 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:24.146 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:24.146 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 21:55:24.146 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:24.146 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:24.146 | INFO | Uploaded 36GiB to GPU in 2.11412s (17.0284GiB/s) | 20260505 21:55:24.146 | INFO | Downloaded 34GiB to GPU in 2.00445s (16.9622GiB/s) | 20260505 21:55:24.275 | INFO | Starting a round of Baz IO | 20260505 21:55:27.444 | INFO | Populating RTMetrics report | 20260505 21:55:27.444 | INFO | RTMetrics report populated | 20260505 21:55:27.444 | INFO | Opening tmp RTMetrics json file | 20260505 21:55:27.444 | INFO | Serializing RTMetrics to json | 20260505 21:55:27.446 | INFO | Serialization done, swapping tmp file now... | 20260505 21:55:27.446 | INFO | RTMetrics done | 20260505 21:55:27.524 | INFO | Finished a round of Baz IO: Wrote 1.95 GiB in 3.249 seconds | 20260505 21:55:27.524 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.54% / 0.00% / 78.46% | 20260505 21:55:27.524 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.135863% / 0% / 8.34465e-05% | 20260505 21:55:27.524 | INFO | Baz IO overall: | 20260505 21:55:27.524 | INFO | BytesWritten : 3811333744805 | 20260505 21:55:27.524 | INFO | NumEvents : 1798768186844 | 20260505 21:55:29.289 | INFO | Analyzing chunk frames = [6598656,6599168) | 20260505 21:55:29.289 | INFO | MemInfo: { | 20260505 21:55:29.289 | INFO | "gpu" : " 17 GB", | 20260505 21:55:29.289 | INFO | "huge" : "", | 20260505 21:55:29.289 | INFO | "huge_shared" : " 27 GB", | 20260505 21:55:29.289 | INFO | "malloc" : " 14 GB", | 20260505 21:55:29.289 | INFO | "pinned" : " 29 GB" | 20260505 21:55:29.289 | INFO | } | 20260505 21:55:29.289 | INFO | Perf report for the last 5.14314 seconds: | 20260505 21:55:29.289 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:55:29.289 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:29.289 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:29.289 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:29.289 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 21:55:29.289 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:29.289 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260505 21:55:29.289 | INFO | PRE_HQ : 60.0%, 1.7, 54.5ms, 0 | 20260505 21:55:29.289 | INFO | BAZWRITER : 62.7%, 3.6, 115.4ms, 0 | 20260505 21:55:29.289 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 21:55:29.289 | INFO | Uploaded 36GiB to GPU in 2.16434s (16.6332GiB/s) | 20260505 21:55:29.289 | INFO | Downloaded 38GiB to GPU in 2.23738s (16.9842GiB/s) | 20260505 21:55:34.430 | INFO | Analyzing chunk frames = [6599168,6599680) | 20260505 21:55:34.430 | INFO | MemInfo: { | 20260505 21:55:34.430 | INFO | "gpu" : " 17 GB", | 20260505 21:55:34.430 | INFO | "huge" : "", | 20260505 21:55:34.430 | INFO | "huge_shared" : " 27 GB", | 20260505 21:55:34.430 | INFO | "malloc" : " 14 GB", | 20260505 21:55:34.430 | INFO | "pinned" : " 29 GB" | 20260505 21:55:34.430 | INFO | } | 20260505 21:55:34.430 | INFO | Perf report for the last 5.1414 seconds: | 20260505 21:55:34.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:55:34.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:34.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:34.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:34.430 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 21:55:34.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:34.430 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:34.430 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 21:55:34.430 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:34.430 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:34.431 | INFO | Uploaded 36GiB to GPU in 2.07736s (17.3297GiB/s) | 20260505 21:55:34.431 | INFO | Downloaded 34GiB to GPU in 2.01784s (16.8497GiB/s) | 20260505 21:55:39.577 | INFO | Analyzing chunk frames = [6599680,6600192) | 20260505 21:55:39.577 | INFO | MemInfo: { | 20260505 21:55:39.577 | INFO | "gpu" : " 17 GB", | 20260505 21:55:39.577 | INFO | "huge" : "", | 20260505 21:55:39.577 | INFO | "huge_shared" : " 27 GB", | 20260505 21:55:39.577 | INFO | "malloc" : " 14 GB", | 20260505 21:55:39.577 | INFO | "pinned" : " 29 GB" | 20260505 21:55:39.577 | INFO | } | 20260505 21:55:39.577 | INFO | Perf report for the last 5.14683 seconds: | 20260505 21:55:39.577 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:55:39.577 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:39.577 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:39.577 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:39.577 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260505 21:55:39.577 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:39.577 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:39.577 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 21:55:39.577 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:39.577 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:39.577 | INFO | Uploaded 36GiB to GPU in 2.13911s (16.8294GiB/s) | 20260505 21:55:39.578 | INFO | Downloaded 34GiB to GPU in 2.02313s (16.8056GiB/s) | 20260505 21:55:44.706 | INFO | Analyzing chunk frames = [6600192,6600704) | 20260505 21:55:44.706 | INFO | MemInfo: { | 20260505 21:55:44.706 | INFO | "gpu" : " 17 GB", | 20260505 21:55:44.706 | INFO | "huge" : "", | 20260505 21:55:44.706 | INFO | "huge_shared" : " 27 GB", | 20260505 21:55:44.706 | INFO | "malloc" : " 14 GB", | 20260505 21:55:44.706 | INFO | "pinned" : " 29 GB" | 20260505 21:55:44.706 | INFO | } | 20260505 21:55:44.706 | INFO | Perf report for the last 5.12865 seconds: | 20260505 21:55:44.706 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:55:44.706 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:44.706 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:44.706 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:44.706 | INFO | ANALYSIS : 61.9%, 3.0, 96.0ms, 0 | 20260505 21:55:44.706 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:44.706 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:44.706 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 21:55:44.706 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:44.706 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:44.706 | INFO | Uploaded 36GiB to GPU in 2.13402s (16.8696GiB/s) | 20260505 21:55:44.706 | INFO | Downloaded 34GiB to GPU in 1.99973s (17.0023GiB/s) | 20260505 21:55:49.847 | INFO | Analyzing chunk frames = [6600704,6601216) | 20260505 21:55:49.847 | INFO | MemInfo: { | 20260505 21:55:49.847 | INFO | "gpu" : " 17 GB", | 20260505 21:55:49.847 | INFO | "huge" : "", | 20260505 21:55:49.847 | INFO | "huge_shared" : " 27 GB", | 20260505 21:55:49.847 | INFO | "malloc" : " 14 GB", | 20260505 21:55:49.848 | INFO | "pinned" : " 29 GB" | 20260505 21:55:49.848 | INFO | } | 20260505 21:55:49.848 | INFO | Perf report for the last 5.14177 seconds: | 20260505 21:55:49.848 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:55:49.848 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:49.848 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:49.848 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:49.848 | INFO | ANALYSIS : 61.6%, 3.0, 96.2ms, 0 | 20260505 21:55:49.848 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:49.848 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:49.848 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:55:49.848 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:49.848 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:49.848 | INFO | Uploaded 36GiB to GPU in 2.13556s (16.8574GiB/s) | 20260505 21:55:49.848 | INFO | Downloaded 34GiB to GPU in 2.05661s (16.532GiB/s) | 20260505 21:55:54.990 | INFO | Analyzing chunk frames = [6601216,6601728) | 20260505 21:55:54.990 | INFO | MemInfo: { | 20260505 21:55:54.990 | INFO | "gpu" : " 17 GB", | 20260505 21:55:54.990 | INFO | "huge" : "", | 20260505 21:55:54.990 | INFO | "huge_shared" : " 27 GB", | 20260505 21:55:54.990 | INFO | "malloc" : " 14 GB", | 20260505 21:55:54.990 | INFO | "pinned" : " 29 GB" | 20260505 21:55:54.990 | INFO | } | 20260505 21:55:54.990 | INFO | Perf report for the last 5.14269 seconds: | 20260505 21:55:54.990 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:55:54.990 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:54.990 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:54.990 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:54.990 | INFO | ANALYSIS : 61.5%, 3.0, 96.1ms, 0 | 20260505 21:55:54.990 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:54.990 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:54.990 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 21:55:54.990 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:54.990 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:55:54.991 | INFO | Uploaded 36GiB to GPU in 2.08458s (17.2697GiB/s) | 20260505 21:55:54.991 | INFO | Downloaded 34GiB to GPU in 2.03455s (16.7113GiB/s) | 20260505 21:56:00.130 | INFO | Analyzing chunk frames = [6601728,6602240) | 20260505 21:56:00.130 | INFO | MemInfo: { | 20260505 21:56:00.130 | INFO | "gpu" : " 17 GB", | 20260505 21:56:00.130 | INFO | "huge" : "", | 20260505 21:56:00.130 | INFO | "huge_shared" : " 27 GB", | 20260505 21:56:00.130 | INFO | "malloc" : " 14 GB", | 20260505 21:56:00.130 | INFO | "pinned" : " 29 GB" | 20260505 21:56:00.130 | INFO | } | 20260505 21:56:00.130 | INFO | Perf report for the last 5.14004 seconds: | 20260505 21:56:00.130 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:56:00.130 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:00.130 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:00.130 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:00.130 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 21:56:00.130 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:00.130 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:00.130 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260505 21:56:00.130 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:00.130 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:00.131 | INFO | Uploaded 36GiB to GPU in 2.02108s (17.8123GiB/s) | 20260505 21:56:00.131 | INFO | Downloaded 34GiB to GPU in 2.03908s (16.6742GiB/s) | 20260505 21:56:05.258 | INFO | Analyzing chunk frames = [6602240,6602752) | 20260505 21:56:05.258 | INFO | MemInfo: { | 20260505 21:56:05.258 | INFO | "gpu" : " 17 GB", | 20260505 21:56:05.258 | INFO | "huge" : "", | 20260505 21:56:05.258 | INFO | "huge_shared" : " 27 GB", | 20260505 21:56:05.258 | INFO | "malloc" : " 14 GB", | 20260505 21:56:05.258 | INFO | "pinned" : " 29 GB" | 20260505 21:56:05.258 | INFO | } | 20260505 21:56:05.259 | INFO | Perf report for the last 5.12817 seconds: | 20260505 21:56:05.259 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:56:05.259 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:05.259 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:05.259 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:05.259 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260505 21:56:05.259 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:05.259 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:05.259 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 21:56:05.259 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:05.259 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:05.259 | INFO | Uploaded 36GiB to GPU in 2.11125s (17.0515GiB/s) | 20260505 21:56:05.259 | INFO | Downloaded 34GiB to GPU in 2.00243s (16.9793GiB/s) | 20260505 21:56:05.259 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 21:56:05.259 | INFO | Total mem usage: 15.3 GiB | 20260505 21:56:05.259 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 21:56:05.259 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.18%) | 20260505 21:56:05.259 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 21:56:05.259 | INFO | High Water Breakdown: | 20260505 21:56:05.259 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 21:56:05.259 | INFO | 38.8% : 5.5 GiB ( 82.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 21:56:05.259 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 21:56:05.259 | INFO | ------------------------------------------ | 20260505 21:56:05.259 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:56:05.259 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 21:56:05.259 | INFO | Total mem usage: 26.5 GiB | 20260505 21:56:05.259 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 21:56:05.259 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 21:56:05.259 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 21:56:05.259 | INFO | High Water Breakdown: | 20260505 21:56:05.259 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:56:05.259 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:56:05.259 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:56:05.259 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:56:05.259 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:56:05.259 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:56:05.259 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 21:56:05.259 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 21:56:05.259 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:56:05.259 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 21:56:05.259 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 21:56:05.259 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 21:56:05.259 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 21:56:05.259 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 21:56:05.259 | INFO | ------------------------------------------ | 20260505 21:56:05.259 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:56:05.259 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 21:56:05.259 | INFO | Total mem usage: 25.6 GiB | 20260505 21:56:05.259 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 21:56:05.259 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 21:56:05.259 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 21:56:05.259 | INFO | High Water Breakdown: | 20260505 21:56:05.259 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 21:56:05.259 | INFO | ------------------------------------------ | 20260505 21:56:05.259 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:56:05.259 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 21:56:05.259 | INFO | Total mem usage: 15.7 GiB | 20260505 21:56:05.259 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 21:56:05.259 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 21:56:05.259 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 21:56:05.259 | INFO | High Water Breakdown: | 20260505 21:56:05.259 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 21:56:05.259 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 21:56:05.259 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 21:56:05.259 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 21:56:05.259 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 21:56:05.259 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 21:56:05.259 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 21:56:05.259 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 21:56:05.259 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 21:56:05.259 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:56:05.259 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 21:56:05.259 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:56:05.259 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 21:56:05.259 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 21:56:05.259 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 21:56:05.259 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 21:56:05.259 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:56:05.259 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 21:56:05.259 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:56:05.259 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:56:05.259 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:56:05.259 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 21:56:05.259 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 21:56:05.259 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:56:05.259 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 21:56:05.259 | INFO | ------------------------------------------ | 20260505 21:56:05.259 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:56:05.260 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 21:56:05.260 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:56:05.260 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:56:05.260 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.40e+01, 1.35e+01 / 8.64e-01 | 20260505 21:56:05.260 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.05e-01, 1.04e-02 / 5.53e-03 | 20260505 21:56:05.260 | INFO | PerfReport: 11.84%, Baseline -- 3.72e+00 / 4.56e+00, 5.79e-01 / 1.18e-01 | 20260505 21:56:05.260 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.55e+00, 1.92e-01 / 7.95e-02 | 20260505 21:56:05.260 | INFO | PerfReport: 15.99%, DME -- 4.02e+01 / 4.42e+01, 3.76e+00 / 8.83e-01 | 20260505 21:56:05.260 | INFO | PerfReport: 57.10%, FrameLabeling -- 1.79e+01 / 1.91e+01, 1.47e+00 / 9.59e-01 | 20260505 21:56:05.260 | INFO | PerfReport: 7.20%, PulseAccumulating -- 2.26e+00 / 2.81e+00, 4.34e-01 / 2.17e-01 | 20260505 21:56:05.260 | INFO | PerfReport: 3.19%, Metrics -- 1.00e+00 / 1.29e+00, 2.71e-01 / 1.36e-01 | 20260505 21:56:05.260 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.47e-01 / 4.01e-01, 5.89e-02 / 3.71e-02 | 20260505 21:56:05.260 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 21:56:05.260 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:56:05.260 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:56:05.260 | INFO | PerfReport: Full Execution -- 9.88e+01 / -nan, 2.49e+01 / 0.00e+00 | 20260505 21:56:05.260 | INFO | PerfReport: 22.33%, Upload -- 2.21e+01 / 0.00e+00, 7.60e+00 / 0.00e+00 | 20260505 21:56:05.260 | INFO | PerfReport: 21.69%, Download -- 2.14e+01 / 0.00e+00, 7.56e+00 / 0.00e+00 | 20260505 21:56:05.260 | INFO | PerfReport: 31.78%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 21:56:05.260 | INFO | PerfReport: 2.24%, LockUpload -- 2.22e+00 / 0.00e+00, 7.98e+00 / 0.00e+00 | 20260505 21:56:05.260 | INFO | PerfReport: 1.29%, LockDownload -- 1.27e+00 / 0.00e+00, 2.82e+00 / 0.00e+00 | 20260505 21:56:05.260 | INFO | PerfReport: 20.65%, LockCompute -- 2.04e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 21:56:05.260 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.99e-02 / 0.00e+00 | 20260505 21:56:05.389 | INFO | Starting a round of Baz IO | 20260505 21:56:08.554 | INFO | Populating RTMetrics report | 20260505 21:56:08.555 | INFO | RTMetrics report populated | 20260505 21:56:08.555 | INFO | Opening tmp RTMetrics json file | 20260505 21:56:08.557 | INFO | Serializing RTMetrics to json | 20260505 21:56:08.558 | INFO | Serialization done, swapping tmp file now... | 20260505 21:56:08.558 | INFO | RTMetrics done | 20260505 21:56:08.638 | INFO | Finished a round of Baz IO: Wrote 1.951 GiB in 3.249 seconds | 20260505 21:56:08.638 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.54% / 0.00% / 78.46% | 20260505 21:56:08.638 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.135807% / 3.97364e-06% / 0.000103315% | 20260505 21:56:08.639 | INFO | Baz IO overall: | 20260505 21:56:08.639 | INFO | BytesWritten : 3813428393306 | 20260505 21:56:08.639 | INFO | NumEvents : 1799733750300 | 20260505 21:56:10.397 | INFO | Analyzing chunk frames = [6602752,6603264) | 20260505 21:56:10.397 | INFO | MemInfo: { | 20260505 21:56:10.397 | INFO | "gpu" : " 17 GB", | 20260505 21:56:10.397 | INFO | "huge" : "", | 20260505 21:56:10.397 | INFO | "huge_shared" : " 27 GB", | 20260505 21:56:10.397 | INFO | "malloc" : " 14 GB", | 20260505 21:56:10.397 | INFO | "pinned" : " 29 GB" | 20260505 21:56:10.398 | INFO | } | 20260505 21:56:10.398 | INFO | Perf report for the last 5.13902 seconds: | 20260505 21:56:10.398 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:56:10.398 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:10.398 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:10.398 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:10.398 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 21:56:10.398 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:10.398 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260505 21:56:10.398 | INFO | PRE_HQ : 59.9%, 1.7, 54.4ms, 0 | 20260505 21:56:10.398 | INFO | BAZWRITER : 62.7%, 3.6, 115.1ms, 0 | 20260505 21:56:10.398 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 21:56:10.398 | INFO | Uploaded 36GiB to GPU in 2.15858s (16.6776GiB/s) | 20260505 21:56:10.398 | INFO | Downloaded 38GiB to GPU in 2.24391s (16.9347GiB/s) | 20260505 21:56:15.545 | INFO | Analyzing chunk frames = [6603264,6603776) | 20260505 21:56:15.545 | INFO | MemInfo: { | 20260505 21:56:15.545 | INFO | "gpu" : " 17 GB", | 20260505 21:56:15.545 | INFO | "huge" : "", | 20260505 21:56:15.545 | INFO | "huge_shared" : " 27 GB", | 20260505 21:56:15.545 | INFO | "malloc" : " 14 GB", | 20260505 21:56:15.545 | INFO | "pinned" : " 29 GB" | 20260505 21:56:15.545 | INFO | } | 20260505 21:56:15.545 | INFO | Perf report for the last 5.14758 seconds: | 20260505 21:56:15.545 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:56:15.545 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:15.545 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:15.545 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:15.545 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 21:56:15.545 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:15.545 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:15.545 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 21:56:15.545 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:15.545 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:15.545 | INFO | Uploaded 36GiB to GPU in 2.09862s (17.1541GiB/s) | 20260505 21:56:15.545 | INFO | Downloaded 34GiB to GPU in 2.02651s (16.7776GiB/s) | 20260505 21:56:20.688 | INFO | Analyzing chunk frames = [6603776,6604288) | 20260505 21:56:20.688 | INFO | MemInfo: { | 20260505 21:56:20.688 | INFO | "gpu" : " 17 GB", | 20260505 21:56:20.688 | INFO | "huge" : "", | 20260505 21:56:20.688 | INFO | "huge_shared" : " 27 GB", | 20260505 21:56:20.688 | INFO | "malloc" : " 14 GB", | 20260505 21:56:20.689 | INFO | "pinned" : " 29 GB" | 20260505 21:56:20.689 | INFO | } | 20260505 21:56:20.689 | INFO | Perf report for the last 5.14342 seconds: | 20260505 21:56:20.689 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:56:20.689 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:20.689 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:20.689 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:20.689 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260505 21:56:20.689 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:20.689 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:20.689 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 21:56:20.689 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:20.689 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:20.689 | INFO | Uploaded 36GiB to GPU in 2.15029s (16.742GiB/s) | 20260505 21:56:20.689 | INFO | Downloaded 34GiB to GPU in 2.04897s (16.5937GiB/s) | 20260505 21:56:25.824 | INFO | Analyzing chunk frames = [6604288,6604800) | 20260505 21:56:25.824 | INFO | MemInfo: { | 20260505 21:56:25.824 | INFO | "gpu" : " 17 GB", | 20260505 21:56:25.824 | INFO | "huge" : "", | 20260505 21:56:25.824 | INFO | "huge_shared" : " 27 GB", | 20260505 21:56:25.824 | INFO | "malloc" : " 14 GB", | 20260505 21:56:25.824 | INFO | "pinned" : " 29 GB" | 20260505 21:56:25.824 | INFO | } | 20260505 21:56:25.824 | INFO | Perf report for the last 5.13564 seconds: | 20260505 21:56:25.824 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:56:25.824 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:25.824 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:25.824 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:25.824 | INFO | ANALYSIS : 61.8%, 3.0, 95.9ms, 0 | 20260505 21:56:25.824 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:25.824 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:25.824 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 21:56:25.824 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:25.824 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:25.824 | INFO | Uploaded 36GiB to GPU in 2.1418s (16.8083GiB/s) | 20260505 21:56:25.825 | INFO | Downloaded 34GiB to GPU in 2.01989s (16.8326GiB/s) | 20260505 21:56:30.958 | INFO | Analyzing chunk frames = [6604800,6605312) | 20260505 21:56:30.958 | INFO | MemInfo: { | 20260505 21:56:30.958 | INFO | "gpu" : " 17 GB", | 20260505 21:56:30.958 | INFO | "huge" : "", | 20260505 21:56:30.958 | INFO | "huge_shared" : " 27 GB", | 20260505 21:56:30.958 | INFO | "malloc" : " 14 GB", | 20260505 21:56:30.958 | INFO | "pinned" : " 29 GB" | 20260505 21:56:30.958 | INFO | } | 20260505 21:56:30.958 | INFO | Perf report for the last 5.13418 seconds: | 20260505 21:56:30.958 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:56:30.958 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:30.958 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:30.958 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:30.958 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260505 21:56:30.958 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:30.958 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:30.958 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:56:30.959 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:30.959 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:30.959 | INFO | Uploaded 36GiB to GPU in 2.13022s (16.8996GiB/s) | 20260505 21:56:30.959 | INFO | Downloaded 34GiB to GPU in 2.06301s (16.4808GiB/s) | 20260505 21:56:36.099 | INFO | Analyzing chunk frames = [6605312,6605824) | 20260505 21:56:36.099 | INFO | MemInfo: { | 20260505 21:56:36.099 | INFO | "gpu" : " 17 GB", | 20260505 21:56:36.099 | INFO | "huge" : "", | 20260505 21:56:36.099 | INFO | "huge_shared" : " 27 GB", | 20260505 21:56:36.099 | INFO | "malloc" : " 14 GB", | 20260505 21:56:36.099 | INFO | "pinned" : " 29 GB" | 20260505 21:56:36.099 | INFO | } | 20260505 21:56:36.099 | INFO | Perf report for the last 5.14048 seconds: | 20260505 21:56:36.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:56:36.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:36.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:36.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:36.099 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 21:56:36.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:36.099 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:36.099 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:56:36.099 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:36.099 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:36.099 | INFO | Uploaded 36GiB to GPU in 2.16405s (16.6355GiB/s) | 20260505 21:56:36.099 | INFO | Downloaded 34GiB to GPU in 2.06926s (16.431GiB/s) | 20260505 21:56:41.240 | INFO | Analyzing chunk frames = [6605824,6606336) | 20260505 21:56:41.240 | INFO | MemInfo: { | 20260505 21:56:41.240 | INFO | "gpu" : " 17 GB", | 20260505 21:56:41.240 | INFO | "huge" : "", | 20260505 21:56:41.240 | INFO | "huge_shared" : " 27 GB", | 20260505 21:56:41.240 | INFO | "malloc" : " 14 GB", | 20260505 21:56:41.240 | INFO | "pinned" : " 29 GB" | 20260505 21:56:41.240 | INFO | } | 20260505 21:56:41.240 | INFO | Perf report for the last 5.14087 seconds: | 20260505 21:56:41.240 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:56:41.240 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:41.240 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:41.240 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:41.240 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 21:56:41.240 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:41.240 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:41.240 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 21:56:41.240 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:41.240 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:41.240 | INFO | Uploaded 36GiB to GPU in 2.16489s (16.629GiB/s) | 20260505 21:56:41.240 | INFO | Downloaded 34GiB to GPU in 2.13178s (15.9491GiB/s) | 20260505 21:56:46.383 | INFO | Analyzing chunk frames = [6606336,6606848) | 20260505 21:56:46.383 | INFO | MemInfo: { | 20260505 21:56:46.383 | INFO | "gpu" : " 17 GB", | 20260505 21:56:46.383 | INFO | "huge" : "", | 20260505 21:56:46.383 | INFO | "huge_shared" : " 27 GB", | 20260505 21:56:46.383 | INFO | "malloc" : " 14 GB", | 20260505 21:56:46.383 | INFO | "pinned" : " 29 GB" | 20260505 21:56:46.383 | INFO | } | 20260505 21:56:46.383 | INFO | Perf report for the last 5.14347 seconds: | 20260505 21:56:46.383 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:56:46.383 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:46.383 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:46.383 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:46.383 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 21:56:46.383 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:46.383 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:46.383 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 21:56:46.383 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:46.383 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:46.384 | INFO | Uploaded 36GiB to GPU in 2.10704s (17.0855GiB/s) | 20260505 21:56:46.384 | INFO | Downloaded 34GiB to GPU in 1.99876s (17.0106GiB/s) | 20260505 21:56:46.509 | INFO | Starting a round of Baz IO | 20260505 21:56:49.661 | INFO | Populating RTMetrics report | 20260505 21:56:49.661 | INFO | RTMetrics report populated | 20260505 21:56:49.661 | INFO | Opening tmp RTMetrics json file | 20260505 21:56:49.663 | INFO | Serializing RTMetrics to json | 20260505 21:56:49.664 | INFO | Serialization done, swapping tmp file now... | 20260505 21:56:49.664 | INFO | RTMetrics done | 20260505 21:56:49.741 | INFO | Finished a round of Baz IO: Wrote 1.952 GiB in 3.231 seconds | 20260505 21:56:49.741 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.54% / 0.00% / 78.46% | 20260505 21:56:49.741 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.136173% / 0% / 9.13938e-05% | 20260505 21:56:49.741 | INFO | Baz IO overall: | 20260505 21:56:49.741 | INFO | BytesWritten : 3815524296518 | 20260505 21:56:49.741 | INFO | NumEvents : 1800700130582 | 20260505 21:56:51.522 | INFO | Analyzing chunk frames = [6606848,6607360) | 20260505 21:56:51.522 | INFO | MemInfo: { | 20260505 21:56:51.522 | INFO | "gpu" : " 17 GB", | 20260505 21:56:51.522 | INFO | "huge" : "", | 20260505 21:56:51.522 | INFO | "huge_shared" : " 27 GB", | 20260505 21:56:51.522 | INFO | "malloc" : " 14 GB", | 20260505 21:56:51.522 | INFO | "pinned" : " 29 GB" | 20260505 21:56:51.522 | INFO | } | 20260505 21:56:51.522 | INFO | Perf report for the last 5.13906 seconds: | 20260505 21:56:51.522 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:56:51.522 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:51.522 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:51.522 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:51.522 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 21:56:51.522 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:51.522 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260505 21:56:51.522 | INFO | PRE_HQ : 60.0%, 1.8, 54.8ms, 0 | 20260505 21:56:51.522 | INFO | BAZWRITER : 62.1%, 3.6, 116.3ms, 0 | 20260505 21:56:51.522 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 21:56:51.523 | INFO | Uploaded 36GiB to GPU in 2.13859s (16.8336GiB/s) | 20260505 21:56:51.523 | INFO | Downloaded 38GiB to GPU in 2.23122s (17.031GiB/s) | 20260505 21:56:56.651 | INFO | Analyzing chunk frames = [6607360,6607872) | 20260505 21:56:56.651 | INFO | MemInfo: { | 20260505 21:56:56.651 | INFO | "gpu" : " 17 GB", | 20260505 21:56:56.651 | INFO | "huge" : "", | 20260505 21:56:56.651 | INFO | "huge_shared" : " 27 GB", | 20260505 21:56:56.651 | INFO | "malloc" : " 14 GB", | 20260505 21:56:56.651 | INFO | "pinned" : " 29 GB" | 20260505 21:56:56.651 | INFO | } | 20260505 21:56:56.651 | INFO | Perf report for the last 5.1289 seconds: | 20260505 21:56:56.651 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:56:56.651 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:56.651 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:56.651 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:56.651 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260505 21:56:56.651 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:56.651 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:56.651 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260505 21:56:56.651 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:56.651 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:56:56.652 | INFO | Uploaded 36GiB to GPU in 2.0898s (17.2265GiB/s) | 20260505 21:56:56.652 | INFO | Downloaded 34GiB to GPU in 2.02607s (16.7813GiB/s) | 20260505 21:57:01.803 | INFO | Analyzing chunk frames = [6607872,6608384) | 20260505 21:57:01.803 | INFO | MemInfo: { | 20260505 21:57:01.803 | INFO | "gpu" : " 17 GB", | 20260505 21:57:01.803 | INFO | "huge" : "", | 20260505 21:57:01.803 | INFO | "huge_shared" : " 27 GB", | 20260505 21:57:01.804 | INFO | "malloc" : " 14 GB", | 20260505 21:57:01.804 | INFO | "pinned" : " 29 GB" | 20260505 21:57:01.804 | INFO | } | 20260505 21:57:01.804 | INFO | Perf report for the last 5.15235 seconds: | 20260505 21:57:01.804 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:57:01.804 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:01.804 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:01.804 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:01.804 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260505 21:57:01.804 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:01.804 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:01.804 | INFO | PRE_HQ : 26.8%, 1.0, 14.0ms, 0 | 20260505 21:57:01.804 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:01.804 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:01.804 | INFO | Uploaded 36GiB to GPU in 2.13257s (16.881GiB/s) | 20260505 21:57:01.804 | INFO | Downloaded 34GiB to GPU in 2.01472s (16.8758GiB/s) | 20260505 21:57:06.942 | INFO | Analyzing chunk frames = [6608384,6608896) | 20260505 21:57:06.942 | INFO | MemInfo: { | 20260505 21:57:06.942 | INFO | "gpu" : " 17 GB", | 20260505 21:57:06.942 | INFO | "huge" : "", | 20260505 21:57:06.942 | INFO | "huge_shared" : " 27 GB", | 20260505 21:57:06.942 | INFO | "malloc" : " 14 GB", | 20260505 21:57:06.942 | INFO | "pinned" : " 29 GB" | 20260505 21:57:06.943 | INFO | } | 20260505 21:57:06.943 | INFO | Perf report for the last 5.13892 seconds: | 20260505 21:57:06.943 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:57:06.943 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:06.943 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:06.943 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:06.943 | INFO | ANALYSIS : 61.8%, 3.0, 95.9ms, 0 | 20260505 21:57:06.943 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:06.943 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:06.943 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 21:57:06.943 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:06.943 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:06.943 | INFO | Uploaded 36GiB to GPU in 2.13201s (16.8855GiB/s) | 20260505 21:57:06.943 | INFO | Downloaded 34GiB to GPU in 2.01191s (16.8994GiB/s) | 20260505 21:57:12.074 | INFO | Analyzing chunk frames = [6608896,6609408) | 20260505 21:57:12.074 | INFO | MemInfo: { | 20260505 21:57:12.074 | INFO | "gpu" : " 17 GB", | 20260505 21:57:12.074 | INFO | "huge" : "", | 20260505 21:57:12.074 | INFO | "huge_shared" : " 27 GB", | 20260505 21:57:12.074 | INFO | "malloc" : " 14 GB", | 20260505 21:57:12.074 | INFO | "pinned" : " 29 GB" | 20260505 21:57:12.074 | INFO | } | 20260505 21:57:12.074 | INFO | Perf report for the last 5.13194 seconds: | 20260505 21:57:12.075 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:57:12.075 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:12.075 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:12.075 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:12.075 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260505 21:57:12.075 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:12.075 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:12.075 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:57:12.075 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:12.075 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:12.075 | INFO | Uploaded 36GiB to GPU in 2.14569s (16.7778GiB/s) | 20260505 21:57:12.075 | INFO | Downloaded 34GiB to GPU in 2.06259s (16.4841GiB/s) | 20260505 21:57:17.211 | INFO | Analyzing chunk frames = [6609408,6609920) | 20260505 21:57:17.211 | INFO | MemInfo: { | 20260505 21:57:17.211 | INFO | "gpu" : " 17 GB", | 20260505 21:57:17.211 | INFO | "huge" : "", | 20260505 21:57:17.211 | INFO | "huge_shared" : " 27 GB", | 20260505 21:57:17.211 | INFO | "malloc" : " 14 GB", | 20260505 21:57:17.211 | INFO | "pinned" : " 29 GB" | 20260505 21:57:17.211 | INFO | } | 20260505 21:57:17.211 | INFO | Perf report for the last 5.1369 seconds: | 20260505 21:57:17.211 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:57:17.211 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:17.211 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:17.211 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:17.211 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260505 21:57:17.211 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:17.211 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:17.211 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 21:57:17.211 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:17.211 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:17.212 | INFO | Uploaded 36GiB to GPU in 2.08765s (17.2443GiB/s) | 20260505 21:57:17.212 | INFO | Downloaded 34GiB to GPU in 2.03331s (16.7215GiB/s) | 20260505 21:57:22.355 | INFO | Analyzing chunk frames = [6609920,6610432) | 20260505 21:57:22.356 | INFO | MemInfo: { | 20260505 21:57:22.356 | INFO | "gpu" : " 17 GB", | 20260505 21:57:22.356 | INFO | "huge" : "", | 20260505 21:57:22.356 | INFO | "huge_shared" : " 27 GB", | 20260505 21:57:22.356 | INFO | "malloc" : " 14 GB", | 20260505 21:57:22.356 | INFO | "pinned" : " 29 GB" | 20260505 21:57:22.356 | INFO | } | 20260505 21:57:22.356 | INFO | Perf report for the last 5.14425 seconds: | 20260505 21:57:22.356 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:57:22.356 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:22.356 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:22.356 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:22.356 | INFO | ANALYSIS : 62.9%, 3.0, 98.4ms, 0 | 20260505 21:57:22.356 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:22.356 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:22.356 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 21:57:22.356 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:22.356 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:22.356 | INFO | Uploaded 36GiB to GPU in 2.14044s (16.819GiB/s) | 20260505 21:57:22.356 | INFO | Downloaded 34GiB to GPU in 2.1392s (15.8938GiB/s) | 20260505 21:57:27.494 | INFO | Analyzing chunk frames = [6610432,6610944) | 20260505 21:57:27.494 | INFO | MemInfo: { | 20260505 21:57:27.494 | INFO | "gpu" : " 17 GB", | 20260505 21:57:27.494 | INFO | "huge" : "", | 20260505 21:57:27.494 | INFO | "huge_shared" : " 27 GB", | 20260505 21:57:27.494 | INFO | "malloc" : " 14 GB", | 20260505 21:57:27.494 | INFO | "pinned" : " 29 GB" | 20260505 21:57:27.494 | INFO | } | 20260505 21:57:27.494 | INFO | Perf report for the last 5.13837 seconds: | 20260505 21:57:27.494 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:57:27.494 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:27.494 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:27.494 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:27.494 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 21:57:27.494 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:27.494 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:27.494 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 21:57:27.494 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:27.494 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:27.494 | INFO | Uploaded 36GiB to GPU in 2.12137s (16.9702GiB/s) | 20260505 21:57:27.494 | INFO | Downloaded 34GiB to GPU in 2.00289s (16.9754GiB/s) | 20260505 21:57:27.495 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 21:57:27.495 | INFO | Total mem usage: 15.3 GiB | 20260505 21:57:27.495 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 21:57:27.495 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.97%) | 20260505 21:57:27.495 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 21:57:27.495 | INFO | High Water Breakdown: | 20260505 21:57:27.495 | INFO | 60.9% : 8.57 GiB ( 90% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 21:57:27.495 | INFO | 38.9% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 21:57:27.495 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 21:57:27.495 | INFO | ------------------------------------------ | 20260505 21:57:27.495 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:57:27.495 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 21:57:27.495 | INFO | Total mem usage: 26.5 GiB | 20260505 21:57:27.495 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 21:57:27.495 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 21:57:27.495 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 21:57:27.495 | INFO | High Water Breakdown: | 20260505 21:57:27.495 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:57:27.495 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:57:27.495 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:57:27.495 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:57:27.495 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:57:27.495 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:57:27.495 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 21:57:27.495 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 21:57:27.495 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:57:27.495 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 21:57:27.495 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 21:57:27.495 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 21:57:27.495 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 21:57:27.495 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 21:57:27.495 | INFO | ------------------------------------------ | 20260505 21:57:27.495 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:57:27.495 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 21:57:27.495 | INFO | Total mem usage: 25.6 GiB | 20260505 21:57:27.495 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 21:57:27.495 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 21:57:27.495 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 21:57:27.495 | INFO | High Water Breakdown: | 20260505 21:57:27.495 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 21:57:27.495 | INFO | ------------------------------------------ | 20260505 21:57:27.495 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:57:27.495 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 21:57:27.495 | INFO | Total mem usage: 15.7 GiB | 20260505 21:57:27.495 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 21:57:27.495 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 21:57:27.495 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 21:57:27.495 | INFO | High Water Breakdown: | 20260505 21:57:27.495 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 21:57:27.495 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 21:57:27.495 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 21:57:27.495 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 21:57:27.495 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 21:57:27.495 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 21:57:27.495 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 21:57:27.495 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 21:57:27.495 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 21:57:27.495 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:57:27.495 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 21:57:27.495 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:57:27.495 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 21:57:27.495 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 21:57:27.495 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 21:57:27.495 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 21:57:27.495 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:57:27.495 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 21:57:27.495 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:57:27.495 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:57:27.495 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:57:27.495 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 21:57:27.495 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 21:57:27.495 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:57:27.495 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 21:57:27.495 | INFO | ------------------------------------------ | 20260505 21:57:27.495 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:57:27.495 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 21:57:27.495 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:57:27.495 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:57:27.495 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.43e+01, 1.35e+01 / 1.64e+00 | 20260505 21:57:27.495 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.03e-01, 1.01e-02 / 2.80e-03 | 20260505 21:57:27.495 | INFO | PerfReport: 11.82%, Baseline -- 3.71e+00 / 4.55e+00, 5.56e-01 / 1.47e-01 | 20260505 21:57:27.495 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.53e+00, 1.87e-01 / 9.34e-02 | 20260505 21:57:27.495 | INFO | PerfReport: 15.99%, DME -- 4.01e+01 / 4.43e+01, 4.17e+00 / 1.68e+00 | 20260505 21:57:27.495 | INFO | PerfReport: 57.13%, FrameLabeling -- 1.79e+01 / 1.93e+01, 1.43e+00 / 1.05e+00 | 20260505 21:57:27.495 | INFO | PerfReport: 7.19%, PulseAccumulating -- 2.25e+00 / 2.82e+00, 4.24e-01 / 2.25e-01 | 20260505 21:57:27.495 | INFO | PerfReport: 3.19%, Metrics -- 1.00e+00 / 1.31e+00, 2.70e-01 / 1.41e-01 | 20260505 21:57:27.496 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.47e-01 / 4.04e-01, 5.96e-02 / 3.58e-02 | 20260505 21:57:27.496 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 21:57:27.496 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:57:27.496 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:57:27.496 | INFO | PerfReport: Full Execution -- 9.89e+01 / -nan, 2.50e+01 / 0.00e+00 | 20260505 21:57:27.496 | INFO | PerfReport: 22.44%, Upload -- 2.22e+01 / 0.00e+00, 7.56e+00 / 0.00e+00 | 20260505 21:57:27.496 | INFO | PerfReport: 21.80%, Download -- 2.16e+01 / 0.00e+00, 7.47e+00 / 0.00e+00 | 20260505 21:57:27.496 | INFO | PerfReport: 31.72%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 21:57:27.496 | INFO | PerfReport: 2.26%, LockUpload -- 2.23e+00 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260505 21:57:27.496 | INFO | PerfReport: 1.34%, LockDownload -- 1.32e+00 / 0.00e+00, 2.85e+00 / 0.00e+00 | 20260505 21:57:27.496 | INFO | PerfReport: 20.44%, LockCompute -- 2.02e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 21:57:27.496 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.01e-02 / 0.00e+00 | 20260505 21:57:27.618 | INFO | Starting a round of Baz IO | 20260505 21:57:30.770 | INFO | Populating RTMetrics report | 20260505 21:57:30.771 | INFO | RTMetrics report populated | 20260505 21:57:30.771 | INFO | Opening tmp RTMetrics json file | 20260505 21:57:30.772 | INFO | Serializing RTMetrics to json | 20260505 21:57:30.773 | INFO | Serialization done, swapping tmp file now... | 20260505 21:57:30.773 | INFO | RTMetrics done | 20260505 21:57:30.853 | INFO | Finished a round of Baz IO: Wrote 1.951 GiB in 3.234 seconds | 20260505 21:57:30.853 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.53% / 0.00% / 78.46% | 20260505 21:57:30.854 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.135835% / 0% / 9.53674e-05% | 20260505 21:57:30.854 | INFO | Baz IO overall: | 20260505 21:57:30.854 | INFO | BytesWritten : 3817619083592 | 20260505 21:57:30.854 | INFO | NumEvents : 1801665868170 | 20260505 21:57:32.635 | INFO | Analyzing chunk frames = [6610944,6611456) | 20260505 21:57:32.635 | INFO | MemInfo: { | 20260505 21:57:32.635 | INFO | "gpu" : " 17 GB", | 20260505 21:57:32.635 | INFO | "huge" : "", | 20260505 21:57:32.635 | INFO | "huge_shared" : " 27 GB", | 20260505 21:57:32.635 | INFO | "malloc" : " 14 GB", | 20260505 21:57:32.635 | INFO | "pinned" : " 29 GB" | 20260505 21:57:32.635 | INFO | } | 20260505 21:57:32.635 | INFO | Perf report for the last 5.14128 seconds: | 20260505 21:57:32.635 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:57:32.635 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:32.635 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:32.636 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:32.636 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260505 21:57:32.636 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:32.636 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260505 21:57:32.636 | INFO | PRE_HQ : 60.1%, 1.8, 55.3ms, 0 | 20260505 21:57:32.636 | INFO | BAZWRITER : 62.0%, 3.7, 116.9ms, 0 | 20260505 21:57:32.636 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 21:57:32.636 | INFO | Uploaded 36GiB to GPU in 2.13557s (16.8573GiB/s) | 20260505 21:57:32.636 | INFO | Downloaded 38GiB to GPU in 2.2209s (17.1102GiB/s) | 20260505 21:57:37.771 | INFO | Analyzing chunk frames = [6611456,6611968) | 20260505 21:57:37.771 | INFO | MemInfo: { | 20260505 21:57:37.771 | INFO | "gpu" : " 17 GB", | 20260505 21:57:37.771 | INFO | "huge" : "", | 20260505 21:57:37.771 | INFO | "huge_shared" : " 27 GB", | 20260505 21:57:37.771 | INFO | "malloc" : " 14 GB", | 20260505 21:57:37.771 | INFO | "pinned" : " 29 GB" | 20260505 21:57:37.771 | INFO | } | 20260505 21:57:37.771 | INFO | Perf report for the last 5.13557 seconds: | 20260505 21:57:37.771 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:57:37.771 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:37.771 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:37.771 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:37.771 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260505 21:57:37.771 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:37.771 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:37.771 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 21:57:37.771 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:37.771 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:37.771 | INFO | Uploaded 36GiB to GPU in 2.08175s (17.2931GiB/s) | 20260505 21:57:37.771 | INFO | Downloaded 34GiB to GPU in 2.0219s (16.8159GiB/s) | 20260505 21:57:42.905 | INFO | Analyzing chunk frames = [6611968,6612480) | 20260505 21:57:42.905 | INFO | MemInfo: { | 20260505 21:57:42.905 | INFO | "gpu" : " 17 GB", | 20260505 21:57:42.905 | INFO | "huge" : "", | 20260505 21:57:42.905 | INFO | "huge_shared" : " 27 GB", | 20260505 21:57:42.905 | INFO | "malloc" : " 14 GB", | 20260505 21:57:42.905 | INFO | "pinned" : " 29 GB" | 20260505 21:57:42.905 | INFO | } | 20260505 21:57:42.905 | INFO | Perf report for the last 5.13417 seconds: | 20260505 21:57:42.905 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:57:42.905 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:42.905 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:42.905 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:42.905 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260505 21:57:42.905 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:42.905 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:42.905 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 21:57:42.905 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:42.905 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:42.905 | INFO | Uploaded 36GiB to GPU in 2.13716s (16.8448GiB/s) | 20260505 21:57:42.905 | INFO | Downloaded 34GiB to GPU in 2.01948s (16.836GiB/s) | 20260505 21:57:48.051 | INFO | Analyzing chunk frames = [6612480,6612992) | 20260505 21:57:48.051 | INFO | MemInfo: { | 20260505 21:57:48.051 | INFO | "gpu" : " 17 GB", | 20260505 21:57:48.051 | INFO | "huge" : "", | 20260505 21:57:48.051 | INFO | "huge_shared" : " 27 GB", | 20260505 21:57:48.051 | INFO | "malloc" : " 14 GB", | 20260505 21:57:48.051 | INFO | "pinned" : " 29 GB" | 20260505 21:57:48.051 | INFO | } | 20260505 21:57:48.051 | INFO | Perf report for the last 5.1462 seconds: | 20260505 21:57:48.051 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:57:48.051 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:48.051 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:48.051 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:48.051 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260505 21:57:48.051 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:48.051 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:48.051 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:57:48.051 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:48.052 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:48.052 | INFO | Uploaded 36GiB to GPU in 2.20529s (16.3244GiB/s) | 20260505 21:57:48.052 | INFO | Downloaded 34GiB to GPU in 2.04544s (16.6224GiB/s) | 20260505 21:57:53.202 | INFO | Analyzing chunk frames = [6612992,6613504) | 20260505 21:57:53.202 | INFO | MemInfo: { | 20260505 21:57:53.202 | INFO | "gpu" : " 17 GB", | 20260505 21:57:53.202 | INFO | "huge" : "", | 20260505 21:57:53.202 | INFO | "huge_shared" : " 27 GB", | 20260505 21:57:53.202 | INFO | "malloc" : " 14 GB", | 20260505 21:57:53.202 | INFO | "pinned" : " 29 GB" | 20260505 21:57:53.202 | INFO | } | 20260505 21:57:53.202 | INFO | Perf report for the last 5.15067 seconds: | 20260505 21:57:53.202 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:57:53.202 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:53.202 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:53.202 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:53.202 | INFO | ANALYSIS : 61.6%, 3.0, 96.3ms, 0 | 20260505 21:57:53.202 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:53.202 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:53.202 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 21:57:53.202 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:53.202 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:53.202 | INFO | Uploaded 36GiB to GPU in 2.16694s (16.6133GiB/s) | 20260505 21:57:53.202 | INFO | Downloaded 34GiB to GPU in 2.07311s (16.4005GiB/s) | 20260505 21:57:58.335 | INFO | Analyzing chunk frames = [6613504,6614016) | 20260505 21:57:58.335 | INFO | MemInfo: { | 20260505 21:57:58.335 | INFO | "gpu" : " 17 GB", | 20260505 21:57:58.335 | INFO | "huge" : "", | 20260505 21:57:58.335 | INFO | "huge_shared" : " 27 GB", | 20260505 21:57:58.335 | INFO | "malloc" : " 14 GB", | 20260505 21:57:58.335 | INFO | "pinned" : " 29 GB" | 20260505 21:57:58.335 | INFO | } | 20260505 21:57:58.335 | INFO | Perf report for the last 5.13326 seconds: | 20260505 21:57:58.335 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:57:58.335 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:58.335 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:58.336 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:58.336 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260505 21:57:58.336 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:58.336 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:58.336 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 21:57:58.336 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:58.336 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:57:58.336 | INFO | Uploaded 36GiB to GPU in 2.06389s (17.4428GiB/s) | 20260505 21:57:58.336 | INFO | Downloaded 34GiB to GPU in 2.03516s (16.7063GiB/s) | 20260505 21:58:03.469 | INFO | Analyzing chunk frames = [6614016,6614528) | 20260505 21:58:03.469 | INFO | MemInfo: { | 20260505 21:58:03.469 | INFO | "gpu" : " 17 GB", | 20260505 21:58:03.469 | INFO | "huge" : "", | 20260505 21:58:03.469 | INFO | "huge_shared" : " 27 GB", | 20260505 21:58:03.469 | INFO | "malloc" : " 14 GB", | 20260505 21:58:03.469 | INFO | "pinned" : " 29 GB" | 20260505 21:58:03.469 | INFO | } | 20260505 21:58:03.469 | INFO | Perf report for the last 5.13369 seconds: | 20260505 21:58:03.469 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:58:03.469 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:03.469 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:03.469 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:03.469 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 21:58:03.469 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:03.469 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:03.469 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 21:58:03.469 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:03.469 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:03.469 | INFO | Uploaded 36GiB to GPU in 2.08254s (17.2866GiB/s) | 20260505 21:58:03.470 | INFO | Downloaded 34GiB to GPU in 2.07037s (16.4222GiB/s) | 20260505 21:58:08.606 | INFO | Analyzing chunk frames = [6614528,6615040) | 20260505 21:58:08.606 | INFO | MemInfo: { | 20260505 21:58:08.606 | INFO | "gpu" : " 17 GB", | 20260505 21:58:08.606 | INFO | "huge" : "", | 20260505 21:58:08.606 | INFO | "huge_shared" : " 27 GB", | 20260505 21:58:08.606 | INFO | "malloc" : " 14 GB", | 20260505 21:58:08.606 | INFO | "pinned" : " 29 GB" | 20260505 21:58:08.606 | INFO | } | 20260505 21:58:08.606 | INFO | Perf report for the last 5.13682 seconds: | 20260505 21:58:08.606 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:58:08.606 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:08.606 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:08.606 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:08.606 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 21:58:08.606 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:08.606 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:08.606 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 21:58:08.606 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:08.606 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:08.606 | INFO | Uploaded 36GiB to GPU in 2.10185s (17.1277GiB/s) | 20260505 21:58:08.606 | INFO | Downloaded 34GiB to GPU in 2.00318s (16.973GiB/s) | 20260505 21:58:08.741 | INFO | Starting a round of Baz IO | 20260505 21:58:11.893 | INFO | Populating RTMetrics report | 20260505 21:58:11.893 | INFO | RTMetrics report populated | 20260505 21:58:11.893 | INFO | Opening tmp RTMetrics json file | 20260505 21:58:11.894 | INFO | Serializing RTMetrics to json | 20260505 21:58:11.895 | INFO | Serialization done, swapping tmp file now... | 20260505 21:58:11.895 | INFO | RTMetrics done | 20260505 21:58:11.972 | INFO | Finished a round of Baz IO: Wrote 1.95 GiB in 3.23 seconds | 20260505 21:58:11.972 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.53% / 0.00% / 78.47% | 20260505 21:58:11.972 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.135008% / 7.94729e-06% / 7.15256e-05% | 20260505 21:58:11.972 | INFO | Baz IO overall: | 20260505 21:58:11.972 | INFO | BytesWritten : 3819713369946 | 20260505 21:58:11.972 | INFO | NumEvents : 1802631201631 | 20260505 21:58:13.750 | INFO | Analyzing chunk frames = [6615040,6615552) | 20260505 21:58:13.750 | INFO | MemInfo: { | 20260505 21:58:13.750 | INFO | "gpu" : " 17 GB", | 20260505 21:58:13.750 | INFO | "huge" : "", | 20260505 21:58:13.750 | INFO | "huge_shared" : " 27 GB", | 20260505 21:58:13.750 | INFO | "malloc" : " 14 GB", | 20260505 21:58:13.750 | INFO | "pinned" : " 29 GB" | 20260505 21:58:13.750 | INFO | } | 20260505 21:58:13.750 | INFO | Perf report for the last 5.14427 seconds: | 20260505 21:58:13.750 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:58:13.750 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:13.750 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:13.750 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:13.750 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 21:58:13.750 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:13.750 | INFO | RT_METRICS : 43.7%, 1.0, 23.0ms, 0 | 20260505 21:58:13.750 | INFO | PRE_HQ : 59.6%, 1.8, 55.1ms, 0 | 20260505 21:58:13.750 | INFO | BAZWRITER : 62.3%, 3.6, 116.1ms, 0 | 20260505 21:58:13.750 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 21:58:13.750 | INFO | Uploaded 36GiB to GPU in 2.15977s (16.6684GiB/s) | 20260505 21:58:13.750 | INFO | Downloaded 38GiB to GPU in 2.23948s (16.9682GiB/s) | 20260505 21:58:18.889 | INFO | Analyzing chunk frames = [6615552,6616064) | 20260505 21:58:18.889 | INFO | MemInfo: { | 20260505 21:58:18.889 | INFO | "gpu" : " 17 GB", | 20260505 21:58:18.889 | INFO | "huge" : "", | 20260505 21:58:18.889 | INFO | "huge_shared" : " 27 GB", | 20260505 21:58:18.889 | INFO | "malloc" : " 14 GB", | 20260505 21:58:18.889 | INFO | "pinned" : " 29 GB" | 20260505 21:58:18.889 | INFO | } | 20260505 21:58:18.889 | INFO | Perf report for the last 5.13935 seconds: | 20260505 21:58:18.890 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:58:18.890 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:18.890 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:18.890 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:18.890 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 21:58:18.890 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:18.890 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:18.890 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 21:58:18.890 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:18.890 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:18.890 | INFO | Uploaded 36GiB to GPU in 2.08039s (17.3044GiB/s) | 20260505 21:58:18.890 | INFO | Downloaded 34GiB to GPU in 2.02044s (16.828GiB/s) | 20260505 21:58:24.024 | INFO | Analyzing chunk frames = [6616064,6616576) | 20260505 21:58:24.024 | INFO | MemInfo: { | 20260505 21:58:24.024 | INFO | "gpu" : " 17 GB", | 20260505 21:58:24.024 | INFO | "huge" : "", | 20260505 21:58:24.024 | INFO | "huge_shared" : " 27 GB", | 20260505 21:58:24.024 | INFO | "malloc" : " 14 GB", | 20260505 21:58:24.024 | INFO | "pinned" : " 29 GB" | 20260505 21:58:24.024 | INFO | } | 20260505 21:58:24.024 | INFO | Perf report for the last 5.13447 seconds: | 20260505 21:58:24.024 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:58:24.024 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:24.024 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:24.024 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:24.024 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260505 21:58:24.024 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:24.024 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:24.024 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 21:58:24.024 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:24.024 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:24.024 | INFO | Uploaded 36GiB to GPU in 2.1244s (16.9459GiB/s) | 20260505 21:58:24.024 | INFO | Downloaded 34GiB to GPU in 2.01026s (16.9133GiB/s) | 20260505 21:58:29.165 | INFO | Analyzing chunk frames = [6616576,6617088) | 20260505 21:58:29.165 | INFO | MemInfo: { | 20260505 21:58:29.165 | INFO | "gpu" : " 17 GB", | 20260505 21:58:29.165 | INFO | "huge" : "", | 20260505 21:58:29.165 | INFO | "huge_shared" : " 27 GB", | 20260505 21:58:29.165 | INFO | "malloc" : " 14 GB", | 20260505 21:58:29.165 | INFO | "pinned" : " 29 GB" | 20260505 21:58:29.165 | INFO | } | 20260505 21:58:29.165 | INFO | Perf report for the last 5.14105 seconds: | 20260505 21:58:29.165 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:58:29.165 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:29.165 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:29.165 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:29.165 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260505 21:58:29.165 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:29.165 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:29.165 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:58:29.165 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:29.165 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:29.165 | INFO | Uploaded 36GiB to GPU in 2.21005s (16.2892GiB/s) | 20260505 21:58:29.165 | INFO | Downloaded 34GiB to GPU in 2.06809s (16.4403GiB/s) | 20260505 21:58:34.313 | INFO | Analyzing chunk frames = [6617088,6617600) | 20260505 21:58:34.313 | INFO | MemInfo: { | 20260505 21:58:34.313 | INFO | "gpu" : " 17 GB", | 20260505 21:58:34.313 | INFO | "huge" : "", | 20260505 21:58:34.313 | INFO | "huge_shared" : " 27 GB", | 20260505 21:58:34.313 | INFO | "malloc" : " 14 GB", | 20260505 21:58:34.313 | INFO | "pinned" : " 29 GB" | 20260505 21:58:34.313 | INFO | } | 20260505 21:58:34.313 | INFO | Perf report for the last 5.14775 seconds: | 20260505 21:58:34.313 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:58:34.313 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:34.313 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:34.313 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:34.313 | INFO | ANALYSIS : 61.5%, 3.0, 96.2ms, 0 | 20260505 21:58:34.313 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:34.313 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:34.313 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:58:34.313 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:34.313 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:34.313 | INFO | Uploaded 36GiB to GPU in 2.13952s (16.8262GiB/s) | 20260505 21:58:34.313 | INFO | Downloaded 34GiB to GPU in 2.06431s (16.4704GiB/s) | 20260505 21:58:39.449 | INFO | Analyzing chunk frames = [6617600,6618112) | 20260505 21:58:39.449 | INFO | MemInfo: { | 20260505 21:58:39.449 | INFO | "gpu" : " 17 GB", | 20260505 21:58:39.449 | INFO | "huge" : "", | 20260505 21:58:39.449 | INFO | "huge_shared" : " 27 GB", | 20260505 21:58:39.449 | INFO | "malloc" : " 14 GB", | 20260505 21:58:39.449 | INFO | "pinned" : " 29 GB" | 20260505 21:58:39.449 | INFO | } | 20260505 21:58:39.449 | INFO | Perf report for the last 5.13592 seconds: | 20260505 21:58:39.449 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:58:39.449 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:39.449 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:39.449 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:39.449 | INFO | ANALYSIS : 61.5%, 3.0, 96.0ms, 0 | 20260505 21:58:39.449 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:39.449 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:39.449 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:58:39.449 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:39.449 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:39.449 | INFO | Uploaded 36GiB to GPU in 2.07057s (17.3865GiB/s) | 20260505 21:58:39.449 | INFO | Downloaded 34GiB to GPU in 2.03027s (16.7465GiB/s) | 20260505 21:58:44.582 | INFO | Analyzing chunk frames = [6618112,6618624) | 20260505 21:58:44.582 | INFO | MemInfo: { | 20260505 21:58:44.582 | INFO | "gpu" : " 17 GB", | 20260505 21:58:44.582 | INFO | "huge" : "", | 20260505 21:58:44.582 | INFO | "huge_shared" : " 27 GB", | 20260505 21:58:44.582 | INFO | "malloc" : " 14 GB", | 20260505 21:58:44.582 | INFO | "pinned" : " 29 GB" | 20260505 21:58:44.582 | INFO | } | 20260505 21:58:44.582 | INFO | Perf report for the last 5.13339 seconds: | 20260505 21:58:44.582 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:58:44.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:44.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:44.582 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:44.582 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260505 21:58:44.582 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:44.582 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:44.582 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 21:58:44.582 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:44.582 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:44.582 | INFO | Uploaded 36GiB to GPU in 2.08444s (17.2708GiB/s) | 20260505 21:58:44.582 | INFO | Downloaded 34GiB to GPU in 2.07916s (16.3528GiB/s) | 20260505 21:58:49.716 | INFO | Analyzing chunk frames = [6618624,6619136) | 20260505 21:58:49.716 | INFO | MemInfo: { | 20260505 21:58:49.716 | INFO | "gpu" : " 17 GB", | 20260505 21:58:49.716 | INFO | "huge" : "", | 20260505 21:58:49.716 | INFO | "huge_shared" : " 27 GB", | 20260505 21:58:49.716 | INFO | "malloc" : " 14 GB", | 20260505 21:58:49.716 | INFO | "pinned" : " 29 GB" | 20260505 21:58:49.716 | INFO | } | 20260505 21:58:49.716 | INFO | Perf report for the last 5.13413 seconds: | 20260505 21:58:49.716 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:58:49.716 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:49.716 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:49.716 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:49.716 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 21:58:49.716 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:49.716 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:49.716 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 21:58:49.716 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:49.716 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:49.716 | INFO | Uploaded 36GiB to GPU in 2.10326s (17.1163GiB/s) | 20260505 21:58:49.716 | INFO | Downloaded 34GiB to GPU in 2.00427s (16.9638GiB/s) | 20260505 21:58:49.717 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 21:58:49.717 | INFO | Total mem usage: 15.3 GiB | 20260505 21:58:49.717 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 21:58:49.717 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.18%) | 20260505 21:58:49.717 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 21:58:49.717 | INFO | High Water Breakdown: | 20260505 21:58:49.717 | INFO | 61% : 8.65 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 21:58:49.717 | INFO | 38.8% : 5.5 GiB ( 82.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 21:58:49.717 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 21:58:49.717 | INFO | ------------------------------------------ | 20260505 21:58:49.717 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:58:49.717 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 21:58:49.717 | INFO | Total mem usage: 26.5 GiB | 20260505 21:58:49.717 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 21:58:49.717 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 21:58:49.717 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 21:58:49.717 | INFO | High Water Breakdown: | 20260505 21:58:49.717 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:58:49.717 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:58:49.717 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:58:49.717 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:58:49.717 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:58:49.717 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:58:49.717 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 21:58:49.717 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 21:58:49.717 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:58:49.717 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 21:58:49.717 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 21:58:49.717 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 21:58:49.717 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 21:58:49.717 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 21:58:49.717 | INFO | ------------------------------------------ | 20260505 21:58:49.717 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:58:49.717 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 21:58:49.717 | INFO | Total mem usage: 25.6 GiB | 20260505 21:58:49.717 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 21:58:49.717 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 21:58:49.717 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 21:58:49.717 | INFO | High Water Breakdown: | 20260505 21:58:49.717 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 21:58:49.717 | INFO | ------------------------------------------ | 20260505 21:58:49.717 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:58:49.717 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 21:58:49.717 | INFO | Total mem usage: 15.7 GiB | 20260505 21:58:49.717 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 21:58:49.717 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 21:58:49.717 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 21:58:49.717 | INFO | High Water Breakdown: | 20260505 21:58:49.717 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 21:58:49.717 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 21:58:49.717 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 21:58:49.717 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 21:58:49.717 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 21:58:49.717 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 21:58:49.717 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 21:58:49.717 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 21:58:49.717 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 21:58:49.717 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 21:58:49.717 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 21:58:49.717 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 21:58:49.717 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 21:58:49.717 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 21:58:49.717 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 21:58:49.717 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 21:58:49.717 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 21:58:49.717 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 21:58:49.718 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 21:58:49.718 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 21:58:49.718 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 21:58:49.718 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 21:58:49.718 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 21:58:49.718 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 21:58:49.718 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 21:58:49.718 | INFO | ------------------------------------------ | 20260505 21:58:49.718 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 21:58:49.718 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 21:58:49.718 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:58:49.718 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:58:49.718 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.40e+01, 1.34e+01 / 8.01e-01 | 20260505 21:58:49.718 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.03e-01, 1.03e-02 / 3.35e-03 | 20260505 21:58:49.718 | INFO | PerfReport: 11.81%, Baseline -- 3.71e+00 / 4.51e+00, 5.68e-01 / 2.23e-01 | 20260505 21:58:49.718 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.53e+00, 1.87e-01 / 9.61e-02 | 20260505 21:58:49.718 | INFO | PerfReport: 15.95%, DME -- 4.01e+01 / 4.40e+01, 3.87e+00 / 1.01e+00 | 20260505 21:58:49.718 | INFO | PerfReport: 57.19%, FrameLabeling -- 1.80e+01 / 1.93e+01, 1.50e+00 / 1.13e+00 | 20260505 21:58:49.718 | INFO | PerfReport: 7.20%, PulseAccumulating -- 2.26e+00 / 2.79e+00, 4.34e-01 / 2.41e-01 | 20260505 21:58:49.718 | INFO | PerfReport: 3.19%, Metrics -- 1.00e+00 / 1.30e+00, 2.70e-01 / 1.13e-01 | 20260505 21:58:49.718 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.47e-01 / 4.01e-01, 5.89e-02 / 3.28e-02 | 20260505 21:58:49.718 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 21:58:49.718 | INFO | PerfReport: Gathered 1536 samples. | 20260505 21:58:49.718 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 21:58:49.718 | INFO | PerfReport: Full Execution -- 9.88e+01 / -nan, 2.50e+01 / 0.00e+00 | 20260505 21:58:49.718 | INFO | PerfReport: 22.36%, Upload -- 2.21e+01 / 0.00e+00, 7.60e+00 / 0.00e+00 | 20260505 21:58:49.718 | INFO | PerfReport: 21.74%, Download -- 2.15e+01 / 0.00e+00, 7.56e+00 / 0.00e+00 | 20260505 21:58:49.718 | INFO | PerfReport: 31.79%, Compute -- 3.14e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 21:58:49.718 | INFO | PerfReport: 2.22%, LockUpload -- 2.19e+00 / 0.00e+00, 7.81e+00 / 0.00e+00 | 20260505 21:58:49.718 | INFO | PerfReport: 1.28%, LockDownload -- 1.26e+00 / 0.00e+00, 2.79e+00 / 0.00e+00 | 20260505 21:58:49.718 | INFO | PerfReport: 20.60%, LockCompute -- 2.04e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 21:58:49.718 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.71e-02 / 0.00e+00 | 20260505 21:58:49.845 | INFO | Starting a round of Baz IO | 20260505 21:58:53.011 | INFO | Populating RTMetrics report | 20260505 21:58:53.011 | INFO | RTMetrics report populated | 20260505 21:58:53.011 | INFO | Opening tmp RTMetrics json file | 20260505 21:58:53.015 | INFO | Serializing RTMetrics to json | 20260505 21:58:53.016 | INFO | Serialization done, swapping tmp file now... | 20260505 21:58:53.016 | INFO | RTMetrics done | 20260505 21:58:53.092 | INFO | Finished a round of Baz IO: Wrote 1.949 GiB in 3.246 seconds | 20260505 21:58:53.092 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.53% / 0.00% / 78.47% | 20260505 21:58:53.092 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.134623% / 0% / 7.54992e-05% | 20260505 21:58:53.092 | INFO | Baz IO overall: | 20260505 21:58:53.092 | INFO | BytesWritten : 3821806287029 | 20260505 21:58:53.092 | INFO | NumEvents : 1803595719853 | 20260505 21:58:54.862 | INFO | Analyzing chunk frames = [6619136,6619648) | 20260505 21:58:54.862 | INFO | MemInfo: { | 20260505 21:58:54.862 | INFO | "gpu" : " 17 GB", | 20260505 21:58:54.862 | INFO | "huge" : "", | 20260505 21:58:54.862 | INFO | "huge_shared" : " 27 GB", | 20260505 21:58:54.862 | INFO | "malloc" : " 14 GB", | 20260505 21:58:54.862 | INFO | "pinned" : " 29 GB" | 20260505 21:58:54.862 | INFO | } | 20260505 21:58:54.862 | INFO | Perf report for the last 5.14557 seconds: | 20260505 21:58:54.862 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:58:54.862 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:54.862 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:54.862 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:54.862 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 21:58:54.862 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:58:54.862 | INFO | RT_METRICS : 43.7%, 1.0, 23.0ms, 0 | 20260505 21:58:54.862 | INFO | PRE_HQ : 60.0%, 1.8, 54.6ms, 0 | 20260505 21:58:54.862 | INFO | BAZWRITER : 62.6%, 3.6, 115.4ms, 0 | 20260505 21:58:54.862 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 21:58:54.862 | INFO | Uploaded 36GiB to GPU in 2.1722s (16.573GiB/s) | 20260505 21:58:54.862 | INFO | Downloaded 38GiB to GPU in 2.25953s (16.8177GiB/s) | 20260505 21:59:00.001 | INFO | Analyzing chunk frames = [6619648,6620160) | 20260505 21:59:00.001 | INFO | MemInfo: { | 20260505 21:59:00.001 | INFO | "gpu" : " 17 GB", | 20260505 21:59:00.001 | INFO | "huge" : "", | 20260505 21:59:00.001 | INFO | "huge_shared" : " 27 GB", | 20260505 21:59:00.001 | INFO | "malloc" : " 14 GB", | 20260505 21:59:00.001 | INFO | "pinned" : " 29 GB" | 20260505 21:59:00.001 | INFO | } | 20260505 21:59:00.001 | INFO | Perf report for the last 5.13952 seconds: | 20260505 21:59:00.001 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:59:00.001 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:00.001 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:00.001 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:00.001 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 21:59:00.001 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:00.001 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:00.001 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 21:59:00.002 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:00.002 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:00.002 | INFO | Uploaded 36GiB to GPU in 2.08442s (17.271GiB/s) | 20260505 21:59:00.002 | INFO | Downloaded 34GiB to GPU in 2.02673s (16.7758GiB/s) | 20260505 21:59:05.149 | INFO | Analyzing chunk frames = [6620160,6620672) | 20260505 21:59:05.149 | INFO | MemInfo: { | 20260505 21:59:05.149 | INFO | "gpu" : " 17 GB", | 20260505 21:59:05.149 | INFO | "huge" : "", | 20260505 21:59:05.149 | INFO | "huge_shared" : " 27 GB", | 20260505 21:59:05.150 | INFO | "malloc" : " 14 GB", | 20260505 21:59:05.150 | INFO | "pinned" : " 29 GB" | 20260505 21:59:05.150 | INFO | } | 20260505 21:59:05.150 | INFO | Perf report for the last 5.1483 seconds: | 20260505 21:59:05.150 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:59:05.150 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:05.150 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:05.150 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:05.150 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260505 21:59:05.150 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:05.150 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:05.150 | INFO | PRE_HQ : 26.8%, 1.0, 14.0ms, 0 | 20260505 21:59:05.150 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:05.150 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:05.150 | INFO | Uploaded 36GiB to GPU in 2.13439s (16.8666GiB/s) | 20260505 21:59:05.150 | INFO | Downloaded 34GiB to GPU in 2.02016s (16.8304GiB/s) | 20260505 21:59:10.283 | INFO | Analyzing chunk frames = [6620672,6621184) | 20260505 21:59:10.283 | INFO | MemInfo: { | 20260505 21:59:10.283 | INFO | "gpu" : " 17 GB", | 20260505 21:59:10.283 | INFO | "huge" : "", | 20260505 21:59:10.283 | INFO | "huge_shared" : " 27 GB", | 20260505 21:59:10.284 | INFO | "malloc" : " 14 GB", | 20260505 21:59:10.284 | INFO | "pinned" : " 29 GB" | 20260505 21:59:10.284 | INFO | } | 20260505 21:59:10.284 | INFO | Perf report for the last 5.1339 seconds: | 20260505 21:59:10.284 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:59:10.284 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:10.284 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:10.284 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:10.284 | INFO | ANALYSIS : 62.0%, 3.0, 96.1ms, 0 | 20260505 21:59:10.284 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:10.284 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:10.284 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:59:10.284 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:10.284 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:10.284 | INFO | Uploaded 36GiB to GPU in 2.13657s (16.8494GiB/s) | 20260505 21:59:10.284 | INFO | Downloaded 34GiB to GPU in 2.03315s (16.7229GiB/s) | 20260505 21:59:15.416 | INFO | Analyzing chunk frames = [6621184,6621696) | 20260505 21:59:15.416 | INFO | MemInfo: { | 20260505 21:59:15.416 | INFO | "gpu" : " 17 GB", | 20260505 21:59:15.416 | INFO | "huge" : "", | 20260505 21:59:15.416 | INFO | "huge_shared" : " 27 GB", | 20260505 21:59:15.416 | INFO | "malloc" : " 14 GB", | 20260505 21:59:15.417 | INFO | "pinned" : " 29 GB" | 20260505 21:59:15.417 | INFO | } | 20260505 21:59:15.417 | INFO | Perf report for the last 5.13303 seconds: | 20260505 21:59:15.417 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:59:15.417 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:15.417 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:15.417 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:15.417 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260505 21:59:15.417 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:15.417 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:15.417 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 21:59:15.417 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:15.417 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:15.417 | INFO | Uploaded 36GiB to GPU in 2.13771s (16.8405GiB/s) | 20260505 21:59:15.417 | INFO | Downloaded 34GiB to GPU in 2.05679s (16.5306GiB/s) | 20260505 21:59:20.563 | INFO | Analyzing chunk frames = [6621696,6622208) | 20260505 21:59:20.563 | INFO | MemInfo: { | 20260505 21:59:20.563 | INFO | "gpu" : " 17 GB", | 20260505 21:59:20.563 | INFO | "huge" : "", | 20260505 21:59:20.563 | INFO | "huge_shared" : " 27 GB", | 20260505 21:59:20.563 | INFO | "malloc" : " 14 GB", | 20260505 21:59:20.563 | INFO | "pinned" : " 29 GB" | 20260505 21:59:20.563 | INFO | } | 20260505 21:59:20.563 | INFO | Perf report for the last 5.14641 seconds: | 20260505 21:59:20.563 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:59:20.563 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:20.563 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:20.563 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:20.563 | INFO | ANALYSIS : 61.6%, 3.0, 96.3ms, 0 | 20260505 21:59:20.563 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:20.563 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:20.563 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 21:59:20.563 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:20.563 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:20.563 | INFO | Uploaded 36GiB to GPU in 2.07598s (17.3412GiB/s) | 20260505 21:59:20.563 | INFO | Downloaded 34GiB to GPU in 2.03652s (16.6952GiB/s) | 20260505 21:59:25.701 | INFO | Analyzing chunk frames = [6622208,6622720) | 20260505 21:59:25.701 | INFO | MemInfo: { | 20260505 21:59:25.701 | INFO | "gpu" : " 17 GB", | 20260505 21:59:25.701 | INFO | "huge" : "", | 20260505 21:59:25.701 | INFO | "huge_shared" : " 27 GB", | 20260505 21:59:25.701 | INFO | "malloc" : " 14 GB", | 20260505 21:59:25.701 | INFO | "pinned" : " 29 GB" | 20260505 21:59:25.701 | INFO | } | 20260505 21:59:25.701 | INFO | Perf report for the last 5.13827 seconds: | 20260505 21:59:25.701 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:59:25.701 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:25.701 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:25.701 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:25.701 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 21:59:25.701 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:25.701 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:25.701 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 21:59:25.701 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:25.701 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:25.702 | INFO | Uploaded 36GiB to GPU in 2.10254s (17.1222GiB/s) | 20260505 21:59:25.702 | INFO | Downloaded 34GiB to GPU in 2.07595s (16.378GiB/s) | 20260505 21:59:30.841 | INFO | Analyzing chunk frames = [6622720,6623232) | 20260505 21:59:30.841 | INFO | MemInfo: { | 20260505 21:59:30.841 | INFO | "gpu" : " 17 GB", | 20260505 21:59:30.841 | INFO | "huge" : "", | 20260505 21:59:30.841 | INFO | "huge_shared" : " 27 GB", | 20260505 21:59:30.841 | INFO | "malloc" : " 14 GB", | 20260505 21:59:30.841 | INFO | "pinned" : " 29 GB" | 20260505 21:59:30.841 | INFO | } | 20260505 21:59:30.841 | INFO | Perf report for the last 5.13944 seconds: | 20260505 21:59:30.841 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:59:30.841 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:30.841 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:30.841 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:30.841 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 21:59:30.841 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:30.841 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:30.841 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 21:59:30.841 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:30.841 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:30.841 | INFO | Uploaded 36GiB to GPU in 2.1147s (17.0237GiB/s) | 20260505 21:59:30.841 | INFO | Downloaded 34GiB to GPU in 2.00645s (16.9453GiB/s) | 20260505 21:59:30.972 | INFO | Starting a round of Baz IO | 20260505 21:59:34.124 | INFO | Populating RTMetrics report | 20260505 21:59:34.124 | INFO | RTMetrics report populated | 20260505 21:59:34.124 | INFO | Opening tmp RTMetrics json file | 20260505 21:59:34.125 | INFO | Serializing RTMetrics to json | 20260505 21:59:34.126 | INFO | Serialization done, swapping tmp file now... | 20260505 21:59:34.127 | INFO | RTMetrics done | 20260505 21:59:34.209 | INFO | Finished a round of Baz IO: Wrote 1.951 GiB in 3.236 seconds | 20260505 21:59:34.209 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.53% / 0.00% / 78.47% | 20260505 21:59:34.209 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.134246% / 0% / 0.000103315% | 20260505 21:59:34.209 | INFO | Baz IO overall: | 20260505 21:59:34.209 | INFO | BytesWritten : 3823900873144 | 20260505 21:59:34.209 | INFO | NumEvents : 1804561419924 | 20260505 21:59:35.974 | INFO | Analyzing chunk frames = [6623232,6623744) | 20260505 21:59:35.974 | INFO | MemInfo: { | 20260505 21:59:35.974 | INFO | "gpu" : " 17 GB", | 20260505 21:59:35.974 | INFO | "huge" : "", | 20260505 21:59:35.974 | INFO | "huge_shared" : " 27 GB", | 20260505 21:59:35.974 | INFO | "malloc" : " 14 GB", | 20260505 21:59:35.974 | INFO | "pinned" : " 29 GB" | 20260505 21:59:35.974 | INFO | } | 20260505 21:59:35.974 | INFO | Perf report for the last 5.13328 seconds: | 20260505 21:59:35.974 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:59:35.974 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:35.974 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:35.974 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:35.974 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260505 21:59:35.974 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:35.974 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260505 21:59:35.974 | INFO | PRE_HQ : 60.1%, 1.8, 54.7ms, 0 | 20260505 21:59:35.974 | INFO | BAZWRITER : 62.6%, 3.6, 115.7ms, 0 | 20260505 21:59:35.974 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 21:59:35.975 | INFO | Uploaded 36GiB to GPU in 2.16736s (16.6101GiB/s) | 20260505 21:59:35.975 | INFO | Downloaded 38GiB to GPU in 2.24285s (16.9428GiB/s) | 20260505 21:59:41.120 | INFO | Analyzing chunk frames = [6623744,6624256) | 20260505 21:59:41.120 | INFO | MemInfo: { | 20260505 21:59:41.120 | INFO | "gpu" : " 17 GB", | 20260505 21:59:41.120 | INFO | "huge" : "", | 20260505 21:59:41.120 | INFO | "huge_shared" : " 27 GB", | 20260505 21:59:41.120 | INFO | "malloc" : " 14 GB", | 20260505 21:59:41.120 | INFO | "pinned" : " 29 GB" | 20260505 21:59:41.120 | INFO | } | 20260505 21:59:41.120 | INFO | Perf report for the last 5.14629 seconds: | 20260505 21:59:41.120 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:59:41.120 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:41.120 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:41.120 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:41.120 | INFO | ANALYSIS : 62.0%, 3.0, 97.0ms, 0 | 20260505 21:59:41.120 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:41.121 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:41.121 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 21:59:41.121 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:41.121 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:41.121 | INFO | Uploaded 36GiB to GPU in 2.08562s (17.2611GiB/s) | 20260505 21:59:41.121 | INFO | Downloaded 34GiB to GPU in 2.02444s (16.7948GiB/s) | 20260505 21:59:46.260 | INFO | Analyzing chunk frames = [6624256,6624768) | 20260505 21:59:46.260 | INFO | MemInfo: { | 20260505 21:59:46.260 | INFO | "gpu" : " 17 GB", | 20260505 21:59:46.260 | INFO | "huge" : "", | 20260505 21:59:46.261 | INFO | "huge_shared" : " 27 GB", | 20260505 21:59:46.261 | INFO | "malloc" : " 14 GB", | 20260505 21:59:46.261 | INFO | "pinned" : " 29 GB" | 20260505 21:59:46.261 | INFO | } | 20260505 21:59:46.261 | INFO | Perf report for the last 5.14031 seconds: | 20260505 21:59:46.261 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:59:46.261 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:46.261 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:46.261 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:46.261 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260505 21:59:46.261 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:46.261 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:46.261 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 21:59:46.261 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:46.261 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:46.261 | INFO | Uploaded 36GiB to GPU in 2.16043s (16.6633GiB/s) | 20260505 21:59:46.261 | INFO | Downloaded 34GiB to GPU in 2.04563s (16.6208GiB/s) | 20260505 21:59:51.394 | INFO | Analyzing chunk frames = [6624768,6625280) | 20260505 21:59:51.394 | INFO | MemInfo: { | 20260505 21:59:51.394 | INFO | "gpu" : " 17 GB", | 20260505 21:59:51.394 | INFO | "huge" : "", | 20260505 21:59:51.394 | INFO | "huge_shared" : " 27 GB", | 20260505 21:59:51.394 | INFO | "malloc" : " 14 GB", | 20260505 21:59:51.394 | INFO | "pinned" : " 29 GB" | 20260505 21:59:51.394 | INFO | } | 20260505 21:59:51.394 | INFO | Perf report for the last 5.13352 seconds: | 20260505 21:59:51.394 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:59:51.394 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:51.394 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:51.394 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:51.394 | INFO | ANALYSIS : 61.8%, 3.0, 95.9ms, 0 | 20260505 21:59:51.394 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:51.394 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:51.394 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 21:59:51.394 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:51.394 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:51.395 | INFO | Uploaded 36GiB to GPU in 2.12738s (16.9222GiB/s) | 20260505 21:59:51.395 | INFO | Downloaded 34GiB to GPU in 2.01839s (16.8451GiB/s) | 20260505 21:59:56.530 | INFO | Analyzing chunk frames = [6625280,6625792) | 20260505 21:59:56.530 | INFO | MemInfo: { | 20260505 21:59:56.530 | INFO | "gpu" : " 17 GB", | 20260505 21:59:56.530 | INFO | "huge" : "", | 20260505 21:59:56.530 | INFO | "huge_shared" : " 27 GB", | 20260505 21:59:56.530 | INFO | "malloc" : " 14 GB", | 20260505 21:59:56.530 | INFO | "pinned" : " 29 GB" | 20260505 21:59:56.530 | INFO | } | 20260505 21:59:56.530 | INFO | Perf report for the last 5.13624 seconds: | 20260505 21:59:56.530 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 21:59:56.530 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:56.531 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:56.531 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:56.531 | INFO | ANALYSIS : 61.6%, 3.0, 96.2ms, 0 | 20260505 21:59:56.531 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:56.531 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:56.531 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 21:59:56.531 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:56.531 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 21:59:56.531 | INFO | Uploaded 36GiB to GPU in 2.13761s (16.8412GiB/s) | 20260505 21:59:56.531 | INFO | Downloaded 34GiB to GPU in 2.06355s (16.4765GiB/s) | 20260505 22:00:01.670 | INFO | Analyzing chunk frames = [6625792,6626304) | 20260505 22:00:01.670 | INFO | MemInfo: { | 20260505 22:00:01.670 | INFO | "gpu" : " 17 GB", | 20260505 22:00:01.670 | INFO | "huge" : "", | 20260505 22:00:01.670 | INFO | "huge_shared" : " 27 GB", | 20260505 22:00:01.670 | INFO | "malloc" : " 14 GB", | 20260505 22:00:01.670 | INFO | "pinned" : " 29 GB" | 20260505 22:00:01.670 | INFO | } | 20260505 22:00:01.670 | INFO | Perf report for the last 5.13929 seconds: | 20260505 22:00:01.670 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:00:01.670 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:01.670 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:01.670 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:01.670 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260505 22:00:01.670 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:01.670 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:01.670 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:00:01.670 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:01.670 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:01.670 | INFO | Uploaded 36GiB to GPU in 2.09824s (17.1572GiB/s) | 20260505 22:00:01.670 | INFO | Downloaded 34GiB to GPU in 2.03689s (16.6921GiB/s) | 20260505 22:00:06.814 | INFO | Analyzing chunk frames = [6626304,6626816) | 20260505 22:00:06.814 | INFO | MemInfo: { | 20260505 22:00:06.814 | INFO | "gpu" : " 17 GB", | 20260505 22:00:06.814 | INFO | "huge" : "", | 20260505 22:00:06.814 | INFO | "huge_shared" : " 27 GB", | 20260505 22:00:06.814 | INFO | "malloc" : " 14 GB", | 20260505 22:00:06.814 | INFO | "pinned" : " 29 GB" | 20260505 22:00:06.814 | INFO | } | 20260505 22:00:06.814 | INFO | Perf report for the last 5.14418 seconds: | 20260505 22:00:06.814 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:00:06.814 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:06.814 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:06.814 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:06.814 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 22:00:06.814 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:06.814 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:06.814 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 22:00:06.814 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:06.814 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:06.814 | INFO | Uploaded 36GiB to GPU in 2.15072s (16.7385GiB/s) | 20260505 22:00:06.814 | INFO | Downloaded 34GiB to GPU in 2.11602s (16.0679GiB/s) | 20260505 22:00:11.959 | INFO | Analyzing chunk frames = [6626816,6627328) | 20260505 22:00:11.959 | INFO | MemInfo: { | 20260505 22:00:11.959 | INFO | "gpu" : " 17 GB", | 20260505 22:00:11.959 | INFO | "huge" : "", | 20260505 22:00:11.959 | INFO | "huge_shared" : " 27 GB", | 20260505 22:00:11.959 | INFO | "malloc" : " 14 GB", | 20260505 22:00:11.959 | INFO | "pinned" : " 29 GB" | 20260505 22:00:11.959 | INFO | } | 20260505 22:00:11.959 | INFO | Perf report for the last 5.14492 seconds: | 20260505 22:00:11.959 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:00:11.959 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:11.959 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:11.959 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:11.959 | INFO | ANALYSIS : 61.9%, 3.0, 96.8ms, 0 | 20260505 22:00:11.959 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:11.959 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:11.959 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:00:11.959 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:11.959 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:11.959 | INFO | Uploaded 36GiB to GPU in 2.11275s (17.0394GiB/s) | 20260505 22:00:11.959 | INFO | Downloaded 34GiB to GPU in 2.00209s (16.9822GiB/s) | 20260505 22:00:11.959 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 22:00:11.959 | INFO | Total mem usage: 15.3 GiB | 20260505 22:00:11.959 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 22:00:11.959 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -9.01%) | 20260505 22:00:11.959 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 22:00:11.959 | INFO | High Water Breakdown: | 20260505 22:00:11.959 | INFO | 60.9% : 8.56 GiB ( 90% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 22:00:11.959 | INFO | 39% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 22:00:11.959 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 22:00:11.960 | INFO | ------------------------------------------ | 20260505 22:00:11.960 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:00:11.960 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 22:00:11.960 | INFO | Total mem usage: 26.5 GiB | 20260505 22:00:11.960 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 22:00:11.960 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 22:00:11.960 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 22:00:11.960 | INFO | High Water Breakdown: | 20260505 22:00:11.960 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:00:11.960 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:00:11.960 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:00:11.960 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:00:11.960 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:00:11.960 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:00:11.960 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 22:00:11.960 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 22:00:11.960 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:00:11.960 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 22:00:11.960 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 22:00:11.960 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 22:00:11.960 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 22:00:11.960 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 22:00:11.960 | INFO | ------------------------------------------ | 20260505 22:00:11.960 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:00:11.960 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 22:00:11.960 | INFO | Total mem usage: 25.6 GiB | 20260505 22:00:11.960 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 22:00:11.960 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 22:00:11.960 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 22:00:11.960 | INFO | High Water Breakdown: | 20260505 22:00:11.960 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 22:00:11.960 | INFO | ------------------------------------------ | 20260505 22:00:11.960 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:00:11.960 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 22:00:11.960 | INFO | Total mem usage: 15.7 GiB | 20260505 22:00:11.960 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 22:00:11.960 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 22:00:11.960 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 22:00:11.960 | INFO | High Water Breakdown: | 20260505 22:00:11.960 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 22:00:11.960 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 22:00:11.960 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 22:00:11.960 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 22:00:11.960 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 22:00:11.960 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 22:00:11.960 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 22:00:11.960 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 22:00:11.960 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 22:00:11.960 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:00:11.960 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 22:00:11.960 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:00:11.960 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 22:00:11.960 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 22:00:11.960 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 22:00:11.960 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 22:00:11.960 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:00:11.960 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 22:00:11.960 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:00:11.960 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:00:11.960 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:00:11.960 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 22:00:11.960 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 22:00:11.960 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:00:11.960 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 22:00:11.960 | INFO | ------------------------------------------ | 20260505 22:00:11.960 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:00:11.960 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 22:00:11.960 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:00:11.960 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:00:11.960 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.39e+01, 1.35e+01 / 7.58e-01 | 20260505 22:00:11.960 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.04e-01, 1.05e-02 / 5.00e-03 | 20260505 22:00:11.960 | INFO | PerfReport: 11.80%, Baseline -- 3.71e+00 / 4.53e+00, 5.72e-01 / 1.76e-01 | 20260505 22:00:11.960 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.52e+00, 1.92e-01 / 8.46e-02 | 20260505 22:00:11.960 | INFO | PerfReport: 15.93%, DME -- 4.01e+01 / 4.38e+01, 3.81e+00 / 8.63e-01 | 20260505 22:00:11.960 | INFO | PerfReport: 57.27%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.56e+00 / 1.19e+00 | 20260505 22:00:11.960 | INFO | PerfReport: 7.15%, PulseAccumulating -- 2.25e+00 / 2.78e+00, 4.29e-01 / 1.87e-01 | 20260505 22:00:11.960 | INFO | PerfReport: 3.17%, Metrics -- 9.98e-01 / 1.29e+00, 2.74e-01 / 1.39e-01 | 20260505 22:00:11.960 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.47e-01 / 4.04e-01, 5.96e-02 / 3.27e-02 | 20260505 22:00:11.960 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 22:00:11.960 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:00:11.960 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:00:11.960 | INFO | PerfReport: Full Execution -- 9.89e+01 / -nan, 2.51e+01 / 0.00e+00 | 20260505 22:00:11.960 | INFO | PerfReport: 22.39%, Upload -- 2.21e+01 / 0.00e+00, 7.65e+00 / 0.00e+00 | 20260505 22:00:11.960 | INFO | PerfReport: 21.77%, Download -- 2.15e+01 / 0.00e+00, 7.63e+00 / 0.00e+00 | 20260505 22:00:11.960 | INFO | PerfReport: 31.80%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 22:00:11.961 | INFO | PerfReport: 2.18%, LockUpload -- 2.16e+00 / 0.00e+00, 7.86e+00 / 0.00e+00 | 20260505 22:00:11.961 | INFO | PerfReport: 1.31%, LockDownload -- 1.30e+00 / 0.00e+00, 2.90e+00 / 0.00e+00 | 20260505 22:00:11.961 | INFO | PerfReport: 20.53%, LockCompute -- 2.03e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 22:00:11.961 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 2.17e-02 / 0.00e+00 | 20260505 22:00:12.085 | INFO | Starting a round of Baz IO | 20260505 22:00:15.234 | INFO | Populating RTMetrics report | 20260505 22:00:15.235 | INFO | RTMetrics report populated | 20260505 22:00:15.235 | INFO | Opening tmp RTMetrics json file | 20260505 22:00:15.237 | INFO | Serializing RTMetrics to json | 20260505 22:00:15.240 | INFO | Serialization done, swapping tmp file now... | 20260505 22:00:15.242 | INFO | RTMetrics done | 20260505 22:00:15.320 | INFO | Finished a round of Baz IO: Wrote 1.947 GiB in 3.235 seconds | 20260505 22:00:15.320 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.53% / 0.00% / 78.47% | 20260505 22:00:15.320 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.134079% / 0% / 6.75519e-05% | 20260505 22:00:15.320 | INFO | Baz IO overall: | 20260505 22:00:15.320 | INFO | BytesWritten : 3825991148834 | 20260505 22:00:15.320 | INFO | NumEvents : 1805524047153 | 20260505 22:00:17.096 | INFO | Analyzing chunk frames = [6627328,6627840) | 20260505 22:00:17.096 | INFO | MemInfo: { | 20260505 22:00:17.096 | INFO | "gpu" : " 17 GB", | 20260505 22:00:17.096 | INFO | "huge" : "", | 20260505 22:00:17.096 | INFO | "huge_shared" : " 27 GB", | 20260505 22:00:17.096 | INFO | "malloc" : " 14 GB", | 20260505 22:00:17.096 | INFO | "pinned" : " 29 GB" | 20260505 22:00:17.096 | INFO | } | 20260505 22:00:17.096 | INFO | Perf report for the last 5.13758 seconds: | 20260505 22:00:17.097 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:00:17.097 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:17.097 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:17.097 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:17.097 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 22:00:17.097 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:17.097 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260505 22:00:17.097 | INFO | PRE_HQ : 60.1%, 1.8, 54.8ms, 0 | 20260505 22:00:17.097 | INFO | BAZWRITER : 62.2%, 3.6, 116.3ms, 0 | 20260505 22:00:17.097 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 22:00:17.097 | INFO | Uploaded 36GiB to GPU in 2.1579s (16.6829GiB/s) | 20260505 22:00:17.097 | INFO | Downloaded 38GiB to GPU in 2.23522s (17.0006GiB/s) | 20260505 22:00:22.230 | INFO | Analyzing chunk frames = [6627840,6628352) | 20260505 22:00:22.230 | INFO | MemInfo: { | 20260505 22:00:22.230 | INFO | "gpu" : " 17 GB", | 20260505 22:00:22.230 | INFO | "huge" : "", | 20260505 22:00:22.230 | INFO | "huge_shared" : " 27 GB", | 20260505 22:00:22.230 | INFO | "malloc" : " 14 GB", | 20260505 22:00:22.230 | INFO | "pinned" : " 29 GB" | 20260505 22:00:22.230 | INFO | } | 20260505 22:00:22.230 | INFO | Perf report for the last 5.1336 seconds: | 20260505 22:00:22.230 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:00:22.230 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:22.230 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:22.230 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:22.230 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260505 22:00:22.230 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:22.230 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:22.230 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:00:22.230 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:22.230 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:22.230 | INFO | Uploaded 36GiB to GPU in 2.09154s (17.2122GiB/s) | 20260505 22:00:22.230 | INFO | Downloaded 34GiB to GPU in 2.01808s (16.8477GiB/s) | 20260505 22:00:27.371 | INFO | Analyzing chunk frames = [6628352,6628864) | 20260505 22:00:27.371 | INFO | MemInfo: { | 20260505 22:00:27.371 | INFO | "gpu" : " 17 GB", | 20260505 22:00:27.371 | INFO | "huge" : "", | 20260505 22:00:27.371 | INFO | "huge_shared" : " 27 GB", | 20260505 22:00:27.371 | INFO | "malloc" : " 14 GB", | 20260505 22:00:27.371 | INFO | "pinned" : " 29 GB" | 20260505 22:00:27.371 | INFO | } | 20260505 22:00:27.371 | INFO | Perf report for the last 5.14058 seconds: | 20260505 22:00:27.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:00:27.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:27.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:27.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:27.371 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260505 22:00:27.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:27.371 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:27.371 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:00:27.371 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:27.371 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:27.371 | INFO | Uploaded 36GiB to GPU in 2.13382s (16.8711GiB/s) | 20260505 22:00:27.371 | INFO | Downloaded 34GiB to GPU in 2.02024s (16.8297GiB/s) | 20260505 22:00:32.514 | INFO | Analyzing chunk frames = [6628864,6629376) | 20260505 22:00:32.514 | INFO | MemInfo: { | 20260505 22:00:32.514 | INFO | "gpu" : " 17 GB", | 20260505 22:00:32.514 | INFO | "huge" : "", | 20260505 22:00:32.514 | INFO | "huge_shared" : " 27 GB", | 20260505 22:00:32.514 | INFO | "malloc" : " 14 GB", | 20260505 22:00:32.514 | INFO | "pinned" : " 29 GB" | 20260505 22:00:32.514 | INFO | } | 20260505 22:00:32.514 | INFO | Perf report for the last 5.14364 seconds: | 20260505 22:00:32.514 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:00:32.514 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:32.514 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:32.514 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:32.514 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260505 22:00:32.514 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:32.514 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:32.514 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:00:32.514 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:32.514 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:32.515 | INFO | Uploaded 36GiB to GPU in 2.12766s (16.92GiB/s) | 20260505 22:00:32.515 | INFO | Downloaded 34GiB to GPU in 2.02719s (16.772GiB/s) | 20260505 22:00:37.651 | INFO | Analyzing chunk frames = [6629376,6629888) | 20260505 22:00:37.651 | INFO | MemInfo: { | 20260505 22:00:37.651 | INFO | "gpu" : " 17 GB", | 20260505 22:00:37.651 | INFO | "huge" : "", | 20260505 22:00:37.652 | INFO | "huge_shared" : " 27 GB", | 20260505 22:00:37.652 | INFO | "malloc" : " 14 GB", | 20260505 22:00:37.652 | INFO | "pinned" : " 29 GB" | 20260505 22:00:37.652 | INFO | } | 20260505 22:00:37.652 | INFO | Perf report for the last 5.1374 seconds: | 20260505 22:00:37.652 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:00:37.652 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:37.652 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:37.652 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:37.652 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260505 22:00:37.652 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:37.652 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:37.652 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 22:00:37.652 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:37.652 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:37.652 | INFO | Uploaded 36GiB to GPU in 2.09381s (17.1935GiB/s) | 20260505 22:00:37.652 | INFO | Downloaded 34GiB to GPU in 2.04309s (16.6415GiB/s) | 20260505 22:00:42.789 | INFO | Analyzing chunk frames = [6629888,6630400) | 20260505 22:00:42.789 | INFO | MemInfo: { | 20260505 22:00:42.789 | INFO | "gpu" : " 17 GB", | 20260505 22:00:42.789 | INFO | "huge" : "", | 20260505 22:00:42.789 | INFO | "huge_shared" : " 27 GB", | 20260505 22:00:42.789 | INFO | "malloc" : " 14 GB", | 20260505 22:00:42.789 | INFO | "pinned" : " 29 GB" | 20260505 22:00:42.789 | INFO | } | 20260505 22:00:42.789 | INFO | Perf report for the last 5.13758 seconds: | 20260505 22:00:42.789 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:00:42.789 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:42.789 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:42.789 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:42.789 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260505 22:00:42.789 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:42.789 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:42.789 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:00:42.790 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:42.790 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:42.790 | INFO | Uploaded 36GiB to GPU in 2.07393s (17.3584GiB/s) | 20260505 22:00:42.790 | INFO | Downloaded 34GiB to GPU in 2.0348s (16.7093GiB/s) | 20260505 22:00:47.927 | INFO | Analyzing chunk frames = [6630400,6630912) | 20260505 22:00:47.927 | INFO | MemInfo: { | 20260505 22:00:47.927 | INFO | "gpu" : " 17 GB", | 20260505 22:00:47.927 | INFO | "huge" : "", | 20260505 22:00:47.927 | INFO | "huge_shared" : " 27 GB", | 20260505 22:00:47.927 | INFO | "malloc" : " 14 GB", | 20260505 22:00:47.927 | INFO | "pinned" : " 29 GB" | 20260505 22:00:47.927 | INFO | } | 20260505 22:00:47.927 | INFO | Perf report for the last 5.13794 seconds: | 20260505 22:00:47.927 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:00:47.927 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:47.927 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:47.927 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:47.927 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260505 22:00:47.927 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:47.927 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:47.927 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 22:00:47.927 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:47.927 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:47.928 | INFO | Uploaded 36GiB to GPU in 2.13395s (16.8701GiB/s) | 20260505 22:00:47.928 | INFO | Downloaded 34GiB to GPU in 2.1216s (16.0256GiB/s) | 20260505 22:00:53.072 | INFO | Analyzing chunk frames = [6630912,6631424) | 20260505 22:00:53.072 | INFO | MemInfo: { | 20260505 22:00:53.072 | INFO | "gpu" : " 17 GB", | 20260505 22:00:53.072 | INFO | "huge" : "", | 20260505 22:00:53.072 | INFO | "huge_shared" : " 27 GB", | 20260505 22:00:53.072 | INFO | "malloc" : " 14 GB", | 20260505 22:00:53.072 | INFO | "pinned" : " 29 GB" | 20260505 22:00:53.072 | INFO | } | 20260505 22:00:53.072 | INFO | Perf report for the last 5.14449 seconds: | 20260505 22:00:53.072 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:00:53.072 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:53.072 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:53.072 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:53.072 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260505 22:00:53.072 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:53.072 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:53.072 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:00:53.072 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:53.072 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:53.072 | INFO | Uploaded 36GiB to GPU in 2.10525s (17.1001GiB/s) | 20260505 22:00:53.072 | INFO | Downloaded 34GiB to GPU in 2.00257s (16.9782GiB/s) | 20260505 22:00:53.193 | INFO | Starting a round of Baz IO | 20260505 22:00:56.342 | INFO | Populating RTMetrics report | 20260505 22:00:56.343 | INFO | RTMetrics report populated | 20260505 22:00:56.343 | INFO | Opening tmp RTMetrics json file | 20260505 22:00:56.345 | INFO | Serializing RTMetrics to json | 20260505 22:00:56.346 | INFO | Serialization done, swapping tmp file now... | 20260505 22:00:56.346 | INFO | RTMetrics done | 20260505 22:00:56.424 | INFO | Finished a round of Baz IO: Wrote 1.949 GiB in 3.23 seconds | 20260505 22:00:56.424 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.53% / 0.00% / 78.47% | 20260505 22:00:56.424 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.133876% / 3.97364e-06% / 0.000123183% | 20260505 22:00:56.424 | INFO | Baz IO overall: | 20260505 22:00:56.424 | INFO | BytesWritten : 3828083807423 | 20260505 22:00:56.424 | INFO | NumEvents : 1806488860699 | 20260505 22:00:58.210 | INFO | Analyzing chunk frames = [6631424,6631936) | 20260505 22:00:58.210 | INFO | MemInfo: { | 20260505 22:00:58.210 | INFO | "gpu" : " 17 GB", | 20260505 22:00:58.210 | INFO | "huge" : "", | 20260505 22:00:58.210 | INFO | "huge_shared" : " 27 GB", | 20260505 22:00:58.210 | INFO | "malloc" : " 14 GB", | 20260505 22:00:58.210 | INFO | "pinned" : " 29 GB" | 20260505 22:00:58.210 | INFO | } | 20260505 22:00:58.211 | INFO | Perf report for the last 5.13877 seconds: | 20260505 22:00:58.211 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:00:58.211 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:58.211 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:58.211 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:58.211 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 22:00:58.211 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:00:58.211 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260505 22:00:58.211 | INFO | PRE_HQ : 59.9%, 1.8, 54.5ms, 0 | 20260505 22:00:58.211 | INFO | BAZWRITER : 62.1%, 3.6, 114.9ms, 0 | 20260505 22:00:58.211 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 22:00:58.211 | INFO | Uploaded 36GiB to GPU in 2.16646s (16.617GiB/s) | 20260505 22:00:58.211 | INFO | Downloaded 38GiB to GPU in 2.23781s (16.9809GiB/s) | 20260505 22:01:03.349 | INFO | Analyzing chunk frames = [6631936,6632448) | 20260505 22:01:03.349 | INFO | MemInfo: { | 20260505 22:01:03.349 | INFO | "gpu" : " 17 GB", | 20260505 22:01:03.349 | INFO | "huge" : "", | 20260505 22:01:03.349 | INFO | "huge_shared" : " 27 GB", | 20260505 22:01:03.349 | INFO | "malloc" : " 14 GB", | 20260505 22:01:03.349 | INFO | "pinned" : " 29 GB" | 20260505 22:01:03.349 | INFO | } | 20260505 22:01:03.349 | INFO | Perf report for the last 5.13866 seconds: | 20260505 22:01:03.349 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:01:03.349 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:03.349 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:03.349 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:03.349 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260505 22:01:03.349 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:03.349 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:03.349 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:01:03.349 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:03.349 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:03.350 | INFO | Uploaded 36GiB to GPU in 2.09724s (17.1654GiB/s) | 20260505 22:01:03.350 | INFO | Downloaded 34GiB to GPU in 2.03299s (16.7241GiB/s) | 20260505 22:01:08.479 | INFO | Analyzing chunk frames = [6632448,6632960) | 20260505 22:01:08.479 | INFO | MemInfo: { | 20260505 22:01:08.479 | INFO | "gpu" : " 17 GB", | 20260505 22:01:08.479 | INFO | "huge" : "", | 20260505 22:01:08.479 | INFO | "huge_shared" : " 27 GB", | 20260505 22:01:08.480 | INFO | "malloc" : " 14 GB", | 20260505 22:01:08.480 | INFO | "pinned" : " 29 GB" | 20260505 22:01:08.480 | INFO | } | 20260505 22:01:08.480 | INFO | Perf report for the last 5.13034 seconds: | 20260505 22:01:08.480 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:01:08.480 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:08.480 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:08.480 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:08.480 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260505 22:01:08.480 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:08.480 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:08.480 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 22:01:08.480 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:08.480 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:08.480 | INFO | Uploaded 36GiB to GPU in 2.14803s (16.7595GiB/s) | 20260505 22:01:08.480 | INFO | Downloaded 34GiB to GPU in 2.0297s (16.7512GiB/s) | 20260505 22:01:13.625 | INFO | Analyzing chunk frames = [6632960,6633472) | 20260505 22:01:13.625 | INFO | MemInfo: { | 20260505 22:01:13.625 | INFO | "gpu" : " 17 GB", | 20260505 22:01:13.625 | INFO | "huge" : "", | 20260505 22:01:13.625 | INFO | "huge_shared" : " 27 GB", | 20260505 22:01:13.625 | INFO | "malloc" : " 14 GB", | 20260505 22:01:13.625 | INFO | "pinned" : " 29 GB" | 20260505 22:01:13.625 | INFO | } | 20260505 22:01:13.625 | INFO | Perf report for the last 5.14519 seconds: | 20260505 22:01:13.625 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:01:13.625 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:13.625 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:13.625 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:13.625 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260505 22:01:13.625 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:13.625 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:13.625 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:01:13.625 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:13.625 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:13.625 | INFO | Uploaded 36GiB to GPU in 2.20259s (16.3444GiB/s) | 20260505 22:01:13.625 | INFO | Downloaded 34GiB to GPU in 2.0393s (16.6724GiB/s) | 20260505 22:01:18.769 | INFO | Analyzing chunk frames = [6633472,6633984) | 20260505 22:01:18.769 | INFO | MemInfo: { | 20260505 22:01:18.769 | INFO | "gpu" : " 17 GB", | 20260505 22:01:18.769 | INFO | "huge" : "", | 20260505 22:01:18.769 | INFO | "huge_shared" : " 27 GB", | 20260505 22:01:18.769 | INFO | "malloc" : " 14 GB", | 20260505 22:01:18.769 | INFO | "pinned" : " 29 GB" | 20260505 22:01:18.769 | INFO | } | 20260505 22:01:18.769 | INFO | Perf report for the last 5.14406 seconds: | 20260505 22:01:18.769 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:01:18.769 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:18.769 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:18.769 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:18.769 | INFO | ANALYSIS : 61.6%, 3.0, 96.2ms, 0 | 20260505 22:01:18.769 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:18.769 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:18.769 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 22:01:18.769 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:18.769 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:18.769 | INFO | Uploaded 36GiB to GPU in 2.16096s (16.6593GiB/s) | 20260505 22:01:18.769 | INFO | Downloaded 34GiB to GPU in 2.06715s (16.4477GiB/s) | 20260505 22:01:23.910 | INFO | Analyzing chunk frames = [6633984,6634496) | 20260505 22:01:23.910 | INFO | MemInfo: { | 20260505 22:01:23.910 | INFO | "gpu" : " 17 GB", | 20260505 22:01:23.910 | INFO | "huge" : "", | 20260505 22:01:23.910 | INFO | "huge_shared" : " 27 GB", | 20260505 22:01:23.910 | INFO | "malloc" : " 14 GB", | 20260505 22:01:23.910 | INFO | "pinned" : " 29 GB" | 20260505 22:01:23.910 | INFO | } | 20260505 22:01:23.910 | INFO | Perf report for the last 5.14089 seconds: | 20260505 22:01:23.910 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:01:23.910 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:23.910 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:23.910 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:23.910 | INFO | ANALYSIS : 61.5%, 3.0, 96.0ms, 0 | 20260505 22:01:23.910 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:23.910 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:23.910 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260505 22:01:23.910 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:23.910 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:23.910 | INFO | Uploaded 36GiB to GPU in 2.07774s (17.3265GiB/s) | 20260505 22:01:23.910 | INFO | Downloaded 34GiB to GPU in 2.03564s (16.7023GiB/s) | 20260505 22:01:29.042 | INFO | Analyzing chunk frames = [6634496,6635008) | 20260505 22:01:29.042 | INFO | MemInfo: { | 20260505 22:01:29.042 | INFO | "gpu" : " 17 GB", | 20260505 22:01:29.042 | INFO | "huge" : "", | 20260505 22:01:29.042 | INFO | "huge_shared" : " 27 GB", | 20260505 22:01:29.042 | INFO | "malloc" : " 14 GB", | 20260505 22:01:29.042 | INFO | "pinned" : " 29 GB" | 20260505 22:01:29.042 | INFO | } | 20260505 22:01:29.042 | INFO | Perf report for the last 5.13248 seconds: | 20260505 22:01:29.042 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:01:29.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:29.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:29.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:29.042 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260505 22:01:29.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:29.042 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:29.042 | INFO | PRE_HQ : 27.7%, 1.0, 14.3ms, 0 | 20260505 22:01:29.042 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:29.042 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:29.043 | INFO | Uploaded 36GiB to GPU in 2.11374s (17.0314GiB/s) | 20260505 22:01:29.043 | INFO | Downloaded 34GiB to GPU in 2.10503s (16.1518GiB/s) | 20260505 22:01:34.181 | INFO | Analyzing chunk frames = [6635008,6635520) | 20260505 22:01:34.181 | INFO | MemInfo: { | 20260505 22:01:34.181 | INFO | "gpu" : " 17 GB", | 20260505 22:01:34.181 | INFO | "huge" : "", | 20260505 22:01:34.181 | INFO | "huge_shared" : " 27 GB", | 20260505 22:01:34.181 | INFO | "malloc" : " 14 GB", | 20260505 22:01:34.181 | INFO | "pinned" : " 29 GB" | 20260505 22:01:34.181 | INFO | } | 20260505 22:01:34.181 | INFO | Perf report for the last 5.13876 seconds: | 20260505 22:01:34.181 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:01:34.181 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:34.181 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:34.181 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:34.181 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 22:01:34.181 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:34.181 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:34.181 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:01:34.181 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:34.181 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:34.181 | INFO | Uploaded 36GiB to GPU in 2.09822s (17.1574GiB/s) | 20260505 22:01:34.181 | INFO | Downloaded 34GiB to GPU in 1.99877s (17.0105GiB/s) | 20260505 22:01:34.182 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 22:01:34.182 | INFO | Total mem usage: 15.3 GiB | 20260505 22:01:34.182 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 22:01:34.182 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -9.02%) | 20260505 22:01:34.182 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 22:01:34.182 | INFO | High Water Breakdown: | 20260505 22:01:34.182 | INFO | 60.9% : 8.56 GiB ( 90% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 22:01:34.182 | INFO | 39% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 22:01:34.182 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 22:01:34.182 | INFO | ------------------------------------------ | 20260505 22:01:34.182 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:01:34.182 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 22:01:34.182 | INFO | Total mem usage: 26.5 GiB | 20260505 22:01:34.182 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 22:01:34.182 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 22:01:34.182 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 22:01:34.182 | INFO | High Water Breakdown: | 20260505 22:01:34.182 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:01:34.182 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:01:34.182 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:01:34.182 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:01:34.182 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:01:34.182 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:01:34.182 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 22:01:34.182 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 22:01:34.182 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:01:34.182 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 22:01:34.182 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 22:01:34.182 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 22:01:34.182 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 22:01:34.182 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 22:01:34.182 | INFO | ------------------------------------------ | 20260505 22:01:34.182 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:01:34.182 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 22:01:34.182 | INFO | Total mem usage: 25.6 GiB | 20260505 22:01:34.182 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 22:01:34.182 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 22:01:34.182 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 22:01:34.182 | INFO | High Water Breakdown: | 20260505 22:01:34.182 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 22:01:34.182 | INFO | ------------------------------------------ | 20260505 22:01:34.182 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:01:34.182 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 22:01:34.182 | INFO | Total mem usage: 15.7 GiB | 20260505 22:01:34.182 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 22:01:34.182 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 22:01:34.182 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 22:01:34.182 | INFO | High Water Breakdown: | 20260505 22:01:34.182 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 22:01:34.182 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 22:01:34.182 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 22:01:34.182 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 22:01:34.182 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 22:01:34.182 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 22:01:34.182 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 22:01:34.182 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 22:01:34.182 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 22:01:34.182 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:01:34.182 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 22:01:34.182 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:01:34.182 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 22:01:34.182 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 22:01:34.182 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 22:01:34.182 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 22:01:34.182 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:01:34.182 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 22:01:34.182 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:01:34.182 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:01:34.182 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:01:34.182 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 22:01:34.182 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 22:01:34.182 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:01:34.182 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 22:01:34.183 | INFO | ------------------------------------------ | 20260505 22:01:34.183 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:01:34.183 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 22:01:34.183 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:01:34.183 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:01:34.183 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.41e+01, 1.35e+01 / 1.16e+00 | 20260505 22:01:34.183 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.04e-01, 1.10e-02 / 4.69e-03 | 20260505 22:01:34.183 | INFO | PerfReport: 11.82%, Baseline -- 3.71e+00 / 4.54e+00, 5.71e-01 / 1.83e-01 | 20260505 22:01:34.183 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.53e+00, 1.89e-01 / 8.99e-02 | 20260505 22:01:34.183 | INFO | PerfReport: 15.98%, DME -- 4.01e+01 / 4.41e+01, 3.98e+00 / 1.21e+00 | 20260505 22:01:34.183 | INFO | PerfReport: 57.17%, FrameLabeling -- 1.79e+01 / 1.94e+01, 1.47e+00 / 1.09e+00 | 20260505 22:01:34.183 | INFO | PerfReport: 7.18%, PulseAccumulating -- 2.26e+00 / 2.78e+00, 4.34e-01 / 2.34e-01 | 20260505 22:01:34.183 | INFO | PerfReport: 3.17%, Metrics -- 9.96e-01 / 1.29e+00, 2.71e-01 / 1.00e-01 | 20260505 22:01:34.183 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.46e-01 / 3.98e-01, 5.88e-02 / 2.94e-02 | 20260505 22:01:34.183 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 22:01:34.183 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:01:34.183 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:01:34.183 | INFO | PerfReport: Full Execution -- 9.88e+01 / -nan, 2.48e+01 / 0.00e+00 | 20260505 22:01:34.183 | INFO | PerfReport: 22.40%, Upload -- 2.21e+01 / 0.00e+00, 7.62e+00 / 0.00e+00 | 20260505 22:01:34.183 | INFO | PerfReport: 21.78%, Download -- 2.15e+01 / 0.00e+00, 7.64e+00 / 0.00e+00 | 20260505 22:01:34.183 | INFO | PerfReport: 31.79%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 22:01:34.183 | INFO | PerfReport: 2.21%, LockUpload -- 2.18e+00 / 0.00e+00, 7.97e+00 / 0.00e+00 | 20260505 22:01:34.183 | INFO | PerfReport: 1.34%, LockDownload -- 1.33e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260505 22:01:34.183 | INFO | PerfReport: 20.47%, LockCompute -- 2.02e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 22:01:34.183 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.94e-02 / 0.00e+00 | 20260505 22:01:34.313 | INFO | Starting a round of Baz IO | 20260505 22:01:37.469 | INFO | Populating RTMetrics report | 20260505 22:01:37.469 | INFO | RTMetrics report populated | 20260505 22:01:37.469 | INFO | Opening tmp RTMetrics json file | 20260505 22:01:37.470 | INFO | Serializing RTMetrics to json | 20260505 22:01:37.471 | INFO | Serialization done, swapping tmp file now... | 20260505 22:01:37.472 | INFO | RTMetrics done | 20260505 22:01:37.553 | INFO | Finished a round of Baz IO: Wrote 1.948 GiB in 3.24 seconds | 20260505 22:01:37.554 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.53% / 0.00% / 78.47% | 20260505 22:01:37.554 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.133888% / 3.97364e-06% / 9.93411e-05% | 20260505 22:01:37.554 | INFO | Baz IO overall: | 20260505 22:01:37.554 | INFO | BytesWritten : 3830174926097 | 20260505 22:01:37.554 | INFO | NumEvents : 1807452633111 | 20260505 22:01:39.319 | INFO | Analyzing chunk frames = [6635520,6636032) | 20260505 22:01:39.319 | INFO | MemInfo: { | 20260505 22:01:39.319 | INFO | "gpu" : " 17 GB", | 20260505 22:01:39.319 | INFO | "huge" : "", | 20260505 22:01:39.319 | INFO | "huge_shared" : " 27 GB", | 20260505 22:01:39.319 | INFO | "malloc" : " 14 GB", | 20260505 22:01:39.319 | INFO | "pinned" : " 29 GB" | 20260505 22:01:39.319 | INFO | } | 20260505 22:01:39.319 | INFO | Perf report for the last 5.13829 seconds: | 20260505 22:01:39.320 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:01:39.320 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:39.320 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:39.320 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:39.320 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 22:01:39.320 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:39.320 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260505 22:01:39.320 | INFO | PRE_HQ : 59.8%, 1.8, 54.4ms, 0 | 20260505 22:01:39.320 | INFO | BAZWRITER : 62.6%, 3.6, 116.6ms, 0 | 20260505 22:01:39.320 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 22:01:39.320 | INFO | Uploaded 36GiB to GPU in 2.15578s (16.6993GiB/s) | 20260505 22:01:39.320 | INFO | Downloaded 38GiB to GPU in 2.22893s (17.0486GiB/s) | 20260505 22:01:44.468 | INFO | Analyzing chunk frames = [6636032,6636544) | 20260505 22:01:44.468 | INFO | MemInfo: { | 20260505 22:01:44.468 | INFO | "gpu" : " 17 GB", | 20260505 22:01:44.468 | INFO | "huge" : "", | 20260505 22:01:44.468 | INFO | "huge_shared" : " 27 GB", | 20260505 22:01:44.468 | INFO | "malloc" : " 14 GB", | 20260505 22:01:44.468 | INFO | "pinned" : " 29 GB" | 20260505 22:01:44.468 | INFO | } | 20260505 22:01:44.468 | INFO | Perf report for the last 5.14859 seconds: | 20260505 22:01:44.468 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:01:44.468 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:44.468 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:44.468 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:44.468 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260505 22:01:44.468 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:44.468 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:44.468 | INFO | PRE_HQ : 27.0%, 1.0, 14.1ms, 0 | 20260505 22:01:44.468 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:44.468 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:44.468 | INFO | Uploaded 36GiB to GPU in 2.08438s (17.2714GiB/s) | 20260505 22:01:44.468 | INFO | Downloaded 34GiB to GPU in 2.02893s (16.7576GiB/s) | 20260505 22:01:49.600 | INFO | Analyzing chunk frames = [6636544,6637056) | 20260505 22:01:49.600 | INFO | MemInfo: { | 20260505 22:01:49.600 | INFO | "gpu" : " 17 GB", | 20260505 22:01:49.600 | INFO | "huge" : "", | 20260505 22:01:49.600 | INFO | "huge_shared" : " 27 GB", | 20260505 22:01:49.600 | INFO | "malloc" : " 14 GB", | 20260505 22:01:49.601 | INFO | "pinned" : " 29 GB" | 20260505 22:01:49.601 | INFO | } | 20260505 22:01:49.601 | INFO | Perf report for the last 5.13256 seconds: | 20260505 22:01:49.601 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:01:49.601 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:49.601 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:49.601 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:49.601 | INFO | ANALYSIS : 62.1%, 3.0, 96.4ms, 0 | 20260505 22:01:49.601 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:49.601 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:49.601 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 22:01:49.601 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:49.601 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:49.601 | INFO | Uploaded 36GiB to GPU in 2.15325s (16.7189GiB/s) | 20260505 22:01:49.601 | INFO | Downloaded 34GiB to GPU in 2.05614s (16.5358GiB/s) | 20260505 22:01:54.735 | INFO | Analyzing chunk frames = [6637056,6637568) | 20260505 22:01:54.735 | INFO | MemInfo: { | 20260505 22:01:54.735 | INFO | "gpu" : " 17 GB", | 20260505 22:01:54.735 | INFO | "huge" : "", | 20260505 22:01:54.735 | INFO | "huge_shared" : " 27 GB", | 20260505 22:01:54.735 | INFO | "malloc" : " 14 GB", | 20260505 22:01:54.736 | INFO | "pinned" : " 29 GB" | 20260505 22:01:54.736 | INFO | } | 20260505 22:01:54.736 | INFO | Perf report for the last 5.13509 seconds: | 20260505 22:01:54.736 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:01:54.736 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:54.736 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:54.736 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:54.736 | INFO | ANALYSIS : 61.9%, 3.0, 96.1ms, 0 | 20260505 22:01:54.736 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:54.736 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:54.736 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:01:54.736 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:54.736 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:54.736 | INFO | Uploaded 36GiB to GPU in 2.13697s (16.8463GiB/s) | 20260505 22:01:54.736 | INFO | Downloaded 34GiB to GPU in 2.03775s (16.6851GiB/s) | 20260505 22:01:59.879 | INFO | Analyzing chunk frames = [6637568,6638080) | 20260505 22:01:59.879 | INFO | MemInfo: { | 20260505 22:01:59.879 | INFO | "gpu" : " 17 GB", | 20260505 22:01:59.879 | INFO | "huge" : "", | 20260505 22:01:59.879 | INFO | "huge_shared" : " 27 GB", | 20260505 22:01:59.879 | INFO | "malloc" : " 14 GB", | 20260505 22:01:59.879 | INFO | "pinned" : " 29 GB" | 20260505 22:01:59.879 | INFO | } | 20260505 22:01:59.879 | INFO | Perf report for the last 5.14358 seconds: | 20260505 22:01:59.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:01:59.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:59.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:59.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:59.879 | INFO | ANALYSIS : 61.6%, 3.0, 96.2ms, 0 | 20260505 22:01:59.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:59.879 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:59.879 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:01:59.879 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:59.879 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:01:59.880 | INFO | Uploaded 36GiB to GPU in 2.13759s (16.8414GiB/s) | 20260505 22:01:59.880 | INFO | Downloaded 34GiB to GPU in 2.06563s (16.4599GiB/s) | 20260505 22:02:05.022 | INFO | Analyzing chunk frames = [6638080,6638592) | 20260505 22:02:05.022 | INFO | MemInfo: { | 20260505 22:02:05.022 | INFO | "gpu" : " 17 GB", | 20260505 22:02:05.022 | INFO | "huge" : "", | 20260505 22:02:05.022 | INFO | "huge_shared" : " 27 GB", | 20260505 22:02:05.022 | INFO | "malloc" : " 14 GB", | 20260505 22:02:05.022 | INFO | "pinned" : " 29 GB" | 20260505 22:02:05.022 | INFO | } | 20260505 22:02:05.022 | INFO | Perf report for the last 5.14308 seconds: | 20260505 22:02:05.022 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:02:05.022 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:05.022 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:05.022 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:05.022 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260505 22:02:05.022 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:05.022 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:05.022 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 22:02:05.022 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:05.023 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:05.023 | INFO | Uploaded 36GiB to GPU in 2.08441s (17.2711GiB/s) | 20260505 22:02:05.023 | INFO | Downloaded 34GiB to GPU in 2.03807s (16.6825GiB/s) | 20260505 22:02:10.158 | INFO | Analyzing chunk frames = [6638592,6639104) | 20260505 22:02:10.158 | INFO | MemInfo: { | 20260505 22:02:10.158 | INFO | "gpu" : " 17 GB", | 20260505 22:02:10.158 | INFO | "huge" : "", | 20260505 22:02:10.158 | INFO | "huge_shared" : " 27 GB", | 20260505 22:02:10.158 | INFO | "malloc" : " 14 GB", | 20260505 22:02:10.158 | INFO | "pinned" : " 29 GB" | 20260505 22:02:10.158 | INFO | } | 20260505 22:02:10.158 | INFO | Perf report for the last 5.13565 seconds: | 20260505 22:02:10.158 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:02:10.158 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:10.158 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:10.158 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:10.158 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 22:02:10.158 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:10.158 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:10.158 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 22:02:10.158 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:10.158 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:10.158 | INFO | Uploaded 36GiB to GPU in 2.02724s (17.7581GiB/s) | 20260505 22:02:10.158 | INFO | Downloaded 34GiB to GPU in 2.0377s (16.6855GiB/s) | 20260505 22:02:15.296 | INFO | Analyzing chunk frames = [6639104,6639616) | 20260505 22:02:15.296 | INFO | MemInfo: { | 20260505 22:02:15.296 | INFO | "gpu" : " 17 GB", | 20260505 22:02:15.296 | INFO | "huge" : "", | 20260505 22:02:15.296 | INFO | "huge_shared" : " 27 GB", | 20260505 22:02:15.296 | INFO | "malloc" : " 14 GB", | 20260505 22:02:15.296 | INFO | "pinned" : " 29 GB" | 20260505 22:02:15.296 | INFO | } | 20260505 22:02:15.296 | INFO | Perf report for the last 5.13845 seconds: | 20260505 22:02:15.296 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:02:15.296 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:15.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:15.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:15.297 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 22:02:15.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:15.297 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:15.297 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 22:02:15.297 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:15.297 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:15.297 | INFO | Uploaded 36GiB to GPU in 2.10421s (17.1086GiB/s) | 20260505 22:02:15.297 | INFO | Downloaded 34GiB to GPU in 2.00698s (16.9408GiB/s) | 20260505 22:02:15.430 | INFO | Starting a round of Baz IO | 20260505 22:02:18.586 | INFO | Populating RTMetrics report | 20260505 22:02:18.587 | INFO | RTMetrics report populated | 20260505 22:02:18.587 | INFO | Opening tmp RTMetrics json file | 20260505 22:02:18.587 | INFO | Serializing RTMetrics to json | 20260505 22:02:18.589 | INFO | Serialization done, swapping tmp file now... | 20260505 22:02:18.589 | INFO | RTMetrics done | 20260505 22:02:18.676 | INFO | Finished a round of Baz IO: Wrote 1.947 GiB in 3.246 seconds | 20260505 22:02:18.676 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.53% / 0.00% / 78.47% | 20260505 22:02:18.676 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.133383% / 0% / 0.000115236% | 20260505 22:02:18.676 | INFO | Baz IO overall: | 20260505 22:02:18.676 | INFO | BytesWritten : 3832265906340 | 20260505 22:02:18.676 | INFO | NumEvents : 1808416299478 | 20260505 22:02:20.430 | INFO | Analyzing chunk frames = [6639616,6640128) | 20260505 22:02:20.430 | INFO | MemInfo: { | 20260505 22:02:20.430 | INFO | "gpu" : " 17 GB", | 20260505 22:02:20.430 | INFO | "huge" : "", | 20260505 22:02:20.430 | INFO | "huge_shared" : " 27 GB", | 20260505 22:02:20.431 | INFO | "malloc" : " 14 GB", | 20260505 22:02:20.431 | INFO | "pinned" : " 29 GB" | 20260505 22:02:20.431 | INFO | } | 20260505 22:02:20.431 | INFO | Perf report for the last 5.1342 seconds: | 20260505 22:02:20.431 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:02:20.431 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:20.431 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:20.431 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:20.431 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 22:02:20.431 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:20.431 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260505 22:02:20.431 | INFO | PRE_HQ : 60.2%, 1.8, 55.1ms, 0 | 20260505 22:02:20.431 | INFO | BAZWRITER : 62.7%, 3.6, 115.8ms, 0 | 20260505 22:02:20.431 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 22:02:20.431 | INFO | Uploaded 36GiB to GPU in 2.16748s (16.6091GiB/s) | 20260505 22:02:20.431 | INFO | Downloaded 38GiB to GPU in 2.24808s (16.9033GiB/s) | 20260505 22:02:25.575 | INFO | Analyzing chunk frames = [6640128,6640640) | 20260505 22:02:25.575 | INFO | MemInfo: { | 20260505 22:02:25.575 | INFO | "gpu" : " 17 GB", | 20260505 22:02:25.575 | INFO | "huge" : "", | 20260505 22:02:25.575 | INFO | "huge_shared" : " 27 GB", | 20260505 22:02:25.575 | INFO | "malloc" : " 14 GB", | 20260505 22:02:25.575 | INFO | "pinned" : " 29 GB" | 20260505 22:02:25.575 | INFO | } | 20260505 22:02:25.575 | INFO | Perf report for the last 5.14432 seconds: | 20260505 22:02:25.575 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:02:25.575 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:25.575 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:25.575 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:25.575 | INFO | ANALYSIS : 62.5%, 3.0, 97.7ms, 0 | 20260505 22:02:25.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:25.575 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:25.575 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:02:25.575 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:25.575 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:25.575 | INFO | Uploaded 36GiB to GPU in 2.1633s (16.6413GiB/s) | 20260505 22:02:25.575 | INFO | Downloaded 34GiB to GPU in 2.05638s (16.5339GiB/s) | 20260505 22:02:30.719 | INFO | Analyzing chunk frames = [6640640,6641152) | 20260505 22:02:30.719 | INFO | MemInfo: { | 20260505 22:02:30.719 | INFO | "gpu" : " 17 GB", | 20260505 22:02:30.719 | INFO | "huge" : "", | 20260505 22:02:30.719 | INFO | "huge_shared" : " 27 GB", | 20260505 22:02:30.719 | INFO | "malloc" : " 14 GB", | 20260505 22:02:30.719 | INFO | "pinned" : " 29 GB" | 20260505 22:02:30.719 | INFO | } | 20260505 22:02:30.719 | INFO | Perf report for the last 5.14421 seconds: | 20260505 22:02:30.719 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:02:30.719 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:30.719 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:30.719 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:30.719 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260505 22:02:30.719 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:30.719 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:30.719 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:02:30.719 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:30.719 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:30.720 | INFO | Uploaded 36GiB to GPU in 2.14391s (16.7918GiB/s) | 20260505 22:02:30.720 | INFO | Downloaded 34GiB to GPU in 2.02933s (16.7543GiB/s) | 20260505 22:02:35.854 | INFO | Analyzing chunk frames = [6641152,6641664) | 20260505 22:02:35.854 | INFO | MemInfo: { | 20260505 22:02:35.854 | INFO | "gpu" : " 17 GB", | 20260505 22:02:35.854 | INFO | "huge" : "", | 20260505 22:02:35.854 | INFO | "huge_shared" : " 27 GB", | 20260505 22:02:35.854 | INFO | "malloc" : " 14 GB", | 20260505 22:02:35.854 | INFO | "pinned" : " 29 GB" | 20260505 22:02:35.854 | INFO | } | 20260505 22:02:35.854 | INFO | Perf report for the last 5.13467 seconds: | 20260505 22:02:35.854 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:02:35.854 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:35.854 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:35.854 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:35.854 | INFO | ANALYSIS : 61.9%, 3.0, 96.1ms, 0 | 20260505 22:02:35.854 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:35.854 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:35.854 | INFO | PRE_HQ : 27.5%, 1.0, 14.2ms, 0 | 20260505 22:02:35.854 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:35.854 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:35.854 | INFO | Uploaded 36GiB to GPU in 2.15191s (16.7293GiB/s) | 20260505 22:02:35.854 | INFO | Downloaded 34GiB to GPU in 2.02588s (16.7829GiB/s) | 20260505 22:02:40.989 | INFO | Analyzing chunk frames = [6641664,6642176) | 20260505 22:02:40.989 | INFO | MemInfo: { | 20260505 22:02:40.989 | INFO | "gpu" : " 17 GB", | 20260505 22:02:40.989 | INFO | "huge" : "", | 20260505 22:02:40.989 | INFO | "huge_shared" : " 27 GB", | 20260505 22:02:40.989 | INFO | "malloc" : " 14 GB", | 20260505 22:02:40.989 | INFO | "pinned" : " 29 GB" | 20260505 22:02:40.989 | INFO | } | 20260505 22:02:40.989 | INFO | Perf report for the last 5.13536 seconds: | 20260505 22:02:40.989 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:02:40.989 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:40.989 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:40.989 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:40.989 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260505 22:02:40.989 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:40.989 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:40.989 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:02:40.989 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:40.989 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:40.990 | INFO | Uploaded 36GiB to GPU in 2.13527s (16.8597GiB/s) | 20260505 22:02:40.990 | INFO | Downloaded 34GiB to GPU in 2.05982s (16.5063GiB/s) | 20260505 22:02:46.135 | INFO | Analyzing chunk frames = [6642176,6642688) | 20260505 22:02:46.135 | INFO | MemInfo: { | 20260505 22:02:46.135 | INFO | "gpu" : " 17 GB", | 20260505 22:02:46.135 | INFO | "huge" : "", | 20260505 22:02:46.135 | INFO | "huge_shared" : " 27 GB", | 20260505 22:02:46.135 | INFO | "malloc" : " 14 GB", | 20260505 22:02:46.135 | INFO | "pinned" : " 29 GB" | 20260505 22:02:46.135 | INFO | } | 20260505 22:02:46.135 | INFO | Perf report for the last 5.14537 seconds: | 20260505 22:02:46.135 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:02:46.135 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:46.135 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:46.135 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:46.135 | INFO | ANALYSIS : 62.0%, 3.0, 97.0ms, 0 | 20260505 22:02:46.135 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:46.135 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:46.135 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 22:02:46.135 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:46.135 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:46.135 | INFO | Uploaded 36GiB to GPU in 2.157s (16.6899GiB/s) | 20260505 22:02:46.135 | INFO | Downloaded 34GiB to GPU in 2.06522s (16.4631GiB/s) | 20260505 22:02:51.274 | INFO | Analyzing chunk frames = [6642688,6643200) | 20260505 22:02:51.274 | INFO | MemInfo: { | 20260505 22:02:51.274 | INFO | "gpu" : " 17 GB", | 20260505 22:02:51.274 | INFO | "huge" : "", | 20260505 22:02:51.274 | INFO | "huge_shared" : " 27 GB", | 20260505 22:02:51.274 | INFO | "malloc" : " 14 GB", | 20260505 22:02:51.274 | INFO | "pinned" : " 29 GB" | 20260505 22:02:51.274 | INFO | } | 20260505 22:02:51.274 | INFO | Perf report for the last 5.13958 seconds: | 20260505 22:02:51.274 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:02:51.274 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:51.274 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:51.274 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:51.274 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 22:02:51.274 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:51.274 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:51.274 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 22:02:51.274 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:51.274 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:51.275 | INFO | Uploaded 36GiB to GPU in 2.15788s (16.6831GiB/s) | 20260505 22:02:51.275 | INFO | Downloaded 34GiB to GPU in 2.12831s (15.9751GiB/s) | 20260505 22:02:56.416 | INFO | Analyzing chunk frames = [6643200,6643712) | 20260505 22:02:56.416 | INFO | MemInfo: { | 20260505 22:02:56.416 | INFO | "gpu" : " 17 GB", | 20260505 22:02:56.416 | INFO | "huge" : "", | 20260505 22:02:56.416 | INFO | "huge_shared" : " 27 GB", | 20260505 22:02:56.417 | INFO | "malloc" : " 14 GB", | 20260505 22:02:56.417 | INFO | "pinned" : " 29 GB" | 20260505 22:02:56.417 | INFO | } | 20260505 22:02:56.417 | INFO | Perf report for the last 5.14244 seconds: | 20260505 22:02:56.417 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:02:56.417 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:56.417 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:56.417 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:56.417 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260505 22:02:56.417 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:56.417 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:56.417 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 22:02:56.417 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:56.417 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:02:56.417 | INFO | Uploaded 36GiB to GPU in 2.12487s (16.9422GiB/s) | 20260505 22:02:56.417 | INFO | Downloaded 34GiB to GPU in 1.9957s (17.0367GiB/s) | 20260505 22:02:56.417 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 22:02:56.417 | INFO | Total mem usage: 15.3 GiB | 20260505 22:02:56.417 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 22:02:56.417 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.48%) | 20260505 22:02:56.417 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 22:02:56.417 | INFO | High Water Breakdown: | 20260505 22:02:56.417 | INFO | 61.1% : 8.63 GiB ( 90.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 22:02:56.417 | INFO | 38.8% : 5.48 GiB ( 82.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 22:02:56.417 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 22:02:56.417 | INFO | ------------------------------------------ | 20260505 22:02:56.417 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:02:56.417 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 22:02:56.417 | INFO | Total mem usage: 26.5 GiB | 20260505 22:02:56.417 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 22:02:56.417 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 22:02:56.417 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 22:02:56.417 | INFO | High Water Breakdown: | 20260505 22:02:56.418 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:02:56.418 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:02:56.418 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:02:56.418 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:02:56.418 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:02:56.418 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:02:56.418 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 22:02:56.418 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 22:02:56.418 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:02:56.418 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 22:02:56.418 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 22:02:56.418 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 22:02:56.418 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 22:02:56.418 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 22:02:56.418 | INFO | ------------------------------------------ | 20260505 22:02:56.418 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:02:56.418 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 22:02:56.418 | INFO | Total mem usage: 25.6 GiB | 20260505 22:02:56.418 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 22:02:56.418 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 22:02:56.418 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 22:02:56.418 | INFO | High Water Breakdown: | 20260505 22:02:56.418 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 22:02:56.418 | INFO | ------------------------------------------ | 20260505 22:02:56.418 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:02:56.418 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 22:02:56.418 | INFO | Total mem usage: 15.7 GiB | 20260505 22:02:56.418 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 22:02:56.418 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 22:02:56.418 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 22:02:56.418 | INFO | High Water Breakdown: | 20260505 22:02:56.418 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 22:02:56.418 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 22:02:56.418 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 22:02:56.418 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 22:02:56.418 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 22:02:56.418 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 22:02:56.418 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 22:02:56.418 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 22:02:56.418 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 22:02:56.418 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:02:56.418 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 22:02:56.418 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:02:56.418 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 22:02:56.418 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 22:02:56.418 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 22:02:56.418 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 22:02:56.418 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:02:56.418 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 22:02:56.418 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:02:56.418 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:02:56.418 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:02:56.418 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 22:02:56.418 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 22:02:56.418 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:02:56.418 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 22:02:56.418 | INFO | ------------------------------------------ | 20260505 22:02:56.418 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:02:56.418 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 22:02:56.418 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:02:56.418 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:02:56.418 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.41e+01, 1.35e+01 / 1.03e+00 | 20260505 22:02:56.418 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.04e-01, 1.05e-02 / 5.06e-03 | 20260505 22:02:56.418 | INFO | PerfReport: 11.80%, Baseline -- 3.70e+00 / 4.51e+00, 5.64e-01 / 2.04e-01 | 20260505 22:02:56.418 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.54e+00, 1.90e-01 / 9.21e-02 | 20260505 22:02:56.418 | INFO | PerfReport: 15.97%, DME -- 4.01e+01 / 4.41e+01, 4.00e+00 / 1.19e+00 | 20260505 22:02:56.418 | INFO | PerfReport: 57.18%, FrameLabeling -- 1.79e+01 / 1.93e+01, 1.47e+00 / 9.55e-01 | 20260505 22:02:56.418 | INFO | PerfReport: 7.19%, PulseAccumulating -- 2.26e+00 / 2.80e+00, 4.31e-01 / 2.12e-01 | 20260505 22:02:56.418 | INFO | PerfReport: 3.18%, Metrics -- 9.98e-01 / 1.31e+00, 2.72e-01 / 1.40e-01 | 20260505 22:02:56.418 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.47e-01 / 4.10e-01, 5.93e-02 / 3.87e-02 | 20260505 22:02:56.418 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 22:02:56.418 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:02:56.418 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:02:56.418 | INFO | PerfReport: Full Execution -- 9.89e+01 / -nan, 2.50e+01 / 0.00e+00 | 20260505 22:02:56.418 | INFO | PerfReport: 22.43%, Upload -- 2.22e+01 / 0.00e+00, 7.67e+00 / 0.00e+00 | 20260505 22:02:56.418 | INFO | PerfReport: 21.79%, Download -- 2.16e+01 / 0.00e+00, 7.66e+00 / 0.00e+00 | 20260505 22:02:56.418 | INFO | PerfReport: 31.72%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 22:02:56.418 | INFO | PerfReport: 2.24%, LockUpload -- 2.22e+00 / 0.00e+00, 8.02e+00 / 0.00e+00 | 20260505 22:02:56.418 | INFO | PerfReport: 1.33%, LockDownload -- 1.31e+00 / 0.00e+00, 2.86e+00 / 0.00e+00 | 20260505 22:02:56.418 | INFO | PerfReport: 20.48%, LockCompute -- 2.03e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 22:02:56.418 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.80e-02 / 0.00e+00 | 20260505 22:02:56.544 | INFO | Starting a round of Baz IO | 20260505 22:02:59.690 | INFO | Populating RTMetrics report | 20260505 22:02:59.690 | INFO | RTMetrics report populated | 20260505 22:02:59.690 | INFO | Opening tmp RTMetrics json file | 20260505 22:02:59.694 | INFO | Serializing RTMetrics to json | 20260505 22:02:59.696 | INFO | Serialization done, swapping tmp file now... | 20260505 22:02:59.697 | INFO | RTMetrics done | 20260505 22:02:59.777 | INFO | Finished a round of Baz IO: Wrote 1.945 GiB in 3.232 seconds | 20260505 22:02:59.777 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.53% / 0.00% / 78.47% | 20260505 22:02:59.777 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.133463% / 3.97364e-06% / 0.000135104% | 20260505 22:02:59.777 | INFO | Baz IO overall: | 20260505 22:02:59.777 | INFO | BytesWritten : 3834354700039 | 20260505 22:02:59.777 | INFO | NumEvents : 1809378802436 | 20260505 22:03:01.552 | INFO | Analyzing chunk frames = [6643712,6644224) | 20260505 22:03:01.552 | INFO | MemInfo: { | 20260505 22:03:01.552 | INFO | "gpu" : " 17 GB", | 20260505 22:03:01.552 | INFO | "huge" : "", | 20260505 22:03:01.552 | INFO | "huge_shared" : " 27 GB", | 20260505 22:03:01.552 | INFO | "malloc" : " 14 GB", | 20260505 22:03:01.552 | INFO | "pinned" : " 29 GB" | 20260505 22:03:01.552 | INFO | } | 20260505 22:03:01.552 | INFO | Perf report for the last 5.13505 seconds: | 20260505 22:03:01.552 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:03:01.552 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:01.552 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:01.552 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:01.552 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260505 22:03:01.552 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:01.552 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260505 22:03:01.552 | INFO | PRE_HQ : 59.8%, 1.8, 55.2ms, 0 | 20260505 22:03:01.552 | INFO | BAZWRITER : 62.2%, 3.7, 116.8ms, 0 | 20260505 22:03:01.552 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 22:03:01.552 | INFO | Uploaded 36GiB to GPU in 2.12776s (16.9192GiB/s) | 20260505 22:03:01.552 | INFO | Downloaded 38GiB to GPU in 2.22848s (17.052GiB/s) | 20260505 22:03:06.688 | INFO | Analyzing chunk frames = [6644224,6644736) | 20260505 22:03:06.688 | INFO | MemInfo: { | 20260505 22:03:06.688 | INFO | "gpu" : " 17 GB", | 20260505 22:03:06.688 | INFO | "huge" : "", | 20260505 22:03:06.688 | INFO | "huge_shared" : " 27 GB", | 20260505 22:03:06.688 | INFO | "malloc" : " 14 GB", | 20260505 22:03:06.688 | INFO | "pinned" : " 29 GB" | 20260505 22:03:06.688 | INFO | } | 20260505 22:03:06.688 | INFO | Perf report for the last 5.13616 seconds: | 20260505 22:03:06.688 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:03:06.688 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:06.688 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:06.688 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:06.688 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 22:03:06.688 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:06.688 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:06.688 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:03:06.688 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:06.688 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:06.688 | INFO | Uploaded 36GiB to GPU in 2.08974s (17.227GiB/s) | 20260505 22:03:06.688 | INFO | Downloaded 34GiB to GPU in 2.02594s (16.7823GiB/s) | 20260505 22:03:11.834 | INFO | Analyzing chunk frames = [6644736,6645248) | 20260505 22:03:11.834 | INFO | MemInfo: { | 20260505 22:03:11.834 | INFO | "gpu" : " 17 GB", | 20260505 22:03:11.834 | INFO | "huge" : "", | 20260505 22:03:11.834 | INFO | "huge_shared" : " 27 GB", | 20260505 22:03:11.834 | INFO | "malloc" : " 14 GB", | 20260505 22:03:11.834 | INFO | "pinned" : " 29 GB" | 20260505 22:03:11.834 | INFO | } | 20260505 22:03:11.835 | INFO | Perf report for the last 5.14658 seconds: | 20260505 22:03:11.835 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:03:11.835 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:11.835 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:11.835 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:11.835 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260505 22:03:11.835 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:11.835 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:11.835 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 22:03:11.835 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:11.835 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:11.835 | INFO | Uploaded 36GiB to GPU in 2.13975s (16.8244GiB/s) | 20260505 22:03:11.835 | INFO | Downloaded 34GiB to GPU in 2.02049s (16.8276GiB/s) | 20260505 22:03:16.969 | INFO | Analyzing chunk frames = [6645248,6645760) | 20260505 22:03:16.969 | INFO | MemInfo: { | 20260505 22:03:16.969 | INFO | "gpu" : " 17 GB", | 20260505 22:03:16.969 | INFO | "huge" : "", | 20260505 22:03:16.969 | INFO | "huge_shared" : " 27 GB", | 20260505 22:03:16.969 | INFO | "malloc" : " 14 GB", | 20260505 22:03:16.969 | INFO | "pinned" : " 29 GB" | 20260505 22:03:16.969 | INFO | } | 20260505 22:03:16.969 | INFO | Perf report for the last 5.13464 seconds: | 20260505 22:03:16.969 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:03:16.969 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:16.969 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:16.969 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:16.969 | INFO | ANALYSIS : 61.8%, 3.0, 96.0ms, 0 | 20260505 22:03:16.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:16.969 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:16.969 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 22:03:16.969 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:16.969 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:16.970 | INFO | Uploaded 36GiB to GPU in 2.14058s (16.8179GiB/s) | 20260505 22:03:16.970 | INFO | Downloaded 34GiB to GPU in 2.01768s (16.8511GiB/s) | 20260505 22:03:22.105 | INFO | Analyzing chunk frames = [6645760,6646272) | 20260505 22:03:22.105 | INFO | MemInfo: { | 20260505 22:03:22.105 | INFO | "gpu" : " 17 GB", | 20260505 22:03:22.105 | INFO | "huge" : "", | 20260505 22:03:22.105 | INFO | "huge_shared" : " 27 GB", | 20260505 22:03:22.106 | INFO | "malloc" : " 14 GB", | 20260505 22:03:22.106 | INFO | "pinned" : " 29 GB" | 20260505 22:03:22.106 | INFO | } | 20260505 22:03:22.106 | INFO | Perf report for the last 5.13634 seconds: | 20260505 22:03:22.106 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:03:22.106 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:22.106 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:22.106 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:22.106 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260505 22:03:22.106 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:22.106 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:22.106 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:03:22.106 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:22.106 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:22.106 | INFO | Uploaded 36GiB to GPU in 2.13144s (16.89GiB/s) | 20260505 22:03:22.106 | INFO | Downloaded 34GiB to GPU in 2.06064s (16.4997GiB/s) | 20260505 22:03:27.246 | INFO | Analyzing chunk frames = [6646272,6646784) | 20260505 22:03:27.246 | INFO | MemInfo: { | 20260505 22:03:27.246 | INFO | "gpu" : " 17 GB", | 20260505 22:03:27.246 | INFO | "huge" : "", | 20260505 22:03:27.246 | INFO | "huge_shared" : " 27 GB", | 20260505 22:03:27.246 | INFO | "malloc" : " 14 GB", | 20260505 22:03:27.246 | INFO | "pinned" : " 29 GB" | 20260505 22:03:27.246 | INFO | } | 20260505 22:03:27.246 | INFO | Perf report for the last 5.14044 seconds: | 20260505 22:03:27.246 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:03:27.246 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:27.246 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:27.246 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:27.246 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 22:03:27.246 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:27.246 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:27.246 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 22:03:27.246 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:27.246 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:27.246 | INFO | Uploaded 36GiB to GPU in 2.1605s (16.6628GiB/s) | 20260505 22:03:27.246 | INFO | Downloaded 34GiB to GPU in 2.06241s (16.4856GiB/s) | 20260505 22:03:32.388 | INFO | Analyzing chunk frames = [6646784,6647296) | 20260505 22:03:32.388 | INFO | MemInfo: { | 20260505 22:03:32.388 | INFO | "gpu" : " 17 GB", | 20260505 22:03:32.388 | INFO | "huge" : "", | 20260505 22:03:32.388 | INFO | "huge_shared" : " 27 GB", | 20260505 22:03:32.388 | INFO | "malloc" : " 14 GB", | 20260505 22:03:32.388 | INFO | "pinned" : " 29 GB" | 20260505 22:03:32.388 | INFO | } | 20260505 22:03:32.388 | INFO | Perf report for the last 5.14167 seconds: | 20260505 22:03:32.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:03:32.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:32.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:32.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:32.388 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 22:03:32.388 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:32.388 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:32.388 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 22:03:32.388 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:32.388 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:32.388 | INFO | Uploaded 36GiB to GPU in 2.12859s (16.9126GiB/s) | 20260505 22:03:32.388 | INFO | Downloaded 34GiB to GPU in 2.15149s (15.803GiB/s) | 20260505 22:03:37.530 | INFO | Analyzing chunk frames = [6647296,6647808) | 20260505 22:03:37.530 | INFO | MemInfo: { | 20260505 22:03:37.530 | INFO | "gpu" : " 17 GB", | 20260505 22:03:37.530 | INFO | "huge" : "", | 20260505 22:03:37.530 | INFO | "huge_shared" : " 27 GB", | 20260505 22:03:37.530 | INFO | "malloc" : " 14 GB", | 20260505 22:03:37.530 | INFO | "pinned" : " 29 GB" | 20260505 22:03:37.530 | INFO | } | 20260505 22:03:37.531 | INFO | Perf report for the last 5.1429 seconds: | 20260505 22:03:37.531 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:03:37.531 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:37.531 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:37.531 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:37.531 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260505 22:03:37.531 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:37.531 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:37.531 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 22:03:37.531 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:37.531 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:37.531 | INFO | Uploaded 36GiB to GPU in 2.11863s (16.9921GiB/s) | 20260505 22:03:37.531 | INFO | Downloaded 34GiB to GPU in 2.00013s (16.9989GiB/s) | 20260505 22:03:37.656 | INFO | Starting a round of Baz IO | 20260505 22:03:40.808 | INFO | Populating RTMetrics report | 20260505 22:03:40.808 | INFO | RTMetrics report populated | 20260505 22:03:40.808 | INFO | Opening tmp RTMetrics json file | 20260505 22:03:40.809 | INFO | Serializing RTMetrics to json | 20260505 22:03:40.813 | INFO | Serialization done, swapping tmp file now... | 20260505 22:03:40.814 | INFO | RTMetrics done | 20260505 22:03:40.887 | INFO | Finished a round of Baz IO: Wrote 1.944 GiB in 3.231 seconds | 20260505 22:03:40.887 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.53% / 0.00% / 78.47% | 20260505 22:03:40.887 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.132914% / 0% / 9.13938e-05% | 20260505 22:03:40.887 | INFO | Baz IO overall: | 20260505 22:03:40.887 | INFO | BytesWritten : 3836441860970 | 20260505 22:03:40.887 | INFO | NumEvents : 1810339930721 | 20260505 22:03:42.668 | INFO | Analyzing chunk frames = [6647808,6648320) | 20260505 22:03:42.668 | INFO | MemInfo: { | 20260505 22:03:42.668 | INFO | "gpu" : " 17 GB", | 20260505 22:03:42.668 | INFO | "huge" : "", | 20260505 22:03:42.668 | INFO | "huge_shared" : " 27 GB", | 20260505 22:03:42.668 | INFO | "malloc" : " 14 GB", | 20260505 22:03:42.668 | INFO | "pinned" : " 29 GB" | 20260505 22:03:42.668 | INFO | } | 20260505 22:03:42.668 | INFO | Perf report for the last 5.13722 seconds: | 20260505 22:03:42.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:03:42.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:42.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:42.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:42.668 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 22:03:42.668 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:42.668 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260505 22:03:42.668 | INFO | PRE_HQ : 59.9%, 1.8, 55.0ms, 0 | 20260505 22:03:42.668 | INFO | BAZWRITER : 62.1%, 3.7, 116.7ms, 0 | 20260505 22:03:42.668 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 22:03:42.668 | INFO | Uploaded 36GiB to GPU in 2.14138s (16.8116GiB/s) | 20260505 22:03:42.668 | INFO | Downloaded 38GiB to GPU in 2.22197s (17.1019GiB/s) | 20260505 22:03:47.801 | INFO | Analyzing chunk frames = [6648320,6648832) | 20260505 22:03:47.801 | INFO | MemInfo: { | 20260505 22:03:47.801 | INFO | "gpu" : " 17 GB", | 20260505 22:03:47.801 | INFO | "huge" : "", | 20260505 22:03:47.801 | INFO | "huge_shared" : " 27 GB", | 20260505 22:03:47.801 | INFO | "malloc" : " 14 GB", | 20260505 22:03:47.801 | INFO | "pinned" : " 29 GB" | 20260505 22:03:47.801 | INFO | } | 20260505 22:03:47.801 | INFO | Perf report for the last 5.13322 seconds: | 20260505 22:03:47.801 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:03:47.801 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:47.801 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:47.801 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:47.801 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260505 22:03:47.801 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:47.801 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:47.801 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 22:03:47.801 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:47.801 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:47.802 | INFO | Uploaded 36GiB to GPU in 2.08663s (17.2527GiB/s) | 20260505 22:03:47.802 | INFO | Downloaded 34GiB to GPU in 2.02973s (16.751GiB/s) | 20260505 22:03:52.940 | INFO | Analyzing chunk frames = [6648832,6649344) | 20260505 22:03:52.940 | INFO | MemInfo: { | 20260505 22:03:52.940 | INFO | "gpu" : " 17 GB", | 20260505 22:03:52.940 | INFO | "huge" : "", | 20260505 22:03:52.940 | INFO | "huge_shared" : " 27 GB", | 20260505 22:03:52.940 | INFO | "malloc" : " 14 GB", | 20260505 22:03:52.940 | INFO | "pinned" : " 29 GB" | 20260505 22:03:52.940 | INFO | } | 20260505 22:03:52.940 | INFO | Perf report for the last 5.13916 seconds: | 20260505 22:03:52.940 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:03:52.940 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:52.940 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:52.940 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:52.940 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260505 22:03:52.940 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:52.940 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:52.940 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:03:52.940 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:52.940 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:52.941 | INFO | Uploaded 36GiB to GPU in 2.13874s (16.8323GiB/s) | 20260505 22:03:52.941 | INFO | Downloaded 34GiB to GPU in 2.02759s (16.7686GiB/s) | 20260505 22:03:58.089 | INFO | Analyzing chunk frames = [6649344,6649856) | 20260505 22:03:58.089 | INFO | MemInfo: { | 20260505 22:03:58.089 | INFO | "gpu" : " 17 GB", | 20260505 22:03:58.089 | INFO | "huge" : "", | 20260505 22:03:58.089 | INFO | "huge_shared" : " 27 GB", | 20260505 22:03:58.089 | INFO | "malloc" : " 14 GB", | 20260505 22:03:58.089 | INFO | "pinned" : " 29 GB" | 20260505 22:03:58.089 | INFO | } | 20260505 22:03:58.089 | INFO | Perf report for the last 5.14855 seconds: | 20260505 22:03:58.089 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:03:58.089 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:58.089 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:58.089 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:58.089 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260505 22:03:58.089 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:58.089 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:58.089 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:03:58.089 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:58.089 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:03:58.089 | INFO | Uploaded 36GiB to GPU in 2.13354s (16.8734GiB/s) | 20260505 22:03:58.089 | INFO | Downloaded 34GiB to GPU in 2.0488s (16.5951GiB/s) | 20260505 22:04:03.228 | INFO | Analyzing chunk frames = [6649856,6650368) | 20260505 22:04:03.228 | INFO | MemInfo: { | 20260505 22:04:03.228 | INFO | "gpu" : " 17 GB", | 20260505 22:04:03.228 | INFO | "huge" : "", | 20260505 22:04:03.228 | INFO | "huge_shared" : " 27 GB", | 20260505 22:04:03.228 | INFO | "malloc" : " 14 GB", | 20260505 22:04:03.228 | INFO | "pinned" : " 29 GB" | 20260505 22:04:03.228 | INFO | } | 20260505 22:04:03.228 | INFO | Perf report for the last 5.13959 seconds: | 20260505 22:04:03.229 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:04:03.229 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:03.229 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:03.229 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:03.229 | INFO | ANALYSIS : 61.6%, 3.0, 96.2ms, 0 | 20260505 22:04:03.229 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:03.229 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:03.229 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 22:04:03.229 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:03.229 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:03.229 | INFO | Uploaded 36GiB to GPU in 2.1525s (16.7247GiB/s) | 20260505 22:04:03.229 | INFO | Downloaded 34GiB to GPU in 2.06868s (16.4356GiB/s) | 20260505 22:04:08.366 | INFO | Analyzing chunk frames = [6650368,6650880) | 20260505 22:04:08.366 | INFO | MemInfo: { | 20260505 22:04:08.366 | INFO | "gpu" : " 17 GB", | 20260505 22:04:08.366 | INFO | "huge" : "", | 20260505 22:04:08.366 | INFO | "huge_shared" : " 27 GB", | 20260505 22:04:08.366 | INFO | "malloc" : " 14 GB", | 20260505 22:04:08.366 | INFO | "pinned" : " 29 GB" | 20260505 22:04:08.366 | INFO | } | 20260505 22:04:08.367 | INFO | Perf report for the last 5.13812 seconds: | 20260505 22:04:08.367 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:04:08.367 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:08.367 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:08.367 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:08.367 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260505 22:04:08.367 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:08.367 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:08.367 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:04:08.367 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:08.367 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:08.367 | INFO | Uploaded 36GiB to GPU in 2.07536s (17.3464GiB/s) | 20260505 22:04:08.367 | INFO | Downloaded 34GiB to GPU in 2.03314s (16.7229GiB/s) | 20260505 22:04:13.503 | INFO | Analyzing chunk frames = [6650880,6651392) | 20260505 22:04:13.503 | INFO | MemInfo: { | 20260505 22:04:13.503 | INFO | "gpu" : " 17 GB", | 20260505 22:04:13.503 | INFO | "huge" : "", | 20260505 22:04:13.503 | INFO | "huge_shared" : " 27 GB", | 20260505 22:04:13.503 | INFO | "malloc" : " 14 GB", | 20260505 22:04:13.503 | INFO | "pinned" : " 29 GB" | 20260505 22:04:13.503 | INFO | } | 20260505 22:04:13.503 | INFO | Perf report for the last 5.13656 seconds: | 20260505 22:04:13.503 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:04:13.503 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:13.503 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:13.503 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:13.503 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 22:04:13.503 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:13.503 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:13.503 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 22:04:13.503 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:13.503 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:13.504 | INFO | Uploaded 36GiB to GPU in 2.16407s (16.6353GiB/s) | 20260505 22:04:13.504 | INFO | Downloaded 34GiB to GPU in 2.11957s (16.041GiB/s) | 20260505 22:04:18.644 | INFO | Analyzing chunk frames = [6651392,6651904) | 20260505 22:04:18.644 | INFO | MemInfo: { | 20260505 22:04:18.644 | INFO | "gpu" : " 17 GB", | 20260505 22:04:18.644 | INFO | "huge" : "", | 20260505 22:04:18.644 | INFO | "huge_shared" : " 27 GB", | 20260505 22:04:18.644 | INFO | "malloc" : " 14 GB", | 20260505 22:04:18.644 | INFO | "pinned" : " 29 GB" | 20260505 22:04:18.644 | INFO | } | 20260505 22:04:18.644 | INFO | Perf report for the last 5.14121 seconds: | 20260505 22:04:18.644 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:04:18.644 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:18.644 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:18.644 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:18.644 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 22:04:18.644 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:18.644 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:18.644 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:04:18.644 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:18.644 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:18.645 | INFO | Uploaded 36GiB to GPU in 2.11189s (17.0463GiB/s) | 20260505 22:04:18.645 | INFO | Downloaded 34GiB to GPU in 2.00777s (16.9342GiB/s) | 20260505 22:04:18.645 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 22:04:18.645 | INFO | Total mem usage: 15.3 GiB | 20260505 22:04:18.645 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 22:04:18.645 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.75%) | 20260505 22:04:18.645 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 22:04:18.645 | INFO | High Water Breakdown: | 20260505 22:04:18.645 | INFO | 61% : 8.59 GiB ( 90.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 22:04:18.645 | INFO | 38.9% : 5.48 GiB ( 82.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 22:04:18.645 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 22:04:18.645 | INFO | ------------------------------------------ | 20260505 22:04:18.645 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:04:18.645 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 22:04:18.645 | INFO | Total mem usage: 26.5 GiB | 20260505 22:04:18.645 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 22:04:18.645 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 22:04:18.645 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 22:04:18.645 | INFO | High Water Breakdown: | 20260505 22:04:18.645 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:04:18.645 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:04:18.645 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:04:18.645 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:04:18.645 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:04:18.645 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:04:18.645 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 22:04:18.645 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 22:04:18.645 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:04:18.645 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 22:04:18.645 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 22:04:18.645 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 22:04:18.645 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 22:04:18.645 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 22:04:18.645 | INFO | ------------------------------------------ | 20260505 22:04:18.645 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:04:18.645 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 22:04:18.645 | INFO | Total mem usage: 25.6 GiB | 20260505 22:04:18.645 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 22:04:18.645 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 22:04:18.645 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 22:04:18.645 | INFO | High Water Breakdown: | 20260505 22:04:18.645 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 22:04:18.645 | INFO | ------------------------------------------ | 20260505 22:04:18.645 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:04:18.645 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 22:04:18.645 | INFO | Total mem usage: 15.7 GiB | 20260505 22:04:18.645 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 22:04:18.645 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 22:04:18.645 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 22:04:18.645 | INFO | High Water Breakdown: | 20260505 22:04:18.645 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 22:04:18.645 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 22:04:18.645 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 22:04:18.645 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 22:04:18.645 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 22:04:18.645 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 22:04:18.645 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 22:04:18.645 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 22:04:18.645 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 22:04:18.645 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:04:18.645 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 22:04:18.645 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:04:18.645 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 22:04:18.646 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 22:04:18.646 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 22:04:18.646 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 22:04:18.646 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:04:18.646 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 22:04:18.646 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:04:18.646 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:04:18.646 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:04:18.646 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 22:04:18.646 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 22:04:18.646 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:04:18.646 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 22:04:18.646 | INFO | ------------------------------------------ | 20260505 22:04:18.646 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:04:18.646 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 22:04:18.646 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:04:18.646 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:04:18.646 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.40e+01, 1.35e+01 / 1.24e+00 | 20260505 22:04:18.646 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.03e-01, 1.05e-02 / 4.09e-03 | 20260505 22:04:18.646 | INFO | PerfReport: 11.80%, Baseline -- 3.70e+00 / 4.52e+00, 5.59e-01 / 2.06e-01 | 20260505 22:04:18.646 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.53e+00, 1.89e-01 / 9.04e-02 | 20260505 22:04:18.646 | INFO | PerfReport: 15.95%, DME -- 4.00e+01 / 4.39e+01, 3.86e+00 / 1.23e+00 | 20260505 22:04:18.646 | INFO | PerfReport: 57.20%, FrameLabeling -- 1.79e+01 / 1.94e+01, 1.46e+00 / 1.01e+00 | 20260505 22:04:18.646 | INFO | PerfReport: 7.20%, PulseAccumulating -- 2.26e+00 / 2.81e+00, 4.31e-01 / 1.92e-01 | 20260505 22:04:18.646 | INFO | PerfReport: 3.18%, Metrics -- 9.97e-01 / 1.31e+00, 2.73e-01 / 1.63e-01 | 20260505 22:04:18.646 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.46e-01 / 4.06e-01, 5.94e-02 / 3.80e-02 | 20260505 22:04:18.646 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 22:04:18.646 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:04:18.646 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:04:18.646 | INFO | PerfReport: Full Execution -- 9.88e+01 / -nan, 2.50e+01 / 0.00e+00 | 20260505 22:04:18.646 | INFO | PerfReport: 22.42%, Upload -- 2.22e+01 / 0.00e+00, 7.59e+00 / 0.00e+00 | 20260505 22:04:18.646 | INFO | PerfReport: 21.82%, Download -- 2.16e+01 / 0.00e+00, 7.61e+00 / 0.00e+00 | 20260505 22:04:18.646 | INFO | PerfReport: 31.74%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 22:04:18.646 | INFO | PerfReport: 2.24%, LockUpload -- 2.21e+00 / 0.00e+00, 8.02e+00 / 0.00e+00 | 20260505 22:04:18.646 | INFO | PerfReport: 1.37%, LockDownload -- 1.35e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260505 22:04:18.646 | INFO | PerfReport: 20.40%, LockCompute -- 2.02e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 22:04:18.646 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.76e-02 / 0.00e+00 | 20260505 22:04:18.767 | INFO | Starting a round of Baz IO | 20260505 22:04:21.920 | INFO | Populating RTMetrics report | 20260505 22:04:21.920 | INFO | RTMetrics report populated | 20260505 22:04:21.920 | INFO | Opening tmp RTMetrics json file | 20260505 22:04:21.921 | INFO | Serializing RTMetrics to json | 20260505 22:04:21.922 | INFO | Serialization done, swapping tmp file now... | 20260505 22:04:21.922 | INFO | RTMetrics done | 20260505 22:04:22.002 | INFO | Finished a round of Baz IO: Wrote 1.944 GiB in 3.234 seconds | 20260505 22:04:22.002 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.53% / 0.00% / 78.47% | 20260505 22:04:22.002 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.132251% / 0% / 8.74201e-05% | 20260505 22:04:22.002 | INFO | Baz IO overall: | 20260505 22:04:22.002 | INFO | BytesWritten : 3838529265576 | 20260505 22:04:22.002 | INFO | NumEvents : 1811301259662 | 20260505 22:04:23.781 | INFO | Analyzing chunk frames = [6651904,6652416) | 20260505 22:04:23.781 | INFO | MemInfo: { | 20260505 22:04:23.781 | INFO | "gpu" : " 17 GB", | 20260505 22:04:23.781 | INFO | "huge" : "", | 20260505 22:04:23.781 | INFO | "huge_shared" : " 27 GB", | 20260505 22:04:23.781 | INFO | "malloc" : " 14 GB", | 20260505 22:04:23.781 | INFO | "pinned" : " 29 GB" | 20260505 22:04:23.781 | INFO | } | 20260505 22:04:23.781 | INFO | Perf report for the last 5.13701 seconds: | 20260505 22:04:23.782 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:04:23.782 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:23.782 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:23.782 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:23.782 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 22:04:23.782 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:23.782 | INFO | RT_METRICS : 43.8%, 1.0, 22.9ms, 0 | 20260505 22:04:23.782 | INFO | PRE_HQ : 60.1%, 1.8, 54.9ms, 0 | 20260505 22:04:23.782 | INFO | BAZWRITER : 62.2%, 3.6, 115.8ms, 0 | 20260505 22:04:23.782 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 22:04:23.782 | INFO | Uploaded 36GiB to GPU in 2.15941s (16.6712GiB/s) | 20260505 22:04:23.782 | INFO | Downloaded 38GiB to GPU in 2.23339s (17.0145GiB/s) | 20260505 22:04:28.917 | INFO | Analyzing chunk frames = [6652416,6652928) | 20260505 22:04:28.917 | INFO | MemInfo: { | 20260505 22:04:28.917 | INFO | "gpu" : " 17 GB", | 20260505 22:04:28.917 | INFO | "huge" : "", | 20260505 22:04:28.917 | INFO | "huge_shared" : " 27 GB", | 20260505 22:04:28.917 | INFO | "malloc" : " 14 GB", | 20260505 22:04:28.917 | INFO | "pinned" : " 29 GB" | 20260505 22:04:28.917 | INFO | } | 20260505 22:04:28.917 | INFO | Perf report for the last 5.13549 seconds: | 20260505 22:04:28.917 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:04:28.917 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:28.917 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:28.917 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:28.917 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260505 22:04:28.917 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:28.917 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:28.917 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 22:04:28.917 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:28.917 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:28.917 | INFO | Uploaded 36GiB to GPU in 2.09107s (17.2161GiB/s) | 20260505 22:04:28.917 | INFO | Downloaded 34GiB to GPU in 2.03036s (16.7458GiB/s) | 20260505 22:04:34.057 | INFO | Analyzing chunk frames = [6652928,6653440) | 20260505 22:04:34.057 | INFO | MemInfo: { | 20260505 22:04:34.057 | INFO | "gpu" : " 17 GB", | 20260505 22:04:34.057 | INFO | "huge" : "", | 20260505 22:04:34.057 | INFO | "huge_shared" : " 27 GB", | 20260505 22:04:34.057 | INFO | "malloc" : " 14 GB", | 20260505 22:04:34.057 | INFO | "pinned" : " 29 GB" | 20260505 22:04:34.057 | INFO | } | 20260505 22:04:34.057 | INFO | Perf report for the last 5.14044 seconds: | 20260505 22:04:34.057 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:04:34.057 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:34.057 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:34.057 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:34.057 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260505 22:04:34.057 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:34.057 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:34.057 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 22:04:34.057 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:34.057 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:34.058 | INFO | Uploaded 36GiB to GPU in 2.1321s (16.8848GiB/s) | 20260505 22:04:34.058 | INFO | Downloaded 34GiB to GPU in 2.0153s (16.8709GiB/s) | 20260505 22:04:39.195 | INFO | Analyzing chunk frames = [6653440,6653952) | 20260505 22:04:39.195 | INFO | MemInfo: { | 20260505 22:04:39.195 | INFO | "gpu" : " 17 GB", | 20260505 22:04:39.195 | INFO | "huge" : "", | 20260505 22:04:39.195 | INFO | "huge_shared" : " 27 GB", | 20260505 22:04:39.195 | INFO | "malloc" : " 14 GB", | 20260505 22:04:39.195 | INFO | "pinned" : " 29 GB" | 20260505 22:04:39.195 | INFO | } | 20260505 22:04:39.195 | INFO | Perf report for the last 5.13741 seconds: | 20260505 22:04:39.195 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:04:39.195 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:39.195 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:39.195 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:39.195 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260505 22:04:39.195 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:39.195 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:39.195 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 22:04:39.195 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:39.195 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:39.195 | INFO | Uploaded 36GiB to GPU in 2.21022s (16.288GiB/s) | 20260505 22:04:39.195 | INFO | Downloaded 34GiB to GPU in 2.0489s (16.5943GiB/s) | 20260505 22:04:44.337 | INFO | Analyzing chunk frames = [6653952,6654464) | 20260505 22:04:44.337 | INFO | MemInfo: { | 20260505 22:04:44.337 | INFO | "gpu" : " 17 GB", | 20260505 22:04:44.337 | INFO | "huge" : "", | 20260505 22:04:44.337 | INFO | "huge_shared" : " 27 GB", | 20260505 22:04:44.337 | INFO | "malloc" : " 14 GB", | 20260505 22:04:44.337 | INFO | "pinned" : " 29 GB" | 20260505 22:04:44.337 | INFO | } | 20260505 22:04:44.337 | INFO | Perf report for the last 5.14273 seconds: | 20260505 22:04:44.337 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:04:44.337 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:44.337 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:44.337 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:44.337 | INFO | ANALYSIS : 61.6%, 3.0, 96.2ms, 0 | 20260505 22:04:44.337 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:44.338 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:44.338 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:04:44.338 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:44.338 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:44.338 | INFO | Uploaded 36GiB to GPU in 2.13678s (16.8478GiB/s) | 20260505 22:04:44.338 | INFO | Downloaded 34GiB to GPU in 2.06005s (16.5045GiB/s) | 20260505 22:04:49.485 | INFO | Analyzing chunk frames = [6654464,6654976) | 20260505 22:04:49.485 | INFO | MemInfo: { | 20260505 22:04:49.485 | INFO | "gpu" : " 17 GB", | 20260505 22:04:49.485 | INFO | "huge" : "", | 20260505 22:04:49.485 | INFO | "huge_shared" : " 27 GB", | 20260505 22:04:49.485 | INFO | "malloc" : " 14 GB", | 20260505 22:04:49.485 | INFO | "pinned" : " 29 GB" | 20260505 22:04:49.485 | INFO | } | 20260505 22:04:49.485 | INFO | Perf report for the last 5.14743 seconds: | 20260505 22:04:49.485 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:04:49.485 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:49.485 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:49.485 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:49.485 | INFO | ANALYSIS : 61.6%, 3.0, 96.3ms, 0 | 20260505 22:04:49.485 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:49.485 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:49.485 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 22:04:49.485 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:49.485 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:49.485 | INFO | Uploaded 36GiB to GPU in 2.08604s (17.2576GiB/s) | 20260505 22:04:49.485 | INFO | Downloaded 34GiB to GPU in 2.0347s (16.7101GiB/s) | 20260505 22:04:54.619 | INFO | Analyzing chunk frames = [6654976,6655488) | 20260505 22:04:54.619 | INFO | MemInfo: { | 20260505 22:04:54.619 | INFO | "gpu" : " 17 GB", | 20260505 22:04:54.619 | INFO | "huge" : "", | 20260505 22:04:54.619 | INFO | "huge_shared" : " 27 GB", | 20260505 22:04:54.619 | INFO | "malloc" : " 14 GB", | 20260505 22:04:54.619 | INFO | "pinned" : " 29 GB" | 20260505 22:04:54.619 | INFO | } | 20260505 22:04:54.619 | INFO | Perf report for the last 5.1342 seconds: | 20260505 22:04:54.619 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:04:54.619 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:54.619 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:54.619 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:54.619 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 22:04:54.619 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:54.619 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:54.619 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 22:04:54.619 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:54.619 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:54.619 | INFO | Uploaded 36GiB to GPU in 2.01954s (17.8258GiB/s) | 20260505 22:04:54.620 | INFO | Downloaded 34GiB to GPU in 2.03363s (16.7189GiB/s) | 20260505 22:04:59.749 | INFO | Analyzing chunk frames = [6655488,6656000) | 20260505 22:04:59.749 | INFO | MemInfo: { | 20260505 22:04:59.749 | INFO | "gpu" : " 17 GB", | 20260505 22:04:59.749 | INFO | "huge" : "", | 20260505 22:04:59.749 | INFO | "huge_shared" : " 27 GB", | 20260505 22:04:59.749 | INFO | "malloc" : " 14 GB", | 20260505 22:04:59.749 | INFO | "pinned" : " 29 GB" | 20260505 22:04:59.749 | INFO | } | 20260505 22:04:59.749 | INFO | Perf report for the last 5.13021 seconds: | 20260505 22:04:59.749 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:04:59.749 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:59.749 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:59.749 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:59.749 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 22:04:59.749 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:59.749 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:59.749 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:04:59.749 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:59.749 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:04:59.750 | INFO | Uploaded 36GiB to GPU in 2.19752s (16.3821GiB/s) | 20260505 22:04:59.750 | INFO | Downloaded 34GiB to GPU in 2.05692s (16.5296GiB/s) | 20260505 22:04:59.884 | INFO | Starting a round of Baz IO | 20260505 22:05:03.042 | INFO | Populating RTMetrics report | 20260505 22:05:03.042 | INFO | RTMetrics report populated | 20260505 22:05:03.042 | INFO | Opening tmp RTMetrics json file | 20260505 22:05:03.043 | INFO | Serializing RTMetrics to json | 20260505 22:05:03.045 | INFO | Serialization done, swapping tmp file now... | 20260505 22:05:03.047 | INFO | RTMetrics done | 20260505 22:05:03.130 | INFO | Finished a round of Baz IO: Wrote 1.945 GiB in 3.246 seconds | 20260505 22:05:03.130 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.52% / 0.00% / 78.47% | 20260505 22:05:03.130 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.132489% / 0% / 7.54992e-05% | 20260505 22:05:03.130 | INFO | Baz IO overall: | 20260505 22:05:03.130 | INFO | BytesWritten : 3840617607442 | 20260505 22:05:03.130 | INFO | NumEvents : 1812263461796 | 20260505 22:05:04.895 | INFO | Analyzing chunk frames = [6656000,6656512) | 20260505 22:05:04.895 | INFO | MemInfo: { | 20260505 22:05:04.895 | INFO | "gpu" : " 17 GB", | 20260505 22:05:04.895 | INFO | "huge" : "", | 20260505 22:05:04.895 | INFO | "huge_shared" : " 27 GB", | 20260505 22:05:04.895 | INFO | "malloc" : " 14 GB", | 20260505 22:05:04.895 | INFO | "pinned" : " 29 GB" | 20260505 22:05:04.895 | INFO | } | 20260505 22:05:04.895 | INFO | Perf report for the last 5.14566 seconds: | 20260505 22:05:04.895 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:05:04.895 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:04.895 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:04.895 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:04.895 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 22:05:04.895 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:04.895 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260505 22:05:04.895 | INFO | PRE_HQ : 59.6%, 1.8, 54.8ms, 0 | 20260505 22:05:04.895 | INFO | BAZWRITER : 62.6%, 3.6, 115.9ms, 0 | 20260505 22:05:04.895 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 22:05:04.895 | INFO | Uploaded 36GiB to GPU in 2.15905s (16.674GiB/s) | 20260505 22:05:04.895 | INFO | Downloaded 38GiB to GPU in 2.24787s (16.9049GiB/s) | 20260505 22:05:10.037 | INFO | Analyzing chunk frames = [6656512,6657024) | 20260505 22:05:10.037 | INFO | MemInfo: { | 20260505 22:05:10.037 | INFO | "gpu" : " 17 GB", | 20260505 22:05:10.037 | INFO | "huge" : "", | 20260505 22:05:10.037 | INFO | "huge_shared" : " 27 GB", | 20260505 22:05:10.037 | INFO | "malloc" : " 14 GB", | 20260505 22:05:10.037 | INFO | "pinned" : " 29 GB" | 20260505 22:05:10.037 | INFO | } | 20260505 22:05:10.037 | INFO | Perf report for the last 5.14236 seconds: | 20260505 22:05:10.037 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:05:10.037 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:10.037 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:10.037 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:10.037 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 22:05:10.037 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:10.037 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:10.038 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 22:05:10.038 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:10.038 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:10.038 | INFO | Uploaded 36GiB to GPU in 2.0839s (17.2753GiB/s) | 20260505 22:05:10.038 | INFO | Downloaded 34GiB to GPU in 2.02409s (16.7977GiB/s) | 20260505 22:05:15.176 | INFO | Analyzing chunk frames = [6657024,6657536) | 20260505 22:05:15.176 | INFO | MemInfo: { | 20260505 22:05:15.176 | INFO | "gpu" : " 17 GB", | 20260505 22:05:15.176 | INFO | "huge" : "", | 20260505 22:05:15.176 | INFO | "huge_shared" : " 27 GB", | 20260505 22:05:15.176 | INFO | "malloc" : " 14 GB", | 20260505 22:05:15.176 | INFO | "pinned" : " 29 GB" | 20260505 22:05:15.176 | INFO | } | 20260505 22:05:15.176 | INFO | Perf report for the last 5.13828 seconds: | 20260505 22:05:15.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:05:15.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:15.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:15.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:15.176 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260505 22:05:15.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:15.176 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:15.176 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:05:15.176 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:15.176 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:15.176 | INFO | Uploaded 36GiB to GPU in 2.14319s (16.7974GiB/s) | 20260505 22:05:15.176 | INFO | Downloaded 34GiB to GPU in 2.04967s (16.588GiB/s) | 20260505 22:05:20.315 | INFO | Analyzing chunk frames = [6657536,6658048) | 20260505 22:05:20.315 | INFO | MemInfo: { | 20260505 22:05:20.315 | INFO | "gpu" : " 17 GB", | 20260505 22:05:20.315 | INFO | "huge" : "", | 20260505 22:05:20.315 | INFO | "huge_shared" : " 27 GB", | 20260505 22:05:20.315 | INFO | "malloc" : " 14 GB", | 20260505 22:05:20.315 | INFO | "pinned" : " 29 GB" | 20260505 22:05:20.315 | INFO | } | 20260505 22:05:20.315 | INFO | Perf report for the last 5.13917 seconds: | 20260505 22:05:20.315 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:05:20.315 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:20.315 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:20.315 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:20.315 | INFO | ANALYSIS : 61.8%, 3.0, 96.0ms, 0 | 20260505 22:05:20.315 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:20.315 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:20.315 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:05:20.315 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:20.315 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:20.315 | INFO | Uploaded 36GiB to GPU in 2.13136s (16.8906GiB/s) | 20260505 22:05:20.315 | INFO | Downloaded 34GiB to GPU in 2.03192s (16.7329GiB/s) | 20260505 22:05:25.456 | INFO | Analyzing chunk frames = [6658048,6658560) | 20260505 22:05:25.456 | INFO | MemInfo: { | 20260505 22:05:25.456 | INFO | "gpu" : " 17 GB", | 20260505 22:05:25.456 | INFO | "huge" : "", | 20260505 22:05:25.456 | INFO | "huge_shared" : " 27 GB", | 20260505 22:05:25.456 | INFO | "malloc" : " 14 GB", | 20260505 22:05:25.456 | INFO | "pinned" : " 29 GB" | 20260505 22:05:25.456 | INFO | } | 20260505 22:05:25.456 | INFO | Perf report for the last 5.14072 seconds: | 20260505 22:05:25.456 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:05:25.456 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:25.456 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:25.456 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:25.456 | INFO | ANALYSIS : 61.6%, 3.0, 96.2ms, 0 | 20260505 22:05:25.456 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:25.456 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:25.456 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:05:25.456 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:25.456 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:25.456 | INFO | Uploaded 36GiB to GPU in 2.13616s (16.8527GiB/s) | 20260505 22:05:25.456 | INFO | Downloaded 34GiB to GPU in 2.06389s (16.4737GiB/s) | 20260505 22:05:30.597 | INFO | Analyzing chunk frames = [6658560,6659072) | 20260505 22:05:30.597 | INFO | MemInfo: { | 20260505 22:05:30.597 | INFO | "gpu" : " 17 GB", | 20260505 22:05:30.597 | INFO | "huge" : "", | 20260505 22:05:30.598 | INFO | "huge_shared" : " 27 GB", | 20260505 22:05:30.598 | INFO | "malloc" : " 14 GB", | 20260505 22:05:30.598 | INFO | "pinned" : " 29 GB" | 20260505 22:05:30.598 | INFO | } | 20260505 22:05:30.598 | INFO | Perf report for the last 5.14198 seconds: | 20260505 22:05:30.598 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:05:30.598 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:30.598 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:30.598 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:30.598 | INFO | ANALYSIS : 61.6%, 3.0, 96.2ms, 0 | 20260505 22:05:30.598 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:30.598 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:30.598 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 22:05:30.598 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:30.598 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:30.598 | INFO | Uploaded 36GiB to GPU in 2.07934s (17.3132GiB/s) | 20260505 22:05:30.598 | INFO | Downloaded 34GiB to GPU in 2.03598s (16.6995GiB/s) | 20260505 22:05:35.734 | INFO | Analyzing chunk frames = [6659072,6659584) | 20260505 22:05:35.734 | INFO | MemInfo: { | 20260505 22:05:35.734 | INFO | "gpu" : " 17 GB", | 20260505 22:05:35.734 | INFO | "huge" : "", | 20260505 22:05:35.734 | INFO | "huge_shared" : " 27 GB", | 20260505 22:05:35.734 | INFO | "malloc" : " 14 GB", | 20260505 22:05:35.734 | INFO | "pinned" : " 29 GB" | 20260505 22:05:35.734 | INFO | } | 20260505 22:05:35.734 | INFO | Perf report for the last 5.13602 seconds: | 20260505 22:05:35.734 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:05:35.734 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:35.734 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:35.734 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:35.734 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260505 22:05:35.734 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:35.734 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:35.734 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 22:05:35.734 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:35.734 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:35.734 | INFO | Uploaded 36GiB to GPU in 2.07316s (17.3648GiB/s) | 20260505 22:05:35.734 | INFO | Downloaded 34GiB to GPU in 2.07592s (16.3783GiB/s) | 20260505 22:05:40.866 | INFO | Analyzing chunk frames = [6659584,6660096) | 20260505 22:05:40.866 | INFO | MemInfo: { | 20260505 22:05:40.866 | INFO | "gpu" : " 17 GB", | 20260505 22:05:40.866 | INFO | "huge" : "", | 20260505 22:05:40.866 | INFO | "huge_shared" : " 27 GB", | 20260505 22:05:40.866 | INFO | "malloc" : " 14 GB", | 20260505 22:05:40.866 | INFO | "pinned" : " 29 GB" | 20260505 22:05:40.866 | INFO | } | 20260505 22:05:40.866 | INFO | Perf report for the last 5.13207 seconds: | 20260505 22:05:40.866 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:05:40.866 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:40.866 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:40.866 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:40.866 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 22:05:40.866 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:40.866 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:40.866 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 22:05:40.866 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:40.866 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:40.866 | INFO | Uploaded 36GiB to GPU in 2.11329s (17.035GiB/s) | 20260505 22:05:40.866 | INFO | Downloaded 34GiB to GPU in 2.00026s (16.9978GiB/s) | 20260505 22:05:40.866 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 22:05:40.866 | INFO | Total mem usage: 15.3 GiB | 20260505 22:05:40.866 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 22:05:40.866 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.2%) | 20260505 22:05:40.866 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 22:05:40.866 | INFO | High Water Breakdown: | 20260505 22:05:40.866 | INFO | 61% : 8.64 GiB ( 90.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 22:05:40.866 | INFO | 38.8% : 5.5 GiB ( 82.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 22:05:40.866 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 22:05:40.866 | INFO | ------------------------------------------ | 20260505 22:05:40.866 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:05:40.866 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 22:05:40.867 | INFO | Total mem usage: 26.5 GiB | 20260505 22:05:40.867 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 22:05:40.867 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 22:05:40.867 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 22:05:40.867 | INFO | High Water Breakdown: | 20260505 22:05:40.867 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:05:40.867 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:05:40.867 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:05:40.867 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:05:40.867 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:05:40.867 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:05:40.867 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 22:05:40.867 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 22:05:40.867 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:05:40.867 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 22:05:40.867 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 22:05:40.867 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 22:05:40.867 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 22:05:40.867 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 22:05:40.867 | INFO | ------------------------------------------ | 20260505 22:05:40.867 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:05:40.867 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 22:05:40.867 | INFO | Total mem usage: 25.6 GiB | 20260505 22:05:40.867 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 22:05:40.867 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 22:05:40.867 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 22:05:40.867 | INFO | High Water Breakdown: | 20260505 22:05:40.867 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 22:05:40.867 | INFO | ------------------------------------------ | 20260505 22:05:40.867 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:05:40.867 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 22:05:40.867 | INFO | Total mem usage: 15.7 GiB | 20260505 22:05:40.867 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 22:05:40.867 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 22:05:40.867 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 22:05:40.867 | INFO | High Water Breakdown: | 20260505 22:05:40.867 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 22:05:40.867 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 22:05:40.867 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 22:05:40.867 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 22:05:40.867 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 22:05:40.867 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 22:05:40.867 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 22:05:40.867 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 22:05:40.867 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 22:05:40.867 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:05:40.867 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 22:05:40.867 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:05:40.867 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 22:05:40.867 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 22:05:40.867 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 22:05:40.867 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 22:05:40.867 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:05:40.867 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 22:05:40.867 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:05:40.867 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:05:40.867 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:05:40.867 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 22:05:40.867 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 22:05:40.867 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:05:40.867 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 22:05:40.867 | INFO | ------------------------------------------ | 20260505 22:05:40.867 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:05:40.867 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 22:05:40.867 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:05:40.867 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:05:40.867 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.41e+01, 1.35e+01 / 9.57e-01 | 20260505 22:05:40.867 | INFO | PerfReport: 0.29%, AggregateStats -- 9.17e-02 / 1.03e-01, 1.02e-02 / 3.63e-03 | 20260505 22:05:40.867 | INFO | PerfReport: 11.80%, Baseline -- 3.70e+00 / 4.52e+00, 5.63e-01 / 1.94e-01 | 20260505 22:05:40.867 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.53e+00, 1.88e-01 / 7.51e-02 | 20260505 22:05:40.867 | INFO | PerfReport: 15.99%, DME -- 4.01e+01 / 4.41e+01, 3.91e+00 / 1.08e+00 | 20260505 22:05:40.867 | INFO | PerfReport: 57.17%, FrameLabeling -- 1.79e+01 / 1.93e+01, 1.46e+00 / 1.02e+00 | 20260505 22:05:40.867 | INFO | PerfReport: 7.20%, PulseAccumulating -- 2.26e+00 / 2.81e+00, 4.30e-01 / 2.30e-01 | 20260505 22:05:40.867 | INFO | PerfReport: 3.18%, Metrics -- 9.96e-01 / 1.30e+00, 2.68e-01 / 1.09e-01 | 20260505 22:05:40.867 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.46e-01 / 4.09e-01, 5.85e-02 / 3.38e-02 | 20260505 22:05:40.867 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 22:05:40.867 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:05:40.867 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:05:40.867 | INFO | PerfReport: Full Execution -- 9.88e+01 / -nan, 2.49e+01 / 0.00e+00 | 20260505 22:05:40.867 | INFO | PerfReport: 22.36%, Upload -- 2.21e+01 / 0.00e+00, 7.60e+00 / 0.00e+00 | 20260505 22:05:40.867 | INFO | PerfReport: 21.77%, Download -- 2.15e+01 / 0.00e+00, 7.58e+00 / 0.00e+00 | 20260505 22:05:40.867 | INFO | PerfReport: 31.74%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 22:05:40.867 | INFO | PerfReport: 2.24%, LockUpload -- 2.21e+00 / 0.00e+00, 7.93e+00 / 0.00e+00 | 20260505 22:05:40.867 | INFO | PerfReport: 1.30%, LockDownload -- 1.29e+00 / 0.00e+00, 2.84e+00 / 0.00e+00 | 20260505 22:05:40.868 | INFO | PerfReport: 20.57%, LockCompute -- 2.03e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260505 22:05:40.868 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.43e-02 / 0.00e+00 | 20260505 22:05:41.000 | INFO | Starting a round of Baz IO | 20260505 22:05:44.167 | INFO | Populating RTMetrics report | 20260505 22:05:44.168 | INFO | RTMetrics report populated | 20260505 22:05:44.168 | INFO | Opening tmp RTMetrics json file | 20260505 22:05:44.169 | INFO | Serializing RTMetrics to json | 20260505 22:05:44.170 | INFO | Serialization done, swapping tmp file now... | 20260505 22:05:44.170 | INFO | RTMetrics done | 20260505 22:05:44.253 | INFO | Finished a round of Baz IO: Wrote 1.945 GiB in 3.253 seconds | 20260505 22:05:44.253 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.52% / 0.00% / 78.48% | 20260505 22:05:44.253 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.132267% / 0% / 8.34465e-05% | 20260505 22:05:44.253 | INFO | Baz IO overall: | 20260505 22:05:44.253 | INFO | BytesWritten : 3842705571235 | 20260505 22:05:44.253 | INFO | NumEvents : 1813225489868 | 20260505 22:05:46.004 | INFO | Analyzing chunk frames = [6660096,6660608) | 20260505 22:05:46.004 | INFO | MemInfo: { | 20260505 22:05:46.004 | INFO | "gpu" : " 17 GB", | 20260505 22:05:46.004 | INFO | "huge" : "", | 20260505 22:05:46.004 | INFO | "huge_shared" : " 27 GB", | 20260505 22:05:46.004 | INFO | "malloc" : " 14 GB", | 20260505 22:05:46.004 | INFO | "pinned" : " 29 GB" | 20260505 22:05:46.004 | INFO | } | 20260505 22:05:46.004 | INFO | Perf report for the last 5.13787 seconds: | 20260505 22:05:46.004 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:05:46.004 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:46.004 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:46.004 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:46.004 | INFO | ANALYSIS : 63.2%, 3.0, 98.6ms, 0 | 20260505 22:05:46.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:46.004 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260505 22:05:46.004 | INFO | PRE_HQ : 60.2%, 1.8, 54.7ms, 0 | 20260505 22:05:46.004 | INFO | BAZWRITER : 62.9%, 3.6, 115.5ms, 0 | 20260505 22:05:46.004 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 22:05:46.004 | INFO | Uploaded 36GiB to GPU in 2.18388s (16.4844GiB/s) | 20260505 22:05:46.004 | INFO | Downloaded 38GiB to GPU in 2.26696s (16.7626GiB/s) | 20260505 22:05:51.151 | INFO | Analyzing chunk frames = [6660608,6661120) | 20260505 22:05:51.151 | INFO | MemInfo: { | 20260505 22:05:51.151 | INFO | "gpu" : " 17 GB", | 20260505 22:05:51.151 | INFO | "huge" : "", | 20260505 22:05:51.151 | INFO | "huge_shared" : " 27 GB", | 20260505 22:05:51.151 | INFO | "malloc" : " 14 GB", | 20260505 22:05:51.151 | INFO | "pinned" : " 29 GB" | 20260505 22:05:51.151 | INFO | } | 20260505 22:05:51.151 | INFO | Perf report for the last 5.14749 seconds: | 20260505 22:05:51.151 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:05:51.151 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:51.151 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:51.151 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:51.151 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260505 22:05:51.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:51.151 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:51.151 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 22:05:51.151 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:51.151 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:51.151 | INFO | Uploaded 36GiB to GPU in 2.07904s (17.3157GiB/s) | 20260505 22:05:51.152 | INFO | Downloaded 34GiB to GPU in 2.02116s (16.822GiB/s) | 20260505 22:05:56.295 | INFO | Analyzing chunk frames = [6661120,6661632) | 20260505 22:05:56.295 | INFO | MemInfo: { | 20260505 22:05:56.295 | INFO | "gpu" : " 17 GB", | 20260505 22:05:56.295 | INFO | "huge" : "", | 20260505 22:05:56.295 | INFO | "huge_shared" : " 27 GB", | 20260505 22:05:56.295 | INFO | "malloc" : " 14 GB", | 20260505 22:05:56.295 | INFO | "pinned" : " 29 GB" | 20260505 22:05:56.295 | INFO | } | 20260505 22:05:56.295 | INFO | Perf report for the last 5.14383 seconds: | 20260505 22:05:56.295 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:05:56.295 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:56.295 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:56.295 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:56.295 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260505 22:05:56.295 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:56.295 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:56.295 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:05:56.295 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:56.295 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:05:56.295 | INFO | Uploaded 36GiB to GPU in 2.14546s (16.7796GiB/s) | 20260505 22:05:56.295 | INFO | Downloaded 34GiB to GPU in 2.04534s (16.6232GiB/s) | 20260505 22:06:01.432 | INFO | Analyzing chunk frames = [6661632,6662144) | 20260505 22:06:01.432 | INFO | MemInfo: { | 20260505 22:06:01.432 | INFO | "gpu" : " 17 GB", | 20260505 22:06:01.432 | INFO | "huge" : "", | 20260505 22:06:01.432 | INFO | "huge_shared" : " 27 GB", | 20260505 22:06:01.432 | INFO | "malloc" : " 14 GB", | 20260505 22:06:01.432 | INFO | "pinned" : " 29 GB" | 20260505 22:06:01.432 | INFO | } | 20260505 22:06:01.433 | INFO | Perf report for the last 5.13772 seconds: | 20260505 22:06:01.433 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:06:01.433 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:01.433 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:01.433 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:01.433 | INFO | ANALYSIS : 61.8%, 3.0, 95.9ms, 0 | 20260505 22:06:01.433 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:01.433 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:01.433 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:06:01.433 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:01.433 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:01.433 | INFO | Uploaded 36GiB to GPU in 2.13545s (16.8582GiB/s) | 20260505 22:06:01.433 | INFO | Downloaded 34GiB to GPU in 2.011s (16.907GiB/s) | 20260505 22:06:06.565 | INFO | Analyzing chunk frames = [6662144,6662656) | 20260505 22:06:06.565 | INFO | MemInfo: { | 20260505 22:06:06.565 | INFO | "gpu" : " 17 GB", | 20260505 22:06:06.565 | INFO | "huge" : "", | 20260505 22:06:06.565 | INFO | "huge_shared" : " 27 GB", | 20260505 22:06:06.565 | INFO | "malloc" : " 14 GB", | 20260505 22:06:06.565 | INFO | "pinned" : " 29 GB" | 20260505 22:06:06.565 | INFO | } | 20260505 22:06:06.565 | INFO | Perf report for the last 5.13274 seconds: | 20260505 22:06:06.565 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:06:06.565 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:06.565 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:06.565 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:06.565 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260505 22:06:06.566 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:06.566 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:06.566 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:06:06.566 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:06.566 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:06.566 | INFO | Uploaded 36GiB to GPU in 2.13732s (16.8435GiB/s) | 20260505 22:06:06.566 | INFO | Downloaded 34GiB to GPU in 2.06677s (16.4508GiB/s) | 20260505 22:06:11.703 | INFO | Analyzing chunk frames = [6662656,6663168) | 20260505 22:06:11.703 | INFO | MemInfo: { | 20260505 22:06:11.703 | INFO | "gpu" : " 17 GB", | 20260505 22:06:11.703 | INFO | "huge" : "", | 20260505 22:06:11.703 | INFO | "huge_shared" : " 27 GB", | 20260505 22:06:11.703 | INFO | "malloc" : " 14 GB", | 20260505 22:06:11.703 | INFO | "pinned" : " 29 GB" | 20260505 22:06:11.703 | INFO | } | 20260505 22:06:11.703 | INFO | Perf report for the last 5.1374 seconds: | 20260505 22:06:11.703 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:06:11.703 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:11.703 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:11.703 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:11.703 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 22:06:11.703 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:11.703 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:11.703 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 22:06:11.703 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:11.703 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:11.703 | INFO | Uploaded 36GiB to GPU in 2.15576s (16.6995GiB/s) | 20260505 22:06:11.703 | INFO | Downloaded 34GiB to GPU in 2.06514s (16.4637GiB/s) | 20260505 22:06:16.846 | INFO | Analyzing chunk frames = [6663168,6663680) | 20260505 22:06:16.846 | INFO | MemInfo: { | 20260505 22:06:16.846 | INFO | "gpu" : " 17 GB", | 20260505 22:06:16.846 | INFO | "huge" : "", | 20260505 22:06:16.846 | INFO | "huge_shared" : " 27 GB", | 20260505 22:06:16.846 | INFO | "malloc" : " 14 GB", | 20260505 22:06:16.846 | INFO | "pinned" : " 29 GB" | 20260505 22:06:16.846 | INFO | } | 20260505 22:06:16.846 | INFO | Perf report for the last 5.14326 seconds: | 20260505 22:06:16.846 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:06:16.846 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:16.846 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:16.846 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:16.846 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 22:06:16.846 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:16.846 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:16.846 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 22:06:16.846 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:16.846 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:16.846 | INFO | Uploaded 36GiB to GPU in 2.13425s (16.8677GiB/s) | 20260505 22:06:16.846 | INFO | Downloaded 34GiB to GPU in 2.12369s (16.0099GiB/s) | 20260505 22:06:21.986 | INFO | Analyzing chunk frames = [6663680,6664192) | 20260505 22:06:21.986 | INFO | MemInfo: { | 20260505 22:06:21.986 | INFO | "gpu" : " 17 GB", | 20260505 22:06:21.986 | INFO | "huge" : "", | 20260505 22:06:21.986 | INFO | "huge_shared" : " 27 GB", | 20260505 22:06:21.986 | INFO | "malloc" : " 14 GB", | 20260505 22:06:21.986 | INFO | "pinned" : " 29 GB" | 20260505 22:06:21.986 | INFO | } | 20260505 22:06:21.987 | INFO | Perf report for the last 5.14057 seconds: | 20260505 22:06:21.987 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:06:21.987 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:21.987 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:21.987 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:21.987 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260505 22:06:21.987 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:21.987 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:21.987 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 22:06:21.987 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:21.987 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:21.987 | INFO | Uploaded 36GiB to GPU in 2.1027s (17.1209GiB/s) | 20260505 22:06:21.987 | INFO | Downloaded 34GiB to GPU in 2.00339s (16.9712GiB/s) | 20260505 22:06:22.117 | INFO | Starting a round of Baz IO | 20260505 22:06:25.268 | INFO | Populating RTMetrics report | 20260505 22:06:25.269 | INFO | RTMetrics report populated | 20260505 22:06:25.269 | INFO | Opening tmp RTMetrics json file | 20260505 22:06:25.271 | INFO | Serializing RTMetrics to json | 20260505 22:06:25.273 | INFO | Serialization done, swapping tmp file now... | 20260505 22:06:25.275 | INFO | RTMetrics done | 20260505 22:06:25.351 | INFO | Finished a round of Baz IO: Wrote 1.945 GiB in 3.233 seconds | 20260505 22:06:25.351 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.52% / 0.00% / 78.48% | 20260505 22:06:25.351 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.132056% / 0% / 0.000103315% | 20260505 22:06:25.351 | INFO | Baz IO overall: | 20260505 22:06:25.351 | INFO | BytesWritten : 3844794461101 | 20260505 22:06:25.351 | INFO | NumEvents : 1814188152010 | 20260505 22:06:27.126 | INFO | Analyzing chunk frames = [6664192,6664704) | 20260505 22:06:27.126 | INFO | MemInfo: { | 20260505 22:06:27.126 | INFO | "gpu" : " 17 GB", | 20260505 22:06:27.126 | INFO | "huge" : "", | 20260505 22:06:27.126 | INFO | "huge_shared" : " 27 GB", | 20260505 22:06:27.126 | INFO | "malloc" : " 14 GB", | 20260505 22:06:27.126 | INFO | "pinned" : " 29 GB" | 20260505 22:06:27.126 | INFO | } | 20260505 22:06:27.126 | INFO | Perf report for the last 5.13909 seconds: | 20260505 22:06:27.126 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:06:27.126 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:27.126 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:27.126 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:27.126 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 22:06:27.126 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:27.126 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260505 22:06:27.126 | INFO | PRE_HQ : 59.8%, 1.8, 54.5ms, 0 | 20260505 22:06:27.126 | INFO | BAZWRITER : 62.3%, 3.6, 116.6ms, 0 | 20260505 22:06:27.126 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 22:06:27.126 | INFO | Uploaded 36GiB to GPU in 2.16463s (16.631GiB/s) | 20260505 22:06:27.126 | INFO | Downloaded 38GiB to GPU in 2.23297s (17.0177GiB/s) | 20260505 22:06:32.266 | INFO | Analyzing chunk frames = [6664704,6665216) | 20260505 22:06:32.266 | INFO | MemInfo: { | 20260505 22:06:32.266 | INFO | "gpu" : " 17 GB", | 20260505 22:06:32.266 | INFO | "huge" : "", | 20260505 22:06:32.266 | INFO | "huge_shared" : " 27 GB", | 20260505 22:06:32.266 | INFO | "malloc" : " 14 GB", | 20260505 22:06:32.266 | INFO | "pinned" : " 29 GB" | 20260505 22:06:32.266 | INFO | } | 20260505 22:06:32.266 | INFO | Perf report for the last 5.14043 seconds: | 20260505 22:06:32.266 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:06:32.266 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:32.266 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:32.266 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:32.266 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 22:06:32.266 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:32.266 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:32.266 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 22:06:32.266 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:32.266 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:32.267 | INFO | Uploaded 36GiB to GPU in 2.0897s (17.2274GiB/s) | 20260505 22:06:32.267 | INFO | Downloaded 34GiB to GPU in 2.03084s (16.7419GiB/s) | 20260505 22:06:37.408 | INFO | Analyzing chunk frames = [6665216,6665728) | 20260505 22:06:37.408 | INFO | MemInfo: { | 20260505 22:06:37.408 | INFO | "gpu" : " 17 GB", | 20260505 22:06:37.408 | INFO | "huge" : "", | 20260505 22:06:37.408 | INFO | "huge_shared" : " 27 GB", | 20260505 22:06:37.408 | INFO | "malloc" : " 14 GB", | 20260505 22:06:37.408 | INFO | "pinned" : " 29 GB" | 20260505 22:06:37.408 | INFO | } | 20260505 22:06:37.408 | INFO | Perf report for the last 5.14217 seconds: | 20260505 22:06:37.409 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:06:37.409 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:37.409 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:37.409 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:37.409 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260505 22:06:37.409 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:37.409 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:37.409 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:06:37.409 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:37.409 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:37.409 | INFO | Uploaded 36GiB to GPU in 2.1367s (16.8484GiB/s) | 20260505 22:06:37.409 | INFO | Downloaded 34GiB to GPU in 2.02234s (16.8122GiB/s) | 20260505 22:06:42.546 | INFO | Analyzing chunk frames = [6665728,6666240) | 20260505 22:06:42.546 | INFO | MemInfo: { | 20260505 22:06:42.546 | INFO | "gpu" : " 17 GB", | 20260505 22:06:42.546 | INFO | "huge" : "", | 20260505 22:06:42.546 | INFO | "huge_shared" : " 27 GB", | 20260505 22:06:42.546 | INFO | "malloc" : " 14 GB", | 20260505 22:06:42.546 | INFO | "pinned" : " 29 GB" | 20260505 22:06:42.546 | INFO | } | 20260505 22:06:42.546 | INFO | Perf report for the last 5.13736 seconds: | 20260505 22:06:42.546 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:06:42.546 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:42.546 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:42.546 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:42.546 | INFO | ANALYSIS : 61.8%, 3.0, 96.0ms, 0 | 20260505 22:06:42.546 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:42.546 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:42.546 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:06:42.546 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:42.546 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:42.546 | INFO | Uploaded 36GiB to GPU in 2.12511s (16.9403GiB/s) | 20260505 22:06:42.546 | INFO | Downloaded 34GiB to GPU in 2.02861s (16.7603GiB/s) | 20260505 22:06:47.680 | INFO | Analyzing chunk frames = [6666240,6666752) | 20260505 22:06:47.680 | INFO | MemInfo: { | 20260505 22:06:47.680 | INFO | "gpu" : " 17 GB", | 20260505 22:06:47.680 | INFO | "huge" : "", | 20260505 22:06:47.680 | INFO | "huge_shared" : " 27 GB", | 20260505 22:06:47.680 | INFO | "malloc" : " 14 GB", | 20260505 22:06:47.680 | INFO | "pinned" : " 29 GB" | 20260505 22:06:47.680 | INFO | } | 20260505 22:06:47.680 | INFO | Perf report for the last 5.13425 seconds: | 20260505 22:06:47.680 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:06:47.680 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:47.680 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:47.680 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:47.680 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260505 22:06:47.680 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:47.680 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:47.680 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 22:06:47.680 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:47.680 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:47.680 | INFO | Uploaded 36GiB to GPU in 2.09493s (17.1843GiB/s) | 20260505 22:06:47.680 | INFO | Downloaded 34GiB to GPU in 2.04683s (16.611GiB/s) | 20260505 22:06:52.818 | INFO | Analyzing chunk frames = [6666752,6667264) | 20260505 22:06:52.818 | INFO | MemInfo: { | 20260505 22:06:52.818 | INFO | "gpu" : " 17 GB", | 20260505 22:06:52.818 | INFO | "huge" : "", | 20260505 22:06:52.818 | INFO | "huge_shared" : " 27 GB", | 20260505 22:06:52.818 | INFO | "malloc" : " 14 GB", | 20260505 22:06:52.818 | INFO | "pinned" : " 29 GB" | 20260505 22:06:52.818 | INFO | } | 20260505 22:06:52.818 | INFO | Perf report for the last 5.13796 seconds: | 20260505 22:06:52.818 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:06:52.818 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:52.818 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:52.818 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:52.818 | INFO | ANALYSIS : 61.7%, 3.0, 95.4ms, 0 | 20260505 22:06:52.818 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:52.818 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:52.818 | INFO | PRE_HQ : 26.7%, 1.0, 13.7ms, 0 | 20260505 22:06:52.818 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:52.818 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:52.818 | INFO | Uploaded 36GiB to GPU in 2.09028s (17.2226GiB/s) | 20260505 22:06:52.818 | INFO | Downloaded 34GiB to GPU in 2.03959s (16.67GiB/s) | 20260505 22:06:57.959 | INFO | Analyzing chunk frames = [6667264,6667776) | 20260505 22:06:57.959 | INFO | MemInfo: { | 20260505 22:06:57.959 | INFO | "gpu" : " 17 GB", | 20260505 22:06:57.959 | INFO | "huge" : "", | 20260505 22:06:57.959 | INFO | "huge_shared" : " 27 GB", | 20260505 22:06:57.959 | INFO | "malloc" : " 14 GB", | 20260505 22:06:57.959 | INFO | "pinned" : " 29 GB" | 20260505 22:06:57.960 | INFO | } | 20260505 22:06:57.960 | INFO | Perf report for the last 5.14167 seconds: | 20260505 22:06:57.960 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:06:57.960 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:57.960 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:57.960 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:57.960 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 22:06:57.960 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:57.960 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:57.960 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 22:06:57.960 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:57.960 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:06:57.960 | INFO | Uploaded 36GiB to GPU in 2.12575s (16.9352GiB/s) | 20260505 22:06:57.960 | INFO | Downloaded 34GiB to GPU in 2.13154s (15.9509GiB/s) | 20260505 22:07:03.103 | INFO | Analyzing chunk frames = [6667776,6668288) | 20260505 22:07:03.103 | INFO | MemInfo: { | 20260505 22:07:03.103 | INFO | "gpu" : " 17 GB", | 20260505 22:07:03.103 | INFO | "huge" : "", | 20260505 22:07:03.103 | INFO | "huge_shared" : " 27 GB", | 20260505 22:07:03.103 | INFO | "malloc" : " 14 GB", | 20260505 22:07:03.103 | INFO | "pinned" : " 29 GB" | 20260505 22:07:03.103 | INFO | } | 20260505 22:07:03.103 | INFO | Perf report for the last 5.14325 seconds: | 20260505 22:07:03.103 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:07:03.103 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:03.103 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:03.103 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:03.103 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260505 22:07:03.103 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:03.103 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:03.103 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260505 22:07:03.103 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:03.103 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:03.103 | INFO | Uploaded 36GiB to GPU in 2.11932s (16.9866GiB/s) | 20260505 22:07:03.103 | INFO | Downloaded 34GiB to GPU in 1.99655s (17.0294GiB/s) | 20260505 22:07:03.104 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 22:07:03.104 | INFO | Total mem usage: 15.3 GiB | 20260505 22:07:03.104 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 22:07:03.104 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.51%) | 20260505 22:07:03.104 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 22:07:03.104 | INFO | High Water Breakdown: | 20260505 22:07:03.104 | INFO | 60.9% : 8.6 GiB ( 90.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 22:07:03.104 | INFO | 39% : 5.5 GiB ( 82.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 22:07:03.104 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 22:07:03.104 | INFO | ------------------------------------------ | 20260505 22:07:03.104 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:07:03.104 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 22:07:03.104 | INFO | Total mem usage: 26.5 GiB | 20260505 22:07:03.104 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 22:07:03.104 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 22:07:03.104 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 22:07:03.104 | INFO | High Water Breakdown: | 20260505 22:07:03.104 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:07:03.104 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:07:03.104 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:07:03.104 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:07:03.104 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:07:03.104 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:07:03.104 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 22:07:03.104 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 22:07:03.104 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:07:03.104 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 22:07:03.104 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 22:07:03.104 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 22:07:03.104 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 22:07:03.104 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 22:07:03.104 | INFO | ------------------------------------------ | 20260505 22:07:03.104 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:07:03.104 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 22:07:03.104 | INFO | Total mem usage: 25.6 GiB | 20260505 22:07:03.104 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 22:07:03.104 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 22:07:03.104 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 22:07:03.104 | INFO | High Water Breakdown: | 20260505 22:07:03.104 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 22:07:03.104 | INFO | ------------------------------------------ | 20260505 22:07:03.104 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:07:03.104 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 22:07:03.104 | INFO | Total mem usage: 15.7 GiB | 20260505 22:07:03.104 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 22:07:03.104 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 22:07:03.104 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 22:07:03.104 | INFO | High Water Breakdown: | 20260505 22:07:03.104 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 22:07:03.104 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 22:07:03.104 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 22:07:03.104 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 22:07:03.104 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 22:07:03.104 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 22:07:03.104 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 22:07:03.104 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 22:07:03.104 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 22:07:03.104 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:07:03.104 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 22:07:03.104 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:07:03.104 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 22:07:03.104 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 22:07:03.104 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 22:07:03.104 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 22:07:03.104 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:07:03.104 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 22:07:03.104 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:07:03.104 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:07:03.104 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:07:03.104 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 22:07:03.104 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 22:07:03.104 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:07:03.104 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 22:07:03.104 | INFO | ------------------------------------------ | 20260505 22:07:03.104 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:07:03.104 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 22:07:03.104 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:07:03.104 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:07:03.104 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.42e+01, 1.36e+01 / 1.35e+00 | 20260505 22:07:03.104 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.03e-01, 1.03e-02 / 2.73e-03 | 20260505 22:07:03.104 | INFO | PerfReport: 11.82%, Baseline -- 3.71e+00 / 4.57e+00, 5.70e-01 / 1.22e-01 | 20260505 22:07:03.105 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.53e+00, 1.89e-01 / 9.77e-02 | 20260505 22:07:03.105 | INFO | PerfReport: 15.97%, DME -- 4.01e+01 / 4.41e+01, 3.94e+00 / 1.42e+00 | 20260505 22:07:03.105 | INFO | PerfReport: 57.21%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.49e+00 / 1.07e+00 | 20260505 22:07:03.105 | INFO | PerfReport: 7.17%, PulseAccumulating -- 2.25e+00 / 2.81e+00, 4.29e-01 / 2.40e-01 | 20260505 22:07:03.105 | INFO | PerfReport: 3.16%, Metrics -- 9.93e-01 / 1.27e+00, 2.65e-01 / 1.21e-01 | 20260505 22:07:03.105 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.46e-01 / 4.03e-01, 5.94e-02 / 3.79e-02 | 20260505 22:07:03.105 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 22:07:03.105 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:07:03.105 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:07:03.105 | INFO | PerfReport: Full Execution -- 9.89e+01 / -nan, 2.52e+01 / 0.00e+00 | 20260505 22:07:03.105 | INFO | PerfReport: 22.40%, Upload -- 2.21e+01 / 0.00e+00, 7.64e+00 / 0.00e+00 | 20260505 22:07:03.105 | INFO | PerfReport: 21.81%, Download -- 2.16e+01 / 0.00e+00, 7.66e+00 / 0.00e+00 | 20260505 22:07:03.105 | INFO | PerfReport: 31.76%, Compute -- 3.14e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 22:07:03.105 | INFO | PerfReport: 2.22%, LockUpload -- 2.20e+00 / 0.00e+00, 8.02e+00 / 0.00e+00 | 20260505 22:07:03.105 | INFO | PerfReport: 1.36%, LockDownload -- 1.35e+00 / 0.00e+00, 2.93e+00 / 0.00e+00 | 20260505 22:07:03.105 | INFO | PerfReport: 20.44%, LockCompute -- 2.02e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 22:07:03.105 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.74e-02 / 0.00e+00 | 20260505 22:07:03.229 | INFO | Starting a round of Baz IO | 20260505 22:07:06.381 | INFO | Populating RTMetrics report | 20260505 22:07:06.381 | INFO | RTMetrics report populated | 20260505 22:07:06.381 | INFO | Opening tmp RTMetrics json file | 20260505 22:07:06.385 | INFO | Serializing RTMetrics to json | 20260505 22:07:06.387 | INFO | Serialization done, swapping tmp file now... | 20260505 22:07:06.387 | INFO | RTMetrics done | 20260505 22:07:06.469 | INFO | Finished a round of Baz IO: Wrote 1.945 GiB in 3.24 seconds | 20260505 22:07:06.469 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.52% / 0.00% / 78.48% | 20260505 22:07:06.469 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.132179% / 0% / 7.94729e-05% | 20260505 22:07:06.469 | INFO | Baz IO overall: | 20260505 22:07:06.469 | INFO | BytesWritten : 3846882793292 | 20260505 22:07:06.469 | INFO | NumEvents : 1815150186828 | 20260505 22:07:08.248 | INFO | Analyzing chunk frames = [6668288,6668800) | 20260505 22:07:08.248 | INFO | MemInfo: { | 20260505 22:07:08.248 | INFO | "gpu" : " 17 GB", | 20260505 22:07:08.248 | INFO | "huge" : "", | 20260505 22:07:08.248 | INFO | "huge_shared" : " 27 GB", | 20260505 22:07:08.248 | INFO | "malloc" : " 14 GB", | 20260505 22:07:08.248 | INFO | "pinned" : " 29 GB" | 20260505 22:07:08.248 | INFO | } | 20260505 22:07:08.248 | INFO | Perf report for the last 5.14508 seconds: | 20260505 22:07:08.248 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:07:08.248 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:08.248 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:08.248 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:08.248 | INFO | ANALYSIS : 62.7%, 3.0, 98.0ms, 0 | 20260505 22:07:08.248 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:08.248 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260505 22:07:08.248 | INFO | PRE_HQ : 60.2%, 1.8, 55.1ms, 0 | 20260505 22:07:08.248 | INFO | BAZWRITER : 62.2%, 3.6, 115.7ms, 0 | 20260505 22:07:08.248 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 22:07:08.248 | INFO | Uploaded 36GiB to GPU in 2.16605s (16.6201GiB/s) | 20260505 22:07:08.248 | INFO | Downloaded 38GiB to GPU in 2.2309s (17.0335GiB/s) | 20260505 22:07:13.378 | INFO | Analyzing chunk frames = [6668800,6669312) | 20260505 22:07:13.378 | INFO | MemInfo: { | 20260505 22:07:13.378 | INFO | "gpu" : " 17 GB", | 20260505 22:07:13.378 | INFO | "huge" : "", | 20260505 22:07:13.378 | INFO | "huge_shared" : " 27 GB", | 20260505 22:07:13.378 | INFO | "malloc" : " 14 GB", | 20260505 22:07:13.378 | INFO | "pinned" : " 29 GB" | 20260505 22:07:13.378 | INFO | } | 20260505 22:07:13.379 | INFO | Perf report for the last 5.13061 seconds: | 20260505 22:07:13.379 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:07:13.379 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:13.379 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:13.379 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:13.379 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260505 22:07:13.379 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:13.379 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:13.379 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:07:13.379 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:13.379 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:13.379 | INFO | Uploaded 36GiB to GPU in 2.06859s (17.4032GiB/s) | 20260505 22:07:13.379 | INFO | Downloaded 34GiB to GPU in 2.024s (16.7984GiB/s) | 20260505 22:07:18.515 | INFO | Analyzing chunk frames = [6669312,6669824) | 20260505 22:07:18.515 | INFO | MemInfo: { | 20260505 22:07:18.515 | INFO | "gpu" : " 17 GB", | 20260505 22:07:18.515 | INFO | "huge" : "", | 20260505 22:07:18.515 | INFO | "huge_shared" : " 27 GB", | 20260505 22:07:18.515 | INFO | "malloc" : " 14 GB", | 20260505 22:07:18.515 | INFO | "pinned" : " 29 GB" | 20260505 22:07:18.515 | INFO | } | 20260505 22:07:18.515 | INFO | Perf report for the last 5.13606 seconds: | 20260505 22:07:18.515 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:07:18.515 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:18.515 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:18.515 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:18.515 | INFO | ANALYSIS : 62.1%, 3.0, 96.5ms, 0 | 20260505 22:07:18.515 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:18.515 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:18.515 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 22:07:18.515 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:18.515 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:18.515 | INFO | Uploaded 36GiB to GPU in 2.14024s (16.8206GiB/s) | 20260505 22:07:18.515 | INFO | Downloaded 34GiB to GPU in 2.02672s (16.7759GiB/s) | 20260505 22:07:23.657 | INFO | Analyzing chunk frames = [6669824,6670336) | 20260505 22:07:23.657 | INFO | MemInfo: { | 20260505 22:07:23.657 | INFO | "gpu" : " 17 GB", | 20260505 22:07:23.657 | INFO | "huge" : "", | 20260505 22:07:23.657 | INFO | "huge_shared" : " 27 GB", | 20260505 22:07:23.657 | INFO | "malloc" : " 14 GB", | 20260505 22:07:23.657 | INFO | "pinned" : " 29 GB" | 20260505 22:07:23.657 | INFO | } | 20260505 22:07:23.657 | INFO | Perf report for the last 5.1421 seconds: | 20260505 22:07:23.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:07:23.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:23.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:23.657 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:23.657 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260505 22:07:23.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:23.657 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:23.657 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 22:07:23.657 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:23.657 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:23.657 | INFO | Uploaded 36GiB to GPU in 2.20487s (16.3275GiB/s) | 20260505 22:07:23.657 | INFO | Downloaded 34GiB to GPU in 2.05103s (16.577GiB/s) | 20260505 22:07:28.802 | INFO | Analyzing chunk frames = [6670336,6670848) | 20260505 22:07:28.802 | INFO | MemInfo: { | 20260505 22:07:28.802 | INFO | "gpu" : " 17 GB", | 20260505 22:07:28.802 | INFO | "huge" : "", | 20260505 22:07:28.802 | INFO | "huge_shared" : " 27 GB", | 20260505 22:07:28.802 | INFO | "malloc" : " 14 GB", | 20260505 22:07:28.802 | INFO | "pinned" : " 29 GB" | 20260505 22:07:28.802 | INFO | } | 20260505 22:07:28.802 | INFO | Perf report for the last 5.1456 seconds: | 20260505 22:07:28.802 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:07:28.802 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:28.803 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:28.803 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:28.803 | INFO | ANALYSIS : 61.6%, 3.0, 96.3ms, 0 | 20260505 22:07:28.803 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:28.803 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:28.803 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 22:07:28.803 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:28.803 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:28.803 | INFO | Uploaded 36GiB to GPU in 2.16446s (16.6323GiB/s) | 20260505 22:07:28.803 | INFO | Downloaded 34GiB to GPU in 2.07374s (16.3955GiB/s) | 20260505 22:07:33.939 | INFO | Analyzing chunk frames = [6670848,6671360) | 20260505 22:07:33.939 | INFO | MemInfo: { | 20260505 22:07:33.939 | INFO | "gpu" : " 17 GB", | 20260505 22:07:33.939 | INFO | "huge" : "", | 20260505 22:07:33.939 | INFO | "huge_shared" : " 27 GB", | 20260505 22:07:33.939 | INFO | "malloc" : " 14 GB", | 20260505 22:07:33.939 | INFO | "pinned" : " 29 GB" | 20260505 22:07:33.939 | INFO | } | 20260505 22:07:33.939 | INFO | Perf report for the last 5.13634 seconds: | 20260505 22:07:33.939 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:07:33.939 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:33.939 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:33.939 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:33.939 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260505 22:07:33.939 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:33.939 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:33.939 | INFO | PRE_HQ : 27.5%, 1.0, 14.2ms, 0 | 20260505 22:07:33.939 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:33.939 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:33.939 | INFO | Uploaded 36GiB to GPU in 2.07105s (17.3825GiB/s) | 20260505 22:07:33.939 | INFO | Downloaded 34GiB to GPU in 2.03811s (16.6822GiB/s) | 20260505 22:07:39.075 | INFO | Analyzing chunk frames = [6671360,6671872) | 20260505 22:07:39.075 | INFO | MemInfo: { | 20260505 22:07:39.075 | INFO | "gpu" : " 17 GB", | 20260505 22:07:39.075 | INFO | "huge" : "", | 20260505 22:07:39.075 | INFO | "huge_shared" : " 27 GB", | 20260505 22:07:39.075 | INFO | "malloc" : " 14 GB", | 20260505 22:07:39.075 | INFO | "pinned" : " 29 GB" | 20260505 22:07:39.076 | INFO | } | 20260505 22:07:39.076 | INFO | Perf report for the last 5.13673 seconds: | 20260505 22:07:39.076 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:07:39.076 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:39.076 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:39.076 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:39.076 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260505 22:07:39.076 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:39.076 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:39.076 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 22:07:39.076 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:39.076 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:39.076 | INFO | Uploaded 36GiB to GPU in 2.10461s (17.1053GiB/s) | 20260505 22:07:39.076 | INFO | Downloaded 34GiB to GPU in 2.07836s (16.3591GiB/s) | 20260505 22:07:44.217 | INFO | Analyzing chunk frames = [6671872,6672384) | 20260505 22:07:44.217 | INFO | MemInfo: { | 20260505 22:07:44.217 | INFO | "gpu" : " 17 GB", | 20260505 22:07:44.217 | INFO | "huge" : "", | 20260505 22:07:44.217 | INFO | "huge_shared" : " 27 GB", | 20260505 22:07:44.217 | INFO | "malloc" : " 14 GB", | 20260505 22:07:44.217 | INFO | "pinned" : " 29 GB" | 20260505 22:07:44.217 | INFO | } | 20260505 22:07:44.217 | INFO | Perf report for the last 5.14147 seconds: | 20260505 22:07:44.217 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:07:44.217 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:44.217 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:44.217 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:44.217 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 22:07:44.217 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:44.217 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:44.217 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 22:07:44.217 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:44.217 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:44.217 | INFO | Uploaded 36GiB to GPU in 2.10651s (17.0899GiB/s) | 20260505 22:07:44.217 | INFO | Downloaded 34GiB to GPU in 2.00476s (16.9596GiB/s) | 20260505 22:07:44.345 | INFO | Starting a round of Baz IO | 20260505 22:07:47.497 | INFO | Populating RTMetrics report | 20260505 22:07:47.498 | INFO | RTMetrics report populated | 20260505 22:07:47.498 | INFO | Opening tmp RTMetrics json file | 20260505 22:07:47.498 | INFO | Serializing RTMetrics to json | 20260505 22:07:47.499 | INFO | Serialization done, swapping tmp file now... | 20260505 22:07:47.500 | INFO | RTMetrics done | 20260505 22:07:47.579 | INFO | Finished a round of Baz IO: Wrote 1.945 GiB in 3.234 seconds | 20260505 22:07:47.579 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.52% / 0.00% / 78.48% | 20260505 22:07:47.579 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.131933% / 3.97364e-06% / 9.93411e-05% | 20260505 22:07:47.579 | INFO | Baz IO overall: | 20260505 22:07:47.579 | INFO | BytesWritten : 3848971646675 | 20260505 22:07:47.579 | INFO | NumEvents : 1816112793336 | 20260505 22:07:49.359 | INFO | Analyzing chunk frames = [6672384,6672896) | 20260505 22:07:49.359 | INFO | MemInfo: { | 20260505 22:07:49.359 | INFO | "gpu" : " 17 GB", | 20260505 22:07:49.359 | INFO | "huge" : "", | 20260505 22:07:49.359 | INFO | "huge_shared" : " 27 GB", | 20260505 22:07:49.359 | INFO | "malloc" : " 14 GB", | 20260505 22:07:49.359 | INFO | "pinned" : " 29 GB" | 20260505 22:07:49.359 | INFO | } | 20260505 22:07:49.359 | INFO | Perf report for the last 5.14209 seconds: | 20260505 22:07:49.359 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:07:49.359 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:49.359 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:49.359 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:49.359 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 22:07:49.359 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:49.359 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260505 22:07:49.359 | INFO | PRE_HQ : 60.0%, 1.8, 54.7ms, 0 | 20260505 22:07:49.359 | INFO | BAZWRITER : 62.2%, 3.6, 116.4ms, 0 | 20260505 22:07:49.359 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 22:07:49.359 | INFO | Uploaded 36GiB to GPU in 2.16805s (16.6048GiB/s) | 20260505 22:07:49.359 | INFO | Downloaded 38GiB to GPU in 2.23862s (16.9748GiB/s) | 20260505 22:07:54.495 | INFO | Analyzing chunk frames = [6672896,6673408) | 20260505 22:07:54.495 | INFO | MemInfo: { | 20260505 22:07:54.495 | INFO | "gpu" : " 17 GB", | 20260505 22:07:54.495 | INFO | "huge" : "", | 20260505 22:07:54.495 | INFO | "huge_shared" : " 27 GB", | 20260505 22:07:54.495 | INFO | "malloc" : " 14 GB", | 20260505 22:07:54.495 | INFO | "pinned" : " 29 GB" | 20260505 22:07:54.495 | INFO | } | 20260505 22:07:54.495 | INFO | Perf report for the last 5.13587 seconds: | 20260505 22:07:54.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:07:54.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:54.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:54.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:54.495 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 22:07:54.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:54.495 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:54.495 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 22:07:54.495 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:54.495 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:54.495 | INFO | Uploaded 36GiB to GPU in 2.09368s (17.1946GiB/s) | 20260505 22:07:54.495 | INFO | Downloaded 34GiB to GPU in 2.03261s (16.7273GiB/s) | 20260505 22:07:59.630 | INFO | Analyzing chunk frames = [6673408,6673920) | 20260505 22:07:59.630 | INFO | MemInfo: { | 20260505 22:07:59.630 | INFO | "gpu" : " 17 GB", | 20260505 22:07:59.630 | INFO | "huge" : "", | 20260505 22:07:59.630 | INFO | "huge_shared" : " 27 GB", | 20260505 22:07:59.630 | INFO | "malloc" : " 14 GB", | 20260505 22:07:59.630 | INFO | "pinned" : " 29 GB" | 20260505 22:07:59.630 | INFO | } | 20260505 22:07:59.630 | INFO | Perf report for the last 5.13498 seconds: | 20260505 22:07:59.630 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:07:59.630 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:59.630 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:59.630 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:59.630 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260505 22:07:59.630 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:59.630 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:59.630 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 22:07:59.630 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:59.630 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:07:59.630 | INFO | Uploaded 36GiB to GPU in 2.1466s (16.7707GiB/s) | 20260505 22:07:59.630 | INFO | Downloaded 34GiB to GPU in 2.04476s (16.6279GiB/s) | 20260505 22:08:04.770 | INFO | Analyzing chunk frames = [6673920,6674432) | 20260505 22:08:04.770 | INFO | MemInfo: { | 20260505 22:08:04.770 | INFO | "gpu" : " 17 GB", | 20260505 22:08:04.770 | INFO | "huge" : "", | 20260505 22:08:04.770 | INFO | "huge_shared" : " 27 GB", | 20260505 22:08:04.770 | INFO | "malloc" : " 14 GB", | 20260505 22:08:04.770 | INFO | "pinned" : " 29 GB" | 20260505 22:08:04.770 | INFO | } | 20260505 22:08:04.770 | INFO | Perf report for the last 5.13983 seconds: | 20260505 22:08:04.770 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:08:04.770 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:04.770 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:04.770 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:04.770 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260505 22:08:04.770 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:04.770 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:04.770 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 22:08:04.770 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:04.770 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:04.770 | INFO | Uploaded 36GiB to GPU in 2.20919s (16.2956GiB/s) | 20260505 22:08:04.770 | INFO | Downloaded 34GiB to GPU in 2.06818s (16.4396GiB/s) | 20260505 22:08:09.911 | INFO | Analyzing chunk frames = [6674432,6674944) | 20260505 22:08:09.911 | INFO | MemInfo: { | 20260505 22:08:09.911 | INFO | "gpu" : " 17 GB", | 20260505 22:08:09.911 | INFO | "huge" : "", | 20260505 22:08:09.911 | INFO | "huge_shared" : " 27 GB", | 20260505 22:08:09.911 | INFO | "malloc" : " 14 GB", | 20260505 22:08:09.911 | INFO | "pinned" : " 29 GB" | 20260505 22:08:09.911 | INFO | } | 20260505 22:08:09.911 | INFO | Perf report for the last 5.14154 seconds: | 20260505 22:08:09.911 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:08:09.911 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:09.911 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:09.911 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:09.911 | INFO | ANALYSIS : 61.5%, 3.0, 96.1ms, 0 | 20260505 22:08:09.911 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:09.911 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:09.911 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 22:08:09.911 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:09.911 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:09.912 | INFO | Uploaded 36GiB to GPU in 2.13336s (16.8748GiB/s) | 20260505 22:08:09.912 | INFO | Downloaded 34GiB to GPU in 2.0626s (16.484GiB/s) | 20260505 22:08:15.056 | INFO | Analyzing chunk frames = [6674944,6675456) | 20260505 22:08:15.056 | INFO | MemInfo: { | 20260505 22:08:15.056 | INFO | "gpu" : " 17 GB", | 20260505 22:08:15.056 | INFO | "huge" : "", | 20260505 22:08:15.056 | INFO | "huge_shared" : " 27 GB", | 20260505 22:08:15.056 | INFO | "malloc" : " 14 GB", | 20260505 22:08:15.056 | INFO | "pinned" : " 29 GB" | 20260505 22:08:15.056 | INFO | } | 20260505 22:08:15.056 | INFO | Perf report for the last 5.14481 seconds: | 20260505 22:08:15.056 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:08:15.056 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:15.056 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:15.056 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:15.056 | INFO | ANALYSIS : 61.4%, 3.0, 96.0ms, 0 | 20260505 22:08:15.056 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:15.056 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:15.056 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:08:15.056 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:15.056 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:15.056 | INFO | Uploaded 36GiB to GPU in 2.07395s (17.3582GiB/s) | 20260505 22:08:15.056 | INFO | Downloaded 34GiB to GPU in 2.03178s (16.7341GiB/s) | 20260505 22:08:20.194 | INFO | Analyzing chunk frames = [6675456,6675968) | 20260505 22:08:20.194 | INFO | MemInfo: { | 20260505 22:08:20.194 | INFO | "gpu" : " 17 GB", | 20260505 22:08:20.194 | INFO | "huge" : "", | 20260505 22:08:20.194 | INFO | "huge_shared" : " 27 GB", | 20260505 22:08:20.194 | INFO | "malloc" : " 14 GB", | 20260505 22:08:20.194 | INFO | "pinned" : " 29 GB" | 20260505 22:08:20.194 | INFO | } | 20260505 22:08:20.194 | INFO | Perf report for the last 5.1377 seconds: | 20260505 22:08:20.194 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:08:20.194 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:20.194 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:20.194 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:20.194 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260505 22:08:20.194 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:20.194 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:20.194 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 22:08:20.194 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:20.194 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:20.194 | INFO | Uploaded 36GiB to GPU in 2.09844s (17.1556GiB/s) | 20260505 22:08:20.194 | INFO | Downloaded 34GiB to GPU in 2.07532s (16.383GiB/s) | 20260505 22:08:25.326 | INFO | Analyzing chunk frames = [6675968,6676480) | 20260505 22:08:25.326 | INFO | MemInfo: { | 20260505 22:08:25.326 | INFO | "gpu" : " 17 GB", | 20260505 22:08:25.326 | INFO | "huge" : "", | 20260505 22:08:25.326 | INFO | "huge_shared" : " 27 GB", | 20260505 22:08:25.327 | INFO | "malloc" : " 14 GB", | 20260505 22:08:25.327 | INFO | "pinned" : " 29 GB" | 20260505 22:08:25.327 | INFO | } | 20260505 22:08:25.327 | INFO | Perf report for the last 5.13262 seconds: | 20260505 22:08:25.327 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:08:25.327 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:25.327 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:25.327 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:25.327 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260505 22:08:25.327 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:25.327 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:25.327 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 22:08:25.327 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:25.327 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:25.327 | INFO | Uploaded 36GiB to GPU in 2.11443s (17.0258GiB/s) | 20260505 22:08:25.327 | INFO | Downloaded 34GiB to GPU in 1.99495s (17.043GiB/s) | 20260505 22:08:25.327 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 22:08:25.327 | INFO | Total mem usage: 15.3 GiB | 20260505 22:08:25.327 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 22:08:25.327 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.91%) | 20260505 22:08:25.327 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 22:08:25.327 | INFO | High Water Breakdown: | 20260505 22:08:25.327 | INFO | 60.8% : 8.56 GiB ( 90% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 22:08:25.327 | INFO | 39% : 5.49 GiB ( 82.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 22:08:25.327 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 22:08:25.327 | INFO | ------------------------------------------ | 20260505 22:08:25.327 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:08:25.327 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 22:08:25.327 | INFO | Total mem usage: 26.5 GiB | 20260505 22:08:25.327 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 22:08:25.327 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 22:08:25.327 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 22:08:25.327 | INFO | High Water Breakdown: | 20260505 22:08:25.327 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:08:25.327 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:08:25.327 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:08:25.327 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:08:25.327 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:08:25.327 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:08:25.327 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 22:08:25.327 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 22:08:25.327 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:08:25.327 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 22:08:25.327 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 22:08:25.327 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 22:08:25.327 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 22:08:25.327 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 22:08:25.327 | INFO | ------------------------------------------ | 20260505 22:08:25.327 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:08:25.327 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 22:08:25.327 | INFO | Total mem usage: 25.6 GiB | 20260505 22:08:25.327 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 22:08:25.327 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 22:08:25.327 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 22:08:25.327 | INFO | High Water Breakdown: | 20260505 22:08:25.327 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 22:08:25.327 | INFO | ------------------------------------------ | 20260505 22:08:25.328 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:08:25.328 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 22:08:25.328 | INFO | Total mem usage: 15.7 GiB | 20260505 22:08:25.328 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 22:08:25.328 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 22:08:25.328 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 22:08:25.328 | INFO | High Water Breakdown: | 20260505 22:08:25.328 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 22:08:25.328 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 22:08:25.328 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 22:08:25.328 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 22:08:25.328 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 22:08:25.328 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 22:08:25.328 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 22:08:25.328 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 22:08:25.328 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 22:08:25.328 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:08:25.328 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 22:08:25.328 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:08:25.328 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 22:08:25.328 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 22:08:25.328 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 22:08:25.328 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 22:08:25.328 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:08:25.328 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 22:08:25.328 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:08:25.328 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:08:25.328 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:08:25.328 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 22:08:25.328 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 22:08:25.328 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:08:25.328 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 22:08:25.328 | INFO | ------------------------------------------ | 20260505 22:08:25.328 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:08:25.328 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 22:08:25.328 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:08:25.328 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:08:25.328 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.38e+01, 1.35e+01 / 6.95e-01 | 20260505 22:08:25.328 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.05e-01, 1.05e-02 / 3.56e-03 | 20260505 22:08:25.328 | INFO | PerfReport: 11.79%, Baseline -- 3.70e+00 / 4.53e+00, 5.59e-01 / 1.92e-01 | 20260505 22:08:25.328 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.51e+00, 1.85e-01 / 9.81e-02 | 20260505 22:08:25.328 | INFO | PerfReport: 15.95%, DME -- 4.00e+01 / 4.38e+01, 3.85e+00 / 8.26e-01 | 20260505 22:08:25.328 | INFO | PerfReport: 57.22%, FrameLabeling -- 1.79e+01 / 1.94e+01, 1.47e+00 / 1.06e+00 | 20260505 22:08:25.328 | INFO | PerfReport: 7.21%, PulseAccumulating -- 2.26e+00 / 2.76e+00, 4.33e-01 / 2.37e-01 | 20260505 22:08:25.328 | INFO | PerfReport: 3.18%, Metrics -- 9.96e-01 / 1.28e+00, 2.69e-01 / 1.27e-01 | 20260505 22:08:25.328 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.46e-01 / 4.00e-01, 5.96e-02 / 3.56e-02 | 20260505 22:08:25.328 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 22:08:25.328 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:08:25.328 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:08:25.328 | INFO | PerfReport: Full Execution -- 9.88e+01 / -nan, 2.48e+01 / 0.00e+00 | 20260505 22:08:25.328 | INFO | PerfReport: 22.44%, Upload -- 2.22e+01 / 0.00e+00, 7.64e+00 / 0.00e+00 | 20260505 22:08:25.328 | INFO | PerfReport: 21.79%, Download -- 2.15e+01 / 0.00e+00, 7.62e+00 / 0.00e+00 | 20260505 22:08:25.328 | INFO | PerfReport: 31.74%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 22:08:25.328 | INFO | PerfReport: 2.22%, LockUpload -- 2.19e+00 / 0.00e+00, 7.87e+00 / 0.00e+00 | 20260505 22:08:25.328 | INFO | PerfReport: 1.30%, LockDownload -- 1.28e+00 / 0.00e+00, 2.77e+00 / 0.00e+00 | 20260505 22:08:25.328 | INFO | PerfReport: 20.49%, LockCompute -- 2.03e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 22:08:25.328 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.48e-02 / 0.00e+00 | 20260505 22:08:25.464 | INFO | Starting a round of Baz IO | 20260505 22:08:28.623 | INFO | Populating RTMetrics report | 20260505 22:08:28.623 | INFO | RTMetrics report populated | 20260505 22:08:28.623 | INFO | Opening tmp RTMetrics json file | 20260505 22:08:28.624 | INFO | Serializing RTMetrics to json | 20260505 22:08:28.625 | INFO | Serialization done, swapping tmp file now... | 20260505 22:08:28.626 | INFO | RTMetrics done | 20260505 22:08:28.702 | INFO | Finished a round of Baz IO: Wrote 1.945 GiB in 3.238 seconds | 20260505 22:08:28.702 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.52% / 0.00% / 78.48% | 20260505 22:08:28.702 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.131611% / 0% / 9.93411e-05% | 20260505 22:08:28.702 | INFO | Baz IO overall: | 20260505 22:08:28.702 | INFO | BytesWritten : 3851059616413 | 20260505 22:08:28.702 | INFO | NumEvents : 1817074769383 | 20260505 22:08:30.468 | INFO | Analyzing chunk frames = [6676480,6676992) | 20260505 22:08:30.468 | INFO | MemInfo: { | 20260505 22:08:30.468 | INFO | "gpu" : " 17 GB", | 20260505 22:08:30.468 | INFO | "huge" : "", | 20260505 22:08:30.468 | INFO | "huge_shared" : " 27 GB", | 20260505 22:08:30.468 | INFO | "malloc" : " 14 GB", | 20260505 22:08:30.468 | INFO | "pinned" : " 29 GB" | 20260505 22:08:30.468 | INFO | } | 20260505 22:08:30.468 | INFO | Perf report for the last 5.14141 seconds: | 20260505 22:08:30.468 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:08:30.468 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:30.468 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:30.468 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:30.468 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 22:08:30.468 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:30.468 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260505 22:08:30.468 | INFO | PRE_HQ : 59.9%, 1.8, 54.7ms, 0 | 20260505 22:08:30.468 | INFO | BAZWRITER : 62.5%, 3.6, 116.6ms, 0 | 20260505 22:08:30.468 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 22:08:30.468 | INFO | Uploaded 36GiB to GPU in 2.17382s (16.5607GiB/s) | 20260505 22:08:30.468 | INFO | Downloaded 38GiB to GPU in 2.24674s (16.9134GiB/s) | 20260505 22:08:35.609 | INFO | Analyzing chunk frames = [6676992,6677504) | 20260505 22:08:35.609 | INFO | MemInfo: { | 20260505 22:08:35.609 | INFO | "gpu" : " 17 GB", | 20260505 22:08:35.609 | INFO | "huge" : "", | 20260505 22:08:35.609 | INFO | "huge_shared" : " 27 GB", | 20260505 22:08:35.609 | INFO | "malloc" : " 14 GB", | 20260505 22:08:35.609 | INFO | "pinned" : " 29 GB" | 20260505 22:08:35.609 | INFO | } | 20260505 22:08:35.609 | INFO | Perf report for the last 5.14101 seconds: | 20260505 22:08:35.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:08:35.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:35.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:35.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:35.609 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260505 22:08:35.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:35.609 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:35.609 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:08:35.609 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:35.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:35.609 | INFO | Uploaded 36GiB to GPU in 2.08033s (17.3049GiB/s) | 20260505 22:08:35.609 | INFO | Downloaded 34GiB to GPU in 2.02493s (16.7907GiB/s) | 20260505 22:08:40.753 | INFO | Analyzing chunk frames = [6677504,6678016) | 20260505 22:08:40.753 | INFO | MemInfo: { | 20260505 22:08:40.753 | INFO | "gpu" : " 17 GB", | 20260505 22:08:40.753 | INFO | "huge" : "", | 20260505 22:08:40.753 | INFO | "huge_shared" : " 27 GB", | 20260505 22:08:40.753 | INFO | "malloc" : " 14 GB", | 20260505 22:08:40.754 | INFO | "pinned" : " 29 GB" | 20260505 22:08:40.754 | INFO | } | 20260505 22:08:40.754 | INFO | Perf report for the last 5.14456 seconds: | 20260505 22:08:40.754 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:08:40.754 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:40.754 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:40.754 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:40.754 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260505 22:08:40.754 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:40.754 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:40.754 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:08:40.754 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:40.754 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:40.754 | INFO | Uploaded 36GiB to GPU in 2.15221s (16.727GiB/s) | 20260505 22:08:40.754 | INFO | Downloaded 34GiB to GPU in 2.04695s (16.61GiB/s) | 20260505 22:08:45.886 | INFO | Analyzing chunk frames = [6678016,6678528) | 20260505 22:08:45.886 | INFO | MemInfo: { | 20260505 22:08:45.886 | INFO | "gpu" : " 17 GB", | 20260505 22:08:45.886 | INFO | "huge" : "", | 20260505 22:08:45.886 | INFO | "huge_shared" : " 27 GB", | 20260505 22:08:45.886 | INFO | "malloc" : " 14 GB", | 20260505 22:08:45.886 | INFO | "pinned" : " 29 GB" | 20260505 22:08:45.886 | INFO | } | 20260505 22:08:45.886 | INFO | Perf report for the last 5.13245 seconds: | 20260505 22:08:45.886 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:08:45.886 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:45.886 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:45.886 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:45.886 | INFO | ANALYSIS : 61.9%, 3.0, 96.0ms, 0 | 20260505 22:08:45.886 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:45.886 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:45.886 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 22:08:45.886 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:45.886 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:45.886 | INFO | Uploaded 36GiB to GPU in 2.13952s (16.8262GiB/s) | 20260505 22:08:45.886 | INFO | Downloaded 34GiB to GPU in 2.02081s (16.8249GiB/s) | 20260505 22:08:51.021 | INFO | Analyzing chunk frames = [6678528,6679040) | 20260505 22:08:51.021 | INFO | MemInfo: { | 20260505 22:08:51.021 | INFO | "gpu" : " 17 GB", | 20260505 22:08:51.021 | INFO | "huge" : "", | 20260505 22:08:51.021 | INFO | "huge_shared" : " 27 GB", | 20260505 22:08:51.022 | INFO | "malloc" : " 14 GB", | 20260505 22:08:51.022 | INFO | "pinned" : " 29 GB" | 20260505 22:08:51.022 | INFO | } | 20260505 22:08:51.022 | INFO | Perf report for the last 5.13555 seconds: | 20260505 22:08:51.022 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:08:51.022 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:51.022 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:51.022 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:51.022 | INFO | ANALYSIS : 61.6%, 3.0, 96.2ms, 0 | 20260505 22:08:51.022 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:51.022 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:51.022 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 22:08:51.022 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:51.022 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:51.022 | INFO | Uploaded 36GiB to GPU in 2.13119s (16.8919GiB/s) | 20260505 22:08:51.022 | INFO | Downloaded 34GiB to GPU in 2.06112s (16.4959GiB/s) | 20260505 22:08:56.170 | INFO | Analyzing chunk frames = [6679040,6679552) | 20260505 22:08:56.170 | INFO | MemInfo: { | 20260505 22:08:56.170 | INFO | "gpu" : " 17 GB", | 20260505 22:08:56.170 | INFO | "huge" : "", | 20260505 22:08:56.170 | INFO | "huge_shared" : " 27 GB", | 20260505 22:08:56.170 | INFO | "malloc" : " 14 GB", | 20260505 22:08:56.170 | INFO | "pinned" : " 29 GB" | 20260505 22:08:56.170 | INFO | } | 20260505 22:08:56.170 | INFO | Perf report for the last 5.14885 seconds: | 20260505 22:08:56.170 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:08:56.170 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:56.170 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:56.170 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:56.171 | INFO | ANALYSIS : 61.6%, 3.0, 96.3ms, 0 | 20260505 22:08:56.171 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:56.171 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:56.171 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:08:56.171 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:56.171 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:08:56.171 | INFO | Uploaded 36GiB to GPU in 2.07943s (17.3125GiB/s) | 20260505 22:08:56.171 | INFO | Downloaded 34GiB to GPU in 2.03374s (16.718GiB/s) | 20260505 22:09:01.305 | INFO | Analyzing chunk frames = [6679552,6680064) | 20260505 22:09:01.305 | INFO | MemInfo: { | 20260505 22:09:01.305 | INFO | "gpu" : " 17 GB", | 20260505 22:09:01.305 | INFO | "huge" : "", | 20260505 22:09:01.305 | INFO | "huge_shared" : " 27 GB", | 20260505 22:09:01.305 | INFO | "malloc" : " 14 GB", | 20260505 22:09:01.305 | INFO | "pinned" : " 29 GB" | 20260505 22:09:01.305 | INFO | } | 20260505 22:09:01.305 | INFO | Perf report for the last 5.13463 seconds: | 20260505 22:09:01.305 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:09:01.305 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:01.305 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:01.305 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:01.305 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 22:09:01.305 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:01.305 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:01.305 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 22:09:01.305 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:01.305 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:01.305 | INFO | Uploaded 36GiB to GPU in 2.02488s (17.7789GiB/s) | 20260505 22:09:01.305 | INFO | Downloaded 34GiB to GPU in 2.03842s (16.6796GiB/s) | 20260505 22:09:06.449 | INFO | Analyzing chunk frames = [6680064,6680576) | 20260505 22:09:06.449 | INFO | MemInfo: { | 20260505 22:09:06.449 | INFO | "gpu" : " 17 GB", | 20260505 22:09:06.449 | INFO | "huge" : "", | 20260505 22:09:06.449 | INFO | "huge_shared" : " 27 GB", | 20260505 22:09:06.449 | INFO | "malloc" : " 14 GB", | 20260505 22:09:06.449 | INFO | "pinned" : " 29 GB" | 20260505 22:09:06.449 | INFO | } | 20260505 22:09:06.449 | INFO | Perf report for the last 5.14418 seconds: | 20260505 22:09:06.449 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:09:06.449 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:06.449 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:06.449 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:06.449 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 22:09:06.449 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:06.449 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:06.449 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 22:09:06.449 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:06.449 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:06.449 | INFO | Uploaded 36GiB to GPU in 2.11548s (17.0174GiB/s) | 20260505 22:09:06.450 | INFO | Downloaded 34GiB to GPU in 2.00138s (16.9883GiB/s) | 20260505 22:09:06.576 | INFO | Starting a round of Baz IO | 20260505 22:09:09.724 | INFO | Populating RTMetrics report | 20260505 22:09:09.724 | INFO | RTMetrics report populated | 20260505 22:09:09.724 | INFO | Opening tmp RTMetrics json file | 20260505 22:09:09.727 | INFO | Serializing RTMetrics to json | 20260505 22:09:09.729 | INFO | Serialization done, swapping tmp file now... | 20260505 22:09:09.729 | INFO | RTMetrics done | 20260505 22:09:09.805 | INFO | Finished a round of Baz IO: Wrote 1.944 GiB in 3.228 seconds | 20260505 22:09:09.805 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.52% / 0.00% / 78.48% | 20260505 22:09:09.805 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.130999% / 0% / 0.000107288% | 20260505 22:09:09.805 | INFO | Baz IO overall: | 20260505 22:09:09.805 | INFO | BytesWritten : 3853146676461 | 20260505 22:09:09.805 | INFO | NumEvents : 1818036119026 | 20260505 22:09:11.584 | INFO | Analyzing chunk frames = [6680576,6681088) | 20260505 22:09:11.584 | INFO | MemInfo: { | 20260505 22:09:11.584 | INFO | "gpu" : " 17 GB", | 20260505 22:09:11.584 | INFO | "huge" : "", | 20260505 22:09:11.584 | INFO | "huge_shared" : " 27 GB", | 20260505 22:09:11.584 | INFO | "malloc" : " 14 GB", | 20260505 22:09:11.584 | INFO | "pinned" : " 29 GB" | 20260505 22:09:11.584 | INFO | } | 20260505 22:09:11.584 | INFO | Perf report for the last 5.13507 seconds: | 20260505 22:09:11.584 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:09:11.584 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:11.584 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:11.584 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:11.584 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 22:09:11.584 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:11.584 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260505 22:09:11.584 | INFO | PRE_HQ : 59.9%, 1.8, 54.9ms, 0 | 20260505 22:09:11.584 | INFO | BAZWRITER : 62.1%, 3.7, 116.7ms, 0 | 20260505 22:09:11.584 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 22:09:11.585 | INFO | Uploaded 36GiB to GPU in 2.13117s (16.8921GiB/s) | 20260505 22:09:11.585 | INFO | Downloaded 38GiB to GPU in 2.2302s (17.0389GiB/s) | 20260505 22:09:16.725 | INFO | Analyzing chunk frames = [6681088,6681600) | 20260505 22:09:16.725 | INFO | MemInfo: { | 20260505 22:09:16.725 | INFO | "gpu" : " 17 GB", | 20260505 22:09:16.725 | INFO | "huge" : "", | 20260505 22:09:16.725 | INFO | "huge_shared" : " 27 GB", | 20260505 22:09:16.725 | INFO | "malloc" : " 14 GB", | 20260505 22:09:16.725 | INFO | "pinned" : " 29 GB" | 20260505 22:09:16.725 | INFO | } | 20260505 22:09:16.725 | INFO | Perf report for the last 5.14072 seconds: | 20260505 22:09:16.725 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:09:16.725 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:16.725 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:16.725 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:16.725 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260505 22:09:16.725 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:16.725 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:16.725 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 22:09:16.725 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:16.725 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:16.725 | INFO | Uploaded 36GiB to GPU in 2.08694s (17.2502GiB/s) | 20260505 22:09:16.726 | INFO | Downloaded 34GiB to GPU in 2.01736s (16.8537GiB/s) | 20260505 22:09:21.867 | INFO | Analyzing chunk frames = [6681600,6682112) | 20260505 22:09:21.867 | INFO | MemInfo: { | 20260505 22:09:21.867 | INFO | "gpu" : " 17 GB", | 20260505 22:09:21.867 | INFO | "huge" : "", | 20260505 22:09:21.867 | INFO | "huge_shared" : " 27 GB", | 20260505 22:09:21.867 | INFO | "malloc" : " 14 GB", | 20260505 22:09:21.867 | INFO | "pinned" : " 29 GB" | 20260505 22:09:21.867 | INFO | } | 20260505 22:09:21.867 | INFO | Perf report for the last 5.14154 seconds: | 20260505 22:09:21.867 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:09:21.867 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:21.867 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:21.867 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:21.867 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260505 22:09:21.867 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:21.867 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:21.867 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 22:09:21.867 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:21.867 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:21.867 | INFO | Uploaded 36GiB to GPU in 2.09519s (17.1822GiB/s) | 20260505 22:09:21.867 | INFO | Downloaded 34GiB to GPU in 2.03606s (16.6989GiB/s) | 20260505 22:09:27.001 | INFO | Analyzing chunk frames = [6682112,6682624) | 20260505 22:09:27.001 | INFO | MemInfo: { | 20260505 22:09:27.001 | INFO | "gpu" : " 17 GB", | 20260505 22:09:27.001 | INFO | "huge" : "", | 20260505 22:09:27.001 | INFO | "huge_shared" : " 27 GB", | 20260505 22:09:27.001 | INFO | "malloc" : " 14 GB", | 20260505 22:09:27.001 | INFO | "pinned" : " 29 GB" | 20260505 22:09:27.002 | INFO | } | 20260505 22:09:27.002 | INFO | Perf report for the last 5.13486 seconds: | 20260505 22:09:27.002 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:09:27.002 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:27.002 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:27.002 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:27.002 | INFO | ANALYSIS : 61.9%, 3.0, 96.0ms, 0 | 20260505 22:09:27.002 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:27.002 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:27.002 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:09:27.002 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:27.002 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:27.002 | INFO | Uploaded 36GiB to GPU in 2.14847s (16.7561GiB/s) | 20260505 22:09:27.002 | INFO | Downloaded 34GiB to GPU in 2.00896s (16.9242GiB/s) | 20260505 22:09:32.138 | INFO | Analyzing chunk frames = [6682624,6683136) | 20260505 22:09:32.138 | INFO | MemInfo: { | 20260505 22:09:32.138 | INFO | "gpu" : " 17 GB", | 20260505 22:09:32.138 | INFO | "huge" : "", | 20260505 22:09:32.138 | INFO | "huge_shared" : " 27 GB", | 20260505 22:09:32.138 | INFO | "malloc" : " 14 GB", | 20260505 22:09:32.138 | INFO | "pinned" : " 29 GB" | 20260505 22:09:32.138 | INFO | } | 20260505 22:09:32.138 | INFO | Perf report for the last 5.13672 seconds: | 20260505 22:09:32.138 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:09:32.138 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:32.138 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:32.138 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:32.138 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260505 22:09:32.138 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:32.138 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:32.138 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 22:09:32.138 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:32.138 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:32.138 | INFO | Uploaded 36GiB to GPU in 2.13657s (16.8494GiB/s) | 20260505 22:09:32.138 | INFO | Downloaded 34GiB to GPU in 2.0631s (16.4801GiB/s) | 20260505 22:09:37.279 | INFO | Analyzing chunk frames = [6683136,6683648) | 20260505 22:09:37.279 | INFO | MemInfo: { | 20260505 22:09:37.279 | INFO | "gpu" : " 17 GB", | 20260505 22:09:37.279 | INFO | "huge" : "", | 20260505 22:09:37.279 | INFO | "huge_shared" : " 27 GB", | 20260505 22:09:37.279 | INFO | "malloc" : " 14 GB", | 20260505 22:09:37.279 | INFO | "pinned" : " 29 GB" | 20260505 22:09:37.279 | INFO | } | 20260505 22:09:37.279 | INFO | Perf report for the last 5.14057 seconds: | 20260505 22:09:37.279 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:09:37.279 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:37.279 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:37.279 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:37.279 | INFO | ANALYSIS : 61.6%, 3.0, 96.3ms, 0 | 20260505 22:09:37.279 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:37.279 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:37.279 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:09:37.279 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:37.279 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:37.279 | INFO | Uploaded 36GiB to GPU in 2.07923s (17.3141GiB/s) | 20260505 22:09:37.279 | INFO | Downloaded 34GiB to GPU in 2.03409s (16.7151GiB/s) | 20260505 22:09:42.422 | INFO | Analyzing chunk frames = [6683648,6684160) | 20260505 22:09:42.422 | INFO | MemInfo: { | 20260505 22:09:42.422 | INFO | "gpu" : " 17 GB", | 20260505 22:09:42.422 | INFO | "huge" : "", | 20260505 22:09:42.422 | INFO | "huge_shared" : " 27 GB", | 20260505 22:09:42.422 | INFO | "malloc" : " 14 GB", | 20260505 22:09:42.422 | INFO | "pinned" : " 29 GB" | 20260505 22:09:42.422 | INFO | } | 20260505 22:09:42.422 | INFO | Perf report for the last 5.1429 seconds: | 20260505 22:09:42.422 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:09:42.422 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:42.422 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:42.422 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:42.422 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 22:09:42.422 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:42.422 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:42.422 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 22:09:42.422 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:42.422 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:42.422 | INFO | Uploaded 36GiB to GPU in 2.17408s (16.5587GiB/s) | 20260505 22:09:42.422 | INFO | Downloaded 34GiB to GPU in 2.14168s (15.8754GiB/s) | 20260505 22:09:47.566 | INFO | Analyzing chunk frames = [6684160,6684672) | 20260505 22:09:47.566 | INFO | MemInfo: { | 20260505 22:09:47.566 | INFO | "gpu" : " 17 GB", | 20260505 22:09:47.566 | INFO | "huge" : "", | 20260505 22:09:47.566 | INFO | "huge_shared" : " 27 GB", | 20260505 22:09:47.566 | INFO | "malloc" : " 14 GB", | 20260505 22:09:47.566 | INFO | "pinned" : " 29 GB" | 20260505 22:09:47.566 | INFO | } | 20260505 22:09:47.566 | INFO | Perf report for the last 5.14452 seconds: | 20260505 22:09:47.566 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:09:47.566 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:47.566 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:47.566 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:47.566 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260505 22:09:47.566 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:47.566 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:47.566 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:09:47.566 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:47.566 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:47.567 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 22:09:47.567 | INFO | Total mem usage: 15.3 GiB | 20260505 22:09:47.567 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 22:09:47.567 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.48%) | 20260505 22:09:47.567 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 22:09:47.567 | INFO | High Water Breakdown: | 20260505 22:09:47.567 | INFO | 61.1% : 8.63 GiB ( 90.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 22:09:47.567 | INFO | 38.8% : 5.48 GiB ( 82.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 22:09:47.567 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 22:09:47.567 | INFO | ------------------------------------------ | 20260505 22:09:47.567 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:09:47.567 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 22:09:47.567 | INFO | Total mem usage: 26.5 GiB | 20260505 22:09:47.567 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 22:09:47.567 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 22:09:47.567 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 22:09:47.567 | INFO | High Water Breakdown: | 20260505 22:09:47.567 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:09:47.567 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:09:47.567 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:09:47.567 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:09:47.567 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:09:47.567 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:09:47.567 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 22:09:47.567 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 22:09:47.567 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:09:47.567 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 22:09:47.567 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 22:09:47.567 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 22:09:47.567 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 22:09:47.567 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 22:09:47.567 | INFO | ------------------------------------------ | 20260505 22:09:47.567 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:09:47.567 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 22:09:47.567 | INFO | Uploaded 36GiB to GPU in 2.11523s (17.0194GiB/s) | 20260505 22:09:47.567 | INFO | Total mem usage: 25.6 GiB | 20260505 22:09:47.567 | INFO | Downloaded 34GiB to GPU in 2.00583s (16.9506GiB/s) | 20260505 22:09:47.567 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 22:09:47.567 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 22:09:47.567 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 22:09:47.567 | INFO | High Water Breakdown: | 20260505 22:09:47.567 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 22:09:47.567 | INFO | ------------------------------------------ | 20260505 22:09:47.567 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:09:47.567 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 22:09:47.567 | INFO | Total mem usage: 15.7 GiB | 20260505 22:09:47.567 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 22:09:47.567 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 22:09:47.567 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 22:09:47.567 | INFO | High Water Breakdown: | 20260505 22:09:47.567 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 22:09:47.567 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 22:09:47.567 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 22:09:47.567 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 22:09:47.567 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 22:09:47.567 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 22:09:47.567 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 22:09:47.567 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 22:09:47.567 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 22:09:47.567 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:09:47.567 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 22:09:47.567 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:09:47.567 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 22:09:47.567 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 22:09:47.567 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 22:09:47.567 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 22:09:47.567 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:09:47.567 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 22:09:47.567 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:09:47.567 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:09:47.567 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:09:47.567 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 22:09:47.567 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 22:09:47.567 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:09:47.567 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 22:09:47.567 | INFO | ------------------------------------------ | 20260505 22:09:47.567 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:09:47.567 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 22:09:47.567 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:09:47.567 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:09:47.567 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.43e+01, 1.36e+01 / 1.22e+00 | 20260505 22:09:47.567 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.04e-01, 1.15e-02 / 4.27e-03 | 20260505 22:09:47.567 | INFO | PerfReport: 11.80%, Baseline -- 3.71e+00 / 4.51e+00, 5.74e-01 / 2.04e-01 | 20260505 22:09:47.567 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.54e+00, 1.91e-01 / 7.09e-02 | 20260505 22:09:47.567 | INFO | PerfReport: 15.98%, DME -- 4.02e+01 / 4.43e+01, 4.11e+00 / 1.38e+00 | 20260505 22:09:47.567 | INFO | PerfReport: 57.21%, FrameLabeling -- 1.80e+01 / 1.93e+01, 1.53e+00 / 8.99e-01 | 20260505 22:09:47.567 | INFO | PerfReport: 7.15%, PulseAccumulating -- 2.25e+00 / 2.84e+00, 4.27e-01 / 1.56e-01 | 20260505 22:09:47.567 | INFO | PerfReport: 3.17%, Metrics -- 9.98e-01 / 1.32e+00, 2.72e-01 / 1.41e-01 | 20260505 22:09:47.567 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.46e-01 / 4.11e-01, 5.92e-02 / 3.98e-02 | 20260505 22:09:47.568 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 22:09:47.568 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:09:47.568 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:09:47.568 | INFO | PerfReport: Full Execution -- 9.88e+01 / -nan, 2.50e+01 / 0.00e+00 | 20260505 22:09:47.568 | INFO | PerfReport: 22.31%, Upload -- 2.20e+01 / 0.00e+00, 7.54e+00 / 0.00e+00 | 20260505 22:09:47.568 | INFO | PerfReport: 21.75%, Download -- 2.15e+01 / 0.00e+00, 7.61e+00 / 0.00e+00 | 20260505 22:09:47.568 | INFO | PerfReport: 31.81%, Compute -- 3.14e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 22:09:47.568 | INFO | PerfReport: 2.19%, LockUpload -- 2.16e+00 / 0.00e+00, 7.93e+00 / 0.00e+00 | 20260505 22:09:47.568 | INFO | PerfReport: 1.33%, LockDownload -- 1.31e+00 / 0.00e+00, 2.93e+00 / 0.00e+00 | 20260505 22:09:47.568 | INFO | PerfReport: 20.59%, LockCompute -- 2.03e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 22:09:47.568 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.95e-02 / 0.00e+00 | 20260505 22:09:47.690 | INFO | Starting a round of Baz IO | 20260505 22:09:50.840 | INFO | Populating RTMetrics report | 20260505 22:09:50.841 | INFO | RTMetrics report populated | 20260505 22:09:50.841 | INFO | Opening tmp RTMetrics json file | 20260505 22:09:50.842 | INFO | Serializing RTMetrics to json | 20260505 22:09:50.843 | INFO | Serialization done, swapping tmp file now... | 20260505 22:09:50.843 | INFO | RTMetrics done | 20260505 22:09:50.926 | INFO | Finished a round of Baz IO: Wrote 1.944 GiB in 3.236 seconds | 20260505 22:09:50.926 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.52% / 0.00% / 78.48% | 20260505 22:09:50.926 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.131285% / 0% / 9.93411e-05% | 20260505 22:09:50.927 | INFO | Baz IO overall: | 20260505 22:09:50.927 | INFO | BytesWritten : 3855233656427 | 20260505 22:09:50.927 | INFO | NumEvents : 1818997332950 | 20260505 22:09:52.701 | INFO | Analyzing chunk frames = [6684672,6685184) | 20260505 22:09:52.701 | INFO | MemInfo: { | 20260505 22:09:52.701 | INFO | "gpu" : " 17 GB", | 20260505 22:09:52.701 | INFO | "huge" : "", | 20260505 22:09:52.701 | INFO | "huge_shared" : " 27 GB", | 20260505 22:09:52.701 | INFO | "malloc" : " 14 GB", | 20260505 22:09:52.701 | INFO | "pinned" : " 29 GB" | 20260505 22:09:52.701 | INFO | } | 20260505 22:09:52.701 | INFO | Perf report for the last 5.13428 seconds: | 20260505 22:09:52.701 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:09:52.701 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:52.701 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:52.701 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:52.701 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260505 22:09:52.701 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:52.701 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260505 22:09:52.701 | INFO | PRE_HQ : 59.9%, 1.8, 54.6ms, 0 | 20260505 22:09:52.701 | INFO | BAZWRITER : 62.3%, 3.7, 117.2ms, 0 | 20260505 22:09:52.701 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 22:09:52.701 | INFO | Uploaded 36GiB to GPU in 2.13562s (16.8569GiB/s) | 20260505 22:09:52.701 | INFO | Downloaded 38GiB to GPU in 2.23127s (17.0306GiB/s) | 20260505 22:09:57.837 | INFO | Analyzing chunk frames = [6685184,6685696) | 20260505 22:09:57.837 | INFO | MemInfo: { | 20260505 22:09:57.837 | INFO | "gpu" : " 17 GB", | 20260505 22:09:57.837 | INFO | "huge" : "", | 20260505 22:09:57.838 | INFO | "huge_shared" : " 27 GB", | 20260505 22:09:57.838 | INFO | "malloc" : " 14 GB", | 20260505 22:09:57.838 | INFO | "pinned" : " 29 GB" | 20260505 22:09:57.838 | INFO | } | 20260505 22:09:57.838 | INFO | Perf report for the last 5.13704 seconds: | 20260505 22:09:57.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:09:57.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:57.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:57.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:57.838 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 22:09:57.838 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:57.838 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:57.838 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:09:57.838 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:57.838 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:09:57.838 | INFO | Uploaded 36GiB to GPU in 2.08565s (17.2608GiB/s) | 20260505 22:09:57.838 | INFO | Downloaded 34GiB to GPU in 2.0314s (16.7372GiB/s) | 20260505 22:10:02.977 | INFO | Analyzing chunk frames = [6685696,6686208) | 20260505 22:10:02.977 | INFO | MemInfo: { | 20260505 22:10:02.977 | INFO | "gpu" : " 17 GB", | 20260505 22:10:02.977 | INFO | "huge" : "", | 20260505 22:10:02.977 | INFO | "huge_shared" : " 27 GB", | 20260505 22:10:02.977 | INFO | "malloc" : " 14 GB", | 20260505 22:10:02.977 | INFO | "pinned" : " 29 GB" | 20260505 22:10:02.977 | INFO | } | 20260505 22:10:02.977 | INFO | Perf report for the last 5.13939 seconds: | 20260505 22:10:02.977 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:10:02.977 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:02.977 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:02.977 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:02.977 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260505 22:10:02.977 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:02.977 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:02.977 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:10:02.977 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:02.977 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:02.977 | INFO | Uploaded 36GiB to GPU in 2.14147s (16.8109GiB/s) | 20260505 22:10:02.977 | INFO | Downloaded 34GiB to GPU in 2.02258s (16.8102GiB/s) | 20260505 22:10:08.117 | INFO | Analyzing chunk frames = [6686208,6686720) | 20260505 22:10:08.117 | INFO | MemInfo: { | 20260505 22:10:08.117 | INFO | "gpu" : " 17 GB", | 20260505 22:10:08.117 | INFO | "huge" : "", | 20260505 22:10:08.117 | INFO | "huge_shared" : " 27 GB", | 20260505 22:10:08.117 | INFO | "malloc" : " 14 GB", | 20260505 22:10:08.117 | INFO | "pinned" : " 29 GB" | 20260505 22:10:08.117 | INFO | } | 20260505 22:10:08.117 | INFO | Perf report for the last 5.14016 seconds: | 20260505 22:10:08.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:10:08.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:08.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:08.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:08.117 | INFO | ANALYSIS : 61.9%, 3.0, 96.2ms, 0 | 20260505 22:10:08.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:08.117 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:08.117 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:10:08.117 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:08.117 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:08.117 | INFO | Uploaded 36GiB to GPU in 2.1427s (16.8012GiB/s) | 20260505 22:10:08.118 | INFO | Downloaded 34GiB to GPU in 2.0468s (16.6113GiB/s) | 20260505 22:10:13.259 | INFO | Analyzing chunk frames = [6686720,6687232) | 20260505 22:10:13.259 | INFO | MemInfo: { | 20260505 22:10:13.259 | INFO | "gpu" : " 17 GB", | 20260505 22:10:13.259 | INFO | "huge" : "", | 20260505 22:10:13.259 | INFO | "huge_shared" : " 27 GB", | 20260505 22:10:13.259 | INFO | "malloc" : " 14 GB", | 20260505 22:10:13.259 | INFO | "pinned" : " 29 GB" | 20260505 22:10:13.259 | INFO | } | 20260505 22:10:13.260 | INFO | Perf report for the last 5.14224 seconds: | 20260505 22:10:13.260 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:10:13.260 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:13.260 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:13.260 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:13.260 | INFO | ANALYSIS : 61.3%, 3.0, 95.8ms, 0 | 20260505 22:10:13.260 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:13.260 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:13.260 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:10:13.260 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:13.260 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:13.260 | INFO | Uploaded 36GiB to GPU in 2.09139s (17.2134GiB/s) | 20260505 22:10:13.260 | INFO | Downloaded 34GiB to GPU in 2.0477s (16.604GiB/s) | 20260505 22:10:18.399 | INFO | Analyzing chunk frames = [6687232,6687744) | 20260505 22:10:18.399 | INFO | MemInfo: { | 20260505 22:10:18.399 | INFO | "gpu" : " 17 GB", | 20260505 22:10:18.399 | INFO | "huge" : "", | 20260505 22:10:18.399 | INFO | "huge_shared" : " 27 GB", | 20260505 22:10:18.399 | INFO | "malloc" : " 14 GB", | 20260505 22:10:18.399 | INFO | "pinned" : " 29 GB" | 20260505 22:10:18.400 | INFO | } | 20260505 22:10:18.400 | INFO | Perf report for the last 5.14005 seconds: | 20260505 22:10:18.400 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:10:18.400 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:18.400 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:18.400 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:18.400 | INFO | ANALYSIS : 61.6%, 3.0, 96.2ms, 0 | 20260505 22:10:18.400 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:18.400 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:18.400 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 22:10:18.400 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:18.400 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:18.400 | INFO | Uploaded 36GiB to GPU in 2.08529s (17.2638GiB/s) | 20260505 22:10:18.400 | INFO | Downloaded 34GiB to GPU in 2.03967s (16.6694GiB/s) | 20260505 22:10:23.533 | INFO | Analyzing chunk frames = [6687744,6688256) | 20260505 22:10:23.533 | INFO | MemInfo: { | 20260505 22:10:23.533 | INFO | "gpu" : " 17 GB", | 20260505 22:10:23.533 | INFO | "huge" : "", | 20260505 22:10:23.533 | INFO | "huge_shared" : " 27 GB", | 20260505 22:10:23.533 | INFO | "malloc" : " 14 GB", | 20260505 22:10:23.533 | INFO | "pinned" : " 29 GB" | 20260505 22:10:23.533 | INFO | } | 20260505 22:10:23.533 | INFO | Perf report for the last 5.13384 seconds: | 20260505 22:10:23.533 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:10:23.533 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:23.533 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:23.533 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:23.533 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 22:10:23.533 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:23.534 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:23.534 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 22:10:23.534 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:23.534 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:23.534 | INFO | Uploaded 36GiB to GPU in 2.15691s (16.6906GiB/s) | 20260505 22:10:23.534 | INFO | Downloaded 34GiB to GPU in 2.11273s (16.093GiB/s) | 20260505 22:10:28.679 | INFO | Analyzing chunk frames = [6688256,6688768) | 20260505 22:10:28.679 | INFO | MemInfo: { | 20260505 22:10:28.679 | INFO | "gpu" : " 17 GB", | 20260505 22:10:28.679 | INFO | "huge" : "", | 20260505 22:10:28.679 | INFO | "huge_shared" : " 27 GB", | 20260505 22:10:28.679 | INFO | "malloc" : " 14 GB", | 20260505 22:10:28.679 | INFO | "pinned" : " 29 GB" | 20260505 22:10:28.679 | INFO | } | 20260505 22:10:28.679 | INFO | Perf report for the last 5.14606 seconds: | 20260505 22:10:28.679 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:10:28.679 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:28.680 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:28.680 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:28.680 | INFO | ANALYSIS : 62.1%, 3.0, 97.1ms, 0 | 20260505 22:10:28.680 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:28.680 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:28.680 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:10:28.680 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:28.680 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:28.680 | INFO | Uploaded 36GiB to GPU in 2.10508s (17.1015GiB/s) | 20260505 22:10:28.680 | INFO | Downloaded 34GiB to GPU in 2.01206s (16.8981GiB/s) | 20260505 22:10:28.803 | INFO | Starting a round of Baz IO | 20260505 22:10:31.948 | INFO | Populating RTMetrics report | 20260505 22:10:31.949 | INFO | RTMetrics report populated | 20260505 22:10:31.949 | INFO | Opening tmp RTMetrics json file | 20260505 22:10:31.952 | INFO | Serializing RTMetrics to json | 20260505 22:10:31.954 | INFO | Serialization done, swapping tmp file now... | 20260505 22:10:31.956 | INFO | RTMetrics done | 20260505 22:10:32.030 | INFO | Finished a round of Baz IO: Wrote 1.944 GiB in 3.227 seconds | 20260505 22:10:32.030 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.52% / 0.00% / 78.48% | 20260505 22:10:32.030 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.130884% / 0% / 7.54992e-05% | 20260505 22:10:32.030 | INFO | Baz IO overall: | 20260505 22:10:32.030 | INFO | BytesWritten : 3857320677758 | 20260505 22:10:32.030 | INFO | NumEvents : 1819958800255 | 20260505 22:10:33.824 | INFO | Analyzing chunk frames = [6688768,6689280) | 20260505 22:10:33.824 | INFO | MemInfo: { | 20260505 22:10:33.824 | INFO | "gpu" : " 17 GB", | 20260505 22:10:33.824 | INFO | "huge" : "", | 20260505 22:10:33.824 | INFO | "huge_shared" : " 27 GB", | 20260505 22:10:33.824 | INFO | "malloc" : " 14 GB", | 20260505 22:10:33.824 | INFO | "pinned" : " 29 GB" | 20260505 22:10:33.824 | INFO | } | 20260505 22:10:33.824 | INFO | Perf report for the last 5.1447 seconds: | 20260505 22:10:33.824 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:10:33.824 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:33.824 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:33.824 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:33.824 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260505 22:10:33.824 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:33.824 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260505 22:10:33.824 | INFO | PRE_HQ : 59.7%, 1.8, 54.8ms, 0 | 20260505 22:10:33.824 | INFO | BAZWRITER : 62.0%, 3.7, 116.7ms, 0 | 20260505 22:10:33.824 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 22:10:33.824 | INFO | Uploaded 36GiB to GPU in 2.11533s (17.0187GiB/s) | 20260505 22:10:33.824 | INFO | Downloaded 38GiB to GPU in 2.21654s (17.1438GiB/s) | 20260505 22:10:38.954 | INFO | Analyzing chunk frames = [6689280,6689792) | 20260505 22:10:38.954 | INFO | MemInfo: { | 20260505 22:10:38.954 | INFO | "gpu" : " 17 GB", | 20260505 22:10:38.954 | INFO | "huge" : "", | 20260505 22:10:38.954 | INFO | "huge_shared" : " 27 GB", | 20260505 22:10:38.954 | INFO | "malloc" : " 14 GB", | 20260505 22:10:38.954 | INFO | "pinned" : " 29 GB" | 20260505 22:10:38.954 | INFO | } | 20260505 22:10:38.954 | INFO | Perf report for the last 5.12997 seconds: | 20260505 22:10:38.954 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:10:38.954 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:38.954 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:38.954 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:38.954 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260505 22:10:38.954 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:38.954 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:38.954 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:10:38.954 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:38.954 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:38.954 | INFO | Uploaded 36GiB to GPU in 2.10783s (17.0792GiB/s) | 20260505 22:10:38.954 | INFO | Downloaded 34GiB to GPU in 2.03833s (16.6803GiB/s) | 20260505 22:10:44.086 | INFO | Analyzing chunk frames = [6689792,6690304) | 20260505 22:10:44.086 | INFO | MemInfo: { | 20260505 22:10:44.086 | INFO | "gpu" : " 17 GB", | 20260505 22:10:44.086 | INFO | "huge" : "", | 20260505 22:10:44.086 | INFO | "huge_shared" : " 27 GB", | 20260505 22:10:44.086 | INFO | "malloc" : " 14 GB", | 20260505 22:10:44.086 | INFO | "pinned" : " 29 GB" | 20260505 22:10:44.086 | INFO | } | 20260505 22:10:44.087 | INFO | Perf report for the last 5.13232 seconds: | 20260505 22:10:44.087 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:10:44.087 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:44.087 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:44.087 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:44.087 | INFO | ANALYSIS : 62.1%, 3.0, 96.5ms, 0 | 20260505 22:10:44.087 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:44.087 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:44.087 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:10:44.087 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:44.087 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:44.087 | INFO | Uploaded 36GiB to GPU in 2.14067s (16.8172GiB/s) | 20260505 22:10:44.087 | INFO | Downloaded 34GiB to GPU in 2.02436s (16.7955GiB/s) | 20260505 22:10:49.232 | INFO | Analyzing chunk frames = [6690304,6690816) | 20260505 22:10:49.232 | INFO | MemInfo: { | 20260505 22:10:49.232 | INFO | "gpu" : " 17 GB", | 20260505 22:10:49.232 | INFO | "huge" : "", | 20260505 22:10:49.232 | INFO | "huge_shared" : " 27 GB", | 20260505 22:10:49.232 | INFO | "malloc" : " 14 GB", | 20260505 22:10:49.232 | INFO | "pinned" : " 29 GB" | 20260505 22:10:49.232 | INFO | } | 20260505 22:10:49.232 | INFO | Perf report for the last 5.14594 seconds: | 20260505 22:10:49.232 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:10:49.232 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:49.233 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:49.233 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:49.233 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260505 22:10:49.233 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:49.233 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:49.233 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:10:49.233 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:49.233 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:49.233 | INFO | Uploaded 36GiB to GPU in 2.20639s (16.3162GiB/s) | 20260505 22:10:49.233 | INFO | Downloaded 34GiB to GPU in 2.05018s (16.5839GiB/s) | 20260505 22:10:54.374 | INFO | Analyzing chunk frames = [6690816,6691328) | 20260505 22:10:54.374 | INFO | MemInfo: { | 20260505 22:10:54.374 | INFO | "gpu" : " 17 GB", | 20260505 22:10:54.374 | INFO | "huge" : "", | 20260505 22:10:54.374 | INFO | "huge_shared" : " 27 GB", | 20260505 22:10:54.374 | INFO | "malloc" : " 14 GB", | 20260505 22:10:54.375 | INFO | "pinned" : " 29 GB" | 20260505 22:10:54.375 | INFO | } | 20260505 22:10:54.375 | INFO | Perf report for the last 5.14212 seconds: | 20260505 22:10:54.375 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:10:54.375 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:54.375 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:54.375 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:54.375 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260505 22:10:54.375 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:54.375 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:54.375 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:10:54.375 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:54.375 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:54.375 | INFO | Uploaded 36GiB to GPU in 2.08916s (17.2318GiB/s) | 20260505 22:10:54.375 | INFO | Downloaded 34GiB to GPU in 2.04518s (16.6244GiB/s) | 20260505 22:10:59.515 | INFO | Analyzing chunk frames = [6691328,6691840) | 20260505 22:10:59.515 | INFO | MemInfo: { | 20260505 22:10:59.515 | INFO | "gpu" : " 17 GB", | 20260505 22:10:59.515 | INFO | "huge" : "", | 20260505 22:10:59.516 | INFO | "huge_shared" : " 27 GB", | 20260505 22:10:59.516 | INFO | "malloc" : " 14 GB", | 20260505 22:10:59.516 | INFO | "pinned" : " 29 GB" | 20260505 22:10:59.516 | INFO | } | 20260505 22:10:59.516 | INFO | Perf report for the last 5.14102 seconds: | 20260505 22:10:59.516 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:10:59.516 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:59.516 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:59.516 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:59.516 | INFO | ANALYSIS : 61.5%, 3.0, 96.0ms, 0 | 20260505 22:10:59.516 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:59.516 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:59.516 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:10:59.516 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:59.516 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:10:59.516 | INFO | Uploaded 36GiB to GPU in 2.08553s (17.2618GiB/s) | 20260505 22:10:59.516 | INFO | Downloaded 34GiB to GPU in 2.04476s (16.6278GiB/s) | 20260505 22:11:04.645 | INFO | Analyzing chunk frames = [6691840,6692352) | 20260505 22:11:04.645 | INFO | MemInfo: { | 20260505 22:11:04.645 | INFO | "gpu" : " 17 GB", | 20260505 22:11:04.645 | INFO | "huge" : "", | 20260505 22:11:04.645 | INFO | "huge_shared" : " 27 GB", | 20260505 22:11:04.645 | INFO | "malloc" : " 14 GB", | 20260505 22:11:04.645 | INFO | "pinned" : " 29 GB" | 20260505 22:11:04.646 | INFO | } | 20260505 22:11:04.646 | INFO | Perf report for the last 5.12994 seconds: | 20260505 22:11:04.646 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:11:04.646 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:04.646 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:04.646 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:04.646 | INFO | ANALYSIS : 62.5%, 3.0, 96.5ms, 0 | 20260505 22:11:04.646 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:04.646 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:04.646 | INFO | PRE_HQ : 27.7%, 1.0, 14.2ms, 0 | 20260505 22:11:04.646 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:04.646 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:04.646 | INFO | Uploaded 36GiB to GPU in 2.09357s (17.1955GiB/s) | 20260505 22:11:04.646 | INFO | Downloaded 34GiB to GPU in 2.08715s (16.2902GiB/s) | 20260505 22:11:09.785 | INFO | Analyzing chunk frames = [6692352,6692864) | 20260505 22:11:09.785 | INFO | MemInfo: { | 20260505 22:11:09.785 | INFO | "gpu" : " 17 GB", | 20260505 22:11:09.785 | INFO | "huge" : "", | 20260505 22:11:09.785 | INFO | "huge_shared" : " 27 GB", | 20260505 22:11:09.785 | INFO | "malloc" : " 14 GB", | 20260505 22:11:09.785 | INFO | "pinned" : " 29 GB" | 20260505 22:11:09.785 | INFO | } | 20260505 22:11:09.785 | INFO | Perf report for the last 5.13984 seconds: | 20260505 22:11:09.785 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:11:09.785 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:09.785 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:09.785 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:09.785 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 22:11:09.786 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:09.786 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:09.786 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 22:11:09.786 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:09.786 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:09.786 | INFO | Uploaded 36GiB to GPU in 2.10347s (17.1146GiB/s) | 20260505 22:11:09.786 | INFO | Downloaded 34GiB to GPU in 2.00813s (16.9312GiB/s) | 20260505 22:11:09.786 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 22:11:09.786 | INFO | Total mem usage: 15.3 GiB | 20260505 22:11:09.786 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 22:11:09.786 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.5%) | 20260505 22:11:09.786 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 22:11:09.786 | INFO | High Water Breakdown: | 20260505 22:11:09.786 | INFO | 61.1% : 8.63 GiB ( 90.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 22:11:09.786 | INFO | 38.8% : 5.48 GiB ( 82.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 22:11:09.786 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 22:11:09.786 | INFO | ------------------------------------------ | 20260505 22:11:09.786 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:11:09.786 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 22:11:09.786 | INFO | Total mem usage: 26.5 GiB | 20260505 22:11:09.786 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 22:11:09.786 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 22:11:09.786 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 22:11:09.786 | INFO | High Water Breakdown: | 20260505 22:11:09.786 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:11:09.786 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:11:09.786 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:11:09.786 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:11:09.786 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:11:09.786 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:11:09.786 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 22:11:09.786 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 22:11:09.786 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:11:09.786 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 22:11:09.786 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 22:11:09.786 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 22:11:09.786 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 22:11:09.786 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 22:11:09.786 | INFO | ------------------------------------------ | 20260505 22:11:09.786 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:11:09.786 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 22:11:09.786 | INFO | Total mem usage: 25.6 GiB | 20260505 22:11:09.786 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 22:11:09.786 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 22:11:09.786 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 22:11:09.786 | INFO | High Water Breakdown: | 20260505 22:11:09.786 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 22:11:09.786 | INFO | ------------------------------------------ | 20260505 22:11:09.786 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:11:09.786 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 22:11:09.786 | INFO | Total mem usage: 15.7 GiB | 20260505 22:11:09.786 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 22:11:09.786 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 22:11:09.786 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 22:11:09.787 | INFO | High Water Breakdown: | 20260505 22:11:09.787 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 22:11:09.787 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 22:11:09.787 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 22:11:09.787 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 22:11:09.787 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 22:11:09.787 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 22:11:09.787 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 22:11:09.787 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 22:11:09.787 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 22:11:09.787 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:11:09.787 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 22:11:09.787 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:11:09.787 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 22:11:09.787 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 22:11:09.787 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 22:11:09.787 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 22:11:09.787 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:11:09.787 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 22:11:09.787 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:11:09.787 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:11:09.787 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:11:09.787 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 22:11:09.787 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 22:11:09.787 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:11:09.787 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 22:11:09.787 | INFO | ------------------------------------------ | 20260505 22:11:09.787 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:11:09.787 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 22:11:09.787 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:11:09.787 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:11:09.787 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.41e+01, 1.35e+01 / 8.65e-01 | 20260505 22:11:09.787 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.05e-01, 1.06e-02 / 4.67e-03 | 20260505 22:11:09.787 | INFO | PerfReport: 11.79%, Baseline -- 3.70e+00 / 4.54e+00, 5.72e-01 / 2.27e-01 | 20260505 22:11:09.787 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.51e+00, 1.90e-01 / 1.02e-01 | 20260505 22:11:09.787 | INFO | PerfReport: 15.94%, DME -- 4.01e+01 / 4.39e+01, 3.76e+00 / 9.39e-01 | 20260505 22:11:09.787 | INFO | PerfReport: 57.30%, FrameLabeling -- 1.80e+01 / 1.96e+01, 1.56e+00 / 1.14e+00 | 20260505 22:11:09.787 | INFO | PerfReport: 7.14%, PulseAccumulating -- 2.25e+00 / 2.79e+00, 4.31e-01 / 2.33e-01 | 20260505 22:11:09.787 | INFO | PerfReport: 3.16%, Metrics -- 9.93e-01 / 1.29e+00, 2.71e-01 / 1.26e-01 | 20260505 22:11:09.787 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.46e-01 / 4.06e-01, 5.97e-02 / 3.53e-02 | 20260505 22:11:09.787 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 22:11:09.787 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:11:09.787 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:11:09.787 | INFO | PerfReport: Full Execution -- 9.87e+01 / -nan, 2.51e+01 / 0.00e+00 | 20260505 22:11:09.787 | INFO | PerfReport: 22.34%, Upload -- 2.21e+01 / 0.00e+00, 7.61e+00 / 0.00e+00 | 20260505 22:11:09.787 | INFO | PerfReport: 21.80%, Download -- 2.15e+01 / 0.00e+00, 7.64e+00 / 0.00e+00 | 20260505 22:11:09.787 | INFO | PerfReport: 31.83%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 22:11:09.787 | INFO | PerfReport: 2.19%, LockUpload -- 2.16e+00 / 0.00e+00, 7.91e+00 / 0.00e+00 | 20260505 22:11:09.787 | INFO | PerfReport: 1.29%, LockDownload -- 1.27e+00 / 0.00e+00, 2.82e+00 / 0.00e+00 | 20260505 22:11:09.787 | INFO | PerfReport: 20.53%, LockCompute -- 2.03e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 22:11:09.787 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.04e-02 / 0.00e+00 | 20260505 22:11:09.920 | INFO | Starting a round of Baz IO | 20260505 22:11:13.082 | INFO | Populating RTMetrics report | 20260505 22:11:13.082 | INFO | RTMetrics report populated | 20260505 22:11:13.082 | INFO | Opening tmp RTMetrics json file | 20260505 22:11:13.085 | INFO | Serializing RTMetrics to json | 20260505 22:11:13.088 | INFO | Serialization done, swapping tmp file now... | 20260505 22:11:13.089 | INFO | RTMetrics done | 20260505 22:11:13.166 | INFO | Finished a round of Baz IO: Wrote 1.941 GiB in 3.246 seconds | 20260505 22:11:13.167 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.52% / 0.00% / 78.48% | 20260505 22:11:13.167 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.130415% / 0% / 9.13938e-05% | 20260505 22:11:13.167 | INFO | Baz IO overall: | 20260505 22:11:13.167 | INFO | BytesWritten : 3859405307677 | 20260505 22:11:13.167 | INFO | NumEvents : 1820918666013 | 20260505 22:11:14.927 | INFO | Analyzing chunk frames = [6692864,6693376) | 20260505 22:11:14.927 | INFO | MemInfo: { | 20260505 22:11:14.927 | INFO | "gpu" : " 17 GB", | 20260505 22:11:14.927 | INFO | "huge" : "", | 20260505 22:11:14.928 | INFO | "huge_shared" : " 27 GB", | 20260505 22:11:14.928 | INFO | "malloc" : " 14 GB", | 20260505 22:11:14.928 | INFO | "pinned" : " 29 GB" | 20260505 22:11:14.928 | INFO | } | 20260505 22:11:14.928 | INFO | Perf report for the last 5.14222 seconds: | 20260505 22:11:14.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:11:14.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:14.928 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:14.928 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:14.928 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 22:11:14.928 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:14.928 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260505 22:11:14.928 | INFO | PRE_HQ : 59.7%, 1.8, 54.5ms, 0 | 20260505 22:11:14.928 | INFO | BAZWRITER : 62.6%, 3.6, 116.8ms, 0 | 20260505 22:11:14.928 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 22:11:14.928 | INFO | Uploaded 36GiB to GPU in 2.17397s (16.5596GiB/s) | 20260505 22:11:14.928 | INFO | Downloaded 38GiB to GPU in 2.24707s (16.9109GiB/s) | 20260505 22:11:20.068 | INFO | Analyzing chunk frames = [6693376,6693888) | 20260505 22:11:20.068 | INFO | MemInfo: { | 20260505 22:11:20.068 | INFO | "gpu" : " 17 GB", | 20260505 22:11:20.068 | INFO | "huge" : "", | 20260505 22:11:20.068 | INFO | "huge_shared" : " 27 GB", | 20260505 22:11:20.068 | INFO | "malloc" : " 14 GB", | 20260505 22:11:20.068 | INFO | "pinned" : " 29 GB" | 20260505 22:11:20.068 | INFO | } | 20260505 22:11:20.068 | INFO | Perf report for the last 5.14005 seconds: | 20260505 22:11:20.068 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:11:20.068 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:20.068 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:20.068 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:20.068 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 22:11:20.068 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:20.068 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:20.068 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 22:11:20.068 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:20.068 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:20.068 | INFO | Uploaded 36GiB to GPU in 2.07487s (17.3505GiB/s) | 20260505 22:11:20.068 | INFO | Downloaded 34GiB to GPU in 2.0219s (16.8158GiB/s) | 20260505 22:11:25.206 | INFO | Analyzing chunk frames = [6693888,6694400) | 20260505 22:11:25.206 | INFO | MemInfo: { | 20260505 22:11:25.206 | INFO | "gpu" : " 17 GB", | 20260505 22:11:25.206 | INFO | "huge" : "", | 20260505 22:11:25.206 | INFO | "huge_shared" : " 27 GB", | 20260505 22:11:25.206 | INFO | "malloc" : " 14 GB", | 20260505 22:11:25.206 | INFO | "pinned" : " 29 GB" | 20260505 22:11:25.206 | INFO | } | 20260505 22:11:25.206 | INFO | Perf report for the last 5.13808 seconds: | 20260505 22:11:25.206 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:11:25.206 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:25.206 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:25.206 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:25.206 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260505 22:11:25.206 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:25.206 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:25.206 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:11:25.206 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:25.206 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:25.206 | INFO | Uploaded 36GiB to GPU in 2.15275s (16.7228GiB/s) | 20260505 22:11:25.206 | INFO | Downloaded 34GiB to GPU in 2.04819s (16.6GiB/s) | 20260505 22:11:30.346 | INFO | Analyzing chunk frames = [6694400,6694912) | 20260505 22:11:30.346 | INFO | MemInfo: { | 20260505 22:11:30.346 | INFO | "gpu" : " 17 GB", | 20260505 22:11:30.346 | INFO | "huge" : "", | 20260505 22:11:30.346 | INFO | "huge_shared" : " 27 GB", | 20260505 22:11:30.346 | INFO | "malloc" : " 14 GB", | 20260505 22:11:30.346 | INFO | "pinned" : " 29 GB" | 20260505 22:11:30.346 | INFO | } | 20260505 22:11:30.346 | INFO | Perf report for the last 5.14046 seconds: | 20260505 22:11:30.346 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:11:30.346 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:30.346 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:30.346 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:30.346 | INFO | ANALYSIS : 62.0%, 3.0, 96.3ms, 0 | 20260505 22:11:30.346 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:30.346 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:30.346 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:11:30.346 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:30.346 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:30.347 | INFO | Uploaded 36GiB to GPU in 2.14935s (16.7492GiB/s) | 20260505 22:11:30.347 | INFO | Downloaded 34GiB to GPU in 2.04854s (16.5972GiB/s) | 20260505 22:11:35.488 | INFO | Analyzing chunk frames = [6694912,6695424) | 20260505 22:11:35.488 | INFO | MemInfo: { | 20260505 22:11:35.488 | INFO | "gpu" : " 17 GB", | 20260505 22:11:35.488 | INFO | "huge" : "", | 20260505 22:11:35.488 | INFO | "huge_shared" : " 27 GB", | 20260505 22:11:35.488 | INFO | "malloc" : " 14 GB", | 20260505 22:11:35.488 | INFO | "pinned" : " 29 GB" | 20260505 22:11:35.488 | INFO | } | 20260505 22:11:35.488 | INFO | Perf report for the last 5.14177 seconds: | 20260505 22:11:35.488 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:11:35.488 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:35.488 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:35.488 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:35.488 | INFO | ANALYSIS : 61.5%, 3.0, 96.1ms, 0 | 20260505 22:11:35.488 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:35.488 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:35.488 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:11:35.488 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:35.488 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:35.488 | INFO | Uploaded 36GiB to GPU in 2.13766s (16.8409GiB/s) | 20260505 22:11:35.488 | INFO | Downloaded 34GiB to GPU in 2.06119s (16.4953GiB/s) | 20260505 22:11:40.625 | INFO | Analyzing chunk frames = [6695424,6695936) | 20260505 22:11:40.625 | INFO | MemInfo: { | 20260505 22:11:40.625 | INFO | "gpu" : " 17 GB", | 20260505 22:11:40.625 | INFO | "huge" : "", | 20260505 22:11:40.625 | INFO | "huge_shared" : " 27 GB", | 20260505 22:11:40.625 | INFO | "malloc" : " 14 GB", | 20260505 22:11:40.625 | INFO | "pinned" : " 29 GB" | 20260505 22:11:40.625 | INFO | } | 20260505 22:11:40.625 | INFO | Perf report for the last 5.13737 seconds: | 20260505 22:11:40.625 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:11:40.625 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:40.625 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:40.625 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:40.625 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260505 22:11:40.626 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:40.626 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:40.626 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:11:40.626 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:40.626 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:40.626 | INFO | Uploaded 36GiB to GPU in 2.08591s (17.2586GiB/s) | 20260505 22:11:40.626 | INFO | Downloaded 34GiB to GPU in 2.03853s (16.6787GiB/s) | 20260505 22:11:45.766 | INFO | Analyzing chunk frames = [6695936,6696448) | 20260505 22:11:45.766 | INFO | MemInfo: { | 20260505 22:11:45.766 | INFO | "gpu" : " 17 GB", | 20260505 22:11:45.766 | INFO | "huge" : "", | 20260505 22:11:45.766 | INFO | "huge_shared" : " 27 GB", | 20260505 22:11:45.766 | INFO | "malloc" : " 14 GB", | 20260505 22:11:45.766 | INFO | "pinned" : " 29 GB" | 20260505 22:11:45.766 | INFO | } | 20260505 22:11:45.766 | INFO | Perf report for the last 5.14086 seconds: | 20260505 22:11:45.766 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:11:45.766 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:45.766 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:45.766 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:45.766 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260505 22:11:45.766 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:45.766 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:45.766 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 22:11:45.766 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:45.766 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:45.767 | INFO | Uploaded 36GiB to GPU in 2.08288s (17.2838GiB/s) | 20260505 22:11:45.767 | INFO | Downloaded 34GiB to GPU in 2.07144s (16.4137GiB/s) | 20260505 22:11:50.904 | INFO | Analyzing chunk frames = [6696448,6696960) | 20260505 22:11:50.904 | INFO | MemInfo: { | 20260505 22:11:50.904 | INFO | "gpu" : " 17 GB", | 20260505 22:11:50.904 | INFO | "huge" : "", | 20260505 22:11:50.904 | INFO | "huge_shared" : " 27 GB", | 20260505 22:11:50.904 | INFO | "malloc" : " 14 GB", | 20260505 22:11:50.904 | INFO | "pinned" : " 29 GB" | 20260505 22:11:50.904 | INFO | } | 20260505 22:11:50.904 | INFO | Perf report for the last 5.13789 seconds: | 20260505 22:11:50.904 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:11:50.904 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:50.904 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:50.904 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:50.904 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 22:11:50.904 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:50.904 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:50.904 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 22:11:50.904 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:50.904 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:50.905 | INFO | Uploaded 36GiB to GPU in 2.11457s (17.0247GiB/s) | 20260505 22:11:50.905 | INFO | Downloaded 34GiB to GPU in 2.01122s (16.9051GiB/s) | 20260505 22:11:51.039 | INFO | Starting a round of Baz IO | 20260505 22:11:54.187 | INFO | Populating RTMetrics report | 20260505 22:11:54.187 | INFO | RTMetrics report populated | 20260505 22:11:54.187 | INFO | Opening tmp RTMetrics json file | 20260505 22:11:54.188 | INFO | Serializing RTMetrics to json | 20260505 22:11:54.189 | INFO | Serialization done, swapping tmp file now... | 20260505 22:11:54.190 | INFO | RTMetrics done | 20260505 22:11:54.273 | INFO | Finished a round of Baz IO: Wrote 1.941 GiB in 3.233 seconds | 20260505 22:11:54.273 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.52% / 0.00% / 78.48% | 20260505 22:11:54.273 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.130435% / 0% / 7.15256e-05% | 20260505 22:11:54.273 | INFO | Baz IO overall: | 20260505 22:11:54.273 | INFO | BytesWritten : 3861489854036 | 20260505 22:11:54.273 | INFO | NumEvents : 1821879017935 | 20260505 22:11:56.038 | INFO | Analyzing chunk frames = [6696960,6697472) | 20260505 22:11:56.038 | INFO | MemInfo: { | 20260505 22:11:56.038 | INFO | "gpu" : " 17 GB", | 20260505 22:11:56.038 | INFO | "huge" : "", | 20260505 22:11:56.038 | INFO | "huge_shared" : " 27 GB", | 20260505 22:11:56.038 | INFO | "malloc" : " 14 GB", | 20260505 22:11:56.038 | INFO | "pinned" : " 29 GB" | 20260505 22:11:56.038 | INFO | } | 20260505 22:11:56.038 | INFO | Perf report for the last 5.13376 seconds: | 20260505 22:11:56.038 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:11:56.038 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:56.038 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:56.038 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:56.038 | INFO | ANALYSIS : 62.9%, 3.0, 98.0ms, 0 | 20260505 22:11:56.038 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:11:56.038 | INFO | RT_METRICS : 43.8%, 1.0, 22.9ms, 0 | 20260505 22:11:56.038 | INFO | PRE_HQ : 59.8%, 1.8, 54.6ms, 0 | 20260505 22:11:56.038 | INFO | BAZWRITER : 62.6%, 3.6, 116.9ms, 0 | 20260505 22:11:56.038 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 22:11:56.038 | INFO | Uploaded 36GiB to GPU in 2.16171s (16.6535GiB/s) | 20260505 22:11:56.038 | INFO | Downloaded 38GiB to GPU in 2.24801s (16.9038GiB/s) | 20260505 22:12:01.185 | INFO | Analyzing chunk frames = [6697472,6697984) | 20260505 22:12:01.185 | INFO | MemInfo: { | 20260505 22:12:01.185 | INFO | "gpu" : " 17 GB", | 20260505 22:12:01.185 | INFO | "huge" : "", | 20260505 22:12:01.185 | INFO | "huge_shared" : " 27 GB", | 20260505 22:12:01.185 | INFO | "malloc" : " 14 GB", | 20260505 22:12:01.185 | INFO | "pinned" : " 29 GB" | 20260505 22:12:01.185 | INFO | } | 20260505 22:12:01.185 | INFO | Perf report for the last 5.14714 seconds: | 20260505 22:12:01.185 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:12:01.185 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:01.185 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:01.185 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:01.185 | INFO | ANALYSIS : 61.9%, 3.0, 96.8ms, 0 | 20260505 22:12:01.185 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:01.185 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:01.185 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:12:01.185 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:01.185 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:01.185 | INFO | Uploaded 36GiB to GPU in 2.09576s (17.1775GiB/s) | 20260505 22:12:01.185 | INFO | Downloaded 34GiB to GPU in 2.0245s (16.7943GiB/s) | 20260505 22:12:06.329 | INFO | Analyzing chunk frames = [6697984,6698496) | 20260505 22:12:06.329 | INFO | MemInfo: { | 20260505 22:12:06.329 | INFO | "gpu" : " 17 GB", | 20260505 22:12:06.329 | INFO | "huge" : "", | 20260505 22:12:06.329 | INFO | "huge_shared" : " 27 GB", | 20260505 22:12:06.329 | INFO | "malloc" : " 14 GB", | 20260505 22:12:06.329 | INFO | "pinned" : " 29 GB" | 20260505 22:12:06.329 | INFO | } | 20260505 22:12:06.329 | INFO | Perf report for the last 5.14401 seconds: | 20260505 22:12:06.329 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:12:06.329 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:06.329 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:06.329 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:06.329 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260505 22:12:06.329 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:06.329 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:06.329 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:12:06.329 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:06.329 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:06.329 | INFO | Uploaded 36GiB to GPU in 2.09841s (17.1559GiB/s) | 20260505 22:12:06.329 | INFO | Downloaded 34GiB to GPU in 2.02793s (16.7659GiB/s) | 20260505 22:12:11.461 | INFO | Analyzing chunk frames = [6698496,6699008) | 20260505 22:12:11.461 | INFO | MemInfo: { | 20260505 22:12:11.461 | INFO | "gpu" : " 17 GB", | 20260505 22:12:11.461 | INFO | "huge" : "", | 20260505 22:12:11.461 | INFO | "huge_shared" : " 27 GB", | 20260505 22:12:11.461 | INFO | "malloc" : " 14 GB", | 20260505 22:12:11.461 | INFO | "pinned" : " 29 GB" | 20260505 22:12:11.461 | INFO | } | 20260505 22:12:11.461 | INFO | Perf report for the last 5.13192 seconds: | 20260505 22:12:11.461 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:12:11.461 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:11.461 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:11.461 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:11.461 | INFO | ANALYSIS : 61.9%, 3.0, 95.9ms, 0 | 20260505 22:12:11.461 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:11.461 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:11.461 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:12:11.461 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:11.461 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:11.461 | INFO | Uploaded 36GiB to GPU in 2.13801s (16.8381GiB/s) | 20260505 22:12:11.461 | INFO | Downloaded 34GiB to GPU in 2.00715s (16.9394GiB/s) | 20260505 22:12:16.592 | INFO | Analyzing chunk frames = [6699008,6699520) | 20260505 22:12:16.592 | INFO | MemInfo: { | 20260505 22:12:16.592 | INFO | "gpu" : " 17 GB", | 20260505 22:12:16.592 | INFO | "huge" : "", | 20260505 22:12:16.592 | INFO | "huge_shared" : " 27 GB", | 20260505 22:12:16.592 | INFO | "malloc" : " 14 GB", | 20260505 22:12:16.592 | INFO | "pinned" : " 29 GB" | 20260505 22:12:16.592 | INFO | } | 20260505 22:12:16.592 | INFO | Perf report for the last 5.13136 seconds: | 20260505 22:12:16.592 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:12:16.592 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:16.592 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:16.592 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:16.592 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260505 22:12:16.592 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:16.592 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:16.592 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 22:12:16.593 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:16.593 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:16.593 | INFO | Uploaded 36GiB to GPU in 2.09311s (17.1993GiB/s) | 20260505 22:12:16.593 | INFO | Downloaded 34GiB to GPU in 2.0402s (16.665GiB/s) | 20260505 22:12:21.737 | INFO | Analyzing chunk frames = [6699520,6700032) | 20260505 22:12:21.737 | INFO | MemInfo: { | 20260505 22:12:21.737 | INFO | "gpu" : " 17 GB", | 20260505 22:12:21.737 | INFO | "huge" : "", | 20260505 22:12:21.738 | INFO | "huge_shared" : " 27 GB", | 20260505 22:12:21.738 | INFO | "malloc" : " 14 GB", | 20260505 22:12:21.738 | INFO | "pinned" : " 29 GB" | 20260505 22:12:21.738 | INFO | } | 20260505 22:12:21.738 | INFO | Perf report for the last 5.14515 seconds: | 20260505 22:12:21.738 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:12:21.738 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:21.738 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:21.738 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:21.738 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260505 22:12:21.738 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:21.738 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:21.738 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:12:21.738 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:21.738 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:21.738 | INFO | Uploaded 36GiB to GPU in 2.15669s (16.6923GiB/s) | 20260505 22:12:21.738 | INFO | Downloaded 34GiB to GPU in 2.06005s (16.5044GiB/s) | 20260505 22:12:26.883 | INFO | Analyzing chunk frames = [6700032,6700544) | 20260505 22:12:26.883 | INFO | MemInfo: { | 20260505 22:12:26.883 | INFO | "gpu" : " 17 GB", | 20260505 22:12:26.883 | INFO | "huge" : "", | 20260505 22:12:26.883 | INFO | "huge_shared" : " 27 GB", | 20260505 22:12:26.883 | INFO | "malloc" : " 14 GB", | 20260505 22:12:26.883 | INFO | "pinned" : " 29 GB" | 20260505 22:12:26.883 | INFO | } | 20260505 22:12:26.883 | INFO | Perf report for the last 5.14575 seconds: | 20260505 22:12:26.883 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:12:26.883 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:26.883 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:26.883 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:26.883 | INFO | ANALYSIS : 62.7%, 3.0, 98.1ms, 0 | 20260505 22:12:26.883 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:26.884 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:26.884 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 22:12:26.884 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:26.884 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:26.884 | INFO | Uploaded 36GiB to GPU in 2.16299s (16.6436GiB/s) | 20260505 22:12:26.884 | INFO | Downloaded 34GiB to GPU in 2.12203s (16.0224GiB/s) | 20260505 22:12:32.018 | INFO | Analyzing chunk frames = [6700544,6701056) | 20260505 22:12:32.018 | INFO | MemInfo: { | 20260505 22:12:32.018 | INFO | "gpu" : " 17 GB", | 20260505 22:12:32.018 | INFO | "huge" : "", | 20260505 22:12:32.018 | INFO | "huge_shared" : " 27 GB", | 20260505 22:12:32.018 | INFO | "malloc" : " 14 GB", | 20260505 22:12:32.018 | INFO | "pinned" : " 29 GB" | 20260505 22:12:32.018 | INFO | } | 20260505 22:12:32.018 | INFO | Perf report for the last 5.13496 seconds: | 20260505 22:12:32.018 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:12:32.018 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:32.018 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:32.018 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:32.018 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 22:12:32.018 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:32.018 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:32.018 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 22:12:32.019 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:32.019 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:32.019 | INFO | Uploaded 36GiB to GPU in 2.11231s (17.0429GiB/s) | 20260505 22:12:32.019 | INFO | Downloaded 34GiB to GPU in 1.99713s (17.0244GiB/s) | 20260505 22:12:32.019 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 22:12:32.019 | INFO | Total mem usage: 15.3 GiB | 20260505 22:12:32.019 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 22:12:32.019 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -9.03%) | 20260505 22:12:32.019 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 22:12:32.019 | INFO | High Water Breakdown: | 20260505 22:12:32.019 | INFO | 60.9% : 8.56 GiB ( 90% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 22:12:32.019 | INFO | 39% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 22:12:32.019 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 22:12:32.019 | INFO | ------------------------------------------ | 20260505 22:12:32.019 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:12:32.019 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 22:12:32.019 | INFO | Total mem usage: 26.5 GiB | 20260505 22:12:32.019 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 22:12:32.019 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 22:12:32.019 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 22:12:32.019 | INFO | High Water Breakdown: | 20260505 22:12:32.019 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:12:32.019 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:12:32.019 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:12:32.019 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:12:32.019 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:12:32.019 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:12:32.019 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 22:12:32.019 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 22:12:32.019 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:12:32.019 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 22:12:32.019 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 22:12:32.019 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 22:12:32.019 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 22:12:32.019 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 22:12:32.019 | INFO | ------------------------------------------ | 20260505 22:12:32.019 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:12:32.019 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 22:12:32.019 | INFO | Total mem usage: 25.6 GiB | 20260505 22:12:32.019 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 22:12:32.019 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 22:12:32.019 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 22:12:32.019 | INFO | High Water Breakdown: | 20260505 22:12:32.019 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 22:12:32.019 | INFO | ------------------------------------------ | 20260505 22:12:32.020 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:12:32.020 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 22:12:32.020 | INFO | Total mem usage: 15.7 GiB | 20260505 22:12:32.020 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 22:12:32.020 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 22:12:32.020 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 22:12:32.020 | INFO | High Water Breakdown: | 20260505 22:12:32.020 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 22:12:32.020 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 22:12:32.020 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 22:12:32.020 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 22:12:32.020 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 22:12:32.020 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 22:12:32.020 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 22:12:32.020 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 22:12:32.020 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 22:12:32.020 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:12:32.020 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 22:12:32.020 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:12:32.020 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 22:12:32.020 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 22:12:32.020 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 22:12:32.020 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 22:12:32.020 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:12:32.020 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 22:12:32.020 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:12:32.020 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:12:32.020 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:12:32.020 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 22:12:32.020 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 22:12:32.020 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:12:32.020 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 22:12:32.020 | INFO | ------------------------------------------ | 20260505 22:12:32.020 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:12:32.020 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 22:12:32.020 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:12:32.020 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:12:32.020 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.41e+01, 1.35e+01 / 1.19e+00 | 20260505 22:12:32.020 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.03e-01, 1.03e-02 / 2.93e-03 | 20260505 22:12:32.020 | INFO | PerfReport: 11.81%, Baseline -- 3.71e+00 / 4.53e+00, 5.75e-01 / 1.99e-01 | 20260505 22:12:32.020 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.50e+00, 1.89e-01 / 1.07e-01 | 20260505 22:12:32.020 | INFO | PerfReport: 15.96%, DME -- 4.01e+01 / 4.41e+01, 3.96e+00 / 1.34e+00 | 20260505 22:12:32.020 | INFO | PerfReport: 57.24%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.55e+00 / 1.05e+00 | 20260505 22:12:32.020 | INFO | PerfReport: 7.15%, PulseAccumulating -- 2.25e+00 / 2.81e+00, 4.33e-01 / 1.93e-01 | 20260505 22:12:32.020 | INFO | PerfReport: 3.17%, Metrics -- 9.96e-01 / 1.30e+00, 2.72e-01 / 1.38e-01 | 20260505 22:12:32.020 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.46e-01 / 4.04e-01, 5.91e-02 / 4.02e-02 | 20260505 22:12:32.020 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 22:12:32.020 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:12:32.020 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:12:32.020 | INFO | PerfReport: Full Execution -- 9.88e+01 / -nan, 2.51e+01 / 0.00e+00 | 20260505 22:12:32.020 | INFO | PerfReport: 22.40%, Upload -- 2.21e+01 / 0.00e+00, 7.66e+00 / 0.00e+00 | 20260505 22:12:32.020 | INFO | PerfReport: 21.80%, Download -- 2.15e+01 / 0.00e+00, 7.68e+00 / 0.00e+00 | 20260505 22:12:32.020 | INFO | PerfReport: 31.81%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 22:12:32.020 | INFO | PerfReport: 2.17%, LockUpload -- 2.15e+00 / 0.00e+00, 7.91e+00 / 0.00e+00 | 20260505 22:12:32.021 | INFO | PerfReport: 1.30%, LockDownload -- 1.28e+00 / 0.00e+00, 2.85e+00 / 0.00e+00 | 20260505 22:12:32.021 | INFO | PerfReport: 20.51%, LockCompute -- 2.03e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 22:12:32.021 | INFO | PerfReport: 0.01%, Misc -- 1.46e-02 / 0.00e+00, 2.25e-02 / 0.00e+00 | 20260505 22:12:32.144 | INFO | Starting a round of Baz IO | 20260505 22:12:35.291 | INFO | Populating RTMetrics report | 20260505 22:12:35.292 | INFO | RTMetrics report populated | 20260505 22:12:35.292 | INFO | Opening tmp RTMetrics json file | 20260505 22:12:35.292 | INFO | Serializing RTMetrics to json | 20260505 22:12:35.294 | INFO | Serialization done, swapping tmp file now... | 20260505 22:12:35.299 | INFO | RTMetrics done | 20260505 22:12:35.378 | INFO | Finished a round of Baz IO: Wrote 1.942 GiB in 3.233 seconds | 20260505 22:12:35.378 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.51% / 0.00% / 78.48% | 20260505 22:12:35.378 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.130351% / 0% / 7.94729e-05% | 20260505 22:12:35.378 | INFO | Baz IO overall: | 20260505 22:12:35.378 | INFO | BytesWritten : 3863575492902 | 20260505 22:12:35.378 | INFO | NumEvents : 1822840446643 | 20260505 22:12:37.156 | INFO | Analyzing chunk frames = [6701056,6701568) | 20260505 22:12:37.156 | INFO | MemInfo: { | 20260505 22:12:37.156 | INFO | "gpu" : " 17 GB", | 20260505 22:12:37.156 | INFO | "huge" : "", | 20260505 22:12:37.156 | INFO | "huge_shared" : " 27 GB", | 20260505 22:12:37.156 | INFO | "malloc" : " 14 GB", | 20260505 22:12:37.156 | INFO | "pinned" : " 29 GB" | 20260505 22:12:37.156 | INFO | } | 20260505 22:12:37.156 | INFO | Perf report for the last 5.13768 seconds: | 20260505 22:12:37.156 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:12:37.156 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:37.156 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:37.156 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:37.156 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 22:12:37.156 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:37.156 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260505 22:12:37.156 | INFO | PRE_HQ : 59.7%, 1.8, 54.5ms, 0 | 20260505 22:12:37.156 | INFO | BAZWRITER : 62.2%, 3.7, 116.6ms, 0 | 20260505 22:12:37.156 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 22:12:37.156 | INFO | Uploaded 36GiB to GPU in 2.12759s (16.9205GiB/s) | 20260505 22:12:37.156 | INFO | Downloaded 38GiB to GPU in 2.23233s (17.0225GiB/s) | 20260505 22:12:42.291 | INFO | Analyzing chunk frames = [6701568,6702080) | 20260505 22:12:42.291 | INFO | MemInfo: { | 20260505 22:12:42.291 | INFO | "gpu" : " 17 GB", | 20260505 22:12:42.291 | INFO | "huge" : "", | 20260505 22:12:42.291 | INFO | "huge_shared" : " 27 GB", | 20260505 22:12:42.291 | INFO | "malloc" : " 14 GB", | 20260505 22:12:42.291 | INFO | "pinned" : " 29 GB" | 20260505 22:12:42.291 | INFO | } | 20260505 22:12:42.291 | INFO | Perf report for the last 5.13484 seconds: | 20260505 22:12:42.291 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:12:42.291 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:42.291 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:42.291 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:42.291 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260505 22:12:42.291 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:42.291 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:42.291 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 22:12:42.291 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:42.291 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:42.291 | INFO | Uploaded 36GiB to GPU in 2.0777s (17.3269GiB/s) | 20260505 22:12:42.291 | INFO | Downloaded 34GiB to GPU in 2.0237s (16.8009GiB/s) | 20260505 22:12:47.438 | INFO | Analyzing chunk frames = [6702080,6702592) | 20260505 22:12:47.438 | INFO | MemInfo: { | 20260505 22:12:47.438 | INFO | "gpu" : " 17 GB", | 20260505 22:12:47.438 | INFO | "huge" : "", | 20260505 22:12:47.438 | INFO | "huge_shared" : " 27 GB", | 20260505 22:12:47.438 | INFO | "malloc" : " 14 GB", | 20260505 22:12:47.438 | INFO | "pinned" : " 29 GB" | 20260505 22:12:47.438 | INFO | } | 20260505 22:12:47.438 | INFO | Perf report for the last 5.14754 seconds: | 20260505 22:12:47.438 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:12:47.438 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:47.439 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:47.439 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:47.439 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260505 22:12:47.439 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:47.439 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:47.439 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 22:12:47.439 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:47.439 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:47.439 | INFO | Uploaded 36GiB to GPU in 2.14236s (16.8039GiB/s) | 20260505 22:12:47.439 | INFO | Downloaded 34GiB to GPU in 2.02897s (16.7573GiB/s) | 20260505 22:12:52.575 | INFO | Analyzing chunk frames = [6702592,6703104) | 20260505 22:12:52.575 | INFO | MemInfo: { | 20260505 22:12:52.575 | INFO | "gpu" : " 17 GB", | 20260505 22:12:52.575 | INFO | "huge" : "", | 20260505 22:12:52.575 | INFO | "huge_shared" : " 27 GB", | 20260505 22:12:52.575 | INFO | "malloc" : " 14 GB", | 20260505 22:12:52.575 | INFO | "pinned" : " 29 GB" | 20260505 22:12:52.575 | INFO | } | 20260505 22:12:52.575 | INFO | Perf report for the last 5.1369 seconds: | 20260505 22:12:52.575 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:12:52.575 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:52.575 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:52.575 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:52.575 | INFO | ANALYSIS : 61.8%, 3.0, 95.9ms, 0 | 20260505 22:12:52.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:52.575 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:52.575 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 22:12:52.575 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:52.576 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:52.576 | INFO | Uploaded 36GiB to GPU in 2.13276s (16.8795GiB/s) | 20260505 22:12:52.576 | INFO | Downloaded 34GiB to GPU in 2.0096s (16.9188GiB/s) | 20260505 22:12:57.714 | INFO | Analyzing chunk frames = [6703104,6703616) | 20260505 22:12:57.714 | INFO | MemInfo: { | 20260505 22:12:57.714 | INFO | "gpu" : " 17 GB", | 20260505 22:12:57.714 | INFO | "huge" : "", | 20260505 22:12:57.714 | INFO | "huge_shared" : " 27 GB", | 20260505 22:12:57.714 | INFO | "malloc" : " 14 GB", | 20260505 22:12:57.714 | INFO | "pinned" : " 29 GB" | 20260505 22:12:57.714 | INFO | } | 20260505 22:12:57.714 | INFO | Perf report for the last 5.13861 seconds: | 20260505 22:12:57.714 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:12:57.714 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:57.714 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:57.714 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:57.714 | INFO | ANALYSIS : 61.6%, 3.0, 96.2ms, 0 | 20260505 22:12:57.714 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:57.714 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:57.714 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 22:12:57.714 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:57.714 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:12:57.714 | INFO | Uploaded 36GiB to GPU in 2.13484s (16.8631GiB/s) | 20260505 22:12:57.714 | INFO | Downloaded 34GiB to GPU in 2.06594s (16.4574GiB/s) | 20260505 22:13:02.857 | INFO | Analyzing chunk frames = [6703616,6704128) | 20260505 22:13:02.857 | INFO | MemInfo: { | 20260505 22:13:02.857 | INFO | "gpu" : " 17 GB", | 20260505 22:13:02.857 | INFO | "huge" : "", | 20260505 22:13:02.857 | INFO | "huge_shared" : " 27 GB", | 20260505 22:13:02.857 | INFO | "malloc" : " 14 GB", | 20260505 22:13:02.857 | INFO | "pinned" : " 29 GB" | 20260505 22:13:02.857 | INFO | } | 20260505 22:13:02.857 | INFO | Perf report for the last 5.14314 seconds: | 20260505 22:13:02.857 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:13:02.857 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:02.857 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:02.857 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:02.857 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 22:13:02.857 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:02.857 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:02.857 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 22:13:02.857 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:02.857 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:02.858 | INFO | Uploaded 36GiB to GPU in 2.15746s (16.6863GiB/s) | 20260505 22:13:02.858 | INFO | Downloaded 34GiB to GPU in 2.06727s (16.4468GiB/s) | 20260505 22:13:07.996 | INFO | Analyzing chunk frames = [6704128,6704640) | 20260505 22:13:07.996 | INFO | MemInfo: { | 20260505 22:13:07.996 | INFO | "gpu" : " 17 GB", | 20260505 22:13:07.996 | INFO | "huge" : "", | 20260505 22:13:07.996 | INFO | "huge_shared" : " 27 GB", | 20260505 22:13:07.996 | INFO | "malloc" : " 14 GB", | 20260505 22:13:07.996 | INFO | "pinned" : " 29 GB" | 20260505 22:13:07.996 | INFO | } | 20260505 22:13:07.996 | INFO | Perf report for the last 5.13915 seconds: | 20260505 22:13:07.996 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:13:07.996 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:07.996 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:07.996 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:07.996 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 22:13:07.996 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:07.996 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:07.996 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 22:13:07.996 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:07.997 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:07.997 | INFO | Uploaded 36GiB to GPU in 2.14536s (16.7804GiB/s) | 20260505 22:13:07.997 | INFO | Downloaded 34GiB to GPU in 2.12642s (15.9893GiB/s) | 20260505 22:13:13.138 | INFO | Analyzing chunk frames = [6704640,6705152) | 20260505 22:13:13.138 | INFO | MemInfo: { | 20260505 22:13:13.138 | INFO | "gpu" : " 17 GB", | 20260505 22:13:13.138 | INFO | "huge" : "", | 20260505 22:13:13.138 | INFO | "huge_shared" : " 27 GB", | 20260505 22:13:13.138 | INFO | "malloc" : " 14 GB", | 20260505 22:13:13.138 | INFO | "pinned" : " 29 GB" | 20260505 22:13:13.138 | INFO | } | 20260505 22:13:13.138 | INFO | Perf report for the last 5.14201 seconds: | 20260505 22:13:13.138 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:13:13.138 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:13.138 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:13.139 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:13.139 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 22:13:13.139 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:13.139 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:13.139 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 22:13:13.139 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:13.139 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:13.139 | INFO | Uploaded 36GiB to GPU in 2.11354s (17.033GiB/s) | 20260505 22:13:13.139 | INFO | Downloaded 34GiB to GPU in 2.00864s (16.9269GiB/s) | 20260505 22:13:13.258 | INFO | Starting a round of Baz IO | 20260505 22:13:16.408 | INFO | Populating RTMetrics report | 20260505 22:13:16.408 | INFO | RTMetrics report populated | 20260505 22:13:16.408 | INFO | Opening tmp RTMetrics json file | 20260505 22:13:16.409 | INFO | Serializing RTMetrics to json | 20260505 22:13:16.410 | INFO | Serialization done, swapping tmp file now... | 20260505 22:13:16.411 | INFO | RTMetrics done | 20260505 22:13:16.494 | INFO | Finished a round of Baz IO: Wrote 1.939 GiB in 3.236 seconds | 20260505 22:13:16.495 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.51% / 0.00% / 78.49% | 20260505 22:13:16.495 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.128742% / 0% / 9.93411e-05% | 20260505 22:13:16.495 | INFO | Baz IO overall: | 20260505 22:13:16.495 | INFO | BytesWritten : 3865657290803 | 20260505 22:13:16.495 | INFO | NumEvents : 1823799025215 | 20260505 22:13:18.275 | INFO | Analyzing chunk frames = [6705152,6705664) | 20260505 22:13:18.275 | INFO | MemInfo: { | 20260505 22:13:18.275 | INFO | "gpu" : " 17 GB", | 20260505 22:13:18.275 | INFO | "huge" : "", | 20260505 22:13:18.275 | INFO | "huge_shared" : " 27 GB", | 20260505 22:13:18.275 | INFO | "malloc" : " 14 GB", | 20260505 22:13:18.275 | INFO | "pinned" : " 29 GB" | 20260505 22:13:18.275 | INFO | } | 20260505 22:13:18.275 | INFO | Perf report for the last 5.13708 seconds: | 20260505 22:13:18.275 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:13:18.275 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:18.275 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:18.275 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:18.275 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 22:13:18.276 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:18.276 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260505 22:13:18.276 | INFO | PRE_HQ : 60.2%, 1.8, 55.2ms, 0 | 20260505 22:13:18.276 | INFO | BAZWRITER : 62.2%, 3.7, 117.7ms, 0 | 20260505 22:13:18.276 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 22:13:18.276 | INFO | Uploaded 36GiB to GPU in 2.16352s (16.6396GiB/s) | 20260505 22:13:18.276 | INFO | Downloaded 38GiB to GPU in 2.23929s (16.9697GiB/s) | 20260505 22:13:23.407 | INFO | Analyzing chunk frames = [6705664,6706176) | 20260505 22:13:23.407 | INFO | MemInfo: { | 20260505 22:13:23.407 | INFO | "gpu" : " 17 GB", | 20260505 22:13:23.407 | INFO | "huge" : "", | 20260505 22:13:23.407 | INFO | "huge_shared" : " 27 GB", | 20260505 22:13:23.407 | INFO | "malloc" : " 14 GB", | 20260505 22:13:23.407 | INFO | "pinned" : " 29 GB" | 20260505 22:13:23.407 | INFO | } | 20260505 22:13:23.407 | INFO | Perf report for the last 5.13146 seconds: | 20260505 22:13:23.407 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:13:23.407 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:23.407 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:23.407 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:23.407 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260505 22:13:23.407 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:23.407 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:23.407 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 22:13:23.407 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:23.407 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:23.407 | INFO | Uploaded 36GiB to GPU in 2.10445s (17.1066GiB/s) | 20260505 22:13:23.407 | INFO | Downloaded 34GiB to GPU in 2.03989s (16.6676GiB/s) | 20260505 22:13:28.550 | INFO | Analyzing chunk frames = [6706176,6706688) | 20260505 22:13:28.550 | INFO | MemInfo: { | 20260505 22:13:28.550 | INFO | "gpu" : " 17 GB", | 20260505 22:13:28.551 | INFO | "huge" : "", | 20260505 22:13:28.551 | INFO | "huge_shared" : " 27 GB", | 20260505 22:13:28.551 | INFO | "malloc" : " 14 GB", | 20260505 22:13:28.551 | INFO | "pinned" : " 29 GB" | 20260505 22:13:28.551 | INFO | } | 20260505 22:13:28.551 | INFO | Perf report for the last 5.14371 seconds: | 20260505 22:13:28.551 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:13:28.551 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:28.551 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:28.551 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:28.551 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260505 22:13:28.551 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:28.551 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:28.551 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:13:28.551 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:28.551 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:28.551 | INFO | Uploaded 36GiB to GPU in 2.1213s (16.9707GiB/s) | 20260505 22:13:28.551 | INFO | Downloaded 34GiB to GPU in 2.01104s (16.9066GiB/s) | 20260505 22:13:33.692 | INFO | Analyzing chunk frames = [6706688,6707200) | 20260505 22:13:33.692 | INFO | MemInfo: { | 20260505 22:13:33.692 | INFO | "gpu" : " 17 GB", | 20260505 22:13:33.692 | INFO | "huge" : "", | 20260505 22:13:33.692 | INFO | "huge_shared" : " 27 GB", | 20260505 22:13:33.692 | INFO | "malloc" : " 14 GB", | 20260505 22:13:33.692 | INFO | "pinned" : " 29 GB" | 20260505 22:13:33.692 | INFO | } | 20260505 22:13:33.692 | INFO | Perf report for the last 5.1411 seconds: | 20260505 22:13:33.692 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:13:33.692 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:33.692 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:33.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:33.692 | INFO | ANALYSIS : 61.9%, 3.0, 96.2ms, 0 | 20260505 22:13:33.692 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:33.692 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:33.692 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 22:13:33.692 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:33.692 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:33.692 | INFO | Uploaded 36GiB to GPU in 2.15266s (16.7235GiB/s) | 20260505 22:13:33.692 | INFO | Downloaded 34GiB to GPU in 2.05162s (16.5723GiB/s) | 20260505 22:13:38.835 | INFO | Analyzing chunk frames = [6707200,6707712) | 20260505 22:13:38.835 | INFO | MemInfo: { | 20260505 22:13:38.835 | INFO | "gpu" : " 17 GB", | 20260505 22:13:38.835 | INFO | "huge" : "", | 20260505 22:13:38.835 | INFO | "huge_shared" : " 27 GB", | 20260505 22:13:38.835 | INFO | "malloc" : " 14 GB", | 20260505 22:13:38.835 | INFO | "pinned" : " 29 GB" | 20260505 22:13:38.835 | INFO | } | 20260505 22:13:38.835 | INFO | Perf report for the last 5.14306 seconds: | 20260505 22:13:38.835 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:13:38.835 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:38.835 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:38.835 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:38.835 | INFO | ANALYSIS : 61.5%, 3.0, 96.1ms, 0 | 20260505 22:13:38.835 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:38.835 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:38.835 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:13:38.835 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:38.835 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:38.835 | INFO | Uploaded 36GiB to GPU in 2.13107s (16.8929GiB/s) | 20260505 22:13:38.835 | INFO | Downloaded 34GiB to GPU in 2.06382s (16.4743GiB/s) | 20260505 22:13:43.967 | INFO | Analyzing chunk frames = [6707712,6708224) | 20260505 22:13:43.967 | INFO | MemInfo: { | 20260505 22:13:43.967 | INFO | "gpu" : " 17 GB", | 20260505 22:13:43.967 | INFO | "huge" : "", | 20260505 22:13:43.967 | INFO | "huge_shared" : " 27 GB", | 20260505 22:13:43.967 | INFO | "malloc" : " 14 GB", | 20260505 22:13:43.967 | INFO | "pinned" : " 29 GB" | 20260505 22:13:43.967 | INFO | } | 20260505 22:13:43.967 | INFO | Perf report for the last 5.13234 seconds: | 20260505 22:13:43.967 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:13:43.967 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:43.967 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:43.967 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:43.967 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260505 22:13:43.967 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:43.967 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:43.967 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 22:13:43.967 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:43.967 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:43.967 | INFO | Uploaded 36GiB to GPU in 2.07903s (17.3158GiB/s) | 20260505 22:13:43.967 | INFO | Downloaded 34GiB to GPU in 2.03781s (16.6845GiB/s) | 20260505 22:13:49.105 | INFO | Analyzing chunk frames = [6708224,6708736) | 20260505 22:13:49.105 | INFO | MemInfo: { | 20260505 22:13:49.105 | INFO | "gpu" : " 17 GB", | 20260505 22:13:49.105 | INFO | "huge" : "", | 20260505 22:13:49.105 | INFO | "huge_shared" : " 27 GB", | 20260505 22:13:49.105 | INFO | "malloc" : " 14 GB", | 20260505 22:13:49.105 | INFO | "pinned" : " 29 GB" | 20260505 22:13:49.105 | INFO | } | 20260505 22:13:49.106 | INFO | Perf report for the last 5.13836 seconds: | 20260505 22:13:49.106 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:13:49.106 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:49.106 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:49.106 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:49.106 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 22:13:49.106 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:49.106 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:49.106 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 22:13:49.106 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:49.106 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:49.106 | INFO | Uploaded 36GiB to GPU in 2.14296s (16.7992GiB/s) | 20260505 22:13:49.106 | INFO | Downloaded 34GiB to GPU in 2.10401s (16.1596GiB/s) | 20260505 22:13:54.248 | INFO | Analyzing chunk frames = [6708736,6709248) | 20260505 22:13:54.248 | INFO | MemInfo: { | 20260505 22:13:54.248 | INFO | "gpu" : " 17 GB", | 20260505 22:13:54.248 | INFO | "huge" : "", | 20260505 22:13:54.248 | INFO | "huge_shared" : " 27 GB", | 20260505 22:13:54.248 | INFO | "malloc" : " 14 GB", | 20260505 22:13:54.248 | INFO | "pinned" : " 29 GB" | 20260505 22:13:54.248 | INFO | } | 20260505 22:13:54.248 | INFO | Perf report for the last 5.14236 seconds: | 20260505 22:13:54.248 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:13:54.248 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:54.248 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:54.248 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:54.248 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260505 22:13:54.248 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:54.248 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:54.248 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 22:13:54.248 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:54.248 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:54.248 | INFO | Uploaded 36GiB to GPU in 2.10642s (17.0906GiB/s) | 20260505 22:13:54.248 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 22:13:54.248 | INFO | Total mem usage: 15.3 GiB | 20260505 22:13:54.248 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 22:13:54.248 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.2%) | 20260505 22:13:54.248 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 22:13:54.248 | INFO | High Water Breakdown: | 20260505 22:13:54.248 | INFO | 61% : 8.64 GiB ( 90.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 22:13:54.248 | INFO | 38.8% : 5.5 GiB ( 82.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 22:13:54.248 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 22:13:54.248 | INFO | ------------------------------------------ | 20260505 22:13:54.248 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:13:54.248 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 22:13:54.248 | INFO | Total mem usage: 26.5 GiB | 20260505 22:13:54.248 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 22:13:54.248 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 22:13:54.248 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 22:13:54.248 | INFO | High Water Breakdown: | 20260505 22:13:54.248 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:13:54.248 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:13:54.248 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:13:54.248 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:13:54.248 | INFO | Downloaded 34GiB to GPU in 2.01s (16.9154GiB/s) | 20260505 22:13:54.248 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:13:54.249 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:13:54.249 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 22:13:54.249 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 22:13:54.249 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:13:54.249 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 22:13:54.249 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 22:13:54.249 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 22:13:54.249 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 22:13:54.249 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 22:13:54.249 | INFO | ------------------------------------------ | 20260505 22:13:54.249 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:13:54.249 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 22:13:54.249 | INFO | Total mem usage: 25.6 GiB | 20260505 22:13:54.249 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 22:13:54.249 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 22:13:54.249 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 22:13:54.249 | INFO | High Water Breakdown: | 20260505 22:13:54.249 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 22:13:54.249 | INFO | ------------------------------------------ | 20260505 22:13:54.249 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:13:54.249 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 22:13:54.249 | INFO | Total mem usage: 15.7 GiB | 20260505 22:13:54.249 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 22:13:54.249 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 22:13:54.249 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 22:13:54.249 | INFO | High Water Breakdown: | 20260505 22:13:54.249 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 22:13:54.249 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 22:13:54.249 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 22:13:54.249 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 22:13:54.249 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 22:13:54.249 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 22:13:54.249 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 22:13:54.249 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 22:13:54.249 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 22:13:54.249 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:13:54.249 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 22:13:54.249 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:13:54.249 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 22:13:54.249 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 22:13:54.249 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 22:13:54.249 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 22:13:54.249 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:13:54.249 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 22:13:54.249 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:13:54.249 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:13:54.249 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:13:54.249 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 22:13:54.249 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 22:13:54.249 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:13:54.249 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 22:13:54.249 | INFO | ------------------------------------------ | 20260505 22:13:54.249 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:13:54.249 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 22:13:54.249 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:13:54.249 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:13:54.249 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.39e+01, 1.34e+01 / 1.28e+00 | 20260505 22:13:54.249 | INFO | PerfReport: 0.30%, AggregateStats -- 9.27e-02 / 1.04e-01, 1.13e-02 / 4.93e-03 | 20260505 22:13:54.249 | INFO | PerfReport: 11.82%, Baseline -- 3.71e+00 / 4.53e+00, 5.69e-01 / 1.64e-01 | 20260505 22:13:54.249 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.52e+00, 1.87e-01 / 9.15e-02 | 20260505 22:13:54.249 | INFO | PerfReport: 15.95%, DME -- 4.00e+01 / 4.38e+01, 3.85e+00 / 1.27e+00 | 20260505 22:13:54.249 | INFO | PerfReport: 57.20%, FrameLabeling -- 1.79e+01 / 1.95e+01, 1.48e+00 / 1.07e+00 | 20260505 22:13:54.249 | INFO | PerfReport: 7.19%, PulseAccumulating -- 2.25e+00 / 2.78e+00, 4.31e-01 / 2.18e-01 | 20260505 22:13:54.249 | INFO | PerfReport: 3.17%, Metrics -- 9.95e-01 / 1.26e+00, 2.68e-01 / 1.32e-01 | 20260505 22:13:54.249 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.45e-01 / 4.01e-01, 5.89e-02 / 3.44e-02 | 20260505 22:13:54.249 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 22:13:54.249 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:13:54.249 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:13:54.249 | INFO | PerfReport: Full Execution -- 9.88e+01 / -nan, 2.50e+01 / 0.00e+00 | 20260505 22:13:54.249 | INFO | PerfReport: 22.42%, Upload -- 2.22e+01 / 0.00e+00, 7.67e+00 / 0.00e+00 | 20260505 22:13:54.249 | INFO | PerfReport: 21.82%, Download -- 2.16e+01 / 0.00e+00, 7.66e+00 / 0.00e+00 | 20260505 22:13:54.250 | INFO | PerfReport: 31.75%, Compute -- 3.14e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 22:13:54.250 | INFO | PerfReport: 2.19%, LockUpload -- 2.17e+00 / 0.00e+00, 7.93e+00 / 0.00e+00 | 20260505 22:13:54.250 | INFO | PerfReport: 1.38%, LockDownload -- 1.36e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260505 22:13:54.250 | INFO | PerfReport: 20.43%, LockCompute -- 2.02e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 22:13:54.250 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.38e-02 / 0.00e+00 | 20260505 22:13:54.375 | INFO | Starting a round of Baz IO | 20260505 22:13:57.522 | INFO | Populating RTMetrics report | 20260505 22:13:57.522 | INFO | RTMetrics report populated | 20260505 22:13:57.522 | INFO | Opening tmp RTMetrics json file | 20260505 22:13:57.525 | INFO | Serializing RTMetrics to json | 20260505 22:13:57.527 | INFO | Serialization done, swapping tmp file now... | 20260505 22:13:57.529 | INFO | RTMetrics done | 20260505 22:13:57.604 | INFO | Finished a round of Baz IO: Wrote 1.942 GiB in 3.229 seconds | 20260505 22:13:57.604 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.51% / 0.00% / 78.49% | 20260505 22:13:57.604 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.129728% / 0% / 9.53674e-05% | 20260505 22:13:57.604 | INFO | Baz IO overall: | 20260505 22:13:57.604 | INFO | BytesWritten : 3867742228446 | 20260505 22:13:57.605 | INFO | NumEvents : 1824759793674 | 20260505 22:13:59.386 | INFO | Analyzing chunk frames = [6709248,6709760) | 20260505 22:13:59.386 | INFO | MemInfo: { | 20260505 22:13:59.386 | INFO | "gpu" : " 17 GB", | 20260505 22:13:59.386 | INFO | "huge" : "", | 20260505 22:13:59.386 | INFO | "huge_shared" : " 27 GB", | 20260505 22:13:59.386 | INFO | "malloc" : " 14 GB", | 20260505 22:13:59.386 | INFO | "pinned" : " 29 GB" | 20260505 22:13:59.386 | INFO | } | 20260505 22:13:59.386 | INFO | Perf report for the last 5.13843 seconds: | 20260505 22:13:59.386 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:13:59.386 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:59.386 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:59.386 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:59.387 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 22:13:59.387 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:13:59.387 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260505 22:13:59.387 | INFO | PRE_HQ : 59.7%, 1.8, 54.4ms, 0 | 20260505 22:13:59.387 | INFO | BAZWRITER : 62.1%, 3.6, 116.4ms, 0 | 20260505 22:13:59.387 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 22:13:59.387 | INFO | Uploaded 36GiB to GPU in 2.13414s (16.8686GiB/s) | 20260505 22:13:59.387 | INFO | Downloaded 38GiB to GPU in 2.2268s (17.0648GiB/s) | 20260505 22:14:04.527 | INFO | Analyzing chunk frames = [6709760,6710272) | 20260505 22:14:04.527 | INFO | MemInfo: { | 20260505 22:14:04.527 | INFO | "gpu" : " 17 GB", | 20260505 22:14:04.527 | INFO | "huge" : "", | 20260505 22:14:04.527 | INFO | "huge_shared" : " 27 GB", | 20260505 22:14:04.527 | INFO | "malloc" : " 14 GB", | 20260505 22:14:04.527 | INFO | "pinned" : " 29 GB" | 20260505 22:14:04.527 | INFO | } | 20260505 22:14:04.527 | INFO | Perf report for the last 5.14102 seconds: | 20260505 22:14:04.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:14:04.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:04.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:04.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:04.527 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 22:14:04.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:04.527 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:04.527 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 22:14:04.527 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:04.528 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:04.528 | INFO | Uploaded 36GiB to GPU in 2.08952s (17.2289GiB/s) | 20260505 22:14:04.528 | INFO | Downloaded 34GiB to GPU in 2.03339s (16.7208GiB/s) | 20260505 22:14:09.662 | INFO | Analyzing chunk frames = [6710272,6710784) | 20260505 22:14:09.662 | INFO | MemInfo: { | 20260505 22:14:09.662 | INFO | "gpu" : " 17 GB", | 20260505 22:14:09.662 | INFO | "huge" : "", | 20260505 22:14:09.662 | INFO | "huge_shared" : " 27 GB", | 20260505 22:14:09.662 | INFO | "malloc" : " 14 GB", | 20260505 22:14:09.662 | INFO | "pinned" : " 29 GB" | 20260505 22:14:09.662 | INFO | } | 20260505 22:14:09.662 | INFO | Perf report for the last 5.13508 seconds: | 20260505 22:14:09.662 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:14:09.662 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:09.662 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:09.663 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:09.663 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260505 22:14:09.663 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:09.663 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:09.663 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 22:14:09.663 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:09.663 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:09.663 | INFO | Uploaded 36GiB to GPU in 2.14151s (16.8106GiB/s) | 20260505 22:14:09.663 | INFO | Downloaded 34GiB to GPU in 2.02492s (16.7908GiB/s) | 20260505 22:14:14.801 | INFO | Analyzing chunk frames = [6710784,6711296) | 20260505 22:14:14.801 | INFO | MemInfo: { | 20260505 22:14:14.801 | INFO | "gpu" : " 17 GB", | 20260505 22:14:14.801 | INFO | "huge" : "", | 20260505 22:14:14.801 | INFO | "huge_shared" : " 27 GB", | 20260505 22:14:14.801 | INFO | "malloc" : " 14 GB", | 20260505 22:14:14.801 | INFO | "pinned" : " 29 GB" | 20260505 22:14:14.801 | INFO | } | 20260505 22:14:14.801 | INFO | Perf report for the last 5.13868 seconds: | 20260505 22:14:14.801 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:14:14.801 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:14.801 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:14.801 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:14.801 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260505 22:14:14.801 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:14.801 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:14.801 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:14:14.801 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:14.801 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:14.801 | INFO | Uploaded 36GiB to GPU in 2.20787s (16.3053GiB/s) | 20260505 22:14:14.801 | INFO | Downloaded 34GiB to GPU in 2.05222s (16.5674GiB/s) | 20260505 22:14:19.951 | INFO | Analyzing chunk frames = [6711296,6711808) | 20260505 22:14:19.951 | INFO | MemInfo: { | 20260505 22:14:19.951 | INFO | "gpu" : " 17 GB", | 20260505 22:14:19.951 | INFO | "huge" : "", | 20260505 22:14:19.951 | INFO | "huge_shared" : " 27 GB", | 20260505 22:14:19.951 | INFO | "malloc" : " 14 GB", | 20260505 22:14:19.951 | INFO | "pinned" : " 29 GB" | 20260505 22:14:19.951 | INFO | } | 20260505 22:14:19.951 | INFO | Perf report for the last 5.14964 seconds: | 20260505 22:14:19.951 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:14:19.951 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:19.951 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:19.951 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:19.951 | INFO | ANALYSIS : 61.4%, 3.0, 96.1ms, 0 | 20260505 22:14:19.951 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:19.951 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:19.951 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 22:14:19.951 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:19.951 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:19.951 | INFO | Uploaded 36GiB to GPU in 2.12978s (16.9031GiB/s) | 20260505 22:14:19.951 | INFO | Downloaded 34GiB to GPU in 2.05767s (16.5235GiB/s) | 20260505 22:14:25.095 | INFO | Analyzing chunk frames = [6711808,6712320) | 20260505 22:14:25.095 | INFO | MemInfo: { | 20260505 22:14:25.095 | INFO | "gpu" : " 17 GB", | 20260505 22:14:25.095 | INFO | "huge" : "", | 20260505 22:14:25.095 | INFO | "huge_shared" : " 27 GB", | 20260505 22:14:25.095 | INFO | "malloc" : " 14 GB", | 20260505 22:14:25.095 | INFO | "pinned" : " 29 GB" | 20260505 22:14:25.095 | INFO | } | 20260505 22:14:25.096 | INFO | Perf report for the last 5.14472 seconds: | 20260505 22:14:25.096 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:14:25.096 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:25.096 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:25.096 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:25.096 | INFO | ANALYSIS : 61.2%, 3.0, 95.6ms, 0 | 20260505 22:14:25.096 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:25.096 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:25.096 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 22:14:25.096 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:25.096 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:25.096 | INFO | Uploaded 36GiB to GPU in 2.05849s (17.4885GiB/s) | 20260505 22:14:25.096 | INFO | Downloaded 34GiB to GPU in 2.03576s (16.7014GiB/s) | 20260505 22:14:30.223 | INFO | Analyzing chunk frames = [6712320,6712832) | 20260505 22:14:30.223 | INFO | MemInfo: { | 20260505 22:14:30.223 | INFO | "gpu" : " 17 GB", | 20260505 22:14:30.223 | INFO | "huge" : "", | 20260505 22:14:30.223 | INFO | "huge_shared" : " 27 GB", | 20260505 22:14:30.223 | INFO | "malloc" : " 14 GB", | 20260505 22:14:30.223 | INFO | "pinned" : " 29 GB" | 20260505 22:14:30.223 | INFO | } | 20260505 22:14:30.223 | INFO | Perf report for the last 5.12753 seconds: | 20260505 22:14:30.223 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:14:30.223 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:30.223 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:30.223 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:30.223 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260505 22:14:30.223 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:30.223 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:30.223 | INFO | PRE_HQ : 27.7%, 1.0, 14.3ms, 0 | 20260505 22:14:30.223 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:30.223 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:30.223 | INFO | Uploaded 36GiB to GPU in 2.01678s (17.8503GiB/s) | 20260505 22:14:30.224 | INFO | Downloaded 34GiB to GPU in 2.03938s (16.6717GiB/s) | 20260505 22:14:35.357 | INFO | Analyzing chunk frames = [6712832,6713344) | 20260505 22:14:35.357 | INFO | MemInfo: { | 20260505 22:14:35.357 | INFO | "gpu" : " 17 GB", | 20260505 22:14:35.357 | INFO | "huge" : "", | 20260505 22:14:35.357 | INFO | "huge_shared" : " 27 GB", | 20260505 22:14:35.357 | INFO | "malloc" : " 14 GB", | 20260505 22:14:35.357 | INFO | "pinned" : " 29 GB" | 20260505 22:14:35.357 | INFO | } | 20260505 22:14:35.357 | INFO | Perf report for the last 5.13405 seconds: | 20260505 22:14:35.357 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:14:35.357 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:35.357 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:35.357 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:35.357 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 22:14:35.357 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:35.357 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:35.357 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:14:35.357 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:35.357 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:35.357 | INFO | Uploaded 36GiB to GPU in 2.10452s (17.1061GiB/s) | 20260505 22:14:35.357 | INFO | Downloaded 34GiB to GPU in 2.00674s (16.9429GiB/s) | 20260505 22:14:35.486 | INFO | Starting a round of Baz IO | 20260505 22:14:38.652 | INFO | Populating RTMetrics report | 20260505 22:14:38.653 | INFO | RTMetrics report populated | 20260505 22:14:38.653 | INFO | Opening tmp RTMetrics json file | 20260505 22:14:38.653 | INFO | Serializing RTMetrics to json | 20260505 22:14:38.654 | INFO | Serialization done, swapping tmp file now... | 20260505 22:14:38.655 | INFO | RTMetrics done | 20260505 22:14:38.741 | INFO | Finished a round of Baz IO: Wrote 1.939 GiB in 3.254 seconds | 20260505 22:14:38.741 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.51% / 0.00% / 78.49% | 20260505 22:14:38.741 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.129624% / 3.97364e-06% / 9.13938e-05% | 20260505 22:14:38.741 | INFO | Baz IO overall: | 20260505 22:14:38.741 | INFO | BytesWritten : 3869824344996 | 20260505 22:14:38.741 | INFO | NumEvents : 1825719020449 | 20260505 22:14:40.503 | INFO | Analyzing chunk frames = [6713344,6713856) | 20260505 22:14:40.503 | INFO | MemInfo: { | 20260505 22:14:40.503 | INFO | "gpu" : " 17 GB", | 20260505 22:14:40.503 | INFO | "huge" : "", | 20260505 22:14:40.503 | INFO | "huge_shared" : " 27 GB", | 20260505 22:14:40.503 | INFO | "malloc" : " 14 GB", | 20260505 22:14:40.503 | INFO | "pinned" : " 29 GB" | 20260505 22:14:40.503 | INFO | } | 20260505 22:14:40.503 | INFO | Perf report for the last 5.14575 seconds: | 20260505 22:14:40.503 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:14:40.503 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:40.503 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:40.503 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:40.503 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 22:14:40.503 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:40.503 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260505 22:14:40.503 | INFO | PRE_HQ : 59.9%, 1.8, 54.5ms, 0 | 20260505 22:14:40.503 | INFO | BAZWRITER : 62.8%, 3.6, 116.9ms, 0 | 20260505 22:14:40.503 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 22:14:40.503 | INFO | Uploaded 36GiB to GPU in 2.16851s (16.6013GiB/s) | 20260505 22:14:40.503 | INFO | Downloaded 38GiB to GPU in 2.24854s (16.8999GiB/s) | 20260505 22:14:45.643 | INFO | Analyzing chunk frames = [6713856,6714368) | 20260505 22:14:45.643 | INFO | MemInfo: { | 20260505 22:14:45.643 | INFO | "gpu" : " 17 GB", | 20260505 22:14:45.643 | INFO | "huge" : "", | 20260505 22:14:45.643 | INFO | "huge_shared" : " 27 GB", | 20260505 22:14:45.644 | INFO | "malloc" : " 14 GB", | 20260505 22:14:45.644 | INFO | "pinned" : " 29 GB" | 20260505 22:14:45.644 | INFO | } | 20260505 22:14:45.644 | INFO | Perf report for the last 5.14066 seconds: | 20260505 22:14:45.644 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:14:45.644 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:45.644 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:45.644 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:45.644 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260505 22:14:45.644 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:45.644 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:45.644 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 22:14:45.644 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:45.644 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:45.644 | INFO | Uploaded 36GiB to GPU in 2.08198s (17.2913GiB/s) | 20260505 22:14:45.644 | INFO | Downloaded 34GiB to GPU in 2.02861s (16.7602GiB/s) | 20260505 22:14:50.781 | INFO | Analyzing chunk frames = [6714368,6714880) | 20260505 22:14:50.781 | INFO | MemInfo: { | 20260505 22:14:50.781 | INFO | "gpu" : " 17 GB", | 20260505 22:14:50.781 | INFO | "huge" : "", | 20260505 22:14:50.781 | INFO | "huge_shared" : " 27 GB", | 20260505 22:14:50.781 | INFO | "malloc" : " 14 GB", | 20260505 22:14:50.781 | INFO | "pinned" : " 29 GB" | 20260505 22:14:50.781 | INFO | } | 20260505 22:14:50.781 | INFO | Perf report for the last 5.13719 seconds: | 20260505 22:14:50.781 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:14:50.781 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:50.781 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:50.781 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:50.781 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260505 22:14:50.781 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:50.781 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:50.781 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:14:50.781 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:50.781 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:50.781 | INFO | Uploaded 36GiB to GPU in 2.14691s (16.7683GiB/s) | 20260505 22:14:50.781 | INFO | Downloaded 34GiB to GPU in 2.0557s (16.5394GiB/s) | 20260505 22:14:55.920 | INFO | Analyzing chunk frames = [6714880,6715392) | 20260505 22:14:55.920 | INFO | MemInfo: { | 20260505 22:14:55.920 | INFO | "gpu" : " 17 GB", | 20260505 22:14:55.920 | INFO | "huge" : "", | 20260505 22:14:55.920 | INFO | "huge_shared" : " 27 GB", | 20260505 22:14:55.920 | INFO | "malloc" : " 14 GB", | 20260505 22:14:55.920 | INFO | "pinned" : " 29 GB" | 20260505 22:14:55.920 | INFO | } | 20260505 22:14:55.920 | INFO | Perf report for the last 5.1392 seconds: | 20260505 22:14:55.920 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:14:55.920 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:55.920 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:55.920 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:55.920 | INFO | ANALYSIS : 61.8%, 3.0, 96.0ms, 0 | 20260505 22:14:55.920 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:55.920 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:55.920 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 22:14:55.920 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:55.920 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:14:55.920 | INFO | Uploaded 36GiB to GPU in 2.13091s (16.8942GiB/s) | 20260505 22:14:55.920 | INFO | Downloaded 34GiB to GPU in 2.03555s (16.7031GiB/s) | 20260505 22:15:01.059 | INFO | Analyzing chunk frames = [6715392,6715904) | 20260505 22:15:01.059 | INFO | MemInfo: { | 20260505 22:15:01.059 | INFO | "gpu" : " 17 GB", | 20260505 22:15:01.059 | INFO | "huge" : "", | 20260505 22:15:01.059 | INFO | "huge_shared" : " 27 GB", | 20260505 22:15:01.059 | INFO | "malloc" : " 14 GB", | 20260505 22:15:01.059 | INFO | "pinned" : " 29 GB" | 20260505 22:15:01.059 | INFO | } | 20260505 22:15:01.059 | INFO | Perf report for the last 5.13945 seconds: | 20260505 22:15:01.060 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:15:01.060 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:01.060 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:01.060 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:01.060 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260505 22:15:01.060 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:01.060 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:01.060 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 22:15:01.060 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:01.060 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:01.060 | INFO | Uploaded 36GiB to GPU in 2.13678s (16.8478GiB/s) | 20260505 22:15:01.060 | INFO | Downloaded 34GiB to GPU in 2.06516s (16.4636GiB/s) | 20260505 22:15:06.201 | INFO | Analyzing chunk frames = [6715904,6716416) | 20260505 22:15:06.201 | INFO | MemInfo: { | 20260505 22:15:06.201 | INFO | "gpu" : " 17 GB", | 20260505 22:15:06.201 | INFO | "huge" : "", | 20260505 22:15:06.201 | INFO | "huge_shared" : " 27 GB", | 20260505 22:15:06.201 | INFO | "malloc" : " 14 GB", | 20260505 22:15:06.201 | INFO | "pinned" : " 29 GB" | 20260505 22:15:06.201 | INFO | } | 20260505 22:15:06.201 | INFO | Perf report for the last 5.14194 seconds: | 20260505 22:15:06.201 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:15:06.201 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:06.201 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:06.202 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:06.202 | INFO | ANALYSIS : 61.7%, 3.0, 95.4ms, 0 | 20260505 22:15:06.202 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:06.202 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:06.202 | INFO | PRE_HQ : 26.8%, 1.0, 13.8ms, 0 | 20260505 22:15:06.202 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:06.202 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:06.202 | INFO | Uploaded 36GiB to GPU in 2.08604s (17.2576GiB/s) | 20260505 22:15:06.202 | INFO | Downloaded 34GiB to GPU in 2.04044s (16.663GiB/s) | 20260505 22:15:11.338 | INFO | Analyzing chunk frames = [6716416,6716928) | 20260505 22:15:11.338 | INFO | MemInfo: { | 20260505 22:15:11.338 | INFO | "gpu" : " 17 GB", | 20260505 22:15:11.338 | INFO | "huge" : "", | 20260505 22:15:11.338 | INFO | "huge_shared" : " 27 GB", | 20260505 22:15:11.338 | INFO | "malloc" : " 14 GB", | 20260505 22:15:11.338 | INFO | "pinned" : " 29 GB" | 20260505 22:15:11.338 | INFO | } | 20260505 22:15:11.338 | INFO | Perf report for the last 5.1365 seconds: | 20260505 22:15:11.338 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:15:11.338 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:11.338 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:11.338 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:11.338 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260505 22:15:11.338 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:11.338 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:11.338 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 22:15:11.338 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:11.338 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:11.338 | INFO | Uploaded 36GiB to GPU in 2.06678s (17.4184GiB/s) | 20260505 22:15:11.338 | INFO | Downloaded 34GiB to GPU in 2.0637s (16.4753GiB/s) | 20260505 22:15:16.478 | INFO | Analyzing chunk frames = [6716928,6717440) | 20260505 22:15:16.478 | INFO | MemInfo: { | 20260505 22:15:16.478 | INFO | "gpu" : " 17 GB", | 20260505 22:15:16.478 | INFO | "huge" : "", | 20260505 22:15:16.478 | INFO | "huge_shared" : " 27 GB", | 20260505 22:15:16.478 | INFO | "malloc" : " 14 GB", | 20260505 22:15:16.478 | INFO | "pinned" : " 29 GB" | 20260505 22:15:16.478 | INFO | } | 20260505 22:15:16.478 | INFO | Perf report for the last 5.14007 seconds: | 20260505 22:15:16.478 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:15:16.478 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:16.478 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:16.478 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:16.478 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260505 22:15:16.478 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:16.478 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:16.478 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:15:16.478 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:16.478 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:16.478 | INFO | Uploaded 36GiB to GPU in 2.10971s (17.064GiB/s) | 20260505 22:15:16.478 | INFO | Downloaded 34GiB to GPU in 2.00095s (16.9919GiB/s) | 20260505 22:15:16.479 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 22:15:16.479 | INFO | Total mem usage: 15.3 GiB | 20260505 22:15:16.479 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 22:15:16.479 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -9%) | 20260505 22:15:16.479 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 22:15:16.479 | INFO | High Water Breakdown: | 20260505 22:15:16.479 | INFO | 60.9% : 8.56 GiB ( 90% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 22:15:16.479 | INFO | 38.9% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 22:15:16.479 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 22:15:16.479 | INFO | ------------------------------------------ | 20260505 22:15:16.479 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:15:16.479 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 22:15:16.479 | INFO | Total mem usage: 26.5 GiB | 20260505 22:15:16.479 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 22:15:16.479 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 22:15:16.479 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 22:15:16.479 | INFO | High Water Breakdown: | 20260505 22:15:16.479 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:15:16.479 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:15:16.479 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:15:16.479 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:15:16.479 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:15:16.479 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:15:16.479 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 22:15:16.479 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 22:15:16.479 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:15:16.479 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 22:15:16.479 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 22:15:16.479 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 22:15:16.479 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 22:15:16.479 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 22:15:16.479 | INFO | ------------------------------------------ | 20260505 22:15:16.479 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:15:16.479 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 22:15:16.479 | INFO | Total mem usage: 25.6 GiB | 20260505 22:15:16.479 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 22:15:16.479 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 22:15:16.479 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 22:15:16.479 | INFO | High Water Breakdown: | 20260505 22:15:16.479 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 22:15:16.479 | INFO | ------------------------------------------ | 20260505 22:15:16.479 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:15:16.479 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 22:15:16.479 | INFO | Total mem usage: 15.7 GiB | 20260505 22:15:16.479 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 22:15:16.479 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 22:15:16.479 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 22:15:16.479 | INFO | High Water Breakdown: | 20260505 22:15:16.479 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 22:15:16.479 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 22:15:16.479 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 22:15:16.479 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 22:15:16.479 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 22:15:16.479 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 22:15:16.479 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 22:15:16.479 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 22:15:16.479 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 22:15:16.479 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:15:16.479 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 22:15:16.479 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:15:16.479 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 22:15:16.479 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 22:15:16.479 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 22:15:16.479 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 22:15:16.479 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:15:16.479 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 22:15:16.479 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:15:16.479 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:15:16.479 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:15:16.479 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 22:15:16.479 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 22:15:16.479 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:15:16.479 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 22:15:16.479 | INFO | ------------------------------------------ | 20260505 22:15:16.479 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:15:16.479 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 22:15:16.479 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:15:16.479 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:15:16.479 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.39e+01, 1.36e+01 / 8.28e-01 | 20260505 22:15:16.479 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.04e-01, 1.02e-02 / 3.28e-03 | 20260505 22:15:16.479 | INFO | PerfReport: 11.81%, Baseline -- 3.71e+00 / 4.50e+00, 5.67e-01 / 2.05e-01 | 20260505 22:15:16.479 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.52e+00, 1.90e-01 / 1.04e-01 | 20260505 22:15:16.479 | INFO | PerfReport: 15.99%, DME -- 4.01e+01 / 4.42e+01, 3.92e+00 / 1.06e+00 | 20260505 22:15:16.479 | INFO | PerfReport: 57.17%, FrameLabeling -- 1.79e+01 / 1.91e+01, 1.46e+00 / 8.47e-01 | 20260505 22:15:16.479 | INFO | PerfReport: 7.19%, PulseAccumulating -- 2.26e+00 / 2.83e+00, 4.30e-01 / 1.66e-01 | 20260505 22:15:16.479 | INFO | PerfReport: 3.17%, Metrics -- 9.96e-01 / 1.31e+00, 2.73e-01 / 1.36e-01 | 20260505 22:15:16.480 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.45e-01 / 4.04e-01, 5.92e-02 / 3.56e-02 | 20260505 22:15:16.480 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 22:15:16.480 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:15:16.480 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:15:16.480 | INFO | PerfReport: Full Execution -- 9.87e+01 / -nan, 2.49e+01 / 0.00e+00 | 20260505 22:15:16.480 | INFO | PerfReport: 22.30%, Upload -- 2.20e+01 / 0.00e+00, 7.53e+00 / 0.00e+00 | 20260505 22:15:16.480 | INFO | PerfReport: 21.77%, Download -- 2.15e+01 / 0.00e+00, 7.58e+00 / 0.00e+00 | 20260505 22:15:16.480 | INFO | PerfReport: 31.79%, Compute -- 3.14e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 22:15:16.480 | INFO | PerfReport: 2.18%, LockUpload -- 2.15e+00 / 0.00e+00, 7.83e+00 / 0.00e+00 | 20260505 22:15:16.480 | INFO | PerfReport: 1.33%, LockDownload -- 1.31e+00 / 0.00e+00, 2.95e+00 / 0.00e+00 | 20260505 22:15:16.480 | INFO | PerfReport: 20.62%, LockCompute -- 2.04e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 22:15:16.480 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.08e-02 / 0.00e+00 | 20260505 22:15:16.608 | INFO | Starting a round of Baz IO | 20260505 22:15:19.760 | INFO | Populating RTMetrics report | 20260505 22:15:19.760 | INFO | RTMetrics report populated | 20260505 22:15:19.760 | INFO | Opening tmp RTMetrics json file | 20260505 22:15:19.761 | INFO | Serializing RTMetrics to json | 20260505 22:15:19.762 | INFO | Serialization done, swapping tmp file now... | 20260505 22:15:19.763 | INFO | RTMetrics done | 20260505 22:15:20.052 | INFO | Finished a round of Baz IO: Wrote 1.939 GiB in 3.444 seconds | 20260505 22:15:20.052 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.51% / 0.00% / 78.49% | 20260505 22:15:20.052 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.128921% / 0% / 8.74201e-05% | 20260505 22:15:20.052 | INFO | Baz IO overall: | 20260505 22:15:20.052 | INFO | BytesWritten : 3871906537648 | 20260505 22:15:20.052 | INFO | NumEvents : 1826678123726 | 20260505 22:15:21.615 | INFO | Analyzing chunk frames = [6717440,6717952) | 20260505 22:15:21.615 | INFO | MemInfo: { | 20260505 22:15:21.615 | INFO | "gpu" : " 17 GB", | 20260505 22:15:21.615 | INFO | "huge" : "", | 20260505 22:15:21.615 | INFO | "huge_shared" : " 27 GB", | 20260505 22:15:21.615 | INFO | "malloc" : " 14 GB", | 20260505 22:15:21.615 | INFO | "pinned" : " 29 GB" | 20260505 22:15:21.615 | INFO | } | 20260505 22:15:21.616 | INFO | Perf report for the last 5.13748 seconds: | 20260505 22:15:21.616 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:15:21.616 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:21.616 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:21.616 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:21.616 | INFO | ANALYSIS : 62.9%, 3.0, 97.8ms, 0 | 20260505 22:15:21.616 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:21.616 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260505 22:15:21.616 | INFO | PRE_HQ : 59.9%, 1.7, 54.3ms, 0 | 20260505 22:15:21.616 | INFO | BAZWRITER : 66.5%, 3.8, 128.6ms, 0 | 20260505 22:15:21.616 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 22:15:21.616 | INFO | Uploaded 36GiB to GPU in 2.18298s (16.4912GiB/s) | 20260505 22:15:21.616 | INFO | Downloaded 38GiB to GPU in 2.25186s (16.875GiB/s) | 20260505 22:15:26.755 | INFO | Analyzing chunk frames = [6717952,6718464) | 20260505 22:15:26.755 | INFO | MemInfo: { | 20260505 22:15:26.755 | INFO | "gpu" : " 17 GB", | 20260505 22:15:26.755 | INFO | "huge" : "", | 20260505 22:15:26.755 | INFO | "huge_shared" : " 27 GB", | 20260505 22:15:26.755 | INFO | "malloc" : " 14 GB", | 20260505 22:15:26.755 | INFO | "pinned" : " 29 GB" | 20260505 22:15:26.755 | INFO | } | 20260505 22:15:26.755 | INFO | Perf report for the last 5.13977 seconds: | 20260505 22:15:26.755 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:15:26.755 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:26.755 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:26.755 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:26.755 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260505 22:15:26.755 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:26.755 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:26.755 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 22:15:26.755 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:26.755 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:26.756 | INFO | Uploaded 36GiB to GPU in 2.07533s (17.3467GiB/s) | 20260505 22:15:26.756 | INFO | Downloaded 34GiB to GPU in 2.02025s (16.8296GiB/s) | 20260505 22:15:31.899 | INFO | Analyzing chunk frames = [6718464,6718976) | 20260505 22:15:31.899 | INFO | MemInfo: { | 20260505 22:15:31.899 | INFO | "gpu" : " 17 GB", | 20260505 22:15:31.899 | INFO | "huge" : "", | 20260505 22:15:31.899 | INFO | "huge_shared" : " 27 GB", | 20260505 22:15:31.899 | INFO | "malloc" : " 14 GB", | 20260505 22:15:31.899 | INFO | "pinned" : " 29 GB" | 20260505 22:15:31.899 | INFO | } | 20260505 22:15:31.899 | INFO | Perf report for the last 5.14392 seconds: | 20260505 22:15:31.899 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:15:31.899 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:31.899 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:31.899 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:31.899 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260505 22:15:31.899 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:31.899 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:31.899 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:15:31.899 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:31.899 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:31.900 | INFO | Uploaded 36GiB to GPU in 2.14965s (16.7469GiB/s) | 20260505 22:15:31.900 | INFO | Downloaded 34GiB to GPU in 2.04974s (16.5875GiB/s) | 20260505 22:15:37.036 | INFO | Analyzing chunk frames = [6718976,6719488) | 20260505 22:15:37.036 | INFO | MemInfo: { | 20260505 22:15:37.036 | INFO | "gpu" : " 17 GB", | 20260505 22:15:37.036 | INFO | "huge" : "", | 20260505 22:15:37.036 | INFO | "huge_shared" : " 27 GB", | 20260505 22:15:37.036 | INFO | "malloc" : " 14 GB", | 20260505 22:15:37.036 | INFO | "pinned" : " 29 GB" | 20260505 22:15:37.036 | INFO | } | 20260505 22:15:37.036 | INFO | Perf report for the last 5.13721 seconds: | 20260505 22:15:37.036 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:15:37.036 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:37.036 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:37.036 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:37.037 | INFO | ANALYSIS : 61.8%, 3.0, 95.9ms, 0 | 20260505 22:15:37.037 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:37.037 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:37.037 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:15:37.037 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:37.037 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:37.037 | INFO | Uploaded 36GiB to GPU in 2.12068s (16.9757GiB/s) | 20260505 22:15:37.037 | INFO | Downloaded 34GiB to GPU in 2.02526s (16.788GiB/s) | 20260505 22:15:42.169 | INFO | Analyzing chunk frames = [6719488,6720000) | 20260505 22:15:42.169 | INFO | MemInfo: { | 20260505 22:15:42.169 | INFO | "gpu" : " 17 GB", | 20260505 22:15:42.169 | INFO | "huge" : "", | 20260505 22:15:42.169 | INFO | "huge_shared" : " 27 GB", | 20260505 22:15:42.169 | INFO | "malloc" : " 14 GB", | 20260505 22:15:42.169 | INFO | "pinned" : " 29 GB" | 20260505 22:15:42.170 | INFO | } | 20260505 22:15:42.170 | INFO | Perf report for the last 5.13308 seconds: | 20260505 22:15:42.170 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:15:42.170 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:42.170 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:42.170 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:42.170 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260505 22:15:42.170 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:42.170 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:42.170 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 22:15:42.170 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:42.170 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:42.170 | INFO | Uploaded 36GiB to GPU in 2.08412s (17.2735GiB/s) | 20260505 22:15:42.170 | INFO | Downloaded 34GiB to GPU in 2.03911s (16.674GiB/s) | 20260505 22:15:47.309 | INFO | Analyzing chunk frames = [6720000,6720512) | 20260505 22:15:47.309 | INFO | MemInfo: { | 20260505 22:15:47.309 | INFO | "gpu" : " 17 GB", | 20260505 22:15:47.309 | INFO | "huge" : "", | 20260505 22:15:47.309 | INFO | "huge_shared" : " 27 GB", | 20260505 22:15:47.310 | INFO | "malloc" : " 14 GB", | 20260505 22:15:47.310 | INFO | "pinned" : " 29 GB" | 20260505 22:15:47.310 | INFO | } | 20260505 22:15:47.310 | INFO | Perf report for the last 5.14014 seconds: | 20260505 22:15:47.310 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:15:47.310 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:47.310 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:47.310 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:47.310 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 22:15:47.310 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:47.310 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:47.310 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:15:47.310 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:47.310 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:47.310 | INFO | Uploaded 36GiB to GPU in 2.16235s (16.6485GiB/s) | 20260505 22:15:47.310 | INFO | Downloaded 34GiB to GPU in 2.06913s (16.432GiB/s) | 20260505 22:15:52.456 | INFO | Analyzing chunk frames = [6720512,6721024) | 20260505 22:15:52.456 | INFO | MemInfo: { | 20260505 22:15:52.456 | INFO | "gpu" : " 17 GB", | 20260505 22:15:52.456 | INFO | "huge" : "", | 20260505 22:15:52.456 | INFO | "huge_shared" : " 27 GB", | 20260505 22:15:52.456 | INFO | "malloc" : " 14 GB", | 20260505 22:15:52.456 | INFO | "pinned" : " 29 GB" | 20260505 22:15:52.456 | INFO | } | 20260505 22:15:52.456 | INFO | Perf report for the last 5.14602 seconds: | 20260505 22:15:52.456 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:15:52.456 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:52.456 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:52.456 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:52.456 | INFO | ANALYSIS : 62.7%, 3.0, 98.0ms, 0 | 20260505 22:15:52.456 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:52.456 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:52.456 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260505 22:15:52.456 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:52.456 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:52.456 | INFO | Uploaded 36GiB to GPU in 2.13586s (16.8551GiB/s) | 20260505 22:15:52.456 | INFO | Downloaded 34GiB to GPU in 2.10969s (16.1161GiB/s) | 20260505 22:15:57.597 | INFO | Analyzing chunk frames = [6721024,6721536) | 20260505 22:15:57.597 | INFO | MemInfo: { | 20260505 22:15:57.597 | INFO | "gpu" : " 17 GB", | 20260505 22:15:57.597 | INFO | "huge" : "", | 20260505 22:15:57.597 | INFO | "huge_shared" : " 27 GB", | 20260505 22:15:57.597 | INFO | "malloc" : " 14 GB", | 20260505 22:15:57.597 | INFO | "pinned" : " 29 GB" | 20260505 22:15:57.597 | INFO | } | 20260505 22:15:57.597 | INFO | Perf report for the last 5.14119 seconds: | 20260505 22:15:57.597 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:15:57.597 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:57.597 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:57.597 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:57.597 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 22:15:57.597 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:57.597 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:57.597 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:15:57.597 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:57.597 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:15:57.597 | INFO | Uploaded 36GiB to GPU in 2.11525s (17.0192GiB/s) | 20260505 22:15:57.597 | INFO | Downloaded 34GiB to GPU in 2.002s (16.983GiB/s) | 20260505 22:15:57.714 | INFO | Starting a round of Baz IO | 20260505 22:16:00.866 | INFO | Populating RTMetrics report | 20260505 22:16:00.867 | INFO | RTMetrics report populated | 20260505 22:16:00.867 | INFO | Opening tmp RTMetrics json file | 20260505 22:16:00.867 | INFO | Serializing RTMetrics to json | 20260505 22:16:00.868 | INFO | Serialization done, swapping tmp file now... | 20260505 22:16:00.869 | INFO | RTMetrics done | 20260505 22:16:00.949 | INFO | Finished a round of Baz IO: Wrote 1.936 GiB in 3.234 seconds | 20260505 22:16:00.949 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.51% / 0.00% / 78.49% | 20260505 22:16:00.949 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.128349% / 0% / 0.000107288% | 20260505 22:16:00.949 | INFO | Baz IO overall: | 20260505 22:16:00.949 | INFO | BytesWritten : 3873985696695 | 20260505 22:16:00.949 | INFO | NumEvents : 1827635107919 | 20260505 22:16:02.730 | INFO | Analyzing chunk frames = [6721536,6722048) | 20260505 22:16:02.730 | INFO | MemInfo: { | 20260505 22:16:02.730 | INFO | "gpu" : " 17 GB", | 20260505 22:16:02.730 | INFO | "huge" : "", | 20260505 22:16:02.730 | INFO | "huge_shared" : " 27 GB", | 20260505 22:16:02.730 | INFO | "malloc" : " 14 GB", | 20260505 22:16:02.730 | INFO | "pinned" : " 29 GB" | 20260505 22:16:02.730 | INFO | } | 20260505 22:16:02.730 | INFO | Perf report for the last 5.1334 seconds: | 20260505 22:16:02.730 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:16:02.730 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:02.730 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:02.730 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:02.730 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 22:16:02.730 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:02.730 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260505 22:16:02.730 | INFO | PRE_HQ : 60.1%, 1.8, 54.7ms, 0 | 20260505 22:16:02.731 | INFO | BAZWRITER : 62.2%, 3.6, 116.0ms, 0 | 20260505 22:16:02.731 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 22:16:02.731 | INFO | Uploaded 36GiB to GPU in 2.16661s (16.6158GiB/s) | 20260505 22:16:02.731 | INFO | Downloaded 38GiB to GPU in 2.23868s (16.9743GiB/s) | 20260505 22:16:07.864 | INFO | Analyzing chunk frames = [6722048,6722560) | 20260505 22:16:07.864 | INFO | MemInfo: { | 20260505 22:16:07.864 | INFO | "gpu" : " 17 GB", | 20260505 22:16:07.864 | INFO | "huge" : "", | 20260505 22:16:07.864 | INFO | "huge_shared" : " 27 GB", | 20260505 22:16:07.864 | INFO | "malloc" : " 14 GB", | 20260505 22:16:07.864 | INFO | "pinned" : " 29 GB" | 20260505 22:16:07.864 | INFO | } | 20260505 22:16:07.864 | INFO | Perf report for the last 5.13408 seconds: | 20260505 22:16:07.864 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:16:07.864 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:07.864 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:07.864 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:07.865 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 22:16:07.865 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:07.865 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:07.865 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 22:16:07.865 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:07.865 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:07.865 | INFO | Uploaded 36GiB to GPU in 2.07813s (17.3233GiB/s) | 20260505 22:16:07.865 | INFO | Downloaded 34GiB to GPU in 2.02053s (16.8273GiB/s) | 20260505 22:16:13.011 | INFO | Analyzing chunk frames = [6722560,6723072) | 20260505 22:16:13.011 | INFO | MemInfo: { | 20260505 22:16:13.011 | INFO | "gpu" : " 17 GB", | 20260505 22:16:13.011 | INFO | "huge" : "", | 20260505 22:16:13.011 | INFO | "huge_shared" : " 27 GB", | 20260505 22:16:13.012 | INFO | "malloc" : " 14 GB", | 20260505 22:16:13.012 | INFO | "pinned" : " 29 GB" | 20260505 22:16:13.012 | INFO | } | 20260505 22:16:13.012 | INFO | Perf report for the last 5.14724 seconds: | 20260505 22:16:13.012 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:16:13.012 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:13.012 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:13.012 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:13.012 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260505 22:16:13.012 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:13.012 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:13.012 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 22:16:13.012 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:13.012 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:13.012 | INFO | Uploaded 36GiB to GPU in 2.13306s (16.8772GiB/s) | 20260505 22:16:13.012 | INFO | Downloaded 34GiB to GPU in 2.01837s (16.8453GiB/s) | 20260505 22:16:18.147 | INFO | Analyzing chunk frames = [6723072,6723584) | 20260505 22:16:18.147 | INFO | MemInfo: { | 20260505 22:16:18.147 | INFO | "gpu" : " 17 GB", | 20260505 22:16:18.147 | INFO | "huge" : "", | 20260505 22:16:18.147 | INFO | "huge_shared" : " 27 GB", | 20260505 22:16:18.147 | INFO | "malloc" : " 14 GB", | 20260505 22:16:18.147 | INFO | "pinned" : " 29 GB" | 20260505 22:16:18.147 | INFO | } | 20260505 22:16:18.147 | INFO | Perf report for the last 5.13542 seconds: | 20260505 22:16:18.147 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:16:18.147 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:18.147 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:18.147 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:18.147 | INFO | ANALYSIS : 61.8%, 3.0, 95.9ms, 0 | 20260505 22:16:18.147 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:18.147 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:18.147 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 22:16:18.147 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:18.147 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:18.147 | INFO | Uploaded 36GiB to GPU in 2.12503s (16.9409GiB/s) | 20260505 22:16:18.147 | INFO | Downloaded 34GiB to GPU in 2.01216s (16.8973GiB/s) | 20260505 22:16:23.289 | INFO | Analyzing chunk frames = [6723584,6724096) | 20260505 22:16:23.289 | INFO | MemInfo: { | 20260505 22:16:23.289 | INFO | "gpu" : " 17 GB", | 20260505 22:16:23.289 | INFO | "huge" : "", | 20260505 22:16:23.289 | INFO | "huge_shared" : " 27 GB", | 20260505 22:16:23.289 | INFO | "malloc" : " 14 GB", | 20260505 22:16:23.289 | INFO | "pinned" : " 29 GB" | 20260505 22:16:23.289 | INFO | } | 20260505 22:16:23.289 | INFO | Perf report for the last 5.14212 seconds: | 20260505 22:16:23.289 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:16:23.289 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:23.289 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:23.289 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:23.289 | INFO | ANALYSIS : 61.6%, 3.0, 96.3ms, 0 | 20260505 22:16:23.289 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:23.289 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:23.289 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:16:23.289 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:23.289 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:23.290 | INFO | Uploaded 36GiB to GPU in 2.16818s (16.6038GiB/s) | 20260505 22:16:23.290 | INFO | Downloaded 34GiB to GPU in 2.07177s (16.4111GiB/s) | 20260505 22:16:28.427 | INFO | Analyzing chunk frames = [6724096,6724608) | 20260505 22:16:28.427 | INFO | MemInfo: { | 20260505 22:16:28.427 | INFO | "gpu" : " 17 GB", | 20260505 22:16:28.427 | INFO | "huge" : "", | 20260505 22:16:28.427 | INFO | "huge_shared" : " 27 GB", | 20260505 22:16:28.427 | INFO | "malloc" : " 14 GB", | 20260505 22:16:28.427 | INFO | "pinned" : " 29 GB" | 20260505 22:16:28.427 | INFO | } | 20260505 22:16:28.427 | INFO | Perf report for the last 5.1382 seconds: | 20260505 22:16:28.427 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:16:28.427 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:28.427 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:28.428 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:28.428 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260505 22:16:28.428 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:28.428 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:28.428 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:16:28.428 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:28.428 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:28.428 | INFO | Uploaded 36GiB to GPU in 2.08259s (17.2862GiB/s) | 20260505 22:16:28.428 | INFO | Downloaded 34GiB to GPU in 2.03385s (16.717GiB/s) | 20260505 22:16:33.564 | INFO | Analyzing chunk frames = [6724608,6725120) | 20260505 22:16:33.564 | INFO | MemInfo: { | 20260505 22:16:33.564 | INFO | "gpu" : " 17 GB", | 20260505 22:16:33.564 | INFO | "huge" : "", | 20260505 22:16:33.564 | INFO | "huge_shared" : " 27 GB", | 20260505 22:16:33.564 | INFO | "malloc" : " 14 GB", | 20260505 22:16:33.564 | INFO | "pinned" : " 29 GB" | 20260505 22:16:33.564 | INFO | } | 20260505 22:16:33.564 | INFO | Perf report for the last 5.13653 seconds: | 20260505 22:16:33.564 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:16:33.564 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:33.564 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:33.564 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:33.564 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 22:16:33.564 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:33.564 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:33.564 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 22:16:33.564 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:33.564 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:33.564 | INFO | Uploaded 36GiB to GPU in 2.13521s (16.8602GiB/s) | 20260505 22:16:33.565 | INFO | Downloaded 34GiB to GPU in 2.10709s (16.136GiB/s) | 20260505 22:16:38.712 | INFO | Analyzing chunk frames = [6725120,6725632) | 20260505 22:16:38.712 | INFO | MemInfo: { | 20260505 22:16:38.712 | INFO | "gpu" : " 17 GB", | 20260505 22:16:38.712 | INFO | "huge" : "", | 20260505 22:16:38.712 | INFO | "huge_shared" : " 27 GB", | 20260505 22:16:38.712 | INFO | "malloc" : " 14 GB", | 20260505 22:16:38.712 | INFO | "pinned" : " 29 GB" | 20260505 22:16:38.712 | INFO | } | 20260505 22:16:38.712 | INFO | Perf report for the last 5.14834 seconds: | 20260505 22:16:38.712 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:16:38.712 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:38.712 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:38.712 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:38.712 | INFO | ANALYSIS : 62.0%, 3.0, 97.0ms, 0 | 20260505 22:16:38.712 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:38.712 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:38.713 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:16:38.713 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:38.713 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:38.713 | INFO | Uploaded 36GiB to GPU in 2.10076s (17.1366GiB/s) | 20260505 22:16:38.713 | INFO | Downloaded 34GiB to GPU in 2.00313s (16.9734GiB/s) | 20260505 22:16:38.713 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 22:16:38.713 | INFO | Total mem usage: 15.3 GiB | 20260505 22:16:38.713 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 22:16:38.713 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -3.62%) | 20260505 22:16:38.713 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 22:16:38.713 | INFO | High Water Breakdown: | 20260505 22:16:38.713 | INFO | 61.6% : 9.12 GiB ( 95.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 22:16:38.713 | INFO | 38.2% : 5.65 GiB ( 85.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 22:16:38.713 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 22:16:38.713 | INFO | ------------------------------------------ | 20260505 22:16:38.713 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:16:38.713 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 22:16:38.713 | INFO | Total mem usage: 26.5 GiB | 20260505 22:16:38.713 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 22:16:38.713 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 22:16:38.713 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 22:16:38.713 | INFO | High Water Breakdown: | 20260505 22:16:38.713 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:16:38.713 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:16:38.713 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:16:38.713 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:16:38.713 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:16:38.713 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:16:38.713 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 22:16:38.713 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 22:16:38.713 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:16:38.713 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 22:16:38.713 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 22:16:38.713 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 22:16:38.713 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 22:16:38.713 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 22:16:38.713 | INFO | ------------------------------------------ | 20260505 22:16:38.713 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:16:38.713 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 22:16:38.713 | INFO | Total mem usage: 25.6 GiB | 20260505 22:16:38.713 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 22:16:38.713 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 22:16:38.713 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 22:16:38.713 | INFO | High Water Breakdown: | 20260505 22:16:38.713 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 22:16:38.713 | INFO | ------------------------------------------ | 20260505 22:16:38.713 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:16:38.713 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 22:16:38.713 | INFO | Total mem usage: 15.7 GiB | 20260505 22:16:38.713 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 22:16:38.713 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 22:16:38.713 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 22:16:38.713 | INFO | High Water Breakdown: | 20260505 22:16:38.713 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 22:16:38.713 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 22:16:38.713 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 22:16:38.713 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 22:16:38.713 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 22:16:38.713 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 22:16:38.714 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 22:16:38.714 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 22:16:38.714 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 22:16:38.714 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:16:38.714 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 22:16:38.714 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:16:38.714 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 22:16:38.714 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 22:16:38.714 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 22:16:38.714 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 22:16:38.714 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:16:38.714 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 22:16:38.714 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:16:38.714 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:16:38.714 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:16:38.714 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 22:16:38.714 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 22:16:38.714 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:16:38.714 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 22:16:38.714 | INFO | ------------------------------------------ | 20260505 22:16:38.714 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:16:38.714 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 22:16:38.714 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:16:38.714 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:16:38.714 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.39e+01, 1.35e+01 / 8.80e-01 | 20260505 22:16:38.714 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.03e-01, 1.03e-02 / 3.94e-03 | 20260505 22:16:38.714 | INFO | PerfReport: 11.80%, Baseline -- 3.70e+00 / 4.53e+00, 5.67e-01 / 1.64e-01 | 20260505 22:16:38.714 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.51e+00, 1.87e-01 / 9.21e-02 | 20260505 22:16:38.714 | INFO | PerfReport: 15.96%, DME -- 4.01e+01 / 4.39e+01, 3.87e+00 / 1.13e+00 | 20260505 22:16:38.714 | INFO | PerfReport: 57.25%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.50e+00 / 1.05e+00 | 20260505 22:16:38.714 | INFO | PerfReport: 7.16%, PulseAccumulating -- 2.25e+00 / 2.79e+00, 4.33e-01 / 2.31e-01 | 20260505 22:16:38.714 | INFO | PerfReport: 3.16%, Metrics -- 9.92e-01 / 1.28e+00, 2.68e-01 / 1.45e-01 | 20260505 22:16:38.714 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.45e-01 / 3.97e-01, 5.79e-02 / 3.21e-02 | 20260505 22:16:38.714 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 22:16:38.714 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:16:38.714 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:16:38.714 | INFO | PerfReport: Full Execution -- 9.88e+01 / -nan, 2.49e+01 / 0.00e+00 | 20260505 22:16:38.714 | INFO | PerfReport: 22.42%, Upload -- 2.21e+01 / 0.00e+00, 7.65e+00 / 0.00e+00 | 20260505 22:16:38.714 | INFO | PerfReport: 21.80%, Download -- 2.15e+01 / 0.00e+00, 7.62e+00 / 0.00e+00 | 20260505 22:16:38.714 | INFO | PerfReport: 31.77%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 22:16:38.714 | INFO | PerfReport: 2.23%, LockUpload -- 2.20e+00 / 0.00e+00, 7.93e+00 / 0.00e+00 | 20260505 22:16:38.714 | INFO | PerfReport: 1.32%, LockDownload -- 1.31e+00 / 0.00e+00, 2.84e+00 / 0.00e+00 | 20260505 22:16:38.714 | INFO | PerfReport: 20.45%, LockCompute -- 2.02e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260505 22:16:38.714 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.58e-02 / 0.00e+00 | 20260505 22:16:38.832 | INFO | Starting a round of Baz IO | 20260505 22:16:41.987 | INFO | Populating RTMetrics report | 20260505 22:16:41.988 | INFO | RTMetrics report populated | 20260505 22:16:41.988 | INFO | Opening tmp RTMetrics json file | 20260505 22:16:41.990 | INFO | Serializing RTMetrics to json | 20260505 22:16:41.992 | INFO | Serialization done, swapping tmp file now... | 20260505 22:16:41.992 | INFO | RTMetrics done | 20260505 22:16:42.075 | INFO | Finished a round of Baz IO: Wrote 1.938 GiB in 3.242 seconds | 20260505 22:16:42.075 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.51% / 0.00% / 78.49% | 20260505 22:16:42.075 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.128369% / 0% / 8.34465e-05% | 20260505 22:16:42.075 | INFO | Baz IO overall: | 20260505 22:16:42.075 | INFO | BytesWritten : 3876066379862 | 20260505 22:16:42.075 | INFO | NumEvents : 1828593405081 | 20260505 22:16:43.849 | INFO | Analyzing chunk frames = [6725632,6726144) | 20260505 22:16:43.849 | INFO | MemInfo: { | 20260505 22:16:43.849 | INFO | "gpu" : " 17 GB", | 20260505 22:16:43.849 | INFO | "huge" : "", | 20260505 22:16:43.849 | INFO | "huge_shared" : " 27 GB", | 20260505 22:16:43.849 | INFO | "malloc" : " 14 GB", | 20260505 22:16:43.849 | INFO | "pinned" : " 29 GB" | 20260505 22:16:43.849 | INFO | } | 20260505 22:16:43.849 | INFO | Perf report for the last 5.13696 seconds: | 20260505 22:16:43.850 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:16:43.850 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:43.850 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:43.850 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:43.850 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 22:16:43.850 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:43.850 | INFO | RT_METRICS : 43.8%, 1.0, 22.9ms, 0 | 20260505 22:16:43.850 | INFO | PRE_HQ : 60.0%, 1.8, 54.9ms, 0 | 20260505 22:16:43.850 | INFO | BAZWRITER : 62.4%, 3.6, 116.3ms, 0 | 20260505 22:16:43.850 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 22:16:43.850 | INFO | Uploaded 36GiB to GPU in 2.16493s (16.6287GiB/s) | 20260505 22:16:43.850 | INFO | Downloaded 38GiB to GPU in 2.23638s (16.9917GiB/s) | 20260505 22:16:48.982 | INFO | Analyzing chunk frames = [6726144,6726656) | 20260505 22:16:48.982 | INFO | MemInfo: { | 20260505 22:16:48.982 | INFO | "gpu" : " 17 GB", | 20260505 22:16:48.982 | INFO | "huge" : "", | 20260505 22:16:48.982 | INFO | "huge_shared" : " 27 GB", | 20260505 22:16:48.982 | INFO | "malloc" : " 14 GB", | 20260505 22:16:48.982 | INFO | "pinned" : " 29 GB" | 20260505 22:16:48.983 | INFO | } | 20260505 22:16:48.983 | INFO | Perf report for the last 5.13326 seconds: | 20260505 22:16:48.983 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:16:48.983 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:48.983 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:48.983 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:48.983 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260505 22:16:48.983 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:48.983 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:48.983 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:16:48.983 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:48.983 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:48.983 | INFO | Uploaded 36GiB to GPU in 2.08692s (17.2503GiB/s) | 20260505 22:16:48.983 | INFO | Downloaded 34GiB to GPU in 2.0295s (16.7529GiB/s) | 20260505 22:16:54.124 | INFO | Analyzing chunk frames = [6726656,6727168) | 20260505 22:16:54.124 | INFO | MemInfo: { | 20260505 22:16:54.124 | INFO | "gpu" : " 17 GB", | 20260505 22:16:54.124 | INFO | "huge" : "", | 20260505 22:16:54.125 | INFO | "huge_shared" : " 27 GB", | 20260505 22:16:54.125 | INFO | "malloc" : " 14 GB", | 20260505 22:16:54.125 | INFO | "pinned" : " 29 GB" | 20260505 22:16:54.125 | INFO | } | 20260505 22:16:54.125 | INFO | Perf report for the last 5.14201 seconds: | 20260505 22:16:54.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:16:54.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:54.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:54.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:54.125 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260505 22:16:54.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:54.125 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:54.125 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:16:54.125 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:54.125 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:54.125 | INFO | Uploaded 36GiB to GPU in 2.13269s (16.8801GiB/s) | 20260505 22:16:54.125 | INFO | Downloaded 34GiB to GPU in 2.02186s (16.8162GiB/s) | 20260505 22:16:59.263 | INFO | Analyzing chunk frames = [6727168,6727680) | 20260505 22:16:59.263 | INFO | MemInfo: { | 20260505 22:16:59.263 | INFO | "gpu" : " 17 GB", | 20260505 22:16:59.263 | INFO | "huge" : "", | 20260505 22:16:59.263 | INFO | "huge_shared" : " 27 GB", | 20260505 22:16:59.263 | INFO | "malloc" : " 14 GB", | 20260505 22:16:59.263 | INFO | "pinned" : " 29 GB" | 20260505 22:16:59.263 | INFO | } | 20260505 22:16:59.263 | INFO | Perf report for the last 5.13826 seconds: | 20260505 22:16:59.263 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:16:59.263 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:59.263 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:59.263 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:59.263 | INFO | ANALYSIS : 61.8%, 3.0, 96.0ms, 0 | 20260505 22:16:59.263 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:59.263 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:59.263 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:16:59.263 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:59.263 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:16:59.263 | INFO | Uploaded 36GiB to GPU in 2.12448s (16.9453GiB/s) | 20260505 22:16:59.263 | INFO | Downloaded 34GiB to GPU in 2.02645s (16.7781GiB/s) | 20260505 22:17:04.403 | INFO | Analyzing chunk frames = [6727680,6728192) | 20260505 22:17:04.403 | INFO | MemInfo: { | 20260505 22:17:04.403 | INFO | "gpu" : " 17 GB", | 20260505 22:17:04.403 | INFO | "huge" : "", | 20260505 22:17:04.403 | INFO | "huge_shared" : " 27 GB", | 20260505 22:17:04.403 | INFO | "malloc" : " 14 GB", | 20260505 22:17:04.404 | INFO | "pinned" : " 29 GB" | 20260505 22:17:04.404 | INFO | } | 20260505 22:17:04.404 | INFO | Perf report for the last 5.14077 seconds: | 20260505 22:17:04.404 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:17:04.404 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:04.404 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:04.404 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:04.404 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260505 22:17:04.404 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:04.404 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:04.404 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:17:04.404 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:04.404 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:04.404 | INFO | Uploaded 36GiB to GPU in 2.0925s (17.2043GiB/s) | 20260505 22:17:04.404 | INFO | Downloaded 34GiB to GPU in 2.04805s (16.6011GiB/s) | 20260505 22:17:09.546 | INFO | Analyzing chunk frames = [6728192,6728704) | 20260505 22:17:09.546 | INFO | MemInfo: { | 20260505 22:17:09.546 | INFO | "gpu" : " 17 GB", | 20260505 22:17:09.546 | INFO | "huge" : "", | 20260505 22:17:09.547 | INFO | "huge_shared" : " 27 GB", | 20260505 22:17:09.547 | INFO | "malloc" : " 14 GB", | 20260505 22:17:09.547 | INFO | "pinned" : " 29 GB" | 20260505 22:17:09.547 | INFO | } | 20260505 22:17:09.547 | INFO | Perf report for the last 5.14348 seconds: | 20260505 22:17:09.547 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:17:09.547 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:09.547 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:09.547 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:09.547 | INFO | ANALYSIS : 61.5%, 3.0, 96.1ms, 0 | 20260505 22:17:09.547 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:09.547 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:09.547 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:17:09.547 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:09.547 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:09.547 | INFO | Uploaded 36GiB to GPU in 2.07751s (17.3285GiB/s) | 20260505 22:17:09.548 | INFO | Downloaded 34GiB to GPU in 2.03941s (16.6715GiB/s) | 20260505 22:17:14.681 | INFO | Analyzing chunk frames = [6728704,6729216) | 20260505 22:17:14.681 | INFO | MemInfo: { | 20260505 22:17:14.681 | INFO | "gpu" : " 17 GB", | 20260505 22:17:14.681 | INFO | "huge" : "", | 20260505 22:17:14.681 | INFO | "huge_shared" : " 27 GB", | 20260505 22:17:14.681 | INFO | "malloc" : " 14 GB", | 20260505 22:17:14.681 | INFO | "pinned" : " 29 GB" | 20260505 22:17:14.681 | INFO | } | 20260505 22:17:14.681 | INFO | Perf report for the last 5.13387 seconds: | 20260505 22:17:14.681 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:17:14.681 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:14.681 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:14.681 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:14.681 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260505 22:17:14.681 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:14.681 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:14.681 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 22:17:14.681 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:14.681 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:14.681 | INFO | Uploaded 36GiB to GPU in 2.03524s (17.6884GiB/s) | 20260505 22:17:14.681 | INFO | Downloaded 34GiB to GPU in 2.04445s (16.6304GiB/s) | 20260505 22:17:19.820 | INFO | Analyzing chunk frames = [6729216,6729728) | 20260505 22:17:19.820 | INFO | MemInfo: { | 20260505 22:17:19.820 | INFO | "gpu" : " 17 GB", | 20260505 22:17:19.820 | INFO | "huge" : "", | 20260505 22:17:19.820 | INFO | "huge_shared" : " 27 GB", | 20260505 22:17:19.820 | INFO | "malloc" : " 14 GB", | 20260505 22:17:19.820 | INFO | "pinned" : " 29 GB" | 20260505 22:17:19.820 | INFO | } | 20260505 22:17:19.820 | INFO | Perf report for the last 5.13936 seconds: | 20260505 22:17:19.820 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:17:19.820 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:19.820 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:19.820 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:19.820 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 22:17:19.820 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:19.820 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:19.821 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:17:19.821 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:19.821 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:19.821 | INFO | Uploaded 36GiB to GPU in 2.10679s (17.0876GiB/s) | 20260505 22:17:19.821 | INFO | Downloaded 34GiB to GPU in 2.00646s (16.9453GiB/s) | 20260505 22:17:19.957 | INFO | Starting a round of Baz IO | 20260505 22:17:23.097 | INFO | Populating RTMetrics report | 20260505 22:17:23.097 | INFO | RTMetrics report populated | 20260505 22:17:23.097 | INFO | Opening tmp RTMetrics json file | 20260505 22:17:23.098 | INFO | Serializing RTMetrics to json | 20260505 22:17:23.099 | INFO | Serialization done, swapping tmp file now... | 20260505 22:17:23.100 | INFO | RTMetrics done | 20260505 22:17:23.180 | INFO | Finished a round of Baz IO: Wrote 1.937 GiB in 3.223 seconds | 20260505 22:17:23.180 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.51% / 0.00% / 78.49% | 20260505 22:17:23.180 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.128078% / 0% / 9.53674e-05% | 20260505 22:17:23.180 | INFO | Baz IO overall: | 20260505 22:17:23.180 | INFO | BytesWritten : 3878146575086 | 20260505 22:17:23.180 | INFO | NumEvents : 1829551313466 | 20260505 22:17:24.962 | INFO | Analyzing chunk frames = [6729728,6730240) | 20260505 22:17:24.962 | INFO | MemInfo: { | 20260505 22:17:24.962 | INFO | "gpu" : " 17 GB", | 20260505 22:17:24.962 | INFO | "huge" : "", | 20260505 22:17:24.962 | INFO | "huge_shared" : " 27 GB", | 20260505 22:17:24.962 | INFO | "malloc" : " 14 GB", | 20260505 22:17:24.962 | INFO | "pinned" : " 29 GB" | 20260505 22:17:24.962 | INFO | } | 20260505 22:17:24.962 | INFO | Perf report for the last 5.14171 seconds: | 20260505 22:17:24.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:17:24.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:24.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:24.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:24.962 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 22:17:24.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:24.962 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260505 22:17:24.962 | INFO | PRE_HQ : 59.7%, 1.8, 54.7ms, 0 | 20260505 22:17:24.962 | INFO | BAZWRITER : 62.2%, 3.7, 117.3ms, 0 | 20260505 22:17:24.962 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 22:17:24.962 | INFO | Uploaded 36GiB to GPU in 2.12281s (16.9587GiB/s) | 20260505 22:17:24.962 | INFO | Downloaded 38GiB to GPU in 2.22246s (17.0982GiB/s) | 20260505 22:17:30.100 | INFO | Analyzing chunk frames = [6730240,6730752) | 20260505 22:17:30.100 | INFO | MemInfo: { | 20260505 22:17:30.100 | INFO | "gpu" : " 17 GB", | 20260505 22:17:30.100 | INFO | "huge" : "", | 20260505 22:17:30.100 | INFO | "huge_shared" : " 27 GB", | 20260505 22:17:30.100 | INFO | "malloc" : " 14 GB", | 20260505 22:17:30.100 | INFO | "pinned" : " 29 GB" | 20260505 22:17:30.100 | INFO | } | 20260505 22:17:30.100 | INFO | Perf report for the last 5.13813 seconds: | 20260505 22:17:30.100 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:17:30.100 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:30.100 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:30.100 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:30.100 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260505 22:17:30.100 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:30.100 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:30.100 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:17:30.100 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:30.100 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:30.101 | INFO | Uploaded 36GiB to GPU in 2.08816s (17.24GiB/s) | 20260505 22:17:30.101 | INFO | Downloaded 34GiB to GPU in 2.02796s (16.7656GiB/s) | 20260505 22:17:35.236 | INFO | Analyzing chunk frames = [6730752,6731264) | 20260505 22:17:35.236 | INFO | MemInfo: { | 20260505 22:17:35.236 | INFO | "gpu" : " 17 GB", | 20260505 22:17:35.236 | INFO | "huge" : "", | 20260505 22:17:35.236 | INFO | "huge_shared" : " 27 GB", | 20260505 22:17:35.236 | INFO | "malloc" : " 14 GB", | 20260505 22:17:35.236 | INFO | "pinned" : " 29 GB" | 20260505 22:17:35.236 | INFO | } | 20260505 22:17:35.236 | INFO | Perf report for the last 5.13605 seconds: | 20260505 22:17:35.236 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:17:35.236 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:35.236 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:35.236 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:35.236 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260505 22:17:35.236 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:35.237 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:35.237 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:17:35.237 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:35.237 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:35.237 | INFO | Uploaded 36GiB to GPU in 2.13506s (16.8613GiB/s) | 20260505 22:17:35.237 | INFO | Downloaded 34GiB to GPU in 2.02075s (16.8254GiB/s) | 20260505 22:17:40.373 | INFO | Analyzing chunk frames = [6731264,6731776) | 20260505 22:17:40.373 | INFO | MemInfo: { | 20260505 22:17:40.373 | INFO | "gpu" : " 17 GB", | 20260505 22:17:40.373 | INFO | "huge" : "", | 20260505 22:17:40.373 | INFO | "huge_shared" : " 27 GB", | 20260505 22:17:40.373 | INFO | "malloc" : " 14 GB", | 20260505 22:17:40.373 | INFO | "pinned" : " 29 GB" | 20260505 22:17:40.373 | INFO | } | 20260505 22:17:40.373 | INFO | Perf report for the last 5.13647 seconds: | 20260505 22:17:40.373 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:17:40.373 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:40.373 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:40.373 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:40.373 | INFO | ANALYSIS : 62.3%, 3.0, 96.7ms, 0 | 20260505 22:17:40.373 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:40.373 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:40.373 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:17:40.373 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:40.373 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:40.373 | INFO | Uploaded 36GiB to GPU in 2.20682s (16.313GiB/s) | 20260505 22:17:40.373 | INFO | Downloaded 34GiB to GPU in 2.06294s (16.4813GiB/s) | 20260505 22:17:45.520 | INFO | Analyzing chunk frames = [6731776,6732288) | 20260505 22:17:45.520 | INFO | MemInfo: { | 20260505 22:17:45.520 | INFO | "gpu" : " 17 GB", | 20260505 22:17:45.520 | INFO | "huge" : "", | 20260505 22:17:45.520 | INFO | "huge_shared" : " 27 GB", | 20260505 22:17:45.520 | INFO | "malloc" : " 14 GB", | 20260505 22:17:45.520 | INFO | "pinned" : " 29 GB" | 20260505 22:17:45.520 | INFO | } | 20260505 22:17:45.520 | INFO | Perf report for the last 5.14749 seconds: | 20260505 22:17:45.520 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:17:45.520 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:45.520 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:45.520 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:45.520 | INFO | ANALYSIS : 61.5%, 3.0, 96.1ms, 0 | 20260505 22:17:45.520 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:45.520 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:45.521 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:17:45.521 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:45.521 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:45.521 | INFO | Uploaded 36GiB to GPU in 2.1313s (16.8911GiB/s) | 20260505 22:17:45.521 | INFO | Downloaded 34GiB to GPU in 2.0604s (16.5017GiB/s) | 20260505 22:17:50.663 | INFO | Analyzing chunk frames = [6732288,6732800) | 20260505 22:17:50.663 | INFO | MemInfo: { | 20260505 22:17:50.663 | INFO | "gpu" : " 17 GB", | 20260505 22:17:50.663 | INFO | "huge" : "", | 20260505 22:17:50.663 | INFO | "huge_shared" : " 27 GB", | 20260505 22:17:50.663 | INFO | "malloc" : " 14 GB", | 20260505 22:17:50.663 | INFO | "pinned" : " 29 GB" | 20260505 22:17:50.663 | INFO | } | 20260505 22:17:50.663 | INFO | Perf report for the last 5.14268 seconds: | 20260505 22:17:50.663 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:17:50.663 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:50.663 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:50.663 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:50.663 | INFO | ANALYSIS : 61.5%, 3.0, 96.1ms, 0 | 20260505 22:17:50.663 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:50.663 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:50.663 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 22:17:50.663 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:50.663 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:50.663 | INFO | Uploaded 36GiB to GPU in 2.09158s (17.2118GiB/s) | 20260505 22:17:50.663 | INFO | Downloaded 34GiB to GPU in 2.04759s (16.6049GiB/s) | 20260505 22:17:55.794 | INFO | Analyzing chunk frames = [6732800,6733312) | 20260505 22:17:55.795 | INFO | MemInfo: { | 20260505 22:17:55.795 | INFO | "gpu" : " 17 GB", | 20260505 22:17:55.795 | INFO | "huge" : "", | 20260505 22:17:55.795 | INFO | "huge_shared" : " 27 GB", | 20260505 22:17:55.795 | INFO | "malloc" : " 14 GB", | 20260505 22:17:55.795 | INFO | "pinned" : " 29 GB" | 20260505 22:17:55.795 | INFO | } | 20260505 22:17:55.795 | INFO | Perf report for the last 5.13172 seconds: | 20260505 22:17:55.795 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:17:55.795 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:55.795 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:55.795 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:55.795 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 22:17:55.795 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:55.795 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:55.795 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 22:17:55.795 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:55.795 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:17:55.795 | INFO | Uploaded 36GiB to GPU in 2.10402s (17.1101GiB/s) | 20260505 22:17:55.795 | INFO | Downloaded 34GiB to GPU in 2.07401s (16.3933GiB/s) | 20260505 22:18:00.930 | INFO | Analyzing chunk frames = [6733312,6733824) | 20260505 22:18:00.930 | INFO | MemInfo: { | 20260505 22:18:00.930 | INFO | "gpu" : " 17 GB", | 20260505 22:18:00.930 | INFO | "huge" : "", | 20260505 22:18:00.930 | INFO | "huge_shared" : " 27 GB", | 20260505 22:18:00.930 | INFO | "malloc" : " 14 GB", | 20260505 22:18:00.930 | INFO | "pinned" : " 29 GB" | 20260505 22:18:00.930 | INFO | } | 20260505 22:18:00.930 | INFO | Perf report for the last 5.13562 seconds: | 20260505 22:18:00.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:18:00.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:00.930 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:00.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:00.930 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 22:18:00.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:00.931 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:00.931 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:18:00.931 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:00.931 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:00.931 | INFO | Uploaded 36GiB to GPU in 2.11659s (17.0085GiB/s) | 20260505 22:18:00.931 | INFO | Downloaded 34GiB to GPU in 2.00143s (16.9878GiB/s) | 20260505 22:18:00.931 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 22:18:00.931 | INFO | Total mem usage: 15.3 GiB | 20260505 22:18:00.931 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 22:18:00.931 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -9%) | 20260505 22:18:00.931 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 22:18:00.931 | INFO | High Water Breakdown: | 20260505 22:18:00.931 | INFO | 60.9% : 8.56 GiB ( 90% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 22:18:00.931 | INFO | 38.9% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 22:18:00.931 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 22:18:00.931 | INFO | ------------------------------------------ | 20260505 22:18:00.931 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:18:00.931 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 22:18:00.931 | INFO | Total mem usage: 26.5 GiB | 20260505 22:18:00.931 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 22:18:00.931 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 22:18:00.931 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 22:18:00.931 | INFO | High Water Breakdown: | 20260505 22:18:00.931 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:18:00.931 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:18:00.931 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:18:00.931 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:18:00.931 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:18:00.931 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:18:00.931 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 22:18:00.931 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 22:18:00.931 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:18:00.931 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 22:18:00.931 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 22:18:00.931 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 22:18:00.931 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 22:18:00.931 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 22:18:00.931 | INFO | ------------------------------------------ | 20260505 22:18:00.931 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:18:00.931 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 22:18:00.931 | INFO | Total mem usage: 25.6 GiB | 20260505 22:18:00.931 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 22:18:00.931 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 22:18:00.931 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 22:18:00.931 | INFO | High Water Breakdown: | 20260505 22:18:00.931 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 22:18:00.931 | INFO | ------------------------------------------ | 20260505 22:18:00.931 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:18:00.931 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 22:18:00.931 | INFO | Total mem usage: 15.7 GiB | 20260505 22:18:00.931 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 22:18:00.931 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 22:18:00.931 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 22:18:00.931 | INFO | High Water Breakdown: | 20260505 22:18:00.932 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 22:18:00.932 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 22:18:00.932 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 22:18:00.932 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 22:18:00.932 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 22:18:00.932 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 22:18:00.932 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 22:18:00.932 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 22:18:00.932 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 22:18:00.932 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:18:00.932 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 22:18:00.932 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:18:00.932 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 22:18:00.932 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 22:18:00.932 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 22:18:00.932 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 22:18:00.932 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:18:00.932 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 22:18:00.932 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:18:00.932 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:18:00.932 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:18:00.932 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 22:18:00.932 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 22:18:00.932 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:18:00.932 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 22:18:00.932 | INFO | ------------------------------------------ | 20260505 22:18:00.932 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:18:00.932 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 22:18:00.932 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:18:00.932 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:18:00.932 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.42e+01, 1.35e+01 / 8.85e-01 | 20260505 22:18:00.932 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.05e-01, 1.02e-02 / 5.70e-03 | 20260505 22:18:00.932 | INFO | PerfReport: 11.77%, Baseline -- 3.70e+00 / 4.47e+00, 5.61e-01 / 2.03e-01 | 20260505 22:18:00.932 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.53e+00, 1.92e-01 / 8.48e-02 | 20260505 22:18:00.932 | INFO | PerfReport: 15.99%, DME -- 4.02e+01 / 4.41e+01, 3.89e+00 / 1.14e+00 | 20260505 22:18:00.932 | INFO | PerfReport: 57.25%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.52e+00 / 1.12e+00 | 20260505 22:18:00.932 | INFO | PerfReport: 7.16%, PulseAccumulating -- 2.25e+00 / 2.79e+00, 4.30e-01 / 2.30e-01 | 20260505 22:18:00.932 | INFO | PerfReport: 3.16%, Metrics -- 9.92e-01 / 1.28e+00, 2.67e-01 / 1.32e-01 | 20260505 22:18:00.932 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.45e-01 / 4.03e-01, 5.88e-02 / 3.64e-02 | 20260505 22:18:00.932 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 22:18:00.932 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:18:00.932 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:18:00.932 | INFO | PerfReport: Full Execution -- 9.87e+01 / -nan, 2.50e+01 / 0.00e+00 | 20260505 22:18:00.932 | INFO | PerfReport: 22.30%, Upload -- 2.20e+01 / 0.00e+00, 7.50e+00 / 0.00e+00 | 20260505 22:18:00.932 | INFO | PerfReport: 21.74%, Download -- 2.15e+01 / 0.00e+00, 7.55e+00 / 0.00e+00 | 20260505 22:18:00.932 | INFO | PerfReport: 31.83%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 22:18:00.932 | INFO | PerfReport: 2.18%, LockUpload -- 2.15e+00 / 0.00e+00, 7.89e+00 / 0.00e+00 | 20260505 22:18:00.932 | INFO | PerfReport: 1.28%, LockDownload -- 1.26e+00 / 0.00e+00, 2.85e+00 / 0.00e+00 | 20260505 22:18:00.932 | INFO | PerfReport: 20.65%, LockCompute -- 2.04e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 22:18:00.932 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.49e-02 / 0.00e+00 | 20260505 22:18:01.064 | INFO | Starting a round of Baz IO | 20260505 22:18:04.227 | INFO | Populating RTMetrics report | 20260505 22:18:04.228 | INFO | RTMetrics report populated | 20260505 22:18:04.228 | INFO | Opening tmp RTMetrics json file | 20260505 22:18:04.230 | INFO | Serializing RTMetrics to json | 20260505 22:18:04.232 | INFO | Serialization done, swapping tmp file now... | 20260505 22:18:04.234 | INFO | RTMetrics done | 20260505 22:18:04.310 | INFO | Finished a round of Baz IO: Wrote 1.937 GiB in 3.246 seconds | 20260505 22:18:04.310 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.51% / 0.00% / 78.49% | 20260505 22:18:04.310 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.127844% / 0% / 9.13938e-05% | 20260505 22:18:04.310 | INFO | Baz IO overall: | 20260505 22:18:04.310 | INFO | BytesWritten : 3880226301964 | 20260505 22:18:04.310 | INFO | NumEvents : 1830509069881 | 20260505 22:18:06.077 | INFO | Analyzing chunk frames = [6733824,6734336) | 20260505 22:18:06.077 | INFO | MemInfo: { | 20260505 22:18:06.077 | INFO | "gpu" : " 17 GB", | 20260505 22:18:06.077 | INFO | "huge" : "", | 20260505 22:18:06.077 | INFO | "huge_shared" : " 27 GB", | 20260505 22:18:06.077 | INFO | "malloc" : " 14 GB", | 20260505 22:18:06.077 | INFO | "pinned" : " 29 GB" | 20260505 22:18:06.077 | INFO | } | 20260505 22:18:06.077 | INFO | Perf report for the last 5.14632 seconds: | 20260505 22:18:06.077 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:18:06.077 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:06.077 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:06.077 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:06.077 | INFO | ANALYSIS : 63.0%, 3.0, 98.5ms, 0 | 20260505 22:18:06.077 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:06.077 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260505 22:18:06.077 | INFO | PRE_HQ : 59.8%, 1.8, 54.6ms, 0 | 20260505 22:18:06.077 | INFO | BAZWRITER : 62.6%, 3.6, 115.5ms, 0 | 20260505 22:18:06.077 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 22:18:06.077 | INFO | Uploaded 36GiB to GPU in 2.16932s (16.5951GiB/s) | 20260505 22:18:06.077 | INFO | Downloaded 38GiB to GPU in 2.25009s (16.8882GiB/s) | 20260505 22:18:11.217 | INFO | Analyzing chunk frames = [6734336,6734848) | 20260505 22:18:11.217 | INFO | MemInfo: { | 20260505 22:18:11.217 | INFO | "gpu" : " 17 GB", | 20260505 22:18:11.217 | INFO | "huge" : "", | 20260505 22:18:11.217 | INFO | "huge_shared" : " 27 GB", | 20260505 22:18:11.218 | INFO | "malloc" : " 14 GB", | 20260505 22:18:11.218 | INFO | "pinned" : " 29 GB" | 20260505 22:18:11.218 | INFO | } | 20260505 22:18:11.218 | INFO | Perf report for the last 5.14086 seconds: | 20260505 22:18:11.218 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:18:11.218 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:11.218 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:11.218 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:11.218 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260505 22:18:11.218 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:11.218 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:11.218 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 22:18:11.218 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:11.218 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:11.218 | INFO | Uploaded 36GiB to GPU in 2.08165s (17.294GiB/s) | 20260505 22:18:11.218 | INFO | Downloaded 34GiB to GPU in 2.02355s (16.8022GiB/s) | 20260505 22:18:16.360 | INFO | Analyzing chunk frames = [6734848,6735360) | 20260505 22:18:16.360 | INFO | MemInfo: { | 20260505 22:18:16.360 | INFO | "gpu" : " 17 GB", | 20260505 22:18:16.360 | INFO | "huge" : "", | 20260505 22:18:16.360 | INFO | "huge_shared" : " 27 GB", | 20260505 22:18:16.360 | INFO | "malloc" : " 14 GB", | 20260505 22:18:16.360 | INFO | "pinned" : " 29 GB" | 20260505 22:18:16.360 | INFO | } | 20260505 22:18:16.361 | INFO | Perf report for the last 5.1429 seconds: | 20260505 22:18:16.361 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:18:16.361 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:16.361 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:16.361 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:16.361 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260505 22:18:16.361 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:16.361 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:16.361 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:18:16.361 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:16.361 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:16.361 | INFO | Uploaded 36GiB to GPU in 2.11045s (17.0579GiB/s) | 20260505 22:18:16.361 | INFO | Downloaded 34GiB to GPU in 2.03455s (16.7113GiB/s) | 20260505 22:18:21.495 | INFO | Analyzing chunk frames = [6735360,6735872) | 20260505 22:18:21.495 | INFO | MemInfo: { | 20260505 22:18:21.495 | INFO | "gpu" : " 17 GB", | 20260505 22:18:21.495 | INFO | "huge" : "", | 20260505 22:18:21.495 | INFO | "huge_shared" : " 27 GB", | 20260505 22:18:21.495 | INFO | "malloc" : " 14 GB", | 20260505 22:18:21.495 | INFO | "pinned" : " 29 GB" | 20260505 22:18:21.495 | INFO | } | 20260505 22:18:21.496 | INFO | Perf report for the last 5.13505 seconds: | 20260505 22:18:21.496 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:18:21.496 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:21.496 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:21.496 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:21.496 | INFO | ANALYSIS : 61.8%, 3.0, 95.8ms, 0 | 20260505 22:18:21.496 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:21.496 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:21.496 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:18:21.496 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:21.496 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:21.496 | INFO | Uploaded 36GiB to GPU in 2.12699s (16.9254GiB/s) | 20260505 22:18:21.496 | INFO | Downloaded 34GiB to GPU in 2.00813s (16.9312GiB/s) | 20260505 22:18:26.627 | INFO | Analyzing chunk frames = [6735872,6736384) | 20260505 22:18:26.627 | INFO | MemInfo: { | 20260505 22:18:26.627 | INFO | "gpu" : " 17 GB", | 20260505 22:18:26.627 | INFO | "huge" : "", | 20260505 22:18:26.627 | INFO | "huge_shared" : " 27 GB", | 20260505 22:18:26.627 | INFO | "malloc" : " 14 GB", | 20260505 22:18:26.627 | INFO | "pinned" : " 29 GB" | 20260505 22:18:26.627 | INFO | } | 20260505 22:18:26.627 | INFO | Perf report for the last 5.13139 seconds: | 20260505 22:18:26.627 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:18:26.627 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:26.627 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:26.627 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:26.627 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260505 22:18:26.627 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:26.627 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:26.627 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 22:18:26.627 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:26.627 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:26.627 | INFO | Uploaded 36GiB to GPU in 2.15936s (16.6716GiB/s) | 20260505 22:18:26.627 | INFO | Downloaded 34GiB to GPU in 2.07002s (16.425GiB/s) | 20260505 22:18:31.774 | INFO | Analyzing chunk frames = [6736384,6736896) | 20260505 22:18:31.774 | INFO | MemInfo: { | 20260505 22:18:31.774 | INFO | "gpu" : " 17 GB", | 20260505 22:18:31.774 | INFO | "huge" : "", | 20260505 22:18:31.774 | INFO | "huge_shared" : " 27 GB", | 20260505 22:18:31.774 | INFO | "malloc" : " 14 GB", | 20260505 22:18:31.774 | INFO | "pinned" : " 29 GB" | 20260505 22:18:31.774 | INFO | } | 20260505 22:18:31.774 | INFO | Perf report for the last 5.147 seconds: | 20260505 22:18:31.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:18:31.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:31.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:31.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:31.774 | INFO | ANALYSIS : 62.3%, 3.0, 97.4ms, 0 | 20260505 22:18:31.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:31.774 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:31.774 | INFO | PRE_HQ : 27.0%, 1.0, 14.1ms, 0 | 20260505 22:18:31.774 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:31.774 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:31.774 | INFO | Uploaded 36GiB to GPU in 2.1697s (16.5922GiB/s) | 20260505 22:18:31.774 | INFO | Downloaded 34GiB to GPU in 2.07562s (16.3807GiB/s) | 20260505 22:18:36.918 | INFO | Analyzing chunk frames = [6736896,6737408) | 20260505 22:18:36.918 | INFO | MemInfo: { | 20260505 22:18:36.918 | INFO | "gpu" : " 17 GB", | 20260505 22:18:36.918 | INFO | "huge" : "", | 20260505 22:18:36.918 | INFO | "huge_shared" : " 27 GB", | 20260505 22:18:36.918 | INFO | "malloc" : " 14 GB", | 20260505 22:18:36.918 | INFO | "pinned" : " 29 GB" | 20260505 22:18:36.918 | INFO | } | 20260505 22:18:36.918 | INFO | Perf report for the last 5.14447 seconds: | 20260505 22:18:36.918 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:18:36.918 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:36.918 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:36.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:36.919 | INFO | ANALYSIS : 62.2%, 3.0, 97.3ms, 0 | 20260505 22:18:36.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:36.919 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:36.919 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 22:18:36.919 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:36.919 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:36.919 | INFO | Uploaded 36GiB to GPU in 2.03915s (17.6545GiB/s) | 20260505 22:18:36.919 | INFO | Downloaded 34GiB to GPU in 2.04622s (16.616GiB/s) | 20260505 22:18:42.050 | INFO | Analyzing chunk frames = [6737408,6737920) | 20260505 22:18:42.050 | INFO | MemInfo: { | 20260505 22:18:42.050 | INFO | "gpu" : " 17 GB", | 20260505 22:18:42.050 | INFO | "huge" : "", | 20260505 22:18:42.050 | INFO | "huge_shared" : " 27 GB", | 20260505 22:18:42.050 | INFO | "malloc" : " 14 GB", | 20260505 22:18:42.050 | INFO | "pinned" : " 29 GB" | 20260505 22:18:42.050 | INFO | } | 20260505 22:18:42.050 | INFO | Perf report for the last 5.13161 seconds: | 20260505 22:18:42.050 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:18:42.050 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:42.050 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:42.050 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:42.050 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260505 22:18:42.050 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:42.050 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:42.050 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:18:42.050 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:42.050 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:42.050 | INFO | Uploaded 36GiB to GPU in 2.10895s (17.0701GiB/s) | 20260505 22:18:42.050 | INFO | Downloaded 34GiB to GPU in 2.00334s (16.9717GiB/s) | 20260505 22:18:42.187 | INFO | Starting a round of Baz IO | 20260505 22:18:45.333 | INFO | Populating RTMetrics report | 20260505 22:18:45.333 | INFO | RTMetrics report populated | 20260505 22:18:45.333 | INFO | Opening tmp RTMetrics json file | 20260505 22:18:45.334 | INFO | Serializing RTMetrics to json | 20260505 22:18:45.335 | INFO | Serialization done, swapping tmp file now... | 20260505 22:18:45.335 | INFO | RTMetrics done | 20260505 22:18:45.419 | INFO | Finished a round of Baz IO: Wrote 1.936 GiB in 3.231 seconds | 20260505 22:18:45.419 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.51% / 0.00% / 78.49% | 20260505 22:18:45.419 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.127335% / 3.97364e-06% / 9.13938e-05% | 20260505 22:18:45.419 | INFO | Baz IO overall: | 20260505 22:18:45.419 | INFO | BytesWritten : 3882304992432 | 20260505 22:18:45.419 | INFO | NumEvents : 1831466123227 | 20260505 22:18:47.188 | INFO | Analyzing chunk frames = [6737920,6738432) | 20260505 22:18:47.188 | INFO | MemInfo: { | 20260505 22:18:47.188 | INFO | "gpu" : " 17 GB", | 20260505 22:18:47.188 | INFO | "huge" : "", | 20260505 22:18:47.188 | INFO | "huge_shared" : " 27 GB", | 20260505 22:18:47.188 | INFO | "malloc" : " 14 GB", | 20260505 22:18:47.188 | INFO | "pinned" : " 29 GB" | 20260505 22:18:47.188 | INFO | } | 20260505 22:18:47.188 | INFO | Perf report for the last 5.13828 seconds: | 20260505 22:18:47.188 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:18:47.188 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:47.188 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:47.188 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:47.188 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260505 22:18:47.188 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:47.188 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260505 22:18:47.188 | INFO | PRE_HQ : 59.9%, 1.8, 55.1ms, 0 | 20260505 22:18:47.188 | INFO | BAZWRITER : 62.5%, 3.6, 116.5ms, 0 | 20260505 22:18:47.188 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 22:18:47.189 | INFO | Uploaded 36GiB to GPU in 2.17556s (16.5475GiB/s) | 20260505 22:18:47.189 | INFO | Downloaded 38GiB to GPU in 2.25379s (16.8605GiB/s) | 20260505 22:18:52.327 | INFO | Analyzing chunk frames = [6738432,6738944) | 20260505 22:18:52.327 | INFO | MemInfo: { | 20260505 22:18:52.327 | INFO | "gpu" : " 17 GB", | 20260505 22:18:52.327 | INFO | "huge" : "", | 20260505 22:18:52.328 | INFO | "huge_shared" : " 27 GB", | 20260505 22:18:52.328 | INFO | "malloc" : " 14 GB", | 20260505 22:18:52.328 | INFO | "pinned" : " 29 GB" | 20260505 22:18:52.328 | INFO | } | 20260505 22:18:52.328 | INFO | Perf report for the last 5.13929 seconds: | 20260505 22:18:52.328 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:18:52.328 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:52.328 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:52.328 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:52.328 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 22:18:52.328 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:52.328 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:52.328 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 22:18:52.328 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:52.328 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:52.328 | INFO | Uploaded 36GiB to GPU in 2.10306s (17.1179GiB/s) | 20260505 22:18:52.328 | INFO | Downloaded 34GiB to GPU in 2.02653s (16.7774GiB/s) | 20260505 22:18:57.470 | INFO | Analyzing chunk frames = [6738944,6739456) | 20260505 22:18:57.470 | INFO | MemInfo: { | 20260505 22:18:57.470 | INFO | "gpu" : " 17 GB", | 20260505 22:18:57.470 | INFO | "huge" : "", | 20260505 22:18:57.470 | INFO | "huge_shared" : " 27 GB", | 20260505 22:18:57.470 | INFO | "malloc" : " 14 GB", | 20260505 22:18:57.470 | INFO | "pinned" : " 29 GB" | 20260505 22:18:57.470 | INFO | } | 20260505 22:18:57.470 | INFO | Perf report for the last 5.14229 seconds: | 20260505 22:18:57.470 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:18:57.470 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:57.470 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:57.470 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:57.470 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260505 22:18:57.470 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:57.470 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:57.470 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 22:18:57.470 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:57.470 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:18:57.470 | INFO | Uploaded 36GiB to GPU in 2.14056s (16.818GiB/s) | 20260505 22:18:57.470 | INFO | Downloaded 34GiB to GPU in 2.03563s (16.7024GiB/s) | 20260505 22:19:02.609 | INFO | Analyzing chunk frames = [6739456,6739968) | 20260505 22:19:02.609 | INFO | MemInfo: { | 20260505 22:19:02.609 | INFO | "gpu" : " 17 GB", | 20260505 22:19:02.609 | INFO | "huge" : "", | 20260505 22:19:02.609 | INFO | "huge_shared" : " 27 GB", | 20260505 22:19:02.609 | INFO | "malloc" : " 14 GB", | 20260505 22:19:02.609 | INFO | "pinned" : " 29 GB" | 20260505 22:19:02.609 | INFO | } | 20260505 22:19:02.609 | INFO | Perf report for the last 5.13913 seconds: | 20260505 22:19:02.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:19:02.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:02.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:02.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:02.609 | INFO | ANALYSIS : 61.9%, 3.0, 96.1ms, 0 | 20260505 22:19:02.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:02.609 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:02.609 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 22:19:02.609 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:02.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:02.609 | INFO | Uploaded 36GiB to GPU in 2.14141s (16.8113GiB/s) | 20260505 22:19:02.609 | INFO | Downloaded 34GiB to GPU in 2.01631s (16.8625GiB/s) | 20260505 22:19:07.745 | INFO | Analyzing chunk frames = [6739968,6740480) | 20260505 22:19:07.745 | INFO | MemInfo: { | 20260505 22:19:07.745 | INFO | "gpu" : " 17 GB", | 20260505 22:19:07.745 | INFO | "huge" : "", | 20260505 22:19:07.745 | INFO | "huge_shared" : " 27 GB", | 20260505 22:19:07.745 | INFO | "malloc" : " 14 GB", | 20260505 22:19:07.746 | INFO | "pinned" : " 29 GB" | 20260505 22:19:07.746 | INFO | } | 20260505 22:19:07.746 | INFO | Perf report for the last 5.1365 seconds: | 20260505 22:19:07.746 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:19:07.746 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:07.746 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:07.746 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:07.746 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260505 22:19:07.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:07.746 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:07.746 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 22:19:07.746 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:07.746 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:07.746 | INFO | Uploaded 36GiB to GPU in 2.1552s (16.7037GiB/s) | 20260505 22:19:07.746 | INFO | Downloaded 34GiB to GPU in 2.06606s (16.4564GiB/s) | 20260505 22:19:12.880 | INFO | Analyzing chunk frames = [6740480,6740992) | 20260505 22:19:12.880 | INFO | MemInfo: { | 20260505 22:19:12.880 | INFO | "gpu" : " 17 GB", | 20260505 22:19:12.880 | INFO | "huge" : "", | 20260505 22:19:12.880 | INFO | "huge_shared" : " 27 GB", | 20260505 22:19:12.880 | INFO | "malloc" : " 14 GB", | 20260505 22:19:12.880 | INFO | "pinned" : " 29 GB" | 20260505 22:19:12.880 | INFO | } | 20260505 22:19:12.880 | INFO | Perf report for the last 5.13418 seconds: | 20260505 22:19:12.880 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:19:12.880 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:12.880 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:12.880 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:12.880 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260505 22:19:12.880 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:12.880 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:12.880 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:19:12.880 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:12.880 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:12.880 | INFO | Uploaded 36GiB to GPU in 2.08333s (17.2801GiB/s) | 20260505 22:19:12.880 | INFO | Downloaded 34GiB to GPU in 2.03873s (16.677GiB/s) | 20260505 22:19:18.030 | INFO | Analyzing chunk frames = [6740992,6741504) | 20260505 22:19:18.030 | INFO | MemInfo: { | 20260505 22:19:18.030 | INFO | "gpu" : " 17 GB", | 20260505 22:19:18.030 | INFO | "huge" : "", | 20260505 22:19:18.030 | INFO | "huge_shared" : " 27 GB", | 20260505 22:19:18.031 | INFO | "malloc" : " 14 GB", | 20260505 22:19:18.031 | INFO | "pinned" : " 29 GB" | 20260505 22:19:18.031 | INFO | } | 20260505 22:19:18.031 | INFO | Perf report for the last 5.15069 seconds: | 20260505 22:19:18.031 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:19:18.031 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:18.031 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:18.031 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:18.031 | INFO | ANALYSIS : 62.7%, 3.0, 98.1ms, 0 | 20260505 22:19:18.031 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:18.031 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:18.031 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 22:19:18.031 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:18.031 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:18.031 | INFO | Uploaded 36GiB to GPU in 2.14348s (16.7951GiB/s) | 20260505 22:19:18.031 | INFO | Downloaded 34GiB to GPU in 2.15582s (15.7713GiB/s) | 20260505 22:19:23.173 | INFO | Analyzing chunk frames = [6741504,6742016) | 20260505 22:19:23.173 | INFO | MemInfo: { | 20260505 22:19:23.173 | INFO | "gpu" : " 17 GB", | 20260505 22:19:23.173 | INFO | "huge" : "", | 20260505 22:19:23.173 | INFO | "huge_shared" : " 27 GB", | 20260505 22:19:23.173 | INFO | "malloc" : " 14 GB", | 20260505 22:19:23.173 | INFO | "pinned" : " 29 GB" | 20260505 22:19:23.173 | INFO | } | 20260505 22:19:23.173 | INFO | Perf report for the last 5.14289 seconds: | 20260505 22:19:23.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:19:23.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:23.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:23.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:23.173 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 22:19:23.174 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:23.174 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:23.174 | INFO | PRE_HQ : 27.2%, 1.0, 14.2ms, 0 | 20260505 22:19:23.174 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:23.174 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:23.174 | INFO | Uploaded 36GiB to GPU in 2.11166s (17.0482GiB/s) | 20260505 22:19:23.174 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 22:19:23.174 | INFO | Total mem usage: 15.3 GiB | 20260505 22:19:23.174 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 22:19:23.174 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -9.04%) | 20260505 22:19:23.174 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 22:19:23.174 | INFO | Downloaded 34GiB to GPU in 2.00795s (16.9327GiB/s) | 20260505 22:19:23.174 | INFO | High Water Breakdown: | 20260505 22:19:23.174 | INFO | 60.9% : 8.56 GiB ( 90% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 22:19:23.174 | INFO | 39% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 22:19:23.174 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 22:19:23.174 | INFO | ------------------------------------------ | 20260505 22:19:23.174 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:19:23.174 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 22:19:23.174 | INFO | Total mem usage: 26.5 GiB | 20260505 22:19:23.174 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 22:19:23.174 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 22:19:23.174 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 22:19:23.174 | INFO | High Water Breakdown: | 20260505 22:19:23.174 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:19:23.174 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:19:23.174 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:19:23.174 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:19:23.174 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:19:23.174 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:19:23.174 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 22:19:23.174 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 22:19:23.174 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:19:23.174 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 22:19:23.174 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 22:19:23.174 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 22:19:23.174 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 22:19:23.174 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 22:19:23.174 | INFO | ------------------------------------------ | 20260505 22:19:23.174 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:19:23.174 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 22:19:23.174 | INFO | Total mem usage: 25.6 GiB | 20260505 22:19:23.174 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 22:19:23.174 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 22:19:23.174 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 22:19:23.174 | INFO | High Water Breakdown: | 20260505 22:19:23.174 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 22:19:23.174 | INFO | ------------------------------------------ | 20260505 22:19:23.174 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:19:23.174 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 22:19:23.174 | INFO | Total mem usage: 15.7 GiB | 20260505 22:19:23.174 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 22:19:23.174 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 22:19:23.174 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 22:19:23.174 | INFO | High Water Breakdown: | 20260505 22:19:23.174 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 22:19:23.174 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 22:19:23.174 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 22:19:23.174 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 22:19:23.174 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 22:19:23.174 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 22:19:23.174 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 22:19:23.174 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 22:19:23.174 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 22:19:23.174 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:19:23.174 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 22:19:23.174 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:19:23.174 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 22:19:23.174 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 22:19:23.174 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 22:19:23.174 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 22:19:23.174 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:19:23.174 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 22:19:23.174 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:19:23.174 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:19:23.174 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:19:23.174 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 22:19:23.174 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 22:19:23.174 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:19:23.174 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 22:19:23.175 | INFO | ------------------------------------------ | 20260505 22:19:23.175 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:19:23.175 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 22:19:23.175 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:19:23.175 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:19:23.175 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.43e+01, 1.36e+01 / 1.11e+00 | 20260505 22:19:23.175 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.04e-01, 1.08e-02 / 3.95e-03 | 20260505 22:19:23.175 | INFO | PerfReport: 11.82%, Baseline -- 3.71e+00 / 4.54e+00, 5.71e-01 / 1.82e-01 | 20260505 22:19:23.175 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.53e+00, 1.91e-01 / 8.51e-02 | 20260505 22:19:23.175 | INFO | PerfReport: 15.97%, DME -- 4.01e+01 / 4.42e+01, 4.02e+00 / 1.17e+00 | 20260505 22:19:23.175 | INFO | PerfReport: 57.21%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.49e+00 / 1.07e+00 | 20260505 22:19:23.175 | INFO | PerfReport: 7.16%, PulseAccumulating -- 2.25e+00 / 2.79e+00, 4.30e-01 / 2.31e-01 | 20260505 22:19:23.175 | INFO | PerfReport: 3.16%, Metrics -- 9.94e-01 / 1.32e+00, 2.71e-01 / 1.47e-01 | 20260505 22:19:23.175 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.45e-01 / 4.08e-01, 5.96e-02 / 3.38e-02 | 20260505 22:19:23.175 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 22:19:23.175 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:19:23.175 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:19:23.175 | INFO | PerfReport: Full Execution -- 9.89e+01 / -nan, 2.51e+01 / 0.00e+00 | 20260505 22:19:23.175 | INFO | PerfReport: 22.39%, Upload -- 2.21e+01 / 0.00e+00, 7.67e+00 / 0.00e+00 | 20260505 22:19:23.175 | INFO | PerfReport: 21.80%, Download -- 2.16e+01 / 0.00e+00, 7.68e+00 / 0.00e+00 | 20260505 22:19:23.175 | INFO | PerfReport: 31.76%, Compute -- 3.14e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 22:19:23.175 | INFO | PerfReport: 2.23%, LockUpload -- 2.20e+00 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260505 22:19:23.175 | INFO | PerfReport: 1.32%, LockDownload -- 1.30e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260505 22:19:23.175 | INFO | PerfReport: 20.49%, LockCompute -- 2.03e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 22:19:23.175 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.62e-02 / 0.00e+00 | 20260505 22:19:23.291 | INFO | Starting a round of Baz IO | 20260505 22:19:26.444 | INFO | Populating RTMetrics report | 20260505 22:19:26.445 | INFO | RTMetrics report populated | 20260505 22:19:26.445 | INFO | Opening tmp RTMetrics json file | 20260505 22:19:26.446 | INFO | Serializing RTMetrics to json | 20260505 22:19:26.447 | INFO | Serialization done, swapping tmp file now... | 20260505 22:19:26.447 | INFO | RTMetrics done | 20260505 22:19:26.529 | INFO | Finished a round of Baz IO: Wrote 1.935 GiB in 3.237 seconds | 20260505 22:19:26.529 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.51% / 0.00% / 78.49% | 20260505 22:19:26.529 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.127053% / 0% / 7.94729e-05% | 20260505 22:19:26.529 | INFO | Baz IO overall: | 20260505 22:19:26.529 | INFO | BytesWritten : 3884382968487 | 20260505 22:19:26.529 | INFO | NumEvents : 1832422898576 | 20260505 22:19:28.310 | INFO | Analyzing chunk frames = [6742016,6742528) | 20260505 22:19:28.310 | INFO | MemInfo: { | 20260505 22:19:28.310 | INFO | "gpu" : " 17 GB", | 20260505 22:19:28.310 | INFO | "huge" : "", | 20260505 22:19:28.310 | INFO | "huge_shared" : " 27 GB", | 20260505 22:19:28.310 | INFO | "malloc" : " 14 GB", | 20260505 22:19:28.310 | INFO | "pinned" : " 29 GB" | 20260505 22:19:28.310 | INFO | } | 20260505 22:19:28.310 | INFO | Perf report for the last 5.1365 seconds: | 20260505 22:19:28.310 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:19:28.310 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:28.310 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:28.310 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:28.310 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 22:19:28.310 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:28.310 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260505 22:19:28.310 | INFO | PRE_HQ : 60.1%, 1.8, 55.1ms, 0 | 20260505 22:19:28.310 | INFO | BAZWRITER : 62.2%, 3.7, 116.9ms, 0 | 20260505 22:19:28.310 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 22:19:28.310 | INFO | Uploaded 36GiB to GPU in 2.13507s (16.8612GiB/s) | 20260505 22:19:28.310 | INFO | Downloaded 38GiB to GPU in 2.23623s (16.9929GiB/s) | 20260505 22:19:33.444 | INFO | Analyzing chunk frames = [6742528,6743040) | 20260505 22:19:33.444 | INFO | MemInfo: { | 20260505 22:19:33.444 | INFO | "gpu" : " 17 GB", | 20260505 22:19:33.444 | INFO | "huge" : "", | 20260505 22:19:33.444 | INFO | "huge_shared" : " 27 GB", | 20260505 22:19:33.444 | INFO | "malloc" : " 14 GB", | 20260505 22:19:33.444 | INFO | "pinned" : " 29 GB" | 20260505 22:19:33.444 | INFO | } | 20260505 22:19:33.444 | INFO | Perf report for the last 5.13452 seconds: | 20260505 22:19:33.444 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:19:33.444 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:33.444 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:33.445 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:33.445 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260505 22:19:33.445 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:33.445 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:33.445 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:19:33.445 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:33.445 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:33.445 | INFO | Uploaded 36GiB to GPU in 2.0914s (17.2133GiB/s) | 20260505 22:19:33.445 | INFO | Downloaded 34GiB to GPU in 2.03075s (16.7426GiB/s) | 20260505 22:19:38.580 | INFO | Analyzing chunk frames = [6743040,6743552) | 20260505 22:19:38.580 | INFO | MemInfo: { | 20260505 22:19:38.580 | INFO | "gpu" : " 17 GB", | 20260505 22:19:38.580 | INFO | "huge" : "", | 20260505 22:19:38.580 | INFO | "huge_shared" : " 27 GB", | 20260505 22:19:38.580 | INFO | "malloc" : " 14 GB", | 20260505 22:19:38.580 | INFO | "pinned" : " 29 GB" | 20260505 22:19:38.580 | INFO | } | 20260505 22:19:38.581 | INFO | Perf report for the last 5.13602 seconds: | 20260505 22:19:38.581 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:19:38.581 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:38.581 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:38.581 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:38.581 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260505 22:19:38.581 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:38.581 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:38.581 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:19:38.581 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:38.581 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:38.581 | INFO | Uploaded 36GiB to GPU in 2.13351s (16.8736GiB/s) | 20260505 22:19:38.581 | INFO | Downloaded 34GiB to GPU in 2.02587s (16.783GiB/s) | 20260505 22:19:43.724 | INFO | Analyzing chunk frames = [6743552,6744064) | 20260505 22:19:43.724 | INFO | MemInfo: { | 20260505 22:19:43.724 | INFO | "gpu" : " 17 GB", | 20260505 22:19:43.724 | INFO | "huge" : "", | 20260505 22:19:43.724 | INFO | "huge_shared" : " 27 GB", | 20260505 22:19:43.724 | INFO | "malloc" : " 14 GB", | 20260505 22:19:43.724 | INFO | "pinned" : " 29 GB" | 20260505 22:19:43.724 | INFO | } | 20260505 22:19:43.724 | INFO | Perf report for the last 5.14354 seconds: | 20260505 22:19:43.724 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:19:43.724 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:43.724 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:43.724 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:43.724 | INFO | ANALYSIS : 61.9%, 3.0, 96.2ms, 0 | 20260505 22:19:43.724 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:43.724 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:43.724 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:19:43.724 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:43.724 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:43.724 | INFO | Uploaded 36GiB to GPU in 2.13507s (16.8613GiB/s) | 20260505 22:19:43.724 | INFO | Downloaded 34GiB to GPU in 2.03887s (16.6759GiB/s) | 20260505 22:19:48.872 | INFO | Analyzing chunk frames = [6744064,6744576) | 20260505 22:19:48.872 | INFO | MemInfo: { | 20260505 22:19:48.872 | INFO | "gpu" : " 17 GB", | 20260505 22:19:48.872 | INFO | "huge" : "", | 20260505 22:19:48.872 | INFO | "huge_shared" : " 27 GB", | 20260505 22:19:48.872 | INFO | "malloc" : " 14 GB", | 20260505 22:19:48.872 | INFO | "pinned" : " 29 GB" | 20260505 22:19:48.872 | INFO | } | 20260505 22:19:48.872 | INFO | Perf report for the last 5.14836 seconds: | 20260505 22:19:48.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:19:48.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:48.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:48.872 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:48.872 | INFO | ANALYSIS : 61.2%, 3.0, 95.7ms, 0 | 20260505 22:19:48.872 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:48.873 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:48.873 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:19:48.873 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:48.873 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:48.873 | INFO | Uploaded 36GiB to GPU in 2.09369s (17.1945GiB/s) | 20260505 22:19:48.873 | INFO | Downloaded 34GiB to GPU in 2.05072s (16.5795GiB/s) | 20260505 22:19:54.002 | INFO | Analyzing chunk frames = [6744576,6745088) | 20260505 22:19:54.002 | INFO | MemInfo: { | 20260505 22:19:54.002 | INFO | "gpu" : " 17 GB", | 20260505 22:19:54.002 | INFO | "huge" : "", | 20260505 22:19:54.002 | INFO | "huge_shared" : " 27 GB", | 20260505 22:19:54.002 | INFO | "malloc" : " 14 GB", | 20260505 22:19:54.002 | INFO | "pinned" : " 29 GB" | 20260505 22:19:54.002 | INFO | } | 20260505 22:19:54.002 | INFO | Perf report for the last 5.12922 seconds: | 20260505 22:19:54.002 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:19:54.002 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:54.002 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:54.002 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:54.002 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260505 22:19:54.002 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:54.002 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:54.002 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:19:54.002 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:54.002 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:54.002 | INFO | Uploaded 36GiB to GPU in 2.07109s (17.3821GiB/s) | 20260505 22:19:54.002 | INFO | Downloaded 34GiB to GPU in 2.03345s (16.7203GiB/s) | 20260505 22:19:59.137 | INFO | Analyzing chunk frames = [6745088,6745600) | 20260505 22:19:59.137 | INFO | MemInfo: { | 20260505 22:19:59.137 | INFO | "gpu" : " 17 GB", | 20260505 22:19:59.137 | INFO | "huge" : "", | 20260505 22:19:59.137 | INFO | "huge_shared" : " 27 GB", | 20260505 22:19:59.137 | INFO | "malloc" : " 14 GB", | 20260505 22:19:59.137 | INFO | "pinned" : " 29 GB" | 20260505 22:19:59.137 | INFO | } | 20260505 22:19:59.137 | INFO | Perf report for the last 5.13529 seconds: | 20260505 22:19:59.137 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:19:59.137 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:59.137 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:59.137 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:59.137 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 22:19:59.137 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:59.137 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:59.137 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 22:19:59.137 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:59.137 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:19:59.137 | INFO | Uploaded 36GiB to GPU in 2.14962s (16.7472GiB/s) | 20260505 22:19:59.137 | INFO | Downloaded 34GiB to GPU in 2.14332s (15.8633GiB/s) | 20260505 22:20:04.283 | INFO | Analyzing chunk frames = [6745600,6746112) | 20260505 22:20:04.283 | INFO | MemInfo: { | 20260505 22:20:04.283 | INFO | "gpu" : " 17 GB", | 20260505 22:20:04.283 | INFO | "huge" : "", | 20260505 22:20:04.283 | INFO | "huge_shared" : " 27 GB", | 20260505 22:20:04.283 | INFO | "malloc" : " 14 GB", | 20260505 22:20:04.283 | INFO | "pinned" : " 29 GB" | 20260505 22:20:04.283 | INFO | } | 20260505 22:20:04.283 | INFO | Perf report for the last 5.1461 seconds: | 20260505 22:20:04.283 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:20:04.283 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:04.283 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:04.283 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:04.283 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 22:20:04.283 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:04.283 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:04.283 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:20:04.283 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:04.283 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:04.283 | INFO | Uploaded 36GiB to GPU in 2.10404s (17.1099GiB/s) | 20260505 22:20:04.283 | INFO | Downloaded 34GiB to GPU in 1.99715s (17.0243GiB/s) | 20260505 22:20:04.408 | INFO | Starting a round of Baz IO | 20260505 22:20:07.557 | INFO | Populating RTMetrics report | 20260505 22:20:07.557 | INFO | RTMetrics report populated | 20260505 22:20:07.557 | INFO | Opening tmp RTMetrics json file | 20260505 22:20:07.560 | INFO | Serializing RTMetrics to json | 20260505 22:20:07.561 | INFO | Serialization done, swapping tmp file now... | 20260505 22:20:07.563 | INFO | RTMetrics done | 20260505 22:20:07.649 | INFO | Finished a round of Baz IO: Wrote 1.936 GiB in 3.241 seconds | 20260505 22:20:07.649 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.51% / 0.00% / 78.49% | 20260505 22:20:07.649 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.12691% / 0% / 0.000115236% | 20260505 22:20:07.649 | INFO | Baz IO overall: | 20260505 22:20:07.649 | INFO | BytesWritten : 3886461346110 | 20260505 22:20:07.649 | INFO | NumEvents : 1833379723091 | 20260505 22:20:09.422 | INFO | Analyzing chunk frames = [6746112,6746624) | 20260505 22:20:09.422 | INFO | MemInfo: { | 20260505 22:20:09.422 | INFO | "gpu" : " 17 GB", | 20260505 22:20:09.422 | INFO | "huge" : "", | 20260505 22:20:09.422 | INFO | "huge_shared" : " 27 GB", | 20260505 22:20:09.422 | INFO | "malloc" : " 14 GB", | 20260505 22:20:09.422 | INFO | "pinned" : " 29 GB" | 20260505 22:20:09.422 | INFO | } | 20260505 22:20:09.422 | INFO | Perf report for the last 5.13936 seconds: | 20260505 22:20:09.422 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:20:09.422 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:09.423 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:09.423 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:09.423 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260505 22:20:09.423 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:09.423 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260505 22:20:09.423 | INFO | PRE_HQ : 60.0%, 1.8, 54.7ms, 0 | 20260505 22:20:09.423 | INFO | BAZWRITER : 62.3%, 3.6, 116.5ms, 0 | 20260505 22:20:09.423 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 22:20:09.423 | INFO | Uploaded 36GiB to GPU in 2.13405s (16.8693GiB/s) | 20260505 22:20:09.423 | INFO | Downloaded 38GiB to GPU in 2.22896s (17.0483GiB/s) | 20260505 22:20:14.556 | INFO | Analyzing chunk frames = [6746624,6747136) | 20260505 22:20:14.556 | INFO | MemInfo: { | 20260505 22:20:14.556 | INFO | "gpu" : " 17 GB", | 20260505 22:20:14.556 | INFO | "huge" : "", | 20260505 22:20:14.556 | INFO | "huge_shared" : " 27 GB", | 20260505 22:20:14.556 | INFO | "malloc" : " 14 GB", | 20260505 22:20:14.556 | INFO | "pinned" : " 29 GB" | 20260505 22:20:14.556 | INFO | } | 20260505 22:20:14.556 | INFO | Perf report for the last 5.13351 seconds: | 20260505 22:20:14.556 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:20:14.556 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:14.556 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:14.556 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:14.556 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 22:20:14.556 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:14.556 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:14.556 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:20:14.556 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:14.556 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:14.556 | INFO | Uploaded 36GiB to GPU in 2.09905s (17.1506GiB/s) | 20260505 22:20:14.556 | INFO | Downloaded 34GiB to GPU in 2.03686s (16.6924GiB/s) | 20260505 22:20:19.692 | INFO | Analyzing chunk frames = [6747136,6747648) | 20260505 22:20:19.692 | INFO | MemInfo: { | 20260505 22:20:19.692 | INFO | "gpu" : " 17 GB", | 20260505 22:20:19.692 | INFO | "huge" : "", | 20260505 22:20:19.692 | INFO | "huge_shared" : " 27 GB", | 20260505 22:20:19.692 | INFO | "malloc" : " 14 GB", | 20260505 22:20:19.692 | INFO | "pinned" : " 29 GB" | 20260505 22:20:19.692 | INFO | } | 20260505 22:20:19.692 | INFO | Perf report for the last 5.1361 seconds: | 20260505 22:20:19.692 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:20:19.692 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:19.692 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:19.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:19.692 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260505 22:20:19.692 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:19.692 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:19.692 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 22:20:19.692 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:19.692 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:19.692 | INFO | Uploaded 36GiB to GPU in 2.13445s (16.8662GiB/s) | 20260505 22:20:19.692 | INFO | Downloaded 34GiB to GPU in 2.01816s (16.8471GiB/s) | 20260505 22:20:24.835 | INFO | Analyzing chunk frames = [6747648,6748160) | 20260505 22:20:24.835 | INFO | MemInfo: { | 20260505 22:20:24.835 | INFO | "gpu" : " 17 GB", | 20260505 22:20:24.835 | INFO | "huge" : "", | 20260505 22:20:24.835 | INFO | "huge_shared" : " 27 GB", | 20260505 22:20:24.835 | INFO | "malloc" : " 14 GB", | 20260505 22:20:24.835 | INFO | "pinned" : " 29 GB" | 20260505 22:20:24.835 | INFO | } | 20260505 22:20:24.835 | INFO | Perf report for the last 5.14298 seconds: | 20260505 22:20:24.835 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:20:24.835 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:24.835 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:24.835 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:24.835 | INFO | ANALYSIS : 62.2%, 3.0, 96.6ms, 0 | 20260505 22:20:24.835 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:24.835 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:24.835 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:20:24.835 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:24.835 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:24.835 | INFO | Uploaded 36GiB to GPU in 2.20456s (16.3298GiB/s) | 20260505 22:20:24.835 | INFO | Downloaded 34GiB to GPU in 2.0511s (16.5765GiB/s) | 20260505 22:20:29.979 | INFO | Analyzing chunk frames = [6748160,6748672) | 20260505 22:20:29.979 | INFO | MemInfo: { | 20260505 22:20:29.979 | INFO | "gpu" : " 17 GB", | 20260505 22:20:29.979 | INFO | "huge" : "", | 20260505 22:20:29.979 | INFO | "huge_shared" : " 27 GB", | 20260505 22:20:29.979 | INFO | "malloc" : " 14 GB", | 20260505 22:20:29.979 | INFO | "pinned" : " 29 GB" | 20260505 22:20:29.979 | INFO | } | 20260505 22:20:29.979 | INFO | Perf report for the last 5.14429 seconds: | 20260505 22:20:29.979 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:20:29.979 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:29.979 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:29.979 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:29.979 | INFO | ANALYSIS : 61.6%, 3.0, 96.2ms, 0 | 20260505 22:20:29.979 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:29.979 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:29.979 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:20:29.979 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:29.979 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:29.980 | INFO | Uploaded 36GiB to GPU in 2.13028s (16.8992GiB/s) | 20260505 22:20:29.980 | INFO | Downloaded 34GiB to GPU in 2.06392s (16.4735GiB/s) | 20260505 22:20:35.125 | INFO | Analyzing chunk frames = [6748672,6749184) | 20260505 22:20:35.125 | INFO | MemInfo: { | 20260505 22:20:35.125 | INFO | "gpu" : " 17 GB", | 20260505 22:20:35.125 | INFO | "huge" : "", | 20260505 22:20:35.125 | INFO | "huge_shared" : " 27 GB", | 20260505 22:20:35.125 | INFO | "malloc" : " 14 GB", | 20260505 22:20:35.125 | INFO | "pinned" : " 29 GB" | 20260505 22:20:35.125 | INFO | } | 20260505 22:20:35.125 | INFO | Perf report for the last 5.14593 seconds: | 20260505 22:20:35.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:20:35.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:35.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:35.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:35.125 | INFO | ANALYSIS : 61.3%, 3.0, 95.8ms, 0 | 20260505 22:20:35.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:35.125 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:35.125 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 22:20:35.125 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:35.125 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:35.126 | INFO | Uploaded 36GiB to GPU in 2.07502s (17.3492GiB/s) | 20260505 22:20:35.126 | INFO | Downloaded 34GiB to GPU in 2.04142s (16.6551GiB/s) | 20260505 22:20:40.256 | INFO | Analyzing chunk frames = [6749184,6749696) | 20260505 22:20:40.256 | INFO | MemInfo: { | 20260505 22:20:40.256 | INFO | "gpu" : " 17 GB", | 20260505 22:20:40.256 | INFO | "huge" : "", | 20260505 22:20:40.256 | INFO | "huge_shared" : " 27 GB", | 20260505 22:20:40.256 | INFO | "malloc" : " 14 GB", | 20260505 22:20:40.256 | INFO | "pinned" : " 29 GB" | 20260505 22:20:40.256 | INFO | } | 20260505 22:20:40.256 | INFO | Perf report for the last 5.1309 seconds: | 20260505 22:20:40.256 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:20:40.256 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:40.256 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:40.256 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:40.256 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 22:20:40.256 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:40.256 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:40.256 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 22:20:40.256 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:40.256 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:40.256 | INFO | Uploaded 36GiB to GPU in 2.01249s (17.8883GiB/s) | 20260505 22:20:40.257 | INFO | Downloaded 34GiB to GPU in 2.04068s (16.6611GiB/s) | 20260505 22:20:45.392 | INFO | Analyzing chunk frames = [6749696,6750208) | 20260505 22:20:45.392 | INFO | MemInfo: { | 20260505 22:20:45.392 | INFO | "gpu" : " 17 GB", | 20260505 22:20:45.392 | INFO | "huge" : "", | 20260505 22:20:45.392 | INFO | "huge_shared" : " 27 GB", | 20260505 22:20:45.392 | INFO | "malloc" : " 14 GB", | 20260505 22:20:45.392 | INFO | "pinned" : " 29 GB" | 20260505 22:20:45.392 | INFO | } | 20260505 22:20:45.392 | INFO | Perf report for the last 5.13595 seconds: | 20260505 22:20:45.392 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:20:45.392 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:45.392 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:45.392 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:45.392 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 22:20:45.392 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:45.392 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:45.392 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:20:45.392 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:45.392 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:45.393 | INFO | Uploaded 36GiB to GPU in 2.12359s (16.9524GiB/s) | 20260505 22:20:45.393 | INFO | Downloaded 34GiB to GPU in 2.0106s (16.9103GiB/s) | 20260505 22:20:45.393 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 22:20:45.393 | INFO | Total mem usage: 15.3 GiB | 20260505 22:20:45.393 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 22:20:45.393 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.39%) | 20260505 22:20:45.393 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 22:20:45.393 | INFO | High Water Breakdown: | 20260505 22:20:45.393 | INFO | 61.1% : 8.64 GiB ( 90.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 22:20:45.393 | INFO | 38.7% : 5.48 GiB ( 82.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 22:20:45.393 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 22:20:45.393 | INFO | ------------------------------------------ | 20260505 22:20:45.393 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:20:45.393 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 22:20:45.393 | INFO | Total mem usage: 26.5 GiB | 20260505 22:20:45.393 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 22:20:45.393 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 22:20:45.393 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 22:20:45.393 | INFO | High Water Breakdown: | 20260505 22:20:45.393 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:20:45.393 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:20:45.393 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:20:45.393 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:20:45.393 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:20:45.393 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:20:45.393 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 22:20:45.393 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 22:20:45.393 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:20:45.393 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 22:20:45.393 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 22:20:45.393 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 22:20:45.393 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 22:20:45.393 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 22:20:45.393 | INFO | ------------------------------------------ | 20260505 22:20:45.393 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:20:45.393 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 22:20:45.393 | INFO | Total mem usage: 25.6 GiB | 20260505 22:20:45.393 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 22:20:45.393 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 22:20:45.393 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 22:20:45.393 | INFO | High Water Breakdown: | 20260505 22:20:45.393 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 22:20:45.393 | INFO | ------------------------------------------ | 20260505 22:20:45.393 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:20:45.393 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 22:20:45.393 | INFO | Total mem usage: 15.7 GiB | 20260505 22:20:45.393 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 22:20:45.393 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 22:20:45.393 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 22:20:45.393 | INFO | High Water Breakdown: | 20260505 22:20:45.393 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 22:20:45.393 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 22:20:45.393 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 22:20:45.393 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 22:20:45.393 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 22:20:45.393 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 22:20:45.393 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 22:20:45.393 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 22:20:45.393 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 22:20:45.393 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:20:45.393 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 22:20:45.393 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:20:45.393 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 22:20:45.393 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 22:20:45.393 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 22:20:45.393 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 22:20:45.393 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:20:45.393 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 22:20:45.393 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:20:45.393 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:20:45.393 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:20:45.393 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 22:20:45.393 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 22:20:45.393 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:20:45.394 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 22:20:45.394 | INFO | ------------------------------------------ | 20260505 22:20:45.394 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:20:45.394 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 22:20:45.394 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:20:45.394 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:20:45.394 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.42e+01, 1.35e+01 / 1.56e+00 | 20260505 22:20:45.394 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.04e-01, 1.06e-02 / 4.10e-03 | 20260505 22:20:45.394 | INFO | PerfReport: 11.81%, Baseline -- 3.71e+00 / 4.52e+00, 5.72e-01 / 2.05e-01 | 20260505 22:20:45.394 | INFO | PerfReport: 3.29%, Histogram -- 1.03e+00 / 1.53e+00, 1.93e-01 / 1.02e-01 | 20260505 22:20:45.394 | INFO | PerfReport: 16.03%, DME -- 4.03e+01 / 4.43e+01, 4.07e+00 / 1.78e+00 | 20260505 22:20:45.394 | INFO | PerfReport: 57.15%, FrameLabeling -- 1.79e+01 / 1.92e+01, 1.47e+00 / 8.83e-01 | 20260505 22:20:45.394 | INFO | PerfReport: 7.17%, PulseAccumulating -- 2.25e+00 / 2.83e+00, 4.32e-01 / 2.00e-01 | 20260505 22:20:45.394 | INFO | PerfReport: 3.16%, Metrics -- 9.92e-01 / 1.28e+00, 2.69e-01 / 1.30e-01 | 20260505 22:20:45.394 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.45e-01 / 3.98e-01, 5.92e-02 / 2.80e-02 | 20260505 22:20:45.394 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 22:20:45.394 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:20:45.394 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:20:45.394 | INFO | PerfReport: Full Execution -- 9.87e+01 / -nan, 2.50e+01 / 0.00e+00 | 20260505 22:20:45.394 | INFO | PerfReport: 22.31%, Upload -- 2.20e+01 / 0.00e+00, 7.54e+00 / 0.00e+00 | 20260505 22:20:45.394 | INFO | PerfReport: 21.80%, Download -- 2.15e+01 / 0.00e+00, 7.65e+00 / 0.00e+00 | 20260505 22:20:45.394 | INFO | PerfReport: 31.81%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 22:20:45.394 | INFO | PerfReport: 2.15%, LockUpload -- 2.12e+00 / 0.00e+00, 7.76e+00 / 0.00e+00 | 20260505 22:20:45.394 | INFO | PerfReport: 1.35%, LockDownload -- 1.33e+00 / 0.00e+00, 2.93e+00 / 0.00e+00 | 20260505 22:20:45.394 | INFO | PerfReport: 20.57%, LockCompute -- 2.03e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 22:20:45.394 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.17e-02 / 0.00e+00 | 20260505 22:20:45.526 | INFO | Starting a round of Baz IO | 20260505 22:20:48.676 | INFO | Populating RTMetrics report | 20260505 22:20:48.677 | INFO | RTMetrics report populated | 20260505 22:20:48.677 | INFO | Opening tmp RTMetrics json file | 20260505 22:20:48.678 | INFO | Serializing RTMetrics to json | 20260505 22:20:48.680 | INFO | Serialization done, swapping tmp file now... | 20260505 22:20:48.683 | INFO | RTMetrics done | 20260505 22:20:48.759 | INFO | Finished a round of Baz IO: Wrote 1.933 GiB in 3.233 seconds | 20260505 22:20:48.759 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.50% / 0.00% / 78.49% | 20260505 22:20:48.759 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.126406% / 3.97364e-06% / 8.74201e-05% | 20260505 22:20:48.759 | INFO | Baz IO overall: | 20260505 22:20:48.759 | INFO | BytesWritten : 3888537335960 | 20260505 22:20:48.759 | INFO | NumEvents : 1834334834416 | 20260505 22:20:50.536 | INFO | Analyzing chunk frames = [6750208,6750720) | 20260505 22:20:50.536 | INFO | MemInfo: { | 20260505 22:20:50.536 | INFO | "gpu" : " 17 GB", | 20260505 22:20:50.536 | INFO | "huge" : "", | 20260505 22:20:50.536 | INFO | "huge_shared" : " 27 GB", | 20260505 22:20:50.536 | INFO | "malloc" : " 14 GB", | 20260505 22:20:50.536 | INFO | "pinned" : " 29 GB" | 20260505 22:20:50.536 | INFO | } | 20260505 22:20:50.536 | INFO | Perf report for the last 5.14386 seconds: | 20260505 22:20:50.536 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:20:50.536 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:50.536 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:50.536 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:50.536 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 22:20:50.536 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:50.536 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260505 22:20:50.536 | INFO | PRE_HQ : 59.8%, 1.7, 54.3ms, 0 | 20260505 22:20:50.536 | INFO | BAZWRITER : 62.4%, 3.6, 116.1ms, 0 | 20260505 22:20:50.537 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 22:20:50.537 | INFO | Uploaded 36GiB to GPU in 2.1712s (16.5807GiB/s) | 20260505 22:20:50.537 | INFO | Downloaded 38GiB to GPU in 2.2402s (16.9628GiB/s) | 20260505 22:20:55.675 | INFO | Analyzing chunk frames = [6750720,6751232) | 20260505 22:20:55.675 | INFO | MemInfo: { | 20260505 22:20:55.675 | INFO | "gpu" : " 17 GB", | 20260505 22:20:55.675 | INFO | "huge" : "", | 20260505 22:20:55.675 | INFO | "huge_shared" : " 27 GB", | 20260505 22:20:55.675 | INFO | "malloc" : " 14 GB", | 20260505 22:20:55.675 | INFO | "pinned" : " 29 GB" | 20260505 22:20:55.675 | INFO | } | 20260505 22:20:55.675 | INFO | Perf report for the last 5.13887 seconds: | 20260505 22:20:55.675 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:20:55.675 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:55.675 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:55.675 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:55.675 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260505 22:20:55.675 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:55.675 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:55.675 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260505 22:20:55.675 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:55.675 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:20:55.676 | INFO | Uploaded 36GiB to GPU in 2.08677s (17.2516GiB/s) | 20260505 22:20:55.676 | INFO | Downloaded 34GiB to GPU in 2.03078s (16.7423GiB/s) | 20260505 22:21:00.815 | INFO | Analyzing chunk frames = [6751232,6751744) | 20260505 22:21:00.815 | INFO | MemInfo: { | 20260505 22:21:00.815 | INFO | "gpu" : " 17 GB", | 20260505 22:21:00.815 | INFO | "huge" : "", | 20260505 22:21:00.815 | INFO | "huge_shared" : " 27 GB", | 20260505 22:21:00.815 | INFO | "malloc" : " 14 GB", | 20260505 22:21:00.815 | INFO | "pinned" : " 29 GB" | 20260505 22:21:00.815 | INFO | } | 20260505 22:21:00.815 | INFO | Perf report for the last 5.14036 seconds: | 20260505 22:21:00.815 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:21:00.815 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:00.815 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:00.815 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:00.815 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260505 22:21:00.815 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:00.815 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:00.815 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:21:00.815 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:00.816 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:00.816 | INFO | Uploaded 36GiB to GPU in 2.13883s (16.8316GiB/s) | 20260505 22:21:00.816 | INFO | Downloaded 34GiB to GPU in 2.02117s (16.8219GiB/s) | 20260505 22:21:05.951 | INFO | Analyzing chunk frames = [6751744,6752256) | 20260505 22:21:05.951 | INFO | MemInfo: { | 20260505 22:21:05.951 | INFO | "gpu" : " 17 GB", | 20260505 22:21:05.951 | INFO | "huge" : "", | 20260505 22:21:05.951 | INFO | "huge_shared" : " 27 GB", | 20260505 22:21:05.951 | INFO | "malloc" : " 14 GB", | 20260505 22:21:05.951 | INFO | "pinned" : " 29 GB" | 20260505 22:21:05.951 | INFO | } | 20260505 22:21:05.951 | INFO | Perf report for the last 5.13597 seconds: | 20260505 22:21:05.951 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:21:05.951 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:05.951 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:05.951 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:05.951 | INFO | ANALYSIS : 61.9%, 3.0, 96.0ms, 0 | 20260505 22:21:05.951 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:05.951 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:05.951 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 22:21:05.951 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:05.951 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:05.952 | INFO | Uploaded 36GiB to GPU in 2.13358s (16.8731GiB/s) | 20260505 22:21:05.952 | INFO | Downloaded 34GiB to GPU in 2.03318s (16.7226GiB/s) | 20260505 22:21:11.093 | INFO | Analyzing chunk frames = [6752256,6752768) | 20260505 22:21:11.093 | INFO | MemInfo: { | 20260505 22:21:11.093 | INFO | "gpu" : " 17 GB", | 20260505 22:21:11.093 | INFO | "huge" : "", | 20260505 22:21:11.093 | INFO | "huge_shared" : " 27 GB", | 20260505 22:21:11.093 | INFO | "malloc" : " 14 GB", | 20260505 22:21:11.093 | INFO | "pinned" : " 29 GB" | 20260505 22:21:11.093 | INFO | } | 20260505 22:21:11.094 | INFO | Perf report for the last 5.14215 seconds: | 20260505 22:21:11.094 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:21:11.094 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:11.094 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:11.094 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:11.094 | INFO | ANALYSIS : 61.5%, 3.0, 96.1ms, 0 | 20260505 22:21:11.094 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:11.094 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:11.094 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:21:11.094 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:11.094 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:11.094 | INFO | Uploaded 36GiB to GPU in 2.12908s (16.9087GiB/s) | 20260505 22:21:11.094 | INFO | Downloaded 34GiB to GPU in 2.06195s (16.4892GiB/s) | 20260505 22:21:16.232 | INFO | Analyzing chunk frames = [6752768,6753280) | 20260505 22:21:16.232 | INFO | MemInfo: { | 20260505 22:21:16.233 | INFO | "gpu" : " 17 GB", | 20260505 22:21:16.233 | INFO | "huge" : "", | 20260505 22:21:16.233 | INFO | "huge_shared" : " 27 GB", | 20260505 22:21:16.233 | INFO | "malloc" : " 14 GB", | 20260505 22:21:16.233 | INFO | "pinned" : " 29 GB" | 20260505 22:21:16.233 | INFO | } | 20260505 22:21:16.233 | INFO | Perf report for the last 5.13911 seconds: | 20260505 22:21:16.233 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:21:16.233 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:16.233 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:16.233 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:16.233 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260505 22:21:16.233 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:16.233 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:16.233 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 22:21:16.233 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:16.233 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:16.233 | INFO | Uploaded 36GiB to GPU in 2.08381s (17.2761GiB/s) | 20260505 22:21:16.233 | INFO | Downloaded 34GiB to GPU in 2.03628s (16.6971GiB/s) | 20260505 22:21:21.370 | INFO | Analyzing chunk frames = [6753280,6753792) | 20260505 22:21:21.370 | INFO | MemInfo: { | 20260505 22:21:21.370 | INFO | "gpu" : " 17 GB", | 20260505 22:21:21.370 | INFO | "huge" : "", | 20260505 22:21:21.370 | INFO | "huge_shared" : " 27 GB", | 20260505 22:21:21.370 | INFO | "malloc" : " 14 GB", | 20260505 22:21:21.370 | INFO | "pinned" : " 29 GB" | 20260505 22:21:21.370 | INFO | } | 20260505 22:21:21.370 | INFO | Perf report for the last 5.13772 seconds: | 20260505 22:21:21.370 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:21:21.370 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:21.370 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:21.370 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:21.370 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 22:21:21.370 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:21.370 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:21.370 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 22:21:21.370 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:21.370 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:21.371 | INFO | Uploaded 36GiB to GPU in 2.02067s (17.8159GiB/s) | 20260505 22:21:21.371 | INFO | Downloaded 34GiB to GPU in 2.03834s (16.6802GiB/s) | 20260505 22:21:26.505 | INFO | Analyzing chunk frames = [6753792,6754304) | 20260505 22:21:26.505 | INFO | MemInfo: { | 20260505 22:21:26.505 | INFO | "gpu" : " 17 GB", | 20260505 22:21:26.505 | INFO | "huge" : "", | 20260505 22:21:26.505 | INFO | "huge_shared" : " 27 GB", | 20260505 22:21:26.505 | INFO | "malloc" : " 14 GB", | 20260505 22:21:26.505 | INFO | "pinned" : " 29 GB" | 20260505 22:21:26.505 | INFO | } | 20260505 22:21:26.505 | INFO | Perf report for the last 5.13494 seconds: | 20260505 22:21:26.505 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:21:26.505 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:26.505 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:26.505 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:26.505 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260505 22:21:26.505 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:26.505 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:26.505 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:21:26.505 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:26.505 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:26.506 | INFO | Uploaded 36GiB to GPU in 2.11238s (17.0424GiB/s) | 20260505 22:21:26.506 | INFO | Downloaded 34GiB to GPU in 2.00383s (16.9675GiB/s) | 20260505 22:21:26.640 | INFO | Starting a round of Baz IO | 20260505 22:21:29.797 | INFO | Populating RTMetrics report | 20260505 22:21:29.797 | INFO | RTMetrics report populated | 20260505 22:21:29.797 | INFO | Opening tmp RTMetrics json file | 20260505 22:21:29.799 | INFO | Serializing RTMetrics to json | 20260505 22:21:29.802 | INFO | Serialization done, swapping tmp file now... | 20260505 22:21:29.802 | INFO | RTMetrics done | 20260505 22:21:29.884 | INFO | Finished a round of Baz IO: Wrote 1.936 GiB in 3.243 seconds | 20260505 22:21:29.884 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.50% / 0.00% / 78.50% | 20260505 22:21:29.884 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.126906% / 0% / 8.74201e-05% | 20260505 22:21:29.884 | INFO | Baz IO overall: | 20260505 22:21:29.884 | INFO | BytesWritten : 3890615991549 | 20260505 22:21:29.884 | INFO | NumEvents : 1835292042177 | 20260505 22:21:31.644 | INFO | Analyzing chunk frames = [6754304,6754816) | 20260505 22:21:31.644 | INFO | MemInfo: { | 20260505 22:21:31.644 | INFO | "gpu" : " 17 GB", | 20260505 22:21:31.644 | INFO | "huge" : "", | 20260505 22:21:31.644 | INFO | "huge_shared" : " 27 GB", | 20260505 22:21:31.644 | INFO | "malloc" : " 14 GB", | 20260505 22:21:31.644 | INFO | "pinned" : " 29 GB" | 20260505 22:21:31.644 | INFO | } | 20260505 22:21:31.644 | INFO | Perf report for the last 5.13833 seconds: | 20260505 22:21:31.644 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:21:31.644 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:31.644 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:31.644 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:31.644 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 22:21:31.644 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:31.644 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260505 22:21:31.644 | INFO | PRE_HQ : 59.9%, 1.8, 54.8ms, 0 | 20260505 22:21:31.644 | INFO | BAZWRITER : 62.6%, 3.6, 117.1ms, 0 | 20260505 22:21:31.644 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 22:21:31.644 | INFO | Uploaded 36GiB to GPU in 2.17871s (16.5235GiB/s) | 20260505 22:21:31.644 | INFO | Downloaded 38GiB to GPU in 2.25466s (16.854GiB/s) | 20260505 22:21:36.789 | INFO | Analyzing chunk frames = [6754816,6755328) | 20260505 22:21:36.789 | INFO | MemInfo: { | 20260505 22:21:36.789 | INFO | "gpu" : " 17 GB", | 20260505 22:21:36.789 | INFO | "huge" : "", | 20260505 22:21:36.789 | INFO | "huge_shared" : " 27 GB", | 20260505 22:21:36.789 | INFO | "malloc" : " 14 GB", | 20260505 22:21:36.789 | INFO | "pinned" : " 29 GB" | 20260505 22:21:36.789 | INFO | } | 20260505 22:21:36.789 | INFO | Perf report for the last 5.14577 seconds: | 20260505 22:21:36.789 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:21:36.789 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:36.789 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:36.789 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:36.789 | INFO | ANALYSIS : 62.1%, 3.0, 97.1ms, 0 | 20260505 22:21:36.789 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:36.789 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:36.790 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 22:21:36.790 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:36.790 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:36.790 | INFO | Uploaded 36GiB to GPU in 2.09947s (17.1472GiB/s) | 20260505 22:21:36.790 | INFO | Downloaded 34GiB to GPU in 2.03266s (16.7268GiB/s) | 20260505 22:21:41.938 | INFO | Analyzing chunk frames = [6755328,6755840) | 20260505 22:21:41.938 | INFO | MemInfo: { | 20260505 22:21:41.938 | INFO | "gpu" : " 17 GB", | 20260505 22:21:41.938 | INFO | "huge" : "", | 20260505 22:21:41.938 | INFO | "huge_shared" : " 27 GB", | 20260505 22:21:41.938 | INFO | "malloc" : " 14 GB", | 20260505 22:21:41.938 | INFO | "pinned" : " 29 GB" | 20260505 22:21:41.938 | INFO | } | 20260505 22:21:41.938 | INFO | Perf report for the last 5.14865 seconds: | 20260505 22:21:41.938 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:21:41.938 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:41.938 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:41.938 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:41.938 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260505 22:21:41.938 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:41.938 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:41.938 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:21:41.938 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:41.938 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:41.939 | INFO | Uploaded 36GiB to GPU in 2.14237s (16.8038GiB/s) | 20260505 22:21:41.939 | INFO | Downloaded 34GiB to GPU in 2.0557s (16.5394GiB/s) | 20260505 22:21:47.076 | INFO | Analyzing chunk frames = [6755840,6756352) | 20260505 22:21:47.076 | INFO | MemInfo: { | 20260505 22:21:47.076 | INFO | "gpu" : " 17 GB", | 20260505 22:21:47.076 | INFO | "huge" : "", | 20260505 22:21:47.076 | INFO | "huge_shared" : " 27 GB", | 20260505 22:21:47.076 | INFO | "malloc" : " 14 GB", | 20260505 22:21:47.076 | INFO | "pinned" : " 29 GB" | 20260505 22:21:47.076 | INFO | } | 20260505 22:21:47.076 | INFO | Perf report for the last 5.13823 seconds: | 20260505 22:21:47.076 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:21:47.076 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:47.076 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:47.076 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:47.076 | INFO | ANALYSIS : 61.8%, 3.0, 95.9ms, 0 | 20260505 22:21:47.076 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:47.076 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:47.076 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:21:47.076 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:47.076 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:47.077 | INFO | Uploaded 36GiB to GPU in 2.14225s (16.8048GiB/s) | 20260505 22:21:47.077 | INFO | Downloaded 34GiB to GPU in 2.02749s (16.7695GiB/s) | 20260505 22:21:52.204 | INFO | Analyzing chunk frames = [6756352,6756864) | 20260505 22:21:52.204 | INFO | MemInfo: { | 20260505 22:21:52.204 | INFO | "gpu" : " 17 GB", | 20260505 22:21:52.204 | INFO | "huge" : "", | 20260505 22:21:52.204 | INFO | "huge_shared" : " 27 GB", | 20260505 22:21:52.204 | INFO | "malloc" : " 14 GB", | 20260505 22:21:52.204 | INFO | "pinned" : " 29 GB" | 20260505 22:21:52.204 | INFO | } | 20260505 22:21:52.204 | INFO | Perf report for the last 5.12771 seconds: | 20260505 22:21:52.204 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:21:52.204 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:52.204 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:52.204 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:52.204 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260505 22:21:52.204 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:52.204 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:52.204 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 22:21:52.204 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:52.204 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:52.204 | INFO | Uploaded 36GiB to GPU in 2.10271s (17.1207GiB/s) | 20260505 22:21:52.204 | INFO | Downloaded 34GiB to GPU in 2.04731s (16.6071GiB/s) | 20260505 22:21:57.347 | INFO | Analyzing chunk frames = [6756864,6757376) | 20260505 22:21:57.347 | INFO | MemInfo: { | 20260505 22:21:57.347 | INFO | "gpu" : " 17 GB", | 20260505 22:21:57.347 | INFO | "huge" : "", | 20260505 22:21:57.347 | INFO | "huge_shared" : " 27 GB", | 20260505 22:21:57.347 | INFO | "malloc" : " 14 GB", | 20260505 22:21:57.347 | INFO | "pinned" : " 29 GB" | 20260505 22:21:57.347 | INFO | } | 20260505 22:21:57.347 | INFO | Perf report for the last 5.14257 seconds: | 20260505 22:21:57.347 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:21:57.347 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:57.347 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:57.347 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:57.347 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260505 22:21:57.347 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:57.347 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:57.347 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:21:57.347 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:57.347 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:21:57.347 | INFO | Uploaded 36GiB to GPU in 2.09611s (17.1746GiB/s) | 20260505 22:21:57.347 | INFO | Downloaded 34GiB to GPU in 2.04331s (16.6397GiB/s) | 20260505 22:22:02.487 | INFO | Analyzing chunk frames = [6757376,6757888) | 20260505 22:22:02.487 | INFO | MemInfo: { | 20260505 22:22:02.487 | INFO | "gpu" : " 17 GB", | 20260505 22:22:02.487 | INFO | "huge" : "", | 20260505 22:22:02.487 | INFO | "huge_shared" : " 27 GB", | 20260505 22:22:02.487 | INFO | "malloc" : " 14 GB", | 20260505 22:22:02.487 | INFO | "pinned" : " 29 GB" | 20260505 22:22:02.487 | INFO | } | 20260505 22:22:02.487 | INFO | Perf report for the last 5.14025 seconds: | 20260505 22:22:02.487 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:22:02.487 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:02.487 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:02.487 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:02.487 | INFO | ANALYSIS : 62.5%, 3.0, 97.7ms, 0 | 20260505 22:22:02.487 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:02.487 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:02.487 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 22:22:02.487 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:02.487 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:02.487 | INFO | Uploaded 36GiB to GPU in 2.12011s (16.9803GiB/s) | 20260505 22:22:02.487 | INFO | Downloaded 34GiB to GPU in 2.08878s (16.2775GiB/s) | 20260505 22:22:07.623 | INFO | Analyzing chunk frames = [6757888,6758400) | 20260505 22:22:07.623 | INFO | MemInfo: { | 20260505 22:22:07.623 | INFO | "gpu" : " 17 GB", | 20260505 22:22:07.623 | INFO | "huge" : "", | 20260505 22:22:07.623 | INFO | "huge_shared" : " 27 GB", | 20260505 22:22:07.624 | INFO | "malloc" : " 14 GB", | 20260505 22:22:07.624 | INFO | "pinned" : " 29 GB" | 20260505 22:22:07.624 | INFO | } | 20260505 22:22:07.624 | INFO | Perf report for the last 5.13662 seconds: | 20260505 22:22:07.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:22:07.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:07.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:07.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:07.624 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 22:22:07.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:07.624 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:07.624 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 22:22:07.624 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:07.624 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:07.624 | INFO | Uploaded 36GiB to GPU in 2.11229s (17.0431GiB/s) | 20260505 22:22:07.624 | INFO | Downloaded 34GiB to GPU in 2.00026s (16.9978GiB/s) | 20260505 22:22:07.624 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 22:22:07.624 | INFO | Total mem usage: 15.3 GiB | 20260505 22:22:07.624 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 22:22:07.624 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -9.04%) | 20260505 22:22:07.624 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 22:22:07.624 | INFO | High Water Breakdown: | 20260505 22:22:07.624 | INFO | 60.9% : 8.56 GiB ( 90% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 22:22:07.624 | INFO | 39% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 22:22:07.624 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 22:22:07.624 | INFO | ------------------------------------------ | 20260505 22:22:07.624 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:22:07.624 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 22:22:07.624 | INFO | Total mem usage: 26.5 GiB | 20260505 22:22:07.624 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 22:22:07.624 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 22:22:07.624 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 22:22:07.624 | INFO | High Water Breakdown: | 20260505 22:22:07.624 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:22:07.625 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:22:07.625 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:22:07.625 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:22:07.625 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:22:07.625 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:22:07.625 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 22:22:07.625 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 22:22:07.625 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:22:07.625 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 22:22:07.625 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 22:22:07.625 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 22:22:07.625 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 22:22:07.625 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 22:22:07.625 | INFO | ------------------------------------------ | 20260505 22:22:07.625 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:22:07.625 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 22:22:07.625 | INFO | Total mem usage: 25.6 GiB | 20260505 22:22:07.625 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 22:22:07.625 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 22:22:07.625 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 22:22:07.625 | INFO | High Water Breakdown: | 20260505 22:22:07.625 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 22:22:07.625 | INFO | ------------------------------------------ | 20260505 22:22:07.625 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:22:07.625 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 22:22:07.625 | INFO | Total mem usage: 15.7 GiB | 20260505 22:22:07.625 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 22:22:07.625 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 22:22:07.625 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 22:22:07.625 | INFO | High Water Breakdown: | 20260505 22:22:07.625 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 22:22:07.625 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 22:22:07.625 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 22:22:07.625 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 22:22:07.625 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 22:22:07.625 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 22:22:07.625 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 22:22:07.625 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 22:22:07.625 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 22:22:07.625 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:22:07.625 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 22:22:07.625 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:22:07.625 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 22:22:07.625 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 22:22:07.625 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 22:22:07.625 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 22:22:07.625 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:22:07.625 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 22:22:07.625 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:22:07.625 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:22:07.625 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:22:07.625 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 22:22:07.625 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 22:22:07.625 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:22:07.625 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 22:22:07.625 | INFO | ------------------------------------------ | 20260505 22:22:07.625 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:22:07.625 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 22:22:07.625 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:22:07.625 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:22:07.625 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.40e+01, 1.33e+01 / 7.24e-01 | 20260505 22:22:07.625 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.04e-01, 1.04e-02 / 3.84e-03 | 20260505 22:22:07.625 | INFO | PerfReport: 11.81%, Baseline -- 3.71e+00 / 4.51e+00, 5.73e-01 / 2.20e-01 | 20260505 22:22:07.625 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.51e+00, 1.89e-01 / 9.38e-02 | 20260505 22:22:07.625 | INFO | PerfReport: 15.97%, DME -- 4.01e+01 / 4.40e+01, 3.80e+00 / 9.53e-01 | 20260505 22:22:07.625 | INFO | PerfReport: 57.24%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.53e+00 / 1.04e+00 | 20260505 22:22:07.625 | INFO | PerfReport: 7.16%, PulseAccumulating -- 2.25e+00 / 2.80e+00, 4.32e-01 / 2.11e-01 | 20260505 22:22:07.625 | INFO | PerfReport: 3.15%, Metrics -- 9.91e-01 / 1.29e+00, 2.69e-01 / 1.34e-01 | 20260505 22:22:07.625 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.45e-01 / 4.09e-01, 5.91e-02 / 3.35e-02 | 20260505 22:22:07.625 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 22:22:07.625 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:22:07.625 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:22:07.625 | INFO | PerfReport: Full Execution -- 9.87e+01 / -nan, 2.49e+01 / 0.00e+00 | 20260505 22:22:07.625 | INFO | PerfReport: 22.34%, Upload -- 2.21e+01 / 0.00e+00, 7.61e+00 / 0.00e+00 | 20260505 22:22:07.625 | INFO | PerfReport: 21.78%, Download -- 2.15e+01 / 0.00e+00, 7.65e+00 / 0.00e+00 | 20260505 22:22:07.625 | INFO | PerfReport: 31.83%, Compute -- 3.14e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 22:22:07.625 | INFO | PerfReport: 2.17%, LockUpload -- 2.15e+00 / 0.00e+00, 7.92e+00 / 0.00e+00 | 20260505 22:22:07.625 | INFO | PerfReport: 1.27%, LockDownload -- 1.26e+00 / 0.00e+00, 2.84e+00 / 0.00e+00 | 20260505 22:22:07.625 | INFO | PerfReport: 20.59%, LockCompute -- 2.03e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260505 22:22:07.625 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.72e-02 / 0.00e+00 | 20260505 22:22:07.750 | INFO | Starting a round of Baz IO | 20260505 22:22:10.898 | INFO | Populating RTMetrics report | 20260505 22:22:10.898 | INFO | RTMetrics report populated | 20260505 22:22:10.898 | INFO | Opening tmp RTMetrics json file | 20260505 22:22:10.902 | INFO | Serializing RTMetrics to json | 20260505 22:22:10.903 | INFO | Serialization done, swapping tmp file now... | 20260505 22:22:10.904 | INFO | RTMetrics done | 20260505 22:22:10.979 | INFO | Finished a round of Baz IO: Wrote 1.934 GiB in 3.228 seconds | 20260505 22:22:10.979 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.50% / 0.00% / 78.50% | 20260505 22:22:10.979 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.126394% / 0% / 7.54992e-05% | 20260505 22:22:10.979 | INFO | Baz IO overall: | 20260505 22:22:10.979 | INFO | BytesWritten : 3892692879968 | 20260505 22:22:10.979 | INFO | NumEvents : 1836248214169 | 20260505 22:22:12.765 | INFO | Analyzing chunk frames = [6758400,6758912) | 20260505 22:22:12.765 | INFO | MemInfo: { | 20260505 22:22:12.765 | INFO | "gpu" : " 17 GB", | 20260505 22:22:12.765 | INFO | "huge" : "", | 20260505 22:22:12.765 | INFO | "huge_shared" : " 27 GB", | 20260505 22:22:12.765 | INFO | "malloc" : " 14 GB", | 20260505 22:22:12.766 | INFO | "pinned" : " 29 GB" | 20260505 22:22:12.766 | INFO | } | 20260505 22:22:12.766 | INFO | Perf report for the last 5.14199 seconds: | 20260505 22:22:12.766 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:22:12.766 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:12.766 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:12.766 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:12.766 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 22:22:12.766 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:12.766 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260505 22:22:12.766 | INFO | PRE_HQ : 59.8%, 1.8, 54.5ms, 0 | 20260505 22:22:12.766 | INFO | BAZWRITER : 62.1%, 3.6, 116.1ms, 0 | 20260505 22:22:12.766 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 22:22:12.766 | INFO | Uploaded 36GiB to GPU in 2.16294s (16.644GiB/s) | 20260505 22:22:12.766 | INFO | Downloaded 38GiB to GPU in 2.23127s (17.0307GiB/s) | 20260505 22:22:17.903 | INFO | Analyzing chunk frames = [6758912,6759424) | 20260505 22:22:17.903 | INFO | MemInfo: { | 20260505 22:22:17.903 | INFO | "gpu" : " 17 GB", | 20260505 22:22:17.903 | INFO | "huge" : "", | 20260505 22:22:17.903 | INFO | "huge_shared" : " 27 GB", | 20260505 22:22:17.903 | INFO | "malloc" : " 14 GB", | 20260505 22:22:17.903 | INFO | "pinned" : " 29 GB" | 20260505 22:22:17.903 | INFO | } | 20260505 22:22:17.903 | INFO | Perf report for the last 5.13715 seconds: | 20260505 22:22:17.903 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:22:17.903 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:17.903 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:17.903 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:17.903 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 22:22:17.903 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:17.903 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:17.903 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:22:17.903 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:17.903 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:17.903 | INFO | Uploaded 36GiB to GPU in 2.07939s (17.3128GiB/s) | 20260505 22:22:17.903 | INFO | Downloaded 34GiB to GPU in 2.03013s (16.7477GiB/s) | 20260505 22:22:23.044 | INFO | Analyzing chunk frames = [6759424,6759936) | 20260505 22:22:23.044 | INFO | MemInfo: { | 20260505 22:22:23.044 | INFO | "gpu" : " 17 GB", | 20260505 22:22:23.044 | INFO | "huge" : "", | 20260505 22:22:23.044 | INFO | "huge_shared" : " 27 GB", | 20260505 22:22:23.044 | INFO | "malloc" : " 14 GB", | 20260505 22:22:23.044 | INFO | "pinned" : " 29 GB" | 20260505 22:22:23.044 | INFO | } | 20260505 22:22:23.044 | INFO | Perf report for the last 5.14131 seconds: | 20260505 22:22:23.044 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:22:23.044 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:23.044 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:23.044 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:23.044 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260505 22:22:23.044 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:23.044 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:23.044 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:22:23.044 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:23.044 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:23.044 | INFO | Uploaded 36GiB to GPU in 2.13228s (16.8833GiB/s) | 20260505 22:22:23.044 | INFO | Downloaded 34GiB to GPU in 2.02076s (16.8254GiB/s) | 20260505 22:22:28.185 | INFO | Analyzing chunk frames = [6759936,6760448) | 20260505 22:22:28.185 | INFO | MemInfo: { | 20260505 22:22:28.185 | INFO | "gpu" : " 17 GB", | 20260505 22:22:28.185 | INFO | "huge" : "", | 20260505 22:22:28.185 | INFO | "huge_shared" : " 27 GB", | 20260505 22:22:28.185 | INFO | "malloc" : " 14 GB", | 20260505 22:22:28.185 | INFO | "pinned" : " 29 GB" | 20260505 22:22:28.185 | INFO | } | 20260505 22:22:28.185 | INFO | Perf report for the last 5.14101 seconds: | 20260505 22:22:28.185 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:22:28.185 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:28.185 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:28.185 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:28.185 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260505 22:22:28.185 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:28.185 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:28.185 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:22:28.185 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:28.185 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:28.185 | INFO | Uploaded 36GiB to GPU in 2.12403s (16.9489GiB/s) | 20260505 22:22:28.185 | INFO | Downloaded 34GiB to GPU in 2.01708s (16.856GiB/s) | 20260505 22:22:33.316 | INFO | Analyzing chunk frames = [6760448,6760960) | 20260505 22:22:33.316 | INFO | MemInfo: { | 20260505 22:22:33.316 | INFO | "gpu" : " 17 GB", | 20260505 22:22:33.316 | INFO | "huge" : "", | 20260505 22:22:33.316 | INFO | "huge_shared" : " 27 GB", | 20260505 22:22:33.316 | INFO | "malloc" : " 14 GB", | 20260505 22:22:33.316 | INFO | "pinned" : " 29 GB" | 20260505 22:22:33.316 | INFO | } | 20260505 22:22:33.316 | INFO | Perf report for the last 5.13121 seconds: | 20260505 22:22:33.316 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:22:33.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:33.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:33.316 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:33.316 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260505 22:22:33.316 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:33.316 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:33.316 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 22:22:33.316 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:33.316 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:33.317 | INFO | Uploaded 36GiB to GPU in 2.09753s (17.163GiB/s) | 20260505 22:22:33.317 | INFO | Downloaded 34GiB to GPU in 2.0473s (16.6073GiB/s) | 20260505 22:22:38.453 | INFO | Analyzing chunk frames = [6760960,6761472) | 20260505 22:22:38.453 | INFO | MemInfo: { | 20260505 22:22:38.454 | INFO | "gpu" : " 17 GB", | 20260505 22:22:38.454 | INFO | "huge" : "", | 20260505 22:22:38.454 | INFO | "huge_shared" : " 27 GB", | 20260505 22:22:38.454 | INFO | "malloc" : " 14 GB", | 20260505 22:22:38.454 | INFO | "pinned" : " 29 GB" | 20260505 22:22:38.454 | INFO | } | 20260505 22:22:38.454 | INFO | Perf report for the last 5.13737 seconds: | 20260505 22:22:38.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:22:38.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:38.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:38.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:38.454 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260505 22:22:38.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:38.454 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:38.454 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:22:38.454 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:38.454 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:38.454 | INFO | Uploaded 36GiB to GPU in 2.0761s (17.3402GiB/s) | 20260505 22:22:38.454 | INFO | Downloaded 34GiB to GPU in 2.03959s (16.67GiB/s) | 20260505 22:22:43.595 | INFO | Analyzing chunk frames = [6761472,6761984) | 20260505 22:22:43.595 | INFO | MemInfo: { | 20260505 22:22:43.595 | INFO | "gpu" : " 17 GB", | 20260505 22:22:43.595 | INFO | "huge" : "", | 20260505 22:22:43.595 | INFO | "huge_shared" : " 27 GB", | 20260505 22:22:43.595 | INFO | "malloc" : " 14 GB", | 20260505 22:22:43.596 | INFO | "pinned" : " 29 GB" | 20260505 22:22:43.596 | INFO | } | 20260505 22:22:43.596 | INFO | Perf report for the last 5.14194 seconds: | 20260505 22:22:43.596 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:22:43.596 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:43.596 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:43.596 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:43.596 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 22:22:43.596 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:43.596 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:43.596 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 22:22:43.596 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:43.596 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:43.596 | INFO | Uploaded 36GiB to GPU in 2.13597s (16.8542GiB/s) | 20260505 22:22:43.596 | INFO | Downloaded 34GiB to GPU in 2.13204s (15.9472GiB/s) | 20260505 22:22:48.742 | INFO | Analyzing chunk frames = [6761984,6762496) | 20260505 22:22:48.742 | INFO | MemInfo: { | 20260505 22:22:48.742 | INFO | "gpu" : " 17 GB", | 20260505 22:22:48.742 | INFO | "huge" : "", | 20260505 22:22:48.742 | INFO | "huge_shared" : " 27 GB", | 20260505 22:22:48.742 | INFO | "malloc" : " 14 GB", | 20260505 22:22:48.742 | INFO | "pinned" : " 29 GB" | 20260505 22:22:48.742 | INFO | } | 20260505 22:22:48.742 | INFO | Perf report for the last 5.14622 seconds: | 20260505 22:22:48.742 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:22:48.742 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:48.742 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:48.742 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:48.742 | INFO | ANALYSIS : 62.0%, 3.0, 97.0ms, 0 | 20260505 22:22:48.742 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:48.742 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:48.742 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:22:48.742 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:48.742 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:48.742 | INFO | Uploaded 36GiB to GPU in 2.11674s (17.0073GiB/s) | 20260505 22:22:48.742 | INFO | Downloaded 34GiB to GPU in 2.00447s (16.9621GiB/s) | 20260505 22:22:48.861 | INFO | Starting a round of Baz IO | 20260505 22:22:52.012 | INFO | Populating RTMetrics report | 20260505 22:22:52.012 | INFO | RTMetrics report populated | 20260505 22:22:52.012 | INFO | Opening tmp RTMetrics json file | 20260505 22:22:52.014 | INFO | Serializing RTMetrics to json | 20260505 22:22:52.015 | INFO | Serialization done, swapping tmp file now... | 20260505 22:22:52.016 | INFO | RTMetrics done | 20260505 22:22:52.094 | INFO | Finished a round of Baz IO: Wrote 1.932 GiB in 3.233 seconds | 20260505 22:22:52.094 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.50% / 0.00% / 78.50% | 20260505 22:22:52.094 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.125396% / 0% / 7.54992e-05% | 20260505 22:22:52.094 | INFO | Baz IO overall: | 20260505 22:22:52.094 | INFO | BytesWritten : 3894767115702 | 20260505 22:22:52.095 | INFO | NumEvents : 1837202561846 | 20260505 22:22:53.883 | INFO | Analyzing chunk frames = [6762496,6763008) | 20260505 22:22:53.883 | INFO | MemInfo: { | 20260505 22:22:53.883 | INFO | "gpu" : " 17 GB", | 20260505 22:22:53.883 | INFO | "huge" : "", | 20260505 22:22:53.883 | INFO | "huge_shared" : " 27 GB", | 20260505 22:22:53.883 | INFO | "malloc" : " 14 GB", | 20260505 22:22:53.883 | INFO | "pinned" : " 29 GB" | 20260505 22:22:53.883 | INFO | } | 20260505 22:22:53.883 | INFO | Perf report for the last 5.14146 seconds: | 20260505 22:22:53.883 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:22:53.883 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:53.883 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:53.883 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:53.883 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 22:22:53.883 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:53.883 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260505 22:22:53.883 | INFO | PRE_HQ : 59.8%, 1.8, 54.5ms, 0 | 20260505 22:22:53.883 | INFO | BAZWRITER : 62.1%, 3.7, 116.5ms, 0 | 20260505 22:22:53.883 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 22:22:53.884 | INFO | Uploaded 36GiB to GPU in 2.16215s (16.6501GiB/s) | 20260505 22:22:53.884 | INFO | Downloaded 38GiB to GPU in 2.23226s (17.0231GiB/s) | 20260505 22:22:59.017 | INFO | Analyzing chunk frames = [6763008,6763520) | 20260505 22:22:59.017 | INFO | MemInfo: { | 20260505 22:22:59.017 | INFO | "gpu" : " 17 GB", | 20260505 22:22:59.017 | INFO | "huge" : "", | 20260505 22:22:59.017 | INFO | "huge_shared" : " 27 GB", | 20260505 22:22:59.017 | INFO | "malloc" : " 14 GB", | 20260505 22:22:59.017 | INFO | "pinned" : " 29 GB" | 20260505 22:22:59.017 | INFO | } | 20260505 22:22:59.017 | INFO | Perf report for the last 5.13351 seconds: | 20260505 22:22:59.017 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:22:59.017 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:59.017 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:59.017 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:59.017 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260505 22:22:59.017 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:59.017 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:59.017 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 22:22:59.017 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:59.017 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:22:59.017 | INFO | Uploaded 36GiB to GPU in 2.08465s (17.2691GiB/s) | 20260505 22:22:59.017 | INFO | Downloaded 34GiB to GPU in 2.03161s (16.7355GiB/s) | 20260505 22:23:04.152 | INFO | Analyzing chunk frames = [6763520,6764032) | 20260505 22:23:04.152 | INFO | MemInfo: { | 20260505 22:23:04.152 | INFO | "gpu" : " 17 GB", | 20260505 22:23:04.152 | INFO | "huge" : "", | 20260505 22:23:04.152 | INFO | "huge_shared" : " 27 GB", | 20260505 22:23:04.152 | INFO | "malloc" : " 14 GB", | 20260505 22:23:04.152 | INFO | "pinned" : " 29 GB" | 20260505 22:23:04.152 | INFO | } | 20260505 22:23:04.152 | INFO | Perf report for the last 5.1356 seconds: | 20260505 22:23:04.152 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:23:04.152 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:04.152 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:04.152 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:04.152 | INFO | ANALYSIS : 62.1%, 3.0, 96.5ms, 0 | 20260505 22:23:04.153 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:04.153 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:04.153 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:23:04.153 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:04.153 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:04.153 | INFO | Uploaded 36GiB to GPU in 2.14131s (16.8121GiB/s) | 20260505 22:23:04.153 | INFO | Downloaded 34GiB to GPU in 2.02145s (16.8196GiB/s) | 20260505 22:23:09.299 | INFO | Analyzing chunk frames = [6764032,6764544) | 20260505 22:23:09.299 | INFO | MemInfo: { | 20260505 22:23:09.299 | INFO | "gpu" : " 17 GB", | 20260505 22:23:09.299 | INFO | "huge" : "", | 20260505 22:23:09.299 | INFO | "huge_shared" : " 27 GB", | 20260505 22:23:09.299 | INFO | "malloc" : " 14 GB", | 20260505 22:23:09.299 | INFO | "pinned" : " 29 GB" | 20260505 22:23:09.299 | INFO | } | 20260505 22:23:09.299 | INFO | Perf report for the last 5.14662 seconds: | 20260505 22:23:09.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:23:09.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:09.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:09.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:09.299 | INFO | ANALYSIS : 61.9%, 3.0, 96.2ms, 0 | 20260505 22:23:09.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:09.299 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:09.299 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:23:09.299 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:09.299 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:09.299 | INFO | Uploaded 36GiB to GPU in 2.13424s (16.8678GiB/s) | 20260505 22:23:09.299 | INFO | Downloaded 34GiB to GPU in 2.04869s (16.596GiB/s) | 20260505 22:23:14.442 | INFO | Analyzing chunk frames = [6764544,6765056) | 20260505 22:23:14.442 | INFO | MemInfo: { | 20260505 22:23:14.442 | INFO | "gpu" : " 17 GB", | 20260505 22:23:14.442 | INFO | "huge" : "", | 20260505 22:23:14.442 | INFO | "huge_shared" : " 27 GB", | 20260505 22:23:14.442 | INFO | "malloc" : " 14 GB", | 20260505 22:23:14.442 | INFO | "pinned" : " 29 GB" | 20260505 22:23:14.442 | INFO | } | 20260505 22:23:14.442 | INFO | Perf report for the last 5.14315 seconds: | 20260505 22:23:14.442 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:23:14.442 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:14.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:14.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:14.442 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260505 22:23:14.442 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:14.442 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:14.442 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 22:23:14.442 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:14.442 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:14.442 | INFO | Uploaded 36GiB to GPU in 2.08982s (17.2263GiB/s) | 20260505 22:23:14.442 | INFO | Downloaded 34GiB to GPU in 2.04359s (16.6374GiB/s) | 20260505 22:23:19.574 | INFO | Analyzing chunk frames = [6765056,6765568) | 20260505 22:23:19.574 | INFO | MemInfo: { | 20260505 22:23:19.574 | INFO | "gpu" : " 17 GB", | 20260505 22:23:19.574 | INFO | "huge" : "", | 20260505 22:23:19.574 | INFO | "huge_shared" : " 27 GB", | 20260505 22:23:19.574 | INFO | "malloc" : " 14 GB", | 20260505 22:23:19.574 | INFO | "pinned" : " 29 GB" | 20260505 22:23:19.574 | INFO | } | 20260505 22:23:19.574 | INFO | Perf report for the last 5.13153 seconds: | 20260505 22:23:19.574 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:23:19.574 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:19.574 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:19.574 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:19.574 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260505 22:23:19.574 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:19.574 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:19.574 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 22:23:19.574 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:19.574 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:19.574 | INFO | Uploaded 36GiB to GPU in 2.07127s (17.3806GiB/s) | 20260505 22:23:19.574 | INFO | Downloaded 34GiB to GPU in 2.03428s (16.7135GiB/s) | 20260505 22:23:24.715 | INFO | Analyzing chunk frames = [6765568,6766080) | 20260505 22:23:24.715 | INFO | MemInfo: { | 20260505 22:23:24.715 | INFO | "gpu" : " 17 GB", | 20260505 22:23:24.715 | INFO | "huge" : "", | 20260505 22:23:24.715 | INFO | "huge_shared" : " 27 GB", | 20260505 22:23:24.715 | INFO | "malloc" : " 14 GB", | 20260505 22:23:24.715 | INFO | "pinned" : " 29 GB" | 20260505 22:23:24.715 | INFO | } | 20260505 22:23:24.715 | INFO | Perf report for the last 5.14113 seconds: | 20260505 22:23:24.715 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:23:24.715 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:24.715 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:24.715 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:24.715 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 22:23:24.715 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:24.715 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:24.715 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 22:23:24.715 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:24.715 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:24.715 | INFO | Uploaded 36GiB to GPU in 2.12885s (16.9105GiB/s) | 20260505 22:23:24.715 | INFO | Downloaded 34GiB to GPU in 2.11092s (16.1067GiB/s) | 20260505 22:23:29.853 | INFO | Analyzing chunk frames = [6766080,6766592) | 20260505 22:23:29.854 | INFO | MemInfo: { | 20260505 22:23:29.854 | INFO | "gpu" : " 17 GB", | 20260505 22:23:29.854 | INFO | "huge" : "", | 20260505 22:23:29.854 | INFO | "huge_shared" : " 27 GB", | 20260505 22:23:29.854 | INFO | "malloc" : " 14 GB", | 20260505 22:23:29.854 | INFO | "pinned" : " 29 GB" | 20260505 22:23:29.854 | INFO | } | 20260505 22:23:29.854 | INFO | Perf report for the last 5.1387 seconds: | 20260505 22:23:29.854 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:23:29.854 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:29.854 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:29.854 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:29.854 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 22:23:29.854 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:29.854 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:29.854 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 22:23:29.854 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:29.854 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:29.854 | INFO | Uploaded 36GiB to GPU in 2.10518s (17.1007GiB/s) | 20260505 22:23:29.854 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 22:23:29.854 | INFO | Total mem usage: 15.3 GiB | 20260505 22:23:29.854 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 22:23:29.854 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.5%) | 20260505 22:23:29.854 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 22:23:29.854 | INFO | Downloaded 34GiB to GPU in 2.00134s (16.9887GiB/s) | 20260505 22:23:29.854 | INFO | High Water Breakdown: | 20260505 22:23:29.854 | INFO | 61.1% : 8.63 GiB ( 90.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 22:23:29.854 | INFO | 38.8% : 5.48 GiB ( 82.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 22:23:29.854 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 22:23:29.854 | INFO | ------------------------------------------ | 20260505 22:23:29.854 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:23:29.854 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 22:23:29.854 | INFO | Total mem usage: 26.5 GiB | 20260505 22:23:29.854 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 22:23:29.854 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 22:23:29.854 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 22:23:29.854 | INFO | High Water Breakdown: | 20260505 22:23:29.854 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:23:29.854 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:23:29.854 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:23:29.854 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:23:29.854 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:23:29.854 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:23:29.854 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 22:23:29.854 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 22:23:29.854 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:23:29.854 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 22:23:29.854 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 22:23:29.854 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 22:23:29.854 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 22:23:29.854 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 22:23:29.854 | INFO | ------------------------------------------ | 20260505 22:23:29.854 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:23:29.854 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 22:23:29.854 | INFO | Total mem usage: 25.6 GiB | 20260505 22:23:29.854 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 22:23:29.854 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 22:23:29.854 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 22:23:29.854 | INFO | High Water Breakdown: | 20260505 22:23:29.854 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 22:23:29.854 | INFO | ------------------------------------------ | 20260505 22:23:29.854 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:23:29.854 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 22:23:29.854 | INFO | Total mem usage: 15.7 GiB | 20260505 22:23:29.854 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 22:23:29.854 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 22:23:29.854 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 22:23:29.854 | INFO | High Water Breakdown: | 20260505 22:23:29.854 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 22:23:29.854 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 22:23:29.854 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 22:23:29.854 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 22:23:29.854 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 22:23:29.854 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 22:23:29.854 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 22:23:29.854 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 22:23:29.854 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 22:23:29.854 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:23:29.854 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 22:23:29.854 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:23:29.854 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 22:23:29.854 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 22:23:29.855 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 22:23:29.855 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 22:23:29.855 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:23:29.855 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 22:23:29.855 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:23:29.855 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:23:29.855 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:23:29.855 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 22:23:29.855 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 22:23:29.855 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:23:29.855 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 22:23:29.855 | INFO | ------------------------------------------ | 20260505 22:23:29.855 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:23:29.855 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 22:23:29.855 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:23:29.855 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:23:29.855 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.43e+01, 1.34e+01 / 1.08e+00 | 20260505 22:23:29.855 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.05e-01, 1.03e-02 / 7.87e-03 | 20260505 22:23:29.855 | INFO | PerfReport: 11.83%, Baseline -- 3.72e+00 / 4.56e+00, 5.78e-01 / 1.25e-01 | 20260505 22:23:29.855 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.54e+00, 1.90e-01 / 7.61e-02 | 20260505 22:23:29.855 | INFO | PerfReport: 15.99%, DME -- 4.02e+01 / 4.42e+01, 3.88e+00 / 1.32e+00 | 20260505 22:23:29.855 | INFO | PerfReport: 57.22%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.51e+00 / 1.11e+00 | 20260505 22:23:29.855 | INFO | PerfReport: 7.15%, PulseAccumulating -- 2.24e+00 / 2.79e+00, 4.30e-01 / 1.96e-01 | 20260505 22:23:29.855 | INFO | PerfReport: 3.15%, Metrics -- 9.89e-01 / 1.28e+00, 2.67e-01 / 1.04e-01 | 20260505 22:23:29.855 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.44e-01 / 4.02e-01, 5.90e-02 / 3.18e-02 | 20260505 22:23:29.855 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 22:23:29.855 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:23:29.855 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:23:29.855 | INFO | PerfReport: Full Execution -- 9.87e+01 / -nan, 2.50e+01 / 0.00e+00 | 20260505 22:23:29.855 | INFO | PerfReport: 22.32%, Upload -- 2.20e+01 / 0.00e+00, 7.58e+00 / 0.00e+00 | 20260505 22:23:29.855 | INFO | PerfReport: 21.80%, Download -- 2.15e+01 / 0.00e+00, 7.63e+00 / 0.00e+00 | 20260505 22:23:29.855 | INFO | PerfReport: 31.83%, Compute -- 3.14e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 22:23:29.855 | INFO | PerfReport: 2.17%, LockUpload -- 2.14e+00 / 0.00e+00, 7.98e+00 / 0.00e+00 | 20260505 22:23:29.855 | INFO | PerfReport: 1.34%, LockDownload -- 1.33e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260505 22:23:29.855 | INFO | PerfReport: 20.52%, LockCompute -- 2.03e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260505 22:23:29.855 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 2.02e-02 / 0.00e+00 | 20260505 22:23:29.985 | INFO | Starting a round of Baz IO | 20260505 22:23:33.140 | INFO | Populating RTMetrics report | 20260505 22:23:33.140 | INFO | RTMetrics report populated | 20260505 22:23:33.140 | INFO | Opening tmp RTMetrics json file | 20260505 22:23:33.141 | INFO | Serializing RTMetrics to json | 20260505 22:23:33.144 | INFO | Serialization done, swapping tmp file now... | 20260505 22:23:33.146 | INFO | RTMetrics done | 20260505 22:23:33.223 | INFO | Finished a round of Baz IO: Wrote 1.932 GiB in 3.237 seconds | 20260505 22:23:33.223 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.50% / 0.00% / 78.50% | 20260505 22:23:33.223 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.125273% / 1.19209e-05% / 0.000107288% | 20260505 22:23:33.223 | INFO | Baz IO overall: | 20260505 22:23:33.223 | INFO | BytesWritten : 3896841588929 | 20260505 22:23:33.223 | INFO | NumEvents : 1838157135046 | 20260505 22:23:34.994 | INFO | Analyzing chunk frames = [6766592,6767104) | 20260505 22:23:34.994 | INFO | MemInfo: { | 20260505 22:23:34.994 | INFO | "gpu" : " 17 GB", | 20260505 22:23:34.994 | INFO | "huge" : "", | 20260505 22:23:34.994 | INFO | "huge_shared" : " 27 GB", | 20260505 22:23:34.994 | INFO | "malloc" : " 14 GB", | 20260505 22:23:34.994 | INFO | "pinned" : " 29 GB" | 20260505 22:23:34.994 | INFO | } | 20260505 22:23:34.994 | INFO | Perf report for the last 5.14043 seconds: | 20260505 22:23:34.994 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:23:34.994 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:34.994 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:34.994 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:34.994 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260505 22:23:34.994 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:34.994 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260505 22:23:34.994 | INFO | PRE_HQ : 59.9%, 1.8, 54.6ms, 0 | 20260505 22:23:34.994 | INFO | BAZWRITER : 62.4%, 3.6, 115.2ms, 0 | 20260505 22:23:34.994 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 22:23:34.994 | INFO | Uploaded 36GiB to GPU in 2.17123s (16.5805GiB/s) | 20260505 22:23:34.994 | INFO | Downloaded 38GiB to GPU in 2.23275s (17.0194GiB/s) | 20260505 22:23:40.131 | INFO | Analyzing chunk frames = [6767104,6767616) | 20260505 22:23:40.131 | INFO | MemInfo: { | 20260505 22:23:40.131 | INFO | "gpu" : " 17 GB", | 20260505 22:23:40.131 | INFO | "huge" : "", | 20260505 22:23:40.131 | INFO | "huge_shared" : " 27 GB", | 20260505 22:23:40.131 | INFO | "malloc" : " 14 GB", | 20260505 22:23:40.131 | INFO | "pinned" : " 29 GB" | 20260505 22:23:40.131 | INFO | } | 20260505 22:23:40.131 | INFO | Perf report for the last 5.13723 seconds: | 20260505 22:23:40.131 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:23:40.131 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:40.131 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:40.131 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:40.131 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 22:23:40.131 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:40.131 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:40.131 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260505 22:23:40.131 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:40.131 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:40.131 | INFO | Uploaded 36GiB to GPU in 2.08441s (17.2711GiB/s) | 20260505 22:23:40.132 | INFO | Downloaded 34GiB to GPU in 2.02863s (16.7601GiB/s) | 20260505 22:23:45.269 | INFO | Analyzing chunk frames = [6767616,6768128) | 20260505 22:23:45.269 | INFO | MemInfo: { | 20260505 22:23:45.269 | INFO | "gpu" : " 17 GB", | 20260505 22:23:45.269 | INFO | "huge" : "", | 20260505 22:23:45.269 | INFO | "huge_shared" : " 27 GB", | 20260505 22:23:45.269 | INFO | "malloc" : " 14 GB", | 20260505 22:23:45.269 | INFO | "pinned" : " 29 GB" | 20260505 22:23:45.269 | INFO | } | 20260505 22:23:45.269 | INFO | Perf report for the last 5.13809 seconds: | 20260505 22:23:45.269 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:23:45.269 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:45.269 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:45.269 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:45.269 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260505 22:23:45.269 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:45.270 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:45.270 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 22:23:45.270 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:45.270 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:45.270 | INFO | Uploaded 36GiB to GPU in 2.12982s (16.9029GiB/s) | 20260505 22:23:45.270 | INFO | Downloaded 34GiB to GPU in 2.02134s (16.8205GiB/s) | 20260505 22:23:50.408 | INFO | Analyzing chunk frames = [6768128,6768640) | 20260505 22:23:50.408 | INFO | MemInfo: { | 20260505 22:23:50.408 | INFO | "gpu" : " 17 GB", | 20260505 22:23:50.408 | INFO | "huge" : "", | 20260505 22:23:50.408 | INFO | "huge_shared" : " 27 GB", | 20260505 22:23:50.408 | INFO | "malloc" : " 14 GB", | 20260505 22:23:50.408 | INFO | "pinned" : " 29 GB" | 20260505 22:23:50.408 | INFO | } | 20260505 22:23:50.409 | INFO | Perf report for the last 5.1391 seconds: | 20260505 22:23:50.409 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:23:50.409 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:50.409 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:50.409 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:50.409 | INFO | ANALYSIS : 61.9%, 3.0, 96.2ms, 0 | 20260505 22:23:50.409 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:50.409 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:50.409 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:23:50.409 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:50.409 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:50.409 | INFO | Uploaded 36GiB to GPU in 2.13224s (16.8836GiB/s) | 20260505 22:23:50.409 | INFO | Downloaded 34GiB to GPU in 2.03892s (16.6755GiB/s) | 20260505 22:23:55.550 | INFO | Analyzing chunk frames = [6768640,6769152) | 20260505 22:23:55.550 | INFO | MemInfo: { | 20260505 22:23:55.550 | INFO | "gpu" : " 17 GB", | 20260505 22:23:55.550 | INFO | "huge" : "", | 20260505 22:23:55.550 | INFO | "huge_shared" : " 27 GB", | 20260505 22:23:55.550 | INFO | "malloc" : " 14 GB", | 20260505 22:23:55.550 | INFO | "pinned" : " 29 GB" | 20260505 22:23:55.550 | INFO | } | 20260505 22:23:55.550 | INFO | Perf report for the last 5.14184 seconds: | 20260505 22:23:55.550 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:23:55.550 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:55.550 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:55.550 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:55.550 | INFO | ANALYSIS : 61.5%, 3.0, 96.0ms, 0 | 20260505 22:23:55.550 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:55.550 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:55.551 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:23:55.551 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:55.551 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:23:55.551 | INFO | Uploaded 36GiB to GPU in 2.1261s (16.9325GiB/s) | 20260505 22:23:55.551 | INFO | Downloaded 34GiB to GPU in 2.06092s (16.4975GiB/s) | 20260505 22:24:00.695 | INFO | Analyzing chunk frames = [6769152,6769664) | 20260505 22:24:00.695 | INFO | MemInfo: { | 20260505 22:24:00.695 | INFO | "gpu" : " 17 GB", | 20260505 22:24:00.695 | INFO | "huge" : "", | 20260505 22:24:00.695 | INFO | "huge_shared" : " 27 GB", | 20260505 22:24:00.695 | INFO | "malloc" : " 14 GB", | 20260505 22:24:00.695 | INFO | "pinned" : " 29 GB" | 20260505 22:24:00.695 | INFO | } | 20260505 22:24:00.695 | INFO | Perf report for the last 5.14456 seconds: | 20260505 22:24:00.695 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:24:00.695 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:00.695 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:00.695 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:00.695 | INFO | ANALYSIS : 61.5%, 3.0, 96.1ms, 0 | 20260505 22:24:00.695 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:00.695 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:00.695 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:24:00.695 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:00.695 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:00.695 | INFO | Uploaded 36GiB to GPU in 2.07814s (17.3232GiB/s) | 20260505 22:24:00.695 | INFO | Downloaded 34GiB to GPU in 2.04181s (16.6519GiB/s) | 20260505 22:24:05.829 | INFO | Analyzing chunk frames = [6769664,6770176) | 20260505 22:24:05.829 | INFO | MemInfo: { | 20260505 22:24:05.829 | INFO | "gpu" : " 17 GB", | 20260505 22:24:05.829 | INFO | "huge" : "", | 20260505 22:24:05.829 | INFO | "huge_shared" : " 27 GB", | 20260505 22:24:05.829 | INFO | "malloc" : " 14 GB", | 20260505 22:24:05.829 | INFO | "pinned" : " 29 GB" | 20260505 22:24:05.829 | INFO | } | 20260505 22:24:05.829 | INFO | Perf report for the last 5.13435 seconds: | 20260505 22:24:05.829 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:24:05.829 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:05.829 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:05.829 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:05.829 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 22:24:05.829 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:05.829 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:05.829 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 22:24:05.829 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:05.829 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:05.830 | INFO | Uploaded 36GiB to GPU in 2.11584s (17.0145GiB/s) | 20260505 22:24:05.830 | INFO | Downloaded 34GiB to GPU in 2.10108s (16.1822GiB/s) | 20260505 22:24:10.962 | INFO | Analyzing chunk frames = [6770176,6770688) | 20260505 22:24:10.962 | INFO | MemInfo: { | 20260505 22:24:10.962 | INFO | "gpu" : " 17 GB", | 20260505 22:24:10.962 | INFO | "huge" : "", | 20260505 22:24:10.962 | INFO | "huge_shared" : " 27 GB", | 20260505 22:24:10.962 | INFO | "malloc" : " 14 GB", | 20260505 22:24:10.962 | INFO | "pinned" : " 29 GB" | 20260505 22:24:10.962 | INFO | } | 20260505 22:24:10.962 | INFO | Perf report for the last 5.13259 seconds: | 20260505 22:24:10.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:24:10.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:10.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:10.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:10.962 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260505 22:24:10.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:10.962 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:10.962 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 22:24:10.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:10.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:10.962 | INFO | Uploaded 36GiB to GPU in 2.11718s (17.0037GiB/s) | 20260505 22:24:10.962 | INFO | Downloaded 34GiB to GPU in 2.01681s (16.8583GiB/s) | 20260505 22:24:11.094 | INFO | Starting a round of Baz IO | 20260505 22:24:14.257 | INFO | Populating RTMetrics report | 20260505 22:24:14.257 | INFO | RTMetrics report populated | 20260505 22:24:14.257 | INFO | Opening tmp RTMetrics json file | 20260505 22:24:14.260 | INFO | Serializing RTMetrics to json | 20260505 22:24:14.261 | INFO | Serialization done, swapping tmp file now... | 20260505 22:24:14.262 | INFO | RTMetrics done | 20260505 22:24:14.340 | INFO | Finished a round of Baz IO: Wrote 1.932 GiB in 3.245 seconds | 20260505 22:24:14.340 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.50% / 0.00% / 78.50% | 20260505 22:24:14.340 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.125531% / 0% / 9.53674e-05% | 20260505 22:24:14.340 | INFO | Baz IO overall: | 20260505 22:24:14.340 | INFO | BytesWritten : 3898915696155 | 20260505 22:24:14.340 | INFO | NumEvents : 1839111688373 | 20260505 22:24:16.108 | INFO | Analyzing chunk frames = [6770688,6771200) | 20260505 22:24:16.108 | INFO | MemInfo: { | 20260505 22:24:16.108 | INFO | "gpu" : " 17 GB", | 20260505 22:24:16.108 | INFO | "huge" : "", | 20260505 22:24:16.108 | INFO | "huge_shared" : " 27 GB", | 20260505 22:24:16.108 | INFO | "malloc" : " 14 GB", | 20260505 22:24:16.108 | INFO | "pinned" : " 29 GB" | 20260505 22:24:16.108 | INFO | } | 20260505 22:24:16.108 | INFO | Perf report for the last 5.14646 seconds: | 20260505 22:24:16.108 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:24:16.108 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:16.108 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:16.108 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:16.108 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 22:24:16.108 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:16.108 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260505 22:24:16.108 | INFO | PRE_HQ : 60.1%, 1.8, 54.8ms, 0 | 20260505 22:24:16.108 | INFO | BAZWRITER : 62.6%, 3.6, 115.8ms, 0 | 20260505 22:24:16.108 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 22:24:16.109 | INFO | Uploaded 36GiB to GPU in 2.17647s (16.5405GiB/s) | 20260505 22:24:16.109 | INFO | Downloaded 38GiB to GPU in 2.25812s (16.8282GiB/s) | 20260505 22:24:21.250 | INFO | Analyzing chunk frames = [6771200,6771712) | 20260505 22:24:21.250 | INFO | MemInfo: { | 20260505 22:24:21.250 | INFO | "gpu" : " 17 GB", | 20260505 22:24:21.250 | INFO | "huge" : "", | 20260505 22:24:21.250 | INFO | "huge_shared" : " 27 GB", | 20260505 22:24:21.250 | INFO | "malloc" : " 14 GB", | 20260505 22:24:21.250 | INFO | "pinned" : " 29 GB" | 20260505 22:24:21.250 | INFO | } | 20260505 22:24:21.250 | INFO | Perf report for the last 5.1419 seconds: | 20260505 22:24:21.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:24:21.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:21.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:21.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:21.250 | INFO | ANALYSIS : 61.8%, 3.0, 96.6ms, 0 | 20260505 22:24:21.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:21.250 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:21.250 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 22:24:21.250 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:21.250 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:21.251 | INFO | Uploaded 36GiB to GPU in 2.09283s (17.2016GiB/s) | 20260505 22:24:21.251 | INFO | Downloaded 34GiB to GPU in 2.03295s (16.7244GiB/s) | 20260505 22:24:26.392 | INFO | Analyzing chunk frames = [6771712,6772224) | 20260505 22:24:26.392 | INFO | MemInfo: { | 20260505 22:24:26.392 | INFO | "gpu" : " 17 GB", | 20260505 22:24:26.392 | INFO | "huge" : "", | 20260505 22:24:26.393 | INFO | "huge_shared" : " 27 GB", | 20260505 22:24:26.393 | INFO | "malloc" : " 14 GB", | 20260505 22:24:26.393 | INFO | "pinned" : " 29 GB" | 20260505 22:24:26.393 | INFO | } | 20260505 22:24:26.393 | INFO | Perf report for the last 5.14228 seconds: | 20260505 22:24:26.393 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:24:26.393 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:26.393 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:26.393 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:26.393 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260505 22:24:26.393 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:26.393 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:26.393 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:24:26.393 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:26.393 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:26.393 | INFO | Uploaded 36GiB to GPU in 2.14727s (16.7655GiB/s) | 20260505 22:24:26.393 | INFO | Downloaded 34GiB to GPU in 2.05123s (16.5754GiB/s) | 20260505 22:24:31.524 | INFO | Analyzing chunk frames = [6772224,6772736) | 20260505 22:24:31.524 | INFO | MemInfo: { | 20260505 22:24:31.524 | INFO | "gpu" : " 17 GB", | 20260505 22:24:31.524 | INFO | "huge" : "", | 20260505 22:24:31.524 | INFO | "huge_shared" : " 27 GB", | 20260505 22:24:31.524 | INFO | "malloc" : " 14 GB", | 20260505 22:24:31.524 | INFO | "pinned" : " 29 GB" | 20260505 22:24:31.524 | INFO | } | 20260505 22:24:31.524 | INFO | Perf report for the last 5.1318 seconds: | 20260505 22:24:31.524 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:24:31.524 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:31.524 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:31.524 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:31.524 | INFO | ANALYSIS : 61.8%, 3.0, 95.8ms, 0 | 20260505 22:24:31.524 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:31.524 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:31.524 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 22:24:31.524 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:31.525 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:31.525 | INFO | Uploaded 36GiB to GPU in 2.13482s (16.8632GiB/s) | 20260505 22:24:31.525 | INFO | Downloaded 34GiB to GPU in 2.00774s (16.9345GiB/s) | 20260505 22:24:36.665 | INFO | Analyzing chunk frames = [6772736,6773248) | 20260505 22:24:36.665 | INFO | MemInfo: { | 20260505 22:24:36.665 | INFO | "gpu" : " 17 GB", | 20260505 22:24:36.665 | INFO | "huge" : "", | 20260505 22:24:36.665 | INFO | "huge_shared" : " 27 GB", | 20260505 22:24:36.665 | INFO | "malloc" : " 14 GB", | 20260505 22:24:36.665 | INFO | "pinned" : " 29 GB" | 20260505 22:24:36.665 | INFO | } | 20260505 22:24:36.665 | INFO | Perf report for the last 5.14035 seconds: | 20260505 22:24:36.665 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:24:36.665 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:36.665 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:36.665 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:36.665 | INFO | ANALYSIS : 61.6%, 3.0, 96.2ms, 0 | 20260505 22:24:36.665 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:36.665 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:36.665 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:24:36.665 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:36.665 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:36.665 | INFO | Uploaded 36GiB to GPU in 2.13408s (16.8691GiB/s) | 20260505 22:24:36.665 | INFO | Downloaded 34GiB to GPU in 2.0642s (16.4712GiB/s) | 20260505 22:24:41.808 | INFO | Analyzing chunk frames = [6773248,6773760) | 20260505 22:24:41.808 | INFO | MemInfo: { | 20260505 22:24:41.808 | INFO | "gpu" : " 17 GB", | 20260505 22:24:41.808 | INFO | "huge" : "", | 20260505 22:24:41.808 | INFO | "huge_shared" : " 27 GB", | 20260505 22:24:41.808 | INFO | "malloc" : " 14 GB", | 20260505 22:24:41.808 | INFO | "pinned" : " 29 GB" | 20260505 22:24:41.808 | INFO | } | 20260505 22:24:41.808 | INFO | Perf report for the last 5.14337 seconds: | 20260505 22:24:41.808 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:24:41.808 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:41.808 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:41.808 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:41.808 | INFO | ANALYSIS : 61.4%, 3.0, 95.9ms, 0 | 20260505 22:24:41.808 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:41.808 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:41.808 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 22:24:41.808 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:41.808 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:41.808 | INFO | Uploaded 36GiB to GPU in 2.08006s (17.3072GiB/s) | 20260505 22:24:41.808 | INFO | Downloaded 34GiB to GPU in 2.03066s (16.7433GiB/s) | 20260505 22:24:46.947 | INFO | Analyzing chunk frames = [6773760,6774272) | 20260505 22:24:46.947 | INFO | MemInfo: { | 20260505 22:24:46.947 | INFO | "gpu" : " 17 GB", | 20260505 22:24:46.947 | INFO | "huge" : "", | 20260505 22:24:46.947 | INFO | "huge_shared" : " 27 GB", | 20260505 22:24:46.948 | INFO | "malloc" : " 14 GB", | 20260505 22:24:46.948 | INFO | "pinned" : " 29 GB" | 20260505 22:24:46.948 | INFO | } | 20260505 22:24:46.948 | INFO | Perf report for the last 5.13952 seconds: | 20260505 22:24:46.948 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:24:46.948 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:46.948 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:46.948 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:46.948 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 22:24:46.948 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:46.948 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:46.948 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 22:24:46.948 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:46.948 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:46.948 | INFO | Uploaded 36GiB to GPU in 2.04129s (17.6359GiB/s) | 20260505 22:24:46.948 | INFO | Downloaded 34GiB to GPU in 2.04711s (16.6088GiB/s) | 20260505 22:24:52.081 | INFO | Analyzing chunk frames = [6774272,6774784) | 20260505 22:24:52.081 | INFO | MemInfo: { | 20260505 22:24:52.081 | INFO | "gpu" : " 17 GB", | 20260505 22:24:52.081 | INFO | "huge" : "", | 20260505 22:24:52.081 | INFO | "huge_shared" : " 27 GB", | 20260505 22:24:52.081 | INFO | "malloc" : " 14 GB", | 20260505 22:24:52.081 | INFO | "pinned" : " 29 GB" | 20260505 22:24:52.081 | INFO | } | 20260505 22:24:52.082 | INFO | Perf report for the last 5.13381 seconds: | 20260505 22:24:52.082 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:24:52.082 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:52.082 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:52.082 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:52.082 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 22:24:52.082 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:52.082 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:52.082 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 22:24:52.082 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:52.082 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:52.082 | INFO | Uploaded 36GiB to GPU in 2.11468s (17.0238GiB/s) | 20260505 22:24:52.082 | INFO | Downloaded 34GiB to GPU in 2.00793s (16.9329GiB/s) | 20260505 22:24:52.082 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 22:24:52.082 | INFO | Total mem usage: 15.3 GiB | 20260505 22:24:52.082 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 22:24:52.082 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.23%) | 20260505 22:24:52.082 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 22:24:52.082 | INFO | High Water Breakdown: | 20260505 22:24:52.082 | INFO | 61% : 8.64 GiB ( 90.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 22:24:52.082 | INFO | 38.8% : 5.5 GiB ( 82.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 22:24:52.082 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 22:24:52.082 | INFO | ------------------------------------------ | 20260505 22:24:52.082 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:24:52.082 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 22:24:52.082 | INFO | Total mem usage: 26.5 GiB | 20260505 22:24:52.082 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 22:24:52.082 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 22:24:52.082 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 22:24:52.082 | INFO | High Water Breakdown: | 20260505 22:24:52.082 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:24:52.082 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:24:52.082 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:24:52.082 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:24:52.082 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:24:52.082 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:24:52.082 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 22:24:52.082 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 22:24:52.082 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:24:52.082 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 22:24:52.082 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 22:24:52.082 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 22:24:52.082 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 22:24:52.082 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 22:24:52.083 | INFO | ------------------------------------------ | 20260505 22:24:52.083 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:24:52.083 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 22:24:52.083 | INFO | Total mem usage: 25.6 GiB | 20260505 22:24:52.083 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 22:24:52.083 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 22:24:52.083 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 22:24:52.083 | INFO | High Water Breakdown: | 20260505 22:24:52.083 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 22:24:52.083 | INFO | ------------------------------------------ | 20260505 22:24:52.083 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:24:52.083 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 22:24:52.083 | INFO | Total mem usage: 15.7 GiB | 20260505 22:24:52.083 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 22:24:52.083 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 22:24:52.083 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 22:24:52.083 | INFO | High Water Breakdown: | 20260505 22:24:52.083 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 22:24:52.083 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 22:24:52.083 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 22:24:52.083 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 22:24:52.083 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 22:24:52.083 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 22:24:52.083 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 22:24:52.083 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 22:24:52.083 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 22:24:52.083 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:24:52.083 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 22:24:52.083 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:24:52.083 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 22:24:52.083 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 22:24:52.083 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 22:24:52.083 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 22:24:52.083 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:24:52.083 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 22:24:52.083 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:24:52.083 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:24:52.083 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:24:52.083 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 22:24:52.083 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 22:24:52.083 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:24:52.083 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 22:24:52.083 | INFO | ------------------------------------------ | 20260505 22:24:52.083 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:24:52.083 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 22:24:52.083 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:24:52.083 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:24:52.083 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.41e+01, 1.31e+01 / 8.67e-01 | 20260505 22:24:52.083 | INFO | PerfReport: 0.29%, AggregateStats -- 9.24e-02 / 1.04e-01, 1.02e-02 / 2.75e-03 | 20260505 22:24:52.083 | INFO | PerfReport: 11.81%, Baseline -- 3.71e+00 / 4.51e+00, 5.67e-01 / 2.16e-01 | 20260505 22:24:52.083 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.53e+00, 1.88e-01 / 9.82e-02 | 20260505 22:24:52.083 | INFO | PerfReport: 15.98%, DME -- 4.01e+01 / 4.40e+01, 3.98e+00 / 9.93e-01 | 20260505 22:24:52.083 | INFO | PerfReport: 57.22%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.51e+00 / 1.10e+00 | 20260505 22:24:52.083 | INFO | PerfReport: 7.16%, PulseAccumulating -- 2.25e+00 / 2.80e+00, 4.27e-01 / 2.32e-01 | 20260505 22:24:52.083 | INFO | PerfReport: 3.16%, Metrics -- 9.92e-01 / 1.30e+00, 2.70e-01 / 1.44e-01 | 20260505 22:24:52.083 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.44e-01 / 4.05e-01, 5.86e-02 / 3.41e-02 | 20260505 22:24:52.083 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 22:24:52.083 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:24:52.083 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:24:52.083 | INFO | PerfReport: Full Execution -- 9.87e+01 / -nan, 2.49e+01 / 0.00e+00 | 20260505 22:24:52.083 | INFO | PerfReport: 22.34%, Upload -- 2.21e+01 / 0.00e+00, 7.56e+00 / 0.00e+00 | 20260505 22:24:52.083 | INFO | PerfReport: 21.79%, Download -- 2.15e+01 / 0.00e+00, 7.61e+00 / 0.00e+00 | 20260505 22:24:52.083 | INFO | PerfReport: 31.80%, Compute -- 3.14e+01 / 0.00e+00, 1.31e+01 / 0.00e+00 | 20260505 22:24:52.083 | INFO | PerfReport: 2.16%, LockUpload -- 2.13e+00 / 0.00e+00, 7.86e+00 / 0.00e+00 | 20260505 22:24:52.083 | INFO | PerfReport: 1.32%, LockDownload -- 1.31e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260505 22:24:52.083 | INFO | PerfReport: 20.58%, LockCompute -- 2.03e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 22:24:52.083 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.85e-02 / 0.00e+00 | 20260505 22:24:52.217 | INFO | Starting a round of Baz IO | 20260505 22:24:55.378 | INFO | Populating RTMetrics report | 20260505 22:24:55.378 | INFO | RTMetrics report populated | 20260505 22:24:55.378 | INFO | Opening tmp RTMetrics json file | 20260505 22:24:55.380 | INFO | Serializing RTMetrics to json | 20260505 22:24:55.381 | INFO | Serialization done, swapping tmp file now... | 20260505 22:24:55.381 | INFO | RTMetrics done | 20260505 22:24:55.462 | INFO | Finished a round of Baz IO: Wrote 1.932 GiB in 3.244 seconds | 20260505 22:24:55.462 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.50% / 0.00% / 78.50% | 20260505 22:24:55.462 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.125917% / 0% / 7.94729e-05% | 20260505 22:24:55.462 | INFO | Baz IO overall: | 20260505 22:24:55.462 | INFO | BytesWritten : 3900990350990 | 20260505 22:24:55.462 | INFO | NumEvents : 1840066634176 | 20260505 22:24:57.217 | INFO | Analyzing chunk frames = [6774784,6775296) | 20260505 22:24:57.217 | INFO | MemInfo: { | 20260505 22:24:57.217 | INFO | "gpu" : " 17 GB", | 20260505 22:24:57.217 | INFO | "huge" : "", | 20260505 22:24:57.217 | INFO | "huge_shared" : " 27 GB", | 20260505 22:24:57.217 | INFO | "malloc" : " 14 GB", | 20260505 22:24:57.217 | INFO | "pinned" : " 29 GB" | 20260505 22:24:57.217 | INFO | } | 20260505 22:24:57.217 | INFO | Perf report for the last 5.13541 seconds: | 20260505 22:24:57.217 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:24:57.217 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:57.217 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:57.217 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:57.217 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 22:24:57.217 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:24:57.217 | INFO | RT_METRICS : 43.6%, 1.0, 22.8ms, 0 | 20260505 22:24:57.217 | INFO | PRE_HQ : 60.0%, 1.8, 54.6ms, 0 | 20260505 22:24:57.217 | INFO | BAZWRITER : 62.7%, 3.6, 114.7ms, 0 | 20260505 22:24:57.217 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 22:24:57.217 | INFO | Uploaded 36GiB to GPU in 2.17582s (16.5455GiB/s) | 20260505 22:24:57.217 | INFO | Downloaded 38GiB to GPU in 2.2532s (16.8649GiB/s) | 20260505 22:25:02.360 | INFO | Analyzing chunk frames = [6775296,6775808) | 20260505 22:25:02.360 | INFO | MemInfo: { | 20260505 22:25:02.360 | INFO | "gpu" : " 17 GB", | 20260505 22:25:02.360 | INFO | "huge" : "", | 20260505 22:25:02.360 | INFO | "huge_shared" : " 27 GB", | 20260505 22:25:02.360 | INFO | "malloc" : " 14 GB", | 20260505 22:25:02.360 | INFO | "pinned" : " 29 GB" | 20260505 22:25:02.360 | INFO | } | 20260505 22:25:02.360 | INFO | Perf report for the last 5.14327 seconds: | 20260505 22:25:02.360 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:25:02.360 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:02.360 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:02.360 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:02.360 | INFO | ANALYSIS : 62.1%, 3.0, 97.1ms, 0 | 20260505 22:25:02.360 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:02.360 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:02.360 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:25:02.360 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:02.360 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:02.360 | INFO | Uploaded 36GiB to GPU in 2.09374s (17.1941GiB/s) | 20260505 22:25:02.361 | INFO | Downloaded 34GiB to GPU in 2.02728s (16.7713GiB/s) | 20260505 22:25:07.504 | INFO | Analyzing chunk frames = [6775808,6776320) | 20260505 22:25:07.504 | INFO | MemInfo: { | 20260505 22:25:07.504 | INFO | "gpu" : " 17 GB", | 20260505 22:25:07.504 | INFO | "huge" : "", | 20260505 22:25:07.504 | INFO | "huge_shared" : " 27 GB", | 20260505 22:25:07.504 | INFO | "malloc" : " 14 GB", | 20260505 22:25:07.504 | INFO | "pinned" : " 29 GB" | 20260505 22:25:07.504 | INFO | } | 20260505 22:25:07.504 | INFO | Perf report for the last 5.14358 seconds: | 20260505 22:25:07.504 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:25:07.504 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:07.504 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:07.504 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:07.504 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260505 22:25:07.504 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:07.504 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:07.504 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:25:07.504 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:07.504 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:07.504 | INFO | Uploaded 36GiB to GPU in 2.14937s (16.7491GiB/s) | 20260505 22:25:07.504 | INFO | Downloaded 34GiB to GPU in 2.04571s (16.6202GiB/s) | 20260505 22:25:12.642 | INFO | Analyzing chunk frames = [6776320,6776832) | 20260505 22:25:12.642 | INFO | MemInfo: { | 20260505 22:25:12.642 | INFO | "gpu" : " 17 GB", | 20260505 22:25:12.642 | INFO | "huge" : "", | 20260505 22:25:12.642 | INFO | "huge_shared" : " 27 GB", | 20260505 22:25:12.642 | INFO | "malloc" : " 14 GB", | 20260505 22:25:12.642 | INFO | "pinned" : " 29 GB" | 20260505 22:25:12.642 | INFO | } | 20260505 22:25:12.642 | INFO | Perf report for the last 5.13843 seconds: | 20260505 22:25:12.642 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:25:12.642 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:12.642 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:12.642 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:12.642 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260505 22:25:12.642 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:12.642 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:12.642 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:25:12.642 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:12.642 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:12.642 | INFO | Uploaded 36GiB to GPU in 2.14613s (16.7744GiB/s) | 20260505 22:25:12.642 | INFO | Downloaded 34GiB to GPU in 2.02704s (16.7732GiB/s) | 20260505 22:25:17.780 | INFO | Analyzing chunk frames = [6776832,6777344) | 20260505 22:25:17.780 | INFO | MemInfo: { | 20260505 22:25:17.780 | INFO | "gpu" : " 17 GB", | 20260505 22:25:17.780 | INFO | "huge" : "", | 20260505 22:25:17.780 | INFO | "huge_shared" : " 27 GB", | 20260505 22:25:17.780 | INFO | "malloc" : " 14 GB", | 20260505 22:25:17.780 | INFO | "pinned" : " 29 GB" | 20260505 22:25:17.780 | INFO | } | 20260505 22:25:17.780 | INFO | Perf report for the last 5.13811 seconds: | 20260505 22:25:17.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:25:17.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:17.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:17.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:17.780 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260505 22:25:17.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:17.780 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:17.780 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:25:17.780 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:17.780 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:17.780 | INFO | Uploaded 36GiB to GPU in 2.09996s (17.1432GiB/s) | 20260505 22:25:17.781 | INFO | Downloaded 34GiB to GPU in 2.04775s (16.6036GiB/s) | 20260505 22:25:22.916 | INFO | Analyzing chunk frames = [6777344,6777856) | 20260505 22:25:22.916 | INFO | MemInfo: { | 20260505 22:25:22.916 | INFO | "gpu" : " 17 GB", | 20260505 22:25:22.916 | INFO | "huge" : "", | 20260505 22:25:22.916 | INFO | "huge_shared" : " 27 GB", | 20260505 22:25:22.916 | INFO | "malloc" : " 14 GB", | 20260505 22:25:22.916 | INFO | "pinned" : " 29 GB" | 20260505 22:25:22.916 | INFO | } | 20260505 22:25:22.916 | INFO | Perf report for the last 5.13545 seconds: | 20260505 22:25:22.916 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:25:22.916 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:22.916 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:22.916 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:22.916 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260505 22:25:22.916 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:22.916 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:22.916 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 22:25:22.916 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:22.916 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:22.916 | INFO | Uploaded 36GiB to GPU in 2.07525s (17.3473GiB/s) | 20260505 22:25:22.916 | INFO | Downloaded 34GiB to GPU in 2.03842s (16.6796GiB/s) | 20260505 22:25:28.061 | INFO | Analyzing chunk frames = [6777856,6778368) | 20260505 22:25:28.061 | INFO | MemInfo: { | 20260505 22:25:28.061 | INFO | "gpu" : " 17 GB", | 20260505 22:25:28.062 | INFO | "huge" : "", | 20260505 22:25:28.062 | INFO | "huge_shared" : " 27 GB", | 20260505 22:25:28.062 | INFO | "malloc" : " 14 GB", | 20260505 22:25:28.062 | INFO | "pinned" : " 29 GB" | 20260505 22:25:28.062 | INFO | } | 20260505 22:25:28.062 | INFO | Perf report for the last 5.14574 seconds: | 20260505 22:25:28.062 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:25:28.062 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:28.062 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:28.062 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:28.062 | INFO | ANALYSIS : 62.7%, 3.0, 98.1ms, 0 | 20260505 22:25:28.062 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:28.062 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:28.062 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 22:25:28.062 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:28.062 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:28.062 | INFO | Uploaded 36GiB to GPU in 2.14766s (16.7624GiB/s) | 20260505 22:25:28.062 | INFO | Downloaded 34GiB to GPU in 2.11126s (16.1041GiB/s) | 20260505 22:25:33.206 | INFO | Analyzing chunk frames = [6778368,6778880) | 20260505 22:25:33.206 | INFO | MemInfo: { | 20260505 22:25:33.206 | INFO | "gpu" : " 17 GB", | 20260505 22:25:33.206 | INFO | "huge" : "", | 20260505 22:25:33.206 | INFO | "huge_shared" : " 27 GB", | 20260505 22:25:33.206 | INFO | "malloc" : " 14 GB", | 20260505 22:25:33.206 | INFO | "pinned" : " 29 GB" | 20260505 22:25:33.206 | INFO | } | 20260505 22:25:33.206 | INFO | Perf report for the last 5.1444 seconds: | 20260505 22:25:33.206 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:25:33.206 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:33.206 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:33.206 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:33.206 | INFO | ANALYSIS : 62.0%, 3.0, 97.0ms, 0 | 20260505 22:25:33.206 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:33.206 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:33.206 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 22:25:33.206 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:33.206 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:33.206 | INFO | Uploaded 36GiB to GPU in 2.116s (17.0132GiB/s) | 20260505 22:25:33.206 | INFO | Downloaded 34GiB to GPU in 2.004s (16.9661GiB/s) | 20260505 22:25:33.325 | INFO | Starting a round of Baz IO | 20260505 22:25:36.476 | INFO | Populating RTMetrics report | 20260505 22:25:36.477 | INFO | RTMetrics report populated | 20260505 22:25:36.477 | INFO | Opening tmp RTMetrics json file | 20260505 22:25:36.479 | INFO | Serializing RTMetrics to json | 20260505 22:25:36.480 | INFO | Serialization done, swapping tmp file now... | 20260505 22:25:36.481 | INFO | RTMetrics done | 20260505 22:25:36.559 | INFO | Finished a round of Baz IO: Wrote 1.931 GiB in 3.234 seconds | 20260505 22:25:36.559 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.50% / 0.00% / 78.50% | 20260505 22:25:36.559 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.12577% / 3.97364e-06% / 7.15256e-05% | 20260505 22:25:36.559 | INFO | Baz IO overall: | 20260505 22:25:36.559 | INFO | BytesWritten : 3903063886124 | 20260505 22:25:36.559 | INFO | NumEvents : 1841020913432 | 20260505 22:25:38.340 | INFO | Analyzing chunk frames = [6778880,6779392) | 20260505 22:25:38.340 | INFO | MemInfo: { | 20260505 22:25:38.340 | INFO | "gpu" : " 17 GB", | 20260505 22:25:38.340 | INFO | "huge" : "", | 20260505 22:25:38.340 | INFO | "huge_shared" : " 27 GB", | 20260505 22:25:38.340 | INFO | "malloc" : " 14 GB", | 20260505 22:25:38.340 | INFO | "pinned" : " 29 GB" | 20260505 22:25:38.340 | INFO | } | 20260505 22:25:38.340 | INFO | Perf report for the last 5.13368 seconds: | 20260505 22:25:38.340 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:25:38.340 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:38.340 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:38.340 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:38.340 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260505 22:25:38.340 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:38.340 | INFO | RT_METRICS : 43.6%, 1.0, 22.8ms, 0 | 20260505 22:25:38.340 | INFO | PRE_HQ : 59.8%, 1.8, 54.6ms, 0 | 20260505 22:25:38.340 | INFO | BAZWRITER : 62.0%, 3.6, 116.1ms, 0 | 20260505 22:25:38.340 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 22:25:38.340 | INFO | Uploaded 36GiB to GPU in 2.11922s (16.9874GiB/s) | 20260505 22:25:38.340 | INFO | Downloaded 38GiB to GPU in 2.21295s (17.1716GiB/s) | 20260505 22:25:43.475 | INFO | Analyzing chunk frames = [6779392,6779904) | 20260505 22:25:43.475 | INFO | MemInfo: { | 20260505 22:25:43.475 | INFO | "gpu" : " 17 GB", | 20260505 22:25:43.475 | INFO | "huge" : "", | 20260505 22:25:43.475 | INFO | "huge_shared" : " 27 GB", | 20260505 22:25:43.475 | INFO | "malloc" : " 14 GB", | 20260505 22:25:43.475 | INFO | "pinned" : " 29 GB" | 20260505 22:25:43.475 | INFO | } | 20260505 22:25:43.475 | INFO | Perf report for the last 5.13535 seconds: | 20260505 22:25:43.475 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:25:43.475 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:43.475 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:43.475 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:43.475 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 22:25:43.475 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:43.475 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:43.475 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 22:25:43.475 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:43.475 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:43.475 | INFO | Uploaded 36GiB to GPU in 2.08752s (17.2453GiB/s) | 20260505 22:25:43.476 | INFO | Downloaded 34GiB to GPU in 2.02847s (16.7614GiB/s) | 20260505 22:25:48.615 | INFO | Analyzing chunk frames = [6779904,6780416) | 20260505 22:25:48.615 | INFO | MemInfo: { | 20260505 22:25:48.615 | INFO | "gpu" : " 17 GB", | 20260505 22:25:48.615 | INFO | "huge" : "", | 20260505 22:25:48.615 | INFO | "huge_shared" : " 27 GB", | 20260505 22:25:48.615 | INFO | "malloc" : " 14 GB", | 20260505 22:25:48.615 | INFO | "pinned" : " 29 GB" | 20260505 22:25:48.615 | INFO | } | 20260505 22:25:48.615 | INFO | Perf report for the last 5.14005 seconds: | 20260505 22:25:48.615 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:25:48.615 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:48.615 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:48.615 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:48.615 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260505 22:25:48.615 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:48.615 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:48.615 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:25:48.615 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:48.615 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:48.615 | INFO | Uploaded 36GiB to GPU in 2.14384s (16.7923GiB/s) | 20260505 22:25:48.615 | INFO | Downloaded 34GiB to GPU in 2.03246s (16.7285GiB/s) | 20260505 22:25:53.758 | INFO | Analyzing chunk frames = [6780416,6780928) | 20260505 22:25:53.758 | INFO | MemInfo: { | 20260505 22:25:53.758 | INFO | "gpu" : " 17 GB", | 20260505 22:25:53.758 | INFO | "huge" : "", | 20260505 22:25:53.758 | INFO | "huge_shared" : " 27 GB", | 20260505 22:25:53.758 | INFO | "malloc" : " 14 GB", | 20260505 22:25:53.758 | INFO | "pinned" : " 29 GB" | 20260505 22:25:53.758 | INFO | } | 20260505 22:25:53.758 | INFO | Perf report for the last 5.14284 seconds: | 20260505 22:25:53.758 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:25:53.758 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:53.758 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:53.758 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:53.758 | INFO | ANALYSIS : 61.8%, 3.0, 96.0ms, 0 | 20260505 22:25:53.758 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:53.758 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:53.758 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:25:53.758 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:53.758 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:53.758 | INFO | Uploaded 36GiB to GPU in 2.13391s (16.8705GiB/s) | 20260505 22:25:53.758 | INFO | Downloaded 34GiB to GPU in 2.02603s (16.7816GiB/s) | 20260505 22:25:58.903 | INFO | Analyzing chunk frames = [6780928,6781440) | 20260505 22:25:58.903 | INFO | MemInfo: { | 20260505 22:25:58.903 | INFO | "gpu" : " 17 GB", | 20260505 22:25:58.903 | INFO | "huge" : "", | 20260505 22:25:58.903 | INFO | "huge_shared" : " 27 GB", | 20260505 22:25:58.903 | INFO | "malloc" : " 14 GB", | 20260505 22:25:58.903 | INFO | "pinned" : " 29 GB" | 20260505 22:25:58.903 | INFO | } | 20260505 22:25:58.903 | INFO | Perf report for the last 5.14511 seconds: | 20260505 22:25:58.903 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:25:58.903 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:58.903 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:58.903 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:58.903 | INFO | ANALYSIS : 61.1%, 3.0, 95.5ms, 0 | 20260505 22:25:58.903 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:58.903 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:58.903 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:25:58.903 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:58.903 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:25:58.903 | INFO | Uploaded 36GiB to GPU in 2.09781s (17.1608GiB/s) | 20260505 22:25:58.903 | INFO | Downloaded 34GiB to GPU in 2.04099s (16.6586GiB/s) | 20260505 22:26:04.032 | INFO | Analyzing chunk frames = [6781440,6781952) | 20260505 22:26:04.032 | INFO | MemInfo: { | 20260505 22:26:04.032 | INFO | "gpu" : " 17 GB", | 20260505 22:26:04.032 | INFO | "huge" : "", | 20260505 22:26:04.032 | INFO | "huge_shared" : " 27 GB", | 20260505 22:26:04.032 | INFO | "malloc" : " 14 GB", | 20260505 22:26:04.032 | INFO | "pinned" : " 29 GB" | 20260505 22:26:04.032 | INFO | } | 20260505 22:26:04.032 | INFO | Perf report for the last 5.12923 seconds: | 20260505 22:26:04.032 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:26:04.032 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:04.032 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:04.032 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:04.032 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260505 22:26:04.032 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:04.032 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:04.032 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260505 22:26:04.032 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:04.032 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:04.033 | INFO | Uploaded 36GiB to GPU in 2.07632s (17.3384GiB/s) | 20260505 22:26:04.033 | INFO | Downloaded 34GiB to GPU in 2.03513s (16.7065GiB/s) | 20260505 22:26:09.167 | INFO | Analyzing chunk frames = [6781952,6782464) | 20260505 22:26:09.167 | INFO | MemInfo: { | 20260505 22:26:09.167 | INFO | "gpu" : " 17 GB", | 20260505 22:26:09.167 | INFO | "huge" : "", | 20260505 22:26:09.167 | INFO | "huge_shared" : " 27 GB", | 20260505 22:26:09.167 | INFO | "malloc" : " 14 GB", | 20260505 22:26:09.167 | INFO | "pinned" : " 29 GB" | 20260505 22:26:09.167 | INFO | } | 20260505 22:26:09.167 | INFO | Perf report for the last 5.13491 seconds: | 20260505 22:26:09.167 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:26:09.167 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:09.167 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:09.167 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:09.167 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 22:26:09.167 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:09.167 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:09.167 | INFO | PRE_HQ : 27.5%, 1.0, 14.2ms, 0 | 20260505 22:26:09.167 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:09.167 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:09.167 | INFO | Uploaded 36GiB to GPU in 2.15321s (16.7192GiB/s) | 20260505 22:26:09.168 | INFO | Downloaded 34GiB to GPU in 2.15188s (15.8002GiB/s) | 20260505 22:26:14.313 | INFO | Analyzing chunk frames = [6782464,6782976) | 20260505 22:26:14.313 | INFO | MemInfo: { | 20260505 22:26:14.313 | INFO | "gpu" : " 17 GB", | 20260505 22:26:14.313 | INFO | "huge" : "", | 20260505 22:26:14.313 | INFO | "huge_shared" : " 27 GB", | 20260505 22:26:14.313 | INFO | "malloc" : " 14 GB", | 20260505 22:26:14.314 | INFO | "pinned" : " 29 GB" | 20260505 22:26:14.314 | INFO | } | 20260505 22:26:14.314 | INFO | Perf report for the last 5.14638 seconds: | 20260505 22:26:14.314 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:26:14.314 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:14.314 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:14.314 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:14.314 | INFO | ANALYSIS : 62.3%, 3.0, 97.4ms, 0 | 20260505 22:26:14.314 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:14.314 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:14.314 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 22:26:14.314 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:14.314 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:14.314 | INFO | Uploaded 36GiB to GPU in 2.12448s (16.9453GiB/s) | 20260505 22:26:14.314 | INFO | Downloaded 34GiB to GPU in 2.01474s (16.8756GiB/s) | 20260505 22:26:14.314 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 22:26:14.314 | INFO | Total mem usage: 15.3 GiB | 20260505 22:26:14.314 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 22:26:14.314 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.23%) | 20260505 22:26:14.314 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 22:26:14.314 | INFO | High Water Breakdown: | 20260505 22:26:14.314 | INFO | 61% : 8.64 GiB ( 90.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 22:26:14.314 | INFO | 38.8% : 5.5 GiB ( 82.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 22:26:14.314 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 22:26:14.314 | INFO | ------------------------------------------ | 20260505 22:26:14.314 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:26:14.314 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 22:26:14.314 | INFO | Total mem usage: 26.5 GiB | 20260505 22:26:14.314 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 22:26:14.314 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 22:26:14.314 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 22:26:14.314 | INFO | High Water Breakdown: | 20260505 22:26:14.314 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:26:14.314 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:26:14.314 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:26:14.314 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:26:14.314 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:26:14.314 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:26:14.314 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 22:26:14.314 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 22:26:14.314 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:26:14.314 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 22:26:14.314 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 22:26:14.314 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 22:26:14.314 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 22:26:14.314 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 22:26:14.315 | INFO | ------------------------------------------ | 20260505 22:26:14.315 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:26:14.315 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 22:26:14.315 | INFO | Total mem usage: 25.6 GiB | 20260505 22:26:14.315 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 22:26:14.315 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 22:26:14.315 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 22:26:14.315 | INFO | High Water Breakdown: | 20260505 22:26:14.315 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 22:26:14.315 | INFO | ------------------------------------------ | 20260505 22:26:14.315 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:26:14.315 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 22:26:14.315 | INFO | Total mem usage: 15.7 GiB | 20260505 22:26:14.315 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 22:26:14.315 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 22:26:14.315 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 22:26:14.315 | INFO | High Water Breakdown: | 20260505 22:26:14.315 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 22:26:14.315 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 22:26:14.315 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 22:26:14.315 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 22:26:14.315 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 22:26:14.315 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 22:26:14.315 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 22:26:14.315 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 22:26:14.315 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 22:26:14.315 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:26:14.315 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 22:26:14.315 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:26:14.315 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 22:26:14.315 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 22:26:14.315 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 22:26:14.315 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 22:26:14.315 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:26:14.315 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 22:26:14.315 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:26:14.315 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:26:14.315 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:26:14.315 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 22:26:14.315 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 22:26:14.315 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:26:14.315 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 22:26:14.315 | INFO | ------------------------------------------ | 20260505 22:26:14.315 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:26:14.315 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 22:26:14.315 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:26:14.315 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:26:14.315 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.43e+01, 1.34e+01 / 1.15e+00 | 20260505 22:26:14.315 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.05e-01, 1.02e-02 / 5.02e-03 | 20260505 22:26:14.315 | INFO | PerfReport: 11.81%, Baseline -- 3.71e+00 / 4.56e+00, 5.70e-01 / 1.62e-01 | 20260505 22:26:14.315 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.51e+00, 1.90e-01 / 1.02e-01 | 20260505 22:26:14.315 | INFO | PerfReport: 15.97%, DME -- 4.01e+01 / 4.41e+01, 3.73e+00 / 1.24e+00 | 20260505 22:26:14.315 | INFO | PerfReport: 57.26%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.50e+00 / 1.15e+00 | 20260505 22:26:14.315 | INFO | PerfReport: 7.14%, PulseAccumulating -- 2.24e+00 / 2.78e+00, 4.27e-01 / 2.16e-01 | 20260505 22:26:14.315 | INFO | PerfReport: 3.16%, Metrics -- 9.91e-01 / 1.27e+00, 2.68e-01 / 1.21e-01 | 20260505 22:26:14.315 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.44e-01 / 4.03e-01, 5.88e-02 / 3.66e-02 | 20260505 22:26:14.315 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 22:26:14.315 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:26:14.315 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:26:14.315 | INFO | PerfReport: Full Execution -- 9.88e+01 / -nan, 2.49e+01 / 0.00e+00 | 20260505 22:26:14.315 | INFO | PerfReport: 22.37%, Upload -- 2.21e+01 / 0.00e+00, 7.62e+00 / 0.00e+00 | 20260505 22:26:14.315 | INFO | PerfReport: 21.82%, Download -- 2.15e+01 / 0.00e+00, 7.63e+00 / 0.00e+00 | 20260505 22:26:14.315 | INFO | PerfReport: 31.78%, Compute -- 3.14e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 22:26:14.315 | INFO | PerfReport: 2.20%, LockUpload -- 2.17e+00 / 0.00e+00, 7.98e+00 / 0.00e+00 | 20260505 22:26:14.315 | INFO | PerfReport: 1.33%, LockDownload -- 1.31e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260505 22:26:14.315 | INFO | PerfReport: 20.49%, LockCompute -- 2.02e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260505 22:26:14.315 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.68e-02 / 0.00e+00 | 20260505 22:26:14.435 | INFO | Starting a round of Baz IO | 20260505 22:26:17.583 | INFO | Populating RTMetrics report | 20260505 22:26:17.583 | INFO | RTMetrics report populated | 20260505 22:26:17.583 | INFO | Opening tmp RTMetrics json file | 20260505 22:26:17.584 | INFO | Serializing RTMetrics to json | 20260505 22:26:17.585 | INFO | Serialization done, swapping tmp file now... | 20260505 22:26:17.586 | INFO | RTMetrics done | 20260505 22:26:17.663 | INFO | Finished a round of Baz IO: Wrote 1.93 GiB in 3.228 seconds | 20260505 22:26:17.664 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.50% / 0.00% / 78.50% | 20260505 22:26:17.664 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.124641% / 3.97364e-06% / 7.94729e-05% | 20260505 22:26:17.664 | INFO | Baz IO overall: | 20260505 22:26:17.664 | INFO | BytesWritten : 3905135858312 | 20260505 22:26:17.664 | INFO | NumEvents : 1841974243711 | 20260505 22:26:19.458 | INFO | Analyzing chunk frames = [6782976,6783488) | 20260505 22:26:19.458 | INFO | MemInfo: { | 20260505 22:26:19.458 | INFO | "gpu" : " 17 GB", | 20260505 22:26:19.458 | INFO | "huge" : "", | 20260505 22:26:19.458 | INFO | "huge_shared" : " 27 GB", | 20260505 22:26:19.458 | INFO | "malloc" : " 14 GB", | 20260505 22:26:19.458 | INFO | "pinned" : " 29 GB" | 20260505 22:26:19.458 | INFO | } | 20260505 22:26:19.458 | INFO | Perf report for the last 5.14436 seconds: | 20260505 22:26:19.458 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:26:19.458 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:19.458 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:19.458 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:19.458 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260505 22:26:19.458 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:19.458 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260505 22:26:19.458 | INFO | PRE_HQ : 59.9%, 1.8, 54.8ms, 0 | 20260505 22:26:19.458 | INFO | BAZWRITER : 61.8%, 3.6, 115.4ms, 0 | 20260505 22:26:19.458 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 22:26:19.458 | INFO | Uploaded 36GiB to GPU in 2.1675s (16.609GiB/s) | 20260505 22:26:19.458 | INFO | Downloaded 38GiB to GPU in 2.23515s (17.0011GiB/s) | 20260505 22:26:24.589 | INFO | Analyzing chunk frames = [6783488,6784000) | 20260505 22:26:24.589 | INFO | MemInfo: { | 20260505 22:26:24.589 | INFO | "gpu" : " 17 GB", | 20260505 22:26:24.589 | INFO | "huge" : "", | 20260505 22:26:24.589 | INFO | "huge_shared" : " 27 GB", | 20260505 22:26:24.589 | INFO | "malloc" : " 14 GB", | 20260505 22:26:24.589 | INFO | "pinned" : " 29 GB" | 20260505 22:26:24.589 | INFO | } | 20260505 22:26:24.589 | INFO | Perf report for the last 5.13075 seconds: | 20260505 22:26:24.589 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:26:24.589 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:24.589 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:24.589 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:24.589 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260505 22:26:24.589 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:24.589 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:24.589 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 22:26:24.589 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:24.589 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:24.589 | INFO | Uploaded 36GiB to GPU in 2.10122s (17.1329GiB/s) | 20260505 22:26:24.589 | INFO | Downloaded 34GiB to GPU in 2.0353s (16.7051GiB/s) | 20260505 22:26:29.723 | INFO | Analyzing chunk frames = [6784000,6784512) | 20260505 22:26:29.723 | INFO | MemInfo: { | 20260505 22:26:29.723 | INFO | "gpu" : " 17 GB", | 20260505 22:26:29.723 | INFO | "huge" : "", | 20260505 22:26:29.723 | INFO | "huge_shared" : " 27 GB", | 20260505 22:26:29.723 | INFO | "malloc" : " 14 GB", | 20260505 22:26:29.723 | INFO | "pinned" : " 29 GB" | 20260505 22:26:29.723 | INFO | } | 20260505 22:26:29.723 | INFO | Perf report for the last 5.13398 seconds: | 20260505 22:26:29.723 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:26:29.723 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:29.723 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:29.723 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:29.723 | INFO | ANALYSIS : 62.0%, 3.0, 96.3ms, 0 | 20260505 22:26:29.723 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:29.723 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:29.723 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260505 22:26:29.723 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:29.723 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:29.723 | INFO | Uploaded 36GiB to GPU in 2.15102s (16.7362GiB/s) | 20260505 22:26:29.723 | INFO | Downloaded 34GiB to GPU in 2.04726s (16.6075GiB/s) | 20260505 22:26:34.867 | INFO | Analyzing chunk frames = [6784512,6785024) | 20260505 22:26:34.867 | INFO | MemInfo: { | 20260505 22:26:34.867 | INFO | "gpu" : " 17 GB", | 20260505 22:26:34.867 | INFO | "huge" : "", | 20260505 22:26:34.867 | INFO | "huge_shared" : " 27 GB", | 20260505 22:26:34.867 | INFO | "malloc" : " 14 GB", | 20260505 22:26:34.867 | INFO | "pinned" : " 29 GB" | 20260505 22:26:34.867 | INFO | } | 20260505 22:26:34.867 | INFO | Perf report for the last 5.14451 seconds: | 20260505 22:26:34.867 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:26:34.867 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:34.867 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:34.867 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:34.867 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260505 22:26:34.867 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:34.867 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:34.867 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:26:34.867 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:34.867 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:34.868 | INFO | Uploaded 36GiB to GPU in 2.21044s (16.2864GiB/s) | 20260505 22:26:34.868 | INFO | Downloaded 34GiB to GPU in 2.06823s (16.4392GiB/s) | 20260505 22:26:40.010 | INFO | Analyzing chunk frames = [6785024,6785536) | 20260505 22:26:40.010 | INFO | MemInfo: { | 20260505 22:26:40.010 | INFO | "gpu" : " 17 GB", | 20260505 22:26:40.011 | INFO | "huge" : "", | 20260505 22:26:40.011 | INFO | "huge_shared" : " 27 GB", | 20260505 22:26:40.011 | INFO | "malloc" : " 14 GB", | 20260505 22:26:40.011 | INFO | "pinned" : " 29 GB" | 20260505 22:26:40.011 | INFO | } | 20260505 22:26:40.011 | INFO | Perf report for the last 5.14367 seconds: | 20260505 22:26:40.011 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:26:40.011 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:40.011 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:40.011 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:40.011 | INFO | ANALYSIS : 61.5%, 3.0, 96.1ms, 0 | 20260505 22:26:40.011 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:40.011 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:40.011 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:26:40.011 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:40.011 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:40.011 | INFO | Uploaded 36GiB to GPU in 2.13213s (16.8845GiB/s) | 20260505 22:26:40.011 | INFO | Downloaded 34GiB to GPU in 2.06035s (16.502GiB/s) | 20260505 22:26:45.150 | INFO | Analyzing chunk frames = [6785536,6786048) | 20260505 22:26:45.150 | INFO | MemInfo: { | 20260505 22:26:45.150 | INFO | "gpu" : " 17 GB", | 20260505 22:26:45.151 | INFO | "huge" : "", | 20260505 22:26:45.151 | INFO | "huge_shared" : " 27 GB", | 20260505 22:26:45.151 | INFO | "malloc" : " 14 GB", | 20260505 22:26:45.151 | INFO | "pinned" : " 29 GB" | 20260505 22:26:45.151 | INFO | } | 20260505 22:26:45.151 | INFO | Perf report for the last 5.13965 seconds: | 20260505 22:26:45.151 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:26:45.151 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:45.151 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:45.151 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:45.151 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260505 22:26:45.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:45.151 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:45.151 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:26:45.151 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:45.151 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:45.151 | INFO | Uploaded 36GiB to GPU in 2.09611s (17.1746GiB/s) | 20260505 22:26:45.151 | INFO | Downloaded 34GiB to GPU in 2.04043s (16.6631GiB/s) | 20260505 22:26:50.290 | INFO | Analyzing chunk frames = [6786048,6786560) | 20260505 22:26:50.290 | INFO | MemInfo: { | 20260505 22:26:50.290 | INFO | "gpu" : " 17 GB", | 20260505 22:26:50.290 | INFO | "huge" : "", | 20260505 22:26:50.290 | INFO | "huge_shared" : " 27 GB", | 20260505 22:26:50.290 | INFO | "malloc" : " 14 GB", | 20260505 22:26:50.290 | INFO | "pinned" : " 29 GB" | 20260505 22:26:50.290 | INFO | } | 20260505 22:26:50.290 | INFO | Perf report for the last 5.13961 seconds: | 20260505 22:26:50.290 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:26:50.290 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:50.290 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:50.290 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:50.290 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 22:26:50.290 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:50.290 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:50.290 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 22:26:50.290 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:50.290 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:50.290 | INFO | Uploaded 36GiB to GPU in 2.1602s (16.6651GiB/s) | 20260505 22:26:50.291 | INFO | Downloaded 34GiB to GPU in 2.12029s (16.0355GiB/s) | 20260505 22:26:55.431 | INFO | Analyzing chunk frames = [6786560,6787072) | 20260505 22:26:55.431 | INFO | MemInfo: { | 20260505 22:26:55.431 | INFO | "gpu" : " 17 GB", | 20260505 22:26:55.431 | INFO | "huge" : "", | 20260505 22:26:55.431 | INFO | "huge_shared" : " 27 GB", | 20260505 22:26:55.431 | INFO | "malloc" : " 14 GB", | 20260505 22:26:55.431 | INFO | "pinned" : " 29 GB" | 20260505 22:26:55.431 | INFO | } | 20260505 22:26:55.431 | INFO | Perf report for the last 5.14116 seconds: | 20260505 22:26:55.431 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:26:55.431 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:55.431 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:55.431 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:55.431 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 22:26:55.431 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:55.431 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:55.431 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 22:26:55.431 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:55.431 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:26:55.432 | INFO | Uploaded 36GiB to GPU in 2.11526s (17.0192GiB/s) | 20260505 22:26:55.432 | INFO | Downloaded 34GiB to GPU in 1.99921s (17.0067GiB/s) | 20260505 22:26:55.556 | INFO | Starting a round of Baz IO | 20260505 22:26:58.706 | INFO | Populating RTMetrics report | 20260505 22:26:58.707 | INFO | RTMetrics report populated | 20260505 22:26:58.707 | INFO | Opening tmp RTMetrics json file | 20260505 22:26:58.709 | INFO | Serializing RTMetrics to json | 20260505 22:26:58.710 | INFO | Serialization done, swapping tmp file now... | 20260505 22:26:58.710 | INFO | RTMetrics done | 20260505 22:26:58.788 | INFO | Finished a round of Baz IO: Wrote 1.929 GiB in 3.232 seconds | 20260505 22:26:58.788 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.50% / 0.00% / 78.50% | 20260505 22:26:58.788 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.124633% / 3.97364e-06% / 0.000107288% | 20260505 22:26:58.788 | INFO | Baz IO overall: | 20260505 22:26:58.788 | INFO | BytesWritten : 3907207038638 | 20260505 22:26:58.788 | INFO | NumEvents : 1842926804288 | 20260505 22:27:00.570 | INFO | Analyzing chunk frames = [6787072,6787584) | 20260505 22:27:00.570 | INFO | MemInfo: { | 20260505 22:27:00.570 | INFO | "gpu" : " 17 GB", | 20260505 22:27:00.570 | INFO | "huge" : "", | 20260505 22:27:00.570 | INFO | "huge_shared" : " 27 GB", | 20260505 22:27:00.570 | INFO | "malloc" : " 14 GB", | 20260505 22:27:00.570 | INFO | "pinned" : " 29 GB" | 20260505 22:27:00.570 | INFO | } | 20260505 22:27:00.570 | INFO | Perf report for the last 5.13839 seconds: | 20260505 22:27:00.570 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:27:00.570 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:00.570 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:00.570 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:00.570 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 22:27:00.570 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:00.570 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260505 22:27:00.570 | INFO | PRE_HQ : 59.9%, 1.8, 54.9ms, 0 | 20260505 22:27:00.570 | INFO | BAZWRITER : 62.1%, 3.6, 116.5ms, 0 | 20260505 22:27:00.570 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 22:27:00.570 | INFO | Uploaded 36GiB to GPU in 2.13424s (16.8679GiB/s) | 20260505 22:27:00.570 | INFO | Downloaded 38GiB to GPU in 2.22452s (17.0824GiB/s) | 20260505 22:27:05.706 | INFO | Analyzing chunk frames = [6787584,6788096) | 20260505 22:27:05.706 | INFO | MemInfo: { | 20260505 22:27:05.706 | INFO | "gpu" : " 17 GB", | 20260505 22:27:05.706 | INFO | "huge" : "", | 20260505 22:27:05.706 | INFO | "huge_shared" : " 27 GB", | 20260505 22:27:05.706 | INFO | "malloc" : " 14 GB", | 20260505 22:27:05.706 | INFO | "pinned" : " 29 GB" | 20260505 22:27:05.706 | INFO | } | 20260505 22:27:05.706 | INFO | Perf report for the last 5.1363 seconds: | 20260505 22:27:05.706 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:27:05.706 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:05.706 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:05.706 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:05.706 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260505 22:27:05.706 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:05.706 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:05.706 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:27:05.706 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:05.706 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:05.706 | INFO | Uploaded 36GiB to GPU in 2.11032s (17.059GiB/s) | 20260505 22:27:05.706 | INFO | Downloaded 34GiB to GPU in 2.04081s (16.66GiB/s) | 20260505 22:27:10.851 | INFO | Analyzing chunk frames = [6788096,6788608) | 20260505 22:27:10.851 | INFO | MemInfo: { | 20260505 22:27:10.851 | INFO | "gpu" : " 17 GB", | 20260505 22:27:10.851 | INFO | "huge" : "", | 20260505 22:27:10.851 | INFO | "huge_shared" : " 27 GB", | 20260505 22:27:10.851 | INFO | "malloc" : " 14 GB", | 20260505 22:27:10.851 | INFO | "pinned" : " 29 GB" | 20260505 22:27:10.851 | INFO | } | 20260505 22:27:10.851 | INFO | Perf report for the last 5.14483 seconds: | 20260505 22:27:10.851 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:27:10.851 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:10.851 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:10.851 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:10.851 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260505 22:27:10.851 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:10.851 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:10.851 | INFO | PRE_HQ : 27.0%, 1.0, 14.1ms, 0 | 20260505 22:27:10.851 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:10.851 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:10.851 | INFO | Uploaded 36GiB to GPU in 2.15042s (16.7409GiB/s) | 20260505 22:27:10.851 | INFO | Downloaded 34GiB to GPU in 2.04656s (16.6132GiB/s) | 20260505 22:27:15.983 | INFO | Analyzing chunk frames = [6788608,6789120) | 20260505 22:27:15.983 | INFO | MemInfo: { | 20260505 22:27:15.983 | INFO | "gpu" : " 17 GB", | 20260505 22:27:15.983 | INFO | "huge" : "", | 20260505 22:27:15.983 | INFO | "huge_shared" : " 27 GB", | 20260505 22:27:15.983 | INFO | "malloc" : " 14 GB", | 20260505 22:27:15.983 | INFO | "pinned" : " 29 GB" | 20260505 22:27:15.983 | INFO | } | 20260505 22:27:15.983 | INFO | Perf report for the last 5.13237 seconds: | 20260505 22:27:15.983 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:27:15.983 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:15.983 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:15.983 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:15.983 | INFO | ANALYSIS : 61.9%, 3.0, 96.0ms, 0 | 20260505 22:27:15.983 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:15.983 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:15.983 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 22:27:15.983 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:15.983 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:15.984 | INFO | Uploaded 36GiB to GPU in 2.13321s (16.876GiB/s) | 20260505 22:27:15.984 | INFO | Downloaded 34GiB to GPU in 2.02574s (16.784GiB/s) | 20260505 22:27:21.124 | INFO | Analyzing chunk frames = [6789120,6789632) | 20260505 22:27:21.124 | INFO | MemInfo: { | 20260505 22:27:21.124 | INFO | "gpu" : " 17 GB", | 20260505 22:27:21.124 | INFO | "huge" : "", | 20260505 22:27:21.124 | INFO | "huge_shared" : " 27 GB", | 20260505 22:27:21.124 | INFO | "malloc" : " 14 GB", | 20260505 22:27:21.124 | INFO | "pinned" : " 29 GB" | 20260505 22:27:21.124 | INFO | } | 20260505 22:27:21.125 | INFO | Perf report for the last 5.14115 seconds: | 20260505 22:27:21.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:27:21.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:21.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:21.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:21.125 | INFO | ANALYSIS : 61.5%, 3.0, 96.1ms, 0 | 20260505 22:27:21.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:21.125 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:21.125 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 22:27:21.125 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:21.125 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:21.125 | INFO | Uploaded 36GiB to GPU in 2.13431s (16.8672GiB/s) | 20260505 22:27:21.125 | INFO | Downloaded 34GiB to GPU in 2.05961s (16.508GiB/s) | 20260505 22:27:26.269 | INFO | Analyzing chunk frames = [6789632,6790144) | 20260505 22:27:26.269 | INFO | MemInfo: { | 20260505 22:27:26.269 | INFO | "gpu" : " 17 GB", | 20260505 22:27:26.269 | INFO | "huge" : "", | 20260505 22:27:26.269 | INFO | "huge_shared" : " 27 GB", | 20260505 22:27:26.269 | INFO | "malloc" : " 14 GB", | 20260505 22:27:26.269 | INFO | "pinned" : " 29 GB" | 20260505 22:27:26.269 | INFO | } | 20260505 22:27:26.269 | INFO | Perf report for the last 5.14455 seconds: | 20260505 22:27:26.269 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:27:26.269 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:26.269 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:26.269 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:26.269 | INFO | ANALYSIS : 61.5%, 3.0, 96.1ms, 0 | 20260505 22:27:26.269 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:26.269 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:26.269 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:27:26.269 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:26.269 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:26.269 | INFO | Uploaded 36GiB to GPU in 2.07982s (17.3091GiB/s) | 20260505 22:27:26.269 | INFO | Downloaded 34GiB to GPU in 2.04086s (16.6596GiB/s) | 20260505 22:27:31.405 | INFO | Analyzing chunk frames = [6790144,6790656) | 20260505 22:27:31.405 | INFO | MemInfo: { | 20260505 22:27:31.405 | INFO | "gpu" : " 17 GB", | 20260505 22:27:31.405 | INFO | "huge" : "", | 20260505 22:27:31.405 | INFO | "huge_shared" : " 27 GB", | 20260505 22:27:31.405 | INFO | "malloc" : " 14 GB", | 20260505 22:27:31.405 | INFO | "pinned" : " 29 GB" | 20260505 22:27:31.405 | INFO | } | 20260505 22:27:31.406 | INFO | Perf report for the last 5.13645 seconds: | 20260505 22:27:31.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:27:31.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:31.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:31.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:31.406 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 22:27:31.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:31.406 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:31.406 | INFO | PRE_HQ : 27.7%, 1.0, 14.3ms, 0 | 20260505 22:27:31.406 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:31.406 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:31.406 | INFO | Uploaded 36GiB to GPU in 2.02011s (17.8208GiB/s) | 20260505 22:27:31.406 | INFO | Downloaded 34GiB to GPU in 2.03883s (16.6763GiB/s) | 20260505 22:27:36.537 | INFO | Analyzing chunk frames = [6790656,6791168) | 20260505 22:27:36.537 | INFO | MemInfo: { | 20260505 22:27:36.537 | INFO | "gpu" : " 17 GB", | 20260505 22:27:36.537 | INFO | "huge" : "", | 20260505 22:27:36.537 | INFO | "huge_shared" : " 27 GB", | 20260505 22:27:36.538 | INFO | "malloc" : " 14 GB", | 20260505 22:27:36.538 | INFO | "pinned" : " 29 GB" | 20260505 22:27:36.538 | INFO | } | 20260505 22:27:36.538 | INFO | Perf report for the last 5.13211 seconds: | 20260505 22:27:36.538 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:27:36.538 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:36.538 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:36.538 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:36.538 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260505 22:27:36.538 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:36.538 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:36.538 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:27:36.538 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:36.538 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:36.538 | INFO | Uploaded 36GiB to GPU in 2.11664s (17.0081GiB/s) | 20260505 22:27:36.538 | INFO | Downloaded 34GiB to GPU in 2.0057s (16.9517GiB/s) | 20260505 22:27:36.538 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 22:27:36.538 | INFO | Total mem usage: 15.3 GiB | 20260505 22:27:36.538 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 22:27:36.538 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.51%) | 20260505 22:27:36.538 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 22:27:36.538 | INFO | High Water Breakdown: | 20260505 22:27:36.538 | INFO | 61.1% : 8.63 GiB ( 90.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 22:27:36.538 | INFO | 38.8% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 22:27:36.538 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 22:27:36.538 | INFO | ------------------------------------------ | 20260505 22:27:36.538 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:27:36.538 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 22:27:36.538 | INFO | Total mem usage: 26.5 GiB | 20260505 22:27:36.538 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 22:27:36.538 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 22:27:36.538 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 22:27:36.538 | INFO | High Water Breakdown: | 20260505 22:27:36.538 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:27:36.538 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:27:36.538 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:27:36.538 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:27:36.538 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:27:36.538 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:27:36.538 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 22:27:36.538 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 22:27:36.538 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:27:36.538 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 22:27:36.538 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 22:27:36.538 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 22:27:36.538 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 22:27:36.538 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 22:27:36.538 | INFO | ------------------------------------------ | 20260505 22:27:36.538 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:27:36.539 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 22:27:36.539 | INFO | Total mem usage: 25.6 GiB | 20260505 22:27:36.539 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 22:27:36.539 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 22:27:36.539 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 22:27:36.539 | INFO | High Water Breakdown: | 20260505 22:27:36.539 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 22:27:36.539 | INFO | ------------------------------------------ | 20260505 22:27:36.539 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:27:36.539 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 22:27:36.539 | INFO | Total mem usage: 15.7 GiB | 20260505 22:27:36.539 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 22:27:36.539 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 22:27:36.539 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 22:27:36.539 | INFO | High Water Breakdown: | 20260505 22:27:36.539 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 22:27:36.539 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 22:27:36.539 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 22:27:36.539 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 22:27:36.539 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 22:27:36.539 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 22:27:36.539 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 22:27:36.539 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 22:27:36.539 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 22:27:36.539 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:27:36.539 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 22:27:36.539 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:27:36.539 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 22:27:36.539 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 22:27:36.539 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 22:27:36.539 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 22:27:36.539 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:27:36.539 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 22:27:36.539 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:27:36.539 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:27:36.539 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:27:36.539 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 22:27:36.539 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 22:27:36.539 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:27:36.539 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 22:27:36.539 | INFO | ------------------------------------------ | 20260505 22:27:36.539 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:27:36.539 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 22:27:36.539 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:27:36.539 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:27:36.539 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.39e+01, 1.35e+01 / 8.14e-01 | 20260505 22:27:36.539 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.04e-01, 1.09e-02 / 3.91e-03 | 20260505 22:27:36.539 | INFO | PerfReport: 11.81%, Baseline -- 3.70e+00 / 4.51e+00, 5.67e-01 / 1.64e-01 | 20260505 22:27:36.539 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.54e+00, 1.89e-01 / 8.62e-02 | 20260505 22:27:36.539 | INFO | PerfReport: 15.98%, DME -- 4.01e+01 / 4.40e+01, 3.82e+00 / 8.43e-01 | 20260505 22:27:36.539 | INFO | PerfReport: 57.20%, FrameLabeling -- 1.79e+01 / 1.92e+01, 1.46e+00 / 1.01e+00 | 20260505 22:27:36.539 | INFO | PerfReport: 7.19%, PulseAccumulating -- 2.25e+00 / 2.80e+00, 4.31e-01 / 1.78e-01 | 20260505 22:27:36.539 | INFO | PerfReport: 3.16%, Metrics -- 9.91e-01 / 1.30e+00, 2.68e-01 / 1.44e-01 | 20260505 22:27:36.539 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.44e-01 / 4.05e-01, 5.82e-02 / 3.08e-02 | 20260505 22:27:36.539 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 22:27:36.539 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:27:36.539 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:27:36.539 | INFO | PerfReport: Full Execution -- 9.87e+01 / -nan, 2.49e+01 / 0.00e+00 | 20260505 22:27:36.539 | INFO | PerfReport: 22.43%, Upload -- 2.21e+01 / 0.00e+00, 7.65e+00 / 0.00e+00 | 20260505 22:27:36.539 | INFO | PerfReport: 21.82%, Download -- 2.15e+01 / 0.00e+00, 7.65e+00 / 0.00e+00 | 20260505 22:27:36.539 | INFO | PerfReport: 31.75%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 22:27:36.539 | INFO | PerfReport: 2.19%, LockUpload -- 2.16e+00 / 0.00e+00, 7.90e+00 / 0.00e+00 | 20260505 22:27:36.539 | INFO | PerfReport: 1.33%, LockDownload -- 1.31e+00 / 0.00e+00, 2.85e+00 / 0.00e+00 | 20260505 22:27:36.539 | INFO | PerfReport: 20.47%, LockCompute -- 2.02e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 22:27:36.539 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.60e-02 / 0.00e+00 | 20260505 22:27:36.672 | INFO | Starting a round of Baz IO | 20260505 22:27:39.832 | INFO | Populating RTMetrics report | 20260505 22:27:39.832 | INFO | RTMetrics report populated | 20260505 22:27:39.834 | INFO | Opening tmp RTMetrics json file | 20260505 22:27:39.834 | INFO | Serializing RTMetrics to json | 20260505 22:27:39.836 | INFO | Serialization done, swapping tmp file now... | 20260505 22:27:39.836 | INFO | RTMetrics done | 20260505 22:27:39.910 | INFO | Finished a round of Baz IO: Wrote 1.929 GiB in 3.238 seconds | 20260505 22:27:39.911 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.50% / 0.00% / 78.50% | 20260505 22:27:39.911 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.124351% / 0% / 7.15256e-05% | 20260505 22:27:39.911 | INFO | Baz IO overall: | 20260505 22:27:39.911 | INFO | BytesWritten : 3909278695401 | 20260505 22:27:39.911 | INFO | NumEvents : 1843879572656 | 20260505 22:27:41.678 | INFO | Analyzing chunk frames = [6791168,6791680) | 20260505 22:27:41.678 | INFO | MemInfo: { | 20260505 22:27:41.678 | INFO | "gpu" : " 17 GB", | 20260505 22:27:41.678 | INFO | "huge" : "", | 20260505 22:27:41.678 | INFO | "huge_shared" : " 27 GB", | 20260505 22:27:41.678 | INFO | "malloc" : " 14 GB", | 20260505 22:27:41.678 | INFO | "pinned" : " 29 GB" | 20260505 22:27:41.678 | INFO | } | 20260505 22:27:41.678 | INFO | Perf report for the last 5.1404 seconds: | 20260505 22:27:41.678 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:27:41.678 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:41.678 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:41.678 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:41.678 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260505 22:27:41.678 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:41.678 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260505 22:27:41.678 | INFO | PRE_HQ : 59.8%, 1.8, 54.7ms, 0 | 20260505 22:27:41.678 | INFO | BAZWRITER : 62.4%, 3.6, 116.4ms, 0 | 20260505 22:27:41.678 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 22:27:41.678 | INFO | Uploaded 36GiB to GPU in 2.17174s (16.5766GiB/s) | 20260505 22:27:41.678 | INFO | Downloaded 38GiB to GPU in 2.25175s (16.8758GiB/s) | 20260505 22:27:46.821 | INFO | Analyzing chunk frames = [6791680,6792192) | 20260505 22:27:46.821 | INFO | MemInfo: { | 20260505 22:27:46.821 | INFO | "gpu" : " 17 GB", | 20260505 22:27:46.821 | INFO | "huge" : "", | 20260505 22:27:46.821 | INFO | "huge_shared" : " 27 GB", | 20260505 22:27:46.822 | INFO | "malloc" : " 14 GB", | 20260505 22:27:46.822 | INFO | "pinned" : " 29 GB" | 20260505 22:27:46.822 | INFO | } | 20260505 22:27:46.822 | INFO | Perf report for the last 5.1435 seconds: | 20260505 22:27:46.822 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:27:46.822 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:46.822 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:46.822 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:46.822 | INFO | ANALYSIS : 61.8%, 3.0, 96.6ms, 0 | 20260505 22:27:46.822 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:46.822 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:46.822 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 22:27:46.822 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:46.822 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:46.822 | INFO | Uploaded 36GiB to GPU in 2.08474s (17.2684GiB/s) | 20260505 22:27:46.822 | INFO | Downloaded 34GiB to GPU in 2.02405s (16.798GiB/s) | 20260505 22:27:51.964 | INFO | Analyzing chunk frames = [6792192,6792704) | 20260505 22:27:51.964 | INFO | MemInfo: { | 20260505 22:27:51.965 | INFO | "gpu" : " 17 GB", | 20260505 22:27:51.965 | INFO | "huge" : "", | 20260505 22:27:51.965 | INFO | "huge_shared" : " 27 GB", | 20260505 22:27:51.965 | INFO | "malloc" : " 14 GB", | 20260505 22:27:51.965 | INFO | "pinned" : " 29 GB" | 20260505 22:27:51.965 | INFO | } | 20260505 22:27:51.965 | INFO | Perf report for the last 5.14305 seconds: | 20260505 22:27:51.965 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:27:51.965 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:51.965 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:51.965 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:51.965 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260505 22:27:51.965 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:51.965 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:51.965 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 22:27:51.965 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:51.965 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:51.965 | INFO | Uploaded 36GiB to GPU in 2.1434s (16.7958GiB/s) | 20260505 22:27:51.965 | INFO | Downloaded 34GiB to GPU in 2.02737s (16.7705GiB/s) | 20260505 22:27:57.102 | INFO | Analyzing chunk frames = [6792704,6793216) | 20260505 22:27:57.102 | INFO | MemInfo: { | 20260505 22:27:57.102 | INFO | "gpu" : " 17 GB", | 20260505 22:27:57.102 | INFO | "huge" : "", | 20260505 22:27:57.102 | INFO | "huge_shared" : " 27 GB", | 20260505 22:27:57.102 | INFO | "malloc" : " 14 GB", | 20260505 22:27:57.102 | INFO | "pinned" : " 29 GB" | 20260505 22:27:57.102 | INFO | } | 20260505 22:27:57.102 | INFO | Perf report for the last 5.13773 seconds: | 20260505 22:27:57.102 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:27:57.102 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:57.102 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:57.102 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:57.102 | INFO | ANALYSIS : 61.7%, 3.0, 95.8ms, 0 | 20260505 22:27:57.102 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:57.102 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:57.102 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:27:57.102 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:57.102 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:27:57.103 | INFO | Uploaded 36GiB to GPU in 2.12407s (16.9486GiB/s) | 20260505 22:27:57.103 | INFO | Downloaded 34GiB to GPU in 2.0214s (16.82GiB/s) | 20260505 22:28:02.237 | INFO | Analyzing chunk frames = [6793216,6793728) | 20260505 22:28:02.237 | INFO | MemInfo: { | 20260505 22:28:02.237 | INFO | "gpu" : " 17 GB", | 20260505 22:28:02.237 | INFO | "huge" : "", | 20260505 22:28:02.237 | INFO | "huge_shared" : " 27 GB", | 20260505 22:28:02.237 | INFO | "malloc" : " 14 GB", | 20260505 22:28:02.237 | INFO | "pinned" : " 29 GB" | 20260505 22:28:02.237 | INFO | } | 20260505 22:28:02.237 | INFO | Perf report for the last 5.13497 seconds: | 20260505 22:28:02.237 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:28:02.237 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:02.237 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:02.237 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:02.237 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260505 22:28:02.237 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:02.237 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:02.237 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:28:02.237 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:02.237 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:02.238 | INFO | Uploaded 36GiB to GPU in 2.14129s (16.8123GiB/s) | 20260505 22:28:02.238 | INFO | Downloaded 34GiB to GPU in 2.06266s (16.4835GiB/s) | 20260505 22:28:07.383 | INFO | Analyzing chunk frames = [6793728,6794240) | 20260505 22:28:07.383 | INFO | MemInfo: { | 20260505 22:28:07.383 | INFO | "gpu" : " 17 GB", | 20260505 22:28:07.383 | INFO | "huge" : "", | 20260505 22:28:07.383 | INFO | "huge_shared" : " 27 GB", | 20260505 22:28:07.383 | INFO | "malloc" : " 14 GB", | 20260505 22:28:07.383 | INFO | "pinned" : " 29 GB" | 20260505 22:28:07.383 | INFO | } | 20260505 22:28:07.383 | INFO | Perf report for the last 5.14546 seconds: | 20260505 22:28:07.383 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:28:07.383 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:07.383 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:07.383 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:07.383 | INFO | ANALYSIS : 61.5%, 3.0, 96.2ms, 0 | 20260505 22:28:07.383 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:07.383 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:07.383 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:28:07.383 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:07.383 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:07.383 | INFO | Uploaded 36GiB to GPU in 2.0869s (17.2504GiB/s) | 20260505 22:28:07.383 | INFO | Downloaded 34GiB to GPU in 2.04093s (16.659GiB/s) | 20260505 22:28:12.519 | INFO | Analyzing chunk frames = [6794240,6794752) | 20260505 22:28:12.519 | INFO | MemInfo: { | 20260505 22:28:12.519 | INFO | "gpu" : " 17 GB", | 20260505 22:28:12.519 | INFO | "huge" : "", | 20260505 22:28:12.519 | INFO | "huge_shared" : " 27 GB", | 20260505 22:28:12.519 | INFO | "malloc" : " 14 GB", | 20260505 22:28:12.519 | INFO | "pinned" : " 29 GB" | 20260505 22:28:12.519 | INFO | } | 20260505 22:28:12.519 | INFO | Perf report for the last 5.13636 seconds: | 20260505 22:28:12.519 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:28:12.519 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:12.519 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:12.519 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:12.519 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260505 22:28:12.519 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:12.519 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:12.519 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260505 22:28:12.519 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:12.519 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:12.519 | INFO | Uploaded 36GiB to GPU in 2.05293s (17.5359GiB/s) | 20260505 22:28:12.519 | INFO | Downloaded 34GiB to GPU in 2.06867s (16.4357GiB/s) | 20260505 22:28:17.659 | INFO | Analyzing chunk frames = [6794752,6795264) | 20260505 22:28:17.659 | INFO | MemInfo: { | 20260505 22:28:17.659 | INFO | "gpu" : " 17 GB", | 20260505 22:28:17.659 | INFO | "huge" : "", | 20260505 22:28:17.659 | INFO | "huge_shared" : " 27 GB", | 20260505 22:28:17.659 | INFO | "malloc" : " 14 GB", | 20260505 22:28:17.659 | INFO | "pinned" : " 29 GB" | 20260505 22:28:17.659 | INFO | } | 20260505 22:28:17.659 | INFO | Perf report for the last 5.13969 seconds: | 20260505 22:28:17.659 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:28:17.659 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:17.659 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:17.659 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:17.659 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 22:28:17.659 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:17.659 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:17.659 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:28:17.659 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:17.659 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:17.659 | INFO | Uploaded 36GiB to GPU in 2.11734s (17.0024GiB/s) | 20260505 22:28:17.659 | INFO | Downloaded 34GiB to GPU in 2.00138s (16.9883GiB/s) | 20260505 22:28:17.788 | INFO | Starting a round of Baz IO | 20260505 22:28:20.935 | INFO | Populating RTMetrics report | 20260505 22:28:20.935 | INFO | RTMetrics report populated | 20260505 22:28:20.935 | INFO | Opening tmp RTMetrics json file | 20260505 22:28:20.937 | INFO | Serializing RTMetrics to json | 20260505 22:28:20.938 | INFO | Serialization done, swapping tmp file now... | 20260505 22:28:20.938 | INFO | RTMetrics done | 20260505 22:28:21.013 | INFO | Finished a round of Baz IO: Wrote 1.928 GiB in 3.224 seconds | 20260505 22:28:21.013 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.50% / 0.00% / 78.50% | 20260505 22:28:21.013 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.123843% / 0% / 7.15256e-05% | 20260505 22:28:21.013 | INFO | Baz IO overall: | 20260505 22:28:21.013 | INFO | BytesWritten : 3911348956293 | 20260505 22:28:21.013 | INFO | NumEvents : 1844831360531 | 20260505 22:28:22.799 | INFO | Analyzing chunk frames = [6795264,6795776) | 20260505 22:28:22.799 | INFO | MemInfo: { | 20260505 22:28:22.799 | INFO | "gpu" : " 17 GB", | 20260505 22:28:22.799 | INFO | "huge" : "", | 20260505 22:28:22.799 | INFO | "huge_shared" : " 27 GB", | 20260505 22:28:22.799 | INFO | "malloc" : " 14 GB", | 20260505 22:28:22.799 | INFO | "pinned" : " 29 GB" | 20260505 22:28:22.799 | INFO | } | 20260505 22:28:22.799 | INFO | Perf report for the last 5.14002 seconds: | 20260505 22:28:22.799 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:28:22.799 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:22.799 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:22.799 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:22.799 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 22:28:22.799 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:22.799 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260505 22:28:22.799 | INFO | PRE_HQ : 59.6%, 1.8, 54.5ms, 0 | 20260505 22:28:22.799 | INFO | BAZWRITER : 62.0%, 3.6, 113.9ms, 0 | 20260505 22:28:22.799 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 22:28:22.799 | INFO | Uploaded 36GiB to GPU in 2.12436s (16.9463GiB/s) | 20260505 22:28:22.799 | INFO | Downloaded 38GiB to GPU in 2.22318s (17.0926GiB/s) | 20260505 22:28:27.933 | INFO | Analyzing chunk frames = [6795776,6796288) | 20260505 22:28:27.933 | INFO | MemInfo: { | 20260505 22:28:27.933 | INFO | "gpu" : " 17 GB", | 20260505 22:28:27.933 | INFO | "huge" : "", | 20260505 22:28:27.933 | INFO | "huge_shared" : " 27 GB", | 20260505 22:28:27.933 | INFO | "malloc" : " 14 GB", | 20260505 22:28:27.933 | INFO | "pinned" : " 29 GB" | 20260505 22:28:27.933 | INFO | } | 20260505 22:28:27.933 | INFO | Perf report for the last 5.1337 seconds: | 20260505 22:28:27.933 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:28:27.933 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:27.933 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:27.933 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:27.933 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260505 22:28:27.933 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:27.933 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:27.933 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260505 22:28:27.933 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:27.933 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:27.933 | INFO | Uploaded 36GiB to GPU in 2.08458s (17.2696GiB/s) | 20260505 22:28:27.933 | INFO | Downloaded 34GiB to GPU in 2.02617s (16.7804GiB/s) | 20260505 22:28:33.078 | INFO | Analyzing chunk frames = [6796288,6796800) | 20260505 22:28:33.078 | INFO | MemInfo: { | 20260505 22:28:33.078 | INFO | "gpu" : " 17 GB", | 20260505 22:28:33.078 | INFO | "huge" : "", | 20260505 22:28:33.078 | INFO | "huge_shared" : " 27 GB", | 20260505 22:28:33.078 | INFO | "malloc" : " 14 GB", | 20260505 22:28:33.078 | INFO | "pinned" : " 29 GB" | 20260505 22:28:33.078 | INFO | } | 20260505 22:28:33.078 | INFO | Perf report for the last 5.14499 seconds: | 20260505 22:28:33.078 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:28:33.078 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:33.078 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:33.078 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:33.078 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260505 22:28:33.078 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:33.078 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:33.078 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:28:33.078 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:33.078 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:33.078 | INFO | Uploaded 36GiB to GPU in 2.12902s (16.9092GiB/s) | 20260505 22:28:33.078 | INFO | Downloaded 34GiB to GPU in 2.01995s (16.8321GiB/s) | 20260505 22:28:38.218 | INFO | Analyzing chunk frames = [6796800,6797312) | 20260505 22:28:38.218 | INFO | MemInfo: { | 20260505 22:28:38.218 | INFO | "gpu" : " 17 GB", | 20260505 22:28:38.218 | INFO | "huge" : "", | 20260505 22:28:38.218 | INFO | "huge_shared" : " 27 GB", | 20260505 22:28:38.218 | INFO | "malloc" : " 14 GB", | 20260505 22:28:38.218 | INFO | "pinned" : " 29 GB" | 20260505 22:28:38.218 | INFO | } | 20260505 22:28:38.218 | INFO | Perf report for the last 5.14051 seconds: | 20260505 22:28:38.218 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:28:38.218 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:38.218 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:38.218 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:38.218 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260505 22:28:38.218 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:38.218 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:38.218 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:28:38.218 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:38.218 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:38.218 | INFO | Uploaded 36GiB to GPU in 2.13691s (16.8467GiB/s) | 20260505 22:28:38.219 | INFO | Downloaded 34GiB to GPU in 2.00943s (16.9202GiB/s) | 20260505 22:28:43.353 | INFO | Analyzing chunk frames = [6797312,6797824) | 20260505 22:28:43.353 | INFO | MemInfo: { | 20260505 22:28:43.353 | INFO | "gpu" : " 17 GB", | 20260505 22:28:43.353 | INFO | "huge" : "", | 20260505 22:28:43.353 | INFO | "huge_shared" : " 27 GB", | 20260505 22:28:43.353 | INFO | "malloc" : " 14 GB", | 20260505 22:28:43.353 | INFO | "pinned" : " 29 GB" | 20260505 22:28:43.353 | INFO | } | 20260505 22:28:43.353 | INFO | Perf report for the last 5.13446 seconds: | 20260505 22:28:43.353 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:28:43.353 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:43.353 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:43.353 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:43.353 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260505 22:28:43.353 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:43.353 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:43.353 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 22:28:43.353 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:43.353 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:43.353 | INFO | Uploaded 36GiB to GPU in 2.08934s (17.2303GiB/s) | 20260505 22:28:43.353 | INFO | Downloaded 34GiB to GPU in 2.04145s (16.6548GiB/s) | 20260505 22:28:48.485 | INFO | Analyzing chunk frames = [6797824,6798336) | 20260505 22:28:48.485 | INFO | MemInfo: { | 20260505 22:28:48.485 | INFO | "gpu" : " 17 GB", | 20260505 22:28:48.485 | INFO | "huge" : "", | 20260505 22:28:48.485 | INFO | "huge_shared" : " 27 GB", | 20260505 22:28:48.485 | INFO | "malloc" : " 14 GB", | 20260505 22:28:48.485 | INFO | "pinned" : " 29 GB" | 20260505 22:28:48.485 | INFO | } | 20260505 22:28:48.485 | INFO | Perf report for the last 5.13278 seconds: | 20260505 22:28:48.485 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:28:48.485 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:48.485 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:48.485 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:48.486 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 22:28:48.486 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:48.486 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:48.486 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:28:48.486 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:48.486 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:48.486 | INFO | Uploaded 36GiB to GPU in 2.15383s (16.7144GiB/s) | 20260505 22:28:48.486 | INFO | Downloaded 34GiB to GPU in 2.06359s (16.4762GiB/s) | 20260505 22:28:53.630 | INFO | Analyzing chunk frames = [6798336,6798848) | 20260505 22:28:53.630 | INFO | MemInfo: { | 20260505 22:28:53.630 | INFO | "gpu" : " 17 GB", | 20260505 22:28:53.630 | INFO | "huge" : "", | 20260505 22:28:53.630 | INFO | "huge_shared" : " 27 GB", | 20260505 22:28:53.630 | INFO | "malloc" : " 14 GB", | 20260505 22:28:53.630 | INFO | "pinned" : " 29 GB" | 20260505 22:28:53.630 | INFO | } | 20260505 22:28:53.630 | INFO | Perf report for the last 5.14428 seconds: | 20260505 22:28:53.630 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:28:53.630 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:53.630 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:53.630 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:53.630 | INFO | ANALYSIS : 62.7%, 3.0, 98.1ms, 0 | 20260505 22:28:53.630 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:53.630 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:53.630 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260505 22:28:53.630 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:53.630 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:53.630 | INFO | Uploaded 36GiB to GPU in 2.14916s (16.7508GiB/s) | 20260505 22:28:53.630 | INFO | Downloaded 34GiB to GPU in 2.1518s (15.8007GiB/s) | 20260505 22:28:58.777 | INFO | Analyzing chunk frames = [6798848,6799360) | 20260505 22:28:58.777 | INFO | MemInfo: { | 20260505 22:28:58.777 | INFO | "gpu" : " 17 GB", | 20260505 22:28:58.777 | INFO | "huge" : "", | 20260505 22:28:58.778 | INFO | "huge_shared" : " 27 GB", | 20260505 22:28:58.778 | INFO | "malloc" : " 14 GB", | 20260505 22:28:58.778 | INFO | "pinned" : " 29 GB" | 20260505 22:28:58.778 | INFO | } | 20260505 22:28:58.778 | INFO | Perf report for the last 5.14788 seconds: | 20260505 22:28:58.778 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:28:58.778 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:58.778 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:58.778 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:58.778 | INFO | ANALYSIS : 61.9%, 3.0, 96.9ms, 0 | 20260505 22:28:58.778 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:58.778 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:58.778 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:28:58.778 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:58.778 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:28:58.778 | INFO | Uploaded 36GiB to GPU in 2.10987s (17.0626GiB/s) | 20260505 22:28:58.778 | INFO | Downloaded 34GiB to GPU in 1.99945s (17.0047GiB/s) | 20260505 22:28:58.778 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 22:28:58.778 | INFO | Total mem usage: 15.3 GiB | 20260505 22:28:58.778 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 22:28:58.778 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.51%) | 20260505 22:28:58.778 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 22:28:58.778 | INFO | High Water Breakdown: | 20260505 22:28:58.778 | INFO | 61.1% : 8.63 GiB ( 90.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 22:28:58.778 | INFO | 38.8% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 22:28:58.778 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 22:28:58.778 | INFO | ------------------------------------------ | 20260505 22:28:58.778 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:28:58.778 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 22:28:58.778 | INFO | Total mem usage: 26.5 GiB | 20260505 22:28:58.778 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 22:28:58.778 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 22:28:58.778 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 22:28:58.778 | INFO | High Water Breakdown: | 20260505 22:28:58.778 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:28:58.778 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:28:58.778 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:28:58.778 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:28:58.778 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:28:58.778 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:28:58.778 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 22:28:58.778 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 22:28:58.778 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:28:58.778 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 22:28:58.778 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 22:28:58.778 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 22:28:58.778 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 22:28:58.778 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 22:28:58.778 | INFO | ------------------------------------------ | 20260505 22:28:58.778 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:28:58.778 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 22:28:58.778 | INFO | Total mem usage: 25.6 GiB | 20260505 22:28:58.778 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 22:28:58.778 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 22:28:58.778 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 22:28:58.778 | INFO | High Water Breakdown: | 20260505 22:28:58.778 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 22:28:58.778 | INFO | ------------------------------------------ | 20260505 22:28:58.778 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:28:58.778 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 22:28:58.778 | INFO | Total mem usage: 15.7 GiB | 20260505 22:28:58.778 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 22:28:58.778 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 22:28:58.778 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 22:28:58.778 | INFO | High Water Breakdown: | 20260505 22:28:58.778 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 22:28:58.778 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 22:28:58.778 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 22:28:58.778 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 22:28:58.778 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 22:28:58.778 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 22:28:58.778 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 22:28:58.778 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 22:28:58.778 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 22:28:58.778 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:28:58.778 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 22:28:58.778 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:28:58.778 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 22:28:58.778 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 22:28:58.779 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 22:28:58.779 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 22:28:58.779 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:28:58.779 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 22:28:58.779 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:28:58.779 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:28:58.779 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:28:58.779 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 22:28:58.779 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 22:28:58.779 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:28:58.779 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 22:28:58.779 | INFO | ------------------------------------------ | 20260505 22:28:58.779 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:28:58.779 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 22:28:58.779 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:28:58.779 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:28:58.779 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.41e+01, 1.35e+01 / 1.43e+00 | 20260505 22:28:58.779 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.04e-01, 1.07e-02 / 5.00e-03 | 20260505 22:28:58.779 | INFO | PerfReport: 11.85%, Baseline -- 3.72e+00 / 4.54e+00, 5.74e-01 / 1.03e-01 | 20260505 22:28:58.779 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.53e+00, 1.89e-01 / 7.33e-02 | 20260505 22:28:58.779 | INFO | PerfReport: 15.99%, DME -- 4.01e+01 / 4.42e+01, 3.96e+00 / 1.45e+00 | 20260505 22:28:58.779 | INFO | PerfReport: 57.15%, FrameLabeling -- 1.79e+01 / 1.93e+01, 1.45e+00 / 8.58e-01 | 20260505 22:28:58.779 | INFO | PerfReport: 7.17%, PulseAccumulating -- 2.25e+00 / 2.81e+00, 4.31e-01 / 1.93e-01 | 20260505 22:28:58.779 | INFO | PerfReport: 3.17%, Metrics -- 9.93e-01 / 1.30e+00, 2.71e-01 / 1.47e-01 | 20260505 22:28:58.779 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.44e-01 / 4.02e-01, 5.82e-02 / 3.37e-02 | 20260505 22:28:58.779 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 22:28:58.779 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:28:58.779 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:28:58.779 | INFO | PerfReport: Full Execution -- 9.87e+01 / -nan, 2.50e+01 / 0.00e+00 | 20260505 22:28:58.779 | INFO | PerfReport: 22.35%, Upload -- 2.21e+01 / 0.00e+00, 7.56e+00 / 0.00e+00 | 20260505 22:28:58.779 | INFO | PerfReport: 21.78%, Download -- 2.15e+01 / 0.00e+00, 7.59e+00 / 0.00e+00 | 20260505 22:28:58.779 | INFO | PerfReport: 31.77%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 22:28:58.779 | INFO | PerfReport: 2.22%, LockUpload -- 2.19e+00 / 0.00e+00, 8.03e+00 / 0.00e+00 | 20260505 22:28:58.779 | INFO | PerfReport: 1.35%, LockDownload -- 1.34e+00 / 0.00e+00, 2.93e+00 / 0.00e+00 | 20260505 22:28:58.779 | INFO | PerfReport: 20.51%, LockCompute -- 2.03e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260505 22:28:58.779 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.88e-02 / 0.00e+00 | 20260505 22:28:58.900 | INFO | Starting a round of Baz IO | 20260505 22:29:02.046 | INFO | Populating RTMetrics report | 20260505 22:29:02.046 | INFO | RTMetrics report populated | 20260505 22:29:02.046 | INFO | Opening tmp RTMetrics json file | 20260505 22:29:02.047 | INFO | Serializing RTMetrics to json | 20260505 22:29:02.049 | INFO | Serialization done, swapping tmp file now... | 20260505 22:29:02.051 | INFO | RTMetrics done | 20260505 22:29:02.128 | INFO | Finished a round of Baz IO: Wrote 1.928 GiB in 3.228 seconds | 20260505 22:29:02.128 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.49% / 0.00% / 78.50% | 20260505 22:29:02.128 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.123906% / 3.97364e-06% / 6.75519e-05% | 20260505 22:29:02.128 | INFO | Baz IO overall: | 20260505 22:29:02.129 | INFO | BytesWritten : 3913419238303 | 20260505 22:29:02.129 | INFO | NumEvents : 1845783198957 | 20260505 22:29:03.917 | INFO | Analyzing chunk frames = [6799360,6799872) | 20260505 22:29:03.917 | INFO | MemInfo: { | 20260505 22:29:03.917 | INFO | "gpu" : " 17 GB", | 20260505 22:29:03.917 | INFO | "huge" : "", | 20260505 22:29:03.917 | INFO | "huge_shared" : " 27 GB", | 20260505 22:29:03.917 | INFO | "malloc" : " 14 GB", | 20260505 22:29:03.917 | INFO | "pinned" : " 29 GB" | 20260505 22:29:03.917 | INFO | } | 20260505 22:29:03.917 | INFO | Perf report for the last 5.13936 seconds: | 20260505 22:29:03.917 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:29:03.917 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:03.917 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:03.917 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:03.917 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 22:29:03.917 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:03.917 | INFO | RT_METRICS : 43.8%, 1.0, 22.9ms, 0 | 20260505 22:29:03.917 | INFO | PRE_HQ : 59.7%, 1.8, 54.7ms, 0 | 20260505 22:29:03.917 | INFO | BAZWRITER : 62.0%, 3.6, 114.4ms, 0 | 20260505 22:29:03.917 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 22:29:03.917 | INFO | Uploaded 36GiB to GPU in 2.13491s (16.8626GiB/s) | 20260505 22:29:03.917 | INFO | Downloaded 38GiB to GPU in 2.23058s (17.0359GiB/s) | 20260505 22:29:09.047 | INFO | Analyzing chunk frames = [6799872,6800384) | 20260505 22:29:09.047 | INFO | MemInfo: { | 20260505 22:29:09.047 | INFO | "gpu" : " 17 GB", | 20260505 22:29:09.047 | INFO | "huge" : "", | 20260505 22:29:09.047 | INFO | "huge_shared" : " 27 GB", | 20260505 22:29:09.047 | INFO | "malloc" : " 14 GB", | 20260505 22:29:09.047 | INFO | "pinned" : " 29 GB" | 20260505 22:29:09.047 | INFO | } | 20260505 22:29:09.047 | INFO | Perf report for the last 5.13023 seconds: | 20260505 22:29:09.047 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:29:09.047 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:09.047 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:09.047 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:09.047 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260505 22:29:09.047 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:09.047 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:09.047 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260505 22:29:09.047 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:09.047 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:09.047 | INFO | Uploaded 36GiB to GPU in 2.08367s (17.2772GiB/s) | 20260505 22:29:09.048 | INFO | Downloaded 34GiB to GPU in 2.02351s (16.8025GiB/s) | 20260505 22:29:14.188 | INFO | Analyzing chunk frames = [6800384,6800896) | 20260505 22:29:14.188 | INFO | MemInfo: { | 20260505 22:29:14.188 | INFO | "gpu" : " 17 GB", | 20260505 22:29:14.189 | INFO | "huge" : "", | 20260505 22:29:14.189 | INFO | "huge_shared" : " 27 GB", | 20260505 22:29:14.189 | INFO | "malloc" : " 14 GB", | 20260505 22:29:14.189 | INFO | "pinned" : " 29 GB" | 20260505 22:29:14.189 | INFO | } | 20260505 22:29:14.189 | INFO | Perf report for the last 5.14139 seconds: | 20260505 22:29:14.189 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:29:14.189 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:14.189 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:14.189 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:14.189 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260505 22:29:14.189 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:14.189 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:14.189 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:29:14.189 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:14.189 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:14.189 | INFO | Uploaded 36GiB to GPU in 2.13676s (16.848GiB/s) | 20260505 22:29:14.189 | INFO | Downloaded 34GiB to GPU in 2.02309s (16.806GiB/s) | 20260505 22:29:19.327 | INFO | Analyzing chunk frames = [6800896,6801408) | 20260505 22:29:19.327 | INFO | MemInfo: { | 20260505 22:29:19.327 | INFO | "gpu" : " 17 GB", | 20260505 22:29:19.327 | INFO | "huge" : "", | 20260505 22:29:19.327 | INFO | "huge_shared" : " 27 GB", | 20260505 22:29:19.327 | INFO | "malloc" : " 14 GB", | 20260505 22:29:19.327 | INFO | "pinned" : " 29 GB" | 20260505 22:29:19.327 | INFO | } | 20260505 22:29:19.327 | INFO | Perf report for the last 5.13828 seconds: | 20260505 22:29:19.327 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:29:19.327 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:19.327 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:19.327 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:19.327 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260505 22:29:19.327 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:19.327 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:19.327 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 22:29:19.327 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:19.327 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:19.327 | INFO | Uploaded 36GiB to GPU in 2.20936s (16.2943GiB/s) | 20260505 22:29:19.327 | INFO | Downloaded 34GiB to GPU in 2.07589s (16.3785GiB/s) | 20260505 22:29:24.473 | INFO | Analyzing chunk frames = [6801408,6801920) | 20260505 22:29:24.473 | INFO | MemInfo: { | 20260505 22:29:24.473 | INFO | "gpu" : " 17 GB", | 20260505 22:29:24.473 | INFO | "huge" : "", | 20260505 22:29:24.473 | INFO | "huge_shared" : " 27 GB", | 20260505 22:29:24.473 | INFO | "malloc" : " 14 GB", | 20260505 22:29:24.473 | INFO | "pinned" : " 29 GB" | 20260505 22:29:24.473 | INFO | } | 20260505 22:29:24.473 | INFO | Perf report for the last 5.14606 seconds: | 20260505 22:29:24.473 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:29:24.473 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:24.473 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:24.473 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:24.473 | INFO | ANALYSIS : 61.5%, 3.0, 96.2ms, 0 | 20260505 22:29:24.473 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:24.473 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:24.473 | INFO | PRE_HQ : 26.8%, 1.0, 14.0ms, 0 | 20260505 22:29:24.473 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:24.473 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:24.473 | INFO | Uploaded 36GiB to GPU in 2.16032s (16.6642GiB/s) | 20260505 22:29:24.473 | INFO | Downloaded 34GiB to GPU in 2.07105s (16.4168GiB/s) | 20260505 22:29:29.607 | INFO | Analyzing chunk frames = [6801920,6802432) | 20260505 22:29:29.607 | INFO | MemInfo: { | 20260505 22:29:29.607 | INFO | "gpu" : " 17 GB", | 20260505 22:29:29.607 | INFO | "huge" : "", | 20260505 22:29:29.607 | INFO | "huge_shared" : " 27 GB", | 20260505 22:29:29.607 | INFO | "malloc" : " 14 GB", | 20260505 22:29:29.607 | INFO | "pinned" : " 29 GB" | 20260505 22:29:29.607 | INFO | } | 20260505 22:29:29.607 | INFO | Perf report for the last 5.13404 seconds: | 20260505 22:29:29.607 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:29:29.607 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:29.607 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:29.607 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:29.607 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260505 22:29:29.607 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:29.607 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:29.607 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:29:29.607 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:29.607 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:29.607 | INFO | Uploaded 36GiB to GPU in 2.08169s (17.2937GiB/s) | 20260505 22:29:29.607 | INFO | Downloaded 34GiB to GPU in 2.04259s (16.6455GiB/s) | 20260505 22:29:34.749 | INFO | Analyzing chunk frames = [6802432,6802944) | 20260505 22:29:34.749 | INFO | MemInfo: { | 20260505 22:29:34.749 | INFO | "gpu" : " 17 GB", | 20260505 22:29:34.749 | INFO | "huge" : "", | 20260505 22:29:34.749 | INFO | "huge_shared" : " 27 GB", | 20260505 22:29:34.749 | INFO | "malloc" : " 14 GB", | 20260505 22:29:34.749 | INFO | "pinned" : " 29 GB" | 20260505 22:29:34.749 | INFO | } | 20260505 22:29:34.749 | INFO | Perf report for the last 5.14189 seconds: | 20260505 22:29:34.749 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:29:34.749 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:34.749 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:34.749 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:34.749 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260505 22:29:34.749 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:34.749 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:34.749 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 22:29:34.749 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:34.749 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:34.749 | INFO | Uploaded 36GiB to GPU in 2.15596s (16.6979GiB/s) | 20260505 22:29:34.749 | INFO | Downloaded 34GiB to GPU in 2.12767s (15.9799GiB/s) | 20260505 22:29:39.887 | INFO | Analyzing chunk frames = [6802944,6803456) | 20260505 22:29:39.887 | INFO | MemInfo: { | 20260505 22:29:39.887 | INFO | "gpu" : " 17 GB", | 20260505 22:29:39.887 | INFO | "huge" : "", | 20260505 22:29:39.887 | INFO | "huge_shared" : " 27 GB", | 20260505 22:29:39.887 | INFO | "malloc" : " 14 GB", | 20260505 22:29:39.887 | INFO | "pinned" : " 29 GB" | 20260505 22:29:39.887 | INFO | } | 20260505 22:29:39.888 | INFO | Perf report for the last 5.13853 seconds: | 20260505 22:29:39.888 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:29:39.888 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:39.888 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:39.888 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:39.888 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 22:29:39.888 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:39.888 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:39.888 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:29:39.888 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:39.888 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:39.888 | INFO | Uploaded 36GiB to GPU in 2.11204s (17.0452GiB/s) | 20260505 22:29:39.888 | INFO | Downloaded 34GiB to GPU in 2.00121s (16.9897GiB/s) | 20260505 22:29:40.010 | INFO | Starting a round of Baz IO | 20260505 22:29:43.156 | INFO | Populating RTMetrics report | 20260505 22:29:43.156 | INFO | RTMetrics report populated | 20260505 22:29:43.156 | INFO | Opening tmp RTMetrics json file | 20260505 22:29:43.158 | INFO | Serializing RTMetrics to json | 20260505 22:29:43.160 | INFO | Serialization done, swapping tmp file now... | 20260505 22:29:43.160 | INFO | RTMetrics done | 20260505 22:29:43.236 | INFO | Finished a round of Baz IO: Wrote 1.93 GiB in 3.226 seconds | 20260505 22:29:43.236 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.49% / 0.00% / 78.51% | 20260505 22:29:43.236 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.123938% / 0% / 6.75519e-05% | 20260505 22:29:43.236 | INFO | Baz IO overall: | 20260505 22:29:43.236 | INFO | BytesWritten : 3915491752585 | 20260505 22:29:43.236 | INFO | NumEvents : 1846736883814 | 20260505 22:29:45.028 | INFO | Analyzing chunk frames = [6803456,6803968) | 20260505 22:29:45.028 | INFO | MemInfo: { | 20260505 22:29:45.028 | INFO | "gpu" : " 17 GB", | 20260505 22:29:45.028 | INFO | "huge" : "", | 20260505 22:29:45.028 | INFO | "huge_shared" : " 27 GB", | 20260505 22:29:45.028 | INFO | "malloc" : " 14 GB", | 20260505 22:29:45.028 | INFO | "pinned" : " 29 GB" | 20260505 22:29:45.028 | INFO | } | 20260505 22:29:45.028 | INFO | Perf report for the last 5.14078 seconds: | 20260505 22:29:45.028 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:29:45.028 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:45.028 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:45.028 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:45.028 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260505 22:29:45.028 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:45.028 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260505 22:29:45.028 | INFO | PRE_HQ : 59.7%, 1.8, 54.3ms, 0 | 20260505 22:29:45.028 | INFO | BAZWRITER : 62.0%, 3.6, 116.1ms, 0 | 20260505 22:29:45.028 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 22:29:45.029 | INFO | Uploaded 36GiB to GPU in 2.12166s (16.9678GiB/s) | 20260505 22:29:45.029 | INFO | Downloaded 38GiB to GPU in 2.23672s (16.9892GiB/s) | 20260505 22:29:50.165 | INFO | Analyzing chunk frames = [6803968,6804480) | 20260505 22:29:50.165 | INFO | MemInfo: { | 20260505 22:29:50.165 | INFO | "gpu" : " 17 GB", | 20260505 22:29:50.165 | INFO | "huge" : "", | 20260505 22:29:50.165 | INFO | "huge_shared" : " 27 GB", | 20260505 22:29:50.166 | INFO | "malloc" : " 14 GB", | 20260505 22:29:50.166 | INFO | "pinned" : " 29 GB" | 20260505 22:29:50.166 | INFO | } | 20260505 22:29:50.166 | INFO | Perf report for the last 5.13725 seconds: | 20260505 22:29:50.166 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:29:50.166 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:50.166 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:50.166 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:50.166 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260505 22:29:50.166 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:50.166 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:50.166 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 22:29:50.166 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:50.166 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:50.166 | INFO | Uploaded 36GiB to GPU in 2.09161s (17.2116GiB/s) | 20260505 22:29:50.166 | INFO | Downloaded 34GiB to GPU in 2.03207s (16.7317GiB/s) | 20260505 22:29:55.299 | INFO | Analyzing chunk frames = [6804480,6804992) | 20260505 22:29:55.299 | INFO | MemInfo: { | 20260505 22:29:55.299 | INFO | "gpu" : " 17 GB", | 20260505 22:29:55.299 | INFO | "huge" : "", | 20260505 22:29:55.299 | INFO | "huge_shared" : " 27 GB", | 20260505 22:29:55.299 | INFO | "malloc" : " 14 GB", | 20260505 22:29:55.299 | INFO | "pinned" : " 29 GB" | 20260505 22:29:55.299 | INFO | } | 20260505 22:29:55.299 | INFO | Perf report for the last 5.13386 seconds: | 20260505 22:29:55.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:29:55.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:55.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:55.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:55.299 | INFO | ANALYSIS : 61.9%, 3.0, 96.2ms, 0 | 20260505 22:29:55.300 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:55.300 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:55.300 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 22:29:55.300 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:55.300 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:29:55.300 | INFO | Uploaded 36GiB to GPU in 2.12497s (16.9414GiB/s) | 20260505 22:29:55.300 | INFO | Downloaded 34GiB to GPU in 2.01404s (16.8815GiB/s) | 20260505 22:30:00.438 | INFO | Analyzing chunk frames = [6804992,6805504) | 20260505 22:30:00.438 | INFO | MemInfo: { | 20260505 22:30:00.438 | INFO | "gpu" : " 17 GB", | 20260505 22:30:00.438 | INFO | "huge" : "", | 20260505 22:30:00.438 | INFO | "huge_shared" : " 27 GB", | 20260505 22:30:00.438 | INFO | "malloc" : " 14 GB", | 20260505 22:30:00.438 | INFO | "pinned" : " 29 GB" | 20260505 22:30:00.438 | INFO | } | 20260505 22:30:00.438 | INFO | Perf report for the last 5.13904 seconds: | 20260505 22:30:00.439 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:30:00.439 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:00.439 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:00.439 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:00.439 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260505 22:30:00.439 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:00.439 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:00.439 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 22:30:00.439 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:00.439 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:00.439 | INFO | Uploaded 36GiB to GPU in 2.20397s (16.3342GiB/s) | 20260505 22:30:00.439 | INFO | Downloaded 34GiB to GPU in 2.04331s (16.6397GiB/s) | 20260505 22:30:05.585 | INFO | Analyzing chunk frames = [6805504,6806016) | 20260505 22:30:05.585 | INFO | MemInfo: { | 20260505 22:30:05.585 | INFO | "gpu" : " 17 GB", | 20260505 22:30:05.585 | INFO | "huge" : "", | 20260505 22:30:05.585 | INFO | "huge_shared" : " 27 GB", | 20260505 22:30:05.585 | INFO | "malloc" : " 14 GB", | 20260505 22:30:05.585 | INFO | "pinned" : " 29 GB" | 20260505 22:30:05.585 | INFO | } | 20260505 22:30:05.585 | INFO | Perf report for the last 5.14625 seconds: | 20260505 22:30:05.585 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:30:05.585 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:05.585 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:05.585 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:05.585 | INFO | ANALYSIS : 61.5%, 3.0, 96.1ms, 0 | 20260505 22:30:05.585 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:05.585 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:05.585 | INFO | PRE_HQ : 26.8%, 1.0, 14.0ms, 0 | 20260505 22:30:05.585 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:05.585 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:05.585 | INFO | Uploaded 36GiB to GPU in 2.1402s (16.8208GiB/s) | 20260505 22:30:05.585 | INFO | Downloaded 34GiB to GPU in 2.06507s (16.4643GiB/s) | 20260505 22:30:10.727 | INFO | Analyzing chunk frames = [6806016,6806528) | 20260505 22:30:10.727 | INFO | MemInfo: { | 20260505 22:30:10.727 | INFO | "gpu" : " 17 GB", | 20260505 22:30:10.727 | INFO | "huge" : "", | 20260505 22:30:10.727 | INFO | "huge_shared" : " 27 GB", | 20260505 22:30:10.727 | INFO | "malloc" : " 14 GB", | 20260505 22:30:10.727 | INFO | "pinned" : " 29 GB" | 20260505 22:30:10.727 | INFO | } | 20260505 22:30:10.727 | INFO | Perf report for the last 5.1425 seconds: | 20260505 22:30:10.727 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:30:10.727 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:10.727 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:10.727 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:10.727 | INFO | ANALYSIS : 61.4%, 3.0, 95.9ms, 0 | 20260505 22:30:10.727 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:10.727 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:10.727 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 22:30:10.727 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:10.727 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:10.727 | INFO | Uploaded 36GiB to GPU in 2.06396s (17.4422GiB/s) | 20260505 22:30:10.727 | INFO | Downloaded 34GiB to GPU in 2.03914s (16.6737GiB/s) | 20260505 22:30:15.859 | INFO | Analyzing chunk frames = [6806528,6807040) | 20260505 22:30:15.859 | INFO | MemInfo: { | 20260505 22:30:15.859 | INFO | "gpu" : " 17 GB", | 20260505 22:30:15.859 | INFO | "huge" : "", | 20260505 22:30:15.859 | INFO | "huge_shared" : " 27 GB", | 20260505 22:30:15.860 | INFO | "malloc" : " 14 GB", | 20260505 22:30:15.860 | INFO | "pinned" : " 29 GB" | 20260505 22:30:15.860 | INFO | } | 20260505 22:30:15.860 | INFO | Perf report for the last 5.13228 seconds: | 20260505 22:30:15.860 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:30:15.860 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:15.860 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:15.860 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:15.860 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 22:30:15.860 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:15.860 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:15.860 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 22:30:15.860 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:15.860 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:15.860 | INFO | Uploaded 36GiB to GPU in 2.09599s (17.1757GiB/s) | 20260505 22:30:15.860 | INFO | Downloaded 34GiB to GPU in 2.07241s (16.4061GiB/s) | 20260505 22:30:20.996 | INFO | Analyzing chunk frames = [6807040,6807552) | 20260505 22:30:20.996 | INFO | MemInfo: { | 20260505 22:30:20.996 | INFO | "gpu" : " 17 GB", | 20260505 22:30:20.996 | INFO | "huge" : "", | 20260505 22:30:20.996 | INFO | "huge_shared" : " 27 GB", | 20260505 22:30:20.996 | INFO | "malloc" : " 14 GB", | 20260505 22:30:20.996 | INFO | "pinned" : " 29 GB" | 20260505 22:30:20.996 | INFO | } | 20260505 22:30:20.996 | INFO | Perf report for the last 5.13626 seconds: | 20260505 22:30:20.996 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:30:20.996 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:20.996 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:20.996 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:20.996 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 22:30:20.996 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:20.996 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:20.996 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:30:20.996 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:20.996 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:20.996 | INFO | Uploaded 36GiB to GPU in 2.11434s (17.0266GiB/s) | 20260505 22:30:20.996 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 22:30:20.996 | INFO | Total mem usage: 15.3 GiB | 20260505 22:30:20.996 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 22:30:20.996 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -9.01%) | 20260505 22:30:20.996 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 22:30:20.996 | INFO | High Water Breakdown: | 20260505 22:30:20.996 | INFO | 60.9% : 8.56 GiB ( 90% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 22:30:20.996 | INFO | 39% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 22:30:20.996 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 22:30:20.996 | INFO | ------------------------------------------ | 20260505 22:30:20.996 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:30:20.996 | INFO | Downloaded 34GiB to GPU in 2.02039s (16.8284GiB/s) | 20260505 22:30:20.996 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 22:30:20.996 | INFO | Total mem usage: 26.5 GiB | 20260505 22:30:20.996 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 22:30:20.996 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 22:30:20.996 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 22:30:20.996 | INFO | High Water Breakdown: | 20260505 22:30:20.996 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:30:20.996 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:30:20.996 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:30:20.996 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:30:20.996 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:30:20.996 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:30:20.996 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 22:30:20.996 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 22:30:20.996 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:30:20.996 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 22:30:20.996 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 22:30:20.996 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 22:30:20.996 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 22:30:20.996 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 22:30:20.996 | INFO | ------------------------------------------ | 20260505 22:30:20.996 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:30:20.996 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 22:30:20.996 | INFO | Total mem usage: 25.6 GiB | 20260505 22:30:20.996 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 22:30:20.996 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 22:30:20.996 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 22:30:20.997 | INFO | High Water Breakdown: | 20260505 22:30:20.997 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 22:30:20.997 | INFO | ------------------------------------------ | 20260505 22:30:20.997 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:30:20.997 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 22:30:20.997 | INFO | Total mem usage: 15.7 GiB | 20260505 22:30:20.997 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 22:30:20.997 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 22:30:20.997 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 22:30:20.997 | INFO | High Water Breakdown: | 20260505 22:30:20.997 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 22:30:20.997 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 22:30:20.997 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 22:30:20.997 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 22:30:20.997 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 22:30:20.997 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 22:30:20.997 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 22:30:20.997 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 22:30:20.997 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 22:30:20.997 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:30:20.997 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 22:30:20.997 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:30:20.997 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 22:30:20.997 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 22:30:20.997 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 22:30:20.997 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 22:30:20.997 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:30:20.997 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 22:30:20.997 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:30:20.997 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:30:20.997 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:30:20.997 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 22:30:20.997 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 22:30:20.997 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:30:20.997 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 22:30:20.997 | INFO | ------------------------------------------ | 20260505 22:30:20.997 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:30:20.997 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 22:30:20.997 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:30:20.997 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:30:20.997 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.39e+01, 1.35e+01 / 9.84e-01 | 20260505 22:30:20.997 | INFO | PerfReport: 0.29%, AggregateStats -- 9.18e-02 / 1.04e-01, 1.02e-02 / 6.05e-03 | 20260505 22:30:20.997 | INFO | PerfReport: 11.80%, Baseline -- 3.70e+00 / 4.49e+00, 5.57e-01 / 2.23e-01 | 20260505 22:30:20.997 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.51e+00, 1.86e-01 / 9.49e-02 | 20260505 22:30:20.997 | INFO | PerfReport: 15.97%, DME -- 4.00e+01 / 4.39e+01, 3.81e+00 / 1.26e+00 | 20260505 22:30:20.997 | INFO | PerfReport: 57.21%, FrameLabeling -- 1.79e+01 / 1.93e+01, 1.45e+00 / 9.81e-01 | 20260505 22:30:20.997 | INFO | PerfReport: 7.19%, PulseAccumulating -- 2.25e+00 / 2.79e+00, 4.28e-01 / 2.37e-01 | 20260505 22:30:20.997 | INFO | PerfReport: 3.17%, Metrics -- 9.93e-01 / 1.30e+00, 2.67e-01 / 1.28e-01 | 20260505 22:30:20.997 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.44e-01 / 4.07e-01, 5.81e-02 / 3.60e-02 | 20260505 22:30:20.997 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 22:30:20.997 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:30:20.997 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:30:20.997 | INFO | PerfReport: Full Execution -- 9.88e+01 / -nan, 2.50e+01 / 0.00e+00 | 20260505 22:30:20.997 | INFO | PerfReport: 22.43%, Upload -- 2.22e+01 / 0.00e+00, 7.58e+00 / 0.00e+00 | 20260505 22:30:20.997 | INFO | PerfReport: 21.83%, Download -- 2.16e+01 / 0.00e+00, 7.57e+00 / 0.00e+00 | 20260505 22:30:20.997 | INFO | PerfReport: 31.73%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 22:30:20.997 | INFO | PerfReport: 2.22%, LockUpload -- 2.19e+00 / 0.00e+00, 7.88e+00 / 0.00e+00 | 20260505 22:30:20.997 | INFO | PerfReport: 1.34%, LockDownload -- 1.32e+00 / 0.00e+00, 2.85e+00 / 0.00e+00 | 20260505 22:30:20.997 | INFO | PerfReport: 20.43%, LockCompute -- 2.02e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260505 22:30:20.997 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.68e-02 / 0.00e+00 | 20260505 22:30:21.130 | INFO | Starting a round of Baz IO | 20260505 22:30:24.288 | INFO | Populating RTMetrics report | 20260505 22:30:24.288 | INFO | RTMetrics report populated | 20260505 22:30:24.288 | INFO | Opening tmp RTMetrics json file | 20260505 22:30:24.289 | INFO | Serializing RTMetrics to json | 20260505 22:30:24.290 | INFO | Serialization done, swapping tmp file now... | 20260505 22:30:24.292 | INFO | RTMetrics done | 20260505 22:30:24.377 | INFO | Finished a round of Baz IO: Wrote 1.93 GiB in 3.246 seconds | 20260505 22:30:24.377 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.49% / 0.00% / 78.51% | 20260505 22:30:24.377 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.123696% / 0% / 7.94729e-05% | 20260505 22:30:24.377 | INFO | Baz IO overall: | 20260505 22:30:24.377 | INFO | BytesWritten : 3917563580559 | 20260505 22:30:24.377 | INFO | NumEvents : 1847690018846 | 20260505 22:30:26.140 | INFO | Analyzing chunk frames = [6807552,6808064) | 20260505 22:30:26.140 | INFO | MemInfo: { | 20260505 22:30:26.140 | INFO | "gpu" : " 17 GB", | 20260505 22:30:26.140 | INFO | "huge" : "", | 20260505 22:30:26.140 | INFO | "huge_shared" : " 27 GB", | 20260505 22:30:26.140 | INFO | "malloc" : " 14 GB", | 20260505 22:30:26.140 | INFO | "pinned" : " 29 GB" | 20260505 22:30:26.140 | INFO | } | 20260505 22:30:26.140 | INFO | Perf report for the last 5.14378 seconds: | 20260505 22:30:26.140 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:30:26.140 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:26.140 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:26.140 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:26.140 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 22:30:26.140 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:26.140 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260505 22:30:26.140 | INFO | PRE_HQ : 59.9%, 1.8, 54.8ms, 0 | 20260505 22:30:26.140 | INFO | BAZWRITER : 62.6%, 3.6, 115.1ms, 0 | 20260505 22:30:26.140 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 22:30:26.140 | INFO | Uploaded 36GiB to GPU in 2.17071s (16.5844GiB/s) | 20260505 22:30:26.140 | INFO | Downloaded 38GiB to GPU in 2.2473s (16.9092GiB/s) | 20260505 22:30:31.278 | INFO | Analyzing chunk frames = [6808064,6808576) | 20260505 22:30:31.278 | INFO | MemInfo: { | 20260505 22:30:31.278 | INFO | "gpu" : " 17 GB", | 20260505 22:30:31.278 | INFO | "huge" : "", | 20260505 22:30:31.279 | INFO | "huge_shared" : " 27 GB", | 20260505 22:30:31.279 | INFO | "malloc" : " 14 GB", | 20260505 22:30:31.279 | INFO | "pinned" : " 29 GB" | 20260505 22:30:31.279 | INFO | } | 20260505 22:30:31.279 | INFO | Perf report for the last 5.13906 seconds: | 20260505 22:30:31.279 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:30:31.279 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:31.279 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:31.279 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:31.279 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 22:30:31.279 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:31.279 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:31.279 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 22:30:31.279 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:31.279 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:31.279 | INFO | Uploaded 36GiB to GPU in 2.08929s (17.2308GiB/s) | 20260505 22:30:31.279 | INFO | Downloaded 34GiB to GPU in 2.02581s (16.7834GiB/s) | 20260505 22:30:36.426 | INFO | Analyzing chunk frames = [6808576,6809088) | 20260505 22:30:36.426 | INFO | MemInfo: { | 20260505 22:30:36.426 | INFO | "gpu" : " 17 GB", | 20260505 22:30:36.426 | INFO | "huge" : "", | 20260505 22:30:36.426 | INFO | "huge_shared" : " 27 GB", | 20260505 22:30:36.426 | INFO | "malloc" : " 14 GB", | 20260505 22:30:36.426 | INFO | "pinned" : " 29 GB" | 20260505 22:30:36.426 | INFO | } | 20260505 22:30:36.426 | INFO | Perf report for the last 5.14737 seconds: | 20260505 22:30:36.426 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:30:36.426 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:36.426 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:36.426 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:36.426 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260505 22:30:36.426 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:36.426 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:36.426 | INFO | PRE_HQ : 27.0%, 1.0, 14.1ms, 0 | 20260505 22:30:36.426 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:36.426 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:36.426 | INFO | Uploaded 36GiB to GPU in 2.14398s (16.7912GiB/s) | 20260505 22:30:36.426 | INFO | Downloaded 34GiB to GPU in 2.02893s (16.7576GiB/s) | 20260505 22:30:41.562 | INFO | Analyzing chunk frames = [6809088,6809600) | 20260505 22:30:41.562 | INFO | MemInfo: { | 20260505 22:30:41.562 | INFO | "gpu" : " 17 GB", | 20260505 22:30:41.562 | INFO | "huge" : "", | 20260505 22:30:41.562 | INFO | "huge_shared" : " 27 GB", | 20260505 22:30:41.562 | INFO | "malloc" : " 14 GB", | 20260505 22:30:41.562 | INFO | "pinned" : " 29 GB" | 20260505 22:30:41.562 | INFO | } | 20260505 22:30:41.562 | INFO | Perf report for the last 5.13562 seconds: | 20260505 22:30:41.562 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:30:41.562 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:41.562 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:41.562 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:41.562 | INFO | ANALYSIS : 61.8%, 3.0, 96.0ms, 0 | 20260505 22:30:41.562 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:41.562 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:41.562 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:30:41.562 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:41.562 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:41.562 | INFO | Uploaded 36GiB to GPU in 2.14196s (16.807GiB/s) | 20260505 22:30:41.562 | INFO | Downloaded 34GiB to GPU in 2.02122s (16.8216GiB/s) | 20260505 22:30:46.701 | INFO | Analyzing chunk frames = [6809600,6810112) | 20260505 22:30:46.701 | INFO | MemInfo: { | 20260505 22:30:46.701 | INFO | "gpu" : " 17 GB", | 20260505 22:30:46.701 | INFO | "huge" : "", | 20260505 22:30:46.701 | INFO | "huge_shared" : " 27 GB", | 20260505 22:30:46.701 | INFO | "malloc" : " 14 GB", | 20260505 22:30:46.701 | INFO | "pinned" : " 29 GB" | 20260505 22:30:46.701 | INFO | } | 20260505 22:30:46.701 | INFO | Perf report for the last 5.13939 seconds: | 20260505 22:30:46.701 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:30:46.701 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:46.701 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:46.701 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:46.701 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260505 22:30:46.701 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:46.701 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:46.701 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 22:30:46.701 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:46.701 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:46.701 | INFO | Uploaded 36GiB to GPU in 2.09146s (17.2128GiB/s) | 20260505 22:30:46.701 | INFO | Downloaded 34GiB to GPU in 2.04747s (16.6059GiB/s) | 20260505 22:30:51.836 | INFO | Analyzing chunk frames = [6810112,6810624) | 20260505 22:30:51.836 | INFO | MemInfo: { | 20260505 22:30:51.836 | INFO | "gpu" : " 17 GB", | 20260505 22:30:51.836 | INFO | "huge" : "", | 20260505 22:30:51.836 | INFO | "huge_shared" : " 27 GB", | 20260505 22:30:51.836 | INFO | "malloc" : " 14 GB", | 20260505 22:30:51.836 | INFO | "pinned" : " 29 GB" | 20260505 22:30:51.836 | INFO | } | 20260505 22:30:51.836 | INFO | Perf report for the last 5.13503 seconds: | 20260505 22:30:51.836 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:30:51.836 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:51.836 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:51.836 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:51.836 | INFO | ANALYSIS : 61.5%, 3.0, 96.0ms, 0 | 20260505 22:30:51.836 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:51.836 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:51.836 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260505 22:30:51.836 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:51.836 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:51.836 | INFO | Uploaded 36GiB to GPU in 2.07416s (17.3565GiB/s) | 20260505 22:30:51.836 | INFO | Downloaded 34GiB to GPU in 2.03498s (16.7078GiB/s) | 20260505 22:30:56.977 | INFO | Analyzing chunk frames = [6810624,6811136) | 20260505 22:30:56.977 | INFO | MemInfo: { | 20260505 22:30:56.977 | INFO | "gpu" : " 17 GB", | 20260505 22:30:56.977 | INFO | "huge" : "", | 20260505 22:30:56.977 | INFO | "huge_shared" : " 27 GB", | 20260505 22:30:56.977 | INFO | "malloc" : " 14 GB", | 20260505 22:30:56.977 | INFO | "pinned" : " 29 GB" | 20260505 22:30:56.977 | INFO | } | 20260505 22:30:56.977 | INFO | Perf report for the last 5.14126 seconds: | 20260505 22:30:56.977 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:30:56.977 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:56.977 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:56.977 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:56.977 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 22:30:56.977 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:56.977 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:56.977 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 22:30:56.978 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:56.978 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:30:56.978 | INFO | Uploaded 36GiB to GPU in 2.16215s (16.6501GiB/s) | 20260505 22:30:56.978 | INFO | Downloaded 34GiB to GPU in 2.07768s (16.3644GiB/s) | 20260505 22:31:02.117 | INFO | Analyzing chunk frames = [6811136,6811648) | 20260505 22:31:02.117 | INFO | MemInfo: { | 20260505 22:31:02.117 | INFO | "gpu" : " 17 GB", | 20260505 22:31:02.117 | INFO | "huge" : "", | 20260505 22:31:02.117 | INFO | "huge_shared" : " 27 GB", | 20260505 22:31:02.117 | INFO | "malloc" : " 14 GB", | 20260505 22:31:02.117 | INFO | "pinned" : " 29 GB" | 20260505 22:31:02.117 | INFO | } | 20260505 22:31:02.117 | INFO | Perf report for the last 5.13945 seconds: | 20260505 22:31:02.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:31:02.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:02.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:02.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:02.117 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260505 22:31:02.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:02.117 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:02.117 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 22:31:02.117 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:02.117 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:02.117 | INFO | Uploaded 36GiB to GPU in 2.11853s (16.9929GiB/s) | 20260505 22:31:02.117 | INFO | Downloaded 34GiB to GPU in 2.00406s (16.9656GiB/s) | 20260505 22:31:02.250 | INFO | Starting a round of Baz IO | 20260505 22:31:05.403 | INFO | Populating RTMetrics report | 20260505 22:31:05.403 | INFO | RTMetrics report populated | 20260505 22:31:05.403 | INFO | Opening tmp RTMetrics json file | 20260505 22:31:05.404 | INFO | Serializing RTMetrics to json | 20260505 22:31:05.405 | INFO | Serialization done, swapping tmp file now... | 20260505 22:31:05.405 | INFO | RTMetrics done | 20260505 22:31:05.484 | INFO | Finished a round of Baz IO: Wrote 1.93 GiB in 3.233 seconds | 20260505 22:31:05.484 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.49% / 0.00% / 78.51% | 20260505 22:31:05.484 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.123425% / 0% / 8.34465e-05% | 20260505 22:31:05.484 | INFO | Baz IO overall: | 20260505 22:31:05.484 | INFO | BytesWritten : 3919636015720 | 20260505 22:31:05.484 | INFO | NumEvents : 1848643782304 | 20260505 22:31:07.250 | INFO | Analyzing chunk frames = [6811648,6812160) | 20260505 22:31:07.250 | INFO | MemInfo: { | 20260505 22:31:07.250 | INFO | "gpu" : " 17 GB", | 20260505 22:31:07.250 | INFO | "huge" : "", | 20260505 22:31:07.250 | INFO | "huge_shared" : " 27 GB", | 20260505 22:31:07.250 | INFO | "malloc" : " 14 GB", | 20260505 22:31:07.250 | INFO | "pinned" : " 29 GB" | 20260505 22:31:07.250 | INFO | } | 20260505 22:31:07.250 | INFO | Perf report for the last 5.13323 seconds: | 20260505 22:31:07.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:31:07.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:07.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:07.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:07.250 | INFO | ANALYSIS : 63.0%, 3.0, 98.1ms, 0 | 20260505 22:31:07.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:07.250 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260505 22:31:07.250 | INFO | PRE_HQ : 60.0%, 1.8, 54.5ms, 0 | 20260505 22:31:07.250 | INFO | BAZWRITER : 62.5%, 3.6, 115.3ms, 0 | 20260505 22:31:07.250 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 22:31:07.250 | INFO | Uploaded 36GiB to GPU in 2.17153s (16.5781GiB/s) | 20260505 22:31:07.250 | INFO | Downloaded 38GiB to GPU in 2.23943s (16.9686GiB/s) | 20260505 22:31:12.398 | INFO | Analyzing chunk frames = [6812160,6812672) | 20260505 22:31:12.398 | INFO | MemInfo: { | 20260505 22:31:12.398 | INFO | "gpu" : " 17 GB", | 20260505 22:31:12.398 | INFO | "huge" : "", | 20260505 22:31:12.398 | INFO | "huge_shared" : " 27 GB", | 20260505 22:31:12.398 | INFO | "malloc" : " 14 GB", | 20260505 22:31:12.398 | INFO | "pinned" : " 29 GB" | 20260505 22:31:12.398 | INFO | } | 20260505 22:31:12.398 | INFO | Perf report for the last 5.14828 seconds: | 20260505 22:31:12.398 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:31:12.398 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:12.398 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:12.398 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:12.399 | INFO | ANALYSIS : 61.8%, 3.0, 96.7ms, 0 | 20260505 22:31:12.399 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:12.399 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:12.399 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 22:31:12.399 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:12.399 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:12.399 | INFO | Uploaded 36GiB to GPU in 2.09586s (17.1767GiB/s) | 20260505 22:31:12.399 | INFO | Downloaded 34GiB to GPU in 2.02205s (16.8146GiB/s) | 20260505 22:31:17.540 | INFO | Analyzing chunk frames = [6812672,6813184) | 20260505 22:31:17.540 | INFO | MemInfo: { | 20260505 22:31:17.540 | INFO | "gpu" : " 17 GB", | 20260505 22:31:17.540 | INFO | "huge" : "", | 20260505 22:31:17.540 | INFO | "huge_shared" : " 27 GB", | 20260505 22:31:17.540 | INFO | "malloc" : " 14 GB", | 20260505 22:31:17.540 | INFO | "pinned" : " 29 GB" | 20260505 22:31:17.540 | INFO | } | 20260505 22:31:17.540 | INFO | Perf report for the last 5.14177 seconds: | 20260505 22:31:17.540 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:31:17.540 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:17.540 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:17.540 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:17.540 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260505 22:31:17.540 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:17.540 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:17.540 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:31:17.540 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:17.540 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:17.540 | INFO | Uploaded 36GiB to GPU in 2.08919s (17.2316GiB/s) | 20260505 22:31:17.540 | INFO | Downloaded 34GiB to GPU in 2.02977s (16.7507GiB/s) | 20260505 22:31:22.673 | INFO | Analyzing chunk frames = [6813184,6813696) | 20260505 22:31:22.673 | INFO | MemInfo: { | 20260505 22:31:22.673 | INFO | "gpu" : " 17 GB", | 20260505 22:31:22.673 | INFO | "huge" : "", | 20260505 22:31:22.673 | INFO | "huge_shared" : " 27 GB", | 20260505 22:31:22.673 | INFO | "malloc" : " 14 GB", | 20260505 22:31:22.673 | INFO | "pinned" : " 29 GB" | 20260505 22:31:22.673 | INFO | } | 20260505 22:31:22.673 | INFO | Perf report for the last 5.13251 seconds: | 20260505 22:31:22.673 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:31:22.673 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:22.673 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:22.673 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:22.673 | INFO | ANALYSIS : 61.8%, 3.0, 95.9ms, 0 | 20260505 22:31:22.673 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:22.673 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:22.673 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 22:31:22.673 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:22.673 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:22.673 | INFO | Uploaded 36GiB to GPU in 2.14359s (16.7943GiB/s) | 20260505 22:31:22.673 | INFO | Downloaded 34GiB to GPU in 2.01549s (16.8694GiB/s) | 20260505 22:31:27.805 | INFO | Analyzing chunk frames = [6813696,6814208) | 20260505 22:31:27.805 | INFO | MemInfo: { | 20260505 22:31:27.805 | INFO | "gpu" : " 17 GB", | 20260505 22:31:27.805 | INFO | "huge" : "", | 20260505 22:31:27.805 | INFO | "huge_shared" : " 27 GB", | 20260505 22:31:27.805 | INFO | "malloc" : " 14 GB", | 20260505 22:31:27.805 | INFO | "pinned" : " 29 GB" | 20260505 22:31:27.805 | INFO | } | 20260505 22:31:27.805 | INFO | Perf report for the last 5.13259 seconds: | 20260505 22:31:27.805 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:31:27.805 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:27.805 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:27.805 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:27.805 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260505 22:31:27.805 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:27.805 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:27.805 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 22:31:27.805 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:27.805 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:27.806 | INFO | Uploaded 36GiB to GPU in 2.13985s (16.8236GiB/s) | 20260505 22:31:27.806 | INFO | Downloaded 34GiB to GPU in 2.06408s (16.4722GiB/s) | 20260505 22:31:32.951 | INFO | Analyzing chunk frames = [6814208,6814720) | 20260505 22:31:32.951 | INFO | MemInfo: { | 20260505 22:31:32.951 | INFO | "gpu" : " 17 GB", | 20260505 22:31:32.951 | INFO | "huge" : "", | 20260505 22:31:32.951 | INFO | "huge_shared" : " 27 GB", | 20260505 22:31:32.951 | INFO | "malloc" : " 14 GB", | 20260505 22:31:32.951 | INFO | "pinned" : " 29 GB" | 20260505 22:31:32.951 | INFO | } | 20260505 22:31:32.951 | INFO | Perf report for the last 5.1456 seconds: | 20260505 22:31:32.951 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:31:32.951 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:32.951 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:32.951 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:32.951 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 22:31:32.951 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:32.951 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:32.951 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 22:31:32.951 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:32.951 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:32.951 | INFO | Uploaded 36GiB to GPU in 2.16561s (16.6235GiB/s) | 20260505 22:31:32.951 | INFO | Downloaded 34GiB to GPU in 2.07027s (16.423GiB/s) | 20260505 22:31:38.091 | INFO | Analyzing chunk frames = [6814720,6815232) | 20260505 22:31:38.091 | INFO | MemInfo: { | 20260505 22:31:38.091 | INFO | "gpu" : " 17 GB", | 20260505 22:31:38.091 | INFO | "huge" : "", | 20260505 22:31:38.091 | INFO | "huge_shared" : " 27 GB", | 20260505 22:31:38.091 | INFO | "malloc" : " 14 GB", | 20260505 22:31:38.091 | INFO | "pinned" : " 29 GB" | 20260505 22:31:38.091 | INFO | } | 20260505 22:31:38.091 | INFO | Perf report for the last 5.13984 seconds: | 20260505 22:31:38.091 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:31:38.091 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:38.091 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:38.091 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:38.091 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260505 22:31:38.091 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:38.091 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:38.091 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 22:31:38.091 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:38.091 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:38.091 | INFO | Uploaded 36GiB to GPU in 2.12045s (16.9775GiB/s) | 20260505 22:31:38.091 | INFO | Downloaded 34GiB to GPU in 2.10147s (16.1792GiB/s) | 20260505 22:31:43.231 | INFO | Analyzing chunk frames = [6815232,6815744) | 20260505 22:31:43.231 | INFO | MemInfo: { | 20260505 22:31:43.231 | INFO | "gpu" : " 17 GB", | 20260505 22:31:43.231 | INFO | "huge" : "", | 20260505 22:31:43.231 | INFO | "huge_shared" : " 27 GB", | 20260505 22:31:43.231 | INFO | "malloc" : " 14 GB", | 20260505 22:31:43.231 | INFO | "pinned" : " 29 GB" | 20260505 22:31:43.231 | INFO | } | 20260505 22:31:43.231 | INFO | Perf report for the last 5.14053 seconds: | 20260505 22:31:43.231 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:31:43.231 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:43.231 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:43.231 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:43.231 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 22:31:43.231 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:43.231 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:43.231 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:31:43.231 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:43.231 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:43.232 | INFO | Uploaded 36GiB to GPU in 2.10981s (17.0631GiB/s) | 20260505 22:31:43.232 | INFO | Downloaded 34GiB to GPU in 1.99731s (17.0229GiB/s) | 20260505 22:31:43.232 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 22:31:43.232 | INFO | Total mem usage: 15.3 GiB | 20260505 22:31:43.232 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 22:31:43.232 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -9.04%) | 20260505 22:31:43.232 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 22:31:43.232 | INFO | High Water Breakdown: | 20260505 22:31:43.232 | INFO | 60.9% : 8.56 GiB ( 90% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 22:31:43.232 | INFO | 39% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 22:31:43.232 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 22:31:43.232 | INFO | ------------------------------------------ | 20260505 22:31:43.232 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:31:43.232 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 22:31:43.232 | INFO | Total mem usage: 26.5 GiB | 20260505 22:31:43.232 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 22:31:43.232 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 22:31:43.232 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 22:31:43.232 | INFO | High Water Breakdown: | 20260505 22:31:43.232 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:31:43.232 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:31:43.232 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:31:43.232 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:31:43.232 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:31:43.232 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:31:43.232 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 22:31:43.232 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 22:31:43.232 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:31:43.232 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 22:31:43.232 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 22:31:43.232 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 22:31:43.232 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 22:31:43.232 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 22:31:43.232 | INFO | ------------------------------------------ | 20260505 22:31:43.232 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:31:43.232 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 22:31:43.232 | INFO | Total mem usage: 25.6 GiB | 20260505 22:31:43.232 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 22:31:43.232 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 22:31:43.232 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 22:31:43.232 | INFO | High Water Breakdown: | 20260505 22:31:43.232 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 22:31:43.232 | INFO | ------------------------------------------ | 20260505 22:31:43.232 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:31:43.232 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 22:31:43.232 | INFO | Total mem usage: 15.7 GiB | 20260505 22:31:43.232 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 22:31:43.232 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 22:31:43.232 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 22:31:43.232 | INFO | High Water Breakdown: | 20260505 22:31:43.232 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 22:31:43.232 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 22:31:43.232 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 22:31:43.232 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 22:31:43.232 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 22:31:43.232 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 22:31:43.232 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 22:31:43.232 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 22:31:43.232 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 22:31:43.232 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:31:43.232 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 22:31:43.232 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:31:43.232 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 22:31:43.232 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 22:31:43.232 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 22:31:43.232 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 22:31:43.232 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:31:43.232 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 22:31:43.232 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:31:43.232 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:31:43.232 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:31:43.232 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 22:31:43.232 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 22:31:43.232 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:31:43.232 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 22:31:43.232 | INFO | ------------------------------------------ | 20260505 22:31:43.232 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:31:43.232 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 22:31:43.232 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:31:43.232 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:31:43.232 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.39e+01, 1.34e+01 / 7.93e-01 | 20260505 22:31:43.232 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.04e-01, 1.03e-02 / 4.23e-03 | 20260505 22:31:43.232 | INFO | PerfReport: 11.80%, Baseline -- 3.70e+00 / 4.53e+00, 5.65e-01 / 1.91e-01 | 20260505 22:31:43.232 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.51e+00, 1.89e-01 / 1.06e-01 | 20260505 22:31:43.232 | INFO | PerfReport: 15.96%, DME -- 4.01e+01 / 4.39e+01, 3.81e+00 / 6.88e-01 | 20260505 22:31:43.232 | INFO | PerfReport: 57.29%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.53e+00 / 1.21e+00 | 20260505 22:31:43.232 | INFO | PerfReport: 7.14%, PulseAccumulating -- 2.24e+00 / 2.77e+00, 4.29e-01 / 2.52e-01 | 20260505 22:31:43.232 | INFO | PerfReport: 3.15%, Metrics -- 9.89e-01 / 1.29e+00, 2.68e-01 / 1.43e-01 | 20260505 22:31:43.232 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.44e-01 / 3.98e-01, 5.81e-02 / 3.82e-02 | 20260505 22:31:43.233 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 22:31:43.233 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:31:43.233 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:31:43.233 | INFO | PerfReport: Full Execution -- 9.88e+01 / -nan, 2.52e+01 / 0.00e+00 | 20260505 22:31:43.233 | INFO | PerfReport: 22.43%, Upload -- 2.22e+01 / 0.00e+00, 7.68e+00 / 0.00e+00 | 20260505 22:31:43.233 | INFO | PerfReport: 21.77%, Download -- 2.15e+01 / 0.00e+00, 7.65e+00 / 0.00e+00 | 20260505 22:31:43.233 | INFO | PerfReport: 31.77%, Compute -- 3.14e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 22:31:43.233 | INFO | PerfReport: 2.23%, LockUpload -- 2.21e+00 / 0.00e+00, 7.98e+00 / 0.00e+00 | 20260505 22:31:43.233 | INFO | PerfReport: 1.28%, LockDownload -- 1.26e+00 / 0.00e+00, 2.79e+00 / 0.00e+00 | 20260505 22:31:43.233 | INFO | PerfReport: 20.50%, LockCompute -- 2.02e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260505 22:31:43.233 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.62e-02 / 0.00e+00 | 20260505 22:31:43.358 | INFO | Starting a round of Baz IO | 20260505 22:31:46.514 | INFO | Populating RTMetrics report | 20260505 22:31:46.514 | INFO | RTMetrics report populated | 20260505 22:31:46.514 | INFO | Opening tmp RTMetrics json file | 20260505 22:31:46.517 | INFO | Serializing RTMetrics to json | 20260505 22:31:46.519 | INFO | Serialization done, swapping tmp file now... | 20260505 22:31:46.521 | INFO | RTMetrics done | 20260505 22:31:46.594 | INFO | Finished a round of Baz IO: Wrote 1.926 GiB in 3.235 seconds | 20260505 22:31:46.594 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.49% / 0.00% / 78.51% | 20260505 22:31:46.594 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.122948% / 0% / 7.94729e-05% | 20260505 22:31:46.594 | INFO | Baz IO overall: | 20260505 22:31:46.594 | INFO | BytesWritten : 3921704049631 | 20260505 22:31:46.594 | INFO | NumEvents : 1849594421668 | 20260505 22:31:48.371 | INFO | Analyzing chunk frames = [6815744,6816256) | 20260505 22:31:48.371 | INFO | MemInfo: { | 20260505 22:31:48.371 | INFO | "gpu" : " 17 GB", | 20260505 22:31:48.371 | INFO | "huge" : "", | 20260505 22:31:48.371 | INFO | "huge_shared" : " 27 GB", | 20260505 22:31:48.371 | INFO | "malloc" : " 14 GB", | 20260505 22:31:48.371 | INFO | "pinned" : " 29 GB" | 20260505 22:31:48.371 | INFO | } | 20260505 22:31:48.371 | INFO | Perf report for the last 5.14004 seconds: | 20260505 22:31:48.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:31:48.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:48.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:48.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:48.371 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 22:31:48.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:48.371 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260505 22:31:48.371 | INFO | PRE_HQ : 60.2%, 1.8, 54.9ms, 0 | 20260505 22:31:48.371 | INFO | BAZWRITER : 62.3%, 3.6, 116.4ms, 0 | 20260505 22:31:48.371 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 22:31:48.372 | INFO | Uploaded 36GiB to GPU in 2.17308s (16.5664GiB/s) | 20260505 22:31:48.372 | INFO | Downloaded 38GiB to GPU in 2.23865s (16.9745GiB/s) | 20260505 22:31:53.510 | INFO | Analyzing chunk frames = [6816256,6816768) | 20260505 22:31:53.510 | INFO | MemInfo: { | 20260505 22:31:53.510 | INFO | "gpu" : " 17 GB", | 20260505 22:31:53.510 | INFO | "huge" : "", | 20260505 22:31:53.510 | INFO | "huge_shared" : " 27 GB", | 20260505 22:31:53.510 | INFO | "malloc" : " 14 GB", | 20260505 22:31:53.511 | INFO | "pinned" : " 29 GB" | 20260505 22:31:53.511 | INFO | } | 20260505 22:31:53.511 | INFO | Perf report for the last 5.13927 seconds: | 20260505 22:31:53.511 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:31:53.511 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:53.511 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:53.511 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:53.511 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 22:31:53.511 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:53.511 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:53.511 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:31:53.511 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:53.511 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:53.511 | INFO | Uploaded 36GiB to GPU in 2.08722s (17.2478GiB/s) | 20260505 22:31:53.511 | INFO | Downloaded 34GiB to GPU in 2.02888s (16.758GiB/s) | 20260505 22:31:58.653 | INFO | Analyzing chunk frames = [6816768,6817280) | 20260505 22:31:58.653 | INFO | MemInfo: { | 20260505 22:31:58.653 | INFO | "gpu" : " 17 GB", | 20260505 22:31:58.653 | INFO | "huge" : "", | 20260505 22:31:58.654 | INFO | "huge_shared" : " 27 GB", | 20260505 22:31:58.654 | INFO | "malloc" : " 14 GB", | 20260505 22:31:58.654 | INFO | "pinned" : " 29 GB" | 20260505 22:31:58.654 | INFO | } | 20260505 22:31:58.654 | INFO | Perf report for the last 5.14296 seconds: | 20260505 22:31:58.654 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:31:58.654 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:58.654 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:58.654 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:58.654 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260505 22:31:58.654 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:58.654 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:58.654 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:31:58.654 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:58.654 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:31:58.654 | INFO | Uploaded 36GiB to GPU in 2.12342s (16.9538GiB/s) | 20260505 22:31:58.654 | INFO | Downloaded 34GiB to GPU in 2.01192s (16.8993GiB/s) | 20260505 22:32:03.788 | INFO | Analyzing chunk frames = [6817280,6817792) | 20260505 22:32:03.788 | INFO | MemInfo: { | 20260505 22:32:03.788 | INFO | "gpu" : " 17 GB", | 20260505 22:32:03.788 | INFO | "huge" : "", | 20260505 22:32:03.789 | INFO | "huge_shared" : " 27 GB", | 20260505 22:32:03.789 | INFO | "malloc" : " 14 GB", | 20260505 22:32:03.789 | INFO | "pinned" : " 29 GB" | 20260505 22:32:03.789 | INFO | } | 20260505 22:32:03.789 | INFO | Perf report for the last 5.13492 seconds: | 20260505 22:32:03.789 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:32:03.789 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:03.789 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:03.789 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:03.789 | INFO | ANALYSIS : 61.7%, 3.0, 95.8ms, 0 | 20260505 22:32:03.789 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:03.789 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:03.789 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 22:32:03.789 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:03.789 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:03.789 | INFO | Uploaded 36GiB to GPU in 2.14124s (16.8127GiB/s) | 20260505 22:32:03.789 | INFO | Downloaded 34GiB to GPU in 2.0254s (16.7868GiB/s) | 20260505 22:32:08.923 | INFO | Analyzing chunk frames = [6817792,6818304) | 20260505 22:32:08.923 | INFO | MemInfo: { | 20260505 22:32:08.923 | INFO | "gpu" : " 17 GB", | 20260505 22:32:08.923 | INFO | "huge" : "", | 20260505 22:32:08.923 | INFO | "huge_shared" : " 27 GB", | 20260505 22:32:08.923 | INFO | "malloc" : " 14 GB", | 20260505 22:32:08.923 | INFO | "pinned" : " 29 GB" | 20260505 22:32:08.923 | INFO | } | 20260505 22:32:08.923 | INFO | Perf report for the last 5.13426 seconds: | 20260505 22:32:08.923 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:32:08.923 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:08.923 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:08.923 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:08.923 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260505 22:32:08.923 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:08.923 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:08.923 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 22:32:08.923 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:08.923 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:08.923 | INFO | Uploaded 36GiB to GPU in 2.09963s (17.1459GiB/s) | 20260505 22:32:08.923 | INFO | Downloaded 34GiB to GPU in 2.04884s (16.5947GiB/s) | 20260505 22:32:14.068 | INFO | Analyzing chunk frames = [6818304,6818816) | 20260505 22:32:14.068 | INFO | MemInfo: { | 20260505 22:32:14.068 | INFO | "gpu" : " 17 GB", | 20260505 22:32:14.068 | INFO | "huge" : "", | 20260505 22:32:14.068 | INFO | "huge_shared" : " 27 GB", | 20260505 22:32:14.068 | INFO | "malloc" : " 14 GB", | 20260505 22:32:14.068 | INFO | "pinned" : " 29 GB" | 20260505 22:32:14.068 | INFO | } | 20260505 22:32:14.068 | INFO | Perf report for the last 5.1449 seconds: | 20260505 22:32:14.068 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:32:14.068 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:14.068 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:14.068 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:14.068 | INFO | ANALYSIS : 61.6%, 3.0, 96.3ms, 0 | 20260505 22:32:14.068 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:14.068 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:14.068 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 22:32:14.068 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:14.068 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:14.068 | INFO | Uploaded 36GiB to GPU in 2.08316s (17.2815GiB/s) | 20260505 22:32:14.068 | INFO | Downloaded 34GiB to GPU in 2.03157s (16.7358GiB/s) | 20260505 22:32:19.205 | INFO | Analyzing chunk frames = [6818816,6819328) | 20260505 22:32:19.205 | INFO | MemInfo: { | 20260505 22:32:19.205 | INFO | "gpu" : " 17 GB", | 20260505 22:32:19.205 | INFO | "huge" : "", | 20260505 22:32:19.206 | INFO | "huge_shared" : " 27 GB", | 20260505 22:32:19.206 | INFO | "malloc" : " 14 GB", | 20260505 22:32:19.206 | INFO | "pinned" : " 29 GB" | 20260505 22:32:19.206 | INFO | } | 20260505 22:32:19.206 | INFO | Perf report for the last 5.13778 seconds: | 20260505 22:32:19.206 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:32:19.206 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:19.206 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:19.206 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:19.206 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 22:32:19.206 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:19.206 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:19.206 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 22:32:19.206 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:19.206 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:19.206 | INFO | Uploaded 36GiB to GPU in 2.14371s (16.7933GiB/s) | 20260505 22:32:19.206 | INFO | Downloaded 34GiB to GPU in 2.1285s (15.9737GiB/s) | 20260505 22:32:24.349 | INFO | Analyzing chunk frames = [6819328,6819840) | 20260505 22:32:24.349 | INFO | MemInfo: { | 20260505 22:32:24.349 | INFO | "gpu" : " 17 GB", | 20260505 22:32:24.349 | INFO | "huge" : "", | 20260505 22:32:24.349 | INFO | "huge_shared" : " 27 GB", | 20260505 22:32:24.349 | INFO | "malloc" : " 14 GB", | 20260505 22:32:24.349 | INFO | "pinned" : " 29 GB" | 20260505 22:32:24.349 | INFO | } | 20260505 22:32:24.349 | INFO | Perf report for the last 5.14371 seconds: | 20260505 22:32:24.349 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:32:24.349 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:24.349 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:24.349 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:24.349 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260505 22:32:24.349 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:24.349 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:24.349 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:32:24.349 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:24.349 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:24.350 | INFO | Uploaded 36GiB to GPU in 2.11206s (17.045GiB/s) | 20260505 22:32:24.350 | INFO | Downloaded 34GiB to GPU in 1.99516s (17.0412GiB/s) | 20260505 22:32:24.470 | INFO | Starting a round of Baz IO | 20260505 22:32:27.620 | INFO | Populating RTMetrics report | 20260505 22:32:27.621 | INFO | RTMetrics report populated | 20260505 22:32:27.621 | INFO | Opening tmp RTMetrics json file | 20260505 22:32:27.621 | INFO | Serializing RTMetrics to json | 20260505 22:32:27.623 | INFO | Serialization done, swapping tmp file now... | 20260505 22:32:27.623 | INFO | RTMetrics done | 20260505 22:32:27.701 | INFO | Finished a round of Baz IO: Wrote 1.928 GiB in 3.231 seconds | 20260505 22:32:27.701 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.49% / 0.00% / 78.51% | 20260505 22:32:27.701 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.122917% / 0% / 8.74201e-05% | 20260505 22:32:27.701 | INFO | Baz IO overall: | 20260505 22:32:27.701 | INFO | BytesWritten : 3923773696493 | 20260505 22:32:27.701 | INFO | NumEvents : 1850546028337 | 20260505 22:32:29.492 | INFO | Analyzing chunk frames = [6819840,6820352) | 20260505 22:32:29.492 | INFO | MemInfo: { | 20260505 22:32:29.492 | INFO | "gpu" : " 17 GB", | 20260505 22:32:29.492 | INFO | "huge" : "", | 20260505 22:32:29.492 | INFO | "huge_shared" : " 27 GB", | 20260505 22:32:29.492 | INFO | "malloc" : " 14 GB", | 20260505 22:32:29.492 | INFO | "pinned" : " 29 GB" | 20260505 22:32:29.492 | INFO | } | 20260505 22:32:29.492 | INFO | Perf report for the last 5.14238 seconds: | 20260505 22:32:29.492 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:32:29.492 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:29.492 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:29.492 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:29.492 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 22:32:29.492 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:29.492 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260505 22:32:29.492 | INFO | PRE_HQ : 59.8%, 1.8, 54.7ms, 0 | 20260505 22:32:29.492 | INFO | BAZWRITER : 62.0%, 3.6, 114.8ms, 0 | 20260505 22:32:29.492 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 22:32:29.492 | INFO | Uploaded 36GiB to GPU in 2.17063s (16.5851GiB/s) | 20260505 22:32:29.492 | INFO | Downloaded 38GiB to GPU in 2.23797s (16.9797GiB/s) | 20260505 22:32:34.623 | INFO | Analyzing chunk frames = [6820352,6820864) | 20260505 22:32:34.623 | INFO | MemInfo: { | 20260505 22:32:34.623 | INFO | "gpu" : " 17 GB", | 20260505 22:32:34.623 | INFO | "huge" : "", | 20260505 22:32:34.623 | INFO | "huge_shared" : " 27 GB", | 20260505 22:32:34.623 | INFO | "malloc" : " 14 GB", | 20260505 22:32:34.623 | INFO | "pinned" : " 29 GB" | 20260505 22:32:34.623 | INFO | } | 20260505 22:32:34.623 | INFO | Perf report for the last 5.13113 seconds: | 20260505 22:32:34.623 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:32:34.623 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:34.623 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:34.623 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:34.623 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260505 22:32:34.623 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:34.623 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:34.623 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:32:34.623 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:34.623 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:34.623 | INFO | Uploaded 36GiB to GPU in 2.08742s (17.2462GiB/s) | 20260505 22:32:34.623 | INFO | Downloaded 34GiB to GPU in 2.02843s (16.7617GiB/s) | 20260505 22:32:39.757 | INFO | Analyzing chunk frames = [6820864,6821376) | 20260505 22:32:39.757 | INFO | MemInfo: { | 20260505 22:32:39.757 | INFO | "gpu" : " 17 GB", | 20260505 22:32:39.757 | INFO | "huge" : "", | 20260505 22:32:39.757 | INFO | "huge_shared" : " 27 GB", | 20260505 22:32:39.757 | INFO | "malloc" : " 14 GB", | 20260505 22:32:39.757 | INFO | "pinned" : " 29 GB" | 20260505 22:32:39.757 | INFO | } | 20260505 22:32:39.757 | INFO | Perf report for the last 5.1345 seconds: | 20260505 22:32:39.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:32:39.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:39.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:39.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:39.757 | INFO | ANALYSIS : 62.0%, 3.0, 96.3ms, 0 | 20260505 22:32:39.757 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:39.757 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:39.757 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:32:39.757 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:39.757 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:39.758 | INFO | Uploaded 36GiB to GPU in 2.13528s (16.8596GiB/s) | 20260505 22:32:39.758 | INFO | Downloaded 34GiB to GPU in 2.02555s (16.7856GiB/s) | 20260505 22:32:44.901 | INFO | Analyzing chunk frames = [6821376,6821888) | 20260505 22:32:44.901 | INFO | MemInfo: { | 20260505 22:32:44.901 | INFO | "gpu" : " 17 GB", | 20260505 22:32:44.901 | INFO | "huge" : "", | 20260505 22:32:44.901 | INFO | "huge_shared" : " 27 GB", | 20260505 22:32:44.901 | INFO | "malloc" : " 14 GB", | 20260505 22:32:44.901 | INFO | "pinned" : " 29 GB" | 20260505 22:32:44.901 | INFO | } | 20260505 22:32:44.901 | INFO | Perf report for the last 5.14355 seconds: | 20260505 22:32:44.901 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:32:44.901 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:44.901 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:44.901 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:44.901 | INFO | ANALYSIS : 62.0%, 3.0, 96.3ms, 0 | 20260505 22:32:44.901 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:44.901 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:44.901 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:32:44.901 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:44.901 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:44.901 | INFO | Uploaded 36GiB to GPU in 2.142s (16.8067GiB/s) | 20260505 22:32:44.901 | INFO | Downloaded 34GiB to GPU in 2.04949s (16.5895GiB/s) | 20260505 22:32:50.049 | INFO | Analyzing chunk frames = [6821888,6822400) | 20260505 22:32:50.049 | INFO | MemInfo: { | 20260505 22:32:50.049 | INFO | "gpu" : " 17 GB", | 20260505 22:32:50.049 | INFO | "huge" : "", | 20260505 22:32:50.049 | INFO | "huge_shared" : " 27 GB", | 20260505 22:32:50.049 | INFO | "malloc" : " 14 GB", | 20260505 22:32:50.049 | INFO | "pinned" : " 29 GB" | 20260505 22:32:50.049 | INFO | } | 20260505 22:32:50.049 | INFO | Perf report for the last 5.14831 seconds: | 20260505 22:32:50.049 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:32:50.049 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:50.049 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:50.049 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:50.049 | INFO | ANALYSIS : 61.1%, 3.0, 95.6ms, 0 | 20260505 22:32:50.049 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:50.049 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:50.049 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:32:50.049 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:50.049 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:50.049 | INFO | Uploaded 36GiB to GPU in 2.08437s (17.2714GiB/s) | 20260505 22:32:50.049 | INFO | Downloaded 34GiB to GPU in 2.0404s (16.6634GiB/s) | 20260505 22:32:55.181 | INFO | Analyzing chunk frames = [6822400,6822912) | 20260505 22:32:55.181 | INFO | MemInfo: { | 20260505 22:32:55.181 | INFO | "gpu" : " 17 GB", | 20260505 22:32:55.181 | INFO | "huge" : "", | 20260505 22:32:55.181 | INFO | "huge_shared" : " 27 GB", | 20260505 22:32:55.181 | INFO | "malloc" : " 14 GB", | 20260505 22:32:55.181 | INFO | "pinned" : " 29 GB" | 20260505 22:32:55.181 | INFO | } | 20260505 22:32:55.181 | INFO | Perf report for the last 5.1316 seconds: | 20260505 22:32:55.181 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:32:55.181 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:55.181 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:55.181 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:55.181 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260505 22:32:55.181 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:55.181 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:55.181 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:32:55.181 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:55.181 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:32:55.181 | INFO | Uploaded 36GiB to GPU in 2.06643s (17.4213GiB/s) | 20260505 22:32:55.181 | INFO | Downloaded 34GiB to GPU in 2.03498s (16.7078GiB/s) | 20260505 22:33:00.319 | INFO | Analyzing chunk frames = [6822912,6823424) | 20260505 22:33:00.319 | INFO | MemInfo: { | 20260505 22:33:00.319 | INFO | "gpu" : " 17 GB", | 20260505 22:33:00.319 | INFO | "huge" : "", | 20260505 22:33:00.319 | INFO | "huge_shared" : " 27 GB", | 20260505 22:33:00.319 | INFO | "malloc" : " 14 GB", | 20260505 22:33:00.319 | INFO | "pinned" : " 29 GB" | 20260505 22:33:00.319 | INFO | } | 20260505 22:33:00.319 | INFO | Perf report for the last 5.13808 seconds: | 20260505 22:33:00.319 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:33:00.319 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:00.319 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:00.319 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:00.319 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260505 22:33:00.319 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:00.319 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:00.319 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 22:33:00.319 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:00.319 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:00.319 | INFO | Uploaded 36GiB to GPU in 2.0546s (17.5216GiB/s) | 20260505 22:33:00.319 | INFO | Downloaded 34GiB to GPU in 2.04355s (16.6377GiB/s) | 20260505 22:33:05.462 | INFO | Analyzing chunk frames = [6823424,6823936) | 20260505 22:33:05.462 | INFO | MemInfo: { | 20260505 22:33:05.462 | INFO | "gpu" : " 17 GB", | 20260505 22:33:05.462 | INFO | "huge" : "", | 20260505 22:33:05.462 | INFO | "huge_shared" : " 27 GB", | 20260505 22:33:05.463 | INFO | "malloc" : " 14 GB", | 20260505 22:33:05.463 | INFO | "pinned" : " 29 GB" | 20260505 22:33:05.463 | INFO | } | 20260505 22:33:05.463 | INFO | Perf report for the last 5.14365 seconds: | 20260505 22:33:05.463 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:33:05.463 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:05.463 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:05.463 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:05.463 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 22:33:05.463 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:05.463 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:05.463 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:33:05.463 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:05.463 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:05.463 | INFO | Uploaded 36GiB to GPU in 2.11792s (16.9978GiB/s) | 20260505 22:33:05.463 | INFO | Downloaded 34GiB to GPU in 2.01066s (16.9098GiB/s) | 20260505 22:33:05.463 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 22:33:05.463 | INFO | Total mem usage: 15.3 GiB | 20260505 22:33:05.463 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 22:33:05.463 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.52%) | 20260505 22:33:05.463 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 22:33:05.463 | INFO | High Water Breakdown: | 20260505 22:33:05.463 | INFO | 61.1% : 8.62 GiB ( 90.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 22:33:05.463 | INFO | 38.8% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 22:33:05.463 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 22:33:05.463 | INFO | ------------------------------------------ | 20260505 22:33:05.463 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:33:05.463 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 22:33:05.463 | INFO | Total mem usage: 26.5 GiB | 20260505 22:33:05.463 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 22:33:05.463 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 22:33:05.463 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 22:33:05.463 | INFO | High Water Breakdown: | 20260505 22:33:05.463 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:33:05.463 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:33:05.463 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:33:05.463 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:33:05.463 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:33:05.463 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:33:05.463 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 22:33:05.463 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 22:33:05.463 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:33:05.463 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 22:33:05.463 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 22:33:05.463 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 22:33:05.463 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 22:33:05.463 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 22:33:05.463 | INFO | ------------------------------------------ | 20260505 22:33:05.463 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:33:05.463 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 22:33:05.464 | INFO | Total mem usage: 25.6 GiB | 20260505 22:33:05.464 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 22:33:05.464 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 22:33:05.464 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 22:33:05.464 | INFO | High Water Breakdown: | 20260505 22:33:05.464 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 22:33:05.464 | INFO | ------------------------------------------ | 20260505 22:33:05.464 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:33:05.464 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 22:33:05.464 | INFO | Total mem usage: 15.7 GiB | 20260505 22:33:05.464 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 22:33:05.464 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 22:33:05.464 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 22:33:05.464 | INFO | High Water Breakdown: | 20260505 22:33:05.464 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 22:33:05.464 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 22:33:05.464 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 22:33:05.464 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 22:33:05.464 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 22:33:05.464 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 22:33:05.464 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 22:33:05.464 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 22:33:05.464 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 22:33:05.464 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:33:05.464 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 22:33:05.464 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:33:05.464 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 22:33:05.464 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 22:33:05.464 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 22:33:05.464 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 22:33:05.464 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:33:05.464 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 22:33:05.464 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:33:05.464 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:33:05.464 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:33:05.464 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 22:33:05.464 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 22:33:05.464 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:33:05.464 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 22:33:05.464 | INFO | ------------------------------------------ | 20260505 22:33:05.464 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:33:05.464 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 22:33:05.464 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:33:05.464 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:33:05.464 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.42e+01, 1.35e+01 / 1.28e+00 | 20260505 22:33:05.464 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.03e-01, 1.02e-02 / 2.73e-03 | 20260505 22:33:05.464 | INFO | PerfReport: 11.79%, Baseline -- 3.70e+00 / 4.50e+00, 5.68e-01 / 1.62e-01 | 20260505 22:33:05.464 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.51e+00, 1.90e-01 / 9.22e-02 | 20260505 22:33:05.464 | INFO | PerfReport: 16.02%, DME -- 4.03e+01 / 4.43e+01, 3.96e+00 / 1.47e+00 | 20260505 22:33:05.464 | INFO | PerfReport: 57.22%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.50e+00 / 1.10e+00 | 20260505 22:33:05.464 | INFO | PerfReport: 7.14%, PulseAccumulating -- 2.24e+00 / 2.78e+00, 4.29e-01 / 2.13e-01 | 20260505 22:33:05.464 | INFO | PerfReport: 3.15%, Metrics -- 9.89e-01 / 1.26e+00, 2.64e-01 / 1.29e-01 | 20260505 22:33:05.464 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.44e-01 / 3.98e-01, 5.81e-02 / 3.71e-02 | 20260505 22:33:05.464 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 22:33:05.464 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:33:05.464 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:33:05.464 | INFO | PerfReport: Full Execution -- 9.87e+01 / -nan, 2.48e+01 / 0.00e+00 | 20260505 22:33:05.464 | INFO | PerfReport: 22.32%, Upload -- 2.20e+01 / 0.00e+00, 7.52e+00 / 0.00e+00 | 20260505 22:33:05.464 | INFO | PerfReport: 21.77%, Download -- 2.15e+01 / 0.00e+00, 7.56e+00 / 0.00e+00 | 20260505 22:33:05.464 | INFO | PerfReport: 31.84%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 22:33:05.464 | INFO | PerfReport: 2.15%, LockUpload -- 2.12e+00 / 0.00e+00, 7.90e+00 / 0.00e+00 | 20260505 22:33:05.464 | INFO | PerfReport: 1.30%, LockDownload -- 1.29e+00 / 0.00e+00, 2.84e+00 / 0.00e+00 | 20260505 22:33:05.464 | INFO | PerfReport: 20.60%, LockCompute -- 2.03e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 22:33:05.464 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.97e-02 / 0.00e+00 | 20260505 22:33:05.587 | INFO | Starting a round of Baz IO | 20260505 22:33:08.736 | INFO | Populating RTMetrics report | 20260505 22:33:08.737 | INFO | RTMetrics report populated | 20260505 22:33:08.737 | INFO | Opening tmp RTMetrics json file | 20260505 22:33:08.739 | INFO | Serializing RTMetrics to json | 20260505 22:33:08.743 | INFO | Serialization done, swapping tmp file now... | 20260505 22:33:08.744 | INFO | RTMetrics done | 20260505 22:33:08.825 | INFO | Finished a round of Baz IO: Wrote 1.929 GiB in 3.238 seconds | 20260505 22:33:08.825 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.49% / 0.00% / 78.51% | 20260505 22:33:08.825 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.123111% / 0% / 7.94729e-05% | 20260505 22:33:08.825 | INFO | Baz IO overall: | 20260505 22:33:08.825 | INFO | BytesWritten : 3925844918984 | 20260505 22:33:08.825 | INFO | NumEvents : 1851499145940 | 20260505 22:33:10.601 | INFO | Analyzing chunk frames = [6823936,6824448) | 20260505 22:33:10.601 | INFO | MemInfo: { | 20260505 22:33:10.601 | INFO | "gpu" : " 17 GB", | 20260505 22:33:10.601 | INFO | "huge" : "", | 20260505 22:33:10.601 | INFO | "huge_shared" : " 27 GB", | 20260505 22:33:10.601 | INFO | "malloc" : " 14 GB", | 20260505 22:33:10.601 | INFO | "pinned" : " 29 GB" | 20260505 22:33:10.601 | INFO | } | 20260505 22:33:10.601 | INFO | Perf report for the last 5.1386 seconds: | 20260505 22:33:10.601 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:33:10.601 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:10.601 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:10.601 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:10.601 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260505 22:33:10.601 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:10.601 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260505 22:33:10.601 | INFO | PRE_HQ : 60.0%, 1.8, 54.7ms, 0 | 20260505 22:33:10.601 | INFO | BAZWRITER : 62.2%, 3.6, 115.9ms, 0 | 20260505 22:33:10.601 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 22:33:10.601 | INFO | Uploaded 36GiB to GPU in 2.16522s (16.6265GiB/s) | 20260505 22:33:10.601 | INFO | Downloaded 38GiB to GPU in 2.23278s (17.0191GiB/s) | 20260505 22:33:15.737 | INFO | Analyzing chunk frames = [6824448,6824960) | 20260505 22:33:15.737 | INFO | MemInfo: { | 20260505 22:33:15.737 | INFO | "gpu" : " 17 GB", | 20260505 22:33:15.737 | INFO | "huge" : "", | 20260505 22:33:15.737 | INFO | "huge_shared" : " 27 GB", | 20260505 22:33:15.737 | INFO | "malloc" : " 14 GB", | 20260505 22:33:15.737 | INFO | "pinned" : " 29 GB" | 20260505 22:33:15.737 | INFO | } | 20260505 22:33:15.737 | INFO | Perf report for the last 5.13578 seconds: | 20260505 22:33:15.737 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:33:15.737 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:15.737 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:15.737 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:15.737 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 22:33:15.737 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:15.737 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:15.737 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260505 22:33:15.737 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:15.737 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:15.737 | INFO | Uploaded 36GiB to GPU in 2.08464s (17.2692GiB/s) | 20260505 22:33:15.737 | INFO | Downloaded 34GiB to GPU in 2.03557s (16.703GiB/s) | 20260505 22:33:20.875 | INFO | Analyzing chunk frames = [6824960,6825472) | 20260505 22:33:20.875 | INFO | MemInfo: { | 20260505 22:33:20.875 | INFO | "gpu" : " 17 GB", | 20260505 22:33:20.875 | INFO | "huge" : "", | 20260505 22:33:20.875 | INFO | "huge_shared" : " 27 GB", | 20260505 22:33:20.875 | INFO | "malloc" : " 14 GB", | 20260505 22:33:20.875 | INFO | "pinned" : " 29 GB" | 20260505 22:33:20.875 | INFO | } | 20260505 22:33:20.875 | INFO | Perf report for the last 5.13804 seconds: | 20260505 22:33:20.875 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:33:20.875 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:20.875 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:20.875 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:20.875 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260505 22:33:20.875 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:20.875 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:20.875 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:33:20.875 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:20.875 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:20.875 | INFO | Uploaded 36GiB to GPU in 2.15353s (16.7168GiB/s) | 20260505 22:33:20.875 | INFO | Downloaded 34GiB to GPU in 2.04849s (16.5976GiB/s) | 20260505 22:33:26.018 | INFO | Analyzing chunk frames = [6825472,6825984) | 20260505 22:33:26.018 | INFO | MemInfo: { | 20260505 22:33:26.018 | INFO | "gpu" : " 17 GB", | 20260505 22:33:26.018 | INFO | "huge" : "", | 20260505 22:33:26.018 | INFO | "huge_shared" : " 27 GB", | 20260505 22:33:26.018 | INFO | "malloc" : " 14 GB", | 20260505 22:33:26.018 | INFO | "pinned" : " 29 GB" | 20260505 22:33:26.018 | INFO | } | 20260505 22:33:26.018 | INFO | Perf report for the last 5.14268 seconds: | 20260505 22:33:26.018 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:33:26.018 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:26.018 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:26.018 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:26.018 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260505 22:33:26.018 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:26.018 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:26.018 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:33:26.018 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:26.018 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:26.018 | INFO | Uploaded 36GiB to GPU in 2.20951s (16.2932GiB/s) | 20260505 22:33:26.018 | INFO | Downloaded 34GiB to GPU in 2.07536s (16.3827GiB/s) | 20260505 22:33:31.159 | INFO | Analyzing chunk frames = [6825984,6826496) | 20260505 22:33:31.159 | INFO | MemInfo: { | 20260505 22:33:31.159 | INFO | "gpu" : " 17 GB", | 20260505 22:33:31.159 | INFO | "huge" : "", | 20260505 22:33:31.159 | INFO | "huge_shared" : " 27 GB", | 20260505 22:33:31.159 | INFO | "malloc" : " 14 GB", | 20260505 22:33:31.159 | INFO | "pinned" : " 29 GB" | 20260505 22:33:31.159 | INFO | } | 20260505 22:33:31.159 | INFO | Perf report for the last 5.14132 seconds: | 20260505 22:33:31.159 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:33:31.159 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:31.159 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:31.159 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:31.159 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260505 22:33:31.159 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:31.159 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:31.159 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 22:33:31.159 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:31.159 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:31.159 | INFO | Uploaded 36GiB to GPU in 2.15086s (16.7375GiB/s) | 20260505 22:33:31.159 | INFO | Downloaded 34GiB to GPU in 2.06964s (16.428GiB/s) | 20260505 22:33:36.299 | INFO | Analyzing chunk frames = [6826496,6827008) | 20260505 22:33:36.299 | INFO | MemInfo: { | 20260505 22:33:36.299 | INFO | "gpu" : " 17 GB", | 20260505 22:33:36.299 | INFO | "huge" : "", | 20260505 22:33:36.299 | INFO | "huge_shared" : " 27 GB", | 20260505 22:33:36.299 | INFO | "malloc" : " 14 GB", | 20260505 22:33:36.299 | INFO | "pinned" : " 29 GB" | 20260505 22:33:36.299 | INFO | } | 20260505 22:33:36.300 | INFO | Perf report for the last 5.14047 seconds: | 20260505 22:33:36.300 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:33:36.300 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:36.300 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:36.300 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:36.300 | INFO | ANALYSIS : 61.5%, 3.0, 96.0ms, 0 | 20260505 22:33:36.300 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:36.300 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:36.300 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 22:33:36.300 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:36.300 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:36.300 | INFO | Uploaded 36GiB to GPU in 2.0749s (17.3503GiB/s) | 20260505 22:33:36.300 | INFO | Downloaded 34GiB to GPU in 2.03582s (16.7009GiB/s) | 20260505 22:33:41.439 | INFO | Analyzing chunk frames = [6827008,6827520) | 20260505 22:33:41.439 | INFO | MemInfo: { | 20260505 22:33:41.439 | INFO | "gpu" : " 17 GB", | 20260505 22:33:41.439 | INFO | "huge" : "", | 20260505 22:33:41.439 | INFO | "huge_shared" : " 27 GB", | 20260505 22:33:41.439 | INFO | "malloc" : " 14 GB", | 20260505 22:33:41.439 | INFO | "pinned" : " 29 GB" | 20260505 22:33:41.439 | INFO | } | 20260505 22:33:41.439 | INFO | Perf report for the last 5.13991 seconds: | 20260505 22:33:41.439 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:33:41.439 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:41.439 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:41.439 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:41.439 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 22:33:41.439 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:41.440 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:41.440 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 22:33:41.440 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:41.440 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:41.440 | INFO | Uploaded 36GiB to GPU in 2.02283s (17.7968GiB/s) | 20260505 22:33:41.440 | INFO | Downloaded 34GiB to GPU in 2.03732s (16.6886GiB/s) | 20260505 22:33:46.569 | INFO | Analyzing chunk frames = [6827520,6828032) | 20260505 22:33:46.569 | INFO | MemInfo: { | 20260505 22:33:46.569 | INFO | "gpu" : " 17 GB", | 20260505 22:33:46.569 | INFO | "huge" : "", | 20260505 22:33:46.569 | INFO | "huge_shared" : " 27 GB", | 20260505 22:33:46.569 | INFO | "malloc" : " 14 GB", | 20260505 22:33:46.569 | INFO | "pinned" : " 29 GB" | 20260505 22:33:46.569 | INFO | } | 20260505 22:33:46.569 | INFO | Perf report for the last 5.12997 seconds: | 20260505 22:33:46.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:33:46.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:46.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:46.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:46.569 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 22:33:46.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:46.569 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:46.570 | INFO | PRE_HQ : 27.5%, 1.0, 14.2ms, 0 | 20260505 22:33:46.570 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:46.570 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:46.570 | INFO | Uploaded 36GiB to GPU in 2.12462s (16.9442GiB/s) | 20260505 22:33:46.570 | INFO | Downloaded 34GiB to GPU in 2.01349s (16.8861GiB/s) | 20260505 22:33:46.706 | INFO | Starting a round of Baz IO | 20260505 22:33:49.855 | INFO | Populating RTMetrics report | 20260505 22:33:49.855 | INFO | RTMetrics report populated | 20260505 22:33:49.855 | INFO | Opening tmp RTMetrics json file | 20260505 22:33:49.857 | INFO | Serializing RTMetrics to json | 20260505 22:33:49.859 | INFO | Serialization done, swapping tmp file now... | 20260505 22:33:49.859 | INFO | RTMetrics done | 20260505 22:33:49.934 | INFO | Finished a round of Baz IO: Wrote 1.927 GiB in 3.228 seconds | 20260505 22:33:49.934 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.49% / 0.00% / 78.51% | 20260505 22:33:49.934 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.122035% / 0% / 7.94729e-05% | 20260505 22:33:49.934 | INFO | Baz IO overall: | 20260505 22:33:49.934 | INFO | BytesWritten : 3927913590448 | 20260505 22:33:49.934 | INFO | NumEvents : 1852450686007 | 20260505 22:33:51.712 | INFO | Analyzing chunk frames = [6828032,6828544) | 20260505 22:33:51.712 | INFO | MemInfo: { | 20260505 22:33:51.712 | INFO | "gpu" : " 17 GB", | 20260505 22:33:51.712 | INFO | "huge" : "", | 20260505 22:33:51.712 | INFO | "huge_shared" : " 27 GB", | 20260505 22:33:51.712 | INFO | "malloc" : " 14 GB", | 20260505 22:33:51.712 | INFO | "pinned" : " 29 GB" | 20260505 22:33:51.712 | INFO | } | 20260505 22:33:51.712 | INFO | Perf report for the last 5.14279 seconds: | 20260505 22:33:51.712 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:33:51.712 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:51.712 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:51.712 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:51.712 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 22:33:51.712 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:51.712 | INFO | RT_METRICS : 43.5%, 1.0, 22.9ms, 0 | 20260505 22:33:51.712 | INFO | PRE_HQ : 59.8%, 1.8, 54.5ms, 0 | 20260505 22:33:51.712 | INFO | BAZWRITER : 62.3%, 3.6, 114.6ms, 0 | 20260505 22:33:51.712 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 22:33:51.712 | INFO | Uploaded 36GiB to GPU in 2.16658s (16.616GiB/s) | 20260505 22:33:51.712 | INFO | Downloaded 38GiB to GPU in 2.24745s (16.908GiB/s) | 20260505 22:33:56.852 | INFO | Analyzing chunk frames = [6828544,6829056) | 20260505 22:33:56.852 | INFO | MemInfo: { | 20260505 22:33:56.852 | INFO | "gpu" : " 17 GB", | 20260505 22:33:56.852 | INFO | "huge" : "", | 20260505 22:33:56.852 | INFO | "huge_shared" : " 27 GB", | 20260505 22:33:56.852 | INFO | "malloc" : " 14 GB", | 20260505 22:33:56.852 | INFO | "pinned" : " 29 GB" | 20260505 22:33:56.852 | INFO | } | 20260505 22:33:56.852 | INFO | Perf report for the last 5.14009 seconds: | 20260505 22:33:56.852 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:33:56.852 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:56.852 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:56.852 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:56.852 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260505 22:33:56.852 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:56.852 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:56.852 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:33:56.852 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:56.852 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:33:56.853 | INFO | Uploaded 36GiB to GPU in 2.08693s (17.2502GiB/s) | 20260505 22:33:56.853 | INFO | Downloaded 34GiB to GPU in 2.02387s (16.7995GiB/s) | 20260505 22:34:01.994 | INFO | Analyzing chunk frames = [6829056,6829568) | 20260505 22:34:01.994 | INFO | MemInfo: { | 20260505 22:34:01.994 | INFO | "gpu" : " 17 GB", | 20260505 22:34:01.994 | INFO | "huge" : "", | 20260505 22:34:01.994 | INFO | "huge_shared" : " 27 GB", | 20260505 22:34:01.994 | INFO | "malloc" : " 14 GB", | 20260505 22:34:01.994 | INFO | "pinned" : " 29 GB" | 20260505 22:34:01.994 | INFO | } | 20260505 22:34:01.994 | INFO | Perf report for the last 5.14159 seconds: | 20260505 22:34:01.994 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:34:01.994 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:01.994 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:01.994 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:01.994 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260505 22:34:01.994 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:01.994 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:01.994 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:34:01.994 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:01.994 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:01.994 | INFO | Uploaded 36GiB to GPU in 2.14313s (16.7979GiB/s) | 20260505 22:34:01.994 | INFO | Downloaded 34GiB to GPU in 2.0331s (16.7232GiB/s) | 20260505 22:34:07.134 | INFO | Analyzing chunk frames = [6829568,6830080) | 20260505 22:34:07.134 | INFO | MemInfo: { | 20260505 22:34:07.134 | INFO | "gpu" : " 17 GB", | 20260505 22:34:07.134 | INFO | "huge" : "", | 20260505 22:34:07.134 | INFO | "huge_shared" : " 27 GB", | 20260505 22:34:07.134 | INFO | "malloc" : " 14 GB", | 20260505 22:34:07.134 | INFO | "pinned" : " 29 GB" | 20260505 22:34:07.134 | INFO | } | 20260505 22:34:07.134 | INFO | Perf report for the last 5.14053 seconds: | 20260505 22:34:07.134 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:34:07.134 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:07.134 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:07.134 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:07.134 | INFO | ANALYSIS : 61.7%, 3.0, 95.8ms, 0 | 20260505 22:34:07.135 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:07.135 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:07.135 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 22:34:07.135 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:07.135 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:07.135 | INFO | Uploaded 36GiB to GPU in 2.12343s (16.9537GiB/s) | 20260505 22:34:07.135 | INFO | Downloaded 34GiB to GPU in 2.01321s (16.8885GiB/s) | 20260505 22:34:12.269 | INFO | Analyzing chunk frames = [6830080,6830592) | 20260505 22:34:12.269 | INFO | MemInfo: { | 20260505 22:34:12.269 | INFO | "gpu" : " 17 GB", | 20260505 22:34:12.269 | INFO | "huge" : "", | 20260505 22:34:12.269 | INFO | "huge_shared" : " 27 GB", | 20260505 22:34:12.269 | INFO | "malloc" : " 14 GB", | 20260505 22:34:12.269 | INFO | "pinned" : " 29 GB" | 20260505 22:34:12.269 | INFO | } | 20260505 22:34:12.269 | INFO | Perf report for the last 5.13501 seconds: | 20260505 22:34:12.269 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:34:12.269 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:12.270 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:12.270 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:12.270 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260505 22:34:12.270 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:12.270 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:12.270 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 22:34:12.270 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:12.270 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:12.270 | INFO | Uploaded 36GiB to GPU in 2.13632s (16.8514GiB/s) | 20260505 22:34:12.270 | INFO | Downloaded 34GiB to GPU in 2.06475s (16.4669GiB/s) | 20260505 22:34:17.412 | INFO | Analyzing chunk frames = [6830592,6831104) | 20260505 22:34:17.412 | INFO | MemInfo: { | 20260505 22:34:17.412 | INFO | "gpu" : " 17 GB", | 20260505 22:34:17.412 | INFO | "huge" : "", | 20260505 22:34:17.412 | INFO | "huge_shared" : " 27 GB", | 20260505 22:34:17.412 | INFO | "malloc" : " 14 GB", | 20260505 22:34:17.412 | INFO | "pinned" : " 29 GB" | 20260505 22:34:17.412 | INFO | } | 20260505 22:34:17.412 | INFO | Perf report for the last 5.14262 seconds: | 20260505 22:34:17.412 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:34:17.412 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:17.412 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:17.412 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:17.412 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260505 22:34:17.412 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:17.412 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:17.412 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:34:17.412 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:17.412 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:17.412 | INFO | Uploaded 36GiB to GPU in 2.09418s (17.1905GiB/s) | 20260505 22:34:17.412 | INFO | Downloaded 34GiB to GPU in 2.0318s (16.7339GiB/s) | 20260505 22:34:22.550 | INFO | Analyzing chunk frames = [6831104,6831616) | 20260505 22:34:22.550 | INFO | MemInfo: { | 20260505 22:34:22.550 | INFO | "gpu" : " 17 GB", | 20260505 22:34:22.550 | INFO | "huge" : "", | 20260505 22:34:22.550 | INFO | "huge_shared" : " 27 GB", | 20260505 22:34:22.550 | INFO | "malloc" : " 14 GB", | 20260505 22:34:22.550 | INFO | "pinned" : " 29 GB" | 20260505 22:34:22.550 | INFO | } | 20260505 22:34:22.550 | INFO | Perf report for the last 5.13823 seconds: | 20260505 22:34:22.550 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:34:22.550 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:22.550 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:22.550 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:22.550 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260505 22:34:22.550 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:22.550 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:22.550 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 22:34:22.550 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:22.550 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:22.551 | INFO | Uploaded 36GiB to GPU in 2.10413s (17.1092GiB/s) | 20260505 22:34:22.551 | INFO | Downloaded 34GiB to GPU in 2.07848s (16.3581GiB/s) | 20260505 22:34:27.691 | INFO | Analyzing chunk frames = [6831616,6832128) | 20260505 22:34:27.691 | INFO | MemInfo: { | 20260505 22:34:27.691 | INFO | "gpu" : " 17 GB", | 20260505 22:34:27.691 | INFO | "huge" : "", | 20260505 22:34:27.691 | INFO | "huge_shared" : " 27 GB", | 20260505 22:34:27.691 | INFO | "malloc" : " 14 GB", | 20260505 22:34:27.691 | INFO | "pinned" : " 29 GB" | 20260505 22:34:27.691 | INFO | } | 20260505 22:34:27.691 | INFO | Perf report for the last 5.14062 seconds: | 20260505 22:34:27.691 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:34:27.691 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:27.691 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:27.691 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:27.691 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260505 22:34:27.691 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:27.691 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:27.691 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:34:27.691 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:27.691 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:27.691 | INFO | Uploaded 36GiB to GPU in 2.11646s (17.0095GiB/s) | 20260505 22:34:27.691 | INFO | Downloaded 34GiB to GPU in 1.99858s (17.0121GiB/s) | 20260505 22:34:27.691 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 22:34:27.691 | INFO | Total mem usage: 15.3 GiB | 20260505 22:34:27.691 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 22:34:27.691 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -9.02%) | 20260505 22:34:27.691 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 22:34:27.691 | INFO | High Water Breakdown: | 20260505 22:34:27.691 | INFO | 60.9% : 8.56 GiB ( 90% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 22:34:27.691 | INFO | 39% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 22:34:27.691 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 22:34:27.691 | INFO | ------------------------------------------ | 20260505 22:34:27.691 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:34:27.691 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 22:34:27.691 | INFO | Total mem usage: 26.5 GiB | 20260505 22:34:27.691 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 22:34:27.691 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 22:34:27.691 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 22:34:27.691 | INFO | High Water Breakdown: | 20260505 22:34:27.691 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:34:27.691 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:34:27.691 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:34:27.691 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:34:27.692 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:34:27.692 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:34:27.692 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 22:34:27.692 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 22:34:27.692 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:34:27.692 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 22:34:27.692 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 22:34:27.692 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 22:34:27.692 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 22:34:27.692 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 22:34:27.692 | INFO | ------------------------------------------ | 20260505 22:34:27.692 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:34:27.692 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 22:34:27.692 | INFO | Total mem usage: 25.6 GiB | 20260505 22:34:27.692 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 22:34:27.692 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 22:34:27.692 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 22:34:27.692 | INFO | High Water Breakdown: | 20260505 22:34:27.692 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 22:34:27.692 | INFO | ------------------------------------------ | 20260505 22:34:27.692 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:34:27.692 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 22:34:27.692 | INFO | Total mem usage: 15.7 GiB | 20260505 22:34:27.692 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 22:34:27.692 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 22:34:27.692 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 22:34:27.692 | INFO | High Water Breakdown: | 20260505 22:34:27.692 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 22:34:27.692 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 22:34:27.692 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 22:34:27.692 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 22:34:27.692 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 22:34:27.692 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 22:34:27.692 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 22:34:27.692 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 22:34:27.692 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 22:34:27.692 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:34:27.692 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 22:34:27.692 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:34:27.692 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 22:34:27.692 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 22:34:27.692 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 22:34:27.692 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 22:34:27.692 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:34:27.692 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 22:34:27.692 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:34:27.692 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:34:27.692 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:34:27.692 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 22:34:27.692 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 22:34:27.692 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:34:27.692 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 22:34:27.692 | INFO | ------------------------------------------ | 20260505 22:34:27.692 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:34:27.692 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 22:34:27.692 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:34:27.692 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:34:27.692 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.39e+01, 1.32e+01 / 9.43e-01 | 20260505 22:34:27.692 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.06e-01, 1.21e-02 / 2.00e-02 | 20260505 22:34:27.692 | INFO | PerfReport: 11.80%, Baseline -- 3.70e+00 / 4.52e+00, 5.64e-01 / 1.80e-01 | 20260505 22:34:27.692 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.54e+00, 1.90e-01 / 8.81e-02 | 20260505 22:34:27.692 | INFO | PerfReport: 15.98%, DME -- 4.01e+01 / 4.40e+01, 3.85e+00 / 1.11e+00 | 20260505 22:34:27.692 | INFO | PerfReport: 57.20%, FrameLabeling -- 1.79e+01 / 1.93e+01, 1.44e+00 / 8.64e-01 | 20260505 22:34:27.692 | INFO | PerfReport: 7.19%, PulseAccumulating -- 2.25e+00 / 2.80e+00, 4.35e-01 / 2.19e-01 | 20260505 22:34:27.692 | INFO | PerfReport: 3.17%, Metrics -- 9.92e-01 / 1.29e+00, 2.68e-01 / 1.49e-01 | 20260505 22:34:27.692 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.44e-01 / 3.99e-01, 5.79e-02 / 3.06e-02 | 20260505 22:34:27.692 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 22:34:27.692 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:34:27.692 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:34:27.692 | INFO | PerfReport: Full Execution -- 9.87e+01 / -nan, 2.49e+01 / 0.00e+00 | 20260505 22:34:27.692 | INFO | PerfReport: 22.39%, Upload -- 2.21e+01 / 0.00e+00, 7.64e+00 / 0.00e+00 | 20260505 22:34:27.692 | INFO | PerfReport: 21.79%, Download -- 2.15e+01 / 0.00e+00, 7.66e+00 / 0.00e+00 | 20260505 22:34:27.692 | INFO | PerfReport: 31.74%, Compute -- 3.13e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260505 22:34:27.692 | INFO | PerfReport: 2.21%, LockUpload -- 2.18e+00 / 0.00e+00, 7.91e+00 / 0.00e+00 | 20260505 22:34:27.692 | INFO | PerfReport: 1.32%, LockDownload -- 1.30e+00 / 0.00e+00, 2.87e+00 / 0.00e+00 | 20260505 22:34:27.692 | INFO | PerfReport: 20.53%, LockCompute -- 2.03e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260505 22:34:27.692 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.98e-02 / 0.00e+00 | 20260505 22:34:27.819 | INFO | Starting a round of Baz IO | 20260505 22:34:30.968 | INFO | Populating RTMetrics report | 20260505 22:34:30.968 | INFO | RTMetrics report populated | 20260505 22:34:30.968 | INFO | Opening tmp RTMetrics json file | 20260505 22:34:30.969 | INFO | Serializing RTMetrics to json | 20260505 22:34:30.970 | INFO | Serialization done, swapping tmp file now... | 20260505 22:34:30.971 | INFO | RTMetrics done | 20260505 22:34:31.046 | INFO | Finished a round of Baz IO: Wrote 1.926 GiB in 3.227 seconds | 20260505 22:34:31.046 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.49% / 0.00% / 78.51% | 20260505 22:34:31.046 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.122329% / 3.97364e-06% / 9.93411e-05% | 20260505 22:34:31.046 | INFO | Baz IO overall: | 20260505 22:34:31.046 | INFO | BytesWritten : 3929981438448 | 20260505 22:34:31.046 | INFO | NumEvents : 1853401882179 | 20260505 22:34:32.822 | INFO | Analyzing chunk frames = [6832128,6832640) | 20260505 22:34:32.822 | INFO | MemInfo: { | 20260505 22:34:32.822 | INFO | "gpu" : " 17 GB", | 20260505 22:34:32.822 | INFO | "huge" : "", | 20260505 22:34:32.822 | INFO | "huge_shared" : " 27 GB", | 20260505 22:34:32.822 | INFO | "malloc" : " 14 GB", | 20260505 22:34:32.822 | INFO | "pinned" : " 29 GB" | 20260505 22:34:32.822 | INFO | } | 20260505 22:34:32.822 | INFO | Perf report for the last 5.13093 seconds: | 20260505 22:34:32.822 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:34:32.822 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:32.822 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:32.822 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:32.822 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260505 22:34:32.822 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:32.822 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260505 22:34:32.822 | INFO | PRE_HQ : 60.0%, 1.8, 54.7ms, 0 | 20260505 22:34:32.822 | INFO | BAZWRITER : 62.0%, 3.6, 115.3ms, 0 | 20260505 22:34:32.822 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 22:34:32.822 | INFO | Uploaded 36GiB to GPU in 2.12434s (16.9464GiB/s) | 20260505 22:34:32.822 | INFO | Downloaded 38GiB to GPU in 2.2316s (17.0281GiB/s) | 20260505 22:34:37.968 | INFO | Analyzing chunk frames = [6832640,6833152) | 20260505 22:34:37.968 | INFO | MemInfo: { | 20260505 22:34:37.968 | INFO | "gpu" : " 17 GB", | 20260505 22:34:37.968 | INFO | "huge" : "", | 20260505 22:34:37.968 | INFO | "huge_shared" : " 27 GB", | 20260505 22:34:37.968 | INFO | "malloc" : " 14 GB", | 20260505 22:34:37.968 | INFO | "pinned" : " 29 GB" | 20260505 22:34:37.968 | INFO | } | 20260505 22:34:37.968 | INFO | Perf report for the last 5.14622 seconds: | 20260505 22:34:37.968 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:34:37.968 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:37.968 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:37.968 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:37.968 | INFO | ANALYSIS : 62.0%, 3.0, 97.0ms, 0 | 20260505 22:34:37.968 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:37.968 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:37.968 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 22:34:37.968 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:37.968 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:37.968 | INFO | Uploaded 36GiB to GPU in 2.10441s (17.1069GiB/s) | 20260505 22:34:37.968 | INFO | Downloaded 34GiB to GPU in 2.02288s (16.8077GiB/s) | 20260505 22:34:43.112 | INFO | Analyzing chunk frames = [6833152,6833664) | 20260505 22:34:43.112 | INFO | MemInfo: { | 20260505 22:34:43.112 | INFO | "gpu" : " 17 GB", | 20260505 22:34:43.112 | INFO | "huge" : "", | 20260505 22:34:43.112 | INFO | "huge_shared" : " 27 GB", | 20260505 22:34:43.112 | INFO | "malloc" : " 14 GB", | 20260505 22:34:43.112 | INFO | "pinned" : " 29 GB" | 20260505 22:34:43.112 | INFO | } | 20260505 22:34:43.112 | INFO | Perf report for the last 5.14396 seconds: | 20260505 22:34:43.112 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:34:43.112 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:43.112 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:43.112 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:43.112 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260505 22:34:43.112 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:43.112 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:43.112 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:34:43.112 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:43.112 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:43.112 | INFO | Uploaded 36GiB to GPU in 2.14302s (16.7987GiB/s) | 20260505 22:34:43.112 | INFO | Downloaded 34GiB to GPU in 2.03204s (16.7319GiB/s) | 20260505 22:34:48.250 | INFO | Analyzing chunk frames = [6833664,6834176) | 20260505 22:34:48.250 | INFO | MemInfo: { | 20260505 22:34:48.250 | INFO | "gpu" : " 17 GB", | 20260505 22:34:48.250 | INFO | "huge" : "", | 20260505 22:34:48.250 | INFO | "huge_shared" : " 27 GB", | 20260505 22:34:48.250 | INFO | "malloc" : " 14 GB", | 20260505 22:34:48.250 | INFO | "pinned" : " 29 GB" | 20260505 22:34:48.250 | INFO | } | 20260505 22:34:48.251 | INFO | Perf report for the last 5.13835 seconds: | 20260505 22:34:48.251 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:34:48.251 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:48.251 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:48.251 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:48.251 | INFO | ANALYSIS : 61.8%, 3.0, 96.0ms, 0 | 20260505 22:34:48.251 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:48.251 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:48.251 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:34:48.251 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:48.251 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:48.251 | INFO | Uploaded 36GiB to GPU in 2.14956s (16.7476GiB/s) | 20260505 22:34:48.251 | INFO | Downloaded 34GiB to GPU in 2.02494s (16.7906GiB/s) | 20260505 22:34:53.386 | INFO | Analyzing chunk frames = [6834176,6834688) | 20260505 22:34:53.386 | INFO | MemInfo: { | 20260505 22:34:53.386 | INFO | "gpu" : " 17 GB", | 20260505 22:34:53.386 | INFO | "huge" : "", | 20260505 22:34:53.386 | INFO | "huge_shared" : " 27 GB", | 20260505 22:34:53.386 | INFO | "malloc" : " 14 GB", | 20260505 22:34:53.386 | INFO | "pinned" : " 29 GB" | 20260505 22:34:53.386 | INFO | } | 20260505 22:34:53.386 | INFO | Perf report for the last 5.13516 seconds: | 20260505 22:34:53.386 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:34:53.386 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:53.386 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:53.386 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:53.386 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260505 22:34:53.386 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:53.386 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:53.386 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 22:34:53.386 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:53.386 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:53.386 | INFO | Uploaded 36GiB to GPU in 2.09843s (17.1557GiB/s) | 20260505 22:34:53.386 | INFO | Downloaded 34GiB to GPU in 2.04521s (16.6242GiB/s) | 20260505 22:34:58.521 | INFO | Analyzing chunk frames = [6834688,6835200) | 20260505 22:34:58.521 | INFO | MemInfo: { | 20260505 22:34:58.521 | INFO | "gpu" : " 17 GB", | 20260505 22:34:58.521 | INFO | "huge" : "", | 20260505 22:34:58.521 | INFO | "huge_shared" : " 27 GB", | 20260505 22:34:58.521 | INFO | "malloc" : " 14 GB", | 20260505 22:34:58.521 | INFO | "pinned" : " 29 GB" | 20260505 22:34:58.521 | INFO | } | 20260505 22:34:58.521 | INFO | Perf report for the last 5.13549 seconds: | 20260505 22:34:58.521 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:34:58.521 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:58.521 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:58.521 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:58.521 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260505 22:34:58.521 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:58.521 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:58.521 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 22:34:58.521 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:58.521 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:34:58.521 | INFO | Uploaded 36GiB to GPU in 2.08735s (17.2468GiB/s) | 20260505 22:34:58.521 | INFO | Downloaded 34GiB to GPU in 2.03516s (16.7063GiB/s) | 20260505 22:35:03.666 | INFO | Analyzing chunk frames = [6835200,6835712) | 20260505 22:35:03.666 | INFO | MemInfo: { | 20260505 22:35:03.666 | INFO | "gpu" : " 17 GB", | 20260505 22:35:03.666 | INFO | "huge" : "", | 20260505 22:35:03.666 | INFO | "huge_shared" : " 27 GB", | 20260505 22:35:03.666 | INFO | "malloc" : " 14 GB", | 20260505 22:35:03.666 | INFO | "pinned" : " 29 GB" | 20260505 22:35:03.666 | INFO | } | 20260505 22:35:03.666 | INFO | Perf report for the last 5.14509 seconds: | 20260505 22:35:03.666 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:35:03.666 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:03.666 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:03.666 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:03.666 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 22:35:03.666 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:03.666 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:03.666 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260505 22:35:03.666 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:03.666 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:03.666 | INFO | Uploaded 36GiB to GPU in 2.15389s (16.7139GiB/s) | 20260505 22:35:03.667 | INFO | Downloaded 34GiB to GPU in 2.12896s (15.9703GiB/s) | 20260505 22:35:08.807 | INFO | Analyzing chunk frames = [6835712,6836224) | 20260505 22:35:08.807 | INFO | MemInfo: { | 20260505 22:35:08.807 | INFO | "gpu" : " 17 GB", | 20260505 22:35:08.807 | INFO | "huge" : "", | 20260505 22:35:08.807 | INFO | "huge_shared" : " 27 GB", | 20260505 22:35:08.807 | INFO | "malloc" : " 14 GB", | 20260505 22:35:08.807 | INFO | "pinned" : " 29 GB" | 20260505 22:35:08.807 | INFO | } | 20260505 22:35:08.807 | INFO | Perf report for the last 5.14047 seconds: | 20260505 22:35:08.807 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:35:08.807 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:08.807 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:08.807 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:08.807 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 22:35:08.807 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:08.807 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:08.807 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:35:08.807 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:08.807 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:08.807 | INFO | Uploaded 36GiB to GPU in 2.12048s (16.9773GiB/s) | 20260505 22:35:08.807 | INFO | Downloaded 34GiB to GPU in 2.01157s (16.9022GiB/s) | 20260505 22:35:08.931 | INFO | Starting a round of Baz IO | 20260505 22:35:12.079 | INFO | Populating RTMetrics report | 20260505 22:35:12.079 | INFO | RTMetrics report populated | 20260505 22:35:12.079 | INFO | Opening tmp RTMetrics json file | 20260505 22:35:12.081 | INFO | Serializing RTMetrics to json | 20260505 22:35:12.082 | INFO | Serialization done, swapping tmp file now... | 20260505 22:35:12.083 | INFO | RTMetrics done | 20260505 22:35:12.158 | INFO | Finished a round of Baz IO: Wrote 1.926 GiB in 3.226 seconds | 20260505 22:35:12.158 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.49% / 0.00% / 78.51% | 20260505 22:35:12.158 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.122277% / 3.97364e-06% / 9.53674e-05% | 20260505 22:35:12.158 | INFO | Baz IO overall: | 20260505 22:35:12.158 | INFO | BytesWritten : 3932049297617 | 20260505 22:35:12.158 | INFO | NumEvents : 1854352957343 | 20260505 22:35:13.944 | INFO | Analyzing chunk frames = [6836224,6836736) | 20260505 22:35:13.944 | INFO | MemInfo: { | 20260505 22:35:13.944 | INFO | "gpu" : " 17 GB", | 20260505 22:35:13.944 | INFO | "huge" : "", | 20260505 22:35:13.944 | INFO | "huge_shared" : " 27 GB", | 20260505 22:35:13.944 | INFO | "malloc" : " 14 GB", | 20260505 22:35:13.944 | INFO | "pinned" : " 29 GB" | 20260505 22:35:13.944 | INFO | } | 20260505 22:35:13.944 | INFO | Perf report for the last 5.13718 seconds: | 20260505 22:35:13.944 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:35:13.944 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:13.944 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:13.944 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:13.944 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 22:35:13.944 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:13.944 | INFO | RT_METRICS : 43.6%, 1.0, 22.8ms, 0 | 20260505 22:35:13.944 | INFO | PRE_HQ : 60.0%, 1.7, 54.4ms, 0 | 20260505 22:35:13.944 | INFO | BAZWRITER : 62.0%, 3.6, 113.2ms, 0 | 20260505 22:35:13.944 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 22:35:13.944 | INFO | Uploaded 36GiB to GPU in 2.16411s (16.635GiB/s) | 20260505 22:35:13.944 | INFO | Downloaded 38GiB to GPU in 2.23655s (16.9904GiB/s) | 20260505 22:35:19.082 | INFO | Analyzing chunk frames = [6836736,6837248) | 20260505 22:35:19.082 | INFO | MemInfo: { | 20260505 22:35:19.082 | INFO | "gpu" : " 17 GB", | 20260505 22:35:19.082 | INFO | "huge" : "", | 20260505 22:35:19.082 | INFO | "huge_shared" : " 27 GB", | 20260505 22:35:19.082 | INFO | "malloc" : " 14 GB", | 20260505 22:35:19.082 | INFO | "pinned" : " 29 GB" | 20260505 22:35:19.082 | INFO | } | 20260505 22:35:19.082 | INFO | Perf report for the last 5.1384 seconds: | 20260505 22:35:19.082 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:35:19.082 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:19.082 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:19.082 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:19.082 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 22:35:19.082 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:19.082 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:19.082 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:35:19.082 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:19.082 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:19.083 | INFO | Uploaded 36GiB to GPU in 2.08905s (17.2327GiB/s) | 20260505 22:35:19.083 | INFO | Downloaded 34GiB to GPU in 2.02889s (16.7579GiB/s) | 20260505 22:35:24.222 | INFO | Analyzing chunk frames = [6837248,6837760) | 20260505 22:35:24.222 | INFO | MemInfo: { | 20260505 22:35:24.222 | INFO | "gpu" : " 17 GB", | 20260505 22:35:24.222 | INFO | "huge" : "", | 20260505 22:35:24.222 | INFO | "huge_shared" : " 27 GB", | 20260505 22:35:24.222 | INFO | "malloc" : " 14 GB", | 20260505 22:35:24.222 | INFO | "pinned" : " 29 GB" | 20260505 22:35:24.222 | INFO | } | 20260505 22:35:24.222 | INFO | Perf report for the last 5.13929 seconds: | 20260505 22:35:24.222 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:35:24.222 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:24.222 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:24.222 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:24.222 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260505 22:35:24.222 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:24.222 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:24.222 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 22:35:24.222 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:24.222 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:24.222 | INFO | Uploaded 36GiB to GPU in 2.13559s (16.8571GiB/s) | 20260505 22:35:24.222 | INFO | Downloaded 34GiB to GPU in 2.02191s (16.8158GiB/s) | 20260505 22:35:29.365 | INFO | Analyzing chunk frames = [6837760,6838272) | 20260505 22:35:29.365 | INFO | MemInfo: { | 20260505 22:35:29.365 | INFO | "gpu" : " 17 GB", | 20260505 22:35:29.365 | INFO | "huge" : "", | 20260505 22:35:29.365 | INFO | "huge_shared" : " 27 GB", | 20260505 22:35:29.365 | INFO | "malloc" : " 14 GB", | 20260505 22:35:29.365 | INFO | "pinned" : " 29 GB" | 20260505 22:35:29.365 | INFO | } | 20260505 22:35:29.365 | INFO | Perf report for the last 5.14347 seconds: | 20260505 22:35:29.365 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:35:29.365 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:29.365 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:29.365 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:29.365 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260505 22:35:29.365 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:29.365 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:29.365 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:35:29.365 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:29.365 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:29.365 | INFO | Uploaded 36GiB to GPU in 2.1358s (16.8555GiB/s) | 20260505 22:35:29.365 | INFO | Downloaded 34GiB to GPU in 2.02842s (16.7618GiB/s) | 20260505 22:35:34.499 | INFO | Analyzing chunk frames = [6838272,6838784) | 20260505 22:35:34.499 | INFO | MemInfo: { | 20260505 22:35:34.499 | INFO | "gpu" : " 17 GB", | 20260505 22:35:34.499 | INFO | "huge" : "", | 20260505 22:35:34.499 | INFO | "huge_shared" : " 27 GB", | 20260505 22:35:34.499 | INFO | "malloc" : " 14 GB", | 20260505 22:35:34.499 | INFO | "pinned" : " 29 GB" | 20260505 22:35:34.499 | INFO | } | 20260505 22:35:34.499 | INFO | Perf report for the last 5.13413 seconds: | 20260505 22:35:34.499 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:35:34.499 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:34.499 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:34.499 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:34.499 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260505 22:35:34.499 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:34.499 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:34.499 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:35:34.499 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:34.499 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:34.500 | INFO | Uploaded 36GiB to GPU in 2.09347s (17.1963GiB/s) | 20260505 22:35:34.500 | INFO | Downloaded 34GiB to GPU in 2.04136s (16.6556GiB/s) | 20260505 22:35:39.639 | INFO | Analyzing chunk frames = [6838784,6839296) | 20260505 22:35:39.639 | INFO | MemInfo: { | 20260505 22:35:39.639 | INFO | "gpu" : " 17 GB", | 20260505 22:35:39.639 | INFO | "huge" : "", | 20260505 22:35:39.639 | INFO | "huge_shared" : " 27 GB", | 20260505 22:35:39.639 | INFO | "malloc" : " 14 GB", | 20260505 22:35:39.639 | INFO | "pinned" : " 29 GB" | 20260505 22:35:39.639 | INFO | } | 20260505 22:35:39.639 | INFO | Perf report for the last 5.1396 seconds: | 20260505 22:35:39.639 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:35:39.639 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:39.639 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:39.639 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:39.639 | INFO | ANALYSIS : 61.6%, 3.0, 96.2ms, 0 | 20260505 22:35:39.639 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:39.639 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:39.639 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 22:35:39.639 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:39.639 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:39.639 | INFO | Uploaded 36GiB to GPU in 2.08387s (17.2756GiB/s) | 20260505 22:35:39.639 | INFO | Downloaded 34GiB to GPU in 2.03852s (16.6788GiB/s) | 20260505 22:35:44.777 | INFO | Analyzing chunk frames = [6839296,6839808) | 20260505 22:35:44.777 | INFO | MemInfo: { | 20260505 22:35:44.777 | INFO | "gpu" : " 17 GB", | 20260505 22:35:44.777 | INFO | "huge" : "", | 20260505 22:35:44.777 | INFO | "huge_shared" : " 27 GB", | 20260505 22:35:44.777 | INFO | "malloc" : " 14 GB", | 20260505 22:35:44.777 | INFO | "pinned" : " 29 GB" | 20260505 22:35:44.777 | INFO | } | 20260505 22:35:44.777 | INFO | Perf report for the last 5.13845 seconds: | 20260505 22:35:44.777 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:35:44.777 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:44.777 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:44.777 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:44.777 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 22:35:44.777 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:44.777 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:44.777 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 22:35:44.777 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:44.777 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:44.778 | INFO | Uploaded 36GiB to GPU in 2.15771s (16.6843GiB/s) | 20260505 22:35:44.778 | INFO | Downloaded 34GiB to GPU in 2.13067s (15.9574GiB/s) | 20260505 22:35:49.920 | INFO | Analyzing chunk frames = [6839808,6840320) | 20260505 22:35:49.921 | INFO | MemInfo: { | 20260505 22:35:49.921 | INFO | "gpu" : " 17 GB", | 20260505 22:35:49.921 | INFO | "huge" : "", | 20260505 22:35:49.921 | INFO | "huge_shared" : " 27 GB", | 20260505 22:35:49.921 | INFO | "malloc" : " 14 GB", | 20260505 22:35:49.921 | INFO | "pinned" : " 29 GB" | 20260505 22:35:49.921 | INFO | } | 20260505 22:35:49.921 | INFO | Perf report for the last 5.14332 seconds: | 20260505 22:35:49.921 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:35:49.921 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:49.921 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:49.921 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:49.921 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 22:35:49.921 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:49.921 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:49.921 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 22:35:49.921 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:49.921 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:49.921 | INFO | Uploaded 36GiB to GPU in 2.11838s (16.9941GiB/s) | 20260505 22:35:49.921 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 22:35:49.921 | INFO | Total mem usage: 15.3 GiB | 20260505 22:35:49.921 | INFO | Downloaded 34GiB to GPU in 1.99896s (17.0088GiB/s) | 20260505 22:35:49.921 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 22:35:49.921 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.51%) | 20260505 22:35:49.921 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 22:35:49.921 | INFO | High Water Breakdown: | 20260505 22:35:49.921 | INFO | 61.1% : 8.63 GiB ( 90.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 22:35:49.921 | INFO | 38.8% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 22:35:49.921 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 22:35:49.921 | INFO | ------------------------------------------ | 20260505 22:35:49.921 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:35:49.921 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 22:35:49.921 | INFO | Total mem usage: 26.5 GiB | 20260505 22:35:49.921 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 22:35:49.921 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 22:35:49.921 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 22:35:49.921 | INFO | High Water Breakdown: | 20260505 22:35:49.921 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:35:49.921 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:35:49.921 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:35:49.921 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:35:49.921 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:35:49.921 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:35:49.921 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 22:35:49.921 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 22:35:49.921 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:35:49.921 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 22:35:49.921 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 22:35:49.921 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 22:35:49.921 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 22:35:49.921 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 22:35:49.921 | INFO | ------------------------------------------ | 20260505 22:35:49.921 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:35:49.921 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 22:35:49.921 | INFO | Total mem usage: 25.6 GiB | 20260505 22:35:49.921 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 22:35:49.921 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 22:35:49.921 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 22:35:49.921 | INFO | High Water Breakdown: | 20260505 22:35:49.921 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 22:35:49.922 | INFO | ------------------------------------------ | 20260505 22:35:49.922 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:35:49.922 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 22:35:49.922 | INFO | Total mem usage: 15.7 GiB | 20260505 22:35:49.922 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 22:35:49.922 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 22:35:49.922 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 22:35:49.922 | INFO | High Water Breakdown: | 20260505 22:35:49.922 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 22:35:49.922 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 22:35:49.922 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 22:35:49.922 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 22:35:49.922 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 22:35:49.922 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 22:35:49.922 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 22:35:49.922 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 22:35:49.922 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 22:35:49.922 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:35:49.922 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 22:35:49.922 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:35:49.922 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 22:35:49.922 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 22:35:49.922 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 22:35:49.922 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 22:35:49.922 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:35:49.922 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 22:35:49.922 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:35:49.922 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:35:49.922 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:35:49.922 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 22:35:49.922 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 22:35:49.922 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:35:49.922 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 22:35:49.922 | INFO | ------------------------------------------ | 20260505 22:35:49.922 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:35:49.922 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 22:35:49.922 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:35:49.922 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:35:49.922 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.42e+01, 1.36e+01 / 1.46e+00 | 20260505 22:35:49.922 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.03e-01, 1.05e-02 / 4.34e-03 | 20260505 22:35:49.922 | INFO | PerfReport: 11.81%, Baseline -- 3.71e+00 / 4.56e+00, 5.69e-01 / 1.84e-01 | 20260505 22:35:49.922 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.52e+00, 1.91e-01 / 1.12e-01 | 20260505 22:35:49.922 | INFO | PerfReport: 15.99%, DME -- 4.02e+01 / 4.41e+01, 3.89e+00 / 1.59e+00 | 20260505 22:35:49.922 | INFO | PerfReport: 57.23%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.49e+00 / 1.16e+00 | 20260505 22:35:49.922 | INFO | PerfReport: 7.14%, PulseAccumulating -- 2.24e+00 / 2.78e+00, 4.29e-01 / 2.23e-01 | 20260505 22:35:49.922 | INFO | PerfReport: 3.15%, Metrics -- 9.90e-01 / 1.27e+00, 2.68e-01 / 1.30e-01 | 20260505 22:35:49.922 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.44e-01 / 3.99e-01, 5.81e-02 / 3.74e-02 | 20260505 22:35:49.922 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 22:35:49.922 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:35:49.922 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:35:49.922 | INFO | PerfReport: Full Execution -- 9.87e+01 / -nan, 2.48e+01 / 0.00e+00 | 20260505 22:35:49.922 | INFO | PerfReport: 22.39%, Upload -- 2.21e+01 / 0.00e+00, 7.59e+00 / 0.00e+00 | 20260505 22:35:49.922 | INFO | PerfReport: 21.80%, Download -- 2.15e+01 / 0.00e+00, 7.55e+00 / 0.00e+00 | 20260505 22:35:49.922 | INFO | PerfReport: 31.80%, Compute -- 3.14e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 22:35:49.922 | INFO | PerfReport: 2.21%, LockUpload -- 2.18e+00 / 0.00e+00, 7.94e+00 / 0.00e+00 | 20260505 22:35:49.922 | INFO | PerfReport: 1.31%, LockDownload -- 1.30e+00 / 0.00e+00, 2.87e+00 / 0.00e+00 | 20260505 22:35:49.922 | INFO | PerfReport: 20.48%, LockCompute -- 2.02e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 22:35:49.922 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.66e-02 / 0.00e+00 | 20260505 22:35:50.041 | INFO | Starting a round of Baz IO | 20260505 22:35:53.194 | INFO | Populating RTMetrics report | 20260505 22:35:53.194 | INFO | RTMetrics report populated | 20260505 22:35:53.194 | INFO | Opening tmp RTMetrics json file | 20260505 22:35:53.195 | INFO | Serializing RTMetrics to json | 20260505 22:35:53.196 | INFO | Serialization done, swapping tmp file now... | 20260505 22:35:53.198 | INFO | RTMetrics done | 20260505 22:35:53.278 | INFO | Finished a round of Baz IO: Wrote 1.924 GiB in 3.236 seconds | 20260505 22:35:53.278 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.49% / 0.00% / 78.51% | 20260505 22:35:53.278 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.122126% / 3.97364e-06% / 9.13938e-05% | 20260505 22:35:53.278 | INFO | Baz IO overall: | 20260505 22:35:53.278 | INFO | BytesWritten : 3934115672223 | 20260505 22:35:53.278 | INFO | NumEvents : 1855302828606 | 20260505 22:35:55.066 | INFO | Analyzing chunk frames = [6840320,6840832) | 20260505 22:35:55.066 | INFO | MemInfo: { | 20260505 22:35:55.066 | INFO | "gpu" : " 17 GB", | 20260505 22:35:55.066 | INFO | "huge" : "", | 20260505 22:35:55.066 | INFO | "huge_shared" : " 27 GB", | 20260505 22:35:55.066 | INFO | "malloc" : " 14 GB", | 20260505 22:35:55.066 | INFO | "pinned" : " 29 GB" | 20260505 22:35:55.066 | INFO | } | 20260505 22:35:55.066 | INFO | Perf report for the last 5.14507 seconds: | 20260505 22:35:55.066 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:35:55.066 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:55.066 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:55.066 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:55.066 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260505 22:35:55.066 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:35:55.066 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260505 22:35:55.066 | INFO | PRE_HQ : 59.9%, 1.8, 54.5ms, 0 | 20260505 22:35:55.066 | INFO | BAZWRITER : 62.0%, 3.6, 115.0ms, 0 | 20260505 22:35:55.066 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 22:35:55.066 | INFO | Uploaded 36GiB to GPU in 2.16572s (16.6227GiB/s) | 20260505 22:35:55.066 | INFO | Downloaded 38GiB to GPU in 2.2391s (16.9711GiB/s) | 20260505 22:36:00.196 | INFO | Analyzing chunk frames = [6840832,6841344) | 20260505 22:36:00.196 | INFO | MemInfo: { | 20260505 22:36:00.196 | INFO | "gpu" : " 17 GB", | 20260505 22:36:00.196 | INFO | "huge" : "", | 20260505 22:36:00.196 | INFO | "huge_shared" : " 27 GB", | 20260505 22:36:00.196 | INFO | "malloc" : " 14 GB", | 20260505 22:36:00.196 | INFO | "pinned" : " 29 GB" | 20260505 22:36:00.196 | INFO | } | 20260505 22:36:00.196 | INFO | Perf report for the last 5.13006 seconds: | 20260505 22:36:00.196 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:36:00.196 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:00.196 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:00.196 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:00.196 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260505 22:36:00.196 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:00.196 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:00.196 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260505 22:36:00.196 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:00.196 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:00.196 | INFO | Uploaded 36GiB to GPU in 2.08837s (17.2383GiB/s) | 20260505 22:36:00.196 | INFO | Downloaded 34GiB to GPU in 2.0307s (16.743GiB/s) | 20260505 22:36:05.335 | INFO | Analyzing chunk frames = [6841344,6841856) | 20260505 22:36:05.335 | INFO | MemInfo: { | 20260505 22:36:05.335 | INFO | "gpu" : " 17 GB", | 20260505 22:36:05.335 | INFO | "huge" : "", | 20260505 22:36:05.335 | INFO | "huge_shared" : " 27 GB", | 20260505 22:36:05.335 | INFO | "malloc" : " 14 GB", | 20260505 22:36:05.335 | INFO | "pinned" : " 29 GB" | 20260505 22:36:05.335 | INFO | } | 20260505 22:36:05.335 | INFO | Perf report for the last 5.13913 seconds: | 20260505 22:36:05.335 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:36:05.335 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:05.335 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:05.335 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:05.335 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260505 22:36:05.335 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:05.335 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:05.335 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:36:05.335 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:05.335 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:05.335 | INFO | Uploaded 36GiB to GPU in 2.13951s (16.8263GiB/s) | 20260505 22:36:05.335 | INFO | Downloaded 34GiB to GPU in 2.02401s (16.7983GiB/s) | 20260505 22:36:10.477 | INFO | Analyzing chunk frames = [6841856,6842368) | 20260505 22:36:10.477 | INFO | MemInfo: { | 20260505 22:36:10.477 | INFO | "gpu" : " 17 GB", | 20260505 22:36:10.477 | INFO | "huge" : "", | 20260505 22:36:10.477 | INFO | "huge_shared" : " 27 GB", | 20260505 22:36:10.477 | INFO | "malloc" : " 14 GB", | 20260505 22:36:10.477 | INFO | "pinned" : " 29 GB" | 20260505 22:36:10.477 | INFO | } | 20260505 22:36:10.477 | INFO | Perf report for the last 5.14189 seconds: | 20260505 22:36:10.477 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:36:10.477 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:10.477 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:10.477 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:10.477 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260505 22:36:10.477 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:10.477 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:10.477 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 22:36:10.477 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:10.477 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:10.477 | INFO | Uploaded 36GiB to GPU in 2.20858s (16.3GiB/s) | 20260505 22:36:10.477 | INFO | Downloaded 34GiB to GPU in 2.04636s (16.6149GiB/s) | 20260505 22:36:15.614 | INFO | Analyzing chunk frames = [6842368,6842880) | 20260505 22:36:15.614 | INFO | MemInfo: { | 20260505 22:36:15.614 | INFO | "gpu" : " 17 GB", | 20260505 22:36:15.614 | INFO | "huge" : "", | 20260505 22:36:15.614 | INFO | "huge_shared" : " 27 GB", | 20260505 22:36:15.614 | INFO | "malloc" : " 14 GB", | 20260505 22:36:15.614 | INFO | "pinned" : " 29 GB" | 20260505 22:36:15.614 | INFO | } | 20260505 22:36:15.614 | INFO | Perf report for the last 5.13735 seconds: | 20260505 22:36:15.614 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:36:15.614 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:15.614 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:15.614 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:15.614 | INFO | ANALYSIS : 61.5%, 3.0, 96.0ms, 0 | 20260505 22:36:15.614 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:15.614 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:15.614 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 22:36:15.614 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:15.614 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:15.614 | INFO | Uploaded 36GiB to GPU in 2.13853s (16.834GiB/s) | 20260505 22:36:15.615 | INFO | Downloaded 34GiB to GPU in 2.0636s (16.4761GiB/s) | 20260505 22:36:20.754 | INFO | Analyzing chunk frames = [6842880,6843392) | 20260505 22:36:20.754 | INFO | MemInfo: { | 20260505 22:36:20.754 | INFO | "gpu" : " 17 GB", | 20260505 22:36:20.754 | INFO | "huge" : "", | 20260505 22:36:20.754 | INFO | "huge_shared" : " 27 GB", | 20260505 22:36:20.754 | INFO | "malloc" : " 14 GB", | 20260505 22:36:20.754 | INFO | "pinned" : " 29 GB" | 20260505 22:36:20.755 | INFO | } | 20260505 22:36:20.755 | INFO | Perf report for the last 5.14028 seconds: | 20260505 22:36:20.755 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:36:20.755 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:20.755 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:20.755 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:20.755 | INFO | ANALYSIS : 61.5%, 3.0, 96.0ms, 0 | 20260505 22:36:20.755 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:20.755 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:20.755 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 22:36:20.755 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:20.755 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:20.755 | INFO | Uploaded 36GiB to GPU in 2.07364s (17.3608GiB/s) | 20260505 22:36:20.755 | INFO | Downloaded 34GiB to GPU in 2.03147s (16.7366GiB/s) | 20260505 22:36:25.895 | INFO | Analyzing chunk frames = [6843392,6843904) | 20260505 22:36:25.895 | INFO | MemInfo: { | 20260505 22:36:25.895 | INFO | "gpu" : " 17 GB", | 20260505 22:36:25.895 | INFO | "huge" : "", | 20260505 22:36:25.896 | INFO | "huge_shared" : " 27 GB", | 20260505 22:36:25.896 | INFO | "malloc" : " 14 GB", | 20260505 22:36:25.896 | INFO | "pinned" : " 29 GB" | 20260505 22:36:25.896 | INFO | } | 20260505 22:36:25.896 | INFO | Perf report for the last 5.14111 seconds: | 20260505 22:36:25.896 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:36:25.896 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:25.896 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:25.896 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:25.896 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 22:36:25.896 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:25.896 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:25.896 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 22:36:25.896 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:25.896 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:25.896 | INFO | Uploaded 36GiB to GPU in 2.04211s (17.6289GiB/s) | 20260505 22:36:25.896 | INFO | Downloaded 34GiB to GPU in 2.04557s (16.6213GiB/s) | 20260505 22:36:31.032 | INFO | Analyzing chunk frames = [6843904,6844416) | 20260505 22:36:31.032 | INFO | MemInfo: { | 20260505 22:36:31.032 | INFO | "gpu" : " 17 GB", | 20260505 22:36:31.032 | INFO | "huge" : "", | 20260505 22:36:31.032 | INFO | "huge_shared" : " 27 GB", | 20260505 22:36:31.032 | INFO | "malloc" : " 14 GB", | 20260505 22:36:31.032 | INFO | "pinned" : " 29 GB" | 20260505 22:36:31.032 | INFO | } | 20260505 22:36:31.032 | INFO | Perf report for the last 5.13611 seconds: | 20260505 22:36:31.032 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:36:31.032 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:31.032 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:31.032 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:31.032 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 22:36:31.032 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:31.032 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:31.032 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 22:36:31.032 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:31.032 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:31.032 | INFO | Uploaded 36GiB to GPU in 2.11746s (17.0015GiB/s) | 20260505 22:36:31.032 | INFO | Downloaded 34GiB to GPU in 2.00142s (16.988GiB/s) | 20260505 22:36:31.158 | INFO | Starting a round of Baz IO | 20260505 22:36:34.305 | INFO | Populating RTMetrics report | 20260505 22:36:34.305 | INFO | RTMetrics report populated | 20260505 22:36:34.305 | INFO | Opening tmp RTMetrics json file | 20260505 22:36:34.308 | INFO | Serializing RTMetrics to json | 20260505 22:36:34.310 | INFO | Serialization done, swapping tmp file now... | 20260505 22:36:34.310 | INFO | RTMetrics done | 20260505 22:36:34.387 | INFO | Finished a round of Baz IO: Wrote 1.925 GiB in 3.228 seconds | 20260505 22:36:34.387 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.49% / 0.00% / 78.51% | 20260505 22:36:34.387 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.121649% / 0% / 7.15256e-05% | 20260505 22:36:34.387 | INFO | Baz IO overall: | 20260505 22:36:34.387 | INFO | BytesWritten : 3936182929264 | 20260505 22:36:34.387 | INFO | NumEvents : 1856253556375 | 20260505 22:36:36.175 | INFO | Analyzing chunk frames = [6844416,6844928) | 20260505 22:36:36.175 | INFO | MemInfo: { | 20260505 22:36:36.175 | INFO | "gpu" : " 17 GB", | 20260505 22:36:36.175 | INFO | "huge" : "", | 20260505 22:36:36.175 | INFO | "huge_shared" : " 27 GB", | 20260505 22:36:36.175 | INFO | "malloc" : " 14 GB", | 20260505 22:36:36.175 | INFO | "pinned" : " 29 GB" | 20260505 22:36:36.175 | INFO | } | 20260505 22:36:36.175 | INFO | Perf report for the last 5.14351 seconds: | 20260505 22:36:36.175 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:36:36.175 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:36.175 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:36.175 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:36.175 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 22:36:36.175 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:36.175 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260505 22:36:36.175 | INFO | PRE_HQ : 59.9%, 1.8, 54.6ms, 0 | 20260505 22:36:36.175 | INFO | BAZWRITER : 62.0%, 3.6, 114.5ms, 0 | 20260505 22:36:36.175 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 22:36:36.176 | INFO | Uploaded 36GiB to GPU in 2.16052s (16.6627GiB/s) | 20260505 22:36:36.176 | INFO | Downloaded 38GiB to GPU in 2.23058s (17.0359GiB/s) | 20260505 22:36:41.316 | INFO | Analyzing chunk frames = [6844928,6845440) | 20260505 22:36:41.316 | INFO | MemInfo: { | 20260505 22:36:41.316 | INFO | "gpu" : " 17 GB", | 20260505 22:36:41.316 | INFO | "huge" : "", | 20260505 22:36:41.316 | INFO | "huge_shared" : " 27 GB", | 20260505 22:36:41.316 | INFO | "malloc" : " 14 GB", | 20260505 22:36:41.316 | INFO | "pinned" : " 29 GB" | 20260505 22:36:41.317 | INFO | } | 20260505 22:36:41.317 | INFO | Perf report for the last 5.14121 seconds: | 20260505 22:36:41.317 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:36:41.317 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:41.317 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:41.317 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:41.317 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 22:36:41.317 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:41.317 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:41.317 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 22:36:41.317 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:41.317 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:41.317 | INFO | Uploaded 36GiB to GPU in 2.08095s (17.2998GiB/s) | 20260505 22:36:41.317 | INFO | Downloaded 34GiB to GPU in 2.02685s (16.7748GiB/s) | 20260505 22:36:46.450 | INFO | Analyzing chunk frames = [6845440,6845952) | 20260505 22:36:46.450 | INFO | MemInfo: { | 20260505 22:36:46.450 | INFO | "gpu" : " 17 GB", | 20260505 22:36:46.450 | INFO | "huge" : "", | 20260505 22:36:46.450 | INFO | "huge_shared" : " 27 GB", | 20260505 22:36:46.450 | INFO | "malloc" : " 14 GB", | 20260505 22:36:46.450 | INFO | "pinned" : " 29 GB" | 20260505 22:36:46.450 | INFO | } | 20260505 22:36:46.450 | INFO | Perf report for the last 5.13391 seconds: | 20260505 22:36:46.450 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:36:46.450 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:46.450 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:46.450 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:46.450 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260505 22:36:46.451 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:46.451 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:46.451 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 22:36:46.451 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:46.451 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:46.451 | INFO | Uploaded 36GiB to GPU in 2.09351s (17.196GiB/s) | 20260505 22:36:46.451 | INFO | Downloaded 34GiB to GPU in 2.02651s (16.7776GiB/s) | 20260505 22:36:51.585 | INFO | Analyzing chunk frames = [6845952,6846464) | 20260505 22:36:51.585 | INFO | MemInfo: { | 20260505 22:36:51.585 | INFO | "gpu" : " 17 GB", | 20260505 22:36:51.585 | INFO | "huge" : "", | 20260505 22:36:51.585 | INFO | "huge_shared" : " 27 GB", | 20260505 22:36:51.585 | INFO | "malloc" : " 14 GB", | 20260505 22:36:51.585 | INFO | "pinned" : " 29 GB" | 20260505 22:36:51.585 | INFO | } | 20260505 22:36:51.585 | INFO | Perf report for the last 5.13487 seconds: | 20260505 22:36:51.585 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:36:51.585 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:51.585 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:51.585 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:51.585 | INFO | ANALYSIS : 62.4%, 3.0, 96.8ms, 0 | 20260505 22:36:51.585 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:51.585 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:51.585 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 22:36:51.585 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:51.585 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:51.586 | INFO | Uploaded 36GiB to GPU in 2.20701s (16.3116GiB/s) | 20260505 22:36:51.586 | INFO | Downloaded 34GiB to GPU in 2.07297s (16.4016GiB/s) | 20260505 22:36:56.730 | INFO | Analyzing chunk frames = [6846464,6846976) | 20260505 22:36:56.730 | INFO | MemInfo: { | 20260505 22:36:56.730 | INFO | "gpu" : " 17 GB", | 20260505 22:36:56.730 | INFO | "huge" : "", | 20260505 22:36:56.730 | INFO | "huge_shared" : " 27 GB", | 20260505 22:36:56.730 | INFO | "malloc" : " 14 GB", | 20260505 22:36:56.730 | INFO | "pinned" : " 29 GB" | 20260505 22:36:56.730 | INFO | } | 20260505 22:36:56.730 | INFO | Perf report for the last 5.14511 seconds: | 20260505 22:36:56.730 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:36:56.730 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:56.730 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:56.730 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:56.730 | INFO | ANALYSIS : 61.5%, 3.0, 96.1ms, 0 | 20260505 22:36:56.731 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:56.731 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:56.731 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 22:36:56.731 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:56.731 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:36:56.731 | INFO | Uploaded 36GiB to GPU in 2.13602s (16.8538GiB/s) | 20260505 22:36:56.731 | INFO | Downloaded 34GiB to GPU in 2.06254s (16.4845GiB/s) | 20260505 22:37:01.872 | INFO | Analyzing chunk frames = [6846976,6847488) | 20260505 22:37:01.872 | INFO | MemInfo: { | 20260505 22:37:01.872 | INFO | "gpu" : " 17 GB", | 20260505 22:37:01.872 | INFO | "huge" : "", | 20260505 22:37:01.872 | INFO | "huge_shared" : " 27 GB", | 20260505 22:37:01.872 | INFO | "malloc" : " 14 GB", | 20260505 22:37:01.872 | INFO | "pinned" : " 29 GB" | 20260505 22:37:01.872 | INFO | } | 20260505 22:37:01.872 | INFO | Perf report for the last 5.14156 seconds: | 20260505 22:37:01.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:37:01.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:01.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:01.872 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:01.872 | INFO | ANALYSIS : 61.5%, 3.0, 96.0ms, 0 | 20260505 22:37:01.872 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:01.872 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:01.872 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 22:37:01.872 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:01.872 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:01.872 | INFO | Uploaded 36GiB to GPU in 2.08054s (17.3032GiB/s) | 20260505 22:37:01.872 | INFO | Downloaded 34GiB to GPU in 2.04705s (16.6093GiB/s) | 20260505 22:37:07.011 | INFO | Analyzing chunk frames = [6847488,6848000) | 20260505 22:37:07.011 | INFO | MemInfo: { | 20260505 22:37:07.011 | INFO | "gpu" : " 17 GB", | 20260505 22:37:07.011 | INFO | "huge" : "", | 20260505 22:37:07.011 | INFO | "huge_shared" : " 27 GB", | 20260505 22:37:07.011 | INFO | "malloc" : " 14 GB", | 20260505 22:37:07.011 | INFO | "pinned" : " 29 GB" | 20260505 22:37:07.011 | INFO | } | 20260505 22:37:07.012 | INFO | Perf report for the last 5.13947 seconds: | 20260505 22:37:07.012 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:37:07.012 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:07.012 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:07.012 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:07.012 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 22:37:07.012 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:07.012 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:07.012 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 22:37:07.012 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:07.012 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:07.012 | INFO | Uploaded 36GiB to GPU in 2.00289s (17.974GiB/s) | 20260505 22:37:07.012 | INFO | Downloaded 34GiB to GPU in 2.03428s (16.7136GiB/s) | 20260505 22:37:12.143 | INFO | Analyzing chunk frames = [6848000,6848512) | 20260505 22:37:12.143 | INFO | MemInfo: { | 20260505 22:37:12.143 | INFO | "gpu" : " 17 GB", | 20260505 22:37:12.143 | INFO | "huge" : "", | 20260505 22:37:12.143 | INFO | "huge_shared" : " 27 GB", | 20260505 22:37:12.143 | INFO | "malloc" : " 14 GB", | 20260505 22:37:12.143 | INFO | "pinned" : " 29 GB" | 20260505 22:37:12.143 | INFO | } | 20260505 22:37:12.143 | INFO | Perf report for the last 5.13157 seconds: | 20260505 22:37:12.143 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:37:12.143 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:12.143 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:12.143 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:12.143 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260505 22:37:12.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:12.143 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:12.143 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:37:12.143 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:12.143 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:12.143 | INFO | Uploaded 36GiB to GPU in 2.1125s (17.0414GiB/s) | 20260505 22:37:12.143 | INFO | Downloaded 34GiB to GPU in 1.99713s (17.0244GiB/s) | 20260505 22:37:12.144 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 22:37:12.144 | INFO | Total mem usage: 15.3 GiB | 20260505 22:37:12.144 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 22:37:12.144 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.52%) | 20260505 22:37:12.144 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 22:37:12.144 | INFO | High Water Breakdown: | 20260505 22:37:12.144 | INFO | 61.1% : 8.62 GiB ( 90.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 22:37:12.144 | INFO | 38.8% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 22:37:12.144 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 22:37:12.144 | INFO | ------------------------------------------ | 20260505 22:37:12.144 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:37:12.144 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 22:37:12.144 | INFO | Total mem usage: 26.5 GiB | 20260505 22:37:12.144 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 22:37:12.144 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 22:37:12.144 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 22:37:12.144 | INFO | High Water Breakdown: | 20260505 22:37:12.144 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:37:12.144 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:37:12.144 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:37:12.144 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:37:12.144 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:37:12.144 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:37:12.144 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 22:37:12.144 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 22:37:12.144 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:37:12.144 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 22:37:12.144 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 22:37:12.144 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 22:37:12.144 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 22:37:12.144 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 22:37:12.144 | INFO | ------------------------------------------ | 20260505 22:37:12.144 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:37:12.144 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 22:37:12.144 | INFO | Total mem usage: 25.6 GiB | 20260505 22:37:12.144 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 22:37:12.144 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 22:37:12.144 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 22:37:12.144 | INFO | High Water Breakdown: | 20260505 22:37:12.144 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 22:37:12.144 | INFO | ------------------------------------------ | 20260505 22:37:12.144 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:37:12.144 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 22:37:12.144 | INFO | Total mem usage: 15.7 GiB | 20260505 22:37:12.144 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 22:37:12.144 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 22:37:12.144 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 22:37:12.144 | INFO | High Water Breakdown: | 20260505 22:37:12.144 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 22:37:12.144 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 22:37:12.144 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 22:37:12.144 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 22:37:12.144 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 22:37:12.144 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 22:37:12.144 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 22:37:12.144 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 22:37:12.144 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 22:37:12.144 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:37:12.144 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 22:37:12.144 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:37:12.144 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 22:37:12.144 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 22:37:12.144 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 22:37:12.144 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 22:37:12.144 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:37:12.144 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 22:37:12.144 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:37:12.144 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:37:12.144 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:37:12.144 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 22:37:12.144 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 22:37:12.144 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:37:12.144 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 22:37:12.144 | INFO | ------------------------------------------ | 20260505 22:37:12.144 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:37:12.144 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 22:37:12.144 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:37:12.144 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:37:12.144 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.41e+01, 1.34e+01 / 8.34e-01 | 20260505 22:37:12.144 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.03e-01, 1.01e-02 / 3.13e-03 | 20260505 22:37:12.144 | INFO | PerfReport: 11.82%, Baseline -- 3.71e+00 / 4.53e+00, 5.71e-01 / 1.46e-01 | 20260505 22:37:12.144 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.54e+00, 1.88e-01 / 7.09e-02 | 20260505 22:37:12.144 | INFO | PerfReport: 16.00%, DME -- 4.01e+01 / 4.42e+01, 3.89e+00 / 8.33e-01 | 20260505 22:37:12.144 | INFO | PerfReport: 57.18%, FrameLabeling -- 1.79e+01 / 1.92e+01, 1.46e+00 / 9.73e-01 | 20260505 22:37:12.144 | INFO | PerfReport: 7.19%, PulseAccumulating -- 2.25e+00 / 2.80e+00, 4.32e-01 / 1.88e-01 | 20260505 22:37:12.144 | INFO | PerfReport: 3.16%, Metrics -- 9.90e-01 / 1.28e+00, 2.64e-01 / 1.06e-01 | 20260505 22:37:12.144 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.44e-01 / 4.06e-01, 5.82e-02 / 3.84e-02 | 20260505 22:37:12.144 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 22:37:12.144 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:37:12.144 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:37:12.144 | INFO | PerfReport: Full Execution -- 9.87e+01 / -nan, 2.48e+01 / 0.00e+00 | 20260505 22:37:12.144 | INFO | PerfReport: 22.33%, Upload -- 2.20e+01 / 0.00e+00, 7.55e+00 / 0.00e+00 | 20260505 22:37:12.144 | INFO | PerfReport: 21.76%, Download -- 2.15e+01 / 0.00e+00, 7.59e+00 / 0.00e+00 | 20260505 22:37:12.144 | INFO | PerfReport: 31.78%, Compute -- 3.14e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 22:37:12.144 | INFO | PerfReport: 2.19%, LockUpload -- 2.16e+00 / 0.00e+00, 7.86e+00 / 0.00e+00 | 20260505 22:37:12.144 | INFO | PerfReport: 1.31%, LockDownload -- 1.29e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260505 22:37:12.145 | INFO | PerfReport: 20.61%, LockCompute -- 2.03e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 22:37:12.145 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.66e-02 / 0.00e+00 | 20260505 22:37:12.274 | INFO | Starting a round of Baz IO | 20260505 22:37:15.438 | INFO | Populating RTMetrics report | 20260505 22:37:15.438 | INFO | RTMetrics report populated | 20260505 22:37:15.438 | INFO | Opening tmp RTMetrics json file | 20260505 22:37:15.439 | INFO | Serializing RTMetrics to json | 20260505 22:37:15.440 | INFO | Serialization done, swapping tmp file now... | 20260505 22:37:15.441 | INFO | RTMetrics done | 20260505 22:37:15.519 | INFO | Finished a round of Baz IO: Wrote 1.925 GiB in 3.244 seconds | 20260505 22:37:15.519 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.48% / 0.00% / 78.51% | 20260505 22:37:15.519 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.122162% / 0% / 0.000111262% | 20260505 22:37:15.519 | INFO | Baz IO overall: | 20260505 22:37:15.519 | INFO | BytesWritten : 3938250235191 | 20260505 22:37:15.519 | INFO | NumEvents : 1857204433842 | 20260505 22:37:17.285 | INFO | Analyzing chunk frames = [6848512,6849024) | 20260505 22:37:17.285 | INFO | MemInfo: { | 20260505 22:37:17.285 | INFO | "gpu" : " 17 GB", | 20260505 22:37:17.285 | INFO | "huge" : "", | 20260505 22:37:17.285 | INFO | "huge_shared" : " 27 GB", | 20260505 22:37:17.285 | INFO | "malloc" : " 14 GB", | 20260505 22:37:17.285 | INFO | "pinned" : " 29 GB" | 20260505 22:37:17.285 | INFO | } | 20260505 22:37:17.285 | INFO | Perf report for the last 5.14185 seconds: | 20260505 22:37:17.285 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:37:17.285 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:17.285 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:17.285 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:17.285 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260505 22:37:17.285 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:17.285 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260505 22:37:17.285 | INFO | PRE_HQ : 60.1%, 1.8, 54.8ms, 0 | 20260505 22:37:17.285 | INFO | BAZWRITER : 62.5%, 3.5, 114.1ms, 0 | 20260505 22:37:17.285 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 22:37:17.285 | INFO | Uploaded 36GiB to GPU in 2.1733s (16.5647GiB/s) | 20260505 22:37:17.285 | INFO | Downloaded 38GiB to GPU in 2.24897s (16.8966GiB/s) | 20260505 22:37:22.428 | INFO | Analyzing chunk frames = [6849024,6849536) | 20260505 22:37:22.428 | INFO | MemInfo: { | 20260505 22:37:22.428 | INFO | "gpu" : " 17 GB", | 20260505 22:37:22.428 | INFO | "huge" : "", | 20260505 22:37:22.429 | INFO | "huge_shared" : " 27 GB", | 20260505 22:37:22.429 | INFO | "malloc" : " 14 GB", | 20260505 22:37:22.429 | INFO | "pinned" : " 29 GB" | 20260505 22:37:22.429 | INFO | } | 20260505 22:37:22.429 | INFO | Perf report for the last 5.14363 seconds: | 20260505 22:37:22.429 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:37:22.429 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:22.429 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:22.429 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:22.429 | INFO | ANALYSIS : 62.1%, 3.0, 97.1ms, 0 | 20260505 22:37:22.429 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:22.429 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:22.429 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:37:22.429 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:22.429 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:22.429 | INFO | Uploaded 36GiB to GPU in 2.09133s (17.2139GiB/s) | 20260505 22:37:22.429 | INFO | Downloaded 34GiB to GPU in 2.02321s (16.805GiB/s) | 20260505 22:37:27.565 | INFO | Analyzing chunk frames = [6849536,6850048) | 20260505 22:37:27.565 | INFO | MemInfo: { | 20260505 22:37:27.565 | INFO | "gpu" : " 17 GB", | 20260505 22:37:27.565 | INFO | "huge" : "", | 20260505 22:37:27.565 | INFO | "huge_shared" : " 27 GB", | 20260505 22:37:27.565 | INFO | "malloc" : " 14 GB", | 20260505 22:37:27.565 | INFO | "pinned" : " 29 GB" | 20260505 22:37:27.565 | INFO | } | 20260505 22:37:27.565 | INFO | Perf report for the last 5.13648 seconds: | 20260505 22:37:27.565 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:37:27.565 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:27.565 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:27.565 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:27.565 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260505 22:37:27.565 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:27.565 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:27.565 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:37:27.565 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:27.565 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:27.565 | INFO | Uploaded 36GiB to GPU in 2.1231s (16.9563GiB/s) | 20260505 22:37:27.565 | INFO | Downloaded 34GiB to GPU in 2.01057s (16.9106GiB/s) | 20260505 22:37:32.706 | INFO | Analyzing chunk frames = [6850048,6850560) | 20260505 22:37:32.706 | INFO | MemInfo: { | 20260505 22:37:32.706 | INFO | "gpu" : " 17 GB", | 20260505 22:37:32.706 | INFO | "huge" : "", | 20260505 22:37:32.706 | INFO | "huge_shared" : " 27 GB", | 20260505 22:37:32.706 | INFO | "malloc" : " 14 GB", | 20260505 22:37:32.706 | INFO | "pinned" : " 29 GB" | 20260505 22:37:32.706 | INFO | } | 20260505 22:37:32.706 | INFO | Perf report for the last 5.14057 seconds: | 20260505 22:37:32.706 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:37:32.706 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:32.706 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:32.706 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:32.706 | INFO | ANALYSIS : 61.8%, 3.0, 95.9ms, 0 | 20260505 22:37:32.706 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:32.706 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:32.706 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:37:32.706 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:32.706 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:32.706 | INFO | Uploaded 36GiB to GPU in 2.13124s (16.8916GiB/s) | 20260505 22:37:32.706 | INFO | Downloaded 34GiB to GPU in 2.0219s (16.8158GiB/s) | 20260505 22:37:37.843 | INFO | Analyzing chunk frames = [6850560,6851072) | 20260505 22:37:37.843 | INFO | MemInfo: { | 20260505 22:37:37.843 | INFO | "gpu" : " 17 GB", | 20260505 22:37:37.843 | INFO | "huge" : "", | 20260505 22:37:37.843 | INFO | "huge_shared" : " 27 GB", | 20260505 22:37:37.843 | INFO | "malloc" : " 14 GB", | 20260505 22:37:37.843 | INFO | "pinned" : " 29 GB" | 20260505 22:37:37.843 | INFO | } | 20260505 22:37:37.843 | INFO | Perf report for the last 5.13778 seconds: | 20260505 22:37:37.843 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:37:37.843 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:37.843 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:37.844 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:37.844 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260505 22:37:37.844 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:37.844 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:37.844 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 22:37:37.844 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:37.844 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:37.844 | INFO | Uploaded 36GiB to GPU in 2.09715s (17.1661GiB/s) | 20260505 22:37:37.844 | INFO | Downloaded 34GiB to GPU in 2.05166s (16.572GiB/s) | 20260505 22:37:42.986 | INFO | Analyzing chunk frames = [6851072,6851584) | 20260505 22:37:42.986 | INFO | MemInfo: { | 20260505 22:37:42.986 | INFO | "gpu" : " 17 GB", | 20260505 22:37:42.986 | INFO | "huge" : "", | 20260505 22:37:42.986 | INFO | "huge_shared" : " 27 GB", | 20260505 22:37:42.986 | INFO | "malloc" : " 14 GB", | 20260505 22:37:42.986 | INFO | "pinned" : " 29 GB" | 20260505 22:37:42.987 | INFO | } | 20260505 22:37:42.987 | INFO | Perf report for the last 5.14304 seconds: | 20260505 22:37:42.987 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:37:42.987 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:42.987 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:42.987 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:42.987 | INFO | ANALYSIS : 61.5%, 3.0, 96.1ms, 0 | 20260505 22:37:42.987 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:42.987 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:42.987 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:37:42.987 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:42.987 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:42.987 | INFO | Uploaded 36GiB to GPU in 2.08396s (17.2748GiB/s) | 20260505 22:37:42.987 | INFO | Downloaded 34GiB to GPU in 2.03826s (16.6809GiB/s) | 20260505 22:37:48.126 | INFO | Analyzing chunk frames = [6851584,6852096) | 20260505 22:37:48.126 | INFO | MemInfo: { | 20260505 22:37:48.126 | INFO | "gpu" : " 17 GB", | 20260505 22:37:48.126 | INFO | "huge" : "", | 20260505 22:37:48.126 | INFO | "huge_shared" : " 27 GB", | 20260505 22:37:48.126 | INFO | "malloc" : " 14 GB", | 20260505 22:37:48.126 | INFO | "pinned" : " 29 GB" | 20260505 22:37:48.126 | INFO | } | 20260505 22:37:48.126 | INFO | Perf report for the last 5.13916 seconds: | 20260505 22:37:48.126 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:37:48.126 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:48.126 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:48.126 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:48.126 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 22:37:48.126 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:48.126 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:48.126 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 22:37:48.126 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:48.126 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:48.126 | INFO | Uploaded 36GiB to GPU in 2.03509s (17.6896GiB/s) | 20260505 22:37:48.126 | INFO | Downloaded 34GiB to GPU in 2.0391s (16.674GiB/s) | 20260505 22:37:53.262 | INFO | Analyzing chunk frames = [6852096,6852608) | 20260505 22:37:53.262 | INFO | MemInfo: { | 20260505 22:37:53.262 | INFO | "gpu" : " 17 GB", | 20260505 22:37:53.262 | INFO | "huge" : "", | 20260505 22:37:53.262 | INFO | "huge_shared" : " 27 GB", | 20260505 22:37:53.262 | INFO | "malloc" : " 14 GB", | 20260505 22:37:53.262 | INFO | "pinned" : " 29 GB" | 20260505 22:37:53.262 | INFO | } | 20260505 22:37:53.262 | INFO | Perf report for the last 5.13628 seconds: | 20260505 22:37:53.262 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:37:53.262 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:53.262 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:53.262 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:53.262 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 22:37:53.262 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:53.262 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:53.262 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 22:37:53.262 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:53.262 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:53.262 | INFO | Uploaded 36GiB to GPU in 2.1163s (17.0108GiB/s) | 20260505 22:37:53.262 | INFO | Downloaded 34GiB to GPU in 2.00218s (16.9815GiB/s) | 20260505 22:37:53.394 | INFO | Starting a round of Baz IO | 20260505 22:37:56.545 | INFO | Populating RTMetrics report | 20260505 22:37:56.545 | INFO | RTMetrics report populated | 20260505 22:37:56.545 | INFO | Opening tmp RTMetrics json file | 20260505 22:37:56.546 | INFO | Serializing RTMetrics to json | 20260505 22:37:56.547 | INFO | Serialization done, swapping tmp file now... | 20260505 22:37:56.548 | INFO | RTMetrics done | 20260505 22:37:56.625 | INFO | Finished a round of Baz IO: Wrote 1.925 GiB in 3.23 seconds | 20260505 22:37:56.625 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.48% / 0.00% / 78.52% | 20260505 22:37:56.625 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.121812% / 0% / 8.34465e-05% | 20260505 22:37:56.625 | INFO | Baz IO overall: | 20260505 22:37:56.625 | INFO | BytesWritten : 3940317458123 | 20260505 22:37:56.625 | INFO | NumEvents : 1858155131452 | 20260505 22:37:58.401 | INFO | Analyzing chunk frames = [6852608,6853120) | 20260505 22:37:58.401 | INFO | MemInfo: { | 20260505 22:37:58.401 | INFO | "gpu" : " 17 GB", | 20260505 22:37:58.401 | INFO | "huge" : "", | 20260505 22:37:58.401 | INFO | "huge_shared" : " 27 GB", | 20260505 22:37:58.401 | INFO | "malloc" : " 14 GB", | 20260505 22:37:58.401 | INFO | "pinned" : " 29 GB" | 20260505 22:37:58.401 | INFO | } | 20260505 22:37:58.401 | INFO | Perf report for the last 5.13896 seconds: | 20260505 22:37:58.401 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:37:58.401 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:58.401 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:58.401 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:58.401 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 22:37:58.401 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:37:58.401 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260505 22:37:58.401 | INFO | PRE_HQ : 59.7%, 1.8, 54.5ms, 0 | 20260505 22:37:58.401 | INFO | BAZWRITER : 62.3%, 3.6, 114.1ms, 0 | 20260505 22:37:58.401 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 22:37:58.401 | INFO | Uploaded 36GiB to GPU in 2.163s (16.6436GiB/s) | 20260505 22:37:58.401 | INFO | Downloaded 38GiB to GPU in 2.23388s (17.0107GiB/s) | 20260505 22:38:03.538 | INFO | Analyzing chunk frames = [6853120,6853632) | 20260505 22:38:03.538 | INFO | MemInfo: { | 20260505 22:38:03.538 | INFO | "gpu" : " 17 GB", | 20260505 22:38:03.538 | INFO | "huge" : "", | 20260505 22:38:03.538 | INFO | "huge_shared" : " 27 GB", | 20260505 22:38:03.538 | INFO | "malloc" : " 14 GB", | 20260505 22:38:03.538 | INFO | "pinned" : " 29 GB" | 20260505 22:38:03.538 | INFO | } | 20260505 22:38:03.538 | INFO | Perf report for the last 5.1367 seconds: | 20260505 22:38:03.538 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:38:03.538 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:03.538 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:03.538 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:03.538 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 22:38:03.538 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:03.538 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:03.538 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:38:03.538 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:03.538 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:03.538 | INFO | Uploaded 36GiB to GPU in 2.08495s (17.2666GiB/s) | 20260505 22:38:03.538 | INFO | Downloaded 34GiB to GPU in 2.02683s (16.7749GiB/s) | 20260505 22:38:08.687 | INFO | Analyzing chunk frames = [6853632,6854144) | 20260505 22:38:08.687 | INFO | MemInfo: { | 20260505 22:38:08.687 | INFO | "gpu" : " 17 GB", | 20260505 22:38:08.687 | INFO | "huge" : "", | 20260505 22:38:08.687 | INFO | "huge_shared" : " 27 GB", | 20260505 22:38:08.687 | INFO | "malloc" : " 14 GB", | 20260505 22:38:08.687 | INFO | "pinned" : " 29 GB" | 20260505 22:38:08.687 | INFO | } | 20260505 22:38:08.687 | INFO | Perf report for the last 5.14976 seconds: | 20260505 22:38:08.687 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:38:08.687 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:08.687 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:08.687 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:08.688 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260505 22:38:08.688 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:08.688 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:08.688 | INFO | PRE_HQ : 26.8%, 1.0, 14.0ms, 0 | 20260505 22:38:08.688 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:08.688 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:08.688 | INFO | Uploaded 36GiB to GPU in 2.14049s (16.8186GiB/s) | 20260505 22:38:08.688 | INFO | Downloaded 34GiB to GPU in 2.03425s (16.7138GiB/s) | 20260505 22:38:13.828 | INFO | Analyzing chunk frames = [6854144,6854656) | 20260505 22:38:13.828 | INFO | MemInfo: { | 20260505 22:38:13.828 | INFO | "gpu" : " 17 GB", | 20260505 22:38:13.828 | INFO | "huge" : "", | 20260505 22:38:13.828 | INFO | "huge_shared" : " 27 GB", | 20260505 22:38:13.828 | INFO | "malloc" : " 14 GB", | 20260505 22:38:13.828 | INFO | "pinned" : " 29 GB" | 20260505 22:38:13.828 | INFO | } | 20260505 22:38:13.829 | INFO | Perf report for the last 5.14104 seconds: | 20260505 22:38:13.829 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:38:13.829 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:13.829 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:13.829 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:13.829 | INFO | ANALYSIS : 61.6%, 3.0, 95.7ms, 0 | 20260505 22:38:13.829 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:13.829 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:13.829 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 22:38:13.829 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:13.829 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:13.829 | INFO | Uploaded 36GiB to GPU in 2.14294s (16.7993GiB/s) | 20260505 22:38:13.829 | INFO | Downloaded 34GiB to GPU in 2.02604s (16.7815GiB/s) | 20260505 22:38:18.958 | INFO | Analyzing chunk frames = [6854656,6855168) | 20260505 22:38:18.958 | INFO | MemInfo: { | 20260505 22:38:18.958 | INFO | "gpu" : " 17 GB", | 20260505 22:38:18.958 | INFO | "huge" : "", | 20260505 22:38:18.958 | INFO | "huge_shared" : " 27 GB", | 20260505 22:38:18.958 | INFO | "malloc" : " 14 GB", | 20260505 22:38:18.958 | INFO | "pinned" : " 29 GB" | 20260505 22:38:18.958 | INFO | } | 20260505 22:38:18.958 | INFO | Perf report for the last 5.12941 seconds: | 20260505 22:38:18.958 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:38:18.958 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:18.958 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:18.958 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:18.958 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260505 22:38:18.958 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:18.958 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:18.958 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 22:38:18.958 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:18.958 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:18.958 | INFO | Uploaded 36GiB to GPU in 2.09671s (17.1698GiB/s) | 20260505 22:38:18.959 | INFO | Downloaded 34GiB to GPU in 2.04397s (16.6343GiB/s) | 20260505 22:38:24.092 | INFO | Analyzing chunk frames = [6855168,6855680) | 20260505 22:38:24.092 | INFO | MemInfo: { | 20260505 22:38:24.092 | INFO | "gpu" : " 17 GB", | 20260505 22:38:24.092 | INFO | "huge" : "", | 20260505 22:38:24.092 | INFO | "huge_shared" : " 27 GB", | 20260505 22:38:24.092 | INFO | "malloc" : " 14 GB", | 20260505 22:38:24.092 | INFO | "pinned" : " 29 GB" | 20260505 22:38:24.092 | INFO | } | 20260505 22:38:24.092 | INFO | Perf report for the last 5.13446 seconds: | 20260505 22:38:24.092 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:38:24.092 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:24.092 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:24.092 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:24.092 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260505 22:38:24.092 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:24.092 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:24.092 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 22:38:24.092 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:24.092 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:24.093 | INFO | Uploaded 36GiB to GPU in 2.08707s (17.249GiB/s) | 20260505 22:38:24.093 | INFO | Downloaded 34GiB to GPU in 2.04061s (16.6617GiB/s) | 20260505 22:38:29.238 | INFO | Analyzing chunk frames = [6855680,6856192) | 20260505 22:38:29.238 | INFO | MemInfo: { | 20260505 22:38:29.238 | INFO | "gpu" : " 17 GB", | 20260505 22:38:29.238 | INFO | "huge" : "", | 20260505 22:38:29.238 | INFO | "huge_shared" : " 27 GB", | 20260505 22:38:29.238 | INFO | "malloc" : " 14 GB", | 20260505 22:38:29.238 | INFO | "pinned" : " 29 GB" | 20260505 22:38:29.238 | INFO | } | 20260505 22:38:29.238 | INFO | Perf report for the last 5.14579 seconds: | 20260505 22:38:29.238 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:38:29.238 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:29.238 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:29.238 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:29.238 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 22:38:29.238 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:29.238 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:29.238 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 22:38:29.238 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:29.238 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:29.238 | INFO | Uploaded 36GiB to GPU in 2.13606s (16.8535GiB/s) | 20260505 22:38:29.239 | INFO | Downloaded 34GiB to GPU in 2.13715s (15.909GiB/s) | 20260505 22:38:34.379 | INFO | Analyzing chunk frames = [6856192,6856704) | 20260505 22:38:34.379 | INFO | MemInfo: { | 20260505 22:38:34.379 | INFO | "gpu" : " 17 GB", | 20260505 22:38:34.379 | INFO | "huge" : "", | 20260505 22:38:34.379 | INFO | "huge_shared" : " 27 GB", | 20260505 22:38:34.379 | INFO | "malloc" : " 14 GB", | 20260505 22:38:34.379 | INFO | "pinned" : " 29 GB" | 20260505 22:38:34.379 | INFO | } | 20260505 22:38:34.379 | INFO | Perf report for the last 5.14081 seconds: | 20260505 22:38:34.379 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:38:34.379 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:34.379 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:34.379 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:34.379 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 22:38:34.379 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:34.379 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:34.379 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 22:38:34.379 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:34.379 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:34.379 | INFO | Uploaded 36GiB to GPU in 2.11394s (17.0298GiB/s) | 20260505 22:38:34.379 | INFO | Downloaded 34GiB to GPU in 2.00378s (16.9679GiB/s) | 20260505 22:38:34.379 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 22:38:34.379 | INFO | Total mem usage: 15.3 GiB | 20260505 22:38:34.379 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 22:38:34.379 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -9.05%) | 20260505 22:38:34.379 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 22:38:34.379 | INFO | High Water Breakdown: | 20260505 22:38:34.379 | INFO | 60.9% : 8.56 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 22:38:34.379 | INFO | 39% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 22:38:34.379 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 22:38:34.379 | INFO | ------------------------------------------ | 20260505 22:38:34.379 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:38:34.379 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 22:38:34.379 | INFO | Total mem usage: 26.5 GiB | 20260505 22:38:34.379 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 22:38:34.379 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 22:38:34.379 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 22:38:34.379 | INFO | High Water Breakdown: | 20260505 22:38:34.379 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:38:34.379 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:38:34.379 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:38:34.379 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:38:34.380 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:38:34.380 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:38:34.380 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 22:38:34.380 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 22:38:34.380 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:38:34.380 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 22:38:34.380 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 22:38:34.380 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 22:38:34.380 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 22:38:34.380 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 22:38:34.380 | INFO | ------------------------------------------ | 20260505 22:38:34.380 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:38:34.380 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 22:38:34.380 | INFO | Total mem usage: 25.6 GiB | 20260505 22:38:34.380 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 22:38:34.380 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 22:38:34.380 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 22:38:34.380 | INFO | High Water Breakdown: | 20260505 22:38:34.380 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 22:38:34.380 | INFO | ------------------------------------------ | 20260505 22:38:34.380 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:38:34.380 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 22:38:34.380 | INFO | Total mem usage: 15.7 GiB | 20260505 22:38:34.380 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 22:38:34.380 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 22:38:34.380 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 22:38:34.380 | INFO | High Water Breakdown: | 20260505 22:38:34.380 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 22:38:34.380 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 22:38:34.380 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 22:38:34.380 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 22:38:34.380 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 22:38:34.380 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 22:38:34.380 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 22:38:34.380 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 22:38:34.380 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 22:38:34.380 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:38:34.380 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 22:38:34.380 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:38:34.380 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 22:38:34.380 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 22:38:34.380 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 22:38:34.380 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 22:38:34.380 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:38:34.380 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 22:38:34.380 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:38:34.380 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:38:34.380 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:38:34.380 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 22:38:34.380 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 22:38:34.380 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:38:34.380 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 22:38:34.380 | INFO | ------------------------------------------ | 20260505 22:38:34.380 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:38:34.380 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 22:38:34.380 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:38:34.380 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:38:34.380 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.44e+01, 1.33e+01 / 1.28e+00 | 20260505 22:38:34.380 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.04e-01, 1.08e-02 / 2.78e-03 | 20260505 22:38:34.380 | INFO | PerfReport: 11.83%, Baseline -- 3.72e+00 / 4.55e+00, 5.74e-01 / 1.69e-01 | 20260505 22:38:34.380 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.53e+00, 1.92e-01 / 7.68e-02 | 20260505 22:38:34.380 | INFO | PerfReport: 16.02%, DME -- 4.03e+01 / 4.44e+01, 4.07e+00 / 1.51e+00 | 20260505 22:38:34.380 | INFO | PerfReport: 57.18%, FrameLabeling -- 1.80e+01 / 1.92e+01, 1.48e+00 / 1.02e+00 | 20260505 22:38:34.380 | INFO | PerfReport: 7.14%, PulseAccumulating -- 2.24e+00 / 2.81e+00, 4.29e-01 / 2.26e-01 | 20260505 22:38:34.380 | INFO | PerfReport: 3.15%, Metrics -- 9.91e-01 / 1.31e+00, 2.72e-01 / 1.31e-01 | 20260505 22:38:34.380 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.44e-01 / 4.05e-01, 5.84e-02 / 3.43e-02 | 20260505 22:38:34.380 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 22:38:34.380 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:38:34.380 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:38:34.380 | INFO | PerfReport: Full Execution -- 9.87e+01 / -nan, 2.49e+01 / 0.00e+00 | 20260505 22:38:34.380 | INFO | PerfReport: 22.30%, Upload -- 2.20e+01 / 0.00e+00, 7.50e+00 / 0.00e+00 | 20260505 22:38:34.380 | INFO | PerfReport: 21.75%, Download -- 2.15e+01 / 0.00e+00, 7.54e+00 / 0.00e+00 | 20260505 22:38:34.380 | INFO | PerfReport: 31.82%, Compute -- 3.14e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260505 22:38:34.380 | INFO | PerfReport: 2.20%, LockUpload -- 2.17e+00 / 0.00e+00, 8.00e+00 / 0.00e+00 | 20260505 22:38:34.380 | INFO | PerfReport: 1.30%, LockDownload -- 1.29e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260505 22:38:34.380 | INFO | PerfReport: 20.62%, LockCompute -- 2.04e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 22:38:34.380 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.40e-02 / 0.00e+00 | 20260505 22:38:34.506 | INFO | Starting a round of Baz IO | 20260505 22:38:37.657 | INFO | Populating RTMetrics report | 20260505 22:38:37.658 | INFO | RTMetrics report populated | 20260505 22:38:37.658 | INFO | Opening tmp RTMetrics json file | 20260505 22:38:37.660 | INFO | Serializing RTMetrics to json | 20260505 22:38:37.662 | INFO | Serialization done, swapping tmp file now... | 20260505 22:38:37.662 | INFO | RTMetrics done | 20260505 22:38:37.741 | INFO | Finished a round of Baz IO: Wrote 1.922 GiB in 3.235 seconds | 20260505 22:38:37.741 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.48% / 0.00% / 78.52% | 20260505 22:38:37.741 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.121041% / 7.94729e-06% / 0.000119209% | 20260505 22:38:37.741 | INFO | Baz IO overall: | 20260505 22:38:37.741 | INFO | BytesWritten : 3942381502810 | 20260505 22:38:37.741 | INFO | NumEvents : 1859103492856 | 20260505 22:38:39.524 | INFO | Analyzing chunk frames = [6856704,6857216) | 20260505 22:38:39.524 | INFO | MemInfo: { | 20260505 22:38:39.524 | INFO | "gpu" : " 17 GB", | 20260505 22:38:39.524 | INFO | "huge" : "", | 20260505 22:38:39.524 | INFO | "huge_shared" : " 27 GB", | 20260505 22:38:39.524 | INFO | "malloc" : " 14 GB", | 20260505 22:38:39.524 | INFO | "pinned" : " 29 GB" | 20260505 22:38:39.524 | INFO | } | 20260505 22:38:39.524 | INFO | Perf report for the last 5.14508 seconds: | 20260505 22:38:39.524 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:38:39.524 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:39.524 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:39.524 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:39.524 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 22:38:39.524 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:39.524 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260505 22:38:39.524 | INFO | PRE_HQ : 59.9%, 1.8, 54.9ms, 0 | 20260505 22:38:39.524 | INFO | BAZWRITER : 62.2%, 3.6, 114.1ms, 0 | 20260505 22:38:39.524 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 22:38:39.524 | INFO | Uploaded 36GiB to GPU in 2.14927s (16.7499GiB/s) | 20260505 22:38:39.524 | INFO | Downloaded 38GiB to GPU in 2.23503s (17.002GiB/s) | 20260505 22:38:44.657 | INFO | Analyzing chunk frames = [6857216,6857728) | 20260505 22:38:44.657 | INFO | MemInfo: { | 20260505 22:38:44.657 | INFO | "gpu" : " 17 GB", | 20260505 22:38:44.657 | INFO | "huge" : "", | 20260505 22:38:44.657 | INFO | "huge_shared" : " 27 GB", | 20260505 22:38:44.657 | INFO | "malloc" : " 14 GB", | 20260505 22:38:44.657 | INFO | "pinned" : " 29 GB" | 20260505 22:38:44.657 | INFO | } | 20260505 22:38:44.657 | INFO | Perf report for the last 5.13273 seconds: | 20260505 22:38:44.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:38:44.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:44.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:44.657 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:44.657 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260505 22:38:44.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:44.657 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:44.657 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 22:38:44.657 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:44.657 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:44.657 | INFO | Uploaded 36GiB to GPU in 2.09387s (17.193GiB/s) | 20260505 22:38:44.657 | INFO | Downloaded 34GiB to GPU in 2.03941s (16.6715GiB/s) | 20260505 22:38:49.793 | INFO | Analyzing chunk frames = [6857728,6858240) | 20260505 22:38:49.793 | INFO | MemInfo: { | 20260505 22:38:49.793 | INFO | "gpu" : " 17 GB", | 20260505 22:38:49.793 | INFO | "huge" : "", | 20260505 22:38:49.793 | INFO | "huge_shared" : " 27 GB", | 20260505 22:38:49.793 | INFO | "malloc" : " 14 GB", | 20260505 22:38:49.793 | INFO | "pinned" : " 29 GB" | 20260505 22:38:49.793 | INFO | } | 20260505 22:38:49.793 | INFO | Perf report for the last 5.13589 seconds: | 20260505 22:38:49.793 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:38:49.793 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:49.793 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:49.793 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:49.793 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260505 22:38:49.793 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:49.793 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:49.793 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 22:38:49.793 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:49.793 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:49.793 | INFO | Uploaded 36GiB to GPU in 2.13377s (16.8716GiB/s) | 20260505 22:38:49.793 | INFO | Downloaded 34GiB to GPU in 2.02259s (16.8101GiB/s) | 20260505 22:38:54.934 | INFO | Analyzing chunk frames = [6858240,6858752) | 20260505 22:38:54.934 | INFO | MemInfo: { | 20260505 22:38:54.934 | INFO | "gpu" : " 17 GB", | 20260505 22:38:54.934 | INFO | "huge" : "", | 20260505 22:38:54.934 | INFO | "huge_shared" : " 27 GB", | 20260505 22:38:54.934 | INFO | "malloc" : " 14 GB", | 20260505 22:38:54.934 | INFO | "pinned" : " 29 GB" | 20260505 22:38:54.934 | INFO | } | 20260505 22:38:54.934 | INFO | Perf report for the last 5.14106 seconds: | 20260505 22:38:54.934 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:38:54.934 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:54.934 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:54.934 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:54.934 | INFO | ANALYSIS : 61.9%, 3.0, 96.1ms, 0 | 20260505 22:38:54.934 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:54.934 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:54.934 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 22:38:54.934 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:54.934 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:38:54.934 | INFO | Uploaded 36GiB to GPU in 2.1376s (16.8413GiB/s) | 20260505 22:38:54.934 | INFO | Downloaded 34GiB to GPU in 2.04578s (16.6196GiB/s) | 20260505 22:39:00.074 | INFO | Analyzing chunk frames = [6858752,6859264) | 20260505 22:39:00.074 | INFO | MemInfo: { | 20260505 22:39:00.074 | INFO | "gpu" : " 17 GB", | 20260505 22:39:00.074 | INFO | "huge" : "", | 20260505 22:39:00.074 | INFO | "huge_shared" : " 27 GB", | 20260505 22:39:00.074 | INFO | "malloc" : " 14 GB", | 20260505 22:39:00.074 | INFO | "pinned" : " 29 GB" | 20260505 22:39:00.074 | INFO | } | 20260505 22:39:00.075 | INFO | Perf report for the last 5.14066 seconds: | 20260505 22:39:00.075 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:39:00.075 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:00.075 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:00.075 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:00.075 | INFO | ANALYSIS : 61.2%, 3.0, 95.6ms, 0 | 20260505 22:39:00.075 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:00.075 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:00.075 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 22:39:00.075 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:00.075 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:00.075 | INFO | Uploaded 36GiB to GPU in 2.10201s (17.1265GiB/s) | 20260505 22:39:00.075 | INFO | Downloaded 34GiB to GPU in 2.04807s (16.601GiB/s) | 20260505 22:39:05.210 | INFO | Analyzing chunk frames = [6859264,6859776) | 20260505 22:39:05.210 | INFO | MemInfo: { | 20260505 22:39:05.210 | INFO | "gpu" : " 17 GB", | 20260505 22:39:05.210 | INFO | "huge" : "", | 20260505 22:39:05.210 | INFO | "huge_shared" : " 27 GB", | 20260505 22:39:05.210 | INFO | "malloc" : " 14 GB", | 20260505 22:39:05.210 | INFO | "pinned" : " 29 GB" | 20260505 22:39:05.210 | INFO | } | 20260505 22:39:05.210 | INFO | Perf report for the last 5.13521 seconds: | 20260505 22:39:05.210 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:39:05.210 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:05.210 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:05.210 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:05.210 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260505 22:39:05.210 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:05.210 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:05.210 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 22:39:05.210 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:05.210 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:05.210 | INFO | Uploaded 36GiB to GPU in 2.07228s (17.3722GiB/s) | 20260505 22:39:05.210 | INFO | Downloaded 34GiB to GPU in 2.03725s (16.6891GiB/s) | 20260505 22:39:10.350 | INFO | Analyzing chunk frames = [6859776,6860288) | 20260505 22:39:10.350 | INFO | MemInfo: { | 20260505 22:39:10.350 | INFO | "gpu" : " 17 GB", | 20260505 22:39:10.350 | INFO | "huge" : "", | 20260505 22:39:10.350 | INFO | "huge_shared" : " 27 GB", | 20260505 22:39:10.350 | INFO | "malloc" : " 14 GB", | 20260505 22:39:10.350 | INFO | "pinned" : " 29 GB" | 20260505 22:39:10.350 | INFO | } | 20260505 22:39:10.350 | INFO | Perf report for the last 5.14067 seconds: | 20260505 22:39:10.350 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:39:10.350 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:10.350 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:10.350 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:10.350 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 22:39:10.350 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:10.350 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:10.350 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 22:39:10.350 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:10.350 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:10.351 | INFO | Uploaded 36GiB to GPU in 2.14098s (16.8147GiB/s) | 20260505 22:39:10.351 | INFO | Downloaded 34GiB to GPU in 2.12951s (15.9662GiB/s) | 20260505 22:39:15.495 | INFO | Analyzing chunk frames = [6860288,6860800) | 20260505 22:39:15.495 | INFO | MemInfo: { | 20260505 22:39:15.495 | INFO | "gpu" : " 17 GB", | 20260505 22:39:15.495 | INFO | "huge" : "", | 20260505 22:39:15.495 | INFO | "huge_shared" : " 27 GB", | 20260505 22:39:15.495 | INFO | "malloc" : " 14 GB", | 20260505 22:39:15.495 | INFO | "pinned" : " 29 GB" | 20260505 22:39:15.495 | INFO | } | 20260505 22:39:15.495 | INFO | Perf report for the last 5.14456 seconds: | 20260505 22:39:15.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:39:15.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:15.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:15.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:15.495 | INFO | ANALYSIS : 62.0%, 3.0, 97.0ms, 0 | 20260505 22:39:15.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:15.495 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:15.495 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 22:39:15.495 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:15.495 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:15.495 | INFO | Uploaded 36GiB to GPU in 2.1154s (17.0181GiB/s) | 20260505 22:39:15.495 | INFO | Downloaded 34GiB to GPU in 2.01233s (16.8958GiB/s) | 20260505 22:39:15.621 | INFO | Starting a round of Baz IO | 20260505 22:39:18.773 | INFO | Populating RTMetrics report | 20260505 22:39:18.774 | INFO | RTMetrics report populated | 20260505 22:39:18.774 | INFO | Opening tmp RTMetrics json file | 20260505 22:39:18.774 | INFO | Serializing RTMetrics to json | 20260505 22:39:18.775 | INFO | Serialization done, swapping tmp file now... | 20260505 22:39:18.776 | INFO | RTMetrics done | 20260505 22:39:18.853 | INFO | Finished a round of Baz IO: Wrote 1.924 GiB in 3.232 seconds | 20260505 22:39:18.853 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.48% / 0.00% / 78.52% | 20260505 22:39:18.853 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.121133% / 3.97364e-06% / 0.000107288% | 20260505 22:39:18.853 | INFO | Baz IO overall: | 20260505 22:39:18.853 | INFO | BytesWritten : 3944447235519 | 20260505 22:39:18.853 | INFO | NumEvents : 1860053240502 | 20260505 22:39:20.642 | INFO | Analyzing chunk frames = [6860800,6861312) | 20260505 22:39:20.642 | INFO | MemInfo: { | 20260505 22:39:20.642 | INFO | "gpu" : " 17 GB", | 20260505 22:39:20.642 | INFO | "huge" : "", | 20260505 22:39:20.642 | INFO | "huge_shared" : " 27 GB", | 20260505 22:39:20.642 | INFO | "malloc" : " 14 GB", | 20260505 22:39:20.642 | INFO | "pinned" : " 29 GB" | 20260505 22:39:20.642 | INFO | } | 20260505 22:39:20.642 | INFO | Perf report for the last 5.14727 seconds: | 20260505 22:39:20.642 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:39:20.642 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:20.642 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:20.642 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:20.642 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260505 22:39:20.642 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:20.642 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260505 22:39:20.642 | INFO | PRE_HQ : 59.9%, 1.8, 54.7ms, 0 | 20260505 22:39:20.642 | INFO | BAZWRITER : 62.0%, 3.6, 114.4ms, 0 | 20260505 22:39:20.642 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 22:39:20.642 | INFO | Uploaded 36GiB to GPU in 2.16454s (16.6317GiB/s) | 20260505 22:39:20.642 | INFO | Downloaded 38GiB to GPU in 2.23746s (16.9835GiB/s) | 20260505 22:39:25.774 | INFO | Analyzing chunk frames = [6861312,6861824) | 20260505 22:39:25.774 | INFO | MemInfo: { | 20260505 22:39:25.774 | INFO | "gpu" : " 17 GB", | 20260505 22:39:25.774 | INFO | "huge" : "", | 20260505 22:39:25.774 | INFO | "huge_shared" : " 27 GB", | 20260505 22:39:25.774 | INFO | "malloc" : " 14 GB", | 20260505 22:39:25.774 | INFO | "pinned" : " 29 GB" | 20260505 22:39:25.774 | INFO | } | 20260505 22:39:25.774 | INFO | Perf report for the last 5.13162 seconds: | 20260505 22:39:25.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:39:25.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:25.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:25.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:25.774 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260505 22:39:25.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:25.774 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:25.774 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 22:39:25.774 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:25.774 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:25.774 | INFO | Uploaded 36GiB to GPU in 2.09843s (17.1557GiB/s) | 20260505 22:39:25.774 | INFO | Downloaded 34GiB to GPU in 2.03677s (16.6931GiB/s) | 20260505 22:39:30.904 | INFO | Analyzing chunk frames = [6861824,6862336) | 20260505 22:39:30.904 | INFO | MemInfo: { | 20260505 22:39:30.904 | INFO | "gpu" : " 17 GB", | 20260505 22:39:30.904 | INFO | "huge" : "", | 20260505 22:39:30.904 | INFO | "huge_shared" : " 27 GB", | 20260505 22:39:30.904 | INFO | "malloc" : " 14 GB", | 20260505 22:39:30.904 | INFO | "pinned" : " 29 GB" | 20260505 22:39:30.904 | INFO | } | 20260505 22:39:30.904 | INFO | Perf report for the last 5.12989 seconds: | 20260505 22:39:30.904 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:39:30.904 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:30.904 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:30.904 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:30.904 | INFO | ANALYSIS : 62.0%, 3.0, 96.3ms, 0 | 20260505 22:39:30.904 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:30.904 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:30.904 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 22:39:30.904 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:30.904 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:30.904 | INFO | Uploaded 36GiB to GPU in 2.15626s (16.6955GiB/s) | 20260505 22:39:30.904 | INFO | Downloaded 34GiB to GPU in 2.05747s (16.5251GiB/s) | 20260505 22:39:36.048 | INFO | Analyzing chunk frames = [6862336,6862848) | 20260505 22:39:36.048 | INFO | MemInfo: { | 20260505 22:39:36.048 | INFO | "gpu" : " 17 GB", | 20260505 22:39:36.048 | INFO | "huge" : "", | 20260505 22:39:36.048 | INFO | "huge_shared" : " 27 GB", | 20260505 22:39:36.048 | INFO | "malloc" : " 14 GB", | 20260505 22:39:36.048 | INFO | "pinned" : " 29 GB" | 20260505 22:39:36.048 | INFO | } | 20260505 22:39:36.048 | INFO | Perf report for the last 5.14413 seconds: | 20260505 22:39:36.048 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:39:36.048 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:36.048 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:36.048 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:36.048 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260505 22:39:36.048 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:36.048 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:36.048 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:39:36.048 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:36.048 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:36.048 | INFO | Uploaded 36GiB to GPU in 2.20684s (16.3129GiB/s) | 20260505 22:39:36.048 | INFO | Downloaded 34GiB to GPU in 2.0452s (16.6243GiB/s) | 20260505 22:39:41.188 | INFO | Analyzing chunk frames = [6862848,6863360) | 20260505 22:39:41.188 | INFO | MemInfo: { | 20260505 22:39:41.188 | INFO | "gpu" : " 17 GB", | 20260505 22:39:41.188 | INFO | "huge" : "", | 20260505 22:39:41.188 | INFO | "huge_shared" : " 27 GB", | 20260505 22:39:41.188 | INFO | "malloc" : " 14 GB", | 20260505 22:39:41.188 | INFO | "pinned" : " 29 GB" | 20260505 22:39:41.188 | INFO | } | 20260505 22:39:41.188 | INFO | Perf report for the last 5.13975 seconds: | 20260505 22:39:41.188 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:39:41.188 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:41.188 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:41.188 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:41.188 | INFO | ANALYSIS : 61.6%, 3.0, 96.2ms, 0 | 20260505 22:39:41.188 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:41.188 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:41.188 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:39:41.188 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:41.188 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:41.188 | INFO | Uploaded 36GiB to GPU in 2.16083s (16.6603GiB/s) | 20260505 22:39:41.188 | INFO | Downloaded 34GiB to GPU in 2.07034s (16.4224GiB/s) | 20260505 22:39:46.332 | INFO | Analyzing chunk frames = [6863360,6863872) | 20260505 22:39:46.332 | INFO | MemInfo: { | 20260505 22:39:46.332 | INFO | "gpu" : " 17 GB", | 20260505 22:39:46.332 | INFO | "huge" : "", | 20260505 22:39:46.332 | INFO | "huge_shared" : " 27 GB", | 20260505 22:39:46.332 | INFO | "malloc" : " 14 GB", | 20260505 22:39:46.332 | INFO | "pinned" : " 29 GB" | 20260505 22:39:46.332 | INFO | } | 20260505 22:39:46.332 | INFO | Perf report for the last 5.14442 seconds: | 20260505 22:39:46.332 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:39:46.332 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:46.332 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:46.332 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:46.332 | INFO | ANALYSIS : 61.4%, 3.0, 96.0ms, 0 | 20260505 22:39:46.332 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:46.332 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:46.332 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:39:46.332 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:46.332 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:46.332 | INFO | Uploaded 36GiB to GPU in 2.07284s (17.3675GiB/s) | 20260505 22:39:46.332 | INFO | Downloaded 34GiB to GPU in 2.03345s (16.7203GiB/s) | 20260505 22:39:51.470 | INFO | Analyzing chunk frames = [6863872,6864384) | 20260505 22:39:51.470 | INFO | MemInfo: { | 20260505 22:39:51.470 | INFO | "gpu" : " 17 GB", | 20260505 22:39:51.470 | INFO | "huge" : "", | 20260505 22:39:51.470 | INFO | "huge_shared" : " 27 GB", | 20260505 22:39:51.470 | INFO | "malloc" : " 14 GB", | 20260505 22:39:51.470 | INFO | "pinned" : " 29 GB" | 20260505 22:39:51.470 | INFO | } | 20260505 22:39:51.470 | INFO | Perf report for the last 5.13742 seconds: | 20260505 22:39:51.470 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:39:51.470 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:51.470 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:51.470 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:51.470 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 22:39:51.470 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:51.470 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:51.470 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 22:39:51.470 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:51.470 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:51.470 | INFO | Uploaded 36GiB to GPU in 2.02078s (17.8149GiB/s) | 20260505 22:39:51.470 | INFO | Downloaded 34GiB to GPU in 2.03334s (16.7212GiB/s) | 20260505 22:39:56.605 | INFO | Analyzing chunk frames = [6864384,6864896) | 20260505 22:39:56.605 | INFO | MemInfo: { | 20260505 22:39:56.605 | INFO | "gpu" : " 17 GB", | 20260505 22:39:56.605 | INFO | "huge" : "", | 20260505 22:39:56.605 | INFO | "huge_shared" : " 27 GB", | 20260505 22:39:56.605 | INFO | "malloc" : " 14 GB", | 20260505 22:39:56.605 | INFO | "pinned" : " 29 GB" | 20260505 22:39:56.605 | INFO | } | 20260505 22:39:56.605 | INFO | Perf report for the last 5.13531 seconds: | 20260505 22:39:56.605 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:39:56.605 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:56.605 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:56.605 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:56.605 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 22:39:56.605 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:56.605 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:56.605 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 22:39:56.605 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:56.605 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:39:56.605 | INFO | Uploaded 36GiB to GPU in 2.11983s (16.9825GiB/s) | 20260505 22:39:56.605 | INFO | Downloaded 34GiB to GPU in 1.99751s (17.0212GiB/s) | 20260505 22:39:56.605 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 22:39:56.605 | INFO | Total mem usage: 15.3 GiB | 20260505 22:39:56.605 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 22:39:56.605 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.3%) | 20260505 22:39:56.605 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 22:39:56.605 | INFO | High Water Breakdown: | 20260505 22:39:56.605 | INFO | 61% : 8.63 GiB ( 90.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 22:39:56.605 | INFO | 38.9% : 5.5 GiB ( 82.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 22:39:56.605 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 22:39:56.605 | INFO | ------------------------------------------ | 20260505 22:39:56.605 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:39:56.605 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 22:39:56.605 | INFO | Total mem usage: 26.5 GiB | 20260505 22:39:56.605 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 22:39:56.605 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 22:39:56.605 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 22:39:56.605 | INFO | High Water Breakdown: | 20260505 22:39:56.605 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:39:56.605 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:39:56.605 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:39:56.605 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:39:56.605 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:39:56.605 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:39:56.605 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 22:39:56.605 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 22:39:56.605 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:39:56.605 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 22:39:56.605 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 22:39:56.605 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 22:39:56.605 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 22:39:56.605 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 22:39:56.605 | INFO | ------------------------------------------ | 20260505 22:39:56.605 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:39:56.605 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 22:39:56.605 | INFO | Total mem usage: 25.6 GiB | 20260505 22:39:56.605 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 22:39:56.606 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 22:39:56.606 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 22:39:56.606 | INFO | High Water Breakdown: | 20260505 22:39:56.606 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 22:39:56.606 | INFO | ------------------------------------------ | 20260505 22:39:56.606 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:39:56.606 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 22:39:56.606 | INFO | Total mem usage: 15.7 GiB | 20260505 22:39:56.606 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 22:39:56.606 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 22:39:56.606 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 22:39:56.606 | INFO | High Water Breakdown: | 20260505 22:39:56.606 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 22:39:56.606 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 22:39:56.606 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 22:39:56.606 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 22:39:56.606 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 22:39:56.606 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 22:39:56.606 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 22:39:56.606 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 22:39:56.606 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 22:39:56.606 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:39:56.606 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 22:39:56.606 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:39:56.606 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 22:39:56.606 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 22:39:56.606 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 22:39:56.606 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 22:39:56.606 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:39:56.606 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 22:39:56.606 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:39:56.606 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:39:56.606 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:39:56.606 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 22:39:56.606 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 22:39:56.606 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:39:56.606 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 22:39:56.606 | INFO | ------------------------------------------ | 20260505 22:39:56.606 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:39:56.606 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 22:39:56.606 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:39:56.606 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:39:56.606 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.40e+01, 1.35e+01 / 1.23e+00 | 20260505 22:39:56.606 | INFO | PerfReport: 0.29%, AggregateStats -- 9.24e-02 / 1.04e-01, 1.07e-02 / 4.99e-03 | 20260505 22:39:56.606 | INFO | PerfReport: 11.80%, Baseline -- 3.70e+00 / 4.53e+00, 5.63e-01 / 1.48e-01 | 20260505 22:39:56.606 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.53e+00, 1.91e-01 / 7.42e-02 | 20260505 22:39:56.606 | INFO | PerfReport: 16.00%, DME -- 4.01e+01 / 4.41e+01, 4.05e+00 / 1.32e+00 | 20260505 22:39:56.606 | INFO | PerfReport: 57.18%, FrameLabeling -- 1.79e+01 / 1.93e+01, 1.45e+00 / 1.02e+00 | 20260505 22:39:56.606 | INFO | PerfReport: 7.18%, PulseAccumulating -- 2.25e+00 / 2.78e+00, 4.30e-01 / 1.94e-01 | 20260505 22:39:56.606 | INFO | PerfReport: 3.16%, Metrics -- 9.90e-01 / 1.26e+00, 2.66e-01 / 9.89e-02 | 20260505 22:39:56.606 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.44e-01 / 3.96e-01, 5.79e-02 / 2.94e-02 | 20260505 22:39:56.606 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 22:39:56.606 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:39:56.606 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:39:56.606 | INFO | PerfReport: Full Execution -- 9.87e+01 / -nan, 2.49e+01 / 0.00e+00 | 20260505 22:39:56.606 | INFO | PerfReport: 22.40%, Upload -- 2.21e+01 / 0.00e+00, 7.58e+00 / 0.00e+00 | 20260505 22:39:56.606 | INFO | PerfReport: 21.83%, Download -- 2.15e+01 / 0.00e+00, 7.64e+00 / 0.00e+00 | 20260505 22:39:56.606 | INFO | PerfReport: 31.77%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 22:39:56.606 | INFO | PerfReport: 2.19%, LockUpload -- 2.16e+00 / 0.00e+00, 7.90e+00 / 0.00e+00 | 20260505 22:39:56.606 | INFO | PerfReport: 1.33%, LockDownload -- 1.32e+00 / 0.00e+00, 2.87e+00 / 0.00e+00 | 20260505 22:39:56.606 | INFO | PerfReport: 20.47%, LockCompute -- 2.02e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260505 22:39:56.606 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 2.12e-02 / 0.00e+00 | 20260505 22:39:56.736 | INFO | Starting a round of Baz IO | 20260505 22:39:59.893 | INFO | Populating RTMetrics report | 20260505 22:39:59.893 | INFO | RTMetrics report populated | 20260505 22:39:59.893 | INFO | Opening tmp RTMetrics json file | 20260505 22:39:59.894 | INFO | Serializing RTMetrics to json | 20260505 22:39:59.895 | INFO | Serialization done, swapping tmp file now... | 20260505 22:39:59.895 | INFO | RTMetrics done | 20260505 22:39:59.975 | INFO | Finished a round of Baz IO: Wrote 1.922 GiB in 3.239 seconds | 20260505 22:39:59.975 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.48% / 0.00% / 78.52% | 20260505 22:39:59.975 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.120266% / 0% / 7.94729e-05% | 20260505 22:39:59.975 | INFO | Baz IO overall: | 20260505 22:39:59.975 | INFO | BytesWritten : 3946511369269 | 20260505 22:39:59.975 | INFO | NumEvents : 1861002152793 | 20260505 22:40:01.747 | INFO | Analyzing chunk frames = [6864896,6865408) | 20260505 22:40:01.747 | INFO | MemInfo: { | 20260505 22:40:01.747 | INFO | "gpu" : " 17 GB", | 20260505 22:40:01.747 | INFO | "huge" : "", | 20260505 22:40:01.747 | INFO | "huge_shared" : " 27 GB", | 20260505 22:40:01.747 | INFO | "malloc" : " 14 GB", | 20260505 22:40:01.747 | INFO | "pinned" : " 29 GB" | 20260505 22:40:01.747 | INFO | } | 20260505 22:40:01.747 | INFO | Perf report for the last 5.14257 seconds: | 20260505 22:40:01.747 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:40:01.747 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:01.747 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:01.748 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:01.748 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 22:40:01.748 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:01.748 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260505 22:40:01.748 | INFO | PRE_HQ : 59.9%, 1.8, 54.8ms, 0 | 20260505 22:40:01.748 | INFO | BAZWRITER : 62.5%, 3.6, 115.5ms, 0 | 20260505 22:40:01.748 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 22:40:01.748 | INFO | Uploaded 36GiB to GPU in 2.16677s (16.6146GiB/s) | 20260505 22:40:01.748 | INFO | Downloaded 38GiB to GPU in 2.24537s (16.9237GiB/s) | 20260505 22:40:06.884 | INFO | Analyzing chunk frames = [6865408,6865920) | 20260505 22:40:06.884 | INFO | MemInfo: { | 20260505 22:40:06.884 | INFO | "gpu" : " 17 GB", | 20260505 22:40:06.884 | INFO | "huge" : "", | 20260505 22:40:06.884 | INFO | "huge_shared" : " 27 GB", | 20260505 22:40:06.884 | INFO | "malloc" : " 14 GB", | 20260505 22:40:06.884 | INFO | "pinned" : " 29 GB" | 20260505 22:40:06.884 | INFO | } | 20260505 22:40:06.884 | INFO | Perf report for the last 5.13685 seconds: | 20260505 22:40:06.884 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:40:06.884 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:06.884 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:06.884 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:06.884 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260505 22:40:06.884 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:06.884 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:06.884 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260505 22:40:06.884 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:06.884 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:06.885 | INFO | Uploaded 36GiB to GPU in 2.0863s (17.2554GiB/s) | 20260505 22:40:06.885 | INFO | Downloaded 34GiB to GPU in 2.02933s (16.7543GiB/s) | 20260505 22:40:12.024 | INFO | Analyzing chunk frames = [6865920,6866432) | 20260505 22:40:12.024 | INFO | MemInfo: { | 20260505 22:40:12.024 | INFO | "gpu" : " 17 GB", | 20260505 22:40:12.024 | INFO | "huge" : "", | 20260505 22:40:12.024 | INFO | "huge_shared" : " 27 GB", | 20260505 22:40:12.024 | INFO | "malloc" : " 14 GB", | 20260505 22:40:12.024 | INFO | "pinned" : " 29 GB" | 20260505 22:40:12.024 | INFO | } | 20260505 22:40:12.024 | INFO | Perf report for the last 5.14006 seconds: | 20260505 22:40:12.025 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:40:12.025 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:12.025 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:12.025 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:12.025 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260505 22:40:12.025 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:12.025 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:12.025 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:40:12.025 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:12.025 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:12.025 | INFO | Uploaded 36GiB to GPU in 2.14877s (16.7538GiB/s) | 20260505 22:40:12.025 | INFO | Downloaded 34GiB to GPU in 2.05644s (16.5334GiB/s) | 20260505 22:40:17.163 | INFO | Analyzing chunk frames = [6866432,6866944) | 20260505 22:40:17.163 | INFO | MemInfo: { | 20260505 22:40:17.163 | INFO | "gpu" : " 17 GB", | 20260505 22:40:17.163 | INFO | "huge" : "", | 20260505 22:40:17.163 | INFO | "huge_shared" : " 27 GB", | 20260505 22:40:17.163 | INFO | "malloc" : " 14 GB", | 20260505 22:40:17.163 | INFO | "pinned" : " 29 GB" | 20260505 22:40:17.163 | INFO | } | 20260505 22:40:17.163 | INFO | Perf report for the last 5.13841 seconds: | 20260505 22:40:17.163 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:40:17.163 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:17.163 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:17.163 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:17.163 | INFO | ANALYSIS : 61.8%, 3.0, 95.9ms, 0 | 20260505 22:40:17.163 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:17.163 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:17.163 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 22:40:17.163 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:17.163 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:17.163 | INFO | Uploaded 36GiB to GPU in 2.13032s (16.8989GiB/s) | 20260505 22:40:17.163 | INFO | Downloaded 34GiB to GPU in 2.02607s (16.7813GiB/s) | 20260505 22:40:22.302 | INFO | Analyzing chunk frames = [6866944,6867456) | 20260505 22:40:22.302 | INFO | MemInfo: { | 20260505 22:40:22.302 | INFO | "gpu" : " 17 GB", | 20260505 22:40:22.302 | INFO | "huge" : "", | 20260505 22:40:22.302 | INFO | "huge_shared" : " 27 GB", | 20260505 22:40:22.302 | INFO | "malloc" : " 14 GB", | 20260505 22:40:22.302 | INFO | "pinned" : " 29 GB" | 20260505 22:40:22.302 | INFO | } | 20260505 22:40:22.302 | INFO | Perf report for the last 5.13901 seconds: | 20260505 22:40:22.302 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:40:22.302 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:22.302 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:22.302 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:22.302 | INFO | ANALYSIS : 61.2%, 3.0, 95.6ms, 0 | 20260505 22:40:22.302 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:22.302 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:22.302 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 22:40:22.302 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:22.302 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:22.302 | INFO | Uploaded 36GiB to GPU in 2.0936s (17.1952GiB/s) | 20260505 22:40:22.302 | INFO | Downloaded 34GiB to GPU in 2.05028s (16.5831GiB/s) | 20260505 22:40:27.448 | INFO | Analyzing chunk frames = [6867456,6867968) | 20260505 22:40:27.448 | INFO | MemInfo: { | 20260505 22:40:27.448 | INFO | "gpu" : " 17 GB", | 20260505 22:40:27.448 | INFO | "huge" : "", | 20260505 22:40:27.448 | INFO | "huge_shared" : " 27 GB", | 20260505 22:40:27.448 | INFO | "malloc" : " 14 GB", | 20260505 22:40:27.448 | INFO | "pinned" : " 29 GB" | 20260505 22:40:27.448 | INFO | } | 20260505 22:40:27.448 | INFO | Perf report for the last 5.14634 seconds: | 20260505 22:40:27.448 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:40:27.448 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:27.448 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:27.448 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:27.448 | INFO | ANALYSIS : 61.4%, 3.0, 96.0ms, 0 | 20260505 22:40:27.448 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:27.448 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:27.448 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:40:27.448 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:27.448 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:27.448 | INFO | Uploaded 36GiB to GPU in 2.06999s (17.3914GiB/s) | 20260505 22:40:27.448 | INFO | Downloaded 34GiB to GPU in 2.03355s (16.7196GiB/s) | 20260505 22:40:32.589 | INFO | Analyzing chunk frames = [6867968,6868480) | 20260505 22:40:32.589 | INFO | MemInfo: { | 20260505 22:40:32.589 | INFO | "gpu" : " 17 GB", | 20260505 22:40:32.589 | INFO | "huge" : "", | 20260505 22:40:32.589 | INFO | "huge_shared" : " 27 GB", | 20260505 22:40:32.589 | INFO | "malloc" : " 14 GB", | 20260505 22:40:32.589 | INFO | "pinned" : " 29 GB" | 20260505 22:40:32.589 | INFO | } | 20260505 22:40:32.589 | INFO | Perf report for the last 5.14109 seconds: | 20260505 22:40:32.589 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:40:32.589 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:32.589 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:32.589 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:32.589 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 22:40:32.589 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:32.589 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:32.589 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 22:40:32.589 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:32.589 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:32.590 | INFO | Uploaded 36GiB to GPU in 2.02925s (17.7405GiB/s) | 20260505 22:40:32.590 | INFO | Downloaded 34GiB to GPU in 2.05819s (16.5194GiB/s) | 20260505 22:40:37.720 | INFO | Analyzing chunk frames = [6868480,6868992) | 20260505 22:40:37.720 | INFO | MemInfo: { | 20260505 22:40:37.720 | INFO | "gpu" : " 17 GB", | 20260505 22:40:37.720 | INFO | "huge" : "", | 20260505 22:40:37.720 | INFO | "huge_shared" : " 27 GB", | 20260505 22:40:37.720 | INFO | "malloc" : " 14 GB", | 20260505 22:40:37.720 | INFO | "pinned" : " 29 GB" | 20260505 22:40:37.720 | INFO | } | 20260505 22:40:37.720 | INFO | Perf report for the last 5.13076 seconds: | 20260505 22:40:37.720 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:40:37.720 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:37.720 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:37.720 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:37.720 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 22:40:37.720 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:37.720 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:37.720 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 22:40:37.720 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:37.720 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:37.720 | INFO | Uploaded 36GiB to GPU in 2.11125s (17.0515GiB/s) | 20260505 22:40:37.720 | INFO | Downloaded 34GiB to GPU in 2.01452s (16.8774GiB/s) | 20260505 22:40:37.854 | INFO | Starting a round of Baz IO | 20260505 22:40:41.004 | INFO | Populating RTMetrics report | 20260505 22:40:41.004 | INFO | RTMetrics report populated | 20260505 22:40:41.004 | INFO | Opening tmp RTMetrics json file | 20260505 22:40:41.006 | INFO | Serializing RTMetrics to json | 20260505 22:40:41.008 | INFO | Serialization done, swapping tmp file now... | 20260505 22:40:41.010 | INFO | RTMetrics done | 20260505 22:40:41.090 | INFO | Finished a round of Baz IO: Wrote 1.923 GiB in 3.235 seconds | 20260505 22:40:41.090 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.48% / 0.00% / 78.52% | 20260505 22:40:41.090 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.120497% / 0% / 9.93411e-05% | 20260505 22:40:41.090 | INFO | Baz IO overall: | 20260505 22:40:41.090 | INFO | BytesWritten : 3948576182283 | 20260505 22:40:41.090 | INFO | NumEvents : 1861951669997 | 20260505 22:40:42.855 | INFO | Analyzing chunk frames = [6868992,6869504) | 20260505 22:40:42.855 | INFO | MemInfo: { | 20260505 22:40:42.855 | INFO | "gpu" : " 17 GB", | 20260505 22:40:42.855 | INFO | "huge" : "", | 20260505 22:40:42.855 | INFO | "huge_shared" : " 27 GB", | 20260505 22:40:42.855 | INFO | "malloc" : " 14 GB", | 20260505 22:40:42.855 | INFO | "pinned" : " 29 GB" | 20260505 22:40:42.855 | INFO | } | 20260505 22:40:42.855 | INFO | Perf report for the last 5.13513 seconds: | 20260505 22:40:42.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:40:42.855 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:42.855 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:42.855 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:42.855 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260505 22:40:42.855 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:42.855 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260505 22:40:42.855 | INFO | PRE_HQ : 59.5%, 1.8, 54.3ms, 0 | 20260505 22:40:42.855 | INFO | BAZWRITER : 62.5%, 3.6, 115.4ms, 0 | 20260505 22:40:42.855 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 22:40:42.855 | INFO | Uploaded 36GiB to GPU in 2.16395s (16.6362GiB/s) | 20260505 22:40:42.856 | INFO | Downloaded 38GiB to GPU in 2.24783s (16.9052GiB/s) | 20260505 22:40:48.003 | INFO | Analyzing chunk frames = [6869504,6870016) | 20260505 22:40:48.003 | INFO | MemInfo: { | 20260505 22:40:48.003 | INFO | "gpu" : " 17 GB", | 20260505 22:40:48.003 | INFO | "huge" : "", | 20260505 22:40:48.003 | INFO | "huge_shared" : " 27 GB", | 20260505 22:40:48.003 | INFO | "malloc" : " 14 GB", | 20260505 22:40:48.003 | INFO | "pinned" : " 29 GB" | 20260505 22:40:48.003 | INFO | } | 20260505 22:40:48.003 | INFO | Perf report for the last 5.14758 seconds: | 20260505 22:40:48.003 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:40:48.003 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:48.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:48.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:48.003 | INFO | ANALYSIS : 61.8%, 3.0, 96.6ms, 0 | 20260505 22:40:48.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:48.003 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:48.003 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:40:48.003 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:48.003 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:48.003 | INFO | Uploaded 36GiB to GPU in 2.0864s (17.2546GiB/s) | 20260505 22:40:48.003 | INFO | Downloaded 34GiB to GPU in 2.02741s (16.7702GiB/s) | 20260505 22:40:53.143 | INFO | Analyzing chunk frames = [6870016,6870528) | 20260505 22:40:53.143 | INFO | MemInfo: { | 20260505 22:40:53.143 | INFO | "gpu" : " 17 GB", | 20260505 22:40:53.143 | INFO | "huge" : "", | 20260505 22:40:53.143 | INFO | "huge_shared" : " 27 GB", | 20260505 22:40:53.144 | INFO | "malloc" : " 14 GB", | 20260505 22:40:53.144 | INFO | "pinned" : " 29 GB" | 20260505 22:40:53.144 | INFO | } | 20260505 22:40:53.144 | INFO | Perf report for the last 5.14075 seconds: | 20260505 22:40:53.144 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:40:53.144 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:53.144 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:53.144 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:53.144 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260505 22:40:53.144 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:53.144 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:53.144 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:40:53.144 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:53.144 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:53.144 | INFO | Uploaded 36GiB to GPU in 2.14749s (16.7638GiB/s) | 20260505 22:40:53.144 | INFO | Downloaded 34GiB to GPU in 2.05155s (16.5729GiB/s) | 20260505 22:40:58.280 | INFO | Analyzing chunk frames = [6870528,6871040) | 20260505 22:40:58.280 | INFO | MemInfo: { | 20260505 22:40:58.280 | INFO | "gpu" : " 17 GB", | 20260505 22:40:58.280 | INFO | "huge" : "", | 20260505 22:40:58.280 | INFO | "huge_shared" : " 27 GB", | 20260505 22:40:58.280 | INFO | "malloc" : " 14 GB", | 20260505 22:40:58.280 | INFO | "pinned" : " 29 GB" | 20260505 22:40:58.280 | INFO | } | 20260505 22:40:58.280 | INFO | Perf report for the last 5.13625 seconds: | 20260505 22:40:58.280 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:40:58.280 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:58.280 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:58.280 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:58.280 | INFO | ANALYSIS : 61.8%, 3.0, 95.9ms, 0 | 20260505 22:40:58.280 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:58.280 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:58.280 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260505 22:40:58.280 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:58.280 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:40:58.280 | INFO | Uploaded 36GiB to GPU in 2.13949s (16.8264GiB/s) | 20260505 22:40:58.280 | INFO | Downloaded 34GiB to GPU in 2.02039s (16.8284GiB/s) | 20260505 22:41:03.416 | INFO | Analyzing chunk frames = [6871040,6871552) | 20260505 22:41:03.416 | INFO | MemInfo: { | 20260505 22:41:03.416 | INFO | "gpu" : " 17 GB", | 20260505 22:41:03.416 | INFO | "huge" : "", | 20260505 22:41:03.416 | INFO | "huge_shared" : " 27 GB", | 20260505 22:41:03.416 | INFO | "malloc" : " 14 GB", | 20260505 22:41:03.416 | INFO | "pinned" : " 29 GB" | 20260505 22:41:03.416 | INFO | } | 20260505 22:41:03.416 | INFO | Perf report for the last 5.13619 seconds: | 20260505 22:41:03.416 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:41:03.416 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:03.416 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:03.416 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:03.416 | INFO | ANALYSIS : 61.5%, 3.0, 96.0ms, 0 | 20260505 22:41:03.416 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:03.416 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:03.416 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 22:41:03.416 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:03.416 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:03.416 | INFO | Uploaded 36GiB to GPU in 2.14325s (16.7969GiB/s) | 20260505 22:41:03.416 | INFO | Downloaded 34GiB to GPU in 2.06518s (16.4634GiB/s) | 20260505 22:41:08.559 | INFO | Analyzing chunk frames = [6871552,6872064) | 20260505 22:41:08.559 | INFO | MemInfo: { | 20260505 22:41:08.559 | INFO | "gpu" : " 17 GB", | 20260505 22:41:08.559 | INFO | "huge" : "", | 20260505 22:41:08.559 | INFO | "huge_shared" : " 27 GB", | 20260505 22:41:08.559 | INFO | "malloc" : " 14 GB", | 20260505 22:41:08.559 | INFO | "pinned" : " 29 GB" | 20260505 22:41:08.559 | INFO | } | 20260505 22:41:08.559 | INFO | Perf report for the last 5.14282 seconds: | 20260505 22:41:08.559 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:41:08.559 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:08.559 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:08.559 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:08.559 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260505 22:41:08.559 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:08.559 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:08.559 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260505 22:41:08.559 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:08.559 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:08.559 | INFO | Uploaded 36GiB to GPU in 2.08662s (17.2528GiB/s) | 20260505 22:41:08.559 | INFO | Downloaded 34GiB to GPU in 2.0374s (16.6879GiB/s) | 20260505 22:41:13.699 | INFO | Analyzing chunk frames = [6872064,6872576) | 20260505 22:41:13.699 | INFO | MemInfo: { | 20260505 22:41:13.699 | INFO | "gpu" : " 17 GB", | 20260505 22:41:13.699 | INFO | "huge" : "", | 20260505 22:41:13.699 | INFO | "huge_shared" : " 27 GB", | 20260505 22:41:13.699 | INFO | "malloc" : " 14 GB", | 20260505 22:41:13.699 | INFO | "pinned" : " 29 GB" | 20260505 22:41:13.699 | INFO | } | 20260505 22:41:13.699 | INFO | Perf report for the last 5.14045 seconds: | 20260505 22:41:13.699 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:41:13.699 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:13.699 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:13.699 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:13.699 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 22:41:13.699 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:13.699 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:13.699 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 22:41:13.699 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:13.699 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:13.700 | INFO | Uploaded 36GiB to GPU in 2.12245s (16.9615GiB/s) | 20260505 22:41:13.700 | INFO | Downloaded 34GiB to GPU in 2.10121s (16.1812GiB/s) | 20260505 22:41:18.838 | INFO | Analyzing chunk frames = [6872576,6873088) | 20260505 22:41:18.838 | INFO | MemInfo: { | 20260505 22:41:18.838 | INFO | "gpu" : " 17 GB", | 20260505 22:41:18.838 | INFO | "huge" : "", | 20260505 22:41:18.838 | INFO | "huge_shared" : " 27 GB", | 20260505 22:41:18.838 | INFO | "malloc" : " 14 GB", | 20260505 22:41:18.838 | INFO | "pinned" : " 29 GB" | 20260505 22:41:18.838 | INFO | } | 20260505 22:41:18.838 | INFO | Perf report for the last 5.13905 seconds: | 20260505 22:41:18.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:41:18.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:18.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:18.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:18.838 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260505 22:41:18.838 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:18.838 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:18.838 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:41:18.838 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:18.838 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:18.839 | INFO | Uploaded 36GiB to GPU in 2.10941s (17.0664GiB/s) | 20260505 22:41:18.839 | INFO | Downloaded 34GiB to GPU in 2.00737s (16.9376GiB/s) | 20260505 22:41:18.839 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 22:41:18.839 | INFO | Total mem usage: 15.3 GiB | 20260505 22:41:18.839 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 22:41:18.839 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -9.04%) | 20260505 22:41:18.839 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 22:41:18.839 | INFO | High Water Breakdown: | 20260505 22:41:18.839 | INFO | 60.9% : 8.56 GiB ( 90% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 22:41:18.839 | INFO | 39% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 22:41:18.839 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 22:41:18.839 | INFO | ------------------------------------------ | 20260505 22:41:18.839 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:41:18.839 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 22:41:18.839 | INFO | Total mem usage: 26.5 GiB | 20260505 22:41:18.839 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 22:41:18.839 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 22:41:18.839 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 22:41:18.839 | INFO | High Water Breakdown: | 20260505 22:41:18.839 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:41:18.839 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:41:18.839 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:41:18.839 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:41:18.839 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:41:18.839 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:41:18.839 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 22:41:18.839 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 22:41:18.839 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:41:18.839 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 22:41:18.839 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 22:41:18.839 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 22:41:18.839 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 22:41:18.839 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 22:41:18.839 | INFO | ------------------------------------------ | 20260505 22:41:18.839 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:41:18.839 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 22:41:18.839 | INFO | Total mem usage: 25.6 GiB | 20260505 22:41:18.839 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 22:41:18.839 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 22:41:18.839 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 22:41:18.839 | INFO | High Water Breakdown: | 20260505 22:41:18.839 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 22:41:18.839 | INFO | ------------------------------------------ | 20260505 22:41:18.839 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:41:18.839 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 22:41:18.839 | INFO | Total mem usage: 15.7 GiB | 20260505 22:41:18.839 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 22:41:18.839 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 22:41:18.839 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 22:41:18.839 | INFO | High Water Breakdown: | 20260505 22:41:18.839 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 22:41:18.839 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 22:41:18.839 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 22:41:18.839 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 22:41:18.839 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 22:41:18.839 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 22:41:18.839 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 22:41:18.839 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 22:41:18.839 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 22:41:18.839 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:41:18.839 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 22:41:18.839 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:41:18.839 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 22:41:18.839 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 22:41:18.839 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 22:41:18.839 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 22:41:18.839 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:41:18.839 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 22:41:18.839 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:41:18.839 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:41:18.839 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:41:18.840 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 22:41:18.840 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 22:41:18.840 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:41:18.840 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 22:41:18.840 | INFO | ------------------------------------------ | 20260505 22:41:18.840 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:41:18.840 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 22:41:18.840 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:41:18.840 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:41:18.840 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.40e+01, 1.35e+01 / 8.83e-01 | 20260505 22:41:18.840 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.04e-01, 1.20e-02 / 3.07e-03 | 20260505 22:41:18.840 | INFO | PerfReport: 11.80%, Baseline -- 3.70e+00 / 4.51e+00, 5.71e-01 / 2.01e-01 | 20260505 22:41:18.840 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.53e+00, 1.89e-01 / 8.49e-02 | 20260505 22:41:18.840 | INFO | PerfReport: 15.98%, DME -- 4.01e+01 / 4.40e+01, 3.74e+00 / 9.50e-01 | 20260505 22:41:18.840 | INFO | PerfReport: 57.26%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.51e+00 / 9.48e-01 | 20260505 22:41:18.840 | INFO | PerfReport: 7.15%, PulseAccumulating -- 2.24e+00 / 2.78e+00, 4.29e-01 / 1.84e-01 | 20260505 22:41:18.840 | INFO | PerfReport: 3.14%, Metrics -- 9.86e-01 / 1.28e+00, 2.66e-01 / 1.05e-01 | 20260505 22:41:18.840 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.43e-01 / 4.03e-01, 5.79e-02 / 3.10e-02 | 20260505 22:41:18.840 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 22:41:18.840 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:41:18.840 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:41:18.840 | INFO | PerfReport: Full Execution -- 9.87e+01 / -nan, 2.49e+01 / 0.00e+00 | 20260505 22:41:18.840 | INFO | PerfReport: 22.33%, Upload -- 2.20e+01 / 0.00e+00, 7.55e+00 / 0.00e+00 | 20260505 22:41:18.840 | INFO | PerfReport: 21.82%, Download -- 2.15e+01 / 0.00e+00, 7.62e+00 / 0.00e+00 | 20260505 22:41:18.840 | INFO | PerfReport: 31.82%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 22:41:18.840 | INFO | PerfReport: 2.16%, LockUpload -- 2.13e+00 / 0.00e+00, 7.92e+00 / 0.00e+00 | 20260505 22:41:18.840 | INFO | PerfReport: 1.30%, LockDownload -- 1.28e+00 / 0.00e+00, 2.89e+00 / 0.00e+00 | 20260505 22:41:18.840 | INFO | PerfReport: 20.56%, LockCompute -- 2.03e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260505 22:41:18.840 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.84e-02 / 0.00e+00 | 20260505 22:41:18.966 | INFO | Starting a round of Baz IO | 20260505 22:41:22.111 | INFO | Populating RTMetrics report | 20260505 22:41:22.112 | INFO | RTMetrics report populated | 20260505 22:41:22.112 | INFO | Opening tmp RTMetrics json file | 20260505 22:41:22.113 | INFO | Serializing RTMetrics to json | 20260505 22:41:22.114 | INFO | Serialization done, swapping tmp file now... | 20260505 22:41:22.115 | INFO | RTMetrics done | 20260505 22:41:22.197 | INFO | Finished a round of Baz IO: Wrote 1.92 GiB in 3.23 seconds | 20260505 22:41:22.197 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.48% / 0.00% / 78.52% | 20260505 22:41:22.197 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.119789% / 7.94729e-06% / 8.74201e-05% | 20260505 22:41:22.197 | INFO | Baz IO overall: | 20260505 22:41:22.197 | INFO | BytesWritten : 3950637827459 | 20260505 22:41:22.197 | INFO | NumEvents : 1862898912188 | 20260505 22:41:23.975 | INFO | Analyzing chunk frames = [6873088,6873600) | 20260505 22:41:23.975 | INFO | MemInfo: { | 20260505 22:41:23.975 | INFO | "gpu" : " 17 GB", | 20260505 22:41:23.975 | INFO | "huge" : "", | 20260505 22:41:23.975 | INFO | "huge_shared" : " 27 GB", | 20260505 22:41:23.975 | INFO | "malloc" : " 14 GB", | 20260505 22:41:23.975 | INFO | "pinned" : " 29 GB" | 20260505 22:41:23.975 | INFO | } | 20260505 22:41:23.975 | INFO | Perf report for the last 5.13655 seconds: | 20260505 22:41:23.975 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:41:23.975 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:23.975 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:23.975 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:23.975 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 22:41:23.975 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:23.975 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260505 22:41:23.975 | INFO | PRE_HQ : 59.7%, 1.8, 54.3ms, 0 | 20260505 22:41:23.975 | INFO | BAZWRITER : 62.2%, 3.6, 116.1ms, 0 | 20260505 22:41:23.975 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 22:41:23.975 | INFO | Uploaded 36GiB to GPU in 2.14707s (16.767GiB/s) | 20260505 22:41:23.975 | INFO | Downloaded 38GiB to GPU in 2.23645s (16.9912GiB/s) | 20260505 22:41:29.111 | INFO | Analyzing chunk frames = [6873600,6874112) | 20260505 22:41:29.111 | INFO | MemInfo: { | 20260505 22:41:29.111 | INFO | "gpu" : " 17 GB", | 20260505 22:41:29.111 | INFO | "huge" : "", | 20260505 22:41:29.111 | INFO | "huge_shared" : " 27 GB", | 20260505 22:41:29.111 | INFO | "malloc" : " 14 GB", | 20260505 22:41:29.111 | INFO | "pinned" : " 29 GB" | 20260505 22:41:29.111 | INFO | } | 20260505 22:41:29.111 | INFO | Perf report for the last 5.1365 seconds: | 20260505 22:41:29.111 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:41:29.111 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:29.112 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:29.112 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:29.112 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 22:41:29.112 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:29.112 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:29.112 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260505 22:41:29.112 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:29.112 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:29.112 | INFO | Uploaded 36GiB to GPU in 2.08527s (17.264GiB/s) | 20260505 22:41:29.112 | INFO | Downloaded 34GiB to GPU in 2.02512s (16.7891GiB/s) | 20260505 22:41:34.259 | INFO | Analyzing chunk frames = [6874112,6874624) | 20260505 22:41:34.259 | INFO | MemInfo: { | 20260505 22:41:34.259 | INFO | "gpu" : " 17 GB", | 20260505 22:41:34.259 | INFO | "huge" : "", | 20260505 22:41:34.259 | INFO | "huge_shared" : " 27 GB", | 20260505 22:41:34.260 | INFO | "malloc" : " 14 GB", | 20260505 22:41:34.260 | INFO | "pinned" : " 29 GB" | 20260505 22:41:34.260 | INFO | } | 20260505 22:41:34.260 | INFO | Perf report for the last 5.14808 seconds: | 20260505 22:41:34.260 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:41:34.260 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:34.260 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:34.260 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:34.260 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260505 22:41:34.260 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:34.260 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:34.260 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 22:41:34.260 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:34.260 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:34.260 | INFO | Uploaded 36GiB to GPU in 2.13672s (16.8483GiB/s) | 20260505 22:41:34.260 | INFO | Downloaded 34GiB to GPU in 2.02456s (16.7938GiB/s) | 20260505 22:41:39.396 | INFO | Analyzing chunk frames = [6874624,6875136) | 20260505 22:41:39.396 | INFO | MemInfo: { | 20260505 22:41:39.396 | INFO | "gpu" : " 17 GB", | 20260505 22:41:39.396 | INFO | "huge" : "", | 20260505 22:41:39.396 | INFO | "huge_shared" : " 27 GB", | 20260505 22:41:39.396 | INFO | "malloc" : " 14 GB", | 20260505 22:41:39.396 | INFO | "pinned" : " 29 GB" | 20260505 22:41:39.396 | INFO | } | 20260505 22:41:39.396 | INFO | Perf report for the last 5.13688 seconds: | 20260505 22:41:39.396 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:41:39.396 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:39.396 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:39.396 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:39.397 | INFO | ANALYSIS : 61.8%, 3.0, 95.9ms, 0 | 20260505 22:41:39.397 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:39.397 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:39.397 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:41:39.397 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:39.397 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:39.397 | INFO | Uploaded 36GiB to GPU in 2.1457s (16.7777GiB/s) | 20260505 22:41:39.397 | INFO | Downloaded 34GiB to GPU in 2.02234s (16.8122GiB/s) | 20260505 22:41:44.530 | INFO | Analyzing chunk frames = [6875136,6875648) | 20260505 22:41:44.530 | INFO | MemInfo: { | 20260505 22:41:44.530 | INFO | "gpu" : " 17 GB", | 20260505 22:41:44.530 | INFO | "huge" : "", | 20260505 22:41:44.530 | INFO | "huge_shared" : " 27 GB", | 20260505 22:41:44.530 | INFO | "malloc" : " 14 GB", | 20260505 22:41:44.530 | INFO | "pinned" : " 29 GB" | 20260505 22:41:44.530 | INFO | } | 20260505 22:41:44.530 | INFO | Perf report for the last 5.1333 seconds: | 20260505 22:41:44.530 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:41:44.530 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:44.530 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:44.530 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:44.530 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260505 22:41:44.530 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:44.530 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:44.530 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:41:44.530 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:44.530 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:44.530 | INFO | Uploaded 36GiB to GPU in 2.09273s (17.2024GiB/s) | 20260505 22:41:44.530 | INFO | Downloaded 34GiB to GPU in 2.0399s (16.6675GiB/s) | 20260505 22:41:49.676 | INFO | Analyzing chunk frames = [6875648,6876160) | 20260505 22:41:49.676 | INFO | MemInfo: { | 20260505 22:41:49.676 | INFO | "gpu" : " 17 GB", | 20260505 22:41:49.676 | INFO | "huge" : "", | 20260505 22:41:49.676 | INFO | "huge_shared" : " 27 GB", | 20260505 22:41:49.676 | INFO | "malloc" : " 14 GB", | 20260505 22:41:49.676 | INFO | "pinned" : " 29 GB" | 20260505 22:41:49.676 | INFO | } | 20260505 22:41:49.676 | INFO | Perf report for the last 5.14654 seconds: | 20260505 22:41:49.676 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:41:49.676 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:49.676 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:49.676 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:49.676 | INFO | ANALYSIS : 61.5%, 3.0, 96.2ms, 0 | 20260505 22:41:49.676 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:49.676 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:49.676 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 22:41:49.676 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:49.676 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:49.677 | INFO | Uploaded 36GiB to GPU in 2.08327s (17.2805GiB/s) | 20260505 22:41:49.677 | INFO | Downloaded 34GiB to GPU in 2.03878s (16.6767GiB/s) | 20260505 22:41:54.811 | INFO | Analyzing chunk frames = [6876160,6876672) | 20260505 22:41:54.811 | INFO | MemInfo: { | 20260505 22:41:54.811 | INFO | "gpu" : " 17 GB", | 20260505 22:41:54.811 | INFO | "huge" : "", | 20260505 22:41:54.811 | INFO | "huge_shared" : " 27 GB", | 20260505 22:41:54.811 | INFO | "malloc" : " 14 GB", | 20260505 22:41:54.812 | INFO | "pinned" : " 29 GB" | 20260505 22:41:54.812 | INFO | } | 20260505 22:41:54.812 | INFO | Perf report for the last 5.1353 seconds: | 20260505 22:41:54.812 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:41:54.812 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:54.812 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:54.812 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:54.812 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260505 22:41:54.812 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:54.812 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:54.812 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 22:41:54.812 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:54.812 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:54.812 | INFO | Uploaded 36GiB to GPU in 2.03281s (17.7095GiB/s) | 20260505 22:41:54.812 | INFO | Downloaded 34GiB to GPU in 2.04463s (16.6289GiB/s) | 20260505 22:41:59.954 | INFO | Analyzing chunk frames = [6876672,6877184) | 20260505 22:41:59.954 | INFO | MemInfo: { | 20260505 22:41:59.954 | INFO | "gpu" : " 17 GB", | 20260505 22:41:59.954 | INFO | "huge" : "", | 20260505 22:41:59.954 | INFO | "huge_shared" : " 27 GB", | 20260505 22:41:59.954 | INFO | "malloc" : " 14 GB", | 20260505 22:41:59.954 | INFO | "pinned" : " 29 GB" | 20260505 22:41:59.954 | INFO | } | 20260505 22:41:59.954 | INFO | Perf report for the last 5.14285 seconds: | 20260505 22:41:59.954 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:41:59.954 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:59.954 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:59.954 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:59.955 | INFO | ANALYSIS : 61.9%, 3.0, 96.8ms, 0 | 20260505 22:41:59.955 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:59.955 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:59.955 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:41:59.955 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:59.955 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:41:59.955 | INFO | Uploaded 36GiB to GPU in 2.11607s (17.0127GiB/s) | 20260505 22:41:59.955 | INFO | Downloaded 34GiB to GPU in 1.99784s (17.0183GiB/s) | 20260505 22:42:00.074 | INFO | Starting a round of Baz IO | 20260505 22:42:03.229 | INFO | Populating RTMetrics report | 20260505 22:42:03.230 | INFO | RTMetrics report populated | 20260505 22:42:03.230 | INFO | Opening tmp RTMetrics json file | 20260505 22:42:03.230 | INFO | Serializing RTMetrics to json | 20260505 22:42:03.232 | INFO | Serialization done, swapping tmp file now... | 20260505 22:42:03.232 | INFO | RTMetrics done | 20260505 22:42:03.314 | INFO | Finished a round of Baz IO: Wrote 1.92 GiB in 3.24 seconds | 20260505 22:42:03.314 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.48% / 0.00% / 78.52% | 20260505 22:42:03.314 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.11994% / 0% / 8.74201e-05% | 20260505 22:42:03.314 | INFO | Baz IO overall: | 20260505 22:42:03.314 | INFO | BytesWritten : 3952699009218 | 20260505 22:42:03.314 | INFO | NumEvents : 1863845848894 | 20260505 22:42:05.095 | INFO | Analyzing chunk frames = [6877184,6877696) | 20260505 22:42:05.095 | INFO | MemInfo: { | 20260505 22:42:05.095 | INFO | "gpu" : " 17 GB", | 20260505 22:42:05.095 | INFO | "huge" : "", | 20260505 22:42:05.095 | INFO | "huge_shared" : " 27 GB", | 20260505 22:42:05.095 | INFO | "malloc" : " 14 GB", | 20260505 22:42:05.095 | INFO | "pinned" : " 29 GB" | 20260505 22:42:05.095 | INFO | } | 20260505 22:42:05.095 | INFO | Perf report for the last 5.14079 seconds: | 20260505 22:42:05.095 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:42:05.095 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:05.095 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:05.095 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:05.095 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 22:42:05.095 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:05.095 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260505 22:42:05.095 | INFO | PRE_HQ : 59.8%, 1.8, 54.5ms, 0 | 20260505 22:42:05.095 | INFO | BAZWRITER : 62.2%, 3.6, 115.5ms, 0 | 20260505 22:42:05.095 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 22:42:05.096 | INFO | Uploaded 36GiB to GPU in 2.15494s (16.7058GiB/s) | 20260505 22:42:05.096 | INFO | Downloaded 38GiB to GPU in 2.23618s (16.9932GiB/s) | 20260505 22:42:10.229 | INFO | Analyzing chunk frames = [6877696,6878208) | 20260505 22:42:10.229 | INFO | MemInfo: { | 20260505 22:42:10.229 | INFO | "gpu" : " 17 GB", | 20260505 22:42:10.229 | INFO | "huge" : "", | 20260505 22:42:10.229 | INFO | "huge_shared" : " 27 GB", | 20260505 22:42:10.233 | INFO | "malloc" : " 14 GB", | 20260505 22:42:10.233 | INFO | "pinned" : " 29 GB" | 20260505 22:42:10.233 | INFO | } | 20260505 22:42:10.233 | INFO | Perf report for the last 5.13748 seconds: | 20260505 22:42:10.233 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:42:10.233 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:10.233 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:10.233 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:10.233 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260505 22:42:10.233 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:10.233 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:10.233 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 22:42:10.233 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:10.233 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:10.233 | INFO | Uploaded 36GiB to GPU in 2.08343s (17.2792GiB/s) | 20260505 22:42:10.233 | INFO | Downloaded 34GiB to GPU in 2.03782s (16.6845GiB/s) | 20260505 22:42:15.364 | INFO | Analyzing chunk frames = [6878208,6878720) | 20260505 22:42:15.364 | INFO | MemInfo: { | 20260505 22:42:15.364 | INFO | "gpu" : " 17 GB", | 20260505 22:42:15.364 | INFO | "huge" : "", | 20260505 22:42:15.364 | INFO | "huge_shared" : " 27 GB", | 20260505 22:42:15.364 | INFO | "malloc" : " 14 GB", | 20260505 22:42:15.364 | INFO | "pinned" : " 29 GB" | 20260505 22:42:15.364 | INFO | } | 20260505 22:42:15.364 | INFO | Perf report for the last 5.13149 seconds: | 20260505 22:42:15.364 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:42:15.364 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:15.364 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:15.364 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:15.364 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260505 22:42:15.364 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:15.364 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:15.364 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 22:42:15.364 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:15.364 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:15.364 | INFO | Uploaded 36GiB to GPU in 2.1479s (16.7606GiB/s) | 20260505 22:42:15.365 | INFO | Downloaded 34GiB to GPU in 2.03573s (16.7016GiB/s) | 20260505 22:42:20.508 | INFO | Analyzing chunk frames = [6878720,6879232) | 20260505 22:42:20.508 | INFO | MemInfo: { | 20260505 22:42:20.508 | INFO | "gpu" : " 17 GB", | 20260505 22:42:20.508 | INFO | "huge" : "", | 20260505 22:42:20.508 | INFO | "huge_shared" : " 27 GB", | 20260505 22:42:20.508 | INFO | "malloc" : " 14 GB", | 20260505 22:42:20.508 | INFO | "pinned" : " 29 GB" | 20260505 22:42:20.508 | INFO | } | 20260505 22:42:20.508 | INFO | Perf report for the last 5.14406 seconds: | 20260505 22:42:20.508 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:42:20.508 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:20.508 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:20.508 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:20.508 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260505 22:42:20.508 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:20.508 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:20.508 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:42:20.508 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:20.508 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:20.509 | INFO | Uploaded 36GiB to GPU in 2.20559s (16.3222GiB/s) | 20260505 22:42:20.509 | INFO | Downloaded 34GiB to GPU in 2.07385s (16.3947GiB/s) | 20260505 22:42:25.656 | INFO | Analyzing chunk frames = [6879232,6879744) | 20260505 22:42:25.656 | INFO | MemInfo: { | 20260505 22:42:25.656 | INFO | "gpu" : " 17 GB", | 20260505 22:42:25.656 | INFO | "huge" : "", | 20260505 22:42:25.656 | INFO | "huge_shared" : " 27 GB", | 20260505 22:42:25.656 | INFO | "malloc" : " 14 GB", | 20260505 22:42:25.656 | INFO | "pinned" : " 29 GB" | 20260505 22:42:25.656 | INFO | } | 20260505 22:42:25.656 | INFO | Perf report for the last 5.14777 seconds: | 20260505 22:42:25.656 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:42:25.656 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:25.656 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:25.656 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:25.656 | INFO | ANALYSIS : 61.2%, 3.0, 95.8ms, 0 | 20260505 22:42:25.656 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:25.656 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:25.656 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:42:25.656 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:25.656 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:25.656 | INFO | Uploaded 36GiB to GPU in 2.09814s (17.1581GiB/s) | 20260505 22:42:25.656 | INFO | Downloaded 34GiB to GPU in 2.05002s (16.5852GiB/s) | 20260505 22:42:30.794 | INFO | Analyzing chunk frames = [6879744,6880256) | 20260505 22:42:30.794 | INFO | MemInfo: { | 20260505 22:42:30.794 | INFO | "gpu" : " 17 GB", | 20260505 22:42:30.794 | INFO | "huge" : "", | 20260505 22:42:30.794 | INFO | "huge_shared" : " 27 GB", | 20260505 22:42:30.794 | INFO | "malloc" : " 14 GB", | 20260505 22:42:30.794 | INFO | "pinned" : " 29 GB" | 20260505 22:42:30.794 | INFO | } | 20260505 22:42:30.794 | INFO | Perf report for the last 5.13794 seconds: | 20260505 22:42:30.794 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:42:30.794 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:30.794 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:30.794 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:30.795 | INFO | ANALYSIS : 61.5%, 3.0, 96.0ms, 0 | 20260505 22:42:30.795 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:30.795 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:30.795 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 22:42:30.795 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:30.795 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:30.795 | INFO | Uploaded 36GiB to GPU in 2.07811s (17.3234GiB/s) | 20260505 22:42:30.795 | INFO | Downloaded 34GiB to GPU in 2.04216s (16.649GiB/s) | 20260505 22:42:35.924 | INFO | Analyzing chunk frames = [6880256,6880768) | 20260505 22:42:35.924 | INFO | MemInfo: { | 20260505 22:42:35.924 | INFO | "gpu" : " 17 GB", | 20260505 22:42:35.924 | INFO | "huge" : "", | 20260505 22:42:35.924 | INFO | "huge_shared" : " 27 GB", | 20260505 22:42:35.924 | INFO | "malloc" : " 14 GB", | 20260505 22:42:35.924 | INFO | "pinned" : " 29 GB" | 20260505 22:42:35.924 | INFO | } | 20260505 22:42:35.924 | INFO | Perf report for the last 5.13016 seconds: | 20260505 22:42:35.924 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:42:35.924 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:35.924 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:35.924 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:35.924 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260505 22:42:35.924 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:35.924 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:35.924 | INFO | PRE_HQ : 27.5%, 1.0, 14.2ms, 0 | 20260505 22:42:35.924 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:35.924 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:35.924 | INFO | Uploaded 36GiB to GPU in 2.0714s (17.3795GiB/s) | 20260505 22:42:35.925 | INFO | Downloaded 34GiB to GPU in 2.06707s (16.4484GiB/s) | 20260505 22:42:41.064 | INFO | Analyzing chunk frames = [6880768,6881280) | 20260505 22:42:41.064 | INFO | MemInfo: { | 20260505 22:42:41.064 | INFO | "gpu" : " 17 GB", | 20260505 22:42:41.064 | INFO | "huge" : "", | 20260505 22:42:41.064 | INFO | "huge_shared" : " 27 GB", | 20260505 22:42:41.064 | INFO | "malloc" : " 14 GB", | 20260505 22:42:41.064 | INFO | "pinned" : " 29 GB" | 20260505 22:42:41.064 | INFO | } | 20260505 22:42:41.064 | INFO | Perf report for the last 5.13958 seconds: | 20260505 22:42:41.064 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:42:41.064 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:41.064 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:41.064 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:41.064 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 22:42:41.064 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:41.064 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:41.064 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 22:42:41.064 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:41.064 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:41.064 | INFO | Uploaded 36GiB to GPU in 2.1036s (17.1135GiB/s) | 20260505 22:42:41.064 | INFO | Downloaded 34GiB to GPU in 2.00048s (16.9959GiB/s) | 20260505 22:42:41.064 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 22:42:41.064 | INFO | Total mem usage: 15.3 GiB | 20260505 22:42:41.064 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 22:42:41.064 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -9.04%) | 20260505 22:42:41.064 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 22:42:41.064 | INFO | High Water Breakdown: | 20260505 22:42:41.064 | INFO | 60.9% : 8.56 GiB ( 90% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 22:42:41.064 | INFO | 39% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 22:42:41.064 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 22:42:41.064 | INFO | ------------------------------------------ | 20260505 22:42:41.064 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:42:41.064 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 22:42:41.064 | INFO | Total mem usage: 26.5 GiB | 20260505 22:42:41.064 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 22:42:41.064 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 22:42:41.064 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 22:42:41.064 | INFO | High Water Breakdown: | 20260505 22:42:41.064 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:42:41.064 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:42:41.064 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:42:41.064 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:42:41.064 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:42:41.064 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:42:41.064 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 22:42:41.064 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 22:42:41.065 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:42:41.065 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 22:42:41.065 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 22:42:41.065 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 22:42:41.065 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 22:42:41.065 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 22:42:41.065 | INFO | ------------------------------------------ | 20260505 22:42:41.065 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:42:41.065 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 22:42:41.065 | INFO | Total mem usage: 25.6 GiB | 20260505 22:42:41.065 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 22:42:41.065 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 22:42:41.065 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 22:42:41.065 | INFO | High Water Breakdown: | 20260505 22:42:41.065 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 22:42:41.065 | INFO | ------------------------------------------ | 20260505 22:42:41.065 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:42:41.065 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 22:42:41.065 | INFO | Total mem usage: 15.7 GiB | 20260505 22:42:41.065 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 22:42:41.065 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 22:42:41.065 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 22:42:41.065 | INFO | High Water Breakdown: | 20260505 22:42:41.065 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 22:42:41.065 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 22:42:41.065 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 22:42:41.065 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 22:42:41.065 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 22:42:41.065 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 22:42:41.065 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 22:42:41.065 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 22:42:41.065 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 22:42:41.065 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:42:41.065 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 22:42:41.065 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:42:41.065 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 22:42:41.065 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 22:42:41.065 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 22:42:41.065 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 22:42:41.065 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:42:41.065 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 22:42:41.065 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:42:41.065 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:42:41.065 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:42:41.065 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 22:42:41.065 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 22:42:41.065 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:42:41.065 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 22:42:41.065 | INFO | ------------------------------------------ | 20260505 22:42:41.065 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:42:41.065 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 22:42:41.065 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:42:41.065 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:42:41.065 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.41e+01, 1.34e+01 / 9.15e-01 | 20260505 22:42:41.065 | INFO | PerfReport: 0.29%, AggregateStats -- 9.17e-02 / 1.04e-01, 1.08e-02 / 4.70e-03 | 20260505 22:42:41.065 | INFO | PerfReport: 11.80%, Baseline -- 3.70e+00 / 4.49e+00, 5.66e-01 / 2.13e-01 | 20260505 22:42:41.065 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.55e+00, 1.90e-01 / 7.00e-02 | 20260505 22:42:41.065 | INFO | PerfReport: 16.02%, DME -- 4.02e+01 / 4.42e+01, 3.85e+00 / 1.14e+00 | 20260505 22:42:41.065 | INFO | PerfReport: 57.22%, FrameLabeling -- 1.80e+01 / 1.93e+01, 1.48e+00 / 1.01e+00 | 20260505 22:42:41.065 | INFO | PerfReport: 7.16%, PulseAccumulating -- 2.25e+00 / 2.78e+00, 4.29e-01 / 2.24e-01 | 20260505 22:42:41.065 | INFO | PerfReport: 3.14%, Metrics -- 9.87e-01 / 1.27e+00, 2.66e-01 / 1.22e-01 | 20260505 22:42:41.065 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.43e-01 / 4.02e-01, 5.79e-02 / 3.28e-02 | 20260505 22:42:41.065 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 22:42:41.065 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:42:41.065 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:42:41.065 | INFO | PerfReport: Full Execution -- 9.86e+01 / -nan, 2.47e+01 / 0.00e+00 | 20260505 22:42:41.065 | INFO | PerfReport: 22.31%, Upload -- 2.20e+01 / 0.00e+00, 7.54e+00 / 0.00e+00 | 20260505 22:42:41.065 | INFO | PerfReport: 21.77%, Download -- 2.15e+01 / 0.00e+00, 7.58e+00 / 0.00e+00 | 20260505 22:42:41.065 | INFO | PerfReport: 31.82%, Compute -- 3.14e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 22:42:41.065 | INFO | PerfReport: 2.20%, LockUpload -- 2.17e+00 / 0.00e+00, 7.96e+00 / 0.00e+00 | 20260505 22:42:41.065 | INFO | PerfReport: 1.27%, LockDownload -- 1.25e+00 / 0.00e+00, 2.82e+00 / 0.00e+00 | 20260505 22:42:41.065 | INFO | PerfReport: 20.62%, LockCompute -- 2.03e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260505 22:42:41.065 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.75e-02 / 0.00e+00 | 20260505 22:42:41.190 | INFO | Starting a round of Baz IO | 20260505 22:42:44.336 | INFO | Populating RTMetrics report | 20260505 22:42:44.336 | INFO | RTMetrics report populated | 20260505 22:42:44.336 | INFO | Opening tmp RTMetrics json file | 20260505 22:42:44.339 | INFO | Serializing RTMetrics to json | 20260505 22:42:44.340 | INFO | Serialization done, swapping tmp file now... | 20260505 22:42:44.341 | INFO | RTMetrics done | 20260505 22:42:44.422 | INFO | Finished a round of Baz IO: Wrote 1.917 GiB in 3.231 seconds | 20260505 22:42:44.422 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.48% / 0.00% / 78.52% | 20260505 22:42:44.422 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.117775% / 0% / 6.75519e-05% | 20260505 22:42:44.422 | INFO | Baz IO overall: | 20260505 22:42:44.422 | INFO | BytesWritten : 3954757102992 | 20260505 22:42:44.422 | INFO | NumEvents : 1864790133178 | 20260505 22:42:46.208 | INFO | Analyzing chunk frames = [6881280,6881792) | 20260505 22:42:46.208 | INFO | MemInfo: { | 20260505 22:42:46.208 | INFO | "gpu" : " 17 GB", | 20260505 22:42:46.208 | INFO | "huge" : "", | 20260505 22:42:46.208 | INFO | "huge_shared" : " 27 GB", | 20260505 22:42:46.208 | INFO | "malloc" : " 14 GB", | 20260505 22:42:46.208 | INFO | "pinned" : " 29 GB" | 20260505 22:42:46.208 | INFO | } | 20260505 22:42:46.208 | INFO | Perf report for the last 5.14428 seconds: | 20260505 22:42:46.208 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:42:46.208 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:46.208 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:46.208 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:46.208 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 22:42:46.208 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:46.208 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260505 22:42:46.208 | INFO | PRE_HQ : 59.5%, 1.8, 54.7ms, 0 | 20260505 22:42:46.208 | INFO | BAZWRITER : 62.0%, 3.6, 115.3ms, 0 | 20260505 22:42:46.208 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 22:42:46.208 | INFO | Uploaded 36GiB to GPU in 2.13461s (16.8649GiB/s) | 20260505 22:42:46.208 | INFO | Downloaded 38GiB to GPU in 2.22149s (17.1056GiB/s) | 20260505 22:42:51.342 | INFO | Analyzing chunk frames = [6881792,6882304) | 20260505 22:42:51.343 | INFO | MemInfo: { | 20260505 22:42:51.343 | INFO | "gpu" : " 17 GB", | 20260505 22:42:51.343 | INFO | "huge" : "", | 20260505 22:42:51.343 | INFO | "huge_shared" : " 27 GB", | 20260505 22:42:51.343 | INFO | "malloc" : " 14 GB", | 20260505 22:42:51.343 | INFO | "pinned" : " 29 GB" | 20260505 22:42:51.343 | INFO | } | 20260505 22:42:51.343 | INFO | Perf report for the last 5.13459 seconds: | 20260505 22:42:51.343 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:42:51.343 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:51.343 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:51.343 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:51.343 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 22:42:51.343 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:51.343 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:51.343 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 22:42:51.343 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:51.343 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:51.343 | INFO | Uploaded 36GiB to GPU in 2.0987s (17.1534GiB/s) | 20260505 22:42:51.343 | INFO | Downloaded 34GiB to GPU in 2.03905s (16.6744GiB/s) | 20260505 22:42:56.482 | INFO | Analyzing chunk frames = [6882304,6882816) | 20260505 22:42:56.482 | INFO | MemInfo: { | 20260505 22:42:56.482 | INFO | "gpu" : " 17 GB", | 20260505 22:42:56.482 | INFO | "huge" : "", | 20260505 22:42:56.482 | INFO | "huge_shared" : " 27 GB", | 20260505 22:42:56.482 | INFO | "malloc" : " 14 GB", | 20260505 22:42:56.482 | INFO | "pinned" : " 29 GB" | 20260505 22:42:56.482 | INFO | } | 20260505 22:42:56.482 | INFO | Perf report for the last 5.13915 seconds: | 20260505 22:42:56.482 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:42:56.482 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:56.482 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:56.482 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:56.482 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260505 22:42:56.482 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:56.482 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:56.482 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 22:42:56.482 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:56.482 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:42:56.482 | INFO | Uploaded 36GiB to GPU in 2.15155s (16.7321GiB/s) | 20260505 22:42:56.482 | INFO | Downloaded 34GiB to GPU in 2.05365s (16.5559GiB/s) | 20260505 22:43:01.622 | INFO | Analyzing chunk frames = [6882816,6883328) | 20260505 22:43:01.622 | INFO | MemInfo: { | 20260505 22:43:01.622 | INFO | "gpu" : " 17 GB", | 20260505 22:43:01.622 | INFO | "huge" : "", | 20260505 22:43:01.622 | INFO | "huge_shared" : " 27 GB", | 20260505 22:43:01.622 | INFO | "malloc" : " 14 GB", | 20260505 22:43:01.622 | INFO | "pinned" : " 29 GB" | 20260505 22:43:01.622 | INFO | } | 20260505 22:43:01.622 | INFO | Perf report for the last 5.1404 seconds: | 20260505 22:43:01.622 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:43:01.622 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:01.622 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:01.622 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:01.622 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260505 22:43:01.622 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:01.622 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:01.622 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:43:01.622 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:01.622 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:01.623 | INFO | Uploaded 36GiB to GPU in 2.20733s (16.3093GiB/s) | 20260505 22:43:01.623 | INFO | Downloaded 34GiB to GPU in 2.04836s (16.5987GiB/s) | 20260505 22:43:06.769 | INFO | Analyzing chunk frames = [6883328,6883840) | 20260505 22:43:06.769 | INFO | MemInfo: { | 20260505 22:43:06.769 | INFO | "gpu" : " 17 GB", | 20260505 22:43:06.769 | INFO | "huge" : "", | 20260505 22:43:06.770 | INFO | "huge_shared" : " 27 GB", | 20260505 22:43:06.770 | INFO | "malloc" : " 14 GB", | 20260505 22:43:06.770 | INFO | "pinned" : " 29 GB" | 20260505 22:43:06.770 | INFO | } | 20260505 22:43:06.770 | INFO | Perf report for the last 5.14722 seconds: | 20260505 22:43:06.770 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:43:06.770 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:06.770 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:06.770 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:06.770 | INFO | ANALYSIS : 61.5%, 3.0, 96.1ms, 0 | 20260505 22:43:06.770 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:06.770 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:06.770 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:43:06.770 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:06.770 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:06.770 | INFO | Uploaded 36GiB to GPU in 2.13644s (16.8504GiB/s) | 20260505 22:43:06.770 | INFO | Downloaded 34GiB to GPU in 2.06529s (16.4626GiB/s) | 20260505 22:43:11.908 | INFO | Analyzing chunk frames = [6883840,6884352) | 20260505 22:43:11.908 | INFO | MemInfo: { | 20260505 22:43:11.908 | INFO | "gpu" : " 17 GB", | 20260505 22:43:11.908 | INFO | "huge" : "", | 20260505 22:43:11.908 | INFO | "huge_shared" : " 27 GB", | 20260505 22:43:11.908 | INFO | "malloc" : " 14 GB", | 20260505 22:43:11.908 | INFO | "pinned" : " 29 GB" | 20260505 22:43:11.908 | INFO | } | 20260505 22:43:11.908 | INFO | Perf report for the last 5.13854 seconds: | 20260505 22:43:11.908 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:43:11.908 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:11.908 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:11.908 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:11.908 | INFO | ANALYSIS : 61.5%, 3.0, 96.0ms, 0 | 20260505 22:43:11.908 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:11.908 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:11.908 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:43:11.908 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:11.908 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:11.908 | INFO | Uploaded 36GiB to GPU in 2.07295s (17.3665GiB/s) | 20260505 22:43:11.908 | INFO | Downloaded 34GiB to GPU in 2.03838s (16.6799GiB/s) | 20260505 22:43:17.044 | INFO | Analyzing chunk frames = [6884352,6884864) | 20260505 22:43:17.045 | INFO | MemInfo: { | 20260505 22:43:17.045 | INFO | "gpu" : " 17 GB", | 20260505 22:43:17.045 | INFO | "huge" : "", | 20260505 22:43:17.045 | INFO | "huge_shared" : " 27 GB", | 20260505 22:43:17.045 | INFO | "malloc" : " 14 GB", | 20260505 22:43:17.045 | INFO | "pinned" : " 29 GB" | 20260505 22:43:17.045 | INFO | } | 20260505 22:43:17.045 | INFO | Perf report for the last 5.13653 seconds: | 20260505 22:43:17.045 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:43:17.045 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:17.045 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:17.045 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:17.045 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 22:43:17.045 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:17.045 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:17.045 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 22:43:17.045 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:17.045 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:17.045 | INFO | Uploaded 36GiB to GPU in 2.00394s (17.9646GiB/s) | 20260505 22:43:17.045 | INFO | Downloaded 34GiB to GPU in 2.0432s (16.6405GiB/s) | 20260505 22:43:22.178 | INFO | Analyzing chunk frames = [6884864,6885376) | 20260505 22:43:22.178 | INFO | MemInfo: { | 20260505 22:43:22.178 | INFO | "gpu" : " 17 GB", | 20260505 22:43:22.178 | INFO | "huge" : "", | 20260505 22:43:22.178 | INFO | "huge_shared" : " 27 GB", | 20260505 22:43:22.178 | INFO | "malloc" : " 14 GB", | 20260505 22:43:22.178 | INFO | "pinned" : " 29 GB" | 20260505 22:43:22.178 | INFO | } | 20260505 22:43:22.178 | INFO | Perf report for the last 5.13316 seconds: | 20260505 22:43:22.178 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:43:22.178 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:22.178 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:22.178 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:22.178 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 22:43:22.178 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:22.178 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:22.178 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 22:43:22.178 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:22.178 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:22.178 | INFO | Uploaded 36GiB to GPU in 2.11799s (16.9972GiB/s) | 20260505 22:43:22.178 | INFO | Downloaded 34GiB to GPU in 2.0172s (16.8551GiB/s) | 20260505 22:43:22.307 | INFO | Starting a round of Baz IO | 20260505 22:43:25.451 | INFO | Populating RTMetrics report | 20260505 22:43:25.451 | INFO | RTMetrics report populated | 20260505 22:43:25.451 | INFO | Opening tmp RTMetrics json file | 20260505 22:43:25.451 | INFO | Serializing RTMetrics to json | 20260505 22:43:25.453 | INFO | Serialization done, swapping tmp file now... | 20260505 22:43:25.453 | INFO | RTMetrics done | 20260505 22:43:25.534 | INFO | Finished a round of Baz IO: Wrote 1.916 GiB in 3.227 seconds | 20260505 22:43:25.534 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.48% / 0.00% / 78.52% | 20260505 22:43:25.534 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.116813% / 0% / 6.75519e-05% | 20260505 22:43:25.534 | INFO | Baz IO overall: | 20260505 22:43:25.534 | INFO | BytesWritten : 3956814516249 | 20260505 22:43:25.534 | INFO | NumEvents : 1865734196781 | 20260505 22:43:27.320 | INFO | Analyzing chunk frames = [6885376,6885888) | 20260505 22:43:27.320 | INFO | MemInfo: { | 20260505 22:43:27.320 | INFO | "gpu" : " 17 GB", | 20260505 22:43:27.320 | INFO | "huge" : "", | 20260505 22:43:27.320 | INFO | "huge_shared" : " 27 GB", | 20260505 22:43:27.320 | INFO | "malloc" : " 14 GB", | 20260505 22:43:27.320 | INFO | "pinned" : " 29 GB" | 20260505 22:43:27.320 | INFO | } | 20260505 22:43:27.321 | INFO | Perf report for the last 5.14266 seconds: | 20260505 22:43:27.321 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:43:27.321 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:27.321 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:27.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:27.321 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260505 22:43:27.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:27.321 | INFO | RT_METRICS : 43.4%, 1.0, 22.8ms, 0 | 20260505 22:43:27.321 | INFO | PRE_HQ : 59.8%, 1.8, 54.8ms, 0 | 20260505 22:43:27.321 | INFO | BAZWRITER : 62.0%, 3.6, 114.4ms, 0 | 20260505 22:43:27.321 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 22:43:27.321 | INFO | Uploaded 36GiB to GPU in 2.12176s (16.967GiB/s) | 20260505 22:43:27.321 | INFO | Downloaded 38GiB to GPU in 2.21757s (17.1359GiB/s) | 20260505 22:43:32.459 | INFO | Analyzing chunk frames = [6885888,6886400) | 20260505 22:43:32.460 | INFO | MemInfo: { | 20260505 22:43:32.460 | INFO | "gpu" : " 17 GB", | 20260505 22:43:32.460 | INFO | "huge" : "", | 20260505 22:43:32.460 | INFO | "huge_shared" : " 27 GB", | 20260505 22:43:32.460 | INFO | "malloc" : " 14 GB", | 20260505 22:43:32.460 | INFO | "pinned" : " 29 GB" | 20260505 22:43:32.460 | INFO | } | 20260505 22:43:32.460 | INFO | Perf report for the last 5.13928 seconds: | 20260505 22:43:32.460 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:43:32.460 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:32.460 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:32.460 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:32.460 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 22:43:32.460 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:32.460 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:32.460 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:43:32.460 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:32.460 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:32.460 | INFO | Uploaded 36GiB to GPU in 2.0855s (17.262GiB/s) | 20260505 22:43:32.460 | INFO | Downloaded 34GiB to GPU in 2.03106s (16.7401GiB/s) | 20260505 22:43:37.606 | INFO | Analyzing chunk frames = [6886400,6886912) | 20260505 22:43:37.606 | INFO | MemInfo: { | 20260505 22:43:37.606 | INFO | "gpu" : " 17 GB", | 20260505 22:43:37.606 | INFO | "huge" : "", | 20260505 22:43:37.606 | INFO | "huge_shared" : " 27 GB", | 20260505 22:43:37.606 | INFO | "malloc" : " 14 GB", | 20260505 22:43:37.606 | INFO | "pinned" : " 29 GB" | 20260505 22:43:37.606 | INFO | } | 20260505 22:43:37.606 | INFO | Perf report for the last 5.14645 seconds: | 20260505 22:43:37.606 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:43:37.606 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:37.606 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:37.606 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:37.606 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260505 22:43:37.606 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:37.606 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:37.606 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 22:43:37.606 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:37.606 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:37.607 | INFO | Uploaded 36GiB to GPU in 2.15596s (16.6979GiB/s) | 20260505 22:43:37.607 | INFO | Downloaded 34GiB to GPU in 2.05658s (16.5323GiB/s) | 20260505 22:43:42.741 | INFO | Analyzing chunk frames = [6886912,6887424) | 20260505 22:43:42.741 | INFO | MemInfo: { | 20260505 22:43:42.741 | INFO | "gpu" : " 17 GB", | 20260505 22:43:42.741 | INFO | "huge" : "", | 20260505 22:43:42.741 | INFO | "huge_shared" : " 27 GB", | 20260505 22:43:42.741 | INFO | "malloc" : " 14 GB", | 20260505 22:43:42.741 | INFO | "pinned" : " 29 GB" | 20260505 22:43:42.741 | INFO | } | 20260505 22:43:42.742 | INFO | Perf report for the last 5.13523 seconds: | 20260505 22:43:42.742 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:43:42.742 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:42.742 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:42.742 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:42.742 | INFO | ANALYSIS : 61.8%, 3.0, 96.0ms, 0 | 20260505 22:43:42.742 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:42.742 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:42.742 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 22:43:42.742 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:42.742 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:42.742 | INFO | Uploaded 36GiB to GPU in 2.12612s (16.9323GiB/s) | 20260505 22:43:42.742 | INFO | Downloaded 34GiB to GPU in 2.01349s (16.8861GiB/s) | 20260505 22:43:47.870 | INFO | Analyzing chunk frames = [6887424,6887936) | 20260505 22:43:47.870 | INFO | MemInfo: { | 20260505 22:43:47.870 | INFO | "gpu" : " 17 GB", | 20260505 22:43:47.870 | INFO | "huge" : "", | 20260505 22:43:47.870 | INFO | "huge_shared" : " 27 GB", | 20260505 22:43:47.870 | INFO | "malloc" : " 14 GB", | 20260505 22:43:47.870 | INFO | "pinned" : " 29 GB" | 20260505 22:43:47.871 | INFO | } | 20260505 22:43:47.871 | INFO | Perf report for the last 5.12903 seconds: | 20260505 22:43:47.871 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:43:47.871 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:47.871 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:47.871 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:47.871 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260505 22:43:47.871 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:47.871 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:47.871 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:43:47.871 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:47.871 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:47.871 | INFO | Uploaded 36GiB to GPU in 2.13839s (16.8351GiB/s) | 20260505 22:43:47.871 | INFO | Downloaded 34GiB to GPU in 2.06271s (16.4832GiB/s) | 20260505 22:43:53.020 | INFO | Analyzing chunk frames = [6887936,6888448) | 20260505 22:43:53.020 | INFO | MemInfo: { | 20260505 22:43:53.020 | INFO | "gpu" : " 17 GB", | 20260505 22:43:53.020 | INFO | "huge" : "", | 20260505 22:43:53.020 | INFO | "huge_shared" : " 27 GB", | 20260505 22:43:53.020 | INFO | "malloc" : " 14 GB", | 20260505 22:43:53.020 | INFO | "pinned" : " 29 GB" | 20260505 22:43:53.020 | INFO | } | 20260505 22:43:53.020 | INFO | Perf report for the last 5.14926 seconds: | 20260505 22:43:53.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:43:53.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:53.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:53.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:53.020 | INFO | ANALYSIS : 61.6%, 3.0, 96.4ms, 0 | 20260505 22:43:53.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:53.020 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:53.020 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260505 22:43:53.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:53.020 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:53.020 | INFO | Uploaded 36GiB to GPU in 2.08603s (17.2577GiB/s) | 20260505 22:43:53.020 | INFO | Downloaded 34GiB to GPU in 2.03724s (16.6892GiB/s) | 20260505 22:43:58.161 | INFO | Analyzing chunk frames = [6888448,6888960) | 20260505 22:43:58.161 | INFO | MemInfo: { | 20260505 22:43:58.161 | INFO | "gpu" : " 17 GB", | 20260505 22:43:58.161 | INFO | "huge" : "", | 20260505 22:43:58.161 | INFO | "huge_shared" : " 27 GB", | 20260505 22:43:58.161 | INFO | "malloc" : " 14 GB", | 20260505 22:43:58.161 | INFO | "pinned" : " 29 GB" | 20260505 22:43:58.161 | INFO | } | 20260505 22:43:58.161 | INFO | Perf report for the last 5.14151 seconds: | 20260505 22:43:58.162 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:43:58.162 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:58.162 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:58.162 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:58.162 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 22:43:58.162 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:58.162 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:58.162 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 22:43:58.162 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:58.162 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:43:58.162 | INFO | Uploaded 36GiB to GPU in 2.01347s (17.8795GiB/s) | 20260505 22:43:58.162 | INFO | Downloaded 34GiB to GPU in 2.03727s (16.689GiB/s) | 20260505 22:44:03.293 | INFO | Analyzing chunk frames = [6888960,6889472) | 20260505 22:44:03.293 | INFO | MemInfo: { | 20260505 22:44:03.293 | INFO | "gpu" : " 17 GB", | 20260505 22:44:03.293 | INFO | "huge" : "", | 20260505 22:44:03.293 | INFO | "huge_shared" : " 27 GB", | 20260505 22:44:03.293 | INFO | "malloc" : " 14 GB", | 20260505 22:44:03.293 | INFO | "pinned" : " 29 GB" | 20260505 22:44:03.293 | INFO | } | 20260505 22:44:03.293 | INFO | Perf report for the last 5.1317 seconds: | 20260505 22:44:03.293 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:44:03.293 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:03.293 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:03.293 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:03.293 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 22:44:03.293 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:03.293 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:03.293 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 22:44:03.293 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:03.293 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:03.293 | INFO | Uploaded 36GiB to GPU in 2.10971s (17.064GiB/s) | 20260505 22:44:03.293 | INFO | Downloaded 34GiB to GPU in 2.00657s (16.9444GiB/s) | 20260505 22:44:03.293 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 22:44:03.293 | INFO | Total mem usage: 15.3 GiB | 20260505 22:44:03.293 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 22:44:03.293 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.56%) | 20260505 22:44:03.293 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 22:44:03.293 | INFO | High Water Breakdown: | 20260505 22:44:03.294 | INFO | 61.1% : 8.62 GiB ( 90.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 22:44:03.294 | INFO | 38.8% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 22:44:03.294 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 22:44:03.294 | INFO | ------------------------------------------ | 20260505 22:44:03.294 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:44:03.294 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 22:44:03.294 | INFO | Total mem usage: 26.5 GiB | 20260505 22:44:03.294 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 22:44:03.294 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 22:44:03.294 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 22:44:03.294 | INFO | High Water Breakdown: | 20260505 22:44:03.294 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:44:03.294 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:44:03.294 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:44:03.294 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:44:03.294 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:44:03.294 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:44:03.294 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 22:44:03.294 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 22:44:03.294 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:44:03.294 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 22:44:03.294 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 22:44:03.294 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 22:44:03.294 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 22:44:03.294 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 22:44:03.294 | INFO | ------------------------------------------ | 20260505 22:44:03.294 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:44:03.294 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 22:44:03.294 | INFO | Total mem usage: 25.6 GiB | 20260505 22:44:03.294 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 22:44:03.294 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 22:44:03.294 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 22:44:03.294 | INFO | High Water Breakdown: | 20260505 22:44:03.294 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 22:44:03.294 | INFO | ------------------------------------------ | 20260505 22:44:03.294 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:44:03.294 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 22:44:03.294 | INFO | Total mem usage: 15.7 GiB | 20260505 22:44:03.294 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 22:44:03.294 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 22:44:03.294 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 22:44:03.294 | INFO | High Water Breakdown: | 20260505 22:44:03.294 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 22:44:03.294 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 22:44:03.294 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 22:44:03.294 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 22:44:03.294 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 22:44:03.294 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 22:44:03.294 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 22:44:03.294 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 22:44:03.294 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 22:44:03.294 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:44:03.294 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 22:44:03.294 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:44:03.294 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 22:44:03.294 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 22:44:03.294 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 22:44:03.294 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 22:44:03.294 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:44:03.294 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 22:44:03.294 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:44:03.294 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:44:03.294 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:44:03.294 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 22:44:03.294 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 22:44:03.294 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:44:03.294 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 22:44:03.294 | INFO | ------------------------------------------ | 20260505 22:44:03.294 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:44:03.294 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 22:44:03.294 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:44:03.294 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:44:03.294 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.40e+01, 1.36e+01 / 9.33e-01 | 20260505 22:44:03.294 | INFO | PerfReport: 0.29%, AggregateStats -- 9.18e-02 / 1.05e-01, 1.06e-02 / 7.36e-03 | 20260505 22:44:03.294 | INFO | PerfReport: 11.82%, Baseline -- 3.70e+00 / 4.51e+00, 5.66e-01 / 2.32e-01 | 20260505 22:44:03.294 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.54e+00, 1.91e-01 / 6.07e-02 | 20260505 22:44:03.294 | INFO | PerfReport: 16.02%, DME -- 4.02e+01 / 4.42e+01, 3.91e+00 / 1.15e+00 | 20260505 22:44:03.294 | INFO | PerfReport: 57.18%, FrameLabeling -- 1.79e+01 / 1.91e+01, 1.43e+00 / 6.54e-01 | 20260505 22:44:03.294 | INFO | PerfReport: 7.17%, PulseAccumulating -- 2.25e+00 / 2.82e+00, 4.27e-01 / 1.50e-01 | 20260505 22:44:03.294 | INFO | PerfReport: 3.15%, Metrics -- 9.87e-01 / 1.30e+00, 2.66e-01 / 1.06e-01 | 20260505 22:44:03.294 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.41e-01 / 4.02e-01, 5.75e-02 / 2.90e-02 | 20260505 22:44:03.294 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 22:44:03.294 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:44:03.294 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:44:03.294 | INFO | PerfReport: Full Execution -- 9.86e+01 / -nan, 2.49e+01 / 0.00e+00 | 20260505 22:44:03.294 | INFO | PerfReport: 22.28%, Upload -- 2.20e+01 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260505 22:44:03.294 | INFO | PerfReport: 21.77%, Download -- 2.15e+01 / 0.00e+00, 7.56e+00 / 0.00e+00 | 20260505 22:44:03.295 | INFO | PerfReport: 31.77%, Compute -- 3.13e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 22:44:03.295 | INFO | PerfReport: 2.20%, LockUpload -- 2.17e+00 / 0.00e+00, 7.89e+00 / 0.00e+00 | 20260505 22:44:03.295 | INFO | PerfReport: 1.34%, LockDownload -- 1.32e+00 / 0.00e+00, 3.01e+00 / 0.00e+00 | 20260505 22:44:03.295 | INFO | PerfReport: 20.61%, LockCompute -- 2.03e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 22:44:03.295 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.66e-02 / 0.00e+00 | 20260505 22:44:03.417 | INFO | Starting a round of Baz IO | 20260505 22:44:06.565 | INFO | Populating RTMetrics report | 20260505 22:44:06.565 | INFO | RTMetrics report populated | 20260505 22:44:06.565 | INFO | Opening tmp RTMetrics json file | 20260505 22:44:06.567 | INFO | Serializing RTMetrics to json | 20260505 22:44:06.572 | INFO | Serialization done, swapping tmp file now... | 20260505 22:44:06.572 | INFO | RTMetrics done | 20260505 22:44:06.645 | INFO | Finished a round of Baz IO: Wrote 1.915 GiB in 3.228 seconds | 20260505 22:44:06.645 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.48% / 0.00% / 78.52% | 20260505 22:44:06.645 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.11698% / 0% / 7.54992e-05% | 20260505 22:44:06.645 | INFO | Baz IO overall: | 20260505 22:44:06.645 | INFO | BytesWritten : 3958870890070 | 20260505 22:44:06.646 | INFO | NumEvents : 1866678359659 | 20260505 22:44:08.432 | INFO | Analyzing chunk frames = [6889472,6889984) | 20260505 22:44:08.432 | INFO | MemInfo: { | 20260505 22:44:08.432 | INFO | "gpu" : " 17 GB", | 20260505 22:44:08.432 | INFO | "huge" : "", | 20260505 22:44:08.432 | INFO | "huge_shared" : " 27 GB", | 20260505 22:44:08.432 | INFO | "malloc" : " 14 GB", | 20260505 22:44:08.432 | INFO | "pinned" : " 29 GB" | 20260505 22:44:08.432 | INFO | } | 20260505 22:44:08.432 | INFO | Perf report for the last 5.13877 seconds: | 20260505 22:44:08.432 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:44:08.432 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:08.432 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:08.432 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:08.432 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260505 22:44:08.432 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:08.432 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260505 22:44:08.432 | INFO | PRE_HQ : 59.7%, 1.7, 54.2ms, 0 | 20260505 22:44:08.432 | INFO | BAZWRITER : 62.1%, 3.6, 114.4ms, 0 | 20260505 22:44:08.432 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 22:44:08.432 | INFO | Uploaded 36GiB to GPU in 2.16496s (16.6285GiB/s) | 20260505 22:44:08.432 | INFO | Downloaded 38GiB to GPU in 2.23259s (17.0206GiB/s) | 20260505 22:44:13.573 | INFO | Analyzing chunk frames = [6889984,6890496) | 20260505 22:44:13.573 | INFO | MemInfo: { | 20260505 22:44:13.573 | INFO | "gpu" : " 17 GB", | 20260505 22:44:13.573 | INFO | "huge" : "", | 20260505 22:44:13.573 | INFO | "huge_shared" : " 27 GB", | 20260505 22:44:13.573 | INFO | "malloc" : " 14 GB", | 20260505 22:44:13.573 | INFO | "pinned" : " 29 GB" | 20260505 22:44:13.573 | INFO | } | 20260505 22:44:13.573 | INFO | Perf report for the last 5.14133 seconds: | 20260505 22:44:13.573 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:44:13.573 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:13.573 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:13.573 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:13.573 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 22:44:13.573 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:13.573 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:13.573 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260505 22:44:13.573 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:13.573 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:13.573 | INFO | Uploaded 36GiB to GPU in 2.0726s (17.3695GiB/s) | 20260505 22:44:13.574 | INFO | Downloaded 34GiB to GPU in 2.02293s (16.8073GiB/s) | 20260505 22:44:18.719 | INFO | Analyzing chunk frames = [6890496,6891008) | 20260505 22:44:18.719 | INFO | MemInfo: { | 20260505 22:44:18.719 | INFO | "gpu" : " 17 GB", | 20260505 22:44:18.720 | INFO | "huge" : "", | 20260505 22:44:18.720 | INFO | "huge_shared" : " 27 GB", | 20260505 22:44:18.720 | INFO | "malloc" : " 14 GB", | 20260505 22:44:18.720 | INFO | "pinned" : " 29 GB" | 20260505 22:44:18.720 | INFO | } | 20260505 22:44:18.720 | INFO | Perf report for the last 5.14649 seconds: | 20260505 22:44:18.720 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:44:18.720 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:18.720 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:18.720 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:18.720 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260505 22:44:18.720 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:18.720 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:18.720 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:44:18.720 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:18.720 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:18.720 | INFO | Uploaded 36GiB to GPU in 2.13748s (16.8423GiB/s) | 20260505 22:44:18.720 | INFO | Downloaded 34GiB to GPU in 2.03201s (16.7322GiB/s) | 20260505 22:44:23.857 | INFO | Analyzing chunk frames = [6891008,6891520) | 20260505 22:44:23.857 | INFO | MemInfo: { | 20260505 22:44:23.857 | INFO | "gpu" : " 17 GB", | 20260505 22:44:23.857 | INFO | "huge" : "", | 20260505 22:44:23.857 | INFO | "huge_shared" : " 27 GB", | 20260505 22:44:23.857 | INFO | "malloc" : " 14 GB", | 20260505 22:44:23.857 | INFO | "pinned" : " 29 GB" | 20260505 22:44:23.857 | INFO | } | 20260505 22:44:23.857 | INFO | Perf report for the last 5.13759 seconds: | 20260505 22:44:23.857 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:44:23.857 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:23.857 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:23.857 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:23.857 | INFO | ANALYSIS : 61.8%, 3.0, 96.0ms, 0 | 20260505 22:44:23.857 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:23.857 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:23.857 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:44:23.857 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:23.857 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:23.858 | INFO | Uploaded 36GiB to GPU in 2.14607s (16.7749GiB/s) | 20260505 22:44:23.858 | INFO | Downloaded 34GiB to GPU in 2.02975s (16.7509GiB/s) | 20260505 22:44:28.992 | INFO | Analyzing chunk frames = [6891520,6892032) | 20260505 22:44:28.992 | INFO | MemInfo: { | 20260505 22:44:28.992 | INFO | "gpu" : " 17 GB", | 20260505 22:44:28.992 | INFO | "huge" : "", | 20260505 22:44:28.992 | INFO | "huge_shared" : " 27 GB", | 20260505 22:44:28.992 | INFO | "malloc" : " 14 GB", | 20260505 22:44:28.992 | INFO | "pinned" : " 29 GB" | 20260505 22:44:28.992 | INFO | } | 20260505 22:44:28.992 | INFO | Perf report for the last 5.1349 seconds: | 20260505 22:44:28.992 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:44:28.992 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:28.992 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:28.992 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:28.992 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260505 22:44:28.992 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:28.992 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:28.992 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260505 22:44:28.992 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:28.992 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:28.992 | INFO | Uploaded 36GiB to GPU in 2.09451s (17.1878GiB/s) | 20260505 22:44:28.992 | INFO | Downloaded 34GiB to GPU in 2.03922s (16.6731GiB/s) | 20260505 22:44:34.129 | INFO | Analyzing chunk frames = [6892032,6892544) | 20260505 22:44:34.129 | INFO | MemInfo: { | 20260505 22:44:34.129 | INFO | "gpu" : " 17 GB", | 20260505 22:44:34.129 | INFO | "huge" : "", | 20260505 22:44:34.129 | INFO | "huge_shared" : " 27 GB", | 20260505 22:44:34.129 | INFO | "malloc" : " 14 GB", | 20260505 22:44:34.129 | INFO | "pinned" : " 29 GB" | 20260505 22:44:34.129 | INFO | } | 20260505 22:44:34.129 | INFO | Perf report for the last 5.13652 seconds: | 20260505 22:44:34.129 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:44:34.129 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:34.129 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:34.129 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:34.129 | INFO | ANALYSIS : 61.6%, 3.0, 96.2ms, 0 | 20260505 22:44:34.129 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:34.129 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:34.129 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 22:44:34.129 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:34.129 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:34.129 | INFO | Uploaded 36GiB to GPU in 2.07868s (17.3187GiB/s) | 20260505 22:44:34.129 | INFO | Downloaded 34GiB to GPU in 2.03247s (16.7284GiB/s) | 20260505 22:44:39.268 | INFO | Analyzing chunk frames = [6892544,6893056) | 20260505 22:44:39.268 | INFO | MemInfo: { | 20260505 22:44:39.268 | INFO | "gpu" : " 17 GB", | 20260505 22:44:39.268 | INFO | "huge" : "", | 20260505 22:44:39.269 | INFO | "huge_shared" : " 27 GB", | 20260505 22:44:39.269 | INFO | "malloc" : " 14 GB", | 20260505 22:44:39.269 | INFO | "pinned" : " 29 GB" | 20260505 22:44:39.269 | INFO | } | 20260505 22:44:39.269 | INFO | Perf report for the last 5.13994 seconds: | 20260505 22:44:39.269 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:44:39.269 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:39.269 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:39.269 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:39.269 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 22:44:39.269 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:39.269 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:39.269 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 22:44:39.269 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:39.269 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:39.269 | INFO | Uploaded 36GiB to GPU in 2.1503s (16.7419GiB/s) | 20260505 22:44:39.269 | INFO | Downloaded 34GiB to GPU in 2.11202s (16.0983GiB/s) | 20260505 22:44:44.414 | INFO | Analyzing chunk frames = [6893056,6893568) | 20260505 22:44:44.414 | INFO | MemInfo: { | 20260505 22:44:44.414 | INFO | "gpu" : " 17 GB", | 20260505 22:44:44.414 | INFO | "huge" : "", | 20260505 22:44:44.414 | INFO | "huge_shared" : " 27 GB", | 20260505 22:44:44.414 | INFO | "malloc" : " 14 GB", | 20260505 22:44:44.414 | INFO | "pinned" : " 29 GB" | 20260505 22:44:44.414 | INFO | } | 20260505 22:44:44.414 | INFO | Perf report for the last 5.14556 seconds: | 20260505 22:44:44.414 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:44:44.414 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:44.414 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:44.414 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:44.414 | INFO | ANALYSIS : 61.9%, 3.0, 96.8ms, 0 | 20260505 22:44:44.414 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:44.414 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:44.414 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 22:44:44.414 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:44.414 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:44.415 | INFO | Uploaded 36GiB to GPU in 2.11383s (17.0307GiB/s) | 20260505 22:44:44.415 | INFO | Downloaded 34GiB to GPU in 1.99918s (17.007GiB/s) | 20260505 22:44:44.538 | INFO | Starting a round of Baz IO | 20260505 22:44:47.687 | INFO | Populating RTMetrics report | 20260505 22:44:47.687 | INFO | RTMetrics report populated | 20260505 22:44:47.687 | INFO | Opening tmp RTMetrics json file | 20260505 22:44:47.690 | INFO | Serializing RTMetrics to json | 20260505 22:44:47.692 | INFO | Serialization done, swapping tmp file now... | 20260505 22:44:47.692 | INFO | RTMetrics done | 20260505 22:44:47.771 | INFO | Finished a round of Baz IO: Wrote 1.916 GiB in 3.232 seconds | 20260505 22:44:47.771 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.48% / 0.00% / 78.52% | 20260505 22:44:47.771 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.116944% / 0% / 6.75519e-05% | 20260505 22:44:47.771 | INFO | Baz IO overall: | 20260505 22:44:47.771 | INFO | BytesWritten : 3960928127320 | 20260505 22:44:47.771 | INFO | NumEvents : 1867623456061 | 20260505 22:44:49.553 | INFO | Analyzing chunk frames = [6893568,6894080) | 20260505 22:44:49.553 | INFO | MemInfo: { | 20260505 22:44:49.553 | INFO | "gpu" : " 17 GB", | 20260505 22:44:49.553 | INFO | "huge" : "", | 20260505 22:44:49.553 | INFO | "huge_shared" : " 27 GB", | 20260505 22:44:49.553 | INFO | "malloc" : " 14 GB", | 20260505 22:44:49.553 | INFO | "pinned" : " 29 GB" | 20260505 22:44:49.553 | INFO | } | 20260505 22:44:49.553 | INFO | Perf report for the last 5.13899 seconds: | 20260505 22:44:49.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:44:49.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:49.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:49.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:49.553 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 22:44:49.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:49.553 | INFO | RT_METRICS : 43.6%, 1.0, 22.8ms, 0 | 20260505 22:44:49.553 | INFO | PRE_HQ : 59.9%, 1.8, 54.6ms, 0 | 20260505 22:44:49.553 | INFO | BAZWRITER : 62.1%, 3.6, 115.2ms, 0 | 20260505 22:44:49.553 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 22:44:49.554 | INFO | Uploaded 36GiB to GPU in 2.16793s (16.6057GiB/s) | 20260505 22:44:49.554 | INFO | Downloaded 38GiB to GPU in 2.23406s (17.0094GiB/s) | 20260505 22:44:54.688 | INFO | Analyzing chunk frames = [6894080,6894592) | 20260505 22:44:54.688 | INFO | MemInfo: { | 20260505 22:44:54.688 | INFO | "gpu" : " 17 GB", | 20260505 22:44:54.688 | INFO | "huge" : "", | 20260505 22:44:54.688 | INFO | "huge_shared" : " 27 GB", | 20260505 22:44:54.688 | INFO | "malloc" : " 14 GB", | 20260505 22:44:54.688 | INFO | "pinned" : " 29 GB" | 20260505 22:44:54.688 | INFO | } | 20260505 22:44:54.688 | INFO | Perf report for the last 5.1347 seconds: | 20260505 22:44:54.688 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:44:54.688 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:54.688 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:54.688 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:54.688 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260505 22:44:54.688 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:54.688 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:54.688 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260505 22:44:54.688 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:54.688 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:54.688 | INFO | Uploaded 36GiB to GPU in 2.08905s (17.2327GiB/s) | 20260505 22:44:54.688 | INFO | Downloaded 34GiB to GPU in 2.02649s (16.7778GiB/s) | 20260505 22:44:59.829 | INFO | Analyzing chunk frames = [6894592,6895104) | 20260505 22:44:59.829 | INFO | MemInfo: { | 20260505 22:44:59.829 | INFO | "gpu" : " 17 GB", | 20260505 22:44:59.829 | INFO | "huge" : "", | 20260505 22:44:59.829 | INFO | "huge_shared" : " 27 GB", | 20260505 22:44:59.829 | INFO | "malloc" : " 14 GB", | 20260505 22:44:59.829 | INFO | "pinned" : " 29 GB" | 20260505 22:44:59.829 | INFO | } | 20260505 22:44:59.829 | INFO | Perf report for the last 5.14096 seconds: | 20260505 22:44:59.829 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:44:59.829 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:59.829 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:59.829 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:59.829 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260505 22:44:59.829 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:59.829 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:59.829 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:44:59.829 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:59.829 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:44:59.829 | INFO | Uploaded 36GiB to GPU in 2.14468s (16.7857GiB/s) | 20260505 22:44:59.829 | INFO | Downloaded 34GiB to GPU in 2.02278s (16.8085GiB/s) | 20260505 22:45:04.969 | INFO | Analyzing chunk frames = [6895104,6895616) | 20260505 22:45:04.969 | INFO | MemInfo: { | 20260505 22:45:04.969 | INFO | "gpu" : " 17 GB", | 20260505 22:45:04.969 | INFO | "huge" : "", | 20260505 22:45:04.969 | INFO | "huge_shared" : " 27 GB", | 20260505 22:45:04.969 | INFO | "malloc" : " 14 GB", | 20260505 22:45:04.969 | INFO | "pinned" : " 29 GB" | 20260505 22:45:04.969 | INFO | } | 20260505 22:45:04.969 | INFO | Perf report for the last 5.14025 seconds: | 20260505 22:45:04.969 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:45:04.969 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:04.969 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:04.969 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:04.969 | INFO | ANALYSIS : 61.7%, 3.0, 95.8ms, 0 | 20260505 22:45:04.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:04.969 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:04.969 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:45:04.969 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:04.969 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:04.969 | INFO | Uploaded 36GiB to GPU in 2.13748s (16.8422GiB/s) | 20260505 22:45:04.970 | INFO | Downloaded 34GiB to GPU in 2.00822s (16.9304GiB/s) | 20260505 22:45:10.105 | INFO | Analyzing chunk frames = [6895616,6896128) | 20260505 22:45:10.105 | INFO | MemInfo: { | 20260505 22:45:10.105 | INFO | "gpu" : " 17 GB", | 20260505 22:45:10.105 | INFO | "huge" : "", | 20260505 22:45:10.105 | INFO | "huge_shared" : " 27 GB", | 20260505 22:45:10.106 | INFO | "malloc" : " 14 GB", | 20260505 22:45:10.106 | INFO | "pinned" : " 29 GB" | 20260505 22:45:10.106 | INFO | } | 20260505 22:45:10.106 | INFO | Perf report for the last 5.1363 seconds: | 20260505 22:45:10.106 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:45:10.106 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:10.106 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:10.106 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:10.106 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260505 22:45:10.106 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:10.106 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:10.106 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:45:10.106 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:10.106 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:10.106 | INFO | Uploaded 36GiB to GPU in 2.10233s (17.1238GiB/s) | 20260505 22:45:10.106 | INFO | Downloaded 34GiB to GPU in 2.04647s (16.6139GiB/s) | 20260505 22:45:15.243 | INFO | Analyzing chunk frames = [6896128,6896640) | 20260505 22:45:15.243 | INFO | MemInfo: { | 20260505 22:45:15.243 | INFO | "gpu" : " 17 GB", | 20260505 22:45:15.243 | INFO | "huge" : "", | 20260505 22:45:15.243 | INFO | "huge_shared" : " 27 GB", | 20260505 22:45:15.243 | INFO | "malloc" : " 14 GB", | 20260505 22:45:15.243 | INFO | "pinned" : " 29 GB" | 20260505 22:45:15.243 | INFO | } | 20260505 22:45:15.243 | INFO | Perf report for the last 5.13719 seconds: | 20260505 22:45:15.243 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:45:15.243 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:15.243 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:15.243 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:15.243 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260505 22:45:15.243 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:15.243 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:15.243 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 22:45:15.243 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:15.243 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:15.243 | INFO | Uploaded 36GiB to GPU in 2.08733s (17.247GiB/s) | 20260505 22:45:15.243 | INFO | Downloaded 34GiB to GPU in 2.03597s (16.6996GiB/s) | 20260505 22:45:20.380 | INFO | Analyzing chunk frames = [6896640,6897152) | 20260505 22:45:20.380 | INFO | MemInfo: { | 20260505 22:45:20.380 | INFO | "gpu" : " 17 GB", | 20260505 22:45:20.380 | INFO | "huge" : "", | 20260505 22:45:20.380 | INFO | "huge_shared" : " 27 GB", | 20260505 22:45:20.380 | INFO | "malloc" : " 14 GB", | 20260505 22:45:20.380 | INFO | "pinned" : " 29 GB" | 20260505 22:45:20.380 | INFO | } | 20260505 22:45:20.380 | INFO | Perf report for the last 5.13741 seconds: | 20260505 22:45:20.380 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:45:20.380 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:20.380 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:20.380 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:20.380 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 22:45:20.380 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:20.380 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:20.380 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 22:45:20.380 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:20.380 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:20.380 | INFO | Uploaded 36GiB to GPU in 2.16796s (16.6055GiB/s) | 20260505 22:45:20.380 | INFO | Downloaded 34GiB to GPU in 2.12752s (15.981GiB/s) | 20260505 22:45:25.527 | INFO | Analyzing chunk frames = [6897152,6897664) | 20260505 22:45:25.527 | INFO | MemInfo: { | 20260505 22:45:25.527 | INFO | "gpu" : " 17 GB", | 20260505 22:45:25.527 | INFO | "huge" : "", | 20260505 22:45:25.527 | INFO | "huge_shared" : " 27 GB", | 20260505 22:45:25.527 | INFO | "malloc" : " 14 GB", | 20260505 22:45:25.527 | INFO | "pinned" : " 29 GB" | 20260505 22:45:25.527 | INFO | } | 20260505 22:45:25.527 | INFO | Perf report for the last 5.14728 seconds: | 20260505 22:45:25.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:45:25.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:25.528 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:25.528 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:25.528 | INFO | ANALYSIS : 61.9%, 3.0, 96.9ms, 0 | 20260505 22:45:25.528 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:25.528 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:25.528 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:45:25.528 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:25.528 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:25.528 | INFO | Uploaded 36GiB to GPU in 2.11697s (17.0054GiB/s) | 20260505 22:45:25.528 | INFO | Downloaded 34GiB to GPU in 1.99951s (17.0042GiB/s) | 20260505 22:45:25.528 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 22:45:25.528 | INFO | Total mem usage: 15.3 GiB | 20260505 22:45:25.528 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 22:45:25.528 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -9.08%) | 20260505 22:45:25.528 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 22:45:25.528 | INFO | High Water Breakdown: | 20260505 22:45:25.528 | INFO | 60.9% : 8.56 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 22:45:25.528 | INFO | 39% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 22:45:25.528 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 22:45:25.528 | INFO | ------------------------------------------ | 20260505 22:45:25.528 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:45:25.528 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 22:45:25.528 | INFO | Total mem usage: 26.5 GiB | 20260505 22:45:25.528 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 22:45:25.528 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 22:45:25.528 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 22:45:25.528 | INFO | High Water Breakdown: | 20260505 22:45:25.528 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:45:25.528 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:45:25.528 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:45:25.528 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:45:25.528 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:45:25.528 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:45:25.528 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 22:45:25.528 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 22:45:25.528 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:45:25.528 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 22:45:25.528 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 22:45:25.528 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 22:45:25.528 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 22:45:25.528 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 22:45:25.528 | INFO | ------------------------------------------ | 20260505 22:45:25.528 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:45:25.528 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 22:45:25.528 | INFO | Total mem usage: 25.6 GiB | 20260505 22:45:25.528 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 22:45:25.528 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 22:45:25.528 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 22:45:25.528 | INFO | High Water Breakdown: | 20260505 22:45:25.528 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 22:45:25.528 | INFO | ------------------------------------------ | 20260505 22:45:25.528 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:45:25.528 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 22:45:25.529 | INFO | Total mem usage: 15.7 GiB | 20260505 22:45:25.529 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 22:45:25.529 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 22:45:25.529 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 22:45:25.529 | INFO | High Water Breakdown: | 20260505 22:45:25.529 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 22:45:25.529 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 22:45:25.529 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 22:45:25.529 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 22:45:25.529 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 22:45:25.529 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 22:45:25.529 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 22:45:25.529 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 22:45:25.529 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 22:45:25.529 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:45:25.529 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 22:45:25.529 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:45:25.529 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 22:45:25.529 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 22:45:25.529 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 22:45:25.529 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 22:45:25.529 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:45:25.529 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 22:45:25.529 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:45:25.529 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:45:25.529 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:45:25.529 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 22:45:25.529 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 22:45:25.529 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:45:25.529 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 22:45:25.529 | INFO | ------------------------------------------ | 20260505 22:45:25.529 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:45:25.529 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 22:45:25.529 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:45:25.529 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:45:25.529 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.40e+01, 1.32e+01 / 1.30e+00 | 20260505 22:45:25.529 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.04e-01, 1.05e-02 / 9.51e-03 | 20260505 22:45:25.529 | INFO | PerfReport: 11.82%, Baseline -- 3.70e+00 / 4.53e+00, 5.66e-01 / 1.62e-01 | 20260505 22:45:25.529 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.52e+00, 1.88e-01 / 1.04e-01 | 20260505 22:45:25.529 | INFO | PerfReport: 16.02%, DME -- 4.02e+01 / 4.41e+01, 3.74e+00 / 1.34e+00 | 20260505 22:45:25.529 | INFO | PerfReport: 57.19%, FrameLabeling -- 1.79e+01 / 1.94e+01, 1.44e+00 / 1.23e+00 | 20260505 22:45:25.529 | INFO | PerfReport: 7.17%, PulseAccumulating -- 2.25e+00 / 2.76e+00, 4.32e-01 / 2.55e-01 | 20260505 22:45:25.529 | INFO | PerfReport: 3.14%, Metrics -- 9.85e-01 / 1.26e+00, 2.64e-01 / 1.37e-01 | 20260505 22:45:25.529 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.41e-01 / 3.88e-01, 5.75e-02 / 3.09e-02 | 20260505 22:45:25.529 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 22:45:25.529 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:45:25.529 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:45:25.529 | INFO | PerfReport: Full Execution -- 9.87e+01 / -nan, 2.50e+01 / 0.00e+00 | 20260505 22:45:25.529 | INFO | PerfReport: 22.41%, Upload -- 2.21e+01 / 0.00e+00, 7.67e+00 / 0.00e+00 | 20260505 22:45:25.529 | INFO | PerfReport: 21.77%, Download -- 2.15e+01 / 0.00e+00, 7.61e+00 / 0.00e+00 | 20260505 22:45:25.529 | INFO | PerfReport: 31.76%, Compute -- 3.13e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260505 22:45:25.529 | INFO | PerfReport: 2.24%, LockUpload -- 2.21e+00 / 0.00e+00, 8.08e+00 / 0.00e+00 | 20260505 22:45:25.529 | INFO | PerfReport: 1.32%, LockDownload -- 1.30e+00 / 0.00e+00, 2.80e+00 / 0.00e+00 | 20260505 22:45:25.529 | INFO | PerfReport: 20.49%, LockCompute -- 2.02e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260505 22:45:25.529 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.87e-02 / 0.00e+00 | 20260505 22:45:25.650 | INFO | Starting a round of Baz IO | 20260505 22:45:28.797 | INFO | Populating RTMetrics report | 20260505 22:45:28.797 | INFO | RTMetrics report populated | 20260505 22:45:28.797 | INFO | Opening tmp RTMetrics json file | 20260505 22:45:28.801 | INFO | Serializing RTMetrics to json | 20260505 22:45:28.803 | INFO | Serialization done, swapping tmp file now... | 20260505 22:45:28.804 | INFO | RTMetrics done | 20260505 22:45:28.883 | INFO | Finished a round of Baz IO: Wrote 1.915 GiB in 3.232 seconds | 20260505 22:45:28.883 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.48% / 0.00% / 78.52% | 20260505 22:45:28.883 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.116873% / 0% / 6.75519e-05% | 20260505 22:45:28.883 | INFO | Baz IO overall: | 20260505 22:45:28.883 | INFO | BytesWritten : 3962984763126 | 20260505 22:45:28.883 | INFO | NumEvents : 1868568429608 | 20260505 22:45:30.670 | INFO | Analyzing chunk frames = [6897664,6898176) | 20260505 22:45:30.670 | INFO | MemInfo: { | 20260505 22:45:30.670 | INFO | "gpu" : " 17 GB", | 20260505 22:45:30.670 | INFO | "huge" : "", | 20260505 22:45:30.670 | INFO | "huge_shared" : " 27 GB", | 20260505 22:45:30.670 | INFO | "malloc" : " 14 GB", | 20260505 22:45:30.670 | INFO | "pinned" : " 29 GB" | 20260505 22:45:30.670 | INFO | } | 20260505 22:45:30.670 | INFO | Perf report for the last 5.14302 seconds: | 20260505 22:45:30.670 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:45:30.670 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:30.671 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:30.671 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:30.671 | INFO | ANALYSIS : 62.5%, 3.0, 97.7ms, 0 | 20260505 22:45:30.671 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:30.671 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260505 22:45:30.671 | INFO | PRE_HQ : 59.9%, 1.8, 54.8ms, 0 | 20260505 22:45:30.671 | INFO | BAZWRITER : 62.0%, 3.6, 115.5ms, 0 | 20260505 22:45:30.671 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 22:45:30.671 | INFO | Uploaded 36GiB to GPU in 2.16414s (16.6348GiB/s) | 20260505 22:45:30.671 | INFO | Downloaded 38GiB to GPU in 2.2366s (16.9901GiB/s) | 20260505 22:45:35.808 | INFO | Analyzing chunk frames = [6898176,6898688) | 20260505 22:45:35.808 | INFO | MemInfo: { | 20260505 22:45:35.808 | INFO | "gpu" : " 17 GB", | 20260505 22:45:35.808 | INFO | "huge" : "", | 20260505 22:45:35.808 | INFO | "huge_shared" : " 27 GB", | 20260505 22:45:35.808 | INFO | "malloc" : " 14 GB", | 20260505 22:45:35.808 | INFO | "pinned" : " 29 GB" | 20260505 22:45:35.808 | INFO | } | 20260505 22:45:35.808 | INFO | Perf report for the last 5.13741 seconds: | 20260505 22:45:35.808 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:45:35.808 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:35.808 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:35.808 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:35.808 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260505 22:45:35.808 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:35.808 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:35.808 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 22:45:35.808 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:35.808 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:35.808 | INFO | Uploaded 36GiB to GPU in 2.09966s (17.1456GiB/s) | 20260505 22:45:35.808 | INFO | Downloaded 34GiB to GPU in 2.03931s (16.6723GiB/s) | 20260505 22:45:40.942 | INFO | Analyzing chunk frames = [6898688,6899200) | 20260505 22:45:40.942 | INFO | MemInfo: { | 20260505 22:45:40.942 | INFO | "gpu" : " 17 GB", | 20260505 22:45:40.942 | INFO | "huge" : "", | 20260505 22:45:40.942 | INFO | "huge_shared" : " 27 GB", | 20260505 22:45:40.942 | INFO | "malloc" : " 14 GB", | 20260505 22:45:40.942 | INFO | "pinned" : " 29 GB" | 20260505 22:45:40.942 | INFO | } | 20260505 22:45:40.942 | INFO | Perf report for the last 5.13394 seconds: | 20260505 22:45:40.942 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:45:40.942 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:40.942 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:40.942 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:40.942 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260505 22:45:40.942 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:40.942 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:40.942 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 22:45:40.942 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:40.942 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:40.942 | INFO | Uploaded 36GiB to GPU in 2.13935s (16.8275GiB/s) | 20260505 22:45:40.942 | INFO | Downloaded 34GiB to GPU in 2.03276s (16.726GiB/s) | 20260505 22:45:46.082 | INFO | Analyzing chunk frames = [6899200,6899712) | 20260505 22:45:46.082 | INFO | MemInfo: { | 20260505 22:45:46.082 | INFO | "gpu" : " 17 GB", | 20260505 22:45:46.082 | INFO | "huge" : "", | 20260505 22:45:46.082 | INFO | "huge_shared" : " 27 GB", | 20260505 22:45:46.082 | INFO | "malloc" : " 14 GB", | 20260505 22:45:46.082 | INFO | "pinned" : " 29 GB" | 20260505 22:45:46.082 | INFO | } | 20260505 22:45:46.083 | INFO | Perf report for the last 5.14064 seconds: | 20260505 22:45:46.083 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:45:46.083 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:46.083 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:46.083 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:46.083 | INFO | ANALYSIS : 61.9%, 3.0, 96.1ms, 0 | 20260505 22:45:46.083 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:46.083 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:46.083 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:45:46.083 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:46.083 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:46.083 | INFO | Uploaded 36GiB to GPU in 2.14052s (16.8183GiB/s) | 20260505 22:45:46.083 | INFO | Downloaded 34GiB to GPU in 2.04339s (16.639GiB/s) | 20260505 22:45:51.220 | INFO | Analyzing chunk frames = [6899712,6900224) | 20260505 22:45:51.220 | INFO | MemInfo: { | 20260505 22:45:51.220 | INFO | "gpu" : " 17 GB", | 20260505 22:45:51.220 | INFO | "huge" : "", | 20260505 22:45:51.220 | INFO | "huge_shared" : " 27 GB", | 20260505 22:45:51.220 | INFO | "malloc" : " 14 GB", | 20260505 22:45:51.220 | INFO | "pinned" : " 29 GB" | 20260505 22:45:51.220 | INFO | } | 20260505 22:45:51.220 | INFO | Perf report for the last 5.13743 seconds: | 20260505 22:45:51.220 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:45:51.220 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:51.220 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:51.220 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:51.220 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260505 22:45:51.220 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:51.220 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:51.220 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 22:45:51.220 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:51.220 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:51.220 | INFO | Uploaded 36GiB to GPU in 2.10635s (17.0912GiB/s) | 20260505 22:45:51.220 | INFO | Downloaded 34GiB to GPU in 2.04867s (16.5961GiB/s) | 20260505 22:45:56.366 | INFO | Analyzing chunk frames = [6900224,6900736) | 20260505 22:45:56.366 | INFO | MemInfo: { | 20260505 22:45:56.366 | INFO | "gpu" : " 17 GB", | 20260505 22:45:56.366 | INFO | "huge" : "", | 20260505 22:45:56.366 | INFO | "huge_shared" : " 27 GB", | 20260505 22:45:56.366 | INFO | "malloc" : " 14 GB", | 20260505 22:45:56.366 | INFO | "pinned" : " 29 GB" | 20260505 22:45:56.366 | INFO | } | 20260505 22:45:56.366 | INFO | Perf report for the last 5.14646 seconds: | 20260505 22:45:56.366 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:45:56.366 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:56.366 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:56.366 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:56.366 | INFO | ANALYSIS : 61.5%, 3.0, 96.1ms, 0 | 20260505 22:45:56.366 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:56.366 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:56.367 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:45:56.367 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:56.367 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:45:56.367 | INFO | Uploaded 36GiB to GPU in 2.09029s (17.2225GiB/s) | 20260505 22:45:56.367 | INFO | Downloaded 34GiB to GPU in 2.04264s (16.6451GiB/s) | 20260505 22:46:01.501 | INFO | Analyzing chunk frames = [6900736,6901248) | 20260505 22:46:01.501 | INFO | MemInfo: { | 20260505 22:46:01.501 | INFO | "gpu" : " 17 GB", | 20260505 22:46:01.501 | INFO | "huge" : "", | 20260505 22:46:01.501 | INFO | "huge_shared" : " 27 GB", | 20260505 22:46:01.501 | INFO | "malloc" : " 14 GB", | 20260505 22:46:01.501 | INFO | "pinned" : " 29 GB" | 20260505 22:46:01.501 | INFO | } | 20260505 22:46:01.501 | INFO | Perf report for the last 5.13436 seconds: | 20260505 22:46:01.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:46:01.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:01.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:01.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:01.501 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 22:46:01.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:01.501 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:01.501 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 22:46:01.501 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:01.501 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:01.501 | INFO | Uploaded 36GiB to GPU in 2.02164s (17.8073GiB/s) | 20260505 22:46:01.501 | INFO | Downloaded 34GiB to GPU in 2.0538s (16.5546GiB/s) | 20260505 22:46:06.636 | INFO | Analyzing chunk frames = [6901248,6901760) | 20260505 22:46:06.636 | INFO | MemInfo: { | 20260505 22:46:06.636 | INFO | "gpu" : " 17 GB", | 20260505 22:46:06.636 | INFO | "huge" : "", | 20260505 22:46:06.636 | INFO | "huge_shared" : " 27 GB", | 20260505 22:46:06.636 | INFO | "malloc" : " 14 GB", | 20260505 22:46:06.636 | INFO | "pinned" : " 29 GB" | 20260505 22:46:06.636 | INFO | } | 20260505 22:46:06.636 | INFO | Perf report for the last 5.13528 seconds: | 20260505 22:46:06.636 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:46:06.636 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:06.636 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:06.636 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:06.636 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 22:46:06.636 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:06.636 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:06.636 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 22:46:06.636 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:06.636 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:06.636 | INFO | Uploaded 36GiB to GPU in 2.11993s (16.9817GiB/s) | 20260505 22:46:06.636 | INFO | Downloaded 34GiB to GPU in 2.00182s (16.9846GiB/s) | 20260505 22:46:06.780 | INFO | Starting a round of Baz IO | 20260505 22:46:09.921 | INFO | Populating RTMetrics report | 20260505 22:46:09.921 | INFO | RTMetrics report populated | 20260505 22:46:09.921 | INFO | Opening tmp RTMetrics json file | 20260505 22:46:09.922 | INFO | Serializing RTMetrics to json | 20260505 22:46:09.924 | INFO | Serialization done, swapping tmp file now... | 20260505 22:46:09.926 | INFO | RTMetrics done | 20260505 22:46:10.007 | INFO | Finished a round of Baz IO: Wrote 1.915 GiB in 3.226 seconds | 20260505 22:46:10.007 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.47% / 0.00% / 78.52% | 20260505 22:46:10.007 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.117612% / 0% / 9.53674e-05% | 20260505 22:46:10.007 | INFO | Baz IO overall: | 20260505 22:46:10.007 | INFO | BytesWritten : 3965040898357 | 20260505 22:46:10.007 | INFO | NumEvents : 1869513322557 | 20260505 22:46:11.781 | INFO | Analyzing chunk frames = [6901760,6902272) | 20260505 22:46:11.781 | INFO | MemInfo: { | 20260505 22:46:11.781 | INFO | "gpu" : " 17 GB", | 20260505 22:46:11.781 | INFO | "huge" : "", | 20260505 22:46:11.781 | INFO | "huge_shared" : " 27 GB", | 20260505 22:46:11.781 | INFO | "malloc" : " 14 GB", | 20260505 22:46:11.781 | INFO | "pinned" : " 29 GB" | 20260505 22:46:11.781 | INFO | } | 20260505 22:46:11.781 | INFO | Perf report for the last 5.14495 seconds: | 20260505 22:46:11.781 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:46:11.781 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:11.781 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:11.781 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:11.781 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 22:46:11.781 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:11.781 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260505 22:46:11.781 | INFO | PRE_HQ : 59.7%, 1.8, 54.7ms, 0 | 20260505 22:46:11.781 | INFO | BAZWRITER : 62.2%, 3.6, 115.5ms, 0 | 20260505 22:46:11.781 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 22:46:11.781 | INFO | Uploaded 36GiB to GPU in 2.17139s (16.5793GiB/s) | 20260505 22:46:11.781 | INFO | Downloaded 38GiB to GPU in 2.23935s (16.9692GiB/s) | 20260505 22:46:16.925 | INFO | Analyzing chunk frames = [6902272,6902784) | 20260505 22:46:16.925 | INFO | MemInfo: { | 20260505 22:46:16.925 | INFO | "gpu" : " 17 GB", | 20260505 22:46:16.925 | INFO | "huge" : "", | 20260505 22:46:16.925 | INFO | "huge_shared" : " 27 GB", | 20260505 22:46:16.925 | INFO | "malloc" : " 14 GB", | 20260505 22:46:16.925 | INFO | "pinned" : " 29 GB" | 20260505 22:46:16.925 | INFO | } | 20260505 22:46:16.925 | INFO | Perf report for the last 5.14362 seconds: | 20260505 22:46:16.925 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:46:16.925 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:16.925 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:16.925 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:16.925 | INFO | ANALYSIS : 61.8%, 3.0, 96.6ms, 0 | 20260505 22:46:16.925 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:16.925 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:16.925 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:46:16.925 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:16.925 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:16.925 | INFO | Uploaded 36GiB to GPU in 2.08384s (17.2758GiB/s) | 20260505 22:46:16.925 | INFO | Downloaded 34GiB to GPU in 2.02875s (16.7591GiB/s) | 20260505 22:46:22.056 | INFO | Analyzing chunk frames = [6902784,6903296) | 20260505 22:46:22.056 | INFO | MemInfo: { | 20260505 22:46:22.056 | INFO | "gpu" : " 17 GB", | 20260505 22:46:22.056 | INFO | "huge" : "", | 20260505 22:46:22.056 | INFO | "huge_shared" : " 27 GB", | 20260505 22:46:22.056 | INFO | "malloc" : " 14 GB", | 20260505 22:46:22.056 | INFO | "pinned" : " 29 GB" | 20260505 22:46:22.056 | INFO | } | 20260505 22:46:22.056 | INFO | Perf report for the last 5.13137 seconds: | 20260505 22:46:22.056 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:46:22.056 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:22.056 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:22.056 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:22.056 | INFO | ANALYSIS : 61.6%, 3.0, 95.6ms, 0 | 20260505 22:46:22.056 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:22.056 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:22.056 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 22:46:22.056 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:22.056 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:22.056 | INFO | Uploaded 36GiB to GPU in 2.07065s (17.3858GiB/s) | 20260505 22:46:22.056 | INFO | Downloaded 34GiB to GPU in 2.01156s (16.9023GiB/s) | 20260505 22:46:27.190 | INFO | Analyzing chunk frames = [6903296,6903808) | 20260505 22:46:27.190 | INFO | MemInfo: { | 20260505 22:46:27.190 | INFO | "gpu" : " 17 GB", | 20260505 22:46:27.190 | INFO | "huge" : "", | 20260505 22:46:27.190 | INFO | "huge_shared" : " 27 GB", | 20260505 22:46:27.190 | INFO | "malloc" : " 14 GB", | 20260505 22:46:27.190 | INFO | "pinned" : " 29 GB" | 20260505 22:46:27.190 | INFO | } | 20260505 22:46:27.190 | INFO | Perf report for the last 5.13381 seconds: | 20260505 22:46:27.190 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:46:27.190 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:27.190 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:27.190 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:27.190 | INFO | ANALYSIS : 62.0%, 3.0, 96.1ms, 0 | 20260505 22:46:27.190 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:27.190 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:27.190 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:46:27.190 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:27.190 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:27.190 | INFO | Uploaded 36GiB to GPU in 2.13407s (16.8692GiB/s) | 20260505 22:46:27.190 | INFO | Downloaded 34GiB to GPU in 2.04167s (16.653GiB/s) | 20260505 22:46:32.337 | INFO | Analyzing chunk frames = [6903808,6904320) | 20260505 22:46:32.337 | INFO | MemInfo: { | 20260505 22:46:32.337 | INFO | "gpu" : " 17 GB", | 20260505 22:46:32.337 | INFO | "huge" : "", | 20260505 22:46:32.337 | INFO | "huge_shared" : " 27 GB", | 20260505 22:46:32.337 | INFO | "malloc" : " 14 GB", | 20260505 22:46:32.337 | INFO | "pinned" : " 29 GB" | 20260505 22:46:32.337 | INFO | } | 20260505 22:46:32.337 | INFO | Perf report for the last 5.14716 seconds: | 20260505 22:46:32.337 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:46:32.337 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:32.337 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:32.337 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:32.337 | INFO | ANALYSIS : 61.5%, 3.0, 96.2ms, 0 | 20260505 22:46:32.337 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:32.337 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:32.337 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 22:46:32.337 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:32.337 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:32.337 | INFO | Uploaded 36GiB to GPU in 2.13892s (16.8309GiB/s) | 20260505 22:46:32.337 | INFO | Downloaded 34GiB to GPU in 2.0689s (16.4339GiB/s) | 20260505 22:46:37.483 | INFO | Analyzing chunk frames = [6904320,6904832) | 20260505 22:46:37.483 | INFO | MemInfo: { | 20260505 22:46:37.483 | INFO | "gpu" : " 17 GB", | 20260505 22:46:37.483 | INFO | "huge" : "", | 20260505 22:46:37.483 | INFO | "huge_shared" : " 27 GB", | 20260505 22:46:37.483 | INFO | "malloc" : " 14 GB", | 20260505 22:46:37.483 | INFO | "pinned" : " 29 GB" | 20260505 22:46:37.483 | INFO | } | 20260505 22:46:37.483 | INFO | Perf report for the last 5.14568 seconds: | 20260505 22:46:37.483 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:46:37.483 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:37.483 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:37.483 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:37.483 | INFO | ANALYSIS : 61.4%, 3.0, 96.0ms, 0 | 20260505 22:46:37.483 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:37.483 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:37.483 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 22:46:37.483 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:37.483 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:37.483 | INFO | Uploaded 36GiB to GPU in 2.07786s (17.3255GiB/s) | 20260505 22:46:37.483 | INFO | Downloaded 34GiB to GPU in 2.0374s (16.6879GiB/s) | 20260505 22:46:42.622 | INFO | Analyzing chunk frames = [6904832,6905344) | 20260505 22:46:42.622 | INFO | MemInfo: { | 20260505 22:46:42.622 | INFO | "gpu" : " 17 GB", | 20260505 22:46:42.622 | INFO | "huge" : "", | 20260505 22:46:42.622 | INFO | "huge_shared" : " 27 GB", | 20260505 22:46:42.622 | INFO | "malloc" : " 14 GB", | 20260505 22:46:42.622 | INFO | "pinned" : " 29 GB" | 20260505 22:46:42.622 | INFO | } | 20260505 22:46:42.622 | INFO | Perf report for the last 5.13903 seconds: | 20260505 22:46:42.622 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:46:42.622 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:42.622 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:42.622 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:42.622 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 22:46:42.622 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:42.622 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:42.622 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 22:46:42.622 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:42.622 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:42.622 | INFO | Uploaded 36GiB to GPU in 2.0822s (17.2894GiB/s) | 20260505 22:46:42.622 | INFO | Downloaded 34GiB to GPU in 2.08293s (16.3231GiB/s) | 20260505 22:46:47.755 | INFO | Analyzing chunk frames = [6905344,6905856) | 20260505 22:46:47.755 | INFO | MemInfo: { | 20260505 22:46:47.755 | INFO | "gpu" : " 17 GB", | 20260505 22:46:47.755 | INFO | "huge" : "", | 20260505 22:46:47.755 | INFO | "huge_shared" : " 27 GB", | 20260505 22:46:47.755 | INFO | "malloc" : " 14 GB", | 20260505 22:46:47.755 | INFO | "pinned" : " 29 GB" | 20260505 22:46:47.755 | INFO | } | 20260505 22:46:47.755 | INFO | Perf report for the last 5.13331 seconds: | 20260505 22:46:47.755 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:46:47.755 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:47.755 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:47.755 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:47.755 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260505 22:46:47.755 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:47.755 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:47.755 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 22:46:47.755 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:47.755 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:47.755 | INFO | Uploaded 36GiB to GPU in 2.11943s (16.9857GiB/s) | 20260505 22:46:47.755 | INFO | Downloaded 34GiB to GPU in 2.01765s (16.8513GiB/s) | 20260505 22:46:47.756 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 22:46:47.756 | INFO | Total mem usage: 15.3 GiB | 20260505 22:46:47.756 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 22:46:47.756 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -9.08%) | 20260505 22:46:47.756 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 22:46:47.756 | INFO | High Water Breakdown: | 20260505 22:46:47.756 | INFO | 60.9% : 8.56 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 22:46:47.756 | INFO | 39% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 22:46:47.756 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 22:46:47.756 | INFO | ------------------------------------------ | 20260505 22:46:47.756 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:46:47.756 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 22:46:47.756 | INFO | Total mem usage: 26.5 GiB | 20260505 22:46:47.756 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 22:46:47.756 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 22:46:47.756 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 22:46:47.756 | INFO | High Water Breakdown: | 20260505 22:46:47.756 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:46:47.756 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:46:47.756 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:46:47.756 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:46:47.756 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:46:47.756 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:46:47.756 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 22:46:47.756 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 22:46:47.756 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:46:47.756 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 22:46:47.756 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 22:46:47.756 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 22:46:47.756 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 22:46:47.756 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 22:46:47.756 | INFO | ------------------------------------------ | 20260505 22:46:47.756 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:46:47.756 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 22:46:47.756 | INFO | Total mem usage: 25.6 GiB | 20260505 22:46:47.756 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 22:46:47.756 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 22:46:47.756 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 22:46:47.756 | INFO | High Water Breakdown: | 20260505 22:46:47.756 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 22:46:47.756 | INFO | ------------------------------------------ | 20260505 22:46:47.756 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:46:47.756 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 22:46:47.756 | INFO | Total mem usage: 15.7 GiB | 20260505 22:46:47.756 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 22:46:47.757 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 22:46:47.757 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 22:46:47.757 | INFO | High Water Breakdown: | 20260505 22:46:47.757 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 22:46:47.757 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 22:46:47.757 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 22:46:47.757 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 22:46:47.757 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 22:46:47.757 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 22:46:47.757 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 22:46:47.757 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 22:46:47.757 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 22:46:47.757 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:46:47.757 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 22:46:47.757 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:46:47.757 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 22:46:47.757 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 22:46:47.757 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 22:46:47.757 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 22:46:47.757 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:46:47.757 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 22:46:47.757 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:46:47.757 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:46:47.757 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:46:47.757 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 22:46:47.757 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 22:46:47.757 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:46:47.757 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 22:46:47.757 | INFO | ------------------------------------------ | 20260505 22:46:47.757 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:46:47.757 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 22:46:47.757 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:46:47.757 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:46:47.757 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.41e+01, 1.36e+01 / 8.18e-01 | 20260505 22:46:47.757 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.04e-01, 1.07e-02 / 5.57e-03 | 20260505 22:46:47.757 | INFO | PerfReport: 11.80%, Baseline -- 3.71e+00 / 4.52e+00, 5.67e-01 / 1.62e-01 | 20260505 22:46:47.757 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.53e+00, 1.92e-01 / 9.35e-02 | 20260505 22:46:47.757 | INFO | PerfReport: 15.99%, DME -- 4.01e+01 / 4.41e+01, 3.76e+00 / 9.67e-01 | 20260505 22:46:47.757 | INFO | PerfReport: 57.29%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.52e+00 / 1.07e+00 | 20260505 22:46:47.757 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.24e+00 / 2.78e+00, 4.27e-01 / 2.08e-01 | 20260505 22:46:47.757 | INFO | PerfReport: 3.13%, Metrics -- 9.83e-01 / 1.28e+00, 2.68e-01 / 1.30e-01 | 20260505 22:46:47.757 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.42e-01 / 4.00e-01, 5.86e-02 / 3.48e-02 | 20260505 22:46:47.757 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 22:46:47.757 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:46:47.757 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:46:47.757 | INFO | PerfReport: Full Execution -- 9.86e+01 / -nan, 2.49e+01 / 0.00e+00 | 20260505 22:46:47.757 | INFO | PerfReport: 22.30%, Upload -- 2.20e+01 / 0.00e+00, 7.54e+00 / 0.00e+00 | 20260505 22:46:47.757 | INFO | PerfReport: 21.82%, Download -- 2.15e+01 / 0.00e+00, 7.58e+00 / 0.00e+00 | 20260505 22:46:47.757 | INFO | PerfReport: 31.85%, Compute -- 3.14e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 22:46:47.757 | INFO | PerfReport: 2.12%, LockUpload -- 2.08e+00 / 0.00e+00, 7.82e+00 / 0.00e+00 | 20260505 22:46:47.757 | INFO | PerfReport: 1.31%, LockDownload -- 1.29e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260505 22:46:47.757 | INFO | PerfReport: 20.59%, LockCompute -- 2.03e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260505 22:46:47.757 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.59e-02 / 0.00e+00 | 20260505 22:46:47.890 | INFO | Starting a round of Baz IO | 20260505 22:46:51.036 | INFO | Populating RTMetrics report | 20260505 22:46:51.036 | INFO | RTMetrics report populated | 20260505 22:46:51.036 | INFO | Opening tmp RTMetrics json file | 20260505 22:46:51.037 | INFO | Serializing RTMetrics to json | 20260505 22:46:51.041 | INFO | Serialization done, swapping tmp file now... | 20260505 22:46:51.044 | INFO | RTMetrics done | 20260505 22:46:51.123 | INFO | Finished a round of Baz IO: Wrote 1.914 GiB in 3.232 seconds | 20260505 22:46:51.123 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.47% / 0.00% / 78.53% | 20260505 22:46:51.123 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.116877% / 7.94729e-06% / 7.54992e-05% | 20260505 22:46:51.123 | INFO | Baz IO overall: | 20260505 22:46:51.123 | INFO | BytesWritten : 3967096486756 | 20260505 22:46:51.123 | INFO | NumEvents : 1870457958738 | 20260505 22:46:52.891 | INFO | Analyzing chunk frames = [6905856,6906368) | 20260505 22:46:52.891 | INFO | MemInfo: { | 20260505 22:46:52.891 | INFO | "gpu" : " 17 GB", | 20260505 22:46:52.891 | INFO | "huge" : "", | 20260505 22:46:52.891 | INFO | "huge_shared" : " 27 GB", | 20260505 22:46:52.891 | INFO | "malloc" : " 14 GB", | 20260505 22:46:52.891 | INFO | "pinned" : " 29 GB" | 20260505 22:46:52.891 | INFO | } | 20260505 22:46:52.891 | INFO | Perf report for the last 5.13594 seconds: | 20260505 22:46:52.891 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:46:52.891 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:52.891 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:52.891 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:52.891 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 22:46:52.891 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:52.891 | INFO | RT_METRICS : 43.8%, 1.0, 22.9ms, 0 | 20260505 22:46:52.891 | INFO | PRE_HQ : 59.7%, 1.8, 54.6ms, 0 | 20260505 22:46:52.891 | INFO | BAZWRITER : 62.4%, 3.6, 115.6ms, 0 | 20260505 22:46:52.891 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 22:46:52.891 | INFO | Uploaded 36GiB to GPU in 2.16533s (16.6256GiB/s) | 20260505 22:46:52.891 | INFO | Downloaded 38GiB to GPU in 2.24218s (16.9478GiB/s) | 20260505 22:46:58.037 | INFO | Analyzing chunk frames = [6906368,6906880) | 20260505 22:46:58.037 | INFO | MemInfo: { | 20260505 22:46:58.037 | INFO | "gpu" : " 17 GB", | 20260505 22:46:58.037 | INFO | "huge" : "", | 20260505 22:46:58.037 | INFO | "huge_shared" : " 27 GB", | 20260505 22:46:58.037 | INFO | "malloc" : " 14 GB", | 20260505 22:46:58.037 | INFO | "pinned" : " 29 GB" | 20260505 22:46:58.037 | INFO | } | 20260505 22:46:58.037 | INFO | Perf report for the last 5.1459 seconds: | 20260505 22:46:58.037 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:46:58.037 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:58.037 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:58.037 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:58.037 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260505 22:46:58.037 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:58.037 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:58.037 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260505 22:46:58.037 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:58.037 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:46:58.037 | INFO | Uploaded 36GiB to GPU in 2.07666s (17.3356GiB/s) | 20260505 22:46:58.037 | INFO | Downloaded 34GiB to GPU in 2.02379s (16.8002GiB/s) | 20260505 22:47:03.175 | INFO | Analyzing chunk frames = [6906880,6907392) | 20260505 22:47:03.175 | INFO | MemInfo: { | 20260505 22:47:03.175 | INFO | "gpu" : " 17 GB", | 20260505 22:47:03.175 | INFO | "huge" : "", | 20260505 22:47:03.175 | INFO | "huge_shared" : " 27 GB", | 20260505 22:47:03.176 | INFO | "malloc" : " 14 GB", | 20260505 22:47:03.176 | INFO | "pinned" : " 29 GB" | 20260505 22:47:03.176 | INFO | } | 20260505 22:47:03.176 | INFO | Perf report for the last 5.13872 seconds: | 20260505 22:47:03.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:47:03.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:03.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:03.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:03.176 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260505 22:47:03.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:03.176 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:03.176 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 22:47:03.176 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:03.176 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:03.176 | INFO | Uploaded 36GiB to GPU in 2.07175s (17.3766GiB/s) | 20260505 22:47:03.176 | INFO | Downloaded 34GiB to GPU in 2.01477s (16.8754GiB/s) | 20260505 22:47:08.316 | INFO | Analyzing chunk frames = [6907392,6907904) | 20260505 22:47:08.316 | INFO | MemInfo: { | 20260505 22:47:08.316 | INFO | "gpu" : " 17 GB", | 20260505 22:47:08.316 | INFO | "huge" : "", | 20260505 22:47:08.316 | INFO | "huge_shared" : " 27 GB", | 20260505 22:47:08.316 | INFO | "malloc" : " 14 GB", | 20260505 22:47:08.316 | INFO | "pinned" : " 29 GB" | 20260505 22:47:08.316 | INFO | } | 20260505 22:47:08.316 | INFO | Perf report for the last 5.14071 seconds: | 20260505 22:47:08.316 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:47:08.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:08.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:08.317 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:08.317 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260505 22:47:08.317 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:08.317 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:08.317 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 22:47:08.317 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:08.317 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:08.317 | INFO | Uploaded 36GiB to GPU in 2.12652s (16.929GiB/s) | 20260505 22:47:08.317 | INFO | Downloaded 34GiB to GPU in 2.00365s (16.969GiB/s) | 20260505 22:47:13.449 | INFO | Analyzing chunk frames = [6907904,6908416) | 20260505 22:47:13.449 | INFO | MemInfo: { | 20260505 22:47:13.449 | INFO | "gpu" : " 17 GB", | 20260505 22:47:13.449 | INFO | "huge" : "", | 20260505 22:47:13.449 | INFO | "huge_shared" : " 27 GB", | 20260505 22:47:13.450 | INFO | "malloc" : " 14 GB", | 20260505 22:47:13.450 | INFO | "pinned" : " 29 GB" | 20260505 22:47:13.450 | INFO | } | 20260505 22:47:13.450 | INFO | Perf report for the last 5.13311 seconds: | 20260505 22:47:13.450 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:47:13.450 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:13.450 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:13.450 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:13.450 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260505 22:47:13.450 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:13.450 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:13.450 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:47:13.450 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:13.450 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:13.450 | INFO | Uploaded 36GiB to GPU in 2.08889s (17.2341GiB/s) | 20260505 22:47:13.450 | INFO | Downloaded 34GiB to GPU in 2.03979s (16.6684GiB/s) | 20260505 22:47:18.592 | INFO | Analyzing chunk frames = [6908416,6908928) | 20260505 22:47:18.592 | INFO | MemInfo: { | 20260505 22:47:18.592 | INFO | "gpu" : " 17 GB", | 20260505 22:47:18.592 | INFO | "huge" : "", | 20260505 22:47:18.592 | INFO | "huge_shared" : " 27 GB", | 20260505 22:47:18.593 | INFO | "malloc" : " 14 GB", | 20260505 22:47:18.593 | INFO | "pinned" : " 29 GB" | 20260505 22:47:18.593 | INFO | } | 20260505 22:47:18.593 | INFO | Perf report for the last 5.14316 seconds: | 20260505 22:47:18.593 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:47:18.593 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:18.593 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:18.593 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:18.593 | INFO | ANALYSIS : 61.5%, 3.0, 96.1ms, 0 | 20260505 22:47:18.593 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:18.593 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:18.593 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:47:18.593 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:18.593 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:18.593 | INFO | Uploaded 36GiB to GPU in 2.08308s (17.2821GiB/s) | 20260505 22:47:18.593 | INFO | Downloaded 34GiB to GPU in 2.03656s (16.6948GiB/s) | 20260505 22:47:23.733 | INFO | Analyzing chunk frames = [6908928,6909440) | 20260505 22:47:23.733 | INFO | MemInfo: { | 20260505 22:47:23.733 | INFO | "gpu" : " 17 GB", | 20260505 22:47:23.733 | INFO | "huge" : "", | 20260505 22:47:23.733 | INFO | "huge_shared" : " 27 GB", | 20260505 22:47:23.734 | INFO | "malloc" : " 14 GB", | 20260505 22:47:23.734 | INFO | "pinned" : " 29 GB" | 20260505 22:47:23.734 | INFO | } | 20260505 22:47:23.734 | INFO | Perf report for the last 5.14088 seconds: | 20260505 22:47:23.734 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:47:23.734 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:23.734 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:23.734 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:23.734 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260505 22:47:23.734 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:23.734 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:23.734 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 22:47:23.734 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:23.734 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:23.734 | INFO | Uploaded 36GiB to GPU in 2.11326s (17.0353GiB/s) | 20260505 22:47:23.734 | INFO | Downloaded 34GiB to GPU in 2.08733s (16.2888GiB/s) | 20260505 22:47:28.867 | INFO | Analyzing chunk frames = [6909440,6909952) | 20260505 22:47:28.867 | INFO | MemInfo: { | 20260505 22:47:28.867 | INFO | "gpu" : " 17 GB", | 20260505 22:47:28.867 | INFO | "huge" : "", | 20260505 22:47:28.867 | INFO | "huge_shared" : " 27 GB", | 20260505 22:47:28.867 | INFO | "malloc" : " 14 GB", | 20260505 22:47:28.867 | INFO | "pinned" : " 29 GB" | 20260505 22:47:28.867 | INFO | } | 20260505 22:47:28.867 | INFO | Perf report for the last 5.13337 seconds: | 20260505 22:47:28.867 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:47:28.867 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:28.867 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:28.867 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:28.867 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 22:47:28.867 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:28.867 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:28.867 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 22:47:28.867 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:28.867 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:28.867 | INFO | Uploaded 36GiB to GPU in 2.10688s (17.0868GiB/s) | 20260505 22:47:28.868 | INFO | Downloaded 34GiB to GPU in 2.0079s (16.9331GiB/s) | 20260505 22:47:29.002 | INFO | Starting a round of Baz IO | 20260505 22:47:32.133 | INFO | Populating RTMetrics report | 20260505 22:47:32.134 | INFO | RTMetrics report populated | 20260505 22:47:32.134 | INFO | Opening tmp RTMetrics json file | 20260505 22:47:32.148 | INFO | Serializing RTMetrics to json | 20260505 22:47:32.186 | INFO | Serialization done, swapping tmp file now... | 20260505 22:47:32.187 | INFO | RTMetrics done | 20260505 22:47:32.269 | INFO | Finished a round of Baz IO: Wrote 1.916 GiB in 3.266 seconds | 20260505 22:47:32.269 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.47% / 0.00% / 78.53% | 20260505 22:47:32.269 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.117199% / 3.97364e-06% / 6.75519e-05% | 20260505 22:47:32.269 | INFO | Baz IO overall: | 20260505 22:47:32.269 | INFO | BytesWritten : 3969153749153 | 20260505 22:47:32.269 | INFO | NumEvents : 1871403843146 | 20260505 22:47:34.006 | INFO | Analyzing chunk frames = [6909952,6910464) | 20260505 22:47:34.006 | INFO | MemInfo: { | 20260505 22:47:34.006 | INFO | "gpu" : " 17 GB", | 20260505 22:47:34.006 | INFO | "huge" : "", | 20260505 22:47:34.006 | INFO | "huge_shared" : " 27 GB", | 20260505 22:47:34.006 | INFO | "malloc" : " 14 GB", | 20260505 22:47:34.006 | INFO | "pinned" : " 29 GB" | 20260505 22:47:34.006 | INFO | } | 20260505 22:47:34.006 | INFO | Perf report for the last 5.13911 seconds: | 20260505 22:47:34.006 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:47:34.006 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:34.006 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:34.006 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:34.006 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260505 22:47:34.006 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:34.006 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260505 22:47:34.006 | INFO | PRE_HQ : 60.0%, 1.8, 54.6ms, 0 | 20260505 22:47:34.006 | INFO | BAZWRITER : 63.0%, 3.5, 114.8ms, 0 | 20260505 22:47:34.006 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.5ms, 0 | 20260505 22:47:34.006 | INFO | Uploaded 36GiB to GPU in 2.12462s (16.9442GiB/s) | 20260505 22:47:34.007 | INFO | Downloaded 38GiB to GPU in 2.22795s (17.056GiB/s) | 20260505 22:47:39.142 | INFO | Analyzing chunk frames = [6910464,6910976) | 20260505 22:47:39.142 | INFO | MemInfo: { | 20260505 22:47:39.142 | INFO | "gpu" : " 17 GB", | 20260505 22:47:39.142 | INFO | "huge" : "", | 20260505 22:47:39.142 | INFO | "huge_shared" : " 27 GB", | 20260505 22:47:39.142 | INFO | "malloc" : " 14 GB", | 20260505 22:47:39.142 | INFO | "pinned" : " 29 GB" | 20260505 22:47:39.142 | INFO | } | 20260505 22:47:39.143 | INFO | Perf report for the last 5.13643 seconds: | 20260505 22:47:39.143 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:47:39.143 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:39.143 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:39.143 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:39.143 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 22:47:39.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:39.143 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:39.143 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:47:39.143 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:39.143 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:39.143 | INFO | Uploaded 36GiB to GPU in 2.07905s (17.3156GiB/s) | 20260505 22:47:39.143 | INFO | Downloaded 34GiB to GPU in 2.02271s (16.8092GiB/s) | 20260505 22:47:44.285 | INFO | Analyzing chunk frames = [6910976,6911488) | 20260505 22:47:44.285 | INFO | MemInfo: { | 20260505 22:47:44.285 | INFO | "gpu" : " 17 GB", | 20260505 22:47:44.285 | INFO | "huge" : "", | 20260505 22:47:44.285 | INFO | "huge_shared" : " 27 GB", | 20260505 22:47:44.285 | INFO | "malloc" : " 14 GB", | 20260505 22:47:44.285 | INFO | "pinned" : " 29 GB" | 20260505 22:47:44.285 | INFO | } | 20260505 22:47:44.285 | INFO | Perf report for the last 5.14258 seconds: | 20260505 22:47:44.285 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:47:44.285 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:44.285 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:44.285 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:44.285 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260505 22:47:44.285 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:44.285 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:44.285 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 22:47:44.285 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:44.285 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:44.285 | INFO | Uploaded 36GiB to GPU in 2.1402s (16.8209GiB/s) | 20260505 22:47:44.285 | INFO | Downloaded 34GiB to GPU in 2.03227s (16.7301GiB/s) | 20260505 22:47:49.431 | INFO | Analyzing chunk frames = [6911488,6912000) | 20260505 22:47:49.431 | INFO | MemInfo: { | 20260505 22:47:49.431 | INFO | "gpu" : " 17 GB", | 20260505 22:47:49.431 | INFO | "huge" : "", | 20260505 22:47:49.431 | INFO | "huge_shared" : " 27 GB", | 20260505 22:47:49.431 | INFO | "malloc" : " 14 GB", | 20260505 22:47:49.431 | INFO | "pinned" : " 29 GB" | 20260505 22:47:49.431 | INFO | } | 20260505 22:47:49.431 | INFO | Perf report for the last 5.14623 seconds: | 20260505 22:47:49.431 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:47:49.431 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:49.431 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:49.431 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:49.431 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260505 22:47:49.431 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:49.431 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:49.431 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:47:49.432 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:49.432 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:49.432 | INFO | Uploaded 36GiB to GPU in 2.12939s (16.9062GiB/s) | 20260505 22:47:49.432 | INFO | Downloaded 34GiB to GPU in 2.00696s (16.941GiB/s) | 20260505 22:47:54.572 | INFO | Analyzing chunk frames = [6912000,6912512) | 20260505 22:47:54.572 | INFO | MemInfo: { | 20260505 22:47:54.572 | INFO | "gpu" : " 17 GB", | 20260505 22:47:54.572 | INFO | "huge" : "", | 20260505 22:47:54.572 | INFO | "huge_shared" : " 27 GB", | 20260505 22:47:54.572 | INFO | "malloc" : " 14 GB", | 20260505 22:47:54.572 | INFO | "pinned" : " 29 GB" | 20260505 22:47:54.572 | INFO | } | 20260505 22:47:54.572 | INFO | Perf report for the last 5.1409 seconds: | 20260505 22:47:54.572 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:47:54.572 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:54.572 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:54.572 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:54.572 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260505 22:47:54.572 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:54.572 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:54.572 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260505 22:47:54.572 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:54.572 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:54.573 | INFO | Uploaded 36GiB to GPU in 2.09423s (17.19GiB/s) | 20260505 22:47:54.573 | INFO | Downloaded 34GiB to GPU in 2.0451s (16.6251GiB/s) | 20260505 22:47:59.699 | INFO | Analyzing chunk frames = [6912512,6913024) | 20260505 22:47:59.699 | INFO | MemInfo: { | 20260505 22:47:59.699 | INFO | "gpu" : " 17 GB", | 20260505 22:47:59.699 | INFO | "huge" : "", | 20260505 22:47:59.699 | INFO | "huge_shared" : " 27 GB", | 20260505 22:47:59.699 | INFO | "malloc" : " 14 GB", | 20260505 22:47:59.699 | INFO | "pinned" : " 29 GB" | 20260505 22:47:59.699 | INFO | } | 20260505 22:47:59.699 | INFO | Perf report for the last 5.12653 seconds: | 20260505 22:47:59.699 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:47:59.699 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:59.699 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:59.699 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:59.699 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260505 22:47:59.699 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:59.699 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:59.699 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:47:59.699 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:59.699 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:47:59.699 | INFO | Uploaded 36GiB to GPU in 2.07159s (17.3779GiB/s) | 20260505 22:47:59.699 | INFO | Downloaded 34GiB to GPU in 2.03369s (16.7184GiB/s) | 20260505 22:48:04.847 | INFO | Analyzing chunk frames = [6913024,6913536) | 20260505 22:48:04.847 | INFO | MemInfo: { | 20260505 22:48:04.847 | INFO | "gpu" : " 17 GB", | 20260505 22:48:04.847 | INFO | "huge" : "", | 20260505 22:48:04.847 | INFO | "huge_shared" : " 27 GB", | 20260505 22:48:04.847 | INFO | "malloc" : " 14 GB", | 20260505 22:48:04.847 | INFO | "pinned" : " 29 GB" | 20260505 22:48:04.847 | INFO | } | 20260505 22:48:04.847 | INFO | Perf report for the last 5.14827 seconds: | 20260505 22:48:04.847 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:48:04.847 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:04.847 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:04.847 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:04.847 | INFO | ANALYSIS : 62.9%, 3.0, 98.3ms, 0 | 20260505 22:48:04.847 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:04.847 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:04.847 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260505 22:48:04.847 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:04.847 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:04.847 | INFO | Uploaded 36GiB to GPU in 2.13267s (16.8803GiB/s) | 20260505 22:48:04.847 | INFO | Downloaded 34GiB to GPU in 2.17087s (15.6619GiB/s) | 20260505 22:48:09.995 | INFO | Analyzing chunk frames = [6913536,6914048) | 20260505 22:48:09.995 | INFO | MemInfo: { | 20260505 22:48:09.995 | INFO | "gpu" : " 17 GB", | 20260505 22:48:09.995 | INFO | "huge" : "", | 20260505 22:48:09.995 | INFO | "huge_shared" : " 27 GB", | 20260505 22:48:09.995 | INFO | "malloc" : " 14 GB", | 20260505 22:48:09.995 | INFO | "pinned" : " 29 GB" | 20260505 22:48:09.995 | INFO | } | 20260505 22:48:09.996 | INFO | Perf report for the last 5.14837 seconds: | 20260505 22:48:09.996 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:48:09.996 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:09.996 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:09.996 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:09.996 | INFO | ANALYSIS : 61.9%, 3.0, 96.8ms, 0 | 20260505 22:48:09.996 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:09.996 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:09.996 | INFO | PRE_HQ : 27.0%, 1.0, 14.1ms, 0 | 20260505 22:48:09.996 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:09.996 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:09.996 | INFO | Uploaded 36GiB to GPU in 2.11489s (17.0222GiB/s) | 20260505 22:48:09.996 | INFO | Downloaded 34GiB to GPU in 1.99751s (17.0212GiB/s) | 20260505 22:48:09.996 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 22:48:09.996 | INFO | Total mem usage: 15.3 GiB | 20260505 22:48:09.996 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 22:48:09.996 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -9.05%) | 20260505 22:48:09.996 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 22:48:09.996 | INFO | High Water Breakdown: | 20260505 22:48:09.996 | INFO | 60.9% : 8.56 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 22:48:09.996 | INFO | 39% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 22:48:09.996 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 22:48:09.996 | INFO | ------------------------------------------ | 20260505 22:48:09.996 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:48:09.996 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 22:48:09.996 | INFO | Total mem usage: 26.5 GiB | 20260505 22:48:09.996 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 22:48:09.996 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 22:48:09.996 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260505 22:48:09.996 | INFO | High Water Breakdown: | 20260505 22:48:09.996 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:48:09.996 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:48:09.996 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:48:09.996 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:48:09.996 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:48:09.996 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:48:09.996 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 22:48:09.996 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 22:48:09.996 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:48:09.996 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 22:48:09.996 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 22:48:09.996 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 22:48:09.996 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 22:48:09.996 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 22:48:09.996 | INFO | ------------------------------------------ | 20260505 22:48:09.996 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:48:09.996 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 22:48:09.996 | INFO | Total mem usage: 25.6 GiB | 20260505 22:48:09.996 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 22:48:09.996 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 22:48:09.996 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 22:48:09.996 | INFO | High Water Breakdown: | 20260505 22:48:09.996 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 22:48:09.997 | INFO | ------------------------------------------ | 20260505 22:48:09.997 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:48:09.997 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 22:48:09.997 | INFO | Total mem usage: 15.7 GiB | 20260505 22:48:09.997 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 22:48:09.997 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 22:48:09.997 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 22:48:09.997 | INFO | High Water Breakdown: | 20260505 22:48:09.997 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 22:48:09.997 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 22:48:09.997 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 22:48:09.997 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 22:48:09.997 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 22:48:09.997 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 22:48:09.997 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 22:48:09.997 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 22:48:09.997 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 22:48:09.997 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:48:09.997 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 22:48:09.997 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:48:09.997 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 22:48:09.997 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 22:48:09.997 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 22:48:09.997 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 22:48:09.997 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:48:09.997 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 22:48:09.997 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:48:09.997 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:48:09.997 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:48:09.997 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 22:48:09.997 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 22:48:09.997 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:48:09.997 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 22:48:09.997 | INFO | ------------------------------------------ | 20260505 22:48:09.997 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:48:09.997 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 22:48:09.997 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:48:09.997 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:48:09.997 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.41e+01, 1.35e+01 / 1.20e+00 | 20260505 22:48:09.997 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.04e-01, 1.05e-02 / 2.52e-03 | 20260505 22:48:09.997 | INFO | PerfReport: 11.86%, Baseline -- 3.72e+00 / 4.54e+00, 5.83e-01 / 1.79e-01 | 20260505 22:48:09.997 | INFO | PerfReport: 3.29%, Histogram -- 1.03e+00 / 1.54e+00, 1.94e-01 / 6.51e-02 | 20260505 22:48:09.997 | INFO | PerfReport: 15.99%, DME -- 4.02e+01 / 4.42e+01, 3.84e+00 / 1.31e+00 | 20260505 22:48:09.997 | INFO | PerfReport: 57.21%, FrameLabeling -- 1.80e+01 / 1.93e+01, 1.49e+00 / 9.74e-01 | 20260505 22:48:09.997 | INFO | PerfReport: 7.13%, PulseAccumulating -- 2.24e+00 / 2.79e+00, 4.29e-01 / 2.28e-01 | 20260505 22:48:09.997 | INFO | PerfReport: 3.14%, Metrics -- 9.86e-01 / 1.29e+00, 2.68e-01 / 1.26e-01 | 20260505 22:48:09.997 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.41e-01 / 3.98e-01, 5.83e-02 / 3.34e-02 | 20260505 22:48:09.997 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 22:48:09.997 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:48:09.997 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:48:09.997 | INFO | PerfReport: Full Execution -- 9.86e+01 / -nan, 2.49e+01 / 0.00e+00 | 20260505 22:48:09.997 | INFO | PerfReport: 22.27%, Upload -- 2.20e+01 / 0.00e+00, 7.52e+00 / 0.00e+00 | 20260505 22:48:09.997 | INFO | PerfReport: 21.79%, Download -- 2.15e+01 / 0.00e+00, 7.60e+00 / 0.00e+00 | 20260505 22:48:09.997 | INFO | PerfReport: 31.86%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 22:48:09.997 | INFO | PerfReport: 2.16%, LockUpload -- 2.13e+00 / 0.00e+00, 8.00e+00 / 0.00e+00 | 20260505 22:48:09.997 | INFO | PerfReport: 1.36%, LockDownload -- 1.34e+00 / 0.00e+00, 2.99e+00 / 0.00e+00 | 20260505 22:48:09.997 | INFO | PerfReport: 20.55%, LockCompute -- 2.03e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260505 22:48:09.997 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 3.39e-02 / 0.00e+00 | 20260505 22:48:10.121 | INFO | Starting a round of Baz IO | 20260505 22:48:13.261 | INFO | Populating RTMetrics report | 20260505 22:48:13.261 | INFO | RTMetrics report populated | 20260505 22:48:13.261 | INFO | Opening tmp RTMetrics json file | 20260505 22:48:13.262 | INFO | Serializing RTMetrics to json | 20260505 22:48:13.264 | INFO | Serialization done, swapping tmp file now... | 20260505 22:48:13.267 | INFO | RTMetrics done | 20260505 22:48:13.341 | INFO | Finished a round of Baz IO: Wrote 1.918 GiB in 3.219 seconds | 20260505 22:48:13.341 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.47% / 0.00% / 78.53% | 20260505 22:48:13.341 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.118117% / 0% / 7.54992e-05% | 20260505 22:48:13.341 | INFO | Baz IO overall: | 20260505 22:48:13.341 | INFO | BytesWritten : 3971212943838 | 20260505 22:48:13.341 | INFO | NumEvents : 1872350274265 | 20260505 22:48:15.128 | INFO | Analyzing chunk frames = [6914048,6914560) | 20260505 22:48:15.128 | INFO | MemInfo: { | 20260505 22:48:15.128 | INFO | "gpu" : " 17 GB", | 20260505 22:48:15.128 | INFO | "huge" : "", | 20260505 22:48:15.128 | INFO | "huge_shared" : " 27 GB", | 20260505 22:48:15.128 | INFO | "malloc" : " 14 GB", | 20260505 22:48:15.128 | INFO | "pinned" : " 29 GB" | 20260505 22:48:15.128 | INFO | } | 20260505 22:48:15.128 | INFO | Perf report for the last 5.13212 seconds: | 20260505 22:48:15.128 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:48:15.128 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:15.128 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:15.128 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:15.128 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 22:48:15.128 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:15.128 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260505 22:48:15.128 | INFO | PRE_HQ : 59.7%, 1.8, 54.3ms, 0 | 20260505 22:48:15.128 | INFO | BAZWRITER : 62.0%, 3.6, 115.0ms, 0 | 20260505 22:48:15.128 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 22:48:15.128 | INFO | Uploaded 36GiB to GPU in 2.14462s (16.7862GiB/s) | 20260505 22:48:15.128 | INFO | Downloaded 38GiB to GPU in 2.23347s (17.0139GiB/s) | 20260505 22:48:20.262 | INFO | Analyzing chunk frames = [6914560,6915072) | 20260505 22:48:20.262 | INFO | MemInfo: { | 20260505 22:48:20.262 | INFO | "gpu" : " 17 GB", | 20260505 22:48:20.262 | INFO | "huge" : "", | 20260505 22:48:20.262 | INFO | "huge_shared" : " 27 GB", | 20260505 22:48:20.262 | INFO | "malloc" : " 14 GB", | 20260505 22:48:20.262 | INFO | "pinned" : " 29 GB" | 20260505 22:48:20.262 | INFO | } | 20260505 22:48:20.262 | INFO | Perf report for the last 5.13438 seconds: | 20260505 22:48:20.262 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:48:20.262 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:20.262 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:20.262 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:20.262 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260505 22:48:20.262 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:20.262 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:20.262 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 22:48:20.262 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:20.262 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:20.262 | INFO | Uploaded 36GiB to GPU in 2.08864s (17.2361GiB/s) | 20260505 22:48:20.262 | INFO | Downloaded 34GiB to GPU in 2.03364s (16.7188GiB/s) | 20260505 22:48:25.399 | INFO | Analyzing chunk frames = [6915072,6915584) | 20260505 22:48:25.399 | INFO | MemInfo: { | 20260505 22:48:25.399 | INFO | "gpu" : " 17 GB", | 20260505 22:48:25.399 | INFO | "huge" : "", | 20260505 22:48:25.399 | INFO | "huge_shared" : " 27 GB", | 20260505 22:48:25.399 | INFO | "malloc" : " 14 GB", | 20260505 22:48:25.399 | INFO | "pinned" : " 29 GB" | 20260505 22:48:25.399 | INFO | } | 20260505 22:48:25.399 | INFO | Perf report for the last 5.13729 seconds: | 20260505 22:48:25.399 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:48:25.399 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:25.399 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:25.399 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:25.399 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260505 22:48:25.399 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:25.399 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:25.400 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:48:25.400 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:25.400 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:25.400 | INFO | Uploaded 36GiB to GPU in 2.14606s (16.7749GiB/s) | 20260505 22:48:25.400 | INFO | Downloaded 34GiB to GPU in 2.0214s (16.82GiB/s) | 20260505 22:48:30.544 | INFO | Analyzing chunk frames = [6915584,6916096) | 20260505 22:48:30.544 | INFO | MemInfo: { | 20260505 22:48:30.544 | INFO | "gpu" : " 17 GB", | 20260505 22:48:30.544 | INFO | "huge" : "", | 20260505 22:48:30.544 | INFO | "huge_shared" : " 27 GB", | 20260505 22:48:30.544 | INFO | "malloc" : " 14 GB", | 20260505 22:48:30.544 | INFO | "pinned" : " 29 GB" | 20260505 22:48:30.544 | INFO | } | 20260505 22:48:30.544 | INFO | Perf report for the last 5.14449 seconds: | 20260505 22:48:30.544 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:48:30.544 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:30.544 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:30.544 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:30.544 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260505 22:48:30.544 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:30.544 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:30.544 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:48:30.544 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:30.544 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:30.544 | INFO | Uploaded 36GiB to GPU in 2.12547s (16.9374GiB/s) | 20260505 22:48:30.544 | INFO | Downloaded 34GiB to GPU in 2.01911s (16.8391GiB/s) | 20260505 22:48:35.682 | INFO | Analyzing chunk frames = [6916096,6916608) | 20260505 22:48:35.682 | INFO | MemInfo: { | 20260505 22:48:35.682 | INFO | "gpu" : " 17 GB", | 20260505 22:48:35.682 | INFO | "huge" : "", | 20260505 22:48:35.682 | INFO | "huge_shared" : " 27 GB", | 20260505 22:48:35.682 | INFO | "malloc" : " 14 GB", | 20260505 22:48:35.682 | INFO | "pinned" : " 29 GB" | 20260505 22:48:35.682 | INFO | } | 20260505 22:48:35.682 | INFO | Perf report for the last 5.13789 seconds: | 20260505 22:48:35.682 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:48:35.682 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:35.682 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:35.682 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:35.682 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260505 22:48:35.682 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:35.682 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:35.682 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:48:35.682 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:35.682 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:35.682 | INFO | Uploaded 36GiB to GPU in 2.09948s (17.1471GiB/s) | 20260505 22:48:35.682 | INFO | Downloaded 34GiB to GPU in 2.03953s (16.6705GiB/s) | 20260505 22:48:40.818 | INFO | Analyzing chunk frames = [6916608,6917120) | 20260505 22:48:40.818 | INFO | MemInfo: { | 20260505 22:48:40.818 | INFO | "gpu" : " 17 GB", | 20260505 22:48:40.818 | INFO | "huge" : "", | 20260505 22:48:40.818 | INFO | "huge_shared" : " 27 GB", | 20260505 22:48:40.818 | INFO | "malloc" : " 14 GB", | 20260505 22:48:40.818 | INFO | "pinned" : " 29 GB" | 20260505 22:48:40.818 | INFO | } | 20260505 22:48:40.818 | INFO | Perf report for the last 5.13627 seconds: | 20260505 22:48:40.818 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:48:40.818 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:40.818 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:40.818 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:40.818 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260505 22:48:40.818 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:40.818 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:40.818 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:48:40.818 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:40.818 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:40.818 | INFO | Uploaded 36GiB to GPU in 2.082s (17.291GiB/s) | 20260505 22:48:40.818 | INFO | Downloaded 34GiB to GPU in 2.04223s (16.6485GiB/s) | 20260505 22:48:45.954 | INFO | Analyzing chunk frames = [6917120,6917632) | 20260505 22:48:45.954 | INFO | MemInfo: { | 20260505 22:48:45.954 | INFO | "gpu" : " 17 GB", | 20260505 22:48:45.954 | INFO | "huge" : "", | 20260505 22:48:45.954 | INFO | "huge_shared" : " 27 GB", | 20260505 22:48:45.954 | INFO | "malloc" : " 14 GB", | 20260505 22:48:45.954 | INFO | "pinned" : " 29 GB" | 20260505 22:48:45.954 | INFO | } | 20260505 22:48:45.954 | INFO | Perf report for the last 5.13561 seconds: | 20260505 22:48:45.954 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:48:45.954 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:45.954 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:45.954 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:45.954 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 22:48:45.954 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:45.954 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:45.954 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 22:48:45.954 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:45.954 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:45.954 | INFO | Uploaded 36GiB to GPU in 2.14839s (16.7568GiB/s) | 20260505 22:48:45.954 | INFO | Downloaded 34GiB to GPU in 2.15285s (15.793GiB/s) | 20260505 22:48:51.099 | INFO | Analyzing chunk frames = [6917632,6918144) | 20260505 22:48:51.099 | INFO | MemInfo: { | 20260505 22:48:51.099 | INFO | "gpu" : " 17 GB", | 20260505 22:48:51.099 | INFO | "huge" : "", | 20260505 22:48:51.099 | INFO | "huge_shared" : " 27 GB", | 20260505 22:48:51.099 | INFO | "malloc" : " 14 GB", | 20260505 22:48:51.099 | INFO | "pinned" : " 29 GB" | 20260505 22:48:51.099 | INFO | } | 20260505 22:48:51.100 | INFO | Perf report for the last 5.14583 seconds: | 20260505 22:48:51.100 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:48:51.100 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:51.100 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:51.100 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:51.100 | INFO | ANALYSIS : 62.1%, 3.0, 97.1ms, 0 | 20260505 22:48:51.100 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:51.100 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:51.100 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:48:51.100 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:51.100 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:51.100 | INFO | Uploaded 36GiB to GPU in 2.11925s (16.9872GiB/s) | 20260505 22:48:51.100 | INFO | Downloaded 34GiB to GPU in 2.01848s (16.8444GiB/s) | 20260505 22:48:51.226 | INFO | Starting a round of Baz IO | 20260505 22:48:54.371 | INFO | Populating RTMetrics report | 20260505 22:48:54.371 | INFO | RTMetrics report populated | 20260505 22:48:54.371 | INFO | Opening tmp RTMetrics json file | 20260505 22:48:54.372 | INFO | Serializing RTMetrics to json | 20260505 22:48:54.373 | INFO | Serialization done, swapping tmp file now... | 20260505 22:48:54.373 | INFO | RTMetrics done | 20260505 22:48:54.453 | INFO | Finished a round of Baz IO: Wrote 1.917 GiB in 3.227 seconds | 20260505 22:48:54.453 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.47% / 0.00% / 78.53% | 20260505 22:48:54.453 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.116881% / 0% / 7.15256e-05% | 20260505 22:48:54.453 | INFO | Baz IO overall: | 20260505 22:48:54.453 | INFO | BytesWritten : 3973271261210 | 20260505 22:48:54.453 | INFO | NumEvents : 1873296476143 | 20260505 22:48:56.241 | INFO | Analyzing chunk frames = [6918144,6918656) | 20260505 22:48:56.241 | INFO | MemInfo: { | 20260505 22:48:56.241 | INFO | "gpu" : " 17 GB", | 20260505 22:48:56.241 | INFO | "huge" : "", | 20260505 22:48:56.241 | INFO | "huge_shared" : " 27 GB", | 20260505 22:48:56.241 | INFO | "malloc" : " 14 GB", | 20260505 22:48:56.241 | INFO | "pinned" : " 29 GB" | 20260505 22:48:56.241 | INFO | } | 20260505 22:48:56.241 | INFO | Perf report for the last 5.14126 seconds: | 20260505 22:48:56.241 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:48:56.241 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:56.241 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:56.241 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:56.241 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 22:48:56.241 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:48:56.241 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260505 22:48:56.241 | INFO | PRE_HQ : 59.7%, 1.8, 54.5ms, 0 | 20260505 22:48:56.241 | INFO | BAZWRITER : 62.0%, 3.6, 115.1ms, 0 | 20260505 22:48:56.241 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 22:48:56.241 | INFO | Uploaded 36GiB to GPU in 2.16043s (16.6633GiB/s) | 20260505 22:48:56.241 | INFO | Downloaded 38GiB to GPU in 2.23248s (17.0214GiB/s) | 20260505 22:49:01.377 | INFO | Analyzing chunk frames = [6918656,6919168) | 20260505 22:49:01.377 | INFO | MemInfo: { | 20260505 22:49:01.377 | INFO | "gpu" : " 17 GB", | 20260505 22:49:01.377 | INFO | "huge" : "", | 20260505 22:49:01.377 | INFO | "huge_shared" : " 27 GB", | 20260505 22:49:01.377 | INFO | "malloc" : " 14 GB", | 20260505 22:49:01.377 | INFO | "pinned" : " 29 GB" | 20260505 22:49:01.377 | INFO | } | 20260505 22:49:01.377 | INFO | Perf report for the last 5.1363 seconds: | 20260505 22:49:01.377 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:49:01.377 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:01.377 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:01.377 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:01.377 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260505 22:49:01.377 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:01.377 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:01.377 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:49:01.377 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:01.377 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:01.377 | INFO | Uploaded 36GiB to GPU in 2.09348s (17.1962GiB/s) | 20260505 22:49:01.377 | INFO | Downloaded 34GiB to GPU in 2.03807s (16.6824GiB/s) | 20260505 22:49:06.514 | INFO | Analyzing chunk frames = [6919168,6919680) | 20260505 22:49:06.514 | INFO | MemInfo: { | 20260505 22:49:06.514 | INFO | "gpu" : " 17 GB", | 20260505 22:49:06.514 | INFO | "huge" : "", | 20260505 22:49:06.514 | INFO | "huge_shared" : " 27 GB", | 20260505 22:49:06.514 | INFO | "malloc" : " 14 GB", | 20260505 22:49:06.514 | INFO | "pinned" : " 29 GB" | 20260505 22:49:06.514 | INFO | } | 20260505 22:49:06.514 | INFO | Perf report for the last 5.13699 seconds: | 20260505 22:49:06.514 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:49:06.514 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:06.514 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:06.514 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:06.514 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260505 22:49:06.514 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:06.514 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:06.514 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:49:06.514 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:06.514 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:06.514 | INFO | Uploaded 36GiB to GPU in 2.16104s (16.6587GiB/s) | 20260505 22:49:06.514 | INFO | Downloaded 34GiB to GPU in 2.05116s (16.576GiB/s) | 20260505 22:49:11.654 | INFO | Analyzing chunk frames = [6919680,6920192) | 20260505 22:49:11.654 | INFO | MemInfo: { | 20260505 22:49:11.654 | INFO | "gpu" : " 17 GB", | 20260505 22:49:11.654 | INFO | "huge" : "", | 20260505 22:49:11.654 | INFO | "huge_shared" : " 27 GB", | 20260505 22:49:11.654 | INFO | "malloc" : " 14 GB", | 20260505 22:49:11.654 | INFO | "pinned" : " 29 GB" | 20260505 22:49:11.654 | INFO | } | 20260505 22:49:11.654 | INFO | Perf report for the last 5.13964 seconds: | 20260505 22:49:11.654 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:49:11.654 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:11.654 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:11.654 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:11.654 | INFO | ANALYSIS : 61.8%, 3.0, 96.0ms, 0 | 20260505 22:49:11.654 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:11.654 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:11.654 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:49:11.654 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:11.654 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:11.654 | INFO | Uploaded 36GiB to GPU in 2.143s (16.7989GiB/s) | 20260505 22:49:11.654 | INFO | Downloaded 34GiB to GPU in 2.03979s (16.6683GiB/s) | 20260505 22:49:16.796 | INFO | Analyzing chunk frames = [6920192,6920704) | 20260505 22:49:16.796 | INFO | MemInfo: { | 20260505 22:49:16.796 | INFO | "gpu" : " 17 GB", | 20260505 22:49:16.796 | INFO | "huge" : "", | 20260505 22:49:16.796 | INFO | "huge_shared" : " 27 GB", | 20260505 22:49:16.796 | INFO | "malloc" : " 14 GB", | 20260505 22:49:16.796 | INFO | "pinned" : " 29 GB" | 20260505 22:49:16.796 | INFO | } | 20260505 22:49:16.796 | INFO | Perf report for the last 5.14257 seconds: | 20260505 22:49:16.796 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:49:16.796 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:16.796 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:16.796 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:16.796 | INFO | ANALYSIS : 61.5%, 3.0, 96.1ms, 0 | 20260505 22:49:16.796 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:16.796 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:16.796 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 22:49:16.796 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:16.796 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:16.797 | INFO | Uploaded 36GiB to GPU in 2.15372s (16.7152GiB/s) | 20260505 22:49:16.797 | INFO | Downloaded 34GiB to GPU in 2.06339s (16.4777GiB/s) | 20260505 22:49:21.947 | INFO | Analyzing chunk frames = [6920704,6921216) | 20260505 22:49:21.947 | INFO | MemInfo: { | 20260505 22:49:21.947 | INFO | "gpu" : " 17 GB", | 20260505 22:49:21.947 | INFO | "huge" : "", | 20260505 22:49:21.947 | INFO | "huge_shared" : " 27 GB", | 20260505 22:49:21.947 | INFO | "malloc" : " 14 GB", | 20260505 22:49:21.947 | INFO | "pinned" : " 29 GB" | 20260505 22:49:21.947 | INFO | } | 20260505 22:49:21.947 | INFO | Perf report for the last 5.15058 seconds: | 20260505 22:49:21.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:49:21.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:21.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:21.947 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:21.947 | INFO | ANALYSIS : 61.4%, 3.0, 96.0ms, 0 | 20260505 22:49:21.947 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:21.947 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:21.947 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 22:49:21.947 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:21.947 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:21.947 | INFO | Uploaded 36GiB to GPU in 2.08141s (17.2959GiB/s) | 20260505 22:49:21.947 | INFO | Downloaded 34GiB to GPU in 2.03903s (16.6746GiB/s) | 20260505 22:49:27.078 | INFO | Analyzing chunk frames = [6921216,6921728) | 20260505 22:49:27.078 | INFO | MemInfo: { | 20260505 22:49:27.078 | INFO | "gpu" : " 17 GB", | 20260505 22:49:27.078 | INFO | "huge" : "", | 20260505 22:49:27.078 | INFO | "huge_shared" : " 27 GB", | 20260505 22:49:27.078 | INFO | "malloc" : " 14 GB", | 20260505 22:49:27.078 | INFO | "pinned" : " 29 GB" | 20260505 22:49:27.078 | INFO | } | 20260505 22:49:27.078 | INFO | Perf report for the last 5.13145 seconds: | 20260505 22:49:27.078 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:49:27.078 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:27.078 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:27.078 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:27.078 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260505 22:49:27.078 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:27.078 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:27.078 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 22:49:27.079 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:27.079 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:27.079 | INFO | Uploaded 36GiB to GPU in 2.09969s (17.1453GiB/s) | 20260505 22:49:27.079 | INFO | Downloaded 34GiB to GPU in 2.06741s (16.4457GiB/s) | 20260505 22:49:32.211 | INFO | Analyzing chunk frames = [6921728,6922240) | 20260505 22:49:32.211 | INFO | MemInfo: { | 20260505 22:49:32.211 | INFO | "gpu" : " 17 GB", | 20260505 22:49:32.211 | INFO | "huge" : "", | 20260505 22:49:32.211 | INFO | "huge_shared" : " 27 GB", | 20260505 22:49:32.211 | INFO | "malloc" : " 14 GB", | 20260505 22:49:32.211 | INFO | "pinned" : " 29 GB" | 20260505 22:49:32.211 | INFO | } | 20260505 22:49:32.211 | INFO | Perf report for the last 5.1324 seconds: | 20260505 22:49:32.211 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:49:32.211 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:32.211 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:32.211 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:32.211 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260505 22:49:32.211 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:32.211 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:32.211 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:49:32.211 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:32.211 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:32.211 | INFO | Uploaded 36GiB to GPU in 2.11097s (17.0537GiB/s) | 20260505 22:49:32.211 | INFO | Downloaded 34GiB to GPU in 1.99965s (17.003GiB/s) | 20260505 22:49:32.211 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 22:49:32.211 | INFO | Total mem usage: 15.3 GiB | 20260505 22:49:32.211 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 22:49:32.211 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -9.08%) | 20260505 22:49:32.211 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 22:49:32.211 | INFO | High Water Breakdown: | 20260505 22:49:32.211 | INFO | 60.9% : 8.56 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 22:49:32.211 | INFO | 39% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 22:49:32.211 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 22:49:32.211 | INFO | ------------------------------------------ | 20260505 22:49:32.211 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:49:32.211 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 22:49:32.211 | INFO | Total mem usage: 26.5 GiB | 20260505 22:49:32.211 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 22:49:32.211 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 22:49:32.211 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 22:49:32.211 | INFO | High Water Breakdown: | 20260505 22:49:32.211 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:49:32.211 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:49:32.211 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:49:32.211 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:49:32.211 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:49:32.211 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:49:32.211 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 22:49:32.212 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 22:49:32.212 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:49:32.212 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 22:49:32.212 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 22:49:32.212 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 22:49:32.212 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 22:49:32.212 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 22:49:32.212 | INFO | ------------------------------------------ | 20260505 22:49:32.212 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:49:32.212 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 22:49:32.212 | INFO | Total mem usage: 25.6 GiB | 20260505 22:49:32.212 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 22:49:32.212 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 22:49:32.212 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 22:49:32.212 | INFO | High Water Breakdown: | 20260505 22:49:32.212 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 22:49:32.212 | INFO | ------------------------------------------ | 20260505 22:49:32.212 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:49:32.212 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 22:49:32.212 | INFO | Total mem usage: 15.7 GiB | 20260505 22:49:32.212 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 22:49:32.212 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 22:49:32.212 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 22:49:32.212 | INFO | High Water Breakdown: | 20260505 22:49:32.212 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 22:49:32.212 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 22:49:32.212 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 22:49:32.212 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 22:49:32.212 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 22:49:32.212 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 22:49:32.212 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 22:49:32.212 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 22:49:32.212 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 22:49:32.212 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:49:32.212 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 22:49:32.212 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:49:32.212 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 22:49:32.212 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 22:49:32.212 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 22:49:32.212 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 22:49:32.212 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:49:32.212 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 22:49:32.212 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:49:32.212 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:49:32.212 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:49:32.212 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 22:49:32.212 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 22:49:32.212 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:49:32.212 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 22:49:32.212 | INFO | ------------------------------------------ | 20260505 22:49:32.212 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:49:32.212 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 22:49:32.212 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:49:32.212 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:49:32.212 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.40e+01, 1.34e+01 / 1.27e+00 | 20260505 22:49:32.212 | INFO | PerfReport: 0.29%, AggregateStats -- 9.24e-02 / 1.04e-01, 1.06e-02 / 4.88e-03 | 20260505 22:49:32.212 | INFO | PerfReport: 11.82%, Baseline -- 3.71e+00 / 4.49e+00, 5.66e-01 / 1.58e-01 | 20260505 22:49:32.212 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.49e+00, 1.86e-01 / 9.28e-02 | 20260505 22:49:32.212 | INFO | PerfReport: 15.96%, DME -- 4.00e+01 / 4.40e+01, 3.82e+00 / 1.41e+00 | 20260505 22:49:32.212 | INFO | PerfReport: 57.30%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.51e+00 / 1.05e+00 | 20260505 22:49:32.212 | INFO | PerfReport: 7.13%, PulseAccumulating -- 2.24e+00 / 2.74e+00, 4.27e-01 / 2.19e-01 | 20260505 22:49:32.212 | INFO | PerfReport: 3.13%, Metrics -- 9.83e-01 / 1.26e+00, 2.69e-01 / 9.05e-02 | 20260505 22:49:32.212 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.41e-01 / 4.04e-01, 5.90e-02 / 3.32e-02 | 20260505 22:49:32.212 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 22:49:32.212 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:49:32.212 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:49:32.212 | INFO | PerfReport: Full Execution -- 9.86e+01 / -nan, 2.47e+01 / 0.00e+00 | 20260505 22:49:32.212 | INFO | PerfReport: 22.43%, Upload -- 2.21e+01 / 0.00e+00, 7.56e+00 / 0.00e+00 | 20260505 22:49:32.212 | INFO | PerfReport: 21.86%, Download -- 2.15e+01 / 0.00e+00, 7.64e+00 / 0.00e+00 | 20260505 22:49:32.212 | INFO | PerfReport: 31.83%, Compute -- 3.14e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 22:49:32.212 | INFO | PerfReport: 2.13%, LockUpload -- 2.10e+00 / 0.00e+00, 7.84e+00 / 0.00e+00 | 20260505 22:49:32.212 | INFO | PerfReport: 1.35%, LockDownload -- 1.33e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260505 22:49:32.212 | INFO | PerfReport: 20.40%, LockCompute -- 2.01e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260505 22:49:32.212 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.64e-02 / 0.00e+00 | 20260505 22:49:32.339 | INFO | Starting a round of Baz IO | 20260505 22:49:35.485 | INFO | Populating RTMetrics report | 20260505 22:49:35.485 | INFO | RTMetrics report populated | 20260505 22:49:35.485 | INFO | Opening tmp RTMetrics json file | 20260505 22:49:35.488 | INFO | Serializing RTMetrics to json | 20260505 22:49:35.490 | INFO | Serialization done, swapping tmp file now... | 20260505 22:49:35.492 | INFO | RTMetrics done | 20260505 22:49:35.564 | INFO | Finished a round of Baz IO: Wrote 1.916 GiB in 3.225 seconds | 20260505 22:49:35.565 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.47% / 0.00% / 78.53% | 20260505 22:49:35.565 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.116515% / 0% / 9.53674e-05% | 20260505 22:49:35.565 | INFO | Baz IO overall: | 20260505 22:49:35.565 | INFO | BytesWritten : 3975328043755 | 20260505 22:49:35.565 | INFO | NumEvents : 1874241680009 | 20260505 22:49:37.356 | INFO | Analyzing chunk frames = [6922240,6922752) | 20260505 22:49:37.356 | INFO | MemInfo: { | 20260505 22:49:37.356 | INFO | "gpu" : " 17 GB", | 20260505 22:49:37.356 | INFO | "huge" : "", | 20260505 22:49:37.356 | INFO | "huge_shared" : " 27 GB", | 20260505 22:49:37.356 | INFO | "malloc" : " 14 GB", | 20260505 22:49:37.356 | INFO | "pinned" : " 29 GB" | 20260505 22:49:37.356 | INFO | } | 20260505 22:49:37.356 | INFO | Perf report for the last 5.14544 seconds: | 20260505 22:49:37.356 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:49:37.356 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:37.356 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:37.356 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:37.356 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260505 22:49:37.356 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:37.356 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260505 22:49:37.357 | INFO | PRE_HQ : 59.6%, 1.8, 54.4ms, 0 | 20260505 22:49:37.357 | INFO | BAZWRITER : 62.0%, 3.6, 114.5ms, 0 | 20260505 22:49:37.357 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 22:49:37.357 | INFO | Uploaded 36GiB to GPU in 2.12966s (16.9041GiB/s) | 20260505 22:49:37.357 | INFO | Downloaded 38GiB to GPU in 2.22584s (17.0722GiB/s) | 20260505 22:49:42.494 | INFO | Analyzing chunk frames = [6922752,6923264) | 20260505 22:49:42.494 | INFO | MemInfo: { | 20260505 22:49:42.494 | INFO | "gpu" : " 17 GB", | 20260505 22:49:42.494 | INFO | "huge" : "", | 20260505 22:49:42.494 | INFO | "huge_shared" : " 27 GB", | 20260505 22:49:42.494 | INFO | "malloc" : " 14 GB", | 20260505 22:49:42.494 | INFO | "pinned" : " 29 GB" | 20260505 22:49:42.494 | INFO | } | 20260505 22:49:42.494 | INFO | Perf report for the last 5.13733 seconds: | 20260505 22:49:42.494 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:49:42.494 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:42.494 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:42.494 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:42.494 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260505 22:49:42.494 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:42.494 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:42.494 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260505 22:49:42.494 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:42.494 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:42.494 | INFO | Uploaded 36GiB to GPU in 2.07854s (17.3199GiB/s) | 20260505 22:49:42.494 | INFO | Downloaded 34GiB to GPU in 2.02474s (16.7923GiB/s) | 20260505 22:49:47.630 | INFO | Analyzing chunk frames = [6923264,6923776) | 20260505 22:49:47.630 | INFO | MemInfo: { | 20260505 22:49:47.630 | INFO | "gpu" : " 17 GB", | 20260505 22:49:47.630 | INFO | "huge" : "", | 20260505 22:49:47.630 | INFO | "huge_shared" : " 27 GB", | 20260505 22:49:47.630 | INFO | "malloc" : " 14 GB", | 20260505 22:49:47.630 | INFO | "pinned" : " 29 GB" | 20260505 22:49:47.630 | INFO | } | 20260505 22:49:47.630 | INFO | Perf report for the last 5.13645 seconds: | 20260505 22:49:47.630 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:49:47.630 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:47.630 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:47.630 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:47.630 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260505 22:49:47.630 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:47.630 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:47.630 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 22:49:47.630 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:47.630 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:47.630 | INFO | Uploaded 36GiB to GPU in 2.10776s (17.0798GiB/s) | 20260505 22:49:47.630 | INFO | Downloaded 34GiB to GPU in 2.02645s (16.7781GiB/s) | 20260505 22:49:52.772 | INFO | Analyzing chunk frames = [6923776,6924288) | 20260505 22:49:52.772 | INFO | MemInfo: { | 20260505 22:49:52.772 | INFO | "gpu" : " 17 GB", | 20260505 22:49:52.772 | INFO | "huge" : "", | 20260505 22:49:52.772 | INFO | "huge_shared" : " 27 GB", | 20260505 22:49:52.772 | INFO | "malloc" : " 14 GB", | 20260505 22:49:52.772 | INFO | "pinned" : " 29 GB" | 20260505 22:49:52.772 | INFO | } | 20260505 22:49:52.772 | INFO | Perf report for the last 5.1422 seconds: | 20260505 22:49:52.772 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:49:52.772 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:52.772 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:52.772 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:52.772 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260505 22:49:52.772 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:52.772 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:52.772 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:49:52.772 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:52.772 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:52.773 | INFO | Uploaded 36GiB to GPU in 2.14106s (16.8141GiB/s) | 20260505 22:49:52.773 | INFO | Downloaded 34GiB to GPU in 2.04041s (16.6633GiB/s) | 20260505 22:49:57.907 | INFO | Analyzing chunk frames = [6924288,6924800) | 20260505 22:49:57.907 | INFO | MemInfo: { | 20260505 22:49:57.907 | INFO | "gpu" : " 17 GB", | 20260505 22:49:57.907 | INFO | "huge" : "", | 20260505 22:49:57.907 | INFO | "huge_shared" : " 27 GB", | 20260505 22:49:57.907 | INFO | "malloc" : " 14 GB", | 20260505 22:49:57.907 | INFO | "pinned" : " 29 GB" | 20260505 22:49:57.907 | INFO | } | 20260505 22:49:57.908 | INFO | Perf report for the last 5.13524 seconds: | 20260505 22:49:57.908 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:49:57.908 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:57.908 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:57.908 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:57.908 | INFO | ANALYSIS : 61.6%, 3.0, 96.2ms, 0 | 20260505 22:49:57.908 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:57.908 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:57.908 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:49:57.908 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:57.908 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:49:57.908 | INFO | Uploaded 36GiB to GPU in 2.15663s (16.6927GiB/s) | 20260505 22:49:57.908 | INFO | Downloaded 34GiB to GPU in 2.06926s (16.431GiB/s) | 20260505 22:50:03.049 | INFO | Analyzing chunk frames = [6924800,6925312) | 20260505 22:50:03.049 | INFO | MemInfo: { | 20260505 22:50:03.049 | INFO | "gpu" : " 17 GB", | 20260505 22:50:03.049 | INFO | "huge" : "", | 20260505 22:50:03.049 | INFO | "huge_shared" : " 27 GB", | 20260505 22:50:03.049 | INFO | "malloc" : " 14 GB", | 20260505 22:50:03.049 | INFO | "pinned" : " 29 GB" | 20260505 22:50:03.049 | INFO | } | 20260505 22:50:03.049 | INFO | Perf report for the last 5.14168 seconds: | 20260505 22:50:03.049 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:50:03.049 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:03.049 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:03.049 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:03.049 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260505 22:50:03.049 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:03.049 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:03.049 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 22:50:03.049 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:03.049 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:03.050 | INFO | Uploaded 36GiB to GPU in 2.08895s (17.2335GiB/s) | 20260505 22:50:03.050 | INFO | Downloaded 34GiB to GPU in 2.04811s (16.6007GiB/s) | 20260505 22:50:08.194 | INFO | Analyzing chunk frames = [6925312,6925824) | 20260505 22:50:08.194 | INFO | MemInfo: { | 20260505 22:50:08.194 | INFO | "gpu" : " 17 GB", | 20260505 22:50:08.194 | INFO | "huge" : "", | 20260505 22:50:08.194 | INFO | "huge_shared" : " 27 GB", | 20260505 22:50:08.194 | INFO | "malloc" : " 14 GB", | 20260505 22:50:08.194 | INFO | "pinned" : " 29 GB" | 20260505 22:50:08.194 | INFO | } | 20260505 22:50:08.194 | INFO | Perf report for the last 5.14489 seconds: | 20260505 22:50:08.194 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:50:08.194 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:08.194 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:08.194 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:08.194 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260505 22:50:08.194 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:08.194 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:08.194 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260505 22:50:08.194 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:08.194 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:08.194 | INFO | Uploaded 36GiB to GPU in 2.10385s (17.1114GiB/s) | 20260505 22:50:08.195 | INFO | Downloaded 34GiB to GPU in 2.09196s (16.2527GiB/s) | 20260505 22:50:13.326 | INFO | Analyzing chunk frames = [6925824,6926336) | 20260505 22:50:13.326 | INFO | MemInfo: { | 20260505 22:50:13.326 | INFO | "gpu" : " 17 GB", | 20260505 22:50:13.326 | INFO | "huge" : "", | 20260505 22:50:13.326 | INFO | "huge_shared" : " 27 GB", | 20260505 22:50:13.326 | INFO | "malloc" : " 14 GB", | 20260505 22:50:13.326 | INFO | "pinned" : " 29 GB" | 20260505 22:50:13.326 | INFO | } | 20260505 22:50:13.326 | INFO | Perf report for the last 5.13225 seconds: | 20260505 22:50:13.326 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:50:13.326 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:13.326 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:13.326 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:13.326 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260505 22:50:13.326 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:13.326 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:13.327 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 22:50:13.327 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:13.327 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:13.327 | INFO | Uploaded 36GiB to GPU in 2.12692s (16.9259GiB/s) | 20260505 22:50:13.327 | INFO | Downloaded 34GiB to GPU in 2.0101s (16.9145GiB/s) | 20260505 22:50:13.452 | INFO | Starting a round of Baz IO | 20260505 22:50:16.596 | INFO | Populating RTMetrics report | 20260505 22:50:16.597 | INFO | RTMetrics report populated | 20260505 22:50:16.597 | INFO | Opening tmp RTMetrics json file | 20260505 22:50:16.597 | INFO | Serializing RTMetrics to json | 20260505 22:50:16.600 | INFO | Serialization done, swapping tmp file now... | 20260505 22:50:16.600 | INFO | RTMetrics done | 20260505 22:50:16.683 | INFO | Finished a round of Baz IO: Wrote 1.913 GiB in 3.23 seconds | 20260505 22:50:16.683 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.47% / 0.00% / 78.53% | 20260505 22:50:16.683 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.11675% / 3.97364e-06% / 7.94729e-05% | 20260505 22:50:16.683 | INFO | Baz IO overall: | 20260505 22:50:16.683 | INFO | BytesWritten : 3977381988003 | 20260505 22:50:16.683 | INFO | NumEvents : 1875184847001 | 20260505 22:50:18.461 | INFO | Analyzing chunk frames = [6926336,6926848) | 20260505 22:50:18.461 | INFO | MemInfo: { | 20260505 22:50:18.461 | INFO | "gpu" : " 17 GB", | 20260505 22:50:18.461 | INFO | "huge" : "", | 20260505 22:50:18.461 | INFO | "huge_shared" : " 27 GB", | 20260505 22:50:18.461 | INFO | "malloc" : " 14 GB", | 20260505 22:50:18.461 | INFO | "pinned" : " 29 GB" | 20260505 22:50:18.462 | INFO | } | 20260505 22:50:18.462 | INFO | Perf report for the last 5.13515 seconds: | 20260505 22:50:18.462 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:50:18.462 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:18.462 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:18.462 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:18.462 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 22:50:18.462 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:18.462 | INFO | RT_METRICS : 43.6%, 1.0, 22.8ms, 0 | 20260505 22:50:18.462 | INFO | PRE_HQ : 60.2%, 1.8, 55.0ms, 0 | 20260505 22:50:18.462 | INFO | BAZWRITER : 62.1%, 3.6, 114.7ms, 0 | 20260505 22:50:18.462 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 22:50:18.462 | INFO | Uploaded 36GiB to GPU in 2.165s (16.6281GiB/s) | 20260505 22:50:18.462 | INFO | Downloaded 38GiB to GPU in 2.23497s (17.0025GiB/s) | 20260505 22:50:23.608 | INFO | Analyzing chunk frames = [6926848,6927360) | 20260505 22:50:23.608 | INFO | MemInfo: { | 20260505 22:50:23.608 | INFO | "gpu" : " 17 GB", | 20260505 22:50:23.608 | INFO | "huge" : "", | 20260505 22:50:23.608 | INFO | "huge_shared" : " 27 GB", | 20260505 22:50:23.608 | INFO | "malloc" : " 14 GB", | 20260505 22:50:23.608 | INFO | "pinned" : " 29 GB" | 20260505 22:50:23.608 | INFO | } | 20260505 22:50:23.608 | INFO | Perf report for the last 5.14669 seconds: | 20260505 22:50:23.608 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:50:23.608 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:23.608 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:23.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:23.608 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260505 22:50:23.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:23.608 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:23.608 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:50:23.608 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:23.608 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:23.609 | INFO | Uploaded 36GiB to GPU in 2.08258s (17.2863GiB/s) | 20260505 22:50:23.609 | INFO | Downloaded 34GiB to GPU in 2.01949s (16.8359GiB/s) | 20260505 22:50:28.750 | INFO | Analyzing chunk frames = [6927360,6927872) | 20260505 22:50:28.750 | INFO | MemInfo: { | 20260505 22:50:28.750 | INFO | "gpu" : " 17 GB", | 20260505 22:50:28.750 | INFO | "huge" : "", | 20260505 22:50:28.750 | INFO | "huge_shared" : " 27 GB", | 20260505 22:50:28.750 | INFO | "malloc" : " 14 GB", | 20260505 22:50:28.750 | INFO | "pinned" : " 29 GB" | 20260505 22:50:28.750 | INFO | } | 20260505 22:50:28.750 | INFO | Perf report for the last 5.14164 seconds: | 20260505 22:50:28.750 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:50:28.750 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:28.750 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:28.750 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:28.750 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260505 22:50:28.750 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:28.750 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:28.750 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 22:50:28.750 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:28.750 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:28.750 | INFO | Uploaded 36GiB to GPU in 2.08879s (17.2348GiB/s) | 20260505 22:50:28.750 | INFO | Downloaded 34GiB to GPU in 2.03739s (16.688GiB/s) | 20260505 22:50:33.889 | INFO | Analyzing chunk frames = [6927872,6928384) | 20260505 22:50:33.889 | INFO | MemInfo: { | 20260505 22:50:33.889 | INFO | "gpu" : " 17 GB", | 20260505 22:50:33.889 | INFO | "huge" : "", | 20260505 22:50:33.889 | INFO | "huge_shared" : " 27 GB", | 20260505 22:50:33.889 | INFO | "malloc" : " 14 GB", | 20260505 22:50:33.889 | INFO | "pinned" : " 29 GB" | 20260505 22:50:33.889 | INFO | } | 20260505 22:50:33.889 | INFO | Perf report for the last 5.13919 seconds: | 20260505 22:50:33.889 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:50:33.889 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:33.889 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:33.889 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:33.889 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260505 22:50:33.889 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:33.889 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:33.889 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:50:33.889 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:33.889 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:33.889 | INFO | Uploaded 36GiB to GPU in 2.12946s (16.9057GiB/s) | 20260505 22:50:33.889 | INFO | Downloaded 34GiB to GPU in 2.01624s (16.8631GiB/s) | 20260505 22:50:39.028 | INFO | Analyzing chunk frames = [6928384,6928896) | 20260505 22:50:39.028 | INFO | MemInfo: { | 20260505 22:50:39.028 | INFO | "gpu" : " 17 GB", | 20260505 22:50:39.028 | INFO | "huge" : "", | 20260505 22:50:39.028 | INFO | "huge_shared" : " 27 GB", | 20260505 22:50:39.028 | INFO | "malloc" : " 14 GB", | 20260505 22:50:39.028 | INFO | "pinned" : " 29 GB" | 20260505 22:50:39.028 | INFO | } | 20260505 22:50:39.028 | INFO | Perf report for the last 5.13931 seconds: | 20260505 22:50:39.028 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:50:39.028 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:39.028 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:39.028 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:39.029 | INFO | ANALYSIS : 61.2%, 3.0, 95.6ms, 0 | 20260505 22:50:39.029 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:39.029 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:39.029 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 22:50:39.029 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:39.029 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:39.029 | INFO | Uploaded 36GiB to GPU in 2.1071s (17.0851GiB/s) | 20260505 22:50:39.029 | INFO | Downloaded 34GiB to GPU in 2.05253s (16.5649GiB/s) | 20260505 22:50:44.167 | INFO | Analyzing chunk frames = [6928896,6929408) | 20260505 22:50:44.167 | INFO | MemInfo: { | 20260505 22:50:44.167 | INFO | "gpu" : " 17 GB", | 20260505 22:50:44.167 | INFO | "huge" : "", | 20260505 22:50:44.167 | INFO | "huge_shared" : " 27 GB", | 20260505 22:50:44.167 | INFO | "malloc" : " 14 GB", | 20260505 22:50:44.167 | INFO | "pinned" : " 29 GB" | 20260505 22:50:44.167 | INFO | } | 20260505 22:50:44.167 | INFO | Perf report for the last 5.1389 seconds: | 20260505 22:50:44.167 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:50:44.167 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:44.167 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:44.167 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:44.167 | INFO | ANALYSIS : 61.4%, 3.0, 95.9ms, 0 | 20260505 22:50:44.167 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:44.167 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:44.167 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 22:50:44.167 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:44.167 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:44.168 | INFO | Uploaded 36GiB to GPU in 2.06505s (17.433GiB/s) | 20260505 22:50:44.168 | INFO | Downloaded 34GiB to GPU in 2.03129s (16.7382GiB/s) | 20260505 22:50:49.305 | INFO | Analyzing chunk frames = [6929408,6929920) | 20260505 22:50:49.305 | INFO | MemInfo: { | 20260505 22:50:49.305 | INFO | "gpu" : " 17 GB", | 20260505 22:50:49.305 | INFO | "huge" : "", | 20260505 22:50:49.305 | INFO | "huge_shared" : " 27 GB", | 20260505 22:50:49.305 | INFO | "malloc" : " 14 GB", | 20260505 22:50:49.305 | INFO | "pinned" : " 29 GB" | 20260505 22:50:49.305 | INFO | } | 20260505 22:50:49.305 | INFO | Perf report for the last 5.13769 seconds: | 20260505 22:50:49.305 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:50:49.305 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:49.305 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:49.305 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:49.305 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 22:50:49.305 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:49.305 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:49.305 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 22:50:49.305 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:49.305 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:49.305 | INFO | Uploaded 36GiB to GPU in 2.01561s (17.8606GiB/s) | 20260505 22:50:49.305 | INFO | Downloaded 34GiB to GPU in 2.03998s (16.6668GiB/s) | 20260505 22:50:54.448 | INFO | Analyzing chunk frames = [6929920,6930432) | 20260505 22:50:54.448 | INFO | MemInfo: { | 20260505 22:50:54.448 | INFO | "gpu" : " 17 GB", | 20260505 22:50:54.448 | INFO | "huge" : "", | 20260505 22:50:54.448 | INFO | "huge_shared" : " 27 GB", | 20260505 22:50:54.448 | INFO | "malloc" : " 14 GB", | 20260505 22:50:54.448 | INFO | "pinned" : " 29 GB" | 20260505 22:50:54.448 | INFO | } | 20260505 22:50:54.448 | INFO | Perf report for the last 5.14298 seconds: | 20260505 22:50:54.448 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:50:54.448 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:54.448 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:54.448 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:54.448 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260505 22:50:54.448 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:54.448 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:54.448 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 22:50:54.448 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:54.448 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:54.449 | INFO | Uploaded 36GiB to GPU in 2.11059s (17.0568GiB/s) | 20260505 22:50:54.449 | INFO | Downloaded 34GiB to GPU in 2.00881s (16.9255GiB/s) | 20260505 22:50:54.449 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 22:50:54.449 | INFO | Total mem usage: 15.3 GiB | 20260505 22:50:54.449 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 22:50:54.449 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -9.07%) | 20260505 22:50:54.449 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 22:50:54.449 | INFO | High Water Breakdown: | 20260505 22:50:54.449 | INFO | 60.9% : 8.56 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 22:50:54.449 | INFO | 39% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 22:50:54.449 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 22:50:54.449 | INFO | ------------------------------------------ | 20260505 22:50:54.449 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:50:54.449 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 22:50:54.449 | INFO | Total mem usage: 26.5 GiB | 20260505 22:50:54.449 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 22:50:54.449 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 22:50:54.449 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 22:50:54.449 | INFO | High Water Breakdown: | 20260505 22:50:54.449 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:50:54.449 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:50:54.449 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:50:54.449 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:50:54.449 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:50:54.449 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:50:54.449 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 22:50:54.449 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 22:50:54.449 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:50:54.449 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 22:50:54.449 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 22:50:54.449 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 22:50:54.449 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 22:50:54.449 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 22:50:54.449 | INFO | ------------------------------------------ | 20260505 22:50:54.449 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:50:54.449 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 22:50:54.449 | INFO | Total mem usage: 25.6 GiB | 20260505 22:50:54.449 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 22:50:54.449 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 22:50:54.449 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 22:50:54.449 | INFO | High Water Breakdown: | 20260505 22:50:54.449 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 22:50:54.449 | INFO | ------------------------------------------ | 20260505 22:50:54.449 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:50:54.450 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 22:50:54.450 | INFO | Total mem usage: 15.7 GiB | 20260505 22:50:54.450 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 22:50:54.450 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 22:50:54.450 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 22:50:54.450 | INFO | High Water Breakdown: | 20260505 22:50:54.450 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 22:50:54.450 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 22:50:54.450 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 22:50:54.450 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 22:50:54.450 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 22:50:54.450 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 22:50:54.450 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 22:50:54.450 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 22:50:54.450 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 22:50:54.450 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:50:54.450 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 22:50:54.450 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:50:54.450 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 22:50:54.450 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 22:50:54.450 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 22:50:54.450 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 22:50:54.450 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:50:54.450 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 22:50:54.450 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:50:54.450 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:50:54.450 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:50:54.450 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 22:50:54.450 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 22:50:54.450 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:50:54.450 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 22:50:54.450 | INFO | ------------------------------------------ | 20260505 22:50:54.450 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:50:54.450 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 22:50:54.450 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:50:54.450 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:50:54.450 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.42e+01, 1.35e+01 / 8.93e-01 | 20260505 22:50:54.450 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.03e-01, 1.06e-02 / 3.94e-03 | 20260505 22:50:54.450 | INFO | PerfReport: 11.82%, Baseline -- 3.71e+00 / 4.51e+00, 5.75e-01 / 2.00e-01 | 20260505 22:50:54.450 | INFO | PerfReport: 3.29%, Histogram -- 1.03e+00 / 1.53e+00, 1.94e-01 / 8.48e-02 | 20260505 22:50:54.450 | INFO | PerfReport: 15.96%, DME -- 4.01e+01 / 4.42e+01, 3.96e+00 / 1.11e+00 | 20260505 22:50:54.450 | INFO | PerfReport: 57.31%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.56e+00 / 1.04e+00 | 20260505 22:50:54.450 | INFO | PerfReport: 7.10%, PulseAccumulating -- 2.23e+00 / 2.76e+00, 4.27e-01 / 2.26e-01 | 20260505 22:50:54.450 | INFO | PerfReport: 3.13%, Metrics -- 9.82e-01 / 1.27e+00, 2.68e-01 / 1.11e-01 | 20260505 22:50:54.450 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.41e-01 / 4.02e-01, 5.93e-02 / 3.47e-02 | 20260505 22:50:54.450 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 22:50:54.450 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:50:54.450 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:50:54.450 | INFO | PerfReport: Full Execution -- 9.85e+01 / -nan, 2.48e+01 / 0.00e+00 | 20260505 22:50:54.450 | INFO | PerfReport: 22.26%, Upload -- 2.19e+01 / 0.00e+00, 7.47e+00 / 0.00e+00 | 20260505 22:50:54.450 | INFO | PerfReport: 21.79%, Download -- 2.15e+01 / 0.00e+00, 7.62e+00 / 0.00e+00 | 20260505 22:50:54.450 | INFO | PerfReport: 31.89%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 22:50:54.450 | INFO | PerfReport: 2.13%, LockUpload -- 2.10e+00 / 0.00e+00, 7.85e+00 / 0.00e+00 | 20260505 22:50:54.450 | INFO | PerfReport: 1.31%, LockDownload -- 1.29e+00 / 0.00e+00, 2.96e+00 / 0.00e+00 | 20260505 22:50:54.450 | INFO | PerfReport: 20.60%, LockCompute -- 2.03e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260505 22:50:54.450 | INFO | PerfReport: 0.01%, Misc -- 1.27e-02 / 0.00e+00, 1.51e-02 / 0.00e+00 | 20260505 22:50:54.568 | INFO | Starting a round of Baz IO | 20260505 22:50:57.719 | INFO | Populating RTMetrics report | 20260505 22:50:57.719 | INFO | RTMetrics report populated | 20260505 22:50:57.719 | INFO | Opening tmp RTMetrics json file | 20260505 22:50:57.720 | INFO | Serializing RTMetrics to json | 20260505 22:50:57.724 | INFO | Serialization done, swapping tmp file now... | 20260505 22:50:57.728 | INFO | RTMetrics done | 20260505 22:50:57.800 | INFO | Finished a round of Baz IO: Wrote 1.914 GiB in 3.231 seconds | 20260505 22:50:57.800 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.47% / 0.00% / 78.53% | 20260505 22:50:57.800 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.117% / 0% / 9.93411e-05% | 20260505 22:50:57.800 | INFO | Baz IO overall: | 20260505 22:50:57.800 | INFO | BytesWritten : 3979436668759 | 20260505 22:50:57.800 | INFO | NumEvents : 1876128547357 | 20260505 22:50:59.578 | INFO | Analyzing chunk frames = [6930432,6930944) | 20260505 22:50:59.578 | INFO | MemInfo: { | 20260505 22:50:59.578 | INFO | "gpu" : " 17 GB", | 20260505 22:50:59.578 | INFO | "huge" : "", | 20260505 22:50:59.578 | INFO | "huge_shared" : " 27 GB", | 20260505 22:50:59.578 | INFO | "malloc" : " 14 GB", | 20260505 22:50:59.578 | INFO | "pinned" : " 29 GB" | 20260505 22:50:59.578 | INFO | } | 20260505 22:50:59.578 | INFO | Perf report for the last 5.1303 seconds: | 20260505 22:50:59.578 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:50:59.578 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:59.578 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:59.578 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:59.578 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260505 22:50:59.578 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:50:59.578 | INFO | RT_METRICS : 43.6%, 1.0, 22.8ms, 0 | 20260505 22:50:59.578 | INFO | PRE_HQ : 59.9%, 1.7, 54.2ms, 0 | 20260505 22:50:59.578 | INFO | BAZWRITER : 62.1%, 3.6, 114.0ms, 0 | 20260505 22:50:59.578 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 22:50:59.579 | INFO | Uploaded 36GiB to GPU in 2.13381s (16.8712GiB/s) | 20260505 22:50:59.579 | INFO | Downloaded 38GiB to GPU in 2.23675s (16.9889GiB/s) | 20260505 22:51:04.722 | INFO | Analyzing chunk frames = [6930944,6931456) | 20260505 22:51:04.722 | INFO | MemInfo: { | 20260505 22:51:04.722 | INFO | "gpu" : " 17 GB", | 20260505 22:51:04.722 | INFO | "huge" : "", | 20260505 22:51:04.722 | INFO | "huge_shared" : " 27 GB", | 20260505 22:51:04.722 | INFO | "malloc" : " 14 GB", | 20260505 22:51:04.722 | INFO | "pinned" : " 29 GB" | 20260505 22:51:04.722 | INFO | } | 20260505 22:51:04.722 | INFO | Perf report for the last 5.14394 seconds: | 20260505 22:51:04.722 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:51:04.722 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:04.722 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:04.722 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:04.722 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 22:51:04.722 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:04.722 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:04.722 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260505 22:51:04.722 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:04.722 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:04.723 | INFO | Uploaded 36GiB to GPU in 2.082s (17.291GiB/s) | 20260505 22:51:04.723 | INFO | Downloaded 34GiB to GPU in 2.02002s (16.8315GiB/s) | 20260505 22:51:09.863 | INFO | Analyzing chunk frames = [6931456,6931968) | 20260505 22:51:09.863 | INFO | MemInfo: { | 20260505 22:51:09.863 | INFO | "gpu" : " 17 GB", | 20260505 22:51:09.863 | INFO | "huge" : "", | 20260505 22:51:09.863 | INFO | "huge_shared" : " 27 GB", | 20260505 22:51:09.863 | INFO | "malloc" : " 14 GB", | 20260505 22:51:09.863 | INFO | "pinned" : " 29 GB" | 20260505 22:51:09.863 | INFO | } | 20260505 22:51:09.863 | INFO | Perf report for the last 5.1406 seconds: | 20260505 22:51:09.863 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:51:09.863 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:09.863 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:09.863 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:09.863 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260505 22:51:09.863 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:09.863 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:09.863 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:51:09.863 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:09.863 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:09.863 | INFO | Uploaded 36GiB to GPU in 2.15017s (16.7428GiB/s) | 20260505 22:51:09.863 | INFO | Downloaded 34GiB to GPU in 2.04892s (16.5941GiB/s) | 20260505 22:51:15.005 | INFO | Analyzing chunk frames = [6931968,6932480) | 20260505 22:51:15.005 | INFO | MemInfo: { | 20260505 22:51:15.005 | INFO | "gpu" : " 17 GB", | 20260505 22:51:15.005 | INFO | "huge" : "", | 20260505 22:51:15.005 | INFO | "huge_shared" : " 27 GB", | 20260505 22:51:15.005 | INFO | "malloc" : " 14 GB", | 20260505 22:51:15.005 | INFO | "pinned" : " 29 GB" | 20260505 22:51:15.005 | INFO | } | 20260505 22:51:15.005 | INFO | Perf report for the last 5.14203 seconds: | 20260505 22:51:15.005 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:51:15.005 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:15.005 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:15.005 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:15.005 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260505 22:51:15.005 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:15.005 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:15.005 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:51:15.005 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:15.005 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:15.005 | INFO | Uploaded 36GiB to GPU in 2.1421s (16.806GiB/s) | 20260505 22:51:15.005 | INFO | Downloaded 34GiB to GPU in 2.01701s (16.8566GiB/s) | 20260505 22:51:20.139 | INFO | Analyzing chunk frames = [6932480,6932992) | 20260505 22:51:20.139 | INFO | MemInfo: { | 20260505 22:51:20.139 | INFO | "gpu" : " 17 GB", | 20260505 22:51:20.139 | INFO | "huge" : "", | 20260505 22:51:20.139 | INFO | "huge_shared" : " 27 GB", | 20260505 22:51:20.140 | INFO | "malloc" : " 14 GB", | 20260505 22:51:20.140 | INFO | "pinned" : " 29 GB" | 20260505 22:51:20.140 | INFO | } | 20260505 22:51:20.140 | INFO | Perf report for the last 5.1346 seconds: | 20260505 22:51:20.140 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:51:20.140 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:20.140 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:20.140 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:20.140 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260505 22:51:20.140 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:20.140 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:20.140 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:51:20.140 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:20.140 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:20.140 | INFO | Uploaded 36GiB to GPU in 2.10531s (17.0997GiB/s) | 20260505 22:51:20.140 | INFO | Downloaded 34GiB to GPU in 2.04794s (16.602GiB/s) | 20260505 22:51:25.272 | INFO | Analyzing chunk frames = [6932992,6933504) | 20260505 22:51:25.273 | INFO | MemInfo: { | 20260505 22:51:25.273 | INFO | "gpu" : " 17 GB", | 20260505 22:51:25.273 | INFO | "huge" : "", | 20260505 22:51:25.273 | INFO | "huge_shared" : " 27 GB", | 20260505 22:51:25.273 | INFO | "malloc" : " 14 GB", | 20260505 22:51:25.273 | INFO | "pinned" : " 29 GB" | 20260505 22:51:25.273 | INFO | } | 20260505 22:51:25.273 | INFO | Perf report for the last 5.13313 seconds: | 20260505 22:51:25.273 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:51:25.273 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:25.273 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:25.273 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:25.273 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260505 22:51:25.273 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:25.273 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:25.273 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 22:51:25.273 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:25.273 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:25.273 | INFO | Uploaded 36GiB to GPU in 2.0811s (17.2985GiB/s) | 20260505 22:51:25.273 | INFO | Downloaded 34GiB to GPU in 2.03561s (16.7026GiB/s) | 20260505 22:51:30.415 | INFO | Analyzing chunk frames = [6933504,6934016) | 20260505 22:51:30.415 | INFO | MemInfo: { | 20260505 22:51:30.415 | INFO | "gpu" : " 17 GB", | 20260505 22:51:30.415 | INFO | "huge" : "", | 20260505 22:51:30.415 | INFO | "huge_shared" : " 27 GB", | 20260505 22:51:30.415 | INFO | "malloc" : " 14 GB", | 20260505 22:51:30.415 | INFO | "pinned" : " 29 GB" | 20260505 22:51:30.415 | INFO | } | 20260505 22:51:30.415 | INFO | Perf report for the last 5.14205 seconds: | 20260505 22:51:30.415 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:51:30.415 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:30.415 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:30.415 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:30.415 | INFO | ANALYSIS : 63.1%, 3.0, 98.5ms, 0 | 20260505 22:51:30.415 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:30.415 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:30.415 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 22:51:30.415 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:30.415 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:30.415 | INFO | Uploaded 36GiB to GPU in 2.15457s (16.7087GiB/s) | 20260505 22:51:30.415 | INFO | Downloaded 34GiB to GPU in 2.14307s (15.8651GiB/s) | 20260505 22:51:35.556 | INFO | Analyzing chunk frames = [6934016,6934528) | 20260505 22:51:35.556 | INFO | MemInfo: { | 20260505 22:51:35.556 | INFO | "gpu" : " 17 GB", | 20260505 22:51:35.556 | INFO | "huge" : "", | 20260505 22:51:35.556 | INFO | "huge_shared" : " 27 GB", | 20260505 22:51:35.556 | INFO | "malloc" : " 14 GB", | 20260505 22:51:35.556 | INFO | "pinned" : " 29 GB" | 20260505 22:51:35.557 | INFO | } | 20260505 22:51:35.557 | INFO | Perf report for the last 5.14176 seconds: | 20260505 22:51:35.557 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:51:35.557 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:35.557 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:35.557 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:35.557 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 22:51:35.557 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:35.557 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:35.557 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 22:51:35.557 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:35.557 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:35.557 | INFO | Uploaded 36GiB to GPU in 2.12031s (16.9787GiB/s) | 20260505 22:51:35.557 | INFO | Downloaded 34GiB to GPU in 2.00975s (16.9175GiB/s) | 20260505 22:51:35.682 | INFO | Starting a round of Baz IO | 20260505 22:51:38.828 | INFO | Populating RTMetrics report | 20260505 22:51:38.828 | INFO | RTMetrics report populated | 20260505 22:51:38.828 | INFO | Opening tmp RTMetrics json file | 20260505 22:51:38.829 | INFO | Serializing RTMetrics to json | 20260505 22:51:38.830 | INFO | Serialization done, swapping tmp file now... | 20260505 22:51:38.831 | INFO | RTMetrics done | 20260505 22:51:38.912 | INFO | Finished a round of Baz IO: Wrote 1.912 GiB in 3.23 seconds | 20260505 22:51:38.912 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.47% / 0.00% / 78.53% | 20260505 22:51:38.912 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.116237% / 0% / 9.53674e-05% | 20260505 22:51:38.912 | INFO | Baz IO overall: | 20260505 22:51:38.912 | INFO | BytesWritten : 3981489148575 | 20260505 22:51:38.912 | INFO | NumEvents : 1877070742977 | 20260505 22:51:40.701 | INFO | Analyzing chunk frames = [6934528,6935040) | 20260505 22:51:40.701 | INFO | MemInfo: { | 20260505 22:51:40.701 | INFO | "gpu" : " 17 GB", | 20260505 22:51:40.701 | INFO | "huge" : "", | 20260505 22:51:40.701 | INFO | "huge_shared" : " 27 GB", | 20260505 22:51:40.701 | INFO | "malloc" : " 14 GB", | 20260505 22:51:40.701 | INFO | "pinned" : " 29 GB" | 20260505 22:51:40.701 | INFO | } | 20260505 22:51:40.701 | INFO | Perf report for the last 5.14439 seconds: | 20260505 22:51:40.701 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:51:40.701 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:40.701 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:40.701 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:40.701 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 22:51:40.701 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:40.701 | INFO | RT_METRICS : 43.4%, 1.0, 22.8ms, 0 | 20260505 22:51:40.701 | INFO | PRE_HQ : 59.8%, 1.7, 54.1ms, 0 | 20260505 22:51:40.701 | INFO | BAZWRITER : 62.0%, 3.6, 114.3ms, 0 | 20260505 22:51:40.701 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 22:51:40.701 | INFO | Uploaded 36GiB to GPU in 2.16781s (16.6066GiB/s) | 20260505 22:51:40.701 | INFO | Downloaded 38GiB to GPU in 2.23545s (16.9988GiB/s) | 20260505 22:51:45.841 | INFO | Analyzing chunk frames = [6935040,6935552) | 20260505 22:51:45.841 | INFO | MemInfo: { | 20260505 22:51:45.841 | INFO | "gpu" : " 17 GB", | 20260505 22:51:45.841 | INFO | "huge" : "", | 20260505 22:51:45.841 | INFO | "huge_shared" : " 27 GB", | 20260505 22:51:45.841 | INFO | "malloc" : " 14 GB", | 20260505 22:51:45.841 | INFO | "pinned" : " 29 GB" | 20260505 22:51:45.841 | INFO | } | 20260505 22:51:45.841 | INFO | Perf report for the last 5.13997 seconds: | 20260505 22:51:45.841 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:51:45.841 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:45.841 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:45.841 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:45.841 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260505 22:51:45.841 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:45.841 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:45.841 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 22:51:45.841 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:45.841 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:45.841 | INFO | Uploaded 36GiB to GPU in 2.10046s (17.1391GiB/s) | 20260505 22:51:45.841 | INFO | Downloaded 34GiB to GPU in 2.04001s (16.6666GiB/s) | 20260505 22:51:50.975 | INFO | Analyzing chunk frames = [6935552,6936064) | 20260505 22:51:50.975 | INFO | MemInfo: { | 20260505 22:51:50.975 | INFO | "gpu" : " 17 GB", | 20260505 22:51:50.975 | INFO | "huge" : "", | 20260505 22:51:50.975 | INFO | "huge_shared" : " 27 GB", | 20260505 22:51:50.975 | INFO | "malloc" : " 14 GB", | 20260505 22:51:50.975 | INFO | "pinned" : " 29 GB" | 20260505 22:51:50.975 | INFO | } | 20260505 22:51:50.975 | INFO | Perf report for the last 5.13401 seconds: | 20260505 22:51:50.975 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:51:50.975 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:50.975 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:50.975 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:50.975 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260505 22:51:50.975 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:50.975 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:50.975 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:51:50.975 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:50.975 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:50.975 | INFO | Uploaded 36GiB to GPU in 2.14511s (16.7823GiB/s) | 20260505 22:51:50.975 | INFO | Downloaded 34GiB to GPU in 2.04456s (16.6295GiB/s) | 20260505 22:51:56.118 | INFO | Analyzing chunk frames = [6936064,6936576) | 20260505 22:51:56.118 | INFO | MemInfo: { | 20260505 22:51:56.118 | INFO | "gpu" : " 17 GB", | 20260505 22:51:56.118 | INFO | "huge" : "", | 20260505 22:51:56.118 | INFO | "huge_shared" : " 27 GB", | 20260505 22:51:56.118 | INFO | "malloc" : " 14 GB", | 20260505 22:51:56.118 | INFO | "pinned" : " 29 GB" | 20260505 22:51:56.119 | INFO | } | 20260505 22:51:56.119 | INFO | Perf report for the last 5.14358 seconds: | 20260505 22:51:56.119 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:51:56.119 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:56.119 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:56.119 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:56.119 | INFO | ANALYSIS : 61.8%, 3.0, 96.0ms, 0 | 20260505 22:51:56.119 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:56.119 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:56.119 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:51:56.119 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:56.119 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:51:56.119 | INFO | Uploaded 36GiB to GPU in 2.13652s (16.8498GiB/s) | 20260505 22:51:56.119 | INFO | Downloaded 34GiB to GPU in 2.03983s (16.668GiB/s) | 20260505 22:52:01.256 | INFO | Analyzing chunk frames = [6936576,6937088) | 20260505 22:52:01.256 | INFO | MemInfo: { | 20260505 22:52:01.256 | INFO | "gpu" : " 17 GB", | 20260505 22:52:01.256 | INFO | "huge" : "", | 20260505 22:52:01.256 | INFO | "huge_shared" : " 27 GB", | 20260505 22:52:01.256 | INFO | "malloc" : " 14 GB", | 20260505 22:52:01.256 | INFO | "pinned" : " 29 GB" | 20260505 22:52:01.256 | INFO | } | 20260505 22:52:01.256 | INFO | Perf report for the last 5.13764 seconds: | 20260505 22:52:01.256 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:52:01.256 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:01.256 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:01.256 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:01.256 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260505 22:52:01.256 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:01.256 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:01.256 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:52:01.256 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:01.256 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:01.256 | INFO | Uploaded 36GiB to GPU in 2.10572s (17.0963GiB/s) | 20260505 22:52:01.256 | INFO | Downloaded 34GiB to GPU in 2.04997s (16.5856GiB/s) | 20260505 22:52:06.400 | INFO | Analyzing chunk frames = [6937088,6937600) | 20260505 22:52:06.400 | INFO | MemInfo: { | 20260505 22:52:06.400 | INFO | "gpu" : " 17 GB", | 20260505 22:52:06.400 | INFO | "huge" : "", | 20260505 22:52:06.400 | INFO | "huge_shared" : " 27 GB", | 20260505 22:52:06.400 | INFO | "malloc" : " 14 GB", | 20260505 22:52:06.400 | INFO | "pinned" : " 29 GB" | 20260505 22:52:06.400 | INFO | } | 20260505 22:52:06.400 | INFO | Perf report for the last 5.14378 seconds: | 20260505 22:52:06.400 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:52:06.400 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:06.400 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:06.400 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:06.400 | INFO | ANALYSIS : 61.4%, 3.0, 95.9ms, 0 | 20260505 22:52:06.400 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:06.400 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:06.400 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260505 22:52:06.400 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:06.400 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:06.400 | INFO | Uploaded 36GiB to GPU in 2.07009s (17.3905GiB/s) | 20260505 22:52:06.400 | INFO | Downloaded 34GiB to GPU in 2.0374s (16.688GiB/s) | 20260505 22:52:11.528 | INFO | Analyzing chunk frames = [6937600,6938112) | 20260505 22:52:11.528 | INFO | MemInfo: { | 20260505 22:52:11.528 | INFO | "gpu" : " 17 GB", | 20260505 22:52:11.528 | INFO | "huge" : "", | 20260505 22:52:11.528 | INFO | "huge_shared" : " 27 GB", | 20260505 22:52:11.528 | INFO | "malloc" : " 14 GB", | 20260505 22:52:11.528 | INFO | "pinned" : " 29 GB" | 20260505 22:52:11.528 | INFO | } | 20260505 22:52:11.528 | INFO | Perf report for the last 5.1281 seconds: | 20260505 22:52:11.528 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:52:11.528 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:11.528 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:11.528 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:11.528 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260505 22:52:11.528 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:11.528 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:11.528 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 22:52:11.528 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:11.528 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:11.528 | INFO | Uploaded 36GiB to GPU in 2.10916s (17.0684GiB/s) | 20260505 22:52:11.528 | INFO | Downloaded 34GiB to GPU in 2.09388s (16.2378GiB/s) | 20260505 22:52:16.671 | INFO | Analyzing chunk frames = [6938112,6938624) | 20260505 22:52:16.671 | INFO | MemInfo: { | 20260505 22:52:16.671 | INFO | "gpu" : " 17 GB", | 20260505 22:52:16.671 | INFO | "huge" : "", | 20260505 22:52:16.671 | INFO | "huge_shared" : " 27 GB", | 20260505 22:52:16.671 | INFO | "malloc" : " 14 GB", | 20260505 22:52:16.671 | INFO | "pinned" : " 29 GB" | 20260505 22:52:16.671 | INFO | } | 20260505 22:52:16.671 | INFO | Perf report for the last 5.14309 seconds: | 20260505 22:52:16.671 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:52:16.671 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:16.671 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:16.671 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:16.671 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260505 22:52:16.671 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:16.671 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:16.672 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:52:16.672 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:16.672 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:16.672 | INFO | Uploaded 36GiB to GPU in 2.11109s (17.0528GiB/s) | 20260505 22:52:16.672 | INFO | Downloaded 34GiB to GPU in 2.00794s (16.9327GiB/s) | 20260505 22:52:16.672 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 22:52:16.672 | INFO | Total mem usage: 15.3 GiB | 20260505 22:52:16.672 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 22:52:16.672 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -9.07%) | 20260505 22:52:16.672 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 22:52:16.672 | INFO | High Water Breakdown: | 20260505 22:52:16.672 | INFO | 60.9% : 8.56 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 22:52:16.672 | INFO | 39% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 22:52:16.672 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 22:52:16.672 | INFO | ------------------------------------------ | 20260505 22:52:16.672 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:52:16.672 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 22:52:16.672 | INFO | Total mem usage: 26.5 GiB | 20260505 22:52:16.672 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 22:52:16.672 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 22:52:16.672 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260505 22:52:16.672 | INFO | High Water Breakdown: | 20260505 22:52:16.672 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:52:16.672 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:52:16.672 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:52:16.672 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:52:16.672 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:52:16.672 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:52:16.672 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 22:52:16.672 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 22:52:16.672 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:52:16.672 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 22:52:16.672 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 22:52:16.672 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 22:52:16.672 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 22:52:16.672 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 22:52:16.672 | INFO | ------------------------------------------ | 20260505 22:52:16.672 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:52:16.672 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 22:52:16.672 | INFO | Total mem usage: 25.6 GiB | 20260505 22:52:16.672 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 22:52:16.672 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 22:52:16.672 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 22:52:16.672 | INFO | High Water Breakdown: | 20260505 22:52:16.672 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 22:52:16.672 | INFO | ------------------------------------------ | 20260505 22:52:16.672 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:52:16.672 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 22:52:16.672 | INFO | Total mem usage: 15.7 GiB | 20260505 22:52:16.672 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 22:52:16.672 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 22:52:16.672 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 22:52:16.672 | INFO | High Water Breakdown: | 20260505 22:52:16.672 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 22:52:16.673 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 22:52:16.673 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 22:52:16.673 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 22:52:16.673 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 22:52:16.673 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 22:52:16.673 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 22:52:16.673 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 22:52:16.673 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 22:52:16.673 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:52:16.673 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 22:52:16.673 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:52:16.673 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 22:52:16.673 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 22:52:16.673 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 22:52:16.673 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 22:52:16.673 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:52:16.673 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 22:52:16.673 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:52:16.673 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:52:16.673 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:52:16.673 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 22:52:16.673 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 22:52:16.673 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:52:16.673 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 22:52:16.673 | INFO | ------------------------------------------ | 20260505 22:52:16.673 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:52:16.673 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 22:52:16.673 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:52:16.673 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:52:16.673 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.41e+01, 1.35e+01 / 1.21e+00 | 20260505 22:52:16.673 | INFO | PerfReport: 0.29%, AggregateStats -- 9.18e-02 / 1.03e-01, 1.03e-02 / 2.74e-03 | 20260505 22:52:16.673 | INFO | PerfReport: 11.81%, Baseline -- 3.70e+00 / 4.55e+00, 5.68e-01 / 1.56e-01 | 20260505 22:52:16.673 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.53e+00, 1.89e-01 / 7.62e-02 | 20260505 22:52:16.673 | INFO | PerfReport: 15.98%, DME -- 4.01e+01 / 4.40e+01, 3.80e+00 / 1.22e+00 | 20260505 22:52:16.673 | INFO | PerfReport: 57.28%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.47e+00 / 8.73e-01 | 20260505 22:52:16.673 | INFO | PerfReport: 7.14%, PulseAccumulating -- 2.24e+00 / 2.76e+00, 4.28e-01 / 2.02e-01 | 20260505 22:52:16.673 | INFO | PerfReport: 3.14%, Metrics -- 9.83e-01 / 1.29e+00, 2.70e-01 / 9.14e-02 | 20260505 22:52:16.673 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.41e-01 / 4.05e-01, 5.87e-02 / 3.26e-02 | 20260505 22:52:16.673 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 22:52:16.673 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:52:16.673 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:52:16.673 | INFO | PerfReport: Full Execution -- 9.86e+01 / -nan, 2.50e+01 / 0.00e+00 | 20260505 22:52:16.673 | INFO | PerfReport: 22.39%, Upload -- 2.21e+01 / 0.00e+00, 7.58e+00 / 0.00e+00 | 20260505 22:52:16.673 | INFO | PerfReport: 21.85%, Download -- 2.16e+01 / 0.00e+00, 7.66e+00 / 0.00e+00 | 20260505 22:52:16.673 | INFO | PerfReport: 31.78%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 22:52:16.673 | INFO | PerfReport: 2.18%, LockUpload -- 2.15e+00 / 0.00e+00, 7.98e+00 / 0.00e+00 | 20260505 22:52:16.673 | INFO | PerfReport: 1.34%, LockDownload -- 1.33e+00 / 0.00e+00, 2.89e+00 / 0.00e+00 | 20260505 22:52:16.673 | INFO | PerfReport: 20.43%, LockCompute -- 2.02e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260505 22:52:16.673 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.90e-02 / 0.00e+00 | 20260505 22:52:16.796 | INFO | Starting a round of Baz IO | 20260505 22:52:19.943 | INFO | Populating RTMetrics report | 20260505 22:52:19.944 | INFO | RTMetrics report populated | 20260505 22:52:19.944 | INFO | Opening tmp RTMetrics json file | 20260505 22:52:19.945 | INFO | Serializing RTMetrics to json | 20260505 22:52:19.948 | INFO | Serialization done, swapping tmp file now... | 20260505 22:52:19.950 | INFO | RTMetrics done | 20260505 22:52:20.023 | INFO | Finished a round of Baz IO: Wrote 1.914 GiB in 3.226 seconds | 20260505 22:52:20.023 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.47% / 0.00% / 78.53% | 20260505 22:52:20.023 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.116332% / 0% / 8.34465e-05% | 20260505 22:52:20.023 | INFO | Baz IO overall: | 20260505 22:52:20.023 | INFO | BytesWritten : 3983544003038 | 20260505 22:52:20.023 | INFO | NumEvents : 1878014775585 | 20260505 22:52:21.813 | INFO | Analyzing chunk frames = [6938624,6939136) | 20260505 22:52:21.813 | INFO | MemInfo: { | 20260505 22:52:21.813 | INFO | "gpu" : " 17 GB", | 20260505 22:52:21.813 | INFO | "huge" : "", | 20260505 22:52:21.813 | INFO | "huge_shared" : " 27 GB", | 20260505 22:52:21.813 | INFO | "malloc" : " 14 GB", | 20260505 22:52:21.813 | INFO | "pinned" : " 29 GB" | 20260505 22:52:21.813 | INFO | } | 20260505 22:52:21.813 | INFO | Perf report for the last 5.14143 seconds: | 20260505 22:52:21.813 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:52:21.813 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:21.813 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:21.813 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:21.813 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260505 22:52:21.813 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:21.813 | INFO | RT_METRICS : 43.4%, 1.0, 22.8ms, 0 | 20260505 22:52:21.813 | INFO | PRE_HQ : 59.9%, 1.7, 54.2ms, 0 | 20260505 22:52:21.813 | INFO | BAZWRITER : 62.0%, 3.6, 114.8ms, 0 | 20260505 22:52:21.813 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 22:52:21.813 | INFO | Uploaded 36GiB to GPU in 2.16641s (16.6173GiB/s) | 20260505 22:52:21.813 | INFO | Downloaded 38GiB to GPU in 2.23751s (16.9832GiB/s) | 20260505 22:52:26.956 | INFO | Analyzing chunk frames = [6939136,6939648) | 20260505 22:52:26.956 | INFO | MemInfo: { | 20260505 22:52:26.956 | INFO | "gpu" : " 17 GB", | 20260505 22:52:26.956 | INFO | "huge" : "", | 20260505 22:52:26.956 | INFO | "huge_shared" : " 27 GB", | 20260505 22:52:26.956 | INFO | "malloc" : " 14 GB", | 20260505 22:52:26.956 | INFO | "pinned" : " 29 GB" | 20260505 22:52:26.956 | INFO | } | 20260505 22:52:26.956 | INFO | Perf report for the last 5.14299 seconds: | 20260505 22:52:26.956 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:52:26.956 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:26.956 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:26.956 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:26.956 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 22:52:26.956 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:26.956 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:26.956 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 22:52:26.956 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:26.956 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:26.956 | INFO | Uploaded 36GiB to GPU in 2.08504s (17.2658GiB/s) | 20260505 22:52:26.956 | INFO | Downloaded 34GiB to GPU in 2.03094s (16.741GiB/s) | 20260505 22:52:32.086 | INFO | Analyzing chunk frames = [6939648,6940160) | 20260505 22:52:32.087 | INFO | MemInfo: { | 20260505 22:52:32.087 | INFO | "gpu" : " 17 GB", | 20260505 22:52:32.087 | INFO | "huge" : "", | 20260505 22:52:32.087 | INFO | "huge_shared" : " 27 GB", | 20260505 22:52:32.087 | INFO | "malloc" : " 14 GB", | 20260505 22:52:32.087 | INFO | "pinned" : " 29 GB" | 20260505 22:52:32.087 | INFO | } | 20260505 22:52:32.087 | INFO | Perf report for the last 5.13115 seconds: | 20260505 22:52:32.087 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:52:32.087 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:32.087 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:32.087 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:32.087 | INFO | ANALYSIS : 62.1%, 3.0, 96.4ms, 0 | 20260505 22:52:32.087 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:32.087 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:32.087 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 22:52:32.087 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:32.087 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:32.087 | INFO | Uploaded 36GiB to GPU in 2.161s (16.659GiB/s) | 20260505 22:52:32.087 | INFO | Downloaded 34GiB to GPU in 2.05363s (16.556GiB/s) | 20260505 22:52:37.222 | INFO | Analyzing chunk frames = [6940160,6940672) | 20260505 22:52:37.222 | INFO | MemInfo: { | 20260505 22:52:37.222 | INFO | "gpu" : " 17 GB", | 20260505 22:52:37.222 | INFO | "huge" : "", | 20260505 22:52:37.223 | INFO | "huge_shared" : " 27 GB", | 20260505 22:52:37.223 | INFO | "malloc" : " 14 GB", | 20260505 22:52:37.223 | INFO | "pinned" : " 29 GB" | 20260505 22:52:37.223 | INFO | } | 20260505 22:52:37.223 | INFO | Perf report for the last 5.13583 seconds: | 20260505 22:52:37.223 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:52:37.223 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:37.223 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:37.223 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:37.223 | INFO | ANALYSIS : 62.0%, 3.0, 96.2ms, 0 | 20260505 22:52:37.223 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:37.223 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:37.223 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260505 22:52:37.223 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:37.223 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:37.223 | INFO | Uploaded 36GiB to GPU in 2.15243s (16.7252GiB/s) | 20260505 22:52:37.223 | INFO | Downloaded 34GiB to GPU in 2.04254s (16.646GiB/s) | 20260505 22:52:42.372 | INFO | Analyzing chunk frames = [6940672,6941184) | 20260505 22:52:42.372 | INFO | MemInfo: { | 20260505 22:52:42.372 | INFO | "gpu" : " 17 GB", | 20260505 22:52:42.372 | INFO | "huge" : "", | 20260505 22:52:42.372 | INFO | "huge_shared" : " 27 GB", | 20260505 22:52:42.372 | INFO | "malloc" : " 14 GB", | 20260505 22:52:42.372 | INFO | "pinned" : " 29 GB" | 20260505 22:52:42.372 | INFO | } | 20260505 22:52:42.372 | INFO | Perf report for the last 5.1496 seconds: | 20260505 22:52:42.372 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:52:42.372 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:42.372 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:42.372 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:42.372 | INFO | ANALYSIS : 61.4%, 3.0, 96.1ms, 0 | 20260505 22:52:42.372 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:42.372 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:42.372 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 22:52:42.372 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:42.372 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:42.372 | INFO | Uploaded 36GiB to GPU in 2.13219s (16.884GiB/s) | 20260505 22:52:42.373 | INFO | Downloaded 34GiB to GPU in 2.0638s (16.4745GiB/s) | 20260505 22:52:47.510 | INFO | Analyzing chunk frames = [6941184,6941696) | 20260505 22:52:47.510 | INFO | MemInfo: { | 20260505 22:52:47.510 | INFO | "gpu" : " 17 GB", | 20260505 22:52:47.510 | INFO | "huge" : "", | 20260505 22:52:47.510 | INFO | "huge_shared" : " 27 GB", | 20260505 22:52:47.510 | INFO | "malloc" : " 14 GB", | 20260505 22:52:47.510 | INFO | "pinned" : " 29 GB" | 20260505 22:52:47.510 | INFO | } | 20260505 22:52:47.510 | INFO | Perf report for the last 5.13768 seconds: | 20260505 22:52:47.510 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:52:47.510 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:47.510 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:47.510 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:47.510 | INFO | ANALYSIS : 61.4%, 3.0, 95.9ms, 0 | 20260505 22:52:47.510 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:47.510 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:47.510 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 22:52:47.510 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:47.510 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:47.510 | INFO | Uploaded 36GiB to GPU in 2.08147s (17.2955GiB/s) | 20260505 22:52:47.510 | INFO | Downloaded 34GiB to GPU in 2.04614s (16.6166GiB/s) | 20260505 22:52:52.649 | INFO | Analyzing chunk frames = [6941696,6942208) | 20260505 22:52:52.649 | INFO | MemInfo: { | 20260505 22:52:52.649 | INFO | "gpu" : " 17 GB", | 20260505 22:52:52.649 | INFO | "huge" : "", | 20260505 22:52:52.649 | INFO | "huge_shared" : " 27 GB", | 20260505 22:52:52.649 | INFO | "malloc" : " 14 GB", | 20260505 22:52:52.649 | INFO | "pinned" : " 29 GB" | 20260505 22:52:52.649 | INFO | } | 20260505 22:52:52.649 | INFO | Perf report for the last 5.1389 seconds: | 20260505 22:52:52.649 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:52:52.649 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:52.649 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:52.649 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:52.649 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 22:52:52.649 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:52.649 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:52.649 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 22:52:52.649 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:52.649 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:52.649 | INFO | Uploaded 36GiB to GPU in 2.09587s (17.1766GiB/s) | 20260505 22:52:52.649 | INFO | Downloaded 34GiB to GPU in 2.06503s (16.4647GiB/s) | 20260505 22:52:57.786 | INFO | Analyzing chunk frames = [6942208,6942720) | 20260505 22:52:57.786 | INFO | MemInfo: { | 20260505 22:52:57.786 | INFO | "gpu" : " 17 GB", | 20260505 22:52:57.786 | INFO | "huge" : "", | 20260505 22:52:57.786 | INFO | "huge_shared" : " 27 GB", | 20260505 22:52:57.786 | INFO | "malloc" : " 14 GB", | 20260505 22:52:57.786 | INFO | "pinned" : " 29 GB" | 20260505 22:52:57.786 | INFO | } | 20260505 22:52:57.786 | INFO | Perf report for the last 5.13742 seconds: | 20260505 22:52:57.786 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:52:57.786 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:57.786 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:57.786 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:57.786 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 22:52:57.786 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:57.786 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:57.786 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:52:57.786 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:57.786 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:52:57.787 | INFO | Uploaded 36GiB to GPU in 2.11322s (17.0356GiB/s) | 20260505 22:52:57.787 | INFO | Downloaded 34GiB to GPU in 1.99927s (17.0062GiB/s) | 20260505 22:52:57.916 | INFO | Starting a round of Baz IO | 20260505 22:53:01.062 | INFO | Populating RTMetrics report | 20260505 22:53:01.063 | INFO | RTMetrics report populated | 20260505 22:53:01.063 | INFO | Opening tmp RTMetrics json file | 20260505 22:53:01.063 | INFO | Serializing RTMetrics to json | 20260505 22:53:01.064 | INFO | Serialization done, swapping tmp file now... | 20260505 22:53:01.066 | INFO | RTMetrics done | 20260505 22:53:01.145 | INFO | Finished a round of Baz IO: Wrote 1.914 GiB in 3.228 seconds | 20260505 22:53:01.145 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.47% / 0.00% / 78.53% | 20260505 22:53:01.145 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.116404% / 0% / 5.16574e-05% | 20260505 22:53:01.145 | INFO | Baz IO overall: | 20260505 22:53:01.145 | INFO | BytesWritten : 3985599407400 | 20260505 22:53:01.145 | INFO | NumEvents : 1878959277206 | 20260505 22:53:02.927 | INFO | Analyzing chunk frames = [6942720,6943232) | 20260505 22:53:02.927 | INFO | MemInfo: { | 20260505 22:53:02.927 | INFO | "gpu" : " 17 GB", | 20260505 22:53:02.927 | INFO | "huge" : "", | 20260505 22:53:02.927 | INFO | "huge_shared" : " 27 GB", | 20260505 22:53:02.927 | INFO | "malloc" : " 14 GB", | 20260505 22:53:02.928 | INFO | "pinned" : " 29 GB" | 20260505 22:53:02.928 | INFO | } | 20260505 22:53:02.928 | INFO | Perf report for the last 5.14128 seconds: | 20260505 22:53:02.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:53:02.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:02.928 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:02.928 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:02.928 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 22:53:02.928 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:02.928 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260505 22:53:02.928 | INFO | PRE_HQ : 59.5%, 1.8, 54.5ms, 0 | 20260505 22:53:02.928 | INFO | BAZWRITER : 62.1%, 3.6, 114.8ms, 0 | 20260505 22:53:02.928 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 22:53:02.928 | INFO | Uploaded 36GiB to GPU in 2.12401s (16.9491GiB/s) | 20260505 22:53:02.928 | INFO | Downloaded 38GiB to GPU in 2.23077s (17.0345GiB/s) | 20260505 22:53:08.067 | INFO | Analyzing chunk frames = [6943232,6943744) | 20260505 22:53:08.067 | INFO | MemInfo: { | 20260505 22:53:08.067 | INFO | "gpu" : " 17 GB", | 20260505 22:53:08.067 | INFO | "huge" : "", | 20260505 22:53:08.067 | INFO | "huge_shared" : " 27 GB", | 20260505 22:53:08.067 | INFO | "malloc" : " 14 GB", | 20260505 22:53:08.067 | INFO | "pinned" : " 29 GB" | 20260505 22:53:08.067 | INFO | } | 20260505 22:53:08.067 | INFO | Perf report for the last 5.13928 seconds: | 20260505 22:53:08.067 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:53:08.067 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:08.067 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:08.067 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:08.067 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 22:53:08.067 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:08.067 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:08.067 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 22:53:08.067 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:08.067 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:08.067 | INFO | Uploaded 36GiB to GPU in 2.07692s (17.3333GiB/s) | 20260505 22:53:08.067 | INFO | Downloaded 34GiB to GPU in 2.02352s (16.8024GiB/s) | 20260505 22:53:13.205 | INFO | Analyzing chunk frames = [6943744,6944256) | 20260505 22:53:13.205 | INFO | MemInfo: { | 20260505 22:53:13.205 | INFO | "gpu" : " 17 GB", | 20260505 22:53:13.205 | INFO | "huge" : "", | 20260505 22:53:13.205 | INFO | "huge_shared" : " 27 GB", | 20260505 22:53:13.205 | INFO | "malloc" : " 14 GB", | 20260505 22:53:13.205 | INFO | "pinned" : " 29 GB" | 20260505 22:53:13.205 | INFO | } | 20260505 22:53:13.205 | INFO | Perf report for the last 5.13847 seconds: | 20260505 22:53:13.205 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:53:13.205 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:13.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:13.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:13.205 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260505 22:53:13.205 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:13.205 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:13.206 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 22:53:13.206 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:13.206 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:13.206 | INFO | Uploaded 36GiB to GPU in 2.096s (17.1756GiB/s) | 20260505 22:53:13.206 | INFO | Downloaded 34GiB to GPU in 2.03196s (16.7326GiB/s) | 20260505 22:53:18.349 | INFO | Analyzing chunk frames = [6944256,6944768) | 20260505 22:53:18.349 | INFO | MemInfo: { | 20260505 22:53:18.349 | INFO | "gpu" : " 17 GB", | 20260505 22:53:18.349 | INFO | "huge" : "", | 20260505 22:53:18.349 | INFO | "huge_shared" : " 27 GB", | 20260505 22:53:18.350 | INFO | "malloc" : " 14 GB", | 20260505 22:53:18.350 | INFO | "pinned" : " 29 GB" | 20260505 22:53:18.350 | INFO | } | 20260505 22:53:18.350 | INFO | Perf report for the last 5.1443 seconds: | 20260505 22:53:18.350 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:53:18.350 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:18.350 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:18.350 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:18.350 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260505 22:53:18.350 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:18.350 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:18.350 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:53:18.350 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:18.350 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:18.350 | INFO | Uploaded 36GiB to GPU in 2.12306s (16.9567GiB/s) | 20260505 22:53:18.350 | INFO | Downloaded 34GiB to GPU in 2.0195s (16.8359GiB/s) | 20260505 22:53:23.482 | INFO | Analyzing chunk frames = [6944768,6945280) | 20260505 22:53:23.482 | INFO | MemInfo: { | 20260505 22:53:23.482 | INFO | "gpu" : " 17 GB", | 20260505 22:53:23.482 | INFO | "huge" : "", | 20260505 22:53:23.482 | INFO | "huge_shared" : " 27 GB", | 20260505 22:53:23.482 | INFO | "malloc" : " 14 GB", | 20260505 22:53:23.482 | INFO | "pinned" : " 29 GB" | 20260505 22:53:23.482 | INFO | } | 20260505 22:53:23.482 | INFO | Perf report for the last 5.13253 seconds: | 20260505 22:53:23.482 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:53:23.482 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:23.482 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:23.482 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:23.482 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260505 22:53:23.482 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:23.482 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:23.482 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:53:23.482 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:23.482 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:23.483 | INFO | Uploaded 36GiB to GPU in 2.09291s (17.2009GiB/s) | 20260505 22:53:23.483 | INFO | Downloaded 34GiB to GPU in 2.04395s (16.6345GiB/s) | 20260505 22:53:28.624 | INFO | Analyzing chunk frames = [6945280,6945792) | 20260505 22:53:28.624 | INFO | MemInfo: { | 20260505 22:53:28.624 | INFO | "gpu" : " 17 GB", | 20260505 22:53:28.624 | INFO | "huge" : "", | 20260505 22:53:28.624 | INFO | "huge_shared" : " 27 GB", | 20260505 22:53:28.624 | INFO | "malloc" : " 14 GB", | 20260505 22:53:28.624 | INFO | "pinned" : " 29 GB" | 20260505 22:53:28.624 | INFO | } | 20260505 22:53:28.624 | INFO | Perf report for the last 5.1421 seconds: | 20260505 22:53:28.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:53:28.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:28.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:28.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:28.624 | INFO | ANALYSIS : 61.4%, 3.0, 96.0ms, 0 | 20260505 22:53:28.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:28.624 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:28.624 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260505 22:53:28.624 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:28.625 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:28.625 | INFO | Uploaded 36GiB to GPU in 2.07674s (17.3349GiB/s) | 20260505 22:53:28.625 | INFO | Downloaded 34GiB to GPU in 2.03945s (16.6711GiB/s) | 20260505 22:53:33.767 | INFO | Analyzing chunk frames = [6945792,6946304) | 20260505 22:53:33.767 | INFO | MemInfo: { | 20260505 22:53:33.767 | INFO | "gpu" : " 17 GB", | 20260505 22:53:33.767 | INFO | "huge" : "", | 20260505 22:53:33.767 | INFO | "huge_shared" : " 27 GB", | 20260505 22:53:33.767 | INFO | "malloc" : " 14 GB", | 20260505 22:53:33.767 | INFO | "pinned" : " 29 GB" | 20260505 22:53:33.767 | INFO | } | 20260505 22:53:33.767 | INFO | Perf report for the last 5.14305 seconds: | 20260505 22:53:33.767 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:53:33.767 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:33.767 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:33.767 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:33.767 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260505 22:53:33.767 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:33.768 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:33.768 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 22:53:33.768 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:33.768 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:33.768 | INFO | Uploaded 36GiB to GPU in 2.06851s (17.4039GiB/s) | 20260505 22:53:33.768 | INFO | Downloaded 34GiB to GPU in 2.06779s (16.4426GiB/s) | 20260505 22:53:38.900 | INFO | Analyzing chunk frames = [6946304,6946816) | 20260505 22:53:38.900 | INFO | MemInfo: { | 20260505 22:53:38.900 | INFO | "gpu" : " 17 GB", | 20260505 22:53:38.900 | INFO | "huge" : "", | 20260505 22:53:38.900 | INFO | "huge_shared" : " 27 GB", | 20260505 22:53:38.900 | INFO | "malloc" : " 14 GB", | 20260505 22:53:38.901 | INFO | "pinned" : " 29 GB" | 20260505 22:53:38.901 | INFO | } | 20260505 22:53:38.901 | INFO | Perf report for the last 5.13331 seconds: | 20260505 22:53:38.901 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:53:38.901 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:38.901 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:38.901 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:38.901 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260505 22:53:38.901 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:38.901 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:38.901 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 22:53:38.901 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:38.901 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:38.901 | INFO | Uploaded 36GiB to GPU in 2.10906s (17.0692GiB/s) | 20260505 22:53:38.901 | INFO | Downloaded 34GiB to GPU in 2.00747s (16.9367GiB/s) | 20260505 22:53:38.901 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 22:53:38.901 | INFO | Total mem usage: 15.3 GiB | 20260505 22:53:38.901 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 22:53:38.901 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.54%) | 20260505 22:53:38.901 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 22:53:38.901 | INFO | High Water Breakdown: | 20260505 22:53:38.901 | INFO | 61.1% : 8.62 GiB ( 90.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 22:53:38.901 | INFO | 38.8% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 22:53:38.901 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 22:53:38.901 | INFO | ------------------------------------------ | 20260505 22:53:38.901 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:53:38.901 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 22:53:38.901 | INFO | Total mem usage: 26.5 GiB | 20260505 22:53:38.901 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 22:53:38.901 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 22:53:38.901 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 22:53:38.901 | INFO | High Water Breakdown: | 20260505 22:53:38.901 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:53:38.902 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:53:38.902 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:53:38.902 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:53:38.902 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:53:38.902 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:53:38.902 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 22:53:38.902 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 22:53:38.902 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:53:38.902 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 22:53:38.902 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 22:53:38.902 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 22:53:38.902 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 22:53:38.902 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 22:53:38.902 | INFO | ------------------------------------------ | 20260505 22:53:38.902 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:53:38.902 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 22:53:38.902 | INFO | Total mem usage: 25.6 GiB | 20260505 22:53:38.902 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 22:53:38.902 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 22:53:38.902 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 22:53:38.902 | INFO | High Water Breakdown: | 20260505 22:53:38.902 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 22:53:38.902 | INFO | ------------------------------------------ | 20260505 22:53:38.902 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:53:38.902 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 22:53:38.902 | INFO | Total mem usage: 15.7 GiB | 20260505 22:53:38.902 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 22:53:38.902 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 22:53:38.902 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 22:53:38.902 | INFO | High Water Breakdown: | 20260505 22:53:38.902 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 22:53:38.902 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 22:53:38.902 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 22:53:38.902 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 22:53:38.902 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 22:53:38.902 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 22:53:38.902 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 22:53:38.902 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 22:53:38.902 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 22:53:38.902 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:53:38.902 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 22:53:38.902 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:53:38.902 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 22:53:38.902 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 22:53:38.902 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 22:53:38.902 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 22:53:38.902 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:53:38.902 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 22:53:38.902 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:53:38.902 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:53:38.902 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:53:38.902 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 22:53:38.902 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 22:53:38.902 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:53:38.902 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 22:53:38.902 | INFO | ------------------------------------------ | 20260505 22:53:38.902 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:53:38.902 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 22:53:38.902 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:53:38.903 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:53:38.903 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.41e+01, 1.35e+01 / 8.81e-01 | 20260505 22:53:38.903 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.05e-01, 1.02e-02 / 3.04e-03 | 20260505 22:53:38.903 | INFO | PerfReport: 11.81%, Baseline -- 3.71e+00 / 4.48e+00, 5.70e-01 / 2.46e-01 | 20260505 22:53:38.903 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.53e+00, 1.93e-01 / 8.34e-02 | 20260505 22:53:38.903 | INFO | PerfReport: 15.99%, DME -- 4.02e+01 / 4.42e+01, 3.92e+00 / 1.06e+00 | 20260505 22:53:38.903 | INFO | PerfReport: 57.30%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.54e+00 / 1.08e+00 | 20260505 22:53:38.903 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.24e+00 / 2.77e+00, 4.27e-01 / 2.45e-01 | 20260505 22:53:38.903 | INFO | PerfReport: 3.13%, Metrics -- 9.83e-01 / 1.27e+00, 2.69e-01 / 1.11e-01 | 20260505 22:53:38.903 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.41e-01 / 3.98e-01, 5.87e-02 / 3.09e-02 | 20260505 22:53:38.903 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 22:53:38.903 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:53:38.903 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:53:38.903 | INFO | PerfReport: Full Execution -- 9.86e+01 / -nan, 2.48e+01 / 0.00e+00 | 20260505 22:53:38.903 | INFO | PerfReport: 22.30%, Upload -- 2.20e+01 / 0.00e+00, 7.50e+00 / 0.00e+00 | 20260505 22:53:38.903 | INFO | PerfReport: 21.80%, Download -- 2.15e+01 / 0.00e+00, 7.60e+00 / 0.00e+00 | 20260505 22:53:38.903 | INFO | PerfReport: 31.87%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 22:53:38.903 | INFO | PerfReport: 2.12%, LockUpload -- 2.09e+00 / 0.00e+00, 7.81e+00 / 0.00e+00 | 20260505 22:53:38.903 | INFO | PerfReport: 1.30%, LockDownload -- 1.28e+00 / 0.00e+00, 2.92e+00 / 0.00e+00 | 20260505 22:53:38.903 | INFO | PerfReport: 20.60%, LockCompute -- 2.03e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 22:53:38.903 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.44e-02 / 0.00e+00 | 20260505 22:53:39.030 | INFO | Starting a round of Baz IO | 20260505 22:53:42.175 | INFO | Populating RTMetrics report | 20260505 22:53:42.176 | INFO | RTMetrics report populated | 20260505 22:53:42.176 | INFO | Opening tmp RTMetrics json file | 20260505 22:53:42.178 | INFO | Serializing RTMetrics to json | 20260505 22:53:42.181 | INFO | Serialization done, swapping tmp file now... | 20260505 22:53:42.181 | INFO | RTMetrics done | 20260505 22:53:42.260 | INFO | Finished a round of Baz IO: Wrote 1.912 GiB in 3.23 seconds | 20260505 22:53:42.261 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.47% / 0.00% / 78.53% | 20260505 22:53:42.261 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.115875% / 3.97364e-06% / 9.53674e-05% | 20260505 22:53:42.261 | INFO | Baz IO overall: | 20260505 22:53:42.261 | INFO | BytesWritten : 3987652749579 | 20260505 22:53:42.261 | INFO | NumEvents : 1879902363157 | 20260505 22:53:44.034 | INFO | Analyzing chunk frames = [6946816,6947328) | 20260505 22:53:44.034 | INFO | MemInfo: { | 20260505 22:53:44.034 | INFO | "gpu" : " 17 GB", | 20260505 22:53:44.034 | INFO | "huge" : "", | 20260505 22:53:44.034 | INFO | "huge_shared" : " 27 GB", | 20260505 22:53:44.034 | INFO | "malloc" : " 14 GB", | 20260505 22:53:44.034 | INFO | "pinned" : " 29 GB" | 20260505 22:53:44.034 | INFO | } | 20260505 22:53:44.035 | INFO | Perf report for the last 5.13377 seconds: | 20260505 22:53:44.035 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:53:44.035 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:44.035 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:44.035 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:44.035 | INFO | ANALYSIS : 62.8%, 3.0, 97.8ms, 0 | 20260505 22:53:44.035 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:44.035 | INFO | RT_METRICS : 43.6%, 1.0, 22.8ms, 0 | 20260505 22:53:44.035 | INFO | PRE_HQ : 59.8%, 1.7, 54.2ms, 0 | 20260505 22:53:44.035 | INFO | BAZWRITER : 62.2%, 3.6, 115.1ms, 0 | 20260505 22:53:44.035 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 22:53:44.035 | INFO | Uploaded 36GiB to GPU in 2.15259s (16.7241GiB/s) | 20260505 22:53:44.035 | INFO | Downloaded 38GiB to GPU in 2.24148s (16.9531GiB/s) | 20260505 22:53:49.178 | INFO | Analyzing chunk frames = [6947328,6947840) | 20260505 22:53:49.178 | INFO | MemInfo: { | 20260505 22:53:49.178 | INFO | "gpu" : " 17 GB", | 20260505 22:53:49.178 | INFO | "huge" : "", | 20260505 22:53:49.178 | INFO | "huge_shared" : " 27 GB", | 20260505 22:53:49.178 | INFO | "malloc" : " 14 GB", | 20260505 22:53:49.178 | INFO | "pinned" : " 29 GB" | 20260505 22:53:49.178 | INFO | } | 20260505 22:53:49.178 | INFO | Perf report for the last 5.1434 seconds: | 20260505 22:53:49.178 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:53:49.178 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:49.178 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:49.178 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:49.178 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 22:53:49.178 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:49.178 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:49.178 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 22:53:49.178 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:49.178 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:49.178 | INFO | Uploaded 36GiB to GPU in 2.06337s (17.4472GiB/s) | 20260505 22:53:49.178 | INFO | Downloaded 34GiB to GPU in 2.02066s (16.8262GiB/s) | 20260505 22:53:54.324 | INFO | Analyzing chunk frames = [6947840,6948352) | 20260505 22:53:54.324 | INFO | MemInfo: { | 20260505 22:53:54.324 | INFO | "gpu" : " 17 GB", | 20260505 22:53:54.324 | INFO | "huge" : "", | 20260505 22:53:54.324 | INFO | "huge_shared" : " 27 GB", | 20260505 22:53:54.324 | INFO | "malloc" : " 14 GB", | 20260505 22:53:54.324 | INFO | "pinned" : " 29 GB" | 20260505 22:53:54.325 | INFO | } | 20260505 22:53:54.325 | INFO | Perf report for the last 5.14662 seconds: | 20260505 22:53:54.325 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:53:54.325 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:54.325 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:54.325 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:54.325 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260505 22:53:54.325 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:54.325 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:54.325 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:53:54.325 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:54.325 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:54.325 | INFO | Uploaded 36GiB to GPU in 2.15409s (16.7124GiB/s) | 20260505 22:53:54.325 | INFO | Downloaded 34GiB to GPU in 2.04921s (16.5918GiB/s) | 20260505 22:53:59.462 | INFO | Analyzing chunk frames = [6948352,6948864) | 20260505 22:53:59.462 | INFO | MemInfo: { | 20260505 22:53:59.462 | INFO | "gpu" : " 17 GB", | 20260505 22:53:59.462 | INFO | "huge" : "", | 20260505 22:53:59.462 | INFO | "huge_shared" : " 27 GB", | 20260505 22:53:59.462 | INFO | "malloc" : " 14 GB", | 20260505 22:53:59.462 | INFO | "pinned" : " 29 GB" | 20260505 22:53:59.462 | INFO | } | 20260505 22:53:59.462 | INFO | Perf report for the last 5.13739 seconds: | 20260505 22:53:59.462 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:53:59.462 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:59.462 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:59.462 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:59.462 | INFO | ANALYSIS : 61.8%, 3.0, 95.9ms, 0 | 20260505 22:53:59.462 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:59.462 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:59.462 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:53:59.462 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:59.462 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:53:59.462 | INFO | Uploaded 36GiB to GPU in 2.13938s (16.8273GiB/s) | 20260505 22:53:59.462 | INFO | Downloaded 34GiB to GPU in 2.01696s (16.8571GiB/s) | 20260505 22:54:04.595 | INFO | Analyzing chunk frames = [6948864,6949376) | 20260505 22:54:04.595 | INFO | MemInfo: { | 20260505 22:54:04.595 | INFO | "gpu" : " 17 GB", | 20260505 22:54:04.595 | INFO | "huge" : "", | 20260505 22:54:04.595 | INFO | "huge_shared" : " 27 GB", | 20260505 22:54:04.595 | INFO | "malloc" : " 14 GB", | 20260505 22:54:04.595 | INFO | "pinned" : " 29 GB" | 20260505 22:54:04.596 | INFO | } | 20260505 22:54:04.596 | INFO | Perf report for the last 5.13366 seconds: | 20260505 22:54:04.596 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:54:04.596 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:04.596 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:04.596 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:04.596 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260505 22:54:04.596 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:04.596 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:04.596 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 22:54:04.596 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:04.596 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:04.596 | INFO | Uploaded 36GiB to GPU in 2.09992s (17.1435GiB/s) | 20260505 22:54:04.596 | INFO | Downloaded 34GiB to GPU in 2.05178s (16.571GiB/s) | 20260505 22:54:09.735 | INFO | Analyzing chunk frames = [6949376,6949888) | 20260505 22:54:09.735 | INFO | MemInfo: { | 20260505 22:54:09.735 | INFO | "gpu" : " 17 GB", | 20260505 22:54:09.735 | INFO | "huge" : "", | 20260505 22:54:09.735 | INFO | "huge_shared" : " 27 GB", | 20260505 22:54:09.735 | INFO | "malloc" : " 14 GB", | 20260505 22:54:09.735 | INFO | "pinned" : " 29 GB" | 20260505 22:54:09.735 | INFO | } | 20260505 22:54:09.735 | INFO | Perf report for the last 5.13909 seconds: | 20260505 22:54:09.735 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:54:09.735 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:09.735 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:09.735 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:09.735 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 22:54:09.735 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:09.735 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:09.735 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:54:09.735 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:09.735 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:09.735 | INFO | Uploaded 36GiB to GPU in 2.10099s (17.1348GiB/s) | 20260505 22:54:09.735 | INFO | Downloaded 34GiB to GPU in 2.0335s (16.7199GiB/s) | 20260505 22:54:14.874 | INFO | Analyzing chunk frames = [6949888,6950400) | 20260505 22:54:14.874 | INFO | MemInfo: { | 20260505 22:54:14.874 | INFO | "gpu" : " 17 GB", | 20260505 22:54:14.874 | INFO | "huge" : "", | 20260505 22:54:14.874 | INFO | "huge_shared" : " 27 GB", | 20260505 22:54:14.874 | INFO | "malloc" : " 14 GB", | 20260505 22:54:14.874 | INFO | "pinned" : " 29 GB" | 20260505 22:54:14.874 | INFO | } | 20260505 22:54:14.874 | INFO | Perf report for the last 5.13933 seconds: | 20260505 22:54:14.874 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:54:14.874 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:14.874 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:14.874 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:14.874 | INFO | ANALYSIS : 62.7%, 3.0, 98.0ms, 0 | 20260505 22:54:14.874 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:14.874 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:14.874 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 22:54:14.874 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:14.874 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:14.874 | INFO | Uploaded 36GiB to GPU in 2.13915s (16.8291GiB/s) | 20260505 22:54:14.874 | INFO | Downloaded 34GiB to GPU in 2.10578s (16.146GiB/s) | 20260505 22:54:20.020 | INFO | Analyzing chunk frames = [6950400,6950912) | 20260505 22:54:20.020 | INFO | MemInfo: { | 20260505 22:54:20.020 | INFO | "gpu" : " 17 GB", | 20260505 22:54:20.020 | INFO | "huge" : "", | 20260505 22:54:20.020 | INFO | "huge_shared" : " 27 GB", | 20260505 22:54:20.020 | INFO | "malloc" : " 14 GB", | 20260505 22:54:20.021 | INFO | "pinned" : " 29 GB" | 20260505 22:54:20.021 | INFO | } | 20260505 22:54:20.021 | INFO | Perf report for the last 5.14655 seconds: | 20260505 22:54:20.021 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:54:20.021 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:20.021 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:20.021 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:20.021 | INFO | ANALYSIS : 62.1%, 3.0, 97.1ms, 0 | 20260505 22:54:20.021 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:20.021 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:20.021 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 22:54:20.021 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:20.021 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:20.021 | INFO | Uploaded 36GiB to GPU in 2.11068s (17.0561GiB/s) | 20260505 22:54:20.021 | INFO | Downloaded 34GiB to GPU in 2.01182s (16.9001GiB/s) | 20260505 22:54:20.138 | INFO | Starting a round of Baz IO | 20260505 22:54:23.287 | INFO | Populating RTMetrics report | 20260505 22:54:23.287 | INFO | RTMetrics report populated | 20260505 22:54:23.287 | INFO | Opening tmp RTMetrics json file | 20260505 22:54:23.288 | INFO | Serializing RTMetrics to json | 20260505 22:54:23.289 | INFO | Serialization done, swapping tmp file now... | 20260505 22:54:23.290 | INFO | RTMetrics done | 20260505 22:54:23.366 | INFO | Finished a round of Baz IO: Wrote 1.911 GiB in 3.227 seconds | 20260505 22:54:23.366 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.47% / 0.00% / 78.53% | 20260505 22:54:23.366 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.115045% / 0% / 9.13938e-05% | 20260505 22:54:23.366 | INFO | Baz IO overall: | 20260505 22:54:23.366 | INFO | BytesWritten : 3989704397522 | 20260505 22:54:23.366 | INFO | NumEvents : 1880844147283 | 20260505 22:54:25.159 | INFO | Analyzing chunk frames = [6950912,6951424) | 20260505 22:54:25.159 | INFO | MemInfo: { | 20260505 22:54:25.159 | INFO | "gpu" : " 17 GB", | 20260505 22:54:25.159 | INFO | "huge" : "", | 20260505 22:54:25.159 | INFO | "huge_shared" : " 27 GB", | 20260505 22:54:25.159 | INFO | "malloc" : " 14 GB", | 20260505 22:54:25.159 | INFO | "pinned" : " 29 GB" | 20260505 22:54:25.159 | INFO | } | 20260505 22:54:25.159 | INFO | Perf report for the last 5.13832 seconds: | 20260505 22:54:25.159 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:54:25.159 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:25.159 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:25.159 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:25.159 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260505 22:54:25.159 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:25.159 | INFO | RT_METRICS : 43.6%, 1.0, 22.8ms, 0 | 20260505 22:54:25.159 | INFO | PRE_HQ : 60.1%, 1.7, 54.5ms, 0 | 20260505 22:54:25.159 | INFO | BAZWRITER : 62.0%, 3.6, 114.4ms, 0 | 20260505 22:54:25.159 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 22:54:25.159 | INFO | Uploaded 36GiB to GPU in 2.1652s (16.6266GiB/s) | 20260505 22:54:25.159 | INFO | Downloaded 38GiB to GPU in 2.23675s (16.9889GiB/s) | 20260505 22:54:30.300 | INFO | Analyzing chunk frames = [6951424,6951936) | 20260505 22:54:30.300 | INFO | MemInfo: { | 20260505 22:54:30.300 | INFO | "gpu" : " 17 GB", | 20260505 22:54:30.300 | INFO | "huge" : "", | 20260505 22:54:30.300 | INFO | "huge_shared" : " 27 GB", | 20260505 22:54:30.300 | INFO | "malloc" : " 14 GB", | 20260505 22:54:30.300 | INFO | "pinned" : " 29 GB" | 20260505 22:54:30.300 | INFO | } | 20260505 22:54:30.300 | INFO | Perf report for the last 5.14074 seconds: | 20260505 22:54:30.300 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:54:30.300 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:30.300 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:30.300 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:30.300 | INFO | ANALYSIS : 61.8%, 3.0, 96.6ms, 0 | 20260505 22:54:30.300 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:30.300 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:30.300 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:54:30.300 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:30.300 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:30.300 | INFO | Uploaded 36GiB to GPU in 2.08255s (17.2865GiB/s) | 20260505 22:54:30.300 | INFO | Downloaded 34GiB to GPU in 2.02797s (16.7655GiB/s) | 20260505 22:54:35.437 | INFO | Analyzing chunk frames = [6951936,6952448) | 20260505 22:54:35.437 | INFO | MemInfo: { | 20260505 22:54:35.437 | INFO | "gpu" : " 17 GB", | 20260505 22:54:35.437 | INFO | "huge" : "", | 20260505 22:54:35.437 | INFO | "huge_shared" : " 27 GB", | 20260505 22:54:35.437 | INFO | "malloc" : " 14 GB", | 20260505 22:54:35.437 | INFO | "pinned" : " 29 GB" | 20260505 22:54:35.437 | INFO | } | 20260505 22:54:35.437 | INFO | Perf report for the last 5.1374 seconds: | 20260505 22:54:35.437 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:54:35.437 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:35.437 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:35.437 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:35.437 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260505 22:54:35.437 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:35.437 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:35.437 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 22:54:35.437 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:35.437 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:35.438 | INFO | Uploaded 36GiB to GPU in 2.15222s (16.7269GiB/s) | 20260505 22:54:35.438 | INFO | Downloaded 34GiB to GPU in 2.05809s (16.5202GiB/s) | 20260505 22:54:40.577 | INFO | Analyzing chunk frames = [6952448,6952960) | 20260505 22:54:40.577 | INFO | MemInfo: { | 20260505 22:54:40.577 | INFO | "gpu" : " 17 GB", | 20260505 22:54:40.577 | INFO | "huge" : "", | 20260505 22:54:40.577 | INFO | "huge_shared" : " 27 GB", | 20260505 22:54:40.577 | INFO | "malloc" : " 14 GB", | 20260505 22:54:40.577 | INFO | "pinned" : " 29 GB" | 20260505 22:54:40.577 | INFO | } | 20260505 22:54:40.577 | INFO | Perf report for the last 5.13966 seconds: | 20260505 22:54:40.577 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:54:40.577 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:40.577 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:40.577 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:40.577 | INFO | ANALYSIS : 61.8%, 3.0, 96.0ms, 0 | 20260505 22:54:40.577 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:40.577 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:40.577 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:54:40.577 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:40.577 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:40.577 | INFO | Uploaded 36GiB to GPU in 2.14463s (16.7861GiB/s) | 20260505 22:54:40.577 | INFO | Downloaded 34GiB to GPU in 2.01967s (16.8344GiB/s) | 20260505 22:54:45.717 | INFO | Analyzing chunk frames = [6952960,6953472) | 20260505 22:54:45.717 | INFO | MemInfo: { | 20260505 22:54:45.717 | INFO | "gpu" : " 17 GB", | 20260505 22:54:45.717 | INFO | "huge" : "", | 20260505 22:54:45.717 | INFO | "huge_shared" : " 27 GB", | 20260505 22:54:45.717 | INFO | "malloc" : " 14 GB", | 20260505 22:54:45.717 | INFO | "pinned" : " 29 GB" | 20260505 22:54:45.717 | INFO | } | 20260505 22:54:45.717 | INFO | Perf report for the last 5.14067 seconds: | 20260505 22:54:45.717 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:54:45.717 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:45.717 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:45.717 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:45.718 | INFO | ANALYSIS : 61.2%, 3.0, 95.6ms, 0 | 20260505 22:54:45.718 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:45.718 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:45.718 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:54:45.718 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:45.718 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:45.718 | INFO | Uploaded 36GiB to GPU in 2.10363s (17.1133GiB/s) | 20260505 22:54:45.718 | INFO | Downloaded 34GiB to GPU in 2.04869s (16.596GiB/s) | 20260505 22:54:50.849 | INFO | Analyzing chunk frames = [6953472,6953984) | 20260505 22:54:50.849 | INFO | MemInfo: { | 20260505 22:54:50.849 | INFO | "gpu" : " 17 GB", | 20260505 22:54:50.849 | INFO | "huge" : "", | 20260505 22:54:50.850 | INFO | "huge_shared" : " 27 GB", | 20260505 22:54:50.850 | INFO | "malloc" : " 14 GB", | 20260505 22:54:50.850 | INFO | "pinned" : " 29 GB" | 20260505 22:54:50.850 | INFO | } | 20260505 22:54:50.850 | INFO | Perf report for the last 5.13221 seconds: | 20260505 22:54:50.850 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:54:50.850 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:50.850 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:50.850 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:50.850 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260505 22:54:50.850 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:50.850 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:50.850 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:54:50.850 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:50.850 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:50.850 | INFO | Uploaded 36GiB to GPU in 2.08216s (17.2897GiB/s) | 20260505 22:54:50.850 | INFO | Downloaded 34GiB to GPU in 2.04058s (16.662GiB/s) | 20260505 22:54:55.990 | INFO | Analyzing chunk frames = [6953984,6954496) | 20260505 22:54:55.990 | INFO | MemInfo: { | 20260505 22:54:55.990 | INFO | "gpu" : " 17 GB", | 20260505 22:54:55.990 | INFO | "huge" : "", | 20260505 22:54:55.990 | INFO | "huge_shared" : " 27 GB", | 20260505 22:54:55.990 | INFO | "malloc" : " 14 GB", | 20260505 22:54:55.990 | INFO | "pinned" : " 29 GB" | 20260505 22:54:55.990 | INFO | } | 20260505 22:54:55.990 | INFO | Perf report for the last 5.1401 seconds: | 20260505 22:54:55.990 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:54:55.990 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:55.990 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:55.990 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:55.990 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 22:54:55.990 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:55.990 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:55.990 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 22:54:55.990 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:55.990 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:54:55.990 | INFO | Uploaded 36GiB to GPU in 2.14164s (16.8095GiB/s) | 20260505 22:54:55.990 | INFO | Downloaded 34GiB to GPU in 2.14277s (15.8673GiB/s) | 20260505 22:55:01.133 | INFO | Analyzing chunk frames = [6954496,6955008) | 20260505 22:55:01.133 | INFO | MemInfo: { | 20260505 22:55:01.133 | INFO | "gpu" : " 17 GB", | 20260505 22:55:01.133 | INFO | "huge" : "", | 20260505 22:55:01.133 | INFO | "huge_shared" : " 27 GB", | 20260505 22:55:01.133 | INFO | "malloc" : " 14 GB", | 20260505 22:55:01.134 | INFO | "pinned" : " 29 GB" | 20260505 22:55:01.134 | INFO | } | 20260505 22:55:01.134 | INFO | Perf report for the last 5.14388 seconds: | 20260505 22:55:01.134 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:55:01.134 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:01.134 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:01.134 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:01.134 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260505 22:55:01.134 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:01.134 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:01.134 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:55:01.134 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:01.134 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:01.134 | INFO | Uploaded 36GiB to GPU in 2.10367s (17.113GiB/s) | 20260505 22:55:01.134 | INFO | Downloaded 34GiB to GPU in 2.00501s (16.9575GiB/s) | 20260505 22:55:01.134 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 22:55:01.134 | INFO | Total mem usage: 15.3 GiB | 20260505 22:55:01.134 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 22:55:01.134 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.1%) | 20260505 22:55:01.134 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 22:55:01.134 | INFO | High Water Breakdown: | 20260505 22:55:01.134 | INFO | 60.9% : 8.55 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 22:55:01.134 | INFO | 39% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 22:55:01.134 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 22:55:01.134 | INFO | ------------------------------------------ | 20260505 22:55:01.134 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:55:01.134 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 22:55:01.134 | INFO | Total mem usage: 26.5 GiB | 20260505 22:55:01.134 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 22:55:01.134 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 22:55:01.134 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 22:55:01.134 | INFO | High Water Breakdown: | 20260505 22:55:01.134 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:55:01.134 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:55:01.135 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:55:01.135 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:55:01.135 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:55:01.135 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:55:01.135 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 22:55:01.135 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 22:55:01.135 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:55:01.135 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 22:55:01.135 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 22:55:01.135 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 22:55:01.135 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 22:55:01.135 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 22:55:01.135 | INFO | ------------------------------------------ | 20260505 22:55:01.135 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:55:01.135 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 22:55:01.135 | INFO | Total mem usage: 25.6 GiB | 20260505 22:55:01.135 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 22:55:01.135 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 22:55:01.135 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 22:55:01.135 | INFO | High Water Breakdown: | 20260505 22:55:01.135 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 22:55:01.135 | INFO | ------------------------------------------ | 20260505 22:55:01.135 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:55:01.135 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 22:55:01.135 | INFO | Total mem usage: 15.7 GiB | 20260505 22:55:01.135 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 22:55:01.135 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 22:55:01.135 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 22:55:01.135 | INFO | High Water Breakdown: | 20260505 22:55:01.135 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 22:55:01.135 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 22:55:01.135 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 22:55:01.135 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 22:55:01.135 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 22:55:01.135 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 22:55:01.135 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 22:55:01.135 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 22:55:01.135 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 22:55:01.135 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:55:01.135 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 22:55:01.135 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:55:01.135 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 22:55:01.135 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 22:55:01.135 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 22:55:01.135 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 22:55:01.135 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:55:01.135 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 22:55:01.135 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:55:01.135 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:55:01.135 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:55:01.135 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 22:55:01.135 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 22:55:01.135 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:55:01.135 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 22:55:01.135 | INFO | ------------------------------------------ | 20260505 22:55:01.135 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:55:01.135 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 22:55:01.135 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:55:01.135 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:55:01.135 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.42e+01, 1.35e+01 / 1.53e+00 | 20260505 22:55:01.135 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.04e-01, 1.03e-02 / 2.41e-03 | 20260505 22:55:01.135 | INFO | PerfReport: 11.80%, Baseline -- 3.70e+00 / 4.56e+00, 5.65e-01 / 1.44e-01 | 20260505 22:55:01.135 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.53e+00, 1.88e-01 / 9.50e-02 | 20260505 22:55:01.135 | INFO | PerfReport: 16.02%, DME -- 4.02e+01 / 4.42e+01, 3.81e+00 / 1.61e+00 | 20260505 22:55:01.135 | INFO | PerfReport: 57.24%, FrameLabeling -- 1.79e+01 / 1.94e+01, 1.45e+00 / 7.71e-01 | 20260505 22:55:01.135 | INFO | PerfReport: 7.15%, PulseAccumulating -- 2.24e+00 / 2.78e+00, 4.32e-01 / 1.76e-01 | 20260505 22:55:01.135 | INFO | PerfReport: 3.14%, Metrics -- 9.83e-01 / 1.26e+00, 2.65e-01 / 1.43e-01 | 20260505 22:55:01.135 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.41e-01 / 3.96e-01, 5.83e-02 / 3.54e-02 | 20260505 22:55:01.135 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 22:55:01.135 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:55:01.135 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:55:01.135 | INFO | PerfReport: Full Execution -- 9.87e+01 / -nan, 2.50e+01 / 0.00e+00 | 20260505 22:55:01.136 | INFO | PerfReport: 22.39%, Upload -- 2.21e+01 / 0.00e+00, 7.63e+00 / 0.00e+00 | 20260505 22:55:01.136 | INFO | PerfReport: 21.84%, Download -- 2.16e+01 / 0.00e+00, 7.65e+00 / 0.00e+00 | 20260505 22:55:01.136 | INFO | PerfReport: 31.77%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 22:55:01.136 | INFO | PerfReport: 2.20%, LockUpload -- 2.17e+00 / 0.00e+00, 8.06e+00 / 0.00e+00 | 20260505 22:55:01.136 | INFO | PerfReport: 1.34%, LockDownload -- 1.32e+00 / 0.00e+00, 2.87e+00 / 0.00e+00 | 20260505 22:55:01.136 | INFO | PerfReport: 20.45%, LockCompute -- 2.02e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260505 22:55:01.136 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.55e-02 / 0.00e+00 | 20260505 22:55:01.257 | INFO | Starting a round of Baz IO | 20260505 22:55:04.404 | INFO | Populating RTMetrics report | 20260505 22:55:04.404 | INFO | RTMetrics report populated | 20260505 22:55:04.404 | INFO | Opening tmp RTMetrics json file | 20260505 22:55:04.405 | INFO | Serializing RTMetrics to json | 20260505 22:55:04.406 | INFO | Serialization done, swapping tmp file now... | 20260505 22:55:04.406 | INFO | RTMetrics done | 20260505 22:55:04.490 | INFO | Finished a round of Baz IO: Wrote 1.911 GiB in 3.232 seconds | 20260505 22:55:04.490 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.46% / 0.00% / 78.53% | 20260505 22:55:04.490 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.115903% / 3.97364e-06% / 7.94729e-05% | 20260505 22:55:04.490 | INFO | Baz IO overall: | 20260505 22:55:04.490 | INFO | BytesWritten : 3991756592321 | 20260505 22:55:04.490 | INFO | NumEvents : 1881786437505 | 20260505 22:55:06.274 | INFO | Analyzing chunk frames = [6955008,6955520) | 20260505 22:55:06.274 | INFO | MemInfo: { | 20260505 22:55:06.274 | INFO | "gpu" : " 17 GB", | 20260505 22:55:06.274 | INFO | "huge" : "", | 20260505 22:55:06.274 | INFO | "huge_shared" : " 27 GB", | 20260505 22:55:06.274 | INFO | "malloc" : " 14 GB", | 20260505 22:55:06.274 | INFO | "pinned" : " 29 GB" | 20260505 22:55:06.274 | INFO | } | 20260505 22:55:06.274 | INFO | Perf report for the last 5.14037 seconds: | 20260505 22:55:06.274 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:55:06.274 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:06.274 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:06.274 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:06.274 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 22:55:06.274 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:06.274 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260505 22:55:06.274 | INFO | PRE_HQ : 60.0%, 1.8, 54.8ms, 0 | 20260505 22:55:06.274 | INFO | BAZWRITER : 62.1%, 3.6, 115.5ms, 0 | 20260505 22:55:06.274 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 22:55:06.274 | INFO | Uploaded 36GiB to GPU in 2.16605s (16.6201GiB/s) | 20260505 22:55:06.274 | INFO | Downloaded 38GiB to GPU in 2.23395s (17.0102GiB/s) | 20260505 22:55:11.411 | INFO | Analyzing chunk frames = [6955520,6956032) | 20260505 22:55:11.411 | INFO | MemInfo: { | 20260505 22:55:11.411 | INFO | "gpu" : " 17 GB", | 20260505 22:55:11.411 | INFO | "huge" : "", | 20260505 22:55:11.411 | INFO | "huge_shared" : " 27 GB", | 20260505 22:55:11.411 | INFO | "malloc" : " 14 GB", | 20260505 22:55:11.411 | INFO | "pinned" : " 29 GB" | 20260505 22:55:11.411 | INFO | } | 20260505 22:55:11.411 | INFO | Perf report for the last 5.13674 seconds: | 20260505 22:55:11.411 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:55:11.411 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:11.411 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:11.411 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:11.411 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 22:55:11.411 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:11.411 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:11.411 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260505 22:55:11.411 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:11.411 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:11.411 | INFO | Uploaded 36GiB to GPU in 2.08431s (17.2719GiB/s) | 20260505 22:55:11.411 | INFO | Downloaded 34GiB to GPU in 2.03541s (16.7042GiB/s) | 20260505 22:55:16.544 | INFO | Analyzing chunk frames = [6956032,6956544) | 20260505 22:55:16.544 | INFO | MemInfo: { | 20260505 22:55:16.544 | INFO | "gpu" : " 17 GB", | 20260505 22:55:16.544 | INFO | "huge" : "", | 20260505 22:55:16.544 | INFO | "huge_shared" : " 27 GB", | 20260505 22:55:16.544 | INFO | "malloc" : " 14 GB", | 20260505 22:55:16.545 | INFO | "pinned" : " 29 GB" | 20260505 22:55:16.545 | INFO | } | 20260505 22:55:16.545 | INFO | Perf report for the last 5.13372 seconds: | 20260505 22:55:16.545 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:55:16.545 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:16.545 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:16.545 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:16.545 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260505 22:55:16.545 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:16.545 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:16.545 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 22:55:16.545 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:16.545 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:16.545 | INFO | Uploaded 36GiB to GPU in 2.14745s (16.7641GiB/s) | 20260505 22:55:16.545 | INFO | Downloaded 34GiB to GPU in 2.04849s (16.5976GiB/s) | 20260505 22:55:21.685 | INFO | Analyzing chunk frames = [6956544,6957056) | 20260505 22:55:21.685 | INFO | MemInfo: { | 20260505 22:55:21.685 | INFO | "gpu" : " 17 GB", | 20260505 22:55:21.685 | INFO | "huge" : "", | 20260505 22:55:21.685 | INFO | "huge_shared" : " 27 GB", | 20260505 22:55:21.685 | INFO | "malloc" : " 14 GB", | 20260505 22:55:21.685 | INFO | "pinned" : " 29 GB" | 20260505 22:55:21.685 | INFO | } | 20260505 22:55:21.685 | INFO | Perf report for the last 5.1408 seconds: | 20260505 22:55:21.685 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:55:21.685 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:21.685 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:21.685 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:21.685 | INFO | ANALYSIS : 61.9%, 3.0, 96.2ms, 0 | 20260505 22:55:21.685 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:21.685 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:21.686 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 22:55:21.686 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:21.686 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:21.686 | INFO | Uploaded 36GiB to GPU in 2.14033s (16.8198GiB/s) | 20260505 22:55:21.686 | INFO | Downloaded 34GiB to GPU in 2.0563s (16.5346GiB/s) | 20260505 22:55:26.825 | INFO | Analyzing chunk frames = [6957056,6957568) | 20260505 22:55:26.825 | INFO | MemInfo: { | 20260505 22:55:26.825 | INFO | "gpu" : " 17 GB", | 20260505 22:55:26.825 | INFO | "huge" : "", | 20260505 22:55:26.825 | INFO | "huge_shared" : " 27 GB", | 20260505 22:55:26.825 | INFO | "malloc" : " 14 GB", | 20260505 22:55:26.825 | INFO | "pinned" : " 29 GB" | 20260505 22:55:26.825 | INFO | } | 20260505 22:55:26.825 | INFO | Perf report for the last 5.13978 seconds: | 20260505 22:55:26.825 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:55:26.825 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:26.825 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:26.825 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:26.825 | INFO | ANALYSIS : 61.6%, 3.0, 96.2ms, 0 | 20260505 22:55:26.825 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:26.825 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:26.825 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:55:26.825 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:26.825 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:26.825 | INFO | Uploaded 36GiB to GPU in 2.16171s (16.6535GiB/s) | 20260505 22:55:26.826 | INFO | Downloaded 34GiB to GPU in 2.07182s (16.4107GiB/s) | 20260505 22:55:31.970 | INFO | Analyzing chunk frames = [6957568,6958080) | 20260505 22:55:31.970 | INFO | MemInfo: { | 20260505 22:55:31.970 | INFO | "gpu" : " 17 GB", | 20260505 22:55:31.970 | INFO | "huge" : "", | 20260505 22:55:31.970 | INFO | "huge_shared" : " 27 GB", | 20260505 22:55:31.970 | INFO | "malloc" : " 14 GB", | 20260505 22:55:31.970 | INFO | "pinned" : " 29 GB" | 20260505 22:55:31.970 | INFO | } | 20260505 22:55:31.970 | INFO | Perf report for the last 5.14452 seconds: | 20260505 22:55:31.970 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:55:31.970 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:31.970 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:31.970 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:31.970 | INFO | ANALYSIS : 61.3%, 3.0, 95.8ms, 0 | 20260505 22:55:31.970 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:31.970 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:31.970 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:55:31.970 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:31.970 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:31.970 | INFO | Uploaded 36GiB to GPU in 2.07049s (17.3872GiB/s) | 20260505 22:55:31.970 | INFO | Downloaded 34GiB to GPU in 2.03383s (16.7172GiB/s) | 20260505 22:55:37.106 | INFO | Analyzing chunk frames = [6958080,6958592) | 20260505 22:55:37.106 | INFO | MemInfo: { | 20260505 22:55:37.106 | INFO | "gpu" : " 17 GB", | 20260505 22:55:37.106 | INFO | "huge" : "", | 20260505 22:55:37.106 | INFO | "huge_shared" : " 27 GB", | 20260505 22:55:37.106 | INFO | "malloc" : " 14 GB", | 20260505 22:55:37.106 | INFO | "pinned" : " 29 GB" | 20260505 22:55:37.107 | INFO | } | 20260505 22:55:37.107 | INFO | Perf report for the last 5.13699 seconds: | 20260505 22:55:37.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:55:37.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:37.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:37.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:37.107 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 22:55:37.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:37.107 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:37.107 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 22:55:37.107 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:37.107 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:37.107 | INFO | Uploaded 36GiB to GPU in 2.02871s (17.7453GiB/s) | 20260505 22:55:37.107 | INFO | Downloaded 34GiB to GPU in 2.03971s (16.669GiB/s) | 20260505 22:55:42.246 | INFO | Analyzing chunk frames = [6958592,6959104) | 20260505 22:55:42.246 | INFO | MemInfo: { | 20260505 22:55:42.246 | INFO | "gpu" : " 17 GB", | 20260505 22:55:42.246 | INFO | "huge" : "", | 20260505 22:55:42.246 | INFO | "huge_shared" : " 27 GB", | 20260505 22:55:42.247 | INFO | "malloc" : " 14 GB", | 20260505 22:55:42.247 | INFO | "pinned" : " 29 GB" | 20260505 22:55:42.247 | INFO | } | 20260505 22:55:42.247 | INFO | Perf report for the last 5.13996 seconds: | 20260505 22:55:42.247 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:55:42.247 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:42.247 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:42.247 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:42.247 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 22:55:42.247 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:42.247 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:42.247 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 22:55:42.247 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:42.247 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:42.247 | INFO | Uploaded 36GiB to GPU in 2.11118s (17.0521GiB/s) | 20260505 22:55:42.247 | INFO | Downloaded 34GiB to GPU in 1.99804s (17.0167GiB/s) | 20260505 22:55:42.374 | INFO | Starting a round of Baz IO | 20260505 22:55:45.524 | INFO | Populating RTMetrics report | 20260505 22:55:45.524 | INFO | RTMetrics report populated | 20260505 22:55:45.524 | INFO | Opening tmp RTMetrics json file | 20260505 22:55:45.525 | INFO | Serializing RTMetrics to json | 20260505 22:55:45.526 | INFO | Serialization done, swapping tmp file now... | 20260505 22:55:45.527 | INFO | RTMetrics done | 20260505 22:55:45.610 | INFO | Finished a round of Baz IO: Wrote 1.911 GiB in 3.236 seconds | 20260505 22:55:45.610 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.46% / 0.00% / 78.54% | 20260505 22:55:45.610 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.115236% / 0% / 6.75519e-05% | 20260505 22:55:45.610 | INFO | Baz IO overall: | 20260505 22:55:45.610 | INFO | BytesWritten : 3993808111528 | 20260505 22:55:45.610 | INFO | NumEvents : 1882728586008 | 20260505 22:55:47.388 | INFO | Analyzing chunk frames = [6959104,6959616) | 20260505 22:55:47.388 | INFO | MemInfo: { | 20260505 22:55:47.388 | INFO | "gpu" : " 17 GB", | 20260505 22:55:47.388 | INFO | "huge" : "", | 20260505 22:55:47.388 | INFO | "huge_shared" : " 27 GB", | 20260505 22:55:47.388 | INFO | "malloc" : " 14 GB", | 20260505 22:55:47.388 | INFO | "pinned" : " 29 GB" | 20260505 22:55:47.388 | INFO | } | 20260505 22:55:47.388 | INFO | Perf report for the last 5.14145 seconds: | 20260505 22:55:47.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:55:47.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:47.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:47.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:47.388 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 22:55:47.388 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:47.388 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260505 22:55:47.388 | INFO | PRE_HQ : 59.8%, 1.8, 54.7ms, 0 | 20260505 22:55:47.388 | INFO | BAZWRITER : 62.2%, 3.6, 116.0ms, 0 | 20260505 22:55:47.388 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 22:55:47.388 | INFO | Uploaded 36GiB to GPU in 2.13777s (16.84GiB/s) | 20260505 22:55:47.388 | INFO | Downloaded 38GiB to GPU in 2.23027s (17.0383GiB/s) | 20260505 22:55:52.527 | INFO | Analyzing chunk frames = [6959616,6960128) | 20260505 22:55:52.527 | INFO | MemInfo: { | 20260505 22:55:52.527 | INFO | "gpu" : " 17 GB", | 20260505 22:55:52.527 | INFO | "huge" : "", | 20260505 22:55:52.527 | INFO | "huge_shared" : " 27 GB", | 20260505 22:55:52.527 | INFO | "malloc" : " 14 GB", | 20260505 22:55:52.527 | INFO | "pinned" : " 29 GB" | 20260505 22:55:52.527 | INFO | } | 20260505 22:55:52.527 | INFO | Perf report for the last 5.13903 seconds: | 20260505 22:55:52.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:55:52.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:52.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:52.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:52.527 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260505 22:55:52.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:52.527 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:52.527 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:55:52.527 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:52.527 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:52.527 | INFO | Uploaded 36GiB to GPU in 2.11308s (17.0368GiB/s) | 20260505 22:55:52.528 | INFO | Downloaded 34GiB to GPU in 2.04511s (16.6251GiB/s) | 20260505 22:55:57.664 | INFO | Analyzing chunk frames = [6960128,6960640) | 20260505 22:55:57.664 | INFO | MemInfo: { | 20260505 22:55:57.664 | INFO | "gpu" : " 17 GB", | 20260505 22:55:57.664 | INFO | "huge" : "", | 20260505 22:55:57.664 | INFO | "huge_shared" : " 27 GB", | 20260505 22:55:57.664 | INFO | "malloc" : " 14 GB", | 20260505 22:55:57.664 | INFO | "pinned" : " 29 GB" | 20260505 22:55:57.664 | INFO | } | 20260505 22:55:57.664 | INFO | Perf report for the last 5.13728 seconds: | 20260505 22:55:57.664 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:55:57.664 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:57.665 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:57.665 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:57.665 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260505 22:55:57.665 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:57.665 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:57.665 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 22:55:57.665 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:57.665 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:55:57.665 | INFO | Uploaded 36GiB to GPU in 2.10966s (17.0644GiB/s) | 20260505 22:55:57.665 | INFO | Downloaded 34GiB to GPU in 2.0285s (16.7611GiB/s) | 20260505 22:56:02.798 | INFO | Analyzing chunk frames = [6960640,6961152) | 20260505 22:56:02.798 | INFO | MemInfo: { | 20260505 22:56:02.798 | INFO | "gpu" : " 17 GB", | 20260505 22:56:02.798 | INFO | "huge" : "", | 20260505 22:56:02.798 | INFO | "huge_shared" : " 27 GB", | 20260505 22:56:02.798 | INFO | "malloc" : " 14 GB", | 20260505 22:56:02.798 | INFO | "pinned" : " 29 GB" | 20260505 22:56:02.798 | INFO | } | 20260505 22:56:02.798 | INFO | Perf report for the last 5.1335 seconds: | 20260505 22:56:02.798 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:56:02.798 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:02.798 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:02.798 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:02.798 | INFO | ANALYSIS : 61.8%, 3.0, 95.9ms, 0 | 20260505 22:56:02.798 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:02.798 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:02.798 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:56:02.798 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:02.798 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:02.798 | INFO | Uploaded 36GiB to GPU in 2.12891s (16.9101GiB/s) | 20260505 22:56:02.798 | INFO | Downloaded 34GiB to GPU in 2.0325s (16.7281GiB/s) | 20260505 22:56:07.940 | INFO | Analyzing chunk frames = [6961152,6961664) | 20260505 22:56:07.940 | INFO | MemInfo: { | 20260505 22:56:07.940 | INFO | "gpu" : " 17 GB", | 20260505 22:56:07.940 | INFO | "huge" : "", | 20260505 22:56:07.940 | INFO | "huge_shared" : " 27 GB", | 20260505 22:56:07.940 | INFO | "malloc" : " 14 GB", | 20260505 22:56:07.940 | INFO | "pinned" : " 29 GB" | 20260505 22:56:07.940 | INFO | } | 20260505 22:56:07.940 | INFO | Perf report for the last 5.14235 seconds: | 20260505 22:56:07.940 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:56:07.940 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:07.940 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:07.940 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:07.940 | INFO | ANALYSIS : 61.5%, 3.0, 96.1ms, 0 | 20260505 22:56:07.940 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:07.940 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:07.940 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 22:56:07.941 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:07.941 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:07.941 | INFO | Uploaded 36GiB to GPU in 2.13322s (16.8759GiB/s) | 20260505 22:56:07.941 | INFO | Downloaded 34GiB to GPU in 2.0583s (16.5185GiB/s) | 20260505 22:56:13.088 | INFO | Analyzing chunk frames = [6961664,6962176) | 20260505 22:56:13.088 | INFO | MemInfo: { | 20260505 22:56:13.088 | INFO | "gpu" : " 17 GB", | 20260505 22:56:13.088 | INFO | "huge" : "", | 20260505 22:56:13.088 | INFO | "huge_shared" : " 27 GB", | 20260505 22:56:13.088 | INFO | "malloc" : " 14 GB", | 20260505 22:56:13.088 | INFO | "pinned" : " 29 GB" | 20260505 22:56:13.088 | INFO | } | 20260505 22:56:13.088 | INFO | Perf report for the last 5.14766 seconds: | 20260505 22:56:13.088 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:56:13.088 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:13.088 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:13.088 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:13.088 | INFO | ANALYSIS : 61.5%, 3.0, 96.2ms, 0 | 20260505 22:56:13.088 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:13.088 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:13.088 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260505 22:56:13.088 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:13.088 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:13.088 | INFO | Uploaded 36GiB to GPU in 2.08975s (17.227GiB/s) | 20260505 22:56:13.089 | INFO | Downloaded 34GiB to GPU in 2.04052s (16.6624GiB/s) | 20260505 22:56:18.226 | INFO | Analyzing chunk frames = [6962176,6962688) | 20260505 22:56:18.226 | INFO | MemInfo: { | 20260505 22:56:18.226 | INFO | "gpu" : " 17 GB", | 20260505 22:56:18.226 | INFO | "huge" : "", | 20260505 22:56:18.226 | INFO | "huge_shared" : " 27 GB", | 20260505 22:56:18.226 | INFO | "malloc" : " 14 GB", | 20260505 22:56:18.226 | INFO | "pinned" : " 29 GB" | 20260505 22:56:18.226 | INFO | } | 20260505 22:56:18.226 | INFO | Perf report for the last 5.13807 seconds: | 20260505 22:56:18.226 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:56:18.226 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:18.226 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:18.226 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:18.226 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260505 22:56:18.226 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:18.226 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:18.226 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 22:56:18.226 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:18.226 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:18.226 | INFO | Uploaded 36GiB to GPU in 2.08813s (17.2403GiB/s) | 20260505 22:56:18.227 | INFO | Downloaded 34GiB to GPU in 2.08601s (16.299GiB/s) | 20260505 22:56:23.358 | INFO | Analyzing chunk frames = [6962688,6963200) | 20260505 22:56:23.358 | INFO | MemInfo: { | 20260505 22:56:23.358 | INFO | "gpu" : " 17 GB", | 20260505 22:56:23.358 | INFO | "huge" : "", | 20260505 22:56:23.358 | INFO | "huge_shared" : " 27 GB", | 20260505 22:56:23.358 | INFO | "malloc" : " 14 GB", | 20260505 22:56:23.358 | INFO | "pinned" : " 29 GB" | 20260505 22:56:23.358 | INFO | } | 20260505 22:56:23.358 | INFO | Perf report for the last 5.1321 seconds: | 20260505 22:56:23.358 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:56:23.358 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:23.358 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:23.358 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:23.358 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 22:56:23.358 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:23.358 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:23.358 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 22:56:23.358 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:23.358 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:23.358 | INFO | Uploaded 36GiB to GPU in 2.12157s (16.9686GiB/s) | 20260505 22:56:23.359 | INFO | Downloaded 34GiB to GPU in 2.01425s (16.8797GiB/s) | 20260505 22:56:23.359 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 22:56:23.359 | INFO | Total mem usage: 15.3 GiB | 20260505 22:56:23.359 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 22:56:23.359 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.26%) | 20260505 22:56:23.359 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 22:56:23.359 | INFO | High Water Breakdown: | 20260505 22:56:23.359 | INFO | 61% : 8.64 GiB ( 90.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 22:56:23.359 | INFO | 38.9% : 5.5 GiB ( 82.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 22:56:23.359 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 22:56:23.359 | INFO | ------------------------------------------ | 20260505 22:56:23.359 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:56:23.359 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 22:56:23.359 | INFO | Total mem usage: 26.5 GiB | 20260505 22:56:23.359 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 22:56:23.359 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 22:56:23.359 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 22:56:23.359 | INFO | High Water Breakdown: | 20260505 22:56:23.359 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:56:23.359 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:56:23.359 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:56:23.359 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:56:23.359 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:56:23.359 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:56:23.359 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 22:56:23.359 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 22:56:23.359 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:56:23.359 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 22:56:23.359 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 22:56:23.359 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 22:56:23.359 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 22:56:23.359 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 22:56:23.359 | INFO | ------------------------------------------ | 20260505 22:56:23.359 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:56:23.359 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 22:56:23.359 | INFO | Total mem usage: 25.6 GiB | 20260505 22:56:23.359 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 22:56:23.359 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 22:56:23.359 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 22:56:23.359 | INFO | High Water Breakdown: | 20260505 22:56:23.359 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 22:56:23.359 | INFO | ------------------------------------------ | 20260505 22:56:23.359 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:56:23.359 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 22:56:23.359 | INFO | Total mem usage: 15.7 GiB | 20260505 22:56:23.359 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 22:56:23.359 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 22:56:23.359 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 22:56:23.359 | INFO | High Water Breakdown: | 20260505 22:56:23.359 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 22:56:23.359 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 22:56:23.359 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 22:56:23.359 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 22:56:23.359 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 22:56:23.360 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 22:56:23.360 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 22:56:23.360 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 22:56:23.360 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 22:56:23.360 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:56:23.360 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 22:56:23.360 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:56:23.360 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 22:56:23.360 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 22:56:23.360 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 22:56:23.360 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 22:56:23.360 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:56:23.360 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 22:56:23.360 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:56:23.360 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:56:23.360 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:56:23.360 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 22:56:23.360 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 22:56:23.360 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:56:23.360 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 22:56:23.360 | INFO | ------------------------------------------ | 20260505 22:56:23.360 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:56:23.360 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 22:56:23.360 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:56:23.360 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:56:23.360 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.40e+01, 1.34e+01 / 8.72e-01 | 20260505 22:56:23.360 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.04e-01, 1.01e-02 / 3.35e-03 | 20260505 22:56:23.360 | INFO | PerfReport: 11.81%, Baseline -- 3.71e+00 / 4.48e+00, 5.69e-01 / 2.52e-01 | 20260505 22:56:23.360 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.53e+00, 1.90e-01 / 9.76e-02 | 20260505 22:56:23.360 | INFO | PerfReport: 15.97%, DME -- 4.01e+01 / 4.40e+01, 4.00e+00 / 1.04e+00 | 20260505 22:56:23.360 | INFO | PerfReport: 57.31%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.52e+00 / 1.13e+00 | 20260505 22:56:23.360 | INFO | PerfReport: 7.13%, PulseAccumulating -- 2.24e+00 / 2.76e+00, 4.28e-01 / 2.52e-01 | 20260505 22:56:23.360 | INFO | PerfReport: 3.13%, Metrics -- 9.81e-01 / 1.28e+00, 2.69e-01 / 9.39e-02 | 20260505 22:56:23.360 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.41e-01 / 4.01e-01, 5.93e-02 / 3.72e-02 | 20260505 22:56:23.360 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 22:56:23.360 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:56:23.360 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:56:23.360 | INFO | PerfReport: Full Execution -- 9.86e+01 / -nan, 2.49e+01 / 0.00e+00 | 20260505 22:56:23.360 | INFO | PerfReport: 22.34%, Upload -- 2.20e+01 / 0.00e+00, 7.57e+00 / 0.00e+00 | 20260505 22:56:23.360 | INFO | PerfReport: 21.82%, Download -- 2.15e+01 / 0.00e+00, 7.68e+00 / 0.00e+00 | 20260505 22:56:23.360 | INFO | PerfReport: 31.82%, Compute -- 3.14e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 22:56:23.360 | INFO | PerfReport: 2.13%, LockUpload -- 2.10e+00 / 0.00e+00, 7.85e+00 / 0.00e+00 | 20260505 22:56:23.360 | INFO | PerfReport: 1.35%, LockDownload -- 1.33e+00 / 0.00e+00, 2.95e+00 / 0.00e+00 | 20260505 22:56:23.360 | INFO | PerfReport: 20.53%, LockCompute -- 2.02e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260505 22:56:23.360 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.38e-02 / 0.00e+00 | 20260505 22:56:23.490 | INFO | Starting a round of Baz IO | 20260505 22:56:26.641 | INFO | Populating RTMetrics report | 20260505 22:56:26.641 | INFO | RTMetrics report populated | 20260505 22:56:26.641 | INFO | Opening tmp RTMetrics json file | 20260505 22:56:26.642 | INFO | Serializing RTMetrics to json | 20260505 22:56:26.645 | INFO | Serialization done, swapping tmp file now... | 20260505 22:56:26.647 | INFO | RTMetrics done | 20260505 22:56:26.723 | INFO | Finished a round of Baz IO: Wrote 1.909 GiB in 3.233 seconds | 20260505 22:56:26.723 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.46% / 0.00% / 78.54% | 20260505 22:56:26.723 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.115037% / 0% / 7.54992e-05% | 20260505 22:56:26.723 | INFO | Baz IO overall: | 20260505 22:56:26.723 | INFO | BytesWritten : 3995857912410 | 20260505 22:56:26.723 | INFO | NumEvents : 1883669744148 | 20260505 22:56:28.495 | INFO | Analyzing chunk frames = [6963200,6963712) | 20260505 22:56:28.495 | INFO | MemInfo: { | 20260505 22:56:28.495 | INFO | "gpu" : " 17 GB", | 20260505 22:56:28.495 | INFO | "huge" : "", | 20260505 22:56:28.495 | INFO | "huge_shared" : " 27 GB", | 20260505 22:56:28.495 | INFO | "malloc" : " 14 GB", | 20260505 22:56:28.495 | INFO | "pinned" : " 29 GB" | 20260505 22:56:28.495 | INFO | } | 20260505 22:56:28.495 | INFO | Perf report for the last 5.13679 seconds: | 20260505 22:56:28.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:56:28.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:28.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:28.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:28.495 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260505 22:56:28.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:28.495 | INFO | RT_METRICS : 43.4%, 1.0, 22.8ms, 0 | 20260505 22:56:28.495 | INFO | PRE_HQ : 59.9%, 1.7, 54.2ms, 0 | 20260505 22:56:28.495 | INFO | BAZWRITER : 62.4%, 3.6, 115.5ms, 0 | 20260505 22:56:28.495 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 22:56:28.495 | INFO | Uploaded 36GiB to GPU in 2.17143s (16.5789GiB/s) | 20260505 22:56:28.495 | INFO | Downloaded 38GiB to GPU in 2.24685s (16.9126GiB/s) | 20260505 22:56:33.640 | INFO | Analyzing chunk frames = [6963712,6964224) | 20260505 22:56:33.640 | INFO | MemInfo: { | 20260505 22:56:33.640 | INFO | "gpu" : " 17 GB", | 20260505 22:56:33.640 | INFO | "huge" : "", | 20260505 22:56:33.640 | INFO | "huge_shared" : " 27 GB", | 20260505 22:56:33.640 | INFO | "malloc" : " 14 GB", | 20260505 22:56:33.640 | INFO | "pinned" : " 29 GB" | 20260505 22:56:33.640 | INFO | } | 20260505 22:56:33.640 | INFO | Perf report for the last 5.14544 seconds: | 20260505 22:56:33.640 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:56:33.640 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:33.640 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:33.641 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:33.641 | INFO | ANALYSIS : 61.8%, 3.0, 96.6ms, 0 | 20260505 22:56:33.641 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:33.641 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:33.641 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 22:56:33.641 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:33.641 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:33.641 | INFO | Uploaded 36GiB to GPU in 2.08334s (17.2799GiB/s) | 20260505 22:56:33.641 | INFO | Downloaded 34GiB to GPU in 2.02554s (16.7857GiB/s) | 20260505 22:56:38.783 | INFO | Analyzing chunk frames = [6964224,6964736) | 20260505 22:56:38.783 | INFO | MemInfo: { | 20260505 22:56:38.783 | INFO | "gpu" : " 17 GB", | 20260505 22:56:38.783 | INFO | "huge" : "", | 20260505 22:56:38.783 | INFO | "huge_shared" : " 27 GB", | 20260505 22:56:38.783 | INFO | "malloc" : " 14 GB", | 20260505 22:56:38.783 | INFO | "pinned" : " 29 GB" | 20260505 22:56:38.783 | INFO | } | 20260505 22:56:38.784 | INFO | Perf report for the last 5.14301 seconds: | 20260505 22:56:38.784 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:56:38.784 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:38.784 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:38.784 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:38.784 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260505 22:56:38.784 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:38.784 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:38.784 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 22:56:38.784 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:38.784 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:38.784 | INFO | Uploaded 36GiB to GPU in 2.05345s (17.5315GiB/s) | 20260505 22:56:38.784 | INFO | Downloaded 34GiB to GPU in 2.01622s (16.8632GiB/s) | 20260505 22:56:43.922 | INFO | Analyzing chunk frames = [6964736,6965248) | 20260505 22:56:43.922 | INFO | MemInfo: { | 20260505 22:56:43.922 | INFO | "gpu" : " 17 GB", | 20260505 22:56:43.922 | INFO | "huge" : "", | 20260505 22:56:43.922 | INFO | "huge_shared" : " 27 GB", | 20260505 22:56:43.922 | INFO | "malloc" : " 14 GB", | 20260505 22:56:43.922 | INFO | "pinned" : " 29 GB" | 20260505 22:56:43.922 | INFO | } | 20260505 22:56:43.922 | INFO | Perf report for the last 5.13843 seconds: | 20260505 22:56:43.922 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:56:43.922 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:43.922 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:43.922 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:43.922 | INFO | ANALYSIS : 61.7%, 3.0, 95.8ms, 0 | 20260505 22:56:43.922 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:43.922 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:43.922 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 22:56:43.922 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:43.922 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:43.922 | INFO | Uploaded 36GiB to GPU in 2.13552s (16.8577GiB/s) | 20260505 22:56:43.922 | INFO | Downloaded 34GiB to GPU in 2.0107s (16.9095GiB/s) | 20260505 22:56:49.060 | INFO | Analyzing chunk frames = [6965248,6965760) | 20260505 22:56:49.060 | INFO | MemInfo: { | 20260505 22:56:49.060 | INFO | "gpu" : " 17 GB", | 20260505 22:56:49.060 | INFO | "huge" : "", | 20260505 22:56:49.060 | INFO | "huge_shared" : " 27 GB", | 20260505 22:56:49.060 | INFO | "malloc" : " 14 GB", | 20260505 22:56:49.060 | INFO | "pinned" : " 29 GB" | 20260505 22:56:49.060 | INFO | } | 20260505 22:56:49.060 | INFO | Perf report for the last 5.13799 seconds: | 20260505 22:56:49.060 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:56:49.060 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:49.060 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:49.060 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:49.060 | INFO | ANALYSIS : 61.2%, 3.0, 95.6ms, 0 | 20260505 22:56:49.060 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:49.060 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:49.060 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 22:56:49.060 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:49.060 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:49.060 | INFO | Uploaded 36GiB to GPU in 2.09226s (17.2063GiB/s) | 20260505 22:56:49.060 | INFO | Downloaded 34GiB to GPU in 2.04521s (16.6242GiB/s) | 20260505 22:56:54.199 | INFO | Analyzing chunk frames = [6965760,6966272) | 20260505 22:56:54.199 | INFO | MemInfo: { | 20260505 22:56:54.199 | INFO | "gpu" : " 17 GB", | 20260505 22:56:54.199 | INFO | "huge" : "", | 20260505 22:56:54.199 | INFO | "huge_shared" : " 27 GB", | 20260505 22:56:54.200 | INFO | "malloc" : " 14 GB", | 20260505 22:56:54.200 | INFO | "pinned" : " 29 GB" | 20260505 22:56:54.200 | INFO | } | 20260505 22:56:54.200 | INFO | Perf report for the last 5.13976 seconds: | 20260505 22:56:54.200 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:56:54.200 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:54.200 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:54.200 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:54.200 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260505 22:56:54.200 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:54.200 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:54.200 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 22:56:54.200 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:54.200 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:54.200 | INFO | Uploaded 36GiB to GPU in 2.07451s (17.3535GiB/s) | 20260505 22:56:54.200 | INFO | Downloaded 34GiB to GPU in 2.03693s (16.6918GiB/s) | 20260505 22:56:59.343 | INFO | Analyzing chunk frames = [6966272,6966784) | 20260505 22:56:59.344 | INFO | MemInfo: { | 20260505 22:56:59.344 | INFO | "gpu" : " 17 GB", | 20260505 22:56:59.344 | INFO | "huge" : "", | 20260505 22:56:59.344 | INFO | "huge_shared" : " 27 GB", | 20260505 22:56:59.344 | INFO | "malloc" : " 14 GB", | 20260505 22:56:59.344 | INFO | "pinned" : " 29 GB" | 20260505 22:56:59.344 | INFO | } | 20260505 22:56:59.344 | INFO | Perf report for the last 5.14401 seconds: | 20260505 22:56:59.344 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:56:59.344 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:59.344 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:59.344 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:59.344 | INFO | ANALYSIS : 62.1%, 3.0, 97.1ms, 0 | 20260505 22:56:59.344 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:59.344 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:59.344 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 22:56:59.344 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:59.344 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:56:59.344 | INFO | Uploaded 36GiB to GPU in 2.02174s (17.8065GiB/s) | 20260505 22:56:59.344 | INFO | Downloaded 34GiB to GPU in 2.04012s (16.6657GiB/s) | 20260505 22:57:04.473 | INFO | Analyzing chunk frames = [6966784,6967296) | 20260505 22:57:04.473 | INFO | MemInfo: { | 20260505 22:57:04.473 | INFO | "gpu" : " 17 GB", | 20260505 22:57:04.473 | INFO | "huge" : "", | 20260505 22:57:04.473 | INFO | "huge_shared" : " 27 GB", | 20260505 22:57:04.473 | INFO | "malloc" : " 14 GB", | 20260505 22:57:04.473 | INFO | "pinned" : " 29 GB" | 20260505 22:57:04.473 | INFO | } | 20260505 22:57:04.473 | INFO | Perf report for the last 5.12937 seconds: | 20260505 22:57:04.473 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:57:04.473 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:04.473 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:04.473 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:04.473 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260505 22:57:04.473 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:04.473 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:04.473 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 22:57:04.473 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:04.473 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:04.473 | INFO | Uploaded 36GiB to GPU in 2.1385s (16.8343GiB/s) | 20260505 22:57:04.473 | INFO | Downloaded 34GiB to GPU in 2.02808s (16.7646GiB/s) | 20260505 22:57:04.601 | INFO | Starting a round of Baz IO | 20260505 22:57:07.741 | INFO | Populating RTMetrics report | 20260505 22:57:07.741 | INFO | RTMetrics report populated | 20260505 22:57:07.741 | INFO | Opening tmp RTMetrics json file | 20260505 22:57:07.742 | INFO | Serializing RTMetrics to json | 20260505 22:57:07.744 | INFO | Serialization done, swapping tmp file now... | 20260505 22:57:07.744 | INFO | RTMetrics done | 20260505 22:57:07.823 | INFO | Finished a round of Baz IO: Wrote 1.91 GiB in 3.222 seconds | 20260505 22:57:07.823 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.46% / 0.00% / 78.54% | 20260505 22:57:07.823 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.115005% / 0% / 8.34465e-05% | 20260505 22:57:07.823 | INFO | Baz IO overall: | 20260505 22:57:07.823 | INFO | BytesWritten : 3997909251317 | 20260505 22:57:07.823 | INFO | NumEvents : 1884612173826 | 20260505 22:57:09.615 | INFO | Analyzing chunk frames = [6967296,6967808) | 20260505 22:57:09.615 | INFO | MemInfo: { | 20260505 22:57:09.615 | INFO | "gpu" : " 17 GB", | 20260505 22:57:09.615 | INFO | "huge" : "", | 20260505 22:57:09.615 | INFO | "huge_shared" : " 27 GB", | 20260505 22:57:09.615 | INFO | "malloc" : " 14 GB", | 20260505 22:57:09.615 | INFO | "pinned" : " 29 GB" | 20260505 22:57:09.615 | INFO | } | 20260505 22:57:09.615 | INFO | Perf report for the last 5.1421 seconds: | 20260505 22:57:09.615 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:57:09.615 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:09.615 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:09.615 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:09.615 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260505 22:57:09.615 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:09.615 | INFO | RT_METRICS : 43.4%, 1.0, 22.8ms, 0 | 20260505 22:57:09.615 | INFO | PRE_HQ : 59.6%, 1.8, 54.2ms, 0 | 20260505 22:57:09.615 | INFO | BAZWRITER : 62.0%, 3.6, 114.4ms, 0 | 20260505 22:57:09.615 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 22:57:09.616 | INFO | Uploaded 36GiB to GPU in 2.11515s (17.0201GiB/s) | 20260505 22:57:09.616 | INFO | Downloaded 38GiB to GPU in 2.22592s (17.0716GiB/s) | 20260505 22:57:14.752 | INFO | Analyzing chunk frames = [6967808,6968320) | 20260505 22:57:14.753 | INFO | MemInfo: { | 20260505 22:57:14.753 | INFO | "gpu" : " 17 GB", | 20260505 22:57:14.753 | INFO | "huge" : "", | 20260505 22:57:14.753 | INFO | "huge_shared" : " 27 GB", | 20260505 22:57:14.753 | INFO | "malloc" : " 14 GB", | 20260505 22:57:14.753 | INFO | "pinned" : " 29 GB" | 20260505 22:57:14.753 | INFO | } | 20260505 22:57:14.753 | INFO | Perf report for the last 5.13757 seconds: | 20260505 22:57:14.753 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:57:14.753 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:14.753 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:14.753 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:14.753 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 22:57:14.753 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:14.753 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:14.753 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 22:57:14.753 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:14.753 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:14.753 | INFO | Uploaded 36GiB to GPU in 2.07756s (17.3281GiB/s) | 20260505 22:57:14.753 | INFO | Downloaded 34GiB to GPU in 2.02613s (16.7807GiB/s) | 20260505 22:57:19.899 | INFO | Analyzing chunk frames = [6968320,6968832) | 20260505 22:57:19.899 | INFO | MemInfo: { | 20260505 22:57:19.899 | INFO | "gpu" : " 17 GB", | 20260505 22:57:19.899 | INFO | "huge" : "", | 20260505 22:57:19.899 | INFO | "huge_shared" : " 27 GB", | 20260505 22:57:19.899 | INFO | "malloc" : " 14 GB", | 20260505 22:57:19.899 | INFO | "pinned" : " 29 GB" | 20260505 22:57:19.899 | INFO | } | 20260505 22:57:19.899 | INFO | Perf report for the last 5.14596 seconds: | 20260505 22:57:19.899 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:57:19.899 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:19.899 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:19.899 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:19.899 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260505 22:57:19.899 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:19.899 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:19.899 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:57:19.899 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:19.899 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:19.899 | INFO | Uploaded 36GiB to GPU in 2.1572s (16.6883GiB/s) | 20260505 22:57:19.899 | INFO | Downloaded 34GiB to GPU in 2.05205s (16.5688GiB/s) | 20260505 22:57:25.037 | INFO | Analyzing chunk frames = [6968832,6969344) | 20260505 22:57:25.037 | INFO | MemInfo: { | 20260505 22:57:25.037 | INFO | "gpu" : " 17 GB", | 20260505 22:57:25.038 | INFO | "huge" : "", | 20260505 22:57:25.038 | INFO | "huge_shared" : " 27 GB", | 20260505 22:57:25.038 | INFO | "malloc" : " 14 GB", | 20260505 22:57:25.038 | INFO | "pinned" : " 29 GB" | 20260505 22:57:25.038 | INFO | } | 20260505 22:57:25.038 | INFO | Perf report for the last 5.13896 seconds: | 20260505 22:57:25.038 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:57:25.038 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:25.038 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:25.038 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:25.038 | INFO | ANALYSIS : 61.8%, 3.0, 95.9ms, 0 | 20260505 22:57:25.038 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:25.038 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:25.038 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:57:25.038 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:25.038 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:25.038 | INFO | Uploaded 36GiB to GPU in 2.14471s (16.7855GiB/s) | 20260505 22:57:25.038 | INFO | Downloaded 34GiB to GPU in 2.01856s (16.8437GiB/s) | 20260505 22:57:30.172 | INFO | Analyzing chunk frames = [6969344,6969856) | 20260505 22:57:30.172 | INFO | MemInfo: { | 20260505 22:57:30.172 | INFO | "gpu" : " 17 GB", | 20260505 22:57:30.172 | INFO | "huge" : "", | 20260505 22:57:30.172 | INFO | "huge_shared" : " 27 GB", | 20260505 22:57:30.172 | INFO | "malloc" : " 14 GB", | 20260505 22:57:30.172 | INFO | "pinned" : " 29 GB" | 20260505 22:57:30.172 | INFO | } | 20260505 22:57:30.172 | INFO | Perf report for the last 5.13438 seconds: | 20260505 22:57:30.172 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:57:30.172 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:30.172 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:30.172 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:30.172 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260505 22:57:30.172 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:30.172 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:30.172 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:57:30.172 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:30.172 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:30.172 | INFO | Uploaded 36GiB to GPU in 2.08567s (17.2606GiB/s) | 20260505 22:57:30.172 | INFO | Downloaded 34GiB to GPU in 2.04161s (16.6535GiB/s) | 20260505 22:57:35.308 | INFO | Analyzing chunk frames = [6969856,6970368) | 20260505 22:57:35.308 | INFO | MemInfo: { | 20260505 22:57:35.308 | INFO | "gpu" : " 17 GB", | 20260505 22:57:35.308 | INFO | "huge" : "", | 20260505 22:57:35.308 | INFO | "huge_shared" : " 27 GB", | 20260505 22:57:35.308 | INFO | "malloc" : " 14 GB", | 20260505 22:57:35.308 | INFO | "pinned" : " 29 GB" | 20260505 22:57:35.308 | INFO | } | 20260505 22:57:35.308 | INFO | Perf report for the last 5.13581 seconds: | 20260505 22:57:35.308 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:57:35.308 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:35.308 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:35.308 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:35.308 | INFO | ANALYSIS : 61.5%, 3.0, 96.0ms, 0 | 20260505 22:57:35.308 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:35.308 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:35.308 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 22:57:35.308 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:35.308 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:35.308 | INFO | Uploaded 36GiB to GPU in 2.07128s (17.3806GiB/s) | 20260505 22:57:35.308 | INFO | Downloaded 34GiB to GPU in 2.03036s (16.7458GiB/s) | 20260505 22:57:40.450 | INFO | Analyzing chunk frames = [6970368,6970880) | 20260505 22:57:40.450 | INFO | MemInfo: { | 20260505 22:57:40.450 | INFO | "gpu" : " 17 GB", | 20260505 22:57:40.450 | INFO | "huge" : "", | 20260505 22:57:40.450 | INFO | "huge_shared" : " 27 GB", | 20260505 22:57:40.451 | INFO | "malloc" : " 14 GB", | 20260505 22:57:40.451 | INFO | "pinned" : " 29 GB" | 20260505 22:57:40.451 | INFO | } | 20260505 22:57:40.451 | INFO | Perf report for the last 5.1427 seconds: | 20260505 22:57:40.451 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:57:40.451 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:40.451 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:40.451 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:40.451 | INFO | ANALYSIS : 62.7%, 3.0, 98.0ms, 0 | 20260505 22:57:40.451 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:40.451 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:40.451 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 22:57:40.451 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:40.451 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:40.451 | INFO | Uploaded 36GiB to GPU in 2.15183s (16.73GiB/s) | 20260505 22:57:40.451 | INFO | Downloaded 34GiB to GPU in 2.12503s (15.9998GiB/s) | 20260505 22:57:45.590 | INFO | Analyzing chunk frames = [6970880,6971392) | 20260505 22:57:45.590 | INFO | MemInfo: { | 20260505 22:57:45.590 | INFO | "gpu" : " 17 GB", | 20260505 22:57:45.590 | INFO | "huge" : "", | 20260505 22:57:45.590 | INFO | "huge_shared" : " 27 GB", | 20260505 22:57:45.590 | INFO | "malloc" : " 14 GB", | 20260505 22:57:45.590 | INFO | "pinned" : " 29 GB" | 20260505 22:57:45.590 | INFO | } | 20260505 22:57:45.590 | INFO | Perf report for the last 5.13952 seconds: | 20260505 22:57:45.590 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:57:45.590 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:45.590 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:45.590 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:45.590 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 22:57:45.590 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:45.590 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:45.590 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:57:45.590 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:45.590 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:45.591 | INFO | Uploaded 36GiB to GPU in 2.1117s (17.0479GiB/s) | 20260505 22:57:45.591 | INFO | Downloaded 34GiB to GPU in 1.99926s (17.0063GiB/s) | 20260505 22:57:45.591 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 22:57:45.591 | INFO | Total mem usage: 15.3 GiB | 20260505 22:57:45.591 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 22:57:45.591 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.55%) | 20260505 22:57:45.591 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 22:57:45.591 | INFO | High Water Breakdown: | 20260505 22:57:45.591 | INFO | 61.1% : 8.62 GiB ( 90.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 22:57:45.591 | INFO | 38.8% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 22:57:45.591 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 22:57:45.591 | INFO | ------------------------------------------ | 20260505 22:57:45.591 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:57:45.591 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 22:57:45.591 | INFO | Total mem usage: 26.5 GiB | 20260505 22:57:45.591 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 22:57:45.591 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 22:57:45.591 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 22:57:45.591 | INFO | High Water Breakdown: | 20260505 22:57:45.591 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:57:45.591 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:57:45.591 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:57:45.591 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:57:45.591 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:57:45.591 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:57:45.591 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 22:57:45.591 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 22:57:45.591 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:57:45.591 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 22:57:45.591 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 22:57:45.591 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 22:57:45.591 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 22:57:45.591 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 22:57:45.591 | INFO | ------------------------------------------ | 20260505 22:57:45.591 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:57:45.591 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 22:57:45.591 | INFO | Total mem usage: 25.6 GiB | 20260505 22:57:45.591 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 22:57:45.591 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 22:57:45.591 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 22:57:45.591 | INFO | High Water Breakdown: | 20260505 22:57:45.591 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 22:57:45.591 | INFO | ------------------------------------------ | 20260505 22:57:45.591 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:57:45.591 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 22:57:45.591 | INFO | Total mem usage: 15.7 GiB | 20260505 22:57:45.591 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 22:57:45.591 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 22:57:45.591 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 22:57:45.591 | INFO | High Water Breakdown: | 20260505 22:57:45.591 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 22:57:45.591 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 22:57:45.591 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 22:57:45.591 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 22:57:45.591 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 22:57:45.591 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 22:57:45.591 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 22:57:45.591 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 22:57:45.591 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 22:57:45.591 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:57:45.591 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 22:57:45.591 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:57:45.591 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 22:57:45.591 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 22:57:45.591 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 22:57:45.591 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 22:57:45.591 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:57:45.591 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 22:57:45.591 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:57:45.591 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:57:45.591 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:57:45.591 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 22:57:45.592 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 22:57:45.592 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:57:45.592 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 22:57:45.592 | INFO | ------------------------------------------ | 20260505 22:57:45.592 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:57:45.592 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 22:57:45.592 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:57:45.592 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:57:45.592 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.41e+01, 1.35e+01 / 1.37e+00 | 20260505 22:57:45.592 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.05e-01, 1.05e-02 / 6.79e-03 | 20260505 22:57:45.592 | INFO | PerfReport: 11.85%, Baseline -- 3.72e+00 / 4.54e+00, 5.78e-01 / 1.90e-01 | 20260505 22:57:45.592 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.52e+00, 1.90e-01 / 8.98e-02 | 20260505 22:57:45.592 | INFO | PerfReport: 15.99%, DME -- 4.02e+01 / 4.42e+01, 3.90e+00 / 1.46e+00 | 20260505 22:57:45.592 | INFO | PerfReport: 57.23%, FrameLabeling -- 1.80e+01 / 1.93e+01, 1.50e+00 / 8.27e-01 | 20260505 22:57:45.592 | INFO | PerfReport: 7.14%, PulseAccumulating -- 2.24e+00 / 2.80e+00, 4.30e-01 / 1.72e-01 | 20260505 22:57:45.592 | INFO | PerfReport: 3.13%, Metrics -- 9.83e-01 / 1.28e+00, 2.68e-01 / 1.32e-01 | 20260505 22:57:45.592 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.41e-01 / 4.03e-01, 5.90e-02 / 3.69e-02 | 20260505 22:57:45.592 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 22:57:45.592 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:57:45.592 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:57:45.592 | INFO | PerfReport: Full Execution -- 9.85e+01 / -nan, 2.49e+01 / 0.00e+00 | 20260505 22:57:45.592 | INFO | PerfReport: 22.26%, Upload -- 2.19e+01 / 0.00e+00, 7.48e+00 / 0.00e+00 | 20260505 22:57:45.592 | INFO | PerfReport: 21.79%, Download -- 2.15e+01 / 0.00e+00, 7.58e+00 / 0.00e+00 | 20260505 22:57:45.592 | INFO | PerfReport: 31.86%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 22:57:45.592 | INFO | PerfReport: 2.17%, LockUpload -- 2.14e+00 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260505 22:57:45.592 | INFO | PerfReport: 1.31%, LockDownload -- 1.29e+00 / 0.00e+00, 2.94e+00 / 0.00e+00 | 20260505 22:57:45.592 | INFO | PerfReport: 20.60%, LockCompute -- 2.03e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260505 22:57:45.592 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.28e-02 / 0.00e+00 | 20260505 22:57:45.712 | INFO | Starting a round of Baz IO | 20260505 22:57:48.860 | INFO | Populating RTMetrics report | 20260505 22:57:48.860 | INFO | RTMetrics report populated | 20260505 22:57:48.861 | INFO | Opening tmp RTMetrics json file | 20260505 22:57:48.861 | INFO | Serializing RTMetrics to json | 20260505 22:57:48.863 | INFO | Serialization done, swapping tmp file now... | 20260505 22:57:48.866 | INFO | RTMetrics done | 20260505 22:57:48.947 | INFO | Finished a round of Baz IO: Wrote 1.912 GiB in 3.235 seconds | 20260505 22:57:48.947 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.46% / 0.00% / 78.54% | 20260505 22:57:48.947 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.11518% / 3.97364e-06% / 8.34465e-05% | 20260505 22:57:48.947 | INFO | Baz IO overall: | 20260505 22:57:48.947 | INFO | BytesWritten : 3999961782915 | 20260505 22:57:48.947 | INFO | NumEvents : 1885555699379 | 20260505 22:57:50.735 | INFO | Analyzing chunk frames = [6971392,6971904) | 20260505 22:57:50.735 | INFO | MemInfo: { | 20260505 22:57:50.735 | INFO | "gpu" : " 17 GB", | 20260505 22:57:50.735 | INFO | "huge" : "", | 20260505 22:57:50.735 | INFO | "huge_shared" : " 27 GB", | 20260505 22:57:50.735 | INFO | "malloc" : " 14 GB", | 20260505 22:57:50.735 | INFO | "pinned" : " 29 GB" | 20260505 22:57:50.735 | INFO | } | 20260505 22:57:50.735 | INFO | Perf report for the last 5.1446 seconds: | 20260505 22:57:50.735 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:57:50.735 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:50.735 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:50.735 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:50.735 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260505 22:57:50.735 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:50.735 | INFO | RT_METRICS : 43.4%, 1.0, 22.8ms, 0 | 20260505 22:57:50.735 | INFO | PRE_HQ : 59.8%, 1.7, 54.3ms, 0 | 20260505 22:57:50.735 | INFO | BAZWRITER : 62.1%, 3.6, 114.3ms, 0 | 20260505 22:57:50.735 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 22:57:50.735 | INFO | Uploaded 36GiB to GPU in 2.17295s (16.5673GiB/s) | 20260505 22:57:50.735 | INFO | Downloaded 38GiB to GPU in 2.23631s (16.9922GiB/s) | 20260505 22:57:55.869 | INFO | Analyzing chunk frames = [6971904,6972416) | 20260505 22:57:55.869 | INFO | MemInfo: { | 20260505 22:57:55.869 | INFO | "gpu" : " 17 GB", | 20260505 22:57:55.869 | INFO | "huge" : "", | 20260505 22:57:55.869 | INFO | "huge_shared" : " 27 GB", | 20260505 22:57:55.869 | INFO | "malloc" : " 14 GB", | 20260505 22:57:55.869 | INFO | "pinned" : " 29 GB" | 20260505 22:57:55.869 | INFO | } | 20260505 22:57:55.869 | INFO | Perf report for the last 5.13381 seconds: | 20260505 22:57:55.869 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:57:55.869 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:55.869 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:55.869 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:55.869 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260505 22:57:55.869 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:55.869 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:55.869 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 22:57:55.869 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:55.869 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:57:55.869 | INFO | Uploaded 36GiB to GPU in 2.10544s (17.0986GiB/s) | 20260505 22:57:55.869 | INFO | Downloaded 34GiB to GPU in 2.03938s (16.6718GiB/s) | 20260505 22:58:01.006 | INFO | Analyzing chunk frames = [6972416,6972928) | 20260505 22:58:01.006 | INFO | MemInfo: { | 20260505 22:58:01.006 | INFO | "gpu" : " 17 GB", | 20260505 22:58:01.006 | INFO | "huge" : "", | 20260505 22:58:01.006 | INFO | "huge_shared" : " 27 GB", | 20260505 22:58:01.006 | INFO | "malloc" : " 14 GB", | 20260505 22:58:01.006 | INFO | "pinned" : " 29 GB" | 20260505 22:58:01.006 | INFO | } | 20260505 22:58:01.006 | INFO | Perf report for the last 5.13774 seconds: | 20260505 22:58:01.006 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:58:01.006 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:01.006 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:01.006 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:01.006 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260505 22:58:01.006 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:01.006 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:01.006 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:58:01.006 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:01.006 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:01.007 | INFO | Uploaded 36GiB to GPU in 2.15297s (16.7211GiB/s) | 20260505 22:58:01.007 | INFO | Downloaded 34GiB to GPU in 2.05023s (16.5835GiB/s) | 20260505 22:58:06.150 | INFO | Analyzing chunk frames = [6972928,6973440) | 20260505 22:58:06.150 | INFO | MemInfo: { | 20260505 22:58:06.150 | INFO | "gpu" : " 17 GB", | 20260505 22:58:06.150 | INFO | "huge" : "", | 20260505 22:58:06.150 | INFO | "huge_shared" : " 27 GB", | 20260505 22:58:06.150 | INFO | "malloc" : " 14 GB", | 20260505 22:58:06.150 | INFO | "pinned" : " 29 GB" | 20260505 22:58:06.150 | INFO | } | 20260505 22:58:06.150 | INFO | Perf report for the last 5.14402 seconds: | 20260505 22:58:06.150 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:58:06.150 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:06.150 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:06.151 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:06.151 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260505 22:58:06.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:06.151 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:06.151 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 22:58:06.151 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:06.151 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:06.151 | INFO | Uploaded 36GiB to GPU in 2.13801s (16.8381GiB/s) | 20260505 22:58:06.151 | INFO | Downloaded 34GiB to GPU in 2.01761s (16.8516GiB/s) | 20260505 22:58:11.290 | INFO | Analyzing chunk frames = [6973440,6973952) | 20260505 22:58:11.290 | INFO | MemInfo: { | 20260505 22:58:11.290 | INFO | "gpu" : " 17 GB", | 20260505 22:58:11.290 | INFO | "huge" : "", | 20260505 22:58:11.290 | INFO | "huge_shared" : " 27 GB", | 20260505 22:58:11.290 | INFO | "malloc" : " 14 GB", | 20260505 22:58:11.290 | INFO | "pinned" : " 29 GB" | 20260505 22:58:11.290 | INFO | } | 20260505 22:58:11.290 | INFO | Perf report for the last 5.13984 seconds: | 20260505 22:58:11.290 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:58:11.290 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:11.290 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:11.290 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:11.290 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260505 22:58:11.290 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:11.290 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:11.290 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:58:11.290 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:11.290 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:11.291 | INFO | Uploaded 36GiB to GPU in 2.0896s (17.2282GiB/s) | 20260505 22:58:11.291 | INFO | Downloaded 34GiB to GPU in 2.04735s (16.6068GiB/s) | 20260505 22:58:16.427 | INFO | Analyzing chunk frames = [6973952,6974464) | 20260505 22:58:16.427 | INFO | MemInfo: { | 20260505 22:58:16.427 | INFO | "gpu" : " 17 GB", | 20260505 22:58:16.427 | INFO | "huge" : "", | 20260505 22:58:16.427 | INFO | "huge_shared" : " 27 GB", | 20260505 22:58:16.427 | INFO | "malloc" : " 14 GB", | 20260505 22:58:16.427 | INFO | "pinned" : " 29 GB" | 20260505 22:58:16.427 | INFO | } | 20260505 22:58:16.427 | INFO | Perf report for the last 5.13678 seconds: | 20260505 22:58:16.427 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:58:16.427 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:16.427 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:16.427 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:16.427 | INFO | ANALYSIS : 61.5%, 3.0, 96.0ms, 0 | 20260505 22:58:16.427 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:16.427 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:16.427 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260505 22:58:16.427 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:16.427 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:16.427 | INFO | Uploaded 36GiB to GPU in 2.07552s (17.345GiB/s) | 20260505 22:58:16.427 | INFO | Downloaded 34GiB to GPU in 2.0358s (16.701GiB/s) | 20260505 22:58:21.568 | INFO | Analyzing chunk frames = [6974464,6974976) | 20260505 22:58:21.568 | INFO | MemInfo: { | 20260505 22:58:21.568 | INFO | "gpu" : " 17 GB", | 20260505 22:58:21.568 | INFO | "huge" : "", | 20260505 22:58:21.568 | INFO | "huge_shared" : " 27 GB", | 20260505 22:58:21.568 | INFO | "malloc" : " 14 GB", | 20260505 22:58:21.568 | INFO | "pinned" : " 29 GB" | 20260505 22:58:21.568 | INFO | } | 20260505 22:58:21.568 | INFO | Perf report for the last 5.14079 seconds: | 20260505 22:58:21.568 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:58:21.568 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:21.568 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:21.568 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:21.568 | INFO | ANALYSIS : 62.7%, 3.0, 98.0ms, 0 | 20260505 22:58:21.568 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:21.568 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:21.568 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 22:58:21.568 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:21.568 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:21.568 | INFO | Uploaded 36GiB to GPU in 2.14309s (16.7981GiB/s) | 20260505 22:58:21.568 | INFO | Downloaded 34GiB to GPU in 2.15173s (15.8012GiB/s) | 20260505 22:58:26.708 | INFO | Analyzing chunk frames = [6974976,6975488) | 20260505 22:58:26.708 | INFO | MemInfo: { | 20260505 22:58:26.708 | INFO | "gpu" : " 17 GB", | 20260505 22:58:26.708 | INFO | "huge" : "", | 20260505 22:58:26.708 | INFO | "huge_shared" : " 27 GB", | 20260505 22:58:26.708 | INFO | "malloc" : " 14 GB", | 20260505 22:58:26.708 | INFO | "pinned" : " 29 GB" | 20260505 22:58:26.708 | INFO | } | 20260505 22:58:26.709 | INFO | Perf report for the last 5.14065 seconds: | 20260505 22:58:26.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:58:26.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:26.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:26.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:26.709 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260505 22:58:26.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:26.709 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:26.709 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 22:58:26.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:26.709 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:26.709 | INFO | Uploaded 36GiB to GPU in 2.11737s (17.0022GiB/s) | 20260505 22:58:26.709 | INFO | Downloaded 34GiB to GPU in 1.99934s (17.0056GiB/s) | 20260505 22:58:26.825 | INFO | Starting a round of Baz IO | 20260505 22:58:29.975 | INFO | Populating RTMetrics report | 20260505 22:58:29.975 | INFO | RTMetrics report populated | 20260505 22:58:29.975 | INFO | Opening tmp RTMetrics json file | 20260505 22:58:29.978 | INFO | Serializing RTMetrics to json | 20260505 22:58:29.979 | INFO | Serialization done, swapping tmp file now... | 20260505 22:58:29.980 | INFO | RTMetrics done | 20260505 22:58:30.057 | INFO | Finished a round of Baz IO: Wrote 1.908 GiB in 3.232 seconds | 20260505 22:58:30.057 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.46% / 0.00% / 78.54% | 20260505 22:58:30.057 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.114934% / 0% / 9.53674e-05% | 20260505 22:58:30.057 | INFO | Baz IO overall: | 20260505 22:58:30.057 | INFO | BytesWritten : 4002010239135 | 20260505 22:58:30.057 | INFO | NumEvents : 1886496180654 | 20260505 22:58:31.847 | INFO | Analyzing chunk frames = [6975488,6976000) | 20260505 22:58:31.847 | INFO | MemInfo: { | 20260505 22:58:31.847 | INFO | "gpu" : " 17 GB", | 20260505 22:58:31.847 | INFO | "huge" : "", | 20260505 22:58:31.847 | INFO | "huge_shared" : " 27 GB", | 20260505 22:58:31.847 | INFO | "malloc" : " 14 GB", | 20260505 22:58:31.847 | INFO | "pinned" : " 29 GB" | 20260505 22:58:31.847 | INFO | } | 20260505 22:58:31.847 | INFO | Perf report for the last 5.1387 seconds: | 20260505 22:58:31.847 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:58:31.847 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:31.847 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:31.847 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:31.847 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260505 22:58:31.847 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:31.847 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260505 22:58:31.847 | INFO | PRE_HQ : 60.0%, 1.7, 54.3ms, 0 | 20260505 22:58:31.847 | INFO | BAZWRITER : 62.0%, 3.6, 115.0ms, 0 | 20260505 22:58:31.847 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 22:58:31.848 | INFO | Uploaded 36GiB to GPU in 2.16426s (16.6338GiB/s) | 20260505 22:58:31.848 | INFO | Downloaded 38GiB to GPU in 2.23941s (16.9688GiB/s) | 20260505 22:58:36.984 | INFO | Analyzing chunk frames = [6976000,6976512) | 20260505 22:58:36.984 | INFO | MemInfo: { | 20260505 22:58:36.984 | INFO | "gpu" : " 17 GB", | 20260505 22:58:36.984 | INFO | "huge" : "", | 20260505 22:58:36.984 | INFO | "huge_shared" : " 27 GB", | 20260505 22:58:36.984 | INFO | "malloc" : " 14 GB", | 20260505 22:58:36.984 | INFO | "pinned" : " 29 GB" | 20260505 22:58:36.984 | INFO | } | 20260505 22:58:36.984 | INFO | Perf report for the last 5.13677 seconds: | 20260505 22:58:36.984 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:58:36.984 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:36.984 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:36.984 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:36.984 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 22:58:36.984 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:36.984 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:36.984 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 22:58:36.984 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:36.984 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:36.984 | INFO | Uploaded 36GiB to GPU in 2.1052s (17.1005GiB/s) | 20260505 22:58:36.984 | INFO | Downloaded 34GiB to GPU in 2.03907s (16.6743GiB/s) | 20260505 22:58:42.127 | INFO | Analyzing chunk frames = [6976512,6977024) | 20260505 22:58:42.127 | INFO | MemInfo: { | 20260505 22:58:42.127 | INFO | "gpu" : " 17 GB", | 20260505 22:58:42.127 | INFO | "huge" : "", | 20260505 22:58:42.127 | INFO | "huge_shared" : " 27 GB", | 20260505 22:58:42.127 | INFO | "malloc" : " 14 GB", | 20260505 22:58:42.127 | INFO | "pinned" : " 29 GB" | 20260505 22:58:42.127 | INFO | } | 20260505 22:58:42.127 | INFO | Perf report for the last 5.14339 seconds: | 20260505 22:58:42.127 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:58:42.127 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:42.127 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:42.127 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:42.128 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260505 22:58:42.128 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:42.128 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:42.128 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 22:58:42.128 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:42.128 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:42.128 | INFO | Uploaded 36GiB to GPU in 2.15999s (16.6667GiB/s) | 20260505 22:58:42.128 | INFO | Downloaded 34GiB to GPU in 2.05254s (16.5649GiB/s) | 20260505 22:58:47.258 | INFO | Analyzing chunk frames = [6977024,6977536) | 20260505 22:58:47.258 | INFO | MemInfo: { | 20260505 22:58:47.258 | INFO | "gpu" : " 17 GB", | 20260505 22:58:47.258 | INFO | "huge" : "", | 20260505 22:58:47.258 | INFO | "huge_shared" : " 27 GB", | 20260505 22:58:47.258 | INFO | "malloc" : " 14 GB", | 20260505 22:58:47.258 | INFO | "pinned" : " 29 GB" | 20260505 22:58:47.258 | INFO | } | 20260505 22:58:47.258 | INFO | Perf report for the last 5.1307 seconds: | 20260505 22:58:47.258 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:58:47.258 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:47.258 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:47.258 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:47.258 | INFO | ANALYSIS : 61.8%, 3.0, 95.9ms, 0 | 20260505 22:58:47.258 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:47.258 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:47.258 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:58:47.258 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:47.258 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:47.258 | INFO | Uploaded 36GiB to GPU in 2.12955s (16.905GiB/s) | 20260505 22:58:47.258 | INFO | Downloaded 34GiB to GPU in 2.01994s (16.8322GiB/s) | 20260505 22:58:52.401 | INFO | Analyzing chunk frames = [6977536,6978048) | 20260505 22:58:52.401 | INFO | MemInfo: { | 20260505 22:58:52.401 | INFO | "gpu" : " 17 GB", | 20260505 22:58:52.401 | INFO | "huge" : "", | 20260505 22:58:52.401 | INFO | "huge_shared" : " 27 GB", | 20260505 22:58:52.401 | INFO | "malloc" : " 14 GB", | 20260505 22:58:52.401 | INFO | "pinned" : " 29 GB" | 20260505 22:58:52.401 | INFO | } | 20260505 22:58:52.401 | INFO | Perf report for the last 5.14311 seconds: | 20260505 22:58:52.401 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:58:52.401 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:52.401 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:52.401 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:52.401 | INFO | ANALYSIS : 61.5%, 3.0, 96.1ms, 0 | 20260505 22:58:52.402 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:52.402 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:52.402 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:58:52.402 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:52.402 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:52.402 | INFO | Uploaded 36GiB to GPU in 2.13746s (16.8424GiB/s) | 20260505 22:58:52.402 | INFO | Downloaded 34GiB to GPU in 2.05814s (16.5198GiB/s) | 20260505 22:58:57.544 | INFO | Analyzing chunk frames = [6978048,6978560) | 20260505 22:58:57.544 | INFO | MemInfo: { | 20260505 22:58:57.544 | INFO | "gpu" : " 17 GB", | 20260505 22:58:57.544 | INFO | "huge" : "", | 20260505 22:58:57.544 | INFO | "huge_shared" : " 27 GB", | 20260505 22:58:57.544 | INFO | "malloc" : " 14 GB", | 20260505 22:58:57.544 | INFO | "pinned" : " 29 GB" | 20260505 22:58:57.544 | INFO | } | 20260505 22:58:57.544 | INFO | Perf report for the last 5.14233 seconds: | 20260505 22:58:57.544 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:58:57.544 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:57.544 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:57.544 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:57.544 | INFO | ANALYSIS : 61.4%, 3.0, 95.9ms, 0 | 20260505 22:58:57.544 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:57.544 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:57.544 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 22:58:57.544 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:57.544 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:58:57.544 | INFO | Uploaded 36GiB to GPU in 2.07578s (17.3428GiB/s) | 20260505 22:58:57.544 | INFO | Downloaded 34GiB to GPU in 2.0365s (16.6953GiB/s) | 20260505 22:59:02.682 | INFO | Analyzing chunk frames = [6978560,6979072) | 20260505 22:59:02.682 | INFO | MemInfo: { | 20260505 22:59:02.682 | INFO | "gpu" : " 17 GB", | 20260505 22:59:02.682 | INFO | "huge" : "", | 20260505 22:59:02.682 | INFO | "huge_shared" : " 27 GB", | 20260505 22:59:02.682 | INFO | "malloc" : " 14 GB", | 20260505 22:59:02.682 | INFO | "pinned" : " 29 GB" | 20260505 22:59:02.682 | INFO | } | 20260505 22:59:02.682 | INFO | Perf report for the last 5.13845 seconds: | 20260505 22:59:02.682 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:59:02.682 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:02.682 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:02.682 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:02.682 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260505 22:59:02.682 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:02.682 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:02.682 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 22:59:02.682 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:02.682 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:02.683 | INFO | Uploaded 36GiB to GPU in 2.13427s (16.8676GiB/s) | 20260505 22:59:02.683 | INFO | Downloaded 34GiB to GPU in 2.07838s (16.3589GiB/s) | 20260505 22:59:07.815 | INFO | Analyzing chunk frames = [6979072,6979584) | 20260505 22:59:07.815 | INFO | MemInfo: { | 20260505 22:59:07.815 | INFO | "gpu" : " 17 GB", | 20260505 22:59:07.815 | INFO | "huge" : "", | 20260505 22:59:07.815 | INFO | "huge_shared" : " 27 GB", | 20260505 22:59:07.815 | INFO | "malloc" : " 14 GB", | 20260505 22:59:07.815 | INFO | "pinned" : " 29 GB" | 20260505 22:59:07.815 | INFO | } | 20260505 22:59:07.815 | INFO | Perf report for the last 5.13337 seconds: | 20260505 22:59:07.816 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:59:07.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:07.816 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:07.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:07.816 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260505 22:59:07.816 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:07.816 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:07.816 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:59:07.816 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:07.816 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:07.816 | INFO | Uploaded 36GiB to GPU in 2.1124s (17.0423GiB/s) | 20260505 22:59:07.816 | INFO | Downloaded 34GiB to GPU in 2.00493s (16.9582GiB/s) | 20260505 22:59:07.816 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 22:59:07.816 | INFO | Total mem usage: 15.3 GiB | 20260505 22:59:07.816 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 22:59:07.816 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.57%) | 20260505 22:59:07.816 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 22:59:07.816 | INFO | High Water Breakdown: | 20260505 22:59:07.816 | INFO | 61.1% : 8.62 GiB ( 90.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 22:59:07.816 | INFO | 38.8% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 22:59:07.816 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 22:59:07.816 | INFO | ------------------------------------------ | 20260505 22:59:07.816 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:59:07.816 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 22:59:07.816 | INFO | Total mem usage: 26.5 GiB | 20260505 22:59:07.816 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 22:59:07.816 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 22:59:07.816 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 22:59:07.816 | INFO | High Water Breakdown: | 20260505 22:59:07.816 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:59:07.816 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:59:07.816 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:59:07.816 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:59:07.816 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:59:07.816 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:59:07.816 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 22:59:07.816 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 22:59:07.816 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:59:07.816 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 22:59:07.816 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 22:59:07.816 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 22:59:07.816 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 22:59:07.816 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 22:59:07.816 | INFO | ------------------------------------------ | 20260505 22:59:07.816 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:59:07.816 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 22:59:07.817 | INFO | Total mem usage: 25.6 GiB | 20260505 22:59:07.817 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 22:59:07.817 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 22:59:07.817 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 22:59:07.817 | INFO | High Water Breakdown: | 20260505 22:59:07.817 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 22:59:07.817 | INFO | ------------------------------------------ | 20260505 22:59:07.817 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:59:07.817 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 22:59:07.817 | INFO | Total mem usage: 15.7 GiB | 20260505 22:59:07.817 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 22:59:07.817 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 22:59:07.817 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 22:59:07.817 | INFO | High Water Breakdown: | 20260505 22:59:07.817 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 22:59:07.817 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 22:59:07.817 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 22:59:07.817 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 22:59:07.817 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 22:59:07.817 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 22:59:07.817 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 22:59:07.817 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 22:59:07.817 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 22:59:07.817 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 22:59:07.817 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 22:59:07.817 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 22:59:07.817 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 22:59:07.817 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 22:59:07.817 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 22:59:07.817 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 22:59:07.817 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 22:59:07.817 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 22:59:07.817 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 22:59:07.817 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 22:59:07.817 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 22:59:07.817 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 22:59:07.817 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 22:59:07.817 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 22:59:07.817 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 22:59:07.817 | INFO | ------------------------------------------ | 20260505 22:59:07.817 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 22:59:07.817 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 22:59:07.817 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:59:07.817 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:59:07.817 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.39e+01, 1.34e+01 / 1.25e+00 | 20260505 22:59:07.817 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.05e-01, 1.05e-02 / 5.14e-03 | 20260505 22:59:07.817 | INFO | PerfReport: 11.84%, Baseline -- 3.71e+00 / 4.51e+00, 5.72e-01 / 2.19e-01 | 20260505 22:59:07.817 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.51e+00, 1.87e-01 / 9.67e-02 | 20260505 22:59:07.817 | INFO | PerfReport: 15.99%, DME -- 4.00e+01 / 4.39e+01, 3.90e+00 / 1.29e+00 | 20260505 22:59:07.817 | INFO | PerfReport: 57.19%, FrameLabeling -- 1.79e+01 / 1.94e+01, 1.40e+00 / 9.82e-01 | 20260505 22:59:07.817 | INFO | PerfReport: 7.17%, PulseAccumulating -- 2.24e+00 / 2.77e+00, 4.30e-01 / 2.12e-01 | 20260505 22:59:07.817 | INFO | PerfReport: 3.15%, Metrics -- 9.85e-01 / 1.29e+00, 2.69e-01 / 1.40e-01 | 20260505 22:59:07.817 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.41e-01 / 3.95e-01, 5.80e-02 / 3.10e-02 | 20260505 22:59:07.817 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 22:59:07.817 | INFO | PerfReport: Gathered 1536 samples. | 20260505 22:59:07.817 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 22:59:07.817 | INFO | PerfReport: Full Execution -- 9.86e+01 / -nan, 2.47e+01 / 0.00e+00 | 20260505 22:59:07.817 | INFO | PerfReport: 22.47%, Upload -- 2.21e+01 / 0.00e+00, 7.65e+00 / 0.00e+00 | 20260505 22:59:07.817 | INFO | PerfReport: 21.87%, Download -- 2.16e+01 / 0.00e+00, 7.69e+00 / 0.00e+00 | 20260505 22:59:07.817 | INFO | PerfReport: 31.75%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 22:59:07.817 | INFO | PerfReport: 2.16%, LockUpload -- 2.13e+00 / 0.00e+00, 7.94e+00 / 0.00e+00 | 20260505 22:59:07.817 | INFO | PerfReport: 1.37%, LockDownload -- 1.35e+00 / 0.00e+00, 2.90e+00 / 0.00e+00 | 20260505 22:59:07.817 | INFO | PerfReport: 20.37%, LockCompute -- 2.01e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260505 22:59:07.817 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 2.20e-02 / 0.00e+00 | 20260505 22:59:07.947 | INFO | Starting a round of Baz IO | 20260505 22:59:11.099 | INFO | Populating RTMetrics report | 20260505 22:59:11.100 | INFO | RTMetrics report populated | 20260505 22:59:11.100 | INFO | Opening tmp RTMetrics json file | 20260505 22:59:11.102 | INFO | Serializing RTMetrics to json | 20260505 22:59:11.103 | INFO | Serialization done, swapping tmp file now... | 20260505 22:59:11.103 | INFO | RTMetrics done | 20260505 22:59:11.177 | INFO | Finished a round of Baz IO: Wrote 1.91 GiB in 3.23 seconds | 20260505 22:59:11.177 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.46% / 0.00% / 78.54% | 20260505 22:59:11.177 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.114961% / 3.97364e-06% / 7.54992e-05% | 20260505 22:59:11.178 | INFO | Baz IO overall: | 20260505 22:59:11.178 | INFO | BytesWritten : 4004061440992 | 20260505 22:59:11.178 | INFO | NumEvents : 1887438368433 | 20260505 22:59:12.958 | INFO | Analyzing chunk frames = [6979584,6980096) | 20260505 22:59:12.958 | INFO | MemInfo: { | 20260505 22:59:12.958 | INFO | "gpu" : " 17 GB", | 20260505 22:59:12.958 | INFO | "huge" : "", | 20260505 22:59:12.958 | INFO | "huge_shared" : " 27 GB", | 20260505 22:59:12.958 | INFO | "malloc" : " 14 GB", | 20260505 22:59:12.958 | INFO | "pinned" : " 29 GB" | 20260505 22:59:12.958 | INFO | } | 20260505 22:59:12.958 | INFO | Perf report for the last 5.14271 seconds: | 20260505 22:59:12.958 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:59:12.958 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:12.958 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:12.958 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:12.958 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 22:59:12.958 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:12.958 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260505 22:59:12.958 | INFO | PRE_HQ : 59.6%, 1.8, 54.2ms, 0 | 20260505 22:59:12.958 | INFO | BAZWRITER : 62.2%, 3.6, 114.2ms, 0 | 20260505 22:59:12.958 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 22:59:12.959 | INFO | Uploaded 36GiB to GPU in 2.15574s (16.6996GiB/s) | 20260505 22:59:12.959 | INFO | Downloaded 38GiB to GPU in 2.23145s (17.0293GiB/s) | 20260505 22:59:18.100 | INFO | Analyzing chunk frames = [6980096,6980608) | 20260505 22:59:18.100 | INFO | MemInfo: { | 20260505 22:59:18.100 | INFO | "gpu" : " 17 GB", | 20260505 22:59:18.100 | INFO | "huge" : "", | 20260505 22:59:18.100 | INFO | "huge_shared" : " 27 GB", | 20260505 22:59:18.100 | INFO | "malloc" : " 14 GB", | 20260505 22:59:18.100 | INFO | "pinned" : " 29 GB" | 20260505 22:59:18.100 | INFO | } | 20260505 22:59:18.100 | INFO | Perf report for the last 5.14194 seconds: | 20260505 22:59:18.100 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:59:18.100 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:18.100 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:18.100 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:18.100 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260505 22:59:18.100 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:18.100 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:18.100 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260505 22:59:18.100 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:18.100 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:18.101 | INFO | Uploaded 36GiB to GPU in 2.08228s (17.2888GiB/s) | 20260505 22:59:18.101 | INFO | Downloaded 34GiB to GPU in 2.03056s (16.7441GiB/s) | 20260505 22:59:23.243 | INFO | Analyzing chunk frames = [6980608,6981120) | 20260505 22:59:23.243 | INFO | MemInfo: { | 20260505 22:59:23.243 | INFO | "gpu" : " 17 GB", | 20260505 22:59:23.243 | INFO | "huge" : "", | 20260505 22:59:23.243 | INFO | "huge_shared" : " 27 GB", | 20260505 22:59:23.243 | INFO | "malloc" : " 14 GB", | 20260505 22:59:23.243 | INFO | "pinned" : " 29 GB" | 20260505 22:59:23.243 | INFO | } | 20260505 22:59:23.243 | INFO | Perf report for the last 5.14303 seconds: | 20260505 22:59:23.243 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:59:23.243 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:23.243 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:23.243 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:23.243 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260505 22:59:23.243 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:23.243 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:23.243 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:59:23.243 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:23.243 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:23.243 | INFO | Uploaded 36GiB to GPU in 2.15517s (16.704GiB/s) | 20260505 22:59:23.243 | INFO | Downloaded 34GiB to GPU in 2.04907s (16.5929GiB/s) | 20260505 22:59:28.376 | INFO | Analyzing chunk frames = [6981120,6981632) | 20260505 22:59:28.376 | INFO | MemInfo: { | 20260505 22:59:28.377 | INFO | "gpu" : " 17 GB", | 20260505 22:59:28.377 | INFO | "huge" : "", | 20260505 22:59:28.377 | INFO | "huge_shared" : " 27 GB", | 20260505 22:59:28.377 | INFO | "malloc" : " 14 GB", | 20260505 22:59:28.377 | INFO | "pinned" : " 29 GB" | 20260505 22:59:28.377 | INFO | } | 20260505 22:59:28.377 | INFO | Perf report for the last 5.1334 seconds: | 20260505 22:59:28.377 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:59:28.377 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:28.377 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:28.377 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:28.377 | INFO | ANALYSIS : 61.8%, 3.0, 95.9ms, 0 | 20260505 22:59:28.377 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:28.377 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:28.377 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 22:59:28.377 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:28.377 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:28.377 | INFO | Uploaded 36GiB to GPU in 2.12182s (16.9665GiB/s) | 20260505 22:59:28.377 | INFO | Downloaded 34GiB to GPU in 2.01162s (16.9018GiB/s) | 20260505 22:59:33.515 | INFO | Analyzing chunk frames = [6981632,6982144) | 20260505 22:59:33.515 | INFO | MemInfo: { | 20260505 22:59:33.515 | INFO | "gpu" : " 17 GB", | 20260505 22:59:33.515 | INFO | "huge" : "", | 20260505 22:59:33.515 | INFO | "huge_shared" : " 27 GB", | 20260505 22:59:33.515 | INFO | "malloc" : " 14 GB", | 20260505 22:59:33.515 | INFO | "pinned" : " 29 GB" | 20260505 22:59:33.516 | INFO | } | 20260505 22:59:33.516 | INFO | Perf report for the last 5.13901 seconds: | 20260505 22:59:33.516 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:59:33.516 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:33.516 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:33.516 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:33.516 | INFO | ANALYSIS : 61.5%, 3.0, 96.1ms, 0 | 20260505 22:59:33.516 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:33.516 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:33.516 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 22:59:33.516 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:33.516 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:33.516 | INFO | Uploaded 36GiB to GPU in 2.13423s (16.8679GiB/s) | 20260505 22:59:33.516 | INFO | Downloaded 34GiB to GPU in 2.0609s (16.4977GiB/s) | 20260505 22:59:38.661 | INFO | Analyzing chunk frames = [6982144,6982656) | 20260505 22:59:38.661 | INFO | MemInfo: { | 20260505 22:59:38.661 | INFO | "gpu" : " 17 GB", | 20260505 22:59:38.661 | INFO | "huge" : "", | 20260505 22:59:38.661 | INFO | "huge_shared" : " 27 GB", | 20260505 22:59:38.661 | INFO | "malloc" : " 14 GB", | 20260505 22:59:38.661 | INFO | "pinned" : " 29 GB" | 20260505 22:59:38.661 | INFO | } | 20260505 22:59:38.661 | INFO | Perf report for the last 5.14498 seconds: | 20260505 22:59:38.661 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:59:38.661 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:38.661 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:38.661 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:38.661 | INFO | ANALYSIS : 61.5%, 3.0, 96.1ms, 0 | 20260505 22:59:38.661 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:38.661 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:38.661 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:59:38.661 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:38.661 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:38.661 | INFO | Uploaded 36GiB to GPU in 2.07538s (17.3462GiB/s) | 20260505 22:59:38.661 | INFO | Downloaded 34GiB to GPU in 2.03746s (16.6874GiB/s) | 20260505 22:59:43.796 | INFO | Analyzing chunk frames = [6982656,6983168) | 20260505 22:59:43.796 | INFO | MemInfo: { | 20260505 22:59:43.796 | INFO | "gpu" : " 17 GB", | 20260505 22:59:43.796 | INFO | "huge" : "", | 20260505 22:59:43.796 | INFO | "huge_shared" : " 27 GB", | 20260505 22:59:43.796 | INFO | "malloc" : " 14 GB", | 20260505 22:59:43.796 | INFO | "pinned" : " 29 GB" | 20260505 22:59:43.796 | INFO | } | 20260505 22:59:43.796 | INFO | Perf report for the last 5.13579 seconds: | 20260505 22:59:43.796 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:59:43.796 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:43.796 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:43.796 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:43.796 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 22:59:43.796 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:43.797 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:43.797 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 22:59:43.797 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:43.797 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:43.797 | INFO | Uploaded 36GiB to GPU in 2.01864s (17.8338GiB/s) | 20260505 22:59:43.797 | INFO | Downloaded 34GiB to GPU in 2.04455s (16.6296GiB/s) | 20260505 22:59:48.930 | INFO | Analyzing chunk frames = [6983168,6983680) | 20260505 22:59:48.930 | INFO | MemInfo: { | 20260505 22:59:48.930 | INFO | "gpu" : " 17 GB", | 20260505 22:59:48.930 | INFO | "huge" : "", | 20260505 22:59:48.930 | INFO | "huge_shared" : " 27 GB", | 20260505 22:59:48.930 | INFO | "malloc" : " 14 GB", | 20260505 22:59:48.930 | INFO | "pinned" : " 29 GB" | 20260505 22:59:48.930 | INFO | } | 20260505 22:59:48.930 | INFO | Perf report for the last 5.13394 seconds: | 20260505 22:59:48.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:59:48.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:48.930 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:48.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:48.930 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260505 22:59:48.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:48.930 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:48.930 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 22:59:48.931 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:48.931 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:48.931 | INFO | Uploaded 36GiB to GPU in 2.11079s (17.0552GiB/s) | 20260505 22:59:48.931 | INFO | Downloaded 34GiB to GPU in 2.01268s (16.8929GiB/s) | 20260505 22:59:49.059 | INFO | Starting a round of Baz IO | 20260505 22:59:52.206 | INFO | Populating RTMetrics report | 20260505 22:59:52.206 | INFO | RTMetrics report populated | 20260505 22:59:52.206 | INFO | Opening tmp RTMetrics json file | 20260505 22:59:52.207 | INFO | Serializing RTMetrics to json | 20260505 22:59:52.208 | INFO | Serialization done, swapping tmp file now... | 20260505 22:59:52.208 | INFO | RTMetrics done | 20260505 22:59:52.286 | INFO | Finished a round of Baz IO: Wrote 1.912 GiB in 3.227 seconds | 20260505 22:59:52.286 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.46% / 0.00% / 78.54% | 20260505 22:59:52.286 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.115319% / 0% / 7.94729e-05% | 20260505 22:59:52.286 | INFO | Baz IO overall: | 20260505 22:59:52.286 | INFO | BytesWritten : 4006114225380 | 20260505 22:59:52.286 | INFO | NumEvents : 1888382212795 | 20260505 22:59:54.074 | INFO | Analyzing chunk frames = [6983680,6984192) | 20260505 22:59:54.074 | INFO | MemInfo: { | 20260505 22:59:54.074 | INFO | "gpu" : " 17 GB", | 20260505 22:59:54.074 | INFO | "huge" : "", | 20260505 22:59:54.074 | INFO | "huge_shared" : " 27 GB", | 20260505 22:59:54.074 | INFO | "malloc" : " 14 GB", | 20260505 22:59:54.074 | INFO | "pinned" : " 29 GB" | 20260505 22:59:54.074 | INFO | } | 20260505 22:59:54.074 | INFO | Perf report for the last 5.14395 seconds: | 20260505 22:59:54.074 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:59:54.074 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:54.074 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:54.074 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:54.074 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260505 22:59:54.074 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:54.074 | INFO | RT_METRICS : 43.4%, 1.0, 22.8ms, 0 | 20260505 22:59:54.074 | INFO | PRE_HQ : 59.9%, 1.8, 54.5ms, 0 | 20260505 22:59:54.074 | INFO | BAZWRITER : 62.0%, 3.6, 113.6ms, 0 | 20260505 22:59:54.074 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 22:59:54.075 | INFO | Uploaded 36GiB to GPU in 2.1584s (16.679GiB/s) | 20260505 22:59:54.075 | INFO | Downloaded 38GiB to GPU in 2.23445s (17.0064GiB/s) | 20260505 22:59:59.214 | INFO | Analyzing chunk frames = [6984192,6984704) | 20260505 22:59:59.214 | INFO | MemInfo: { | 20260505 22:59:59.214 | INFO | "gpu" : " 17 GB", | 20260505 22:59:59.214 | INFO | "huge" : "", | 20260505 22:59:59.214 | INFO | "huge_shared" : " 27 GB", | 20260505 22:59:59.214 | INFO | "malloc" : " 14 GB", | 20260505 22:59:59.215 | INFO | "pinned" : " 29 GB" | 20260505 22:59:59.215 | INFO | } | 20260505 22:59:59.215 | INFO | Perf report for the last 5.14027 seconds: | 20260505 22:59:59.215 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 22:59:59.215 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:59.215 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:59.215 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:59.215 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 22:59:59.215 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:59.215 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:59.215 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 22:59:59.215 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:59.215 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 22:59:59.215 | INFO | Uploaded 36GiB to GPU in 2.08792s (17.2421GiB/s) | 20260505 22:59:59.215 | INFO | Downloaded 34GiB to GPU in 2.02392s (16.7991GiB/s) | 20260505 23:00:04.360 | INFO | Analyzing chunk frames = [6984704,6985216) | 20260505 23:00:04.360 | INFO | MemInfo: { | 20260505 23:00:04.360 | INFO | "gpu" : " 17 GB", | 20260505 23:00:04.360 | INFO | "huge" : "", | 20260505 23:00:04.360 | INFO | "huge_shared" : " 27 GB", | 20260505 23:00:04.360 | INFO | "malloc" : " 14 GB", | 20260505 23:00:04.360 | INFO | "pinned" : " 29 GB" | 20260505 23:00:04.360 | INFO | } | 20260505 23:00:04.360 | INFO | Perf report for the last 5.14546 seconds: | 20260505 23:00:04.360 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:00:04.360 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:04.360 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:04.360 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:04.360 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260505 23:00:04.360 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:04.360 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:04.360 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:00:04.360 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:04.360 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:04.360 | INFO | Uploaded 36GiB to GPU in 2.10029s (17.1405GiB/s) | 20260505 23:00:04.360 | INFO | Downloaded 34GiB to GPU in 2.03024s (16.7468GiB/s) | 20260505 23:00:09.494 | INFO | Analyzing chunk frames = [6985216,6985728) | 20260505 23:00:09.494 | INFO | MemInfo: { | 20260505 23:00:09.494 | INFO | "gpu" : " 17 GB", | 20260505 23:00:09.494 | INFO | "huge" : "", | 20260505 23:00:09.494 | INFO | "huge_shared" : " 27 GB", | 20260505 23:00:09.494 | INFO | "malloc" : " 14 GB", | 20260505 23:00:09.494 | INFO | "pinned" : " 29 GB" | 20260505 23:00:09.494 | INFO | } | 20260505 23:00:09.494 | INFO | Perf report for the last 5.13377 seconds: | 20260505 23:00:09.494 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:00:09.494 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:09.494 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:09.494 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:09.494 | INFO | ANALYSIS : 61.7%, 3.0, 95.8ms, 0 | 20260505 23:00:09.494 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:09.494 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:09.494 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 23:00:09.494 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:09.494 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:09.494 | INFO | Uploaded 36GiB to GPU in 2.1494s (16.7489GiB/s) | 20260505 23:00:09.494 | INFO | Downloaded 34GiB to GPU in 2.03346s (16.7203GiB/s) | 20260505 23:00:14.628 | INFO | Analyzing chunk frames = [6985728,6986240) | 20260505 23:00:14.628 | INFO | MemInfo: { | 20260505 23:00:14.628 | INFO | "gpu" : " 17 GB", | 20260505 23:00:14.628 | INFO | "huge" : "", | 20260505 23:00:14.628 | INFO | "huge_shared" : " 27 GB", | 20260505 23:00:14.628 | INFO | "malloc" : " 14 GB", | 20260505 23:00:14.629 | INFO | "pinned" : " 29 GB" | 20260505 23:00:14.629 | INFO | } | 20260505 23:00:14.629 | INFO | Perf report for the last 5.13483 seconds: | 20260505 23:00:14.629 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:00:14.629 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:14.629 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:14.629 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:14.629 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260505 23:00:14.629 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:14.629 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:14.629 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:00:14.629 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:14.629 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:14.629 | INFO | Uploaded 36GiB to GPU in 2.11155s (17.0491GiB/s) | 20260505 23:00:14.629 | INFO | Downloaded 34GiB to GPU in 2.04984s (16.5867GiB/s) | 20260505 23:00:19.767 | INFO | Analyzing chunk frames = [6986240,6986752) | 20260505 23:00:19.767 | INFO | MemInfo: { | 20260505 23:00:19.767 | INFO | "gpu" : " 17 GB", | 20260505 23:00:19.767 | INFO | "huge" : "", | 20260505 23:00:19.767 | INFO | "huge_shared" : " 27 GB", | 20260505 23:00:19.767 | INFO | "malloc" : " 14 GB", | 20260505 23:00:19.767 | INFO | "pinned" : " 29 GB" | 20260505 23:00:19.767 | INFO | } | 20260505 23:00:19.767 | INFO | Perf report for the last 5.13833 seconds: | 20260505 23:00:19.767 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:00:19.767 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:19.767 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:19.767 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:19.767 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260505 23:00:19.767 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:19.767 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:19.767 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260505 23:00:19.767 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:19.767 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:19.767 | INFO | Uploaded 36GiB to GPU in 2.09005s (17.2244GiB/s) | 20260505 23:00:19.767 | INFO | Downloaded 34GiB to GPU in 2.04146s (16.6548GiB/s) | 20260505 23:00:24.909 | INFO | Analyzing chunk frames = [6986752,6987264) | 20260505 23:00:24.909 | INFO | MemInfo: { | 20260505 23:00:24.909 | INFO | "gpu" : " 17 GB", | 20260505 23:00:24.909 | INFO | "huge" : "", | 20260505 23:00:24.909 | INFO | "huge_shared" : " 27 GB", | 20260505 23:00:24.909 | INFO | "malloc" : " 14 GB", | 20260505 23:00:24.909 | INFO | "pinned" : " 29 GB" | 20260505 23:00:24.909 | INFO | } | 20260505 23:00:24.909 | INFO | Perf report for the last 5.14209 seconds: | 20260505 23:00:24.909 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:00:24.909 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:24.909 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:24.909 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:24.909 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 23:00:24.909 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:24.909 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:24.909 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 23:00:24.909 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:24.909 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:24.909 | INFO | Uploaded 36GiB to GPU in 2.14619s (16.7739GiB/s) | 20260505 23:00:24.909 | INFO | Downloaded 34GiB to GPU in 2.09982s (16.1919GiB/s) | 20260505 23:00:30.052 | INFO | Analyzing chunk frames = [6987264,6987776) | 20260505 23:00:30.052 | INFO | MemInfo: { | 20260505 23:00:30.052 | INFO | "gpu" : " 17 GB", | 20260505 23:00:30.052 | INFO | "huge" : "", | 20260505 23:00:30.052 | INFO | "huge_shared" : " 27 GB", | 20260505 23:00:30.052 | INFO | "malloc" : " 14 GB", | 20260505 23:00:30.052 | INFO | "pinned" : " 29 GB" | 20260505 23:00:30.052 | INFO | } | 20260505 23:00:30.053 | INFO | Perf report for the last 5.14334 seconds: | 20260505 23:00:30.053 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:00:30.053 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:30.053 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:30.053 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:30.053 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 23:00:30.053 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:30.053 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:30.053 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 23:00:30.053 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:30.053 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:30.053 | INFO | Uploaded 36GiB to GPU in 2.12441s (16.9459GiB/s) | 20260505 23:00:30.053 | INFO | Downloaded 34GiB to GPU in 2.01512s (16.8724GiB/s) | 20260505 23:00:30.053 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 23:00:30.053 | INFO | Total mem usage: 15.3 GiB | 20260505 23:00:30.053 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 23:00:30.053 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -9.05%) | 20260505 23:00:30.053 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 23:00:30.053 | INFO | High Water Breakdown: | 20260505 23:00:30.053 | INFO | 60.9% : 8.56 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 23:00:30.053 | INFO | 39% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 23:00:30.053 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 23:00:30.053 | INFO | ------------------------------------------ | 20260505 23:00:30.053 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:00:30.053 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 23:00:30.053 | INFO | Total mem usage: 26.5 GiB | 20260505 23:00:30.053 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 23:00:30.053 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 23:00:30.053 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 23:00:30.053 | INFO | High Water Breakdown: | 20260505 23:00:30.053 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:00:30.053 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:00:30.053 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:00:30.053 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:00:30.053 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:00:30.053 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:00:30.053 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 23:00:30.053 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 23:00:30.053 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:00:30.053 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 23:00:30.053 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 23:00:30.053 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 23:00:30.053 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 23:00:30.053 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 23:00:30.053 | INFO | ------------------------------------------ | 20260505 23:00:30.054 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:00:30.054 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 23:00:30.054 | INFO | Total mem usage: 25.6 GiB | 20260505 23:00:30.054 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 23:00:30.054 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 23:00:30.054 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 23:00:30.054 | INFO | High Water Breakdown: | 20260505 23:00:30.054 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 23:00:30.054 | INFO | ------------------------------------------ | 20260505 23:00:30.054 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:00:30.054 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 23:00:30.054 | INFO | Total mem usage: 15.7 GiB | 20260505 23:00:30.054 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 23:00:30.054 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 23:00:30.054 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 23:00:30.054 | INFO | High Water Breakdown: | 20260505 23:00:30.054 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 23:00:30.054 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 23:00:30.054 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 23:00:30.054 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 23:00:30.054 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 23:00:30.054 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 23:00:30.054 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 23:00:30.054 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 23:00:30.054 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 23:00:30.054 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:00:30.054 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 23:00:30.054 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:00:30.054 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 23:00:30.054 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 23:00:30.054 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 23:00:30.054 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 23:00:30.054 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:00:30.054 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 23:00:30.054 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:00:30.054 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:00:30.054 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:00:30.054 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 23:00:30.054 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 23:00:30.054 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:00:30.054 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 23:00:30.054 | INFO | ------------------------------------------ | 20260505 23:00:30.054 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:00:30.054 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 23:00:30.054 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:00:30.054 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:00:30.054 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.40e+01, 1.33e+01 / 8.12e-01 | 20260505 23:00:30.054 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.04e-01, 1.06e-02 / 5.63e-03 | 20260505 23:00:30.054 | INFO | PerfReport: 11.82%, Baseline -- 3.70e+00 / 4.54e+00, 5.71e-01 / 1.65e-01 | 20260505 23:00:30.054 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.53e+00, 1.91e-01 / 6.31e-02 | 20260505 23:00:30.054 | INFO | PerfReport: 15.97%, DME -- 4.00e+01 / 4.39e+01, 3.89e+00 / 9.72e-01 | 20260505 23:00:30.054 | INFO | PerfReport: 57.27%, FrameLabeling -- 1.79e+01 / 1.94e+01, 1.47e+00 / 9.92e-01 | 20260505 23:00:30.054 | INFO | PerfReport: 7.14%, PulseAccumulating -- 2.24e+00 / 2.80e+00, 4.28e-01 / 1.73e-01 | 20260505 23:00:30.054 | INFO | PerfReport: 3.14%, Metrics -- 9.84e-01 / 1.28e+00, 2.67e-01 / 1.04e-01 | 20260505 23:00:30.054 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.41e-01 / 4.04e-01, 5.83e-02 / 3.59e-02 | 20260505 23:00:30.054 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 23:00:30.054 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:00:30.054 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:00:30.054 | INFO | PerfReport: Full Execution -- 9.86e+01 / -nan, 2.49e+01 / 0.00e+00 | 20260505 23:00:30.054 | INFO | PerfReport: 22.33%, Upload -- 2.20e+01 / 0.00e+00, 7.60e+00 / 0.00e+00 | 20260505 23:00:30.054 | INFO | PerfReport: 21.79%, Download -- 2.15e+01 / 0.00e+00, 7.65e+00 / 0.00e+00 | 20260505 23:00:30.054 | INFO | PerfReport: 31.79%, Compute -- 3.13e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260505 23:00:30.054 | INFO | PerfReport: 2.20%, LockUpload -- 2.17e+00 / 0.00e+00, 7.96e+00 / 0.00e+00 | 20260505 23:00:30.054 | INFO | PerfReport: 1.31%, LockDownload -- 1.29e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260505 23:00:30.054 | INFO | PerfReport: 20.56%, LockCompute -- 2.03e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260505 23:00:30.054 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.66e-02 / 0.00e+00 | 20260505 23:00:30.173 | INFO | Starting a round of Baz IO | 20260505 23:00:33.323 | INFO | Populating RTMetrics report | 20260505 23:00:33.324 | INFO | RTMetrics report populated | 20260505 23:00:33.324 | INFO | Opening tmp RTMetrics json file | 20260505 23:00:33.325 | INFO | Serializing RTMetrics to json | 20260505 23:00:33.330 | INFO | Serialization done, swapping tmp file now... | 20260505 23:00:33.332 | INFO | RTMetrics done | 20260505 23:00:33.403 | INFO | Finished a round of Baz IO: Wrote 1.911 GiB in 3.23 seconds | 20260505 23:00:33.403 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.46% / 0.00% / 78.54% | 20260505 23:00:33.403 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.114457% / 0% / 9.13938e-05% | 20260505 23:00:33.403 | INFO | Baz IO overall: | 20260505 23:00:33.403 | INFO | BytesWritten : 4008165904756 | 20260505 23:00:33.403 | INFO | NumEvents : 1889325108100 | 20260505 23:00:35.194 | INFO | Analyzing chunk frames = [6987776,6988288) | 20260505 23:00:35.194 | INFO | MemInfo: { | 20260505 23:00:35.194 | INFO | "gpu" : " 17 GB", | 20260505 23:00:35.194 | INFO | "huge" : "", | 20260505 23:00:35.194 | INFO | "huge_shared" : " 27 GB", | 20260505 23:00:35.194 | INFO | "malloc" : " 14 GB", | 20260505 23:00:35.194 | INFO | "pinned" : " 29 GB" | 20260505 23:00:35.194 | INFO | } | 20260505 23:00:35.194 | INFO | Perf report for the last 5.14125 seconds: | 20260505 23:00:35.194 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:00:35.194 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:35.194 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:35.194 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:35.194 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 23:00:35.194 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:35.194 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260505 23:00:35.194 | INFO | PRE_HQ : 60.0%, 1.7, 54.4ms, 0 | 20260505 23:00:35.194 | INFO | BAZWRITER : 62.0%, 3.6, 113.6ms, 0 | 20260505 23:00:35.194 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 23:00:35.194 | INFO | Uploaded 36GiB to GPU in 2.17012s (16.5889GiB/s) | 20260505 23:00:35.194 | INFO | Downloaded 38GiB to GPU in 2.23945s (16.9685GiB/s) | 20260505 23:00:40.326 | INFO | Analyzing chunk frames = [6988288,6988800) | 20260505 23:00:40.326 | INFO | MemInfo: { | 20260505 23:00:40.326 | INFO | "gpu" : " 17 GB", | 20260505 23:00:40.326 | INFO | "huge" : "", | 20260505 23:00:40.327 | INFO | "huge_shared" : " 27 GB", | 20260505 23:00:40.327 | INFO | "malloc" : " 14 GB", | 20260505 23:00:40.327 | INFO | "pinned" : " 29 GB" | 20260505 23:00:40.327 | INFO | } | 20260505 23:00:40.327 | INFO | Perf report for the last 5.13294 seconds: | 20260505 23:00:40.327 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:00:40.327 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:40.327 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:40.327 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:40.327 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 23:00:40.327 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:40.327 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:40.327 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260505 23:00:40.327 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:40.327 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:40.327 | INFO | Uploaded 36GiB to GPU in 2.07401s (17.3577GiB/s) | 20260505 23:00:40.327 | INFO | Downloaded 34GiB to GPU in 2.02686s (16.7747GiB/s) | 20260505 23:00:45.468 | INFO | Analyzing chunk frames = [6988800,6989312) | 20260505 23:00:45.468 | INFO | MemInfo: { | 20260505 23:00:45.468 | INFO | "gpu" : " 17 GB", | 20260505 23:00:45.468 | INFO | "huge" : "", | 20260505 23:00:45.468 | INFO | "huge_shared" : " 27 GB", | 20260505 23:00:45.468 | INFO | "malloc" : " 14 GB", | 20260505 23:00:45.468 | INFO | "pinned" : " 29 GB" | 20260505 23:00:45.468 | INFO | } | 20260505 23:00:45.468 | INFO | Perf report for the last 5.14165 seconds: | 20260505 23:00:45.468 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:00:45.468 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:45.468 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:45.468 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:45.468 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260505 23:00:45.468 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:45.468 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:45.468 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:00:45.469 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:45.469 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:45.469 | INFO | Uploaded 36GiB to GPU in 2.14311s (16.798GiB/s) | 20260505 23:00:45.469 | INFO | Downloaded 34GiB to GPU in 2.04879s (16.5952GiB/s) | 20260505 23:00:50.608 | INFO | Analyzing chunk frames = [6989312,6989824) | 20260505 23:00:50.608 | INFO | MemInfo: { | 20260505 23:00:50.608 | INFO | "gpu" : " 17 GB", | 20260505 23:00:50.608 | INFO | "huge" : "", | 20260505 23:00:50.608 | INFO | "huge_shared" : " 27 GB", | 20260505 23:00:50.608 | INFO | "malloc" : " 14 GB", | 20260505 23:00:50.608 | INFO | "pinned" : " 29 GB" | 20260505 23:00:50.608 | INFO | } | 20260505 23:00:50.608 | INFO | Perf report for the last 5.14013 seconds: | 20260505 23:00:50.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:00:50.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:50.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:50.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:50.609 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260505 23:00:50.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:50.609 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:50.609 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:00:50.609 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:50.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:50.609 | INFO | Uploaded 36GiB to GPU in 2.13938s (16.8273GiB/s) | 20260505 23:00:50.609 | INFO | Downloaded 34GiB to GPU in 2.01193s (16.8992GiB/s) | 20260505 23:00:55.745 | INFO | Analyzing chunk frames = [6989824,6990336) | 20260505 23:00:55.745 | INFO | MemInfo: { | 20260505 23:00:55.745 | INFO | "gpu" : " 17 GB", | 20260505 23:00:55.745 | INFO | "huge" : "", | 20260505 23:00:55.745 | INFO | "huge_shared" : " 27 GB", | 20260505 23:00:55.745 | INFO | "malloc" : " 14 GB", | 20260505 23:00:55.745 | INFO | "pinned" : " 29 GB" | 20260505 23:00:55.745 | INFO | } | 20260505 23:00:55.745 | INFO | Perf report for the last 5.13643 seconds: | 20260505 23:00:55.745 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:00:55.745 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:55.745 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:55.745 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:55.745 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260505 23:00:55.745 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:55.745 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:55.745 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 23:00:55.745 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:55.745 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:00:55.745 | INFO | Uploaded 36GiB to GPU in 2.10536s (17.0992GiB/s) | 20260505 23:00:55.745 | INFO | Downloaded 34GiB to GPU in 2.05333s (16.5585GiB/s) | 20260505 23:01:00.883 | INFO | Analyzing chunk frames = [6990336,6990848) | 20260505 23:01:00.883 | INFO | MemInfo: { | 20260505 23:01:00.883 | INFO | "gpu" : " 17 GB", | 20260505 23:01:00.883 | INFO | "huge" : "", | 20260505 23:01:00.883 | INFO | "huge_shared" : " 27 GB", | 20260505 23:01:00.883 | INFO | "malloc" : " 14 GB", | 20260505 23:01:00.883 | INFO | "pinned" : " 29 GB" | 20260505 23:01:00.883 | INFO | } | 20260505 23:01:00.883 | INFO | Perf report for the last 5.13824 seconds: | 20260505 23:01:00.883 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:01:00.883 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:00.883 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:00.883 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:00.883 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260505 23:01:00.883 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:00.883 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:00.883 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 23:01:00.883 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:00.883 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:00.883 | INFO | Uploaded 36GiB to GPU in 2.07548s (17.3454GiB/s) | 20260505 23:01:00.884 | INFO | Downloaded 34GiB to GPU in 2.03612s (16.6984GiB/s) | 20260505 23:01:06.027 | INFO | Analyzing chunk frames = [6990848,6991360) | 20260505 23:01:06.027 | INFO | MemInfo: { | 20260505 23:01:06.027 | INFO | "gpu" : " 17 GB", | 20260505 23:01:06.027 | INFO | "huge" : "", | 20260505 23:01:06.027 | INFO | "huge_shared" : " 27 GB", | 20260505 23:01:06.027 | INFO | "malloc" : " 14 GB", | 20260505 23:01:06.027 | INFO | "pinned" : " 29 GB" | 20260505 23:01:06.027 | INFO | } | 20260505 23:01:06.027 | INFO | Perf report for the last 5.14415 seconds: | 20260505 23:01:06.027 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:01:06.027 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:06.027 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:06.027 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:06.027 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 23:01:06.027 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:06.028 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:06.028 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 23:01:06.028 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:06.028 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:06.028 | INFO | Uploaded 36GiB to GPU in 2.17557s (16.5474GiB/s) | 20260505 23:01:06.028 | INFO | Downloaded 34GiB to GPU in 2.1216s (16.0256GiB/s) | 20260505 23:01:11.164 | INFO | Analyzing chunk frames = [6991360,6991872) | 20260505 23:01:11.164 | INFO | MemInfo: { | 20260505 23:01:11.164 | INFO | "gpu" : " 17 GB", | 20260505 23:01:11.164 | INFO | "huge" : "", | 20260505 23:01:11.164 | INFO | "huge_shared" : " 27 GB", | 20260505 23:01:11.164 | INFO | "malloc" : " 14 GB", | 20260505 23:01:11.164 | INFO | "pinned" : " 29 GB" | 20260505 23:01:11.164 | INFO | } | 20260505 23:01:11.164 | INFO | Perf report for the last 5.13637 seconds: | 20260505 23:01:11.164 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:01:11.164 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:11.164 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:11.164 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:11.164 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 23:01:11.164 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:11.164 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:11.164 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 23:01:11.164 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:11.164 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:11.164 | INFO | Uploaded 36GiB to GPU in 2.11585s (17.0144GiB/s) | 20260505 23:01:11.164 | INFO | Downloaded 34GiB to GPU in 2.00469s (16.9602GiB/s) | 20260505 23:01:11.282 | INFO | Starting a round of Baz IO | 20260505 23:01:14.429 | INFO | Populating RTMetrics report | 20260505 23:01:14.429 | INFO | RTMetrics report populated | 20260505 23:01:14.429 | INFO | Opening tmp RTMetrics json file | 20260505 23:01:14.431 | INFO | Serializing RTMetrics to json | 20260505 23:01:14.433 | INFO | Serialization done, swapping tmp file now... | 20260505 23:01:14.434 | INFO | RTMetrics done | 20260505 23:01:14.516 | INFO | Finished a round of Baz IO: Wrote 1.909 GiB in 3.233 seconds | 20260505 23:01:14.516 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.46% / 0.00% / 78.54% | 20260505 23:01:14.516 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.114095% / 0% / 9.93411e-05% | 20260505 23:01:14.516 | INFO | Baz IO overall: | 20260505 23:01:14.516 | INFO | BytesWritten : 4010215560723 | 20260505 23:01:14.516 | INFO | NumEvents : 1890266786908 | 20260505 23:01:16.307 | INFO | Analyzing chunk frames = [6991872,6992384) | 20260505 23:01:16.307 | INFO | MemInfo: { | 20260505 23:01:16.307 | INFO | "gpu" : " 17 GB", | 20260505 23:01:16.307 | INFO | "huge" : "", | 20260505 23:01:16.307 | INFO | "huge_shared" : " 27 GB", | 20260505 23:01:16.307 | INFO | "malloc" : " 14 GB", | 20260505 23:01:16.307 | INFO | "pinned" : " 29 GB" | 20260505 23:01:16.307 | INFO | } | 20260505 23:01:16.307 | INFO | Perf report for the last 5.14347 seconds: | 20260505 23:01:16.307 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:01:16.307 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:16.307 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:16.307 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:16.307 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260505 23:01:16.307 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:16.307 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260505 23:01:16.307 | INFO | PRE_HQ : 59.9%, 1.8, 54.8ms, 0 | 20260505 23:01:16.307 | INFO | BAZWRITER : 61.9%, 3.6, 114.5ms, 0 | 20260505 23:01:16.307 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 23:01:16.308 | INFO | Uploaded 36GiB to GPU in 2.15785s (16.6833GiB/s) | 20260505 23:01:16.308 | INFO | Downloaded 38GiB to GPU in 2.23584s (16.9958GiB/s) | 20260505 23:01:21.448 | INFO | Analyzing chunk frames = [6992384,6992896) | 20260505 23:01:21.448 | INFO | MemInfo: { | 20260505 23:01:21.448 | INFO | "gpu" : " 17 GB", | 20260505 23:01:21.448 | INFO | "huge" : "", | 20260505 23:01:21.448 | INFO | "huge_shared" : " 27 GB", | 20260505 23:01:21.448 | INFO | "malloc" : " 14 GB", | 20260505 23:01:21.448 | INFO | "pinned" : " 29 GB" | 20260505 23:01:21.448 | INFO | } | 20260505 23:01:21.448 | INFO | Perf report for the last 5.14112 seconds: | 20260505 23:01:21.448 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:01:21.448 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:21.448 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:21.448 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:21.448 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 23:01:21.448 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:21.448 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:21.448 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 23:01:21.449 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:21.449 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:21.449 | INFO | Uploaded 36GiB to GPU in 2.10348s (17.1145GiB/s) | 20260505 23:01:21.449 | INFO | Downloaded 34GiB to GPU in 2.04064s (16.6614GiB/s) | 20260505 23:01:26.578 | INFO | Analyzing chunk frames = [6992896,6993408) | 20260505 23:01:26.578 | INFO | MemInfo: { | 20260505 23:01:26.578 | INFO | "gpu" : " 17 GB", | 20260505 23:01:26.578 | INFO | "huge" : "", | 20260505 23:01:26.578 | INFO | "huge_shared" : " 27 GB", | 20260505 23:01:26.578 | INFO | "malloc" : " 14 GB", | 20260505 23:01:26.578 | INFO | "pinned" : " 29 GB" | 20260505 23:01:26.578 | INFO | } | 20260505 23:01:26.578 | INFO | Perf report for the last 5.12996 seconds: | 20260505 23:01:26.578 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:01:26.578 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:26.578 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:26.578 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:26.578 | INFO | ANALYSIS : 62.0%, 3.0, 96.3ms, 0 | 20260505 23:01:26.578 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:26.578 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:26.578 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 23:01:26.578 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:26.579 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:26.579 | INFO | Uploaded 36GiB to GPU in 2.15416s (16.7119GiB/s) | 20260505 23:01:26.579 | INFO | Downloaded 34GiB to GPU in 2.05847s (16.5171GiB/s) | 20260505 23:01:31.720 | INFO | Analyzing chunk frames = [6993408,6993920) | 20260505 23:01:31.720 | INFO | MemInfo: { | 20260505 23:01:31.720 | INFO | "gpu" : " 17 GB", | 20260505 23:01:31.720 | INFO | "huge" : "", | 20260505 23:01:31.720 | INFO | "huge_shared" : " 27 GB", | 20260505 23:01:31.720 | INFO | "malloc" : " 14 GB", | 20260505 23:01:31.721 | INFO | "pinned" : " 29 GB" | 20260505 23:01:31.721 | INFO | } | 20260505 23:01:31.721 | INFO | Perf report for the last 5.14222 seconds: | 20260505 23:01:31.721 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:01:31.721 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:31.721 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:31.721 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:31.721 | INFO | ANALYSIS : 61.9%, 3.0, 96.1ms, 0 | 20260505 23:01:31.721 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:31.721 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:31.721 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 23:01:31.721 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:31.721 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:31.721 | INFO | Uploaded 36GiB to GPU in 2.14331s (16.7964GiB/s) | 20260505 23:01:31.721 | INFO | Downloaded 34GiB to GPU in 2.04334s (16.6394GiB/s) | 20260505 23:01:36.860 | INFO | Analyzing chunk frames = [6993920,6994432) | 20260505 23:01:36.860 | INFO | MemInfo: { | 20260505 23:01:36.860 | INFO | "gpu" : " 17 GB", | 20260505 23:01:36.860 | INFO | "huge" : "", | 20260505 23:01:36.860 | INFO | "huge_shared" : " 27 GB", | 20260505 23:01:36.860 | INFO | "malloc" : " 14 GB", | 20260505 23:01:36.860 | INFO | "pinned" : " 29 GB" | 20260505 23:01:36.860 | INFO | } | 20260505 23:01:36.860 | INFO | Perf report for the last 5.13991 seconds: | 20260505 23:01:36.861 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:01:36.861 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:36.861 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:36.861 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:36.861 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260505 23:01:36.861 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:36.861 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:36.861 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 23:01:36.861 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:36.861 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:36.861 | INFO | Uploaded 36GiB to GPU in 2.09301s (17.2001GiB/s) | 20260505 23:01:36.861 | INFO | Downloaded 34GiB to GPU in 2.04359s (16.6374GiB/s) | 20260505 23:01:42.003 | INFO | Analyzing chunk frames = [6994432,6994944) | 20260505 23:01:42.003 | INFO | MemInfo: { | 20260505 23:01:42.003 | INFO | "gpu" : " 17 GB", | 20260505 23:01:42.003 | INFO | "huge" : "", | 20260505 23:01:42.003 | INFO | "huge_shared" : " 27 GB", | 20260505 23:01:42.003 | INFO | "malloc" : " 14 GB", | 20260505 23:01:42.003 | INFO | "pinned" : " 29 GB" | 20260505 23:01:42.003 | INFO | } | 20260505 23:01:42.003 | INFO | Perf report for the last 5.14258 seconds: | 20260505 23:01:42.003 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:01:42.003 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:42.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:42.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:42.003 | INFO | ANALYSIS : 61.4%, 3.0, 95.9ms, 0 | 20260505 23:01:42.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:42.003 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:42.003 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:01:42.003 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:42.003 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:42.003 | INFO | Uploaded 36GiB to GPU in 2.0699s (17.3921GiB/s) | 20260505 23:01:42.003 | INFO | Downloaded 34GiB to GPU in 2.03942s (16.6714GiB/s) | 20260505 23:01:47.137 | INFO | Analyzing chunk frames = [6994944,6995456) | 20260505 23:01:47.137 | INFO | MemInfo: { | 20260505 23:01:47.137 | INFO | "gpu" : " 17 GB", | 20260505 23:01:47.137 | INFO | "huge" : "", | 20260505 23:01:47.137 | INFO | "huge_shared" : " 27 GB", | 20260505 23:01:47.137 | INFO | "malloc" : " 14 GB", | 20260505 23:01:47.137 | INFO | "pinned" : " 29 GB" | 20260505 23:01:47.137 | INFO | } | 20260505 23:01:47.137 | INFO | Perf report for the last 5.13428 seconds: | 20260505 23:01:47.137 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:01:47.137 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:47.137 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:47.137 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:47.137 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 23:01:47.137 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:47.138 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:47.138 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 23:01:47.138 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:47.138 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:47.138 | INFO | Uploaded 36GiB to GPU in 2.11882s (16.9906GiB/s) | 20260505 23:01:47.138 | INFO | Downloaded 34GiB to GPU in 2.11026s (16.1118GiB/s) | 20260505 23:01:52.277 | INFO | Analyzing chunk frames = [6995456,6995968) | 20260505 23:01:52.277 | INFO | MemInfo: { | 20260505 23:01:52.277 | INFO | "gpu" : " 17 GB", | 20260505 23:01:52.277 | INFO | "huge" : "", | 20260505 23:01:52.277 | INFO | "huge_shared" : " 27 GB", | 20260505 23:01:52.277 | INFO | "malloc" : " 14 GB", | 20260505 23:01:52.277 | INFO | "pinned" : " 29 GB" | 20260505 23:01:52.277 | INFO | } | 20260505 23:01:52.277 | INFO | Perf report for the last 5.13997 seconds: | 20260505 23:01:52.277 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:01:52.277 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:52.277 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:52.277 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:52.277 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 23:01:52.277 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:52.277 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:52.277 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 23:01:52.277 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:52.278 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:52.278 | INFO | Uploaded 36GiB to GPU in 2.11418s (17.0279GiB/s) | 20260505 23:01:52.278 | INFO | Downloaded 34GiB to GPU in 2.0126s (16.8936GiB/s) | 20260505 23:01:52.278 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 23:01:52.278 | INFO | Total mem usage: 15.3 GiB | 20260505 23:01:52.278 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 23:01:52.278 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -9.08%) | 20260505 23:01:52.278 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 23:01:52.278 | INFO | High Water Breakdown: | 20260505 23:01:52.278 | INFO | 60.9% : 8.56 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 23:01:52.278 | INFO | 39% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 23:01:52.278 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 23:01:52.278 | INFO | ------------------------------------------ | 20260505 23:01:52.278 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:01:52.278 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 23:01:52.278 | INFO | Total mem usage: 26.5 GiB | 20260505 23:01:52.278 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 23:01:52.278 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 23:01:52.278 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 23:01:52.278 | INFO | High Water Breakdown: | 20260505 23:01:52.278 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:01:52.278 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:01:52.278 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:01:52.278 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:01:52.278 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:01:52.278 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:01:52.278 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 23:01:52.278 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 23:01:52.278 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:01:52.278 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 23:01:52.278 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 23:01:52.278 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 23:01:52.278 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 23:01:52.278 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 23:01:52.278 | INFO | ------------------------------------------ | 20260505 23:01:52.278 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:01:52.278 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 23:01:52.278 | INFO | Total mem usage: 25.6 GiB | 20260505 23:01:52.279 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 23:01:52.279 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 23:01:52.279 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 23:01:52.279 | INFO | High Water Breakdown: | 20260505 23:01:52.279 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 23:01:52.279 | INFO | ------------------------------------------ | 20260505 23:01:52.279 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:01:52.279 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 23:01:52.279 | INFO | Total mem usage: 15.7 GiB | 20260505 23:01:52.279 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 23:01:52.279 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 23:01:52.279 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 23:01:52.279 | INFO | High Water Breakdown: | 20260505 23:01:52.279 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 23:01:52.279 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 23:01:52.279 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 23:01:52.279 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 23:01:52.279 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 23:01:52.279 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 23:01:52.279 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 23:01:52.279 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 23:01:52.279 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 23:01:52.279 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:01:52.279 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 23:01:52.279 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:01:52.279 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 23:01:52.279 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 23:01:52.279 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 23:01:52.279 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 23:01:52.279 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:01:52.279 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 23:01:52.279 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:01:52.279 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:01:52.279 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:01:52.279 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 23:01:52.279 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 23:01:52.279 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:01:52.279 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 23:01:52.279 | INFO | ------------------------------------------ | 20260505 23:01:52.279 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:01:52.279 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 23:01:52.279 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:01:52.279 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:01:52.279 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.39e+01, 1.33e+01 / 5.68e-01 | 20260505 23:01:52.279 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.04e-01, 1.02e-02 / 4.27e-03 | 20260505 23:01:52.279 | INFO | PerfReport: 11.81%, Baseline -- 3.70e+00 / 4.52e+00, 5.63e-01 / 2.16e-01 | 20260505 23:01:52.279 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.52e+00, 1.87e-01 / 1.04e-01 | 20260505 23:01:52.279 | INFO | PerfReport: 15.96%, DME -- 4.00e+01 / 4.37e+01, 3.81e+00 / 4.65e-01 | 20260505 23:01:52.279 | INFO | PerfReport: 57.29%, FrameLabeling -- 1.79e+01 / 1.97e+01, 1.47e+00 / 9.98e-01 | 20260505 23:01:52.279 | INFO | PerfReport: 7.15%, PulseAccumulating -- 2.24e+00 / 2.72e+00, 4.28e-01 / 2.53e-01 | 20260505 23:01:52.279 | INFO | PerfReport: 3.13%, Metrics -- 9.82e-01 / 1.26e+00, 2.66e-01 / 1.25e-01 | 20260505 23:01:52.279 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.40e-01 / 3.89e-01, 5.78e-02 / 2.97e-02 | 20260505 23:01:52.279 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 23:01:52.279 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:01:52.279 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:01:52.279 | INFO | PerfReport: Full Execution -- 9.86e+01 / -nan, 2.48e+01 / 0.00e+00 | 20260505 23:01:52.279 | INFO | PerfReport: 22.42%, Upload -- 2.21e+01 / 0.00e+00, 7.66e+00 / 0.00e+00 | 20260505 23:01:52.279 | INFO | PerfReport: 21.87%, Download -- 2.16e+01 / 0.00e+00, 7.66e+00 / 0.00e+00 | 20260505 23:01:52.279 | INFO | PerfReport: 31.77%, Compute -- 3.13e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260505 23:01:52.279 | INFO | PerfReport: 2.15%, LockUpload -- 2.12e+00 / 0.00e+00, 7.91e+00 / 0.00e+00 | 20260505 23:01:52.279 | INFO | PerfReport: 1.33%, LockDownload -- 1.31e+00 / 0.00e+00, 2.82e+00 / 0.00e+00 | 20260505 23:01:52.279 | INFO | PerfReport: 20.44%, LockCompute -- 2.02e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260505 23:01:52.279 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.79e-02 / 0.00e+00 | 20260505 23:01:52.405 | INFO | Starting a round of Baz IO | 20260505 23:01:55.553 | INFO | Populating RTMetrics report | 20260505 23:01:55.553 | INFO | RTMetrics report populated | 20260505 23:01:55.553 | INFO | Opening tmp RTMetrics json file | 20260505 23:01:55.556 | INFO | Serializing RTMetrics to json | 20260505 23:01:55.558 | INFO | Serialization done, swapping tmp file now... | 20260505 23:01:55.558 | INFO | RTMetrics done | 20260505 23:01:55.631 | INFO | Finished a round of Baz IO: Wrote 1.91 GiB in 3.226 seconds | 20260505 23:01:55.632 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.46% / 0.00% / 78.54% | 20260505 23:01:55.632 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.114107% / 0% / 7.54992e-05% | 20260505 23:01:55.632 | INFO | Baz IO overall: | 20260505 23:01:55.632 | INFO | BytesWritten : 4012266314770 | 20260505 23:01:55.632 | INFO | NumEvents : 1891209317223 | 20260505 23:01:57.421 | INFO | Analyzing chunk frames = [6995968,6996480) | 20260505 23:01:57.421 | INFO | MemInfo: { | 20260505 23:01:57.421 | INFO | "gpu" : " 17 GB", | 20260505 23:01:57.421 | INFO | "huge" : "", | 20260505 23:01:57.421 | INFO | "huge_shared" : " 27 GB", | 20260505 23:01:57.421 | INFO | "malloc" : " 14 GB", | 20260505 23:01:57.421 | INFO | "pinned" : " 29 GB" | 20260505 23:01:57.422 | INFO | } | 20260505 23:01:57.422 | INFO | Perf report for the last 5.14417 seconds: | 20260505 23:01:57.422 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:01:57.422 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:57.422 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:57.422 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:57.422 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260505 23:01:57.422 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:01:57.422 | INFO | RT_METRICS : 43.4%, 1.0, 22.8ms, 0 | 20260505 23:01:57.422 | INFO | PRE_HQ : 60.0%, 1.8, 54.5ms, 0 | 20260505 23:01:57.422 | INFO | BAZWRITER : 62.0%, 3.6, 114.3ms, 0 | 20260505 23:01:57.422 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 23:01:57.422 | INFO | Uploaded 36GiB to GPU in 2.16676s (16.6146GiB/s) | 20260505 23:01:57.422 | INFO | Downloaded 38GiB to GPU in 2.23112s (17.0318GiB/s) | 20260505 23:02:02.558 | INFO | Analyzing chunk frames = [6996480,6996992) | 20260505 23:02:02.558 | INFO | MemInfo: { | 20260505 23:02:02.559 | INFO | "gpu" : " 17 GB", | 20260505 23:02:02.559 | INFO | "huge" : "", | 20260505 23:02:02.559 | INFO | "huge_shared" : " 27 GB", | 20260505 23:02:02.559 | INFO | "malloc" : " 14 GB", | 20260505 23:02:02.559 | INFO | "pinned" : " 29 GB" | 20260505 23:02:02.559 | INFO | } | 20260505 23:02:02.559 | INFO | Perf report for the last 5.13716 seconds: | 20260505 23:02:02.559 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:02:02.559 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:02.559 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:02.559 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:02.559 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 23:02:02.559 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:02.559 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:02.559 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:02:02.559 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:02.559 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:02.559 | INFO | Uploaded 36GiB to GPU in 2.10377s (17.1122GiB/s) | 20260505 23:02:02.559 | INFO | Downloaded 34GiB to GPU in 2.03955s (16.6704GiB/s) | 20260505 23:02:07.692 | INFO | Analyzing chunk frames = [6996992,6997504) | 20260505 23:02:07.692 | INFO | MemInfo: { | 20260505 23:02:07.692 | INFO | "gpu" : " 17 GB", | 20260505 23:02:07.692 | INFO | "huge" : "", | 20260505 23:02:07.692 | INFO | "huge_shared" : " 27 GB", | 20260505 23:02:07.692 | INFO | "malloc" : " 14 GB", | 20260505 23:02:07.692 | INFO | "pinned" : " 29 GB" | 20260505 23:02:07.692 | INFO | } | 20260505 23:02:07.692 | INFO | Perf report for the last 5.13335 seconds: | 20260505 23:02:07.692 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:02:07.692 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:07.692 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:07.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:07.692 | INFO | ANALYSIS : 62.0%, 3.0, 96.3ms, 0 | 20260505 23:02:07.692 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:07.692 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:07.692 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 23:02:07.692 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:07.692 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:07.692 | INFO | Uploaded 36GiB to GPU in 2.15713s (16.6888GiB/s) | 20260505 23:02:07.692 | INFO | Downloaded 34GiB to GPU in 2.05768s (16.5234GiB/s) | 20260505 23:02:12.831 | INFO | Analyzing chunk frames = [6997504,6998016) | 20260505 23:02:12.831 | INFO | MemInfo: { | 20260505 23:02:12.831 | INFO | "gpu" : " 17 GB", | 20260505 23:02:12.831 | INFO | "huge" : "", | 20260505 23:02:12.831 | INFO | "huge_shared" : " 27 GB", | 20260505 23:02:12.831 | INFO | "malloc" : " 14 GB", | 20260505 23:02:12.831 | INFO | "pinned" : " 29 GB" | 20260505 23:02:12.831 | INFO | } | 20260505 23:02:12.832 | INFO | Perf report for the last 5.1394 seconds: | 20260505 23:02:12.832 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:02:12.832 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:12.832 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:12.832 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:12.832 | INFO | ANALYSIS : 61.8%, 3.0, 96.0ms, 0 | 20260505 23:02:12.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:12.832 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:12.832 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 23:02:12.832 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:12.832 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:12.832 | INFO | Uploaded 36GiB to GPU in 2.13345s (16.8741GiB/s) | 20260505 23:02:12.832 | INFO | Downloaded 34GiB to GPU in 2.03451s (16.7116GiB/s) | 20260505 23:02:17.979 | INFO | Analyzing chunk frames = [6998016,6998528) | 20260505 23:02:17.979 | INFO | MemInfo: { | 20260505 23:02:17.979 | INFO | "gpu" : " 17 GB", | 20260505 23:02:17.979 | INFO | "huge" : "", | 20260505 23:02:17.979 | INFO | "huge_shared" : " 27 GB", | 20260505 23:02:17.979 | INFO | "malloc" : " 14 GB", | 20260505 23:02:17.979 | INFO | "pinned" : " 29 GB" | 20260505 23:02:17.979 | INFO | } | 20260505 23:02:17.979 | INFO | Perf report for the last 5.14771 seconds: | 20260505 23:02:17.979 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:02:17.979 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:17.979 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:17.979 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:17.979 | INFO | ANALYSIS : 61.5%, 3.0, 96.1ms, 0 | 20260505 23:02:17.979 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:17.979 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:17.980 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 23:02:17.980 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:17.980 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:17.980 | INFO | Uploaded 36GiB to GPU in 2.13875s (16.8322GiB/s) | 20260505 23:02:17.980 | INFO | Downloaded 34GiB to GPU in 2.06483s (16.4662GiB/s) | 20260505 23:02:23.119 | INFO | Analyzing chunk frames = [6998528,6999040) | 20260505 23:02:23.119 | INFO | MemInfo: { | 20260505 23:02:23.119 | INFO | "gpu" : " 17 GB", | 20260505 23:02:23.119 | INFO | "huge" : "", | 20260505 23:02:23.119 | INFO | "huge_shared" : " 27 GB", | 20260505 23:02:23.119 | INFO | "malloc" : " 14 GB", | 20260505 23:02:23.119 | INFO | "pinned" : " 29 GB" | 20260505 23:02:23.119 | INFO | } | 20260505 23:02:23.119 | INFO | Perf report for the last 5.13968 seconds: | 20260505 23:02:23.119 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:02:23.119 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:23.119 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:23.119 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:23.119 | INFO | ANALYSIS : 61.4%, 3.0, 95.9ms, 0 | 20260505 23:02:23.119 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:23.119 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:23.119 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:02:23.119 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:23.119 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:23.119 | INFO | Uploaded 36GiB to GPU in 2.08366s (17.2773GiB/s) | 20260505 23:02:23.119 | INFO | Downloaded 34GiB to GPU in 2.04101s (16.6585GiB/s) | 20260505 23:02:28.256 | INFO | Analyzing chunk frames = [6999040,6999552) | 20260505 23:02:28.256 | INFO | MemInfo: { | 20260505 23:02:28.256 | INFO | "gpu" : " 17 GB", | 20260505 23:02:28.256 | INFO | "huge" : "", | 20260505 23:02:28.256 | INFO | "huge_shared" : " 27 GB", | 20260505 23:02:28.256 | INFO | "malloc" : " 14 GB", | 20260505 23:02:28.257 | INFO | "pinned" : " 29 GB" | 20260505 23:02:28.257 | INFO | } | 20260505 23:02:28.257 | INFO | Perf report for the last 5.13777 seconds: | 20260505 23:02:28.257 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:02:28.257 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:28.257 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:28.257 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:28.257 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 23:02:28.257 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:28.257 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:28.257 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 23:02:28.257 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:28.257 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:28.257 | INFO | Uploaded 36GiB to GPU in 2.06165s (17.4618GiB/s) | 20260505 23:02:28.257 | INFO | Downloaded 34GiB to GPU in 2.06841s (16.4377GiB/s) | 20260505 23:02:33.394 | INFO | Analyzing chunk frames = [6999552,7000064) | 20260505 23:02:33.394 | INFO | MemInfo: { | 20260505 23:02:33.394 | INFO | "gpu" : " 17 GB", | 20260505 23:02:33.394 | INFO | "huge" : "", | 20260505 23:02:33.394 | INFO | "huge_shared" : " 27 GB", | 20260505 23:02:33.394 | INFO | "malloc" : " 14 GB", | 20260505 23:02:33.394 | INFO | "pinned" : " 29 GB" | 20260505 23:02:33.394 | INFO | } | 20260505 23:02:33.394 | INFO | Perf report for the last 5.13728 seconds: | 20260505 23:02:33.394 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:02:33.394 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:33.394 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:33.394 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:33.394 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 23:02:33.394 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:33.394 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:33.394 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 23:02:33.394 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:33.395 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:33.395 | INFO | Uploaded 36GiB to GPU in 2.11808s (16.9965GiB/s) | 20260505 23:02:33.395 | INFO | Downloaded 34GiB to GPU in 2.0132s (16.8885GiB/s) | 20260505 23:02:33.528 | INFO | Starting a round of Baz IO | 20260505 23:02:36.675 | INFO | Populating RTMetrics report | 20260505 23:02:36.676 | INFO | RTMetrics report populated | 20260505 23:02:36.676 | INFO | Opening tmp RTMetrics json file | 20260505 23:02:36.678 | INFO | Serializing RTMetrics to json | 20260505 23:02:36.680 | INFO | Serialization done, swapping tmp file now... | 20260505 23:02:36.681 | INFO | RTMetrics done | 20260505 23:02:36.757 | INFO | Finished a round of Baz IO: Wrote 1.91 GiB in 3.229 seconds | 20260505 23:02:36.757 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.46% / 0.00% / 78.54% | 20260505 23:02:36.757 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.114155% / 0% / 9.53674e-05% | 20260505 23:02:36.757 | INFO | Baz IO overall: | 20260505 23:02:36.757 | INFO | BytesWritten : 4014317697567 | 20260505 23:02:36.757 | INFO | NumEvents : 1892152334378 | 20260505 23:02:38.528 | INFO | Analyzing chunk frames = [7000064,7000576) | 20260505 23:02:38.528 | INFO | MemInfo: { | 20260505 23:02:38.528 | INFO | "gpu" : " 17 GB", | 20260505 23:02:38.528 | INFO | "huge" : "", | 20260505 23:02:38.528 | INFO | "huge_shared" : " 27 GB", | 20260505 23:02:38.528 | INFO | "malloc" : " 14 GB", | 20260505 23:02:38.528 | INFO | "pinned" : " 29 GB" | 20260505 23:02:38.528 | INFO | } | 20260505 23:02:38.528 | INFO | Perf report for the last 5.13382 seconds: | 20260505 23:02:38.528 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:02:38.528 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:38.528 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:38.528 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:38.528 | INFO | ANALYSIS : 62.9%, 3.0, 98.0ms, 0 | 20260505 23:02:38.528 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:38.528 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260505 23:02:38.528 | INFO | PRE_HQ : 59.7%, 1.8, 54.3ms, 0 | 20260505 23:02:38.528 | INFO | BAZWRITER : 62.4%, 3.6, 114.9ms, 0 | 20260505 23:02:38.528 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 23:02:38.528 | INFO | Uploaded 36GiB to GPU in 2.13967s (16.825GiB/s) | 20260505 23:02:38.528 | INFO | Downloaded 38GiB to GPU in 2.2408s (16.9582GiB/s) | 20260505 23:02:43.672 | INFO | Analyzing chunk frames = [7000576,7001088) | 20260505 23:02:43.672 | INFO | MemInfo: { | 20260505 23:02:43.672 | INFO | "gpu" : " 17 GB", | 20260505 23:02:43.672 | INFO | "huge" : "", | 20260505 23:02:43.672 | INFO | "huge_shared" : " 27 GB", | 20260505 23:02:43.672 | INFO | "malloc" : " 14 GB", | 20260505 23:02:43.672 | INFO | "pinned" : " 29 GB" | 20260505 23:02:43.672 | INFO | } | 20260505 23:02:43.672 | INFO | Perf report for the last 5.14423 seconds: | 20260505 23:02:43.672 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:02:43.672 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:43.672 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:43.672 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:43.672 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 23:02:43.672 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:43.672 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:43.672 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 23:02:43.672 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:43.672 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:43.672 | INFO | Uploaded 36GiB to GPU in 2.08345s (17.2791GiB/s) | 20260505 23:02:43.672 | INFO | Downloaded 34GiB to GPU in 2.03024s (16.7468GiB/s) | 20260505 23:02:48.814 | INFO | Analyzing chunk frames = [7001088,7001600) | 20260505 23:02:48.814 | INFO | MemInfo: { | 20260505 23:02:48.814 | INFO | "gpu" : " 17 GB", | 20260505 23:02:48.814 | INFO | "huge" : "", | 20260505 23:02:48.814 | INFO | "huge_shared" : " 27 GB", | 20260505 23:02:48.814 | INFO | "malloc" : " 14 GB", | 20260505 23:02:48.814 | INFO | "pinned" : " 29 GB" | 20260505 23:02:48.814 | INFO | } | 20260505 23:02:48.814 | INFO | Perf report for the last 5.14198 seconds: | 20260505 23:02:48.814 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:02:48.814 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:48.814 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:48.814 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:48.814 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260505 23:02:48.814 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:48.814 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:48.814 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 23:02:48.814 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:48.814 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:48.814 | INFO | Uploaded 36GiB to GPU in 2.11051s (17.0575GiB/s) | 20260505 23:02:48.814 | INFO | Downloaded 34GiB to GPU in 2.02941s (16.7536GiB/s) | 20260505 23:02:53.952 | INFO | Analyzing chunk frames = [7001600,7002112) | 20260505 23:02:53.952 | INFO | MemInfo: { | 20260505 23:02:53.952 | INFO | "gpu" : " 17 GB", | 20260505 23:02:53.952 | INFO | "huge" : "", | 20260505 23:02:53.952 | INFO | "huge_shared" : " 27 GB", | 20260505 23:02:53.952 | INFO | "malloc" : " 14 GB", | 20260505 23:02:53.952 | INFO | "pinned" : " 29 GB" | 20260505 23:02:53.952 | INFO | } | 20260505 23:02:53.952 | INFO | Perf report for the last 5.13804 seconds: | 20260505 23:02:53.952 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:02:53.952 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:53.952 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:53.952 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:53.952 | INFO | ANALYSIS : 61.6%, 3.0, 95.7ms, 0 | 20260505 23:02:53.952 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:53.952 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:53.952 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 23:02:53.952 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:53.952 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:53.952 | INFO | Uploaded 36GiB to GPU in 2.12049s (16.9772GiB/s) | 20260505 23:02:53.953 | INFO | Downloaded 34GiB to GPU in 2.01548s (16.8695GiB/s) | 20260505 23:02:59.088 | INFO | Analyzing chunk frames = [7002112,7002624) | 20260505 23:02:59.088 | INFO | MemInfo: { | 20260505 23:02:59.088 | INFO | "gpu" : " 17 GB", | 20260505 23:02:59.088 | INFO | "huge" : "", | 20260505 23:02:59.088 | INFO | "huge_shared" : " 27 GB", | 20260505 23:02:59.088 | INFO | "malloc" : " 14 GB", | 20260505 23:02:59.088 | INFO | "pinned" : " 29 GB" | 20260505 23:02:59.088 | INFO | } | 20260505 23:02:59.088 | INFO | Perf report for the last 5.1363 seconds: | 20260505 23:02:59.088 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:02:59.088 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:59.088 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:59.088 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:59.088 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260505 23:02:59.088 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:59.089 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:59.089 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 23:02:59.089 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:59.089 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:02:59.089 | INFO | Uploaded 36GiB to GPU in 2.0844s (17.2711GiB/s) | 20260505 23:02:59.089 | INFO | Downloaded 34GiB to GPU in 2.04401s (16.634GiB/s) | 20260505 23:03:04.235 | INFO | Analyzing chunk frames = [7002624,7003136) | 20260505 23:03:04.235 | INFO | MemInfo: { | 20260505 23:03:04.235 | INFO | "gpu" : " 17 GB", | 20260505 23:03:04.235 | INFO | "huge" : "", | 20260505 23:03:04.235 | INFO | "huge_shared" : " 27 GB", | 20260505 23:03:04.235 | INFO | "malloc" : " 14 GB", | 20260505 23:03:04.235 | INFO | "pinned" : " 29 GB" | 20260505 23:03:04.235 | INFO | } | 20260505 23:03:04.235 | INFO | Perf report for the last 5.14688 seconds: | 20260505 23:03:04.235 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:03:04.235 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:04.235 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:04.235 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:04.236 | INFO | ANALYSIS : 61.4%, 3.0, 96.1ms, 0 | 20260505 23:03:04.236 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:04.236 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:04.236 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:03:04.236 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:04.236 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:04.236 | INFO | Uploaded 36GiB to GPU in 2.08239s (17.2879GiB/s) | 20260505 23:03:04.236 | INFO | Downloaded 34GiB to GPU in 2.04344s (16.6386GiB/s) | 20260505 23:03:09.368 | INFO | Analyzing chunk frames = [7003136,7003648) | 20260505 23:03:09.368 | INFO | MemInfo: { | 20260505 23:03:09.368 | INFO | "gpu" : " 17 GB", | 20260505 23:03:09.368 | INFO | "huge" : "", | 20260505 23:03:09.368 | INFO | "huge_shared" : " 27 GB", | 20260505 23:03:09.368 | INFO | "malloc" : " 14 GB", | 20260505 23:03:09.368 | INFO | "pinned" : " 29 GB" | 20260505 23:03:09.368 | INFO | } | 20260505 23:03:09.368 | INFO | Perf report for the last 5.13255 seconds: | 20260505 23:03:09.368 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:03:09.368 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:09.368 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:09.368 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:09.368 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260505 23:03:09.368 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:09.368 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:09.368 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 23:03:09.368 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:09.368 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:09.368 | INFO | Uploaded 36GiB to GPU in 2.05226s (17.5417GiB/s) | 20260505 23:03:09.368 | INFO | Downloaded 34GiB to GPU in 2.05899s (16.513GiB/s) | 20260505 23:03:14.507 | INFO | Analyzing chunk frames = [7003648,7004160) | 20260505 23:03:14.507 | INFO | MemInfo: { | 20260505 23:03:14.507 | INFO | "gpu" : " 17 GB", | 20260505 23:03:14.507 | INFO | "huge" : "", | 20260505 23:03:14.507 | INFO | "huge_shared" : " 27 GB", | 20260505 23:03:14.507 | INFO | "malloc" : " 14 GB", | 20260505 23:03:14.507 | INFO | "pinned" : " 29 GB" | 20260505 23:03:14.507 | INFO | } | 20260505 23:03:14.507 | INFO | Perf report for the last 5.1395 seconds: | 20260505 23:03:14.507 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:03:14.507 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:14.507 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:14.507 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:14.507 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 23:03:14.507 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:14.507 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:14.507 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 23:03:14.507 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:14.507 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:14.508 | INFO | Uploaded 36GiB to GPU in 2.11521s (17.0196GiB/s) | 20260505 23:03:14.508 | INFO | Downloaded 34GiB to GPU in 2.00777s (16.9342GiB/s) | 20260505 23:03:14.508 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 23:03:14.508 | INFO | Total mem usage: 15.3 GiB | 20260505 23:03:14.508 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 23:03:14.508 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -9.04%) | 20260505 23:03:14.508 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 23:03:14.508 | INFO | High Water Breakdown: | 20260505 23:03:14.508 | INFO | 60.9% : 8.56 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 23:03:14.508 | INFO | 39% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 23:03:14.508 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 23:03:14.508 | INFO | ------------------------------------------ | 20260505 23:03:14.508 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:03:14.508 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 23:03:14.508 | INFO | Total mem usage: 26.5 GiB | 20260505 23:03:14.508 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 23:03:14.508 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 23:03:14.508 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 23:03:14.508 | INFO | High Water Breakdown: | 20260505 23:03:14.508 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:03:14.508 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:03:14.508 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:03:14.508 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:03:14.508 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:03:14.508 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:03:14.508 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 23:03:14.508 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 23:03:14.508 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:03:14.508 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 23:03:14.508 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 23:03:14.508 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 23:03:14.508 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 23:03:14.509 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 23:03:14.509 | INFO | ------------------------------------------ | 20260505 23:03:14.509 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:03:14.509 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 23:03:14.509 | INFO | Total mem usage: 25.6 GiB | 20260505 23:03:14.509 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 23:03:14.509 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 23:03:14.509 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 23:03:14.509 | INFO | High Water Breakdown: | 20260505 23:03:14.509 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 23:03:14.509 | INFO | ------------------------------------------ | 20260505 23:03:14.509 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:03:14.509 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 23:03:14.509 | INFO | Total mem usage: 15.7 GiB | 20260505 23:03:14.509 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 23:03:14.509 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 23:03:14.509 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 23:03:14.509 | INFO | High Water Breakdown: | 20260505 23:03:14.509 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 23:03:14.509 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 23:03:14.509 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 23:03:14.509 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 23:03:14.509 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 23:03:14.509 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 23:03:14.509 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 23:03:14.509 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 23:03:14.509 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 23:03:14.509 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:03:14.509 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 23:03:14.509 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:03:14.509 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 23:03:14.509 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 23:03:14.509 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 23:03:14.509 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 23:03:14.509 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:03:14.509 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 23:03:14.509 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:03:14.509 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:03:14.509 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:03:14.509 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 23:03:14.509 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 23:03:14.509 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:03:14.509 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 23:03:14.509 | INFO | ------------------------------------------ | 20260505 23:03:14.509 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:03:14.509 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 23:03:14.509 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:03:14.509 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:03:14.509 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.40e+01, 1.34e+01 / 6.30e-01 | 20260505 23:03:14.509 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.04e-01, 1.01e-02 / 3.73e-03 | 20260505 23:03:14.509 | INFO | PerfReport: 11.81%, Baseline -- 3.71e+00 / 4.51e+00, 5.70e-01 / 1.85e-01 | 20260505 23:03:14.509 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.52e+00, 1.92e-01 / 8.22e-02 | 20260505 23:03:14.509 | INFO | PerfReport: 15.95%, DME -- 4.01e+01 / 4.40e+01, 3.84e+00 / 9.52e-01 | 20260505 23:03:14.509 | INFO | PerfReport: 57.34%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.56e+00 / 1.17e+00 | 20260505 23:03:14.509 | INFO | PerfReport: 7.11%, PulseAccumulating -- 2.23e+00 / 2.77e+00, 4.28e-01 / 2.52e-01 | 20260505 23:03:14.509 | INFO | PerfReport: 3.13%, Metrics -- 9.82e-01 / 1.29e+00, 2.68e-01 / 1.15e-01 | 20260505 23:03:14.509 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.41e-01 / 4.01e-01, 5.90e-02 / 3.62e-02 | 20260505 23:03:14.509 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 23:03:14.509 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:03:14.509 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:03:14.509 | INFO | PerfReport: Full Execution -- 9.85e+01 / -nan, 2.47e+01 / 0.00e+00 | 20260505 23:03:14.509 | INFO | PerfReport: 22.31%, Upload -- 2.20e+01 / 0.00e+00, 7.54e+00 / 0.00e+00 | 20260505 23:03:14.509 | INFO | PerfReport: 21.82%, Download -- 2.15e+01 / 0.00e+00, 7.61e+00 / 0.00e+00 | 20260505 23:03:14.509 | INFO | PerfReport: 31.87%, Compute -- 3.14e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 23:03:14.509 | INFO | PerfReport: 2.10%, LockUpload -- 2.07e+00 / 0.00e+00, 7.75e+00 / 0.00e+00 | 20260505 23:03:14.509 | INFO | PerfReport: 1.31%, LockDownload -- 1.29e+00 / 0.00e+00, 2.93e+00 / 0.00e+00 | 20260505 23:03:14.509 | INFO | PerfReport: 20.57%, LockCompute -- 2.03e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260505 23:03:14.509 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.88e-02 / 0.00e+00 | 20260505 23:03:14.632 | INFO | Starting a round of Baz IO | 20260505 23:03:17.782 | INFO | Populating RTMetrics report | 20260505 23:03:17.782 | INFO | RTMetrics report populated | 20260505 23:03:17.782 | INFO | Opening tmp RTMetrics json file | 20260505 23:03:17.793 | INFO | Serializing RTMetrics to json | 20260505 23:03:17.795 | INFO | Serialization done, swapping tmp file now... | 20260505 23:03:17.797 | INFO | RTMetrics done | 20260505 23:03:17.866 | INFO | Finished a round of Baz IO: Wrote 1.908 GiB in 3.233 seconds | 20260505 23:03:17.866 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.46% / 0.00% / 78.54% | 20260505 23:03:17.866 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.113825% / 0% / 0.000103315% | 20260505 23:03:17.866 | INFO | Baz IO overall: | 20260505 23:03:17.866 | INFO | BytesWritten : 4016366854152 | 20260505 23:03:17.866 | INFO | NumEvents : 1893093655665 | 20260505 23:03:19.647 | INFO | Analyzing chunk frames = [7004160,7004672) | 20260505 23:03:19.647 | INFO | MemInfo: { | 20260505 23:03:19.647 | INFO | "gpu" : " 17 GB", | 20260505 23:03:19.647 | INFO | "huge" : "", | 20260505 23:03:19.647 | INFO | "huge_shared" : " 27 GB", | 20260505 23:03:19.647 | INFO | "malloc" : " 14 GB", | 20260505 23:03:19.647 | INFO | "pinned" : " 29 GB" | 20260505 23:03:19.647 | INFO | } | 20260505 23:03:19.648 | INFO | Perf report for the last 5.14016 seconds: | 20260505 23:03:19.648 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:03:19.648 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:19.648 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:19.648 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:19.648 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 23:03:19.648 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:19.648 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260505 23:03:19.648 | INFO | PRE_HQ : 59.7%, 1.8, 54.8ms, 0 | 20260505 23:03:19.648 | INFO | BAZWRITER : 62.1%, 3.6, 115.1ms, 0 | 20260505 23:03:19.648 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260505 23:03:19.648 | INFO | Uploaded 36GiB to GPU in 2.1328s (16.8792GiB/s) | 20260505 23:03:19.648 | INFO | Downloaded 38GiB to GPU in 2.22148s (17.1057GiB/s) | 20260505 23:03:24.781 | INFO | Analyzing chunk frames = [7004672,7005184) | 20260505 23:03:24.781 | INFO | MemInfo: { | 20260505 23:03:24.781 | INFO | "gpu" : " 17 GB", | 20260505 23:03:24.781 | INFO | "huge" : "", | 20260505 23:03:24.781 | INFO | "huge_shared" : " 27 GB", | 20260505 23:03:24.781 | INFO | "malloc" : " 14 GB", | 20260505 23:03:24.781 | INFO | "pinned" : " 29 GB" | 20260505 23:03:24.781 | INFO | } | 20260505 23:03:24.781 | INFO | Perf report for the last 5.13382 seconds: | 20260505 23:03:24.781 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:03:24.781 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:24.781 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:24.781 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:24.781 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260505 23:03:24.781 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:24.782 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:24.782 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:03:24.782 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:24.782 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:24.782 | INFO | Uploaded 36GiB to GPU in 2.08638s (17.2547GiB/s) | 20260505 23:03:24.782 | INFO | Downloaded 34GiB to GPU in 2.03409s (16.7151GiB/s) | 20260505 23:03:29.927 | INFO | Analyzing chunk frames = [7005184,7005696) | 20260505 23:03:29.927 | INFO | MemInfo: { | 20260505 23:03:29.927 | INFO | "gpu" : " 17 GB", | 20260505 23:03:29.927 | INFO | "huge" : "", | 20260505 23:03:29.927 | INFO | "huge_shared" : " 27 GB", | 20260505 23:03:29.927 | INFO | "malloc" : " 14 GB", | 20260505 23:03:29.927 | INFO | "pinned" : " 29 GB" | 20260505 23:03:29.927 | INFO | } | 20260505 23:03:29.927 | INFO | Perf report for the last 5.14545 seconds: | 20260505 23:03:29.927 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:03:29.927 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:29.927 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:29.927 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:29.927 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260505 23:03:29.927 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:29.927 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:29.927 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:03:29.927 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:29.927 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:29.927 | INFO | Uploaded 36GiB to GPU in 2.13043s (16.898GiB/s) | 20260505 23:03:29.927 | INFO | Downloaded 34GiB to GPU in 2.02004s (16.8313GiB/s) | 20260505 23:03:35.071 | INFO | Analyzing chunk frames = [7005696,7006208) | 20260505 23:03:35.071 | INFO | MemInfo: { | 20260505 23:03:35.071 | INFO | "gpu" : " 17 GB", | 20260505 23:03:35.071 | INFO | "huge" : "", | 20260505 23:03:35.071 | INFO | "huge_shared" : " 27 GB", | 20260505 23:03:35.071 | INFO | "malloc" : " 14 GB", | 20260505 23:03:35.071 | INFO | "pinned" : " 29 GB" | 20260505 23:03:35.071 | INFO | } | 20260505 23:03:35.071 | INFO | Perf report for the last 5.14444 seconds: | 20260505 23:03:35.071 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:03:35.071 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:35.071 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:35.071 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:35.071 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260505 23:03:35.071 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:35.071 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:35.071 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:03:35.071 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:35.071 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:35.072 | INFO | Uploaded 36GiB to GPU in 2.14183s (16.8081GiB/s) | 20260505 23:03:35.072 | INFO | Downloaded 34GiB to GPU in 2.02635s (16.7789GiB/s) | 20260505 23:03:40.203 | INFO | Analyzing chunk frames = [7006208,7006720) | 20260505 23:03:40.203 | INFO | MemInfo: { | 20260505 23:03:40.203 | INFO | "gpu" : " 17 GB", | 20260505 23:03:40.203 | INFO | "huge" : "", | 20260505 23:03:40.203 | INFO | "huge_shared" : " 27 GB", | 20260505 23:03:40.203 | INFO | "malloc" : " 14 GB", | 20260505 23:03:40.203 | INFO | "pinned" : " 29 GB" | 20260505 23:03:40.203 | INFO | } | 20260505 23:03:40.203 | INFO | Perf report for the last 5.13211 seconds: | 20260505 23:03:40.203 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:03:40.203 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:40.203 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:40.203 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:40.203 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260505 23:03:40.204 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:40.204 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:40.204 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 23:03:40.204 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:40.204 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:40.204 | INFO | Uploaded 36GiB to GPU in 2.0883s (17.2389GiB/s) | 20260505 23:03:40.204 | INFO | Downloaded 34GiB to GPU in 2.04646s (16.614GiB/s) | 20260505 23:03:45.339 | INFO | Analyzing chunk frames = [7006720,7007232) | 20260505 23:03:45.339 | INFO | MemInfo: { | 20260505 23:03:45.339 | INFO | "gpu" : " 17 GB", | 20260505 23:03:45.339 | INFO | "huge" : "", | 20260505 23:03:45.339 | INFO | "huge_shared" : " 27 GB", | 20260505 23:03:45.339 | INFO | "malloc" : " 14 GB", | 20260505 23:03:45.339 | INFO | "pinned" : " 29 GB" | 20260505 23:03:45.339 | INFO | } | 20260505 23:03:45.339 | INFO | Perf report for the last 5.13574 seconds: | 20260505 23:03:45.339 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:03:45.339 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:45.339 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:45.339 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:45.339 | INFO | ANALYSIS : 61.5%, 3.0, 96.0ms, 0 | 20260505 23:03:45.339 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:45.339 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:45.339 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 23:03:45.339 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:45.339 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:45.339 | INFO | Uploaded 36GiB to GPU in 2.09675s (17.1695GiB/s) | 20260505 23:03:45.340 | INFO | Downloaded 34GiB to GPU in 2.03193s (16.7329GiB/s) | 20260505 23:03:50.486 | INFO | Analyzing chunk frames = [7007232,7007744) | 20260505 23:03:50.486 | INFO | MemInfo: { | 20260505 23:03:50.486 | INFO | "gpu" : " 17 GB", | 20260505 23:03:50.486 | INFO | "huge" : "", | 20260505 23:03:50.486 | INFO | "huge_shared" : " 27 GB", | 20260505 23:03:50.487 | INFO | "malloc" : " 14 GB", | 20260505 23:03:50.487 | INFO | "pinned" : " 29 GB" | 20260505 23:03:50.487 | INFO | } | 20260505 23:03:50.487 | INFO | Perf report for the last 5.14743 seconds: | 20260505 23:03:50.487 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:03:50.487 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:50.487 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:50.487 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:50.487 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 23:03:50.487 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:50.487 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:50.487 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260505 23:03:50.487 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:50.487 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:50.487 | INFO | Uploaded 36GiB to GPU in 2.16017s (16.6653GiB/s) | 20260505 23:03:50.487 | INFO | Downloaded 34GiB to GPU in 2.12016s (16.0365GiB/s) | 20260505 23:03:55.624 | INFO | Analyzing chunk frames = [7007744,7008256) | 20260505 23:03:55.624 | INFO | MemInfo: { | 20260505 23:03:55.624 | INFO | "gpu" : " 17 GB", | 20260505 23:03:55.624 | INFO | "huge" : "", | 20260505 23:03:55.624 | INFO | "huge_shared" : " 27 GB", | 20260505 23:03:55.624 | INFO | "malloc" : " 14 GB", | 20260505 23:03:55.624 | INFO | "pinned" : " 29 GB" | 20260505 23:03:55.624 | INFO | } | 20260505 23:03:55.624 | INFO | Perf report for the last 5.13726 seconds: | 20260505 23:03:55.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:03:55.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:55.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:55.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:55.624 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 23:03:55.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:55.624 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:55.624 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 23:03:55.624 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:55.624 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:03:55.624 | INFO | Uploaded 36GiB to GPU in 2.1126s (17.0406GiB/s) | 20260505 23:03:55.624 | INFO | Downloaded 34GiB to GPU in 2.00542s (16.9541GiB/s) | 20260505 23:03:55.744 | INFO | Starting a round of Baz IO | 20260505 23:03:58.898 | INFO | Populating RTMetrics report | 20260505 23:03:58.899 | INFO | RTMetrics report populated | 20260505 23:03:58.899 | INFO | Opening tmp RTMetrics json file | 20260505 23:03:58.900 | INFO | Serializing RTMetrics to json | 20260505 23:03:58.904 | INFO | Serialization done, swapping tmp file now... | 20260505 23:03:58.905 | INFO | RTMetrics done | 20260505 23:03:58.994 | INFO | Finished a round of Baz IO: Wrote 1.909 GiB in 3.25 seconds | 20260505 23:03:58.994 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.45% / 0.00% / 78.54% | 20260505 23:03:58.994 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.113535% / 0% / 8.74201e-05% | 20260505 23:03:58.994 | INFO | Baz IO overall: | 20260505 23:03:58.994 | INFO | BytesWritten : 4018416922829 | 20260505 23:03:58.994 | INFO | NumEvents : 1894035158091 | 20260505 23:04:00.764 | INFO | Analyzing chunk frames = [7008256,7008768) | 20260505 23:04:00.764 | INFO | MemInfo: { | 20260505 23:04:00.764 | INFO | "gpu" : " 17 GB", | 20260505 23:04:00.764 | INFO | "huge" : "", | 20260505 23:04:00.764 | INFO | "huge_shared" : " 27 GB", | 20260505 23:04:00.764 | INFO | "malloc" : " 14 GB", | 20260505 23:04:00.764 | INFO | "pinned" : " 29 GB" | 20260505 23:04:00.764 | INFO | } | 20260505 23:04:00.764 | INFO | Perf report for the last 5.14057 seconds: | 20260505 23:04:00.764 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:04:00.764 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:00.764 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:00.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:00.765 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260505 23:04:00.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:00.765 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260505 23:04:00.765 | INFO | PRE_HQ : 60.0%, 1.7, 54.4ms, 0 | 20260505 23:04:00.765 | INFO | BAZWRITER : 62.5%, 3.6, 114.8ms, 0 | 20260505 23:04:00.765 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 23:04:00.765 | INFO | Uploaded 36GiB to GPU in 2.17004s (16.5896GiB/s) | 20260505 23:04:00.765 | INFO | Downloaded 38GiB to GPU in 2.24196s (16.9494GiB/s) | 20260505 23:04:05.899 | INFO | Analyzing chunk frames = [7008768,7009280) | 20260505 23:04:05.899 | INFO | MemInfo: { | 20260505 23:04:05.899 | INFO | "gpu" : " 17 GB", | 20260505 23:04:05.899 | INFO | "huge" : "", | 20260505 23:04:05.899 | INFO | "huge_shared" : " 27 GB", | 20260505 23:04:05.899 | INFO | "malloc" : " 14 GB", | 20260505 23:04:05.899 | INFO | "pinned" : " 29 GB" | 20260505 23:04:05.899 | INFO | } | 20260505 23:04:05.900 | INFO | Perf report for the last 5.13503 seconds: | 20260505 23:04:05.900 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:04:05.900 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:05.900 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:05.900 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:05.900 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260505 23:04:05.900 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:05.900 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:05.900 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:04:05.900 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:05.900 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:05.900 | INFO | Uploaded 36GiB to GPU in 2.07752s (17.3283GiB/s) | 20260505 23:04:05.900 | INFO | Downloaded 34GiB to GPU in 2.02481s (16.7917GiB/s) | 20260505 23:04:11.038 | INFO | Analyzing chunk frames = [7009280,7009792) | 20260505 23:04:11.038 | INFO | MemInfo: { | 20260505 23:04:11.038 | INFO | "gpu" : " 17 GB", | 20260505 23:04:11.038 | INFO | "huge" : "", | 20260505 23:04:11.038 | INFO | "huge_shared" : " 27 GB", | 20260505 23:04:11.038 | INFO | "malloc" : " 14 GB", | 20260505 23:04:11.038 | INFO | "pinned" : " 29 GB" | 20260505 23:04:11.038 | INFO | } | 20260505 23:04:11.038 | INFO | Perf report for the last 5.13833 seconds: | 20260505 23:04:11.038 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:04:11.038 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:11.038 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:11.038 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:11.038 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260505 23:04:11.038 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:11.038 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:11.038 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:04:11.038 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:11.038 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:11.038 | INFO | Uploaded 36GiB to GPU in 2.15017s (16.7429GiB/s) | 20260505 23:04:11.038 | INFO | Downloaded 34GiB to GPU in 2.05531s (16.5425GiB/s) | 20260505 23:04:16.181 | INFO | Analyzing chunk frames = [7009792,7010304) | 20260505 23:04:16.181 | INFO | MemInfo: { | 20260505 23:04:16.181 | INFO | "gpu" : " 17 GB", | 20260505 23:04:16.181 | INFO | "huge" : "", | 20260505 23:04:16.181 | INFO | "huge_shared" : " 27 GB", | 20260505 23:04:16.181 | INFO | "malloc" : " 14 GB", | 20260505 23:04:16.181 | INFO | "pinned" : " 29 GB" | 20260505 23:04:16.181 | INFO | } | 20260505 23:04:16.181 | INFO | Perf report for the last 5.14356 seconds: | 20260505 23:04:16.181 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:04:16.181 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:16.181 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:16.181 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:16.182 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260505 23:04:16.182 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:16.182 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:16.182 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:04:16.182 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:16.182 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:16.182 | INFO | Uploaded 36GiB to GPU in 2.1262s (16.9317GiB/s) | 20260505 23:04:16.182 | INFO | Downloaded 34GiB to GPU in 2.02259s (16.8102GiB/s) | 20260505 23:04:21.319 | INFO | Analyzing chunk frames = [7010304,7010816) | 20260505 23:04:21.319 | INFO | MemInfo: { | 20260505 23:04:21.319 | INFO | "gpu" : " 17 GB", | 20260505 23:04:21.319 | INFO | "huge" : "", | 20260505 23:04:21.319 | INFO | "huge_shared" : " 27 GB", | 20260505 23:04:21.319 | INFO | "malloc" : " 14 GB", | 20260505 23:04:21.320 | INFO | "pinned" : " 29 GB" | 20260505 23:04:21.320 | INFO | } | 20260505 23:04:21.320 | INFO | Perf report for the last 5.13829 seconds: | 20260505 23:04:21.320 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:04:21.320 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:21.320 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:21.320 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:21.320 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260505 23:04:21.320 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:21.320 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:21.320 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:04:21.320 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:21.320 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:21.320 | INFO | Uploaded 36GiB to GPU in 2.08729s (17.2472GiB/s) | 20260505 23:04:21.320 | INFO | Downloaded 34GiB to GPU in 2.04699s (16.6097GiB/s) | 20260505 23:04:26.455 | INFO | Analyzing chunk frames = [7010816,7011328) | 20260505 23:04:26.455 | INFO | MemInfo: { | 20260505 23:04:26.455 | INFO | "gpu" : " 17 GB", | 20260505 23:04:26.455 | INFO | "huge" : "", | 20260505 23:04:26.455 | INFO | "huge_shared" : " 27 GB", | 20260505 23:04:26.455 | INFO | "malloc" : " 14 GB", | 20260505 23:04:26.455 | INFO | "pinned" : " 29 GB" | 20260505 23:04:26.455 | INFO | } | 20260505 23:04:26.455 | INFO | Perf report for the last 5.13553 seconds: | 20260505 23:04:26.455 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:04:26.455 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:26.455 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:26.455 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:26.455 | INFO | ANALYSIS : 61.5%, 3.0, 96.0ms, 0 | 20260505 23:04:26.455 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:26.456 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:26.456 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260505 23:04:26.456 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:26.456 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:26.456 | INFO | Uploaded 36GiB to GPU in 2.07234s (17.3716GiB/s) | 20260505 23:04:26.456 | INFO | Downloaded 34GiB to GPU in 2.03422s (16.714GiB/s) | 20260505 23:04:31.594 | INFO | Analyzing chunk frames = [7011328,7011840) | 20260505 23:04:31.594 | INFO | MemInfo: { | 20260505 23:04:31.594 | INFO | "gpu" : " 17 GB", | 20260505 23:04:31.594 | INFO | "huge" : "", | 20260505 23:04:31.594 | INFO | "huge_shared" : " 27 GB", | 20260505 23:04:31.594 | INFO | "malloc" : " 14 GB", | 20260505 23:04:31.594 | INFO | "pinned" : " 29 GB" | 20260505 23:04:31.594 | INFO | } | 20260505 23:04:31.594 | INFO | Perf report for the last 5.13918 seconds: | 20260505 23:04:31.595 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:04:31.595 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:31.595 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:31.595 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:31.595 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 23:04:31.595 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:31.595 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:31.595 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 23:04:31.595 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:31.595 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:31.595 | INFO | Uploaded 36GiB to GPU in 2.15445s (16.7096GiB/s) | 20260505 23:04:31.595 | INFO | Downloaded 34GiB to GPU in 2.11612s (16.0672GiB/s) | 20260505 23:04:36.741 | INFO | Analyzing chunk frames = [7011840,7012352) | 20260505 23:04:36.741 | INFO | MemInfo: { | 20260505 23:04:36.741 | INFO | "gpu" : " 17 GB", | 20260505 23:04:36.741 | INFO | "huge" : "", | 20260505 23:04:36.741 | INFO | "huge_shared" : " 27 GB", | 20260505 23:04:36.741 | INFO | "malloc" : " 14 GB", | 20260505 23:04:36.741 | INFO | "pinned" : " 29 GB" | 20260505 23:04:36.741 | INFO | } | 20260505 23:04:36.741 | INFO | Perf report for the last 5.14685 seconds: | 20260505 23:04:36.741 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:04:36.741 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:36.741 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:36.741 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:36.741 | INFO | ANALYSIS : 62.0%, 3.0, 97.0ms, 0 | 20260505 23:04:36.741 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:36.741 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:36.741 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 23:04:36.741 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:36.741 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:36.742 | INFO | Uploaded 36GiB to GPU in 2.10562s (17.0971GiB/s) | 20260505 23:04:36.742 | INFO | Downloaded 34GiB to GPU in 2.00755s (16.9361GiB/s) | 20260505 23:04:36.742 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 23:04:36.742 | INFO | Total mem usage: 15.3 GiB | 20260505 23:04:36.742 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 23:04:36.742 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -9.06%) | 20260505 23:04:36.742 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 23:04:36.742 | INFO | High Water Breakdown: | 20260505 23:04:36.742 | INFO | 60.9% : 8.56 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 23:04:36.742 | INFO | 39% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 23:04:36.742 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 23:04:36.742 | INFO | ------------------------------------------ | 20260505 23:04:36.742 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:04:36.742 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 23:04:36.742 | INFO | Total mem usage: 26.5 GiB | 20260505 23:04:36.742 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 23:04:36.742 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 23:04:36.742 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 23:04:36.742 | INFO | High Water Breakdown: | 20260505 23:04:36.742 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:04:36.742 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:04:36.742 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:04:36.742 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:04:36.742 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:04:36.742 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:04:36.742 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 23:04:36.742 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 23:04:36.742 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:04:36.742 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 23:04:36.742 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 23:04:36.742 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 23:04:36.742 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 23:04:36.742 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 23:04:36.742 | INFO | ------------------------------------------ | 20260505 23:04:36.742 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:04:36.742 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 23:04:36.742 | INFO | Total mem usage: 25.6 GiB | 20260505 23:04:36.742 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 23:04:36.742 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 23:04:36.742 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 23:04:36.742 | INFO | High Water Breakdown: | 20260505 23:04:36.742 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 23:04:36.742 | INFO | ------------------------------------------ | 20260505 23:04:36.742 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:04:36.742 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 23:04:36.742 | INFO | Total mem usage: 15.7 GiB | 20260505 23:04:36.742 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 23:04:36.742 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 23:04:36.742 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 23:04:36.742 | INFO | High Water Breakdown: | 20260505 23:04:36.742 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 23:04:36.742 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 23:04:36.742 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 23:04:36.742 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 23:04:36.742 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 23:04:36.742 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 23:04:36.742 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 23:04:36.742 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 23:04:36.742 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 23:04:36.743 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:04:36.743 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 23:04:36.743 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:04:36.743 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 23:04:36.743 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 23:04:36.743 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 23:04:36.743 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 23:04:36.743 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:04:36.743 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 23:04:36.743 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:04:36.743 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:04:36.743 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:04:36.743 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 23:04:36.743 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 23:04:36.743 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:04:36.743 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 23:04:36.743 | INFO | ------------------------------------------ | 20260505 23:04:36.743 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:04:36.743 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 23:04:36.743 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:04:36.743 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:04:36.743 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.39e+01, 1.34e+01 / 7.63e-01 | 20260505 23:04:36.743 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.04e-01, 1.05e-02 / 5.29e-03 | 20260505 23:04:36.743 | INFO | PerfReport: 11.82%, Baseline -- 3.70e+00 / 4.54e+00, 5.70e-01 / 1.41e-01 | 20260505 23:04:36.743 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.53e+00, 1.88e-01 / 8.49e-02 | 20260505 23:04:36.743 | INFO | PerfReport: 15.95%, DME -- 4.00e+01 / 4.37e+01, 3.64e+00 / 6.85e-01 | 20260505 23:04:36.743 | INFO | PerfReport: 57.33%, FrameLabeling -- 1.80e+01 / 1.96e+01, 1.52e+00 / 1.21e+00 | 20260505 23:04:36.743 | INFO | PerfReport: 7.14%, PulseAccumulating -- 2.24e+00 / 2.76e+00, 4.28e-01 / 2.25e-01 | 20260505 23:04:36.743 | INFO | PerfReport: 3.12%, Metrics -- 9.80e-01 / 1.25e+00, 2.63e-01 / 1.01e-01 | 20260505 23:04:36.743 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.40e-01 / 3.96e-01, 5.85e-02 / 3.48e-02 | 20260505 23:04:36.743 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 23:04:36.743 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:04:36.743 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:04:36.743 | INFO | PerfReport: Full Execution -- 9.86e+01 / -nan, 2.49e+01 / 0.00e+00 | 20260505 23:04:36.743 | INFO | PerfReport: 22.37%, Upload -- 2.21e+01 / 0.00e+00, 7.61e+00 / 0.00e+00 | 20260505 23:04:36.743 | INFO | PerfReport: 21.82%, Download -- 2.15e+01 / 0.00e+00, 7.63e+00 / 0.00e+00 | 20260505 23:04:36.743 | INFO | PerfReport: 31.79%, Compute -- 3.14e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 23:04:36.743 | INFO | PerfReport: 2.20%, LockUpload -- 2.17e+00 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260505 23:04:36.743 | INFO | PerfReport: 1.33%, LockDownload -- 1.31e+00 / 0.00e+00, 2.86e+00 / 0.00e+00 | 20260505 23:04:36.743 | INFO | PerfReport: 20.47%, LockCompute -- 2.02e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260505 23:04:36.743 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 2.21e-02 / 0.00e+00 | 20260505 23:04:36.860 | INFO | Starting a round of Baz IO | 20260505 23:04:40.006 | INFO | Populating RTMetrics report | 20260505 23:04:40.006 | INFO | RTMetrics report populated | 20260505 23:04:40.006 | INFO | Opening tmp RTMetrics json file | 20260505 23:04:40.008 | INFO | Serializing RTMetrics to json | 20260505 23:04:40.010 | INFO | Serialization done, swapping tmp file now... | 20260505 23:04:40.010 | INFO | RTMetrics done | 20260505 23:04:40.085 | INFO | Finished a round of Baz IO: Wrote 1.909 GiB in 3.224 seconds | 20260505 23:04:40.085 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.45% / 0.00% / 78.55% | 20260505 23:04:40.085 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.112911% / 3.97364e-06% / 0.000119209% | 20260505 23:04:40.085 | INFO | Baz IO overall: | 20260505 23:04:40.085 | INFO | BytesWritten : 4020466321425 | 20260505 23:04:40.085 | INFO | NumEvents : 1894976836745 | 20260505 23:04:41.879 | INFO | Analyzing chunk frames = [7012352,7012864) | 20260505 23:04:41.879 | INFO | MemInfo: { | 20260505 23:04:41.879 | INFO | "gpu" : " 17 GB", | 20260505 23:04:41.879 | INFO | "huge" : "", | 20260505 23:04:41.879 | INFO | "huge_shared" : " 27 GB", | 20260505 23:04:41.879 | INFO | "malloc" : " 14 GB", | 20260505 23:04:41.879 | INFO | "pinned" : " 29 GB" | 20260505 23:04:41.879 | INFO | } | 20260505 23:04:41.879 | INFO | Perf report for the last 5.13775 seconds: | 20260505 23:04:41.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:04:41.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:41.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:41.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:41.879 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260505 23:04:41.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:41.879 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260505 23:04:41.879 | INFO | PRE_HQ : 59.8%, 1.7, 54.3ms, 0 | 20260505 23:04:41.879 | INFO | BAZWRITER : 61.9%, 3.6, 113.3ms, 0 | 20260505 23:04:41.879 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 23:04:41.879 | INFO | Uploaded 36GiB to GPU in 2.1757s (16.5464GiB/s) | 20260505 23:04:41.880 | INFO | Downloaded 38GiB to GPU in 2.24542s (16.9233GiB/s) | 20260505 23:04:47.019 | INFO | Analyzing chunk frames = [7012864,7013376) | 20260505 23:04:47.019 | INFO | MemInfo: { | 20260505 23:04:47.019 | INFO | "gpu" : " 17 GB", | 20260505 23:04:47.019 | INFO | "huge" : "", | 20260505 23:04:47.019 | INFO | "huge_shared" : " 27 GB", | 20260505 23:04:47.019 | INFO | "malloc" : " 14 GB", | 20260505 23:04:47.019 | INFO | "pinned" : " 29 GB" | 20260505 23:04:47.019 | INFO | } | 20260505 23:04:47.019 | INFO | Perf report for the last 5.14029 seconds: | 20260505 23:04:47.019 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:04:47.019 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:47.019 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:47.019 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:47.019 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260505 23:04:47.019 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:47.019 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:47.020 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:04:47.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:47.020 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:47.020 | INFO | Uploaded 36GiB to GPU in 2.09513s (17.1827GiB/s) | 20260505 23:04:47.020 | INFO | Downloaded 34GiB to GPU in 2.04117s (16.6571GiB/s) | 20260505 23:04:52.150 | INFO | Analyzing chunk frames = [7013376,7013888) | 20260505 23:04:52.150 | INFO | MemInfo: { | 20260505 23:04:52.150 | INFO | "gpu" : " 17 GB", | 20260505 23:04:52.150 | INFO | "huge" : "", | 20260505 23:04:52.150 | INFO | "huge_shared" : " 27 GB", | 20260505 23:04:52.150 | INFO | "malloc" : " 14 GB", | 20260505 23:04:52.150 | INFO | "pinned" : " 29 GB" | 20260505 23:04:52.150 | INFO | } | 20260505 23:04:52.150 | INFO | Perf report for the last 5.13043 seconds: | 20260505 23:04:52.150 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:04:52.150 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:52.150 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:52.150 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:52.150 | INFO | ANALYSIS : 62.0%, 3.0, 96.2ms, 0 | 20260505 23:04:52.150 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:52.150 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:52.150 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 23:04:52.150 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:52.150 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:52.150 | INFO | Uploaded 36GiB to GPU in 2.15004s (16.7438GiB/s) | 20260505 23:04:52.150 | INFO | Downloaded 34GiB to GPU in 2.05399s (16.5531GiB/s) | 20260505 23:04:57.292 | INFO | Analyzing chunk frames = [7013888,7014400) | 20260505 23:04:57.292 | INFO | MemInfo: { | 20260505 23:04:57.292 | INFO | "gpu" : " 17 GB", | 20260505 23:04:57.292 | INFO | "huge" : "", | 20260505 23:04:57.292 | INFO | "huge_shared" : " 27 GB", | 20260505 23:04:57.293 | INFO | "malloc" : " 14 GB", | 20260505 23:04:57.293 | INFO | "pinned" : " 29 GB" | 20260505 23:04:57.293 | INFO | } | 20260505 23:04:57.293 | INFO | Perf report for the last 5.14277 seconds: | 20260505 23:04:57.293 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:04:57.293 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:57.293 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:57.293 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:57.293 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260505 23:04:57.293 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:57.293 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:57.293 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 23:04:57.293 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:57.293 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:04:57.293 | INFO | Uploaded 36GiB to GPU in 2.20987s (16.2906GiB/s) | 20260505 23:04:57.293 | INFO | Downloaded 34GiB to GPU in 2.0696s (16.4283GiB/s) | 20260505 23:05:02.445 | INFO | Analyzing chunk frames = [7014400,7014912) | 20260505 23:05:02.445 | INFO | MemInfo: { | 20260505 23:05:02.445 | INFO | "gpu" : " 17 GB", | 20260505 23:05:02.445 | INFO | "huge" : "", | 20260505 23:05:02.445 | INFO | "huge_shared" : " 27 GB", | 20260505 23:05:02.445 | INFO | "malloc" : " 14 GB", | 20260505 23:05:02.445 | INFO | "pinned" : " 29 GB" | 20260505 23:05:02.445 | INFO | } | 20260505 23:05:02.445 | INFO | Perf report for the last 5.15211 seconds: | 20260505 23:05:02.445 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:05:02.445 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:02.445 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:02.445 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:02.445 | INFO | ANALYSIS : 61.1%, 3.0, 95.7ms, 0 | 20260505 23:05:02.445 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:02.445 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:02.445 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 23:05:02.445 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:02.445 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:02.445 | INFO | Uploaded 36GiB to GPU in 2.09551s (17.1796GiB/s) | 20260505 23:05:02.445 | INFO | Downloaded 34GiB to GPU in 2.05359s (16.5564GiB/s) | 20260505 23:05:07.574 | INFO | Analyzing chunk frames = [7014912,7015424) | 20260505 23:05:07.574 | INFO | MemInfo: { | 20260505 23:05:07.574 | INFO | "gpu" : " 17 GB", | 20260505 23:05:07.574 | INFO | "huge" : "", | 20260505 23:05:07.574 | INFO | "huge_shared" : " 27 GB", | 20260505 23:05:07.574 | INFO | "malloc" : " 14 GB", | 20260505 23:05:07.575 | INFO | "pinned" : " 29 GB" | 20260505 23:05:07.575 | INFO | } | 20260505 23:05:07.575 | INFO | Perf report for the last 5.12992 seconds: | 20260505 23:05:07.575 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:05:07.575 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:07.575 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:07.575 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:07.575 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260505 23:05:07.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:07.575 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:07.575 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 23:05:07.575 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:07.575 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:07.575 | INFO | Uploaded 36GiB to GPU in 2.06896s (17.4GiB/s) | 20260505 23:05:07.575 | INFO | Downloaded 34GiB to GPU in 2.03607s (16.6988GiB/s) | 20260505 23:05:12.712 | INFO | Analyzing chunk frames = [7015424,7015936) | 20260505 23:05:12.712 | INFO | MemInfo: { | 20260505 23:05:12.712 | INFO | "gpu" : " 17 GB", | 20260505 23:05:12.712 | INFO | "huge" : "", | 20260505 23:05:12.712 | INFO | "huge_shared" : " 27 GB", | 20260505 23:05:12.712 | INFO | "malloc" : " 14 GB", | 20260505 23:05:12.712 | INFO | "pinned" : " 29 GB" | 20260505 23:05:12.712 | INFO | } | 20260505 23:05:12.712 | INFO | Perf report for the last 5.13706 seconds: | 20260505 23:05:12.712 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:05:12.712 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:12.712 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:12.712 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:12.712 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 23:05:12.712 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:12.712 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:12.712 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 23:05:12.712 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:12.712 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:12.712 | INFO | Uploaded 36GiB to GPU in 2.01728s (17.8458GiB/s) | 20260505 23:05:12.712 | INFO | Downloaded 34GiB to GPU in 2.03766s (16.6858GiB/s) | 20260505 23:05:17.850 | INFO | Analyzing chunk frames = [7015936,7016448) | 20260505 23:05:17.850 | INFO | MemInfo: { | 20260505 23:05:17.850 | INFO | "gpu" : " 17 GB", | 20260505 23:05:17.850 | INFO | "huge" : "", | 20260505 23:05:17.850 | INFO | "huge_shared" : " 27 GB", | 20260505 23:05:17.851 | INFO | "malloc" : " 14 GB", | 20260505 23:05:17.851 | INFO | "pinned" : " 29 GB" | 20260505 23:05:17.851 | INFO | } | 20260505 23:05:17.851 | INFO | Perf report for the last 5.1389 seconds: | 20260505 23:05:17.851 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:05:17.851 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:17.851 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:17.851 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:17.851 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 23:05:17.851 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:17.851 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:17.851 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 23:05:17.851 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:17.852 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:17.852 | INFO | Uploaded 36GiB to GPU in 2.11459s (17.0245GiB/s) | 20260505 23:05:17.852 | INFO | Downloaded 34GiB to GPU in 2.00572s (16.9515GiB/s) | 20260505 23:05:17.975 | INFO | Starting a round of Baz IO | 20260505 23:05:21.125 | INFO | Populating RTMetrics report | 20260505 23:05:21.125 | INFO | RTMetrics report populated | 20260505 23:05:21.125 | INFO | Opening tmp RTMetrics json file | 20260505 23:05:21.127 | INFO | Serializing RTMetrics to json | 20260505 23:05:21.129 | INFO | Serialization done, swapping tmp file now... | 20260505 23:05:21.132 | INFO | RTMetrics done | 20260505 23:05:21.205 | INFO | Finished a round of Baz IO: Wrote 1.906 GiB in 3.229 seconds | 20260505 23:05:21.205 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.45% / 0.00% / 78.55% | 20260505 23:05:21.205 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.112696% / 0% / 9.53674e-05% | 20260505 23:05:21.205 | INFO | Baz IO overall: | 20260505 23:05:21.205 | INFO | BytesWritten : 4022512653775 | 20260505 23:05:21.205 | INFO | NumEvents : 1895916599241 | 20260505 23:05:22.994 | INFO | Analyzing chunk frames = [7016448,7016960) | 20260505 23:05:22.994 | INFO | MemInfo: { | 20260505 23:05:22.994 | INFO | "gpu" : " 17 GB", | 20260505 23:05:22.994 | INFO | "huge" : "", | 20260505 23:05:22.994 | INFO | "huge_shared" : " 27 GB", | 20260505 23:05:22.994 | INFO | "malloc" : " 14 GB", | 20260505 23:05:22.994 | INFO | "pinned" : " 29 GB" | 20260505 23:05:22.994 | INFO | } | 20260505 23:05:22.994 | INFO | Perf report for the last 5.14375 seconds: | 20260505 23:05:22.994 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:05:22.994 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:22.994 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:22.994 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:22.994 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260505 23:05:22.994 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:22.994 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260505 23:05:22.994 | INFO | PRE_HQ : 59.8%, 1.8, 54.8ms, 0 | 20260505 23:05:22.995 | INFO | BAZWRITER : 62.0%, 3.6, 114.7ms, 0 | 20260505 23:05:22.995 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 23:05:22.995 | INFO | Uploaded 36GiB to GPU in 2.13904s (16.83GiB/s) | 20260505 23:05:22.995 | INFO | Downloaded 38GiB to GPU in 2.21908s (17.1242GiB/s) | 20260505 23:05:28.131 | INFO | Analyzing chunk frames = [7016960,7017472) | 20260505 23:05:28.131 | INFO | MemInfo: { | 20260505 23:05:28.131 | INFO | "gpu" : " 17 GB", | 20260505 23:05:28.131 | INFO | "huge" : "", | 20260505 23:05:28.131 | INFO | "huge_shared" : " 27 GB", | 20260505 23:05:28.131 | INFO | "malloc" : " 14 GB", | 20260505 23:05:28.131 | INFO | "pinned" : " 29 GB" | 20260505 23:05:28.131 | INFO | } | 20260505 23:05:28.131 | INFO | Perf report for the last 5.13677 seconds: | 20260505 23:05:28.131 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:05:28.131 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:28.131 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:28.131 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:28.131 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260505 23:05:28.131 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:28.131 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:28.131 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:05:28.131 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:28.131 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:28.131 | INFO | Uploaded 36GiB to GPU in 2.11105s (17.0531GiB/s) | 20260505 23:05:28.132 | INFO | Downloaded 34GiB to GPU in 2.03905s (16.6745GiB/s) | 20260505 23:05:33.270 | INFO | Analyzing chunk frames = [7017472,7017984) | 20260505 23:05:33.270 | INFO | MemInfo: { | 20260505 23:05:33.270 | INFO | "gpu" : " 17 GB", | 20260505 23:05:33.270 | INFO | "huge" : "", | 20260505 23:05:33.270 | INFO | "huge_shared" : " 27 GB", | 20260505 23:05:33.270 | INFO | "malloc" : " 14 GB", | 20260505 23:05:33.270 | INFO | "pinned" : " 29 GB" | 20260505 23:05:33.270 | INFO | } | 20260505 23:05:33.270 | INFO | Perf report for the last 5.13884 seconds: | 20260505 23:05:33.270 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:05:33.270 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:33.270 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:33.270 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:33.270 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260505 23:05:33.270 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:33.270 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:33.270 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:05:33.270 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:33.270 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:33.270 | INFO | Uploaded 36GiB to GPU in 2.16184s (16.6525GiB/s) | 20260505 23:05:33.270 | INFO | Downloaded 34GiB to GPU in 2.05456s (16.5486GiB/s) | 20260505 23:05:38.409 | INFO | Analyzing chunk frames = [7017984,7018496) | 20260505 23:05:38.409 | INFO | MemInfo: { | 20260505 23:05:38.409 | INFO | "gpu" : " 17 GB", | 20260505 23:05:38.409 | INFO | "huge" : "", | 20260505 23:05:38.409 | INFO | "huge_shared" : " 27 GB", | 20260505 23:05:38.409 | INFO | "malloc" : " 14 GB", | 20260505 23:05:38.409 | INFO | "pinned" : " 29 GB" | 20260505 23:05:38.409 | INFO | } | 20260505 23:05:38.409 | INFO | Perf report for the last 5.13872 seconds: | 20260505 23:05:38.409 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:05:38.409 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:38.409 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:38.409 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:38.409 | INFO | ANALYSIS : 61.7%, 3.0, 95.8ms, 0 | 20260505 23:05:38.409 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:38.409 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:38.409 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:05:38.409 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:38.409 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:38.409 | INFO | Uploaded 36GiB to GPU in 2.12035s (16.9783GiB/s) | 20260505 23:05:38.409 | INFO | Downloaded 34GiB to GPU in 2.02108s (16.8227GiB/s) | 20260505 23:05:43.547 | INFO | Analyzing chunk frames = [7018496,7019008) | 20260505 23:05:43.547 | INFO | MemInfo: { | 20260505 23:05:43.547 | INFO | "gpu" : " 17 GB", | 20260505 23:05:43.547 | INFO | "huge" : "", | 20260505 23:05:43.547 | INFO | "huge_shared" : " 27 GB", | 20260505 23:05:43.547 | INFO | "malloc" : " 14 GB", | 20260505 23:05:43.547 | INFO | "pinned" : " 29 GB" | 20260505 23:05:43.547 | INFO | } | 20260505 23:05:43.547 | INFO | Perf report for the last 5.13845 seconds: | 20260505 23:05:43.547 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:05:43.547 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:43.547 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:43.547 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:43.547 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260505 23:05:43.547 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:43.547 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:43.547 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 23:05:43.547 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:43.547 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:43.548 | INFO | Uploaded 36GiB to GPU in 2.13145s (16.8899GiB/s) | 20260505 23:05:43.548 | INFO | Downloaded 34GiB to GPU in 2.06357s (16.4763GiB/s) | 20260505 23:05:48.696 | INFO | Analyzing chunk frames = [7019008,7019520) | 20260505 23:05:48.696 | INFO | MemInfo: { | 20260505 23:05:48.696 | INFO | "gpu" : " 17 GB", | 20260505 23:05:48.696 | INFO | "huge" : "", | 20260505 23:05:48.696 | INFO | "huge_shared" : " 27 GB", | 20260505 23:05:48.696 | INFO | "malloc" : " 14 GB", | 20260505 23:05:48.696 | INFO | "pinned" : " 29 GB" | 20260505 23:05:48.696 | INFO | } | 20260505 23:05:48.696 | INFO | Perf report for the last 5.14893 seconds: | 20260505 23:05:48.696 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:05:48.696 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:48.696 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:48.696 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:48.696 | INFO | ANALYSIS : 61.3%, 3.0, 95.9ms, 0 | 20260505 23:05:48.696 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:48.696 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:48.696 | INFO | PRE_HQ : 27.2%, 1.0, 14.2ms, 0 | 20260505 23:05:48.696 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:48.696 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:48.697 | INFO | Uploaded 36GiB to GPU in 2.07116s (17.3815GiB/s) | 20260505 23:05:48.697 | INFO | Downloaded 34GiB to GPU in 2.03844s (16.6794GiB/s) | 20260505 23:05:53.836 | INFO | Analyzing chunk frames = [7019520,7020032) | 20260505 23:05:53.836 | INFO | MemInfo: { | 20260505 23:05:53.836 | INFO | "gpu" : " 17 GB", | 20260505 23:05:53.836 | INFO | "huge" : "", | 20260505 23:05:53.836 | INFO | "huge_shared" : " 27 GB", | 20260505 23:05:53.836 | INFO | "malloc" : " 14 GB", | 20260505 23:05:53.836 | INFO | "pinned" : " 29 GB" | 20260505 23:05:53.836 | INFO | } | 20260505 23:05:53.836 | INFO | Perf report for the last 5.13976 seconds: | 20260505 23:05:53.836 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:05:53.836 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:53.836 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:53.836 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:53.836 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 23:05:53.836 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:53.836 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:53.836 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 23:05:53.836 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:53.836 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:53.836 | INFO | Uploaded 36GiB to GPU in 2.01295s (17.8842GiB/s) | 20260505 23:05:53.836 | INFO | Downloaded 34GiB to GPU in 2.05144s (16.5737GiB/s) | 20260505 23:05:58.964 | INFO | Analyzing chunk frames = [7020032,7020544) | 20260505 23:05:58.964 | INFO | MemInfo: { | 20260505 23:05:58.964 | INFO | "gpu" : " 17 GB", | 20260505 23:05:58.964 | INFO | "huge" : "", | 20260505 23:05:58.964 | INFO | "huge_shared" : " 27 GB", | 20260505 23:05:58.964 | INFO | "malloc" : " 14 GB", | 20260505 23:05:58.964 | INFO | "pinned" : " 29 GB" | 20260505 23:05:58.964 | INFO | } | 20260505 23:05:58.964 | INFO | Perf report for the last 5.12839 seconds: | 20260505 23:05:58.965 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:05:58.965 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:58.965 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:58.965 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:58.965 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260505 23:05:58.965 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:58.965 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:58.965 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 23:05:58.965 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:58.965 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:05:58.965 | INFO | Uploaded 36GiB to GPU in 2.13211s (16.8847GiB/s) | 20260505 23:05:58.965 | INFO | Downloaded 34GiB to GPU in 2.02491s (16.7909GiB/s) | 20260505 23:05:58.965 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 23:05:58.965 | INFO | Total mem usage: 15.3 GiB | 20260505 23:05:58.965 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 23:05:58.965 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -9.08%) | 20260505 23:05:58.965 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 23:05:58.965 | INFO | High Water Breakdown: | 20260505 23:05:58.965 | INFO | 60.9% : 8.56 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 23:05:58.965 | INFO | 39% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 23:05:58.965 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 23:05:58.965 | INFO | ------------------------------------------ | 20260505 23:05:58.965 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:05:58.965 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 23:05:58.965 | INFO | Total mem usage: 26.5 GiB | 20260505 23:05:58.965 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 23:05:58.965 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 23:05:58.965 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 23:05:58.965 | INFO | High Water Breakdown: | 20260505 23:05:58.965 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:05:58.965 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:05:58.965 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:05:58.965 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:05:58.965 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:05:58.965 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:05:58.965 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 23:05:58.965 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 23:05:58.965 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:05:58.965 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 23:05:58.965 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 23:05:58.965 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 23:05:58.965 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 23:05:58.965 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 23:05:58.965 | INFO | ------------------------------------------ | 20260505 23:05:58.965 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:05:58.965 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 23:05:58.965 | INFO | Total mem usage: 25.6 GiB | 20260505 23:05:58.965 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 23:05:58.965 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 23:05:58.965 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 23:05:58.965 | INFO | High Water Breakdown: | 20260505 23:05:58.965 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 23:05:58.965 | INFO | ------------------------------------------ | 20260505 23:05:58.965 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:05:58.965 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 23:05:58.965 | INFO | Total mem usage: 15.7 GiB | 20260505 23:05:58.965 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 23:05:58.965 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 23:05:58.965 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 23:05:58.965 | INFO | High Water Breakdown: | 20260505 23:05:58.965 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 23:05:58.965 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 23:05:58.965 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 23:05:58.965 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 23:05:58.965 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 23:05:58.965 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 23:05:58.965 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 23:05:58.965 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 23:05:58.965 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 23:05:58.965 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:05:58.965 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 23:05:58.965 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:05:58.965 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 23:05:58.965 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 23:05:58.966 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 23:05:58.966 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 23:05:58.966 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:05:58.966 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 23:05:58.966 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:05:58.966 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:05:58.966 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:05:58.966 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 23:05:58.966 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 23:05:58.966 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:05:58.966 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 23:05:58.966 | INFO | ------------------------------------------ | 20260505 23:05:58.966 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:05:58.966 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 23:05:58.966 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:05:58.966 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:05:58.966 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.40e+01, 1.33e+01 / 8.00e-01 | 20260505 23:05:58.966 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.05e-01, 1.09e-02 / 5.39e-03 | 20260505 23:05:58.966 | INFO | PerfReport: 11.80%, Baseline -- 3.69e+00 / 4.51e+00, 5.61e-01 / 1.53e-01 | 20260505 23:05:58.966 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.53e+00, 1.86e-01 / 6.76e-02 | 20260505 23:05:58.966 | INFO | PerfReport: 16.02%, DME -- 4.01e+01 / 4.41e+01, 3.77e+00 / 8.61e-01 | 20260505 23:05:58.966 | INFO | PerfReport: 57.23%, FrameLabeling -- 1.79e+01 / 1.93e+01, 1.43e+00 / 8.71e-01 | 20260505 23:05:58.966 | INFO | PerfReport: 7.17%, PulseAccumulating -- 2.24e+00 / 2.80e+00, 4.28e-01 / 1.78e-01 | 20260505 23:05:58.966 | INFO | PerfReport: 3.14%, Metrics -- 9.82e-01 / 1.27e+00, 2.66e-01 / 1.27e-01 | 20260505 23:05:58.966 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.41e-01 / 4.02e-01, 5.86e-02 / 3.39e-02 | 20260505 23:05:58.966 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 23:05:58.966 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:05:58.966 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:05:58.966 | INFO | PerfReport: Full Execution -- 9.85e+01 / -nan, 2.48e+01 / 0.00e+00 | 20260505 23:05:58.966 | INFO | PerfReport: 22.34%, Upload -- 2.20e+01 / 0.00e+00, 7.51e+00 / 0.00e+00 | 20260505 23:05:58.966 | INFO | PerfReport: 21.84%, Download -- 2.15e+01 / 0.00e+00, 7.56e+00 / 0.00e+00 | 20260505 23:05:58.966 | INFO | PerfReport: 31.78%, Compute -- 3.13e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260505 23:05:58.966 | INFO | PerfReport: 2.15%, LockUpload -- 2.12e+00 / 0.00e+00, 7.83e+00 / 0.00e+00 | 20260505 23:05:58.966 | INFO | PerfReport: 1.33%, LockDownload -- 1.31e+00 / 0.00e+00, 2.92e+00 / 0.00e+00 | 20260505 23:05:58.966 | INFO | PerfReport: 20.56%, LockCompute -- 2.03e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260505 23:05:58.966 | INFO | PerfReport: 0.01%, Misc -- 1.27e-02 / 0.00e+00, 1.32e-02 / 0.00e+00 | 20260505 23:05:59.099 | INFO | Starting a round of Baz IO | 20260505 23:06:02.245 | INFO | Populating RTMetrics report | 20260505 23:06:02.246 | INFO | RTMetrics report populated | 20260505 23:06:02.246 | INFO | Opening tmp RTMetrics json file | 20260505 23:06:02.246 | INFO | Serializing RTMetrics to json | 20260505 23:06:02.248 | INFO | Serialization done, swapping tmp file now... | 20260505 23:06:02.248 | INFO | RTMetrics done | 20260505 23:06:02.326 | INFO | Finished a round of Baz IO: Wrote 1.906 GiB in 3.227 seconds | 20260505 23:06:02.327 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.45% / 0.00% / 78.55% | 20260505 23:06:02.327 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.112335% / 0% / 8.34465e-05% | 20260505 23:06:02.327 | INFO | Baz IO overall: | 20260505 23:06:02.327 | INFO | BytesWritten : 4024559675368 | 20260505 23:06:02.327 | INFO | NumEvents : 1896856804028 | 20260505 23:06:04.098 | INFO | Analyzing chunk frames = [7020544,7021056) | 20260505 23:06:04.099 | INFO | MemInfo: { | 20260505 23:06:04.099 | INFO | "gpu" : " 17 GB", | 20260505 23:06:04.099 | INFO | "huge" : "", | 20260505 23:06:04.099 | INFO | "huge_shared" : " 27 GB", | 20260505 23:06:04.099 | INFO | "malloc" : " 14 GB", | 20260505 23:06:04.099 | INFO | "pinned" : " 29 GB" | 20260505 23:06:04.099 | INFO | } | 20260505 23:06:04.099 | INFO | Perf report for the last 5.13414 seconds: | 20260505 23:06:04.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:06:04.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:04.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:04.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:04.099 | INFO | ANALYSIS : 62.9%, 3.0, 98.0ms, 0 | 20260505 23:06:04.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:04.099 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260505 23:06:04.099 | INFO | PRE_HQ : 59.8%, 1.8, 54.5ms, 0 | 20260505 23:06:04.099 | INFO | BAZWRITER : 62.3%, 3.6, 113.8ms, 0 | 20260505 23:06:04.099 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 23:06:04.099 | INFO | Uploaded 36GiB to GPU in 2.16869s (16.5999GiB/s) | 20260505 23:06:04.099 | INFO | Downloaded 38GiB to GPU in 2.24652s (16.915GiB/s) | 20260505 23:06:09.248 | INFO | Analyzing chunk frames = [7021056,7021568) | 20260505 23:06:09.248 | INFO | MemInfo: { | 20260505 23:06:09.248 | INFO | "gpu" : " 17 GB", | 20260505 23:06:09.248 | INFO | "huge" : "", | 20260505 23:06:09.248 | INFO | "huge_shared" : " 27 GB", | 20260505 23:06:09.248 | INFO | "malloc" : " 14 GB", | 20260505 23:06:09.248 | INFO | "pinned" : " 29 GB" | 20260505 23:06:09.248 | INFO | } | 20260505 23:06:09.248 | INFO | Perf report for the last 5.14976 seconds: | 20260505 23:06:09.248 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:06:09.248 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:09.248 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:09.248 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:09.248 | INFO | ANALYSIS : 61.8%, 3.0, 96.6ms, 0 | 20260505 23:06:09.248 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:09.248 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:09.248 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260505 23:06:09.248 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:09.248 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:09.249 | INFO | Uploaded 36GiB to GPU in 2.07704s (17.3323GiB/s) | 20260505 23:06:09.249 | INFO | Downloaded 34GiB to GPU in 2.02239s (16.8118GiB/s) | 20260505 23:06:14.398 | INFO | Analyzing chunk frames = [7021568,7022080) | 20260505 23:06:14.398 | INFO | MemInfo: { | 20260505 23:06:14.398 | INFO | "gpu" : " 17 GB", | 20260505 23:06:14.398 | INFO | "huge" : "", | 20260505 23:06:14.398 | INFO | "huge_shared" : " 27 GB", | 20260505 23:06:14.398 | INFO | "malloc" : " 14 GB", | 20260505 23:06:14.398 | INFO | "pinned" : " 29 GB" | 20260505 23:06:14.398 | INFO | } | 20260505 23:06:14.398 | INFO | Perf report for the last 5.14985 seconds: | 20260505 23:06:14.398 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:06:14.398 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:14.398 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:14.398 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:14.398 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260505 23:06:14.398 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:14.398 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:14.398 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260505 23:06:14.398 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:14.398 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:14.399 | INFO | Uploaded 36GiB to GPU in 2.10799s (17.0779GiB/s) | 20260505 23:06:14.399 | INFO | Downloaded 34GiB to GPU in 2.02216s (16.8137GiB/s) | 20260505 23:06:19.526 | INFO | Analyzing chunk frames = [7022080,7022592) | 20260505 23:06:19.526 | INFO | MemInfo: { | 20260505 23:06:19.526 | INFO | "gpu" : " 17 GB", | 20260505 23:06:19.526 | INFO | "huge" : "", | 20260505 23:06:19.526 | INFO | "huge_shared" : " 27 GB", | 20260505 23:06:19.526 | INFO | "malloc" : " 14 GB", | 20260505 23:06:19.526 | INFO | "pinned" : " 29 GB" | 20260505 23:06:19.526 | INFO | } | 20260505 23:06:19.526 | INFO | Perf report for the last 5.12771 seconds: | 20260505 23:06:19.526 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:06:19.526 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:19.526 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:19.526 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:19.526 | INFO | ANALYSIS : 61.8%, 3.0, 95.8ms, 0 | 20260505 23:06:19.526 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:19.526 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:19.526 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260505 23:06:19.526 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:19.526 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:19.526 | INFO | Uploaded 36GiB to GPU in 2.13475s (16.8638GiB/s) | 20260505 23:06:19.526 | INFO | Downloaded 34GiB to GPU in 2.02413s (16.7974GiB/s) | 20260505 23:06:24.663 | INFO | Analyzing chunk frames = [7022592,7023104) | 20260505 23:06:24.663 | INFO | MemInfo: { | 20260505 23:06:24.663 | INFO | "gpu" : " 17 GB", | 20260505 23:06:24.663 | INFO | "huge" : "", | 20260505 23:06:24.663 | INFO | "huge_shared" : " 27 GB", | 20260505 23:06:24.663 | INFO | "malloc" : " 14 GB", | 20260505 23:06:24.663 | INFO | "pinned" : " 29 GB" | 20260505 23:06:24.663 | INFO | } | 20260505 23:06:24.663 | INFO | Perf report for the last 5.1373 seconds: | 20260505 23:06:24.663 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:06:24.663 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:24.663 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:24.663 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:24.663 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260505 23:06:24.663 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:24.663 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:24.663 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 23:06:24.663 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:24.663 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:24.663 | INFO | Uploaded 36GiB to GPU in 2.10353s (17.1141GiB/s) | 20260505 23:06:24.664 | INFO | Downloaded 34GiB to GPU in 2.04501s (16.6258GiB/s) | 20260505 23:06:29.803 | INFO | Analyzing chunk frames = [7023104,7023616) | 20260505 23:06:29.803 | INFO | MemInfo: { | 20260505 23:06:29.803 | INFO | "gpu" : " 17 GB", | 20260505 23:06:29.803 | INFO | "huge" : "", | 20260505 23:06:29.803 | INFO | "huge_shared" : " 27 GB", | 20260505 23:06:29.803 | INFO | "malloc" : " 14 GB", | 20260505 23:06:29.803 | INFO | "pinned" : " 29 GB" | 20260505 23:06:29.803 | INFO | } | 20260505 23:06:29.803 | INFO | Perf report for the last 5.13966 seconds: | 20260505 23:06:29.803 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:06:29.803 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:29.803 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:29.803 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:29.803 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260505 23:06:29.803 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:29.803 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:29.803 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260505 23:06:29.803 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:29.803 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:29.803 | INFO | Uploaded 36GiB to GPU in 2.08454s (17.27GiB/s) | 20260505 23:06:29.803 | INFO | Downloaded 34GiB to GPU in 2.03494s (16.7081GiB/s) | 20260505 23:06:34.943 | INFO | Analyzing chunk frames = [7023616,7024128) | 20260505 23:06:34.943 | INFO | MemInfo: { | 20260505 23:06:34.943 | INFO | "gpu" : " 17 GB", | 20260505 23:06:34.943 | INFO | "huge" : "", | 20260505 23:06:34.943 | INFO | "huge_shared" : " 27 GB", | 20260505 23:06:34.943 | INFO | "malloc" : " 14 GB", | 20260505 23:06:34.943 | INFO | "pinned" : " 29 GB" | 20260505 23:06:34.943 | INFO | } | 20260505 23:06:34.943 | INFO | Perf report for the last 5.14056 seconds: | 20260505 23:06:34.944 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:06:34.944 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:34.944 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:34.944 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:34.944 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260505 23:06:34.944 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:34.944 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:34.944 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 23:06:34.944 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:34.944 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:34.944 | INFO | Uploaded 36GiB to GPU in 2.04064s (17.6415GiB/s) | 20260505 23:06:34.944 | INFO | Downloaded 34GiB to GPU in 2.0546s (16.5482GiB/s) | 20260505 23:06:40.080 | INFO | Analyzing chunk frames = [7024128,7024640) | 20260505 23:06:40.080 | INFO | MemInfo: { | 20260505 23:06:40.080 | INFO | "gpu" : " 17 GB", | 20260505 23:06:40.080 | INFO | "huge" : "", | 20260505 23:06:40.080 | INFO | "huge_shared" : " 27 GB", | 20260505 23:06:40.080 | INFO | "malloc" : " 14 GB", | 20260505 23:06:40.080 | INFO | "pinned" : " 29 GB" | 20260505 23:06:40.080 | INFO | } | 20260505 23:06:40.080 | INFO | Perf report for the last 5.13693 seconds: | 20260505 23:06:40.080 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:06:40.080 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:40.080 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:40.080 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:40.080 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260505 23:06:40.081 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:40.081 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:40.081 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:06:40.081 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:40.081 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:40.081 | INFO | Uploaded 36GiB to GPU in 2.1011s (17.1339GiB/s) | 20260505 23:06:40.081 | INFO | Downloaded 34GiB to GPU in 2.02614s (16.7807GiB/s) | 20260505 23:06:40.209 | INFO | Starting a round of Baz IO | 20260505 23:06:43.353 | INFO | Populating RTMetrics report | 20260505 23:06:43.353 | INFO | RTMetrics report populated | 20260505 23:06:43.353 | INFO | Opening tmp RTMetrics json file | 20260505 23:06:43.355 | INFO | Serializing RTMetrics to json | 20260505 23:06:43.358 | INFO | Serialization done, swapping tmp file now... | 20260505 23:06:43.360 | INFO | RTMetrics done | 20260505 23:06:43.437 | INFO | Finished a round of Baz IO: Wrote 1.906 GiB in 3.228 seconds | 20260505 23:06:43.438 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.45% / 0.00% / 78.55% | 20260505 23:06:43.438 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.112494% / 0% / 8.34465e-05% | 20260505 23:06:43.438 | INFO | Baz IO overall: | 20260505 23:06:43.438 | INFO | BytesWritten : 4026605920114 | 20260505 23:06:43.438 | INFO | NumEvents : 1897796478145 | 20260505 23:06:45.220 | INFO | Analyzing chunk frames = [7024640,7025152) | 20260505 23:06:45.220 | INFO | MemInfo: { | 20260505 23:06:45.220 | INFO | "gpu" : " 17 GB", | 20260505 23:06:45.220 | INFO | "huge" : "", | 20260505 23:06:45.220 | INFO | "huge_shared" : " 27 GB", | 20260505 23:06:45.220 | INFO | "malloc" : " 14 GB", | 20260505 23:06:45.220 | INFO | "pinned" : " 29 GB" | 20260505 23:06:45.220 | INFO | } | 20260505 23:06:45.220 | INFO | Perf report for the last 5.13942 seconds: | 20260505 23:06:45.220 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:06:45.220 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:45.220 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:45.220 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:45.220 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260505 23:06:45.220 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:45.220 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260505 23:06:45.220 | INFO | PRE_HQ : 59.6%, 1.8, 54.2ms, 0 | 20260505 23:06:45.220 | INFO | BAZWRITER : 62.1%, 3.6, 113.3ms, 0 | 20260505 23:06:45.220 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 23:06:45.220 | INFO | Uploaded 36GiB to GPU in 2.13743s (16.8427GiB/s) | 20260505 23:06:45.220 | INFO | Downloaded 38GiB to GPU in 2.22379s (17.0879GiB/s) | 20260505 23:06:50.361 | INFO | Analyzing chunk frames = [7025152,7025664) | 20260505 23:06:50.361 | INFO | MemInfo: { | 20260505 23:06:50.361 | INFO | "gpu" : " 17 GB", | 20260505 23:06:50.361 | INFO | "huge" : "", | 20260505 23:06:50.361 | INFO | "huge_shared" : " 27 GB", | 20260505 23:06:50.361 | INFO | "malloc" : " 14 GB", | 20260505 23:06:50.361 | INFO | "pinned" : " 29 GB" | 20260505 23:06:50.361 | INFO | } | 20260505 23:06:50.361 | INFO | Perf report for the last 5.14135 seconds: | 20260505 23:06:50.361 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:06:50.361 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:50.361 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:50.361 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:50.361 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 23:06:50.361 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:50.361 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:50.361 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:06:50.361 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:50.361 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:50.362 | INFO | Uploaded 36GiB to GPU in 2.08789s (17.2423GiB/s) | 20260505 23:06:50.362 | INFO | Downloaded 34GiB to GPU in 2.02375s (16.8005GiB/s) | 20260505 23:06:55.499 | INFO | Analyzing chunk frames = [7025664,7026176) | 20260505 23:06:55.499 | INFO | MemInfo: { | 20260505 23:06:55.499 | INFO | "gpu" : " 17 GB", | 20260505 23:06:55.499 | INFO | "huge" : "", | 20260505 23:06:55.499 | INFO | "huge_shared" : " 27 GB", | 20260505 23:06:55.499 | INFO | "malloc" : " 14 GB", | 20260505 23:06:55.499 | INFO | "pinned" : " 29 GB" | 20260505 23:06:55.500 | INFO | } | 20260505 23:06:55.500 | INFO | Perf report for the last 5.13833 seconds: | 20260505 23:06:55.500 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:06:55.500 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:55.500 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:55.500 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:55.500 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260505 23:06:55.500 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:55.500 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:55.500 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 23:06:55.500 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:55.500 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:06:55.500 | INFO | Uploaded 36GiB to GPU in 2.09358s (17.1954GiB/s) | 20260505 23:06:55.500 | INFO | Downloaded 34GiB to GPU in 2.02875s (16.7591GiB/s) | 20260505 23:07:00.644 | INFO | Analyzing chunk frames = [7026176,7026688) | 20260505 23:07:00.644 | INFO | MemInfo: { | 20260505 23:07:00.644 | INFO | "gpu" : " 17 GB", | 20260505 23:07:00.644 | INFO | "huge" : "", | 20260505 23:07:00.644 | INFO | "huge_shared" : " 27 GB", | 20260505 23:07:00.644 | INFO | "malloc" : " 14 GB", | 20260505 23:07:00.644 | INFO | "pinned" : " 29 GB" | 20260505 23:07:00.644 | INFO | } | 20260505 23:07:00.644 | INFO | Perf report for the last 5.14453 seconds: | 20260505 23:07:00.644 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:07:00.644 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:00.644 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:00.644 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:00.644 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260505 23:07:00.644 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:00.644 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:00.644 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:07:00.644 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:00.644 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:00.644 | INFO | Uploaded 36GiB to GPU in 2.14454s (16.7869GiB/s) | 20260505 23:07:00.644 | INFO | Downloaded 34GiB to GPU in 2.01224s (16.8966GiB/s) | 20260505 23:07:05.781 | INFO | Analyzing chunk frames = [7026688,7027200) | 20260505 23:07:05.781 | INFO | MemInfo: { | 20260505 23:07:05.781 | INFO | "gpu" : " 17 GB", | 20260505 23:07:05.781 | INFO | "huge" : "", | 20260505 23:07:05.781 | INFO | "huge_shared" : " 27 GB", | 20260505 23:07:05.781 | INFO | "malloc" : " 14 GB", | 20260505 23:07:05.781 | INFO | "pinned" : " 29 GB" | 20260505 23:07:05.781 | INFO | } | 20260505 23:07:05.781 | INFO | Perf report for the last 5.13684 seconds: | 20260505 23:07:05.781 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:07:05.781 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:05.781 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:05.781 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:05.781 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260505 23:07:05.781 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:05.781 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:05.781 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 23:07:05.781 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:05.781 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:05.781 | INFO | Uploaded 36GiB to GPU in 2.09145s (17.2129GiB/s) | 20260505 23:07:05.781 | INFO | Downloaded 34GiB to GPU in 2.04041s (16.6633GiB/s) | 20260505 23:07:10.913 | INFO | Analyzing chunk frames = [7027200,7027712) | 20260505 23:07:10.913 | INFO | MemInfo: { | 20260505 23:07:10.913 | INFO | "gpu" : " 17 GB", | 20260505 23:07:10.913 | INFO | "huge" : "", | 20260505 23:07:10.913 | INFO | "huge_shared" : " 27 GB", | 20260505 23:07:10.913 | INFO | "malloc" : " 14 GB", | 20260505 23:07:10.913 | INFO | "pinned" : " 29 GB" | 20260505 23:07:10.913 | INFO | } | 20260505 23:07:10.913 | INFO | Perf report for the last 5.1324 seconds: | 20260505 23:07:10.913 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:07:10.913 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:10.913 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:10.913 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:10.913 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260505 23:07:10.913 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:10.913 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:10.913 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260505 23:07:10.914 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:10.914 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:10.914 | INFO | Uploaded 36GiB to GPU in 2.07492s (17.3501GiB/s) | 20260505 23:07:10.914 | INFO | Downloaded 34GiB to GPU in 2.03512s (16.7066GiB/s) | 20260505 23:07:16.055 | INFO | Analyzing chunk frames = [7027712,7028224) | 20260505 23:07:16.055 | INFO | MemInfo: { | 20260505 23:07:16.055 | INFO | "gpu" : " 17 GB", | 20260505 23:07:16.055 | INFO | "huge" : "", | 20260505 23:07:16.055 | INFO | "huge_shared" : " 27 GB", | 20260505 23:07:16.055 | INFO | "malloc" : " 14 GB", | 20260505 23:07:16.055 | INFO | "pinned" : " 29 GB" | 20260505 23:07:16.055 | INFO | } | 20260505 23:07:16.055 | INFO | Perf report for the last 5.14141 seconds: | 20260505 23:07:16.055 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:07:16.055 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:16.055 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:16.055 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:16.055 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 23:07:16.055 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:16.055 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:16.055 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 23:07:16.055 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:16.055 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:16.055 | INFO | Uploaded 36GiB to GPU in 2.16052s (16.6626GiB/s) | 20260505 23:07:16.055 | INFO | Downloaded 34GiB to GPU in 2.11625s (16.0662GiB/s) | 20260505 23:07:21.196 | INFO | Analyzing chunk frames = [7028224,7028736) | 20260505 23:07:21.196 | INFO | MemInfo: { | 20260505 23:07:21.196 | INFO | "gpu" : " 17 GB", | 20260505 23:07:21.196 | INFO | "huge" : "", | 20260505 23:07:21.196 | INFO | "huge_shared" : " 27 GB", | 20260505 23:07:21.196 | INFO | "malloc" : " 14 GB", | 20260505 23:07:21.196 | INFO | "pinned" : " 29 GB" | 20260505 23:07:21.196 | INFO | } | 20260505 23:07:21.196 | INFO | Perf report for the last 5.14163 seconds: | 20260505 23:07:21.196 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:07:21.196 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:21.196 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:21.197 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:21.197 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 23:07:21.197 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:21.197 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:21.197 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:07:21.197 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:21.197 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:21.197 | INFO | Uploaded 36GiB to GPU in 2.11762s (17.0002GiB/s) | 20260505 23:07:21.197 | INFO | Downloaded 34GiB to GPU in 1.99453s (17.0466GiB/s) | 20260505 23:07:21.197 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 23:07:21.197 | INFO | Total mem usage: 15.3 GiB | 20260505 23:07:21.197 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 23:07:21.197 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -9.08%) | 20260505 23:07:21.197 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 23:07:21.197 | INFO | High Water Breakdown: | 20260505 23:07:21.197 | INFO | 60.9% : 8.56 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 23:07:21.197 | INFO | 39% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 23:07:21.197 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 23:07:21.197 | INFO | ------------------------------------------ | 20260505 23:07:21.197 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:07:21.197 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 23:07:21.197 | INFO | Total mem usage: 26.5 GiB | 20260505 23:07:21.197 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 23:07:21.197 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 23:07:21.197 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 23:07:21.197 | INFO | High Water Breakdown: | 20260505 23:07:21.197 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:07:21.197 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:07:21.197 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:07:21.197 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:07:21.197 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:07:21.197 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:07:21.197 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 23:07:21.197 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 23:07:21.197 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:07:21.197 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 23:07:21.197 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 23:07:21.197 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 23:07:21.197 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 23:07:21.197 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 23:07:21.197 | INFO | ------------------------------------------ | 20260505 23:07:21.197 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:07:21.197 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 23:07:21.197 | INFO | Total mem usage: 25.6 GiB | 20260505 23:07:21.197 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 23:07:21.197 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 23:07:21.197 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 23:07:21.197 | INFO | High Water Breakdown: | 20260505 23:07:21.197 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 23:07:21.197 | INFO | ------------------------------------------ | 20260505 23:07:21.197 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:07:21.197 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 23:07:21.197 | INFO | Total mem usage: 15.7 GiB | 20260505 23:07:21.198 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 23:07:21.198 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 23:07:21.198 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 23:07:21.198 | INFO | High Water Breakdown: | 20260505 23:07:21.198 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 23:07:21.198 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 23:07:21.198 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 23:07:21.198 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 23:07:21.198 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 23:07:21.198 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 23:07:21.198 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 23:07:21.198 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 23:07:21.198 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 23:07:21.198 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:07:21.198 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 23:07:21.198 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:07:21.198 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 23:07:21.198 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 23:07:21.198 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 23:07:21.198 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 23:07:21.198 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:07:21.198 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 23:07:21.198 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:07:21.198 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:07:21.198 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:07:21.198 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 23:07:21.198 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 23:07:21.198 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:07:21.198 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 23:07:21.198 | INFO | ------------------------------------------ | 20260505 23:07:21.198 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:07:21.198 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 23:07:21.198 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:07:21.198 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:07:21.198 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.42e+01, 1.34e+01 / 8.15e-01 | 20260505 23:07:21.198 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.03e-01, 1.00e-02 / 3.22e-03 | 20260505 23:07:21.198 | INFO | PerfReport: 11.82%, Baseline -- 3.72e+00 / 4.51e+00, 5.79e-01 / 2.26e-01 | 20260505 23:07:21.198 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.51e+00, 1.93e-01 / 9.37e-02 | 20260505 23:07:21.198 | INFO | PerfReport: 15.94%, DME -- 4.01e+01 / 4.41e+01, 3.89e+00 / 9.11e-01 | 20260505 23:07:21.198 | INFO | PerfReport: 57.41%, FrameLabeling -- 1.81e+01 / 1.96e+01, 1.64e+00 / 1.12e+00 | 20260505 23:07:21.198 | INFO | PerfReport: 7.06%, PulseAccumulating -- 2.22e+00 / 2.78e+00, 4.18e-01 / 1.66e-01 | 20260505 23:07:21.198 | INFO | PerfReport: 3.11%, Metrics -- 9.78e-01 / 1.28e+00, 2.67e-01 / 1.03e-01 | 20260505 23:07:21.198 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.40e-01 / 4.00e-01, 5.95e-02 / 3.41e-02 | 20260505 23:07:21.198 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 23:07:21.198 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:07:21.198 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:07:21.198 | INFO | PerfReport: Full Execution -- 9.85e+01 / -nan, 2.49e+01 / 0.00e+00 | 20260505 23:07:21.198 | INFO | PerfReport: 22.29%, Upload -- 2.20e+01 / 0.00e+00, 7.43e+00 / 0.00e+00 | 20260505 23:07:21.198 | INFO | PerfReport: 21.78%, Download -- 2.15e+01 / 0.00e+00, 7.58e+00 / 0.00e+00 | 20260505 23:07:21.198 | INFO | PerfReport: 31.94%, Compute -- 3.15e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 23:07:21.198 | INFO | PerfReport: 2.18%, LockUpload -- 2.14e+00 / 0.00e+00, 7.92e+00 / 0.00e+00 | 20260505 23:07:21.198 | INFO | PerfReport: 1.24%, LockDownload -- 1.22e+00 / 0.00e+00, 2.86e+00 / 0.00e+00 | 20260505 23:07:21.198 | INFO | PerfReport: 20.57%, LockCompute -- 2.03e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260505 23:07:21.198 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.84e-02 / 0.00e+00 | 20260505 23:07:21.313 | INFO | Starting a round of Baz IO | 20260505 23:07:24.465 | INFO | Populating RTMetrics report | 20260505 23:07:24.465 | INFO | RTMetrics report populated | 20260505 23:07:24.465 | INFO | Opening tmp RTMetrics json file | 20260505 23:07:24.466 | INFO | Serializing RTMetrics to json | 20260505 23:07:24.467 | INFO | Serialization done, swapping tmp file now... | 20260505 23:07:24.467 | INFO | RTMetrics done | 20260505 23:07:24.542 | INFO | Finished a round of Baz IO: Wrote 1.905 GiB in 3.228 seconds | 20260505 23:07:24.542 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.45% / 0.00% / 78.55% | 20260505 23:07:24.542 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.112228% / 0% / 6.75519e-05% | 20260505 23:07:24.542 | INFO | Baz IO overall: | 20260505 23:07:24.542 | INFO | BytesWritten : 4028651115776 | 20260505 23:07:24.542 | INFO | NumEvents : 1898735581076 | 20260505 23:07:26.343 | INFO | Analyzing chunk frames = [7028736,7029248) | 20260505 23:07:26.343 | INFO | MemInfo: { | 20260505 23:07:26.343 | INFO | "gpu" : " 17 GB", | 20260505 23:07:26.343 | INFO | "huge" : "", | 20260505 23:07:26.343 | INFO | "huge_shared" : " 27 GB", | 20260505 23:07:26.343 | INFO | "malloc" : " 14 GB", | 20260505 23:07:26.343 | INFO | "pinned" : " 29 GB" | 20260505 23:07:26.343 | INFO | } | 20260505 23:07:26.343 | INFO | Perf report for the last 5.14636 seconds: | 20260505 23:07:26.343 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:07:26.343 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:26.343 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:26.343 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:26.343 | INFO | ANALYSIS : 62.5%, 3.0, 97.7ms, 0 | 20260505 23:07:26.343 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:26.343 | INFO | RT_METRICS : 43.3%, 1.0, 22.8ms, 0 | 20260505 23:07:26.343 | INFO | PRE_HQ : 59.7%, 1.7, 54.0ms, 0 | 20260505 23:07:26.343 | INFO | BAZWRITER : 61.9%, 3.5, 112.9ms, 0 | 20260505 23:07:26.343 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 23:07:26.343 | INFO | Uploaded 36GiB to GPU in 2.16528s (16.626GiB/s) | 20260505 23:07:26.343 | INFO | Downloaded 38GiB to GPU in 2.23734s (16.9845GiB/s) | 20260505 23:07:31.476 | INFO | Analyzing chunk frames = [7029248,7029760) | 20260505 23:07:31.476 | INFO | MemInfo: { | 20260505 23:07:31.476 | INFO | "gpu" : " 17 GB", | 20260505 23:07:31.476 | INFO | "huge" : "", | 20260505 23:07:31.476 | INFO | "huge_shared" : " 27 GB", | 20260505 23:07:31.476 | INFO | "malloc" : " 14 GB", | 20260505 23:07:31.476 | INFO | "pinned" : " 29 GB" | 20260505 23:07:31.476 | INFO | } | 20260505 23:07:31.476 | INFO | Perf report for the last 5.13357 seconds: | 20260505 23:07:31.476 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:07:31.476 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:31.476 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:31.476 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:31.476 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260505 23:07:31.476 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:31.476 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:31.476 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:07:31.477 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:31.477 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:31.477 | INFO | Uploaded 36GiB to GPU in 2.11175s (17.0475GiB/s) | 20260505 23:07:31.477 | INFO | Downloaded 34GiB to GPU in 2.04351s (16.638GiB/s) | 20260505 23:07:36.612 | INFO | Analyzing chunk frames = [7029760,7030272) | 20260505 23:07:36.612 | INFO | MemInfo: { | 20260505 23:07:36.612 | INFO | "gpu" : " 17 GB", | 20260505 23:07:36.612 | INFO | "huge" : "", | 20260505 23:07:36.612 | INFO | "huge_shared" : " 27 GB", | 20260505 23:07:36.612 | INFO | "malloc" : " 14 GB", | 20260505 23:07:36.612 | INFO | "pinned" : " 29 GB" | 20260505 23:07:36.612 | INFO | } | 20260505 23:07:36.612 | INFO | Perf report for the last 5.13606 seconds: | 20260505 23:07:36.612 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:07:36.612 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:36.612 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:36.613 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:36.613 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260505 23:07:36.613 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:36.613 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:36.613 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:07:36.613 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:36.613 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:36.613 | INFO | Uploaded 36GiB to GPU in 2.15007s (16.7437GiB/s) | 20260505 23:07:36.613 | INFO | Downloaded 34GiB to GPU in 2.05465s (16.5478GiB/s) | 20260505 23:07:41.755 | INFO | Analyzing chunk frames = [7030272,7030784) | 20260505 23:07:41.755 | INFO | MemInfo: { | 20260505 23:07:41.755 | INFO | "gpu" : " 17 GB", | 20260505 23:07:41.755 | INFO | "huge" : "", | 20260505 23:07:41.755 | INFO | "huge_shared" : " 27 GB", | 20260505 23:07:41.755 | INFO | "malloc" : " 14 GB", | 20260505 23:07:41.755 | INFO | "pinned" : " 29 GB" | 20260505 23:07:41.755 | INFO | } | 20260505 23:07:41.755 | INFO | Perf report for the last 5.14264 seconds: | 20260505 23:07:41.755 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:07:41.755 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:41.755 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:41.755 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:41.755 | INFO | ANALYSIS : 61.9%, 3.0, 96.1ms, 0 | 20260505 23:07:41.755 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:41.755 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:41.755 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260505 23:07:41.755 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:41.755 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:41.755 | INFO | Uploaded 36GiB to GPU in 2.13578s (16.8556GiB/s) | 20260505 23:07:41.755 | INFO | Downloaded 34GiB to GPU in 2.04428s (16.6317GiB/s) | 20260505 23:07:46.893 | INFO | Analyzing chunk frames = [7030784,7031296) | 20260505 23:07:46.893 | INFO | MemInfo: { | 20260505 23:07:46.893 | INFO | "gpu" : " 17 GB", | 20260505 23:07:46.893 | INFO | "huge" : "", | 20260505 23:07:46.893 | INFO | "huge_shared" : " 27 GB", | 20260505 23:07:46.893 | INFO | "malloc" : " 14 GB", | 20260505 23:07:46.893 | INFO | "pinned" : " 29 GB" | 20260505 23:07:46.893 | INFO | } | 20260505 23:07:46.893 | INFO | Perf report for the last 5.13817 seconds: | 20260505 23:07:46.893 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:07:46.893 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:46.893 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:46.893 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:46.893 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260505 23:07:46.893 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:46.894 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:46.894 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 23:07:46.894 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:46.894 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:46.894 | INFO | Uploaded 36GiB to GPU in 2.09501s (17.1837GiB/s) | 20260505 23:07:46.894 | INFO | Downloaded 34GiB to GPU in 2.04492s (16.6266GiB/s) | 20260505 23:07:52.031 | INFO | Analyzing chunk frames = [7031296,7031808) | 20260505 23:07:52.031 | INFO | MemInfo: { | 20260505 23:07:52.031 | INFO | "gpu" : " 17 GB", | 20260505 23:07:52.031 | INFO | "huge" : "", | 20260505 23:07:52.031 | INFO | "huge_shared" : " 27 GB", | 20260505 23:07:52.031 | INFO | "malloc" : " 14 GB", | 20260505 23:07:52.031 | INFO | "pinned" : " 29 GB" | 20260505 23:07:52.031 | INFO | } | 20260505 23:07:52.031 | INFO | Perf report for the last 5.13802 seconds: | 20260505 23:07:52.031 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:07:52.031 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:52.031 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:52.031 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:52.031 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260505 23:07:52.031 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:52.031 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:52.031 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260505 23:07:52.031 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:52.031 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:52.032 | INFO | Uploaded 36GiB to GPU in 2.05922s (17.4823GiB/s) | 20260505 23:07:52.032 | INFO | Downloaded 34GiB to GPU in 2.03488s (16.7086GiB/s) | 20260505 23:07:57.169 | INFO | Analyzing chunk frames = [7031808,7032320) | 20260505 23:07:57.169 | INFO | MemInfo: { | 20260505 23:07:57.169 | INFO | "gpu" : " 17 GB", | 20260505 23:07:57.169 | INFO | "huge" : "", | 20260505 23:07:57.169 | INFO | "huge_shared" : " 27 GB", | 20260505 23:07:57.169 | INFO | "malloc" : " 14 GB", | 20260505 23:07:57.169 | INFO | "pinned" : " 29 GB" | 20260505 23:07:57.169 | INFO | } | 20260505 23:07:57.169 | INFO | Perf report for the last 5.13752 seconds: | 20260505 23:07:57.169 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:07:57.169 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:57.169 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:57.169 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:57.169 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 23:07:57.169 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:57.169 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:57.169 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 23:07:57.169 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:57.169 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:07:57.169 | INFO | Uploaded 36GiB to GPU in 2.03171s (17.7191GiB/s) | 20260505 23:07:57.169 | INFO | Downloaded 34GiB to GPU in 2.03844s (16.6794GiB/s) | 20260505 23:08:02.311 | INFO | Analyzing chunk frames = [7032320,7032832) | 20260505 23:08:02.311 | INFO | MemInfo: { | 20260505 23:08:02.311 | INFO | "gpu" : " 17 GB", | 20260505 23:08:02.311 | INFO | "huge" : "", | 20260505 23:08:02.311 | INFO | "huge_shared" : " 27 GB", | 20260505 23:08:02.311 | INFO | "malloc" : " 14 GB", | 20260505 23:08:02.312 | INFO | "pinned" : " 29 GB" | 20260505 23:08:02.312 | INFO | } | 20260505 23:08:02.312 | INFO | Perf report for the last 5.14284 seconds: | 20260505 23:08:02.312 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:08:02.312 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:02.312 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:02.312 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:02.312 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 23:08:02.312 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:02.312 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:02.312 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:08:02.312 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:02.312 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:02.312 | INFO | Uploaded 36GiB to GPU in 2.10915s (17.0685GiB/s) | 20260505 23:08:02.312 | INFO | Downloaded 34GiB to GPU in 2.0058s (16.9509GiB/s) | 20260505 23:08:02.429 | INFO | Starting a round of Baz IO | 20260505 23:08:05.577 | INFO | Populating RTMetrics report | 20260505 23:08:05.577 | INFO | RTMetrics report populated | 20260505 23:08:05.577 | INFO | Opening tmp RTMetrics json file | 20260505 23:08:05.582 | INFO | Serializing RTMetrics to json | 20260505 23:08:05.597 | INFO | Serialization done, swapping tmp file now... | 20260505 23:08:05.604 | INFO | RTMetrics done | 20260505 23:08:05.639 | INFO | Finished a round of Baz IO: Wrote 1.906 GiB in 3.209 seconds | 20260505 23:08:05.639 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.45% / 0.00% / 78.55% | 20260505 23:08:05.639 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.111318% / 3.97364e-06% / 7.54992e-05% | 20260505 23:08:05.639 | INFO | Baz IO overall: | 20260505 23:08:05.639 | INFO | BytesWritten : 4030697376711 | 20260505 23:08:05.639 | INFO | NumEvents : 1899675566746 | 20260505 23:08:07.452 | INFO | Analyzing chunk frames = [7032832,7033344) | 20260505 23:08:07.452 | INFO | MemInfo: { | 20260505 23:08:07.452 | INFO | "gpu" : " 17 GB", | 20260505 23:08:07.452 | INFO | "huge" : "", | 20260505 23:08:07.452 | INFO | "huge_shared" : " 27 GB", | 20260505 23:08:07.452 | INFO | "malloc" : " 14 GB", | 20260505 23:08:07.452 | INFO | "pinned" : " 29 GB" | 20260505 23:08:07.452 | INFO | } | 20260505 23:08:07.452 | INFO | Perf report for the last 5.14069 seconds: | 20260505 23:08:07.452 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:08:07.452 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:07.452 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:07.452 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:07.452 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260505 23:08:07.453 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:07.453 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260505 23:08:07.453 | INFO | PRE_HQ : 59.7%, 1.7, 54.1ms, 0 | 20260505 23:08:07.453 | INFO | BAZWRITER : 61.6%, 3.6, 112.9ms, 0 | 20260505 23:08:07.453 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260505 23:08:07.453 | INFO | Uploaded 36GiB to GPU in 2.13167s (16.8882GiB/s) | 20260505 23:08:07.453 | INFO | Downloaded 38GiB to GPU in 2.23017s (17.0391GiB/s) | 20260505 23:08:12.589 | INFO | Analyzing chunk frames = [7033344,7033856) | 20260505 23:08:12.589 | INFO | MemInfo: { | 20260505 23:08:12.589 | INFO | "gpu" : " 17 GB", | 20260505 23:08:12.589 | INFO | "huge" : "", | 20260505 23:08:12.589 | INFO | "huge_shared" : " 27 GB", | 20260505 23:08:12.589 | INFO | "malloc" : " 14 GB", | 20260505 23:08:12.589 | INFO | "pinned" : " 29 GB" | 20260505 23:08:12.589 | INFO | } | 20260505 23:08:12.589 | INFO | Perf report for the last 5.13695 seconds: | 20260505 23:08:12.589 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:08:12.589 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:12.589 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:12.589 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:12.589 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260505 23:08:12.590 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:12.590 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:12.590 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:08:12.590 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:12.590 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:12.590 | INFO | Uploaded 36GiB to GPU in 2.10604s (17.0937GiB/s) | 20260505 23:08:12.590 | INFO | Downloaded 34GiB to GPU in 2.0427s (16.6446GiB/s) | 20260505 23:08:17.723 | INFO | Analyzing chunk frames = [7033856,7034368) | 20260505 23:08:17.723 | INFO | MemInfo: { | 20260505 23:08:17.723 | INFO | "gpu" : " 17 GB", | 20260505 23:08:17.723 | INFO | "huge" : "", | 20260505 23:08:17.723 | INFO | "huge_shared" : " 27 GB", | 20260505 23:08:17.723 | INFO | "malloc" : " 14 GB", | 20260505 23:08:17.723 | INFO | "pinned" : " 29 GB" | 20260505 23:08:17.723 | INFO | } | 20260505 23:08:17.723 | INFO | Perf report for the last 5.13364 seconds: | 20260505 23:08:17.723 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:08:17.723 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:17.723 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:17.723 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:17.723 | INFO | ANALYSIS : 62.0%, 3.0, 96.3ms, 0 | 20260505 23:08:17.723 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:17.723 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:17.723 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260505 23:08:17.723 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:17.723 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:17.723 | INFO | Uploaded 36GiB to GPU in 2.15033s (16.7416GiB/s) | 20260505 23:08:17.723 | INFO | Downloaded 34GiB to GPU in 2.04992s (16.586GiB/s) | 20260505 23:08:22.866 | INFO | Analyzing chunk frames = [7034368,7034880) | 20260505 23:08:22.866 | INFO | MemInfo: { | 20260505 23:08:22.866 | INFO | "gpu" : " 17 GB", | 20260505 23:08:22.866 | INFO | "huge" : "", | 20260505 23:08:22.866 | INFO | "huge_shared" : " 27 GB", | 20260505 23:08:22.866 | INFO | "malloc" : " 14 GB", | 20260505 23:08:22.866 | INFO | "pinned" : " 29 GB" | 20260505 23:08:22.866 | INFO | } | 20260505 23:08:22.866 | INFO | Perf report for the last 5.14332 seconds: | 20260505 23:08:22.866 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:08:22.866 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:22.866 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:22.866 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:22.866 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260505 23:08:22.866 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:22.866 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:22.866 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:08:22.866 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:22.866 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:22.867 | INFO | Uploaded 36GiB to GPU in 2.12531s (16.9387GiB/s) | 20260505 23:08:22.867 | INFO | Downloaded 34GiB to GPU in 2.049s (16.5935GiB/s) | 20260505 23:08:28.012 | INFO | Analyzing chunk frames = [7034880,7035392) | 20260505 23:08:28.012 | INFO | MemInfo: { | 20260505 23:08:28.012 | INFO | "gpu" : " 17 GB", | 20260505 23:08:28.012 | INFO | "huge" : "", | 20260505 23:08:28.012 | INFO | "huge_shared" : " 27 GB", | 20260505 23:08:28.012 | INFO | "malloc" : " 14 GB", | 20260505 23:08:28.012 | INFO | "pinned" : " 29 GB" | 20260505 23:08:28.012 | INFO | } | 20260505 23:08:28.012 | INFO | Perf report for the last 5.14595 seconds: | 20260505 23:08:28.012 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:08:28.012 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:28.012 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:28.012 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:28.012 | INFO | ANALYSIS : 61.1%, 3.0, 95.5ms, 0 | 20260505 23:08:28.012 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:28.012 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:28.012 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 23:08:28.012 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:28.012 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:28.013 | INFO | Uploaded 36GiB to GPU in 2.08784s (17.2427GiB/s) | 20260505 23:08:28.013 | INFO | Downloaded 34GiB to GPU in 2.04482s (16.6273GiB/s) | 20260505 23:08:33.151 | INFO | Analyzing chunk frames = [7035392,7035904) | 20260505 23:08:33.151 | INFO | MemInfo: { | 20260505 23:08:33.151 | INFO | "gpu" : " 17 GB", | 20260505 23:08:33.151 | INFO | "huge" : "", | 20260505 23:08:33.151 | INFO | "huge_shared" : " 27 GB", | 20260505 23:08:33.151 | INFO | "malloc" : " 14 GB", | 20260505 23:08:33.152 | INFO | "pinned" : " 29 GB" | 20260505 23:08:33.152 | INFO | } | 20260505 23:08:33.152 | INFO | Perf report for the last 5.13934 seconds: | 20260505 23:08:33.152 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:08:33.152 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:33.152 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:33.152 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:33.152 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260505 23:08:33.152 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:33.152 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:33.152 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:08:33.152 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:33.152 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:33.152 | INFO | Uploaded 36GiB to GPU in 2.06776s (17.4101GiB/s) | 20260505 23:08:33.152 | INFO | Downloaded 34GiB to GPU in 2.03639s (16.6962GiB/s) | 20260505 23:08:38.285 | INFO | Analyzing chunk frames = [7035904,7036416) | 20260505 23:08:38.285 | INFO | MemInfo: { | 20260505 23:08:38.285 | INFO | "gpu" : " 17 GB", | 20260505 23:08:38.285 | INFO | "huge" : "", | 20260505 23:08:38.285 | INFO | "huge_shared" : " 27 GB", | 20260505 23:08:38.285 | INFO | "malloc" : " 14 GB", | 20260505 23:08:38.285 | INFO | "pinned" : " 29 GB" | 20260505 23:08:38.285 | INFO | } | 20260505 23:08:38.285 | INFO | Perf report for the last 5.1333 seconds: | 20260505 23:08:38.285 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:08:38.285 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:38.285 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:38.285 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:38.285 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 23:08:38.285 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:38.285 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:38.285 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 23:08:38.285 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:38.285 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:38.285 | INFO | Uploaded 36GiB to GPU in 2.01185s (17.894GiB/s) | 20260505 23:08:38.285 | INFO | Downloaded 34GiB to GPU in 2.03797s (16.6833GiB/s) | 20260505 23:08:43.421 | INFO | Analyzing chunk frames = [7036416,7036928) | 20260505 23:08:43.421 | INFO | MemInfo: { | 20260505 23:08:43.421 | INFO | "gpu" : " 17 GB", | 20260505 23:08:43.421 | INFO | "huge" : "", | 20260505 23:08:43.421 | INFO | "huge_shared" : " 27 GB", | 20260505 23:08:43.422 | INFO | "malloc" : " 14 GB", | 20260505 23:08:43.422 | INFO | "pinned" : " 29 GB" | 20260505 23:08:43.422 | INFO | } | 20260505 23:08:43.422 | INFO | Perf report for the last 5.13665 seconds: | 20260505 23:08:43.422 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:08:43.422 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:43.422 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:43.422 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:43.422 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 23:08:43.422 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:43.422 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:43.422 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:08:43.422 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:43.422 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:43.422 | INFO | Uploaded 36GiB to GPU in 2.11859s (16.9925GiB/s) | 20260505 23:08:43.422 | INFO | Downloaded 34GiB to GPU in 2.01277s (16.8922GiB/s) | 20260505 23:08:43.422 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 23:08:43.422 | INFO | Total mem usage: 15.3 GiB | 20260505 23:08:43.422 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 23:08:43.422 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -9.05%) | 20260505 23:08:43.422 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 23:08:43.422 | INFO | High Water Breakdown: | 20260505 23:08:43.422 | INFO | 60.9% : 8.56 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 23:08:43.422 | INFO | 39% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 23:08:43.422 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 23:08:43.422 | INFO | ------------------------------------------ | 20260505 23:08:43.422 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:08:43.422 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 23:08:43.422 | INFO | Total mem usage: 26.5 GiB | 20260505 23:08:43.422 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 23:08:43.422 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 23:08:43.422 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 23:08:43.422 | INFO | High Water Breakdown: | 20260505 23:08:43.422 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:08:43.422 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:08:43.422 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:08:43.422 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:08:43.422 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:08:43.422 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:08:43.422 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 23:08:43.422 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 23:08:43.422 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:08:43.422 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 23:08:43.422 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 23:08:43.422 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 23:08:43.422 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 23:08:43.422 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 23:08:43.422 | INFO | ------------------------------------------ | 20260505 23:08:43.422 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:08:43.422 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 23:08:43.422 | INFO | Total mem usage: 25.6 GiB | 20260505 23:08:43.423 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 23:08:43.423 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 23:08:43.423 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 23:08:43.423 | INFO | High Water Breakdown: | 20260505 23:08:43.423 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 23:08:43.423 | INFO | ------------------------------------------ | 20260505 23:08:43.423 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:08:43.423 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 23:08:43.423 | INFO | Total mem usage: 15.7 GiB | 20260505 23:08:43.423 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 23:08:43.423 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 23:08:43.423 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 23:08:43.423 | INFO | High Water Breakdown: | 20260505 23:08:43.423 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 23:08:43.423 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 23:08:43.423 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 23:08:43.423 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 23:08:43.423 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 23:08:43.423 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 23:08:43.423 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 23:08:43.423 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 23:08:43.423 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 23:08:43.423 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:08:43.423 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 23:08:43.423 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:08:43.423 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 23:08:43.423 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 23:08:43.423 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 23:08:43.423 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 23:08:43.423 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:08:43.423 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 23:08:43.423 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:08:43.423 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:08:43.423 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:08:43.423 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 23:08:43.423 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 23:08:43.423 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:08:43.423 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 23:08:43.423 | INFO | ------------------------------------------ | 20260505 23:08:43.423 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:08:43.423 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 23:08:43.423 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:08:43.423 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:08:43.423 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.40e+01, 1.35e+01 / 8.35e-01 | 20260505 23:08:43.423 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.04e-01, 1.03e-02 / 4.66e-03 | 20260505 23:08:43.423 | INFO | PerfReport: 11.81%, Baseline -- 3.71e+00 / 4.48e+00, 5.64e-01 / 2.44e-01 | 20260505 23:08:43.423 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.52e+00, 1.90e-01 / 1.06e-01 | 20260505 23:08:43.423 | INFO | PerfReport: 16.03%, DME -- 4.02e+01 / 4.43e+01, 3.92e+00 / 1.10e+00 | 20260505 23:08:43.423 | INFO | PerfReport: 57.23%, FrameLabeling -- 1.80e+01 / 1.92e+01, 1.48e+00 / 1.08e+00 | 20260505 23:08:43.423 | INFO | PerfReport: 7.14%, PulseAccumulating -- 2.24e+00 / 2.81e+00, 4.22e-01 / 2.03e-01 | 20260505 23:08:43.423 | INFO | PerfReport: 3.13%, Metrics -- 9.82e-01 / 1.27e+00, 2.68e-01 / 1.46e-01 | 20260505 23:08:43.423 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.40e-01 / 3.97e-01, 5.89e-02 / 3.10e-02 | 20260505 23:08:43.423 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 23:08:43.423 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:08:43.423 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:08:43.423 | INFO | PerfReport: Full Execution -- 9.85e+01 / -nan, 2.46e+01 / 0.00e+00 | 20260505 23:08:43.423 | INFO | PerfReport: 22.25%, Upload -- 2.19e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260505 23:08:43.423 | INFO | PerfReport: 21.83%, Download -- 2.15e+01 / 0.00e+00, 7.53e+00 / 0.00e+00 | 20260505 23:08:43.423 | INFO | PerfReport: 31.87%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 23:08:43.423 | INFO | PerfReport: 2.08%, LockUpload -- 2.05e+00 / 0.00e+00, 7.77e+00 / 0.00e+00 | 20260505 23:08:43.423 | INFO | PerfReport: 1.32%, LockDownload -- 1.30e+00 / 0.00e+00, 2.94e+00 / 0.00e+00 | 20260505 23:08:43.423 | INFO | PerfReport: 20.63%, LockCompute -- 2.03e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260505 23:08:43.423 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.76e-02 / 0.00e+00 | 20260505 23:08:43.568 | INFO | Starting a round of Baz IO | 20260505 23:08:46.704 | INFO | Populating RTMetrics report | 20260505 23:08:46.704 | INFO | RTMetrics report populated | 20260505 23:08:46.704 | INFO | Opening tmp RTMetrics json file | 20260505 23:08:46.705 | INFO | Serializing RTMetrics to json | 20260505 23:08:46.708 | INFO | Serialization done, swapping tmp file now... | 20260505 23:08:46.710 | INFO | RTMetrics done | 20260505 23:08:46.787 | INFO | Finished a round of Baz IO: Wrote 1.906 GiB in 3.219 seconds | 20260505 23:08:46.788 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.45% / 0.00% / 78.55% | 20260505 23:08:46.788 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.112057% / 0% / 7.15256e-05% | 20260505 23:08:46.788 | INFO | Baz IO overall: | 20260505 23:08:46.788 | INFO | BytesWritten : 4032743596015 | 20260505 23:08:46.788 | INFO | NumEvents : 1900615535493 | 20260505 23:08:48.566 | INFO | Analyzing chunk frames = [7036928,7037440) | 20260505 23:08:48.566 | INFO | MemInfo: { | 20260505 23:08:48.566 | INFO | "gpu" : " 17 GB", | 20260505 23:08:48.566 | INFO | "huge" : "", | 20260505 23:08:48.566 | INFO | "huge_shared" : " 27 GB", | 20260505 23:08:48.566 | INFO | "malloc" : " 14 GB", | 20260505 23:08:48.566 | INFO | "pinned" : " 29 GB" | 20260505 23:08:48.566 | INFO | } | 20260505 23:08:48.567 | INFO | Perf report for the last 5.14483 seconds: | 20260505 23:08:48.567 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:08:48.567 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:48.567 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:48.567 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:48.567 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260505 23:08:48.567 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:48.567 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260505 23:08:48.567 | INFO | PRE_HQ : 59.4%, 1.8, 54.5ms, 0 | 20260505 23:08:48.567 | INFO | BAZWRITER : 62.6%, 3.6, 114.9ms, 0 | 20260505 23:08:48.567 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 23:08:48.567 | INFO | Uploaded 36GiB to GPU in 2.15563s (16.7005GiB/s) | 20260505 23:08:48.567 | INFO | Downloaded 38GiB to GPU in 2.24517s (16.9252GiB/s) | 20260505 23:08:53.709 | INFO | Analyzing chunk frames = [7037440,7037952) | 20260505 23:08:53.709 | INFO | MemInfo: { | 20260505 23:08:53.709 | INFO | "gpu" : " 17 GB", | 20260505 23:08:53.709 | INFO | "huge" : "", | 20260505 23:08:53.709 | INFO | "huge_shared" : " 27 GB", | 20260505 23:08:53.709 | INFO | "malloc" : " 14 GB", | 20260505 23:08:53.709 | INFO | "pinned" : " 29 GB" | 20260505 23:08:53.709 | INFO | } | 20260505 23:08:53.709 | INFO | Perf report for the last 5.14256 seconds: | 20260505 23:08:53.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:08:53.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:53.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:53.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:53.709 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 23:08:53.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:53.709 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:53.709 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:08:53.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:53.709 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:53.709 | INFO | Uploaded 36GiB to GPU in 2.09742s (17.1639GiB/s) | 20260505 23:08:53.709 | INFO | Downloaded 34GiB to GPU in 2.03736s (16.6883GiB/s) | 20260505 23:08:58.848 | INFO | Analyzing chunk frames = [7037952,7038464) | 20260505 23:08:58.848 | INFO | MemInfo: { | 20260505 23:08:58.848 | INFO | "gpu" : " 17 GB", | 20260505 23:08:58.848 | INFO | "huge" : "", | 20260505 23:08:58.848 | INFO | "huge_shared" : " 27 GB", | 20260505 23:08:58.849 | INFO | "malloc" : " 14 GB", | 20260505 23:08:58.849 | INFO | "pinned" : " 29 GB" | 20260505 23:08:58.849 | INFO | } | 20260505 23:08:58.849 | INFO | Perf report for the last 5.13953 seconds: | 20260505 23:08:58.849 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:08:58.849 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:58.849 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:58.849 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:58.849 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260505 23:08:58.849 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:58.849 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:58.849 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:08:58.849 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:58.849 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:08:58.849 | INFO | Uploaded 36GiB to GPU in 2.09465s (17.1866GiB/s) | 20260505 23:08:58.849 | INFO | Downloaded 34GiB to GPU in 2.02324s (16.8047GiB/s) | 20260505 23:09:03.985 | INFO | Analyzing chunk frames = [7038464,7038976) | 20260505 23:09:03.985 | INFO | MemInfo: { | 20260505 23:09:03.985 | INFO | "gpu" : " 17 GB", | 20260505 23:09:03.985 | INFO | "huge" : "", | 20260505 23:09:03.985 | INFO | "huge_shared" : " 27 GB", | 20260505 23:09:03.985 | INFO | "malloc" : " 14 GB", | 20260505 23:09:03.986 | INFO | "pinned" : " 29 GB" | 20260505 23:09:03.986 | INFO | } | 20260505 23:09:03.986 | INFO | Perf report for the last 5.13707 seconds: | 20260505 23:09:03.986 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:09:03.986 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:03.986 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:03.986 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:03.986 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260505 23:09:03.986 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:03.986 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:03.986 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 23:09:03.986 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:03.986 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:03.986 | INFO | Uploaded 36GiB to GPU in 2.14431s (16.7886GiB/s) | 20260505 23:09:03.986 | INFO | Downloaded 34GiB to GPU in 2.018s (16.8483GiB/s) | 20260505 23:09:09.119 | INFO | Analyzing chunk frames = [7038976,7039488) | 20260505 23:09:09.119 | INFO | MemInfo: { | 20260505 23:09:09.119 | INFO | "gpu" : " 17 GB", | 20260505 23:09:09.119 | INFO | "huge" : "", | 20260505 23:09:09.119 | INFO | "huge_shared" : " 27 GB", | 20260505 23:09:09.119 | INFO | "malloc" : " 14 GB", | 20260505 23:09:09.119 | INFO | "pinned" : " 29 GB" | 20260505 23:09:09.119 | INFO | } | 20260505 23:09:09.119 | INFO | Perf report for the last 5.13348 seconds: | 20260505 23:09:09.119 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:09:09.119 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:09.119 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:09.119 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:09.119 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260505 23:09:09.119 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:09.119 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:09.119 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 23:09:09.119 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:09.119 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:09.119 | INFO | Uploaded 36GiB to GPU in 2.09085s (17.2179GiB/s) | 20260505 23:09:09.119 | INFO | Downloaded 34GiB to GPU in 2.04174s (16.6525GiB/s) | 20260505 23:09:14.264 | INFO | Analyzing chunk frames = [7039488,7040000) | 20260505 23:09:14.264 | INFO | MemInfo: { | 20260505 23:09:14.264 | INFO | "gpu" : " 17 GB", | 20260505 23:09:14.264 | INFO | "huge" : "", | 20260505 23:09:14.264 | INFO | "huge_shared" : " 27 GB", | 20260505 23:09:14.265 | INFO | "malloc" : " 14 GB", | 20260505 23:09:14.265 | INFO | "pinned" : " 29 GB" | 20260505 23:09:14.265 | INFO | } | 20260505 23:09:14.265 | INFO | Perf report for the last 5.14547 seconds: | 20260505 23:09:14.265 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:09:14.265 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:14.265 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:14.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:14.265 | INFO | ANALYSIS : 61.4%, 3.0, 96.0ms, 0 | 20260505 23:09:14.265 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:14.265 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:14.265 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260505 23:09:14.265 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:14.265 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:14.265 | INFO | Uploaded 36GiB to GPU in 2.08134s (17.2965GiB/s) | 20260505 23:09:14.265 | INFO | Downloaded 34GiB to GPU in 2.03875s (16.6769GiB/s) | 20260505 23:09:19.401 | INFO | Analyzing chunk frames = [7040000,7040512) | 20260505 23:09:19.401 | INFO | MemInfo: { | 20260505 23:09:19.401 | INFO | "gpu" : " 17 GB", | 20260505 23:09:19.401 | INFO | "huge" : "", | 20260505 23:09:19.401 | INFO | "huge_shared" : " 27 GB", | 20260505 23:09:19.401 | INFO | "malloc" : " 14 GB", | 20260505 23:09:19.401 | INFO | "pinned" : " 29 GB" | 20260505 23:09:19.401 | INFO | } | 20260505 23:09:19.401 | INFO | Perf report for the last 5.13667 seconds: | 20260505 23:09:19.401 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:09:19.401 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:19.401 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:19.401 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:19.401 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 23:09:19.401 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:19.401 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:19.401 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 23:09:19.401 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:19.402 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:19.402 | INFO | Uploaded 36GiB to GPU in 2.0282s (17.7498GiB/s) | 20260505 23:09:19.402 | INFO | Downloaded 34GiB to GPU in 2.05948s (16.509GiB/s) | 20260505 23:09:24.538 | INFO | Analyzing chunk frames = [7040512,7041024) | 20260505 23:09:24.538 | INFO | MemInfo: { | 20260505 23:09:24.538 | INFO | "gpu" : " 17 GB", | 20260505 23:09:24.538 | INFO | "huge" : "", | 20260505 23:09:24.538 | INFO | "huge_shared" : " 27 GB", | 20260505 23:09:24.538 | INFO | "malloc" : " 14 GB", | 20260505 23:09:24.538 | INFO | "pinned" : " 29 GB" | 20260505 23:09:24.538 | INFO | } | 20260505 23:09:24.538 | INFO | Perf report for the last 5.13666 seconds: | 20260505 23:09:24.538 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:09:24.538 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:24.538 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:24.538 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:24.538 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260505 23:09:24.538 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:24.538 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:24.538 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:09:24.538 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:24.538 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:24.538 | INFO | Uploaded 36GiB to GPU in 2.11167s (17.0481GiB/s) | 20260505 23:09:24.538 | INFO | Downloaded 34GiB to GPU in 2.00465s (16.9606GiB/s) | 20260505 23:09:24.665 | INFO | Starting a round of Baz IO | 20260505 23:09:27.809 | INFO | Populating RTMetrics report | 20260505 23:09:27.810 | INFO | RTMetrics report populated | 20260505 23:09:27.810 | INFO | Opening tmp RTMetrics json file | 20260505 23:09:27.812 | INFO | Serializing RTMetrics to json | 20260505 23:09:27.814 | INFO | Serialization done, swapping tmp file now... | 20260505 23:09:27.814 | INFO | RTMetrics done | 20260505 23:09:27.894 | INFO | Finished a round of Baz IO: Wrote 1.905 GiB in 3.229 seconds | 20260505 23:09:27.894 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.45% / 0.00% / 78.55% | 20260505 23:09:27.894 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.11218% / 3.97364e-06% / 5.96046e-05% | 20260505 23:09:27.894 | INFO | Baz IO overall: | 20260505 23:09:27.894 | INFO | BytesWritten : 4034789548938 | 20260505 23:09:27.894 | INFO | NumEvents : 1901555254255 | 20260505 23:09:29.679 | INFO | Analyzing chunk frames = [7041024,7041536) | 20260505 23:09:29.679 | INFO | MemInfo: { | 20260505 23:09:29.679 | INFO | "gpu" : " 17 GB", | 20260505 23:09:29.679 | INFO | "huge" : "", | 20260505 23:09:29.679 | INFO | "huge_shared" : " 27 GB", | 20260505 23:09:29.679 | INFO | "malloc" : " 14 GB", | 20260505 23:09:29.679 | INFO | "pinned" : " 29 GB" | 20260505 23:09:29.679 | INFO | } | 20260505 23:09:29.679 | INFO | Perf report for the last 5.14071 seconds: | 20260505 23:09:29.679 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:09:29.679 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:29.679 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:29.679 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:29.679 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 23:09:29.679 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:29.679 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260505 23:09:29.679 | INFO | PRE_HQ : 59.8%, 1.7, 54.2ms, 0 | 20260505 23:09:29.679 | INFO | BAZWRITER : 62.1%, 3.6, 114.1ms, 0 | 20260505 23:09:29.679 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 23:09:29.679 | INFO | Uploaded 36GiB to GPU in 2.15988s (16.6676GiB/s) | 20260505 23:09:29.679 | INFO | Downloaded 38GiB to GPU in 2.23284s (17.0187GiB/s) | 20260505 23:09:34.819 | INFO | Analyzing chunk frames = [7041536,7042048) | 20260505 23:09:34.819 | INFO | MemInfo: { | 20260505 23:09:34.819 | INFO | "gpu" : " 17 GB", | 20260505 23:09:34.819 | INFO | "huge" : "", | 20260505 23:09:34.819 | INFO | "huge_shared" : " 27 GB", | 20260505 23:09:34.819 | INFO | "malloc" : " 14 GB", | 20260505 23:09:34.819 | INFO | "pinned" : " 29 GB" | 20260505 23:09:34.819 | INFO | } | 20260505 23:09:34.819 | INFO | Perf report for the last 5.14059 seconds: | 20260505 23:09:34.819 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:09:34.819 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:34.819 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:34.819 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:34.819 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 23:09:34.819 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:34.819 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:34.819 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:09:34.819 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:34.820 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:34.820 | INFO | Uploaded 36GiB to GPU in 2.08663s (17.2527GiB/s) | 20260505 23:09:34.820 | INFO | Downloaded 34GiB to GPU in 2.0261s (16.781GiB/s) | 20260505 23:09:39.969 | INFO | Analyzing chunk frames = [7042048,7042560) | 20260505 23:09:39.969 | INFO | MemInfo: { | 20260505 23:09:39.969 | INFO | "gpu" : " 17 GB", | 20260505 23:09:39.969 | INFO | "huge" : "", | 20260505 23:09:39.969 | INFO | "huge_shared" : " 27 GB", | 20260505 23:09:39.969 | INFO | "malloc" : " 14 GB", | 20260505 23:09:39.969 | INFO | "pinned" : " 29 GB" | 20260505 23:09:39.969 | INFO | } | 20260505 23:09:39.969 | INFO | Perf report for the last 5.1493 seconds: | 20260505 23:09:39.969 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:09:39.969 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:39.969 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:39.969 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:39.969 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260505 23:09:39.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:39.969 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:39.969 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:09:39.969 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:39.969 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:39.969 | INFO | Uploaded 36GiB to GPU in 2.0937s (17.1944GiB/s) | 20260505 23:09:39.969 | INFO | Downloaded 34GiB to GPU in 2.02524s (16.7881GiB/s) | 20260505 23:09:45.103 | INFO | Analyzing chunk frames = [7042560,7043072) | 20260505 23:09:45.103 | INFO | MemInfo: { | 20260505 23:09:45.103 | INFO | "gpu" : " 17 GB", | 20260505 23:09:45.103 | INFO | "huge" : "", | 20260505 23:09:45.103 | INFO | "huge_shared" : " 27 GB", | 20260505 23:09:45.103 | INFO | "malloc" : " 14 GB", | 20260505 23:09:45.103 | INFO | "pinned" : " 29 GB" | 20260505 23:09:45.103 | INFO | } | 20260505 23:09:45.103 | INFO | Perf report for the last 5.13445 seconds: | 20260505 23:09:45.103 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:09:45.103 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:45.103 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:45.103 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:45.103 | INFO | ANALYSIS : 61.7%, 3.0, 95.7ms, 0 | 20260505 23:09:45.103 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:45.103 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:45.103 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 23:09:45.103 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:45.103 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:45.103 | INFO | Uploaded 36GiB to GPU in 2.15026s (16.7422GiB/s) | 20260505 23:09:45.103 | INFO | Downloaded 34GiB to GPU in 2.04948s (16.5896GiB/s) | 20260505 23:09:50.233 | INFO | Analyzing chunk frames = [7043072,7043584) | 20260505 23:09:50.233 | INFO | MemInfo: { | 20260505 23:09:50.233 | INFO | "gpu" : " 17 GB", | 20260505 23:09:50.233 | INFO | "huge" : "", | 20260505 23:09:50.233 | INFO | "huge_shared" : " 27 GB", | 20260505 23:09:50.233 | INFO | "malloc" : " 14 GB", | 20260505 23:09:50.233 | INFO | "pinned" : " 29 GB" | 20260505 23:09:50.233 | INFO | } | 20260505 23:09:50.234 | INFO | Perf report for the last 5.1304 seconds: | 20260505 23:09:50.234 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:09:50.234 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:50.234 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:50.234 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:50.234 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260505 23:09:50.234 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:50.234 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:50.234 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:09:50.234 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:50.234 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:50.234 | INFO | Uploaded 36GiB to GPU in 2.0913s (17.2142GiB/s) | 20260505 23:09:50.234 | INFO | Downloaded 34GiB to GPU in 2.04196s (16.6507GiB/s) | 20260505 23:09:55.372 | INFO | Analyzing chunk frames = [7043584,7044096) | 20260505 23:09:55.373 | INFO | MemInfo: { | 20260505 23:09:55.373 | INFO | "gpu" : " 17 GB", | 20260505 23:09:55.373 | INFO | "huge" : "", | 20260505 23:09:55.373 | INFO | "huge_shared" : " 27 GB", | 20260505 23:09:55.373 | INFO | "malloc" : " 14 GB", | 20260505 23:09:55.373 | INFO | "pinned" : " 29 GB" | 20260505 23:09:55.373 | INFO | } | 20260505 23:09:55.373 | INFO | Perf report for the last 5.13921 seconds: | 20260505 23:09:55.373 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:09:55.373 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:55.373 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:55.373 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:55.373 | INFO | ANALYSIS : 61.4%, 3.0, 95.9ms, 0 | 20260505 23:09:55.373 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:55.373 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:55.373 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 23:09:55.373 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:55.373 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:09:55.373 | INFO | Uploaded 36GiB to GPU in 2.06926s (17.3975GiB/s) | 20260505 23:09:55.373 | INFO | Downloaded 34GiB to GPU in 2.03465s (16.7105GiB/s) | 20260505 23:10:00.514 | INFO | Analyzing chunk frames = [7044096,7044608) | 20260505 23:10:00.514 | INFO | MemInfo: { | 20260505 23:10:00.514 | INFO | "gpu" : " 17 GB", | 20260505 23:10:00.514 | INFO | "huge" : "", | 20260505 23:10:00.514 | INFO | "huge_shared" : " 27 GB", | 20260505 23:10:00.514 | INFO | "malloc" : " 14 GB", | 20260505 23:10:00.515 | INFO | "pinned" : " 29 GB" | 20260505 23:10:00.515 | INFO | } | 20260505 23:10:00.515 | INFO | Perf report for the last 5.14192 seconds: | 20260505 23:10:00.515 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:10:00.515 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:00.515 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:00.515 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:00.515 | INFO | ANALYSIS : 62.7%, 3.0, 98.0ms, 0 | 20260505 23:10:00.515 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:00.515 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:00.515 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 23:10:00.515 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:00.515 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:00.515 | INFO | Uploaded 36GiB to GPU in 2.14027s (16.8203GiB/s) | 20260505 23:10:00.515 | INFO | Downloaded 34GiB to GPU in 2.13279s (15.9416GiB/s) | 20260505 23:10:05.665 | INFO | Analyzing chunk frames = [7044608,7045120) | 20260505 23:10:05.665 | INFO | MemInfo: { | 20260505 23:10:05.665 | INFO | "gpu" : " 17 GB", | 20260505 23:10:05.665 | INFO | "huge" : "", | 20260505 23:10:05.665 | INFO | "huge_shared" : " 27 GB", | 20260505 23:10:05.665 | INFO | "malloc" : " 14 GB", | 20260505 23:10:05.665 | INFO | "pinned" : " 29 GB" | 20260505 23:10:05.665 | INFO | } | 20260505 23:10:05.665 | INFO | Perf report for the last 5.15002 seconds: | 20260505 23:10:05.665 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:10:05.665 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:05.665 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:05.665 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:05.665 | INFO | ANALYSIS : 61.9%, 3.0, 96.9ms, 0 | 20260505 23:10:05.665 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:05.665 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:05.665 | INFO | PRE_HQ : 26.8%, 1.0, 14.0ms, 0 | 20260505 23:10:05.665 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:05.665 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:05.665 | INFO | Uploaded 36GiB to GPU in 2.1341s (16.8689GiB/s) | 20260505 23:10:05.665 | INFO | Downloaded 34GiB to GPU in 2.0228s (16.8084GiB/s) | 20260505 23:10:05.665 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 23:10:05.665 | INFO | Total mem usage: 15.3 GiB | 20260505 23:10:05.665 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 23:10:05.665 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -9.07%) | 20260505 23:10:05.665 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 23:10:05.665 | INFO | High Water Breakdown: | 20260505 23:10:05.665 | INFO | 60.9% : 8.56 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 23:10:05.665 | INFO | 39% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 23:10:05.665 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 23:10:05.665 | INFO | ------------------------------------------ | 20260505 23:10:05.665 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:10:05.665 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 23:10:05.665 | INFO | Total mem usage: 26.5 GiB | 20260505 23:10:05.665 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 23:10:05.665 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 23:10:05.665 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 23:10:05.665 | INFO | High Water Breakdown: | 20260505 23:10:05.665 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:10:05.665 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:10:05.665 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:10:05.665 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:10:05.665 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:10:05.665 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:10:05.665 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 23:10:05.665 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 23:10:05.665 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:10:05.666 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 23:10:05.666 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 23:10:05.666 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 23:10:05.666 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 23:10:05.666 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 23:10:05.666 | INFO | ------------------------------------------ | 20260505 23:10:05.666 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:10:05.666 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 23:10:05.666 | INFO | Total mem usage: 25.6 GiB | 20260505 23:10:05.666 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 23:10:05.666 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 23:10:05.666 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 23:10:05.666 | INFO | High Water Breakdown: | 20260505 23:10:05.666 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 23:10:05.666 | INFO | ------------------------------------------ | 20260505 23:10:05.666 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:10:05.666 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 23:10:05.666 | INFO | Total mem usage: 15.7 GiB | 20260505 23:10:05.666 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 23:10:05.666 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 23:10:05.666 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 23:10:05.666 | INFO | High Water Breakdown: | 20260505 23:10:05.666 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 23:10:05.666 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 23:10:05.666 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 23:10:05.666 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 23:10:05.666 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 23:10:05.666 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 23:10:05.666 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 23:10:05.666 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 23:10:05.666 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 23:10:05.666 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:10:05.666 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 23:10:05.666 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:10:05.666 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 23:10:05.666 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 23:10:05.666 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 23:10:05.666 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 23:10:05.666 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:10:05.666 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 23:10:05.666 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:10:05.666 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:10:05.666 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:10:05.666 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 23:10:05.666 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 23:10:05.666 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:10:05.666 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 23:10:05.666 | INFO | ------------------------------------------ | 20260505 23:10:05.666 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:10:05.666 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 23:10:05.666 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:10:05.666 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:10:05.666 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.42e+01, 1.34e+01 / 1.26e+00 | 20260505 23:10:05.666 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.05e-01, 1.05e-02 / 4.43e-03 | 20260505 23:10:05.666 | INFO | PerfReport: 11.81%, Baseline -- 3.71e+00 / 4.49e+00, 5.73e-01 / 2.19e-01 | 20260505 23:10:05.666 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.53e+00, 1.90e-01 / 8.18e-02 | 20260505 23:10:05.666 | INFO | PerfReport: 15.98%, DME -- 4.01e+01 / 4.41e+01, 3.78e+00 / 1.41e+00 | 20260505 23:10:05.666 | INFO | PerfReport: 57.30%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.52e+00 / 1.01e+00 | 20260505 23:10:05.666 | INFO | PerfReport: 7.11%, PulseAccumulating -- 2.23e+00 / 2.77e+00, 4.23e-01 / 2.02e-01 | 20260505 23:10:05.666 | INFO | PerfReport: 3.13%, Metrics -- 9.82e-01 / 1.30e+00, 2.70e-01 / 1.37e-01 | 20260505 23:10:05.666 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.40e-01 / 3.94e-01, 5.87e-02 / 2.82e-02 | 20260505 23:10:05.666 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 23:10:05.666 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:10:05.666 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:10:05.666 | INFO | PerfReport: Full Execution -- 9.85e+01 / -nan, 2.49e+01 / 0.00e+00 | 20260505 23:10:05.666 | INFO | PerfReport: 22.29%, Upload -- 2.20e+01 / 0.00e+00, 7.49e+00 / 0.00e+00 | 20260505 23:10:05.666 | INFO | PerfReport: 21.84%, Download -- 2.15e+01 / 0.00e+00, 7.59e+00 / 0.00e+00 | 20260505 23:10:05.666 | INFO | PerfReport: 31.87%, Compute -- 3.14e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260505 23:10:05.666 | INFO | PerfReport: 2.14%, LockUpload -- 2.10e+00 / 0.00e+00, 7.92e+00 / 0.00e+00 | 20260505 23:10:05.666 | INFO | PerfReport: 1.30%, LockDownload -- 1.28e+00 / 0.00e+00, 2.89e+00 / 0.00e+00 | 20260505 23:10:05.666 | INFO | PerfReport: 20.55%, LockCompute -- 2.02e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260505 23:10:05.666 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 2.40e-02 / 0.00e+00 | 20260505 23:10:05.782 | INFO | Starting a round of Baz IO | 20260505 23:10:08.931 | INFO | Populating RTMetrics report | 20260505 23:10:08.931 | INFO | RTMetrics report populated | 20260505 23:10:08.931 | INFO | Opening tmp RTMetrics json file | 20260505 23:10:08.934 | INFO | Serializing RTMetrics to json | 20260505 23:10:08.936 | INFO | Serialization done, swapping tmp file now... | 20260505 23:10:08.939 | INFO | RTMetrics done | 20260505 23:10:09.010 | INFO | Finished a round of Baz IO: Wrote 1.905 GiB in 3.228 seconds | 20260505 23:10:09.011 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.45% / 0.00% / 78.55% | 20260505 23:10:09.011 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.111298% / 0% / 9.13938e-05% | 20260505 23:10:09.011 | INFO | Baz IO overall: | 20260505 23:10:09.011 | INFO | BytesWritten : 4036834731308 | 20260505 23:10:09.011 | INFO | NumEvents : 1902494524718 | 20260505 23:10:10.800 | INFO | Analyzing chunk frames = [7045120,7045632) | 20260505 23:10:10.800 | INFO | MemInfo: { | 20260505 23:10:10.800 | INFO | "gpu" : " 17 GB", | 20260505 23:10:10.800 | INFO | "huge" : "", | 20260505 23:10:10.800 | INFO | "huge_shared" : " 27 GB", | 20260505 23:10:10.800 | INFO | "malloc" : " 14 GB", | 20260505 23:10:10.800 | INFO | "pinned" : " 29 GB" | 20260505 23:10:10.801 | INFO | } | 20260505 23:10:10.801 | INFO | Perf report for the last 5.13586 seconds: | 20260505 23:10:10.801 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:10:10.801 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:10.801 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:10.801 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:10.801 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 23:10:10.801 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:10.801 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260505 23:10:10.801 | INFO | PRE_HQ : 59.7%, 1.7, 54.2ms, 0 | 20260505 23:10:10.801 | INFO | BAZWRITER : 61.9%, 3.6, 114.5ms, 0 | 20260505 23:10:10.801 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 23:10:10.801 | INFO | Uploaded 36GiB to GPU in 2.18076s (16.508GiB/s) | 20260505 23:10:10.801 | INFO | Downloaded 38GiB to GPU in 2.24794s (16.9044GiB/s) | 20260505 23:10:15.934 | INFO | Analyzing chunk frames = [7045632,7046144) | 20260505 23:10:15.934 | INFO | MemInfo: { | 20260505 23:10:15.934 | INFO | "gpu" : " 17 GB", | 20260505 23:10:15.934 | INFO | "huge" : "", | 20260505 23:10:15.934 | INFO | "huge_shared" : " 27 GB", | 20260505 23:10:15.934 | INFO | "malloc" : " 14 GB", | 20260505 23:10:15.934 | INFO | "pinned" : " 29 GB" | 20260505 23:10:15.934 | INFO | } | 20260505 23:10:15.934 | INFO | Perf report for the last 5.13313 seconds: | 20260505 23:10:15.934 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:10:15.934 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:15.934 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:15.934 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:15.934 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260505 23:10:15.934 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:15.934 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:15.934 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:10:15.934 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:15.934 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:15.934 | INFO | Uploaded 36GiB to GPU in 2.10705s (17.0855GiB/s) | 20260505 23:10:15.934 | INFO | Downloaded 34GiB to GPU in 2.03956s (16.6702GiB/s) | 20260505 23:10:21.073 | INFO | Analyzing chunk frames = [7046144,7046656) | 20260505 23:10:21.073 | INFO | MemInfo: { | 20260505 23:10:21.073 | INFO | "gpu" : " 17 GB", | 20260505 23:10:21.073 | INFO | "huge" : "", | 20260505 23:10:21.073 | INFO | "huge_shared" : " 27 GB", | 20260505 23:10:21.074 | INFO | "malloc" : " 14 GB", | 20260505 23:10:21.074 | INFO | "pinned" : " 29 GB" | 20260505 23:10:21.074 | INFO | } | 20260505 23:10:21.074 | INFO | Perf report for the last 5.13988 seconds: | 20260505 23:10:21.074 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:10:21.074 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:21.074 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:21.074 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:21.074 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260505 23:10:21.074 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:21.074 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:21.074 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 23:10:21.074 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:21.074 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:21.074 | INFO | Uploaded 36GiB to GPU in 2.15748s (16.6861GiB/s) | 20260505 23:10:21.074 | INFO | Downloaded 34GiB to GPU in 2.04612s (16.6168GiB/s) | 20260505 23:10:26.212 | INFO | Analyzing chunk frames = [7046656,7047168) | 20260505 23:10:26.212 | INFO | MemInfo: { | 20260505 23:10:26.212 | INFO | "gpu" : " 17 GB", | 20260505 23:10:26.212 | INFO | "huge" : "", | 20260505 23:10:26.212 | INFO | "huge_shared" : " 27 GB", | 20260505 23:10:26.212 | INFO | "malloc" : " 14 GB", | 20260505 23:10:26.212 | INFO | "pinned" : " 29 GB" | 20260505 23:10:26.212 | INFO | } | 20260505 23:10:26.212 | INFO | Perf report for the last 5.13854 seconds: | 20260505 23:10:26.212 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:10:26.212 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:26.212 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:26.212 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:26.212 | INFO | ANALYSIS : 61.7%, 3.0, 95.8ms, 0 | 20260505 23:10:26.212 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:26.212 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:26.212 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:10:26.212 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:26.212 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:26.212 | INFO | Uploaded 36GiB to GPU in 2.14443s (16.7877GiB/s) | 20260505 23:10:26.212 | INFO | Downloaded 34GiB to GPU in 2.01364s (16.8848GiB/s) | 20260505 23:10:31.349 | INFO | Analyzing chunk frames = [7047168,7047680) | 20260505 23:10:31.349 | INFO | MemInfo: { | 20260505 23:10:31.349 | INFO | "gpu" : " 17 GB", | 20260505 23:10:31.349 | INFO | "huge" : "", | 20260505 23:10:31.349 | INFO | "huge_shared" : " 27 GB", | 20260505 23:10:31.349 | INFO | "malloc" : " 14 GB", | 20260505 23:10:31.349 | INFO | "pinned" : " 29 GB" | 20260505 23:10:31.350 | INFO | } | 20260505 23:10:31.350 | INFO | Perf report for the last 5.13747 seconds: | 20260505 23:10:31.350 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:10:31.350 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:31.350 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:31.350 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:31.350 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260505 23:10:31.350 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:31.350 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:31.350 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 23:10:31.350 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:31.350 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:31.350 | INFO | Uploaded 36GiB to GPU in 2.08399s (17.2745GiB/s) | 20260505 23:10:31.350 | INFO | Downloaded 34GiB to GPU in 2.04425s (16.632GiB/s) | 20260505 23:10:36.488 | INFO | Analyzing chunk frames = [7047680,7048192) | 20260505 23:10:36.488 | INFO | MemInfo: { | 20260505 23:10:36.488 | INFO | "gpu" : " 17 GB", | 20260505 23:10:36.488 | INFO | "huge" : "", | 20260505 23:10:36.488 | INFO | "huge_shared" : " 27 GB", | 20260505 23:10:36.488 | INFO | "malloc" : " 14 GB", | 20260505 23:10:36.488 | INFO | "pinned" : " 29 GB" | 20260505 23:10:36.488 | INFO | } | 20260505 23:10:36.488 | INFO | Perf report for the last 5.13853 seconds: | 20260505 23:10:36.488 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:10:36.488 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:36.488 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:36.488 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:36.488 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260505 23:10:36.488 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:36.488 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:36.488 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:10:36.488 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:36.488 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:36.488 | INFO | Uploaded 36GiB to GPU in 2.08s (17.3077GiB/s) | 20260505 23:10:36.489 | INFO | Downloaded 34GiB to GPU in 2.03613s (16.6983GiB/s) | 20260505 23:10:41.630 | INFO | Analyzing chunk frames = [7048192,7048704) | 20260505 23:10:41.630 | INFO | MemInfo: { | 20260505 23:10:41.630 | INFO | "gpu" : " 17 GB", | 20260505 23:10:41.630 | INFO | "huge" : "", | 20260505 23:10:41.630 | INFO | "huge_shared" : " 27 GB", | 20260505 23:10:41.630 | INFO | "malloc" : " 14 GB", | 20260505 23:10:41.630 | INFO | "pinned" : " 29 GB" | 20260505 23:10:41.630 | INFO | } | 20260505 23:10:41.630 | INFO | Perf report for the last 5.14211 seconds: | 20260505 23:10:41.630 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:10:41.630 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:41.630 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:41.630 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:41.630 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260505 23:10:41.630 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:41.630 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:41.630 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 23:10:41.630 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:41.631 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:41.631 | INFO | Uploaded 36GiB to GPU in 2.1574s (16.6868GiB/s) | 20260505 23:10:41.631 | INFO | Downloaded 34GiB to GPU in 2.10985s (16.1149GiB/s) | 20260505 23:10:46.774 | INFO | Analyzing chunk frames = [7048704,7049216) | 20260505 23:10:46.774 | INFO | MemInfo: { | 20260505 23:10:46.774 | INFO | "gpu" : " 17 GB", | 20260505 23:10:46.774 | INFO | "huge" : "", | 20260505 23:10:46.774 | INFO | "huge_shared" : " 27 GB", | 20260505 23:10:46.774 | INFO | "malloc" : " 14 GB", | 20260505 23:10:46.774 | INFO | "pinned" : " 29 GB" | 20260505 23:10:46.774 | INFO | } | 20260505 23:10:46.774 | INFO | Perf report for the last 5.14373 seconds: | 20260505 23:10:46.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:10:46.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:46.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:46.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:46.774 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 23:10:46.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:46.774 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:46.774 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 23:10:46.774 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:46.774 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:46.774 | INFO | Uploaded 36GiB to GPU in 2.11168s (17.048GiB/s) | 20260505 23:10:46.774 | INFO | Downloaded 34GiB to GPU in 2.00358s (16.9696GiB/s) | 20260505 23:10:46.892 | INFO | Starting a round of Baz IO | 20260505 23:10:50.038 | INFO | Populating RTMetrics report | 20260505 23:10:50.039 | INFO | RTMetrics report populated | 20260505 23:10:50.039 | INFO | Opening tmp RTMetrics json file | 20260505 23:10:50.040 | INFO | Serializing RTMetrics to json | 20260505 23:10:50.043 | INFO | Serialization done, swapping tmp file now... | 20260505 23:10:50.045 | INFO | RTMetrics done | 20260505 23:10:50.120 | INFO | Finished a round of Baz IO: Wrote 1.906 GiB in 3.228 seconds | 20260505 23:10:50.120 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.45% / 0.00% / 78.55% | 20260505 23:10:50.120 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.111747% / 0% / 0.000103315% | 20260505 23:10:50.120 | INFO | Baz IO overall: | 20260505 23:10:50.120 | INFO | BytesWritten : 4038880986479 | 20260505 23:10:50.120 | INFO | NumEvents : 1903434859889 | 20260505 23:10:51.913 | INFO | Analyzing chunk frames = [7049216,7049728) | 20260505 23:10:51.913 | INFO | MemInfo: { | 20260505 23:10:51.913 | INFO | "gpu" : " 17 GB", | 20260505 23:10:51.913 | INFO | "huge" : "", | 20260505 23:10:51.913 | INFO | "huge_shared" : " 27 GB", | 20260505 23:10:51.913 | INFO | "malloc" : " 14 GB", | 20260505 23:10:51.913 | INFO | "pinned" : " 29 GB" | 20260505 23:10:51.913 | INFO | } | 20260505 23:10:51.913 | INFO | Perf report for the last 5.13921 seconds: | 20260505 23:10:51.913 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:10:51.913 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:51.913 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:51.913 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:51.913 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260505 23:10:51.913 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:51.913 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260505 23:10:51.913 | INFO | PRE_HQ : 59.7%, 1.8, 54.2ms, 0 | 20260505 23:10:51.913 | INFO | BAZWRITER : 62.0%, 3.6, 114.1ms, 0 | 20260505 23:10:51.913 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 23:10:51.914 | INFO | Uploaded 36GiB to GPU in 2.15076s (16.7383GiB/s) | 20260505 23:10:51.914 | INFO | Downloaded 38GiB to GPU in 2.23562s (16.9975GiB/s) | 20260505 23:10:57.049 | INFO | Analyzing chunk frames = [7049728,7050240) | 20260505 23:10:57.049 | INFO | MemInfo: { | 20260505 23:10:57.049 | INFO | "gpu" : " 17 GB", | 20260505 23:10:57.049 | INFO | "huge" : "", | 20260505 23:10:57.049 | INFO | "huge_shared" : " 27 GB", | 20260505 23:10:57.049 | INFO | "malloc" : " 14 GB", | 20260505 23:10:57.049 | INFO | "pinned" : " 29 GB" | 20260505 23:10:57.049 | INFO | } | 20260505 23:10:57.049 | INFO | Perf report for the last 5.13585 seconds: | 20260505 23:10:57.049 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:10:57.049 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:57.049 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:57.049 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:57.049 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260505 23:10:57.049 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:57.049 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:57.049 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260505 23:10:57.049 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:57.049 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:10:57.049 | INFO | Uploaded 36GiB to GPU in 2.09867s (17.1537GiB/s) | 20260505 23:10:57.049 | INFO | Downloaded 34GiB to GPU in 2.03785s (16.6843GiB/s) | 20260505 23:11:02.182 | INFO | Analyzing chunk frames = [7050240,7050752) | 20260505 23:11:02.182 | INFO | MemInfo: { | 20260505 23:11:02.182 | INFO | "gpu" : " 17 GB", | 20260505 23:11:02.182 | INFO | "huge" : "", | 20260505 23:11:02.182 | INFO | "huge_shared" : " 27 GB", | 20260505 23:11:02.182 | INFO | "malloc" : " 14 GB", | 20260505 23:11:02.182 | INFO | "pinned" : " 29 GB" | 20260505 23:11:02.182 | INFO | } | 20260505 23:11:02.182 | INFO | Perf report for the last 5.13315 seconds: | 20260505 23:11:02.182 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:11:02.182 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:02.182 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:02.182 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:02.182 | INFO | ANALYSIS : 62.0%, 3.0, 96.3ms, 0 | 20260505 23:11:02.182 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:02.182 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:02.182 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:11:02.182 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:02.182 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:02.183 | INFO | Uploaded 36GiB to GPU in 2.15157s (16.732GiB/s) | 20260505 23:11:02.183 | INFO | Downloaded 34GiB to GPU in 2.03582s (16.7009GiB/s) | 20260505 23:11:07.327 | INFO | Analyzing chunk frames = [7050752,7051264) | 20260505 23:11:07.327 | INFO | MemInfo: { | 20260505 23:11:07.327 | INFO | "gpu" : " 17 GB", | 20260505 23:11:07.327 | INFO | "huge" : "", | 20260505 23:11:07.327 | INFO | "huge_shared" : " 27 GB", | 20260505 23:11:07.327 | INFO | "malloc" : " 14 GB", | 20260505 23:11:07.328 | INFO | "pinned" : " 29 GB" | 20260505 23:11:07.328 | INFO | } | 20260505 23:11:07.328 | INFO | Perf report for the last 5.14528 seconds: | 20260505 23:11:07.328 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:11:07.328 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:07.328 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:07.328 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:07.328 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260505 23:11:07.328 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:07.328 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:07.328 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:11:07.328 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:07.328 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:07.328 | INFO | Uploaded 36GiB to GPU in 2.21043s (16.2864GiB/s) | 20260505 23:11:07.328 | INFO | Downloaded 34GiB to GPU in 2.06613s (16.4559GiB/s) | 20260505 23:11:12.465 | INFO | Analyzing chunk frames = [7051264,7051776) | 20260505 23:11:12.465 | INFO | MemInfo: { | 20260505 23:11:12.465 | INFO | "gpu" : " 17 GB", | 20260505 23:11:12.465 | INFO | "huge" : "", | 20260505 23:11:12.465 | INFO | "huge_shared" : " 27 GB", | 20260505 23:11:12.465 | INFO | "malloc" : " 14 GB", | 20260505 23:11:12.465 | INFO | "pinned" : " 29 GB" | 20260505 23:11:12.465 | INFO | } | 20260505 23:11:12.465 | INFO | Perf report for the last 5.13776 seconds: | 20260505 23:11:12.465 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:11:12.465 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:12.465 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:12.465 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:12.465 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260505 23:11:12.465 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:12.465 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:12.465 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:11:12.466 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:12.466 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:12.466 | INFO | Uploaded 36GiB to GPU in 2.09163s (17.2115GiB/s) | 20260505 23:11:12.466 | INFO | Downloaded 34GiB to GPU in 2.04133s (16.6558GiB/s) | 20260505 23:11:17.611 | INFO | Analyzing chunk frames = [7051776,7052288) | 20260505 23:11:17.611 | INFO | MemInfo: { | 20260505 23:11:17.611 | INFO | "gpu" : " 17 GB", | 20260505 23:11:17.611 | INFO | "huge" : "", | 20260505 23:11:17.611 | INFO | "huge_shared" : " 27 GB", | 20260505 23:11:17.611 | INFO | "malloc" : " 14 GB", | 20260505 23:11:17.611 | INFO | "pinned" : " 29 GB" | 20260505 23:11:17.611 | INFO | } | 20260505 23:11:17.611 | INFO | Perf report for the last 5.14577 seconds: | 20260505 23:11:17.611 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:11:17.611 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:17.611 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:17.611 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:17.611 | INFO | ANALYSIS : 61.4%, 3.0, 95.9ms, 0 | 20260505 23:11:17.611 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:17.611 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:17.611 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260505 23:11:17.611 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:17.611 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:17.611 | INFO | Uploaded 36GiB to GPU in 2.07498s (17.3496GiB/s) | 20260505 23:11:17.611 | INFO | Downloaded 34GiB to GPU in 2.03668s (16.6938GiB/s) | 20260505 23:11:22.749 | INFO | Analyzing chunk frames = [7052288,7052800) | 20260505 23:11:22.749 | INFO | MemInfo: { | 20260505 23:11:22.749 | INFO | "gpu" : " 17 GB", | 20260505 23:11:22.749 | INFO | "huge" : "", | 20260505 23:11:22.749 | INFO | "huge_shared" : " 27 GB", | 20260505 23:11:22.749 | INFO | "malloc" : " 14 GB", | 20260505 23:11:22.749 | INFO | "pinned" : " 29 GB" | 20260505 23:11:22.749 | INFO | } | 20260505 23:11:22.749 | INFO | Perf report for the last 5.1379 seconds: | 20260505 23:11:22.749 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:11:22.749 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:22.749 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:22.749 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:22.749 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 23:11:22.749 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:22.749 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:22.749 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 23:11:22.749 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:22.749 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:22.749 | INFO | Uploaded 36GiB to GPU in 2.06626s (17.4228GiB/s) | 20260505 23:11:22.749 | INFO | Downloaded 34GiB to GPU in 2.06052s (16.5007GiB/s) | 20260505 23:11:27.880 | INFO | Analyzing chunk frames = [7052800,7053312) | 20260505 23:11:27.880 | INFO | MemInfo: { | 20260505 23:11:27.880 | INFO | "gpu" : " 17 GB", | 20260505 23:11:27.880 | INFO | "huge" : "", | 20260505 23:11:27.880 | INFO | "huge_shared" : " 27 GB", | 20260505 23:11:27.880 | INFO | "malloc" : " 14 GB", | 20260505 23:11:27.880 | INFO | "pinned" : " 29 GB" | 20260505 23:11:27.880 | INFO | } | 20260505 23:11:27.880 | INFO | Perf report for the last 5.13073 seconds: | 20260505 23:11:27.880 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:11:27.880 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:27.880 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:27.880 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:27.880 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260505 23:11:27.880 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:27.880 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:27.880 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260505 23:11:27.880 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:27.880 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:27.880 | INFO | Uploaded 36GiB to GPU in 2.11115s (17.0523GiB/s) | 20260505 23:11:27.880 | INFO | Downloaded 34GiB to GPU in 2.00142s (16.988GiB/s) | 20260505 23:11:27.880 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 23:11:27.880 | INFO | Total mem usage: 15.3 GiB | 20260505 23:11:27.880 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 23:11:27.880 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -9.03%) | 20260505 23:11:27.880 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 23:11:27.880 | INFO | High Water Breakdown: | 20260505 23:11:27.880 | INFO | 60.9% : 8.56 GiB ( 90% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 23:11:27.880 | INFO | 39% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 23:11:27.880 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 23:11:27.880 | INFO | ------------------------------------------ | 20260505 23:11:27.880 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:11:27.880 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 23:11:27.880 | INFO | Total mem usage: 26.5 GiB | 20260505 23:11:27.880 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 23:11:27.880 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 23:11:27.880 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 23:11:27.880 | INFO | High Water Breakdown: | 20260505 23:11:27.880 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:11:27.880 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:11:27.880 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:11:27.880 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:11:27.880 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:11:27.880 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:11:27.881 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 23:11:27.881 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 23:11:27.881 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:11:27.881 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 23:11:27.881 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 23:11:27.881 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 23:11:27.881 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 23:11:27.881 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 23:11:27.881 | INFO | ------------------------------------------ | 20260505 23:11:27.881 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:11:27.881 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 23:11:27.881 | INFO | Total mem usage: 25.6 GiB | 20260505 23:11:27.881 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 23:11:27.881 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 23:11:27.881 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 23:11:27.881 | INFO | High Water Breakdown: | 20260505 23:11:27.881 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 23:11:27.881 | INFO | ------------------------------------------ | 20260505 23:11:27.881 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:11:27.881 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 23:11:27.881 | INFO | Total mem usage: 15.7 GiB | 20260505 23:11:27.881 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 23:11:27.881 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 23:11:27.881 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 23:11:27.881 | INFO | High Water Breakdown: | 20260505 23:11:27.881 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 23:11:27.881 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 23:11:27.881 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 23:11:27.881 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 23:11:27.881 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 23:11:27.881 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 23:11:27.881 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 23:11:27.881 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 23:11:27.881 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 23:11:27.881 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:11:27.881 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 23:11:27.881 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:11:27.881 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 23:11:27.881 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 23:11:27.881 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 23:11:27.881 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 23:11:27.881 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:11:27.881 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 23:11:27.881 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:11:27.881 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:11:27.881 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:11:27.881 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 23:11:27.882 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 23:11:27.882 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:11:27.882 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 23:11:27.882 | INFO | ------------------------------------------ | 20260505 23:11:27.882 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:11:27.882 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 23:11:27.882 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:11:27.882 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:11:27.882 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.39e+01, 1.35e+01 / 8.14e-01 | 20260505 23:11:27.882 | INFO | PerfReport: 0.29%, AggregateStats -- 9.17e-02 / 1.04e-01, 1.02e-02 / 5.01e-03 | 20260505 23:11:27.882 | INFO | PerfReport: 11.82%, Baseline -- 3.70e+00 / 4.53e+00, 5.71e-01 / 1.67e-01 | 20260505 23:11:27.882 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.53e+00, 1.88e-01 / 8.80e-02 | 20260505 23:11:27.882 | INFO | PerfReport: 16.00%, DME -- 4.01e+01 / 4.39e+01, 3.85e+00 / 9.68e-01 | 20260505 23:11:27.882 | INFO | PerfReport: 57.24%, FrameLabeling -- 1.79e+01 / 1.94e+01, 1.45e+00 / 1.24e+00 | 20260505 23:11:27.882 | INFO | PerfReport: 7.16%, PulseAccumulating -- 2.24e+00 / 2.77e+00, 4.29e-01 / 2.42e-01 | 20260505 23:11:27.882 | INFO | PerfReport: 3.14%, Metrics -- 9.83e-01 / 1.28e+00, 2.67e-01 / 1.33e-01 | 20260505 23:11:27.882 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.40e-01 / 3.96e-01, 5.84e-02 / 3.04e-02 | 20260505 23:11:27.882 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 23:11:27.882 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:11:27.882 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:11:27.882 | INFO | PerfReport: Full Execution -- 9.86e+01 / -nan, 2.49e+01 / 0.00e+00 | 20260505 23:11:27.882 | INFO | PerfReport: 22.44%, Upload -- 2.21e+01 / 0.00e+00, 7.62e+00 / 0.00e+00 | 20260505 23:11:27.882 | INFO | PerfReport: 21.83%, Download -- 2.15e+01 / 0.00e+00, 7.64e+00 / 0.00e+00 | 20260505 23:11:27.882 | INFO | PerfReport: 31.77%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 23:11:27.882 | INFO | PerfReport: 2.21%, LockUpload -- 2.17e+00 / 0.00e+00, 7.98e+00 / 0.00e+00 | 20260505 23:11:27.882 | INFO | PerfReport: 1.30%, LockDownload -- 1.28e+00 / 0.00e+00, 2.81e+00 / 0.00e+00 | 20260505 23:11:27.882 | INFO | PerfReport: 20.44%, LockCompute -- 2.01e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 23:11:27.882 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 2.03e-02 / 0.00e+00 | 20260505 23:11:28.008 | INFO | Starting a round of Baz IO | 20260505 23:11:31.152 | INFO | Populating RTMetrics report | 20260505 23:11:31.152 | INFO | RTMetrics report populated | 20260505 23:11:31.152 | INFO | Opening tmp RTMetrics json file | 20260505 23:11:31.153 | INFO | Serializing RTMetrics to json | 20260505 23:11:31.154 | INFO | Serialization done, swapping tmp file now... | 20260505 23:11:31.155 | INFO | RTMetrics done | 20260505 23:11:31.231 | INFO | Finished a round of Baz IO: Wrote 1.904 GiB in 3.223 seconds | 20260505 23:11:31.231 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.45% / 0.00% / 78.55% | 20260505 23:11:31.231 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.111298% / 0% / 9.13938e-05% | 20260505 23:11:31.231 | INFO | Baz IO overall: | 20260505 23:11:31.231 | INFO | BytesWritten : 4040925293174 | 20260505 23:11:31.231 | INFO | NumEvents : 1904373755838 | 20260505 23:11:33.028 | INFO | Analyzing chunk frames = [7053312,7053824) | 20260505 23:11:33.028 | INFO | MemInfo: { | 20260505 23:11:33.028 | INFO | "gpu" : " 17 GB", | 20260505 23:11:33.028 | INFO | "huge" : "", | 20260505 23:11:33.028 | INFO | "huge_shared" : " 27 GB", | 20260505 23:11:33.028 | INFO | "malloc" : " 14 GB", | 20260505 23:11:33.028 | INFO | "pinned" : " 29 GB" | 20260505 23:11:33.028 | INFO | } | 20260505 23:11:33.028 | INFO | Perf report for the last 5.14831 seconds: | 20260505 23:11:33.028 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:11:33.028 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:33.028 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:33.028 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:33.028 | INFO | ANALYSIS : 62.5%, 3.0, 97.7ms, 0 | 20260505 23:11:33.028 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:33.028 | INFO | RT_METRICS : 43.4%, 1.0, 22.8ms, 0 | 20260505 23:11:33.028 | INFO | PRE_HQ : 59.5%, 1.8, 54.3ms, 0 | 20260505 23:11:33.028 | INFO | BAZWRITER : 61.9%, 3.6, 114.4ms, 0 | 20260505 23:11:33.028 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 23:11:33.028 | INFO | Uploaded 36GiB to GPU in 2.12919s (16.9078GiB/s) | 20260505 23:11:33.028 | INFO | Downloaded 38GiB to GPU in 2.2333s (17.0152GiB/s) | 20260505 23:11:38.164 | INFO | Analyzing chunk frames = [7053824,7054336) | 20260505 23:11:38.164 | INFO | MemInfo: { | 20260505 23:11:38.164 | INFO | "gpu" : " 17 GB", | 20260505 23:11:38.164 | INFO | "huge" : "", | 20260505 23:11:38.164 | INFO | "huge_shared" : " 27 GB", | 20260505 23:11:38.164 | INFO | "malloc" : " 14 GB", | 20260505 23:11:38.164 | INFO | "pinned" : " 29 GB" | 20260505 23:11:38.165 | INFO | } | 20260505 23:11:38.165 | INFO | Perf report for the last 5.13641 seconds: | 20260505 23:11:38.165 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:11:38.165 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:38.165 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:38.165 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:38.165 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 23:11:38.165 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:38.165 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:38.165 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260505 23:11:38.165 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:38.165 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:38.165 | INFO | Uploaded 36GiB to GPU in 2.08399s (17.2746GiB/s) | 20260505 23:11:38.165 | INFO | Downloaded 34GiB to GPU in 2.03515s (16.7064GiB/s) | 20260505 23:11:43.299 | INFO | Analyzing chunk frames = [7054336,7054848) | 20260505 23:11:43.299 | INFO | MemInfo: { | 20260505 23:11:43.299 | INFO | "gpu" : " 17 GB", | 20260505 23:11:43.299 | INFO | "huge" : "", | 20260505 23:11:43.299 | INFO | "huge_shared" : " 27 GB", | 20260505 23:11:43.299 | INFO | "malloc" : " 14 GB", | 20260505 23:11:43.299 | INFO | "pinned" : " 29 GB" | 20260505 23:11:43.299 | INFO | } | 20260505 23:11:43.299 | INFO | Perf report for the last 5.1348 seconds: | 20260505 23:11:43.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:11:43.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:43.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:43.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:43.299 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260505 23:11:43.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:43.300 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:43.300 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260505 23:11:43.300 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:43.300 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:43.300 | INFO | Uploaded 36GiB to GPU in 2.07575s (17.3431GiB/s) | 20260505 23:11:43.300 | INFO | Downloaded 34GiB to GPU in 2.02295s (16.8071GiB/s) | 20260505 23:11:48.437 | INFO | Analyzing chunk frames = [7054848,7055360) | 20260505 23:11:48.437 | INFO | MemInfo: { | 20260505 23:11:48.437 | INFO | "gpu" : " 17 GB", | 20260505 23:11:48.437 | INFO | "huge" : "", | 20260505 23:11:48.437 | INFO | "huge_shared" : " 27 GB", | 20260505 23:11:48.437 | INFO | "malloc" : " 14 GB", | 20260505 23:11:48.437 | INFO | "pinned" : " 29 GB" | 20260505 23:11:48.437 | INFO | } | 20260505 23:11:48.437 | INFO | Perf report for the last 5.13791 seconds: | 20260505 23:11:48.437 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:11:48.437 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:48.437 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:48.437 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:48.437 | INFO | ANALYSIS : 61.9%, 3.0, 96.1ms, 0 | 20260505 23:11:48.437 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:48.437 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:48.437 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 23:11:48.437 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:48.437 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:48.438 | INFO | Uploaded 36GiB to GPU in 2.13611s (16.853GiB/s) | 20260505 23:11:48.438 | INFO | Downloaded 34GiB to GPU in 2.03042s (16.7453GiB/s) | 20260505 23:11:53.583 | INFO | Analyzing chunk frames = [7055360,7055872) | 20260505 23:11:53.583 | INFO | MemInfo: { | 20260505 23:11:53.583 | INFO | "gpu" : " 17 GB", | 20260505 23:11:53.583 | INFO | "huge" : "", | 20260505 23:11:53.583 | INFO | "huge_shared" : " 27 GB", | 20260505 23:11:53.583 | INFO | "malloc" : " 14 GB", | 20260505 23:11:53.583 | INFO | "pinned" : " 29 GB" | 20260505 23:11:53.583 | INFO | } | 20260505 23:11:53.583 | INFO | Perf report for the last 5.14593 seconds: | 20260505 23:11:53.583 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:11:53.583 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:53.583 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:53.583 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:53.583 | INFO | ANALYSIS : 61.4%, 3.0, 96.0ms, 0 | 20260505 23:11:53.583 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:53.583 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:53.583 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:11:53.583 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:53.583 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:53.583 | INFO | Uploaded 36GiB to GPU in 2.13221s (16.8839GiB/s) | 20260505 23:11:53.583 | INFO | Downloaded 34GiB to GPU in 2.06183s (16.4902GiB/s) | 20260505 23:11:58.726 | INFO | Analyzing chunk frames = [7055872,7056384) | 20260505 23:11:58.726 | INFO | MemInfo: { | 20260505 23:11:58.726 | INFO | "gpu" : " 17 GB", | 20260505 23:11:58.726 | INFO | "huge" : "", | 20260505 23:11:58.726 | INFO | "huge_shared" : " 27 GB", | 20260505 23:11:58.726 | INFO | "malloc" : " 14 GB", | 20260505 23:11:58.726 | INFO | "pinned" : " 29 GB" | 20260505 23:11:58.726 | INFO | } | 20260505 23:11:58.726 | INFO | Perf report for the last 5.14257 seconds: | 20260505 23:11:58.726 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:11:58.726 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:58.726 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:58.726 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:58.726 | INFO | ANALYSIS : 61.3%, 3.0, 95.8ms, 0 | 20260505 23:11:58.726 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:58.726 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:58.726 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260505 23:11:58.726 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:58.726 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:11:58.726 | INFO | Uploaded 36GiB to GPU in 2.06818s (17.4066GiB/s) | 20260505 23:11:58.726 | INFO | Downloaded 34GiB to GPU in 2.03824s (16.6811GiB/s) | 20260505 23:12:03.857 | INFO | Analyzing chunk frames = [7056384,7056896) | 20260505 23:12:03.857 | INFO | MemInfo: { | 20260505 23:12:03.857 | INFO | "gpu" : " 17 GB", | 20260505 23:12:03.857 | INFO | "huge" : "", | 20260505 23:12:03.857 | INFO | "huge_shared" : " 27 GB", | 20260505 23:12:03.857 | INFO | "malloc" : " 14 GB", | 20260505 23:12:03.857 | INFO | "pinned" : " 29 GB" | 20260505 23:12:03.857 | INFO | } | 20260505 23:12:03.857 | INFO | Perf report for the last 5.13154 seconds: | 20260505 23:12:03.857 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:12:03.857 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:03.857 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:03.857 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:03.857 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260505 23:12:03.857 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:03.857 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:03.857 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 23:12:03.857 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:03.858 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:03.858 | INFO | Uploaded 36GiB to GPU in 2.11664s (17.0081GiB/s) | 20260505 23:12:03.858 | INFO | Downloaded 34GiB to GPU in 2.08034s (16.3435GiB/s) | 20260505 23:12:08.996 | INFO | Analyzing chunk frames = [7056896,7057408) | 20260505 23:12:08.996 | INFO | MemInfo: { | 20260505 23:12:08.996 | INFO | "gpu" : " 17 GB", | 20260505 23:12:08.996 | INFO | "huge" : "", | 20260505 23:12:08.996 | INFO | "huge_shared" : " 27 GB", | 20260505 23:12:08.996 | INFO | "malloc" : " 14 GB", | 20260505 23:12:08.996 | INFO | "pinned" : " 29 GB" | 20260505 23:12:08.996 | INFO | } | 20260505 23:12:08.996 | INFO | Perf report for the last 5.13861 seconds: | 20260505 23:12:08.996 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:12:08.996 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:08.996 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:08.996 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:08.996 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 23:12:08.996 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:08.996 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:08.996 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 23:12:08.996 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:08.996 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:08.996 | INFO | Uploaded 36GiB to GPU in 2.12309s (16.9564GiB/s) | 20260505 23:12:08.996 | INFO | Downloaded 34GiB to GPU in 2.00899s (16.924GiB/s) | 20260505 23:12:09.126 | INFO | Starting a round of Baz IO | 20260505 23:12:12.270 | INFO | Populating RTMetrics report | 20260505 23:12:12.270 | INFO | RTMetrics report populated | 20260505 23:12:12.270 | INFO | Opening tmp RTMetrics json file | 20260505 23:12:12.276 | INFO | Serializing RTMetrics to json | 20260505 23:12:12.283 | INFO | Serialization done, swapping tmp file now... | 20260505 23:12:12.294 | INFO | RTMetrics done | 20260505 23:12:12.351 | INFO | Finished a round of Baz IO: Wrote 1.904 GiB in 3.225 seconds | 20260505 23:12:12.351 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.45% / 0.00% / 78.55% | 20260505 23:12:12.351 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.111508% / 7.94729e-06% / 8.74201e-05% | 20260505 23:12:12.351 | INFO | Baz IO overall: | 20260505 23:12:12.351 | INFO | BytesWritten : 4042969314809 | 20260505 23:12:12.351 | INFO | NumEvents : 1905312505488 | 20260505 23:12:14.140 | INFO | Analyzing chunk frames = [7057408,7057920) | 20260505 23:12:14.140 | INFO | MemInfo: { | 20260505 23:12:14.140 | INFO | "gpu" : " 17 GB", | 20260505 23:12:14.140 | INFO | "huge" : "", | 20260505 23:12:14.140 | INFO | "huge_shared" : " 27 GB", | 20260505 23:12:14.140 | INFO | "malloc" : " 14 GB", | 20260505 23:12:14.140 | INFO | "pinned" : " 29 GB" | 20260505 23:12:14.140 | INFO | } | 20260505 23:12:14.140 | INFO | Perf report for the last 5.1443 seconds: | 20260505 23:12:14.140 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:12:14.140 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:14.140 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:14.140 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:14.140 | INFO | ANALYSIS : 62.6%, 3.0, 97.9ms, 0 | 20260505 23:12:14.140 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:14.140 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260505 23:12:14.140 | INFO | PRE_HQ : 59.7%, 1.7, 54.2ms, 0 | 20260505 23:12:14.140 | INFO | BAZWRITER : 62.0%, 3.6, 114.5ms, 0 | 20260505 23:12:14.140 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 23:12:14.141 | INFO | Uploaded 36GiB to GPU in 2.1477s (16.7621GiB/s) | 20260505 23:12:14.141 | INFO | Downloaded 38GiB to GPU in 2.2368s (16.9885GiB/s) | 20260505 23:12:19.277 | INFO | Analyzing chunk frames = [7057920,7058432) | 20260505 23:12:19.277 | INFO | MemInfo: { | 20260505 23:12:19.277 | INFO | "gpu" : " 17 GB", | 20260505 23:12:19.277 | INFO | "huge" : "", | 20260505 23:12:19.277 | INFO | "huge_shared" : " 27 GB", | 20260505 23:12:19.277 | INFO | "malloc" : " 14 GB", | 20260505 23:12:19.277 | INFO | "pinned" : " 29 GB" | 20260505 23:12:19.277 | INFO | } | 20260505 23:12:19.277 | INFO | Perf report for the last 5.13664 seconds: | 20260505 23:12:19.277 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:12:19.277 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:19.277 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:19.277 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:19.277 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 23:12:19.277 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:19.277 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:19.277 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:12:19.277 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:19.277 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:19.277 | INFO | Uploaded 36GiB to GPU in 2.09965s (17.1458GiB/s) | 20260505 23:12:19.277 | INFO | Downloaded 34GiB to GPU in 2.03557s (16.7029GiB/s) | 20260505 23:12:24.421 | INFO | Analyzing chunk frames = [7058432,7058944) | 20260505 23:12:24.421 | INFO | MemInfo: { | 20260505 23:12:24.421 | INFO | "gpu" : " 17 GB", | 20260505 23:12:24.421 | INFO | "huge" : "", | 20260505 23:12:24.421 | INFO | "huge_shared" : " 27 GB", | 20260505 23:12:24.421 | INFO | "malloc" : " 14 GB", | 20260505 23:12:24.421 | INFO | "pinned" : " 29 GB" | 20260505 23:12:24.421 | INFO | } | 20260505 23:12:24.421 | INFO | Perf report for the last 5.14428 seconds: | 20260505 23:12:24.421 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:12:24.421 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:24.421 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:24.421 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:24.421 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260505 23:12:24.421 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:24.421 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:24.421 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:12:24.421 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:24.421 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:24.422 | INFO | Uploaded 36GiB to GPU in 2.15552s (16.7013GiB/s) | 20260505 23:12:24.422 | INFO | Downloaded 34GiB to GPU in 2.05548s (16.5412GiB/s) | 20260505 23:12:29.555 | INFO | Analyzing chunk frames = [7058944,7059456) | 20260505 23:12:29.555 | INFO | MemInfo: { | 20260505 23:12:29.555 | INFO | "gpu" : " 17 GB", | 20260505 23:12:29.555 | INFO | "huge" : "", | 20260505 23:12:29.555 | INFO | "huge_shared" : " 27 GB", | 20260505 23:12:29.555 | INFO | "malloc" : " 14 GB", | 20260505 23:12:29.555 | INFO | "pinned" : " 29 GB" | 20260505 23:12:29.555 | INFO | } | 20260505 23:12:29.555 | INFO | Perf report for the last 5.13391 seconds: | 20260505 23:12:29.555 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:12:29.555 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:29.555 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:29.555 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:29.555 | INFO | ANALYSIS : 61.7%, 3.0, 95.8ms, 0 | 20260505 23:12:29.555 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:29.555 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:29.555 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:12:29.555 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:29.555 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:29.556 | INFO | Uploaded 36GiB to GPU in 2.1486s (16.7551GiB/s) | 20260505 23:12:29.556 | INFO | Downloaded 34GiB to GPU in 2.02326s (16.8046GiB/s) | 20260505 23:12:34.695 | INFO | Analyzing chunk frames = [7059456,7059968) | 20260505 23:12:34.695 | INFO | MemInfo: { | 20260505 23:12:34.695 | INFO | "gpu" : " 17 GB", | 20260505 23:12:34.695 | INFO | "huge" : "", | 20260505 23:12:34.695 | INFO | "huge_shared" : " 27 GB", | 20260505 23:12:34.695 | INFO | "malloc" : " 14 GB", | 20260505 23:12:34.695 | INFO | "pinned" : " 29 GB" | 20260505 23:12:34.695 | INFO | } | 20260505 23:12:34.695 | INFO | Perf report for the last 5.14006 seconds: | 20260505 23:12:34.695 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:12:34.695 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:34.695 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:34.695 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:34.695 | INFO | ANALYSIS : 61.5%, 3.0, 96.1ms, 0 | 20260505 23:12:34.695 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:34.695 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:34.695 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:12:34.695 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:34.695 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:34.695 | INFO | Uploaded 36GiB to GPU in 2.15817s (16.6808GiB/s) | 20260505 23:12:34.696 | INFO | Downloaded 34GiB to GPU in 2.06856s (16.4365GiB/s) | 20260505 23:12:39.836 | INFO | Analyzing chunk frames = [7059968,7060480) | 20260505 23:12:39.836 | INFO | MemInfo: { | 20260505 23:12:39.836 | INFO | "gpu" : " 17 GB", | 20260505 23:12:39.836 | INFO | "huge" : "", | 20260505 23:12:39.836 | INFO | "huge_shared" : " 27 GB", | 20260505 23:12:39.836 | INFO | "malloc" : " 14 GB", | 20260505 23:12:39.836 | INFO | "pinned" : " 29 GB" | 20260505 23:12:39.836 | INFO | } | 20260505 23:12:39.837 | INFO | Perf report for the last 5.14126 seconds: | 20260505 23:12:39.837 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:12:39.837 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:39.837 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:39.837 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:39.837 | INFO | ANALYSIS : 61.4%, 3.0, 95.9ms, 0 | 20260505 23:12:39.837 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:39.837 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:39.837 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:12:39.837 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:39.837 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:39.837 | INFO | Uploaded 36GiB to GPU in 2.08838s (17.2382GiB/s) | 20260505 23:12:39.837 | INFO | Downloaded 34GiB to GPU in 2.04321s (16.6405GiB/s) | 20260505 23:12:44.977 | INFO | Analyzing chunk frames = [7060480,7060992) | 20260505 23:12:44.977 | INFO | MemInfo: { | 20260505 23:12:44.977 | INFO | "gpu" : " 17 GB", | 20260505 23:12:44.977 | INFO | "huge" : "", | 20260505 23:12:44.977 | INFO | "huge_shared" : " 27 GB", | 20260505 23:12:44.977 | INFO | "malloc" : " 14 GB", | 20260505 23:12:44.977 | INFO | "pinned" : " 29 GB" | 20260505 23:12:44.977 | INFO | } | 20260505 23:12:44.978 | INFO | Perf report for the last 5.14116 seconds: | 20260505 23:12:44.978 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:12:44.978 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:44.978 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:44.978 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:44.978 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 23:12:44.978 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:44.978 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:44.978 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 23:12:44.978 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:44.978 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:44.978 | INFO | Uploaded 36GiB to GPU in 2.10024s (17.1409GiB/s) | 20260505 23:12:44.978 | INFO | Downloaded 34GiB to GPU in 2.07107s (16.4166GiB/s) | 20260505 23:12:50.115 | INFO | Analyzing chunk frames = [7060992,7061504) | 20260505 23:12:50.115 | INFO | MemInfo: { | 20260505 23:12:50.115 | INFO | "gpu" : " 17 GB", | 20260505 23:12:50.115 | INFO | "huge" : "", | 20260505 23:12:50.115 | INFO | "huge_shared" : " 27 GB", | 20260505 23:12:50.115 | INFO | "malloc" : " 14 GB", | 20260505 23:12:50.115 | INFO | "pinned" : " 29 GB" | 20260505 23:12:50.115 | INFO | } | 20260505 23:12:50.115 | INFO | Perf report for the last 5.13735 seconds: | 20260505 23:12:50.115 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:12:50.115 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:50.115 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:50.115 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:50.115 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 23:12:50.115 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:50.115 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:50.115 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:12:50.115 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:50.115 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:50.115 | INFO | Uploaded 36GiB to GPU in 2.11094s (17.054GiB/s) | 20260505 23:12:50.115 | INFO | Downloaded 34GiB to GPU in 2.00494s (16.9581GiB/s) | 20260505 23:12:50.116 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 23:12:50.116 | INFO | Total mem usage: 15.3 GiB | 20260505 23:12:50.116 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 23:12:50.116 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -9.07%) | 20260505 23:12:50.116 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 23:12:50.116 | INFO | High Water Breakdown: | 20260505 23:12:50.116 | INFO | 60.9% : 8.56 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 23:12:50.116 | INFO | 39% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 23:12:50.116 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 23:12:50.116 | INFO | ------------------------------------------ | 20260505 23:12:50.116 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:12:50.116 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 23:12:50.116 | INFO | Total mem usage: 26.5 GiB | 20260505 23:12:50.116 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 23:12:50.116 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 23:12:50.116 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260505 23:12:50.116 | INFO | High Water Breakdown: | 20260505 23:12:50.116 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:12:50.116 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:12:50.116 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:12:50.116 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:12:50.116 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:12:50.116 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:12:50.116 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 23:12:50.116 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 23:12:50.116 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:12:50.116 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 23:12:50.116 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 23:12:50.116 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 23:12:50.116 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 23:12:50.116 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 23:12:50.116 | INFO | ------------------------------------------ | 20260505 23:12:50.116 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:12:50.116 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 23:12:50.116 | INFO | Total mem usage: 25.6 GiB | 20260505 23:12:50.116 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 23:12:50.116 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 23:12:50.116 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 23:12:50.116 | INFO | High Water Breakdown: | 20260505 23:12:50.116 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 23:12:50.116 | INFO | ------------------------------------------ | 20260505 23:12:50.116 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:12:50.116 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 23:12:50.116 | INFO | Total mem usage: 15.7 GiB | 20260505 23:12:50.116 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 23:12:50.116 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 23:12:50.116 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 23:12:50.116 | INFO | High Water Breakdown: | 20260505 23:12:50.116 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 23:12:50.116 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 23:12:50.116 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 23:12:50.116 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 23:12:50.116 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 23:12:50.116 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 23:12:50.116 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 23:12:50.116 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 23:12:50.116 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 23:12:50.116 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:12:50.116 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 23:12:50.116 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:12:50.116 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 23:12:50.116 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 23:12:50.116 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 23:12:50.116 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 23:12:50.116 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:12:50.116 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 23:12:50.116 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:12:50.116 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:12:50.116 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:12:50.116 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 23:12:50.116 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 23:12:50.116 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:12:50.116 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 23:12:50.116 | INFO | ------------------------------------------ | 20260505 23:12:50.116 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:12:50.116 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 23:12:50.116 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:12:50.117 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:12:50.117 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.39e+01, 1.34e+01 / 8.41e-01 | 20260505 23:12:50.117 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.05e-01, 1.04e-02 / 7.44e-03 | 20260505 23:12:50.117 | INFO | PerfReport: 11.82%, Baseline -- 3.70e+00 / 4.50e+00, 5.68e-01 / 1.73e-01 | 20260505 23:12:50.117 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.51e+00, 1.88e-01 / 7.67e-02 | 20260505 23:12:50.117 | INFO | PerfReport: 15.96%, DME -- 4.00e+01 / 4.40e+01, 3.85e+00 / 1.06e+00 | 20260505 23:12:50.117 | INFO | PerfReport: 57.30%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.49e+00 / 8.83e-01 | 20260505 23:12:50.117 | INFO | PerfReport: 7.14%, PulseAccumulating -- 2.24e+00 / 2.77e+00, 4.27e-01 / 2.07e-01 | 20260505 23:12:50.117 | INFO | PerfReport: 3.13%, Metrics -- 9.80e-01 / 1.28e+00, 2.66e-01 / 1.15e-01 | 20260505 23:12:50.117 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.40e-01 / 3.96e-01, 5.85e-02 / 2.97e-02 | 20260505 23:12:50.117 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 23:12:50.117 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:12:50.117 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:12:50.117 | INFO | PerfReport: Full Execution -- 9.85e+01 / -nan, 2.49e+01 / 0.00e+00 | 20260505 23:12:50.117 | INFO | PerfReport: 22.38%, Upload -- 2.21e+01 / 0.00e+00, 7.55e+00 / 0.00e+00 | 20260505 23:12:50.117 | INFO | PerfReport: 21.84%, Download -- 2.15e+01 / 0.00e+00, 7.61e+00 / 0.00e+00 | 20260505 23:12:50.117 | INFO | PerfReport: 31.80%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 23:12:50.117 | INFO | PerfReport: 2.14%, LockUpload -- 2.10e+00 / 0.00e+00, 7.79e+00 / 0.00e+00 | 20260505 23:12:50.117 | INFO | PerfReport: 1.35%, LockDownload -- 1.33e+00 / 0.00e+00, 2.94e+00 / 0.00e+00 | 20260505 23:12:50.117 | INFO | PerfReport: 20.48%, LockCompute -- 2.02e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260505 23:12:50.117 | INFO | PerfReport: 0.01%, Misc -- 1.27e-02 / 0.00e+00, 1.43e-02 / 0.00e+00 | 20260505 23:12:50.243 | INFO | Starting a round of Baz IO | 20260505 23:12:53.388 | INFO | Populating RTMetrics report | 20260505 23:12:53.388 | INFO | RTMetrics report populated | 20260505 23:12:53.388 | INFO | Opening tmp RTMetrics json file | 20260505 23:12:53.389 | INFO | Serializing RTMetrics to json | 20260505 23:12:53.390 | INFO | Serialization done, swapping tmp file now... | 20260505 23:12:53.390 | INFO | RTMetrics done | 20260505 23:12:53.466 | INFO | Finished a round of Baz IO: Wrote 1.903 GiB in 3.223 seconds | 20260505 23:12:53.467 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.44% / 0.00% / 78.55% | 20260505 23:12:53.467 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.111421% / 0% / 0.000107288% | 20260505 23:12:53.467 | INFO | Baz IO overall: | 20260505 23:12:53.467 | INFO | BytesWritten : 4045012689274 | 20260505 23:12:53.467 | INFO | NumEvents : 1906250679056 | 20260505 23:12:55.252 | INFO | Analyzing chunk frames = [7061504,7062016) | 20260505 23:12:55.252 | INFO | MemInfo: { | 20260505 23:12:55.252 | INFO | "gpu" : " 17 GB", | 20260505 23:12:55.252 | INFO | "huge" : "", | 20260505 23:12:55.252 | INFO | "huge_shared" : " 27 GB", | 20260505 23:12:55.252 | INFO | "malloc" : " 14 GB", | 20260505 23:12:55.252 | INFO | "pinned" : " 29 GB" | 20260505 23:12:55.252 | INFO | } | 20260505 23:12:55.252 | INFO | Perf report for the last 5.13703 seconds: | 20260505 23:12:55.252 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:12:55.252 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:55.252 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:55.252 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:55.252 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 23:12:55.252 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:12:55.252 | INFO | RT_METRICS : 43.4%, 1.0, 22.7ms, 0 | 20260505 23:12:55.252 | INFO | PRE_HQ : 59.8%, 1.7, 54.1ms, 0 | 20260505 23:12:55.252 | INFO | BAZWRITER : 62.0%, 3.6, 114.0ms, 0 | 20260505 23:12:55.252 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 23:12:55.252 | INFO | Uploaded 36GiB to GPU in 2.15217s (16.7273GiB/s) | 20260505 23:12:55.252 | INFO | Downloaded 38GiB to GPU in 2.23032s (17.0379GiB/s) | 20260505 23:13:00.390 | INFO | Analyzing chunk frames = [7062016,7062528) | 20260505 23:13:00.390 | INFO | MemInfo: { | 20260505 23:13:00.390 | INFO | "gpu" : " 17 GB", | 20260505 23:13:00.390 | INFO | "huge" : "", | 20260505 23:13:00.390 | INFO | "huge_shared" : " 27 GB", | 20260505 23:13:00.390 | INFO | "malloc" : " 14 GB", | 20260505 23:13:00.390 | INFO | "pinned" : " 29 GB" | 20260505 23:13:00.390 | INFO | } | 20260505 23:13:00.390 | INFO | Perf report for the last 5.13778 seconds: | 20260505 23:13:00.390 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:13:00.390 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:00.390 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:00.390 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:00.390 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 23:13:00.390 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:00.390 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:00.390 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:13:00.390 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:00.390 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:00.390 | INFO | Uploaded 36GiB to GPU in 2.09447s (17.1882GiB/s) | 20260505 23:13:00.390 | INFO | Downloaded 34GiB to GPU in 2.03063s (16.7436GiB/s) | 20260505 23:13:05.532 | INFO | Analyzing chunk frames = [7062528,7063040) | 20260505 23:13:05.532 | INFO | MemInfo: { | 20260505 23:13:05.532 | INFO | "gpu" : " 17 GB", | 20260505 23:13:05.532 | INFO | "huge" : "", | 20260505 23:13:05.532 | INFO | "huge_shared" : " 27 GB", | 20260505 23:13:05.532 | INFO | "malloc" : " 14 GB", | 20260505 23:13:05.532 | INFO | "pinned" : " 29 GB" | 20260505 23:13:05.532 | INFO | } | 20260505 23:13:05.532 | INFO | Perf report for the last 5.14251 seconds: | 20260505 23:13:05.532 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:13:05.532 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:05.532 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:05.532 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:05.532 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260505 23:13:05.532 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:05.532 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:05.532 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:13:05.532 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:05.532 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:05.533 | INFO | Uploaded 36GiB to GPU in 2.15795s (16.6825GiB/s) | 20260505 23:13:05.533 | INFO | Downloaded 34GiB to GPU in 2.05268s (16.5637GiB/s) | 20260505 23:13:10.675 | INFO | Analyzing chunk frames = [7063040,7063552) | 20260505 23:13:10.675 | INFO | MemInfo: { | 20260505 23:13:10.675 | INFO | "gpu" : " 17 GB", | 20260505 23:13:10.675 | INFO | "huge" : "", | 20260505 23:13:10.675 | INFO | "huge_shared" : " 27 GB", | 20260505 23:13:10.675 | INFO | "malloc" : " 14 GB", | 20260505 23:13:10.675 | INFO | "pinned" : " 29 GB" | 20260505 23:13:10.675 | INFO | } | 20260505 23:13:10.676 | INFO | Perf report for the last 5.14309 seconds: | 20260505 23:13:10.676 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:13:10.676 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:10.676 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:10.676 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:10.676 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260505 23:13:10.676 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:10.676 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:10.676 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 23:13:10.676 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:10.676 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:10.676 | INFO | Uploaded 36GiB to GPU in 2.13718s (16.8447GiB/s) | 20260505 23:13:10.676 | INFO | Downloaded 34GiB to GPU in 2.01618s (16.8636GiB/s) | 20260505 23:13:15.805 | INFO | Analyzing chunk frames = [7063552,7064064) | 20260505 23:13:15.805 | INFO | MemInfo: { | 20260505 23:13:15.805 | INFO | "gpu" : " 17 GB", | 20260505 23:13:15.805 | INFO | "huge" : "", | 20260505 23:13:15.805 | INFO | "huge_shared" : " 27 GB", | 20260505 23:13:15.805 | INFO | "malloc" : " 14 GB", | 20260505 23:13:15.805 | INFO | "pinned" : " 29 GB" | 20260505 23:13:15.805 | INFO | } | 20260505 23:13:15.805 | INFO | Perf report for the last 5.12966 seconds: | 20260505 23:13:15.805 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:13:15.805 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:15.805 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:15.805 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:15.805 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260505 23:13:15.805 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:15.805 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:15.805 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:13:15.805 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:15.805 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:15.805 | INFO | Uploaded 36GiB to GPU in 2.08407s (17.2739GiB/s) | 20260505 23:13:15.805 | INFO | Downloaded 34GiB to GPU in 2.04114s (16.6574GiB/s) | 20260505 23:13:20.946 | INFO | Analyzing chunk frames = [7064064,7064576) | 20260505 23:13:20.946 | INFO | MemInfo: { | 20260505 23:13:20.946 | INFO | "gpu" : " 17 GB", | 20260505 23:13:20.946 | INFO | "huge" : "", | 20260505 23:13:20.946 | INFO | "huge_shared" : " 27 GB", | 20260505 23:13:20.946 | INFO | "malloc" : " 14 GB", | 20260505 23:13:20.946 | INFO | "pinned" : " 29 GB" | 20260505 23:13:20.946 | INFO | } | 20260505 23:13:20.946 | INFO | Perf report for the last 5.14051 seconds: | 20260505 23:13:20.946 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:13:20.946 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:20.946 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:20.946 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:20.946 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260505 23:13:20.946 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:20.946 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:20.946 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 23:13:20.946 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:20.946 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:20.946 | INFO | Uploaded 36GiB to GPU in 2.08441s (17.271GiB/s) | 20260505 23:13:20.946 | INFO | Downloaded 34GiB to GPU in 2.03923s (16.6729GiB/s) | 20260505 23:13:26.092 | INFO | Analyzing chunk frames = [7064576,7065088) | 20260505 23:13:26.092 | INFO | MemInfo: { | 20260505 23:13:26.093 | INFO | "gpu" : " 17 GB", | 20260505 23:13:26.093 | INFO | "huge" : "", | 20260505 23:13:26.093 | INFO | "huge_shared" : " 27 GB", | 20260505 23:13:26.093 | INFO | "malloc" : " 14 GB", | 20260505 23:13:26.093 | INFO | "pinned" : " 29 GB" | 20260505 23:13:26.093 | INFO | } | 20260505 23:13:26.093 | INFO | Perf report for the last 5.14684 seconds: | 20260505 23:13:26.093 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:13:26.093 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:26.093 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:26.093 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:26.093 | INFO | ANALYSIS : 62.8%, 3.0, 98.2ms, 0 | 20260505 23:13:26.093 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:26.093 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:26.093 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 23:13:26.093 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:26.093 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:26.093 | INFO | Uploaded 36GiB to GPU in 2.16872s (16.5996GiB/s) | 20260505 23:13:26.093 | INFO | Downloaded 34GiB to GPU in 2.13275s (15.9418GiB/s) | 20260505 23:13:31.235 | INFO | Analyzing chunk frames = [7065088,7065600) | 20260505 23:13:31.235 | INFO | MemInfo: { | 20260505 23:13:31.235 | INFO | "gpu" : " 17 GB", | 20260505 23:13:31.235 | INFO | "huge" : "", | 20260505 23:13:31.235 | INFO | "huge_shared" : " 27 GB", | 20260505 23:13:31.235 | INFO | "malloc" : " 14 GB", | 20260505 23:13:31.235 | INFO | "pinned" : " 29 GB" | 20260505 23:13:31.235 | INFO | } | 20260505 23:13:31.235 | INFO | Perf report for the last 5.14211 seconds: | 20260505 23:13:31.235 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:13:31.235 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:31.235 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:31.235 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:31.235 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260505 23:13:31.235 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:31.235 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:31.235 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 23:13:31.235 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:31.235 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:31.235 | INFO | Uploaded 36GiB to GPU in 2.13553s (16.8577GiB/s) | 20260505 23:13:31.235 | INFO | Downloaded 34GiB to GPU in 2.01676s (16.8588GiB/s) | 20260505 23:13:31.364 | INFO | Starting a round of Baz IO | 20260505 23:13:34.501 | INFO | Populating RTMetrics report | 20260505 23:13:34.501 | INFO | RTMetrics report populated | 20260505 23:13:34.501 | INFO | Opening tmp RTMetrics json file | 20260505 23:13:34.504 | INFO | Serializing RTMetrics to json | 20260505 23:13:34.507 | INFO | Serialization done, swapping tmp file now... | 20260505 23:13:34.508 | INFO | RTMetrics done | 20260505 23:13:34.580 | INFO | Finished a round of Baz IO: Wrote 1.904 GiB in 3.216 seconds | 20260505 23:13:34.580 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.44% / 0.00% / 78.56% | 20260505 23:13:34.580 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.110892% / 0% / 9.53674e-05% | 20260505 23:13:34.580 | INFO | Baz IO overall: | 20260505 23:13:34.580 | INFO | BytesWritten : 4047057228687 | 20260505 23:13:34.580 | INFO | NumEvents : 1907189772545 | 20260505 23:13:36.373 | INFO | Analyzing chunk frames = [7065600,7066112) | 20260505 23:13:36.373 | INFO | MemInfo: { | 20260505 23:13:36.373 | INFO | "gpu" : " 17 GB", | 20260505 23:13:36.373 | INFO | "huge" : "", | 20260505 23:13:36.373 | INFO | "huge_shared" : " 27 GB", | 20260505 23:13:36.373 | INFO | "malloc" : " 14 GB", | 20260505 23:13:36.373 | INFO | "pinned" : " 29 GB" | 20260505 23:13:36.373 | INFO | } | 20260505 23:13:36.373 | INFO | Perf report for the last 5.13837 seconds: | 20260505 23:13:36.373 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:13:36.373 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:36.373 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:36.373 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:36.373 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260505 23:13:36.373 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:36.373 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260505 23:13:36.373 | INFO | PRE_HQ : 59.8%, 1.7, 54.2ms, 0 | 20260505 23:13:36.373 | INFO | BAZWRITER : 62.6%, 3.6, 114.7ms, 0 | 20260505 23:13:36.373 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 23:13:36.373 | INFO | Uploaded 36GiB to GPU in 2.1637s (16.6382GiB/s) | 20260505 23:13:36.373 | INFO | Downloaded 38GiB to GPU in 2.23679s (16.9886GiB/s) | 20260505 23:13:41.505 | INFO | Analyzing chunk frames = [7066112,7066624) | 20260505 23:13:41.505 | INFO | MemInfo: { | 20260505 23:13:41.505 | INFO | "gpu" : " 17 GB", | 20260505 23:13:41.505 | INFO | "huge" : "", | 20260505 23:13:41.505 | INFO | "huge_shared" : " 27 GB", | 20260505 23:13:41.505 | INFO | "malloc" : " 14 GB", | 20260505 23:13:41.505 | INFO | "pinned" : " 29 GB" | 20260505 23:13:41.505 | INFO | } | 20260505 23:13:41.505 | INFO | Perf report for the last 5.13176 seconds: | 20260505 23:13:41.505 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:13:41.505 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:41.505 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:41.505 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:41.505 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260505 23:13:41.505 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:41.505 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:41.505 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:13:41.505 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:41.505 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:41.505 | INFO | Uploaded 36GiB to GPU in 2.11027s (17.0594GiB/s) | 20260505 23:13:41.505 | INFO | Downloaded 34GiB to GPU in 2.03747s (16.6874GiB/s) | 20260505 23:13:46.648 | INFO | Analyzing chunk frames = [7066624,7067136) | 20260505 23:13:46.648 | INFO | MemInfo: { | 20260505 23:13:46.648 | INFO | "gpu" : " 17 GB", | 20260505 23:13:46.648 | INFO | "huge" : "", | 20260505 23:13:46.648 | INFO | "huge_shared" : " 27 GB", | 20260505 23:13:46.648 | INFO | "malloc" : " 14 GB", | 20260505 23:13:46.648 | INFO | "pinned" : " 29 GB" | 20260505 23:13:46.648 | INFO | } | 20260505 23:13:46.648 | INFO | Perf report for the last 5.1434 seconds: | 20260505 23:13:46.648 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:13:46.648 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:46.648 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:46.648 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:46.648 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260505 23:13:46.648 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:46.648 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:46.648 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 23:13:46.649 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:46.649 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:46.649 | INFO | Uploaded 36GiB to GPU in 2.12264s (16.96GiB/s) | 20260505 23:13:46.649 | INFO | Downloaded 34GiB to GPU in 2.01291s (16.891GiB/s) | 20260505 23:13:51.793 | INFO | Analyzing chunk frames = [7067136,7067648) | 20260505 23:13:51.793 | INFO | MemInfo: { | 20260505 23:13:51.793 | INFO | "gpu" : " 17 GB", | 20260505 23:13:51.793 | INFO | "huge" : "", | 20260505 23:13:51.793 | INFO | "huge_shared" : " 27 GB", | 20260505 23:13:51.793 | INFO | "malloc" : " 14 GB", | 20260505 23:13:51.793 | INFO | "pinned" : " 29 GB" | 20260505 23:13:51.793 | INFO | } | 20260505 23:13:51.793 | INFO | Perf report for the last 5.14454 seconds: | 20260505 23:13:51.793 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:13:51.793 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:51.793 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:51.793 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:51.793 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260505 23:13:51.793 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:51.793 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:51.793 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:13:51.793 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:51.793 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:51.793 | INFO | Uploaded 36GiB to GPU in 2.14515s (16.782GiB/s) | 20260505 23:13:51.793 | INFO | Downloaded 34GiB to GPU in 2.0196s (16.835GiB/s) | 20260505 23:13:56.930 | INFO | Analyzing chunk frames = [7067648,7068160) | 20260505 23:13:56.930 | INFO | MemInfo: { | 20260505 23:13:56.930 | INFO | "gpu" : " 17 GB", | 20260505 23:13:56.930 | INFO | "huge" : "", | 20260505 23:13:56.930 | INFO | "huge_shared" : " 27 GB", | 20260505 23:13:56.930 | INFO | "malloc" : " 14 GB", | 20260505 23:13:56.930 | INFO | "pinned" : " 29 GB" | 20260505 23:13:56.930 | INFO | } | 20260505 23:13:56.930 | INFO | Perf report for the last 5.13696 seconds: | 20260505 23:13:56.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:13:56.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:56.930 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:56.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:56.930 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260505 23:13:56.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:56.930 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:56.930 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260505 23:13:56.930 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:56.930 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:13:56.930 | INFO | Uploaded 36GiB to GPU in 2.09005s (17.2245GiB/s) | 20260505 23:13:56.930 | INFO | Downloaded 34GiB to GPU in 2.04462s (16.629GiB/s) | 20260505 23:14:02.064 | INFO | Analyzing chunk frames = [7068160,7068672) | 20260505 23:14:02.064 | INFO | MemInfo: { | 20260505 23:14:02.064 | INFO | "gpu" : " 17 GB", | 20260505 23:14:02.064 | INFO | "huge" : "", | 20260505 23:14:02.064 | INFO | "huge_shared" : " 27 GB", | 20260505 23:14:02.064 | INFO | "malloc" : " 14 GB", | 20260505 23:14:02.064 | INFO | "pinned" : " 29 GB" | 20260505 23:14:02.064 | INFO | } | 20260505 23:14:02.065 | INFO | Perf report for the last 5.13483 seconds: | 20260505 23:14:02.065 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:14:02.065 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:02.065 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:02.065 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:02.065 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260505 23:14:02.065 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:02.065 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:02.065 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 23:14:02.065 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:02.065 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:02.065 | INFO | Uploaded 36GiB to GPU in 2.07741s (17.3292GiB/s) | 20260505 23:14:02.065 | INFO | Downloaded 34GiB to GPU in 2.03904s (16.6745GiB/s) | 20260505 23:14:07.202 | INFO | Analyzing chunk frames = [7068672,7069184) | 20260505 23:14:07.202 | INFO | MemInfo: { | 20260505 23:14:07.202 | INFO | "gpu" : " 17 GB", | 20260505 23:14:07.202 | INFO | "huge" : "", | 20260505 23:14:07.202 | INFO | "huge_shared" : " 27 GB", | 20260505 23:14:07.202 | INFO | "malloc" : " 14 GB", | 20260505 23:14:07.202 | INFO | "pinned" : " 29 GB" | 20260505 23:14:07.202 | INFO | } | 20260505 23:14:07.202 | INFO | Perf report for the last 5.13747 seconds: | 20260505 23:14:07.202 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:14:07.202 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:07.202 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:07.202 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:07.202 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 23:14:07.202 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:07.202 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:07.202 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260505 23:14:07.202 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:07.202 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:07.202 | INFO | Uploaded 36GiB to GPU in 2.14744s (16.7641GiB/s) | 20260505 23:14:07.202 | INFO | Downloaded 34GiB to GPU in 2.11331s (16.0885GiB/s) | 20260505 23:14:12.348 | INFO | Analyzing chunk frames = [7069184,7069696) | 20260505 23:14:12.348 | INFO | MemInfo: { | 20260505 23:14:12.348 | INFO | "gpu" : " 17 GB", | 20260505 23:14:12.348 | INFO | "huge" : "", | 20260505 23:14:12.348 | INFO | "huge_shared" : " 27 GB", | 20260505 23:14:12.348 | INFO | "malloc" : " 14 GB", | 20260505 23:14:12.348 | INFO | "pinned" : " 29 GB" | 20260505 23:14:12.348 | INFO | } | 20260505 23:14:12.348 | INFO | Perf report for the last 5.14587 seconds: | 20260505 23:14:12.348 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:14:12.348 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:12.348 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:12.348 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:12.348 | INFO | ANALYSIS : 61.9%, 3.0, 96.8ms, 0 | 20260505 23:14:12.348 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:12.348 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:12.348 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 23:14:12.348 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:12.348 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:12.348 | INFO | Uploaded 36GiB to GPU in 2.11716s (17.0039GiB/s) | 20260505 23:14:12.348 | INFO | Downloaded 34GiB to GPU in 2.00064s (16.9945GiB/s) | 20260505 23:14:12.348 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 23:14:12.348 | INFO | Total mem usage: 15.3 GiB | 20260505 23:14:12.348 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 23:14:12.348 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.1%) | 20260505 23:14:12.348 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 23:14:12.348 | INFO | High Water Breakdown: | 20260505 23:14:12.348 | INFO | 60.9% : 8.55 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 23:14:12.348 | INFO | 39% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 23:14:12.348 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 23:14:12.348 | INFO | ------------------------------------------ | 20260505 23:14:12.349 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:14:12.349 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 23:14:12.349 | INFO | Total mem usage: 26.5 GiB | 20260505 23:14:12.349 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 23:14:12.349 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 23:14:12.349 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 23:14:12.349 | INFO | High Water Breakdown: | 20260505 23:14:12.349 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:14:12.349 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:14:12.349 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:14:12.349 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:14:12.349 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:14:12.349 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:14:12.349 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 23:14:12.349 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 23:14:12.349 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:14:12.349 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 23:14:12.349 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 23:14:12.349 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 23:14:12.349 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 23:14:12.349 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 23:14:12.349 | INFO | ------------------------------------------ | 20260505 23:14:12.349 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:14:12.349 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 23:14:12.349 | INFO | Total mem usage: 25.6 GiB | 20260505 23:14:12.349 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 23:14:12.349 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 23:14:12.349 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 23:14:12.349 | INFO | High Water Breakdown: | 20260505 23:14:12.349 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 23:14:12.349 | INFO | ------------------------------------------ | 20260505 23:14:12.349 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:14:12.349 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 23:14:12.349 | INFO | Total mem usage: 15.7 GiB | 20260505 23:14:12.349 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 23:14:12.349 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 23:14:12.349 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 23:14:12.349 | INFO | High Water Breakdown: | 20260505 23:14:12.349 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 23:14:12.349 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 23:14:12.349 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 23:14:12.349 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 23:14:12.349 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 23:14:12.349 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 23:14:12.349 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 23:14:12.349 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 23:14:12.349 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 23:14:12.349 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:14:12.349 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 23:14:12.349 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:14:12.349 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 23:14:12.349 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 23:14:12.349 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 23:14:12.349 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 23:14:12.349 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:14:12.349 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 23:14:12.349 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:14:12.349 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:14:12.349 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:14:12.349 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 23:14:12.349 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 23:14:12.349 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:14:12.349 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 23:14:12.349 | INFO | ------------------------------------------ | 20260505 23:14:12.349 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:14:12.349 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 23:14:12.349 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:14:12.349 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:14:12.349 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.39e+01, 1.31e+01 / 1.16e+00 | 20260505 23:14:12.349 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.05e-01, 1.07e-02 / 6.17e-03 | 20260505 23:14:12.349 | INFO | PerfReport: 11.82%, Baseline -- 3.70e+00 / 4.54e+00, 5.64e-01 / 1.73e-01 | 20260505 23:14:12.349 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.53e+00, 1.85e-01 / 9.17e-02 | 20260505 23:14:12.349 | INFO | PerfReport: 15.99%, DME -- 4.00e+01 / 4.39e+01, 3.92e+00 / 1.32e+00 | 20260505 23:14:12.349 | INFO | PerfReport: 57.23%, FrameLabeling -- 1.79e+01 / 1.94e+01, 1.43e+00 / 1.14e+00 | 20260505 23:14:12.349 | INFO | PerfReport: 7.18%, PulseAccumulating -- 2.25e+00 / 2.77e+00, 4.29e-01 / 2.43e-01 | 20260505 23:14:12.349 | INFO | PerfReport: 3.13%, Metrics -- 9.81e-01 / 1.25e+00, 2.63e-01 / 1.16e-01 | 20260505 23:14:12.349 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.40e-01 / 3.95e-01, 5.78e-02 / 3.13e-02 | 20260505 23:14:12.349 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 23:14:12.349 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:14:12.349 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:14:12.349 | INFO | PerfReport: Full Execution -- 9.86e+01 / -nan, 2.48e+01 / 0.00e+00 | 20260505 23:14:12.350 | INFO | PerfReport: 22.45%, Upload -- 2.21e+01 / 0.00e+00, 7.62e+00 / 0.00e+00 | 20260505 23:14:12.350 | INFO | PerfReport: 21.84%, Download -- 2.15e+01 / 0.00e+00, 7.61e+00 / 0.00e+00 | 20260505 23:14:12.350 | INFO | PerfReport: 31.76%, Compute -- 3.13e+01 / 0.00e+00, 1.31e+01 / 0.00e+00 | 20260505 23:14:12.350 | INFO | PerfReport: 2.21%, LockUpload -- 2.18e+00 / 0.00e+00, 8.06e+00 / 0.00e+00 | 20260505 23:14:12.350 | INFO | PerfReport: 1.32%, LockDownload -- 1.30e+00 / 0.00e+00, 2.83e+00 / 0.00e+00 | 20260505 23:14:12.350 | INFO | PerfReport: 20.40%, LockCompute -- 2.01e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260505 23:14:12.350 | INFO | PerfReport: 0.01%, Misc -- 1.46e-02 / 0.00e+00, 2.94e-02 / 0.00e+00 | 20260505 23:14:12.466 | INFO | Starting a round of Baz IO | 20260505 23:14:15.613 | INFO | Populating RTMetrics report | 20260505 23:14:15.613 | INFO | RTMetrics report populated | 20260505 23:14:15.613 | INFO | Opening tmp RTMetrics json file | 20260505 23:14:15.614 | INFO | Serializing RTMetrics to json | 20260505 23:14:15.617 | INFO | Serialization done, swapping tmp file now... | 20260505 23:14:15.618 | INFO | RTMetrics done | 20260505 23:14:15.694 | INFO | Finished a round of Baz IO: Wrote 1.901 GiB in 3.228 seconds | 20260505 23:14:15.694 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.44% / 0.00% / 78.56% | 20260505 23:14:15.694 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.109375% / 3.97364e-06% / 9.93411e-05% | 20260505 23:14:15.694 | INFO | Baz IO overall: | 20260505 23:14:15.694 | INFO | BytesWritten : 4049097973819 | 20260505 23:14:15.694 | INFO | NumEvents : 1908125920166 | 20260505 23:14:17.486 | INFO | Analyzing chunk frames = [7069696,7070208) | 20260505 23:14:17.486 | INFO | MemInfo: { | 20260505 23:14:17.486 | INFO | "gpu" : " 17 GB", | 20260505 23:14:17.486 | INFO | "huge" : "", | 20260505 23:14:17.486 | INFO | "huge_shared" : " 27 GB", | 20260505 23:14:17.486 | INFO | "malloc" : " 14 GB", | 20260505 23:14:17.486 | INFO | "pinned" : " 29 GB" | 20260505 23:14:17.486 | INFO | } | 20260505 23:14:17.486 | INFO | Perf report for the last 5.1377 seconds: | 20260505 23:14:17.486 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:14:17.486 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:17.486 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:17.486 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:17.486 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260505 23:14:17.486 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:17.486 | INFO | RT_METRICS : 43.6%, 1.0, 22.8ms, 0 | 20260505 23:14:17.486 | INFO | PRE_HQ : 59.6%, 1.8, 54.9ms, 0 | 20260505 23:14:17.486 | INFO | BAZWRITER : 61.9%, 3.6, 115.2ms, 0 | 20260505 23:14:17.486 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 23:14:17.486 | INFO | Uploaded 36GiB to GPU in 2.14391s (16.7918GiB/s) | 20260505 23:14:17.486 | INFO | Downloaded 38GiB to GPU in 2.23839s (16.9765GiB/s) | 20260505 23:14:22.620 | INFO | Analyzing chunk frames = [7070208,7070720) | 20260505 23:14:22.620 | INFO | MemInfo: { | 20260505 23:14:22.620 | INFO | "gpu" : " 17 GB", | 20260505 23:14:22.620 | INFO | "huge" : "", | 20260505 23:14:22.620 | INFO | "huge_shared" : " 27 GB", | 20260505 23:14:22.620 | INFO | "malloc" : " 14 GB", | 20260505 23:14:22.620 | INFO | "pinned" : " 29 GB" | 20260505 23:14:22.620 | INFO | } | 20260505 23:14:22.620 | INFO | Perf report for the last 5.13436 seconds: | 20260505 23:14:22.620 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:14:22.620 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:22.620 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:22.620 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:22.620 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260505 23:14:22.620 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:22.620 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:22.620 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:14:22.620 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:22.620 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:22.620 | INFO | Uploaded 36GiB to GPU in 2.08948s (17.2292GiB/s) | 20260505 23:14:22.620 | INFO | Downloaded 34GiB to GPU in 2.0396s (16.6699GiB/s) | 20260505 23:14:27.758 | INFO | Analyzing chunk frames = [7070720,7071232) | 20260505 23:14:27.758 | INFO | MemInfo: { | 20260505 23:14:27.758 | INFO | "gpu" : " 17 GB", | 20260505 23:14:27.758 | INFO | "huge" : "", | 20260505 23:14:27.758 | INFO | "huge_shared" : " 27 GB", | 20260505 23:14:27.758 | INFO | "malloc" : " 14 GB", | 20260505 23:14:27.758 | INFO | "pinned" : " 29 GB" | 20260505 23:14:27.758 | INFO | } | 20260505 23:14:27.758 | INFO | Perf report for the last 5.13784 seconds: | 20260505 23:14:27.758 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:14:27.758 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:27.758 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:27.758 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:27.758 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260505 23:14:27.758 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:27.758 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:27.758 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 23:14:27.758 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:27.758 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:27.758 | INFO | Uploaded 36GiB to GPU in 2.15322s (16.7191GiB/s) | 20260505 23:14:27.758 | INFO | Downloaded 34GiB to GPU in 2.05668s (16.5315GiB/s) | 20260505 23:14:32.897 | INFO | Analyzing chunk frames = [7071232,7071744) | 20260505 23:14:32.897 | INFO | MemInfo: { | 20260505 23:14:32.897 | INFO | "gpu" : " 17 GB", | 20260505 23:14:32.897 | INFO | "huge" : "", | 20260505 23:14:32.897 | INFO | "huge_shared" : " 27 GB", | 20260505 23:14:32.897 | INFO | "malloc" : " 14 GB", | 20260505 23:14:32.897 | INFO | "pinned" : " 29 GB" | 20260505 23:14:32.897 | INFO | } | 20260505 23:14:32.898 | INFO | Perf report for the last 5.13956 seconds: | 20260505 23:14:32.898 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:14:32.898 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:32.898 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:32.898 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:32.898 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260505 23:14:32.898 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:32.898 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:32.898 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 23:14:32.898 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:32.898 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:32.898 | INFO | Uploaded 36GiB to GPU in 2.13295s (16.8781GiB/s) | 20260505 23:14:32.898 | INFO | Downloaded 34GiB to GPU in 2.05116s (16.576GiB/s) | 20260505 23:14:38.041 | INFO | Analyzing chunk frames = [7071744,7072256) | 20260505 23:14:38.041 | INFO | MemInfo: { | 20260505 23:14:38.041 | INFO | "gpu" : " 17 GB", | 20260505 23:14:38.041 | INFO | "huge" : "", | 20260505 23:14:38.041 | INFO | "huge_shared" : " 27 GB", | 20260505 23:14:38.041 | INFO | "malloc" : " 14 GB", | 20260505 23:14:38.041 | INFO | "pinned" : " 29 GB" | 20260505 23:14:38.041 | INFO | } | 20260505 23:14:38.041 | INFO | Perf report for the last 5.14345 seconds: | 20260505 23:14:38.041 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:14:38.041 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:38.041 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:38.041 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:38.041 | INFO | ANALYSIS : 61.5%, 3.0, 96.0ms, 0 | 20260505 23:14:38.041 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:38.041 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:38.041 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 23:14:38.041 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:38.041 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:38.041 | INFO | Uploaded 36GiB to GPU in 2.13663s (16.8489GiB/s) | 20260505 23:14:38.041 | INFO | Downloaded 34GiB to GPU in 2.06357s (16.4763GiB/s) | 20260505 23:14:43.188 | INFO | Analyzing chunk frames = [7072256,7072768) | 20260505 23:14:43.188 | INFO | MemInfo: { | 20260505 23:14:43.188 | INFO | "gpu" : " 17 GB", | 20260505 23:14:43.188 | INFO | "huge" : "", | 20260505 23:14:43.188 | INFO | "huge_shared" : " 27 GB", | 20260505 23:14:43.188 | INFO | "malloc" : " 14 GB", | 20260505 23:14:43.188 | INFO | "pinned" : " 29 GB" | 20260505 23:14:43.188 | INFO | } | 20260505 23:14:43.188 | INFO | Perf report for the last 5.14724 seconds: | 20260505 23:14:43.188 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:14:43.188 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:43.188 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:43.188 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:43.188 | INFO | ANALYSIS : 61.2%, 3.0, 95.7ms, 0 | 20260505 23:14:43.188 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:43.188 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:43.188 | INFO | PRE_HQ : 26.8%, 1.0, 14.0ms, 0 | 20260505 23:14:43.188 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:43.188 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:43.189 | INFO | Uploaded 36GiB to GPU in 2.07329s (17.3637GiB/s) | 20260505 23:14:43.189 | INFO | Downloaded 34GiB to GPU in 2.03901s (16.6747GiB/s) | 20260505 23:14:48.322 | INFO | Analyzing chunk frames = [7072768,7073280) | 20260505 23:14:48.322 | INFO | MemInfo: { | 20260505 23:14:48.322 | INFO | "gpu" : " 17 GB", | 20260505 23:14:48.322 | INFO | "huge" : "", | 20260505 23:14:48.322 | INFO | "huge_shared" : " 27 GB", | 20260505 23:14:48.322 | INFO | "malloc" : " 14 GB", | 20260505 23:14:48.323 | INFO | "pinned" : " 29 GB" | 20260505 23:14:48.323 | INFO | } | 20260505 23:14:48.323 | INFO | Perf report for the last 5.13447 seconds: | 20260505 23:14:48.323 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:14:48.323 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:48.323 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:48.323 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:48.323 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 23:14:48.323 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:48.323 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:48.323 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 23:14:48.323 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:48.323 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:48.323 | INFO | Uploaded 36GiB to GPU in 2.01647s (17.853GiB/s) | 20260505 23:14:48.323 | INFO | Downloaded 34GiB to GPU in 2.04835s (16.5988GiB/s) | 20260505 23:14:53.452 | INFO | Analyzing chunk frames = [7073280,7073792) | 20260505 23:14:53.452 | INFO | MemInfo: { | 20260505 23:14:53.452 | INFO | "gpu" : " 17 GB", | 20260505 23:14:53.452 | INFO | "huge" : "", | 20260505 23:14:53.452 | INFO | "huge_shared" : " 27 GB", | 20260505 23:14:53.452 | INFO | "malloc" : " 14 GB", | 20260505 23:14:53.452 | INFO | "pinned" : " 29 GB" | 20260505 23:14:53.452 | INFO | } | 20260505 23:14:53.452 | INFO | Perf report for the last 5.12929 seconds: | 20260505 23:14:53.452 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:14:53.452 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:53.452 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:53.452 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:53.452 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260505 23:14:53.452 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:53.452 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:53.452 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 23:14:53.452 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:53.452 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:53.453 | INFO | Uploaded 36GiB to GPU in 2.11586s (17.0144GiB/s) | 20260505 23:14:53.453 | INFO | Downloaded 34GiB to GPU in 1.99962s (17.0033GiB/s) | 20260505 23:14:53.583 | INFO | Starting a round of Baz IO | 20260505 23:14:56.729 | INFO | Populating RTMetrics report | 20260505 23:14:56.729 | INFO | RTMetrics report populated | 20260505 23:14:56.729 | INFO | Opening tmp RTMetrics json file | 20260505 23:14:56.730 | INFO | Serializing RTMetrics to json | 20260505 23:14:56.734 | INFO | Serialization done, swapping tmp file now... | 20260505 23:14:56.736 | INFO | RTMetrics done | 20260505 23:14:56.814 | INFO | Finished a round of Baz IO: Wrote 1.9 GiB in 3.23 seconds | 20260505 23:14:56.814 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.44% / 0.00% / 78.56% | 20260505 23:14:56.814 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.110614% / 0% / 7.94729e-05% | 20260505 23:14:56.814 | INFO | Baz IO overall: | 20260505 23:14:56.814 | INFO | BytesWritten : 4051138074539 | 20260505 23:14:56.814 | INFO | NumEvents : 1909061546767 | 20260505 23:14:58.598 | INFO | Analyzing chunk frames = [7073792,7074304) | 20260505 23:14:58.598 | INFO | MemInfo: { | 20260505 23:14:58.598 | INFO | "gpu" : " 17 GB", | 20260505 23:14:58.598 | INFO | "huge" : "", | 20260505 23:14:58.598 | INFO | "huge_shared" : " 27 GB", | 20260505 23:14:58.598 | INFO | "malloc" : " 14 GB", | 20260505 23:14:58.598 | INFO | "pinned" : " 29 GB" | 20260505 23:14:58.598 | INFO | } | 20260505 23:14:58.598 | INFO | Perf report for the last 5.1463 seconds: | 20260505 23:14:58.598 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:14:58.598 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:58.598 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:58.598 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:58.598 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260505 23:14:58.598 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:14:58.598 | INFO | RT_METRICS : 43.4%, 1.0, 22.8ms, 0 | 20260505 23:14:58.598 | INFO | PRE_HQ : 59.5%, 1.8, 54.3ms, 0 | 20260505 23:14:58.598 | INFO | BAZWRITER : 62.1%, 3.6, 113.8ms, 0 | 20260505 23:14:58.598 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 23:14:58.599 | INFO | Uploaded 36GiB to GPU in 2.12051s (16.977GiB/s) | 20260505 23:14:58.599 | INFO | Downloaded 38GiB to GPU in 2.23222s (17.0234GiB/s) | 20260505 23:15:03.743 | INFO | Analyzing chunk frames = [7074304,7074816) | 20260505 23:15:03.743 | INFO | MemInfo: { | 20260505 23:15:03.743 | INFO | "gpu" : " 17 GB", | 20260505 23:15:03.743 | INFO | "huge" : "", | 20260505 23:15:03.743 | INFO | "huge_shared" : " 27 GB", | 20260505 23:15:03.743 | INFO | "malloc" : " 14 GB", | 20260505 23:15:03.744 | INFO | "pinned" : " 29 GB" | 20260505 23:15:03.744 | INFO | } | 20260505 23:15:03.744 | INFO | Perf report for the last 5.1454 seconds: | 20260505 23:15:03.744 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:15:03.744 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:03.744 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:03.744 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:03.744 | INFO | ANALYSIS : 61.8%, 3.0, 96.6ms, 0 | 20260505 23:15:03.744 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:03.744 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:03.744 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:15:03.744 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:03.744 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:03.744 | INFO | Uploaded 36GiB to GPU in 2.08752s (17.2454GiB/s) | 20260505 23:15:03.744 | INFO | Downloaded 34GiB to GPU in 2.02657s (16.7771GiB/s) | 20260505 23:15:08.877 | INFO | Analyzing chunk frames = [7074816,7075328) | 20260505 23:15:08.877 | INFO | MemInfo: { | 20260505 23:15:08.877 | INFO | "gpu" : " 17 GB", | 20260505 23:15:08.877 | INFO | "huge" : "", | 20260505 23:15:08.877 | INFO | "huge_shared" : " 27 GB", | 20260505 23:15:08.877 | INFO | "malloc" : " 14 GB", | 20260505 23:15:08.877 | INFO | "pinned" : " 29 GB" | 20260505 23:15:08.877 | INFO | } | 20260505 23:15:08.877 | INFO | Perf report for the last 5.13347 seconds: | 20260505 23:15:08.877 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:15:08.877 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:08.877 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:08.877 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:08.877 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260505 23:15:08.878 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:08.878 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:08.878 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:15:08.878 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:08.878 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:08.878 | INFO | Uploaded 36GiB to GPU in 2.07561s (17.3443GiB/s) | 20260505 23:15:08.878 | INFO | Downloaded 34GiB to GPU in 2.01133s (16.9042GiB/s) | 20260505 23:15:14.016 | INFO | Analyzing chunk frames = [7075328,7075840) | 20260505 23:15:14.016 | INFO | MemInfo: { | 20260505 23:15:14.016 | INFO | "gpu" : " 17 GB", | 20260505 23:15:14.016 | INFO | "huge" : "", | 20260505 23:15:14.016 | INFO | "huge_shared" : " 27 GB", | 20260505 23:15:14.016 | INFO | "malloc" : " 14 GB", | 20260505 23:15:14.016 | INFO | "pinned" : " 29 GB" | 20260505 23:15:14.016 | INFO | } | 20260505 23:15:14.016 | INFO | Perf report for the last 5.13871 seconds: | 20260505 23:15:14.016 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:15:14.016 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:14.016 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:14.016 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:14.016 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260505 23:15:14.016 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:14.016 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:14.016 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 23:15:14.016 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:14.016 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:14.016 | INFO | Uploaded 36GiB to GPU in 2.13904s (16.83GiB/s) | 20260505 23:15:14.016 | INFO | Downloaded 34GiB to GPU in 2.00605s (16.9488GiB/s) | 20260505 23:15:19.157 | INFO | Analyzing chunk frames = [7075840,7076352) | 20260505 23:15:19.157 | INFO | MemInfo: { | 20260505 23:15:19.157 | INFO | "gpu" : " 17 GB", | 20260505 23:15:19.157 | INFO | "huge" : "", | 20260505 23:15:19.157 | INFO | "huge_shared" : " 27 GB", | 20260505 23:15:19.157 | INFO | "malloc" : " 14 GB", | 20260505 23:15:19.157 | INFO | "pinned" : " 29 GB" | 20260505 23:15:19.157 | INFO | } | 20260505 23:15:19.157 | INFO | Perf report for the last 5.1414 seconds: | 20260505 23:15:19.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:15:19.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:19.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:19.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:19.157 | INFO | ANALYSIS : 61.2%, 3.0, 95.6ms, 0 | 20260505 23:15:19.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:19.157 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:19.157 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 23:15:19.157 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:19.157 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:19.157 | INFO | Uploaded 36GiB to GPU in 2.09108s (17.216GiB/s) | 20260505 23:15:19.158 | INFO | Downloaded 34GiB to GPU in 2.04507s (16.6254GiB/s) | 20260505 23:15:24.299 | INFO | Analyzing chunk frames = [7076352,7076864) | 20260505 23:15:24.299 | INFO | MemInfo: { | 20260505 23:15:24.299 | INFO | "gpu" : " 17 GB", | 20260505 23:15:24.299 | INFO | "huge" : "", | 20260505 23:15:24.299 | INFO | "huge_shared" : " 27 GB", | 20260505 23:15:24.299 | INFO | "malloc" : " 14 GB", | 20260505 23:15:24.299 | INFO | "pinned" : " 29 GB" | 20260505 23:15:24.299 | INFO | } | 20260505 23:15:24.299 | INFO | Perf report for the last 5.14152 seconds: | 20260505 23:15:24.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:15:24.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:24.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:24.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:24.299 | INFO | ANALYSIS : 61.2%, 3.0, 95.6ms, 0 | 20260505 23:15:24.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:24.299 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:24.299 | INFO | PRE_HQ : 27.2%, 1.0, 14.2ms, 0 | 20260505 23:15:24.299 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:24.299 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:24.299 | INFO | Uploaded 36GiB to GPU in 2.07084s (17.3843GiB/s) | 20260505 23:15:24.299 | INFO | Downloaded 34GiB to GPU in 2.03336s (16.7211GiB/s) | 20260505 23:15:29.433 | INFO | Analyzing chunk frames = [7076864,7077376) | 20260505 23:15:29.433 | INFO | MemInfo: { | 20260505 23:15:29.433 | INFO | "gpu" : " 17 GB", | 20260505 23:15:29.433 | INFO | "huge" : "", | 20260505 23:15:29.433 | INFO | "huge_shared" : " 27 GB", | 20260505 23:15:29.433 | INFO | "malloc" : " 14 GB", | 20260505 23:15:29.433 | INFO | "pinned" : " 29 GB" | 20260505 23:15:29.433 | INFO | } | 20260505 23:15:29.433 | INFO | Perf report for the last 5.13429 seconds: | 20260505 23:15:29.433 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:15:29.433 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:29.433 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:29.433 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:29.433 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 23:15:29.433 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:29.433 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:29.433 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 23:15:29.433 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:29.433 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:29.433 | INFO | Uploaded 36GiB to GPU in 2.01766s (17.8425GiB/s) | 20260505 23:15:29.433 | INFO | Downloaded 34GiB to GPU in 2.03975s (16.6687GiB/s) | 20260505 23:15:34.570 | INFO | Analyzing chunk frames = [7077376,7077888) | 20260505 23:15:34.570 | INFO | MemInfo: { | 20260505 23:15:34.570 | INFO | "gpu" : " 17 GB", | 20260505 23:15:34.570 | INFO | "huge" : "", | 20260505 23:15:34.570 | INFO | "huge_shared" : " 27 GB", | 20260505 23:15:34.570 | INFO | "malloc" : " 14 GB", | 20260505 23:15:34.570 | INFO | "pinned" : " 29 GB" | 20260505 23:15:34.570 | INFO | } | 20260505 23:15:34.570 | INFO | Perf report for the last 5.13677 seconds: | 20260505 23:15:34.570 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:15:34.570 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:34.570 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:34.570 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:34.570 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260505 23:15:34.570 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:34.570 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:34.570 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:15:34.570 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:34.570 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:34.570 | INFO | Uploaded 36GiB to GPU in 2.11489s (17.0222GiB/s) | 20260505 23:15:34.570 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 23:15:34.570 | INFO | Total mem usage: 15.3 GiB | 20260505 23:15:34.570 | INFO | Downloaded 34GiB to GPU in 2.00702s (16.9406GiB/s) | 20260505 23:15:34.570 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 23:15:34.570 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -9.08%) | 20260505 23:15:34.570 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 23:15:34.570 | INFO | High Water Breakdown: | 20260505 23:15:34.570 | INFO | 60.9% : 8.55 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 23:15:34.570 | INFO | 39% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 23:15:34.570 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 23:15:34.570 | INFO | ------------------------------------------ | 20260505 23:15:34.570 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:15:34.570 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 23:15:34.570 | INFO | Total mem usage: 26.5 GiB | 20260505 23:15:34.570 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 23:15:34.570 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 23:15:34.570 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 23:15:34.570 | INFO | High Water Breakdown: | 20260505 23:15:34.570 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:15:34.570 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:15:34.570 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:15:34.570 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:15:34.570 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:15:34.570 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:15:34.570 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 23:15:34.570 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 23:15:34.570 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:15:34.570 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 23:15:34.570 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 23:15:34.571 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 23:15:34.571 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 23:15:34.571 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 23:15:34.571 | INFO | ------------------------------------------ | 20260505 23:15:34.571 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:15:34.571 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 23:15:34.571 | INFO | Total mem usage: 25.6 GiB | 20260505 23:15:34.571 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 23:15:34.571 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 23:15:34.571 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 23:15:34.571 | INFO | High Water Breakdown: | 20260505 23:15:34.571 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 23:15:34.571 | INFO | ------------------------------------------ | 20260505 23:15:34.571 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:15:34.571 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 23:15:34.571 | INFO | Total mem usage: 15.7 GiB | 20260505 23:15:34.571 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 23:15:34.571 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 23:15:34.571 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 23:15:34.571 | INFO | High Water Breakdown: | 20260505 23:15:34.571 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 23:15:34.571 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 23:15:34.571 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 23:15:34.571 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 23:15:34.571 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 23:15:34.571 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 23:15:34.571 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 23:15:34.571 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 23:15:34.571 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 23:15:34.571 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:15:34.571 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 23:15:34.571 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:15:34.571 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 23:15:34.571 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 23:15:34.571 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 23:15:34.571 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 23:15:34.571 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:15:34.571 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 23:15:34.571 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:15:34.571 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:15:34.571 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:15:34.571 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 23:15:34.571 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 23:15:34.571 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:15:34.571 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 23:15:34.571 | INFO | ------------------------------------------ | 20260505 23:15:34.571 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:15:34.571 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 23:15:34.571 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:15:34.571 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:15:34.571 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.42e+01, 1.34e+01 / 9.38e-01 | 20260505 23:15:34.571 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.04e-01, 1.02e-02 / 2.84e-03 | 20260505 23:15:34.571 | INFO | PerfReport: 11.85%, Baseline -- 3.72e+00 / 4.54e+00, 5.81e-01 / 1.94e-01 | 20260505 23:15:34.571 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.54e+00, 1.92e-01 / 7.69e-02 | 20260505 23:15:34.571 | INFO | PerfReport: 16.01%, DME -- 4.02e+01 / 4.44e+01, 3.86e+00 / 1.21e+00 | 20260505 23:15:34.571 | INFO | PerfReport: 57.24%, FrameLabeling -- 1.80e+01 / 1.90e+01, 1.49e+00 / 8.98e-01 | 20260505 23:15:34.571 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.23e+00 / 2.82e+00, 4.26e-01 / 1.83e-01 | 20260505 23:15:34.571 | INFO | PerfReport: 3.12%, Metrics -- 9.78e-01 / 1.29e+00, 2.64e-01 / 1.28e-01 | 20260505 23:15:34.571 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.39e-01 / 3.99e-01, 5.81e-02 / 2.96e-02 | 20260505 23:15:34.571 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 23:15:34.571 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:15:34.571 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:15:34.571 | INFO | PerfReport: Full Execution -- 9.84e+01 / -nan, 2.47e+01 / 0.00e+00 | 20260505 23:15:34.571 | INFO | PerfReport: 22.22%, Upload -- 2.19e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260505 23:15:34.571 | INFO | PerfReport: 21.79%, Download -- 2.14e+01 / 0.00e+00, 7.59e+00 / 0.00e+00 | 20260505 23:15:34.571 | INFO | PerfReport: 31.88%, Compute -- 3.14e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 23:15:34.571 | INFO | PerfReport: 2.12%, LockUpload -- 2.09e+00 / 0.00e+00, 7.81e+00 / 0.00e+00 | 20260505 23:15:34.571 | INFO | PerfReport: 1.31%, LockDownload -- 1.29e+00 / 0.00e+00, 2.98e+00 / 0.00e+00 | 20260505 23:15:34.571 | INFO | PerfReport: 20.66%, LockCompute -- 2.03e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260505 23:15:34.571 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 2.01e-02 / 0.00e+00 | 20260505 23:15:34.704 | INFO | Starting a round of Baz IO | 20260505 23:15:37.849 | INFO | Populating RTMetrics report | 20260505 23:15:37.850 | INFO | RTMetrics report populated | 20260505 23:15:37.850 | INFO | Opening tmp RTMetrics json file | 20260505 23:15:37.851 | INFO | Serializing RTMetrics to json | 20260505 23:15:37.852 | INFO | Serialization done, swapping tmp file now... | 20260505 23:15:37.852 | INFO | RTMetrics done | 20260505 23:15:37.927 | INFO | Finished a round of Baz IO: Wrote 1.9 GiB in 3.222 seconds | 20260505 23:15:37.927 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.44% / 0.00% / 78.56% | 20260505 23:15:37.927 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.109935% / 3.97364e-06% / 0.000107288% | 20260505 23:15:37.927 | INFO | Baz IO overall: | 20260505 23:15:37.927 | INFO | BytesWritten : 4053178525635 | 20260505 23:15:37.927 | INFO | NumEvents : 1909997746704 | 20260505 23:15:39.708 | INFO | Analyzing chunk frames = [7077888,7078400) | 20260505 23:15:39.708 | INFO | MemInfo: { | 20260505 23:15:39.708 | INFO | "gpu" : " 17 GB", | 20260505 23:15:39.708 | INFO | "huge" : "", | 20260505 23:15:39.708 | INFO | "huge_shared" : " 27 GB", | 20260505 23:15:39.708 | INFO | "malloc" : " 14 GB", | 20260505 23:15:39.708 | INFO | "pinned" : " 29 GB" | 20260505 23:15:39.708 | INFO | } | 20260505 23:15:39.708 | INFO | Perf report for the last 5.13773 seconds: | 20260505 23:15:39.708 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:15:39.708 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:39.708 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:39.708 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:39.708 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260505 23:15:39.708 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:39.708 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260505 23:15:39.708 | INFO | PRE_HQ : 59.6%, 1.8, 54.5ms, 0 | 20260505 23:15:39.708 | INFO | BAZWRITER : 62.2%, 3.6, 113.5ms, 0 | 20260505 23:15:39.708 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 23:15:39.708 | INFO | Uploaded 36GiB to GPU in 2.17352s (16.563GiB/s) | 20260505 23:15:39.708 | INFO | Downloaded 38GiB to GPU in 2.24369s (16.9364GiB/s) | 20260505 23:15:44.850 | INFO | Analyzing chunk frames = [7078400,7078912) | 20260505 23:15:44.850 | INFO | MemInfo: { | 20260505 23:15:44.850 | INFO | "gpu" : " 17 GB", | 20260505 23:15:44.850 | INFO | "huge" : "", | 20260505 23:15:44.850 | INFO | "huge_shared" : " 27 GB", | 20260505 23:15:44.850 | INFO | "malloc" : " 14 GB", | 20260505 23:15:44.850 | INFO | "pinned" : " 29 GB" | 20260505 23:15:44.850 | INFO | } | 20260505 23:15:44.851 | INFO | Perf report for the last 5.14282 seconds: | 20260505 23:15:44.851 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:15:44.851 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:44.851 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:44.851 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:44.851 | INFO | ANALYSIS : 61.8%, 3.0, 96.6ms, 0 | 20260505 23:15:44.851 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:44.851 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:44.851 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:15:44.851 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:44.851 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:44.851 | INFO | Uploaded 36GiB to GPU in 2.08542s (17.2627GiB/s) | 20260505 23:15:44.851 | INFO | Downloaded 34GiB to GPU in 2.02316s (16.8054GiB/s) | 20260505 23:15:49.998 | INFO | Analyzing chunk frames = [7078912,7079424) | 20260505 23:15:49.998 | INFO | MemInfo: { | 20260505 23:15:49.998 | INFO | "gpu" : " 17 GB", | 20260505 23:15:49.998 | INFO | "huge" : "", | 20260505 23:15:49.998 | INFO | "huge_shared" : " 27 GB", | 20260505 23:15:49.998 | INFO | "malloc" : " 14 GB", | 20260505 23:15:49.998 | INFO | "pinned" : " 29 GB" | 20260505 23:15:49.998 | INFO | } | 20260505 23:15:49.998 | INFO | Perf report for the last 5.14755 seconds: | 20260505 23:15:49.998 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:15:49.998 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:49.998 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:49.998 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:49.998 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260505 23:15:49.998 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:49.998 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:49.998 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:15:49.998 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:49.998 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:49.998 | INFO | Uploaded 36GiB to GPU in 2.1561s (16.6968GiB/s) | 20260505 23:15:49.998 | INFO | Downloaded 34GiB to GPU in 2.05661s (16.5321GiB/s) | 20260505 23:15:55.134 | INFO | Analyzing chunk frames = [7079424,7079936) | 20260505 23:15:55.134 | INFO | MemInfo: { | 20260505 23:15:55.134 | INFO | "gpu" : " 17 GB", | 20260505 23:15:55.134 | INFO | "huge" : "", | 20260505 23:15:55.134 | INFO | "huge_shared" : " 27 GB", | 20260505 23:15:55.134 | INFO | "malloc" : " 14 GB", | 20260505 23:15:55.134 | INFO | "pinned" : " 29 GB" | 20260505 23:15:55.134 | INFO | } | 20260505 23:15:55.134 | INFO | Perf report for the last 5.13582 seconds: | 20260505 23:15:55.134 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:15:55.134 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:55.134 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:55.134 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:55.134 | INFO | ANALYSIS : 61.7%, 3.0, 95.8ms, 0 | 20260505 23:15:55.134 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:55.134 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:55.134 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 23:15:55.134 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:55.134 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:15:55.134 | INFO | Uploaded 36GiB to GPU in 2.14213s (16.8057GiB/s) | 20260505 23:15:55.134 | INFO | Downloaded 34GiB to GPU in 2.02266s (16.8095GiB/s) | 20260505 23:16:00.268 | INFO | Analyzing chunk frames = [7079936,7080448) | 20260505 23:16:00.268 | INFO | MemInfo: { | 20260505 23:16:00.268 | INFO | "gpu" : " 17 GB", | 20260505 23:16:00.269 | INFO | "huge" : "", | 20260505 23:16:00.269 | INFO | "huge_shared" : " 27 GB", | 20260505 23:16:00.269 | INFO | "malloc" : " 14 GB", | 20260505 23:16:00.269 | INFO | "pinned" : " 29 GB" | 20260505 23:16:00.269 | INFO | } | 20260505 23:16:00.269 | INFO | Perf report for the last 5.13472 seconds: | 20260505 23:16:00.269 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:16:00.269 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:00.269 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:00.269 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:00.269 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260505 23:16:00.269 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:00.269 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:00.269 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 23:16:00.269 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:00.269 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:00.269 | INFO | Uploaded 36GiB to GPU in 2.09669s (17.1699GiB/s) | 20260505 23:16:00.269 | INFO | Downloaded 34GiB to GPU in 2.0406s (16.6617GiB/s) | 20260505 23:16:05.407 | INFO | Analyzing chunk frames = [7080448,7080960) | 20260505 23:16:05.407 | INFO | MemInfo: { | 20260505 23:16:05.407 | INFO | "gpu" : " 17 GB", | 20260505 23:16:05.407 | INFO | "huge" : "", | 20260505 23:16:05.407 | INFO | "huge_shared" : " 27 GB", | 20260505 23:16:05.407 | INFO | "malloc" : " 14 GB", | 20260505 23:16:05.407 | INFO | "pinned" : " 29 GB" | 20260505 23:16:05.407 | INFO | } | 20260505 23:16:05.407 | INFO | Perf report for the last 5.13879 seconds: | 20260505 23:16:05.407 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:16:05.407 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:05.407 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:05.408 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:05.408 | INFO | ANALYSIS : 61.5%, 3.0, 96.0ms, 0 | 20260505 23:16:05.408 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:05.408 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:05.408 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260505 23:16:05.408 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:05.408 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:05.408 | INFO | Uploaded 36GiB to GPU in 2.08028s (17.3053GiB/s) | 20260505 23:16:05.408 | INFO | Downloaded 34GiB to GPU in 2.03819s (16.6815GiB/s) | 20260505 23:16:10.548 | INFO | Analyzing chunk frames = [7080960,7081472) | 20260505 23:16:10.548 | INFO | MemInfo: { | 20260505 23:16:10.548 | INFO | "gpu" : " 17 GB", | 20260505 23:16:10.548 | INFO | "huge" : "", | 20260505 23:16:10.548 | INFO | "huge_shared" : " 27 GB", | 20260505 23:16:10.548 | INFO | "malloc" : " 14 GB", | 20260505 23:16:10.548 | INFO | "pinned" : " 29 GB" | 20260505 23:16:10.548 | INFO | } | 20260505 23:16:10.548 | INFO | Perf report for the last 5.14059 seconds: | 20260505 23:16:10.548 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:16:10.548 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:10.548 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:10.548 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:10.548 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260505 23:16:10.548 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:10.548 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:10.548 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 23:16:10.548 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:10.548 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:10.548 | INFO | Uploaded 36GiB to GPU in 2.10796s (17.0781GiB/s) | 20260505 23:16:10.548 | INFO | Downloaded 34GiB to GPU in 2.08723s (16.2895GiB/s) | 20260505 23:16:15.691 | INFO | Analyzing chunk frames = [7081472,7081984) | 20260505 23:16:15.691 | INFO | MemInfo: { | 20260505 23:16:15.691 | INFO | "gpu" : " 17 GB", | 20260505 23:16:15.691 | INFO | "huge" : "", | 20260505 23:16:15.691 | INFO | "huge_shared" : " 27 GB", | 20260505 23:16:15.691 | INFO | "malloc" : " 14 GB", | 20260505 23:16:15.692 | INFO | "pinned" : " 29 GB" | 20260505 23:16:15.692 | INFO | } | 20260505 23:16:15.692 | INFO | Perf report for the last 5.14348 seconds: | 20260505 23:16:15.692 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:16:15.692 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:15.692 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:15.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:15.692 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 23:16:15.692 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:15.692 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:15.692 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 23:16:15.692 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:15.692 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:15.692 | INFO | Uploaded 36GiB to GPU in 2.1031s (17.1176GiB/s) | 20260505 23:16:15.692 | INFO | Downloaded 34GiB to GPU in 1.99854s (17.0124GiB/s) | 20260505 23:16:15.815 | INFO | Starting a round of Baz IO | 20260505 23:16:18.965 | INFO | Populating RTMetrics report | 20260505 23:16:18.965 | INFO | RTMetrics report populated | 20260505 23:16:18.965 | INFO | Opening tmp RTMetrics json file | 20260505 23:16:18.971 | INFO | Serializing RTMetrics to json | 20260505 23:16:18.973 | INFO | Serialization done, swapping tmp file now... | 20260505 23:16:18.974 | INFO | RTMetrics done | 20260505 23:16:19.042 | INFO | Finished a round of Baz IO: Wrote 1.9 GiB in 3.227 seconds | 20260505 23:16:19.042 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.44% / 0.00% / 78.56% | 20260505 23:16:19.042 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.109847% / 3.97364e-06% / 8.74201e-05% | 20260505 23:16:19.042 | INFO | Baz IO overall: | 20260505 23:16:19.042 | INFO | BytesWritten : 4055218424043 | 20260505 23:16:19.042 | INFO | NumEvents : 1910933561749 | 20260505 23:16:20.826 | INFO | Analyzing chunk frames = [7081984,7082496) | 20260505 23:16:20.826 | INFO | MemInfo: { | 20260505 23:16:20.826 | INFO | "gpu" : " 17 GB", | 20260505 23:16:20.826 | INFO | "huge" : "", | 20260505 23:16:20.826 | INFO | "huge_shared" : " 27 GB", | 20260505 23:16:20.826 | INFO | "malloc" : " 14 GB", | 20260505 23:16:20.826 | INFO | "pinned" : " 29 GB" | 20260505 23:16:20.826 | INFO | } | 20260505 23:16:20.826 | INFO | Perf report for the last 5.13463 seconds: | 20260505 23:16:20.826 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:16:20.826 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:20.826 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:20.826 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:20.826 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 23:16:20.826 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:20.826 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260505 23:16:20.826 | INFO | PRE_HQ : 60.2%, 1.8, 54.7ms, 0 | 20260505 23:16:20.826 | INFO | BAZWRITER : 62.0%, 3.6, 114.6ms, 0 | 20260505 23:16:20.826 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 23:16:20.826 | INFO | Uploaded 36GiB to GPU in 2.16791s (16.6058GiB/s) | 20260505 23:16:20.826 | INFO | Downloaded 38GiB to GPU in 2.24186s (16.9502GiB/s) | 20260505 23:16:25.964 | INFO | Analyzing chunk frames = [7082496,7083008) | 20260505 23:16:25.964 | INFO | MemInfo: { | 20260505 23:16:25.964 | INFO | "gpu" : " 17 GB", | 20260505 23:16:25.964 | INFO | "huge" : "", | 20260505 23:16:25.964 | INFO | "huge_shared" : " 27 GB", | 20260505 23:16:25.964 | INFO | "malloc" : " 14 GB", | 20260505 23:16:25.964 | INFO | "pinned" : " 29 GB" | 20260505 23:16:25.964 | INFO | } | 20260505 23:16:25.964 | INFO | Perf report for the last 5.13819 seconds: | 20260505 23:16:25.964 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:16:25.964 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:25.964 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:25.964 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:25.964 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 23:16:25.964 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:25.964 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:25.964 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260505 23:16:25.964 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:25.964 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:25.965 | INFO | Uploaded 36GiB to GPU in 2.0824s (17.2877GiB/s) | 20260505 23:16:25.965 | INFO | Downloaded 34GiB to GPU in 2.02793s (16.7659GiB/s) | 20260505 23:16:31.113 | INFO | Analyzing chunk frames = [7083008,7083520) | 20260505 23:16:31.113 | INFO | MemInfo: { | 20260505 23:16:31.113 | INFO | "gpu" : " 17 GB", | 20260505 23:16:31.113 | INFO | "huge" : "", | 20260505 23:16:31.113 | INFO | "huge_shared" : " 27 GB", | 20260505 23:16:31.113 | INFO | "malloc" : " 14 GB", | 20260505 23:16:31.113 | INFO | "pinned" : " 29 GB" | 20260505 23:16:31.113 | INFO | } | 20260505 23:16:31.113 | INFO | Perf report for the last 5.14891 seconds: | 20260505 23:16:31.113 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:16:31.113 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:31.113 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:31.113 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:31.113 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260505 23:16:31.113 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:31.113 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:31.113 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:16:31.113 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:31.113 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:31.114 | INFO | Uploaded 36GiB to GPU in 2.14612s (16.7745GiB/s) | 20260505 23:16:31.114 | INFO | Downloaded 34GiB to GPU in 2.0511s (16.5765GiB/s) | 20260505 23:16:36.248 | INFO | Analyzing chunk frames = [7083520,7084032) | 20260505 23:16:36.248 | INFO | MemInfo: { | 20260505 23:16:36.248 | INFO | "gpu" : " 17 GB", | 20260505 23:16:36.248 | INFO | "huge" : "", | 20260505 23:16:36.248 | INFO | "huge_shared" : " 27 GB", | 20260505 23:16:36.248 | INFO | "malloc" : " 14 GB", | 20260505 23:16:36.248 | INFO | "pinned" : " 29 GB" | 20260505 23:16:36.248 | INFO | } | 20260505 23:16:36.248 | INFO | Perf report for the last 5.13447 seconds: | 20260505 23:16:36.248 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:16:36.248 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:36.248 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:36.248 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:36.248 | INFO | ANALYSIS : 61.7%, 3.0, 95.8ms, 0 | 20260505 23:16:36.248 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:36.248 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:36.248 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 23:16:36.248 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:36.248 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:36.248 | INFO | Uploaded 36GiB to GPU in 2.13602s (16.8538GiB/s) | 20260505 23:16:36.248 | INFO | Downloaded 34GiB to GPU in 2.00868s (16.9265GiB/s) | 20260505 23:16:41.378 | INFO | Analyzing chunk frames = [7084032,7084544) | 20260505 23:16:41.378 | INFO | MemInfo: { | 20260505 23:16:41.378 | INFO | "gpu" : " 17 GB", | 20260505 23:16:41.378 | INFO | "huge" : "", | 20260505 23:16:41.378 | INFO | "huge_shared" : " 27 GB", | 20260505 23:16:41.378 | INFO | "malloc" : " 14 GB", | 20260505 23:16:41.378 | INFO | "pinned" : " 29 GB" | 20260505 23:16:41.378 | INFO | } | 20260505 23:16:41.378 | INFO | Perf report for the last 5.13069 seconds: | 20260505 23:16:41.379 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:16:41.379 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:41.379 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:41.379 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:41.379 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260505 23:16:41.379 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:41.379 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:41.379 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:16:41.379 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:41.379 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:41.379 | INFO | Uploaded 36GiB to GPU in 2.08926s (17.231GiB/s) | 20260505 23:16:41.379 | INFO | Downloaded 34GiB to GPU in 2.04528s (16.6236GiB/s) | 20260505 23:16:46.517 | INFO | Analyzing chunk frames = [7084544,7085056) | 20260505 23:16:46.517 | INFO | MemInfo: { | 20260505 23:16:46.517 | INFO | "gpu" : " 17 GB", | 20260505 23:16:46.517 | INFO | "huge" : "", | 20260505 23:16:46.517 | INFO | "huge_shared" : " 27 GB", | 20260505 23:16:46.517 | INFO | "malloc" : " 14 GB", | 20260505 23:16:46.517 | INFO | "pinned" : " 29 GB" | 20260505 23:16:46.517 | INFO | } | 20260505 23:16:46.517 | INFO | Perf report for the last 5.13822 seconds: | 20260505 23:16:46.517 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:16:46.517 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:46.517 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:46.517 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:46.517 | INFO | ANALYSIS : 61.6%, 3.0, 96.2ms, 0 | 20260505 23:16:46.517 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:46.517 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:46.517 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:16:46.517 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:46.517 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:46.517 | INFO | Uploaded 36GiB to GPU in 2.07658s (17.3362GiB/s) | 20260505 23:16:46.517 | INFO | Downloaded 34GiB to GPU in 2.03543s (16.7041GiB/s) | 20260505 23:16:51.663 | INFO | Analyzing chunk frames = [7085056,7085568) | 20260505 23:16:51.663 | INFO | MemInfo: { | 20260505 23:16:51.663 | INFO | "gpu" : " 17 GB", | 20260505 23:16:51.663 | INFO | "huge" : "", | 20260505 23:16:51.663 | INFO | "huge_shared" : " 27 GB", | 20260505 23:16:51.663 | INFO | "malloc" : " 14 GB", | 20260505 23:16:51.663 | INFO | "pinned" : " 29 GB" | 20260505 23:16:51.663 | INFO | } | 20260505 23:16:51.663 | INFO | Perf report for the last 5.14647 seconds: | 20260505 23:16:51.663 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:16:51.663 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:51.663 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:51.663 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:51.663 | INFO | ANALYSIS : 62.7%, 3.0, 98.0ms, 0 | 20260505 23:16:51.663 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:51.663 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:51.663 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 23:16:51.663 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:51.663 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:51.663 | INFO | Uploaded 36GiB to GPU in 2.14504s (16.7829GiB/s) | 20260505 23:16:51.663 | INFO | Downloaded 34GiB to GPU in 2.13252s (15.9436GiB/s) | 20260505 23:16:56.802 | INFO | Analyzing chunk frames = [7085568,7086080) | 20260505 23:16:56.802 | INFO | MemInfo: { | 20260505 23:16:56.802 | INFO | "gpu" : " 17 GB", | 20260505 23:16:56.802 | INFO | "huge" : "", | 20260505 23:16:56.802 | INFO | "huge_shared" : " 27 GB", | 20260505 23:16:56.802 | INFO | "malloc" : " 14 GB", | 20260505 23:16:56.802 | INFO | "pinned" : " 29 GB" | 20260505 23:16:56.802 | INFO | } | 20260505 23:16:56.803 | INFO | Perf report for the last 5.1393 seconds: | 20260505 23:16:56.803 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:16:56.803 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:56.803 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:56.803 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:56.803 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260505 23:16:56.803 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:56.803 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:56.803 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 23:16:56.803 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:56.803 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:16:56.803 | INFO | Uploaded 36GiB to GPU in 2.11074s (17.0556GiB/s) | 20260505 23:16:56.803 | INFO | Downloaded 34GiB to GPU in 2.00843s (16.9287GiB/s) | 20260505 23:16:56.803 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 23:16:56.803 | INFO | Total mem usage: 15.3 GiB | 20260505 23:16:56.803 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 23:16:56.803 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.11%) | 20260505 23:16:56.803 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 23:16:56.803 | INFO | High Water Breakdown: | 20260505 23:16:56.803 | INFO | 60.9% : 8.55 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 23:16:56.803 | INFO | 39% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 23:16:56.803 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 23:16:56.803 | INFO | ------------------------------------------ | 20260505 23:16:56.803 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:16:56.803 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 23:16:56.803 | INFO | Total mem usage: 26.5 GiB | 20260505 23:16:56.803 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 23:16:56.803 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 23:16:56.803 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 23:16:56.803 | INFO | High Water Breakdown: | 20260505 23:16:56.803 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:16:56.803 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:16:56.803 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:16:56.803 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:16:56.803 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:16:56.803 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:16:56.803 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 23:16:56.803 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 23:16:56.803 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:16:56.803 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 23:16:56.803 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 23:16:56.803 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 23:16:56.803 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 23:16:56.803 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 23:16:56.803 | INFO | ------------------------------------------ | 20260505 23:16:56.803 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:16:56.803 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 23:16:56.803 | INFO | Total mem usage: 25.6 GiB | 20260505 23:16:56.803 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 23:16:56.803 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 23:16:56.803 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 23:16:56.803 | INFO | High Water Breakdown: | 20260505 23:16:56.803 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 23:16:56.803 | INFO | ------------------------------------------ | 20260505 23:16:56.803 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:16:56.803 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 23:16:56.803 | INFO | Total mem usage: 15.7 GiB | 20260505 23:16:56.803 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 23:16:56.803 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 23:16:56.803 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 23:16:56.803 | INFO | High Water Breakdown: | 20260505 23:16:56.803 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 23:16:56.803 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 23:16:56.803 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 23:16:56.803 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 23:16:56.803 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 23:16:56.803 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 23:16:56.803 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 23:16:56.803 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 23:16:56.803 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 23:16:56.803 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:16:56.803 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 23:16:56.803 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:16:56.803 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 23:16:56.803 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 23:16:56.803 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 23:16:56.803 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 23:16:56.803 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:16:56.803 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 23:16:56.803 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:16:56.803 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:16:56.803 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:16:56.803 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 23:16:56.803 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 23:16:56.803 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:16:56.803 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 23:16:56.803 | INFO | ------------------------------------------ | 20260505 23:16:56.803 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:16:56.804 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 23:16:56.804 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:16:56.804 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:16:56.804 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.39e+01, 1.34e+01 / 1.09e+00 | 20260505 23:16:56.804 | INFO | PerfReport: 0.30%, AggregateStats -- 9.25e-02 / 1.04e-01, 1.05e-02 / 5.45e-03 | 20260505 23:16:56.804 | INFO | PerfReport: 11.84%, Baseline -- 3.71e+00 / 4.55e+00, 5.61e-01 / 1.33e-01 | 20260505 23:16:56.804 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.52e+00, 1.86e-01 / 7.98e-02 | 20260505 23:16:56.804 | INFO | PerfReport: 15.99%, DME -- 4.01e+01 / 4.39e+01, 3.77e+00 / 1.23e+00 | 20260505 23:16:56.804 | INFO | PerfReport: 57.24%, FrameLabeling -- 1.79e+01 / 1.95e+01, 1.42e+00 / 1.08e+00 | 20260505 23:16:56.804 | INFO | PerfReport: 7.15%, PulseAccumulating -- 2.24e+00 / 2.75e+00, 4.22e-01 / 2.30e-01 | 20260505 23:16:56.804 | INFO | PerfReport: 3.13%, Metrics -- 9.80e-01 / 1.28e+00, 2.64e-01 / 1.56e-01 | 20260505 23:16:56.804 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.39e-01 / 3.99e-01, 5.78e-02 / 3.10e-02 | 20260505 23:16:56.804 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 23:16:56.804 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:16:56.804 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:16:56.804 | INFO | PerfReport: Full Execution -- 9.86e+01 / -nan, 2.46e+01 / 0.00e+00 | 20260505 23:16:56.804 | INFO | PerfReport: 22.39%, Upload -- 2.21e+01 / 0.00e+00, 7.50e+00 / 0.00e+00 | 20260505 23:16:56.804 | INFO | PerfReport: 21.84%, Download -- 2.15e+01 / 0.00e+00, 7.54e+00 / 0.00e+00 | 20260505 23:16:56.804 | INFO | PerfReport: 31.77%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 23:16:56.804 | INFO | PerfReport: 2.18%, LockUpload -- 2.15e+00 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260505 23:16:56.804 | INFO | PerfReport: 1.35%, LockDownload -- 1.33e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260505 23:16:56.804 | INFO | PerfReport: 20.47%, LockCompute -- 2.02e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260505 23:16:56.804 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.71e-02 / 0.00e+00 | 20260505 23:16:56.929 | INFO | Starting a round of Baz IO | 20260505 23:17:00.079 | INFO | Populating RTMetrics report | 20260505 23:17:00.079 | INFO | RTMetrics report populated | 20260505 23:17:00.079 | INFO | Opening tmp RTMetrics json file | 20260505 23:17:00.083 | INFO | Serializing RTMetrics to json | 20260505 23:17:00.084 | INFO | Serialization done, swapping tmp file now... | 20260505 23:17:00.085 | INFO | RTMetrics done | 20260505 23:17:00.156 | INFO | Finished a round of Baz IO: Wrote 1.899 GiB in 3.227 seconds | 20260505 23:17:00.157 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.44% / 0.00% / 78.56% | 20260505 23:17:00.157 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.109704% / 0% / 9.53674e-05% | 20260505 23:17:00.157 | INFO | Baz IO overall: | 20260505 23:17:00.157 | INFO | BytesWritten : 4057257706469 | 20260505 23:17:00.157 | INFO | NumEvents : 1911869244611 | 20260505 23:17:01.947 | INFO | Analyzing chunk frames = [7086080,7086592) | 20260505 23:17:01.947 | INFO | MemInfo: { | 20260505 23:17:01.947 | INFO | "gpu" : " 17 GB", | 20260505 23:17:01.947 | INFO | "huge" : "", | 20260505 23:17:01.947 | INFO | "huge_shared" : " 27 GB", | 20260505 23:17:01.947 | INFO | "malloc" : " 14 GB", | 20260505 23:17:01.947 | INFO | "pinned" : " 29 GB" | 20260505 23:17:01.947 | INFO | } | 20260505 23:17:01.947 | INFO | Perf report for the last 5.14424 seconds: | 20260505 23:17:01.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:17:01.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:01.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:01.947 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:01.947 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260505 23:17:01.947 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:01.947 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260505 23:17:01.947 | INFO | PRE_HQ : 59.9%, 1.8, 54.8ms, 0 | 20260505 23:17:01.947 | INFO | BAZWRITER : 62.0%, 3.6, 115.7ms, 0 | 20260505 23:17:01.947 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 23:17:01.947 | INFO | Uploaded 36GiB to GPU in 2.16239s (16.6482GiB/s) | 20260505 23:17:01.947 | INFO | Downloaded 38GiB to GPU in 2.2309s (17.0335GiB/s) | 20260505 23:17:07.081 | INFO | Analyzing chunk frames = [7086592,7087104) | 20260505 23:17:07.081 | INFO | MemInfo: { | 20260505 23:17:07.081 | INFO | "gpu" : " 17 GB", | 20260505 23:17:07.081 | INFO | "huge" : "", | 20260505 23:17:07.081 | INFO | "huge_shared" : " 27 GB", | 20260505 23:17:07.082 | INFO | "malloc" : " 14 GB", | 20260505 23:17:07.082 | INFO | "pinned" : " 29 GB" | 20260505 23:17:07.082 | INFO | } | 20260505 23:17:07.082 | INFO | Perf report for the last 5.13484 seconds: | 20260505 23:17:07.082 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:17:07.082 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:07.082 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:07.082 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:07.082 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 23:17:07.082 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:07.082 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:07.082 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260505 23:17:07.082 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:07.082 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:07.082 | INFO | Uploaded 36GiB to GPU in 2.08818s (17.2399GiB/s) | 20260505 23:17:07.082 | INFO | Downloaded 34GiB to GPU in 2.0332s (16.7224GiB/s) | 20260505 23:17:12.214 | INFO | Analyzing chunk frames = [7087104,7087616) | 20260505 23:17:12.214 | INFO | MemInfo: { | 20260505 23:17:12.214 | INFO | "gpu" : " 17 GB", | 20260505 23:17:12.214 | INFO | "huge" : "", | 20260505 23:17:12.214 | INFO | "huge_shared" : " 27 GB", | 20260505 23:17:12.214 | INFO | "malloc" : " 14 GB", | 20260505 23:17:12.214 | INFO | "pinned" : " 29 GB" | 20260505 23:17:12.214 | INFO | } | 20260505 23:17:12.214 | INFO | Perf report for the last 5.13225 seconds: | 20260505 23:17:12.214 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:17:12.214 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:12.214 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:12.214 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:12.214 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260505 23:17:12.214 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:12.214 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:12.214 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:17:12.214 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:12.214 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:12.214 | INFO | Uploaded 36GiB to GPU in 2.14153s (16.8104GiB/s) | 20260505 23:17:12.214 | INFO | Downloaded 34GiB to GPU in 2.01946s (16.8362GiB/s) | 20260505 23:17:17.364 | INFO | Analyzing chunk frames = [7087616,7088128) | 20260505 23:17:17.364 | INFO | MemInfo: { | 20260505 23:17:17.364 | INFO | "gpu" : " 17 GB", | 20260505 23:17:17.364 | INFO | "huge" : "", | 20260505 23:17:17.364 | INFO | "huge_shared" : " 27 GB", | 20260505 23:17:17.364 | INFO | "malloc" : " 14 GB", | 20260505 23:17:17.364 | INFO | "pinned" : " 29 GB" | 20260505 23:17:17.364 | INFO | } | 20260505 23:17:17.364 | INFO | Perf report for the last 5.15051 seconds: | 20260505 23:17:17.364 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:17:17.364 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:17.364 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:17.364 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:17.365 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260505 23:17:17.365 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:17.365 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:17.365 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 23:17:17.365 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:17.365 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:17.365 | INFO | Uploaded 36GiB to GPU in 2.14027s (16.8203GiB/s) | 20260505 23:17:17.365 | INFO | Downloaded 34GiB to GPU in 2.02879s (16.7588GiB/s) | 20260505 23:17:22.502 | INFO | Analyzing chunk frames = [7088128,7088640) | 20260505 23:17:22.502 | INFO | MemInfo: { | 20260505 23:17:22.502 | INFO | "gpu" : " 17 GB", | 20260505 23:17:22.502 | INFO | "huge" : "", | 20260505 23:17:22.502 | INFO | "huge_shared" : " 27 GB", | 20260505 23:17:22.502 | INFO | "malloc" : " 14 GB", | 20260505 23:17:22.502 | INFO | "pinned" : " 29 GB" | 20260505 23:17:22.502 | INFO | } | 20260505 23:17:22.502 | INFO | Perf report for the last 5.13788 seconds: | 20260505 23:17:22.502 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:17:22.502 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:22.502 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:22.502 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:22.502 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260505 23:17:22.502 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:22.502 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:22.502 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 23:17:22.502 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:22.502 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:22.503 | INFO | Uploaded 36GiB to GPU in 2.09842s (17.1558GiB/s) | 20260505 23:17:22.503 | INFO | Downloaded 34GiB to GPU in 2.04487s (16.627GiB/s) | 20260505 23:17:27.643 | INFO | Analyzing chunk frames = [7088640,7089152) | 20260505 23:17:27.643 | INFO | MemInfo: { | 20260505 23:17:27.643 | INFO | "gpu" : " 17 GB", | 20260505 23:17:27.643 | INFO | "huge" : "", | 20260505 23:17:27.643 | INFO | "huge_shared" : " 27 GB", | 20260505 23:17:27.643 | INFO | "malloc" : " 14 GB", | 20260505 23:17:27.643 | INFO | "pinned" : " 29 GB" | 20260505 23:17:27.643 | INFO | } | 20260505 23:17:27.643 | INFO | Perf report for the last 5.14048 seconds: | 20260505 23:17:27.643 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:17:27.643 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:27.643 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:27.643 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:27.643 | INFO | ANALYSIS : 61.3%, 3.0, 95.8ms, 0 | 20260505 23:17:27.643 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:27.643 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:27.643 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 23:17:27.643 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:27.643 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:27.643 | INFO | Uploaded 36GiB to GPU in 2.07619s (17.3394GiB/s) | 20260505 23:17:27.643 | INFO | Downloaded 34GiB to GPU in 2.03312s (16.7231GiB/s) | 20260505 23:17:32.776 | INFO | Analyzing chunk frames = [7089152,7089664) | 20260505 23:17:32.776 | INFO | MemInfo: { | 20260505 23:17:32.776 | INFO | "gpu" : " 17 GB", | 20260505 23:17:32.776 | INFO | "huge" : "", | 20260505 23:17:32.776 | INFO | "huge_shared" : " 27 GB", | 20260505 23:17:32.776 | INFO | "malloc" : " 14 GB", | 20260505 23:17:32.776 | INFO | "pinned" : " 29 GB" | 20260505 23:17:32.776 | INFO | } | 20260505 23:17:32.776 | INFO | Perf report for the last 5.13307 seconds: | 20260505 23:17:32.776 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:17:32.776 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:32.776 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:32.776 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:32.776 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 23:17:32.776 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:32.776 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:32.776 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 23:17:32.776 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:32.776 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:32.776 | INFO | Uploaded 36GiB to GPU in 2.00634s (17.9431GiB/s) | 20260505 23:17:32.776 | INFO | Downloaded 34GiB to GPU in 2.03321s (16.7223GiB/s) | 20260505 23:17:37.919 | INFO | Analyzing chunk frames = [7089664,7090176) | 20260505 23:17:37.919 | INFO | MemInfo: { | 20260505 23:17:37.919 | INFO | "gpu" : " 17 GB", | 20260505 23:17:37.919 | INFO | "huge" : "", | 20260505 23:17:37.919 | INFO | "huge_shared" : " 27 GB", | 20260505 23:17:37.919 | INFO | "malloc" : " 14 GB", | 20260505 23:17:37.919 | INFO | "pinned" : " 29 GB" | 20260505 23:17:37.919 | INFO | } | 20260505 23:17:37.919 | INFO | Perf report for the last 5.14344 seconds: | 20260505 23:17:37.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:17:37.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:37.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:37.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:37.919 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 23:17:37.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:37.919 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:37.919 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 23:17:37.919 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:37.919 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:37.920 | INFO | Uploaded 36GiB to GPU in 2.12327s (16.955GiB/s) | 20260505 23:17:37.920 | INFO | Downloaded 34GiB to GPU in 2.00639s (16.9459GiB/s) | 20260505 23:17:38.038 | INFO | Starting a round of Baz IO | 20260505 23:17:41.189 | INFO | Populating RTMetrics report | 20260505 23:17:41.190 | INFO | RTMetrics report populated | 20260505 23:17:41.190 | INFO | Opening tmp RTMetrics json file | 20260505 23:17:41.191 | INFO | Serializing RTMetrics to json | 20260505 23:17:41.192 | INFO | Serialization done, swapping tmp file now... | 20260505 23:17:41.192 | INFO | RTMetrics done | 20260505 23:17:41.269 | INFO | Finished a round of Baz IO: Wrote 1.897 GiB in 3.23 seconds | 20260505 23:17:41.269 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.44% / 0.00% / 78.56% | 20260505 23:17:41.269 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.109661% / 0% / 7.15256e-05% | 20260505 23:17:41.269 | INFO | Baz IO overall: | 20260505 23:17:41.269 | INFO | BytesWritten : 4059294142982 | 20260505 23:17:41.269 | INFO | NumEvents : 1912803340731 | 20260505 23:17:43.060 | INFO | Analyzing chunk frames = [7090176,7090688) | 20260505 23:17:43.060 | INFO | MemInfo: { | 20260505 23:17:43.060 | INFO | "gpu" : " 17 GB", | 20260505 23:17:43.060 | INFO | "huge" : "", | 20260505 23:17:43.060 | INFO | "huge_shared" : " 27 GB", | 20260505 23:17:43.060 | INFO | "malloc" : " 14 GB", | 20260505 23:17:43.060 | INFO | "pinned" : " 29 GB" | 20260505 23:17:43.060 | INFO | } | 20260505 23:17:43.060 | INFO | Perf report for the last 5.14028 seconds: | 20260505 23:17:43.060 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:17:43.060 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:43.060 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:43.060 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:43.060 | INFO | ANALYSIS : 62.6%, 3.0, 97.2ms, 0 | 20260505 23:17:43.060 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:43.060 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260505 23:17:43.060 | INFO | PRE_HQ : 60.0%, 1.8, 54.5ms, 0 | 20260505 23:17:43.060 | INFO | BAZWRITER : 61.9%, 3.9, 124.6ms, 0 | 20260505 23:17:43.060 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 23:17:43.060 | INFO | Uploaded 36GiB to GPU in 2.17171s (16.5768GiB/s) | 20260505 23:17:43.060 | INFO | Downloaded 38GiB to GPU in 2.24635s (16.9163GiB/s) | 20260505 23:17:48.195 | INFO | Analyzing chunk frames = [7090688,7091200) | 20260505 23:17:48.195 | INFO | MemInfo: { | 20260505 23:17:48.195 | INFO | "gpu" : " 17 GB", | 20260505 23:17:48.195 | INFO | "huge" : "", | 20260505 23:17:48.195 | INFO | "huge_shared" : " 27 GB", | 20260505 23:17:48.195 | INFO | "malloc" : " 14 GB", | 20260505 23:17:48.195 | INFO | "pinned" : " 29 GB" | 20260505 23:17:48.195 | INFO | } | 20260505 23:17:48.195 | INFO | Perf report for the last 5.13549 seconds: | 20260505 23:17:48.195 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:17:48.195 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:48.195 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:48.195 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:48.195 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260505 23:17:48.195 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:48.195 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:48.195 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:17:48.195 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:48.195 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:48.195 | INFO | Uploaded 36GiB to GPU in 2.09366s (17.1948GiB/s) | 20260505 23:17:48.195 | INFO | Downloaded 34GiB to GPU in 2.03779s (16.6847GiB/s) | 20260505 23:17:53.333 | INFO | Analyzing chunk frames = [7091200,7091712) | 20260505 23:17:53.333 | INFO | MemInfo: { | 20260505 23:17:53.333 | INFO | "gpu" : " 17 GB", | 20260505 23:17:53.333 | INFO | "huge" : "", | 20260505 23:17:53.333 | INFO | "huge_shared" : " 27 GB", | 20260505 23:17:53.333 | INFO | "malloc" : " 14 GB", | 20260505 23:17:53.333 | INFO | "pinned" : " 29 GB" | 20260505 23:17:53.333 | INFO | } | 20260505 23:17:53.333 | INFO | Perf report for the last 5.1382 seconds: | 20260505 23:17:53.333 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:17:53.333 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:53.333 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:53.333 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:53.333 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260505 23:17:53.333 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:53.333 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:53.333 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:17:53.333 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:53.333 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:53.334 | INFO | Uploaded 36GiB to GPU in 2.08848s (17.2374GiB/s) | 20260505 23:17:53.334 | INFO | Downloaded 34GiB to GPU in 2.02253s (16.8106GiB/s) | 20260505 23:17:58.469 | INFO | Analyzing chunk frames = [7091712,7092224) | 20260505 23:17:58.469 | INFO | MemInfo: { | 20260505 23:17:58.469 | INFO | "gpu" : " 17 GB", | 20260505 23:17:58.469 | INFO | "huge" : "", | 20260505 23:17:58.469 | INFO | "huge_shared" : " 27 GB", | 20260505 23:17:58.469 | INFO | "malloc" : " 14 GB", | 20260505 23:17:58.469 | INFO | "pinned" : " 29 GB" | 20260505 23:17:58.469 | INFO | } | 20260505 23:17:58.469 | INFO | Perf report for the last 5.13575 seconds: | 20260505 23:17:58.469 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:17:58.469 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:58.469 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:58.469 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:58.469 | INFO | ANALYSIS : 61.9%, 3.0, 96.1ms, 0 | 20260505 23:17:58.469 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:58.469 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:58.469 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260505 23:17:58.469 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:58.469 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:17:58.469 | INFO | Uploaded 36GiB to GPU in 2.14472s (16.7854GiB/s) | 20260505 23:17:58.469 | INFO | Downloaded 34GiB to GPU in 2.04092s (16.6591GiB/s) | 20260505 23:18:03.617 | INFO | Analyzing chunk frames = [7092224,7092736) | 20260505 23:18:03.617 | INFO | MemInfo: { | 20260505 23:18:03.617 | INFO | "gpu" : " 17 GB", | 20260505 23:18:03.617 | INFO | "huge" : "", | 20260505 23:18:03.617 | INFO | "huge_shared" : " 27 GB", | 20260505 23:18:03.617 | INFO | "malloc" : " 14 GB", | 20260505 23:18:03.617 | INFO | "pinned" : " 29 GB" | 20260505 23:18:03.617 | INFO | } | 20260505 23:18:03.617 | INFO | Perf report for the last 5.14803 seconds: | 20260505 23:18:03.617 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:18:03.617 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:03.617 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:03.617 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:03.617 | INFO | ANALYSIS : 61.4%, 3.0, 96.0ms, 0 | 20260505 23:18:03.617 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:03.617 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:03.617 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 23:18:03.617 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:03.617 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:03.617 | INFO | Uploaded 36GiB to GPU in 2.14029s (16.8201GiB/s) | 20260505 23:18:03.617 | INFO | Downloaded 34GiB to GPU in 2.06481s (16.4664GiB/s) | 20260505 23:18:08.758 | INFO | Analyzing chunk frames = [7092736,7093248) | 20260505 23:18:08.758 | INFO | MemInfo: { | 20260505 23:18:08.758 | INFO | "gpu" : " 17 GB", | 20260505 23:18:08.758 | INFO | "huge" : "", | 20260505 23:18:08.758 | INFO | "huge_shared" : " 27 GB", | 20260505 23:18:08.759 | INFO | "malloc" : " 14 GB", | 20260505 23:18:08.759 | INFO | "pinned" : " 29 GB" | 20260505 23:18:08.759 | INFO | } | 20260505 23:18:08.759 | INFO | Perf report for the last 5.14143 seconds: | 20260505 23:18:08.759 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:18:08.759 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:08.759 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:08.759 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:08.759 | INFO | ANALYSIS : 61.2%, 3.0, 95.6ms, 0 | 20260505 23:18:08.759 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:08.759 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:08.759 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260505 23:18:08.759 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:08.759 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:08.759 | INFO | Uploaded 36GiB to GPU in 2.06413s (17.4408GiB/s) | 20260505 23:18:08.759 | INFO | Downloaded 34GiB to GPU in 2.0303s (16.7463GiB/s) | 20260505 23:18:13.893 | INFO | Analyzing chunk frames = [7093248,7093760) | 20260505 23:18:13.893 | INFO | MemInfo: { | 20260505 23:18:13.893 | INFO | "gpu" : " 17 GB", | 20260505 23:18:13.893 | INFO | "huge" : "", | 20260505 23:18:13.893 | INFO | "huge_shared" : " 27 GB", | 20260505 23:18:13.893 | INFO | "malloc" : " 14 GB", | 20260505 23:18:13.893 | INFO | "pinned" : " 29 GB" | 20260505 23:18:13.893 | INFO | } | 20260505 23:18:13.893 | INFO | Perf report for the last 5.13448 seconds: | 20260505 23:18:13.893 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:18:13.893 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:13.893 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:13.893 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:13.893 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 23:18:13.893 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:13.893 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:13.893 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 23:18:13.893 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:13.893 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:13.893 | INFO | Uploaded 36GiB to GPU in 2.02452s (17.782GiB/s) | 20260505 23:18:13.893 | INFO | Downloaded 34GiB to GPU in 2.04545s (16.6222GiB/s) | 20260505 23:18:19.025 | INFO | Analyzing chunk frames = [7093760,7094272) | 20260505 23:18:19.025 | INFO | MemInfo: { | 20260505 23:18:19.025 | INFO | "gpu" : " 17 GB", | 20260505 23:18:19.025 | INFO | "huge" : "", | 20260505 23:18:19.025 | INFO | "huge_shared" : " 27 GB", | 20260505 23:18:19.025 | INFO | "malloc" : " 14 GB", | 20260505 23:18:19.025 | INFO | "pinned" : " 29 GB" | 20260505 23:18:19.025 | INFO | } | 20260505 23:18:19.025 | INFO | Perf report for the last 5.13238 seconds: | 20260505 23:18:19.025 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:18:19.025 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:19.026 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:19.026 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:19.026 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260505 23:18:19.026 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:19.026 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:19.026 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:18:19.026 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:19.026 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:19.026 | INFO | Uploaded 36GiB to GPU in 2.10783s (17.0792GiB/s) | 20260505 23:18:19.026 | INFO | Downloaded 34GiB to GPU in 1.99668s (17.0283GiB/s) | 20260505 23:18:19.026 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 23:18:19.026 | INFO | Total mem usage: 15.3 GiB | 20260505 23:18:19.026 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 23:18:19.026 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -3.59%) | 20260505 23:18:19.026 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 23:18:19.026 | INFO | High Water Breakdown: | 20260505 23:18:19.026 | INFO | 61.7% : 9.12 GiB ( 95.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 23:18:19.026 | INFO | 38.2% : 5.65 GiB ( 85.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 23:18:19.026 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 23:18:19.026 | INFO | ------------------------------------------ | 20260505 23:18:19.026 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:18:19.026 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 23:18:19.026 | INFO | Total mem usage: 26.5 GiB | 20260505 23:18:19.026 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 23:18:19.026 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 23:18:19.026 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 23:18:19.026 | INFO | High Water Breakdown: | 20260505 23:18:19.026 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:18:19.026 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:18:19.026 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:18:19.026 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:18:19.026 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:18:19.026 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:18:19.026 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 23:18:19.026 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 23:18:19.026 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:18:19.026 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 23:18:19.026 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 23:18:19.026 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 23:18:19.026 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 23:18:19.026 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 23:18:19.026 | INFO | ------------------------------------------ | 20260505 23:18:19.026 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:18:19.026 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 23:18:19.026 | INFO | Total mem usage: 25.6 GiB | 20260505 23:18:19.026 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 23:18:19.026 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 23:18:19.026 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 23:18:19.026 | INFO | High Water Breakdown: | 20260505 23:18:19.026 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 23:18:19.026 | INFO | ------------------------------------------ | 20260505 23:18:19.026 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:18:19.026 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 23:18:19.026 | INFO | Total mem usage: 15.7 GiB | 20260505 23:18:19.026 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 23:18:19.027 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 23:18:19.027 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 23:18:19.027 | INFO | High Water Breakdown: | 20260505 23:18:19.027 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 23:18:19.027 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 23:18:19.027 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 23:18:19.027 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 23:18:19.027 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 23:18:19.027 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 23:18:19.027 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 23:18:19.027 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 23:18:19.027 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 23:18:19.027 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:18:19.027 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 23:18:19.027 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:18:19.027 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 23:18:19.027 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 23:18:19.027 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 23:18:19.027 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 23:18:19.027 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:18:19.027 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 23:18:19.027 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:18:19.027 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:18:19.027 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:18:19.027 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 23:18:19.027 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 23:18:19.027 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:18:19.027 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 23:18:19.027 | INFO | ------------------------------------------ | 20260505 23:18:19.027 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:18:19.027 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 23:18:19.027 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:18:19.027 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:18:19.027 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.41e+01, 1.35e+01 / 7.94e-01 | 20260505 23:18:19.027 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.04e-01, 1.03e-02 / 4.18e-03 | 20260505 23:18:19.027 | INFO | PerfReport: 11.83%, Baseline -- 3.71e+00 / 4.50e+00, 5.75e-01 / 1.50e-01 | 20260505 23:18:19.027 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.52e+00, 1.93e-01 / 8.51e-02 | 20260505 23:18:19.027 | INFO | PerfReport: 16.00%, DME -- 4.02e+01 / 4.43e+01, 3.85e+00 / 9.24e-01 | 20260505 23:18:19.027 | INFO | PerfReport: 57.29%, FrameLabeling -- 1.80e+01 / 1.93e+01, 1.53e+00 / 1.01e+00 | 20260505 23:18:19.027 | INFO | PerfReport: 7.11%, PulseAccumulating -- 2.23e+00 / 2.77e+00, 4.27e-01 / 1.98e-01 | 20260505 23:18:19.027 | INFO | PerfReport: 3.11%, Metrics -- 9.75e-01 / 1.26e+00, 2.66e-01 / 1.04e-01 | 20260505 23:18:19.027 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.39e-01 / 3.95e-01, 5.88e-02 / 2.91e-02 | 20260505 23:18:19.027 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 23:18:19.027 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:18:19.027 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:18:19.027 | INFO | PerfReport: Full Execution -- 9.84e+01 / -nan, 2.47e+01 / 0.00e+00 | 20260505 23:18:19.027 | INFO | PerfReport: 22.28%, Upload -- 2.19e+01 / 0.00e+00, 7.49e+00 / 0.00e+00 | 20260505 23:18:19.027 | INFO | PerfReport: 21.78%, Download -- 2.14e+01 / 0.00e+00, 7.55e+00 / 0.00e+00 | 20260505 23:18:19.027 | INFO | PerfReport: 31.89%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 23:18:19.027 | INFO | PerfReport: 2.11%, LockUpload -- 2.08e+00 / 0.00e+00, 7.75e+00 / 0.00e+00 | 20260505 23:18:19.027 | INFO | PerfReport: 1.31%, LockDownload -- 1.29e+00 / 0.00e+00, 2.93e+00 / 0.00e+00 | 20260505 23:18:19.027 | INFO | PerfReport: 20.62%, LockCompute -- 2.03e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260505 23:18:19.027 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.78e-02 / 0.00e+00 | 20260505 23:18:19.155 | INFO | Starting a round of Baz IO | 20260505 23:18:22.305 | INFO | Populating RTMetrics report | 20260505 23:18:22.305 | INFO | RTMetrics report populated | 20260505 23:18:22.305 | INFO | Opening tmp RTMetrics json file | 20260505 23:18:22.308 | INFO | Serializing RTMetrics to json | 20260505 23:18:22.311 | INFO | Serialization done, swapping tmp file now... | 20260505 23:18:22.312 | INFO | RTMetrics done | 20260505 23:18:22.386 | INFO | Finished a round of Baz IO: Wrote 1.898 GiB in 3.23 seconds | 20260505 23:18:22.386 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.44% / 0.00% / 78.56% | 20260505 23:18:22.386 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.109239% / 0% / 9.93411e-05% | 20260505 23:18:22.386 | INFO | Baz IO overall: | 20260505 23:18:22.386 | INFO | BytesWritten : 4061331978115 | 20260505 23:18:22.386 | INFO | NumEvents : 1913738393281 | 20260505 23:18:24.172 | INFO | Analyzing chunk frames = [7094272,7094784) | 20260505 23:18:24.172 | INFO | MemInfo: { | 20260505 23:18:24.172 | INFO | "gpu" : " 17 GB", | 20260505 23:18:24.172 | INFO | "huge" : "", | 20260505 23:18:24.172 | INFO | "huge_shared" : " 27 GB", | 20260505 23:18:24.172 | INFO | "malloc" : " 14 GB", | 20260505 23:18:24.172 | INFO | "pinned" : " 29 GB" | 20260505 23:18:24.172 | INFO | } | 20260505 23:18:24.172 | INFO | Perf report for the last 5.14644 seconds: | 20260505 23:18:24.172 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:18:24.172 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:24.172 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:24.172 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:24.172 | INFO | ANALYSIS : 62.7%, 3.0, 98.0ms, 0 | 20260505 23:18:24.172 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:24.172 | INFO | RT_METRICS : 43.3%, 1.0, 22.8ms, 0 | 20260505 23:18:24.172 | INFO | PRE_HQ : 59.4%, 1.8, 54.1ms, 0 | 20260505 23:18:24.172 | INFO | BAZWRITER : 62.1%, 3.6, 114.7ms, 0 | 20260505 23:18:24.172 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 23:18:24.172 | INFO | Uploaded 36GiB to GPU in 2.12816s (16.916GiB/s) | 20260505 23:18:24.172 | INFO | Downloaded 38GiB to GPU in 2.23814s (16.9784GiB/s) | 20260505 23:18:29.315 | INFO | Analyzing chunk frames = [7094784,7095296) | 20260505 23:18:29.315 | INFO | MemInfo: { | 20260505 23:18:29.315 | INFO | "gpu" : " 17 GB", | 20260505 23:18:29.315 | INFO | "huge" : "", | 20260505 23:18:29.315 | INFO | "huge_shared" : " 27 GB", | 20260505 23:18:29.315 | INFO | "malloc" : " 14 GB", | 20260505 23:18:29.315 | INFO | "pinned" : " 29 GB" | 20260505 23:18:29.315 | INFO | } | 20260505 23:18:29.315 | INFO | Perf report for the last 5.14347 seconds: | 20260505 23:18:29.315 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:18:29.315 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:29.315 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:29.315 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:29.315 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 23:18:29.315 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:29.315 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:29.316 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260505 23:18:29.316 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:29.316 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:29.316 | INFO | Uploaded 36GiB to GPU in 2.0902s (17.2232GiB/s) | 20260505 23:18:29.316 | INFO | Downloaded 34GiB to GPU in 2.03653s (16.6951GiB/s) | 20260505 23:18:34.455 | INFO | Analyzing chunk frames = [7095296,7095808) | 20260505 23:18:34.455 | INFO | MemInfo: { | 20260505 23:18:34.455 | INFO | "gpu" : " 17 GB", | 20260505 23:18:34.455 | INFO | "huge" : "", | 20260505 23:18:34.455 | INFO | "huge_shared" : " 27 GB", | 20260505 23:18:34.455 | INFO | "malloc" : " 14 GB", | 20260505 23:18:34.455 | INFO | "pinned" : " 29 GB" | 20260505 23:18:34.455 | INFO | } | 20260505 23:18:34.455 | INFO | Perf report for the last 5.13949 seconds: | 20260505 23:18:34.455 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:18:34.455 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:34.455 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:34.455 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:34.455 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260505 23:18:34.455 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:34.455 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:34.455 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:18:34.455 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:34.455 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:34.455 | INFO | Uploaded 36GiB to GPU in 2.06953s (17.3953GiB/s) | 20260505 23:18:34.455 | INFO | Downloaded 34GiB to GPU in 2.01592s (16.8657GiB/s) | 20260505 23:18:39.593 | INFO | Analyzing chunk frames = [7095808,7096320) | 20260505 23:18:39.593 | INFO | MemInfo: { | 20260505 23:18:39.593 | INFO | "gpu" : " 17 GB", | 20260505 23:18:39.593 | INFO | "huge" : "", | 20260505 23:18:39.593 | INFO | "huge_shared" : " 27 GB", | 20260505 23:18:39.593 | INFO | "malloc" : " 14 GB", | 20260505 23:18:39.593 | INFO | "pinned" : " 29 GB" | 20260505 23:18:39.593 | INFO | } | 20260505 23:18:39.593 | INFO | Perf report for the last 5.13805 seconds: | 20260505 23:18:39.593 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:18:39.593 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:39.593 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:39.593 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:39.593 | INFO | ANALYSIS : 61.7%, 3.0, 95.8ms, 0 | 20260505 23:18:39.593 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:39.593 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:39.593 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:18:39.593 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:39.593 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:39.593 | INFO | Uploaded 36GiB to GPU in 2.14066s (16.8172GiB/s) | 20260505 23:18:39.593 | INFO | Downloaded 34GiB to GPU in 2.02718s (16.7721GiB/s) | 20260505 23:18:44.726 | INFO | Analyzing chunk frames = [7096320,7096832) | 20260505 23:18:44.726 | INFO | MemInfo: { | 20260505 23:18:44.726 | INFO | "gpu" : " 17 GB", | 20260505 23:18:44.726 | INFO | "huge" : "", | 20260505 23:18:44.726 | INFO | "huge_shared" : " 27 GB", | 20260505 23:18:44.726 | INFO | "malloc" : " 14 GB", | 20260505 23:18:44.726 | INFO | "pinned" : " 29 GB" | 20260505 23:18:44.726 | INFO | } | 20260505 23:18:44.726 | INFO | Perf report for the last 5.13304 seconds: | 20260505 23:18:44.726 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:18:44.726 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:44.726 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:44.726 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:44.726 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260505 23:18:44.726 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:44.726 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:44.726 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 23:18:44.726 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:44.726 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:44.726 | INFO | Uploaded 36GiB to GPU in 2.10059s (17.138GiB/s) | 20260505 23:18:44.726 | INFO | Downloaded 34GiB to GPU in 2.04983s (16.5867GiB/s) | 20260505 23:18:49.870 | INFO | Analyzing chunk frames = [7096832,7097344) | 20260505 23:18:49.871 | INFO | MemInfo: { | 20260505 23:18:49.871 | INFO | "gpu" : " 17 GB", | 20260505 23:18:49.871 | INFO | "huge" : "", | 20260505 23:18:49.871 | INFO | "huge_shared" : " 27 GB", | 20260505 23:18:49.871 | INFO | "malloc" : " 14 GB", | 20260505 23:18:49.871 | INFO | "pinned" : " 29 GB" | 20260505 23:18:49.871 | INFO | } | 20260505 23:18:49.871 | INFO | Perf report for the last 5.14467 seconds: | 20260505 23:18:49.871 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:18:49.871 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:49.871 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:49.871 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:49.871 | INFO | ANALYSIS : 61.4%, 3.0, 96.0ms, 0 | 20260505 23:18:49.871 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:49.871 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:49.871 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:18:49.871 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:49.871 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:49.871 | INFO | Uploaded 36GiB to GPU in 2.07168s (17.3772GiB/s) | 20260505 23:18:49.871 | INFO | Downloaded 34GiB to GPU in 2.03618s (16.6979GiB/s) | 20260505 23:18:55.009 | INFO | Analyzing chunk frames = [7097344,7097856) | 20260505 23:18:55.009 | INFO | MemInfo: { | 20260505 23:18:55.009 | INFO | "gpu" : " 17 GB", | 20260505 23:18:55.009 | INFO | "huge" : "", | 20260505 23:18:55.009 | INFO | "huge_shared" : " 27 GB", | 20260505 23:18:55.009 | INFO | "malloc" : " 14 GB", | 20260505 23:18:55.009 | INFO | "pinned" : " 29 GB" | 20260505 23:18:55.009 | INFO | } | 20260505 23:18:55.009 | INFO | Perf report for the last 5.13859 seconds: | 20260505 23:18:55.009 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:18:55.009 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:55.009 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:55.009 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:55.009 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 23:18:55.009 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:55.009 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:55.009 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 23:18:55.009 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:55.009 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:18:55.010 | INFO | Uploaded 36GiB to GPU in 2.01879s (17.8325GiB/s) | 20260505 23:18:55.010 | INFO | Downloaded 34GiB to GPU in 2.03782s (16.6845GiB/s) | 20260505 23:19:00.154 | INFO | Analyzing chunk frames = [7097856,7098368) | 20260505 23:19:00.154 | INFO | MemInfo: { | 20260505 23:19:00.154 | INFO | "gpu" : " 17 GB", | 20260505 23:19:00.154 | INFO | "huge" : "", | 20260505 23:19:00.154 | INFO | "huge_shared" : " 27 GB", | 20260505 23:19:00.154 | INFO | "malloc" : " 14 GB", | 20260505 23:19:00.154 | INFO | "pinned" : " 29 GB" | 20260505 23:19:00.154 | INFO | } | 20260505 23:19:00.154 | INFO | Perf report for the last 5.14516 seconds: | 20260505 23:19:00.155 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:19:00.155 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:00.155 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:00.155 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:00.155 | INFO | ANALYSIS : 61.9%, 3.0, 96.8ms, 0 | 20260505 23:19:00.155 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:00.155 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:00.155 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 23:19:00.155 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:00.155 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:00.155 | INFO | Uploaded 36GiB to GPU in 2.11219s (17.0439GiB/s) | 20260505 23:19:00.155 | INFO | Downloaded 34GiB to GPU in 2.01074s (16.9092GiB/s) | 20260505 23:19:00.278 | INFO | Starting a round of Baz IO | 20260505 23:19:03.422 | INFO | Populating RTMetrics report | 20260505 23:19:03.422 | INFO | RTMetrics report populated | 20260505 23:19:03.422 | INFO | Opening tmp RTMetrics json file | 20260505 23:19:03.424 | INFO | Serializing RTMetrics to json | 20260505 23:19:03.425 | INFO | Serialization done, swapping tmp file now... | 20260505 23:19:03.426 | INFO | RTMetrics done | 20260505 23:19:03.506 | INFO | Finished a round of Baz IO: Wrote 1.897 GiB in 3.228 seconds | 20260505 23:19:03.506 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.44% / 0.00% / 78.56% | 20260505 23:19:03.506 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.109176% / 7.94729e-06% / 9.13938e-05% | 20260505 23:19:03.506 | INFO | Baz IO overall: | 20260505 23:19:03.506 | INFO | BytesWritten : 4063368994191 | 20260505 23:19:03.506 | INFO | NumEvents : 1914672835582 | 20260505 23:19:05.283 | INFO | Analyzing chunk frames = [7098368,7098880) | 20260505 23:19:05.283 | INFO | MemInfo: { | 20260505 23:19:05.283 | INFO | "gpu" : " 17 GB", | 20260505 23:19:05.283 | INFO | "huge" : "", | 20260505 23:19:05.283 | INFO | "huge_shared" : " 27 GB", | 20260505 23:19:05.283 | INFO | "malloc" : " 14 GB", | 20260505 23:19:05.283 | INFO | "pinned" : " 29 GB" | 20260505 23:19:05.283 | INFO | } | 20260505 23:19:05.283 | INFO | Perf report for the last 5.12871 seconds: | 20260505 23:19:05.283 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:19:05.283 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:05.283 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:05.283 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:05.283 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260505 23:19:05.283 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:05.283 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260505 23:19:05.283 | INFO | PRE_HQ : 60.1%, 1.7, 54.4ms, 0 | 20260505 23:19:05.283 | INFO | BAZWRITER : 62.2%, 3.6, 114.3ms, 0 | 20260505 23:19:05.283 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 23:19:05.283 | INFO | Uploaded 36GiB to GPU in 2.17142s (16.579GiB/s) | 20260505 23:19:05.283 | INFO | Downloaded 38GiB to GPU in 2.24212s (16.9483GiB/s) | 20260505 23:19:10.424 | INFO | Analyzing chunk frames = [7098880,7099392) | 20260505 23:19:10.424 | INFO | MemInfo: { | 20260505 23:19:10.424 | INFO | "gpu" : " 17 GB", | 20260505 23:19:10.424 | INFO | "huge" : "", | 20260505 23:19:10.424 | INFO | "huge_shared" : " 27 GB", | 20260505 23:19:10.424 | INFO | "malloc" : " 14 GB", | 20260505 23:19:10.424 | INFO | "pinned" : " 29 GB" | 20260505 23:19:10.424 | INFO | } | 20260505 23:19:10.424 | INFO | Perf report for the last 5.14098 seconds: | 20260505 23:19:10.424 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:19:10.424 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:10.424 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:10.424 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:10.424 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 23:19:10.424 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:10.424 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:10.424 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260505 23:19:10.424 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:10.424 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:10.424 | INFO | Uploaded 36GiB to GPU in 2.07432s (17.3551GiB/s) | 20260505 23:19:10.424 | INFO | Downloaded 34GiB to GPU in 2.01934s (16.8372GiB/s) | 20260505 23:19:15.566 | INFO | Analyzing chunk frames = [7099392,7099904) | 20260505 23:19:15.566 | INFO | MemInfo: { | 20260505 23:19:15.566 | INFO | "gpu" : " 17 GB", | 20260505 23:19:15.566 | INFO | "huge" : "", | 20260505 23:19:15.566 | INFO | "huge_shared" : " 27 GB", | 20260505 23:19:15.566 | INFO | "malloc" : " 14 GB", | 20260505 23:19:15.566 | INFO | "pinned" : " 29 GB" | 20260505 23:19:15.566 | INFO | } | 20260505 23:19:15.566 | INFO | Perf report for the last 5.14199 seconds: | 20260505 23:19:15.566 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:19:15.566 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:15.566 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:15.566 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:15.566 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260505 23:19:15.566 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:15.566 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:15.566 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 23:19:15.566 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:15.566 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:15.566 | INFO | Uploaded 36GiB to GPU in 2.10108s (17.1341GiB/s) | 20260505 23:19:15.567 | INFO | Downloaded 34GiB to GPU in 2.02985s (16.75GiB/s) | 20260505 23:19:20.704 | INFO | Analyzing chunk frames = [7099904,7100416) | 20260505 23:19:20.704 | INFO | MemInfo: { | 20260505 23:19:20.704 | INFO | "gpu" : " 17 GB", | 20260505 23:19:20.704 | INFO | "huge" : "", | 20260505 23:19:20.704 | INFO | "huge_shared" : " 27 GB", | 20260505 23:19:20.704 | INFO | "malloc" : " 14 GB", | 20260505 23:19:20.704 | INFO | "pinned" : " 29 GB" | 20260505 23:19:20.704 | INFO | } | 20260505 23:19:20.704 | INFO | Perf report for the last 5.13784 seconds: | 20260505 23:19:20.704 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:19:20.704 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:20.704 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:20.704 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:20.704 | INFO | ANALYSIS : 61.8%, 3.0, 95.9ms, 0 | 20260505 23:19:20.704 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:20.704 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:20.704 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 23:19:20.704 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:20.704 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:20.704 | INFO | Uploaded 36GiB to GPU in 2.15848s (16.6784GiB/s) | 20260505 23:19:20.704 | INFO | Downloaded 34GiB to GPU in 2.04433s (16.6314GiB/s) | 20260505 23:19:25.840 | INFO | Analyzing chunk frames = [7100416,7100928) | 20260505 23:19:25.840 | INFO | MemInfo: { | 20260505 23:19:25.840 | INFO | "gpu" : " 17 GB", | 20260505 23:19:25.840 | INFO | "huge" : "", | 20260505 23:19:25.840 | INFO | "huge_shared" : " 27 GB", | 20260505 23:19:25.840 | INFO | "malloc" : " 14 GB", | 20260505 23:19:25.840 | INFO | "pinned" : " 29 GB" | 20260505 23:19:25.840 | INFO | } | 20260505 23:19:25.840 | INFO | Perf report for the last 5.13562 seconds: | 20260505 23:19:25.840 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:19:25.840 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:25.840 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:25.840 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:25.840 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260505 23:19:25.840 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:25.840 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:25.840 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260505 23:19:25.840 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:25.840 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:25.840 | INFO | Uploaded 36GiB to GPU in 2.10774s (17.0799GiB/s) | 20260505 23:19:25.840 | INFO | Downloaded 34GiB to GPU in 2.04984s (16.5866GiB/s) | 20260505 23:19:30.977 | INFO | Analyzing chunk frames = [7100928,7101440) | 20260505 23:19:30.977 | INFO | MemInfo: { | 20260505 23:19:30.977 | INFO | "gpu" : " 17 GB", | 20260505 23:19:30.977 | INFO | "huge" : "", | 20260505 23:19:30.977 | INFO | "huge_shared" : " 27 GB", | 20260505 23:19:30.977 | INFO | "malloc" : " 14 GB", | 20260505 23:19:30.977 | INFO | "pinned" : " 29 GB" | 20260505 23:19:30.977 | INFO | } | 20260505 23:19:30.977 | INFO | Perf report for the last 5.13733 seconds: | 20260505 23:19:30.977 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:19:30.977 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:30.977 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:30.977 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:30.977 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260505 23:19:30.977 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:30.977 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:30.977 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:19:30.977 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:30.977 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:30.977 | INFO | Uploaded 36GiB to GPU in 2.08313s (17.2817GiB/s) | 20260505 23:19:30.977 | INFO | Downloaded 34GiB to GPU in 2.03386s (16.7169GiB/s) | 20260505 23:19:36.122 | INFO | Analyzing chunk frames = [7101440,7101952) | 20260505 23:19:36.122 | INFO | MemInfo: { | 20260505 23:19:36.122 | INFO | "gpu" : " 17 GB", | 20260505 23:19:36.122 | INFO | "huge" : "", | 20260505 23:19:36.122 | INFO | "huge_shared" : " 27 GB", | 20260505 23:19:36.122 | INFO | "malloc" : " 14 GB", | 20260505 23:19:36.122 | INFO | "pinned" : " 29 GB" | 20260505 23:19:36.122 | INFO | } | 20260505 23:19:36.123 | INFO | Perf report for the last 5.14552 seconds: | 20260505 23:19:36.123 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:19:36.123 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:36.123 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:36.123 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:36.123 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260505 23:19:36.123 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:36.123 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:36.123 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 23:19:36.123 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:36.123 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:36.123 | INFO | Uploaded 36GiB to GPU in 2.12667s (16.9279GiB/s) | 20260505 23:19:36.123 | INFO | Downloaded 34GiB to GPU in 2.10392s (16.1603GiB/s) | 20260505 23:19:41.268 | INFO | Analyzing chunk frames = [7101952,7102464) | 20260505 23:19:41.268 | INFO | MemInfo: { | 20260505 23:19:41.268 | INFO | "gpu" : " 17 GB", | 20260505 23:19:41.268 | INFO | "huge" : "", | 20260505 23:19:41.268 | INFO | "huge_shared" : " 27 GB", | 20260505 23:19:41.268 | INFO | "malloc" : " 14 GB", | 20260505 23:19:41.268 | INFO | "pinned" : " 29 GB" | 20260505 23:19:41.268 | INFO | } | 20260505 23:19:41.268 | INFO | Perf report for the last 5.14527 seconds: | 20260505 23:19:41.268 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:19:41.268 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:41.268 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:41.268 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:41.268 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260505 23:19:41.268 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:41.268 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:41.268 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:19:41.268 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:41.268 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:41.268 | INFO | Uploaded 36GiB to GPU in 2.11809s (16.9964GiB/s) | 20260505 23:19:41.268 | INFO | Downloaded 34GiB to GPU in 2.00734s (16.9378GiB/s) | 20260505 23:19:41.268 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 23:19:41.268 | INFO | Total mem usage: 15.3 GiB | 20260505 23:19:41.268 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 23:19:41.268 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.09%) | 20260505 23:19:41.268 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 23:19:41.268 | INFO | High Water Breakdown: | 20260505 23:19:41.268 | INFO | 60.9% : 8.55 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 23:19:41.268 | INFO | 39% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 23:19:41.268 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 23:19:41.268 | INFO | ------------------------------------------ | 20260505 23:19:41.268 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:19:41.268 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 23:19:41.268 | INFO | Total mem usage: 26.5 GiB | 20260505 23:19:41.268 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 23:19:41.268 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 23:19:41.269 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 23:19:41.269 | INFO | High Water Breakdown: | 20260505 23:19:41.269 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:19:41.269 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:19:41.269 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:19:41.269 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:19:41.269 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:19:41.269 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:19:41.269 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 23:19:41.269 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 23:19:41.269 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:19:41.269 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 23:19:41.269 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 23:19:41.269 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 23:19:41.269 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 23:19:41.269 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 23:19:41.269 | INFO | ------------------------------------------ | 20260505 23:19:41.269 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:19:41.269 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 23:19:41.269 | INFO | Total mem usage: 25.6 GiB | 20260505 23:19:41.269 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 23:19:41.269 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 23:19:41.269 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 23:19:41.269 | INFO | High Water Breakdown: | 20260505 23:19:41.269 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 23:19:41.269 | INFO | ------------------------------------------ | 20260505 23:19:41.269 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:19:41.269 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 23:19:41.269 | INFO | Total mem usage: 15.7 GiB | 20260505 23:19:41.269 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 23:19:41.269 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 23:19:41.269 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 23:19:41.269 | INFO | High Water Breakdown: | 20260505 23:19:41.269 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 23:19:41.269 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 23:19:41.269 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 23:19:41.269 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 23:19:41.269 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 23:19:41.269 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 23:19:41.269 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 23:19:41.269 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 23:19:41.269 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 23:19:41.269 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:19:41.269 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 23:19:41.269 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:19:41.269 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 23:19:41.269 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 23:19:41.269 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 23:19:41.269 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 23:19:41.269 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:19:41.269 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 23:19:41.269 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:19:41.269 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:19:41.269 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:19:41.269 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 23:19:41.269 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 23:19:41.269 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:19:41.269 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 23:19:41.269 | INFO | ------------------------------------------ | 20260505 23:19:41.269 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:19:41.269 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 23:19:41.269 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:19:41.269 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:19:41.269 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.40e+01, 1.33e+01 / 7.56e-01 | 20260505 23:19:41.269 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.04e-01, 1.03e-02 / 4.69e-03 | 20260505 23:19:41.269 | INFO | PerfReport: 11.82%, Baseline -- 3.71e+00 / 4.51e+00, 5.70e-01 / 2.12e-01 | 20260505 23:19:41.269 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.53e+00, 1.91e-01 / 9.50e-02 | 20260505 23:19:41.269 | INFO | PerfReport: 15.99%, DME -- 4.01e+01 / 4.41e+01, 3.71e+00 / 7.81e-01 | 20260505 23:19:41.269 | INFO | PerfReport: 57.30%, FrameLabeling -- 1.80e+01 / 1.93e+01, 1.50e+00 / 1.10e+00 | 20260505 23:19:41.269 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.23e+00 / 2.77e+00, 4.24e-01 / 2.37e-01 | 20260505 23:19:41.269 | INFO | PerfReport: 3.12%, Metrics -- 9.78e-01 / 1.26e+00, 2.61e-01 / 1.29e-01 | 20260505 23:19:41.269 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.39e-01 / 3.91e-01, 5.75e-02 / 2.90e-02 | 20260505 23:19:41.269 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 23:19:41.269 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:19:41.269 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:19:41.269 | INFO | PerfReport: Full Execution -- 9.84e+01 / -nan, 2.48e+01 / 0.00e+00 | 20260505 23:19:41.269 | INFO | PerfReport: 22.27%, Upload -- 2.19e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260505 23:19:41.269 | INFO | PerfReport: 21.81%, Download -- 2.15e+01 / 0.00e+00, 7.52e+00 / 0.00e+00 | 20260505 23:19:41.269 | INFO | PerfReport: 31.87%, Compute -- 3.14e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260505 23:19:41.269 | INFO | PerfReport: 2.14%, LockUpload -- 2.11e+00 / 0.00e+00, 7.98e+00 / 0.00e+00 | 20260505 23:19:41.269 | INFO | PerfReport: 1.31%, LockDownload -- 1.29e+00 / 0.00e+00, 2.90e+00 / 0.00e+00 | 20260505 23:19:41.269 | INFO | PerfReport: 20.59%, LockCompute -- 2.03e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260505 23:19:41.269 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.74e-02 / 0.00e+00 | 20260505 23:19:41.396 | INFO | Starting a round of Baz IO | 20260505 23:19:44.546 | INFO | Populating RTMetrics report | 20260505 23:19:44.546 | INFO | RTMetrics report populated | 20260505 23:19:44.546 | INFO | Opening tmp RTMetrics json file | 20260505 23:19:44.548 | INFO | Serializing RTMetrics to json | 20260505 23:19:44.550 | INFO | Serialization done, swapping tmp file now... | 20260505 23:19:44.552 | INFO | RTMetrics done | 20260505 23:19:44.630 | INFO | Finished a round of Baz IO: Wrote 1.896 GiB in 3.234 seconds | 20260505 23:19:44.630 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.44% / 0.00% / 78.56% | 20260505 23:19:44.630 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.108826% / 0% / 6.75519e-05% | 20260505 23:19:44.630 | INFO | Baz IO overall: | 20260505 23:19:44.630 | INFO | BytesWritten : 4065404392360 | 20260505 23:19:44.630 | INFO | NumEvents : 1915606446733 | 20260505 23:19:46.407 | INFO | Analyzing chunk frames = [7102464,7102976) | 20260505 23:19:46.407 | INFO | MemInfo: { | 20260505 23:19:46.407 | INFO | "gpu" : " 17 GB", | 20260505 23:19:46.407 | INFO | "huge" : "", | 20260505 23:19:46.407 | INFO | "huge_shared" : " 27 GB", | 20260505 23:19:46.407 | INFO | "malloc" : " 14 GB", | 20260505 23:19:46.407 | INFO | "pinned" : " 29 GB" | 20260505 23:19:46.407 | INFO | } | 20260505 23:19:46.407 | INFO | Perf report for the last 5.13919 seconds: | 20260505 23:19:46.407 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:19:46.407 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:46.407 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:46.407 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:46.407 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260505 23:19:46.407 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:46.407 | INFO | RT_METRICS : 43.4%, 1.0, 22.8ms, 0 | 20260505 23:19:46.407 | INFO | PRE_HQ : 59.9%, 1.7, 54.2ms, 0 | 20260505 23:19:46.407 | INFO | BAZWRITER : 62.2%, 3.6, 115.1ms, 0 | 20260505 23:19:46.407 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 23:19:46.407 | INFO | Uploaded 36GiB to GPU in 2.16794s (16.6056GiB/s) | 20260505 23:19:46.407 | INFO | Downloaded 38GiB to GPU in 2.23509s (17.0015GiB/s) | 20260505 23:19:51.538 | INFO | Analyzing chunk frames = [7102976,7103488) | 20260505 23:19:51.538 | INFO | MemInfo: { | 20260505 23:19:51.538 | INFO | "gpu" : " 17 GB", | 20260505 23:19:51.538 | INFO | "huge" : "", | 20260505 23:19:51.538 | INFO | "huge_shared" : " 27 GB", | 20260505 23:19:51.538 | INFO | "malloc" : " 14 GB", | 20260505 23:19:51.538 | INFO | "pinned" : " 29 GB" | 20260505 23:19:51.538 | INFO | } | 20260505 23:19:51.538 | INFO | Perf report for the last 5.13112 seconds: | 20260505 23:19:51.538 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:19:51.538 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:51.538 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:51.538 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:51.538 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260505 23:19:51.538 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:51.538 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:51.538 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:19:51.538 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:51.538 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:51.539 | INFO | Uploaded 36GiB to GPU in 2.08713s (17.2486GiB/s) | 20260505 23:19:51.539 | INFO | Downloaded 34GiB to GPU in 2.02904s (16.7567GiB/s) | 20260505 23:19:56.683 | INFO | Analyzing chunk frames = [7103488,7104000) | 20260505 23:19:56.683 | INFO | MemInfo: { | 20260505 23:19:56.683 | INFO | "gpu" : " 17 GB", | 20260505 23:19:56.683 | INFO | "huge" : "", | 20260505 23:19:56.683 | INFO | "huge_shared" : " 27 GB", | 20260505 23:19:56.683 | INFO | "malloc" : " 14 GB", | 20260505 23:19:56.683 | INFO | "pinned" : " 29 GB" | 20260505 23:19:56.683 | INFO | } | 20260505 23:19:56.683 | INFO | Perf report for the last 5.14458 seconds: | 20260505 23:19:56.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:19:56.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:56.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:56.683 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:56.683 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260505 23:19:56.683 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:56.683 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:56.683 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:19:56.683 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:56.683 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:19:56.683 | INFO | Uploaded 36GiB to GPU in 2.1418s (16.8083GiB/s) | 20260505 23:19:56.683 | INFO | Downloaded 34GiB to GPU in 2.02957s (16.7523GiB/s) | 20260505 23:20:01.818 | INFO | Analyzing chunk frames = [7104000,7104512) | 20260505 23:20:01.818 | INFO | MemInfo: { | 20260505 23:20:01.818 | INFO | "gpu" : " 17 GB", | 20260505 23:20:01.818 | INFO | "huge" : "", | 20260505 23:20:01.818 | INFO | "huge_shared" : " 27 GB", | 20260505 23:20:01.818 | INFO | "malloc" : " 14 GB", | 20260505 23:20:01.818 | INFO | "pinned" : " 29 GB" | 20260505 23:20:01.818 | INFO | } | 20260505 23:20:01.818 | INFO | Perf report for the last 5.13503 seconds: | 20260505 23:20:01.818 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:20:01.818 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:01.818 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:01.818 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:01.818 | INFO | ANALYSIS : 61.7%, 3.0, 95.8ms, 0 | 20260505 23:20:01.818 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:01.818 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:01.818 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:20:01.818 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:01.818 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:01.818 | INFO | Uploaded 36GiB to GPU in 2.14357s (16.7944GiB/s) | 20260505 23:20:01.818 | INFO | Downloaded 34GiB to GPU in 2.02194s (16.8156GiB/s) | 20260505 23:20:06.960 | INFO | Analyzing chunk frames = [7104512,7105024) | 20260505 23:20:06.960 | INFO | MemInfo: { | 20260505 23:20:06.960 | INFO | "gpu" : " 17 GB", | 20260505 23:20:06.960 | INFO | "huge" : "", | 20260505 23:20:06.960 | INFO | "huge_shared" : " 27 GB", | 20260505 23:20:06.960 | INFO | "malloc" : " 14 GB", | 20260505 23:20:06.960 | INFO | "pinned" : " 29 GB" | 20260505 23:20:06.960 | INFO | } | 20260505 23:20:06.960 | INFO | Perf report for the last 5.1425 seconds: | 20260505 23:20:06.960 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:20:06.960 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:06.960 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:06.960 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:06.960 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260505 23:20:06.960 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:06.960 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:06.960 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:20:06.960 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:06.960 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:06.961 | INFO | Uploaded 36GiB to GPU in 2.11239s (17.0423GiB/s) | 20260505 23:20:06.961 | INFO | Downloaded 34GiB to GPU in 2.05132s (16.5747GiB/s) | 20260505 23:20:12.099 | INFO | Analyzing chunk frames = [7105024,7105536) | 20260505 23:20:12.099 | INFO | MemInfo: { | 20260505 23:20:12.099 | INFO | "gpu" : " 17 GB", | 20260505 23:20:12.099 | INFO | "huge" : "", | 20260505 23:20:12.099 | INFO | "huge_shared" : " 27 GB", | 20260505 23:20:12.099 | INFO | "malloc" : " 14 GB", | 20260505 23:20:12.100 | INFO | "pinned" : " 29 GB" | 20260505 23:20:12.100 | INFO | } | 20260505 23:20:12.100 | INFO | Perf report for the last 5.13922 seconds: | 20260505 23:20:12.100 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:20:12.100 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:12.100 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:12.100 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:12.100 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260505 23:20:12.100 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:12.100 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:12.100 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260505 23:20:12.100 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:12.100 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:12.100 | INFO | Uploaded 36GiB to GPU in 2.0667s (17.4191GiB/s) | 20260505 23:20:12.100 | INFO | Downloaded 34GiB to GPU in 2.03437s (16.7128GiB/s) | 20260505 23:20:17.234 | INFO | Analyzing chunk frames = [7105536,7106048) | 20260505 23:20:17.234 | INFO | MemInfo: { | 20260505 23:20:17.234 | INFO | "gpu" : " 17 GB", | 20260505 23:20:17.234 | INFO | "huge" : "", | 20260505 23:20:17.234 | INFO | "huge_shared" : " 27 GB", | 20260505 23:20:17.234 | INFO | "malloc" : " 14 GB", | 20260505 23:20:17.234 | INFO | "pinned" : " 29 GB" | 20260505 23:20:17.234 | INFO | } | 20260505 23:20:17.234 | INFO | Perf report for the last 5.13454 seconds: | 20260505 23:20:17.234 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:20:17.234 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:17.234 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:17.234 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:17.234 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260505 23:20:17.234 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:17.234 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:17.234 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 23:20:17.234 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:17.234 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:17.234 | INFO | Uploaded 36GiB to GPU in 2.14428s (16.7889GiB/s) | 20260505 23:20:17.234 | INFO | Downloaded 34GiB to GPU in 2.09654s (16.2172GiB/s) | 20260505 23:20:22.379 | INFO | Analyzing chunk frames = [7106048,7106560) | 20260505 23:20:22.379 | INFO | MemInfo: { | 20260505 23:20:22.379 | INFO | "gpu" : " 17 GB", | 20260505 23:20:22.379 | INFO | "huge" : "", | 20260505 23:20:22.379 | INFO | "huge_shared" : " 27 GB", | 20260505 23:20:22.379 | INFO | "malloc" : " 14 GB", | 20260505 23:20:22.379 | INFO | "pinned" : " 29 GB" | 20260505 23:20:22.379 | INFO | } | 20260505 23:20:22.379 | INFO | Perf report for the last 5.14485 seconds: | 20260505 23:20:22.379 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:20:22.379 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:22.379 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:22.379 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:22.379 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260505 23:20:22.379 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:22.379 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:22.379 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:20:22.379 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:22.379 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:22.379 | INFO | Uploaded 36GiB to GPU in 2.12088s (16.9741GiB/s) | 20260505 23:20:22.379 | INFO | Downloaded 34GiB to GPU in 2.00945s (16.9201GiB/s) | 20260505 23:20:22.496 | INFO | Starting a round of Baz IO | 20260505 23:20:25.645 | INFO | Populating RTMetrics report | 20260505 23:20:25.646 | INFO | RTMetrics report populated | 20260505 23:20:25.646 | INFO | Opening tmp RTMetrics json file | 20260505 23:20:25.646 | INFO | Serializing RTMetrics to json | 20260505 23:20:25.647 | INFO | Serialization done, swapping tmp file now... | 20260505 23:20:25.648 | INFO | RTMetrics done | 20260505 23:20:25.722 | INFO | Finished a round of Baz IO: Wrote 1.896 GiB in 3.226 seconds | 20260505 23:20:25.722 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.44% / 0.00% / 78.56% | 20260505 23:20:25.722 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.109502% / 0% / 4.76837e-05% | 20260505 23:20:25.722 | INFO | Baz IO overall: | 20260505 23:20:25.722 | INFO | BytesWritten : 4067439931696 | 20260505 23:20:25.722 | INFO | NumEvents : 1916540159996 | 20260505 23:20:27.520 | INFO | Analyzing chunk frames = [7106560,7107072) | 20260505 23:20:27.520 | INFO | MemInfo: { | 20260505 23:20:27.520 | INFO | "gpu" : " 17 GB", | 20260505 23:20:27.520 | INFO | "huge" : "", | 20260505 23:20:27.520 | INFO | "huge_shared" : " 27 GB", | 20260505 23:20:27.520 | INFO | "malloc" : " 14 GB", | 20260505 23:20:27.520 | INFO | "pinned" : " 29 GB" | 20260505 23:20:27.520 | INFO | } | 20260505 23:20:27.521 | INFO | Perf report for the last 5.14154 seconds: | 20260505 23:20:27.521 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:20:27.521 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:27.521 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:27.521 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:27.521 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260505 23:20:27.521 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:27.521 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260505 23:20:27.521 | INFO | PRE_HQ : 59.8%, 1.7, 54.1ms, 0 | 20260505 23:20:27.521 | INFO | BAZWRITER : 61.9%, 3.6, 113.7ms, 0 | 20260505 23:20:27.521 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 23:20:27.521 | INFO | Uploaded 36GiB to GPU in 2.16762s (16.6081GiB/s) | 20260505 23:20:27.521 | INFO | Downloaded 38GiB to GPU in 2.23979s (16.9659GiB/s) | 20260505 23:20:32.654 | INFO | Analyzing chunk frames = [7107072,7107584) | 20260505 23:20:32.654 | INFO | MemInfo: { | 20260505 23:20:32.654 | INFO | "gpu" : " 17 GB", | 20260505 23:20:32.654 | INFO | "huge" : "", | 20260505 23:20:32.655 | INFO | "huge_shared" : " 27 GB", | 20260505 23:20:32.655 | INFO | "malloc" : " 14 GB", | 20260505 23:20:32.655 | INFO | "pinned" : " 29 GB" | 20260505 23:20:32.655 | INFO | } | 20260505 23:20:32.655 | INFO | Perf report for the last 5.13411 seconds: | 20260505 23:20:32.655 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:20:32.655 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:32.655 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:32.655 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:32.655 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260505 23:20:32.655 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:32.655 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:32.655 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:20:32.655 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:32.655 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:32.655 | INFO | Uploaded 36GiB to GPU in 2.10879s (17.0714GiB/s) | 20260505 23:20:32.655 | INFO | Downloaded 34GiB to GPU in 2.04012s (16.6657GiB/s) | 20260505 23:20:37.788 | INFO | Analyzing chunk frames = [7107584,7108096) | 20260505 23:20:37.788 | INFO | MemInfo: { | 20260505 23:20:37.788 | INFO | "gpu" : " 17 GB", | 20260505 23:20:37.788 | INFO | "huge" : "", | 20260505 23:20:37.788 | INFO | "huge_shared" : " 27 GB", | 20260505 23:20:37.788 | INFO | "malloc" : " 14 GB", | 20260505 23:20:37.789 | INFO | "pinned" : " 29 GB" | 20260505 23:20:37.789 | INFO | } | 20260505 23:20:37.789 | INFO | Perf report for the last 5.1339 seconds: | 20260505 23:20:37.789 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:20:37.789 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:37.789 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:37.789 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:37.789 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260505 23:20:37.789 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:37.789 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:37.789 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260505 23:20:37.789 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:37.789 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:37.789 | INFO | Uploaded 36GiB to GPU in 2.15569s (16.7GiB/s) | 20260505 23:20:37.789 | INFO | Downloaded 34GiB to GPU in 2.05235s (16.5664GiB/s) | 20260505 23:20:42.932 | INFO | Analyzing chunk frames = [7108096,7108608) | 20260505 23:20:42.932 | INFO | MemInfo: { | 20260505 23:20:42.932 | INFO | "gpu" : " 17 GB", | 20260505 23:20:42.932 | INFO | "huge" : "", | 20260505 23:20:42.932 | INFO | "huge_shared" : " 27 GB", | 20260505 23:20:42.932 | INFO | "malloc" : " 14 GB", | 20260505 23:20:42.932 | INFO | "pinned" : " 29 GB" | 20260505 23:20:42.932 | INFO | } | 20260505 23:20:42.932 | INFO | Perf report for the last 5.14369 seconds: | 20260505 23:20:42.932 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:20:42.932 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:42.932 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:42.932 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:42.932 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260505 23:20:42.932 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:42.932 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:42.932 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:20:42.932 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:42.932 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:42.933 | INFO | Uploaded 36GiB to GPU in 2.14178s (16.8085GiB/s) | 20260505 23:20:42.933 | INFO | Downloaded 34GiB to GPU in 2.04517s (16.6246GiB/s) | 20260505 23:20:48.077 | INFO | Analyzing chunk frames = [7108608,7109120) | 20260505 23:20:48.077 | INFO | MemInfo: { | 20260505 23:20:48.077 | INFO | "gpu" : " 17 GB", | 20260505 23:20:48.077 | INFO | "huge" : "", | 20260505 23:20:48.077 | INFO | "huge_shared" : " 27 GB", | 20260505 23:20:48.077 | INFO | "malloc" : " 14 GB", | 20260505 23:20:48.077 | INFO | "pinned" : " 29 GB" | 20260505 23:20:48.077 | INFO | } | 20260505 23:20:48.077 | INFO | Perf report for the last 5.14458 seconds: | 20260505 23:20:48.077 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:20:48.077 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:48.077 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:48.077 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:48.077 | INFO | ANALYSIS : 61.1%, 3.0, 95.5ms, 0 | 20260505 23:20:48.077 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:48.077 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:48.077 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 23:20:48.077 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:48.077 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:48.077 | INFO | Uploaded 36GiB to GPU in 2.10523s (17.1003GiB/s) | 20260505 23:20:48.077 | INFO | Downloaded 34GiB to GPU in 2.04678s (16.6114GiB/s) | 20260505 23:20:53.214 | INFO | Analyzing chunk frames = [7109120,7109632) | 20260505 23:20:53.214 | INFO | MemInfo: { | 20260505 23:20:53.214 | INFO | "gpu" : " 17 GB", | 20260505 23:20:53.214 | INFO | "huge" : "", | 20260505 23:20:53.214 | INFO | "huge_shared" : " 27 GB", | 20260505 23:20:53.214 | INFO | "malloc" : " 14 GB", | 20260505 23:20:53.214 | INFO | "pinned" : " 29 GB" | 20260505 23:20:53.214 | INFO | } | 20260505 23:20:53.214 | INFO | Perf report for the last 5.13713 seconds: | 20260505 23:20:53.214 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:20:53.214 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:53.214 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:53.214 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:53.214 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260505 23:20:53.214 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:53.214 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:53.214 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:20:53.214 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:53.214 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:53.214 | INFO | Uploaded 36GiB to GPU in 2.08054s (17.3032GiB/s) | 20260505 23:20:53.214 | INFO | Downloaded 34GiB to GPU in 2.03734s (16.6884GiB/s) | 20260505 23:20:58.349 | INFO | Analyzing chunk frames = [7109632,7110144) | 20260505 23:20:58.349 | INFO | MemInfo: { | 20260505 23:20:58.349 | INFO | "gpu" : " 17 GB", | 20260505 23:20:58.349 | INFO | "huge" : "", | 20260505 23:20:58.349 | INFO | "huge_shared" : " 27 GB", | 20260505 23:20:58.349 | INFO | "malloc" : " 14 GB", | 20260505 23:20:58.349 | INFO | "pinned" : " 29 GB" | 20260505 23:20:58.349 | INFO | } | 20260505 23:20:58.349 | INFO | Perf report for the last 5.1347 seconds: | 20260505 23:20:58.349 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:20:58.349 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:58.349 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:58.349 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:58.349 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260505 23:20:58.349 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:58.349 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:58.349 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 23:20:58.349 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:58.349 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:20:58.349 | INFO | Uploaded 36GiB to GPU in 2.1172s (17.0036GiB/s) | 20260505 23:20:58.349 | INFO | Downloaded 34GiB to GPU in 2.07025s (16.4232GiB/s) | 20260505 23:21:03.490 | INFO | Analyzing chunk frames = [7110144,7110656) | 20260505 23:21:03.490 | INFO | MemInfo: { | 20260505 23:21:03.490 | INFO | "gpu" : " 17 GB", | 20260505 23:21:03.490 | INFO | "huge" : "", | 20260505 23:21:03.490 | INFO | "huge_shared" : " 27 GB", | 20260505 23:21:03.490 | INFO | "malloc" : " 14 GB", | 20260505 23:21:03.490 | INFO | "pinned" : " 29 GB" | 20260505 23:21:03.490 | INFO | } | 20260505 23:21:03.490 | INFO | Perf report for the last 5.1417 seconds: | 20260505 23:21:03.490 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:21:03.490 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:03.490 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:03.490 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:03.490 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260505 23:21:03.491 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:03.491 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:03.491 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:21:03.491 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:03.491 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:03.491 | INFO | Uploaded 36GiB to GPU in 2.12252s (16.9609GiB/s) | 20260505 23:21:03.491 | INFO | Downloaded 34GiB to GPU in 2.01615s (16.8638GiB/s) | 20260505 23:21:03.491 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 23:21:03.491 | INFO | Total mem usage: 15.3 GiB | 20260505 23:21:03.491 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 23:21:03.491 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.1%) | 20260505 23:21:03.491 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 23:21:03.491 | INFO | High Water Breakdown: | 20260505 23:21:03.491 | INFO | 60.9% : 8.55 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 23:21:03.491 | INFO | 39% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 23:21:03.491 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 23:21:03.491 | INFO | ------------------------------------------ | 20260505 23:21:03.491 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:21:03.491 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 23:21:03.491 | INFO | Total mem usage: 26.5 GiB | 20260505 23:21:03.491 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 23:21:03.491 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 23:21:03.491 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 23:21:03.491 | INFO | High Water Breakdown: | 20260505 23:21:03.491 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:21:03.491 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:21:03.491 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:21:03.491 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:21:03.491 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:21:03.491 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:21:03.491 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 23:21:03.491 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 23:21:03.491 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:21:03.491 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 23:21:03.491 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 23:21:03.491 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 23:21:03.491 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 23:21:03.491 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 23:21:03.491 | INFO | ------------------------------------------ | 20260505 23:21:03.491 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:21:03.491 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 23:21:03.491 | INFO | Total mem usage: 25.6 GiB | 20260505 23:21:03.491 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 23:21:03.491 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 23:21:03.491 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 23:21:03.491 | INFO | High Water Breakdown: | 20260505 23:21:03.491 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 23:21:03.491 | INFO | ------------------------------------------ | 20260505 23:21:03.491 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:21:03.491 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 23:21:03.491 | INFO | Total mem usage: 15.7 GiB | 20260505 23:21:03.491 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 23:21:03.491 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 23:21:03.491 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 23:21:03.491 | INFO | High Water Breakdown: | 20260505 23:21:03.491 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 23:21:03.492 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 23:21:03.492 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 23:21:03.492 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 23:21:03.492 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 23:21:03.492 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 23:21:03.492 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 23:21:03.492 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 23:21:03.492 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 23:21:03.492 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:21:03.492 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 23:21:03.492 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:21:03.492 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 23:21:03.492 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 23:21:03.492 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 23:21:03.492 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 23:21:03.492 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:21:03.492 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 23:21:03.492 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:21:03.492 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:21:03.492 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:21:03.492 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 23:21:03.492 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 23:21:03.492 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:21:03.492 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 23:21:03.492 | INFO | ------------------------------------------ | 20260505 23:21:03.492 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:21:03.492 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 23:21:03.492 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:21:03.492 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:21:03.492 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.40e+01, 1.36e+01 / 8.16e-01 | 20260505 23:21:03.492 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.04e-01, 1.09e-02 / 4.83e-03 | 20260505 23:21:03.492 | INFO | PerfReport: 11.80%, Baseline -- 3.70e+00 / 4.51e+00, 5.59e-01 / 1.98e-01 | 20260505 23:21:03.492 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.53e+00, 1.88e-01 / 1.06e-01 | 20260505 23:21:03.492 | INFO | PerfReport: 15.98%, DME -- 4.01e+01 / 4.39e+01, 3.68e+00 / 8.91e-01 | 20260505 23:21:03.492 | INFO | PerfReport: 57.34%, FrameLabeling -- 1.80e+01 / 1.96e+01, 1.51e+00 / 1.19e+00 | 20260505 23:21:03.492 | INFO | PerfReport: 7.13%, PulseAccumulating -- 2.23e+00 / 2.74e+00, 4.27e-01 / 2.58e-01 | 20260505 23:21:03.492 | INFO | PerfReport: 3.12%, Metrics -- 9.77e-01 / 1.26e+00, 2.65e-01 / 9.05e-02 | 20260505 23:21:03.492 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.39e-01 / 3.98e-01, 5.84e-02 / 3.00e-02 | 20260505 23:21:03.492 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 23:21:03.492 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:21:03.492 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:21:03.492 | INFO | PerfReport: Full Execution -- 9.85e+01 / -nan, 2.49e+01 / 0.00e+00 | 20260505 23:21:03.492 | INFO | PerfReport: 22.45%, Upload -- 2.21e+01 / 0.00e+00, 7.61e+00 / 0.00e+00 | 20260505 23:21:03.492 | INFO | PerfReport: 21.84%, Download -- 2.15e+01 / 0.00e+00, 7.61e+00 / 0.00e+00 | 20260505 23:21:03.492 | INFO | PerfReport: 31.81%, Compute -- 3.13e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 23:21:03.492 | INFO | PerfReport: 2.17%, LockUpload -- 2.14e+00 / 0.00e+00, 7.89e+00 / 0.00e+00 | 20260505 23:21:03.492 | INFO | PerfReport: 1.27%, LockDownload -- 1.25e+00 / 0.00e+00, 2.73e+00 / 0.00e+00 | 20260505 23:21:03.492 | INFO | PerfReport: 20.45%, LockCompute -- 2.01e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260505 23:21:03.492 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.14e-02 / 0.00e+00 | 20260505 23:21:03.613 | INFO | Starting a round of Baz IO | 20260505 23:21:06.761 | INFO | Populating RTMetrics report | 20260505 23:21:06.762 | INFO | RTMetrics report populated | 20260505 23:21:06.762 | INFO | Opening tmp RTMetrics json file | 20260505 23:21:06.766 | INFO | Serializing RTMetrics to json | 20260505 23:21:06.767 | INFO | Serialization done, swapping tmp file now... | 20260505 23:21:06.768 | INFO | RTMetrics done | 20260505 23:21:06.844 | INFO | Finished a round of Baz IO: Wrote 1.897 GiB in 3.231 seconds | 20260505 23:21:06.844 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.44% / 0.00% / 78.56% | 20260505 23:21:06.844 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.108639% / 3.97364e-06% / 8.34465e-05% | 20260505 23:21:06.844 | INFO | Baz IO overall: | 20260505 23:21:06.844 | INFO | BytesWritten : 4069476647932 | 20260505 23:21:06.844 | INFO | NumEvents : 1917474466154 | 20260505 23:21:08.633 | INFO | Analyzing chunk frames = [7110656,7111168) | 20260505 23:21:08.633 | INFO | MemInfo: { | 20260505 23:21:08.633 | INFO | "gpu" : " 17 GB", | 20260505 23:21:08.633 | INFO | "huge" : "", | 20260505 23:21:08.633 | INFO | "huge_shared" : " 27 GB", | 20260505 23:21:08.633 | INFO | "malloc" : " 14 GB", | 20260505 23:21:08.633 | INFO | "pinned" : " 29 GB" | 20260505 23:21:08.633 | INFO | } | 20260505 23:21:08.633 | INFO | Perf report for the last 5.14248 seconds: | 20260505 23:21:08.633 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:21:08.633 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:08.633 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:08.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:08.633 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260505 23:21:08.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:08.633 | INFO | RT_METRICS : 43.4%, 1.0, 22.8ms, 0 | 20260505 23:21:08.633 | INFO | PRE_HQ : 59.8%, 1.7, 54.0ms, 0 | 20260505 23:21:08.633 | INFO | BAZWRITER : 62.0%, 3.6, 113.6ms, 0 | 20260505 23:21:08.633 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 23:21:08.633 | INFO | Uploaded 36GiB to GPU in 2.17031s (16.5875GiB/s) | 20260505 23:21:08.633 | INFO | Downloaded 38GiB to GPU in 2.24009s (16.9636GiB/s) | 20260505 23:21:13.773 | INFO | Analyzing chunk frames = [7111168,7111680) | 20260505 23:21:13.773 | INFO | MemInfo: { | 20260505 23:21:13.773 | INFO | "gpu" : " 17 GB", | 20260505 23:21:13.773 | INFO | "huge" : "", | 20260505 23:21:13.773 | INFO | "huge_shared" : " 27 GB", | 20260505 23:21:13.773 | INFO | "malloc" : " 14 GB", | 20260505 23:21:13.773 | INFO | "pinned" : " 29 GB" | 20260505 23:21:13.773 | INFO | } | 20260505 23:21:13.773 | INFO | Perf report for the last 5.13996 seconds: | 20260505 23:21:13.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:21:13.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:13.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:13.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:13.773 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 23:21:13.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:13.773 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:13.773 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260505 23:21:13.773 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:13.773 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:13.773 | INFO | Uploaded 36GiB to GPU in 2.11234s (17.0427GiB/s) | 20260505 23:21:13.773 | INFO | Downloaded 34GiB to GPU in 2.03888s (16.6758GiB/s) | 20260505 23:21:18.912 | INFO | Analyzing chunk frames = [7111680,7112192) | 20260505 23:21:18.912 | INFO | MemInfo: { | 20260505 23:21:18.912 | INFO | "gpu" : " 17 GB", | 20260505 23:21:18.912 | INFO | "huge" : "", | 20260505 23:21:18.912 | INFO | "huge_shared" : " 27 GB", | 20260505 23:21:18.912 | INFO | "malloc" : " 14 GB", | 20260505 23:21:18.912 | INFO | "pinned" : " 29 GB" | 20260505 23:21:18.912 | INFO | } | 20260505 23:21:18.913 | INFO | Perf report for the last 5.13963 seconds: | 20260505 23:21:18.913 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:21:18.913 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:18.913 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:18.913 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:18.913 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260505 23:21:18.913 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:18.913 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:18.913 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260505 23:21:18.913 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:18.913 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:18.913 | INFO | Uploaded 36GiB to GPU in 2.07571s (17.3435GiB/s) | 20260505 23:21:18.913 | INFO | Downloaded 34GiB to GPU in 2.01945s (16.8363GiB/s) | 20260505 23:21:24.049 | INFO | Analyzing chunk frames = [7112192,7112704) | 20260505 23:21:24.049 | INFO | MemInfo: { | 20260505 23:21:24.049 | INFO | "gpu" : " 17 GB", | 20260505 23:21:24.049 | INFO | "huge" : "", | 20260505 23:21:24.049 | INFO | "huge_shared" : " 27 GB", | 20260505 23:21:24.049 | INFO | "malloc" : " 14 GB", | 20260505 23:21:24.049 | INFO | "pinned" : " 29 GB" | 20260505 23:21:24.049 | INFO | } | 20260505 23:21:24.049 | INFO | Perf report for the last 5.13643 seconds: | 20260505 23:21:24.049 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:21:24.049 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:24.049 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:24.049 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:24.049 | INFO | ANALYSIS : 61.7%, 3.0, 95.8ms, 0 | 20260505 23:21:24.049 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:24.049 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:24.049 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:21:24.049 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:24.049 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:24.049 | INFO | Uploaded 36GiB to GPU in 2.12815s (16.9161GiB/s) | 20260505 23:21:24.049 | INFO | Downloaded 34GiB to GPU in 2.00348s (16.9705GiB/s) | 20260505 23:21:29.188 | INFO | Analyzing chunk frames = [7112704,7113216) | 20260505 23:21:29.188 | INFO | MemInfo: { | 20260505 23:21:29.188 | INFO | "gpu" : " 17 GB", | 20260505 23:21:29.188 | INFO | "huge" : "", | 20260505 23:21:29.188 | INFO | "huge_shared" : " 27 GB", | 20260505 23:21:29.188 | INFO | "malloc" : " 14 GB", | 20260505 23:21:29.188 | INFO | "pinned" : " 29 GB" | 20260505 23:21:29.188 | INFO | } | 20260505 23:21:29.188 | INFO | Perf report for the last 5.13929 seconds: | 20260505 23:21:29.188 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:21:29.188 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:29.188 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:29.188 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:29.188 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260505 23:21:29.188 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:29.188 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:29.188 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:21:29.188 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:29.188 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:29.189 | INFO | Uploaded 36GiB to GPU in 2.10393s (17.1108GiB/s) | 20260505 23:21:29.189 | INFO | Downloaded 34GiB to GPU in 2.04422s (16.6323GiB/s) | 20260505 23:21:34.328 | INFO | Analyzing chunk frames = [7113216,7113728) | 20260505 23:21:34.328 | INFO | MemInfo: { | 20260505 23:21:34.328 | INFO | "gpu" : " 17 GB", | 20260505 23:21:34.328 | INFO | "huge" : "", | 20260505 23:21:34.328 | INFO | "huge_shared" : " 27 GB", | 20260505 23:21:34.328 | INFO | "malloc" : " 14 GB", | 20260505 23:21:34.328 | INFO | "pinned" : " 29 GB" | 20260505 23:21:34.328 | INFO | } | 20260505 23:21:34.328 | INFO | Perf report for the last 5.13968 seconds: | 20260505 23:21:34.328 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:21:34.328 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:34.328 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:34.328 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:34.328 | INFO | ANALYSIS : 61.4%, 3.0, 95.9ms, 0 | 20260505 23:21:34.328 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:34.328 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:34.328 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:21:34.328 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:34.328 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:34.328 | INFO | Uploaded 36GiB to GPU in 2.07212s (17.3735GiB/s) | 20260505 23:21:34.328 | INFO | Downloaded 34GiB to GPU in 2.03466s (16.7104GiB/s) | 20260505 23:21:39.475 | INFO | Analyzing chunk frames = [7113728,7114240) | 20260505 23:21:39.475 | INFO | MemInfo: { | 20260505 23:21:39.475 | INFO | "gpu" : " 17 GB", | 20260505 23:21:39.475 | INFO | "huge" : "", | 20260505 23:21:39.475 | INFO | "huge_shared" : " 27 GB", | 20260505 23:21:39.475 | INFO | "malloc" : " 14 GB", | 20260505 23:21:39.475 | INFO | "pinned" : " 29 GB" | 20260505 23:21:39.475 | INFO | } | 20260505 23:21:39.475 | INFO | Perf report for the last 5.14686 seconds: | 20260505 23:21:39.475 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:21:39.475 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:39.475 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:39.475 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:39.475 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260505 23:21:39.475 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:39.475 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:39.475 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260505 23:21:39.475 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:39.475 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:39.475 | INFO | Uploaded 36GiB to GPU in 2.05486s (17.5195GiB/s) | 20260505 23:21:39.475 | INFO | Downloaded 34GiB to GPU in 2.059s (16.5129GiB/s) | 20260505 23:21:44.604 | INFO | Analyzing chunk frames = [7114240,7114752) | 20260505 23:21:44.604 | INFO | MemInfo: { | 20260505 23:21:44.604 | INFO | "gpu" : " 17 GB", | 20260505 23:21:44.605 | INFO | "huge" : "", | 20260505 23:21:44.605 | INFO | "huge_shared" : " 27 GB", | 20260505 23:21:44.605 | INFO | "malloc" : " 14 GB", | 20260505 23:21:44.605 | INFO | "pinned" : " 29 GB" | 20260505 23:21:44.605 | INFO | } | 20260505 23:21:44.605 | INFO | Perf report for the last 5.12979 seconds: | 20260505 23:21:44.605 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:21:44.605 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:44.605 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:44.605 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:44.605 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260505 23:21:44.605 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:44.605 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:44.605 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:21:44.605 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:44.605 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:44.605 | INFO | Uploaded 36GiB to GPU in 2.14928s (16.7498GiB/s) | 20260505 23:21:44.605 | INFO | Downloaded 34GiB to GPU in 2.02542s (16.7866GiB/s) | 20260505 23:21:44.738 | INFO | Starting a round of Baz IO | 20260505 23:21:47.888 | INFO | Populating RTMetrics report | 20260505 23:21:47.889 | INFO | RTMetrics report populated | 20260505 23:21:47.889 | INFO | Opening tmp RTMetrics json file | 20260505 23:21:47.889 | INFO | Serializing RTMetrics to json | 20260505 23:21:47.890 | INFO | Serialization done, swapping tmp file now... | 20260505 23:21:47.891 | INFO | RTMetrics done | 20260505 23:21:47.970 | INFO | Finished a round of Baz IO: Wrote 1.897 GiB in 3.231 seconds | 20260505 23:21:47.970 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.43% / 0.00% / 78.56% | 20260505 23:21:47.970 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.108711% / 3.97364e-06% / 7.15256e-05% | 20260505 23:21:47.970 | INFO | Baz IO overall: | 20260505 23:21:47.970 | INFO | BytesWritten : 4071513448375 | 20260505 23:21:47.970 | INFO | NumEvents : 1918409021421 | 20260505 23:21:49.740 | INFO | Analyzing chunk frames = [7114752,7115264) | 20260505 23:21:49.740 | INFO | MemInfo: { | 20260505 23:21:49.740 | INFO | "gpu" : " 17 GB", | 20260505 23:21:49.740 | INFO | "huge" : "", | 20260505 23:21:49.740 | INFO | "huge_shared" : " 27 GB", | 20260505 23:21:49.740 | INFO | "malloc" : " 14 GB", | 20260505 23:21:49.740 | INFO | "pinned" : " 29 GB" | 20260505 23:21:49.740 | INFO | } | 20260505 23:21:49.740 | INFO | Perf report for the last 5.13545 seconds: | 20260505 23:21:49.740 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:21:49.740 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:49.740 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:49.740 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:49.740 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260505 23:21:49.740 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:49.740 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260505 23:21:49.740 | INFO | PRE_HQ : 59.8%, 1.7, 54.2ms, 0 | 20260505 23:21:49.740 | INFO | BAZWRITER : 62.4%, 3.5, 113.7ms, 0 | 20260505 23:21:49.740 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 23:21:49.740 | INFO | Uploaded 36GiB to GPU in 2.16906s (16.597GiB/s) | 20260505 23:21:49.740 | INFO | Downloaded 38GiB to GPU in 2.24182s (16.9505GiB/s) | 20260505 23:21:54.884 | INFO | Analyzing chunk frames = [7115264,7115776) | 20260505 23:21:54.884 | INFO | MemInfo: { | 20260505 23:21:54.884 | INFO | "gpu" : " 17 GB", | 20260505 23:21:54.884 | INFO | "huge" : "", | 20260505 23:21:54.884 | INFO | "huge_shared" : " 27 GB", | 20260505 23:21:54.884 | INFO | "malloc" : " 14 GB", | 20260505 23:21:54.884 | INFO | "pinned" : " 29 GB" | 20260505 23:21:54.884 | INFO | } | 20260505 23:21:54.884 | INFO | Perf report for the last 5.14398 seconds: | 20260505 23:21:54.884 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:21:54.884 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:54.884 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:54.884 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:54.884 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 23:21:54.884 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:54.884 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:54.884 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260505 23:21:54.884 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:54.884 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:21:54.884 | INFO | Uploaded 36GiB to GPU in 2.09036s (17.2219GiB/s) | 20260505 23:21:54.884 | INFO | Downloaded 34GiB to GPU in 2.03257s (16.7276GiB/s) | 20260505 23:22:00.026 | INFO | Analyzing chunk frames = [7115776,7116288) | 20260505 23:22:00.026 | INFO | MemInfo: { | 20260505 23:22:00.026 | INFO | "gpu" : " 17 GB", | 20260505 23:22:00.026 | INFO | "huge" : "", | 20260505 23:22:00.026 | INFO | "huge_shared" : " 27 GB", | 20260505 23:22:00.026 | INFO | "malloc" : " 14 GB", | 20260505 23:22:00.026 | INFO | "pinned" : " 29 GB" | 20260505 23:22:00.026 | INFO | } | 20260505 23:22:00.026 | INFO | Perf report for the last 5.14197 seconds: | 20260505 23:22:00.026 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:22:00.026 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:00.026 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:00.026 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:00.026 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260505 23:22:00.026 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:00.026 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:00.026 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:22:00.026 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:00.026 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:00.026 | INFO | Uploaded 36GiB to GPU in 2.0832s (17.2811GiB/s) | 20260505 23:22:00.026 | INFO | Downloaded 34GiB to GPU in 2.02378s (16.8002GiB/s) | 20260505 23:22:05.162 | INFO | Analyzing chunk frames = [7116288,7116800) | 20260505 23:22:05.162 | INFO | MemInfo: { | 20260505 23:22:05.162 | INFO | "gpu" : " 17 GB", | 20260505 23:22:05.162 | INFO | "huge" : "", | 20260505 23:22:05.162 | INFO | "huge_shared" : " 27 GB", | 20260505 23:22:05.162 | INFO | "malloc" : " 14 GB", | 20260505 23:22:05.162 | INFO | "pinned" : " 29 GB" | 20260505 23:22:05.162 | INFO | } | 20260505 23:22:05.163 | INFO | Perf report for the last 5.13641 seconds: | 20260505 23:22:05.163 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:22:05.163 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:05.163 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:05.163 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:05.163 | INFO | ANALYSIS : 61.8%, 3.0, 95.9ms, 0 | 20260505 23:22:05.163 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:05.163 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:05.163 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:22:05.163 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:05.163 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:05.163 | INFO | Uploaded 36GiB to GPU in 2.1422s (16.8051GiB/s) | 20260505 23:22:05.163 | INFO | Downloaded 34GiB to GPU in 2.02661s (16.7768GiB/s) | 20260505 23:22:10.296 | INFO | Analyzing chunk frames = [7116800,7117312) | 20260505 23:22:10.296 | INFO | MemInfo: { | 20260505 23:22:10.296 | INFO | "gpu" : " 17 GB", | 20260505 23:22:10.296 | INFO | "huge" : "", | 20260505 23:22:10.296 | INFO | "huge_shared" : " 27 GB", | 20260505 23:22:10.296 | INFO | "malloc" : " 14 GB", | 20260505 23:22:10.296 | INFO | "pinned" : " 29 GB" | 20260505 23:22:10.296 | INFO | } | 20260505 23:22:10.296 | INFO | Perf report for the last 5.13369 seconds: | 20260505 23:22:10.296 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:22:10.296 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:10.296 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:10.296 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:10.296 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260505 23:22:10.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:10.296 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:10.296 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:22:10.296 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:10.296 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:10.296 | INFO | Uploaded 36GiB to GPU in 2.10122s (17.1329GiB/s) | 20260505 23:22:10.296 | INFO | Downloaded 34GiB to GPU in 2.05134s (16.5746GiB/s) | 20260505 23:22:15.440 | INFO | Analyzing chunk frames = [7117312,7117824) | 20260505 23:22:15.440 | INFO | MemInfo: { | 20260505 23:22:15.440 | INFO | "gpu" : " 17 GB", | 20260505 23:22:15.440 | INFO | "huge" : "", | 20260505 23:22:15.440 | INFO | "huge_shared" : " 27 GB", | 20260505 23:22:15.440 | INFO | "malloc" : " 14 GB", | 20260505 23:22:15.440 | INFO | "pinned" : " 29 GB" | 20260505 23:22:15.440 | INFO | } | 20260505 23:22:15.440 | INFO | Perf report for the last 5.14359 seconds: | 20260505 23:22:15.440 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:22:15.440 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:15.440 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:15.440 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:15.440 | INFO | ANALYSIS : 61.6%, 3.0, 96.2ms, 0 | 20260505 23:22:15.440 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:15.440 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:15.440 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:22:15.440 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:15.440 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:15.440 | INFO | Uploaded 36GiB to GPU in 2.08248s (17.2871GiB/s) | 20260505 23:22:15.440 | INFO | Downloaded 34GiB to GPU in 2.03508s (16.707GiB/s) | 20260505 23:22:20.586 | INFO | Analyzing chunk frames = [7117824,7118336) | 20260505 23:22:20.586 | INFO | MemInfo: { | 20260505 23:22:20.586 | INFO | "gpu" : " 17 GB", | 20260505 23:22:20.586 | INFO | "huge" : "", | 20260505 23:22:20.586 | INFO | "huge_shared" : " 27 GB", | 20260505 23:22:20.586 | INFO | "malloc" : " 14 GB", | 20260505 23:22:20.586 | INFO | "pinned" : " 29 GB" | 20260505 23:22:20.586 | INFO | } | 20260505 23:22:20.586 | INFO | Perf report for the last 5.1461 seconds: | 20260505 23:22:20.586 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:22:20.586 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:20.586 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:20.586 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:20.586 | INFO | ANALYSIS : 62.5%, 3.0, 97.7ms, 0 | 20260505 23:22:20.586 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:20.586 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:20.586 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 23:22:20.586 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:20.586 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:20.586 | INFO | Uploaded 36GiB to GPU in 2.12163s (16.9681GiB/s) | 20260505 23:22:20.586 | INFO | Downloaded 34GiB to GPU in 2.10072s (16.1849GiB/s) | 20260505 23:22:25.726 | INFO | Analyzing chunk frames = [7118336,7118848) | 20260505 23:22:25.726 | INFO | MemInfo: { | 20260505 23:22:25.726 | INFO | "gpu" : " 17 GB", | 20260505 23:22:25.726 | INFO | "huge" : "", | 20260505 23:22:25.726 | INFO | "huge_shared" : " 27 GB", | 20260505 23:22:25.726 | INFO | "malloc" : " 14 GB", | 20260505 23:22:25.726 | INFO | "pinned" : " 29 GB" | 20260505 23:22:25.726 | INFO | } | 20260505 23:22:25.726 | INFO | Perf report for the last 5.13977 seconds: | 20260505 23:22:25.726 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:22:25.726 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:25.726 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:25.726 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:25.726 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 23:22:25.726 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:25.726 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:25.726 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:22:25.726 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:25.726 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:25.726 | INFO | Uploaded 36GiB to GPU in 2.12639s (16.9301GiB/s) | 20260505 23:22:25.726 | INFO | Downloaded 34GiB to GPU in 2.01647s (16.8611GiB/s) | 20260505 23:22:25.726 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 23:22:25.726 | INFO | Total mem usage: 15.3 GiB | 20260505 23:22:25.726 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 23:22:25.726 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -9.07%) | 20260505 23:22:25.726 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 23:22:25.726 | INFO | High Water Breakdown: | 20260505 23:22:25.726 | INFO | 60.9% : 8.55 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 23:22:25.726 | INFO | 39% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 23:22:25.726 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 23:22:25.726 | INFO | ------------------------------------------ | 20260505 23:22:25.726 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:22:25.726 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 23:22:25.726 | INFO | Total mem usage: 26.5 GiB | 20260505 23:22:25.726 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 23:22:25.726 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 23:22:25.726 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 23:22:25.726 | INFO | High Water Breakdown: | 20260505 23:22:25.726 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:22:25.726 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:22:25.726 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:22:25.726 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:22:25.726 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:22:25.726 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:22:25.726 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 23:22:25.726 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 23:22:25.726 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:22:25.726 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 23:22:25.726 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 23:22:25.726 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 23:22:25.726 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 23:22:25.726 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 23:22:25.726 | INFO | ------------------------------------------ | 20260505 23:22:25.726 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:22:25.726 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 23:22:25.726 | INFO | Total mem usage: 25.6 GiB | 20260505 23:22:25.726 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 23:22:25.726 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 23:22:25.726 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 23:22:25.726 | INFO | High Water Breakdown: | 20260505 23:22:25.726 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 23:22:25.726 | INFO | ------------------------------------------ | 20260505 23:22:25.726 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:22:25.726 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 23:22:25.726 | INFO | Total mem usage: 15.7 GiB | 20260505 23:22:25.726 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 23:22:25.726 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 23:22:25.726 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 23:22:25.726 | INFO | High Water Breakdown: | 20260505 23:22:25.727 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 23:22:25.727 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 23:22:25.727 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 23:22:25.727 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 23:22:25.727 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 23:22:25.727 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 23:22:25.727 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 23:22:25.727 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 23:22:25.727 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 23:22:25.727 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:22:25.727 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 23:22:25.727 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:22:25.727 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 23:22:25.727 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 23:22:25.727 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 23:22:25.727 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 23:22:25.727 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:22:25.727 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 23:22:25.727 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:22:25.727 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:22:25.727 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:22:25.727 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 23:22:25.727 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 23:22:25.727 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:22:25.727 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 23:22:25.727 | INFO | ------------------------------------------ | 20260505 23:22:25.727 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:22:25.727 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 23:22:25.727 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:22:25.727 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:22:25.727 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.40e+01, 1.35e+01 / 7.96e-01 | 20260505 23:22:25.727 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.04e-01, 1.03e-02 / 3.77e-03 | 20260505 23:22:25.727 | INFO | PerfReport: 11.82%, Baseline -- 3.71e+00 / 4.51e+00, 5.68e-01 / 1.78e-01 | 20260505 23:22:25.727 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.53e+00, 1.91e-01 / 8.66e-02 | 20260505 23:22:25.727 | INFO | PerfReport: 15.97%, DME -- 4.01e+01 / 4.38e+01, 3.63e+00 / 7.43e-01 | 20260505 23:22:25.727 | INFO | PerfReport: 57.35%, FrameLabeling -- 1.80e+01 / 1.96e+01, 1.54e+00 / 9.94e-01 | 20260505 23:22:25.727 | INFO | PerfReport: 7.11%, PulseAccumulating -- 2.23e+00 / 2.75e+00, 4.25e-01 / 2.41e-01 | 20260505 23:22:25.727 | INFO | PerfReport: 3.11%, Metrics -- 9.76e-01 / 1.26e+00, 2.64e-01 / 1.15e-01 | 20260505 23:22:25.727 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.39e-01 / 3.98e-01, 5.84e-02 / 2.87e-02 | 20260505 23:22:25.727 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 23:22:25.727 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:22:25.727 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:22:25.727 | INFO | PerfReport: Full Execution -- 9.85e+01 / -nan, 2.48e+01 / 0.00e+00 | 20260505 23:22:25.727 | INFO | PerfReport: 22.34%, Upload -- 2.20e+01 / 0.00e+00, 7.53e+00 / 0.00e+00 | 20260505 23:22:25.727 | INFO | PerfReport: 21.82%, Download -- 2.15e+01 / 0.00e+00, 7.60e+00 / 0.00e+00 | 20260505 23:22:25.727 | INFO | PerfReport: 31.85%, Compute -- 3.14e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 23:22:25.727 | INFO | PerfReport: 2.15%, LockUpload -- 2.12e+00 / 0.00e+00, 7.93e+00 / 0.00e+00 | 20260505 23:22:25.727 | INFO | PerfReport: 1.29%, LockDownload -- 1.27e+00 / 0.00e+00, 2.82e+00 / 0.00e+00 | 20260505 23:22:25.727 | INFO | PerfReport: 20.54%, LockCompute -- 2.02e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260505 23:22:25.727 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.03e-02 / 0.00e+00 | 20260505 23:22:25.843 | INFO | Starting a round of Baz IO | 20260505 23:22:28.991 | INFO | Populating RTMetrics report | 20260505 23:22:28.991 | INFO | RTMetrics report populated | 20260505 23:22:28.991 | INFO | Opening tmp RTMetrics json file | 20260505 23:22:28.993 | INFO | Serializing RTMetrics to json | 20260505 23:22:28.996 | INFO | Serialization done, swapping tmp file now... | 20260505 23:22:28.997 | INFO | RTMetrics done | 20260505 23:22:29.073 | INFO | Finished a round of Baz IO: Wrote 1.896 GiB in 3.23 seconds | 20260505 23:22:29.074 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.43% / 0.00% / 78.57% | 20260505 23:22:29.074 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.108592% / 3.97364e-06% / 0.000107288% | 20260505 23:22:29.074 | INFO | Baz IO overall: | 20260505 23:22:29.074 | INFO | BytesWritten : 4073549052612 | 20260505 23:22:29.074 | INFO | NumEvents : 1919342895288 | 20260505 23:22:30.859 | INFO | Analyzing chunk frames = [7118848,7119360) | 20260505 23:22:30.859 | INFO | MemInfo: { | 20260505 23:22:30.859 | INFO | "gpu" : " 17 GB", | 20260505 23:22:30.859 | INFO | "huge" : "", | 20260505 23:22:30.859 | INFO | "huge_shared" : " 27 GB", | 20260505 23:22:30.859 | INFO | "malloc" : " 14 GB", | 20260505 23:22:30.859 | INFO | "pinned" : " 29 GB" | 20260505 23:22:30.859 | INFO | } | 20260505 23:22:30.859 | INFO | Perf report for the last 5.13312 seconds: | 20260505 23:22:30.859 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:22:30.859 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:30.859 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:30.859 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:30.859 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260505 23:22:30.859 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:30.859 | INFO | RT_METRICS : 43.6%, 1.0, 22.8ms, 0 | 20260505 23:22:30.859 | INFO | PRE_HQ : 59.9%, 1.7, 54.2ms, 0 | 20260505 23:22:30.859 | INFO | BAZWRITER : 62.1%, 3.6, 113.6ms, 0 | 20260505 23:22:30.859 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 23:22:30.859 | INFO | Uploaded 36GiB to GPU in 2.16871s (16.5997GiB/s) | 20260505 23:22:30.859 | INFO | Downloaded 38GiB to GPU in 2.24344s (16.9383GiB/s) | 20260505 23:22:36.000 | INFO | Analyzing chunk frames = [7119360,7119872) | 20260505 23:22:36.000 | INFO | MemInfo: { | 20260505 23:22:36.000 | INFO | "gpu" : " 17 GB", | 20260505 23:22:36.000 | INFO | "huge" : "", | 20260505 23:22:36.000 | INFO | "huge_shared" : " 27 GB", | 20260505 23:22:36.000 | INFO | "malloc" : " 14 GB", | 20260505 23:22:36.000 | INFO | "pinned" : " 29 GB" | 20260505 23:22:36.000 | INFO | } | 20260505 23:22:36.000 | INFO | Perf report for the last 5.1413 seconds: | 20260505 23:22:36.000 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:22:36.000 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:36.000 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:36.000 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:36.000 | INFO | ANALYSIS : 61.8%, 3.0, 96.6ms, 0 | 20260505 23:22:36.000 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:36.000 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:36.000 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:22:36.000 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:36.000 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:36.000 | INFO | Uploaded 36GiB to GPU in 2.09218s (17.207GiB/s) | 20260505 23:22:36.000 | INFO | Downloaded 34GiB to GPU in 2.03155s (16.736GiB/s) | 20260505 23:22:41.136 | INFO | Analyzing chunk frames = [7119872,7120384) | 20260505 23:22:41.136 | INFO | MemInfo: { | 20260505 23:22:41.136 | INFO | "gpu" : " 17 GB", | 20260505 23:22:41.136 | INFO | "huge" : "", | 20260505 23:22:41.136 | INFO | "huge_shared" : " 27 GB", | 20260505 23:22:41.136 | INFO | "malloc" : " 14 GB", | 20260505 23:22:41.136 | INFO | "pinned" : " 29 GB" | 20260505 23:22:41.136 | INFO | } | 20260505 23:22:41.136 | INFO | Perf report for the last 5.13616 seconds: | 20260505 23:22:41.136 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:22:41.136 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:41.136 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:41.136 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:41.136 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260505 23:22:41.136 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:41.136 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:41.137 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:22:41.137 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:41.137 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:41.137 | INFO | Uploaded 36GiB to GPU in 2.15185s (16.7298GiB/s) | 20260505 23:22:41.137 | INFO | Downloaded 34GiB to GPU in 2.04262s (16.6453GiB/s) | 20260505 23:22:46.279 | INFO | Analyzing chunk frames = [7120384,7120896) | 20260505 23:22:46.279 | INFO | MemInfo: { | 20260505 23:22:46.279 | INFO | "gpu" : " 17 GB", | 20260505 23:22:46.279 | INFO | "huge" : "", | 20260505 23:22:46.279 | INFO | "huge_shared" : " 27 GB", | 20260505 23:22:46.279 | INFO | "malloc" : " 14 GB", | 20260505 23:22:46.279 | INFO | "pinned" : " 29 GB" | 20260505 23:22:46.279 | INFO | } | 20260505 23:22:46.279 | INFO | Perf report for the last 5.14275 seconds: | 20260505 23:22:46.279 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:22:46.279 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:46.279 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:46.279 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:46.279 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260505 23:22:46.279 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:46.279 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:46.279 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:22:46.279 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:46.279 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:46.279 | INFO | Uploaded 36GiB to GPU in 2.14731s (16.7652GiB/s) | 20260505 23:22:46.279 | INFO | Downloaded 34GiB to GPU in 2.0268s (16.7752GiB/s) | 20260505 23:22:51.423 | INFO | Analyzing chunk frames = [7120896,7121408) | 20260505 23:22:51.423 | INFO | MemInfo: { | 20260505 23:22:51.423 | INFO | "gpu" : " 17 GB", | 20260505 23:22:51.423 | INFO | "huge" : "", | 20260505 23:22:51.423 | INFO | "huge_shared" : " 27 GB", | 20260505 23:22:51.423 | INFO | "malloc" : " 14 GB", | 20260505 23:22:51.423 | INFO | "pinned" : " 29 GB" | 20260505 23:22:51.423 | INFO | } | 20260505 23:22:51.423 | INFO | Perf report for the last 5.14371 seconds: | 20260505 23:22:51.423 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:22:51.423 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:51.423 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:51.423 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:51.423 | INFO | ANALYSIS : 61.2%, 3.0, 95.6ms, 0 | 20260505 23:22:51.423 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:51.423 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:51.423 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:22:51.423 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:51.423 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:51.423 | INFO | Uploaded 36GiB to GPU in 2.09435s (17.1891GiB/s) | 20260505 23:22:51.423 | INFO | Downloaded 34GiB to GPU in 2.04786s (16.6027GiB/s) | 20260505 23:22:56.552 | INFO | Analyzing chunk frames = [7121408,7121920) | 20260505 23:22:56.552 | INFO | MemInfo: { | 20260505 23:22:56.552 | INFO | "gpu" : " 17 GB", | 20260505 23:22:56.552 | INFO | "huge" : "", | 20260505 23:22:56.553 | INFO | "huge_shared" : " 27 GB", | 20260505 23:22:56.553 | INFO | "malloc" : " 14 GB", | 20260505 23:22:56.553 | INFO | "pinned" : " 29 GB" | 20260505 23:22:56.553 | INFO | } | 20260505 23:22:56.553 | INFO | Perf report for the last 5.12975 seconds: | 20260505 23:22:56.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:22:56.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:56.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:56.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:56.553 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260505 23:22:56.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:56.553 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:56.553 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260505 23:22:56.553 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:56.553 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:22:56.553 | INFO | Uploaded 36GiB to GPU in 2.08328s (17.2804GiB/s) | 20260505 23:22:56.553 | INFO | Downloaded 34GiB to GPU in 2.03768s (16.6857GiB/s) | 20260505 23:23:01.693 | INFO | Analyzing chunk frames = [7121920,7122432) | 20260505 23:23:01.693 | INFO | MemInfo: { | 20260505 23:23:01.693 | INFO | "gpu" : " 17 GB", | 20260505 23:23:01.693 | INFO | "huge" : "", | 20260505 23:23:01.693 | INFO | "huge_shared" : " 27 GB", | 20260505 23:23:01.693 | INFO | "malloc" : " 14 GB", | 20260505 23:23:01.693 | INFO | "pinned" : " 29 GB" | 20260505 23:23:01.693 | INFO | } | 20260505 23:23:01.693 | INFO | Perf report for the last 5.14008 seconds: | 20260505 23:23:01.693 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:23:01.693 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:01.693 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:01.693 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:01.693 | INFO | ANALYSIS : 62.7%, 3.0, 98.0ms, 0 | 20260505 23:23:01.693 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:01.693 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:01.693 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 23:23:01.693 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:01.693 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:01.693 | INFO | Uploaded 36GiB to GPU in 2.1582s (16.6805GiB/s) | 20260505 23:23:01.693 | INFO | Downloaded 34GiB to GPU in 2.13209s (15.9468GiB/s) | 20260505 23:23:06.840 | INFO | Analyzing chunk frames = [7122432,7122944) | 20260505 23:23:06.840 | INFO | MemInfo: { | 20260505 23:23:06.840 | INFO | "gpu" : " 17 GB", | 20260505 23:23:06.840 | INFO | "huge" : "", | 20260505 23:23:06.840 | INFO | "huge_shared" : " 27 GB", | 20260505 23:23:06.840 | INFO | "malloc" : " 14 GB", | 20260505 23:23:06.840 | INFO | "pinned" : " 29 GB" | 20260505 23:23:06.840 | INFO | } | 20260505 23:23:06.840 | INFO | Perf report for the last 5.14739 seconds: | 20260505 23:23:06.840 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:23:06.840 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:06.840 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:06.840 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:06.840 | INFO | ANALYSIS : 61.9%, 3.0, 96.9ms, 0 | 20260505 23:23:06.840 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:06.840 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:06.840 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:23:06.840 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:06.840 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:06.840 | INFO | Uploaded 36GiB to GPU in 2.11715s (17.004GiB/s) | 20260505 23:23:06.840 | INFO | Downloaded 34GiB to GPU in 2.00296s (16.9749GiB/s) | 20260505 23:23:06.957 | INFO | Starting a round of Baz IO | 20260505 23:23:10.113 | INFO | Populating RTMetrics report | 20260505 23:23:10.113 | INFO | RTMetrics report populated | 20260505 23:23:10.113 | INFO | Opening tmp RTMetrics json file | 20260505 23:23:10.119 | INFO | Serializing RTMetrics to json | 20260505 23:23:10.121 | INFO | Serialization done, swapping tmp file now... | 20260505 23:23:10.123 | INFO | RTMetrics done | 20260505 23:23:10.192 | INFO | Finished a round of Baz IO: Wrote 1.896 GiB in 3.234 seconds | 20260505 23:23:10.192 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.43% / 0.00% / 78.57% | 20260505 23:23:10.192 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.108365% / 0% / 9.93411e-05% | 20260505 23:23:10.192 | INFO | Baz IO overall: | 20260505 23:23:10.192 | INFO | BytesWritten : 4075585089352 | 20260505 23:23:10.192 | INFO | NumEvents : 1920277235634 | 20260505 23:23:11.973 | INFO | Analyzing chunk frames = [7122944,7123456) | 20260505 23:23:11.973 | INFO | MemInfo: { | 20260505 23:23:11.973 | INFO | "gpu" : " 17 GB", | 20260505 23:23:11.973 | INFO | "huge" : "", | 20260505 23:23:11.973 | INFO | "huge_shared" : " 27 GB", | 20260505 23:23:11.973 | INFO | "malloc" : " 14 GB", | 20260505 23:23:11.973 | INFO | "pinned" : " 29 GB" | 20260505 23:23:11.973 | INFO | } | 20260505 23:23:11.973 | INFO | Perf report for the last 5.13337 seconds: | 20260505 23:23:11.973 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:23:11.973 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:11.973 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:11.973 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:11.973 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260505 23:23:11.973 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:11.974 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260505 23:23:11.974 | INFO | PRE_HQ : 60.0%, 1.7, 54.1ms, 0 | 20260505 23:23:11.974 | INFO | BAZWRITER : 62.1%, 3.6, 114.4ms, 0 | 20260505 23:23:11.974 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 23:23:11.974 | INFO | Uploaded 36GiB to GPU in 2.18416s (16.4823GiB/s) | 20260505 23:23:11.974 | INFO | Downloaded 38GiB to GPU in 2.2416s (16.9522GiB/s) | 20260505 23:23:17.106 | INFO | Analyzing chunk frames = [7123456,7123968) | 20260505 23:23:17.106 | INFO | MemInfo: { | 20260505 23:23:17.106 | INFO | "gpu" : " 17 GB", | 20260505 23:23:17.106 | INFO | "huge" : "", | 20260505 23:23:17.106 | INFO | "huge_shared" : " 27 GB", | 20260505 23:23:17.106 | INFO | "malloc" : " 14 GB", | 20260505 23:23:17.106 | INFO | "pinned" : " 29 GB" | 20260505 23:23:17.106 | INFO | } | 20260505 23:23:17.107 | INFO | Perf report for the last 5.13304 seconds: | 20260505 23:23:17.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:23:17.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:17.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:17.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:17.107 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260505 23:23:17.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:17.107 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:17.107 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:23:17.107 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:17.107 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:17.107 | INFO | Uploaded 36GiB to GPU in 2.08708s (17.249GiB/s) | 20260505 23:23:17.107 | INFO | Downloaded 34GiB to GPU in 2.02965s (16.7517GiB/s) | 20260505 23:23:22.250 | INFO | Analyzing chunk frames = [7123968,7124480) | 20260505 23:23:22.250 | INFO | MemInfo: { | 20260505 23:23:22.250 | INFO | "gpu" : " 17 GB", | 20260505 23:23:22.250 | INFO | "huge" : "", | 20260505 23:23:22.250 | INFO | "huge_shared" : " 27 GB", | 20260505 23:23:22.250 | INFO | "malloc" : " 14 GB", | 20260505 23:23:22.250 | INFO | "pinned" : " 29 GB" | 20260505 23:23:22.250 | INFO | } | 20260505 23:23:22.250 | INFO | Perf report for the last 5.14342 seconds: | 20260505 23:23:22.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:23:22.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:22.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:22.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:22.250 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260505 23:23:22.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:22.250 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:22.250 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260505 23:23:22.250 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:22.250 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:22.250 | INFO | Uploaded 36GiB to GPU in 2.14527s (16.7811GiB/s) | 20260505 23:23:22.250 | INFO | Downloaded 34GiB to GPU in 2.02257s (16.8103GiB/s) | 20260505 23:23:27.392 | INFO | Analyzing chunk frames = [7124480,7124992) | 20260505 23:23:27.392 | INFO | MemInfo: { | 20260505 23:23:27.392 | INFO | "gpu" : " 17 GB", | 20260505 23:23:27.392 | INFO | "huge" : "", | 20260505 23:23:27.392 | INFO | "huge_shared" : " 27 GB", | 20260505 23:23:27.392 | INFO | "malloc" : " 14 GB", | 20260505 23:23:27.392 | INFO | "pinned" : " 29 GB" | 20260505 23:23:27.392 | INFO | } | 20260505 23:23:27.392 | INFO | Perf report for the last 5.14193 seconds: | 20260505 23:23:27.392 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:23:27.392 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:27.392 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:27.392 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:27.392 | INFO | ANALYSIS : 61.7%, 3.0, 95.8ms, 0 | 20260505 23:23:27.392 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:27.392 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:27.392 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:23:27.392 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:27.392 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:27.392 | INFO | Uploaded 36GiB to GPU in 2.12404s (16.9488GiB/s) | 20260505 23:23:27.392 | INFO | Downloaded 34GiB to GPU in 2.02155s (16.8188GiB/s) | 20260505 23:23:32.535 | INFO | Analyzing chunk frames = [7124992,7125504) | 20260505 23:23:32.535 | INFO | MemInfo: { | 20260505 23:23:32.535 | INFO | "gpu" : " 17 GB", | 20260505 23:23:32.535 | INFO | "huge" : "", | 20260505 23:23:32.535 | INFO | "huge_shared" : " 27 GB", | 20260505 23:23:32.535 | INFO | "malloc" : " 14 GB", | 20260505 23:23:32.535 | INFO | "pinned" : " 29 GB" | 20260505 23:23:32.535 | INFO | } | 20260505 23:23:32.535 | INFO | Perf report for the last 5.14289 seconds: | 20260505 23:23:32.535 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:23:32.535 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:32.535 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:32.535 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:32.535 | INFO | ANALYSIS : 61.2%, 3.0, 95.6ms, 0 | 20260505 23:23:32.535 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:32.535 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:32.535 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:23:32.535 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:32.535 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:32.535 | INFO | Uploaded 36GiB to GPU in 2.10149s (17.1307GiB/s) | 20260505 23:23:32.535 | INFO | Downloaded 34GiB to GPU in 2.04424s (16.6321GiB/s) | 20260505 23:23:37.677 | INFO | Analyzing chunk frames = [7125504,7126016) | 20260505 23:23:37.677 | INFO | MemInfo: { | 20260505 23:23:37.677 | INFO | "gpu" : " 17 GB", | 20260505 23:23:37.677 | INFO | "huge" : "", | 20260505 23:23:37.677 | INFO | "huge_shared" : " 27 GB", | 20260505 23:23:37.677 | INFO | "malloc" : " 14 GB", | 20260505 23:23:37.677 | INFO | "pinned" : " 29 GB" | 20260505 23:23:37.677 | INFO | } | 20260505 23:23:37.677 | INFO | Perf report for the last 5.14259 seconds: | 20260505 23:23:37.677 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:23:37.677 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:37.677 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:37.677 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:37.677 | INFO | ANALYSIS : 61.4%, 3.0, 95.9ms, 0 | 20260505 23:23:37.677 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:37.677 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:37.677 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:23:37.677 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:37.677 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:37.678 | INFO | Uploaded 36GiB to GPU in 2.07943s (17.3124GiB/s) | 20260505 23:23:37.678 | INFO | Downloaded 34GiB to GPU in 2.03902s (16.6747GiB/s) | 20260505 23:23:42.809 | INFO | Analyzing chunk frames = [7126016,7126528) | 20260505 23:23:42.809 | INFO | MemInfo: { | 20260505 23:23:42.809 | INFO | "gpu" : " 17 GB", | 20260505 23:23:42.809 | INFO | "huge" : "", | 20260505 23:23:42.810 | INFO | "huge_shared" : " 27 GB", | 20260505 23:23:42.810 | INFO | "malloc" : " 14 GB", | 20260505 23:23:42.810 | INFO | "pinned" : " 29 GB" | 20260505 23:23:42.810 | INFO | } | 20260505 23:23:42.810 | INFO | Perf report for the last 5.13216 seconds: | 20260505 23:23:42.810 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:23:42.810 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:42.810 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:42.810 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:42.810 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 23:23:42.810 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:42.810 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:42.810 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 23:23:42.810 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:42.810 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:42.810 | INFO | Uploaded 36GiB to GPU in 2.0332s (17.706GiB/s) | 20260505 23:23:42.810 | INFO | Downloaded 34GiB to GPU in 2.03854s (16.6786GiB/s) | 20260505 23:23:47.956 | INFO | Analyzing chunk frames = [7126528,7127040) | 20260505 23:23:47.956 | INFO | MemInfo: { | 20260505 23:23:47.956 | INFO | "gpu" : " 17 GB", | 20260505 23:23:47.956 | INFO | "huge" : "", | 20260505 23:23:47.956 | INFO | "huge_shared" : " 27 GB", | 20260505 23:23:47.956 | INFO | "malloc" : " 14 GB", | 20260505 23:23:47.956 | INFO | "pinned" : " 29 GB" | 20260505 23:23:47.956 | INFO | } | 20260505 23:23:47.956 | INFO | Perf report for the last 5.14639 seconds: | 20260505 23:23:47.956 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:23:47.956 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:47.956 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:47.956 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:47.956 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260505 23:23:47.956 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:47.956 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:47.956 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:23:47.956 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:47.956 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:47.956 | INFO | Uploaded 36GiB to GPU in 2.11699s (17.0052GiB/s) | 20260505 23:23:47.956 | INFO | Downloaded 34GiB to GPU in 2.00418s (16.9645GiB/s) | 20260505 23:23:47.956 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 23:23:47.956 | INFO | Total mem usage: 15.3 GiB | 20260505 23:23:47.956 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 23:23:47.956 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.11%) | 20260505 23:23:47.956 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 23:23:47.956 | INFO | High Water Breakdown: | 20260505 23:23:47.957 | INFO | 60.9% : 8.55 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 23:23:47.957 | INFO | 39% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 23:23:47.957 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 23:23:47.957 | INFO | ------------------------------------------ | 20260505 23:23:47.957 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:23:47.957 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 23:23:47.957 | INFO | Total mem usage: 26.5 GiB | 20260505 23:23:47.957 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 23:23:47.957 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 23:23:47.957 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 23:23:47.957 | INFO | High Water Breakdown: | 20260505 23:23:47.957 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:23:47.957 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:23:47.957 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:23:47.957 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:23:47.957 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:23:47.957 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:23:47.957 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 23:23:47.957 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 23:23:47.957 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:23:47.957 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 23:23:47.957 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 23:23:47.957 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 23:23:47.957 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 23:23:47.957 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 23:23:47.957 | INFO | ------------------------------------------ | 20260505 23:23:47.957 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:23:47.957 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 23:23:47.957 | INFO | Total mem usage: 25.6 GiB | 20260505 23:23:47.957 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 23:23:47.957 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 23:23:47.957 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 23:23:47.957 | INFO | High Water Breakdown: | 20260505 23:23:47.957 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 23:23:47.957 | INFO | ------------------------------------------ | 20260505 23:23:47.957 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:23:47.957 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 23:23:47.957 | INFO | Total mem usage: 15.7 GiB | 20260505 23:23:47.957 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 23:23:47.957 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 23:23:47.957 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 23:23:47.957 | INFO | High Water Breakdown: | 20260505 23:23:47.957 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 23:23:47.957 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 23:23:47.957 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 23:23:47.957 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 23:23:47.957 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 23:23:47.957 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 23:23:47.957 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 23:23:47.957 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 23:23:47.957 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 23:23:47.957 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:23:47.957 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 23:23:47.957 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:23:47.957 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 23:23:47.957 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 23:23:47.957 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 23:23:47.957 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 23:23:47.957 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:23:47.957 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 23:23:47.957 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:23:47.957 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:23:47.957 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:23:47.957 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 23:23:47.957 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 23:23:47.957 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:23:47.957 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 23:23:47.957 | INFO | ------------------------------------------ | 20260505 23:23:47.957 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:23:47.957 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 23:23:47.957 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:23:47.957 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:23:47.957 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.40e+01, 1.35e+01 / 1.05e+00 | 20260505 23:23:47.957 | INFO | PerfReport: 0.29%, AggregateStats -- 9.18e-02 / 1.03e-01, 1.02e-02 / 4.26e-03 | 20260505 23:23:47.957 | INFO | PerfReport: 11.82%, Baseline -- 3.70e+00 / 4.54e+00, 5.60e-01 / 1.36e-01 | 20260505 23:23:47.957 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.52e+00, 1.86e-01 / 9.67e-02 | 20260505 23:23:47.957 | INFO | PerfReport: 16.03%, DME -- 4.02e+01 / 4.41e+01, 3.76e+00 / 1.22e+00 | 20260505 23:23:47.957 | INFO | PerfReport: 57.24%, FrameLabeling -- 1.79e+01 / 1.93e+01, 1.44e+00 / 1.01e+00 | 20260505 23:23:47.957 | INFO | PerfReport: 7.15%, PulseAccumulating -- 2.24e+00 / 2.77e+00, 4.25e-01 / 2.28e-01 | 20260505 23:23:47.957 | INFO | PerfReport: 3.12%, Metrics -- 9.77e-01 / 1.26e+00, 2.64e-01 / 9.44e-02 | 20260505 23:23:47.958 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.39e-01 / 3.91e-01, 5.80e-02 / 2.94e-02 | 20260505 23:23:47.958 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 23:23:47.958 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:23:47.958 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:23:47.958 | INFO | PerfReport: Full Execution -- 9.85e+01 / -nan, 2.47e+01 / 0.00e+00 | 20260505 23:23:47.958 | INFO | PerfReport: 22.39%, Upload -- 2.21e+01 / 0.00e+00, 7.55e+00 / 0.00e+00 | 20260505 23:23:47.958 | INFO | PerfReport: 21.81%, Download -- 2.15e+01 / 0.00e+00, 7.50e+00 / 0.00e+00 | 20260505 23:23:47.958 | INFO | PerfReport: 31.79%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 23:23:47.958 | INFO | PerfReport: 2.17%, LockUpload -- 2.14e+00 / 0.00e+00, 7.92e+00 / 0.00e+00 | 20260505 23:23:47.958 | INFO | PerfReport: 1.30%, LockDownload -- 1.28e+00 / 0.00e+00, 2.82e+00 / 0.00e+00 | 20260505 23:23:47.958 | INFO | PerfReport: 20.52%, LockCompute -- 2.02e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260505 23:23:47.958 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.81e-02 / 0.00e+00 | 20260505 23:23:48.074 | INFO | Starting a round of Baz IO | 20260505 23:23:51.220 | INFO | Populating RTMetrics report | 20260505 23:23:51.220 | INFO | RTMetrics report populated | 20260505 23:23:51.220 | INFO | Opening tmp RTMetrics json file | 20260505 23:23:51.223 | INFO | Serializing RTMetrics to json | 20260505 23:23:51.226 | INFO | Serialization done, swapping tmp file now... | 20260505 23:23:51.226 | INFO | RTMetrics done | 20260505 23:23:51.301 | INFO | Finished a round of Baz IO: Wrote 1.896 GiB in 3.226 seconds | 20260505 23:23:51.301 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.43% / 0.00% / 78.57% | 20260505 23:23:51.301 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.10821% / 0% / 9.53674e-05% | 20260505 23:23:51.301 | INFO | Baz IO overall: | 20260505 23:23:51.301 | INFO | BytesWritten : 4077620979302 | 20260505 23:23:51.301 | INFO | NumEvents : 1921211408002 | 20260505 23:23:53.089 | INFO | Analyzing chunk frames = [7127040,7127552) | 20260505 23:23:53.089 | INFO | MemInfo: { | 20260505 23:23:53.089 | INFO | "gpu" : " 17 GB", | 20260505 23:23:53.089 | INFO | "huge" : "", | 20260505 23:23:53.089 | INFO | "huge_shared" : " 27 GB", | 20260505 23:23:53.089 | INFO | "malloc" : " 14 GB", | 20260505 23:23:53.089 | INFO | "pinned" : " 29 GB" | 20260505 23:23:53.089 | INFO | } | 20260505 23:23:53.089 | INFO | Perf report for the last 5.1332 seconds: | 20260505 23:23:53.089 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:23:53.089 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:53.089 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:53.089 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:53.089 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 23:23:53.089 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:53.089 | INFO | RT_METRICS : 43.4%, 1.0, 22.8ms, 0 | 20260505 23:23:53.089 | INFO | PRE_HQ : 59.5%, 1.8, 54.2ms, 0 | 20260505 23:23:53.089 | INFO | BAZWRITER : 62.0%, 3.6, 113.3ms, 0 | 20260505 23:23:53.089 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 23:23:53.089 | INFO | Uploaded 36GiB to GPU in 2.11822s (16.9954GiB/s) | 20260505 23:23:53.090 | INFO | Downloaded 38GiB to GPU in 2.22554s (17.0745GiB/s) | 20260505 23:23:58.231 | INFO | Analyzing chunk frames = [7127552,7128064) | 20260505 23:23:58.231 | INFO | MemInfo: { | 20260505 23:23:58.231 | INFO | "gpu" : " 17 GB", | 20260505 23:23:58.231 | INFO | "huge" : "", | 20260505 23:23:58.231 | INFO | "huge_shared" : " 27 GB", | 20260505 23:23:58.231 | INFO | "malloc" : " 14 GB", | 20260505 23:23:58.231 | INFO | "pinned" : " 29 GB" | 20260505 23:23:58.231 | INFO | } | 20260505 23:23:58.232 | INFO | Perf report for the last 5.14231 seconds: | 20260505 23:23:58.232 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:23:58.232 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:58.232 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:58.232 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:58.232 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260505 23:23:58.232 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:58.232 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:58.232 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260505 23:23:58.232 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:58.232 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:23:58.232 | INFO | Uploaded 36GiB to GPU in 2.09052s (17.2206GiB/s) | 20260505 23:23:58.232 | INFO | Downloaded 34GiB to GPU in 2.03572s (16.7017GiB/s) | 20260505 23:24:03.371 | INFO | Analyzing chunk frames = [7128064,7128576) | 20260505 23:24:03.371 | INFO | MemInfo: { | 20260505 23:24:03.371 | INFO | "gpu" : " 17 GB", | 20260505 23:24:03.371 | INFO | "huge" : "", | 20260505 23:24:03.371 | INFO | "huge_shared" : " 27 GB", | 20260505 23:24:03.371 | INFO | "malloc" : " 14 GB", | 20260505 23:24:03.371 | INFO | "pinned" : " 29 GB" | 20260505 23:24:03.371 | INFO | } | 20260505 23:24:03.371 | INFO | Perf report for the last 5.13979 seconds: | 20260505 23:24:03.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:24:03.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:03.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:03.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:03.371 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260505 23:24:03.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:03.371 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:03.371 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:24:03.372 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:03.372 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:03.372 | INFO | Uploaded 36GiB to GPU in 2.10918s (17.0683GiB/s) | 20260505 23:24:03.372 | INFO | Downloaded 34GiB to GPU in 2.03092s (16.7412GiB/s) | 20260505 23:24:08.503 | INFO | Analyzing chunk frames = [7128576,7129088) | 20260505 23:24:08.503 | INFO | MemInfo: { | 20260505 23:24:08.503 | INFO | "gpu" : " 17 GB", | 20260505 23:24:08.503 | INFO | "huge" : "", | 20260505 23:24:08.503 | INFO | "huge_shared" : " 27 GB", | 20260505 23:24:08.503 | INFO | "malloc" : " 14 GB", | 20260505 23:24:08.503 | INFO | "pinned" : " 29 GB" | 20260505 23:24:08.503 | INFO | } | 20260505 23:24:08.503 | INFO | Perf report for the last 5.13198 seconds: | 20260505 23:24:08.503 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:24:08.503 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:08.503 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:08.503 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:08.503 | INFO | ANALYSIS : 61.8%, 3.0, 95.8ms, 0 | 20260505 23:24:08.503 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:08.503 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:08.503 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 23:24:08.503 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:08.503 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:08.504 | INFO | Uploaded 36GiB to GPU in 2.13334s (16.8749GiB/s) | 20260505 23:24:08.504 | INFO | Downloaded 34GiB to GPU in 2.01655s (16.8604GiB/s) | 20260505 23:24:13.648 | INFO | Analyzing chunk frames = [7129088,7129600) | 20260505 23:24:13.648 | INFO | MemInfo: { | 20260505 23:24:13.648 | INFO | "gpu" : " 17 GB", | 20260505 23:24:13.648 | INFO | "huge" : "", | 20260505 23:24:13.648 | INFO | "huge_shared" : " 27 GB", | 20260505 23:24:13.648 | INFO | "malloc" : " 14 GB", | 20260505 23:24:13.648 | INFO | "pinned" : " 29 GB" | 20260505 23:24:13.648 | INFO | } | 20260505 23:24:13.648 | INFO | Perf report for the last 5.14514 seconds: | 20260505 23:24:13.649 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:24:13.649 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:13.649 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:13.649 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:13.649 | INFO | ANALYSIS : 61.1%, 3.0, 95.6ms, 0 | 20260505 23:24:13.649 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:13.649 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:13.649 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:24:13.649 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:13.649 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:13.649 | INFO | Uploaded 36GiB to GPU in 2.10033s (17.1402GiB/s) | 20260505 23:24:13.649 | INFO | Downloaded 34GiB to GPU in 2.04668s (16.6122GiB/s) | 20260505 23:24:18.787 | INFO | Analyzing chunk frames = [7129600,7130112) | 20260505 23:24:18.787 | INFO | MemInfo: { | 20260505 23:24:18.787 | INFO | "gpu" : " 17 GB", | 20260505 23:24:18.787 | INFO | "huge" : "", | 20260505 23:24:18.787 | INFO | "huge_shared" : " 27 GB", | 20260505 23:24:18.787 | INFO | "malloc" : " 14 GB", | 20260505 23:24:18.787 | INFO | "pinned" : " 29 GB" | 20260505 23:24:18.787 | INFO | } | 20260505 23:24:18.788 | INFO | Perf report for the last 5.13905 seconds: | 20260505 23:24:18.788 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:24:18.788 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:18.788 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:18.788 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:18.788 | INFO | ANALYSIS : 61.2%, 3.0, 95.6ms, 0 | 20260505 23:24:18.788 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:18.788 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:18.788 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 23:24:18.788 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:18.788 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:18.788 | INFO | Uploaded 36GiB to GPU in 2.07078s (17.3848GiB/s) | 20260505 23:24:18.788 | INFO | Downloaded 34GiB to GPU in 2.03215s (16.7311GiB/s) | 20260505 23:24:23.923 | INFO | Analyzing chunk frames = [7130112,7130624) | 20260505 23:24:23.923 | INFO | MemInfo: { | 20260505 23:24:23.923 | INFO | "gpu" : " 17 GB", | 20260505 23:24:23.923 | INFO | "huge" : "", | 20260505 23:24:23.923 | INFO | "huge_shared" : " 27 GB", | 20260505 23:24:23.923 | INFO | "malloc" : " 14 GB", | 20260505 23:24:23.923 | INFO | "pinned" : " 29 GB" | 20260505 23:24:23.923 | INFO | } | 20260505 23:24:23.923 | INFO | Perf report for the last 5.13572 seconds: | 20260505 23:24:23.923 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:24:23.923 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:23.923 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:23.923 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:23.923 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260505 23:24:23.923 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:23.923 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:23.923 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260505 23:24:23.923 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:23.923 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:23.924 | INFO | Uploaded 36GiB to GPU in 2.01993s (17.8224GiB/s) | 20260505 23:24:23.924 | INFO | Downloaded 34GiB to GPU in 2.05818s (16.5194GiB/s) | 20260505 23:24:29.057 | INFO | Analyzing chunk frames = [7130624,7131136) | 20260505 23:24:29.057 | INFO | MemInfo: { | 20260505 23:24:29.057 | INFO | "gpu" : " 17 GB", | 20260505 23:24:29.057 | INFO | "huge" : "", | 20260505 23:24:29.057 | INFO | "huge_shared" : " 27 GB", | 20260505 23:24:29.057 | INFO | "malloc" : " 14 GB", | 20260505 23:24:29.057 | INFO | "pinned" : " 29 GB" | 20260505 23:24:29.057 | INFO | } | 20260505 23:24:29.057 | INFO | Perf report for the last 5.13404 seconds: | 20260505 23:24:29.057 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:24:29.057 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:29.057 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:29.057 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:29.057 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260505 23:24:29.057 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:29.057 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:29.057 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 23:24:29.057 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:29.057 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:29.057 | INFO | Uploaded 36GiB to GPU in 2.0911s (17.2158GiB/s) | 20260505 23:24:29.058 | INFO | Downloaded 34GiB to GPU in 2.00195s (16.9834GiB/s) | 20260505 23:24:29.188 | INFO | Starting a round of Baz IO | 20260505 23:24:32.340 | INFO | Populating RTMetrics report | 20260505 23:24:32.340 | INFO | RTMetrics report populated | 20260505 23:24:32.340 | INFO | Opening tmp RTMetrics json file | 20260505 23:24:32.345 | INFO | Serializing RTMetrics to json | 20260505 23:24:32.348 | INFO | Serialization done, swapping tmp file now... | 20260505 23:24:32.348 | INFO | RTMetrics done | 20260505 23:24:32.419 | INFO | Finished a round of Baz IO: Wrote 1.895 GiB in 3.23 seconds | 20260505 23:24:32.419 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.43% / 0.00% / 78.57% | 20260505 23:24:32.419 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.108254% / 0% / 0.000111262% | 20260505 23:24:32.419 | INFO | Baz IO overall: | 20260505 23:24:32.419 | INFO | BytesWritten : 4079656037920 | 20260505 23:24:32.419 | INFO | NumEvents : 1922144844266 | 20260505 23:24:34.205 | INFO | Analyzing chunk frames = [7131136,7131648) | 20260505 23:24:34.205 | INFO | MemInfo: { | 20260505 23:24:34.205 | INFO | "gpu" : " 17 GB", | 20260505 23:24:34.205 | INFO | "huge" : "", | 20260505 23:24:34.205 | INFO | "huge_shared" : " 27 GB", | 20260505 23:24:34.205 | INFO | "malloc" : " 14 GB", | 20260505 23:24:34.205 | INFO | "pinned" : " 29 GB" | 20260505 23:24:34.205 | INFO | } | 20260505 23:24:34.205 | INFO | Perf report for the last 5.14734 seconds: | 20260505 23:24:34.205 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:24:34.205 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:34.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:34.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:34.205 | INFO | ANALYSIS : 62.7%, 3.0, 98.1ms, 0 | 20260505 23:24:34.205 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:34.205 | INFO | RT_METRICS : 43.4%, 1.0, 22.8ms, 0 | 20260505 23:24:34.205 | INFO | PRE_HQ : 59.7%, 1.7, 54.1ms, 0 | 20260505 23:24:34.205 | INFO | BAZWRITER : 62.2%, 3.5, 113.1ms, 0 | 20260505 23:24:34.205 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 23:24:34.205 | INFO | Uploaded 36GiB to GPU in 2.15958s (16.6699GiB/s) | 20260505 23:24:34.205 | INFO | Downloaded 38GiB to GPU in 2.24683s (16.9127GiB/s) | 20260505 23:24:39.347 | INFO | Analyzing chunk frames = [7131648,7132160) | 20260505 23:24:39.347 | INFO | MemInfo: { | 20260505 23:24:39.347 | INFO | "gpu" : " 17 GB", | 20260505 23:24:39.347 | INFO | "huge" : "", | 20260505 23:24:39.347 | INFO | "huge_shared" : " 27 GB", | 20260505 23:24:39.347 | INFO | "malloc" : " 14 GB", | 20260505 23:24:39.347 | INFO | "pinned" : " 29 GB" | 20260505 23:24:39.347 | INFO | } | 20260505 23:24:39.347 | INFO | Perf report for the last 5.14268 seconds: | 20260505 23:24:39.347 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:24:39.347 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:39.347 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:39.347 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:39.347 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260505 23:24:39.347 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:39.347 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:39.347 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260505 23:24:39.347 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:39.347 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:39.348 | INFO | Uploaded 36GiB to GPU in 2.10137s (17.1317GiB/s) | 20260505 23:24:39.348 | INFO | Downloaded 34GiB to GPU in 2.03811s (16.6821GiB/s) | 20260505 23:24:44.485 | INFO | Analyzing chunk frames = [7132160,7132672) | 20260505 23:24:44.485 | INFO | MemInfo: { | 20260505 23:24:44.485 | INFO | "gpu" : " 17 GB", | 20260505 23:24:44.485 | INFO | "huge" : "", | 20260505 23:24:44.485 | INFO | "huge_shared" : " 27 GB", | 20260505 23:24:44.485 | INFO | "malloc" : " 14 GB", | 20260505 23:24:44.485 | INFO | "pinned" : " 29 GB" | 20260505 23:24:44.485 | INFO | } | 20260505 23:24:44.485 | INFO | Perf report for the last 5.13757 seconds: | 20260505 23:24:44.485 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:24:44.485 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:44.485 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:44.485 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:44.485 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260505 23:24:44.485 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:44.485 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:44.485 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:24:44.485 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:44.485 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:44.485 | INFO | Uploaded 36GiB to GPU in 2.08252s (17.2867GiB/s) | 20260505 23:24:44.485 | INFO | Downloaded 34GiB to GPU in 2.01792s (16.8491GiB/s) | 20260505 23:24:49.619 | INFO | Analyzing chunk frames = [7132672,7133184) | 20260505 23:24:49.619 | INFO | MemInfo: { | 20260505 23:24:49.619 | INFO | "gpu" : " 17 GB", | 20260505 23:24:49.619 | INFO | "huge" : "", | 20260505 23:24:49.619 | INFO | "huge_shared" : " 27 GB", | 20260505 23:24:49.619 | INFO | "malloc" : " 14 GB", | 20260505 23:24:49.619 | INFO | "pinned" : " 29 GB" | 20260505 23:24:49.619 | INFO | } | 20260505 23:24:49.619 | INFO | Perf report for the last 5.13436 seconds: | 20260505 23:24:49.619 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:24:49.619 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:49.619 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:49.619 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:49.619 | INFO | ANALYSIS : 61.8%, 3.0, 95.8ms, 0 | 20260505 23:24:49.619 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:49.619 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:49.619 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 23:24:49.619 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:49.620 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:49.620 | INFO | Uploaded 36GiB to GPU in 2.13425s (16.8677GiB/s) | 20260505 23:24:49.620 | INFO | Downloaded 34GiB to GPU in 2.00937s (16.9208GiB/s) | 20260505 23:24:54.759 | INFO | Analyzing chunk frames = [7133184,7133696) | 20260505 23:24:54.759 | INFO | MemInfo: { | 20260505 23:24:54.759 | INFO | "gpu" : " 17 GB", | 20260505 23:24:54.759 | INFO | "huge" : "", | 20260505 23:24:54.759 | INFO | "huge_shared" : " 27 GB", | 20260505 23:24:54.759 | INFO | "malloc" : " 14 GB", | 20260505 23:24:54.759 | INFO | "pinned" : " 29 GB" | 20260505 23:24:54.759 | INFO | } | 20260505 23:24:54.759 | INFO | Perf report for the last 5.13991 seconds: | 20260505 23:24:54.759 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:24:54.759 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:54.759 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:54.759 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:54.759 | INFO | ANALYSIS : 61.4%, 3.0, 95.9ms, 0 | 20260505 23:24:54.759 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:54.759 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:54.759 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:24:54.759 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:54.759 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:54.759 | INFO | Uploaded 36GiB to GPU in 2.12903s (16.9091GiB/s) | 20260505 23:24:54.759 | INFO | Downloaded 34GiB to GPU in 2.06136s (16.494GiB/s) | 20260505 23:24:59.903 | INFO | Analyzing chunk frames = [7133696,7134208) | 20260505 23:24:59.903 | INFO | MemInfo: { | 20260505 23:24:59.903 | INFO | "gpu" : " 17 GB", | 20260505 23:24:59.903 | INFO | "huge" : "", | 20260505 23:24:59.903 | INFO | "huge_shared" : " 27 GB", | 20260505 23:24:59.903 | INFO | "malloc" : " 14 GB", | 20260505 23:24:59.903 | INFO | "pinned" : " 29 GB" | 20260505 23:24:59.903 | INFO | } | 20260505 23:24:59.903 | INFO | Perf report for the last 5.14359 seconds: | 20260505 23:24:59.903 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:24:59.903 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:59.903 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:59.903 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:59.903 | INFO | ANALYSIS : 61.5%, 3.0, 96.1ms, 0 | 20260505 23:24:59.903 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:59.903 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:59.903 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:24:59.903 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:59.903 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:24:59.903 | INFO | Uploaded 36GiB to GPU in 2.08878s (17.2349GiB/s) | 20260505 23:24:59.903 | INFO | Downloaded 34GiB to GPU in 2.04178s (16.6521GiB/s) | 20260505 23:25:05.041 | INFO | Analyzing chunk frames = [7134208,7134720) | 20260505 23:25:05.041 | INFO | MemInfo: { | 20260505 23:25:05.041 | INFO | "gpu" : " 17 GB", | 20260505 23:25:05.041 | INFO | "huge" : "", | 20260505 23:25:05.041 | INFO | "huge_shared" : " 27 GB", | 20260505 23:25:05.041 | INFO | "malloc" : " 14 GB", | 20260505 23:25:05.041 | INFO | "pinned" : " 29 GB" | 20260505 23:25:05.041 | INFO | } | 20260505 23:25:05.041 | INFO | Perf report for the last 5.1383 seconds: | 20260505 23:25:05.041 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:25:05.041 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:05.041 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:05.041 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:05.041 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 23:25:05.041 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:05.041 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:05.041 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 23:25:05.041 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:05.041 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:05.041 | INFO | Uploaded 36GiB to GPU in 2.0226s (17.7988GiB/s) | 20260505 23:25:05.041 | INFO | Downloaded 34GiB to GPU in 2.045s (16.6259GiB/s) | 20260505 23:25:10.178 | INFO | Analyzing chunk frames = [7134720,7135232) | 20260505 23:25:10.178 | INFO | MemInfo: { | 20260505 23:25:10.178 | INFO | "gpu" : " 17 GB", | 20260505 23:25:10.178 | INFO | "huge" : "", | 20260505 23:25:10.178 | INFO | "huge_shared" : " 27 GB", | 20260505 23:25:10.178 | INFO | "malloc" : " 14 GB", | 20260505 23:25:10.178 | INFO | "pinned" : " 29 GB" | 20260505 23:25:10.178 | INFO | } | 20260505 23:25:10.178 | INFO | Perf report for the last 5.13688 seconds: | 20260505 23:25:10.178 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:25:10.178 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:10.178 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:10.178 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:10.178 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 23:25:10.178 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:10.178 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:10.178 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260505 23:25:10.178 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:10.178 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:10.178 | INFO | Uploaded 36GiB to GPU in 2.12048s (16.9773GiB/s) | 20260505 23:25:10.178 | INFO | Downloaded 34GiB to GPU in 2.01188s (16.8996GiB/s) | 20260505 23:25:10.178 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 23:25:10.178 | INFO | Total mem usage: 15.3 GiB | 20260505 23:25:10.178 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 23:25:10.178 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.26%) | 20260505 23:25:10.178 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 23:25:10.178 | INFO | High Water Breakdown: | 20260505 23:25:10.178 | INFO | 61% : 8.64 GiB ( 90.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 23:25:10.178 | INFO | 38.9% : 5.5 GiB ( 82.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 23:25:10.178 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 23:25:10.179 | INFO | ------------------------------------------ | 20260505 23:25:10.179 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:25:10.179 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 23:25:10.179 | INFO | Total mem usage: 26.5 GiB | 20260505 23:25:10.179 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 23:25:10.179 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 23:25:10.179 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 23:25:10.179 | INFO | High Water Breakdown: | 20260505 23:25:10.179 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:25:10.179 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:25:10.179 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:25:10.179 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:25:10.179 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:25:10.179 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:25:10.179 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 23:25:10.179 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 23:25:10.179 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:25:10.179 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 23:25:10.179 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 23:25:10.179 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 23:25:10.179 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 23:25:10.179 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 23:25:10.179 | INFO | ------------------------------------------ | 20260505 23:25:10.179 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:25:10.179 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 23:25:10.179 | INFO | Total mem usage: 25.6 GiB | 20260505 23:25:10.179 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 23:25:10.179 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 23:25:10.179 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 23:25:10.179 | INFO | High Water Breakdown: | 20260505 23:25:10.179 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 23:25:10.179 | INFO | ------------------------------------------ | 20260505 23:25:10.179 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:25:10.179 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 23:25:10.179 | INFO | Total mem usage: 15.7 GiB | 20260505 23:25:10.179 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 23:25:10.179 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 23:25:10.179 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 23:25:10.179 | INFO | High Water Breakdown: | 20260505 23:25:10.179 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 23:25:10.179 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 23:25:10.179 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 23:25:10.179 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 23:25:10.179 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 23:25:10.179 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 23:25:10.179 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 23:25:10.179 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 23:25:10.179 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 23:25:10.179 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:25:10.179 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 23:25:10.179 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:25:10.179 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 23:25:10.179 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 23:25:10.179 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 23:25:10.179 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 23:25:10.179 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:25:10.179 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 23:25:10.179 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:25:10.179 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:25:10.179 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:25:10.179 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 23:25:10.179 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 23:25:10.179 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:25:10.179 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 23:25:10.179 | INFO | ------------------------------------------ | 20260505 23:25:10.179 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:25:10.179 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 23:25:10.179 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:25:10.179 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:25:10.179 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.41e+01, 1.35e+01 / 8.12e-01 | 20260505 23:25:10.179 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.03e-01, 1.08e-02 / 2.57e-03 | 20260505 23:25:10.179 | INFO | PerfReport: 11.85%, Baseline -- 3.72e+00 / 4.51e+00, 5.79e-01 / 2.10e-01 | 20260505 23:25:10.179 | INFO | PerfReport: 3.29%, Histogram -- 1.03e+00 / 1.54e+00, 1.93e-01 / 7.97e-02 | 20260505 23:25:10.179 | INFO | PerfReport: 16.00%, DME -- 4.02e+01 / 4.42e+01, 3.92e+00 / 9.32e-01 | 20260505 23:25:10.179 | INFO | PerfReport: 57.26%, FrameLabeling -- 1.80e+01 / 1.93e+01, 1.51e+00 / 1.08e+00 | 20260505 23:25:10.179 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.23e+00 / 2.77e+00, 4.24e-01 / 2.39e-01 | 20260505 23:25:10.179 | INFO | PerfReport: 3.11%, Metrics -- 9.77e-01 / 1.27e+00, 2.60e-01 / 1.25e-01 | 20260505 23:25:10.179 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.39e-01 / 3.96e-01, 5.81e-02 / 2.99e-02 | 20260505 23:25:10.179 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 23:25:10.179 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:25:10.179 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:25:10.179 | INFO | PerfReport: Full Execution -- 9.84e+01 / -nan, 2.45e+01 / 0.00e+00 | 20260505 23:25:10.179 | INFO | PerfReport: 22.22%, Upload -- 2.19e+01 / 0.00e+00, 7.38e+00 / 0.00e+00 | 20260505 23:25:10.179 | INFO | PerfReport: 21.79%, Download -- 2.14e+01 / 0.00e+00, 7.55e+00 / 0.00e+00 | 20260505 23:25:10.179 | INFO | PerfReport: 31.90%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 23:25:10.179 | INFO | PerfReport: 2.13%, LockUpload -- 2.09e+00 / 0.00e+00, 7.89e+00 / 0.00e+00 | 20260505 23:25:10.179 | INFO | PerfReport: 1.31%, LockDownload -- 1.29e+00 / 0.00e+00, 2.98e+00 / 0.00e+00 | 20260505 23:25:10.179 | INFO | PerfReport: 20.64%, LockCompute -- 2.03e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260505 23:25:10.179 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.59e-02 / 0.00e+00 | 20260505 23:25:10.299 | INFO | Starting a round of Baz IO | 20260505 23:25:13.453 | INFO | Populating RTMetrics report | 20260505 23:25:13.453 | INFO | RTMetrics report populated | 20260505 23:25:13.453 | INFO | Opening tmp RTMetrics json file | 20260505 23:25:13.456 | INFO | Serializing RTMetrics to json | 20260505 23:25:13.459 | INFO | Serialization done, swapping tmp file now... | 20260505 23:25:13.459 | INFO | RTMetrics done | 20260505 23:25:13.532 | INFO | Finished a round of Baz IO: Wrote 1.897 GiB in 3.233 seconds | 20260505 23:25:13.532 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.43% / 0.00% / 78.57% | 20260505 23:25:13.532 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.107936% / 0% / 0.000103315% | 20260505 23:25:13.532 | INFO | Baz IO overall: | 20260505 23:25:13.532 | INFO | BytesWritten : 4081693373455 | 20260505 23:25:13.532 | INFO | NumEvents : 1923080055828 | 20260505 23:25:15.320 | INFO | Analyzing chunk frames = [7135232,7135744) | 20260505 23:25:15.320 | INFO | MemInfo: { | 20260505 23:25:15.320 | INFO | "gpu" : " 17 GB", | 20260505 23:25:15.320 | INFO | "huge" : "", | 20260505 23:25:15.320 | INFO | "huge_shared" : " 27 GB", | 20260505 23:25:15.320 | INFO | "malloc" : " 14 GB", | 20260505 23:25:15.320 | INFO | "pinned" : " 29 GB" | 20260505 23:25:15.320 | INFO | } | 20260505 23:25:15.320 | INFO | Perf report for the last 5.14191 seconds: | 20260505 23:25:15.320 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:25:15.320 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:15.320 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:15.320 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:15.320 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260505 23:25:15.320 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:15.320 | INFO | RT_METRICS : 43.4%, 1.0, 22.8ms, 0 | 20260505 23:25:15.320 | INFO | PRE_HQ : 59.9%, 1.7, 54.2ms, 0 | 20260505 23:25:15.320 | INFO | BAZWRITER : 62.0%, 3.6, 113.9ms, 0 | 20260505 23:25:15.320 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 23:25:15.320 | INFO | Uploaded 36GiB to GPU in 2.1723s (16.5723GiB/s) | 20260505 23:25:15.320 | INFO | Downloaded 38GiB to GPU in 2.23633s (16.9921GiB/s) | 20260505 23:25:20.459 | INFO | Analyzing chunk frames = [7135744,7136256) | 20260505 23:25:20.459 | INFO | MemInfo: { | 20260505 23:25:20.459 | INFO | "gpu" : " 17 GB", | 20260505 23:25:20.459 | INFO | "huge" : "", | 20260505 23:25:20.459 | INFO | "huge_shared" : " 27 GB", | 20260505 23:25:20.459 | INFO | "malloc" : " 14 GB", | 20260505 23:25:20.459 | INFO | "pinned" : " 29 GB" | 20260505 23:25:20.459 | INFO | } | 20260505 23:25:20.459 | INFO | Perf report for the last 5.13895 seconds: | 20260505 23:25:20.459 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:25:20.459 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:20.459 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:20.459 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:20.459 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 23:25:20.459 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:20.459 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:20.459 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:25:20.459 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:20.459 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:20.459 | INFO | Uploaded 36GiB to GPU in 2.08443s (17.2709GiB/s) | 20260505 23:25:20.459 | INFO | Downloaded 34GiB to GPU in 2.02745s (16.7699GiB/s) | 20260505 23:25:25.597 | INFO | Analyzing chunk frames = [7136256,7136768) | 20260505 23:25:25.597 | INFO | MemInfo: { | 20260505 23:25:25.597 | INFO | "gpu" : " 17 GB", | 20260505 23:25:25.598 | INFO | "huge" : "", | 20260505 23:25:25.598 | INFO | "huge_shared" : " 27 GB", | 20260505 23:25:25.598 | INFO | "malloc" : " 14 GB", | 20260505 23:25:25.598 | INFO | "pinned" : " 29 GB" | 20260505 23:25:25.598 | INFO | } | 20260505 23:25:25.598 | INFO | Perf report for the last 5.13865 seconds: | 20260505 23:25:25.598 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:25:25.598 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:25.598 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:25.598 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:25.598 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260505 23:25:25.598 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:25.598 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:25.598 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:25:25.598 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:25.598 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:25.598 | INFO | Uploaded 36GiB to GPU in 2.05928s (17.4818GiB/s) | 20260505 23:25:25.598 | INFO | Downloaded 34GiB to GPU in 2.01547s (16.8695GiB/s) | 20260505 23:25:30.736 | INFO | Analyzing chunk frames = [7136768,7137280) | 20260505 23:25:30.736 | INFO | MemInfo: { | 20260505 23:25:30.736 | INFO | "gpu" : " 17 GB", | 20260505 23:25:30.736 | INFO | "huge" : "", | 20260505 23:25:30.736 | INFO | "huge_shared" : " 27 GB", | 20260505 23:25:30.736 | INFO | "malloc" : " 14 GB", | 20260505 23:25:30.736 | INFO | "pinned" : " 29 GB" | 20260505 23:25:30.736 | INFO | } | 20260505 23:25:30.736 | INFO | Perf report for the last 5.13862 seconds: | 20260505 23:25:30.736 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:25:30.736 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:30.736 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:30.736 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:30.736 | INFO | ANALYSIS : 61.8%, 3.0, 95.9ms, 0 | 20260505 23:25:30.736 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:30.736 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:30.736 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 23:25:30.736 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:30.736 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:30.736 | INFO | Uploaded 36GiB to GPU in 2.14741s (16.7644GiB/s) | 20260505 23:25:30.737 | INFO | Downloaded 34GiB to GPU in 2.01819s (16.8468GiB/s) | 20260505 23:25:35.871 | INFO | Analyzing chunk frames = [7137280,7137792) | 20260505 23:25:35.871 | INFO | MemInfo: { | 20260505 23:25:35.871 | INFO | "gpu" : " 17 GB", | 20260505 23:25:35.871 | INFO | "huge" : "", | 20260505 23:25:35.871 | INFO | "huge_shared" : " 27 GB", | 20260505 23:25:35.871 | INFO | "malloc" : " 14 GB", | 20260505 23:25:35.871 | INFO | "pinned" : " 29 GB" | 20260505 23:25:35.871 | INFO | } | 20260505 23:25:35.871 | INFO | Perf report for the last 5.13459 seconds: | 20260505 23:25:35.871 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:25:35.871 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:35.871 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:35.871 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:35.871 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260505 23:25:35.871 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:35.871 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:35.871 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 23:25:35.871 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:35.871 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:35.871 | INFO | Uploaded 36GiB to GPU in 2.09666s (17.1702GiB/s) | 20260505 23:25:35.871 | INFO | Downloaded 34GiB to GPU in 2.04141s (16.6552GiB/s) | 20260505 23:25:41.010 | INFO | Analyzing chunk frames = [7137792,7138304) | 20260505 23:25:41.010 | INFO | MemInfo: { | 20260505 23:25:41.010 | INFO | "gpu" : " 17 GB", | 20260505 23:25:41.010 | INFO | "huge" : "", | 20260505 23:25:41.010 | INFO | "huge_shared" : " 27 GB", | 20260505 23:25:41.010 | INFO | "malloc" : " 14 GB", | 20260505 23:25:41.010 | INFO | "pinned" : " 29 GB" | 20260505 23:25:41.010 | INFO | } | 20260505 23:25:41.010 | INFO | Perf report for the last 5.13919 seconds: | 20260505 23:25:41.010 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:25:41.010 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:41.010 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:41.010 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:41.010 | INFO | ANALYSIS : 61.6%, 3.0, 96.2ms, 0 | 20260505 23:25:41.010 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:41.010 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:41.010 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:25:41.010 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:41.010 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:41.010 | INFO | Uploaded 36GiB to GPU in 2.08487s (17.2673GiB/s) | 20260505 23:25:41.010 | INFO | Downloaded 34GiB to GPU in 2.0362s (16.6978GiB/s) | 20260505 23:25:46.156 | INFO | Analyzing chunk frames = [7138304,7138816) | 20260505 23:25:46.156 | INFO | MemInfo: { | 20260505 23:25:46.156 | INFO | "gpu" : " 17 GB", | 20260505 23:25:46.156 | INFO | "huge" : "", | 20260505 23:25:46.156 | INFO | "huge_shared" : " 27 GB", | 20260505 23:25:46.156 | INFO | "malloc" : " 14 GB", | 20260505 23:25:46.156 | INFO | "pinned" : " 29 GB" | 20260505 23:25:46.156 | INFO | } | 20260505 23:25:46.156 | INFO | Perf report for the last 5.14581 seconds: | 20260505 23:25:46.156 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:25:46.156 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:46.156 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:46.156 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:46.156 | INFO | ANALYSIS : 62.0%, 3.0, 97.0ms, 0 | 20260505 23:25:46.156 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:46.156 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:46.156 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 23:25:46.156 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:46.156 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:46.156 | INFO | Uploaded 36GiB to GPU in 2.01411s (17.8739GiB/s) | 20260505 23:25:46.156 | INFO | Downloaded 34GiB to GPU in 2.03001s (16.7487GiB/s) | 20260505 23:25:51.297 | INFO | Analyzing chunk frames = [7138816,7139328) | 20260505 23:25:51.297 | INFO | MemInfo: { | 20260505 23:25:51.297 | INFO | "gpu" : " 17 GB", | 20260505 23:25:51.297 | INFO | "huge" : "", | 20260505 23:25:51.297 | INFO | "huge_shared" : " 27 GB", | 20260505 23:25:51.297 | INFO | "malloc" : " 14 GB", | 20260505 23:25:51.297 | INFO | "pinned" : " 29 GB" | 20260505 23:25:51.297 | INFO | } | 20260505 23:25:51.297 | INFO | Perf report for the last 5.14134 seconds: | 20260505 23:25:51.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:25:51.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:51.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:51.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:51.297 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 23:25:51.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:51.297 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:51.297 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:25:51.297 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:51.297 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:51.297 | INFO | Uploaded 36GiB to GPU in 2.13943s (16.8269GiB/s) | 20260505 23:25:51.297 | INFO | Downloaded 34GiB to GPU in 2.02153s (16.819GiB/s) | 20260505 23:25:51.415 | INFO | Starting a round of Baz IO | 20260505 23:25:54.563 | INFO | Populating RTMetrics report | 20260505 23:25:54.563 | INFO | RTMetrics report populated | 20260505 23:25:54.563 | INFO | Opening tmp RTMetrics json file | 20260505 23:25:54.564 | INFO | Serializing RTMetrics to json | 20260505 23:25:54.566 | INFO | Serialization done, swapping tmp file now... | 20260505 23:25:54.566 | INFO | RTMetrics done | 20260505 23:25:54.644 | INFO | Finished a round of Baz IO: Wrote 1.896 GiB in 3.228 seconds | 20260505 23:25:54.644 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.43% / 0.00% / 78.57% | 20260505 23:25:54.644 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.108143% / 0% / 8.74201e-05% | 20260505 23:25:54.644 | INFO | Baz IO overall: | 20260505 23:25:54.644 | INFO | BytesWritten : 4083728978115 | 20260505 23:25:54.644 | INFO | NumEvents : 1924014360945 | 20260505 23:25:56.430 | INFO | Analyzing chunk frames = [7139328,7139840) | 20260505 23:25:56.430 | INFO | MemInfo: { | 20260505 23:25:56.430 | INFO | "gpu" : " 17 GB", | 20260505 23:25:56.430 | INFO | "huge" : "", | 20260505 23:25:56.430 | INFO | "huge_shared" : " 27 GB", | 20260505 23:25:56.430 | INFO | "malloc" : " 14 GB", | 20260505 23:25:56.430 | INFO | "pinned" : " 29 GB" | 20260505 23:25:56.430 | INFO | } | 20260505 23:25:56.430 | INFO | Perf report for the last 5.13257 seconds: | 20260505 23:25:56.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:25:56.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:56.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:56.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:56.430 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 23:25:56.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:25:56.430 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260505 23:25:56.430 | INFO | PRE_HQ : 59.6%, 1.8, 54.1ms, 0 | 20260505 23:25:56.430 | INFO | BAZWRITER : 62.0%, 3.5, 112.8ms, 0 | 20260505 23:25:56.430 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 23:25:56.430 | INFO | Uploaded 36GiB to GPU in 2.1467s (16.7699GiB/s) | 20260505 23:25:56.430 | INFO | Downloaded 38GiB to GPU in 2.23557s (16.9979GiB/s) | 20260505 23:26:01.569 | INFO | Analyzing chunk frames = [7139840,7140352) | 20260505 23:26:01.569 | INFO | MemInfo: { | 20260505 23:26:01.569 | INFO | "gpu" : " 17 GB", | 20260505 23:26:01.569 | INFO | "huge" : "", | 20260505 23:26:01.569 | INFO | "huge_shared" : " 27 GB", | 20260505 23:26:01.569 | INFO | "malloc" : " 14 GB", | 20260505 23:26:01.569 | INFO | "pinned" : " 29 GB" | 20260505 23:26:01.569 | INFO | } | 20260505 23:26:01.569 | INFO | Perf report for the last 5.13929 seconds: | 20260505 23:26:01.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:26:01.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:01.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:01.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:01.569 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 23:26:01.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:01.569 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:01.569 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:26:01.569 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:01.569 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:01.569 | INFO | Uploaded 36GiB to GPU in 2.08089s (17.3003GiB/s) | 20260505 23:26:01.569 | INFO | Downloaded 34GiB to GPU in 2.022s (16.815GiB/s) | 20260505 23:26:06.711 | INFO | Analyzing chunk frames = [7140352,7140864) | 20260505 23:26:06.711 | INFO | MemInfo: { | 20260505 23:26:06.711 | INFO | "gpu" : " 17 GB", | 20260505 23:26:06.711 | INFO | "huge" : "", | 20260505 23:26:06.711 | INFO | "huge_shared" : " 27 GB", | 20260505 23:26:06.711 | INFO | "malloc" : " 14 GB", | 20260505 23:26:06.711 | INFO | "pinned" : " 29 GB" | 20260505 23:26:06.711 | INFO | } | 20260505 23:26:06.711 | INFO | Perf report for the last 5.14182 seconds: | 20260505 23:26:06.711 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:26:06.711 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:06.711 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:06.711 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:06.711 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260505 23:26:06.711 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:06.711 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:06.711 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:26:06.711 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:06.711 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:06.711 | INFO | Uploaded 36GiB to GPU in 2.14652s (16.7713GiB/s) | 20260505 23:26:06.711 | INFO | Downloaded 34GiB to GPU in 2.02491s (16.7909GiB/s) | 20260505 23:26:11.855 | INFO | Analyzing chunk frames = [7140864,7141376) | 20260505 23:26:11.855 | INFO | MemInfo: { | 20260505 23:26:11.855 | INFO | "gpu" : " 17 GB", | 20260505 23:26:11.855 | INFO | "huge" : "", | 20260505 23:26:11.855 | INFO | "huge_shared" : " 27 GB", | 20260505 23:26:11.855 | INFO | "malloc" : " 14 GB", | 20260505 23:26:11.855 | INFO | "pinned" : " 29 GB" | 20260505 23:26:11.855 | INFO | } | 20260505 23:26:11.855 | INFO | Perf report for the last 5.14418 seconds: | 20260505 23:26:11.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:26:11.855 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:11.855 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:11.855 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:11.855 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260505 23:26:11.855 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:11.855 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:11.855 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:26:11.855 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:11.855 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:11.855 | INFO | Uploaded 36GiB to GPU in 2.14204s (16.8064GiB/s) | 20260505 23:26:11.855 | INFO | Downloaded 34GiB to GPU in 2.01123s (16.9051GiB/s) | 20260505 23:26:16.999 | INFO | Analyzing chunk frames = [7141376,7141888) | 20260505 23:26:16.999 | INFO | MemInfo: { | 20260505 23:26:16.999 | INFO | "gpu" : " 17 GB", | 20260505 23:26:16.999 | INFO | "huge" : "", | 20260505 23:26:16.999 | INFO | "huge_shared" : " 27 GB", | 20260505 23:26:16.999 | INFO | "malloc" : " 14 GB", | 20260505 23:26:16.999 | INFO | "pinned" : " 29 GB" | 20260505 23:26:16.999 | INFO | } | 20260505 23:26:16.999 | INFO | Perf report for the last 5.1439 seconds: | 20260505 23:26:16.999 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:26:16.999 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:16.999 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:16.999 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:16.999 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260505 23:26:16.999 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:16.999 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:16.999 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:26:16.999 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:16.999 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:16.999 | INFO | Uploaded 36GiB to GPU in 2.09829s (17.1568GiB/s) | 20260505 23:26:16.999 | INFO | Downloaded 34GiB to GPU in 2.0498s (16.587GiB/s) | 20260505 23:26:22.130 | INFO | Analyzing chunk frames = [7141888,7142400) | 20260505 23:26:22.130 | INFO | MemInfo: { | 20260505 23:26:22.130 | INFO | "gpu" : " 17 GB", | 20260505 23:26:22.130 | INFO | "huge" : "", | 20260505 23:26:22.130 | INFO | "huge_shared" : " 27 GB", | 20260505 23:26:22.130 | INFO | "malloc" : " 14 GB", | 20260505 23:26:22.130 | INFO | "pinned" : " 29 GB" | 20260505 23:26:22.130 | INFO | } | 20260505 23:26:22.130 | INFO | Perf report for the last 5.13082 seconds: | 20260505 23:26:22.130 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:26:22.130 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:22.130 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:22.130 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:22.130 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260505 23:26:22.130 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:22.130 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:22.130 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:26:22.130 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:22.130 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:22.130 | INFO | Uploaded 36GiB to GPU in 2.06077s (17.4692GiB/s) | 20260505 23:26:22.130 | INFO | Downloaded 34GiB to GPU in 2.03035s (16.7459GiB/s) | 20260505 23:26:27.271 | INFO | Analyzing chunk frames = [7142400,7142912) | 20260505 23:26:27.271 | INFO | MemInfo: { | 20260505 23:26:27.271 | INFO | "gpu" : " 17 GB", | 20260505 23:26:27.271 | INFO | "huge" : "", | 20260505 23:26:27.271 | INFO | "huge_shared" : " 27 GB", | 20260505 23:26:27.271 | INFO | "malloc" : " 14 GB", | 20260505 23:26:27.271 | INFO | "pinned" : " 29 GB" | 20260505 23:26:27.271 | INFO | } | 20260505 23:26:27.271 | INFO | Perf report for the last 5.14122 seconds: | 20260505 23:26:27.271 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:26:27.271 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:27.271 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:27.271 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:27.271 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260505 23:26:27.271 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:27.271 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:27.271 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 23:26:27.271 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:27.271 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:27.271 | INFO | Uploaded 36GiB to GPU in 2.13821s (16.8365GiB/s) | 20260505 23:26:27.271 | INFO | Downloaded 34GiB to GPU in 2.07831s (16.3595GiB/s) | 20260505 23:26:32.411 | INFO | Analyzing chunk frames = [7142912,7143424) | 20260505 23:26:32.411 | INFO | MemInfo: { | 20260505 23:26:32.411 | INFO | "gpu" : " 17 GB", | 20260505 23:26:32.411 | INFO | "huge" : "", | 20260505 23:26:32.411 | INFO | "huge_shared" : " 27 GB", | 20260505 23:26:32.411 | INFO | "malloc" : " 14 GB", | 20260505 23:26:32.411 | INFO | "pinned" : " 29 GB" | 20260505 23:26:32.411 | INFO | } | 20260505 23:26:32.411 | INFO | Perf report for the last 5.13976 seconds: | 20260505 23:26:32.411 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:26:32.411 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:32.411 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:32.411 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:32.411 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 23:26:32.411 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:32.411 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:32.411 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 23:26:32.411 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:32.411 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:32.411 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 23:26:32.411 | INFO | Total mem usage: 15.3 GiB | 20260505 23:26:32.411 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 23:26:32.411 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -9.08%) | 20260505 23:26:32.411 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 23:26:32.411 | INFO | High Water Breakdown: | 20260505 23:26:32.411 | INFO | 60.9% : 8.56 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 23:26:32.411 | INFO | 39% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 23:26:32.411 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 23:26:32.411 | INFO | ------------------------------------------ | 20260505 23:26:32.411 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:26:32.411 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 23:26:32.411 | INFO | Total mem usage: 26.5 GiB | 20260505 23:26:32.411 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 23:26:32.411 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 23:26:32.411 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260505 23:26:32.411 | INFO | High Water Breakdown: | 20260505 23:26:32.411 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:26:32.411 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:26:32.411 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:26:32.411 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:26:32.411 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:26:32.411 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:26:32.411 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 23:26:32.411 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 23:26:32.411 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:26:32.411 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 23:26:32.411 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 23:26:32.411 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 23:26:32.411 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 23:26:32.411 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 23:26:32.411 | INFO | ------------------------------------------ | 20260505 23:26:32.411 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:26:32.411 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 23:26:32.411 | INFO | Total mem usage: 25.6 GiB | 20260505 23:26:32.411 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 23:26:32.411 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 23:26:32.411 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 23:26:32.411 | INFO | High Water Breakdown: | 20260505 23:26:32.411 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 23:26:32.411 | INFO | ------------------------------------------ | 20260505 23:26:32.411 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:26:32.411 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 23:26:32.411 | INFO | Total mem usage: 15.7 GiB | 20260505 23:26:32.411 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 23:26:32.411 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 23:26:32.411 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 23:26:32.411 | INFO | High Water Breakdown: | 20260505 23:26:32.411 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 23:26:32.411 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 23:26:32.411 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 23:26:32.411 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 23:26:32.411 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 23:26:32.411 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 23:26:32.411 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 23:26:32.411 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 23:26:32.411 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 23:26:32.412 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:26:32.411 | INFO | Uploaded 36GiB to GPU in 2.13622s (16.8522GiB/s) | 20260505 23:26:32.412 | INFO | Downloaded 34GiB to GPU in 2.02466s (16.7929GiB/s) | 20260505 23:26:32.412 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 23:26:32.412 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:26:32.412 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 23:26:32.412 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 23:26:32.412 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 23:26:32.412 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 23:26:32.412 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:26:32.412 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 23:26:32.412 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:26:32.412 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:26:32.412 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:26:32.412 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 23:26:32.412 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 23:26:32.412 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:26:32.412 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 23:26:32.412 | INFO | ------------------------------------------ | 20260505 23:26:32.412 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:26:32.412 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 23:26:32.412 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:26:32.412 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:26:32.412 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.40e+01, 1.34e+01 / 6.69e-01 | 20260505 23:26:32.412 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.04e-01, 1.04e-02 / 3.69e-03 | 20260505 23:26:32.412 | INFO | PerfReport: 11.84%, Baseline -- 3.71e+00 / 4.51e+00, 5.77e-01 / 2.16e-01 | 20260505 23:26:32.412 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.54e+00, 1.93e-01 / 7.00e-02 | 20260505 23:26:32.412 | INFO | PerfReport: 15.99%, DME -- 4.01e+01 / 4.40e+01, 3.75e+00 / 9.82e-01 | 20260505 23:26:32.412 | INFO | PerfReport: 57.27%, FrameLabeling -- 1.79e+01 / 1.93e+01, 1.50e+00 / 8.50e-01 | 20260505 23:26:32.412 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.23e+00 / 2.79e+00, 4.30e-01 / 2.12e-01 | 20260505 23:26:32.412 | INFO | PerfReport: 3.12%, Metrics -- 9.78e-01 / 1.29e+00, 2.66e-01 / 1.30e-01 | 20260505 23:26:32.412 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.38e-01 / 4.02e-01, 5.81e-02 / 3.64e-02 | 20260505 23:26:32.412 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 23:26:32.412 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:26:32.412 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:26:32.412 | INFO | PerfReport: Full Execution -- 9.85e+01 / -nan, 2.49e+01 / 0.00e+00 | 20260505 23:26:32.412 | INFO | PerfReport: 22.32%, Upload -- 2.20e+01 / 0.00e+00, 7.53e+00 / 0.00e+00 | 20260505 23:26:32.412 | INFO | PerfReport: 21.76%, Download -- 2.14e+01 / 0.00e+00, 7.61e+00 / 0.00e+00 | 20260505 23:26:32.412 | INFO | PerfReport: 31.83%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 23:26:32.412 | INFO | PerfReport: 2.20%, LockUpload -- 2.16e+00 / 0.00e+00, 8.00e+00 / 0.00e+00 | 20260505 23:26:32.412 | INFO | PerfReport: 1.30%, LockDownload -- 1.28e+00 / 0.00e+00, 2.89e+00 / 0.00e+00 | 20260505 23:26:32.412 | INFO | PerfReport: 20.58%, LockCompute -- 2.03e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260505 23:26:32.412 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.77e-02 / 0.00e+00 | 20260505 23:26:32.533 | INFO | Starting a round of Baz IO | 20260505 23:26:35.683 | INFO | Populating RTMetrics report | 20260505 23:26:35.683 | INFO | RTMetrics report populated | 20260505 23:26:35.683 | INFO | Opening tmp RTMetrics json file | 20260505 23:26:35.685 | INFO | Serializing RTMetrics to json | 20260505 23:26:35.688 | INFO | Serialization done, swapping tmp file now... | 20260505 23:26:35.688 | INFO | RTMetrics done | 20260505 23:26:35.767 | INFO | Finished a round of Baz IO: Wrote 1.895 GiB in 3.234 seconds | 20260505 23:26:35.767 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.43% / 0.00% / 78.57% | 20260505 23:26:35.767 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.107682% / 0% / 4.76837e-05% | 20260505 23:26:35.767 | INFO | Baz IO overall: | 20260505 23:26:35.767 | INFO | BytesWritten : 4085763671100 | 20260505 23:26:35.767 | INFO | NumEvents : 1924947697134 | 20260505 23:26:37.548 | INFO | Analyzing chunk frames = [7143424,7143936) | 20260505 23:26:37.548 | INFO | MemInfo: { | 20260505 23:26:37.548 | INFO | "gpu" : " 17 GB", | 20260505 23:26:37.548 | INFO | "huge" : "", | 20260505 23:26:37.548 | INFO | "huge_shared" : " 27 GB", | 20260505 23:26:37.548 | INFO | "malloc" : " 14 GB", | 20260505 23:26:37.548 | INFO | "pinned" : " 29 GB" | 20260505 23:26:37.548 | INFO | } | 20260505 23:26:37.548 | INFO | Perf report for the last 5.13691 seconds: | 20260505 23:26:37.548 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:26:37.548 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:37.548 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:37.548 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:37.548 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260505 23:26:37.548 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:37.548 | INFO | RT_METRICS : 43.4%, 1.0, 22.7ms, 0 | 20260505 23:26:37.548 | INFO | PRE_HQ : 59.8%, 1.7, 54.3ms, 0 | 20260505 23:26:37.548 | INFO | BAZWRITER : 62.1%, 3.5, 113.2ms, 0 | 20260505 23:26:37.548 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 23:26:37.548 | INFO | Uploaded 36GiB to GPU in 2.12898s (16.9095GiB/s) | 20260505 23:26:37.548 | INFO | Downloaded 38GiB to GPU in 2.23365s (17.0125GiB/s) | 20260505 23:26:42.685 | INFO | Analyzing chunk frames = [7143936,7144448) | 20260505 23:26:42.685 | INFO | MemInfo: { | 20260505 23:26:42.685 | INFO | "gpu" : " 17 GB", | 20260505 23:26:42.685 | INFO | "huge" : "", | 20260505 23:26:42.685 | INFO | "huge_shared" : " 27 GB", | 20260505 23:26:42.685 | INFO | "malloc" : " 14 GB", | 20260505 23:26:42.685 | INFO | "pinned" : " 29 GB" | 20260505 23:26:42.685 | INFO | } | 20260505 23:26:42.685 | INFO | Perf report for the last 5.13741 seconds: | 20260505 23:26:42.685 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:26:42.685 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:42.685 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:42.685 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:42.685 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 23:26:42.685 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:42.685 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:42.685 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:26:42.685 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:42.685 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:42.685 | INFO | Uploaded 36GiB to GPU in 2.08422s (17.2727GiB/s) | 20260505 23:26:42.685 | INFO | Downloaded 34GiB to GPU in 2.02868s (16.7597GiB/s) | 20260505 23:26:47.823 | INFO | Analyzing chunk frames = [7144448,7144960) | 20260505 23:26:47.823 | INFO | MemInfo: { | 20260505 23:26:47.823 | INFO | "gpu" : " 17 GB", | 20260505 23:26:47.823 | INFO | "huge" : "", | 20260505 23:26:47.823 | INFO | "huge_shared" : " 27 GB", | 20260505 23:26:47.823 | INFO | "malloc" : " 14 GB", | 20260505 23:26:47.823 | INFO | "pinned" : " 29 GB" | 20260505 23:26:47.823 | INFO | } | 20260505 23:26:47.823 | INFO | Perf report for the last 5.13786 seconds: | 20260505 23:26:47.823 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:26:47.823 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:47.823 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:47.823 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:47.823 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260505 23:26:47.823 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:47.823 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:47.823 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:26:47.823 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:47.823 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:47.823 | INFO | Uploaded 36GiB to GPU in 2.13937s (16.8274GiB/s) | 20260505 23:26:47.823 | INFO | Downloaded 34GiB to GPU in 2.0198s (16.8333GiB/s) | 20260505 23:26:52.967 | INFO | Analyzing chunk frames = [7144960,7145472) | 20260505 23:26:52.967 | INFO | MemInfo: { | 20260505 23:26:52.967 | INFO | "gpu" : " 17 GB", | 20260505 23:26:52.967 | INFO | "huge" : "", | 20260505 23:26:52.967 | INFO | "huge_shared" : " 27 GB", | 20260505 23:26:52.967 | INFO | "malloc" : " 14 GB", | 20260505 23:26:52.967 | INFO | "pinned" : " 29 GB" | 20260505 23:26:52.967 | INFO | } | 20260505 23:26:52.968 | INFO | Perf report for the last 5.14443 seconds: | 20260505 23:26:52.968 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:26:52.968 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:52.968 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:52.968 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:52.968 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260505 23:26:52.968 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:52.968 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:52.968 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 23:26:52.968 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:52.968 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:52.968 | INFO | Uploaded 36GiB to GPU in 2.13881s (16.8318GiB/s) | 20260505 23:26:52.968 | INFO | Downloaded 34GiB to GPU in 2.01493s (16.874GiB/s) | 20260505 23:26:58.107 | INFO | Analyzing chunk frames = [7145472,7145984) | 20260505 23:26:58.107 | INFO | MemInfo: { | 20260505 23:26:58.107 | INFO | "gpu" : " 17 GB", | 20260505 23:26:58.107 | INFO | "huge" : "", | 20260505 23:26:58.107 | INFO | "huge_shared" : " 27 GB", | 20260505 23:26:58.107 | INFO | "malloc" : " 14 GB", | 20260505 23:26:58.107 | INFO | "pinned" : " 29 GB" | 20260505 23:26:58.107 | INFO | } | 20260505 23:26:58.107 | INFO | Perf report for the last 5.13965 seconds: | 20260505 23:26:58.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:26:58.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:58.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:58.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:58.107 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260505 23:26:58.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:58.107 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:58.107 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:26:58.107 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:58.107 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:26:58.107 | INFO | Uploaded 36GiB to GPU in 2.08525s (17.2641GiB/s) | 20260505 23:26:58.107 | INFO | Downloaded 34GiB to GPU in 2.04177s (16.6522GiB/s) | 20260505 23:27:03.244 | INFO | Analyzing chunk frames = [7145984,7146496) | 20260505 23:27:03.244 | INFO | MemInfo: { | 20260505 23:27:03.244 | INFO | "gpu" : " 17 GB", | 20260505 23:27:03.244 | INFO | "huge" : "", | 20260505 23:27:03.244 | INFO | "huge_shared" : " 27 GB", | 20260505 23:27:03.244 | INFO | "malloc" : " 14 GB", | 20260505 23:27:03.244 | INFO | "pinned" : " 29 GB" | 20260505 23:27:03.244 | INFO | } | 20260505 23:27:03.244 | INFO | Perf report for the last 5.13661 seconds: | 20260505 23:27:03.244 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:27:03.244 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:03.244 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:03.244 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:03.244 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260505 23:27:03.244 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:03.244 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:03.244 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 23:27:03.244 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:03.244 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:03.244 | INFO | Uploaded 36GiB to GPU in 2.07574s (17.3432GiB/s) | 20260505 23:27:03.244 | INFO | Downloaded 34GiB to GPU in 2.04564s (16.6207GiB/s) | 20260505 23:27:08.380 | INFO | Analyzing chunk frames = [7146496,7147008) | 20260505 23:27:08.380 | INFO | MemInfo: { | 20260505 23:27:08.380 | INFO | "gpu" : " 17 GB", | 20260505 23:27:08.380 | INFO | "huge" : "", | 20260505 23:27:08.380 | INFO | "huge_shared" : " 27 GB", | 20260505 23:27:08.380 | INFO | "malloc" : " 14 GB", | 20260505 23:27:08.380 | INFO | "pinned" : " 29 GB" | 20260505 23:27:08.380 | INFO | } | 20260505 23:27:08.380 | INFO | Perf report for the last 5.13587 seconds: | 20260505 23:27:08.380 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:27:08.380 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:08.380 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:08.380 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:08.380 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 23:27:08.380 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:08.380 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:08.380 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 23:27:08.380 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:08.380 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:08.380 | INFO | Uploaded 36GiB to GPU in 2.10191s (17.1273GiB/s) | 20260505 23:27:08.380 | INFO | Downloaded 34GiB to GPU in 2.06361s (16.476GiB/s) | 20260505 23:27:13.521 | INFO | Analyzing chunk frames = [7147008,7147520) | 20260505 23:27:13.521 | INFO | MemInfo: { | 20260505 23:27:13.521 | INFO | "gpu" : " 17 GB", | 20260505 23:27:13.521 | INFO | "huge" : "", | 20260505 23:27:13.521 | INFO | "huge_shared" : " 27 GB", | 20260505 23:27:13.521 | INFO | "malloc" : " 14 GB", | 20260505 23:27:13.521 | INFO | "pinned" : " 29 GB" | 20260505 23:27:13.521 | INFO | } | 20260505 23:27:13.521 | INFO | Perf report for the last 5.14154 seconds: | 20260505 23:27:13.521 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:27:13.521 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:13.521 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:13.521 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:13.521 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 23:27:13.521 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:13.521 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:13.521 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 23:27:13.521 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:13.521 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:13.522 | INFO | Uploaded 36GiB to GPU in 2.11187s (17.0465GiB/s) | 20260505 23:27:13.522 | INFO | Downloaded 34GiB to GPU in 2.00681s (16.9423GiB/s) | 20260505 23:27:13.649 | INFO | Starting a round of Baz IO | 20260505 23:27:16.796 | INFO | Populating RTMetrics report | 20260505 23:27:16.796 | INFO | RTMetrics report populated | 20260505 23:27:16.796 | INFO | Opening tmp RTMetrics json file | 20260505 23:27:16.797 | INFO | Serializing RTMetrics to json | 20260505 23:27:16.798 | INFO | Serialization done, swapping tmp file now... | 20260505 23:27:16.798 | INFO | RTMetrics done | 20260505 23:27:16.875 | INFO | Finished a round of Baz IO: Wrote 1.893 GiB in 3.225 seconds | 20260505 23:27:16.875 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.43% / 0.00% / 78.57% | 20260505 23:27:16.875 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.107849% / 0% / 7.54992e-05% | 20260505 23:27:16.875 | INFO | Baz IO overall: | 20260505 23:27:16.875 | INFO | BytesWritten : 4087796608788 | 20260505 23:27:16.875 | INFO | NumEvents : 1925879935469 | 20260505 23:27:18.661 | INFO | Analyzing chunk frames = [7147520,7148032) | 20260505 23:27:18.661 | INFO | MemInfo: { | 20260505 23:27:18.661 | INFO | "gpu" : " 17 GB", | 20260505 23:27:18.661 | INFO | "huge" : "", | 20260505 23:27:18.661 | INFO | "huge_shared" : " 27 GB", | 20260505 23:27:18.661 | INFO | "malloc" : " 14 GB", | 20260505 23:27:18.661 | INFO | "pinned" : " 29 GB" | 20260505 23:27:18.661 | INFO | } | 20260505 23:27:18.661 | INFO | Perf report for the last 5.14003 seconds: | 20260505 23:27:18.661 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:27:18.661 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:18.661 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:18.661 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:18.661 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 23:27:18.661 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:18.661 | INFO | RT_METRICS : 43.4%, 1.0, 22.8ms, 0 | 20260505 23:27:18.661 | INFO | PRE_HQ : 59.7%, 1.8, 54.9ms, 0 | 20260505 23:27:18.661 | INFO | BAZWRITER : 62.0%, 3.5, 113.0ms, 0 | 20260505 23:27:18.661 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 23:27:18.662 | INFO | Uploaded 36GiB to GPU in 2.13301s (16.8775GiB/s) | 20260505 23:27:18.662 | INFO | Downloaded 38GiB to GPU in 2.22209s (17.101GiB/s) | 20260505 23:27:23.808 | INFO | Analyzing chunk frames = [7148032,7148544) | 20260505 23:27:23.808 | INFO | MemInfo: { | 20260505 23:27:23.808 | INFO | "gpu" : " 17 GB", | 20260505 23:27:23.808 | INFO | "huge" : "", | 20260505 23:27:23.808 | INFO | "huge_shared" : " 27 GB", | 20260505 23:27:23.808 | INFO | "malloc" : " 14 GB", | 20260505 23:27:23.808 | INFO | "pinned" : " 29 GB" | 20260505 23:27:23.808 | INFO | } | 20260505 23:27:23.808 | INFO | Perf report for the last 5.1466 seconds: | 20260505 23:27:23.808 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:27:23.808 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:23.808 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:23.808 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:23.808 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260505 23:27:23.808 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:23.808 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:23.808 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:27:23.808 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:23.808 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:23.808 | INFO | Uploaded 36GiB to GPU in 2.10588s (17.095GiB/s) | 20260505 23:27:23.808 | INFO | Downloaded 34GiB to GPU in 2.04052s (16.6624GiB/s) | 20260505 23:27:28.949 | INFO | Analyzing chunk frames = [7148544,7149056) | 20260505 23:27:28.949 | INFO | MemInfo: { | 20260505 23:27:28.949 | INFO | "gpu" : " 17 GB", | 20260505 23:27:28.949 | INFO | "huge" : "", | 20260505 23:27:28.949 | INFO | "huge_shared" : " 27 GB", | 20260505 23:27:28.949 | INFO | "malloc" : " 14 GB", | 20260505 23:27:28.949 | INFO | "pinned" : " 29 GB" | 20260505 23:27:28.949 | INFO | } | 20260505 23:27:28.949 | INFO | Perf report for the last 5.14093 seconds: | 20260505 23:27:28.949 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:27:28.949 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:28.949 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:28.949 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:28.949 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260505 23:27:28.949 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:28.949 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:28.949 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:27:28.949 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:28.949 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:28.949 | INFO | Uploaded 36GiB to GPU in 2.07906s (17.3155GiB/s) | 20260505 23:27:28.949 | INFO | Downloaded 34GiB to GPU in 2.01043s (16.9118GiB/s) | 20260505 23:27:34.077 | INFO | Analyzing chunk frames = [7149056,7149568) | 20260505 23:27:34.077 | INFO | MemInfo: { | 20260505 23:27:34.077 | INFO | "gpu" : " 17 GB", | 20260505 23:27:34.077 | INFO | "huge" : "", | 20260505 23:27:34.077 | INFO | "huge_shared" : " 27 GB", | 20260505 23:27:34.077 | INFO | "malloc" : " 14 GB", | 20260505 23:27:34.077 | INFO | "pinned" : " 29 GB" | 20260505 23:27:34.077 | INFO | } | 20260505 23:27:34.077 | INFO | Perf report for the last 5.12823 seconds: | 20260505 23:27:34.077 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:27:34.077 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:34.077 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:34.077 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:34.077 | INFO | ANALYSIS : 61.8%, 3.0, 95.8ms, 0 | 20260505 23:27:34.077 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:34.077 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:34.077 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 23:27:34.077 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:34.077 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:34.077 | INFO | Uploaded 36GiB to GPU in 2.14041s (16.8192GiB/s) | 20260505 23:27:34.077 | INFO | Downloaded 34GiB to GPU in 2.01662s (16.8599GiB/s) | 20260505 23:27:39.222 | INFO | Analyzing chunk frames = [7149568,7150080) | 20260505 23:27:39.222 | INFO | MemInfo: { | 20260505 23:27:39.222 | INFO | "gpu" : " 17 GB", | 20260505 23:27:39.222 | INFO | "huge" : "", | 20260505 23:27:39.222 | INFO | "huge_shared" : " 27 GB", | 20260505 23:27:39.222 | INFO | "malloc" : " 14 GB", | 20260505 23:27:39.222 | INFO | "pinned" : " 29 GB" | 20260505 23:27:39.222 | INFO | } | 20260505 23:27:39.222 | INFO | Perf report for the last 5.14491 seconds: | 20260505 23:27:39.222 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:27:39.222 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:39.222 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:39.222 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:39.222 | INFO | ANALYSIS : 61.1%, 3.0, 95.5ms, 0 | 20260505 23:27:39.222 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:39.222 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:39.222 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 23:27:39.222 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:39.222 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:39.222 | INFO | Uploaded 36GiB to GPU in 2.09314s (17.1991GiB/s) | 20260505 23:27:39.222 | INFO | Downloaded 34GiB to GPU in 2.04253s (16.646GiB/s) | 20260505 23:27:44.361 | INFO | Analyzing chunk frames = [7150080,7150592) | 20260505 23:27:44.361 | INFO | MemInfo: { | 20260505 23:27:44.361 | INFO | "gpu" : " 17 GB", | 20260505 23:27:44.361 | INFO | "huge" : "", | 20260505 23:27:44.361 | INFO | "huge_shared" : " 27 GB", | 20260505 23:27:44.361 | INFO | "malloc" : " 14 GB", | 20260505 23:27:44.361 | INFO | "pinned" : " 29 GB" | 20260505 23:27:44.361 | INFO | } | 20260505 23:27:44.361 | INFO | Perf report for the last 5.13903 seconds: | 20260505 23:27:44.361 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:27:44.361 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:44.361 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:44.361 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:44.361 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260505 23:27:44.361 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:44.361 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:44.361 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:27:44.361 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:44.361 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:44.361 | INFO | Uploaded 36GiB to GPU in 2.0676s (17.4115GiB/s) | 20260505 23:27:44.361 | INFO | Downloaded 34GiB to GPU in 2.04021s (16.6649GiB/s) | 20260505 23:27:49.496 | INFO | Analyzing chunk frames = [7150592,7151104) | 20260505 23:27:49.496 | INFO | MemInfo: { | 20260505 23:27:49.496 | INFO | "gpu" : " 17 GB", | 20260505 23:27:49.496 | INFO | "huge" : "", | 20260505 23:27:49.496 | INFO | "huge_shared" : " 27 GB", | 20260505 23:27:49.497 | INFO | "malloc" : " 14 GB", | 20260505 23:27:49.497 | INFO | "pinned" : " 29 GB" | 20260505 23:27:49.497 | INFO | } | 20260505 23:27:49.497 | INFO | Perf report for the last 5.1355 seconds: | 20260505 23:27:49.497 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:27:49.497 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:49.497 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:49.497 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:49.497 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260505 23:27:49.497 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:49.497 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:49.497 | INFO | PRE_HQ : 27.5%, 1.0, 14.2ms, 0 | 20260505 23:27:49.497 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:49.497 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:49.497 | INFO | Uploaded 36GiB to GPU in 2.01136s (17.8984GiB/s) | 20260505 23:27:49.497 | INFO | Downloaded 34GiB to GPU in 2.0398s (16.6683GiB/s) | 20260505 23:27:54.636 | INFO | Analyzing chunk frames = [7151104,7151616) | 20260505 23:27:54.636 | INFO | MemInfo: { | 20260505 23:27:54.636 | INFO | "gpu" : " 17 GB", | 20260505 23:27:54.636 | INFO | "huge" : "", | 20260505 23:27:54.636 | INFO | "huge_shared" : " 27 GB", | 20260505 23:27:54.636 | INFO | "malloc" : " 14 GB", | 20260505 23:27:54.636 | INFO | "pinned" : " 29 GB" | 20260505 23:27:54.636 | INFO | } | 20260505 23:27:54.636 | INFO | Perf report for the last 5.13927 seconds: | 20260505 23:27:54.636 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:27:54.636 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:54.636 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:54.636 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:54.636 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 23:27:54.636 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:54.636 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:54.636 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:27:54.636 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:54.636 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:54.636 | INFO | Uploaded 36GiB to GPU in 2.10581s (17.0956GiB/s) | 20260505 23:27:54.636 | INFO | Downloaded 34GiB to GPU in 2.00124s (16.9894GiB/s) | 20260505 23:27:54.636 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 23:27:54.636 | INFO | Total mem usage: 15.3 GiB | 20260505 23:27:54.636 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 23:27:54.636 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.28%) | 20260505 23:27:54.636 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 23:27:54.636 | INFO | High Water Breakdown: | 20260505 23:27:54.636 | INFO | 61% : 8.63 GiB ( 90.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 23:27:54.636 | INFO | 38.9% : 5.5 GiB ( 82.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 23:27:54.636 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 23:27:54.636 | INFO | ------------------------------------------ | 20260505 23:27:54.636 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:27:54.636 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 23:27:54.636 | INFO | Total mem usage: 26.5 GiB | 20260505 23:27:54.636 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 23:27:54.636 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 23:27:54.636 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 23:27:54.636 | INFO | High Water Breakdown: | 20260505 23:27:54.636 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:27:54.636 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:27:54.636 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:27:54.636 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:27:54.636 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:27:54.636 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:27:54.636 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 23:27:54.636 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 23:27:54.636 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:27:54.636 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 23:27:54.636 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 23:27:54.636 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 23:27:54.636 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 23:27:54.636 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 23:27:54.636 | INFO | ------------------------------------------ | 20260505 23:27:54.637 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:27:54.637 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 23:27:54.637 | INFO | Total mem usage: 25.6 GiB | 20260505 23:27:54.637 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 23:27:54.637 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 23:27:54.637 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 23:27:54.637 | INFO | High Water Breakdown: | 20260505 23:27:54.637 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 23:27:54.637 | INFO | ------------------------------------------ | 20260505 23:27:54.637 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:27:54.637 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 23:27:54.637 | INFO | Total mem usage: 15.7 GiB | 20260505 23:27:54.637 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 23:27:54.637 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 23:27:54.637 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 23:27:54.637 | INFO | High Water Breakdown: | 20260505 23:27:54.637 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 23:27:54.637 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 23:27:54.637 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 23:27:54.637 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 23:27:54.637 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 23:27:54.637 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 23:27:54.637 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 23:27:54.637 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 23:27:54.637 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 23:27:54.637 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:27:54.637 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 23:27:54.637 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:27:54.637 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 23:27:54.637 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 23:27:54.637 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 23:27:54.637 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 23:27:54.637 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:27:54.637 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 23:27:54.637 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:27:54.637 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:27:54.637 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:27:54.637 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 23:27:54.637 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 23:27:54.637 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:27:54.637 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 23:27:54.637 | INFO | ------------------------------------------ | 20260505 23:27:54.637 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:27:54.637 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 23:27:54.637 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:27:54.637 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:27:54.637 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.40e+01, 1.36e+01 / 7.93e-01 | 20260505 23:27:54.637 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.04e-01, 1.06e-02 / 4.27e-03 | 20260505 23:27:54.637 | INFO | PerfReport: 11.85%, Baseline -- 3.72e+00 / 4.51e+00, 5.72e-01 / 2.20e-01 | 20260505 23:27:54.637 | INFO | PerfReport: 3.29%, Histogram -- 1.03e+00 / 1.54e+00, 1.93e-01 / 8.17e-02 | 20260505 23:27:54.637 | INFO | PerfReport: 16.00%, DME -- 4.02e+01 / 4.42e+01, 3.94e+00 / 1.07e+00 | 20260505 23:27:54.637 | INFO | PerfReport: 57.23%, FrameLabeling -- 1.80e+01 / 1.92e+01, 1.46e+00 / 7.94e-01 | 20260505 23:27:54.637 | INFO | PerfReport: 7.13%, PulseAccumulating -- 2.24e+00 / 2.81e+00, 4.22e-01 / 1.71e-01 | 20260505 23:27:54.637 | INFO | PerfReport: 3.12%, Metrics -- 9.78e-01 / 1.27e+00, 2.64e-01 / 1.19e-01 | 20260505 23:27:54.637 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.39e-01 / 3.97e-01, 5.80e-02 / 2.98e-02 | 20260505 23:27:54.637 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 23:27:54.637 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:27:54.637 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:27:54.637 | INFO | PerfReport: Full Execution -- 9.84e+01 / -nan, 2.47e+01 / 0.00e+00 | 20260505 23:27:54.637 | INFO | PerfReport: 22.24%, Upload -- 2.19e+01 / 0.00e+00, 7.28e+00 / 0.00e+00 | 20260505 23:27:54.637 | INFO | PerfReport: 21.75%, Download -- 2.14e+01 / 0.00e+00, 7.47e+00 / 0.00e+00 | 20260505 23:27:54.637 | INFO | PerfReport: 31.89%, Compute -- 3.14e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 23:27:54.637 | INFO | PerfReport: 2.15%, LockUpload -- 2.12e+00 / 0.00e+00, 7.89e+00 / 0.00e+00 | 20260505 23:27:54.637 | INFO | PerfReport: 1.32%, LockDownload -- 1.29e+00 / 0.00e+00, 2.94e+00 / 0.00e+00 | 20260505 23:27:54.637 | INFO | PerfReport: 20.64%, LockCompute -- 2.03e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260505 23:27:54.637 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 2.07e-02 / 0.00e+00 | 20260505 23:27:54.765 | INFO | Starting a round of Baz IO | 20260505 23:27:57.909 | INFO | Populating RTMetrics report | 20260505 23:27:57.910 | INFO | RTMetrics report populated | 20260505 23:27:57.910 | INFO | Opening tmp RTMetrics json file | 20260505 23:27:57.911 | INFO | Serializing RTMetrics to json | 20260505 23:27:57.912 | INFO | Serialization done, swapping tmp file now... | 20260505 23:27:57.913 | INFO | RTMetrics done | 20260505 23:27:57.990 | INFO | Finished a round of Baz IO: Wrote 1.894 GiB in 3.224 seconds | 20260505 23:27:57.990 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.43% / 0.00% / 78.57% | 20260505 23:27:57.990 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.106839% / 0% / 7.94729e-05% | 20260505 23:27:57.990 | INFO | Baz IO overall: | 20260505 23:27:57.990 | INFO | BytesWritten : 4089829955869 | 20260505 23:27:57.990 | INFO | NumEvents : 1926812610586 | 20260505 23:27:59.780 | INFO | Analyzing chunk frames = [7151616,7152128) | 20260505 23:27:59.780 | INFO | MemInfo: { | 20260505 23:27:59.780 | INFO | "gpu" : " 17 GB", | 20260505 23:27:59.780 | INFO | "huge" : "", | 20260505 23:27:59.780 | INFO | "huge_shared" : " 27 GB", | 20260505 23:27:59.780 | INFO | "malloc" : " 14 GB", | 20260505 23:27:59.780 | INFO | "pinned" : " 29 GB" | 20260505 23:27:59.780 | INFO | } | 20260505 23:27:59.780 | INFO | Perf report for the last 5.14439 seconds: | 20260505 23:27:59.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:27:59.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:59.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:59.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:59.780 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260505 23:27:59.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:27:59.780 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260505 23:27:59.780 | INFO | PRE_HQ : 59.7%, 1.7, 54.2ms, 0 | 20260505 23:27:59.780 | INFO | BAZWRITER : 62.0%, 3.6, 114.0ms, 0 | 20260505 23:27:59.780 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 23:27:59.780 | INFO | Uploaded 36GiB to GPU in 2.15777s (16.6839GiB/s) | 20260505 23:27:59.781 | INFO | Downloaded 38GiB to GPU in 2.24037s (16.9615GiB/s) | 20260505 23:28:04.918 | INFO | Analyzing chunk frames = [7152128,7152640) | 20260505 23:28:04.918 | INFO | MemInfo: { | 20260505 23:28:04.918 | INFO | "gpu" : " 17 GB", | 20260505 23:28:04.918 | INFO | "huge" : "", | 20260505 23:28:04.918 | INFO | "huge_shared" : " 27 GB", | 20260505 23:28:04.918 | INFO | "malloc" : " 14 GB", | 20260505 23:28:04.918 | INFO | "pinned" : " 29 GB" | 20260505 23:28:04.918 | INFO | } | 20260505 23:28:04.918 | INFO | Perf report for the last 5.13805 seconds: | 20260505 23:28:04.918 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:28:04.918 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:04.918 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:04.918 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:04.918 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260505 23:28:04.918 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:04.918 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:04.918 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260505 23:28:04.918 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:04.918 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:04.919 | INFO | Uploaded 36GiB to GPU in 2.10271s (17.1208GiB/s) | 20260505 23:28:04.919 | INFO | Downloaded 34GiB to GPU in 2.03803s (16.6827GiB/s) | 20260505 23:28:10.059 | INFO | Analyzing chunk frames = [7152640,7153152) | 20260505 23:28:10.059 | INFO | MemInfo: { | 20260505 23:28:10.059 | INFO | "gpu" : " 17 GB", | 20260505 23:28:10.059 | INFO | "huge" : "", | 20260505 23:28:10.059 | INFO | "huge_shared" : " 27 GB", | 20260505 23:28:10.059 | INFO | "malloc" : " 14 GB", | 20260505 23:28:10.059 | INFO | "pinned" : " 29 GB" | 20260505 23:28:10.059 | INFO | } | 20260505 23:28:10.059 | INFO | Perf report for the last 5.14031 seconds: | 20260505 23:28:10.059 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:28:10.059 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:10.059 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:10.059 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:10.059 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260505 23:28:10.059 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:10.059 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:10.059 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:28:10.059 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:10.059 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:10.059 | INFO | Uploaded 36GiB to GPU in 2.06814s (17.4069GiB/s) | 20260505 23:28:10.059 | INFO | Downloaded 34GiB to GPU in 2.01559s (16.8685GiB/s) | 20260505 23:28:15.191 | INFO | Analyzing chunk frames = [7153152,7153664) | 20260505 23:28:15.191 | INFO | MemInfo: { | 20260505 23:28:15.191 | INFO | "gpu" : " 17 GB", | 20260505 23:28:15.191 | INFO | "huge" : "", | 20260505 23:28:15.191 | INFO | "huge_shared" : " 27 GB", | 20260505 23:28:15.191 | INFO | "malloc" : " 14 GB", | 20260505 23:28:15.191 | INFO | "pinned" : " 29 GB" | 20260505 23:28:15.191 | INFO | } | 20260505 23:28:15.191 | INFO | Perf report for the last 5.1326 seconds: | 20260505 23:28:15.191 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:28:15.191 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:15.191 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:15.191 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:15.191 | INFO | ANALYSIS : 61.8%, 3.0, 95.9ms, 0 | 20260505 23:28:15.191 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:15.191 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:15.191 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:28:15.191 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:15.191 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:15.191 | INFO | Uploaded 36GiB to GPU in 2.13353s (16.8734GiB/s) | 20260505 23:28:15.192 | INFO | Downloaded 34GiB to GPU in 2.01945s (16.8363GiB/s) | 20260505 23:28:20.328 | INFO | Analyzing chunk frames = [7153664,7154176) | 20260505 23:28:20.328 | INFO | MemInfo: { | 20260505 23:28:20.328 | INFO | "gpu" : " 17 GB", | 20260505 23:28:20.329 | INFO | "huge" : "", | 20260505 23:28:20.329 | INFO | "huge_shared" : " 27 GB", | 20260505 23:28:20.329 | INFO | "malloc" : " 14 GB", | 20260505 23:28:20.329 | INFO | "pinned" : " 29 GB" | 20260505 23:28:20.329 | INFO | } | 20260505 23:28:20.329 | INFO | Perf report for the last 5.13743 seconds: | 20260505 23:28:20.329 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:28:20.329 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:20.329 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:20.329 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:20.329 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260505 23:28:20.329 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:20.329 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:20.329 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260505 23:28:20.329 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:20.329 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:20.329 | INFO | Uploaded 36GiB to GPU in 2.09606s (17.1751GiB/s) | 20260505 23:28:20.329 | INFO | Downloaded 34GiB to GPU in 2.04845s (16.5979GiB/s) | 20260505 23:28:25.472 | INFO | Analyzing chunk frames = [7154176,7154688) | 20260505 23:28:25.472 | INFO | MemInfo: { | 20260505 23:28:25.472 | INFO | "gpu" : " 17 GB", | 20260505 23:28:25.472 | INFO | "huge" : "", | 20260505 23:28:25.472 | INFO | "huge_shared" : " 27 GB", | 20260505 23:28:25.472 | INFO | "malloc" : " 14 GB", | 20260505 23:28:25.472 | INFO | "pinned" : " 29 GB" | 20260505 23:28:25.472 | INFO | } | 20260505 23:28:25.472 | INFO | Perf report for the last 5.14355 seconds: | 20260505 23:28:25.472 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:28:25.472 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:25.472 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:25.472 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:25.472 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260505 23:28:25.472 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:25.472 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:25.472 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260505 23:28:25.472 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:25.472 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:25.472 | INFO | Uploaded 36GiB to GPU in 2.09326s (17.1981GiB/s) | 20260505 23:28:25.472 | INFO | Downloaded 34GiB to GPU in 2.03373s (16.718GiB/s) | 20260505 23:28:30.618 | INFO | Analyzing chunk frames = [7154688,7155200) | 20260505 23:28:30.618 | INFO | MemInfo: { | 20260505 23:28:30.618 | INFO | "gpu" : " 17 GB", | 20260505 23:28:30.618 | INFO | "huge" : "", | 20260505 23:28:30.618 | INFO | "huge_shared" : " 27 GB", | 20260505 23:28:30.618 | INFO | "malloc" : " 14 GB", | 20260505 23:28:30.618 | INFO | "pinned" : " 29 GB" | 20260505 23:28:30.618 | INFO | } | 20260505 23:28:30.618 | INFO | Perf report for the last 5.14607 seconds: | 20260505 23:28:30.618 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:28:30.618 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:30.618 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:30.618 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:30.618 | INFO | ANALYSIS : 62.3%, 3.0, 97.4ms, 0 | 20260505 23:28:30.618 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:30.618 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:30.618 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 23:28:30.618 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:30.618 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:30.619 | INFO | Uploaded 36GiB to GPU in 2.09189s (17.2093GiB/s) | 20260505 23:28:30.619 | INFO | Downloaded 34GiB to GPU in 2.06586s (16.458GiB/s) | 20260505 23:28:35.756 | INFO | Analyzing chunk frames = [7155200,7155712) | 20260505 23:28:35.756 | INFO | MemInfo: { | 20260505 23:28:35.756 | INFO | "gpu" : " 17 GB", | 20260505 23:28:35.756 | INFO | "huge" : "", | 20260505 23:28:35.756 | INFO | "huge_shared" : " 27 GB", | 20260505 23:28:35.756 | INFO | "malloc" : " 14 GB", | 20260505 23:28:35.756 | INFO | "pinned" : " 29 GB" | 20260505 23:28:35.756 | INFO | } | 20260505 23:28:35.756 | INFO | Perf report for the last 5.13806 seconds: | 20260505 23:28:35.756 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:28:35.756 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:35.756 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:35.756 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:35.756 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260505 23:28:35.756 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:35.756 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:35.756 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:28:35.756 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:35.756 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:35.757 | INFO | Uploaded 36GiB to GPU in 2.13973s (16.8245GiB/s) | 20260505 23:28:35.757 | INFO | Downloaded 34GiB to GPU in 2.0142s (16.8802GiB/s) | 20260505 23:28:35.879 | INFO | Starting a round of Baz IO | 20260505 23:28:39.024 | INFO | Populating RTMetrics report | 20260505 23:28:39.024 | INFO | RTMetrics report populated | 20260505 23:28:39.024 | INFO | Opening tmp RTMetrics json file | 20260505 23:28:39.025 | INFO | Serializing RTMetrics to json | 20260505 23:28:39.026 | INFO | Serialization done, swapping tmp file now... | 20260505 23:28:39.027 | INFO | RTMetrics done | 20260505 23:28:39.102 | INFO | Finished a round of Baz IO: Wrote 1.891 GiB in 3.222 seconds | 20260505 23:28:39.102 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.43% / 0.00% / 78.57% | 20260505 23:28:39.102 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.106815% / 3.97364e-06% / 7.15256e-05% | 20260505 23:28:39.102 | INFO | Baz IO overall: | 20260505 23:28:39.102 | INFO | BytesWritten : 4091860935700 | 20260505 23:28:39.102 | INFO | NumEvents : 1927743748954 | 20260505 23:28:40.891 | INFO | Analyzing chunk frames = [7155712,7156224) | 20260505 23:28:40.892 | INFO | MemInfo: { | 20260505 23:28:40.892 | INFO | "gpu" : " 17 GB", | 20260505 23:28:40.892 | INFO | "huge" : "", | 20260505 23:28:40.892 | INFO | "huge_shared" : " 27 GB", | 20260505 23:28:40.892 | INFO | "malloc" : " 14 GB", | 20260505 23:28:40.892 | INFO | "pinned" : " 29 GB" | 20260505 23:28:40.892 | INFO | } | 20260505 23:28:40.892 | INFO | Perf report for the last 5.13522 seconds: | 20260505 23:28:40.892 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:28:40.892 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:40.892 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:40.892 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:40.892 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 23:28:40.892 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:40.892 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260505 23:28:40.892 | INFO | PRE_HQ : 59.9%, 1.7, 54.1ms, 0 | 20260505 23:28:40.892 | INFO | BAZWRITER : 61.9%, 3.6, 113.3ms, 0 | 20260505 23:28:40.892 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 23:28:40.892 | INFO | Uploaded 36GiB to GPU in 2.16407s (16.6354GiB/s) | 20260505 23:28:40.892 | INFO | Downloaded 38GiB to GPU in 2.23335s (17.0148GiB/s) | 20260505 23:28:46.033 | INFO | Analyzing chunk frames = [7156224,7156736) | 20260505 23:28:46.033 | INFO | MemInfo: { | 20260505 23:28:46.033 | INFO | "gpu" : " 17 GB", | 20260505 23:28:46.033 | INFO | "huge" : "", | 20260505 23:28:46.033 | INFO | "huge_shared" : " 27 GB", | 20260505 23:28:46.033 | INFO | "malloc" : " 14 GB", | 20260505 23:28:46.033 | INFO | "pinned" : " 29 GB" | 20260505 23:28:46.033 | INFO | } | 20260505 23:28:46.033 | INFO | Perf report for the last 5.14173 seconds: | 20260505 23:28:46.033 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:28:46.033 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:46.033 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:46.033 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:46.033 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 23:28:46.033 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:46.033 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:46.033 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260505 23:28:46.033 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:46.033 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:46.034 | INFO | Uploaded 36GiB to GPU in 2.09255s (17.2039GiB/s) | 20260505 23:28:46.034 | INFO | Downloaded 34GiB to GPU in 2.02991s (16.7495GiB/s) | 20260505 23:28:51.170 | INFO | Analyzing chunk frames = [7156736,7157248) | 20260505 23:28:51.170 | INFO | MemInfo: { | 20260505 23:28:51.170 | INFO | "gpu" : " 17 GB", | 20260505 23:28:51.170 | INFO | "huge" : "", | 20260505 23:28:51.170 | INFO | "huge_shared" : " 27 GB", | 20260505 23:28:51.170 | INFO | "malloc" : " 14 GB", | 20260505 23:28:51.170 | INFO | "pinned" : " 29 GB" | 20260505 23:28:51.170 | INFO | } | 20260505 23:28:51.170 | INFO | Perf report for the last 5.13648 seconds: | 20260505 23:28:51.170 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:28:51.170 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:51.170 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:51.170 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:51.170 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260505 23:28:51.170 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:51.170 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:51.170 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:28:51.170 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:51.170 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:51.170 | INFO | Uploaded 36GiB to GPU in 2.08305s (17.2824GiB/s) | 20260505 23:28:51.170 | INFO | Downloaded 34GiB to GPU in 2.0084s (16.9289GiB/s) | 20260505 23:28:56.309 | INFO | Analyzing chunk frames = [7157248,7157760) | 20260505 23:28:56.309 | INFO | MemInfo: { | 20260505 23:28:56.309 | INFO | "gpu" : " 17 GB", | 20260505 23:28:56.309 | INFO | "huge" : "", | 20260505 23:28:56.309 | INFO | "huge_shared" : " 27 GB", | 20260505 23:28:56.309 | INFO | "malloc" : " 14 GB", | 20260505 23:28:56.309 | INFO | "pinned" : " 29 GB" | 20260505 23:28:56.309 | INFO | } | 20260505 23:28:56.309 | INFO | Perf report for the last 5.13927 seconds: | 20260505 23:28:56.309 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:28:56.309 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:56.309 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:56.309 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:56.309 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260505 23:28:56.309 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:56.309 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:56.309 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:28:56.309 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:56.309 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:28:56.309 | INFO | Uploaded 36GiB to GPU in 2.14362s (16.794GiB/s) | 20260505 23:28:56.309 | INFO | Downloaded 34GiB to GPU in 2.02332s (16.8041GiB/s) | 20260505 23:29:01.449 | INFO | Analyzing chunk frames = [7157760,7158272) | 20260505 23:29:01.449 | INFO | MemInfo: { | 20260505 23:29:01.449 | INFO | "gpu" : " 17 GB", | 20260505 23:29:01.449 | INFO | "huge" : "", | 20260505 23:29:01.449 | INFO | "huge_shared" : " 27 GB", | 20260505 23:29:01.449 | INFO | "malloc" : " 14 GB", | 20260505 23:29:01.449 | INFO | "pinned" : " 29 GB" | 20260505 23:29:01.449 | INFO | } | 20260505 23:29:01.449 | INFO | Perf report for the last 5.13998 seconds: | 20260505 23:29:01.449 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:29:01.449 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:01.449 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:01.449 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:01.449 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260505 23:29:01.449 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:01.449 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:01.449 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:29:01.449 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:01.449 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:01.449 | INFO | Uploaded 36GiB to GPU in 2.09291s (17.2009GiB/s) | 20260505 23:29:01.449 | INFO | Downloaded 34GiB to GPU in 2.04593s (16.6184GiB/s) | 20260505 23:29:06.586 | INFO | Analyzing chunk frames = [7158272,7158784) | 20260505 23:29:06.586 | INFO | MemInfo: { | 20260505 23:29:06.586 | INFO | "gpu" : " 17 GB", | 20260505 23:29:06.586 | INFO | "huge" : "", | 20260505 23:29:06.586 | INFO | "huge_shared" : " 27 GB", | 20260505 23:29:06.586 | INFO | "malloc" : " 14 GB", | 20260505 23:29:06.586 | INFO | "pinned" : " 29 GB" | 20260505 23:29:06.586 | INFO | } | 20260505 23:29:06.586 | INFO | Perf report for the last 5.13684 seconds: | 20260505 23:29:06.586 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:29:06.586 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:06.586 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:06.586 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:06.586 | INFO | ANALYSIS : 61.5%, 3.0, 96.0ms, 0 | 20260505 23:29:06.586 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:06.586 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:06.586 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:29:06.586 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:06.586 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:06.586 | INFO | Uploaded 36GiB to GPU in 2.07952s (17.3117GiB/s) | 20260505 23:29:06.586 | INFO | Downloaded 34GiB to GPU in 2.04025s (16.6646GiB/s) | 20260505 23:29:11.736 | INFO | Analyzing chunk frames = [7158784,7159296) | 20260505 23:29:11.736 | INFO | MemInfo: { | 20260505 23:29:11.736 | INFO | "gpu" : " 17 GB", | 20260505 23:29:11.736 | INFO | "huge" : "", | 20260505 23:29:11.736 | INFO | "huge_shared" : " 27 GB", | 20260505 23:29:11.736 | INFO | "malloc" : " 14 GB", | 20260505 23:29:11.736 | INFO | "pinned" : " 29 GB" | 20260505 23:29:11.736 | INFO | } | 20260505 23:29:11.736 | INFO | Perf report for the last 5.14989 seconds: | 20260505 23:29:11.736 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:29:11.736 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:11.736 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:11.736 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:11.736 | INFO | ANALYSIS : 62.4%, 3.0, 97.6ms, 0 | 20260505 23:29:11.736 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:11.736 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:11.736 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 23:29:11.736 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:11.736 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:11.736 | INFO | Uploaded 36GiB to GPU in 2.12184s (16.9664GiB/s) | 20260505 23:29:11.736 | INFO | Downloaded 34GiB to GPU in 2.10697s (16.1369GiB/s) | 20260505 23:29:16.877 | INFO | Analyzing chunk frames = [7159296,7159808) | 20260505 23:29:16.877 | INFO | MemInfo: { | 20260505 23:29:16.877 | INFO | "gpu" : " 17 GB", | 20260505 23:29:16.877 | INFO | "huge" : "", | 20260505 23:29:16.877 | INFO | "huge_shared" : " 27 GB", | 20260505 23:29:16.877 | INFO | "malloc" : " 14 GB", | 20260505 23:29:16.877 | INFO | "pinned" : " 29 GB" | 20260505 23:29:16.877 | INFO | } | 20260505 23:29:16.877 | INFO | Perf report for the last 5.14101 seconds: | 20260505 23:29:16.877 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:29:16.877 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:16.877 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:16.877 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:16.877 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260505 23:29:16.877 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:16.877 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:16.877 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:29:16.877 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:16.877 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:16.877 | INFO | Uploaded 36GiB to GPU in 2.14431s (16.7886GiB/s) | 20260505 23:29:16.877 | INFO | Downloaded 34GiB to GPU in 2.02185s (16.8162GiB/s) | 20260505 23:29:16.877 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 23:29:16.877 | INFO | Total mem usage: 15.3 GiB | 20260505 23:29:16.877 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 23:29:16.877 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -9.08%) | 20260505 23:29:16.877 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 23:29:16.877 | INFO | High Water Breakdown: | 20260505 23:29:16.877 | INFO | 60.9% : 8.55 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 23:29:16.877 | INFO | 39% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 23:29:16.877 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 23:29:16.877 | INFO | ------------------------------------------ | 20260505 23:29:16.877 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:29:16.877 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 23:29:16.877 | INFO | Total mem usage: 26.5 GiB | 20260505 23:29:16.877 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 23:29:16.877 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 23:29:16.877 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 23:29:16.877 | INFO | High Water Breakdown: | 20260505 23:29:16.877 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:29:16.877 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:29:16.877 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:29:16.877 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:29:16.877 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:29:16.877 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:29:16.877 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 23:29:16.877 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 23:29:16.877 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:29:16.877 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 23:29:16.877 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 23:29:16.877 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 23:29:16.878 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 23:29:16.878 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 23:29:16.878 | INFO | ------------------------------------------ | 20260505 23:29:16.878 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:29:16.878 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 23:29:16.878 | INFO | Total mem usage: 25.6 GiB | 20260505 23:29:16.878 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 23:29:16.878 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 23:29:16.878 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 23:29:16.878 | INFO | High Water Breakdown: | 20260505 23:29:16.878 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 23:29:16.878 | INFO | ------------------------------------------ | 20260505 23:29:16.878 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:29:16.878 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 23:29:16.878 | INFO | Total mem usage: 15.7 GiB | 20260505 23:29:16.878 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 23:29:16.878 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 23:29:16.878 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 23:29:16.878 | INFO | High Water Breakdown: | 20260505 23:29:16.878 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 23:29:16.878 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 23:29:16.878 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 23:29:16.878 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 23:29:16.878 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 23:29:16.878 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 23:29:16.878 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 23:29:16.878 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 23:29:16.878 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 23:29:16.878 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:29:16.878 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 23:29:16.878 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:29:16.878 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 23:29:16.878 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 23:29:16.878 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 23:29:16.878 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 23:29:16.878 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:29:16.878 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 23:29:16.878 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:29:16.878 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:29:16.878 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:29:16.878 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 23:29:16.878 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 23:29:16.878 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:29:16.878 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 23:29:16.878 | INFO | ------------------------------------------ | 20260505 23:29:16.878 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:29:16.878 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 23:29:16.878 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:29:16.878 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:29:16.878 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.39e+01, 1.34e+01 / 5.85e-01 | 20260505 23:29:16.878 | INFO | PerfReport: 0.29%, AggregateStats -- 9.18e-02 / 1.04e-01, 1.06e-02 / 5.07e-03 | 20260505 23:29:16.878 | INFO | PerfReport: 11.82%, Baseline -- 3.71e+00 / 4.53e+00, 5.67e-01 / 1.42e-01 | 20260505 23:29:16.878 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.51e+00, 1.87e-01 / 8.45e-02 | 20260505 23:29:16.878 | INFO | PerfReport: 15.97%, DME -- 4.01e+01 / 4.38e+01, 3.61e+00 / 7.76e-01 | 20260505 23:29:16.878 | INFO | PerfReport: 57.33%, FrameLabeling -- 1.80e+01 / 1.96e+01, 1.49e+00 / 1.09e+00 | 20260505 23:29:16.878 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.23e+00 / 2.77e+00, 4.26e-01 / 2.39e-01 | 20260505 23:29:16.878 | INFO | PerfReport: 3.11%, Metrics -- 9.75e-01 / 1.26e+00, 2.63e-01 / 1.03e-01 | 20260505 23:29:16.878 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.38e-01 / 3.97e-01, 5.82e-02 / 3.00e-02 | 20260505 23:29:16.878 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 23:29:16.878 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:29:16.878 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:29:16.878 | INFO | PerfReport: Full Execution -- 9.84e+01 / -nan, 2.46e+01 / 0.00e+00 | 20260505 23:29:16.878 | INFO | PerfReport: 22.36%, Upload -- 2.20e+01 / 0.00e+00, 7.53e+00 / 0.00e+00 | 20260505 23:29:16.878 | INFO | PerfReport: 21.81%, Download -- 2.15e+01 / 0.00e+00, 7.51e+00 / 0.00e+00 | 20260505 23:29:16.878 | INFO | PerfReport: 31.85%, Compute -- 3.14e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 23:29:16.878 | INFO | PerfReport: 2.17%, LockUpload -- 2.13e+00 / 0.00e+00, 7.96e+00 / 0.00e+00 | 20260505 23:29:16.878 | INFO | PerfReport: 1.28%, LockDownload -- 1.26e+00 / 0.00e+00, 2.81e+00 / 0.00e+00 | 20260505 23:29:16.878 | INFO | PerfReport: 20.52%, LockCompute -- 2.02e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260505 23:29:16.878 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.71e-02 / 0.00e+00 | 20260505 23:29:16.995 | INFO | Starting a round of Baz IO | 20260505 23:29:20.146 | INFO | Populating RTMetrics report | 20260505 23:29:20.146 | INFO | RTMetrics report populated | 20260505 23:29:20.146 | INFO | Opening tmp RTMetrics json file | 20260505 23:29:20.147 | INFO | Serializing RTMetrics to json | 20260505 23:29:20.148 | INFO | Serialization done, swapping tmp file now... | 20260505 23:29:20.149 | INFO | RTMetrics done | 20260505 23:29:20.228 | INFO | Finished a round of Baz IO: Wrote 1.891 GiB in 3.233 seconds | 20260505 23:29:20.228 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.43% / 0.00% / 78.57% | 20260505 23:29:20.228 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.106124% / 0% / 8.74201e-05% | 20260505 23:29:20.228 | INFO | Baz IO overall: | 20260505 23:29:20.228 | INFO | BytesWritten : 4093891763881 | 20260505 23:29:20.228 | INFO | NumEvents : 1928675301940 | 20260505 23:29:22.005 | INFO | Analyzing chunk frames = [7159808,7160320) | 20260505 23:29:22.005 | INFO | MemInfo: { | 20260505 23:29:22.005 | INFO | "gpu" : " 17 GB", | 20260505 23:29:22.005 | INFO | "huge" : "", | 20260505 23:29:22.005 | INFO | "huge_shared" : " 27 GB", | 20260505 23:29:22.005 | INFO | "malloc" : " 14 GB", | 20260505 23:29:22.005 | INFO | "pinned" : " 29 GB" | 20260505 23:29:22.005 | INFO | } | 20260505 23:29:22.005 | INFO | Perf report for the last 5.12796 seconds: | 20260505 23:29:22.005 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:29:22.005 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:22.005 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:22.005 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:22.005 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260505 23:29:22.005 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:22.005 | INFO | RT_METRICS : 43.5%, 1.0, 22.7ms, 0 | 20260505 23:29:22.005 | INFO | PRE_HQ : 59.8%, 1.7, 53.9ms, 0 | 20260505 23:29:22.005 | INFO | BAZWRITER : 62.1%, 3.5, 112.7ms, 0 | 20260505 23:29:22.005 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 23:29:22.005 | INFO | Uploaded 36GiB to GPU in 2.17953s (16.5173GiB/s) | 20260505 23:29:22.005 | INFO | Downloaded 38GiB to GPU in 2.23263s (17.0203GiB/s) | 20260505 23:29:27.139 | INFO | Analyzing chunk frames = [7160320,7160832) | 20260505 23:29:27.139 | INFO | MemInfo: { | 20260505 23:29:27.139 | INFO | "gpu" : " 17 GB", | 20260505 23:29:27.139 | INFO | "huge" : "", | 20260505 23:29:27.139 | INFO | "huge_shared" : " 27 GB", | 20260505 23:29:27.139 | INFO | "malloc" : " 14 GB", | 20260505 23:29:27.139 | INFO | "pinned" : " 29 GB" | 20260505 23:29:27.139 | INFO | } | 20260505 23:29:27.139 | INFO | Perf report for the last 5.13382 seconds: | 20260505 23:29:27.139 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:29:27.139 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:27.139 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:27.139 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:27.139 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 23:29:27.139 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:27.139 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:27.139 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260505 23:29:27.139 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:27.139 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:27.139 | INFO | Uploaded 36GiB to GPU in 2.09107s (17.216GiB/s) | 20260505 23:29:27.139 | INFO | Downloaded 34GiB to GPU in 2.02955s (16.7525GiB/s) | 20260505 23:29:32.285 | INFO | Analyzing chunk frames = [7160832,7161344) | 20260505 23:29:32.285 | INFO | MemInfo: { | 20260505 23:29:32.285 | INFO | "gpu" : " 17 GB", | 20260505 23:29:32.285 | INFO | "huge" : "", | 20260505 23:29:32.285 | INFO | "huge_shared" : " 27 GB", | 20260505 23:29:32.285 | INFO | "malloc" : " 14 GB", | 20260505 23:29:32.285 | INFO | "pinned" : " 29 GB" | 20260505 23:29:32.285 | INFO | } | 20260505 23:29:32.285 | INFO | Perf report for the last 5.1466 seconds: | 20260505 23:29:32.285 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:29:32.285 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:32.285 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:32.285 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:32.285 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260505 23:29:32.285 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:32.285 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:32.285 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:29:32.285 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:32.285 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:32.286 | INFO | Uploaded 36GiB to GPU in 2.14105s (16.8142GiB/s) | 20260505 23:29:32.286 | INFO | Downloaded 34GiB to GPU in 2.02232s (16.8124GiB/s) | 20260505 23:29:37.424 | INFO | Analyzing chunk frames = [7161344,7161856) | 20260505 23:29:37.424 | INFO | MemInfo: { | 20260505 23:29:37.424 | INFO | "gpu" : " 17 GB", | 20260505 23:29:37.424 | INFO | "huge" : "", | 20260505 23:29:37.424 | INFO | "huge_shared" : " 27 GB", | 20260505 23:29:37.424 | INFO | "malloc" : " 14 GB", | 20260505 23:29:37.424 | INFO | "pinned" : " 29 GB" | 20260505 23:29:37.424 | INFO | } | 20260505 23:29:37.424 | INFO | Perf report for the last 5.13908 seconds: | 20260505 23:29:37.424 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:29:37.424 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:37.424 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:37.424 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:37.424 | INFO | ANALYSIS : 61.8%, 3.0, 95.9ms, 0 | 20260505 23:29:37.424 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:37.424 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:37.424 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 23:29:37.425 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:37.425 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:37.425 | INFO | Uploaded 36GiB to GPU in 2.14809s (16.7591GiB/s) | 20260505 23:29:37.425 | INFO | Downloaded 34GiB to GPU in 2.00906s (16.9233GiB/s) | 20260505 23:29:42.569 | INFO | Analyzing chunk frames = [7161856,7162368) | 20260505 23:29:42.569 | INFO | MemInfo: { | 20260505 23:29:42.569 | INFO | "gpu" : " 17 GB", | 20260505 23:29:42.569 | INFO | "huge" : "", | 20260505 23:29:42.569 | INFO | "huge_shared" : " 27 GB", | 20260505 23:29:42.569 | INFO | "malloc" : " 14 GB", | 20260505 23:29:42.569 | INFO | "pinned" : " 29 GB" | 20260505 23:29:42.569 | INFO | } | 20260505 23:29:42.569 | INFO | Perf report for the last 5.14465 seconds: | 20260505 23:29:42.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:29:42.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:42.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:42.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:42.569 | INFO | ANALYSIS : 61.0%, 3.0, 95.4ms, 0 | 20260505 23:29:42.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:42.569 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:42.569 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 23:29:42.569 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:42.569 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:42.569 | INFO | Uploaded 36GiB to GPU in 2.09114s (17.2155GiB/s) | 20260505 23:29:42.569 | INFO | Downloaded 34GiB to GPU in 2.04276s (16.6441GiB/s) | 20260505 23:29:47.703 | INFO | Analyzing chunk frames = [7162368,7162880) | 20260505 23:29:47.703 | INFO | MemInfo: { | 20260505 23:29:47.703 | INFO | "gpu" : " 17 GB", | 20260505 23:29:47.703 | INFO | "huge" : "", | 20260505 23:29:47.703 | INFO | "huge_shared" : " 27 GB", | 20260505 23:29:47.703 | INFO | "malloc" : " 14 GB", | 20260505 23:29:47.703 | INFO | "pinned" : " 29 GB" | 20260505 23:29:47.703 | INFO | } | 20260505 23:29:47.703 | INFO | Perf report for the last 5.13428 seconds: | 20260505 23:29:47.703 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:29:47.703 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:47.703 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:47.703 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:47.703 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260505 23:29:47.703 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:47.704 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:47.704 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 23:29:47.704 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:47.704 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:47.704 | INFO | Uploaded 36GiB to GPU in 2.07449s (17.3536GiB/s) | 20260505 23:29:47.704 | INFO | Downloaded 34GiB to GPU in 2.03966s (16.6694GiB/s) | 20260505 23:29:52.838 | INFO | Analyzing chunk frames = [7162880,7163392) | 20260505 23:29:52.838 | INFO | MemInfo: { | 20260505 23:29:52.838 | INFO | "gpu" : " 17 GB", | 20260505 23:29:52.838 | INFO | "huge" : "", | 20260505 23:29:52.838 | INFO | "huge_shared" : " 27 GB", | 20260505 23:29:52.838 | INFO | "malloc" : " 14 GB", | 20260505 23:29:52.839 | INFO | "pinned" : " 29 GB" | 20260505 23:29:52.839 | INFO | } | 20260505 23:29:52.839 | INFO | Perf report for the last 5.13518 seconds: | 20260505 23:29:52.839 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:29:52.839 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:52.839 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:52.839 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:52.839 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260505 23:29:52.839 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:52.839 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:52.839 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 23:29:52.839 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:52.839 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:52.839 | INFO | Uploaded 36GiB to GPU in 2.16188s (16.6522GiB/s) | 20260505 23:29:52.839 | INFO | Downloaded 34GiB to GPU in 2.13284s (15.9412GiB/s) | 20260505 23:29:57.984 | INFO | Analyzing chunk frames = [7163392,7163904) | 20260505 23:29:57.984 | INFO | MemInfo: { | 20260505 23:29:57.984 | INFO | "gpu" : " 17 GB", | 20260505 23:29:57.984 | INFO | "huge" : "", | 20260505 23:29:57.984 | INFO | "huge_shared" : " 27 GB", | 20260505 23:29:57.984 | INFO | "malloc" : " 14 GB", | 20260505 23:29:57.984 | INFO | "pinned" : " 29 GB" | 20260505 23:29:57.984 | INFO | } | 20260505 23:29:57.984 | INFO | Perf report for the last 5.14535 seconds: | 20260505 23:29:57.984 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:29:57.984 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:57.984 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:57.984 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:57.984 | INFO | ANALYSIS : 61.9%, 3.0, 96.8ms, 0 | 20260505 23:29:57.984 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:57.984 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:57.984 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:29:57.984 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:57.984 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:29:57.984 | INFO | Uploaded 36GiB to GPU in 2.1178s (16.9988GiB/s) | 20260505 23:29:57.984 | INFO | Downloaded 34GiB to GPU in 2.00649s (16.945GiB/s) | 20260505 23:29:58.102 | INFO | Starting a round of Baz IO | 20260505 23:30:01.251 | INFO | Populating RTMetrics report | 20260505 23:30:01.251 | INFO | RTMetrics report populated | 20260505 23:30:01.251 | INFO | Opening tmp RTMetrics json file | 20260505 23:30:01.252 | INFO | Serializing RTMetrics to json | 20260505 23:30:01.253 | INFO | Serialization done, swapping tmp file now... | 20260505 23:30:01.253 | INFO | RTMetrics done | 20260505 23:30:01.329 | INFO | Finished a round of Baz IO: Wrote 1.893 GiB in 3.226 seconds | 20260505 23:30:01.329 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.43% / 0.00% / 78.57% | 20260505 23:30:01.329 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.106096% / 0% / 7.94729e-05% | 20260505 23:30:01.329 | INFO | Baz IO overall: | 20260505 23:30:01.329 | INFO | BytesWritten : 4095924233951 | 20260505 23:30:01.329 | INFO | NumEvents : 1929608132869 | 20260505 23:30:03.123 | INFO | Analyzing chunk frames = [7163904,7164416) | 20260505 23:30:03.123 | INFO | MemInfo: { | 20260505 23:30:03.123 | INFO | "gpu" : " 17 GB", | 20260505 23:30:03.123 | INFO | "huge" : "", | 20260505 23:30:03.123 | INFO | "huge_shared" : " 27 GB", | 20260505 23:30:03.123 | INFO | "malloc" : " 14 GB", | 20260505 23:30:03.123 | INFO | "pinned" : " 29 GB" | 20260505 23:30:03.123 | INFO | } | 20260505 23:30:03.123 | INFO | Perf report for the last 5.13928 seconds: | 20260505 23:30:03.123 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:30:03.123 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:03.123 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:03.123 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:03.123 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260505 23:30:03.123 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:03.123 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260505 23:30:03.123 | INFO | PRE_HQ : 59.8%, 1.7, 54.0ms, 0 | 20260505 23:30:03.123 | INFO | BAZWRITER : 61.9%, 3.5, 112.0ms, 0 | 20260505 23:30:03.123 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 23:30:03.124 | INFO | Uploaded 36GiB to GPU in 2.16601s (16.6204GiB/s) | 20260505 23:30:03.124 | INFO | Downloaded 38GiB to GPU in 2.23926s (16.9699GiB/s) | 20260505 23:30:08.264 | INFO | Analyzing chunk frames = [7164416,7164928) | 20260505 23:30:08.264 | INFO | MemInfo: { | 20260505 23:30:08.264 | INFO | "gpu" : " 17 GB", | 20260505 23:30:08.264 | INFO | "huge" : "", | 20260505 23:30:08.264 | INFO | "huge_shared" : " 27 GB", | 20260505 23:30:08.264 | INFO | "malloc" : " 14 GB", | 20260505 23:30:08.264 | INFO | "pinned" : " 29 GB" | 20260505 23:30:08.264 | INFO | } | 20260505 23:30:08.264 | INFO | Perf report for the last 5.14073 seconds: | 20260505 23:30:08.264 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:30:08.264 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:08.264 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:08.264 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:08.264 | INFO | ANALYSIS : 61.6%, 3.0, 96.2ms, 0 | 20260505 23:30:08.264 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:08.264 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:08.264 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260505 23:30:08.264 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:08.264 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:08.264 | INFO | Uploaded 36GiB to GPU in 2.08866s (17.2359GiB/s) | 20260505 23:30:08.264 | INFO | Downloaded 34GiB to GPU in 2.03782s (16.6845GiB/s) | 20260505 23:30:13.396 | INFO | Analyzing chunk frames = [7164928,7165440) | 20260505 23:30:13.396 | INFO | MemInfo: { | 20260505 23:30:13.396 | INFO | "gpu" : " 17 GB", | 20260505 23:30:13.396 | INFO | "huge" : "", | 20260505 23:30:13.396 | INFO | "huge_shared" : " 27 GB", | 20260505 23:30:13.396 | INFO | "malloc" : " 14 GB", | 20260505 23:30:13.396 | INFO | "pinned" : " 29 GB" | 20260505 23:30:13.397 | INFO | } | 20260505 23:30:13.397 | INFO | Perf report for the last 5.13258 seconds: | 20260505 23:30:13.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:30:13.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:13.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:13.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:13.397 | INFO | ANALYSIS : 62.0%, 3.0, 96.3ms, 0 | 20260505 23:30:13.397 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:13.397 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:13.397 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260505 23:30:13.397 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:13.397 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:13.397 | INFO | Uploaded 36GiB to GPU in 2.14808s (16.7592GiB/s) | 20260505 23:30:13.397 | INFO | Downloaded 34GiB to GPU in 2.04819s (16.6001GiB/s) | 20260505 23:30:18.540 | INFO | Analyzing chunk frames = [7165440,7165952) | 20260505 23:30:18.540 | INFO | MemInfo: { | 20260505 23:30:18.540 | INFO | "gpu" : " 17 GB", | 20260505 23:30:18.540 | INFO | "huge" : "", | 20260505 23:30:18.540 | INFO | "huge_shared" : " 27 GB", | 20260505 23:30:18.540 | INFO | "malloc" : " 14 GB", | 20260505 23:30:18.540 | INFO | "pinned" : " 29 GB" | 20260505 23:30:18.540 | INFO | } | 20260505 23:30:18.540 | INFO | Perf report for the last 5.14349 seconds: | 20260505 23:30:18.540 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:30:18.540 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:18.540 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:18.540 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:18.540 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260505 23:30:18.540 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:18.540 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:18.540 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 23:30:18.540 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:18.540 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:18.540 | INFO | Uploaded 36GiB to GPU in 2.12234s (16.9624GiB/s) | 20260505 23:30:18.540 | INFO | Downloaded 34GiB to GPU in 2.01824s (16.8464GiB/s) | 20260505 23:30:23.685 | INFO | Analyzing chunk frames = [7165952,7166464) | 20260505 23:30:23.685 | INFO | MemInfo: { | 20260505 23:30:23.685 | INFO | "gpu" : " 17 GB", | 20260505 23:30:23.685 | INFO | "huge" : "", | 20260505 23:30:23.685 | INFO | "huge_shared" : " 27 GB", | 20260505 23:30:23.685 | INFO | "malloc" : " 14 GB", | 20260505 23:30:23.685 | INFO | "pinned" : " 29 GB" | 20260505 23:30:23.685 | INFO | } | 20260505 23:30:23.685 | INFO | Perf report for the last 5.14469 seconds: | 20260505 23:30:23.685 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:30:23.685 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:23.685 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:23.685 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:23.685 | INFO | ANALYSIS : 61.1%, 3.0, 95.5ms, 0 | 20260505 23:30:23.685 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:23.685 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:23.685 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:30:23.685 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:23.685 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:23.685 | INFO | Uploaded 36GiB to GPU in 2.09605s (17.1752GiB/s) | 20260505 23:30:23.685 | INFO | Downloaded 34GiB to GPU in 2.04615s (16.6166GiB/s) | 20260505 23:30:28.816 | INFO | Analyzing chunk frames = [7166464,7166976) | 20260505 23:30:28.816 | INFO | MemInfo: { | 20260505 23:30:28.816 | INFO | "gpu" : " 17 GB", | 20260505 23:30:28.816 | INFO | "huge" : "", | 20260505 23:30:28.816 | INFO | "huge_shared" : " 27 GB", | 20260505 23:30:28.816 | INFO | "malloc" : " 14 GB", | 20260505 23:30:28.816 | INFO | "pinned" : " 29 GB" | 20260505 23:30:28.816 | INFO | } | 20260505 23:30:28.816 | INFO | Perf report for the last 5.13095 seconds: | 20260505 23:30:28.816 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:30:28.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:28.816 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:28.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:28.816 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260505 23:30:28.816 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:28.816 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:28.816 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 23:30:28.816 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:28.816 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:28.816 | INFO | Uploaded 36GiB to GPU in 2.06789s (17.4091GiB/s) | 20260505 23:30:28.816 | INFO | Downloaded 34GiB to GPU in 2.03887s (16.6759GiB/s) | 20260505 23:30:33.953 | INFO | Analyzing chunk frames = [7166976,7167488) | 20260505 23:30:33.953 | INFO | MemInfo: { | 20260505 23:30:33.953 | INFO | "gpu" : " 17 GB", | 20260505 23:30:33.953 | INFO | "huge" : "", | 20260505 23:30:33.953 | INFO | "huge_shared" : " 27 GB", | 20260505 23:30:33.953 | INFO | "malloc" : " 14 GB", | 20260505 23:30:33.953 | INFO | "pinned" : " 29 GB" | 20260505 23:30:33.953 | INFO | } | 20260505 23:30:33.953 | INFO | Perf report for the last 5.13774 seconds: | 20260505 23:30:33.953 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:30:33.953 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:33.954 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:33.954 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:33.954 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260505 23:30:33.954 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:33.954 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:33.954 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260505 23:30:33.954 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:33.954 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:33.954 | INFO | Uploaded 36GiB to GPU in 2.11794s (16.9976GiB/s) | 20260505 23:30:33.954 | INFO | Downloaded 34GiB to GPU in 2.08995s (16.2684GiB/s) | 20260505 23:30:39.094 | INFO | Analyzing chunk frames = [7167488,7168000) | 20260505 23:30:39.094 | INFO | MemInfo: { | 20260505 23:30:39.094 | INFO | "gpu" : " 17 GB", | 20260505 23:30:39.094 | INFO | "huge" : "", | 20260505 23:30:39.094 | INFO | "huge_shared" : " 27 GB", | 20260505 23:30:39.094 | INFO | "malloc" : " 14 GB", | 20260505 23:30:39.094 | INFO | "pinned" : " 29 GB" | 20260505 23:30:39.094 | INFO | } | 20260505 23:30:39.094 | INFO | Perf report for the last 5.14028 seconds: | 20260505 23:30:39.094 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:30:39.094 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:39.094 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:39.094 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:39.094 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 23:30:39.094 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:39.094 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:39.094 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:30:39.094 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:39.094 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:39.094 | INFO | Uploaded 36GiB to GPU in 2.11198s (17.0456GiB/s) | 20260505 23:30:39.094 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 23:30:39.094 | INFO | Total mem usage: 15.3 GiB | 20260505 23:30:39.094 | INFO | Downloaded 34GiB to GPU in 1.99655s (17.0294GiB/s) | 20260505 23:30:39.094 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 23:30:39.094 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.12%) | 20260505 23:30:39.094 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 23:30:39.094 | INFO | High Water Breakdown: | 20260505 23:30:39.094 | INFO | 60.9% : 8.55 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 23:30:39.094 | INFO | 39% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 23:30:39.094 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 23:30:39.094 | INFO | ------------------------------------------ | 20260505 23:30:39.094 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:30:39.094 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 23:30:39.094 | INFO | Total mem usage: 26.5 GiB | 20260505 23:30:39.094 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 23:30:39.094 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 23:30:39.094 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 23:30:39.094 | INFO | High Water Breakdown: | 20260505 23:30:39.094 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:30:39.094 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:30:39.094 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:30:39.094 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:30:39.094 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:30:39.094 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:30:39.094 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 23:30:39.094 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 23:30:39.094 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:30:39.094 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 23:30:39.094 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 23:30:39.094 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 23:30:39.094 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 23:30:39.094 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 23:30:39.094 | INFO | ------------------------------------------ | 20260505 23:30:39.094 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:30:39.094 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 23:30:39.094 | INFO | Total mem usage: 25.6 GiB | 20260505 23:30:39.094 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 23:30:39.094 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 23:30:39.094 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 23:30:39.094 | INFO | High Water Breakdown: | 20260505 23:30:39.094 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 23:30:39.094 | INFO | ------------------------------------------ | 20260505 23:30:39.094 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:30:39.094 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 23:30:39.094 | INFO | Total mem usage: 15.7 GiB | 20260505 23:30:39.094 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 23:30:39.095 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 23:30:39.095 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 23:30:39.095 | INFO | High Water Breakdown: | 20260505 23:30:39.095 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 23:30:39.095 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 23:30:39.095 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 23:30:39.095 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 23:30:39.095 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 23:30:39.095 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 23:30:39.095 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 23:30:39.095 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 23:30:39.095 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 23:30:39.095 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:30:39.095 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 23:30:39.095 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:30:39.095 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 23:30:39.095 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 23:30:39.095 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 23:30:39.095 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 23:30:39.095 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:30:39.095 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 23:30:39.095 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:30:39.095 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:30:39.095 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:30:39.095 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 23:30:39.095 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 23:30:39.095 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:30:39.095 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 23:30:39.095 | INFO | ------------------------------------------ | 20260505 23:30:39.095 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:30:39.095 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 23:30:39.095 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:30:39.095 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:30:39.095 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.39e+01, 1.33e+01 / 1.16e+00 | 20260505 23:30:39.095 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.04e-01, 1.03e-02 / 4.13e-03 | 20260505 23:30:39.095 | INFO | PerfReport: 11.82%, Baseline -- 3.70e+00 / 4.53e+00, 5.67e-01 / 1.67e-01 | 20260505 23:30:39.095 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.53e+00, 1.88e-01 / 1.02e-01 | 20260505 23:30:39.095 | INFO | PerfReport: 16.00%, DME -- 4.01e+01 / 4.39e+01, 3.82e+00 / 1.23e+00 | 20260505 23:30:39.095 | INFO | PerfReport: 57.28%, FrameLabeling -- 1.79e+01 / 1.94e+01, 1.46e+00 / 1.19e+00 | 20260505 23:30:39.095 | INFO | PerfReport: 7.13%, PulseAccumulating -- 2.23e+00 / 2.75e+00, 4.27e-01 / 2.67e-01 | 20260505 23:30:39.095 | INFO | PerfReport: 3.12%, Metrics -- 9.76e-01 / 1.25e+00, 2.63e-01 / 1.15e-01 | 20260505 23:30:39.095 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.38e-01 / 3.94e-01, 5.83e-02 / 3.63e-02 | 20260505 23:30:39.095 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 23:30:39.095 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:30:39.095 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:30:39.095 | INFO | PerfReport: Full Execution -- 9.85e+01 / -nan, 2.48e+01 / 0.00e+00 | 20260505 23:30:39.095 | INFO | PerfReport: 22.43%, Upload -- 2.21e+01 / 0.00e+00, 7.59e+00 / 0.00e+00 | 20260505 23:30:39.095 | INFO | PerfReport: 21.84%, Download -- 2.15e+01 / 0.00e+00, 7.62e+00 / 0.00e+00 | 20260505 23:30:39.095 | INFO | PerfReport: 31.81%, Compute -- 3.13e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260505 23:30:39.095 | INFO | PerfReport: 2.16%, LockUpload -- 2.13e+00 / 0.00e+00, 7.91e+00 / 0.00e+00 | 20260505 23:30:39.095 | INFO | PerfReport: 1.32%, LockDownload -- 1.30e+00 / 0.00e+00, 2.82e+00 / 0.00e+00 | 20260505 23:30:39.095 | INFO | PerfReport: 20.44%, LockCompute -- 2.01e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260505 23:30:39.095 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.86e-02 / 0.00e+00 | 20260505 23:30:39.218 | INFO | Starting a round of Baz IO | 20260505 23:30:42.368 | INFO | Populating RTMetrics report | 20260505 23:30:42.368 | INFO | RTMetrics report populated | 20260505 23:30:42.368 | INFO | Opening tmp RTMetrics json file | 20260505 23:30:42.369 | INFO | Serializing RTMetrics to json | 20260505 23:30:42.370 | INFO | Serialization done, swapping tmp file now... | 20260505 23:30:42.371 | INFO | RTMetrics done | 20260505 23:30:42.451 | INFO | Finished a round of Baz IO: Wrote 1.893 GiB in 3.232 seconds | 20260505 23:30:42.451 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.43% / 0.00% / 78.57% | 20260505 23:30:42.451 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.106454% / 0% / 5.5631e-05% | 20260505 23:30:42.451 | INFO | Baz IO overall: | 20260505 23:30:42.451 | INFO | BytesWritten : 4097956482416 | 20260505 23:30:42.451 | INFO | NumEvents : 1930540526315 | 20260505 23:30:44.235 | INFO | Analyzing chunk frames = [7168000,7168512) | 20260505 23:30:44.235 | INFO | MemInfo: { | 20260505 23:30:44.235 | INFO | "gpu" : " 17 GB", | 20260505 23:30:44.235 | INFO | "huge" : "", | 20260505 23:30:44.235 | INFO | "huge_shared" : " 27 GB", | 20260505 23:30:44.235 | INFO | "malloc" : " 14 GB", | 20260505 23:30:44.235 | INFO | "pinned" : " 29 GB" | 20260505 23:30:44.235 | INFO | } | 20260505 23:30:44.235 | INFO | Perf report for the last 5.14089 seconds: | 20260505 23:30:44.235 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:30:44.235 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:44.235 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:44.235 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:44.235 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 23:30:44.235 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:44.235 | INFO | RT_METRICS : 43.4%, 1.0, 22.8ms, 0 | 20260505 23:30:44.235 | INFO | PRE_HQ : 59.9%, 1.8, 54.5ms, 0 | 20260505 23:30:44.235 | INFO | BAZWRITER : 62.0%, 3.5, 112.6ms, 0 | 20260505 23:30:44.235 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 23:30:44.235 | INFO | Uploaded 36GiB to GPU in 2.17076s (16.584GiB/s) | 20260505 23:30:44.235 | INFO | Downloaded 38GiB to GPU in 2.23342s (17.0142GiB/s) | 20260505 23:30:49.379 | INFO | Analyzing chunk frames = [7168512,7169024) | 20260505 23:30:49.379 | INFO | MemInfo: { | 20260505 23:30:49.379 | INFO | "gpu" : " 17 GB", | 20260505 23:30:49.379 | INFO | "huge" : "", | 20260505 23:30:49.379 | INFO | "huge_shared" : " 27 GB", | 20260505 23:30:49.379 | INFO | "malloc" : " 14 GB", | 20260505 23:30:49.379 | INFO | "pinned" : " 29 GB" | 20260505 23:30:49.379 | INFO | } | 20260505 23:30:49.379 | INFO | Perf report for the last 5.14461 seconds: | 20260505 23:30:49.379 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:30:49.379 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:49.379 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:49.379 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:49.379 | INFO | ANALYSIS : 61.6%, 3.0, 96.3ms, 0 | 20260505 23:30:49.379 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:49.379 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:49.379 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:30:49.379 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:49.379 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:49.380 | INFO | Uploaded 36GiB to GPU in 2.08888s (17.2341GiB/s) | 20260505 23:30:49.380 | INFO | Downloaded 34GiB to GPU in 2.03565s (16.7023GiB/s) | 20260505 23:30:54.516 | INFO | Analyzing chunk frames = [7169024,7169536) | 20260505 23:30:54.516 | INFO | MemInfo: { | 20260505 23:30:54.516 | INFO | "gpu" : " 17 GB", | 20260505 23:30:54.516 | INFO | "huge" : "", | 20260505 23:30:54.516 | INFO | "huge_shared" : " 27 GB", | 20260505 23:30:54.516 | INFO | "malloc" : " 14 GB", | 20260505 23:30:54.516 | INFO | "pinned" : " 29 GB" | 20260505 23:30:54.516 | INFO | } | 20260505 23:30:54.516 | INFO | Perf report for the last 5.13646 seconds: | 20260505 23:30:54.516 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:30:54.516 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:54.516 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:54.516 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:54.516 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260505 23:30:54.516 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:54.516 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:54.516 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:30:54.516 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:54.516 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:54.516 | INFO | Uploaded 36GiB to GPU in 2.06104s (17.4669GiB/s) | 20260505 23:30:54.516 | INFO | Downloaded 34GiB to GPU in 2.01814s (16.8472GiB/s) | 20260505 23:30:59.650 | INFO | Analyzing chunk frames = [7169536,7170048) | 20260505 23:30:59.650 | INFO | MemInfo: { | 20260505 23:30:59.650 | INFO | "gpu" : " 17 GB", | 20260505 23:30:59.650 | INFO | "huge" : "", | 20260505 23:30:59.650 | INFO | "huge_shared" : " 27 GB", | 20260505 23:30:59.650 | INFO | "malloc" : " 14 GB", | 20260505 23:30:59.650 | INFO | "pinned" : " 29 GB" | 20260505 23:30:59.650 | INFO | } | 20260505 23:30:59.650 | INFO | Perf report for the last 5.13429 seconds: | 20260505 23:30:59.650 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:30:59.650 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:59.650 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:59.650 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:59.650 | INFO | ANALYSIS : 61.8%, 3.0, 95.8ms, 0 | 20260505 23:30:59.650 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:59.650 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:59.650 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:30:59.650 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:59.650 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:30:59.650 | INFO | Uploaded 36GiB to GPU in 2.13501s (16.8617GiB/s) | 20260505 23:30:59.650 | INFO | Downloaded 34GiB to GPU in 2.0072s (16.939GiB/s) | 20260505 23:31:04.793 | INFO | Analyzing chunk frames = [7170048,7170560) | 20260505 23:31:04.793 | INFO | MemInfo: { | 20260505 23:31:04.793 | INFO | "gpu" : " 17 GB", | 20260505 23:31:04.793 | INFO | "huge" : "", | 20260505 23:31:04.793 | INFO | "huge_shared" : " 27 GB", | 20260505 23:31:04.793 | INFO | "malloc" : " 14 GB", | 20260505 23:31:04.793 | INFO | "pinned" : " 29 GB" | 20260505 23:31:04.793 | INFO | } | 20260505 23:31:04.793 | INFO | Perf report for the last 5.14313 seconds: | 20260505 23:31:04.793 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:31:04.793 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:04.793 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:04.793 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:04.793 | INFO | ANALYSIS : 61.5%, 3.0, 96.0ms, 0 | 20260505 23:31:04.793 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:04.793 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:04.793 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:31:04.793 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:04.793 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:04.793 | INFO | Uploaded 36GiB to GPU in 2.13751s (16.8421GiB/s) | 20260505 23:31:04.793 | INFO | Downloaded 34GiB to GPU in 2.06227s (16.4867GiB/s) | 20260505 23:31:09.936 | INFO | Analyzing chunk frames = [7170560,7171072) | 20260505 23:31:09.936 | INFO | MemInfo: { | 20260505 23:31:09.936 | INFO | "gpu" : " 17 GB", | 20260505 23:31:09.936 | INFO | "huge" : "", | 20260505 23:31:09.936 | INFO | "huge_shared" : " 27 GB", | 20260505 23:31:09.936 | INFO | "malloc" : " 14 GB", | 20260505 23:31:09.936 | INFO | "pinned" : " 29 GB" | 20260505 23:31:09.937 | INFO | } | 20260505 23:31:09.937 | INFO | Perf report for the last 5.14331 seconds: | 20260505 23:31:09.937 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:31:09.937 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:09.937 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:09.937 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:09.937 | INFO | ANALYSIS : 61.3%, 3.0, 95.8ms, 0 | 20260505 23:31:09.937 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:09.937 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:09.937 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:31:09.937 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:09.937 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:09.937 | INFO | Uploaded 36GiB to GPU in 2.07302s (17.366GiB/s) | 20260505 23:31:09.937 | INFO | Downloaded 34GiB to GPU in 2.04027s (16.6645GiB/s) | 20260505 23:31:15.077 | INFO | Analyzing chunk frames = [7171072,7171584) | 20260505 23:31:15.077 | INFO | MemInfo: { | 20260505 23:31:15.077 | INFO | "gpu" : " 17 GB", | 20260505 23:31:15.077 | INFO | "huge" : "", | 20260505 23:31:15.077 | INFO | "huge_shared" : " 27 GB", | 20260505 23:31:15.077 | INFO | "malloc" : " 14 GB", | 20260505 23:31:15.077 | INFO | "pinned" : " 29 GB" | 20260505 23:31:15.077 | INFO | } | 20260505 23:31:15.077 | INFO | Perf report for the last 5.14066 seconds: | 20260505 23:31:15.077 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:31:15.077 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:15.077 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:15.077 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:15.077 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 23:31:15.077 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:15.077 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:15.077 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 23:31:15.077 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:15.077 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:15.077 | INFO | Uploaded 36GiB to GPU in 2.02434s (17.7835GiB/s) | 20260505 23:31:15.077 | INFO | Downloaded 34GiB to GPU in 2.04316s (16.6409GiB/s) | 20260505 23:31:20.214 | INFO | Analyzing chunk frames = [7171584,7172096) | 20260505 23:31:20.214 | INFO | MemInfo: { | 20260505 23:31:20.214 | INFO | "gpu" : " 17 GB", | 20260505 23:31:20.214 | INFO | "huge" : "", | 20260505 23:31:20.214 | INFO | "huge_shared" : " 27 GB", | 20260505 23:31:20.214 | INFO | "malloc" : " 14 GB", | 20260505 23:31:20.214 | INFO | "pinned" : " 29 GB" | 20260505 23:31:20.214 | INFO | } | 20260505 23:31:20.214 | INFO | Perf report for the last 5.13651 seconds: | 20260505 23:31:20.214 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:31:20.214 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:20.214 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:20.214 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:20.214 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 23:31:20.214 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:20.214 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:20.214 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:31:20.214 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:20.214 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:20.214 | INFO | Uploaded 36GiB to GPU in 2.11447s (17.0256GiB/s) | 20260505 23:31:20.214 | INFO | Downloaded 34GiB to GPU in 2.00848s (16.9282GiB/s) | 20260505 23:31:20.339 | INFO | Starting a round of Baz IO | 20260505 23:31:23.482 | INFO | Populating RTMetrics report | 20260505 23:31:23.482 | INFO | RTMetrics report populated | 20260505 23:31:23.482 | INFO | Opening tmp RTMetrics json file | 20260505 23:31:23.483 | INFO | Serializing RTMetrics to json | 20260505 23:31:23.484 | INFO | Serialization done, swapping tmp file now... | 20260505 23:31:23.484 | INFO | RTMetrics done | 20260505 23:31:23.561 | INFO | Finished a round of Baz IO: Wrote 1.893 GiB in 3.221 seconds | 20260505 23:31:23.561 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.42% / 0.00% / 78.57% | 20260505 23:31:23.561 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.106514% / 0% / 7.15256e-05% | 20260505 23:31:23.561 | INFO | Baz IO overall: | 20260505 23:31:23.561 | INFO | BytesWritten : 4099989126857 | 20260505 23:31:23.561 | INFO | NumEvents : 1931473341348 | 20260505 23:31:25.349 | INFO | Analyzing chunk frames = [7172096,7172608) | 20260505 23:31:25.349 | INFO | MemInfo: { | 20260505 23:31:25.349 | INFO | "gpu" : " 17 GB", | 20260505 23:31:25.349 | INFO | "huge" : "", | 20260505 23:31:25.349 | INFO | "huge_shared" : " 27 GB", | 20260505 23:31:25.349 | INFO | "malloc" : " 14 GB", | 20260505 23:31:25.349 | INFO | "pinned" : " 29 GB" | 20260505 23:31:25.349 | INFO | } | 20260505 23:31:25.349 | INFO | Perf report for the last 5.13498 seconds: | 20260505 23:31:25.349 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:31:25.349 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:25.349 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:25.349 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:25.349 | INFO | ANALYSIS : 62.6%, 3.0, 96.8ms, 0 | 20260505 23:31:25.349 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:25.349 | INFO | RT_METRICS : 43.4%, 1.0, 22.7ms, 0 | 20260505 23:31:25.349 | INFO | PRE_HQ : 59.9%, 1.7, 53.9ms, 0 | 20260505 23:31:25.349 | INFO | BAZWRITER : 61.9%, 4.1, 129.6ms, 0 | 20260505 23:31:25.349 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 23:31:25.349 | INFO | Uploaded 36GiB to GPU in 2.18091s (16.5069GiB/s) | 20260505 23:31:25.349 | INFO | Downloaded 38GiB to GPU in 2.25397s (16.8592GiB/s) | 20260505 23:31:30.491 | INFO | Analyzing chunk frames = [7172608,7173120) | 20260505 23:31:30.491 | INFO | MemInfo: { | 20260505 23:31:30.491 | INFO | "gpu" : " 17 GB", | 20260505 23:31:30.491 | INFO | "huge" : "", | 20260505 23:31:30.491 | INFO | "huge_shared" : " 27 GB", | 20260505 23:31:30.491 | INFO | "malloc" : " 14 GB", | 20260505 23:31:30.491 | INFO | "pinned" : " 29 GB" | 20260505 23:31:30.491 | INFO | } | 20260505 23:31:30.491 | INFO | Perf report for the last 5.14196 seconds: | 20260505 23:31:30.491 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:31:30.491 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:30.491 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:30.491 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:30.491 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260505 23:31:30.491 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:30.491 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:30.491 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:31:30.491 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:30.491 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:30.491 | INFO | Uploaded 36GiB to GPU in 2.09737s (17.1644GiB/s) | 20260505 23:31:30.491 | INFO | Downloaded 34GiB to GPU in 2.04039s (16.6635GiB/s) | 20260505 23:31:35.635 | INFO | Analyzing chunk frames = [7173120,7173632) | 20260505 23:31:35.635 | INFO | MemInfo: { | 20260505 23:31:35.635 | INFO | "gpu" : " 17 GB", | 20260505 23:31:35.635 | INFO | "huge" : "", | 20260505 23:31:35.635 | INFO | "huge_shared" : " 27 GB", | 20260505 23:31:35.635 | INFO | "malloc" : " 14 GB", | 20260505 23:31:35.635 | INFO | "pinned" : " 29 GB" | 20260505 23:31:35.635 | INFO | } | 20260505 23:31:35.635 | INFO | Perf report for the last 5.14478 seconds: | 20260505 23:31:35.635 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:31:35.635 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:35.636 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:35.636 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:35.636 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260505 23:31:35.636 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:35.636 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:35.636 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:31:35.636 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:35.636 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:35.636 | INFO | Uploaded 36GiB to GPU in 2.07469s (17.352GiB/s) | 20260505 23:31:35.636 | INFO | Downloaded 34GiB to GPU in 2.02274s (16.8089GiB/s) | 20260505 23:31:40.769 | INFO | Analyzing chunk frames = [7173632,7174144) | 20260505 23:31:40.769 | INFO | MemInfo: { | 20260505 23:31:40.769 | INFO | "gpu" : " 17 GB", | 20260505 23:31:40.769 | INFO | "huge" : "", | 20260505 23:31:40.769 | INFO | "huge_shared" : " 27 GB", | 20260505 23:31:40.769 | INFO | "malloc" : " 14 GB", | 20260505 23:31:40.769 | INFO | "pinned" : " 29 GB" | 20260505 23:31:40.769 | INFO | } | 20260505 23:31:40.769 | INFO | Perf report for the last 5.13345 seconds: | 20260505 23:31:40.769 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:31:40.769 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:40.769 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:40.769 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:40.769 | INFO | ANALYSIS : 61.8%, 3.0, 95.9ms, 0 | 20260505 23:31:40.769 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:40.769 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:40.769 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 23:31:40.769 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:40.769 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:40.769 | INFO | Uploaded 36GiB to GPU in 2.1447s (16.7855GiB/s) | 20260505 23:31:40.769 | INFO | Downloaded 34GiB to GPU in 2.02677s (16.7755GiB/s) | 20260505 23:31:45.905 | INFO | Analyzing chunk frames = [7174144,7174656) | 20260505 23:31:45.905 | INFO | MemInfo: { | 20260505 23:31:45.905 | INFO | "gpu" : " 17 GB", | 20260505 23:31:45.905 | INFO | "huge" : "", | 20260505 23:31:45.905 | INFO | "huge_shared" : " 27 GB", | 20260505 23:31:45.905 | INFO | "malloc" : " 14 GB", | 20260505 23:31:45.905 | INFO | "pinned" : " 29 GB" | 20260505 23:31:45.905 | INFO | } | 20260505 23:31:45.905 | INFO | Perf report for the last 5.13646 seconds: | 20260505 23:31:45.905 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:31:45.905 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:45.905 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:45.905 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:45.905 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260505 23:31:45.905 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:45.905 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:45.905 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 23:31:45.906 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:45.906 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:45.906 | INFO | Uploaded 36GiB to GPU in 2.08186s (17.2923GiB/s) | 20260505 23:31:45.906 | INFO | Downloaded 34GiB to GPU in 2.03922s (16.673GiB/s) | 20260505 23:31:51.046 | INFO | Analyzing chunk frames = [7174656,7175168) | 20260505 23:31:51.046 | INFO | MemInfo: { | 20260505 23:31:51.046 | INFO | "gpu" : " 17 GB", | 20260505 23:31:51.046 | INFO | "huge" : "", | 20260505 23:31:51.046 | INFO | "huge_shared" : " 27 GB", | 20260505 23:31:51.046 | INFO | "malloc" : " 14 GB", | 20260505 23:31:51.046 | INFO | "pinned" : " 29 GB" | 20260505 23:31:51.046 | INFO | } | 20260505 23:31:51.046 | INFO | Perf report for the last 5.14037 seconds: | 20260505 23:31:51.046 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:31:51.046 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:51.046 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:51.046 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:51.046 | INFO | ANALYSIS : 61.6%, 3.0, 96.2ms, 0 | 20260505 23:31:51.046 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:51.046 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:51.046 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 23:31:51.046 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:51.046 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:51.046 | INFO | Uploaded 36GiB to GPU in 2.07725s (17.3306GiB/s) | 20260505 23:31:51.046 | INFO | Downloaded 34GiB to GPU in 2.03295s (16.7245GiB/s) | 20260505 23:31:56.191 | INFO | Analyzing chunk frames = [7175168,7175680) | 20260505 23:31:56.191 | INFO | MemInfo: { | 20260505 23:31:56.191 | INFO | "gpu" : " 17 GB", | 20260505 23:31:56.191 | INFO | "huge" : "", | 20260505 23:31:56.191 | INFO | "huge_shared" : " 27 GB", | 20260505 23:31:56.191 | INFO | "malloc" : " 14 GB", | 20260505 23:31:56.191 | INFO | "pinned" : " 29 GB" | 20260505 23:31:56.191 | INFO | } | 20260505 23:31:56.191 | INFO | Perf report for the last 5.14501 seconds: | 20260505 23:31:56.191 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:31:56.191 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:56.191 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:56.191 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:56.191 | INFO | ANALYSIS : 62.5%, 3.0, 97.8ms, 0 | 20260505 23:31:56.191 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:56.191 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:56.191 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 23:31:56.191 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:56.191 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:31:56.191 | INFO | Uploaded 36GiB to GPU in 2.15399s (16.7132GiB/s) | 20260505 23:31:56.191 | INFO | Downloaded 34GiB to GPU in 2.13089s (15.9557GiB/s) | 20260505 23:32:01.326 | INFO | Analyzing chunk frames = [7175680,7176192) | 20260505 23:32:01.326 | INFO | MemInfo: { | 20260505 23:32:01.326 | INFO | "gpu" : " 17 GB", | 20260505 23:32:01.326 | INFO | "huge" : "", | 20260505 23:32:01.326 | INFO | "huge_shared" : " 27 GB", | 20260505 23:32:01.326 | INFO | "malloc" : " 14 GB", | 20260505 23:32:01.326 | INFO | "pinned" : " 29 GB" | 20260505 23:32:01.326 | INFO | } | 20260505 23:32:01.326 | INFO | Perf report for the last 5.13543 seconds: | 20260505 23:32:01.326 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:32:01.326 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:01.326 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:01.326 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:01.326 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 23:32:01.326 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:01.326 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:01.326 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 23:32:01.326 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:01.326 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:01.327 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 23:32:01.327 | INFO | Total mem usage: 15.3 GiB | 20260505 23:32:01.327 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 23:32:01.327 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -1.98%) | 20260505 23:32:01.326 | INFO | Uploaded 36GiB to GPU in 2.1334s (16.8745GiB/s) | 20260505 23:32:01.327 | INFO | Downloaded 34GiB to GPU in 2.02775s (16.7674GiB/s) | 20260505 23:32:01.327 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 23:32:01.327 | INFO | High Water Breakdown: | 20260505 23:32:01.327 | INFO | 61.9% : 9.31 GiB ( 97.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 23:32:01.327 | INFO | 38% : 5.7 GiB ( 85.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 23:32:01.327 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 23:32:01.327 | INFO | ------------------------------------------ | 20260505 23:32:01.327 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:32:01.327 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 23:32:01.327 | INFO | Total mem usage: 26.5 GiB | 20260505 23:32:01.327 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 23:32:01.327 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 23:32:01.327 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 23:32:01.327 | INFO | High Water Breakdown: | 20260505 23:32:01.327 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:32:01.327 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:32:01.327 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:32:01.327 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:32:01.327 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:32:01.327 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:32:01.327 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 23:32:01.327 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 23:32:01.327 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:32:01.327 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 23:32:01.327 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 23:32:01.327 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 23:32:01.327 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 23:32:01.327 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 23:32:01.327 | INFO | ------------------------------------------ | 20260505 23:32:01.327 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:32:01.327 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 23:32:01.327 | INFO | Total mem usage: 25.6 GiB | 20260505 23:32:01.327 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 23:32:01.327 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 23:32:01.327 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 23:32:01.327 | INFO | High Water Breakdown: | 20260505 23:32:01.327 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 23:32:01.327 | INFO | ------------------------------------------ | 20260505 23:32:01.327 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:32:01.327 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 23:32:01.327 | INFO | Total mem usage: 15.7 GiB | 20260505 23:32:01.327 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 23:32:01.327 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 23:32:01.327 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 23:32:01.327 | INFO | High Water Breakdown: | 20260505 23:32:01.327 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 23:32:01.327 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 23:32:01.327 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 23:32:01.327 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 23:32:01.327 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 23:32:01.327 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 23:32:01.327 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 23:32:01.327 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 23:32:01.327 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 23:32:01.327 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:32:01.327 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 23:32:01.327 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:32:01.327 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 23:32:01.327 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 23:32:01.327 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 23:32:01.327 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 23:32:01.327 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:32:01.327 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 23:32:01.327 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:32:01.327 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:32:01.327 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:32:01.327 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 23:32:01.327 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 23:32:01.327 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:32:01.327 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 23:32:01.327 | INFO | ------------------------------------------ | 20260505 23:32:01.327 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:32:01.327 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 23:32:01.327 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:32:01.327 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:32:01.327 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.41e+01, 1.35e+01 / 1.17e+00 | 20260505 23:32:01.327 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.04e-01, 1.05e-02 / 5.92e-03 | 20260505 23:32:01.327 | INFO | PerfReport: 11.84%, Baseline -- 3.71e+00 / 4.53e+00, 5.73e-01 / 1.51e-01 | 20260505 23:32:01.327 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.52e+00, 1.89e-01 / 9.05e-02 | 20260505 23:32:01.327 | INFO | PerfReport: 16.01%, DME -- 4.01e+01 / 4.42e+01, 3.90e+00 / 1.35e+00 | 20260505 23:32:01.327 | INFO | PerfReport: 57.28%, FrameLabeling -- 1.79e+01 / 1.93e+01, 1.48e+00 / 1.16e+00 | 20260505 23:32:01.327 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.23e+00 / 2.77e+00, 4.29e-01 / 2.45e-01 | 20260505 23:32:01.327 | INFO | PerfReport: 3.11%, Metrics -- 9.76e-01 / 1.25e+00, 2.65e-01 / 1.11e-01 | 20260505 23:32:01.327 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.38e-01 / 3.90e-01, 5.80e-02 / 3.17e-02 | 20260505 23:32:01.327 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 23:32:01.327 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:32:01.327 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:32:01.327 | INFO | PerfReport: Full Execution -- 9.84e+01 / -nan, 2.46e+01 / 0.00e+00 | 20260505 23:32:01.327 | INFO | PerfReport: 22.33%, Upload -- 2.20e+01 / 0.00e+00, 7.52e+00 / 0.00e+00 | 20260505 23:32:01.327 | INFO | PerfReport: 21.85%, Download -- 2.15e+01 / 0.00e+00, 7.60e+00 / 0.00e+00 | 20260505 23:32:01.328 | INFO | PerfReport: 31.85%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 23:32:01.328 | INFO | PerfReport: 2.14%, LockUpload -- 2.11e+00 / 0.00e+00, 7.93e+00 / 0.00e+00 | 20260505 23:32:01.328 | INFO | PerfReport: 1.33%, LockDownload -- 1.31e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260505 23:32:01.328 | INFO | PerfReport: 20.48%, LockCompute -- 2.01e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260505 23:32:01.328 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.55e-02 / 0.00e+00 | 20260505 23:32:01.444 | INFO | Starting a round of Baz IO | 20260505 23:32:04.595 | INFO | Populating RTMetrics report | 20260505 23:32:04.595 | INFO | RTMetrics report populated | 20260505 23:32:04.595 | INFO | Opening tmp RTMetrics json file | 20260505 23:32:04.598 | INFO | Serializing RTMetrics to json | 20260505 23:32:04.600 | INFO | Serialization done, swapping tmp file now... | 20260505 23:32:04.601 | INFO | RTMetrics done | 20260505 23:32:04.675 | INFO | Finished a round of Baz IO: Wrote 1.891 GiB in 3.231 seconds | 20260505 23:32:04.675 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.42% / 0.00% / 78.58% | 20260505 23:32:04.676 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.10585% / 0% / 7.54992e-05% | 20260505 23:32:04.676 | INFO | Baz IO overall: | 20260505 23:32:04.676 | INFO | BytesWritten : 4102019489021 | 20260505 23:32:04.676 | INFO | NumEvents : 1932404585816 | 20260505 23:32:06.462 | INFO | Analyzing chunk frames = [7176192,7176704) | 20260505 23:32:06.462 | INFO | MemInfo: { | 20260505 23:32:06.462 | INFO | "gpu" : " 17 GB", | 20260505 23:32:06.462 | INFO | "huge" : "", | 20260505 23:32:06.462 | INFO | "huge_shared" : " 27 GB", | 20260505 23:32:06.462 | INFO | "malloc" : " 14 GB", | 20260505 23:32:06.462 | INFO | "pinned" : " 29 GB" | 20260505 23:32:06.462 | INFO | } | 20260505 23:32:06.462 | INFO | Perf report for the last 5.1358 seconds: | 20260505 23:32:06.462 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:32:06.462 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:06.462 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:06.462 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:06.462 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 23:32:06.462 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:06.462 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260505 23:32:06.462 | INFO | PRE_HQ : 59.7%, 1.7, 54.1ms, 0 | 20260505 23:32:06.462 | INFO | BAZWRITER : 62.1%, 3.6, 116.0ms, 0 | 20260505 23:32:06.462 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 23:32:06.462 | INFO | Uploaded 36GiB to GPU in 2.13069s (16.8959GiB/s) | 20260505 23:32:06.462 | INFO | Downloaded 38GiB to GPU in 2.22734s (17.0607GiB/s) | 20260505 23:32:11.602 | INFO | Analyzing chunk frames = [7176704,7177216) | 20260505 23:32:11.602 | INFO | MemInfo: { | 20260505 23:32:11.602 | INFO | "gpu" : " 17 GB", | 20260505 23:32:11.602 | INFO | "huge" : "", | 20260505 23:32:11.602 | INFO | "huge_shared" : " 27 GB", | 20260505 23:32:11.602 | INFO | "malloc" : " 14 GB", | 20260505 23:32:11.602 | INFO | "pinned" : " 29 GB" | 20260505 23:32:11.602 | INFO | } | 20260505 23:32:11.602 | INFO | Perf report for the last 5.14009 seconds: | 20260505 23:32:11.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:32:11.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:11.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:11.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:11.602 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 23:32:11.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:11.602 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:11.602 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260505 23:32:11.602 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:11.602 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:11.602 | INFO | Uploaded 36GiB to GPU in 2.08443s (17.2709GiB/s) | 20260505 23:32:11.602 | INFO | Downloaded 34GiB to GPU in 2.02446s (16.7946GiB/s) | 20260505 23:32:16.743 | INFO | Analyzing chunk frames = [7177216,7177728) | 20260505 23:32:16.743 | INFO | MemInfo: { | 20260505 23:32:16.743 | INFO | "gpu" : " 17 GB", | 20260505 23:32:16.743 | INFO | "huge" : "", | 20260505 23:32:16.743 | INFO | "huge_shared" : " 27 GB", | 20260505 23:32:16.743 | INFO | "malloc" : " 14 GB", | 20260505 23:32:16.744 | INFO | "pinned" : " 29 GB" | 20260505 23:32:16.744 | INFO | } | 20260505 23:32:16.744 | INFO | Perf report for the last 5.14136 seconds: | 20260505 23:32:16.744 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:32:16.744 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:16.744 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:16.744 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:16.744 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260505 23:32:16.744 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:16.744 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:16.744 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 23:32:16.744 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:16.744 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:16.744 | INFO | Uploaded 36GiB to GPU in 2.11545s (17.0177GiB/s) | 20260505 23:32:16.744 | INFO | Downloaded 34GiB to GPU in 2.03507s (16.707GiB/s) | 20260505 23:32:21.885 | INFO | Analyzing chunk frames = [7177728,7178240) | 20260505 23:32:21.885 | INFO | MemInfo: { | 20260505 23:32:21.885 | INFO | "gpu" : " 17 GB", | 20260505 23:32:21.885 | INFO | "huge" : "", | 20260505 23:32:21.885 | INFO | "huge_shared" : " 27 GB", | 20260505 23:32:21.885 | INFO | "malloc" : " 14 GB", | 20260505 23:32:21.885 | INFO | "pinned" : " 29 GB" | 20260505 23:32:21.885 | INFO | } | 20260505 23:32:21.885 | INFO | Perf report for the last 5.14184 seconds: | 20260505 23:32:21.885 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:32:21.885 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:21.885 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:21.885 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:21.885 | INFO | ANALYSIS : 61.8%, 3.0, 96.0ms, 0 | 20260505 23:32:21.885 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:21.885 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:21.885 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:32:21.885 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:21.885 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:21.886 | INFO | Uploaded 36GiB to GPU in 2.13572s (16.8561GiB/s) | 20260505 23:32:21.886 | INFO | Downloaded 34GiB to GPU in 2.03581s (16.701GiB/s) | 20260505 23:32:27.028 | INFO | Analyzing chunk frames = [7178240,7178752) | 20260505 23:32:27.028 | INFO | MemInfo: { | 20260505 23:32:27.028 | INFO | "gpu" : " 17 GB", | 20260505 23:32:27.028 | INFO | "huge" : "", | 20260505 23:32:27.028 | INFO | "huge_shared" : " 27 GB", | 20260505 23:32:27.028 | INFO | "malloc" : " 14 GB", | 20260505 23:32:27.028 | INFO | "pinned" : " 29 GB" | 20260505 23:32:27.028 | INFO | } | 20260505 23:32:27.028 | INFO | Perf report for the last 5.14247 seconds: | 20260505 23:32:27.028 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:32:27.028 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:27.028 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:27.028 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:27.028 | INFO | ANALYSIS : 61.6%, 3.0, 96.2ms, 0 | 20260505 23:32:27.028 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:27.028 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:27.028 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:32:27.028 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:27.028 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:27.028 | INFO | Uploaded 36GiB to GPU in 2.16248s (16.6476GiB/s) | 20260505 23:32:27.028 | INFO | Downloaded 34GiB to GPU in 2.07435s (16.3907GiB/s) | 20260505 23:32:32.165 | INFO | Analyzing chunk frames = [7178752,7179264) | 20260505 23:32:32.165 | INFO | MemInfo: { | 20260505 23:32:32.165 | INFO | "gpu" : " 17 GB", | 20260505 23:32:32.165 | INFO | "huge" : "", | 20260505 23:32:32.165 | INFO | "huge_shared" : " 27 GB", | 20260505 23:32:32.165 | INFO | "malloc" : " 14 GB", | 20260505 23:32:32.165 | INFO | "pinned" : " 29 GB" | 20260505 23:32:32.165 | INFO | } | 20260505 23:32:32.165 | INFO | Perf report for the last 5.13707 seconds: | 20260505 23:32:32.165 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:32:32.165 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:32.165 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:32.165 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:32.165 | INFO | ANALYSIS : 61.5%, 3.0, 96.0ms, 0 | 20260505 23:32:32.165 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:32.165 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:32.165 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260505 23:32:32.165 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:32.165 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:32.165 | INFO | Uploaded 36GiB to GPU in 2.06901s (17.3996GiB/s) | 20260505 23:32:32.165 | INFO | Downloaded 34GiB to GPU in 2.03335s (16.7211GiB/s) | 20260505 23:32:37.309 | INFO | Analyzing chunk frames = [7179264,7179776) | 20260505 23:32:37.309 | INFO | MemInfo: { | 20260505 23:32:37.309 | INFO | "gpu" : " 17 GB", | 20260505 23:32:37.309 | INFO | "huge" : "", | 20260505 23:32:37.309 | INFO | "huge_shared" : " 27 GB", | 20260505 23:32:37.309 | INFO | "malloc" : " 14 GB", | 20260505 23:32:37.309 | INFO | "pinned" : " 29 GB" | 20260505 23:32:37.309 | INFO | } | 20260505 23:32:37.310 | INFO | Perf report for the last 5.14457 seconds: | 20260505 23:32:37.310 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:32:37.310 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:37.310 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:37.310 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:37.310 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 23:32:37.310 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:37.310 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:37.310 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 23:32:37.310 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:37.310 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:37.310 | INFO | Uploaded 36GiB to GPU in 2.07165s (17.3775GiB/s) | 20260505 23:32:37.310 | INFO | Downloaded 34GiB to GPU in 2.0718s (16.4109GiB/s) | 20260505 23:32:42.442 | INFO | Analyzing chunk frames = [7179776,7180288) | 20260505 23:32:42.442 | INFO | MemInfo: { | 20260505 23:32:42.442 | INFO | "gpu" : " 17 GB", | 20260505 23:32:42.443 | INFO | "huge" : "", | 20260505 23:32:42.443 | INFO | "huge_shared" : " 27 GB", | 20260505 23:32:42.443 | INFO | "malloc" : " 14 GB", | 20260505 23:32:42.443 | INFO | "pinned" : " 29 GB" | 20260505 23:32:42.443 | INFO | } | 20260505 23:32:42.443 | INFO | Perf report for the last 5.13303 seconds: | 20260505 23:32:42.443 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:32:42.443 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:42.443 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:42.443 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:42.443 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 23:32:42.443 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:42.443 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:42.443 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:32:42.443 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:42.443 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:42.443 | INFO | Uploaded 36GiB to GPU in 2.13689s (16.8469GiB/s) | 20260505 23:32:42.443 | INFO | Downloaded 34GiB to GPU in 2.02731s (16.771GiB/s) | 20260505 23:32:42.559 | INFO | Starting a round of Baz IO | 20260505 23:32:45.707 | INFO | Populating RTMetrics report | 20260505 23:32:45.707 | INFO | RTMetrics report populated | 20260505 23:32:45.707 | INFO | Opening tmp RTMetrics json file | 20260505 23:32:45.710 | INFO | Serializing RTMetrics to json | 20260505 23:32:45.713 | INFO | Serialization done, swapping tmp file now... | 20260505 23:32:45.713 | INFO | RTMetrics done | 20260505 23:32:45.787 | INFO | Finished a round of Baz IO: Wrote 1.89 GiB in 3.227 seconds | 20260505 23:32:45.787 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.42% / 0.00% / 78.58% | 20260505 23:32:45.787 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.106076% / 0% / 6.75519e-05% | 20260505 23:32:45.787 | INFO | Baz IO overall: | 20260505 23:32:45.787 | INFO | BytesWritten : 4104048816078 | 20260505 23:32:45.787 | INFO | NumEvents : 1933335057106 | 20260505 23:32:47.580 | INFO | Analyzing chunk frames = [7180288,7180800) | 20260505 23:32:47.580 | INFO | MemInfo: { | 20260505 23:32:47.580 | INFO | "gpu" : " 17 GB", | 20260505 23:32:47.580 | INFO | "huge" : "", | 20260505 23:32:47.580 | INFO | "huge_shared" : " 27 GB", | 20260505 23:32:47.580 | INFO | "malloc" : " 14 GB", | 20260505 23:32:47.580 | INFO | "pinned" : " 29 GB" | 20260505 23:32:47.580 | INFO | } | 20260505 23:32:47.580 | INFO | Perf report for the last 5.13742 seconds: | 20260505 23:32:47.580 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:32:47.580 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:47.580 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:47.580 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:47.580 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260505 23:32:47.580 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:47.580 | INFO | RT_METRICS : 43.4%, 1.0, 22.8ms, 0 | 20260505 23:32:47.580 | INFO | PRE_HQ : 59.9%, 1.7, 54.3ms, 0 | 20260505 23:32:47.580 | INFO | BAZWRITER : 61.9%, 3.5, 112.7ms, 0 | 20260505 23:32:47.580 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 23:32:47.580 | INFO | Uploaded 36GiB to GPU in 2.17162s (16.5775GiB/s) | 20260505 23:32:47.580 | INFO | Downloaded 38GiB to GPU in 2.23949s (16.9681GiB/s) | 20260505 23:32:52.725 | INFO | Analyzing chunk frames = [7180800,7181312) | 20260505 23:32:52.725 | INFO | MemInfo: { | 20260505 23:32:52.725 | INFO | "gpu" : " 17 GB", | 20260505 23:32:52.725 | INFO | "huge" : "", | 20260505 23:32:52.725 | INFO | "huge_shared" : " 27 GB", | 20260505 23:32:52.725 | INFO | "malloc" : " 14 GB", | 20260505 23:32:52.725 | INFO | "pinned" : " 29 GB" | 20260505 23:32:52.725 | INFO | } | 20260505 23:32:52.725 | INFO | Perf report for the last 5.14485 seconds: | 20260505 23:32:52.725 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:32:52.725 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:52.725 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:52.725 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:52.725 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 23:32:52.725 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:52.725 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:52.725 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260505 23:32:52.725 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:52.725 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:52.725 | INFO | Uploaded 36GiB to GPU in 2.07683s (17.3341GiB/s) | 20260505 23:32:52.725 | INFO | Downloaded 34GiB to GPU in 2.01589s (16.866GiB/s) | 20260505 23:32:57.857 | INFO | Analyzing chunk frames = [7181312,7181824) | 20260505 23:32:57.857 | INFO | MemInfo: { | 20260505 23:32:57.857 | INFO | "gpu" : " 17 GB", | 20260505 23:32:57.857 | INFO | "huge" : "", | 20260505 23:32:57.857 | INFO | "huge_shared" : " 27 GB", | 20260505 23:32:57.857 | INFO | "malloc" : " 14 GB", | 20260505 23:32:57.857 | INFO | "pinned" : " 29 GB" | 20260505 23:32:57.857 | INFO | } | 20260505 23:32:57.857 | INFO | Perf report for the last 5.13218 seconds: | 20260505 23:32:57.857 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:32:57.857 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:57.857 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:57.857 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:57.857 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260505 23:32:57.857 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:57.857 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:57.857 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 23:32:57.857 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:57.857 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:32:57.857 | INFO | Uploaded 36GiB to GPU in 2.10834s (17.0751GiB/s) | 20260505 23:32:57.857 | INFO | Downloaded 34GiB to GPU in 2.02774s (16.7675GiB/s) | 20260505 23:33:03.001 | INFO | Analyzing chunk frames = [7181824,7182336) | 20260505 23:33:03.001 | INFO | MemInfo: { | 20260505 23:33:03.001 | INFO | "gpu" : " 17 GB", | 20260505 23:33:03.001 | INFO | "huge" : "", | 20260505 23:33:03.001 | INFO | "huge_shared" : " 27 GB", | 20260505 23:33:03.001 | INFO | "malloc" : " 14 GB", | 20260505 23:33:03.001 | INFO | "pinned" : " 29 GB" | 20260505 23:33:03.001 | INFO | } | 20260505 23:33:03.001 | INFO | Perf report for the last 5.14403 seconds: | 20260505 23:33:03.001 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:33:03.001 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:03.001 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:03.001 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:03.001 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260505 23:33:03.001 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:03.001 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:03.001 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 23:33:03.001 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:03.001 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:03.001 | INFO | Uploaded 36GiB to GPU in 2.13238s (16.8825GiB/s) | 20260505 23:33:03.001 | INFO | Downloaded 34GiB to GPU in 2.02616s (16.7805GiB/s) | 20260505 23:33:08.142 | INFO | Analyzing chunk frames = [7182336,7182848) | 20260505 23:33:08.142 | INFO | MemInfo: { | 20260505 23:33:08.142 | INFO | "gpu" : " 17 GB", | 20260505 23:33:08.142 | INFO | "huge" : "", | 20260505 23:33:08.143 | INFO | "huge_shared" : " 27 GB", | 20260505 23:33:08.143 | INFO | "malloc" : " 14 GB", | 20260505 23:33:08.143 | INFO | "pinned" : " 29 GB" | 20260505 23:33:08.143 | INFO | } | 20260505 23:33:08.143 | INFO | Perf report for the last 5.14145 seconds: | 20260505 23:33:08.143 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:33:08.143 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:08.143 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:08.143 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:08.143 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260505 23:33:08.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:08.143 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:08.143 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 23:33:08.143 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:08.143 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:08.143 | INFO | Uploaded 36GiB to GPU in 2.10652s (17.0898GiB/s) | 20260505 23:33:08.143 | INFO | Downloaded 34GiB to GPU in 2.04869s (16.596GiB/s) | 20260505 23:33:13.278 | INFO | Analyzing chunk frames = [7182848,7183360) | 20260505 23:33:13.278 | INFO | MemInfo: { | 20260505 23:33:13.278 | INFO | "gpu" : " 17 GB", | 20260505 23:33:13.278 | INFO | "huge" : "", | 20260505 23:33:13.278 | INFO | "huge_shared" : " 27 GB", | 20260505 23:33:13.278 | INFO | "malloc" : " 14 GB", | 20260505 23:33:13.278 | INFO | "pinned" : " 29 GB" | 20260505 23:33:13.278 | INFO | } | 20260505 23:33:13.278 | INFO | Perf report for the last 5.13514 seconds: | 20260505 23:33:13.278 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:33:13.278 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:13.278 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:13.278 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:13.278 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260505 23:33:13.278 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:13.278 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:13.278 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 23:33:13.278 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:13.278 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:13.278 | INFO | Uploaded 36GiB to GPU in 2.08206s (17.2906GiB/s) | 20260505 23:33:13.278 | INFO | Downloaded 34GiB to GPU in 2.03888s (16.6759GiB/s) | 20260505 23:33:18.409 | INFO | Analyzing chunk frames = [7183360,7183872) | 20260505 23:33:18.409 | INFO | MemInfo: { | 20260505 23:33:18.409 | INFO | "gpu" : " 17 GB", | 20260505 23:33:18.409 | INFO | "huge" : "", | 20260505 23:33:18.409 | INFO | "huge_shared" : " 27 GB", | 20260505 23:33:18.409 | INFO | "malloc" : " 14 GB", | 20260505 23:33:18.409 | INFO | "pinned" : " 29 GB" | 20260505 23:33:18.409 | INFO | } | 20260505 23:33:18.409 | INFO | Perf report for the last 5.13168 seconds: | 20260505 23:33:18.409 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:33:18.409 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:18.409 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:18.409 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:18.409 | INFO | ANALYSIS : 62.8%, 3.0, 97.8ms, 0 | 20260505 23:33:18.409 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:18.409 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:18.409 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 23:33:18.410 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:18.410 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:18.410 | INFO | Uploaded 36GiB to GPU in 2.15229s (16.7263GiB/s) | 20260505 23:33:18.410 | INFO | Downloaded 34GiB to GPU in 2.10356s (16.1631GiB/s) | 20260505 23:33:23.555 | INFO | Analyzing chunk frames = [7183872,7184384) | 20260505 23:33:23.555 | INFO | MemInfo: { | 20260505 23:33:23.555 | INFO | "gpu" : " 17 GB", | 20260505 23:33:23.555 | INFO | "huge" : "", | 20260505 23:33:23.555 | INFO | "huge_shared" : " 27 GB", | 20260505 23:33:23.555 | INFO | "malloc" : " 14 GB", | 20260505 23:33:23.555 | INFO | "pinned" : " 29 GB" | 20260505 23:33:23.555 | INFO | } | 20260505 23:33:23.555 | INFO | Perf report for the last 5.146 seconds: | 20260505 23:33:23.555 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:33:23.555 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:23.555 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:23.556 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:23.556 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260505 23:33:23.556 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:23.556 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:23.556 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:33:23.556 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:23.556 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:23.556 | INFO | Uploaded 36GiB to GPU in 2.11115s (17.0523GiB/s) | 20260505 23:33:23.556 | INFO | Downloaded 34GiB to GPU in 2.01063s (16.9101GiB/s) | 20260505 23:33:23.556 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 23:33:23.556 | INFO | Total mem usage: 15.3 GiB | 20260505 23:33:23.556 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 23:33:23.556 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.28%) | 20260505 23:33:23.556 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 23:33:23.556 | INFO | High Water Breakdown: | 20260505 23:33:23.556 | INFO | 61% : 8.64 GiB ( 90.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 23:33:23.556 | INFO | 38.9% : 5.5 GiB ( 82.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 23:33:23.556 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 23:33:23.556 | INFO | ------------------------------------------ | 20260505 23:33:23.556 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:33:23.556 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 23:33:23.556 | INFO | Total mem usage: 26.5 GiB | 20260505 23:33:23.556 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 23:33:23.556 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 23:33:23.556 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 23:33:23.556 | INFO | High Water Breakdown: | 20260505 23:33:23.556 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:33:23.556 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:33:23.556 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:33:23.556 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:33:23.556 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:33:23.556 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:33:23.556 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 23:33:23.556 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 23:33:23.556 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:33:23.556 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 23:33:23.556 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 23:33:23.556 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 23:33:23.556 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 23:33:23.556 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 23:33:23.556 | INFO | ------------------------------------------ | 20260505 23:33:23.556 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:33:23.556 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 23:33:23.556 | INFO | Total mem usage: 25.6 GiB | 20260505 23:33:23.556 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 23:33:23.556 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 23:33:23.556 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 23:33:23.556 | INFO | High Water Breakdown: | 20260505 23:33:23.556 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 23:33:23.557 | INFO | ------------------------------------------ | 20260505 23:33:23.557 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:33:23.557 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 23:33:23.557 | INFO | Total mem usage: 15.7 GiB | 20260505 23:33:23.557 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 23:33:23.557 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 23:33:23.557 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 23:33:23.557 | INFO | High Water Breakdown: | 20260505 23:33:23.557 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 23:33:23.557 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 23:33:23.557 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 23:33:23.557 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 23:33:23.557 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 23:33:23.557 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 23:33:23.557 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 23:33:23.557 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 23:33:23.557 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 23:33:23.557 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:33:23.557 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 23:33:23.557 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:33:23.557 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 23:33:23.557 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 23:33:23.557 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 23:33:23.557 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 23:33:23.557 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:33:23.557 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 23:33:23.557 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:33:23.557 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:33:23.557 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:33:23.557 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 23:33:23.557 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 23:33:23.557 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:33:23.557 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 23:33:23.557 | INFO | ------------------------------------------ | 20260505 23:33:23.557 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:33:23.557 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 23:33:23.557 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:33:23.557 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:33:23.557 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.42e+01, 1.35e+01 / 7.04e-01 | 20260505 23:33:23.557 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.04e-01, 1.08e-02 / 4.37e-03 | 20260505 23:33:23.557 | INFO | PerfReport: 11.80%, Baseline -- 3.71e+00 / 4.45e+00, 5.71e-01 / 2.44e-01 | 20260505 23:33:23.557 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.51e+00, 1.91e-01 / 1.13e-01 | 20260505 23:33:23.557 | INFO | PerfReport: 15.94%, DME -- 4.01e+01 / 4.40e+01, 3.88e+00 / 9.34e-01 | 20260505 23:33:23.557 | INFO | PerfReport: 57.46%, FrameLabeling -- 1.81e+01 / 1.98e+01, 1.63e+00 / 1.32e+00 | 20260505 23:33:23.557 | INFO | PerfReport: 7.06%, PulseAccumulating -- 2.22e+00 / 2.71e+00, 4.21e-01 / 2.47e-01 | 20260505 23:33:23.557 | INFO | PerfReport: 3.09%, Metrics -- 9.71e-01 / 1.25e+00, 2.65e-01 / 9.57e-02 | 20260505 23:33:23.557 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.37e-01 / 3.88e-01, 5.88e-02 / 3.30e-02 | 20260505 23:33:23.557 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 23:33:23.557 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:33:23.557 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:33:23.557 | INFO | PerfReport: Full Execution -- 9.85e+01 / -nan, 2.49e+01 / 0.00e+00 | 20260505 23:33:23.557 | INFO | PerfReport: 22.36%, Upload -- 2.20e+01 / 0.00e+00, 7.55e+00 / 0.00e+00 | 20260505 23:33:23.557 | INFO | PerfReport: 21.83%, Download -- 2.15e+01 / 0.00e+00, 7.67e+00 / 0.00e+00 | 20260505 23:33:23.557 | INFO | PerfReport: 31.89%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 23:33:23.557 | INFO | PerfReport: 2.11%, LockUpload -- 2.08e+00 / 0.00e+00, 7.76e+00 / 0.00e+00 | 20260505 23:33:23.557 | INFO | PerfReport: 1.29%, LockDownload -- 1.27e+00 / 0.00e+00, 2.87e+00 / 0.00e+00 | 20260505 23:33:23.557 | INFO | PerfReport: 20.49%, LockCompute -- 2.02e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260505 23:33:23.557 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.95e-02 / 0.00e+00 | 20260505 23:33:23.675 | INFO | Starting a round of Baz IO | 20260505 23:33:26.826 | INFO | Populating RTMetrics report | 20260505 23:33:26.826 | INFO | RTMetrics report populated | 20260505 23:33:26.826 | INFO | Opening tmp RTMetrics json file | 20260505 23:33:26.827 | INFO | Serializing RTMetrics to json | 20260505 23:33:26.830 | INFO | Serialization done, swapping tmp file now... | 20260505 23:33:26.831 | INFO | RTMetrics done | 20260505 23:33:26.905 | INFO | Finished a round of Baz IO: Wrote 1.893 GiB in 3.229 seconds | 20260505 23:33:26.905 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.42% / 0.00% / 78.58% | 20260505 23:33:26.905 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.105886% / 0% / 5.96046e-05% | 20260505 23:33:26.905 | INFO | Baz IO overall: | 20260505 23:33:26.905 | INFO | BytesWritten : 4106081064210 | 20260505 23:33:26.905 | INFO | NumEvents : 1934268029506 | 20260505 23:33:28.699 | INFO | Analyzing chunk frames = [7184384,7184896) | 20260505 23:33:28.699 | INFO | MemInfo: { | 20260505 23:33:28.699 | INFO | "gpu" : " 17 GB", | 20260505 23:33:28.699 | INFO | "huge" : "", | 20260505 23:33:28.699 | INFO | "huge_shared" : " 27 GB", | 20260505 23:33:28.699 | INFO | "malloc" : " 14 GB", | 20260505 23:33:28.700 | INFO | "pinned" : " 29 GB" | 20260505 23:33:28.700 | INFO | } | 20260505 23:33:28.700 | INFO | Perf report for the last 5.14408 seconds: | 20260505 23:33:28.700 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:33:28.700 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:28.700 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:28.700 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:28.700 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 23:33:28.700 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:28.700 | INFO | RT_METRICS : 43.4%, 1.0, 22.8ms, 0 | 20260505 23:33:28.700 | INFO | PRE_HQ : 59.9%, 1.7, 54.4ms, 0 | 20260505 23:33:28.700 | INFO | BAZWRITER : 61.9%, 3.5, 113.0ms, 0 | 20260505 23:33:28.700 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 23:33:28.700 | INFO | Uploaded 36GiB to GPU in 2.1584s (16.679GiB/s) | 20260505 23:33:28.700 | INFO | Downloaded 38GiB to GPU in 2.23429s (17.0076GiB/s) | 20260505 23:33:33.836 | INFO | Analyzing chunk frames = [7184896,7185408) | 20260505 23:33:33.836 | INFO | MemInfo: { | 20260505 23:33:33.836 | INFO | "gpu" : " 17 GB", | 20260505 23:33:33.836 | INFO | "huge" : "", | 20260505 23:33:33.836 | INFO | "huge_shared" : " 27 GB", | 20260505 23:33:33.836 | INFO | "malloc" : " 14 GB", | 20260505 23:33:33.836 | INFO | "pinned" : " 29 GB" | 20260505 23:33:33.836 | INFO | } | 20260505 23:33:33.836 | INFO | Perf report for the last 5.13684 seconds: | 20260505 23:33:33.836 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:33:33.836 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:33.836 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:33.836 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:33.836 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260505 23:33:33.836 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:33.836 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:33.837 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260505 23:33:33.837 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:33.837 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:33.837 | INFO | Uploaded 36GiB to GPU in 2.08996s (17.2252GiB/s) | 20260505 23:33:33.837 | INFO | Downloaded 34GiB to GPU in 2.03888s (16.6758GiB/s) | 20260505 23:33:38.971 | INFO | Analyzing chunk frames = [7185408,7185920) | 20260505 23:33:38.971 | INFO | MemInfo: { | 20260505 23:33:38.971 | INFO | "gpu" : " 17 GB", | 20260505 23:33:38.971 | INFO | "huge" : "", | 20260505 23:33:38.972 | INFO | "huge_shared" : " 27 GB", | 20260505 23:33:38.972 | INFO | "malloc" : " 14 GB", | 20260505 23:33:38.972 | INFO | "pinned" : " 29 GB" | 20260505 23:33:38.972 | INFO | } | 20260505 23:33:38.972 | INFO | Perf report for the last 5.13522 seconds: | 20260505 23:33:38.972 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:33:38.972 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:38.972 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:38.972 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:38.972 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260505 23:33:38.972 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:38.972 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:38.972 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:33:38.972 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:38.972 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:38.972 | INFO | Uploaded 36GiB to GPU in 2.0912s (17.215GiB/s) | 20260505 23:33:38.972 | INFO | Downloaded 34GiB to GPU in 2.02613s (16.7808GiB/s) | 20260505 23:33:44.114 | INFO | Analyzing chunk frames = [7185920,7186432) | 20260505 23:33:44.114 | INFO | MemInfo: { | 20260505 23:33:44.114 | INFO | "gpu" : " 17 GB", | 20260505 23:33:44.114 | INFO | "huge" : "", | 20260505 23:33:44.114 | INFO | "huge_shared" : " 27 GB", | 20260505 23:33:44.114 | INFO | "malloc" : " 14 GB", | 20260505 23:33:44.114 | INFO | "pinned" : " 29 GB" | 20260505 23:33:44.114 | INFO | } | 20260505 23:33:44.114 | INFO | Perf report for the last 5.14238 seconds: | 20260505 23:33:44.114 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:33:44.114 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:44.114 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:44.114 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:44.114 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260505 23:33:44.114 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:44.114 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:44.114 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:33:44.114 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:44.114 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:44.114 | INFO | Uploaded 36GiB to GPU in 2.135s (16.8618GiB/s) | 20260505 23:33:44.114 | INFO | Downloaded 34GiB to GPU in 2.03996s (16.667GiB/s) | 20260505 23:33:49.259 | INFO | Analyzing chunk frames = [7186432,7186944) | 20260505 23:33:49.259 | INFO | MemInfo: { | 20260505 23:33:49.259 | INFO | "gpu" : " 17 GB", | 20260505 23:33:49.259 | INFO | "huge" : "", | 20260505 23:33:49.259 | INFO | "huge_shared" : " 27 GB", | 20260505 23:33:49.259 | INFO | "malloc" : " 14 GB", | 20260505 23:33:49.259 | INFO | "pinned" : " 29 GB" | 20260505 23:33:49.259 | INFO | } | 20260505 23:33:49.259 | INFO | Perf report for the last 5.14521 seconds: | 20260505 23:33:49.259 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:33:49.259 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:49.259 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:49.259 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:49.259 | INFO | ANALYSIS : 61.1%, 3.0, 95.6ms, 0 | 20260505 23:33:49.259 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:49.259 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:49.259 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 23:33:49.259 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:49.259 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:49.259 | INFO | Uploaded 36GiB to GPU in 2.10305s (17.118GiB/s) | 20260505 23:33:49.260 | INFO | Downloaded 34GiB to GPU in 2.05338s (16.5581GiB/s) | 20260505 23:33:54.393 | INFO | Analyzing chunk frames = [7186944,7187456) | 20260505 23:33:54.393 | INFO | MemInfo: { | 20260505 23:33:54.393 | INFO | "gpu" : " 17 GB", | 20260505 23:33:54.393 | INFO | "huge" : "", | 20260505 23:33:54.393 | INFO | "huge_shared" : " 27 GB", | 20260505 23:33:54.393 | INFO | "malloc" : " 14 GB", | 20260505 23:33:54.393 | INFO | "pinned" : " 29 GB" | 20260505 23:33:54.393 | INFO | } | 20260505 23:33:54.393 | INFO | Perf report for the last 5.13378 seconds: | 20260505 23:33:54.393 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:33:54.393 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:54.393 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:54.393 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:54.393 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260505 23:33:54.393 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:54.393 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:54.393 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260505 23:33:54.393 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:54.393 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:54.393 | INFO | Uploaded 36GiB to GPU in 2.06552s (17.429GiB/s) | 20260505 23:33:54.393 | INFO | Downloaded 34GiB to GPU in 2.03028s (16.7464GiB/s) | 20260505 23:33:59.529 | INFO | Analyzing chunk frames = [7187456,7187968) | 20260505 23:33:59.529 | INFO | MemInfo: { | 20260505 23:33:59.529 | INFO | "gpu" : " 17 GB", | 20260505 23:33:59.529 | INFO | "huge" : "", | 20260505 23:33:59.529 | INFO | "huge_shared" : " 27 GB", | 20260505 23:33:59.529 | INFO | "malloc" : " 14 GB", | 20260505 23:33:59.529 | INFO | "pinned" : " 29 GB" | 20260505 23:33:59.529 | INFO | } | 20260505 23:33:59.529 | INFO | Perf report for the last 5.13616 seconds: | 20260505 23:33:59.529 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:33:59.529 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:59.529 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:59.529 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:59.529 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260505 23:33:59.529 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:59.529 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:59.529 | INFO | PRE_HQ : 27.7%, 1.0, 14.3ms, 0 | 20260505 23:33:59.529 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:59.529 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:33:59.529 | INFO | Uploaded 36GiB to GPU in 2.1398s (16.824GiB/s) | 20260505 23:33:59.530 | INFO | Downloaded 34GiB to GPU in 2.13452s (15.9287GiB/s) | 20260505 23:34:04.672 | INFO | Analyzing chunk frames = [7187968,7188480) | 20260505 23:34:04.672 | INFO | MemInfo: { | 20260505 23:34:04.672 | INFO | "gpu" : " 17 GB", | 20260505 23:34:04.672 | INFO | "huge" : "", | 20260505 23:34:04.672 | INFO | "huge_shared" : " 27 GB", | 20260505 23:34:04.672 | INFO | "malloc" : " 14 GB", | 20260505 23:34:04.672 | INFO | "pinned" : " 29 GB" | 20260505 23:34:04.672 | INFO | } | 20260505 23:34:04.672 | INFO | Perf report for the last 5.14324 seconds: | 20260505 23:34:04.672 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:34:04.672 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:04.672 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:04.672 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:04.673 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260505 23:34:04.673 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:04.673 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:04.673 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260505 23:34:04.673 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:04.673 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:04.673 | INFO | Uploaded 36GiB to GPU in 2.11503s (17.0211GiB/s) | 20260505 23:34:04.673 | INFO | Downloaded 34GiB to GPU in 2.00881s (16.9254GiB/s) | 20260505 23:34:04.793 | INFO | Starting a round of Baz IO | 20260505 23:34:07.940 | INFO | Populating RTMetrics report | 20260505 23:34:07.940 | INFO | RTMetrics report populated | 20260505 23:34:07.940 | INFO | Opening tmp RTMetrics json file | 20260505 23:34:07.942 | INFO | Serializing RTMetrics to json | 20260505 23:34:07.944 | INFO | Serialization done, swapping tmp file now... | 20260505 23:34:07.947 | INFO | RTMetrics done | 20260505 23:34:08.037 | INFO | Finished a round of Baz IO: Wrote 1.891 GiB in 3.244 seconds | 20260505 23:34:08.037 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.42% / 0.00% / 78.58% | 20260505 23:34:08.037 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.105818% / 0% / 6.75519e-05% | 20260505 23:34:08.038 | INFO | Baz IO overall: | 20260505 23:34:08.038 | INFO | BytesWritten : 4108111560147 | 20260505 23:34:08.038 | INFO | NumEvents : 1935199249893 | 20260505 23:34:09.810 | INFO | Analyzing chunk frames = [7188480,7188992) | 20260505 23:34:09.810 | INFO | MemInfo: { | 20260505 23:34:09.810 | INFO | "gpu" : " 17 GB", | 20260505 23:34:09.810 | INFO | "huge" : "", | 20260505 23:34:09.810 | INFO | "huge_shared" : " 27 GB", | 20260505 23:34:09.810 | INFO | "malloc" : " 14 GB", | 20260505 23:34:09.810 | INFO | "pinned" : " 29 GB" | 20260505 23:34:09.810 | INFO | } | 20260505 23:34:09.810 | INFO | Perf report for the last 5.13793 seconds: | 20260505 23:34:09.810 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:34:09.810 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:09.810 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:09.810 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:09.810 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 23:34:09.810 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:09.810 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260505 23:34:09.810 | INFO | PRE_HQ : 59.9%, 1.7, 54.3ms, 0 | 20260505 23:34:09.810 | INFO | BAZWRITER : 62.3%, 3.6, 116.8ms, 0 | 20260505 23:34:09.810 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 23:34:09.811 | INFO | Uploaded 36GiB to GPU in 2.16808s (16.6045GiB/s) | 20260505 23:34:09.811 | INFO | Downloaded 38GiB to GPU in 2.24188s (16.9501GiB/s) | 20260505 23:34:14.951 | INFO | Analyzing chunk frames = [7188992,7189504) | 20260505 23:34:14.951 | INFO | MemInfo: { | 20260505 23:34:14.951 | INFO | "gpu" : " 17 GB", | 20260505 23:34:14.951 | INFO | "huge" : "", | 20260505 23:34:14.951 | INFO | "huge_shared" : " 27 GB", | 20260505 23:34:14.951 | INFO | "malloc" : " 14 GB", | 20260505 23:34:14.951 | INFO | "pinned" : " 29 GB" | 20260505 23:34:14.951 | INFO | } | 20260505 23:34:14.951 | INFO | Perf report for the last 5.14031 seconds: | 20260505 23:34:14.951 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:34:14.951 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:14.951 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:14.951 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:14.951 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260505 23:34:14.951 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:14.951 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:14.951 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:34:14.951 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:14.951 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:14.951 | INFO | Uploaded 36GiB to GPU in 2.11166s (17.0482GiB/s) | 20260505 23:34:14.951 | INFO | Downloaded 34GiB to GPU in 2.04304s (16.6418GiB/s) | 20260505 23:34:20.093 | INFO | Analyzing chunk frames = [7189504,7190016) | 20260505 23:34:20.093 | INFO | MemInfo: { | 20260505 23:34:20.093 | INFO | "gpu" : " 17 GB", | 20260505 23:34:20.094 | INFO | "huge" : "", | 20260505 23:34:20.094 | INFO | "huge_shared" : " 27 GB", | 20260505 23:34:20.094 | INFO | "malloc" : " 14 GB", | 20260505 23:34:20.094 | INFO | "pinned" : " 29 GB" | 20260505 23:34:20.094 | INFO | } | 20260505 23:34:20.094 | INFO | Perf report for the last 5.14286 seconds: | 20260505 23:34:20.094 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:34:20.094 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:20.094 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:20.094 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:20.094 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260505 23:34:20.094 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:20.094 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:20.094 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:34:20.094 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:20.094 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:20.094 | INFO | Uploaded 36GiB to GPU in 2.0863s (17.2555GiB/s) | 20260505 23:34:20.094 | INFO | Downloaded 34GiB to GPU in 2.02081s (16.825GiB/s) | 20260505 23:34:25.223 | INFO | Analyzing chunk frames = [7190016,7190528) | 20260505 23:34:25.223 | INFO | MemInfo: { | 20260505 23:34:25.223 | INFO | "gpu" : " 17 GB", | 20260505 23:34:25.223 | INFO | "huge" : "", | 20260505 23:34:25.223 | INFO | "huge_shared" : " 27 GB", | 20260505 23:34:25.223 | INFO | "malloc" : " 14 GB", | 20260505 23:34:25.223 | INFO | "pinned" : " 29 GB" | 20260505 23:34:25.223 | INFO | } | 20260505 23:34:25.223 | INFO | Perf report for the last 5.12985 seconds: | 20260505 23:34:25.223 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:34:25.223 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:25.223 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:25.223 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:25.223 | INFO | ANALYSIS : 61.7%, 3.0, 95.7ms, 0 | 20260505 23:34:25.224 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:25.224 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:25.224 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 23:34:25.224 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:25.224 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:25.224 | INFO | Uploaded 36GiB to GPU in 2.13621s (16.8523GiB/s) | 20260505 23:34:25.224 | INFO | Downloaded 34GiB to GPU in 2.01391s (16.8826GiB/s) | 20260505 23:34:30.364 | INFO | Analyzing chunk frames = [7190528,7191040) | 20260505 23:34:30.364 | INFO | MemInfo: { | 20260505 23:34:30.364 | INFO | "gpu" : " 17 GB", | 20260505 23:34:30.365 | INFO | "huge" : "", | 20260505 23:34:30.365 | INFO | "huge_shared" : " 27 GB", | 20260505 23:34:30.365 | INFO | "malloc" : " 14 GB", | 20260505 23:34:30.365 | INFO | "pinned" : " 29 GB" | 20260505 23:34:30.365 | INFO | } | 20260505 23:34:30.365 | INFO | Perf report for the last 5.14118 seconds: | 20260505 23:34:30.365 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:34:30.365 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:30.365 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:30.365 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:30.365 | INFO | ANALYSIS : 61.6%, 3.0, 96.2ms, 0 | 20260505 23:34:30.365 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:30.365 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:30.365 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260505 23:34:30.365 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:30.365 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:30.365 | INFO | Uploaded 36GiB to GPU in 2.13932s (16.8277GiB/s) | 20260505 23:34:30.365 | INFO | Downloaded 34GiB to GPU in 2.06353s (16.4766GiB/s) | 20260505 23:34:35.505 | INFO | Analyzing chunk frames = [7191040,7191552) | 20260505 23:34:35.505 | INFO | MemInfo: { | 20260505 23:34:35.505 | INFO | "gpu" : " 17 GB", | 20260505 23:34:35.505 | INFO | "huge" : "", | 20260505 23:34:35.505 | INFO | "huge_shared" : " 27 GB", | 20260505 23:34:35.505 | INFO | "malloc" : " 14 GB", | 20260505 23:34:35.505 | INFO | "pinned" : " 29 GB" | 20260505 23:34:35.505 | INFO | } | 20260505 23:34:35.505 | INFO | Perf report for the last 5.14029 seconds: | 20260505 23:34:35.505 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:34:35.505 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:35.505 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:35.505 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:35.505 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260505 23:34:35.505 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:35.505 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:35.505 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260505 23:34:35.505 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:35.505 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:35.505 | INFO | Uploaded 36GiB to GPU in 2.08523s (17.2642GiB/s) | 20260505 23:34:35.505 | INFO | Downloaded 34GiB to GPU in 2.03745s (16.6875GiB/s) | 20260505 23:34:40.653 | INFO | Analyzing chunk frames = [7191552,7192064) | 20260505 23:34:40.653 | INFO | MemInfo: { | 20260505 23:34:40.653 | INFO | "gpu" : " 17 GB", | 20260505 23:34:40.653 | INFO | "huge" : "", | 20260505 23:34:40.653 | INFO | "huge_shared" : " 27 GB", | 20260505 23:34:40.653 | INFO | "malloc" : " 14 GB", | 20260505 23:34:40.653 | INFO | "pinned" : " 29 GB" | 20260505 23:34:40.653 | INFO | } | 20260505 23:34:40.653 | INFO | Perf report for the last 5.14777 seconds: | 20260505 23:34:40.653 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:34:40.653 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:40.653 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:40.653 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:40.653 | INFO | ANALYSIS : 62.0%, 3.0, 97.0ms, 0 | 20260505 23:34:40.653 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:40.653 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:40.653 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 23:34:40.653 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:40.653 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:40.653 | INFO | Uploaded 36GiB to GPU in 2.01437s (17.8716GiB/s) | 20260505 23:34:40.653 | INFO | Downloaded 34GiB to GPU in 2.03291s (16.7248GiB/s) | 20260505 23:34:45.786 | INFO | Analyzing chunk frames = [7192064,7192576) | 20260505 23:34:45.786 | INFO | MemInfo: { | 20260505 23:34:45.786 | INFO | "gpu" : " 17 GB", | 20260505 23:34:45.786 | INFO | "huge" : "", | 20260505 23:34:45.786 | INFO | "huge_shared" : " 27 GB", | 20260505 23:34:45.786 | INFO | "malloc" : " 14 GB", | 20260505 23:34:45.786 | INFO | "pinned" : " 29 GB" | 20260505 23:34:45.786 | INFO | } | 20260505 23:34:45.786 | INFO | Perf report for the last 5.133 seconds: | 20260505 23:34:45.786 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:34:45.786 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:45.786 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:45.786 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:45.786 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260505 23:34:45.786 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:45.786 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:45.786 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 23:34:45.786 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:45.786 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:45.786 | INFO | Uploaded 36GiB to GPU in 2.14164s (16.8095GiB/s) | 20260505 23:34:45.786 | INFO | Downloaded 34GiB to GPU in 2.03246s (16.7285GiB/s) | 20260505 23:34:45.786 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 23:34:45.786 | INFO | Total mem usage: 15.3 GiB | 20260505 23:34:45.786 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 23:34:45.786 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.12%) | 20260505 23:34:45.786 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 23:34:45.786 | INFO | High Water Breakdown: | 20260505 23:34:45.786 | INFO | 60.9% : 8.55 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 23:34:45.786 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 23:34:45.786 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 23:34:45.786 | INFO | ------------------------------------------ | 20260505 23:34:45.786 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:34:45.786 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 23:34:45.786 | INFO | Total mem usage: 26.5 GiB | 20260505 23:34:45.786 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 23:34:45.786 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 23:34:45.786 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 23:34:45.786 | INFO | High Water Breakdown: | 20260505 23:34:45.786 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:34:45.786 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:34:45.786 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:34:45.786 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:34:45.786 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:34:45.786 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:34:45.786 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 23:34:45.786 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 23:34:45.786 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:34:45.786 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 23:34:45.786 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 23:34:45.786 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 23:34:45.786 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 23:34:45.786 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 23:34:45.786 | INFO | ------------------------------------------ | 20260505 23:34:45.786 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:34:45.786 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 23:34:45.786 | INFO | Total mem usage: 25.6 GiB | 20260505 23:34:45.786 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 23:34:45.786 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 23:34:45.786 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 23:34:45.786 | INFO | High Water Breakdown: | 20260505 23:34:45.787 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 23:34:45.787 | INFO | ------------------------------------------ | 20260505 23:34:45.787 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:34:45.787 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 23:34:45.787 | INFO | Total mem usage: 15.7 GiB | 20260505 23:34:45.787 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 23:34:45.787 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 23:34:45.787 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 23:34:45.787 | INFO | High Water Breakdown: | 20260505 23:34:45.787 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 23:34:45.787 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 23:34:45.787 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 23:34:45.787 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 23:34:45.787 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 23:34:45.787 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 23:34:45.787 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 23:34:45.787 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 23:34:45.787 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 23:34:45.787 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:34:45.787 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 23:34:45.787 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:34:45.787 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 23:34:45.787 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 23:34:45.787 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 23:34:45.787 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 23:34:45.787 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:34:45.787 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 23:34:45.787 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:34:45.787 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:34:45.787 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:34:45.787 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 23:34:45.787 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 23:34:45.787 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:34:45.787 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 23:34:45.787 | INFO | ------------------------------------------ | 20260505 23:34:45.787 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:34:45.787 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 23:34:45.787 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:34:45.787 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:34:45.787 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.42e+01, 1.35e+01 / 1.34e+00 | 20260505 23:34:45.787 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.03e-01, 1.06e-02 / 3.22e-03 | 20260505 23:34:45.787 | INFO | PerfReport: 11.83%, Baseline -- 3.71e+00 / 4.53e+00, 5.72e-01 / 1.84e-01 | 20260505 23:34:45.787 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.51e+00, 1.90e-01 / 1.10e-01 | 20260505 23:34:45.787 | INFO | PerfReport: 16.00%, DME -- 4.01e+01 / 4.42e+01, 3.97e+00 / 1.41e+00 | 20260505 23:34:45.787 | INFO | PerfReport: 57.30%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.50e+00 / 1.05e+00 | 20260505 23:34:45.787 | INFO | PerfReport: 7.11%, PulseAccumulating -- 2.23e+00 / 2.76e+00, 4.24e-01 / 2.29e-01 | 20260505 23:34:45.787 | INFO | PerfReport: 3.10%, Metrics -- 9.73e-01 / 1.24e+00, 2.65e-01 / 1.12e-01 | 20260505 23:34:45.787 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.38e-01 / 3.95e-01, 5.89e-02 / 3.56e-02 | 20260505 23:34:45.787 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 23:34:45.787 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:34:45.787 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:34:45.787 | INFO | PerfReport: Full Execution -- 9.85e+01 / -nan, 2.47e+01 / 0.00e+00 | 20260505 23:34:45.787 | INFO | PerfReport: 22.34%, Upload -- 2.20e+01 / 0.00e+00, 7.56e+00 / 0.00e+00 | 20260505 23:34:45.787 | INFO | PerfReport: 21.85%, Download -- 2.15e+01 / 0.00e+00, 7.61e+00 / 0.00e+00 | 20260505 23:34:45.787 | INFO | PerfReport: 31.84%, Compute -- 3.14e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 23:34:45.787 | INFO | PerfReport: 2.11%, LockUpload -- 2.07e+00 / 0.00e+00, 7.79e+00 / 0.00e+00 | 20260505 23:34:45.787 | INFO | PerfReport: 1.35%, LockDownload -- 1.33e+00 / 0.00e+00, 2.94e+00 / 0.00e+00 | 20260505 23:34:45.787 | INFO | PerfReport: 20.50%, LockCompute -- 2.02e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260505 23:34:45.787 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.58e-02 / 0.00e+00 | 20260505 23:34:45.915 | INFO | Starting a round of Baz IO | 20260505 23:34:49.060 | INFO | Populating RTMetrics report | 20260505 23:34:49.060 | INFO | RTMetrics report populated | 20260505 23:34:49.060 | INFO | Opening tmp RTMetrics json file | 20260505 23:34:49.061 | INFO | Serializing RTMetrics to json | 20260505 23:34:49.062 | INFO | Serialization done, swapping tmp file now... | 20260505 23:34:49.063 | INFO | RTMetrics done | 20260505 23:34:49.140 | INFO | Finished a round of Baz IO: Wrote 1.889 GiB in 3.224 seconds | 20260505 23:34:49.140 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.42% / 0.00% / 78.58% | 20260505 23:34:49.140 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.104443% / 0% / 7.54992e-05% | 20260505 23:34:49.140 | INFO | Baz IO overall: | 20260505 23:34:49.140 | INFO | BytesWritten : 4110139980154 | 20260505 23:34:49.140 | INFO | NumEvents : 1936129398940 | 20260505 23:34:50.921 | INFO | Analyzing chunk frames = [7192576,7193088) | 20260505 23:34:50.921 | INFO | MemInfo: { | 20260505 23:34:50.921 | INFO | "gpu" : " 17 GB", | 20260505 23:34:50.922 | INFO | "huge" : "", | 20260505 23:34:50.922 | INFO | "huge_shared" : " 27 GB", | 20260505 23:34:50.922 | INFO | "malloc" : " 14 GB", | 20260505 23:34:50.922 | INFO | "pinned" : " 29 GB" | 20260505 23:34:50.922 | INFO | } | 20260505 23:34:50.922 | INFO | Perf report for the last 5.13579 seconds: | 20260505 23:34:50.922 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:34:50.922 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:50.922 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:50.922 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:50.922 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 23:34:50.922 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:50.922 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260505 23:34:50.922 | INFO | PRE_HQ : 59.9%, 1.7, 54.1ms, 0 | 20260505 23:34:50.922 | INFO | BAZWRITER : 62.1%, 3.6, 114.0ms, 0 | 20260505 23:34:50.922 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 23:34:50.922 | INFO | Uploaded 36GiB to GPU in 2.16995s (16.5902GiB/s) | 20260505 23:34:50.922 | INFO | Downloaded 38GiB to GPU in 2.23887s (16.9729GiB/s) | 20260505 23:34:56.064 | INFO | Analyzing chunk frames = [7193088,7193600) | 20260505 23:34:56.065 | INFO | MemInfo: { | 20260505 23:34:56.065 | INFO | "gpu" : " 17 GB", | 20260505 23:34:56.065 | INFO | "huge" : "", | 20260505 23:34:56.065 | INFO | "huge_shared" : " 27 GB", | 20260505 23:34:56.065 | INFO | "malloc" : " 14 GB", | 20260505 23:34:56.065 | INFO | "pinned" : " 29 GB" | 20260505 23:34:56.065 | INFO | } | 20260505 23:34:56.065 | INFO | Perf report for the last 5.14313 seconds: | 20260505 23:34:56.065 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:34:56.065 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:56.065 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:56.065 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:56.065 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260505 23:34:56.065 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:56.065 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:56.065 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:34:56.065 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:56.065 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:34:56.065 | INFO | Uploaded 36GiB to GPU in 2.07658s (17.3362GiB/s) | 20260505 23:34:56.065 | INFO | Downloaded 34GiB to GPU in 2.02708s (16.7729GiB/s) | 20260505 23:35:01.207 | INFO | Analyzing chunk frames = [7193600,7194112) | 20260505 23:35:01.207 | INFO | MemInfo: { | 20260505 23:35:01.207 | INFO | "gpu" : " 17 GB", | 20260505 23:35:01.207 | INFO | "huge" : "", | 20260505 23:35:01.207 | INFO | "huge_shared" : " 27 GB", | 20260505 23:35:01.207 | INFO | "malloc" : " 14 GB", | 20260505 23:35:01.207 | INFO | "pinned" : " 29 GB" | 20260505 23:35:01.207 | INFO | } | 20260505 23:35:01.207 | INFO | Perf report for the last 5.1427 seconds: | 20260505 23:35:01.207 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:35:01.207 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:01.207 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:01.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:01.207 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260505 23:35:01.207 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:01.207 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:01.207 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:35:01.207 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:01.207 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:01.208 | INFO | Uploaded 36GiB to GPU in 2.09094s (17.2172GiB/s) | 20260505 23:35:01.208 | INFO | Downloaded 34GiB to GPU in 2.03441s (16.7125GiB/s) | 20260505 23:35:06.343 | INFO | Analyzing chunk frames = [7194112,7194624) | 20260505 23:35:06.343 | INFO | MemInfo: { | 20260505 23:35:06.343 | INFO | "gpu" : " 17 GB", | 20260505 23:35:06.343 | INFO | "huge" : "", | 20260505 23:35:06.343 | INFO | "huge_shared" : " 27 GB", | 20260505 23:35:06.343 | INFO | "malloc" : " 14 GB", | 20260505 23:35:06.343 | INFO | "pinned" : " 29 GB" | 20260505 23:35:06.343 | INFO | } | 20260505 23:35:06.343 | INFO | Perf report for the last 5.13606 seconds: | 20260505 23:35:06.344 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:35:06.344 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:06.344 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:06.344 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:06.344 | INFO | ANALYSIS : 61.8%, 3.0, 95.9ms, 0 | 20260505 23:35:06.344 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:06.344 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:06.344 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:35:06.344 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:06.344 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:06.344 | INFO | Uploaded 36GiB to GPU in 2.1317s (16.888GiB/s) | 20260505 23:35:06.344 | INFO | Downloaded 34GiB to GPU in 2.01758s (16.8519GiB/s) | 20260505 23:35:11.481 | INFO | Analyzing chunk frames = [7194624,7195136) | 20260505 23:35:11.481 | INFO | MemInfo: { | 20260505 23:35:11.481 | INFO | "gpu" : " 17 GB", | 20260505 23:35:11.481 | INFO | "huge" : "", | 20260505 23:35:11.481 | INFO | "huge_shared" : " 27 GB", | 20260505 23:35:11.481 | INFO | "malloc" : " 14 GB", | 20260505 23:35:11.481 | INFO | "pinned" : " 29 GB" | 20260505 23:35:11.481 | INFO | } | 20260505 23:35:11.481 | INFO | Perf report for the last 5.13785 seconds: | 20260505 23:35:11.481 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:35:11.481 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:11.481 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:11.481 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:11.481 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260505 23:35:11.481 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:11.481 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:11.481 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:35:11.481 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:11.481 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:11.482 | INFO | Uploaded 36GiB to GPU in 2.09737s (17.1644GiB/s) | 20260505 23:35:11.482 | INFO | Downloaded 34GiB to GPU in 2.04439s (16.6309GiB/s) | 20260505 23:35:16.622 | INFO | Analyzing chunk frames = [7195136,7195648) | 20260505 23:35:16.622 | INFO | MemInfo: { | 20260505 23:35:16.622 | INFO | "gpu" : " 17 GB", | 20260505 23:35:16.622 | INFO | "huge" : "", | 20260505 23:35:16.622 | INFO | "huge_shared" : " 27 GB", | 20260505 23:35:16.622 | INFO | "malloc" : " 14 GB", | 20260505 23:35:16.622 | INFO | "pinned" : " 29 GB" | 20260505 23:35:16.622 | INFO | } | 20260505 23:35:16.622 | INFO | Perf report for the last 5.14042 seconds: | 20260505 23:35:16.622 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:35:16.622 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:16.622 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:16.622 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:16.622 | INFO | ANALYSIS : 61.4%, 3.0, 95.9ms, 0 | 20260505 23:35:16.622 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:16.622 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:16.622 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:35:16.622 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:16.622 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:16.622 | INFO | Uploaded 36GiB to GPU in 2.07021s (17.3895GiB/s) | 20260505 23:35:16.622 | INFO | Downloaded 34GiB to GPU in 2.03396s (16.7161GiB/s) | 20260505 23:35:21.763 | INFO | Analyzing chunk frames = [7195648,7196160) | 20260505 23:35:21.763 | INFO | MemInfo: { | 20260505 23:35:21.763 | INFO | "gpu" : " 17 GB", | 20260505 23:35:21.763 | INFO | "huge" : "", | 20260505 23:35:21.763 | INFO | "huge_shared" : " 27 GB", | 20260505 23:35:21.763 | INFO | "malloc" : " 14 GB", | 20260505 23:35:21.763 | INFO | "pinned" : " 29 GB" | 20260505 23:35:21.763 | INFO | } | 20260505 23:35:21.763 | INFO | Perf report for the last 5.14107 seconds: | 20260505 23:35:21.763 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:35:21.763 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:21.763 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:21.763 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:21.763 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 23:35:21.763 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:21.763 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:21.763 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 23:35:21.763 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:21.763 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:21.763 | INFO | Uploaded 36GiB to GPU in 2.01337s (17.8804GiB/s) | 20260505 23:35:21.763 | INFO | Downloaded 34GiB to GPU in 2.03838s (16.6799GiB/s) | 20260505 23:35:26.902 | INFO | Analyzing chunk frames = [7196160,7196672) | 20260505 23:35:26.902 | INFO | MemInfo: { | 20260505 23:35:26.902 | INFO | "gpu" : " 17 GB", | 20260505 23:35:26.902 | INFO | "huge" : "", | 20260505 23:35:26.902 | INFO | "huge_shared" : " 27 GB", | 20260505 23:35:26.902 | INFO | "malloc" : " 14 GB", | 20260505 23:35:26.902 | INFO | "pinned" : " 29 GB" | 20260505 23:35:26.902 | INFO | } | 20260505 23:35:26.902 | INFO | Perf report for the last 5.13906 seconds: | 20260505 23:35:26.902 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:35:26.902 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:26.902 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:26.902 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:26.902 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 23:35:26.902 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:26.902 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:26.902 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:35:26.902 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:26.902 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:26.902 | INFO | Uploaded 36GiB to GPU in 2.13786s (16.8393GiB/s) | 20260505 23:35:26.902 | INFO | Downloaded 34GiB to GPU in 2.02542s (16.7866GiB/s) | 20260505 23:35:27.020 | INFO | Starting a round of Baz IO | 20260505 23:35:30.170 | INFO | Populating RTMetrics report | 20260505 23:35:30.171 | INFO | RTMetrics report populated | 20260505 23:35:30.171 | INFO | Opening tmp RTMetrics json file | 20260505 23:35:30.171 | INFO | Serializing RTMetrics to json | 20260505 23:35:30.172 | INFO | Serialization done, swapping tmp file now... | 20260505 23:35:30.173 | INFO | RTMetrics done | 20260505 23:35:30.249 | INFO | Finished a round of Baz IO: Wrote 1.891 GiB in 3.229 seconds | 20260505 23:35:30.249 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.42% / 0.00% / 78.58% | 20260505 23:35:30.249 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.10529% / 3.97364e-06% / 6.75519e-05% | 20260505 23:35:30.249 | INFO | Baz IO overall: | 20260505 23:35:30.249 | INFO | BytesWritten : 4112170237885 | 20260505 23:35:30.249 | INFO | NumEvents : 1937060907393 | 20260505 23:35:32.043 | INFO | Analyzing chunk frames = [7196672,7197184) | 20260505 23:35:32.043 | INFO | MemInfo: { | 20260505 23:35:32.043 | INFO | "gpu" : " 17 GB", | 20260505 23:35:32.043 | INFO | "huge" : "", | 20260505 23:35:32.043 | INFO | "huge_shared" : " 27 GB", | 20260505 23:35:32.043 | INFO | "malloc" : " 14 GB", | 20260505 23:35:32.043 | INFO | "pinned" : " 29 GB" | 20260505 23:35:32.043 | INFO | } | 20260505 23:35:32.043 | INFO | Perf report for the last 5.14095 seconds: | 20260505 23:35:32.043 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:35:32.043 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:32.043 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:32.043 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:32.043 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 23:35:32.043 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:32.043 | INFO | RT_METRICS : 43.4%, 1.0, 22.8ms, 0 | 20260505 23:35:32.043 | INFO | PRE_HQ : 59.8%, 1.7, 54.3ms, 0 | 20260505 23:35:32.043 | INFO | BAZWRITER : 62.0%, 3.6, 113.4ms, 0 | 20260505 23:35:32.043 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 23:35:32.043 | INFO | Uploaded 36GiB to GPU in 2.14305s (16.7985GiB/s) | 20260505 23:35:32.043 | INFO | Downloaded 38GiB to GPU in 2.23194s (17.0256GiB/s) | 20260505 23:35:37.174 | INFO | Analyzing chunk frames = [7197184,7197696) | 20260505 23:35:37.174 | INFO | MemInfo: { | 20260505 23:35:37.174 | INFO | "gpu" : " 17 GB", | 20260505 23:35:37.174 | INFO | "huge" : "", | 20260505 23:35:37.174 | INFO | "huge_shared" : " 27 GB", | 20260505 23:35:37.174 | INFO | "malloc" : " 14 GB", | 20260505 23:35:37.174 | INFO | "pinned" : " 29 GB" | 20260505 23:35:37.174 | INFO | } | 20260505 23:35:37.174 | INFO | Perf report for the last 5.13139 seconds: | 20260505 23:35:37.174 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:35:37.174 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:37.174 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:37.174 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:37.174 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260505 23:35:37.174 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:37.174 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:37.174 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260505 23:35:37.174 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:37.174 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:37.175 | INFO | Uploaded 36GiB to GPU in 2.08104s (17.299GiB/s) | 20260505 23:35:37.175 | INFO | Downloaded 34GiB to GPU in 2.02672s (16.7759GiB/s) | 20260505 23:35:42.316 | INFO | Analyzing chunk frames = [7197696,7198208) | 20260505 23:35:42.316 | INFO | MemInfo: { | 20260505 23:35:42.316 | INFO | "gpu" : " 17 GB", | 20260505 23:35:42.316 | INFO | "huge" : "", | 20260505 23:35:42.316 | INFO | "huge_shared" : " 27 GB", | 20260505 23:35:42.316 | INFO | "malloc" : " 14 GB", | 20260505 23:35:42.316 | INFO | "pinned" : " 29 GB" | 20260505 23:35:42.316 | INFO | } | 20260505 23:35:42.316 | INFO | Perf report for the last 5.14209 seconds: | 20260505 23:35:42.316 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:35:42.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:42.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:42.316 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:42.316 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260505 23:35:42.316 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:42.316 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:42.316 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260505 23:35:42.316 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:42.316 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:42.317 | INFO | Uploaded 36GiB to GPU in 2.13764s (16.841GiB/s) | 20260505 23:35:42.317 | INFO | Downloaded 34GiB to GPU in 2.0298s (16.7504GiB/s) | 20260505 23:35:47.456 | INFO | Analyzing chunk frames = [7198208,7198720) | 20260505 23:35:47.456 | INFO | MemInfo: { | 20260505 23:35:47.456 | INFO | "gpu" : " 17 GB", | 20260505 23:35:47.456 | INFO | "huge" : "", | 20260505 23:35:47.456 | INFO | "huge_shared" : " 27 GB", | 20260505 23:35:47.456 | INFO | "malloc" : " 14 GB", | 20260505 23:35:47.456 | INFO | "pinned" : " 29 GB" | 20260505 23:35:47.456 | INFO | } | 20260505 23:35:47.456 | INFO | Perf report for the last 5.13986 seconds: | 20260505 23:35:47.456 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:35:47.456 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:47.456 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:47.456 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:47.456 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260505 23:35:47.456 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:47.456 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:47.456 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:35:47.456 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:47.456 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:47.456 | INFO | Uploaded 36GiB to GPU in 2.13272s (16.8798GiB/s) | 20260505 23:35:47.456 | INFO | Downloaded 34GiB to GPU in 2.00959s (16.9189GiB/s) | 20260505 23:35:52.595 | INFO | Analyzing chunk frames = [7198720,7199232) | 20260505 23:35:52.595 | INFO | MemInfo: { | 20260505 23:35:52.595 | INFO | "gpu" : " 17 GB", | 20260505 23:35:52.595 | INFO | "huge" : "", | 20260505 23:35:52.595 | INFO | "huge_shared" : " 27 GB", | 20260505 23:35:52.595 | INFO | "malloc" : " 14 GB", | 20260505 23:35:52.595 | INFO | "pinned" : " 29 GB" | 20260505 23:35:52.595 | INFO | } | 20260505 23:35:52.595 | INFO | Perf report for the last 5.13909 seconds: | 20260505 23:35:52.595 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:35:52.595 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:52.595 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:52.595 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:52.595 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260505 23:35:52.595 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:52.595 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:52.595 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:35:52.596 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:52.596 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:52.596 | INFO | Uploaded 36GiB to GPU in 2.10086s (17.1359GiB/s) | 20260505 23:35:52.596 | INFO | Downloaded 34GiB to GPU in 2.04906s (16.593GiB/s) | 20260505 23:35:57.733 | INFO | Analyzing chunk frames = [7199232,7199744) | 20260505 23:35:57.733 | INFO | MemInfo: { | 20260505 23:35:57.733 | INFO | "gpu" : " 17 GB", | 20260505 23:35:57.733 | INFO | "huge" : "", | 20260505 23:35:57.733 | INFO | "huge_shared" : " 27 GB", | 20260505 23:35:57.733 | INFO | "malloc" : " 14 GB", | 20260505 23:35:57.733 | INFO | "pinned" : " 29 GB" | 20260505 23:35:57.733 | INFO | } | 20260505 23:35:57.733 | INFO | Perf report for the last 5.13741 seconds: | 20260505 23:35:57.733 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:35:57.733 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:57.733 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:57.733 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:57.733 | INFO | ANALYSIS : 61.5%, 3.0, 96.0ms, 0 | 20260505 23:35:57.733 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:57.733 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:57.733 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260505 23:35:57.733 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:57.733 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:35:57.733 | INFO | Uploaded 36GiB to GPU in 2.07459s (17.3528GiB/s) | 20260505 23:35:57.733 | INFO | Downloaded 34GiB to GPU in 2.03714s (16.6901GiB/s) | 20260505 23:36:02.874 | INFO | Analyzing chunk frames = [7199744,7200256) | 20260505 23:36:02.874 | INFO | MemInfo: { | 20260505 23:36:02.874 | INFO | "gpu" : " 17 GB", | 20260505 23:36:02.874 | INFO | "huge" : "", | 20260505 23:36:02.874 | INFO | "huge_shared" : " 27 GB", | 20260505 23:36:02.874 | INFO | "malloc" : " 14 GB", | 20260505 23:36:02.874 | INFO | "pinned" : " 29 GB" | 20260505 23:36:02.874 | INFO | } | 20260505 23:36:02.874 | INFO | Perf report for the last 5.14112 seconds: | 20260505 23:36:02.874 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:36:02.874 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:02.874 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:02.874 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:02.874 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 23:36:02.874 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:02.874 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:02.874 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 23:36:02.874 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:02.874 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:02.874 | INFO | Uploaded 36GiB to GPU in 2.13671s (16.8483GiB/s) | 20260505 23:36:02.874 | INFO | Downloaded 34GiB to GPU in 2.1066s (16.1398GiB/s) | 20260505 23:36:08.017 | INFO | Analyzing chunk frames = [7200256,7200768) | 20260505 23:36:08.017 | INFO | MemInfo: { | 20260505 23:36:08.017 | INFO | "gpu" : " 17 GB", | 20260505 23:36:08.017 | INFO | "huge" : "", | 20260505 23:36:08.017 | INFO | "huge_shared" : " 27 GB", | 20260505 23:36:08.017 | INFO | "malloc" : " 14 GB", | 20260505 23:36:08.017 | INFO | "pinned" : " 29 GB" | 20260505 23:36:08.017 | INFO | } | 20260505 23:36:08.017 | INFO | Perf report for the last 5.14327 seconds: | 20260505 23:36:08.017 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:36:08.017 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:08.017 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:08.017 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:08.017 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 23:36:08.017 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:08.017 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:08.017 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 23:36:08.017 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:08.017 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:08.018 | INFO | Uploaded 36GiB to GPU in 2.11501s (17.0212GiB/s) | 20260505 23:36:08.018 | INFO | Downloaded 34GiB to GPU in 2.00821s (16.9305GiB/s) | 20260505 23:36:08.018 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 23:36:08.018 | INFO | Total mem usage: 15.3 GiB | 20260505 23:36:08.018 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 23:36:08.018 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.3%) | 20260505 23:36:08.018 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 23:36:08.018 | INFO | High Water Breakdown: | 20260505 23:36:08.018 | INFO | 61% : 8.63 GiB ( 90.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 23:36:08.018 | INFO | 38.9% : 5.5 GiB ( 82.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 23:36:08.018 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 23:36:08.018 | INFO | ------------------------------------------ | 20260505 23:36:08.018 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:36:08.018 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 23:36:08.018 | INFO | Total mem usage: 26.5 GiB | 20260505 23:36:08.018 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 23:36:08.018 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 23:36:08.018 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 23:36:08.018 | INFO | High Water Breakdown: | 20260505 23:36:08.018 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:36:08.018 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:36:08.018 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:36:08.018 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:36:08.018 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:36:08.018 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:36:08.018 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 23:36:08.018 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 23:36:08.018 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:36:08.018 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 23:36:08.018 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 23:36:08.018 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 23:36:08.018 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 23:36:08.018 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 23:36:08.018 | INFO | ------------------------------------------ | 20260505 23:36:08.018 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:36:08.018 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 23:36:08.018 | INFO | Total mem usage: 25.6 GiB | 20260505 23:36:08.018 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 23:36:08.018 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 23:36:08.018 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 23:36:08.018 | INFO | High Water Breakdown: | 20260505 23:36:08.018 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 23:36:08.018 | INFO | ------------------------------------------ | 20260505 23:36:08.018 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:36:08.018 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 23:36:08.018 | INFO | Total mem usage: 15.7 GiB | 20260505 23:36:08.018 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 23:36:08.018 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 23:36:08.018 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 23:36:08.018 | INFO | High Water Breakdown: | 20260505 23:36:08.018 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 23:36:08.018 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 23:36:08.018 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 23:36:08.018 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 23:36:08.018 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 23:36:08.018 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 23:36:08.018 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 23:36:08.018 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 23:36:08.018 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 23:36:08.018 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:36:08.018 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 23:36:08.019 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:36:08.019 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 23:36:08.019 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 23:36:08.019 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 23:36:08.019 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 23:36:08.019 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:36:08.019 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 23:36:08.019 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:36:08.019 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:36:08.019 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:36:08.019 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 23:36:08.019 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 23:36:08.019 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:36:08.019 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 23:36:08.019 | INFO | ------------------------------------------ | 20260505 23:36:08.019 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:36:08.019 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 23:36:08.019 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:36:08.019 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:36:08.019 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.40e+01, 1.35e+01 / 7.19e-01 | 20260505 23:36:08.019 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.04e-01, 1.09e-02 / 3.37e-03 | 20260505 23:36:08.019 | INFO | PerfReport: 11.83%, Baseline -- 3.71e+00 / 4.53e+00, 5.69e-01 / 1.98e-01 | 20260505 23:36:08.019 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.55e+00, 1.92e-01 / 6.11e-02 | 20260505 23:36:08.019 | INFO | PerfReport: 16.01%, DME -- 4.01e+01 / 4.40e+01, 3.79e+00 / 8.32e-01 | 20260505 23:36:08.019 | INFO | PerfReport: 57.27%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.48e+00 / 1.04e+00 | 20260505 23:36:08.019 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.23e+00 / 2.75e+00, 4.26e-01 / 2.36e-01 | 20260505 23:36:08.019 | INFO | PerfReport: 3.11%, Metrics -- 9.76e-01 / 1.27e+00, 2.66e-01 / 1.41e-01 | 20260505 23:36:08.019 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.38e-01 / 3.95e-01, 5.84e-02 / 3.13e-02 | 20260505 23:36:08.019 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 23:36:08.019 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:36:08.019 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:36:08.019 | INFO | PerfReport: Full Execution -- 9.85e+01 / -nan, 2.47e+01 / 0.00e+00 | 20260505 23:36:08.019 | INFO | PerfReport: 22.28%, Upload -- 2.19e+01 / 0.00e+00, 7.54e+00 / 0.00e+00 | 20260505 23:36:08.019 | INFO | PerfReport: 21.79%, Download -- 2.15e+01 / 0.00e+00, 7.60e+00 / 0.00e+00 | 20260505 23:36:08.019 | INFO | PerfReport: 31.83%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 23:36:08.019 | INFO | PerfReport: 2.16%, LockUpload -- 2.13e+00 / 0.00e+00, 7.89e+00 / 0.00e+00 | 20260505 23:36:08.019 | INFO | PerfReport: 1.30%, LockDownload -- 1.28e+00 / 0.00e+00, 2.85e+00 / 0.00e+00 | 20260505 23:36:08.019 | INFO | PerfReport: 20.62%, LockCompute -- 2.03e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260505 23:36:08.019 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.93e-02 / 0.00e+00 | 20260505 23:36:08.137 | INFO | Starting a round of Baz IO | 20260505 23:36:11.283 | INFO | Populating RTMetrics report | 20260505 23:36:11.283 | INFO | RTMetrics report populated | 20260505 23:36:11.283 | INFO | Opening tmp RTMetrics json file | 20260505 23:36:11.285 | INFO | Serializing RTMetrics to json | 20260505 23:36:11.286 | INFO | Serialization done, swapping tmp file now... | 20260505 23:36:11.287 | INFO | RTMetrics done | 20260505 23:36:11.368 | INFO | Finished a round of Baz IO: Wrote 1.889 GiB in 3.231 seconds | 20260505 23:36:11.368 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.42% / 0.00% / 78.58% | 20260505 23:36:11.368 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.104451% / 0% / 5.5631e-05% | 20260505 23:36:11.368 | INFO | Baz IO overall: | 20260505 23:36:11.368 | INFO | BytesWritten : 4114199027794 | 20260505 23:36:11.368 | INFO | NumEvents : 1937991478466 | 20260505 23:36:13.159 | INFO | Analyzing chunk frames = [7200768,7201280) | 20260505 23:36:13.159 | INFO | MemInfo: { | 20260505 23:36:13.159 | INFO | "gpu" : " 17 GB", | 20260505 23:36:13.159 | INFO | "huge" : "", | 20260505 23:36:13.159 | INFO | "huge_shared" : " 27 GB", | 20260505 23:36:13.159 | INFO | "malloc" : " 14 GB", | 20260505 23:36:13.159 | INFO | "pinned" : " 29 GB" | 20260505 23:36:13.159 | INFO | } | 20260505 23:36:13.159 | INFO | Perf report for the last 5.14138 seconds: | 20260505 23:36:13.159 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:36:13.159 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:13.159 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:13.159 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:13.159 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260505 23:36:13.159 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:13.159 | INFO | RT_METRICS : 43.4%, 1.0, 22.8ms, 0 | 20260505 23:36:13.159 | INFO | PRE_HQ : 59.8%, 1.8, 54.4ms, 0 | 20260505 23:36:13.159 | INFO | BAZWRITER : 62.0%, 3.6, 115.7ms, 0 | 20260505 23:36:13.159 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 23:36:13.159 | INFO | Uploaded 36GiB to GPU in 2.17134s (16.5796GiB/s) | 20260505 23:36:13.159 | INFO | Downloaded 38GiB to GPU in 2.23802s (16.9793GiB/s) | 20260505 23:36:18.299 | INFO | Analyzing chunk frames = [7201280,7201792) | 20260505 23:36:18.299 | INFO | MemInfo: { | 20260505 23:36:18.299 | INFO | "gpu" : " 17 GB", | 20260505 23:36:18.299 | INFO | "huge" : "", | 20260505 23:36:18.299 | INFO | "huge_shared" : " 27 GB", | 20260505 23:36:18.299 | INFO | "malloc" : " 14 GB", | 20260505 23:36:18.299 | INFO | "pinned" : " 29 GB" | 20260505 23:36:18.299 | INFO | } | 20260505 23:36:18.299 | INFO | Perf report for the last 5.14041 seconds: | 20260505 23:36:18.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:36:18.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:18.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:18.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:18.299 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 23:36:18.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:18.299 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:18.299 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260505 23:36:18.299 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:18.299 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:18.299 | INFO | Uploaded 36GiB to GPU in 2.07995s (17.3081GiB/s) | 20260505 23:36:18.299 | INFO | Downloaded 34GiB to GPU in 2.02861s (16.7602GiB/s) | 20260505 23:36:23.431 | INFO | Analyzing chunk frames = [7201792,7202304) | 20260505 23:36:23.431 | INFO | MemInfo: { | 20260505 23:36:23.431 | INFO | "gpu" : " 17 GB", | 20260505 23:36:23.431 | INFO | "huge" : "", | 20260505 23:36:23.431 | INFO | "huge_shared" : " 27 GB", | 20260505 23:36:23.431 | INFO | "malloc" : " 14 GB", | 20260505 23:36:23.431 | INFO | "pinned" : " 29 GB" | 20260505 23:36:23.431 | INFO | } | 20260505 23:36:23.431 | INFO | Perf report for the last 5.13178 seconds: | 20260505 23:36:23.431 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:36:23.431 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:23.431 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:23.431 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:23.431 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260505 23:36:23.431 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:23.431 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:23.431 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 23:36:23.431 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:23.431 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:23.431 | INFO | Uploaded 36GiB to GPU in 2.08338s (17.2796GiB/s) | 20260505 23:36:23.431 | INFO | Downloaded 34GiB to GPU in 2.03215s (16.7311GiB/s) | 20260505 23:36:28.579 | INFO | Analyzing chunk frames = [7202304,7202816) | 20260505 23:36:28.579 | INFO | MemInfo: { | 20260505 23:36:28.579 | INFO | "gpu" : " 17 GB", | 20260505 23:36:28.579 | INFO | "huge" : "", | 20260505 23:36:28.579 | INFO | "huge_shared" : " 27 GB", | 20260505 23:36:28.579 | INFO | "malloc" : " 14 GB", | 20260505 23:36:28.579 | INFO | "pinned" : " 29 GB" | 20260505 23:36:28.579 | INFO | } | 20260505 23:36:28.579 | INFO | Perf report for the last 5.14852 seconds: | 20260505 23:36:28.579 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:36:28.579 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:28.579 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:28.579 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:28.579 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260505 23:36:28.579 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:28.579 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:28.579 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:36:28.579 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:28.579 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:28.580 | INFO | Uploaded 36GiB to GPU in 2.13101s (16.8934GiB/s) | 20260505 23:36:28.580 | INFO | Downloaded 34GiB to GPU in 2.01341s (16.8867GiB/s) | 20260505 23:36:33.712 | INFO | Analyzing chunk frames = [7202816,7203328) | 20260505 23:36:33.712 | INFO | MemInfo: { | 20260505 23:36:33.712 | INFO | "gpu" : " 17 GB", | 20260505 23:36:33.712 | INFO | "huge" : "", | 20260505 23:36:33.712 | INFO | "huge_shared" : " 27 GB", | 20260505 23:36:33.712 | INFO | "malloc" : " 14 GB", | 20260505 23:36:33.712 | INFO | "pinned" : " 29 GB" | 20260505 23:36:33.713 | INFO | } | 20260505 23:36:33.713 | INFO | Perf report for the last 5.13318 seconds: | 20260505 23:36:33.713 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:36:33.713 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:33.713 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:33.713 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:33.713 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260505 23:36:33.713 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:33.713 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:33.713 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:36:33.713 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:33.713 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:33.713 | INFO | Uploaded 36GiB to GPU in 2.09139s (17.2134GiB/s) | 20260505 23:36:33.713 | INFO | Downloaded 34GiB to GPU in 2.04358s (16.6374GiB/s) | 20260505 23:36:38.849 | INFO | Analyzing chunk frames = [7203328,7203840) | 20260505 23:36:38.849 | INFO | MemInfo: { | 20260505 23:36:38.849 | INFO | "gpu" : " 17 GB", | 20260505 23:36:38.849 | INFO | "huge" : "", | 20260505 23:36:38.849 | INFO | "huge_shared" : " 27 GB", | 20260505 23:36:38.849 | INFO | "malloc" : " 14 GB", | 20260505 23:36:38.849 | INFO | "pinned" : " 29 GB" | 20260505 23:36:38.849 | INFO | } | 20260505 23:36:38.849 | INFO | Perf report for the last 5.13637 seconds: | 20260505 23:36:38.849 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:36:38.849 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:38.849 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:38.849 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:38.849 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260505 23:36:38.849 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:38.849 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:38.849 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:36:38.849 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:38.849 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:38.849 | INFO | Uploaded 36GiB to GPU in 2.06432s (17.4391GiB/s) | 20260505 23:36:38.849 | INFO | Downloaded 34GiB to GPU in 2.03614s (16.6982GiB/s) | 20260505 23:36:43.991 | INFO | Analyzing chunk frames = [7203840,7204352) | 20260505 23:36:43.991 | INFO | MemInfo: { | 20260505 23:36:43.991 | INFO | "gpu" : " 17 GB", | 20260505 23:36:43.991 | INFO | "huge" : "", | 20260505 23:36:43.991 | INFO | "huge_shared" : " 27 GB", | 20260505 23:36:43.991 | INFO | "malloc" : " 14 GB", | 20260505 23:36:43.991 | INFO | "pinned" : " 29 GB" | 20260505 23:36:43.991 | INFO | } | 20260505 23:36:43.991 | INFO | Perf report for the last 5.14198 seconds: | 20260505 23:36:43.991 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:36:43.991 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:43.991 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:43.991 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:43.991 | INFO | ANALYSIS : 62.5%, 3.0, 97.7ms, 0 | 20260505 23:36:43.991 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:43.991 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:43.991 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 23:36:43.991 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:43.991 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:43.991 | INFO | Uploaded 36GiB to GPU in 2.14674s (16.7696GiB/s) | 20260505 23:36:43.991 | INFO | Downloaded 34GiB to GPU in 2.12897s (15.9701GiB/s) | 20260505 23:36:49.129 | INFO | Analyzing chunk frames = [7204352,7204864) | 20260505 23:36:49.129 | INFO | MemInfo: { | 20260505 23:36:49.129 | INFO | "gpu" : " 17 GB", | 20260505 23:36:49.129 | INFO | "huge" : "", | 20260505 23:36:49.129 | INFO | "huge_shared" : " 27 GB", | 20260505 23:36:49.129 | INFO | "malloc" : " 14 GB", | 20260505 23:36:49.129 | INFO | "pinned" : " 29 GB" | 20260505 23:36:49.129 | INFO | } | 20260505 23:36:49.129 | INFO | Perf report for the last 5.13802 seconds: | 20260505 23:36:49.129 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:36:49.129 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:49.129 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:49.129 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:49.129 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 23:36:49.129 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:49.129 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:49.129 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:36:49.129 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:49.129 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:49.129 | INFO | Uploaded 36GiB to GPU in 2.10672s (17.0881GiB/s) | 20260505 23:36:49.129 | INFO | Downloaded 34GiB to GPU in 2.00697s (16.941GiB/s) | 20260505 23:36:49.253 | INFO | Starting a round of Baz IO | 20260505 23:36:52.401 | INFO | Populating RTMetrics report | 20260505 23:36:52.401 | INFO | RTMetrics report populated | 20260505 23:36:52.401 | INFO | Opening tmp RTMetrics json file | 20260505 23:36:52.402 | INFO | Serializing RTMetrics to json | 20260505 23:36:52.403 | INFO | Serialization done, swapping tmp file now... | 20260505 23:36:52.404 | INFO | RTMetrics done | 20260505 23:36:52.483 | INFO | Finished a round of Baz IO: Wrote 1.89 GiB in 3.23 seconds | 20260505 23:36:52.483 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.42% / 0.00% / 78.58% | 20260505 23:36:52.483 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.104292% / 3.97364e-06% / 8.74201e-05% | 20260505 23:36:52.483 | INFO | Baz IO overall: | 20260505 23:36:52.483 | INFO | BytesWritten : 4116228326092 | 20260505 23:36:52.483 | INFO | NumEvents : 1938922150969 | 20260505 23:36:54.270 | INFO | Analyzing chunk frames = [7204864,7205376) | 20260505 23:36:54.270 | INFO | MemInfo: { | 20260505 23:36:54.270 | INFO | "gpu" : " 17 GB", | 20260505 23:36:54.270 | INFO | "huge" : "", | 20260505 23:36:54.270 | INFO | "huge_shared" : " 27 GB", | 20260505 23:36:54.270 | INFO | "malloc" : " 14 GB", | 20260505 23:36:54.270 | INFO | "pinned" : " 29 GB" | 20260505 23:36:54.270 | INFO | } | 20260505 23:36:54.270 | INFO | Perf report for the last 5.14148 seconds: | 20260505 23:36:54.270 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:36:54.271 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:54.271 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:54.271 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:54.271 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 23:36:54.271 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:54.271 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260505 23:36:54.271 | INFO | PRE_HQ : 59.5%, 1.8, 54.1ms, 0 | 20260505 23:36:54.271 | INFO | BAZWRITER : 62.0%, 3.6, 115.9ms, 0 | 20260505 23:36:54.271 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 23:36:54.271 | INFO | Uploaded 36GiB to GPU in 2.14077s (16.8163GiB/s) | 20260505 23:36:54.271 | INFO | Downloaded 38GiB to GPU in 2.23089s (17.0336GiB/s) | 20260505 23:36:59.409 | INFO | Analyzing chunk frames = [7205376,7205888) | 20260505 23:36:59.409 | INFO | MemInfo: { | 20260505 23:36:59.409 | INFO | "gpu" : " 17 GB", | 20260505 23:36:59.409 | INFO | "huge" : "", | 20260505 23:36:59.409 | INFO | "huge_shared" : " 27 GB", | 20260505 23:36:59.409 | INFO | "malloc" : " 14 GB", | 20260505 23:36:59.409 | INFO | "pinned" : " 29 GB" | 20260505 23:36:59.409 | INFO | } | 20260505 23:36:59.409 | INFO | Perf report for the last 5.13889 seconds: | 20260505 23:36:59.409 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:36:59.409 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:59.409 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:59.409 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:59.409 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260505 23:36:59.409 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:59.409 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:59.409 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260505 23:36:59.409 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:59.409 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:36:59.410 | INFO | Uploaded 36GiB to GPU in 2.10766s (17.0805GiB/s) | 20260505 23:36:59.410 | INFO | Downloaded 34GiB to GPU in 2.03821s (16.6813GiB/s) | 20260505 23:37:04.545 | INFO | Analyzing chunk frames = [7205888,7206400) | 20260505 23:37:04.545 | INFO | MemInfo: { | 20260505 23:37:04.545 | INFO | "gpu" : " 17 GB", | 20260505 23:37:04.545 | INFO | "huge" : "", | 20260505 23:37:04.545 | INFO | "huge_shared" : " 27 GB", | 20260505 23:37:04.545 | INFO | "malloc" : " 14 GB", | 20260505 23:37:04.545 | INFO | "pinned" : " 29 GB" | 20260505 23:37:04.545 | INFO | } | 20260505 23:37:04.545 | INFO | Perf report for the last 5.13574 seconds: | 20260505 23:37:04.545 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:37:04.545 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:04.545 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:04.545 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:04.545 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260505 23:37:04.545 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:04.545 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:04.545 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260505 23:37:04.545 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:04.545 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:04.545 | INFO | Uploaded 36GiB to GPU in 2.0887s (17.2356GiB/s) | 20260505 23:37:04.545 | INFO | Downloaded 34GiB to GPU in 2.03142s (16.737GiB/s) | 20260505 23:37:09.683 | INFO | Analyzing chunk frames = [7206400,7206912) | 20260505 23:37:09.683 | INFO | MemInfo: { | 20260505 23:37:09.683 | INFO | "gpu" : " 17 GB", | 20260505 23:37:09.683 | INFO | "huge" : "", | 20260505 23:37:09.683 | INFO | "huge_shared" : " 27 GB", | 20260505 23:37:09.683 | INFO | "malloc" : " 14 GB", | 20260505 23:37:09.683 | INFO | "pinned" : " 29 GB" | 20260505 23:37:09.683 | INFO | } | 20260505 23:37:09.683 | INFO | Perf report for the last 5.13801 seconds: | 20260505 23:37:09.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:37:09.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:09.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:09.683 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:09.683 | INFO | ANALYSIS : 61.7%, 3.0, 95.8ms, 0 | 20260505 23:37:09.683 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:09.683 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:09.683 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:37:09.683 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:09.683 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:09.683 | INFO | Uploaded 36GiB to GPU in 2.12648s (16.9294GiB/s) | 20260505 23:37:09.683 | INFO | Downloaded 34GiB to GPU in 2.01147s (16.9031GiB/s) | 20260505 23:37:14.825 | INFO | Analyzing chunk frames = [7206912,7207424) | 20260505 23:37:14.825 | INFO | MemInfo: { | 20260505 23:37:14.825 | INFO | "gpu" : " 17 GB", | 20260505 23:37:14.825 | INFO | "huge" : "", | 20260505 23:37:14.825 | INFO | "huge_shared" : " 27 GB", | 20260505 23:37:14.825 | INFO | "malloc" : " 14 GB", | 20260505 23:37:14.825 | INFO | "pinned" : " 29 GB" | 20260505 23:37:14.825 | INFO | } | 20260505 23:37:14.825 | INFO | Perf report for the last 5.14197 seconds: | 20260505 23:37:14.825 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:37:14.825 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:14.825 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:14.825 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:14.825 | INFO | ANALYSIS : 61.2%, 3.0, 95.6ms, 0 | 20260505 23:37:14.825 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:14.825 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:14.825 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260505 23:37:14.825 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:14.825 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:14.825 | INFO | Uploaded 36GiB to GPU in 2.09207s (17.2079GiB/s) | 20260505 23:37:14.825 | INFO | Downloaded 34GiB to GPU in 2.04701s (16.6096GiB/s) | 20260505 23:37:19.969 | INFO | Analyzing chunk frames = [7207424,7207936) | 20260505 23:37:19.969 | INFO | MemInfo: { | 20260505 23:37:19.969 | INFO | "gpu" : " 17 GB", | 20260505 23:37:19.969 | INFO | "huge" : "", | 20260505 23:37:19.969 | INFO | "huge_shared" : " 27 GB", | 20260505 23:37:19.969 | INFO | "malloc" : " 14 GB", | 20260505 23:37:19.969 | INFO | "pinned" : " 29 GB" | 20260505 23:37:19.969 | INFO | } | 20260505 23:37:19.969 | INFO | Perf report for the last 5.14404 seconds: | 20260505 23:37:19.969 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:37:19.969 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:19.969 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:19.969 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:19.969 | INFO | ANALYSIS : 61.2%, 3.0, 95.7ms, 0 | 20260505 23:37:19.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:19.969 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:19.969 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:37:19.969 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:19.969 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:19.969 | INFO | Uploaded 36GiB to GPU in 2.06965s (17.3943GiB/s) | 20260505 23:37:19.969 | INFO | Downloaded 34GiB to GPU in 2.03877s (16.6767GiB/s) | 20260505 23:37:25.108 | INFO | Analyzing chunk frames = [7207936,7208448) | 20260505 23:37:25.108 | INFO | MemInfo: { | 20260505 23:37:25.108 | INFO | "gpu" : " 17 GB", | 20260505 23:37:25.108 | INFO | "huge" : "", | 20260505 23:37:25.108 | INFO | "huge_shared" : " 27 GB", | 20260505 23:37:25.108 | INFO | "malloc" : " 14 GB", | 20260505 23:37:25.108 | INFO | "pinned" : " 29 GB" | 20260505 23:37:25.108 | INFO | } | 20260505 23:37:25.108 | INFO | Perf report for the last 5.13861 seconds: | 20260505 23:37:25.108 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:37:25.108 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:25.108 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:25.108 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:25.108 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 23:37:25.108 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:25.108 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:25.108 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260505 23:37:25.108 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:25.108 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:25.108 | INFO | Uploaded 36GiB to GPU in 2.01647s (17.8529GiB/s) | 20260505 23:37:25.108 | INFO | Downloaded 34GiB to GPU in 2.03913s (16.6738GiB/s) | 20260505 23:37:30.243 | INFO | Analyzing chunk frames = [7208448,7208960) | 20260505 23:37:30.243 | INFO | MemInfo: { | 20260505 23:37:30.243 | INFO | "gpu" : " 17 GB", | 20260505 23:37:30.244 | INFO | "huge" : "", | 20260505 23:37:30.244 | INFO | "huge_shared" : " 27 GB", | 20260505 23:37:30.244 | INFO | "malloc" : " 14 GB", | 20260505 23:37:30.244 | INFO | "pinned" : " 29 GB" | 20260505 23:37:30.244 | INFO | } | 20260505 23:37:30.244 | INFO | Perf report for the last 5.13591 seconds: | 20260505 23:37:30.244 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:37:30.244 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:30.244 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:30.244 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:30.244 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260505 23:37:30.244 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:30.244 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:30.244 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:37:30.244 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:30.244 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:30.244 | INFO | Uploaded 36GiB to GPU in 2.11048s (17.0577GiB/s) | 20260505 23:37:30.244 | INFO | Downloaded 34GiB to GPU in 1.9986s (17.0119GiB/s) | 20260505 23:37:30.244 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 23:37:30.244 | INFO | Total mem usage: 15.3 GiB | 20260505 23:37:30.244 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 23:37:30.244 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.29%) | 20260505 23:37:30.244 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 23:37:30.244 | INFO | High Water Breakdown: | 20260505 23:37:30.244 | INFO | 61% : 8.63 GiB ( 90.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 23:37:30.244 | INFO | 38.9% : 5.5 GiB ( 82.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 23:37:30.244 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 23:37:30.244 | INFO | ------------------------------------------ | 20260505 23:37:30.244 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:37:30.244 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 23:37:30.244 | INFO | Total mem usage: 26.5 GiB | 20260505 23:37:30.244 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 23:37:30.244 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 23:37:30.244 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 23:37:30.244 | INFO | High Water Breakdown: | 20260505 23:37:30.244 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:37:30.244 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:37:30.244 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:37:30.244 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:37:30.244 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:37:30.244 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:37:30.244 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 23:37:30.244 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 23:37:30.244 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:37:30.244 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 23:37:30.244 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 23:37:30.245 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 23:37:30.245 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 23:37:30.245 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 23:37:30.245 | INFO | ------------------------------------------ | 20260505 23:37:30.245 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:37:30.245 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 23:37:30.245 | INFO | Total mem usage: 25.6 GiB | 20260505 23:37:30.245 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 23:37:30.245 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 23:37:30.245 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 23:37:30.245 | INFO | High Water Breakdown: | 20260505 23:37:30.245 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 23:37:30.245 | INFO | ------------------------------------------ | 20260505 23:37:30.245 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:37:30.245 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 23:37:30.245 | INFO | Total mem usage: 15.7 GiB | 20260505 23:37:30.245 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 23:37:30.245 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 23:37:30.245 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 23:37:30.245 | INFO | High Water Breakdown: | 20260505 23:37:30.245 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 23:37:30.245 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 23:37:30.245 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 23:37:30.245 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 23:37:30.245 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 23:37:30.245 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 23:37:30.245 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 23:37:30.245 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 23:37:30.245 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 23:37:30.245 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:37:30.245 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 23:37:30.245 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:37:30.245 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 23:37:30.245 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 23:37:30.245 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 23:37:30.245 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 23:37:30.245 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:37:30.245 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 23:37:30.245 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:37:30.245 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:37:30.245 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:37:30.245 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 23:37:30.245 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 23:37:30.245 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:37:30.245 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 23:37:30.245 | INFO | ------------------------------------------ | 20260505 23:37:30.245 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:37:30.245 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 23:37:30.245 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:37:30.245 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:37:30.245 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.42e+01, 1.34e+01 / 1.32e+00 | 20260505 23:37:30.245 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.04e-01, 1.04e-02 / 4.41e-03 | 20260505 23:37:30.245 | INFO | PerfReport: 11.85%, Baseline -- 3.72e+00 / 4.53e+00, 5.78e-01 / 2.11e-01 | 20260505 23:37:30.245 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.52e+00, 1.91e-01 / 1.06e-01 | 20260505 23:37:30.245 | INFO | PerfReport: 16.00%, DME -- 4.02e+01 / 4.42e+01, 3.88e+00 / 1.34e+00 | 20260505 23:37:30.245 | INFO | PerfReport: 57.29%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.52e+00 / 1.12e+00 | 20260505 23:37:30.245 | INFO | PerfReport: 7.09%, PulseAccumulating -- 2.23e+00 / 2.77e+00, 4.23e-01 / 2.27e-01 | 20260505 23:37:30.245 | INFO | PerfReport: 3.11%, Metrics -- 9.75e-01 / 1.25e+00, 2.66e-01 / 1.02e-01 | 20260505 23:37:30.245 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.38e-01 / 3.93e-01, 5.86e-02 / 3.31e-02 | 20260505 23:37:30.245 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 23:37:30.245 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:37:30.245 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:37:30.245 | INFO | PerfReport: Full Execution -- 9.84e+01 / -nan, 2.47e+01 / 0.00e+00 | 20260505 23:37:30.245 | INFO | PerfReport: 22.25%, Upload -- 2.19e+01 / 0.00e+00, 7.44e+00 / 0.00e+00 | 20260505 23:37:30.245 | INFO | PerfReport: 21.81%, Download -- 2.15e+01 / 0.00e+00, 7.50e+00 / 0.00e+00 | 20260505 23:37:30.245 | INFO | PerfReport: 31.90%, Compute -- 3.14e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 23:37:30.245 | INFO | PerfReport: 2.09%, LockUpload -- 2.06e+00 / 0.00e+00, 7.79e+00 / 0.00e+00 | 20260505 23:37:30.245 | INFO | PerfReport: 1.34%, LockDownload -- 1.32e+00 / 0.00e+00, 2.95e+00 / 0.00e+00 | 20260505 23:37:30.245 | INFO | PerfReport: 20.59%, LockCompute -- 2.03e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260505 23:37:30.245 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.33e-02 / 0.00e+00 | 20260505 23:37:30.369 | INFO | Starting a round of Baz IO | 20260505 23:37:33.512 | INFO | Populating RTMetrics report | 20260505 23:37:33.512 | INFO | RTMetrics report populated | 20260505 23:37:33.512 | INFO | Opening tmp RTMetrics json file | 20260505 23:37:33.513 | INFO | Serializing RTMetrics to json | 20260505 23:37:33.514 | INFO | Serialization done, swapping tmp file now... | 20260505 23:37:33.515 | INFO | RTMetrics done | 20260505 23:37:33.592 | INFO | Finished a round of Baz IO: Wrote 1.886 GiB in 3.222 seconds | 20260505 23:37:33.592 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.42% / 0.00% / 78.58% | 20260505 23:37:33.592 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.103482% / 3.97364e-06% / 7.15256e-05% | 20260505 23:37:33.592 | INFO | Baz IO overall: | 20260505 23:37:33.592 | INFO | BytesWritten : 4118253151745 | 20260505 23:37:33.592 | INFO | NumEvents : 1939849639132 | 20260505 23:37:35.383 | INFO | Analyzing chunk frames = [7208960,7209472) | 20260505 23:37:35.383 | INFO | MemInfo: { | 20260505 23:37:35.383 | INFO | "gpu" : " 17 GB", | 20260505 23:37:35.383 | INFO | "huge" : "", | 20260505 23:37:35.383 | INFO | "huge_shared" : " 27 GB", | 20260505 23:37:35.383 | INFO | "malloc" : " 14 GB", | 20260505 23:37:35.383 | INFO | "pinned" : " 29 GB" | 20260505 23:37:35.383 | INFO | } | 20260505 23:37:35.383 | INFO | Perf report for the last 5.13956 seconds: | 20260505 23:37:35.383 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:37:35.383 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:35.383 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:35.383 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:35.383 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 23:37:35.383 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:35.383 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260505 23:37:35.383 | INFO | PRE_HQ : 59.9%, 1.7, 54.2ms, 0 | 20260505 23:37:35.383 | INFO | BAZWRITER : 61.9%, 3.6, 113.4ms, 0 | 20260505 23:37:35.383 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 23:37:35.383 | INFO | Uploaded 36GiB to GPU in 2.16614s (16.6194GiB/s) | 20260505 23:37:35.383 | INFO | Downloaded 38GiB to GPU in 2.24031s (16.962GiB/s) | 20260505 23:37:40.526 | INFO | Analyzing chunk frames = [7209472,7209984) | 20260505 23:37:40.526 | INFO | MemInfo: { | 20260505 23:37:40.526 | INFO | "gpu" : " 17 GB", | 20260505 23:37:40.526 | INFO | "huge" : "", | 20260505 23:37:40.526 | INFO | "huge_shared" : " 27 GB", | 20260505 23:37:40.526 | INFO | "malloc" : " 14 GB", | 20260505 23:37:40.526 | INFO | "pinned" : " 29 GB" | 20260505 23:37:40.526 | INFO | } | 20260505 23:37:40.526 | INFO | Perf report for the last 5.14294 seconds: | 20260505 23:37:40.526 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:37:40.526 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:40.526 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:40.526 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:40.526 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260505 23:37:40.526 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:40.526 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:40.526 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260505 23:37:40.526 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:40.526 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:40.526 | INFO | Uploaded 36GiB to GPU in 2.09007s (17.2243GiB/s) | 20260505 23:37:40.527 | INFO | Downloaded 34GiB to GPU in 2.03194s (16.7328GiB/s) | 20260505 23:37:45.662 | INFO | Analyzing chunk frames = [7209984,7210496) | 20260505 23:37:45.662 | INFO | MemInfo: { | 20260505 23:37:45.662 | INFO | "gpu" : " 17 GB", | 20260505 23:37:45.662 | INFO | "huge" : "", | 20260505 23:37:45.662 | INFO | "huge_shared" : " 27 GB", | 20260505 23:37:45.662 | INFO | "malloc" : " 14 GB", | 20260505 23:37:45.662 | INFO | "pinned" : " 29 GB" | 20260505 23:37:45.662 | INFO | } | 20260505 23:37:45.662 | INFO | Perf report for the last 5.13607 seconds: | 20260505 23:37:45.662 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:37:45.662 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:45.662 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:45.662 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:45.662 | INFO | ANALYSIS : 61.4%, 3.0, 95.4ms, 0 | 20260505 23:37:45.662 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:45.662 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:45.662 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:37:45.662 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:45.662 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:45.663 | INFO | Uploaded 36GiB to GPU in 2.08372s (17.2768GiB/s) | 20260505 23:37:45.663 | INFO | Downloaded 34GiB to GPU in 2.01261s (16.8935GiB/s) | 20260505 23:37:50.800 | INFO | Analyzing chunk frames = [7210496,7211008) | 20260505 23:37:50.800 | INFO | MemInfo: { | 20260505 23:37:50.800 | INFO | "gpu" : " 17 GB", | 20260505 23:37:50.801 | INFO | "huge" : "", | 20260505 23:37:50.801 | INFO | "huge_shared" : " 27 GB", | 20260505 23:37:50.801 | INFO | "malloc" : " 14 GB", | 20260505 23:37:50.801 | INFO | "pinned" : " 29 GB" | 20260505 23:37:50.801 | INFO | } | 20260505 23:37:50.801 | INFO | Perf report for the last 5.13831 seconds: | 20260505 23:37:50.801 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:37:50.801 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:50.801 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:50.801 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:50.801 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260505 23:37:50.801 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:50.801 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:50.801 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 23:37:50.801 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:50.801 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:50.801 | INFO | Uploaded 36GiB to GPU in 2.14569s (16.7779GiB/s) | 20260505 23:37:50.801 | INFO | Downloaded 34GiB to GPU in 2.02686s (16.7747GiB/s) | 20260505 23:37:55.939 | INFO | Analyzing chunk frames = [7211008,7211520) | 20260505 23:37:55.939 | INFO | MemInfo: { | 20260505 23:37:55.939 | INFO | "gpu" : " 17 GB", | 20260505 23:37:55.939 | INFO | "huge" : "", | 20260505 23:37:55.939 | INFO | "huge_shared" : " 27 GB", | 20260505 23:37:55.939 | INFO | "malloc" : " 14 GB", | 20260505 23:37:55.939 | INFO | "pinned" : " 29 GB" | 20260505 23:37:55.939 | INFO | } | 20260505 23:37:55.940 | INFO | Perf report for the last 5.13884 seconds: | 20260505 23:37:55.940 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:37:55.940 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:55.940 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:55.940 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:55.940 | INFO | ANALYSIS : 61.6%, 3.0, 96.2ms, 0 | 20260505 23:37:55.940 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:55.940 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:55.940 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260505 23:37:55.940 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:55.940 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:37:55.940 | INFO | Uploaded 36GiB to GPU in 2.15966s (16.6693GiB/s) | 20260505 23:37:55.940 | INFO | Downloaded 34GiB to GPU in 2.07272s (16.4036GiB/s) | 20260505 23:38:01.080 | INFO | Analyzing chunk frames = [7211520,7212032) | 20260505 23:38:01.080 | INFO | MemInfo: { | 20260505 23:38:01.080 | INFO | "gpu" : " 17 GB", | 20260505 23:38:01.080 | INFO | "huge" : "", | 20260505 23:38:01.080 | INFO | "huge_shared" : " 27 GB", | 20260505 23:38:01.080 | INFO | "malloc" : " 14 GB", | 20260505 23:38:01.080 | INFO | "pinned" : " 29 GB" | 20260505 23:38:01.080 | INFO | } | 20260505 23:38:01.080 | INFO | Perf report for the last 5.14094 seconds: | 20260505 23:38:01.080 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:38:01.080 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:01.080 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:01.080 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:01.080 | INFO | ANALYSIS : 61.3%, 3.0, 95.8ms, 0 | 20260505 23:38:01.080 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:01.080 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:01.081 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:38:01.081 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:01.081 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:01.081 | INFO | Uploaded 36GiB to GPU in 2.07791s (17.3251GiB/s) | 20260505 23:38:01.081 | INFO | Downloaded 34GiB to GPU in 2.03966s (16.6694GiB/s) | 20260505 23:38:06.224 | INFO | Analyzing chunk frames = [7212032,7212544) | 20260505 23:38:06.224 | INFO | MemInfo: { | 20260505 23:38:06.224 | INFO | "gpu" : " 17 GB", | 20260505 23:38:06.224 | INFO | "huge" : "", | 20260505 23:38:06.224 | INFO | "huge_shared" : " 27 GB", | 20260505 23:38:06.224 | INFO | "malloc" : " 14 GB", | 20260505 23:38:06.224 | INFO | "pinned" : " 29 GB" | 20260505 23:38:06.224 | INFO | } | 20260505 23:38:06.224 | INFO | Perf report for the last 5.14378 seconds: | 20260505 23:38:06.224 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:38:06.224 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:06.224 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:06.224 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:06.224 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 23:38:06.224 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:06.224 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:06.224 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260505 23:38:06.224 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:06.224 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:06.225 | INFO | Uploaded 36GiB to GPU in 2.017s (17.8483GiB/s) | 20260505 23:38:06.225 | INFO | Downloaded 34GiB to GPU in 2.0352s (16.706GiB/s) | 20260505 23:38:11.360 | INFO | Analyzing chunk frames = [7212544,7213056) | 20260505 23:38:11.360 | INFO | MemInfo: { | 20260505 23:38:11.360 | INFO | "gpu" : " 17 GB", | 20260505 23:38:11.360 | INFO | "huge" : "", | 20260505 23:38:11.360 | INFO | "huge_shared" : " 27 GB", | 20260505 23:38:11.360 | INFO | "malloc" : " 14 GB", | 20260505 23:38:11.360 | INFO | "pinned" : " 29 GB" | 20260505 23:38:11.360 | INFO | } | 20260505 23:38:11.360 | INFO | Perf report for the last 5.13614 seconds: | 20260505 23:38:11.360 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:38:11.360 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:11.360 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:11.360 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:11.360 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260505 23:38:11.360 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:11.361 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:11.361 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:38:11.361 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:11.361 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:11.361 | INFO | Uploaded 36GiB to GPU in 2.13175s (16.8875GiB/s) | 20260505 23:38:11.361 | INFO | Downloaded 34GiB to GPU in 2.02383s (16.7998GiB/s) | 20260505 23:38:11.479 | INFO | Starting a round of Baz IO | 20260505 23:38:14.632 | INFO | Populating RTMetrics report | 20260505 23:38:14.633 | INFO | RTMetrics report populated | 20260505 23:38:14.633 | INFO | Opening tmp RTMetrics json file | 20260505 23:38:14.634 | INFO | Serializing RTMetrics to json | 20260505 23:38:14.635 | INFO | Serialization done, swapping tmp file now... | 20260505 23:38:14.635 | INFO | RTMetrics done | 20260505 23:38:14.710 | INFO | Finished a round of Baz IO: Wrote 1.887 GiB in 3.23 seconds | 20260505 23:38:14.710 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.42% / 0.00% / 78.58% | 20260505 23:38:14.710 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.104252% / 0% / 7.15256e-05% | 20260505 23:38:14.710 | INFO | Baz IO overall: | 20260505 23:38:14.710 | INFO | BytesWritten : 4120279666234 | 20260505 23:38:14.710 | INFO | NumEvents : 1940778218254 | 20260505 23:38:16.491 | INFO | Analyzing chunk frames = [7213056,7213568) | 20260505 23:38:16.491 | INFO | MemInfo: { | 20260505 23:38:16.491 | INFO | "gpu" : " 17 GB", | 20260505 23:38:16.491 | INFO | "huge" : "", | 20260505 23:38:16.492 | INFO | "huge_shared" : " 27 GB", | 20260505 23:38:16.492 | INFO | "malloc" : " 14 GB", | 20260505 23:38:16.492 | INFO | "pinned" : " 29 GB" | 20260505 23:38:16.492 | INFO | } | 20260505 23:38:16.492 | INFO | Perf report for the last 5.13118 seconds: | 20260505 23:38:16.492 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:38:16.492 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:16.492 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:16.492 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:16.492 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260505 23:38:16.492 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:16.492 | INFO | RT_METRICS : 43.4%, 1.0, 22.7ms, 0 | 20260505 23:38:16.492 | INFO | PRE_HQ : 60.0%, 1.8, 54.7ms, 0 | 20260505 23:38:16.492 | INFO | BAZWRITER : 62.1%, 3.6, 115.7ms, 0 | 20260505 23:38:16.492 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 23:38:16.492 | INFO | Uploaded 36GiB to GPU in 2.1675s (16.609GiB/s) | 20260505 23:38:16.492 | INFO | Downloaded 38GiB to GPU in 2.23495s (17.0026GiB/s) | 20260505 23:38:21.630 | INFO | Analyzing chunk frames = [7213568,7214080) | 20260505 23:38:21.630 | INFO | MemInfo: { | 20260505 23:38:21.630 | INFO | "gpu" : " 17 GB", | 20260505 23:38:21.630 | INFO | "huge" : "", | 20260505 23:38:21.630 | INFO | "huge_shared" : " 27 GB", | 20260505 23:38:21.630 | INFO | "malloc" : " 14 GB", | 20260505 23:38:21.630 | INFO | "pinned" : " 29 GB" | 20260505 23:38:21.630 | INFO | } | 20260505 23:38:21.630 | INFO | Perf report for the last 5.13871 seconds: | 20260505 23:38:21.630 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:38:21.630 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:21.630 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:21.630 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:21.630 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 23:38:21.630 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:21.630 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:21.630 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:38:21.630 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:21.630 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:21.631 | INFO | Uploaded 36GiB to GPU in 2.09802s (17.159GiB/s) | 20260505 23:38:21.631 | INFO | Downloaded 34GiB to GPU in 2.02704s (16.7732GiB/s) | 20260505 23:38:26.774 | INFO | Analyzing chunk frames = [7214080,7214592) | 20260505 23:38:26.774 | INFO | MemInfo: { | 20260505 23:38:26.774 | INFO | "gpu" : " 17 GB", | 20260505 23:38:26.774 | INFO | "huge" : "", | 20260505 23:38:26.774 | INFO | "huge_shared" : " 27 GB", | 20260505 23:38:26.774 | INFO | "malloc" : " 14 GB", | 20260505 23:38:26.774 | INFO | "pinned" : " 29 GB" | 20260505 23:38:26.774 | INFO | } | 20260505 23:38:26.774 | INFO | Perf report for the last 5.14384 seconds: | 20260505 23:38:26.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:38:26.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:26.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:26.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:26.774 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260505 23:38:26.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:26.774 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:26.774 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:38:26.774 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:26.774 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:26.774 | INFO | Uploaded 36GiB to GPU in 2.15349s (16.7171GiB/s) | 20260505 23:38:26.774 | INFO | Downloaded 34GiB to GPU in 2.04805s (16.6011GiB/s) | 20260505 23:38:31.920 | INFO | Analyzing chunk frames = [7214592,7215104) | 20260505 23:38:31.920 | INFO | MemInfo: { | 20260505 23:38:31.920 | INFO | "gpu" : " 17 GB", | 20260505 23:38:31.920 | INFO | "huge" : "", | 20260505 23:38:31.920 | INFO | "huge_shared" : " 27 GB", | 20260505 23:38:31.920 | INFO | "malloc" : " 14 GB", | 20260505 23:38:31.920 | INFO | "pinned" : " 29 GB" | 20260505 23:38:31.920 | INFO | } | 20260505 23:38:31.920 | INFO | Perf report for the last 5.14588 seconds: | 20260505 23:38:31.920 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:38:31.920 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:31.920 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:31.920 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:31.920 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260505 23:38:31.920 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:31.920 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:31.920 | INFO | PRE_HQ : 26.8%, 1.0, 14.0ms, 0 | 20260505 23:38:31.920 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:31.920 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:31.920 | INFO | Uploaded 36GiB to GPU in 2.14569s (16.7778GiB/s) | 20260505 23:38:31.920 | INFO | Downloaded 34GiB to GPU in 2.02306s (16.8062GiB/s) | 20260505 23:38:37.057 | INFO | Analyzing chunk frames = [7215104,7215616) | 20260505 23:38:37.057 | INFO | MemInfo: { | 20260505 23:38:37.057 | INFO | "gpu" : " 17 GB", | 20260505 23:38:37.057 | INFO | "huge" : "", | 20260505 23:38:37.057 | INFO | "huge_shared" : " 27 GB", | 20260505 23:38:37.057 | INFO | "malloc" : " 14 GB", | 20260505 23:38:37.057 | INFO | "pinned" : " 29 GB" | 20260505 23:38:37.057 | INFO | } | 20260505 23:38:37.057 | INFO | Perf report for the last 5.13674 seconds: | 20260505 23:38:37.057 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:38:37.057 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:37.057 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:37.057 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:37.057 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260505 23:38:37.057 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:37.057 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:37.057 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:38:37.057 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:37.057 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:37.057 | INFO | Uploaded 36GiB to GPU in 2.09065s (17.2195GiB/s) | 20260505 23:38:37.057 | INFO | Downloaded 34GiB to GPU in 2.04599s (16.6179GiB/s) | 20260505 23:38:42.191 | INFO | Analyzing chunk frames = [7215616,7216128) | 20260505 23:38:42.191 | INFO | MemInfo: { | 20260505 23:38:42.191 | INFO | "gpu" : " 17 GB", | 20260505 23:38:42.191 | INFO | "huge" : "", | 20260505 23:38:42.191 | INFO | "huge_shared" : " 27 GB", | 20260505 23:38:42.192 | INFO | "malloc" : " 14 GB", | 20260505 23:38:42.192 | INFO | "pinned" : " 29 GB" | 20260505 23:38:42.192 | INFO | } | 20260505 23:38:42.192 | INFO | Perf report for the last 5.13473 seconds: | 20260505 23:38:42.192 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:38:42.192 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:42.192 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:42.192 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:42.192 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260505 23:38:42.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:42.192 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:42.192 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:38:42.192 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:42.192 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:42.192 | INFO | Uploaded 36GiB to GPU in 2.08487s (17.2673GiB/s) | 20260505 23:38:42.192 | INFO | Downloaded 34GiB to GPU in 2.03625s (16.6974GiB/s) | 20260505 23:38:47.335 | INFO | Analyzing chunk frames = [7216128,7216640) | 20260505 23:38:47.335 | INFO | MemInfo: { | 20260505 23:38:47.335 | INFO | "gpu" : " 17 GB", | 20260505 23:38:47.335 | INFO | "huge" : "", | 20260505 23:38:47.335 | INFO | "huge_shared" : " 27 GB", | 20260505 23:38:47.335 | INFO | "malloc" : " 14 GB", | 20260505 23:38:47.335 | INFO | "pinned" : " 29 GB" | 20260505 23:38:47.335 | INFO | } | 20260505 23:38:47.335 | INFO | Perf report for the last 5.14377 seconds: | 20260505 23:38:47.335 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:38:47.335 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:47.335 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:47.335 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:47.335 | INFO | ANALYSIS : 62.2%, 3.0, 97.3ms, 0 | 20260505 23:38:47.335 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:47.335 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:47.335 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 23:38:47.335 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:47.335 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:47.336 | INFO | Uploaded 36GiB to GPU in 2.04405s (17.6121GiB/s) | 20260505 23:38:47.336 | INFO | Downloaded 34GiB to GPU in 2.04433s (16.6314GiB/s) | 20260505 23:38:52.473 | INFO | Analyzing chunk frames = [7216640,7217152) | 20260505 23:38:52.473 | INFO | MemInfo: { | 20260505 23:38:52.473 | INFO | "gpu" : " 17 GB", | 20260505 23:38:52.473 | INFO | "huge" : "", | 20260505 23:38:52.473 | INFO | "huge_shared" : " 27 GB", | 20260505 23:38:52.473 | INFO | "malloc" : " 14 GB", | 20260505 23:38:52.473 | INFO | "pinned" : " 29 GB" | 20260505 23:38:52.473 | INFO | } | 20260505 23:38:52.473 | INFO | Perf report for the last 5.13796 seconds: | 20260505 23:38:52.473 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:38:52.473 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:52.473 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:52.473 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:52.473 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260505 23:38:52.473 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:52.473 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:52.473 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 23:38:52.473 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:52.473 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:52.473 | INFO | Uploaded 36GiB to GPU in 2.11028s (17.0593GiB/s) | 20260505 23:38:52.474 | INFO | Downloaded 34GiB to GPU in 2.00163s (16.9862GiB/s) | 20260505 23:38:52.474 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 23:38:52.474 | INFO | Total mem usage: 15.3 GiB | 20260505 23:38:52.474 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 23:38:52.474 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.13%) | 20260505 23:38:52.474 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 23:38:52.474 | INFO | High Water Breakdown: | 20260505 23:38:52.474 | INFO | 60.9% : 8.55 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 23:38:52.474 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 23:38:52.474 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 23:38:52.474 | INFO | ------------------------------------------ | 20260505 23:38:52.474 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:38:52.474 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 23:38:52.474 | INFO | Total mem usage: 26.5 GiB | 20260505 23:38:52.474 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 23:38:52.474 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 23:38:52.474 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 23:38:52.474 | INFO | High Water Breakdown: | 20260505 23:38:52.474 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:38:52.474 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:38:52.474 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:38:52.474 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:38:52.474 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:38:52.474 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:38:52.474 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 23:38:52.474 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 23:38:52.474 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:38:52.474 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 23:38:52.474 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 23:38:52.474 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 23:38:52.474 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 23:38:52.474 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 23:38:52.474 | INFO | ------------------------------------------ | 20260505 23:38:52.474 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:38:52.474 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 23:38:52.474 | INFO | Total mem usage: 25.6 GiB | 20260505 23:38:52.474 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 23:38:52.474 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 23:38:52.474 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 23:38:52.474 | INFO | High Water Breakdown: | 20260505 23:38:52.474 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 23:38:52.474 | INFO | ------------------------------------------ | 20260505 23:38:52.474 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:38:52.474 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 23:38:52.474 | INFO | Total mem usage: 15.7 GiB | 20260505 23:38:52.474 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 23:38:52.474 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 23:38:52.474 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 23:38:52.474 | INFO | High Water Breakdown: | 20260505 23:38:52.474 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 23:38:52.474 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 23:38:52.474 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 23:38:52.474 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 23:38:52.474 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 23:38:52.474 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 23:38:52.474 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 23:38:52.474 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 23:38:52.474 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 23:38:52.474 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:38:52.474 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 23:38:52.474 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:38:52.474 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 23:38:52.474 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 23:38:52.474 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 23:38:52.474 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 23:38:52.474 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:38:52.474 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 23:38:52.474 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:38:52.474 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:38:52.474 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:38:52.474 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 23:38:52.474 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 23:38:52.474 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:38:52.474 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 23:38:52.474 | INFO | ------------------------------------------ | 20260505 23:38:52.474 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:38:52.474 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 23:38:52.474 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:38:52.474 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:38:52.474 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.40e+01, 1.35e+01 / 8.34e-01 | 20260505 23:38:52.474 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.03e-01, 1.03e-02 / 2.96e-03 | 20260505 23:38:52.474 | INFO | PerfReport: 11.82%, Baseline -- 3.70e+00 / 4.51e+00, 5.61e-01 / 1.55e-01 | 20260505 23:38:52.474 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.54e+00, 1.91e-01 / 9.07e-02 | 20260505 23:38:52.474 | INFO | PerfReport: 16.01%, DME -- 4.01e+01 / 4.41e+01, 3.84e+00 / 1.00e+00 | 20260505 23:38:52.474 | INFO | PerfReport: 57.27%, FrameLabeling -- 1.79e+01 / 1.93e+01, 1.45e+00 / 1.06e+00 | 20260505 23:38:52.474 | INFO | PerfReport: 7.14%, PulseAccumulating -- 2.24e+00 / 2.78e+00, 4.26e-01 / 2.34e-01 | 20260505 23:38:52.474 | INFO | PerfReport: 3.11%, Metrics -- 9.74e-01 / 1.26e+00, 2.62e-01 / 1.36e-01 | 20260505 23:38:52.474 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.38e-01 / 3.97e-01, 5.76e-02 / 3.38e-02 | 20260505 23:38:52.474 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 23:38:52.474 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:38:52.474 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:38:52.475 | INFO | PerfReport: Full Execution -- 9.85e+01 / -nan, 2.47e+01 / 0.00e+00 | 20260505 23:38:52.475 | INFO | PerfReport: 22.33%, Upload -- 2.20e+01 / 0.00e+00, 7.47e+00 / 0.00e+00 | 20260505 23:38:52.475 | INFO | PerfReport: 21.78%, Download -- 2.14e+01 / 0.00e+00, 7.58e+00 / 0.00e+00 | 20260505 23:38:52.475 | INFO | PerfReport: 31.81%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 23:38:52.475 | INFO | PerfReport: 2.19%, LockUpload -- 2.15e+00 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260505 23:38:52.475 | INFO | PerfReport: 1.31%, LockDownload -- 1.29e+00 / 0.00e+00, 2.90e+00 / 0.00e+00 | 20260505 23:38:52.475 | INFO | PerfReport: 20.56%, LockCompute -- 2.02e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260505 23:38:52.475 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 1.94e-02 / 0.00e+00 | 20260505 23:38:52.597 | INFO | Starting a round of Baz IO | 20260505 23:38:55.739 | INFO | Populating RTMetrics report | 20260505 23:38:55.739 | INFO | RTMetrics report populated | 20260505 23:38:55.739 | INFO | Opening tmp RTMetrics json file | 20260505 23:38:55.741 | INFO | Serializing RTMetrics to json | 20260505 23:38:55.746 | INFO | Serialization done, swapping tmp file now... | 20260505 23:38:55.746 | INFO | RTMetrics done | 20260505 23:38:55.822 | INFO | Finished a round of Baz IO: Wrote 1.887 GiB in 3.224 seconds | 20260505 23:38:55.822 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.42% / 0.00% / 78.58% | 20260505 23:38:55.822 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.103954% / 3.97364e-06% / 7.15256e-05% | 20260505 23:38:55.822 | INFO | Baz IO overall: | 20260505 23:38:55.822 | INFO | BytesWritten : 4122306332239 | 20260505 23:38:55.822 | INFO | NumEvents : 1941707112638 | 20260505 23:38:57.615 | INFO | Analyzing chunk frames = [7217152,7217664) | 20260505 23:38:57.615 | INFO | MemInfo: { | 20260505 23:38:57.615 | INFO | "gpu" : " 17 GB", | 20260505 23:38:57.615 | INFO | "huge" : "", | 20260505 23:38:57.615 | INFO | "huge_shared" : " 27 GB", | 20260505 23:38:57.615 | INFO | "malloc" : " 14 GB", | 20260505 23:38:57.615 | INFO | "pinned" : " 29 GB" | 20260505 23:38:57.615 | INFO | } | 20260505 23:38:57.615 | INFO | Perf report for the last 5.14183 seconds: | 20260505 23:38:57.615 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:38:57.615 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:57.615 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:57.615 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:57.615 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260505 23:38:57.615 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:38:57.615 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260505 23:38:57.615 | INFO | PRE_HQ : 59.7%, 1.8, 54.4ms, 0 | 20260505 23:38:57.615 | INFO | BAZWRITER : 61.9%, 3.6, 114.0ms, 0 | 20260505 23:38:57.615 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 23:38:57.615 | INFO | Uploaded 36GiB to GPU in 2.16513s (16.6272GiB/s) | 20260505 23:38:57.616 | INFO | Downloaded 38GiB to GPU in 2.23292s (17.0181GiB/s) | 20260505 23:39:02.750 | INFO | Analyzing chunk frames = [7217664,7218176) | 20260505 23:39:02.750 | INFO | MemInfo: { | 20260505 23:39:02.750 | INFO | "gpu" : " 17 GB", | 20260505 23:39:02.750 | INFO | "huge" : "", | 20260505 23:39:02.750 | INFO | "huge_shared" : " 27 GB", | 20260505 23:39:02.750 | INFO | "malloc" : " 14 GB", | 20260505 23:39:02.750 | INFO | "pinned" : " 29 GB" | 20260505 23:39:02.750 | INFO | } | 20260505 23:39:02.750 | INFO | Perf report for the last 5.135 seconds: | 20260505 23:39:02.750 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:39:02.750 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:02.750 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:02.750 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:02.750 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 23:39:02.750 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:02.750 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:02.750 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 23:39:02.750 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:02.750 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:02.750 | INFO | Uploaded 36GiB to GPU in 2.08613s (17.2568GiB/s) | 20260505 23:39:02.750 | INFO | Downloaded 34GiB to GPU in 2.02564s (16.7848GiB/s) | 20260505 23:39:07.897 | INFO | Analyzing chunk frames = [7218176,7218688) | 20260505 23:39:07.897 | INFO | MemInfo: { | 20260505 23:39:07.897 | INFO | "gpu" : " 17 GB", | 20260505 23:39:07.897 | INFO | "huge" : "", | 20260505 23:39:07.897 | INFO | "huge_shared" : " 27 GB", | 20260505 23:39:07.898 | INFO | "malloc" : " 14 GB", | 20260505 23:39:07.898 | INFO | "pinned" : " 29 GB" | 20260505 23:39:07.898 | INFO | } | 20260505 23:39:07.898 | INFO | Perf report for the last 5.14738 seconds: | 20260505 23:39:07.898 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:39:07.898 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:07.898 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:07.898 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:07.898 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260505 23:39:07.898 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:07.898 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:07.898 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:39:07.898 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:07.898 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:07.898 | INFO | Uploaded 36GiB to GPU in 2.1626s (16.6466GiB/s) | 20260505 23:39:07.898 | INFO | Downloaded 34GiB to GPU in 2.05389s (16.5539GiB/s) | 20260505 23:39:13.033 | INFO | Analyzing chunk frames = [7218688,7219200) | 20260505 23:39:13.033 | INFO | MemInfo: { | 20260505 23:39:13.033 | INFO | "gpu" : " 17 GB", | 20260505 23:39:13.033 | INFO | "huge" : "", | 20260505 23:39:13.033 | INFO | "huge_shared" : " 27 GB", | 20260505 23:39:13.033 | INFO | "malloc" : " 14 GB", | 20260505 23:39:13.033 | INFO | "pinned" : " 29 GB" | 20260505 23:39:13.033 | INFO | } | 20260505 23:39:13.033 | INFO | Perf report for the last 5.13509 seconds: | 20260505 23:39:13.033 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:39:13.033 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:13.033 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:13.033 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:13.033 | INFO | ANALYSIS : 61.8%, 3.0, 95.8ms, 0 | 20260505 23:39:13.033 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:13.033 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:13.033 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:39:13.033 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:13.033 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:13.033 | INFO | Uploaded 36GiB to GPU in 2.14164s (16.8095GiB/s) | 20260505 23:39:13.033 | INFO | Downloaded 34GiB to GPU in 2.02566s (16.7846GiB/s) | 20260505 23:39:18.172 | INFO | Analyzing chunk frames = [7219200,7219712) | 20260505 23:39:18.172 | INFO | MemInfo: { | 20260505 23:39:18.172 | INFO | "gpu" : " 17 GB", | 20260505 23:39:18.172 | INFO | "huge" : "", | 20260505 23:39:18.173 | INFO | "huge_shared" : " 27 GB", | 20260505 23:39:18.173 | INFO | "malloc" : " 14 GB", | 20260505 23:39:18.173 | INFO | "pinned" : " 29 GB" | 20260505 23:39:18.173 | INFO | } | 20260505 23:39:18.173 | INFO | Perf report for the last 5.14001 seconds: | 20260505 23:39:18.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:39:18.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:18.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:18.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:18.173 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260505 23:39:18.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:18.173 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:18.173 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 23:39:18.173 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:18.173 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:18.173 | INFO | Uploaded 36GiB to GPU in 2.08683s (17.251GiB/s) | 20260505 23:39:18.173 | INFO | Downloaded 34GiB to GPU in 2.04461s (16.6291GiB/s) | 20260505 23:39:23.310 | INFO | Analyzing chunk frames = [7219712,7220224) | 20260505 23:39:23.310 | INFO | MemInfo: { | 20260505 23:39:23.310 | INFO | "gpu" : " 17 GB", | 20260505 23:39:23.310 | INFO | "huge" : "", | 20260505 23:39:23.310 | INFO | "huge_shared" : " 27 GB", | 20260505 23:39:23.310 | INFO | "malloc" : " 14 GB", | 20260505 23:39:23.310 | INFO | "pinned" : " 29 GB" | 20260505 23:39:23.310 | INFO | } | 20260505 23:39:23.310 | INFO | Perf report for the last 5.13772 seconds: | 20260505 23:39:23.310 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:39:23.310 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:23.310 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:23.310 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:23.310 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260505 23:39:23.310 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:23.310 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:23.311 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 23:39:23.311 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:23.311 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:23.311 | INFO | Uploaded 36GiB to GPU in 2.08016s (17.3064GiB/s) | 20260505 23:39:23.311 | INFO | Downloaded 34GiB to GPU in 2.03516s (16.7063GiB/s) | 20260505 23:39:28.443 | INFO | Analyzing chunk frames = [7220224,7220736) | 20260505 23:39:28.443 | INFO | MemInfo: { | 20260505 23:39:28.443 | INFO | "gpu" : " 17 GB", | 20260505 23:39:28.443 | INFO | "huge" : "", | 20260505 23:39:28.443 | INFO | "huge_shared" : " 27 GB", | 20260505 23:39:28.443 | INFO | "malloc" : " 14 GB", | 20260505 23:39:28.443 | INFO | "pinned" : " 29 GB" | 20260505 23:39:28.443 | INFO | } | 20260505 23:39:28.443 | INFO | Perf report for the last 5.13289 seconds: | 20260505 23:39:28.443 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:39:28.443 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:28.443 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:28.443 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:28.443 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 23:39:28.443 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:28.443 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:28.443 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260505 23:39:28.443 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:28.443 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:28.444 | INFO | Uploaded 36GiB to GPU in 2.15763s (16.685GiB/s) | 20260505 23:39:28.444 | INFO | Downloaded 34GiB to GPU in 2.12932s (15.9675GiB/s) | 20260505 23:39:33.585 | INFO | Analyzing chunk frames = [7220736,7221248) | 20260505 23:39:33.585 | INFO | MemInfo: { | 20260505 23:39:33.585 | INFO | "gpu" : " 17 GB", | 20260505 23:39:33.585 | INFO | "huge" : "", | 20260505 23:39:33.585 | INFO | "huge_shared" : " 27 GB", | 20260505 23:39:33.585 | INFO | "malloc" : " 14 GB", | 20260505 23:39:33.585 | INFO | "pinned" : " 29 GB" | 20260505 23:39:33.585 | INFO | } | 20260505 23:39:33.586 | INFO | Perf report for the last 5.14219 seconds: | 20260505 23:39:33.586 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:39:33.586 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:33.586 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:33.586 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:33.586 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260505 23:39:33.586 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:33.586 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:33.586 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 23:39:33.586 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:33.586 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:33.586 | INFO | Uploaded 36GiB to GPU in 2.10518s (17.1006GiB/s) | 20260505 23:39:33.586 | INFO | Downloaded 34GiB to GPU in 2.01084s (16.9084GiB/s) | 20260505 23:39:33.711 | INFO | Starting a round of Baz IO | 20260505 23:39:36.856 | INFO | Populating RTMetrics report | 20260505 23:39:36.856 | INFO | RTMetrics report populated | 20260505 23:39:36.856 | INFO | Opening tmp RTMetrics json file | 20260505 23:39:36.857 | INFO | Serializing RTMetrics to json | 20260505 23:39:36.858 | INFO | Serialization done, swapping tmp file now... | 20260505 23:39:36.859 | INFO | RTMetrics done | 20260505 23:39:36.940 | INFO | Finished a round of Baz IO: Wrote 1.886 GiB in 3.229 seconds | 20260505 23:39:36.940 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.42% / 0.00% / 78.58% | 20260505 23:39:36.940 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.103426% / 3.97364e-06% / 7.15256e-05% | 20260505 23:39:36.940 | INFO | Baz IO overall: | 20260505 23:39:36.940 | INFO | BytesWritten : 4124331173287 | 20260505 23:39:36.941 | INFO | NumEvents : 1942635284977 | 20260505 23:39:38.731 | INFO | Analyzing chunk frames = [7221248,7221760) | 20260505 23:39:38.731 | INFO | MemInfo: { | 20260505 23:39:38.731 | INFO | "gpu" : " 17 GB", | 20260505 23:39:38.731 | INFO | "huge" : "", | 20260505 23:39:38.731 | INFO | "huge_shared" : " 27 GB", | 20260505 23:39:38.731 | INFO | "malloc" : " 14 GB", | 20260505 23:39:38.731 | INFO | "pinned" : " 29 GB" | 20260505 23:39:38.731 | INFO | } | 20260505 23:39:38.731 | INFO | Perf report for the last 5.14575 seconds: | 20260505 23:39:38.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:39:38.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:38.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:38.731 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:38.731 | INFO | ANALYSIS : 62.5%, 3.0, 97.7ms, 0 | 20260505 23:39:38.731 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:38.731 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260505 23:39:38.731 | INFO | PRE_HQ : 59.9%, 1.8, 54.6ms, 0 | 20260505 23:39:38.731 | INFO | BAZWRITER : 62.0%, 3.6, 113.5ms, 0 | 20260505 23:39:38.731 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 23:39:38.732 | INFO | Uploaded 36GiB to GPU in 2.14076s (16.8165GiB/s) | 20260505 23:39:38.732 | INFO | Downloaded 38GiB to GPU in 2.22682s (17.0647GiB/s) | 20260505 23:39:43.871 | INFO | Analyzing chunk frames = [7221760,7222272) | 20260505 23:39:43.871 | INFO | MemInfo: { | 20260505 23:39:43.871 | INFO | "gpu" : " 17 GB", | 20260505 23:39:43.871 | INFO | "huge" : "", | 20260505 23:39:43.871 | INFO | "huge_shared" : " 27 GB", | 20260505 23:39:43.871 | INFO | "malloc" : " 14 GB", | 20260505 23:39:43.871 | INFO | "pinned" : " 29 GB" | 20260505 23:39:43.871 | INFO | } | 20260505 23:39:43.871 | INFO | Perf report for the last 5.13938 seconds: | 20260505 23:39:43.871 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:39:43.871 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:43.871 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:43.871 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:43.871 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260505 23:39:43.871 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:43.871 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:43.871 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 23:39:43.871 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:43.871 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:43.871 | INFO | Uploaded 36GiB to GPU in 2.09738s (17.1643GiB/s) | 20260505 23:39:43.871 | INFO | Downloaded 34GiB to GPU in 2.03792s (16.6837GiB/s) | 20260505 23:39:49.003 | INFO | Analyzing chunk frames = [7222272,7222784) | 20260505 23:39:49.003 | INFO | MemInfo: { | 20260505 23:39:49.004 | INFO | "gpu" : " 17 GB", | 20260505 23:39:49.004 | INFO | "huge" : "", | 20260505 23:39:49.004 | INFO | "huge_shared" : " 27 GB", | 20260505 23:39:49.004 | INFO | "malloc" : " 14 GB", | 20260505 23:39:49.004 | INFO | "pinned" : " 29 GB" | 20260505 23:39:49.004 | INFO | } | 20260505 23:39:49.004 | INFO | Perf report for the last 5.13292 seconds: | 20260505 23:39:49.004 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:39:49.004 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:49.004 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:49.004 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:49.004 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260505 23:39:49.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:49.004 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:49.004 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:39:49.004 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:49.004 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:49.004 | INFO | Uploaded 36GiB to GPU in 2.07593s (17.3416GiB/s) | 20260505 23:39:49.004 | INFO | Downloaded 34GiB to GPU in 2.00901s (16.9237GiB/s) | 20260505 23:39:54.144 | INFO | Analyzing chunk frames = [7222784,7223296) | 20260505 23:39:54.144 | INFO | MemInfo: { | 20260505 23:39:54.144 | INFO | "gpu" : " 17 GB", | 20260505 23:39:54.144 | INFO | "huge" : "", | 20260505 23:39:54.144 | INFO | "huge_shared" : " 27 GB", | 20260505 23:39:54.144 | INFO | "malloc" : " 14 GB", | 20260505 23:39:54.144 | INFO | "pinned" : " 29 GB" | 20260505 23:39:54.144 | INFO | } | 20260505 23:39:54.144 | INFO | Perf report for the last 5.14075 seconds: | 20260505 23:39:54.144 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:39:54.144 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:54.144 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:54.144 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:54.144 | INFO | ANALYSIS : 61.8%, 3.0, 96.0ms, 0 | 20260505 23:39:54.144 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:54.144 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:54.144 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:39:54.144 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:54.144 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:54.145 | INFO | Uploaded 36GiB to GPU in 2.13785s (16.8393GiB/s) | 20260505 23:39:54.145 | INFO | Downloaded 34GiB to GPU in 2.02906s (16.7566GiB/s) | 20260505 23:39:59.285 | INFO | Analyzing chunk frames = [7223296,7223808) | 20260505 23:39:59.285 | INFO | MemInfo: { | 20260505 23:39:59.285 | INFO | "gpu" : " 17 GB", | 20260505 23:39:59.285 | INFO | "huge" : "", | 20260505 23:39:59.285 | INFO | "huge_shared" : " 27 GB", | 20260505 23:39:59.285 | INFO | "malloc" : " 14 GB", | 20260505 23:39:59.285 | INFO | "pinned" : " 29 GB" | 20260505 23:39:59.285 | INFO | } | 20260505 23:39:59.285 | INFO | Perf report for the last 5.1408 seconds: | 20260505 23:39:59.285 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:39:59.285 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:59.285 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:59.285 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:59.285 | INFO | ANALYSIS : 61.2%, 3.0, 95.6ms, 0 | 20260505 23:39:59.285 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:59.285 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:59.285 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:39:59.285 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:59.285 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:39:59.285 | INFO | Uploaded 36GiB to GPU in 2.10064s (17.1377GiB/s) | 20260505 23:39:59.285 | INFO | Downloaded 34GiB to GPU in 2.05355s (16.5567GiB/s) | 20260505 23:40:04.422 | INFO | Analyzing chunk frames = [7223808,7224320) | 20260505 23:40:04.422 | INFO | MemInfo: { | 20260505 23:40:04.422 | INFO | "gpu" : " 17 GB", | 20260505 23:40:04.422 | INFO | "huge" : "", | 20260505 23:40:04.422 | INFO | "huge_shared" : " 27 GB", | 20260505 23:40:04.422 | INFO | "malloc" : " 14 GB", | 20260505 23:40:04.422 | INFO | "pinned" : " 29 GB" | 20260505 23:40:04.422 | INFO | } | 20260505 23:40:04.422 | INFO | Perf report for the last 5.137 seconds: | 20260505 23:40:04.422 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:40:04.422 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:04.422 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:04.422 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:04.422 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260505 23:40:04.422 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:04.422 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:04.422 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:40:04.422 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:04.422 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:04.423 | INFO | Uploaded 36GiB to GPU in 2.0677s (17.4106GiB/s) | 20260505 23:40:04.423 | INFO | Downloaded 34GiB to GPU in 2.03514s (16.7064GiB/s) | 20260505 23:40:09.564 | INFO | Analyzing chunk frames = [7224320,7224832) | 20260505 23:40:09.564 | INFO | MemInfo: { | 20260505 23:40:09.564 | INFO | "gpu" : " 17 GB", | 20260505 23:40:09.564 | INFO | "huge" : "", | 20260505 23:40:09.564 | INFO | "huge_shared" : " 27 GB", | 20260505 23:40:09.564 | INFO | "malloc" : " 14 GB", | 20260505 23:40:09.564 | INFO | "pinned" : " 29 GB" | 20260505 23:40:09.564 | INFO | } | 20260505 23:40:09.564 | INFO | Perf report for the last 5.14206 seconds: | 20260505 23:40:09.564 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:40:09.564 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:09.564 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:09.564 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:09.564 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260505 23:40:09.564 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:09.564 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:09.564 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 23:40:09.564 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:09.564 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:09.564 | INFO | Uploaded 36GiB to GPU in 2.03712s (17.672GiB/s) | 20260505 23:40:09.565 | INFO | Downloaded 34GiB to GPU in 2.03977s (16.6685GiB/s) | 20260505 23:40:14.704 | INFO | Analyzing chunk frames = [7224832,7225344) | 20260505 23:40:14.704 | INFO | MemInfo: { | 20260505 23:40:14.704 | INFO | "gpu" : " 17 GB", | 20260505 23:40:14.704 | INFO | "huge" : "", | 20260505 23:40:14.704 | INFO | "huge_shared" : " 27 GB", | 20260505 23:40:14.704 | INFO | "malloc" : " 14 GB", | 20260505 23:40:14.704 | INFO | "pinned" : " 29 GB" | 20260505 23:40:14.704 | INFO | } | 20260505 23:40:14.704 | INFO | Perf report for the last 5.1397 seconds: | 20260505 23:40:14.704 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:40:14.704 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:14.704 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:14.704 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:14.704 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260505 23:40:14.704 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:14.704 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:14.704 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:40:14.704 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:14.704 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:14.704 | INFO | Uploaded 36GiB to GPU in 2.11221s (17.0438GiB/s) | 20260505 23:40:14.704 | INFO | Downloaded 34GiB to GPU in 2.00631s (16.9466GiB/s) | 20260505 23:40:14.704 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 23:40:14.704 | INFO | Total mem usage: 15.3 GiB | 20260505 23:40:14.704 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 23:40:14.704 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.11%) | 20260505 23:40:14.704 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 23:40:14.704 | INFO | High Water Breakdown: | 20260505 23:40:14.704 | INFO | 60.9% : 8.55 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 23:40:14.704 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 23:40:14.704 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 23:40:14.704 | INFO | ------------------------------------------ | 20260505 23:40:14.704 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:40:14.704 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 23:40:14.704 | INFO | Total mem usage: 26.5 GiB | 20260505 23:40:14.704 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 23:40:14.704 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 23:40:14.704 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 23:40:14.704 | INFO | High Water Breakdown: | 20260505 23:40:14.705 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:40:14.705 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:40:14.705 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:40:14.705 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:40:14.705 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:40:14.705 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:40:14.705 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 23:40:14.705 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 23:40:14.705 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:40:14.705 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 23:40:14.705 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 23:40:14.705 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 23:40:14.705 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 23:40:14.705 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 23:40:14.705 | INFO | ------------------------------------------ | 20260505 23:40:14.705 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:40:14.705 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 23:40:14.705 | INFO | Total mem usage: 25.6 GiB | 20260505 23:40:14.705 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 23:40:14.705 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 23:40:14.705 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 23:40:14.705 | INFO | High Water Breakdown: | 20260505 23:40:14.705 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 23:40:14.705 | INFO | ------------------------------------------ | 20260505 23:40:14.705 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:40:14.705 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 23:40:14.705 | INFO | Total mem usage: 15.7 GiB | 20260505 23:40:14.705 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 23:40:14.705 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 23:40:14.705 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 23:40:14.705 | INFO | High Water Breakdown: | 20260505 23:40:14.705 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 23:40:14.705 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 23:40:14.705 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 23:40:14.705 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 23:40:14.705 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 23:40:14.705 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 23:40:14.705 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 23:40:14.705 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 23:40:14.705 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 23:40:14.705 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:40:14.705 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 23:40:14.705 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:40:14.705 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 23:40:14.705 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 23:40:14.705 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 23:40:14.705 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 23:40:14.705 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:40:14.705 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 23:40:14.705 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:40:14.705 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:40:14.705 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:40:14.705 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 23:40:14.705 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 23:40:14.705 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:40:14.705 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 23:40:14.705 | INFO | ------------------------------------------ | 20260505 23:40:14.705 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:40:14.705 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 23:40:14.705 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:40:14.705 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:40:14.705 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.40e+01, 1.35e+01 / 7.82e-01 | 20260505 23:40:14.705 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.03e-01, 1.03e-02 / 3.25e-03 | 20260505 23:40:14.705 | INFO | PerfReport: 11.85%, Baseline -- 3.71e+00 / 4.52e+00, 5.79e-01 / 1.29e-01 | 20260505 23:40:14.705 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.54e+00, 1.92e-01 / 7.83e-02 | 20260505 23:40:14.705 | INFO | PerfReport: 16.02%, DME -- 4.02e+01 / 4.41e+01, 3.89e+00 / 1.10e+00 | 20260505 23:40:14.705 | INFO | PerfReport: 57.24%, FrameLabeling -- 1.79e+01 / 1.93e+01, 1.45e+00 / 9.33e-01 | 20260505 23:40:14.705 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.23e+00 / 2.78e+00, 4.28e-01 / 1.81e-01 | 20260505 23:40:14.705 | INFO | PerfReport: 3.11%, Metrics -- 9.75e-01 / 1.25e+00, 2.67e-01 / 1.27e-01 | 20260505 23:40:14.705 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.38e-01 / 3.95e-01, 5.86e-02 / 3.57e-02 | 20260505 23:40:14.705 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 23:40:14.705 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:40:14.705 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:40:14.705 | INFO | PerfReport: Full Execution -- 9.84e+01 / -nan, 2.47e+01 / 0.00e+00 | 20260505 23:40:14.705 | INFO | PerfReport: 22.32%, Upload -- 2.20e+01 / 0.00e+00, 7.47e+00 / 0.00e+00 | 20260505 23:40:14.705 | INFO | PerfReport: 21.82%, Download -- 2.15e+01 / 0.00e+00, 7.57e+00 / 0.00e+00 | 20260505 23:40:14.705 | INFO | PerfReport: 31.84%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 23:40:14.705 | INFO | PerfReport: 2.15%, LockUpload -- 2.11e+00 / 0.00e+00, 7.92e+00 / 0.00e+00 | 20260505 23:40:14.705 | INFO | PerfReport: 1.33%, LockDownload -- 1.30e+00 / 0.00e+00, 2.90e+00 / 0.00e+00 | 20260505 23:40:14.705 | INFO | PerfReport: 20.53%, LockCompute -- 2.02e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260505 23:40:14.705 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.92e-02 / 0.00e+00 | 20260505 23:40:14.829 | INFO | Starting a round of Baz IO | 20260505 23:40:17.976 | INFO | Populating RTMetrics report | 20260505 23:40:17.976 | INFO | RTMetrics report populated | 20260505 23:40:17.976 | INFO | Opening tmp RTMetrics json file | 20260505 23:40:17.977 | INFO | Serializing RTMetrics to json | 20260505 23:40:17.979 | INFO | Serialization done, swapping tmp file now... | 20260505 23:40:17.979 | INFO | RTMetrics done | 20260505 23:40:18.062 | INFO | Finished a round of Baz IO: Wrote 1.885 GiB in 3.233 seconds | 20260505 23:40:18.062 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.42% / 0.00% / 78.58% | 20260505 23:40:18.062 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.103935% / 0% / 8.74201e-05% | 20260505 23:40:18.062 | INFO | Baz IO overall: | 20260505 23:40:18.062 | INFO | BytesWritten : 4126355316450 | 20260505 23:40:18.062 | INFO | NumEvents : 1943562991121 | 20260505 23:40:19.848 | INFO | Analyzing chunk frames = [7225344,7225856) | 20260505 23:40:19.848 | INFO | MemInfo: { | 20260505 23:40:19.848 | INFO | "gpu" : " 17 GB", | 20260505 23:40:19.848 | INFO | "huge" : "", | 20260505 23:40:19.849 | INFO | "huge_shared" : " 27 GB", | 20260505 23:40:19.849 | INFO | "malloc" : " 14 GB", | 20260505 23:40:19.849 | INFO | "pinned" : " 29 GB" | 20260505 23:40:19.849 | INFO | } | 20260505 23:40:19.849 | INFO | Perf report for the last 5.14456 seconds: | 20260505 23:40:19.849 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:40:19.849 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:19.849 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:19.849 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:19.849 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260505 23:40:19.849 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:19.849 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260505 23:40:19.849 | INFO | PRE_HQ : 59.6%, 1.8, 54.6ms, 0 | 20260505 23:40:19.849 | INFO | BAZWRITER : 61.8%, 3.6, 113.9ms, 0 | 20260505 23:40:19.849 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 23:40:19.849 | INFO | Uploaded 36GiB to GPU in 2.14501s (16.7831GiB/s) | 20260505 23:40:19.849 | INFO | Downloaded 38GiB to GPU in 2.2469s (16.9122GiB/s) | 20260505 23:40:24.986 | INFO | Analyzing chunk frames = [7225856,7226368) | 20260505 23:40:24.986 | INFO | MemInfo: { | 20260505 23:40:24.986 | INFO | "gpu" : " 17 GB", | 20260505 23:40:24.987 | INFO | "huge" : "", | 20260505 23:40:24.987 | INFO | "huge_shared" : " 27 GB", | 20260505 23:40:24.987 | INFO | "malloc" : " 14 GB", | 20260505 23:40:24.987 | INFO | "pinned" : " 29 GB" | 20260505 23:40:24.987 | INFO | } | 20260505 23:40:24.987 | INFO | Perf report for the last 5.13804 seconds: | 20260505 23:40:24.987 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:40:24.987 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:24.987 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:24.987 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:24.987 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260505 23:40:24.987 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:24.987 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:24.987 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:40:24.987 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:24.987 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:24.987 | INFO | Uploaded 36GiB to GPU in 1.98265s (18.1575GiB/s) | 20260505 23:40:24.987 | INFO | Downloaded 34GiB to GPU in 1.99575s (17.0362GiB/s) | 20260505 23:40:30.118 | INFO | Analyzing chunk frames = [7226368,7226880) | 20260505 23:40:30.118 | INFO | MemInfo: { | 20260505 23:40:30.118 | INFO | "gpu" : " 17 GB", | 20260505 23:40:30.118 | INFO | "huge" : "", | 20260505 23:40:30.118 | INFO | "huge_shared" : " 27 GB", | 20260505 23:40:30.118 | INFO | "malloc" : " 14 GB", | 20260505 23:40:30.118 | INFO | "pinned" : " 29 GB" | 20260505 23:40:30.118 | INFO | } | 20260505 23:40:30.118 | INFO | Perf report for the last 5.13147 seconds: | 20260505 23:40:30.118 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:40:30.118 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:30.118 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:30.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:30.118 | INFO | ANALYSIS : 61.5%, 3.0, 95.5ms, 0 | 20260505 23:40:30.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:30.118 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:30.118 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260505 23:40:30.118 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:30.118 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:30.118 | INFO | Uploaded 36GiB to GPU in 2.07336s (17.3631GiB/s) | 20260505 23:40:30.118 | INFO | Downloaded 34GiB to GPU in 2.01594s (16.8656GiB/s) | 20260505 23:40:35.260 | INFO | Analyzing chunk frames = [7226880,7227392) | 20260505 23:40:35.260 | INFO | MemInfo: { | 20260505 23:40:35.260 | INFO | "gpu" : " 17 GB", | 20260505 23:40:35.260 | INFO | "huge" : "", | 20260505 23:40:35.260 | INFO | "huge_shared" : " 27 GB", | 20260505 23:40:35.260 | INFO | "malloc" : " 14 GB", | 20260505 23:40:35.260 | INFO | "pinned" : " 29 GB" | 20260505 23:40:35.260 | INFO | } | 20260505 23:40:35.260 | INFO | Perf report for the last 5.14186 seconds: | 20260505 23:40:35.260 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:40:35.260 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:35.260 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:35.260 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:35.260 | INFO | ANALYSIS : 61.6%, 3.0, 95.7ms, 0 | 20260505 23:40:35.260 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:35.260 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:35.260 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 23:40:35.260 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:35.260 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:35.260 | INFO | Uploaded 36GiB to GPU in 2.11411s (17.0284GiB/s) | 20260505 23:40:35.260 | INFO | Downloaded 34GiB to GPU in 2.02181s (16.8166GiB/s) | 20260505 23:40:40.397 | INFO | Analyzing chunk frames = [7227392,7227904) | 20260505 23:40:40.397 | INFO | MemInfo: { | 20260505 23:40:40.397 | INFO | "gpu" : " 17 GB", | 20260505 23:40:40.397 | INFO | "huge" : "", | 20260505 23:40:40.397 | INFO | "huge_shared" : " 27 GB", | 20260505 23:40:40.397 | INFO | "malloc" : " 14 GB", | 20260505 23:40:40.397 | INFO | "pinned" : " 29 GB" | 20260505 23:40:40.397 | INFO | } | 20260505 23:40:40.397 | INFO | Perf report for the last 5.13681 seconds: | 20260505 23:40:40.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:40:40.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:40.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:40.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:40.397 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260505 23:40:40.397 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:40.397 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:40.397 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 23:40:40.397 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:40.397 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:40.397 | INFO | Uploaded 36GiB to GPU in 2.13385s (16.8709GiB/s) | 20260505 23:40:40.397 | INFO | Downloaded 34GiB to GPU in 2.0684s (16.4378GiB/s) | 20260505 23:40:45.538 | INFO | Analyzing chunk frames = [7227904,7228416) | 20260505 23:40:45.538 | INFO | MemInfo: { | 20260505 23:40:45.538 | INFO | "gpu" : " 17 GB", | 20260505 23:40:45.538 | INFO | "huge" : "", | 20260505 23:40:45.538 | INFO | "huge_shared" : " 27 GB", | 20260505 23:40:45.538 | INFO | "malloc" : " 14 GB", | 20260505 23:40:45.538 | INFO | "pinned" : " 29 GB" | 20260505 23:40:45.538 | INFO | } | 20260505 23:40:45.538 | INFO | Perf report for the last 5.1412 seconds: | 20260505 23:40:45.538 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:40:45.538 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:45.538 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:45.538 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:45.538 | INFO | ANALYSIS : 61.3%, 3.0, 95.8ms, 0 | 20260505 23:40:45.538 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:45.538 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:45.538 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:40:45.538 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:45.538 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:45.538 | INFO | Uploaded 36GiB to GPU in 2.06905s (17.3993GiB/s) | 20260505 23:40:45.538 | INFO | Downloaded 34GiB to GPU in 2.03627s (16.6972GiB/s) | 20260505 23:40:50.679 | INFO | Analyzing chunk frames = [7228416,7228928) | 20260505 23:40:50.679 | INFO | MemInfo: { | 20260505 23:40:50.679 | INFO | "gpu" : " 17 GB", | 20260505 23:40:50.680 | INFO | "huge" : "", | 20260505 23:40:50.680 | INFO | "huge_shared" : " 27 GB", | 20260505 23:40:50.680 | INFO | "malloc" : " 14 GB", | 20260505 23:40:50.680 | INFO | "pinned" : " 29 GB" | 20260505 23:40:50.680 | INFO | } | 20260505 23:40:50.680 | INFO | Perf report for the last 5.14159 seconds: | 20260505 23:40:50.680 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:40:50.680 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:50.680 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:50.680 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:50.680 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260505 23:40:50.680 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:50.680 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:50.680 | INFO | PRE_HQ : 27.2%, 1.0, 14.2ms, 0 | 20260505 23:40:50.680 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:50.680 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:50.680 | INFO | Uploaded 36GiB to GPU in 2.09898s (17.1512GiB/s) | 20260505 23:40:50.680 | INFO | Downloaded 34GiB to GPU in 2.08703s (16.2911GiB/s) | 20260505 23:40:55.818 | INFO | Analyzing chunk frames = [7228928,7229440) | 20260505 23:40:55.818 | INFO | MemInfo: { | 20260505 23:40:55.818 | INFO | "gpu" : " 17 GB", | 20260505 23:40:55.818 | INFO | "huge" : "", | 20260505 23:40:55.818 | INFO | "huge_shared" : " 27 GB", | 20260505 23:40:55.818 | INFO | "malloc" : " 14 GB", | 20260505 23:40:55.818 | INFO | "pinned" : " 29 GB" | 20260505 23:40:55.818 | INFO | } | 20260505 23:40:55.818 | INFO | Perf report for the last 5.13878 seconds: | 20260505 23:40:55.818 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:40:55.818 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:55.818 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:55.818 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:55.818 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 23:40:55.818 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:55.818 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:55.818 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:40:55.819 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:55.819 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:40:55.819 | INFO | Uploaded 36GiB to GPU in 2.12905s (16.909GiB/s) | 20260505 23:40:55.819 | INFO | Downloaded 34GiB to GPU in 2.01596s (16.8654GiB/s) | 20260505 23:40:55.944 | INFO | Starting a round of Baz IO | 20260505 23:40:59.088 | INFO | Populating RTMetrics report | 20260505 23:40:59.088 | INFO | RTMetrics report populated | 20260505 23:40:59.088 | INFO | Opening tmp RTMetrics json file | 20260505 23:40:59.090 | INFO | Serializing RTMetrics to json | 20260505 23:40:59.091 | INFO | Serialization done, swapping tmp file now... | 20260505 23:40:59.092 | INFO | RTMetrics done | 20260505 23:40:59.171 | INFO | Finished a round of Baz IO: Wrote 1.885 GiB in 3.227 seconds | 20260505 23:40:59.171 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.41% / 0.00% / 78.58% | 20260505 23:40:59.171 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.103223% / 0% / 6.35783e-05% | 20260505 23:40:59.171 | INFO | Baz IO overall: | 20260505 23:40:59.171 | INFO | BytesWritten : 4128379437825 | 20260505 23:40:59.171 | INFO | NumEvents : 1944490738879 | 20260505 23:41:00.954 | INFO | Analyzing chunk frames = [7229440,7229952) | 20260505 23:41:00.954 | INFO | MemInfo: { | 20260505 23:41:00.954 | INFO | "gpu" : " 17 GB", | 20260505 23:41:00.954 | INFO | "huge" : "", | 20260505 23:41:00.954 | INFO | "huge_shared" : " 27 GB", | 20260505 23:41:00.954 | INFO | "malloc" : " 14 GB", | 20260505 23:41:00.954 | INFO | "pinned" : " 29 GB" | 20260505 23:41:00.954 | INFO | } | 20260505 23:41:00.954 | INFO | Perf report for the last 5.13534 seconds: | 20260505 23:41:00.954 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:41:00.954 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:00.954 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:00.954 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:00.954 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 23:41:00.954 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:00.954 | INFO | RT_METRICS : 43.4%, 1.0, 22.8ms, 0 | 20260505 23:41:00.954 | INFO | PRE_HQ : 59.9%, 1.7, 54.3ms, 0 | 20260505 23:41:00.954 | INFO | BAZWRITER : 62.1%, 3.6, 113.9ms, 0 | 20260505 23:41:00.954 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 23:41:00.954 | INFO | Uploaded 36GiB to GPU in 2.16885s (16.5986GiB/s) | 20260505 23:41:00.954 | INFO | Downloaded 38GiB to GPU in 2.23656s (16.9904GiB/s) | 20260505 23:41:06.094 | INFO | Analyzing chunk frames = [7229952,7230464) | 20260505 23:41:06.094 | INFO | MemInfo: { | 20260505 23:41:06.094 | INFO | "gpu" : " 17 GB", | 20260505 23:41:06.094 | INFO | "huge" : "", | 20260505 23:41:06.094 | INFO | "huge_shared" : " 27 GB", | 20260505 23:41:06.094 | INFO | "malloc" : " 14 GB", | 20260505 23:41:06.094 | INFO | "pinned" : " 29 GB" | 20260505 23:41:06.094 | INFO | } | 20260505 23:41:06.094 | INFO | Perf report for the last 5.14068 seconds: | 20260505 23:41:06.094 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:41:06.094 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:06.094 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:06.094 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:06.095 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 23:41:06.095 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:06.095 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:06.095 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260505 23:41:06.095 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:06.095 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:06.095 | INFO | Uploaded 36GiB to GPU in 2.08518s (17.2647GiB/s) | 20260505 23:41:06.095 | INFO | Downloaded 34GiB to GPU in 2.02419s (16.7968GiB/s) | 20260505 23:41:11.235 | INFO | Analyzing chunk frames = [7230464,7230976) | 20260505 23:41:11.235 | INFO | MemInfo: { | 20260505 23:41:11.235 | INFO | "gpu" : " 17 GB", | 20260505 23:41:11.235 | INFO | "huge" : "", | 20260505 23:41:11.235 | INFO | "huge_shared" : " 27 GB", | 20260505 23:41:11.235 | INFO | "malloc" : " 14 GB", | 20260505 23:41:11.235 | INFO | "pinned" : " 29 GB" | 20260505 23:41:11.235 | INFO | } | 20260505 23:41:11.235 | INFO | Perf report for the last 5.14048 seconds: | 20260505 23:41:11.235 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:41:11.235 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:11.235 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:11.235 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:11.235 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260505 23:41:11.235 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:11.235 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:11.235 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:41:11.235 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:11.235 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:11.235 | INFO | Uploaded 36GiB to GPU in 2.16003s (16.6664GiB/s) | 20260505 23:41:11.235 | INFO | Downloaded 34GiB to GPU in 2.05799s (16.521GiB/s) | 20260505 23:41:16.373 | INFO | Analyzing chunk frames = [7230976,7231488) | 20260505 23:41:16.373 | INFO | MemInfo: { | 20260505 23:41:16.373 | INFO | "gpu" : " 17 GB", | 20260505 23:41:16.373 | INFO | "huge" : "", | 20260505 23:41:16.373 | INFO | "huge_shared" : " 27 GB", | 20260505 23:41:16.373 | INFO | "malloc" : " 14 GB", | 20260505 23:41:16.373 | INFO | "pinned" : " 29 GB" | 20260505 23:41:16.373 | INFO | } | 20260505 23:41:16.373 | INFO | Perf report for the last 5.13787 seconds: | 20260505 23:41:16.373 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:41:16.373 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:16.373 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:16.373 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:16.373 | INFO | ANALYSIS : 61.7%, 3.0, 95.8ms, 0 | 20260505 23:41:16.373 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:16.373 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:16.373 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 23:41:16.373 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:16.373 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:16.373 | INFO | Uploaded 36GiB to GPU in 2.13729s (16.8438GiB/s) | 20260505 23:41:16.373 | INFO | Downloaded 34GiB to GPU in 2.00153s (16.987GiB/s) | 20260505 23:41:21.520 | INFO | Analyzing chunk frames = [7231488,7232000) | 20260505 23:41:21.520 | INFO | MemInfo: { | 20260505 23:41:21.520 | INFO | "gpu" : " 17 GB", | 20260505 23:41:21.520 | INFO | "huge" : "", | 20260505 23:41:21.520 | INFO | "huge_shared" : " 27 GB", | 20260505 23:41:21.520 | INFO | "malloc" : " 14 GB", | 20260505 23:41:21.520 | INFO | "pinned" : " 29 GB" | 20260505 23:41:21.520 | INFO | } | 20260505 23:41:21.520 | INFO | Perf report for the last 5.14715 seconds: | 20260505 23:41:21.520 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:41:21.520 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:21.520 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:21.520 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:21.520 | INFO | ANALYSIS : 61.2%, 3.0, 95.6ms, 0 | 20260505 23:41:21.520 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:21.520 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:21.520 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 23:41:21.520 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:21.520 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:21.520 | INFO | Uploaded 36GiB to GPU in 2.10769s (17.0803GiB/s) | 20260505 23:41:21.520 | INFO | Downloaded 34GiB to GPU in 2.0492s (16.5918GiB/s) | 20260505 23:41:26.657 | INFO | Analyzing chunk frames = [7232000,7232512) | 20260505 23:41:26.657 | INFO | MemInfo: { | 20260505 23:41:26.657 | INFO | "gpu" : " 17 GB", | 20260505 23:41:26.657 | INFO | "huge" : "", | 20260505 23:41:26.657 | INFO | "huge_shared" : " 27 GB", | 20260505 23:41:26.657 | INFO | "malloc" : " 14 GB", | 20260505 23:41:26.657 | INFO | "pinned" : " 29 GB" | 20260505 23:41:26.657 | INFO | } | 20260505 23:41:26.657 | INFO | Perf report for the last 5.13744 seconds: | 20260505 23:41:26.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:41:26.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:26.658 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:26.658 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:26.658 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260505 23:41:26.658 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:26.658 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:26.658 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 23:41:26.658 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:26.658 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:26.658 | INFO | Uploaded 36GiB to GPU in 2.07356s (17.3615GiB/s) | 20260505 23:41:26.658 | INFO | Downloaded 34GiB to GPU in 2.03611s (16.6985GiB/s) | 20260505 23:41:31.798 | INFO | Analyzing chunk frames = [7232512,7233024) | 20260505 23:41:31.798 | INFO | MemInfo: { | 20260505 23:41:31.798 | INFO | "gpu" : " 17 GB", | 20260505 23:41:31.799 | INFO | "huge" : "", | 20260505 23:41:31.799 | INFO | "huge_shared" : " 27 GB", | 20260505 23:41:31.799 | INFO | "malloc" : " 14 GB", | 20260505 23:41:31.799 | INFO | "pinned" : " 29 GB" | 20260505 23:41:31.799 | INFO | } | 20260505 23:41:31.799 | INFO | Perf report for the last 5.14117 seconds: | 20260505 23:41:31.799 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:41:31.799 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:31.799 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:31.799 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:31.799 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260505 23:41:31.799 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:31.799 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:31.799 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 23:41:31.799 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:31.799 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:31.799 | INFO | Uploaded 36GiB to GPU in 2.09422s (17.1901GiB/s) | 20260505 23:41:31.799 | INFO | Downloaded 34GiB to GPU in 2.08908s (16.2751GiB/s) | 20260505 23:41:36.937 | INFO | Analyzing chunk frames = [7233024,7233536) | 20260505 23:41:36.937 | INFO | MemInfo: { | 20260505 23:41:36.937 | INFO | "gpu" : " 17 GB", | 20260505 23:41:36.937 | INFO | "huge" : "", | 20260505 23:41:36.937 | INFO | "huge_shared" : " 27 GB", | 20260505 23:41:36.937 | INFO | "malloc" : " 14 GB", | 20260505 23:41:36.937 | INFO | "pinned" : " 29 GB" | 20260505 23:41:36.937 | INFO | } | 20260505 23:41:36.937 | INFO | Perf report for the last 5.13809 seconds: | 20260505 23:41:36.937 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:41:36.937 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:36.937 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:36.937 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:36.937 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 23:41:36.937 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:36.937 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:36.937 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 23:41:36.937 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:36.937 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:36.937 | INFO | Uploaded 36GiB to GPU in 2.13646s (16.8503GiB/s) | 20260505 23:41:36.937 | INFO | Downloaded 34GiB to GPU in 2.02891s (16.7577GiB/s) | 20260505 23:41:36.937 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 23:41:36.937 | INFO | Total mem usage: 15.3 GiB | 20260505 23:41:36.937 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 23:41:36.937 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.11%) | 20260505 23:41:36.937 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 23:41:36.937 | INFO | High Water Breakdown: | 20260505 23:41:36.937 | INFO | 60.9% : 8.55 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 23:41:36.937 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 23:41:36.937 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 23:41:36.937 | INFO | ------------------------------------------ | 20260505 23:41:36.937 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:41:36.937 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 23:41:36.937 | INFO | Total mem usage: 26.5 GiB | 20260505 23:41:36.937 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 23:41:36.937 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 23:41:36.937 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 23:41:36.937 | INFO | High Water Breakdown: | 20260505 23:41:36.937 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:41:36.937 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:41:36.937 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:41:36.937 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:41:36.937 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:41:36.937 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:41:36.937 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 23:41:36.937 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 23:41:36.937 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:41:36.937 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 23:41:36.937 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 23:41:36.937 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 23:41:36.937 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 23:41:36.937 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 23:41:36.937 | INFO | ------------------------------------------ | 20260505 23:41:36.937 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:41:36.937 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 23:41:36.937 | INFO | Total mem usage: 25.6 GiB | 20260505 23:41:36.937 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 23:41:36.937 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 23:41:36.937 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 23:41:36.938 | INFO | High Water Breakdown: | 20260505 23:41:36.938 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 23:41:36.938 | INFO | ------------------------------------------ | 20260505 23:41:36.938 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:41:36.938 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 23:41:36.938 | INFO | Total mem usage: 15.7 GiB | 20260505 23:41:36.938 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 23:41:36.938 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 23:41:36.938 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 23:41:36.938 | INFO | High Water Breakdown: | 20260505 23:41:36.938 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 23:41:36.938 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 23:41:36.938 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 23:41:36.938 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 23:41:36.938 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 23:41:36.938 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 23:41:36.938 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 23:41:36.938 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 23:41:36.938 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 23:41:36.938 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:41:36.938 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 23:41:36.938 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:41:36.938 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 23:41:36.938 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 23:41:36.938 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 23:41:36.938 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 23:41:36.938 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:41:36.938 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 23:41:36.938 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:41:36.938 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:41:36.938 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:41:36.938 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 23:41:36.938 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 23:41:36.938 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:41:36.938 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 23:41:36.938 | INFO | ------------------------------------------ | 20260505 23:41:36.938 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:41:36.938 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 23:41:36.938 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:41:36.938 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:41:36.938 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.39e+01, 1.34e+01 / 7.93e-01 | 20260505 23:41:36.938 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.05e-01, 1.02e-02 / 4.64e-03 | 20260505 23:41:36.938 | INFO | PerfReport: 11.83%, Baseline -- 3.71e+00 / 4.51e+00, 5.68e-01 / 1.94e-01 | 20260505 23:41:36.938 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.52e+00, 1.92e-01 / 9.06e-02 | 20260505 23:41:36.938 | INFO | PerfReport: 15.98%, DME -- 4.01e+01 / 4.39e+01, 3.82e+00 / 9.36e-01 | 20260505 23:41:36.938 | INFO | PerfReport: 57.33%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.53e+00 / 9.93e-01 | 20260505 23:41:36.938 | INFO | PerfReport: 7.10%, PulseAccumulating -- 2.23e+00 / 2.75e+00, 4.25e-01 / 2.28e-01 | 20260505 23:41:36.938 | INFO | PerfReport: 3.11%, Metrics -- 9.73e-01 / 1.25e+00, 2.64e-01 / 8.99e-02 | 20260505 23:41:36.938 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.37e-01 / 3.92e-01, 5.76e-02 / 2.68e-02 | 20260505 23:41:36.938 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 23:41:36.938 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:41:36.938 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:41:36.938 | INFO | PerfReport: Full Execution -- 9.84e+01 / -nan, 2.47e+01 / 0.00e+00 | 20260505 23:41:36.938 | INFO | PerfReport: 22.30%, Upload -- 2.19e+01 / 0.00e+00, 7.52e+00 / 0.00e+00 | 20260505 23:41:36.938 | INFO | PerfReport: 21.84%, Download -- 2.15e+01 / 0.00e+00, 7.63e+00 / 0.00e+00 | 20260505 23:41:36.938 | INFO | PerfReport: 31.85%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 23:41:36.938 | INFO | PerfReport: 2.13%, LockUpload -- 2.10e+00 / 0.00e+00, 7.81e+00 / 0.00e+00 | 20260505 23:41:36.938 | INFO | PerfReport: 1.35%, LockDownload -- 1.32e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260505 23:41:36.938 | INFO | PerfReport: 20.51%, LockCompute -- 2.02e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260505 23:41:36.938 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.60e-02 / 0.00e+00 | 20260505 23:41:37.053 | INFO | Starting a round of Baz IO | 20260505 23:41:40.204 | INFO | Populating RTMetrics report | 20260505 23:41:40.204 | INFO | RTMetrics report populated | 20260505 23:41:40.204 | INFO | Opening tmp RTMetrics json file | 20260505 23:41:40.205 | INFO | Serializing RTMetrics to json | 20260505 23:41:40.206 | INFO | Serialization done, swapping tmp file now... | 20260505 23:41:40.207 | INFO | RTMetrics done | 20260505 23:41:40.283 | INFO | Finished a round of Baz IO: Wrote 1.885 GiB in 3.23 seconds | 20260505 23:41:40.283 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.41% / 0.00% / 78.59% | 20260505 23:41:40.283 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.103398% / 0% / 8.34465e-05% | 20260505 23:41:40.283 | INFO | Baz IO overall: | 20260505 23:41:40.283 | INFO | BytesWritten : 4130403923966 | 20260505 23:41:40.283 | INFO | NumEvents : 1945418850012 | 20260505 23:41:42.065 | INFO | Analyzing chunk frames = [7233536,7234048) | 20260505 23:41:42.065 | INFO | MemInfo: { | 20260505 23:41:42.065 | INFO | "gpu" : " 17 GB", | 20260505 23:41:42.065 | INFO | "huge" : "", | 20260505 23:41:42.065 | INFO | "huge_shared" : " 27 GB", | 20260505 23:41:42.065 | INFO | "malloc" : " 14 GB", | 20260505 23:41:42.065 | INFO | "pinned" : " 29 GB" | 20260505 23:41:42.066 | INFO | } | 20260505 23:41:42.066 | INFO | Perf report for the last 5.12878 seconds: | 20260505 23:41:42.066 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:41:42.066 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:42.066 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:42.066 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:42.066 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260505 23:41:42.066 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:42.066 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260505 23:41:42.066 | INFO | PRE_HQ : 59.9%, 1.7, 54.3ms, 0 | 20260505 23:41:42.066 | INFO | BAZWRITER : 62.1%, 3.6, 114.0ms, 0 | 20260505 23:41:42.066 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 23:41:42.066 | INFO | Uploaded 36GiB to GPU in 2.14544s (16.7798GiB/s) | 20260505 23:41:42.066 | INFO | Downloaded 38GiB to GPU in 2.22792s (17.0563GiB/s) | 20260505 23:41:47.208 | INFO | Analyzing chunk frames = [7234048,7234560) | 20260505 23:41:47.208 | INFO | MemInfo: { | 20260505 23:41:47.208 | INFO | "gpu" : " 17 GB", | 20260505 23:41:47.208 | INFO | "huge" : "", | 20260505 23:41:47.208 | INFO | "huge_shared" : " 27 GB", | 20260505 23:41:47.208 | INFO | "malloc" : " 14 GB", | 20260505 23:41:47.208 | INFO | "pinned" : " 29 GB" | 20260505 23:41:47.208 | INFO | } | 20260505 23:41:47.208 | INFO | Perf report for the last 5.14241 seconds: | 20260505 23:41:47.208 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:41:47.208 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:47.208 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:47.208 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:47.208 | INFO | ANALYSIS : 61.8%, 3.0, 96.6ms, 0 | 20260505 23:41:47.208 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:47.208 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:47.208 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:41:47.208 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:47.208 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:47.208 | INFO | Uploaded 36GiB to GPU in 2.09295s (17.2006GiB/s) | 20260505 23:41:47.208 | INFO | Downloaded 34GiB to GPU in 2.02321s (16.805GiB/s) | 20260505 23:41:52.353 | INFO | Analyzing chunk frames = [7234560,7235072) | 20260505 23:41:52.353 | INFO | MemInfo: { | 20260505 23:41:52.353 | INFO | "gpu" : " 17 GB", | 20260505 23:41:52.353 | INFO | "huge" : "", | 20260505 23:41:52.353 | INFO | "huge_shared" : " 27 GB", | 20260505 23:41:52.353 | INFO | "malloc" : " 14 GB", | 20260505 23:41:52.354 | INFO | "pinned" : " 29 GB" | 20260505 23:41:52.354 | INFO | } | 20260505 23:41:52.354 | INFO | Perf report for the last 5.1456 seconds: | 20260505 23:41:52.354 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:41:52.354 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:52.354 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:52.354 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:52.354 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260505 23:41:52.354 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:52.354 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:52.354 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:41:52.354 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:52.354 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:52.354 | INFO | Uploaded 36GiB to GPU in 2.16605s (16.6201GiB/s) | 20260505 23:41:52.354 | INFO | Downloaded 34GiB to GPU in 2.05959s (16.5081GiB/s) | 20260505 23:41:57.497 | INFO | Analyzing chunk frames = [7235072,7235584) | 20260505 23:41:57.497 | INFO | MemInfo: { | 20260505 23:41:57.497 | INFO | "gpu" : " 17 GB", | 20260505 23:41:57.497 | INFO | "huge" : "", | 20260505 23:41:57.497 | INFO | "huge_shared" : " 27 GB", | 20260505 23:41:57.497 | INFO | "malloc" : " 14 GB", | 20260505 23:41:57.497 | INFO | "pinned" : " 29 GB" | 20260505 23:41:57.497 | INFO | } | 20260505 23:41:57.497 | INFO | Perf report for the last 5.14327 seconds: | 20260505 23:41:57.497 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:41:57.497 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:57.497 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:57.497 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:57.497 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260505 23:41:57.497 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:57.497 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:57.497 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 23:41:57.497 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:57.497 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:41:57.497 | INFO | Uploaded 36GiB to GPU in 2.14148s (16.8108GiB/s) | 20260505 23:41:57.497 | INFO | Downloaded 34GiB to GPU in 2.02228s (16.8127GiB/s) | 20260505 23:42:02.628 | INFO | Analyzing chunk frames = [7235584,7236096) | 20260505 23:42:02.628 | INFO | MemInfo: { | 20260505 23:42:02.628 | INFO | "gpu" : " 17 GB", | 20260505 23:42:02.628 | INFO | "huge" : "", | 20260505 23:42:02.628 | INFO | "huge_shared" : " 27 GB", | 20260505 23:42:02.628 | INFO | "malloc" : " 14 GB", | 20260505 23:42:02.628 | INFO | "pinned" : " 29 GB" | 20260505 23:42:02.628 | INFO | } | 20260505 23:42:02.628 | INFO | Perf report for the last 5.13158 seconds: | 20260505 23:42:02.628 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:42:02.628 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:02.628 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:02.628 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:02.628 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260505 23:42:02.628 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:02.629 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:02.629 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 23:42:02.629 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:02.629 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:02.629 | INFO | Uploaded 36GiB to GPU in 2.09504s (17.1834GiB/s) | 20260505 23:42:02.629 | INFO | Downloaded 34GiB to GPU in 2.045s (16.6259GiB/s) | 20260505 23:42:07.765 | INFO | Analyzing chunk frames = [7236096,7236608) | 20260505 23:42:07.765 | INFO | MemInfo: { | 20260505 23:42:07.765 | INFO | "gpu" : " 17 GB", | 20260505 23:42:07.765 | INFO | "huge" : "", | 20260505 23:42:07.765 | INFO | "huge_shared" : " 27 GB", | 20260505 23:42:07.765 | INFO | "malloc" : " 14 GB", | 20260505 23:42:07.765 | INFO | "pinned" : " 29 GB" | 20260505 23:42:07.765 | INFO | } | 20260505 23:42:07.765 | INFO | Perf report for the last 5.13695 seconds: | 20260505 23:42:07.765 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:42:07.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:07.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:07.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:07.765 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260505 23:42:07.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:07.765 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:07.765 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:42:07.765 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:07.766 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:07.766 | INFO | Uploaded 36GiB to GPU in 2.08708s (17.2489GiB/s) | 20260505 23:42:07.766 | INFO | Downloaded 34GiB to GPU in 2.04062s (16.6616GiB/s) | 20260505 23:42:12.908 | INFO | Analyzing chunk frames = [7236608,7237120) | 20260505 23:42:12.908 | INFO | MemInfo: { | 20260505 23:42:12.908 | INFO | "gpu" : " 17 GB", | 20260505 23:42:12.908 | INFO | "huge" : "", | 20260505 23:42:12.908 | INFO | "huge_shared" : " 27 GB", | 20260505 23:42:12.908 | INFO | "malloc" : " 14 GB", | 20260505 23:42:12.908 | INFO | "pinned" : " 29 GB" | 20260505 23:42:12.908 | INFO | } | 20260505 23:42:12.908 | INFO | Perf report for the last 5.14255 seconds: | 20260505 23:42:12.908 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:42:12.908 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:12.908 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:12.908 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:12.908 | INFO | ANALYSIS : 62.7%, 3.0, 98.0ms, 0 | 20260505 23:42:12.908 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:12.908 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:12.908 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 23:42:12.908 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:12.908 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:12.908 | INFO | Uploaded 36GiB to GPU in 2.13102s (16.8934GiB/s) | 20260505 23:42:12.908 | INFO | Downloaded 34GiB to GPU in 2.11257s (16.0941GiB/s) | 20260505 23:42:18.046 | INFO | Analyzing chunk frames = [7237120,7237632) | 20260505 23:42:18.046 | INFO | MemInfo: { | 20260505 23:42:18.046 | INFO | "gpu" : " 17 GB", | 20260505 23:42:18.046 | INFO | "huge" : "", | 20260505 23:42:18.046 | INFO | "huge_shared" : " 27 GB", | 20260505 23:42:18.046 | INFO | "malloc" : " 14 GB", | 20260505 23:42:18.046 | INFO | "pinned" : " 29 GB" | 20260505 23:42:18.046 | INFO | } | 20260505 23:42:18.046 | INFO | Perf report for the last 5.13781 seconds: | 20260505 23:42:18.046 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:42:18.046 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:18.046 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:18.046 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:18.046 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 23:42:18.046 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:18.046 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:18.046 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 23:42:18.046 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:18.046 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:18.046 | INFO | Uploaded 36GiB to GPU in 2.11747s (17.0015GiB/s) | 20260505 23:42:18.046 | INFO | Downloaded 34GiB to GPU in 2.0137s (16.8843GiB/s) | 20260505 23:42:18.168 | INFO | Starting a round of Baz IO | 20260505 23:42:21.313 | INFO | Populating RTMetrics report | 20260505 23:42:21.313 | INFO | RTMetrics report populated | 20260505 23:42:21.313 | INFO | Opening tmp RTMetrics json file | 20260505 23:42:21.314 | INFO | Serializing RTMetrics to json | 20260505 23:42:21.317 | INFO | Serialization done, swapping tmp file now... | 20260505 23:42:21.320 | INFO | RTMetrics done | 20260505 23:42:21.389 | INFO | Finished a round of Baz IO: Wrote 1.883 GiB in 3.221 seconds | 20260505 23:42:21.390 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.41% / 0.00% / 78.59% | 20260505 23:42:21.390 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.103068% / 3.97364e-06% / 7.15256e-05% | 20260505 23:42:21.390 | INFO | Baz IO overall: | 20260505 23:42:21.390 | INFO | BytesWritten : 4132426206193 | 20260505 23:42:21.390 | INFO | NumEvents : 1946345579470 | 20260505 23:42:23.186 | INFO | Analyzing chunk frames = [7237632,7238144) | 20260505 23:42:23.186 | INFO | MemInfo: { | 20260505 23:42:23.186 | INFO | "gpu" : " 17 GB", | 20260505 23:42:23.186 | INFO | "huge" : "", | 20260505 23:42:23.186 | INFO | "huge_shared" : " 27 GB", | 20260505 23:42:23.186 | INFO | "malloc" : " 14 GB", | 20260505 23:42:23.186 | INFO | "pinned" : " 29 GB" | 20260505 23:42:23.186 | INFO | } | 20260505 23:42:23.186 | INFO | Perf report for the last 5.1405 seconds: | 20260505 23:42:23.186 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:42:23.186 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:23.186 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:23.186 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:23.186 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260505 23:42:23.186 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:23.186 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260505 23:42:23.186 | INFO | PRE_HQ : 59.9%, 1.7, 54.3ms, 0 | 20260505 23:42:23.186 | INFO | BAZWRITER : 61.9%, 3.6, 114.6ms, 0 | 20260505 23:42:23.187 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 23:42:23.187 | INFO | Uploaded 36GiB to GPU in 2.16748s (16.6091GiB/s) | 20260505 23:42:23.187 | INFO | Downloaded 38GiB to GPU in 2.23745s (16.9837GiB/s) | 20260505 23:42:28.326 | INFO | Analyzing chunk frames = [7238144,7238656) | 20260505 23:42:28.326 | INFO | MemInfo: { | 20260505 23:42:28.326 | INFO | "gpu" : " 17 GB", | 20260505 23:42:28.326 | INFO | "huge" : "", | 20260505 23:42:28.326 | INFO | "huge_shared" : " 27 GB", | 20260505 23:42:28.326 | INFO | "malloc" : " 14 GB", | 20260505 23:42:28.326 | INFO | "pinned" : " 29 GB" | 20260505 23:42:28.326 | INFO | } | 20260505 23:42:28.326 | INFO | Perf report for the last 5.13938 seconds: | 20260505 23:42:28.326 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:42:28.326 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:28.326 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:28.326 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:28.326 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 23:42:28.326 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:28.326 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:28.326 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260505 23:42:28.326 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:28.326 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:28.326 | INFO | Uploaded 36GiB to GPU in 2.08399s (17.2746GiB/s) | 20260505 23:42:28.326 | INFO | Downloaded 34GiB to GPU in 2.03028s (16.7465GiB/s) | 20260505 23:42:33.467 | INFO | Analyzing chunk frames = [7238656,7239168) | 20260505 23:42:33.467 | INFO | MemInfo: { | 20260505 23:42:33.467 | INFO | "gpu" : " 17 GB", | 20260505 23:42:33.467 | INFO | "huge" : "", | 20260505 23:42:33.467 | INFO | "huge_shared" : " 27 GB", | 20260505 23:42:33.467 | INFO | "malloc" : " 14 GB", | 20260505 23:42:33.467 | INFO | "pinned" : " 29 GB" | 20260505 23:42:33.467 | INFO | } | 20260505 23:42:33.467 | INFO | Perf report for the last 5.14163 seconds: | 20260505 23:42:33.467 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:42:33.467 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:33.467 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:33.467 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:33.467 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260505 23:42:33.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:33.467 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:33.467 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260505 23:42:33.467 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:33.467 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:33.468 | INFO | Uploaded 36GiB to GPU in 2.14491s (16.784GiB/s) | 20260505 23:42:33.468 | INFO | Downloaded 34GiB to GPU in 2.03352s (16.7197GiB/s) | 20260505 23:42:38.607 | INFO | Analyzing chunk frames = [7239168,7239680) | 20260505 23:42:38.607 | INFO | MemInfo: { | 20260505 23:42:38.607 | INFO | "gpu" : " 17 GB", | 20260505 23:42:38.607 | INFO | "huge" : "", | 20260505 23:42:38.607 | INFO | "huge_shared" : " 27 GB", | 20260505 23:42:38.608 | INFO | "malloc" : " 14 GB", | 20260505 23:42:38.608 | INFO | "pinned" : " 29 GB" | 20260505 23:42:38.608 | INFO | } | 20260505 23:42:38.608 | INFO | Perf report for the last 5.14024 seconds: | 20260505 23:42:38.608 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:42:38.608 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:38.608 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:38.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:38.608 | INFO | ANALYSIS : 61.9%, 3.0, 96.0ms, 0 | 20260505 23:42:38.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:38.608 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:38.608 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260505 23:42:38.608 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:38.608 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:38.608 | INFO | Uploaded 36GiB to GPU in 2.13193s (16.8861GiB/s) | 20260505 23:42:38.608 | INFO | Downloaded 34GiB to GPU in 2.02673s (16.7758GiB/s) | 20260505 23:42:43.748 | INFO | Analyzing chunk frames = [7239680,7240192) | 20260505 23:42:43.748 | INFO | MemInfo: { | 20260505 23:42:43.748 | INFO | "gpu" : " 17 GB", | 20260505 23:42:43.748 | INFO | "huge" : "", | 20260505 23:42:43.748 | INFO | "huge_shared" : " 27 GB", | 20260505 23:42:43.748 | INFO | "malloc" : " 14 GB", | 20260505 23:42:43.748 | INFO | "pinned" : " 29 GB" | 20260505 23:42:43.748 | INFO | } | 20260505 23:42:43.748 | INFO | Perf report for the last 5.14031 seconds: | 20260505 23:42:43.748 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:42:43.748 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:43.748 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:43.748 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:43.748 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260505 23:42:43.748 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:43.748 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:43.748 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 23:42:43.748 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:43.748 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:43.748 | INFO | Uploaded 36GiB to GPU in 2.0868s (17.2513GiB/s) | 20260505 23:42:43.748 | INFO | Downloaded 34GiB to GPU in 2.04483s (16.6273GiB/s) | 20260505 23:42:48.883 | INFO | Analyzing chunk frames = [7240192,7240704) | 20260505 23:42:48.883 | INFO | MemInfo: { | 20260505 23:42:48.883 | INFO | "gpu" : " 17 GB", | 20260505 23:42:48.883 | INFO | "huge" : "", | 20260505 23:42:48.883 | INFO | "huge_shared" : " 27 GB", | 20260505 23:42:48.883 | INFO | "malloc" : " 14 GB", | 20260505 23:42:48.883 | INFO | "pinned" : " 29 GB" | 20260505 23:42:48.883 | INFO | } | 20260505 23:42:48.883 | INFO | Perf report for the last 5.13507 seconds: | 20260505 23:42:48.883 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:42:48.883 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:48.883 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:48.883 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:48.883 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260505 23:42:48.883 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:48.883 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:48.883 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:42:48.883 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:48.883 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:48.883 | INFO | Uploaded 36GiB to GPU in 2.06965s (17.3943GiB/s) | 20260505 23:42:48.883 | INFO | Downloaded 34GiB to GPU in 2.04117s (16.6571GiB/s) | 20260505 23:42:54.016 | INFO | Analyzing chunk frames = [7240704,7241216) | 20260505 23:42:54.016 | INFO | MemInfo: { | 20260505 23:42:54.016 | INFO | "gpu" : " 17 GB", | 20260505 23:42:54.016 | INFO | "huge" : "", | 20260505 23:42:54.016 | INFO | "huge_shared" : " 27 GB", | 20260505 23:42:54.016 | INFO | "malloc" : " 14 GB", | 20260505 23:42:54.016 | INFO | "pinned" : " 29 GB" | 20260505 23:42:54.016 | INFO | } | 20260505 23:42:54.016 | INFO | Perf report for the last 5.13339 seconds: | 20260505 23:42:54.016 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:42:54.016 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:54.016 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:54.016 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:54.017 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260505 23:42:54.017 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:54.017 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:54.017 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260505 23:42:54.017 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:54.017 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:54.017 | INFO | Uploaded 36GiB to GPU in 2.12741s (16.922GiB/s) | 20260505 23:42:54.017 | INFO | Downloaded 34GiB to GPU in 2.07974s (16.3482GiB/s) | 20260505 23:42:59.160 | INFO | Analyzing chunk frames = [7241216,7241728) | 20260505 23:42:59.160 | INFO | MemInfo: { | 20260505 23:42:59.160 | INFO | "gpu" : " 17 GB", | 20260505 23:42:59.160 | INFO | "huge" : "", | 20260505 23:42:59.160 | INFO | "huge_shared" : " 27 GB", | 20260505 23:42:59.160 | INFO | "malloc" : " 14 GB", | 20260505 23:42:59.160 | INFO | "pinned" : " 29 GB" | 20260505 23:42:59.160 | INFO | } | 20260505 23:42:59.160 | INFO | Perf report for the last 5.14337 seconds: | 20260505 23:42:59.160 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:42:59.160 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:59.160 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:59.160 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:59.160 | INFO | ANALYSIS : 61.9%, 3.0, 96.8ms, 0 | 20260505 23:42:59.160 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:59.160 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:59.160 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 23:42:59.160 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:59.160 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:42:59.160 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 23:42:59.160 | INFO | Uploaded 36GiB to GPU in 2.10868s (17.0723GiB/s) | 20260505 23:42:59.160 | INFO | Total mem usage: 15.3 GiB | 20260505 23:42:59.160 | INFO | Downloaded 34GiB to GPU in 2.00741s (16.9372GiB/s) | 20260505 23:42:59.160 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 23:42:59.160 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.1%) | 20260505 23:42:59.160 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 23:42:59.160 | INFO | High Water Breakdown: | 20260505 23:42:59.160 | INFO | 60.9% : 8.55 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 23:42:59.160 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 23:42:59.160 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 23:42:59.160 | INFO | ------------------------------------------ | 20260505 23:42:59.160 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:42:59.160 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 23:42:59.160 | INFO | Total mem usage: 26.5 GiB | 20260505 23:42:59.160 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 23:42:59.160 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 23:42:59.160 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 23:42:59.160 | INFO | High Water Breakdown: | 20260505 23:42:59.160 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:42:59.160 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:42:59.160 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:42:59.160 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:42:59.160 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:42:59.160 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:42:59.160 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 23:42:59.160 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 23:42:59.160 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:42:59.160 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 23:42:59.161 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 23:42:59.161 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 23:42:59.161 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 23:42:59.161 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 23:42:59.161 | INFO | ------------------------------------------ | 20260505 23:42:59.161 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:42:59.161 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 23:42:59.161 | INFO | Total mem usage: 25.6 GiB | 20260505 23:42:59.161 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 23:42:59.161 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 23:42:59.161 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 23:42:59.161 | INFO | High Water Breakdown: | 20260505 23:42:59.161 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 23:42:59.161 | INFO | ------------------------------------------ | 20260505 23:42:59.161 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:42:59.161 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 23:42:59.161 | INFO | Total mem usage: 15.7 GiB | 20260505 23:42:59.161 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 23:42:59.161 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 23:42:59.161 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 23:42:59.161 | INFO | High Water Breakdown: | 20260505 23:42:59.161 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 23:42:59.161 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 23:42:59.161 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 23:42:59.161 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 23:42:59.161 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 23:42:59.161 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 23:42:59.161 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 23:42:59.161 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 23:42:59.161 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 23:42:59.161 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:42:59.161 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 23:42:59.161 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:42:59.161 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 23:42:59.161 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 23:42:59.161 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 23:42:59.161 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 23:42:59.161 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:42:59.161 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 23:42:59.161 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:42:59.161 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:42:59.161 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:42:59.161 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 23:42:59.161 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 23:42:59.161 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:42:59.161 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 23:42:59.161 | INFO | ------------------------------------------ | 20260505 23:42:59.161 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:42:59.161 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 23:42:59.161 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:42:59.161 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:42:59.161 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.39e+01, 1.34e+01 / 6.64e-01 | 20260505 23:42:59.161 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.03e-01, 1.05e-02 / 4.09e-03 | 20260505 23:42:59.161 | INFO | PerfReport: 11.82%, Baseline -- 3.70e+00 / 4.51e+00, 5.64e-01 / 1.80e-01 | 20260505 23:42:59.161 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.52e+00, 1.88e-01 / 9.76e-02 | 20260505 23:42:59.161 | INFO | PerfReport: 15.98%, DME -- 4.01e+01 / 4.39e+01, 3.80e+00 / 8.66e-01 | 20260505 23:42:59.161 | INFO | PerfReport: 57.34%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.49e+00 / 1.15e+00 | 20260505 23:42:59.161 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.23e+00 / 2.72e+00, 4.25e-01 / 2.49e-01 | 20260505 23:42:59.161 | INFO | PerfReport: 3.10%, Metrics -- 9.72e-01 / 1.24e+00, 2.61e-01 / 1.27e-01 | 20260505 23:42:59.161 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.37e-01 / 3.92e-01, 5.73e-02 / 3.17e-02 | 20260505 23:42:59.161 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 23:42:59.161 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:42:59.161 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:42:59.161 | INFO | PerfReport: Full Execution -- 9.85e+01 / -nan, 2.49e+01 / 0.00e+00 | 20260505 23:42:59.161 | INFO | PerfReport: 22.40%, Upload -- 2.21e+01 / 0.00e+00, 7.61e+00 / 0.00e+00 | 20260505 23:42:59.161 | INFO | PerfReport: 21.84%, Download -- 2.15e+01 / 0.00e+00, 7.63e+00 / 0.00e+00 | 20260505 23:42:59.161 | INFO | PerfReport: 31.80%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 23:42:59.161 | INFO | PerfReport: 2.17%, LockUpload -- 2.14e+00 / 0.00e+00, 8.00e+00 / 0.00e+00 | 20260505 23:42:59.161 | INFO | PerfReport: 1.32%, LockDownload -- 1.30e+00 / 0.00e+00, 2.86e+00 / 0.00e+00 | 20260505 23:42:59.161 | INFO | PerfReport: 20.45%, LockCompute -- 2.01e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260505 23:42:59.161 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.91e-02 / 0.00e+00 | 20260505 23:42:59.281 | INFO | Starting a round of Baz IO | 20260505 23:43:02.429 | INFO | Populating RTMetrics report | 20260505 23:43:02.429 | INFO | RTMetrics report populated | 20260505 23:43:02.429 | INFO | Opening tmp RTMetrics json file | 20260505 23:43:02.432 | INFO | Serializing RTMetrics to json | 20260505 23:43:02.435 | INFO | Serialization done, swapping tmp file now... | 20260505 23:43:02.436 | INFO | RTMetrics done | 20260505 23:43:02.514 | INFO | Finished a round of Baz IO: Wrote 1.888 GiB in 3.232 seconds | 20260505 23:43:02.514 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.41% / 0.00% / 78.59% | 20260505 23:43:02.514 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.103009% / 3.97364e-06% / 7.94729e-05% | 20260505 23:43:02.514 | INFO | Baz IO overall: | 20260505 23:43:02.514 | INFO | BytesWritten : 4134453589050 | 20260505 23:43:02.514 | INFO | NumEvents : 1947274382885 | 20260505 23:43:04.305 | INFO | Analyzing chunk frames = [7241728,7242240) | 20260505 23:43:04.305 | INFO | MemInfo: { | 20260505 23:43:04.305 | INFO | "gpu" : " 17 GB", | 20260505 23:43:04.305 | INFO | "huge" : "", | 20260505 23:43:04.305 | INFO | "huge_shared" : " 27 GB", | 20260505 23:43:04.306 | INFO | "malloc" : " 14 GB", | 20260505 23:43:04.306 | INFO | "pinned" : " 29 GB" | 20260505 23:43:04.306 | INFO | } | 20260505 23:43:04.306 | INFO | Perf report for the last 5.14579 seconds: | 20260505 23:43:04.306 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:43:04.306 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:04.306 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:04.306 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:04.306 | INFO | ANALYSIS : 62.5%, 3.0, 97.7ms, 0 | 20260505 23:43:04.306 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:04.306 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260505 23:43:04.306 | INFO | PRE_HQ : 59.8%, 1.7, 54.3ms, 0 | 20260505 23:43:04.306 | INFO | BAZWRITER : 62.0%, 3.6, 114.9ms, 0 | 20260505 23:43:04.306 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 23:43:04.306 | INFO | Uploaded 36GiB to GPU in 2.16202s (16.6511GiB/s) | 20260505 23:43:04.306 | INFO | Downloaded 38GiB to GPU in 2.23619s (16.9932GiB/s) | 20260505 23:43:09.444 | INFO | Analyzing chunk frames = [7242240,7242752) | 20260505 23:43:09.444 | INFO | MemInfo: { | 20260505 23:43:09.444 | INFO | "gpu" : " 17 GB", | 20260505 23:43:09.444 | INFO | "huge" : "", | 20260505 23:43:09.444 | INFO | "huge_shared" : " 27 GB", | 20260505 23:43:09.444 | INFO | "malloc" : " 14 GB", | 20260505 23:43:09.444 | INFO | "pinned" : " 29 GB" | 20260505 23:43:09.444 | INFO | } | 20260505 23:43:09.444 | INFO | Perf report for the last 5.13822 seconds: | 20260505 23:43:09.444 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:43:09.444 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:09.444 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:09.444 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:09.444 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260505 23:43:09.444 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:09.444 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:09.444 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:43:09.444 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:09.444 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:09.444 | INFO | Uploaded 36GiB to GPU in 2.09238s (17.2052GiB/s) | 20260505 23:43:09.444 | INFO | Downloaded 34GiB to GPU in 2.03583s (16.7008GiB/s) | 20260505 23:43:14.580 | INFO | Analyzing chunk frames = [7242752,7243264) | 20260505 23:43:14.580 | INFO | MemInfo: { | 20260505 23:43:14.580 | INFO | "gpu" : " 17 GB", | 20260505 23:43:14.580 | INFO | "huge" : "", | 20260505 23:43:14.580 | INFO | "huge_shared" : " 27 GB", | 20260505 23:43:14.580 | INFO | "malloc" : " 14 GB", | 20260505 23:43:14.580 | INFO | "pinned" : " 29 GB" | 20260505 23:43:14.580 | INFO | } | 20260505 23:43:14.580 | INFO | Perf report for the last 5.13603 seconds: | 20260505 23:43:14.580 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:43:14.580 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:14.580 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:14.580 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:14.580 | INFO | ANALYSIS : 61.6%, 3.0, 95.7ms, 0 | 20260505 23:43:14.580 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:14.580 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:14.580 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:43:14.580 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:14.580 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:14.580 | INFO | Uploaded 36GiB to GPU in 2.08243s (17.2875GiB/s) | 20260505 23:43:14.580 | INFO | Downloaded 34GiB to GPU in 2.02349s (16.8026GiB/s) | 20260505 23:43:19.717 | INFO | Analyzing chunk frames = [7243264,7243776) | 20260505 23:43:19.717 | INFO | MemInfo: { | 20260505 23:43:19.717 | INFO | "gpu" : " 17 GB", | 20260505 23:43:19.717 | INFO | "huge" : "", | 20260505 23:43:19.717 | INFO | "huge_shared" : " 27 GB", | 20260505 23:43:19.717 | INFO | "malloc" : " 14 GB", | 20260505 23:43:19.717 | INFO | "pinned" : " 29 GB" | 20260505 23:43:19.717 | INFO | } | 20260505 23:43:19.717 | INFO | Perf report for the last 5.13687 seconds: | 20260505 23:43:19.717 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:43:19.717 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:19.717 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:19.717 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:19.717 | INFO | ANALYSIS : 61.8%, 3.0, 95.9ms, 0 | 20260505 23:43:19.717 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:19.717 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:19.717 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:43:19.717 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:19.717 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:19.717 | INFO | Uploaded 36GiB to GPU in 2.12528s (16.939GiB/s) | 20260505 23:43:19.717 | INFO | Downloaded 34GiB to GPU in 2.02669s (16.7761GiB/s) | 20260505 23:43:24.861 | INFO | Analyzing chunk frames = [7243776,7244288) | 20260505 23:43:24.861 | INFO | MemInfo: { | 20260505 23:43:24.861 | INFO | "gpu" : " 17 GB", | 20260505 23:43:24.861 | INFO | "huge" : "", | 20260505 23:43:24.862 | INFO | "huge_shared" : " 27 GB", | 20260505 23:43:24.862 | INFO | "malloc" : " 14 GB", | 20260505 23:43:24.862 | INFO | "pinned" : " 29 GB" | 20260505 23:43:24.862 | INFO | } | 20260505 23:43:24.862 | INFO | Perf report for the last 5.14493 seconds: | 20260505 23:43:24.862 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:43:24.862 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:24.862 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:24.862 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:24.862 | INFO | ANALYSIS : 61.2%, 3.0, 95.6ms, 0 | 20260505 23:43:24.862 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:24.862 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:24.862 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 23:43:24.862 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:24.862 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:24.862 | INFO | Uploaded 36GiB to GPU in 2.10906s (17.0692GiB/s) | 20260505 23:43:24.862 | INFO | Downloaded 34GiB to GPU in 2.05226s (16.5671GiB/s) | 20260505 23:43:29.998 | INFO | Analyzing chunk frames = [7244288,7244800) | 20260505 23:43:29.998 | INFO | MemInfo: { | 20260505 23:43:29.998 | INFO | "gpu" : " 17 GB", | 20260505 23:43:29.998 | INFO | "huge" : "", | 20260505 23:43:29.998 | INFO | "huge_shared" : " 27 GB", | 20260505 23:43:29.998 | INFO | "malloc" : " 14 GB", | 20260505 23:43:29.998 | INFO | "pinned" : " 29 GB" | 20260505 23:43:29.998 | INFO | } | 20260505 23:43:29.999 | INFO | Perf report for the last 5.13681 seconds: | 20260505 23:43:29.999 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:43:29.999 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:29.999 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:29.999 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:29.999 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260505 23:43:29.999 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:29.999 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:29.999 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260505 23:43:29.999 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:29.999 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:29.999 | INFO | Uploaded 36GiB to GPU in 2.05714s (17.5GiB/s) | 20260505 23:43:29.999 | INFO | Downloaded 34GiB to GPU in 2.03295s (16.7245GiB/s) | 20260505 23:43:35.135 | INFO | Analyzing chunk frames = [7244800,7245312) | 20260505 23:43:35.135 | INFO | MemInfo: { | 20260505 23:43:35.135 | INFO | "gpu" : " 17 GB", | 20260505 23:43:35.135 | INFO | "huge" : "", | 20260505 23:43:35.135 | INFO | "huge_shared" : " 27 GB", | 20260505 23:43:35.135 | INFO | "malloc" : " 14 GB", | 20260505 23:43:35.135 | INFO | "pinned" : " 29 GB" | 20260505 23:43:35.135 | INFO | } | 20260505 23:43:35.135 | INFO | Perf report for the last 5.13617 seconds: | 20260505 23:43:35.135 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:43:35.135 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:35.135 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:35.135 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:35.135 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 23:43:35.135 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:35.135 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:35.135 | INFO | PRE_HQ : 27.5%, 1.0, 14.2ms, 0 | 20260505 23:43:35.135 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:35.135 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:35.135 | INFO | Uploaded 36GiB to GPU in 2.01726s (17.846GiB/s) | 20260505 23:43:35.135 | INFO | Downloaded 34GiB to GPU in 2.04131s (16.6559GiB/s) | 20260505 23:43:40.274 | INFO | Analyzing chunk frames = [7245312,7245824) | 20260505 23:43:40.274 | INFO | MemInfo: { | 20260505 23:43:40.274 | INFO | "gpu" : " 17 GB", | 20260505 23:43:40.274 | INFO | "huge" : "", | 20260505 23:43:40.274 | INFO | "huge_shared" : " 27 GB", | 20260505 23:43:40.274 | INFO | "malloc" : " 14 GB", | 20260505 23:43:40.274 | INFO | "pinned" : " 29 GB" | 20260505 23:43:40.274 | INFO | } | 20260505 23:43:40.274 | INFO | Perf report for the last 5.13951 seconds: | 20260505 23:43:40.274 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:43:40.274 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:40.274 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:40.274 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:40.274 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 23:43:40.274 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:40.274 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:40.274 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 23:43:40.274 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:40.274 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:40.275 | INFO | Uploaded 36GiB to GPU in 2.12356s (16.9526GiB/s) | 20260505 23:43:40.275 | INFO | Downloaded 34GiB to GPU in 2.01509s (16.8727GiB/s) | 20260505 23:43:40.404 | INFO | Starting a round of Baz IO | 20260505 23:43:43.545 | INFO | Populating RTMetrics report | 20260505 23:43:43.546 | INFO | RTMetrics report populated | 20260505 23:43:43.546 | INFO | Opening tmp RTMetrics json file | 20260505 23:43:43.546 | INFO | Serializing RTMetrics to json | 20260505 23:43:43.548 | INFO | Serialization done, swapping tmp file now... | 20260505 23:43:43.549 | INFO | RTMetrics done | 20260505 23:43:43.627 | INFO | Finished a round of Baz IO: Wrote 1.885 GiB in 3.223 seconds | 20260505 23:43:43.627 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.41% / 0.00% / 78.59% | 20260505 23:43:43.627 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.10248% / 0% / 6.75519e-05% | 20260505 23:43:43.627 | INFO | Baz IO overall: | 20260505 23:43:43.627 | INFO | BytesWritten : 4136477691591 | 20260505 23:43:43.627 | INFO | NumEvents : 1948201061731 | 20260505 23:43:45.415 | INFO | Analyzing chunk frames = [7245824,7246336) | 20260505 23:43:45.415 | INFO | MemInfo: { | 20260505 23:43:45.415 | INFO | "gpu" : " 17 GB", | 20260505 23:43:45.415 | INFO | "huge" : "", | 20260505 23:43:45.416 | INFO | "huge_shared" : " 27 GB", | 20260505 23:43:45.416 | INFO | "malloc" : " 14 GB", | 20260505 23:43:45.416 | INFO | "pinned" : " 29 GB" | 20260505 23:43:45.416 | INFO | } | 20260505 23:43:45.416 | INFO | Perf report for the last 5.14138 seconds: | 20260505 23:43:45.416 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:43:45.416 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:45.416 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:45.416 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:45.416 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 23:43:45.416 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:45.416 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260505 23:43:45.416 | INFO | PRE_HQ : 59.6%, 1.8, 54.2ms, 0 | 20260505 23:43:45.416 | INFO | BAZWRITER : 62.0%, 3.6, 114.3ms, 0 | 20260505 23:43:45.416 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 23:43:45.416 | INFO | Uploaded 36GiB to GPU in 2.1218s (16.9668GiB/s) | 20260505 23:43:45.416 | INFO | Downloaded 38GiB to GPU in 2.2349s (17.003GiB/s) | 20260505 23:43:50.561 | INFO | Analyzing chunk frames = [7246336,7246848) | 20260505 23:43:50.561 | INFO | MemInfo: { | 20260505 23:43:50.561 | INFO | "gpu" : " 17 GB", | 20260505 23:43:50.561 | INFO | "huge" : "", | 20260505 23:43:50.561 | INFO | "huge_shared" : " 27 GB", | 20260505 23:43:50.561 | INFO | "malloc" : " 14 GB", | 20260505 23:43:50.561 | INFO | "pinned" : " 29 GB" | 20260505 23:43:50.561 | INFO | } | 20260505 23:43:50.561 | INFO | Perf report for the last 5.14526 seconds: | 20260505 23:43:50.561 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:43:50.561 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:50.561 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:50.561 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:50.561 | INFO | ANALYSIS : 61.8%, 3.0, 96.6ms, 0 | 20260505 23:43:50.561 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:50.561 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:50.561 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260505 23:43:50.561 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:50.561 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:50.561 | INFO | Uploaded 36GiB to GPU in 2.09689s (17.1683GiB/s) | 20260505 23:43:50.561 | INFO | Downloaded 34GiB to GPU in 2.033s (16.724GiB/s) | 20260505 23:43:55.698 | INFO | Analyzing chunk frames = [7246848,7247360) | 20260505 23:43:55.698 | INFO | MemInfo: { | 20260505 23:43:55.698 | INFO | "gpu" : " 17 GB", | 20260505 23:43:55.698 | INFO | "huge" : "", | 20260505 23:43:55.698 | INFO | "huge_shared" : " 27 GB", | 20260505 23:43:55.698 | INFO | "malloc" : " 14 GB", | 20260505 23:43:55.698 | INFO | "pinned" : " 29 GB" | 20260505 23:43:55.698 | INFO | } | 20260505 23:43:55.698 | INFO | Perf report for the last 5.13695 seconds: | 20260505 23:43:55.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:43:55.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:55.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:55.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:55.698 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260505 23:43:55.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:55.698 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:55.698 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260505 23:43:55.698 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:55.698 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:43:55.698 | INFO | Uploaded 36GiB to GPU in 2.08403s (17.2742GiB/s) | 20260505 23:43:55.698 | INFO | Downloaded 34GiB to GPU in 2.01739s (16.8535GiB/s) | 20260505 23:44:00.832 | INFO | Analyzing chunk frames = [7247360,7247872) | 20260505 23:44:00.832 | INFO | MemInfo: { | 20260505 23:44:00.832 | INFO | "gpu" : " 17 GB", | 20260505 23:44:00.832 | INFO | "huge" : "", | 20260505 23:44:00.832 | INFO | "huge_shared" : " 27 GB", | 20260505 23:44:00.832 | INFO | "malloc" : " 14 GB", | 20260505 23:44:00.832 | INFO | "pinned" : " 29 GB" | 20260505 23:44:00.832 | INFO | } | 20260505 23:44:00.832 | INFO | Perf report for the last 5.13406 seconds: | 20260505 23:44:00.832 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:44:00.832 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:00.832 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:00.832 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:00.832 | INFO | ANALYSIS : 61.8%, 3.0, 95.8ms, 0 | 20260505 23:44:00.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:00.832 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:00.832 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:44:00.832 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:00.832 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:00.832 | INFO | Uploaded 36GiB to GPU in 2.14651s (16.7714GiB/s) | 20260505 23:44:00.832 | INFO | Downloaded 34GiB to GPU in 2.02563s (16.7849GiB/s) | 20260505 23:44:05.969 | INFO | Analyzing chunk frames = [7247872,7248384) | 20260505 23:44:05.969 | INFO | MemInfo: { | 20260505 23:44:05.969 | INFO | "gpu" : " 17 GB", | 20260505 23:44:05.969 | INFO | "huge" : "", | 20260505 23:44:05.969 | INFO | "huge_shared" : " 27 GB", | 20260505 23:44:05.969 | INFO | "malloc" : " 14 GB", | 20260505 23:44:05.969 | INFO | "pinned" : " 29 GB" | 20260505 23:44:05.969 | INFO | } | 20260505 23:44:05.969 | INFO | Perf report for the last 5.13677 seconds: | 20260505 23:44:05.969 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:44:05.969 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:05.969 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:05.969 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:05.969 | INFO | ANALYSIS : 61.5%, 3.0, 96.0ms, 0 | 20260505 23:44:05.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:05.969 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:05.969 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 23:44:05.969 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:05.969 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:05.969 | INFO | Uploaded 36GiB to GPU in 2.14993s (16.7447GiB/s) | 20260505 23:44:05.969 | INFO | Downloaded 34GiB to GPU in 2.06416s (16.4716GiB/s) | 20260505 23:44:11.114 | INFO | Analyzing chunk frames = [7248384,7248896) | 20260505 23:44:11.114 | INFO | MemInfo: { | 20260505 23:44:11.114 | INFO | "gpu" : " 17 GB", | 20260505 23:44:11.114 | INFO | "huge" : "", | 20260505 23:44:11.114 | INFO | "huge_shared" : " 27 GB", | 20260505 23:44:11.114 | INFO | "malloc" : " 14 GB", | 20260505 23:44:11.114 | INFO | "pinned" : " 29 GB" | 20260505 23:44:11.114 | INFO | } | 20260505 23:44:11.114 | INFO | Perf report for the last 5.14529 seconds: | 20260505 23:44:11.114 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:44:11.114 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:11.114 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:11.114 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:11.114 | INFO | ANALYSIS : 61.3%, 3.0, 95.8ms, 0 | 20260505 23:44:11.114 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:11.114 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:11.114 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260505 23:44:11.114 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:11.114 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:11.114 | INFO | Uploaded 36GiB to GPU in 2.07504s (17.3491GiB/s) | 20260505 23:44:11.114 | INFO | Downloaded 34GiB to GPU in 2.03311s (16.7232GiB/s) | 20260505 23:44:16.260 | INFO | Analyzing chunk frames = [7248896,7249408) | 20260505 23:44:16.260 | INFO | MemInfo: { | 20260505 23:44:16.260 | INFO | "gpu" : " 17 GB", | 20260505 23:44:16.260 | INFO | "huge" : "", | 20260505 23:44:16.260 | INFO | "huge_shared" : " 27 GB", | 20260505 23:44:16.261 | INFO | "malloc" : " 14 GB", | 20260505 23:44:16.261 | INFO | "pinned" : " 29 GB" | 20260505 23:44:16.261 | INFO | } | 20260505 23:44:16.261 | INFO | Perf report for the last 5.14655 seconds: | 20260505 23:44:16.261 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:44:16.261 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:16.261 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:16.261 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:16.261 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 23:44:16.261 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:16.261 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:16.261 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260505 23:44:16.261 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:16.261 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:16.261 | INFO | Uploaded 36GiB to GPU in 2.09753s (17.1631GiB/s) | 20260505 23:44:16.261 | INFO | Downloaded 34GiB to GPU in 2.07102s (16.417GiB/s) | 20260505 23:44:21.391 | INFO | Analyzing chunk frames = [7249408,7249920) | 20260505 23:44:21.391 | INFO | MemInfo: { | 20260505 23:44:21.391 | INFO | "gpu" : " 17 GB", | 20260505 23:44:21.391 | INFO | "huge" : "", | 20260505 23:44:21.391 | INFO | "huge_shared" : " 27 GB", | 20260505 23:44:21.391 | INFO | "malloc" : " 14 GB", | 20260505 23:44:21.392 | INFO | "pinned" : " 29 GB" | 20260505 23:44:21.392 | INFO | } | 20260505 23:44:21.392 | INFO | Perf report for the last 5.13097 seconds: | 20260505 23:44:21.392 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:44:21.392 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:21.392 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:21.392 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:21.392 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260505 23:44:21.392 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:21.392 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:21.392 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:44:21.392 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:21.392 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:21.392 | INFO | Uploaded 36GiB to GPU in 2.15127s (16.7343GiB/s) | 20260505 23:44:21.392 | INFO | Downloaded 34GiB to GPU in 2.03735s (16.6884GiB/s) | 20260505 23:44:21.392 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 23:44:21.392 | INFO | Total mem usage: 15.3 GiB | 20260505 23:44:21.392 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 23:44:21.392 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -9.09%) | 20260505 23:44:21.392 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 23:44:21.392 | INFO | High Water Breakdown: | 20260505 23:44:21.392 | INFO | 60.9% : 8.55 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 23:44:21.392 | INFO | 39% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 23:44:21.392 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 23:44:21.392 | INFO | ------------------------------------------ | 20260505 23:44:21.392 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:44:21.392 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 23:44:21.392 | INFO | Total mem usage: 26.5 GiB | 20260505 23:44:21.392 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 23:44:21.392 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 23:44:21.392 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 23:44:21.392 | INFO | High Water Breakdown: | 20260505 23:44:21.392 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:44:21.392 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:44:21.392 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:44:21.392 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:44:21.392 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:44:21.392 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:44:21.392 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 23:44:21.392 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 23:44:21.392 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:44:21.392 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 23:44:21.392 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 23:44:21.392 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 23:44:21.392 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 23:44:21.392 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 23:44:21.392 | INFO | ------------------------------------------ | 20260505 23:44:21.392 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:44:21.392 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 23:44:21.392 | INFO | Total mem usage: 25.6 GiB | 20260505 23:44:21.392 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 23:44:21.392 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 23:44:21.392 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 23:44:21.392 | INFO | High Water Breakdown: | 20260505 23:44:21.392 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 23:44:21.392 | INFO | ------------------------------------------ | 20260505 23:44:21.392 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:44:21.392 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 23:44:21.392 | INFO | Total mem usage: 15.7 GiB | 20260505 23:44:21.392 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 23:44:21.392 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 23:44:21.392 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 23:44:21.392 | INFO | High Water Breakdown: | 20260505 23:44:21.392 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 23:44:21.392 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 23:44:21.392 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 23:44:21.392 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 23:44:21.392 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 23:44:21.392 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 23:44:21.392 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 23:44:21.392 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 23:44:21.392 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 23:44:21.392 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:44:21.392 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 23:44:21.393 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:44:21.393 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 23:44:21.393 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 23:44:21.393 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 23:44:21.393 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 23:44:21.393 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:44:21.393 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 23:44:21.393 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:44:21.393 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:44:21.393 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:44:21.393 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 23:44:21.393 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 23:44:21.393 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:44:21.393 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 23:44:21.393 | INFO | ------------------------------------------ | 20260505 23:44:21.393 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:44:21.393 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 23:44:21.393 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:44:21.393 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:44:21.393 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.41e+01, 1.34e+01 / 9.18e-01 | 20260505 23:44:21.393 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.05e-01, 1.01e-02 / 5.11e-03 | 20260505 23:44:21.393 | INFO | PerfReport: 11.83%, Baseline -- 3.71e+00 / 4.48e+00, 5.69e-01 / 2.36e-01 | 20260505 23:44:21.393 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.53e+00, 1.91e-01 / 1.02e-01 | 20260505 23:44:21.393 | INFO | PerfReport: 15.98%, DME -- 4.01e+01 / 4.41e+01, 3.74e+00 / 1.10e+00 | 20260505 23:44:21.393 | INFO | PerfReport: 57.33%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.49e+00 / 1.15e+00 | 20260505 23:44:21.393 | INFO | PerfReport: 7.11%, PulseAccumulating -- 2.23e+00 / 2.77e+00, 4.22e-01 / 2.29e-01 | 20260505 23:44:21.393 | INFO | PerfReport: 3.10%, Metrics -- 9.74e-01 / 1.29e+00, 2.63e-01 / 8.60e-02 | 20260505 23:44:21.393 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.37e-01 / 3.97e-01, 5.77e-02 / 2.96e-02 | 20260505 23:44:21.393 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 23:44:21.393 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:44:21.393 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:44:21.393 | INFO | PerfReport: Full Execution -- 9.84e+01 / -nan, 2.44e+01 / 0.00e+00 | 20260505 23:44:21.393 | INFO | PerfReport: 22.29%, Upload -- 2.19e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260505 23:44:21.393 | INFO | PerfReport: 21.82%, Download -- 2.15e+01 / 0.00e+00, 7.55e+00 / 0.00e+00 | 20260505 23:44:21.393 | INFO | PerfReport: 31.87%, Compute -- 3.14e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 23:44:21.393 | INFO | PerfReport: 2.13%, LockUpload -- 2.10e+00 / 0.00e+00, 7.86e+00 / 0.00e+00 | 20260505 23:44:21.393 | INFO | PerfReport: 1.30%, LockDownload -- 1.28e+00 / 0.00e+00, 2.92e+00 / 0.00e+00 | 20260505 23:44:21.393 | INFO | PerfReport: 20.58%, LockCompute -- 2.03e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260505 23:44:21.393 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.81e-02 / 0.00e+00 | 20260505 23:44:21.516 | INFO | Starting a round of Baz IO | 20260505 23:44:24.662 | INFO | Populating RTMetrics report | 20260505 23:44:24.662 | INFO | RTMetrics report populated | 20260505 23:44:24.662 | INFO | Opening tmp RTMetrics json file | 20260505 23:44:24.665 | INFO | Serializing RTMetrics to json | 20260505 23:44:24.667 | INFO | Serialization done, swapping tmp file now... | 20260505 23:44:24.668 | INFO | RTMetrics done | 20260505 23:44:24.741 | INFO | Finished a round of Baz IO: Wrote 1.883 GiB in 3.225 seconds | 20260505 23:44:24.741 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.41% / 0.00% / 78.59% | 20260505 23:44:24.741 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.101872% / 0% / 6.35783e-05% | 20260505 23:44:24.741 | INFO | Baz IO overall: | 20260505 23:44:24.741 | INFO | BytesWritten : 4138500053315 | 20260505 23:44:24.741 | INFO | NumEvents : 1949127188407 | 20260505 23:44:26.527 | INFO | Analyzing chunk frames = [7249920,7250432) | 20260505 23:44:26.527 | INFO | MemInfo: { | 20260505 23:44:26.527 | INFO | "gpu" : " 17 GB", | 20260505 23:44:26.527 | INFO | "huge" : "", | 20260505 23:44:26.527 | INFO | "huge_shared" : " 27 GB", | 20260505 23:44:26.527 | INFO | "malloc" : " 14 GB", | 20260505 23:44:26.527 | INFO | "pinned" : " 29 GB" | 20260505 23:44:26.527 | INFO | } | 20260505 23:44:26.527 | INFO | Perf report for the last 5.13587 seconds: | 20260505 23:44:26.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:44:26.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:26.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:26.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:26.527 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 23:44:26.528 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:26.528 | INFO | RT_METRICS : 43.6%, 1.0, 22.8ms, 0 | 20260505 23:44:26.528 | INFO | PRE_HQ : 60.0%, 1.8, 54.5ms, 0 | 20260505 23:44:26.528 | INFO | BAZWRITER : 62.0%, 3.6, 114.8ms, 0 | 20260505 23:44:26.528 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 23:44:26.528 | INFO | Uploaded 36GiB to GPU in 2.16416s (16.6346GiB/s) | 20260505 23:44:26.528 | INFO | Downloaded 38GiB to GPU in 2.227s (17.0633GiB/s) | 20260505 23:44:31.671 | INFO | Analyzing chunk frames = [7250432,7250944) | 20260505 23:44:31.671 | INFO | MemInfo: { | 20260505 23:44:31.671 | INFO | "gpu" : " 17 GB", | 20260505 23:44:31.671 | INFO | "huge" : "", | 20260505 23:44:31.671 | INFO | "huge_shared" : " 27 GB", | 20260505 23:44:31.671 | INFO | "malloc" : " 14 GB", | 20260505 23:44:31.671 | INFO | "pinned" : " 29 GB" | 20260505 23:44:31.671 | INFO | } | 20260505 23:44:31.671 | INFO | Perf report for the last 5.14358 seconds: | 20260505 23:44:31.671 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:44:31.671 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:31.671 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:31.671 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:31.671 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 23:44:31.671 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:31.671 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:31.671 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:44:31.671 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:31.671 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:31.671 | INFO | Uploaded 36GiB to GPU in 2.08981s (17.2265GiB/s) | 20260505 23:44:31.671 | INFO | Downloaded 34GiB to GPU in 2.03063s (16.7436GiB/s) | 20260505 23:44:36.810 | INFO | Analyzing chunk frames = [7250944,7251456) | 20260505 23:44:36.810 | INFO | MemInfo: { | 20260505 23:44:36.810 | INFO | "gpu" : " 17 GB", | 20260505 23:44:36.810 | INFO | "huge" : "", | 20260505 23:44:36.810 | INFO | "huge_shared" : " 27 GB", | 20260505 23:44:36.810 | INFO | "malloc" : " 14 GB", | 20260505 23:44:36.810 | INFO | "pinned" : " 29 GB" | 20260505 23:44:36.810 | INFO | } | 20260505 23:44:36.810 | INFO | Perf report for the last 5.13894 seconds: | 20260505 23:44:36.810 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:44:36.810 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:36.810 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:36.810 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:36.810 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260505 23:44:36.810 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:36.810 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:36.810 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:44:36.810 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:36.810 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:36.810 | INFO | Uploaded 36GiB to GPU in 2.08682s (17.2512GiB/s) | 20260505 23:44:36.810 | INFO | Downloaded 34GiB to GPU in 2.02444s (16.7947GiB/s) | 20260505 23:44:41.947 | INFO | Analyzing chunk frames = [7251456,7251968) | 20260505 23:44:41.947 | INFO | MemInfo: { | 20260505 23:44:41.947 | INFO | "gpu" : " 17 GB", | 20260505 23:44:41.947 | INFO | "huge" : "", | 20260505 23:44:41.947 | INFO | "huge_shared" : " 27 GB", | 20260505 23:44:41.947 | INFO | "malloc" : " 14 GB", | 20260505 23:44:41.947 | INFO | "pinned" : " 29 GB" | 20260505 23:44:41.947 | INFO | } | 20260505 23:44:41.947 | INFO | Perf report for the last 5.13734 seconds: | 20260505 23:44:41.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:44:41.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:41.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:41.947 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:41.947 | INFO | ANALYSIS : 61.8%, 3.0, 95.9ms, 0 | 20260505 23:44:41.947 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:41.947 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:41.948 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:44:41.948 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:41.948 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:41.948 | INFO | Uploaded 36GiB to GPU in 2.14637s (16.7725GiB/s) | 20260505 23:44:41.948 | INFO | Downloaded 34GiB to GPU in 2.02285s (16.8079GiB/s) | 20260505 23:44:47.087 | INFO | Analyzing chunk frames = [7251968,7252480) | 20260505 23:44:47.088 | INFO | MemInfo: { | 20260505 23:44:47.088 | INFO | "gpu" : " 17 GB", | 20260505 23:44:47.088 | INFO | "huge" : "", | 20260505 23:44:47.088 | INFO | "huge_shared" : " 27 GB", | 20260505 23:44:47.088 | INFO | "malloc" : " 14 GB", | 20260505 23:44:47.088 | INFO | "pinned" : " 29 GB" | 20260505 23:44:47.088 | INFO | } | 20260505 23:44:47.088 | INFO | Perf report for the last 5.14031 seconds: | 20260505 23:44:47.088 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:44:47.088 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:47.088 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:47.088 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:47.088 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260505 23:44:47.088 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:47.088 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:47.088 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 23:44:47.088 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:47.088 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:47.088 | INFO | Uploaded 36GiB to GPU in 2.10157s (17.1301GiB/s) | 20260505 23:44:47.088 | INFO | Downloaded 34GiB to GPU in 2.04961s (16.5885GiB/s) | 20260505 23:44:52.224 | INFO | Analyzing chunk frames = [7252480,7252992) | 20260505 23:44:52.224 | INFO | MemInfo: { | 20260505 23:44:52.224 | INFO | "gpu" : " 17 GB", | 20260505 23:44:52.224 | INFO | "huge" : "", | 20260505 23:44:52.224 | INFO | "huge_shared" : " 27 GB", | 20260505 23:44:52.224 | INFO | "malloc" : " 14 GB", | 20260505 23:44:52.224 | INFO | "pinned" : " 29 GB" | 20260505 23:44:52.224 | INFO | } | 20260505 23:44:52.224 | INFO | Perf report for the last 5.13672 seconds: | 20260505 23:44:52.224 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:44:52.224 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:52.224 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:52.224 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:52.224 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260505 23:44:52.224 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:52.224 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:52.224 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:44:52.224 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:52.224 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:52.225 | INFO | Uploaded 36GiB to GPU in 2.07792s (17.325GiB/s) | 20260505 23:44:52.225 | INFO | Downloaded 34GiB to GPU in 2.03729s (16.6888GiB/s) | 20260505 23:44:57.370 | INFO | Analyzing chunk frames = [7252992,7253504) | 20260505 23:44:57.370 | INFO | MemInfo: { | 20260505 23:44:57.370 | INFO | "gpu" : " 17 GB", | 20260505 23:44:57.370 | INFO | "huge" : "", | 20260505 23:44:57.370 | INFO | "huge_shared" : " 27 GB", | 20260505 23:44:57.370 | INFO | "malloc" : " 14 GB", | 20260505 23:44:57.370 | INFO | "pinned" : " 29 GB" | 20260505 23:44:57.370 | INFO | } | 20260505 23:44:57.370 | INFO | Perf report for the last 5.14603 seconds: | 20260505 23:44:57.370 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:44:57.370 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:57.370 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:57.370 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:57.370 | INFO | ANALYSIS : 62.2%, 3.0, 97.3ms, 0 | 20260505 23:44:57.370 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:57.371 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:57.371 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 23:44:57.371 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:57.371 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:44:57.371 | INFO | Uploaded 36GiB to GPU in 2.04404s (17.6122GiB/s) | 20260505 23:44:57.371 | INFO | Downloaded 34GiB to GPU in 2.04521s (16.6242GiB/s) | 20260505 23:45:02.508 | INFO | Analyzing chunk frames = [7253504,7254016) | 20260505 23:45:02.508 | INFO | MemInfo: { | 20260505 23:45:02.508 | INFO | "gpu" : " 17 GB", | 20260505 23:45:02.508 | INFO | "huge" : "", | 20260505 23:45:02.508 | INFO | "huge_shared" : " 27 GB", | 20260505 23:45:02.508 | INFO | "malloc" : " 14 GB", | 20260505 23:45:02.508 | INFO | "pinned" : " 29 GB" | 20260505 23:45:02.508 | INFO | } | 20260505 23:45:02.508 | INFO | Perf report for the last 5.13794 seconds: | 20260505 23:45:02.508 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:45:02.508 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:02.508 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:02.508 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:02.508 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260505 23:45:02.508 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:02.508 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:02.508 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 23:45:02.508 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:02.508 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:02.509 | INFO | Uploaded 36GiB to GPU in 2.13357s (16.8732GiB/s) | 20260505 23:45:02.509 | INFO | Downloaded 34GiB to GPU in 2.02259s (16.8101GiB/s) | 20260505 23:45:02.626 | INFO | Starting a round of Baz IO | 20260505 23:45:05.773 | INFO | Populating RTMetrics report | 20260505 23:45:05.774 | INFO | RTMetrics report populated | 20260505 23:45:05.774 | INFO | Opening tmp RTMetrics json file | 20260505 23:45:05.775 | INFO | Serializing RTMetrics to json | 20260505 23:45:05.777 | INFO | Serialization done, swapping tmp file now... | 20260505 23:45:05.777 | INFO | RTMetrics done | 20260505 23:45:05.854 | INFO | Finished a round of Baz IO: Wrote 1.883 GiB in 3.227 seconds | 20260505 23:45:05.854 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.41% / 0.00% / 78.59% | 20260505 23:45:05.854 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.102532% / 0% / 7.54992e-05% | 20260505 23:45:05.854 | INFO | Baz IO overall: | 20260505 23:45:05.854 | INFO | BytesWritten : 4140521581807 | 20260505 23:45:05.854 | INFO | NumEvents : 1950053124902 | 20260505 23:45:07.644 | INFO | Analyzing chunk frames = [7254016,7254528) | 20260505 23:45:07.644 | INFO | MemInfo: { | 20260505 23:45:07.644 | INFO | "gpu" : " 17 GB", | 20260505 23:45:07.644 | INFO | "huge" : "", | 20260505 23:45:07.644 | INFO | "huge_shared" : " 27 GB", | 20260505 23:45:07.644 | INFO | "malloc" : " 14 GB", | 20260505 23:45:07.644 | INFO | "pinned" : " 29 GB" | 20260505 23:45:07.644 | INFO | } | 20260505 23:45:07.644 | INFO | Perf report for the last 5.13525 seconds: | 20260505 23:45:07.644 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:45:07.644 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:07.644 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:07.644 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:07.644 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260505 23:45:07.644 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:07.644 | INFO | RT_METRICS : 43.4%, 1.0, 22.7ms, 0 | 20260505 23:45:07.644 | INFO | PRE_HQ : 59.9%, 1.7, 54.1ms, 0 | 20260505 23:45:07.644 | INFO | BAZWRITER : 62.0%, 3.6, 114.1ms, 0 | 20260505 23:45:07.644 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 23:45:07.644 | INFO | Uploaded 36GiB to GPU in 2.18105s (16.5058GiB/s) | 20260505 23:45:07.644 | INFO | Downloaded 38GiB to GPU in 2.24522s (16.9248GiB/s) | 20260505 23:45:12.782 | INFO | Analyzing chunk frames = [7254528,7255040) | 20260505 23:45:12.782 | INFO | MemInfo: { | 20260505 23:45:12.782 | INFO | "gpu" : " 17 GB", | 20260505 23:45:12.782 | INFO | "huge" : "", | 20260505 23:45:12.782 | INFO | "huge_shared" : " 27 GB", | 20260505 23:45:12.782 | INFO | "malloc" : " 14 GB", | 20260505 23:45:12.782 | INFO | "pinned" : " 29 GB" | 20260505 23:45:12.782 | INFO | } | 20260505 23:45:12.782 | INFO | Perf report for the last 5.13818 seconds: | 20260505 23:45:12.782 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:45:12.782 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:12.782 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:12.782 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:12.782 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260505 23:45:12.782 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:12.782 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:12.782 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260505 23:45:12.782 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:12.782 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:12.782 | INFO | Uploaded 36GiB to GPU in 2.11077s (17.0554GiB/s) | 20260505 23:45:12.783 | INFO | Downloaded 34GiB to GPU in 2.00408s (16.9654GiB/s) | 20260505 23:45:17.922 | INFO | Analyzing chunk frames = [7255040,7255552) | 20260505 23:45:17.922 | INFO | MemInfo: { | 20260505 23:45:17.922 | INFO | "gpu" : " 17 GB", | 20260505 23:45:17.922 | INFO | "huge" : "", | 20260505 23:45:17.922 | INFO | "huge_shared" : " 27 GB", | 20260505 23:45:17.922 | INFO | "malloc" : " 14 GB", | 20260505 23:45:17.922 | INFO | "pinned" : " 29 GB" | 20260505 23:45:17.922 | INFO | } | 20260505 23:45:17.922 | INFO | Perf report for the last 5.14061 seconds: | 20260505 23:45:17.922 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:45:17.922 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:17.922 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:17.922 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:17.922 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260505 23:45:17.923 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:17.923 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:17.923 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:45:17.923 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:17.923 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:17.923 | INFO | Uploaded 36GiB to GPU in 2.16003s (16.6664GiB/s) | 20260505 23:45:17.923 | INFO | Downloaded 34GiB to GPU in 2.06121s (16.4952GiB/s) | 20260505 23:45:23.067 | INFO | Analyzing chunk frames = [7255552,7256064) | 20260505 23:45:23.067 | INFO | MemInfo: { | 20260505 23:45:23.067 | INFO | "gpu" : " 17 GB", | 20260505 23:45:23.067 | INFO | "huge" : "", | 20260505 23:45:23.067 | INFO | "huge_shared" : " 27 GB", | 20260505 23:45:23.067 | INFO | "malloc" : " 14 GB", | 20260505 23:45:23.067 | INFO | "pinned" : " 29 GB" | 20260505 23:45:23.067 | INFO | } | 20260505 23:45:23.067 | INFO | Perf report for the last 5.14485 seconds: | 20260505 23:45:23.067 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:45:23.067 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:23.067 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:23.067 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:23.067 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260505 23:45:23.067 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:23.067 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:23.067 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:45:23.067 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:23.067 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:23.067 | INFO | Uploaded 36GiB to GPU in 2.14985s (16.7453GiB/s) | 20260505 23:45:23.068 | INFO | Downloaded 34GiB to GPU in 2.03103s (16.7402GiB/s) | 20260505 23:45:28.209 | INFO | Analyzing chunk frames = [7256064,7256576) | 20260505 23:45:28.209 | INFO | MemInfo: { | 20260505 23:45:28.209 | INFO | "gpu" : " 17 GB", | 20260505 23:45:28.209 | INFO | "huge" : "", | 20260505 23:45:28.209 | INFO | "huge_shared" : " 27 GB", | 20260505 23:45:28.209 | INFO | "malloc" : " 14 GB", | 20260505 23:45:28.209 | INFO | "pinned" : " 29 GB" | 20260505 23:45:28.209 | INFO | } | 20260505 23:45:28.209 | INFO | Perf report for the last 5.14193 seconds: | 20260505 23:45:28.209 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:45:28.209 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:28.209 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:28.209 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:28.209 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260505 23:45:28.209 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:28.209 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:28.209 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:45:28.209 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:28.209 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:28.210 | INFO | Uploaded 36GiB to GPU in 2.08957s (17.2284GiB/s) | 20260505 23:45:28.210 | INFO | Downloaded 34GiB to GPU in 2.04561s (16.6209GiB/s) | 20260505 23:45:33.343 | INFO | Analyzing chunk frames = [7256576,7257088) | 20260505 23:45:33.344 | INFO | MemInfo: { | 20260505 23:45:33.344 | INFO | "gpu" : " 17 GB", | 20260505 23:45:33.344 | INFO | "huge" : "", | 20260505 23:45:33.344 | INFO | "huge_shared" : " 27 GB", | 20260505 23:45:33.344 | INFO | "malloc" : " 14 GB", | 20260505 23:45:33.344 | INFO | "pinned" : " 29 GB" | 20260505 23:45:33.344 | INFO | } | 20260505 23:45:33.344 | INFO | Perf report for the last 5.13443 seconds: | 20260505 23:45:33.344 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:45:33.344 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:33.344 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:33.344 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:33.344 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260505 23:45:33.344 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:33.344 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:33.344 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:45:33.344 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:33.344 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:33.344 | INFO | Uploaded 36GiB to GPU in 2.06946s (17.3958GiB/s) | 20260505 23:45:33.344 | INFO | Downloaded 34GiB to GPU in 2.04215s (16.6491GiB/s) | 20260505 23:45:38.478 | INFO | Analyzing chunk frames = [7257088,7257600) | 20260505 23:45:38.478 | INFO | MemInfo: { | 20260505 23:45:38.478 | INFO | "gpu" : " 17 GB", | 20260505 23:45:38.478 | INFO | "huge" : "", | 20260505 23:45:38.478 | INFO | "huge_shared" : " 27 GB", | 20260505 23:45:38.478 | INFO | "malloc" : " 14 GB", | 20260505 23:45:38.478 | INFO | "pinned" : " 29 GB" | 20260505 23:45:38.478 | INFO | } | 20260505 23:45:38.478 | INFO | Perf report for the last 5.13466 seconds: | 20260505 23:45:38.478 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:45:38.478 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:38.478 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:38.478 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:38.478 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260505 23:45:38.478 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:38.478 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:38.478 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 23:45:38.479 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:38.479 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:38.479 | INFO | Uploaded 36GiB to GPU in 2.09004s (17.2246GiB/s) | 20260505 23:45:38.479 | INFO | Downloaded 34GiB to GPU in 2.07027s (16.423GiB/s) | 20260505 23:45:43.621 | INFO | Analyzing chunk frames = [7257600,7258112) | 20260505 23:45:43.621 | INFO | MemInfo: { | 20260505 23:45:43.622 | INFO | "gpu" : " 17 GB", | 20260505 23:45:43.622 | INFO | "huge" : "", | 20260505 23:45:43.622 | INFO | "huge_shared" : " 27 GB", | 20260505 23:45:43.622 | INFO | "malloc" : " 14 GB", | 20260505 23:45:43.622 | INFO | "pinned" : " 29 GB" | 20260505 23:45:43.622 | INFO | } | 20260505 23:45:43.622 | INFO | Perf report for the last 5.14329 seconds: | 20260505 23:45:43.622 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:45:43.622 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:43.622 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:43.622 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:43.622 | INFO | ANALYSIS : 61.9%, 3.0, 96.8ms, 0 | 20260505 23:45:43.622 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:43.622 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:43.622 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:45:43.622 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:43.622 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:43.622 | INFO | Uploaded 36GiB to GPU in 2.11127s (17.0514GiB/s) | 20260505 23:45:43.622 | INFO | Downloaded 34GiB to GPU in 1.9979s (17.0178GiB/s) | 20260505 23:45:43.622 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 23:45:43.622 | INFO | Total mem usage: 15.3 GiB | 20260505 23:45:43.622 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 23:45:43.622 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.12%) | 20260505 23:45:43.622 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 23:45:43.622 | INFO | High Water Breakdown: | 20260505 23:45:43.622 | INFO | 60.9% : 8.55 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 23:45:43.622 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 23:45:43.622 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 23:45:43.622 | INFO | ------------------------------------------ | 20260505 23:45:43.622 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:45:43.622 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 23:45:43.622 | INFO | Total mem usage: 26.5 GiB | 20260505 23:45:43.622 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 23:45:43.622 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 23:45:43.622 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 23:45:43.622 | INFO | High Water Breakdown: | 20260505 23:45:43.622 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:45:43.622 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:45:43.622 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:45:43.622 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:45:43.623 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:45:43.623 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:45:43.623 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 23:45:43.623 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 23:45:43.623 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:45:43.623 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 23:45:43.623 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 23:45:43.623 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 23:45:43.623 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 23:45:43.623 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 23:45:43.623 | INFO | ------------------------------------------ | 20260505 23:45:43.623 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:45:43.623 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 23:45:43.623 | INFO | Total mem usage: 25.6 GiB | 20260505 23:45:43.623 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 23:45:43.623 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 23:45:43.623 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 23:45:43.623 | INFO | High Water Breakdown: | 20260505 23:45:43.623 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 23:45:43.623 | INFO | ------------------------------------------ | 20260505 23:45:43.623 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:45:43.623 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 23:45:43.623 | INFO | Total mem usage: 15.7 GiB | 20260505 23:45:43.623 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 23:45:43.623 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 23:45:43.623 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 23:45:43.623 | INFO | High Water Breakdown: | 20260505 23:45:43.623 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 23:45:43.623 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 23:45:43.623 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 23:45:43.623 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 23:45:43.623 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 23:45:43.623 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 23:45:43.623 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 23:45:43.623 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 23:45:43.623 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 23:45:43.623 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:45:43.623 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 23:45:43.623 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:45:43.623 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 23:45:43.623 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 23:45:43.623 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 23:45:43.623 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 23:45:43.623 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:45:43.623 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 23:45:43.623 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:45:43.623 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:45:43.623 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:45:43.623 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 23:45:43.623 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 23:45:43.623 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:45:43.623 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 23:45:43.623 | INFO | ------------------------------------------ | 20260505 23:45:43.623 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:45:43.623 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 23:45:43.623 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:45:43.623 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:45:43.623 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.40e+01, 1.34e+01 / 7.70e-01 | 20260505 23:45:43.623 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.04e-01, 1.09e-02 / 3.62e-03 | 20260505 23:45:43.623 | INFO | PerfReport: 11.82%, Baseline -- 3.70e+00 / 4.49e+00, 5.65e-01 / 1.41e-01 | 20260505 23:45:43.623 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.52e+00, 1.87e-01 / 8.85e-02 | 20260505 23:45:43.624 | INFO | PerfReport: 16.01%, DME -- 4.01e+01 / 4.41e+01, 3.90e+00 / 8.78e-01 | 20260505 23:45:43.624 | INFO | PerfReport: 57.32%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.51e+00 / 1.19e+00 | 20260505 23:45:43.624 | INFO | PerfReport: 7.11%, PulseAccumulating -- 2.23e+00 / 2.77e+00, 4.24e-01 / 2.22e-01 | 20260505 23:45:43.624 | INFO | PerfReport: 3.10%, Metrics -- 9.72e-01 / 1.27e+00, 2.61e-01 / 1.26e-01 | 20260505 23:45:43.624 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.37e-01 / 3.93e-01, 5.79e-02 / 2.66e-02 | 20260505 23:45:43.624 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 23:45:43.624 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:45:43.624 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:45:43.624 | INFO | PerfReport: Full Execution -- 9.85e+01 / -nan, 2.49e+01 / 0.00e+00 | 20260505 23:45:43.624 | INFO | PerfReport: 22.35%, Upload -- 2.20e+01 / 0.00e+00, 7.59e+00 / 0.00e+00 | 20260505 23:45:43.624 | INFO | PerfReport: 21.79%, Download -- 2.15e+01 / 0.00e+00, 7.60e+00 / 0.00e+00 | 20260505 23:45:43.624 | INFO | PerfReport: 31.83%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 23:45:43.624 | INFO | PerfReport: 2.17%, LockUpload -- 2.14e+00 / 0.00e+00, 7.98e+00 / 0.00e+00 | 20260505 23:45:43.624 | INFO | PerfReport: 1.28%, LockDownload -- 1.26e+00 / 0.00e+00, 2.84e+00 / 0.00e+00 | 20260505 23:45:43.624 | INFO | PerfReport: 20.56%, LockCompute -- 2.03e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260505 23:45:43.624 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.63e-02 / 0.00e+00 | 20260505 23:45:43.742 | INFO | Starting a round of Baz IO | 20260505 23:45:46.891 | INFO | Populating RTMetrics report | 20260505 23:45:46.892 | INFO | RTMetrics report populated | 20260505 23:45:46.892 | INFO | Opening tmp RTMetrics json file | 20260505 23:45:46.893 | INFO | Serializing RTMetrics to json | 20260505 23:45:46.894 | INFO | Serialization done, swapping tmp file now... | 20260505 23:45:46.894 | INFO | RTMetrics done | 20260505 23:45:46.970 | INFO | Finished a round of Baz IO: Wrote 1.882 GiB in 3.228 seconds | 20260505 23:45:46.970 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.41% / 0.00% / 78.59% | 20260505 23:45:46.970 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.101316% / 0% / 7.15256e-05% | 20260505 23:45:46.971 | INFO | Baz IO overall: | 20260505 23:45:46.971 | INFO | BytesWritten : 4142541847079 | 20260505 23:45:46.971 | INFO | NumEvents : 1950978332233 | 20260505 23:45:48.760 | INFO | Analyzing chunk frames = [7258112,7258624) | 20260505 23:45:48.760 | INFO | MemInfo: { | 20260505 23:45:48.760 | INFO | "gpu" : " 17 GB", | 20260505 23:45:48.760 | INFO | "huge" : "", | 20260505 23:45:48.760 | INFO | "huge_shared" : " 27 GB", | 20260505 23:45:48.760 | INFO | "malloc" : " 14 GB", | 20260505 23:45:48.760 | INFO | "pinned" : " 29 GB" | 20260505 23:45:48.760 | INFO | } | 20260505 23:45:48.760 | INFO | Perf report for the last 5.13874 seconds: | 20260505 23:45:48.760 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:45:48.760 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:48.760 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:48.760 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:48.760 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 23:45:48.760 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:48.761 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260505 23:45:48.761 | INFO | PRE_HQ : 59.8%, 1.7, 54.1ms, 0 | 20260505 23:45:48.761 | INFO | BAZWRITER : 62.0%, 3.6, 114.7ms, 0 | 20260505 23:45:48.761 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 23:45:48.761 | INFO | Uploaded 36GiB to GPU in 2.13382s (16.8711GiB/s) | 20260505 23:45:48.761 | INFO | Downloaded 38GiB to GPU in 2.23492s (17.0029GiB/s) | 20260505 23:45:53.895 | INFO | Analyzing chunk frames = [7258624,7259136) | 20260505 23:45:53.895 | INFO | MemInfo: { | 20260505 23:45:53.895 | INFO | "gpu" : " 17 GB", | 20260505 23:45:53.895 | INFO | "huge" : "", | 20260505 23:45:53.895 | INFO | "huge_shared" : " 27 GB", | 20260505 23:45:53.895 | INFO | "malloc" : " 14 GB", | 20260505 23:45:53.895 | INFO | "pinned" : " 29 GB" | 20260505 23:45:53.895 | INFO | } | 20260505 23:45:53.895 | INFO | Perf report for the last 5.13493 seconds: | 20260505 23:45:53.895 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:45:53.895 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:53.895 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:53.895 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:53.895 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260505 23:45:53.895 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:53.895 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:53.896 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260505 23:45:53.896 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:53.896 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:53.896 | INFO | Uploaded 36GiB to GPU in 2.10865s (17.0726GiB/s) | 20260505 23:45:53.896 | INFO | Downloaded 34GiB to GPU in 2.04191s (16.6511GiB/s) | 20260505 23:45:59.035 | INFO | Analyzing chunk frames = [7259136,7259648) | 20260505 23:45:59.035 | INFO | MemInfo: { | 20260505 23:45:59.035 | INFO | "gpu" : " 17 GB", | 20260505 23:45:59.035 | INFO | "huge" : "", | 20260505 23:45:59.035 | INFO | "huge_shared" : " 27 GB", | 20260505 23:45:59.035 | INFO | "malloc" : " 14 GB", | 20260505 23:45:59.035 | INFO | "pinned" : " 29 GB" | 20260505 23:45:59.035 | INFO | } | 20260505 23:45:59.035 | INFO | Perf report for the last 5.13997 seconds: | 20260505 23:45:59.035 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:45:59.035 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:59.035 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:59.035 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:59.035 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260505 23:45:59.035 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:59.035 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:59.035 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:45:59.035 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:59.035 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:45:59.036 | INFO | Uploaded 36GiB to GPU in 2.12989s (16.9023GiB/s) | 20260505 23:45:59.036 | INFO | Downloaded 34GiB to GPU in 2.01657s (16.8603GiB/s) | 20260505 23:46:04.178 | INFO | Analyzing chunk frames = [7259648,7260160) | 20260505 23:46:04.178 | INFO | MemInfo: { | 20260505 23:46:04.178 | INFO | "gpu" : " 17 GB", | 20260505 23:46:04.178 | INFO | "huge" : "", | 20260505 23:46:04.178 | INFO | "huge_shared" : " 27 GB", | 20260505 23:46:04.178 | INFO | "malloc" : " 14 GB", | 20260505 23:46:04.178 | INFO | "pinned" : " 29 GB" | 20260505 23:46:04.178 | INFO | } | 20260505 23:46:04.178 | INFO | Perf report for the last 5.14249 seconds: | 20260505 23:46:04.178 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:46:04.178 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:04.178 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:04.178 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:04.178 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260505 23:46:04.178 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:04.178 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:04.178 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260505 23:46:04.178 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:04.178 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:04.178 | INFO | Uploaded 36GiB to GPU in 2.14006s (16.8219GiB/s) | 20260505 23:46:04.178 | INFO | Downloaded 34GiB to GPU in 2.01227s (16.8963GiB/s) | 20260505 23:46:09.323 | INFO | Analyzing chunk frames = [7260160,7260672) | 20260505 23:46:09.323 | INFO | MemInfo: { | 20260505 23:46:09.323 | INFO | "gpu" : " 17 GB", | 20260505 23:46:09.323 | INFO | "huge" : "", | 20260505 23:46:09.323 | INFO | "huge_shared" : " 27 GB", | 20260505 23:46:09.323 | INFO | "malloc" : " 14 GB", | 20260505 23:46:09.323 | INFO | "pinned" : " 29 GB" | 20260505 23:46:09.323 | INFO | } | 20260505 23:46:09.323 | INFO | Perf report for the last 5.14489 seconds: | 20260505 23:46:09.323 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:46:09.323 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:09.323 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:09.323 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:09.323 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260505 23:46:09.323 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:09.323 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:09.323 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:46:09.323 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:09.323 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:09.323 | INFO | Uploaded 36GiB to GPU in 2.08524s (17.2642GiB/s) | 20260505 23:46:09.323 | INFO | Downloaded 34GiB to GPU in 2.04703s (16.6094GiB/s) | 20260505 23:46:14.456 | INFO | Analyzing chunk frames = [7260672,7261184) | 20260505 23:46:14.456 | INFO | MemInfo: { | 20260505 23:46:14.456 | INFO | "gpu" : " 17 GB", | 20260505 23:46:14.456 | INFO | "huge" : "", | 20260505 23:46:14.456 | INFO | "huge_shared" : " 27 GB", | 20260505 23:46:14.456 | INFO | "malloc" : " 14 GB", | 20260505 23:46:14.456 | INFO | "pinned" : " 29 GB" | 20260505 23:46:14.456 | INFO | } | 20260505 23:46:14.456 | INFO | Perf report for the last 5.13327 seconds: | 20260505 23:46:14.456 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:46:14.456 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:14.456 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:14.456 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:14.456 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260505 23:46:14.456 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:14.456 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:14.456 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:46:14.456 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:14.456 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:14.456 | INFO | Uploaded 36GiB to GPU in 2.07904s (17.3157GiB/s) | 20260505 23:46:14.456 | INFO | Downloaded 34GiB to GPU in 2.0415s (16.6544GiB/s) | 20260505 23:46:19.590 | INFO | Analyzing chunk frames = [7261184,7261696) | 20260505 23:46:19.590 | INFO | MemInfo: { | 20260505 23:46:19.590 | INFO | "gpu" : " 17 GB", | 20260505 23:46:19.590 | INFO | "huge" : "", | 20260505 23:46:19.590 | INFO | "huge_shared" : " 27 GB", | 20260505 23:46:19.590 | INFO | "malloc" : " 14 GB", | 20260505 23:46:19.590 | INFO | "pinned" : " 29 GB" | 20260505 23:46:19.590 | INFO | } | 20260505 23:46:19.590 | INFO | Perf report for the last 5.13413 seconds: | 20260505 23:46:19.590 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:46:19.590 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:19.590 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:19.590 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:19.590 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260505 23:46:19.590 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:19.590 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:19.590 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 23:46:19.590 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:19.590 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:19.591 | INFO | Uploaded 36GiB to GPU in 2.12168s (16.9677GiB/s) | 20260505 23:46:19.591 | INFO | Downloaded 34GiB to GPU in 2.0732s (16.3997GiB/s) | 20260505 23:46:24.733 | INFO | Analyzing chunk frames = [7261696,7262208) | 20260505 23:46:24.733 | INFO | MemInfo: { | 20260505 23:46:24.733 | INFO | "gpu" : " 17 GB", | 20260505 23:46:24.733 | INFO | "huge" : "", | 20260505 23:46:24.733 | INFO | "huge_shared" : " 27 GB", | 20260505 23:46:24.733 | INFO | "malloc" : " 14 GB", | 20260505 23:46:24.733 | INFO | "pinned" : " 29 GB" | 20260505 23:46:24.733 | INFO | } | 20260505 23:46:24.733 | INFO | Perf report for the last 5.14288 seconds: | 20260505 23:46:24.733 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:46:24.733 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:24.733 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:24.733 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:24.733 | INFO | ANALYSIS : 61.6%, 3.0, 96.3ms, 0 | 20260505 23:46:24.733 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:24.733 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:24.733 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260505 23:46:24.733 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:24.733 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:24.734 | INFO | Uploaded 36GiB to GPU in 2.05997s (17.476GiB/s) | 20260505 23:46:24.734 | INFO | Downloaded 34GiB to GPU in 2.01165s (16.9016GiB/s) | 20260505 23:46:24.853 | INFO | Starting a round of Baz IO | 20260505 23:46:27.997 | INFO | Populating RTMetrics report | 20260505 23:46:27.997 | INFO | RTMetrics report populated | 20260505 23:46:27.997 | INFO | Opening tmp RTMetrics json file | 20260505 23:46:27.998 | INFO | Serializing RTMetrics to json | 20260505 23:46:28.000 | INFO | Serialization done, swapping tmp file now... | 20260505 23:46:28.000 | INFO | RTMetrics done | 20260505 23:46:28.079 | INFO | Finished a round of Baz IO: Wrote 1.881 GiB in 3.226 seconds | 20260505 23:46:28.079 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.41% / 0.00% / 78.59% | 20260505 23:46:28.079 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.101443% / 0% / 5.96046e-05% | 20260505 23:46:28.079 | INFO | Baz IO overall: | 20260505 23:46:28.079 | INFO | BytesWritten : 4144561908340 | 20260505 23:46:28.079 | INFO | NumEvents : 1951903593899 | 20260505 23:46:29.882 | INFO | Analyzing chunk frames = [7262208,7262720) | 20260505 23:46:29.882 | INFO | MemInfo: { | 20260505 23:46:29.882 | INFO | "gpu" : " 17 GB", | 20260505 23:46:29.882 | INFO | "huge" : "", | 20260505 23:46:29.882 | INFO | "huge_shared" : " 27 GB", | 20260505 23:46:29.882 | INFO | "malloc" : " 14 GB", | 20260505 23:46:29.882 | INFO | "pinned" : " 29 GB" | 20260505 23:46:29.882 | INFO | } | 20260505 23:46:29.882 | INFO | Perf report for the last 5.14876 seconds: | 20260505 23:46:29.882 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:46:29.882 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:29.882 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:29.882 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:29.882 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260505 23:46:29.882 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:29.882 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260505 23:46:29.882 | INFO | PRE_HQ : 59.7%, 1.7, 53.9ms, 0 | 20260505 23:46:29.882 | INFO | BAZWRITER : 61.9%, 3.5, 113.0ms, 0 | 20260505 23:46:29.882 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 23:46:29.882 | INFO | Uploaded 36GiB to GPU in 2.17174s (16.5766GiB/s) | 20260505 23:46:29.883 | INFO | Downloaded 38GiB to GPU in 2.23965s (16.967GiB/s) | 20260505 23:46:35.017 | INFO | Analyzing chunk frames = [7262720,7263232) | 20260505 23:46:35.017 | INFO | MemInfo: { | 20260505 23:46:35.017 | INFO | "gpu" : " 17 GB", | 20260505 23:46:35.017 | INFO | "huge" : "", | 20260505 23:46:35.017 | INFO | "huge_shared" : " 27 GB", | 20260505 23:46:35.017 | INFO | "malloc" : " 14 GB", | 20260505 23:46:35.017 | INFO | "pinned" : " 29 GB" | 20260505 23:46:35.017 | INFO | } | 20260505 23:46:35.017 | INFO | Perf report for the last 5.13495 seconds: | 20260505 23:46:35.017 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:46:35.017 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:35.017 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:35.017 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:35.017 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260505 23:46:35.017 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:35.017 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:35.017 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260505 23:46:35.017 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:35.017 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:35.017 | INFO | Uploaded 36GiB to GPU in 1.97191s (18.2564GiB/s) | 20260505 23:46:35.017 | INFO | Downloaded 34GiB to GPU in 1.98517s (17.127GiB/s) | 20260505 23:46:40.155 | INFO | Analyzing chunk frames = [7263232,7263744) | 20260505 23:46:40.155 | INFO | MemInfo: { | 20260505 23:46:40.155 | INFO | "gpu" : " 17 GB", | 20260505 23:46:40.155 | INFO | "huge" : "", | 20260505 23:46:40.155 | INFO | "huge_shared" : " 27 GB", | 20260505 23:46:40.155 | INFO | "malloc" : " 14 GB", | 20260505 23:46:40.155 | INFO | "pinned" : " 29 GB" | 20260505 23:46:40.155 | INFO | } | 20260505 23:46:40.155 | INFO | Perf report for the last 5.13823 seconds: | 20260505 23:46:40.155 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:46:40.155 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:40.155 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:40.155 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:40.155 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260505 23:46:40.155 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:40.155 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:40.155 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:46:40.155 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:40.155 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:40.155 | INFO | Uploaded 36GiB to GPU in 2.08776s (17.2434GiB/s) | 20260505 23:46:40.155 | INFO | Downloaded 34GiB to GPU in 2.02061s (16.8266GiB/s) | 20260505 23:46:45.290 | INFO | Analyzing chunk frames = [7263744,7264256) | 20260505 23:46:45.290 | INFO | MemInfo: { | 20260505 23:46:45.290 | INFO | "gpu" : " 17 GB", | 20260505 23:46:45.290 | INFO | "huge" : "", | 20260505 23:46:45.290 | INFO | "huge_shared" : " 27 GB", | 20260505 23:46:45.290 | INFO | "malloc" : " 14 GB", | 20260505 23:46:45.290 | INFO | "pinned" : " 29 GB" | 20260505 23:46:45.290 | INFO | } | 20260505 23:46:45.290 | INFO | Perf report for the last 5.13525 seconds: | 20260505 23:46:45.290 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:46:45.290 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:45.290 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:45.290 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:45.290 | INFO | ANALYSIS : 61.8%, 3.0, 95.9ms, 0 | 20260505 23:46:45.290 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:45.290 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:45.290 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260505 23:46:45.290 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:45.290 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:45.291 | INFO | Uploaded 36GiB to GPU in 2.13804s (16.8378GiB/s) | 20260505 23:46:45.291 | INFO | Downloaded 34GiB to GPU in 2.0144s (16.8785GiB/s) | 20260505 23:46:50.434 | INFO | Analyzing chunk frames = [7264256,7264768) | 20260505 23:46:50.434 | INFO | MemInfo: { | 20260505 23:46:50.434 | INFO | "gpu" : " 17 GB", | 20260505 23:46:50.434 | INFO | "huge" : "", | 20260505 23:46:50.434 | INFO | "huge_shared" : " 27 GB", | 20260505 23:46:50.434 | INFO | "malloc" : " 14 GB", | 20260505 23:46:50.434 | INFO | "pinned" : " 29 GB" | 20260505 23:46:50.434 | INFO | } | 20260505 23:46:50.434 | INFO | Perf report for the last 5.14344 seconds: | 20260505 23:46:50.434 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:46:50.434 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:50.434 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:50.434 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:50.434 | INFO | ANALYSIS : 61.5%, 3.0, 96.1ms, 0 | 20260505 23:46:50.434 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:50.434 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:50.434 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260505 23:46:50.434 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:50.434 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:50.434 | INFO | Uploaded 36GiB to GPU in 2.15989s (16.6675GiB/s) | 20260505 23:46:50.434 | INFO | Downloaded 34GiB to GPU in 2.07129s (16.4149GiB/s) | 20260505 23:46:55.569 | INFO | Analyzing chunk frames = [7264768,7265280) | 20260505 23:46:55.569 | INFO | MemInfo: { | 20260505 23:46:55.569 | INFO | "gpu" : " 17 GB", | 20260505 23:46:55.569 | INFO | "huge" : "", | 20260505 23:46:55.569 | INFO | "huge_shared" : " 27 GB", | 20260505 23:46:55.569 | INFO | "malloc" : " 14 GB", | 20260505 23:46:55.569 | INFO | "pinned" : " 29 GB" | 20260505 23:46:55.569 | INFO | } | 20260505 23:46:55.569 | INFO | Perf report for the last 5.13538 seconds: | 20260505 23:46:55.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:46:55.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:55.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:55.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:55.569 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260505 23:46:55.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:55.569 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:55.569 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:46:55.569 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:55.569 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:46:55.569 | INFO | Uploaded 36GiB to GPU in 2.0773s (17.3302GiB/s) | 20260505 23:46:55.569 | INFO | Downloaded 34GiB to GPU in 2.04214s (16.6492GiB/s) | 20260505 23:47:00.709 | INFO | Analyzing chunk frames = [7265280,7265792) | 20260505 23:47:00.709 | INFO | MemInfo: { | 20260505 23:47:00.709 | INFO | "gpu" : " 17 GB", | 20260505 23:47:00.709 | INFO | "huge" : "", | 20260505 23:47:00.709 | INFO | "huge_shared" : " 27 GB", | 20260505 23:47:00.709 | INFO | "malloc" : " 14 GB", | 20260505 23:47:00.709 | INFO | "pinned" : " 29 GB" | 20260505 23:47:00.709 | INFO | } | 20260505 23:47:00.709 | INFO | Perf report for the last 5.14004 seconds: | 20260505 23:47:00.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:47:00.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:00.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:00.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:00.709 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260505 23:47:00.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:00.709 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:00.709 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 23:47:00.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:00.709 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:00.709 | INFO | Uploaded 36GiB to GPU in 2.01673s (17.8507GiB/s) | 20260505 23:47:00.710 | INFO | Downloaded 34GiB to GPU in 2.03463s (16.7107GiB/s) | 20260505 23:47:05.850 | INFO | Analyzing chunk frames = [7265792,7266304) | 20260505 23:47:05.850 | INFO | MemInfo: { | 20260505 23:47:05.850 | INFO | "gpu" : " 17 GB", | 20260505 23:47:05.850 | INFO | "huge" : "", | 20260505 23:47:05.850 | INFO | "huge_shared" : " 27 GB", | 20260505 23:47:05.850 | INFO | "malloc" : " 14 GB", | 20260505 23:47:05.850 | INFO | "pinned" : " 29 GB" | 20260505 23:47:05.851 | INFO | } | 20260505 23:47:05.851 | INFO | Perf report for the last 5.14131 seconds: | 20260505 23:47:05.851 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:47:05.851 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:05.851 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:05.851 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:05.851 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260505 23:47:05.851 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:05.851 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:05.851 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:47:05.851 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:05.851 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:05.851 | INFO | Uploaded 36GiB to GPU in 2.10825s (17.0758GiB/s) | 20260505 23:47:05.851 | INFO | Downloaded 34GiB to GPU in 2.00293s (16.9751GiB/s) | 20260505 23:47:05.851 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 23:47:05.851 | INFO | Total mem usage: 15.3 GiB | 20260505 23:47:05.851 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 23:47:05.851 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.11%) | 20260505 23:47:05.851 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 23:47:05.851 | INFO | High Water Breakdown: | 20260505 23:47:05.851 | INFO | 60.9% : 8.55 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 23:47:05.851 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 23:47:05.851 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 23:47:05.851 | INFO | ------------------------------------------ | 20260505 23:47:05.851 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:47:05.851 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 23:47:05.851 | INFO | Total mem usage: 26.5 GiB | 20260505 23:47:05.851 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 23:47:05.851 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 23:47:05.851 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260505 23:47:05.851 | INFO | High Water Breakdown: | 20260505 23:47:05.851 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:47:05.851 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:47:05.851 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:47:05.851 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:47:05.851 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:47:05.851 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:47:05.851 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 23:47:05.851 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 23:47:05.851 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:47:05.851 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 23:47:05.851 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 23:47:05.851 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 23:47:05.851 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 23:47:05.851 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 23:47:05.851 | INFO | ------------------------------------------ | 20260505 23:47:05.851 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:47:05.851 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 23:47:05.851 | INFO | Total mem usage: 25.6 GiB | 20260505 23:47:05.851 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 23:47:05.851 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 23:47:05.851 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 23:47:05.851 | INFO | High Water Breakdown: | 20260505 23:47:05.851 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 23:47:05.851 | INFO | ------------------------------------------ | 20260505 23:47:05.851 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:47:05.851 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 23:47:05.851 | INFO | Total mem usage: 15.7 GiB | 20260505 23:47:05.851 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 23:47:05.851 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 23:47:05.851 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 23:47:05.851 | INFO | High Water Breakdown: | 20260505 23:47:05.851 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 23:47:05.851 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 23:47:05.851 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 23:47:05.851 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 23:47:05.851 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 23:47:05.851 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 23:47:05.851 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 23:47:05.851 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 23:47:05.851 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 23:47:05.851 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:47:05.851 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 23:47:05.851 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:47:05.851 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 23:47:05.851 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 23:47:05.851 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 23:47:05.851 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 23:47:05.852 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:47:05.852 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 23:47:05.852 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:47:05.852 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:47:05.852 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:47:05.852 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 23:47:05.852 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 23:47:05.852 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:47:05.852 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 23:47:05.852 | INFO | ------------------------------------------ | 20260505 23:47:05.852 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:47:05.852 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 23:47:05.852 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:47:05.852 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:47:05.852 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.39e+01, 1.36e+01 / 6.46e-01 | 20260505 23:47:05.852 | INFO | PerfReport: 0.29%, AggregateStats -- 9.18e-02 / 1.04e-01, 9.99e-03 / 4.67e-03 | 20260505 23:47:05.852 | INFO | PerfReport: 11.84%, Baseline -- 3.71e+00 / 4.52e+00, 5.78e-01 / 1.93e-01 | 20260505 23:47:05.852 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.53e+00, 1.94e-01 / 1.04e-01 | 20260505 23:47:05.852 | INFO | PerfReport: 15.98%, DME -- 4.01e+01 / 4.40e+01, 3.73e+00 / 8.54e-01 | 20260505 23:47:05.852 | INFO | PerfReport: 57.30%, FrameLabeling -- 1.80e+01 / 1.93e+01, 1.50e+00 / 1.02e+00 | 20260505 23:47:05.852 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.23e+00 / 2.80e+00, 4.28e-01 / 2.15e-01 | 20260505 23:47:05.852 | INFO | PerfReport: 3.10%, Metrics -- 9.72e-01 / 1.26e+00, 2.62e-01 / 1.29e-01 | 20260505 23:47:05.852 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.36e-01 / 3.90e-01, 5.77e-02 / 3.14e-02 | 20260505 23:47:05.852 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 23:47:05.852 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:47:05.852 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:47:05.852 | INFO | PerfReport: Full Execution -- 9.83e+01 / -nan, 2.47e+01 / 0.00e+00 | 20260505 23:47:05.852 | INFO | PerfReport: 22.24%, Upload -- 2.19e+01 / 0.00e+00, 7.48e+00 / 0.00e+00 | 20260505 23:47:05.852 | INFO | PerfReport: 21.78%, Download -- 2.14e+01 / 0.00e+00, 7.55e+00 / 0.00e+00 | 20260505 23:47:05.852 | INFO | PerfReport: 31.89%, Compute -- 3.14e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 23:47:05.852 | INFO | PerfReport: 2.15%, LockUpload -- 2.11e+00 / 0.00e+00, 7.89e+00 / 0.00e+00 | 20260505 23:47:05.852 | INFO | PerfReport: 1.30%, LockDownload -- 1.28e+00 / 0.00e+00, 2.90e+00 / 0.00e+00 | 20260505 23:47:05.852 | INFO | PerfReport: 20.63%, LockCompute -- 2.03e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260505 23:47:05.852 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.26e-02 / 0.00e+00 | 20260505 23:47:05.975 | INFO | Starting a round of Baz IO | 20260505 23:47:09.121 | INFO | Populating RTMetrics report | 20260505 23:47:09.122 | INFO | RTMetrics report populated | 20260505 23:47:09.122 | INFO | Opening tmp RTMetrics json file | 20260505 23:47:09.123 | INFO | Serializing RTMetrics to json | 20260505 23:47:09.126 | INFO | Serialization done, swapping tmp file now... | 20260505 23:47:09.130 | INFO | RTMetrics done | 20260505 23:47:09.202 | INFO | Finished a round of Baz IO: Wrote 1.88 GiB in 3.226 seconds | 20260505 23:47:09.202 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.41% / 0.00% / 78.59% | 20260505 23:47:09.202 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.100784% / 3.97364e-06% / 9.53674e-05% | 20260505 23:47:09.202 | INFO | Baz IO overall: | 20260505 23:47:09.202 | INFO | BytesWritten : 4146581023853 | 20260505 23:47:09.202 | INFO | NumEvents : 1952828543241 | 20260505 23:47:10.986 | INFO | Analyzing chunk frames = [7266304,7266816) | 20260505 23:47:10.986 | INFO | MemInfo: { | 20260505 23:47:10.986 | INFO | "gpu" : " 17 GB", | 20260505 23:47:10.986 | INFO | "huge" : "", | 20260505 23:47:10.986 | INFO | "huge_shared" : " 27 GB", | 20260505 23:47:10.986 | INFO | "malloc" : " 14 GB", | 20260505 23:47:10.986 | INFO | "pinned" : " 29 GB" | 20260505 23:47:10.986 | INFO | } | 20260505 23:47:10.986 | INFO | Perf report for the last 5.13574 seconds: | 20260505 23:47:10.986 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:47:10.986 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:10.986 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:10.986 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:10.986 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260505 23:47:10.986 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:10.986 | INFO | RT_METRICS : 43.6%, 1.0, 22.8ms, 0 | 20260505 23:47:10.986 | INFO | PRE_HQ : 59.8%, 1.7, 54.2ms, 0 | 20260505 23:47:10.986 | INFO | BAZWRITER : 62.1%, 3.6, 113.4ms, 0 | 20260505 23:47:10.986 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 23:47:10.987 | INFO | Uploaded 36GiB to GPU in 2.16827s (16.6031GiB/s) | 20260505 23:47:10.987 | INFO | Downloaded 38GiB to GPU in 2.23219s (17.0236GiB/s) | 20260505 23:47:16.129 | INFO | Analyzing chunk frames = [7266816,7267328) | 20260505 23:47:16.129 | INFO | MemInfo: { | 20260505 23:47:16.129 | INFO | "gpu" : " 17 GB", | 20260505 23:47:16.129 | INFO | "huge" : "", | 20260505 23:47:16.129 | INFO | "huge_shared" : " 27 GB", | 20260505 23:47:16.129 | INFO | "malloc" : " 14 GB", | 20260505 23:47:16.129 | INFO | "pinned" : " 29 GB" | 20260505 23:47:16.129 | INFO | } | 20260505 23:47:16.129 | INFO | Perf report for the last 5.14267 seconds: | 20260505 23:47:16.129 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:47:16.129 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:16.129 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:16.129 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:16.129 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 23:47:16.129 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:16.129 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:16.129 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:47:16.129 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:16.129 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:16.130 | INFO | Uploaded 36GiB to GPU in 2.10764s (17.0807GiB/s) | 20260505 23:47:16.130 | INFO | Downloaded 34GiB to GPU in 2.03925s (16.6728GiB/s) | 20260505 23:47:21.271 | INFO | Analyzing chunk frames = [7267328,7267840) | 20260505 23:47:21.271 | INFO | MemInfo: { | 20260505 23:47:21.271 | INFO | "gpu" : " 17 GB", | 20260505 23:47:21.271 | INFO | "huge" : "", | 20260505 23:47:21.271 | INFO | "huge_shared" : " 27 GB", | 20260505 23:47:21.271 | INFO | "malloc" : " 14 GB", | 20260505 23:47:21.271 | INFO | "pinned" : " 29 GB" | 20260505 23:47:21.271 | INFO | } | 20260505 23:47:21.271 | INFO | Perf report for the last 5.14171 seconds: | 20260505 23:47:21.271 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:47:21.271 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:21.271 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:21.271 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:21.271 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260505 23:47:21.271 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:21.271 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:21.271 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:47:21.271 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:21.271 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:21.271 | INFO | Uploaded 36GiB to GPU in 2.06073s (17.4696GiB/s) | 20260505 23:47:21.271 | INFO | Downloaded 34GiB to GPU in 2.01511s (16.8725GiB/s) | 20260505 23:47:26.404 | INFO | Analyzing chunk frames = [7267840,7268352) | 20260505 23:47:26.404 | INFO | MemInfo: { | 20260505 23:47:26.404 | INFO | "gpu" : " 17 GB", | 20260505 23:47:26.404 | INFO | "huge" : "", | 20260505 23:47:26.404 | INFO | "huge_shared" : " 27 GB", | 20260505 23:47:26.404 | INFO | "malloc" : " 14 GB", | 20260505 23:47:26.404 | INFO | "pinned" : " 29 GB" | 20260505 23:47:26.404 | INFO | } | 20260505 23:47:26.404 | INFO | Perf report for the last 5.13312 seconds: | 20260505 23:47:26.404 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:47:26.404 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:26.404 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:26.404 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:26.404 | INFO | ANALYSIS : 61.8%, 3.0, 95.8ms, 0 | 20260505 23:47:26.404 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:26.404 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:26.404 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:47:26.404 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:26.404 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:26.404 | INFO | Uploaded 36GiB to GPU in 2.14053s (16.8183GiB/s) | 20260505 23:47:26.404 | INFO | Downloaded 34GiB to GPU in 2.00975s (16.9176GiB/s) | 20260505 23:47:31.542 | INFO | Analyzing chunk frames = [7268352,7268864) | 20260505 23:47:31.542 | INFO | MemInfo: { | 20260505 23:47:31.542 | INFO | "gpu" : " 17 GB", | 20260505 23:47:31.542 | INFO | "huge" : "", | 20260505 23:47:31.542 | INFO | "huge_shared" : " 27 GB", | 20260505 23:47:31.542 | INFO | "malloc" : " 14 GB", | 20260505 23:47:31.542 | INFO | "pinned" : " 29 GB" | 20260505 23:47:31.542 | INFO | } | 20260505 23:47:31.542 | INFO | Perf report for the last 5.13856 seconds: | 20260505 23:47:31.542 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:47:31.542 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:31.542 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:31.542 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:31.542 | INFO | ANALYSIS : 61.6%, 3.0, 96.2ms, 0 | 20260505 23:47:31.542 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:31.543 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:31.543 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 23:47:31.543 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:31.543 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:31.543 | INFO | Uploaded 36GiB to GPU in 2.15044s (16.7407GiB/s) | 20260505 23:47:31.543 | INFO | Downloaded 34GiB to GPU in 2.0681s (16.4402GiB/s) | 20260505 23:47:36.690 | INFO | Analyzing chunk frames = [7268864,7269376) | 20260505 23:47:36.690 | INFO | MemInfo: { | 20260505 23:47:36.690 | INFO | "gpu" : " 17 GB", | 20260505 23:47:36.690 | INFO | "huge" : "", | 20260505 23:47:36.690 | INFO | "huge_shared" : " 27 GB", | 20260505 23:47:36.690 | INFO | "malloc" : " 14 GB", | 20260505 23:47:36.690 | INFO | "pinned" : " 29 GB" | 20260505 23:47:36.690 | INFO | } | 20260505 23:47:36.690 | INFO | Perf report for the last 5.14752 seconds: | 20260505 23:47:36.690 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:47:36.690 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:36.690 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:36.690 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:36.690 | INFO | ANALYSIS : 61.4%, 3.0, 96.1ms, 0 | 20260505 23:47:36.690 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:36.690 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:36.690 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:47:36.690 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:36.690 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:36.690 | INFO | Uploaded 36GiB to GPU in 2.08459s (17.2696GiB/s) | 20260505 23:47:36.690 | INFO | Downloaded 34GiB to GPU in 2.04424s (16.6321GiB/s) | 20260505 23:47:41.838 | INFO | Analyzing chunk frames = [7269376,7269888) | 20260505 23:47:41.838 | INFO | MemInfo: { | 20260505 23:47:41.838 | INFO | "gpu" : " 17 GB", | 20260505 23:47:41.838 | INFO | "huge" : "", | 20260505 23:47:41.838 | INFO | "huge_shared" : " 27 GB", | 20260505 23:47:41.838 | INFO | "malloc" : " 14 GB", | 20260505 23:47:41.838 | INFO | "pinned" : " 29 GB" | 20260505 23:47:41.838 | INFO | } | 20260505 23:47:41.838 | INFO | Perf report for the last 5.14826 seconds: | 20260505 23:47:41.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:47:41.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:41.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:41.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:41.838 | INFO | ANALYSIS : 62.1%, 3.0, 97.1ms, 0 | 20260505 23:47:41.838 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:41.838 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:41.838 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 23:47:41.838 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:41.838 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:41.839 | INFO | Uploaded 36GiB to GPU in 2.0611s (17.4664GiB/s) | 20260505 23:47:41.839 | INFO | Downloaded 34GiB to GPU in 2.06297s (16.4811GiB/s) | 20260505 23:47:46.969 | INFO | Analyzing chunk frames = [7269888,7270400) | 20260505 23:47:46.970 | INFO | MemInfo: { | 20260505 23:47:46.970 | INFO | "gpu" : " 17 GB", | 20260505 23:47:46.970 | INFO | "huge" : "", | 20260505 23:47:46.970 | INFO | "huge_shared" : " 27 GB", | 20260505 23:47:46.970 | INFO | "malloc" : " 14 GB", | 20260505 23:47:46.970 | INFO | "pinned" : " 29 GB" | 20260505 23:47:46.970 | INFO | } | 20260505 23:47:46.970 | INFO | Perf report for the last 5.13144 seconds: | 20260505 23:47:46.970 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:47:46.970 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:46.970 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:46.970 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:46.970 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260505 23:47:46.970 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:46.970 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:46.970 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 23:47:46.970 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:46.970 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:46.970 | INFO | Uploaded 36GiB to GPU in 2.0633s (17.4478GiB/s) | 20260505 23:47:46.970 | INFO | Downloaded 34GiB to GPU in 2.0216s (16.8184GiB/s) | 20260505 23:47:47.088 | INFO | Starting a round of Baz IO | 20260505 23:47:50.238 | INFO | Populating RTMetrics report | 20260505 23:47:50.238 | INFO | RTMetrics report populated | 20260505 23:47:50.238 | INFO | Opening tmp RTMetrics json file | 20260505 23:47:50.240 | INFO | Serializing RTMetrics to json | 20260505 23:47:50.241 | INFO | Serialization done, swapping tmp file now... | 20260505 23:47:50.241 | INFO | RTMetrics done | 20260505 23:47:50.315 | INFO | Finished a round of Baz IO: Wrote 1.882 GiB in 3.227 seconds | 20260505 23:47:50.316 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.41% / 0.00% / 78.59% | 20260505 23:47:50.316 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.101336% / 3.97364e-06% / 5.96046e-05% | 20260505 23:47:50.316 | INFO | Baz IO overall: | 20260505 23:47:50.316 | INFO | BytesWritten : 4148601344339 | 20260505 23:47:50.316 | INFO | NumEvents : 1953754213452 | 20260505 23:47:52.101 | INFO | Analyzing chunk frames = [7270400,7270912) | 20260505 23:47:52.101 | INFO | MemInfo: { | 20260505 23:47:52.101 | INFO | "gpu" : " 17 GB", | 20260505 23:47:52.101 | INFO | "huge" : "", | 20260505 23:47:52.101 | INFO | "huge_shared" : " 27 GB", | 20260505 23:47:52.101 | INFO | "malloc" : " 14 GB", | 20260505 23:47:52.101 | INFO | "pinned" : " 29 GB" | 20260505 23:47:52.101 | INFO | } | 20260505 23:47:52.101 | INFO | Perf report for the last 5.13106 seconds: | 20260505 23:47:52.101 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:47:52.101 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:52.101 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:52.101 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:52.101 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260505 23:47:52.101 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:52.101 | INFO | RT_METRICS : 43.4%, 1.0, 22.7ms, 0 | 20260505 23:47:52.101 | INFO | PRE_HQ : 59.9%, 1.7, 54.0ms, 0 | 20260505 23:47:52.101 | INFO | BAZWRITER : 62.0%, 3.6, 114.3ms, 0 | 20260505 23:47:52.101 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 23:47:52.101 | INFO | Uploaded 36GiB to GPU in 2.1735s (16.5631GiB/s) | 20260505 23:47:52.101 | INFO | Downloaded 38GiB to GPU in 2.24008s (16.9637GiB/s) | 20260505 23:47:57.245 | INFO | Analyzing chunk frames = [7270912,7271424) | 20260505 23:47:57.245 | INFO | MemInfo: { | 20260505 23:47:57.245 | INFO | "gpu" : " 17 GB", | 20260505 23:47:57.245 | INFO | "huge" : "", | 20260505 23:47:57.245 | INFO | "huge_shared" : " 27 GB", | 20260505 23:47:57.245 | INFO | "malloc" : " 14 GB", | 20260505 23:47:57.245 | INFO | "pinned" : " 29 GB" | 20260505 23:47:57.245 | INFO | } | 20260505 23:47:57.245 | INFO | Perf report for the last 5.14456 seconds: | 20260505 23:47:57.245 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:47:57.245 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:57.245 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:57.245 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:57.245 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 23:47:57.245 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:57.245 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:57.245 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260505 23:47:57.245 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:57.245 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:47:57.246 | INFO | Uploaded 36GiB to GPU in 2.09558s (17.179GiB/s) | 20260505 23:47:57.246 | INFO | Downloaded 34GiB to GPU in 2.03343s (16.7205GiB/s) | 20260505 23:48:02.382 | INFO | Analyzing chunk frames = [7271424,7271936) | 20260505 23:48:02.382 | INFO | MemInfo: { | 20260505 23:48:02.382 | INFO | "gpu" : " 17 GB", | 20260505 23:48:02.382 | INFO | "huge" : "", | 20260505 23:48:02.382 | INFO | "huge_shared" : " 27 GB", | 20260505 23:48:02.382 | INFO | "malloc" : " 14 GB", | 20260505 23:48:02.382 | INFO | "pinned" : " 29 GB" | 20260505 23:48:02.382 | INFO | } | 20260505 23:48:02.382 | INFO | Perf report for the last 5.1366 seconds: | 20260505 23:48:02.382 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:48:02.382 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:02.382 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:02.382 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:02.382 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260505 23:48:02.382 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:02.382 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:02.382 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:48:02.382 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:02.382 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:02.382 | INFO | Uploaded 36GiB to GPU in 2.08313s (17.2817GiB/s) | 20260505 23:48:02.382 | INFO | Downloaded 34GiB to GPU in 2.02683s (16.7749GiB/s) | 20260505 23:48:07.529 | INFO | Analyzing chunk frames = [7271936,7272448) | 20260505 23:48:07.529 | INFO | MemInfo: { | 20260505 23:48:07.529 | INFO | "gpu" : " 17 GB", | 20260505 23:48:07.529 | INFO | "huge" : "", | 20260505 23:48:07.529 | INFO | "huge_shared" : " 27 GB", | 20260505 23:48:07.529 | INFO | "malloc" : " 14 GB", | 20260505 23:48:07.529 | INFO | "pinned" : " 29 GB" | 20260505 23:48:07.529 | INFO | } | 20260505 23:48:07.529 | INFO | Perf report for the last 5.14685 seconds: | 20260505 23:48:07.529 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:48:07.529 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:07.529 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:07.529 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:07.529 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260505 23:48:07.529 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:07.529 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:07.529 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:48:07.529 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:07.529 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:07.529 | INFO | Uploaded 36GiB to GPU in 2.14068s (16.8171GiB/s) | 20260505 23:48:07.529 | INFO | Downloaded 34GiB to GPU in 2.02943s (16.7534GiB/s) | 20260505 23:48:12.664 | INFO | Analyzing chunk frames = [7272448,7272960) | 20260505 23:48:12.664 | INFO | MemInfo: { | 20260505 23:48:12.664 | INFO | "gpu" : " 17 GB", | 20260505 23:48:12.664 | INFO | "huge" : "", | 20260505 23:48:12.664 | INFO | "huge_shared" : " 27 GB", | 20260505 23:48:12.664 | INFO | "malloc" : " 14 GB", | 20260505 23:48:12.664 | INFO | "pinned" : " 29 GB" | 20260505 23:48:12.664 | INFO | } | 20260505 23:48:12.664 | INFO | Perf report for the last 5.13523 seconds: | 20260505 23:48:12.664 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:48:12.664 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:12.664 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:12.664 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:12.664 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260505 23:48:12.664 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:12.664 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:12.664 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:48:12.664 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:12.664 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:12.665 | INFO | Uploaded 36GiB to GPU in 2.08241s (17.2876GiB/s) | 20260505 23:48:12.665 | INFO | Downloaded 34GiB to GPU in 2.04164s (16.6533GiB/s) | 20260505 23:48:17.799 | INFO | Analyzing chunk frames = [7272960,7273472) | 20260505 23:48:17.799 | INFO | MemInfo: { | 20260505 23:48:17.799 | INFO | "gpu" : " 17 GB", | 20260505 23:48:17.799 | INFO | "huge" : "", | 20260505 23:48:17.799 | INFO | "huge_shared" : " 27 GB", | 20260505 23:48:17.799 | INFO | "malloc" : " 14 GB", | 20260505 23:48:17.799 | INFO | "pinned" : " 29 GB" | 20260505 23:48:17.799 | INFO | } | 20260505 23:48:17.799 | INFO | Perf report for the last 5.13511 seconds: | 20260505 23:48:17.799 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:48:17.799 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:17.799 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:17.799 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:17.799 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260505 23:48:17.799 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:17.799 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:17.799 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:48:17.799 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:17.799 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:17.799 | INFO | Uploaded 36GiB to GPU in 2.06888s (17.4007GiB/s) | 20260505 23:48:17.799 | INFO | Downloaded 34GiB to GPU in 2.03666s (16.694GiB/s) | 20260505 23:48:22.944 | INFO | Analyzing chunk frames = [7273472,7273984) | 20260505 23:48:22.944 | INFO | MemInfo: { | 20260505 23:48:22.944 | INFO | "gpu" : " 17 GB", | 20260505 23:48:22.944 | INFO | "huge" : "", | 20260505 23:48:22.944 | INFO | "huge_shared" : " 27 GB", | 20260505 23:48:22.944 | INFO | "malloc" : " 14 GB", | 20260505 23:48:22.944 | INFO | "pinned" : " 29 GB" | 20260505 23:48:22.944 | INFO | } | 20260505 23:48:22.944 | INFO | Perf report for the last 5.14534 seconds: | 20260505 23:48:22.944 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:48:22.944 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:22.945 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:22.945 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:22.945 | INFO | ANALYSIS : 62.6%, 3.0, 97.9ms, 0 | 20260505 23:48:22.945 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:22.945 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:22.945 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260505 23:48:22.945 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:22.945 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:22.945 | INFO | Uploaded 36GiB to GPU in 2.14879s (16.7536GiB/s) | 20260505 23:48:22.945 | INFO | Downloaded 34GiB to GPU in 2.11314s (16.0898GiB/s) | 20260505 23:48:28.080 | INFO | Analyzing chunk frames = [7273984,7274496) | 20260505 23:48:28.080 | INFO | MemInfo: { | 20260505 23:48:28.080 | INFO | "gpu" : " 17 GB", | 20260505 23:48:28.081 | INFO | "huge" : "", | 20260505 23:48:28.081 | INFO | "huge_shared" : " 27 GB", | 20260505 23:48:28.081 | INFO | "malloc" : " 14 GB", | 20260505 23:48:28.081 | INFO | "pinned" : " 29 GB" | 20260505 23:48:28.081 | INFO | } | 20260505 23:48:28.081 | INFO | Perf report for the last 5.1361 seconds: | 20260505 23:48:28.081 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:48:28.081 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:28.081 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:28.081 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:28.081 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 23:48:28.081 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:28.081 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:28.081 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:48:28.081 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:28.081 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:28.081 | INFO | Uploaded 36GiB to GPU in 2.13106s (16.893GiB/s) | 20260505 23:48:28.081 | INFO | Downloaded 34GiB to GPU in 2.02877s (16.7589GiB/s) | 20260505 23:48:28.081 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 23:48:28.081 | INFO | Total mem usage: 15.3 GiB | 20260505 23:48:28.081 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 23:48:28.081 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.11%) | 20260505 23:48:28.081 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 23:48:28.081 | INFO | High Water Breakdown: | 20260505 23:48:28.081 | INFO | 60.9% : 8.55 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 23:48:28.081 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 23:48:28.081 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 23:48:28.081 | INFO | ------------------------------------------ | 20260505 23:48:28.081 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:48:28.081 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 23:48:28.081 | INFO | Total mem usage: 26.5 GiB | 20260505 23:48:28.081 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 23:48:28.081 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 23:48:28.081 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 23:48:28.081 | INFO | High Water Breakdown: | 20260505 23:48:28.081 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:48:28.081 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:48:28.081 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:48:28.081 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:48:28.081 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:48:28.081 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:48:28.081 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 23:48:28.081 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 23:48:28.081 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:48:28.081 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 23:48:28.081 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 23:48:28.081 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 23:48:28.081 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 23:48:28.081 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 23:48:28.081 | INFO | ------------------------------------------ | 20260505 23:48:28.081 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:48:28.081 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 23:48:28.081 | INFO | Total mem usage: 25.6 GiB | 20260505 23:48:28.081 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 23:48:28.081 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 23:48:28.081 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 23:48:28.081 | INFO | High Water Breakdown: | 20260505 23:48:28.081 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 23:48:28.081 | INFO | ------------------------------------------ | 20260505 23:48:28.081 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:48:28.081 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 23:48:28.081 | INFO | Total mem usage: 15.7 GiB | 20260505 23:48:28.081 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 23:48:28.081 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 23:48:28.081 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 23:48:28.081 | INFO | High Water Breakdown: | 20260505 23:48:28.081 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 23:48:28.081 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 23:48:28.081 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 23:48:28.081 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 23:48:28.081 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 23:48:28.081 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 23:48:28.081 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 23:48:28.081 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 23:48:28.081 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 23:48:28.081 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:48:28.081 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 23:48:28.081 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:48:28.081 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 23:48:28.081 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 23:48:28.081 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 23:48:28.081 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 23:48:28.081 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:48:28.081 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 23:48:28.081 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:48:28.081 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:48:28.081 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:48:28.082 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 23:48:28.082 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 23:48:28.082 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:48:28.082 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 23:48:28.082 | INFO | ------------------------------------------ | 20260505 23:48:28.082 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:48:28.082 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 23:48:28.082 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:48:28.082 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:48:28.082 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.40e+01, 1.34e+01 / 8.47e-01 | 20260505 23:48:28.082 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.03e-01, 1.05e-02 / 4.88e-03 | 20260505 23:48:28.082 | INFO | PerfReport: 11.84%, Baseline -- 3.71e+00 / 4.53e+00, 5.73e-01 / 1.92e-01 | 20260505 23:48:28.082 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.53e+00, 1.88e-01 / 8.29e-02 | 20260505 23:48:28.082 | INFO | PerfReport: 15.97%, DME -- 4.00e+01 / 4.39e+01, 3.79e+00 / 1.02e+00 | 20260505 23:48:28.082 | INFO | PerfReport: 57.34%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.51e+00 / 1.16e+00 | 20260505 23:48:28.082 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.23e+00 / 2.74e+00, 4.25e-01 / 2.46e-01 | 20260505 23:48:28.082 | INFO | PerfReport: 3.10%, Metrics -- 9.72e-01 / 1.26e+00, 2.64e-01 / 1.24e-01 | 20260505 23:48:28.082 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.36e-01 / 3.90e-01, 5.71e-02 / 3.39e-02 | 20260505 23:48:28.082 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 23:48:28.082 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:48:28.082 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:48:28.082 | INFO | PerfReport: Full Execution -- 9.84e+01 / -nan, 2.47e+01 / 0.00e+00 | 20260505 23:48:28.082 | INFO | PerfReport: 22.34%, Upload -- 2.20e+01 / 0.00e+00, 7.55e+00 / 0.00e+00 | 20260505 23:48:28.082 | INFO | PerfReport: 21.87%, Download -- 2.15e+01 / 0.00e+00, 7.64e+00 / 0.00e+00 | 20260505 23:48:28.082 | INFO | PerfReport: 31.85%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 23:48:28.082 | INFO | PerfReport: 2.12%, LockUpload -- 2.09e+00 / 0.00e+00, 7.88e+00 / 0.00e+00 | 20260505 23:48:28.082 | INFO | PerfReport: 1.33%, LockDownload -- 1.30e+00 / 0.00e+00, 2.87e+00 / 0.00e+00 | 20260505 23:48:28.082 | INFO | PerfReport: 20.48%, LockCompute -- 2.01e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260505 23:48:28.082 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.32e-02 / 0.00e+00 | 20260505 23:48:28.198 | INFO | Starting a round of Baz IO | 20260505 23:48:31.350 | INFO | Populating RTMetrics report | 20260505 23:48:31.350 | INFO | RTMetrics report populated | 20260505 23:48:31.350 | INFO | Opening tmp RTMetrics json file | 20260505 23:48:31.351 | INFO | Serializing RTMetrics to json | 20260505 23:48:31.352 | INFO | Serialization done, swapping tmp file now... | 20260505 23:48:31.353 | INFO | RTMetrics done | 20260505 23:48:31.434 | INFO | Finished a round of Baz IO: Wrote 1.882 GiB in 3.235 seconds | 20260505 23:48:31.434 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.41% / 0.00% / 78.59% | 20260505 23:48:31.434 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.101554% / 0% / 8.74201e-05% | 20260505 23:48:31.434 | INFO | Baz IO overall: | 20260505 23:48:31.434 | INFO | BytesWritten : 4150622077535 | 20260505 23:48:31.434 | INFO | NumEvents : 1954680148011 | 20260505 23:48:33.217 | INFO | Analyzing chunk frames = [7274496,7275008) | 20260505 23:48:33.217 | INFO | MemInfo: { | 20260505 23:48:33.217 | INFO | "gpu" : " 17 GB", | 20260505 23:48:33.217 | INFO | "huge" : "", | 20260505 23:48:33.217 | INFO | "huge_shared" : " 27 GB", | 20260505 23:48:33.217 | INFO | "malloc" : " 14 GB", | 20260505 23:48:33.217 | INFO | "pinned" : " 29 GB" | 20260505 23:48:33.217 | INFO | } | 20260505 23:48:33.217 | INFO | Perf report for the last 5.1365 seconds: | 20260505 23:48:33.217 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:48:33.217 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:33.217 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:33.217 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:33.217 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 23:48:33.217 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:33.217 | INFO | RT_METRICS : 43.4%, 1.0, 22.7ms, 0 | 20260505 23:48:33.217 | INFO | PRE_HQ : 59.7%, 1.7, 54.2ms, 0 | 20260505 23:48:33.217 | INFO | BAZWRITER : 62.1%, 3.6, 114.4ms, 0 | 20260505 23:48:33.217 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 23:48:33.217 | INFO | Uploaded 36GiB to GPU in 2.16348s (16.6399GiB/s) | 20260505 23:48:33.217 | INFO | Downloaded 38GiB to GPU in 2.24311s (16.9407GiB/s) | 20260505 23:48:38.351 | INFO | Analyzing chunk frames = [7275008,7275520) | 20260505 23:48:38.351 | INFO | MemInfo: { | 20260505 23:48:38.351 | INFO | "gpu" : " 17 GB", | 20260505 23:48:38.351 | INFO | "huge" : "", | 20260505 23:48:38.351 | INFO | "huge_shared" : " 27 GB", | 20260505 23:48:38.351 | INFO | "malloc" : " 14 GB", | 20260505 23:48:38.351 | INFO | "pinned" : " 29 GB" | 20260505 23:48:38.351 | INFO | } | 20260505 23:48:38.351 | INFO | Perf report for the last 5.13416 seconds: | 20260505 23:48:38.351 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:48:38.351 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:38.351 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:38.351 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:38.351 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260505 23:48:38.351 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:38.351 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:38.351 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260505 23:48:38.351 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:38.351 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:38.352 | INFO | Uploaded 36GiB to GPU in 2.08494s (17.2667GiB/s) | 20260505 23:48:38.352 | INFO | Downloaded 34GiB to GPU in 2.02838s (16.7622GiB/s) | 20260505 23:48:43.496 | INFO | Analyzing chunk frames = [7275520,7276032) | 20260505 23:48:43.496 | INFO | MemInfo: { | 20260505 23:48:43.496 | INFO | "gpu" : " 17 GB", | 20260505 23:48:43.496 | INFO | "huge" : "", | 20260505 23:48:43.496 | INFO | "huge_shared" : " 27 GB", | 20260505 23:48:43.496 | INFO | "malloc" : " 14 GB", | 20260505 23:48:43.496 | INFO | "pinned" : " 29 GB" | 20260505 23:48:43.496 | INFO | } | 20260505 23:48:43.496 | INFO | Perf report for the last 5.14485 seconds: | 20260505 23:48:43.496 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:48:43.496 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:43.496 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:43.496 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:43.496 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260505 23:48:43.496 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:43.496 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:43.496 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260505 23:48:43.496 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:43.496 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:43.497 | INFO | Uploaded 36GiB to GPU in 2.13915s (16.8291GiB/s) | 20260505 23:48:43.497 | INFO | Downloaded 34GiB to GPU in 2.03248s (16.7283GiB/s) | 20260505 23:48:48.640 | INFO | Analyzing chunk frames = [7276032,7276544) | 20260505 23:48:48.640 | INFO | MemInfo: { | 20260505 23:48:48.640 | INFO | "gpu" : " 17 GB", | 20260505 23:48:48.640 | INFO | "huge" : "", | 20260505 23:48:48.640 | INFO | "huge_shared" : " 27 GB", | 20260505 23:48:48.640 | INFO | "malloc" : " 14 GB", | 20260505 23:48:48.640 | INFO | "pinned" : " 29 GB" | 20260505 23:48:48.640 | INFO | } | 20260505 23:48:48.640 | INFO | Perf report for the last 5.1443 seconds: | 20260505 23:48:48.640 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:48:48.640 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:48.640 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:48.640 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:48.641 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260505 23:48:48.641 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:48.641 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:48.641 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:48:48.641 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:48.641 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:48.641 | INFO | Uploaded 36GiB to GPU in 2.13745s (16.8425GiB/s) | 20260505 23:48:48.641 | INFO | Downloaded 34GiB to GPU in 2.01513s (16.8724GiB/s) | 20260505 23:48:53.782 | INFO | Analyzing chunk frames = [7276544,7277056) | 20260505 23:48:53.782 | INFO | MemInfo: { | 20260505 23:48:53.782 | INFO | "gpu" : " 17 GB", | 20260505 23:48:53.782 | INFO | "huge" : "", | 20260505 23:48:53.782 | INFO | "huge_shared" : " 27 GB", | 20260505 23:48:53.782 | INFO | "malloc" : " 14 GB", | 20260505 23:48:53.782 | INFO | "pinned" : " 29 GB" | 20260505 23:48:53.782 | INFO | } | 20260505 23:48:53.782 | INFO | Perf report for the last 5.14134 seconds: | 20260505 23:48:53.782 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:48:53.782 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:53.782 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:53.782 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:53.782 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260505 23:48:53.782 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:53.782 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:53.782 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260505 23:48:53.782 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:53.782 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:53.782 | INFO | Uploaded 36GiB to GPU in 2.0949s (17.1846GiB/s) | 20260505 23:48:53.782 | INFO | Downloaded 34GiB to GPU in 2.0516s (16.5724GiB/s) | 20260505 23:48:58.917 | INFO | Analyzing chunk frames = [7277056,7277568) | 20260505 23:48:58.917 | INFO | MemInfo: { | 20260505 23:48:58.917 | INFO | "gpu" : " 17 GB", | 20260505 23:48:58.917 | INFO | "huge" : "", | 20260505 23:48:58.917 | INFO | "huge_shared" : " 27 GB", | 20260505 23:48:58.917 | INFO | "malloc" : " 14 GB", | 20260505 23:48:58.917 | INFO | "pinned" : " 29 GB" | 20260505 23:48:58.917 | INFO | } | 20260505 23:48:58.917 | INFO | Perf report for the last 5.13531 seconds: | 20260505 23:48:58.917 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:48:58.917 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:58.917 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:58.917 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:58.917 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260505 23:48:58.917 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:58.917 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:58.917 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:48:58.917 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:58.917 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:48:58.917 | INFO | Uploaded 36GiB to GPU in 2.06603s (17.4247GiB/s) | 20260505 23:48:58.917 | INFO | Downloaded 34GiB to GPU in 2.03571s (16.7018GiB/s) | 20260505 23:49:04.052 | INFO | Analyzing chunk frames = [7277568,7278080) | 20260505 23:49:04.052 | INFO | MemInfo: { | 20260505 23:49:04.052 | INFO | "gpu" : " 17 GB", | 20260505 23:49:04.052 | INFO | "huge" : "", | 20260505 23:49:04.052 | INFO | "huge_shared" : " 27 GB", | 20260505 23:49:04.052 | INFO | "malloc" : " 14 GB", | 20260505 23:49:04.052 | INFO | "pinned" : " 29 GB" | 20260505 23:49:04.052 | INFO | } | 20260505 23:49:04.052 | INFO | Perf report for the last 5.13477 seconds: | 20260505 23:49:04.052 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:49:04.052 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:04.052 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:04.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:04.052 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260505 23:49:04.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:04.052 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:04.052 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 23:49:04.052 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:04.052 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:04.052 | INFO | Uploaded 36GiB to GPU in 2.06108s (17.4666GiB/s) | 20260505 23:49:04.052 | INFO | Downloaded 34GiB to GPU in 2.06144s (16.4933GiB/s) | 20260505 23:49:09.195 | INFO | Analyzing chunk frames = [7278080,7278592) | 20260505 23:49:09.195 | INFO | MemInfo: { | 20260505 23:49:09.195 | INFO | "gpu" : " 17 GB", | 20260505 23:49:09.195 | INFO | "huge" : "", | 20260505 23:49:09.195 | INFO | "huge_shared" : " 27 GB", | 20260505 23:49:09.195 | INFO | "malloc" : " 14 GB", | 20260505 23:49:09.195 | INFO | "pinned" : " 29 GB" | 20260505 23:49:09.195 | INFO | } | 20260505 23:49:09.195 | INFO | Perf report for the last 5.14341 seconds: | 20260505 23:49:09.195 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:49:09.195 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:09.195 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:09.195 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:09.195 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 23:49:09.195 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:09.195 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:09.195 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:49:09.195 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:09.195 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:09.196 | INFO | Uploaded 36GiB to GPU in 2.11294s (17.0379GiB/s) | 20260505 23:49:09.196 | INFO | Downloaded 34GiB to GPU in 2.00027s (16.9977GiB/s) | 20260505 23:49:09.313 | INFO | Starting a round of Baz IO | 20260505 23:49:12.459 | INFO | Populating RTMetrics report | 20260505 23:49:12.460 | INFO | RTMetrics report populated | 20260505 23:49:12.460 | INFO | Opening tmp RTMetrics json file | 20260505 23:49:12.460 | INFO | Serializing RTMetrics to json | 20260505 23:49:12.462 | INFO | Serialization done, swapping tmp file now... | 20260505 23:49:12.462 | INFO | RTMetrics done | 20260505 23:49:12.536 | INFO | Finished a round of Baz IO: Wrote 1.88 GiB in 3.223 seconds | 20260505 23:49:12.536 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.41% / 0.00% / 78.59% | 20260505 23:49:12.536 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.100299% / 0% / 6.35783e-05% | 20260505 23:49:12.536 | INFO | Baz IO overall: | 20260505 23:49:12.536 | INFO | BytesWritten : 4152640198630 | 20260505 23:49:12.536 | INFO | NumEvents : 1955604407573 | 20260505 23:49:14.336 | INFO | Analyzing chunk frames = [7278592,7279104) | 20260505 23:49:14.336 | INFO | MemInfo: { | 20260505 23:49:14.336 | INFO | "gpu" : " 17 GB", | 20260505 23:49:14.336 | INFO | "huge" : "", | 20260505 23:49:14.336 | INFO | "huge_shared" : " 27 GB", | 20260505 23:49:14.336 | INFO | "malloc" : " 14 GB", | 20260505 23:49:14.336 | INFO | "pinned" : " 29 GB" | 20260505 23:49:14.336 | INFO | } | 20260505 23:49:14.336 | INFO | Perf report for the last 5.14092 seconds: | 20260505 23:49:14.336 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:49:14.336 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:14.336 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:14.336 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:14.336 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260505 23:49:14.336 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:14.336 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260505 23:49:14.336 | INFO | PRE_HQ : 59.7%, 1.7, 53.9ms, 0 | 20260505 23:49:14.336 | INFO | BAZWRITER : 61.9%, 3.5, 111.8ms, 0 | 20260505 23:49:14.336 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 23:49:14.337 | INFO | Uploaded 36GiB to GPU in 2.1634s (16.6405GiB/s) | 20260505 23:49:14.337 | INFO | Downloaded 38GiB to GPU in 2.2306s (17.0358GiB/s) | 20260505 23:49:19.478 | INFO | Analyzing chunk frames = [7279104,7279616) | 20260505 23:49:19.478 | INFO | MemInfo: { | 20260505 23:49:19.478 | INFO | "gpu" : " 17 GB", | 20260505 23:49:19.478 | INFO | "huge" : "", | 20260505 23:49:19.478 | INFO | "huge_shared" : " 27 GB", | 20260505 23:49:19.478 | INFO | "malloc" : " 14 GB", | 20260505 23:49:19.478 | INFO | "pinned" : " 29 GB" | 20260505 23:49:19.478 | INFO | } | 20260505 23:49:19.478 | INFO | Perf report for the last 5.14165 seconds: | 20260505 23:49:19.478 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:49:19.478 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:19.478 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:19.478 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:19.478 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 23:49:19.478 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:19.478 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:19.478 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260505 23:49:19.478 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:19.478 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:19.478 | INFO | Uploaded 36GiB to GPU in 2.11505s (17.0209GiB/s) | 20260505 23:49:19.478 | INFO | Downloaded 34GiB to GPU in 2.04069s (16.661GiB/s) | 20260505 23:49:24.616 | INFO | Analyzing chunk frames = [7279616,7280128) | 20260505 23:49:24.616 | INFO | MemInfo: { | 20260505 23:49:24.616 | INFO | "gpu" : " 17 GB", | 20260505 23:49:24.616 | INFO | "huge" : "", | 20260505 23:49:24.616 | INFO | "huge_shared" : " 27 GB", | 20260505 23:49:24.616 | INFO | "malloc" : " 14 GB", | 20260505 23:49:24.616 | INFO | "pinned" : " 29 GB" | 20260505 23:49:24.616 | INFO | } | 20260505 23:49:24.616 | INFO | Perf report for the last 5.13777 seconds: | 20260505 23:49:24.616 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:49:24.616 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:24.616 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:24.616 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:24.616 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260505 23:49:24.616 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:24.616 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:24.616 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:49:24.616 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:24.616 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:24.616 | INFO | Uploaded 36GiB to GPU in 2.0758s (17.3427GiB/s) | 20260505 23:49:24.616 | INFO | Downloaded 34GiB to GPU in 2.02496s (16.7905GiB/s) | 20260505 23:49:29.752 | INFO | Analyzing chunk frames = [7280128,7280640) | 20260505 23:49:29.752 | INFO | MemInfo: { | 20260505 23:49:29.752 | INFO | "gpu" : " 17 GB", | 20260505 23:49:29.752 | INFO | "huge" : "", | 20260505 23:49:29.752 | INFO | "huge_shared" : " 27 GB", | 20260505 23:49:29.752 | INFO | "malloc" : " 14 GB", | 20260505 23:49:29.752 | INFO | "pinned" : " 29 GB" | 20260505 23:49:29.752 | INFO | } | 20260505 23:49:29.752 | INFO | Perf report for the last 5.13658 seconds: | 20260505 23:49:29.752 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:49:29.752 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:29.752 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:29.752 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:29.752 | INFO | ANALYSIS : 61.7%, 3.0, 95.8ms, 0 | 20260505 23:49:29.752 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:29.752 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:29.752 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:49:29.752 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:29.752 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:29.753 | INFO | Uploaded 36GiB to GPU in 2.1349s (16.8626GiB/s) | 20260505 23:49:29.753 | INFO | Downloaded 34GiB to GPU in 2.00957s (16.919GiB/s) | 20260505 23:49:34.890 | INFO | Analyzing chunk frames = [7280640,7281152) | 20260505 23:49:34.890 | INFO | MemInfo: { | 20260505 23:49:34.890 | INFO | "gpu" : " 17 GB", | 20260505 23:49:34.890 | INFO | "huge" : "", | 20260505 23:49:34.890 | INFO | "huge_shared" : " 27 GB", | 20260505 23:49:34.890 | INFO | "malloc" : " 14 GB", | 20260505 23:49:34.890 | INFO | "pinned" : " 29 GB" | 20260505 23:49:34.890 | INFO | } | 20260505 23:49:34.890 | INFO | Perf report for the last 5.13809 seconds: | 20260505 23:49:34.890 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:49:34.890 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:34.890 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:34.890 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:34.890 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260505 23:49:34.890 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:34.890 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:34.890 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:49:34.891 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:34.891 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:34.891 | INFO | Uploaded 36GiB to GPU in 2.09708s (17.1667GiB/s) | 20260505 23:49:34.891 | INFO | Downloaded 34GiB to GPU in 2.04797s (16.6018GiB/s) | 20260505 23:49:40.029 | INFO | Analyzing chunk frames = [7281152,7281664) | 20260505 23:49:40.029 | INFO | MemInfo: { | 20260505 23:49:40.029 | INFO | "gpu" : " 17 GB", | 20260505 23:49:40.029 | INFO | "huge" : "", | 20260505 23:49:40.029 | INFO | "huge_shared" : " 27 GB", | 20260505 23:49:40.029 | INFO | "malloc" : " 14 GB", | 20260505 23:49:40.029 | INFO | "pinned" : " 29 GB" | 20260505 23:49:40.029 | INFO | } | 20260505 23:49:40.030 | INFO | Perf report for the last 5.13909 seconds: | 20260505 23:49:40.030 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:49:40.030 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:40.030 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:40.030 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:40.030 | INFO | ANALYSIS : 61.4%, 3.0, 95.9ms, 0 | 20260505 23:49:40.030 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:40.030 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:40.030 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:49:40.030 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:40.030 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:40.030 | INFO | Uploaded 36GiB to GPU in 2.07442s (17.3542GiB/s) | 20260505 23:49:40.030 | INFO | Downloaded 34GiB to GPU in 2.03651s (16.6953GiB/s) | 20260505 23:49:45.168 | INFO | Analyzing chunk frames = [7281664,7282176) | 20260505 23:49:45.168 | INFO | MemInfo: { | 20260505 23:49:45.168 | INFO | "gpu" : " 17 GB", | 20260505 23:49:45.168 | INFO | "huge" : "", | 20260505 23:49:45.168 | INFO | "huge_shared" : " 27 GB", | 20260505 23:49:45.168 | INFO | "malloc" : " 14 GB", | 20260505 23:49:45.168 | INFO | "pinned" : " 29 GB" | 20260505 23:49:45.168 | INFO | } | 20260505 23:49:45.168 | INFO | Perf report for the last 5.13868 seconds: | 20260505 23:49:45.168 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:49:45.168 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:45.168 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:45.168 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:45.168 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 23:49:45.168 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:45.168 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:45.168 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 23:49:45.168 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:45.168 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:45.169 | INFO | Uploaded 36GiB to GPU in 2.07655s (17.3365GiB/s) | 20260505 23:49:45.169 | INFO | Downloaded 34GiB to GPU in 2.06414s (16.4717GiB/s) | 20260505 23:49:50.303 | INFO | Analyzing chunk frames = [7282176,7282688) | 20260505 23:49:50.303 | INFO | MemInfo: { | 20260505 23:49:50.303 | INFO | "gpu" : " 17 GB", | 20260505 23:49:50.304 | INFO | "huge" : "", | 20260505 23:49:50.304 | INFO | "huge_shared" : " 27 GB", | 20260505 23:49:50.304 | INFO | "malloc" : " 14 GB", | 20260505 23:49:50.304 | INFO | "pinned" : " 29 GB" | 20260505 23:49:50.304 | INFO | } | 20260505 23:49:50.304 | INFO | Perf report for the last 5.13549 seconds: | 20260505 23:49:50.304 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:49:50.304 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:50.304 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:50.304 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:50.304 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260505 23:49:50.304 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:50.304 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:50.304 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:49:50.304 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:50.304 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:50.304 | INFO | Uploaded 36GiB to GPU in 2.11398s (17.0295GiB/s) | 20260505 23:49:50.304 | INFO | Downloaded 34GiB to GPU in 2.00692s (16.9414GiB/s) | 20260505 23:49:50.304 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 23:49:50.304 | INFO | Total mem usage: 15.3 GiB | 20260505 23:49:50.304 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 23:49:50.304 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.09%) | 20260505 23:49:50.304 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 23:49:50.304 | INFO | High Water Breakdown: | 20260505 23:49:50.304 | INFO | 60.9% : 8.55 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 23:49:50.305 | INFO | 39% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 23:49:50.305 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 23:49:50.305 | INFO | ------------------------------------------ | 20260505 23:49:50.305 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:49:50.305 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 23:49:50.305 | INFO | Total mem usage: 26.5 GiB | 20260505 23:49:50.305 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 23:49:50.305 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 23:49:50.305 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 23:49:50.305 | INFO | High Water Breakdown: | 20260505 23:49:50.305 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:49:50.305 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:49:50.305 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:49:50.305 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:49:50.305 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:49:50.305 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:49:50.305 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 23:49:50.305 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 23:49:50.305 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:49:50.305 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 23:49:50.305 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 23:49:50.305 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 23:49:50.305 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 23:49:50.305 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 23:49:50.305 | INFO | ------------------------------------------ | 20260505 23:49:50.305 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:49:50.305 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 23:49:50.305 | INFO | Total mem usage: 25.6 GiB | 20260505 23:49:50.305 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 23:49:50.305 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 23:49:50.305 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 23:49:50.305 | INFO | High Water Breakdown: | 20260505 23:49:50.305 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 23:49:50.305 | INFO | ------------------------------------------ | 20260505 23:49:50.305 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:49:50.305 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 23:49:50.305 | INFO | Total mem usage: 15.7 GiB | 20260505 23:49:50.305 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 23:49:50.305 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 23:49:50.305 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 23:49:50.305 | INFO | High Water Breakdown: | 20260505 23:49:50.305 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 23:49:50.305 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 23:49:50.305 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 23:49:50.305 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 23:49:50.305 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 23:49:50.305 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 23:49:50.305 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 23:49:50.305 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 23:49:50.305 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 23:49:50.305 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:49:50.305 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 23:49:50.305 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:49:50.305 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 23:49:50.305 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 23:49:50.305 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 23:49:50.305 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 23:49:50.305 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:49:50.305 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 23:49:50.305 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:49:50.305 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:49:50.305 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:49:50.305 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 23:49:50.305 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 23:49:50.305 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:49:50.305 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 23:49:50.305 | INFO | ------------------------------------------ | 20260505 23:49:50.305 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:49:50.305 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 23:49:50.305 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:49:50.305 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:49:50.305 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.39e+01, 1.35e+01 / 8.10e-01 | 20260505 23:49:50.305 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.04e-01, 1.02e-02 / 2.75e-03 | 20260505 23:49:50.305 | INFO | PerfReport: 11.83%, Baseline -- 3.71e+00 / 4.48e+00, 5.70e-01 / 2.35e-01 | 20260505 23:49:50.305 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.53e+00, 1.91e-01 / 9.39e-02 | 20260505 23:49:50.305 | INFO | PerfReport: 15.97%, DME -- 4.01e+01 / 4.39e+01, 3.74e+00 / 1.11e+00 | 20260505 23:49:50.305 | INFO | PerfReport: 57.33%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.51e+00 / 1.11e+00 | 20260505 23:49:50.305 | INFO | PerfReport: 7.11%, PulseAccumulating -- 2.23e+00 / 2.75e+00, 4.22e-01 / 2.54e-01 | 20260505 23:49:50.305 | INFO | PerfReport: 3.10%, Metrics -- 9.72e-01 / 1.25e+00, 2.62e-01 / 1.01e-01 | 20260505 23:49:50.305 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.36e-01 / 3.93e-01, 5.78e-02 / 3.15e-02 | 20260505 23:49:50.305 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 23:49:50.305 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:49:50.305 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:49:50.305 | INFO | PerfReport: Full Execution -- 9.84e+01 / -nan, 2.46e+01 / 0.00e+00 | 20260505 23:49:50.305 | INFO | PerfReport: 22.31%, Upload -- 2.19e+01 / 0.00e+00, 7.51e+00 / 0.00e+00 | 20260505 23:49:50.305 | INFO | PerfReport: 21.79%, Download -- 2.14e+01 / 0.00e+00, 7.55e+00 / 0.00e+00 | 20260505 23:49:50.306 | INFO | PerfReport: 31.87%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260505 23:49:50.306 | INFO | PerfReport: 2.15%, LockUpload -- 2.12e+00 / 0.00e+00, 7.95e+00 / 0.00e+00 | 20260505 23:49:50.306 | INFO | PerfReport: 1.26%, LockDownload -- 1.24e+00 / 0.00e+00, 2.80e+00 / 0.00e+00 | 20260505 23:49:50.306 | INFO | PerfReport: 20.60%, LockCompute -- 2.03e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260505 23:49:50.306 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.93e-02 / 0.00e+00 | 20260505 23:49:50.429 | INFO | Starting a round of Baz IO | 20260505 23:49:53.575 | INFO | Populating RTMetrics report | 20260505 23:49:53.575 | INFO | RTMetrics report populated | 20260505 23:49:53.575 | INFO | Opening tmp RTMetrics json file | 20260505 23:49:53.576 | INFO | Serializing RTMetrics to json | 20260505 23:49:53.577 | INFO | Serialization done, swapping tmp file now... | 20260505 23:49:53.577 | INFO | RTMetrics done | 20260505 23:49:53.652 | INFO | Finished a round of Baz IO: Wrote 1.882 GiB in 3.222 seconds | 20260505 23:49:53.652 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.41% / 0.00% / 78.59% | 20260505 23:49:53.652 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.100907% / 0% / 6.75519e-05% | 20260505 23:49:53.652 | INFO | Baz IO overall: | 20260505 23:49:53.652 | INFO | BytesWritten : 4154660475434 | 20260505 23:49:53.652 | INFO | NumEvents : 1956530230863 | 20260505 23:49:55.450 | INFO | Analyzing chunk frames = [7282688,7283200) | 20260505 23:49:55.450 | INFO | MemInfo: { | 20260505 23:49:55.450 | INFO | "gpu" : " 17 GB", | 20260505 23:49:55.450 | INFO | "huge" : "", | 20260505 23:49:55.450 | INFO | "huge_shared" : " 27 GB", | 20260505 23:49:55.451 | INFO | "malloc" : " 14 GB", | 20260505 23:49:55.451 | INFO | "pinned" : " 29 GB" | 20260505 23:49:55.451 | INFO | } | 20260505 23:49:55.451 | INFO | Perf report for the last 5.14686 seconds: | 20260505 23:49:55.451 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:49:55.451 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:55.451 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:55.451 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:55.451 | INFO | ANALYSIS : 62.5%, 3.0, 97.7ms, 0 | 20260505 23:49:55.451 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:49:55.451 | INFO | RT_METRICS : 43.5%, 1.0, 22.9ms, 0 | 20260505 23:49:55.451 | INFO | PRE_HQ : 59.9%, 1.7, 54.4ms, 0 | 20260505 23:49:55.451 | INFO | BAZWRITER : 61.8%, 3.5, 112.0ms, 0 | 20260505 23:49:55.451 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 23:49:55.451 | INFO | Uploaded 36GiB to GPU in 2.17165s (16.5773GiB/s) | 20260505 23:49:55.451 | INFO | Downloaded 38GiB to GPU in 2.23961s (16.9672GiB/s) | 20260505 23:50:00.592 | INFO | Analyzing chunk frames = [7283200,7283712) | 20260505 23:50:00.592 | INFO | MemInfo: { | 20260505 23:50:00.592 | INFO | "gpu" : " 17 GB", | 20260505 23:50:00.592 | INFO | "huge" : "", | 20260505 23:50:00.592 | INFO | "huge_shared" : " 27 GB", | 20260505 23:50:00.592 | INFO | "malloc" : " 14 GB", | 20260505 23:50:00.592 | INFO | "pinned" : " 29 GB" | 20260505 23:50:00.592 | INFO | } | 20260505 23:50:00.592 | INFO | Perf report for the last 5.14176 seconds: | 20260505 23:50:00.592 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:50:00.592 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:00.592 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:00.592 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:00.592 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 23:50:00.592 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:00.592 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:00.592 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260505 23:50:00.592 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:00.592 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:00.593 | INFO | Uploaded 36GiB to GPU in 2.11158s (17.0489GiB/s) | 20260505 23:50:00.593 | INFO | Downloaded 34GiB to GPU in 2.04273s (16.6444GiB/s) | 20260505 23:50:05.724 | INFO | Analyzing chunk frames = [7283712,7284224) | 20260505 23:50:05.724 | INFO | MemInfo: { | 20260505 23:50:05.724 | INFO | "gpu" : " 17 GB", | 20260505 23:50:05.724 | INFO | "huge" : "", | 20260505 23:50:05.724 | INFO | "huge_shared" : " 27 GB", | 20260505 23:50:05.724 | INFO | "malloc" : " 14 GB", | 20260505 23:50:05.724 | INFO | "pinned" : " 29 GB" | 20260505 23:50:05.724 | INFO | } | 20260505 23:50:05.724 | INFO | Perf report for the last 5.13164 seconds: | 20260505 23:50:05.724 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:50:05.724 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:05.724 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:05.724 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:05.724 | INFO | ANALYSIS : 61.4%, 3.0, 95.4ms, 0 | 20260505 23:50:05.724 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:05.724 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:05.724 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:50:05.724 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:05.724 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:05.724 | INFO | Uploaded 36GiB to GPU in 2.08167s (17.2938GiB/s) | 20260505 23:50:05.724 | INFO | Downloaded 34GiB to GPU in 2.0113s (16.9045GiB/s) | 20260505 23:50:10.861 | INFO | Analyzing chunk frames = [7284224,7284736) | 20260505 23:50:10.861 | INFO | MemInfo: { | 20260505 23:50:10.861 | INFO | "gpu" : " 17 GB", | 20260505 23:50:10.861 | INFO | "huge" : "", | 20260505 23:50:10.861 | INFO | "huge_shared" : " 27 GB", | 20260505 23:50:10.861 | INFO | "malloc" : " 14 GB", | 20260505 23:50:10.861 | INFO | "pinned" : " 29 GB" | 20260505 23:50:10.861 | INFO | } | 20260505 23:50:10.861 | INFO | Perf report for the last 5.13734 seconds: | 20260505 23:50:10.861 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:50:10.861 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:10.861 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:10.861 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:10.861 | INFO | ANALYSIS : 61.9%, 3.0, 96.1ms, 0 | 20260505 23:50:10.861 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:10.861 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:10.861 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:50:10.861 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:10.861 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:10.862 | INFO | Uploaded 36GiB to GPU in 2.15428s (16.7109GiB/s) | 20260505 23:50:10.862 | INFO | Downloaded 34GiB to GPU in 2.03461s (16.7108GiB/s) | 20260505 23:50:16.006 | INFO | Analyzing chunk frames = [7284736,7285248) | 20260505 23:50:16.006 | INFO | MemInfo: { | 20260505 23:50:16.006 | INFO | "gpu" : " 17 GB", | 20260505 23:50:16.006 | INFO | "huge" : "", | 20260505 23:50:16.006 | INFO | "huge_shared" : " 27 GB", | 20260505 23:50:16.006 | INFO | "malloc" : " 14 GB", | 20260505 23:50:16.006 | INFO | "pinned" : " 29 GB" | 20260505 23:50:16.006 | INFO | } | 20260505 23:50:16.006 | INFO | Perf report for the last 5.14489 seconds: | 20260505 23:50:16.006 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:50:16.006 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:16.006 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:16.006 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:16.006 | INFO | ANALYSIS : 61.4%, 3.0, 96.0ms, 0 | 20260505 23:50:16.006 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:16.006 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:16.006 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260505 23:50:16.006 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:16.006 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:16.007 | INFO | Uploaded 36GiB to GPU in 2.15048s (16.7404GiB/s) | 20260505 23:50:16.007 | INFO | Downloaded 34GiB to GPU in 2.06785s (16.4422GiB/s) | 20260505 23:50:21.150 | INFO | Analyzing chunk frames = [7285248,7285760) | 20260505 23:50:21.150 | INFO | MemInfo: { | 20260505 23:50:21.150 | INFO | "gpu" : " 17 GB", | 20260505 23:50:21.150 | INFO | "huge" : "", | 20260505 23:50:21.150 | INFO | "huge_shared" : " 27 GB", | 20260505 23:50:21.150 | INFO | "malloc" : " 14 GB", | 20260505 23:50:21.150 | INFO | "pinned" : " 29 GB" | 20260505 23:50:21.150 | INFO | } | 20260505 23:50:21.150 | INFO | Perf report for the last 5.14373 seconds: | 20260505 23:50:21.150 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:50:21.150 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:21.150 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:21.150 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:21.150 | INFO | ANALYSIS : 61.3%, 3.0, 95.8ms, 0 | 20260505 23:50:21.150 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:21.150 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:21.150 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:50:21.150 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:21.150 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:21.150 | INFO | Uploaded 36GiB to GPU in 2.06818s (17.4066GiB/s) | 20260505 23:50:21.150 | INFO | Downloaded 34GiB to GPU in 2.03615s (16.6981GiB/s) | 20260505 23:50:26.288 | INFO | Analyzing chunk frames = [7285760,7286272) | 20260505 23:50:26.288 | INFO | MemInfo: { | 20260505 23:50:26.288 | INFO | "gpu" : " 17 GB", | 20260505 23:50:26.288 | INFO | "huge" : "", | 20260505 23:50:26.288 | INFO | "huge_shared" : " 27 GB", | 20260505 23:50:26.288 | INFO | "malloc" : " 14 GB", | 20260505 23:50:26.288 | INFO | "pinned" : " 29 GB" | 20260505 23:50:26.288 | INFO | } | 20260505 23:50:26.288 | INFO | Perf report for the last 5.13844 seconds: | 20260505 23:50:26.288 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:50:26.288 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:26.288 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:26.288 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:26.288 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260505 23:50:26.289 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:26.289 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:26.289 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 23:50:26.289 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:26.289 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:26.289 | INFO | Uploaded 36GiB to GPU in 2.10793s (17.0784GiB/s) | 20260505 23:50:26.289 | INFO | Downloaded 34GiB to GPU in 2.07451s (16.3894GiB/s) | 20260505 23:50:31.425 | INFO | Analyzing chunk frames = [7286272,7286784) | 20260505 23:50:31.425 | INFO | MemInfo: { | 20260505 23:50:31.425 | INFO | "gpu" : " 17 GB", | 20260505 23:50:31.425 | INFO | "huge" : "", | 20260505 23:50:31.425 | INFO | "huge_shared" : " 27 GB", | 20260505 23:50:31.425 | INFO | "malloc" : " 14 GB", | 20260505 23:50:31.425 | INFO | "pinned" : " 29 GB" | 20260505 23:50:31.425 | INFO | } | 20260505 23:50:31.425 | INFO | Perf report for the last 5.13661 seconds: | 20260505 23:50:31.425 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:50:31.425 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:31.425 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:31.425 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:31.425 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260505 23:50:31.425 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:31.425 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:31.425 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 23:50:31.425 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:31.425 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:31.425 | INFO | Uploaded 36GiB to GPU in 2.13229s (16.8832GiB/s) | 20260505 23:50:31.425 | INFO | Downloaded 34GiB to GPU in 2.01761s (16.8516GiB/s) | 20260505 23:50:31.552 | INFO | Starting a round of Baz IO | 20260505 23:50:34.701 | INFO | Populating RTMetrics report | 20260505 23:50:34.702 | INFO | RTMetrics report populated | 20260505 23:50:34.702 | INFO | Opening tmp RTMetrics json file | 20260505 23:50:34.702 | INFO | Serializing RTMetrics to json | 20260505 23:50:34.703 | INFO | Serialization done, swapping tmp file now... | 20260505 23:50:34.704 | INFO | RTMetrics done | 20260505 23:50:34.779 | INFO | Finished a round of Baz IO: Wrote 1.879 GiB in 3.226 seconds | 20260505 23:50:34.779 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.40% / 0.00% / 78.59% | 20260505 23:50:34.779 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.100299% / 3.97364e-06% / 4.76837e-05% | 20260505 23:50:34.779 | INFO | Baz IO overall: | 20260505 23:50:34.779 | INFO | BytesWritten : 4156677906923 | 20260505 23:50:34.779 | INFO | NumEvents : 1957454219818 | 20260505 23:50:36.559 | INFO | Analyzing chunk frames = [7286784,7287296) | 20260505 23:50:36.560 | INFO | MemInfo: { | 20260505 23:50:36.560 | INFO | "gpu" : " 17 GB", | 20260505 23:50:36.560 | INFO | "huge" : "", | 20260505 23:50:36.560 | INFO | "huge_shared" : " 27 GB", | 20260505 23:50:36.560 | INFO | "malloc" : " 14 GB", | 20260505 23:50:36.560 | INFO | "pinned" : " 29 GB" | 20260505 23:50:36.560 | INFO | } | 20260505 23:50:36.560 | INFO | Perf report for the last 5.13463 seconds: | 20260505 23:50:36.560 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:50:36.560 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:36.560 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:36.560 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:36.560 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260505 23:50:36.560 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:36.560 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260505 23:50:36.560 | INFO | PRE_HQ : 60.0%, 1.7, 54.1ms, 0 | 20260505 23:50:36.560 | INFO | BAZWRITER : 62.0%, 3.5, 111.6ms, 0 | 20260505 23:50:36.560 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 23:50:36.560 | INFO | Uploaded 36GiB to GPU in 2.17157s (16.5779GiB/s) | 20260505 23:50:36.560 | INFO | Downloaded 38GiB to GPU in 2.24213s (16.9482GiB/s) | 20260505 23:50:41.705 | INFO | Analyzing chunk frames = [7287296,7287808) | 20260505 23:50:41.705 | INFO | MemInfo: { | 20260505 23:50:41.705 | INFO | "gpu" : " 17 GB", | 20260505 23:50:41.705 | INFO | "huge" : "", | 20260505 23:50:41.705 | INFO | "huge_shared" : " 27 GB", | 20260505 23:50:41.705 | INFO | "malloc" : " 14 GB", | 20260505 23:50:41.705 | INFO | "pinned" : " 29 GB" | 20260505 23:50:41.705 | INFO | } | 20260505 23:50:41.705 | INFO | Perf report for the last 5.14528 seconds: | 20260505 23:50:41.705 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:50:41.705 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:41.705 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:41.705 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:41.705 | INFO | ANALYSIS : 61.8%, 3.0, 96.6ms, 0 | 20260505 23:50:41.705 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:41.705 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:41.705 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260505 23:50:41.705 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:41.705 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:41.705 | INFO | Uploaded 36GiB to GPU in 2.10022s (17.141GiB/s) | 20260505 23:50:41.705 | INFO | Downloaded 34GiB to GPU in 2.03209s (16.7316GiB/s) | 20260505 23:50:46.843 | INFO | Analyzing chunk frames = [7287808,7288320) | 20260505 23:50:46.843 | INFO | MemInfo: { | 20260505 23:50:46.843 | INFO | "gpu" : " 17 GB", | 20260505 23:50:46.843 | INFO | "huge" : "", | 20260505 23:50:46.843 | INFO | "huge_shared" : " 27 GB", | 20260505 23:50:46.843 | INFO | "malloc" : " 14 GB", | 20260505 23:50:46.843 | INFO | "pinned" : " 29 GB" | 20260505 23:50:46.843 | INFO | } | 20260505 23:50:46.844 | INFO | Perf report for the last 5.13865 seconds: | 20260505 23:50:46.844 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:50:46.844 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:46.844 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:46.844 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:46.844 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260505 23:50:46.844 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:46.844 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:46.844 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260505 23:50:46.844 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:46.844 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:46.844 | INFO | Uploaded 36GiB to GPU in 2.08453s (17.2701GiB/s) | 20260505 23:50:46.844 | INFO | Downloaded 34GiB to GPU in 2.01534s (16.8706GiB/s) | 20260505 23:50:51.982 | INFO | Analyzing chunk frames = [7288320,7288832) | 20260505 23:50:51.982 | INFO | MemInfo: { | 20260505 23:50:51.982 | INFO | "gpu" : " 17 GB", | 20260505 23:50:51.982 | INFO | "huge" : "", | 20260505 23:50:51.982 | INFO | "huge_shared" : " 27 GB", | 20260505 23:50:51.982 | INFO | "malloc" : " 14 GB", | 20260505 23:50:51.982 | INFO | "pinned" : " 29 GB" | 20260505 23:50:51.982 | INFO | } | 20260505 23:50:51.982 | INFO | Perf report for the last 5.13801 seconds: | 20260505 23:50:51.982 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:50:51.982 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:51.982 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:51.982 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:51.982 | INFO | ANALYSIS : 61.7%, 3.0, 95.8ms, 0 | 20260505 23:50:51.982 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:51.982 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:51.982 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:50:51.982 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:51.982 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:51.982 | INFO | Uploaded 36GiB to GPU in 2.1368s (16.8476GiB/s) | 20260505 23:50:51.982 | INFO | Downloaded 34GiB to GPU in 2.01838s (16.8452GiB/s) | 20260505 23:50:57.118 | INFO | Analyzing chunk frames = [7288832,7289344) | 20260505 23:50:57.118 | INFO | MemInfo: { | 20260505 23:50:57.118 | INFO | "gpu" : " 17 GB", | 20260505 23:50:57.118 | INFO | "huge" : "", | 20260505 23:50:57.118 | INFO | "huge_shared" : " 27 GB", | 20260505 23:50:57.118 | INFO | "malloc" : " 14 GB", | 20260505 23:50:57.118 | INFO | "pinned" : " 29 GB" | 20260505 23:50:57.118 | INFO | } | 20260505 23:50:57.118 | INFO | Perf report for the last 5.13674 seconds: | 20260505 23:50:57.118 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:50:57.118 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:57.118 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:57.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:57.118 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260505 23:50:57.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:57.119 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:57.119 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:50:57.119 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:57.119 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:50:57.119 | INFO | Uploaded 36GiB to GPU in 2.09277s (17.2021GiB/s) | 20260505 23:50:57.119 | INFO | Downloaded 34GiB to GPU in 2.04024s (16.6647GiB/s) | 20260505 23:51:02.257 | INFO | Analyzing chunk frames = [7289344,7289856) | 20260505 23:51:02.257 | INFO | MemInfo: { | 20260505 23:51:02.257 | INFO | "gpu" : " 17 GB", | 20260505 23:51:02.257 | INFO | "huge" : "", | 20260505 23:51:02.257 | INFO | "huge_shared" : " 27 GB", | 20260505 23:51:02.257 | INFO | "malloc" : " 14 GB", | 20260505 23:51:02.257 | INFO | "pinned" : " 29 GB" | 20260505 23:51:02.257 | INFO | } | 20260505 23:51:02.257 | INFO | Perf report for the last 5.13837 seconds: | 20260505 23:51:02.257 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:51:02.257 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:02.257 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:02.257 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:02.257 | INFO | ANALYSIS : 61.5%, 3.0, 96.0ms, 0 | 20260505 23:51:02.257 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:02.257 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:02.257 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:51:02.257 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:02.257 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:02.257 | INFO | Uploaded 36GiB to GPU in 2.08552s (17.2619GiB/s) | 20260505 23:51:02.257 | INFO | Downloaded 34GiB to GPU in 2.04386s (16.6352GiB/s) | 20260505 23:51:07.402 | INFO | Analyzing chunk frames = [7289856,7290368) | 20260505 23:51:07.402 | INFO | MemInfo: { | 20260505 23:51:07.402 | INFO | "gpu" : " 17 GB", | 20260505 23:51:07.402 | INFO | "huge" : "", | 20260505 23:51:07.402 | INFO | "huge_shared" : " 27 GB", | 20260505 23:51:07.402 | INFO | "malloc" : " 14 GB", | 20260505 23:51:07.402 | INFO | "pinned" : " 29 GB" | 20260505 23:51:07.402 | INFO | } | 20260505 23:51:07.402 | INFO | Perf report for the last 5.14486 seconds: | 20260505 23:51:07.402 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:51:07.402 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:07.402 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:07.402 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:07.402 | INFO | ANALYSIS : 62.5%, 3.0, 97.7ms, 0 | 20260505 23:51:07.402 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:07.402 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:07.402 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260505 23:51:07.402 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:07.402 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:07.402 | INFO | Uploaded 36GiB to GPU in 2.11762s (17.0002GiB/s) | 20260505 23:51:07.402 | INFO | Downloaded 34GiB to GPU in 2.06977s (16.4269GiB/s) | 20260505 23:51:12.539 | INFO | Analyzing chunk frames = [7290368,7290880) | 20260505 23:51:12.539 | INFO | MemInfo: { | 20260505 23:51:12.539 | INFO | "gpu" : " 17 GB", | 20260505 23:51:12.539 | INFO | "huge" : "", | 20260505 23:51:12.540 | INFO | "huge_shared" : " 27 GB", | 20260505 23:51:12.540 | INFO | "malloc" : " 14 GB", | 20260505 23:51:12.540 | INFO | "pinned" : " 29 GB" | 20260505 23:51:12.540 | INFO | } | 20260505 23:51:12.540 | INFO | Perf report for the last 5.13792 seconds: | 20260505 23:51:12.540 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:51:12.540 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:12.540 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:12.540 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:12.540 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260505 23:51:12.540 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:12.540 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:12.540 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:51:12.540 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:12.540 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:12.540 | INFO | Uploaded 36GiB to GPU in 2.13698s (16.8462GiB/s) | 20260505 23:51:12.540 | INFO | Downloaded 34GiB to GPU in 2.02749s (16.7695GiB/s) | 20260505 23:51:12.540 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 23:51:12.540 | INFO | Total mem usage: 15.3 GiB | 20260505 23:51:12.540 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 23:51:12.540 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.12%) | 20260505 23:51:12.540 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 23:51:12.540 | INFO | High Water Breakdown: | 20260505 23:51:12.540 | INFO | 60.9% : 8.55 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 23:51:12.540 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 23:51:12.540 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 23:51:12.540 | INFO | ------------------------------------------ | 20260505 23:51:12.540 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:51:12.540 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 23:51:12.540 | INFO | Total mem usage: 26.5 GiB | 20260505 23:51:12.540 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 23:51:12.540 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 23:51:12.540 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 23:51:12.540 | INFO | High Water Breakdown: | 20260505 23:51:12.540 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:51:12.540 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:51:12.540 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:51:12.540 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:51:12.540 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:51:12.540 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:51:12.540 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 23:51:12.540 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 23:51:12.540 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:51:12.540 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 23:51:12.540 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 23:51:12.540 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 23:51:12.540 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 23:51:12.540 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 23:51:12.541 | INFO | ------------------------------------------ | 20260505 23:51:12.541 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:51:12.541 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 23:51:12.541 | INFO | Total mem usage: 25.6 GiB | 20260505 23:51:12.541 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 23:51:12.541 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 23:51:12.541 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 23:51:12.541 | INFO | High Water Breakdown: | 20260505 23:51:12.541 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 23:51:12.541 | INFO | ------------------------------------------ | 20260505 23:51:12.541 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:51:12.541 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 23:51:12.541 | INFO | Total mem usage: 15.7 GiB | 20260505 23:51:12.541 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 23:51:12.541 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 23:51:12.541 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 23:51:12.541 | INFO | High Water Breakdown: | 20260505 23:51:12.541 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 23:51:12.541 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 23:51:12.541 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 23:51:12.541 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 23:51:12.541 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 23:51:12.541 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 23:51:12.541 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 23:51:12.541 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 23:51:12.541 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 23:51:12.541 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:51:12.541 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 23:51:12.541 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:51:12.541 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 23:51:12.541 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 23:51:12.541 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 23:51:12.541 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 23:51:12.541 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:51:12.541 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 23:51:12.541 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:51:12.541 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:51:12.541 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:51:12.541 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 23:51:12.541 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 23:51:12.541 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:51:12.541 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 23:51:12.541 | INFO | ------------------------------------------ | 20260505 23:51:12.541 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:51:12.541 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 23:51:12.541 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:51:12.541 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:51:12.541 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.39e+01, 1.33e+01 / 7.23e-01 | 20260505 23:51:12.541 | INFO | PerfReport: 0.29%, AggregateStats -- 9.25e-02 / 1.06e-01, 1.07e-02 / 5.82e-03 | 20260505 23:51:12.541 | INFO | PerfReport: 11.81%, Baseline -- 3.70e+00 / 4.48e+00, 5.65e-01 / 2.18e-01 | 20260505 23:51:12.541 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.51e+00, 1.89e-01 / 9.38e-02 | 20260505 23:51:12.541 | INFO | PerfReport: 15.95%, DME -- 4.00e+01 / 4.38e+01, 3.68e+00 / 9.10e-01 | 20260505 23:51:12.541 | INFO | PerfReport: 57.40%, FrameLabeling -- 1.80e+01 / 1.96e+01, 1.54e+00 / 1.00e+00 | 20260505 23:51:12.541 | INFO | PerfReport: 7.09%, PulseAccumulating -- 2.22e+00 / 2.72e+00, 4.23e-01 / 2.18e-01 | 20260505 23:51:12.541 | INFO | PerfReport: 3.09%, Metrics -- 9.69e-01 / 1.25e+00, 2.63e-01 / 1.35e-01 | 20260505 23:51:12.541 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.36e-01 / 3.88e-01, 5.73e-02 / 3.11e-02 | 20260505 23:51:12.541 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 23:51:12.541 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:51:12.541 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:51:12.541 | INFO | PerfReport: Full Execution -- 9.85e+01 / -nan, 2.46e+01 / 0.00e+00 | 20260505 23:51:12.541 | INFO | PerfReport: 22.42%, Upload -- 2.21e+01 / 0.00e+00, 7.62e+00 / 0.00e+00 | 20260505 23:51:12.541 | INFO | PerfReport: 21.83%, Download -- 2.15e+01 / 0.00e+00, 7.63e+00 / 0.00e+00 | 20260505 23:51:12.541 | INFO | PerfReport: 31.84%, Compute -- 3.13e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260505 23:51:12.541 | INFO | PerfReport: 2.15%, LockUpload -- 2.12e+00 / 0.00e+00, 7.86e+00 / 0.00e+00 | 20260505 23:51:12.541 | INFO | PerfReport: 1.25%, LockDownload -- 1.23e+00 / 0.00e+00, 2.75e+00 / 0.00e+00 | 20260505 23:51:12.541 | INFO | PerfReport: 20.50%, LockCompute -- 2.02e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260505 23:51:12.541 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.78e-02 / 0.00e+00 | 20260505 23:51:12.659 | INFO | Starting a round of Baz IO | 20260505 23:51:15.811 | INFO | Populating RTMetrics report | 20260505 23:51:15.811 | INFO | RTMetrics report populated | 20260505 23:51:15.811 | INFO | Opening tmp RTMetrics json file | 20260505 23:51:15.812 | INFO | Serializing RTMetrics to json | 20260505 23:51:15.813 | INFO | Serialization done, swapping tmp file now... | 20260505 23:51:15.814 | INFO | RTMetrics done | 20260505 23:51:15.891 | INFO | Finished a round of Baz IO: Wrote 1.879 GiB in 3.232 seconds | 20260505 23:51:15.891 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.40% / 0.00% / 78.60% | 20260505 23:51:15.891 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.100299% / 0% / 6.35783e-05% | 20260505 23:51:15.891 | INFO | Baz IO overall: | 20260505 23:51:15.891 | INFO | BytesWritten : 4158695900146 | 20260505 23:51:15.891 | INFO | NumEvents : 1958378885669 | 20260505 23:51:17.673 | INFO | Analyzing chunk frames = [7290880,7291392) | 20260505 23:51:17.673 | INFO | MemInfo: { | 20260505 23:51:17.673 | INFO | "gpu" : " 17 GB", | 20260505 23:51:17.673 | INFO | "huge" : "", | 20260505 23:51:17.673 | INFO | "huge_shared" : " 27 GB", | 20260505 23:51:17.673 | INFO | "malloc" : " 14 GB", | 20260505 23:51:17.673 | INFO | "pinned" : " 29 GB" | 20260505 23:51:17.673 | INFO | } | 20260505 23:51:17.673 | INFO | Perf report for the last 5.13358 seconds: | 20260505 23:51:17.673 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:51:17.673 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:17.673 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:17.673 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:17.673 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 23:51:17.673 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:17.673 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260505 23:51:17.673 | INFO | PRE_HQ : 60.1%, 1.7, 54.5ms, 0 | 20260505 23:51:17.673 | INFO | BAZWRITER : 62.1%, 3.6, 113.4ms, 0 | 20260505 23:51:17.673 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 23:51:17.673 | INFO | Uploaded 36GiB to GPU in 2.17515s (16.5506GiB/s) | 20260505 23:51:17.673 | INFO | Downloaded 38GiB to GPU in 2.24412s (16.9331GiB/s) | 20260505 23:51:22.812 | INFO | Analyzing chunk frames = [7291392,7291904) | 20260505 23:51:22.812 | INFO | MemInfo: { | 20260505 23:51:22.812 | INFO | "gpu" : " 17 GB", | 20260505 23:51:22.812 | INFO | "huge" : "", | 20260505 23:51:22.812 | INFO | "huge_shared" : " 27 GB", | 20260505 23:51:22.812 | INFO | "malloc" : " 14 GB", | 20260505 23:51:22.813 | INFO | "pinned" : " 29 GB" | 20260505 23:51:22.813 | INFO | } | 20260505 23:51:22.813 | INFO | Perf report for the last 5.13939 seconds: | 20260505 23:51:22.813 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:51:22.813 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:22.813 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:22.813 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:22.813 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 23:51:22.813 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:22.813 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:22.813 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:51:22.813 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:22.813 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:22.813 | INFO | Uploaded 36GiB to GPU in 2.08552s (17.2619GiB/s) | 20260505 23:51:22.813 | INFO | Downloaded 34GiB to GPU in 2.02768s (16.7679GiB/s) | 20260505 23:51:27.959 | INFO | Analyzing chunk frames = [7291904,7292416) | 20260505 23:51:27.959 | INFO | MemInfo: { | 20260505 23:51:27.959 | INFO | "gpu" : " 17 GB", | 20260505 23:51:27.959 | INFO | "huge" : "", | 20260505 23:51:27.959 | INFO | "huge_shared" : " 27 GB", | 20260505 23:51:27.959 | INFO | "malloc" : " 14 GB", | 20260505 23:51:27.959 | INFO | "pinned" : " 29 GB" | 20260505 23:51:27.959 | INFO | } | 20260505 23:51:27.959 | INFO | Perf report for the last 5.14668 seconds: | 20260505 23:51:27.959 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:51:27.959 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:27.959 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:27.959 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:27.959 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260505 23:51:27.959 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:27.959 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:27.959 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 23:51:27.959 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:27.959 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:27.960 | INFO | Uploaded 36GiB to GPU in 2.13395s (16.8701GiB/s) | 20260505 23:51:27.960 | INFO | Downloaded 34GiB to GPU in 2.03256s (16.7276GiB/s) | 20260505 23:51:33.107 | INFO | Analyzing chunk frames = [7292416,7292928) | 20260505 23:51:33.107 | INFO | MemInfo: { | 20260505 23:51:33.107 | INFO | "gpu" : " 17 GB", | 20260505 23:51:33.107 | INFO | "huge" : "", | 20260505 23:51:33.107 | INFO | "huge_shared" : " 27 GB", | 20260505 23:51:33.107 | INFO | "malloc" : " 14 GB", | 20260505 23:51:33.107 | INFO | "pinned" : " 29 GB" | 20260505 23:51:33.107 | INFO | } | 20260505 23:51:33.107 | INFO | Perf report for the last 5.14752 seconds: | 20260505 23:51:33.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:51:33.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:33.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:33.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:33.107 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260505 23:51:33.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:33.107 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:33.107 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:51:33.107 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:33.107 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:33.107 | INFO | Uploaded 36GiB to GPU in 2.13874s (16.8324GiB/s) | 20260505 23:51:33.107 | INFO | Downloaded 34GiB to GPU in 2.03062s (16.7437GiB/s) | 20260505 23:51:38.239 | INFO | Analyzing chunk frames = [7292928,7293440) | 20260505 23:51:38.239 | INFO | MemInfo: { | 20260505 23:51:38.239 | INFO | "gpu" : " 17 GB", | 20260505 23:51:38.239 | INFO | "huge" : "", | 20260505 23:51:38.239 | INFO | "huge_shared" : " 27 GB", | 20260505 23:51:38.239 | INFO | "malloc" : " 14 GB", | 20260505 23:51:38.239 | INFO | "pinned" : " 29 GB" | 20260505 23:51:38.239 | INFO | } | 20260505 23:51:38.239 | INFO | Perf report for the last 5.13228 seconds: | 20260505 23:51:38.239 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:51:38.239 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:38.239 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:38.239 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:38.239 | INFO | ANALYSIS : 60.9%, 3.0, 95.0ms, 0 | 20260505 23:51:38.239 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:38.239 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:38.239 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 23:51:38.239 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:38.239 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:38.239 | INFO | Uploaded 36GiB to GPU in 2.07178s (17.3764GiB/s) | 20260505 23:51:38.239 | INFO | Downloaded 34GiB to GPU in 2.03416s (16.7145GiB/s) | 20260505 23:51:43.370 | INFO | Analyzing chunk frames = [7293440,7293952) | 20260505 23:51:43.370 | INFO | MemInfo: { | 20260505 23:51:43.370 | INFO | "gpu" : " 17 GB", | 20260505 23:51:43.370 | INFO | "huge" : "", | 20260505 23:51:43.370 | INFO | "huge_shared" : " 27 GB", | 20260505 23:51:43.370 | INFO | "malloc" : " 14 GB", | 20260505 23:51:43.370 | INFO | "pinned" : " 29 GB" | 20260505 23:51:43.370 | INFO | } | 20260505 23:51:43.370 | INFO | Perf report for the last 5.13095 seconds: | 20260505 23:51:43.370 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:51:43.370 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:43.370 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:43.370 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:43.370 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260505 23:51:43.370 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:43.370 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:43.370 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 23:51:43.370 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:43.370 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:43.370 | INFO | Uploaded 36GiB to GPU in 2.07236s (17.3715GiB/s) | 20260505 23:51:43.370 | INFO | Downloaded 34GiB to GPU in 2.03957s (16.6702GiB/s) | 20260505 23:51:48.518 | INFO | Analyzing chunk frames = [7293952,7294464) | 20260505 23:51:48.518 | INFO | MemInfo: { | 20260505 23:51:48.518 | INFO | "gpu" : " 17 GB", | 20260505 23:51:48.518 | INFO | "huge" : "", | 20260505 23:51:48.518 | INFO | "huge_shared" : " 27 GB", | 20260505 23:51:48.518 | INFO | "malloc" : " 14 GB", | 20260505 23:51:48.518 | INFO | "pinned" : " 29 GB" | 20260505 23:51:48.518 | INFO | } | 20260505 23:51:48.518 | INFO | Perf report for the last 5.14829 seconds: | 20260505 23:51:48.518 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:51:48.518 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:48.518 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:48.518 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:48.518 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260505 23:51:48.518 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:48.518 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:48.519 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260505 23:51:48.519 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:48.519 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:48.519 | INFO | Uploaded 36GiB to GPU in 2.0914s (17.2134GiB/s) | 20260505 23:51:48.519 | INFO | Downloaded 34GiB to GPU in 2.07336s (16.3985GiB/s) | 20260505 23:51:53.659 | INFO | Analyzing chunk frames = [7294464,7294976) | 20260505 23:51:53.659 | INFO | MemInfo: { | 20260505 23:51:53.659 | INFO | "gpu" : " 17 GB", | 20260505 23:51:53.659 | INFO | "huge" : "", | 20260505 23:51:53.659 | INFO | "huge_shared" : " 27 GB", | 20260505 23:51:53.659 | INFO | "malloc" : " 14 GB", | 20260505 23:51:53.659 | INFO | "pinned" : " 29 GB" | 20260505 23:51:53.659 | INFO | } | 20260505 23:51:53.659 | INFO | Perf report for the last 5.14051 seconds: | 20260505 23:51:53.659 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:51:53.659 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:53.659 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:53.659 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:53.659 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260505 23:51:53.659 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:53.659 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:53.659 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 23:51:53.659 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:53.659 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:53.659 | INFO | Uploaded 36GiB to GPU in 2.13132s (16.8909GiB/s) | 20260505 23:51:53.659 | INFO | Downloaded 34GiB to GPU in 2.02048s (16.8277GiB/s) | 20260505 23:51:53.776 | INFO | Starting a round of Baz IO | 20260505 23:51:56.930 | INFO | Populating RTMetrics report | 20260505 23:51:56.930 | INFO | RTMetrics report populated | 20260505 23:51:56.930 | INFO | Opening tmp RTMetrics json file | 20260505 23:51:56.932 | INFO | Serializing RTMetrics to json | 20260505 23:51:56.934 | INFO | Serialization done, swapping tmp file now... | 20260505 23:51:56.936 | INFO | RTMetrics done | 20260505 23:51:57.009 | INFO | Finished a round of Baz IO: Wrote 1.879 GiB in 3.233 seconds | 20260505 23:51:57.010 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.40% / 0.00% / 78.60% | 20260505 23:51:57.010 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.100764% / 0% / 6.75519e-05% | 20260505 23:51:57.010 | INFO | Baz IO overall: | 20260505 23:51:57.010 | INFO | BytesWritten : 4160713703067 | 20260505 23:51:57.010 | INFO | NumEvents : 1959303623516 | 20260505 23:51:58.796 | INFO | Analyzing chunk frames = [7294976,7295488) | 20260505 23:51:58.796 | INFO | MemInfo: { | 20260505 23:51:58.796 | INFO | "gpu" : " 17 GB", | 20260505 23:51:58.796 | INFO | "huge" : "", | 20260505 23:51:58.796 | INFO | "huge_shared" : " 27 GB", | 20260505 23:51:58.796 | INFO | "malloc" : " 14 GB", | 20260505 23:51:58.796 | INFO | "pinned" : " 29 GB" | 20260505 23:51:58.796 | INFO | } | 20260505 23:51:58.796 | INFO | Perf report for the last 5.13741 seconds: | 20260505 23:51:58.796 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:51:58.796 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:58.796 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:58.796 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:58.796 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 23:51:58.796 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:51:58.796 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260505 23:51:58.796 | INFO | PRE_HQ : 59.8%, 1.7, 53.9ms, 0 | 20260505 23:51:58.797 | INFO | BAZWRITER : 62.0%, 3.5, 112.2ms, 0 | 20260505 23:51:58.797 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 23:51:58.797 | INFO | Uploaded 36GiB to GPU in 2.18661s (16.4639GiB/s) | 20260505 23:51:58.797 | INFO | Downloaded 38GiB to GPU in 2.23869s (16.9742GiB/s) | 20260505 23:52:03.933 | INFO | Analyzing chunk frames = [7295488,7296000) | 20260505 23:52:03.933 | INFO | MemInfo: { | 20260505 23:52:03.933 | INFO | "gpu" : " 17 GB", | 20260505 23:52:03.933 | INFO | "huge" : "", | 20260505 23:52:03.933 | INFO | "huge_shared" : " 27 GB", | 20260505 23:52:03.933 | INFO | "malloc" : " 14 GB", | 20260505 23:52:03.933 | INFO | "pinned" : " 29 GB" | 20260505 23:52:03.933 | INFO | } | 20260505 23:52:03.933 | INFO | Perf report for the last 5.13656 seconds: | 20260505 23:52:03.933 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:52:03.933 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:03.933 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:03.933 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:03.933 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260505 23:52:03.933 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:03.933 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:03.933 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260505 23:52:03.933 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:03.933 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:03.933 | INFO | Uploaded 36GiB to GPU in 2.09962s (17.146GiB/s) | 20260505 23:52:03.933 | INFO | Downloaded 34GiB to GPU in 2.03869s (16.6773GiB/s) | 20260505 23:52:09.069 | INFO | Analyzing chunk frames = [7296000,7296512) | 20260505 23:52:09.069 | INFO | MemInfo: { | 20260505 23:52:09.069 | INFO | "gpu" : " 17 GB", | 20260505 23:52:09.069 | INFO | "huge" : "", | 20260505 23:52:09.069 | INFO | "huge_shared" : " 27 GB", | 20260505 23:52:09.069 | INFO | "malloc" : " 14 GB", | 20260505 23:52:09.069 | INFO | "pinned" : " 29 GB" | 20260505 23:52:09.069 | INFO | } | 20260505 23:52:09.069 | INFO | Perf report for the last 5.13583 seconds: | 20260505 23:52:09.069 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:52:09.069 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:09.069 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:09.069 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:09.069 | INFO | ANALYSIS : 61.9%, 3.0, 96.2ms, 0 | 20260505 23:52:09.069 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:09.069 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:09.069 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:52:09.069 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:09.069 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:09.069 | INFO | Uploaded 36GiB to GPU in 2.1428s (16.8004GiB/s) | 20260505 23:52:09.069 | INFO | Downloaded 34GiB to GPU in 2.03699s (16.6913GiB/s) | 20260505 23:52:14.211 | INFO | Analyzing chunk frames = [7296512,7297024) | 20260505 23:52:14.211 | INFO | MemInfo: { | 20260505 23:52:14.211 | INFO | "gpu" : " 17 GB", | 20260505 23:52:14.211 | INFO | "huge" : "", | 20260505 23:52:14.211 | INFO | "huge_shared" : " 27 GB", | 20260505 23:52:14.211 | INFO | "malloc" : " 14 GB", | 20260505 23:52:14.211 | INFO | "pinned" : " 29 GB" | 20260505 23:52:14.211 | INFO | } | 20260505 23:52:14.211 | INFO | Perf report for the last 5.14192 seconds: | 20260505 23:52:14.211 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:52:14.211 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:14.211 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:14.211 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:14.211 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260505 23:52:14.211 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:14.211 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:14.211 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:52:14.211 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:14.211 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:14.211 | INFO | Uploaded 36GiB to GPU in 2.1286s (16.9125GiB/s) | 20260505 23:52:14.211 | INFO | Downloaded 34GiB to GPU in 2.01712s (16.8557GiB/s) | 20260505 23:52:19.350 | INFO | Analyzing chunk frames = [7297024,7297536) | 20260505 23:52:19.350 | INFO | MemInfo: { | 20260505 23:52:19.350 | INFO | "gpu" : " 17 GB", | 20260505 23:52:19.350 | INFO | "huge" : "", | 20260505 23:52:19.350 | INFO | "huge_shared" : " 27 GB", | 20260505 23:52:19.350 | INFO | "malloc" : " 14 GB", | 20260505 23:52:19.350 | INFO | "pinned" : " 29 GB" | 20260505 23:52:19.350 | INFO | } | 20260505 23:52:19.350 | INFO | Perf report for the last 5.13896 seconds: | 20260505 23:52:19.350 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:52:19.350 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:19.350 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:19.350 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:19.350 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260505 23:52:19.350 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:19.350 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:19.350 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:52:19.350 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:19.350 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:19.350 | INFO | Uploaded 36GiB to GPU in 2.08682s (17.2511GiB/s) | 20260505 23:52:19.350 | INFO | Downloaded 34GiB to GPU in 2.04797s (16.6018GiB/s) | 20260505 23:52:24.487 | INFO | Analyzing chunk frames = [7297536,7298048) | 20260505 23:52:24.487 | INFO | MemInfo: { | 20260505 23:52:24.487 | INFO | "gpu" : " 17 GB", | 20260505 23:52:24.487 | INFO | "huge" : "", | 20260505 23:52:24.487 | INFO | "huge_shared" : " 27 GB", | 20260505 23:52:24.487 | INFO | "malloc" : " 14 GB", | 20260505 23:52:24.487 | INFO | "pinned" : " 29 GB" | 20260505 23:52:24.487 | INFO | } | 20260505 23:52:24.488 | INFO | Perf report for the last 5.13779 seconds: | 20260505 23:52:24.488 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:52:24.488 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:24.488 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:24.488 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:24.488 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260505 23:52:24.488 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:24.488 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:24.488 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260505 23:52:24.488 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:24.488 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:24.488 | INFO | Uploaded 36GiB to GPU in 2.07088s (17.3839GiB/s) | 20260505 23:52:24.488 | INFO | Downloaded 34GiB to GPU in 2.03643s (16.6959GiB/s) | 20260505 23:52:29.623 | INFO | Analyzing chunk frames = [7298048,7298560) | 20260505 23:52:29.623 | INFO | MemInfo: { | 20260505 23:52:29.623 | INFO | "gpu" : " 17 GB", | 20260505 23:52:29.623 | INFO | "huge" : "", | 20260505 23:52:29.623 | INFO | "huge_shared" : " 27 GB", | 20260505 23:52:29.623 | INFO | "malloc" : " 14 GB", | 20260505 23:52:29.624 | INFO | "pinned" : " 29 GB" | 20260505 23:52:29.624 | INFO | } | 20260505 23:52:29.624 | INFO | Perf report for the last 5.13603 seconds: | 20260505 23:52:29.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:52:29.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:29.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:29.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:29.624 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260505 23:52:29.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:29.624 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:29.624 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 23:52:29.624 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:29.624 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:29.624 | INFO | Uploaded 36GiB to GPU in 2.0721s (17.3737GiB/s) | 20260505 23:52:29.624 | INFO | Downloaded 34GiB to GPU in 2.06319s (16.4794GiB/s) | 20260505 23:52:34.768 | INFO | Analyzing chunk frames = [7298560,7299072) | 20260505 23:52:34.768 | INFO | MemInfo: { | 20260505 23:52:34.768 | INFO | "gpu" : " 17 GB", | 20260505 23:52:34.768 | INFO | "huge" : "", | 20260505 23:52:34.768 | INFO | "huge_shared" : " 27 GB", | 20260505 23:52:34.768 | INFO | "malloc" : " 14 GB", | 20260505 23:52:34.768 | INFO | "pinned" : " 29 GB" | 20260505 23:52:34.768 | INFO | } | 20260505 23:52:34.768 | INFO | Perf report for the last 5.14421 seconds: | 20260505 23:52:34.768 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:52:34.768 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:34.768 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:34.768 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:34.768 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260505 23:52:34.768 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:34.768 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:34.768 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 23:52:34.768 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:34.768 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:34.768 | INFO | Uploaded 36GiB to GPU in 2.1014s (17.1315GiB/s) | 20260505 23:52:34.768 | INFO | Downloaded 34GiB to GPU in 2.00849s (16.9282GiB/s) | 20260505 23:52:34.768 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 23:52:34.768 | INFO | Total mem usage: 15.3 GiB | 20260505 23:52:34.768 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 23:52:34.768 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.12%) | 20260505 23:52:34.768 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 23:52:34.768 | INFO | High Water Breakdown: | 20260505 23:52:34.768 | INFO | 60.9% : 8.55 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 23:52:34.768 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 23:52:34.768 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 23:52:34.768 | INFO | ------------------------------------------ | 20260505 23:52:34.768 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:52:34.768 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 23:52:34.768 | INFO | Total mem usage: 26.5 GiB | 20260505 23:52:34.768 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 23:52:34.768 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 23:52:34.768 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 23:52:34.768 | INFO | High Water Breakdown: | 20260505 23:52:34.768 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:52:34.768 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:52:34.768 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:52:34.768 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:52:34.768 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:52:34.768 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:52:34.769 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 23:52:34.769 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 23:52:34.769 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:52:34.769 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 23:52:34.769 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 23:52:34.769 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 23:52:34.769 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 23:52:34.769 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 23:52:34.769 | INFO | ------------------------------------------ | 20260505 23:52:34.769 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:52:34.769 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 23:52:34.769 | INFO | Total mem usage: 25.6 GiB | 20260505 23:52:34.769 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 23:52:34.769 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 23:52:34.769 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 23:52:34.769 | INFO | High Water Breakdown: | 20260505 23:52:34.769 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 23:52:34.769 | INFO | ------------------------------------------ | 20260505 23:52:34.769 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:52:34.769 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 23:52:34.769 | INFO | Total mem usage: 15.7 GiB | 20260505 23:52:34.769 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 23:52:34.769 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 23:52:34.769 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 23:52:34.769 | INFO | High Water Breakdown: | 20260505 23:52:34.769 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 23:52:34.769 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 23:52:34.769 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 23:52:34.769 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 23:52:34.769 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 23:52:34.769 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 23:52:34.769 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 23:52:34.769 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 23:52:34.769 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 23:52:34.769 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:52:34.769 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 23:52:34.769 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:52:34.769 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 23:52:34.769 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 23:52:34.769 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 23:52:34.769 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 23:52:34.769 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:52:34.769 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 23:52:34.769 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:52:34.769 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:52:34.769 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:52:34.769 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 23:52:34.769 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 23:52:34.769 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:52:34.769 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 23:52:34.769 | INFO | ------------------------------------------ | 20260505 23:52:34.769 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:52:34.769 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 23:52:34.769 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:52:34.769 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:52:34.769 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.40e+01, 1.34e+01 / 7.75e-01 | 20260505 23:52:34.769 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.04e-01, 1.02e-02 / 4.64e-03 | 20260505 23:52:34.769 | INFO | PerfReport: 11.81%, Baseline -- 3.70e+00 / 4.50e+00, 5.61e-01 / 1.90e-01 | 20260505 23:52:34.769 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.52e+00, 1.87e-01 / 8.78e-02 | 20260505 23:52:34.769 | INFO | PerfReport: 15.99%, DME -- 4.01e+01 / 4.39e+01, 3.84e+00 / 1.14e+00 | 20260505 23:52:34.769 | INFO | PerfReport: 57.34%, FrameLabeling -- 1.80e+01 / 1.96e+01, 1.48e+00 / 1.15e+00 | 20260505 23:52:34.769 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.23e+00 / 2.74e+00, 4.23e-01 / 2.59e-01 | 20260505 23:52:34.769 | INFO | PerfReport: 3.10%, Metrics -- 9.70e-01 / 1.25e+00, 2.59e-01 / 1.12e-01 | 20260505 23:52:34.769 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.36e-01 / 3.91e-01, 5.70e-02 / 2.73e-02 | 20260505 23:52:34.769 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 23:52:34.769 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:52:34.769 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:52:34.769 | INFO | PerfReport: Full Execution -- 9.84e+01 / -nan, 2.44e+01 / 0.00e+00 | 20260505 23:52:34.769 | INFO | PerfReport: 22.35%, Upload -- 2.20e+01 / 0.00e+00, 7.56e+00 / 0.00e+00 | 20260505 23:52:34.769 | INFO | PerfReport: 21.82%, Download -- 2.15e+01 / 0.00e+00, 7.56e+00 / 0.00e+00 | 20260505 23:52:34.769 | INFO | PerfReport: 31.83%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 23:52:34.769 | INFO | PerfReport: 2.17%, LockUpload -- 2.14e+00 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260505 23:52:34.769 | INFO | PerfReport: 1.27%, LockDownload -- 1.25e+00 / 0.00e+00, 2.79e+00 / 0.00e+00 | 20260505 23:52:34.769 | INFO | PerfReport: 20.54%, LockCompute -- 2.02e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260505 23:52:34.769 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.50e-02 / 0.00e+00 | 20260505 23:52:34.886 | INFO | Starting a round of Baz IO | 20260505 23:52:38.033 | INFO | Populating RTMetrics report | 20260505 23:52:38.033 | INFO | RTMetrics report populated | 20260505 23:52:38.033 | INFO | Opening tmp RTMetrics json file | 20260505 23:52:38.035 | INFO | Serializing RTMetrics to json | 20260505 23:52:38.040 | INFO | Serialization done, swapping tmp file now... | 20260505 23:52:38.041 | INFO | RTMetrics done | 20260505 23:52:38.110 | INFO | Finished a round of Baz IO: Wrote 1.876 GiB in 3.224 seconds | 20260505 23:52:38.110 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.40% / 0.00% / 78.60% | 20260505 23:52:38.110 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.100303% / 0% / 8.34465e-05% | 20260505 23:52:38.110 | INFO | Baz IO overall: | 20260505 23:52:38.110 | INFO | BytesWritten : 4162728166581 | 20260505 23:52:38.110 | INFO | NumEvents : 1960225786818 | 20260505 23:52:39.911 | INFO | Analyzing chunk frames = [7299072,7299584) | 20260505 23:52:39.911 | INFO | MemInfo: { | 20260505 23:52:39.911 | INFO | "gpu" : " 17 GB", | 20260505 23:52:39.911 | INFO | "huge" : "", | 20260505 23:52:39.911 | INFO | "huge_shared" : " 27 GB", | 20260505 23:52:39.911 | INFO | "malloc" : " 14 GB", | 20260505 23:52:39.911 | INFO | "pinned" : " 29 GB" | 20260505 23:52:39.911 | INFO | } | 20260505 23:52:39.912 | INFO | Perf report for the last 5.14371 seconds: | 20260505 23:52:39.912 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:52:39.912 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:39.912 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:39.912 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:39.912 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260505 23:52:39.912 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:39.912 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260505 23:52:39.912 | INFO | PRE_HQ : 59.8%, 1.7, 54.1ms, 0 | 20260505 23:52:39.912 | INFO | BAZWRITER : 61.9%, 3.5, 111.9ms, 0 | 20260505 23:52:39.912 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260505 23:52:39.912 | INFO | Uploaded 36GiB to GPU in 2.1724s (16.5716GiB/s) | 20260505 23:52:39.912 | INFO | Downloaded 38GiB to GPU in 2.24007s (16.9638GiB/s) | 20260505 23:52:45.052 | INFO | Analyzing chunk frames = [7299584,7300096) | 20260505 23:52:45.052 | INFO | MemInfo: { | 20260505 23:52:45.052 | INFO | "gpu" : " 17 GB", | 20260505 23:52:45.053 | INFO | "huge" : "", | 20260505 23:52:45.053 | INFO | "huge_shared" : " 27 GB", | 20260505 23:52:45.053 | INFO | "malloc" : " 14 GB", | 20260505 23:52:45.053 | INFO | "pinned" : " 29 GB" | 20260505 23:52:45.053 | INFO | } | 20260505 23:52:45.053 | INFO | Perf report for the last 5.14113 seconds: | 20260505 23:52:45.053 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:52:45.053 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:45.053 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:45.053 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:45.053 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 23:52:45.053 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:45.053 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:45.053 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260505 23:52:45.053 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:45.053 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:45.053 | INFO | Uploaded 36GiB to GPU in 2.11618s (17.0118GiB/s) | 20260505 23:52:45.053 | INFO | Downloaded 34GiB to GPU in 2.04301s (16.6421GiB/s) | 20260505 23:52:50.190 | INFO | Analyzing chunk frames = [7300096,7300608) | 20260505 23:52:50.190 | INFO | MemInfo: { | 20260505 23:52:50.190 | INFO | "gpu" : " 17 GB", | 20260505 23:52:50.190 | INFO | "huge" : "", | 20260505 23:52:50.190 | INFO | "huge_shared" : " 27 GB", | 20260505 23:52:50.190 | INFO | "malloc" : " 14 GB", | 20260505 23:52:50.190 | INFO | "pinned" : " 29 GB" | 20260505 23:52:50.190 | INFO | } | 20260505 23:52:50.190 | INFO | Perf report for the last 5.13741 seconds: | 20260505 23:52:50.190 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:52:50.190 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:50.190 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:50.190 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:50.190 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260505 23:52:50.190 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:50.190 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:50.190 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260505 23:52:50.190 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:50.190 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:50.190 | INFO | Uploaded 36GiB to GPU in 2.07599s (17.3411GiB/s) | 20260505 23:52:50.190 | INFO | Downloaded 34GiB to GPU in 2.01568s (16.8677GiB/s) | 20260505 23:52:55.322 | INFO | Analyzing chunk frames = [7300608,7301120) | 20260505 23:52:55.322 | INFO | MemInfo: { | 20260505 23:52:55.322 | INFO | "gpu" : " 17 GB", | 20260505 23:52:55.322 | INFO | "huge" : "", | 20260505 23:52:55.322 | INFO | "huge_shared" : " 27 GB", | 20260505 23:52:55.322 | INFO | "malloc" : " 14 GB", | 20260505 23:52:55.322 | INFO | "pinned" : " 29 GB" | 20260505 23:52:55.322 | INFO | } | 20260505 23:52:55.323 | INFO | Perf report for the last 5.13246 seconds: | 20260505 23:52:55.323 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:52:55.323 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:55.323 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:55.323 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:55.323 | INFO | ANALYSIS : 61.8%, 3.0, 95.8ms, 0 | 20260505 23:52:55.323 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:55.323 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:55.323 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 23:52:55.323 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:55.323 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:52:55.323 | INFO | Uploaded 36GiB to GPU in 2.13067s (16.8961GiB/s) | 20260505 23:52:55.323 | INFO | Downloaded 34GiB to GPU in 2.01496s (16.8738GiB/s) | 20260505 23:53:00.464 | INFO | Analyzing chunk frames = [7301120,7301632) | 20260505 23:53:00.464 | INFO | MemInfo: { | 20260505 23:53:00.464 | INFO | "gpu" : " 17 GB", | 20260505 23:53:00.464 | INFO | "huge" : "", | 20260505 23:53:00.464 | INFO | "huge_shared" : " 27 GB", | 20260505 23:53:00.464 | INFO | "malloc" : " 14 GB", | 20260505 23:53:00.464 | INFO | "pinned" : " 29 GB" | 20260505 23:53:00.464 | INFO | } | 20260505 23:53:00.464 | INFO | Perf report for the last 5.14166 seconds: | 20260505 23:53:00.464 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:53:00.464 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:00.464 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:00.464 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:00.464 | INFO | ANALYSIS : 61.2%, 3.0, 95.6ms, 0 | 20260505 23:53:00.464 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:00.464 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:00.464 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:53:00.464 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:00.464 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:00.465 | INFO | Uploaded 36GiB to GPU in 2.10406s (17.1098GiB/s) | 20260505 23:53:00.465 | INFO | Downloaded 34GiB to GPU in 2.05004s (16.585GiB/s) | 20260505 23:53:05.613 | INFO | Analyzing chunk frames = [7301632,7302144) | 20260505 23:53:05.613 | INFO | MemInfo: { | 20260505 23:53:05.613 | INFO | "gpu" : " 17 GB", | 20260505 23:53:05.613 | INFO | "huge" : "", | 20260505 23:53:05.613 | INFO | "huge_shared" : " 27 GB", | 20260505 23:53:05.613 | INFO | "malloc" : " 14 GB", | 20260505 23:53:05.613 | INFO | "pinned" : " 29 GB" | 20260505 23:53:05.613 | INFO | } | 20260505 23:53:05.613 | INFO | Perf report for the last 5.1492 seconds: | 20260505 23:53:05.613 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:53:05.613 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:05.613 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:05.613 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:05.613 | INFO | ANALYSIS : 61.2%, 3.0, 95.7ms, 0 | 20260505 23:53:05.613 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:05.613 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:05.614 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:53:05.614 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:05.614 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:05.614 | INFO | Uploaded 36GiB to GPU in 2.06575s (17.427GiB/s) | 20260505 23:53:05.614 | INFO | Downloaded 34GiB to GPU in 2.03621s (16.6977GiB/s) | 20260505 23:53:10.745 | INFO | Analyzing chunk frames = [7302144,7302656) | 20260505 23:53:10.745 | INFO | MemInfo: { | 20260505 23:53:10.745 | INFO | "gpu" : " 17 GB", | 20260505 23:53:10.745 | INFO | "huge" : "", | 20260505 23:53:10.746 | INFO | "huge_shared" : " 27 GB", | 20260505 23:53:10.746 | INFO | "malloc" : " 14 GB", | 20260505 23:53:10.746 | INFO | "pinned" : " 29 GB" | 20260505 23:53:10.746 | INFO | } | 20260505 23:53:10.746 | INFO | Perf report for the last 5.13224 seconds: | 20260505 23:53:10.746 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:53:10.746 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:10.746 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:10.746 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:10.746 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 23:53:10.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:10.746 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:10.746 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 23:53:10.746 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:10.746 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:10.746 | INFO | Uploaded 36GiB to GPU in 2.10447s (17.1064GiB/s) | 20260505 23:53:10.746 | INFO | Downloaded 34GiB to GPU in 2.08267s (16.3252GiB/s) | 20260505 23:53:15.880 | INFO | Analyzing chunk frames = [7302656,7303168) | 20260505 23:53:15.880 | INFO | MemInfo: { | 20260505 23:53:15.880 | INFO | "gpu" : " 17 GB", | 20260505 23:53:15.880 | INFO | "huge" : "", | 20260505 23:53:15.880 | INFO | "huge_shared" : " 27 GB", | 20260505 23:53:15.880 | INFO | "malloc" : " 14 GB", | 20260505 23:53:15.880 | INFO | "pinned" : " 29 GB" | 20260505 23:53:15.880 | INFO | } | 20260505 23:53:15.880 | INFO | Perf report for the last 5.13441 seconds: | 20260505 23:53:15.880 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:53:15.880 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:15.880 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:15.880 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:15.880 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260505 23:53:15.880 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:15.880 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:15.880 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 23:53:15.880 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:15.880 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:15.880 | INFO | Uploaded 36GiB to GPU in 2.11327s (17.0352GiB/s) | 20260505 23:53:15.880 | INFO | Downloaded 34GiB to GPU in 2.01587s (16.8662GiB/s) | 20260505 23:53:16.008 | INFO | Starting a round of Baz IO | 20260505 23:53:19.147 | INFO | Populating RTMetrics report | 20260505 23:53:19.148 | INFO | RTMetrics report populated | 20260505 23:53:19.148 | INFO | Opening tmp RTMetrics json file | 20260505 23:53:19.148 | INFO | Serializing RTMetrics to json | 20260505 23:53:19.150 | INFO | Serialization done, swapping tmp file now... | 20260505 23:53:19.150 | INFO | RTMetrics done | 20260505 23:53:19.225 | INFO | Finished a round of Baz IO: Wrote 1.878 GiB in 3.217 seconds | 20260505 23:53:19.225 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.40% / 0.00% / 78.60% | 20260505 23:53:19.225 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.100183% / 0% / 8.34465e-05% | 20260505 23:53:19.225 | INFO | Baz IO overall: | 20260505 23:53:19.225 | INFO | BytesWritten : 4164744213830 | 20260505 23:53:19.225 | INFO | NumEvents : 1961149052878 | 20260505 23:53:21.022 | INFO | Analyzing chunk frames = [7303168,7303680) | 20260505 23:53:21.022 | INFO | MemInfo: { | 20260505 23:53:21.022 | INFO | "gpu" : " 17 GB", | 20260505 23:53:21.022 | INFO | "huge" : "", | 20260505 23:53:21.023 | INFO | "huge_shared" : " 27 GB", | 20260505 23:53:21.023 | INFO | "malloc" : " 14 GB", | 20260505 23:53:21.023 | INFO | "pinned" : " 29 GB" | 20260505 23:53:21.023 | INFO | } | 20260505 23:53:21.023 | INFO | Perf report for the last 5.14251 seconds: | 20260505 23:53:21.023 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:53:21.023 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:21.023 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:21.023 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:21.023 | INFO | ANALYSIS : 62.5%, 3.0, 97.7ms, 0 | 20260505 23:53:21.023 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:21.023 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260505 23:53:21.023 | INFO | PRE_HQ : 59.8%, 1.7, 54.1ms, 0 | 20260505 23:53:21.023 | INFO | BAZWRITER : 61.8%, 3.5, 112.1ms, 0 | 20260505 23:53:21.023 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 23:53:21.023 | INFO | Uploaded 36GiB to GPU in 2.17083s (16.5835GiB/s) | 20260505 23:53:21.023 | INFO | Downloaded 38GiB to GPU in 2.24369s (16.9364GiB/s) | 20260505 23:53:26.161 | INFO | Analyzing chunk frames = [7303680,7304192) | 20260505 23:53:26.161 | INFO | MemInfo: { | 20260505 23:53:26.161 | INFO | "gpu" : " 17 GB", | 20260505 23:53:26.161 | INFO | "huge" : "", | 20260505 23:53:26.161 | INFO | "huge_shared" : " 27 GB", | 20260505 23:53:26.161 | INFO | "malloc" : " 14 GB", | 20260505 23:53:26.161 | INFO | "pinned" : " 29 GB" | 20260505 23:53:26.161 | INFO | } | 20260505 23:53:26.161 | INFO | Perf report for the last 5.1383 seconds: | 20260505 23:53:26.161 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:53:26.161 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:26.161 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:26.161 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:26.161 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 23:53:26.161 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:26.161 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:26.161 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:53:26.161 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:26.161 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:26.161 | INFO | Uploaded 36GiB to GPU in 2.10122s (17.1329GiB/s) | 20260505 23:53:26.161 | INFO | Downloaded 34GiB to GPU in 2.03986s (16.6678GiB/s) | 20260505 23:53:31.299 | INFO | Analyzing chunk frames = [7304192,7304704) | 20260505 23:53:31.299 | INFO | MemInfo: { | 20260505 23:53:31.299 | INFO | "gpu" : " 17 GB", | 20260505 23:53:31.299 | INFO | "huge" : "", | 20260505 23:53:31.299 | INFO | "huge_shared" : " 27 GB", | 20260505 23:53:31.299 | INFO | "malloc" : " 14 GB", | 20260505 23:53:31.299 | INFO | "pinned" : " 29 GB" | 20260505 23:53:31.299 | INFO | } | 20260505 23:53:31.299 | INFO | Perf report for the last 5.13819 seconds: | 20260505 23:53:31.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:53:31.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:31.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:31.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:31.299 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260505 23:53:31.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:31.299 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:31.299 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 23:53:31.299 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:31.299 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:31.299 | INFO | Uploaded 36GiB to GPU in 2.08617s (17.2565GiB/s) | 20260505 23:53:31.299 | INFO | Downloaded 34GiB to GPU in 2.03041s (16.7454GiB/s) | 20260505 23:53:36.439 | INFO | Analyzing chunk frames = [7304704,7305216) | 20260505 23:53:36.439 | INFO | MemInfo: { | 20260505 23:53:36.439 | INFO | "gpu" : " 17 GB", | 20260505 23:53:36.439 | INFO | "huge" : "", | 20260505 23:53:36.439 | INFO | "huge_shared" : " 27 GB", | 20260505 23:53:36.439 | INFO | "malloc" : " 14 GB", | 20260505 23:53:36.439 | INFO | "pinned" : " 29 GB" | 20260505 23:53:36.439 | INFO | } | 20260505 23:53:36.439 | INFO | Perf report for the last 5.13982 seconds: | 20260505 23:53:36.439 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:53:36.439 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:36.439 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:36.439 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:36.439 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260505 23:53:36.439 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:36.439 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:36.439 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 23:53:36.439 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:36.439 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:36.440 | INFO | Uploaded 36GiB to GPU in 2.12738s (16.9222GiB/s) | 20260505 23:53:36.440 | INFO | Downloaded 34GiB to GPU in 2.00822s (16.9304GiB/s) | 20260505 23:53:41.578 | INFO | Analyzing chunk frames = [7305216,7305728) | 20260505 23:53:41.578 | INFO | MemInfo: { | 20260505 23:53:41.578 | INFO | "gpu" : " 17 GB", | 20260505 23:53:41.578 | INFO | "huge" : "", | 20260505 23:53:41.578 | INFO | "huge_shared" : " 27 GB", | 20260505 23:53:41.578 | INFO | "malloc" : " 14 GB", | 20260505 23:53:41.578 | INFO | "pinned" : " 29 GB" | 20260505 23:53:41.578 | INFO | } | 20260505 23:53:41.578 | INFO | Perf report for the last 5.13876 seconds: | 20260505 23:53:41.578 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:53:41.578 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:41.578 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:41.578 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:41.578 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260505 23:53:41.578 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:41.578 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:41.578 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 23:53:41.578 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:41.578 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:41.578 | INFO | Uploaded 36GiB to GPU in 2.17038s (16.5869GiB/s) | 20260505 23:53:41.578 | INFO | Downloaded 34GiB to GPU in 2.07022s (16.4234GiB/s) | 20260505 23:53:46.720 | INFO | Analyzing chunk frames = [7305728,7306240) | 20260505 23:53:46.720 | INFO | MemInfo: { | 20260505 23:53:46.720 | INFO | "gpu" : " 17 GB", | 20260505 23:53:46.720 | INFO | "huge" : "", | 20260505 23:53:46.720 | INFO | "huge_shared" : " 27 GB", | 20260505 23:53:46.720 | INFO | "malloc" : " 14 GB", | 20260505 23:53:46.720 | INFO | "pinned" : " 29 GB" | 20260505 23:53:46.720 | INFO | } | 20260505 23:53:46.720 | INFO | Perf report for the last 5.14216 seconds: | 20260505 23:53:46.720 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:53:46.720 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:46.720 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:46.720 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:46.720 | INFO | ANALYSIS : 61.3%, 3.0, 95.8ms, 0 | 20260505 23:53:46.720 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:46.720 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:46.720 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 23:53:46.720 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:46.720 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:46.720 | INFO | Uploaded 36GiB to GPU in 2.09304s (17.1999GiB/s) | 20260505 23:53:46.720 | INFO | Downloaded 34GiB to GPU in 2.04174s (16.6524GiB/s) | 20260505 23:53:51.862 | INFO | Analyzing chunk frames = [7306240,7306752) | 20260505 23:53:51.862 | INFO | MemInfo: { | 20260505 23:53:51.862 | INFO | "gpu" : " 17 GB", | 20260505 23:53:51.862 | INFO | "huge" : "", | 20260505 23:53:51.862 | INFO | "huge_shared" : " 27 GB", | 20260505 23:53:51.862 | INFO | "malloc" : " 14 GB", | 20260505 23:53:51.862 | INFO | "pinned" : " 29 GB" | 20260505 23:53:51.862 | INFO | } | 20260505 23:53:51.862 | INFO | Perf report for the last 5.1419 seconds: | 20260505 23:53:51.862 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:53:51.862 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:51.862 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:51.862 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:51.862 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 23:53:51.862 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:51.862 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:51.862 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 23:53:51.862 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:51.862 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:51.862 | INFO | Uploaded 36GiB to GPU in 2.07105s (17.3825GiB/s) | 20260505 23:53:51.862 | INFO | Downloaded 34GiB to GPU in 2.06644s (16.4534GiB/s) | 20260505 23:53:56.996 | INFO | Analyzing chunk frames = [7306752,7307264) | 20260505 23:53:56.996 | INFO | MemInfo: { | 20260505 23:53:56.996 | INFO | "gpu" : " 17 GB", | 20260505 23:53:56.996 | INFO | "huge" : "", | 20260505 23:53:56.996 | INFO | "huge_shared" : " 27 GB", | 20260505 23:53:56.996 | INFO | "malloc" : " 14 GB", | 20260505 23:53:56.996 | INFO | "pinned" : " 29 GB" | 20260505 23:53:56.996 | INFO | } | 20260505 23:53:56.996 | INFO | Perf report for the last 5.13468 seconds: | 20260505 23:53:56.996 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:53:56.996 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:56.997 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:56.997 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:56.997 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260505 23:53:56.997 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:56.997 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:56.997 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260505 23:53:56.997 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:56.997 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:53:56.997 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 23:53:56.997 | INFO | Total mem usage: 15.3 GiB | 20260505 23:53:56.997 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 23:53:56.997 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.13%) | 20260505 23:53:56.997 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 23:53:56.997 | INFO | High Water Breakdown: | 20260505 23:53:56.997 | INFO | 60.9% : 8.55 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 23:53:56.997 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 23:53:56.997 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 23:53:56.997 | INFO | ------------------------------------------ | 20260505 23:53:56.997 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:53:56.997 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 23:53:56.997 | INFO | Total mem usage: 26.5 GiB | 20260505 23:53:56.997 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 23:53:56.997 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 23:53:56.997 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 23:53:56.997 | INFO | High Water Breakdown: | 20260505 23:53:56.997 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:53:56.997 | INFO | Uploaded 36GiB to GPU in 2.12981s (16.9029GiB/s) | 20260505 23:53:56.997 | INFO | Downloaded 34GiB to GPU in 2.01478s (16.8753GiB/s) | 20260505 23:53:56.997 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:53:56.997 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:53:56.997 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:53:56.997 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:53:56.997 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:53:56.997 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 23:53:56.997 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 23:53:56.997 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:53:56.997 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 23:53:56.997 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 23:53:56.997 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 23:53:56.997 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 23:53:56.997 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 23:53:56.997 | INFO | ------------------------------------------ | 20260505 23:53:56.997 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:53:56.997 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 23:53:56.997 | INFO | Total mem usage: 25.6 GiB | 20260505 23:53:56.997 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 23:53:56.997 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 23:53:56.997 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 23:53:56.997 | INFO | High Water Breakdown: | 20260505 23:53:56.997 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 23:53:56.997 | INFO | ------------------------------------------ | 20260505 23:53:56.997 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:53:56.997 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 23:53:56.997 | INFO | Total mem usage: 15.7 GiB | 20260505 23:53:56.997 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 23:53:56.997 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 23:53:56.997 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 23:53:56.997 | INFO | High Water Breakdown: | 20260505 23:53:56.997 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 23:53:56.997 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 23:53:56.997 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 23:53:56.997 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 23:53:56.997 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 23:53:56.997 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 23:53:56.997 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 23:53:56.997 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 23:53:56.997 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 23:53:56.997 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:53:56.997 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 23:53:56.997 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:53:56.997 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 23:53:56.997 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 23:53:56.998 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 23:53:56.998 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 23:53:56.998 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:53:56.998 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 23:53:56.998 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:53:56.998 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:53:56.998 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:53:56.998 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 23:53:56.998 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 23:53:56.998 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:53:56.998 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 23:53:56.998 | INFO | ------------------------------------------ | 20260505 23:53:56.998 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:53:56.998 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 23:53:56.998 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:53:56.998 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:53:56.998 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.40e+01, 1.33e+01 / 7.96e-01 | 20260505 23:53:56.998 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.05e-01, 1.04e-02 / 3.82e-03 | 20260505 23:53:56.998 | INFO | PerfReport: 11.82%, Baseline -- 3.70e+00 / 4.51e+00, 5.61e-01 / 1.50e-01 | 20260505 23:53:56.998 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.53e+00, 1.86e-01 / 7.50e-02 | 20260505 23:53:56.998 | INFO | PerfReport: 15.99%, DME -- 4.01e+01 / 4.39e+01, 3.75e+00 / 1.04e+00 | 20260505 23:53:56.998 | INFO | PerfReport: 57.32%, FrameLabeling -- 1.79e+01 / 1.95e+01, 1.46e+00 / 1.22e+00 | 20260505 23:53:56.998 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.23e+00 / 2.76e+00, 4.26e-01 / 2.41e-01 | 20260505 23:53:56.998 | INFO | PerfReport: 3.10%, Metrics -- 9.72e-01 / 1.26e+00, 2.64e-01 / 1.28e-01 | 20260505 23:53:56.998 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.37e-01 / 3.90e-01, 5.77e-02 / 3.07e-02 | 20260505 23:53:56.998 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 23:53:56.998 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:53:56.998 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:53:56.998 | INFO | PerfReport: Full Execution -- 9.84e+01 / -nan, 2.45e+01 / 0.00e+00 | 20260505 23:53:56.998 | INFO | PerfReport: 22.39%, Upload -- 2.20e+01 / 0.00e+00, 7.56e+00 / 0.00e+00 | 20260505 23:53:56.998 | INFO | PerfReport: 21.85%, Download -- 2.15e+01 / 0.00e+00, 7.50e+00 / 0.00e+00 | 20260505 23:53:56.998 | INFO | PerfReport: 31.83%, Compute -- 3.13e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260505 23:53:56.998 | INFO | PerfReport: 2.12%, LockUpload -- 2.09e+00 / 0.00e+00, 7.84e+00 / 0.00e+00 | 20260505 23:53:56.998 | INFO | PerfReport: 1.29%, LockDownload -- 1.27e+00 / 0.00e+00, 2.81e+00 / 0.00e+00 | 20260505 23:53:56.998 | INFO | PerfReport: 20.51%, LockCompute -- 2.02e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260505 23:53:56.998 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.82e-02 / 0.00e+00 | 20260505 23:53:57.123 | INFO | Starting a round of Baz IO | 20260505 23:54:00.267 | INFO | Populating RTMetrics report | 20260505 23:54:00.267 | INFO | RTMetrics report populated | 20260505 23:54:00.267 | INFO | Opening tmp RTMetrics json file | 20260505 23:54:00.268 | INFO | Serializing RTMetrics to json | 20260505 23:54:00.269 | INFO | Serialization done, swapping tmp file now... | 20260505 23:54:00.270 | INFO | RTMetrics done | 20260505 23:54:00.346 | INFO | Finished a round of Baz IO: Wrote 1.877 GiB in 3.223 seconds | 20260505 23:54:00.347 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.40% / 0.00% / 78.60% | 20260505 23:54:00.347 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0998815% / 0% / 6.35783e-05% | 20260505 23:54:00.347 | INFO | Baz IO overall: | 20260505 23:54:00.347 | INFO | BytesWritten : 4166760075214 | 20260505 23:54:00.347 | INFO | NumEvents : 1962072135558 | 20260505 23:54:02.135 | INFO | Analyzing chunk frames = [7307264,7307776) | 20260505 23:54:02.135 | INFO | MemInfo: { | 20260505 23:54:02.135 | INFO | "gpu" : " 17 GB", | 20260505 23:54:02.135 | INFO | "huge" : "", | 20260505 23:54:02.135 | INFO | "huge_shared" : " 27 GB", | 20260505 23:54:02.135 | INFO | "malloc" : " 14 GB", | 20260505 23:54:02.135 | INFO | "pinned" : " 29 GB" | 20260505 23:54:02.135 | INFO | } | 20260505 23:54:02.135 | INFO | Perf report for the last 5.13883 seconds: | 20260505 23:54:02.135 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:54:02.135 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:02.135 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:02.135 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:02.135 | INFO | ANALYSIS : 62.6%, 3.0, 96.6ms, 0 | 20260505 23:54:02.135 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:02.135 | INFO | RT_METRICS : 43.4%, 1.0, 22.5ms, 0 | 20260505 23:54:02.135 | INFO | PRE_HQ : 59.9%, 1.7, 53.6ms, 0 | 20260505 23:54:02.136 | INFO | BAZWRITER : 62.0%, 3.5, 111.2ms, 0 | 20260505 23:54:02.136 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 23:54:02.136 | INFO | Uploaded 36GiB to GPU in 2.17379s (16.561GiB/s) | 20260505 23:54:02.136 | INFO | Downloaded 38GiB to GPU in 2.23924s (16.9701GiB/s) | 20260505 23:54:07.277 | INFO | Analyzing chunk frames = [7307776,7308288) | 20260505 23:54:07.277 | INFO | MemInfo: { | 20260505 23:54:07.277 | INFO | "gpu" : " 17 GB", | 20260505 23:54:07.277 | INFO | "huge" : "", | 20260505 23:54:07.277 | INFO | "huge_shared" : " 27 GB", | 20260505 23:54:07.277 | INFO | "malloc" : " 14 GB", | 20260505 23:54:07.277 | INFO | "pinned" : " 29 GB" | 20260505 23:54:07.277 | INFO | } | 20260505 23:54:07.277 | INFO | Perf report for the last 5.14189 seconds: | 20260505 23:54:07.277 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:54:07.277 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:07.277 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:07.277 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:07.277 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260505 23:54:07.277 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:07.277 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:07.277 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260505 23:54:07.277 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:07.277 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:07.278 | INFO | Uploaded 36GiB to GPU in 2.08704s (17.2493GiB/s) | 20260505 23:54:07.278 | INFO | Downloaded 34GiB to GPU in 2.02672s (16.7759GiB/s) | 20260505 23:54:12.420 | INFO | Analyzing chunk frames = [7308288,7308800) | 20260505 23:54:12.420 | INFO | MemInfo: { | 20260505 23:54:12.420 | INFO | "gpu" : " 17 GB", | 20260505 23:54:12.420 | INFO | "huge" : "", | 20260505 23:54:12.420 | INFO | "huge_shared" : " 27 GB", | 20260505 23:54:12.420 | INFO | "malloc" : " 14 GB", | 20260505 23:54:12.420 | INFO | "pinned" : " 29 GB" | 20260505 23:54:12.420 | INFO | } | 20260505 23:54:12.420 | INFO | Perf report for the last 5.14306 seconds: | 20260505 23:54:12.420 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:54:12.420 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:12.420 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:12.420 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:12.420 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260505 23:54:12.420 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:12.420 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:12.420 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260505 23:54:12.420 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:12.420 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:12.421 | INFO | Uploaded 36GiB to GPU in 2.0844s (17.2712GiB/s) | 20260505 23:54:12.421 | INFO | Downloaded 34GiB to GPU in 2.02451s (16.7942GiB/s) | 20260505 23:54:17.561 | INFO | Analyzing chunk frames = [7308800,7309312) | 20260505 23:54:17.561 | INFO | MemInfo: { | 20260505 23:54:17.561 | INFO | "gpu" : " 17 GB", | 20260505 23:54:17.561 | INFO | "huge" : "", | 20260505 23:54:17.561 | INFO | "huge_shared" : " 27 GB", | 20260505 23:54:17.561 | INFO | "malloc" : " 14 GB", | 20260505 23:54:17.561 | INFO | "pinned" : " 29 GB" | 20260505 23:54:17.561 | INFO | } | 20260505 23:54:17.561 | INFO | Perf report for the last 5.14092 seconds: | 20260505 23:54:17.561 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:54:17.561 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:17.561 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:17.561 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:17.561 | INFO | ANALYSIS : 61.6%, 3.0, 95.6ms, 0 | 20260505 23:54:17.561 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:17.561 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:17.561 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 23:54:17.561 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:17.561 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:17.561 | INFO | Uploaded 36GiB to GPU in 2.14684s (16.7688GiB/s) | 20260505 23:54:17.561 | INFO | Downloaded 34GiB to GPU in 2.04761s (16.6047GiB/s) | 20260505 23:54:22.693 | INFO | Analyzing chunk frames = [7309312,7309824) | 20260505 23:54:22.693 | INFO | MemInfo: { | 20260505 23:54:22.693 | INFO | "gpu" : " 17 GB", | 20260505 23:54:22.693 | INFO | "huge" : "", | 20260505 23:54:22.693 | INFO | "huge_shared" : " 27 GB", | 20260505 23:54:22.693 | INFO | "malloc" : " 14 GB", | 20260505 23:54:22.693 | INFO | "pinned" : " 29 GB" | 20260505 23:54:22.693 | INFO | } | 20260505 23:54:22.693 | INFO | Perf report for the last 5.13143 seconds: | 20260505 23:54:22.693 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:54:22.693 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:22.693 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:22.693 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:22.693 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260505 23:54:22.693 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:22.693 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:22.693 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:54:22.693 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:22.693 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:22.693 | INFO | Uploaded 36GiB to GPU in 2.09915s (17.1498GiB/s) | 20260505 23:54:22.693 | INFO | Downloaded 34GiB to GPU in 2.04321s (16.6405GiB/s) | 20260505 23:54:27.832 | INFO | Analyzing chunk frames = [7309824,7310336) | 20260505 23:54:27.832 | INFO | MemInfo: { | 20260505 23:54:27.832 | INFO | "gpu" : " 17 GB", | 20260505 23:54:27.832 | INFO | "huge" : "", | 20260505 23:54:27.832 | INFO | "huge_shared" : " 27 GB", | 20260505 23:54:27.832 | INFO | "malloc" : " 14 GB", | 20260505 23:54:27.832 | INFO | "pinned" : " 29 GB" | 20260505 23:54:27.832 | INFO | } | 20260505 23:54:27.832 | INFO | Perf report for the last 5.1394 seconds: | 20260505 23:54:27.832 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:54:27.832 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:27.832 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:27.832 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:27.832 | INFO | ANALYSIS : 61.4%, 3.0, 95.9ms, 0 | 20260505 23:54:27.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:27.832 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:27.832 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:54:27.832 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:27.832 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:27.832 | INFO | Uploaded 36GiB to GPU in 2.07685s (17.3339GiB/s) | 20260505 23:54:27.832 | INFO | Downloaded 34GiB to GPU in 2.03813s (16.6819GiB/s) | 20260505 23:54:32.974 | INFO | Analyzing chunk frames = [7310336,7310848) | 20260505 23:54:32.974 | INFO | MemInfo: { | 20260505 23:54:32.974 | INFO | "gpu" : " 17 GB", | 20260505 23:54:32.974 | INFO | "huge" : "", | 20260505 23:54:32.974 | INFO | "huge_shared" : " 27 GB", | 20260505 23:54:32.974 | INFO | "malloc" : " 14 GB", | 20260505 23:54:32.974 | INFO | "pinned" : " 29 GB" | 20260505 23:54:32.974 | INFO | } | 20260505 23:54:32.974 | INFO | Perf report for the last 5.14203 seconds: | 20260505 23:54:32.974 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:54:32.974 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:32.974 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:32.974 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:32.974 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260505 23:54:32.974 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:32.974 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:32.974 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260505 23:54:32.974 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:32.974 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:32.974 | INFO | Uploaded 36GiB to GPU in 2.07281s (17.3677GiB/s) | 20260505 23:54:32.974 | INFO | Downloaded 34GiB to GPU in 2.06437s (16.4699GiB/s) | 20260505 23:54:38.113 | INFO | Analyzing chunk frames = [7310848,7311360) | 20260505 23:54:38.113 | INFO | MemInfo: { | 20260505 23:54:38.113 | INFO | "gpu" : " 17 GB", | 20260505 23:54:38.113 | INFO | "huge" : "", | 20260505 23:54:38.113 | INFO | "huge_shared" : " 27 GB", | 20260505 23:54:38.114 | INFO | "malloc" : " 14 GB", | 20260505 23:54:38.114 | INFO | "pinned" : " 29 GB" | 20260505 23:54:38.114 | INFO | } | 20260505 23:54:38.114 | INFO | Perf report for the last 5.13944 seconds: | 20260505 23:54:38.114 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:54:38.114 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:38.114 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:38.114 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:38.114 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 23:54:38.114 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:38.114 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:38.114 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 23:54:38.114 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:38.114 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:38.114 | INFO | Uploaded 36GiB to GPU in 2.12833s (16.9147GiB/s) | 20260505 23:54:38.114 | INFO | Downloaded 34GiB to GPU in 2.02047s (16.8278GiB/s) | 20260505 23:54:38.235 | INFO | Starting a round of Baz IO | 20260505 23:54:41.386 | INFO | Populating RTMetrics report | 20260505 23:54:41.387 | INFO | RTMetrics report populated | 20260505 23:54:41.387 | INFO | Opening tmp RTMetrics json file | 20260505 23:54:41.387 | INFO | Serializing RTMetrics to json | 20260505 23:54:41.389 | INFO | Serialization done, swapping tmp file now... | 20260505 23:54:41.389 | INFO | RTMetrics done | 20260505 23:54:41.468 | INFO | Finished a round of Baz IO: Wrote 1.879 GiB in 3.232 seconds | 20260505 23:54:41.468 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.40% / 0.00% / 78.60% | 20260505 23:54:41.468 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.100605% / 3.97364e-06% / 5.96046e-05% | 20260505 23:54:41.468 | INFO | Baz IO overall: | 20260505 23:54:41.468 | INFO | BytesWritten : 4168777499517 | 20260505 23:54:41.468 | INFO | NumEvents : 1962996260545 | 20260505 23:54:43.249 | INFO | Analyzing chunk frames = [7311360,7311872) | 20260505 23:54:43.249 | INFO | MemInfo: { | 20260505 23:54:43.249 | INFO | "gpu" : " 17 GB", | 20260505 23:54:43.249 | INFO | "huge" : "", | 20260505 23:54:43.250 | INFO | "huge_shared" : " 27 GB", | 20260505 23:54:43.250 | INFO | "malloc" : " 14 GB", | 20260505 23:54:43.250 | INFO | "pinned" : " 29 GB" | 20260505 23:54:43.250 | INFO | } | 20260505 23:54:43.250 | INFO | Perf report for the last 5.13599 seconds: | 20260505 23:54:43.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:54:43.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:43.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:43.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:43.250 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 23:54:43.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:43.250 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260505 23:54:43.250 | INFO | PRE_HQ : 59.9%, 1.7, 54.1ms, 0 | 20260505 23:54:43.250 | INFO | BAZWRITER : 62.1%, 3.5, 112.3ms, 0 | 20260505 23:54:43.250 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 23:54:43.250 | INFO | Uploaded 36GiB to GPU in 2.17179s (16.5762GiB/s) | 20260505 23:54:43.250 | INFO | Downloaded 38GiB to GPU in 2.2377s (16.9817GiB/s) | 20260505 23:54:48.384 | INFO | Analyzing chunk frames = [7311872,7312384) | 20260505 23:54:48.384 | INFO | MemInfo: { | 20260505 23:54:48.384 | INFO | "gpu" : " 17 GB", | 20260505 23:54:48.384 | INFO | "huge" : "", | 20260505 23:54:48.384 | INFO | "huge_shared" : " 27 GB", | 20260505 23:54:48.384 | INFO | "malloc" : " 14 GB", | 20260505 23:54:48.384 | INFO | "pinned" : " 29 GB" | 20260505 23:54:48.384 | INFO | } | 20260505 23:54:48.384 | INFO | Perf report for the last 5.13445 seconds: | 20260505 23:54:48.384 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:54:48.384 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:48.384 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:48.384 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:48.384 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260505 23:54:48.384 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:48.384 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:48.384 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260505 23:54:48.384 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:48.384 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:48.384 | INFO | Uploaded 36GiB to GPU in 2.09259s (17.2035GiB/s) | 20260505 23:54:48.384 | INFO | Downloaded 34GiB to GPU in 2.02615s (16.7806GiB/s) | 20260505 23:54:53.532 | INFO | Analyzing chunk frames = [7312384,7312896) | 20260505 23:54:53.532 | INFO | MemInfo: { | 20260505 23:54:53.532 | INFO | "gpu" : " 17 GB", | 20260505 23:54:53.532 | INFO | "huge" : "", | 20260505 23:54:53.532 | INFO | "huge_shared" : " 27 GB", | 20260505 23:54:53.532 | INFO | "malloc" : " 14 GB", | 20260505 23:54:53.532 | INFO | "pinned" : " 29 GB" | 20260505 23:54:53.532 | INFO | } | 20260505 23:54:53.532 | INFO | Perf report for the last 5.14796 seconds: | 20260505 23:54:53.532 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:54:53.532 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:53.532 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:53.532 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:53.532 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260505 23:54:53.532 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:53.532 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:53.532 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:54:53.532 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:53.532 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:53.532 | INFO | Uploaded 36GiB to GPU in 2.14704s (16.7672GiB/s) | 20260505 23:54:53.532 | INFO | Downloaded 34GiB to GPU in 2.05117s (16.5759GiB/s) | 20260505 23:54:58.674 | INFO | Analyzing chunk frames = [7312896,7313408) | 20260505 23:54:58.674 | INFO | MemInfo: { | 20260505 23:54:58.674 | INFO | "gpu" : " 17 GB", | 20260505 23:54:58.674 | INFO | "huge" : "", | 20260505 23:54:58.674 | INFO | "huge_shared" : " 27 GB", | 20260505 23:54:58.674 | INFO | "malloc" : " 14 GB", | 20260505 23:54:58.674 | INFO | "pinned" : " 29 GB" | 20260505 23:54:58.674 | INFO | } | 20260505 23:54:58.674 | INFO | Perf report for the last 5.14183 seconds: | 20260505 23:54:58.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:54:58.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:58.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:58.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:58.674 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260505 23:54:58.674 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:58.674 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:58.674 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:54:58.674 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:58.674 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:54:58.674 | INFO | Uploaded 36GiB to GPU in 2.13451s (16.8657GiB/s) | 20260505 23:54:58.674 | INFO | Downloaded 34GiB to GPU in 2.02268s (16.8094GiB/s) | 20260505 23:55:03.808 | INFO | Analyzing chunk frames = [7313408,7313920) | 20260505 23:55:03.808 | INFO | MemInfo: { | 20260505 23:55:03.808 | INFO | "gpu" : " 17 GB", | 20260505 23:55:03.809 | INFO | "huge" : "", | 20260505 23:55:03.809 | INFO | "huge_shared" : " 27 GB", | 20260505 23:55:03.809 | INFO | "malloc" : " 14 GB", | 20260505 23:55:03.809 | INFO | "pinned" : " 29 GB" | 20260505 23:55:03.809 | INFO | } | 20260505 23:55:03.809 | INFO | Perf report for the last 5.13478 seconds: | 20260505 23:55:03.809 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:55:03.809 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:03.809 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:03.809 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:03.809 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260505 23:55:03.809 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:03.809 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:03.809 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260505 23:55:03.809 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:03.809 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:03.809 | INFO | Uploaded 36GiB to GPU in 2.0917s (17.2109GiB/s) | 20260505 23:55:03.809 | INFO | Downloaded 34GiB to GPU in 2.03836s (16.6801GiB/s) | 20260505 23:55:08.941 | INFO | Analyzing chunk frames = [7313920,7314432) | 20260505 23:55:08.941 | INFO | MemInfo: { | 20260505 23:55:08.941 | INFO | "gpu" : " 17 GB", | 20260505 23:55:08.941 | INFO | "huge" : "", | 20260505 23:55:08.941 | INFO | "huge_shared" : " 27 GB", | 20260505 23:55:08.941 | INFO | "malloc" : " 14 GB", | 20260505 23:55:08.941 | INFO | "pinned" : " 29 GB" | 20260505 23:55:08.941 | INFO | } | 20260505 23:55:08.941 | INFO | Perf report for the last 5.13247 seconds: | 20260505 23:55:08.941 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:55:08.941 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:08.941 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:08.941 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:08.941 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260505 23:55:08.941 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:08.941 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:08.941 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 23:55:08.941 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:08.941 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:08.941 | INFO | Uploaded 36GiB to GPU in 2.08062s (17.3026GiB/s) | 20260505 23:55:08.941 | INFO | Downloaded 34GiB to GPU in 2.04028s (16.6643GiB/s) | 20260505 23:55:14.084 | INFO | Analyzing chunk frames = [7314432,7314944) | 20260505 23:55:14.084 | INFO | MemInfo: { | 20260505 23:55:14.084 | INFO | "gpu" : " 17 GB", | 20260505 23:55:14.084 | INFO | "huge" : "", | 20260505 23:55:14.084 | INFO | "huge_shared" : " 27 GB", | 20260505 23:55:14.085 | INFO | "malloc" : " 14 GB", | 20260505 23:55:14.085 | INFO | "pinned" : " 29 GB" | 20260505 23:55:14.085 | INFO | } | 20260505 23:55:14.085 | INFO | Perf report for the last 5.1435 seconds: | 20260505 23:55:14.085 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:55:14.085 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:14.085 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:14.085 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:14.085 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260505 23:55:14.085 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:14.085 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:14.085 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 23:55:14.085 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:14.085 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:14.085 | INFO | Uploaded 36GiB to GPU in 2.12677s (16.9271GiB/s) | 20260505 23:55:14.085 | INFO | Downloaded 34GiB to GPU in 2.12388s (16.0084GiB/s) | 20260505 23:55:19.229 | INFO | Analyzing chunk frames = [7314944,7315456) | 20260505 23:55:19.229 | INFO | MemInfo: { | 20260505 23:55:19.229 | INFO | "gpu" : " 17 GB", | 20260505 23:55:19.229 | INFO | "huge" : "", | 20260505 23:55:19.229 | INFO | "huge_shared" : " 27 GB", | 20260505 23:55:19.229 | INFO | "malloc" : " 14 GB", | 20260505 23:55:19.229 | INFO | "pinned" : " 29 GB" | 20260505 23:55:19.229 | INFO | } | 20260505 23:55:19.229 | INFO | Perf report for the last 5.14449 seconds: | 20260505 23:55:19.229 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:55:19.229 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:19.229 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:19.229 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:19.229 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260505 23:55:19.229 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:19.229 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:19.229 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 23:55:19.229 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:19.229 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:19.229 | INFO | Uploaded 36GiB to GPU in 2.12215s (16.964GiB/s) | 20260505 23:55:19.229 | INFO | Downloaded 34GiB to GPU in 1.99874s (17.0107GiB/s) | 20260505 23:55:19.229 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 23:55:19.229 | INFO | Total mem usage: 15.3 GiB | 20260505 23:55:19.229 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 23:55:19.229 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.15%) | 20260505 23:55:19.229 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 23:55:19.229 | INFO | High Water Breakdown: | 20260505 23:55:19.229 | INFO | 60.9% : 8.55 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 23:55:19.230 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 23:55:19.230 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 23:55:19.230 | INFO | ------------------------------------------ | 20260505 23:55:19.230 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:55:19.230 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 23:55:19.230 | INFO | Total mem usage: 26.5 GiB | 20260505 23:55:19.230 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 23:55:19.230 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 23:55:19.230 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 23:55:19.230 | INFO | High Water Breakdown: | 20260505 23:55:19.230 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:55:19.230 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:55:19.230 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:55:19.230 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:55:19.230 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:55:19.230 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:55:19.230 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 23:55:19.230 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 23:55:19.230 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:55:19.230 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 23:55:19.230 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 23:55:19.230 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 23:55:19.230 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 23:55:19.230 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 23:55:19.230 | INFO | ------------------------------------------ | 20260505 23:55:19.230 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:55:19.230 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 23:55:19.230 | INFO | Total mem usage: 25.6 GiB | 20260505 23:55:19.230 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 23:55:19.230 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 23:55:19.230 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 23:55:19.230 | INFO | High Water Breakdown: | 20260505 23:55:19.230 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 23:55:19.230 | INFO | ------------------------------------------ | 20260505 23:55:19.230 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:55:19.230 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 23:55:19.230 | INFO | Total mem usage: 15.7 GiB | 20260505 23:55:19.230 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 23:55:19.230 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 23:55:19.230 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 23:55:19.230 | INFO | High Water Breakdown: | 20260505 23:55:19.230 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 23:55:19.230 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 23:55:19.230 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 23:55:19.230 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 23:55:19.230 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 23:55:19.230 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 23:55:19.230 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 23:55:19.230 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 23:55:19.230 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 23:55:19.230 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:55:19.230 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 23:55:19.230 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:55:19.230 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 23:55:19.230 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 23:55:19.230 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 23:55:19.230 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 23:55:19.230 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:55:19.230 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 23:55:19.230 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:55:19.230 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:55:19.230 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:55:19.230 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 23:55:19.230 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 23:55:19.230 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:55:19.230 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 23:55:19.230 | INFO | ------------------------------------------ | 20260505 23:55:19.230 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:55:19.230 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 23:55:19.230 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:55:19.230 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:55:19.230 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.41e+01, 1.36e+01 / 1.29e+00 | 20260505 23:55:19.230 | INFO | PerfReport: 0.29%, AggregateStats -- 9.18e-02 / 1.04e-01, 1.05e-02 / 3.48e-03 | 20260505 23:55:19.230 | INFO | PerfReport: 11.82%, Baseline -- 3.70e+00 / 4.51e+00, 5.66e-01 / 1.98e-01 | 20260505 23:55:19.230 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.52e+00, 1.88e-01 / 8.15e-02 | 20260505 23:55:19.230 | INFO | PerfReport: 16.00%, DME -- 4.01e+01 / 4.41e+01, 3.99e+00 / 1.34e+00 | 20260505 23:55:19.230 | INFO | PerfReport: 57.31%, FrameLabeling -- 1.79e+01 / 1.95e+01, 1.45e+00 / 1.06e+00 | 20260505 23:55:19.230 | INFO | PerfReport: 7.13%, PulseAccumulating -- 2.23e+00 / 2.75e+00, 4.24e-01 / 2.26e-01 | 20260505 23:55:19.230 | INFO | PerfReport: 3.10%, Metrics -- 9.71e-01 / 1.26e+00, 2.63e-01 / 1.08e-01 | 20260505 23:55:19.230 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.36e-01 / 3.92e-01, 5.69e-02 / 2.76e-02 | 20260505 23:55:19.230 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 23:55:19.230 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:55:19.230 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:55:19.230 | INFO | PerfReport: Full Execution -- 9.84e+01 / -nan, 2.47e+01 / 0.00e+00 | 20260505 23:55:19.230 | INFO | PerfReport: 22.38%, Upload -- 2.20e+01 / 0.00e+00, 7.57e+00 / 0.00e+00 | 20260505 23:55:19.230 | INFO | PerfReport: 21.86%, Download -- 2.15e+01 / 0.00e+00, 7.62e+00 / 0.00e+00 | 20260505 23:55:19.230 | INFO | PerfReport: 31.80%, Compute -- 3.13e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 23:55:19.230 | INFO | PerfReport: 2.17%, LockUpload -- 2.14e+00 / 0.00e+00, 8.07e+00 / 0.00e+00 | 20260505 23:55:19.230 | INFO | PerfReport: 1.32%, LockDownload -- 1.30e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260505 23:55:19.230 | INFO | PerfReport: 20.45%, LockCompute -- 2.01e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260505 23:55:19.230 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.97e-02 / 0.00e+00 | 20260505 23:55:19.347 | INFO | Starting a round of Baz IO | 20260505 23:55:22.499 | INFO | Populating RTMetrics report | 20260505 23:55:22.499 | INFO | RTMetrics report populated | 20260505 23:55:22.499 | INFO | Opening tmp RTMetrics json file | 20260505 23:55:22.501 | INFO | Serializing RTMetrics to json | 20260505 23:55:22.502 | INFO | Serialization done, swapping tmp file now... | 20260505 23:55:22.503 | INFO | RTMetrics done | 20260505 23:55:22.578 | INFO | Finished a round of Baz IO: Wrote 1.877 GiB in 3.231 seconds | 20260505 23:55:22.578 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.40% / 0.00% / 78.60% | 20260505 23:55:22.578 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.100009% / 0% / 7.15256e-05% | 20260505 23:55:22.578 | INFO | Baz IO overall: | 20260505 23:55:22.578 | INFO | BytesWritten : 4170792412733 | 20260505 23:55:22.578 | INFO | NumEvents : 1963918457287 | 20260505 23:55:24.370 | INFO | Analyzing chunk frames = [7315456,7315968) | 20260505 23:55:24.370 | INFO | MemInfo: { | 20260505 23:55:24.370 | INFO | "gpu" : " 17 GB", | 20260505 23:55:24.371 | INFO | "huge" : "", | 20260505 23:55:24.371 | INFO | "huge_shared" : " 27 GB", | 20260505 23:55:24.371 | INFO | "malloc" : " 14 GB", | 20260505 23:55:24.371 | INFO | "pinned" : " 29 GB" | 20260505 23:55:24.371 | INFO | } | 20260505 23:55:24.371 | INFO | Perf report for the last 5.14151 seconds: | 20260505 23:55:24.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:55:24.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:24.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:24.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:24.371 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 23:55:24.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:24.371 | INFO | RT_METRICS : 43.4%, 1.0, 22.7ms, 0 | 20260505 23:55:24.371 | INFO | PRE_HQ : 59.8%, 1.7, 54.3ms, 0 | 20260505 23:55:24.371 | INFO | BAZWRITER : 61.9%, 3.6, 113.4ms, 0 | 20260505 23:55:24.371 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 23:55:24.371 | INFO | Uploaded 36GiB to GPU in 2.18865s (16.4485GiB/s) | 20260505 23:55:24.371 | INFO | Downloaded 38GiB to GPU in 2.24249s (16.9455GiB/s) | 20260505 23:55:29.512 | INFO | Analyzing chunk frames = [7315968,7316480) | 20260505 23:55:29.512 | INFO | MemInfo: { | 20260505 23:55:29.512 | INFO | "gpu" : " 17 GB", | 20260505 23:55:29.512 | INFO | "huge" : "", | 20260505 23:55:29.512 | INFO | "huge_shared" : " 27 GB", | 20260505 23:55:29.512 | INFO | "malloc" : " 14 GB", | 20260505 23:55:29.512 | INFO | "pinned" : " 29 GB" | 20260505 23:55:29.513 | INFO | } | 20260505 23:55:29.513 | INFO | Perf report for the last 5.14185 seconds: | 20260505 23:55:29.513 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:55:29.513 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:29.513 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:29.513 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:29.513 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260505 23:55:29.513 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:29.513 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:29.513 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:55:29.513 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:29.513 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:29.513 | INFO | Uploaded 36GiB to GPU in 1.9975s (18.0226GiB/s) | 20260505 23:55:29.513 | INFO | Downloaded 34GiB to GPU in 1.99645s (17.0302GiB/s) | 20260505 23:55:34.640 | INFO | Analyzing chunk frames = [7316480,7316992) | 20260505 23:55:34.640 | INFO | MemInfo: { | 20260505 23:55:34.640 | INFO | "gpu" : " 17 GB", | 20260505 23:55:34.640 | INFO | "huge" : "", | 20260505 23:55:34.640 | INFO | "huge_shared" : " 27 GB", | 20260505 23:55:34.640 | INFO | "malloc" : " 14 GB", | 20260505 23:55:34.640 | INFO | "pinned" : " 29 GB" | 20260505 23:55:34.640 | INFO | } | 20260505 23:55:34.640 | INFO | Perf report for the last 5.12784 seconds: | 20260505 23:55:34.640 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:55:34.640 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:34.640 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:34.640 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:34.640 | INFO | ANALYSIS : 61.4%, 3.0, 95.4ms, 0 | 20260505 23:55:34.640 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:34.640 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:34.640 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 23:55:34.640 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:34.640 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:34.641 | INFO | Uploaded 36GiB to GPU in 2.08869s (17.2357GiB/s) | 20260505 23:55:34.641 | INFO | Downloaded 34GiB to GPU in 2.02334s (16.8039GiB/s) | 20260505 23:55:39.783 | INFO | Analyzing chunk frames = [7316992,7317504) | 20260505 23:55:39.783 | INFO | MemInfo: { | 20260505 23:55:39.784 | INFO | "gpu" : " 17 GB", | 20260505 23:55:39.784 | INFO | "huge" : "", | 20260505 23:55:39.784 | INFO | "huge_shared" : " 27 GB", | 20260505 23:55:39.784 | INFO | "malloc" : " 14 GB", | 20260505 23:55:39.784 | INFO | "pinned" : " 29 GB" | 20260505 23:55:39.784 | INFO | } | 20260505 23:55:39.784 | INFO | Perf report for the last 5.14327 seconds: | 20260505 23:55:39.784 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:55:39.784 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:39.784 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:39.784 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:39.784 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260505 23:55:39.784 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:39.784 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:39.784 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:55:39.784 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:39.784 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:39.784 | INFO | Uploaded 36GiB to GPU in 2.13138s (16.8904GiB/s) | 20260505 23:55:39.784 | INFO | Downloaded 34GiB to GPU in 2.0097s (16.9179GiB/s) | 20260505 23:55:44.928 | INFO | Analyzing chunk frames = [7317504,7318016) | 20260505 23:55:44.928 | INFO | MemInfo: { | 20260505 23:55:44.928 | INFO | "gpu" : " 17 GB", | 20260505 23:55:44.928 | INFO | "huge" : "", | 20260505 23:55:44.928 | INFO | "huge_shared" : " 27 GB", | 20260505 23:55:44.928 | INFO | "malloc" : " 14 GB", | 20260505 23:55:44.928 | INFO | "pinned" : " 29 GB" | 20260505 23:55:44.928 | INFO | } | 20260505 23:55:44.928 | INFO | Perf report for the last 5.14472 seconds: | 20260505 23:55:44.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:55:44.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:44.928 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:44.928 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:44.928 | INFO | ANALYSIS : 61.1%, 3.0, 95.5ms, 0 | 20260505 23:55:44.928 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:44.928 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:44.928 | INFO | PRE_HQ : 26.8%, 1.0, 14.0ms, 0 | 20260505 23:55:44.929 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:44.929 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:44.929 | INFO | Uploaded 36GiB to GPU in 2.09348s (17.1963GiB/s) | 20260505 23:55:44.929 | INFO | Downloaded 34GiB to GPU in 2.05085s (16.5785GiB/s) | 20260505 23:55:50.065 | INFO | Analyzing chunk frames = [7318016,7318528) | 20260505 23:55:50.065 | INFO | MemInfo: { | 20260505 23:55:50.065 | INFO | "gpu" : " 17 GB", | 20260505 23:55:50.065 | INFO | "huge" : "", | 20260505 23:55:50.065 | INFO | "huge_shared" : " 27 GB", | 20260505 23:55:50.065 | INFO | "malloc" : " 14 GB", | 20260505 23:55:50.065 | INFO | "pinned" : " 29 GB" | 20260505 23:55:50.065 | INFO | } | 20260505 23:55:50.065 | INFO | Perf report for the last 5.13664 seconds: | 20260505 23:55:50.065 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:55:50.065 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:50.065 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:50.065 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:50.065 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260505 23:55:50.065 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:50.065 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:50.065 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:55:50.065 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:50.065 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:50.065 | INFO | Uploaded 36GiB to GPU in 2.07835s (17.3214GiB/s) | 20260505 23:55:50.065 | INFO | Downloaded 34GiB to GPU in 2.04331s (16.6397GiB/s) | 20260505 23:55:55.201 | INFO | Analyzing chunk frames = [7318528,7319040) | 20260505 23:55:55.201 | INFO | MemInfo: { | 20260505 23:55:55.201 | INFO | "gpu" : " 17 GB", | 20260505 23:55:55.201 | INFO | "huge" : "", | 20260505 23:55:55.201 | INFO | "huge_shared" : " 27 GB", | 20260505 23:55:55.201 | INFO | "malloc" : " 14 GB", | 20260505 23:55:55.201 | INFO | "pinned" : " 29 GB" | 20260505 23:55:55.201 | INFO | } | 20260505 23:55:55.201 | INFO | Perf report for the last 5.13593 seconds: | 20260505 23:55:55.201 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:55:55.201 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:55.201 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:55.201 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:55.201 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 23:55:55.201 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:55.201 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:55.201 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 23:55:55.201 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:55.201 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:55:55.201 | INFO | Uploaded 36GiB to GPU in 2.03698s (17.6732GiB/s) | 20260505 23:55:55.201 | INFO | Downloaded 34GiB to GPU in 2.05487s (16.546GiB/s) | 20260505 23:56:00.339 | INFO | Analyzing chunk frames = [7319040,7319552) | 20260505 23:56:00.339 | INFO | MemInfo: { | 20260505 23:56:00.339 | INFO | "gpu" : " 17 GB", | 20260505 23:56:00.339 | INFO | "huge" : "", | 20260505 23:56:00.339 | INFO | "huge_shared" : " 27 GB", | 20260505 23:56:00.339 | INFO | "malloc" : " 14 GB", | 20260505 23:56:00.339 | INFO | "pinned" : " 29 GB" | 20260505 23:56:00.339 | INFO | } | 20260505 23:56:00.339 | INFO | Perf report for the last 5.13799 seconds: | 20260505 23:56:00.339 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:56:00.339 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:00.339 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:00.339 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:00.339 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 23:56:00.339 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:00.339 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:00.339 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:56:00.339 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:00.339 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:00.339 | INFO | Uploaded 36GiB to GPU in 2.11953s (16.9849GiB/s) | 20260505 23:56:00.339 | INFO | Downloaded 34GiB to GPU in 2.00898s (16.924GiB/s) | 20260505 23:56:00.459 | INFO | Starting a round of Baz IO | 20260505 23:56:03.607 | INFO | Populating RTMetrics report | 20260505 23:56:03.607 | INFO | RTMetrics report populated | 20260505 23:56:03.607 | INFO | Opening tmp RTMetrics json file | 20260505 23:56:03.612 | INFO | Serializing RTMetrics to json | 20260505 23:56:03.614 | INFO | Serialization done, swapping tmp file now... | 20260505 23:56:03.616 | INFO | RTMetrics done | 20260505 23:56:03.687 | INFO | Finished a round of Baz IO: Wrote 1.876 GiB in 3.227 seconds | 20260505 23:56:03.687 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.40% / 0.00% / 78.60% | 20260505 23:56:03.687 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.100279% / 0% / 8.74201e-05% | 20260505 23:56:03.687 | INFO | Baz IO overall: | 20260505 23:56:03.687 | INFO | BytesWritten : 4172806324335 | 20260505 23:56:03.687 | INFO | NumEvents : 1964840233741 | 20260505 23:56:05.485 | INFO | Analyzing chunk frames = [7319552,7320064) | 20260505 23:56:05.485 | INFO | MemInfo: { | 20260505 23:56:05.485 | INFO | "gpu" : " 17 GB", | 20260505 23:56:05.485 | INFO | "huge" : "", | 20260505 23:56:05.485 | INFO | "huge_shared" : " 27 GB", | 20260505 23:56:05.485 | INFO | "malloc" : " 14 GB", | 20260505 23:56:05.485 | INFO | "pinned" : " 29 GB" | 20260505 23:56:05.485 | INFO | } | 20260505 23:56:05.485 | INFO | Perf report for the last 5.14579 seconds: | 20260505 23:56:05.485 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:56:05.485 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:05.485 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:05.485 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:05.485 | INFO | ANALYSIS : 62.4%, 3.0, 97.6ms, 0 | 20260505 23:56:05.485 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:05.485 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260505 23:56:05.485 | INFO | PRE_HQ : 60.0%, 1.7, 54.4ms, 0 | 20260505 23:56:05.485 | INFO | BAZWRITER : 61.9%, 3.6, 113.8ms, 0 | 20260505 23:56:05.485 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260505 23:56:05.485 | INFO | Uploaded 36GiB to GPU in 2.16616s (16.6193GiB/s) | 20260505 23:56:05.485 | INFO | Downloaded 38GiB to GPU in 2.23502s (17.0021GiB/s) | 20260505 23:56:10.623 | INFO | Analyzing chunk frames = [7320064,7320576) | 20260505 23:56:10.623 | INFO | MemInfo: { | 20260505 23:56:10.623 | INFO | "gpu" : " 17 GB", | 20260505 23:56:10.623 | INFO | "huge" : "", | 20260505 23:56:10.623 | INFO | "huge_shared" : " 27 GB", | 20260505 23:56:10.623 | INFO | "malloc" : " 14 GB", | 20260505 23:56:10.623 | INFO | "pinned" : " 29 GB" | 20260505 23:56:10.623 | INFO | } | 20260505 23:56:10.623 | INFO | Perf report for the last 5.13852 seconds: | 20260505 23:56:10.623 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:56:10.623 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:10.623 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:10.623 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:10.623 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260505 23:56:10.623 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:10.623 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:10.623 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260505 23:56:10.623 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:10.623 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:10.624 | INFO | Uploaded 36GiB to GPU in 2.10518s (17.1007GiB/s) | 20260505 23:56:10.624 | INFO | Downloaded 34GiB to GPU in 2.04106s (16.658GiB/s) | 20260505 23:56:15.757 | INFO | Analyzing chunk frames = [7320576,7321088) | 20260505 23:56:15.757 | INFO | MemInfo: { | 20260505 23:56:15.757 | INFO | "gpu" : " 17 GB", | 20260505 23:56:15.757 | INFO | "huge" : "", | 20260505 23:56:15.757 | INFO | "huge_shared" : " 27 GB", | 20260505 23:56:15.757 | INFO | "malloc" : " 14 GB", | 20260505 23:56:15.757 | INFO | "pinned" : " 29 GB" | 20260505 23:56:15.757 | INFO | } | 20260505 23:56:15.757 | INFO | Perf report for the last 5.13412 seconds: | 20260505 23:56:15.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:56:15.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:15.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:15.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:15.758 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260505 23:56:15.758 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:15.758 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:15.758 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:56:15.758 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:15.758 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:15.758 | INFO | Uploaded 36GiB to GPU in 2.07163s (17.3777GiB/s) | 20260505 23:56:15.758 | INFO | Downloaded 34GiB to GPU in 2.0198s (16.8334GiB/s) | 20260505 23:56:20.894 | INFO | Analyzing chunk frames = [7321088,7321600) | 20260505 23:56:20.894 | INFO | MemInfo: { | 20260505 23:56:20.894 | INFO | "gpu" : " 17 GB", | 20260505 23:56:20.894 | INFO | "huge" : "", | 20260505 23:56:20.894 | INFO | "huge_shared" : " 27 GB", | 20260505 23:56:20.894 | INFO | "malloc" : " 14 GB", | 20260505 23:56:20.894 | INFO | "pinned" : " 29 GB" | 20260505 23:56:20.894 | INFO | } | 20260505 23:56:20.894 | INFO | Perf report for the last 5.13648 seconds: | 20260505 23:56:20.894 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:56:20.894 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:20.894 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:20.894 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:20.894 | INFO | ANALYSIS : 61.8%, 3.0, 95.9ms, 0 | 20260505 23:56:20.894 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:20.894 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:20.894 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260505 23:56:20.894 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:20.894 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:20.894 | INFO | Uploaded 36GiB to GPU in 2.12339s (16.954GiB/s) | 20260505 23:56:20.894 | INFO | Downloaded 34GiB to GPU in 2.01789s (16.8493GiB/s) | 20260505 23:56:26.036 | INFO | Analyzing chunk frames = [7321600,7322112) | 20260505 23:56:26.036 | INFO | MemInfo: { | 20260505 23:56:26.036 | INFO | "gpu" : " 17 GB", | 20260505 23:56:26.036 | INFO | "huge" : "", | 20260505 23:56:26.036 | INFO | "huge_shared" : " 27 GB", | 20260505 23:56:26.036 | INFO | "malloc" : " 14 GB", | 20260505 23:56:26.036 | INFO | "pinned" : " 29 GB" | 20260505 23:56:26.036 | INFO | } | 20260505 23:56:26.036 | INFO | Perf report for the last 5.14213 seconds: | 20260505 23:56:26.036 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:56:26.036 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:26.036 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:26.036 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:26.036 | INFO | ANALYSIS : 61.4%, 3.0, 95.9ms, 0 | 20260505 23:56:26.036 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:26.036 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:26.036 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 23:56:26.036 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:26.036 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:26.036 | INFO | Uploaded 36GiB to GPU in 2.12366s (16.9519GiB/s) | 20260505 23:56:26.036 | INFO | Downloaded 34GiB to GPU in 2.06492s (16.4655GiB/s) | 20260505 23:56:31.187 | INFO | Analyzing chunk frames = [7322112,7322624) | 20260505 23:56:31.187 | INFO | MemInfo: { | 20260505 23:56:31.187 | INFO | "gpu" : " 17 GB", | 20260505 23:56:31.187 | INFO | "huge" : "", | 20260505 23:56:31.187 | INFO | "huge_shared" : " 27 GB", | 20260505 23:56:31.187 | INFO | "malloc" : " 14 GB", | 20260505 23:56:31.187 | INFO | "pinned" : " 29 GB" | 20260505 23:56:31.187 | INFO | } | 20260505 23:56:31.187 | INFO | Perf report for the last 5.15094 seconds: | 20260505 23:56:31.187 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:56:31.187 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:31.187 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:31.187 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:31.187 | INFO | ANALYSIS : 61.3%, 3.0, 95.9ms, 0 | 20260505 23:56:31.187 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:31.187 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:31.187 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:56:31.187 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:31.187 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:31.187 | INFO | Uploaded 36GiB to GPU in 2.07765s (17.3273GiB/s) | 20260505 23:56:31.187 | INFO | Downloaded 34GiB to GPU in 2.04172s (16.6526GiB/s) | 20260505 23:56:36.322 | INFO | Analyzing chunk frames = [7322624,7323136) | 20260505 23:56:36.322 | INFO | MemInfo: { | 20260505 23:56:36.323 | INFO | "gpu" : " 17 GB", | 20260505 23:56:36.323 | INFO | "huge" : "", | 20260505 23:56:36.323 | INFO | "huge_shared" : " 27 GB", | 20260505 23:56:36.323 | INFO | "malloc" : " 14 GB", | 20260505 23:56:36.323 | INFO | "pinned" : " 29 GB" | 20260505 23:56:36.323 | INFO | } | 20260505 23:56:36.323 | INFO | Perf report for the last 5.13555 seconds: | 20260505 23:56:36.323 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:56:36.323 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:36.323 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:36.323 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:36.323 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260505 23:56:36.323 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:36.323 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:36.323 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 23:56:36.323 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:36.323 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:36.323 | INFO | Uploaded 36GiB to GPU in 2.07424s (17.3557GiB/s) | 20260505 23:56:36.323 | INFO | Downloaded 34GiB to GPU in 2.06752s (16.4448GiB/s) | 20260505 23:56:41.459 | INFO | Analyzing chunk frames = [7323136,7323648) | 20260505 23:56:41.459 | INFO | MemInfo: { | 20260505 23:56:41.459 | INFO | "gpu" : " 17 GB", | 20260505 23:56:41.459 | INFO | "huge" : "", | 20260505 23:56:41.459 | INFO | "huge_shared" : " 27 GB", | 20260505 23:56:41.459 | INFO | "malloc" : " 14 GB", | 20260505 23:56:41.459 | INFO | "pinned" : " 29 GB" | 20260505 23:56:41.459 | INFO | } | 20260505 23:56:41.459 | INFO | Perf report for the last 5.13671 seconds: | 20260505 23:56:41.459 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:56:41.459 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:41.459 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:41.459 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:41.459 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260505 23:56:41.459 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:41.459 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:41.459 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 23:56:41.459 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:41.460 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:41.460 | INFO | Uploaded 36GiB to GPU in 2.13457s (16.8653GiB/s) | 20260505 23:56:41.460 | INFO | Downloaded 34GiB to GPU in 2.03029s (16.7464GiB/s) | 20260505 23:56:41.460 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 23:56:41.460 | INFO | Total mem usage: 15.3 GiB | 20260505 23:56:41.460 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 23:56:41.460 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.13%) | 20260505 23:56:41.460 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 23:56:41.460 | INFO | High Water Breakdown: | 20260505 23:56:41.460 | INFO | 60.9% : 8.55 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 23:56:41.460 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 23:56:41.460 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 23:56:41.460 | INFO | ------------------------------------------ | 20260505 23:56:41.460 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:56:41.460 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 23:56:41.460 | INFO | Total mem usage: 26.5 GiB | 20260505 23:56:41.460 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 23:56:41.460 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 23:56:41.460 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 23:56:41.460 | INFO | High Water Breakdown: | 20260505 23:56:41.460 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:56:41.460 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:56:41.460 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:56:41.460 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:56:41.460 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:56:41.460 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:56:41.460 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 23:56:41.460 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 23:56:41.460 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:56:41.460 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 23:56:41.460 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 23:56:41.460 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 23:56:41.460 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 23:56:41.460 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 23:56:41.460 | INFO | ------------------------------------------ | 20260505 23:56:41.460 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:56:41.460 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 23:56:41.460 | INFO | Total mem usage: 25.6 GiB | 20260505 23:56:41.460 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 23:56:41.460 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260505 23:56:41.460 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260505 23:56:41.460 | INFO | High Water Breakdown: | 20260505 23:56:41.460 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260505 23:56:41.460 | INFO | ------------------------------------------ | 20260505 23:56:41.460 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:56:41.460 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 23:56:41.460 | INFO | Total mem usage: 15.7 GiB | 20260505 23:56:41.460 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 23:56:41.460 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 23:56:41.460 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 23:56:41.460 | INFO | High Water Breakdown: | 20260505 23:56:41.460 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 23:56:41.460 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 23:56:41.460 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 23:56:41.460 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 23:56:41.460 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 23:56:41.460 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 23:56:41.461 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 23:56:41.461 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 23:56:41.461 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 23:56:41.461 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:56:41.461 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 23:56:41.461 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:56:41.461 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 23:56:41.461 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 23:56:41.461 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 23:56:41.461 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 23:56:41.461 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:56:41.461 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 23:56:41.461 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:56:41.461 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:56:41.461 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:56:41.461 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 23:56:41.461 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 23:56:41.461 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:56:41.461 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 23:56:41.461 | INFO | ------------------------------------------ | 20260505 23:56:41.461 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:56:41.461 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 23:56:41.461 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:56:41.461 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:56:41.461 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.39e+01, 1.34e+01 / 7.80e-01 | 20260505 23:56:41.461 | INFO | PerfReport: 0.29%, AggregateStats -- 9.18e-02 / 1.04e-01, 9.99e-03 / 2.67e-03 | 20260505 23:56:41.461 | INFO | PerfReport: 11.84%, Baseline -- 3.71e+00 / 4.47e+00, 5.75e-01 / 2.52e-01 | 20260505 23:56:41.461 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.52e+00, 1.91e-01 / 1.08e-01 | 20260505 23:56:41.461 | INFO | PerfReport: 16.00%, DME -- 4.01e+01 / 4.40e+01, 3.95e+00 / 1.14e+00 | 20260505 23:56:41.461 | INFO | PerfReport: 57.32%, FrameLabeling -- 1.80e+01 / 1.93e+01, 1.50e+00 / 1.16e+00 | 20260505 23:56:41.461 | INFO | PerfReport: 7.10%, PulseAccumulating -- 2.23e+00 / 2.76e+00, 4.28e-01 / 2.58e-01 | 20260505 23:56:41.461 | INFO | PerfReport: 3.10%, Metrics -- 9.70e-01 / 1.24e+00, 2.61e-01 / 1.03e-01 | 20260505 23:56:41.461 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.36e-01 / 3.91e-01, 5.79e-02 / 3.59e-02 | 20260505 23:56:41.461 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 23:56:41.461 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:56:41.461 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:56:41.461 | INFO | PerfReport: Full Execution -- 9.83e+01 / -nan, 2.46e+01 / 0.00e+00 | 20260505 23:56:41.461 | INFO | PerfReport: 22.27%, Upload -- 2.19e+01 / 0.00e+00, 7.47e+00 / 0.00e+00 | 20260505 23:56:41.461 | INFO | PerfReport: 21.83%, Download -- 2.15e+01 / 0.00e+00, 7.61e+00 / 0.00e+00 | 20260505 23:56:41.461 | INFO | PerfReport: 31.88%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 23:56:41.461 | INFO | PerfReport: 2.10%, LockUpload -- 2.06e+00 / 0.00e+00, 7.81e+00 / 0.00e+00 | 20260505 23:56:41.461 | INFO | PerfReport: 1.32%, LockDownload -- 1.30e+00 / 0.00e+00, 2.89e+00 / 0.00e+00 | 20260505 23:56:41.461 | INFO | PerfReport: 20.59%, LockCompute -- 2.02e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260505 23:56:41.461 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.95e-02 / 0.00e+00 | 20260505 23:56:41.581 | INFO | Starting a round of Baz IO | 20260505 23:56:44.727 | INFO | Populating RTMetrics report | 20260505 23:56:44.728 | INFO | RTMetrics report populated | 20260505 23:56:44.728 | INFO | Opening tmp RTMetrics json file | 20260505 23:56:44.730 | INFO | Serializing RTMetrics to json | 20260505 23:56:44.731 | INFO | Serialization done, swapping tmp file now... | 20260505 23:56:44.731 | INFO | RTMetrics done | 20260505 23:56:44.809 | INFO | Finished a round of Baz IO: Wrote 1.876 GiB in 3.228 seconds | 20260505 23:56:44.809 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.40% / 0.00% / 78.60% | 20260505 23:56:44.809 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0996033% / 0% / 7.15256e-05% | 20260505 23:56:44.809 | INFO | Baz IO overall: | 20260505 23:56:44.809 | INFO | BytesWritten : 4174820368022 | 20260505 23:56:44.809 | INFO | NumEvents : 1965762385985 | 20260505 23:56:46.591 | INFO | Analyzing chunk frames = [7323648,7324160) | 20260505 23:56:46.591 | INFO | MemInfo: { | 20260505 23:56:46.591 | INFO | "gpu" : " 17 GB", | 20260505 23:56:46.591 | INFO | "huge" : "", | 20260505 23:56:46.591 | INFO | "huge_shared" : " 27 GB", | 20260505 23:56:46.591 | INFO | "malloc" : " 14 GB", | 20260505 23:56:46.591 | INFO | "pinned" : " 29 GB" | 20260505 23:56:46.592 | INFO | } | 20260505 23:56:46.592 | INFO | Perf report for the last 5.13218 seconds: | 20260505 23:56:46.592 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:56:46.592 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:46.592 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:46.592 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:46.592 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 23:56:46.592 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:46.592 | INFO | RT_METRICS : 43.6%, 1.0, 22.8ms, 0 | 20260505 23:56:46.592 | INFO | PRE_HQ : 60.0%, 1.8, 54.9ms, 0 | 20260505 23:56:46.592 | INFO | BAZWRITER : 62.1%, 3.6, 114.9ms, 0 | 20260505 23:56:46.592 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 23:56:46.592 | INFO | Uploaded 36GiB to GPU in 2.13253s (16.8814GiB/s) | 20260505 23:56:46.592 | INFO | Downloaded 38GiB to GPU in 2.22108s (17.1088GiB/s) | 20260505 23:56:51.735 | INFO | Analyzing chunk frames = [7324160,7324672) | 20260505 23:56:51.735 | INFO | MemInfo: { | 20260505 23:56:51.735 | INFO | "gpu" : " 17 GB", | 20260505 23:56:51.735 | INFO | "huge" : "", | 20260505 23:56:51.735 | INFO | "huge_shared" : " 27 GB", | 20260505 23:56:51.735 | INFO | "malloc" : " 14 GB", | 20260505 23:56:51.735 | INFO | "pinned" : " 29 GB" | 20260505 23:56:51.735 | INFO | } | 20260505 23:56:51.735 | INFO | Perf report for the last 5.14381 seconds: | 20260505 23:56:51.735 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:56:51.735 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:51.735 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:51.735 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:51.735 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260505 23:56:51.736 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:51.736 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:51.736 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260505 23:56:51.736 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:51.736 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:51.736 | INFO | Uploaded 36GiB to GPU in 2.09237s (17.2054GiB/s) | 20260505 23:56:51.736 | INFO | Downloaded 34GiB to GPU in 2.03263s (16.7271GiB/s) | 20260505 23:56:56.881 | INFO | Analyzing chunk frames = [7324672,7325184) | 20260505 23:56:56.881 | INFO | MemInfo: { | 20260505 23:56:56.881 | INFO | "gpu" : " 17 GB", | 20260505 23:56:56.881 | INFO | "huge" : "", | 20260505 23:56:56.881 | INFO | "huge_shared" : " 27 GB", | 20260505 23:56:56.881 | INFO | "malloc" : " 14 GB", | 20260505 23:56:56.881 | INFO | "pinned" : " 29 GB" | 20260505 23:56:56.881 | INFO | } | 20260505 23:56:56.881 | INFO | Perf report for the last 5.14558 seconds: | 20260505 23:56:56.881 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:56:56.881 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:56.881 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:56.881 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:56.881 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260505 23:56:56.881 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:56.881 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:56.881 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260505 23:56:56.881 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:56.881 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:56:56.881 | INFO | Uploaded 36GiB to GPU in 2.08846s (17.2375GiB/s) | 20260505 23:56:56.881 | INFO | Downloaded 34GiB to GPU in 2.0265s (16.7777GiB/s) | 20260505 23:57:02.017 | INFO | Analyzing chunk frames = [7325184,7325696) | 20260505 23:57:02.017 | INFO | MemInfo: { | 20260505 23:57:02.017 | INFO | "gpu" : " 17 GB", | 20260505 23:57:02.017 | INFO | "huge" : "", | 20260505 23:57:02.017 | INFO | "huge_shared" : " 27 GB", | 20260505 23:57:02.017 | INFO | "malloc" : " 14 GB", | 20260505 23:57:02.017 | INFO | "pinned" : " 29 GB" | 20260505 23:57:02.017 | INFO | } | 20260505 23:57:02.017 | INFO | Perf report for the last 5.13572 seconds: | 20260505 23:57:02.017 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:57:02.017 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:02.017 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:02.017 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:02.017 | INFO | ANALYSIS : 61.7%, 3.0, 95.8ms, 0 | 20260505 23:57:02.017 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:02.017 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:02.017 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 23:57:02.017 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:02.017 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:02.017 | INFO | Uploaded 36GiB to GPU in 2.13215s (16.8844GiB/s) | 20260505 23:57:02.017 | INFO | Downloaded 34GiB to GPU in 2.01545s (16.8697GiB/s) | 20260505 23:57:07.152 | INFO | Analyzing chunk frames = [7325696,7326208) | 20260505 23:57:07.152 | INFO | MemInfo: { | 20260505 23:57:07.152 | INFO | "gpu" : " 17 GB", | 20260505 23:57:07.152 | INFO | "huge" : "", | 20260505 23:57:07.152 | INFO | "huge_shared" : " 27 GB", | 20260505 23:57:07.152 | INFO | "malloc" : " 14 GB", | 20260505 23:57:07.152 | INFO | "pinned" : " 29 GB" | 20260505 23:57:07.152 | INFO | } | 20260505 23:57:07.152 | INFO | Perf report for the last 5.13531 seconds: | 20260505 23:57:07.152 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:57:07.152 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:07.152 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:07.152 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:07.152 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260505 23:57:07.152 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:07.152 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:07.152 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260505 23:57:07.152 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:07.152 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:07.152 | INFO | Uploaded 36GiB to GPU in 2.09675s (17.1694GiB/s) | 20260505 23:57:07.152 | INFO | Downloaded 34GiB to GPU in 2.04768s (16.6042GiB/s) | 20260505 23:57:12.294 | INFO | Analyzing chunk frames = [7326208,7326720) | 20260505 23:57:12.294 | INFO | MemInfo: { | 20260505 23:57:12.294 | INFO | "gpu" : " 17 GB", | 20260505 23:57:12.294 | INFO | "huge" : "", | 20260505 23:57:12.294 | INFO | "huge_shared" : " 27 GB", | 20260505 23:57:12.294 | INFO | "malloc" : " 14 GB", | 20260505 23:57:12.294 | INFO | "pinned" : " 29 GB" | 20260505 23:57:12.294 | INFO | } | 20260505 23:57:12.294 | INFO | Perf report for the last 5.14197 seconds: | 20260505 23:57:12.294 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:57:12.294 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:12.294 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:12.294 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:12.294 | INFO | ANALYSIS : 61.5%, 3.0, 96.0ms, 0 | 20260505 23:57:12.294 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:12.294 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:12.294 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 23:57:12.294 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:12.294 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:12.294 | INFO | Uploaded 36GiB to GPU in 2.07633s (17.3383GiB/s) | 20260505 23:57:12.294 | INFO | Downloaded 34GiB to GPU in 2.04013s (16.6656GiB/s) | 20260505 23:57:17.434 | INFO | Analyzing chunk frames = [7326720,7327232) | 20260505 23:57:17.434 | INFO | MemInfo: { | 20260505 23:57:17.434 | INFO | "gpu" : " 17 GB", | 20260505 23:57:17.434 | INFO | "huge" : "", | 20260505 23:57:17.434 | INFO | "huge_shared" : " 27 GB", | 20260505 23:57:17.434 | INFO | "malloc" : " 14 GB", | 20260505 23:57:17.434 | INFO | "pinned" : " 29 GB" | 20260505 23:57:17.434 | INFO | } | 20260505 23:57:17.434 | INFO | Perf report for the last 5.14029 seconds: | 20260505 23:57:17.434 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:57:17.434 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:17.434 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:17.434 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:17.434 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260505 23:57:17.434 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:17.434 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:17.434 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260505 23:57:17.434 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:17.434 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:17.435 | INFO | Uploaded 36GiB to GPU in 2.0647s (17.4359GiB/s) | 20260505 23:57:17.435 | INFO | Downloaded 34GiB to GPU in 2.06878s (16.4348GiB/s) | 20260505 23:57:22.569 | INFO | Analyzing chunk frames = [7327232,7327744) | 20260505 23:57:22.569 | INFO | MemInfo: { | 20260505 23:57:22.569 | INFO | "gpu" : " 17 GB", | 20260505 23:57:22.569 | INFO | "huge" : "", | 20260505 23:57:22.569 | INFO | "huge_shared" : " 27 GB", | 20260505 23:57:22.569 | INFO | "malloc" : " 14 GB", | 20260505 23:57:22.569 | INFO | "pinned" : " 29 GB" | 20260505 23:57:22.569 | INFO | } | 20260505 23:57:22.569 | INFO | Perf report for the last 5.13488 seconds: | 20260505 23:57:22.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:57:22.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:22.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:22.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:22.569 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260505 23:57:22.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:22.569 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:22.569 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:57:22.569 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:22.569 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:22.569 | INFO | Uploaded 36GiB to GPU in 2.1351s (16.861GiB/s) | 20260505 23:57:22.569 | INFO | Downloaded 34GiB to GPU in 2.02649s (16.7778GiB/s) | 20260505 23:57:22.693 | INFO | Starting a round of Baz IO | 20260505 23:57:25.839 | INFO | Populating RTMetrics report | 20260505 23:57:25.839 | INFO | RTMetrics report populated | 20260505 23:57:25.839 | INFO | Opening tmp RTMetrics json file | 20260505 23:57:25.841 | INFO | Serializing RTMetrics to json | 20260505 23:57:25.844 | INFO | Serialization done, swapping tmp file now... | 20260505 23:57:25.845 | INFO | RTMetrics done | 20260505 23:57:25.922 | INFO | Finished a round of Baz IO: Wrote 1.876 GiB in 3.229 seconds | 20260505 23:57:25.922 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.40% / 0.00% / 78.60% | 20260505 23:57:25.922 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0996232% / 7.94729e-06% / 6.75519e-05% | 20260505 23:57:25.922 | INFO | Baz IO overall: | 20260505 23:57:25.922 | INFO | BytesWritten : 4176834309381 | 20260505 23:57:25.922 | INFO | NumEvents : 1966684551084 | 20260505 23:57:27.704 | INFO | Analyzing chunk frames = [7327744,7328256) | 20260505 23:57:27.704 | INFO | MemInfo: { | 20260505 23:57:27.704 | INFO | "gpu" : " 17 GB", | 20260505 23:57:27.704 | INFO | "huge" : "", | 20260505 23:57:27.704 | INFO | "huge_shared" : " 27 GB", | 20260505 23:57:27.704 | INFO | "malloc" : " 14 GB", | 20260505 23:57:27.704 | INFO | "pinned" : " 29 GB" | 20260505 23:57:27.704 | INFO | } | 20260505 23:57:27.704 | INFO | Perf report for the last 5.13465 seconds: | 20260505 23:57:27.704 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:57:27.704 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:27.704 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:27.704 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:27.704 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260505 23:57:27.704 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:27.704 | INFO | RT_METRICS : 43.4%, 1.0, 22.7ms, 0 | 20260505 23:57:27.704 | INFO | PRE_HQ : 60.0%, 1.7, 54.4ms, 0 | 20260505 23:57:27.704 | INFO | BAZWRITER : 62.1%, 3.5, 113.0ms, 0 | 20260505 23:57:27.704 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 23:57:27.704 | INFO | Uploaded 36GiB to GPU in 2.17247s (16.571GiB/s) | 20260505 23:57:27.704 | INFO | Downloaded 38GiB to GPU in 2.24189s (16.95GiB/s) | 20260505 23:57:32.848 | INFO | Analyzing chunk frames = [7328256,7328768) | 20260505 23:57:32.848 | INFO | MemInfo: { | 20260505 23:57:32.848 | INFO | "gpu" : " 17 GB", | 20260505 23:57:32.848 | INFO | "huge" : "", | 20260505 23:57:32.848 | INFO | "huge_shared" : " 27 GB", | 20260505 23:57:32.848 | INFO | "malloc" : " 14 GB", | 20260505 23:57:32.848 | INFO | "pinned" : " 29 GB" | 20260505 23:57:32.848 | INFO | } | 20260505 23:57:32.848 | INFO | Perf report for the last 5.14398 seconds: | 20260505 23:57:32.848 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:57:32.848 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:32.848 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:32.848 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:32.848 | INFO | ANALYSIS : 61.8%, 3.0, 96.6ms, 0 | 20260505 23:57:32.848 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:32.848 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:32.848 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:57:32.848 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:32.848 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:32.848 | INFO | Uploaded 36GiB to GPU in 2.0892s (17.2315GiB/s) | 20260505 23:57:32.848 | INFO | Downloaded 34GiB to GPU in 2.02859s (16.7604GiB/s) | 20260505 23:57:37.989 | INFO | Analyzing chunk frames = [7328768,7329280) | 20260505 23:57:37.989 | INFO | MemInfo: { | 20260505 23:57:37.989 | INFO | "gpu" : " 17 GB", | 20260505 23:57:37.989 | INFO | "huge" : "", | 20260505 23:57:37.989 | INFO | "huge_shared" : " 27 GB", | 20260505 23:57:37.989 | INFO | "malloc" : " 14 GB", | 20260505 23:57:37.989 | INFO | "pinned" : " 29 GB" | 20260505 23:57:37.989 | INFO | } | 20260505 23:57:37.989 | INFO | Perf report for the last 5.1411 seconds: | 20260505 23:57:37.989 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:57:37.989 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:37.989 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:37.989 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:37.989 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260505 23:57:37.989 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:37.989 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:37.989 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:57:37.989 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:37.989 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:37.989 | INFO | Uploaded 36GiB to GPU in 2.08439s (17.2712GiB/s) | 20260505 23:57:37.989 | INFO | Downloaded 34GiB to GPU in 2.03085s (16.7417GiB/s) | 20260505 23:57:43.137 | INFO | Analyzing chunk frames = [7329280,7329792) | 20260505 23:57:43.137 | INFO | MemInfo: { | 20260505 23:57:43.137 | INFO | "gpu" : " 17 GB", | 20260505 23:57:43.137 | INFO | "huge" : "", | 20260505 23:57:43.137 | INFO | "huge_shared" : " 27 GB", | 20260505 23:57:43.137 | INFO | "malloc" : " 14 GB", | 20260505 23:57:43.137 | INFO | "pinned" : " 29 GB" | 20260505 23:57:43.137 | INFO | } | 20260505 23:57:43.137 | INFO | Perf report for the last 5.14785 seconds: | 20260505 23:57:43.137 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:57:43.137 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:43.137 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:43.137 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:43.137 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260505 23:57:43.137 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:43.137 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:43.137 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:57:43.137 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:43.137 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:43.137 | INFO | Uploaded 36GiB to GPU in 2.13686s (16.8472GiB/s) | 20260505 23:57:43.137 | INFO | Downloaded 34GiB to GPU in 2.02359s (16.8018GiB/s) | 20260505 23:57:48.273 | INFO | Analyzing chunk frames = [7329792,7330304) | 20260505 23:57:48.273 | INFO | MemInfo: { | 20260505 23:57:48.273 | INFO | "gpu" : " 17 GB", | 20260505 23:57:48.273 | INFO | "huge" : "", | 20260505 23:57:48.273 | INFO | "huge_shared" : " 27 GB", | 20260505 23:57:48.273 | INFO | "malloc" : " 14 GB", | 20260505 23:57:48.273 | INFO | "pinned" : " 29 GB" | 20260505 23:57:48.273 | INFO | } | 20260505 23:57:48.273 | INFO | Perf report for the last 5.13614 seconds: | 20260505 23:57:48.273 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:57:48.273 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:48.273 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:48.273 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:48.273 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260505 23:57:48.273 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:48.273 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:48.273 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260505 23:57:48.273 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:48.273 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:48.273 | INFO | Uploaded 36GiB to GPU in 2.09368s (17.1946GiB/s) | 20260505 23:57:48.273 | INFO | Downloaded 34GiB to GPU in 2.05186s (16.5703GiB/s) | 20260505 23:57:53.406 | INFO | Analyzing chunk frames = [7330304,7330816) | 20260505 23:57:53.406 | INFO | MemInfo: { | 20260505 23:57:53.406 | INFO | "gpu" : " 17 GB", | 20260505 23:57:53.406 | INFO | "huge" : "", | 20260505 23:57:53.407 | INFO | "huge_shared" : " 27 GB", | 20260505 23:57:53.407 | INFO | "malloc" : " 14 GB", | 20260505 23:57:53.407 | INFO | "pinned" : " 29 GB" | 20260505 23:57:53.407 | INFO | } | 20260505 23:57:53.407 | INFO | Perf report for the last 5.13356 seconds: | 20260505 23:57:53.407 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:57:53.407 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:53.407 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:53.407 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:53.407 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260505 23:57:53.407 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:53.407 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:53.407 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260505 23:57:53.407 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:53.407 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:53.407 | INFO | Uploaded 36GiB to GPU in 2.07937s (17.313GiB/s) | 20260505 23:57:53.407 | INFO | Downloaded 34GiB to GPU in 2.04695s (16.61GiB/s) | 20260505 23:57:58.548 | INFO | Analyzing chunk frames = [7330816,7331328) | 20260505 23:57:58.548 | INFO | MemInfo: { | 20260505 23:57:58.548 | INFO | "gpu" : " 17 GB", | 20260505 23:57:58.548 | INFO | "huge" : "", | 20260505 23:57:58.548 | INFO | "huge_shared" : " 27 GB", | 20260505 23:57:58.548 | INFO | "malloc" : " 14 GB", | 20260505 23:57:58.548 | INFO | "pinned" : " 29 GB" | 20260505 23:57:58.548 | INFO | } | 20260505 23:57:58.548 | INFO | Perf report for the last 5.14143 seconds: | 20260505 23:57:58.548 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:57:58.548 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:58.548 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:58.548 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:58.548 | INFO | ANALYSIS : 62.5%, 3.0, 97.7ms, 0 | 20260505 23:57:58.548 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:58.548 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:58.548 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260505 23:57:58.548 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:58.548 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:57:58.548 | INFO | Uploaded 36GiB to GPU in 2.11276s (17.0393GiB/s) | 20260505 23:57:58.548 | INFO | Downloaded 34GiB to GPU in 2.07563s (16.3805GiB/s) | 20260505 23:58:03.685 | INFO | Analyzing chunk frames = [7331328,7331840) | 20260505 23:58:03.685 | INFO | MemInfo: { | 20260505 23:58:03.685 | INFO | "gpu" : " 17 GB", | 20260505 23:58:03.685 | INFO | "huge" : "", | 20260505 23:58:03.685 | INFO | "huge_shared" : " 27 GB", | 20260505 23:58:03.685 | INFO | "malloc" : " 14 GB", | 20260505 23:58:03.685 | INFO | "pinned" : " 29 GB" | 20260505 23:58:03.685 | INFO | } | 20260505 23:58:03.685 | INFO | Perf report for the last 5.13705 seconds: | 20260505 23:58:03.685 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:58:03.685 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:03.685 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:03.685 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:03.685 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 23:58:03.685 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:03.685 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:03.685 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260505 23:58:03.685 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:03.685 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:03.685 | INFO | Uploaded 36GiB to GPU in 2.13766s (16.8409GiB/s) | 20260505 23:58:03.685 | INFO | Downloaded 34GiB to GPU in 2.02813s (16.7642GiB/s) | 20260505 23:58:03.686 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 23:58:03.686 | INFO | Total mem usage: 15.3 GiB | 20260505 23:58:03.686 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 23:58:03.686 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.13%) | 20260505 23:58:03.686 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 23:58:03.686 | INFO | High Water Breakdown: | 20260505 23:58:03.686 | INFO | 60.9% : 8.55 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 23:58:03.686 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 23:58:03.686 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 23:58:03.686 | INFO | ------------------------------------------ | 20260505 23:58:03.686 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:58:03.686 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 23:58:03.686 | INFO | Total mem usage: 26.5 GiB | 20260505 23:58:03.686 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 23:58:03.686 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 23:58:03.686 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260505 23:58:03.686 | INFO | High Water Breakdown: | 20260505 23:58:03.686 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:58:03.686 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:58:03.686 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:58:03.686 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:58:03.686 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:58:03.686 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:58:03.686 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 23:58:03.686 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 23:58:03.686 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:58:03.686 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 23:58:03.686 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 23:58:03.686 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 23:58:03.686 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 23:58:03.686 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 23:58:03.686 | INFO | ------------------------------------------ | 20260505 23:58:03.686 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:58:03.686 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 23:58:03.686 | INFO | Total mem usage: 25.6 GiB | 20260505 23:58:03.686 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 23:58:03.686 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 23:58:03.686 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 23:58:03.686 | INFO | High Water Breakdown: | 20260505 23:58:03.686 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 23:58:03.686 | INFO | ------------------------------------------ | 20260505 23:58:03.686 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:58:03.686 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 23:58:03.686 | INFO | Total mem usage: 15.7 GiB | 20260505 23:58:03.686 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 23:58:03.686 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 23:58:03.686 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 23:58:03.686 | INFO | High Water Breakdown: | 20260505 23:58:03.686 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 23:58:03.686 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 23:58:03.686 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 23:58:03.686 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 23:58:03.686 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 23:58:03.686 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 23:58:03.686 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 23:58:03.686 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 23:58:03.686 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 23:58:03.686 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:58:03.686 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 23:58:03.686 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:58:03.686 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 23:58:03.686 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 23:58:03.686 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 23:58:03.686 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 23:58:03.686 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:58:03.686 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 23:58:03.686 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:58:03.686 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:58:03.686 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:58:03.686 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 23:58:03.686 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 23:58:03.686 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:58:03.686 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 23:58:03.686 | INFO | ------------------------------------------ | 20260505 23:58:03.686 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:58:03.686 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 23:58:03.686 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:58:03.686 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:58:03.686 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.39e+01, 1.34e+01 / 7.87e-01 | 20260505 23:58:03.686 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.05e-01, 1.03e-02 / 3.71e-03 | 20260505 23:58:03.686 | INFO | PerfReport: 11.81%, Baseline -- 3.70e+00 / 4.50e+00, 5.65e-01 / 2.23e-01 | 20260505 23:58:03.686 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.51e+00, 1.88e-01 / 1.18e-01 | 20260505 23:58:03.686 | INFO | PerfReport: 15.96%, DME -- 4.01e+01 / 4.38e+01, 3.80e+00 / 9.74e-01 | 20260505 23:58:03.686 | INFO | PerfReport: 57.41%, FrameLabeling -- 1.80e+01 / 1.96e+01, 1.55e+00 / 1.19e+00 | 20260505 23:58:03.687 | INFO | PerfReport: 7.08%, PulseAccumulating -- 2.22e+00 / 2.76e+00, 4.17e-01 / 2.25e-01 | 20260505 23:58:03.687 | INFO | PerfReport: 3.09%, Metrics -- 9.69e-01 / 1.25e+00, 2.63e-01 / 1.33e-01 | 20260505 23:58:03.687 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.36e-01 / 3.91e-01, 5.74e-02 / 3.13e-02 | 20260505 23:58:03.687 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 23:58:03.687 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:58:03.687 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:58:03.687 | INFO | PerfReport: Full Execution -- 9.84e+01 / -nan, 2.46e+01 / 0.00e+00 | 20260505 23:58:03.687 | INFO | PerfReport: 22.31%, Upload -- 2.20e+01 / 0.00e+00, 7.50e+00 / 0.00e+00 | 20260505 23:58:03.687 | INFO | PerfReport: 21.84%, Download -- 2.15e+01 / 0.00e+00, 7.55e+00 / 0.00e+00 | 20260505 23:58:03.687 | INFO | PerfReport: 31.87%, Compute -- 3.14e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260505 23:58:03.687 | INFO | PerfReport: 2.13%, LockUpload -- 2.09e+00 / 0.00e+00, 7.90e+00 / 0.00e+00 | 20260505 23:58:03.687 | INFO | PerfReport: 1.27%, LockDownload -- 1.25e+00 / 0.00e+00, 2.83e+00 / 0.00e+00 | 20260505 23:58:03.687 | INFO | PerfReport: 20.57%, LockCompute -- 2.02e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260505 23:58:03.687 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.74e-02 / 0.00e+00 | 20260505 23:58:03.807 | INFO | Starting a round of Baz IO | 20260505 23:58:06.957 | INFO | Populating RTMetrics report | 20260505 23:58:06.958 | INFO | RTMetrics report populated | 20260505 23:58:06.958 | INFO | Opening tmp RTMetrics json file | 20260505 23:58:06.959 | INFO | Serializing RTMetrics to json | 20260505 23:58:06.962 | INFO | Serialization done, swapping tmp file now... | 20260505 23:58:06.963 | INFO | RTMetrics done | 20260505 23:58:07.036 | INFO | Finished a round of Baz IO: Wrote 1.873 GiB in 3.229 seconds | 20260505 23:58:07.036 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.40% / 0.00% / 78.60% | 20260505 23:58:07.036 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0992934% / 0% / 6.35783e-05% | 20260505 23:58:07.036 | INFO | Baz IO overall: | 20260505 23:58:07.036 | INFO | BytesWritten : 4178845594348 | 20260505 23:58:07.036 | INFO | NumEvents : 1967605039081 | 20260505 23:58:08.826 | INFO | Analyzing chunk frames = [7331840,7332352) | 20260505 23:58:08.826 | INFO | MemInfo: { | 20260505 23:58:08.826 | INFO | "gpu" : " 17 GB", | 20260505 23:58:08.826 | INFO | "huge" : "", | 20260505 23:58:08.826 | INFO | "huge_shared" : " 27 GB", | 20260505 23:58:08.826 | INFO | "malloc" : " 14 GB", | 20260505 23:58:08.826 | INFO | "pinned" : " 29 GB" | 20260505 23:58:08.826 | INFO | } | 20260505 23:58:08.826 | INFO | Perf report for the last 5.14112 seconds: | 20260505 23:58:08.826 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:58:08.826 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:08.826 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:08.826 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:08.826 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260505 23:58:08.826 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:08.826 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260505 23:58:08.826 | INFO | PRE_HQ : 59.8%, 1.7, 54.0ms, 0 | 20260505 23:58:08.826 | INFO | BAZWRITER : 62.0%, 3.6, 113.4ms, 0 | 20260505 23:58:08.826 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 23:58:08.826 | INFO | Uploaded 36GiB to GPU in 2.16113s (16.6579GiB/s) | 20260505 23:58:08.826 | INFO | Downloaded 38GiB to GPU in 2.23229s (17.0229GiB/s) | 20260505 23:58:13.967 | INFO | Analyzing chunk frames = [7332352,7332864) | 20260505 23:58:13.967 | INFO | MemInfo: { | 20260505 23:58:13.967 | INFO | "gpu" : " 17 GB", | 20260505 23:58:13.967 | INFO | "huge" : "", | 20260505 23:58:13.967 | INFO | "huge_shared" : " 27 GB", | 20260505 23:58:13.967 | INFO | "malloc" : " 14 GB", | 20260505 23:58:13.967 | INFO | "pinned" : " 29 GB" | 20260505 23:58:13.967 | INFO | } | 20260505 23:58:13.968 | INFO | Perf report for the last 5.14133 seconds: | 20260505 23:58:13.968 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:58:13.968 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:13.968 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:13.968 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:13.968 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260505 23:58:13.968 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:13.968 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:13.968 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260505 23:58:13.968 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:13.968 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:13.968 | INFO | Uploaded 36GiB to GPU in 2.11388s (17.0303GiB/s) | 20260505 23:58:13.968 | INFO | Downloaded 34GiB to GPU in 2.04533s (16.6232GiB/s) | 20260505 23:58:19.101 | INFO | Analyzing chunk frames = [7332864,7333376) | 20260505 23:58:19.101 | INFO | MemInfo: { | 20260505 23:58:19.101 | INFO | "gpu" : " 17 GB", | 20260505 23:58:19.102 | INFO | "huge" : "", | 20260505 23:58:19.102 | INFO | "huge_shared" : " 27 GB", | 20260505 23:58:19.102 | INFO | "malloc" : " 14 GB", | 20260505 23:58:19.102 | INFO | "pinned" : " 29 GB" | 20260505 23:58:19.102 | INFO | } | 20260505 23:58:19.102 | INFO | Perf report for the last 5.1341 seconds: | 20260505 23:58:19.102 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:58:19.102 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:19.102 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:19.102 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:19.102 | INFO | ANALYSIS : 61.6%, 3.0, 95.7ms, 0 | 20260505 23:58:19.102 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:19.102 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:19.102 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:58:19.102 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:19.102 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:19.102 | INFO | Uploaded 36GiB to GPU in 2.08284s (17.2841GiB/s) | 20260505 23:58:19.102 | INFO | Downloaded 34GiB to GPU in 2.02264s (16.8097GiB/s) | 20260505 23:58:24.245 | INFO | Analyzing chunk frames = [7333376,7333888) | 20260505 23:58:24.245 | INFO | MemInfo: { | 20260505 23:58:24.245 | INFO | "gpu" : " 17 GB", | 20260505 23:58:24.245 | INFO | "huge" : "", | 20260505 23:58:24.245 | INFO | "huge_shared" : " 27 GB", | 20260505 23:58:24.245 | INFO | "malloc" : " 14 GB", | 20260505 23:58:24.245 | INFO | "pinned" : " 29 GB" | 20260505 23:58:24.245 | INFO | } | 20260505 23:58:24.245 | INFO | Perf report for the last 5.14361 seconds: | 20260505 23:58:24.245 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:58:24.245 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:24.245 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:24.245 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:24.245 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260505 23:58:24.245 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:24.245 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:24.245 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:58:24.245 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:24.245 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:24.245 | INFO | Uploaded 36GiB to GPU in 2.14233s (16.8041GiB/s) | 20260505 23:58:24.246 | INFO | Downloaded 34GiB to GPU in 2.0182s (16.8467GiB/s) | 20260505 23:58:29.385 | INFO | Analyzing chunk frames = [7333888,7334400) | 20260505 23:58:29.385 | INFO | MemInfo: { | 20260505 23:58:29.385 | INFO | "gpu" : " 17 GB", | 20260505 23:58:29.385 | INFO | "huge" : "", | 20260505 23:58:29.385 | INFO | "huge_shared" : " 27 GB", | 20260505 23:58:29.385 | INFO | "malloc" : " 14 GB", | 20260505 23:58:29.385 | INFO | "pinned" : " 29 GB" | 20260505 23:58:29.385 | INFO | } | 20260505 23:58:29.385 | INFO | Perf report for the last 5.13959 seconds: | 20260505 23:58:29.385 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:58:29.385 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:29.385 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:29.385 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:29.385 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260505 23:58:29.385 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:29.385 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:29.385 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:58:29.385 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:29.385 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:29.385 | INFO | Uploaded 36GiB to GPU in 2.08716s (17.2483GiB/s) | 20260505 23:58:29.385 | INFO | Downloaded 34GiB to GPU in 2.03833s (16.6803GiB/s) | 20260505 23:58:34.520 | INFO | Analyzing chunk frames = [7334400,7334912) | 20260505 23:58:34.520 | INFO | MemInfo: { | 20260505 23:58:34.520 | INFO | "gpu" : " 17 GB", | 20260505 23:58:34.520 | INFO | "huge" : "", | 20260505 23:58:34.520 | INFO | "huge_shared" : " 27 GB", | 20260505 23:58:34.520 | INFO | "malloc" : " 14 GB", | 20260505 23:58:34.520 | INFO | "pinned" : " 29 GB" | 20260505 23:58:34.520 | INFO | } | 20260505 23:58:34.520 | INFO | Perf report for the last 5.13561 seconds: | 20260505 23:58:34.521 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:58:34.521 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:34.521 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:34.521 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:34.521 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260505 23:58:34.521 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:34.521 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:34.521 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260505 23:58:34.521 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:34.521 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:34.521 | INFO | Uploaded 36GiB to GPU in 2.07699s (17.3328GiB/s) | 20260505 23:58:34.521 | INFO | Downloaded 34GiB to GPU in 2.03601s (16.6993GiB/s) | 20260505 23:58:39.654 | INFO | Analyzing chunk frames = [7334912,7335424) | 20260505 23:58:39.654 | INFO | MemInfo: { | 20260505 23:58:39.654 | INFO | "gpu" : " 17 GB", | 20260505 23:58:39.654 | INFO | "huge" : "", | 20260505 23:58:39.654 | INFO | "huge_shared" : " 27 GB", | 20260505 23:58:39.654 | INFO | "malloc" : " 14 GB", | 20260505 23:58:39.654 | INFO | "pinned" : " 29 GB" | 20260505 23:58:39.654 | INFO | } | 20260505 23:58:39.654 | INFO | Perf report for the last 5.13393 seconds: | 20260505 23:58:39.654 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:58:39.654 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:39.654 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:39.655 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:39.655 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260505 23:58:39.655 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:39.655 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:39.655 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260505 23:58:39.655 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:39.655 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:39.655 | INFO | Uploaded 36GiB to GPU in 2.07492s (17.3501GiB/s) | 20260505 23:58:39.655 | INFO | Downloaded 34GiB to GPU in 2.06254s (16.4845GiB/s) | 20260505 23:58:44.801 | INFO | Analyzing chunk frames = [7335424,7335936) | 20260505 23:58:44.801 | INFO | MemInfo: { | 20260505 23:58:44.801 | INFO | "gpu" : " 17 GB", | 20260505 23:58:44.801 | INFO | "huge" : "", | 20260505 23:58:44.801 | INFO | "huge_shared" : " 27 GB", | 20260505 23:58:44.802 | INFO | "malloc" : " 14 GB", | 20260505 23:58:44.802 | INFO | "pinned" : " 29 GB" | 20260505 23:58:44.802 | INFO | } | 20260505 23:58:44.802 | INFO | Perf report for the last 5.14724 seconds: | 20260505 23:58:44.802 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:58:44.802 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:44.802 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:44.802 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:44.802 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260505 23:58:44.802 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:44.802 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:44.802 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260505 23:58:44.802 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:44.802 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:44.802 | INFO | Uploaded 36GiB to GPU in 2.12069s (16.9756GiB/s) | 20260505 23:58:44.802 | INFO | Downloaded 34GiB to GPU in 2.00965s (16.9184GiB/s) | 20260505 23:58:44.918 | INFO | Starting a round of Baz IO | 20260505 23:58:48.066 | INFO | Populating RTMetrics report | 20260505 23:58:48.067 | INFO | RTMetrics report populated | 20260505 23:58:48.067 | INFO | Opening tmp RTMetrics json file | 20260505 23:58:48.067 | INFO | Serializing RTMetrics to json | 20260505 23:58:48.068 | INFO | Serialization done, swapping tmp file now... | 20260505 23:58:48.069 | INFO | RTMetrics done | 20260505 23:58:48.142 | INFO | Finished a round of Baz IO: Wrote 1.872 GiB in 3.223 seconds | 20260505 23:58:48.142 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.40% / 0.00% / 78.60% | 20260505 23:58:48.142 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0984629% / 0% / 7.94729e-05% | 20260505 23:58:48.142 | INFO | Baz IO overall: | 20260505 23:58:48.142 | INFO | BytesWritten : 4180855940849 | 20260505 23:58:48.142 | INFO | NumEvents : 1968524966112 | 20260505 23:58:49.940 | INFO | Analyzing chunk frames = [7335936,7336448) | 20260505 23:58:49.940 | INFO | MemInfo: { | 20260505 23:58:49.940 | INFO | "gpu" : " 17 GB", | 20260505 23:58:49.940 | INFO | "huge" : "", | 20260505 23:58:49.940 | INFO | "huge_shared" : " 27 GB", | 20260505 23:58:49.940 | INFO | "malloc" : " 14 GB", | 20260505 23:58:49.940 | INFO | "pinned" : " 29 GB" | 20260505 23:58:49.940 | INFO | } | 20260505 23:58:49.940 | INFO | Perf report for the last 5.13857 seconds: | 20260505 23:58:49.940 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:58:49.940 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:49.940 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:49.940 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:49.940 | INFO | ANALYSIS : 62.5%, 3.0, 97.0ms, 0 | 20260505 23:58:49.940 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:49.940 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260505 23:58:49.940 | INFO | PRE_HQ : 59.8%, 1.7, 53.8ms, 0 | 20260505 23:58:49.940 | INFO | BAZWRITER : 61.9%, 3.8, 121.5ms, 0 | 20260505 23:58:49.940 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 23:58:49.940 | INFO | Uploaded 36GiB to GPU in 2.17386s (16.5604GiB/s) | 20260505 23:58:49.941 | INFO | Downloaded 38GiB to GPU in 2.2507s (16.8837GiB/s) | 20260505 23:58:55.084 | INFO | Analyzing chunk frames = [7336448,7336960) | 20260505 23:58:55.084 | INFO | MemInfo: { | 20260505 23:58:55.084 | INFO | "gpu" : " 17 GB", | 20260505 23:58:55.085 | INFO | "huge" : "", | 20260505 23:58:55.085 | INFO | "huge_shared" : " 27 GB", | 20260505 23:58:55.085 | INFO | "malloc" : " 14 GB", | 20260505 23:58:55.085 | INFO | "pinned" : " 29 GB" | 20260505 23:58:55.085 | INFO | } | 20260505 23:58:55.085 | INFO | Perf report for the last 5.1444 seconds: | 20260505 23:58:55.085 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:58:55.085 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:55.085 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:55.085 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:55.085 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260505 23:58:55.085 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:55.085 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:55.085 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260505 23:58:55.085 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:55.085 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:58:55.085 | INFO | Uploaded 36GiB to GPU in 2.10146s (17.1309GiB/s) | 20260505 23:58:55.085 | INFO | Downloaded 34GiB to GPU in 2.03709s (16.6904GiB/s) | 20260505 23:59:00.218 | INFO | Analyzing chunk frames = [7336960,7337472) | 20260505 23:59:00.218 | INFO | MemInfo: { | 20260505 23:59:00.218 | INFO | "gpu" : " 17 GB", | 20260505 23:59:00.218 | INFO | "huge" : "", | 20260505 23:59:00.218 | INFO | "huge_shared" : " 27 GB", | 20260505 23:59:00.218 | INFO | "malloc" : " 14 GB", | 20260505 23:59:00.218 | INFO | "pinned" : " 29 GB" | 20260505 23:59:00.218 | INFO | } | 20260505 23:59:00.218 | INFO | Perf report for the last 5.13315 seconds: | 20260505 23:59:00.218 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:59:00.218 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:00.218 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:00.218 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:00.218 | INFO | ANALYSIS : 61.7%, 3.0, 95.8ms, 0 | 20260505 23:59:00.218 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:00.218 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:00.218 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:59:00.218 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:00.218 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:00.218 | INFO | Uploaded 36GiB to GPU in 2.10006s (17.1423GiB/s) | 20260505 23:59:00.218 | INFO | Downloaded 34GiB to GPU in 2.03384s (16.7172GiB/s) | 20260505 23:59:05.359 | INFO | Analyzing chunk frames = [7337472,7337984) | 20260505 23:59:05.359 | INFO | MemInfo: { | 20260505 23:59:05.359 | INFO | "gpu" : " 17 GB", | 20260505 23:59:05.359 | INFO | "huge" : "", | 20260505 23:59:05.359 | INFO | "huge_shared" : " 27 GB", | 20260505 23:59:05.359 | INFO | "malloc" : " 14 GB", | 20260505 23:59:05.359 | INFO | "pinned" : " 29 GB" | 20260505 23:59:05.359 | INFO | } | 20260505 23:59:05.359 | INFO | Perf report for the last 5.14139 seconds: | 20260505 23:59:05.359 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:59:05.359 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:05.359 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:05.359 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:05.359 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260505 23:59:05.359 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:05.359 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:05.359 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260505 23:59:05.359 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:05.359 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:05.359 | INFO | Uploaded 36GiB to GPU in 2.13334s (16.8749GiB/s) | 20260505 23:59:05.359 | INFO | Downloaded 34GiB to GPU in 2.00673s (16.943GiB/s) | 20260505 23:59:10.500 | INFO | Analyzing chunk frames = [7337984,7338496) | 20260505 23:59:10.500 | INFO | MemInfo: { | 20260505 23:59:10.500 | INFO | "gpu" : " 17 GB", | 20260505 23:59:10.500 | INFO | "huge" : "", | 20260505 23:59:10.500 | INFO | "huge_shared" : " 27 GB", | 20260505 23:59:10.500 | INFO | "malloc" : " 14 GB", | 20260505 23:59:10.500 | INFO | "pinned" : " 29 GB" | 20260505 23:59:10.500 | INFO | } | 20260505 23:59:10.501 | INFO | Perf report for the last 5.14126 seconds: | 20260505 23:59:10.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:59:10.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:10.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:10.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:10.501 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260505 23:59:10.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:10.501 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:10.501 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:59:10.501 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:10.501 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:10.501 | INFO | Uploaded 36GiB to GPU in 2.11362s (17.0324GiB/s) | 20260505 23:59:10.501 | INFO | Downloaded 34GiB to GPU in 2.02165s (16.818GiB/s) | 20260505 23:59:15.638 | INFO | Analyzing chunk frames = [7338496,7339008) | 20260505 23:59:15.638 | INFO | MemInfo: { | 20260505 23:59:15.638 | INFO | "gpu" : " 17 GB", | 20260505 23:59:15.638 | INFO | "huge" : "", | 20260505 23:59:15.638 | INFO | "huge_shared" : " 27 GB", | 20260505 23:59:15.638 | INFO | "malloc" : " 14 GB", | 20260505 23:59:15.638 | INFO | "pinned" : " 29 GB" | 20260505 23:59:15.638 | INFO | } | 20260505 23:59:15.638 | INFO | Perf report for the last 5.13766 seconds: | 20260505 23:59:15.638 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:59:15.638 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:15.638 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:15.638 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:15.638 | INFO | ANALYSIS : 61.2%, 3.0, 95.6ms, 0 | 20260505 23:59:15.638 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:15.638 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:15.638 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260505 23:59:15.638 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:15.638 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:15.638 | INFO | Uploaded 36GiB to GPU in 2.06615s (17.4237GiB/s) | 20260505 23:59:15.638 | INFO | Downloaded 34GiB to GPU in 2.0366s (16.6945GiB/s) | 20260505 23:59:20.776 | INFO | Analyzing chunk frames = [7339008,7339520) | 20260505 23:59:20.776 | INFO | MemInfo: { | 20260505 23:59:20.776 | INFO | "gpu" : " 17 GB", | 20260505 23:59:20.776 | INFO | "huge" : "", | 20260505 23:59:20.776 | INFO | "huge_shared" : " 27 GB", | 20260505 23:59:20.776 | INFO | "malloc" : " 14 GB", | 20260505 23:59:20.776 | INFO | "pinned" : " 29 GB" | 20260505 23:59:20.776 | INFO | } | 20260505 23:59:20.776 | INFO | Perf report for the last 5.13768 seconds: | 20260505 23:59:20.776 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:59:20.776 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:20.776 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:20.776 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:20.776 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260505 23:59:20.776 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:20.776 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:20.776 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260505 23:59:20.776 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:20.776 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:20.776 | INFO | Uploaded 36GiB to GPU in 2.10561s (17.0972GiB/s) | 20260505 23:59:20.776 | INFO | Downloaded 34GiB to GPU in 2.06754s (16.4446GiB/s) | 20260505 23:59:25.917 | INFO | Analyzing chunk frames = [7339520,7340032) | 20260505 23:59:25.917 | INFO | MemInfo: { | 20260505 23:59:25.917 | INFO | "gpu" : " 17 GB", | 20260505 23:59:25.917 | INFO | "huge" : "", | 20260505 23:59:25.917 | INFO | "huge_shared" : " 27 GB", | 20260505 23:59:25.917 | INFO | "malloc" : " 14 GB", | 20260505 23:59:25.917 | INFO | "pinned" : " 29 GB" | 20260505 23:59:25.917 | INFO | } | 20260505 23:59:25.917 | INFO | Perf report for the last 5.14086 seconds: | 20260505 23:59:25.917 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:59:25.917 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:25.917 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:25.917 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:25.917 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260505 23:59:25.917 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:25.917 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:25.917 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260505 23:59:25.917 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:25.917 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:25.917 | INFO | Uploaded 36GiB to GPU in 2.11382s (17.0308GiB/s) | 20260505 23:59:25.917 | INFO | Downloaded 34GiB to GPU in 2.00091s (16.9923GiB/s) | 20260505 23:59:25.917 | INFO | ----Memory usage report for Standard Host Memory----- | 20260505 23:59:25.917 | INFO | Total mem usage: 15.3 GiB | 20260505 23:59:25.917 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260505 23:59:25.917 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -5.95%) | 20260505 23:59:25.917 | INFO | Unused Cache: 0B (Waste: 0%) | 20260505 23:59:25.917 | INFO | High Water Breakdown: | 20260505 23:59:25.917 | INFO | 61.3% : 8.87 GiB ( 93.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260505 23:59:25.917 | INFO | 38.5% : 5.58 GiB ( 83.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260505 23:59:25.917 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260505 23:59:25.917 | INFO | ------------------------------------------ | 20260505 23:59:25.917 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:59:25.917 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260505 23:59:25.917 | INFO | Total mem usage: 26.5 GiB | 20260505 23:59:25.917 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260505 23:59:25.917 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260505 23:59:25.917 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260505 23:59:25.917 | INFO | High Water Breakdown: | 20260505 23:59:25.917 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:59:25.917 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:59:25.917 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:59:25.917 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:59:25.917 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:59:25.918 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:59:25.918 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260505 23:59:25.918 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260505 23:59:25.918 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:59:25.918 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260505 23:59:25.918 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260505 23:59:25.918 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260505 23:59:25.918 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260505 23:59:25.918 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260505 23:59:25.918 | INFO | ------------------------------------------ | 20260505 23:59:25.918 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:59:25.918 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260505 23:59:25.918 | INFO | Total mem usage: 25.6 GiB | 20260505 23:59:25.918 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260505 23:59:25.918 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260505 23:59:25.918 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260505 23:59:25.918 | INFO | High Water Breakdown: | 20260505 23:59:25.918 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260505 23:59:25.918 | INFO | ------------------------------------------ | 20260505 23:59:25.918 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:59:25.918 | INFO | ----Memory usage report for Gpu Memory----- | 20260505 23:59:25.918 | INFO | Total mem usage: 15.7 GiB | 20260505 23:59:25.918 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260505 23:59:25.918 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260505 23:59:25.918 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260505 23:59:25.918 | INFO | High Water Breakdown: | 20260505 23:59:25.918 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260505 23:59:25.918 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260505 23:59:25.918 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260505 23:59:25.918 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260505 23:59:25.918 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260505 23:59:25.918 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260505 23:59:25.918 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260505 23:59:25.918 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260505 23:59:25.918 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260505 23:59:25.918 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260505 23:59:25.918 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260505 23:59:25.918 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260505 23:59:25.918 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260505 23:59:25.918 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260505 23:59:25.918 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260505 23:59:25.918 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260505 23:59:25.918 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260505 23:59:25.918 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260505 23:59:25.918 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260505 23:59:25.918 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260505 23:59:25.918 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260505 23:59:25.918 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260505 23:59:25.918 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260505 23:59:25.918 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260505 23:59:25.918 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260505 23:59:25.918 | INFO | ------------------------------------------ | 20260505 23:59:25.918 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260505 23:59:25.918 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260505 23:59:25.918 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:59:25.918 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:59:25.918 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.41e+01, 1.35e+01 / 7.24e-01 | 20260505 23:59:25.918 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.05e-01, 1.05e-02 / 3.63e-03 | 20260505 23:59:25.918 | INFO | PerfReport: 11.85%, Baseline -- 3.71e+00 / 4.54e+00, 5.77e-01 / 1.34e-01 | 20260505 23:59:25.918 | INFO | PerfReport: 3.29%, Histogram -- 1.03e+00 / 1.53e+00, 1.91e-01 / 6.39e-02 | 20260505 23:59:25.918 | INFO | PerfReport: 15.98%, DME -- 4.01e+01 / 4.41e+01, 3.91e+00 / 8.10e-01 | 20260505 23:59:25.918 | INFO | PerfReport: 57.32%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.51e+00 / 1.13e+00 | 20260505 23:59:25.918 | INFO | PerfReport: 7.10%, PulseAccumulating -- 2.23e+00 / 2.75e+00, 4.24e-01 / 2.32e-01 | 20260505 23:59:25.918 | INFO | PerfReport: 3.09%, Metrics -- 9.70e-01 / 1.26e+00, 2.63e-01 / 1.10e-01 | 20260505 23:59:25.918 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.36e-01 / 3.86e-01, 5.71e-02 / 2.42e-02 | 20260505 23:59:25.918 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260505 23:59:25.918 | INFO | PerfReport: Gathered 1536 samples. | 20260505 23:59:25.918 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260505 23:59:25.918 | INFO | PerfReport: Full Execution -- 9.83e+01 / -nan, 2.45e+01 / 0.00e+00 | 20260505 23:59:25.918 | INFO | PerfReport: 22.37%, Upload -- 2.20e+01 / 0.00e+00, 7.54e+00 / 0.00e+00 | 20260505 23:59:25.918 | INFO | PerfReport: 21.81%, Download -- 2.14e+01 / 0.00e+00, 7.58e+00 / 0.00e+00 | 20260505 23:59:25.919 | INFO | PerfReport: 31.90%, Compute -- 3.14e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260505 23:59:25.919 | INFO | PerfReport: 2.14%, LockUpload -- 2.11e+00 / 0.00e+00, 7.92e+00 / 0.00e+00 | 20260505 23:59:25.919 | INFO | PerfReport: 1.27%, LockDownload -- 1.25e+00 / 0.00e+00, 2.82e+00 / 0.00e+00 | 20260505 23:59:25.919 | INFO | PerfReport: 20.50%, LockCompute -- 2.02e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260505 23:59:25.919 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.89e-02 / 0.00e+00 | 20260505 23:59:26.038 | INFO | Starting a round of Baz IO | 20260505 23:59:29.190 | INFO | Populating RTMetrics report | 20260505 23:59:29.190 | INFO | RTMetrics report populated | 20260505 23:59:29.190 | INFO | Opening tmp RTMetrics json file | 20260505 23:59:29.191 | INFO | Serializing RTMetrics to json | 20260505 23:59:29.192 | INFO | Serialization done, swapping tmp file now... | 20260505 23:59:29.192 | INFO | RTMetrics done | 20260505 23:59:29.269 | INFO | Finished a round of Baz IO: Wrote 1.876 GiB in 3.231 seconds | 20260505 23:59:29.269 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.40% / 0.00% / 78.60% | 20260505 23:59:29.269 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0993093% / 0% / 8.34465e-05% | 20260505 23:59:29.269 | INFO | Baz IO overall: | 20260505 23:59:29.269 | INFO | BytesWritten : 4182870679756 | 20260505 23:59:29.269 | INFO | NumEvents : 1969448022896 | 20260505 23:59:31.059 | INFO | Analyzing chunk frames = [7340032,7340544) | 20260505 23:59:31.059 | INFO | MemInfo: { | 20260505 23:59:31.059 | INFO | "gpu" : " 17 GB", | 20260505 23:59:31.059 | INFO | "huge" : "", | 20260505 23:59:31.059 | INFO | "huge_shared" : " 27 GB", | 20260505 23:59:31.059 | INFO | "malloc" : " 14 GB", | 20260505 23:59:31.059 | INFO | "pinned" : " 29 GB" | 20260505 23:59:31.059 | INFO | } | 20260505 23:59:31.059 | INFO | Perf report for the last 5.14193 seconds: | 20260505 23:59:31.059 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:59:31.059 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:31.059 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:31.059 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:31.059 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260505 23:59:31.059 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:31.059 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260505 23:59:31.059 | INFO | PRE_HQ : 59.6%, 1.7, 54.0ms, 0 | 20260505 23:59:31.059 | INFO | BAZWRITER : 62.0%, 3.6, 113.8ms, 0 | 20260505 23:59:31.059 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260505 23:59:31.059 | INFO | Uploaded 36GiB to GPU in 2.14526s (16.7812GiB/s) | 20260505 23:59:31.059 | INFO | Downloaded 38GiB to GPU in 2.23982s (16.9656GiB/s) | 20260505 23:59:36.198 | INFO | Analyzing chunk frames = [7340544,7341056) | 20260505 23:59:36.198 | INFO | MemInfo: { | 20260505 23:59:36.198 | INFO | "gpu" : " 17 GB", | 20260505 23:59:36.198 | INFO | "huge" : "", | 20260505 23:59:36.198 | INFO | "huge_shared" : " 27 GB", | 20260505 23:59:36.198 | INFO | "malloc" : " 14 GB", | 20260505 23:59:36.199 | INFO | "pinned" : " 29 GB" | 20260505 23:59:36.199 | INFO | } | 20260505 23:59:36.199 | INFO | Perf report for the last 5.13993 seconds: | 20260505 23:59:36.199 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:59:36.199 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:36.199 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:36.199 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:36.199 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260505 23:59:36.199 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:36.199 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:36.199 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260505 23:59:36.199 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:36.199 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:36.199 | INFO | Uploaded 36GiB to GPU in 1.9805s (18.1772GiB/s) | 20260505 23:59:36.199 | INFO | Downloaded 34GiB to GPU in 1.99017s (17.0839GiB/s) | 20260505 23:59:41.340 | INFO | Analyzing chunk frames = [7341056,7341568) | 20260505 23:59:41.340 | INFO | MemInfo: { | 20260505 23:59:41.340 | INFO | "gpu" : " 17 GB", | 20260505 23:59:41.340 | INFO | "huge" : "", | 20260505 23:59:41.340 | INFO | "huge_shared" : " 27 GB", | 20260505 23:59:41.340 | INFO | "malloc" : " 14 GB", | 20260505 23:59:41.340 | INFO | "pinned" : " 29 GB" | 20260505 23:59:41.340 | INFO | } | 20260505 23:59:41.340 | INFO | Perf report for the last 5.14156 seconds: | 20260505 23:59:41.340 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:59:41.340 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:41.340 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:41.340 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:41.340 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260505 23:59:41.340 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:41.340 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:41.340 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260505 23:59:41.340 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:41.340 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:41.340 | INFO | Uploaded 36GiB to GPU in 2.07763s (17.3274GiB/s) | 20260505 23:59:41.341 | INFO | Downloaded 34GiB to GPU in 2.0209s (16.8242GiB/s) | 20260505 23:59:46.466 | INFO | Analyzing chunk frames = [7341568,7342080) | 20260505 23:59:46.466 | INFO | MemInfo: { | 20260505 23:59:46.466 | INFO | "gpu" : " 17 GB", | 20260505 23:59:46.466 | INFO | "huge" : "", | 20260505 23:59:46.466 | INFO | "huge_shared" : " 27 GB", | 20260505 23:59:46.466 | INFO | "malloc" : " 14 GB", | 20260505 23:59:46.466 | INFO | "pinned" : " 29 GB" | 20260505 23:59:46.466 | INFO | } | 20260505 23:59:46.466 | INFO | Perf report for the last 5.12614 seconds: | 20260505 23:59:46.467 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:59:46.467 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:46.467 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:46.467 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:46.467 | INFO | ANALYSIS : 61.8%, 3.0, 95.7ms, 0 | 20260505 23:59:46.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:46.467 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:46.467 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260505 23:59:46.467 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:46.467 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:46.467 | INFO | Uploaded 36GiB to GPU in 2.14105s (16.8142GiB/s) | 20260505 23:59:46.467 | INFO | Downloaded 34GiB to GPU in 2.03632s (16.6968GiB/s) | 20260505 23:59:51.608 | INFO | Analyzing chunk frames = [7342080,7342592) | 20260505 23:59:51.608 | INFO | MemInfo: { | 20260505 23:59:51.608 | INFO | "gpu" : " 17 GB", | 20260505 23:59:51.608 | INFO | "huge" : "", | 20260505 23:59:51.608 | INFO | "huge_shared" : " 27 GB", | 20260505 23:59:51.608 | INFO | "malloc" : " 14 GB", | 20260505 23:59:51.608 | INFO | "pinned" : " 29 GB" | 20260505 23:59:51.608 | INFO | } | 20260505 23:59:51.608 | INFO | Perf report for the last 5.14162 seconds: | 20260505 23:59:51.608 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:59:51.608 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:51.608 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:51.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:51.608 | INFO | ANALYSIS : 61.6%, 3.0, 96.2ms, 0 | 20260505 23:59:51.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:51.608 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:51.608 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260505 23:59:51.608 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:51.608 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:51.608 | INFO | Uploaded 36GiB to GPU in 2.17291s (16.5676GiB/s) | 20260505 23:59:51.609 | INFO | Downloaded 34GiB to GPU in 2.0764s (16.3745GiB/s) | 20260505 23:59:56.752 | INFO | Analyzing chunk frames = [7342592,7343104) | 20260505 23:59:56.752 | INFO | MemInfo: { | 20260505 23:59:56.752 | INFO | "gpu" : " 17 GB", | 20260505 23:59:56.752 | INFO | "huge" : "", | 20260505 23:59:56.752 | INFO | "huge_shared" : " 27 GB", | 20260505 23:59:56.752 | INFO | "malloc" : " 14 GB", | 20260505 23:59:56.752 | INFO | "pinned" : " 29 GB" | 20260505 23:59:56.752 | INFO | } | 20260505 23:59:56.753 | INFO | Perf report for the last 5.1445 seconds: | 20260505 23:59:56.753 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260505 23:59:56.753 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:56.753 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:56.753 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:56.753 | INFO | ANALYSIS : 61.4%, 3.0, 96.0ms, 0 | 20260505 23:59:56.753 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:56.753 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:56.753 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260505 23:59:56.753 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:56.753 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260505 23:59:56.753 | INFO | Uploaded 36GiB to GPU in 2.07652s (17.3367GiB/s) | 20260505 23:59:56.753 | INFO | Downloaded 34GiB to GPU in 2.03623s (16.6975GiB/s) | 20260506 00:00:01.892 | INFO | Analyzing chunk frames = [7343104,7343616) | 20260506 00:00:01.892 | INFO | MemInfo: { | 20260506 00:00:01.892 | INFO | "gpu" : " 17 GB", | 20260506 00:00:01.892 | INFO | "huge" : "", | 20260506 00:00:01.892 | INFO | "huge_shared" : " 27 GB", | 20260506 00:00:01.892 | INFO | "malloc" : " 14 GB", | 20260506 00:00:01.892 | INFO | "pinned" : " 29 GB" | 20260506 00:00:01.892 | INFO | } | 20260506 00:00:01.892 | INFO | Perf report for the last 5.1397 seconds: | 20260506 00:00:01.893 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:00:01.893 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:01.893 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:01.893 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:01.893 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260506 00:00:01.893 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:01.893 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:01.893 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 00:00:01.893 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:01.893 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:01.893 | INFO | Uploaded 36GiB to GPU in 2.05453s (17.5222GiB/s) | 20260506 00:00:01.893 | INFO | Downloaded 34GiB to GPU in 2.07152s (16.413GiB/s) | 20260506 00:00:07.030 | INFO | Analyzing chunk frames = [7343616,7344128) | 20260506 00:00:07.030 | INFO | MemInfo: { | 20260506 00:00:07.030 | INFO | "gpu" : " 17 GB", | 20260506 00:00:07.030 | INFO | "huge" : "", | 20260506 00:00:07.030 | INFO | "huge_shared" : " 27 GB", | 20260506 00:00:07.030 | INFO | "malloc" : " 14 GB", | 20260506 00:00:07.030 | INFO | "pinned" : " 29 GB" | 20260506 00:00:07.030 | INFO | } | 20260506 00:00:07.030 | INFO | Perf report for the last 5.13815 seconds: | 20260506 00:00:07.030 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:00:07.030 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:07.030 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:07.030 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:07.030 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260506 00:00:07.030 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:07.030 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:07.030 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 00:00:07.030 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:07.031 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:07.031 | INFO | Uploaded 36GiB to GPU in 2.11662s (17.0082GiB/s) | 20260506 00:00:07.031 | INFO | Downloaded 34GiB to GPU in 2.0156s (16.8684GiB/s) | 20260506 00:00:07.156 | INFO | Starting a round of Baz IO | 20260506 00:00:10.301 | INFO | Populating RTMetrics report | 20260506 00:00:10.301 | INFO | RTMetrics report populated | 20260506 00:00:10.301 | INFO | Opening tmp RTMetrics json file | 20260506 00:00:10.304 | INFO | Serializing RTMetrics to json | 20260506 00:00:10.308 | INFO | Serialization done, swapping tmp file now... | 20260506 00:00:10.308 | INFO | RTMetrics done | 20260506 00:00:10.380 | INFO | Finished a round of Baz IO: Wrote 1.874 GiB in 3.223 seconds | 20260506 00:00:10.380 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.40% / 0.00% / 78.60% | 20260506 00:00:10.380 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0989397% / 0% / 8.34465e-05% | 20260506 00:00:10.380 | INFO | Baz IO overall: | 20260506 00:00:10.380 | INFO | BytesWritten : 4184883185574 | 20260506 00:00:10.380 | INFO | NumEvents : 1970369369096 | 20260506 00:00:12.169 | INFO | Analyzing chunk frames = [7344128,7344640) | 20260506 00:00:12.169 | INFO | MemInfo: { | 20260506 00:00:12.169 | INFO | "gpu" : " 17 GB", | 20260506 00:00:12.169 | INFO | "huge" : "", | 20260506 00:00:12.169 | INFO | "huge_shared" : " 27 GB", | 20260506 00:00:12.169 | INFO | "malloc" : " 14 GB", | 20260506 00:00:12.169 | INFO | "pinned" : " 29 GB" | 20260506 00:00:12.169 | INFO | } | 20260506 00:00:12.169 | INFO | Perf report for the last 5.13842 seconds: | 20260506 00:00:12.169 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:00:12.169 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:12.169 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:12.169 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:12.169 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260506 00:00:12.169 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:12.169 | INFO | RT_METRICS : 43.4%, 1.0, 22.7ms, 0 | 20260506 00:00:12.169 | INFO | PRE_HQ : 60.0%, 1.7, 54.2ms, 0 | 20260506 00:00:12.169 | INFO | BAZWRITER : 62.0%, 3.6, 113.4ms, 0 | 20260506 00:00:12.169 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 00:00:12.169 | INFO | Uploaded 36GiB to GPU in 2.16976s (16.5917GiB/s) | 20260506 00:00:12.169 | INFO | Downloaded 38GiB to GPU in 2.23654s (16.9905GiB/s) | 20260506 00:00:17.310 | INFO | Analyzing chunk frames = [7344640,7345152) | 20260506 00:00:17.310 | INFO | MemInfo: { | 20260506 00:00:17.310 | INFO | "gpu" : " 17 GB", | 20260506 00:00:17.310 | INFO | "huge" : "", | 20260506 00:00:17.310 | INFO | "huge_shared" : " 27 GB", | 20260506 00:00:17.310 | INFO | "malloc" : " 14 GB", | 20260506 00:00:17.310 | INFO | "pinned" : " 29 GB" | 20260506 00:00:17.310 | INFO | } | 20260506 00:00:17.310 | INFO | Perf report for the last 5.14151 seconds: | 20260506 00:00:17.311 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:00:17.311 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:17.311 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:17.311 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:17.311 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 00:00:17.311 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:17.311 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:17.311 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:00:17.311 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:17.311 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:17.311 | INFO | Uploaded 36GiB to GPU in 2.09124s (17.2147GiB/s) | 20260506 00:00:17.311 | INFO | Downloaded 34GiB to GPU in 2.02971s (16.7511GiB/s) | 20260506 00:00:22.448 | INFO | Analyzing chunk frames = [7345152,7345664) | 20260506 00:00:22.448 | INFO | MemInfo: { | 20260506 00:00:22.448 | INFO | "gpu" : " 17 GB", | 20260506 00:00:22.448 | INFO | "huge" : "", | 20260506 00:00:22.448 | INFO | "huge_shared" : " 27 GB", | 20260506 00:00:22.448 | INFO | "malloc" : " 14 GB", | 20260506 00:00:22.448 | INFO | "pinned" : " 29 GB" | 20260506 00:00:22.448 | INFO | } | 20260506 00:00:22.448 | INFO | Perf report for the last 5.13724 seconds: | 20260506 00:00:22.448 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:00:22.448 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:22.448 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:22.448 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:22.448 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 00:00:22.448 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:22.448 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:22.448 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 00:00:22.448 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:22.448 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:22.448 | INFO | Uploaded 36GiB to GPU in 2.08212s (17.29GiB/s) | 20260506 00:00:22.448 | INFO | Downloaded 34GiB to GPU in 2.01578s (16.8669GiB/s) | 20260506 00:00:27.589 | INFO | Analyzing chunk frames = [7345664,7346176) | 20260506 00:00:27.589 | INFO | MemInfo: { | 20260506 00:00:27.589 | INFO | "gpu" : " 17 GB", | 20260506 00:00:27.589 | INFO | "huge" : "", | 20260506 00:00:27.589 | INFO | "huge_shared" : " 27 GB", | 20260506 00:00:27.589 | INFO | "malloc" : " 14 GB", | 20260506 00:00:27.589 | INFO | "pinned" : " 29 GB" | 20260506 00:00:27.589 | INFO | } | 20260506 00:00:27.589 | INFO | Perf report for the last 5.14185 seconds: | 20260506 00:00:27.590 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:00:27.590 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:27.590 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:27.590 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:27.590 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 00:00:27.590 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:27.590 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:27.590 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:00:27.590 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:27.590 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:27.590 | INFO | Uploaded 36GiB to GPU in 2.13754s (16.8418GiB/s) | 20260506 00:00:27.590 | INFO | Downloaded 34GiB to GPU in 2.0128s (16.8919GiB/s) | 20260506 00:00:32.723 | INFO | Analyzing chunk frames = [7346176,7346688) | 20260506 00:00:32.723 | INFO | MemInfo: { | 20260506 00:00:32.723 | INFO | "gpu" : " 17 GB", | 20260506 00:00:32.723 | INFO | "huge" : "", | 20260506 00:00:32.723 | INFO | "huge_shared" : " 27 GB", | 20260506 00:00:32.723 | INFO | "malloc" : " 14 GB", | 20260506 00:00:32.723 | INFO | "pinned" : " 29 GB" | 20260506 00:00:32.723 | INFO | } | 20260506 00:00:32.723 | INFO | Perf report for the last 5.13364 seconds: | 20260506 00:00:32.723 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:00:32.723 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:32.723 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:32.723 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:32.723 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 00:00:32.723 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:32.723 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:32.723 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 00:00:32.723 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:32.723 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:32.724 | INFO | Uploaded 36GiB to GPU in 2.08445s (17.2708GiB/s) | 20260506 00:00:32.724 | INFO | Downloaded 34GiB to GPU in 2.04627s (16.6156GiB/s) | 20260506 00:00:37.865 | INFO | Analyzing chunk frames = [7346688,7347200) | 20260506 00:00:37.865 | INFO | MemInfo: { | 20260506 00:00:37.865 | INFO | "gpu" : " 17 GB", | 20260506 00:00:37.865 | INFO | "huge" : "", | 20260506 00:00:37.865 | INFO | "huge_shared" : " 27 GB", | 20260506 00:00:37.865 | INFO | "malloc" : " 14 GB", | 20260506 00:00:37.865 | INFO | "pinned" : " 29 GB" | 20260506 00:00:37.865 | INFO | } | 20260506 00:00:37.865 | INFO | Perf report for the last 5.14232 seconds: | 20260506 00:00:37.866 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:00:37.866 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:37.866 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:37.866 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:37.866 | INFO | ANALYSIS : 61.5%, 3.0, 96.0ms, 0 | 20260506 00:00:37.866 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:37.866 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:37.866 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 00:00:37.866 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:37.866 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:37.866 | INFO | Uploaded 36GiB to GPU in 2.07986s (17.3089GiB/s) | 20260506 00:00:37.866 | INFO | Downloaded 34GiB to GPU in 2.03257s (16.7276GiB/s) | 20260506 00:00:43.006 | INFO | Analyzing chunk frames = [7347200,7347712) | 20260506 00:00:43.006 | INFO | MemInfo: { | 20260506 00:00:43.006 | INFO | "gpu" : " 17 GB", | 20260506 00:00:43.006 | INFO | "huge" : "", | 20260506 00:00:43.006 | INFO | "huge_shared" : " 27 GB", | 20260506 00:00:43.006 | INFO | "malloc" : " 14 GB", | 20260506 00:00:43.006 | INFO | "pinned" : " 29 GB" | 20260506 00:00:43.006 | INFO | } | 20260506 00:00:43.006 | INFO | Perf report for the last 5.14055 seconds: | 20260506 00:00:43.006 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:00:43.006 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:43.006 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:43.006 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:43.006 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260506 00:00:43.006 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:43.006 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:43.006 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 00:00:43.006 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:43.006 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:43.006 | INFO | Uploaded 36GiB to GPU in 2.10416s (17.109GiB/s) | 20260506 00:00:43.006 | INFO | Downloaded 34GiB to GPU in 2.09095s (16.2606GiB/s) | 20260506 00:00:48.148 | INFO | Analyzing chunk frames = [7347712,7348224) | 20260506 00:00:48.148 | INFO | MemInfo: { | 20260506 00:00:48.148 | INFO | "gpu" : " 17 GB", | 20260506 00:00:48.148 | INFO | "huge" : "", | 20260506 00:00:48.148 | INFO | "huge_shared" : " 27 GB", | 20260506 00:00:48.148 | INFO | "malloc" : " 14 GB", | 20260506 00:00:48.148 | INFO | "pinned" : " 29 GB" | 20260506 00:00:48.148 | INFO | } | 20260506 00:00:48.148 | INFO | Perf report for the last 5.14229 seconds: | 20260506 00:00:48.149 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:00:48.149 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:48.149 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:48.149 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:48.149 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260506 00:00:48.149 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:48.149 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:48.149 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 00:00:48.149 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:48.149 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:48.149 | INFO | Uploaded 36GiB to GPU in 2.14447s (16.7873GiB/s) | 20260506 00:00:48.149 | INFO | Downloaded 34GiB to GPU in 2.03124s (16.7386GiB/s) | 20260506 00:00:48.149 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 00:00:48.149 | INFO | Total mem usage: 15.3 GiB | 20260506 00:00:48.149 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 00:00:48.149 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.11%) | 20260506 00:00:48.149 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 00:00:48.149 | INFO | High Water Breakdown: | 20260506 00:00:48.149 | INFO | 60.9% : 8.55 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 00:00:48.149 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 00:00:48.149 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 00:00:48.149 | INFO | ------------------------------------------ | 20260506 00:00:48.149 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:00:48.149 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 00:00:48.149 | INFO | Total mem usage: 26.5 GiB | 20260506 00:00:48.149 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 00:00:48.149 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 00:00:48.149 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 00:00:48.149 | INFO | High Water Breakdown: | 20260506 00:00:48.149 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:00:48.149 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:00:48.149 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:00:48.149 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:00:48.149 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:00:48.149 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:00:48.149 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 00:00:48.149 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 00:00:48.149 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:00:48.149 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 00:00:48.149 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 00:00:48.149 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 00:00:48.149 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 00:00:48.149 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 00:00:48.149 | INFO | ------------------------------------------ | 20260506 00:00:48.149 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:00:48.149 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 00:00:48.149 | INFO | Total mem usage: 25.6 GiB | 20260506 00:00:48.149 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 00:00:48.150 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 00:00:48.150 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 00:00:48.150 | INFO | High Water Breakdown: | 20260506 00:00:48.150 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 00:00:48.150 | INFO | ------------------------------------------ | 20260506 00:00:48.150 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:00:48.150 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 00:00:48.150 | INFO | Total mem usage: 15.7 GiB | 20260506 00:00:48.150 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 00:00:48.150 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 00:00:48.150 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 00:00:48.150 | INFO | High Water Breakdown: | 20260506 00:00:48.150 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 00:00:48.150 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 00:00:48.150 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 00:00:48.150 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 00:00:48.150 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 00:00:48.150 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 00:00:48.150 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 00:00:48.150 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 00:00:48.150 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 00:00:48.150 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:00:48.150 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 00:00:48.150 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:00:48.150 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 00:00:48.150 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 00:00:48.150 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 00:00:48.150 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 00:00:48.150 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:00:48.150 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 00:00:48.150 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:00:48.150 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:00:48.150 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:00:48.150 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 00:00:48.150 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 00:00:48.150 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:00:48.150 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 00:00:48.150 | INFO | ------------------------------------------ | 20260506 00:00:48.150 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:00:48.150 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 00:00:48.150 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:00:48.150 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:00:48.150 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.38e+01, 1.35e+01 / 7.32e-01 | 20260506 00:00:48.150 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.03e-01, 1.08e-02 / 4.28e-03 | 20260506 00:00:48.150 | INFO | PerfReport: 11.83%, Baseline -- 3.70e+00 / 4.49e+00, 5.67e-01 / 2.25e-01 | 20260506 00:00:48.150 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.53e+00, 1.91e-01 / 7.69e-02 | 20260506 00:00:48.150 | INFO | PerfReport: 15.97%, DME -- 4.00e+01 / 4.38e+01, 3.82e+00 / 8.74e-01 | 20260506 00:00:48.150 | INFO | PerfReport: 57.34%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.47e+00 / 1.02e+00 | 20260506 00:00:48.150 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.23e+00 / 2.75e+00, 4.24e-01 / 2.32e-01 | 20260506 00:00:48.150 | INFO | PerfReport: 3.10%, Metrics -- 9.69e-01 / 1.25e+00, 2.61e-01 / 8.35e-02 | 20260506 00:00:48.150 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.35e-01 / 3.87e-01, 5.64e-02 / 2.53e-02 | 20260506 00:00:48.150 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 00:00:48.150 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:00:48.150 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:00:48.150 | INFO | PerfReport: Full Execution -- 9.83e+01 / -nan, 2.46e+01 / 0.00e+00 | 20260506 00:00:48.150 | INFO | PerfReport: 22.28%, Upload -- 2.19e+01 / 0.00e+00, 7.49e+00 / 0.00e+00 | 20260506 00:00:48.150 | INFO | PerfReport: 21.84%, Download -- 2.15e+01 / 0.00e+00, 7.61e+00 / 0.00e+00 | 20260506 00:00:48.150 | INFO | PerfReport: 31.84%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 00:00:48.150 | INFO | PerfReport: 2.16%, LockUpload -- 2.13e+00 / 0.00e+00, 7.93e+00 / 0.00e+00 | 20260506 00:00:48.150 | INFO | PerfReport: 1.31%, LockDownload -- 1.29e+00 / 0.00e+00, 2.86e+00 / 0.00e+00 | 20260506 00:00:48.150 | INFO | PerfReport: 20.55%, LockCompute -- 2.02e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 00:00:48.150 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.27e-02 / 0.00e+00 | 20260506 00:00:48.266 | INFO | Starting a round of Baz IO | 20260506 00:00:51.416 | INFO | Populating RTMetrics report | 20260506 00:00:51.416 | INFO | RTMetrics report populated | 20260506 00:00:51.416 | INFO | Opening tmp RTMetrics json file | 20260506 00:00:51.417 | INFO | Serializing RTMetrics to json | 20260506 00:00:51.420 | INFO | Serialization done, swapping tmp file now... | 20260506 00:00:51.421 | INFO | RTMetrics done | 20260506 00:00:51.494 | INFO | Finished a round of Baz IO: Wrote 1.873 GiB in 3.227 seconds | 20260506 00:00:51.494 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.39% / 0.00% / 78.60% | 20260506 00:00:51.494 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0983874% / 0% / 4.37101e-05% | 20260506 00:00:51.494 | INFO | Baz IO overall: | 20260506 00:00:51.494 | INFO | BytesWritten : 4186894120472 | 20260506 00:00:51.494 | INFO | NumEvents : 1971289935983 | 20260506 00:00:53.286 | INFO | Analyzing chunk frames = [7348224,7348736) | 20260506 00:00:53.286 | INFO | MemInfo: { | 20260506 00:00:53.286 | INFO | "gpu" : " 17 GB", | 20260506 00:00:53.286 | INFO | "huge" : "", | 20260506 00:00:53.286 | INFO | "huge_shared" : " 27 GB", | 20260506 00:00:53.286 | INFO | "malloc" : " 14 GB", | 20260506 00:00:53.286 | INFO | "pinned" : " 29 GB" | 20260506 00:00:53.286 | INFO | } | 20260506 00:00:53.286 | INFO | Perf report for the last 5.13771 seconds: | 20260506 00:00:53.286 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:00:53.286 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:53.286 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:53.286 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:53.286 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260506 00:00:53.286 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:53.286 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260506 00:00:53.286 | INFO | PRE_HQ : 59.9%, 1.7, 53.8ms, 0 | 20260506 00:00:53.286 | INFO | BAZWRITER : 62.0%, 3.6, 113.0ms, 0 | 20260506 00:00:53.286 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:00:53.286 | INFO | Uploaded 36GiB to GPU in 2.1668s (16.6144GiB/s) | 20260506 00:00:53.287 | INFO | Downloaded 38GiB to GPU in 2.2417s (16.9514GiB/s) | 20260506 00:00:58.415 | INFO | Analyzing chunk frames = [7348736,7349248) | 20260506 00:00:58.415 | INFO | MemInfo: { | 20260506 00:00:58.415 | INFO | "gpu" : " 17 GB", | 20260506 00:00:58.415 | INFO | "huge" : "", | 20260506 00:00:58.415 | INFO | "huge_shared" : " 27 GB", | 20260506 00:00:58.415 | INFO | "malloc" : " 14 GB", | 20260506 00:00:58.415 | INFO | "pinned" : " 29 GB" | 20260506 00:00:58.415 | INFO | } | 20260506 00:00:58.415 | INFO | Perf report for the last 5.12914 seconds: | 20260506 00:00:58.415 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:00:58.415 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:58.415 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:58.415 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:58.415 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 00:00:58.415 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:58.415 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:58.415 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 00:00:58.416 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:58.416 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:00:58.416 | INFO | Uploaded 36GiB to GPU in 2.09353s (17.1959GiB/s) | 20260506 00:00:58.416 | INFO | Downloaded 34GiB to GPU in 2.03014s (16.7476GiB/s) | 20260506 00:01:03.563 | INFO | Analyzing chunk frames = [7349248,7349760) | 20260506 00:01:03.563 | INFO | MemInfo: { | 20260506 00:01:03.563 | INFO | "gpu" : " 17 GB", | 20260506 00:01:03.563 | INFO | "huge" : "", | 20260506 00:01:03.563 | INFO | "huge_shared" : " 27 GB", | 20260506 00:01:03.563 | INFO | "malloc" : " 14 GB", | 20260506 00:01:03.563 | INFO | "pinned" : " 29 GB" | 20260506 00:01:03.563 | INFO | } | 20260506 00:01:03.563 | INFO | Perf report for the last 5.14767 seconds: | 20260506 00:01:03.563 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:01:03.563 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:03.563 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:03.563 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:03.563 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 00:01:03.563 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:03.563 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:03.563 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:01:03.563 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:03.563 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:03.563 | INFO | Uploaded 36GiB to GPU in 2.14898s (16.7522GiB/s) | 20260506 00:01:03.563 | INFO | Downloaded 34GiB to GPU in 2.04908s (16.5928GiB/s) | 20260506 00:01:08.704 | INFO | Analyzing chunk frames = [7349760,7350272) | 20260506 00:01:08.704 | INFO | MemInfo: { | 20260506 00:01:08.704 | INFO | "gpu" : " 17 GB", | 20260506 00:01:08.704 | INFO | "huge" : "", | 20260506 00:01:08.704 | INFO | "huge_shared" : " 27 GB", | 20260506 00:01:08.704 | INFO | "malloc" : " 14 GB", | 20260506 00:01:08.704 | INFO | "pinned" : " 29 GB" | 20260506 00:01:08.704 | INFO | } | 20260506 00:01:08.704 | INFO | Perf report for the last 5.14149 seconds: | 20260506 00:01:08.704 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:01:08.705 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:08.705 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:08.705 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:08.705 | INFO | ANALYSIS : 61.6%, 3.0, 95.6ms, 0 | 20260506 00:01:08.705 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:08.705 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:08.705 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:01:08.705 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:08.705 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:08.705 | INFO | Uploaded 36GiB to GPU in 2.13878s (16.832GiB/s) | 20260506 00:01:08.705 | INFO | Downloaded 34GiB to GPU in 2.03699s (16.6913GiB/s) | 20260506 00:01:13.845 | INFO | Analyzing chunk frames = [7350272,7350784) | 20260506 00:01:13.845 | INFO | MemInfo: { | 20260506 00:01:13.845 | INFO | "gpu" : " 17 GB", | 20260506 00:01:13.845 | INFO | "huge" : "", | 20260506 00:01:13.845 | INFO | "huge_shared" : " 27 GB", | 20260506 00:01:13.845 | INFO | "malloc" : " 14 GB", | 20260506 00:01:13.845 | INFO | "pinned" : " 29 GB" | 20260506 00:01:13.845 | INFO | } | 20260506 00:01:13.846 | INFO | Perf report for the last 5.14112 seconds: | 20260506 00:01:13.846 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:01:13.846 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:13.846 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:13.846 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:13.846 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 00:01:13.846 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:13.846 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:13.846 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 00:01:13.846 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:13.846 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:13.846 | INFO | Uploaded 36GiB to GPU in 2.09092s (17.2173GiB/s) | 20260506 00:01:13.846 | INFO | Downloaded 34GiB to GPU in 2.0464s (16.6145GiB/s) | 20260506 00:01:18.978 | INFO | Analyzing chunk frames = [7350784,7351296) | 20260506 00:01:18.978 | INFO | MemInfo: { | 20260506 00:01:18.978 | INFO | "gpu" : " 17 GB", | 20260506 00:01:18.978 | INFO | "huge" : "", | 20260506 00:01:18.978 | INFO | "huge_shared" : " 27 GB", | 20260506 00:01:18.978 | INFO | "malloc" : " 14 GB", | 20260506 00:01:18.978 | INFO | "pinned" : " 29 GB" | 20260506 00:01:18.978 | INFO | } | 20260506 00:01:18.978 | INFO | Perf report for the last 5.13268 seconds: | 20260506 00:01:18.978 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:01:18.978 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:18.978 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:18.978 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:18.978 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 00:01:18.978 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:18.978 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:18.978 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 00:01:18.978 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:18.978 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:18.979 | INFO | Uploaded 36GiB to GPU in 2.07151s (17.3786GiB/s) | 20260506 00:01:18.979 | INFO | Downloaded 34GiB to GPU in 2.04213s (16.6493GiB/s) | 20260506 00:01:24.119 | INFO | Analyzing chunk frames = [7351296,7351808) | 20260506 00:01:24.119 | INFO | MemInfo: { | 20260506 00:01:24.119 | INFO | "gpu" : " 17 GB", | 20260506 00:01:24.119 | INFO | "huge" : "", | 20260506 00:01:24.119 | INFO | "huge_shared" : " 27 GB", | 20260506 00:01:24.119 | INFO | "malloc" : " 14 GB", | 20260506 00:01:24.119 | INFO | "pinned" : " 29 GB" | 20260506 00:01:24.119 | INFO | } | 20260506 00:01:24.119 | INFO | Perf report for the last 5.14111 seconds: | 20260506 00:01:24.119 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:01:24.119 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:24.119 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:24.119 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:24.119 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260506 00:01:24.119 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:24.119 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:24.119 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 00:01:24.119 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:24.119 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:24.120 | INFO | Uploaded 36GiB to GPU in 2.13047s (16.8977GiB/s) | 20260506 00:01:24.120 | INFO | Downloaded 34GiB to GPU in 2.08387s (16.3158GiB/s) | 20260506 00:01:29.261 | INFO | Analyzing chunk frames = [7351808,7352320) | 20260506 00:01:29.261 | INFO | MemInfo: { | 20260506 00:01:29.261 | INFO | "gpu" : " 17 GB", | 20260506 00:01:29.261 | INFO | "huge" : "", | 20260506 00:01:29.261 | INFO | "huge_shared" : " 27 GB", | 20260506 00:01:29.261 | INFO | "malloc" : " 14 GB", | 20260506 00:01:29.261 | INFO | "pinned" : " 29 GB" | 20260506 00:01:29.261 | INFO | } | 20260506 00:01:29.261 | INFO | Perf report for the last 5.14129 seconds: | 20260506 00:01:29.261 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:01:29.261 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:29.261 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:29.261 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:29.261 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260506 00:01:29.261 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:29.261 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:29.261 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 00:01:29.261 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:29.261 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:29.261 | INFO | Uploaded 36GiB to GPU in 2.11888s (16.9901GiB/s) | 20260506 00:01:29.261 | INFO | Downloaded 34GiB to GPU in 2.0089s (16.9247GiB/s) | 20260506 00:01:29.378 | INFO | Starting a round of Baz IO | 20260506 00:01:32.529 | INFO | Populating RTMetrics report | 20260506 00:01:32.530 | INFO | RTMetrics report populated | 20260506 00:01:32.530 | INFO | Opening tmp RTMetrics json file | 20260506 00:01:32.533 | INFO | Serializing RTMetrics to json | 20260506 00:01:32.535 | INFO | Serialization done, swapping tmp file now... | 20260506 00:01:32.536 | INFO | RTMetrics done | 20260506 00:01:32.611 | INFO | Finished a round of Baz IO: Wrote 1.872 GiB in 3.233 seconds | 20260506 00:01:32.611 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.39% / 0.00% / 78.61% | 20260506 00:01:32.611 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0982881% / 3.97364e-06% / 7.54992e-05% | 20260506 00:01:32.611 | INFO | Baz IO overall: | 20260506 00:01:32.611 | INFO | BytesWritten : 4188903785377 | 20260506 00:01:32.611 | INFO | NumEvents : 1972209838746 | 20260506 00:01:34.396 | INFO | Analyzing chunk frames = [7352320,7352832) | 20260506 00:01:34.396 | INFO | MemInfo: { | 20260506 00:01:34.396 | INFO | "gpu" : " 17 GB", | 20260506 00:01:34.396 | INFO | "huge" : "", | 20260506 00:01:34.396 | INFO | "huge_shared" : " 27 GB", | 20260506 00:01:34.396 | INFO | "malloc" : " 14 GB", | 20260506 00:01:34.396 | INFO | "pinned" : " 29 GB" | 20260506 00:01:34.397 | INFO | } | 20260506 00:01:34.397 | INFO | Perf report for the last 5.13601 seconds: | 20260506 00:01:34.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:01:34.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:34.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:34.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:34.397 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260506 00:01:34.397 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:34.397 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260506 00:01:34.397 | INFO | PRE_HQ : 59.7%, 1.7, 53.8ms, 0 | 20260506 00:01:34.397 | INFO | BAZWRITER : 62.0%, 3.6, 113.7ms, 0 | 20260506 00:01:34.397 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 00:01:34.397 | INFO | Uploaded 36GiB to GPU in 2.1398s (16.824GiB/s) | 20260506 00:01:34.397 | INFO | Downloaded 38GiB to GPU in 2.23576s (16.9964GiB/s) | 20260506 00:01:39.539 | INFO | Analyzing chunk frames = [7352832,7353344) | 20260506 00:01:39.539 | INFO | MemInfo: { | 20260506 00:01:39.539 | INFO | "gpu" : " 17 GB", | 20260506 00:01:39.539 | INFO | "huge" : "", | 20260506 00:01:39.539 | INFO | "huge_shared" : " 27 GB", | 20260506 00:01:39.539 | INFO | "malloc" : " 14 GB", | 20260506 00:01:39.539 | INFO | "pinned" : " 29 GB" | 20260506 00:01:39.539 | INFO | } | 20260506 00:01:39.539 | INFO | Perf report for the last 5.14223 seconds: | 20260506 00:01:39.539 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:01:39.539 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:39.539 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:39.539 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:39.539 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260506 00:01:39.539 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:39.539 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:39.539 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 00:01:39.539 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:39.539 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:39.539 | INFO | Uploaded 36GiB to GPU in 2.09071s (17.219GiB/s) | 20260506 00:01:39.539 | INFO | Downloaded 34GiB to GPU in 2.02892s (16.7577GiB/s) | 20260506 00:01:44.675 | INFO | Analyzing chunk frames = [7353344,7353856) | 20260506 00:01:44.675 | INFO | MemInfo: { | 20260506 00:01:44.675 | INFO | "gpu" : " 17 GB", | 20260506 00:01:44.675 | INFO | "huge" : "", | 20260506 00:01:44.675 | INFO | "huge_shared" : " 27 GB", | 20260506 00:01:44.675 | INFO | "malloc" : " 14 GB", | 20260506 00:01:44.675 | INFO | "pinned" : " 29 GB" | 20260506 00:01:44.675 | INFO | } | 20260506 00:01:44.675 | INFO | Perf report for the last 5.13587 seconds: | 20260506 00:01:44.675 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:01:44.675 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:44.675 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:44.675 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:44.675 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 00:01:44.675 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:44.675 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:44.675 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:01:44.675 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:44.675 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:44.675 | INFO | Uploaded 36GiB to GPU in 2.09103s (17.2164GiB/s) | 20260506 00:01:44.675 | INFO | Downloaded 34GiB to GPU in 2.03384s (16.7172GiB/s) | 20260506 00:01:49.819 | INFO | Analyzing chunk frames = [7353856,7354368) | 20260506 00:01:49.819 | INFO | MemInfo: { | 20260506 00:01:49.819 | INFO | "gpu" : " 17 GB", | 20260506 00:01:49.819 | INFO | "huge" : "", | 20260506 00:01:49.819 | INFO | "huge_shared" : " 27 GB", | 20260506 00:01:49.819 | INFO | "malloc" : " 14 GB", | 20260506 00:01:49.819 | INFO | "pinned" : " 29 GB" | 20260506 00:01:49.819 | INFO | } | 20260506 00:01:49.819 | INFO | Perf report for the last 5.14396 seconds: | 20260506 00:01:49.819 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:01:49.819 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:49.819 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:49.819 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:49.819 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 00:01:49.819 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:49.819 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:49.819 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:01:49.819 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:49.819 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:49.819 | INFO | Uploaded 36GiB to GPU in 2.13936s (16.8275GiB/s) | 20260506 00:01:49.819 | INFO | Downloaded 34GiB to GPU in 2.02139s (16.8201GiB/s) | 20260506 00:01:54.960 | INFO | Analyzing chunk frames = [7354368,7354880) | 20260506 00:01:54.960 | INFO | MemInfo: { | 20260506 00:01:54.960 | INFO | "gpu" : " 17 GB", | 20260506 00:01:54.960 | INFO | "huge" : "", | 20260506 00:01:54.960 | INFO | "huge_shared" : " 27 GB", | 20260506 00:01:54.960 | INFO | "malloc" : " 14 GB", | 20260506 00:01:54.960 | INFO | "pinned" : " 29 GB" | 20260506 00:01:54.960 | INFO | } | 20260506 00:01:54.960 | INFO | Perf report for the last 5.14109 seconds: | 20260506 00:01:54.960 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:01:54.960 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:54.960 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:54.960 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:54.960 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 00:01:54.960 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:54.960 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:54.960 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 00:01:54.960 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:54.960 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:01:54.960 | INFO | Uploaded 36GiB to GPU in 2.09311s (17.1993GiB/s) | 20260506 00:01:54.960 | INFO | Downloaded 34GiB to GPU in 2.04575s (16.6198GiB/s) | 20260506 00:02:00.094 | INFO | Analyzing chunk frames = [7354880,7355392) | 20260506 00:02:00.094 | INFO | MemInfo: { | 20260506 00:02:00.094 | INFO | "gpu" : " 17 GB", | 20260506 00:02:00.094 | INFO | "huge" : "", | 20260506 00:02:00.095 | INFO | "huge_shared" : " 27 GB", | 20260506 00:02:00.095 | INFO | "malloc" : " 14 GB", | 20260506 00:02:00.095 | INFO | "pinned" : " 29 GB" | 20260506 00:02:00.095 | INFO | } | 20260506 00:02:00.095 | INFO | Perf report for the last 5.13473 seconds: | 20260506 00:02:00.095 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:02:00.095 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:00.095 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:00.095 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:00.095 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260506 00:02:00.095 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:00.095 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:00.095 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:02:00.095 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:00.095 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:00.095 | INFO | Uploaded 36GiB to GPU in 2.07233s (17.3717GiB/s) | 20260506 00:02:00.095 | INFO | Downloaded 34GiB to GPU in 2.04093s (16.6591GiB/s) | 20260506 00:02:05.229 | INFO | Analyzing chunk frames = [7355392,7355904) | 20260506 00:02:05.229 | INFO | MemInfo: { | 20260506 00:02:05.229 | INFO | "gpu" : " 17 GB", | 20260506 00:02:05.229 | INFO | "huge" : "", | 20260506 00:02:05.229 | INFO | "huge_shared" : " 27 GB", | 20260506 00:02:05.229 | INFO | "malloc" : " 14 GB", | 20260506 00:02:05.229 | INFO | "pinned" : " 29 GB" | 20260506 00:02:05.229 | INFO | } | 20260506 00:02:05.229 | INFO | Perf report for the last 5.13439 seconds: | 20260506 00:02:05.229 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:02:05.229 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:05.229 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:05.229 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:05.229 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260506 00:02:05.229 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:05.229 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:05.229 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 00:02:05.229 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:05.229 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:05.229 | INFO | Uploaded 36GiB to GPU in 2.13055s (16.8971GiB/s) | 20260506 00:02:05.229 | INFO | Downloaded 34GiB to GPU in 2.11172s (16.1006GiB/s) | 20260506 00:02:10.371 | INFO | Analyzing chunk frames = [7355904,7356416) | 20260506 00:02:10.371 | INFO | MemInfo: { | 20260506 00:02:10.371 | INFO | "gpu" : " 17 GB", | 20260506 00:02:10.371 | INFO | "huge" : "", | 20260506 00:02:10.371 | INFO | "huge_shared" : " 27 GB", | 20260506 00:02:10.371 | INFO | "malloc" : " 14 GB", | 20260506 00:02:10.371 | INFO | "pinned" : " 29 GB" | 20260506 00:02:10.371 | INFO | } | 20260506 00:02:10.371 | INFO | Perf report for the last 5.14212 seconds: | 20260506 00:02:10.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:02:10.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:10.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:10.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:10.371 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260506 00:02:10.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:10.371 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:10.371 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:02:10.371 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:10.371 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:10.371 | INFO | Uploaded 36GiB to GPU in 2.11628s (17.011GiB/s) | 20260506 00:02:10.371 | INFO | Downloaded 34GiB to GPU in 2.00396s (16.9664GiB/s) | 20260506 00:02:10.372 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 00:02:10.372 | INFO | Total mem usage: 15.3 GiB | 20260506 00:02:10.372 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 00:02:10.372 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.13%) | 20260506 00:02:10.372 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 00:02:10.372 | INFO | High Water Breakdown: | 20260506 00:02:10.372 | INFO | 60.9% : 8.55 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 00:02:10.372 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 00:02:10.372 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 00:02:10.372 | INFO | ------------------------------------------ | 20260506 00:02:10.372 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:02:10.372 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 00:02:10.372 | INFO | Total mem usage: 26.5 GiB | 20260506 00:02:10.372 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 00:02:10.372 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 00:02:10.372 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260506 00:02:10.372 | INFO | High Water Breakdown: | 20260506 00:02:10.372 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:02:10.372 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:02:10.372 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:02:10.372 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:02:10.372 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:02:10.372 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:02:10.372 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 00:02:10.372 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 00:02:10.372 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:02:10.372 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 00:02:10.372 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 00:02:10.372 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 00:02:10.372 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 00:02:10.372 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 00:02:10.372 | INFO | ------------------------------------------ | 20260506 00:02:10.372 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:02:10.372 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 00:02:10.372 | INFO | Total mem usage: 25.6 GiB | 20260506 00:02:10.372 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 00:02:10.372 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 00:02:10.372 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 00:02:10.372 | INFO | High Water Breakdown: | 20260506 00:02:10.372 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 00:02:10.372 | INFO | ------------------------------------------ | 20260506 00:02:10.372 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:02:10.372 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 00:02:10.372 | INFO | Total mem usage: 15.7 GiB | 20260506 00:02:10.372 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 00:02:10.372 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 00:02:10.372 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 00:02:10.372 | INFO | High Water Breakdown: | 20260506 00:02:10.372 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 00:02:10.372 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 00:02:10.372 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 00:02:10.372 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 00:02:10.372 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 00:02:10.372 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 00:02:10.372 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 00:02:10.372 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 00:02:10.372 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 00:02:10.372 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:02:10.372 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 00:02:10.372 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:02:10.372 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 00:02:10.372 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 00:02:10.372 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 00:02:10.372 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 00:02:10.372 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:02:10.372 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 00:02:10.372 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:02:10.372 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:02:10.372 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:02:10.372 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 00:02:10.372 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 00:02:10.372 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:02:10.372 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 00:02:10.372 | INFO | ------------------------------------------ | 20260506 00:02:10.372 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:02:10.372 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 00:02:10.372 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:02:10.372 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:02:10.372 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.38e+01, 1.34e+01 / 6.95e-01 | 20260506 00:02:10.372 | INFO | PerfReport: 0.29%, AggregateStats -- 9.18e-02 / 1.03e-01, 1.06e-02 / 3.36e-03 | 20260506 00:02:10.372 | INFO | PerfReport: 11.82%, Baseline -- 3.70e+00 / 4.54e+00, 5.66e-01 / 1.40e-01 | 20260506 00:02:10.372 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.52e+00, 1.88e-01 / 7.29e-02 | 20260506 00:02:10.372 | INFO | PerfReport: 15.97%, DME -- 4.00e+01 / 4.38e+01, 3.80e+00 / 8.43e-01 | 20260506 00:02:10.372 | INFO | PerfReport: 57.37%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.50e+00 / 9.82e-01 | 20260506 00:02:10.372 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.23e+00 / 2.76e+00, 4.24e-01 / 2.04e-01 | 20260506 00:02:10.372 | INFO | PerfReport: 3.09%, Metrics -- 9.69e-01 / 1.23e+00, 2.62e-01 / 9.79e-02 | 20260506 00:02:10.372 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.35e-01 / 3.92e-01, 5.72e-02 / 3.80e-02 | 20260506 00:02:10.372 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 00:02:10.372 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:02:10.372 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:02:10.373 | INFO | PerfReport: Full Execution -- 9.84e+01 / -nan, 2.48e+01 / 0.00e+00 | 20260506 00:02:10.373 | INFO | PerfReport: 22.38%, Upload -- 2.20e+01 / 0.00e+00, 7.57e+00 / 0.00e+00 | 20260506 00:02:10.373 | INFO | PerfReport: 21.87%, Download -- 2.15e+01 / 0.00e+00, 7.63e+00 / 0.00e+00 | 20260506 00:02:10.373 | INFO | PerfReport: 31.82%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 00:02:10.373 | INFO | PerfReport: 2.15%, LockUpload -- 2.12e+00 / 0.00e+00, 7.91e+00 / 0.00e+00 | 20260506 00:02:10.373 | INFO | PerfReport: 1.32%, LockDownload -- 1.30e+00 / 0.00e+00, 2.84e+00 / 0.00e+00 | 20260506 00:02:10.373 | INFO | PerfReport: 20.45%, LockCompute -- 2.01e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 00:02:10.373 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 2.00e-02 / 0.00e+00 | 20260506 00:02:10.488 | INFO | Starting a round of Baz IO | 20260506 00:02:13.637 | INFO | Populating RTMetrics report | 20260506 00:02:13.637 | INFO | RTMetrics report populated | 20260506 00:02:13.637 | INFO | Opening tmp RTMetrics json file | 20260506 00:02:13.638 | INFO | Serializing RTMetrics to json | 20260506 00:02:13.639 | INFO | Serialization done, swapping tmp file now... | 20260506 00:02:13.640 | INFO | RTMetrics done | 20260506 00:02:13.717 | INFO | Finished a round of Baz IO: Wrote 1.871 GiB in 3.228 seconds | 20260506 00:02:13.717 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.39% / 0.00% / 78.61% | 20260506 00:02:13.717 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0976125% / 0% / 7.94729e-05% | 20260506 00:02:13.717 | INFO | Baz IO overall: | 20260506 00:02:13.717 | INFO | BytesWritten : 4190912253916 | 20260506 00:02:13.717 | INFO | NumEvents : 1973129093936 | 20260506 00:02:15.516 | INFO | Analyzing chunk frames = [7356416,7356928) | 20260506 00:02:15.516 | INFO | MemInfo: { | 20260506 00:02:15.516 | INFO | "gpu" : " 17 GB", | 20260506 00:02:15.516 | INFO | "huge" : "", | 20260506 00:02:15.516 | INFO | "huge_shared" : " 27 GB", | 20260506 00:02:15.516 | INFO | "malloc" : " 14 GB", | 20260506 00:02:15.516 | INFO | "pinned" : " 29 GB" | 20260506 00:02:15.516 | INFO | } | 20260506 00:02:15.516 | INFO | Perf report for the last 5.14482 seconds: | 20260506 00:02:15.516 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:02:15.516 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:15.516 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:15.516 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:15.516 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260506 00:02:15.516 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:15.516 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260506 00:02:15.516 | INFO | PRE_HQ : 59.7%, 1.7, 53.8ms, 0 | 20260506 00:02:15.516 | INFO | BAZWRITER : 61.9%, 3.6, 114.4ms, 0 | 20260506 00:02:15.516 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 00:02:15.516 | INFO | Uploaded 36GiB to GPU in 2.16458s (16.6314GiB/s) | 20260506 00:02:15.516 | INFO | Downloaded 38GiB to GPU in 2.23552s (16.9983GiB/s) | 20260506 00:02:20.656 | INFO | Analyzing chunk frames = [7356928,7357440) | 20260506 00:02:20.656 | INFO | MemInfo: { | 20260506 00:02:20.656 | INFO | "gpu" : " 17 GB", | 20260506 00:02:20.656 | INFO | "huge" : "", | 20260506 00:02:20.656 | INFO | "huge_shared" : " 27 GB", | 20260506 00:02:20.656 | INFO | "malloc" : " 14 GB", | 20260506 00:02:20.656 | INFO | "pinned" : " 29 GB" | 20260506 00:02:20.656 | INFO | } | 20260506 00:02:20.656 | INFO | Perf report for the last 5.13985 seconds: | 20260506 00:02:20.656 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:02:20.656 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:20.656 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:20.656 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:20.656 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 00:02:20.656 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:20.656 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:20.656 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:02:20.656 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:20.656 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:20.656 | INFO | Uploaded 36GiB to GPU in 1.97671s (18.2121GiB/s) | 20260506 00:02:20.656 | INFO | Downloaded 34GiB to GPU in 1.99008s (17.0847GiB/s) | 20260506 00:02:25.794 | INFO | Analyzing chunk frames = [7357440,7357952) | 20260506 00:02:25.794 | INFO | MemInfo: { | 20260506 00:02:25.794 | INFO | "gpu" : " 17 GB", | 20260506 00:02:25.794 | INFO | "huge" : "", | 20260506 00:02:25.794 | INFO | "huge_shared" : " 27 GB", | 20260506 00:02:25.794 | INFO | "malloc" : " 14 GB", | 20260506 00:02:25.794 | INFO | "pinned" : " 29 GB" | 20260506 00:02:25.794 | INFO | } | 20260506 00:02:25.794 | INFO | Perf report for the last 5.1383 seconds: | 20260506 00:02:25.794 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:02:25.794 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:25.794 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:25.794 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:25.794 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 00:02:25.794 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:25.794 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:25.794 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 00:02:25.794 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:25.794 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:25.794 | INFO | Uploaded 36GiB to GPU in 2.08777s (17.2433GiB/s) | 20260506 00:02:25.794 | INFO | Downloaded 34GiB to GPU in 2.0105s (16.9113GiB/s) | 20260506 00:02:30.930 | INFO | Analyzing chunk frames = [7357952,7358464) | 20260506 00:02:30.930 | INFO | MemInfo: { | 20260506 00:02:30.930 | INFO | "gpu" : " 17 GB", | 20260506 00:02:30.930 | INFO | "huge" : "", | 20260506 00:02:30.930 | INFO | "huge_shared" : " 27 GB", | 20260506 00:02:30.930 | INFO | "malloc" : " 14 GB", | 20260506 00:02:30.930 | INFO | "pinned" : " 29 GB" | 20260506 00:02:30.930 | INFO | } | 20260506 00:02:30.930 | INFO | Perf report for the last 5.13573 seconds: | 20260506 00:02:30.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:02:30.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:30.930 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:30.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:30.930 | INFO | ANALYSIS : 61.7%, 3.0, 95.8ms, 0 | 20260506 00:02:30.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:30.930 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:30.930 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 00:02:30.930 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:30.930 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:30.930 | INFO | Uploaded 36GiB to GPU in 2.13612s (16.853GiB/s) | 20260506 00:02:30.930 | INFO | Downloaded 34GiB to GPU in 2.02212s (16.814GiB/s) | 20260506 00:02:36.073 | INFO | Analyzing chunk frames = [7358464,7358976) | 20260506 00:02:36.073 | INFO | MemInfo: { | 20260506 00:02:36.073 | INFO | "gpu" : " 17 GB", | 20260506 00:02:36.073 | INFO | "huge" : "", | 20260506 00:02:36.073 | INFO | "huge_shared" : " 27 GB", | 20260506 00:02:36.074 | INFO | "malloc" : " 14 GB", | 20260506 00:02:36.074 | INFO | "pinned" : " 29 GB" | 20260506 00:02:36.074 | INFO | } | 20260506 00:02:36.074 | INFO | Perf report for the last 5.14382 seconds: | 20260506 00:02:36.074 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:02:36.074 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:36.074 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:36.074 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:36.074 | INFO | ANALYSIS : 61.2%, 3.0, 95.7ms, 0 | 20260506 00:02:36.074 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:36.074 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:36.074 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 00:02:36.074 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:36.074 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:36.074 | INFO | Uploaded 36GiB to GPU in 2.10605s (17.0936GiB/s) | 20260506 00:02:36.074 | INFO | Downloaded 34GiB to GPU in 2.05048s (16.5815GiB/s) | 20260506 00:02:41.210 | INFO | Analyzing chunk frames = [7358976,7359488) | 20260506 00:02:41.210 | INFO | MemInfo: { | 20260506 00:02:41.210 | INFO | "gpu" : " 17 GB", | 20260506 00:02:41.210 | INFO | "huge" : "", | 20260506 00:02:41.210 | INFO | "huge_shared" : " 27 GB", | 20260506 00:02:41.210 | INFO | "malloc" : " 14 GB", | 20260506 00:02:41.210 | INFO | "pinned" : " 29 GB" | 20260506 00:02:41.210 | INFO | } | 20260506 00:02:41.210 | INFO | Perf report for the last 5.13664 seconds: | 20260506 00:02:41.210 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:02:41.210 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:41.210 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:41.210 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:41.210 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260506 00:02:41.210 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:41.210 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:41.210 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 00:02:41.210 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:41.211 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:41.211 | INFO | Uploaded 36GiB to GPU in 2.07241s (17.3711GiB/s) | 20260506 00:02:41.211 | INFO | Downloaded 34GiB to GPU in 2.04391s (16.6348GiB/s) | 20260506 00:02:46.349 | INFO | Analyzing chunk frames = [7359488,7360000) | 20260506 00:02:46.349 | INFO | MemInfo: { | 20260506 00:02:46.349 | INFO | "gpu" : " 17 GB", | 20260506 00:02:46.349 | INFO | "huge" : "", | 20260506 00:02:46.349 | INFO | "huge_shared" : " 27 GB", | 20260506 00:02:46.349 | INFO | "malloc" : " 14 GB", | 20260506 00:02:46.349 | INFO | "pinned" : " 29 GB" | 20260506 00:02:46.349 | INFO | } | 20260506 00:02:46.349 | INFO | Perf report for the last 5.13908 seconds: | 20260506 00:02:46.349 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:02:46.349 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:46.350 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:46.350 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:46.350 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260506 00:02:46.350 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:46.350 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:46.350 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 00:02:46.350 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:46.350 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:46.350 | INFO | Uploaded 36GiB to GPU in 2.0421s (17.6289GiB/s) | 20260506 00:02:46.350 | INFO | Downloaded 34GiB to GPU in 2.06337s (16.4779GiB/s) | 20260506 00:02:51.491 | INFO | Analyzing chunk frames = [7360000,7360512) | 20260506 00:02:51.491 | INFO | MemInfo: { | 20260506 00:02:51.491 | INFO | "gpu" : " 17 GB", | 20260506 00:02:51.491 | INFO | "huge" : "", | 20260506 00:02:51.492 | INFO | "huge_shared" : " 27 GB", | 20260506 00:02:51.492 | INFO | "malloc" : " 14 GB", | 20260506 00:02:51.492 | INFO | "pinned" : " 29 GB" | 20260506 00:02:51.492 | INFO | } | 20260506 00:02:51.492 | INFO | Perf report for the last 5.14208 seconds: | 20260506 00:02:51.492 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:02:51.492 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:51.492 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:51.492 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:51.492 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260506 00:02:51.492 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:51.492 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:51.492 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 00:02:51.492 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:51.492 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:51.492 | INFO | Uploaded 36GiB to GPU in 2.11029s (17.0593GiB/s) | 20260506 00:02:51.492 | INFO | Downloaded 34GiB to GPU in 2.01079s (16.9088GiB/s) | 20260506 00:02:51.608 | INFO | Starting a round of Baz IO | 20260506 00:02:54.756 | INFO | Populating RTMetrics report | 20260506 00:02:54.756 | INFO | RTMetrics report populated | 20260506 00:02:54.756 | INFO | Opening tmp RTMetrics json file | 20260506 00:02:54.757 | INFO | Serializing RTMetrics to json | 20260506 00:02:54.759 | INFO | Serialization done, swapping tmp file now... | 20260506 00:02:54.759 | INFO | RTMetrics done | 20260506 00:02:54.836 | INFO | Finished a round of Baz IO: Wrote 1.871 GiB in 3.227 seconds | 20260506 00:02:54.836 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.39% / 0.00% / 78.61% | 20260506 00:02:54.836 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0975211% / 0% / 6.35783e-05% | 20260506 00:02:54.836 | INFO | Baz IO overall: | 20260506 00:02:54.836 | INFO | BytesWritten : 4192920940119 | 20260506 00:02:54.836 | INFO | NumEvents : 1974048905327 | 20260506 00:02:56.628 | INFO | Analyzing chunk frames = [7360512,7361024) | 20260506 00:02:56.628 | INFO | MemInfo: { | 20260506 00:02:56.628 | INFO | "gpu" : " 17 GB", | 20260506 00:02:56.628 | INFO | "huge" : "", | 20260506 00:02:56.628 | INFO | "huge_shared" : " 27 GB", | 20260506 00:02:56.628 | INFO | "malloc" : " 14 GB", | 20260506 00:02:56.628 | INFO | "pinned" : " 29 GB" | 20260506 00:02:56.628 | INFO | } | 20260506 00:02:56.628 | INFO | Perf report for the last 5.13642 seconds: | 20260506 00:02:56.628 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:02:56.628 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:56.628 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:56.628 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:56.628 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260506 00:02:56.628 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:02:56.628 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260506 00:02:56.628 | INFO | PRE_HQ : 59.8%, 1.7, 53.9ms, 0 | 20260506 00:02:56.628 | INFO | BAZWRITER : 62.0%, 3.5, 113.1ms, 0 | 20260506 00:02:56.628 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:02:56.628 | INFO | Uploaded 36GiB to GPU in 2.16774s (16.6072GiB/s) | 20260506 00:02:56.628 | INFO | Downloaded 38GiB to GPU in 2.23585s (16.9958GiB/s) | 20260506 00:03:01.770 | INFO | Analyzing chunk frames = [7361024,7361536) | 20260506 00:03:01.770 | INFO | MemInfo: { | 20260506 00:03:01.770 | INFO | "gpu" : " 17 GB", | 20260506 00:03:01.770 | INFO | "huge" : "", | 20260506 00:03:01.770 | INFO | "huge_shared" : " 27 GB", | 20260506 00:03:01.770 | INFO | "malloc" : " 14 GB", | 20260506 00:03:01.770 | INFO | "pinned" : " 29 GB" | 20260506 00:03:01.770 | INFO | } | 20260506 00:03:01.770 | INFO | Perf report for the last 5.14159 seconds: | 20260506 00:03:01.770 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:03:01.770 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:01.770 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:01.770 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:01.770 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 00:03:01.770 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:01.770 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:01.770 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:03:01.770 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:01.770 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:01.770 | INFO | Uploaded 36GiB to GPU in 2.11282s (17.0388GiB/s) | 20260506 00:03:01.770 | INFO | Downloaded 34GiB to GPU in 2.04253s (16.646GiB/s) | 20260506 00:03:06.908 | INFO | Analyzing chunk frames = [7361536,7362048) | 20260506 00:03:06.908 | INFO | MemInfo: { | 20260506 00:03:06.908 | INFO | "gpu" : " 17 GB", | 20260506 00:03:06.908 | INFO | "huge" : "", | 20260506 00:03:06.908 | INFO | "huge_shared" : " 27 GB", | 20260506 00:03:06.908 | INFO | "malloc" : " 14 GB", | 20260506 00:03:06.908 | INFO | "pinned" : " 29 GB" | 20260506 00:03:06.908 | INFO | } | 20260506 00:03:06.908 | INFO | Perf report for the last 5.13858 seconds: | 20260506 00:03:06.908 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:03:06.908 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:06.908 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:06.908 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:06.908 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 00:03:06.908 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:06.908 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:06.908 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:03:06.908 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:06.908 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:06.909 | INFO | Uploaded 36GiB to GPU in 2.07743s (17.3291GiB/s) | 20260506 00:03:06.909 | INFO | Downloaded 34GiB to GPU in 2.02123s (16.8214GiB/s) | 20260506 00:03:12.042 | INFO | Analyzing chunk frames = [7362048,7362560) | 20260506 00:03:12.042 | INFO | MemInfo: { | 20260506 00:03:12.042 | INFO | "gpu" : " 17 GB", | 20260506 00:03:12.042 | INFO | "huge" : "", | 20260506 00:03:12.042 | INFO | "huge_shared" : " 27 GB", | 20260506 00:03:12.042 | INFO | "malloc" : " 14 GB", | 20260506 00:03:12.042 | INFO | "pinned" : " 29 GB" | 20260506 00:03:12.042 | INFO | } | 20260506 00:03:12.042 | INFO | Perf report for the last 5.13386 seconds: | 20260506 00:03:12.042 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:03:12.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:12.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:12.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:12.042 | INFO | ANALYSIS : 61.8%, 3.0, 95.8ms, 0 | 20260506 00:03:12.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:12.042 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:12.042 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 00:03:12.042 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:12.042 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:12.042 | INFO | Uploaded 36GiB to GPU in 2.13475s (16.8638GiB/s) | 20260506 00:03:12.042 | INFO | Downloaded 34GiB to GPU in 2.02242s (16.8116GiB/s) | 20260506 00:03:17.181 | INFO | Analyzing chunk frames = [7362560,7363072) | 20260506 00:03:17.181 | INFO | MemInfo: { | 20260506 00:03:17.181 | INFO | "gpu" : " 17 GB", | 20260506 00:03:17.181 | INFO | "huge" : "", | 20260506 00:03:17.181 | INFO | "huge_shared" : " 27 GB", | 20260506 00:03:17.181 | INFO | "malloc" : " 14 GB", | 20260506 00:03:17.181 | INFO | "pinned" : " 29 GB" | 20260506 00:03:17.181 | INFO | } | 20260506 00:03:17.181 | INFO | Perf report for the last 5.1389 seconds: | 20260506 00:03:17.181 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:03:17.181 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:17.181 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:17.181 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:17.181 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260506 00:03:17.181 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:17.181 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:17.181 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:03:17.181 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:17.181 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:17.181 | INFO | Uploaded 36GiB to GPU in 2.15969s (16.669GiB/s) | 20260506 00:03:17.181 | INFO | Downloaded 34GiB to GPU in 2.07117s (16.4159GiB/s) | 20260506 00:03:22.327 | INFO | Analyzing chunk frames = [7363072,7363584) | 20260506 00:03:22.327 | INFO | MemInfo: { | 20260506 00:03:22.327 | INFO | "gpu" : " 17 GB", | 20260506 00:03:22.327 | INFO | "huge" : "", | 20260506 00:03:22.327 | INFO | "huge_shared" : " 27 GB", | 20260506 00:03:22.327 | INFO | "malloc" : " 14 GB", | 20260506 00:03:22.327 | INFO | "pinned" : " 29 GB" | 20260506 00:03:22.327 | INFO | } | 20260506 00:03:22.327 | INFO | Perf report for the last 5.14625 seconds: | 20260506 00:03:22.327 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:03:22.327 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:22.327 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:22.327 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:22.327 | INFO | ANALYSIS : 61.5%, 3.0, 96.1ms, 0 | 20260506 00:03:22.327 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:22.327 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:22.327 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:03:22.327 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:22.327 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:22.328 | INFO | Uploaded 36GiB to GPU in 2.07948s (17.312GiB/s) | 20260506 00:03:22.328 | INFO | Downloaded 34GiB to GPU in 2.03768s (16.6856GiB/s) | 20260506 00:03:27.470 | INFO | Analyzing chunk frames = [7363584,7364096) | 20260506 00:03:27.470 | INFO | MemInfo: { | 20260506 00:03:27.470 | INFO | "gpu" : " 17 GB", | 20260506 00:03:27.470 | INFO | "huge" : "", | 20260506 00:03:27.470 | INFO | "huge_shared" : " 27 GB", | 20260506 00:03:27.470 | INFO | "malloc" : " 14 GB", | 20260506 00:03:27.470 | INFO | "pinned" : " 29 GB" | 20260506 00:03:27.470 | INFO | } | 20260506 00:03:27.470 | INFO | Perf report for the last 5.14248 seconds: | 20260506 00:03:27.470 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:03:27.470 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:27.470 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:27.470 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:27.470 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260506 00:03:27.470 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:27.470 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:27.470 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 00:03:27.470 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:27.470 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:27.470 | INFO | Uploaded 36GiB to GPU in 2.09567s (17.1782GiB/s) | 20260506 00:03:27.470 | INFO | Downloaded 34GiB to GPU in 2.07041s (16.4219GiB/s) | 20260506 00:03:32.604 | INFO | Analyzing chunk frames = [7364096,7364608) | 20260506 00:03:32.604 | INFO | MemInfo: { | 20260506 00:03:32.604 | INFO | "gpu" : " 17 GB", | 20260506 00:03:32.604 | INFO | "huge" : "", | 20260506 00:03:32.604 | INFO | "huge_shared" : " 27 GB", | 20260506 00:03:32.604 | INFO | "malloc" : " 14 GB", | 20260506 00:03:32.604 | INFO | "pinned" : " 29 GB" | 20260506 00:03:32.605 | INFO | } | 20260506 00:03:32.605 | INFO | Perf report for the last 5.13486 seconds: | 20260506 00:03:32.605 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:03:32.605 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:32.605 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:32.605 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:32.605 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260506 00:03:32.605 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:32.605 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:32.605 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 00:03:32.605 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:32.605 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:32.605 | INFO | Uploaded 36GiB to GPU in 2.12977s (16.9033GiB/s) | 20260506 00:03:32.605 | INFO | Downloaded 34GiB to GPU in 2.02539s (16.7869GiB/s) | 20260506 00:03:32.605 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 00:03:32.605 | INFO | Total mem usage: 15.3 GiB | 20260506 00:03:32.605 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 00:03:32.605 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.14%) | 20260506 00:03:32.605 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 00:03:32.605 | INFO | High Water Breakdown: | 20260506 00:03:32.605 | INFO | 60.9% : 8.55 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 00:03:32.605 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 00:03:32.605 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 00:03:32.605 | INFO | ------------------------------------------ | 20260506 00:03:32.605 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:03:32.605 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 00:03:32.605 | INFO | Total mem usage: 26.5 GiB | 20260506 00:03:32.605 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 00:03:32.605 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 00:03:32.605 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260506 00:03:32.605 | INFO | High Water Breakdown: | 20260506 00:03:32.605 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:03:32.605 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:03:32.605 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:03:32.605 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:03:32.605 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:03:32.605 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:03:32.605 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 00:03:32.605 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 00:03:32.605 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:03:32.605 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 00:03:32.605 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 00:03:32.605 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 00:03:32.605 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 00:03:32.605 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 00:03:32.605 | INFO | ------------------------------------------ | 20260506 00:03:32.605 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:03:32.605 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 00:03:32.605 | INFO | Total mem usage: 25.6 GiB | 20260506 00:03:32.605 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 00:03:32.605 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 00:03:32.605 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 00:03:32.605 | INFO | High Water Breakdown: | 20260506 00:03:32.605 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 00:03:32.605 | INFO | ------------------------------------------ | 20260506 00:03:32.605 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:03:32.605 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 00:03:32.605 | INFO | Total mem usage: 15.7 GiB | 20260506 00:03:32.605 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 00:03:32.605 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 00:03:32.605 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 00:03:32.605 | INFO | High Water Breakdown: | 20260506 00:03:32.605 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 00:03:32.605 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 00:03:32.605 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 00:03:32.606 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 00:03:32.606 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 00:03:32.606 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 00:03:32.606 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 00:03:32.606 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 00:03:32.606 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 00:03:32.606 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:03:32.606 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 00:03:32.606 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:03:32.606 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 00:03:32.606 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 00:03:32.606 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 00:03:32.606 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 00:03:32.606 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:03:32.606 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 00:03:32.606 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:03:32.606 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:03:32.606 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:03:32.606 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 00:03:32.606 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 00:03:32.606 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:03:32.606 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 00:03:32.606 | INFO | ------------------------------------------ | 20260506 00:03:32.606 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:03:32.606 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 00:03:32.606 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:03:32.606 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:03:32.606 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.38e+01, 1.34e+01 / 7.98e-01 | 20260506 00:03:32.606 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.04e-01, 1.06e-02 / 3.11e-03 | 20260506 00:03:32.606 | INFO | PerfReport: 11.86%, Baseline -- 3.71e+00 / 4.52e+00, 5.74e-01 / 1.88e-01 | 20260506 00:03:32.606 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.53e+00, 1.92e-01 / 9.03e-02 | 20260506 00:03:32.606 | INFO | PerfReport: 15.99%, DME -- 4.00e+01 / 4.39e+01, 3.83e+00 / 1.07e+00 | 20260506 00:03:32.606 | INFO | PerfReport: 57.29%, FrameLabeling -- 1.79e+01 / 1.94e+01, 1.46e+00 / 1.02e+00 | 20260506 00:03:32.606 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.23e+00 / 2.75e+00, 4.26e-01 / 2.55e-01 | 20260506 00:03:32.606 | INFO | PerfReport: 3.10%, Metrics -- 9.69e-01 / 1.25e+00, 2.61e-01 / 1.13e-01 | 20260506 00:03:32.606 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.35e-01 / 3.84e-01, 5.66e-02 / 2.59e-02 | 20260506 00:03:32.606 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 00:03:32.606 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:03:32.606 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:03:32.606 | INFO | PerfReport: Full Execution -- 9.82e+01 / -nan, 2.46e+01 / 0.00e+00 | 20260506 00:03:32.606 | INFO | PerfReport: 22.30%, Upload -- 2.19e+01 / 0.00e+00, 7.50e+00 / 0.00e+00 | 20260506 00:03:32.606 | INFO | PerfReport: 21.84%, Download -- 2.15e+01 / 0.00e+00, 7.60e+00 / 0.00e+00 | 20260506 00:03:32.606 | INFO | PerfReport: 31.87%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 00:03:32.606 | INFO | PerfReport: 2.11%, LockUpload -- 2.08e+00 / 0.00e+00, 7.88e+00 / 0.00e+00 | 20260506 00:03:32.606 | INFO | PerfReport: 1.31%, LockDownload -- 1.28e+00 / 0.00e+00, 2.86e+00 / 0.00e+00 | 20260506 00:03:32.606 | INFO | PerfReport: 20.56%, LockCompute -- 2.02e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 00:03:32.606 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.18e-02 / 0.00e+00 | 20260506 00:03:32.722 | INFO | Starting a round of Baz IO | 20260506 00:03:35.872 | INFO | Populating RTMetrics report | 20260506 00:03:35.872 | INFO | RTMetrics report populated | 20260506 00:03:35.872 | INFO | Opening tmp RTMetrics json file | 20260506 00:03:35.873 | INFO | Serializing RTMetrics to json | 20260506 00:03:35.875 | INFO | Serialization done, swapping tmp file now... | 20260506 00:03:35.875 | INFO | RTMetrics done | 20260506 00:03:35.952 | INFO | Finished a round of Baz IO: Wrote 1.87 GiB in 3.23 seconds | 20260506 00:03:35.952 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.39% / 0.00% / 78.61% | 20260506 00:03:35.952 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0976165% / 3.97364e-06% / 6.35783e-05% | 20260506 00:03:35.952 | INFO | Baz IO overall: | 20260506 00:03:35.952 | INFO | BytesWritten : 4194928840615 | 20260506 00:03:35.952 | INFO | NumEvents : 1974968341889 | 20260506 00:03:37.738 | INFO | Analyzing chunk frames = [7364608,7365120) | 20260506 00:03:37.738 | INFO | MemInfo: { | 20260506 00:03:37.738 | INFO | "gpu" : " 17 GB", | 20260506 00:03:37.738 | INFO | "huge" : "", | 20260506 00:03:37.738 | INFO | "huge_shared" : " 27 GB", | 20260506 00:03:37.738 | INFO | "malloc" : " 14 GB", | 20260506 00:03:37.738 | INFO | "pinned" : " 29 GB" | 20260506 00:03:37.738 | INFO | } | 20260506 00:03:37.738 | INFO | Perf report for the last 5.13376 seconds: | 20260506 00:03:37.738 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:03:37.738 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:37.739 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:37.739 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:37.739 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260506 00:03:37.739 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:37.739 | INFO | RT_METRICS : 43.2%, 1.0, 22.6ms, 0 | 20260506 00:03:37.739 | INFO | PRE_HQ : 59.7%, 1.7, 54.0ms, 0 | 20260506 00:03:37.739 | INFO | BAZWRITER : 62.1%, 3.6, 113.7ms, 0 | 20260506 00:03:37.739 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:03:37.739 | INFO | Uploaded 36GiB to GPU in 2.15496s (16.7056GiB/s) | 20260506 00:03:37.739 | INFO | Downloaded 38GiB to GPU in 2.23975s (16.9661GiB/s) | 20260506 00:03:42.880 | INFO | Analyzing chunk frames = [7365120,7365632) | 20260506 00:03:42.880 | INFO | MemInfo: { | 20260506 00:03:42.880 | INFO | "gpu" : " 17 GB", | 20260506 00:03:42.880 | INFO | "huge" : "", | 20260506 00:03:42.880 | INFO | "huge_shared" : " 27 GB", | 20260506 00:03:42.880 | INFO | "malloc" : " 14 GB", | 20260506 00:03:42.880 | INFO | "pinned" : " 29 GB" | 20260506 00:03:42.880 | INFO | } | 20260506 00:03:42.880 | INFO | Perf report for the last 5.1419 seconds: | 20260506 00:03:42.880 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:03:42.880 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:42.880 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:42.880 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:42.880 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 00:03:42.880 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:42.880 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:42.880 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:03:42.880 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:42.880 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:42.881 | INFO | Uploaded 36GiB to GPU in 2.09698s (17.1675GiB/s) | 20260506 00:03:42.881 | INFO | Downloaded 34GiB to GPU in 2.02922s (16.7552GiB/s) | 20260506 00:03:48.021 | INFO | Analyzing chunk frames = [7365632,7366144) | 20260506 00:03:48.021 | INFO | MemInfo: { | 20260506 00:03:48.021 | INFO | "gpu" : " 17 GB", | 20260506 00:03:48.021 | INFO | "huge" : "", | 20260506 00:03:48.021 | INFO | "huge_shared" : " 27 GB", | 20260506 00:03:48.021 | INFO | "malloc" : " 14 GB", | 20260506 00:03:48.021 | INFO | "pinned" : " 29 GB" | 20260506 00:03:48.021 | INFO | } | 20260506 00:03:48.021 | INFO | Perf report for the last 5.14117 seconds: | 20260506 00:03:48.021 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:03:48.021 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:48.021 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:48.021 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:48.022 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 00:03:48.022 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:48.022 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:48.022 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 00:03:48.022 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:48.022 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:48.022 | INFO | Uploaded 36GiB to GPU in 2.08255s (17.2865GiB/s) | 20260506 00:03:48.022 | INFO | Downloaded 34GiB to GPU in 2.02716s (16.7723GiB/s) | 20260506 00:03:53.165 | INFO | Analyzing chunk frames = [7366144,7366656) | 20260506 00:03:53.165 | INFO | MemInfo: { | 20260506 00:03:53.165 | INFO | "gpu" : " 17 GB", | 20260506 00:03:53.165 | INFO | "huge" : "", | 20260506 00:03:53.165 | INFO | "huge_shared" : " 27 GB", | 20260506 00:03:53.165 | INFO | "malloc" : " 14 GB", | 20260506 00:03:53.165 | INFO | "pinned" : " 29 GB" | 20260506 00:03:53.165 | INFO | } | 20260506 00:03:53.165 | INFO | Perf report for the last 5.14347 seconds: | 20260506 00:03:53.165 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:03:53.165 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:53.165 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:53.165 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:53.165 | INFO | ANALYSIS : 61.6%, 3.0, 95.7ms, 0 | 20260506 00:03:53.165 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:53.165 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:53.165 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 00:03:53.165 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:53.165 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:53.165 | INFO | Uploaded 36GiB to GPU in 2.1379s (16.839GiB/s) | 20260506 00:03:53.165 | INFO | Downloaded 34GiB to GPU in 2.01368s (16.8845GiB/s) | 20260506 00:03:58.300 | INFO | Analyzing chunk frames = [7366656,7367168) | 20260506 00:03:58.300 | INFO | MemInfo: { | 20260506 00:03:58.300 | INFO | "gpu" : " 17 GB", | 20260506 00:03:58.300 | INFO | "huge" : "", | 20260506 00:03:58.300 | INFO | "huge_shared" : " 27 GB", | 20260506 00:03:58.300 | INFO | "malloc" : " 14 GB", | 20260506 00:03:58.300 | INFO | "pinned" : " 29 GB" | 20260506 00:03:58.300 | INFO | } | 20260506 00:03:58.300 | INFO | Perf report for the last 5.13539 seconds: | 20260506 00:03:58.300 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:03:58.300 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:58.300 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:58.300 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:58.300 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 00:03:58.300 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:58.300 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:58.300 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260506 00:03:58.300 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:58.300 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:03:58.301 | INFO | Uploaded 36GiB to GPU in 2.09535s (17.1809GiB/s) | 20260506 00:03:58.301 | INFO | Downloaded 34GiB to GPU in 2.0477s (16.604GiB/s) | 20260506 00:04:03.443 | INFO | Analyzing chunk frames = [7367168,7367680) | 20260506 00:04:03.443 | INFO | MemInfo: { | 20260506 00:04:03.443 | INFO | "gpu" : " 17 GB", | 20260506 00:04:03.443 | INFO | "huge" : "", | 20260506 00:04:03.443 | INFO | "huge_shared" : " 27 GB", | 20260506 00:04:03.443 | INFO | "malloc" : " 14 GB", | 20260506 00:04:03.443 | INFO | "pinned" : " 29 GB" | 20260506 00:04:03.443 | INFO | } | 20260506 00:04:03.443 | INFO | Perf report for the last 5.14278 seconds: | 20260506 00:04:03.443 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:04:03.443 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:03.443 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:03.443 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:03.443 | INFO | ANALYSIS : 61.4%, 3.0, 95.9ms, 0 | 20260506 00:04:03.443 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:03.443 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:03.443 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 00:04:03.443 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:03.443 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:03.444 | INFO | Uploaded 36GiB to GPU in 2.08609s (17.2571GiB/s) | 20260506 00:04:03.444 | INFO | Downloaded 34GiB to GPU in 2.04576s (16.6198GiB/s) | 20260506 00:04:08.582 | INFO | Analyzing chunk frames = [7367680,7368192) | 20260506 00:04:08.582 | INFO | MemInfo: { | 20260506 00:04:08.582 | INFO | "gpu" : " 17 GB", | 20260506 00:04:08.582 | INFO | "huge" : "", | 20260506 00:04:08.582 | INFO | "huge_shared" : " 27 GB", | 20260506 00:04:08.582 | INFO | "malloc" : " 14 GB", | 20260506 00:04:08.582 | INFO | "pinned" : " 29 GB" | 20260506 00:04:08.582 | INFO | } | 20260506 00:04:08.582 | INFO | Perf report for the last 5.13927 seconds: | 20260506 00:04:08.582 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:04:08.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:08.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:08.582 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:08.582 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260506 00:04:08.582 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:08.582 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:08.582 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 00:04:08.583 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:08.583 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:08.583 | INFO | Uploaded 36GiB to GPU in 2.07s (17.3913GiB/s) | 20260506 00:04:08.583 | INFO | Downloaded 34GiB to GPU in 2.06905s (16.4326GiB/s) | 20260506 00:04:13.719 | INFO | Analyzing chunk frames = [7368192,7368704) | 20260506 00:04:13.719 | INFO | MemInfo: { | 20260506 00:04:13.719 | INFO | "gpu" : " 17 GB", | 20260506 00:04:13.719 | INFO | "huge" : "", | 20260506 00:04:13.719 | INFO | "huge_shared" : " 27 GB", | 20260506 00:04:13.719 | INFO | "malloc" : " 14 GB", | 20260506 00:04:13.719 | INFO | "pinned" : " 29 GB" | 20260506 00:04:13.719 | INFO | } | 20260506 00:04:13.719 | INFO | Perf report for the last 5.13686 seconds: | 20260506 00:04:13.719 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:04:13.719 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:13.719 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:13.719 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:13.719 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260506 00:04:13.720 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:13.720 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:13.720 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260506 00:04:13.720 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:13.720 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:13.720 | INFO | Uploaded 36GiB to GPU in 2.13109s (16.8928GiB/s) | 20260506 00:04:13.720 | INFO | Downloaded 34GiB to GPU in 2.02375s (16.8005GiB/s) | 20260506 00:04:13.837 | INFO | Starting a round of Baz IO | 20260506 00:04:16.982 | INFO | Populating RTMetrics report | 20260506 00:04:16.983 | INFO | RTMetrics report populated | 20260506 00:04:16.983 | INFO | Opening tmp RTMetrics json file | 20260506 00:04:16.983 | INFO | Serializing RTMetrics to json | 20260506 00:04:16.984 | INFO | Serialization done, swapping tmp file now... | 20260506 00:04:16.984 | INFO | RTMetrics done | 20260506 00:04:17.067 | INFO | Finished a round of Baz IO: Wrote 1.87 GiB in 3.23 seconds | 20260506 00:04:17.067 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.39% / 0.00% / 78.61% | 20260506 00:04:17.067 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0974298% / 0% / 5.96046e-05% | 20260506 00:04:17.067 | INFO | Baz IO overall: | 20260506 00:04:17.067 | INFO | BytesWritten : 4196936981343 | 20260506 00:04:17.067 | INFO | NumEvents : 1975887752698 | 20260506 00:04:18.856 | INFO | Analyzing chunk frames = [7368704,7369216) | 20260506 00:04:18.856 | INFO | MemInfo: { | 20260506 00:04:18.856 | INFO | "gpu" : " 17 GB", | 20260506 00:04:18.856 | INFO | "huge" : "", | 20260506 00:04:18.856 | INFO | "huge_shared" : " 27 GB", | 20260506 00:04:18.856 | INFO | "malloc" : " 14 GB", | 20260506 00:04:18.856 | INFO | "pinned" : " 29 GB" | 20260506 00:04:18.856 | INFO | } | 20260506 00:04:18.857 | INFO | Perf report for the last 5.13724 seconds: | 20260506 00:04:18.857 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:04:18.857 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:18.857 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:18.857 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:18.857 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260506 00:04:18.857 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:18.857 | INFO | RT_METRICS : 43.4%, 1.0, 22.8ms, 0 | 20260506 00:04:18.857 | INFO | PRE_HQ : 59.8%, 1.8, 54.6ms, 0 | 20260506 00:04:18.857 | INFO | BAZWRITER : 62.0%, 3.6, 114.2ms, 0 | 20260506 00:04:18.857 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:04:18.857 | INFO | Uploaded 36GiB to GPU in 2.16499s (16.6282GiB/s) | 20260506 00:04:18.857 | INFO | Downloaded 38GiB to GPU in 2.23585s (16.9958GiB/s) | 20260506 00:04:23.992 | INFO | Analyzing chunk frames = [7369216,7369728) | 20260506 00:04:23.992 | INFO | MemInfo: { | 20260506 00:04:23.992 | INFO | "gpu" : " 17 GB", | 20260506 00:04:23.992 | INFO | "huge" : "", | 20260506 00:04:23.992 | INFO | "huge_shared" : " 27 GB", | 20260506 00:04:23.992 | INFO | "malloc" : " 14 GB", | 20260506 00:04:23.992 | INFO | "pinned" : " 29 GB" | 20260506 00:04:23.993 | INFO | } | 20260506 00:04:23.993 | INFO | Perf report for the last 5.1362 seconds: | 20260506 00:04:23.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:04:23.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:23.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:23.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:23.993 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 00:04:23.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:23.993 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:23.993 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 00:04:23.993 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:23.993 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:23.993 | INFO | Uploaded 36GiB to GPU in 2.09098s (17.2168GiB/s) | 20260506 00:04:23.993 | INFO | Downloaded 34GiB to GPU in 2.02945s (16.7533GiB/s) | 20260506 00:04:29.137 | INFO | Analyzing chunk frames = [7369728,7370240) | 20260506 00:04:29.137 | INFO | MemInfo: { | 20260506 00:04:29.137 | INFO | "gpu" : " 17 GB", | 20260506 00:04:29.137 | INFO | "huge" : "", | 20260506 00:04:29.137 | INFO | "huge_shared" : " 27 GB", | 20260506 00:04:29.137 | INFO | "malloc" : " 14 GB", | 20260506 00:04:29.137 | INFO | "pinned" : " 29 GB" | 20260506 00:04:29.137 | INFO | } | 20260506 00:04:29.137 | INFO | Perf report for the last 5.1444 seconds: | 20260506 00:04:29.137 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:04:29.137 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:29.137 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:29.137 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:29.137 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 00:04:29.137 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:29.137 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:29.137 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 00:04:29.137 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:29.137 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:29.137 | INFO | Uploaded 36GiB to GPU in 2.14645s (16.7719GiB/s) | 20260506 00:04:29.137 | INFO | Downloaded 34GiB to GPU in 2.04695s (16.6101GiB/s) | 20260506 00:04:34.280 | INFO | Analyzing chunk frames = [7370240,7370752) | 20260506 00:04:34.280 | INFO | MemInfo: { | 20260506 00:04:34.280 | INFO | "gpu" : " 17 GB", | 20260506 00:04:34.280 | INFO | "huge" : "", | 20260506 00:04:34.280 | INFO | "huge_shared" : " 27 GB", | 20260506 00:04:34.280 | INFO | "malloc" : " 14 GB", | 20260506 00:04:34.280 | INFO | "pinned" : " 29 GB" | 20260506 00:04:34.280 | INFO | } | 20260506 00:04:34.280 | INFO | Perf report for the last 5.14309 seconds: | 20260506 00:04:34.280 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:04:34.280 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:34.280 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:34.280 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:34.280 | INFO | ANALYSIS : 61.7%, 3.0, 95.8ms, 0 | 20260506 00:04:34.280 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:34.280 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:34.280 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 00:04:34.280 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:34.280 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:34.281 | INFO | Uploaded 36GiB to GPU in 2.13893s (16.8308GiB/s) | 20260506 00:04:34.281 | INFO | Downloaded 34GiB to GPU in 2.01299s (16.8903GiB/s) | 20260506 00:04:39.418 | INFO | Analyzing chunk frames = [7370752,7371264) | 20260506 00:04:39.418 | INFO | MemInfo: { | 20260506 00:04:39.418 | INFO | "gpu" : " 17 GB", | 20260506 00:04:39.418 | INFO | "huge" : "", | 20260506 00:04:39.418 | INFO | "huge_shared" : " 27 GB", | 20260506 00:04:39.418 | INFO | "malloc" : " 14 GB", | 20260506 00:04:39.418 | INFO | "pinned" : " 29 GB" | 20260506 00:04:39.418 | INFO | } | 20260506 00:04:39.418 | INFO | Perf report for the last 5.13796 seconds: | 20260506 00:04:39.418 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:04:39.418 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:39.418 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:39.418 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:39.418 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 00:04:39.418 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:39.418 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:39.418 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 00:04:39.418 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:39.418 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:39.418 | INFO | Uploaded 36GiB to GPU in 2.08685s (17.2509GiB/s) | 20260506 00:04:39.418 | INFO | Downloaded 34GiB to GPU in 2.04294s (16.6427GiB/s) | 20260506 00:04:44.553 | INFO | Analyzing chunk frames = [7371264,7371776) | 20260506 00:04:44.553 | INFO | MemInfo: { | 20260506 00:04:44.553 | INFO | "gpu" : " 17 GB", | 20260506 00:04:44.553 | INFO | "huge" : "", | 20260506 00:04:44.553 | INFO | "huge_shared" : " 27 GB", | 20260506 00:04:44.553 | INFO | "malloc" : " 14 GB", | 20260506 00:04:44.553 | INFO | "pinned" : " 29 GB" | 20260506 00:04:44.553 | INFO | } | 20260506 00:04:44.553 | INFO | Perf report for the last 5.13435 seconds: | 20260506 00:04:44.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:04:44.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:44.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:44.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:44.553 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 00:04:44.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:44.553 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:44.553 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 00:04:44.553 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:44.553 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:44.553 | INFO | Uploaded 36GiB to GPU in 2.06596s (17.4253GiB/s) | 20260506 00:04:44.553 | INFO | Downloaded 34GiB to GPU in 2.0377s (16.6854GiB/s) | 20260506 00:04:49.692 | INFO | Analyzing chunk frames = [7371776,7372288) | 20260506 00:04:49.692 | INFO | MemInfo: { | 20260506 00:04:49.692 | INFO | "gpu" : " 17 GB", | 20260506 00:04:49.692 | INFO | "huge" : "", | 20260506 00:04:49.692 | INFO | "huge_shared" : " 27 GB", | 20260506 00:04:49.692 | INFO | "malloc" : " 14 GB", | 20260506 00:04:49.692 | INFO | "pinned" : " 29 GB" | 20260506 00:04:49.692 | INFO | } | 20260506 00:04:49.692 | INFO | Perf report for the last 5.13936 seconds: | 20260506 00:04:49.692 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:04:49.692 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:49.692 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:49.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:49.692 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260506 00:04:49.692 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:49.692 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:49.692 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 00:04:49.692 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:49.692 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:49.692 | INFO | Uploaded 36GiB to GPU in 2.08111s (17.2984GiB/s) | 20260506 00:04:49.692 | INFO | Downloaded 34GiB to GPU in 2.06022s (16.5031GiB/s) | 20260506 00:04:54.832 | INFO | Analyzing chunk frames = [7372288,7372800) | 20260506 00:04:54.832 | INFO | MemInfo: { | 20260506 00:04:54.832 | INFO | "gpu" : " 17 GB", | 20260506 00:04:54.832 | INFO | "huge" : "", | 20260506 00:04:54.832 | INFO | "huge_shared" : " 27 GB", | 20260506 00:04:54.832 | INFO | "malloc" : " 14 GB", | 20260506 00:04:54.832 | INFO | "pinned" : " 29 GB" | 20260506 00:04:54.833 | INFO | } | 20260506 00:04:54.833 | INFO | Perf report for the last 5.14062 seconds: | 20260506 00:04:54.833 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:04:54.833 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:54.833 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:54.833 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:54.833 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260506 00:04:54.833 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:54.833 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:54.833 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 00:04:54.833 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:54.833 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:54.833 | INFO | Uploaded 36GiB to GPU in 2.1216s (16.9683GiB/s) | 20260506 00:04:54.833 | INFO | Downloaded 34GiB to GPU in 2.01355s (16.8856GiB/s) | 20260506 00:04:54.833 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 00:04:54.833 | INFO | Total mem usage: 15.3 GiB | 20260506 00:04:54.833 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 00:04:54.833 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.12%) | 20260506 00:04:54.833 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 00:04:54.833 | INFO | High Water Breakdown: | 20260506 00:04:54.833 | INFO | 60.9% : 8.55 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 00:04:54.833 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 00:04:54.833 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 00:04:54.833 | INFO | ------------------------------------------ | 20260506 00:04:54.833 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:04:54.833 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 00:04:54.833 | INFO | Total mem usage: 26.5 GiB | 20260506 00:04:54.833 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 00:04:54.833 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 00:04:54.834 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 00:04:54.834 | INFO | High Water Breakdown: | 20260506 00:04:54.834 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:04:54.834 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:04:54.834 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:04:54.834 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:04:54.834 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:04:54.834 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:04:54.834 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 00:04:54.834 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 00:04:54.834 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:04:54.834 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 00:04:54.834 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 00:04:54.834 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 00:04:54.834 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 00:04:54.834 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 00:04:54.834 | INFO | ------------------------------------------ | 20260506 00:04:54.834 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:04:54.834 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 00:04:54.834 | INFO | Total mem usage: 25.6 GiB | 20260506 00:04:54.834 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 00:04:54.834 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 00:04:54.834 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 00:04:54.834 | INFO | High Water Breakdown: | 20260506 00:04:54.834 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 00:04:54.834 | INFO | ------------------------------------------ | 20260506 00:04:54.834 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:04:54.834 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 00:04:54.834 | INFO | Total mem usage: 15.7 GiB | 20260506 00:04:54.834 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 00:04:54.834 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 00:04:54.834 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 00:04:54.834 | INFO | High Water Breakdown: | 20260506 00:04:54.834 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 00:04:54.834 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 00:04:54.834 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 00:04:54.834 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 00:04:54.834 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 00:04:54.834 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 00:04:54.834 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 00:04:54.834 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 00:04:54.834 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 00:04:54.834 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:04:54.834 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 00:04:54.834 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:04:54.834 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 00:04:54.834 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 00:04:54.834 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 00:04:54.834 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 00:04:54.834 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:04:54.834 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 00:04:54.834 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:04:54.834 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:04:54.834 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:04:54.834 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 00:04:54.834 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 00:04:54.834 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:04:54.834 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 00:04:54.834 | INFO | ------------------------------------------ | 20260506 00:04:54.834 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:04:54.834 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 00:04:54.834 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:04:54.834 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:04:54.834 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.39e+01, 1.35e+01 / 8.09e-01 | 20260506 00:04:54.834 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.04e-01, 1.03e-02 / 2.61e-03 | 20260506 00:04:54.834 | INFO | PerfReport: 11.82%, Baseline -- 3.70e+00 / 4.48e+00, 5.68e-01 / 2.45e-01 | 20260506 00:04:54.834 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.53e+00, 1.89e-01 / 8.24e-02 | 20260506 00:04:54.834 | INFO | PerfReport: 15.99%, DME -- 4.01e+01 / 4.39e+01, 3.88e+00 / 9.23e-01 | 20260506 00:04:54.834 | INFO | PerfReport: 57.35%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.50e+00 / 1.20e+00 | 20260506 00:04:54.834 | INFO | PerfReport: 7.11%, PulseAccumulating -- 2.23e+00 / 2.75e+00, 4.28e-01 / 2.47e-01 | 20260506 00:04:54.834 | INFO | PerfReport: 3.09%, Metrics -- 9.68e-01 / 1.24e+00, 2.59e-01 / 1.04e-01 | 20260506 00:04:54.834 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.35e-01 / 3.86e-01, 5.61e-02 / 2.80e-02 | 20260506 00:04:54.834 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 00:04:54.834 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:04:54.834 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:04:54.834 | INFO | PerfReport: Full Execution -- 9.83e+01 / -nan, 2.47e+01 / 0.00e+00 | 20260506 00:04:54.834 | INFO | PerfReport: 22.35%, Upload -- 2.20e+01 / 0.00e+00, 7.55e+00 / 0.00e+00 | 20260506 00:04:54.835 | INFO | PerfReport: 21.83%, Download -- 2.15e+01 / 0.00e+00, 7.56e+00 / 0.00e+00 | 20260506 00:04:54.835 | INFO | PerfReport: 31.85%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 00:04:54.835 | INFO | PerfReport: 2.15%, LockUpload -- 2.11e+00 / 0.00e+00, 7.91e+00 / 0.00e+00 | 20260506 00:04:54.835 | INFO | PerfReport: 1.27%, LockDownload -- 1.25e+00 / 0.00e+00, 2.80e+00 / 0.00e+00 | 20260506 00:04:54.835 | INFO | PerfReport: 20.54%, LockCompute -- 2.02e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260506 00:04:54.835 | INFO | PerfReport: 0.01%, Misc -- 1.27e-02 / 0.00e+00, 1.40e-02 / 0.00e+00 | 20260506 00:04:54.950 | INFO | Starting a round of Baz IO | 20260506 00:04:58.100 | INFO | Populating RTMetrics report | 20260506 00:04:58.101 | INFO | RTMetrics report populated | 20260506 00:04:58.101 | INFO | Opening tmp RTMetrics json file | 20260506 00:04:58.102 | INFO | Serializing RTMetrics to json | 20260506 00:04:58.104 | INFO | Serialization done, swapping tmp file now... | 20260506 00:04:58.104 | INFO | RTMetrics done | 20260506 00:04:58.186 | INFO | Finished a round of Baz IO: Wrote 1.87 GiB in 3.235 seconds | 20260506 00:04:58.186 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.39% / 0.00% / 78.61% | 20260506 00:04:58.186 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0972907% / 0% / 8.74201e-05% | 20260506 00:04:58.186 | INFO | Baz IO overall: | 20260506 00:04:58.186 | INFO | BytesWritten : 4198944380682 | 20260506 00:04:58.186 | INFO | NumEvents : 1976806856856 | 20260506 00:04:59.975 | INFO | Analyzing chunk frames = [7372800,7373312) | 20260506 00:04:59.975 | INFO | MemInfo: { | 20260506 00:04:59.975 | INFO | "gpu" : " 17 GB", | 20260506 00:04:59.975 | INFO | "huge" : "", | 20260506 00:04:59.975 | INFO | "huge_shared" : " 27 GB", | 20260506 00:04:59.975 | INFO | "malloc" : " 14 GB", | 20260506 00:04:59.975 | INFO | "pinned" : " 29 GB" | 20260506 00:04:59.975 | INFO | } | 20260506 00:04:59.975 | INFO | Perf report for the last 5.14284 seconds: | 20260506 00:04:59.976 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:04:59.976 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:59.976 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:59.976 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:59.976 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260506 00:04:59.976 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:04:59.976 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260506 00:04:59.976 | INFO | PRE_HQ : 59.6%, 1.7, 54.1ms, 0 | 20260506 00:04:59.976 | INFO | BAZWRITER : 61.9%, 3.6, 113.5ms, 0 | 20260506 00:04:59.976 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:04:59.976 | INFO | Uploaded 36GiB to GPU in 2.18146s (16.5027GiB/s) | 20260506 00:04:59.976 | INFO | Downloaded 38GiB to GPU in 2.23804s (16.9791GiB/s) | 20260506 00:05:05.116 | INFO | Analyzing chunk frames = [7373312,7373824) | 20260506 00:05:05.116 | INFO | MemInfo: { | 20260506 00:05:05.116 | INFO | "gpu" : " 17 GB", | 20260506 00:05:05.116 | INFO | "huge" : "", | 20260506 00:05:05.116 | INFO | "huge_shared" : " 27 GB", | 20260506 00:05:05.116 | INFO | "malloc" : " 14 GB", | 20260506 00:05:05.116 | INFO | "pinned" : " 29 GB" | 20260506 00:05:05.116 | INFO | } | 20260506 00:05:05.116 | INFO | Perf report for the last 5.14072 seconds: | 20260506 00:05:05.116 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:05:05.116 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:05.116 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:05.116 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:05.116 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260506 00:05:05.116 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:05.116 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:05.116 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:05:05.116 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:05.116 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:05.116 | INFO | Uploaded 36GiB to GPU in 2.11447s (17.0255GiB/s) | 20260506 00:05:05.117 | INFO | Downloaded 34GiB to GPU in 2.04009s (16.6659GiB/s) | 20260506 00:05:10.253 | INFO | Analyzing chunk frames = [7373824,7374336) | 20260506 00:05:10.253 | INFO | MemInfo: { | 20260506 00:05:10.253 | INFO | "gpu" : " 17 GB", | 20260506 00:05:10.253 | INFO | "huge" : "", | 20260506 00:05:10.253 | INFO | "huge_shared" : " 27 GB", | 20260506 00:05:10.253 | INFO | "malloc" : " 14 GB", | 20260506 00:05:10.253 | INFO | "pinned" : " 29 GB" | 20260506 00:05:10.253 | INFO | } | 20260506 00:05:10.253 | INFO | Perf report for the last 5.13676 seconds: | 20260506 00:05:10.253 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:05:10.253 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:10.253 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:10.253 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:10.253 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 00:05:10.253 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:10.253 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:10.253 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 00:05:10.253 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:10.253 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:10.253 | INFO | Uploaded 36GiB to GPU in 2.09139s (17.2134GiB/s) | 20260506 00:05:10.253 | INFO | Downloaded 34GiB to GPU in 2.03041s (16.7454GiB/s) | 20260506 00:05:15.391 | INFO | Analyzing chunk frames = [7374336,7374848) | 20260506 00:05:15.391 | INFO | MemInfo: { | 20260506 00:05:15.391 | INFO | "gpu" : " 17 GB", | 20260506 00:05:15.391 | INFO | "huge" : "", | 20260506 00:05:15.391 | INFO | "huge_shared" : " 27 GB", | 20260506 00:05:15.392 | INFO | "malloc" : " 14 GB", | 20260506 00:05:15.392 | INFO | "pinned" : " 29 GB" | 20260506 00:05:15.392 | INFO | } | 20260506 00:05:15.392 | INFO | Perf report for the last 5.13879 seconds: | 20260506 00:05:15.392 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:05:15.392 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:15.392 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:15.392 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:15.392 | INFO | ANALYSIS : 61.7%, 3.0, 95.8ms, 0 | 20260506 00:05:15.392 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:15.392 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:15.392 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 00:05:15.392 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:15.392 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:15.392 | INFO | Uploaded 36GiB to GPU in 2.13199s (16.8856GiB/s) | 20260506 00:05:15.392 | INFO | Downloaded 34GiB to GPU in 2.00894s (16.9243GiB/s) | 20260506 00:05:20.534 | INFO | Analyzing chunk frames = [7374848,7375360) | 20260506 00:05:20.534 | INFO | MemInfo: { | 20260506 00:05:20.534 | INFO | "gpu" : " 17 GB", | 20260506 00:05:20.534 | INFO | "huge" : "", | 20260506 00:05:20.534 | INFO | "huge_shared" : " 27 GB", | 20260506 00:05:20.534 | INFO | "malloc" : " 14 GB", | 20260506 00:05:20.534 | INFO | "pinned" : " 29 GB" | 20260506 00:05:20.534 | INFO | } | 20260506 00:05:20.535 | INFO | Perf report for the last 5.14277 seconds: | 20260506 00:05:20.535 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:05:20.535 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:20.535 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:20.535 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:20.535 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 00:05:20.535 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:20.535 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:20.535 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 00:05:20.535 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:20.535 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:20.535 | INFO | Uploaded 36GiB to GPU in 2.08849s (17.2373GiB/s) | 20260506 00:05:20.535 | INFO | Downloaded 34GiB to GPU in 2.04723s (16.6078GiB/s) | 20260506 00:05:25.669 | INFO | Analyzing chunk frames = [7375360,7375872) | 20260506 00:05:25.669 | INFO | MemInfo: { | 20260506 00:05:25.669 | INFO | "gpu" : " 17 GB", | 20260506 00:05:25.669 | INFO | "huge" : "", | 20260506 00:05:25.669 | INFO | "huge_shared" : " 27 GB", | 20260506 00:05:25.669 | INFO | "malloc" : " 14 GB", | 20260506 00:05:25.669 | INFO | "pinned" : " 29 GB" | 20260506 00:05:25.669 | INFO | } | 20260506 00:05:25.669 | INFO | Perf report for the last 5.13475 seconds: | 20260506 00:05:25.669 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:05:25.669 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:25.669 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:25.670 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:25.670 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 00:05:25.670 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:25.670 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:25.670 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:05:25.670 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:25.670 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:25.670 | INFO | Uploaded 36GiB to GPU in 2.06944s (17.396GiB/s) | 20260506 00:05:25.670 | INFO | Downloaded 34GiB to GPU in 2.02996s (16.7491GiB/s) | 20260506 00:05:30.803 | INFO | Analyzing chunk frames = [7375872,7376384) | 20260506 00:05:30.803 | INFO | MemInfo: { | 20260506 00:05:30.803 | INFO | "gpu" : " 17 GB", | 20260506 00:05:30.803 | INFO | "huge" : "", | 20260506 00:05:30.803 | INFO | "huge_shared" : " 27 GB", | 20260506 00:05:30.803 | INFO | "malloc" : " 14 GB", | 20260506 00:05:30.803 | INFO | "pinned" : " 29 GB" | 20260506 00:05:30.803 | INFO | } | 20260506 00:05:30.803 | INFO | Perf report for the last 5.13365 seconds: | 20260506 00:05:30.803 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:05:30.803 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:30.803 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:30.803 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:30.803 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260506 00:05:30.803 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:30.803 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:30.803 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 00:05:30.803 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:30.803 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:30.803 | INFO | Uploaded 36GiB to GPU in 2.09258s (17.2036GiB/s) | 20260506 00:05:30.803 | INFO | Downloaded 34GiB to GPU in 2.07118s (16.4158GiB/s) | 20260506 00:05:35.944 | INFO | Analyzing chunk frames = [7376384,7376896) | 20260506 00:05:35.944 | INFO | MemInfo: { | 20260506 00:05:35.944 | INFO | "gpu" : " 17 GB", | 20260506 00:05:35.944 | INFO | "huge" : "", | 20260506 00:05:35.944 | INFO | "huge_shared" : " 27 GB", | 20260506 00:05:35.944 | INFO | "malloc" : " 14 GB", | 20260506 00:05:35.944 | INFO | "pinned" : " 29 GB" | 20260506 00:05:35.945 | INFO | } | 20260506 00:05:35.945 | INFO | Perf report for the last 5.14176 seconds: | 20260506 00:05:35.945 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:05:35.945 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:35.945 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:35.945 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:35.945 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260506 00:05:35.945 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:35.945 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:35.945 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 00:05:35.945 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:35.945 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:35.945 | INFO | Uploaded 36GiB to GPU in 2.10824s (17.0758GiB/s) | 20260506 00:05:35.945 | INFO | Downloaded 34GiB to GPU in 2.00008s (16.9993GiB/s) | 20260506 00:05:36.068 | INFO | Starting a round of Baz IO | 20260506 00:05:39.216 | INFO | Populating RTMetrics report | 20260506 00:05:39.216 | INFO | RTMetrics report populated | 20260506 00:05:39.216 | INFO | Opening tmp RTMetrics json file | 20260506 00:05:39.219 | INFO | Serializing RTMetrics to json | 20260506 00:05:39.222 | INFO | Serialization done, swapping tmp file now... | 20260506 00:05:39.222 | INFO | RTMetrics done | 20260506 00:05:39.293 | INFO | Finished a round of Baz IO: Wrote 1.869 GiB in 3.225 seconds | 20260506 00:05:39.293 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.39% / 0.00% / 78.61% | 20260506 00:05:39.293 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0966748% / 0% / 8.34465e-05% | 20260506 00:05:39.293 | INFO | Baz IO overall: | 20260506 00:05:39.293 | INFO | BytesWritten : 4200950852802 | 20260506 00:05:39.293 | INFO | NumEvents : 1977724923731 | 20260506 00:05:41.090 | INFO | Analyzing chunk frames = [7376896,7377408) | 20260506 00:05:41.090 | INFO | MemInfo: { | 20260506 00:05:41.090 | INFO | "gpu" : " 17 GB", | 20260506 00:05:41.090 | INFO | "huge" : "", | 20260506 00:05:41.090 | INFO | "huge_shared" : " 27 GB", | 20260506 00:05:41.090 | INFO | "malloc" : " 14 GB", | 20260506 00:05:41.090 | INFO | "pinned" : " 29 GB" | 20260506 00:05:41.090 | INFO | } | 20260506 00:05:41.091 | INFO | Perf report for the last 5.14577 seconds: | 20260506 00:05:41.091 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:05:41.091 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:41.091 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:41.091 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:41.091 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260506 00:05:41.091 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:41.091 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260506 00:05:41.091 | INFO | PRE_HQ : 59.6%, 1.7, 54.0ms, 0 | 20260506 00:05:41.091 | INFO | BAZWRITER : 61.9%, 3.6, 113.5ms, 0 | 20260506 00:05:41.091 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 00:05:41.091 | INFO | Uploaded 36GiB to GPU in 2.15221s (16.727GiB/s) | 20260506 00:05:41.091 | INFO | Downloaded 38GiB to GPU in 2.23442s (17.0066GiB/s) | 20260506 00:05:46.232 | INFO | Analyzing chunk frames = [7377408,7377920) | 20260506 00:05:46.232 | INFO | MemInfo: { | 20260506 00:05:46.232 | INFO | "gpu" : " 17 GB", | 20260506 00:05:46.232 | INFO | "huge" : "", | 20260506 00:05:46.232 | INFO | "huge_shared" : " 27 GB", | 20260506 00:05:46.233 | INFO | "malloc" : " 14 GB", | 20260506 00:05:46.233 | INFO | "pinned" : " 29 GB" | 20260506 00:05:46.233 | INFO | } | 20260506 00:05:46.233 | INFO | Perf report for the last 5.14222 seconds: | 20260506 00:05:46.233 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:05:46.233 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:46.233 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:46.233 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:46.233 | INFO | ANALYSIS : 61.1%, 3.0, 95.5ms, 0 | 20260506 00:05:46.233 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:46.233 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:46.233 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:05:46.233 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:46.233 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:46.233 | INFO | Uploaded 36GiB to GPU in 1.99093s (18.082GiB/s) | 20260506 00:05:46.233 | INFO | Downloaded 34GiB to GPU in 1.99008s (17.0847GiB/s) | 20260506 00:05:51.364 | INFO | Analyzing chunk frames = [7377920,7378432) | 20260506 00:05:51.364 | INFO | MemInfo: { | 20260506 00:05:51.364 | INFO | "gpu" : " 17 GB", | 20260506 00:05:51.364 | INFO | "huge" : "", | 20260506 00:05:51.364 | INFO | "huge_shared" : " 27 GB", | 20260506 00:05:51.364 | INFO | "malloc" : " 14 GB", | 20260506 00:05:51.364 | INFO | "pinned" : " 29 GB" | 20260506 00:05:51.364 | INFO | } | 20260506 00:05:51.364 | INFO | Perf report for the last 5.13087 seconds: | 20260506 00:05:51.364 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:05:51.364 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:51.364 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:51.364 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:51.364 | INFO | ANALYSIS : 61.4%, 3.0, 95.4ms, 0 | 20260506 00:05:51.364 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:51.364 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:51.364 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 00:05:51.364 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:51.364 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:51.364 | INFO | Uploaded 36GiB to GPU in 2.08516s (17.2648GiB/s) | 20260506 00:05:51.364 | INFO | Downloaded 34GiB to GPU in 2.0193s (16.8375GiB/s) | 20260506 00:05:56.500 | INFO | Analyzing chunk frames = [7378432,7378944) | 20260506 00:05:56.500 | INFO | MemInfo: { | 20260506 00:05:56.500 | INFO | "gpu" : " 17 GB", | 20260506 00:05:56.500 | INFO | "huge" : "", | 20260506 00:05:56.500 | INFO | "huge_shared" : " 27 GB", | 20260506 00:05:56.500 | INFO | "malloc" : " 14 GB", | 20260506 00:05:56.500 | INFO | "pinned" : " 29 GB" | 20260506 00:05:56.500 | INFO | } | 20260506 00:05:56.500 | INFO | Perf report for the last 5.13684 seconds: | 20260506 00:05:56.500 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:05:56.500 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:56.500 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:56.500 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:56.500 | INFO | ANALYSIS : 61.7%, 3.0, 95.8ms, 0 | 20260506 00:05:56.500 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:56.501 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:56.501 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260506 00:05:56.501 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:56.501 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:05:56.501 | INFO | Uploaded 36GiB to GPU in 2.13007s (16.9009GiB/s) | 20260506 00:05:56.501 | INFO | Downloaded 34GiB to GPU in 2.0243s (16.7959GiB/s) | 20260506 00:06:01.645 | INFO | Analyzing chunk frames = [7378944,7379456) | 20260506 00:06:01.645 | INFO | MemInfo: { | 20260506 00:06:01.645 | INFO | "gpu" : " 17 GB", | 20260506 00:06:01.645 | INFO | "huge" : "", | 20260506 00:06:01.645 | INFO | "huge_shared" : " 27 GB", | 20260506 00:06:01.645 | INFO | "malloc" : " 14 GB", | 20260506 00:06:01.645 | INFO | "pinned" : " 29 GB" | 20260506 00:06:01.645 | INFO | } | 20260506 00:06:01.645 | INFO | Perf report for the last 5.14429 seconds: | 20260506 00:06:01.645 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:06:01.645 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:01.645 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:01.645 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:01.645 | INFO | ANALYSIS : 61.2%, 3.0, 95.7ms, 0 | 20260506 00:06:01.645 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:01.645 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:01.645 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 00:06:01.645 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:01.645 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:01.645 | INFO | Uploaded 36GiB to GPU in 2.11531s (17.0188GiB/s) | 20260506 00:06:01.645 | INFO | Downloaded 34GiB to GPU in 2.05562s (16.54GiB/s) | 20260506 00:06:06.782 | INFO | Analyzing chunk frames = [7379456,7379968) | 20260506 00:06:06.782 | INFO | MemInfo: { | 20260506 00:06:06.782 | INFO | "gpu" : " 17 GB", | 20260506 00:06:06.782 | INFO | "huge" : "", | 20260506 00:06:06.782 | INFO | "huge_shared" : " 27 GB", | 20260506 00:06:06.782 | INFO | "malloc" : " 14 GB", | 20260506 00:06:06.782 | INFO | "pinned" : " 29 GB" | 20260506 00:06:06.782 | INFO | } | 20260506 00:06:06.782 | INFO | Perf report for the last 5.13716 seconds: | 20260506 00:06:06.782 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:06:06.782 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:06.782 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:06.782 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:06.782 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260506 00:06:06.782 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:06.782 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:06.782 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 00:06:06.782 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:06.782 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:06.782 | INFO | Uploaded 36GiB to GPU in 2.07608s (17.3403GiB/s) | 20260506 00:06:06.782 | INFO | Downloaded 34GiB to GPU in 2.04168s (16.6529GiB/s) | 20260506 00:06:11.923 | INFO | Analyzing chunk frames = [7379968,7380480) | 20260506 00:06:11.923 | INFO | MemInfo: { | 20260506 00:06:11.923 | INFO | "gpu" : " 17 GB", | 20260506 00:06:11.923 | INFO | "huge" : "", | 20260506 00:06:11.923 | INFO | "huge_shared" : " 27 GB", | 20260506 00:06:11.923 | INFO | "malloc" : " 14 GB", | 20260506 00:06:11.923 | INFO | "pinned" : " 29 GB" | 20260506 00:06:11.923 | INFO | } | 20260506 00:06:11.923 | INFO | Perf report for the last 5.14139 seconds: | 20260506 00:06:11.923 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:06:11.923 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:11.923 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:11.923 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:11.923 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260506 00:06:11.923 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:11.923 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:11.923 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 00:06:11.924 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:11.924 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:11.924 | INFO | Uploaded 36GiB to GPU in 2.0276s (17.755GiB/s) | 20260506 00:06:11.924 | INFO | Downloaded 34GiB to GPU in 2.03775s (16.6851GiB/s) | 20260506 00:06:17.065 | INFO | Analyzing chunk frames = [7380480,7380992) | 20260506 00:06:17.065 | INFO | MemInfo: { | 20260506 00:06:17.065 | INFO | "gpu" : " 17 GB", | 20260506 00:06:17.065 | INFO | "huge" : "", | 20260506 00:06:17.065 | INFO | "huge_shared" : " 27 GB", | 20260506 00:06:17.065 | INFO | "malloc" : " 14 GB", | 20260506 00:06:17.065 | INFO | "pinned" : " 29 GB" | 20260506 00:06:17.065 | INFO | } | 20260506 00:06:17.065 | INFO | Perf report for the last 5.14182 seconds: | 20260506 00:06:17.065 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:06:17.065 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:17.065 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:17.065 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:17.065 | INFO | ANALYSIS : 61.9%, 3.0, 96.8ms, 0 | 20260506 00:06:17.065 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:17.065 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:17.065 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 00:06:17.065 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:17.065 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:17.066 | INFO | Uploaded 36GiB to GPU in 2.11707s (17.0047GiB/s) | 20260506 00:06:17.066 | INFO | Downloaded 34GiB to GPU in 2.006s (16.9491GiB/s) | 20260506 00:06:17.066 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 00:06:17.066 | INFO | Total mem usage: 15.3 GiB | 20260506 00:06:17.066 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 00:06:17.066 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.12%) | 20260506 00:06:17.066 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 00:06:17.066 | INFO | High Water Breakdown: | 20260506 00:06:17.066 | INFO | 60.9% : 8.55 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 00:06:17.066 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 00:06:17.066 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 00:06:17.066 | INFO | ------------------------------------------ | 20260506 00:06:17.066 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:06:17.066 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 00:06:17.066 | INFO | Total mem usage: 26.5 GiB | 20260506 00:06:17.066 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 00:06:17.066 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 00:06:17.066 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 00:06:17.066 | INFO | High Water Breakdown: | 20260506 00:06:17.066 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:06:17.066 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:06:17.066 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:06:17.066 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:06:17.066 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:06:17.066 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:06:17.066 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 00:06:17.066 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 00:06:17.066 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:06:17.066 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 00:06:17.066 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 00:06:17.066 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 00:06:17.066 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 00:06:17.066 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 00:06:17.066 | INFO | ------------------------------------------ | 20260506 00:06:17.066 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:06:17.066 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 00:06:17.066 | INFO | Total mem usage: 25.6 GiB | 20260506 00:06:17.066 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 00:06:17.066 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 00:06:17.066 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 00:06:17.066 | INFO | High Water Breakdown: | 20260506 00:06:17.066 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 00:06:17.066 | INFO | ------------------------------------------ | 20260506 00:06:17.066 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:06:17.066 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 00:06:17.066 | INFO | Total mem usage: 15.7 GiB | 20260506 00:06:17.066 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 00:06:17.066 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 00:06:17.067 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 00:06:17.067 | INFO | High Water Breakdown: | 20260506 00:06:17.067 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 00:06:17.067 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 00:06:17.067 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 00:06:17.067 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 00:06:17.067 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 00:06:17.067 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 00:06:17.067 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 00:06:17.067 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 00:06:17.067 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 00:06:17.067 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:06:17.067 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 00:06:17.067 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:06:17.067 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 00:06:17.067 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 00:06:17.067 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 00:06:17.067 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 00:06:17.067 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:06:17.067 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 00:06:17.067 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:06:17.067 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:06:17.067 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:06:17.067 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 00:06:17.067 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 00:06:17.067 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:06:17.067 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 00:06:17.067 | INFO | ------------------------------------------ | 20260506 00:06:17.067 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:06:17.067 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 00:06:17.067 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:06:17.067 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:06:17.067 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.41e+01, 1.34e+01 / 9.67e-01 | 20260506 00:06:17.067 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.03e-01, 1.01e-02 / 3.42e-03 | 20260506 00:06:17.067 | INFO | PerfReport: 11.83%, Baseline -- 3.71e+00 / 4.51e+00, 5.75e-01 / 1.82e-01 | 20260506 00:06:17.067 | INFO | PerfReport: 3.29%, Histogram -- 1.03e+00 / 1.53e+00, 1.94e-01 / 9.16e-02 | 20260506 00:06:17.067 | INFO | PerfReport: 16.00%, DME -- 4.02e+01 / 4.42e+01, 3.87e+00 / 1.23e+00 | 20260506 00:06:17.067 | INFO | PerfReport: 57.36%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.53e+00 / 1.00e+00 | 20260506 00:06:17.067 | INFO | PerfReport: 7.07%, PulseAccumulating -- 2.22e+00 / 2.74e+00, 4.21e-01 / 2.19e-01 | 20260506 00:06:17.067 | INFO | PerfReport: 3.08%, Metrics -- 9.66e-01 / 1.24e+00, 2.56e-01 / 1.11e-01 | 20260506 00:06:17.067 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.35e-01 / 3.84e-01, 5.63e-02 / 2.71e-02 | 20260506 00:06:17.067 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 00:06:17.067 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:06:17.067 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:06:17.067 | INFO | PerfReport: Full Execution -- 9.82e+01 / -nan, 2.46e+01 / 0.00e+00 | 20260506 00:06:17.067 | INFO | PerfReport: 22.25%, Upload -- 2.19e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260506 00:06:17.067 | INFO | PerfReport: 21.79%, Download -- 2.14e+01 / 0.00e+00, 7.52e+00 / 0.00e+00 | 20260506 00:06:17.067 | INFO | PerfReport: 31.94%, Compute -- 3.14e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 00:06:17.067 | INFO | PerfReport: 2.11%, LockUpload -- 2.07e+00 / 0.00e+00, 7.85e+00 / 0.00e+00 | 20260506 00:06:17.067 | INFO | PerfReport: 1.29%, LockDownload -- 1.26e+00 / 0.00e+00, 2.86e+00 / 0.00e+00 | 20260506 00:06:17.067 | INFO | PerfReport: 20.61%, LockCompute -- 2.02e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 00:06:17.067 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.62e-02 / 0.00e+00 | 20260506 00:06:17.182 | INFO | Starting a round of Baz IO | 20260506 00:06:20.331 | INFO | Populating RTMetrics report | 20260506 00:06:20.331 | INFO | RTMetrics report populated | 20260506 00:06:20.331 | INFO | Opening tmp RTMetrics json file | 20260506 00:06:20.332 | INFO | Serializing RTMetrics to json | 20260506 00:06:20.333 | INFO | Serialization done, swapping tmp file now... | 20260506 00:06:20.337 | INFO | RTMetrics done | 20260506 00:06:20.413 | INFO | Finished a round of Baz IO: Wrote 1.867 GiB in 3.231 seconds | 20260506 00:06:20.413 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.39% / 0.00% / 78.61% | 20260506 00:06:20.413 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0964204% / 0% / 6.75519e-05% | 20260506 00:06:20.413 | INFO | Baz IO overall: | 20260506 00:06:20.413 | INFO | BytesWritten : 4202955975936 | 20260506 00:06:20.413 | INFO | NumEvents : 1978642296555 | 20260506 00:06:22.202 | INFO | Analyzing chunk frames = [7380992,7381504) | 20260506 00:06:22.202 | INFO | MemInfo: { | 20260506 00:06:22.202 | INFO | "gpu" : " 17 GB", | 20260506 00:06:22.202 | INFO | "huge" : "", | 20260506 00:06:22.202 | INFO | "huge_shared" : " 27 GB", | 20260506 00:06:22.202 | INFO | "malloc" : " 14 GB", | 20260506 00:06:22.202 | INFO | "pinned" : " 29 GB" | 20260506 00:06:22.202 | INFO | } | 20260506 00:06:22.202 | INFO | Perf report for the last 5.13715 seconds: | 20260506 00:06:22.202 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:06:22.202 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:22.202 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:22.203 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:22.203 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260506 00:06:22.203 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:22.203 | INFO | RT_METRICS : 43.4%, 1.0, 22.7ms, 0 | 20260506 00:06:22.203 | INFO | PRE_HQ : 59.9%, 1.7, 54.2ms, 0 | 20260506 00:06:22.203 | INFO | BAZWRITER : 62.0%, 3.6, 113.3ms, 0 | 20260506 00:06:22.203 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 00:06:22.203 | INFO | Uploaded 36GiB to GPU in 2.16891s (16.5982GiB/s) | 20260506 00:06:22.203 | INFO | Downloaded 38GiB to GPU in 2.24196s (16.9494GiB/s) | 20260506 00:06:27.342 | INFO | Analyzing chunk frames = [7381504,7382016) | 20260506 00:06:27.342 | INFO | MemInfo: { | 20260506 00:06:27.342 | INFO | "gpu" : " 17 GB", | 20260506 00:06:27.342 | INFO | "huge" : "", | 20260506 00:06:27.342 | INFO | "huge_shared" : " 27 GB", | 20260506 00:06:27.342 | INFO | "malloc" : " 14 GB", | 20260506 00:06:27.342 | INFO | "pinned" : " 29 GB" | 20260506 00:06:27.342 | INFO | } | 20260506 00:06:27.342 | INFO | Perf report for the last 5.13944 seconds: | 20260506 00:06:27.342 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:06:27.342 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:27.342 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:27.342 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:27.342 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 00:06:27.342 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:27.342 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:27.342 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:06:27.342 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:27.342 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:27.342 | INFO | Uploaded 36GiB to GPU in 2.11506s (17.0208GiB/s) | 20260506 00:06:27.342 | INFO | Downloaded 34GiB to GPU in 2.04338s (16.6391GiB/s) | 20260506 00:06:32.482 | INFO | Analyzing chunk frames = [7382016,7382528) | 20260506 00:06:32.482 | INFO | MemInfo: { | 20260506 00:06:32.482 | INFO | "gpu" : " 17 GB", | 20260506 00:06:32.482 | INFO | "huge" : "", | 20260506 00:06:32.482 | INFO | "huge_shared" : " 27 GB", | 20260506 00:06:32.482 | INFO | "malloc" : " 14 GB", | 20260506 00:06:32.482 | INFO | "pinned" : " 29 GB" | 20260506 00:06:32.482 | INFO | } | 20260506 00:06:32.482 | INFO | Perf report for the last 5.14035 seconds: | 20260506 00:06:32.482 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:06:32.482 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:32.482 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:32.482 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:32.482 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 00:06:32.482 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:32.482 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:32.482 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 00:06:32.482 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:32.482 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:32.483 | INFO | Uploaded 36GiB to GPU in 2.06759s (17.4116GiB/s) | 20260506 00:06:32.483 | INFO | Downloaded 34GiB to GPU in 2.01649s (16.861GiB/s) | 20260506 00:06:37.624 | INFO | Analyzing chunk frames = [7382528,7383040) | 20260506 00:06:37.624 | INFO | MemInfo: { | 20260506 00:06:37.624 | INFO | "gpu" : " 17 GB", | 20260506 00:06:37.624 | INFO | "huge" : "", | 20260506 00:06:37.624 | INFO | "huge_shared" : " 27 GB", | 20260506 00:06:37.624 | INFO | "malloc" : " 14 GB", | 20260506 00:06:37.624 | INFO | "pinned" : " 29 GB" | 20260506 00:06:37.624 | INFO | } | 20260506 00:06:37.624 | INFO | Perf report for the last 5.14154 seconds: | 20260506 00:06:37.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:06:37.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:37.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:37.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:37.624 | INFO | ANALYSIS : 61.7%, 3.0, 95.8ms, 0 | 20260506 00:06:37.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:37.624 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:37.624 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 00:06:37.624 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:37.624 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:37.624 | INFO | Uploaded 36GiB to GPU in 2.13876s (16.8322GiB/s) | 20260506 00:06:37.624 | INFO | Downloaded 34GiB to GPU in 2.02565s (16.7848GiB/s) | 20260506 00:06:42.754 | INFO | Analyzing chunk frames = [7383040,7383552) | 20260506 00:06:42.754 | INFO | MemInfo: { | 20260506 00:06:42.754 | INFO | "gpu" : " 17 GB", | 20260506 00:06:42.754 | INFO | "huge" : "", | 20260506 00:06:42.754 | INFO | "huge_shared" : " 27 GB", | 20260506 00:06:42.754 | INFO | "malloc" : " 14 GB", | 20260506 00:06:42.754 | INFO | "pinned" : " 29 GB" | 20260506 00:06:42.754 | INFO | } | 20260506 00:06:42.754 | INFO | Perf report for the last 5.13045 seconds: | 20260506 00:06:42.754 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:06:42.754 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:42.754 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:42.754 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:42.754 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 00:06:42.754 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:42.754 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:42.754 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260506 00:06:42.754 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:42.754 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:42.754 | INFO | Uploaded 36GiB to GPU in 2.09527s (17.1816GiB/s) | 20260506 00:06:42.754 | INFO | Downloaded 34GiB to GPU in 2.04874s (16.5956GiB/s) | 20260506 00:06:47.897 | INFO | Analyzing chunk frames = [7383552,7384064) | 20260506 00:06:47.897 | INFO | MemInfo: { | 20260506 00:06:47.897 | INFO | "gpu" : " 17 GB", | 20260506 00:06:47.897 | INFO | "huge" : "", | 20260506 00:06:47.897 | INFO | "huge_shared" : " 27 GB", | 20260506 00:06:47.897 | INFO | "malloc" : " 14 GB", | 20260506 00:06:47.897 | INFO | "pinned" : " 29 GB" | 20260506 00:06:47.897 | INFO | } | 20260506 00:06:47.897 | INFO | Perf report for the last 5.14266 seconds: | 20260506 00:06:47.897 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:06:47.897 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:47.897 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:47.897 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:47.897 | INFO | ANALYSIS : 61.3%, 3.0, 95.8ms, 0 | 20260506 00:06:47.897 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:47.897 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:47.897 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 00:06:47.897 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:47.897 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:47.897 | INFO | Uploaded 36GiB to GPU in 2.08302s (17.2826GiB/s) | 20260506 00:06:47.897 | INFO | Downloaded 34GiB to GPU in 2.03652s (16.6952GiB/s) | 20260506 00:06:53.040 | INFO | Analyzing chunk frames = [7384064,7384576) | 20260506 00:06:53.040 | INFO | MemInfo: { | 20260506 00:06:53.040 | INFO | "gpu" : " 17 GB", | 20260506 00:06:53.040 | INFO | "huge" : "", | 20260506 00:06:53.040 | INFO | "huge_shared" : " 27 GB", | 20260506 00:06:53.040 | INFO | "malloc" : " 14 GB", | 20260506 00:06:53.040 | INFO | "pinned" : " 29 GB" | 20260506 00:06:53.040 | INFO | } | 20260506 00:06:53.040 | INFO | Perf report for the last 5.14332 seconds: | 20260506 00:06:53.040 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:06:53.040 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:53.040 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:53.040 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:53.040 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260506 00:06:53.040 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:53.040 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:53.040 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 00:06:53.040 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:53.040 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:53.040 | INFO | Uploaded 36GiB to GPU in 2.02253s (17.7995GiB/s) | 20260506 00:06:53.040 | INFO | Downloaded 34GiB to GPU in 2.03595s (16.6998GiB/s) | 20260506 00:06:58.177 | INFO | Analyzing chunk frames = [7384576,7385088) | 20260506 00:06:58.178 | INFO | MemInfo: { | 20260506 00:06:58.178 | INFO | "gpu" : " 17 GB", | 20260506 00:06:58.178 | INFO | "huge" : "", | 20260506 00:06:58.178 | INFO | "huge_shared" : " 27 GB", | 20260506 00:06:58.178 | INFO | "malloc" : " 14 GB", | 20260506 00:06:58.178 | INFO | "pinned" : " 29 GB" | 20260506 00:06:58.178 | INFO | } | 20260506 00:06:58.178 | INFO | Perf report for the last 5.1375 seconds: | 20260506 00:06:58.178 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:06:58.178 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:58.178 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:58.178 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:58.178 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260506 00:06:58.178 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:58.178 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:58.178 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 00:06:58.178 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:58.178 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:06:58.178 | INFO | Uploaded 36GiB to GPU in 2.13141s (16.8903GiB/s) | 20260506 00:06:58.178 | INFO | Downloaded 34GiB to GPU in 2.01994s (16.8322GiB/s) | 20260506 00:06:58.295 | INFO | Starting a round of Baz IO | 20260506 00:07:01.446 | INFO | Populating RTMetrics report | 20260506 00:07:01.446 | INFO | RTMetrics report populated | 20260506 00:07:01.446 | INFO | Opening tmp RTMetrics json file | 20260506 00:07:01.447 | INFO | Serializing RTMetrics to json | 20260506 00:07:01.448 | INFO | Serialization done, swapping tmp file now... | 20260506 00:07:01.450 | INFO | RTMetrics done | 20260506 00:07:01.526 | INFO | Finished a round of Baz IO: Wrote 1.87 GiB in 3.231 seconds | 20260506 00:07:01.526 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.39% / 0.00% / 78.61% | 20260506 00:07:01.526 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0970523% / 0% / 6.75519e-05% | 20260506 00:07:01.526 | INFO | Baz IO overall: | 20260506 00:07:01.526 | INFO | BytesWritten : 4204964227229 | 20260506 00:07:01.526 | INFO | NumEvents : 1979561635546 | 20260506 00:07:03.313 | INFO | Analyzing chunk frames = [7385088,7385600) | 20260506 00:07:03.313 | INFO | MemInfo: { | 20260506 00:07:03.313 | INFO | "gpu" : " 17 GB", | 20260506 00:07:03.313 | INFO | "huge" : "", | 20260506 00:07:03.313 | INFO | "huge_shared" : " 27 GB", | 20260506 00:07:03.313 | INFO | "malloc" : " 14 GB", | 20260506 00:07:03.313 | INFO | "pinned" : " 29 GB" | 20260506 00:07:03.313 | INFO | } | 20260506 00:07:03.313 | INFO | Perf report for the last 5.13511 seconds: | 20260506 00:07:03.313 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:07:03.313 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:03.313 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:03.313 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:03.313 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260506 00:07:03.313 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:03.313 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260506 00:07:03.313 | INFO | PRE_HQ : 59.9%, 1.7, 54.2ms, 0 | 20260506 00:07:03.313 | INFO | BAZWRITER : 62.0%, 3.5, 113.1ms, 0 | 20260506 00:07:03.313 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 00:07:03.313 | INFO | Uploaded 36GiB to GPU in 2.17228s (16.5724GiB/s) | 20260506 00:07:03.313 | INFO | Downloaded 38GiB to GPU in 2.24189s (16.95GiB/s) | 20260506 00:07:08.454 | INFO | Analyzing chunk frames = [7385600,7386112) | 20260506 00:07:08.454 | INFO | MemInfo: { | 20260506 00:07:08.454 | INFO | "gpu" : " 17 GB", | 20260506 00:07:08.454 | INFO | "huge" : "", | 20260506 00:07:08.454 | INFO | "huge_shared" : " 27 GB", | 20260506 00:07:08.454 | INFO | "malloc" : " 14 GB", | 20260506 00:07:08.454 | INFO | "pinned" : " 29 GB" | 20260506 00:07:08.454 | INFO | } | 20260506 00:07:08.454 | INFO | Perf report for the last 5.14149 seconds: | 20260506 00:07:08.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:07:08.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:08.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:08.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:08.454 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 00:07:08.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:08.454 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:08.454 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 00:07:08.454 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:08.454 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:08.455 | INFO | Uploaded 36GiB to GPU in 2.08834s (17.2386GiB/s) | 20260506 00:07:08.455 | INFO | Downloaded 34GiB to GPU in 2.02729s (16.7712GiB/s) | 20260506 00:07:13.596 | INFO | Analyzing chunk frames = [7386112,7386624) | 20260506 00:07:13.596 | INFO | MemInfo: { | 20260506 00:07:13.596 | INFO | "gpu" : " 17 GB", | 20260506 00:07:13.596 | INFO | "huge" : "", | 20260506 00:07:13.596 | INFO | "huge_shared" : " 27 GB", | 20260506 00:07:13.596 | INFO | "malloc" : " 14 GB", | 20260506 00:07:13.596 | INFO | "pinned" : " 29 GB" | 20260506 00:07:13.597 | INFO | } | 20260506 00:07:13.597 | INFO | Perf report for the last 5.14221 seconds: | 20260506 00:07:13.597 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:07:13.597 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:13.597 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:13.597 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:13.597 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 00:07:13.597 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:13.597 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:13.597 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:07:13.597 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:13.597 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:13.597 | INFO | Uploaded 36GiB to GPU in 2.09187s (17.2095GiB/s) | 20260506 00:07:13.597 | INFO | Downloaded 34GiB to GPU in 2.0303s (16.7463GiB/s) | 20260506 00:07:18.734 | INFO | Analyzing chunk frames = [7386624,7387136) | 20260506 00:07:18.734 | INFO | MemInfo: { | 20260506 00:07:18.734 | INFO | "gpu" : " 17 GB", | 20260506 00:07:18.734 | INFO | "huge" : "", | 20260506 00:07:18.734 | INFO | "huge_shared" : " 27 GB", | 20260506 00:07:18.734 | INFO | "malloc" : " 14 GB", | 20260506 00:07:18.734 | INFO | "pinned" : " 29 GB" | 20260506 00:07:18.734 | INFO | } | 20260506 00:07:18.735 | INFO | Perf report for the last 5.13796 seconds: | 20260506 00:07:18.735 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:07:18.735 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:18.735 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:18.735 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:18.735 | INFO | ANALYSIS : 61.7%, 3.0, 95.8ms, 0 | 20260506 00:07:18.735 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:18.735 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:18.735 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 00:07:18.735 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:18.735 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:18.735 | INFO | Uploaded 36GiB to GPU in 2.13594s (16.8544GiB/s) | 20260506 00:07:18.735 | INFO | Downloaded 34GiB to GPU in 2.0232s (16.8051GiB/s) | 20260506 00:07:23.871 | INFO | Analyzing chunk frames = [7387136,7387648) | 20260506 00:07:23.871 | INFO | MemInfo: { | 20260506 00:07:23.871 | INFO | "gpu" : " 17 GB", | 20260506 00:07:23.871 | INFO | "huge" : "", | 20260506 00:07:23.871 | INFO | "huge_shared" : " 27 GB", | 20260506 00:07:23.871 | INFO | "malloc" : " 14 GB", | 20260506 00:07:23.871 | INFO | "pinned" : " 29 GB" | 20260506 00:07:23.871 | INFO | } | 20260506 00:07:23.871 | INFO | Perf report for the last 5.13671 seconds: | 20260506 00:07:23.871 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:07:23.871 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:23.871 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:23.871 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:23.871 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 00:07:23.871 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:23.871 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:23.871 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 00:07:23.871 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:23.871 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:23.871 | INFO | Uploaded 36GiB to GPU in 2.08663s (17.2527GiB/s) | 20260506 00:07:23.871 | INFO | Downloaded 34GiB to GPU in 2.04195s (16.6508GiB/s) | 20260506 00:07:29.011 | INFO | Analyzing chunk frames = [7387648,7388160) | 20260506 00:07:29.011 | INFO | MemInfo: { | 20260506 00:07:29.011 | INFO | "gpu" : " 17 GB", | 20260506 00:07:29.011 | INFO | "huge" : "", | 20260506 00:07:29.011 | INFO | "huge_shared" : " 27 GB", | 20260506 00:07:29.011 | INFO | "malloc" : " 14 GB", | 20260506 00:07:29.011 | INFO | "pinned" : " 29 GB" | 20260506 00:07:29.011 | INFO | } | 20260506 00:07:29.011 | INFO | Perf report for the last 5.1395 seconds: | 20260506 00:07:29.011 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:07:29.011 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:29.011 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:29.011 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:29.011 | INFO | ANALYSIS : 61.3%, 3.0, 95.8ms, 0 | 20260506 00:07:29.011 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:29.011 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:29.011 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:07:29.011 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:29.011 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:29.011 | INFO | Uploaded 36GiB to GPU in 2.06984s (17.3927GiB/s) | 20260506 00:07:29.011 | INFO | Downloaded 34GiB to GPU in 2.0346s (16.7109GiB/s) | 20260506 00:07:34.154 | INFO | Analyzing chunk frames = [7388160,7388672) | 20260506 00:07:34.154 | INFO | MemInfo: { | 20260506 00:07:34.154 | INFO | "gpu" : " 17 GB", | 20260506 00:07:34.154 | INFO | "huge" : "", | 20260506 00:07:34.154 | INFO | "huge_shared" : " 27 GB", | 20260506 00:07:34.154 | INFO | "malloc" : " 14 GB", | 20260506 00:07:34.154 | INFO | "pinned" : " 29 GB" | 20260506 00:07:34.154 | INFO | } | 20260506 00:07:34.154 | INFO | Perf report for the last 5.14358 seconds: | 20260506 00:07:34.154 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:07:34.154 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:34.154 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:34.154 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:34.154 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260506 00:07:34.154 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:34.154 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:34.154 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 00:07:34.154 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:34.154 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:34.155 | INFO | Uploaded 36GiB to GPU in 2.074s (17.3578GiB/s) | 20260506 00:07:34.155 | INFO | Downloaded 34GiB to GPU in 2.06259s (16.4841GiB/s) | 20260506 00:07:39.292 | INFO | Analyzing chunk frames = [7388672,7389184) | 20260506 00:07:39.292 | INFO | MemInfo: { | 20260506 00:07:39.292 | INFO | "gpu" : " 17 GB", | 20260506 00:07:39.292 | INFO | "huge" : "", | 20260506 00:07:39.292 | INFO | "huge_shared" : " 27 GB", | 20260506 00:07:39.292 | INFO | "malloc" : " 14 GB", | 20260506 00:07:39.292 | INFO | "pinned" : " 29 GB" | 20260506 00:07:39.292 | INFO | } | 20260506 00:07:39.292 | INFO | Perf report for the last 5.13791 seconds: | 20260506 00:07:39.292 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:07:39.292 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:39.292 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:39.292 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:39.292 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260506 00:07:39.292 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:39.292 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:39.292 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:07:39.292 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:39.292 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:39.293 | INFO | Uploaded 36GiB to GPU in 2.14368s (16.7936GiB/s) | 20260506 00:07:39.293 | INFO | Downloaded 34GiB to GPU in 2.03447s (16.712GiB/s) | 20260506 00:07:39.293 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 00:07:39.293 | INFO | Total mem usage: 15.3 GiB | 20260506 00:07:39.293 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 00:07:39.293 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.13%) | 20260506 00:07:39.293 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 00:07:39.293 | INFO | High Water Breakdown: | 20260506 00:07:39.293 | INFO | 60.9% : 8.55 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 00:07:39.293 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 00:07:39.293 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 00:07:39.293 | INFO | ------------------------------------------ | 20260506 00:07:39.293 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:07:39.293 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 00:07:39.293 | INFO | Total mem usage: 26.5 GiB | 20260506 00:07:39.293 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 00:07:39.293 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 00:07:39.293 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 00:07:39.293 | INFO | High Water Breakdown: | 20260506 00:07:39.293 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:07:39.293 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:07:39.293 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:07:39.293 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:07:39.293 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:07:39.293 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:07:39.293 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 00:07:39.293 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 00:07:39.293 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:07:39.293 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 00:07:39.293 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 00:07:39.293 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 00:07:39.293 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 00:07:39.293 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 00:07:39.293 | INFO | ------------------------------------------ | 20260506 00:07:39.293 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:07:39.293 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 00:07:39.293 | INFO | Total mem usage: 25.6 GiB | 20260506 00:07:39.293 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 00:07:39.293 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 00:07:39.293 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 00:07:39.293 | INFO | High Water Breakdown: | 20260506 00:07:39.293 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 00:07:39.293 | INFO | ------------------------------------------ | 20260506 00:07:39.293 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:07:39.293 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 00:07:39.293 | INFO | Total mem usage: 15.7 GiB | 20260506 00:07:39.293 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 00:07:39.293 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 00:07:39.293 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 00:07:39.293 | INFO | High Water Breakdown: | 20260506 00:07:39.293 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 00:07:39.293 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 00:07:39.293 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 00:07:39.293 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 00:07:39.293 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 00:07:39.293 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 00:07:39.293 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 00:07:39.293 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 00:07:39.293 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 00:07:39.293 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:07:39.293 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 00:07:39.293 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:07:39.293 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 00:07:39.293 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 00:07:39.293 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 00:07:39.293 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 00:07:39.293 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:07:39.293 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 00:07:39.293 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:07:39.293 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:07:39.293 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:07:39.293 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 00:07:39.293 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 00:07:39.293 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:07:39.293 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 00:07:39.293 | INFO | ------------------------------------------ | 20260506 00:07:39.293 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:07:39.293 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 00:07:39.293 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:07:39.293 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:07:39.293 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.40e+01, 1.34e+01 / 7.56e-01 | 20260506 00:07:39.293 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.04e-01, 1.05e-02 / 3.47e-03 | 20260506 00:07:39.293 | INFO | PerfReport: 11.84%, Baseline -- 3.71e+00 / 4.51e+00, 5.66e-01 / 1.71e-01 | 20260506 00:07:39.293 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.51e+00, 1.87e-01 / 9.26e-02 | 20260506 00:07:39.293 | INFO | PerfReport: 16.01%, DME -- 4.01e+01 / 4.41e+01, 3.90e+00 / 1.06e+00 | 20260506 00:07:39.293 | INFO | PerfReport: 57.29%, FrameLabeling -- 1.79e+01 / 1.94e+01, 1.47e+00 / 9.15e-01 | 20260506 00:07:39.293 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.23e+00 / 2.77e+00, 4.24e-01 / 2.07e-01 | 20260506 00:07:39.293 | INFO | PerfReport: 3.09%, Metrics -- 9.69e-01 / 1.25e+00, 2.60e-01 / 1.35e-01 | 20260506 00:07:39.293 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.35e-01 / 3.87e-01, 5.62e-02 / 3.13e-02 | 20260506 00:07:39.293 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 00:07:39.293 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:07:39.293 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:07:39.293 | INFO | PerfReport: Full Execution -- 9.83e+01 / -nan, 2.44e+01 / 0.00e+00 | 20260506 00:07:39.293 | INFO | PerfReport: 22.31%, Upload -- 2.19e+01 / 0.00e+00, 7.49e+00 / 0.00e+00 | 20260506 00:07:39.294 | INFO | PerfReport: 21.83%, Download -- 2.15e+01 / 0.00e+00, 7.57e+00 / 0.00e+00 | 20260506 00:07:39.294 | INFO | PerfReport: 31.86%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 00:07:39.294 | INFO | PerfReport: 2.11%, LockUpload -- 2.08e+00 / 0.00e+00, 7.88e+00 / 0.00e+00 | 20260506 00:07:39.294 | INFO | PerfReport: 1.31%, LockDownload -- 1.29e+00 / 0.00e+00, 2.90e+00 / 0.00e+00 | 20260506 00:07:39.294 | INFO | PerfReport: 20.56%, LockCompute -- 2.02e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 00:07:39.294 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.70e-02 / 0.00e+00 | 20260506 00:07:39.414 | INFO | Starting a round of Baz IO | 20260506 00:07:42.557 | INFO | Populating RTMetrics report | 20260506 00:07:42.557 | INFO | RTMetrics report populated | 20260506 00:07:42.557 | INFO | Opening tmp RTMetrics json file | 20260506 00:07:42.558 | INFO | Serializing RTMetrics to json | 20260506 00:07:42.559 | INFO | Serialization done, swapping tmp file now... | 20260506 00:07:42.560 | INFO | RTMetrics done | 20260506 00:07:42.638 | INFO | Finished a round of Baz IO: Wrote 1.87 GiB in 3.223 seconds | 20260506 00:07:42.638 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.39% / 0.00% / 78.61% | 20260506 00:07:42.638 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0971834% / 7.94729e-06% / 8.74201e-05% | 20260506 00:07:42.638 | INFO | Baz IO overall: | 20260506 00:07:42.638 | INFO | BytesWritten : 4206971598251 | 20260506 00:07:42.638 | INFO | NumEvents : 1980480425092 | 20260506 00:07:44.434 | INFO | Analyzing chunk frames = [7389184,7389696) | 20260506 00:07:44.434 | INFO | MemInfo: { | 20260506 00:07:44.434 | INFO | "gpu" : " 17 GB", | 20260506 00:07:44.434 | INFO | "huge" : "", | 20260506 00:07:44.434 | INFO | "huge_shared" : " 27 GB", | 20260506 00:07:44.434 | INFO | "malloc" : " 14 GB", | 20260506 00:07:44.434 | INFO | "pinned" : " 29 GB" | 20260506 00:07:44.434 | INFO | } | 20260506 00:07:44.434 | INFO | Perf report for the last 5.14167 seconds: | 20260506 00:07:44.434 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:07:44.434 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:44.434 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:44.434 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:44.434 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260506 00:07:44.434 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:44.434 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260506 00:07:44.434 | INFO | PRE_HQ : 59.8%, 1.7, 54.1ms, 0 | 20260506 00:07:44.434 | INFO | BAZWRITER : 61.9%, 3.6, 113.6ms, 0 | 20260506 00:07:44.434 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:07:44.434 | INFO | Uploaded 36GiB to GPU in 2.17059s (16.5854GiB/s) | 20260506 00:07:44.434 | INFO | Downloaded 38GiB to GPU in 2.23635s (16.992GiB/s) | 20260506 00:07:49.575 | INFO | Analyzing chunk frames = [7389696,7390208) | 20260506 00:07:49.575 | INFO | MemInfo: { | 20260506 00:07:49.575 | INFO | "gpu" : " 17 GB", | 20260506 00:07:49.575 | INFO | "huge" : "", | 20260506 00:07:49.575 | INFO | "huge_shared" : " 27 GB", | 20260506 00:07:49.575 | INFO | "malloc" : " 14 GB", | 20260506 00:07:49.575 | INFO | "pinned" : " 29 GB" | 20260506 00:07:49.575 | INFO | } | 20260506 00:07:49.575 | INFO | Perf report for the last 5.14075 seconds: | 20260506 00:07:49.575 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:07:49.575 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:49.575 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:49.575 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:49.575 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 00:07:49.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:49.575 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:49.575 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:07:49.575 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:49.575 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:49.575 | INFO | Uploaded 36GiB to GPU in 2.09499s (17.1839GiB/s) | 20260506 00:07:49.575 | INFO | Downloaded 34GiB to GPU in 2.03006s (16.7483GiB/s) | 20260506 00:07:54.709 | INFO | Analyzing chunk frames = [7390208,7390720) | 20260506 00:07:54.709 | INFO | MemInfo: { | 20260506 00:07:54.709 | INFO | "gpu" : " 17 GB", | 20260506 00:07:54.709 | INFO | "huge" : "", | 20260506 00:07:54.709 | INFO | "huge_shared" : " 27 GB", | 20260506 00:07:54.709 | INFO | "malloc" : " 14 GB", | 20260506 00:07:54.709 | INFO | "pinned" : " 29 GB" | 20260506 00:07:54.709 | INFO | } | 20260506 00:07:54.709 | INFO | Perf report for the last 5.13464 seconds: | 20260506 00:07:54.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:07:54.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:54.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:54.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:54.709 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 00:07:54.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:54.709 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:54.709 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:07:54.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:54.709 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:54.710 | INFO | Uploaded 36GiB to GPU in 2.09997s (17.1431GiB/s) | 20260506 00:07:54.710 | INFO | Downloaded 34GiB to GPU in 2.03052s (16.7445GiB/s) | 20260506 00:07:59.856 | INFO | Analyzing chunk frames = [7390720,7391232) | 20260506 00:07:59.856 | INFO | MemInfo: { | 20260506 00:07:59.856 | INFO | "gpu" : " 17 GB", | 20260506 00:07:59.856 | INFO | "huge" : "", | 20260506 00:07:59.856 | INFO | "huge_shared" : " 27 GB", | 20260506 00:07:59.856 | INFO | "malloc" : " 14 GB", | 20260506 00:07:59.856 | INFO | "pinned" : " 29 GB" | 20260506 00:07:59.856 | INFO | } | 20260506 00:07:59.856 | INFO | Perf report for the last 5.14683 seconds: | 20260506 00:07:59.856 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:07:59.856 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:59.856 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:59.856 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:59.856 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 00:07:59.856 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:59.856 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:59.856 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:07:59.856 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:59.856 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:07:59.856 | INFO | Uploaded 36GiB to GPU in 2.1405s (16.8185GiB/s) | 20260506 00:07:59.856 | INFO | Downloaded 34GiB to GPU in 2.01999s (16.8318GiB/s) | 20260506 00:08:04.991 | INFO | Analyzing chunk frames = [7391232,7391744) | 20260506 00:08:04.991 | INFO | MemInfo: { | 20260506 00:08:04.991 | INFO | "gpu" : " 17 GB", | 20260506 00:08:04.991 | INFO | "huge" : "", | 20260506 00:08:04.991 | INFO | "huge_shared" : " 27 GB", | 20260506 00:08:04.991 | INFO | "malloc" : " 14 GB", | 20260506 00:08:04.991 | INFO | "pinned" : " 29 GB" | 20260506 00:08:04.991 | INFO | } | 20260506 00:08:04.991 | INFO | Perf report for the last 5.13508 seconds: | 20260506 00:08:04.991 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:08:04.991 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:04.991 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:04.991 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:04.991 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260506 00:08:04.991 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:04.991 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:04.991 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 00:08:04.991 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:04.991 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:04.992 | INFO | Uploaded 36GiB to GPU in 2.08287s (17.2838GiB/s) | 20260506 00:08:04.992 | INFO | Downloaded 34GiB to GPU in 2.03252s (16.728GiB/s) | 20260506 00:08:10.122 | INFO | Analyzing chunk frames = [7391744,7392256) | 20260506 00:08:10.122 | INFO | MemInfo: { | 20260506 00:08:10.122 | INFO | "gpu" : " 17 GB", | 20260506 00:08:10.122 | INFO | "huge" : "", | 20260506 00:08:10.122 | INFO | "huge_shared" : " 27 GB", | 20260506 00:08:10.122 | INFO | "malloc" : " 14 GB", | 20260506 00:08:10.122 | INFO | "pinned" : " 29 GB" | 20260506 00:08:10.122 | INFO | } | 20260506 00:08:10.122 | INFO | Perf report for the last 5.13066 seconds: | 20260506 00:08:10.122 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:08:10.122 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:10.122 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:10.122 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:10.122 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 00:08:10.122 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:10.122 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:10.122 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 00:08:10.122 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:10.122 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:10.122 | INFO | Uploaded 36GiB to GPU in 2.06959s (17.3947GiB/s) | 20260506 00:08:10.122 | INFO | Downloaded 34GiB to GPU in 2.03965s (16.6695GiB/s) | 20260506 00:08:15.265 | INFO | Analyzing chunk frames = [7392256,7392768) | 20260506 00:08:15.265 | INFO | MemInfo: { | 20260506 00:08:15.265 | INFO | "gpu" : " 17 GB", | 20260506 00:08:15.265 | INFO | "huge" : "", | 20260506 00:08:15.265 | INFO | "huge_shared" : " 27 GB", | 20260506 00:08:15.265 | INFO | "malloc" : " 14 GB", | 20260506 00:08:15.265 | INFO | "pinned" : " 29 GB" | 20260506 00:08:15.265 | INFO | } | 20260506 00:08:15.265 | INFO | Perf report for the last 5.14333 seconds: | 20260506 00:08:15.265 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:08:15.265 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:15.266 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:15.266 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:15.266 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260506 00:08:15.266 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:15.266 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:15.266 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 00:08:15.266 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:15.266 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:15.266 | INFO | Uploaded 36GiB to GPU in 2.14457s (16.7866GiB/s) | 20260506 00:08:15.266 | INFO | Downloaded 34GiB to GPU in 2.10277s (16.1692GiB/s) | 20260506 00:08:20.406 | INFO | Analyzing chunk frames = [7392768,7393280) | 20260506 00:08:20.406 | INFO | MemInfo: { | 20260506 00:08:20.406 | INFO | "gpu" : " 17 GB", | 20260506 00:08:20.406 | INFO | "huge" : "", | 20260506 00:08:20.406 | INFO | "huge_shared" : " 27 GB", | 20260506 00:08:20.406 | INFO | "malloc" : " 14 GB", | 20260506 00:08:20.406 | INFO | "pinned" : " 29 GB" | 20260506 00:08:20.406 | INFO | } | 20260506 00:08:20.406 | INFO | Perf report for the last 5.14061 seconds: | 20260506 00:08:20.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:08:20.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:20.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:20.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:20.406 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260506 00:08:20.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:20.406 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:20.406 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:08:20.406 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:20.406 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:20.406 | INFO | Uploaded 36GiB to GPU in 2.11789s (16.998GiB/s) | 20260506 00:08:20.406 | INFO | Downloaded 34GiB to GPU in 2.00406s (16.9656GiB/s) | 20260506 00:08:20.526 | INFO | Starting a round of Baz IO | 20260506 00:08:23.678 | INFO | Populating RTMetrics report | 20260506 00:08:23.678 | INFO | RTMetrics report populated | 20260506 00:08:23.678 | INFO | Opening tmp RTMetrics json file | 20260506 00:08:23.680 | INFO | Serializing RTMetrics to json | 20260506 00:08:23.682 | INFO | Serialization done, swapping tmp file now... | 20260506 00:08:23.683 | INFO | RTMetrics done | 20260506 00:08:23.759 | INFO | Finished a round of Baz IO: Wrote 1.871 GiB in 3.233 seconds | 20260506 00:08:23.759 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.39% / 0.00% / 78.61% | 20260506 00:08:23.759 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0969172% / 0% / 8.74201e-05% | 20260506 00:08:23.759 | INFO | Baz IO overall: | 20260506 00:08:23.759 | INFO | BytesWritten : 4208980629284 | 20260506 00:08:23.759 | INFO | NumEvents : 1981400270282 | 20260506 00:08:25.551 | INFO | Analyzing chunk frames = [7393280,7393792) | 20260506 00:08:25.551 | INFO | MemInfo: { | 20260506 00:08:25.551 | INFO | "gpu" : " 17 GB", | 20260506 00:08:25.551 | INFO | "huge" : "", | 20260506 00:08:25.551 | INFO | "huge_shared" : " 27 GB", | 20260506 00:08:25.551 | INFO | "malloc" : " 14 GB", | 20260506 00:08:25.551 | INFO | "pinned" : " 29 GB" | 20260506 00:08:25.551 | INFO | } | 20260506 00:08:25.551 | INFO | Perf report for the last 5.14508 seconds: | 20260506 00:08:25.551 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:08:25.551 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:25.551 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:25.551 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:25.551 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260506 00:08:25.551 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:25.551 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260506 00:08:25.551 | INFO | PRE_HQ : 59.6%, 1.7, 53.8ms, 0 | 20260506 00:08:25.551 | INFO | BAZWRITER : 62.0%, 3.6, 113.9ms, 0 | 20260506 00:08:25.551 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:08:25.551 | INFO | Uploaded 36GiB to GPU in 2.15317s (16.7195GiB/s) | 20260506 00:08:25.551 | INFO | Downloaded 38GiB to GPU in 2.24475s (16.9284GiB/s) | 20260506 00:08:30.688 | INFO | Analyzing chunk frames = [7393792,7394304) | 20260506 00:08:30.688 | INFO | MemInfo: { | 20260506 00:08:30.688 | INFO | "gpu" : " 17 GB", | 20260506 00:08:30.688 | INFO | "huge" : "", | 20260506 00:08:30.688 | INFO | "huge_shared" : " 27 GB", | 20260506 00:08:30.688 | INFO | "malloc" : " 14 GB", | 20260506 00:08:30.688 | INFO | "pinned" : " 29 GB" | 20260506 00:08:30.688 | INFO | } | 20260506 00:08:30.688 | INFO | Perf report for the last 5.13695 seconds: | 20260506 00:08:30.688 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:08:30.688 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:30.688 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:30.688 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:30.688 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 00:08:30.688 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:30.688 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:30.688 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 00:08:30.688 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:30.688 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:30.688 | INFO | Uploaded 36GiB to GPU in 2.11609s (17.0125GiB/s) | 20260506 00:08:30.688 | INFO | Downloaded 34GiB to GPU in 2.03733s (16.6885GiB/s) | 20260506 00:08:35.821 | INFO | Analyzing chunk frames = [7394304,7394816) | 20260506 00:08:35.821 | INFO | MemInfo: { | 20260506 00:08:35.821 | INFO | "gpu" : " 17 GB", | 20260506 00:08:35.821 | INFO | "huge" : "", | 20260506 00:08:35.821 | INFO | "huge_shared" : " 27 GB", | 20260506 00:08:35.822 | INFO | "malloc" : " 14 GB", | 20260506 00:08:35.822 | INFO | "pinned" : " 29 GB" | 20260506 00:08:35.822 | INFO | } | 20260506 00:08:35.822 | INFO | Perf report for the last 5.13404 seconds: | 20260506 00:08:35.822 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:08:35.822 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:35.822 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:35.822 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:35.822 | INFO | ANALYSIS : 61.4%, 3.0, 94.1ms, 0 | 20260506 00:08:35.822 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:35.822 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:35.822 | INFO | PRE_HQ : 26.6%, 1.0, 13.7ms, 0 | 20260506 00:08:35.822 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:35.822 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:35.822 | INFO | Uploaded 36GiB to GPU in 2.07261s (17.3694GiB/s) | 20260506 00:08:35.822 | INFO | Downloaded 34GiB to GPU in 2.01558s (16.8686GiB/s) | 20260506 00:08:40.961 | INFO | Analyzing chunk frames = [7394816,7395328) | 20260506 00:08:40.961 | INFO | MemInfo: { | 20260506 00:08:40.961 | INFO | "gpu" : " 17 GB", | 20260506 00:08:40.961 | INFO | "huge" : "", | 20260506 00:08:40.961 | INFO | "huge_shared" : " 27 GB", | 20260506 00:08:40.961 | INFO | "malloc" : " 14 GB", | 20260506 00:08:40.961 | INFO | "pinned" : " 29 GB" | 20260506 00:08:40.961 | INFO | } | 20260506 00:08:40.961 | INFO | Perf report for the last 5.13916 seconds: | 20260506 00:08:40.961 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:08:40.961 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:40.961 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:40.961 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:40.961 | INFO | ANALYSIS : 61.7%, 3.0, 95.8ms, 0 | 20260506 00:08:40.961 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:40.961 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:40.961 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:08:40.961 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:40.961 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:40.961 | INFO | Uploaded 36GiB to GPU in 2.13826s (16.8361GiB/s) | 20260506 00:08:40.962 | INFO | Downloaded 34GiB to GPU in 2.0177s (16.8508GiB/s) | 20260506 00:08:46.102 | INFO | Analyzing chunk frames = [7395328,7395840) | 20260506 00:08:46.102 | INFO | MemInfo: { | 20260506 00:08:46.102 | INFO | "gpu" : " 17 GB", | 20260506 00:08:46.102 | INFO | "huge" : "", | 20260506 00:08:46.102 | INFO | "huge_shared" : " 27 GB", | 20260506 00:08:46.102 | INFO | "malloc" : " 14 GB", | 20260506 00:08:46.102 | INFO | "pinned" : " 29 GB" | 20260506 00:08:46.102 | INFO | } | 20260506 00:08:46.102 | INFO | Perf report for the last 5.14049 seconds: | 20260506 00:08:46.102 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:08:46.102 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:46.102 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:46.102 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:46.102 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 00:08:46.102 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:46.102 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:46.102 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:08:46.102 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:46.102 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:46.102 | INFO | Uploaded 36GiB to GPU in 2.10381s (17.1118GiB/s) | 20260506 00:08:46.102 | INFO | Downloaded 34GiB to GPU in 2.04435s (16.6312GiB/s) | 20260506 00:08:51.247 | INFO | Analyzing chunk frames = [7395840,7396352) | 20260506 00:08:51.247 | INFO | MemInfo: { | 20260506 00:08:51.247 | INFO | "gpu" : " 17 GB", | 20260506 00:08:51.247 | INFO | "huge" : "", | 20260506 00:08:51.247 | INFO | "huge_shared" : " 27 GB", | 20260506 00:08:51.247 | INFO | "malloc" : " 14 GB", | 20260506 00:08:51.247 | INFO | "pinned" : " 29 GB" | 20260506 00:08:51.247 | INFO | } | 20260506 00:08:51.247 | INFO | Perf report for the last 5.14535 seconds: | 20260506 00:08:51.247 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:08:51.247 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:51.247 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:51.247 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:51.247 | INFO | ANALYSIS : 61.2%, 3.0, 95.6ms, 0 | 20260506 00:08:51.247 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:51.247 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:51.247 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:08:51.247 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:51.247 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:51.247 | INFO | Uploaded 36GiB to GPU in 2.06569s (17.4276GiB/s) | 20260506 00:08:51.247 | INFO | Downloaded 34GiB to GPU in 2.03568s (16.7021GiB/s) | 20260506 00:08:56.383 | INFO | Analyzing chunk frames = [7396352,7396864) | 20260506 00:08:56.383 | INFO | MemInfo: { | 20260506 00:08:56.383 | INFO | "gpu" : " 17 GB", | 20260506 00:08:56.383 | INFO | "huge" : "", | 20260506 00:08:56.383 | INFO | "huge_shared" : " 27 GB", | 20260506 00:08:56.383 | INFO | "malloc" : " 14 GB", | 20260506 00:08:56.383 | INFO | "pinned" : " 29 GB" | 20260506 00:08:56.383 | INFO | } | 20260506 00:08:56.383 | INFO | Perf report for the last 5.1362 seconds: | 20260506 00:08:56.383 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:08:56.383 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:56.383 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:56.383 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:56.383 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260506 00:08:56.383 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:56.383 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:56.383 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 00:08:56.383 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:56.383 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:08:56.383 | INFO | Uploaded 36GiB to GPU in 2.10403s (17.11GiB/s) | 20260506 00:08:56.383 | INFO | Downloaded 34GiB to GPU in 2.07359s (16.3967GiB/s) | 20260506 00:09:01.521 | INFO | Analyzing chunk frames = [7396864,7397376) | 20260506 00:09:01.521 | INFO | MemInfo: { | 20260506 00:09:01.521 | INFO | "gpu" : " 17 GB", | 20260506 00:09:01.521 | INFO | "huge" : "", | 20260506 00:09:01.521 | INFO | "huge_shared" : " 27 GB", | 20260506 00:09:01.521 | INFO | "malloc" : " 14 GB", | 20260506 00:09:01.521 | INFO | "pinned" : " 29 GB" | 20260506 00:09:01.521 | INFO | } | 20260506 00:09:01.521 | INFO | Perf report for the last 5.13767 seconds: | 20260506 00:09:01.521 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:09:01.521 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:01.521 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:01.521 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:01.521 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260506 00:09:01.521 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:01.521 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:01.521 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 00:09:01.521 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:01.521 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:01.521 | INFO | Uploaded 36GiB to GPU in 2.11562s (17.0163GiB/s) | 20260506 00:09:01.521 | INFO | Downloaded 34GiB to GPU in 2.01111s (16.9061GiB/s) | 20260506 00:09:01.521 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 00:09:01.521 | INFO | Total mem usage: 15.3 GiB | 20260506 00:09:01.521 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 00:09:01.521 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.11%) | 20260506 00:09:01.521 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 00:09:01.521 | INFO | High Water Breakdown: | 20260506 00:09:01.521 | INFO | 60.9% : 8.55 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 00:09:01.521 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 00:09:01.521 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 00:09:01.521 | INFO | ------------------------------------------ | 20260506 00:09:01.521 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:09:01.521 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 00:09:01.521 | INFO | Total mem usage: 26.5 GiB | 20260506 00:09:01.521 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 00:09:01.521 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 00:09:01.521 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260506 00:09:01.521 | INFO | High Water Breakdown: | 20260506 00:09:01.521 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:09:01.521 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:09:01.521 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:09:01.521 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:09:01.521 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:09:01.521 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:09:01.522 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 00:09:01.522 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 00:09:01.522 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:09:01.522 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 00:09:01.522 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 00:09:01.522 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 00:09:01.522 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 00:09:01.522 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 00:09:01.522 | INFO | ------------------------------------------ | 20260506 00:09:01.522 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:09:01.522 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 00:09:01.522 | INFO | Total mem usage: 25.6 GiB | 20260506 00:09:01.522 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 00:09:01.522 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 00:09:01.522 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 00:09:01.522 | INFO | High Water Breakdown: | 20260506 00:09:01.522 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 00:09:01.522 | INFO | ------------------------------------------ | 20260506 00:09:01.522 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:09:01.522 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 00:09:01.522 | INFO | Total mem usage: 15.7 GiB | 20260506 00:09:01.522 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 00:09:01.522 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 00:09:01.522 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 00:09:01.522 | INFO | High Water Breakdown: | 20260506 00:09:01.522 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 00:09:01.522 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 00:09:01.522 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 00:09:01.522 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 00:09:01.522 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 00:09:01.522 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 00:09:01.522 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 00:09:01.522 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 00:09:01.522 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 00:09:01.522 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:09:01.522 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 00:09:01.522 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:09:01.522 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 00:09:01.522 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 00:09:01.522 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 00:09:01.522 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 00:09:01.522 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:09:01.522 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 00:09:01.522 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:09:01.522 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:09:01.522 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:09:01.522 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 00:09:01.522 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 00:09:01.522 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:09:01.522 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 00:09:01.522 | INFO | ------------------------------------------ | 20260506 00:09:01.522 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:09:01.522 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 00:09:01.522 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:09:01.522 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:09:01.522 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.38e+01, 1.31e+01 / 6.48e-01 | 20260506 00:09:01.522 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.04e-01, 1.13e-02 / 5.48e-03 | 20260506 00:09:01.522 | INFO | PerfReport: 11.84%, Baseline -- 3.71e+00 / 4.51e+00, 5.70e-01 / 1.70e-01 | 20260506 00:09:01.522 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.52e+00, 1.90e-01 / 9.88e-02 | 20260506 00:09:01.522 | INFO | PerfReport: 15.96%, DME -- 4.00e+01 / 4.38e+01, 3.75e+00 / 7.93e-01 | 20260506 00:09:01.522 | INFO | PerfReport: 57.36%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.49e+00 / 7.88e-01 | 20260506 00:09:01.522 | INFO | PerfReport: 7.10%, PulseAccumulating -- 2.22e+00 / 2.75e+00, 4.23e-01 / 1.81e-01 | 20260506 00:09:01.522 | INFO | PerfReport: 3.09%, Metrics -- 9.68e-01 / 1.24e+00, 2.60e-01 / 1.20e-01 | 20260506 00:09:01.522 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.35e-01 / 3.88e-01, 5.65e-02 / 3.08e-02 | 20260506 00:09:01.522 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 00:09:01.522 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:09:01.522 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:09:01.522 | INFO | PerfReport: Full Execution -- 9.83e+01 / -nan, 2.46e+01 / 0.00e+00 | 20260506 00:09:01.522 | INFO | PerfReport: 22.39%, Upload -- 2.20e+01 / 0.00e+00, 7.54e+00 / 0.00e+00 | 20260506 00:09:01.522 | INFO | PerfReport: 21.85%, Download -- 2.15e+01 / 0.00e+00, 7.59e+00 / 0.00e+00 | 20260506 00:09:01.522 | INFO | PerfReport: 31.87%, Compute -- 3.13e+01 / 0.00e+00, 1.31e+01 / 0.00e+00 | 20260506 00:09:01.522 | INFO | PerfReport: 2.13%, LockUpload -- 2.09e+00 / 0.00e+00, 7.96e+00 / 0.00e+00 | 20260506 00:09:01.522 | INFO | PerfReport: 1.31%, LockDownload -- 1.29e+00 / 0.00e+00, 2.86e+00 / 0.00e+00 | 20260506 00:09:01.522 | INFO | PerfReport: 20.44%, LockCompute -- 2.01e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 00:09:01.522 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.84e-02 / 0.00e+00 | 20260506 00:09:01.646 | INFO | Starting a round of Baz IO | 20260506 00:09:04.793 | INFO | Populating RTMetrics report | 20260506 00:09:04.793 | INFO | RTMetrics report populated | 20260506 00:09:04.793 | INFO | Opening tmp RTMetrics json file | 20260506 00:09:04.794 | INFO | Serializing RTMetrics to json | 20260506 00:09:04.797 | INFO | Serialization done, swapping tmp file now... | 20260506 00:09:04.799 | INFO | RTMetrics done | 20260506 00:09:04.875 | INFO | Finished a round of Baz IO: Wrote 1.87 GiB in 3.228 seconds | 20260506 00:09:04.875 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.39% / 0.00% / 78.61% | 20260506 00:09:04.875 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.097084% / 0% / 5.5631e-05% | 20260506 00:09:04.875 | INFO | Baz IO overall: | 20260506 00:09:04.875 | INFO | BytesWritten : 4210988051192 | 20260506 00:09:04.875 | INFO | NumEvents : 1982318735207 | 20260506 00:09:06.665 | INFO | Analyzing chunk frames = [7397376,7397888) | 20260506 00:09:06.665 | INFO | MemInfo: { | 20260506 00:09:06.665 | INFO | "gpu" : " 17 GB", | 20260506 00:09:06.665 | INFO | "huge" : "", | 20260506 00:09:06.665 | INFO | "huge_shared" : " 27 GB", | 20260506 00:09:06.665 | INFO | "malloc" : " 14 GB", | 20260506 00:09:06.665 | INFO | "pinned" : " 29 GB" | 20260506 00:09:06.665 | INFO | } | 20260506 00:09:06.665 | INFO | Perf report for the last 5.14441 seconds: | 20260506 00:09:06.665 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:09:06.665 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:06.665 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:06.665 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:06.665 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260506 00:09:06.665 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:06.665 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260506 00:09:06.666 | INFO | PRE_HQ : 59.8%, 1.7, 53.9ms, 0 | 20260506 00:09:06.666 | INFO | BAZWRITER : 62.0%, 3.5, 112.8ms, 0 | 20260506 00:09:06.666 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 00:09:06.666 | INFO | Uploaded 36GiB to GPU in 2.17318s (16.5656GiB/s) | 20260506 00:09:06.666 | INFO | Downloaded 38GiB to GPU in 2.24048s (16.9607GiB/s) | 20260506 00:09:11.803 | INFO | Analyzing chunk frames = [7397888,7398400) | 20260506 00:09:11.803 | INFO | MemInfo: { | 20260506 00:09:11.803 | INFO | "gpu" : " 17 GB", | 20260506 00:09:11.803 | INFO | "huge" : "", | 20260506 00:09:11.804 | INFO | "huge_shared" : " 27 GB", | 20260506 00:09:11.804 | INFO | "malloc" : " 14 GB", | 20260506 00:09:11.804 | INFO | "pinned" : " 29 GB" | 20260506 00:09:11.804 | INFO | } | 20260506 00:09:11.804 | INFO | Perf report for the last 5.13819 seconds: | 20260506 00:09:11.804 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:09:11.804 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:11.804 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:11.804 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:11.804 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 00:09:11.804 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:11.804 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:11.804 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:09:11.804 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:11.804 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:11.804 | INFO | Uploaded 36GiB to GPU in 1.98369s (18.148GiB/s) | 20260506 00:09:11.804 | INFO | Downloaded 34GiB to GPU in 1.99108s (17.0762GiB/s) | 20260506 00:09:16.938 | INFO | Analyzing chunk frames = [7398400,7398912) | 20260506 00:09:16.938 | INFO | MemInfo: { | 20260506 00:09:16.938 | INFO | "gpu" : " 17 GB", | 20260506 00:09:16.938 | INFO | "huge" : "", | 20260506 00:09:16.938 | INFO | "huge_shared" : " 27 GB", | 20260506 00:09:16.938 | INFO | "malloc" : " 14 GB", | 20260506 00:09:16.938 | INFO | "pinned" : " 29 GB" | 20260506 00:09:16.938 | INFO | } | 20260506 00:09:16.938 | INFO | Perf report for the last 5.13445 seconds: | 20260506 00:09:16.938 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:09:16.938 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:16.938 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:16.938 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:16.938 | INFO | ANALYSIS : 61.4%, 3.0, 95.4ms, 0 | 20260506 00:09:16.938 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:16.938 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:16.938 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:09:16.938 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:16.938 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:16.938 | INFO | Uploaded 36GiB to GPU in 2.09259s (17.2036GiB/s) | 20260506 00:09:16.938 | INFO | Downloaded 34GiB to GPU in 2.02118s (16.8219GiB/s) | 20260506 00:09:22.072 | INFO | Analyzing chunk frames = [7398912,7399424) | 20260506 00:09:22.072 | INFO | MemInfo: { | 20260506 00:09:22.072 | INFO | "gpu" : " 17 GB", | 20260506 00:09:22.072 | INFO | "huge" : "", | 20260506 00:09:22.072 | INFO | "huge_shared" : " 27 GB", | 20260506 00:09:22.072 | INFO | "malloc" : " 14 GB", | 20260506 00:09:22.072 | INFO | "pinned" : " 29 GB" | 20260506 00:09:22.072 | INFO | } | 20260506 00:09:22.072 | INFO | Perf report for the last 5.13433 seconds: | 20260506 00:09:22.072 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:09:22.072 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:22.072 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:22.072 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:22.072 | INFO | ANALYSIS : 61.7%, 3.0, 95.7ms, 0 | 20260506 00:09:22.072 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:22.072 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:22.072 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 00:09:22.072 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:22.073 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:22.073 | INFO | Uploaded 36GiB to GPU in 2.13718s (16.8446GiB/s) | 20260506 00:09:22.073 | INFO | Downloaded 34GiB to GPU in 2.00884s (16.9252GiB/s) | 20260506 00:09:27.216 | INFO | Analyzing chunk frames = [7399424,7399936) | 20260506 00:09:27.216 | INFO | MemInfo: { | 20260506 00:09:27.216 | INFO | "gpu" : " 17 GB", | 20260506 00:09:27.216 | INFO | "huge" : "", | 20260506 00:09:27.216 | INFO | "huge_shared" : " 27 GB", | 20260506 00:09:27.216 | INFO | "malloc" : " 14 GB", | 20260506 00:09:27.216 | INFO | "pinned" : " 29 GB" | 20260506 00:09:27.216 | INFO | } | 20260506 00:09:27.216 | INFO | Perf report for the last 5.14363 seconds: | 20260506 00:09:27.216 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:09:27.216 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:27.216 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:27.216 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:27.216 | INFO | ANALYSIS : 61.5%, 3.0, 96.1ms, 0 | 20260506 00:09:27.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:27.216 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:27.216 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:09:27.216 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:27.216 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:27.216 | INFO | Uploaded 36GiB to GPU in 2.1428s (16.8005GiB/s) | 20260506 00:09:27.216 | INFO | Downloaded 34GiB to GPU in 2.06552s (16.4607GiB/s) | 20260506 00:09:32.357 | INFO | Analyzing chunk frames = [7399936,7400448) | 20260506 00:09:32.357 | INFO | MemInfo: { | 20260506 00:09:32.357 | INFO | "gpu" : " 17 GB", | 20260506 00:09:32.357 | INFO | "huge" : "", | 20260506 00:09:32.357 | INFO | "huge_shared" : " 27 GB", | 20260506 00:09:32.358 | INFO | "malloc" : " 14 GB", | 20260506 00:09:32.358 | INFO | "pinned" : " 29 GB" | 20260506 00:09:32.358 | INFO | } | 20260506 00:09:32.358 | INFO | Perf report for the last 5.14154 seconds: | 20260506 00:09:32.358 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:09:32.358 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:32.358 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:32.358 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:32.358 | INFO | ANALYSIS : 61.2%, 3.0, 95.7ms, 0 | 20260506 00:09:32.358 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:32.358 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:32.358 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:09:32.358 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:32.358 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:32.358 | INFO | Uploaded 36GiB to GPU in 2.06696s (17.4169GiB/s) | 20260506 00:09:32.358 | INFO | Downloaded 34GiB to GPU in 2.03127s (16.7383GiB/s) | 20260506 00:09:37.501 | INFO | Analyzing chunk frames = [7400448,7400960) | 20260506 00:09:37.501 | INFO | MemInfo: { | 20260506 00:09:37.501 | INFO | "gpu" : " 17 GB", | 20260506 00:09:37.501 | INFO | "huge" : "", | 20260506 00:09:37.501 | INFO | "huge_shared" : " 27 GB", | 20260506 00:09:37.501 | INFO | "malloc" : " 14 GB", | 20260506 00:09:37.501 | INFO | "pinned" : " 29 GB" | 20260506 00:09:37.501 | INFO | } | 20260506 00:09:37.501 | INFO | Perf report for the last 5.14369 seconds: | 20260506 00:09:37.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:09:37.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:37.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:37.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:37.501 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260506 00:09:37.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:37.501 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:37.501 | INFO | PRE_HQ : 27.2%, 1.0, 14.2ms, 0 | 20260506 00:09:37.501 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:37.501 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:37.502 | INFO | Uploaded 36GiB to GPU in 2.0696s (17.3947GiB/s) | 20260506 00:09:37.502 | INFO | Downloaded 34GiB to GPU in 2.073s (16.4013GiB/s) | 20260506 00:09:42.637 | INFO | Analyzing chunk frames = [7400960,7401472) | 20260506 00:09:42.637 | INFO | MemInfo: { | 20260506 00:09:42.637 | INFO | "gpu" : " 17 GB", | 20260506 00:09:42.637 | INFO | "huge" : "", | 20260506 00:09:42.637 | INFO | "huge_shared" : " 27 GB", | 20260506 00:09:42.637 | INFO | "malloc" : " 14 GB", | 20260506 00:09:42.637 | INFO | "pinned" : " 29 GB" | 20260506 00:09:42.637 | INFO | } | 20260506 00:09:42.637 | INFO | Perf report for the last 5.13553 seconds: | 20260506 00:09:42.637 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:09:42.637 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:42.637 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:42.637 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:42.637 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 00:09:42.637 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:42.637 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:42.637 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 00:09:42.637 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:42.637 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:42.637 | INFO | Uploaded 36GiB to GPU in 2.15494s (16.7058GiB/s) | 20260506 00:09:42.637 | INFO | Downloaded 34GiB to GPU in 2.02647s (16.7779GiB/s) | 20260506 00:09:42.757 | INFO | Starting a round of Baz IO | 20260506 00:09:45.904 | INFO | Populating RTMetrics report | 20260506 00:09:45.905 | INFO | RTMetrics report populated | 20260506 00:09:45.905 | INFO | Opening tmp RTMetrics json file | 20260506 00:09:45.907 | INFO | Serializing RTMetrics to json | 20260506 00:09:45.909 | INFO | Serialization done, swapping tmp file now... | 20260506 00:09:45.910 | INFO | RTMetrics done | 20260506 00:09:45.980 | INFO | Finished a round of Baz IO: Wrote 1.868 GiB in 3.223 seconds | 20260506 00:09:45.980 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.39% / 0.00% / 78.61% | 20260506 00:09:45.980 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0961304% / 0% / 4.76837e-05% | 20260506 00:09:45.980 | INFO | Baz IO overall: | 20260506 00:09:45.981 | INFO | BytesWritten : 4212993689100 | 20260506 00:09:45.981 | INFO | NumEvents : 1983235977641 | 20260506 00:09:47.774 | INFO | Analyzing chunk frames = [7401472,7401984) | 20260506 00:09:47.774 | INFO | MemInfo: { | 20260506 00:09:47.774 | INFO | "gpu" : " 17 GB", | 20260506 00:09:47.774 | INFO | "huge" : "", | 20260506 00:09:47.774 | INFO | "huge_shared" : " 27 GB", | 20260506 00:09:47.774 | INFO | "malloc" : " 14 GB", | 20260506 00:09:47.774 | INFO | "pinned" : " 29 GB" | 20260506 00:09:47.774 | INFO | } | 20260506 00:09:47.774 | INFO | Perf report for the last 5.13712 seconds: | 20260506 00:09:47.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:09:47.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:47.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:47.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:47.774 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260506 00:09:47.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:47.774 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260506 00:09:47.774 | INFO | PRE_HQ : 59.9%, 1.7, 54.0ms, 0 | 20260506 00:09:47.774 | INFO | BAZWRITER : 62.0%, 3.5, 112.8ms, 0 | 20260506 00:09:47.774 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:09:47.774 | INFO | Uploaded 36GiB to GPU in 2.16713s (16.6118GiB/s) | 20260506 00:09:47.774 | INFO | Downloaded 38GiB to GPU in 2.23674s (16.989GiB/s) | 20260506 00:09:52.916 | INFO | Analyzing chunk frames = [7401984,7402496) | 20260506 00:09:52.916 | INFO | MemInfo: { | 20260506 00:09:52.916 | INFO | "gpu" : " 17 GB", | 20260506 00:09:52.916 | INFO | "huge" : "", | 20260506 00:09:52.916 | INFO | "huge_shared" : " 27 GB", | 20260506 00:09:52.916 | INFO | "malloc" : " 14 GB", | 20260506 00:09:52.916 | INFO | "pinned" : " 29 GB" | 20260506 00:09:52.916 | INFO | } | 20260506 00:09:52.916 | INFO | Perf report for the last 5.14238 seconds: | 20260506 00:09:52.916 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:09:52.916 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:52.916 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:52.916 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:52.916 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260506 00:09:52.916 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:52.916 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:52.916 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:09:52.916 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:52.916 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:52.917 | INFO | Uploaded 36GiB to GPU in 2.0974s (17.1641GiB/s) | 20260506 00:09:52.917 | INFO | Downloaded 34GiB to GPU in 2.03457s (16.7111GiB/s) | 20260506 00:09:58.053 | INFO | Analyzing chunk frames = [7402496,7403008) | 20260506 00:09:58.053 | INFO | MemInfo: { | 20260506 00:09:58.053 | INFO | "gpu" : " 17 GB", | 20260506 00:09:58.053 | INFO | "huge" : "", | 20260506 00:09:58.053 | INFO | "huge_shared" : " 27 GB", | 20260506 00:09:58.053 | INFO | "malloc" : " 14 GB", | 20260506 00:09:58.053 | INFO | "pinned" : " 29 GB" | 20260506 00:09:58.053 | INFO | } | 20260506 00:09:58.053 | INFO | Perf report for the last 5.13671 seconds: | 20260506 00:09:58.053 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:09:58.053 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:58.053 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:58.053 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:58.053 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 00:09:58.053 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:58.053 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:58.053 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:09:58.053 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:58.053 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:09:58.053 | INFO | Uploaded 36GiB to GPU in 2.08216s (17.2898GiB/s) | 20260506 00:09:58.053 | INFO | Downloaded 34GiB to GPU in 2.01847s (16.8444GiB/s) | 20260506 00:10:03.194 | INFO | Analyzing chunk frames = [7403008,7403520) | 20260506 00:10:03.194 | INFO | MemInfo: { | 20260506 00:10:03.194 | INFO | "gpu" : " 17 GB", | 20260506 00:10:03.194 | INFO | "huge" : "", | 20260506 00:10:03.194 | INFO | "huge_shared" : " 27 GB", | 20260506 00:10:03.194 | INFO | "malloc" : " 14 GB", | 20260506 00:10:03.194 | INFO | "pinned" : " 29 GB" | 20260506 00:10:03.194 | INFO | } | 20260506 00:10:03.194 | INFO | Perf report for the last 5.14066 seconds: | 20260506 00:10:03.194 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:10:03.194 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:03.194 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:03.194 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:03.194 | INFO | ANALYSIS : 61.7%, 3.0, 95.8ms, 0 | 20260506 00:10:03.194 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:03.194 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:03.194 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 00:10:03.194 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:03.194 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:03.194 | INFO | Uploaded 36GiB to GPU in 2.14358s (16.7944GiB/s) | 20260506 00:10:03.194 | INFO | Downloaded 34GiB to GPU in 2.02544s (16.7865GiB/s) | 20260506 00:10:08.334 | INFO | Analyzing chunk frames = [7403520,7404032) | 20260506 00:10:08.334 | INFO | MemInfo: { | 20260506 00:10:08.334 | INFO | "gpu" : " 17 GB", | 20260506 00:10:08.334 | INFO | "huge" : "", | 20260506 00:10:08.334 | INFO | "huge_shared" : " 27 GB", | 20260506 00:10:08.334 | INFO | "malloc" : " 14 GB", | 20260506 00:10:08.334 | INFO | "pinned" : " 29 GB" | 20260506 00:10:08.334 | INFO | } | 20260506 00:10:08.334 | INFO | Perf report for the last 5.14046 seconds: | 20260506 00:10:08.334 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:10:08.334 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:08.334 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:08.334 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:08.334 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 00:10:08.334 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:08.334 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:08.334 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:10:08.334 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:08.334 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:08.334 | INFO | Uploaded 36GiB to GPU in 2.09405s (17.1916GiB/s) | 20260506 00:10:08.334 | INFO | Downloaded 34GiB to GPU in 2.04716s (16.6084GiB/s) | 20260506 00:10:13.474 | INFO | Analyzing chunk frames = [7404032,7404544) | 20260506 00:10:13.474 | INFO | MemInfo: { | 20260506 00:10:13.474 | INFO | "gpu" : " 17 GB", | 20260506 00:10:13.474 | INFO | "huge" : "", | 20260506 00:10:13.474 | INFO | "huge_shared" : " 27 GB", | 20260506 00:10:13.474 | INFO | "malloc" : " 14 GB", | 20260506 00:10:13.474 | INFO | "pinned" : " 29 GB" | 20260506 00:10:13.474 | INFO | } | 20260506 00:10:13.474 | INFO | Perf report for the last 5.1401 seconds: | 20260506 00:10:13.474 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:10:13.474 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:13.474 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:13.474 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:13.474 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260506 00:10:13.474 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:13.474 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:13.474 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:10:13.474 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:13.474 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:13.475 | INFO | Uploaded 36GiB to GPU in 2.07751s (17.3284GiB/s) | 20260506 00:10:13.475 | INFO | Downloaded 34GiB to GPU in 2.03821s (16.6813GiB/s) | 20260506 00:10:18.616 | INFO | Analyzing chunk frames = [7404544,7405056) | 20260506 00:10:18.616 | INFO | MemInfo: { | 20260506 00:10:18.616 | INFO | "gpu" : " 17 GB", | 20260506 00:10:18.616 | INFO | "huge" : "", | 20260506 00:10:18.616 | INFO | "huge_shared" : " 27 GB", | 20260506 00:10:18.616 | INFO | "malloc" : " 14 GB", | 20260506 00:10:18.616 | INFO | "pinned" : " 29 GB" | 20260506 00:10:18.616 | INFO | } | 20260506 00:10:18.616 | INFO | Perf report for the last 5.14135 seconds: | 20260506 00:10:18.616 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:10:18.616 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:18.616 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:18.616 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:18.616 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260506 00:10:18.616 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:18.616 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:18.616 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260506 00:10:18.616 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:18.616 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:18.616 | INFO | Uploaded 36GiB to GPU in 2.09879s (17.1528GiB/s) | 20260506 00:10:18.616 | INFO | Downloaded 34GiB to GPU in 2.07666s (16.3725GiB/s) | 20260506 00:10:23.755 | INFO | Analyzing chunk frames = [7405056,7405568) | 20260506 00:10:23.755 | INFO | MemInfo: { | 20260506 00:10:23.755 | INFO | "gpu" : " 17 GB", | 20260506 00:10:23.755 | INFO | "huge" : "", | 20260506 00:10:23.755 | INFO | "huge_shared" : " 27 GB", | 20260506 00:10:23.755 | INFO | "malloc" : " 14 GB", | 20260506 00:10:23.755 | INFO | "pinned" : " 29 GB" | 20260506 00:10:23.755 | INFO | } | 20260506 00:10:23.755 | INFO | Perf report for the last 5.13909 seconds: | 20260506 00:10:23.755 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:10:23.755 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:23.755 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:23.755 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:23.755 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260506 00:10:23.755 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:23.755 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:23.755 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 00:10:23.755 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:23.755 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:23.755 | INFO | Uploaded 36GiB to GPU in 2.13455s (16.8654GiB/s) | 20260506 00:10:23.755 | INFO | Downloaded 34GiB to GPU in 2.01729s (16.8543GiB/s) | 20260506 00:10:23.755 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 00:10:23.755 | INFO | Total mem usage: 15.3 GiB | 20260506 00:10:23.755 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 00:10:23.755 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.13%) | 20260506 00:10:23.755 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 00:10:23.755 | INFO | High Water Breakdown: | 20260506 00:10:23.755 | INFO | 60.9% : 8.55 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 00:10:23.755 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 00:10:23.755 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 00:10:23.755 | INFO | ------------------------------------------ | 20260506 00:10:23.755 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:10:23.755 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 00:10:23.755 | INFO | Total mem usage: 26.5 GiB | 20260506 00:10:23.755 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 00:10:23.755 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 00:10:23.755 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260506 00:10:23.755 | INFO | High Water Breakdown: | 20260506 00:10:23.755 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:10:23.755 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:10:23.755 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:10:23.755 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:10:23.755 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:10:23.755 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:10:23.755 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 00:10:23.756 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 00:10:23.756 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:10:23.756 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 00:10:23.756 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 00:10:23.756 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 00:10:23.756 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 00:10:23.756 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 00:10:23.756 | INFO | ------------------------------------------ | 20260506 00:10:23.756 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:10:23.756 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 00:10:23.756 | INFO | Total mem usage: 25.6 GiB | 20260506 00:10:23.756 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 00:10:23.756 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 00:10:23.756 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 00:10:23.756 | INFO | High Water Breakdown: | 20260506 00:10:23.756 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 00:10:23.756 | INFO | ------------------------------------------ | 20260506 00:10:23.756 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:10:23.756 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 00:10:23.756 | INFO | Total mem usage: 15.7 GiB | 20260506 00:10:23.756 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 00:10:23.756 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 00:10:23.756 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 00:10:23.756 | INFO | High Water Breakdown: | 20260506 00:10:23.756 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 00:10:23.756 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 00:10:23.756 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 00:10:23.756 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 00:10:23.756 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 00:10:23.756 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 00:10:23.756 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 00:10:23.756 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 00:10:23.756 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 00:10:23.756 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:10:23.756 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 00:10:23.756 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:10:23.756 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 00:10:23.756 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 00:10:23.756 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 00:10:23.756 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 00:10:23.756 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:10:23.756 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 00:10:23.756 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:10:23.756 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:10:23.756 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:10:23.756 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 00:10:23.756 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 00:10:23.756 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:10:23.756 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 00:10:23.756 | INFO | ------------------------------------------ | 20260506 00:10:23.756 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:10:23.756 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 00:10:23.756 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:10:23.756 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:10:23.756 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.38e+01, 1.31e+01 / 6.55e-01 | 20260506 00:10:23.756 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.04e-01, 1.06e-02 / 3.92e-03 | 20260506 00:10:23.756 | INFO | PerfReport: 11.84%, Baseline -- 3.71e+00 / 4.50e+00, 5.68e-01 / 2.29e-01 | 20260506 00:10:23.756 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.52e+00, 1.89e-01 / 1.12e-01 | 20260506 00:10:23.756 | INFO | PerfReport: 15.96%, DME -- 4.00e+01 / 4.38e+01, 3.82e+00 / 8.72e-01 | 20260506 00:10:23.756 | INFO | PerfReport: 57.38%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.51e+00 / 1.30e+00 | 20260506 00:10:23.756 | INFO | PerfReport: 7.09%, PulseAccumulating -- 2.22e+00 / 2.72e+00, 4.20e-01 / 2.67e-01 | 20260506 00:10:23.756 | INFO | PerfReport: 3.09%, Metrics -- 9.67e-01 / 1.24e+00, 2.60e-01 / 1.27e-01 | 20260506 00:10:23.756 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.34e-01 / 3.82e-01, 5.65e-02 / 2.63e-02 | 20260506 00:10:23.756 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 00:10:23.756 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:10:23.756 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:10:23.756 | INFO | PerfReport: Full Execution -- 9.83e+01 / -nan, 2.45e+01 / 0.00e+00 | 20260506 00:10:23.756 | INFO | PerfReport: 22.33%, Upload -- 2.20e+01 / 0.00e+00, 7.54e+00 / 0.00e+00 | 20260506 00:10:23.756 | INFO | PerfReport: 21.83%, Download -- 2.15e+01 / 0.00e+00, 7.56e+00 / 0.00e+00 | 20260506 00:10:23.756 | INFO | PerfReport: 31.86%, Compute -- 3.13e+01 / 0.00e+00, 1.31e+01 / 0.00e+00 | 20260506 00:10:23.756 | INFO | PerfReport: 2.15%, LockUpload -- 2.12e+00 / 0.00e+00, 7.89e+00 / 0.00e+00 | 20260506 00:10:23.756 | INFO | PerfReport: 1.29%, LockDownload -- 1.27e+00 / 0.00e+00, 2.82e+00 / 0.00e+00 | 20260506 00:10:23.756 | INFO | PerfReport: 20.53%, LockCompute -- 2.02e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260506 00:10:23.756 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.84e-02 / 0.00e+00 | 20260506 00:10:23.873 | INFO | Starting a round of Baz IO | 20260506 00:10:27.018 | INFO | Populating RTMetrics report | 20260506 00:10:27.019 | INFO | RTMetrics report populated | 20260506 00:10:27.019 | INFO | Opening tmp RTMetrics json file | 20260506 00:10:27.020 | INFO | Serializing RTMetrics to json | 20260506 00:10:27.022 | INFO | Serialization done, swapping tmp file now... | 20260506 00:10:27.022 | INFO | RTMetrics done | 20260506 00:10:27.096 | INFO | Finished a round of Baz IO: Wrote 1.869 GiB in 3.223 seconds | 20260506 00:10:27.097 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.39% / 0.00% / 78.61% | 20260506 00:10:27.097 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0963052% / 0% / 7.94729e-05% | 20260506 00:10:27.097 | INFO | Baz IO overall: | 20260506 00:10:27.097 | INFO | BytesWritten : 4215000150528 | 20260506 00:10:27.097 | INFO | NumEvents : 1984154123099 | 20260506 00:10:28.889 | INFO | Analyzing chunk frames = [7405568,7406080) | 20260506 00:10:28.889 | INFO | MemInfo: { | 20260506 00:10:28.889 | INFO | "gpu" : " 17 GB", | 20260506 00:10:28.889 | INFO | "huge" : "", | 20260506 00:10:28.889 | INFO | "huge_shared" : " 27 GB", | 20260506 00:10:28.889 | INFO | "malloc" : " 14 GB", | 20260506 00:10:28.889 | INFO | "pinned" : " 29 GB" | 20260506 00:10:28.889 | INFO | } | 20260506 00:10:28.889 | INFO | Perf report for the last 5.13464 seconds: | 20260506 00:10:28.889 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:10:28.889 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:28.889 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:28.889 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:28.889 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260506 00:10:28.889 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:28.890 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260506 00:10:28.890 | INFO | PRE_HQ : 59.8%, 1.7, 53.8ms, 0 | 20260506 00:10:28.890 | INFO | BAZWRITER : 61.8%, 3.6, 113.3ms, 0 | 20260506 00:10:28.890 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:10:28.890 | INFO | Uploaded 36GiB to GPU in 2.18458s (16.4792GiB/s) | 20260506 00:10:28.890 | INFO | Downloaded 38GiB to GPU in 2.23941s (16.9688GiB/s) | 20260506 00:10:34.024 | INFO | Analyzing chunk frames = [7406080,7406592) | 20260506 00:10:34.024 | INFO | MemInfo: { | 20260506 00:10:34.024 | INFO | "gpu" : " 17 GB", | 20260506 00:10:34.024 | INFO | "huge" : "", | 20260506 00:10:34.024 | INFO | "huge_shared" : " 27 GB", | 20260506 00:10:34.024 | INFO | "malloc" : " 14 GB", | 20260506 00:10:34.024 | INFO | "pinned" : " 29 GB" | 20260506 00:10:34.024 | INFO | } | 20260506 00:10:34.024 | INFO | Perf report for the last 5.13462 seconds: | 20260506 00:10:34.024 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:10:34.024 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:34.024 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:34.024 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:34.024 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 00:10:34.024 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:34.024 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:34.024 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:10:34.024 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:34.024 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:34.024 | INFO | Uploaded 36GiB to GPU in 2.10954s (17.0653GiB/s) | 20260506 00:10:34.024 | INFO | Downloaded 34GiB to GPU in 2.04323s (16.6403GiB/s) | 20260506 00:10:39.169 | INFO | Analyzing chunk frames = [7406592,7407104) | 20260506 00:10:39.169 | INFO | MemInfo: { | 20260506 00:10:39.169 | INFO | "gpu" : " 17 GB", | 20260506 00:10:39.169 | INFO | "huge" : "", | 20260506 00:10:39.169 | INFO | "huge_shared" : " 27 GB", | 20260506 00:10:39.169 | INFO | "malloc" : " 14 GB", | 20260506 00:10:39.169 | INFO | "pinned" : " 29 GB" | 20260506 00:10:39.169 | INFO | } | 20260506 00:10:39.169 | INFO | Perf report for the last 5.14476 seconds: | 20260506 00:10:39.169 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:10:39.169 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:39.169 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:39.169 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:39.169 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 00:10:39.169 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:39.169 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:39.169 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:10:39.169 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:39.169 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:39.169 | INFO | Uploaded 36GiB to GPU in 2.10228s (17.1243GiB/s) | 20260506 00:10:39.169 | INFO | Downloaded 34GiB to GPU in 2.03544s (16.704GiB/s) | 20260506 00:10:44.310 | INFO | Analyzing chunk frames = [7407104,7407616) | 20260506 00:10:44.310 | INFO | MemInfo: { | 20260506 00:10:44.310 | INFO | "gpu" : " 17 GB", | 20260506 00:10:44.310 | INFO | "huge" : "", | 20260506 00:10:44.310 | INFO | "huge_shared" : " 27 GB", | 20260506 00:10:44.310 | INFO | "malloc" : " 14 GB", | 20260506 00:10:44.310 | INFO | "pinned" : " 29 GB" | 20260506 00:10:44.310 | INFO | } | 20260506 00:10:44.310 | INFO | Perf report for the last 5.141 seconds: | 20260506 00:10:44.310 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:10:44.310 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:44.310 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:44.310 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:44.310 | INFO | ANALYSIS : 61.6%, 3.0, 95.7ms, 0 | 20260506 00:10:44.310 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:44.310 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:44.310 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 00:10:44.310 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:44.310 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:44.310 | INFO | Uploaded 36GiB to GPU in 2.13755s (16.8417GiB/s) | 20260506 00:10:44.310 | INFO | Downloaded 34GiB to GPU in 2.02308s (16.8061GiB/s) | 20260506 00:10:49.452 | INFO | Analyzing chunk frames = [7407616,7408128) | 20260506 00:10:49.452 | INFO | MemInfo: { | 20260506 00:10:49.452 | INFO | "gpu" : " 17 GB", | 20260506 00:10:49.452 | INFO | "huge" : "", | 20260506 00:10:49.452 | INFO | "huge_shared" : " 27 GB", | 20260506 00:10:49.452 | INFO | "malloc" : " 14 GB", | 20260506 00:10:49.452 | INFO | "pinned" : " 29 GB" | 20260506 00:10:49.452 | INFO | } | 20260506 00:10:49.452 | INFO | Perf report for the last 5.14184 seconds: | 20260506 00:10:49.452 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:10:49.452 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:49.452 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:49.452 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:49.452 | INFO | ANALYSIS : 60.9%, 3.0, 95.2ms, 0 | 20260506 00:10:49.452 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:49.452 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:49.452 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:10:49.452 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:49.452 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:49.452 | INFO | Uploaded 36GiB to GPU in 2.09153s (17.2123GiB/s) | 20260506 00:10:49.452 | INFO | Downloaded 34GiB to GPU in 2.04509s (16.6252GiB/s) | 20260506 00:10:54.585 | INFO | Analyzing chunk frames = [7408128,7408640) | 20260506 00:10:54.585 | INFO | MemInfo: { | 20260506 00:10:54.585 | INFO | "gpu" : " 17 GB", | 20260506 00:10:54.585 | INFO | "huge" : "", | 20260506 00:10:54.585 | INFO | "huge_shared" : " 27 GB", | 20260506 00:10:54.585 | INFO | "malloc" : " 14 GB", | 20260506 00:10:54.585 | INFO | "pinned" : " 29 GB" | 20260506 00:10:54.585 | INFO | } | 20260506 00:10:54.585 | INFO | Perf report for the last 5.1333 seconds: | 20260506 00:10:54.585 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:10:54.585 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:54.585 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:54.585 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:54.585 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 00:10:54.585 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:54.585 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:54.585 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 00:10:54.585 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:54.585 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:54.585 | INFO | Uploaded 36GiB to GPU in 2.06827s (17.4059GiB/s) | 20260506 00:10:54.585 | INFO | Downloaded 34GiB to GPU in 2.02787s (16.7663GiB/s) | 20260506 00:10:59.721 | INFO | Analyzing chunk frames = [7408640,7409152) | 20260506 00:10:59.721 | INFO | MemInfo: { | 20260506 00:10:59.721 | INFO | "gpu" : " 17 GB", | 20260506 00:10:59.721 | INFO | "huge" : "", | 20260506 00:10:59.721 | INFO | "huge_shared" : " 27 GB", | 20260506 00:10:59.721 | INFO | "malloc" : " 14 GB", | 20260506 00:10:59.721 | INFO | "pinned" : " 29 GB" | 20260506 00:10:59.721 | INFO | } | 20260506 00:10:59.721 | INFO | Perf report for the last 5.13607 seconds: | 20260506 00:10:59.721 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:10:59.721 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:59.721 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:59.721 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:59.721 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260506 00:10:59.721 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:59.721 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:59.721 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 00:10:59.721 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:59.721 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:10:59.721 | INFO | Uploaded 36GiB to GPU in 2.12135s (16.9703GiB/s) | 20260506 00:10:59.721 | INFO | Downloaded 34GiB to GPU in 2.08841s (16.2803GiB/s) | 20260506 00:11:04.866 | INFO | Analyzing chunk frames = [7409152,7409664) | 20260506 00:11:04.866 | INFO | MemInfo: { | 20260506 00:11:04.866 | INFO | "gpu" : " 17 GB", | 20260506 00:11:04.866 | INFO | "huge" : "", | 20260506 00:11:04.866 | INFO | "huge_shared" : " 27 GB", | 20260506 00:11:04.866 | INFO | "malloc" : " 14 GB", | 20260506 00:11:04.866 | INFO | "pinned" : " 29 GB" | 20260506 00:11:04.866 | INFO | } | 20260506 00:11:04.866 | INFO | Perf report for the last 5.14508 seconds: | 20260506 00:11:04.866 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:11:04.866 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:04.866 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:04.866 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:04.866 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260506 00:11:04.866 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:04.866 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:04.866 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:11:04.866 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:04.866 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:04.867 | INFO | Uploaded 36GiB to GPU in 2.11584s (17.0145GiB/s) | 20260506 00:11:04.867 | INFO | Downloaded 34GiB to GPU in 2.0007s (16.9941GiB/s) | 20260506 00:11:04.983 | INFO | Starting a round of Baz IO | 20260506 00:11:08.129 | INFO | Populating RTMetrics report | 20260506 00:11:08.129 | INFO | RTMetrics report populated | 20260506 00:11:08.129 | INFO | Opening tmp RTMetrics json file | 20260506 00:11:08.131 | INFO | Serializing RTMetrics to json | 20260506 00:11:08.133 | INFO | Serialization done, swapping tmp file now... | 20260506 00:11:08.135 | INFO | RTMetrics done | 20260506 00:11:08.207 | INFO | Finished a round of Baz IO: Wrote 1.866 GiB in 3.223 seconds | 20260506 00:11:08.207 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.38% / 0.00% / 78.61% | 20260506 00:11:08.207 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0959277% / 7.94729e-06% / 9.93411e-05% | 20260506 00:11:08.207 | INFO | Baz IO overall: | 20260506 00:11:08.207 | INFO | BytesWritten : 4217004037211 | 20260506 00:11:08.207 | INFO | NumEvents : 1985070772627 | 20260506 00:11:10.004 | INFO | Analyzing chunk frames = [7409664,7410176) | 20260506 00:11:10.004 | INFO | MemInfo: { | 20260506 00:11:10.004 | INFO | "gpu" : " 17 GB", | 20260506 00:11:10.004 | INFO | "huge" : "", | 20260506 00:11:10.004 | INFO | "huge_shared" : " 27 GB", | 20260506 00:11:10.004 | INFO | "malloc" : " 14 GB", | 20260506 00:11:10.004 | INFO | "pinned" : " 29 GB" | 20260506 00:11:10.004 | INFO | } | 20260506 00:11:10.004 | INFO | Perf report for the last 5.1381 seconds: | 20260506 00:11:10.004 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:11:10.004 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:10.004 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:10.004 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:10.004 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260506 00:11:10.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:10.004 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260506 00:11:10.004 | INFO | PRE_HQ : 59.7%, 1.7, 53.7ms, 0 | 20260506 00:11:10.004 | INFO | BAZWRITER : 61.9%, 3.6, 113.2ms, 0 | 20260506 00:11:10.004 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:11:10.005 | INFO | Uploaded 36GiB to GPU in 2.16318s (16.6422GiB/s) | 20260506 00:11:10.005 | INFO | Downloaded 38GiB to GPU in 2.23489s (17.0031GiB/s) | 20260506 00:11:15.143 | INFO | Analyzing chunk frames = [7410176,7410688) | 20260506 00:11:15.143 | INFO | MemInfo: { | 20260506 00:11:15.143 | INFO | "gpu" : " 17 GB", | 20260506 00:11:15.143 | INFO | "huge" : "", | 20260506 00:11:15.143 | INFO | "huge_shared" : " 27 GB", | 20260506 00:11:15.143 | INFO | "malloc" : " 14 GB", | 20260506 00:11:15.143 | INFO | "pinned" : " 29 GB" | 20260506 00:11:15.143 | INFO | } | 20260506 00:11:15.143 | INFO | Perf report for the last 5.1387 seconds: | 20260506 00:11:15.143 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:11:15.143 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:15.143 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:15.143 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:15.143 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260506 00:11:15.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:15.143 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:15.143 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 00:11:15.143 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:15.143 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:15.143 | INFO | Uploaded 36GiB to GPU in 2.10192s (17.1272GiB/s) | 20260506 00:11:15.143 | INFO | Downloaded 34GiB to GPU in 2.03992s (16.6674GiB/s) | 20260506 00:11:20.280 | INFO | Analyzing chunk frames = [7410688,7411200) | 20260506 00:11:20.280 | INFO | MemInfo: { | 20260506 00:11:20.280 | INFO | "gpu" : " 17 GB", | 20260506 00:11:20.280 | INFO | "huge" : "", | 20260506 00:11:20.280 | INFO | "huge_shared" : " 27 GB", | 20260506 00:11:20.280 | INFO | "malloc" : " 14 GB", | 20260506 00:11:20.280 | INFO | "pinned" : " 29 GB" | 20260506 00:11:20.280 | INFO | } | 20260506 00:11:20.280 | INFO | Perf report for the last 5.13668 seconds: | 20260506 00:11:20.280 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:11:20.280 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:20.280 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:20.280 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:20.280 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260506 00:11:20.280 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:20.280 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:20.280 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:11:20.280 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:20.280 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:20.280 | INFO | Uploaded 36GiB to GPU in 2.14469s (16.7856GiB/s) | 20260506 00:11:20.280 | INFO | Downloaded 34GiB to GPU in 2.05543s (16.5416GiB/s) | 20260506 00:11:25.427 | INFO | Analyzing chunk frames = [7411200,7411712) | 20260506 00:11:25.427 | INFO | MemInfo: { | 20260506 00:11:25.427 | INFO | "gpu" : " 17 GB", | 20260506 00:11:25.427 | INFO | "huge" : "", | 20260506 00:11:25.427 | INFO | "huge_shared" : " 27 GB", | 20260506 00:11:25.427 | INFO | "malloc" : " 14 GB", | 20260506 00:11:25.427 | INFO | "pinned" : " 29 GB" | 20260506 00:11:25.427 | INFO | } | 20260506 00:11:25.427 | INFO | Perf report for the last 5.14726 seconds: | 20260506 00:11:25.427 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:11:25.427 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:25.427 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:25.427 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:25.427 | INFO | ANALYSIS : 61.6%, 3.0, 95.7ms, 0 | 20260506 00:11:25.427 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:25.427 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:25.427 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:11:25.427 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:25.427 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:25.427 | INFO | Uploaded 36GiB to GPU in 2.12116s (16.9719GiB/s) | 20260506 00:11:25.427 | INFO | Downloaded 34GiB to GPU in 2.01868s (16.8427GiB/s) | 20260506 00:11:30.561 | INFO | Analyzing chunk frames = [7411712,7412224) | 20260506 00:11:30.561 | INFO | MemInfo: { | 20260506 00:11:30.561 | INFO | "gpu" : " 17 GB", | 20260506 00:11:30.561 | INFO | "huge" : "", | 20260506 00:11:30.561 | INFO | "huge_shared" : " 27 GB", | 20260506 00:11:30.561 | INFO | "malloc" : " 14 GB", | 20260506 00:11:30.561 | INFO | "pinned" : " 29 GB" | 20260506 00:11:30.561 | INFO | } | 20260506 00:11:30.561 | INFO | Perf report for the last 5.13448 seconds: | 20260506 00:11:30.561 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:11:30.562 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:30.562 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:30.562 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:30.562 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 00:11:30.562 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:30.562 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:30.562 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 00:11:30.562 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:30.562 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:30.562 | INFO | Uploaded 36GiB to GPU in 2.09041s (17.2215GiB/s) | 20260506 00:11:30.562 | INFO | Downloaded 34GiB to GPU in 2.04036s (16.6637GiB/s) | 20260506 00:11:35.699 | INFO | Analyzing chunk frames = [7412224,7412736) | 20260506 00:11:35.699 | INFO | MemInfo: { | 20260506 00:11:35.699 | INFO | "gpu" : " 17 GB", | 20260506 00:11:35.699 | INFO | "huge" : "", | 20260506 00:11:35.699 | INFO | "huge_shared" : " 27 GB", | 20260506 00:11:35.699 | INFO | "malloc" : " 14 GB", | 20260506 00:11:35.700 | INFO | "pinned" : " 29 GB" | 20260506 00:11:35.700 | INFO | } | 20260506 00:11:35.700 | INFO | Perf report for the last 5.13806 seconds: | 20260506 00:11:35.700 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:11:35.700 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:35.700 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:35.700 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:35.700 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260506 00:11:35.700 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:35.700 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:35.700 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:11:35.700 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:35.700 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:35.700 | INFO | Uploaded 36GiB to GPU in 2.07454s (17.3533GiB/s) | 20260506 00:11:35.700 | INFO | Downloaded 34GiB to GPU in 2.0426s (16.6454GiB/s) | 20260506 00:11:40.847 | INFO | Analyzing chunk frames = [7412736,7413248) | 20260506 00:11:40.847 | INFO | MemInfo: { | 20260506 00:11:40.847 | INFO | "gpu" : " 17 GB", | 20260506 00:11:40.847 | INFO | "huge" : "", | 20260506 00:11:40.847 | INFO | "huge_shared" : " 27 GB", | 20260506 00:11:40.847 | INFO | "malloc" : " 14 GB", | 20260506 00:11:40.847 | INFO | "pinned" : " 29 GB" | 20260506 00:11:40.847 | INFO | } | 20260506 00:11:40.847 | INFO | Perf report for the last 5.14787 seconds: | 20260506 00:11:40.847 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:11:40.847 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:40.847 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:40.847 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:40.847 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260506 00:11:40.847 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:40.847 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:40.847 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 00:11:40.847 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:40.847 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:40.848 | INFO | Uploaded 36GiB to GPU in 2.03461s (17.6938GiB/s) | 20260506 00:11:40.848 | INFO | Downloaded 34GiB to GPU in 2.04299s (16.6423GiB/s) | 20260506 00:11:45.979 | INFO | Analyzing chunk frames = [7413248,7413760) | 20260506 00:11:45.979 | INFO | MemInfo: { | 20260506 00:11:45.979 | INFO | "gpu" : " 17 GB", | 20260506 00:11:45.979 | INFO | "huge" : "", | 20260506 00:11:45.979 | INFO | "huge_shared" : " 27 GB", | 20260506 00:11:45.979 | INFO | "malloc" : " 14 GB", | 20260506 00:11:45.979 | INFO | "pinned" : " 29 GB" | 20260506 00:11:45.979 | INFO | } | 20260506 00:11:45.979 | INFO | Perf report for the last 5.13203 seconds: | 20260506 00:11:45.979 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:11:45.979 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:45.979 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:45.979 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:45.979 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260506 00:11:45.979 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:45.980 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:45.980 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 00:11:45.980 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:45.980 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:45.980 | INFO | Uploaded 36GiB to GPU in 2.12973s (16.9036GiB/s) | 20260506 00:11:45.980 | INFO | Downloaded 34GiB to GPU in 2.02164s (16.818GiB/s) | 20260506 00:11:45.980 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 00:11:45.980 | INFO | Total mem usage: 15.3 GiB | 20260506 00:11:45.980 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 00:11:45.980 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.13%) | 20260506 00:11:45.980 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 00:11:45.980 | INFO | High Water Breakdown: | 20260506 00:11:45.980 | INFO | 60.9% : 8.55 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 00:11:45.980 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 00:11:45.980 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 00:11:45.980 | INFO | ------------------------------------------ | 20260506 00:11:45.980 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:11:45.980 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 00:11:45.980 | INFO | Total mem usage: 26.5 GiB | 20260506 00:11:45.980 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 00:11:45.980 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 00:11:45.980 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260506 00:11:45.980 | INFO | High Water Breakdown: | 20260506 00:11:45.980 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:11:45.980 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:11:45.980 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:11:45.980 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:11:45.980 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:11:45.980 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:11:45.980 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 00:11:45.980 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 00:11:45.980 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:11:45.980 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 00:11:45.980 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 00:11:45.980 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 00:11:45.980 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 00:11:45.980 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 00:11:45.980 | INFO | ------------------------------------------ | 20260506 00:11:45.980 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:11:45.980 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 00:11:45.980 | INFO | Total mem usage: 25.6 GiB | 20260506 00:11:45.980 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 00:11:45.980 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 00:11:45.980 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 00:11:45.980 | INFO | High Water Breakdown: | 20260506 00:11:45.980 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 00:11:45.980 | INFO | ------------------------------------------ | 20260506 00:11:45.980 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:11:45.980 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 00:11:45.980 | INFO | Total mem usage: 15.7 GiB | 20260506 00:11:45.980 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 00:11:45.980 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 00:11:45.980 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 00:11:45.980 | INFO | High Water Breakdown: | 20260506 00:11:45.980 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 00:11:45.980 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 00:11:45.980 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 00:11:45.980 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 00:11:45.980 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 00:11:45.980 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 00:11:45.980 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 00:11:45.980 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 00:11:45.980 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 00:11:45.980 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:11:45.980 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 00:11:45.980 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:11:45.980 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 00:11:45.980 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 00:11:45.980 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 00:11:45.980 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 00:11:45.980 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:11:45.980 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 00:11:45.980 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:11:45.980 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:11:45.980 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:11:45.980 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 00:11:45.980 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 00:11:45.980 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:11:45.981 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 00:11:45.981 | INFO | ------------------------------------------ | 20260506 00:11:45.981 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:11:45.981 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 00:11:45.981 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:11:45.981 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:11:45.981 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.40e+01, 1.36e+01 / 8.13e-01 | 20260506 00:11:45.981 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.04e-01, 1.02e-02 / 3.76e-03 | 20260506 00:11:45.981 | INFO | PerfReport: 11.83%, Baseline -- 3.70e+00 / 4.51e+00, 5.66e-01 / 1.50e-01 | 20260506 00:11:45.981 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.52e+00, 1.89e-01 / 8.08e-02 | 20260506 00:11:45.981 | INFO | PerfReport: 15.99%, DME -- 4.01e+01 / 4.40e+01, 3.84e+00 / 9.49e-01 | 20260506 00:11:45.981 | INFO | PerfReport: 57.35%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.48e+00 / 1.06e+00 | 20260506 00:11:45.981 | INFO | PerfReport: 7.11%, PulseAccumulating -- 2.22e+00 / 2.74e+00, 4.26e-01 / 2.40e-01 | 20260506 00:11:45.981 | INFO | PerfReport: 3.09%, Metrics -- 9.67e-01 / 1.24e+00, 2.59e-01 / 1.10e-01 | 20260506 00:11:45.981 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.35e-01 / 3.92e-01, 5.75e-02 / 3.25e-02 | 20260506 00:11:45.981 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 00:11:45.981 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:11:45.981 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:11:45.981 | INFO | PerfReport: Full Execution -- 9.83e+01 / -nan, 2.46e+01 / 0.00e+00 | 20260506 00:11:45.981 | INFO | PerfReport: 22.38%, Upload -- 2.20e+01 / 0.00e+00, 7.54e+00 / 0.00e+00 | 20260506 00:11:45.981 | INFO | PerfReport: 21.85%, Download -- 2.15e+01 / 0.00e+00, 7.60e+00 / 0.00e+00 | 20260506 00:11:45.981 | INFO | PerfReport: 31.85%, Compute -- 3.13e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 00:11:45.981 | INFO | PerfReport: 2.11%, LockUpload -- 2.08e+00 / 0.00e+00, 7.90e+00 / 0.00e+00 | 20260506 00:11:45.981 | INFO | PerfReport: 1.32%, LockDownload -- 1.30e+00 / 0.00e+00, 2.87e+00 / 0.00e+00 | 20260506 00:11:45.981 | INFO | PerfReport: 20.47%, LockCompute -- 2.01e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260506 00:11:45.981 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.81e-02 / 0.00e+00 | 20260506 00:11:46.096 | INFO | Starting a round of Baz IO | 20260506 00:11:49.249 | INFO | Populating RTMetrics report | 20260506 00:11:49.249 | INFO | RTMetrics report populated | 20260506 00:11:49.249 | INFO | Opening tmp RTMetrics json file | 20260506 00:11:49.250 | INFO | Serializing RTMetrics to json | 20260506 00:11:49.251 | INFO | Serialization done, swapping tmp file now... | 20260506 00:11:49.251 | INFO | RTMetrics done | 20260506 00:11:49.326 | INFO | Finished a round of Baz IO: Wrote 1.867 GiB in 3.23 seconds | 20260506 00:11:49.326 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.38% / 0.00% / 78.62% | 20260506 00:11:49.326 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0958443% / 0% / 8.34465e-05% | 20260506 00:11:49.326 | INFO | Baz IO overall: | 20260506 00:11:49.326 | INFO | BytesWritten : 4219009158979 | 20260506 00:11:49.326 | INFO | NumEvents : 1985988394708 | 20260506 00:11:51.122 | INFO | Analyzing chunk frames = [7413760,7414272) | 20260506 00:11:51.122 | INFO | MemInfo: { | 20260506 00:11:51.122 | INFO | "gpu" : " 17 GB", | 20260506 00:11:51.122 | INFO | "huge" : "", | 20260506 00:11:51.122 | INFO | "huge_shared" : " 27 GB", | 20260506 00:11:51.122 | INFO | "malloc" : " 14 GB", | 20260506 00:11:51.122 | INFO | "pinned" : " 29 GB" | 20260506 00:11:51.122 | INFO | } | 20260506 00:11:51.122 | INFO | Perf report for the last 5.14285 seconds: | 20260506 00:11:51.122 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:11:51.122 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:51.122 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:51.122 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:51.122 | INFO | ANALYSIS : 62.5%, 3.0, 97.7ms, 0 | 20260506 00:11:51.122 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:51.122 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260506 00:11:51.122 | INFO | PRE_HQ : 59.9%, 1.7, 54.1ms, 0 | 20260506 00:11:51.122 | INFO | BAZWRITER : 61.9%, 3.5, 112.8ms, 0 | 20260506 00:11:51.122 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:11:51.123 | INFO | Uploaded 36GiB to GPU in 2.16416s (16.6346GiB/s) | 20260506 00:11:51.123 | INFO | Downloaded 38GiB to GPU in 2.23069s (17.0351GiB/s) | 20260506 00:11:56.262 | INFO | Analyzing chunk frames = [7414272,7414784) | 20260506 00:11:56.262 | INFO | MemInfo: { | 20260506 00:11:56.262 | INFO | "gpu" : " 17 GB", | 20260506 00:11:56.262 | INFO | "huge" : "", | 20260506 00:11:56.262 | INFO | "huge_shared" : " 27 GB", | 20260506 00:11:56.262 | INFO | "malloc" : " 14 GB", | 20260506 00:11:56.262 | INFO | "pinned" : " 29 GB" | 20260506 00:11:56.262 | INFO | } | 20260506 00:11:56.262 | INFO | Perf report for the last 5.13975 seconds: | 20260506 00:11:56.262 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:11:56.262 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:56.262 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:56.262 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:56.262 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 00:11:56.262 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:56.262 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:56.262 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:11:56.262 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:56.262 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:11:56.262 | INFO | Uploaded 36GiB to GPU in 1.96744s (18.2979GiB/s) | 20260506 00:11:56.262 | INFO | Downloaded 34GiB to GPU in 1.99205s (17.0678GiB/s) | 20260506 00:12:01.397 | INFO | Analyzing chunk frames = [7414784,7415296) | 20260506 00:12:01.397 | INFO | MemInfo: { | 20260506 00:12:01.397 | INFO | "gpu" : " 17 GB", | 20260506 00:12:01.397 | INFO | "huge" : "", | 20260506 00:12:01.397 | INFO | "huge_shared" : " 27 GB", | 20260506 00:12:01.397 | INFO | "malloc" : " 14 GB", | 20260506 00:12:01.397 | INFO | "pinned" : " 29 GB" | 20260506 00:12:01.397 | INFO | } | 20260506 00:12:01.398 | INFO | Perf report for the last 5.13539 seconds: | 20260506 00:12:01.398 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:12:01.398 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:01.398 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:01.398 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:01.398 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 00:12:01.398 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:01.398 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:01.398 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260506 00:12:01.398 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:01.398 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:01.398 | INFO | Uploaded 36GiB to GPU in 2.08093s (17.3GiB/s) | 20260506 00:12:01.398 | INFO | Downloaded 34GiB to GPU in 2.02219s (16.8134GiB/s) | 20260506 00:12:06.533 | INFO | Analyzing chunk frames = [7415296,7415808) | 20260506 00:12:06.533 | INFO | MemInfo: { | 20260506 00:12:06.533 | INFO | "gpu" : " 17 GB", | 20260506 00:12:06.533 | INFO | "huge" : "", | 20260506 00:12:06.533 | INFO | "huge_shared" : " 27 GB", | 20260506 00:12:06.533 | INFO | "malloc" : " 14 GB", | 20260506 00:12:06.533 | INFO | "pinned" : " 29 GB" | 20260506 00:12:06.533 | INFO | } | 20260506 00:12:06.533 | INFO | Perf report for the last 5.13587 seconds: | 20260506 00:12:06.533 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:12:06.533 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:06.533 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:06.533 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:06.533 | INFO | ANALYSIS : 61.7%, 3.0, 95.8ms, 0 | 20260506 00:12:06.534 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:06.534 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:06.534 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 00:12:06.534 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:06.534 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:06.534 | INFO | Uploaded 36GiB to GPU in 2.12809s (16.9165GiB/s) | 20260506 00:12:06.534 | INFO | Downloaded 34GiB to GPU in 2.01687s (16.8578GiB/s) | 20260506 00:12:11.677 | INFO | Analyzing chunk frames = [7415808,7416320) | 20260506 00:12:11.677 | INFO | MemInfo: { | 20260506 00:12:11.677 | INFO | "gpu" : " 17 GB", | 20260506 00:12:11.677 | INFO | "huge" : "", | 20260506 00:12:11.677 | INFO | "huge_shared" : " 27 GB", | 20260506 00:12:11.677 | INFO | "malloc" : " 14 GB", | 20260506 00:12:11.677 | INFO | "pinned" : " 29 GB" | 20260506 00:12:11.677 | INFO | } | 20260506 00:12:11.677 | INFO | Perf report for the last 5.14376 seconds: | 20260506 00:12:11.677 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:12:11.677 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:11.677 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:11.677 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:11.677 | INFO | ANALYSIS : 61.5%, 3.0, 96.2ms, 0 | 20260506 00:12:11.677 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:11.677 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:11.677 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:12:11.677 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:11.677 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:11.678 | INFO | Uploaded 36GiB to GPU in 2.16823s (16.6034GiB/s) | 20260506 00:12:11.678 | INFO | Downloaded 34GiB to GPU in 2.07153s (16.413GiB/s) | 20260506 00:12:16.821 | INFO | Analyzing chunk frames = [7416320,7416832) | 20260506 00:12:16.821 | INFO | MemInfo: { | 20260506 00:12:16.821 | INFO | "gpu" : " 17 GB", | 20260506 00:12:16.821 | INFO | "huge" : "", | 20260506 00:12:16.821 | INFO | "huge_shared" : " 27 GB", | 20260506 00:12:16.821 | INFO | "malloc" : " 14 GB", | 20260506 00:12:16.821 | INFO | "pinned" : " 29 GB" | 20260506 00:12:16.821 | INFO | } | 20260506 00:12:16.821 | INFO | Perf report for the last 5.14399 seconds: | 20260506 00:12:16.821 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:12:16.821 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:16.821 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:16.821 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:16.821 | INFO | ANALYSIS : 61.2%, 3.0, 95.6ms, 0 | 20260506 00:12:16.821 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:16.821 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:16.821 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 00:12:16.821 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:16.821 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:16.821 | INFO | Uploaded 36GiB to GPU in 2.07242s (17.371GiB/s) | 20260506 00:12:16.821 | INFO | Downloaded 34GiB to GPU in 2.04172s (16.6526GiB/s) | 20260506 00:12:21.957 | INFO | Analyzing chunk frames = [7416832,7417344) | 20260506 00:12:21.957 | INFO | MemInfo: { | 20260506 00:12:21.957 | INFO | "gpu" : " 17 GB", | 20260506 00:12:21.957 | INFO | "huge" : "", | 20260506 00:12:21.957 | INFO | "huge_shared" : " 27 GB", | 20260506 00:12:21.957 | INFO | "malloc" : " 14 GB", | 20260506 00:12:21.957 | INFO | "pinned" : " 29 GB" | 20260506 00:12:21.957 | INFO | } | 20260506 00:12:21.958 | INFO | Perf report for the last 5.13637 seconds: | 20260506 00:12:21.958 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:12:21.958 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:21.958 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:21.958 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:21.958 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260506 00:12:21.958 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:21.958 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:21.958 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 00:12:21.958 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:21.958 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:21.958 | INFO | Uploaded 36GiB to GPU in 2.07014s (17.3902GiB/s) | 20260506 00:12:21.958 | INFO | Downloaded 34GiB to GPU in 2.06191s (16.4895GiB/s) | 20260506 00:12:27.092 | INFO | Analyzing chunk frames = [7417344,7417856) | 20260506 00:12:27.092 | INFO | MemInfo: { | 20260506 00:12:27.092 | INFO | "gpu" : " 17 GB", | 20260506 00:12:27.092 | INFO | "huge" : "", | 20260506 00:12:27.092 | INFO | "huge_shared" : " 27 GB", | 20260506 00:12:27.092 | INFO | "malloc" : " 14 GB", | 20260506 00:12:27.092 | INFO | "pinned" : " 29 GB" | 20260506 00:12:27.092 | INFO | } | 20260506 00:12:27.092 | INFO | Perf report for the last 5.13478 seconds: | 20260506 00:12:27.092 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:12:27.092 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:27.092 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:27.092 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:27.092 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260506 00:12:27.092 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:27.092 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:27.092 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 00:12:27.092 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:27.092 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:27.093 | INFO | Uploaded 36GiB to GPU in 2.13018s (16.9GiB/s) | 20260506 00:12:27.093 | INFO | Downloaded 34GiB to GPU in 2.01667s (16.8595GiB/s) | 20260506 00:12:27.215 | INFO | Starting a round of Baz IO | 20260506 00:12:30.366 | INFO | Populating RTMetrics report | 20260506 00:12:30.366 | INFO | RTMetrics report populated | 20260506 00:12:30.366 | INFO | Opening tmp RTMetrics json file | 20260506 00:12:30.368 | INFO | Serializing RTMetrics to json | 20260506 00:12:30.370 | INFO | Serialization done, swapping tmp file now... | 20260506 00:12:30.370 | INFO | RTMetrics done | 20260506 00:12:30.441 | INFO | Finished a round of Baz IO: Wrote 1.866 GiB in 3.225 seconds | 20260506 00:12:30.441 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.38% / 0.00% / 78.62% | 20260506 00:12:30.441 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0956734% / 0% / 0.000115236% | 20260506 00:12:30.441 | INFO | Baz IO overall: | 20260506 00:12:30.441 | INFO | BytesWritten : 4221012323922 | 20260506 00:12:30.441 | INFO | NumEvents : 1986904769641 | 20260506 00:12:32.232 | INFO | Analyzing chunk frames = [7417856,7418368) | 20260506 00:12:32.232 | INFO | MemInfo: { | 20260506 00:12:32.232 | INFO | "gpu" : " 17 GB", | 20260506 00:12:32.232 | INFO | "huge" : "", | 20260506 00:12:32.232 | INFO | "huge_shared" : " 27 GB", | 20260506 00:12:32.232 | INFO | "malloc" : " 14 GB", | 20260506 00:12:32.232 | INFO | "pinned" : " 29 GB" | 20260506 00:12:32.232 | INFO | } | 20260506 00:12:32.232 | INFO | Perf report for the last 5.13977 seconds: | 20260506 00:12:32.232 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:12:32.232 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:32.232 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:32.232 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:32.232 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260506 00:12:32.232 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:32.232 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260506 00:12:32.232 | INFO | PRE_HQ : 59.6%, 1.7, 53.9ms, 0 | 20260506 00:12:32.232 | INFO | BAZWRITER : 61.9%, 3.5, 112.7ms, 0 | 20260506 00:12:32.232 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:12:32.232 | INFO | Uploaded 36GiB to GPU in 2.1518s (16.7302GiB/s) | 20260506 00:12:32.232 | INFO | Downloaded 38GiB to GPU in 2.24279s (16.9432GiB/s) | 20260506 00:12:37.373 | INFO | Analyzing chunk frames = [7418368,7418880) | 20260506 00:12:37.373 | INFO | MemInfo: { | 20260506 00:12:37.373 | INFO | "gpu" : " 17 GB", | 20260506 00:12:37.373 | INFO | "huge" : "", | 20260506 00:12:37.373 | INFO | "huge_shared" : " 27 GB", | 20260506 00:12:37.373 | INFO | "malloc" : " 14 GB", | 20260506 00:12:37.373 | INFO | "pinned" : " 29 GB" | 20260506 00:12:37.373 | INFO | } | 20260506 00:12:37.373 | INFO | Perf report for the last 5.14067 seconds: | 20260506 00:12:37.373 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:12:37.373 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:37.373 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:37.373 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:37.373 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 00:12:37.373 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:37.373 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:37.373 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 00:12:37.373 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:37.373 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:37.373 | INFO | Uploaded 36GiB to GPU in 2.10947s (17.0659GiB/s) | 20260506 00:12:37.373 | INFO | Downloaded 34GiB to GPU in 2.04215s (16.6491GiB/s) | 20260506 00:12:42.515 | INFO | Analyzing chunk frames = [7418880,7419392) | 20260506 00:12:42.515 | INFO | MemInfo: { | 20260506 00:12:42.515 | INFO | "gpu" : " 17 GB", | 20260506 00:12:42.515 | INFO | "huge" : "", | 20260506 00:12:42.515 | INFO | "huge_shared" : " 27 GB", | 20260506 00:12:42.515 | INFO | "malloc" : " 14 GB", | 20260506 00:12:42.515 | INFO | "pinned" : " 29 GB" | 20260506 00:12:42.515 | INFO | } | 20260506 00:12:42.515 | INFO | Perf report for the last 5.14193 seconds: | 20260506 00:12:42.515 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:12:42.515 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:42.515 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:42.515 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:42.515 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 00:12:42.515 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:42.515 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:42.515 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:12:42.515 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:42.515 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:42.515 | INFO | Uploaded 36GiB to GPU in 2.08221s (17.2894GiB/s) | 20260506 00:12:42.515 | INFO | Downloaded 34GiB to GPU in 2.02222s (16.8132GiB/s) | 20260506 00:12:47.651 | INFO | Analyzing chunk frames = [7419392,7419904) | 20260506 00:12:47.651 | INFO | MemInfo: { | 20260506 00:12:47.651 | INFO | "gpu" : " 17 GB", | 20260506 00:12:47.651 | INFO | "huge" : "", | 20260506 00:12:47.651 | INFO | "huge_shared" : " 27 GB", | 20260506 00:12:47.651 | INFO | "malloc" : " 14 GB", | 20260506 00:12:47.651 | INFO | "pinned" : " 29 GB" | 20260506 00:12:47.651 | INFO | } | 20260506 00:12:47.651 | INFO | Perf report for the last 5.13623 seconds: | 20260506 00:12:47.651 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:12:47.651 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:47.651 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:47.651 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:47.651 | INFO | ANALYSIS : 61.6%, 3.0, 95.6ms, 0 | 20260506 00:12:47.651 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:47.651 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:47.651 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 00:12:47.651 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:47.651 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:47.651 | INFO | Uploaded 36GiB to GPU in 2.12985s (16.9026GiB/s) | 20260506 00:12:47.651 | INFO | Downloaded 34GiB to GPU in 2.0134s (16.8868GiB/s) | 20260506 00:12:52.795 | INFO | Analyzing chunk frames = [7419904,7420416) | 20260506 00:12:52.795 | INFO | MemInfo: { | 20260506 00:12:52.795 | INFO | "gpu" : " 17 GB", | 20260506 00:12:52.795 | INFO | "huge" : "", | 20260506 00:12:52.795 | INFO | "huge_shared" : " 27 GB", | 20260506 00:12:52.795 | INFO | "malloc" : " 14 GB", | 20260506 00:12:52.795 | INFO | "pinned" : " 29 GB" | 20260506 00:12:52.795 | INFO | } | 20260506 00:12:52.795 | INFO | Perf report for the last 5.14443 seconds: | 20260506 00:12:52.795 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:12:52.795 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:52.795 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:52.795 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:52.795 | INFO | ANALYSIS : 61.2%, 3.0, 95.6ms, 0 | 20260506 00:12:52.795 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:52.795 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:52.795 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:12:52.796 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:52.796 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:52.796 | INFO | Uploaded 36GiB to GPU in 2.10581s (17.0956GiB/s) | 20260506 00:12:52.796 | INFO | Downloaded 34GiB to GPU in 2.04915s (16.5922GiB/s) | 20260506 00:12:57.936 | INFO | Analyzing chunk frames = [7420416,7420928) | 20260506 00:12:57.936 | INFO | MemInfo: { | 20260506 00:12:57.936 | INFO | "gpu" : " 17 GB", | 20260506 00:12:57.936 | INFO | "huge" : "", | 20260506 00:12:57.936 | INFO | "huge_shared" : " 27 GB", | 20260506 00:12:57.936 | INFO | "malloc" : " 14 GB", | 20260506 00:12:57.936 | INFO | "pinned" : " 29 GB" | 20260506 00:12:57.936 | INFO | } | 20260506 00:12:57.936 | INFO | Perf report for the last 5.1403 seconds: | 20260506 00:12:57.936 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:12:57.936 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:57.936 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:57.936 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:57.936 | INFO | ANALYSIS : 61.3%, 3.0, 95.8ms, 0 | 20260506 00:12:57.936 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:57.936 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:57.936 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 00:12:57.936 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:57.936 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:12:57.936 | INFO | Uploaded 36GiB to GPU in 2.07893s (17.3166GiB/s) | 20260506 00:12:57.936 | INFO | Downloaded 34GiB to GPU in 2.044s (16.6341GiB/s) | 20260506 00:13:03.076 | INFO | Analyzing chunk frames = [7420928,7421440) | 20260506 00:13:03.076 | INFO | MemInfo: { | 20260506 00:13:03.076 | INFO | "gpu" : " 17 GB", | 20260506 00:13:03.076 | INFO | "huge" : "", | 20260506 00:13:03.076 | INFO | "huge_shared" : " 27 GB", | 20260506 00:13:03.077 | INFO | "malloc" : " 14 GB", | 20260506 00:13:03.077 | INFO | "pinned" : " 29 GB" | 20260506 00:13:03.077 | INFO | } | 20260506 00:13:03.077 | INFO | Perf report for the last 5.14079 seconds: | 20260506 00:13:03.077 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:13:03.077 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:03.077 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:03.077 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:03.077 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260506 00:13:03.077 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:03.077 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:03.077 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 00:13:03.077 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:03.077 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:03.077 | INFO | Uploaded 36GiB to GPU in 2.05957s (17.4794GiB/s) | 20260506 00:13:03.077 | INFO | Downloaded 34GiB to GPU in 2.07867s (16.3566GiB/s) | 20260506 00:13:08.211 | INFO | Analyzing chunk frames = [7421440,7421952) | 20260506 00:13:08.211 | INFO | MemInfo: { | 20260506 00:13:08.211 | INFO | "gpu" : " 17 GB", | 20260506 00:13:08.211 | INFO | "huge" : "", | 20260506 00:13:08.211 | INFO | "huge_shared" : " 27 GB", | 20260506 00:13:08.211 | INFO | "malloc" : " 14 GB", | 20260506 00:13:08.211 | INFO | "pinned" : " 29 GB" | 20260506 00:13:08.211 | INFO | } | 20260506 00:13:08.211 | INFO | Perf report for the last 5.13489 seconds: | 20260506 00:13:08.211 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:13:08.211 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:08.211 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:08.211 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:08.211 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 00:13:08.212 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:08.212 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:08.212 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 00:13:08.212 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:08.212 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:08.212 | INFO | Uploaded 36GiB to GPU in 2.12491s (16.9419GiB/s) | 20260506 00:13:08.212 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 00:13:08.212 | INFO | Total mem usage: 15.3 GiB | 20260506 00:13:08.212 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 00:13:08.212 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.12%) | 20260506 00:13:08.212 | INFO | Downloaded 34GiB to GPU in 2.01782s (16.8499GiB/s) | 20260506 00:13:08.212 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 00:13:08.212 | INFO | High Water Breakdown: | 20260506 00:13:08.212 | INFO | 60.9% : 8.55 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 00:13:08.212 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 00:13:08.212 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 00:13:08.212 | INFO | ------------------------------------------ | 20260506 00:13:08.212 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:13:08.212 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 00:13:08.212 | INFO | Total mem usage: 26.5 GiB | 20260506 00:13:08.212 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 00:13:08.212 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 00:13:08.212 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 00:13:08.212 | INFO | High Water Breakdown: | 20260506 00:13:08.212 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:13:08.212 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:13:08.212 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:13:08.212 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:13:08.212 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:13:08.212 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:13:08.212 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 00:13:08.212 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 00:13:08.212 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:13:08.212 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 00:13:08.212 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 00:13:08.212 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 00:13:08.212 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 00:13:08.212 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 00:13:08.212 | INFO | ------------------------------------------ | 20260506 00:13:08.212 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:13:08.212 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 00:13:08.212 | INFO | Total mem usage: 25.6 GiB | 20260506 00:13:08.212 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 00:13:08.212 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 00:13:08.212 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 00:13:08.212 | INFO | High Water Breakdown: | 20260506 00:13:08.212 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 00:13:08.212 | INFO | ------------------------------------------ | 20260506 00:13:08.212 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:13:08.212 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 00:13:08.212 | INFO | Total mem usage: 15.7 GiB | 20260506 00:13:08.212 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 00:13:08.212 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 00:13:08.212 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 00:13:08.212 | INFO | High Water Breakdown: | 20260506 00:13:08.212 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 00:13:08.212 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 00:13:08.212 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 00:13:08.212 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 00:13:08.212 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 00:13:08.212 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 00:13:08.212 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 00:13:08.212 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 00:13:08.212 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 00:13:08.212 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:13:08.212 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 00:13:08.212 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:13:08.212 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 00:13:08.212 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 00:13:08.212 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 00:13:08.212 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 00:13:08.212 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:13:08.212 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 00:13:08.212 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:13:08.212 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:13:08.212 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:13:08.212 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 00:13:08.212 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 00:13:08.212 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:13:08.212 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 00:13:08.213 | INFO | ------------------------------------------ | 20260506 00:13:08.213 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:13:08.213 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 00:13:08.213 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:13:08.213 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:13:08.213 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.39e+01, 1.34e+01 / 9.57e-01 | 20260506 00:13:08.213 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.03e-01, 1.07e-02 / 3.62e-03 | 20260506 00:13:08.213 | INFO | PerfReport: 11.82%, Baseline -- 3.70e+00 / 4.51e+00, 5.67e-01 / 1.77e-01 | 20260506 00:13:08.213 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.51e+00, 1.91e-01 / 8.83e-02 | 20260506 00:13:08.213 | INFO | PerfReport: 15.98%, DME -- 4.01e+01 / 4.40e+01, 3.81e+00 / 1.00e+00 | 20260506 00:13:08.213 | INFO | PerfReport: 57.42%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.56e+00 / 1.15e+00 | 20260506 00:13:08.213 | INFO | PerfReport: 7.07%, PulseAccumulating -- 2.21e+00 / 2.74e+00, 4.19e-01 / 2.32e-01 | 20260506 00:13:08.213 | INFO | PerfReport: 3.08%, Metrics -- 9.64e-01 / 1.24e+00, 2.60e-01 / 1.22e-01 | 20260506 00:13:08.213 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.34e-01 / 3.85e-01, 5.69e-02 / 2.67e-02 | 20260506 00:13:08.213 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 00:13:08.213 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:13:08.213 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:13:08.213 | INFO | PerfReport: Full Execution -- 9.83e+01 / -nan, 2.46e+01 / 0.00e+00 | 20260506 00:13:08.213 | INFO | PerfReport: 22.27%, Upload -- 2.19e+01 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260506 00:13:08.213 | INFO | PerfReport: 21.84%, Download -- 2.15e+01 / 0.00e+00, 7.60e+00 / 0.00e+00 | 20260506 00:13:08.213 | INFO | PerfReport: 31.89%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 00:13:08.213 | INFO | PerfReport: 2.11%, LockUpload -- 2.08e+00 / 0.00e+00, 7.77e+00 / 0.00e+00 | 20260506 00:13:08.213 | INFO | PerfReport: 1.31%, LockDownload -- 1.28e+00 / 0.00e+00, 2.89e+00 / 0.00e+00 | 20260506 00:13:08.213 | INFO | PerfReport: 20.56%, LockCompute -- 2.02e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260506 00:13:08.213 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.03e-02 / 0.00e+00 | 20260506 00:13:08.329 | INFO | Starting a round of Baz IO | 20260506 00:13:11.475 | INFO | Populating RTMetrics report | 20260506 00:13:11.476 | INFO | RTMetrics report populated | 20260506 00:13:11.476 | INFO | Opening tmp RTMetrics json file | 20260506 00:13:11.479 | INFO | Serializing RTMetrics to json | 20260506 00:13:11.480 | INFO | Serialization done, swapping tmp file now... | 20260506 00:13:11.481 | INFO | RTMetrics done | 20260506 00:13:11.556 | INFO | Finished a round of Baz IO: Wrote 1.865 GiB in 3.227 seconds | 20260506 00:13:11.556 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.38% / 0.00% / 78.62% | 20260506 00:13:11.556 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0957211% / 0% / 8.74201e-05% | 20260506 00:13:11.556 | INFO | Baz IO overall: | 20260506 00:13:11.556 | INFO | BytesWritten : 4223015176870 | 20260506 00:13:11.556 | INFO | NumEvents : 1987821236452 | 20260506 00:13:13.342 | INFO | Analyzing chunk frames = [7421952,7422464) | 20260506 00:13:13.342 | INFO | MemInfo: { | 20260506 00:13:13.342 | INFO | "gpu" : " 17 GB", | 20260506 00:13:13.342 | INFO | "huge" : "", | 20260506 00:13:13.342 | INFO | "huge_shared" : " 27 GB", | 20260506 00:13:13.342 | INFO | "malloc" : " 14 GB", | 20260506 00:13:13.342 | INFO | "pinned" : " 29 GB" | 20260506 00:13:13.342 | INFO | } | 20260506 00:13:13.342 | INFO | Perf report for the last 5.13089 seconds: | 20260506 00:13:13.342 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:13:13.342 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:13.342 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:13.342 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:13.342 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 00:13:13.342 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:13.342 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260506 00:13:13.342 | INFO | PRE_HQ : 59.9%, 1.7, 54.0ms, 0 | 20260506 00:13:13.342 | INFO | BAZWRITER : 62.0%, 3.5, 112.8ms, 0 | 20260506 00:13:13.342 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:13:13.343 | INFO | Uploaded 36GiB to GPU in 2.16665s (16.6155GiB/s) | 20260506 00:13:13.343 | INFO | Downloaded 38GiB to GPU in 2.23425s (17.008GiB/s) | 20260506 00:13:18.489 | INFO | Analyzing chunk frames = [7422464,7422976) | 20260506 00:13:18.489 | INFO | MemInfo: { | 20260506 00:13:18.489 | INFO | "gpu" : " 17 GB", | 20260506 00:13:18.489 | INFO | "huge" : "", | 20260506 00:13:18.489 | INFO | "huge_shared" : " 27 GB", | 20260506 00:13:18.489 | INFO | "malloc" : " 14 GB", | 20260506 00:13:18.489 | INFO | "pinned" : " 29 GB" | 20260506 00:13:18.489 | INFO | } | 20260506 00:13:18.489 | INFO | Perf report for the last 5.1463 seconds: | 20260506 00:13:18.489 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:13:18.489 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:18.489 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:18.489 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:18.489 | INFO | ANALYSIS : 61.8%, 3.0, 96.6ms, 0 | 20260506 00:13:18.489 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:18.489 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:18.489 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:13:18.489 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:18.489 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:18.489 | INFO | Uploaded 36GiB to GPU in 2.08487s (17.2673GiB/s) | 20260506 00:13:18.489 | INFO | Downloaded 34GiB to GPU in 2.02929s (16.7546GiB/s) | 20260506 00:13:23.634 | INFO | Analyzing chunk frames = [7422976,7423488) | 20260506 00:13:23.634 | INFO | MemInfo: { | 20260506 00:13:23.634 | INFO | "gpu" : " 17 GB", | 20260506 00:13:23.634 | INFO | "huge" : "", | 20260506 00:13:23.634 | INFO | "huge_shared" : " 27 GB", | 20260506 00:13:23.634 | INFO | "malloc" : " 14 GB", | 20260506 00:13:23.634 | INFO | "pinned" : " 29 GB" | 20260506 00:13:23.634 | INFO | } | 20260506 00:13:23.634 | INFO | Perf report for the last 5.14582 seconds: | 20260506 00:13:23.634 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:13:23.634 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:23.635 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:23.635 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:23.635 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 00:13:23.635 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:23.635 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:23.635 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:13:23.635 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:23.635 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:23.635 | INFO | Uploaded 36GiB to GPU in 2.09047s (17.221GiB/s) | 20260506 00:13:23.635 | INFO | Downloaded 34GiB to GPU in 2.01721s (16.8549GiB/s) | 20260506 00:13:28.770 | INFO | Analyzing chunk frames = [7423488,7424000) | 20260506 00:13:28.770 | INFO | MemInfo: { | 20260506 00:13:28.770 | INFO | "gpu" : " 17 GB", | 20260506 00:13:28.770 | INFO | "huge" : "", | 20260506 00:13:28.770 | INFO | "huge_shared" : " 27 GB", | 20260506 00:13:28.770 | INFO | "malloc" : " 14 GB", | 20260506 00:13:28.770 | INFO | "pinned" : " 29 GB" | 20260506 00:13:28.770 | INFO | } | 20260506 00:13:28.770 | INFO | Perf report for the last 5.13551 seconds: | 20260506 00:13:28.770 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:13:28.770 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:28.770 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:28.770 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:28.770 | INFO | ANALYSIS : 61.6%, 3.0, 95.7ms, 0 | 20260506 00:13:28.770 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:28.770 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:28.770 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:13:28.770 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:28.770 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:28.770 | INFO | Uploaded 36GiB to GPU in 2.14486s (16.7843GiB/s) | 20260506 00:13:28.770 | INFO | Downloaded 34GiB to GPU in 2.04802s (16.6014GiB/s) | 20260506 00:13:33.905 | INFO | Analyzing chunk frames = [7424000,7424512) | 20260506 00:13:33.905 | INFO | MemInfo: { | 20260506 00:13:33.905 | INFO | "gpu" : " 17 GB", | 20260506 00:13:33.905 | INFO | "huge" : "", | 20260506 00:13:33.905 | INFO | "huge_shared" : " 27 GB", | 20260506 00:13:33.906 | INFO | "malloc" : " 14 GB", | 20260506 00:13:33.906 | INFO | "pinned" : " 29 GB" | 20260506 00:13:33.906 | INFO | } | 20260506 00:13:33.906 | INFO | Perf report for the last 5.13555 seconds: | 20260506 00:13:33.906 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:13:33.906 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:33.906 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:33.906 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:33.906 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 00:13:33.906 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:33.906 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:33.906 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 00:13:33.906 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:33.906 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:33.906 | INFO | Uploaded 36GiB to GPU in 2.10796s (17.0781GiB/s) | 20260506 00:13:33.906 | INFO | Downloaded 34GiB to GPU in 2.05051s (16.5812GiB/s) | 20260506 00:13:39.043 | INFO | Analyzing chunk frames = [7424512,7425024) | 20260506 00:13:39.043 | INFO | MemInfo: { | 20260506 00:13:39.043 | INFO | "gpu" : " 17 GB", | 20260506 00:13:39.044 | INFO | "huge" : "", | 20260506 00:13:39.044 | INFO | "huge_shared" : " 27 GB", | 20260506 00:13:39.044 | INFO | "malloc" : " 14 GB", | 20260506 00:13:39.044 | INFO | "pinned" : " 29 GB" | 20260506 00:13:39.044 | INFO | } | 20260506 00:13:39.044 | INFO | Perf report for the last 5.13805 seconds: | 20260506 00:13:39.044 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:13:39.044 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:39.044 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:39.044 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:39.044 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260506 00:13:39.044 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:39.044 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:39.044 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:13:39.044 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:39.044 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:39.044 | INFO | Uploaded 36GiB to GPU in 2.06833s (17.4054GiB/s) | 20260506 00:13:39.044 | INFO | Downloaded 34GiB to GPU in 2.03483s (16.709GiB/s) | 20260506 00:13:44.189 | INFO | Analyzing chunk frames = [7425024,7425536) | 20260506 00:13:44.189 | INFO | MemInfo: { | 20260506 00:13:44.189 | INFO | "gpu" : " 17 GB", | 20260506 00:13:44.189 | INFO | "huge" : "", | 20260506 00:13:44.189 | INFO | "huge_shared" : " 27 GB", | 20260506 00:13:44.189 | INFO | "malloc" : " 14 GB", | 20260506 00:13:44.189 | INFO | "pinned" : " 29 GB" | 20260506 00:13:44.189 | INFO | } | 20260506 00:13:44.189 | INFO | Perf report for the last 5.14567 seconds: | 20260506 00:13:44.189 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:13:44.189 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:44.189 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:44.189 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:44.189 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260506 00:13:44.189 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:44.189 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:44.189 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 00:13:44.189 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:44.189 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:44.189 | INFO | Uploaded 36GiB to GPU in 2.03925s (17.6536GiB/s) | 20260506 00:13:44.190 | INFO | Downloaded 34GiB to GPU in 2.04789s (16.6024GiB/s) | 20260506 00:13:49.323 | INFO | Analyzing chunk frames = [7425536,7426048) | 20260506 00:13:49.323 | INFO | MemInfo: { | 20260506 00:13:49.323 | INFO | "gpu" : " 17 GB", | 20260506 00:13:49.323 | INFO | "huge" : "", | 20260506 00:13:49.323 | INFO | "huge_shared" : " 27 GB", | 20260506 00:13:49.323 | INFO | "malloc" : " 14 GB", | 20260506 00:13:49.323 | INFO | "pinned" : " 29 GB" | 20260506 00:13:49.323 | INFO | } | 20260506 00:13:49.323 | INFO | Perf report for the last 5.13358 seconds: | 20260506 00:13:49.323 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:13:49.323 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:49.323 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:49.323 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:49.323 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260506 00:13:49.323 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:49.323 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:49.323 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 00:13:49.323 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:49.323 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:49.323 | INFO | Uploaded 36GiB to GPU in 2.095s (17.1838GiB/s) | 20260506 00:13:49.323 | INFO | Downloaded 34GiB to GPU in 2.03249s (16.7282GiB/s) | 20260506 00:13:49.440 | INFO | Starting a round of Baz IO | 20260506 00:13:52.587 | INFO | Populating RTMetrics report | 20260506 00:13:52.587 | INFO | RTMetrics report populated | 20260506 00:13:52.587 | INFO | Opening tmp RTMetrics json file | 20260506 00:13:52.588 | INFO | Serializing RTMetrics to json | 20260506 00:13:52.590 | INFO | Serialization done, swapping tmp file now... | 20260506 00:13:52.590 | INFO | RTMetrics done | 20260506 00:13:52.667 | INFO | Finished a round of Baz IO: Wrote 1.865 GiB in 3.226 seconds | 20260506 00:13:52.667 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.38% / 0.00% / 78.62% | 20260506 00:13:52.667 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0957529% / 3.97364e-06% / 6.35783e-05% | 20260506 00:13:52.667 | INFO | Baz IO overall: | 20260506 00:13:52.667 | INFO | BytesWritten : 4225017472998 | 20260506 00:13:52.667 | INFO | NumEvents : 1988737440653 | 20260506 00:13:54.463 | INFO | Analyzing chunk frames = [7426048,7426560) | 20260506 00:13:54.463 | INFO | MemInfo: { | 20260506 00:13:54.463 | INFO | "gpu" : " 17 GB", | 20260506 00:13:54.463 | INFO | "huge" : "", | 20260506 00:13:54.463 | INFO | "huge_shared" : " 27 GB", | 20260506 00:13:54.463 | INFO | "malloc" : " 14 GB", | 20260506 00:13:54.463 | INFO | "pinned" : " 29 GB" | 20260506 00:13:54.463 | INFO | } | 20260506 00:13:54.463 | INFO | Perf report for the last 5.14036 seconds: | 20260506 00:13:54.463 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:13:54.463 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:54.463 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:54.463 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:54.463 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260506 00:13:54.463 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:54.463 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260506 00:13:54.463 | INFO | PRE_HQ : 59.7%, 1.7, 53.8ms, 0 | 20260506 00:13:54.463 | INFO | BAZWRITER : 61.9%, 3.5, 112.2ms, 0 | 20260506 00:13:54.463 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:13:54.464 | INFO | Uploaded 36GiB to GPU in 2.16503s (16.6279GiB/s) | 20260506 00:13:54.464 | INFO | Downloaded 38GiB to GPU in 2.23935s (16.9692GiB/s) | 20260506 00:13:59.602 | INFO | Analyzing chunk frames = [7426560,7427072) | 20260506 00:13:59.602 | INFO | MemInfo: { | 20260506 00:13:59.602 | INFO | "gpu" : " 17 GB", | 20260506 00:13:59.602 | INFO | "huge" : "", | 20260506 00:13:59.602 | INFO | "huge_shared" : " 27 GB", | 20260506 00:13:59.602 | INFO | "malloc" : " 14 GB", | 20260506 00:13:59.602 | INFO | "pinned" : " 29 GB" | 20260506 00:13:59.603 | INFO | } | 20260506 00:13:59.603 | INFO | Perf report for the last 5.13924 seconds: | 20260506 00:13:59.603 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:13:59.603 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:59.603 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:59.603 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:59.603 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 00:13:59.603 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:59.603 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:59.603 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:13:59.603 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:59.603 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:13:59.603 | INFO | Uploaded 36GiB to GPU in 2.09498s (17.184GiB/s) | 20260506 00:13:59.603 | INFO | Downloaded 34GiB to GPU in 2.03558s (16.7029GiB/s) | 20260506 00:14:04.744 | INFO | Analyzing chunk frames = [7427072,7427584) | 20260506 00:14:04.744 | INFO | MemInfo: { | 20260506 00:14:04.744 | INFO | "gpu" : " 17 GB", | 20260506 00:14:04.744 | INFO | "huge" : "", | 20260506 00:14:04.744 | INFO | "huge_shared" : " 27 GB", | 20260506 00:14:04.744 | INFO | "malloc" : " 14 GB", | 20260506 00:14:04.744 | INFO | "pinned" : " 29 GB" | 20260506 00:14:04.744 | INFO | } | 20260506 00:14:04.744 | INFO | Perf report for the last 5.1414 seconds: | 20260506 00:14:04.744 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:14:04.744 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:04.744 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:04.744 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:04.744 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 00:14:04.744 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:04.744 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:04.744 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:14:04.744 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:04.744 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:04.744 | INFO | Uploaded 36GiB to GPU in 2.08564s (17.2609GiB/s) | 20260506 00:14:04.744 | INFO | Downloaded 34GiB to GPU in 2.0339s (16.7167GiB/s) | 20260506 00:14:09.883 | INFO | Analyzing chunk frames = [7427584,7428096) | 20260506 00:14:09.883 | INFO | MemInfo: { | 20260506 00:14:09.883 | INFO | "gpu" : " 17 GB", | 20260506 00:14:09.883 | INFO | "huge" : "", | 20260506 00:14:09.883 | INFO | "huge_shared" : " 27 GB", | 20260506 00:14:09.883 | INFO | "malloc" : " 14 GB", | 20260506 00:14:09.883 | INFO | "pinned" : " 29 GB" | 20260506 00:14:09.883 | INFO | } | 20260506 00:14:09.883 | INFO | Perf report for the last 5.13918 seconds: | 20260506 00:14:09.883 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:14:09.883 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:09.883 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:09.883 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:09.883 | INFO | ANALYSIS : 61.6%, 3.0, 95.7ms, 0 | 20260506 00:14:09.883 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:09.883 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:09.883 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 00:14:09.883 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:09.883 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:09.883 | INFO | Uploaded 36GiB to GPU in 2.13382s (16.8712GiB/s) | 20260506 00:14:09.883 | INFO | Downloaded 34GiB to GPU in 2.0236s (16.8018GiB/s) | 20260506 00:14:15.025 | INFO | Analyzing chunk frames = [7428096,7428608) | 20260506 00:14:15.025 | INFO | MemInfo: { | 20260506 00:14:15.025 | INFO | "gpu" : " 17 GB", | 20260506 00:14:15.025 | INFO | "huge" : "", | 20260506 00:14:15.025 | INFO | "huge_shared" : " 27 GB", | 20260506 00:14:15.025 | INFO | "malloc" : " 14 GB", | 20260506 00:14:15.025 | INFO | "pinned" : " 29 GB" | 20260506 00:14:15.025 | INFO | } | 20260506 00:14:15.025 | INFO | Perf report for the last 5.14186 seconds: | 20260506 00:14:15.025 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:14:15.025 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:15.025 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:15.025 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:15.025 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 00:14:15.025 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:15.025 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:15.025 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:14:15.025 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:15.025 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:15.025 | INFO | Uploaded 36GiB to GPU in 2.09926s (17.1489GiB/s) | 20260506 00:14:15.025 | INFO | Downloaded 34GiB to GPU in 2.04475s (16.6279GiB/s) | 20260506 00:14:20.165 | INFO | Analyzing chunk frames = [7428608,7429120) | 20260506 00:14:20.165 | INFO | MemInfo: { | 20260506 00:14:20.165 | INFO | "gpu" : " 17 GB", | 20260506 00:14:20.165 | INFO | "huge" : "", | 20260506 00:14:20.165 | INFO | "huge_shared" : " 27 GB", | 20260506 00:14:20.165 | INFO | "malloc" : " 14 GB", | 20260506 00:14:20.165 | INFO | "pinned" : " 29 GB" | 20260506 00:14:20.165 | INFO | } | 20260506 00:14:20.165 | INFO | Perf report for the last 5.14038 seconds: | 20260506 00:14:20.165 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:14:20.165 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:20.165 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:20.165 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:20.165 | INFO | ANALYSIS : 61.2%, 3.0, 95.6ms, 0 | 20260506 00:14:20.165 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:20.165 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:20.165 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:14:20.165 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:20.166 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:20.166 | INFO | Uploaded 36GiB to GPU in 2.07364s (17.3608GiB/s) | 20260506 00:14:20.166 | INFO | Downloaded 34GiB to GPU in 2.04301s (16.6421GiB/s) | 20260506 00:14:25.294 | INFO | Analyzing chunk frames = [7429120,7429632) | 20260506 00:14:25.294 | INFO | MemInfo: { | 20260506 00:14:25.294 | INFO | "gpu" : " 17 GB", | 20260506 00:14:25.294 | INFO | "huge" : "", | 20260506 00:14:25.294 | INFO | "huge_shared" : " 27 GB", | 20260506 00:14:25.294 | INFO | "malloc" : " 14 GB", | 20260506 00:14:25.294 | INFO | "pinned" : " 29 GB" | 20260506 00:14:25.294 | INFO | } | 20260506 00:14:25.294 | INFO | Perf report for the last 5.12903 seconds: | 20260506 00:14:25.294 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:14:25.294 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:25.294 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:25.294 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:25.295 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260506 00:14:25.295 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:25.295 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:25.295 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 00:14:25.295 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:25.295 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:25.295 | INFO | Uploaded 36GiB to GPU in 2.06909s (17.399GiB/s) | 20260506 00:14:25.295 | INFO | Downloaded 34GiB to GPU in 2.06352s (16.4767GiB/s) | 20260506 00:14:30.440 | INFO | Analyzing chunk frames = [7429632,7430144) | 20260506 00:14:30.440 | INFO | MemInfo: { | 20260506 00:14:30.440 | INFO | "gpu" : " 17 GB", | 20260506 00:14:30.440 | INFO | "huge" : "", | 20260506 00:14:30.440 | INFO | "huge_shared" : " 27 GB", | 20260506 00:14:30.440 | INFO | "malloc" : " 14 GB", | 20260506 00:14:30.440 | INFO | "pinned" : " 29 GB" | 20260506 00:14:30.440 | INFO | } | 20260506 00:14:30.440 | INFO | Perf report for the last 5.14568 seconds: | 20260506 00:14:30.440 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:14:30.440 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:30.440 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:30.440 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:30.440 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260506 00:14:30.440 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:30.440 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:30.440 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:14:30.440 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:30.440 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:30.440 | INFO | Uploaded 36GiB to GPU in 2.11822s (16.9954GiB/s) | 20260506 00:14:30.440 | INFO | Downloaded 34GiB to GPU in 2.00751s (16.9364GiB/s) | 20260506 00:14:30.441 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 00:14:30.441 | INFO | Total mem usage: 15.3 GiB | 20260506 00:14:30.441 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 00:14:30.441 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.15%) | 20260506 00:14:30.441 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 00:14:30.441 | INFO | High Water Breakdown: | 20260506 00:14:30.441 | INFO | 60.9% : 8.55 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 00:14:30.441 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 00:14:30.441 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 00:14:30.441 | INFO | ------------------------------------------ | 20260506 00:14:30.441 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:14:30.441 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 00:14:30.441 | INFO | Total mem usage: 26.5 GiB | 20260506 00:14:30.441 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 00:14:30.441 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 00:14:30.441 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260506 00:14:30.441 | INFO | High Water Breakdown: | 20260506 00:14:30.441 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:14:30.441 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:14:30.441 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:14:30.441 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:14:30.441 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:14:30.441 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:14:30.441 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 00:14:30.441 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 00:14:30.441 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:14:30.441 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 00:14:30.441 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 00:14:30.441 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 00:14:30.441 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 00:14:30.441 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 00:14:30.441 | INFO | ------------------------------------------ | 20260506 00:14:30.441 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:14:30.441 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 00:14:30.441 | INFO | Total mem usage: 25.6 GiB | 20260506 00:14:30.441 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 00:14:30.441 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 00:14:30.441 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 00:14:30.441 | INFO | High Water Breakdown: | 20260506 00:14:30.441 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 00:14:30.441 | INFO | ------------------------------------------ | 20260506 00:14:30.441 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:14:30.441 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 00:14:30.441 | INFO | Total mem usage: 15.7 GiB | 20260506 00:14:30.441 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 00:14:30.441 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 00:14:30.441 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 00:14:30.441 | INFO | High Water Breakdown: | 20260506 00:14:30.441 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 00:14:30.441 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 00:14:30.441 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 00:14:30.441 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 00:14:30.441 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 00:14:30.441 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 00:14:30.441 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 00:14:30.441 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 00:14:30.441 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 00:14:30.441 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:14:30.441 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 00:14:30.441 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:14:30.441 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 00:14:30.441 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 00:14:30.441 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 00:14:30.441 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 00:14:30.441 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:14:30.441 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 00:14:30.441 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:14:30.441 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:14:30.441 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:14:30.441 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 00:14:30.441 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 00:14:30.441 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:14:30.441 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 00:14:30.441 | INFO | ------------------------------------------ | 20260506 00:14:30.441 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:14:30.441 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 00:14:30.441 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:14:30.441 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:14:30.441 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.42e+01, 1.35e+01 / 9.52e-01 | 20260506 00:14:30.441 | INFO | PerfReport: 0.29%, AggregateStats -- 9.17e-02 / 1.04e-01, 1.03e-02 / 3.46e-03 | 20260506 00:14:30.441 | INFO | PerfReport: 11.83%, Baseline -- 3.71e+00 / 4.52e+00, 5.75e-01 / 2.05e-01 | 20260506 00:14:30.441 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.52e+00, 1.91e-01 / 1.05e-01 | 20260506 00:14:30.441 | INFO | PerfReport: 15.99%, DME -- 4.01e+01 / 4.41e+01, 3.82e+00 / 1.10e+00 | 20260506 00:14:30.441 | INFO | PerfReport: 57.38%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.53e+00 / 1.23e+00 | 20260506 00:14:30.441 | INFO | PerfReport: 7.08%, PulseAccumulating -- 2.22e+00 / 2.75e+00, 4.21e-01 / 2.29e-01 | 20260506 00:14:30.441 | INFO | PerfReport: 3.08%, Metrics -- 9.66e-01 / 1.24e+00, 2.61e-01 / 1.15e-01 | 20260506 00:14:30.441 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.34e-01 / 3.90e-01, 5.73e-02 / 3.26e-02 | 20260506 00:14:30.441 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 00:14:30.441 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:14:30.441 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:14:30.441 | INFO | PerfReport: Full Execution -- 9.83e+01 / -nan, 2.46e+01 / 0.00e+00 | 20260506 00:14:30.441 | INFO | PerfReport: 22.28%, Upload -- 2.19e+01 / 0.00e+00, 7.50e+00 / 0.00e+00 | 20260506 00:14:30.441 | INFO | PerfReport: 21.85%, Download -- 2.15e+01 / 0.00e+00, 7.57e+00 / 0.00e+00 | 20260506 00:14:30.442 | INFO | PerfReport: 31.91%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 00:14:30.442 | INFO | PerfReport: 2.09%, LockUpload -- 2.05e+00 / 0.00e+00, 7.78e+00 / 0.00e+00 | 20260506 00:14:30.442 | INFO | PerfReport: 1.27%, LockDownload -- 1.25e+00 / 0.00e+00, 2.84e+00 / 0.00e+00 | 20260506 00:14:30.442 | INFO | PerfReport: 20.58%, LockCompute -- 2.02e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 00:14:30.442 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.30e-02 / 0.00e+00 | 20260506 00:14:30.557 | INFO | Starting a round of Baz IO | 20260506 00:14:33.704 | INFO | Populating RTMetrics report | 20260506 00:14:33.704 | INFO | RTMetrics report populated | 20260506 00:14:33.704 | INFO | Opening tmp RTMetrics json file | 20260506 00:14:33.705 | INFO | Serializing RTMetrics to json | 20260506 00:14:33.706 | INFO | Serialization done, swapping tmp file now... | 20260506 00:14:33.706 | INFO | RTMetrics done | 20260506 00:14:33.780 | INFO | Finished a round of Baz IO: Wrote 1.865 GiB in 3.223 seconds | 20260506 00:14:33.780 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.38% / 0.00% / 78.62% | 20260506 00:14:33.781 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0955343% / 0% / 8.34465e-05% | 20260506 00:14:33.781 | INFO | Baz IO overall: | 20260506 00:14:33.781 | INFO | BytesWritten : 4227020464163 | 20260506 00:14:33.781 | INFO | NumEvents : 1989653844481 | 20260506 00:14:35.587 | INFO | Analyzing chunk frames = [7430144,7430656) | 20260506 00:14:35.587 | INFO | MemInfo: { | 20260506 00:14:35.587 | INFO | "gpu" : " 17 GB", | 20260506 00:14:35.587 | INFO | "huge" : "", | 20260506 00:14:35.587 | INFO | "huge_shared" : " 27 GB", | 20260506 00:14:35.587 | INFO | "malloc" : " 14 GB", | 20260506 00:14:35.587 | INFO | "pinned" : " 29 GB" | 20260506 00:14:35.587 | INFO | } | 20260506 00:14:35.587 | INFO | Perf report for the last 5.14712 seconds: | 20260506 00:14:35.587 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:14:35.587 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:35.587 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:35.587 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:35.587 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260506 00:14:35.588 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:35.588 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260506 00:14:35.588 | INFO | PRE_HQ : 59.6%, 1.7, 53.9ms, 0 | 20260506 00:14:35.588 | INFO | BAZWRITER : 61.8%, 3.5, 112.2ms, 0 | 20260506 00:14:35.588 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:14:35.588 | INFO | Uploaded 36GiB to GPU in 2.17401s (16.5593GiB/s) | 20260506 00:14:35.588 | INFO | Downloaded 38GiB to GPU in 2.24014s (16.9633GiB/s) | 20260506 00:14:40.720 | INFO | Analyzing chunk frames = [7430656,7431168) | 20260506 00:14:40.720 | INFO | MemInfo: { | 20260506 00:14:40.720 | INFO | "gpu" : " 17 GB", | 20260506 00:14:40.720 | INFO | "huge" : "", | 20260506 00:14:40.720 | INFO | "huge_shared" : " 27 GB", | 20260506 00:14:40.720 | INFO | "malloc" : " 14 GB", | 20260506 00:14:40.721 | INFO | "pinned" : " 29 GB" | 20260506 00:14:40.721 | INFO | } | 20260506 00:14:40.721 | INFO | Perf report for the last 5.13329 seconds: | 20260506 00:14:40.721 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:14:40.721 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:40.721 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:40.721 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:40.721 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 00:14:40.721 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:40.721 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:40.721 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:14:40.721 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:40.721 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:40.721 | INFO | Uploaded 36GiB to GPU in 1.98156s (18.1675GiB/s) | 20260506 00:14:40.721 | INFO | Downloaded 34GiB to GPU in 1.9822s (17.1527GiB/s) | 20260506 00:14:45.852 | INFO | Analyzing chunk frames = [7431168,7431680) | 20260506 00:14:45.852 | INFO | MemInfo: { | 20260506 00:14:45.852 | INFO | "gpu" : " 17 GB", | 20260506 00:14:45.852 | INFO | "huge" : "", | 20260506 00:14:45.852 | INFO | "huge_shared" : " 27 GB", | 20260506 00:14:45.852 | INFO | "malloc" : " 14 GB", | 20260506 00:14:45.852 | INFO | "pinned" : " 29 GB" | 20260506 00:14:45.852 | INFO | } | 20260506 00:14:45.853 | INFO | Perf report for the last 5.13201 seconds: | 20260506 00:14:45.853 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:14:45.853 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:45.853 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:45.853 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:45.853 | INFO | ANALYSIS : 61.5%, 3.0, 95.5ms, 0 | 20260506 00:14:45.853 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:45.853 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:45.853 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:14:45.853 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:45.853 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:45.853 | INFO | Uploaded 36GiB to GPU in 2.08464s (17.2692GiB/s) | 20260506 00:14:45.853 | INFO | Downloaded 34GiB to GPU in 2.01938s (16.8369GiB/s) | 20260506 00:14:50.998 | INFO | Analyzing chunk frames = [7431680,7432192) | 20260506 00:14:50.998 | INFO | MemInfo: { | 20260506 00:14:50.998 | INFO | "gpu" : " 17 GB", | 20260506 00:14:50.998 | INFO | "huge" : "", | 20260506 00:14:50.998 | INFO | "huge_shared" : " 27 GB", | 20260506 00:14:50.998 | INFO | "malloc" : " 14 GB", | 20260506 00:14:50.998 | INFO | "pinned" : " 29 GB" | 20260506 00:14:50.998 | INFO | } | 20260506 00:14:50.998 | INFO | Perf report for the last 5.1454 seconds: | 20260506 00:14:50.998 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:14:50.998 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:50.998 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:50.998 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:50.998 | INFO | ANALYSIS : 61.6%, 3.0, 95.7ms, 0 | 20260506 00:14:50.998 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:50.998 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:50.998 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 00:14:50.998 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:50.998 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:50.998 | INFO | Uploaded 36GiB to GPU in 2.12417s (16.9478GiB/s) | 20260506 00:14:50.998 | INFO | Downloaded 34GiB to GPU in 2.01559s (16.8685GiB/s) | 20260506 00:14:56.136 | INFO | Analyzing chunk frames = [7432192,7432704) | 20260506 00:14:56.136 | INFO | MemInfo: { | 20260506 00:14:56.136 | INFO | "gpu" : " 17 GB", | 20260506 00:14:56.136 | INFO | "huge" : "", | 20260506 00:14:56.136 | INFO | "huge_shared" : " 27 GB", | 20260506 00:14:56.136 | INFO | "malloc" : " 14 GB", | 20260506 00:14:56.136 | INFO | "pinned" : " 29 GB" | 20260506 00:14:56.136 | INFO | } | 20260506 00:14:56.136 | INFO | Perf report for the last 5.1378 seconds: | 20260506 00:14:56.136 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:14:56.136 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:56.136 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:56.136 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:56.136 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 00:14:56.136 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:56.136 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:56.136 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:14:56.136 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:56.136 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:14:56.136 | INFO | Uploaded 36GiB to GPU in 2.10315s (17.1172GiB/s) | 20260506 00:14:56.136 | INFO | Downloaded 34GiB to GPU in 2.04277s (16.644GiB/s) | 20260506 00:15:01.274 | INFO | Analyzing chunk frames = [7432704,7433216) | 20260506 00:15:01.274 | INFO | MemInfo: { | 20260506 00:15:01.274 | INFO | "gpu" : " 17 GB", | 20260506 00:15:01.274 | INFO | "huge" : "", | 20260506 00:15:01.274 | INFO | "huge_shared" : " 27 GB", | 20260506 00:15:01.274 | INFO | "malloc" : " 14 GB", | 20260506 00:15:01.274 | INFO | "pinned" : " 29 GB" | 20260506 00:15:01.274 | INFO | } | 20260506 00:15:01.274 | INFO | Perf report for the last 5.13862 seconds: | 20260506 00:15:01.274 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:15:01.274 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:01.274 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:01.275 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:01.275 | INFO | ANALYSIS : 61.2%, 3.0, 95.6ms, 0 | 20260506 00:15:01.275 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:01.275 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:01.275 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:15:01.275 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:01.275 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:01.275 | INFO | Uploaded 36GiB to GPU in 2.07181s (17.3761GiB/s) | 20260506 00:15:01.275 | INFO | Downloaded 34GiB to GPU in 2.04339s (16.639GiB/s) | 20260506 00:15:06.414 | INFO | Analyzing chunk frames = [7433216,7433728) | 20260506 00:15:06.414 | INFO | MemInfo: { | 20260506 00:15:06.414 | INFO | "gpu" : " 17 GB", | 20260506 00:15:06.414 | INFO | "huge" : "", | 20260506 00:15:06.414 | INFO | "huge_shared" : " 27 GB", | 20260506 00:15:06.414 | INFO | "malloc" : " 14 GB", | 20260506 00:15:06.414 | INFO | "pinned" : " 29 GB" | 20260506 00:15:06.415 | INFO | } | 20260506 00:15:06.415 | INFO | Perf report for the last 5.14008 seconds: | 20260506 00:15:06.415 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:15:06.415 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:06.415 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:06.415 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:06.415 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260506 00:15:06.415 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:06.415 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:06.415 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 00:15:06.415 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:06.415 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:06.415 | INFO | Uploaded 36GiB to GPU in 2.08989s (17.2258GiB/s) | 20260506 00:15:06.415 | INFO | Downloaded 34GiB to GPU in 2.08972s (16.2701GiB/s) | 20260506 00:15:11.552 | INFO | Analyzing chunk frames = [7433728,7434240) | 20260506 00:15:11.552 | INFO | MemInfo: { | 20260506 00:15:11.552 | INFO | "gpu" : " 17 GB", | 20260506 00:15:11.552 | INFO | "huge" : "", | 20260506 00:15:11.552 | INFO | "huge_shared" : " 27 GB", | 20260506 00:15:11.552 | INFO | "malloc" : " 14 GB", | 20260506 00:15:11.553 | INFO | "pinned" : " 29 GB" | 20260506 00:15:11.553 | INFO | } | 20260506 00:15:11.553 | INFO | Perf report for the last 5.1381 seconds: | 20260506 00:15:11.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:15:11.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:11.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:11.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:11.553 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260506 00:15:11.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:11.553 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:11.553 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 00:15:11.553 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:11.553 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:11.553 | INFO | Uploaded 36GiB to GPU in 2.11303s (17.0372GiB/s) | 20260506 00:15:11.553 | INFO | Downloaded 34GiB to GPU in 2.00566s (16.9521GiB/s) | 20260506 00:15:11.683 | INFO | Starting a round of Baz IO | 20260506 00:15:14.823 | INFO | Populating RTMetrics report | 20260506 00:15:14.823 | INFO | RTMetrics report populated | 20260506 00:15:14.823 | INFO | Opening tmp RTMetrics json file | 20260506 00:15:14.825 | INFO | Serializing RTMetrics to json | 20260506 00:15:14.828 | INFO | Serialization done, swapping tmp file now... | 20260506 00:15:14.829 | INFO | RTMetrics done | 20260506 00:15:14.899 | INFO | Finished a round of Baz IO: Wrote 1.866 GiB in 3.215 seconds | 20260506 00:15:14.899 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.38% / 0.00% / 78.62% | 20260506 00:15:14.899 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0954469% / 7.94729e-06% / 0.000107288% | 20260506 00:15:14.899 | INFO | Baz IO overall: | 20260506 00:15:14.899 | INFO | BytesWritten : 4229023563598 | 20260506 00:15:14.899 | INFO | NumEvents : 1990570401837 | 20260506 00:15:16.692 | INFO | Analyzing chunk frames = [7434240,7434752) | 20260506 00:15:16.692 | INFO | MemInfo: { | 20260506 00:15:16.692 | INFO | "gpu" : " 17 GB", | 20260506 00:15:16.692 | INFO | "huge" : "", | 20260506 00:15:16.692 | INFO | "huge_shared" : " 27 GB", | 20260506 00:15:16.692 | INFO | "malloc" : " 14 GB", | 20260506 00:15:16.692 | INFO | "pinned" : " 29 GB" | 20260506 00:15:16.692 | INFO | } | 20260506 00:15:16.692 | INFO | Perf report for the last 5.13928 seconds: | 20260506 00:15:16.692 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:15:16.692 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:16.692 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:16.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:16.692 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260506 00:15:16.692 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:16.692 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260506 00:15:16.692 | INFO | PRE_HQ : 59.6%, 1.7, 54.0ms, 0 | 20260506 00:15:16.692 | INFO | BAZWRITER : 61.7%, 3.6, 112.7ms, 0 | 20260506 00:15:16.692 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:15:16.692 | INFO | Uploaded 36GiB to GPU in 2.1686s (16.6006GiB/s) | 20260506 00:15:16.692 | INFO | Downloaded 38GiB to GPU in 2.23175s (17.027GiB/s) | 20260506 00:15:21.835 | INFO | Analyzing chunk frames = [7434752,7435264) | 20260506 00:15:21.835 | INFO | MemInfo: { | 20260506 00:15:21.835 | INFO | "gpu" : " 17 GB", | 20260506 00:15:21.835 | INFO | "huge" : "", | 20260506 00:15:21.835 | INFO | "huge_shared" : " 27 GB", | 20260506 00:15:21.835 | INFO | "malloc" : " 14 GB", | 20260506 00:15:21.835 | INFO | "pinned" : " 29 GB" | 20260506 00:15:21.835 | INFO | } | 20260506 00:15:21.835 | INFO | Perf report for the last 5.14294 seconds: | 20260506 00:15:21.835 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:15:21.835 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:21.835 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:21.835 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:21.835 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260506 00:15:21.835 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:21.835 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:21.835 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 00:15:21.835 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:21.835 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:21.835 | INFO | Uploaded 36GiB to GPU in 2.10916s (17.0684GiB/s) | 20260506 00:15:21.835 | INFO | Downloaded 34GiB to GPU in 2.03718s (16.6898GiB/s) | 20260506 00:15:26.970 | INFO | Analyzing chunk frames = [7435264,7435776) | 20260506 00:15:26.970 | INFO | MemInfo: { | 20260506 00:15:26.970 | INFO | "gpu" : " 17 GB", | 20260506 00:15:26.970 | INFO | "huge" : "", | 20260506 00:15:26.970 | INFO | "huge_shared" : " 27 GB", | 20260506 00:15:26.970 | INFO | "malloc" : " 14 GB", | 20260506 00:15:26.970 | INFO | "pinned" : " 29 GB" | 20260506 00:15:26.970 | INFO | } | 20260506 00:15:26.970 | INFO | Perf report for the last 5.13552 seconds: | 20260506 00:15:26.971 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:15:26.971 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:26.971 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:26.971 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:26.971 | INFO | ANALYSIS : 61.4%, 3.0, 95.4ms, 0 | 20260506 00:15:26.971 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:26.971 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:26.971 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:15:26.971 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:26.971 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:26.971 | INFO | Uploaded 36GiB to GPU in 2.06503s (17.4331GiB/s) | 20260506 00:15:26.971 | INFO | Downloaded 34GiB to GPU in 2.01247s (16.8947GiB/s) | 20260506 00:15:32.108 | INFO | Analyzing chunk frames = [7435776,7436288) | 20260506 00:15:32.108 | INFO | MemInfo: { | 20260506 00:15:32.108 | INFO | "gpu" : " 17 GB", | 20260506 00:15:32.108 | INFO | "huge" : "", | 20260506 00:15:32.108 | INFO | "huge_shared" : " 27 GB", | 20260506 00:15:32.108 | INFO | "malloc" : " 14 GB", | 20260506 00:15:32.108 | INFO | "pinned" : " 29 GB" | 20260506 00:15:32.108 | INFO | } | 20260506 00:15:32.108 | INFO | Perf report for the last 5.13746 seconds: | 20260506 00:15:32.108 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:15:32.108 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:32.108 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:32.108 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:32.108 | INFO | ANALYSIS : 61.7%, 3.0, 95.8ms, 0 | 20260506 00:15:32.108 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:32.108 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:32.108 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 00:15:32.108 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:32.108 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:32.108 | INFO | Uploaded 36GiB to GPU in 2.13604s (16.8536GiB/s) | 20260506 00:15:32.108 | INFO | Downloaded 34GiB to GPU in 2.02104s (16.823GiB/s) | 20260506 00:15:37.249 | INFO | Analyzing chunk frames = [7436288,7436800) | 20260506 00:15:37.249 | INFO | MemInfo: { | 20260506 00:15:37.249 | INFO | "gpu" : " 17 GB", | 20260506 00:15:37.249 | INFO | "huge" : "", | 20260506 00:15:37.249 | INFO | "huge_shared" : " 27 GB", | 20260506 00:15:37.249 | INFO | "malloc" : " 14 GB", | 20260506 00:15:37.249 | INFO | "pinned" : " 29 GB" | 20260506 00:15:37.249 | INFO | } | 20260506 00:15:37.249 | INFO | Perf report for the last 5.1407 seconds: | 20260506 00:15:37.249 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:15:37.249 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:37.249 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:37.249 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:37.249 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260506 00:15:37.249 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:37.249 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:37.249 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 00:15:37.249 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:37.249 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:37.249 | INFO | Uploaded 36GiB to GPU in 2.16119s (16.6575GiB/s) | 20260506 00:15:37.249 | INFO | Downloaded 34GiB to GPU in 2.06999s (16.4252GiB/s) | 20260506 00:15:42.394 | INFO | Analyzing chunk frames = [7436800,7437312) | 20260506 00:15:42.394 | INFO | MemInfo: { | 20260506 00:15:42.394 | INFO | "gpu" : " 17 GB", | 20260506 00:15:42.394 | INFO | "huge" : "", | 20260506 00:15:42.394 | INFO | "huge_shared" : " 27 GB", | 20260506 00:15:42.394 | INFO | "malloc" : " 14 GB", | 20260506 00:15:42.394 | INFO | "pinned" : " 29 GB" | 20260506 00:15:42.394 | INFO | } | 20260506 00:15:42.394 | INFO | Perf report for the last 5.14536 seconds: | 20260506 00:15:42.394 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:15:42.394 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:42.394 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:42.394 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:42.394 | INFO | ANALYSIS : 61.2%, 3.0, 95.6ms, 0 | 20260506 00:15:42.394 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:42.394 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:42.394 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:15:42.394 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:42.394 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:42.394 | INFO | Uploaded 36GiB to GPU in 2.06973s (17.3936GiB/s) | 20260506 00:15:42.394 | INFO | Downloaded 34GiB to GPU in 2.03885s (16.6761GiB/s) | 20260506 00:15:47.533 | INFO | Analyzing chunk frames = [7437312,7437824) | 20260506 00:15:47.533 | INFO | MemInfo: { | 20260506 00:15:47.533 | INFO | "gpu" : " 17 GB", | 20260506 00:15:47.533 | INFO | "huge" : "", | 20260506 00:15:47.533 | INFO | "huge_shared" : " 27 GB", | 20260506 00:15:47.533 | INFO | "malloc" : " 14 GB", | 20260506 00:15:47.533 | INFO | "pinned" : " 29 GB" | 20260506 00:15:47.533 | INFO | } | 20260506 00:15:47.533 | INFO | Perf report for the last 5.13925 seconds: | 20260506 00:15:47.533 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:15:47.533 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:47.533 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:47.533 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:47.533 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 00:15:47.534 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:47.534 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:47.534 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 00:15:47.534 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:47.534 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:47.534 | INFO | Uploaded 36GiB to GPU in 2.01845s (17.8355GiB/s) | 20260506 00:15:47.534 | INFO | Downloaded 34GiB to GPU in 2.04749s (16.6057GiB/s) | 20260506 00:15:52.666 | INFO | Analyzing chunk frames = [7437824,7438336) | 20260506 00:15:52.666 | INFO | MemInfo: { | 20260506 00:15:52.666 | INFO | "gpu" : " 17 GB", | 20260506 00:15:52.666 | INFO | "huge" : "", | 20260506 00:15:52.666 | INFO | "huge_shared" : " 27 GB", | 20260506 00:15:52.666 | INFO | "malloc" : " 14 GB", | 20260506 00:15:52.666 | INFO | "pinned" : " 29 GB" | 20260506 00:15:52.666 | INFO | } | 20260506 00:15:52.666 | INFO | Perf report for the last 5.13276 seconds: | 20260506 00:15:52.666 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:15:52.666 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:52.666 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:52.666 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:52.666 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260506 00:15:52.666 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:52.666 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:52.666 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 00:15:52.666 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:52.666 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:52.666 | INFO | Uploaded 36GiB to GPU in 2.10581s (17.0956GiB/s) | 20260506 00:15:52.666 | INFO | Downloaded 34GiB to GPU in 2.03646s (16.6956GiB/s) | 20260506 00:15:52.666 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 00:15:52.666 | INFO | Total mem usage: 15.3 GiB | 20260506 00:15:52.666 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 00:15:52.666 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.12%) | 20260506 00:15:52.666 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 00:15:52.666 | INFO | High Water Breakdown: | 20260506 00:15:52.667 | INFO | 60.9% : 8.55 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 00:15:52.667 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 00:15:52.667 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 00:15:52.667 | INFO | ------------------------------------------ | 20260506 00:15:52.667 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:15:52.667 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 00:15:52.667 | INFO | Total mem usage: 26.5 GiB | 20260506 00:15:52.667 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 00:15:52.667 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 00:15:52.667 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 00:15:52.667 | INFO | High Water Breakdown: | 20260506 00:15:52.667 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:15:52.667 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:15:52.667 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:15:52.667 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:15:52.667 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:15:52.667 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:15:52.667 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 00:15:52.667 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 00:15:52.667 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:15:52.667 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 00:15:52.667 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 00:15:52.667 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 00:15:52.667 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 00:15:52.667 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 00:15:52.667 | INFO | ------------------------------------------ | 20260506 00:15:52.667 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:15:52.667 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 00:15:52.667 | INFO | Total mem usage: 25.6 GiB | 20260506 00:15:52.667 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 00:15:52.667 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 00:15:52.667 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 00:15:52.667 | INFO | High Water Breakdown: | 20260506 00:15:52.667 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 00:15:52.667 | INFO | ------------------------------------------ | 20260506 00:15:52.667 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:15:52.667 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 00:15:52.667 | INFO | Total mem usage: 15.7 GiB | 20260506 00:15:52.667 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 00:15:52.667 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 00:15:52.667 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 00:15:52.667 | INFO | High Water Breakdown: | 20260506 00:15:52.667 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 00:15:52.667 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 00:15:52.667 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 00:15:52.667 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 00:15:52.667 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 00:15:52.667 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 00:15:52.667 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 00:15:52.667 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 00:15:52.667 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 00:15:52.667 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:15:52.667 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 00:15:52.667 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:15:52.667 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 00:15:52.667 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 00:15:52.667 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 00:15:52.667 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 00:15:52.667 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:15:52.667 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 00:15:52.667 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:15:52.667 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:15:52.667 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:15:52.667 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 00:15:52.667 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 00:15:52.667 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:15:52.667 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 00:15:52.668 | INFO | ------------------------------------------ | 20260506 00:15:52.668 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:15:52.668 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 00:15:52.668 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:15:52.668 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:15:52.668 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.40e+01, 1.32e+01 / 8.97e-01 | 20260506 00:15:52.668 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.06e-01, 1.11e-02 / 1.83e-02 | 20260506 00:15:52.668 | INFO | PerfReport: 11.85%, Baseline -- 3.71e+00 / 4.52e+00, 5.75e-01 / 2.12e-01 | 20260506 00:15:52.668 | INFO | PerfReport: 3.29%, Histogram -- 1.03e+00 / 1.52e+00, 1.92e-01 / 9.15e-02 | 20260506 00:15:52.668 | INFO | PerfReport: 15.97%, DME -- 4.00e+01 / 4.40e+01, 3.87e+00 / 1.10e+00 | 20260506 00:15:52.668 | INFO | PerfReport: 57.37%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.53e+00 / 1.03e+00 | 20260506 00:15:52.668 | INFO | PerfReport: 7.08%, PulseAccumulating -- 2.22e+00 / 2.73e+00, 4.21e-01 / 2.28e-01 | 20260506 00:15:52.668 | INFO | PerfReport: 3.08%, Metrics -- 9.66e-01 / 1.26e+00, 2.59e-01 / 1.27e-01 | 20260506 00:15:52.668 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.35e-01 / 3.91e-01, 5.62e-02 / 3.36e-02 | 20260506 00:15:52.668 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 00:15:52.668 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:15:52.668 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:15:52.668 | INFO | PerfReport: Full Execution -- 9.82e+01 / -nan, 2.43e+01 / 0.00e+00 | 20260506 00:15:52.668 | INFO | PerfReport: 22.26%, Upload -- 2.19e+01 / 0.00e+00, 7.45e+00 / 0.00e+00 | 20260506 00:15:52.668 | INFO | PerfReport: 21.84%, Download -- 2.14e+01 / 0.00e+00, 7.57e+00 / 0.00e+00 | 20260506 00:15:52.668 | INFO | PerfReport: 31.92%, Compute -- 3.13e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260506 00:15:52.668 | INFO | PerfReport: 2.11%, LockUpload -- 2.07e+00 / 0.00e+00, 7.84e+00 / 0.00e+00 | 20260506 00:15:52.668 | INFO | PerfReport: 1.32%, LockDownload -- 1.30e+00 / 0.00e+00, 2.92e+00 / 0.00e+00 | 20260506 00:15:52.668 | INFO | PerfReport: 20.55%, LockCompute -- 2.02e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 00:15:52.668 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.57e-02 / 0.00e+00 | 20260506 00:15:52.791 | INFO | Starting a round of Baz IO | 20260506 00:15:55.940 | INFO | Populating RTMetrics report | 20260506 00:15:55.941 | INFO | RTMetrics report populated | 20260506 00:15:55.941 | INFO | Opening tmp RTMetrics json file | 20260506 00:15:55.943 | INFO | Serializing RTMetrics to json | 20260506 00:15:55.947 | INFO | Serialization done, swapping tmp file now... | 20260506 00:15:55.948 | INFO | RTMetrics done | 20260506 00:15:56.017 | INFO | Finished a round of Baz IO: Wrote 1.864 GiB in 3.225 seconds | 20260506 00:15:56.017 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.38% / 0.00% / 78.62% | 20260506 00:15:56.017 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0945886% / 0% / 7.54992e-05% | 20260506 00:15:56.017 | INFO | Baz IO overall: | 20260506 00:15:56.017 | INFO | BytesWritten : 4231025317088 | 20260506 00:15:56.017 | INFO | NumEvents : 1991485672074 | 20260506 00:15:57.806 | INFO | Analyzing chunk frames = [7438336,7438848) | 20260506 00:15:57.806 | INFO | MemInfo: { | 20260506 00:15:57.806 | INFO | "gpu" : " 17 GB", | 20260506 00:15:57.806 | INFO | "huge" : "", | 20260506 00:15:57.806 | INFO | "huge_shared" : " 27 GB", | 20260506 00:15:57.806 | INFO | "malloc" : " 14 GB", | 20260506 00:15:57.807 | INFO | "pinned" : " 29 GB" | 20260506 00:15:57.807 | INFO | } | 20260506 00:15:57.807 | INFO | Perf report for the last 5.14066 seconds: | 20260506 00:15:57.807 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:15:57.807 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:57.807 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:57.807 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:57.807 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260506 00:15:57.807 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:15:57.807 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260506 00:15:57.807 | INFO | PRE_HQ : 59.6%, 1.7, 53.9ms, 0 | 20260506 00:15:57.807 | INFO | BAZWRITER : 62.0%, 3.6, 113.4ms, 0 | 20260506 00:15:57.807 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 00:15:57.807 | INFO | Uploaded 36GiB to GPU in 2.16431s (16.6335GiB/s) | 20260506 00:15:57.807 | INFO | Downloaded 38GiB to GPU in 2.22662s (17.0662GiB/s) | 20260506 00:16:02.949 | INFO | Analyzing chunk frames = [7438848,7439360) | 20260506 00:16:02.949 | INFO | MemInfo: { | 20260506 00:16:02.949 | INFO | "gpu" : " 17 GB", | 20260506 00:16:02.949 | INFO | "huge" : "", | 20260506 00:16:02.949 | INFO | "huge_shared" : " 27 GB", | 20260506 00:16:02.949 | INFO | "malloc" : " 14 GB", | 20260506 00:16:02.949 | INFO | "pinned" : " 29 GB" | 20260506 00:16:02.950 | INFO | } | 20260506 00:16:02.950 | INFO | Perf report for the last 5.14288 seconds: | 20260506 00:16:02.950 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:16:02.950 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:02.950 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:02.950 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:02.950 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 00:16:02.950 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:02.950 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:02.950 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:16:02.950 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:02.950 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:02.950 | INFO | Uploaded 36GiB to GPU in 2.09666s (17.1701GiB/s) | 20260506 00:16:02.950 | INFO | Downloaded 34GiB to GPU in 2.03391s (16.7165GiB/s) | 20260506 00:16:08.089 | INFO | Analyzing chunk frames = [7439360,7439872) | 20260506 00:16:08.089 | INFO | MemInfo: { | 20260506 00:16:08.089 | INFO | "gpu" : " 17 GB", | 20260506 00:16:08.089 | INFO | "huge" : "", | 20260506 00:16:08.089 | INFO | "huge_shared" : " 27 GB", | 20260506 00:16:08.089 | INFO | "malloc" : " 14 GB", | 20260506 00:16:08.089 | INFO | "pinned" : " 29 GB" | 20260506 00:16:08.089 | INFO | } | 20260506 00:16:08.089 | INFO | Perf report for the last 5.13912 seconds: | 20260506 00:16:08.089 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:16:08.089 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:08.089 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:08.089 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:08.089 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 00:16:08.089 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:08.089 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:08.089 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 00:16:08.089 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:08.089 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:08.089 | INFO | Uploaded 36GiB to GPU in 2.06387s (17.4429GiB/s) | 20260506 00:16:08.089 | INFO | Downloaded 34GiB to GPU in 2.02066s (16.8262GiB/s) | 20260506 00:16:13.225 | INFO | Analyzing chunk frames = [7439872,7440384) | 20260506 00:16:13.225 | INFO | MemInfo: { | 20260506 00:16:13.225 | INFO | "gpu" : " 17 GB", | 20260506 00:16:13.225 | INFO | "huge" : "", | 20260506 00:16:13.225 | INFO | "huge_shared" : " 27 GB", | 20260506 00:16:13.225 | INFO | "malloc" : " 14 GB", | 20260506 00:16:13.225 | INFO | "pinned" : " 29 GB" | 20260506 00:16:13.225 | INFO | } | 20260506 00:16:13.226 | INFO | Perf report for the last 5.13677 seconds: | 20260506 00:16:13.226 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:16:13.226 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:13.226 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:13.226 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:13.226 | INFO | ANALYSIS : 61.6%, 3.0, 95.7ms, 0 | 20260506 00:16:13.226 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:13.226 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:13.226 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:16:13.226 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:13.226 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:13.226 | INFO | Uploaded 36GiB to GPU in 2.13568s (16.8565GiB/s) | 20260506 00:16:13.226 | INFO | Downloaded 34GiB to GPU in 2.02595s (16.7823GiB/s) | 20260506 00:16:18.361 | INFO | Analyzing chunk frames = [7440384,7440896) | 20260506 00:16:18.361 | INFO | MemInfo: { | 20260506 00:16:18.361 | INFO | "gpu" : " 17 GB", | 20260506 00:16:18.361 | INFO | "huge" : "", | 20260506 00:16:18.361 | INFO | "huge_shared" : " 27 GB", | 20260506 00:16:18.361 | INFO | "malloc" : " 14 GB", | 20260506 00:16:18.361 | INFO | "pinned" : " 29 GB" | 20260506 00:16:18.361 | INFO | } | 20260506 00:16:18.361 | INFO | Perf report for the last 5.13521 seconds: | 20260506 00:16:18.361 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:16:18.361 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:18.361 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:18.361 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:18.361 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 00:16:18.361 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:18.361 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:18.361 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 00:16:18.361 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:18.361 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:18.361 | INFO | Uploaded 36GiB to GPU in 2.10534s (17.0994GiB/s) | 20260506 00:16:18.361 | INFO | Downloaded 34GiB to GPU in 2.05095s (16.5777GiB/s) | 20260506 00:16:23.506 | INFO | Analyzing chunk frames = [7440896,7441408) | 20260506 00:16:23.506 | INFO | MemInfo: { | 20260506 00:16:23.506 | INFO | "gpu" : " 17 GB", | 20260506 00:16:23.506 | INFO | "huge" : "", | 20260506 00:16:23.506 | INFO | "huge_shared" : " 27 GB", | 20260506 00:16:23.506 | INFO | "malloc" : " 14 GB", | 20260506 00:16:23.507 | INFO | "pinned" : " 29 GB" | 20260506 00:16:23.507 | INFO | } | 20260506 00:16:23.507 | INFO | Perf report for the last 5.14591 seconds: | 20260506 00:16:23.507 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:16:23.507 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:23.507 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:23.507 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:23.507 | INFO | ANALYSIS : 61.2%, 3.0, 95.7ms, 0 | 20260506 00:16:23.507 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:23.507 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:23.507 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:16:23.507 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:23.507 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:23.507 | INFO | Uploaded 36GiB to GPU in 2.07137s (17.3798GiB/s) | 20260506 00:16:23.507 | INFO | Downloaded 34GiB to GPU in 2.04351s (16.638GiB/s) | 20260506 00:16:28.646 | INFO | Analyzing chunk frames = [7441408,7441920) | 20260506 00:16:28.646 | INFO | MemInfo: { | 20260506 00:16:28.646 | INFO | "gpu" : " 17 GB", | 20260506 00:16:28.646 | INFO | "huge" : "", | 20260506 00:16:28.646 | INFO | "huge_shared" : " 27 GB", | 20260506 00:16:28.646 | INFO | "malloc" : " 14 GB", | 20260506 00:16:28.646 | INFO | "pinned" : " 29 GB" | 20260506 00:16:28.647 | INFO | } | 20260506 00:16:28.647 | INFO | Perf report for the last 5.13994 seconds: | 20260506 00:16:28.647 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:16:28.647 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:28.647 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:28.647 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:28.647 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260506 00:16:28.647 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:28.647 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:28.647 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 00:16:28.647 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:28.647 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:28.647 | INFO | Uploaded 36GiB to GPU in 2.10703s (17.0857GiB/s) | 20260506 00:16:28.647 | INFO | Downloaded 34GiB to GPU in 2.08548s (16.3032GiB/s) | 20260506 00:16:33.779 | INFO | Analyzing chunk frames = [7441920,7442432) | 20260506 00:16:33.779 | INFO | MemInfo: { | 20260506 00:16:33.779 | INFO | "gpu" : " 17 GB", | 20260506 00:16:33.779 | INFO | "huge" : "", | 20260506 00:16:33.779 | INFO | "huge_shared" : " 27 GB", | 20260506 00:16:33.779 | INFO | "malloc" : " 14 GB", | 20260506 00:16:33.780 | INFO | "pinned" : " 29 GB" | 20260506 00:16:33.780 | INFO | } | 20260506 00:16:33.780 | INFO | Perf report for the last 5.13293 seconds: | 20260506 00:16:33.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:16:33.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:33.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:33.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:33.780 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 00:16:33.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:33.780 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:33.780 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 00:16:33.780 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:33.780 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:33.780 | INFO | Uploaded 36GiB to GPU in 2.12515s (16.94GiB/s) | 20260506 00:16:33.780 | INFO | Downloaded 34GiB to GPU in 2.02223s (16.8132GiB/s) | 20260506 00:16:33.905 | INFO | Starting a round of Baz IO | 20260506 00:16:37.048 | INFO | Populating RTMetrics report | 20260506 00:16:37.048 | INFO | RTMetrics report populated | 20260506 00:16:37.048 | INFO | Opening tmp RTMetrics json file | 20260506 00:16:37.048 | INFO | Serializing RTMetrics to json | 20260506 00:16:37.050 | INFO | Serialization done, swapping tmp file now... | 20260506 00:16:37.050 | INFO | RTMetrics done | 20260506 00:16:37.124 | INFO | Finished a round of Baz IO: Wrote 1.864 GiB in 3.218 seconds | 20260506 00:16:37.124 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.38% / 0.00% / 78.62% | 20260506 00:16:37.124 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0948826% / 0% / 8.74201e-05% | 20260506 00:16:37.124 | INFO | Baz IO overall: | 20260506 00:16:37.124 | INFO | BytesWritten : 4233027114543 | 20260506 00:16:37.124 | INFO | NumEvents : 1992401157817 | 20260506 00:16:38.920 | INFO | Analyzing chunk frames = [7442432,7442944) | 20260506 00:16:38.920 | INFO | MemInfo: { | 20260506 00:16:38.920 | INFO | "gpu" : " 17 GB", | 20260506 00:16:38.920 | INFO | "huge" : "", | 20260506 00:16:38.920 | INFO | "huge_shared" : " 27 GB", | 20260506 00:16:38.920 | INFO | "malloc" : " 14 GB", | 20260506 00:16:38.920 | INFO | "pinned" : " 29 GB" | 20260506 00:16:38.920 | INFO | } | 20260506 00:16:38.920 | INFO | Perf report for the last 5.14092 seconds: | 20260506 00:16:38.920 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:16:38.920 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:38.920 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:38.921 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:38.921 | INFO | ANALYSIS : 62.5%, 3.0, 97.7ms, 0 | 20260506 00:16:38.921 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:38.921 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260506 00:16:38.921 | INFO | PRE_HQ : 59.8%, 1.7, 53.9ms, 0 | 20260506 00:16:38.921 | INFO | BAZWRITER : 61.8%, 3.6, 113.7ms, 0 | 20260506 00:16:38.921 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:16:38.921 | INFO | Uploaded 36GiB to GPU in 2.16816s (16.6039GiB/s) | 20260506 00:16:38.921 | INFO | Downloaded 38GiB to GPU in 2.23825s (16.9775GiB/s) | 20260506 00:16:44.060 | INFO | Analyzing chunk frames = [7442944,7443456) | 20260506 00:16:44.060 | INFO | MemInfo: { | 20260506 00:16:44.060 | INFO | "gpu" : " 17 GB", | 20260506 00:16:44.060 | INFO | "huge" : "", | 20260506 00:16:44.060 | INFO | "huge_shared" : " 27 GB", | 20260506 00:16:44.060 | INFO | "malloc" : " 14 GB", | 20260506 00:16:44.060 | INFO | "pinned" : " 29 GB" | 20260506 00:16:44.060 | INFO | } | 20260506 00:16:44.060 | INFO | Perf report for the last 5.13928 seconds: | 20260506 00:16:44.060 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:16:44.060 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:44.060 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:44.060 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:44.060 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260506 00:16:44.060 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:44.060 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:44.060 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 00:16:44.060 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:44.060 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:44.060 | INFO | Uploaded 36GiB to GPU in 2.09608s (17.1749GiB/s) | 20260506 00:16:44.060 | INFO | Downloaded 34GiB to GPU in 2.02781s (16.7669GiB/s) | 20260506 00:16:49.203 | INFO | Analyzing chunk frames = [7443456,7443968) | 20260506 00:16:49.203 | INFO | MemInfo: { | 20260506 00:16:49.203 | INFO | "gpu" : " 17 GB", | 20260506 00:16:49.203 | INFO | "huge" : "", | 20260506 00:16:49.203 | INFO | "huge_shared" : " 27 GB", | 20260506 00:16:49.203 | INFO | "malloc" : " 14 GB", | 20260506 00:16:49.203 | INFO | "pinned" : " 29 GB" | 20260506 00:16:49.203 | INFO | } | 20260506 00:16:49.203 | INFO | Perf report for the last 5.14297 seconds: | 20260506 00:16:49.203 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:16:49.203 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:49.203 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:49.203 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:49.203 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 00:16:49.203 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:49.203 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:49.203 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:16:49.203 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:49.203 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:49.203 | INFO | Uploaded 36GiB to GPU in 2.09329s (17.1978GiB/s) | 20260506 00:16:49.203 | INFO | Downloaded 34GiB to GPU in 2.02767s (16.768GiB/s) | 20260506 00:16:54.342 | INFO | Analyzing chunk frames = [7443968,7444480) | 20260506 00:16:54.342 | INFO | MemInfo: { | 20260506 00:16:54.342 | INFO | "gpu" : " 17 GB", | 20260506 00:16:54.342 | INFO | "huge" : "", | 20260506 00:16:54.342 | INFO | "huge_shared" : " 27 GB", | 20260506 00:16:54.342 | INFO | "malloc" : " 14 GB", | 20260506 00:16:54.342 | INFO | "pinned" : " 29 GB" | 20260506 00:16:54.342 | INFO | } | 20260506 00:16:54.342 | INFO | Perf report for the last 5.13903 seconds: | 20260506 00:16:54.342 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:16:54.342 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:54.342 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:54.342 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:54.342 | INFO | ANALYSIS : 61.6%, 3.0, 95.6ms, 0 | 20260506 00:16:54.342 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:54.342 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:54.342 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 00:16:54.342 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:54.342 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:54.342 | INFO | Uploaded 36GiB to GPU in 2.14722s (16.7659GiB/s) | 20260506 00:16:54.342 | INFO | Downloaded 34GiB to GPU in 2.04569s (16.6203GiB/s) | 20260506 00:16:59.483 | INFO | Analyzing chunk frames = [7444480,7444992) | 20260506 00:16:59.483 | INFO | MemInfo: { | 20260506 00:16:59.483 | INFO | "gpu" : " 17 GB", | 20260506 00:16:59.483 | INFO | "huge" : "", | 20260506 00:16:59.483 | INFO | "huge_shared" : " 27 GB", | 20260506 00:16:59.483 | INFO | "malloc" : " 14 GB", | 20260506 00:16:59.483 | INFO | "pinned" : " 29 GB" | 20260506 00:16:59.483 | INFO | } | 20260506 00:16:59.483 | INFO | Perf report for the last 5.14129 seconds: | 20260506 00:16:59.483 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:16:59.483 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:59.483 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:59.483 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:59.483 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 00:16:59.483 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:59.483 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:59.483 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:16:59.483 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:59.483 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:16:59.484 | INFO | Uploaded 36GiB to GPU in 2.10617s (17.0926GiB/s) | 20260506 00:16:59.484 | INFO | Downloaded 34GiB to GPU in 2.04564s (16.6207GiB/s) | 20260506 00:17:04.613 | INFO | Analyzing chunk frames = [7444992,7445504) | 20260506 00:17:04.613 | INFO | MemInfo: { | 20260506 00:17:04.613 | INFO | "gpu" : " 17 GB", | 20260506 00:17:04.613 | INFO | "huge" : "", | 20260506 00:17:04.613 | INFO | "huge_shared" : " 27 GB", | 20260506 00:17:04.613 | INFO | "malloc" : " 14 GB", | 20260506 00:17:04.613 | INFO | "pinned" : " 29 GB" | 20260506 00:17:04.613 | INFO | } | 20260506 00:17:04.613 | INFO | Perf report for the last 5.1301 seconds: | 20260506 00:17:04.613 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:17:04.613 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:04.613 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:04.613 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:04.613 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 00:17:04.613 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:04.613 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:04.613 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:17:04.614 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:04.614 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:04.614 | INFO | Uploaded 36GiB to GPU in 2.07603s (17.3408GiB/s) | 20260506 00:17:04.614 | INFO | Downloaded 34GiB to GPU in 2.03631s (16.6969GiB/s) | 20260506 00:17:09.759 | INFO | Analyzing chunk frames = [7445504,7446016) | 20260506 00:17:09.759 | INFO | MemInfo: { | 20260506 00:17:09.759 | INFO | "gpu" : " 17 GB", | 20260506 00:17:09.759 | INFO | "huge" : "", | 20260506 00:17:09.759 | INFO | "huge_shared" : " 27 GB", | 20260506 00:17:09.759 | INFO | "malloc" : " 14 GB", | 20260506 00:17:09.759 | INFO | "pinned" : " 29 GB" | 20260506 00:17:09.759 | INFO | } | 20260506 00:17:09.759 | INFO | Perf report for the last 5.14564 seconds: | 20260506 00:17:09.759 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:17:09.759 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:09.759 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:09.759 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:09.759 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260506 00:17:09.759 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:09.759 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:09.759 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 00:17:09.759 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:09.759 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:09.759 | INFO | Uploaded 36GiB to GPU in 2.0242s (17.7848GiB/s) | 20260506 00:17:09.759 | INFO | Downloaded 34GiB to GPU in 2.04038s (16.6636GiB/s) | 20260506 00:17:14.904 | INFO | Analyzing chunk frames = [7446016,7446528) | 20260506 00:17:14.904 | INFO | MemInfo: { | 20260506 00:17:14.904 | INFO | "gpu" : " 17 GB", | 20260506 00:17:14.904 | INFO | "huge" : "", | 20260506 00:17:14.904 | INFO | "huge_shared" : " 27 GB", | 20260506 00:17:14.904 | INFO | "malloc" : " 14 GB", | 20260506 00:17:14.904 | INFO | "pinned" : " 29 GB" | 20260506 00:17:14.904 | INFO | } | 20260506 00:17:14.904 | INFO | Perf report for the last 5.14538 seconds: | 20260506 00:17:14.904 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:17:14.904 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:14.904 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:14.904 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:14.904 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260506 00:17:14.904 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:14.904 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:14.904 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 00:17:14.904 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:14.904 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:14.905 | INFO | Uploaded 36GiB to GPU in 2.12134s (16.9704GiB/s) | 20260506 00:17:14.905 | INFO | Downloaded 34GiB to GPU in 2.01282s (16.8917GiB/s) | 20260506 00:17:14.905 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 00:17:14.905 | INFO | Total mem usage: 15.3 GiB | 20260506 00:17:14.905 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 00:17:14.905 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.11%) | 20260506 00:17:14.905 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 00:17:14.905 | INFO | High Water Breakdown: | 20260506 00:17:14.905 | INFO | 60.9% : 8.55 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 00:17:14.905 | INFO | 39% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 00:17:14.905 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 00:17:14.905 | INFO | ------------------------------------------ | 20260506 00:17:14.905 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:17:14.905 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 00:17:14.905 | INFO | Total mem usage: 26.5 GiB | 20260506 00:17:14.905 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 00:17:14.905 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 00:17:14.905 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260506 00:17:14.905 | INFO | High Water Breakdown: | 20260506 00:17:14.905 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:17:14.905 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:17:14.905 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:17:14.905 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:17:14.905 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:17:14.905 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:17:14.905 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 00:17:14.905 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 00:17:14.905 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:17:14.905 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 00:17:14.905 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 00:17:14.905 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 00:17:14.905 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 00:17:14.905 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 00:17:14.905 | INFO | ------------------------------------------ | 20260506 00:17:14.905 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:17:14.905 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 00:17:14.905 | INFO | Total mem usage: 25.6 GiB | 20260506 00:17:14.905 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 00:17:14.905 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 00:17:14.905 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 00:17:14.905 | INFO | High Water Breakdown: | 20260506 00:17:14.905 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 00:17:14.905 | INFO | ------------------------------------------ | 20260506 00:17:14.905 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:17:14.905 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 00:17:14.905 | INFO | Total mem usage: 15.7 GiB | 20260506 00:17:14.905 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 00:17:14.905 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 00:17:14.905 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 00:17:14.905 | INFO | High Water Breakdown: | 20260506 00:17:14.905 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 00:17:14.905 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 00:17:14.905 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 00:17:14.905 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 00:17:14.905 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 00:17:14.905 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 00:17:14.905 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 00:17:14.905 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 00:17:14.905 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 00:17:14.905 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:17:14.905 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 00:17:14.905 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:17:14.905 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 00:17:14.905 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 00:17:14.905 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 00:17:14.905 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 00:17:14.905 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:17:14.906 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 00:17:14.906 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:17:14.906 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:17:14.906 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:17:14.906 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 00:17:14.906 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 00:17:14.906 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:17:14.906 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 00:17:14.906 | INFO | ------------------------------------------ | 20260506 00:17:14.906 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:17:14.906 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 00:17:14.906 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:17:14.906 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:17:14.906 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.41e+01, 1.34e+01 / 8.33e-01 | 20260506 00:17:14.906 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.04e-01, 1.07e-02 / 4.02e-03 | 20260506 00:17:14.906 | INFO | PerfReport: 11.83%, Baseline -- 3.71e+00 / 4.52e+00, 5.72e-01 / 1.55e-01 | 20260506 00:17:14.906 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.54e+00, 1.90e-01 / 5.82e-02 | 20260506 00:17:14.906 | INFO | PerfReport: 15.99%, DME -- 4.01e+01 / 4.40e+01, 3.88e+00 / 1.11e+00 | 20260506 00:17:14.906 | INFO | PerfReport: 57.35%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.50e+00 / 1.05e+00 | 20260506 00:17:14.906 | INFO | PerfReport: 7.10%, PulseAccumulating -- 2.22e+00 / 2.76e+00, 4.23e-01 / 2.34e-01 | 20260506 00:17:14.906 | INFO | PerfReport: 3.09%, Metrics -- 9.67e-01 / 1.26e+00, 2.60e-01 / 8.79e-02 | 20260506 00:17:14.906 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.35e-01 / 3.90e-01, 5.66e-02 / 2.73e-02 | 20260506 00:17:14.906 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 00:17:14.906 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:17:14.906 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:17:14.906 | INFO | PerfReport: Full Execution -- 9.83e+01 / -nan, 2.45e+01 / 0.00e+00 | 20260506 00:17:14.906 | INFO | PerfReport: 22.32%, Upload -- 2.19e+01 / 0.00e+00, 7.53e+00 / 0.00e+00 | 20260506 00:17:14.906 | INFO | PerfReport: 21.84%, Download -- 2.15e+01 / 0.00e+00, 7.57e+00 / 0.00e+00 | 20260506 00:17:14.906 | INFO | PerfReport: 31.85%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 00:17:14.906 | INFO | PerfReport: 2.13%, LockUpload -- 2.10e+00 / 0.00e+00, 7.91e+00 / 0.00e+00 | 20260506 00:17:14.906 | INFO | PerfReport: 1.28%, LockDownload -- 1.26e+00 / 0.00e+00, 2.82e+00 / 0.00e+00 | 20260506 00:17:14.906 | INFO | PerfReport: 20.56%, LockCompute -- 2.02e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 00:17:14.906 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 2.72e-02 / 0.00e+00 | 20260506 00:17:15.022 | INFO | Starting a round of Baz IO | 20260506 00:17:18.171 | INFO | Populating RTMetrics report | 20260506 00:17:18.172 | INFO | RTMetrics report populated | 20260506 00:17:18.172 | INFO | Opening tmp RTMetrics json file | 20260506 00:17:18.173 | INFO | Serializing RTMetrics to json | 20260506 00:17:18.174 | INFO | Serialization done, swapping tmp file now... | 20260506 00:17:18.175 | INFO | RTMetrics done | 20260506 00:17:18.247 | INFO | Finished a round of Baz IO: Wrote 1.864 GiB in 3.225 seconds | 20260506 00:17:18.247 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.38% / 0.00% / 78.62% | 20260506 00:17:18.247 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0945449% / 0% / 7.94729e-05% | 20260506 00:17:18.247 | INFO | Baz IO overall: | 20260506 00:17:18.247 | INFO | BytesWritten : 4235028379916 | 20260506 00:17:18.247 | INFO | NumEvents : 1993316481390 | 20260506 00:17:20.042 | INFO | Analyzing chunk frames = [7446528,7447040) | 20260506 00:17:20.042 | INFO | MemInfo: { | 20260506 00:17:20.042 | INFO | "gpu" : " 17 GB", | 20260506 00:17:20.042 | INFO | "huge" : "", | 20260506 00:17:20.042 | INFO | "huge_shared" : " 27 GB", | 20260506 00:17:20.042 | INFO | "malloc" : " 14 GB", | 20260506 00:17:20.042 | INFO | "pinned" : " 29 GB" | 20260506 00:17:20.042 | INFO | } | 20260506 00:17:20.042 | INFO | Perf report for the last 5.13766 seconds: | 20260506 00:17:20.042 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:17:20.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:20.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:20.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:20.042 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260506 00:17:20.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:20.042 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260506 00:17:20.042 | INFO | PRE_HQ : 60.0%, 1.7, 54.4ms, 0 | 20260506 00:17:20.042 | INFO | BAZWRITER : 61.9%, 3.5, 112.4ms, 0 | 20260506 00:17:20.042 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:17:20.042 | INFO | Uploaded 36GiB to GPU in 2.18286s (16.4921GiB/s) | 20260506 00:17:20.042 | INFO | Downloaded 38GiB to GPU in 2.24723s (16.9097GiB/s) | 20260506 00:17:25.175 | INFO | Analyzing chunk frames = [7447040,7447552) | 20260506 00:17:25.175 | INFO | MemInfo: { | 20260506 00:17:25.175 | INFO | "gpu" : " 17 GB", | 20260506 00:17:25.175 | INFO | "huge" : "", | 20260506 00:17:25.175 | INFO | "huge_shared" : " 27 GB", | 20260506 00:17:25.175 | INFO | "malloc" : " 14 GB", | 20260506 00:17:25.175 | INFO | "pinned" : " 29 GB" | 20260506 00:17:25.175 | INFO | } | 20260506 00:17:25.175 | INFO | Perf report for the last 5.13295 seconds: | 20260506 00:17:25.175 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:17:25.175 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:25.175 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:25.175 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:25.175 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 00:17:25.175 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:25.175 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:25.175 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:17:25.175 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:25.175 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:25.175 | INFO | Uploaded 36GiB to GPU in 2.10973s (17.0638GiB/s) | 20260506 00:17:25.175 | INFO | Downloaded 34GiB to GPU in 2.03924s (16.6729GiB/s) | 20260506 00:17:30.314 | INFO | Analyzing chunk frames = [7447552,7448064) | 20260506 00:17:30.314 | INFO | MemInfo: { | 20260506 00:17:30.314 | INFO | "gpu" : " 17 GB", | 20260506 00:17:30.314 | INFO | "huge" : "", | 20260506 00:17:30.314 | INFO | "huge_shared" : " 27 GB", | 20260506 00:17:30.314 | INFO | "malloc" : " 14 GB", | 20260506 00:17:30.314 | INFO | "pinned" : " 29 GB" | 20260506 00:17:30.314 | INFO | } | 20260506 00:17:30.314 | INFO | Perf report for the last 5.13928 seconds: | 20260506 00:17:30.314 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:17:30.314 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:30.314 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:30.314 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:30.314 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260506 00:17:30.314 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:30.314 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:30.315 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 00:17:30.315 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:30.315 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:30.315 | INFO | Uploaded 36GiB to GPU in 2.10687s (17.0869GiB/s) | 20260506 00:17:30.315 | INFO | Downloaded 34GiB to GPU in 2.03117s (16.7392GiB/s) | 20260506 00:17:35.457 | INFO | Analyzing chunk frames = [7448064,7448576) | 20260506 00:17:35.457 | INFO | MemInfo: { | 20260506 00:17:35.457 | INFO | "gpu" : " 17 GB", | 20260506 00:17:35.457 | INFO | "huge" : "", | 20260506 00:17:35.457 | INFO | "huge_shared" : " 27 GB", | 20260506 00:17:35.457 | INFO | "malloc" : " 14 GB", | 20260506 00:17:35.457 | INFO | "pinned" : " 29 GB" | 20260506 00:17:35.457 | INFO | } | 20260506 00:17:35.457 | INFO | Perf report for the last 5.1425 seconds: | 20260506 00:17:35.457 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:17:35.457 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:35.457 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:35.457 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:35.457 | INFO | ANALYSIS : 61.6%, 3.0, 95.7ms, 0 | 20260506 00:17:35.457 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:35.457 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:35.457 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 00:17:35.457 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:35.457 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:35.457 | INFO | Uploaded 36GiB to GPU in 2.13486s (16.8629GiB/s) | 20260506 00:17:35.457 | INFO | Downloaded 34GiB to GPU in 2.02647s (16.778GiB/s) | 20260506 00:17:40.593 | INFO | Analyzing chunk frames = [7448576,7449088) | 20260506 00:17:40.593 | INFO | MemInfo: { | 20260506 00:17:40.593 | INFO | "gpu" : " 17 GB", | 20260506 00:17:40.593 | INFO | "huge" : "", | 20260506 00:17:40.593 | INFO | "huge_shared" : " 27 GB", | 20260506 00:17:40.593 | INFO | "malloc" : " 14 GB", | 20260506 00:17:40.593 | INFO | "pinned" : " 29 GB" | 20260506 00:17:40.593 | INFO | } | 20260506 00:17:40.593 | INFO | Perf report for the last 5.13623 seconds: | 20260506 00:17:40.593 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:17:40.593 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:40.593 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:40.593 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:40.593 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 00:17:40.593 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:40.593 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:40.593 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260506 00:17:40.593 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:40.593 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:40.593 | INFO | Uploaded 36GiB to GPU in 2.09522s (17.182GiB/s) | 20260506 00:17:40.594 | INFO | Downloaded 34GiB to GPU in 2.04003s (16.6664GiB/s) | 20260506 00:17:45.733 | INFO | Analyzing chunk frames = [7449088,7449600) | 20260506 00:17:45.733 | INFO | MemInfo: { | 20260506 00:17:45.733 | INFO | "gpu" : " 17 GB", | 20260506 00:17:45.733 | INFO | "huge" : "", | 20260506 00:17:45.733 | INFO | "huge_shared" : " 27 GB", | 20260506 00:17:45.733 | INFO | "malloc" : " 14 GB", | 20260506 00:17:45.733 | INFO | "pinned" : " 29 GB" | 20260506 00:17:45.733 | INFO | } | 20260506 00:17:45.733 | INFO | Perf report for the last 5.13969 seconds: | 20260506 00:17:45.733 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:17:45.733 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:45.733 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:45.733 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:45.733 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260506 00:17:45.733 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:45.733 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:45.733 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 00:17:45.733 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:45.733 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:45.733 | INFO | Uploaded 36GiB to GPU in 2.07247s (17.3706GiB/s) | 20260506 00:17:45.733 | INFO | Downloaded 34GiB to GPU in 2.03639s (16.6962GiB/s) | 20260506 00:17:50.871 | INFO | Analyzing chunk frames = [7449600,7450112) | 20260506 00:17:50.871 | INFO | MemInfo: { | 20260506 00:17:50.871 | INFO | "gpu" : " 17 GB", | 20260506 00:17:50.871 | INFO | "huge" : "", | 20260506 00:17:50.871 | INFO | "huge_shared" : " 27 GB", | 20260506 00:17:50.871 | INFO | "malloc" : " 14 GB", | 20260506 00:17:50.871 | INFO | "pinned" : " 29 GB" | 20260506 00:17:50.871 | INFO | } | 20260506 00:17:50.872 | INFO | Perf report for the last 5.139 seconds: | 20260506 00:17:50.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:17:50.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:50.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:50.872 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:50.872 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260506 00:17:50.872 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:50.872 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:50.872 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 00:17:50.872 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:50.872 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:50.872 | INFO | Uploaded 36GiB to GPU in 2.05431s (17.5241GiB/s) | 20260506 00:17:50.872 | INFO | Downloaded 34GiB to GPU in 2.04247s (16.6465GiB/s) | 20260506 00:17:56.012 | INFO | Analyzing chunk frames = [7450112,7450624) | 20260506 00:17:56.012 | INFO | MemInfo: { | 20260506 00:17:56.012 | INFO | "gpu" : " 17 GB", | 20260506 00:17:56.012 | INFO | "huge" : "", | 20260506 00:17:56.012 | INFO | "huge_shared" : " 27 GB", | 20260506 00:17:56.012 | INFO | "malloc" : " 14 GB", | 20260506 00:17:56.012 | INFO | "pinned" : " 29 GB" | 20260506 00:17:56.013 | INFO | } | 20260506 00:17:56.013 | INFO | Perf report for the last 5.14093 seconds: | 20260506 00:17:56.013 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:17:56.013 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:56.013 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:56.013 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:56.013 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260506 00:17:56.013 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:56.013 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:56.013 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260506 00:17:56.013 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:56.013 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:17:56.013 | INFO | Uploaded 36GiB to GPU in 2.11964s (16.984GiB/s) | 20260506 00:17:56.013 | INFO | Downloaded 34GiB to GPU in 2.01336s (16.8872GiB/s) | 20260506 00:17:56.130 | INFO | Starting a round of Baz IO | 20260506 00:17:59.275 | INFO | Populating RTMetrics report | 20260506 00:17:59.275 | INFO | RTMetrics report populated | 20260506 00:17:59.275 | INFO | Opening tmp RTMetrics json file | 20260506 00:17:59.278 | INFO | Serializing RTMetrics to json | 20260506 00:17:59.279 | INFO | Serialization done, swapping tmp file now... | 20260506 00:17:59.279 | INFO | RTMetrics done | 20260506 00:17:59.358 | INFO | Finished a round of Baz IO: Wrote 1.866 GiB in 3.227 seconds | 20260506 00:17:59.358 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.38% / 0.00% / 78.62% | 20260506 00:17:59.358 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0947197% / 0% / 8.74201e-05% | 20260506 00:17:59.358 | INFO | Baz IO overall: | 20260506 00:17:59.358 | INFO | BytesWritten : 4237031591100 | 20260506 00:17:59.358 | INFO | NumEvents : 1994232917260 | 20260506 00:18:01.157 | INFO | Analyzing chunk frames = [7450624,7451136) | 20260506 00:18:01.157 | INFO | MemInfo: { | 20260506 00:18:01.157 | INFO | "gpu" : " 17 GB", | 20260506 00:18:01.157 | INFO | "huge" : "", | 20260506 00:18:01.157 | INFO | "huge_shared" : " 27 GB", | 20260506 00:18:01.157 | INFO | "malloc" : " 14 GB", | 20260506 00:18:01.157 | INFO | "pinned" : " 29 GB" | 20260506 00:18:01.157 | INFO | } | 20260506 00:18:01.157 | INFO | Perf report for the last 5.14434 seconds: | 20260506 00:18:01.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:18:01.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:01.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:01.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:01.157 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260506 00:18:01.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:01.157 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260506 00:18:01.157 | INFO | PRE_HQ : 59.8%, 1.7, 54.1ms, 0 | 20260506 00:18:01.157 | INFO | BAZWRITER : 61.9%, 3.5, 112.6ms, 0 | 20260506 00:18:01.157 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:18:01.157 | INFO | Uploaded 36GiB to GPU in 2.16824s (16.6033GiB/s) | 20260506 00:18:01.157 | INFO | Downloaded 38GiB to GPU in 2.23976s (16.9661GiB/s) | 20260506 00:18:06.291 | INFO | Analyzing chunk frames = [7451136,7451648) | 20260506 00:18:06.291 | INFO | MemInfo: { | 20260506 00:18:06.291 | INFO | "gpu" : " 17 GB", | 20260506 00:18:06.291 | INFO | "huge" : "", | 20260506 00:18:06.292 | INFO | "huge_shared" : " 27 GB", | 20260506 00:18:06.292 | INFO | "malloc" : " 14 GB", | 20260506 00:18:06.292 | INFO | "pinned" : " 29 GB" | 20260506 00:18:06.292 | INFO | } | 20260506 00:18:06.292 | INFO | Perf report for the last 5.13457 seconds: | 20260506 00:18:06.292 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:18:06.292 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:06.292 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:06.292 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:06.292 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 00:18:06.292 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:06.292 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:06.292 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 00:18:06.292 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:06.292 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:06.292 | INFO | Uploaded 36GiB to GPU in 1.99397s (18.0545GiB/s) | 20260506 00:18:06.292 | INFO | Downloaded 34GiB to GPU in 1.99763s (17.0201GiB/s) | 20260506 00:18:11.427 | INFO | Analyzing chunk frames = [7451648,7452160) | 20260506 00:18:11.427 | INFO | MemInfo: { | 20260506 00:18:11.427 | INFO | "gpu" : " 17 GB", | 20260506 00:18:11.427 | INFO | "huge" : "", | 20260506 00:18:11.427 | INFO | "huge_shared" : " 27 GB", | 20260506 00:18:11.427 | INFO | "malloc" : " 14 GB", | 20260506 00:18:11.427 | INFO | "pinned" : " 29 GB" | 20260506 00:18:11.427 | INFO | } | 20260506 00:18:11.427 | INFO | Perf report for the last 5.13591 seconds: | 20260506 00:18:11.428 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:18:11.428 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:11.428 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:11.428 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:11.428 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 00:18:11.428 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:11.428 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:11.428 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 00:18:11.428 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:11.428 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:11.428 | INFO | Uploaded 36GiB to GPU in 2.07333s (17.3633GiB/s) | 20260506 00:18:11.428 | INFO | Downloaded 34GiB to GPU in 2.02201s (16.815GiB/s) | 20260506 00:18:16.566 | INFO | Analyzing chunk frames = [7452160,7452672) | 20260506 00:18:16.566 | INFO | MemInfo: { | 20260506 00:18:16.566 | INFO | "gpu" : " 17 GB", | 20260506 00:18:16.566 | INFO | "huge" : "", | 20260506 00:18:16.566 | INFO | "huge_shared" : " 27 GB", | 20260506 00:18:16.566 | INFO | "malloc" : " 14 GB", | 20260506 00:18:16.566 | INFO | "pinned" : " 29 GB" | 20260506 00:18:16.566 | INFO | } | 20260506 00:18:16.567 | INFO | Perf report for the last 5.13903 seconds: | 20260506 00:18:16.567 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:18:16.567 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:16.567 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:16.567 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:16.567 | INFO | ANALYSIS : 61.8%, 3.0, 96.0ms, 0 | 20260506 00:18:16.567 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:16.567 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:16.567 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 00:18:16.567 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:16.567 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:16.567 | INFO | Uploaded 36GiB to GPU in 2.14228s (16.8045GiB/s) | 20260506 00:18:16.567 | INFO | Downloaded 34GiB to GPU in 2.03309s (16.7233GiB/s) | 20260506 00:18:21.711 | INFO | Analyzing chunk frames = [7452672,7453184) | 20260506 00:18:21.711 | INFO | MemInfo: { | 20260506 00:18:21.711 | INFO | "gpu" : " 17 GB", | 20260506 00:18:21.711 | INFO | "huge" : "", | 20260506 00:18:21.711 | INFO | "huge_shared" : " 27 GB", | 20260506 00:18:21.711 | INFO | "malloc" : " 14 GB", | 20260506 00:18:21.711 | INFO | "pinned" : " 29 GB" | 20260506 00:18:21.711 | INFO | } | 20260506 00:18:21.711 | INFO | Perf report for the last 5.14443 seconds: | 20260506 00:18:21.711 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:18:21.711 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:21.711 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:21.711 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:21.711 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 00:18:21.711 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:21.711 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:21.711 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 00:18:21.711 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:21.711 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:21.711 | INFO | Uploaded 36GiB to GPU in 2.09876s (17.153GiB/s) | 20260506 00:18:21.711 | INFO | Downloaded 34GiB to GPU in 2.04701s (16.6096GiB/s) | 20260506 00:18:26.858 | INFO | Analyzing chunk frames = [7453184,7453696) | 20260506 00:18:26.858 | INFO | MemInfo: { | 20260506 00:18:26.858 | INFO | "gpu" : " 17 GB", | 20260506 00:18:26.858 | INFO | "huge" : "", | 20260506 00:18:26.858 | INFO | "huge_shared" : " 27 GB", | 20260506 00:18:26.858 | INFO | "malloc" : " 14 GB", | 20260506 00:18:26.858 | INFO | "pinned" : " 29 GB" | 20260506 00:18:26.858 | INFO | } | 20260506 00:18:26.858 | INFO | Perf report for the last 5.14686 seconds: | 20260506 00:18:26.858 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:18:26.858 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:26.858 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:26.858 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:26.858 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 00:18:26.858 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:26.858 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:26.858 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 00:18:26.858 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:26.858 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:26.858 | INFO | Uploaded 36GiB to GPU in 2.0781s (17.3235GiB/s) | 20260506 00:18:26.858 | INFO | Downloaded 34GiB to GPU in 2.04051s (16.6625GiB/s) | 20260506 00:18:31.991 | INFO | Analyzing chunk frames = [7453696,7454208) | 20260506 00:18:31.991 | INFO | MemInfo: { | 20260506 00:18:31.991 | INFO | "gpu" : " 17 GB", | 20260506 00:18:31.991 | INFO | "huge" : "", | 20260506 00:18:31.991 | INFO | "huge_shared" : " 27 GB", | 20260506 00:18:31.991 | INFO | "malloc" : " 14 GB", | 20260506 00:18:31.991 | INFO | "pinned" : " 29 GB" | 20260506 00:18:31.991 | INFO | } | 20260506 00:18:31.992 | INFO | Perf report for the last 5.13377 seconds: | 20260506 00:18:31.992 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:18:31.992 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:31.992 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:31.992 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:31.992 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260506 00:18:31.992 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:31.992 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:31.992 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 00:18:31.992 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:31.992 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:31.992 | INFO | Uploaded 36GiB to GPU in 2.09974s (17.145GiB/s) | 20260506 00:18:31.992 | INFO | Downloaded 34GiB to GPU in 2.0765s (16.3737GiB/s) | 20260506 00:18:37.124 | INFO | Analyzing chunk frames = [7454208,7454720) | 20260506 00:18:37.124 | INFO | MemInfo: { | 20260506 00:18:37.124 | INFO | "gpu" : " 17 GB", | 20260506 00:18:37.124 | INFO | "huge" : "", | 20260506 00:18:37.124 | INFO | "huge_shared" : " 27 GB", | 20260506 00:18:37.124 | INFO | "malloc" : " 14 GB", | 20260506 00:18:37.124 | INFO | "pinned" : " 29 GB" | 20260506 00:18:37.124 | INFO | } | 20260506 00:18:37.124 | INFO | Perf report for the last 5.13273 seconds: | 20260506 00:18:37.124 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:18:37.124 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:37.124 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:37.124 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:37.124 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 00:18:37.124 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:37.125 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:37.125 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 00:18:37.125 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:37.125 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:37.125 | INFO | Uploaded 36GiB to GPU in 2.12742s (16.9219GiB/s) | 20260506 00:18:37.125 | INFO | Downloaded 34GiB to GPU in 2.02764s (16.7682GiB/s) | 20260506 00:18:37.125 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 00:18:37.125 | INFO | Total mem usage: 15.3 GiB | 20260506 00:18:37.125 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 00:18:37.125 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.12%) | 20260506 00:18:37.125 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 00:18:37.125 | INFO | High Water Breakdown: | 20260506 00:18:37.126 | INFO | 60.9% : 8.55 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 00:18:37.126 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 00:18:37.126 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 00:18:37.126 | INFO | ------------------------------------------ | 20260506 00:18:37.126 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:18:37.126 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 00:18:37.126 | INFO | Total mem usage: 26.5 GiB | 20260506 00:18:37.126 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 00:18:37.126 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 00:18:37.126 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260506 00:18:37.126 | INFO | High Water Breakdown: | 20260506 00:18:37.126 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:18:37.126 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:18:37.126 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:18:37.126 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:18:37.126 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:18:37.126 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:18:37.126 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 00:18:37.126 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 00:18:37.126 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:18:37.126 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 00:18:37.126 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 00:18:37.126 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 00:18:37.126 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 00:18:37.126 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 00:18:37.126 | INFO | ------------------------------------------ | 20260506 00:18:37.126 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:18:37.126 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 00:18:37.126 | INFO | Total mem usage: 25.6 GiB | 20260506 00:18:37.126 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 00:18:37.126 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 00:18:37.126 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 00:18:37.126 | INFO | High Water Breakdown: | 20260506 00:18:37.126 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 00:18:37.126 | INFO | ------------------------------------------ | 20260506 00:18:37.126 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:18:37.126 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 00:18:37.126 | INFO | Total mem usage: 15.7 GiB | 20260506 00:18:37.126 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 00:18:37.126 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 00:18:37.126 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 00:18:37.126 | INFO | High Water Breakdown: | 20260506 00:18:37.126 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 00:18:37.126 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 00:18:37.126 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 00:18:37.126 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 00:18:37.126 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 00:18:37.126 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 00:18:37.126 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 00:18:37.126 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 00:18:37.126 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 00:18:37.126 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:18:37.126 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 00:18:37.126 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:18:37.126 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 00:18:37.126 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 00:18:37.126 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 00:18:37.126 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 00:18:37.126 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:18:37.126 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 00:18:37.126 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:18:37.126 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:18:37.126 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:18:37.126 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 00:18:37.126 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 00:18:37.126 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:18:37.126 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 00:18:37.126 | INFO | ------------------------------------------ | 20260506 00:18:37.126 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:18:37.126 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 00:18:37.126 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:18:37.126 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:18:37.126 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.40e+01, 1.33e+01 / 9.86e-01 | 20260506 00:18:37.127 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.04e-01, 1.03e-02 / 4.70e-03 | 20260506 00:18:37.127 | INFO | PerfReport: 11.84%, Baseline -- 3.71e+00 / 4.49e+00, 5.70e-01 / 2.26e-01 | 20260506 00:18:37.127 | INFO | PerfReport: 3.29%, Histogram -- 1.03e+00 / 1.51e+00, 1.90e-01 / 1.09e-01 | 20260506 00:18:37.127 | INFO | PerfReport: 15.98%, DME -- 4.01e+01 / 4.41e+01, 3.83e+00 / 1.20e+00 | 20260506 00:18:37.127 | INFO | PerfReport: 57.37%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.53e+00 / 1.24e+00 | 20260506 00:18:37.127 | INFO | PerfReport: 7.08%, PulseAccumulating -- 2.22e+00 / 2.76e+00, 4.21e-01 / 2.54e-01 | 20260506 00:18:37.127 | INFO | PerfReport: 3.08%, Metrics -- 9.65e-01 / 1.25e+00, 2.58e-01 / 1.33e-01 | 20260506 00:18:37.127 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.35e-01 / 3.87e-01, 5.70e-02 / 3.38e-02 | 20260506 00:18:37.127 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 00:18:37.127 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:18:37.127 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:18:37.127 | INFO | PerfReport: Full Execution -- 9.82e+01 / -nan, 2.44e+01 / 0.00e+00 | 20260506 00:18:37.127 | INFO | PerfReport: 22.31%, Upload -- 2.19e+01 / 0.00e+00, 7.48e+00 / 0.00e+00 | 20260506 00:18:37.127 | INFO | PerfReport: 21.85%, Download -- 2.15e+01 / 0.00e+00, 7.55e+00 / 0.00e+00 | 20260506 00:18:37.127 | INFO | PerfReport: 31.91%, Compute -- 3.13e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 00:18:37.127 | INFO | PerfReport: 2.10%, LockUpload -- 2.06e+00 / 0.00e+00, 7.84e+00 / 0.00e+00 | 20260506 00:18:37.127 | INFO | PerfReport: 1.29%, LockDownload -- 1.27e+00 / 0.00e+00, 2.84e+00 / 0.00e+00 | 20260506 00:18:37.127 | INFO | PerfReport: 20.53%, LockCompute -- 2.02e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 00:18:37.127 | INFO | PerfReport: 0.01%, Misc -- 1.24e-02 / 0.00e+00, 1.14e-02 / 0.00e+00 | 20260506 00:18:37.251 | INFO | Starting a round of Baz IO | 20260506 00:18:40.390 | INFO | Populating RTMetrics report | 20260506 00:18:40.390 | INFO | RTMetrics report populated | 20260506 00:18:40.390 | INFO | Opening tmp RTMetrics json file | 20260506 00:18:40.392 | INFO | Serializing RTMetrics to json | 20260506 00:18:40.395 | INFO | Serialization done, swapping tmp file now... | 20260506 00:18:40.395 | INFO | RTMetrics done | 20260506 00:18:40.468 | INFO | Finished a round of Baz IO: Wrote 1.864 GiB in 3.217 seconds | 20260506 00:18:40.468 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.38% / 0.00% / 78.62% | 20260506 00:18:40.468 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0947475% / 0% / 9.13938e-05% | 20260506 00:18:40.468 | INFO | Baz IO overall: | 20260506 00:18:40.468 | INFO | BytesWritten : 4239033078225 | 20260506 00:18:40.468 | INFO | NumEvents : 1995148410191 | 20260506 00:18:42.268 | INFO | Analyzing chunk frames = [7454720,7455232) | 20260506 00:18:42.268 | INFO | MemInfo: { | 20260506 00:18:42.268 | INFO | "gpu" : " 17 GB", | 20260506 00:18:42.268 | INFO | "huge" : "", | 20260506 00:18:42.268 | INFO | "huge_shared" : " 27 GB", | 20260506 00:18:42.268 | INFO | "malloc" : " 14 GB", | 20260506 00:18:42.268 | INFO | "pinned" : " 29 GB" | 20260506 00:18:42.268 | INFO | } | 20260506 00:18:42.268 | INFO | Perf report for the last 5.14375 seconds: | 20260506 00:18:42.268 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:18:42.268 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:42.268 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:42.268 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:42.268 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260506 00:18:42.268 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:42.268 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260506 00:18:42.268 | INFO | PRE_HQ : 59.8%, 1.7, 54.2ms, 0 | 20260506 00:18:42.268 | INFO | BAZWRITER : 61.8%, 3.6, 113.2ms, 0 | 20260506 00:18:42.269 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 00:18:42.269 | INFO | Uploaded 36GiB to GPU in 2.16702s (16.6127GiB/s) | 20260506 00:18:42.269 | INFO | Downloaded 38GiB to GPU in 2.24231s (16.9468GiB/s) | 20260506 00:18:47.406 | INFO | Analyzing chunk frames = [7455232,7455744) | 20260506 00:18:47.406 | INFO | MemInfo: { | 20260506 00:18:47.406 | INFO | "gpu" : " 17 GB", | 20260506 00:18:47.406 | INFO | "huge" : "", | 20260506 00:18:47.406 | INFO | "huge_shared" : " 27 GB", | 20260506 00:18:47.406 | INFO | "malloc" : " 14 GB", | 20260506 00:18:47.406 | INFO | "pinned" : " 29 GB" | 20260506 00:18:47.406 | INFO | } | 20260506 00:18:47.406 | INFO | Perf report for the last 5.1377 seconds: | 20260506 00:18:47.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:18:47.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:47.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:47.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:47.406 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 00:18:47.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:47.406 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:47.406 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 00:18:47.406 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:47.406 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:47.406 | INFO | Uploaded 36GiB to GPU in 1.97479s (18.2298GiB/s) | 20260506 00:18:47.406 | INFO | Downloaded 34GiB to GPU in 1.99279s (17.0615GiB/s) | 20260506 00:18:52.547 | INFO | Analyzing chunk frames = [7455744,7456256) | 20260506 00:18:52.547 | INFO | MemInfo: { | 20260506 00:18:52.547 | INFO | "gpu" : " 17 GB", | 20260506 00:18:52.547 | INFO | "huge" : "", | 20260506 00:18:52.547 | INFO | "huge_shared" : " 27 GB", | 20260506 00:18:52.547 | INFO | "malloc" : " 14 GB", | 20260506 00:18:52.547 | INFO | "pinned" : " 29 GB" | 20260506 00:18:52.547 | INFO | } | 20260506 00:18:52.547 | INFO | Perf report for the last 5.14099 seconds: | 20260506 00:18:52.547 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:18:52.547 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:52.547 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:52.547 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:52.547 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 00:18:52.547 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:52.547 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:52.547 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 00:18:52.547 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:52.547 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:52.547 | INFO | Uploaded 36GiB to GPU in 2.08526s (17.2641GiB/s) | 20260506 00:18:52.547 | INFO | Downloaded 34GiB to GPU in 2.02506s (16.7896GiB/s) | 20260506 00:18:57.684 | INFO | Analyzing chunk frames = [7456256,7456768) | 20260506 00:18:57.684 | INFO | MemInfo: { | 20260506 00:18:57.684 | INFO | "gpu" : " 17 GB", | 20260506 00:18:57.684 | INFO | "huge" : "", | 20260506 00:18:57.684 | INFO | "huge_shared" : " 27 GB", | 20260506 00:18:57.684 | INFO | "malloc" : " 14 GB", | 20260506 00:18:57.684 | INFO | "pinned" : " 29 GB" | 20260506 00:18:57.684 | INFO | } | 20260506 00:18:57.685 | INFO | Perf report for the last 5.13763 seconds: | 20260506 00:18:57.685 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:18:57.685 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:57.685 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:57.685 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:57.685 | INFO | ANALYSIS : 61.6%, 3.0, 95.6ms, 0 | 20260506 00:18:57.685 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:57.685 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:57.685 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 00:18:57.685 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:57.685 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:18:57.685 | INFO | Uploaded 36GiB to GPU in 2.13454s (16.8655GiB/s) | 20260506 00:18:57.685 | INFO | Downloaded 34GiB to GPU in 2.01223s (16.8967GiB/s) | 20260506 00:19:02.822 | INFO | Analyzing chunk frames = [7456768,7457280) | 20260506 00:19:02.822 | INFO | MemInfo: { | 20260506 00:19:02.822 | INFO | "gpu" : " 17 GB", | 20260506 00:19:02.822 | INFO | "huge" : "", | 20260506 00:19:02.822 | INFO | "huge_shared" : " 27 GB", | 20260506 00:19:02.822 | INFO | "malloc" : " 14 GB", | 20260506 00:19:02.823 | INFO | "pinned" : " 29 GB" | 20260506 00:19:02.823 | INFO | } | 20260506 00:19:02.823 | INFO | Perf report for the last 5.13803 seconds: | 20260506 00:19:02.823 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:19:02.823 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:02.823 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:02.823 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:02.823 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 00:19:02.823 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:02.823 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:02.823 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260506 00:19:02.823 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:02.823 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:02.823 | INFO | Uploaded 36GiB to GPU in 2.10191s (17.1273GiB/s) | 20260506 00:19:02.823 | INFO | Downloaded 34GiB to GPU in 2.04446s (16.6303GiB/s) | 20260506 00:19:07.964 | INFO | Analyzing chunk frames = [7457280,7457792) | 20260506 00:19:07.964 | INFO | MemInfo: { | 20260506 00:19:07.964 | INFO | "gpu" : " 17 GB", | 20260506 00:19:07.964 | INFO | "huge" : "", | 20260506 00:19:07.964 | INFO | "huge_shared" : " 27 GB", | 20260506 00:19:07.964 | INFO | "malloc" : " 14 GB", | 20260506 00:19:07.964 | INFO | "pinned" : " 29 GB" | 20260506 00:19:07.964 | INFO | } | 20260506 00:19:07.964 | INFO | Perf report for the last 5.14182 seconds: | 20260506 00:19:07.965 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:19:07.965 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:07.965 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:07.965 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:07.965 | INFO | ANALYSIS : 61.2%, 3.0, 95.7ms, 0 | 20260506 00:19:07.965 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:07.965 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:07.965 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:19:07.965 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:07.965 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:07.965 | INFO | Uploaded 36GiB to GPU in 2.066s (17.425GiB/s) | 20260506 00:19:07.965 | INFO | Downloaded 34GiB to GPU in 2.03655s (16.6949GiB/s) | 20260506 00:19:13.107 | INFO | Analyzing chunk frames = [7457792,7458304) | 20260506 00:19:13.107 | INFO | MemInfo: { | 20260506 00:19:13.107 | INFO | "gpu" : " 17 GB", | 20260506 00:19:13.107 | INFO | "huge" : "", | 20260506 00:19:13.107 | INFO | "huge_shared" : " 27 GB", | 20260506 00:19:13.107 | INFO | "malloc" : " 14 GB", | 20260506 00:19:13.107 | INFO | "pinned" : " 29 GB" | 20260506 00:19:13.107 | INFO | } | 20260506 00:19:13.107 | INFO | Perf report for the last 5.14279 seconds: | 20260506 00:19:13.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:19:13.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:13.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:13.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:13.107 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260506 00:19:13.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:13.107 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:13.107 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 00:19:13.107 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:13.107 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:13.107 | INFO | Uploaded 36GiB to GPU in 2.0704s (17.388GiB/s) | 20260506 00:19:13.107 | INFO | Downloaded 34GiB to GPU in 2.06584s (16.4582GiB/s) | 20260506 00:19:18.238 | INFO | Analyzing chunk frames = [7458304,7458816) | 20260506 00:19:18.238 | INFO | MemInfo: { | 20260506 00:19:18.238 | INFO | "gpu" : " 17 GB", | 20260506 00:19:18.238 | INFO | "huge" : "", | 20260506 00:19:18.238 | INFO | "huge_shared" : " 27 GB", | 20260506 00:19:18.238 | INFO | "malloc" : " 14 GB", | 20260506 00:19:18.238 | INFO | "pinned" : " 29 GB" | 20260506 00:19:18.238 | INFO | } | 20260506 00:19:18.238 | INFO | Perf report for the last 5.13078 seconds: | 20260506 00:19:18.238 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:19:18.238 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:18.238 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:18.238 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:18.238 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260506 00:19:18.238 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:18.238 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:18.238 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 00:19:18.238 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:18.238 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:18.238 | INFO | Uploaded 36GiB to GPU in 2.13661s (16.8491GiB/s) | 20260506 00:19:18.238 | INFO | Downloaded 34GiB to GPU in 2.02815s (16.7641GiB/s) | 20260506 00:19:18.378 | INFO | Starting a round of Baz IO | 20260506 00:19:21.513 | INFO | Populating RTMetrics report | 20260506 00:19:21.514 | INFO | RTMetrics report populated | 20260506 00:19:21.514 | INFO | Opening tmp RTMetrics json file | 20260506 00:19:21.515 | INFO | Serializing RTMetrics to json | 20260506 00:19:21.517 | INFO | Serialization done, swapping tmp file now... | 20260506 00:19:21.520 | INFO | RTMetrics done | 20260506 00:19:21.595 | INFO | Finished a round of Baz IO: Wrote 1.862 GiB in 3.216 seconds | 20260506 00:19:21.595 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.38% / 0.00% / 78.62% | 20260506 00:19:21.595 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0942111% / 0% / 7.94729e-05% | 20260506 00:19:21.595 | INFO | Baz IO overall: | 20260506 00:19:21.595 | INFO | BytesWritten : 4241032902704 | 20260506 00:19:21.595 | INFO | NumEvents : 1996062753374 | 20260506 00:19:23.376 | INFO | Analyzing chunk frames = [7458816,7459328) | 20260506 00:19:23.376 | INFO | MemInfo: { | 20260506 00:19:23.376 | INFO | "gpu" : " 17 GB", | 20260506 00:19:23.376 | INFO | "huge" : "", | 20260506 00:19:23.376 | INFO | "huge_shared" : " 27 GB", | 20260506 00:19:23.376 | INFO | "malloc" : " 14 GB", | 20260506 00:19:23.376 | INFO | "pinned" : " 29 GB" | 20260506 00:19:23.376 | INFO | } | 20260506 00:19:23.376 | INFO | Perf report for the last 5.13843 seconds: | 20260506 00:19:23.377 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:19:23.377 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:23.377 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:23.377 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:23.377 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260506 00:19:23.377 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:23.377 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260506 00:19:23.377 | INFO | PRE_HQ : 59.6%, 1.7, 53.9ms, 0 | 20260506 00:19:23.377 | INFO | BAZWRITER : 62.0%, 3.6, 113.4ms, 0 | 20260506 00:19:23.377 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 00:19:23.377 | INFO | Uploaded 36GiB to GPU in 2.1778s (16.5304GiB/s) | 20260506 00:19:23.377 | INFO | Downloaded 38GiB to GPU in 2.24295s (16.942GiB/s) | 20260506 00:19:28.521 | INFO | Analyzing chunk frames = [7459328,7459840) | 20260506 00:19:28.521 | INFO | MemInfo: { | 20260506 00:19:28.521 | INFO | "gpu" : " 17 GB", | 20260506 00:19:28.521 | INFO | "huge" : "", | 20260506 00:19:28.521 | INFO | "huge_shared" : " 27 GB", | 20260506 00:19:28.521 | INFO | "malloc" : " 14 GB", | 20260506 00:19:28.521 | INFO | "pinned" : " 29 GB" | 20260506 00:19:28.521 | INFO | } | 20260506 00:19:28.521 | INFO | Perf report for the last 5.14469 seconds: | 20260506 00:19:28.521 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:19:28.521 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:28.521 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:28.521 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:28.521 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260506 00:19:28.521 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:28.521 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:28.521 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:19:28.521 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:28.521 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:28.521 | INFO | Uploaded 36GiB to GPU in 2.0959s (17.1764GiB/s) | 20260506 00:19:28.522 | INFO | Downloaded 34GiB to GPU in 2.03114s (16.7393GiB/s) | 20260506 00:19:33.664 | INFO | Analyzing chunk frames = [7459840,7460352) | 20260506 00:19:33.664 | INFO | MemInfo: { | 20260506 00:19:33.664 | INFO | "gpu" : " 17 GB", | 20260506 00:19:33.664 | INFO | "huge" : "", | 20260506 00:19:33.664 | INFO | "huge_shared" : " 27 GB", | 20260506 00:19:33.664 | INFO | "malloc" : " 14 GB", | 20260506 00:19:33.664 | INFO | "pinned" : " 29 GB" | 20260506 00:19:33.664 | INFO | } | 20260506 00:19:33.664 | INFO | Perf report for the last 5.14245 seconds: | 20260506 00:19:33.664 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:19:33.664 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:33.664 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:33.664 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:33.664 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 00:19:33.664 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:33.664 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:33.664 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:19:33.664 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:33.664 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:33.664 | INFO | Uploaded 36GiB to GPU in 2.07974s (17.3099GiB/s) | 20260506 00:19:33.664 | INFO | Downloaded 34GiB to GPU in 2.02092s (16.824GiB/s) | 20260506 00:19:38.806 | INFO | Analyzing chunk frames = [7460352,7460864) | 20260506 00:19:38.806 | INFO | MemInfo: { | 20260506 00:19:38.806 | INFO | "gpu" : " 17 GB", | 20260506 00:19:38.806 | INFO | "huge" : "", | 20260506 00:19:38.806 | INFO | "huge_shared" : " 27 GB", | 20260506 00:19:38.806 | INFO | "malloc" : " 14 GB", | 20260506 00:19:38.806 | INFO | "pinned" : " 29 GB" | 20260506 00:19:38.806 | INFO | } | 20260506 00:19:38.806 | INFO | Perf report for the last 5.14249 seconds: | 20260506 00:19:38.806 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:19:38.806 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:38.806 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:38.806 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:38.806 | INFO | ANALYSIS : 61.2%, 3.0, 95.1ms, 0 | 20260506 00:19:38.806 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:38.806 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:38.806 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:19:38.806 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:38.806 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:38.806 | INFO | Uploaded 36GiB to GPU in 2.09002s (17.2247GiB/s) | 20260506 00:19:38.806 | INFO | Downloaded 34GiB to GPU in 2.02531s (16.7876GiB/s) | 20260506 00:19:43.934 | INFO | Analyzing chunk frames = [7460864,7461376) | 20260506 00:19:43.934 | INFO | MemInfo: { | 20260506 00:19:43.934 | INFO | "gpu" : " 17 GB", | 20260506 00:19:43.934 | INFO | "huge" : "", | 20260506 00:19:43.934 | INFO | "huge_shared" : " 27 GB", | 20260506 00:19:43.934 | INFO | "malloc" : " 14 GB", | 20260506 00:19:43.934 | INFO | "pinned" : " 29 GB" | 20260506 00:19:43.934 | INFO | } | 20260506 00:19:43.934 | INFO | Perf report for the last 5.12819 seconds: | 20260506 00:19:43.934 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:19:43.934 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:43.934 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:43.934 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:43.934 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 00:19:43.934 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:43.934 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:43.934 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 00:19:43.934 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:43.934 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:43.934 | INFO | Uploaded 36GiB to GPU in 2.10635s (17.0911GiB/s) | 20260506 00:19:43.935 | INFO | Downloaded 34GiB to GPU in 2.04363s (16.6371GiB/s) | 20260506 00:19:49.077 | INFO | Analyzing chunk frames = [7461376,7461888) | 20260506 00:19:49.077 | INFO | MemInfo: { | 20260506 00:19:49.077 | INFO | "gpu" : " 17 GB", | 20260506 00:19:49.077 | INFO | "huge" : "", | 20260506 00:19:49.077 | INFO | "huge_shared" : " 27 GB", | 20260506 00:19:49.077 | INFO | "malloc" : " 14 GB", | 20260506 00:19:49.077 | INFO | "pinned" : " 29 GB" | 20260506 00:19:49.077 | INFO | } | 20260506 00:19:49.078 | INFO | Perf report for the last 5.14327 seconds: | 20260506 00:19:49.078 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:19:49.078 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:49.078 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:49.078 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:49.078 | INFO | ANALYSIS : 61.3%, 3.0, 95.8ms, 0 | 20260506 00:19:49.078 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:49.078 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:49.078 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:19:49.078 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:49.078 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:49.078 | INFO | Uploaded 36GiB to GPU in 2.07775s (17.3264GiB/s) | 20260506 00:19:49.078 | INFO | Downloaded 34GiB to GPU in 2.03589s (16.7003GiB/s) | 20260506 00:19:54.225 | INFO | Analyzing chunk frames = [7461888,7462400) | 20260506 00:19:54.225 | INFO | MemInfo: { | 20260506 00:19:54.225 | INFO | "gpu" : " 17 GB", | 20260506 00:19:54.225 | INFO | "huge" : "", | 20260506 00:19:54.225 | INFO | "huge_shared" : " 27 GB", | 20260506 00:19:54.225 | INFO | "malloc" : " 14 GB", | 20260506 00:19:54.225 | INFO | "pinned" : " 29 GB" | 20260506 00:19:54.225 | INFO | } | 20260506 00:19:54.225 | INFO | Perf report for the last 5.1473 seconds: | 20260506 00:19:54.225 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:19:54.225 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:54.225 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:54.225 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:54.225 | INFO | ANALYSIS : 62.2%, 3.0, 97.3ms, 0 | 20260506 00:19:54.225 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:54.225 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:54.225 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 00:19:54.225 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:54.225 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:54.225 | INFO | Uploaded 36GiB to GPU in 2.07281s (17.3678GiB/s) | 20260506 00:19:54.225 | INFO | Downloaded 34GiB to GPU in 2.06709s (16.4482GiB/s) | 20260506 00:19:59.357 | INFO | Analyzing chunk frames = [7462400,7462912) | 20260506 00:19:59.357 | INFO | MemInfo: { | 20260506 00:19:59.357 | INFO | "gpu" : " 17 GB", | 20260506 00:19:59.357 | INFO | "huge" : "", | 20260506 00:19:59.357 | INFO | "huge_shared" : " 27 GB", | 20260506 00:19:59.357 | INFO | "malloc" : " 14 GB", | 20260506 00:19:59.357 | INFO | "pinned" : " 29 GB" | 20260506 00:19:59.357 | INFO | } | 20260506 00:19:59.357 | INFO | Perf report for the last 5.13238 seconds: | 20260506 00:19:59.357 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:19:59.357 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:59.357 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:59.357 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:59.357 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260506 00:19:59.357 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:59.357 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:59.357 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 00:19:59.357 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:59.357 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:19:59.358 | INFO | Uploaded 36GiB to GPU in 2.13747s (16.8424GiB/s) | 20260506 00:19:59.358 | INFO | Downloaded 34GiB to GPU in 2.03361s (16.719GiB/s) | 20260506 00:19:59.358 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 00:19:59.358 | INFO | Total mem usage: 15.3 GiB | 20260506 00:19:59.358 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 00:19:59.358 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.13%) | 20260506 00:19:59.358 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 00:19:59.358 | INFO | High Water Breakdown: | 20260506 00:19:59.358 | INFO | 60.9% : 8.55 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 00:19:59.358 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 00:19:59.358 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 00:19:59.358 | INFO | ------------------------------------------ | 20260506 00:19:59.358 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:19:59.358 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 00:19:59.358 | INFO | Total mem usage: 26.5 GiB | 20260506 00:19:59.358 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 00:19:59.358 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 00:19:59.358 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260506 00:19:59.358 | INFO | High Water Breakdown: | 20260506 00:19:59.358 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:19:59.358 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:19:59.358 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:19:59.358 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:19:59.358 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:19:59.358 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:19:59.358 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 00:19:59.358 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 00:19:59.358 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:19:59.358 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 00:19:59.358 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 00:19:59.358 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 00:19:59.358 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 00:19:59.358 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 00:19:59.358 | INFO | ------------------------------------------ | 20260506 00:19:59.358 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:19:59.358 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 00:19:59.358 | INFO | Total mem usage: 25.6 GiB | 20260506 00:19:59.358 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 00:19:59.358 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 00:19:59.358 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 00:19:59.358 | INFO | High Water Breakdown: | 20260506 00:19:59.358 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 00:19:59.358 | INFO | ------------------------------------------ | 20260506 00:19:59.358 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:19:59.358 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 00:19:59.358 | INFO | Total mem usage: 15.7 GiB | 20260506 00:19:59.358 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 00:19:59.358 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 00:19:59.358 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 00:19:59.358 | INFO | High Water Breakdown: | 20260506 00:19:59.358 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 00:19:59.358 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 00:19:59.358 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 00:19:59.358 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 00:19:59.358 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 00:19:59.358 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 00:19:59.359 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 00:19:59.359 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 00:19:59.359 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 00:19:59.359 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:19:59.359 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 00:19:59.359 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:19:59.359 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 00:19:59.359 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 00:19:59.359 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 00:19:59.359 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 00:19:59.359 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:19:59.359 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 00:19:59.359 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:19:59.359 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:19:59.359 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:19:59.359 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 00:19:59.359 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 00:19:59.359 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:19:59.359 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 00:19:59.359 | INFO | ------------------------------------------ | 20260506 00:19:59.359 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:19:59.359 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 00:19:59.359 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:19:59.359 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:19:59.359 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.40e+01, 1.35e+01 / 8.46e-01 | 20260506 00:19:59.359 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.04e-01, 1.09e-02 / 3.62e-03 | 20260506 00:19:59.359 | INFO | PerfReport: 11.83%, Baseline -- 3.71e+00 / 4.50e+00, 5.72e-01 / 1.63e-01 | 20260506 00:19:59.359 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.53e+00, 1.90e-01 / 8.63e-02 | 20260506 00:19:59.359 | INFO | PerfReport: 15.95%, DME -- 4.00e+01 / 4.40e+01, 3.93e+00 / 9.84e-01 | 20260506 00:19:59.359 | INFO | PerfReport: 57.44%, FrameLabeling -- 1.80e+01 / 1.96e+01, 1.56e+00 / 1.17e+00 | 20260506 00:19:59.359 | INFO | PerfReport: 7.05%, PulseAccumulating -- 2.21e+00 / 2.73e+00, 4.17e-01 / 2.33e-01 | 20260506 00:19:59.359 | INFO | PerfReport: 3.07%, Metrics -- 9.64e-01 / 1.25e+00, 2.61e-01 / 1.18e-01 | 20260506 00:19:59.359 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.34e-01 / 3.87e-01, 5.74e-02 / 2.83e-02 | 20260506 00:19:59.359 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 00:19:59.359 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:19:59.359 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:19:59.359 | INFO | PerfReport: Full Execution -- 9.82e+01 / -nan, 2.46e+01 / 0.00e+00 | 20260506 00:19:59.359 | INFO | PerfReport: 22.26%, Upload -- 2.19e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260506 00:19:59.359 | INFO | PerfReport: 21.85%, Download -- 2.15e+01 / 0.00e+00, 7.57e+00 / 0.00e+00 | 20260506 00:19:59.359 | INFO | PerfReport: 31.94%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 00:19:59.359 | INFO | PerfReport: 2.10%, LockUpload -- 2.06e+00 / 0.00e+00, 7.82e+00 / 0.00e+00 | 20260506 00:19:59.359 | INFO | PerfReport: 1.26%, LockDownload -- 1.24e+00 / 0.00e+00, 2.82e+00 / 0.00e+00 | 20260506 00:19:59.359 | INFO | PerfReport: 20.58%, LockCompute -- 2.02e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 00:19:59.359 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.68e-02 / 0.00e+00 | 20260506 00:19:59.476 | INFO | Starting a round of Baz IO | 20260506 00:20:02.627 | INFO | Populating RTMetrics report | 20260506 00:20:02.627 | INFO | RTMetrics report populated | 20260506 00:20:02.627 | INFO | Opening tmp RTMetrics json file | 20260506 00:20:02.629 | INFO | Serializing RTMetrics to json | 20260506 00:20:02.632 | INFO | Serialization done, swapping tmp file now... | 20260506 00:20:02.634 | INFO | RTMetrics done | 20260506 00:20:02.706 | INFO | Finished a round of Baz IO: Wrote 1.865 GiB in 3.229 seconds | 20260506 00:20:02.706 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.38% / 0.00% / 78.62% | 20260506 00:20:02.706 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0942826% / 0% / 7.94729e-05% | 20260506 00:20:02.707 | INFO | Baz IO overall: | 20260506 00:20:02.707 | INFO | BytesWritten : 4243035040692 | 20260506 00:20:02.707 | INFO | NumEvents : 1996978619583 | 20260506 00:20:04.496 | INFO | Analyzing chunk frames = [7462912,7463424) | 20260506 00:20:04.496 | INFO | MemInfo: { | 20260506 00:20:04.496 | INFO | "gpu" : " 17 GB", | 20260506 00:20:04.496 | INFO | "huge" : "", | 20260506 00:20:04.496 | INFO | "huge_shared" : " 27 GB", | 20260506 00:20:04.496 | INFO | "malloc" : " 14 GB", | 20260506 00:20:04.496 | INFO | "pinned" : " 29 GB" | 20260506 00:20:04.496 | INFO | } | 20260506 00:20:04.496 | INFO | Perf report for the last 5.13901 seconds: | 20260506 00:20:04.496 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:20:04.496 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:04.496 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:04.496 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:04.496 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260506 00:20:04.496 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:04.496 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260506 00:20:04.496 | INFO | PRE_HQ : 59.7%, 1.7, 54.1ms, 0 | 20260506 00:20:04.496 | INFO | BAZWRITER : 62.0%, 3.5, 112.2ms, 0 | 20260506 00:20:04.496 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:20:04.497 | INFO | Uploaded 36GiB to GPU in 2.15133s (16.7338GiB/s) | 20260506 00:20:04.497 | INFO | Downloaded 38GiB to GPU in 2.23525s (17.0004GiB/s) | 20260506 00:20:09.635 | INFO | Analyzing chunk frames = [7463424,7463936) | 20260506 00:20:09.635 | INFO | MemInfo: { | 20260506 00:20:09.635 | INFO | "gpu" : " 17 GB", | 20260506 00:20:09.635 | INFO | "huge" : "", | 20260506 00:20:09.635 | INFO | "huge_shared" : " 27 GB", | 20260506 00:20:09.635 | INFO | "malloc" : " 14 GB", | 20260506 00:20:09.635 | INFO | "pinned" : " 29 GB" | 20260506 00:20:09.635 | INFO | } | 20260506 00:20:09.635 | INFO | Perf report for the last 5.13867 seconds: | 20260506 00:20:09.635 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:20:09.635 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:09.635 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:09.635 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:09.635 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 00:20:09.635 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:09.635 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:09.635 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:20:09.635 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:09.635 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:09.635 | INFO | Uploaded 36GiB to GPU in 2.09836s (17.1562GiB/s) | 20260506 00:20:09.635 | INFO | Downloaded 34GiB to GPU in 2.03988s (16.6676GiB/s) | 20260506 00:20:14.778 | INFO | Analyzing chunk frames = [7463936,7464448) | 20260506 00:20:14.778 | INFO | MemInfo: { | 20260506 00:20:14.778 | INFO | "gpu" : " 17 GB", | 20260506 00:20:14.778 | INFO | "huge" : "", | 20260506 00:20:14.778 | INFO | "huge_shared" : " 27 GB", | 20260506 00:20:14.778 | INFO | "malloc" : " 14 GB", | 20260506 00:20:14.778 | INFO | "pinned" : " 29 GB" | 20260506 00:20:14.778 | INFO | } | 20260506 00:20:14.778 | INFO | Perf report for the last 5.14305 seconds: | 20260506 00:20:14.778 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:20:14.778 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:14.778 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:14.778 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:14.778 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 00:20:14.778 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:14.778 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:14.778 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 00:20:14.778 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:14.778 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:14.778 | INFO | Uploaded 36GiB to GPU in 2.07231s (17.3719GiB/s) | 20260506 00:20:14.778 | INFO | Downloaded 34GiB to GPU in 2.018s (16.8484GiB/s) | 20260506 00:20:19.917 | INFO | Analyzing chunk frames = [7464448,7464960) | 20260506 00:20:19.917 | INFO | MemInfo: { | 20260506 00:20:19.917 | INFO | "gpu" : " 17 GB", | 20260506 00:20:19.917 | INFO | "huge" : "", | 20260506 00:20:19.917 | INFO | "huge_shared" : " 27 GB", | 20260506 00:20:19.917 | INFO | "malloc" : " 14 GB", | 20260506 00:20:19.917 | INFO | "pinned" : " 29 GB" | 20260506 00:20:19.917 | INFO | } | 20260506 00:20:19.917 | INFO | Perf report for the last 5.13867 seconds: | 20260506 00:20:19.917 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:20:19.917 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:19.917 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:19.917 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:19.917 | INFO | ANALYSIS : 61.5%, 3.0, 95.5ms, 0 | 20260506 00:20:19.917 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:19.917 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:19.917 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 00:20:19.917 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:19.917 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:19.917 | INFO | Uploaded 36GiB to GPU in 2.15092s (16.737GiB/s) | 20260506 00:20:19.917 | INFO | Downloaded 34GiB to GPU in 2.05832s (16.5183GiB/s) | 20260506 00:20:25.052 | INFO | Analyzing chunk frames = [7464960,7465472) | 20260506 00:20:25.052 | INFO | MemInfo: { | 20260506 00:20:25.052 | INFO | "gpu" : " 17 GB", | 20260506 00:20:25.052 | INFO | "huge" : "", | 20260506 00:20:25.052 | INFO | "huge_shared" : " 27 GB", | 20260506 00:20:25.053 | INFO | "malloc" : " 14 GB", | 20260506 00:20:25.053 | INFO | "pinned" : " 29 GB" | 20260506 00:20:25.053 | INFO | } | 20260506 00:20:25.053 | INFO | Perf report for the last 5.13586 seconds: | 20260506 00:20:25.053 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:20:25.053 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:25.053 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:25.053 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:25.053 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 00:20:25.053 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:25.053 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:25.053 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 00:20:25.053 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:25.053 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:25.053 | INFO | Uploaded 36GiB to GPU in 2.09133s (17.2139GiB/s) | 20260506 00:20:25.053 | INFO | Downloaded 34GiB to GPU in 2.04828s (16.5993GiB/s) | 20260506 00:20:30.193 | INFO | Analyzing chunk frames = [7465472,7465984) | 20260506 00:20:30.193 | INFO | MemInfo: { | 20260506 00:20:30.193 | INFO | "gpu" : " 17 GB", | 20260506 00:20:30.193 | INFO | "huge" : "", | 20260506 00:20:30.193 | INFO | "huge_shared" : " 27 GB", | 20260506 00:20:30.193 | INFO | "malloc" : " 14 GB", | 20260506 00:20:30.193 | INFO | "pinned" : " 29 GB" | 20260506 00:20:30.193 | INFO | } | 20260506 00:20:30.193 | INFO | Perf report for the last 5.14009 seconds: | 20260506 00:20:30.193 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:20:30.193 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:30.193 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:30.193 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:30.193 | INFO | ANALYSIS : 61.5%, 3.0, 96.0ms, 0 | 20260506 00:20:30.193 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:30.193 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:30.193 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:20:30.193 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:30.193 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:30.193 | INFO | Uploaded 36GiB to GPU in 2.08312s (17.2818GiB/s) | 20260506 00:20:30.193 | INFO | Downloaded 34GiB to GPU in 2.03906s (16.6743GiB/s) | 20260506 00:20:35.330 | INFO | Analyzing chunk frames = [7465984,7466496) | 20260506 00:20:35.330 | INFO | MemInfo: { | 20260506 00:20:35.330 | INFO | "gpu" : " 17 GB", | 20260506 00:20:35.330 | INFO | "huge" : "", | 20260506 00:20:35.330 | INFO | "huge_shared" : " 27 GB", | 20260506 00:20:35.330 | INFO | "malloc" : " 14 GB", | 20260506 00:20:35.330 | INFO | "pinned" : " 29 GB" | 20260506 00:20:35.330 | INFO | } | 20260506 00:20:35.330 | INFO | Perf report for the last 5.13737 seconds: | 20260506 00:20:35.330 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:20:35.330 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:35.330 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:35.330 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:35.330 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260506 00:20:35.330 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:35.330 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:35.330 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 00:20:35.330 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:35.330 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:35.330 | INFO | Uploaded 36GiB to GPU in 2.10529s (17.0998GiB/s) | 20260506 00:20:35.330 | INFO | Downloaded 34GiB to GPU in 2.08612s (16.2982GiB/s) | 20260506 00:20:40.477 | INFO | Analyzing chunk frames = [7466496,7467008) | 20260506 00:20:40.477 | INFO | MemInfo: { | 20260506 00:20:40.477 | INFO | "gpu" : " 17 GB", | 20260506 00:20:40.477 | INFO | "huge" : "", | 20260506 00:20:40.477 | INFO | "huge_shared" : " 27 GB", | 20260506 00:20:40.477 | INFO | "malloc" : " 14 GB", | 20260506 00:20:40.477 | INFO | "pinned" : " 29 GB" | 20260506 00:20:40.477 | INFO | } | 20260506 00:20:40.478 | INFO | Perf report for the last 5.14746 seconds: | 20260506 00:20:40.478 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:20:40.478 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:40.478 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:40.478 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:40.478 | INFO | ANALYSIS : 61.9%, 3.0, 96.8ms, 0 | 20260506 00:20:40.478 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:40.478 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:40.478 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 00:20:40.478 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:40.478 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:40.478 | INFO | Uploaded 36GiB to GPU in 2.12541s (16.9379GiB/s) | 20260506 00:20:40.478 | INFO | Downloaded 34GiB to GPU in 2.01336s (16.8872GiB/s) | 20260506 00:20:40.604 | INFO | Starting a round of Baz IO | 20260506 00:20:43.739 | INFO | Populating RTMetrics report | 20260506 00:20:43.739 | INFO | RTMetrics report populated | 20260506 00:20:43.739 | INFO | Opening tmp RTMetrics json file | 20260506 00:20:43.742 | INFO | Serializing RTMetrics to json | 20260506 00:20:43.743 | INFO | Serialization done, swapping tmp file now... | 20260506 00:20:43.744 | INFO | RTMetrics done | 20260506 00:20:43.818 | INFO | Finished a round of Baz IO: Wrote 1.865 GiB in 3.214 seconds | 20260506 00:20:43.818 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.38% / 0.00% / 78.62% | 20260506 00:20:43.818 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0940283% / 0% / 9.13938e-05% | 20260506 00:20:43.818 | INFO | Baz IO overall: | 20260506 00:20:43.818 | INFO | BytesWritten : 4245037283248 | 20260506 00:20:43.818 | INFO | NumEvents : 1997894447401 | 20260506 00:20:45.615 | INFO | Analyzing chunk frames = [7467008,7467520) | 20260506 00:20:45.615 | INFO | MemInfo: { | 20260506 00:20:45.615 | INFO | "gpu" : " 17 GB", | 20260506 00:20:45.615 | INFO | "huge" : "", | 20260506 00:20:45.615 | INFO | "huge_shared" : " 27 GB", | 20260506 00:20:45.615 | INFO | "malloc" : " 14 GB", | 20260506 00:20:45.615 | INFO | "pinned" : " 29 GB" | 20260506 00:20:45.615 | INFO | } | 20260506 00:20:45.615 | INFO | Perf report for the last 5.13769 seconds: | 20260506 00:20:45.615 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:20:45.615 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:45.615 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:45.615 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:45.615 | INFO | ANALYSIS : 62.5%, 3.0, 96.5ms, 0 | 20260506 00:20:45.615 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:45.615 | INFO | RT_METRICS : 43.2%, 1.0, 22.4ms, 0 | 20260506 00:20:45.615 | INFO | PRE_HQ : 59.4%, 1.8, 53.6ms, 0 | 20260506 00:20:45.615 | INFO | BAZWRITER : 61.7%, 3.5, 111.4ms, 0 | 20260506 00:20:45.615 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:20:45.616 | INFO | Uploaded 36GiB to GPU in 2.11054s (17.0572GiB/s) | 20260506 00:20:45.616 | INFO | Downloaded 38GiB to GPU in 2.22033s (17.1146GiB/s) | 20260506 00:20:50.752 | INFO | Analyzing chunk frames = [7467520,7468032) | 20260506 00:20:50.752 | INFO | MemInfo: { | 20260506 00:20:50.752 | INFO | "gpu" : " 17 GB", | 20260506 00:20:50.752 | INFO | "huge" : "", | 20260506 00:20:50.752 | INFO | "huge_shared" : " 27 GB", | 20260506 00:20:50.752 | INFO | "malloc" : " 14 GB", | 20260506 00:20:50.752 | INFO | "pinned" : " 29 GB" | 20260506 00:20:50.752 | INFO | } | 20260506 00:20:50.752 | INFO | Perf report for the last 5.13662 seconds: | 20260506 00:20:50.752 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:20:50.752 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:50.752 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:50.752 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:50.752 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 00:20:50.752 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:50.752 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:50.752 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 00:20:50.752 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:50.752 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:50.752 | INFO | Uploaded 36GiB to GPU in 2.1246s (16.9443GiB/s) | 20260506 00:20:50.752 | INFO | Downloaded 34GiB to GPU in 2.04285s (16.6434GiB/s) | 20260506 00:20:55.885 | INFO | Analyzing chunk frames = [7468032,7468544) | 20260506 00:20:55.885 | INFO | MemInfo: { | 20260506 00:20:55.885 | INFO | "gpu" : " 17 GB", | 20260506 00:20:55.885 | INFO | "huge" : "", | 20260506 00:20:55.885 | INFO | "huge_shared" : " 27 GB", | 20260506 00:20:55.885 | INFO | "malloc" : " 14 GB", | 20260506 00:20:55.885 | INFO | "pinned" : " 29 GB" | 20260506 00:20:55.885 | INFO | } | 20260506 00:20:55.885 | INFO | Perf report for the last 5.13294 seconds: | 20260506 00:20:55.885 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:20:55.885 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:55.885 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:55.885 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:55.885 | INFO | ANALYSIS : 61.6%, 3.0, 95.7ms, 0 | 20260506 00:20:55.885 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:55.885 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:55.885 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 00:20:55.885 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:55.885 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:20:55.885 | INFO | Uploaded 36GiB to GPU in 2.11197s (17.0457GiB/s) | 20260506 00:20:55.885 | INFO | Downloaded 34GiB to GPU in 2.02891s (16.7578GiB/s) | 20260506 00:21:01.033 | INFO | Analyzing chunk frames = [7468544,7469056) | 20260506 00:21:01.033 | INFO | MemInfo: { | 20260506 00:21:01.033 | INFO | "gpu" : " 17 GB", | 20260506 00:21:01.033 | INFO | "huge" : "", | 20260506 00:21:01.033 | INFO | "huge_shared" : " 27 GB", | 20260506 00:21:01.033 | INFO | "malloc" : " 14 GB", | 20260506 00:21:01.033 | INFO | "pinned" : " 29 GB" | 20260506 00:21:01.033 | INFO | } | 20260506 00:21:01.033 | INFO | Perf report for the last 5.14802 seconds: | 20260506 00:21:01.033 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:21:01.033 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:01.033 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:01.033 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:01.033 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 00:21:01.033 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:01.033 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:01.033 | INFO | PRE_HQ : 26.8%, 1.0, 14.0ms, 0 | 20260506 00:21:01.033 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:01.033 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:01.033 | INFO | Uploaded 36GiB to GPU in 2.13586s (16.855GiB/s) | 20260506 00:21:01.033 | INFO | Downloaded 34GiB to GPU in 2.01497s (16.8737GiB/s) | 20260506 00:21:06.173 | INFO | Analyzing chunk frames = [7469056,7469568) | 20260506 00:21:06.173 | INFO | MemInfo: { | 20260506 00:21:06.173 | INFO | "gpu" : " 17 GB", | 20260506 00:21:06.173 | INFO | "huge" : "", | 20260506 00:21:06.173 | INFO | "huge_shared" : " 27 GB", | 20260506 00:21:06.173 | INFO | "malloc" : " 14 GB", | 20260506 00:21:06.173 | INFO | "pinned" : " 29 GB" | 20260506 00:21:06.173 | INFO | } | 20260506 00:21:06.173 | INFO | Perf report for the last 5.14059 seconds: | 20260506 00:21:06.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:21:06.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:06.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:06.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:06.173 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 00:21:06.174 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:06.174 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:06.174 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 00:21:06.174 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:06.174 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:06.174 | INFO | Uploaded 36GiB to GPU in 2.09123s (17.2147GiB/s) | 20260506 00:21:06.174 | INFO | Downloaded 34GiB to GPU in 2.04567s (16.6205GiB/s) | 20260506 00:21:11.312 | INFO | Analyzing chunk frames = [7469568,7470080) | 20260506 00:21:11.312 | INFO | MemInfo: { | 20260506 00:21:11.312 | INFO | "gpu" : " 17 GB", | 20260506 00:21:11.312 | INFO | "huge" : "", | 20260506 00:21:11.312 | INFO | "huge_shared" : " 27 GB", | 20260506 00:21:11.312 | INFO | "malloc" : " 14 GB", | 20260506 00:21:11.312 | INFO | "pinned" : " 29 GB" | 20260506 00:21:11.312 | INFO | } | 20260506 00:21:11.312 | INFO | Perf report for the last 5.13849 seconds: | 20260506 00:21:11.312 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:21:11.312 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:11.312 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:11.312 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:11.312 | INFO | ANALYSIS : 61.2%, 3.0, 95.6ms, 0 | 20260506 00:21:11.312 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:11.312 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:11.312 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:21:11.312 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:11.312 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:11.312 | INFO | Uploaded 36GiB to GPU in 2.07077s (17.3848GiB/s) | 20260506 00:21:11.312 | INFO | Downloaded 34GiB to GPU in 2.04163s (16.6534GiB/s) | 20260506 00:21:16.448 | INFO | Analyzing chunk frames = [7470080,7470592) | 20260506 00:21:16.448 | INFO | MemInfo: { | 20260506 00:21:16.448 | INFO | "gpu" : " 17 GB", | 20260506 00:21:16.448 | INFO | "huge" : "", | 20260506 00:21:16.448 | INFO | "huge_shared" : " 27 GB", | 20260506 00:21:16.448 | INFO | "malloc" : " 14 GB", | 20260506 00:21:16.448 | INFO | "pinned" : " 29 GB" | 20260506 00:21:16.448 | INFO | } | 20260506 00:21:16.448 | INFO | Perf report for the last 5.13629 seconds: | 20260506 00:21:16.448 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:21:16.448 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:16.448 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:16.448 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:16.448 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 00:21:16.448 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:16.448 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:16.448 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 00:21:16.448 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:16.448 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:16.448 | INFO | Uploaded 36GiB to GPU in 2.04372s (17.6149GiB/s) | 20260506 00:21:16.448 | INFO | Downloaded 34GiB to GPU in 2.06151s (16.4928GiB/s) | 20260506 00:21:21.584 | INFO | Analyzing chunk frames = [7470592,7471104) | 20260506 00:21:21.585 | INFO | MemInfo: { | 20260506 00:21:21.585 | INFO | "gpu" : " 17 GB", | 20260506 00:21:21.585 | INFO | "huge" : "", | 20260506 00:21:21.585 | INFO | "huge_shared" : " 27 GB", | 20260506 00:21:21.585 | INFO | "malloc" : " 14 GB", | 20260506 00:21:21.585 | INFO | "pinned" : " 29 GB" | 20260506 00:21:21.585 | INFO | } | 20260506 00:21:21.585 | INFO | Perf report for the last 5.13643 seconds: | 20260506 00:21:21.585 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:21:21.585 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:21.585 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:21.585 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:21.585 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260506 00:21:21.585 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:21.585 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:21.585 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 00:21:21.585 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:21.585 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:21.585 | INFO | Uploaded 36GiB to GPU in 2.11241s (17.0421GiB/s) | 20260506 00:21:21.585 | INFO | Downloaded 34GiB to GPU in 2.00916s (16.9225GiB/s) | 20260506 00:21:21.585 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 00:21:21.585 | INFO | Total mem usage: 15.3 GiB | 20260506 00:21:21.585 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 00:21:21.585 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.11%) | 20260506 00:21:21.585 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 00:21:21.585 | INFO | High Water Breakdown: | 20260506 00:21:21.585 | INFO | 60.9% : 8.55 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 00:21:21.585 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 00:21:21.585 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 00:21:21.585 | INFO | ------------------------------------------ | 20260506 00:21:21.585 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:21:21.585 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 00:21:21.585 | INFO | Total mem usage: 26.5 GiB | 20260506 00:21:21.585 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 00:21:21.585 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 00:21:21.585 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 00:21:21.585 | INFO | High Water Breakdown: | 20260506 00:21:21.585 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:21:21.585 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:21:21.585 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:21:21.585 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:21:21.585 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:21:21.585 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:21:21.585 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 00:21:21.585 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 00:21:21.585 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:21:21.585 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 00:21:21.585 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 00:21:21.585 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 00:21:21.585 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 00:21:21.585 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 00:21:21.585 | INFO | ------------------------------------------ | 20260506 00:21:21.585 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:21:21.585 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 00:21:21.585 | INFO | Total mem usage: 25.6 GiB | 20260506 00:21:21.585 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 00:21:21.585 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 00:21:21.585 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 00:21:21.585 | INFO | High Water Breakdown: | 20260506 00:21:21.585 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 00:21:21.585 | INFO | ------------------------------------------ | 20260506 00:21:21.585 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:21:21.585 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 00:21:21.585 | INFO | Total mem usage: 15.7 GiB | 20260506 00:21:21.585 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 00:21:21.585 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 00:21:21.585 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 00:21:21.586 | INFO | High Water Breakdown: | 20260506 00:21:21.586 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 00:21:21.586 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 00:21:21.586 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 00:21:21.586 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 00:21:21.586 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 00:21:21.586 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 00:21:21.586 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 00:21:21.586 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 00:21:21.586 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 00:21:21.586 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:21:21.586 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 00:21:21.586 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:21:21.586 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 00:21:21.586 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 00:21:21.586 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 00:21:21.586 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 00:21:21.586 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:21:21.586 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 00:21:21.586 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:21:21.586 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:21:21.586 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:21:21.586 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 00:21:21.586 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 00:21:21.586 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:21:21.586 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 00:21:21.586 | INFO | ------------------------------------------ | 20260506 00:21:21.586 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:21:21.586 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 00:21:21.586 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:21:21.586 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:21:21.586 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.40e+01, 1.35e+01 / 8.91e-01 | 20260506 00:21:21.586 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.03e-01, 1.03e-02 / 4.20e-03 | 20260506 00:21:21.586 | INFO | PerfReport: 11.83%, Baseline -- 3.71e+00 / 4.47e+00, 5.72e-01 / 2.81e-01 | 20260506 00:21:21.586 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.50e+00, 1.91e-01 / 1.15e-01 | 20260506 00:21:21.586 | INFO | PerfReport: 15.97%, DME -- 4.00e+01 / 4.40e+01, 3.84e+00 / 1.10e+00 | 20260506 00:21:21.586 | INFO | PerfReport: 57.40%, FrameLabeling -- 1.80e+01 / 1.96e+01, 1.55e+00 / 1.13e+00 | 20260506 00:21:21.586 | INFO | PerfReport: 7.07%, PulseAccumulating -- 2.22e+00 / 2.72e+00, 4.20e-01 / 2.46e-01 | 20260506 00:21:21.586 | INFO | PerfReport: 3.08%, Metrics -- 9.66e-01 / 1.25e+00, 2.60e-01 / 1.31e-01 | 20260506 00:21:21.586 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.34e-01 / 3.90e-01, 5.65e-02 / 3.38e-02 | 20260506 00:21:21.586 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 00:21:21.586 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:21:21.586 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:21:21.586 | INFO | PerfReport: Full Execution -- 9.83e+01 / -nan, 2.47e+01 / 0.00e+00 | 20260506 00:21:21.586 | INFO | PerfReport: 22.31%, Upload -- 2.19e+01 / 0.00e+00, 7.48e+00 / 0.00e+00 | 20260506 00:21:21.586 | INFO | PerfReport: 21.86%, Download -- 2.15e+01 / 0.00e+00, 7.61e+00 / 0.00e+00 | 20260506 00:21:21.586 | INFO | PerfReport: 31.89%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 00:21:21.586 | INFO | PerfReport: 2.11%, LockUpload -- 2.08e+00 / 0.00e+00, 7.91e+00 / 0.00e+00 | 20260506 00:21:21.586 | INFO | PerfReport: 1.32%, LockDownload -- 1.30e+00 / 0.00e+00, 2.95e+00 / 0.00e+00 | 20260506 00:21:21.586 | INFO | PerfReport: 20.50%, LockCompute -- 2.02e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 00:21:21.586 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.62e-02 / 0.00e+00 | 20260506 00:21:21.706 | INFO | Starting a round of Baz IO | 20260506 00:21:24.849 | INFO | Populating RTMetrics report | 20260506 00:21:24.849 | INFO | RTMetrics report populated | 20260506 00:21:24.849 | INFO | Opening tmp RTMetrics json file | 20260506 00:21:24.852 | INFO | Serializing RTMetrics to json | 20260506 00:21:24.853 | INFO | Serialization done, swapping tmp file now... | 20260506 00:21:24.853 | INFO | RTMetrics done | 20260506 00:21:24.928 | INFO | Finished a round of Baz IO: Wrote 1.864 GiB in 3.221 seconds | 20260506 00:21:24.928 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.37% / 0.00% / 78.62% | 20260506 00:21:24.928 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0943343% / 0% / 0.000107288% | 20260506 00:21:24.928 | INFO | Baz IO overall: | 20260506 00:21:24.928 | INFO | BytesWritten : 4247038571305 | 20260506 00:21:24.928 | INFO | NumEvents : 1998809625254 | 20260506 00:21:26.726 | INFO | Analyzing chunk frames = [7471104,7471616) | 20260506 00:21:26.727 | INFO | MemInfo: { | 20260506 00:21:26.727 | INFO | "gpu" : " 17 GB", | 20260506 00:21:26.727 | INFO | "huge" : "", | 20260506 00:21:26.727 | INFO | "huge_shared" : " 27 GB", | 20260506 00:21:26.727 | INFO | "malloc" : " 14 GB", | 20260506 00:21:26.727 | INFO | "pinned" : " 29 GB" | 20260506 00:21:26.727 | INFO | } | 20260506 00:21:26.727 | INFO | Perf report for the last 5.14192 seconds: | 20260506 00:21:26.727 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:21:26.727 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:26.727 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:26.727 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:26.727 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260506 00:21:26.727 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:26.727 | INFO | RT_METRICS : 43.4%, 1.0, 22.8ms, 0 | 20260506 00:21:26.727 | INFO | PRE_HQ : 59.9%, 1.8, 54.5ms, 0 | 20260506 00:21:26.727 | INFO | BAZWRITER : 61.8%, 3.5, 112.5ms, 0 | 20260506 00:21:26.727 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:21:26.727 | INFO | Uploaded 36GiB to GPU in 2.16579s (16.6221GiB/s) | 20260506 00:21:26.727 | INFO | Downloaded 38GiB to GPU in 2.23863s (16.9746GiB/s) | 20260506 00:21:31.862 | INFO | Analyzing chunk frames = [7471616,7472128) | 20260506 00:21:31.862 | INFO | MemInfo: { | 20260506 00:21:31.862 | INFO | "gpu" : " 17 GB", | 20260506 00:21:31.862 | INFO | "huge" : "", | 20260506 00:21:31.862 | INFO | "huge_shared" : " 27 GB", | 20260506 00:21:31.862 | INFO | "malloc" : " 14 GB", | 20260506 00:21:31.862 | INFO | "pinned" : " 29 GB" | 20260506 00:21:31.862 | INFO | } | 20260506 00:21:31.862 | INFO | Perf report for the last 5.13549 seconds: | 20260506 00:21:31.862 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:21:31.862 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:31.862 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:31.862 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:31.862 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 00:21:31.862 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:31.862 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:31.862 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:21:31.862 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:31.862 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:31.862 | INFO | Uploaded 36GiB to GPU in 1.96627s (18.3088GiB/s) | 20260506 00:21:31.862 | INFO | Downloaded 34GiB to GPU in 1.99113s (17.0757GiB/s) | 20260506 00:21:37.002 | INFO | Analyzing chunk frames = [7472128,7472640) | 20260506 00:21:37.002 | INFO | MemInfo: { | 20260506 00:21:37.002 | INFO | "gpu" : " 17 GB", | 20260506 00:21:37.002 | INFO | "huge" : "", | 20260506 00:21:37.002 | INFO | "huge_shared" : " 27 GB", | 20260506 00:21:37.002 | INFO | "malloc" : " 14 GB", | 20260506 00:21:37.002 | INFO | "pinned" : " 29 GB" | 20260506 00:21:37.002 | INFO | } | 20260506 00:21:37.003 | INFO | Perf report for the last 5.1404 seconds: | 20260506 00:21:37.003 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:21:37.003 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:37.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:37.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:37.003 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 00:21:37.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:37.003 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:37.003 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:21:37.003 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:37.003 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:37.003 | INFO | Uploaded 36GiB to GPU in 2.09319s (17.1986GiB/s) | 20260506 00:21:37.003 | INFO | Downloaded 34GiB to GPU in 2.03337s (16.721GiB/s) | 20260506 00:21:42.141 | INFO | Analyzing chunk frames = [7472640,7473152) | 20260506 00:21:42.141 | INFO | MemInfo: { | 20260506 00:21:42.141 | INFO | "gpu" : " 17 GB", | 20260506 00:21:42.141 | INFO | "huge" : "", | 20260506 00:21:42.141 | INFO | "huge_shared" : " 27 GB", | 20260506 00:21:42.141 | INFO | "malloc" : " 14 GB", | 20260506 00:21:42.142 | INFO | "pinned" : " 29 GB" | 20260506 00:21:42.142 | INFO | } | 20260506 00:21:42.142 | INFO | Perf report for the last 5.139 seconds: | 20260506 00:21:42.142 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:21:42.142 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:42.142 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:42.142 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:42.142 | INFO | ANALYSIS : 61.7%, 3.0, 95.8ms, 0 | 20260506 00:21:42.142 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:42.142 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:42.142 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 00:21:42.142 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:42.142 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:42.142 | INFO | Uploaded 36GiB to GPU in 2.12881s (16.9108GiB/s) | 20260506 00:21:42.142 | INFO | Downloaded 34GiB to GPU in 2.01322s (16.8883GiB/s) | 20260506 00:21:47.285 | INFO | Analyzing chunk frames = [7473152,7473664) | 20260506 00:21:47.285 | INFO | MemInfo: { | 20260506 00:21:47.285 | INFO | "gpu" : " 17 GB", | 20260506 00:21:47.285 | INFO | "huge" : "", | 20260506 00:21:47.285 | INFO | "huge_shared" : " 27 GB", | 20260506 00:21:47.285 | INFO | "malloc" : " 14 GB", | 20260506 00:21:47.285 | INFO | "pinned" : " 29 GB" | 20260506 00:21:47.285 | INFO | } | 20260506 00:21:47.285 | INFO | Perf report for the last 5.14334 seconds: | 20260506 00:21:47.285 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:21:47.285 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:47.285 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:47.285 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:47.285 | INFO | ANALYSIS : 61.1%, 3.0, 95.5ms, 0 | 20260506 00:21:47.285 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:47.285 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:47.285 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 00:21:47.285 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:47.285 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:47.285 | INFO | Uploaded 36GiB to GPU in 2.10266s (17.1212GiB/s) | 20260506 00:21:47.285 | INFO | Downloaded 34GiB to GPU in 2.05219s (16.5677GiB/s) | 20260506 00:21:52.431 | INFO | Analyzing chunk frames = [7473664,7474176) | 20260506 00:21:52.431 | INFO | MemInfo: { | 20260506 00:21:52.431 | INFO | "gpu" : " 17 GB", | 20260506 00:21:52.431 | INFO | "huge" : "", | 20260506 00:21:52.431 | INFO | "huge_shared" : " 27 GB", | 20260506 00:21:52.431 | INFO | "malloc" : " 14 GB", | 20260506 00:21:52.431 | INFO | "pinned" : " 29 GB" | 20260506 00:21:52.431 | INFO | } | 20260506 00:21:52.431 | INFO | Perf report for the last 5.14645 seconds: | 20260506 00:21:52.431 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:21:52.431 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:52.431 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:52.431 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:52.431 | INFO | ANALYSIS : 61.2%, 3.0, 95.7ms, 0 | 20260506 00:21:52.431 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:52.432 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:52.432 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:21:52.432 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:52.432 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:52.432 | INFO | Uploaded 36GiB to GPU in 2.07165s (17.3774GiB/s) | 20260506 00:21:52.432 | INFO | Downloaded 34GiB to GPU in 2.04254s (16.6459GiB/s) | 20260506 00:21:57.561 | INFO | Analyzing chunk frames = [7474176,7474688) | 20260506 00:21:57.561 | INFO | MemInfo: { | 20260506 00:21:57.561 | INFO | "gpu" : " 17 GB", | 20260506 00:21:57.561 | INFO | "huge" : "", | 20260506 00:21:57.561 | INFO | "huge_shared" : " 27 GB", | 20260506 00:21:57.561 | INFO | "malloc" : " 14 GB", | 20260506 00:21:57.561 | INFO | "pinned" : " 29 GB" | 20260506 00:21:57.561 | INFO | } | 20260506 00:21:57.561 | INFO | Perf report for the last 5.12983 seconds: | 20260506 00:21:57.561 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:21:57.561 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:57.561 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:57.561 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:57.561 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 00:21:57.561 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:57.561 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:57.561 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 00:21:57.561 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:57.561 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:21:57.561 | INFO | Uploaded 36GiB to GPU in 2.06248s (17.4547GiB/s) | 20260506 00:21:57.561 | INFO | Downloaded 34GiB to GPU in 2.07567s (16.3803GiB/s) | 20260506 00:22:02.697 | INFO | Analyzing chunk frames = [7474688,7475200) | 20260506 00:22:02.697 | INFO | MemInfo: { | 20260506 00:22:02.697 | INFO | "gpu" : " 17 GB", | 20260506 00:22:02.697 | INFO | "huge" : "", | 20260506 00:22:02.697 | INFO | "huge_shared" : " 27 GB", | 20260506 00:22:02.697 | INFO | "malloc" : " 14 GB", | 20260506 00:22:02.697 | INFO | "pinned" : " 29 GB" | 20260506 00:22:02.697 | INFO | } | 20260506 00:22:02.698 | INFO | Perf report for the last 5.13631 seconds: | 20260506 00:22:02.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:22:02.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:02.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:02.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:02.698 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260506 00:22:02.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:02.698 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:02.698 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 00:22:02.698 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:02.698 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:02.698 | INFO | Uploaded 36GiB to GPU in 2.12506s (16.9407GiB/s) | 20260506 00:22:02.698 | INFO | Downloaded 34GiB to GPU in 2.01701s (16.8566GiB/s) | 20260506 00:22:02.824 | INFO | Starting a round of Baz IO | 20260506 00:22:05.966 | INFO | Populating RTMetrics report | 20260506 00:22:05.967 | INFO | RTMetrics report populated | 20260506 00:22:05.967 | INFO | Opening tmp RTMetrics json file | 20260506 00:22:05.967 | INFO | Serializing RTMetrics to json | 20260506 00:22:05.969 | INFO | Serialization done, swapping tmp file now... | 20260506 00:22:05.969 | INFO | RTMetrics done | 20260506 00:22:06.045 | INFO | Finished a round of Baz IO: Wrote 1.865 GiB in 3.221 seconds | 20260506 00:22:06.045 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.37% / 0.00% / 78.63% | 20260506 00:22:06.045 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0943263% / 3.97364e-06% / 7.54992e-05% | 20260506 00:22:06.045 | INFO | Baz IO overall: | 20260506 00:22:06.045 | INFO | BytesWritten : 4249040882250 | 20260506 00:22:06.045 | INFO | NumEvents : 1999725838605 | 20260506 00:22:07.840 | INFO | Analyzing chunk frames = [7475200,7475712) | 20260506 00:22:07.840 | INFO | MemInfo: { | 20260506 00:22:07.840 | INFO | "gpu" : " 17 GB", | 20260506 00:22:07.840 | INFO | "huge" : "", | 20260506 00:22:07.840 | INFO | "huge_shared" : " 27 GB", | 20260506 00:22:07.840 | INFO | "malloc" : " 14 GB", | 20260506 00:22:07.841 | INFO | "pinned" : " 29 GB" | 20260506 00:22:07.841 | INFO | } | 20260506 00:22:07.841 | INFO | Perf report for the last 5.14302 seconds: | 20260506 00:22:07.841 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:22:07.841 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:07.841 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:07.841 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:07.841 | INFO | ANALYSIS : 62.5%, 3.0, 97.7ms, 0 | 20260506 00:22:07.841 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:07.841 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260506 00:22:07.841 | INFO | PRE_HQ : 59.8%, 1.7, 53.9ms, 0 | 20260506 00:22:07.841 | INFO | BAZWRITER : 61.8%, 3.5, 112.0ms, 0 | 20260506 00:22:07.841 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:22:07.841 | INFO | Uploaded 36GiB to GPU in 2.16616s (16.6192GiB/s) | 20260506 00:22:07.841 | INFO | Downloaded 38GiB to GPU in 2.23729s (16.9848GiB/s) | 20260506 00:22:12.978 | INFO | Analyzing chunk frames = [7475712,7476224) | 20260506 00:22:12.978 | INFO | MemInfo: { | 20260506 00:22:12.978 | INFO | "gpu" : " 17 GB", | 20260506 00:22:12.978 | INFO | "huge" : "", | 20260506 00:22:12.978 | INFO | "huge_shared" : " 27 GB", | 20260506 00:22:12.978 | INFO | "malloc" : " 14 GB", | 20260506 00:22:12.978 | INFO | "pinned" : " 29 GB" | 20260506 00:22:12.978 | INFO | } | 20260506 00:22:12.978 | INFO | Perf report for the last 5.13759 seconds: | 20260506 00:22:12.978 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:22:12.978 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:12.978 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:12.978 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:12.978 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 00:22:12.978 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:12.978 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:12.978 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:22:12.978 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:12.978 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:12.978 | INFO | Uploaded 36GiB to GPU in 2.10825s (17.0758GiB/s) | 20260506 00:22:12.979 | INFO | Downloaded 34GiB to GPU in 2.04208s (16.6497GiB/s) | 20260506 00:22:18.123 | INFO | Analyzing chunk frames = [7476224,7476736) | 20260506 00:22:18.123 | INFO | MemInfo: { | 20260506 00:22:18.123 | INFO | "gpu" : " 17 GB", | 20260506 00:22:18.123 | INFO | "huge" : "", | 20260506 00:22:18.123 | INFO | "huge_shared" : " 27 GB", | 20260506 00:22:18.123 | INFO | "malloc" : " 14 GB", | 20260506 00:22:18.123 | INFO | "pinned" : " 29 GB" | 20260506 00:22:18.123 | INFO | } | 20260506 00:22:18.123 | INFO | Perf report for the last 5.14498 seconds: | 20260506 00:22:18.123 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:22:18.123 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:18.123 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:18.123 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:18.123 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 00:22:18.123 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:18.123 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:18.123 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 00:22:18.123 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:18.123 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:18.123 | INFO | Uploaded 36GiB to GPU in 2.08392s (17.2752GiB/s) | 20260506 00:22:18.123 | INFO | Downloaded 34GiB to GPU in 2.0115s (16.9028GiB/s) | 20260506 00:22:23.262 | INFO | Analyzing chunk frames = [7476736,7477248) | 20260506 00:22:23.262 | INFO | MemInfo: { | 20260506 00:22:23.262 | INFO | "gpu" : " 17 GB", | 20260506 00:22:23.262 | INFO | "huge" : "", | 20260506 00:22:23.262 | INFO | "huge_shared" : " 27 GB", | 20260506 00:22:23.262 | INFO | "malloc" : " 14 GB", | 20260506 00:22:23.262 | INFO | "pinned" : " 29 GB" | 20260506 00:22:23.262 | INFO | } | 20260506 00:22:23.262 | INFO | Perf report for the last 5.1389 seconds: | 20260506 00:22:23.262 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:22:23.262 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:23.262 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:23.262 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:23.262 | INFO | ANALYSIS : 61.6%, 3.0, 95.6ms, 0 | 20260506 00:22:23.262 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:23.262 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:23.262 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 00:22:23.262 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:23.262 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:23.262 | INFO | Uploaded 36GiB to GPU in 2.14071s (16.8169GiB/s) | 20260506 00:22:23.262 | INFO | Downloaded 34GiB to GPU in 2.02358s (16.8019GiB/s) | 20260506 00:22:28.397 | INFO | Analyzing chunk frames = [7477248,7477760) | 20260506 00:22:28.397 | INFO | MemInfo: { | 20260506 00:22:28.397 | INFO | "gpu" : " 17 GB", | 20260506 00:22:28.397 | INFO | "huge" : "", | 20260506 00:22:28.397 | INFO | "huge_shared" : " 27 GB", | 20260506 00:22:28.397 | INFO | "malloc" : " 14 GB", | 20260506 00:22:28.397 | INFO | "pinned" : " 29 GB" | 20260506 00:22:28.397 | INFO | } | 20260506 00:22:28.397 | INFO | Perf report for the last 5.13519 seconds: | 20260506 00:22:28.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:22:28.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:28.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:28.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:28.397 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 00:22:28.397 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:28.397 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:28.397 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:22:28.397 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:28.397 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:28.397 | INFO | Uploaded 36GiB to GPU in 2.10956s (17.0651GiB/s) | 20260506 00:22:28.398 | INFO | Downloaded 34GiB to GPU in 2.04604s (16.6175GiB/s) | 20260506 00:22:33.538 | INFO | Analyzing chunk frames = [7477760,7478272) | 20260506 00:22:33.538 | INFO | MemInfo: { | 20260506 00:22:33.538 | INFO | "gpu" : " 17 GB", | 20260506 00:22:33.538 | INFO | "huge" : "", | 20260506 00:22:33.538 | INFO | "huge_shared" : " 27 GB", | 20260506 00:22:33.538 | INFO | "malloc" : " 14 GB", | 20260506 00:22:33.538 | INFO | "pinned" : " 29 GB" | 20260506 00:22:33.538 | INFO | } | 20260506 00:22:33.538 | INFO | Perf report for the last 5.14103 seconds: | 20260506 00:22:33.538 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:22:33.538 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:33.538 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:33.538 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:33.538 | INFO | ANALYSIS : 61.3%, 3.0, 95.8ms, 0 | 20260506 00:22:33.538 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:33.538 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:33.538 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 00:22:33.538 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:33.538 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:33.539 | INFO | Uploaded 36GiB to GPU in 2.07188s (17.3756GiB/s) | 20260506 00:22:33.539 | INFO | Downloaded 34GiB to GPU in 2.036s (16.6994GiB/s) | 20260506 00:22:38.676 | INFO | Analyzing chunk frames = [7478272,7478784) | 20260506 00:22:38.676 | INFO | MemInfo: { | 20260506 00:22:38.676 | INFO | "gpu" : " 17 GB", | 20260506 00:22:38.676 | INFO | "huge" : "", | 20260506 00:22:38.676 | INFO | "huge_shared" : " 27 GB", | 20260506 00:22:38.676 | INFO | "malloc" : " 14 GB", | 20260506 00:22:38.676 | INFO | "pinned" : " 29 GB" | 20260506 00:22:38.676 | INFO | } | 20260506 00:22:38.677 | INFO | Perf report for the last 5.1382 seconds: | 20260506 00:22:38.677 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:22:38.677 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:38.677 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:38.677 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:38.677 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260506 00:22:38.677 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:38.677 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:38.677 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 00:22:38.677 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:38.677 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:38.677 | INFO | Uploaded 36GiB to GPU in 2.04665s (17.5897GiB/s) | 20260506 00:22:38.677 | INFO | Downloaded 34GiB to GPU in 2.06969s (16.4276GiB/s) | 20260506 00:22:43.813 | INFO | Analyzing chunk frames = [7478784,7479296) | 20260506 00:22:43.813 | INFO | MemInfo: { | 20260506 00:22:43.813 | INFO | "gpu" : " 17 GB", | 20260506 00:22:43.813 | INFO | "huge" : "", | 20260506 00:22:43.813 | INFO | "huge_shared" : " 27 GB", | 20260506 00:22:43.813 | INFO | "malloc" : " 14 GB", | 20260506 00:22:43.813 | INFO | "pinned" : " 29 GB" | 20260506 00:22:43.813 | INFO | } | 20260506 00:22:43.813 | INFO | Perf report for the last 5.13627 seconds: | 20260506 00:22:43.813 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:22:43.813 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:43.813 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:43.813 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:43.813 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260506 00:22:43.813 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:43.813 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:43.813 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:22:43.813 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:43.813 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:43.813 | INFO | Uploaded 36GiB to GPU in 2.12904s (16.909GiB/s) | 20260506 00:22:43.813 | INFO | Downloaded 34GiB to GPU in 2.02152s (16.8191GiB/s) | 20260506 00:22:43.813 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 00:22:43.813 | INFO | Total mem usage: 15.3 GiB | 20260506 00:22:43.813 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 00:22:43.813 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.13%) | 20260506 00:22:43.813 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 00:22:43.813 | INFO | High Water Breakdown: | 20260506 00:22:43.813 | INFO | 60.9% : 8.55 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 00:22:43.813 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 00:22:43.813 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 00:22:43.813 | INFO | ------------------------------------------ | 20260506 00:22:43.813 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:22:43.813 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 00:22:43.813 | INFO | Total mem usage: 26.5 GiB | 20260506 00:22:43.813 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 00:22:43.813 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 00:22:43.813 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260506 00:22:43.813 | INFO | High Water Breakdown: | 20260506 00:22:43.813 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:22:43.813 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:22:43.813 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:22:43.813 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:22:43.813 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:22:43.813 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:22:43.813 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 00:22:43.813 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 00:22:43.813 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:22:43.813 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 00:22:43.813 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 00:22:43.813 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 00:22:43.813 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 00:22:43.813 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 00:22:43.813 | INFO | ------------------------------------------ | 20260506 00:22:43.813 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:22:43.813 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 00:22:43.813 | INFO | Total mem usage: 25.6 GiB | 20260506 00:22:43.813 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 00:22:43.813 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 00:22:43.813 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 00:22:43.813 | INFO | High Water Breakdown: | 20260506 00:22:43.813 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 00:22:43.813 | INFO | ------------------------------------------ | 20260506 00:22:43.814 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:22:43.814 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 00:22:43.814 | INFO | Total mem usage: 15.7 GiB | 20260506 00:22:43.814 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 00:22:43.814 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 00:22:43.814 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 00:22:43.814 | INFO | High Water Breakdown: | 20260506 00:22:43.814 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 00:22:43.814 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 00:22:43.814 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 00:22:43.814 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 00:22:43.814 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 00:22:43.814 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 00:22:43.814 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 00:22:43.814 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 00:22:43.814 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 00:22:43.814 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:22:43.814 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 00:22:43.814 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:22:43.814 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 00:22:43.814 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 00:22:43.814 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 00:22:43.814 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 00:22:43.814 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:22:43.814 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 00:22:43.814 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:22:43.814 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:22:43.814 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:22:43.814 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 00:22:43.814 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 00:22:43.814 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:22:43.814 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 00:22:43.814 | INFO | ------------------------------------------ | 20260506 00:22:43.814 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:22:43.814 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 00:22:43.814 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:22:43.814 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:22:43.814 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.40e+01, 1.34e+01 / 9.28e-01 | 20260506 00:22:43.814 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.04e-01, 1.02e-02 / 4.71e-03 | 20260506 00:22:43.814 | INFO | PerfReport: 11.84%, Baseline -- 3.71e+00 / 4.52e+00, 5.70e-01 / 1.95e-01 | 20260506 00:22:43.814 | INFO | PerfReport: 3.29%, Histogram -- 1.03e+00 / 1.51e+00, 1.92e-01 / 1.18e-01 | 20260506 00:22:43.814 | INFO | PerfReport: 15.99%, DME -- 4.01e+01 / 4.40e+01, 3.85e+00 / 1.24e+00 | 20260506 00:22:43.814 | INFO | PerfReport: 57.35%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.51e+00 / 1.05e+00 | 20260506 00:22:43.814 | INFO | PerfReport: 7.08%, PulseAccumulating -- 2.22e+00 / 2.76e+00, 4.18e-01 / 2.13e-01 | 20260506 00:22:43.814 | INFO | PerfReport: 3.08%, Metrics -- 9.67e-01 / 1.25e+00, 2.61e-01 / 8.91e-02 | 20260506 00:22:43.814 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.35e-01 / 3.88e-01, 5.69e-02 / 2.83e-02 | 20260506 00:22:43.814 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 00:22:43.814 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:22:43.814 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:22:43.815 | INFO | PerfReport: Full Execution -- 9.82e+01 / -nan, 2.44e+01 / 0.00e+00 | 20260506 00:22:43.815 | INFO | PerfReport: 22.25%, Upload -- 2.19e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260506 00:22:43.815 | INFO | PerfReport: 21.84%, Download -- 2.15e+01 / 0.00e+00, 7.50e+00 / 0.00e+00 | 20260506 00:22:43.815 | INFO | PerfReport: 31.91%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 00:22:43.815 | INFO | PerfReport: 2.10%, LockUpload -- 2.06e+00 / 0.00e+00, 7.83e+00 / 0.00e+00 | 20260506 00:22:43.815 | INFO | PerfReport: 1.30%, LockDownload -- 1.27e+00 / 0.00e+00, 2.87e+00 / 0.00e+00 | 20260506 00:22:43.815 | INFO | PerfReport: 20.59%, LockCompute -- 2.02e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 00:22:43.815 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.38e-02 / 0.00e+00 | 20260506 00:22:43.938 | INFO | Starting a round of Baz IO | 20260506 00:22:47.081 | INFO | Populating RTMetrics report | 20260506 00:22:47.082 | INFO | RTMetrics report populated | 20260506 00:22:47.082 | INFO | Opening tmp RTMetrics json file | 20260506 00:22:47.087 | INFO | Serializing RTMetrics to json | 20260506 00:22:47.089 | INFO | Serialization done, swapping tmp file now... | 20260506 00:22:47.089 | INFO | RTMetrics done | 20260506 00:22:47.173 | INFO | Finished a round of Baz IO: Wrote 1.862 GiB in 3.235 seconds | 20260506 00:22:47.173 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.37% / 0.00% / 78.63% | 20260506 00:22:47.173 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0940522% / 3.97364e-06% / 9.13938e-05% | 20260506 00:22:47.173 | INFO | Baz IO overall: | 20260506 00:22:47.174 | INFO | BytesWritten : 4251040179719 | 20260506 00:22:47.174 | INFO | NumEvents : 2000640092397 | 20260506 00:22:48.951 | INFO | Analyzing chunk frames = [7479296,7479808) | 20260506 00:22:48.951 | INFO | MemInfo: { | 20260506 00:22:48.951 | INFO | "gpu" : " 17 GB", | 20260506 00:22:48.951 | INFO | "huge" : "", | 20260506 00:22:48.951 | INFO | "huge_shared" : " 27 GB", | 20260506 00:22:48.951 | INFO | "malloc" : " 14 GB", | 20260506 00:22:48.951 | INFO | "pinned" : " 29 GB" | 20260506 00:22:48.951 | INFO | } | 20260506 00:22:48.951 | INFO | Perf report for the last 5.13832 seconds: | 20260506 00:22:48.951 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:22:48.951 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:48.951 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:48.951 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:48.951 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260506 00:22:48.951 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:48.951 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260506 00:22:48.951 | INFO | PRE_HQ : 60.1%, 1.7, 54.4ms, 0 | 20260506 00:22:48.951 | INFO | BAZWRITER : 62.1%, 3.6, 114.8ms, 0 | 20260506 00:22:48.951 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260506 00:22:48.951 | INFO | Uploaded 36GiB to GPU in 2.1744s (16.5563GiB/s) | 20260506 00:22:48.951 | INFO | Downloaded 38GiB to GPU in 2.24537s (16.9237GiB/s) | 20260506 00:22:54.093 | INFO | Analyzing chunk frames = [7479808,7480320) | 20260506 00:22:54.093 | INFO | MemInfo: { | 20260506 00:22:54.093 | INFO | "gpu" : " 17 GB", | 20260506 00:22:54.093 | INFO | "huge" : "", | 20260506 00:22:54.093 | INFO | "huge_shared" : " 27 GB", | 20260506 00:22:54.093 | INFO | "malloc" : " 14 GB", | 20260506 00:22:54.093 | INFO | "pinned" : " 29 GB" | 20260506 00:22:54.093 | INFO | } | 20260506 00:22:54.093 | INFO | Perf report for the last 5.14158 seconds: | 20260506 00:22:54.093 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:22:54.093 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:54.093 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:54.093 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:54.093 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260506 00:22:54.093 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:54.093 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:54.093 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:22:54.093 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:54.093 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:54.093 | INFO | Uploaded 36GiB to GPU in 2.08255s (17.2865GiB/s) | 20260506 00:22:54.093 | INFO | Downloaded 34GiB to GPU in 2.02416s (16.7971GiB/s) | 20260506 00:22:59.236 | INFO | Analyzing chunk frames = [7480320,7480832) | 20260506 00:22:59.236 | INFO | MemInfo: { | 20260506 00:22:59.236 | INFO | "gpu" : " 17 GB", | 20260506 00:22:59.236 | INFO | "huge" : "", | 20260506 00:22:59.236 | INFO | "huge_shared" : " 27 GB", | 20260506 00:22:59.236 | INFO | "malloc" : " 14 GB", | 20260506 00:22:59.237 | INFO | "pinned" : " 29 GB" | 20260506 00:22:59.237 | INFO | } | 20260506 00:22:59.237 | INFO | Perf report for the last 5.14386 seconds: | 20260506 00:22:59.237 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:22:59.237 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:59.237 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:59.237 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:59.237 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 00:22:59.237 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:59.237 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:59.237 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:22:59.237 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:59.237 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:22:59.237 | INFO | Uploaded 36GiB to GPU in 2.0791s (17.3152GiB/s) | 20260506 00:22:59.237 | INFO | Downloaded 34GiB to GPU in 2.02302s (16.8066GiB/s) | 20260506 00:23:04.378 | INFO | Analyzing chunk frames = [7480832,7481344) | 20260506 00:23:04.378 | INFO | MemInfo: { | 20260506 00:23:04.378 | INFO | "gpu" : " 17 GB", | 20260506 00:23:04.378 | INFO | "huge" : "", | 20260506 00:23:04.378 | INFO | "huge_shared" : " 27 GB", | 20260506 00:23:04.379 | INFO | "malloc" : " 14 GB", | 20260506 00:23:04.379 | INFO | "pinned" : " 29 GB" | 20260506 00:23:04.379 | INFO | } | 20260506 00:23:04.379 | INFO | Perf report for the last 5.14204 seconds: | 20260506 00:23:04.379 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:23:04.379 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:04.379 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:04.379 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:04.379 | INFO | ANALYSIS : 61.6%, 3.0, 95.7ms, 0 | 20260506 00:23:04.379 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:04.379 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:04.379 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:23:04.379 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:04.379 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:04.379 | INFO | Uploaded 36GiB to GPU in 2.13602s (16.8537GiB/s) | 20260506 00:23:04.379 | INFO | Downloaded 34GiB to GPU in 2.01984s (16.833GiB/s) | 20260506 00:23:09.512 | INFO | Analyzing chunk frames = [7481344,7481856) | 20260506 00:23:09.512 | INFO | MemInfo: { | 20260506 00:23:09.512 | INFO | "gpu" : " 17 GB", | 20260506 00:23:09.512 | INFO | "huge" : "", | 20260506 00:23:09.512 | INFO | "huge_shared" : " 27 GB", | 20260506 00:23:09.512 | INFO | "malloc" : " 14 GB", | 20260506 00:23:09.512 | INFO | "pinned" : " 29 GB" | 20260506 00:23:09.512 | INFO | } | 20260506 00:23:09.512 | INFO | Perf report for the last 5.13377 seconds: | 20260506 00:23:09.512 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:23:09.512 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:09.512 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:09.512 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:09.513 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 00:23:09.513 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:09.513 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:09.513 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 00:23:09.513 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:09.513 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:09.513 | INFO | Uploaded 36GiB to GPU in 2.09391s (17.1927GiB/s) | 20260506 00:23:09.513 | INFO | Downloaded 34GiB to GPU in 2.04692s (16.6103GiB/s) | 20260506 00:23:14.651 | INFO | Analyzing chunk frames = [7481856,7482368) | 20260506 00:23:14.651 | INFO | MemInfo: { | 20260506 00:23:14.651 | INFO | "gpu" : " 17 GB", | 20260506 00:23:14.651 | INFO | "huge" : "", | 20260506 00:23:14.651 | INFO | "huge_shared" : " 27 GB", | 20260506 00:23:14.651 | INFO | "malloc" : " 14 GB", | 20260506 00:23:14.651 | INFO | "pinned" : " 29 GB" | 20260506 00:23:14.651 | INFO | } | 20260506 00:23:14.651 | INFO | Perf report for the last 5.13879 seconds: | 20260506 00:23:14.651 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:23:14.651 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:14.651 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:14.651 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:14.651 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 00:23:14.651 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:14.651 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:14.651 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 00:23:14.651 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:14.651 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:14.651 | INFO | Uploaded 36GiB to GPU in 2.07914s (17.3149GiB/s) | 20260506 00:23:14.652 | INFO | Downloaded 34GiB to GPU in 2.04261s (16.6454GiB/s) | 20260506 00:23:19.792 | INFO | Analyzing chunk frames = [7482368,7482880) | 20260506 00:23:19.792 | INFO | MemInfo: { | 20260506 00:23:19.792 | INFO | "gpu" : " 17 GB", | 20260506 00:23:19.792 | INFO | "huge" : "", | 20260506 00:23:19.792 | INFO | "huge_shared" : " 27 GB", | 20260506 00:23:19.792 | INFO | "malloc" : " 14 GB", | 20260506 00:23:19.792 | INFO | "pinned" : " 29 GB" | 20260506 00:23:19.792 | INFO | } | 20260506 00:23:19.792 | INFO | Perf report for the last 5.14095 seconds: | 20260506 00:23:19.792 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:23:19.792 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:19.792 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:19.792 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:19.792 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260506 00:23:19.792 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:19.792 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:19.792 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 00:23:19.792 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:19.792 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:19.792 | INFO | Uploaded 36GiB to GPU in 2.08154s (17.2949GiB/s) | 20260506 00:23:19.793 | INFO | Downloaded 34GiB to GPU in 2.08111s (16.3374GiB/s) | 20260506 00:23:24.933 | INFO | Analyzing chunk frames = [7482880,7483392) | 20260506 00:23:24.933 | INFO | MemInfo: { | 20260506 00:23:24.933 | INFO | "gpu" : " 17 GB", | 20260506 00:23:24.933 | INFO | "huge" : "", | 20260506 00:23:24.933 | INFO | "huge_shared" : " 27 GB", | 20260506 00:23:24.933 | INFO | "malloc" : " 14 GB", | 20260506 00:23:24.934 | INFO | "pinned" : " 29 GB" | 20260506 00:23:24.934 | INFO | } | 20260506 00:23:24.934 | INFO | Perf report for the last 5.14135 seconds: | 20260506 00:23:24.934 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:23:24.934 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:24.934 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:24.934 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:24.934 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260506 00:23:24.934 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:24.934 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:24.934 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:23:24.934 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:24.934 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:24.934 | INFO | Uploaded 36GiB to GPU in 2.13394s (16.8702GiB/s) | 20260506 00:23:24.934 | INFO | Downloaded 34GiB to GPU in 2.01965s (16.8346GiB/s) | 20260506 00:23:25.052 | INFO | Starting a round of Baz IO | 20260506 00:23:28.198 | INFO | Populating RTMetrics report | 20260506 00:23:28.199 | INFO | RTMetrics report populated | 20260506 00:23:28.199 | INFO | Opening tmp RTMetrics json file | 20260506 00:23:28.200 | INFO | Serializing RTMetrics to json | 20260506 00:23:28.202 | INFO | Serialization done, swapping tmp file now... | 20260506 00:23:28.202 | INFO | RTMetrics done | 20260506 00:23:28.274 | INFO | Finished a round of Baz IO: Wrote 1.862 GiB in 3.221 seconds | 20260506 00:23:28.274 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.37% / 0.00% / 78.63% | 20260506 00:23:28.274 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0937263% / 0% / 8.74201e-05% | 20260506 00:23:28.274 | INFO | Baz IO overall: | 20260506 00:23:28.274 | INFO | BytesWritten : 4253039193338 | 20260506 00:23:28.274 | INFO | NumEvents : 2001554417274 | 20260506 00:23:30.073 | INFO | Analyzing chunk frames = [7483392,7483904) | 20260506 00:23:30.073 | INFO | MemInfo: { | 20260506 00:23:30.073 | INFO | "gpu" : " 17 GB", | 20260506 00:23:30.073 | INFO | "huge" : "", | 20260506 00:23:30.073 | INFO | "huge_shared" : " 27 GB", | 20260506 00:23:30.073 | INFO | "malloc" : " 14 GB", | 20260506 00:23:30.073 | INFO | "pinned" : " 29 GB" | 20260506 00:23:30.073 | INFO | } | 20260506 00:23:30.073 | INFO | Perf report for the last 5.13943 seconds: | 20260506 00:23:30.073 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:23:30.073 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:30.073 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:30.073 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:30.073 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260506 00:23:30.073 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:30.073 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260506 00:23:30.073 | INFO | PRE_HQ : 59.6%, 1.7, 53.8ms, 0 | 20260506 00:23:30.073 | INFO | BAZWRITER : 61.8%, 3.6, 114.9ms, 0 | 20260506 00:23:30.073 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:23:30.073 | INFO | Uploaded 36GiB to GPU in 2.17926s (16.5194GiB/s) | 20260506 00:23:30.073 | INFO | Downloaded 38GiB to GPU in 2.2417s (16.9515GiB/s) | 20260506 00:23:35.211 | INFO | Analyzing chunk frames = [7483904,7484416) | 20260506 00:23:35.211 | INFO | MemInfo: { | 20260506 00:23:35.211 | INFO | "gpu" : " 17 GB", | 20260506 00:23:35.211 | INFO | "huge" : "", | 20260506 00:23:35.211 | INFO | "huge_shared" : " 27 GB", | 20260506 00:23:35.211 | INFO | "malloc" : " 14 GB", | 20260506 00:23:35.211 | INFO | "pinned" : " 29 GB" | 20260506 00:23:35.211 | INFO | } | 20260506 00:23:35.211 | INFO | Perf report for the last 5.13827 seconds: | 20260506 00:23:35.211 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:23:35.211 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:35.211 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:35.211 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:35.211 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260506 00:23:35.211 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:35.211 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:35.211 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:23:35.211 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:35.211 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:35.212 | INFO | Uploaded 36GiB to GPU in 2.10544s (17.0985GiB/s) | 20260506 00:23:35.212 | INFO | Downloaded 34GiB to GPU in 2.04239s (16.6471GiB/s) | 20260506 00:23:40.349 | INFO | Analyzing chunk frames = [7484416,7484928) | 20260506 00:23:40.349 | INFO | MemInfo: { | 20260506 00:23:40.349 | INFO | "gpu" : " 17 GB", | 20260506 00:23:40.350 | INFO | "huge" : "", | 20260506 00:23:40.350 | INFO | "huge_shared" : " 27 GB", | 20260506 00:23:40.350 | INFO | "malloc" : " 14 GB", | 20260506 00:23:40.350 | INFO | "pinned" : " 29 GB" | 20260506 00:23:40.350 | INFO | } | 20260506 00:23:40.350 | INFO | Perf report for the last 5.1384 seconds: | 20260506 00:23:40.350 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:23:40.350 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:40.350 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:40.350 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:40.350 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 00:23:40.350 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:40.350 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:40.350 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:23:40.350 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:40.350 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:40.350 | INFO | Uploaded 36GiB to GPU in 2.15166s (16.7312GiB/s) | 20260506 00:23:40.350 | INFO | Downloaded 34GiB to GPU in 2.06014s (16.5037GiB/s) | 20260506 00:23:45.486 | INFO | Analyzing chunk frames = [7484928,7485440) | 20260506 00:23:45.486 | INFO | MemInfo: { | 20260506 00:23:45.486 | INFO | "gpu" : " 17 GB", | 20260506 00:23:45.486 | INFO | "huge" : "", | 20260506 00:23:45.486 | INFO | "huge_shared" : " 27 GB", | 20260506 00:23:45.486 | INFO | "malloc" : " 14 GB", | 20260506 00:23:45.486 | INFO | "pinned" : " 29 GB" | 20260506 00:23:45.486 | INFO | } | 20260506 00:23:45.487 | INFO | Perf report for the last 5.13679 seconds: | 20260506 00:23:45.487 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:23:45.487 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:45.487 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:45.487 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:45.487 | INFO | ANALYSIS : 61.6%, 3.0, 95.7ms, 0 | 20260506 00:23:45.487 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:45.487 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:45.487 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:23:45.487 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:45.487 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:45.487 | INFO | Uploaded 36GiB to GPU in 2.13572s (16.8561GiB/s) | 20260506 00:23:45.487 | INFO | Downloaded 34GiB to GPU in 2.02538s (16.787GiB/s) | 20260506 00:23:50.628 | INFO | Analyzing chunk frames = [7485440,7485952) | 20260506 00:23:50.628 | INFO | MemInfo: { | 20260506 00:23:50.628 | INFO | "gpu" : " 17 GB", | 20260506 00:23:50.628 | INFO | "huge" : "", | 20260506 00:23:50.628 | INFO | "huge_shared" : " 27 GB", | 20260506 00:23:50.628 | INFO | "malloc" : " 14 GB", | 20260506 00:23:50.628 | INFO | "pinned" : " 29 GB" | 20260506 00:23:50.628 | INFO | } | 20260506 00:23:50.628 | INFO | Perf report for the last 5.14146 seconds: | 20260506 00:23:50.628 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:23:50.628 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:50.628 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:50.628 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:50.628 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 00:23:50.628 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:50.628 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:50.628 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 00:23:50.628 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:50.628 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:50.629 | INFO | Uploaded 36GiB to GPU in 2.0885s (17.2372GiB/s) | 20260506 00:23:50.629 | INFO | Downloaded 34GiB to GPU in 2.04847s (16.5978GiB/s) | 20260506 00:23:55.763 | INFO | Analyzing chunk frames = [7485952,7486464) | 20260506 00:23:55.763 | INFO | MemInfo: { | 20260506 00:23:55.763 | INFO | "gpu" : " 17 GB", | 20260506 00:23:55.763 | INFO | "huge" : "", | 20260506 00:23:55.763 | INFO | "huge_shared" : " 27 GB", | 20260506 00:23:55.763 | INFO | "malloc" : " 14 GB", | 20260506 00:23:55.763 | INFO | "pinned" : " 29 GB" | 20260506 00:23:55.763 | INFO | } | 20260506 00:23:55.763 | INFO | Perf report for the last 5.13515 seconds: | 20260506 00:23:55.763 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:23:55.763 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:55.763 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:55.763 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:55.763 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 00:23:55.763 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:55.763 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:55.763 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:23:55.763 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:55.763 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:23:55.763 | INFO | Uploaded 36GiB to GPU in 2.06904s (17.3994GiB/s) | 20260506 00:23:55.763 | INFO | Downloaded 34GiB to GPU in 2.04062s (16.6616GiB/s) | 20260506 00:24:00.902 | INFO | Analyzing chunk frames = [7486464,7486976) | 20260506 00:24:00.902 | INFO | MemInfo: { | 20260506 00:24:00.902 | INFO | "gpu" : " 17 GB", | 20260506 00:24:00.902 | INFO | "huge" : "", | 20260506 00:24:00.902 | INFO | "huge_shared" : " 27 GB", | 20260506 00:24:00.902 | INFO | "malloc" : " 14 GB", | 20260506 00:24:00.902 | INFO | "pinned" : " 29 GB" | 20260506 00:24:00.902 | INFO | } | 20260506 00:24:00.902 | INFO | Perf report for the last 5.13861 seconds: | 20260506 00:24:00.902 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:24:00.902 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:00.902 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:00.902 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:00.902 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260506 00:24:00.902 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:00.902 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:00.902 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 00:24:00.902 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:00.902 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:00.902 | INFO | Uploaded 36GiB to GPU in 2.03822s (17.6625GiB/s) | 20260506 00:24:00.902 | INFO | Downloaded 34GiB to GPU in 2.04298s (16.6423GiB/s) | 20260506 00:24:06.047 | INFO | Analyzing chunk frames = [7486976,7487488) | 20260506 00:24:06.047 | INFO | MemInfo: { | 20260506 00:24:06.047 | INFO | "gpu" : " 17 GB", | 20260506 00:24:06.047 | INFO | "huge" : "", | 20260506 00:24:06.047 | INFO | "huge_shared" : " 27 GB", | 20260506 00:24:06.047 | INFO | "malloc" : " 14 GB", | 20260506 00:24:06.047 | INFO | "pinned" : " 29 GB" | 20260506 00:24:06.047 | INFO | } | 20260506 00:24:06.047 | INFO | Perf report for the last 5.14518 seconds: | 20260506 00:24:06.047 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:24:06.047 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:06.047 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:06.047 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:06.047 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260506 00:24:06.047 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:06.047 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:06.047 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:24:06.047 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:06.047 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:06.047 | INFO | Uploaded 36GiB to GPU in 2.11179s (17.0471GiB/s) | 20260506 00:24:06.047 | INFO | Downloaded 34GiB to GPU in 2.00062s (16.9947GiB/s) | 20260506 00:24:06.047 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 00:24:06.047 | INFO | Total mem usage: 15.3 GiB | 20260506 00:24:06.047 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 00:24:06.047 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.39%) | 20260506 00:24:06.047 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 00:24:06.047 | INFO | High Water Breakdown: | 20260506 00:24:06.047 | INFO | 61% : 8.62 GiB ( 90.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 00:24:06.047 | INFO | 38.9% : 5.5 GiB ( 82.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 00:24:06.047 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 00:24:06.047 | INFO | ------------------------------------------ | 20260506 00:24:06.047 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:24:06.047 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 00:24:06.047 | INFO | Total mem usage: 26.5 GiB | 20260506 00:24:06.047 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 00:24:06.047 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 00:24:06.047 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 00:24:06.047 | INFO | High Water Breakdown: | 20260506 00:24:06.047 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:24:06.048 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:24:06.048 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:24:06.048 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:24:06.048 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:24:06.048 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:24:06.048 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 00:24:06.048 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 00:24:06.048 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:24:06.048 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 00:24:06.048 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 00:24:06.048 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 00:24:06.048 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 00:24:06.048 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 00:24:06.048 | INFO | ------------------------------------------ | 20260506 00:24:06.048 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:24:06.048 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 00:24:06.048 | INFO | Total mem usage: 25.6 GiB | 20260506 00:24:06.048 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 00:24:06.048 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 00:24:06.048 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 00:24:06.048 | INFO | High Water Breakdown: | 20260506 00:24:06.048 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 00:24:06.048 | INFO | ------------------------------------------ | 20260506 00:24:06.048 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:24:06.048 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 00:24:06.048 | INFO | Total mem usage: 15.7 GiB | 20260506 00:24:06.048 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 00:24:06.048 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 00:24:06.048 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 00:24:06.048 | INFO | High Water Breakdown: | 20260506 00:24:06.048 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 00:24:06.048 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 00:24:06.048 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 00:24:06.048 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 00:24:06.048 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 00:24:06.048 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 00:24:06.048 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 00:24:06.048 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 00:24:06.048 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 00:24:06.048 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:24:06.048 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 00:24:06.048 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:24:06.048 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 00:24:06.048 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 00:24:06.048 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 00:24:06.048 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 00:24:06.048 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:24:06.048 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 00:24:06.048 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:24:06.048 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:24:06.048 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:24:06.048 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 00:24:06.048 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 00:24:06.048 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:24:06.048 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 00:24:06.048 | INFO | ------------------------------------------ | 20260506 00:24:06.048 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:24:06.048 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 00:24:06.048 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:24:06.048 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:24:06.048 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.39e+01, 1.34e+01 / 8.49e-01 | 20260506 00:24:06.048 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.04e-01, 1.08e-02 / 4.08e-03 | 20260506 00:24:06.048 | INFO | PerfReport: 11.84%, Baseline -- 3.70e+00 / 4.51e+00, 5.68e-01 / 1.91e-01 | 20260506 00:24:06.048 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.52e+00, 1.88e-01 / 9.40e-02 | 20260506 00:24:06.048 | INFO | PerfReport: 15.99%, DME -- 4.00e+01 / 4.39e+01, 3.87e+00 / 1.07e+00 | 20260506 00:24:06.048 | INFO | PerfReport: 57.33%, FrameLabeling -- 1.79e+01 / 1.94e+01, 1.47e+00 / 9.60e-01 | 20260506 00:24:06.048 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.23e+00 / 2.76e+00, 4.26e-01 / 2.06e-01 | 20260506 00:24:06.048 | INFO | PerfReport: 3.09%, Metrics -- 9.67e-01 / 1.25e+00, 2.62e-01 / 1.23e-01 | 20260506 00:24:06.048 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.34e-01 / 3.88e-01, 5.70e-02 / 3.00e-02 | 20260506 00:24:06.048 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 00:24:06.048 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:24:06.048 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:24:06.048 | INFO | PerfReport: Full Execution -- 9.83e+01 / -nan, 2.46e+01 / 0.00e+00 | 20260506 00:24:06.048 | INFO | PerfReport: 22.35%, Upload -- 2.20e+01 / 0.00e+00, 7.57e+00 / 0.00e+00 | 20260506 00:24:06.048 | INFO | PerfReport: 21.86%, Download -- 2.15e+01 / 0.00e+00, 7.64e+00 / 0.00e+00 | 20260506 00:24:06.048 | INFO | PerfReport: 31.84%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 00:24:06.048 | INFO | PerfReport: 2.13%, LockUpload -- 2.09e+00 / 0.00e+00, 7.91e+00 / 0.00e+00 | 20260506 00:24:06.048 | INFO | PerfReport: 1.31%, LockDownload -- 1.28e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260506 00:24:06.048 | INFO | PerfReport: 20.50%, LockCompute -- 2.01e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 00:24:06.048 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.81e-02 / 0.00e+00 | 20260506 00:24:06.164 | INFO | Starting a round of Baz IO | 20260506 00:24:09.313 | INFO | Populating RTMetrics report | 20260506 00:24:09.313 | INFO | RTMetrics report populated | 20260506 00:24:09.313 | INFO | Opening tmp RTMetrics json file | 20260506 00:24:09.314 | INFO | Serializing RTMetrics to json | 20260506 00:24:09.315 | INFO | Serialization done, swapping tmp file now... | 20260506 00:24:09.315 | INFO | RTMetrics done | 20260506 00:24:09.390 | INFO | Finished a round of Baz IO: Wrote 1.861 GiB in 3.226 seconds | 20260506 00:24:09.390 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.37% / 0.00% / 78.63% | 20260506 00:24:09.390 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0935475% / 3.97364e-06% / 8.74201e-05% | 20260506 00:24:09.390 | INFO | Baz IO overall: | 20260506 00:24:09.390 | INFO | BytesWritten : 4255037389510 | 20260506 00:24:09.390 | INFO | NumEvents : 2002468145289 | 20260506 00:24:11.189 | INFO | Analyzing chunk frames = [7487488,7488000) | 20260506 00:24:11.189 | INFO | MemInfo: { | 20260506 00:24:11.189 | INFO | "gpu" : " 17 GB", | 20260506 00:24:11.189 | INFO | "huge" : "", | 20260506 00:24:11.189 | INFO | "huge_shared" : " 27 GB", | 20260506 00:24:11.189 | INFO | "malloc" : " 14 GB", | 20260506 00:24:11.189 | INFO | "pinned" : " 29 GB" | 20260506 00:24:11.189 | INFO | } | 20260506 00:24:11.189 | INFO | Perf report for the last 5.14218 seconds: | 20260506 00:24:11.189 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:24:11.189 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:11.189 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:11.189 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:11.189 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260506 00:24:11.189 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:11.189 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260506 00:24:11.189 | INFO | PRE_HQ : 59.6%, 1.7, 53.8ms, 0 | 20260506 00:24:11.189 | INFO | BAZWRITER : 61.6%, 3.5, 112.4ms, 0 | 20260506 00:24:11.189 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:24:11.189 | INFO | Uploaded 36GiB to GPU in 2.1832s (16.4896GiB/s) | 20260506 00:24:11.189 | INFO | Downloaded 38GiB to GPU in 2.24468s (16.9289GiB/s) | 20260506 00:24:16.323 | INFO | Analyzing chunk frames = [7488000,7488512) | 20260506 00:24:16.323 | INFO | MemInfo: { | 20260506 00:24:16.323 | INFO | "gpu" : " 17 GB", | 20260506 00:24:16.323 | INFO | "huge" : "", | 20260506 00:24:16.323 | INFO | "huge_shared" : " 27 GB", | 20260506 00:24:16.323 | INFO | "malloc" : " 14 GB", | 20260506 00:24:16.323 | INFO | "pinned" : " 29 GB" | 20260506 00:24:16.323 | INFO | } | 20260506 00:24:16.323 | INFO | Perf report for the last 5.13425 seconds: | 20260506 00:24:16.323 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:24:16.323 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:16.323 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:16.323 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:16.323 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 00:24:16.323 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:16.323 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:16.324 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:24:16.324 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:16.324 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:16.324 | INFO | Uploaded 36GiB to GPU in 2.11355s (17.0329GiB/s) | 20260506 00:24:16.324 | INFO | Downloaded 34GiB to GPU in 2.04372s (16.6364GiB/s) | 20260506 00:24:21.462 | INFO | Analyzing chunk frames = [7488512,7489024) | 20260506 00:24:21.462 | INFO | MemInfo: { | 20260506 00:24:21.462 | INFO | "gpu" : " 17 GB", | 20260506 00:24:21.462 | INFO | "huge" : "", | 20260506 00:24:21.462 | INFO | "huge_shared" : " 27 GB", | 20260506 00:24:21.462 | INFO | "malloc" : " 14 GB", | 20260506 00:24:21.462 | INFO | "pinned" : " 29 GB" | 20260506 00:24:21.462 | INFO | } | 20260506 00:24:21.462 | INFO | Perf report for the last 5.13846 seconds: | 20260506 00:24:21.462 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:24:21.462 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:21.462 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:21.462 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:21.462 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 00:24:21.462 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:21.462 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:21.462 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:24:21.462 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:21.462 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:21.462 | INFO | Uploaded 36GiB to GPU in 2.10206s (17.1261GiB/s) | 20260506 00:24:21.462 | INFO | Downloaded 34GiB to GPU in 2.03302s (16.7239GiB/s) | 20260506 00:24:26.607 | INFO | Analyzing chunk frames = [7489024,7489536) | 20260506 00:24:26.607 | INFO | MemInfo: { | 20260506 00:24:26.607 | INFO | "gpu" : " 17 GB", | 20260506 00:24:26.607 | INFO | "huge" : "", | 20260506 00:24:26.607 | INFO | "huge_shared" : " 27 GB", | 20260506 00:24:26.607 | INFO | "malloc" : " 14 GB", | 20260506 00:24:26.607 | INFO | "pinned" : " 29 GB" | 20260506 00:24:26.607 | INFO | } | 20260506 00:24:26.607 | INFO | Perf report for the last 5.14481 seconds: | 20260506 00:24:26.607 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:24:26.607 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:26.607 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:26.607 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:26.607 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 00:24:26.607 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:26.607 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:26.607 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:24:26.607 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:26.607 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:26.607 | INFO | Uploaded 36GiB to GPU in 2.14093s (16.8152GiB/s) | 20260506 00:24:26.607 | INFO | Downloaded 34GiB to GPU in 2.02981s (16.7503GiB/s) | 20260506 00:24:31.741 | INFO | Analyzing chunk frames = [7489536,7490048) | 20260506 00:24:31.741 | INFO | MemInfo: { | 20260506 00:24:31.741 | INFO | "gpu" : " 17 GB", | 20260506 00:24:31.741 | INFO | "huge" : "", | 20260506 00:24:31.741 | INFO | "huge_shared" : " 27 GB", | 20260506 00:24:31.741 | INFO | "malloc" : " 14 GB", | 20260506 00:24:31.741 | INFO | "pinned" : " 29 GB" | 20260506 00:24:31.741 | INFO | } | 20260506 00:24:31.741 | INFO | Perf report for the last 5.13442 seconds: | 20260506 00:24:31.741 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:24:31.741 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:31.741 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:31.741 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:31.741 | INFO | ANALYSIS : 60.9%, 3.0, 95.0ms, 0 | 20260506 00:24:31.741 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:31.741 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:31.741 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 00:24:31.741 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:31.741 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:31.741 | INFO | Uploaded 36GiB to GPU in 2.07116s (17.3816GiB/s) | 20260506 00:24:31.741 | INFO | Downloaded 34GiB to GPU in 2.03515s (16.7064GiB/s) | 20260506 00:24:36.884 | INFO | Analyzing chunk frames = [7490048,7490560) | 20260506 00:24:36.884 | INFO | MemInfo: { | 20260506 00:24:36.884 | INFO | "gpu" : " 17 GB", | 20260506 00:24:36.884 | INFO | "huge" : "", | 20260506 00:24:36.884 | INFO | "huge_shared" : " 27 GB", | 20260506 00:24:36.884 | INFO | "malloc" : " 14 GB", | 20260506 00:24:36.884 | INFO | "pinned" : " 29 GB" | 20260506 00:24:36.884 | INFO | } | 20260506 00:24:36.884 | INFO | Perf report for the last 5.14326 seconds: | 20260506 00:24:36.884 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:24:36.884 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:36.884 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:36.884 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:36.884 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260506 00:24:36.884 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:36.885 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:36.885 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:24:36.885 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:36.885 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:36.885 | INFO | Uploaded 36GiB to GPU in 2.06535s (17.4305GiB/s) | 20260506 00:24:36.885 | INFO | Downloaded 34GiB to GPU in 2.03238s (16.7291GiB/s) | 20260506 00:24:42.022 | INFO | Analyzing chunk frames = [7490560,7491072) | 20260506 00:24:42.022 | INFO | MemInfo: { | 20260506 00:24:42.022 | INFO | "gpu" : " 17 GB", | 20260506 00:24:42.022 | INFO | "huge" : "", | 20260506 00:24:42.022 | INFO | "huge_shared" : " 27 GB", | 20260506 00:24:42.022 | INFO | "malloc" : " 14 GB", | 20260506 00:24:42.022 | INFO | "pinned" : " 29 GB" | 20260506 00:24:42.022 | INFO | } | 20260506 00:24:42.022 | INFO | Perf report for the last 5.13779 seconds: | 20260506 00:24:42.022 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:24:42.022 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:42.022 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:42.022 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:42.022 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 00:24:42.022 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:42.022 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:42.022 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 00:24:42.022 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:42.022 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:42.022 | INFO | Uploaded 36GiB to GPU in 2.03001s (17.7339GiB/s) | 20260506 00:24:42.022 | INFO | Downloaded 34GiB to GPU in 2.0402s (16.665GiB/s) | 20260506 00:24:47.158 | INFO | Analyzing chunk frames = [7491072,7491584) | 20260506 00:24:47.158 | INFO | MemInfo: { | 20260506 00:24:47.158 | INFO | "gpu" : " 17 GB", | 20260506 00:24:47.158 | INFO | "huge" : "", | 20260506 00:24:47.158 | INFO | "huge_shared" : " 27 GB", | 20260506 00:24:47.158 | INFO | "malloc" : " 14 GB", | 20260506 00:24:47.158 | INFO | "pinned" : " 29 GB" | 20260506 00:24:47.158 | INFO | } | 20260506 00:24:47.158 | INFO | Perf report for the last 5.13582 seconds: | 20260506 00:24:47.158 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:24:47.158 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:47.158 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:47.158 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:47.158 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 00:24:47.158 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:47.158 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:47.158 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260506 00:24:47.158 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:47.158 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:47.158 | INFO | Uploaded 36GiB to GPU in 2.07041s (17.3878GiB/s) | 20260506 00:24:47.158 | INFO | Downloaded 34GiB to GPU in 1.9947s (17.0451GiB/s) | 20260506 00:24:47.281 | INFO | Starting a round of Baz IO | 20260506 00:24:50.427 | INFO | Populating RTMetrics report | 20260506 00:24:50.428 | INFO | RTMetrics report populated | 20260506 00:24:50.428 | INFO | Opening tmp RTMetrics json file | 20260506 00:24:50.428 | INFO | Serializing RTMetrics to json | 20260506 00:24:50.430 | INFO | Serialization done, swapping tmp file now... | 20260506 00:24:50.430 | INFO | RTMetrics done | 20260506 00:24:50.506 | INFO | Finished a round of Baz IO: Wrote 1.859 GiB in 3.224 seconds | 20260506 00:24:50.506 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.37% / 0.00% / 78.63% | 20260506 00:24:50.506 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0931223% / 0% / 6.35783e-05% | 20260506 00:24:50.506 | INFO | Baz IO overall: | 20260506 00:24:50.506 | INFO | BytesWritten : 4257033742564 | 20260506 00:24:50.506 | INFO | NumEvents : 2003381153330 | 20260506 00:24:52.301 | INFO | Analyzing chunk frames = [7491584,7492096) | 20260506 00:24:52.301 | INFO | MemInfo: { | 20260506 00:24:52.301 | INFO | "gpu" : " 17 GB", | 20260506 00:24:52.301 | INFO | "huge" : "", | 20260506 00:24:52.301 | INFO | "huge_shared" : " 27 GB", | 20260506 00:24:52.301 | INFO | "malloc" : " 14 GB", | 20260506 00:24:52.301 | INFO | "pinned" : " 29 GB" | 20260506 00:24:52.301 | INFO | } | 20260506 00:24:52.301 | INFO | Perf report for the last 5.14268 seconds: | 20260506 00:24:52.301 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:24:52.301 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:52.301 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:52.301 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:52.301 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260506 00:24:52.301 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:52.301 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260506 00:24:52.301 | INFO | PRE_HQ : 59.8%, 1.7, 54.0ms, 0 | 20260506 00:24:52.301 | INFO | BAZWRITER : 61.9%, 3.6, 114.9ms, 0 | 20260506 00:24:52.301 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:24:52.301 | INFO | Uploaded 36GiB to GPU in 2.17238s (16.5716GiB/s) | 20260506 00:24:52.301 | INFO | Downloaded 38GiB to GPU in 2.24358s (16.9372GiB/s) | 20260506 00:24:57.442 | INFO | Analyzing chunk frames = [7492096,7492608) | 20260506 00:24:57.442 | INFO | MemInfo: { | 20260506 00:24:57.442 | INFO | "gpu" : " 17 GB", | 20260506 00:24:57.442 | INFO | "huge" : "", | 20260506 00:24:57.442 | INFO | "huge_shared" : " 27 GB", | 20260506 00:24:57.442 | INFO | "malloc" : " 14 GB", | 20260506 00:24:57.442 | INFO | "pinned" : " 29 GB" | 20260506 00:24:57.442 | INFO | } | 20260506 00:24:57.442 | INFO | Perf report for the last 5.1416 seconds: | 20260506 00:24:57.442 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:24:57.442 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:57.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:57.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:57.442 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260506 00:24:57.442 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:57.442 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:57.442 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 00:24:57.442 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:57.442 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:24:57.443 | INFO | Uploaded 36GiB to GPU in 2.10817s (17.0764GiB/s) | 20260506 00:24:57.443 | INFO | Downloaded 34GiB to GPU in 2.03775s (16.6851GiB/s) | 20260506 00:25:02.583 | INFO | Analyzing chunk frames = [7492608,7493120) | 20260506 00:25:02.583 | INFO | MemInfo: { | 20260506 00:25:02.583 | INFO | "gpu" : " 17 GB", | 20260506 00:25:02.583 | INFO | "huge" : "", | 20260506 00:25:02.583 | INFO | "huge_shared" : " 27 GB", | 20260506 00:25:02.583 | INFO | "malloc" : " 14 GB", | 20260506 00:25:02.583 | INFO | "pinned" : " 29 GB" | 20260506 00:25:02.583 | INFO | } | 20260506 00:25:02.583 | INFO | Perf report for the last 5.14077 seconds: | 20260506 00:25:02.583 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:25:02.583 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:02.583 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:02.583 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:02.583 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 00:25:02.583 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:02.583 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:02.583 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:25:02.583 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:02.583 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:02.583 | INFO | Uploaded 36GiB to GPU in 2.08035s (17.3048GiB/s) | 20260506 00:25:02.583 | INFO | Downloaded 34GiB to GPU in 2.0212s (16.8216GiB/s) | 20260506 00:25:07.714 | INFO | Analyzing chunk frames = [7493120,7493632) | 20260506 00:25:07.714 | INFO | MemInfo: { | 20260506 00:25:07.714 | INFO | "gpu" : " 17 GB", | 20260506 00:25:07.714 | INFO | "huge" : "", | 20260506 00:25:07.714 | INFO | "huge_shared" : " 27 GB", | 20260506 00:25:07.714 | INFO | "malloc" : " 14 GB", | 20260506 00:25:07.714 | INFO | "pinned" : " 29 GB" | 20260506 00:25:07.714 | INFO | } | 20260506 00:25:07.714 | INFO | Perf report for the last 5.13052 seconds: | 20260506 00:25:07.714 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:25:07.714 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:07.714 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:07.714 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:07.714 | INFO | ANALYSIS : 61.7%, 3.0, 95.7ms, 0 | 20260506 00:25:07.714 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:07.714 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:07.714 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 00:25:07.714 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:07.714 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:07.714 | INFO | Uploaded 36GiB to GPU in 2.13613s (16.8529GiB/s) | 20260506 00:25:07.714 | INFO | Downloaded 34GiB to GPU in 2.02314s (16.8055GiB/s) | 20260506 00:25:12.857 | INFO | Analyzing chunk frames = [7493632,7494144) | 20260506 00:25:12.857 | INFO | MemInfo: { | 20260506 00:25:12.857 | INFO | "gpu" : " 17 GB", | 20260506 00:25:12.857 | INFO | "huge" : "", | 20260506 00:25:12.857 | INFO | "huge_shared" : " 27 GB", | 20260506 00:25:12.857 | INFO | "malloc" : " 14 GB", | 20260506 00:25:12.857 | INFO | "pinned" : " 29 GB" | 20260506 00:25:12.857 | INFO | } | 20260506 00:25:12.857 | INFO | Perf report for the last 5.14296 seconds: | 20260506 00:25:12.857 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:25:12.857 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:12.857 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:12.857 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:12.857 | INFO | ANALYSIS : 61.1%, 3.0, 95.5ms, 0 | 20260506 00:25:12.857 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:12.857 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:12.857 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 00:25:12.857 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:12.857 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:12.857 | INFO | Uploaded 36GiB to GPU in 2.10769s (17.0803GiB/s) | 20260506 00:25:12.857 | INFO | Downloaded 34GiB to GPU in 2.05333s (16.5584GiB/s) | 20260506 00:25:18.001 | INFO | Analyzing chunk frames = [7494144,7494656) | 20260506 00:25:18.001 | INFO | MemInfo: { | 20260506 00:25:18.001 | INFO | "gpu" : " 17 GB", | 20260506 00:25:18.001 | INFO | "huge" : "", | 20260506 00:25:18.001 | INFO | "huge_shared" : " 27 GB", | 20260506 00:25:18.001 | INFO | "malloc" : " 14 GB", | 20260506 00:25:18.001 | INFO | "pinned" : " 29 GB" | 20260506 00:25:18.001 | INFO | } | 20260506 00:25:18.001 | INFO | Perf report for the last 5.14445 seconds: | 20260506 00:25:18.001 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:25:18.001 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:18.001 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:18.001 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:18.001 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260506 00:25:18.001 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:18.001 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:18.001 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 00:25:18.001 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:18.001 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:18.001 | INFO | Uploaded 36GiB to GPU in 2.0777s (17.3269GiB/s) | 20260506 00:25:18.001 | INFO | Downloaded 34GiB to GPU in 2.04107s (16.6579GiB/s) | 20260506 00:25:23.136 | INFO | Analyzing chunk frames = [7494656,7495168) | 20260506 00:25:23.136 | INFO | MemInfo: { | 20260506 00:25:23.136 | INFO | "gpu" : " 17 GB", | 20260506 00:25:23.136 | INFO | "huge" : "", | 20260506 00:25:23.136 | INFO | "huge_shared" : " 27 GB", | 20260506 00:25:23.136 | INFO | "malloc" : " 14 GB", | 20260506 00:25:23.136 | INFO | "pinned" : " 29 GB" | 20260506 00:25:23.136 | INFO | } | 20260506 00:25:23.136 | INFO | Perf report for the last 5.13528 seconds: | 20260506 00:25:23.136 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:25:23.136 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:23.136 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:23.136 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:23.136 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 00:25:23.136 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:23.136 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:23.136 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260506 00:25:23.136 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:23.136 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:23.137 | INFO | Uploaded 36GiB to GPU in 2.05324s (17.5332GiB/s) | 20260506 00:25:23.137 | INFO | Downloaded 34GiB to GPU in 2.07518s (16.3841GiB/s) | 20260506 00:25:28.271 | INFO | Analyzing chunk frames = [7495168,7495680) | 20260506 00:25:28.271 | INFO | MemInfo: { | 20260506 00:25:28.271 | INFO | "gpu" : " 17 GB", | 20260506 00:25:28.271 | INFO | "huge" : "", | 20260506 00:25:28.271 | INFO | "huge_shared" : " 27 GB", | 20260506 00:25:28.271 | INFO | "malloc" : " 14 GB", | 20260506 00:25:28.271 | INFO | "pinned" : " 29 GB" | 20260506 00:25:28.271 | INFO | } | 20260506 00:25:28.271 | INFO | Perf report for the last 5.13489 seconds: | 20260506 00:25:28.271 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:25:28.271 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:28.271 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:28.271 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:28.271 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 00:25:28.271 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:28.271 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:28.271 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 00:25:28.271 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:28.271 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:28.271 | INFO | Uploaded 36GiB to GPU in 2.12964s (16.9043GiB/s) | 20260506 00:25:28.272 | INFO | Downloaded 34GiB to GPU in 2.02257s (16.8103GiB/s) | 20260506 00:25:28.272 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 00:25:28.272 | INFO | Total mem usage: 15.3 GiB | 20260506 00:25:28.272 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 00:25:28.272 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.16%) | 20260506 00:25:28.272 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 00:25:28.272 | INFO | High Water Breakdown: | 20260506 00:25:28.272 | INFO | 60.9% : 8.55 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 00:25:28.272 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 00:25:28.272 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 00:25:28.272 | INFO | ------------------------------------------ | 20260506 00:25:28.272 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:25:28.272 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 00:25:28.272 | INFO | Total mem usage: 26.5 GiB | 20260506 00:25:28.272 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 00:25:28.272 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 00:25:28.272 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260506 00:25:28.272 | INFO | High Water Breakdown: | 20260506 00:25:28.272 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:25:28.272 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:25:28.272 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:25:28.272 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:25:28.272 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:25:28.272 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:25:28.272 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 00:25:28.272 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 00:25:28.272 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:25:28.272 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 00:25:28.272 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 00:25:28.272 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 00:25:28.272 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 00:25:28.272 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 00:25:28.272 | INFO | ------------------------------------------ | 20260506 00:25:28.272 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:25:28.272 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 00:25:28.272 | INFO | Total mem usage: 25.6 GiB | 20260506 00:25:28.272 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 00:25:28.272 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 00:25:28.272 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 00:25:28.272 | INFO | High Water Breakdown: | 20260506 00:25:28.272 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 00:25:28.272 | INFO | ------------------------------------------ | 20260506 00:25:28.272 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:25:28.272 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 00:25:28.272 | INFO | Total mem usage: 15.7 GiB | 20260506 00:25:28.272 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 00:25:28.272 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 00:25:28.272 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 00:25:28.272 | INFO | High Water Breakdown: | 20260506 00:25:28.272 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 00:25:28.272 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 00:25:28.272 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 00:25:28.272 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 00:25:28.272 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 00:25:28.272 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 00:25:28.272 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 00:25:28.272 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 00:25:28.272 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 00:25:28.272 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:25:28.272 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 00:25:28.272 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:25:28.272 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 00:25:28.272 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 00:25:28.272 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 00:25:28.272 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 00:25:28.272 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:25:28.272 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 00:25:28.272 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:25:28.272 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:25:28.272 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:25:28.272 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 00:25:28.272 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 00:25:28.272 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:25:28.272 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 00:25:28.272 | INFO | ------------------------------------------ | 20260506 00:25:28.272 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:25:28.272 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 00:25:28.272 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:25:28.272 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:25:28.272 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.40e+01, 1.35e+01 / 7.01e-01 | 20260506 00:25:28.272 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.04e-01, 1.01e-02 / 3.63e-03 | 20260506 00:25:28.272 | INFO | PerfReport: 11.83%, Baseline -- 3.70e+00 / 4.49e+00, 5.68e-01 / 1.69e-01 | 20260506 00:25:28.272 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.53e+00, 1.90e-01 / 6.87e-02 | 20260506 00:25:28.272 | INFO | PerfReport: 16.01%, DME -- 4.01e+01 / 4.41e+01, 3.83e+00 / 1.07e+00 | 20260506 00:25:28.272 | INFO | PerfReport: 57.34%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.48e+00 / 1.10e+00 | 20260506 00:25:28.272 | INFO | PerfReport: 7.09%, PulseAccumulating -- 2.22e+00 / 2.76e+00, 4.23e-01 / 2.23e-01 | 20260506 00:25:28.272 | INFO | PerfReport: 3.08%, Metrics -- 9.64e-01 / 1.23e+00, 2.60e-01 / 9.17e-02 | 20260506 00:25:28.273 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.33e-01 / 3.88e-01, 5.64e-02 / 2.21e-02 | 20260506 00:25:28.273 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 00:25:28.273 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:25:28.273 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:25:28.273 | INFO | PerfReport: Full Execution -- 9.82e+01 / -nan, 2.44e+01 / 0.00e+00 | 20260506 00:25:28.273 | INFO | PerfReport: 22.30%, Upload -- 2.19e+01 / 0.00e+00, 7.48e+00 / 0.00e+00 | 20260506 00:25:28.273 | INFO | PerfReport: 21.86%, Download -- 2.15e+01 / 0.00e+00, 7.58e+00 / 0.00e+00 | 20260506 00:25:28.273 | INFO | PerfReport: 31.88%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 00:25:28.273 | INFO | PerfReport: 2.11%, LockUpload -- 2.07e+00 / 0.00e+00, 7.89e+00 / 0.00e+00 | 20260506 00:25:28.273 | INFO | PerfReport: 1.31%, LockDownload -- 1.28e+00 / 0.00e+00, 2.89e+00 / 0.00e+00 | 20260506 00:25:28.273 | INFO | PerfReport: 20.53%, LockCompute -- 2.02e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 00:25:28.273 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.78e-02 / 0.00e+00 | 20260506 00:25:28.399 | INFO | Starting a round of Baz IO | 20260506 00:25:31.543 | INFO | Populating RTMetrics report | 20260506 00:25:31.543 | INFO | RTMetrics report populated | 20260506 00:25:31.543 | INFO | Opening tmp RTMetrics json file | 20260506 00:25:31.544 | INFO | Serializing RTMetrics to json | 20260506 00:25:31.545 | INFO | Serialization done, swapping tmp file now... | 20260506 00:25:31.545 | INFO | RTMetrics done | 20260506 00:25:31.621 | INFO | Finished a round of Baz IO: Wrote 1.858 GiB in 3.221 seconds | 20260506 00:25:31.621 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.37% / 0.00% / 78.63% | 20260506 00:25:31.621 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0926971% / 3.97364e-06% / 8.34465e-05% | 20260506 00:25:31.621 | INFO | Baz IO overall: | 20260506 00:25:31.621 | INFO | BytesWritten : 4259029013420 | 20260506 00:25:31.621 | INFO | NumEvents : 2004293144712 | 20260506 00:25:33.412 | INFO | Analyzing chunk frames = [7495680,7496192) | 20260506 00:25:33.412 | INFO | MemInfo: { | 20260506 00:25:33.412 | INFO | "gpu" : " 17 GB", | 20260506 00:25:33.412 | INFO | "huge" : "", | 20260506 00:25:33.412 | INFO | "huge_shared" : " 27 GB", | 20260506 00:25:33.412 | INFO | "malloc" : " 14 GB", | 20260506 00:25:33.412 | INFO | "pinned" : " 29 GB" | 20260506 00:25:33.412 | INFO | } | 20260506 00:25:33.412 | INFO | Perf report for the last 5.1405 seconds: | 20260506 00:25:33.412 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:25:33.412 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:33.412 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:33.412 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:33.412 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260506 00:25:33.412 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:33.412 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260506 00:25:33.412 | INFO | PRE_HQ : 59.9%, 1.7, 54.0ms, 0 | 20260506 00:25:33.412 | INFO | BAZWRITER : 61.9%, 3.6, 113.1ms, 0 | 20260506 00:25:33.412 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:25:33.412 | INFO | Uploaded 36GiB to GPU in 2.1668s (16.6144GiB/s) | 20260506 00:25:33.412 | INFO | Downloaded 38GiB to GPU in 2.2437s (16.9363GiB/s) | 20260506 00:25:38.554 | INFO | Analyzing chunk frames = [7496192,7496704) | 20260506 00:25:38.554 | INFO | MemInfo: { | 20260506 00:25:38.554 | INFO | "gpu" : " 17 GB", | 20260506 00:25:38.554 | INFO | "huge" : "", | 20260506 00:25:38.554 | INFO | "huge_shared" : " 27 GB", | 20260506 00:25:38.554 | INFO | "malloc" : " 14 GB", | 20260506 00:25:38.554 | INFO | "pinned" : " 29 GB" | 20260506 00:25:38.554 | INFO | } | 20260506 00:25:38.554 | INFO | Perf report for the last 5.14204 seconds: | 20260506 00:25:38.554 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:25:38.554 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:38.554 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:38.554 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:38.554 | INFO | ANALYSIS : 61.4%, 3.0, 95.9ms, 0 | 20260506 00:25:38.554 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:38.554 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:38.554 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 00:25:38.554 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:38.554 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:38.554 | INFO | Uploaded 36GiB to GPU in 2.04561s (17.5987GiB/s) | 20260506 00:25:38.554 | INFO | Downloaded 34GiB to GPU in 2.0164s (16.8617GiB/s) | 20260506 00:25:43.700 | INFO | Analyzing chunk frames = [7496704,7497216) | 20260506 00:25:43.700 | INFO | MemInfo: { | 20260506 00:25:43.700 | INFO | "gpu" : " 17 GB", | 20260506 00:25:43.700 | INFO | "huge" : "", | 20260506 00:25:43.700 | INFO | "huge_shared" : " 27 GB", | 20260506 00:25:43.700 | INFO | "malloc" : " 14 GB", | 20260506 00:25:43.700 | INFO | "pinned" : " 29 GB" | 20260506 00:25:43.700 | INFO | } | 20260506 00:25:43.700 | INFO | Perf report for the last 5.14613 seconds: | 20260506 00:25:43.700 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:25:43.700 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:43.700 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:43.700 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:43.700 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 00:25:43.700 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:43.700 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:43.700 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:25:43.700 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:43.700 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:43.700 | INFO | Uploaded 36GiB to GPU in 2.07806s (17.3238GiB/s) | 20260506 00:25:43.700 | INFO | Downloaded 34GiB to GPU in 2.02356s (16.8021GiB/s) | 20260506 00:25:48.832 | INFO | Analyzing chunk frames = [7497216,7497728) | 20260506 00:25:48.832 | INFO | MemInfo: { | 20260506 00:25:48.832 | INFO | "gpu" : " 17 GB", | 20260506 00:25:48.832 | INFO | "huge" : "", | 20260506 00:25:48.832 | INFO | "huge_shared" : " 27 GB", | 20260506 00:25:48.832 | INFO | "malloc" : " 14 GB", | 20260506 00:25:48.832 | INFO | "pinned" : " 29 GB" | 20260506 00:25:48.832 | INFO | } | 20260506 00:25:48.832 | INFO | Perf report for the last 5.13243 seconds: | 20260506 00:25:48.832 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:25:48.832 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:48.832 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:48.832 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:48.832 | INFO | ANALYSIS : 61.5%, 3.0, 95.4ms, 0 | 20260506 00:25:48.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:48.833 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:48.833 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 00:25:48.833 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:48.833 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:48.833 | INFO | Uploaded 36GiB to GPU in 2.11876s (16.9911GiB/s) | 20260506 00:25:48.833 | INFO | Downloaded 34GiB to GPU in 2.03033s (16.746GiB/s) | 20260506 00:25:53.968 | INFO | Analyzing chunk frames = [7497728,7498240) | 20260506 00:25:53.968 | INFO | MemInfo: { | 20260506 00:25:53.968 | INFO | "gpu" : " 17 GB", | 20260506 00:25:53.968 | INFO | "huge" : "", | 20260506 00:25:53.969 | INFO | "huge_shared" : " 27 GB", | 20260506 00:25:53.969 | INFO | "malloc" : " 14 GB", | 20260506 00:25:53.969 | INFO | "pinned" : " 29 GB" | 20260506 00:25:53.969 | INFO | } | 20260506 00:25:53.969 | INFO | Perf report for the last 5.13617 seconds: | 20260506 00:25:53.969 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:25:53.969 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:53.969 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:53.969 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:53.969 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 00:25:53.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:53.969 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:53.969 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:25:53.969 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:53.969 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:53.969 | INFO | Uploaded 36GiB to GPU in 2.09456s (17.1874GiB/s) | 20260506 00:25:53.969 | INFO | Downloaded 34GiB to GPU in 2.04293s (16.6428GiB/s) | 20260506 00:25:59.112 | INFO | Analyzing chunk frames = [7498240,7498752) | 20260506 00:25:59.112 | INFO | MemInfo: { | 20260506 00:25:59.112 | INFO | "gpu" : " 17 GB", | 20260506 00:25:59.112 | INFO | "huge" : "", | 20260506 00:25:59.113 | INFO | "huge_shared" : " 27 GB", | 20260506 00:25:59.113 | INFO | "malloc" : " 14 GB", | 20260506 00:25:59.113 | INFO | "pinned" : " 29 GB" | 20260506 00:25:59.113 | INFO | } | 20260506 00:25:59.113 | INFO | Perf report for the last 5.14396 seconds: | 20260506 00:25:59.113 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:25:59.113 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:59.113 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:59.113 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:59.113 | INFO | ANALYSIS : 61.4%, 3.0, 95.9ms, 0 | 20260506 00:25:59.113 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:59.113 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:59.113 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:25:59.113 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:59.113 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:25:59.113 | INFO | Uploaded 36GiB to GPU in 2.08581s (17.2595GiB/s) | 20260506 00:25:59.113 | INFO | Downloaded 34GiB to GPU in 2.04108s (16.6579GiB/s) | 20260506 00:26:04.249 | INFO | Analyzing chunk frames = [7498752,7499264) | 20260506 00:26:04.249 | INFO | MemInfo: { | 20260506 00:26:04.249 | INFO | "gpu" : " 17 GB", | 20260506 00:26:04.249 | INFO | "huge" : "", | 20260506 00:26:04.249 | INFO | "huge_shared" : " 27 GB", | 20260506 00:26:04.249 | INFO | "malloc" : " 14 GB", | 20260506 00:26:04.249 | INFO | "pinned" : " 29 GB" | 20260506 00:26:04.249 | INFO | } | 20260506 00:26:04.249 | INFO | Perf report for the last 5.13629 seconds: | 20260506 00:26:04.249 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:26:04.249 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:04.249 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:04.249 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:04.249 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260506 00:26:04.249 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:04.249 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:04.249 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260506 00:26:04.249 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:04.249 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:04.249 | INFO | Uploaded 36GiB to GPU in 2.08335s (17.2798GiB/s) | 20260506 00:26:04.249 | INFO | Downloaded 34GiB to GPU in 2.07803s (16.3617GiB/s) | 20260506 00:26:09.392 | INFO | Analyzing chunk frames = [7499264,7499776) | 20260506 00:26:09.392 | INFO | MemInfo: { | 20260506 00:26:09.392 | INFO | "gpu" : " 17 GB", | 20260506 00:26:09.392 | INFO | "huge" : "", | 20260506 00:26:09.392 | INFO | "huge_shared" : " 27 GB", | 20260506 00:26:09.392 | INFO | "malloc" : " 14 GB", | 20260506 00:26:09.392 | INFO | "pinned" : " 29 GB" | 20260506 00:26:09.392 | INFO | } | 20260506 00:26:09.392 | INFO | Perf report for the last 5.1428 seconds: | 20260506 00:26:09.392 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:26:09.392 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:09.392 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:09.392 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:09.392 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260506 00:26:09.392 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:09.392 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:09.392 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 00:26:09.392 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:09.392 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:09.392 | INFO | Uploaded 36GiB to GPU in 2.12723s (16.9234GiB/s) | 20260506 00:26:09.392 | INFO | Downloaded 34GiB to GPU in 2.02005s (16.8313GiB/s) | 20260506 00:26:09.513 | INFO | Starting a round of Baz IO | 20260506 00:26:12.662 | INFO | Populating RTMetrics report | 20260506 00:26:12.663 | INFO | RTMetrics report populated | 20260506 00:26:12.663 | INFO | Opening tmp RTMetrics json file | 20260506 00:26:12.663 | INFO | Serializing RTMetrics to json | 20260506 00:26:12.664 | INFO | Serialization done, swapping tmp file now... | 20260506 00:26:12.665 | INFO | RTMetrics done | 20260506 00:26:12.735 | INFO | Finished a round of Baz IO: Wrote 1.858 GiB in 3.222 seconds | 20260506 00:26:12.735 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.37% / 0.00% / 78.63% | 20260506 00:26:12.735 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0925779% / 0% / 7.54992e-05% | 20260506 00:26:12.735 | INFO | Baz IO overall: | 20260506 00:26:12.735 | INFO | BytesWritten : 4261023978861 | 20260506 00:26:12.735 | INFO | NumEvents : 2005205321698 | 20260506 00:26:14.533 | INFO | Analyzing chunk frames = [7499776,7500288) | 20260506 00:26:14.533 | INFO | MemInfo: { | 20260506 00:26:14.533 | INFO | "gpu" : " 17 GB", | 20260506 00:26:14.533 | INFO | "huge" : "", | 20260506 00:26:14.533 | INFO | "huge_shared" : " 27 GB", | 20260506 00:26:14.533 | INFO | "malloc" : " 14 GB", | 20260506 00:26:14.533 | INFO | "pinned" : " 29 GB" | 20260506 00:26:14.533 | INFO | } | 20260506 00:26:14.533 | INFO | Perf report for the last 5.14163 seconds: | 20260506 00:26:14.533 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:26:14.533 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:14.533 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:14.533 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:14.533 | INFO | ANALYSIS : 62.5%, 3.0, 97.7ms, 0 | 20260506 00:26:14.533 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:14.533 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260506 00:26:14.533 | INFO | PRE_HQ : 59.9%, 1.7, 54.3ms, 0 | 20260506 00:26:14.533 | INFO | BAZWRITER : 61.8%, 3.5, 111.1ms, 0 | 20260506 00:26:14.534 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:26:14.534 | INFO | Uploaded 36GiB to GPU in 2.1679s (16.6059GiB/s) | 20260506 00:26:14.534 | INFO | Downloaded 38GiB to GPU in 2.24024s (16.9625GiB/s) | 20260506 00:26:19.663 | INFO | Analyzing chunk frames = [7500288,7500800) | 20260506 00:26:19.663 | INFO | MemInfo: { | 20260506 00:26:19.663 | INFO | "gpu" : " 17 GB", | 20260506 00:26:19.663 | INFO | "huge" : "", | 20260506 00:26:19.663 | INFO | "huge_shared" : " 27 GB", | 20260506 00:26:19.663 | INFO | "malloc" : " 14 GB", | 20260506 00:26:19.663 | INFO | "pinned" : " 29 GB" | 20260506 00:26:19.663 | INFO | } | 20260506 00:26:19.663 | INFO | Perf report for the last 5.12977 seconds: | 20260506 00:26:19.663 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:26:19.663 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:19.663 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:19.663 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:19.663 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 00:26:19.663 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:19.663 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:19.663 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 00:26:19.663 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:19.663 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:19.663 | INFO | Uploaded 36GiB to GPU in 2.0908s (17.2183GiB/s) | 20260506 00:26:19.663 | INFO | Downloaded 34GiB to GPU in 2.03878s (16.6766GiB/s) | 20260506 00:26:24.806 | INFO | Analyzing chunk frames = [7500800,7501312) | 20260506 00:26:24.806 | INFO | MemInfo: { | 20260506 00:26:24.806 | INFO | "gpu" : " 17 GB", | 20260506 00:26:24.806 | INFO | "huge" : "", | 20260506 00:26:24.806 | INFO | "huge_shared" : " 27 GB", | 20260506 00:26:24.806 | INFO | "malloc" : " 14 GB", | 20260506 00:26:24.806 | INFO | "pinned" : " 29 GB" | 20260506 00:26:24.806 | INFO | } | 20260506 00:26:24.806 | INFO | Perf report for the last 5.14297 seconds: | 20260506 00:26:24.806 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:26:24.806 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:24.806 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:24.806 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:24.806 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 00:26:24.806 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:24.806 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:24.806 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:26:24.806 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:24.806 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:24.806 | INFO | Uploaded 36GiB to GPU in 2.09566s (17.1784GiB/s) | 20260506 00:26:24.806 | INFO | Downloaded 34GiB to GPU in 2.03253s (16.728GiB/s) | 20260506 00:26:29.947 | INFO | Analyzing chunk frames = [7501312,7501824) | 20260506 00:26:29.948 | INFO | MemInfo: { | 20260506 00:26:29.948 | INFO | "gpu" : " 17 GB", | 20260506 00:26:29.948 | INFO | "huge" : "", | 20260506 00:26:29.948 | INFO | "huge_shared" : " 27 GB", | 20260506 00:26:29.948 | INFO | "malloc" : " 14 GB", | 20260506 00:26:29.948 | INFO | "pinned" : " 29 GB" | 20260506 00:26:29.948 | INFO | } | 20260506 00:26:29.948 | INFO | Perf report for the last 5.14152 seconds: | 20260506 00:26:29.948 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:26:29.948 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:29.948 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:29.948 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:29.948 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260506 00:26:29.948 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:29.948 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:29.948 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 00:26:29.948 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:29.948 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:29.948 | INFO | Uploaded 36GiB to GPU in 2.13378s (16.8715GiB/s) | 20260506 00:26:29.948 | INFO | Downloaded 34GiB to GPU in 2.01737s (16.8536GiB/s) | 20260506 00:26:35.084 | INFO | Analyzing chunk frames = [7501824,7502336) | 20260506 00:26:35.084 | INFO | MemInfo: { | 20260506 00:26:35.084 | INFO | "gpu" : " 17 GB", | 20260506 00:26:35.084 | INFO | "huge" : "", | 20260506 00:26:35.084 | INFO | "huge_shared" : " 27 GB", | 20260506 00:26:35.084 | INFO | "malloc" : " 14 GB", | 20260506 00:26:35.084 | INFO | "pinned" : " 29 GB" | 20260506 00:26:35.084 | INFO | } | 20260506 00:26:35.084 | INFO | Perf report for the last 5.13612 seconds: | 20260506 00:26:35.084 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:26:35.084 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:35.084 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:35.084 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:35.084 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 00:26:35.084 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:35.084 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:35.084 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 00:26:35.084 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:35.084 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:35.084 | INFO | Uploaded 36GiB to GPU in 2.08649s (17.2538GiB/s) | 20260506 00:26:35.084 | INFO | Downloaded 34GiB to GPU in 2.04636s (16.6149GiB/s) | 20260506 00:26:40.219 | INFO | Analyzing chunk frames = [7502336,7502848) | 20260506 00:26:40.219 | INFO | MemInfo: { | 20260506 00:26:40.219 | INFO | "gpu" : " 17 GB", | 20260506 00:26:40.219 | INFO | "huge" : "", | 20260506 00:26:40.219 | INFO | "huge_shared" : " 27 GB", | 20260506 00:26:40.219 | INFO | "malloc" : " 14 GB", | 20260506 00:26:40.219 | INFO | "pinned" : " 29 GB" | 20260506 00:26:40.219 | INFO | } | 20260506 00:26:40.219 | INFO | Perf report for the last 5.13567 seconds: | 20260506 00:26:40.219 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:26:40.219 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:40.219 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:40.220 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:40.220 | INFO | ANALYSIS : 61.4%, 3.0, 95.9ms, 0 | 20260506 00:26:40.220 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:40.220 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:40.220 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:26:40.220 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:40.220 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:40.220 | INFO | Uploaded 36GiB to GPU in 2.0779s (17.3252GiB/s) | 20260506 00:26:40.220 | INFO | Downloaded 34GiB to GPU in 2.03887s (16.6759GiB/s) | 20260506 00:26:45.365 | INFO | Analyzing chunk frames = [7502848,7503360) | 20260506 00:26:45.365 | INFO | MemInfo: { | 20260506 00:26:45.365 | INFO | "gpu" : " 17 GB", | 20260506 00:26:45.365 | INFO | "huge" : "", | 20260506 00:26:45.365 | INFO | "huge_shared" : " 27 GB", | 20260506 00:26:45.365 | INFO | "malloc" : " 14 GB", | 20260506 00:26:45.365 | INFO | "pinned" : " 29 GB" | 20260506 00:26:45.365 | INFO | } | 20260506 00:26:45.365 | INFO | Perf report for the last 5.14588 seconds: | 20260506 00:26:45.365 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:26:45.365 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:45.365 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:45.365 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:45.365 | INFO | ANALYSIS : 62.7%, 3.0, 98.0ms, 0 | 20260506 00:26:45.365 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:45.365 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:45.365 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 00:26:45.365 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:45.365 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:45.366 | INFO | Uploaded 36GiB to GPU in 2.16694s (16.6133GiB/s) | 20260506 00:26:45.366 | INFO | Downloaded 34GiB to GPU in 2.13162s (15.9503GiB/s) | 20260506 00:26:50.512 | INFO | Analyzing chunk frames = [7503360,7503872) | 20260506 00:26:50.512 | INFO | MemInfo: { | 20260506 00:26:50.512 | INFO | "gpu" : " 17 GB", | 20260506 00:26:50.512 | INFO | "huge" : "", | 20260506 00:26:50.512 | INFO | "huge_shared" : " 27 GB", | 20260506 00:26:50.512 | INFO | "malloc" : " 14 GB", | 20260506 00:26:50.512 | INFO | "pinned" : " 29 GB" | 20260506 00:26:50.512 | INFO | } | 20260506 00:26:50.512 | INFO | Perf report for the last 5.14696 seconds: | 20260506 00:26:50.512 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:26:50.512 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:50.512 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:50.512 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:50.512 | INFO | ANALYSIS : 61.8%, 3.0, 96.6ms, 0 | 20260506 00:26:50.512 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:50.512 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:50.512 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:26:50.512 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:50.512 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:50.513 | INFO | Uploaded 36GiB to GPU in 2.11235s (17.0426GiB/s) | 20260506 00:26:50.513 | INFO | Downloaded 34GiB to GPU in 2.00739s (16.9374GiB/s) | 20260506 00:26:50.513 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 00:26:50.513 | INFO | Total mem usage: 15.3 GiB | 20260506 00:26:50.513 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 00:26:50.513 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.15%) | 20260506 00:26:50.513 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 00:26:50.513 | INFO | High Water Breakdown: | 20260506 00:26:50.513 | INFO | 60.9% : 8.55 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 00:26:50.513 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 00:26:50.513 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 00:26:50.513 | INFO | ------------------------------------------ | 20260506 00:26:50.513 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:26:50.513 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 00:26:50.513 | INFO | Total mem usage: 26.5 GiB | 20260506 00:26:50.513 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 00:26:50.513 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 00:26:50.513 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 00:26:50.513 | INFO | High Water Breakdown: | 20260506 00:26:50.513 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:26:50.513 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:26:50.513 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:26:50.513 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:26:50.513 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:26:50.513 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:26:50.513 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 00:26:50.513 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 00:26:50.513 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:26:50.513 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 00:26:50.513 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 00:26:50.513 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 00:26:50.513 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 00:26:50.513 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 00:26:50.513 | INFO | ------------------------------------------ | 20260506 00:26:50.513 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:26:50.513 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 00:26:50.513 | INFO | Total mem usage: 25.6 GiB | 20260506 00:26:50.513 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 00:26:50.513 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 00:26:50.513 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 00:26:50.513 | INFO | High Water Breakdown: | 20260506 00:26:50.513 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 00:26:50.513 | INFO | ------------------------------------------ | 20260506 00:26:50.513 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:26:50.513 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 00:26:50.513 | INFO | Total mem usage: 15.7 GiB | 20260506 00:26:50.513 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 00:26:50.513 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 00:26:50.513 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 00:26:50.514 | INFO | High Water Breakdown: | 20260506 00:26:50.514 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 00:26:50.514 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 00:26:50.514 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 00:26:50.514 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 00:26:50.514 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 00:26:50.514 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 00:26:50.514 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 00:26:50.514 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 00:26:50.514 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 00:26:50.514 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:26:50.514 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 00:26:50.514 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:26:50.514 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 00:26:50.514 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 00:26:50.514 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 00:26:50.514 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 00:26:50.514 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:26:50.514 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 00:26:50.514 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:26:50.514 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:26:50.514 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:26:50.514 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 00:26:50.514 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 00:26:50.514 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:26:50.514 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 00:26:50.514 | INFO | ------------------------------------------ | 20260506 00:26:50.514 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:26:50.514 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 00:26:50.514 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:26:50.514 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:26:50.514 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.42e+01, 1.32e+01 / 1.37e+00 | 20260506 00:26:50.514 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.04e-01, 1.04e-02 / 3.44e-03 | 20260506 00:26:50.514 | INFO | PerfReport: 11.83%, Baseline -- 3.71e+00 / 4.52e+00, 5.74e-01 / 1.69e-01 | 20260506 00:26:50.514 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.52e+00, 1.91e-01 / 8.55e-02 | 20260506 00:26:50.514 | INFO | PerfReport: 15.97%, DME -- 4.01e+01 / 4.40e+01, 3.88e+00 / 1.38e+00 | 20260506 00:26:50.514 | INFO | PerfReport: 57.45%, FrameLabeling -- 1.80e+01 / 1.97e+01, 1.59e+00 / 1.30e+00 | 20260506 00:26:50.514 | INFO | PerfReport: 7.04%, PulseAccumulating -- 2.21e+00 / 2.71e+00, 4.18e-01 / 2.56e-01 | 20260506 00:26:50.514 | INFO | PerfReport: 3.07%, Metrics -- 9.62e-01 / 1.24e+00, 2.60e-01 / 1.20e-01 | 20260506 00:26:50.514 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.34e-01 / 3.84e-01, 5.71e-02 / 3.22e-02 | 20260506 00:26:50.514 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 00:26:50.514 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:26:50.514 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:26:50.514 | INFO | PerfReport: Full Execution -- 9.83e+01 / -nan, 2.48e+01 / 0.00e+00 | 20260506 00:26:50.514 | INFO | PerfReport: 22.34%, Upload -- 2.20e+01 / 0.00e+00, 7.52e+00 / 0.00e+00 | 20260506 00:26:50.514 | INFO | PerfReport: 21.89%, Download -- 2.15e+01 / 0.00e+00, 7.64e+00 / 0.00e+00 | 20260506 00:26:50.514 | INFO | PerfReport: 31.91%, Compute -- 3.14e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 00:26:50.514 | INFO | PerfReport: 2.10%, LockUpload -- 2.06e+00 / 0.00e+00, 7.86e+00 / 0.00e+00 | 20260506 00:26:50.514 | INFO | PerfReport: 1.31%, LockDownload -- 1.28e+00 / 0.00e+00, 2.87e+00 / 0.00e+00 | 20260506 00:26:50.514 | INFO | PerfReport: 20.45%, LockCompute -- 2.01e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 00:26:50.514 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.61e-02 / 0.00e+00 | 20260506 00:26:50.639 | INFO | Starting a round of Baz IO | 20260506 00:26:53.776 | INFO | Populating RTMetrics report | 20260506 00:26:53.776 | INFO | RTMetrics report populated | 20260506 00:26:53.776 | INFO | Opening tmp RTMetrics json file | 20260506 00:26:53.779 | INFO | Serializing RTMetrics to json | 20260506 00:26:53.783 | INFO | Serialization done, swapping tmp file now... | 20260506 00:26:53.783 | INFO | RTMetrics done | 20260506 00:26:53.859 | INFO | Finished a round of Baz IO: Wrote 1.858 GiB in 3.219 seconds | 20260506 00:26:53.859 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.37% / 0.00% / 78.63% | 20260506 00:26:53.859 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0929356% / 0% / 7.94729e-05% | 20260506 00:26:53.859 | INFO | Baz IO overall: | 20260506 00:26:53.859 | INFO | BytesWritten : 4263018907795 | 20260506 00:26:53.859 | INFO | NumEvents : 2006117536592 | 20260506 00:26:55.645 | INFO | Analyzing chunk frames = [7503872,7504384) | 20260506 00:26:55.645 | INFO | MemInfo: { | 20260506 00:26:55.645 | INFO | "gpu" : " 17 GB", | 20260506 00:26:55.645 | INFO | "huge" : "", | 20260506 00:26:55.645 | INFO | "huge_shared" : " 27 GB", | 20260506 00:26:55.645 | INFO | "malloc" : " 14 GB", | 20260506 00:26:55.645 | INFO | "pinned" : " 29 GB" | 20260506 00:26:55.645 | INFO | } | 20260506 00:26:55.645 | INFO | Perf report for the last 5.13273 seconds: | 20260506 00:26:55.645 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:26:55.645 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:55.645 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:55.645 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:55.645 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260506 00:26:55.645 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:26:55.645 | INFO | RT_METRICS : 43.4%, 1.0, 22.7ms, 0 | 20260506 00:26:55.645 | INFO | PRE_HQ : 59.7%, 1.7, 54.2ms, 0 | 20260506 00:26:55.645 | INFO | BAZWRITER : 62.0%, 3.5, 111.8ms, 0 | 20260506 00:26:55.645 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:26:55.645 | INFO | Uploaded 36GiB to GPU in 2.1764s (16.5411GiB/s) | 20260506 00:26:55.646 | INFO | Downloaded 38GiB to GPU in 2.23739s (16.9841GiB/s) | 20260506 00:27:00.785 | INFO | Analyzing chunk frames = [7504384,7504896) | 20260506 00:27:00.785 | INFO | MemInfo: { | 20260506 00:27:00.785 | INFO | "gpu" : " 17 GB", | 20260506 00:27:00.785 | INFO | "huge" : "", | 20260506 00:27:00.785 | INFO | "huge_shared" : " 27 GB", | 20260506 00:27:00.785 | INFO | "malloc" : " 14 GB", | 20260506 00:27:00.785 | INFO | "pinned" : " 29 GB" | 20260506 00:27:00.786 | INFO | } | 20260506 00:27:00.786 | INFO | Perf report for the last 5.14046 seconds: | 20260506 00:27:00.786 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:27:00.786 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:00.786 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:00.786 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:00.786 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 00:27:00.786 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:00.786 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:00.786 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:27:00.786 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:00.786 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:00.786 | INFO | Uploaded 36GiB to GPU in 1.96933s (18.2803GiB/s) | 20260506 00:27:00.786 | INFO | Downloaded 34GiB to GPU in 1.99493s (17.0432GiB/s) | 20260506 00:27:05.919 | INFO | Analyzing chunk frames = [7504896,7505408) | 20260506 00:27:05.919 | INFO | MemInfo: { | 20260506 00:27:05.919 | INFO | "gpu" : " 17 GB", | 20260506 00:27:05.919 | INFO | "huge" : "", | 20260506 00:27:05.919 | INFO | "huge_shared" : " 27 GB", | 20260506 00:27:05.919 | INFO | "malloc" : " 14 GB", | 20260506 00:27:05.919 | INFO | "pinned" : " 29 GB" | 20260506 00:27:05.919 | INFO | } | 20260506 00:27:05.919 | INFO | Perf report for the last 5.13388 seconds: | 20260506 00:27:05.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:27:05.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:05.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:05.920 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:05.920 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 00:27:05.920 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:05.920 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:05.920 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 00:27:05.920 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:05.920 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:05.920 | INFO | Uploaded 36GiB to GPU in 2.09552s (17.1795GiB/s) | 20260506 00:27:05.920 | INFO | Downloaded 34GiB to GPU in 2.02932s (16.7544GiB/s) | 20260506 00:27:11.064 | INFO | Analyzing chunk frames = [7505408,7505920) | 20260506 00:27:11.064 | INFO | MemInfo: { | 20260506 00:27:11.064 | INFO | "gpu" : " 17 GB", | 20260506 00:27:11.064 | INFO | "huge" : "", | 20260506 00:27:11.064 | INFO | "huge_shared" : " 27 GB", | 20260506 00:27:11.064 | INFO | "malloc" : " 14 GB", | 20260506 00:27:11.065 | INFO | "pinned" : " 29 GB" | 20260506 00:27:11.065 | INFO | } | 20260506 00:27:11.065 | INFO | Perf report for the last 5.14514 seconds: | 20260506 00:27:11.065 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:27:11.065 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:11.065 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:11.065 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:11.065 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 00:27:11.065 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:11.065 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:11.065 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 00:27:11.065 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:11.065 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:11.065 | INFO | Uploaded 36GiB to GPU in 2.13472s (16.864GiB/s) | 20260506 00:27:11.065 | INFO | Downloaded 34GiB to GPU in 2.01792s (16.8491GiB/s) | 20260506 00:27:16.204 | INFO | Analyzing chunk frames = [7505920,7506432) | 20260506 00:27:16.204 | INFO | MemInfo: { | 20260506 00:27:16.204 | INFO | "gpu" : " 17 GB", | 20260506 00:27:16.204 | INFO | "huge" : "", | 20260506 00:27:16.204 | INFO | "huge_shared" : " 27 GB", | 20260506 00:27:16.204 | INFO | "malloc" : " 14 GB", | 20260506 00:27:16.204 | INFO | "pinned" : " 29 GB" | 20260506 00:27:16.204 | INFO | } | 20260506 00:27:16.204 | INFO | Perf report for the last 5.13965 seconds: | 20260506 00:27:16.204 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:27:16.204 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:16.204 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:16.204 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:16.204 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 00:27:16.204 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:16.204 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:16.204 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 00:27:16.204 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:16.204 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:16.205 | INFO | Uploaded 36GiB to GPU in 2.08606s (17.2574GiB/s) | 20260506 00:27:16.205 | INFO | Downloaded 34GiB to GPU in 2.04451s (16.6299GiB/s) | 20260506 00:27:21.344 | INFO | Analyzing chunk frames = [7506432,7506944) | 20260506 00:27:21.344 | INFO | MemInfo: { | 20260506 00:27:21.344 | INFO | "gpu" : " 17 GB", | 20260506 00:27:21.344 | INFO | "huge" : "", | 20260506 00:27:21.344 | INFO | "huge_shared" : " 27 GB", | 20260506 00:27:21.344 | INFO | "malloc" : " 14 GB", | 20260506 00:27:21.344 | INFO | "pinned" : " 29 GB" | 20260506 00:27:21.344 | INFO | } | 20260506 00:27:21.344 | INFO | Perf report for the last 5.13977 seconds: | 20260506 00:27:21.344 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:27:21.344 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:21.344 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:21.344 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:21.344 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260506 00:27:21.344 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:21.344 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:21.344 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 00:27:21.344 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:21.344 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:21.344 | INFO | Uploaded 36GiB to GPU in 2.06869s (17.4023GiB/s) | 20260506 00:27:21.344 | INFO | Downloaded 34GiB to GPU in 2.03712s (16.6903GiB/s) | 20260506 00:27:26.482 | INFO | Analyzing chunk frames = [7506944,7507456) | 20260506 00:27:26.482 | INFO | MemInfo: { | 20260506 00:27:26.482 | INFO | "gpu" : " 17 GB", | 20260506 00:27:26.482 | INFO | "huge" : "", | 20260506 00:27:26.482 | INFO | "huge_shared" : " 27 GB", | 20260506 00:27:26.482 | INFO | "malloc" : " 14 GB", | 20260506 00:27:26.482 | INFO | "pinned" : " 29 GB" | 20260506 00:27:26.482 | INFO | } | 20260506 00:27:26.482 | INFO | Perf report for the last 5.13828 seconds: | 20260506 00:27:26.482 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:27:26.482 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:26.482 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:26.482 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:26.482 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260506 00:27:26.482 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:26.482 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:26.482 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 00:27:26.482 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:26.482 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:26.483 | INFO | Uploaded 36GiB to GPU in 2.11241s (17.0421GiB/s) | 20260506 00:27:26.483 | INFO | Downloaded 34GiB to GPU in 2.11095s (16.1065GiB/s) | 20260506 00:27:31.618 | INFO | Analyzing chunk frames = [7507456,7507968) | 20260506 00:27:31.618 | INFO | MemInfo: { | 20260506 00:27:31.618 | INFO | "gpu" : " 17 GB", | 20260506 00:27:31.618 | INFO | "huge" : "", | 20260506 00:27:31.618 | INFO | "huge_shared" : " 27 GB", | 20260506 00:27:31.618 | INFO | "malloc" : " 14 GB", | 20260506 00:27:31.618 | INFO | "pinned" : " 29 GB" | 20260506 00:27:31.618 | INFO | } | 20260506 00:27:31.618 | INFO | Perf report for the last 5.13575 seconds: | 20260506 00:27:31.618 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:27:31.618 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:31.618 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:31.618 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:31.618 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260506 00:27:31.618 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:31.618 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:31.618 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 00:27:31.618 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:31.618 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:31.618 | INFO | Uploaded 36GiB to GPU in 2.11293s (17.038GiB/s) | 20260506 00:27:31.618 | INFO | Downloaded 34GiB to GPU in 2.00442s (16.9625GiB/s) | 20260506 00:27:31.737 | INFO | Starting a round of Baz IO | 20260506 00:27:34.885 | INFO | Populating RTMetrics report | 20260506 00:27:34.885 | INFO | RTMetrics report populated | 20260506 00:27:34.885 | INFO | Opening tmp RTMetrics json file | 20260506 00:27:34.885 | INFO | Serializing RTMetrics to json | 20260506 00:27:34.888 | INFO | Serialization done, swapping tmp file now... | 20260506 00:27:34.888 | INFO | RTMetrics done | 20260506 00:27:34.962 | INFO | Finished a round of Baz IO: Wrote 1.858 GiB in 3.225 seconds | 20260506 00:27:34.962 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.37% / 0.00% / 78.63% | 20260506 00:27:34.962 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0926137% / 0% / 6.75519e-05% | 20260506 00:27:34.962 | INFO | Baz IO overall: | 20260506 00:27:34.962 | INFO | BytesWritten : 4265013760213 | 20260506 00:27:34.962 | INFO | NumEvents : 2007029735423 | 20260506 00:27:36.760 | INFO | Analyzing chunk frames = [7507968,7508480) | 20260506 00:27:36.760 | INFO | MemInfo: { | 20260506 00:27:36.760 | INFO | "gpu" : " 17 GB", | 20260506 00:27:36.760 | INFO | "huge" : "", | 20260506 00:27:36.760 | INFO | "huge_shared" : " 27 GB", | 20260506 00:27:36.760 | INFO | "malloc" : " 14 GB", | 20260506 00:27:36.760 | INFO | "pinned" : " 29 GB" | 20260506 00:27:36.760 | INFO | } | 20260506 00:27:36.760 | INFO | Perf report for the last 5.14178 seconds: | 20260506 00:27:36.760 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:27:36.760 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:36.760 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:36.760 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:36.760 | INFO | ANALYSIS : 62.5%, 3.0, 97.7ms, 0 | 20260506 00:27:36.760 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:36.760 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260506 00:27:36.760 | INFO | PRE_HQ : 59.3%, 1.7, 53.8ms, 0 | 20260506 00:27:36.760 | INFO | BAZWRITER : 61.8%, 3.5, 111.0ms, 0 | 20260506 00:27:36.760 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:27:36.760 | INFO | Uploaded 36GiB to GPU in 2.16371s (16.6381GiB/s) | 20260506 00:27:36.760 | INFO | Downloaded 38GiB to GPU in 2.22732s (17.0609GiB/s) | 20260506 00:27:41.898 | INFO | Analyzing chunk frames = [7508480,7508992) | 20260506 00:27:41.898 | INFO | MemInfo: { | 20260506 00:27:41.898 | INFO | "gpu" : " 17 GB", | 20260506 00:27:41.898 | INFO | "huge" : "", | 20260506 00:27:41.898 | INFO | "huge_shared" : " 27 GB", | 20260506 00:27:41.898 | INFO | "malloc" : " 14 GB", | 20260506 00:27:41.898 | INFO | "pinned" : " 29 GB" | 20260506 00:27:41.898 | INFO | } | 20260506 00:27:41.899 | INFO | Perf report for the last 5.1387 seconds: | 20260506 00:27:41.899 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:27:41.899 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:41.899 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:41.899 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:41.899 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 00:27:41.899 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:41.899 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:41.899 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 00:27:41.899 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:41.899 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:41.899 | INFO | Uploaded 36GiB to GPU in 1.97923s (18.1889GiB/s) | 20260506 00:27:41.899 | INFO | Downloaded 34GiB to GPU in 1.99081s (17.0785GiB/s) | 20260506 00:27:47.031 | INFO | Analyzing chunk frames = [7508992,7509504) | 20260506 00:27:47.031 | INFO | MemInfo: { | 20260506 00:27:47.031 | INFO | "gpu" : " 17 GB", | 20260506 00:27:47.031 | INFO | "huge" : "", | 20260506 00:27:47.031 | INFO | "huge_shared" : " 27 GB", | 20260506 00:27:47.031 | INFO | "malloc" : " 14 GB", | 20260506 00:27:47.031 | INFO | "pinned" : " 29 GB" | 20260506 00:27:47.031 | INFO | } | 20260506 00:27:47.031 | INFO | Perf report for the last 5.13279 seconds: | 20260506 00:27:47.031 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:27:47.031 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:47.031 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:47.031 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:47.031 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 00:27:47.031 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:47.031 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:47.031 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 00:27:47.031 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:47.031 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:47.032 | INFO | Uploaded 36GiB to GPU in 2.07843s (17.3208GiB/s) | 20260506 00:27:47.032 | INFO | Downloaded 34GiB to GPU in 2.01778s (16.8502GiB/s) | 20260506 00:27:52.173 | INFO | Analyzing chunk frames = [7509504,7510016) | 20260506 00:27:52.173 | INFO | MemInfo: { | 20260506 00:27:52.173 | INFO | "gpu" : " 17 GB", | 20260506 00:27:52.173 | INFO | "huge" : "", | 20260506 00:27:52.173 | INFO | "huge_shared" : " 27 GB", | 20260506 00:27:52.173 | INFO | "malloc" : " 14 GB", | 20260506 00:27:52.173 | INFO | "pinned" : " 29 GB" | 20260506 00:27:52.173 | INFO | } | 20260506 00:27:52.173 | INFO | Perf report for the last 5.1414 seconds: | 20260506 00:27:52.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:27:52.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:52.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:52.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:52.173 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260506 00:27:52.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:52.173 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:52.173 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 00:27:52.173 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:52.173 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:52.173 | INFO | Uploaded 36GiB to GPU in 2.13649s (16.85GiB/s) | 20260506 00:27:52.174 | INFO | Downloaded 34GiB to GPU in 2.01442s (16.8783GiB/s) | 20260506 00:27:57.318 | INFO | Analyzing chunk frames = [7510016,7510528) | 20260506 00:27:57.318 | INFO | MemInfo: { | 20260506 00:27:57.318 | INFO | "gpu" : " 17 GB", | 20260506 00:27:57.318 | INFO | "huge" : "", | 20260506 00:27:57.318 | INFO | "huge_shared" : " 27 GB", | 20260506 00:27:57.318 | INFO | "malloc" : " 14 GB", | 20260506 00:27:57.318 | INFO | "pinned" : " 29 GB" | 20260506 00:27:57.318 | INFO | } | 20260506 00:27:57.319 | INFO | Perf report for the last 5.1457 seconds: | 20260506 00:27:57.319 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:27:57.319 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:57.319 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:57.319 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:57.319 | INFO | ANALYSIS : 61.1%, 3.0, 95.5ms, 0 | 20260506 00:27:57.319 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:57.319 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:57.319 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 00:27:57.319 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:57.319 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:27:57.319 | INFO | Uploaded 36GiB to GPU in 2.09849s (17.1552GiB/s) | 20260506 00:27:57.319 | INFO | Downloaded 34GiB to GPU in 2.04036s (16.6637GiB/s) | 20260506 00:28:02.461 | INFO | Analyzing chunk frames = [7510528,7511040) | 20260506 00:28:02.461 | INFO | MemInfo: { | 20260506 00:28:02.461 | INFO | "gpu" : " 17 GB", | 20260506 00:28:02.461 | INFO | "huge" : "", | 20260506 00:28:02.461 | INFO | "huge_shared" : " 27 GB", | 20260506 00:28:02.461 | INFO | "malloc" : " 14 GB", | 20260506 00:28:02.461 | INFO | "pinned" : " 29 GB" | 20260506 00:28:02.461 | INFO | } | 20260506 00:28:02.461 | INFO | Perf report for the last 5.14293 seconds: | 20260506 00:28:02.461 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:28:02.461 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:02.461 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:02.462 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:02.462 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260506 00:28:02.462 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:02.462 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:02.462 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:28:02.462 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:02.462 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:02.462 | INFO | Uploaded 36GiB to GPU in 2.07004s (17.3909GiB/s) | 20260506 00:28:02.462 | INFO | Downloaded 34GiB to GPU in 2.03946s (16.6711GiB/s) | 20260506 00:28:07.597 | INFO | Analyzing chunk frames = [7511040,7511552) | 20260506 00:28:07.597 | INFO | MemInfo: { | 20260506 00:28:07.597 | INFO | "gpu" : " 17 GB", | 20260506 00:28:07.597 | INFO | "huge" : "", | 20260506 00:28:07.597 | INFO | "huge_shared" : " 27 GB", | 20260506 00:28:07.597 | INFO | "malloc" : " 14 GB", | 20260506 00:28:07.597 | INFO | "pinned" : " 29 GB" | 20260506 00:28:07.597 | INFO | } | 20260506 00:28:07.597 | INFO | Perf report for the last 5.13552 seconds: | 20260506 00:28:07.597 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:28:07.597 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:07.597 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:07.597 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:07.597 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 00:28:07.597 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:07.597 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:07.597 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260506 00:28:07.597 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:07.597 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:07.597 | INFO | Uploaded 36GiB to GPU in 2.04946s (17.5656GiB/s) | 20260506 00:28:07.597 | INFO | Downloaded 34GiB to GPU in 2.0712s (16.4156GiB/s) | 20260506 00:28:12.735 | INFO | Analyzing chunk frames = [7511552,7512064) | 20260506 00:28:12.735 | INFO | MemInfo: { | 20260506 00:28:12.735 | INFO | "gpu" : " 17 GB", | 20260506 00:28:12.735 | INFO | "huge" : "", | 20260506 00:28:12.735 | INFO | "huge_shared" : " 27 GB", | 20260506 00:28:12.735 | INFO | "malloc" : " 14 GB", | 20260506 00:28:12.735 | INFO | "pinned" : " 29 GB" | 20260506 00:28:12.735 | INFO | } | 20260506 00:28:12.735 | INFO | Perf report for the last 5.13766 seconds: | 20260506 00:28:12.735 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:28:12.735 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:12.735 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:12.735 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:12.735 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 00:28:12.735 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:12.735 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:12.735 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:28:12.735 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:12.735 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:12.735 | INFO | Uploaded 36GiB to GPU in 2.12071s (16.9754GiB/s) | 20260506 00:28:12.735 | INFO | Downloaded 34GiB to GPU in 2.01221s (16.8969GiB/s) | 20260506 00:28:12.735 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 00:28:12.735 | INFO | Total mem usage: 15.3 GiB | 20260506 00:28:12.735 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 00:28:12.735 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.15%) | 20260506 00:28:12.735 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 00:28:12.735 | INFO | High Water Breakdown: | 20260506 00:28:12.735 | INFO | 60.9% : 8.55 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 00:28:12.735 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 00:28:12.735 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 00:28:12.735 | INFO | ------------------------------------------ | 20260506 00:28:12.735 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:28:12.735 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 00:28:12.735 | INFO | Total mem usage: 26.5 GiB | 20260506 00:28:12.735 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 00:28:12.735 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 00:28:12.735 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260506 00:28:12.735 | INFO | High Water Breakdown: | 20260506 00:28:12.735 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:28:12.735 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:28:12.735 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:28:12.735 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:28:12.735 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:28:12.735 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:28:12.735 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 00:28:12.735 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 00:28:12.735 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:28:12.735 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 00:28:12.735 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 00:28:12.735 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 00:28:12.735 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 00:28:12.735 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 00:28:12.735 | INFO | ------------------------------------------ | 20260506 00:28:12.735 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:28:12.735 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 00:28:12.735 | INFO | Total mem usage: 25.6 GiB | 20260506 00:28:12.735 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 00:28:12.735 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 00:28:12.735 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 00:28:12.735 | INFO | High Water Breakdown: | 20260506 00:28:12.735 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 00:28:12.735 | INFO | ------------------------------------------ | 20260506 00:28:12.735 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:28:12.735 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 00:28:12.735 | INFO | Total mem usage: 15.7 GiB | 20260506 00:28:12.735 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 00:28:12.735 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 00:28:12.735 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 00:28:12.735 | INFO | High Water Breakdown: | 20260506 00:28:12.735 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 00:28:12.735 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 00:28:12.735 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 00:28:12.735 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 00:28:12.735 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 00:28:12.736 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 00:28:12.736 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 00:28:12.736 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 00:28:12.736 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 00:28:12.736 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:28:12.736 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 00:28:12.736 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:28:12.736 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 00:28:12.736 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 00:28:12.736 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 00:28:12.736 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 00:28:12.736 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:28:12.736 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 00:28:12.736 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:28:12.736 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:28:12.736 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:28:12.736 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 00:28:12.736 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 00:28:12.736 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:28:12.736 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 00:28:12.736 | INFO | ------------------------------------------ | 20260506 00:28:12.736 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:28:12.736 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 00:28:12.736 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:28:12.736 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:28:12.736 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.39e+01, 1.34e+01 / 7.59e-01 | 20260506 00:28:12.736 | INFO | PerfReport: 0.29%, AggregateStats -- 9.25e-02 / 1.03e-01, 1.09e-02 / 3.46e-03 | 20260506 00:28:12.736 | INFO | PerfReport: 11.86%, Baseline -- 3.72e+00 / 4.51e+00, 5.80e-01 / 2.59e-01 | 20260506 00:28:12.736 | INFO | PerfReport: 3.29%, Histogram -- 1.03e+00 / 1.52e+00, 1.92e-01 / 9.93e-02 | 20260506 00:28:12.736 | INFO | PerfReport: 15.96%, DME -- 4.01e+01 / 4.39e+01, 3.75e+00 / 7.39e-01 | 20260506 00:28:12.736 | INFO | PerfReport: 57.40%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.56e+00 / 1.22e+00 | 20260506 00:28:12.736 | INFO | PerfReport: 7.06%, PulseAccumulating -- 2.21e+00 / 2.72e+00, 4.19e-01 / 2.80e-01 | 20260506 00:28:12.736 | INFO | PerfReport: 3.06%, Metrics -- 9.61e-01 / 1.23e+00, 2.58e-01 / 1.24e-01 | 20260506 00:28:12.736 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.33e-01 / 3.77e-01, 5.62e-02 / 2.64e-02 | 20260506 00:28:12.736 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 00:28:12.736 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:28:12.736 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:28:12.736 | INFO | PerfReport: Full Execution -- 9.82e+01 / -nan, 2.46e+01 / 0.00e+00 | 20260506 00:28:12.736 | INFO | PerfReport: 22.18%, Upload -- 2.18e+01 / 0.00e+00, 7.35e+00 / 0.00e+00 | 20260506 00:28:12.736 | INFO | PerfReport: 21.81%, Download -- 2.14e+01 / 0.00e+00, 7.51e+00 / 0.00e+00 | 20260506 00:28:12.736 | INFO | PerfReport: 31.95%, Compute -- 3.14e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 00:28:12.736 | INFO | PerfReport: 2.09%, LockUpload -- 2.06e+00 / 0.00e+00, 7.79e+00 / 0.00e+00 | 20260506 00:28:12.736 | INFO | PerfReport: 1.31%, LockDownload -- 1.29e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260506 00:28:12.736 | INFO | PerfReport: 20.63%, LockCompute -- 2.03e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 00:28:12.736 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.70e-02 / 0.00e+00 | 20260506 00:28:12.851 | INFO | Starting a round of Baz IO | 20260506 00:28:15.997 | INFO | Populating RTMetrics report | 20260506 00:28:15.998 | INFO | RTMetrics report populated | 20260506 00:28:15.998 | INFO | Opening tmp RTMetrics json file | 20260506 00:28:15.999 | INFO | Serializing RTMetrics to json | 20260506 00:28:16.002 | INFO | Serialization done, swapping tmp file now... | 20260506 00:28:16.002 | INFO | RTMetrics done | 20260506 00:28:16.073 | INFO | Finished a round of Baz IO: Wrote 1.856 GiB in 3.221 seconds | 20260506 00:28:16.073 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.37% / 0.00% / 78.63% | 20260506 00:28:16.073 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0920931% / 0% / 9.93411e-05% | 20260506 00:28:16.073 | INFO | Baz IO overall: | 20260506 00:28:16.073 | INFO | BytesWritten : 4267006666648 | 20260506 00:28:16.073 | INFO | NumEvents : 2007940651869 | 20260506 00:28:17.875 | INFO | Analyzing chunk frames = [7512064,7512576) | 20260506 00:28:17.875 | INFO | MemInfo: { | 20260506 00:28:17.875 | INFO | "gpu" : " 17 GB", | 20260506 00:28:17.875 | INFO | "huge" : "", | 20260506 00:28:17.875 | INFO | "huge_shared" : " 27 GB", | 20260506 00:28:17.875 | INFO | "malloc" : " 14 GB", | 20260506 00:28:17.875 | INFO | "pinned" : " 29 GB" | 20260506 00:28:17.875 | INFO | } | 20260506 00:28:17.875 | INFO | Perf report for the last 5.14029 seconds: | 20260506 00:28:17.875 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:28:17.875 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:17.875 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:17.875 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:17.875 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260506 00:28:17.875 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:17.875 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260506 00:28:17.875 | INFO | PRE_HQ : 59.8%, 1.7, 53.9ms, 0 | 20260506 00:28:17.875 | INFO | BAZWRITER : 61.8%, 3.5, 111.7ms, 0 | 20260506 00:28:17.875 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:28:17.875 | INFO | Uploaded 36GiB to GPU in 2.16465s (16.6309GiB/s) | 20260506 00:28:17.875 | INFO | Downloaded 38GiB to GPU in 2.23736s (16.9843GiB/s) | 20260506 00:28:23.014 | INFO | Analyzing chunk frames = [7512576,7513088) | 20260506 00:28:23.014 | INFO | MemInfo: { | 20260506 00:28:23.014 | INFO | "gpu" : " 17 GB", | 20260506 00:28:23.014 | INFO | "huge" : "", | 20260506 00:28:23.014 | INFO | "huge_shared" : " 27 GB", | 20260506 00:28:23.014 | INFO | "malloc" : " 14 GB", | 20260506 00:28:23.014 | INFO | "pinned" : " 29 GB" | 20260506 00:28:23.014 | INFO | } | 20260506 00:28:23.014 | INFO | Perf report for the last 5.13873 seconds: | 20260506 00:28:23.014 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:28:23.014 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:23.014 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:23.014 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:23.014 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 00:28:23.014 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:23.014 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:23.014 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 00:28:23.014 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:23.014 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:23.014 | INFO | Uploaded 36GiB to GPU in 2.12219s (16.9636GiB/s) | 20260506 00:28:23.014 | INFO | Downloaded 34GiB to GPU in 2.04254s (16.6459GiB/s) | 20260506 00:28:28.157 | INFO | Analyzing chunk frames = [7513088,7513600) | 20260506 00:28:28.157 | INFO | MemInfo: { | 20260506 00:28:28.157 | INFO | "gpu" : " 17 GB", | 20260506 00:28:28.157 | INFO | "huge" : "", | 20260506 00:28:28.157 | INFO | "huge_shared" : " 27 GB", | 20260506 00:28:28.157 | INFO | "malloc" : " 14 GB", | 20260506 00:28:28.157 | INFO | "pinned" : " 29 GB" | 20260506 00:28:28.157 | INFO | } | 20260506 00:28:28.157 | INFO | Perf report for the last 5.14339 seconds: | 20260506 00:28:28.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:28:28.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:28.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:28.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:28.157 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 00:28:28.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:28.157 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:28.157 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 00:28:28.157 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:28.157 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:28.158 | INFO | Uploaded 36GiB to GPU in 2.0722s (17.3729GiB/s) | 20260506 00:28:28.158 | INFO | Downloaded 34GiB to GPU in 2.01794s (16.8489GiB/s) | 20260506 00:28:33.290 | INFO | Analyzing chunk frames = [7513600,7514112) | 20260506 00:28:33.290 | INFO | MemInfo: { | 20260506 00:28:33.290 | INFO | "gpu" : " 17 GB", | 20260506 00:28:33.290 | INFO | "huge" : "", | 20260506 00:28:33.290 | INFO | "huge_shared" : " 27 GB", | 20260506 00:28:33.290 | INFO | "malloc" : " 14 GB", | 20260506 00:28:33.290 | INFO | "pinned" : " 29 GB" | 20260506 00:28:33.290 | INFO | } | 20260506 00:28:33.290 | INFO | Perf report for the last 5.13296 seconds: | 20260506 00:28:33.290 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:28:33.290 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:33.290 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:33.290 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:33.290 | INFO | ANALYSIS : 61.7%, 3.0, 95.7ms, 0 | 20260506 00:28:33.290 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:33.290 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:33.290 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 00:28:33.290 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:33.290 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:33.290 | INFO | Uploaded 36GiB to GPU in 2.1442s (16.7895GiB/s) | 20260506 00:28:33.290 | INFO | Downloaded 34GiB to GPU in 2.03139s (16.7373GiB/s) | 20260506 00:28:38.430 | INFO | Analyzing chunk frames = [7514112,7514624) | 20260506 00:28:38.430 | INFO | MemInfo: { | 20260506 00:28:38.430 | INFO | "gpu" : " 17 GB", | 20260506 00:28:38.430 | INFO | "huge" : "", | 20260506 00:28:38.430 | INFO | "huge_shared" : " 27 GB", | 20260506 00:28:38.430 | INFO | "malloc" : " 14 GB", | 20260506 00:28:38.430 | INFO | "pinned" : " 29 GB" | 20260506 00:28:38.430 | INFO | } | 20260506 00:28:38.430 | INFO | Perf report for the last 5.13962 seconds: | 20260506 00:28:38.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:28:38.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:38.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:38.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:38.430 | INFO | ANALYSIS : 61.1%, 3.0, 95.5ms, 0 | 20260506 00:28:38.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:38.430 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:38.430 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:28:38.430 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:38.430 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:38.430 | INFO | Uploaded 36GiB to GPU in 2.09555s (17.1793GiB/s) | 20260506 00:28:38.431 | INFO | Downloaded 34GiB to GPU in 2.04981s (16.5869GiB/s) | 20260506 00:28:43.573 | INFO | Analyzing chunk frames = [7514624,7515136) | 20260506 00:28:43.573 | INFO | MemInfo: { | 20260506 00:28:43.573 | INFO | "gpu" : " 17 GB", | 20260506 00:28:43.573 | INFO | "huge" : "", | 20260506 00:28:43.573 | INFO | "huge_shared" : " 27 GB", | 20260506 00:28:43.573 | INFO | "malloc" : " 14 GB", | 20260506 00:28:43.573 | INFO | "pinned" : " 29 GB" | 20260506 00:28:43.573 | INFO | } | 20260506 00:28:43.573 | INFO | Perf report for the last 5.14356 seconds: | 20260506 00:28:43.573 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:28:43.573 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:43.573 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:43.573 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:43.573 | INFO | ANALYSIS : 61.3%, 3.0, 95.8ms, 0 | 20260506 00:28:43.573 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:43.573 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:43.573 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 00:28:43.573 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:43.573 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:43.574 | INFO | Uploaded 36GiB to GPU in 2.07485s (17.3507GiB/s) | 20260506 00:28:43.574 | INFO | Downloaded 34GiB to GPU in 2.03926s (16.6727GiB/s) | 20260506 00:28:48.709 | INFO | Analyzing chunk frames = [7515136,7515648) | 20260506 00:28:48.709 | INFO | MemInfo: { | 20260506 00:28:48.709 | INFO | "gpu" : " 17 GB", | 20260506 00:28:48.709 | INFO | "huge" : "", | 20260506 00:28:48.709 | INFO | "huge_shared" : " 27 GB", | 20260506 00:28:48.709 | INFO | "malloc" : " 14 GB", | 20260506 00:28:48.709 | INFO | "pinned" : " 29 GB" | 20260506 00:28:48.710 | INFO | } | 20260506 00:28:48.710 | INFO | Perf report for the last 5.13624 seconds: | 20260506 00:28:48.710 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:28:48.710 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:48.710 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:48.710 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:48.710 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 00:28:48.710 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:48.710 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:48.710 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 00:28:48.710 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:48.710 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:48.710 | INFO | Uploaded 36GiB to GPU in 2.02504s (17.7774GiB/s) | 20260506 00:28:48.710 | INFO | Downloaded 34GiB to GPU in 2.04745s (16.6061GiB/s) | 20260506 00:28:53.844 | INFO | Analyzing chunk frames = [7515648,7516160) | 20260506 00:28:53.844 | INFO | MemInfo: { | 20260506 00:28:53.844 | INFO | "gpu" : " 17 GB", | 20260506 00:28:53.844 | INFO | "huge" : "", | 20260506 00:28:53.844 | INFO | "huge_shared" : " 27 GB", | 20260506 00:28:53.844 | INFO | "malloc" : " 14 GB", | 20260506 00:28:53.844 | INFO | "pinned" : " 29 GB" | 20260506 00:28:53.844 | INFO | } | 20260506 00:28:53.844 | INFO | Perf report for the last 5.13431 seconds: | 20260506 00:28:53.844 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:28:53.844 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:53.844 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:53.844 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:53.844 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260506 00:28:53.844 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:53.844 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:53.844 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:28:53.844 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:53.844 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:53.844 | INFO | Uploaded 36GiB to GPU in 2.12618s (16.9317GiB/s) | 20260506 00:28:53.844 | INFO | Downloaded 34GiB to GPU in 2.01129s (16.9046GiB/s) | 20260506 00:28:53.968 | INFO | Starting a round of Baz IO | 20260506 00:28:57.110 | INFO | Populating RTMetrics report | 20260506 00:28:57.111 | INFO | RTMetrics report populated | 20260506 00:28:57.111 | INFO | Opening tmp RTMetrics json file | 20260506 00:28:57.112 | INFO | Serializing RTMetrics to json | 20260506 00:28:57.114 | INFO | Serialization done, swapping tmp file now... | 20260506 00:28:57.114 | INFO | RTMetrics done | 20260506 00:28:57.189 | INFO | Finished a round of Baz IO: Wrote 1.859 GiB in 3.221 seconds | 20260506 00:28:57.190 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.37% / 0.00% / 78.63% | 20260506 00:28:57.190 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0932018% / 0% / 7.15256e-05% | 20260506 00:28:57.190 | INFO | Baz IO overall: | 20260506 00:28:57.190 | INFO | BytesWritten : 4269002813906 | 20260506 00:28:57.190 | INFO | NumEvents : 2008853864595 | 20260506 00:28:58.983 | INFO | Analyzing chunk frames = [7516160,7516672) | 20260506 00:28:58.983 | INFO | MemInfo: { | 20260506 00:28:58.983 | INFO | "gpu" : " 17 GB", | 20260506 00:28:58.983 | INFO | "huge" : "", | 20260506 00:28:58.983 | INFO | "huge_shared" : " 27 GB", | 20260506 00:28:58.983 | INFO | "malloc" : " 14 GB", | 20260506 00:28:58.983 | INFO | "pinned" : " 29 GB" | 20260506 00:28:58.983 | INFO | } | 20260506 00:28:58.983 | INFO | Perf report for the last 5.13921 seconds: | 20260506 00:28:58.983 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:28:58.983 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:58.983 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:58.983 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:58.983 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260506 00:28:58.983 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:28:58.983 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260506 00:28:58.983 | INFO | PRE_HQ : 59.7%, 1.7, 53.6ms, 0 | 20260506 00:28:58.983 | INFO | BAZWRITER : 61.9%, 3.5, 111.7ms, 0 | 20260506 00:28:58.983 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:28:58.983 | INFO | Uploaded 36GiB to GPU in 2.16996s (16.5902GiB/s) | 20260506 00:28:58.983 | INFO | Downloaded 38GiB to GPU in 2.23782s (16.9808GiB/s) | 20260506 00:29:04.128 | INFO | Analyzing chunk frames = [7516672,7517184) | 20260506 00:29:04.128 | INFO | MemInfo: { | 20260506 00:29:04.128 | INFO | "gpu" : " 17 GB", | 20260506 00:29:04.128 | INFO | "huge" : "", | 20260506 00:29:04.128 | INFO | "huge_shared" : " 27 GB", | 20260506 00:29:04.128 | INFO | "malloc" : " 14 GB", | 20260506 00:29:04.128 | INFO | "pinned" : " 29 GB" | 20260506 00:29:04.128 | INFO | } | 20260506 00:29:04.128 | INFO | Perf report for the last 5.14463 seconds: | 20260506 00:29:04.128 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:29:04.128 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:04.128 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:04.128 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:04.128 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260506 00:29:04.128 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:04.128 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:04.128 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 00:29:04.128 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:04.128 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:04.128 | INFO | Uploaded 36GiB to GPU in 2.11057s (17.057GiB/s) | 20260506 00:29:04.128 | INFO | Downloaded 34GiB to GPU in 2.04481s (16.6274GiB/s) | 20260506 00:29:09.272 | INFO | Analyzing chunk frames = [7517184,7517696) | 20260506 00:29:09.272 | INFO | MemInfo: { | 20260506 00:29:09.272 | INFO | "gpu" : " 17 GB", | 20260506 00:29:09.272 | INFO | "huge" : "", | 20260506 00:29:09.272 | INFO | "huge_shared" : " 27 GB", | 20260506 00:29:09.272 | INFO | "malloc" : " 14 GB", | 20260506 00:29:09.272 | INFO | "pinned" : " 29 GB" | 20260506 00:29:09.272 | INFO | } | 20260506 00:29:09.272 | INFO | Perf report for the last 5.14433 seconds: | 20260506 00:29:09.272 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:29:09.272 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:09.272 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:09.272 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:09.272 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 00:29:09.272 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:09.272 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:09.272 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:29:09.272 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:09.272 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:09.272 | INFO | Uploaded 36GiB to GPU in 2.07642s (17.3376GiB/s) | 20260506 00:29:09.272 | INFO | Downloaded 34GiB to GPU in 2.01663s (16.8598GiB/s) | 20260506 00:29:14.411 | INFO | Analyzing chunk frames = [7517696,7518208) | 20260506 00:29:14.411 | INFO | MemInfo: { | 20260506 00:29:14.411 | INFO | "gpu" : " 17 GB", | 20260506 00:29:14.411 | INFO | "huge" : "", | 20260506 00:29:14.411 | INFO | "huge_shared" : " 27 GB", | 20260506 00:29:14.411 | INFO | "malloc" : " 14 GB", | 20260506 00:29:14.411 | INFO | "pinned" : " 29 GB" | 20260506 00:29:14.411 | INFO | } | 20260506 00:29:14.411 | INFO | Perf report for the last 5.13936 seconds: | 20260506 00:29:14.411 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:29:14.411 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:14.411 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:14.411 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:14.411 | INFO | ANALYSIS : 61.6%, 3.0, 95.6ms, 0 | 20260506 00:29:14.411 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:14.411 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:14.412 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:29:14.412 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:14.412 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:14.412 | INFO | Uploaded 36GiB to GPU in 2.15336s (16.7181GiB/s) | 20260506 00:29:14.412 | INFO | Downloaded 34GiB to GPU in 2.05166s (16.572GiB/s) | 20260506 00:29:19.542 | INFO | Analyzing chunk frames = [7518208,7518720) | 20260506 00:29:19.542 | INFO | MemInfo: { | 20260506 00:29:19.542 | INFO | "gpu" : " 17 GB", | 20260506 00:29:19.542 | INFO | "huge" : "", | 20260506 00:29:19.542 | INFO | "huge_shared" : " 27 GB", | 20260506 00:29:19.542 | INFO | "malloc" : " 14 GB", | 20260506 00:29:19.542 | INFO | "pinned" : " 29 GB" | 20260506 00:29:19.542 | INFO | } | 20260506 00:29:19.542 | INFO | Perf report for the last 5.13066 seconds: | 20260506 00:29:19.542 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:29:19.542 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:19.542 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:19.542 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:19.542 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 00:29:19.542 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:19.542 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:19.542 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 00:29:19.542 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:19.542 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:19.542 | INFO | Uploaded 36GiB to GPU in 2.0936s (17.1953GiB/s) | 20260506 00:29:19.542 | INFO | Downloaded 34GiB to GPU in 2.05066s (16.58GiB/s) | 20260506 00:29:24.682 | INFO | Analyzing chunk frames = [7518720,7519232) | 20260506 00:29:24.682 | INFO | MemInfo: { | 20260506 00:29:24.682 | INFO | "gpu" : " 17 GB", | 20260506 00:29:24.682 | INFO | "huge" : "", | 20260506 00:29:24.682 | INFO | "huge_shared" : " 27 GB", | 20260506 00:29:24.682 | INFO | "malloc" : " 14 GB", | 20260506 00:29:24.682 | INFO | "pinned" : " 29 GB" | 20260506 00:29:24.682 | INFO | } | 20260506 00:29:24.682 | INFO | Perf report for the last 5.14006 seconds: | 20260506 00:29:24.682 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:29:24.682 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:24.682 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:24.682 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:24.682 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260506 00:29:24.682 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:24.682 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:24.682 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:29:24.682 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:24.682 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:24.682 | INFO | Uploaded 36GiB to GPU in 2.07794s (17.3248GiB/s) | 20260506 00:29:24.682 | INFO | Downloaded 34GiB to GPU in 2.03602s (16.6993GiB/s) | 20260506 00:29:29.823 | INFO | Analyzing chunk frames = [7519232,7519744) | 20260506 00:29:29.823 | INFO | MemInfo: { | 20260506 00:29:29.823 | INFO | "gpu" : " 17 GB", | 20260506 00:29:29.823 | INFO | "huge" : "", | 20260506 00:29:29.823 | INFO | "huge_shared" : " 27 GB", | 20260506 00:29:29.823 | INFO | "malloc" : " 14 GB", | 20260506 00:29:29.823 | INFO | "pinned" : " 29 GB" | 20260506 00:29:29.823 | INFO | } | 20260506 00:29:29.823 | INFO | Perf report for the last 5.14079 seconds: | 20260506 00:29:29.823 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:29:29.823 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:29.823 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:29.823 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:29.823 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260506 00:29:29.823 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:29.823 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:29.823 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 00:29:29.823 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:29.823 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:29.823 | INFO | Uploaded 36GiB to GPU in 2.06734s (17.4136GiB/s) | 20260506 00:29:29.823 | INFO | Downloaded 34GiB to GPU in 2.06226s (16.4868GiB/s) | 20260506 00:29:34.967 | INFO | Analyzing chunk frames = [7519744,7520256) | 20260506 00:29:34.967 | INFO | MemInfo: { | 20260506 00:29:34.967 | INFO | "gpu" : " 17 GB", | 20260506 00:29:34.967 | INFO | "huge" : "", | 20260506 00:29:34.967 | INFO | "huge_shared" : " 27 GB", | 20260506 00:29:34.967 | INFO | "malloc" : " 14 GB", | 20260506 00:29:34.967 | INFO | "pinned" : " 29 GB" | 20260506 00:29:34.967 | INFO | } | 20260506 00:29:34.967 | INFO | Perf report for the last 5.1444 seconds: | 20260506 00:29:34.967 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:29:34.967 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:34.967 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:34.967 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:34.967 | INFO | ANALYSIS : 61.8%, 3.0, 96.6ms, 0 | 20260506 00:29:34.967 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:34.967 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:34.967 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:29:34.967 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:34.967 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:34.968 | INFO | Uploaded 36GiB to GPU in 2.13001s (16.9014GiB/s) | 20260506 00:29:34.968 | INFO | Downloaded 34GiB to GPU in 2.0215s (16.8192GiB/s) | 20260506 00:29:34.968 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 00:29:34.968 | INFO | Total mem usage: 15.3 GiB | 20260506 00:29:34.968 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 00:29:34.968 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.16%) | 20260506 00:29:34.968 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 00:29:34.968 | INFO | High Water Breakdown: | 20260506 00:29:34.968 | INFO | 60.9% : 8.55 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 00:29:34.968 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 00:29:34.968 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 00:29:34.968 | INFO | ------------------------------------------ | 20260506 00:29:34.968 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:29:34.968 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 00:29:34.968 | INFO | Total mem usage: 26.5 GiB | 20260506 00:29:34.968 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 00:29:34.968 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 00:29:34.968 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260506 00:29:34.968 | INFO | High Water Breakdown: | 20260506 00:29:34.968 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:29:34.968 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:29:34.968 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:29:34.968 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:29:34.968 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:29:34.968 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:29:34.968 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 00:29:34.968 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 00:29:34.968 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:29:34.968 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 00:29:34.968 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 00:29:34.968 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 00:29:34.968 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 00:29:34.968 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 00:29:34.968 | INFO | ------------------------------------------ | 20260506 00:29:34.968 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:29:34.968 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 00:29:34.968 | INFO | Total mem usage: 25.6 GiB | 20260506 00:29:34.968 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 00:29:34.968 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 00:29:34.968 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 00:29:34.968 | INFO | High Water Breakdown: | 20260506 00:29:34.968 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 00:29:34.968 | INFO | ------------------------------------------ | 20260506 00:29:34.968 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:29:34.968 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 00:29:34.968 | INFO | Total mem usage: 15.7 GiB | 20260506 00:29:34.968 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 00:29:34.968 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 00:29:34.968 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 00:29:34.968 | INFO | High Water Breakdown: | 20260506 00:29:34.968 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 00:29:34.968 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 00:29:34.968 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 00:29:34.968 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 00:29:34.968 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 00:29:34.968 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 00:29:34.968 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 00:29:34.968 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 00:29:34.968 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 00:29:34.968 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:29:34.968 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 00:29:34.968 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:29:34.968 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 00:29:34.968 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 00:29:34.968 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 00:29:34.968 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 00:29:34.968 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:29:34.968 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 00:29:34.968 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:29:34.968 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:29:34.968 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:29:34.968 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 00:29:34.968 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 00:29:34.968 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:29:34.968 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 00:29:34.968 | INFO | ------------------------------------------ | 20260506 00:29:34.968 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:29:34.968 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 00:29:34.968 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:29:34.968 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:29:34.968 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.38e+01, 1.35e+01 / 8.28e-01 | 20260506 00:29:34.968 | INFO | PerfReport: 0.29%, AggregateStats -- 9.17e-02 / 1.03e-01, 1.04e-02 / 3.56e-03 | 20260506 00:29:34.968 | INFO | PerfReport: 11.84%, Baseline -- 3.70e+00 / 4.49e+00, 5.68e-01 / 1.98e-01 | 20260506 00:29:34.968 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.52e+00, 1.87e-01 / 9.21e-02 | 20260506 00:29:34.969 | INFO | PerfReport: 16.03%, DME -- 4.01e+01 / 4.40e+01, 3.79e+00 / 1.05e+00 | 20260506 00:29:34.969 | INFO | PerfReport: 57.29%, FrameLabeling -- 1.79e+01 / 1.93e+01, 1.41e+00 / 1.15e+00 | 20260506 00:29:34.969 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.23e+00 / 2.76e+00, 4.23e-01 / 2.28e-01 | 20260506 00:29:34.969 | INFO | PerfReport: 3.09%, Metrics -- 9.65e-01 / 1.25e+00, 2.58e-01 / 1.14e-01 | 20260506 00:29:34.969 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.33e-01 / 3.91e-01, 5.55e-02 / 2.75e-02 | 20260506 00:29:34.969 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 00:29:34.969 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:29:34.969 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:29:34.969 | INFO | PerfReport: Full Execution -- 9.82e+01 / -nan, 2.45e+01 / 0.00e+00 | 20260506 00:29:34.969 | INFO | PerfReport: 22.34%, Upload -- 2.19e+01 / 0.00e+00, 7.45e+00 / 0.00e+00 | 20260506 00:29:34.969 | INFO | PerfReport: 21.87%, Download -- 2.15e+01 / 0.00e+00, 7.59e+00 / 0.00e+00 | 20260506 00:29:34.969 | INFO | PerfReport: 31.83%, Compute -- 3.13e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 00:29:34.969 | INFO | PerfReport: 2.11%, LockUpload -- 2.07e+00 / 0.00e+00, 7.91e+00 / 0.00e+00 | 20260506 00:29:34.969 | INFO | PerfReport: 1.35%, LockDownload -- 1.32e+00 / 0.00e+00, 2.96e+00 / 0.00e+00 | 20260506 00:29:34.969 | INFO | PerfReport: 20.49%, LockCompute -- 2.01e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 00:29:34.969 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.93e-02 / 0.00e+00 | 20260506 00:29:35.085 | INFO | Starting a round of Baz IO | 20260506 00:29:38.235 | INFO | Populating RTMetrics report | 20260506 00:29:38.235 | INFO | RTMetrics report populated | 20260506 00:29:38.235 | INFO | Opening tmp RTMetrics json file | 20260506 00:29:38.237 | INFO | Serializing RTMetrics to json | 20260506 00:29:38.239 | INFO | Serialization done, swapping tmp file now... | 20260506 00:29:38.239 | INFO | RTMetrics done | 20260506 00:29:38.311 | INFO | Finished a round of Baz IO: Wrote 1.859 GiB in 3.226 seconds | 20260506 00:29:38.311 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.37% / 0.00% / 78.63% | 20260506 00:29:38.311 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0925819% / 0% / 4.76837e-05% | 20260506 00:29:38.311 | INFO | Baz IO overall: | 20260506 00:29:38.311 | INFO | BytesWritten : 4270998375211 | 20260506 00:29:38.312 | INFO | NumEvents : 2009766814252 | 20260506 00:29:40.106 | INFO | Analyzing chunk frames = [7520256,7520768) | 20260506 00:29:40.106 | INFO | MemInfo: { | 20260506 00:29:40.106 | INFO | "gpu" : " 17 GB", | 20260506 00:29:40.106 | INFO | "huge" : "", | 20260506 00:29:40.106 | INFO | "huge_shared" : " 27 GB", | 20260506 00:29:40.106 | INFO | "malloc" : " 14 GB", | 20260506 00:29:40.106 | INFO | "pinned" : " 29 GB" | 20260506 00:29:40.106 | INFO | } | 20260506 00:29:40.106 | INFO | Perf report for the last 5.13876 seconds: | 20260506 00:29:40.106 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:29:40.106 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:40.106 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:40.106 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:40.106 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260506 00:29:40.106 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:40.106 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260506 00:29:40.106 | INFO | PRE_HQ : 59.7%, 1.7, 54.0ms, 0 | 20260506 00:29:40.106 | INFO | BAZWRITER : 61.8%, 3.5, 111.0ms, 0 | 20260506 00:29:40.106 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:29:40.106 | INFO | Uploaded 36GiB to GPU in 2.1825s (16.4949GiB/s) | 20260506 00:29:40.106 | INFO | Downloaded 38GiB to GPU in 2.24106s (16.9563GiB/s) | 20260506 00:29:45.240 | INFO | Analyzing chunk frames = [7520768,7521280) | 20260506 00:29:45.240 | INFO | MemInfo: { | 20260506 00:29:45.240 | INFO | "gpu" : " 17 GB", | 20260506 00:29:45.240 | INFO | "huge" : "", | 20260506 00:29:45.240 | INFO | "huge_shared" : " 27 GB", | 20260506 00:29:45.240 | INFO | "malloc" : " 14 GB", | 20260506 00:29:45.240 | INFO | "pinned" : " 29 GB" | 20260506 00:29:45.240 | INFO | } | 20260506 00:29:45.241 | INFO | Perf report for the last 5.13436 seconds: | 20260506 00:29:45.241 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:29:45.241 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:45.241 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:45.241 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:45.241 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 00:29:45.241 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:45.241 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:45.241 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:29:45.241 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:45.241 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:45.241 | INFO | Uploaded 36GiB to GPU in 2.09539s (17.1805GiB/s) | 20260506 00:29:45.241 | INFO | Downloaded 34GiB to GPU in 2.03704s (16.6909GiB/s) | 20260506 00:29:50.384 | INFO | Analyzing chunk frames = [7521280,7521792) | 20260506 00:29:50.384 | INFO | MemInfo: { | 20260506 00:29:50.384 | INFO | "gpu" : " 17 GB", | 20260506 00:29:50.384 | INFO | "huge" : "", | 20260506 00:29:50.384 | INFO | "huge_shared" : " 27 GB", | 20260506 00:29:50.384 | INFO | "malloc" : " 14 GB", | 20260506 00:29:50.384 | INFO | "pinned" : " 29 GB" | 20260506 00:29:50.384 | INFO | } | 20260506 00:29:50.384 | INFO | Perf report for the last 5.14378 seconds: | 20260506 00:29:50.384 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:29:50.384 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:50.384 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:50.384 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:50.384 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 00:29:50.384 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:50.384 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:50.384 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:29:50.384 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:50.384 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:50.384 | INFO | Uploaded 36GiB to GPU in 2.09586s (17.1767GiB/s) | 20260506 00:29:50.384 | INFO | Downloaded 34GiB to GPU in 2.03592s (16.7GiB/s) | 20260506 00:29:55.523 | INFO | Analyzing chunk frames = [7521792,7522304) | 20260506 00:29:55.523 | INFO | MemInfo: { | 20260506 00:29:55.523 | INFO | "gpu" : " 17 GB", | 20260506 00:29:55.523 | INFO | "huge" : "", | 20260506 00:29:55.523 | INFO | "huge_shared" : " 27 GB", | 20260506 00:29:55.523 | INFO | "malloc" : " 14 GB", | 20260506 00:29:55.523 | INFO | "pinned" : " 29 GB" | 20260506 00:29:55.523 | INFO | } | 20260506 00:29:55.523 | INFO | Perf report for the last 5.13845 seconds: | 20260506 00:29:55.523 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:29:55.523 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:55.523 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:55.523 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:55.523 | INFO | ANALYSIS : 61.6%, 3.0, 95.7ms, 0 | 20260506 00:29:55.523 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:55.523 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:55.523 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 00:29:55.523 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:55.523 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:29:55.523 | INFO | Uploaded 36GiB to GPU in 2.13939s (16.8272GiB/s) | 20260506 00:29:55.523 | INFO | Downloaded 34GiB to GPU in 2.02807s (16.7647GiB/s) | 20260506 00:30:00.658 | INFO | Analyzing chunk frames = [7522304,7522816) | 20260506 00:30:00.658 | INFO | MemInfo: { | 20260506 00:30:00.658 | INFO | "gpu" : " 17 GB", | 20260506 00:30:00.658 | INFO | "huge" : "", | 20260506 00:30:00.658 | INFO | "huge_shared" : " 27 GB", | 20260506 00:30:00.658 | INFO | "malloc" : " 14 GB", | 20260506 00:30:00.658 | INFO | "pinned" : " 29 GB" | 20260506 00:30:00.658 | INFO | } | 20260506 00:30:00.658 | INFO | Perf report for the last 5.13505 seconds: | 20260506 00:30:00.658 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:30:00.658 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:00.658 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:00.658 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:00.658 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 00:30:00.658 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:00.658 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:00.658 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 00:30:00.658 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:00.658 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:00.658 | INFO | Uploaded 36GiB to GPU in 2.07895s (17.3164GiB/s) | 20260506 00:30:00.658 | INFO | Downloaded 34GiB to GPU in 2.04397s (16.6343GiB/s) | 20260506 00:30:05.794 | INFO | Analyzing chunk frames = [7522816,7523328) | 20260506 00:30:05.794 | INFO | MemInfo: { | 20260506 00:30:05.794 | INFO | "gpu" : " 17 GB", | 20260506 00:30:05.794 | INFO | "huge" : "", | 20260506 00:30:05.794 | INFO | "huge_shared" : " 27 GB", | 20260506 00:30:05.794 | INFO | "malloc" : " 14 GB", | 20260506 00:30:05.795 | INFO | "pinned" : " 29 GB" | 20260506 00:30:05.795 | INFO | } | 20260506 00:30:05.795 | INFO | Perf report for the last 5.13669 seconds: | 20260506 00:30:05.795 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:30:05.795 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:05.795 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:05.795 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:05.795 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260506 00:30:05.795 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:05.795 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:05.795 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:30:05.795 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:05.795 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:05.795 | INFO | Uploaded 36GiB to GPU in 2.07569s (17.3437GiB/s) | 20260506 00:30:05.795 | INFO | Downloaded 34GiB to GPU in 2.04628s (16.6155GiB/s) | 20260506 00:30:10.933 | INFO | Analyzing chunk frames = [7523328,7523840) | 20260506 00:30:10.933 | INFO | MemInfo: { | 20260506 00:30:10.933 | INFO | "gpu" : " 17 GB", | 20260506 00:30:10.933 | INFO | "huge" : "", | 20260506 00:30:10.933 | INFO | "huge_shared" : " 27 GB", | 20260506 00:30:10.933 | INFO | "malloc" : " 14 GB", | 20260506 00:30:10.933 | INFO | "pinned" : " 29 GB" | 20260506 00:30:10.933 | INFO | } | 20260506 00:30:10.933 | INFO | Perf report for the last 5.13862 seconds: | 20260506 00:30:10.933 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:30:10.933 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:10.933 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:10.933 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:10.933 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260506 00:30:10.933 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:10.933 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:10.933 | INFO | PRE_HQ : 27.5%, 1.0, 14.2ms, 0 | 20260506 00:30:10.933 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:10.933 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:10.933 | INFO | Uploaded 36GiB to GPU in 2.15462s (16.7083GiB/s) | 20260506 00:30:10.934 | INFO | Downloaded 34GiB to GPU in 2.13069s (15.9573GiB/s) | 20260506 00:30:16.081 | INFO | Analyzing chunk frames = [7523840,7524352) | 20260506 00:30:16.081 | INFO | MemInfo: { | 20260506 00:30:16.081 | INFO | "gpu" : " 17 GB", | 20260506 00:30:16.081 | INFO | "huge" : "", | 20260506 00:30:16.081 | INFO | "huge_shared" : " 27 GB", | 20260506 00:30:16.082 | INFO | "malloc" : " 14 GB", | 20260506 00:30:16.082 | INFO | "pinned" : " 29 GB" | 20260506 00:30:16.082 | INFO | } | 20260506 00:30:16.082 | INFO | Perf report for the last 5.14848 seconds: | 20260506 00:30:16.082 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:30:16.082 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:16.082 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:16.082 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:16.082 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260506 00:30:16.082 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:16.082 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:16.082 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 00:30:16.082 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:16.082 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:16.082 | INFO | Uploaded 36GiB to GPU in 2.13246s (16.8819GiB/s) | 20260506 00:30:16.082 | INFO | Downloaded 34GiB to GPU in 2.03113s (16.7394GiB/s) | 20260506 00:30:16.208 | INFO | Starting a round of Baz IO | 20260506 00:30:19.349 | INFO | Populating RTMetrics report | 20260506 00:30:19.350 | INFO | RTMetrics report populated | 20260506 00:30:19.350 | INFO | Opening tmp RTMetrics json file | 20260506 00:30:19.352 | INFO | Serializing RTMetrics to json | 20260506 00:30:19.355 | INFO | Serialization done, swapping tmp file now... | 20260506 00:30:19.356 | INFO | RTMetrics done | 20260506 00:30:19.434 | INFO | Finished a round of Baz IO: Wrote 1.859 GiB in 3.226 seconds | 20260506 00:30:19.434 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.37% / 0.00% / 78.63% | 20260506 00:30:19.434 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0926455% / 0% / 8.74201e-05% | 20260506 00:30:19.434 | INFO | Baz IO overall: | 20260506 00:30:19.434 | INFO | BytesWritten : 4272994157873 | 20260506 00:30:19.434 | INFO | NumEvents : 2010679620005 | 20260506 00:30:21.222 | INFO | Analyzing chunk frames = [7524352,7524864) | 20260506 00:30:21.222 | INFO | MemInfo: { | 20260506 00:30:21.222 | INFO | "gpu" : " 17 GB", | 20260506 00:30:21.222 | INFO | "huge" : "", | 20260506 00:30:21.222 | INFO | "huge_shared" : " 27 GB", | 20260506 00:30:21.222 | INFO | "malloc" : " 14 GB", | 20260506 00:30:21.222 | INFO | "pinned" : " 29 GB" | 20260506 00:30:21.222 | INFO | } | 20260506 00:30:21.223 | INFO | Perf report for the last 5.14081 seconds: | 20260506 00:30:21.223 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:30:21.223 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:21.223 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:21.223 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:21.223 | INFO | ANALYSIS : 62.5%, 3.0, 97.7ms, 0 | 20260506 00:30:21.223 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:21.223 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260506 00:30:21.223 | INFO | PRE_HQ : 60.0%, 1.8, 54.6ms, 0 | 20260506 00:30:21.223 | INFO | BAZWRITER : 62.0%, 3.5, 112.1ms, 0 | 20260506 00:30:21.223 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 00:30:21.223 | INFO | Uploaded 36GiB to GPU in 2.19049s (16.4346GiB/s) | 20260506 00:30:21.223 | INFO | Downloaded 38GiB to GPU in 2.24785s (16.9051GiB/s) | 20260506 00:30:26.354 | INFO | Analyzing chunk frames = [7524864,7525376) | 20260506 00:30:26.354 | INFO | MemInfo: { | 20260506 00:30:26.354 | INFO | "gpu" : " 17 GB", | 20260506 00:30:26.354 | INFO | "huge" : "", | 20260506 00:30:26.354 | INFO | "huge_shared" : " 27 GB", | 20260506 00:30:26.354 | INFO | "malloc" : " 14 GB", | 20260506 00:30:26.354 | INFO | "pinned" : " 29 GB" | 20260506 00:30:26.354 | INFO | } | 20260506 00:30:26.355 | INFO | Perf report for the last 5.132 seconds: | 20260506 00:30:26.355 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:30:26.355 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:26.355 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:26.355 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:26.355 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260506 00:30:26.355 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:26.355 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:26.355 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 00:30:26.355 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:26.355 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:26.355 | INFO | Uploaded 36GiB to GPU in 1.95493s (18.415GiB/s) | 20260506 00:30:26.355 | INFO | Downloaded 34GiB to GPU in 1.97939s (17.177GiB/s) | 20260506 00:30:31.490 | INFO | Analyzing chunk frames = [7525376,7525888) | 20260506 00:30:31.490 | INFO | MemInfo: { | 20260506 00:30:31.490 | INFO | "gpu" : " 17 GB", | 20260506 00:30:31.490 | INFO | "huge" : "", | 20260506 00:30:31.490 | INFO | "huge_shared" : " 27 GB", | 20260506 00:30:31.490 | INFO | "malloc" : " 14 GB", | 20260506 00:30:31.490 | INFO | "pinned" : " 29 GB" | 20260506 00:30:31.490 | INFO | } | 20260506 00:30:31.490 | INFO | Perf report for the last 5.13588 seconds: | 20260506 00:30:31.490 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:30:31.490 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:31.490 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:31.490 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:31.491 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 00:30:31.491 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:31.491 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:31.491 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 00:30:31.491 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:31.491 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:31.491 | INFO | Uploaded 36GiB to GPU in 2.14791s (16.7605GiB/s) | 20260506 00:30:31.491 | INFO | Downloaded 34GiB to GPU in 2.04922s (16.5917GiB/s) | 20260506 00:30:36.632 | INFO | Analyzing chunk frames = [7525888,7526400) | 20260506 00:30:36.632 | INFO | MemInfo: { | 20260506 00:30:36.632 | INFO | "gpu" : " 17 GB", | 20260506 00:30:36.632 | INFO | "huge" : "", | 20260506 00:30:36.632 | INFO | "huge_shared" : " 27 GB", | 20260506 00:30:36.632 | INFO | "malloc" : " 14 GB", | 20260506 00:30:36.632 | INFO | "pinned" : " 29 GB" | 20260506 00:30:36.632 | INFO | } | 20260506 00:30:36.632 | INFO | Perf report for the last 5.1414 seconds: | 20260506 00:30:36.632 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:30:36.632 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:36.632 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:36.632 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:36.632 | INFO | ANALYSIS : 61.7%, 3.0, 95.8ms, 0 | 20260506 00:30:36.632 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:36.632 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:36.632 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:30:36.632 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:36.632 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:36.632 | INFO | Uploaded 36GiB to GPU in 2.12997s (16.9017GiB/s) | 20260506 00:30:36.632 | INFO | Downloaded 34GiB to GPU in 2.02268s (16.8094GiB/s) | 20260506 00:30:41.779 | INFO | Analyzing chunk frames = [7526400,7526912) | 20260506 00:30:41.779 | INFO | MemInfo: { | 20260506 00:30:41.779 | INFO | "gpu" : " 17 GB", | 20260506 00:30:41.779 | INFO | "huge" : "", | 20260506 00:30:41.779 | INFO | "huge_shared" : " 27 GB", | 20260506 00:30:41.779 | INFO | "malloc" : " 14 GB", | 20260506 00:30:41.779 | INFO | "pinned" : " 29 GB" | 20260506 00:30:41.779 | INFO | } | 20260506 00:30:41.779 | INFO | Perf report for the last 5.14739 seconds: | 20260506 00:30:41.779 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:30:41.779 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:41.779 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:41.779 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:41.779 | INFO | ANALYSIS : 61.1%, 3.0, 95.5ms, 0 | 20260506 00:30:41.779 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:41.779 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:41.779 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:30:41.779 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:41.779 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:41.780 | INFO | Uploaded 36GiB to GPU in 2.11068s (17.0561GiB/s) | 20260506 00:30:41.780 | INFO | Downloaded 34GiB to GPU in 2.05065s (16.5801GiB/s) | 20260506 00:30:46.918 | INFO | Analyzing chunk frames = [7526912,7527424) | 20260506 00:30:46.918 | INFO | MemInfo: { | 20260506 00:30:46.918 | INFO | "gpu" : " 17 GB", | 20260506 00:30:46.918 | INFO | "huge" : "", | 20260506 00:30:46.918 | INFO | "huge_shared" : " 27 GB", | 20260506 00:30:46.918 | INFO | "malloc" : " 14 GB", | 20260506 00:30:46.918 | INFO | "pinned" : " 29 GB" | 20260506 00:30:46.918 | INFO | } | 20260506 00:30:46.918 | INFO | Perf report for the last 5.13868 seconds: | 20260506 00:30:46.918 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:30:46.918 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:46.918 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:46.918 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:46.918 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 00:30:46.918 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:46.918 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:46.918 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 00:30:46.918 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:46.918 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:46.918 | INFO | Uploaded 36GiB to GPU in 2.07601s (17.341GiB/s) | 20260506 00:30:46.918 | INFO | Downloaded 34GiB to GPU in 2.03252s (16.728GiB/s) | 20260506 00:30:52.055 | INFO | Analyzing chunk frames = [7527424,7527936) | 20260506 00:30:52.055 | INFO | MemInfo: { | 20260506 00:30:52.055 | INFO | "gpu" : " 17 GB", | 20260506 00:30:52.055 | INFO | "huge" : "", | 20260506 00:30:52.055 | INFO | "huge_shared" : " 27 GB", | 20260506 00:30:52.055 | INFO | "malloc" : " 14 GB", | 20260506 00:30:52.056 | INFO | "pinned" : " 29 GB" | 20260506 00:30:52.056 | INFO | } | 20260506 00:30:52.056 | INFO | Perf report for the last 5.13763 seconds: | 20260506 00:30:52.056 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:30:52.056 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:52.056 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:52.056 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:52.056 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 00:30:52.056 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:52.056 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:52.056 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 00:30:52.056 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:52.056 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:52.056 | INFO | Uploaded 36GiB to GPU in 2.0666s (17.4199GiB/s) | 20260506 00:30:52.056 | INFO | Downloaded 34GiB to GPU in 2.06117s (16.4955GiB/s) | 20260506 00:30:57.189 | INFO | Analyzing chunk frames = [7527936,7528448) | 20260506 00:30:57.189 | INFO | MemInfo: { | 20260506 00:30:57.189 | INFO | "gpu" : " 17 GB", | 20260506 00:30:57.189 | INFO | "huge" : "", | 20260506 00:30:57.189 | INFO | "huge_shared" : " 27 GB", | 20260506 00:30:57.189 | INFO | "malloc" : " 14 GB", | 20260506 00:30:57.189 | INFO | "pinned" : " 29 GB" | 20260506 00:30:57.189 | INFO | } | 20260506 00:30:57.189 | INFO | Perf report for the last 5.13362 seconds: | 20260506 00:30:57.189 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:30:57.189 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:57.189 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:57.189 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:57.189 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260506 00:30:57.189 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:57.189 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:57.189 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 00:30:57.189 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:57.189 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:30:57.189 | INFO | Uploaded 36GiB to GPU in 2.12616s (16.932GiB/s) | 20260506 00:30:57.189 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 00:30:57.189 | INFO | Total mem usage: 15.3 GiB | 20260506 00:30:57.189 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 00:30:57.189 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.15%) | 20260506 00:30:57.189 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 00:30:57.189 | INFO | High Water Breakdown: | 20260506 00:30:57.189 | INFO | 60.9% : 8.55 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 00:30:57.189 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 00:30:57.189 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 00:30:57.189 | INFO | ------------------------------------------ | 20260506 00:30:57.189 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:30:57.189 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 00:30:57.189 | INFO | Total mem usage: 26.5 GiB | 20260506 00:30:57.189 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 00:30:57.189 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 00:30:57.189 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 00:30:57.189 | INFO | High Water Breakdown: | 20260506 00:30:57.189 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:30:57.189 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:30:57.189 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:30:57.189 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:30:57.189 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:30:57.189 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:30:57.190 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 00:30:57.190 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 00:30:57.190 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:30:57.190 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 00:30:57.190 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 00:30:57.190 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 00:30:57.190 | INFO | Downloaded 34GiB to GPU in 2.01055s (16.9108GiB/s) | 20260506 00:30:57.190 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 00:30:57.190 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 00:30:57.190 | INFO | ------------------------------------------ | 20260506 00:30:57.190 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:30:57.190 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 00:30:57.190 | INFO | Total mem usage: 25.6 GiB | 20260506 00:30:57.190 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 00:30:57.190 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 00:30:57.190 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 00:30:57.190 | INFO | High Water Breakdown: | 20260506 00:30:57.190 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 00:30:57.190 | INFO | ------------------------------------------ | 20260506 00:30:57.190 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:30:57.190 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 00:30:57.190 | INFO | Total mem usage: 15.7 GiB | 20260506 00:30:57.190 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 00:30:57.190 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 00:30:57.190 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 00:30:57.190 | INFO | High Water Breakdown: | 20260506 00:30:57.190 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 00:30:57.190 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 00:30:57.190 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 00:30:57.190 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 00:30:57.190 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 00:30:57.190 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 00:30:57.190 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 00:30:57.190 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 00:30:57.190 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 00:30:57.190 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:30:57.190 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 00:30:57.190 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:30:57.190 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 00:30:57.190 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 00:30:57.190 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 00:30:57.190 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 00:30:57.190 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:30:57.190 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 00:30:57.190 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:30:57.190 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:30:57.190 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:30:57.190 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 00:30:57.190 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 00:30:57.190 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:30:57.190 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 00:30:57.190 | INFO | ------------------------------------------ | 20260506 00:30:57.190 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:30:57.190 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 00:30:57.190 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:30:57.190 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:30:57.190 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.38e+01, 1.35e+01 / 1.09e+00 | 20260506 00:30:57.190 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.03e-01, 1.08e-02 / 4.86e-03 | 20260506 00:30:57.190 | INFO | PerfReport: 11.83%, Baseline -- 3.70e+00 / 4.49e+00, 5.65e-01 / 2.25e-01 | 20260506 00:30:57.190 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.50e+00, 1.87e-01 / 1.06e-01 | 20260506 00:30:57.190 | INFO | PerfReport: 15.97%, DME -- 4.00e+01 / 4.38e+01, 3.71e+00 / 1.23e+00 | 20260506 00:30:57.190 | INFO | PerfReport: 57.41%, FrameLabeling -- 1.80e+01 / 1.96e+01, 1.51e+00 / 1.06e+00 | 20260506 00:30:57.190 | INFO | PerfReport: 7.08%, PulseAccumulating -- 2.22e+00 / 2.71e+00, 4.20e-01 / 2.58e-01 | 20260506 00:30:57.190 | INFO | PerfReport: 3.08%, Metrics -- 9.64e-01 / 1.23e+00, 2.59e-01 / 1.21e-01 | 20260506 00:30:57.190 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.34e-01 / 3.85e-01, 5.67e-02 / 3.26e-02 | 20260506 00:30:57.191 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 00:30:57.191 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:30:57.191 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:30:57.191 | INFO | PerfReport: Full Execution -- 9.83e+01 / -nan, 2.46e+01 / 0.00e+00 | 20260506 00:30:57.191 | INFO | PerfReport: 22.36%, Upload -- 2.20e+01 / 0.00e+00, 7.52e+00 / 0.00e+00 | 20260506 00:30:57.191 | INFO | PerfReport: 21.89%, Download -- 2.15e+01 / 0.00e+00, 7.62e+00 / 0.00e+00 | 20260506 00:30:57.191 | INFO | PerfReport: 31.85%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 00:30:57.191 | INFO | PerfReport: 2.10%, LockUpload -- 2.07e+00 / 0.00e+00, 7.86e+00 / 0.00e+00 | 20260506 00:30:57.191 | INFO | PerfReport: 1.33%, LockDownload -- 1.31e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260506 00:30:57.191 | INFO | PerfReport: 20.44%, LockCompute -- 2.01e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260506 00:30:57.191 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.61e-02 / 0.00e+00 | 20260506 00:30:57.306 | INFO | Starting a round of Baz IO | 20260506 00:31:00.452 | INFO | Populating RTMetrics report | 20260506 00:31:00.452 | INFO | RTMetrics report populated | 20260506 00:31:00.452 | INFO | Opening tmp RTMetrics json file | 20260506 00:31:00.455 | INFO | Serializing RTMetrics to json | 20260506 00:31:00.457 | INFO | Serialization done, swapping tmp file now... | 20260506 00:31:00.457 | INFO | RTMetrics done | 20260506 00:31:00.531 | INFO | Finished a round of Baz IO: Wrote 1.857 GiB in 3.225 seconds | 20260506 00:31:00.531 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.37% / 0.00% / 78.63% | 20260506 00:31:00.531 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0922163% / 3.97364e-06% / 6.35783e-05% | 20260506 00:31:00.531 | INFO | Baz IO overall: | 20260506 00:31:00.531 | INFO | BytesWritten : 4274988378875 | 20260506 00:31:00.531 | INFO | NumEvents : 2011590903485 | 20260506 00:31:02.332 | INFO | Analyzing chunk frames = [7528448,7528960) | 20260506 00:31:02.332 | INFO | MemInfo: { | 20260506 00:31:02.332 | INFO | "gpu" : " 17 GB", | 20260506 00:31:02.332 | INFO | "huge" : "", | 20260506 00:31:02.332 | INFO | "huge_shared" : " 27 GB", | 20260506 00:31:02.332 | INFO | "malloc" : " 14 GB", | 20260506 00:31:02.332 | INFO | "pinned" : " 29 GB" | 20260506 00:31:02.332 | INFO | } | 20260506 00:31:02.332 | INFO | Perf report for the last 5.143 seconds: | 20260506 00:31:02.332 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:31:02.332 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:02.332 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:02.332 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:02.332 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260506 00:31:02.332 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:02.332 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260506 00:31:02.332 | INFO | PRE_HQ : 59.8%, 1.7, 53.9ms, 0 | 20260506 00:31:02.332 | INFO | BAZWRITER : 61.9%, 3.5, 112.4ms, 0 | 20260506 00:31:02.332 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:31:02.332 | INFO | Uploaded 36GiB to GPU in 2.16629s (16.6183GiB/s) | 20260506 00:31:02.332 | INFO | Downloaded 38GiB to GPU in 2.23396s (17.0102GiB/s) | 20260506 00:31:07.477 | INFO | Analyzing chunk frames = [7528960,7529472) | 20260506 00:31:07.477 | INFO | MemInfo: { | 20260506 00:31:07.477 | INFO | "gpu" : " 17 GB", | 20260506 00:31:07.477 | INFO | "huge" : "", | 20260506 00:31:07.477 | INFO | "huge_shared" : " 27 GB", | 20260506 00:31:07.477 | INFO | "malloc" : " 14 GB", | 20260506 00:31:07.477 | INFO | "pinned" : " 29 GB" | 20260506 00:31:07.477 | INFO | } | 20260506 00:31:07.477 | INFO | Perf report for the last 5.14449 seconds: | 20260506 00:31:07.477 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:31:07.477 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:07.477 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:07.477 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:07.477 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260506 00:31:07.477 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:07.477 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:07.477 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 00:31:07.477 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:07.477 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:07.477 | INFO | Uploaded 36GiB to GPU in 2.10013s (17.1418GiB/s) | 20260506 00:31:07.477 | INFO | Downloaded 34GiB to GPU in 2.03613s (16.6984GiB/s) | 20260506 00:31:12.611 | INFO | Analyzing chunk frames = [7529472,7529984) | 20260506 00:31:12.611 | INFO | MemInfo: { | 20260506 00:31:12.611 | INFO | "gpu" : " 17 GB", | 20260506 00:31:12.611 | INFO | "huge" : "", | 20260506 00:31:12.611 | INFO | "huge_shared" : " 27 GB", | 20260506 00:31:12.611 | INFO | "malloc" : " 14 GB", | 20260506 00:31:12.611 | INFO | "pinned" : " 29 GB" | 20260506 00:31:12.611 | INFO | } | 20260506 00:31:12.611 | INFO | Perf report for the last 5.13429 seconds: | 20260506 00:31:12.611 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:31:12.611 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:12.611 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:12.611 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:12.611 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 00:31:12.611 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:12.611 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:12.611 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:31:12.611 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:12.611 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:12.611 | INFO | Uploaded 36GiB to GPU in 2.08186s (17.2923GiB/s) | 20260506 00:31:12.611 | INFO | Downloaded 34GiB to GPU in 2.01328s (16.8879GiB/s) | 20260506 00:31:17.749 | INFO | Analyzing chunk frames = [7529984,7530496) | 20260506 00:31:17.749 | INFO | MemInfo: { | 20260506 00:31:17.749 | INFO | "gpu" : " 17 GB", | 20260506 00:31:17.749 | INFO | "huge" : "", | 20260506 00:31:17.749 | INFO | "huge_shared" : " 27 GB", | 20260506 00:31:17.749 | INFO | "malloc" : " 14 GB", | 20260506 00:31:17.749 | INFO | "pinned" : " 29 GB" | 20260506 00:31:17.749 | INFO | } | 20260506 00:31:17.749 | INFO | Perf report for the last 5.13773 seconds: | 20260506 00:31:17.749 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:31:17.749 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:17.749 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:17.749 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:17.749 | INFO | ANALYSIS : 61.7%, 3.0, 95.8ms, 0 | 20260506 00:31:17.749 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:17.749 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:17.749 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 00:31:17.749 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:17.749 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:17.749 | INFO | Uploaded 36GiB to GPU in 2.13585s (16.8551GiB/s) | 20260506 00:31:17.749 | INFO | Downloaded 34GiB to GPU in 2.0252s (16.7885GiB/s) | 20260506 00:31:22.892 | INFO | Analyzing chunk frames = [7530496,7531008) | 20260506 00:31:22.892 | INFO | MemInfo: { | 20260506 00:31:22.892 | INFO | "gpu" : " 17 GB", | 20260506 00:31:22.892 | INFO | "huge" : "", | 20260506 00:31:22.892 | INFO | "huge_shared" : " 27 GB", | 20260506 00:31:22.892 | INFO | "malloc" : " 14 GB", | 20260506 00:31:22.892 | INFO | "pinned" : " 29 GB" | 20260506 00:31:22.892 | INFO | } | 20260506 00:31:22.892 | INFO | Perf report for the last 5.14316 seconds: | 20260506 00:31:22.892 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:31:22.892 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:22.892 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:22.892 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:22.892 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 00:31:22.892 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:22.892 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:22.892 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:31:22.892 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:22.892 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:22.892 | INFO | Uploaded 36GiB to GPU in 2.10067s (17.1374GiB/s) | 20260506 00:31:22.892 | INFO | Downloaded 34GiB to GPU in 2.04214s (16.6492GiB/s) | 20260506 00:31:28.033 | INFO | Analyzing chunk frames = [7531008,7531520) | 20260506 00:31:28.033 | INFO | MemInfo: { | 20260506 00:31:28.033 | INFO | "gpu" : " 17 GB", | 20260506 00:31:28.033 | INFO | "huge" : "", | 20260506 00:31:28.033 | INFO | "huge_shared" : " 27 GB", | 20260506 00:31:28.033 | INFO | "malloc" : " 14 GB", | 20260506 00:31:28.033 | INFO | "pinned" : " 29 GB" | 20260506 00:31:28.033 | INFO | } | 20260506 00:31:28.033 | INFO | Perf report for the last 5.1414 seconds: | 20260506 00:31:28.033 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:31:28.033 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:28.033 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:28.033 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:28.033 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 00:31:28.033 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:28.033 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:28.033 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:31:28.033 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:28.033 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:28.034 | INFO | Uploaded 36GiB to GPU in 2.05954s (17.4796GiB/s) | 20260506 00:31:28.034 | INFO | Downloaded 34GiB to GPU in 2.02939s (16.7538GiB/s) | 20260506 00:31:33.168 | INFO | Analyzing chunk frames = [7531520,7532032) | 20260506 00:31:33.168 | INFO | MemInfo: { | 20260506 00:31:33.168 | INFO | "gpu" : " 17 GB", | 20260506 00:31:33.168 | INFO | "huge" : "", | 20260506 00:31:33.168 | INFO | "huge_shared" : " 27 GB", | 20260506 00:31:33.168 | INFO | "malloc" : " 14 GB", | 20260506 00:31:33.168 | INFO | "pinned" : " 29 GB" | 20260506 00:31:33.168 | INFO | } | 20260506 00:31:33.168 | INFO | Perf report for the last 5.13462 seconds: | 20260506 00:31:33.168 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:31:33.168 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:33.168 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:33.168 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:33.168 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260506 00:31:33.168 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:33.168 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:33.168 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 00:31:33.168 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:33.168 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:33.168 | INFO | Uploaded 36GiB to GPU in 2.0875s (17.2455GiB/s) | 20260506 00:31:33.168 | INFO | Downloaded 34GiB to GPU in 2.0716s (16.4124GiB/s) | 20260506 00:31:38.302 | INFO | Analyzing chunk frames = [7532032,7532544) | 20260506 00:31:38.302 | INFO | MemInfo: { | 20260506 00:31:38.302 | INFO | "gpu" : " 17 GB", | 20260506 00:31:38.302 | INFO | "huge" : "", | 20260506 00:31:38.302 | INFO | "huge_shared" : " 27 GB", | 20260506 00:31:38.302 | INFO | "malloc" : " 14 GB", | 20260506 00:31:38.302 | INFO | "pinned" : " 29 GB" | 20260506 00:31:38.302 | INFO | } | 20260506 00:31:38.302 | INFO | Perf report for the last 5.13392 seconds: | 20260506 00:31:38.302 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:31:38.302 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:38.302 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:38.302 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:38.302 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260506 00:31:38.302 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:38.302 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:38.302 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 00:31:38.302 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:38.302 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:38.302 | INFO | Uploaded 36GiB to GPU in 2.12592s (16.9338GiB/s) | 20260506 00:31:38.302 | INFO | Downloaded 34GiB to GPU in 2.01142s (16.9035GiB/s) | 20260506 00:31:38.425 | INFO | Starting a round of Baz IO | 20260506 00:31:41.573 | INFO | Populating RTMetrics report | 20260506 00:31:41.573 | INFO | RTMetrics report populated | 20260506 00:31:41.573 | INFO | Opening tmp RTMetrics json file | 20260506 00:31:41.574 | INFO | Serializing RTMetrics to json | 20260506 00:31:41.578 | INFO | Serialization done, swapping tmp file now... | 20260506 00:31:41.579 | INFO | RTMetrics done | 20260506 00:31:41.650 | INFO | Finished a round of Baz IO: Wrote 1.857 GiB in 3.224 seconds | 20260506 00:31:41.650 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.36% / 0.00% / 78.63% | 20260506 00:31:41.650 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.092264% / 3.97364e-06% / 6.75519e-05% | 20260506 00:31:41.650 | INFO | Baz IO overall: | 20260506 00:31:41.650 | INFO | BytesWritten : 4276982499501 | 20260506 00:31:41.650 | INFO | NumEvents : 2012502293647 | 20260506 00:31:43.445 | INFO | Analyzing chunk frames = [7532544,7533056) | 20260506 00:31:43.445 | INFO | MemInfo: { | 20260506 00:31:43.445 | INFO | "gpu" : " 17 GB", | 20260506 00:31:43.445 | INFO | "huge" : "", | 20260506 00:31:43.445 | INFO | "huge_shared" : " 27 GB", | 20260506 00:31:43.445 | INFO | "malloc" : " 14 GB", | 20260506 00:31:43.445 | INFO | "pinned" : " 29 GB" | 20260506 00:31:43.445 | INFO | } | 20260506 00:31:43.445 | INFO | Perf report for the last 5.14308 seconds: | 20260506 00:31:43.445 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:31:43.445 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:43.445 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:43.445 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:43.445 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260506 00:31:43.445 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:43.445 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260506 00:31:43.445 | INFO | PRE_HQ : 59.7%, 1.7, 53.7ms, 0 | 20260506 00:31:43.445 | INFO | BAZWRITER : 61.9%, 3.5, 111.2ms, 0 | 20260506 00:31:43.445 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:31:43.445 | INFO | Uploaded 36GiB to GPU in 2.16611s (16.6196GiB/s) | 20260506 00:31:43.445 | INFO | Downloaded 38GiB to GPU in 2.22542s (17.0754GiB/s) | 20260506 00:31:48.586 | INFO | Analyzing chunk frames = [7533056,7533568) | 20260506 00:31:48.586 | INFO | MemInfo: { | 20260506 00:31:48.586 | INFO | "gpu" : " 17 GB", | 20260506 00:31:48.586 | INFO | "huge" : "", | 20260506 00:31:48.586 | INFO | "huge_shared" : " 27 GB", | 20260506 00:31:48.586 | INFO | "malloc" : " 14 GB", | 20260506 00:31:48.586 | INFO | "pinned" : " 29 GB" | 20260506 00:31:48.586 | INFO | } | 20260506 00:31:48.586 | INFO | Perf report for the last 5.14085 seconds: | 20260506 00:31:48.586 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:31:48.586 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:48.586 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:48.586 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:48.586 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 00:31:48.586 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:48.586 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:48.586 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 00:31:48.586 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:48.586 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:48.586 | INFO | Uploaded 36GiB to GPU in 2.09123s (17.2148GiB/s) | 20260506 00:31:48.586 | INFO | Downloaded 34GiB to GPU in 2.03615s (16.6982GiB/s) | 20260506 00:31:53.736 | INFO | Analyzing chunk frames = [7533568,7534080) | 20260506 00:31:53.736 | INFO | MemInfo: { | 20260506 00:31:53.736 | INFO | "gpu" : " 17 GB", | 20260506 00:31:53.736 | INFO | "huge" : "", | 20260506 00:31:53.736 | INFO | "huge_shared" : " 27 GB", | 20260506 00:31:53.736 | INFO | "malloc" : " 14 GB", | 20260506 00:31:53.736 | INFO | "pinned" : " 29 GB" | 20260506 00:31:53.736 | INFO | } | 20260506 00:31:53.736 | INFO | Perf report for the last 5.15064 seconds: | 20260506 00:31:53.737 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:31:53.737 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:53.737 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:53.737 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:53.737 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 00:31:53.737 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:53.737 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:53.737 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:31:53.737 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:53.737 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:53.737 | INFO | Uploaded 36GiB to GPU in 2.08336s (17.2798GiB/s) | 20260506 00:31:53.737 | INFO | Downloaded 34GiB to GPU in 2.02164s (16.8181GiB/s) | 20260506 00:31:58.866 | INFO | Analyzing chunk frames = [7534080,7534592) | 20260506 00:31:58.866 | INFO | MemInfo: { | 20260506 00:31:58.866 | INFO | "gpu" : " 17 GB", | 20260506 00:31:58.866 | INFO | "huge" : "", | 20260506 00:31:58.866 | INFO | "huge_shared" : " 27 GB", | 20260506 00:31:58.866 | INFO | "malloc" : " 14 GB", | 20260506 00:31:58.866 | INFO | "pinned" : " 29 GB" | 20260506 00:31:58.866 | INFO | } | 20260506 00:31:58.866 | INFO | Perf report for the last 5.12927 seconds: | 20260506 00:31:58.866 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:31:58.866 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:58.866 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:58.866 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:58.866 | INFO | ANALYSIS : 61.7%, 3.0, 95.6ms, 0 | 20260506 00:31:58.866 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:58.866 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:58.866 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:31:58.866 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:58.866 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:31:58.866 | INFO | Uploaded 36GiB to GPU in 2.13849s (16.8343GiB/s) | 20260506 00:31:58.866 | INFO | Downloaded 34GiB to GPU in 2.05096s (16.5776GiB/s) | 20260506 00:32:04.001 | INFO | Analyzing chunk frames = [7534592,7535104) | 20260506 00:32:04.001 | INFO | MemInfo: { | 20260506 00:32:04.001 | INFO | "gpu" : " 17 GB", | 20260506 00:32:04.001 | INFO | "huge" : "", | 20260506 00:32:04.001 | INFO | "huge_shared" : " 27 GB", | 20260506 00:32:04.001 | INFO | "malloc" : " 14 GB", | 20260506 00:32:04.001 | INFO | "pinned" : " 29 GB" | 20260506 00:32:04.001 | INFO | } | 20260506 00:32:04.001 | INFO | Perf report for the last 5.13499 seconds: | 20260506 00:32:04.001 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:32:04.001 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:04.001 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:04.001 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:04.001 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 00:32:04.001 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:04.001 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:04.001 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 00:32:04.001 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:04.001 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:04.001 | INFO | Uploaded 36GiB to GPU in 2.09028s (17.2226GiB/s) | 20260506 00:32:04.001 | INFO | Downloaded 34GiB to GPU in 2.04773s (16.6038GiB/s) | 20260506 00:32:09.142 | INFO | Analyzing chunk frames = [7535104,7535616) | 20260506 00:32:09.142 | INFO | MemInfo: { | 20260506 00:32:09.142 | INFO | "gpu" : " 17 GB", | 20260506 00:32:09.142 | INFO | "huge" : "", | 20260506 00:32:09.142 | INFO | "huge_shared" : " 27 GB", | 20260506 00:32:09.142 | INFO | "malloc" : " 14 GB", | 20260506 00:32:09.142 | INFO | "pinned" : " 29 GB" | 20260506 00:32:09.142 | INFO | } | 20260506 00:32:09.142 | INFO | Perf report for the last 5.14089 seconds: | 20260506 00:32:09.142 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:32:09.142 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:09.142 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:09.142 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:09.142 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260506 00:32:09.142 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:09.142 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:09.142 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 00:32:09.142 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:09.142 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:09.142 | INFO | Uploaded 36GiB to GPU in 2.07161s (17.3778GiB/s) | 20260506 00:32:09.142 | INFO | Downloaded 34GiB to GPU in 2.03844s (16.6795GiB/s) | 20260506 00:32:14.283 | INFO | Analyzing chunk frames = [7535616,7536128) | 20260506 00:32:14.283 | INFO | MemInfo: { | 20260506 00:32:14.283 | INFO | "gpu" : " 17 GB", | 20260506 00:32:14.283 | INFO | "huge" : "", | 20260506 00:32:14.283 | INFO | "huge_shared" : " 27 GB", | 20260506 00:32:14.283 | INFO | "malloc" : " 14 GB", | 20260506 00:32:14.283 | INFO | "pinned" : " 29 GB" | 20260506 00:32:14.283 | INFO | } | 20260506 00:32:14.283 | INFO | Perf report for the last 5.14123 seconds: | 20260506 00:32:14.283 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:32:14.283 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:14.283 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:14.283 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:14.283 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260506 00:32:14.283 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:14.283 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:14.283 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 00:32:14.283 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:14.283 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:14.283 | INFO | Uploaded 36GiB to GPU in 2.06964s (17.3943GiB/s) | 20260506 00:32:14.283 | INFO | Downloaded 34GiB to GPU in 2.07512s (16.3846GiB/s) | 20260506 00:32:19.421 | INFO | Analyzing chunk frames = [7536128,7536640) | 20260506 00:32:19.421 | INFO | MemInfo: { | 20260506 00:32:19.421 | INFO | "gpu" : " 17 GB", | 20260506 00:32:19.421 | INFO | "huge" : "", | 20260506 00:32:19.421 | INFO | "huge_shared" : " 27 GB", | 20260506 00:32:19.421 | INFO | "malloc" : " 14 GB", | 20260506 00:32:19.421 | INFO | "pinned" : " 29 GB" | 20260506 00:32:19.421 | INFO | } | 20260506 00:32:19.421 | INFO | Perf report for the last 5.13798 seconds: | 20260506 00:32:19.421 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:32:19.421 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:19.421 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:19.421 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:19.421 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260506 00:32:19.421 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:19.421 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:19.421 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 00:32:19.421 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:19.421 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:19.421 | INFO | Uploaded 36GiB to GPU in 2.12578s (16.935GiB/s) | 20260506 00:32:19.421 | INFO | Downloaded 34GiB to GPU in 2.01739s (16.8535GiB/s) | 20260506 00:32:19.421 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 00:32:19.421 | INFO | Total mem usage: 15.3 GiB | 20260506 00:32:19.421 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 00:32:19.421 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.15%) | 20260506 00:32:19.421 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 00:32:19.421 | INFO | High Water Breakdown: | 20260506 00:32:19.421 | INFO | 60.9% : 8.55 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 00:32:19.421 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 00:32:19.421 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 00:32:19.421 | INFO | ------------------------------------------ | 20260506 00:32:19.421 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:32:19.422 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 00:32:19.422 | INFO | Total mem usage: 26.5 GiB | 20260506 00:32:19.422 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 00:32:19.422 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 00:32:19.422 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260506 00:32:19.422 | INFO | High Water Breakdown: | 20260506 00:32:19.422 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:32:19.422 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:32:19.422 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:32:19.422 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:32:19.422 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:32:19.422 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:32:19.422 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 00:32:19.422 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 00:32:19.422 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:32:19.422 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 00:32:19.422 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 00:32:19.422 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 00:32:19.422 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 00:32:19.422 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 00:32:19.422 | INFO | ------------------------------------------ | 20260506 00:32:19.422 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:32:19.422 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 00:32:19.422 | INFO | Total mem usage: 25.6 GiB | 20260506 00:32:19.422 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 00:32:19.422 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 00:32:19.422 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 00:32:19.422 | INFO | High Water Breakdown: | 20260506 00:32:19.422 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 00:32:19.422 | INFO | ------------------------------------------ | 20260506 00:32:19.422 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:32:19.422 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 00:32:19.422 | INFO | Total mem usage: 15.7 GiB | 20260506 00:32:19.422 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 00:32:19.422 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 00:32:19.422 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 00:32:19.422 | INFO | High Water Breakdown: | 20260506 00:32:19.422 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 00:32:19.422 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 00:32:19.422 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 00:32:19.422 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 00:32:19.422 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 00:32:19.422 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 00:32:19.422 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 00:32:19.422 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 00:32:19.422 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 00:32:19.422 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:32:19.422 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 00:32:19.422 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:32:19.422 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 00:32:19.422 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 00:32:19.422 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 00:32:19.422 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 00:32:19.422 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:32:19.422 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 00:32:19.422 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:32:19.422 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:32:19.422 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:32:19.422 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 00:32:19.422 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 00:32:19.422 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:32:19.422 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 00:32:19.422 | INFO | ------------------------------------------ | 20260506 00:32:19.422 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:32:19.422 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 00:32:19.422 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:32:19.422 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:32:19.422 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.38e+01, 1.34e+01 / 7.74e-01 | 20260506 00:32:19.422 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.04e-01, 1.06e-02 / 2.87e-03 | 20260506 00:32:19.422 | INFO | PerfReport: 11.84%, Baseline -- 3.70e+00 / 4.50e+00, 5.70e-01 / 1.84e-01 | 20260506 00:32:19.422 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.53e+00, 1.91e-01 / 8.13e-02 | 20260506 00:32:19.422 | INFO | PerfReport: 15.98%, DME -- 4.00e+01 / 4.38e+01, 3.85e+00 / 9.33e-01 | 20260506 00:32:19.422 | INFO | PerfReport: 57.34%, FrameLabeling -- 1.79e+01 / 1.95e+01, 1.44e+00 / 9.83e-01 | 20260506 00:32:19.422 | INFO | PerfReport: 7.11%, PulseAccumulating -- 2.22e+00 / 2.73e+00, 4.23e-01 / 2.09e-01 | 20260506 00:32:19.422 | INFO | PerfReport: 3.09%, Metrics -- 9.65e-01 / 1.26e+00, 2.61e-01 / 1.32e-01 | 20260506 00:32:19.423 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.33e-01 / 3.88e-01, 5.56e-02 / 2.62e-02 | 20260506 00:32:19.423 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 00:32:19.423 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:32:19.423 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:32:19.423 | INFO | PerfReport: Full Execution -- 9.82e+01 / -nan, 2.44e+01 / 0.00e+00 | 20260506 00:32:19.423 | INFO | PerfReport: 22.34%, Upload -- 2.19e+01 / 0.00e+00, 7.54e+00 / 0.00e+00 | 20260506 00:32:19.423 | INFO | PerfReport: 21.86%, Download -- 2.15e+01 / 0.00e+00, 7.61e+00 / 0.00e+00 | 20260506 00:32:19.423 | INFO | PerfReport: 31.84%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 00:32:19.423 | INFO | PerfReport: 2.11%, LockUpload -- 2.07e+00 / 0.00e+00, 7.88e+00 / 0.00e+00 | 20260506 00:32:19.423 | INFO | PerfReport: 1.31%, LockDownload -- 1.28e+00 / 0.00e+00, 2.87e+00 / 0.00e+00 | 20260506 00:32:19.423 | INFO | PerfReport: 20.53%, LockCompute -- 2.02e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 00:32:19.423 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.98e-02 / 0.00e+00 | 20260506 00:32:19.543 | INFO | Starting a round of Baz IO | 20260506 00:32:22.684 | INFO | Populating RTMetrics report | 20260506 00:32:22.684 | INFO | RTMetrics report populated | 20260506 00:32:22.684 | INFO | Opening tmp RTMetrics json file | 20260506 00:32:22.685 | INFO | Serializing RTMetrics to json | 20260506 00:32:22.686 | INFO | Serialization done, swapping tmp file now... | 20260506 00:32:22.686 | INFO | RTMetrics done | 20260506 00:32:22.759 | INFO | Finished a round of Baz IO: Wrote 1.856 GiB in 3.216 seconds | 20260506 00:32:22.759 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.36% / 0.00% / 78.64% | 20260506 00:32:22.759 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0917077% / 3.97364e-06% / 8.74201e-05% | 20260506 00:32:22.759 | INFO | Baz IO overall: | 20260506 00:32:22.759 | INFO | BytesWritten : 4278975486570 | 20260506 00:32:22.759 | INFO | NumEvents : 2013413498628 | 20260506 00:32:24.560 | INFO | Analyzing chunk frames = [7536640,7537152) | 20260506 00:32:24.560 | INFO | MemInfo: { | 20260506 00:32:24.560 | INFO | "gpu" : " 17 GB", | 20260506 00:32:24.560 | INFO | "huge" : "", | 20260506 00:32:24.560 | INFO | "huge_shared" : " 27 GB", | 20260506 00:32:24.560 | INFO | "malloc" : " 14 GB", | 20260506 00:32:24.560 | INFO | "pinned" : " 29 GB" | 20260506 00:32:24.560 | INFO | } | 20260506 00:32:24.561 | INFO | Perf report for the last 5.13959 seconds: | 20260506 00:32:24.561 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:32:24.561 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:24.561 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:24.561 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:24.561 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260506 00:32:24.561 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:24.561 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260506 00:32:24.561 | INFO | PRE_HQ : 59.8%, 1.7, 53.9ms, 0 | 20260506 00:32:24.561 | INFO | BAZWRITER : 61.7%, 3.5, 110.9ms, 0 | 20260506 00:32:24.561 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:32:24.561 | INFO | Uploaded 36GiB to GPU in 2.17181s (16.5761GiB/s) | 20260506 00:32:24.561 | INFO | Downloaded 38GiB to GPU in 2.24165s (16.9518GiB/s) | 20260506 00:32:29.702 | INFO | Analyzing chunk frames = [7537152,7537664) | 20260506 00:32:29.702 | INFO | MemInfo: { | 20260506 00:32:29.702 | INFO | "gpu" : " 17 GB", | 20260506 00:32:29.702 | INFO | "huge" : "", | 20260506 00:32:29.702 | INFO | "huge_shared" : " 27 GB", | 20260506 00:32:29.702 | INFO | "malloc" : " 14 GB", | 20260506 00:32:29.702 | INFO | "pinned" : " 29 GB" | 20260506 00:32:29.702 | INFO | } | 20260506 00:32:29.702 | INFO | Perf report for the last 5.14193 seconds: | 20260506 00:32:29.702 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:32:29.702 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:29.702 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:29.703 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:29.703 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 00:32:29.703 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:29.703 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:29.703 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 00:32:29.703 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:29.703 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:29.703 | INFO | Uploaded 36GiB to GPU in 2.09391s (17.1927GiB/s) | 20260506 00:32:29.703 | INFO | Downloaded 34GiB to GPU in 2.03495s (16.7081GiB/s) | 20260506 00:32:34.844 | INFO | Analyzing chunk frames = [7537664,7538176) | 20260506 00:32:34.844 | INFO | MemInfo: { | 20260506 00:32:34.844 | INFO | "gpu" : " 17 GB", | 20260506 00:32:34.844 | INFO | "huge" : "", | 20260506 00:32:34.844 | INFO | "huge_shared" : " 27 GB", | 20260506 00:32:34.844 | INFO | "malloc" : " 14 GB", | 20260506 00:32:34.844 | INFO | "pinned" : " 29 GB" | 20260506 00:32:34.844 | INFO | } | 20260506 00:32:34.845 | INFO | Perf report for the last 5.14206 seconds: | 20260506 00:32:34.845 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:32:34.845 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:34.845 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:34.845 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:34.845 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 00:32:34.845 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:34.845 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:34.845 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 00:32:34.845 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:34.845 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:34.845 | INFO | Uploaded 36GiB to GPU in 2.08532s (17.2635GiB/s) | 20260506 00:32:34.845 | INFO | Downloaded 34GiB to GPU in 2.02714s (16.7724GiB/s) | 20260506 00:32:39.979 | INFO | Analyzing chunk frames = [7538176,7538688) | 20260506 00:32:39.979 | INFO | MemInfo: { | 20260506 00:32:39.979 | INFO | "gpu" : " 17 GB", | 20260506 00:32:39.979 | INFO | "huge" : "", | 20260506 00:32:39.979 | INFO | "huge_shared" : " 27 GB", | 20260506 00:32:39.979 | INFO | "malloc" : " 14 GB", | 20260506 00:32:39.979 | INFO | "pinned" : " 29 GB" | 20260506 00:32:39.979 | INFO | } | 20260506 00:32:39.979 | INFO | Perf report for the last 5.13479 seconds: | 20260506 00:32:39.979 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:32:39.979 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:39.979 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:39.979 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:39.979 | INFO | ANALYSIS : 61.6%, 3.0, 95.6ms, 0 | 20260506 00:32:39.979 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:39.979 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:39.979 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 00:32:39.979 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:39.979 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:39.979 | INFO | Uploaded 36GiB to GPU in 2.14563s (16.7783GiB/s) | 20260506 00:32:39.980 | INFO | Downloaded 34GiB to GPU in 2.05263s (16.5641GiB/s) | 20260506 00:32:45.116 | INFO | Analyzing chunk frames = [7538688,7539200) | 20260506 00:32:45.116 | INFO | MemInfo: { | 20260506 00:32:45.116 | INFO | "gpu" : " 17 GB", | 20260506 00:32:45.116 | INFO | "huge" : "", | 20260506 00:32:45.116 | INFO | "huge_shared" : " 27 GB", | 20260506 00:32:45.116 | INFO | "malloc" : " 14 GB", | 20260506 00:32:45.116 | INFO | "pinned" : " 29 GB" | 20260506 00:32:45.116 | INFO | } | 20260506 00:32:45.116 | INFO | Perf report for the last 5.13656 seconds: | 20260506 00:32:45.116 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:32:45.116 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:45.116 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:45.116 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:45.116 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 00:32:45.116 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:45.116 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:45.116 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 00:32:45.116 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:45.116 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:45.116 | INFO | Uploaded 36GiB to GPU in 2.09105s (17.2162GiB/s) | 20260506 00:32:45.116 | INFO | Downloaded 34GiB to GPU in 2.04978s (16.5871GiB/s) | 20260506 00:32:50.251 | INFO | Analyzing chunk frames = [7539200,7539712) | 20260506 00:32:50.251 | INFO | MemInfo: { | 20260506 00:32:50.251 | INFO | "gpu" : " 17 GB", | 20260506 00:32:50.251 | INFO | "huge" : "", | 20260506 00:32:50.251 | INFO | "huge_shared" : " 27 GB", | 20260506 00:32:50.251 | INFO | "malloc" : " 14 GB", | 20260506 00:32:50.251 | INFO | "pinned" : " 29 GB" | 20260506 00:32:50.251 | INFO | } | 20260506 00:32:50.251 | INFO | Perf report for the last 5.13485 seconds: | 20260506 00:32:50.251 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:32:50.251 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:50.251 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:50.251 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:50.251 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 00:32:50.251 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:50.251 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:50.251 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:32:50.251 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:50.251 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:50.251 | INFO | Uploaded 36GiB to GPU in 2.07652s (17.3367GiB/s) | 20260506 00:32:50.251 | INFO | Downloaded 34GiB to GPU in 2.03687s (16.6922GiB/s) | 20260506 00:32:55.398 | INFO | Analyzing chunk frames = [7539712,7540224) | 20260506 00:32:55.398 | INFO | MemInfo: { | 20260506 00:32:55.398 | INFO | "gpu" : " 17 GB", | 20260506 00:32:55.398 | INFO | "huge" : "", | 20260506 00:32:55.398 | INFO | "huge_shared" : " 27 GB", | 20260506 00:32:55.398 | INFO | "malloc" : " 14 GB", | 20260506 00:32:55.398 | INFO | "pinned" : " 29 GB" | 20260506 00:32:55.398 | INFO | } | 20260506 00:32:55.399 | INFO | Perf report for the last 5.14773 seconds: | 20260506 00:32:55.399 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:32:55.399 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:55.399 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:55.399 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:55.399 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260506 00:32:55.399 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:55.399 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:55.399 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 00:32:55.399 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:55.399 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:32:55.399 | INFO | Uploaded 36GiB to GPU in 2.02899s (17.7429GiB/s) | 20260506 00:32:55.399 | INFO | Downloaded 34GiB to GPU in 2.04064s (16.6614GiB/s) | 20260506 00:33:00.539 | INFO | Analyzing chunk frames = [7540224,7540736) | 20260506 00:33:00.539 | INFO | MemInfo: { | 20260506 00:33:00.539 | INFO | "gpu" : " 17 GB", | 20260506 00:33:00.539 | INFO | "huge" : "", | 20260506 00:33:00.539 | INFO | "huge_shared" : " 27 GB", | 20260506 00:33:00.539 | INFO | "malloc" : " 14 GB", | 20260506 00:33:00.539 | INFO | "pinned" : " 29 GB" | 20260506 00:33:00.539 | INFO | } | 20260506 00:33:00.539 | INFO | Perf report for the last 5.14084 seconds: | 20260506 00:33:00.539 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:33:00.539 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:00.539 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:00.539 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:00.539 | INFO | ANALYSIS : 61.8%, 3.0, 96.6ms, 0 | 20260506 00:33:00.539 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:00.539 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:00.539 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:33:00.539 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:00.539 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:00.540 | INFO | Uploaded 36GiB to GPU in 2.12799s (16.9174GiB/s) | 20260506 00:33:00.540 | INFO | Downloaded 34GiB to GPU in 2.02546s (16.7863GiB/s) | 20260506 00:33:00.657 | INFO | Starting a round of Baz IO | 20260506 00:33:03.809 | INFO | Populating RTMetrics report | 20260506 00:33:03.809 | INFO | RTMetrics report populated | 20260506 00:33:03.810 | INFO | Opening tmp RTMetrics json file | 20260506 00:33:03.810 | INFO | Serializing RTMetrics to json | 20260506 00:33:03.814 | INFO | Serialization done, swapping tmp file now... | 20260506 00:33:03.815 | INFO | RTMetrics done | 20260506 00:33:03.886 | INFO | Finished a round of Baz IO: Wrote 1.855 GiB in 3.229 seconds | 20260506 00:33:03.886 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.36% / 0.00% / 78.64% | 20260506 00:33:03.886 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0916203% / 3.97364e-06% / 9.93411e-05% | 20260506 00:33:03.886 | INFO | Baz IO overall: | 20260506 00:33:03.886 | INFO | BytesWritten : 4280967336275 | 20260506 00:33:03.886 | INFO | NumEvents : 2014323523061 | 20260506 00:33:05.684 | INFO | Analyzing chunk frames = [7540736,7541248) | 20260506 00:33:05.684 | INFO | MemInfo: { | 20260506 00:33:05.684 | INFO | "gpu" : " 17 GB", | 20260506 00:33:05.684 | INFO | "huge" : "", | 20260506 00:33:05.684 | INFO | "huge_shared" : " 27 GB", | 20260506 00:33:05.684 | INFO | "malloc" : " 14 GB", | 20260506 00:33:05.684 | INFO | "pinned" : " 29 GB" | 20260506 00:33:05.684 | INFO | } | 20260506 00:33:05.684 | INFO | Perf report for the last 5.14428 seconds: | 20260506 00:33:05.684 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:33:05.684 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:05.684 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:05.684 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:05.684 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260506 00:33:05.684 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:05.684 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260506 00:33:05.684 | INFO | PRE_HQ : 59.7%, 1.7, 53.8ms, 0 | 20260506 00:33:05.684 | INFO | BAZWRITER : 61.9%, 3.5, 111.0ms, 0 | 20260506 00:33:05.684 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:33:05.684 | INFO | Uploaded 36GiB to GPU in 2.14998s (16.7444GiB/s) | 20260506 00:33:05.684 | INFO | Downloaded 38GiB to GPU in 2.23943s (16.9686GiB/s) | 20260506 00:33:10.817 | INFO | Analyzing chunk frames = [7541248,7541760) | 20260506 00:33:10.817 | INFO | MemInfo: { | 20260506 00:33:10.817 | INFO | "gpu" : " 17 GB", | 20260506 00:33:10.817 | INFO | "huge" : "", | 20260506 00:33:10.817 | INFO | "huge_shared" : " 27 GB", | 20260506 00:33:10.817 | INFO | "malloc" : " 14 GB", | 20260506 00:33:10.817 | INFO | "pinned" : " 29 GB" | 20260506 00:33:10.817 | INFO | } | 20260506 00:33:10.817 | INFO | Perf report for the last 5.13335 seconds: | 20260506 00:33:10.817 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:33:10.817 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:10.817 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:10.817 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:10.817 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 00:33:10.817 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:10.817 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:10.817 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 00:33:10.817 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:10.817 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:10.817 | INFO | Uploaded 36GiB to GPU in 2.12134s (16.9704GiB/s) | 20260506 00:33:10.817 | INFO | Downloaded 34GiB to GPU in 2.0418s (16.652GiB/s) | 20260506 00:33:15.956 | INFO | Analyzing chunk frames = [7541760,7542272) | 20260506 00:33:15.956 | INFO | MemInfo: { | 20260506 00:33:15.956 | INFO | "gpu" : " 17 GB", | 20260506 00:33:15.956 | INFO | "huge" : "", | 20260506 00:33:15.956 | INFO | "huge_shared" : " 27 GB", | 20260506 00:33:15.956 | INFO | "malloc" : " 14 GB", | 20260506 00:33:15.956 | INFO | "pinned" : " 29 GB" | 20260506 00:33:15.956 | INFO | } | 20260506 00:33:15.957 | INFO | Perf report for the last 5.13948 seconds: | 20260506 00:33:15.957 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:33:15.957 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:15.957 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:15.957 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:15.957 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 00:33:15.957 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:15.957 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:15.957 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:33:15.957 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:15.957 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:15.957 | INFO | Uploaded 36GiB to GPU in 2.09862s (17.1541GiB/s) | 20260506 00:33:15.957 | INFO | Downloaded 34GiB to GPU in 2.03099s (16.7406GiB/s) | 20260506 00:33:21.092 | INFO | Analyzing chunk frames = [7542272,7542784) | 20260506 00:33:21.092 | INFO | MemInfo: { | 20260506 00:33:21.092 | INFO | "gpu" : " 17 GB", | 20260506 00:33:21.092 | INFO | "huge" : "", | 20260506 00:33:21.092 | INFO | "huge_shared" : " 27 GB", | 20260506 00:33:21.092 | INFO | "malloc" : " 14 GB", | 20260506 00:33:21.092 | INFO | "pinned" : " 29 GB" | 20260506 00:33:21.092 | INFO | } | 20260506 00:33:21.092 | INFO | Perf report for the last 5.13577 seconds: | 20260506 00:33:21.092 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:33:21.092 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:21.092 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:21.092 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:21.092 | INFO | ANALYSIS : 61.6%, 3.0, 95.6ms, 0 | 20260506 00:33:21.092 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:21.092 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:21.092 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 00:33:21.092 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:21.092 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:21.093 | INFO | Uploaded 36GiB to GPU in 2.15612s (16.6967GiB/s) | 20260506 00:33:21.093 | INFO | Downloaded 34GiB to GPU in 2.05779s (16.5226GiB/s) | 20260506 00:33:26.232 | INFO | Analyzing chunk frames = [7542784,7543296) | 20260506 00:33:26.232 | INFO | MemInfo: { | 20260506 00:33:26.232 | INFO | "gpu" : " 17 GB", | 20260506 00:33:26.232 | INFO | "huge" : "", | 20260506 00:33:26.232 | INFO | "huge_shared" : " 27 GB", | 20260506 00:33:26.232 | INFO | "malloc" : " 14 GB", | 20260506 00:33:26.232 | INFO | "pinned" : " 29 GB" | 20260506 00:33:26.232 | INFO | } | 20260506 00:33:26.232 | INFO | Perf report for the last 5.13938 seconds: | 20260506 00:33:26.232 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:33:26.232 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:26.232 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:26.232 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:26.232 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 00:33:26.232 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:26.232 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:26.232 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:33:26.232 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:26.232 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:26.232 | INFO | Uploaded 36GiB to GPU in 2.09069s (17.2192GiB/s) | 20260506 00:33:26.232 | INFO | Downloaded 34GiB to GPU in 2.0422s (16.6487GiB/s) | 20260506 00:33:31.373 | INFO | Analyzing chunk frames = [7543296,7543808) | 20260506 00:33:31.373 | INFO | MemInfo: { | 20260506 00:33:31.373 | INFO | "gpu" : " 17 GB", | 20260506 00:33:31.373 | INFO | "huge" : "", | 20260506 00:33:31.373 | INFO | "huge_shared" : " 27 GB", | 20260506 00:33:31.373 | INFO | "malloc" : " 14 GB", | 20260506 00:33:31.373 | INFO | "pinned" : " 29 GB" | 20260506 00:33:31.373 | INFO | } | 20260506 00:33:31.373 | INFO | Perf report for the last 5.14155 seconds: | 20260506 00:33:31.373 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:33:31.373 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:31.373 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:31.373 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:31.373 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260506 00:33:31.373 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:31.373 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:31.373 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 00:33:31.373 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:31.373 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:31.374 | INFO | Uploaded 36GiB to GPU in 2.07718s (17.3312GiB/s) | 20260506 00:33:31.374 | INFO | Downloaded 34GiB to GPU in 2.03513s (16.7065GiB/s) | 20260506 00:33:36.509 | INFO | Analyzing chunk frames = [7543808,7544320) | 20260506 00:33:36.509 | INFO | MemInfo: { | 20260506 00:33:36.509 | INFO | "gpu" : " 17 GB", | 20260506 00:33:36.509 | INFO | "huge" : "", | 20260506 00:33:36.509 | INFO | "huge_shared" : " 27 GB", | 20260506 00:33:36.509 | INFO | "malloc" : " 14 GB", | 20260506 00:33:36.509 | INFO | "pinned" : " 29 GB" | 20260506 00:33:36.509 | INFO | } | 20260506 00:33:36.509 | INFO | Perf report for the last 5.13545 seconds: | 20260506 00:33:36.509 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:33:36.509 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:36.509 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:36.509 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:36.509 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260506 00:33:36.509 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:36.509 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:36.509 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 00:33:36.509 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:36.509 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:36.509 | INFO | Uploaded 36GiB to GPU in 2.0598s (17.4775GiB/s) | 20260506 00:33:36.509 | INFO | Downloaded 34GiB to GPU in 2.06127s (16.4947GiB/s) | 20260506 00:33:41.652 | INFO | Analyzing chunk frames = [7544320,7544832) | 20260506 00:33:41.652 | INFO | MemInfo: { | 20260506 00:33:41.652 | INFO | "gpu" : " 17 GB", | 20260506 00:33:41.652 | INFO | "huge" : "", | 20260506 00:33:41.652 | INFO | "huge_shared" : " 27 GB", | 20260506 00:33:41.652 | INFO | "malloc" : " 14 GB", | 20260506 00:33:41.652 | INFO | "pinned" : " 29 GB" | 20260506 00:33:41.652 | INFO | } | 20260506 00:33:41.652 | INFO | Perf report for the last 5.14297 seconds: | 20260506 00:33:41.652 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:33:41.652 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:41.652 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:41.652 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:41.652 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260506 00:33:41.652 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:41.652 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:41.652 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 00:33:41.652 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:41.652 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:41.652 | INFO | Uploaded 36GiB to GPU in 2.11728s (17.003GiB/s) | 20260506 00:33:41.652 | INFO | Downloaded 34GiB to GPU in 2.00381s (16.9677GiB/s) | 20260506 00:33:41.652 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 00:33:41.652 | INFO | Total mem usage: 15.3 GiB | 20260506 00:33:41.652 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 00:33:41.652 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.15%) | 20260506 00:33:41.652 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 00:33:41.652 | INFO | High Water Breakdown: | 20260506 00:33:41.652 | INFO | 60.9% : 8.55 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 00:33:41.652 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 00:33:41.652 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 00:33:41.652 | INFO | ------------------------------------------ | 20260506 00:33:41.652 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:33:41.652 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 00:33:41.652 | INFO | Total mem usage: 26.5 GiB | 20260506 00:33:41.652 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 00:33:41.652 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 00:33:41.652 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260506 00:33:41.652 | INFO | High Water Breakdown: | 20260506 00:33:41.652 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:33:41.652 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:33:41.652 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:33:41.652 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:33:41.652 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:33:41.652 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:33:41.652 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 00:33:41.652 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 00:33:41.652 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:33:41.652 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 00:33:41.652 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 00:33:41.652 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 00:33:41.652 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 00:33:41.652 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 00:33:41.652 | INFO | ------------------------------------------ | 20260506 00:33:41.652 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:33:41.652 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 00:33:41.652 | INFO | Total mem usage: 25.6 GiB | 20260506 00:33:41.652 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 00:33:41.652 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 00:33:41.652 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 00:33:41.652 | INFO | High Water Breakdown: | 20260506 00:33:41.652 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 00:33:41.652 | INFO | ------------------------------------------ | 20260506 00:33:41.652 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:33:41.652 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 00:33:41.652 | INFO | Total mem usage: 15.7 GiB | 20260506 00:33:41.652 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 00:33:41.652 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 00:33:41.652 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 00:33:41.652 | INFO | High Water Breakdown: | 20260506 00:33:41.652 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 00:33:41.652 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 00:33:41.652 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 00:33:41.652 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 00:33:41.652 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 00:33:41.652 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 00:33:41.652 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 00:33:41.652 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 00:33:41.653 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 00:33:41.653 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:33:41.653 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 00:33:41.653 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:33:41.653 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 00:33:41.653 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 00:33:41.653 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 00:33:41.653 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 00:33:41.653 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:33:41.653 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 00:33:41.653 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:33:41.653 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:33:41.653 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:33:41.653 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 00:33:41.653 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 00:33:41.653 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:33:41.653 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 00:33:41.653 | INFO | ------------------------------------------ | 20260506 00:33:41.653 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:33:41.653 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 00:33:41.653 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:33:41.653 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:33:41.653 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.40e+01, 1.34e+01 / 9.00e-01 | 20260506 00:33:41.653 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.03e-01, 1.06e-02 / 2.50e-03 | 20260506 00:33:41.653 | INFO | PerfReport: 11.81%, Baseline -- 3.70e+00 / 4.48e+00, 5.66e-01 / 2.18e-01 | 20260506 00:33:41.653 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.53e+00, 1.90e-01 / 7.56e-02 | 20260506 00:33:41.653 | INFO | PerfReport: 16.00%, DME -- 4.01e+01 / 4.40e+01, 3.86e+00 / 9.63e-01 | 20260506 00:33:41.653 | INFO | PerfReport: 57.39%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.50e+00 / 1.02e+00 | 20260506 00:33:41.653 | INFO | PerfReport: 7.08%, PulseAccumulating -- 2.22e+00 / 2.75e+00, 4.21e-01 / 2.22e-01 | 20260506 00:33:41.653 | INFO | PerfReport: 3.08%, Metrics -- 9.63e-01 / 1.26e+00, 2.63e-01 / 1.41e-01 | 20260506 00:33:41.653 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.34e-01 / 3.84e-01, 5.71e-02 / 2.91e-02 | 20260506 00:33:41.653 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 00:33:41.653 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:33:41.653 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:33:41.653 | INFO | PerfReport: Full Execution -- 9.83e+01 / -nan, 2.46e+01 / 0.00e+00 | 20260506 00:33:41.653 | INFO | PerfReport: 22.32%, Upload -- 2.19e+01 / 0.00e+00, 7.51e+00 / 0.00e+00 | 20260506 00:33:41.653 | INFO | PerfReport: 21.88%, Download -- 2.15e+01 / 0.00e+00, 7.64e+00 / 0.00e+00 | 20260506 00:33:41.653 | INFO | PerfReport: 31.86%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 00:33:41.653 | INFO | PerfReport: 2.10%, LockUpload -- 2.06e+00 / 0.00e+00, 7.85e+00 / 0.00e+00 | 20260506 00:33:41.653 | INFO | PerfReport: 1.31%, LockDownload -- 1.29e+00 / 0.00e+00, 2.93e+00 / 0.00e+00 | 20260506 00:33:41.653 | INFO | PerfReport: 20.52%, LockCompute -- 2.02e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 00:33:41.653 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.73e-02 / 0.00e+00 | 20260506 00:33:41.768 | INFO | Starting a round of Baz IO | 20260506 00:33:44.916 | INFO | Populating RTMetrics report | 20260506 00:33:44.916 | INFO | RTMetrics report populated | 20260506 00:33:44.916 | INFO | Opening tmp RTMetrics json file | 20260506 00:33:44.917 | INFO | Serializing RTMetrics to json | 20260506 00:33:44.921 | INFO | Serialization done, swapping tmp file now... | 20260506 00:33:44.921 | INFO | RTMetrics done | 20260506 00:33:44.999 | INFO | Finished a round of Baz IO: Wrote 1.853 GiB in 3.23 seconds | 20260506 00:33:44.999 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.36% / 0.00% / 78.64% | 20260506 00:33:44.999 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.090452% / 0% / 7.94729e-05% | 20260506 00:33:44.999 | INFO | Baz IO overall: | 20260506 00:33:44.999 | INFO | BytesWritten : 4282956587439 | 20260506 00:33:44.999 | INFO | NumEvents : 2015232052377 | 20260506 00:33:46.796 | INFO | Analyzing chunk frames = [7544832,7545344) | 20260506 00:33:46.796 | INFO | MemInfo: { | 20260506 00:33:46.796 | INFO | "gpu" : " 17 GB", | 20260506 00:33:46.796 | INFO | "huge" : "", | 20260506 00:33:46.796 | INFO | "huge_shared" : " 27 GB", | 20260506 00:33:46.796 | INFO | "malloc" : " 14 GB", | 20260506 00:33:46.796 | INFO | "pinned" : " 29 GB" | 20260506 00:33:46.796 | INFO | } | 20260506 00:33:46.796 | INFO | Perf report for the last 5.1447 seconds: | 20260506 00:33:46.796 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:33:46.796 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:46.796 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:46.796 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:46.796 | INFO | ANALYSIS : 62.4%, 3.0, 97.6ms, 0 | 20260506 00:33:46.796 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:46.797 | INFO | RT_METRICS : 43.4%, 1.0, 22.8ms, 0 | 20260506 00:33:46.797 | INFO | PRE_HQ : 59.7%, 1.7, 54.2ms, 0 | 20260506 00:33:46.797 | INFO | BAZWRITER : 61.9%, 3.5, 111.5ms, 0 | 20260506 00:33:46.797 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:33:46.797 | INFO | Uploaded 36GiB to GPU in 2.16677s (16.6146GiB/s) | 20260506 00:33:46.797 | INFO | Downloaded 38GiB to GPU in 2.23684s (16.9883GiB/s) | 20260506 00:33:51.928 | INFO | Analyzing chunk frames = [7545344,7545856) | 20260506 00:33:51.928 | INFO | MemInfo: { | 20260506 00:33:51.928 | INFO | "gpu" : " 17 GB", | 20260506 00:33:51.928 | INFO | "huge" : "", | 20260506 00:33:51.928 | INFO | "huge_shared" : " 27 GB", | 20260506 00:33:51.928 | INFO | "malloc" : " 14 GB", | 20260506 00:33:51.928 | INFO | "pinned" : " 29 GB" | 20260506 00:33:51.928 | INFO | } | 20260506 00:33:51.928 | INFO | Perf report for the last 5.13166 seconds: | 20260506 00:33:51.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:33:51.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:51.928 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:51.928 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:51.928 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 00:33:51.928 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:51.928 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:51.928 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 00:33:51.928 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:51.928 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:51.928 | INFO | Uploaded 36GiB to GPU in 1.9764s (18.2149GiB/s) | 20260506 00:33:51.929 | INFO | Downloaded 34GiB to GPU in 1.99068s (17.0796GiB/s) | 20260506 00:33:57.064 | INFO | Analyzing chunk frames = [7545856,7546368) | 20260506 00:33:57.064 | INFO | MemInfo: { | 20260506 00:33:57.064 | INFO | "gpu" : " 17 GB", | 20260506 00:33:57.064 | INFO | "huge" : "", | 20260506 00:33:57.064 | INFO | "huge_shared" : " 27 GB", | 20260506 00:33:57.064 | INFO | "malloc" : " 14 GB", | 20260506 00:33:57.064 | INFO | "pinned" : " 29 GB" | 20260506 00:33:57.064 | INFO | } | 20260506 00:33:57.064 | INFO | Perf report for the last 5.13596 seconds: | 20260506 00:33:57.064 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:33:57.064 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:57.064 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:57.064 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:57.064 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 00:33:57.064 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:57.064 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:57.064 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 00:33:57.064 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:57.064 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:33:57.064 | INFO | Uploaded 36GiB to GPU in 2.07308s (17.3655GiB/s) | 20260506 00:33:57.064 | INFO | Downloaded 34GiB to GPU in 2.02237s (16.812GiB/s) | 20260506 00:34:02.208 | INFO | Analyzing chunk frames = [7546368,7546880) | 20260506 00:34:02.208 | INFO | MemInfo: { | 20260506 00:34:02.208 | INFO | "gpu" : " 17 GB", | 20260506 00:34:02.208 | INFO | "huge" : "", | 20260506 00:34:02.208 | INFO | "huge_shared" : " 27 GB", | 20260506 00:34:02.208 | INFO | "malloc" : " 14 GB", | 20260506 00:34:02.208 | INFO | "pinned" : " 29 GB" | 20260506 00:34:02.208 | INFO | } | 20260506 00:34:02.208 | INFO | Perf report for the last 5.14386 seconds: | 20260506 00:34:02.208 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:34:02.208 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:02.208 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:02.208 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:02.208 | INFO | ANALYSIS : 61.6%, 3.0, 95.7ms, 0 | 20260506 00:34:02.208 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:02.208 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:02.208 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:34:02.208 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:02.208 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:02.208 | INFO | Uploaded 36GiB to GPU in 2.13021s (16.8998GiB/s) | 20260506 00:34:02.208 | INFO | Downloaded 34GiB to GPU in 2.01872s (16.8424GiB/s) | 20260506 00:34:07.347 | INFO | Analyzing chunk frames = [7546880,7547392) | 20260506 00:34:07.347 | INFO | MemInfo: { | 20260506 00:34:07.347 | INFO | "gpu" : " 17 GB", | 20260506 00:34:07.347 | INFO | "huge" : "", | 20260506 00:34:07.347 | INFO | "huge_shared" : " 27 GB", | 20260506 00:34:07.347 | INFO | "malloc" : " 14 GB", | 20260506 00:34:07.347 | INFO | "pinned" : " 29 GB" | 20260506 00:34:07.347 | INFO | } | 20260506 00:34:07.347 | INFO | Perf report for the last 5.1389 seconds: | 20260506 00:34:07.347 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:34:07.347 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:07.347 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:07.347 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:07.347 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 00:34:07.347 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:07.347 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:07.347 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 00:34:07.347 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:07.347 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:07.347 | INFO | Uploaded 36GiB to GPU in 2.09225s (17.2064GiB/s) | 20260506 00:34:07.347 | INFO | Downloaded 34GiB to GPU in 2.04784s (16.6029GiB/s) | 20260506 00:34:12.493 | INFO | Analyzing chunk frames = [7547392,7547904) | 20260506 00:34:12.493 | INFO | MemInfo: { | 20260506 00:34:12.493 | INFO | "gpu" : " 17 GB", | 20260506 00:34:12.493 | INFO | "huge" : "", | 20260506 00:34:12.493 | INFO | "huge_shared" : " 27 GB", | 20260506 00:34:12.493 | INFO | "malloc" : " 14 GB", | 20260506 00:34:12.493 | INFO | "pinned" : " 29 GB" | 20260506 00:34:12.493 | INFO | } | 20260506 00:34:12.493 | INFO | Perf report for the last 5.14616 seconds: | 20260506 00:34:12.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:34:12.493 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:12.493 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:12.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:12.493 | INFO | ANALYSIS : 61.1%, 3.0, 95.5ms, 0 | 20260506 00:34:12.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:12.493 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:12.493 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 00:34:12.493 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:12.493 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:12.493 | INFO | Uploaded 36GiB to GPU in 2.07465s (17.3524GiB/s) | 20260506 00:34:12.493 | INFO | Downloaded 34GiB to GPU in 2.04229s (16.648GiB/s) | 20260506 00:34:17.632 | INFO | Analyzing chunk frames = [7547904,7548416) | 20260506 00:34:17.632 | INFO | MemInfo: { | 20260506 00:34:17.632 | INFO | "gpu" : " 17 GB", | 20260506 00:34:17.632 | INFO | "huge" : "", | 20260506 00:34:17.632 | INFO | "huge_shared" : " 27 GB", | 20260506 00:34:17.632 | INFO | "malloc" : " 14 GB", | 20260506 00:34:17.632 | INFO | "pinned" : " 29 GB" | 20260506 00:34:17.632 | INFO | } | 20260506 00:34:17.632 | INFO | Perf report for the last 5.13933 seconds: | 20260506 00:34:17.633 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:34:17.633 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:17.633 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:17.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:17.633 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260506 00:34:17.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:17.633 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:17.633 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 00:34:17.633 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:17.633 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:17.633 | INFO | Uploaded 36GiB to GPU in 2.15103s (16.7362GiB/s) | 20260506 00:34:17.633 | INFO | Downloaded 34GiB to GPU in 2.0956s (16.2245GiB/s) | 20260506 00:34:22.765 | INFO | Analyzing chunk frames = [7548416,7548928) | 20260506 00:34:22.765 | INFO | MemInfo: { | 20260506 00:34:22.765 | INFO | "gpu" : " 17 GB", | 20260506 00:34:22.765 | INFO | "huge" : "", | 20260506 00:34:22.765 | INFO | "huge_shared" : " 27 GB", | 20260506 00:34:22.765 | INFO | "malloc" : " 14 GB", | 20260506 00:34:22.765 | INFO | "pinned" : " 29 GB" | 20260506 00:34:22.765 | INFO | } | 20260506 00:34:22.765 | INFO | Perf report for the last 5.13254 seconds: | 20260506 00:34:22.765 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:34:22.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:22.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:22.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:22.765 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260506 00:34:22.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:22.765 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:22.765 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 00:34:22.765 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:22.765 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:22.765 | INFO | Uploaded 36GiB to GPU in 2.11294s (17.0379GiB/s) | 20260506 00:34:22.765 | INFO | Downloaded 34GiB to GPU in 2.0029s (16.9754GiB/s) | 20260506 00:34:22.883 | INFO | Starting a round of Baz IO | 20260506 00:34:26.026 | INFO | Populating RTMetrics report | 20260506 00:34:26.027 | INFO | RTMetrics report populated | 20260506 00:34:26.027 | INFO | Opening tmp RTMetrics json file | 20260506 00:34:26.028 | INFO | Serializing RTMetrics to json | 20260506 00:34:26.031 | INFO | Serialization done, swapping tmp file now... | 20260506 00:34:26.031 | INFO | RTMetrics done | 20260506 00:34:26.105 | INFO | Finished a round of Baz IO: Wrote 1.854 GiB in 3.222 seconds | 20260506 00:34:26.105 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.36% / 0.00% / 78.64% | 20260506 00:34:26.105 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0914852% / 0% / 0.000115236% | 20260506 00:34:26.106 | INFO | Baz IO overall: | 20260506 00:34:26.106 | INFO | BytesWritten : 4284947219697 | 20260506 00:34:26.106 | INFO | NumEvents : 2016142051601 | 20260506 00:34:27.910 | INFO | Analyzing chunk frames = [7548928,7549440) | 20260506 00:34:27.910 | INFO | MemInfo: { | 20260506 00:34:27.910 | INFO | "gpu" : " 17 GB", | 20260506 00:34:27.910 | INFO | "huge" : "", | 20260506 00:34:27.910 | INFO | "huge_shared" : " 27 GB", | 20260506 00:34:27.910 | INFO | "malloc" : " 14 GB", | 20260506 00:34:27.910 | INFO | "pinned" : " 29 GB" | 20260506 00:34:27.910 | INFO | } | 20260506 00:34:27.910 | INFO | Perf report for the last 5.14531 seconds: | 20260506 00:34:27.910 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:34:27.910 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:27.910 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:27.910 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:27.910 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260506 00:34:27.910 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:27.910 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260506 00:34:27.910 | INFO | PRE_HQ : 59.8%, 1.7, 54.2ms, 0 | 20260506 00:34:27.910 | INFO | BAZWRITER : 61.8%, 3.5, 110.5ms, 0 | 20260506 00:34:27.910 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:34:27.910 | INFO | Uploaded 36GiB to GPU in 2.1685s (16.6014GiB/s) | 20260506 00:34:27.911 | INFO | Downloaded 38GiB to GPU in 2.24035s (16.9616GiB/s) | 20260506 00:34:33.048 | INFO | Analyzing chunk frames = [7549440,7549952) | 20260506 00:34:33.048 | INFO | MemInfo: { | 20260506 00:34:33.048 | INFO | "gpu" : " 17 GB", | 20260506 00:34:33.048 | INFO | "huge" : "", | 20260506 00:34:33.048 | INFO | "huge_shared" : " 27 GB", | 20260506 00:34:33.048 | INFO | "malloc" : " 14 GB", | 20260506 00:34:33.048 | INFO | "pinned" : " 29 GB" | 20260506 00:34:33.048 | INFO | } | 20260506 00:34:33.048 | INFO | Perf report for the last 5.13822 seconds: | 20260506 00:34:33.048 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:34:33.048 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:33.048 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:33.049 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:33.049 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 00:34:33.049 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:33.049 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:33.049 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:34:33.049 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:33.049 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:33.049 | INFO | Uploaded 36GiB to GPU in 1.9903s (18.0877GiB/s) | 20260506 00:34:33.049 | INFO | Downloaded 34GiB to GPU in 1.9936s (17.0546GiB/s) | 20260506 00:34:38.182 | INFO | Analyzing chunk frames = [7549952,7550464) | 20260506 00:34:38.182 | INFO | MemInfo: { | 20260506 00:34:38.182 | INFO | "gpu" : " 17 GB", | 20260506 00:34:38.182 | INFO | "huge" : "", | 20260506 00:34:38.182 | INFO | "huge_shared" : " 27 GB", | 20260506 00:34:38.182 | INFO | "malloc" : " 14 GB", | 20260506 00:34:38.182 | INFO | "pinned" : " 29 GB" | 20260506 00:34:38.182 | INFO | } | 20260506 00:34:38.182 | INFO | Perf report for the last 5.1335 seconds: | 20260506 00:34:38.182 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:34:38.182 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:38.182 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:38.182 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:38.182 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 00:34:38.182 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:38.182 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:38.182 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:34:38.182 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:38.182 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:38.182 | INFO | Uploaded 36GiB to GPU in 2.08805s (17.2409GiB/s) | 20260506 00:34:38.182 | INFO | Downloaded 34GiB to GPU in 2.02759s (16.7687GiB/s) | 20260506 00:34:43.320 | INFO | Analyzing chunk frames = [7550464,7550976) | 20260506 00:34:43.320 | INFO | MemInfo: { | 20260506 00:34:43.320 | INFO | "gpu" : " 17 GB", | 20260506 00:34:43.320 | INFO | "huge" : "", | 20260506 00:34:43.320 | INFO | "huge_shared" : " 27 GB", | 20260506 00:34:43.320 | INFO | "malloc" : " 14 GB", | 20260506 00:34:43.320 | INFO | "pinned" : " 29 GB" | 20260506 00:34:43.320 | INFO | } | 20260506 00:34:43.320 | INFO | Perf report for the last 5.13803 seconds: | 20260506 00:34:43.320 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:34:43.320 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:43.320 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:43.320 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:43.320 | INFO | ANALYSIS : 61.7%, 3.0, 95.7ms, 0 | 20260506 00:34:43.320 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:43.320 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:43.320 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 00:34:43.320 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:43.320 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:43.320 | INFO | Uploaded 36GiB to GPU in 2.13619s (16.8524GiB/s) | 20260506 00:34:43.320 | INFO | Downloaded 34GiB to GPU in 2.02602s (16.7817GiB/s) | 20260506 00:34:48.463 | INFO | Analyzing chunk frames = [7550976,7551488) | 20260506 00:34:48.463 | INFO | MemInfo: { | 20260506 00:34:48.463 | INFO | "gpu" : " 17 GB", | 20260506 00:34:48.463 | INFO | "huge" : "", | 20260506 00:34:48.463 | INFO | "huge_shared" : " 27 GB", | 20260506 00:34:48.463 | INFO | "malloc" : " 14 GB", | 20260506 00:34:48.463 | INFO | "pinned" : " 29 GB" | 20260506 00:34:48.463 | INFO | } | 20260506 00:34:48.463 | INFO | Perf report for the last 5.14304 seconds: | 20260506 00:34:48.463 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:34:48.463 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:48.463 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:48.463 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:48.463 | INFO | ANALYSIS : 61.1%, 3.0, 95.5ms, 0 | 20260506 00:34:48.463 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:48.463 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:48.463 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 00:34:48.463 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:48.463 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:48.463 | INFO | Uploaded 36GiB to GPU in 2.10267s (17.1211GiB/s) | 20260506 00:34:48.463 | INFO | Downloaded 34GiB to GPU in 2.05423s (16.5512GiB/s) | 20260506 00:34:53.607 | INFO | Analyzing chunk frames = [7551488,7552000) | 20260506 00:34:53.607 | INFO | MemInfo: { | 20260506 00:34:53.607 | INFO | "gpu" : " 17 GB", | 20260506 00:34:53.607 | INFO | "huge" : "", | 20260506 00:34:53.608 | INFO | "huge_shared" : " 27 GB", | 20260506 00:34:53.608 | INFO | "malloc" : " 14 GB", | 20260506 00:34:53.608 | INFO | "pinned" : " 29 GB" | 20260506 00:34:53.608 | INFO | } | 20260506 00:34:53.608 | INFO | Perf report for the last 5.14447 seconds: | 20260506 00:34:53.608 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:34:53.608 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:53.608 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:53.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:53.608 | INFO | ANALYSIS : 61.3%, 3.0, 95.8ms, 0 | 20260506 00:34:53.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:53.608 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:53.608 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:34:53.608 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:53.608 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:53.608 | INFO | Uploaded 36GiB to GPU in 2.07513s (17.3483GiB/s) | 20260506 00:34:53.608 | INFO | Downloaded 34GiB to GPU in 2.04297s (16.6425GiB/s) | 20260506 00:34:58.745 | INFO | Analyzing chunk frames = [7552000,7552512) | 20260506 00:34:58.745 | INFO | MemInfo: { | 20260506 00:34:58.745 | INFO | "gpu" : " 17 GB", | 20260506 00:34:58.745 | INFO | "huge" : "", | 20260506 00:34:58.745 | INFO | "huge_shared" : " 27 GB", | 20260506 00:34:58.745 | INFO | "malloc" : " 14 GB", | 20260506 00:34:58.745 | INFO | "pinned" : " 29 GB" | 20260506 00:34:58.745 | INFO | } | 20260506 00:34:58.745 | INFO | Perf report for the last 5.13759 seconds: | 20260506 00:34:58.745 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:34:58.745 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:58.745 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:58.745 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:58.745 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260506 00:34:58.745 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:58.745 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:58.745 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 00:34:58.745 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:58.745 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:34:58.745 | INFO | Uploaded 36GiB to GPU in 2.03088s (17.7263GiB/s) | 20260506 00:34:58.746 | INFO | Downloaded 34GiB to GPU in 2.04312s (16.6412GiB/s) | 20260506 00:35:03.881 | INFO | Analyzing chunk frames = [7552512,7553024) | 20260506 00:35:03.881 | INFO | MemInfo: { | 20260506 00:35:03.881 | INFO | "gpu" : " 17 GB", | 20260506 00:35:03.881 | INFO | "huge" : "", | 20260506 00:35:03.881 | INFO | "huge_shared" : " 27 GB", | 20260506 00:35:03.881 | INFO | "malloc" : " 14 GB", | 20260506 00:35:03.881 | INFO | "pinned" : " 29 GB" | 20260506 00:35:03.881 | INFO | } | 20260506 00:35:03.881 | INFO | Perf report for the last 5.13579 seconds: | 20260506 00:35:03.881 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:35:03.881 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:03.881 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:03.881 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:03.881 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260506 00:35:03.881 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:03.881 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:03.881 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 00:35:03.881 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:03.881 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:03.881 | INFO | Uploaded 36GiB to GPU in 2.10795s (17.0782GiB/s) | 20260506 00:35:03.881 | INFO | Downloaded 34GiB to GPU in 2.03031s (16.7462GiB/s) | 20260506 00:35:03.881 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 00:35:03.881 | INFO | Total mem usage: 15.3 GiB | 20260506 00:35:03.881 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 00:35:03.881 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.15%) | 20260506 00:35:03.881 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 00:35:03.881 | INFO | High Water Breakdown: | 20260506 00:35:03.881 | INFO | 60.9% : 8.55 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 00:35:03.881 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 00:35:03.881 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 00:35:03.881 | INFO | ------------------------------------------ | 20260506 00:35:03.881 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:35:03.881 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 00:35:03.881 | INFO | Total mem usage: 26.5 GiB | 20260506 00:35:03.881 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 00:35:03.881 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 00:35:03.882 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 00:35:03.882 | INFO | High Water Breakdown: | 20260506 00:35:03.882 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:35:03.882 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:35:03.882 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:35:03.882 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:35:03.882 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:35:03.882 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:35:03.882 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 00:35:03.882 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 00:35:03.882 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:35:03.882 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 00:35:03.882 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 00:35:03.882 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 00:35:03.882 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 00:35:03.882 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 00:35:03.882 | INFO | ------------------------------------------ | 20260506 00:35:03.882 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:35:03.882 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 00:35:03.882 | INFO | Total mem usage: 25.6 GiB | 20260506 00:35:03.882 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 00:35:03.882 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 00:35:03.882 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 00:35:03.882 | INFO | High Water Breakdown: | 20260506 00:35:03.882 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 00:35:03.882 | INFO | ------------------------------------------ | 20260506 00:35:03.882 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:35:03.882 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 00:35:03.882 | INFO | Total mem usage: 15.7 GiB | 20260506 00:35:03.882 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 00:35:03.882 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 00:35:03.882 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 00:35:03.882 | INFO | High Water Breakdown: | 20260506 00:35:03.882 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 00:35:03.882 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 00:35:03.882 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 00:35:03.882 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 00:35:03.882 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 00:35:03.882 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 00:35:03.882 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 00:35:03.882 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 00:35:03.882 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 00:35:03.882 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:35:03.882 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 00:35:03.882 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:35:03.882 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 00:35:03.882 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 00:35:03.882 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 00:35:03.882 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 00:35:03.882 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:35:03.882 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 00:35:03.882 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:35:03.882 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:35:03.882 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:35:03.882 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 00:35:03.882 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 00:35:03.882 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:35:03.882 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 00:35:03.882 | INFO | ------------------------------------------ | 20260506 00:35:03.882 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:35:03.883 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 00:35:03.883 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:35:03.883 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:35:03.883 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.40e+01, 1.34e+01 / 8.20e-01 | 20260506 00:35:03.883 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.07e-01, 1.12e-02 / 1.53e-02 | 20260506 00:35:03.883 | INFO | PerfReport: 11.84%, Baseline -- 3.72e+00 / 4.52e+00, 5.77e-01 / 1.84e-01 | 20260506 00:35:03.883 | INFO | PerfReport: 3.29%, Histogram -- 1.03e+00 / 1.51e+00, 1.92e-01 / 8.73e-02 | 20260506 00:35:03.883 | INFO | PerfReport: 15.96%, DME -- 4.01e+01 / 4.40e+01, 3.82e+00 / 8.74e-01 | 20260506 00:35:03.883 | INFO | PerfReport: 57.45%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.59e+00 / 9.60e-01 | 20260506 00:35:03.883 | INFO | PerfReport: 7.04%, PulseAccumulating -- 2.21e+00 / 2.74e+00, 4.16e-01 / 1.79e-01 | 20260506 00:35:03.883 | INFO | PerfReport: 3.06%, Metrics -- 9.60e-01 / 1.24e+00, 2.57e-01 / 1.06e-01 | 20260506 00:35:03.883 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.33e-01 / 3.80e-01, 5.57e-02 / 2.32e-02 | 20260506 00:35:03.883 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 00:35:03.883 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:35:03.883 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:35:03.883 | INFO | PerfReport: Full Execution -- 9.81e+01 / -nan, 2.45e+01 / 0.00e+00 | 20260506 00:35:03.883 | INFO | PerfReport: 22.21%, Upload -- 2.18e+01 / 0.00e+00, 7.36e+00 / 0.00e+00 | 20260506 00:35:03.883 | INFO | PerfReport: 21.84%, Download -- 2.14e+01 / 0.00e+00, 7.53e+00 / 0.00e+00 | 20260506 00:35:03.883 | INFO | PerfReport: 31.98%, Compute -- 3.14e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 00:35:03.883 | INFO | PerfReport: 2.09%, LockUpload -- 2.05e+00 / 0.00e+00, 7.82e+00 / 0.00e+00 | 20260506 00:35:03.883 | INFO | PerfReport: 1.27%, LockDownload -- 1.25e+00 / 0.00e+00, 2.85e+00 / 0.00e+00 | 20260506 00:35:03.883 | INFO | PerfReport: 20.59%, LockCompute -- 2.02e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 00:35:03.883 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 2.20e-02 / 0.00e+00 | 20260506 00:35:04.000 | INFO | Starting a round of Baz IO | 20260506 00:35:07.141 | INFO | Populating RTMetrics report | 20260506 00:35:07.142 | INFO | RTMetrics report populated | 20260506 00:35:07.142 | INFO | Opening tmp RTMetrics json file | 20260506 00:35:07.142 | INFO | Serializing RTMetrics to json | 20260506 00:35:07.144 | INFO | Serialization done, swapping tmp file now... | 20260506 00:35:07.144 | INFO | RTMetrics done | 20260506 00:35:07.216 | INFO | Finished a round of Baz IO: Wrote 1.853 GiB in 3.215 seconds | 20260506 00:35:07.216 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.36% / 0.00% / 78.64% | 20260506 00:35:07.216 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0909328% / 0% / 9.53674e-05% | 20260506 00:35:07.216 | INFO | Baz IO overall: | 20260506 00:35:07.216 | INFO | BytesWritten : 4286936843092 | 20260506 00:35:07.216 | INFO | NumEvents : 2017051246548 | 20260506 00:35:09.016 | INFO | Analyzing chunk frames = [7553024,7553536) | 20260506 00:35:09.016 | INFO | MemInfo: { | 20260506 00:35:09.016 | INFO | "gpu" : " 17 GB", | 20260506 00:35:09.016 | INFO | "huge" : "", | 20260506 00:35:09.016 | INFO | "huge_shared" : " 27 GB", | 20260506 00:35:09.016 | INFO | "malloc" : " 14 GB", | 20260506 00:35:09.016 | INFO | "pinned" : " 29 GB" | 20260506 00:35:09.016 | INFO | } | 20260506 00:35:09.017 | INFO | Perf report for the last 5.13554 seconds: | 20260506 00:35:09.017 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:35:09.017 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:09.017 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:09.017 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:09.017 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260506 00:35:09.017 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:09.017 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260506 00:35:09.017 | INFO | PRE_HQ : 59.6%, 1.7, 53.7ms, 0 | 20260506 00:35:09.017 | INFO | BAZWRITER : 61.7%, 3.5, 110.5ms, 0 | 20260506 00:35:09.017 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:35:09.017 | INFO | Uploaded 36GiB to GPU in 2.16775s (16.6071GiB/s) | 20260506 00:35:09.017 | INFO | Downloaded 38GiB to GPU in 2.23799s (16.9795GiB/s) | 20260506 00:35:14.160 | INFO | Analyzing chunk frames = [7553536,7554048) | 20260506 00:35:14.160 | INFO | MemInfo: { | 20260506 00:35:14.160 | INFO | "gpu" : " 17 GB", | 20260506 00:35:14.160 | INFO | "huge" : "", | 20260506 00:35:14.160 | INFO | "huge_shared" : " 27 GB", | 20260506 00:35:14.160 | INFO | "malloc" : " 14 GB", | 20260506 00:35:14.160 | INFO | "pinned" : " 29 GB" | 20260506 00:35:14.160 | INFO | } | 20260506 00:35:14.160 | INFO | Perf report for the last 5.1438 seconds: | 20260506 00:35:14.160 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:35:14.160 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:14.160 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:14.160 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:14.160 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 00:35:14.160 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:14.160 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:14.160 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 00:35:14.160 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:14.160 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:14.161 | INFO | Uploaded 36GiB to GPU in 2.10511s (17.1013GiB/s) | 20260506 00:35:14.161 | INFO | Downloaded 34GiB to GPU in 2.03902s (16.6747GiB/s) | 20260506 00:35:19.301 | INFO | Analyzing chunk frames = [7554048,7554560) | 20260506 00:35:19.301 | INFO | MemInfo: { | 20260506 00:35:19.301 | INFO | "gpu" : " 17 GB", | 20260506 00:35:19.301 | INFO | "huge" : "", | 20260506 00:35:19.301 | INFO | "huge_shared" : " 27 GB", | 20260506 00:35:19.301 | INFO | "malloc" : " 14 GB", | 20260506 00:35:19.301 | INFO | "pinned" : " 29 GB" | 20260506 00:35:19.301 | INFO | } | 20260506 00:35:19.301 | INFO | Perf report for the last 5.14084 seconds: | 20260506 00:35:19.301 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:35:19.301 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:19.301 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:19.301 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:19.302 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 00:35:19.302 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:19.302 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:19.302 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 00:35:19.302 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:19.302 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:19.302 | INFO | Uploaded 36GiB to GPU in 2.08785s (17.2426GiB/s) | 20260506 00:35:19.302 | INFO | Downloaded 34GiB to GPU in 2.01439s (16.8785GiB/s) | 20260506 00:35:24.443 | INFO | Analyzing chunk frames = [7554560,7555072) | 20260506 00:35:24.443 | INFO | MemInfo: { | 20260506 00:35:24.443 | INFO | "gpu" : " 17 GB", | 20260506 00:35:24.443 | INFO | "huge" : "", | 20260506 00:35:24.443 | INFO | "huge_shared" : " 27 GB", | 20260506 00:35:24.443 | INFO | "malloc" : " 14 GB", | 20260506 00:35:24.443 | INFO | "pinned" : " 29 GB" | 20260506 00:35:24.443 | INFO | } | 20260506 00:35:24.443 | INFO | Perf report for the last 5.14226 seconds: | 20260506 00:35:24.443 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:35:24.443 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:24.443 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:24.443 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:24.444 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260506 00:35:24.444 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:24.444 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:24.444 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:35:24.444 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:24.444 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:24.444 | INFO | Uploaded 36GiB to GPU in 2.13986s (16.8235GiB/s) | 20260506 00:35:24.444 | INFO | Downloaded 34GiB to GPU in 2.02149s (16.8192GiB/s) | 20260506 00:35:29.578 | INFO | Analyzing chunk frames = [7555072,7555584) | 20260506 00:35:29.578 | INFO | MemInfo: { | 20260506 00:35:29.578 | INFO | "gpu" : " 17 GB", | 20260506 00:35:29.578 | INFO | "huge" : "", | 20260506 00:35:29.578 | INFO | "huge_shared" : " 27 GB", | 20260506 00:35:29.578 | INFO | "malloc" : " 14 GB", | 20260506 00:35:29.578 | INFO | "pinned" : " 29 GB" | 20260506 00:35:29.578 | INFO | } | 20260506 00:35:29.578 | INFO | Perf report for the last 5.13454 seconds: | 20260506 00:35:29.578 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:35:29.578 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:29.578 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:29.578 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:29.578 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 00:35:29.578 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:29.578 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:29.578 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 00:35:29.578 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:29.578 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:29.578 | INFO | Uploaded 36GiB to GPU in 2.09244s (17.2048GiB/s) | 20260506 00:35:29.578 | INFO | Downloaded 34GiB to GPU in 2.0454s (16.6227GiB/s) | 20260506 00:35:34.723 | INFO | Analyzing chunk frames = [7555584,7556096) | 20260506 00:35:34.723 | INFO | MemInfo: { | 20260506 00:35:34.723 | INFO | "gpu" : " 17 GB", | 20260506 00:35:34.723 | INFO | "huge" : "", | 20260506 00:35:34.723 | INFO | "huge_shared" : " 27 GB", | 20260506 00:35:34.723 | INFO | "malloc" : " 14 GB", | 20260506 00:35:34.723 | INFO | "pinned" : " 29 GB" | 20260506 00:35:34.723 | INFO | } | 20260506 00:35:34.723 | INFO | Perf report for the last 5.1448 seconds: | 20260506 00:35:34.723 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:35:34.723 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:34.723 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:34.723 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:34.723 | INFO | ANALYSIS : 61.3%, 3.0, 95.8ms, 0 | 20260506 00:35:34.723 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:34.723 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:34.723 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:35:34.723 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:34.723 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:34.723 | INFO | Uploaded 36GiB to GPU in 2.08235s (17.2882GiB/s) | 20260506 00:35:34.723 | INFO | Downloaded 34GiB to GPU in 2.04065s (16.6614GiB/s) | 20260506 00:35:39.862 | INFO | Analyzing chunk frames = [7556096,7556608) | 20260506 00:35:39.862 | INFO | MemInfo: { | 20260506 00:35:39.862 | INFO | "gpu" : " 17 GB", | 20260506 00:35:39.862 | INFO | "huge" : "", | 20260506 00:35:39.862 | INFO | "huge_shared" : " 27 GB", | 20260506 00:35:39.862 | INFO | "malloc" : " 14 GB", | 20260506 00:35:39.862 | INFO | "pinned" : " 29 GB" | 20260506 00:35:39.862 | INFO | } | 20260506 00:35:39.862 | INFO | Perf report for the last 5.13929 seconds: | 20260506 00:35:39.862 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:35:39.862 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:39.862 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:39.862 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:39.862 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 00:35:39.862 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:39.862 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:39.862 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 00:35:39.862 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:39.862 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:39.862 | INFO | Uploaded 36GiB to GPU in 2.03585s (17.683GiB/s) | 20260506 00:35:39.862 | INFO | Downloaded 34GiB to GPU in 2.05194s (16.5697GiB/s) | 20260506 00:35:44.996 | INFO | Analyzing chunk frames = [7556608,7557120) | 20260506 00:35:44.996 | INFO | MemInfo: { | 20260506 00:35:44.996 | INFO | "gpu" : " 17 GB", | 20260506 00:35:44.996 | INFO | "huge" : "", | 20260506 00:35:44.996 | INFO | "huge_shared" : " 27 GB", | 20260506 00:35:44.996 | INFO | "malloc" : " 14 GB", | 20260506 00:35:44.996 | INFO | "pinned" : " 29 GB" | 20260506 00:35:44.996 | INFO | } | 20260506 00:35:44.996 | INFO | Perf report for the last 5.13408 seconds: | 20260506 00:35:44.996 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:35:44.996 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:44.996 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:44.996 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:44.996 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260506 00:35:44.996 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:44.996 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:44.996 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 00:35:44.996 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:44.996 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:44.997 | INFO | Uploaded 36GiB to GPU in 2.0568s (17.5029GiB/s) | 20260506 00:35:44.997 | INFO | Downloaded 34GiB to GPU in 2.01623s (16.8632GiB/s) | 20260506 00:35:45.114 | INFO | Starting a round of Baz IO | 20260506 00:35:48.260 | INFO | Populating RTMetrics report | 20260506 00:35:48.260 | INFO | RTMetrics report populated | 20260506 00:35:48.260 | INFO | Opening tmp RTMetrics json file | 20260506 00:35:48.261 | INFO | Serializing RTMetrics to json | 20260506 00:35:48.265 | INFO | Serialization done, swapping tmp file now... | 20260506 00:35:48.265 | INFO | RTMetrics done | 20260506 00:35:48.338 | INFO | Finished a round of Baz IO: Wrote 1.853 GiB in 3.223 seconds | 20260506 00:35:48.338 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.36% / 0.00% / 78.64% | 20260506 00:35:48.338 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0910918% / 0% / 6.75519e-05% | 20260506 00:35:48.338 | INFO | Baz IO overall: | 20260506 00:35:48.338 | INFO | BytesWritten : 4288926756047 | 20260506 00:35:48.338 | INFO | NumEvents : 2017960683194 | 20260506 00:35:50.135 | INFO | Analyzing chunk frames = [7557120,7557632) | 20260506 00:35:50.135 | INFO | MemInfo: { | 20260506 00:35:50.135 | INFO | "gpu" : " 17 GB", | 20260506 00:35:50.135 | INFO | "huge" : "", | 20260506 00:35:50.135 | INFO | "huge_shared" : " 27 GB", | 20260506 00:35:50.135 | INFO | "malloc" : " 14 GB", | 20260506 00:35:50.135 | INFO | "pinned" : " 29 GB" | 20260506 00:35:50.135 | INFO | } | 20260506 00:35:50.135 | INFO | Perf report for the last 5.13902 seconds: | 20260506 00:35:50.135 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:35:50.135 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:50.135 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:50.135 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:50.135 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260506 00:35:50.135 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:50.135 | INFO | RT_METRICS : 43.2%, 1.0, 22.6ms, 0 | 20260506 00:35:50.135 | INFO | PRE_HQ : 59.8%, 1.7, 54.0ms, 0 | 20260506 00:35:50.135 | INFO | BAZWRITER : 61.8%, 3.5, 110.9ms, 0 | 20260506 00:35:50.135 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:35:50.135 | INFO | Uploaded 36GiB to GPU in 2.18011s (16.5129GiB/s) | 20260506 00:35:50.136 | INFO | Downloaded 38GiB to GPU in 2.2422s (16.9476GiB/s) | 20260506 00:35:55.270 | INFO | Analyzing chunk frames = [7557632,7558144) | 20260506 00:35:55.270 | INFO | MemInfo: { | 20260506 00:35:55.270 | INFO | "gpu" : " 17 GB", | 20260506 00:35:55.270 | INFO | "huge" : "", | 20260506 00:35:55.270 | INFO | "huge_shared" : " 27 GB", | 20260506 00:35:55.270 | INFO | "malloc" : " 14 GB", | 20260506 00:35:55.270 | INFO | "pinned" : " 29 GB" | 20260506 00:35:55.270 | INFO | } | 20260506 00:35:55.270 | INFO | Perf report for the last 5.13473 seconds: | 20260506 00:35:55.270 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:35:55.270 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:55.270 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:55.270 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:55.270 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 00:35:55.270 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:55.270 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:55.270 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:35:55.270 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:55.270 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:35:55.270 | INFO | Uploaded 36GiB to GPU in 2.09737s (17.1644GiB/s) | 20260506 00:35:55.270 | INFO | Downloaded 34GiB to GPU in 2.03208s (16.7316GiB/s) | 20260506 00:36:00.418 | INFO | Analyzing chunk frames = [7558144,7558656) | 20260506 00:36:00.418 | INFO | MemInfo: { | 20260506 00:36:00.418 | INFO | "gpu" : " 17 GB", | 20260506 00:36:00.418 | INFO | "huge" : "", | 20260506 00:36:00.418 | INFO | "huge_shared" : " 27 GB", | 20260506 00:36:00.418 | INFO | "malloc" : " 14 GB", | 20260506 00:36:00.418 | INFO | "pinned" : " 29 GB" | 20260506 00:36:00.418 | INFO | } | 20260506 00:36:00.418 | INFO | Perf report for the last 5.14782 seconds: | 20260506 00:36:00.418 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:36:00.418 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:00.418 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:00.418 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:00.418 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 00:36:00.418 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:00.418 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:00.418 | INFO | PRE_HQ : 26.4%, 1.0, 13.8ms, 0 | 20260506 00:36:00.418 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:00.418 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:00.418 | INFO | Uploaded 36GiB to GPU in 2.10268s (17.121GiB/s) | 20260506 00:36:00.418 | INFO | Downloaded 34GiB to GPU in 2.03567s (16.7021GiB/s) | 20260506 00:36:05.559 | INFO | Analyzing chunk frames = [7558656,7559168) | 20260506 00:36:05.559 | INFO | MemInfo: { | 20260506 00:36:05.559 | INFO | "gpu" : " 17 GB", | 20260506 00:36:05.559 | INFO | "huge" : "", | 20260506 00:36:05.559 | INFO | "huge_shared" : " 27 GB", | 20260506 00:36:05.559 | INFO | "malloc" : " 14 GB", | 20260506 00:36:05.559 | INFO | "pinned" : " 29 GB" | 20260506 00:36:05.559 | INFO | } | 20260506 00:36:05.559 | INFO | Perf report for the last 5.14105 seconds: | 20260506 00:36:05.559 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:36:05.559 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:05.559 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:05.559 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:05.559 | INFO | ANALYSIS : 61.6%, 3.0, 95.7ms, 0 | 20260506 00:36:05.559 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:05.559 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:05.559 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:36:05.559 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:05.559 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:05.559 | INFO | Uploaded 36GiB to GPU in 2.15202s (16.7284GiB/s) | 20260506 00:36:05.559 | INFO | Downloaded 34GiB to GPU in 2.05498s (16.5451GiB/s) | 20260506 00:36:10.698 | INFO | Analyzing chunk frames = [7559168,7559680) | 20260506 00:36:10.698 | INFO | MemInfo: { | 20260506 00:36:10.698 | INFO | "gpu" : " 17 GB", | 20260506 00:36:10.698 | INFO | "huge" : "", | 20260506 00:36:10.698 | INFO | "huge_shared" : " 27 GB", | 20260506 00:36:10.698 | INFO | "malloc" : " 14 GB", | 20260506 00:36:10.698 | INFO | "pinned" : " 29 GB" | 20260506 00:36:10.698 | INFO | } | 20260506 00:36:10.698 | INFO | Perf report for the last 5.13949 seconds: | 20260506 00:36:10.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:36:10.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:10.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:10.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:10.698 | INFO | ANALYSIS : 60.8%, 3.0, 94.9ms, 0 | 20260506 00:36:10.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:10.698 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:10.698 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 00:36:10.699 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:10.699 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:10.699 | INFO | Uploaded 36GiB to GPU in 2.07446s (17.3539GiB/s) | 20260506 00:36:10.699 | INFO | Downloaded 34GiB to GPU in 2.02792s (16.7659GiB/s) | 20260506 00:36:15.827 | INFO | Analyzing chunk frames = [7559680,7560192) | 20260506 00:36:15.827 | INFO | MemInfo: { | 20260506 00:36:15.827 | INFO | "gpu" : " 17 GB", | 20260506 00:36:15.827 | INFO | "huge" : "", | 20260506 00:36:15.827 | INFO | "huge_shared" : " 27 GB", | 20260506 00:36:15.827 | INFO | "malloc" : " 14 GB", | 20260506 00:36:15.827 | INFO | "pinned" : " 29 GB" | 20260506 00:36:15.827 | INFO | } | 20260506 00:36:15.827 | INFO | Perf report for the last 5.12895 seconds: | 20260506 00:36:15.827 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:36:15.827 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:15.827 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:15.827 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:15.827 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 00:36:15.827 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:15.827 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:15.827 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:36:15.827 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:15.827 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:15.828 | INFO | Uploaded 36GiB to GPU in 2.07375s (17.3599GiB/s) | 20260506 00:36:15.828 | INFO | Downloaded 34GiB to GPU in 2.03881s (16.6764GiB/s) | 20260506 00:36:20.968 | INFO | Analyzing chunk frames = [7560192,7560704) | 20260506 00:36:20.968 | INFO | MemInfo: { | 20260506 00:36:20.968 | INFO | "gpu" : " 17 GB", | 20260506 00:36:20.968 | INFO | "huge" : "", | 20260506 00:36:20.968 | INFO | "huge_shared" : " 27 GB", | 20260506 00:36:20.968 | INFO | "malloc" : " 14 GB", | 20260506 00:36:20.968 | INFO | "pinned" : " 29 GB" | 20260506 00:36:20.968 | INFO | } | 20260506 00:36:20.968 | INFO | Perf report for the last 5.14081 seconds: | 20260506 00:36:20.968 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:36:20.968 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:20.968 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:20.968 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:20.968 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260506 00:36:20.968 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:20.968 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:20.968 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 00:36:20.968 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:20.968 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:20.968 | INFO | Uploaded 36GiB to GPU in 2.12307s (16.9565GiB/s) | 20260506 00:36:20.968 | INFO | Downloaded 34GiB to GPU in 2.06945s (16.4295GiB/s) | 20260506 00:36:26.109 | INFO | Analyzing chunk frames = [7560704,7561216) | 20260506 00:36:26.109 | INFO | MemInfo: { | 20260506 00:36:26.109 | INFO | "gpu" : " 17 GB", | 20260506 00:36:26.109 | INFO | "huge" : "", | 20260506 00:36:26.109 | INFO | "huge_shared" : " 27 GB", | 20260506 00:36:26.109 | INFO | "malloc" : " 14 GB", | 20260506 00:36:26.109 | INFO | "pinned" : " 29 GB" | 20260506 00:36:26.109 | INFO | } | 20260506 00:36:26.109 | INFO | Perf report for the last 5.14064 seconds: | 20260506 00:36:26.109 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:36:26.109 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:26.109 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:26.109 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:26.109 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260506 00:36:26.109 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:26.109 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:26.109 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:36:26.109 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:26.109 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:26.109 | INFO | Uploaded 36GiB to GPU in 2.14418s (16.7896GiB/s) | 20260506 00:36:26.109 | INFO | Downloaded 34GiB to GPU in 2.02876s (16.759GiB/s) | 20260506 00:36:26.109 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 00:36:26.109 | INFO | Total mem usage: 15.3 GiB | 20260506 00:36:26.109 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 00:36:26.109 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.15%) | 20260506 00:36:26.109 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 00:36:26.109 | INFO | High Water Breakdown: | 20260506 00:36:26.109 | INFO | 60.9% : 8.55 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 00:36:26.109 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 00:36:26.109 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 00:36:26.109 | INFO | ------------------------------------------ | 20260506 00:36:26.109 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:36:26.109 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 00:36:26.109 | INFO | Total mem usage: 26.5 GiB | 20260506 00:36:26.109 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 00:36:26.109 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 00:36:26.109 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260506 00:36:26.109 | INFO | High Water Breakdown: | 20260506 00:36:26.109 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:36:26.109 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:36:26.109 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:36:26.109 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:36:26.109 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:36:26.109 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:36:26.109 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 00:36:26.109 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 00:36:26.109 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:36:26.109 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 00:36:26.109 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 00:36:26.110 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 00:36:26.110 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 00:36:26.110 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 00:36:26.110 | INFO | ------------------------------------------ | 20260506 00:36:26.110 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:36:26.110 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 00:36:26.110 | INFO | Total mem usage: 25.6 GiB | 20260506 00:36:26.110 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 00:36:26.110 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 00:36:26.110 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 00:36:26.110 | INFO | High Water Breakdown: | 20260506 00:36:26.110 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 00:36:26.110 | INFO | ------------------------------------------ | 20260506 00:36:26.110 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:36:26.110 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 00:36:26.110 | INFO | Total mem usage: 15.7 GiB | 20260506 00:36:26.110 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 00:36:26.110 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 00:36:26.110 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 00:36:26.110 | INFO | High Water Breakdown: | 20260506 00:36:26.110 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 00:36:26.110 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 00:36:26.110 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 00:36:26.110 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 00:36:26.110 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 00:36:26.110 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 00:36:26.110 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 00:36:26.110 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 00:36:26.110 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 00:36:26.110 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:36:26.110 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 00:36:26.110 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:36:26.110 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 00:36:26.110 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 00:36:26.110 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 00:36:26.110 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 00:36:26.110 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:36:26.110 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 00:36:26.110 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:36:26.110 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:36:26.110 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:36:26.110 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 00:36:26.110 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 00:36:26.110 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:36:26.110 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 00:36:26.110 | INFO | ------------------------------------------ | 20260506 00:36:26.110 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:36:26.110 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 00:36:26.110 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:36:26.110 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:36:26.110 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.40e+01, 1.34e+01 / 8.36e-01 | 20260506 00:36:26.110 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.04e-01, 1.14e-02 / 2.76e-03 | 20260506 00:36:26.110 | INFO | PerfReport: 11.82%, Baseline -- 3.70e+00 / 4.53e+00, 5.72e-01 / 1.63e-01 | 20260506 00:36:26.110 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.54e+00, 1.92e-01 / 7.65e-02 | 20260506 00:36:26.110 | INFO | PerfReport: 16.00%, DME -- 4.01e+01 / 4.41e+01, 3.81e+00 / 1.00e+00 | 20260506 00:36:26.110 | INFO | PerfReport: 57.38%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.50e+00 / 1.06e+00 | 20260506 00:36:26.111 | INFO | PerfReport: 7.09%, PulseAccumulating -- 2.22e+00 / 2.76e+00, 4.24e-01 / 2.27e-01 | 20260506 00:36:26.111 | INFO | PerfReport: 3.08%, Metrics -- 9.63e-01 / 1.24e+00, 2.61e-01 / 9.89e-02 | 20260506 00:36:26.111 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.33e-01 / 3.85e-01, 5.66e-02 / 3.02e-02 | 20260506 00:36:26.111 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 00:36:26.111 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:36:26.111 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:36:26.111 | INFO | PerfReport: Full Execution -- 9.82e+01 / -nan, 2.44e+01 / 0.00e+00 | 20260506 00:36:26.111 | INFO | PerfReport: 22.35%, Upload -- 2.20e+01 / 0.00e+00, 7.51e+00 / 0.00e+00 | 20260506 00:36:26.111 | INFO | PerfReport: 21.88%, Download -- 2.15e+01 / 0.00e+00, 7.62e+00 / 0.00e+00 | 20260506 00:36:26.111 | INFO | PerfReport: 31.89%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 00:36:26.111 | INFO | PerfReport: 2.09%, LockUpload -- 2.05e+00 / 0.00e+00, 7.90e+00 / 0.00e+00 | 20260506 00:36:26.111 | INFO | PerfReport: 1.30%, LockDownload -- 1.27e+00 / 0.00e+00, 2.89e+00 / 0.00e+00 | 20260506 00:36:26.111 | INFO | PerfReport: 20.47%, LockCompute -- 2.01e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 00:36:26.111 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.85e-02 / 0.00e+00 | 20260506 00:36:26.228 | INFO | Starting a round of Baz IO | 20260506 00:36:29.373 | INFO | Populating RTMetrics report | 20260506 00:36:29.373 | INFO | RTMetrics report populated | 20260506 00:36:29.373 | INFO | Opening tmp RTMetrics json file | 20260506 00:36:29.374 | INFO | Serializing RTMetrics to json | 20260506 00:36:29.375 | INFO | Serialization done, swapping tmp file now... | 20260506 00:36:29.376 | INFO | RTMetrics done | 20260506 00:36:29.455 | INFO | Finished a round of Baz IO: Wrote 1.853 GiB in 3.227 seconds | 20260506 00:36:29.455 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.36% / 0.00% / 78.64% | 20260506 00:36:29.455 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.091056% / 0% / 0.000111262% | 20260506 00:36:29.455 | INFO | Baz IO overall: | 20260506 00:36:29.455 | INFO | BytesWritten : 4290916926372 | 20260506 00:36:29.455 | INFO | NumEvents : 2018870233730 | 20260506 00:36:31.249 | INFO | Analyzing chunk frames = [7561216,7561728) | 20260506 00:36:31.249 | INFO | MemInfo: { | 20260506 00:36:31.249 | INFO | "gpu" : " 17 GB", | 20260506 00:36:31.249 | INFO | "huge" : "", | 20260506 00:36:31.249 | INFO | "huge_shared" : " 27 GB", | 20260506 00:36:31.249 | INFO | "malloc" : " 14 GB", | 20260506 00:36:31.249 | INFO | "pinned" : " 29 GB" | 20260506 00:36:31.249 | INFO | } | 20260506 00:36:31.249 | INFO | Perf report for the last 5.1401 seconds: | 20260506 00:36:31.249 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:36:31.249 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:31.249 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:31.249 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:31.249 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260506 00:36:31.249 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:31.249 | INFO | RT_METRICS : 43.2%, 1.0, 22.6ms, 0 | 20260506 00:36:31.249 | INFO | PRE_HQ : 59.8%, 1.7, 53.9ms, 0 | 20260506 00:36:31.249 | INFO | BAZWRITER : 62.0%, 3.5, 110.8ms, 0 | 20260506 00:36:31.249 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:36:31.249 | INFO | Uploaded 36GiB to GPU in 2.16275s (16.6454GiB/s) | 20260506 00:36:31.249 | INFO | Downloaded 38GiB to GPU in 2.23822s (16.9778GiB/s) | 20260506 00:36:36.388 | INFO | Analyzing chunk frames = [7561728,7562240) | 20260506 00:36:36.388 | INFO | MemInfo: { | 20260506 00:36:36.388 | INFO | "gpu" : " 17 GB", | 20260506 00:36:36.388 | INFO | "huge" : "", | 20260506 00:36:36.388 | INFO | "huge_shared" : " 27 GB", | 20260506 00:36:36.388 | INFO | "malloc" : " 14 GB", | 20260506 00:36:36.388 | INFO | "pinned" : " 29 GB" | 20260506 00:36:36.388 | INFO | } | 20260506 00:36:36.388 | INFO | Perf report for the last 5.13917 seconds: | 20260506 00:36:36.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:36:36.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:36.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:36.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:36.388 | INFO | ANALYSIS : 61.4%, 3.0, 95.9ms, 0 | 20260506 00:36:36.388 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:36.388 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:36.388 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 00:36:36.388 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:36.388 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:36.388 | INFO | Uploaded 36GiB to GPU in 2.02102s (17.8128GiB/s) | 20260506 00:36:36.388 | INFO | Downloaded 34GiB to GPU in 2.00605s (16.9487GiB/s) | 20260506 00:36:41.528 | INFO | Analyzing chunk frames = [7562240,7562752) | 20260506 00:36:41.528 | INFO | MemInfo: { | 20260506 00:36:41.528 | INFO | "gpu" : " 17 GB", | 20260506 00:36:41.528 | INFO | "huge" : "", | 20260506 00:36:41.528 | INFO | "huge_shared" : " 27 GB", | 20260506 00:36:41.528 | INFO | "malloc" : " 14 GB", | 20260506 00:36:41.528 | INFO | "pinned" : " 29 GB" | 20260506 00:36:41.528 | INFO | } | 20260506 00:36:41.528 | INFO | Perf report for the last 5.14019 seconds: | 20260506 00:36:41.528 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:36:41.528 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:41.528 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:41.528 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:41.528 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260506 00:36:41.528 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:41.528 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:41.528 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 00:36:41.528 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:41.528 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:41.529 | INFO | Uploaded 36GiB to GPU in 2.15228s (16.7264GiB/s) | 20260506 00:36:41.529 | INFO | Downloaded 34GiB to GPU in 2.06324s (16.4789GiB/s) | 20260506 00:36:46.674 | INFO | Analyzing chunk frames = [7562752,7563264) | 20260506 00:36:46.674 | INFO | MemInfo: { | 20260506 00:36:46.674 | INFO | "gpu" : " 17 GB", | 20260506 00:36:46.674 | INFO | "huge" : "", | 20260506 00:36:46.674 | INFO | "huge_shared" : " 27 GB", | 20260506 00:36:46.674 | INFO | "malloc" : " 14 GB", | 20260506 00:36:46.674 | INFO | "pinned" : " 29 GB" | 20260506 00:36:46.674 | INFO | } | 20260506 00:36:46.674 | INFO | Perf report for the last 5.14584 seconds: | 20260506 00:36:46.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:36:46.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:46.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:46.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:46.674 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 00:36:46.674 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:46.674 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:46.674 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:36:46.674 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:46.674 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:46.674 | INFO | Uploaded 36GiB to GPU in 2.14433s (16.7884GiB/s) | 20260506 00:36:46.674 | INFO | Downloaded 34GiB to GPU in 2.02738s (16.7704GiB/s) | 20260506 00:36:51.812 | INFO | Analyzing chunk frames = [7563264,7563776) | 20260506 00:36:51.812 | INFO | MemInfo: { | 20260506 00:36:51.812 | INFO | "gpu" : " 17 GB", | 20260506 00:36:51.812 | INFO | "huge" : "", | 20260506 00:36:51.812 | INFO | "huge_shared" : " 27 GB", | 20260506 00:36:51.812 | INFO | "malloc" : " 14 GB", | 20260506 00:36:51.812 | INFO | "pinned" : " 29 GB" | 20260506 00:36:51.812 | INFO | } | 20260506 00:36:51.812 | INFO | Perf report for the last 5.13782 seconds: | 20260506 00:36:51.812 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:36:51.812 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:51.812 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:51.812 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:51.812 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260506 00:36:51.812 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:51.812 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:51.812 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 00:36:51.812 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:51.812 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:51.812 | INFO | Uploaded 36GiB to GPU in 2.07679s (17.3345GiB/s) | 20260506 00:36:51.812 | INFO | Downloaded 34GiB to GPU in 2.02658s (16.777GiB/s) | 20260506 00:36:56.945 | INFO | Analyzing chunk frames = [7563776,7564288) | 20260506 00:36:56.945 | INFO | MemInfo: { | 20260506 00:36:56.945 | INFO | "gpu" : " 17 GB", | 20260506 00:36:56.945 | INFO | "huge" : "", | 20260506 00:36:56.945 | INFO | "huge_shared" : " 27 GB", | 20260506 00:36:56.945 | INFO | "malloc" : " 14 GB", | 20260506 00:36:56.945 | INFO | "pinned" : " 29 GB" | 20260506 00:36:56.945 | INFO | } | 20260506 00:36:56.945 | INFO | Perf report for the last 5.13336 seconds: | 20260506 00:36:56.945 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:36:56.945 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:56.945 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:56.945 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:56.945 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 00:36:56.945 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:56.945 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:56.945 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:36:56.946 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:56.946 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:36:56.946 | INFO | Uploaded 36GiB to GPU in 2.07527s (17.3471GiB/s) | 20260506 00:36:56.946 | INFO | Downloaded 34GiB to GPU in 2.03691s (16.692GiB/s) | 20260506 00:37:02.087 | INFO | Analyzing chunk frames = [7564288,7564800) | 20260506 00:37:02.087 | INFO | MemInfo: { | 20260506 00:37:02.087 | INFO | "gpu" : " 17 GB", | 20260506 00:37:02.087 | INFO | "huge" : "", | 20260506 00:37:02.087 | INFO | "huge_shared" : " 27 GB", | 20260506 00:37:02.087 | INFO | "malloc" : " 14 GB", | 20260506 00:37:02.087 | INFO | "pinned" : " 29 GB" | 20260506 00:37:02.087 | INFO | } | 20260506 00:37:02.087 | INFO | Perf report for the last 5.142 seconds: | 20260506 00:37:02.087 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:37:02.087 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:02.087 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:02.087 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:02.087 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260506 00:37:02.087 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:02.087 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:02.088 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 00:37:02.088 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:02.088 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:02.088 | INFO | Uploaded 36GiB to GPU in 2.10094s (17.1352GiB/s) | 20260506 00:37:02.088 | INFO | Downloaded 34GiB to GPU in 2.08379s (16.3165GiB/s) | 20260506 00:37:07.221 | INFO | Analyzing chunk frames = [7564800,7565312) | 20260506 00:37:07.221 | INFO | MemInfo: { | 20260506 00:37:07.221 | INFO | "gpu" : " 17 GB", | 20260506 00:37:07.221 | INFO | "huge" : "", | 20260506 00:37:07.221 | INFO | "huge_shared" : " 27 GB", | 20260506 00:37:07.221 | INFO | "malloc" : " 14 GB", | 20260506 00:37:07.221 | INFO | "pinned" : " 29 GB" | 20260506 00:37:07.221 | INFO | } | 20260506 00:37:07.221 | INFO | Perf report for the last 5.13363 seconds: | 20260506 00:37:07.221 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:37:07.221 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:07.221 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:07.221 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:07.221 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260506 00:37:07.221 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:07.221 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:07.221 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 00:37:07.221 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:07.221 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:07.221 | INFO | Uploaded 36GiB to GPU in 2.11078s (17.0553GiB/s) | 20260506 00:37:07.221 | INFO | Downloaded 34GiB to GPU in 2.03037s (16.7457GiB/s) | 20260506 00:37:07.339 | INFO | Starting a round of Baz IO | 20260506 00:37:10.491 | INFO | Populating RTMetrics report | 20260506 00:37:10.491 | INFO | RTMetrics report populated | 20260506 00:37:10.491 | INFO | Opening tmp RTMetrics json file | 20260506 00:37:10.492 | INFO | Serializing RTMetrics to json | 20260506 00:37:10.493 | INFO | Serialization done, swapping tmp file now... | 20260506 00:37:10.494 | INFO | RTMetrics done | 20260506 00:37:10.572 | INFO | Finished a round of Baz IO: Wrote 1.853 GiB in 3.232 seconds | 20260506 00:37:10.572 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.36% / 0.00% / 78.64% | 20260506 00:37:10.572 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0908295% / 0% / 0.000107288% | 20260506 00:37:10.572 | INFO | Baz IO overall: | 20260506 00:37:10.572 | INFO | BytesWritten : 4292906181818 | 20260506 00:37:10.572 | INFO | NumEvents : 2019779311086 | 20260506 00:37:12.368 | INFO | Analyzing chunk frames = [7565312,7565824) | 20260506 00:37:12.368 | INFO | MemInfo: { | 20260506 00:37:12.368 | INFO | "gpu" : " 17 GB", | 20260506 00:37:12.368 | INFO | "huge" : "", | 20260506 00:37:12.368 | INFO | "huge_shared" : " 27 GB", | 20260506 00:37:12.368 | INFO | "malloc" : " 14 GB", | 20260506 00:37:12.369 | INFO | "pinned" : " 29 GB" | 20260506 00:37:12.369 | INFO | } | 20260506 00:37:12.369 | INFO | Perf report for the last 5.14754 seconds: | 20260506 00:37:12.369 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:37:12.369 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:12.369 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:12.369 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:12.369 | INFO | ANALYSIS : 62.5%, 3.0, 97.7ms, 0 | 20260506 00:37:12.369 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:12.369 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260506 00:37:12.369 | INFO | PRE_HQ : 59.8%, 1.7, 54.0ms, 0 | 20260506 00:37:12.369 | INFO | BAZWRITER : 61.9%, 3.5, 111.3ms, 0 | 20260506 00:37:12.369 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:37:12.369 | INFO | Uploaded 36GiB to GPU in 2.15421s (16.7115GiB/s) | 20260506 00:37:12.369 | INFO | Downloaded 38GiB to GPU in 2.24394s (16.9345GiB/s) | 20260506 00:37:17.509 | INFO | Analyzing chunk frames = [7565824,7566336) | 20260506 00:37:17.509 | INFO | MemInfo: { | 20260506 00:37:17.509 | INFO | "gpu" : " 17 GB", | 20260506 00:37:17.509 | INFO | "huge" : "", | 20260506 00:37:17.509 | INFO | "huge_shared" : " 27 GB", | 20260506 00:37:17.509 | INFO | "malloc" : " 14 GB", | 20260506 00:37:17.509 | INFO | "pinned" : " 29 GB" | 20260506 00:37:17.509 | INFO | } | 20260506 00:37:17.509 | INFO | Perf report for the last 5.14048 seconds: | 20260506 00:37:17.509 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:37:17.509 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:17.509 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:17.509 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:17.509 | INFO | ANALYSIS : 60.9%, 3.0, 95.1ms, 0 | 20260506 00:37:17.509 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:17.509 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:17.509 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:37:17.509 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:17.509 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:17.509 | INFO | Uploaded 36GiB to GPU in 1.95958s (18.3713GiB/s) | 20260506 00:37:17.509 | INFO | Downloaded 34GiB to GPU in 1.98799s (17.1027GiB/s) | 20260506 00:37:22.645 | INFO | Analyzing chunk frames = [7566336,7566848) | 20260506 00:37:22.645 | INFO | MemInfo: { | 20260506 00:37:22.645 | INFO | "gpu" : " 17 GB", | 20260506 00:37:22.645 | INFO | "huge" : "", | 20260506 00:37:22.645 | INFO | "huge_shared" : " 27 GB", | 20260506 00:37:22.645 | INFO | "malloc" : " 14 GB", | 20260506 00:37:22.645 | INFO | "pinned" : " 29 GB" | 20260506 00:37:22.645 | INFO | } | 20260506 00:37:22.645 | INFO | Perf report for the last 5.13628 seconds: | 20260506 00:37:22.645 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:37:22.645 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:22.645 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:22.645 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:22.645 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 00:37:22.645 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:22.645 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:22.645 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:37:22.645 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:22.646 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:22.646 | INFO | Uploaded 36GiB to GPU in 2.08744s (17.246GiB/s) | 20260506 00:37:22.646 | INFO | Downloaded 34GiB to GPU in 2.01745s (16.853GiB/s) | 20260506 00:37:27.780 | INFO | Analyzing chunk frames = [7566848,7567360) | 20260506 00:37:27.780 | INFO | MemInfo: { | 20260506 00:37:27.780 | INFO | "gpu" : " 17 GB", | 20260506 00:37:27.780 | INFO | "huge" : "", | 20260506 00:37:27.780 | INFO | "huge_shared" : " 27 GB", | 20260506 00:37:27.780 | INFO | "malloc" : " 14 GB", | 20260506 00:37:27.780 | INFO | "pinned" : " 29 GB" | 20260506 00:37:27.780 | INFO | } | 20260506 00:37:27.780 | INFO | Perf report for the last 5.13484 seconds: | 20260506 00:37:27.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:37:27.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:27.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:27.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:27.780 | INFO | ANALYSIS : 61.8%, 3.0, 95.8ms, 0 | 20260506 00:37:27.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:27.780 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:27.780 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:37:27.780 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:27.780 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:27.780 | INFO | Uploaded 36GiB to GPU in 2.13913s (16.8293GiB/s) | 20260506 00:37:27.781 | INFO | Downloaded 34GiB to GPU in 2.01234s (16.8957GiB/s) | 20260506 00:37:32.922 | INFO | Analyzing chunk frames = [7567360,7567872) | 20260506 00:37:32.922 | INFO | MemInfo: { | 20260506 00:37:32.922 | INFO | "gpu" : " 17 GB", | 20260506 00:37:32.922 | INFO | "huge" : "", | 20260506 00:37:32.922 | INFO | "huge_shared" : " 27 GB", | 20260506 00:37:32.922 | INFO | "malloc" : " 14 GB", | 20260506 00:37:32.922 | INFO | "pinned" : " 29 GB" | 20260506 00:37:32.922 | INFO | } | 20260506 00:37:32.922 | INFO | Perf report for the last 5.14176 seconds: | 20260506 00:37:32.922 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:37:32.922 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:32.922 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:32.922 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:32.922 | INFO | ANALYSIS : 61.1%, 3.0, 95.5ms, 0 | 20260506 00:37:32.922 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:32.922 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:32.922 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 00:37:32.922 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:32.922 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:32.922 | INFO | Uploaded 36GiB to GPU in 2.10363s (17.1133GiB/s) | 20260506 00:37:32.922 | INFO | Downloaded 34GiB to GPU in 2.04723s (16.6078GiB/s) | 20260506 00:37:38.062 | INFO | Analyzing chunk frames = [7567872,7568384) | 20260506 00:37:38.062 | INFO | MemInfo: { | 20260506 00:37:38.062 | INFO | "gpu" : " 17 GB", | 20260506 00:37:38.063 | INFO | "huge" : "", | 20260506 00:37:38.063 | INFO | "huge_shared" : " 27 GB", | 20260506 00:37:38.063 | INFO | "malloc" : " 14 GB", | 20260506 00:37:38.063 | INFO | "pinned" : " 29 GB" | 20260506 00:37:38.063 | INFO | } | 20260506 00:37:38.063 | INFO | Perf report for the last 5.14067 seconds: | 20260506 00:37:38.063 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:37:38.063 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:38.063 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:38.063 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:38.063 | INFO | ANALYSIS : 61.2%, 3.0, 95.6ms, 0 | 20260506 00:37:38.063 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:38.063 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:38.063 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:37:38.063 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:38.063 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:38.063 | INFO | Uploaded 36GiB to GPU in 2.08278s (17.2846GiB/s) | 20260506 00:37:38.063 | INFO | Downloaded 34GiB to GPU in 2.0426s (16.6455GiB/s) | 20260506 00:37:43.199 | INFO | Analyzing chunk frames = [7568384,7568896) | 20260506 00:37:43.199 | INFO | MemInfo: { | 20260506 00:37:43.199 | INFO | "gpu" : " 17 GB", | 20260506 00:37:43.199 | INFO | "huge" : "", | 20260506 00:37:43.199 | INFO | "huge_shared" : " 27 GB", | 20260506 00:37:43.199 | INFO | "malloc" : " 14 GB", | 20260506 00:37:43.199 | INFO | "pinned" : " 29 GB" | 20260506 00:37:43.199 | INFO | } | 20260506 00:37:43.199 | INFO | Perf report for the last 5.13662 seconds: | 20260506 00:37:43.199 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:37:43.199 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:43.199 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:43.199 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:43.199 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 00:37:43.199 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:43.199 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:43.199 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260506 00:37:43.199 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:43.199 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:43.200 | INFO | Uploaded 36GiB to GPU in 2.02409s (17.7858GiB/s) | 20260506 00:37:43.200 | INFO | Downloaded 34GiB to GPU in 2.03422s (16.714GiB/s) | 20260506 00:37:48.339 | INFO | Analyzing chunk frames = [7568896,7569408) | 20260506 00:37:48.339 | INFO | MemInfo: { | 20260506 00:37:48.339 | INFO | "gpu" : " 17 GB", | 20260506 00:37:48.339 | INFO | "huge" : "", | 20260506 00:37:48.339 | INFO | "huge_shared" : " 27 GB", | 20260506 00:37:48.339 | INFO | "malloc" : " 14 GB", | 20260506 00:37:48.340 | INFO | "pinned" : " 29 GB" | 20260506 00:37:48.340 | INFO | } | 20260506 00:37:48.340 | INFO | Perf report for the last 5.14027 seconds: | 20260506 00:37:48.340 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:37:48.340 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:48.340 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:48.340 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:48.340 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260506 00:37:48.340 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:48.340 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:48.340 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 00:37:48.340 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:48.340 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:48.340 | INFO | Uploaded 36GiB to GPU in 2.11809s (16.9965GiB/s) | 20260506 00:37:48.340 | INFO | Downloaded 34GiB to GPU in 2.0033s (16.972GiB/s) | 20260506 00:37:48.340 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 00:37:48.340 | INFO | Total mem usage: 15.3 GiB | 20260506 00:37:48.340 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 00:37:48.340 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.12%) | 20260506 00:37:48.340 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 00:37:48.340 | INFO | High Water Breakdown: | 20260506 00:37:48.340 | INFO | 60.9% : 8.55 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 00:37:48.340 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 00:37:48.340 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 00:37:48.340 | INFO | ------------------------------------------ | 20260506 00:37:48.340 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:37:48.340 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 00:37:48.340 | INFO | Total mem usage: 26.5 GiB | 20260506 00:37:48.340 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 00:37:48.340 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 00:37:48.340 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 00:37:48.340 | INFO | High Water Breakdown: | 20260506 00:37:48.340 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:37:48.340 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:37:48.340 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:37:48.340 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:37:48.340 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:37:48.340 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:37:48.340 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 00:37:48.340 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 00:37:48.340 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:37:48.340 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 00:37:48.340 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 00:37:48.340 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 00:37:48.340 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 00:37:48.340 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 00:37:48.340 | INFO | ------------------------------------------ | 20260506 00:37:48.340 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:37:48.340 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 00:37:48.340 | INFO | Total mem usage: 25.6 GiB | 20260506 00:37:48.340 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 00:37:48.340 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 00:37:48.340 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 00:37:48.340 | INFO | High Water Breakdown: | 20260506 00:37:48.340 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 00:37:48.340 | INFO | ------------------------------------------ | 20260506 00:37:48.340 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:37:48.340 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 00:37:48.340 | INFO | Total mem usage: 15.7 GiB | 20260506 00:37:48.340 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 00:37:48.340 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 00:37:48.340 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 00:37:48.340 | INFO | High Water Breakdown: | 20260506 00:37:48.340 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 00:37:48.340 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 00:37:48.340 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 00:37:48.340 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 00:37:48.340 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 00:37:48.340 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 00:37:48.340 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 00:37:48.340 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 00:37:48.340 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 00:37:48.340 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:37:48.341 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 00:37:48.341 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:37:48.341 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 00:37:48.341 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 00:37:48.341 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 00:37:48.341 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 00:37:48.341 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:37:48.341 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 00:37:48.341 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:37:48.341 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:37:48.341 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:37:48.341 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 00:37:48.341 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 00:37:48.341 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:37:48.341 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 00:37:48.341 | INFO | ------------------------------------------ | 20260506 00:37:48.341 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:37:48.341 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 00:37:48.341 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:37:48.341 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:37:48.341 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.39e+01, 1.32e+01 / 8.61e-01 | 20260506 00:37:48.341 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.04e-01, 1.16e-02 / 2.92e-03 | 20260506 00:37:48.341 | INFO | PerfReport: 11.83%, Baseline -- 3.71e+00 / 4.52e+00, 5.75e-01 / 2.04e-01 | 20260506 00:37:48.341 | INFO | PerfReport: 3.29%, Histogram -- 1.03e+00 / 1.53e+00, 1.94e-01 / 9.24e-02 | 20260506 00:37:48.341 | INFO | PerfReport: 15.98%, DME -- 4.01e+01 / 4.40e+01, 3.80e+00 / 1.06e+00 | 20260506 00:37:48.341 | INFO | PerfReport: 57.41%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.57e+00 / 1.03e+00 | 20260506 00:37:48.341 | INFO | PerfReport: 7.06%, PulseAccumulating -- 2.21e+00 / 2.74e+00, 4.19e-01 / 2.25e-01 | 20260506 00:37:48.341 | INFO | PerfReport: 3.07%, Metrics -- 9.62e-01 / 1.24e+00, 2.60e-01 / 1.23e-01 | 20260506 00:37:48.341 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.33e-01 / 3.85e-01, 5.65e-02 / 3.36e-02 | 20260506 00:37:48.341 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 00:37:48.341 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:37:48.341 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:37:48.341 | INFO | PerfReport: Full Execution -- 9.81e+01 / -nan, 2.46e+01 / 0.00e+00 | 20260506 00:37:48.341 | INFO | PerfReport: 22.23%, Upload -- 2.18e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260506 00:37:48.341 | INFO | PerfReport: 21.83%, Download -- 2.14e+01 / 0.00e+00, 7.60e+00 / 0.00e+00 | 20260506 00:37:48.341 | INFO | PerfReport: 31.95%, Compute -- 3.14e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260506 00:37:48.341 | INFO | PerfReport: 2.12%, LockUpload -- 2.08e+00 / 0.00e+00, 7.87e+00 / 0.00e+00 | 20260506 00:37:48.341 | INFO | PerfReport: 1.29%, LockDownload -- 1.27e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260506 00:37:48.341 | INFO | PerfReport: 20.57%, LockCompute -- 2.02e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 00:37:48.341 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.89e-02 / 0.00e+00 | 20260506 00:37:48.456 | INFO | Starting a round of Baz IO | 20260506 00:37:51.602 | INFO | Populating RTMetrics report | 20260506 00:37:51.602 | INFO | RTMetrics report populated | 20260506 00:37:51.602 | INFO | Opening tmp RTMetrics json file | 20260506 00:37:51.603 | INFO | Serializing RTMetrics to json | 20260506 00:37:51.605 | INFO | Serialization done, swapping tmp file now... | 20260506 00:37:51.605 | INFO | RTMetrics done | 20260506 00:37:51.683 | INFO | Finished a round of Baz IO: Wrote 1.854 GiB in 3.227 seconds | 20260506 00:37:51.683 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.36% / 0.00% / 78.64% | 20260506 00:37:51.683 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0905474% / 0% / 9.13938e-05% | 20260506 00:37:51.683 | INFO | Baz IO overall: | 20260506 00:37:51.683 | INFO | BytesWritten : 4294897415426 | 20260506 00:37:51.683 | INFO | NumEvents : 2020689883616 | 20260506 00:37:53.478 | INFO | Analyzing chunk frames = [7569408,7569920) | 20260506 00:37:53.478 | INFO | MemInfo: { | 20260506 00:37:53.478 | INFO | "gpu" : " 17 GB", | 20260506 00:37:53.478 | INFO | "huge" : "", | 20260506 00:37:53.478 | INFO | "huge_shared" : " 27 GB", | 20260506 00:37:53.478 | INFO | "malloc" : " 14 GB", | 20260506 00:37:53.478 | INFO | "pinned" : " 29 GB" | 20260506 00:37:53.478 | INFO | } | 20260506 00:37:53.478 | INFO | Perf report for the last 5.13877 seconds: | 20260506 00:37:53.478 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:37:53.478 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:53.478 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:53.478 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:53.478 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260506 00:37:53.478 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:53.478 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260506 00:37:53.478 | INFO | PRE_HQ : 59.7%, 1.7, 53.8ms, 0 | 20260506 00:37:53.478 | INFO | BAZWRITER : 61.9%, 3.5, 110.8ms, 0 | 20260506 00:37:53.478 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:37:53.479 | INFO | Uploaded 36GiB to GPU in 2.16601s (16.6204GiB/s) | 20260506 00:37:53.479 | INFO | Downloaded 38GiB to GPU in 2.23767s (16.982GiB/s) | 20260506 00:37:58.623 | INFO | Analyzing chunk frames = [7569920,7570432) | 20260506 00:37:58.623 | INFO | MemInfo: { | 20260506 00:37:58.623 | INFO | "gpu" : " 17 GB", | 20260506 00:37:58.623 | INFO | "huge" : "", | 20260506 00:37:58.623 | INFO | "huge_shared" : " 27 GB", | 20260506 00:37:58.623 | INFO | "malloc" : " 14 GB", | 20260506 00:37:58.623 | INFO | "pinned" : " 29 GB" | 20260506 00:37:58.623 | INFO | } | 20260506 00:37:58.623 | INFO | Perf report for the last 5.14487 seconds: | 20260506 00:37:58.623 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:37:58.623 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:58.623 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:58.623 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:58.623 | INFO | ANALYSIS : 61.7%, 3.0, 96.5ms, 0 | 20260506 00:37:58.623 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:58.623 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:58.623 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 00:37:58.623 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:58.623 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:37:58.624 | INFO | Uploaded 36GiB to GPU in 2.11384s (17.0306GiB/s) | 20260506 00:37:58.624 | INFO | Downloaded 34GiB to GPU in 2.041s (16.6585GiB/s) | 20260506 00:38:03.760 | INFO | Analyzing chunk frames = [7570432,7570944) | 20260506 00:38:03.760 | INFO | MemInfo: { | 20260506 00:38:03.760 | INFO | "gpu" : " 17 GB", | 20260506 00:38:03.760 | INFO | "huge" : "", | 20260506 00:38:03.760 | INFO | "huge_shared" : " 27 GB", | 20260506 00:38:03.760 | INFO | "malloc" : " 14 GB", | 20260506 00:38:03.760 | INFO | "pinned" : " 29 GB" | 20260506 00:38:03.760 | INFO | } | 20260506 00:38:03.760 | INFO | Perf report for the last 5.13661 seconds: | 20260506 00:38:03.760 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:38:03.760 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:03.760 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:03.760 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:03.760 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 00:38:03.760 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:03.760 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:03.760 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:38:03.760 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:03.760 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:03.760 | INFO | Uploaded 36GiB to GPU in 2.08435s (17.2716GiB/s) | 20260506 00:38:03.760 | INFO | Downloaded 34GiB to GPU in 2.01742s (16.8532GiB/s) | 20260506 00:38:08.892 | INFO | Analyzing chunk frames = [7570944,7571456) | 20260506 00:38:08.892 | INFO | MemInfo: { | 20260506 00:38:08.892 | INFO | "gpu" : " 17 GB", | 20260506 00:38:08.892 | INFO | "huge" : "", | 20260506 00:38:08.892 | INFO | "huge_shared" : " 27 GB", | 20260506 00:38:08.892 | INFO | "malloc" : " 14 GB", | 20260506 00:38:08.892 | INFO | "pinned" : " 29 GB" | 20260506 00:38:08.892 | INFO | } | 20260506 00:38:08.892 | INFO | Perf report for the last 5.13217 seconds: | 20260506 00:38:08.892 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:38:08.892 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:08.892 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:08.892 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:08.892 | INFO | ANALYSIS : 61.7%, 3.0, 95.6ms, 0 | 20260506 00:38:08.892 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:08.892 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:08.892 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 00:38:08.892 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:08.892 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:08.892 | INFO | Uploaded 36GiB to GPU in 2.13932s (16.8278GiB/s) | 20260506 00:38:08.892 | INFO | Downloaded 34GiB to GPU in 2.02748s (16.7696GiB/s) | 20260506 00:38:14.034 | INFO | Analyzing chunk frames = [7571456,7571968) | 20260506 00:38:14.034 | INFO | MemInfo: { | 20260506 00:38:14.034 | INFO | "gpu" : " 17 GB", | 20260506 00:38:14.034 | INFO | "huge" : "", | 20260506 00:38:14.034 | INFO | "huge_shared" : " 27 GB", | 20260506 00:38:14.034 | INFO | "malloc" : " 14 GB", | 20260506 00:38:14.034 | INFO | "pinned" : " 29 GB" | 20260506 00:38:14.034 | INFO | } | 20260506 00:38:14.034 | INFO | Perf report for the last 5.14182 seconds: | 20260506 00:38:14.034 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:38:14.034 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:14.034 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:14.034 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:14.034 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 00:38:14.034 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:14.034 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:14.034 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:38:14.034 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:14.034 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:14.034 | INFO | Uploaded 36GiB to GPU in 2.10536s (17.0992GiB/s) | 20260506 00:38:14.034 | INFO | Downloaded 34GiB to GPU in 2.05214s (16.568GiB/s) | 20260506 00:38:19.175 | INFO | Analyzing chunk frames = [7571968,7572480) | 20260506 00:38:19.175 | INFO | MemInfo: { | 20260506 00:38:19.175 | INFO | "gpu" : " 17 GB", | 20260506 00:38:19.175 | INFO | "huge" : "", | 20260506 00:38:19.175 | INFO | "huge_shared" : " 27 GB", | 20260506 00:38:19.175 | INFO | "malloc" : " 14 GB", | 20260506 00:38:19.175 | INFO | "pinned" : " 29 GB" | 20260506 00:38:19.175 | INFO | } | 20260506 00:38:19.175 | INFO | Perf report for the last 5.14145 seconds: | 20260506 00:38:19.175 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:38:19.175 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:19.175 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:19.175 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:19.175 | INFO | ANALYSIS : 61.3%, 3.0, 95.8ms, 0 | 20260506 00:38:19.175 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:19.175 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:19.175 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:38:19.175 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:19.175 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:19.176 | INFO | Uploaded 36GiB to GPU in 2.07189s (17.3754GiB/s) | 20260506 00:38:19.176 | INFO | Downloaded 34GiB to GPU in 2.0299s (16.7496GiB/s) | 20260506 00:38:24.320 | INFO | Analyzing chunk frames = [7572480,7572992) | 20260506 00:38:24.320 | INFO | MemInfo: { | 20260506 00:38:24.320 | INFO | "gpu" : " 17 GB", | 20260506 00:38:24.320 | INFO | "huge" : "", | 20260506 00:38:24.320 | INFO | "huge_shared" : " 27 GB", | 20260506 00:38:24.320 | INFO | "malloc" : " 14 GB", | 20260506 00:38:24.320 | INFO | "pinned" : " 29 GB" | 20260506 00:38:24.320 | INFO | } | 20260506 00:38:24.320 | INFO | Perf report for the last 5.14511 seconds: | 20260506 00:38:24.320 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:38:24.320 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:24.320 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:24.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:24.321 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260506 00:38:24.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:24.321 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:24.321 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 00:38:24.321 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:24.321 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:24.321 | INFO | Uploaded 36GiB to GPU in 2.01233s (17.8898GiB/s) | 20260506 00:38:24.321 | INFO | Downloaded 34GiB to GPU in 2.04195s (16.6507GiB/s) | 20260506 00:38:29.458 | INFO | Analyzing chunk frames = [7572992,7573504) | 20260506 00:38:29.458 | INFO | MemInfo: { | 20260506 00:38:29.458 | INFO | "gpu" : " 17 GB", | 20260506 00:38:29.458 | INFO | "huge" : "", | 20260506 00:38:29.458 | INFO | "huge_shared" : " 27 GB", | 20260506 00:38:29.458 | INFO | "malloc" : " 14 GB", | 20260506 00:38:29.458 | INFO | "pinned" : " 29 GB" | 20260506 00:38:29.458 | INFO | } | 20260506 00:38:29.458 | INFO | Perf report for the last 5.13803 seconds: | 20260506 00:38:29.458 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:38:29.458 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:29.459 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:29.459 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:29.459 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260506 00:38:29.459 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:29.459 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:29.459 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:38:29.459 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:29.459 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:29.459 | INFO | Uploaded 36GiB to GPU in 2.13659s (16.8493GiB/s) | 20260506 00:38:29.459 | INFO | Downloaded 34GiB to GPU in 2.02432s (16.7957GiB/s) | 20260506 00:38:29.577 | INFO | Starting a round of Baz IO | 20260506 00:38:32.716 | INFO | Populating RTMetrics report | 20260506 00:38:32.717 | INFO | RTMetrics report populated | 20260506 00:38:32.717 | INFO | Opening tmp RTMetrics json file | 20260506 00:38:32.717 | INFO | Serializing RTMetrics to json | 20260506 00:38:32.718 | INFO | Serialization done, swapping tmp file now... | 20260506 00:38:32.719 | INFO | RTMetrics done | 20260506 00:38:32.791 | INFO | Finished a round of Baz IO: Wrote 1.851 GiB in 3.214 seconds | 20260506 00:38:32.791 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.36% / 0.00% / 78.64% | 20260506 00:38:32.791 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0900308% / 7.94729e-06% / 7.94729e-05% | 20260506 00:38:32.791 | INFO | Baz IO overall: | 20260506 00:38:32.791 | INFO | BytesWritten : 4296884474189 | 20260506 00:38:32.791 | INFO | NumEvents : 2021597426988 | 20260506 00:38:34.592 | INFO | Analyzing chunk frames = [7573504,7574016) | 20260506 00:38:34.592 | INFO | MemInfo: { | 20260506 00:38:34.592 | INFO | "gpu" : " 17 GB", | 20260506 00:38:34.592 | INFO | "huge" : "", | 20260506 00:38:34.592 | INFO | "huge_shared" : " 27 GB", | 20260506 00:38:34.592 | INFO | "malloc" : " 14 GB", | 20260506 00:38:34.592 | INFO | "pinned" : " 29 GB" | 20260506 00:38:34.592 | INFO | } | 20260506 00:38:34.592 | INFO | Perf report for the last 5.13316 seconds: | 20260506 00:38:34.592 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:38:34.592 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:34.592 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:34.592 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:34.592 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260506 00:38:34.592 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:34.592 | INFO | RT_METRICS : 43.2%, 1.0, 22.6ms, 0 | 20260506 00:38:34.592 | INFO | PRE_HQ : 59.7%, 1.8, 54.2ms, 0 | 20260506 00:38:34.592 | INFO | BAZWRITER : 61.6%, 3.5, 110.4ms, 0 | 20260506 00:38:34.592 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:38:34.592 | INFO | Uploaded 36GiB to GPU in 2.17081s (16.5837GiB/s) | 20260506 00:38:34.592 | INFO | Downloaded 38GiB to GPU in 2.23811s (16.9786GiB/s) | 20260506 00:38:39.732 | INFO | Analyzing chunk frames = [7574016,7574528) | 20260506 00:38:39.732 | INFO | MemInfo: { | 20260506 00:38:39.732 | INFO | "gpu" : " 17 GB", | 20260506 00:38:39.732 | INFO | "huge" : "", | 20260506 00:38:39.732 | INFO | "huge_shared" : " 27 GB", | 20260506 00:38:39.732 | INFO | "malloc" : " 14 GB", | 20260506 00:38:39.732 | INFO | "pinned" : " 29 GB" | 20260506 00:38:39.732 | INFO | } | 20260506 00:38:39.732 | INFO | Perf report for the last 5.13995 seconds: | 20260506 00:38:39.732 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:38:39.732 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:39.732 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:39.732 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:39.732 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 00:38:39.732 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:39.732 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:39.732 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 00:38:39.732 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:39.732 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:39.732 | INFO | Uploaded 36GiB to GPU in 2.10589s (17.0949GiB/s) | 20260506 00:38:39.732 | INFO | Downloaded 34GiB to GPU in 2.03598s (16.6996GiB/s) | 20260506 00:38:44.871 | INFO | Analyzing chunk frames = [7574528,7575040) | 20260506 00:38:44.871 | INFO | MemInfo: { | 20260506 00:38:44.871 | INFO | "gpu" : " 17 GB", | 20260506 00:38:44.871 | INFO | "huge" : "", | 20260506 00:38:44.871 | INFO | "huge_shared" : " 27 GB", | 20260506 00:38:44.871 | INFO | "malloc" : " 14 GB", | 20260506 00:38:44.871 | INFO | "pinned" : " 29 GB" | 20260506 00:38:44.871 | INFO | } | 20260506 00:38:44.871 | INFO | Perf report for the last 5.13985 seconds: | 20260506 00:38:44.871 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:38:44.871 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:44.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:44.872 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:44.872 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 00:38:44.872 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:44.872 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:44.872 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:38:44.872 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:44.872 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:44.872 | INFO | Uploaded 36GiB to GPU in 2.07878s (17.3179GiB/s) | 20260506 00:38:44.872 | INFO | Downloaded 34GiB to GPU in 2.01663s (16.8598GiB/s) | 20260506 00:38:50.012 | INFO | Analyzing chunk frames = [7575040,7575552) | 20260506 00:38:50.012 | INFO | MemInfo: { | 20260506 00:38:50.012 | INFO | "gpu" : " 17 GB", | 20260506 00:38:50.012 | INFO | "huge" : "", | 20260506 00:38:50.013 | INFO | "huge_shared" : " 27 GB", | 20260506 00:38:50.013 | INFO | "malloc" : " 14 GB", | 20260506 00:38:50.013 | INFO | "pinned" : " 29 GB" | 20260506 00:38:50.013 | INFO | } | 20260506 00:38:50.013 | INFO | Perf report for the last 5.14106 seconds: | 20260506 00:38:50.013 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:38:50.013 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:50.013 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:50.013 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:50.013 | INFO | ANALYSIS : 61.6%, 3.0, 95.6ms, 0 | 20260506 00:38:50.013 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:50.013 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:50.013 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 00:38:50.013 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:50.013 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:50.013 | INFO | Uploaded 36GiB to GPU in 2.13969s (16.8249GiB/s) | 20260506 00:38:50.013 | INFO | Downloaded 34GiB to GPU in 2.01576s (16.8671GiB/s) | 20260506 00:38:55.148 | INFO | Analyzing chunk frames = [7575552,7576064) | 20260506 00:38:55.148 | INFO | MemInfo: { | 20260506 00:38:55.148 | INFO | "gpu" : " 17 GB", | 20260506 00:38:55.148 | INFO | "huge" : "", | 20260506 00:38:55.148 | INFO | "huge_shared" : " 27 GB", | 20260506 00:38:55.148 | INFO | "malloc" : " 14 GB", | 20260506 00:38:55.148 | INFO | "pinned" : " 29 GB" | 20260506 00:38:55.148 | INFO | } | 20260506 00:38:55.148 | INFO | Perf report for the last 5.13548 seconds: | 20260506 00:38:55.148 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:38:55.148 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:55.148 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:55.148 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:55.148 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 00:38:55.148 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:55.148 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:55.148 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:38:55.148 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:55.148 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:38:55.148 | INFO | Uploaded 36GiB to GPU in 2.09471s (17.1862GiB/s) | 20260506 00:38:55.148 | INFO | Downloaded 34GiB to GPU in 2.0486s (16.5967GiB/s) | 20260506 00:39:00.289 | INFO | Analyzing chunk frames = [7576064,7576576) | 20260506 00:39:00.289 | INFO | MemInfo: { | 20260506 00:39:00.289 | INFO | "gpu" : " 17 GB", | 20260506 00:39:00.289 | INFO | "huge" : "", | 20260506 00:39:00.289 | INFO | "huge_shared" : " 27 GB", | 20260506 00:39:00.289 | INFO | "malloc" : " 14 GB", | 20260506 00:39:00.289 | INFO | "pinned" : " 29 GB" | 20260506 00:39:00.289 | INFO | } | 20260506 00:39:00.289 | INFO | Perf report for the last 5.14103 seconds: | 20260506 00:39:00.289 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:39:00.289 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:00.289 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:00.289 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:00.289 | INFO | ANALYSIS : 61.4%, 3.0, 95.9ms, 0 | 20260506 00:39:00.289 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:00.289 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:00.289 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:39:00.289 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:00.289 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:00.289 | INFO | Uploaded 36GiB to GPU in 2.07996s (17.308GiB/s) | 20260506 00:39:00.289 | INFO | Downloaded 34GiB to GPU in 2.03432s (16.7132GiB/s) | 20260506 00:39:05.432 | INFO | Analyzing chunk frames = [7576576,7577088) | 20260506 00:39:05.432 | INFO | MemInfo: { | 20260506 00:39:05.432 | INFO | "gpu" : " 17 GB", | 20260506 00:39:05.432 | INFO | "huge" : "", | 20260506 00:39:05.432 | INFO | "huge_shared" : " 27 GB", | 20260506 00:39:05.432 | INFO | "malloc" : " 14 GB", | 20260506 00:39:05.432 | INFO | "pinned" : " 29 GB" | 20260506 00:39:05.432 | INFO | } | 20260506 00:39:05.432 | INFO | Perf report for the last 5.14295 seconds: | 20260506 00:39:05.432 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:39:05.432 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:05.432 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:05.432 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:05.432 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260506 00:39:05.432 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:05.432 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:05.432 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 00:39:05.432 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:05.432 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:05.432 | INFO | Uploaded 36GiB to GPU in 2.11402s (17.0292GiB/s) | 20260506 00:39:05.432 | INFO | Downloaded 34GiB to GPU in 2.08203s (16.3302GiB/s) | 20260506 00:39:10.571 | INFO | Analyzing chunk frames = [7577088,7577600) | 20260506 00:39:10.571 | INFO | MemInfo: { | 20260506 00:39:10.571 | INFO | "gpu" : " 17 GB", | 20260506 00:39:10.571 | INFO | "huge" : "", | 20260506 00:39:10.571 | INFO | "huge_shared" : " 27 GB", | 20260506 00:39:10.571 | INFO | "malloc" : " 14 GB", | 20260506 00:39:10.571 | INFO | "pinned" : " 29 GB" | 20260506 00:39:10.571 | INFO | } | 20260506 00:39:10.571 | INFO | Perf report for the last 5.13887 seconds: | 20260506 00:39:10.571 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:39:10.571 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:10.571 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:10.571 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:10.571 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260506 00:39:10.571 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:10.571 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:10.571 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:39:10.571 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:10.571 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:10.571 | INFO | Uploaded 36GiB to GPU in 2.13695s (16.8464GiB/s) | 20260506 00:39:10.571 | INFO | Downloaded 34GiB to GPU in 2.02348s (16.8027GiB/s) | 20260506 00:39:10.571 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 00:39:10.571 | INFO | Total mem usage: 15.3 GiB | 20260506 00:39:10.571 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 00:39:10.571 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.16%) | 20260506 00:39:10.571 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 00:39:10.571 | INFO | High Water Breakdown: | 20260506 00:39:10.571 | INFO | 60.9% : 8.55 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 00:39:10.571 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 00:39:10.571 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 00:39:10.571 | INFO | ------------------------------------------ | 20260506 00:39:10.571 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:39:10.571 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 00:39:10.571 | INFO | Total mem usage: 26.5 GiB | 20260506 00:39:10.571 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 00:39:10.571 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 00:39:10.571 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260506 00:39:10.571 | INFO | High Water Breakdown: | 20260506 00:39:10.571 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:39:10.571 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:39:10.571 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:39:10.571 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:39:10.571 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:39:10.571 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:39:10.571 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 00:39:10.571 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 00:39:10.571 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:39:10.571 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 00:39:10.572 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 00:39:10.572 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 00:39:10.572 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 00:39:10.572 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 00:39:10.572 | INFO | ------------------------------------------ | 20260506 00:39:10.572 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:39:10.572 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 00:39:10.572 | INFO | Total mem usage: 25.6 GiB | 20260506 00:39:10.572 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 00:39:10.572 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 00:39:10.572 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 00:39:10.572 | INFO | High Water Breakdown: | 20260506 00:39:10.572 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 00:39:10.572 | INFO | ------------------------------------------ | 20260506 00:39:10.572 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:39:10.572 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 00:39:10.572 | INFO | Total mem usage: 15.7 GiB | 20260506 00:39:10.572 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 00:39:10.572 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 00:39:10.572 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 00:39:10.572 | INFO | High Water Breakdown: | 20260506 00:39:10.572 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 00:39:10.572 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 00:39:10.572 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 00:39:10.572 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 00:39:10.572 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 00:39:10.572 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 00:39:10.572 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 00:39:10.572 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 00:39:10.572 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 00:39:10.572 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:39:10.572 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 00:39:10.572 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:39:10.572 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 00:39:10.572 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 00:39:10.572 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 00:39:10.572 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 00:39:10.572 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:39:10.572 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 00:39:10.572 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:39:10.572 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:39:10.572 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:39:10.572 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 00:39:10.572 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 00:39:10.572 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:39:10.572 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 00:39:10.572 | INFO | ------------------------------------------ | 20260506 00:39:10.572 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:39:10.572 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 00:39:10.572 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:39:10.572 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:39:10.572 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.40e+01, 1.33e+01 / 7.87e-01 | 20260506 00:39:10.572 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.06e-01, 1.24e-02 / 1.68e-02 | 20260506 00:39:10.572 | INFO | PerfReport: 11.84%, Baseline -- 3.70e+00 / 4.52e+00, 5.69e-01 / 1.78e-01 | 20260506 00:39:10.572 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.53e+00, 1.87e-01 / 8.77e-02 | 20260506 00:39:10.572 | INFO | PerfReport: 16.02%, DME -- 4.01e+01 / 4.41e+01, 3.87e+00 / 1.10e+00 | 20260506 00:39:10.572 | INFO | PerfReport: 57.31%, FrameLabeling -- 1.79e+01 / 1.94e+01, 1.42e+00 / 1.03e+00 | 20260506 00:39:10.572 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.23e+00 / 2.77e+00, 4.27e-01 / 2.23e-01 | 20260506 00:39:10.572 | INFO | PerfReport: 3.08%, Metrics -- 9.63e-01 / 1.25e+00, 2.61e-01 / 1.16e-01 | 20260506 00:39:10.572 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.32e-01 / 3.92e-01, 5.58e-02 / 2.77e-02 | 20260506 00:39:10.572 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 00:39:10.572 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:39:10.572 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:39:10.572 | INFO | PerfReport: Full Execution -- 9.82e+01 / -nan, 2.44e+01 / 0.00e+00 | 20260506 00:39:10.572 | INFO | PerfReport: 22.37%, Upload -- 2.20e+01 / 0.00e+00, 7.51e+00 / 0.00e+00 | 20260506 00:39:10.572 | INFO | PerfReport: 21.85%, Download -- 2.15e+01 / 0.00e+00, 7.60e+00 / 0.00e+00 | 20260506 00:39:10.572 | INFO | PerfReport: 31.83%, Compute -- 3.13e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 00:39:10.572 | INFO | PerfReport: 2.14%, LockUpload -- 2.10e+00 / 0.00e+00, 7.89e+00 / 0.00e+00 | 20260506 00:39:10.572 | INFO | PerfReport: 1.31%, LockDownload -- 1.29e+00 / 0.00e+00, 2.87e+00 / 0.00e+00 | 20260506 00:39:10.572 | INFO | PerfReport: 20.48%, LockCompute -- 2.01e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 00:39:10.572 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.89e-02 / 0.00e+00 | 20260506 00:39:10.687 | INFO | Starting a round of Baz IO | 20260506 00:39:13.835 | INFO | Populating RTMetrics report | 20260506 00:39:13.836 | INFO | RTMetrics report populated | 20260506 00:39:13.836 | INFO | Opening tmp RTMetrics json file | 20260506 00:39:13.836 | INFO | Serializing RTMetrics to json | 20260506 00:39:13.838 | INFO | Serialization done, swapping tmp file now... | 20260506 00:39:13.839 | INFO | RTMetrics done | 20260506 00:39:13.915 | INFO | Finished a round of Baz IO: Wrote 1.851 GiB in 3.227 seconds | 20260506 00:39:13.915 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.36% / 0.00% / 78.64% | 20260506 00:39:13.915 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0899156% / 0% / 6.75519e-05% | 20260506 00:39:13.915 | INFO | Baz IO overall: | 20260506 00:39:13.915 | INFO | BytesWritten : 4298872426623 | 20260506 00:39:13.915 | INFO | NumEvents : 2022505949291 | 20260506 00:39:15.707 | INFO | Analyzing chunk frames = [7577600,7578112) | 20260506 00:39:15.707 | INFO | MemInfo: { | 20260506 00:39:15.707 | INFO | "gpu" : " 17 GB", | 20260506 00:39:15.707 | INFO | "huge" : "", | 20260506 00:39:15.707 | INFO | "huge_shared" : " 27 GB", | 20260506 00:39:15.707 | INFO | "malloc" : " 14 GB", | 20260506 00:39:15.707 | INFO | "pinned" : " 29 GB" | 20260506 00:39:15.707 | INFO | } | 20260506 00:39:15.707 | INFO | Perf report for the last 5.13621 seconds: | 20260506 00:39:15.707 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:39:15.707 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:15.707 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:15.707 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:15.707 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260506 00:39:15.707 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:15.707 | INFO | RT_METRICS : 43.2%, 1.0, 22.6ms, 0 | 20260506 00:39:15.707 | INFO | PRE_HQ : 59.8%, 1.7, 53.8ms, 0 | 20260506 00:39:15.707 | INFO | BAZWRITER : 61.9%, 3.5, 110.4ms, 0 | 20260506 00:39:15.707 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:39:15.707 | INFO | Uploaded 36GiB to GPU in 2.16643s (16.6172GiB/s) | 20260506 00:39:15.707 | INFO | Downloaded 38GiB to GPU in 2.23974s (16.9663GiB/s) | 20260506 00:39:20.844 | INFO | Analyzing chunk frames = [7578112,7578624) | 20260506 00:39:20.844 | INFO | MemInfo: { | 20260506 00:39:20.844 | INFO | "gpu" : " 17 GB", | 20260506 00:39:20.844 | INFO | "huge" : "", | 20260506 00:39:20.845 | INFO | "huge_shared" : " 27 GB", | 20260506 00:39:20.845 | INFO | "malloc" : " 14 GB", | 20260506 00:39:20.845 | INFO | "pinned" : " 29 GB" | 20260506 00:39:20.845 | INFO | } | 20260506 00:39:20.845 | INFO | Perf report for the last 5.13743 seconds: | 20260506 00:39:20.845 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:39:20.845 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:20.845 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:20.845 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:20.845 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 00:39:20.845 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:20.845 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:20.845 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 00:39:20.845 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:20.845 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:20.845 | INFO | Uploaded 36GiB to GPU in 2.10215s (17.1254GiB/s) | 20260506 00:39:20.845 | INFO | Downloaded 34GiB to GPU in 2.0406s (16.6617GiB/s) | 20260506 00:39:25.986 | INFO | Analyzing chunk frames = [7578624,7579136) | 20260506 00:39:25.986 | INFO | MemInfo: { | 20260506 00:39:25.986 | INFO | "gpu" : " 17 GB", | 20260506 00:39:25.986 | INFO | "huge" : "", | 20260506 00:39:25.986 | INFO | "huge_shared" : " 27 GB", | 20260506 00:39:25.986 | INFO | "malloc" : " 14 GB", | 20260506 00:39:25.986 | INFO | "pinned" : " 29 GB" | 20260506 00:39:25.986 | INFO | } | 20260506 00:39:25.986 | INFO | Perf report for the last 5.14139 seconds: | 20260506 00:39:25.986 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:39:25.986 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:25.986 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:25.986 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:25.986 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 00:39:25.986 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:25.986 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:25.986 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 00:39:25.986 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:25.986 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:25.986 | INFO | Uploaded 36GiB to GPU in 2.15141s (16.7332GiB/s) | 20260506 00:39:25.986 | INFO | Downloaded 34GiB to GPU in 2.05553s (16.5407GiB/s) | 20260506 00:39:31.129 | INFO | Analyzing chunk frames = [7579136,7579648) | 20260506 00:39:31.129 | INFO | MemInfo: { | 20260506 00:39:31.129 | INFO | "gpu" : " 17 GB", | 20260506 00:39:31.129 | INFO | "huge" : "", | 20260506 00:39:31.129 | INFO | "huge_shared" : " 27 GB", | 20260506 00:39:31.129 | INFO | "malloc" : " 14 GB", | 20260506 00:39:31.129 | INFO | "pinned" : " 29 GB" | 20260506 00:39:31.129 | INFO | } | 20260506 00:39:31.129 | INFO | Perf report for the last 5.14298 seconds: | 20260506 00:39:31.129 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:39:31.129 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:31.129 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:31.129 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:31.129 | INFO | ANALYSIS : 61.6%, 3.0, 95.7ms, 0 | 20260506 00:39:31.129 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:31.129 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:31.129 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:39:31.129 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:31.129 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:31.129 | INFO | Uploaded 36GiB to GPU in 2.13768s (16.8407GiB/s) | 20260506 00:39:31.129 | INFO | Downloaded 34GiB to GPU in 2.02247s (16.8111GiB/s) | 20260506 00:39:36.270 | INFO | Analyzing chunk frames = [7579648,7580160) | 20260506 00:39:36.270 | INFO | MemInfo: { | 20260506 00:39:36.270 | INFO | "gpu" : " 17 GB", | 20260506 00:39:36.270 | INFO | "huge" : "", | 20260506 00:39:36.270 | INFO | "huge_shared" : " 27 GB", | 20260506 00:39:36.270 | INFO | "malloc" : " 14 GB", | 20260506 00:39:36.270 | INFO | "pinned" : " 29 GB" | 20260506 00:39:36.270 | INFO | } | 20260506 00:39:36.270 | INFO | Perf report for the last 5.14087 seconds: | 20260506 00:39:36.270 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:39:36.270 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:36.270 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:36.270 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:36.270 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 00:39:36.270 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:36.270 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:36.270 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 00:39:36.270 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:36.270 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:36.270 | INFO | Uploaded 36GiB to GPU in 2.09672s (17.1697GiB/s) | 20260506 00:39:36.270 | INFO | Downloaded 34GiB to GPU in 2.0362s (16.6977GiB/s) | 20260506 00:39:41.408 | INFO | Analyzing chunk frames = [7580160,7580672) | 20260506 00:39:41.408 | INFO | MemInfo: { | 20260506 00:39:41.408 | INFO | "gpu" : " 17 GB", | 20260506 00:39:41.408 | INFO | "huge" : "", | 20260506 00:39:41.408 | INFO | "huge_shared" : " 27 GB", | 20260506 00:39:41.408 | INFO | "malloc" : " 14 GB", | 20260506 00:39:41.408 | INFO | "pinned" : " 29 GB" | 20260506 00:39:41.408 | INFO | } | 20260506 00:39:41.408 | INFO | Perf report for the last 5.13846 seconds: | 20260506 00:39:41.408 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:39:41.408 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:41.408 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:41.408 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:41.408 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 00:39:41.408 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:41.408 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:41.409 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:39:41.409 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:41.409 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:41.409 | INFO | Uploaded 36GiB to GPU in 2.06897s (17.4GiB/s) | 20260506 00:39:41.409 | INFO | Downloaded 34GiB to GPU in 2.03825s (16.681GiB/s) | 20260506 00:39:46.542 | INFO | Analyzing chunk frames = [7580672,7581184) | 20260506 00:39:46.542 | INFO | MemInfo: { | 20260506 00:39:46.542 | INFO | "gpu" : " 17 GB", | 20260506 00:39:46.542 | INFO | "huge" : "", | 20260506 00:39:46.542 | INFO | "huge_shared" : " 27 GB", | 20260506 00:39:46.542 | INFO | "malloc" : " 14 GB", | 20260506 00:39:46.542 | INFO | "pinned" : " 29 GB" | 20260506 00:39:46.542 | INFO | } | 20260506 00:39:46.542 | INFO | Perf report for the last 5.13408 seconds: | 20260506 00:39:46.542 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:39:46.542 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:46.542 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:46.543 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:46.543 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260506 00:39:46.543 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:46.543 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:46.543 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 00:39:46.543 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:46.543 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:46.543 | INFO | Uploaded 36GiB to GPU in 2.10444s (17.1066GiB/s) | 20260506 00:39:46.543 | INFO | Downloaded 34GiB to GPU in 2.07079s (16.4188GiB/s) | 20260506 00:39:51.683 | INFO | Analyzing chunk frames = [7581184,7581696) | 20260506 00:39:51.683 | INFO | MemInfo: { | 20260506 00:39:51.683 | INFO | "gpu" : " 17 GB", | 20260506 00:39:51.683 | INFO | "huge" : "", | 20260506 00:39:51.683 | INFO | "huge_shared" : " 27 GB", | 20260506 00:39:51.683 | INFO | "malloc" : " 14 GB", | 20260506 00:39:51.683 | INFO | "pinned" : " 29 GB" | 20260506 00:39:51.683 | INFO | } | 20260506 00:39:51.683 | INFO | Perf report for the last 5.1408 seconds: | 20260506 00:39:51.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:39:51.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:51.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:51.683 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:51.683 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260506 00:39:51.683 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:51.683 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:51.683 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:39:51.683 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:51.683 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:51.684 | INFO | Uploaded 36GiB to GPU in 2.12961s (16.9045GiB/s) | 20260506 00:39:51.684 | INFO | Downloaded 34GiB to GPU in 2.01389s (16.8827GiB/s) | 20260506 00:39:51.801 | INFO | Starting a round of Baz IO | 20260506 00:39:54.953 | INFO | Populating RTMetrics report | 20260506 00:39:54.954 | INFO | RTMetrics report populated | 20260506 00:39:54.954 | INFO | Opening tmp RTMetrics json file | 20260506 00:39:54.954 | INFO | Serializing RTMetrics to json | 20260506 00:39:54.956 | INFO | Serialization done, swapping tmp file now... | 20260506 00:39:54.956 | INFO | RTMetrics done | 20260506 00:39:55.027 | INFO | Finished a round of Baz IO: Wrote 1.851 GiB in 3.226 seconds | 20260506 00:39:55.027 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.36% / 0.00% / 78.64% | 20260506 00:39:55.027 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0893394% / 0% / 4.76837e-05% | 20260506 00:39:55.027 | INFO | Baz IO overall: | 20260506 00:39:55.027 | INFO | BytesWritten : 4300859970832 | 20260506 00:39:55.027 | INFO | NumEvents : 2023414178270 | 20260506 00:39:56.823 | INFO | Analyzing chunk frames = [7581696,7582208) | 20260506 00:39:56.823 | INFO | MemInfo: { | 20260506 00:39:56.823 | INFO | "gpu" : " 17 GB", | 20260506 00:39:56.823 | INFO | "huge" : "", | 20260506 00:39:56.823 | INFO | "huge_shared" : " 27 GB", | 20260506 00:39:56.823 | INFO | "malloc" : " 14 GB", | 20260506 00:39:56.823 | INFO | "pinned" : " 29 GB" | 20260506 00:39:56.823 | INFO | } | 20260506 00:39:56.823 | INFO | Perf report for the last 5.13999 seconds: | 20260506 00:39:56.823 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:39:56.823 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:56.823 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:56.823 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:56.823 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260506 00:39:56.823 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:39:56.823 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260506 00:39:56.823 | INFO | PRE_HQ : 59.9%, 1.7, 53.8ms, 0 | 20260506 00:39:56.823 | INFO | BAZWRITER : 61.9%, 3.5, 110.8ms, 0 | 20260506 00:39:56.823 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:39:56.824 | INFO | Uploaded 36GiB to GPU in 2.16791s (16.6059GiB/s) | 20260506 00:39:56.824 | INFO | Downloaded 38GiB to GPU in 2.23939s (16.9689GiB/s) | 20260506 00:40:01.960 | INFO | Analyzing chunk frames = [7582208,7582720) | 20260506 00:40:01.960 | INFO | MemInfo: { | 20260506 00:40:01.960 | INFO | "gpu" : " 17 GB", | 20260506 00:40:01.960 | INFO | "huge" : "", | 20260506 00:40:01.960 | INFO | "huge_shared" : " 27 GB", | 20260506 00:40:01.960 | INFO | "malloc" : " 14 GB", | 20260506 00:40:01.960 | INFO | "pinned" : " 29 GB" | 20260506 00:40:01.960 | INFO | } | 20260506 00:40:01.961 | INFO | Perf report for the last 5.13725 seconds: | 20260506 00:40:01.961 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:40:01.961 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:01.961 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:01.961 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:01.961 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 00:40:01.961 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:01.961 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:01.961 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 00:40:01.961 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:01.961 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:01.961 | INFO | Uploaded 36GiB to GPU in 2.11361s (17.0325GiB/s) | 20260506 00:40:01.961 | INFO | Downloaded 34GiB to GPU in 2.04496s (16.6262GiB/s) | 20260506 00:40:07.098 | INFO | Analyzing chunk frames = [7582720,7583232) | 20260506 00:40:07.098 | INFO | MemInfo: { | 20260506 00:40:07.098 | INFO | "gpu" : " 17 GB", | 20260506 00:40:07.098 | INFO | "huge" : "", | 20260506 00:40:07.098 | INFO | "huge_shared" : " 27 GB", | 20260506 00:40:07.098 | INFO | "malloc" : " 14 GB", | 20260506 00:40:07.098 | INFO | "pinned" : " 29 GB" | 20260506 00:40:07.098 | INFO | } | 20260506 00:40:07.098 | INFO | Perf report for the last 5.13779 seconds: | 20260506 00:40:07.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:40:07.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:07.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:07.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:07.099 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 00:40:07.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:07.099 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:07.099 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 00:40:07.099 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:07.099 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:07.099 | INFO | Uploaded 36GiB to GPU in 2.09692s (17.168GiB/s) | 20260506 00:40:07.099 | INFO | Downloaded 34GiB to GPU in 2.03227s (16.7301GiB/s) | 20260506 00:40:12.242 | INFO | Analyzing chunk frames = [7583232,7583744) | 20260506 00:40:12.242 | INFO | MemInfo: { | 20260506 00:40:12.242 | INFO | "gpu" : " 17 GB", | 20260506 00:40:12.242 | INFO | "huge" : "", | 20260506 00:40:12.242 | INFO | "huge_shared" : " 27 GB", | 20260506 00:40:12.242 | INFO | "malloc" : " 14 GB", | 20260506 00:40:12.242 | INFO | "pinned" : " 29 GB" | 20260506 00:40:12.242 | INFO | } | 20260506 00:40:12.242 | INFO | Perf report for the last 5.14379 seconds: | 20260506 00:40:12.242 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:40:12.242 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:12.242 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:12.242 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:12.242 | INFO | ANALYSIS : 61.6%, 3.0, 95.7ms, 0 | 20260506 00:40:12.242 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:12.242 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:12.242 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 00:40:12.242 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:12.242 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:12.243 | INFO | Uploaded 36GiB to GPU in 2.13041s (16.8982GiB/s) | 20260506 00:40:12.243 | INFO | Downloaded 34GiB to GPU in 2.00608s (16.9485GiB/s) | 20260506 00:40:17.385 | INFO | Analyzing chunk frames = [7583744,7584256) | 20260506 00:40:17.385 | INFO | MemInfo: { | 20260506 00:40:17.385 | INFO | "gpu" : " 17 GB", | 20260506 00:40:17.385 | INFO | "huge" : "", | 20260506 00:40:17.385 | INFO | "huge_shared" : " 27 GB", | 20260506 00:40:17.385 | INFO | "malloc" : " 14 GB", | 20260506 00:40:17.385 | INFO | "pinned" : " 29 GB" | 20260506 00:40:17.385 | INFO | } | 20260506 00:40:17.385 | INFO | Perf report for the last 5.14237 seconds: | 20260506 00:40:17.385 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:40:17.385 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:17.385 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:17.385 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:17.385 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 00:40:17.385 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:17.385 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:17.385 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 00:40:17.385 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:17.385 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:17.385 | INFO | Uploaded 36GiB to GPU in 2.09598s (17.1757GiB/s) | 20260506 00:40:17.385 | INFO | Downloaded 34GiB to GPU in 2.04958s (16.5887GiB/s) | 20260506 00:40:22.520 | INFO | Analyzing chunk frames = [7584256,7584768) | 20260506 00:40:22.520 | INFO | MemInfo: { | 20260506 00:40:22.520 | INFO | "gpu" : " 17 GB", | 20260506 00:40:22.520 | INFO | "huge" : "", | 20260506 00:40:22.520 | INFO | "huge_shared" : " 27 GB", | 20260506 00:40:22.520 | INFO | "malloc" : " 14 GB", | 20260506 00:40:22.520 | INFO | "pinned" : " 29 GB" | 20260506 00:40:22.520 | INFO | } | 20260506 00:40:22.520 | INFO | Perf report for the last 5.13582 seconds: | 20260506 00:40:22.520 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:40:22.520 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:22.520 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:22.520 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:22.520 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 00:40:22.520 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:22.520 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:22.520 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 00:40:22.520 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:22.520 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:22.521 | INFO | Uploaded 36GiB to GPU in 2.07294s (17.3666GiB/s) | 20260506 00:40:22.521 | INFO | Downloaded 34GiB to GPU in 2.03692s (16.6919GiB/s) | 20260506 00:40:27.655 | INFO | Analyzing chunk frames = [7584768,7585280) | 20260506 00:40:27.655 | INFO | MemInfo: { | 20260506 00:40:27.655 | INFO | "gpu" : " 17 GB", | 20260506 00:40:27.655 | INFO | "huge" : "", | 20260506 00:40:27.655 | INFO | "huge_shared" : " 27 GB", | 20260506 00:40:27.655 | INFO | "malloc" : " 14 GB", | 20260506 00:40:27.655 | INFO | "pinned" : " 29 GB" | 20260506 00:40:27.655 | INFO | } | 20260506 00:40:27.655 | INFO | Perf report for the last 5.13494 seconds: | 20260506 00:40:27.655 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:40:27.655 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:27.655 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:27.655 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:27.655 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260506 00:40:27.655 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:27.655 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:27.655 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 00:40:27.655 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:27.655 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:27.656 | INFO | Uploaded 36GiB to GPU in 2.07325s (17.364GiB/s) | 20260506 00:40:27.656 | INFO | Downloaded 34GiB to GPU in 2.06251s (16.4848GiB/s) | 20260506 00:40:32.796 | INFO | Analyzing chunk frames = [7585280,7585792) | 20260506 00:40:32.796 | INFO | MemInfo: { | 20260506 00:40:32.796 | INFO | "gpu" : " 17 GB", | 20260506 00:40:32.796 | INFO | "huge" : "", | 20260506 00:40:32.796 | INFO | "huge_shared" : " 27 GB", | 20260506 00:40:32.796 | INFO | "malloc" : " 14 GB", | 20260506 00:40:32.796 | INFO | "pinned" : " 29 GB" | 20260506 00:40:32.797 | INFO | } | 20260506 00:40:32.797 | INFO | Perf report for the last 5.14118 seconds: | 20260506 00:40:32.797 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:40:32.797 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:32.797 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:32.797 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:32.797 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260506 00:40:32.797 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:32.797 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:32.797 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 00:40:32.797 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:32.797 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:32.797 | INFO | Uploaded 36GiB to GPU in 2.11847s (16.9934GiB/s) | 20260506 00:40:32.797 | INFO | Downloaded 34GiB to GPU in 2.00804s (16.932GiB/s) | 20260506 00:40:32.797 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 00:40:32.797 | INFO | Total mem usage: 15.3 GiB | 20260506 00:40:32.797 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 00:40:32.797 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.15%) | 20260506 00:40:32.797 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 00:40:32.797 | INFO | High Water Breakdown: | 20260506 00:40:32.797 | INFO | 60.9% : 8.55 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 00:40:32.797 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 00:40:32.797 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 00:40:32.797 | INFO | ------------------------------------------ | 20260506 00:40:32.797 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:40:32.797 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 00:40:32.797 | INFO | Total mem usage: 26.5 GiB | 20260506 00:40:32.797 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 00:40:32.797 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 00:40:32.797 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260506 00:40:32.797 | INFO | High Water Breakdown: | 20260506 00:40:32.797 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:40:32.797 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:40:32.797 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:40:32.797 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:40:32.797 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:40:32.797 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:40:32.797 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 00:40:32.797 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 00:40:32.797 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:40:32.797 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 00:40:32.797 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 00:40:32.797 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 00:40:32.797 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 00:40:32.797 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 00:40:32.797 | INFO | ------------------------------------------ | 20260506 00:40:32.797 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:40:32.797 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 00:40:32.797 | INFO | Total mem usage: 25.6 GiB | 20260506 00:40:32.797 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 00:40:32.797 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 00:40:32.797 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 00:40:32.797 | INFO | High Water Breakdown: | 20260506 00:40:32.797 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 00:40:32.797 | INFO | ------------------------------------------ | 20260506 00:40:32.797 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:40:32.797 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 00:40:32.797 | INFO | Total mem usage: 15.7 GiB | 20260506 00:40:32.797 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 00:40:32.797 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 00:40:32.797 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 00:40:32.797 | INFO | High Water Breakdown: | 20260506 00:40:32.797 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 00:40:32.797 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 00:40:32.797 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 00:40:32.797 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 00:40:32.797 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 00:40:32.797 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 00:40:32.797 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 00:40:32.798 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 00:40:32.798 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 00:40:32.798 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:40:32.798 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 00:40:32.798 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:40:32.798 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 00:40:32.798 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 00:40:32.798 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 00:40:32.798 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 00:40:32.798 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:40:32.798 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 00:40:32.798 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:40:32.798 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:40:32.798 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:40:32.798 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 00:40:32.798 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 00:40:32.798 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:40:32.798 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 00:40:32.798 | INFO | ------------------------------------------ | 20260506 00:40:32.798 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:40:32.798 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 00:40:32.798 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:40:32.798 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:40:32.798 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.41e+01, 1.36e+01 / 6.72e-01 | 20260506 00:40:32.798 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.03e-01, 1.05e-02 / 4.08e-03 | 20260506 00:40:32.798 | INFO | PerfReport: 11.83%, Baseline -- 3.70e+00 / 4.52e+00, 5.65e-01 / 1.71e-01 | 20260506 00:40:32.798 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.52e+00, 1.89e-01 / 9.90e-02 | 20260506 00:40:32.798 | INFO | PerfReport: 16.00%, DME -- 4.01e+01 / 4.39e+01, 3.82e+00 / 8.44e-01 | 20260506 00:40:32.798 | INFO | PerfReport: 57.38%, FrameLabeling -- 1.80e+01 / 1.96e+01, 1.48e+00 / 1.21e+00 | 20260506 00:40:32.798 | INFO | PerfReport: 7.08%, PulseAccumulating -- 2.22e+00 / 2.74e+00, 4.19e-01 / 2.56e-01 | 20260506 00:40:32.798 | INFO | PerfReport: 3.07%, Metrics -- 9.62e-01 / 1.25e+00, 2.61e-01 / 1.21e-01 | 20260506 00:40:32.798 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.33e-01 / 3.90e-01, 5.71e-02 / 3.28e-02 | 20260506 00:40:32.798 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 00:40:32.798 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:40:32.798 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:40:32.798 | INFO | PerfReport: Full Execution -- 9.83e+01 / -nan, 2.47e+01 / 0.00e+00 | 20260506 00:40:32.798 | INFO | PerfReport: 22.41%, Upload -- 2.20e+01 / 0.00e+00, 7.54e+00 / 0.00e+00 | 20260506 00:40:32.798 | INFO | PerfReport: 21.86%, Download -- 2.15e+01 / 0.00e+00, 7.60e+00 / 0.00e+00 | 20260506 00:40:32.798 | INFO | PerfReport: 31.85%, Compute -- 3.13e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 00:40:32.798 | INFO | PerfReport: 2.13%, LockUpload -- 2.10e+00 / 0.00e+00, 7.93e+00 / 0.00e+00 | 20260506 00:40:32.798 | INFO | PerfReport: 1.26%, LockDownload -- 1.24e+00 / 0.00e+00, 2.78e+00 / 0.00e+00 | 20260506 00:40:32.798 | INFO | PerfReport: 20.48%, LockCompute -- 2.01e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 00:40:32.798 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.85e-02 / 0.00e+00 | 20260506 00:40:32.917 | INFO | Starting a round of Baz IO | 20260506 00:40:36.065 | INFO | Populating RTMetrics report | 20260506 00:40:36.065 | INFO | RTMetrics report populated | 20260506 00:40:36.065 | INFO | Opening tmp RTMetrics json file | 20260506 00:40:36.066 | INFO | Serializing RTMetrics to json | 20260506 00:40:36.068 | INFO | Serialization done, swapping tmp file now... | 20260506 00:40:36.068 | INFO | RTMetrics done | 20260506 00:40:36.142 | INFO | Finished a round of Baz IO: Wrote 1.851 GiB in 3.224 seconds | 20260506 00:40:36.142 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.36% / 0.00% / 78.64% | 20260506 00:40:36.142 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0892798% / 0% / 6.75519e-05% | 20260506 00:40:36.142 | INFO | Baz IO overall: | 20260506 00:40:36.142 | INFO | BytesWritten : 4302847030806 | 20260506 00:40:36.142 | INFO | NumEvents : 2024322086443 | 20260506 00:40:37.940 | INFO | Analyzing chunk frames = [7585792,7586304) | 20260506 00:40:37.940 | INFO | MemInfo: { | 20260506 00:40:37.940 | INFO | "gpu" : " 17 GB", | 20260506 00:40:37.940 | INFO | "huge" : "", | 20260506 00:40:37.940 | INFO | "huge_shared" : " 27 GB", | 20260506 00:40:37.940 | INFO | "malloc" : " 14 GB", | 20260506 00:40:37.940 | INFO | "pinned" : " 29 GB" | 20260506 00:40:37.940 | INFO | } | 20260506 00:40:37.940 | INFO | Perf report for the last 5.1438 seconds: | 20260506 00:40:37.940 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:40:37.940 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:37.940 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:37.940 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:37.940 | INFO | ANALYSIS : 62.5%, 3.0, 97.7ms, 0 | 20260506 00:40:37.940 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:37.940 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260506 00:40:37.940 | INFO | PRE_HQ : 59.7%, 1.7, 53.9ms, 0 | 20260506 00:40:37.940 | INFO | BAZWRITER : 61.8%, 3.5, 110.7ms, 0 | 20260506 00:40:37.940 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:40:37.941 | INFO | Uploaded 36GiB to GPU in 2.16574s (16.6225GiB/s) | 20260506 00:40:37.941 | INFO | Downloaded 38GiB to GPU in 2.23742s (16.9838GiB/s) | 20260506 00:40:43.088 | INFO | Analyzing chunk frames = [7586304,7586816) | 20260506 00:40:43.088 | INFO | MemInfo: { | 20260506 00:40:43.088 | INFO | "gpu" : " 17 GB", | 20260506 00:40:43.088 | INFO | "huge" : "", | 20260506 00:40:43.088 | INFO | "huge_shared" : " 27 GB", | 20260506 00:40:43.088 | INFO | "malloc" : " 14 GB", | 20260506 00:40:43.088 | INFO | "pinned" : " 29 GB" | 20260506 00:40:43.088 | INFO | } | 20260506 00:40:43.088 | INFO | Perf report for the last 5.1474 seconds: | 20260506 00:40:43.088 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:40:43.088 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:43.088 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:43.088 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:43.088 | INFO | ANALYSIS : 60.7%, 3.0, 94.9ms, 0 | 20260506 00:40:43.088 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:43.088 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:43.088 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:40:43.088 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:43.088 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:43.088 | INFO | Uploaded 36GiB to GPU in 1.97427s (18.2346GiB/s) | 20260506 00:40:43.088 | INFO | Downloaded 34GiB to GPU in 1.98484s (17.1298GiB/s) | 20260506 00:40:48.218 | INFO | Analyzing chunk frames = [7586816,7587328) | 20260506 00:40:48.218 | INFO | MemInfo: { | 20260506 00:40:48.218 | INFO | "gpu" : " 17 GB", | 20260506 00:40:48.218 | INFO | "huge" : "", | 20260506 00:40:48.218 | INFO | "huge_shared" : " 27 GB", | 20260506 00:40:48.218 | INFO | "malloc" : " 14 GB", | 20260506 00:40:48.218 | INFO | "pinned" : " 29 GB" | 20260506 00:40:48.218 | INFO | } | 20260506 00:40:48.218 | INFO | Perf report for the last 5.13037 seconds: | 20260506 00:40:48.218 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:40:48.218 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:48.218 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:48.218 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:48.218 | INFO | ANALYSIS : 61.3%, 3.0, 95.2ms, 0 | 20260506 00:40:48.218 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:48.218 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:48.218 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 00:40:48.218 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:48.218 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:48.218 | INFO | Uploaded 36GiB to GPU in 2.1059s (17.0948GiB/s) | 20260506 00:40:48.218 | INFO | Downloaded 34GiB to GPU in 2.04375s (16.6361GiB/s) | 20260506 00:40:53.355 | INFO | Analyzing chunk frames = [7587328,7587840) | 20260506 00:40:53.355 | INFO | MemInfo: { | 20260506 00:40:53.355 | INFO | "gpu" : " 17 GB", | 20260506 00:40:53.355 | INFO | "huge" : "", | 20260506 00:40:53.355 | INFO | "huge_shared" : " 27 GB", | 20260506 00:40:53.355 | INFO | "malloc" : " 14 GB", | 20260506 00:40:53.355 | INFO | "pinned" : " 29 GB" | 20260506 00:40:53.355 | INFO | } | 20260506 00:40:53.355 | INFO | Perf report for the last 5.13706 seconds: | 20260506 00:40:53.355 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:40:53.355 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:53.355 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:53.355 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:53.355 | INFO | ANALYSIS : 61.6%, 3.0, 95.6ms, 0 | 20260506 00:40:53.355 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:53.355 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:53.355 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 00:40:53.355 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:53.355 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:53.355 | INFO | Uploaded 36GiB to GPU in 2.1313s (16.8911GiB/s) | 20260506 00:40:53.355 | INFO | Downloaded 34GiB to GPU in 2.02022s (16.8298GiB/s) | 20260506 00:40:58.498 | INFO | Analyzing chunk frames = [7587840,7588352) | 20260506 00:40:58.498 | INFO | MemInfo: { | 20260506 00:40:58.498 | INFO | "gpu" : " 17 GB", | 20260506 00:40:58.498 | INFO | "huge" : "", | 20260506 00:40:58.498 | INFO | "huge_shared" : " 27 GB", | 20260506 00:40:58.498 | INFO | "malloc" : " 14 GB", | 20260506 00:40:58.498 | INFO | "pinned" : " 29 GB" | 20260506 00:40:58.498 | INFO | } | 20260506 00:40:58.498 | INFO | Perf report for the last 5.14241 seconds: | 20260506 00:40:58.498 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:40:58.498 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:58.498 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:58.498 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:58.498 | INFO | ANALYSIS : 61.1%, 3.0, 95.5ms, 0 | 20260506 00:40:58.498 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:58.498 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:58.498 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:40:58.498 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:58.498 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:40:58.498 | INFO | Uploaded 36GiB to GPU in 2.1023s (17.1241GiB/s) | 20260506 00:40:58.498 | INFO | Downloaded 34GiB to GPU in 2.04746s (16.606GiB/s) | 20260506 00:41:03.635 | INFO | Analyzing chunk frames = [7588352,7588864) | 20260506 00:41:03.635 | INFO | MemInfo: { | 20260506 00:41:03.635 | INFO | "gpu" : " 17 GB", | 20260506 00:41:03.635 | INFO | "huge" : "", | 20260506 00:41:03.635 | INFO | "huge_shared" : " 27 GB", | 20260506 00:41:03.635 | INFO | "malloc" : " 14 GB", | 20260506 00:41:03.635 | INFO | "pinned" : " 29 GB" | 20260506 00:41:03.635 | INFO | } | 20260506 00:41:03.635 | INFO | Perf report for the last 5.137 seconds: | 20260506 00:41:03.635 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:41:03.635 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:03.635 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:03.635 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:03.635 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 00:41:03.635 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:03.635 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:03.635 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 00:41:03.635 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:03.635 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:03.635 | INFO | Uploaded 36GiB to GPU in 2.06594s (17.4255GiB/s) | 20260506 00:41:03.635 | INFO | Downloaded 34GiB to GPU in 2.0273s (16.7711GiB/s) | 20260506 00:41:08.773 | INFO | Analyzing chunk frames = [7588864,7589376) | 20260506 00:41:08.773 | INFO | MemInfo: { | 20260506 00:41:08.773 | INFO | "gpu" : " 17 GB", | 20260506 00:41:08.773 | INFO | "huge" : "", | 20260506 00:41:08.773 | INFO | "huge_shared" : " 27 GB", | 20260506 00:41:08.773 | INFO | "malloc" : " 14 GB", | 20260506 00:41:08.773 | INFO | "pinned" : " 29 GB" | 20260506 00:41:08.773 | INFO | } | 20260506 00:41:08.773 | INFO | Perf report for the last 5.13831 seconds: | 20260506 00:41:08.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:41:08.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:08.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:08.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:08.773 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260506 00:41:08.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:08.773 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:08.773 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260506 00:41:08.773 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:08.773 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:08.773 | INFO | Uploaded 36GiB to GPU in 2.1028s (17.12GiB/s) | 20260506 00:41:08.773 | INFO | Downloaded 34GiB to GPU in 2.06965s (16.4279GiB/s) | 20260506 00:41:13.911 | INFO | Analyzing chunk frames = [7589376,7589888) | 20260506 00:41:13.912 | INFO | MemInfo: { | 20260506 00:41:13.912 | INFO | "gpu" : " 17 GB", | 20260506 00:41:13.912 | INFO | "huge" : "", | 20260506 00:41:13.912 | INFO | "huge_shared" : " 27 GB", | 20260506 00:41:13.912 | INFO | "malloc" : " 14 GB", | 20260506 00:41:13.912 | INFO | "pinned" : " 29 GB" | 20260506 00:41:13.912 | INFO | } | 20260506 00:41:13.912 | INFO | Perf report for the last 5.13865 seconds: | 20260506 00:41:13.912 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:41:13.912 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:13.912 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:13.912 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:13.912 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260506 00:41:13.912 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:13.912 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:13.912 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 00:41:13.912 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:13.912 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:13.912 | INFO | Uploaded 36GiB to GPU in 2.11988s (16.9821GiB/s) | 20260506 00:41:13.912 | INFO | Downloaded 34GiB to GPU in 2.01278s (16.8921GiB/s) | 20260506 00:41:14.037 | INFO | Starting a round of Baz IO | 20260506 00:41:17.174 | INFO | Populating RTMetrics report | 20260506 00:41:17.174 | INFO | RTMetrics report populated | 20260506 00:41:17.174 | INFO | Opening tmp RTMetrics json file | 20260506 00:41:17.175 | INFO | Serializing RTMetrics to json | 20260506 00:41:17.176 | INFO | Serialization done, swapping tmp file now... | 20260506 00:41:17.176 | INFO | RTMetrics done | 20260506 00:41:17.251 | INFO | Finished a round of Baz IO: Wrote 1.848 GiB in 3.214 seconds | 20260506 00:41:17.251 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.36% / 0.00% / 78.64% | 20260506 00:41:17.251 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0884652% / 0% / 5.5631e-05% | 20260506 00:41:17.251 | INFO | Baz IO overall: | 20260506 00:41:17.251 | INFO | BytesWritten : 4304831538650 | 20260506 00:41:17.251 | INFO | NumEvents : 2025228441736 | 20260506 00:41:19.049 | INFO | Analyzing chunk frames = [7589888,7590400) | 20260506 00:41:19.049 | INFO | MemInfo: { | 20260506 00:41:19.049 | INFO | "gpu" : " 17 GB", | 20260506 00:41:19.049 | INFO | "huge" : "", | 20260506 00:41:19.049 | INFO | "huge_shared" : " 27 GB", | 20260506 00:41:19.049 | INFO | "malloc" : " 14 GB", | 20260506 00:41:19.049 | INFO | "pinned" : " 29 GB" | 20260506 00:41:19.049 | INFO | } | 20260506 00:41:19.049 | INFO | Perf report for the last 5.13781 seconds: | 20260506 00:41:19.049 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:41:19.049 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:19.049 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:19.049 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:19.049 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260506 00:41:19.049 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:19.050 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260506 00:41:19.050 | INFO | PRE_HQ : 59.6%, 1.7, 54.0ms, 0 | 20260506 00:41:19.050 | INFO | BAZWRITER : 61.7%, 3.5, 111.1ms, 0 | 20260506 00:41:19.050 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:41:19.050 | INFO | Uploaded 36GiB to GPU in 2.12434s (16.9465GiB/s) | 20260506 00:41:19.050 | INFO | Downloaded 38GiB to GPU in 2.22736s (17.0606GiB/s) | 20260506 00:41:24.193 | INFO | Analyzing chunk frames = [7590400,7590912) | 20260506 00:41:24.193 | INFO | MemInfo: { | 20260506 00:41:24.193 | INFO | "gpu" : " 17 GB", | 20260506 00:41:24.193 | INFO | "huge" : "", | 20260506 00:41:24.193 | INFO | "huge_shared" : " 27 GB", | 20260506 00:41:24.193 | INFO | "malloc" : " 14 GB", | 20260506 00:41:24.193 | INFO | "pinned" : " 29 GB" | 20260506 00:41:24.193 | INFO | } | 20260506 00:41:24.193 | INFO | Perf report for the last 5.14391 seconds: | 20260506 00:41:24.193 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:41:24.193 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:24.193 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:24.193 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:24.193 | INFO | ANALYSIS : 61.7%, 3.0, 96.5ms, 0 | 20260506 00:41:24.193 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:24.193 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:24.193 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 00:41:24.193 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:24.194 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:24.194 | INFO | Uploaded 36GiB to GPU in 2.09736s (17.1644GiB/s) | 20260506 00:41:24.194 | INFO | Downloaded 34GiB to GPU in 2.03908s (16.6742GiB/s) | 20260506 00:41:29.335 | INFO | Analyzing chunk frames = [7590912,7591424) | 20260506 00:41:29.335 | INFO | MemInfo: { | 20260506 00:41:29.335 | INFO | "gpu" : " 17 GB", | 20260506 00:41:29.335 | INFO | "huge" : "", | 20260506 00:41:29.335 | INFO | "huge_shared" : " 27 GB", | 20260506 00:41:29.335 | INFO | "malloc" : " 14 GB", | 20260506 00:41:29.335 | INFO | "pinned" : " 29 GB" | 20260506 00:41:29.335 | INFO | } | 20260506 00:41:29.335 | INFO | Perf report for the last 5.14143 seconds: | 20260506 00:41:29.335 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:41:29.335 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:29.335 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:29.335 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:29.335 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 00:41:29.335 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:29.335 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:29.335 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:41:29.335 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:29.335 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:29.335 | INFO | Uploaded 36GiB to GPU in 2.08705s (17.2492GiB/s) | 20260506 00:41:29.335 | INFO | Downloaded 34GiB to GPU in 2.02668s (16.7762GiB/s) | 20260506 00:41:34.472 | INFO | Analyzing chunk frames = [7591424,7591936) | 20260506 00:41:34.472 | INFO | MemInfo: { | 20260506 00:41:34.472 | INFO | "gpu" : " 17 GB", | 20260506 00:41:34.472 | INFO | "huge" : "", | 20260506 00:41:34.472 | INFO | "huge_shared" : " 27 GB", | 20260506 00:41:34.472 | INFO | "malloc" : " 14 GB", | 20260506 00:41:34.472 | INFO | "pinned" : " 29 GB" | 20260506 00:41:34.472 | INFO | } | 20260506 00:41:34.472 | INFO | Perf report for the last 5.13699 seconds: | 20260506 00:41:34.472 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:41:34.472 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:34.472 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:34.472 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:34.472 | INFO | ANALYSIS : 61.6%, 3.0, 95.6ms, 0 | 20260506 00:41:34.472 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:34.472 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:34.472 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:41:34.472 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:34.472 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:34.472 | INFO | Uploaded 36GiB to GPU in 2.1466s (16.7707GiB/s) | 20260506 00:41:34.472 | INFO | Downloaded 34GiB to GPU in 2.06009s (16.5041GiB/s) | 20260506 00:41:39.606 | INFO | Analyzing chunk frames = [7591936,7592448) | 20260506 00:41:39.606 | INFO | MemInfo: { | 20260506 00:41:39.606 | INFO | "gpu" : " 17 GB", | 20260506 00:41:39.606 | INFO | "huge" : "", | 20260506 00:41:39.606 | INFO | "huge_shared" : " 27 GB", | 20260506 00:41:39.606 | INFO | "malloc" : " 14 GB", | 20260506 00:41:39.606 | INFO | "pinned" : " 29 GB" | 20260506 00:41:39.606 | INFO | } | 20260506 00:41:39.606 | INFO | Perf report for the last 5.13394 seconds: | 20260506 00:41:39.606 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:41:39.606 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:39.606 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:39.606 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:39.606 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 00:41:39.606 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:39.606 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:39.606 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:41:39.606 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:39.606 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:39.606 | INFO | Uploaded 36GiB to GPU in 2.10503s (17.1019GiB/s) | 20260506 00:41:39.606 | INFO | Downloaded 34GiB to GPU in 2.04882s (16.5949GiB/s) | 20260506 00:41:44.750 | INFO | Analyzing chunk frames = [7592448,7592960) | 20260506 00:41:44.750 | INFO | MemInfo: { | 20260506 00:41:44.750 | INFO | "gpu" : " 17 GB", | 20260506 00:41:44.750 | INFO | "huge" : "", | 20260506 00:41:44.750 | INFO | "huge_shared" : " 27 GB", | 20260506 00:41:44.750 | INFO | "malloc" : " 14 GB", | 20260506 00:41:44.750 | INFO | "pinned" : " 29 GB" | 20260506 00:41:44.750 | INFO | } | 20260506 00:41:44.750 | INFO | Perf report for the last 5.1446 seconds: | 20260506 00:41:44.750 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:41:44.750 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:44.750 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:44.750 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:44.750 | INFO | ANALYSIS : 61.4%, 3.0, 96.0ms, 0 | 20260506 00:41:44.750 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:44.750 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:44.750 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:41:44.750 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:44.750 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:44.751 | INFO | Uploaded 36GiB to GPU in 2.07483s (17.3508GiB/s) | 20260506 00:41:44.751 | INFO | Downloaded 34GiB to GPU in 2.03386s (16.717GiB/s) | 20260506 00:41:49.894 | INFO | Analyzing chunk frames = [7592960,7593472) | 20260506 00:41:49.894 | INFO | MemInfo: { | 20260506 00:41:49.894 | INFO | "gpu" : " 17 GB", | 20260506 00:41:49.894 | INFO | "huge" : "", | 20260506 00:41:49.894 | INFO | "huge_shared" : " 27 GB", | 20260506 00:41:49.894 | INFO | "malloc" : " 14 GB", | 20260506 00:41:49.894 | INFO | "pinned" : " 29 GB" | 20260506 00:41:49.894 | INFO | } | 20260506 00:41:49.894 | INFO | Perf report for the last 5.14329 seconds: | 20260506 00:41:49.894 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:41:49.894 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:49.894 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:49.894 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:49.894 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260506 00:41:49.894 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:49.894 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:49.894 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 00:41:49.894 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:49.894 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:49.894 | INFO | Uploaded 36GiB to GPU in 2.02417s (17.7851GiB/s) | 20260506 00:41:49.894 | INFO | Downloaded 34GiB to GPU in 2.04215s (16.6491GiB/s) | 20260506 00:41:55.038 | INFO | Analyzing chunk frames = [7593472,7593984) | 20260506 00:41:55.038 | INFO | MemInfo: { | 20260506 00:41:55.038 | INFO | "gpu" : " 17 GB", | 20260506 00:41:55.038 | INFO | "huge" : "", | 20260506 00:41:55.038 | INFO | "huge_shared" : " 27 GB", | 20260506 00:41:55.038 | INFO | "malloc" : " 14 GB", | 20260506 00:41:55.038 | INFO | "pinned" : " 29 GB" | 20260506 00:41:55.038 | INFO | } | 20260506 00:41:55.038 | INFO | Perf report for the last 5.14435 seconds: | 20260506 00:41:55.038 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:41:55.038 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:55.038 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:55.038 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:55.038 | INFO | ANALYSIS : 61.5%, 3.0, 96.1ms, 0 | 20260506 00:41:55.038 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:55.038 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:55.038 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:41:55.038 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:55.038 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:41:55.038 | INFO | Uploaded 36GiB to GPU in 2.05807s (17.4921GiB/s) | 20260506 00:41:55.038 | INFO | Downloaded 34GiB to GPU in 2.01749s (16.8526GiB/s) | 20260506 00:41:55.039 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 00:41:55.039 | INFO | Total mem usage: 15.3 GiB | 20260506 00:41:55.039 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 00:41:55.039 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.72%) | 20260506 00:41:55.039 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 00:41:55.039 | INFO | High Water Breakdown: | 20260506 00:41:55.039 | INFO | 61% : 8.6 GiB ( 90.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 00:41:55.039 | INFO | 38.8% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 00:41:55.039 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 00:41:55.039 | INFO | ------------------------------------------ | 20260506 00:41:55.039 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:41:55.039 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 00:41:55.039 | INFO | Total mem usage: 26.5 GiB | 20260506 00:41:55.039 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 00:41:55.039 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 00:41:55.039 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260506 00:41:55.039 | INFO | High Water Breakdown: | 20260506 00:41:55.039 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:41:55.039 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:41:55.039 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:41:55.039 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:41:55.039 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:41:55.039 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:41:55.039 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 00:41:55.039 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 00:41:55.039 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:41:55.039 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 00:41:55.039 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 00:41:55.039 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 00:41:55.039 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 00:41:55.039 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 00:41:55.039 | INFO | ------------------------------------------ | 20260506 00:41:55.039 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:41:55.039 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 00:41:55.039 | INFO | Total mem usage: 25.6 GiB | 20260506 00:41:55.039 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 00:41:55.039 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 00:41:55.039 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 00:41:55.039 | INFO | High Water Breakdown: | 20260506 00:41:55.039 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 00:41:55.039 | INFO | ------------------------------------------ | 20260506 00:41:55.039 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:41:55.039 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 00:41:55.039 | INFO | Total mem usage: 15.7 GiB | 20260506 00:41:55.039 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 00:41:55.039 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 00:41:55.039 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 00:41:55.039 | INFO | High Water Breakdown: | 20260506 00:41:55.039 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 00:41:55.039 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 00:41:55.039 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 00:41:55.039 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 00:41:55.039 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 00:41:55.039 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 00:41:55.039 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 00:41:55.039 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 00:41:55.039 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 00:41:55.039 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:41:55.039 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 00:41:55.039 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:41:55.039 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 00:41:55.039 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 00:41:55.039 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 00:41:55.039 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 00:41:55.039 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:41:55.039 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 00:41:55.039 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:41:55.039 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:41:55.039 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:41:55.039 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 00:41:55.039 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 00:41:55.039 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:41:55.039 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 00:41:55.039 | INFO | ------------------------------------------ | 20260506 00:41:55.039 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:41:55.039 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 00:41:55.039 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:41:55.039 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:41:55.039 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.39e+01, 1.34e+01 / 7.56e-01 | 20260506 00:41:55.039 | INFO | PerfReport: 0.29%, AggregateStats -- 9.17e-02 / 1.04e-01, 1.03e-02 / 5.84e-03 | 20260506 00:41:55.039 | INFO | PerfReport: 11.86%, Baseline -- 3.71e+00 / 4.50e+00, 5.78e-01 / 1.87e-01 | 20260506 00:41:55.039 | INFO | PerfReport: 3.29%, Histogram -- 1.03e+00 / 1.52e+00, 1.94e-01 / 1.02e-01 | 20260506 00:41:55.039 | INFO | PerfReport: 16.01%, DME -- 4.01e+01 / 4.41e+01, 3.91e+00 / 9.51e-01 | 20260506 00:41:55.039 | INFO | PerfReport: 57.32%, FrameLabeling -- 1.79e+01 / 1.93e+01, 1.47e+00 / 9.79e-01 | 20260506 00:41:55.039 | INFO | PerfReport: 7.09%, PulseAccumulating -- 2.22e+00 / 2.77e+00, 4.20e-01 / 2.14e-01 | 20260506 00:41:55.039 | INFO | PerfReport: 3.08%, Metrics -- 9.63e-01 / 1.25e+00, 2.60e-01 / 1.08e-01 | 20260506 00:41:55.039 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.32e-01 / 3.85e-01, 5.61e-02 / 2.92e-02 | 20260506 00:41:55.040 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 00:41:55.040 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:41:55.040 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:41:55.040 | INFO | PerfReport: Full Execution -- 9.81e+01 / -nan, 2.45e+01 / 0.00e+00 | 20260506 00:41:55.040 | INFO | PerfReport: 22.22%, Upload -- 2.18e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260506 00:41:55.040 | INFO | PerfReport: 21.86%, Download -- 2.14e+01 / 0.00e+00, 7.59e+00 / 0.00e+00 | 20260506 00:41:55.040 | INFO | PerfReport: 31.92%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 00:41:55.040 | INFO | PerfReport: 2.06%, LockUpload -- 2.02e+00 / 0.00e+00, 7.82e+00 / 0.00e+00 | 20260506 00:41:55.040 | INFO | PerfReport: 1.30%, LockDownload -- 1.28e+00 / 0.00e+00, 2.92e+00 / 0.00e+00 | 20260506 00:41:55.040 | INFO | PerfReport: 20.63%, LockCompute -- 2.02e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 00:41:55.040 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.62e-02 / 0.00e+00 | 20260506 00:41:55.167 | INFO | Starting a round of Baz IO | 20260506 00:41:58.302 | INFO | Populating RTMetrics report | 20260506 00:41:58.302 | INFO | RTMetrics report populated | 20260506 00:41:58.302 | INFO | Opening tmp RTMetrics json file | 20260506 00:41:58.303 | INFO | Serializing RTMetrics to json | 20260506 00:41:58.304 | INFO | Serialization done, swapping tmp file now... | 20260506 00:41:58.305 | INFO | RTMetrics done | 20260506 00:41:58.380 | INFO | Finished a round of Baz IO: Wrote 1.85 GiB in 3.213 seconds | 20260506 00:41:58.380 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.36% / 0.00% / 78.64% | 20260506 00:41:58.380 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0889579% / 0% / 7.54992e-05% | 20260506 00:41:58.380 | INFO | Baz IO overall: | 20260506 00:41:58.380 | INFO | BytesWritten : 4306818234877 | 20260506 00:41:58.380 | INFO | NumEvents : 2026136278872 | 20260506 00:42:00.166 | INFO | Analyzing chunk frames = [7593984,7594496) | 20260506 00:42:00.166 | INFO | MemInfo: { | 20260506 00:42:00.166 | INFO | "gpu" : " 17 GB", | 20260506 00:42:00.166 | INFO | "huge" : "", | 20260506 00:42:00.166 | INFO | "huge_shared" : " 27 GB", | 20260506 00:42:00.166 | INFO | "malloc" : " 14 GB", | 20260506 00:42:00.166 | INFO | "pinned" : " 29 GB" | 20260506 00:42:00.166 | INFO | } | 20260506 00:42:00.166 | INFO | Perf report for the last 5.12798 seconds: | 20260506 00:42:00.166 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:42:00.166 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:00.166 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:00.166 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:00.166 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260506 00:42:00.166 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:00.166 | INFO | RT_METRICS : 43.3%, 1.0, 22.6ms, 0 | 20260506 00:42:00.166 | INFO | PRE_HQ : 59.5%, 1.8, 54.0ms, 0 | 20260506 00:42:00.166 | INFO | BAZWRITER : 61.9%, 3.5, 111.2ms, 0 | 20260506 00:42:00.166 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:42:00.167 | INFO | Uploaded 36GiB to GPU in 2.10835s (17.075GiB/s) | 20260506 00:42:00.167 | INFO | Downloaded 38GiB to GPU in 2.22241s (17.0985GiB/s) | 20260506 00:42:05.305 | INFO | Analyzing chunk frames = [7594496,7595008) | 20260506 00:42:05.305 | INFO | MemInfo: { | 20260506 00:42:05.305 | INFO | "gpu" : " 17 GB", | 20260506 00:42:05.305 | INFO | "huge" : "", | 20260506 00:42:05.305 | INFO | "huge_shared" : " 27 GB", | 20260506 00:42:05.305 | INFO | "malloc" : " 14 GB", | 20260506 00:42:05.305 | INFO | "pinned" : " 29 GB" | 20260506 00:42:05.305 | INFO | } | 20260506 00:42:05.305 | INFO | Perf report for the last 5.13944 seconds: | 20260506 00:42:05.305 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:42:05.305 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:05.305 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:05.306 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:05.306 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 00:42:05.306 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:05.306 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:05.306 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 00:42:05.306 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:05.306 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:05.306 | INFO | Uploaded 36GiB to GPU in 2.10267s (17.1211GiB/s) | 20260506 00:42:05.306 | INFO | Downloaded 34GiB to GPU in 2.04032s (16.664GiB/s) | 20260506 00:42:10.450 | INFO | Analyzing chunk frames = [7595008,7595520) | 20260506 00:42:10.450 | INFO | MemInfo: { | 20260506 00:42:10.450 | INFO | "gpu" : " 17 GB", | 20260506 00:42:10.450 | INFO | "huge" : "", | 20260506 00:42:10.450 | INFO | "huge_shared" : " 27 GB", | 20260506 00:42:10.450 | INFO | "malloc" : " 14 GB", | 20260506 00:42:10.450 | INFO | "pinned" : " 29 GB" | 20260506 00:42:10.450 | INFO | } | 20260506 00:42:10.450 | INFO | Perf report for the last 5.14447 seconds: | 20260506 00:42:10.450 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:42:10.450 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:10.450 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:10.450 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:10.450 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 00:42:10.450 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:10.450 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:10.450 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 00:42:10.450 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:10.450 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:10.450 | INFO | Uploaded 36GiB to GPU in 2.09195s (17.2088GiB/s) | 20260506 00:42:10.450 | INFO | Downloaded 34GiB to GPU in 2.01631s (16.8625GiB/s) | 20260506 00:42:15.588 | INFO | Analyzing chunk frames = [7595520,7596032) | 20260506 00:42:15.588 | INFO | MemInfo: { | 20260506 00:42:15.588 | INFO | "gpu" : " 17 GB", | 20260506 00:42:15.588 | INFO | "huge" : "", | 20260506 00:42:15.588 | INFO | "huge_shared" : " 27 GB", | 20260506 00:42:15.588 | INFO | "malloc" : " 14 GB", | 20260506 00:42:15.588 | INFO | "pinned" : " 29 GB" | 20260506 00:42:15.588 | INFO | } | 20260506 00:42:15.588 | INFO | Perf report for the last 5.13838 seconds: | 20260506 00:42:15.588 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:42:15.588 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:15.588 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:15.588 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:15.588 | INFO | ANALYSIS : 61.6%, 3.0, 95.6ms, 0 | 20260506 00:42:15.588 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:15.588 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:15.588 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 00:42:15.588 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:15.588 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:15.589 | INFO | Uploaded 36GiB to GPU in 2.13947s (16.8266GiB/s) | 20260506 00:42:15.589 | INFO | Downloaded 34GiB to GPU in 2.02124s (16.8214GiB/s) | 20260506 00:42:20.721 | INFO | Analyzing chunk frames = [7596032,7596544) | 20260506 00:42:20.721 | INFO | MemInfo: { | 20260506 00:42:20.721 | INFO | "gpu" : " 17 GB", | 20260506 00:42:20.721 | INFO | "huge" : "", | 20260506 00:42:20.721 | INFO | "huge_shared" : " 27 GB", | 20260506 00:42:20.721 | INFO | "malloc" : " 14 GB", | 20260506 00:42:20.721 | INFO | "pinned" : " 29 GB" | 20260506 00:42:20.721 | INFO | } | 20260506 00:42:20.721 | INFO | Perf report for the last 5.13238 seconds: | 20260506 00:42:20.721 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:42:20.721 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:20.721 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:20.721 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:20.721 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 00:42:20.721 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:20.721 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:20.721 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 00:42:20.721 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:20.721 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:20.721 | INFO | Uploaded 36GiB to GPU in 2.09331s (17.1977GiB/s) | 20260506 00:42:20.721 | INFO | Downloaded 34GiB to GPU in 2.04596s (16.6181GiB/s) | 20260506 00:42:25.861 | INFO | Analyzing chunk frames = [7596544,7597056) | 20260506 00:42:25.861 | INFO | MemInfo: { | 20260506 00:42:25.861 | INFO | "gpu" : " 17 GB", | 20260506 00:42:25.861 | INFO | "huge" : "", | 20260506 00:42:25.861 | INFO | "huge_shared" : " 27 GB", | 20260506 00:42:25.861 | INFO | "malloc" : " 14 GB", | 20260506 00:42:25.861 | INFO | "pinned" : " 29 GB" | 20260506 00:42:25.861 | INFO | } | 20260506 00:42:25.861 | INFO | Perf report for the last 5.14034 seconds: | 20260506 00:42:25.861 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:42:25.861 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:25.861 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:25.861 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:25.861 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260506 00:42:25.861 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:25.861 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:25.861 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:42:25.861 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:25.861 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:25.861 | INFO | Uploaded 36GiB to GPU in 2.0696s (17.3946GiB/s) | 20260506 00:42:25.861 | INFO | Downloaded 34GiB to GPU in 2.02881s (16.7586GiB/s) | 20260506 00:42:31.005 | INFO | Analyzing chunk frames = [7597056,7597568) | 20260506 00:42:31.005 | INFO | MemInfo: { | 20260506 00:42:31.005 | INFO | "gpu" : " 17 GB", | 20260506 00:42:31.005 | INFO | "huge" : "", | 20260506 00:42:31.005 | INFO | "huge_shared" : " 27 GB", | 20260506 00:42:31.005 | INFO | "malloc" : " 14 GB", | 20260506 00:42:31.005 | INFO | "pinned" : " 29 GB" | 20260506 00:42:31.005 | INFO | } | 20260506 00:42:31.005 | INFO | Perf report for the last 5.14426 seconds: | 20260506 00:42:31.005 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:42:31.005 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:31.005 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:31.005 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:31.005 | INFO | ANALYSIS : 62.0%, 3.0, 97.0ms, 0 | 20260506 00:42:31.005 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:31.005 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:31.005 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 00:42:31.005 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:31.005 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:31.006 | INFO | Uploaded 36GiB to GPU in 2.04042s (17.6434GiB/s) | 20260506 00:42:31.006 | INFO | Downloaded 34GiB to GPU in 2.03914s (16.6737GiB/s) | 20260506 00:42:36.144 | INFO | Analyzing chunk frames = [7597568,7598080) | 20260506 00:42:36.144 | INFO | MemInfo: { | 20260506 00:42:36.144 | INFO | "gpu" : " 17 GB", | 20260506 00:42:36.144 | INFO | "huge" : "", | 20260506 00:42:36.144 | INFO | "huge_shared" : " 27 GB", | 20260506 00:42:36.144 | INFO | "malloc" : " 14 GB", | 20260506 00:42:36.144 | INFO | "pinned" : " 29 GB" | 20260506 00:42:36.144 | INFO | } | 20260506 00:42:36.144 | INFO | Perf report for the last 5.13873 seconds: | 20260506 00:42:36.144 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:42:36.144 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:36.144 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:36.144 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:36.144 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260506 00:42:36.144 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:36.144 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:36.144 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 00:42:36.144 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:36.144 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:36.144 | INFO | Uploaded 36GiB to GPU in 2.132s (16.8855GiB/s) | 20260506 00:42:36.144 | INFO | Downloaded 34GiB to GPU in 2.02251s (16.8108GiB/s) | 20260506 00:42:36.273 | INFO | Starting a round of Baz IO | 20260506 00:42:39.405 | INFO | Populating RTMetrics report | 20260506 00:42:39.405 | INFO | RTMetrics report populated | 20260506 00:42:39.405 | INFO | Opening tmp RTMetrics json file | 20260506 00:42:39.405 | INFO | Serializing RTMetrics to json | 20260506 00:42:39.407 | INFO | Serialization done, swapping tmp file now... | 20260506 00:42:39.408 | INFO | RTMetrics done | 20260506 00:42:39.483 | INFO | Finished a round of Baz IO: Wrote 1.851 GiB in 3.209 seconds | 20260506 00:42:39.483 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.35% / 0.00% / 78.64% | 20260506 00:42:39.483 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0889778% / 0% / 7.94729e-05% | 20260506 00:42:39.483 | INFO | Baz IO overall: | 20260506 00:42:39.483 | INFO | BytesWritten : 4308806105067 | 20260506 00:42:39.483 | INFO | NumEvents : 2027044391338 | 20260506 00:42:41.279 | INFO | Analyzing chunk frames = [7598080,7598592) | 20260506 00:42:41.279 | INFO | MemInfo: { | 20260506 00:42:41.279 | INFO | "gpu" : " 17 GB", | 20260506 00:42:41.279 | INFO | "huge" : "", | 20260506 00:42:41.279 | INFO | "huge_shared" : " 27 GB", | 20260506 00:42:41.279 | INFO | "malloc" : " 14 GB", | 20260506 00:42:41.279 | INFO | "pinned" : " 29 GB" | 20260506 00:42:41.279 | INFO | } | 20260506 00:42:41.279 | INFO | Perf report for the last 5.13519 seconds: | 20260506 00:42:41.279 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:42:41.279 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:41.279 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:41.279 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:41.279 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260506 00:42:41.279 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:41.279 | INFO | RT_METRICS : 43.2%, 1.0, 22.6ms, 0 | 20260506 00:42:41.279 | INFO | PRE_HQ : 59.7%, 1.7, 54.2ms, 0 | 20260506 00:42:41.279 | INFO | BAZWRITER : 61.8%, 3.5, 110.4ms, 0 | 20260506 00:42:41.279 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:42:41.279 | INFO | Uploaded 36GiB to GPU in 2.1869s (16.4617GiB/s) | 20260506 00:42:41.279 | INFO | Downloaded 38GiB to GPU in 2.24487s (16.9275GiB/s) | 20260506 00:42:46.419 | INFO | Analyzing chunk frames = [7598592,7599104) | 20260506 00:42:46.419 | INFO | MemInfo: { | 20260506 00:42:46.419 | INFO | "gpu" : " 17 GB", | 20260506 00:42:46.419 | INFO | "huge" : "", | 20260506 00:42:46.419 | INFO | "huge_shared" : " 27 GB", | 20260506 00:42:46.419 | INFO | "malloc" : " 14 GB", | 20260506 00:42:46.419 | INFO | "pinned" : " 29 GB" | 20260506 00:42:46.419 | INFO | } | 20260506 00:42:46.419 | INFO | Perf report for the last 5.13949 seconds: | 20260506 00:42:46.419 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:42:46.419 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:46.419 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:46.419 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:46.419 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 00:42:46.419 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:46.419 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:46.419 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:42:46.419 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:46.419 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:46.419 | INFO | Uploaded 36GiB to GPU in 2.1046s (17.1054GiB/s) | 20260506 00:42:46.419 | INFO | Downloaded 34GiB to GPU in 2.04433s (16.6314GiB/s) | 20260506 00:42:51.557 | INFO | Analyzing chunk frames = [7599104,7599616) | 20260506 00:42:51.557 | INFO | MemInfo: { | 20260506 00:42:51.557 | INFO | "gpu" : " 17 GB", | 20260506 00:42:51.557 | INFO | "huge" : "", | 20260506 00:42:51.557 | INFO | "huge_shared" : " 27 GB", | 20260506 00:42:51.557 | INFO | "malloc" : " 14 GB", | 20260506 00:42:51.557 | INFO | "pinned" : " 29 GB" | 20260506 00:42:51.557 | INFO | } | 20260506 00:42:51.557 | INFO | Perf report for the last 5.13799 seconds: | 20260506 00:42:51.557 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:42:51.557 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:51.557 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:51.557 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:51.557 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 00:42:51.557 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:51.557 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:51.557 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:42:51.557 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:51.557 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:51.557 | INFO | Uploaded 36GiB to GPU in 2.12806s (16.9168GiB/s) | 20260506 00:42:51.557 | INFO | Downloaded 34GiB to GPU in 2.0243s (16.796GiB/s) | 20260506 00:42:56.701 | INFO | Analyzing chunk frames = [7599616,7600128) | 20260506 00:42:56.701 | INFO | MemInfo: { | 20260506 00:42:56.701 | INFO | "gpu" : " 17 GB", | 20260506 00:42:56.701 | INFO | "huge" : "", | 20260506 00:42:56.701 | INFO | "huge_shared" : " 27 GB", | 20260506 00:42:56.701 | INFO | "malloc" : " 14 GB", | 20260506 00:42:56.701 | INFO | "pinned" : " 29 GB" | 20260506 00:42:56.701 | INFO | } | 20260506 00:42:56.701 | INFO | Perf report for the last 5.1446 seconds: | 20260506 00:42:56.701 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:42:56.701 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:56.701 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:56.701 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:56.701 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260506 00:42:56.702 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:56.702 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:56.702 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:42:56.702 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:56.702 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:42:56.702 | INFO | Uploaded 36GiB to GPU in 2.14156s (16.8102GiB/s) | 20260506 00:42:56.702 | INFO | Downloaded 34GiB to GPU in 2.02009s (16.8309GiB/s) | 20260506 00:43:01.846 | INFO | Analyzing chunk frames = [7600128,7600640) | 20260506 00:43:01.846 | INFO | MemInfo: { | 20260506 00:43:01.846 | INFO | "gpu" : " 17 GB", | 20260506 00:43:01.846 | INFO | "huge" : "", | 20260506 00:43:01.846 | INFO | "huge_shared" : " 27 GB", | 20260506 00:43:01.846 | INFO | "malloc" : " 14 GB", | 20260506 00:43:01.846 | INFO | "pinned" : " 29 GB" | 20260506 00:43:01.846 | INFO | } | 20260506 00:43:01.846 | INFO | Perf report for the last 5.14449 seconds: | 20260506 00:43:01.846 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:43:01.846 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:01.846 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:01.846 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:01.846 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 00:43:01.846 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:01.846 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:01.846 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 00:43:01.846 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:01.846 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:01.846 | INFO | Uploaded 36GiB to GPU in 2.09447s (17.1881GiB/s) | 20260506 00:43:01.846 | INFO | Downloaded 34GiB to GPU in 2.04993s (16.5859GiB/s) | 20260506 00:43:06.985 | INFO | Analyzing chunk frames = [7600640,7601152) | 20260506 00:43:06.985 | INFO | MemInfo: { | 20260506 00:43:06.985 | INFO | "gpu" : " 17 GB", | 20260506 00:43:06.985 | INFO | "huge" : "", | 20260506 00:43:06.985 | INFO | "huge_shared" : " 27 GB", | 20260506 00:43:06.985 | INFO | "malloc" : " 14 GB", | 20260506 00:43:06.985 | INFO | "pinned" : " 29 GB" | 20260506 00:43:06.985 | INFO | } | 20260506 00:43:06.985 | INFO | Perf report for the last 5.13869 seconds: | 20260506 00:43:06.985 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:43:06.985 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:06.985 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:06.985 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:06.985 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 00:43:06.985 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:06.985 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:06.985 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 00:43:06.985 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:06.985 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:06.985 | INFO | Uploaded 36GiB to GPU in 2.08279s (17.2845GiB/s) | 20260506 00:43:06.985 | INFO | Downloaded 34GiB to GPU in 2.04505s (16.6255GiB/s) | 20260506 00:43:12.113 | INFO | Analyzing chunk frames = [7601152,7601664) | 20260506 00:43:12.113 | INFO | MemInfo: { | 20260506 00:43:12.113 | INFO | "gpu" : " 17 GB", | 20260506 00:43:12.113 | INFO | "huge" : "", | 20260506 00:43:12.113 | INFO | "huge_shared" : " 27 GB", | 20260506 00:43:12.113 | INFO | "malloc" : " 14 GB", | 20260506 00:43:12.113 | INFO | "pinned" : " 29 GB" | 20260506 00:43:12.113 | INFO | } | 20260506 00:43:12.113 | INFO | Perf report for the last 5.12874 seconds: | 20260506 00:43:12.113 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:43:12.113 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:12.113 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:12.113 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:12.113 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260506 00:43:12.113 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:12.113 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:12.113 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260506 00:43:12.113 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:12.113 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:12.114 | INFO | Uploaded 36GiB to GPU in 2.10346s (17.1146GiB/s) | 20260506 00:43:12.114 | INFO | Downloaded 34GiB to GPU in 2.07083s (16.4185GiB/s) | 20260506 00:43:17.258 | INFO | Analyzing chunk frames = [7601664,7602176) | 20260506 00:43:17.258 | INFO | MemInfo: { | 20260506 00:43:17.258 | INFO | "gpu" : " 17 GB", | 20260506 00:43:17.258 | INFO | "huge" : "", | 20260506 00:43:17.258 | INFO | "huge_shared" : " 27 GB", | 20260506 00:43:17.258 | INFO | "malloc" : " 14 GB", | 20260506 00:43:17.258 | INFO | "pinned" : " 29 GB" | 20260506 00:43:17.258 | INFO | } | 20260506 00:43:17.258 | INFO | Perf report for the last 5.14506 seconds: | 20260506 00:43:17.258 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:43:17.258 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:17.258 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:17.259 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:17.259 | INFO | ANALYSIS : 61.9%, 3.0, 96.8ms, 0 | 20260506 00:43:17.259 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:17.259 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:17.259 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 00:43:17.259 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:17.259 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:17.259 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 00:43:17.259 | INFO | Total mem usage: 15.3 GiB | 20260506 00:43:17.259 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 00:43:17.259 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.14%) | 20260506 00:43:17.259 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 00:43:17.259 | INFO | High Water Breakdown: | 20260506 00:43:17.259 | INFO | 60.9% : 8.55 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 00:43:17.259 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 00:43:17.259 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 00:43:17.259 | INFO | ------------------------------------------ | 20260506 00:43:17.259 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:43:17.259 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 00:43:17.259 | INFO | Total mem usage: 26.5 GiB | 20260506 00:43:17.259 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 00:43:17.259 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 00:43:17.259 | INFO | Uploaded 36GiB to GPU in 2.13358s (16.8731GiB/s) | 20260506 00:43:17.259 | INFO | Downloaded 34GiB to GPU in 2.02063s (16.8264GiB/s) | 20260506 00:43:17.259 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 00:43:17.259 | INFO | High Water Breakdown: | 20260506 00:43:17.259 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:43:17.259 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:43:17.259 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:43:17.259 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:43:17.259 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:43:17.259 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:43:17.259 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 00:43:17.259 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 00:43:17.259 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:43:17.259 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 00:43:17.259 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 00:43:17.259 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 00:43:17.259 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 00:43:17.259 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 00:43:17.259 | INFO | ------------------------------------------ | 20260506 00:43:17.259 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:43:17.259 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 00:43:17.259 | INFO | Total mem usage: 25.6 GiB | 20260506 00:43:17.259 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 00:43:17.259 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 00:43:17.259 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 00:43:17.259 | INFO | High Water Breakdown: | 20260506 00:43:17.259 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 00:43:17.260 | INFO | ------------------------------------------ | 20260506 00:43:17.260 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:43:17.260 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 00:43:17.260 | INFO | Total mem usage: 15.7 GiB | 20260506 00:43:17.260 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 00:43:17.260 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 00:43:17.260 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 00:43:17.260 | INFO | High Water Breakdown: | 20260506 00:43:17.260 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 00:43:17.260 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 00:43:17.260 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 00:43:17.260 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 00:43:17.260 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 00:43:17.260 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 00:43:17.260 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 00:43:17.260 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 00:43:17.260 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 00:43:17.260 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:43:17.260 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 00:43:17.260 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:43:17.260 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 00:43:17.260 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 00:43:17.260 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 00:43:17.260 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 00:43:17.260 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:43:17.260 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 00:43:17.260 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:43:17.260 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:43:17.260 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:43:17.260 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 00:43:17.260 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 00:43:17.260 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:43:17.260 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 00:43:17.260 | INFO | ------------------------------------------ | 20260506 00:43:17.260 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:43:17.260 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 00:43:17.260 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:43:17.260 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:43:17.260 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.39e+01, 1.34e+01 / 6.88e-01 | 20260506 00:43:17.260 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.03e-01, 1.02e-02 / 2.83e-03 | 20260506 00:43:17.260 | INFO | PerfReport: 11.84%, Baseline -- 3.70e+00 / 4.49e+00, 5.67e-01 / 2.06e-01 | 20260506 00:43:17.260 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.53e+00, 1.87e-01 / 7.99e-02 | 20260506 00:43:17.260 | INFO | PerfReport: 16.02%, DME -- 4.01e+01 / 4.41e+01, 3.91e+00 / 9.10e-01 | 20260506 00:43:17.260 | INFO | PerfReport: 57.30%, FrameLabeling -- 1.79e+01 / 1.93e+01, 1.44e+00 / 8.71e-01 | 20260506 00:43:17.260 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.23e+00 / 2.78e+00, 4.24e-01 / 2.10e-01 | 20260506 00:43:17.260 | INFO | PerfReport: 3.08%, Metrics -- 9.63e-01 / 1.26e+00, 2.59e-01 / 1.33e-01 | 20260506 00:43:17.260 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.33e-01 / 3.88e-01, 5.61e-02 / 3.09e-02 | 20260506 00:43:17.260 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 00:43:17.260 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:43:17.260 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:43:17.260 | INFO | PerfReport: Full Execution -- 9.83e+01 / -nan, 2.46e+01 / 0.00e+00 | 20260506 00:43:17.260 | INFO | PerfReport: 22.36%, Upload -- 2.20e+01 / 0.00e+00, 7.52e+00 / 0.00e+00 | 20260506 00:43:17.260 | INFO | PerfReport: 21.83%, Download -- 2.15e+01 / 0.00e+00, 7.57e+00 / 0.00e+00 | 20260506 00:43:17.260 | INFO | PerfReport: 31.82%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 00:43:17.260 | INFO | PerfReport: 2.14%, LockUpload -- 2.10e+00 / 0.00e+00, 7.91e+00 / 0.00e+00 | 20260506 00:43:17.260 | INFO | PerfReport: 1.33%, LockDownload -- 1.31e+00 / 0.00e+00, 2.93e+00 / 0.00e+00 | 20260506 00:43:17.260 | INFO | PerfReport: 20.51%, LockCompute -- 2.02e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 00:43:17.260 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.56e-02 / 0.00e+00 | 20260506 00:43:17.377 | INFO | Starting a round of Baz IO | 20260506 00:43:20.523 | INFO | Populating RTMetrics report | 20260506 00:43:20.523 | INFO | RTMetrics report populated | 20260506 00:43:20.523 | INFO | Opening tmp RTMetrics json file | 20260506 00:43:20.525 | INFO | Serializing RTMetrics to json | 20260506 00:43:20.528 | INFO | Serialization done, swapping tmp file now... | 20260506 00:43:20.528 | INFO | RTMetrics done | 20260506 00:43:20.602 | INFO | Finished a round of Baz IO: Wrote 1.853 GiB in 3.225 seconds | 20260506 00:43:20.602 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.35% / 0.00% / 78.65% | 20260506 00:43:20.602 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0888944% / 0% / 8.74201e-05% | 20260506 00:43:20.602 | INFO | Baz IO overall: | 20260506 00:43:20.603 | INFO | BytesWritten : 4310795648529 | 20260506 00:43:20.603 | INFO | NumEvents : 2027952787307 | 20260506 00:43:22.403 | INFO | Analyzing chunk frames = [7602176,7602688) | 20260506 00:43:22.403 | INFO | MemInfo: { | 20260506 00:43:22.403 | INFO | "gpu" : " 17 GB", | 20260506 00:43:22.403 | INFO | "huge" : "", | 20260506 00:43:22.403 | INFO | "huge_shared" : " 27 GB", | 20260506 00:43:22.403 | INFO | "malloc" : " 14 GB", | 20260506 00:43:22.403 | INFO | "pinned" : " 29 GB" | 20260506 00:43:22.403 | INFO | } | 20260506 00:43:22.403 | INFO | Perf report for the last 5.14494 seconds: | 20260506 00:43:22.403 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:43:22.404 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:22.404 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:22.404 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:22.404 | INFO | ANALYSIS : 62.4%, 3.0, 97.6ms, 0 | 20260506 00:43:22.404 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:22.404 | INFO | RT_METRICS : 43.0%, 1.0, 22.6ms, 0 | 20260506 00:43:22.404 | INFO | PRE_HQ : 59.6%, 1.7, 53.8ms, 0 | 20260506 00:43:22.404 | INFO | BAZWRITER : 61.8%, 3.5, 111.6ms, 0 | 20260506 00:43:22.404 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:43:22.404 | INFO | Uploaded 36GiB to GPU in 2.17658s (16.5397GiB/s) | 20260506 00:43:22.404 | INFO | Downloaded 38GiB to GPU in 2.24129s (16.9545GiB/s) | 20260506 00:43:27.541 | INFO | Analyzing chunk frames = [7602688,7603200) | 20260506 00:43:27.541 | INFO | MemInfo: { | 20260506 00:43:27.541 | INFO | "gpu" : " 17 GB", | 20260506 00:43:27.541 | INFO | "huge" : "", | 20260506 00:43:27.541 | INFO | "huge_shared" : " 27 GB", | 20260506 00:43:27.541 | INFO | "malloc" : " 14 GB", | 20260506 00:43:27.541 | INFO | "pinned" : " 29 GB" | 20260506 00:43:27.541 | INFO | } | 20260506 00:43:27.541 | INFO | Perf report for the last 5.13749 seconds: | 20260506 00:43:27.541 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:43:27.541 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:27.541 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:27.541 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:27.541 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 00:43:27.541 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:27.541 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:27.541 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:43:27.541 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:27.541 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:27.542 | INFO | Uploaded 36GiB to GPU in 2.1161s (17.0124GiB/s) | 20260506 00:43:27.542 | INFO | Downloaded 34GiB to GPU in 2.04197s (16.6505GiB/s) | 20260506 00:43:32.674 | INFO | Analyzing chunk frames = [7603200,7603712) | 20260506 00:43:32.674 | INFO | MemInfo: { | 20260506 00:43:32.674 | INFO | "gpu" : " 17 GB", | 20260506 00:43:32.674 | INFO | "huge" : "", | 20260506 00:43:32.674 | INFO | "huge_shared" : " 27 GB", | 20260506 00:43:32.674 | INFO | "malloc" : " 14 GB", | 20260506 00:43:32.674 | INFO | "pinned" : " 29 GB" | 20260506 00:43:32.674 | INFO | } | 20260506 00:43:32.675 | INFO | Perf report for the last 5.13357 seconds: | 20260506 00:43:32.675 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:43:32.675 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:32.675 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:32.675 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:32.675 | INFO | ANALYSIS : 61.4%, 3.0, 95.4ms, 0 | 20260506 00:43:32.675 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:32.675 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:32.675 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 00:43:32.675 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:32.675 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:32.675 | INFO | Uploaded 36GiB to GPU in 2.08354s (17.2783GiB/s) | 20260506 00:43:32.675 | INFO | Downloaded 34GiB to GPU in 2.02395s (16.7988GiB/s) | 20260506 00:43:37.816 | INFO | Analyzing chunk frames = [7603712,7604224) | 20260506 00:43:37.816 | INFO | MemInfo: { | 20260506 00:43:37.816 | INFO | "gpu" : " 17 GB", | 20260506 00:43:37.816 | INFO | "huge" : "", | 20260506 00:43:37.816 | INFO | "huge_shared" : " 27 GB", | 20260506 00:43:37.816 | INFO | "malloc" : " 14 GB", | 20260506 00:43:37.816 | INFO | "pinned" : " 29 GB" | 20260506 00:43:37.816 | INFO | } | 20260506 00:43:37.816 | INFO | Perf report for the last 5.14192 seconds: | 20260506 00:43:37.816 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:43:37.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:37.816 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:37.817 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:37.817 | INFO | ANALYSIS : 61.6%, 3.0, 95.7ms, 0 | 20260506 00:43:37.817 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:37.817 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:37.817 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:43:37.817 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:37.817 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:37.817 | INFO | Uploaded 36GiB to GPU in 2.14092s (16.8152GiB/s) | 20260506 00:43:37.817 | INFO | Downloaded 34GiB to GPU in 2.00607s (16.9485GiB/s) | 20260506 00:43:42.957 | INFO | Analyzing chunk frames = [7604224,7604736) | 20260506 00:43:42.957 | INFO | MemInfo: { | 20260506 00:43:42.957 | INFO | "gpu" : " 17 GB", | 20260506 00:43:42.957 | INFO | "huge" : "", | 20260506 00:43:42.957 | INFO | "huge_shared" : " 27 GB", | 20260506 00:43:42.957 | INFO | "malloc" : " 14 GB", | 20260506 00:43:42.957 | INFO | "pinned" : " 29 GB" | 20260506 00:43:42.957 | INFO | } | 20260506 00:43:42.957 | INFO | Perf report for the last 5.14076 seconds: | 20260506 00:43:42.957 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:43:42.957 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:42.957 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:42.957 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:42.957 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 00:43:42.957 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:42.957 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:42.957 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:43:42.957 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:42.957 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:42.957 | INFO | Uploaded 36GiB to GPU in 2.09803s (17.1589GiB/s) | 20260506 00:43:42.957 | INFO | Downloaded 34GiB to GPU in 2.04656s (16.6132GiB/s) | 20260506 00:43:48.093 | INFO | Analyzing chunk frames = [7604736,7605248) | 20260506 00:43:48.093 | INFO | MemInfo: { | 20260506 00:43:48.093 | INFO | "gpu" : " 17 GB", | 20260506 00:43:48.093 | INFO | "huge" : "", | 20260506 00:43:48.093 | INFO | "huge_shared" : " 27 GB", | 20260506 00:43:48.093 | INFO | "malloc" : " 14 GB", | 20260506 00:43:48.093 | INFO | "pinned" : " 29 GB" | 20260506 00:43:48.093 | INFO | } | 20260506 00:43:48.093 | INFO | Perf report for the last 5.13611 seconds: | 20260506 00:43:48.093 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:43:48.093 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:48.093 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:48.093 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:48.093 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 00:43:48.093 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:48.093 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:48.093 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 00:43:48.093 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:48.093 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:48.094 | INFO | Uploaded 36GiB to GPU in 2.06327s (17.448GiB/s) | 20260506 00:43:48.094 | INFO | Downloaded 34GiB to GPU in 2.03747s (16.6874GiB/s) | 20260506 00:43:53.228 | INFO | Analyzing chunk frames = [7605248,7605760) | 20260506 00:43:53.228 | INFO | MemInfo: { | 20260506 00:43:53.228 | INFO | "gpu" : " 17 GB", | 20260506 00:43:53.228 | INFO | "huge" : "", | 20260506 00:43:53.228 | INFO | "huge_shared" : " 27 GB", | 20260506 00:43:53.228 | INFO | "malloc" : " 14 GB", | 20260506 00:43:53.228 | INFO | "pinned" : " 29 GB" | 20260506 00:43:53.228 | INFO | } | 20260506 00:43:53.228 | INFO | Perf report for the last 5.13468 seconds: | 20260506 00:43:53.228 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:43:53.228 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:53.228 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:53.228 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:53.228 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260506 00:43:53.228 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:53.228 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:53.228 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 00:43:53.228 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:53.228 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:53.228 | INFO | Uploaded 36GiB to GPU in 2.02626s (17.7667GiB/s) | 20260506 00:43:53.228 | INFO | Downloaded 34GiB to GPU in 2.04146s (16.6547GiB/s) | 20260506 00:43:58.369 | INFO | Analyzing chunk frames = [7605760,7606272) | 20260506 00:43:58.369 | INFO | MemInfo: { | 20260506 00:43:58.369 | INFO | "gpu" : " 17 GB", | 20260506 00:43:58.369 | INFO | "huge" : "", | 20260506 00:43:58.369 | INFO | "huge_shared" : " 27 GB", | 20260506 00:43:58.369 | INFO | "malloc" : " 14 GB", | 20260506 00:43:58.369 | INFO | "pinned" : " 29 GB" | 20260506 00:43:58.369 | INFO | } | 20260506 00:43:58.369 | INFO | Perf report for the last 5.14072 seconds: | 20260506 00:43:58.369 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:43:58.369 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:58.369 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:58.369 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:58.369 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260506 00:43:58.369 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:58.369 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:58.369 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 00:43:58.369 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:58.369 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:43:58.369 | INFO | Uploaded 36GiB to GPU in 2.11108s (17.0529GiB/s) | 20260506 00:43:58.369 | INFO | Downloaded 34GiB to GPU in 2.00552s (16.9532GiB/s) | 20260506 00:43:58.491 | INFO | Starting a round of Baz IO | 20260506 00:44:01.637 | INFO | Populating RTMetrics report | 20260506 00:44:01.638 | INFO | RTMetrics report populated | 20260506 00:44:01.638 | INFO | Opening tmp RTMetrics json file | 20260506 00:44:01.638 | INFO | Serializing RTMetrics to json | 20260506 00:44:01.639 | INFO | Serialization done, swapping tmp file now... | 20260506 00:44:01.640 | INFO | RTMetrics done | 20260506 00:44:01.719 | INFO | Finished a round of Baz IO: Wrote 1.85 GiB in 3.228 seconds | 20260506 00:44:01.719 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.35% / 0.00% / 78.65% | 20260506 00:44:01.719 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0886162% / 0% / 6.75519e-05% | 20260506 00:44:01.719 | INFO | Baz IO overall: | 20260506 00:44:01.719 | INFO | BytesWritten : 4312782580007 | 20260506 00:44:01.720 | INFO | NumEvents : 2028859488352 | 20260506 00:44:03.510 | INFO | Analyzing chunk frames = [7606272,7606784) | 20260506 00:44:03.510 | INFO | MemInfo: { | 20260506 00:44:03.510 | INFO | "gpu" : " 17 GB", | 20260506 00:44:03.510 | INFO | "huge" : "", | 20260506 00:44:03.510 | INFO | "huge_shared" : " 27 GB", | 20260506 00:44:03.510 | INFO | "malloc" : " 14 GB", | 20260506 00:44:03.510 | INFO | "pinned" : " 29 GB" | 20260506 00:44:03.510 | INFO | } | 20260506 00:44:03.510 | INFO | Perf report for the last 5.14097 seconds: | 20260506 00:44:03.510 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:44:03.510 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:03.510 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:03.510 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:03.510 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260506 00:44:03.510 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:03.510 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260506 00:44:03.510 | INFO | PRE_HQ : 60.0%, 1.7, 54.4ms, 0 | 20260506 00:44:03.510 | INFO | BAZWRITER : 61.9%, 3.5, 111.1ms, 0 | 20260506 00:44:03.510 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:44:03.510 | INFO | Uploaded 36GiB to GPU in 2.1727s (16.5693GiB/s) | 20260506 00:44:03.510 | INFO | Downloaded 38GiB to GPU in 2.23884s (16.9731GiB/s) | 20260506 00:44:08.659 | INFO | Analyzing chunk frames = [7606784,7607296) | 20260506 00:44:08.659 | INFO | MemInfo: { | 20260506 00:44:08.659 | INFO | "gpu" : " 17 GB", | 20260506 00:44:08.659 | INFO | "huge" : "", | 20260506 00:44:08.659 | INFO | "huge_shared" : " 27 GB", | 20260506 00:44:08.659 | INFO | "malloc" : " 14 GB", | 20260506 00:44:08.659 | INFO | "pinned" : " 29 GB" | 20260506 00:44:08.659 | INFO | } | 20260506 00:44:08.659 | INFO | Perf report for the last 5.14903 seconds: | 20260506 00:44:08.659 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:44:08.659 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:08.659 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:08.659 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:08.659 | INFO | ANALYSIS : 61.6%, 3.0, 96.4ms, 0 | 20260506 00:44:08.659 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:08.659 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:08.659 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 00:44:08.659 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:08.659 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:08.659 | INFO | Uploaded 36GiB to GPU in 2.11731s (17.0027GiB/s) | 20260506 00:44:08.659 | INFO | Downloaded 34GiB to GPU in 2.04058s (16.662GiB/s) | 20260506 00:44:13.794 | INFO | Analyzing chunk frames = [7607296,7607808) | 20260506 00:44:13.794 | INFO | MemInfo: { | 20260506 00:44:13.794 | INFO | "gpu" : " 17 GB", | 20260506 00:44:13.794 | INFO | "huge" : "", | 20260506 00:44:13.794 | INFO | "huge_shared" : " 27 GB", | 20260506 00:44:13.794 | INFO | "malloc" : " 14 GB", | 20260506 00:44:13.794 | INFO | "pinned" : " 29 GB" | 20260506 00:44:13.794 | INFO | } | 20260506 00:44:13.794 | INFO | Perf report for the last 5.13532 seconds: | 20260506 00:44:13.794 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:44:13.794 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:13.794 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:13.794 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:13.794 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 00:44:13.794 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:13.794 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:13.794 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 00:44:13.794 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:13.794 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:13.794 | INFO | Uploaded 36GiB to GPU in 2.08946s (17.2293GiB/s) | 20260506 00:44:13.795 | INFO | Downloaded 34GiB to GPU in 2.02305s (16.8063GiB/s) | 20260506 00:44:18.927 | INFO | Analyzing chunk frames = [7607808,7608320) | 20260506 00:44:18.927 | INFO | MemInfo: { | 20260506 00:44:18.927 | INFO | "gpu" : " 17 GB", | 20260506 00:44:18.927 | INFO | "huge" : "", | 20260506 00:44:18.927 | INFO | "huge_shared" : " 27 GB", | 20260506 00:44:18.927 | INFO | "malloc" : " 14 GB", | 20260506 00:44:18.927 | INFO | "pinned" : " 29 GB" | 20260506 00:44:18.927 | INFO | } | 20260506 00:44:18.927 | INFO | Perf report for the last 5.1327 seconds: | 20260506 00:44:18.927 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:44:18.927 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:18.927 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:18.927 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:18.927 | INFO | ANALYSIS : 61.6%, 3.0, 95.6ms, 0 | 20260506 00:44:18.927 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:18.927 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:18.927 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:44:18.927 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:18.927 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:18.927 | INFO | Uploaded 36GiB to GPU in 2.14488s (16.7841GiB/s) | 20260506 00:44:18.927 | INFO | Downloaded 34GiB to GPU in 2.04966s (16.5881GiB/s) | 20260506 00:44:24.066 | INFO | Analyzing chunk frames = [7608320,7608832) | 20260506 00:44:24.066 | INFO | MemInfo: { | 20260506 00:44:24.066 | INFO | "gpu" : " 17 GB", | 20260506 00:44:24.066 | INFO | "huge" : "", | 20260506 00:44:24.066 | INFO | "huge_shared" : " 27 GB", | 20260506 00:44:24.066 | INFO | "malloc" : " 14 GB", | 20260506 00:44:24.066 | INFO | "pinned" : " 29 GB" | 20260506 00:44:24.066 | INFO | } | 20260506 00:44:24.066 | INFO | Perf report for the last 5.13929 seconds: | 20260506 00:44:24.066 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:44:24.066 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:24.066 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:24.066 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:24.066 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 00:44:24.066 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:24.066 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:24.066 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:44:24.066 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:24.066 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:24.066 | INFO | Uploaded 36GiB to GPU in 2.10738s (17.0828GiB/s) | 20260506 00:44:24.066 | INFO | Downloaded 34GiB to GPU in 2.05057s (16.5808GiB/s) | 20260506 00:44:29.208 | INFO | Analyzing chunk frames = [7608832,7609344) | 20260506 00:44:29.208 | INFO | MemInfo: { | 20260506 00:44:29.208 | INFO | "gpu" : " 17 GB", | 20260506 00:44:29.208 | INFO | "huge" : "", | 20260506 00:44:29.208 | INFO | "huge_shared" : " 27 GB", | 20260506 00:44:29.208 | INFO | "malloc" : " 14 GB", | 20260506 00:44:29.208 | INFO | "pinned" : " 29 GB" | 20260506 00:44:29.208 | INFO | } | 20260506 00:44:29.209 | INFO | Perf report for the last 5.14235 seconds: | 20260506 00:44:29.209 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:44:29.209 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:29.209 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:29.209 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:29.209 | INFO | ANALYSIS : 61.2%, 3.0, 95.6ms, 0 | 20260506 00:44:29.209 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:29.209 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:29.209 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 00:44:29.209 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:29.209 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:29.209 | INFO | Uploaded 36GiB to GPU in 2.0731s (17.3653GiB/s) | 20260506 00:44:29.209 | INFO | Downloaded 34GiB to GPU in 2.03932s (16.6722GiB/s) | 20260506 00:44:34.355 | INFO | Analyzing chunk frames = [7609344,7609856) | 20260506 00:44:34.355 | INFO | MemInfo: { | 20260506 00:44:34.355 | INFO | "gpu" : " 17 GB", | 20260506 00:44:34.355 | INFO | "huge" : "", | 20260506 00:44:34.355 | INFO | "huge_shared" : " 27 GB", | 20260506 00:44:34.355 | INFO | "malloc" : " 14 GB", | 20260506 00:44:34.355 | INFO | "pinned" : " 29 GB" | 20260506 00:44:34.355 | INFO | } | 20260506 00:44:34.355 | INFO | Perf report for the last 5.14645 seconds: | 20260506 00:44:34.355 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:44:34.355 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:34.355 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:34.355 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:34.355 | INFO | ANALYSIS : 62.2%, 3.0, 97.3ms, 0 | 20260506 00:44:34.355 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:34.355 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:34.355 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 00:44:34.355 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:34.355 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:34.355 | INFO | Uploaded 36GiB to GPU in 2.10353s (17.114GiB/s) | 20260506 00:44:34.355 | INFO | Downloaded 34GiB to GPU in 2.07684s (16.371GiB/s) | 20260506 00:44:39.488 | INFO | Analyzing chunk frames = [7609856,7610368) | 20260506 00:44:39.488 | INFO | MemInfo: { | 20260506 00:44:39.488 | INFO | "gpu" : " 17 GB", | 20260506 00:44:39.488 | INFO | "huge" : "", | 20260506 00:44:39.488 | INFO | "huge_shared" : " 27 GB", | 20260506 00:44:39.488 | INFO | "malloc" : " 14 GB", | 20260506 00:44:39.488 | INFO | "pinned" : " 29 GB" | 20260506 00:44:39.488 | INFO | } | 20260506 00:44:39.488 | INFO | Perf report for the last 5.13289 seconds: | 20260506 00:44:39.488 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:44:39.488 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:39.488 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:39.488 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:39.488 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260506 00:44:39.488 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:39.488 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:39.488 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:44:39.488 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:39.488 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:39.488 | INFO | Uploaded 36GiB to GPU in 2.12976s (16.9033GiB/s) | 20260506 00:44:39.488 | INFO | Downloaded 34GiB to GPU in 2.01996s (16.832GiB/s) | 20260506 00:44:39.488 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 00:44:39.488 | INFO | Total mem usage: 15.3 GiB | 20260506 00:44:39.488 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 00:44:39.488 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.16%) | 20260506 00:44:39.488 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 00:44:39.488 | INFO | High Water Breakdown: | 20260506 00:44:39.488 | INFO | 60.9% : 8.55 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 00:44:39.488 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 00:44:39.488 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 00:44:39.488 | INFO | ------------------------------------------ | 20260506 00:44:39.488 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:44:39.488 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 00:44:39.488 | INFO | Total mem usage: 26.5 GiB | 20260506 00:44:39.488 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 00:44:39.488 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 00:44:39.488 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 00:44:39.488 | INFO | High Water Breakdown: | 20260506 00:44:39.489 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:44:39.489 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:44:39.489 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:44:39.489 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:44:39.489 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:44:39.489 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:44:39.489 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 00:44:39.489 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 00:44:39.489 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:44:39.489 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 00:44:39.489 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 00:44:39.489 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 00:44:39.489 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 00:44:39.489 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 00:44:39.489 | INFO | ------------------------------------------ | 20260506 00:44:39.489 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:44:39.489 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 00:44:39.489 | INFO | Total mem usage: 25.6 GiB | 20260506 00:44:39.489 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 00:44:39.489 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 00:44:39.489 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 00:44:39.489 | INFO | High Water Breakdown: | 20260506 00:44:39.489 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 00:44:39.489 | INFO | ------------------------------------------ | 20260506 00:44:39.489 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:44:39.489 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 00:44:39.489 | INFO | Total mem usage: 15.7 GiB | 20260506 00:44:39.489 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 00:44:39.489 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 00:44:39.489 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 00:44:39.489 | INFO | High Water Breakdown: | 20260506 00:44:39.489 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 00:44:39.489 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 00:44:39.489 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 00:44:39.489 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 00:44:39.489 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 00:44:39.489 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 00:44:39.489 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 00:44:39.489 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 00:44:39.489 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 00:44:39.489 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:44:39.489 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 00:44:39.489 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:44:39.489 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 00:44:39.489 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 00:44:39.489 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 00:44:39.489 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 00:44:39.489 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:44:39.489 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 00:44:39.489 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:44:39.489 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:44:39.489 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:44:39.489 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 00:44:39.489 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 00:44:39.489 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:44:39.489 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 00:44:39.489 | INFO | ------------------------------------------ | 20260506 00:44:39.489 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:44:39.489 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 00:44:39.489 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:44:39.489 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:44:39.489 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.39e+01, 1.33e+01 / 8.54e-01 | 20260506 00:44:39.489 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.04e-01, 1.02e-02 / 4.00e-03 | 20260506 00:44:39.489 | INFO | PerfReport: 11.85%, Baseline -- 3.71e+00 / 4.51e+00, 5.76e-01 / 1.76e-01 | 20260506 00:44:39.489 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.53e+00, 1.87e-01 / 1.00e-01 | 20260506 00:44:39.489 | INFO | PerfReport: 16.00%, DME -- 4.00e+01 / 4.39e+01, 3.62e+00 / 1.09e+00 | 20260506 00:44:39.489 | INFO | PerfReport: 57.35%, FrameLabeling -- 1.79e+01 / 1.94e+01, 1.45e+00 / 1.13e+00 | 20260506 00:44:39.489 | INFO | PerfReport: 7.10%, PulseAccumulating -- 2.22e+00 / 2.75e+00, 4.24e-01 / 2.34e-01 | 20260506 00:44:39.489 | INFO | PerfReport: 3.07%, Metrics -- 9.61e-01 / 1.24e+00, 2.59e-01 / 1.30e-01 | 20260506 00:44:39.489 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.33e-01 / 3.87e-01, 5.64e-02 / 2.93e-02 | 20260506 00:44:39.489 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 00:44:39.489 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:44:39.489 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:44:39.489 | INFO | PerfReport: Full Execution -- 9.82e+01 / -nan, 2.43e+01 / 0.00e+00 | 20260506 00:44:39.489 | INFO | PerfReport: 22.38%, Upload -- 2.20e+01 / 0.00e+00, 7.55e+00 / 0.00e+00 | 20260506 00:44:39.490 | INFO | PerfReport: 21.87%, Download -- 2.15e+01 / 0.00e+00, 7.62e+00 / 0.00e+00 | 20260506 00:44:39.490 | INFO | PerfReport: 31.85%, Compute -- 3.13e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 00:44:39.490 | INFO | PerfReport: 2.11%, LockUpload -- 2.07e+00 / 0.00e+00, 7.86e+00 / 0.00e+00 | 20260506 00:44:39.490 | INFO | PerfReport: 1.31%, LockDownload -- 1.29e+00 / 0.00e+00, 2.84e+00 / 0.00e+00 | 20260506 00:44:39.490 | INFO | PerfReport: 20.48%, LockCompute -- 2.01e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 00:44:39.490 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.56e-02 / 0.00e+00 | 20260506 00:44:39.605 | INFO | Starting a round of Baz IO | 20260506 00:44:42.748 | INFO | Populating RTMetrics report | 20260506 00:44:42.748 | INFO | RTMetrics report populated | 20260506 00:44:42.748 | INFO | Opening tmp RTMetrics json file | 20260506 00:44:42.749 | INFO | Serializing RTMetrics to json | 20260506 00:44:42.750 | INFO | Serialization done, swapping tmp file now... | 20260506 00:44:42.750 | INFO | RTMetrics done | 20260506 00:44:42.824 | INFO | Finished a round of Baz IO: Wrote 1.85 GiB in 3.219 seconds | 20260506 00:44:42.824 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.35% / 0.00% / 78.65% | 20260506 00:44:42.824 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0889937% / 0% / 7.54992e-05% | 20260506 00:44:42.824 | INFO | Baz IO overall: | 20260506 00:44:42.824 | INFO | BytesWritten : 4314768904525 | 20260506 00:44:42.824 | INFO | NumEvents : 2029766236276 | 20260506 00:44:44.624 | INFO | Analyzing chunk frames = [7610368,7610880) | 20260506 00:44:44.624 | INFO | MemInfo: { | 20260506 00:44:44.624 | INFO | "gpu" : " 17 GB", | 20260506 00:44:44.624 | INFO | "huge" : "", | 20260506 00:44:44.624 | INFO | "huge_shared" : " 27 GB", | 20260506 00:44:44.624 | INFO | "malloc" : " 14 GB", | 20260506 00:44:44.624 | INFO | "pinned" : " 29 GB" | 20260506 00:44:44.624 | INFO | } | 20260506 00:44:44.624 | INFO | Perf report for the last 5.13656 seconds: | 20260506 00:44:44.625 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:44:44.625 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:44.625 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:44.625 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:44.625 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260506 00:44:44.625 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:44.625 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260506 00:44:44.625 | INFO | PRE_HQ : 59.7%, 1.7, 53.8ms, 0 | 20260506 00:44:44.625 | INFO | BAZWRITER : 61.8%, 3.5, 110.0ms, 0 | 20260506 00:44:44.625 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:44:44.625 | INFO | Uploaded 36GiB to GPU in 2.17127s (16.5802GiB/s) | 20260506 00:44:44.625 | INFO | Downloaded 38GiB to GPU in 2.24083s (16.958GiB/s) | 20260506 00:44:49.773 | INFO | Analyzing chunk frames = [7610880,7611392) | 20260506 00:44:49.773 | INFO | MemInfo: { | 20260506 00:44:49.773 | INFO | "gpu" : " 17 GB", | 20260506 00:44:49.773 | INFO | "huge" : "", | 20260506 00:44:49.773 | INFO | "huge_shared" : " 27 GB", | 20260506 00:44:49.773 | INFO | "malloc" : " 14 GB", | 20260506 00:44:49.773 | INFO | "pinned" : " 29 GB" | 20260506 00:44:49.773 | INFO | } | 20260506 00:44:49.773 | INFO | Perf report for the last 5.1484 seconds: | 20260506 00:44:49.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:44:49.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:49.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:49.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:49.773 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260506 00:44:49.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:49.773 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:49.773 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 00:44:49.773 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:49.773 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:49.773 | INFO | Uploaded 36GiB to GPU in 2.09416s (17.1906GiB/s) | 20260506 00:44:49.773 | INFO | Downloaded 34GiB to GPU in 2.02944s (16.7534GiB/s) | 20260506 00:44:54.904 | INFO | Analyzing chunk frames = [7611392,7611904) | 20260506 00:44:54.905 | INFO | MemInfo: { | 20260506 00:44:54.905 | INFO | "gpu" : " 17 GB", | 20260506 00:44:54.905 | INFO | "huge" : "", | 20260506 00:44:54.905 | INFO | "huge_shared" : " 27 GB", | 20260506 00:44:54.905 | INFO | "malloc" : " 14 GB", | 20260506 00:44:54.905 | INFO | "pinned" : " 29 GB" | 20260506 00:44:54.905 | INFO | } | 20260506 00:44:54.905 | INFO | Perf report for the last 5.13189 seconds: | 20260506 00:44:54.905 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:44:54.905 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:54.905 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:54.905 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:54.905 | INFO | ANALYSIS : 61.3%, 3.0, 95.2ms, 0 | 20260506 00:44:54.905 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:54.905 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:54.905 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 00:44:54.905 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:54.905 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:44:54.906 | INFO | Uploaded 36GiB to GPU in 2.09156s (17.2121GiB/s) | 20260506 00:44:54.906 | INFO | Downloaded 34GiB to GPU in 2.02599s (16.7819GiB/s) | 20260506 00:45:00.040 | INFO | Analyzing chunk frames = [7611904,7612416) | 20260506 00:45:00.040 | INFO | MemInfo: { | 20260506 00:45:00.040 | INFO | "gpu" : " 17 GB", | 20260506 00:45:00.040 | INFO | "huge" : "", | 20260506 00:45:00.040 | INFO | "huge_shared" : " 27 GB", | 20260506 00:45:00.040 | INFO | "malloc" : " 14 GB", | 20260506 00:45:00.040 | INFO | "pinned" : " 29 GB" | 20260506 00:45:00.040 | INFO | } | 20260506 00:45:00.040 | INFO | Perf report for the last 5.13539 seconds: | 20260506 00:45:00.040 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:45:00.040 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:00.040 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:00.040 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:00.040 | INFO | ANALYSIS : 61.6%, 3.0, 95.6ms, 0 | 20260506 00:45:00.040 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:00.040 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:00.040 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:45:00.040 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:00.040 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:00.040 | INFO | Uploaded 36GiB to GPU in 2.12979s (16.9031GiB/s) | 20260506 00:45:00.040 | INFO | Downloaded 34GiB to GPU in 2.00395s (16.9665GiB/s) | 20260506 00:45:05.178 | INFO | Analyzing chunk frames = [7612416,7612928) | 20260506 00:45:05.178 | INFO | MemInfo: { | 20260506 00:45:05.178 | INFO | "gpu" : " 17 GB", | 20260506 00:45:05.178 | INFO | "huge" : "", | 20260506 00:45:05.178 | INFO | "huge_shared" : " 27 GB", | 20260506 00:45:05.178 | INFO | "malloc" : " 14 GB", | 20260506 00:45:05.178 | INFO | "pinned" : " 29 GB" | 20260506 00:45:05.178 | INFO | } | 20260506 00:45:05.178 | INFO | Perf report for the last 5.13777 seconds: | 20260506 00:45:05.178 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:45:05.178 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:05.178 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:05.178 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:05.178 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 00:45:05.178 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:05.178 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:05.178 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:45:05.178 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:05.178 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:05.178 | INFO | Uploaded 36GiB to GPU in 2.095s (17.1837GiB/s) | 20260506 00:45:05.178 | INFO | Downloaded 34GiB to GPU in 2.04555s (16.6214GiB/s) | 20260506 00:45:10.324 | INFO | Analyzing chunk frames = [7612928,7613440) | 20260506 00:45:10.324 | INFO | MemInfo: { | 20260506 00:45:10.324 | INFO | "gpu" : " 17 GB", | 20260506 00:45:10.324 | INFO | "huge" : "", | 20260506 00:45:10.324 | INFO | "huge_shared" : " 27 GB", | 20260506 00:45:10.324 | INFO | "malloc" : " 14 GB", | 20260506 00:45:10.324 | INFO | "pinned" : " 29 GB" | 20260506 00:45:10.324 | INFO | } | 20260506 00:45:10.324 | INFO | Perf report for the last 5.14592 seconds: | 20260506 00:45:10.324 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:45:10.324 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:10.324 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:10.324 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:10.324 | INFO | ANALYSIS : 61.2%, 3.0, 95.7ms, 0 | 20260506 00:45:10.324 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:10.324 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:10.324 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:45:10.324 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:10.324 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:10.324 | INFO | Uploaded 36GiB to GPU in 2.07141s (17.3795GiB/s) | 20260506 00:45:10.324 | INFO | Downloaded 34GiB to GPU in 2.03826s (16.6809GiB/s) | 20260506 00:45:15.463 | INFO | Analyzing chunk frames = [7613440,7613952) | 20260506 00:45:15.463 | INFO | MemInfo: { | 20260506 00:45:15.463 | INFO | "gpu" : " 17 GB", | 20260506 00:45:15.463 | INFO | "huge" : "", | 20260506 00:45:15.463 | INFO | "huge_shared" : " 27 GB", | 20260506 00:45:15.463 | INFO | "malloc" : " 14 GB", | 20260506 00:45:15.463 | INFO | "pinned" : " 29 GB" | 20260506 00:45:15.463 | INFO | } | 20260506 00:45:15.464 | INFO | Perf report for the last 5.13962 seconds: | 20260506 00:45:15.464 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:45:15.464 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:15.464 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:15.464 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:15.464 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 00:45:15.464 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:15.464 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:15.464 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 00:45:15.464 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:15.464 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:15.464 | INFO | Uploaded 36GiB to GPU in 2.00739s (17.9337GiB/s) | 20260506 00:45:15.464 | INFO | Downloaded 34GiB to GPU in 2.0342s (16.7142GiB/s) | 20260506 00:45:20.609 | INFO | Analyzing chunk frames = [7613952,7614464) | 20260506 00:45:20.609 | INFO | MemInfo: { | 20260506 00:45:20.609 | INFO | "gpu" : " 17 GB", | 20260506 00:45:20.609 | INFO | "huge" : "", | 20260506 00:45:20.609 | INFO | "huge_shared" : " 27 GB", | 20260506 00:45:20.609 | INFO | "malloc" : " 14 GB", | 20260506 00:45:20.609 | INFO | "pinned" : " 29 GB" | 20260506 00:45:20.609 | INFO | } | 20260506 00:45:20.610 | INFO | Perf report for the last 5.14622 seconds: | 20260506 00:45:20.610 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:45:20.610 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:20.610 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:20.610 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:20.610 | INFO | ANALYSIS : 61.8%, 3.0, 96.7ms, 0 | 20260506 00:45:20.610 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:20.610 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:20.610 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:45:20.610 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:20.610 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:20.610 | INFO | Uploaded 36GiB to GPU in 2.13603s (16.8537GiB/s) | 20260506 00:45:20.610 | INFO | Downloaded 34GiB to GPU in 2.02692s (16.7742GiB/s) | 20260506 00:45:20.727 | INFO | Starting a round of Baz IO | 20260506 00:45:23.872 | INFO | Populating RTMetrics report | 20260506 00:45:23.873 | INFO | RTMetrics report populated | 20260506 00:45:23.873 | INFO | Opening tmp RTMetrics json file | 20260506 00:45:23.875 | INFO | Serializing RTMetrics to json | 20260506 00:45:23.877 | INFO | Serialization done, swapping tmp file now... | 20260506 00:45:23.880 | INFO | RTMetrics done | 20260506 00:45:23.953 | INFO | Finished a round of Baz IO: Wrote 1.85 GiB in 3.225 seconds | 20260506 00:45:23.953 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.35% / 0.00% / 78.65% | 20260506 00:45:23.953 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.088644% / 0% / 7.15256e-05% | 20260506 00:45:23.953 | INFO | Baz IO overall: | 20260506 00:45:23.953 | INFO | BytesWritten : 4316755504512 | 20260506 00:45:23.953 | INFO | NumEvents : 2030673955416 | 20260506 00:45:25.742 | INFO | Analyzing chunk frames = [7614464,7614976) | 20260506 00:45:25.742 | INFO | MemInfo: { | 20260506 00:45:25.742 | INFO | "gpu" : " 17 GB", | 20260506 00:45:25.742 | INFO | "huge" : "", | 20260506 00:45:25.742 | INFO | "huge_shared" : " 27 GB", | 20260506 00:45:25.742 | INFO | "malloc" : " 14 GB", | 20260506 00:45:25.742 | INFO | "pinned" : " 29 GB" | 20260506 00:45:25.742 | INFO | } | 20260506 00:45:25.742 | INFO | Perf report for the last 5.13204 seconds: | 20260506 00:45:25.742 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:45:25.742 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:25.742 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:25.742 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:25.742 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 00:45:25.742 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:25.742 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260506 00:45:25.742 | INFO | PRE_HQ : 59.9%, 1.7, 54.2ms, 0 | 20260506 00:45:25.742 | INFO | BAZWRITER : 61.9%, 3.5, 111.2ms, 0 | 20260506 00:45:25.742 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 00:45:25.742 | INFO | Uploaded 36GiB to GPU in 2.17987s (16.5148GiB/s) | 20260506 00:45:25.742 | INFO | Downloaded 38GiB to GPU in 2.24502s (16.9264GiB/s) | 20260506 00:45:30.876 | INFO | Analyzing chunk frames = [7614976,7615488) | 20260506 00:45:30.876 | INFO | MemInfo: { | 20260506 00:45:30.876 | INFO | "gpu" : " 17 GB", | 20260506 00:45:30.876 | INFO | "huge" : "", | 20260506 00:45:30.876 | INFO | "huge_shared" : " 27 GB", | 20260506 00:45:30.876 | INFO | "malloc" : " 14 GB", | 20260506 00:45:30.876 | INFO | "pinned" : " 29 GB" | 20260506 00:45:30.876 | INFO | } | 20260506 00:45:30.876 | INFO | Perf report for the last 5.13453 seconds: | 20260506 00:45:30.876 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:45:30.876 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:30.876 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:30.876 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:30.876 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 00:45:30.876 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:30.876 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:30.876 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 00:45:30.876 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:30.876 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:30.877 | INFO | Uploaded 36GiB to GPU in 2.09541s (17.1804GiB/s) | 20260506 00:45:30.877 | INFO | Downloaded 34GiB to GPU in 2.03147s (16.7367GiB/s) | 20260506 00:45:36.020 | INFO | Analyzing chunk frames = [7615488,7616000) | 20260506 00:45:36.020 | INFO | MemInfo: { | 20260506 00:45:36.020 | INFO | "gpu" : " 17 GB", | 20260506 00:45:36.020 | INFO | "huge" : "", | 20260506 00:45:36.020 | INFO | "huge_shared" : " 27 GB", | 20260506 00:45:36.020 | INFO | "malloc" : " 14 GB", | 20260506 00:45:36.020 | INFO | "pinned" : " 29 GB" | 20260506 00:45:36.020 | INFO | } | 20260506 00:45:36.020 | INFO | Perf report for the last 5.14324 seconds: | 20260506 00:45:36.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:45:36.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:36.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:36.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:36.020 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 00:45:36.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:36.020 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:36.020 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:45:36.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:36.020 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:36.020 | INFO | Uploaded 36GiB to GPU in 2.08782s (17.2429GiB/s) | 20260506 00:45:36.020 | INFO | Downloaded 34GiB to GPU in 2.03049s (16.7448GiB/s) | 20260506 00:45:41.161 | INFO | Analyzing chunk frames = [7616000,7616512) | 20260506 00:45:41.161 | INFO | MemInfo: { | 20260506 00:45:41.161 | INFO | "gpu" : " 17 GB", | 20260506 00:45:41.161 | INFO | "huge" : "", | 20260506 00:45:41.161 | INFO | "huge_shared" : " 27 GB", | 20260506 00:45:41.161 | INFO | "malloc" : " 14 GB", | 20260506 00:45:41.161 | INFO | "pinned" : " 29 GB" | 20260506 00:45:41.161 | INFO | } | 20260506 00:45:41.161 | INFO | Perf report for the last 5.14113 seconds: | 20260506 00:45:41.161 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:45:41.161 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:41.161 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:41.161 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:41.161 | INFO | ANALYSIS : 61.6%, 3.0, 95.6ms, 0 | 20260506 00:45:41.161 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:41.161 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:41.161 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 00:45:41.161 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:41.161 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:41.161 | INFO | Uploaded 36GiB to GPU in 2.14284s (16.8001GiB/s) | 20260506 00:45:41.161 | INFO | Downloaded 34GiB to GPU in 2.04325s (16.6402GiB/s) | 20260506 00:45:46.306 | INFO | Analyzing chunk frames = [7616512,7617024) | 20260506 00:45:46.306 | INFO | MemInfo: { | 20260506 00:45:46.306 | INFO | "gpu" : " 17 GB", | 20260506 00:45:46.306 | INFO | "huge" : "", | 20260506 00:45:46.306 | INFO | "huge_shared" : " 27 GB", | 20260506 00:45:46.306 | INFO | "malloc" : " 14 GB", | 20260506 00:45:46.306 | INFO | "pinned" : " 29 GB" | 20260506 00:45:46.306 | INFO | } | 20260506 00:45:46.306 | INFO | Perf report for the last 5.14516 seconds: | 20260506 00:45:46.306 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:45:46.306 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:46.306 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:46.306 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:46.306 | INFO | ANALYSIS : 60.9%, 3.0, 95.2ms, 0 | 20260506 00:45:46.306 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:46.306 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:46.306 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:45:46.306 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:46.306 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:46.306 | INFO | Uploaded 36GiB to GPU in 2.09728s (17.1651GiB/s) | 20260506 00:45:46.306 | INFO | Downloaded 34GiB to GPU in 2.04809s (16.6008GiB/s) | 20260506 00:45:51.435 | INFO | Analyzing chunk frames = [7617024,7617536) | 20260506 00:45:51.435 | INFO | MemInfo: { | 20260506 00:45:51.435 | INFO | "gpu" : " 17 GB", | 20260506 00:45:51.435 | INFO | "huge" : "", | 20260506 00:45:51.435 | INFO | "huge_shared" : " 27 GB", | 20260506 00:45:51.435 | INFO | "malloc" : " 14 GB", | 20260506 00:45:51.435 | INFO | "pinned" : " 29 GB" | 20260506 00:45:51.435 | INFO | } | 20260506 00:45:51.435 | INFO | Perf report for the last 5.12939 seconds: | 20260506 00:45:51.435 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:45:51.436 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:51.436 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:51.436 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:51.436 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 00:45:51.436 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:51.436 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:51.436 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:45:51.436 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:51.436 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:51.436 | INFO | Uploaded 36GiB to GPU in 2.0664s (17.4216GiB/s) | 20260506 00:45:51.436 | INFO | Downloaded 34GiB to GPU in 2.03629s (16.697GiB/s) | 20260506 00:45:56.578 | INFO | Analyzing chunk frames = [7617536,7618048) | 20260506 00:45:56.578 | INFO | MemInfo: { | 20260506 00:45:56.578 | INFO | "gpu" : " 17 GB", | 20260506 00:45:56.578 | INFO | "huge" : "", | 20260506 00:45:56.578 | INFO | "huge_shared" : " 27 GB", | 20260506 00:45:56.578 | INFO | "malloc" : " 14 GB", | 20260506 00:45:56.578 | INFO | "pinned" : " 29 GB" | 20260506 00:45:56.578 | INFO | } | 20260506 00:45:56.578 | INFO | Perf report for the last 5.1423 seconds: | 20260506 00:45:56.578 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:45:56.578 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:56.578 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:56.578 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:56.578 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260506 00:45:56.578 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:56.578 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:56.578 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 00:45:56.578 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:56.578 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:45:56.578 | INFO | Uploaded 36GiB to GPU in 2.02549s (17.7734GiB/s) | 20260506 00:45:56.578 | INFO | Downloaded 34GiB to GPU in 2.03614s (16.6982GiB/s) | 20260506 00:46:01.720 | INFO | Analyzing chunk frames = [7618048,7618560) | 20260506 00:46:01.720 | INFO | MemInfo: { | 20260506 00:46:01.720 | INFO | "gpu" : " 17 GB", | 20260506 00:46:01.720 | INFO | "huge" : "", | 20260506 00:46:01.720 | INFO | "huge_shared" : " 27 GB", | 20260506 00:46:01.720 | INFO | "malloc" : " 14 GB", | 20260506 00:46:01.720 | INFO | "pinned" : " 29 GB" | 20260506 00:46:01.720 | INFO | } | 20260506 00:46:01.720 | INFO | Perf report for the last 5.1425 seconds: | 20260506 00:46:01.720 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:46:01.720 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:01.720 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:01.720 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:01.720 | INFO | ANALYSIS : 61.9%, 3.0, 96.8ms, 0 | 20260506 00:46:01.720 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:01.720 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:01.720 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:46:01.720 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:01.720 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:01.720 | INFO | Uploaded 36GiB to GPU in 2.129s (16.9093GiB/s) | 20260506 00:46:01.721 | INFO | Downloaded 34GiB to GPU in 2.02064s (16.8264GiB/s) | 20260506 00:46:01.721 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 00:46:01.721 | INFO | Total mem usage: 15.3 GiB | 20260506 00:46:01.721 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 00:46:01.721 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.16%) | 20260506 00:46:01.721 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 00:46:01.721 | INFO | High Water Breakdown: | 20260506 00:46:01.721 | INFO | 60.9% : 8.55 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 00:46:01.721 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 00:46:01.721 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 00:46:01.721 | INFO | ------------------------------------------ | 20260506 00:46:01.721 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:46:01.721 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 00:46:01.721 | INFO | Total mem usage: 26.5 GiB | 20260506 00:46:01.721 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 00:46:01.721 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 00:46:01.721 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260506 00:46:01.721 | INFO | High Water Breakdown: | 20260506 00:46:01.721 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:46:01.721 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:46:01.721 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:46:01.721 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:46:01.721 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:46:01.721 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:46:01.721 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 00:46:01.721 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 00:46:01.721 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:46:01.721 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 00:46:01.721 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 00:46:01.721 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 00:46:01.721 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 00:46:01.721 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 00:46:01.721 | INFO | ------------------------------------------ | 20260506 00:46:01.721 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:46:01.721 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 00:46:01.721 | INFO | Total mem usage: 25.6 GiB | 20260506 00:46:01.721 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 00:46:01.721 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 00:46:01.721 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 00:46:01.721 | INFO | High Water Breakdown: | 20260506 00:46:01.721 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 00:46:01.721 | INFO | ------------------------------------------ | 20260506 00:46:01.721 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:46:01.721 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 00:46:01.721 | INFO | Total mem usage: 15.7 GiB | 20260506 00:46:01.721 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 00:46:01.721 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 00:46:01.721 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 00:46:01.721 | INFO | High Water Breakdown: | 20260506 00:46:01.721 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 00:46:01.721 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 00:46:01.721 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 00:46:01.721 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 00:46:01.721 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 00:46:01.721 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 00:46:01.721 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 00:46:01.721 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 00:46:01.721 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 00:46:01.721 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:46:01.721 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 00:46:01.721 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:46:01.721 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 00:46:01.721 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 00:46:01.721 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 00:46:01.721 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 00:46:01.721 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:46:01.721 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 00:46:01.721 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:46:01.721 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:46:01.721 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:46:01.721 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 00:46:01.721 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 00:46:01.721 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:46:01.721 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 00:46:01.721 | INFO | ------------------------------------------ | 20260506 00:46:01.721 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:46:01.721 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 00:46:01.721 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:46:01.721 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:46:01.721 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.39e+01, 1.35e+01 / 6.46e-01 | 20260506 00:46:01.722 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.05e-01, 1.04e-02 / 6.02e-03 | 20260506 00:46:01.722 | INFO | PerfReport: 11.84%, Baseline -- 3.70e+00 / 4.54e+00, 5.71e-01 / 1.75e-01 | 20260506 00:46:01.722 | INFO | PerfReport: 3.28%, Histogram -- 1.02e+00 / 1.53e+00, 1.90e-01 / 8.27e-02 | 20260506 00:46:01.722 | INFO | PerfReport: 16.01%, DME -- 4.01e+01 / 4.40e+01, 3.82e+00 / 6.37e-01 | 20260506 00:46:01.722 | INFO | PerfReport: 57.32%, FrameLabeling -- 1.79e+01 / 1.93e+01, 1.44e+00 / 1.02e+00 | 20260506 00:46:01.722 | INFO | PerfReport: 7.11%, PulseAccumulating -- 2.22e+00 / 2.77e+00, 4.23e-01 / 2.34e-01 | 20260506 00:46:01.722 | INFO | PerfReport: 3.08%, Metrics -- 9.62e-01 / 1.23e+00, 2.59e-01 / 1.07e-01 | 20260506 00:46:01.722 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.32e-01 / 3.90e-01, 5.67e-02 / 3.12e-02 | 20260506 00:46:01.722 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 00:46:01.722 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:46:01.722 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:46:01.722 | INFO | PerfReport: Full Execution -- 9.82e+01 / -nan, 2.44e+01 / 0.00e+00 | 20260506 00:46:01.722 | INFO | PerfReport: 22.30%, Upload -- 2.19e+01 / 0.00e+00, 7.48e+00 / 0.00e+00 | 20260506 00:46:01.722 | INFO | PerfReport: 21.85%, Download -- 2.14e+01 / 0.00e+00, 7.60e+00 / 0.00e+00 | 20260506 00:46:01.722 | INFO | PerfReport: 31.86%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 00:46:01.722 | INFO | PerfReport: 2.11%, LockUpload -- 2.07e+00 / 0.00e+00, 7.87e+00 / 0.00e+00 | 20260506 00:46:01.722 | INFO | PerfReport: 1.32%, LockDownload -- 1.30e+00 / 0.00e+00, 2.92e+00 / 0.00e+00 | 20260506 00:46:01.722 | INFO | PerfReport: 20.55%, LockCompute -- 2.02e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 00:46:01.722 | INFO | PerfReport: 0.01%, Misc -- 1.27e-02 / 0.00e+00, 1.31e-02 / 0.00e+00 | 20260506 00:46:01.838 | INFO | Starting a round of Baz IO | 20260506 00:46:04.982 | INFO | Populating RTMetrics report | 20260506 00:46:04.982 | INFO | RTMetrics report populated | 20260506 00:46:04.982 | INFO | Opening tmp RTMetrics json file | 20260506 00:46:04.983 | INFO | Serializing RTMetrics to json | 20260506 00:46:04.984 | INFO | Serialization done, swapping tmp file now... | 20260506 00:46:04.985 | INFO | RTMetrics done | 20260506 00:46:05.060 | INFO | Finished a round of Baz IO: Wrote 1.849 GiB in 3.221 seconds | 20260506 00:46:05.060 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.35% / 0.00% / 78.65% | 20260506 00:46:05.060 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0889381% / 3.97364e-06% / 7.54992e-05% | 20260506 00:46:05.060 | INFO | Baz IO overall: | 20260506 00:46:05.060 | INFO | BytesWritten : 4318740428794 | 20260506 00:46:05.060 | INFO | NumEvents : 2031580651701 | 20260506 00:46:06.857 | INFO | Analyzing chunk frames = [7618560,7619072) | 20260506 00:46:06.857 | INFO | MemInfo: { | 20260506 00:46:06.857 | INFO | "gpu" : " 17 GB", | 20260506 00:46:06.857 | INFO | "huge" : "", | 20260506 00:46:06.857 | INFO | "huge_shared" : " 27 GB", | 20260506 00:46:06.857 | INFO | "malloc" : " 14 GB", | 20260506 00:46:06.857 | INFO | "pinned" : " 29 GB" | 20260506 00:46:06.857 | INFO | } | 20260506 00:46:06.857 | INFO | Perf report for the last 5.13649 seconds: | 20260506 00:46:06.857 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:46:06.857 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:06.857 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:06.857 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:06.857 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260506 00:46:06.857 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:06.857 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260506 00:46:06.857 | INFO | PRE_HQ : 59.8%, 1.7, 54.0ms, 0 | 20260506 00:46:06.857 | INFO | BAZWRITER : 61.8%, 3.5, 110.9ms, 0 | 20260506 00:46:06.857 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:46:06.857 | INFO | Uploaded 36GiB to GPU in 2.18227s (16.4966GiB/s) | 20260506 00:46:06.857 | INFO | Downloaded 38GiB to GPU in 2.24208s (16.9486GiB/s) | 20260506 00:46:11.993 | INFO | Analyzing chunk frames = [7619072,7619584) | 20260506 00:46:11.993 | INFO | MemInfo: { | 20260506 00:46:11.993 | INFO | "gpu" : " 17 GB", | 20260506 00:46:11.993 | INFO | "huge" : "", | 20260506 00:46:11.993 | INFO | "huge_shared" : " 27 GB", | 20260506 00:46:11.993 | INFO | "malloc" : " 14 GB", | 20260506 00:46:11.993 | INFO | "pinned" : " 29 GB" | 20260506 00:46:11.993 | INFO | } | 20260506 00:46:11.993 | INFO | Perf report for the last 5.13666 seconds: | 20260506 00:46:11.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:46:11.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:11.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:11.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:11.993 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 00:46:11.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:11.993 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:11.993 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:46:11.993 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:11.993 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:11.994 | INFO | Uploaded 36GiB to GPU in 2.10952s (17.0655GiB/s) | 20260506 00:46:11.994 | INFO | Downloaded 34GiB to GPU in 2.04186s (16.6515GiB/s) | 20260506 00:46:17.131 | INFO | Analyzing chunk frames = [7619584,7620096) | 20260506 00:46:17.131 | INFO | MemInfo: { | 20260506 00:46:17.131 | INFO | "gpu" : " 17 GB", | 20260506 00:46:17.131 | INFO | "huge" : "", | 20260506 00:46:17.131 | INFO | "huge_shared" : " 27 GB", | 20260506 00:46:17.131 | INFO | "malloc" : " 14 GB", | 20260506 00:46:17.131 | INFO | "pinned" : " 29 GB" | 20260506 00:46:17.131 | INFO | } | 20260506 00:46:17.131 | INFO | Perf report for the last 5.13785 seconds: | 20260506 00:46:17.131 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:46:17.131 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:17.131 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:17.131 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:17.131 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 00:46:17.131 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:17.131 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:17.131 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:46:17.131 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:17.131 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:17.131 | INFO | Uploaded 36GiB to GPU in 2.0971s (17.1666GiB/s) | 20260506 00:46:17.132 | INFO | Downloaded 34GiB to GPU in 2.03879s (16.6765GiB/s) | 20260506 00:46:22.275 | INFO | Analyzing chunk frames = [7620096,7620608) | 20260506 00:46:22.275 | INFO | MemInfo: { | 20260506 00:46:22.275 | INFO | "gpu" : " 17 GB", | 20260506 00:46:22.275 | INFO | "huge" : "", | 20260506 00:46:22.275 | INFO | "huge_shared" : " 27 GB", | 20260506 00:46:22.275 | INFO | "malloc" : " 14 GB", | 20260506 00:46:22.275 | INFO | "pinned" : " 29 GB" | 20260506 00:46:22.275 | INFO | } | 20260506 00:46:22.275 | INFO | Perf report for the last 5.14407 seconds: | 20260506 00:46:22.275 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:46:22.275 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:22.275 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:22.275 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:22.275 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 00:46:22.275 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:22.275 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:22.275 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 00:46:22.275 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:22.275 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:22.276 | INFO | Uploaded 36GiB to GPU in 2.13248s (16.8818GiB/s) | 20260506 00:46:22.276 | INFO | Downloaded 34GiB to GPU in 2.02193s (16.8156GiB/s) | 20260506 00:46:27.413 | INFO | Analyzing chunk frames = [7620608,7621120) | 20260506 00:46:27.413 | INFO | MemInfo: { | 20260506 00:46:27.413 | INFO | "gpu" : " 17 GB", | 20260506 00:46:27.413 | INFO | "huge" : "", | 20260506 00:46:27.413 | INFO | "huge_shared" : " 27 GB", | 20260506 00:46:27.413 | INFO | "malloc" : " 14 GB", | 20260506 00:46:27.413 | INFO | "pinned" : " 29 GB" | 20260506 00:46:27.413 | INFO | } | 20260506 00:46:27.413 | INFO | Perf report for the last 5.13795 seconds: | 20260506 00:46:27.413 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:46:27.413 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:27.413 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:27.413 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:27.413 | INFO | ANALYSIS : 60.9%, 3.0, 95.1ms, 0 | 20260506 00:46:27.414 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:27.414 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:27.414 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 00:46:27.414 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:27.414 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:27.414 | INFO | Uploaded 36GiB to GPU in 2.08976s (17.2269GiB/s) | 20260506 00:46:27.414 | INFO | Downloaded 34GiB to GPU in 2.03968s (16.6693GiB/s) | 20260506 00:46:32.552 | INFO | Analyzing chunk frames = [7621120,7621632) | 20260506 00:46:32.552 | INFO | MemInfo: { | 20260506 00:46:32.552 | INFO | "gpu" : " 17 GB", | 20260506 00:46:32.553 | INFO | "huge" : "", | 20260506 00:46:32.553 | INFO | "huge_shared" : " 27 GB", | 20260506 00:46:32.553 | INFO | "malloc" : " 14 GB", | 20260506 00:46:32.553 | INFO | "pinned" : " 29 GB" | 20260506 00:46:32.553 | INFO | } | 20260506 00:46:32.553 | INFO | Perf report for the last 5.13948 seconds: | 20260506 00:46:32.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:46:32.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:32.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:32.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:32.553 | INFO | ANALYSIS : 61.2%, 3.0, 95.6ms, 0 | 20260506 00:46:32.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:32.553 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:32.553 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 00:46:32.553 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:32.553 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:32.553 | INFO | Uploaded 36GiB to GPU in 2.06341s (17.4469GiB/s) | 20260506 00:46:32.553 | INFO | Downloaded 34GiB to GPU in 2.0354s (16.7043GiB/s) | 20260506 00:46:37.686 | INFO | Analyzing chunk frames = [7621632,7622144) | 20260506 00:46:37.686 | INFO | MemInfo: { | 20260506 00:46:37.686 | INFO | "gpu" : " 17 GB", | 20260506 00:46:37.686 | INFO | "huge" : "", | 20260506 00:46:37.686 | INFO | "huge_shared" : " 27 GB", | 20260506 00:46:37.686 | INFO | "malloc" : " 14 GB", | 20260506 00:46:37.686 | INFO | "pinned" : " 29 GB" | 20260506 00:46:37.686 | INFO | } | 20260506 00:46:37.686 | INFO | Perf report for the last 5.13368 seconds: | 20260506 00:46:37.686 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:46:37.686 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:37.686 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:37.686 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:37.686 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260506 00:46:37.686 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:37.686 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:37.686 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 00:46:37.687 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:37.687 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:37.687 | INFO | Uploaded 36GiB to GPU in 2.1162s (17.0116GiB/s) | 20260506 00:46:37.687 | INFO | Downloaded 34GiB to GPU in 2.1105s (16.1099GiB/s) | 20260506 00:46:42.829 | INFO | Analyzing chunk frames = [7622144,7622656) | 20260506 00:46:42.829 | INFO | MemInfo: { | 20260506 00:46:42.829 | INFO | "gpu" : " 17 GB", | 20260506 00:46:42.829 | INFO | "huge" : "", | 20260506 00:46:42.829 | INFO | "huge_shared" : " 27 GB", | 20260506 00:46:42.829 | INFO | "malloc" : " 14 GB", | 20260506 00:46:42.829 | INFO | "pinned" : " 29 GB" | 20260506 00:46:42.829 | INFO | } | 20260506 00:46:42.829 | INFO | Perf report for the last 5.1424 seconds: | 20260506 00:46:42.829 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:46:42.829 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:42.829 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:42.829 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:42.829 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260506 00:46:42.829 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:42.829 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:42.829 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:46:42.829 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:42.829 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:42.829 | INFO | Uploaded 36GiB to GPU in 2.11774s (16.9992GiB/s) | 20260506 00:46:42.829 | INFO | Downloaded 34GiB to GPU in 2.00489s (16.9585GiB/s) | 20260506 00:46:42.946 | INFO | Starting a round of Baz IO | 20260506 00:46:46.095 | INFO | Populating RTMetrics report | 20260506 00:46:46.095 | INFO | RTMetrics report populated | 20260506 00:46:46.095 | INFO | Opening tmp RTMetrics json file | 20260506 00:46:46.097 | INFO | Serializing RTMetrics to json | 20260506 00:46:46.098 | INFO | Serialization done, swapping tmp file now... | 20260506 00:46:46.098 | INFO | RTMetrics done | 20260506 00:46:46.171 | INFO | Finished a round of Baz IO: Wrote 1.849 GiB in 3.225 seconds | 20260506 00:46:46.171 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.35% / 0.00% / 78.65% | 20260506 00:46:46.171 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0883619% / 3.97364e-06% / 9.13938e-05% | 20260506 00:46:46.171 | INFO | Baz IO overall: | 20260506 00:46:46.171 | INFO | BytesWritten : 4320725589059 | 20260506 00:46:46.171 | INFO | NumEvents : 2032487874763 | 20260506 00:46:47.973 | INFO | Analyzing chunk frames = [7622656,7623168) | 20260506 00:46:47.973 | INFO | MemInfo: { | 20260506 00:46:47.973 | INFO | "gpu" : " 17 GB", | 20260506 00:46:47.973 | INFO | "huge" : "", | 20260506 00:46:47.973 | INFO | "huge_shared" : " 27 GB", | 20260506 00:46:47.973 | INFO | "malloc" : " 14 GB", | 20260506 00:46:47.973 | INFO | "pinned" : " 29 GB" | 20260506 00:46:47.973 | INFO | } | 20260506 00:46:47.973 | INFO | Perf report for the last 5.14412 seconds: | 20260506 00:46:47.973 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:46:47.973 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:47.973 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:47.973 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:47.973 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260506 00:46:47.973 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:47.973 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260506 00:46:47.973 | INFO | PRE_HQ : 60.0%, 1.7, 54.2ms, 0 | 20260506 00:46:47.973 | INFO | BAZWRITER : 61.8%, 3.5, 110.6ms, 0 | 20260506 00:46:47.973 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:46:47.973 | INFO | Uploaded 36GiB to GPU in 2.17169s (16.5769GiB/s) | 20260506 00:46:47.973 | INFO | Downloaded 38GiB to GPU in 2.2414s (16.9537GiB/s) | 20260506 00:46:53.114 | INFO | Analyzing chunk frames = [7623168,7623680) | 20260506 00:46:53.114 | INFO | MemInfo: { | 20260506 00:46:53.114 | INFO | "gpu" : " 17 GB", | 20260506 00:46:53.114 | INFO | "huge" : "", | 20260506 00:46:53.114 | INFO | "huge_shared" : " 27 GB", | 20260506 00:46:53.114 | INFO | "malloc" : " 14 GB", | 20260506 00:46:53.114 | INFO | "pinned" : " 29 GB" | 20260506 00:46:53.114 | INFO | } | 20260506 00:46:53.114 | INFO | Perf report for the last 5.14147 seconds: | 20260506 00:46:53.114 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:46:53.114 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:53.114 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:53.115 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:53.115 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 00:46:53.115 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:53.115 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:53.115 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 00:46:53.115 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:53.115 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:53.115 | INFO | Uploaded 36GiB to GPU in 1.97203s (18.2553GiB/s) | 20260506 00:46:53.115 | INFO | Downloaded 34GiB to GPU in 1.99205s (17.0678GiB/s) | 20260506 00:46:58.252 | INFO | Analyzing chunk frames = [7623680,7624192) | 20260506 00:46:58.252 | INFO | MemInfo: { | 20260506 00:46:58.252 | INFO | "gpu" : " 17 GB", | 20260506 00:46:58.252 | INFO | "huge" : "", | 20260506 00:46:58.252 | INFO | "huge_shared" : " 27 GB", | 20260506 00:46:58.252 | INFO | "malloc" : " 14 GB", | 20260506 00:46:58.252 | INFO | "pinned" : " 29 GB" | 20260506 00:46:58.252 | INFO | } | 20260506 00:46:58.252 | INFO | Perf report for the last 5.13773 seconds: | 20260506 00:46:58.252 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:46:58.252 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:58.252 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:58.252 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:58.252 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 00:46:58.252 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:58.252 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:58.252 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:46:58.252 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:58.252 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:46:58.253 | INFO | Uploaded 36GiB to GPU in 2.07404s (17.3574GiB/s) | 20260506 00:46:58.253 | INFO | Downloaded 34GiB to GPU in 2.01658s (16.8602GiB/s) | 20260506 00:47:03.386 | INFO | Analyzing chunk frames = [7624192,7624704) | 20260506 00:47:03.386 | INFO | MemInfo: { | 20260506 00:47:03.386 | INFO | "gpu" : " 17 GB", | 20260506 00:47:03.386 | INFO | "huge" : "", | 20260506 00:47:03.386 | INFO | "huge_shared" : " 27 GB", | 20260506 00:47:03.386 | INFO | "malloc" : " 14 GB", | 20260506 00:47:03.386 | INFO | "pinned" : " 29 GB" | 20260506 00:47:03.386 | INFO | } | 20260506 00:47:03.386 | INFO | Perf report for the last 5.13387 seconds: | 20260506 00:47:03.386 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:47:03.386 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:03.386 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:03.386 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:03.386 | INFO | ANALYSIS : 61.6%, 3.0, 95.6ms, 0 | 20260506 00:47:03.386 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:03.386 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:03.386 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:47:03.386 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:03.386 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:03.386 | INFO | Uploaded 36GiB to GPU in 2.13494s (16.8623GiB/s) | 20260506 00:47:03.386 | INFO | Downloaded 34GiB to GPU in 2.02001s (16.8316GiB/s) | 20260506 00:47:08.529 | INFO | Analyzing chunk frames = [7624704,7625216) | 20260506 00:47:08.529 | INFO | MemInfo: { | 20260506 00:47:08.529 | INFO | "gpu" : " 17 GB", | 20260506 00:47:08.529 | INFO | "huge" : "", | 20260506 00:47:08.529 | INFO | "huge_shared" : " 27 GB", | 20260506 00:47:08.529 | INFO | "malloc" : " 14 GB", | 20260506 00:47:08.529 | INFO | "pinned" : " 29 GB" | 20260506 00:47:08.529 | INFO | } | 20260506 00:47:08.529 | INFO | Perf report for the last 5.14328 seconds: | 20260506 00:47:08.529 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:47:08.529 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:08.529 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:08.529 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:08.529 | INFO | ANALYSIS : 61.1%, 3.0, 95.5ms, 0 | 20260506 00:47:08.529 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:08.529 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:08.529 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 00:47:08.529 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:08.529 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:08.530 | INFO | Uploaded 36GiB to GPU in 2.1093s (17.0673GiB/s) | 20260506 00:47:08.530 | INFO | Downloaded 34GiB to GPU in 2.05161s (16.5723GiB/s) | 20260506 00:47:13.672 | INFO | Analyzing chunk frames = [7625216,7625728) | 20260506 00:47:13.672 | INFO | MemInfo: { | 20260506 00:47:13.672 | INFO | "gpu" : " 17 GB", | 20260506 00:47:13.672 | INFO | "huge" : "", | 20260506 00:47:13.672 | INFO | "huge_shared" : " 27 GB", | 20260506 00:47:13.672 | INFO | "malloc" : " 14 GB", | 20260506 00:47:13.672 | INFO | "pinned" : " 29 GB" | 20260506 00:47:13.672 | INFO | } | 20260506 00:47:13.672 | INFO | Perf report for the last 5.14257 seconds: | 20260506 00:47:13.672 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:47:13.672 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:13.672 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:13.672 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:13.672 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 00:47:13.672 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:13.672 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:13.672 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 00:47:13.672 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:13.672 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:13.672 | INFO | Uploaded 36GiB to GPU in 2.06569s (17.4276GiB/s) | 20260506 00:47:13.672 | INFO | Downloaded 34GiB to GPU in 2.04044s (16.6631GiB/s) | 20260506 00:47:18.815 | INFO | Analyzing chunk frames = [7625728,7626240) | 20260506 00:47:18.815 | INFO | MemInfo: { | 20260506 00:47:18.815 | INFO | "gpu" : " 17 GB", | 20260506 00:47:18.815 | INFO | "huge" : "", | 20260506 00:47:18.815 | INFO | "huge_shared" : " 27 GB", | 20260506 00:47:18.815 | INFO | "malloc" : " 14 GB", | 20260506 00:47:18.815 | INFO | "pinned" : " 29 GB" | 20260506 00:47:18.815 | INFO | } | 20260506 00:47:18.815 | INFO | Perf report for the last 5.14317 seconds: | 20260506 00:47:18.815 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:47:18.815 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:18.815 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:18.815 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:18.815 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260506 00:47:18.815 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:18.815 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:18.815 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 00:47:18.815 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:18.815 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:18.815 | INFO | Uploaded 36GiB to GPU in 2.03307s (17.7072GiB/s) | 20260506 00:47:18.815 | INFO | Downloaded 34GiB to GPU in 2.06795s (16.4414GiB/s) | 20260506 00:47:23.945 | INFO | Analyzing chunk frames = [7626240,7626752) | 20260506 00:47:23.945 | INFO | MemInfo: { | 20260506 00:47:23.945 | INFO | "gpu" : " 17 GB", | 20260506 00:47:23.945 | INFO | "huge" : "", | 20260506 00:47:23.945 | INFO | "huge_shared" : " 27 GB", | 20260506 00:47:23.945 | INFO | "malloc" : " 14 GB", | 20260506 00:47:23.945 | INFO | "pinned" : " 29 GB" | 20260506 00:47:23.945 | INFO | } | 20260506 00:47:23.945 | INFO | Perf report for the last 5.13031 seconds: | 20260506 00:47:23.945 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:47:23.945 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:23.945 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:23.945 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:23.946 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 00:47:23.946 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:23.946 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:23.946 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:47:23.946 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:23.946 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:23.946 | INFO | Uploaded 36GiB to GPU in 2.13093s (16.894GiB/s) | 20260506 00:47:23.946 | INFO | Downloaded 34GiB to GPU in 2.03548s (16.7037GiB/s) | 20260506 00:47:23.946 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 00:47:23.946 | INFO | Total mem usage: 15.3 GiB | 20260506 00:47:23.946 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 00:47:23.946 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.15%) | 20260506 00:47:23.946 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 00:47:23.946 | INFO | High Water Breakdown: | 20260506 00:47:23.946 | INFO | 60.9% : 8.55 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 00:47:23.946 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 00:47:23.946 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 00:47:23.946 | INFO | ------------------------------------------ | 20260506 00:47:23.946 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:47:23.946 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 00:47:23.946 | INFO | Total mem usage: 26.5 GiB | 20260506 00:47:23.946 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 00:47:23.946 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 00:47:23.946 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 00:47:23.946 | INFO | High Water Breakdown: | 20260506 00:47:23.946 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:47:23.946 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:47:23.946 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:47:23.946 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:47:23.946 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:47:23.946 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:47:23.946 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 00:47:23.946 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 00:47:23.947 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:47:23.947 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 00:47:23.947 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 00:47:23.947 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 00:47:23.947 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 00:47:23.947 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 00:47:23.947 | INFO | ------------------------------------------ | 20260506 00:47:23.947 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:47:23.947 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 00:47:23.947 | INFO | Total mem usage: 25.6 GiB | 20260506 00:47:23.947 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 00:47:23.947 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 00:47:23.947 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 00:47:23.947 | INFO | High Water Breakdown: | 20260506 00:47:23.947 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 00:47:23.947 | INFO | ------------------------------------------ | 20260506 00:47:23.947 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:47:23.947 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 00:47:23.947 | INFO | Total mem usage: 15.7 GiB | 20260506 00:47:23.947 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 00:47:23.947 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 00:47:23.947 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 00:47:23.947 | INFO | High Water Breakdown: | 20260506 00:47:23.947 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 00:47:23.947 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 00:47:23.947 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 00:47:23.947 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 00:47:23.947 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 00:47:23.947 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 00:47:23.947 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 00:47:23.947 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 00:47:23.947 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 00:47:23.947 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:47:23.947 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 00:47:23.947 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:47:23.947 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 00:47:23.947 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 00:47:23.947 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 00:47:23.947 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 00:47:23.947 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:47:23.947 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 00:47:23.947 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:47:23.947 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:47:23.947 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:47:23.947 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 00:47:23.947 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 00:47:23.947 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:47:23.947 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 00:47:23.947 | INFO | ------------------------------------------ | 20260506 00:47:23.947 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:47:23.947 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 00:47:23.947 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:47:23.947 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:47:23.947 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.39e+01, 1.35e+01 / 8.59e-01 | 20260506 00:47:23.947 | INFO | PerfReport: 0.29%, AggregateStats -- 9.24e-02 / 1.04e-01, 1.20e-02 / 4.20e-03 | 20260506 00:47:23.947 | INFO | PerfReport: 11.82%, Baseline -- 3.70e+00 / 4.54e+00, 5.64e-01 / 2.01e-01 | 20260506 00:47:23.947 | INFO | PerfReport: 3.29%, Histogram -- 1.03e+00 / 1.53e+00, 1.91e-01 / 9.56e-02 | 20260506 00:47:23.947 | INFO | PerfReport: 15.99%, DME -- 4.01e+01 / 4.39e+01, 3.82e+00 / 9.12e-01 | 20260506 00:47:23.947 | INFO | PerfReport: 57.42%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.53e+00 / 1.17e+00 | 20260506 00:47:23.947 | INFO | PerfReport: 7.06%, PulseAccumulating -- 2.21e+00 / 2.75e+00, 4.17e-01 / 2.21e-01 | 20260506 00:47:23.947 | INFO | PerfReport: 3.06%, Metrics -- 9.59e-01 / 1.24e+00, 2.58e-01 / 1.04e-01 | 20260506 00:47:23.947 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.33e-01 / 3.86e-01, 5.67e-02 / 2.98e-02 | 20260506 00:47:23.947 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 00:47:23.947 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:47:23.947 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:47:23.947 | INFO | PerfReport: Full Execution -- 9.82e+01 / -nan, 2.45e+01 / 0.00e+00 | 20260506 00:47:23.947 | INFO | PerfReport: 22.28%, Upload -- 2.19e+01 / 0.00e+00, 7.45e+00 / 0.00e+00 | 20260506 00:47:23.947 | INFO | PerfReport: 21.88%, Download -- 2.15e+01 / 0.00e+00, 7.58e+00 / 0.00e+00 | 20260506 00:47:23.947 | INFO | PerfReport: 31.91%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 00:47:23.947 | INFO | PerfReport: 2.10%, LockUpload -- 2.06e+00 / 0.00e+00, 7.87e+00 / 0.00e+00 | 20260506 00:47:23.947 | INFO | PerfReport: 1.31%, LockDownload -- 1.29e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260506 00:47:23.947 | INFO | PerfReport: 20.50%, LockCompute -- 2.01e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 00:47:23.947 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.65e-02 / 0.00e+00 | 20260506 00:47:24.069 | INFO | Starting a round of Baz IO | 20260506 00:47:27.215 | INFO | Populating RTMetrics report | 20260506 00:47:27.215 | INFO | RTMetrics report populated | 20260506 00:47:27.215 | INFO | Opening tmp RTMetrics json file | 20260506 00:47:27.216 | INFO | Serializing RTMetrics to json | 20260506 00:47:27.219 | INFO | Serialization done, swapping tmp file now... | 20260506 00:47:27.220 | INFO | RTMetrics done | 20260506 00:47:27.292 | INFO | Finished a round of Baz IO: Wrote 1.846 GiB in 3.223 seconds | 20260506 00:47:27.292 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.35% / 0.00% / 78.65% | 20260506 00:47:27.292 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0876705% / 3.97364e-06% / 7.94729e-05% | 20260506 00:47:27.292 | INFO | Baz IO overall: | 20260506 00:47:27.292 | INFO | BytesWritten : 4322707671320 | 20260506 00:47:27.292 | INFO | NumEvents : 2033393025631 | 20260506 00:47:29.083 | INFO | Analyzing chunk frames = [7626752,7627264) | 20260506 00:47:29.083 | INFO | MemInfo: { | 20260506 00:47:29.083 | INFO | "gpu" : " 17 GB", | 20260506 00:47:29.083 | INFO | "huge" : "", | 20260506 00:47:29.083 | INFO | "huge_shared" : " 27 GB", | 20260506 00:47:29.083 | INFO | "malloc" : " 14 GB", | 20260506 00:47:29.084 | INFO | "pinned" : " 29 GB" | 20260506 00:47:29.084 | INFO | } | 20260506 00:47:29.084 | INFO | Perf report for the last 5.1381 seconds: | 20260506 00:47:29.084 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:47:29.084 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:29.084 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:29.084 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:29.084 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260506 00:47:29.084 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:29.084 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260506 00:47:29.084 | INFO | PRE_HQ : 59.8%, 1.7, 53.6ms, 0 | 20260506 00:47:29.084 | INFO | BAZWRITER : 61.9%, 3.5, 110.9ms, 0 | 20260506 00:47:29.084 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:47:29.084 | INFO | Uploaded 36GiB to GPU in 2.16264s (16.6463GiB/s) | 20260506 00:47:29.084 | INFO | Downloaded 38GiB to GPU in 2.23302s (17.0173GiB/s) | 20260506 00:47:34.230 | INFO | Analyzing chunk frames = [7627264,7627776) | 20260506 00:47:34.230 | INFO | MemInfo: { | 20260506 00:47:34.230 | INFO | "gpu" : " 17 GB", | 20260506 00:47:34.230 | INFO | "huge" : "", | 20260506 00:47:34.230 | INFO | "huge_shared" : " 27 GB", | 20260506 00:47:34.230 | INFO | "malloc" : " 14 GB", | 20260506 00:47:34.230 | INFO | "pinned" : " 29 GB" | 20260506 00:47:34.230 | INFO | } | 20260506 00:47:34.230 | INFO | Perf report for the last 5.14623 seconds: | 20260506 00:47:34.230 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:47:34.230 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:34.230 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:34.230 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:34.230 | INFO | ANALYSIS : 61.6%, 3.0, 96.4ms, 0 | 20260506 00:47:34.230 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:34.230 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:34.230 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 00:47:34.230 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:34.230 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:34.230 | INFO | Uploaded 36GiB to GPU in 2.09292s (17.2009GiB/s) | 20260506 00:47:34.230 | INFO | Downloaded 34GiB to GPU in 2.03782s (16.6845GiB/s) | 20260506 00:47:39.365 | INFO | Analyzing chunk frames = [7627776,7628288) | 20260506 00:47:39.365 | INFO | MemInfo: { | 20260506 00:47:39.365 | INFO | "gpu" : " 17 GB", | 20260506 00:47:39.365 | INFO | "huge" : "", | 20260506 00:47:39.365 | INFO | "huge_shared" : " 27 GB", | 20260506 00:47:39.366 | INFO | "malloc" : " 14 GB", | 20260506 00:47:39.366 | INFO | "pinned" : " 29 GB" | 20260506 00:47:39.366 | INFO | } | 20260506 00:47:39.366 | INFO | Perf report for the last 5.13584 seconds: | 20260506 00:47:39.366 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:47:39.366 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:39.366 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:39.366 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:39.366 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 00:47:39.366 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:39.366 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:39.366 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:47:39.366 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:39.366 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:39.366 | INFO | Uploaded 36GiB to GPU in 2.09056s (17.2203GiB/s) | 20260506 00:47:39.366 | INFO | Downloaded 34GiB to GPU in 2.0263s (16.7794GiB/s) | 20260506 00:47:44.499 | INFO | Analyzing chunk frames = [7628288,7628800) | 20260506 00:47:44.499 | INFO | MemInfo: { | 20260506 00:47:44.499 | INFO | "gpu" : " 17 GB", | 20260506 00:47:44.499 | INFO | "huge" : "", | 20260506 00:47:44.499 | INFO | "huge_shared" : " 27 GB", | 20260506 00:47:44.499 | INFO | "malloc" : " 14 GB", | 20260506 00:47:44.499 | INFO | "pinned" : " 29 GB" | 20260506 00:47:44.499 | INFO | } | 20260506 00:47:44.499 | INFO | Perf report for the last 5.13377 seconds: | 20260506 00:47:44.499 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:47:44.499 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:44.499 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:44.499 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:44.500 | INFO | ANALYSIS : 61.6%, 3.0, 95.6ms, 0 | 20260506 00:47:44.500 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:44.500 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:44.500 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:47:44.500 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:44.500 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:44.500 | INFO | Uploaded 36GiB to GPU in 2.13877s (16.8321GiB/s) | 20260506 00:47:44.500 | INFO | Downloaded 34GiB to GPU in 2.04904s (16.5931GiB/s) | 20260506 00:47:49.640 | INFO | Analyzing chunk frames = [7628800,7629312) | 20260506 00:47:49.640 | INFO | MemInfo: { | 20260506 00:47:49.640 | INFO | "gpu" : " 17 GB", | 20260506 00:47:49.640 | INFO | "huge" : "", | 20260506 00:47:49.640 | INFO | "huge_shared" : " 27 GB", | 20260506 00:47:49.640 | INFO | "malloc" : " 14 GB", | 20260506 00:47:49.640 | INFO | "pinned" : " 29 GB" | 20260506 00:47:49.640 | INFO | } | 20260506 00:47:49.640 | INFO | Perf report for the last 5.14056 seconds: | 20260506 00:47:49.640 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:47:49.640 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:49.640 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:49.640 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:49.640 | INFO | ANALYSIS : 61.1%, 3.0, 95.5ms, 0 | 20260506 00:47:49.640 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:49.640 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:49.640 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 00:47:49.640 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:49.640 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:49.640 | INFO | Uploaded 36GiB to GPU in 2.08874s (17.2353GiB/s) | 20260506 00:47:49.640 | INFO | Downloaded 34GiB to GPU in 2.04204s (16.6501GiB/s) | 20260506 00:47:54.783 | INFO | Analyzing chunk frames = [7629312,7629824) | 20260506 00:47:54.783 | INFO | MemInfo: { | 20260506 00:47:54.783 | INFO | "gpu" : " 17 GB", | 20260506 00:47:54.783 | INFO | "huge" : "", | 20260506 00:47:54.783 | INFO | "huge_shared" : " 27 GB", | 20260506 00:47:54.783 | INFO | "malloc" : " 14 GB", | 20260506 00:47:54.783 | INFO | "pinned" : " 29 GB" | 20260506 00:47:54.783 | INFO | } | 20260506 00:47:54.783 | INFO | Perf report for the last 5.14334 seconds: | 20260506 00:47:54.783 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:47:54.783 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:54.783 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:54.783 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:54.783 | INFO | ANALYSIS : 61.2%, 3.0, 95.6ms, 0 | 20260506 00:47:54.783 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:54.784 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:54.784 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:47:54.784 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:54.784 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:54.784 | INFO | Uploaded 36GiB to GPU in 2.0725s (17.3703GiB/s) | 20260506 00:47:54.784 | INFO | Downloaded 34GiB to GPU in 2.04464s (16.6289GiB/s) | 20260506 00:47:59.930 | INFO | Analyzing chunk frames = [7629824,7630336) | 20260506 00:47:59.930 | INFO | MemInfo: { | 20260506 00:47:59.930 | INFO | "gpu" : " 17 GB", | 20260506 00:47:59.930 | INFO | "huge" : "", | 20260506 00:47:59.930 | INFO | "huge_shared" : " 27 GB", | 20260506 00:47:59.930 | INFO | "malloc" : " 14 GB", | 20260506 00:47:59.930 | INFO | "pinned" : " 29 GB" | 20260506 00:47:59.930 | INFO | } | 20260506 00:47:59.931 | INFO | Perf report for the last 5.14717 seconds: | 20260506 00:47:59.931 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:47:59.931 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:59.931 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:59.931 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:59.931 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260506 00:47:59.931 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:59.931 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:59.931 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 00:47:59.931 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:59.931 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:47:59.931 | INFO | Uploaded 36GiB to GPU in 2.11008s (17.061GiB/s) | 20260506 00:47:59.931 | INFO | Downloaded 34GiB to GPU in 2.11689s (16.0613GiB/s) | 20260506 00:48:05.064 | INFO | Analyzing chunk frames = [7630336,7630848) | 20260506 00:48:05.064 | INFO | MemInfo: { | 20260506 00:48:05.064 | INFO | "gpu" : " 17 GB", | 20260506 00:48:05.064 | INFO | "huge" : "", | 20260506 00:48:05.064 | INFO | "huge_shared" : " 27 GB", | 20260506 00:48:05.064 | INFO | "malloc" : " 14 GB", | 20260506 00:48:05.064 | INFO | "pinned" : " 29 GB" | 20260506 00:48:05.064 | INFO | } | 20260506 00:48:05.064 | INFO | Perf report for the last 5.13321 seconds: | 20260506 00:48:05.064 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:48:05.064 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:05.064 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:05.064 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:05.064 | INFO | ANALYSIS : 61.5%, 3.0, 96.0ms, 0 | 20260506 00:48:05.064 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:05.064 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:05.064 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 00:48:05.064 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:05.064 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:05.064 | INFO | Uploaded 36GiB to GPU in 2.05174s (17.5461GiB/s) | 20260506 00:48:05.064 | INFO | Downloaded 34GiB to GPU in 2.02485s (16.7914GiB/s) | 20260506 00:48:05.181 | INFO | Starting a round of Baz IO | 20260506 00:48:08.327 | INFO | Populating RTMetrics report | 20260506 00:48:08.328 | INFO | RTMetrics report populated | 20260506 00:48:08.328 | INFO | Opening tmp RTMetrics json file | 20260506 00:48:08.328 | INFO | Serializing RTMetrics to json | 20260506 00:48:08.329 | INFO | Serialization done, swapping tmp file now... | 20260506 00:48:08.330 | INFO | RTMetrics done | 20260506 00:48:08.403 | INFO | Finished a round of Baz IO: Wrote 1.847 GiB in 3.221 seconds | 20260506 00:48:08.403 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.35% / 0.00% / 78.65% | 20260506 00:48:08.403 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0878215% / 0% / 4.76837e-05% | 20260506 00:48:08.403 | INFO | Baz IO overall: | 20260506 00:48:08.403 | INFO | BytesWritten : 4324691320338 | 20260506 00:48:08.403 | INFO | NumEvents : 2034299316556 | 20260506 00:48:10.200 | INFO | Analyzing chunk frames = [7630848,7631360) | 20260506 00:48:10.200 | INFO | MemInfo: { | 20260506 00:48:10.200 | INFO | "gpu" : " 17 GB", | 20260506 00:48:10.200 | INFO | "huge" : "", | 20260506 00:48:10.200 | INFO | "huge_shared" : " 27 GB", | 20260506 00:48:10.200 | INFO | "malloc" : " 14 GB", | 20260506 00:48:10.200 | INFO | "pinned" : " 29 GB" | 20260506 00:48:10.200 | INFO | } | 20260506 00:48:10.200 | INFO | Perf report for the last 5.13633 seconds: | 20260506 00:48:10.200 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:48:10.200 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:10.200 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:10.200 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:10.200 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260506 00:48:10.200 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:10.200 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260506 00:48:10.200 | INFO | PRE_HQ : 59.7%, 1.7, 53.7ms, 0 | 20260506 00:48:10.200 | INFO | BAZWRITER : 61.8%, 3.5, 111.0ms, 0 | 20260506 00:48:10.200 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:48:10.200 | INFO | Uploaded 36GiB to GPU in 2.18149s (16.5025GiB/s) | 20260506 00:48:10.200 | INFO | Downloaded 38GiB to GPU in 2.24639s (16.9161GiB/s) | 20260506 00:48:15.344 | INFO | Analyzing chunk frames = [7631360,7631872) | 20260506 00:48:15.344 | INFO | MemInfo: { | 20260506 00:48:15.344 | INFO | "gpu" : " 17 GB", | 20260506 00:48:15.344 | INFO | "huge" : "", | 20260506 00:48:15.344 | INFO | "huge_shared" : " 27 GB", | 20260506 00:48:15.344 | INFO | "malloc" : " 14 GB", | 20260506 00:48:15.344 | INFO | "pinned" : " 29 GB" | 20260506 00:48:15.344 | INFO | } | 20260506 00:48:15.344 | INFO | Perf report for the last 5.14365 seconds: | 20260506 00:48:15.344 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:48:15.344 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:15.344 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:15.344 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:15.344 | INFO | ANALYSIS : 61.4%, 3.0, 95.9ms, 0 | 20260506 00:48:15.344 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:15.344 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:15.344 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 00:48:15.344 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:15.344 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:15.344 | INFO | Uploaded 36GiB to GPU in 2.02076s (17.8151GiB/s) | 20260506 00:48:15.344 | INFO | Downloaded 34GiB to GPU in 2.00904s (16.9235GiB/s) | 20260506 00:48:20.478 | INFO | Analyzing chunk frames = [7631872,7632384) | 20260506 00:48:20.478 | INFO | MemInfo: { | 20260506 00:48:20.478 | INFO | "gpu" : " 17 GB", | 20260506 00:48:20.478 | INFO | "huge" : "", | 20260506 00:48:20.478 | INFO | "huge_shared" : " 27 GB", | 20260506 00:48:20.478 | INFO | "malloc" : " 14 GB", | 20260506 00:48:20.478 | INFO | "pinned" : " 29 GB" | 20260506 00:48:20.478 | INFO | } | 20260506 00:48:20.479 | INFO | Perf report for the last 5.13476 seconds: | 20260506 00:48:20.479 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:48:20.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:20.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:20.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:20.479 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 00:48:20.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:20.479 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:20.479 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 00:48:20.479 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:20.479 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:20.479 | INFO | Uploaded 36GiB to GPU in 2.07958s (17.3112GiB/s) | 20260506 00:48:20.479 | INFO | Downloaded 34GiB to GPU in 2.02188s (16.816GiB/s) | 20260506 00:48:25.621 | INFO | Analyzing chunk frames = [7632384,7632896) | 20260506 00:48:25.621 | INFO | MemInfo: { | 20260506 00:48:25.621 | INFO | "gpu" : " 17 GB", | 20260506 00:48:25.621 | INFO | "huge" : "", | 20260506 00:48:25.621 | INFO | "huge_shared" : " 27 GB", | 20260506 00:48:25.621 | INFO | "malloc" : " 14 GB", | 20260506 00:48:25.622 | INFO | "pinned" : " 29 GB" | 20260506 00:48:25.622 | INFO | } | 20260506 00:48:25.622 | INFO | Perf report for the last 5.14298 seconds: | 20260506 00:48:25.622 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:48:25.622 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:25.622 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:25.622 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:25.622 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260506 00:48:25.622 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:25.622 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:25.622 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:48:25.622 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:25.622 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:25.622 | INFO | Uploaded 36GiB to GPU in 2.13205s (16.8852GiB/s) | 20260506 00:48:25.622 | INFO | Downloaded 34GiB to GPU in 2.03387s (16.7169GiB/s) | 20260506 00:48:30.765 | INFO | Analyzing chunk frames = [7632896,7633408) | 20260506 00:48:30.765 | INFO | MemInfo: { | 20260506 00:48:30.765 | INFO | "gpu" : " 17 GB", | 20260506 00:48:30.765 | INFO | "huge" : "", | 20260506 00:48:30.765 | INFO | "huge_shared" : " 27 GB", | 20260506 00:48:30.765 | INFO | "malloc" : " 14 GB", | 20260506 00:48:30.765 | INFO | "pinned" : " 29 GB" | 20260506 00:48:30.765 | INFO | } | 20260506 00:48:30.765 | INFO | Perf report for the last 5.14366 seconds: | 20260506 00:48:30.765 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:48:30.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:30.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:30.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:30.765 | INFO | ANALYSIS : 60.9%, 3.0, 95.2ms, 0 | 20260506 00:48:30.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:30.765 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:30.765 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:48:30.765 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:30.765 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:30.766 | INFO | Uploaded 36GiB to GPU in 2.08797s (17.2416GiB/s) | 20260506 00:48:30.766 | INFO | Downloaded 34GiB to GPU in 2.04859s (16.5968GiB/s) | 20260506 00:48:35.895 | INFO | Analyzing chunk frames = [7633408,7633920) | 20260506 00:48:35.895 | INFO | MemInfo: { | 20260506 00:48:35.895 | INFO | "gpu" : " 17 GB", | 20260506 00:48:35.895 | INFO | "huge" : "", | 20260506 00:48:35.895 | INFO | "huge_shared" : " 27 GB", | 20260506 00:48:35.896 | INFO | "malloc" : " 14 GB", | 20260506 00:48:35.896 | INFO | "pinned" : " 29 GB" | 20260506 00:48:35.896 | INFO | } | 20260506 00:48:35.896 | INFO | Perf report for the last 5.13046 seconds: | 20260506 00:48:35.896 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:48:35.896 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:35.896 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:35.896 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:35.896 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 00:48:35.896 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:35.896 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:35.896 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:48:35.896 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:35.896 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:35.896 | INFO | Uploaded 36GiB to GPU in 2.07103s (17.3826GiB/s) | 20260506 00:48:35.896 | INFO | Downloaded 34GiB to GPU in 2.03967s (16.6694GiB/s) | 20260506 00:48:41.038 | INFO | Analyzing chunk frames = [7633920,7634432) | 20260506 00:48:41.039 | INFO | MemInfo: { | 20260506 00:48:41.039 | INFO | "gpu" : " 17 GB", | 20260506 00:48:41.039 | INFO | "huge" : "", | 20260506 00:48:41.039 | INFO | "huge_shared" : " 27 GB", | 20260506 00:48:41.039 | INFO | "malloc" : " 14 GB", | 20260506 00:48:41.039 | INFO | "pinned" : " 29 GB" | 20260506 00:48:41.039 | INFO | } | 20260506 00:48:41.039 | INFO | Perf report for the last 5.14303 seconds: | 20260506 00:48:41.039 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:48:41.039 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:41.039 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:41.039 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:41.039 | INFO | ANALYSIS : 62.1%, 3.0, 97.1ms, 0 | 20260506 00:48:41.039 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:41.039 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:41.039 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 00:48:41.039 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:41.039 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:41.039 | INFO | Uploaded 36GiB to GPU in 2.06707s (17.416GiB/s) | 20260506 00:48:41.039 | INFO | Downloaded 34GiB to GPU in 2.06041s (16.5015GiB/s) | 20260506 00:48:46.177 | INFO | Analyzing chunk frames = [7634432,7634944) | 20260506 00:48:46.177 | INFO | MemInfo: { | 20260506 00:48:46.177 | INFO | "gpu" : " 17 GB", | 20260506 00:48:46.177 | INFO | "huge" : "", | 20260506 00:48:46.177 | INFO | "huge_shared" : " 27 GB", | 20260506 00:48:46.177 | INFO | "malloc" : " 14 GB", | 20260506 00:48:46.178 | INFO | "pinned" : " 29 GB" | 20260506 00:48:46.178 | INFO | } | 20260506 00:48:46.178 | INFO | Perf report for the last 5.13883 seconds: | 20260506 00:48:46.178 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:48:46.178 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:46.178 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:46.178 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:46.178 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260506 00:48:46.178 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:46.178 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:46.178 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:48:46.178 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:46.178 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:46.178 | INFO | Uploaded 36GiB to GPU in 2.13626s (16.8519GiB/s) | 20260506 00:48:46.178 | INFO | Downloaded 34GiB to GPU in 2.03151s (16.7363GiB/s) | 20260506 00:48:46.178 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 00:48:46.178 | INFO | Total mem usage: 15.3 GiB | 20260506 00:48:46.178 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 00:48:46.178 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.16%) | 20260506 00:48:46.178 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 00:48:46.178 | INFO | High Water Breakdown: | 20260506 00:48:46.178 | INFO | 60.9% : 8.55 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 00:48:46.178 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 00:48:46.178 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 00:48:46.178 | INFO | ------------------------------------------ | 20260506 00:48:46.178 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:48:46.178 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 00:48:46.178 | INFO | Total mem usage: 26.5 GiB | 20260506 00:48:46.178 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 00:48:46.178 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 00:48:46.178 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260506 00:48:46.178 | INFO | High Water Breakdown: | 20260506 00:48:46.178 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:48:46.178 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:48:46.178 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:48:46.178 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:48:46.178 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:48:46.178 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:48:46.178 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 00:48:46.178 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 00:48:46.178 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:48:46.178 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 00:48:46.178 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 00:48:46.178 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 00:48:46.178 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 00:48:46.178 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 00:48:46.178 | INFO | ------------------------------------------ | 20260506 00:48:46.178 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:48:46.178 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 00:48:46.178 | INFO | Total mem usage: 25.6 GiB | 20260506 00:48:46.178 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 00:48:46.178 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 00:48:46.178 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 00:48:46.178 | INFO | High Water Breakdown: | 20260506 00:48:46.179 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 00:48:46.179 | INFO | ------------------------------------------ | 20260506 00:48:46.179 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:48:46.179 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 00:48:46.179 | INFO | Total mem usage: 15.7 GiB | 20260506 00:48:46.179 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 00:48:46.179 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 00:48:46.179 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 00:48:46.179 | INFO | High Water Breakdown: | 20260506 00:48:46.179 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 00:48:46.179 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 00:48:46.179 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 00:48:46.179 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 00:48:46.179 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 00:48:46.179 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 00:48:46.179 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 00:48:46.179 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 00:48:46.179 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 00:48:46.179 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:48:46.179 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 00:48:46.179 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:48:46.179 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 00:48:46.179 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 00:48:46.179 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 00:48:46.179 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 00:48:46.179 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:48:46.179 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 00:48:46.179 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:48:46.179 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:48:46.179 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:48:46.179 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 00:48:46.179 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 00:48:46.179 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:48:46.179 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 00:48:46.179 | INFO | ------------------------------------------ | 20260506 00:48:46.179 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:48:46.179 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 00:48:46.179 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:48:46.179 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:48:46.179 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.38e+01, 1.34e+01 / 7.42e-01 | 20260506 00:48:46.179 | INFO | PerfReport: 0.29%, AggregateStats -- 9.18e-02 / 1.04e-01, 1.07e-02 / 7.03e-03 | 20260506 00:48:46.179 | INFO | PerfReport: 11.83%, Baseline -- 3.70e+00 / 4.52e+00, 5.65e-01 / 1.99e-01 | 20260506 00:48:46.179 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.54e+00, 1.90e-01 / 7.56e-02 | 20260506 00:48:46.179 | INFO | PerfReport: 15.97%, DME -- 4.00e+01 / 4.38e+01, 3.75e+00 / 8.43e-01 | 20260506 00:48:46.179 | INFO | PerfReport: 57.41%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.49e+00 / 1.10e+00 | 20260506 00:48:46.179 | INFO | PerfReport: 7.08%, PulseAccumulating -- 2.21e+00 / 2.72e+00, 4.20e-01 / 2.42e-01 | 20260506 00:48:46.179 | INFO | PerfReport: 3.07%, Metrics -- 9.61e-01 / 1.23e+00, 2.59e-01 / 1.19e-01 | 20260506 00:48:46.179 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.32e-01 / 3.86e-01, 5.61e-02 / 2.92e-02 | 20260506 00:48:46.179 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 00:48:46.179 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:48:46.179 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:48:46.179 | INFO | PerfReport: Full Execution -- 9.82e+01 / -nan, 2.44e+01 / 0.00e+00 | 20260506 00:48:46.179 | INFO | PerfReport: 22.27%, Upload -- 2.19e+01 / 0.00e+00, 7.45e+00 / 0.00e+00 | 20260506 00:48:46.179 | INFO | PerfReport: 21.93%, Download -- 2.15e+01 / 0.00e+00, 7.63e+00 / 0.00e+00 | 20260506 00:48:46.179 | INFO | PerfReport: 31.89%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 00:48:46.179 | INFO | PerfReport: 2.06%, LockUpload -- 2.02e+00 / 0.00e+00, 7.80e+00 / 0.00e+00 | 20260506 00:48:46.179 | INFO | PerfReport: 1.34%, LockDownload -- 1.32e+00 / 0.00e+00, 2.89e+00 / 0.00e+00 | 20260506 00:48:46.179 | INFO | PerfReport: 20.49%, LockCompute -- 2.01e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 00:48:46.179 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.86e-02 / 0.00e+00 | 20260506 00:48:46.303 | INFO | Starting a round of Baz IO | 20260506 00:48:49.441 | INFO | Populating RTMetrics report | 20260506 00:48:49.441 | INFO | RTMetrics report populated | 20260506 00:48:49.441 | INFO | Opening tmp RTMetrics json file | 20260506 00:48:49.444 | INFO | Serializing RTMetrics to json | 20260506 00:48:49.445 | INFO | Serialization done, swapping tmp file now... | 20260506 00:48:49.446 | INFO | RTMetrics done | 20260506 00:48:49.519 | INFO | Finished a round of Baz IO: Wrote 1.849 GiB in 3.215 seconds | 20260506 00:48:49.519 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.35% / 0.00% / 78.65% | 20260506 00:48:49.519 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0880678% / 7.94729e-06% / 4.76837e-05% | 20260506 00:48:49.519 | INFO | Baz IO overall: | 20260506 00:48:49.519 | INFO | BytesWritten : 4326676136213 | 20260506 00:48:49.519 | INFO | NumEvents : 2035206488934 | 20260506 00:48:51.313 | INFO | Analyzing chunk frames = [7634944,7635456) | 20260506 00:48:51.313 | INFO | MemInfo: { | 20260506 00:48:51.313 | INFO | "gpu" : " 17 GB", | 20260506 00:48:51.313 | INFO | "huge" : "", | 20260506 00:48:51.313 | INFO | "huge_shared" : " 27 GB", | 20260506 00:48:51.313 | INFO | "malloc" : " 14 GB", | 20260506 00:48:51.313 | INFO | "pinned" : " 29 GB" | 20260506 00:48:51.313 | INFO | } | 20260506 00:48:51.313 | INFO | Perf report for the last 5.1356 seconds: | 20260506 00:48:51.313 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:48:51.313 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:51.313 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:51.313 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:51.313 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260506 00:48:51.313 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:51.313 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260506 00:48:51.313 | INFO | PRE_HQ : 59.4%, 1.7, 53.8ms, 0 | 20260506 00:48:51.313 | INFO | BAZWRITER : 61.9%, 3.5, 112.0ms, 0 | 20260506 00:48:51.313 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:48:51.314 | INFO | Uploaded 36GiB to GPU in 2.15108s (16.7358GiB/s) | 20260506 00:48:51.314 | INFO | Downloaded 38GiB to GPU in 2.23988s (16.9652GiB/s) | 20260506 00:48:56.448 | INFO | Analyzing chunk frames = [7635456,7635968) | 20260506 00:48:56.449 | INFO | MemInfo: { | 20260506 00:48:56.449 | INFO | "gpu" : " 17 GB", | 20260506 00:48:56.449 | INFO | "huge" : "", | 20260506 00:48:56.449 | INFO | "huge_shared" : " 27 GB", | 20260506 00:48:56.449 | INFO | "malloc" : " 14 GB", | 20260506 00:48:56.449 | INFO | "pinned" : " 29 GB" | 20260506 00:48:56.449 | INFO | } | 20260506 00:48:56.449 | INFO | Perf report for the last 5.13553 seconds: | 20260506 00:48:56.449 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:48:56.449 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:56.449 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:56.449 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:56.449 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 00:48:56.449 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:56.449 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:56.449 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 00:48:56.449 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:56.449 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:48:56.449 | INFO | Uploaded 36GiB to GPU in 2.09801s (17.1591GiB/s) | 20260506 00:48:56.449 | INFO | Downloaded 34GiB to GPU in 2.03906s (16.6743GiB/s) | 20260506 00:49:01.592 | INFO | Analyzing chunk frames = [7635968,7636480) | 20260506 00:49:01.592 | INFO | MemInfo: { | 20260506 00:49:01.592 | INFO | "gpu" : " 17 GB", | 20260506 00:49:01.592 | INFO | "huge" : "", | 20260506 00:49:01.592 | INFO | "huge_shared" : " 27 GB", | 20260506 00:49:01.592 | INFO | "malloc" : " 14 GB", | 20260506 00:49:01.592 | INFO | "pinned" : " 29 GB" | 20260506 00:49:01.592 | INFO | } | 20260506 00:49:01.592 | INFO | Perf report for the last 5.14321 seconds: | 20260506 00:49:01.592 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:49:01.592 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:01.592 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:01.592 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:01.592 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 00:49:01.592 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:01.592 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:01.592 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:49:01.592 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:01.592 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:01.592 | INFO | Uploaded 36GiB to GPU in 2.0887s (17.2356GiB/s) | 20260506 00:49:01.592 | INFO | Downloaded 34GiB to GPU in 2.03296s (16.7244GiB/s) | 20260506 00:49:06.739 | INFO | Analyzing chunk frames = [7636480,7636992) | 20260506 00:49:06.739 | INFO | MemInfo: { | 20260506 00:49:06.739 | INFO | "gpu" : " 17 GB", | 20260506 00:49:06.739 | INFO | "huge" : "", | 20260506 00:49:06.739 | INFO | "huge_shared" : " 27 GB", | 20260506 00:49:06.739 | INFO | "malloc" : " 14 GB", | 20260506 00:49:06.739 | INFO | "pinned" : " 29 GB" | 20260506 00:49:06.739 | INFO | } | 20260506 00:49:06.739 | INFO | Perf report for the last 5.1475 seconds: | 20260506 00:49:06.739 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:49:06.739 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:06.739 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:06.739 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:06.739 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260506 00:49:06.739 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:06.740 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:06.740 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:49:06.740 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:06.740 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:06.740 | INFO | Uploaded 36GiB to GPU in 2.13076s (16.8954GiB/s) | 20260506 00:49:06.740 | INFO | Downloaded 34GiB to GPU in 2.01772s (16.8507GiB/s) | 20260506 00:49:11.876 | INFO | Analyzing chunk frames = [7636992,7637504) | 20260506 00:49:11.877 | INFO | MemInfo: { | 20260506 00:49:11.877 | INFO | "gpu" : " 17 GB", | 20260506 00:49:11.877 | INFO | "huge" : "", | 20260506 00:49:11.877 | INFO | "huge_shared" : " 27 GB", | 20260506 00:49:11.877 | INFO | "malloc" : " 14 GB", | 20260506 00:49:11.877 | INFO | "pinned" : " 29 GB" | 20260506 00:49:11.877 | INFO | } | 20260506 00:49:11.877 | INFO | Perf report for the last 5.13724 seconds: | 20260506 00:49:11.877 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:49:11.877 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:11.877 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:11.877 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:11.877 | INFO | ANALYSIS : 60.8%, 3.0, 94.9ms, 0 | 20260506 00:49:11.877 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:11.877 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:11.877 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:49:11.877 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:11.877 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:11.877 | INFO | Uploaded 36GiB to GPU in 2.08362s (17.2776GiB/s) | 20260506 00:49:11.877 | INFO | Downloaded 34GiB to GPU in 2.03648s (16.6955GiB/s) | 20260506 00:49:17.013 | INFO | Analyzing chunk frames = [7637504,7638016) | 20260506 00:49:17.013 | INFO | MemInfo: { | 20260506 00:49:17.013 | INFO | "gpu" : " 17 GB", | 20260506 00:49:17.013 | INFO | "huge" : "", | 20260506 00:49:17.013 | INFO | "huge_shared" : " 27 GB", | 20260506 00:49:17.013 | INFO | "malloc" : " 14 GB", | 20260506 00:49:17.013 | INFO | "pinned" : " 29 GB" | 20260506 00:49:17.013 | INFO | } | 20260506 00:49:17.013 | INFO | Perf report for the last 5.1361 seconds: | 20260506 00:49:17.013 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:49:17.013 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:17.013 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:17.013 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:17.013 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260506 00:49:17.013 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:17.013 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:17.013 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:49:17.013 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:17.013 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:17.013 | INFO | Uploaded 36GiB to GPU in 2.06513s (17.4323GiB/s) | 20260506 00:49:17.013 | INFO | Downloaded 34GiB to GPU in 2.03647s (16.6956GiB/s) | 20260506 00:49:22.150 | INFO | Analyzing chunk frames = [7638016,7638528) | 20260506 00:49:22.150 | INFO | MemInfo: { | 20260506 00:49:22.150 | INFO | "gpu" : " 17 GB", | 20260506 00:49:22.150 | INFO | "huge" : "", | 20260506 00:49:22.150 | INFO | "huge_shared" : " 27 GB", | 20260506 00:49:22.150 | INFO | "malloc" : " 14 GB", | 20260506 00:49:22.150 | INFO | "pinned" : " 29 GB" | 20260506 00:49:22.150 | INFO | } | 20260506 00:49:22.150 | INFO | Perf report for the last 5.13729 seconds: | 20260506 00:49:22.150 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:49:22.150 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:22.150 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:22.150 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:22.150 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260506 00:49:22.150 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:22.150 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:22.150 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 00:49:22.150 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:22.150 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:22.150 | INFO | Uploaded 36GiB to GPU in 2.07197s (17.3747GiB/s) | 20260506 00:49:22.150 | INFO | Downloaded 34GiB to GPU in 2.0686s (16.4362GiB/s) | 20260506 00:49:27.290 | INFO | Analyzing chunk frames = [7638528,7639040) | 20260506 00:49:27.290 | INFO | MemInfo: { | 20260506 00:49:27.290 | INFO | "gpu" : " 17 GB", | 20260506 00:49:27.290 | INFO | "huge" : "", | 20260506 00:49:27.290 | INFO | "huge_shared" : " 27 GB", | 20260506 00:49:27.290 | INFO | "malloc" : " 14 GB", | 20260506 00:49:27.290 | INFO | "pinned" : " 29 GB" | 20260506 00:49:27.290 | INFO | } | 20260506 00:49:27.291 | INFO | Perf report for the last 5.14035 seconds: | 20260506 00:49:27.291 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:49:27.291 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:27.291 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:27.291 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:27.291 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260506 00:49:27.291 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:27.291 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:27.291 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 00:49:27.291 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:27.291 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:27.291 | INFO | Uploaded 36GiB to GPU in 2.13885s (16.8315GiB/s) | 20260506 00:49:27.291 | INFO | Downloaded 34GiB to GPU in 2.02982s (16.7503GiB/s) | 20260506 00:49:27.409 | INFO | Starting a round of Baz IO | 20260506 00:49:30.556 | INFO | Populating RTMetrics report | 20260506 00:49:30.556 | INFO | RTMetrics report populated | 20260506 00:49:30.556 | INFO | Opening tmp RTMetrics json file | 20260506 00:49:30.557 | INFO | Serializing RTMetrics to json | 20260506 00:49:30.558 | INFO | Serialization done, swapping tmp file now... | 20260506 00:49:30.559 | INFO | RTMetrics done | 20260506 00:49:30.631 | INFO | Finished a round of Baz IO: Wrote 1.849 GiB in 3.222 seconds | 20260506 00:49:30.631 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.35% / 0.00% / 78.65% | 20260506 00:49:30.631 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0882268% / 3.97364e-06% / 4.76837e-05% | 20260506 00:49:30.631 | INFO | Baz IO overall: | 20260506 00:49:30.632 | INFO | BytesWritten : 4328661331152 | 20260506 00:49:30.632 | INFO | NumEvents : 2036113794504 | 20260506 00:49:32.431 | INFO | Analyzing chunk frames = [7639040,7639552) | 20260506 00:49:32.431 | INFO | MemInfo: { | 20260506 00:49:32.431 | INFO | "gpu" : " 17 GB", | 20260506 00:49:32.431 | INFO | "huge" : "", | 20260506 00:49:32.431 | INFO | "huge_shared" : " 27 GB", | 20260506 00:49:32.431 | INFO | "malloc" : " 14 GB", | 20260506 00:49:32.431 | INFO | "pinned" : " 29 GB" | 20260506 00:49:32.431 | INFO | } | 20260506 00:49:32.431 | INFO | Perf report for the last 5.14069 seconds: | 20260506 00:49:32.431 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:49:32.431 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:32.431 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:32.431 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:32.431 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260506 00:49:32.431 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:32.431 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260506 00:49:32.431 | INFO | PRE_HQ : 59.6%, 1.7, 53.7ms, 0 | 20260506 00:49:32.431 | INFO | BAZWRITER : 61.8%, 3.5, 110.1ms, 0 | 20260506 00:49:32.431 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:49:32.432 | INFO | Uploaded 36GiB to GPU in 2.18245s (16.4952GiB/s) | 20260506 00:49:32.432 | INFO | Downloaded 38GiB to GPU in 2.24299s (16.9417GiB/s) | 20260506 00:49:37.572 | INFO | Analyzing chunk frames = [7639552,7640064) | 20260506 00:49:37.572 | INFO | MemInfo: { | 20260506 00:49:37.572 | INFO | "gpu" : " 17 GB", | 20260506 00:49:37.572 | INFO | "huge" : "", | 20260506 00:49:37.572 | INFO | "huge_shared" : " 27 GB", | 20260506 00:49:37.572 | INFO | "malloc" : " 14 GB", | 20260506 00:49:37.572 | INFO | "pinned" : " 29 GB" | 20260506 00:49:37.572 | INFO | } | 20260506 00:49:37.572 | INFO | Perf report for the last 5.14061 seconds: | 20260506 00:49:37.572 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:49:37.572 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:37.572 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:37.572 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:37.572 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 00:49:37.572 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:37.572 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:37.572 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:49:37.572 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:37.572 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:37.572 | INFO | Uploaded 36GiB to GPU in 2.11199s (17.0456GiB/s) | 20260506 00:49:37.572 | INFO | Downloaded 34GiB to GPU in 2.0379s (16.6838GiB/s) | 20260506 00:49:42.707 | INFO | Analyzing chunk frames = [7640064,7640576) | 20260506 00:49:42.707 | INFO | MemInfo: { | 20260506 00:49:42.707 | INFO | "gpu" : " 17 GB", | 20260506 00:49:42.707 | INFO | "huge" : "", | 20260506 00:49:42.707 | INFO | "huge_shared" : " 27 GB", | 20260506 00:49:42.707 | INFO | "malloc" : " 14 GB", | 20260506 00:49:42.707 | INFO | "pinned" : " 29 GB" | 20260506 00:49:42.707 | INFO | } | 20260506 00:49:42.707 | INFO | Perf report for the last 5.13533 seconds: | 20260506 00:49:42.707 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:49:42.707 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:42.707 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:42.707 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:42.707 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 00:49:42.707 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:42.707 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:42.707 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:49:42.707 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:42.707 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:42.707 | INFO | Uploaded 36GiB to GPU in 2.1005s (17.1388GiB/s) | 20260506 00:49:42.708 | INFO | Downloaded 34GiB to GPU in 2.02825s (16.7632GiB/s) | 20260506 00:49:47.847 | INFO | Analyzing chunk frames = [7640576,7641088) | 20260506 00:49:47.847 | INFO | MemInfo: { | 20260506 00:49:47.847 | INFO | "gpu" : " 17 GB", | 20260506 00:49:47.847 | INFO | "huge" : "", | 20260506 00:49:47.847 | INFO | "huge_shared" : " 27 GB", | 20260506 00:49:47.847 | INFO | "malloc" : " 14 GB", | 20260506 00:49:47.847 | INFO | "pinned" : " 29 GB" | 20260506 00:49:47.847 | INFO | } | 20260506 00:49:47.847 | INFO | Perf report for the last 5.14008 seconds: | 20260506 00:49:47.847 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:49:47.847 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:47.847 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:47.847 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:47.847 | INFO | ANALYSIS : 61.7%, 3.0, 95.8ms, 0 | 20260506 00:49:47.847 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:47.847 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:47.847 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:49:47.847 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:47.847 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:47.848 | INFO | Uploaded 36GiB to GPU in 2.14456s (16.7867GiB/s) | 20260506 00:49:47.848 | INFO | Downloaded 34GiB to GPU in 2.02762s (16.7685GiB/s) | 20260506 00:49:52.988 | INFO | Analyzing chunk frames = [7641088,7641600) | 20260506 00:49:52.988 | INFO | MemInfo: { | 20260506 00:49:52.988 | INFO | "gpu" : " 17 GB", | 20260506 00:49:52.988 | INFO | "huge" : "", | 20260506 00:49:52.988 | INFO | "huge_shared" : " 27 GB", | 20260506 00:49:52.988 | INFO | "malloc" : " 14 GB", | 20260506 00:49:52.988 | INFO | "pinned" : " 29 GB" | 20260506 00:49:52.988 | INFO | } | 20260506 00:49:52.988 | INFO | Perf report for the last 5.14116 seconds: | 20260506 00:49:52.988 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:49:52.988 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:52.988 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:52.988 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:52.989 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 00:49:52.989 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:52.989 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:52.989 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:49:52.989 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:52.989 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:52.989 | INFO | Uploaded 36GiB to GPU in 2.08459s (17.2696GiB/s) | 20260506 00:49:52.989 | INFO | Downloaded 34GiB to GPU in 2.04131s (16.656GiB/s) | 20260506 00:49:58.132 | INFO | Analyzing chunk frames = [7641600,7642112) | 20260506 00:49:58.132 | INFO | MemInfo: { | 20260506 00:49:58.132 | INFO | "gpu" : " 17 GB", | 20260506 00:49:58.132 | INFO | "huge" : "", | 20260506 00:49:58.133 | INFO | "huge_shared" : " 27 GB", | 20260506 00:49:58.133 | INFO | "malloc" : " 14 GB", | 20260506 00:49:58.133 | INFO | "pinned" : " 29 GB" | 20260506 00:49:58.133 | INFO | } | 20260506 00:49:58.133 | INFO | Perf report for the last 5.14415 seconds: | 20260506 00:49:58.133 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:49:58.133 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:58.133 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:58.133 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:58.133 | INFO | ANALYSIS : 61.1%, 3.0, 95.6ms, 0 | 20260506 00:49:58.133 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:58.133 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:58.133 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:49:58.133 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:58.133 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:49:58.133 | INFO | Uploaded 36GiB to GPU in 2.06152s (17.4628GiB/s) | 20260506 00:49:58.133 | INFO | Downloaded 34GiB to GPU in 2.03465s (16.7105GiB/s) | 20260506 00:50:03.261 | INFO | Analyzing chunk frames = [7642112,7642624) | 20260506 00:50:03.261 | INFO | MemInfo: { | 20260506 00:50:03.261 | INFO | "gpu" : " 17 GB", | 20260506 00:50:03.262 | INFO | "huge" : "", | 20260506 00:50:03.262 | INFO | "huge_shared" : " 27 GB", | 20260506 00:50:03.262 | INFO | "malloc" : " 14 GB", | 20260506 00:50:03.262 | INFO | "pinned" : " 29 GB" | 20260506 00:50:03.262 | INFO | } | 20260506 00:50:03.262 | INFO | Perf report for the last 5.12908 seconds: | 20260506 00:50:03.262 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:50:03.262 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:03.262 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:03.262 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:03.262 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 00:50:03.262 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:03.262 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:03.262 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 00:50:03.262 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:03.262 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:03.262 | INFO | Uploaded 36GiB to GPU in 2.1064s (17.0908GiB/s) | 20260506 00:50:03.262 | INFO | Downloaded 34GiB to GPU in 2.07786s (16.363GiB/s) | 20260506 00:50:08.400 | INFO | Analyzing chunk frames = [7642624,7643136) | 20260506 00:50:08.400 | INFO | MemInfo: { | 20260506 00:50:08.400 | INFO | "gpu" : " 17 GB", | 20260506 00:50:08.400 | INFO | "huge" : "", | 20260506 00:50:08.400 | INFO | "huge_shared" : " 27 GB", | 20260506 00:50:08.400 | INFO | "malloc" : " 14 GB", | 20260506 00:50:08.400 | INFO | "pinned" : " 29 GB" | 20260506 00:50:08.400 | INFO | } | 20260506 00:50:08.400 | INFO | Perf report for the last 5.13798 seconds: | 20260506 00:50:08.400 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:50:08.400 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:08.400 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:08.400 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:08.400 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260506 00:50:08.400 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:08.400 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:08.400 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 00:50:08.400 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:08.400 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:08.400 | INFO | Uploaded 36GiB to GPU in 2.11975s (16.9831GiB/s) | 20260506 00:50:08.400 | INFO | Downloaded 34GiB to GPU in 2.00906s (16.9233GiB/s) | 20260506 00:50:08.400 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 00:50:08.400 | INFO | Total mem usage: 15.3 GiB | 20260506 00:50:08.400 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 00:50:08.400 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.12%) | 20260506 00:50:08.400 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 00:50:08.400 | INFO | High Water Breakdown: | 20260506 00:50:08.400 | INFO | 60.9% : 8.55 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 00:50:08.400 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 00:50:08.400 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 00:50:08.400 | INFO | ------------------------------------------ | 20260506 00:50:08.400 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:50:08.400 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 00:50:08.400 | INFO | Total mem usage: 26.5 GiB | 20260506 00:50:08.400 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 00:50:08.400 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 00:50:08.400 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 00:50:08.400 | INFO | High Water Breakdown: | 20260506 00:50:08.400 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:50:08.400 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:50:08.400 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:50:08.400 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:50:08.400 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:50:08.400 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:50:08.400 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 00:50:08.400 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 00:50:08.400 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:50:08.400 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 00:50:08.400 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 00:50:08.401 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 00:50:08.401 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 00:50:08.401 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 00:50:08.401 | INFO | ------------------------------------------ | 20260506 00:50:08.401 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:50:08.401 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 00:50:08.401 | INFO | Total mem usage: 25.6 GiB | 20260506 00:50:08.401 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 00:50:08.401 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 00:50:08.401 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 00:50:08.401 | INFO | High Water Breakdown: | 20260506 00:50:08.401 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 00:50:08.401 | INFO | ------------------------------------------ | 20260506 00:50:08.401 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:50:08.401 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 00:50:08.401 | INFO | Total mem usage: 15.7 GiB | 20260506 00:50:08.401 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 00:50:08.401 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 00:50:08.401 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 00:50:08.401 | INFO | High Water Breakdown: | 20260506 00:50:08.401 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 00:50:08.401 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 00:50:08.401 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 00:50:08.401 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 00:50:08.401 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 00:50:08.401 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 00:50:08.401 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 00:50:08.401 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 00:50:08.401 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 00:50:08.401 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:50:08.401 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 00:50:08.401 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:50:08.401 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 00:50:08.401 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 00:50:08.401 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 00:50:08.401 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 00:50:08.401 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:50:08.401 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 00:50:08.401 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:50:08.401 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:50:08.401 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:50:08.401 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 00:50:08.401 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 00:50:08.401 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:50:08.401 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 00:50:08.401 | INFO | ------------------------------------------ | 20260506 00:50:08.401 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:50:08.401 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 00:50:08.401 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:50:08.401 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:50:08.401 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.39e+01, 1.34e+01 / 8.28e-01 | 20260506 00:50:08.401 | INFO | PerfReport: 0.29%, AggregateStats -- 9.18e-02 / 1.04e-01, 1.14e-02 / 4.65e-03 | 20260506 00:50:08.401 | INFO | PerfReport: 11.82%, Baseline -- 3.70e+00 / 4.47e+00, 5.61e-01 / 2.58e-01 | 20260506 00:50:08.401 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.51e+00, 1.87e-01 / 1.05e-01 | 20260506 00:50:08.401 | INFO | PerfReport: 15.99%, DME -- 4.01e+01 / 4.39e+01, 3.67e+00 / 1.02e+00 | 20260506 00:50:08.401 | INFO | PerfReport: 57.44%, FrameLabeling -- 1.80e+01 / 1.96e+01, 1.53e+00 / 1.17e+00 | 20260506 00:50:08.401 | INFO | PerfReport: 7.07%, PulseAccumulating -- 2.21e+00 / 2.74e+00, 4.17e-01 / 2.59e-01 | 20260506 00:50:08.401 | INFO | PerfReport: 3.06%, Metrics -- 9.59e-01 / 1.23e+00, 2.58e-01 / 1.30e-01 | 20260506 00:50:08.401 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.33e-01 / 3.88e-01, 5.73e-02 / 3.34e-02 | 20260506 00:50:08.401 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 00:50:08.401 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:50:08.401 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:50:08.401 | INFO | PerfReport: Full Execution -- 9.82e+01 / -nan, 2.45e+01 / 0.00e+00 | 20260506 00:50:08.401 | INFO | PerfReport: 22.36%, Upload -- 2.20e+01 / 0.00e+00, 7.47e+00 / 0.00e+00 | 20260506 00:50:08.401 | INFO | PerfReport: 21.87%, Download -- 2.15e+01 / 0.00e+00, 7.60e+00 / 0.00e+00 | 20260506 00:50:08.401 | INFO | PerfReport: 31.88%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 00:50:08.401 | INFO | PerfReport: 2.10%, LockUpload -- 2.06e+00 / 0.00e+00, 7.88e+00 / 0.00e+00 | 20260506 00:50:08.401 | INFO | PerfReport: 1.30%, LockDownload -- 1.27e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260506 00:50:08.401 | INFO | PerfReport: 20.48%, LockCompute -- 2.01e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 00:50:08.401 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.93e-02 / 0.00e+00 | 20260506 00:50:08.518 | INFO | Starting a round of Baz IO | 20260506 00:50:11.636 | INFO | Populating RTMetrics report | 20260506 00:50:11.636 | INFO | RTMetrics report populated | 20260506 00:50:11.636 | INFO | Opening tmp RTMetrics json file | 20260506 00:50:11.653 | INFO | Serializing RTMetrics to json | 20260506 00:50:11.657 | INFO | Serialization done, swapping tmp file now... | 20260506 00:50:11.660 | INFO | RTMetrics done | 20260506 00:50:11.746 | INFO | Finished a round of Baz IO: Wrote 1.847 GiB in 3.227 seconds | 20260506 00:50:11.746 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.35% / 0.00% / 78.65% | 20260506 00:50:11.746 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0880202% / 0% / 6.75519e-05% | 20260506 00:50:11.746 | INFO | Baz IO overall: | 20260506 00:50:11.746 | INFO | BytesWritten : 4330644319198 | 20260506 00:50:11.746 | INFO | NumEvents : 2037018968669 | 20260506 00:50:13.543 | INFO | Analyzing chunk frames = [7643136,7643648) | 20260506 00:50:13.543 | INFO | MemInfo: { | 20260506 00:50:13.543 | INFO | "gpu" : " 17 GB", | 20260506 00:50:13.543 | INFO | "huge" : "", | 20260506 00:50:13.543 | INFO | "huge_shared" : " 27 GB", | 20260506 00:50:13.543 | INFO | "malloc" : " 14 GB", | 20260506 00:50:13.543 | INFO | "pinned" : " 29 GB" | 20260506 00:50:13.543 | INFO | } | 20260506 00:50:13.543 | INFO | Perf report for the last 5.1437 seconds: | 20260506 00:50:13.544 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:50:13.544 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:13.544 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:13.544 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:13.544 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260506 00:50:13.544 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:13.544 | INFO | RT_METRICS : 43.4%, 1.0, 22.8ms, 0 | 20260506 00:50:13.544 | INFO | PRE_HQ : 59.8%, 1.7, 53.7ms, 0 | 20260506 00:50:13.544 | INFO | BAZWRITER : 61.9%, 3.5, 110.2ms, 0 | 20260506 00:50:13.544 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:50:13.544 | INFO | Uploaded 36GiB to GPU in 2.16485s (16.6293GiB/s) | 20260506 00:50:13.544 | INFO | Downloaded 38GiB to GPU in 2.23823s (16.9777GiB/s) | 20260506 00:50:18.689 | INFO | Analyzing chunk frames = [7643648,7644160) | 20260506 00:50:18.689 | INFO | MemInfo: { | 20260506 00:50:18.689 | INFO | "gpu" : " 17 GB", | 20260506 00:50:18.689 | INFO | "huge" : "", | 20260506 00:50:18.689 | INFO | "huge_shared" : " 27 GB", | 20260506 00:50:18.689 | INFO | "malloc" : " 14 GB", | 20260506 00:50:18.689 | INFO | "pinned" : " 29 GB" | 20260506 00:50:18.689 | INFO | } | 20260506 00:50:18.689 | INFO | Perf report for the last 5.14559 seconds: | 20260506 00:50:18.689 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:50:18.689 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:18.689 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:18.689 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:18.689 | INFO | ANALYSIS : 60.8%, 3.0, 95.1ms, 0 | 20260506 00:50:18.689 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:18.689 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:18.689 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:50:18.689 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:18.689 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:18.689 | INFO | Uploaded 36GiB to GPU in 1.99208s (18.0716GiB/s) | 20260506 00:50:18.689 | INFO | Downloaded 34GiB to GPU in 1.9967s (17.0281GiB/s) | 20260506 00:50:23.821 | INFO | Analyzing chunk frames = [7644160,7644672) | 20260506 00:50:23.821 | INFO | MemInfo: { | 20260506 00:50:23.821 | INFO | "gpu" : " 17 GB", | 20260506 00:50:23.821 | INFO | "huge" : "", | 20260506 00:50:23.821 | INFO | "huge_shared" : " 27 GB", | 20260506 00:50:23.821 | INFO | "malloc" : " 14 GB", | 20260506 00:50:23.821 | INFO | "pinned" : " 29 GB" | 20260506 00:50:23.821 | INFO | } | 20260506 00:50:23.821 | INFO | Perf report for the last 5.13246 seconds: | 20260506 00:50:23.821 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:50:23.821 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:23.821 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:23.822 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:23.822 | INFO | ANALYSIS : 61.4%, 3.0, 95.4ms, 0 | 20260506 00:50:23.822 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:23.822 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:23.822 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 00:50:23.822 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:23.822 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:23.822 | INFO | Uploaded 36GiB to GPU in 2.09837s (17.1561GiB/s) | 20260506 00:50:23.822 | INFO | Downloaded 34GiB to GPU in 2.02834s (16.7625GiB/s) | 20260506 00:50:28.956 | INFO | Analyzing chunk frames = [7644672,7645184) | 20260506 00:50:28.956 | INFO | MemInfo: { | 20260506 00:50:28.956 | INFO | "gpu" : " 17 GB", | 20260506 00:50:28.956 | INFO | "huge" : "", | 20260506 00:50:28.956 | INFO | "huge_shared" : " 27 GB", | 20260506 00:50:28.956 | INFO | "malloc" : " 14 GB", | 20260506 00:50:28.956 | INFO | "pinned" : " 29 GB" | 20260506 00:50:28.956 | INFO | } | 20260506 00:50:28.956 | INFO | Perf report for the last 5.1347 seconds: | 20260506 00:50:28.956 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:50:28.956 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:28.956 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:28.956 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:28.956 | INFO | ANALYSIS : 61.6%, 3.0, 95.6ms, 0 | 20260506 00:50:28.956 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:28.956 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:28.956 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 00:50:28.956 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:28.956 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:28.956 | INFO | Uploaded 36GiB to GPU in 2.13455s (16.8654GiB/s) | 20260506 00:50:28.956 | INFO | Downloaded 34GiB to GPU in 2.02921s (16.7553GiB/s) | 20260506 00:50:34.102 | INFO | Analyzing chunk frames = [7645184,7645696) | 20260506 00:50:34.102 | INFO | MemInfo: { | 20260506 00:50:34.102 | INFO | "gpu" : " 17 GB", | 20260506 00:50:34.102 | INFO | "huge" : "", | 20260506 00:50:34.102 | INFO | "huge_shared" : " 27 GB", | 20260506 00:50:34.102 | INFO | "malloc" : " 14 GB", | 20260506 00:50:34.102 | INFO | "pinned" : " 29 GB" | 20260506 00:50:34.102 | INFO | } | 20260506 00:50:34.102 | INFO | Perf report for the last 5.14618 seconds: | 20260506 00:50:34.103 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:50:34.103 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:34.103 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:34.103 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:34.103 | INFO | ANALYSIS : 61.6%, 3.0, 96.3ms, 0 | 20260506 00:50:34.103 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:34.103 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:34.103 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:50:34.103 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:34.103 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:34.103 | INFO | Uploaded 36GiB to GPU in 2.17107s (16.5817GiB/s) | 20260506 00:50:34.103 | INFO | Downloaded 34GiB to GPU in 2.076s (16.3776GiB/s) | 20260506 00:50:39.243 | INFO | Analyzing chunk frames = [7645696,7646208) | 20260506 00:50:39.243 | INFO | MemInfo: { | 20260506 00:50:39.243 | INFO | "gpu" : " 17 GB", | 20260506 00:50:39.243 | INFO | "huge" : "", | 20260506 00:50:39.243 | INFO | "huge_shared" : " 27 GB", | 20260506 00:50:39.243 | INFO | "malloc" : " 14 GB", | 20260506 00:50:39.243 | INFO | "pinned" : " 29 GB" | 20260506 00:50:39.243 | INFO | } | 20260506 00:50:39.244 | INFO | Perf report for the last 5.14114 seconds: | 20260506 00:50:39.244 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:50:39.244 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:39.244 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:39.244 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:39.244 | INFO | ANALYSIS : 61.1%, 3.0, 95.5ms, 0 | 20260506 00:50:39.244 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:39.244 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:39.244 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:50:39.244 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:39.244 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:39.244 | INFO | Uploaded 36GiB to GPU in 2.07106s (17.3824GiB/s) | 20260506 00:50:39.244 | INFO | Downloaded 34GiB to GPU in 2.04272s (16.6445GiB/s) | 20260506 00:50:44.388 | INFO | Analyzing chunk frames = [7646208,7646720) | 20260506 00:50:44.388 | INFO | MemInfo: { | 20260506 00:50:44.388 | INFO | "gpu" : " 17 GB", | 20260506 00:50:44.388 | INFO | "huge" : "", | 20260506 00:50:44.388 | INFO | "huge_shared" : " 27 GB", | 20260506 00:50:44.388 | INFO | "malloc" : " 14 GB", | 20260506 00:50:44.388 | INFO | "pinned" : " 29 GB" | 20260506 00:50:44.388 | INFO | } | 20260506 00:50:44.388 | INFO | Perf report for the last 5.14465 seconds: | 20260506 00:50:44.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:50:44.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:44.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:44.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:44.388 | INFO | ANALYSIS : 62.1%, 3.0, 97.1ms, 0 | 20260506 00:50:44.388 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:44.388 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:44.388 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 00:50:44.388 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:44.388 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:44.388 | INFO | Uploaded 36GiB to GPU in 2.06687s (17.4176GiB/s) | 20260506 00:50:44.388 | INFO | Downloaded 34GiB to GPU in 2.06932s (16.4305GiB/s) | 20260506 00:50:49.520 | INFO | Analyzing chunk frames = [7646720,7647232) | 20260506 00:50:49.520 | INFO | MemInfo: { | 20260506 00:50:49.520 | INFO | "gpu" : " 17 GB", | 20260506 00:50:49.520 | INFO | "huge" : "", | 20260506 00:50:49.520 | INFO | "huge_shared" : " 27 GB", | 20260506 00:50:49.520 | INFO | "malloc" : " 14 GB", | 20260506 00:50:49.520 | INFO | "pinned" : " 29 GB" | 20260506 00:50:49.520 | INFO | } | 20260506 00:50:49.520 | INFO | Perf report for the last 5.13212 seconds: | 20260506 00:50:49.521 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:50:49.521 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:49.521 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:49.521 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:49.521 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 00:50:49.521 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:49.521 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:49.521 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 00:50:49.521 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:49.521 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:49.521 | INFO | Uploaded 36GiB to GPU in 2.04141s (17.6349GiB/s) | 20260506 00:50:49.521 | INFO | Downloaded 34GiB to GPU in 2.02211s (16.8141GiB/s) | 20260506 00:50:49.641 | INFO | Starting a round of Baz IO | 20260506 00:50:52.787 | INFO | Populating RTMetrics report | 20260506 00:50:52.788 | INFO | RTMetrics report populated | 20260506 00:50:52.788 | INFO | Opening tmp RTMetrics json file | 20260506 00:50:52.790 | INFO | Serializing RTMetrics to json | 20260506 00:50:52.792 | INFO | Serialization done, swapping tmp file now... | 20260506 00:50:52.794 | INFO | RTMetrics done | 20260506 00:50:52.867 | INFO | Finished a round of Baz IO: Wrote 1.846 GiB in 3.225 seconds | 20260506 00:50:52.867 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.35% / 0.00% / 78.65% | 20260506 00:50:52.867 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0876307% / 0% / 7.94729e-05% | 20260506 00:50:52.867 | INFO | Baz IO overall: | 20260506 00:50:52.867 | INFO | BytesWritten : 4332626517599 | 20260506 00:50:52.867 | INFO | NumEvents : 2037923961531 | 20260506 00:50:54.661 | INFO | Analyzing chunk frames = [7647232,7647744) | 20260506 00:50:54.661 | INFO | MemInfo: { | 20260506 00:50:54.661 | INFO | "gpu" : " 17 GB", | 20260506 00:50:54.661 | INFO | "huge" : "", | 20260506 00:50:54.661 | INFO | "huge_shared" : " 27 GB", | 20260506 00:50:54.661 | INFO | "malloc" : " 14 GB", | 20260506 00:50:54.661 | INFO | "pinned" : " 29 GB" | 20260506 00:50:54.661 | INFO | } | 20260506 00:50:54.661 | INFO | Perf report for the last 5.14077 seconds: | 20260506 00:50:54.661 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:50:54.661 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:54.661 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:54.661 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:54.661 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260506 00:50:54.661 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:54.661 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260506 00:50:54.661 | INFO | PRE_HQ : 60.1%, 1.7, 54.5ms, 0 | 20260506 00:50:54.661 | INFO | BAZWRITER : 61.9%, 3.5, 111.4ms, 0 | 20260506 00:50:54.661 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 00:50:54.661 | INFO | Uploaded 36GiB to GPU in 2.17058s (16.5855GiB/s) | 20260506 00:50:54.661 | INFO | Downloaded 38GiB to GPU in 2.23864s (16.9746GiB/s) | 20260506 00:50:59.800 | INFO | Analyzing chunk frames = [7647744,7648256) | 20260506 00:50:59.800 | INFO | MemInfo: { | 20260506 00:50:59.800 | INFO | "gpu" : " 17 GB", | 20260506 00:50:59.800 | INFO | "huge" : "", | 20260506 00:50:59.800 | INFO | "huge_shared" : " 27 GB", | 20260506 00:50:59.800 | INFO | "malloc" : " 14 GB", | 20260506 00:50:59.800 | INFO | "pinned" : " 29 GB" | 20260506 00:50:59.800 | INFO | } | 20260506 00:50:59.800 | INFO | Perf report for the last 5.13892 seconds: | 20260506 00:50:59.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:50:59.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:59.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:59.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:59.800 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260506 00:50:59.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:59.800 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:59.800 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 00:50:59.800 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:59.800 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:50:59.800 | INFO | Uploaded 36GiB to GPU in 2.11935s (16.9864GiB/s) | 20260506 00:50:59.800 | INFO | Downloaded 34GiB to GPU in 2.04165s (16.6532GiB/s) | 20260506 00:51:04.941 | INFO | Analyzing chunk frames = [7648256,7648768) | 20260506 00:51:04.941 | INFO | MemInfo: { | 20260506 00:51:04.941 | INFO | "gpu" : " 17 GB", | 20260506 00:51:04.941 | INFO | "huge" : "", | 20260506 00:51:04.941 | INFO | "huge_shared" : " 27 GB", | 20260506 00:51:04.941 | INFO | "malloc" : " 14 GB", | 20260506 00:51:04.941 | INFO | "pinned" : " 29 GB" | 20260506 00:51:04.941 | INFO | } | 20260506 00:51:04.941 | INFO | Perf report for the last 5.14101 seconds: | 20260506 00:51:04.941 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:51:04.941 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:04.941 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:04.941 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:04.941 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 00:51:04.941 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:04.941 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:04.941 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:51:04.941 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:04.941 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:04.942 | INFO | Uploaded 36GiB to GPU in 2.08984s (17.2262GiB/s) | 20260506 00:51:04.942 | INFO | Downloaded 34GiB to GPU in 2.02541s (16.7868GiB/s) | 20260506 00:51:10.087 | INFO | Analyzing chunk frames = [7648768,7649280) | 20260506 00:51:10.087 | INFO | MemInfo: { | 20260506 00:51:10.087 | INFO | "gpu" : " 17 GB", | 20260506 00:51:10.087 | INFO | "huge" : "", | 20260506 00:51:10.087 | INFO | "huge_shared" : " 27 GB", | 20260506 00:51:10.087 | INFO | "malloc" : " 14 GB", | 20260506 00:51:10.087 | INFO | "pinned" : " 29 GB" | 20260506 00:51:10.087 | INFO | } | 20260506 00:51:10.087 | INFO | Perf report for the last 5.14598 seconds: | 20260506 00:51:10.087 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:51:10.087 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:10.087 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:10.087 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:10.087 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260506 00:51:10.087 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:10.087 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:10.087 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:51:10.087 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:10.087 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:10.087 | INFO | Uploaded 36GiB to GPU in 2.13288s (16.8786GiB/s) | 20260506 00:51:10.087 | INFO | Downloaded 34GiB to GPU in 2.02673s (16.7758GiB/s) | 20260506 00:51:15.212 | INFO | Analyzing chunk frames = [7649280,7649792) | 20260506 00:51:15.212 | INFO | MemInfo: { | 20260506 00:51:15.212 | INFO | "gpu" : " 17 GB", | 20260506 00:51:15.212 | INFO | "huge" : "", | 20260506 00:51:15.213 | INFO | "huge_shared" : " 27 GB", | 20260506 00:51:15.213 | INFO | "malloc" : " 14 GB", | 20260506 00:51:15.213 | INFO | "pinned" : " 29 GB" | 20260506 00:51:15.213 | INFO | } | 20260506 00:51:15.213 | INFO | Perf report for the last 5.12557 seconds: | 20260506 00:51:15.213 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:51:15.213 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:15.213 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:15.213 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:15.213 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 00:51:15.213 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:15.213 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:15.213 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 00:51:15.213 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:15.213 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:15.213 | INFO | Uploaded 36GiB to GPU in 2.0893s (17.2306GiB/s) | 20260506 00:51:15.213 | INFO | Downloaded 34GiB to GPU in 2.04778s (16.6034GiB/s) | 20260506 00:51:20.353 | INFO | Analyzing chunk frames = [7649792,7650304) | 20260506 00:51:20.353 | INFO | MemInfo: { | 20260506 00:51:20.353 | INFO | "gpu" : " 17 GB", | 20260506 00:51:20.353 | INFO | "huge" : "", | 20260506 00:51:20.353 | INFO | "huge_shared" : " 27 GB", | 20260506 00:51:20.353 | INFO | "malloc" : " 14 GB", | 20260506 00:51:20.353 | INFO | "pinned" : " 29 GB" | 20260506 00:51:20.353 | INFO | } | 20260506 00:51:20.353 | INFO | Perf report for the last 5.14057 seconds: | 20260506 00:51:20.353 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:51:20.353 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:20.353 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:20.353 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:20.353 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260506 00:51:20.353 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:20.353 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:20.353 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:51:20.353 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:20.354 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:20.354 | INFO | Uploaded 36GiB to GPU in 2.07375s (17.3599GiB/s) | 20260506 00:51:20.354 | INFO | Downloaded 34GiB to GPU in 2.03847s (16.6792GiB/s) | 20260506 00:51:25.498 | INFO | Analyzing chunk frames = [7650304,7650816) | 20260506 00:51:25.498 | INFO | MemInfo: { | 20260506 00:51:25.498 | INFO | "gpu" : " 17 GB", | 20260506 00:51:25.498 | INFO | "huge" : "", | 20260506 00:51:25.498 | INFO | "huge_shared" : " 27 GB", | 20260506 00:51:25.498 | INFO | "malloc" : " 14 GB", | 20260506 00:51:25.498 | INFO | "pinned" : " 29 GB" | 20260506 00:51:25.498 | INFO | } | 20260506 00:51:25.499 | INFO | Perf report for the last 5.14528 seconds: | 20260506 00:51:25.499 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:51:25.499 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:25.499 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:25.499 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:25.499 | INFO | ANALYSIS : 62.0%, 3.0, 97.0ms, 0 | 20260506 00:51:25.499 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:25.499 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:25.499 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 00:51:25.499 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:25.499 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:25.499 | INFO | Uploaded 36GiB to GPU in 2.03585s (17.683GiB/s) | 20260506 00:51:25.499 | INFO | Downloaded 34GiB to GPU in 2.04262s (16.6453GiB/s) | 20260506 00:51:30.637 | INFO | Analyzing chunk frames = [7650816,7651328) | 20260506 00:51:30.637 | INFO | MemInfo: { | 20260506 00:51:30.637 | INFO | "gpu" : " 17 GB", | 20260506 00:51:30.637 | INFO | "huge" : "", | 20260506 00:51:30.637 | INFO | "huge_shared" : " 27 GB", | 20260506 00:51:30.637 | INFO | "malloc" : " 14 GB", | 20260506 00:51:30.637 | INFO | "pinned" : " 29 GB" | 20260506 00:51:30.637 | INFO | } | 20260506 00:51:30.637 | INFO | Perf report for the last 5.13885 seconds: | 20260506 00:51:30.638 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:51:30.638 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:30.638 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:30.638 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:30.638 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260506 00:51:30.638 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:30.638 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:30.638 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 00:51:30.638 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:30.638 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:30.638 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 00:51:30.638 | INFO | Total mem usage: 15.3 GiB | 20260506 00:51:30.638 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 00:51:30.638 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.14%) | 20260506 00:51:30.638 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 00:51:30.638 | INFO | High Water Breakdown: | 20260506 00:51:30.638 | INFO | 60.9% : 8.55 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 00:51:30.638 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 00:51:30.638 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 00:51:30.638 | INFO | ------------------------------------------ | 20260506 00:51:30.638 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:51:30.638 | INFO | Uploaded 36GiB to GPU in 2.04482s (17.6055GiB/s) | 20260506 00:51:30.638 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 00:51:30.638 | INFO | Total mem usage: 26.5 GiB | 20260506 00:51:30.638 | INFO | Downloaded 34GiB to GPU in 2.01883s (16.8415GiB/s) | 20260506 00:51:30.638 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 00:51:30.638 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 00:51:30.638 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 00:51:30.638 | INFO | High Water Breakdown: | 20260506 00:51:30.638 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:51:30.638 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:51:30.638 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:51:30.638 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:51:30.638 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:51:30.638 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:51:30.638 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 00:51:30.638 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 00:51:30.638 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:51:30.638 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 00:51:30.638 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 00:51:30.638 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 00:51:30.638 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 00:51:30.638 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 00:51:30.638 | INFO | ------------------------------------------ | 20260506 00:51:30.638 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:51:30.638 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 00:51:30.638 | INFO | Total mem usage: 25.6 GiB | 20260506 00:51:30.638 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 00:51:30.638 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 00:51:30.638 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 00:51:30.638 | INFO | High Water Breakdown: | 20260506 00:51:30.638 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 00:51:30.638 | INFO | ------------------------------------------ | 20260506 00:51:30.638 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:51:30.638 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 00:51:30.638 | INFO | Total mem usage: 15.7 GiB | 20260506 00:51:30.638 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 00:51:30.639 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 00:51:30.639 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 00:51:30.639 | INFO | High Water Breakdown: | 20260506 00:51:30.639 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 00:51:30.639 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 00:51:30.639 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 00:51:30.639 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 00:51:30.639 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 00:51:30.639 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 00:51:30.639 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 00:51:30.639 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 00:51:30.639 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 00:51:30.639 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:51:30.639 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 00:51:30.639 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:51:30.639 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 00:51:30.639 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 00:51:30.639 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 00:51:30.639 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 00:51:30.639 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:51:30.639 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 00:51:30.639 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:51:30.639 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:51:30.639 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:51:30.639 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 00:51:30.639 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 00:51:30.639 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:51:30.639 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 00:51:30.639 | INFO | ------------------------------------------ | 20260506 00:51:30.639 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:51:30.639 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 00:51:30.639 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:51:30.639 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:51:30.639 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.40e+01, 1.33e+01 / 9.23e-01 | 20260506 00:51:30.639 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.04e-01, 1.04e-02 / 3.28e-03 | 20260506 00:51:30.639 | INFO | PerfReport: 11.85%, Baseline -- 3.71e+00 / 4.51e+00, 5.72e-01 / 1.92e-01 | 20260506 00:51:30.639 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.53e+00, 1.92e-01 / 8.81e-02 | 20260506 00:51:30.639 | INFO | PerfReport: 15.98%, DME -- 4.00e+01 / 4.41e+01, 3.91e+00 / 1.06e+00 | 20260506 00:51:30.639 | INFO | PerfReport: 57.35%, FrameLabeling -- 1.79e+01 / 1.94e+01, 1.47e+00 / 9.92e-01 | 20260506 00:51:30.639 | INFO | PerfReport: 7.11%, PulseAccumulating -- 2.23e+00 / 2.74e+00, 4.25e-01 / 2.27e-01 | 20260506 00:51:30.639 | INFO | PerfReport: 3.07%, Metrics -- 9.61e-01 / 1.23e+00, 2.59e-01 / 1.10e-01 | 20260506 00:51:30.639 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.32e-01 / 3.84e-01, 5.65e-02 / 2.67e-02 | 20260506 00:51:30.639 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 00:51:30.639 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:51:30.639 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:51:30.639 | INFO | PerfReport: Full Execution -- 9.81e+01 / -nan, 2.44e+01 / 0.00e+00 | 20260506 00:51:30.639 | INFO | PerfReport: 22.24%, Upload -- 2.18e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260506 00:51:30.639 | INFO | PerfReport: 21.90%, Download -- 2.15e+01 / 0.00e+00, 7.62e+00 / 0.00e+00 | 20260506 00:51:30.639 | INFO | PerfReport: 31.92%, Compute -- 3.13e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 00:51:30.639 | INFO | PerfReport: 2.07%, LockUpload -- 2.03e+00 / 0.00e+00, 7.78e+00 / 0.00e+00 | 20260506 00:51:30.639 | INFO | PerfReport: 1.32%, LockDownload -- 1.30e+00 / 0.00e+00, 2.92e+00 / 0.00e+00 | 20260506 00:51:30.639 | INFO | PerfReport: 20.54%, LockCompute -- 2.01e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 00:51:30.639 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.98e-02 / 0.00e+00 | 20260506 00:51:30.755 | INFO | Starting a round of Baz IO | 20260506 00:51:33.908 | INFO | Populating RTMetrics report | 20260506 00:51:33.908 | INFO | RTMetrics report populated | 20260506 00:51:33.908 | INFO | Opening tmp RTMetrics json file | 20260506 00:51:33.909 | INFO | Serializing RTMetrics to json | 20260506 00:51:33.910 | INFO | Serialization done, swapping tmp file now... | 20260506 00:51:33.911 | INFO | RTMetrics done | 20260506 00:51:33.985 | INFO | Finished a round of Baz IO: Wrote 1.846 GiB in 3.23 seconds | 20260506 00:51:33.985 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.35% / 0.00% / 78.65% | 20260506 00:51:33.985 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0879248% / 3.97364e-06% / 5.16574e-05% | 20260506 00:51:33.985 | INFO | Baz IO overall: | 20260506 00:51:33.985 | INFO | BytesWritten : 4334609020627 | 20260506 00:51:33.985 | INFO | NumEvents : 2038829478092 | 20260506 00:51:35.769 | INFO | Analyzing chunk frames = [7651328,7651840) | 20260506 00:51:35.769 | INFO | MemInfo: { | 20260506 00:51:35.769 | INFO | "gpu" : " 17 GB", | 20260506 00:51:35.769 | INFO | "huge" : "", | 20260506 00:51:35.769 | INFO | "huge_shared" : " 27 GB", | 20260506 00:51:35.770 | INFO | "malloc" : " 14 GB", | 20260506 00:51:35.770 | INFO | "pinned" : " 29 GB" | 20260506 00:51:35.770 | INFO | } | 20260506 00:51:35.770 | INFO | Perf report for the last 5.13217 seconds: | 20260506 00:51:35.770 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:51:35.770 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:35.770 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:35.770 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:35.770 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260506 00:51:35.770 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:35.770 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260506 00:51:35.770 | INFO | PRE_HQ : 59.6%, 1.7, 53.8ms, 0 | 20260506 00:51:35.770 | INFO | BAZWRITER : 62.0%, 3.5, 110.8ms, 0 | 20260506 00:51:35.770 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:51:35.770 | INFO | Uploaded 36GiB to GPU in 2.16302s (16.6434GiB/s) | 20260506 00:51:35.770 | INFO | Downloaded 38GiB to GPU in 2.24028s (16.9622GiB/s) | 20260506 00:51:40.911 | INFO | Analyzing chunk frames = [7651840,7652352) | 20260506 00:51:40.911 | INFO | MemInfo: { | 20260506 00:51:40.911 | INFO | "gpu" : " 17 GB", | 20260506 00:51:40.911 | INFO | "huge" : "", | 20260506 00:51:40.911 | INFO | "huge_shared" : " 27 GB", | 20260506 00:51:40.911 | INFO | "malloc" : " 14 GB", | 20260506 00:51:40.911 | INFO | "pinned" : " 29 GB" | 20260506 00:51:40.911 | INFO | } | 20260506 00:51:40.911 | INFO | Perf report for the last 5.14123 seconds: | 20260506 00:51:40.911 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:51:40.911 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:40.911 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:40.911 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:40.911 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260506 00:51:40.911 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:40.911 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:40.911 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 00:51:40.911 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:40.911 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:40.911 | INFO | Uploaded 36GiB to GPU in 2.10136s (17.1317GiB/s) | 20260506 00:51:40.911 | INFO | Downloaded 34GiB to GPU in 2.03627s (16.6972GiB/s) | 20260506 00:51:46.053 | INFO | Analyzing chunk frames = [7652352,7652864) | 20260506 00:51:46.053 | INFO | MemInfo: { | 20260506 00:51:46.053 | INFO | "gpu" : " 17 GB", | 20260506 00:51:46.053 | INFO | "huge" : "", | 20260506 00:51:46.053 | INFO | "huge_shared" : " 27 GB", | 20260506 00:51:46.053 | INFO | "malloc" : " 14 GB", | 20260506 00:51:46.053 | INFO | "pinned" : " 29 GB" | 20260506 00:51:46.053 | INFO | } | 20260506 00:51:46.053 | INFO | Perf report for the last 5.14199 seconds: | 20260506 00:51:46.053 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:51:46.053 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:46.053 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:46.053 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:46.053 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 00:51:46.053 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:46.053 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:46.053 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:51:46.053 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:46.053 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:46.053 | INFO | Uploaded 36GiB to GPU in 2.10445s (17.1066GiB/s) | 20260506 00:51:46.053 | INFO | Downloaded 34GiB to GPU in 2.02314s (16.8056GiB/s) | 20260506 00:51:51.193 | INFO | Analyzing chunk frames = [7652864,7653376) | 20260506 00:51:51.193 | INFO | MemInfo: { | 20260506 00:51:51.193 | INFO | "gpu" : " 17 GB", | 20260506 00:51:51.193 | INFO | "huge" : "", | 20260506 00:51:51.193 | INFO | "huge_shared" : " 27 GB", | 20260506 00:51:51.193 | INFO | "malloc" : " 14 GB", | 20260506 00:51:51.193 | INFO | "pinned" : " 29 GB" | 20260506 00:51:51.193 | INFO | } | 20260506 00:51:51.193 | INFO | Perf report for the last 5.1404 seconds: | 20260506 00:51:51.193 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:51:51.193 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:51.193 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:51.193 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:51.193 | INFO | ANALYSIS : 61.6%, 3.0, 95.7ms, 0 | 20260506 00:51:51.193 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:51.193 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:51.193 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:51:51.193 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:51.193 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:51.194 | INFO | Uploaded 36GiB to GPU in 2.13292s (16.8783GiB/s) | 20260506 00:51:51.194 | INFO | Downloaded 34GiB to GPU in 2.01568s (16.8677GiB/s) | 20260506 00:51:56.336 | INFO | Analyzing chunk frames = [7653376,7653888) | 20260506 00:51:56.336 | INFO | MemInfo: { | 20260506 00:51:56.336 | INFO | "gpu" : " 17 GB", | 20260506 00:51:56.336 | INFO | "huge" : "", | 20260506 00:51:56.336 | INFO | "huge_shared" : " 27 GB", | 20260506 00:51:56.336 | INFO | "malloc" : " 14 GB", | 20260506 00:51:56.336 | INFO | "pinned" : " 29 GB" | 20260506 00:51:56.336 | INFO | } | 20260506 00:51:56.336 | INFO | Perf report for the last 5.14307 seconds: | 20260506 00:51:56.336 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:51:56.336 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:56.336 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:56.336 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:56.336 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 00:51:56.336 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:56.336 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:56.336 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 00:51:56.336 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:56.336 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:51:56.337 | INFO | Uploaded 36GiB to GPU in 2.09641s (17.1722GiB/s) | 20260506 00:51:56.337 | INFO | Downloaded 34GiB to GPU in 2.04513s (16.6249GiB/s) | 20260506 00:52:01.471 | INFO | Analyzing chunk frames = [7653888,7654400) | 20260506 00:52:01.471 | INFO | MemInfo: { | 20260506 00:52:01.471 | INFO | "gpu" : " 17 GB", | 20260506 00:52:01.471 | INFO | "huge" : "", | 20260506 00:52:01.471 | INFO | "huge_shared" : " 27 GB", | 20260506 00:52:01.471 | INFO | "malloc" : " 14 GB", | 20260506 00:52:01.471 | INFO | "pinned" : " 29 GB" | 20260506 00:52:01.471 | INFO | } | 20260506 00:52:01.471 | INFO | Perf report for the last 5.13519 seconds: | 20260506 00:52:01.472 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:52:01.472 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:01.472 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:01.472 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:01.472 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 00:52:01.472 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:01.472 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:01.472 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:52:01.472 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:01.472 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:01.472 | INFO | Uploaded 36GiB to GPU in 2.0621s (17.4579GiB/s) | 20260506 00:52:01.472 | INFO | Downloaded 34GiB to GPU in 2.03821s (16.6813GiB/s) | 20260506 00:52:06.615 | INFO | Analyzing chunk frames = [7654400,7654912) | 20260506 00:52:06.616 | INFO | MemInfo: { | 20260506 00:52:06.616 | INFO | "gpu" : " 17 GB", | 20260506 00:52:06.616 | INFO | "huge" : "", | 20260506 00:52:06.616 | INFO | "huge_shared" : " 27 GB", | 20260506 00:52:06.616 | INFO | "malloc" : " 14 GB", | 20260506 00:52:06.616 | INFO | "pinned" : " 29 GB" | 20260506 00:52:06.616 | INFO | } | 20260506 00:52:06.616 | INFO | Perf report for the last 5.14407 seconds: | 20260506 00:52:06.616 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:52:06.616 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:06.616 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:06.616 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:06.616 | INFO | ANALYSIS : 62.1%, 3.0, 97.1ms, 0 | 20260506 00:52:06.616 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:06.616 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:06.616 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 00:52:06.616 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:06.616 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:06.616 | INFO | Uploaded 36GiB to GPU in 2.07028s (17.389GiB/s) | 20260506 00:52:06.616 | INFO | Downloaded 34GiB to GPU in 2.07367s (16.396GiB/s) | 20260506 00:52:11.747 | INFO | Analyzing chunk frames = [7654912,7655424) | 20260506 00:52:11.747 | INFO | MemInfo: { | 20260506 00:52:11.747 | INFO | "gpu" : " 17 GB", | 20260506 00:52:11.747 | INFO | "huge" : "", | 20260506 00:52:11.747 | INFO | "huge_shared" : " 27 GB", | 20260506 00:52:11.747 | INFO | "malloc" : " 14 GB", | 20260506 00:52:11.747 | INFO | "pinned" : " 29 GB" | 20260506 00:52:11.747 | INFO | } | 20260506 00:52:11.747 | INFO | Perf report for the last 5.13128 seconds: | 20260506 00:52:11.747 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:52:11.747 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:11.747 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:11.747 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:11.747 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 00:52:11.747 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:11.747 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:11.747 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:52:11.747 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:11.747 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:11.747 | INFO | Uploaded 36GiB to GPU in 2.06317s (17.4489GiB/s) | 20260506 00:52:11.747 | INFO | Downloaded 34GiB to GPU in 2.01732s (16.8541GiB/s) | 20260506 00:52:11.864 | INFO | Starting a round of Baz IO | 20260506 00:52:15.011 | INFO | Populating RTMetrics report | 20260506 00:52:15.011 | INFO | RTMetrics report populated | 20260506 00:52:15.011 | INFO | Opening tmp RTMetrics json file | 20260506 00:52:15.012 | INFO | Serializing RTMetrics to json | 20260506 00:52:15.013 | INFO | Serialization done, swapping tmp file now... | 20260506 00:52:15.013 | INFO | RTMetrics done | 20260506 00:52:15.086 | INFO | Finished a round of Baz IO: Wrote 1.846 GiB in 3.222 seconds | 20260506 00:52:15.086 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.35% / 0.00% / 78.65% | 20260506 00:52:15.086 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0876904% / 3.97364e-06% / 5.96046e-05% | 20260506 00:52:15.086 | INFO | Baz IO overall: | 20260506 00:52:15.086 | INFO | BytesWritten : 4336591358240 | 20260506 00:52:15.086 | INFO | NumEvents : 2039734995886 | 20260506 00:52:16.887 | INFO | Analyzing chunk frames = [7655424,7655936) | 20260506 00:52:16.887 | INFO | MemInfo: { | 20260506 00:52:16.887 | INFO | "gpu" : " 17 GB", | 20260506 00:52:16.888 | INFO | "huge" : "", | 20260506 00:52:16.888 | INFO | "huge_shared" : " 27 GB", | 20260506 00:52:16.888 | INFO | "malloc" : " 14 GB", | 20260506 00:52:16.888 | INFO | "pinned" : " 29 GB" | 20260506 00:52:16.888 | INFO | } | 20260506 00:52:16.888 | INFO | Perf report for the last 5.14079 seconds: | 20260506 00:52:16.888 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:52:16.888 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:16.888 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:16.888 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:16.888 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260506 00:52:16.888 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:16.888 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260506 00:52:16.888 | INFO | PRE_HQ : 59.7%, 1.7, 54.0ms, 0 | 20260506 00:52:16.888 | INFO | BAZWRITER : 61.7%, 3.5, 111.2ms, 0 | 20260506 00:52:16.888 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:52:16.888 | INFO | Uploaded 36GiB to GPU in 2.16388s (16.6368GiB/s) | 20260506 00:52:16.888 | INFO | Downloaded 38GiB to GPU in 2.23986s (16.9654GiB/s) | 20260506 00:52:22.032 | INFO | Analyzing chunk frames = [7655936,7656448) | 20260506 00:52:22.032 | INFO | MemInfo: { | 20260506 00:52:22.032 | INFO | "gpu" : " 17 GB", | 20260506 00:52:22.032 | INFO | "huge" : "", | 20260506 00:52:22.032 | INFO | "huge_shared" : " 27 GB", | 20260506 00:52:22.032 | INFO | "malloc" : " 14 GB", | 20260506 00:52:22.032 | INFO | "pinned" : " 29 GB" | 20260506 00:52:22.032 | INFO | } | 20260506 00:52:22.032 | INFO | Perf report for the last 5.14431 seconds: | 20260506 00:52:22.032 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:52:22.032 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:22.032 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:22.032 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:22.032 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 00:52:22.032 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:22.032 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:22.032 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 00:52:22.032 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:22.032 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:22.032 | INFO | Uploaded 36GiB to GPU in 2.09883s (17.1524GiB/s) | 20260506 00:52:22.032 | INFO | Downloaded 34GiB to GPU in 2.03211s (16.7314GiB/s) | 20260506 00:52:27.161 | INFO | Analyzing chunk frames = [7656448,7656960) | 20260506 00:52:27.161 | INFO | MemInfo: { | 20260506 00:52:27.161 | INFO | "gpu" : " 17 GB", | 20260506 00:52:27.161 | INFO | "huge" : "", | 20260506 00:52:27.161 | INFO | "huge_shared" : " 27 GB", | 20260506 00:52:27.161 | INFO | "malloc" : " 14 GB", | 20260506 00:52:27.161 | INFO | "pinned" : " 29 GB" | 20260506 00:52:27.161 | INFO | } | 20260506 00:52:27.161 | INFO | Perf report for the last 5.12915 seconds: | 20260506 00:52:27.161 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:52:27.161 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:27.161 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:27.161 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:27.161 | INFO | ANALYSIS : 61.6%, 3.0, 95.6ms, 0 | 20260506 00:52:27.161 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:27.161 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:27.161 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:52:27.161 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:27.161 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:27.161 | INFO | Uploaded 36GiB to GPU in 2.09132s (17.214GiB/s) | 20260506 00:52:27.161 | INFO | Downloaded 34GiB to GPU in 2.03086s (16.7417GiB/s) | 20260506 00:52:32.307 | INFO | Analyzing chunk frames = [7656960,7657472) | 20260506 00:52:32.307 | INFO | MemInfo: { | 20260506 00:52:32.307 | INFO | "gpu" : " 17 GB", | 20260506 00:52:32.307 | INFO | "huge" : "", | 20260506 00:52:32.307 | INFO | "huge_shared" : " 27 GB", | 20260506 00:52:32.307 | INFO | "malloc" : " 14 GB", | 20260506 00:52:32.307 | INFO | "pinned" : " 29 GB" | 20260506 00:52:32.307 | INFO | } | 20260506 00:52:32.307 | INFO | Perf report for the last 5.14609 seconds: | 20260506 00:52:32.307 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:52:32.307 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:32.307 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:32.307 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:32.307 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 00:52:32.307 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:32.307 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:32.307 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:52:32.307 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:32.307 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:32.308 | INFO | Uploaded 36GiB to GPU in 2.1362s (16.8524GiB/s) | 20260506 00:52:32.308 | INFO | Downloaded 34GiB to GPU in 2.00786s (16.9335GiB/s) | 20260506 00:52:37.452 | INFO | Analyzing chunk frames = [7657472,7657984) | 20260506 00:52:37.452 | INFO | MemInfo: { | 20260506 00:52:37.452 | INFO | "gpu" : " 17 GB", | 20260506 00:52:37.452 | INFO | "huge" : "", | 20260506 00:52:37.452 | INFO | "huge_shared" : " 27 GB", | 20260506 00:52:37.452 | INFO | "malloc" : " 14 GB", | 20260506 00:52:37.452 | INFO | "pinned" : " 29 GB" | 20260506 00:52:37.452 | INFO | } | 20260506 00:52:37.453 | INFO | Perf report for the last 5.14523 seconds: | 20260506 00:52:37.453 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:52:37.453 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:37.453 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:37.453 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:37.453 | INFO | ANALYSIS : 60.7%, 3.0, 94.9ms, 0 | 20260506 00:52:37.453 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:37.453 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:37.453 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:52:37.453 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:37.453 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:37.453 | INFO | Uploaded 36GiB to GPU in 2.06637s (17.4219GiB/s) | 20260506 00:52:37.453 | INFO | Downloaded 34GiB to GPU in 2.02928s (16.7547GiB/s) | 20260506 00:52:42.587 | INFO | Analyzing chunk frames = [7657984,7658496) | 20260506 00:52:42.587 | INFO | MemInfo: { | 20260506 00:52:42.587 | INFO | "gpu" : " 17 GB", | 20260506 00:52:42.587 | INFO | "huge" : "", | 20260506 00:52:42.587 | INFO | "huge_shared" : " 27 GB", | 20260506 00:52:42.587 | INFO | "malloc" : " 14 GB", | 20260506 00:52:42.587 | INFO | "pinned" : " 29 GB" | 20260506 00:52:42.587 | INFO | } | 20260506 00:52:42.587 | INFO | Perf report for the last 5.1346 seconds: | 20260506 00:52:42.587 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:52:42.587 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:42.587 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:42.587 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:42.587 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 00:52:42.587 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:42.587 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:42.587 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 00:52:42.587 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:42.587 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:42.587 | INFO | Uploaded 36GiB to GPU in 2.09462s (17.1869GiB/s) | 20260506 00:52:42.587 | INFO | Downloaded 34GiB to GPU in 2.03604s (16.6991GiB/s) | 20260506 00:52:47.720 | INFO | Analyzing chunk frames = [7658496,7659008) | 20260506 00:52:47.720 | INFO | MemInfo: { | 20260506 00:52:47.720 | INFO | "gpu" : " 17 GB", | 20260506 00:52:47.720 | INFO | "huge" : "", | 20260506 00:52:47.720 | INFO | "huge_shared" : " 27 GB", | 20260506 00:52:47.720 | INFO | "malloc" : " 14 GB", | 20260506 00:52:47.720 | INFO | "pinned" : " 29 GB" | 20260506 00:52:47.720 | INFO | } | 20260506 00:52:47.720 | INFO | Perf report for the last 5.13297 seconds: | 20260506 00:52:47.720 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:52:47.720 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:47.720 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:47.720 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:47.720 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260506 00:52:47.720 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:47.720 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:47.720 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260506 00:52:47.720 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:47.720 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:47.720 | INFO | Uploaded 36GiB to GPU in 2.06826s (17.4059GiB/s) | 20260506 00:52:47.720 | INFO | Downloaded 34GiB to GPU in 2.06724s (16.447GiB/s) | 20260506 00:52:52.864 | INFO | Analyzing chunk frames = [7659008,7659520) | 20260506 00:52:52.864 | INFO | MemInfo: { | 20260506 00:52:52.864 | INFO | "gpu" : " 17 GB", | 20260506 00:52:52.864 | INFO | "huge" : "", | 20260506 00:52:52.864 | INFO | "huge_shared" : " 27 GB", | 20260506 00:52:52.864 | INFO | "malloc" : " 14 GB", | 20260506 00:52:52.864 | INFO | "pinned" : " 29 GB" | 20260506 00:52:52.864 | INFO | } | 20260506 00:52:52.864 | INFO | Perf report for the last 5.14382 seconds: | 20260506 00:52:52.864 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:52:52.864 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:52.864 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:52.864 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:52.864 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260506 00:52:52.864 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:52.864 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:52.864 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:52:52.864 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:52.864 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:52.864 | INFO | Uploaded 36GiB to GPU in 2.12344s (16.9536GiB/s) | 20260506 00:52:52.864 | INFO | Downloaded 34GiB to GPU in 2.0091s (16.923GiB/s) | 20260506 00:52:52.864 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 00:52:52.864 | INFO | Total mem usage: 15.3 GiB | 20260506 00:52:52.864 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 00:52:52.864 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.12%) | 20260506 00:52:52.864 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 00:52:52.864 | INFO | High Water Breakdown: | 20260506 00:52:52.864 | INFO | 60.9% : 8.55 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 00:52:52.864 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 00:52:52.864 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 00:52:52.864 | INFO | ------------------------------------------ | 20260506 00:52:52.865 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:52:52.865 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 00:52:52.865 | INFO | Total mem usage: 26.5 GiB | 20260506 00:52:52.865 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 00:52:52.865 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 00:52:52.865 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260506 00:52:52.865 | INFO | High Water Breakdown: | 20260506 00:52:52.865 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:52:52.865 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:52:52.865 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:52:52.865 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:52:52.865 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:52:52.865 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:52:52.865 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 00:52:52.865 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 00:52:52.865 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:52:52.865 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 00:52:52.865 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 00:52:52.865 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 00:52:52.865 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 00:52:52.865 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 00:52:52.865 | INFO | ------------------------------------------ | 20260506 00:52:52.865 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:52:52.865 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 00:52:52.865 | INFO | Total mem usage: 25.6 GiB | 20260506 00:52:52.865 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 00:52:52.865 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 00:52:52.865 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 00:52:52.865 | INFO | High Water Breakdown: | 20260506 00:52:52.865 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 00:52:52.865 | INFO | ------------------------------------------ | 20260506 00:52:52.865 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:52:52.865 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 00:52:52.865 | INFO | Total mem usage: 15.7 GiB | 20260506 00:52:52.865 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 00:52:52.865 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 00:52:52.865 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 00:52:52.865 | INFO | High Water Breakdown: | 20260506 00:52:52.865 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 00:52:52.865 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 00:52:52.865 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 00:52:52.865 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 00:52:52.865 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 00:52:52.865 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 00:52:52.865 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 00:52:52.865 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 00:52:52.865 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 00:52:52.865 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:52:52.865 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 00:52:52.865 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:52:52.865 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 00:52:52.865 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 00:52:52.865 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 00:52:52.865 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 00:52:52.865 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:52:52.865 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 00:52:52.865 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:52:52.865 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:52:52.865 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:52:52.865 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 00:52:52.865 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 00:52:52.865 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:52:52.865 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 00:52:52.865 | INFO | ------------------------------------------ | 20260506 00:52:52.865 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:52:52.865 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 00:52:52.865 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:52:52.865 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:52:52.865 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.40e+01, 1.36e+01 / 8.84e-01 | 20260506 00:52:52.865 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.03e-01, 1.07e-02 / 3.80e-03 | 20260506 00:52:52.865 | INFO | PerfReport: 11.83%, Baseline -- 3.71e+00 / 4.48e+00, 5.72e-01 / 2.60e-01 | 20260506 00:52:52.865 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.51e+00, 1.93e-01 / 1.07e-01 | 20260506 00:52:52.865 | INFO | PerfReport: 15.97%, DME -- 4.01e+01 / 4.40e+01, 3.93e+00 / 1.07e+00 | 20260506 00:52:52.865 | INFO | PerfReport: 57.46%, FrameLabeling -- 1.80e+01 / 1.96e+01, 1.58e+00 / 1.13e+00 | 20260506 00:52:52.865 | INFO | PerfReport: 7.04%, PulseAccumulating -- 2.21e+00 / 2.72e+00, 4.17e-01 / 2.58e-01 | 20260506 00:52:52.865 | INFO | PerfReport: 3.06%, Metrics -- 9.58e-01 / 1.25e+00, 2.61e-01 / 1.26e-01 | 20260506 00:52:52.865 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.33e-01 / 3.88e-01, 5.77e-02 / 3.39e-02 | 20260506 00:52:52.865 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 00:52:52.865 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:52:52.865 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:52:52.865 | INFO | PerfReport: Full Execution -- 9.82e+01 / -nan, 2.46e+01 / 0.00e+00 | 20260506 00:52:52.865 | INFO | PerfReport: 22.30%, Upload -- 2.19e+01 / 0.00e+00, 7.43e+00 / 0.00e+00 | 20260506 00:52:52.865 | INFO | PerfReport: 21.84%, Download -- 2.14e+01 / 0.00e+00, 7.59e+00 / 0.00e+00 | 20260506 00:52:52.865 | INFO | PerfReport: 31.94%, Compute -- 3.14e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 00:52:52.866 | INFO | PerfReport: 2.11%, LockUpload -- 2.07e+00 / 0.00e+00, 7.90e+00 / 0.00e+00 | 20260506 00:52:52.866 | INFO | PerfReport: 1.26%, LockDownload -- 1.24e+00 / 0.00e+00, 2.86e+00 / 0.00e+00 | 20260506 00:52:52.866 | INFO | PerfReport: 20.53%, LockCompute -- 2.02e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 00:52:52.866 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.91e-02 / 0.00e+00 | 20260506 00:52:52.981 | INFO | Starting a round of Baz IO | 20260506 00:52:56.127 | INFO | Populating RTMetrics report | 20260506 00:52:56.127 | INFO | RTMetrics report populated | 20260506 00:52:56.127 | INFO | Opening tmp RTMetrics json file | 20260506 00:52:56.128 | INFO | Serializing RTMetrics to json | 20260506 00:52:56.129 | INFO | Serialization done, swapping tmp file now... | 20260506 00:52:56.130 | INFO | RTMetrics done | 20260506 00:52:56.202 | INFO | Finished a round of Baz IO: Wrote 1.845 GiB in 3.221 seconds | 20260506 00:52:56.202 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.35% / 0.00% / 78.65% | 20260506 00:52:56.202 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.087591% / 0% / 7.15256e-05% | 20260506 00:52:56.202 | INFO | Baz IO overall: | 20260506 00:52:56.202 | INFO | BytesWritten : 4338572581033 | 20260506 00:52:56.202 | INFO | NumEvents : 2040639684572 | 20260506 00:52:58.006 | INFO | Analyzing chunk frames = [7659520,7660032) | 20260506 00:52:58.006 | INFO | MemInfo: { | 20260506 00:52:58.006 | INFO | "gpu" : " 17 GB", | 20260506 00:52:58.006 | INFO | "huge" : "", | 20260506 00:52:58.006 | INFO | "huge_shared" : " 27 GB", | 20260506 00:52:58.006 | INFO | "malloc" : " 14 GB", | 20260506 00:52:58.006 | INFO | "pinned" : " 29 GB" | 20260506 00:52:58.006 | INFO | } | 20260506 00:52:58.006 | INFO | Perf report for the last 5.14243 seconds: | 20260506 00:52:58.006 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:52:58.006 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:58.006 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:58.007 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:58.007 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260506 00:52:58.007 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:52:58.007 | INFO | RT_METRICS : 43.0%, 1.0, 22.6ms, 0 | 20260506 00:52:58.007 | INFO | PRE_HQ : 59.5%, 1.7, 53.6ms, 0 | 20260506 00:52:58.007 | INFO | BAZWRITER : 61.8%, 3.5, 110.8ms, 0 | 20260506 00:52:58.007 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:52:58.007 | INFO | Uploaded 36GiB to GPU in 2.13344s (16.8742GiB/s) | 20260506 00:52:58.007 | INFO | Downloaded 38GiB to GPU in 2.23679s (16.9886GiB/s) | 20260506 00:53:03.144 | INFO | Analyzing chunk frames = [7660032,7660544) | 20260506 00:53:03.144 | INFO | MemInfo: { | 20260506 00:53:03.144 | INFO | "gpu" : " 17 GB", | 20260506 00:53:03.144 | INFO | "huge" : "", | 20260506 00:53:03.144 | INFO | "huge_shared" : " 27 GB", | 20260506 00:53:03.144 | INFO | "malloc" : " 14 GB", | 20260506 00:53:03.144 | INFO | "pinned" : " 29 GB" | 20260506 00:53:03.144 | INFO | } | 20260506 00:53:03.144 | INFO | Perf report for the last 5.13745 seconds: | 20260506 00:53:03.144 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:53:03.144 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:03.144 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:03.144 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:03.144 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 00:53:03.144 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:03.144 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:03.144 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 00:53:03.144 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:03.144 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:03.144 | INFO | Uploaded 36GiB to GPU in 2.10755s (17.0814GiB/s) | 20260506 00:53:03.144 | INFO | Downloaded 34GiB to GPU in 2.04062s (16.6616GiB/s) | 20260506 00:53:08.289 | INFO | Analyzing chunk frames = [7660544,7661056) | 20260506 00:53:08.289 | INFO | MemInfo: { | 20260506 00:53:08.289 | INFO | "gpu" : " 17 GB", | 20260506 00:53:08.289 | INFO | "huge" : "", | 20260506 00:53:08.289 | INFO | "huge_shared" : " 27 GB", | 20260506 00:53:08.289 | INFO | "malloc" : " 14 GB", | 20260506 00:53:08.290 | INFO | "pinned" : " 29 GB" | 20260506 00:53:08.290 | INFO | } | 20260506 00:53:08.290 | INFO | Perf report for the last 5.14574 seconds: | 20260506 00:53:08.290 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:53:08.290 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:08.290 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:08.290 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:08.290 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 00:53:08.290 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:08.290 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:08.290 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:53:08.290 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:08.290 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:08.290 | INFO | Uploaded 36GiB to GPU in 2.08414s (17.2734GiB/s) | 20260506 00:53:08.290 | INFO | Downloaded 34GiB to GPU in 2.02216s (16.8137GiB/s) | 20260506 00:53:13.420 | INFO | Analyzing chunk frames = [7661056,7661568) | 20260506 00:53:13.420 | INFO | MemInfo: { | 20260506 00:53:13.420 | INFO | "gpu" : " 17 GB", | 20260506 00:53:13.420 | INFO | "huge" : "", | 20260506 00:53:13.420 | INFO | "huge_shared" : " 27 GB", | 20260506 00:53:13.420 | INFO | "malloc" : " 14 GB", | 20260506 00:53:13.420 | INFO | "pinned" : " 29 GB" | 20260506 00:53:13.421 | INFO | } | 20260506 00:53:13.421 | INFO | Perf report for the last 5.13095 seconds: | 20260506 00:53:13.421 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:53:13.421 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:13.421 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:13.421 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:13.421 | INFO | ANALYSIS : 61.6%, 3.0, 95.6ms, 0 | 20260506 00:53:13.421 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:13.421 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:13.421 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:53:13.421 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:13.421 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:13.421 | INFO | Uploaded 36GiB to GPU in 2.14679s (16.7692GiB/s) | 20260506 00:53:13.421 | INFO | Downloaded 34GiB to GPU in 2.05659s (16.5322GiB/s) | 20260506 00:53:18.563 | INFO | Analyzing chunk frames = [7661568,7662080) | 20260506 00:53:18.563 | INFO | MemInfo: { | 20260506 00:53:18.563 | INFO | "gpu" : " 17 GB", | 20260506 00:53:18.563 | INFO | "huge" : "", | 20260506 00:53:18.563 | INFO | "huge_shared" : " 27 GB", | 20260506 00:53:18.563 | INFO | "malloc" : " 14 GB", | 20260506 00:53:18.563 | INFO | "pinned" : " 29 GB" | 20260506 00:53:18.563 | INFO | } | 20260506 00:53:18.563 | INFO | Perf report for the last 5.14288 seconds: | 20260506 00:53:18.563 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:53:18.563 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:18.563 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:18.564 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:18.564 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 00:53:18.564 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:18.564 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:18.564 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:53:18.564 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:18.564 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:18.564 | INFO | Uploaded 36GiB to GPU in 2.08931s (17.2305GiB/s) | 20260506 00:53:18.564 | INFO | Downloaded 34GiB to GPU in 2.0507s (16.5797GiB/s) | 20260506 00:53:23.701 | INFO | Analyzing chunk frames = [7662080,7662592) | 20260506 00:53:23.701 | INFO | MemInfo: { | 20260506 00:53:23.701 | INFO | "gpu" : " 17 GB", | 20260506 00:53:23.701 | INFO | "huge" : "", | 20260506 00:53:23.701 | INFO | "huge_shared" : " 27 GB", | 20260506 00:53:23.701 | INFO | "malloc" : " 14 GB", | 20260506 00:53:23.701 | INFO | "pinned" : " 29 GB" | 20260506 00:53:23.701 | INFO | } | 20260506 00:53:23.701 | INFO | Perf report for the last 5.13765 seconds: | 20260506 00:53:23.701 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:53:23.701 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:23.701 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:23.701 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:23.701 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 00:53:23.701 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:23.701 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:23.701 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 00:53:23.701 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:23.701 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:23.701 | INFO | Uploaded 36GiB to GPU in 2.07197s (17.3747GiB/s) | 20260506 00:53:23.701 | INFO | Downloaded 34GiB to GPU in 2.03455s (16.7113GiB/s) | 20260506 00:53:28.839 | INFO | Analyzing chunk frames = [7662592,7663104) | 20260506 00:53:28.839 | INFO | MemInfo: { | 20260506 00:53:28.839 | INFO | "gpu" : " 17 GB", | 20260506 00:53:28.839 | INFO | "huge" : "", | 20260506 00:53:28.839 | INFO | "huge_shared" : " 27 GB", | 20260506 00:53:28.839 | INFO | "malloc" : " 14 GB", | 20260506 00:53:28.839 | INFO | "pinned" : " 29 GB" | 20260506 00:53:28.839 | INFO | } | 20260506 00:53:28.839 | INFO | Perf report for the last 5.13767 seconds: | 20260506 00:53:28.839 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:53:28.839 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:28.839 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:28.839 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:28.839 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260506 00:53:28.839 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:28.839 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:28.839 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 00:53:28.839 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:28.839 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:28.839 | INFO | Uploaded 36GiB to GPU in 2.05548s (17.5142GiB/s) | 20260506 00:53:28.839 | INFO | Downloaded 34GiB to GPU in 2.06538s (16.4619GiB/s) | 20260506 00:53:33.977 | INFO | Analyzing chunk frames = [7663104,7663616) | 20260506 00:53:33.977 | INFO | MemInfo: { | 20260506 00:53:33.977 | INFO | "gpu" : " 17 GB", | 20260506 00:53:33.977 | INFO | "huge" : "", | 20260506 00:53:33.977 | INFO | "huge_shared" : " 27 GB", | 20260506 00:53:33.977 | INFO | "malloc" : " 14 GB", | 20260506 00:53:33.977 | INFO | "pinned" : " 29 GB" | 20260506 00:53:33.977 | INFO | } | 20260506 00:53:33.977 | INFO | Perf report for the last 5.13868 seconds: | 20260506 00:53:33.977 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:53:33.977 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:33.977 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:33.977 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:33.978 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260506 00:53:33.978 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:33.978 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:33.978 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 00:53:33.978 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:33.978 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:33.978 | INFO | Uploaded 36GiB to GPU in 2.11317s (17.036GiB/s) | 20260506 00:53:33.978 | INFO | Downloaded 34GiB to GPU in 2.00829s (16.9298GiB/s) | 20260506 00:53:34.094 | INFO | Starting a round of Baz IO | 20260506 00:53:37.240 | INFO | Populating RTMetrics report | 20260506 00:53:37.241 | INFO | RTMetrics report populated | 20260506 00:53:37.241 | INFO | Opening tmp RTMetrics json file | 20260506 00:53:37.241 | INFO | Serializing RTMetrics to json | 20260506 00:53:37.244 | INFO | Serialization done, swapping tmp file now... | 20260506 00:53:37.246 | INFO | RTMetrics done | 20260506 00:53:37.321 | INFO | Finished a round of Baz IO: Wrote 1.844 GiB in 3.227 seconds | 20260506 00:53:37.322 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.35% / 0.00% / 78.65% | 20260506 00:53:37.322 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0877261% / 0% / 6.75519e-05% | 20260506 00:53:37.322 | INFO | Baz IO overall: | 20260506 00:53:37.322 | INFO | BytesWritten : 4340552741504 | 20260506 00:53:37.322 | INFO | NumEvents : 2041543853561 | 20260506 00:53:39.121 | INFO | Analyzing chunk frames = [7663616,7664128) | 20260506 00:53:39.121 | INFO | MemInfo: { | 20260506 00:53:39.121 | INFO | "gpu" : " 17 GB", | 20260506 00:53:39.121 | INFO | "huge" : "", | 20260506 00:53:39.121 | INFO | "huge_shared" : " 27 GB", | 20260506 00:53:39.122 | INFO | "malloc" : " 14 GB", | 20260506 00:53:39.122 | INFO | "pinned" : " 29 GB" | 20260506 00:53:39.122 | INFO | } | 20260506 00:53:39.122 | INFO | Perf report for the last 5.14419 seconds: | 20260506 00:53:39.122 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:53:39.122 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:39.122 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:39.122 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:39.122 | INFO | ANALYSIS : 62.4%, 3.0, 97.6ms, 0 | 20260506 00:53:39.122 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:39.122 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260506 00:53:39.122 | INFO | PRE_HQ : 59.7%, 1.7, 53.8ms, 0 | 20260506 00:53:39.122 | INFO | BAZWRITER : 61.9%, 3.5, 110.7ms, 0 | 20260506 00:53:39.122 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:53:39.122 | INFO | Uploaded 36GiB to GPU in 2.16527s (16.6261GiB/s) | 20260506 00:53:39.122 | INFO | Downloaded 38GiB to GPU in 2.2431s (16.9409GiB/s) | 20260506 00:53:44.261 | INFO | Analyzing chunk frames = [7664128,7664640) | 20260506 00:53:44.261 | INFO | MemInfo: { | 20260506 00:53:44.261 | INFO | "gpu" : " 17 GB", | 20260506 00:53:44.261 | INFO | "huge" : "", | 20260506 00:53:44.261 | INFO | "huge_shared" : " 27 GB", | 20260506 00:53:44.261 | INFO | "malloc" : " 14 GB", | 20260506 00:53:44.261 | INFO | "pinned" : " 29 GB" | 20260506 00:53:44.261 | INFO | } | 20260506 00:53:44.261 | INFO | Perf report for the last 5.13919 seconds: | 20260506 00:53:44.261 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:53:44.261 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:44.261 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:44.261 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:44.261 | INFO | ANALYSIS : 60.8%, 3.0, 94.9ms, 0 | 20260506 00:53:44.261 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:44.261 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:44.261 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 00:53:44.261 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:44.261 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:44.261 | INFO | Uploaded 36GiB to GPU in 1.94257s (18.5321GiB/s) | 20260506 00:53:44.261 | INFO | Downloaded 34GiB to GPU in 1.97911s (17.1795GiB/s) | 20260506 00:53:49.398 | INFO | Analyzing chunk frames = [7664640,7665152) | 20260506 00:53:49.398 | INFO | MemInfo: { | 20260506 00:53:49.398 | INFO | "gpu" : " 17 GB", | 20260506 00:53:49.398 | INFO | "huge" : "", | 20260506 00:53:49.398 | INFO | "huge_shared" : " 27 GB", | 20260506 00:53:49.398 | INFO | "malloc" : " 14 GB", | 20260506 00:53:49.398 | INFO | "pinned" : " 29 GB" | 20260506 00:53:49.398 | INFO | } | 20260506 00:53:49.398 | INFO | Perf report for the last 5.13695 seconds: | 20260506 00:53:49.398 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:53:49.398 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:49.398 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:49.398 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:49.398 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 00:53:49.398 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:49.398 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:49.398 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:53:49.398 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:49.398 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:49.398 | INFO | Uploaded 36GiB to GPU in 2.07789s (17.3253GiB/s) | 20260506 00:53:49.398 | INFO | Downloaded 34GiB to GPU in 2.02688s (16.7745GiB/s) | 20260506 00:53:54.533 | INFO | Analyzing chunk frames = [7665152,7665664) | 20260506 00:53:54.533 | INFO | MemInfo: { | 20260506 00:53:54.533 | INFO | "gpu" : " 17 GB", | 20260506 00:53:54.533 | INFO | "huge" : "", | 20260506 00:53:54.533 | INFO | "huge_shared" : " 27 GB", | 20260506 00:53:54.533 | INFO | "malloc" : " 14 GB", | 20260506 00:53:54.533 | INFO | "pinned" : " 29 GB" | 20260506 00:53:54.533 | INFO | } | 20260506 00:53:54.533 | INFO | Perf report for the last 5.13513 seconds: | 20260506 00:53:54.533 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:53:54.533 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:54.533 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:54.533 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:54.533 | INFO | ANALYSIS : 61.6%, 3.0, 95.6ms, 0 | 20260506 00:53:54.533 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:54.533 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:54.533 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:53:54.533 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:54.533 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:54.533 | INFO | Uploaded 36GiB to GPU in 2.14186s (16.8078GiB/s) | 20260506 00:53:54.533 | INFO | Downloaded 34GiB to GPU in 2.02359s (16.8018GiB/s) | 20260506 00:53:59.672 | INFO | Analyzing chunk frames = [7665664,7666176) | 20260506 00:53:59.672 | INFO | MemInfo: { | 20260506 00:53:59.672 | INFO | "gpu" : " 17 GB", | 20260506 00:53:59.672 | INFO | "huge" : "", | 20260506 00:53:59.672 | INFO | "huge_shared" : " 27 GB", | 20260506 00:53:59.672 | INFO | "malloc" : " 14 GB", | 20260506 00:53:59.672 | INFO | "pinned" : " 29 GB" | 20260506 00:53:59.673 | INFO | } | 20260506 00:53:59.673 | INFO | Perf report for the last 5.13962 seconds: | 20260506 00:53:59.673 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:53:59.673 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:59.673 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:59.673 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:59.673 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 00:53:59.673 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:59.673 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:59.673 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:53:59.673 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:59.673 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:53:59.673 | INFO | Uploaded 36GiB to GPU in 2.10671s (17.0882GiB/s) | 20260506 00:53:59.673 | INFO | Downloaded 34GiB to GPU in 2.0492s (16.5918GiB/s) | 20260506 00:54:04.812 | INFO | Analyzing chunk frames = [7666176,7666688) | 20260506 00:54:04.812 | INFO | MemInfo: { | 20260506 00:54:04.812 | INFO | "gpu" : " 17 GB", | 20260506 00:54:04.812 | INFO | "huge" : "", | 20260506 00:54:04.812 | INFO | "huge_shared" : " 27 GB", | 20260506 00:54:04.812 | INFO | "malloc" : " 14 GB", | 20260506 00:54:04.812 | INFO | "pinned" : " 29 GB" | 20260506 00:54:04.812 | INFO | } | 20260506 00:54:04.812 | INFO | Perf report for the last 5.13986 seconds: | 20260506 00:54:04.813 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:54:04.813 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:04.813 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:04.813 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:04.813 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260506 00:54:04.813 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:04.813 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:04.813 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:54:04.813 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:04.813 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:04.813 | INFO | Uploaded 36GiB to GPU in 2.07127s (17.3807GiB/s) | 20260506 00:54:04.813 | INFO | Downloaded 34GiB to GPU in 2.04175s (16.6524GiB/s) | 20260506 00:54:09.955 | INFO | Analyzing chunk frames = [7666688,7667200) | 20260506 00:54:09.955 | INFO | MemInfo: { | 20260506 00:54:09.955 | INFO | "gpu" : " 17 GB", | 20260506 00:54:09.955 | INFO | "huge" : "", | 20260506 00:54:09.955 | INFO | "huge_shared" : " 27 GB", | 20260506 00:54:09.955 | INFO | "malloc" : " 14 GB", | 20260506 00:54:09.956 | INFO | "pinned" : " 29 GB" | 20260506 00:54:09.956 | INFO | } | 20260506 00:54:09.956 | INFO | Perf report for the last 5.14305 seconds: | 20260506 00:54:09.956 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:54:09.956 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:09.956 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:09.956 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:09.956 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260506 00:54:09.956 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:09.956 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:09.956 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 00:54:09.956 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:09.956 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:09.956 | INFO | Uploaded 36GiB to GPU in 2.07583s (17.3424GiB/s) | 20260506 00:54:09.956 | INFO | Downloaded 34GiB to GPU in 2.06457s (16.4683GiB/s) | 20260506 00:54:15.094 | INFO | Analyzing chunk frames = [7667200,7667712) | 20260506 00:54:15.094 | INFO | MemInfo: { | 20260506 00:54:15.094 | INFO | "gpu" : " 17 GB", | 20260506 00:54:15.094 | INFO | "huge" : "", | 20260506 00:54:15.094 | INFO | "huge_shared" : " 27 GB", | 20260506 00:54:15.094 | INFO | "malloc" : " 14 GB", | 20260506 00:54:15.094 | INFO | "pinned" : " 29 GB" | 20260506 00:54:15.094 | INFO | } | 20260506 00:54:15.094 | INFO | Perf report for the last 5.13868 seconds: | 20260506 00:54:15.094 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:54:15.094 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:15.094 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:15.094 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:15.094 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260506 00:54:15.094 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:15.094 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:15.094 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:54:15.094 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:15.094 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:15.095 | INFO | Uploaded 36GiB to GPU in 2.10941s (17.0663GiB/s) | 20260506 00:54:15.095 | INFO | Downloaded 34GiB to GPU in 2.0248s (16.7918GiB/s) | 20260506 00:54:15.095 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 00:54:15.095 | INFO | Total mem usage: 15.3 GiB | 20260506 00:54:15.095 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 00:54:15.095 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.15%) | 20260506 00:54:15.095 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 00:54:15.095 | INFO | High Water Breakdown: | 20260506 00:54:15.095 | INFO | 60.9% : 8.55 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 00:54:15.095 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 00:54:15.095 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 00:54:15.095 | INFO | ------------------------------------------ | 20260506 00:54:15.095 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:54:15.095 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 00:54:15.095 | INFO | Total mem usage: 26.5 GiB | 20260506 00:54:15.095 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 00:54:15.095 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 00:54:15.095 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260506 00:54:15.095 | INFO | High Water Breakdown: | 20260506 00:54:15.095 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:54:15.095 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:54:15.095 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:54:15.095 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:54:15.095 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:54:15.095 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:54:15.095 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 00:54:15.095 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 00:54:15.095 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:54:15.095 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 00:54:15.095 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 00:54:15.095 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 00:54:15.095 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 00:54:15.095 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 00:54:15.095 | INFO | ------------------------------------------ | 20260506 00:54:15.095 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:54:15.095 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 00:54:15.095 | INFO | Total mem usage: 25.6 GiB | 20260506 00:54:15.095 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 00:54:15.095 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 00:54:15.095 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 00:54:15.095 | INFO | High Water Breakdown: | 20260506 00:54:15.095 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 00:54:15.095 | INFO | ------------------------------------------ | 20260506 00:54:15.095 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:54:15.095 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 00:54:15.095 | INFO | Total mem usage: 15.7 GiB | 20260506 00:54:15.095 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 00:54:15.095 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 00:54:15.095 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 00:54:15.095 | INFO | High Water Breakdown: | 20260506 00:54:15.095 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 00:54:15.095 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 00:54:15.095 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 00:54:15.095 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 00:54:15.095 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 00:54:15.095 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 00:54:15.095 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 00:54:15.095 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 00:54:15.095 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 00:54:15.095 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:54:15.095 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 00:54:15.095 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:54:15.095 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 00:54:15.095 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 00:54:15.095 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 00:54:15.095 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 00:54:15.095 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:54:15.095 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 00:54:15.095 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:54:15.096 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:54:15.096 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:54:15.096 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 00:54:15.096 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 00:54:15.096 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:54:15.096 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 00:54:15.096 | INFO | ------------------------------------------ | 20260506 00:54:15.096 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:54:15.096 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 00:54:15.096 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:54:15.096 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:54:15.096 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.38e+01, 1.34e+01 / 8.33e-01 | 20260506 00:54:15.096 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.03e-01, 1.09e-02 / 4.60e-03 | 20260506 00:54:15.096 | INFO | PerfReport: 11.84%, Baseline -- 3.71e+00 / 4.50e+00, 5.78e-01 / 2.00e-01 | 20260506 00:54:15.096 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.52e+00, 1.92e-01 / 7.76e-02 | 20260506 00:54:15.096 | INFO | PerfReport: 15.97%, DME -- 4.00e+01 / 4.38e+01, 3.84e+00 / 1.02e+00 | 20260506 00:54:15.096 | INFO | PerfReport: 57.41%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.52e+00 / 1.13e+00 | 20260506 00:54:15.096 | INFO | PerfReport: 7.07%, PulseAccumulating -- 2.21e+00 / 2.75e+00, 4.20e-01 / 2.30e-01 | 20260506 00:54:15.096 | INFO | PerfReport: 3.07%, Metrics -- 9.61e-01 / 1.25e+00, 2.61e-01 / 1.25e-01 | 20260506 00:54:15.096 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.32e-01 / 3.90e-01, 5.70e-02 / 3.12e-02 | 20260506 00:54:15.096 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 00:54:15.096 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:54:15.096 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:54:15.096 | INFO | PerfReport: Full Execution -- 9.81e+01 / -nan, 2.44e+01 / 0.00e+00 | 20260506 00:54:15.096 | INFO | PerfReport: 22.23%, Upload -- 2.18e+01 / 0.00e+00, 7.42e+00 / 0.00e+00 | 20260506 00:54:15.096 | INFO | PerfReport: 21.88%, Download -- 2.15e+01 / 0.00e+00, 7.60e+00 / 0.00e+00 | 20260506 00:54:15.096 | INFO | PerfReport: 31.93%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 00:54:15.096 | INFO | PerfReport: 2.08%, LockUpload -- 2.04e+00 / 0.00e+00, 7.80e+00 / 0.00e+00 | 20260506 00:54:15.096 | INFO | PerfReport: 1.29%, LockDownload -- 1.26e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260506 00:54:15.096 | INFO | PerfReport: 20.58%, LockCompute -- 2.02e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 00:54:15.096 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.70e-02 / 0.00e+00 | 20260506 00:54:15.212 | INFO | Starting a round of Baz IO | 20260506 00:54:18.362 | INFO | Populating RTMetrics report | 20260506 00:54:18.362 | INFO | RTMetrics report populated | 20260506 00:54:18.362 | INFO | Opening tmp RTMetrics json file | 20260506 00:54:18.363 | INFO | Serializing RTMetrics to json | 20260506 00:54:18.364 | INFO | Serialization done, swapping tmp file now... | 20260506 00:54:18.365 | INFO | RTMetrics done | 20260506 00:54:18.441 | INFO | Finished a round of Baz IO: Wrote 1.845 GiB in 3.228 seconds | 20260506 00:54:18.441 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.34% / 0.00% / 78.65% | 20260506 00:54:18.441 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0870903% / 0% / 5.5631e-05% | 20260506 00:54:18.441 | INFO | Baz IO overall: | 20260506 00:54:18.441 | INFO | BytesWritten : 4342533992283 | 20260506 00:54:18.441 | INFO | NumEvents : 2042448954926 | 20260506 00:54:20.236 | INFO | Analyzing chunk frames = [7667712,7668224) | 20260506 00:54:20.236 | INFO | MemInfo: { | 20260506 00:54:20.236 | INFO | "gpu" : " 17 GB", | 20260506 00:54:20.236 | INFO | "huge" : "", | 20260506 00:54:20.236 | INFO | "huge_shared" : " 27 GB", | 20260506 00:54:20.236 | INFO | "malloc" : " 14 GB", | 20260506 00:54:20.236 | INFO | "pinned" : " 29 GB" | 20260506 00:54:20.236 | INFO | } | 20260506 00:54:20.236 | INFO | Perf report for the last 5.14162 seconds: | 20260506 00:54:20.236 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:54:20.236 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:20.236 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:20.236 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:20.236 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260506 00:54:20.236 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:20.236 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260506 00:54:20.236 | INFO | PRE_HQ : 59.6%, 1.7, 53.9ms, 0 | 20260506 00:54:20.236 | INFO | BAZWRITER : 62.0%, 3.5, 111.2ms, 0 | 20260506 00:54:20.236 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:54:20.236 | INFO | Uploaded 36GiB to GPU in 2.13184s (16.8869GiB/s) | 20260506 00:54:20.236 | INFO | Downloaded 38GiB to GPU in 2.23419s (17.0084GiB/s) | 20260506 00:54:25.378 | INFO | Analyzing chunk frames = [7668224,7668736) | 20260506 00:54:25.378 | INFO | MemInfo: { | 20260506 00:54:25.378 | INFO | "gpu" : " 17 GB", | 20260506 00:54:25.378 | INFO | "huge" : "", | 20260506 00:54:25.378 | INFO | "huge_shared" : " 27 GB", | 20260506 00:54:25.378 | INFO | "malloc" : " 14 GB", | 20260506 00:54:25.378 | INFO | "pinned" : " 29 GB" | 20260506 00:54:25.378 | INFO | } | 20260506 00:54:25.378 | INFO | Perf report for the last 5.14227 seconds: | 20260506 00:54:25.378 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:54:25.378 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:25.378 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:25.378 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:25.378 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260506 00:54:25.378 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:25.378 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:25.378 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 00:54:25.378 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:25.378 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:25.378 | INFO | Uploaded 36GiB to GPU in 2.12218s (16.9637GiB/s) | 20260506 00:54:25.378 | INFO | Downloaded 34GiB to GPU in 2.03946s (16.6711GiB/s) | 20260506 00:54:30.512 | INFO | Analyzing chunk frames = [7668736,7669248) | 20260506 00:54:30.512 | INFO | MemInfo: { | 20260506 00:54:30.512 | INFO | "gpu" : " 17 GB", | 20260506 00:54:30.512 | INFO | "huge" : "", | 20260506 00:54:30.513 | INFO | "huge_shared" : " 27 GB", | 20260506 00:54:30.513 | INFO | "malloc" : " 14 GB", | 20260506 00:54:30.513 | INFO | "pinned" : " 29 GB" | 20260506 00:54:30.513 | INFO | } | 20260506 00:54:30.513 | INFO | Perf report for the last 5.13455 seconds: | 20260506 00:54:30.513 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:54:30.513 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:30.513 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:30.513 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:30.513 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 00:54:30.513 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:30.513 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:30.513 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:54:30.513 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:30.513 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:30.513 | INFO | Uploaded 36GiB to GPU in 2.08447s (17.2706GiB/s) | 20260506 00:54:30.513 | INFO | Downloaded 34GiB to GPU in 2.03016s (16.7475GiB/s) | 20260506 00:54:35.651 | INFO | Analyzing chunk frames = [7669248,7669760) | 20260506 00:54:35.651 | INFO | MemInfo: { | 20260506 00:54:35.651 | INFO | "gpu" : " 17 GB", | 20260506 00:54:35.651 | INFO | "huge" : "", | 20260506 00:54:35.651 | INFO | "huge_shared" : " 27 GB", | 20260506 00:54:35.651 | INFO | "malloc" : " 14 GB", | 20260506 00:54:35.651 | INFO | "pinned" : " 29 GB" | 20260506 00:54:35.651 | INFO | } | 20260506 00:54:35.651 | INFO | Perf report for the last 5.1385 seconds: | 20260506 00:54:35.651 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:54:35.651 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:35.651 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:35.651 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:35.651 | INFO | ANALYSIS : 61.6%, 3.0, 95.7ms, 0 | 20260506 00:54:35.651 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:35.651 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:35.651 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 00:54:35.652 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:35.652 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:35.652 | INFO | Uploaded 36GiB to GPU in 2.14653s (16.7713GiB/s) | 20260506 00:54:35.652 | INFO | Downloaded 34GiB to GPU in 2.04605s (16.6174GiB/s) | 20260506 00:54:40.789 | INFO | Analyzing chunk frames = [7669760,7670272) | 20260506 00:54:40.789 | INFO | MemInfo: { | 20260506 00:54:40.789 | INFO | "gpu" : " 17 GB", | 20260506 00:54:40.789 | INFO | "huge" : "", | 20260506 00:54:40.789 | INFO | "huge_shared" : " 27 GB", | 20260506 00:54:40.789 | INFO | "malloc" : " 14 GB", | 20260506 00:54:40.789 | INFO | "pinned" : " 29 GB" | 20260506 00:54:40.789 | INFO | } | 20260506 00:54:40.789 | INFO | Perf report for the last 5.138 seconds: | 20260506 00:54:40.789 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:54:40.789 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:40.789 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:40.789 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:40.789 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 00:54:40.789 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:40.789 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:40.789 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 00:54:40.789 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:40.789 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:40.790 | INFO | Uploaded 36GiB to GPU in 2.10159s (17.1299GiB/s) | 20260506 00:54:40.790 | INFO | Downloaded 34GiB to GPU in 2.04432s (16.6314GiB/s) | 20260506 00:54:45.930 | INFO | Analyzing chunk frames = [7670272,7670784) | 20260506 00:54:45.930 | INFO | MemInfo: { | 20260506 00:54:45.930 | INFO | "gpu" : " 17 GB", | 20260506 00:54:45.930 | INFO | "huge" : "", | 20260506 00:54:45.930 | INFO | "huge_shared" : " 27 GB", | 20260506 00:54:45.931 | INFO | "malloc" : " 14 GB", | 20260506 00:54:45.931 | INFO | "pinned" : " 29 GB" | 20260506 00:54:45.931 | INFO | } | 20260506 00:54:45.931 | INFO | Perf report for the last 5.14139 seconds: | 20260506 00:54:45.931 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:54:45.931 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:45.931 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:45.931 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:45.931 | INFO | ANALYSIS : 61.2%, 3.0, 95.6ms, 0 | 20260506 00:54:45.931 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:45.931 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:45.931 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:54:45.931 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:45.931 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:45.931 | INFO | Uploaded 36GiB to GPU in 2.08615s (17.2567GiB/s) | 20260506 00:54:45.931 | INFO | Downloaded 34GiB to GPU in 2.04349s (16.6382GiB/s) | 20260506 00:54:51.076 | INFO | Analyzing chunk frames = [7670784,7671296) | 20260506 00:54:51.076 | INFO | MemInfo: { | 20260506 00:54:51.076 | INFO | "gpu" : " 17 GB", | 20260506 00:54:51.076 | INFO | "huge" : "", | 20260506 00:54:51.076 | INFO | "huge_shared" : " 27 GB", | 20260506 00:54:51.076 | INFO | "malloc" : " 14 GB", | 20260506 00:54:51.076 | INFO | "pinned" : " 29 GB" | 20260506 00:54:51.076 | INFO | } | 20260506 00:54:51.076 | INFO | Perf report for the last 5.14556 seconds: | 20260506 00:54:51.076 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:54:51.076 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:51.076 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:51.076 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:51.076 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260506 00:54:51.076 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:51.076 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:51.076 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 00:54:51.076 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:51.076 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:51.077 | INFO | Uploaded 36GiB to GPU in 2.0639s (17.4427GiB/s) | 20260506 00:54:51.077 | INFO | Downloaded 34GiB to GPU in 2.06638s (16.4539GiB/s) | 20260506 00:54:56.212 | INFO | Analyzing chunk frames = [7671296,7671808) | 20260506 00:54:56.212 | INFO | MemInfo: { | 20260506 00:54:56.212 | INFO | "gpu" : " 17 GB", | 20260506 00:54:56.212 | INFO | "huge" : "", | 20260506 00:54:56.212 | INFO | "huge_shared" : " 27 GB", | 20260506 00:54:56.212 | INFO | "malloc" : " 14 GB", | 20260506 00:54:56.213 | INFO | "pinned" : " 29 GB" | 20260506 00:54:56.213 | INFO | } | 20260506 00:54:56.213 | INFO | Perf report for the last 5.13638 seconds: | 20260506 00:54:56.213 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:54:56.213 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:56.213 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:56.213 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:56.213 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260506 00:54:56.213 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:56.213 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:56.213 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 00:54:56.213 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:56.213 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:54:56.213 | INFO | Uploaded 36GiB to GPU in 2.13581s (16.8554GiB/s) | 20260506 00:54:56.213 | INFO | Downloaded 34GiB to GPU in 2.02856s (16.7606GiB/s) | 20260506 00:54:56.339 | INFO | Starting a round of Baz IO | 20260506 00:54:59.483 | INFO | Populating RTMetrics report | 20260506 00:54:59.484 | INFO | RTMetrics report populated | 20260506 00:54:59.484 | INFO | Opening tmp RTMetrics json file | 20260506 00:54:59.484 | INFO | Serializing RTMetrics to json | 20260506 00:54:59.485 | INFO | Serialization done, swapping tmp file now... | 20260506 00:54:59.486 | INFO | RTMetrics done | 20260506 00:54:59.558 | INFO | Finished a round of Baz IO: Wrote 1.845 GiB in 3.218 seconds | 20260506 00:54:59.558 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.34% / 0.00% / 78.66% | 20260506 00:54:59.558 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0869195% / 3.97364e-06% / 7.54992e-05% | 20260506 00:54:59.558 | INFO | Baz IO overall: | 20260506 00:54:59.558 | INFO | BytesWritten : 4344514673795 | 20260506 00:54:59.558 | INFO | NumEvents : 2043353476111 | 20260506 00:55:01.344 | INFO | Analyzing chunk frames = [7671808,7672320) | 20260506 00:55:01.344 | INFO | MemInfo: { | 20260506 00:55:01.344 | INFO | "gpu" : " 17 GB", | 20260506 00:55:01.344 | INFO | "huge" : "", | 20260506 00:55:01.344 | INFO | "huge_shared" : " 27 GB", | 20260506 00:55:01.344 | INFO | "malloc" : " 14 GB", | 20260506 00:55:01.344 | INFO | "pinned" : " 29 GB" | 20260506 00:55:01.345 | INFO | } | 20260506 00:55:01.345 | INFO | Perf report for the last 5.13194 seconds: | 20260506 00:55:01.345 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:55:01.345 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:01.345 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:01.345 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:01.345 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260506 00:55:01.345 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:01.345 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260506 00:55:01.345 | INFO | PRE_HQ : 59.6%, 1.7, 53.9ms, 0 | 20260506 00:55:01.345 | INFO | BAZWRITER : 62.0%, 3.5, 110.8ms, 0 | 20260506 00:55:01.345 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:55:01.345 | INFO | Uploaded 36GiB to GPU in 2.16876s (16.5994GiB/s) | 20260506 00:55:01.345 | INFO | Downloaded 38GiB to GPU in 2.24731s (16.9091GiB/s) | 20260506 00:55:06.487 | INFO | Analyzing chunk frames = [7672320,7672832) | 20260506 00:55:06.487 | INFO | MemInfo: { | 20260506 00:55:06.487 | INFO | "gpu" : " 17 GB", | 20260506 00:55:06.487 | INFO | "huge" : "", | 20260506 00:55:06.487 | INFO | "huge_shared" : " 27 GB", | 20260506 00:55:06.487 | INFO | "malloc" : " 14 GB", | 20260506 00:55:06.487 | INFO | "pinned" : " 29 GB" | 20260506 00:55:06.487 | INFO | } | 20260506 00:55:06.487 | INFO | Perf report for the last 5.14234 seconds: | 20260506 00:55:06.487 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:55:06.487 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:06.487 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:06.487 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:06.487 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 00:55:06.487 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:06.487 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:06.487 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:55:06.487 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:06.487 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:06.487 | INFO | Uploaded 36GiB to GPU in 2.09604s (17.1752GiB/s) | 20260506 00:55:06.487 | INFO | Downloaded 34GiB to GPU in 2.03778s (16.6848GiB/s) | 20260506 00:55:11.628 | INFO | Analyzing chunk frames = [7672832,7673344) | 20260506 00:55:11.628 | INFO | MemInfo: { | 20260506 00:55:11.628 | INFO | "gpu" : " 17 GB", | 20260506 00:55:11.628 | INFO | "huge" : "", | 20260506 00:55:11.628 | INFO | "huge_shared" : " 27 GB", | 20260506 00:55:11.628 | INFO | "malloc" : " 14 GB", | 20260506 00:55:11.628 | INFO | "pinned" : " 29 GB" | 20260506 00:55:11.628 | INFO | } | 20260506 00:55:11.628 | INFO | Perf report for the last 5.14127 seconds: | 20260506 00:55:11.628 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:55:11.628 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:11.628 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:11.628 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:11.628 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 00:55:11.628 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:11.628 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:11.628 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:55:11.628 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:11.628 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:11.629 | INFO | Uploaded 36GiB to GPU in 2.07812s (17.3234GiB/s) | 20260506 00:55:11.629 | INFO | Downloaded 34GiB to GPU in 2.0191s (16.8392GiB/s) | 20260506 00:55:16.767 | INFO | Analyzing chunk frames = [7673344,7673856) | 20260506 00:55:16.767 | INFO | MemInfo: { | 20260506 00:55:16.767 | INFO | "gpu" : " 17 GB", | 20260506 00:55:16.767 | INFO | "huge" : "", | 20260506 00:55:16.767 | INFO | "huge_shared" : " 27 GB", | 20260506 00:55:16.767 | INFO | "malloc" : " 14 GB", | 20260506 00:55:16.767 | INFO | "pinned" : " 29 GB" | 20260506 00:55:16.767 | INFO | } | 20260506 00:55:16.767 | INFO | Perf report for the last 5.13897 seconds: | 20260506 00:55:16.767 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:55:16.767 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:16.767 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:16.767 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:16.767 | INFO | ANALYSIS : 61.3%, 3.0, 95.2ms, 0 | 20260506 00:55:16.767 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:16.767 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:16.767 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 00:55:16.767 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:16.767 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:16.767 | INFO | Uploaded 36GiB to GPU in 2.09004s (17.2246GiB/s) | 20260506 00:55:16.768 | INFO | Downloaded 34GiB to GPU in 2.03077s (16.7424GiB/s) | 20260506 00:55:21.910 | INFO | Analyzing chunk frames = [7673856,7674368) | 20260506 00:55:21.910 | INFO | MemInfo: { | 20260506 00:55:21.910 | INFO | "gpu" : " 17 GB", | 20260506 00:55:21.910 | INFO | "huge" : "", | 20260506 00:55:21.910 | INFO | "huge_shared" : " 27 GB", | 20260506 00:55:21.910 | INFO | "malloc" : " 14 GB", | 20260506 00:55:21.910 | INFO | "pinned" : " 29 GB" | 20260506 00:55:21.910 | INFO | } | 20260506 00:55:21.910 | INFO | Perf report for the last 5.14266 seconds: | 20260506 00:55:21.910 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:55:21.910 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:21.910 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:21.910 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:21.910 | INFO | ANALYSIS : 60.9%, 3.0, 95.2ms, 0 | 20260506 00:55:21.910 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:21.910 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:21.910 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:55:21.910 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:21.910 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:21.910 | INFO | Uploaded 36GiB to GPU in 2.08645s (17.2542GiB/s) | 20260506 00:55:21.910 | INFO | Downloaded 34GiB to GPU in 2.04648s (16.6139GiB/s) | 20260506 00:55:27.046 | INFO | Analyzing chunk frames = [7674368,7674880) | 20260506 00:55:27.047 | INFO | MemInfo: { | 20260506 00:55:27.047 | INFO | "gpu" : " 17 GB", | 20260506 00:55:27.047 | INFO | "huge" : "", | 20260506 00:55:27.047 | INFO | "huge_shared" : " 27 GB", | 20260506 00:55:27.047 | INFO | "malloc" : " 14 GB", | 20260506 00:55:27.047 | INFO | "pinned" : " 29 GB" | 20260506 00:55:27.047 | INFO | } | 20260506 00:55:27.047 | INFO | Perf report for the last 5.13674 seconds: | 20260506 00:55:27.047 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:55:27.047 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:27.047 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:27.047 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:27.047 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 00:55:27.047 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:27.047 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:27.047 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:55:27.047 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:27.047 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:27.047 | INFO | Uploaded 36GiB to GPU in 2.06857s (17.4034GiB/s) | 20260506 00:55:27.047 | INFO | Downloaded 34GiB to GPU in 2.03659s (16.6946GiB/s) | 20260506 00:55:32.183 | INFO | Analyzing chunk frames = [7674880,7675392) | 20260506 00:55:32.183 | INFO | MemInfo: { | 20260506 00:55:32.183 | INFO | "gpu" : " 17 GB", | 20260506 00:55:32.183 | INFO | "huge" : "", | 20260506 00:55:32.183 | INFO | "huge_shared" : " 27 GB", | 20260506 00:55:32.183 | INFO | "malloc" : " 14 GB", | 20260506 00:55:32.183 | INFO | "pinned" : " 29 GB" | 20260506 00:55:32.183 | INFO | } | 20260506 00:55:32.183 | INFO | Perf report for the last 5.13635 seconds: | 20260506 00:55:32.183 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:55:32.183 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:32.183 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:32.183 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:32.183 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260506 00:55:32.183 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:32.183 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:32.183 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 00:55:32.183 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:32.183 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:32.183 | INFO | Uploaded 36GiB to GPU in 2.06829s (17.4056GiB/s) | 20260506 00:55:32.184 | INFO | Downloaded 34GiB to GPU in 2.06496s (16.4652GiB/s) | 20260506 00:55:37.330 | INFO | Analyzing chunk frames = [7675392,7675904) | 20260506 00:55:37.330 | INFO | MemInfo: { | 20260506 00:55:37.330 | INFO | "gpu" : " 17 GB", | 20260506 00:55:37.330 | INFO | "huge" : "", | 20260506 00:55:37.330 | INFO | "huge_shared" : " 27 GB", | 20260506 00:55:37.331 | INFO | "malloc" : " 14 GB", | 20260506 00:55:37.331 | INFO | "pinned" : " 29 GB" | 20260506 00:55:37.331 | INFO | } | 20260506 00:55:37.331 | INFO | Perf report for the last 5.14774 seconds: | 20260506 00:55:37.331 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:55:37.331 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:37.331 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:37.331 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:37.331 | INFO | ANALYSIS : 61.9%, 3.0, 96.8ms, 0 | 20260506 00:55:37.331 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:37.331 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:37.331 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 00:55:37.331 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:37.331 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:37.331 | INFO | Uploaded 36GiB to GPU in 2.13508s (16.8612GiB/s) | 20260506 00:55:37.331 | INFO | Downloaded 34GiB to GPU in 2.02832s (16.7626GiB/s) | 20260506 00:55:37.331 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 00:55:37.331 | INFO | Total mem usage: 15.3 GiB | 20260506 00:55:37.331 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 00:55:37.331 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.12%) | 20260506 00:55:37.331 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 00:55:37.331 | INFO | High Water Breakdown: | 20260506 00:55:37.331 | INFO | 60.9% : 8.55 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 00:55:37.331 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 00:55:37.331 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 00:55:37.331 | INFO | ------------------------------------------ | 20260506 00:55:37.331 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:55:37.331 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 00:55:37.331 | INFO | Total mem usage: 26.5 GiB | 20260506 00:55:37.331 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 00:55:37.331 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 00:55:37.331 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260506 00:55:37.331 | INFO | High Water Breakdown: | 20260506 00:55:37.331 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:55:37.331 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:55:37.331 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:55:37.331 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:55:37.331 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:55:37.331 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:55:37.331 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 00:55:37.332 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 00:55:37.332 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:55:37.332 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 00:55:37.332 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 00:55:37.332 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 00:55:37.332 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 00:55:37.332 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 00:55:37.332 | INFO | ------------------------------------------ | 20260506 00:55:37.332 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:55:37.332 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 00:55:37.332 | INFO | Total mem usage: 25.6 GiB | 20260506 00:55:37.332 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 00:55:37.332 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 00:55:37.332 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 00:55:37.332 | INFO | High Water Breakdown: | 20260506 00:55:37.332 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 00:55:37.332 | INFO | ------------------------------------------ | 20260506 00:55:37.332 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:55:37.332 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 00:55:37.332 | INFO | Total mem usage: 15.7 GiB | 20260506 00:55:37.332 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 00:55:37.332 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 00:55:37.332 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 00:55:37.332 | INFO | High Water Breakdown: | 20260506 00:55:37.332 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 00:55:37.332 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 00:55:37.332 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 00:55:37.332 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 00:55:37.332 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 00:55:37.332 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 00:55:37.332 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 00:55:37.332 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 00:55:37.332 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 00:55:37.332 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:55:37.332 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 00:55:37.332 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:55:37.332 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 00:55:37.332 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 00:55:37.332 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 00:55:37.332 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 00:55:37.332 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:55:37.332 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 00:55:37.332 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:55:37.332 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:55:37.332 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:55:37.332 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 00:55:37.332 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 00:55:37.332 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:55:37.332 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 00:55:37.332 | INFO | ------------------------------------------ | 20260506 00:55:37.332 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:55:37.332 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 00:55:37.332 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:55:37.332 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:55:37.332 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.39e+01, 1.35e+01 / 7.22e-01 | 20260506 00:55:37.332 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.02e-01, 1.05e-02 / 3.81e-03 | 20260506 00:55:37.332 | INFO | PerfReport: 11.82%, Baseline -- 3.70e+00 / 4.50e+00, 5.73e-01 / 2.55e-01 | 20260506 00:55:37.332 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.52e+00, 1.88e-01 / 1.10e-01 | 20260506 00:55:37.332 | INFO | PerfReport: 15.99%, DME -- 4.01e+01 / 4.39e+01, 3.81e+00 / 8.66e-01 | 20260506 00:55:37.332 | INFO | PerfReport: 57.43%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.53e+00 / 1.07e+00 | 20260506 00:55:37.332 | INFO | PerfReport: 7.07%, PulseAccumulating -- 2.21e+00 / 2.77e+00, 4.19e-01 / 2.36e-01 | 20260506 00:55:37.332 | INFO | PerfReport: 3.06%, Metrics -- 9.59e-01 / 1.24e+00, 2.61e-01 / 1.21e-01 | 20260506 00:55:37.332 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.32e-01 / 3.90e-01, 5.71e-02 / 3.12e-02 | 20260506 00:55:37.332 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 00:55:37.332 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:55:37.332 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:55:37.332 | INFO | PerfReport: Full Execution -- 9.82e+01 / -nan, 2.45e+01 / 0.00e+00 | 20260506 00:55:37.332 | INFO | PerfReport: 22.32%, Upload -- 2.19e+01 / 0.00e+00, 7.47e+00 / 0.00e+00 | 20260506 00:55:37.332 | INFO | PerfReport: 21.91%, Download -- 2.15e+01 / 0.00e+00, 7.64e+00 / 0.00e+00 | 20260506 00:55:37.332 | INFO | PerfReport: 31.91%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 00:55:37.333 | INFO | PerfReport: 2.06%, LockUpload -- 2.02e+00 / 0.00e+00, 7.77e+00 / 0.00e+00 | 20260506 00:55:37.333 | INFO | PerfReport: 1.29%, LockDownload -- 1.26e+00 / 0.00e+00, 2.85e+00 / 0.00e+00 | 20260506 00:55:37.333 | INFO | PerfReport: 20.50%, LockCompute -- 2.01e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 00:55:37.333 | INFO | PerfReport: 0.01%, Misc -- 1.27e-02 / 0.00e+00, 1.30e-02 / 0.00e+00 | 20260506 00:55:37.456 | INFO | Starting a round of Baz IO | 20260506 00:55:40.583 | INFO | Populating RTMetrics report | 20260506 00:55:40.583 | INFO | RTMetrics report populated | 20260506 00:55:40.583 | INFO | Opening tmp RTMetrics json file | 20260506 00:55:40.584 | INFO | Serializing RTMetrics to json | 20260506 00:55:40.585 | INFO | Serialization done, swapping tmp file now... | 20260506 00:55:40.585 | INFO | RTMetrics done | 20260506 00:55:40.657 | INFO | Finished a round of Baz IO: Wrote 1.845 GiB in 3.2 seconds | 20260506 00:55:40.657 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.34% / 0.00% / 78.66% | 20260506 00:55:40.657 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0869989% / 0% / 8.34465e-05% | 20260506 00:55:40.657 | INFO | Baz IO overall: | 20260506 00:55:40.657 | INFO | BytesWritten : 4346495263276 | 20260506 00:55:40.657 | INFO | NumEvents : 2044258400592 | 20260506 00:55:42.465 | INFO | Analyzing chunk frames = [7675904,7676416) | 20260506 00:55:42.465 | INFO | MemInfo: { | 20260506 00:55:42.465 | INFO | "gpu" : " 17 GB", | 20260506 00:55:42.465 | INFO | "huge" : "", | 20260506 00:55:42.465 | INFO | "huge_shared" : " 27 GB", | 20260506 00:55:42.465 | INFO | "malloc" : " 14 GB", | 20260506 00:55:42.465 | INFO | "pinned" : " 29 GB" | 20260506 00:55:42.465 | INFO | } | 20260506 00:55:42.465 | INFO | Perf report for the last 5.13437 seconds: | 20260506 00:55:42.465 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:55:42.465 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:42.465 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:42.465 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:42.465 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260506 00:55:42.465 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:42.465 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260506 00:55:42.465 | INFO | PRE_HQ : 59.2%, 1.8, 53.9ms, 0 | 20260506 00:55:42.465 | INFO | BAZWRITER : 61.5%, 3.5, 111.1ms, 0 | 20260506 00:55:42.465 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:55:42.466 | INFO | Uploaded 36GiB to GPU in 2.12637s (16.9303GiB/s) | 20260506 00:55:42.466 | INFO | Downloaded 38GiB to GPU in 2.22217s (17.1004GiB/s) | 20260506 00:55:47.597 | INFO | Analyzing chunk frames = [7676416,7676928) | 20260506 00:55:47.597 | INFO | MemInfo: { | 20260506 00:55:47.597 | INFO | "gpu" : " 17 GB", | 20260506 00:55:47.597 | INFO | "huge" : "", | 20260506 00:55:47.597 | INFO | "huge_shared" : " 27 GB", | 20260506 00:55:47.597 | INFO | "malloc" : " 14 GB", | 20260506 00:55:47.597 | INFO | "pinned" : " 29 GB" | 20260506 00:55:47.597 | INFO | } | 20260506 00:55:47.597 | INFO | Perf report for the last 5.13191 seconds: | 20260506 00:55:47.597 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:55:47.597 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:47.597 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:47.597 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:47.597 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 00:55:47.597 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:47.597 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:47.597 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:55:47.597 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:47.597 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:47.597 | INFO | Uploaded 36GiB to GPU in 2.11671s (17.0075GiB/s) | 20260506 00:55:47.597 | INFO | Downloaded 34GiB to GPU in 2.04074s (16.6607GiB/s) | 20260506 00:55:52.737 | INFO | Analyzing chunk frames = [7676928,7677440) | 20260506 00:55:52.737 | INFO | MemInfo: { | 20260506 00:55:52.737 | INFO | "gpu" : " 17 GB", | 20260506 00:55:52.737 | INFO | "huge" : "", | 20260506 00:55:52.737 | INFO | "huge_shared" : " 27 GB", | 20260506 00:55:52.737 | INFO | "malloc" : " 14 GB", | 20260506 00:55:52.737 | INFO | "pinned" : " 29 GB" | 20260506 00:55:52.737 | INFO | } | 20260506 00:55:52.737 | INFO | Perf report for the last 5.13983 seconds: | 20260506 00:55:52.737 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:55:52.737 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:52.737 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:52.737 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:52.737 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260506 00:55:52.737 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:52.737 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:52.737 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 00:55:52.737 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:52.737 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:52.737 | INFO | Uploaded 36GiB to GPU in 2.1544s (16.71GiB/s) | 20260506 00:55:52.737 | INFO | Downloaded 34GiB to GPU in 2.05509s (16.5443GiB/s) | 20260506 00:55:57.884 | INFO | Analyzing chunk frames = [7677440,7677952) | 20260506 00:55:57.884 | INFO | MemInfo: { | 20260506 00:55:57.884 | INFO | "gpu" : " 17 GB", | 20260506 00:55:57.884 | INFO | "huge" : "", | 20260506 00:55:57.884 | INFO | "huge_shared" : " 27 GB", | 20260506 00:55:57.884 | INFO | "malloc" : " 14 GB", | 20260506 00:55:57.884 | INFO | "pinned" : " 29 GB" | 20260506 00:55:57.884 | INFO | } | 20260506 00:55:57.884 | INFO | Perf report for the last 5.14712 seconds: | 20260506 00:55:57.884 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:55:57.884 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:57.884 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:57.884 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:57.884 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 00:55:57.884 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:57.884 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:57.884 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:55:57.884 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:57.884 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:55:57.884 | INFO | Uploaded 36GiB to GPU in 2.1361s (16.8532GiB/s) | 20260506 00:55:57.884 | INFO | Downloaded 34GiB to GPU in 2.01035s (16.9125GiB/s) | 20260506 00:56:03.028 | INFO | Analyzing chunk frames = [7677952,7678464) | 20260506 00:56:03.028 | INFO | MemInfo: { | 20260506 00:56:03.028 | INFO | "gpu" : " 17 GB", | 20260506 00:56:03.028 | INFO | "huge" : "", | 20260506 00:56:03.028 | INFO | "huge_shared" : " 27 GB", | 20260506 00:56:03.028 | INFO | "malloc" : " 14 GB", | 20260506 00:56:03.028 | INFO | "pinned" : " 29 GB" | 20260506 00:56:03.028 | INFO | } | 20260506 00:56:03.028 | INFO | Perf report for the last 5.14389 seconds: | 20260506 00:56:03.028 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:56:03.028 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:03.028 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:03.028 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:03.028 | INFO | ANALYSIS : 60.9%, 3.0, 94.3ms, 0 | 20260506 00:56:03.028 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:03.028 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:03.028 | INFO | PRE_HQ : 26.8%, 1.0, 13.8ms, 0 | 20260506 00:56:03.028 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:03.028 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:03.028 | INFO | Uploaded 36GiB to GPU in 2.107s (17.0859GiB/s) | 20260506 00:56:03.028 | INFO | Downloaded 34GiB to GPU in 2.04028s (16.6644GiB/s) | 20260506 00:56:08.160 | INFO | Analyzing chunk frames = [7678464,7678976) | 20260506 00:56:08.160 | INFO | MemInfo: { | 20260506 00:56:08.160 | INFO | "gpu" : " 17 GB", | 20260506 00:56:08.160 | INFO | "huge" : "", | 20260506 00:56:08.160 | INFO | "huge_shared" : " 27 GB", | 20260506 00:56:08.160 | INFO | "malloc" : " 14 GB", | 20260506 00:56:08.160 | INFO | "pinned" : " 29 GB" | 20260506 00:56:08.161 | INFO | } | 20260506 00:56:08.161 | INFO | Perf report for the last 5.13263 seconds: | 20260506 00:56:08.161 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:56:08.161 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:08.161 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:08.161 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:08.161 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 00:56:08.161 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:08.161 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:08.161 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:56:08.161 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:08.161 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:08.161 | INFO | Uploaded 36GiB to GPU in 2.05751s (17.4969GiB/s) | 20260506 00:56:08.161 | INFO | Downloaded 34GiB to GPU in 2.026s (16.7818GiB/s) | 20260506 00:56:13.295 | INFO | Analyzing chunk frames = [7678976,7679488) | 20260506 00:56:13.295 | INFO | MemInfo: { | 20260506 00:56:13.295 | INFO | "gpu" : " 17 GB", | 20260506 00:56:13.295 | INFO | "huge" : "", | 20260506 00:56:13.295 | INFO | "huge_shared" : " 27 GB", | 20260506 00:56:13.295 | INFO | "malloc" : " 14 GB", | 20260506 00:56:13.295 | INFO | "pinned" : " 29 GB" | 20260506 00:56:13.296 | INFO | } | 20260506 00:56:13.296 | INFO | Perf report for the last 5.13499 seconds: | 20260506 00:56:13.296 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:56:13.296 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:13.296 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:13.296 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:13.296 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260506 00:56:13.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:13.296 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:13.296 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 00:56:13.296 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:13.296 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:13.296 | INFO | Uploaded 36GiB to GPU in 2.07078s (17.3847GiB/s) | 20260506 00:56:13.296 | INFO | Downloaded 34GiB to GPU in 2.07235s (16.4065GiB/s) | 20260506 00:56:18.437 | INFO | Analyzing chunk frames = [7679488,7680000) | 20260506 00:56:18.437 | INFO | MemInfo: { | 20260506 00:56:18.437 | INFO | "gpu" : " 17 GB", | 20260506 00:56:18.437 | INFO | "huge" : "", | 20260506 00:56:18.437 | INFO | "huge_shared" : " 27 GB", | 20260506 00:56:18.437 | INFO | "malloc" : " 14 GB", | 20260506 00:56:18.437 | INFO | "pinned" : " 29 GB" | 20260506 00:56:18.437 | INFO | } | 20260506 00:56:18.437 | INFO | Perf report for the last 5.14157 seconds: | 20260506 00:56:18.437 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:56:18.437 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:18.437 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:18.437 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:18.437 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260506 00:56:18.437 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:18.437 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:18.437 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 00:56:18.437 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:18.437 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:18.437 | INFO | Uploaded 36GiB to GPU in 2.12046s (16.9774GiB/s) | 20260506 00:56:18.437 | INFO | Downloaded 34GiB to GPU in 2.01591s (16.8658GiB/s) | 20260506 00:56:18.562 | INFO | Starting a round of Baz IO | 20260506 00:56:21.705 | INFO | Populating RTMetrics report | 20260506 00:56:21.705 | INFO | RTMetrics report populated | 20260506 00:56:21.705 | INFO | Opening tmp RTMetrics json file | 20260506 00:56:21.705 | INFO | Serializing RTMetrics to json | 20260506 00:56:21.706 | INFO | Serialization done, swapping tmp file now... | 20260506 00:56:21.707 | INFO | RTMetrics done | 20260506 00:56:21.782 | INFO | Finished a round of Baz IO: Wrote 1.845 GiB in 3.22 seconds | 20260506 00:56:21.782 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.34% / 0.00% / 78.66% | 20260506 00:56:21.782 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0867009% / 0% / 6.35783e-05% | 20260506 00:56:21.782 | INFO | Baz IO overall: | 20260506 00:56:21.782 | INFO | BytesWritten : 4348476379496 | 20260506 00:56:21.782 | INFO | NumEvents : 2045163204806 | 20260506 00:56:23.575 | INFO | Analyzing chunk frames = [7680000,7680512) | 20260506 00:56:23.575 | INFO | MemInfo: { | 20260506 00:56:23.575 | INFO | "gpu" : " 17 GB", | 20260506 00:56:23.575 | INFO | "huge" : "", | 20260506 00:56:23.575 | INFO | "huge_shared" : " 27 GB", | 20260506 00:56:23.575 | INFO | "malloc" : " 14 GB", | 20260506 00:56:23.575 | INFO | "pinned" : " 29 GB" | 20260506 00:56:23.575 | INFO | } | 20260506 00:56:23.575 | INFO | Perf report for the last 5.13778 seconds: | 20260506 00:56:23.575 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:56:23.575 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:23.575 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:23.575 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:23.575 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260506 00:56:23.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:23.575 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260506 00:56:23.575 | INFO | PRE_HQ : 59.8%, 1.8, 54.5ms, 0 | 20260506 00:56:23.575 | INFO | BAZWRITER : 61.7%, 3.5, 110.3ms, 0 | 20260506 00:56:23.575 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:56:23.575 | INFO | Uploaded 36GiB to GPU in 2.18355s (16.4869GiB/s) | 20260506 00:56:23.575 | INFO | Downloaded 38GiB to GPU in 2.23995s (16.9647GiB/s) | 20260506 00:56:28.719 | INFO | Analyzing chunk frames = [7680512,7681024) | 20260506 00:56:28.719 | INFO | MemInfo: { | 20260506 00:56:28.719 | INFO | "gpu" : " 17 GB", | 20260506 00:56:28.719 | INFO | "huge" : "", | 20260506 00:56:28.719 | INFO | "huge_shared" : " 27 GB", | 20260506 00:56:28.720 | INFO | "malloc" : " 14 GB", | 20260506 00:56:28.720 | INFO | "pinned" : " 29 GB" | 20260506 00:56:28.720 | INFO | } | 20260506 00:56:28.720 | INFO | Perf report for the last 5.14468 seconds: | 20260506 00:56:28.720 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:56:28.720 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:28.720 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:28.720 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:28.720 | INFO | ANALYSIS : 61.1%, 3.0, 95.5ms, 0 | 20260506 00:56:28.720 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:28.720 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:28.720 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:56:28.720 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:28.720 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:28.720 | INFO | Uploaded 36GiB to GPU in 2.00656s (17.9411GiB/s) | 20260506 00:56:28.720 | INFO | Downloaded 34GiB to GPU in 1.98974s (17.0877GiB/s) | 20260506 00:56:33.861 | INFO | Analyzing chunk frames = [7681024,7681536) | 20260506 00:56:33.861 | INFO | MemInfo: { | 20260506 00:56:33.861 | INFO | "gpu" : " 17 GB", | 20260506 00:56:33.861 | INFO | "huge" : "", | 20260506 00:56:33.861 | INFO | "huge_shared" : " 27 GB", | 20260506 00:56:33.861 | INFO | "malloc" : " 14 GB", | 20260506 00:56:33.861 | INFO | "pinned" : " 29 GB" | 20260506 00:56:33.861 | INFO | } | 20260506 00:56:33.861 | INFO | Perf report for the last 5.14146 seconds: | 20260506 00:56:33.861 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:56:33.861 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:33.861 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:33.861 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:33.861 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 00:56:33.861 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:33.861 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:33.861 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:56:33.861 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:33.861 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:33.861 | INFO | Uploaded 36GiB to GPU in 2.07889s (17.3169GiB/s) | 20260506 00:56:33.861 | INFO | Downloaded 34GiB to GPU in 2.01478s (16.8753GiB/s) | 20260506 00:56:38.996 | INFO | Analyzing chunk frames = [7681536,7682048) | 20260506 00:56:38.996 | INFO | MemInfo: { | 20260506 00:56:38.996 | INFO | "gpu" : " 17 GB", | 20260506 00:56:38.996 | INFO | "huge" : "", | 20260506 00:56:38.996 | INFO | "huge_shared" : " 27 GB", | 20260506 00:56:38.997 | INFO | "malloc" : " 14 GB", | 20260506 00:56:38.997 | INFO | "pinned" : " 29 GB" | 20260506 00:56:38.997 | INFO | } | 20260506 00:56:38.997 | INFO | Perf report for the last 5.13554 seconds: | 20260506 00:56:38.997 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:56:38.997 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:38.997 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:38.997 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:38.997 | INFO | ANALYSIS : 61.7%, 3.0, 95.8ms, 0 | 20260506 00:56:38.997 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:38.997 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:38.997 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 00:56:38.997 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:38.997 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:38.997 | INFO | Uploaded 36GiB to GPU in 2.13451s (16.8657GiB/s) | 20260506 00:56:38.997 | INFO | Downloaded 34GiB to GPU in 2.01497s (16.8737GiB/s) | 20260506 00:56:44.140 | INFO | Analyzing chunk frames = [7682048,7682560) | 20260506 00:56:44.140 | INFO | MemInfo: { | 20260506 00:56:44.140 | INFO | "gpu" : " 17 GB", | 20260506 00:56:44.140 | INFO | "huge" : "", | 20260506 00:56:44.140 | INFO | "huge_shared" : " 27 GB", | 20260506 00:56:44.140 | INFO | "malloc" : " 14 GB", | 20260506 00:56:44.140 | INFO | "pinned" : " 29 GB" | 20260506 00:56:44.140 | INFO | } | 20260506 00:56:44.140 | INFO | Perf report for the last 5.14324 seconds: | 20260506 00:56:44.140 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:56:44.140 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:44.140 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:44.140 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:44.140 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 00:56:44.140 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:44.140 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:44.140 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 00:56:44.140 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:44.140 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:44.140 | INFO | Uploaded 36GiB to GPU in 2.07984s (17.309GiB/s) | 20260506 00:56:44.140 | INFO | Downloaded 34GiB to GPU in 2.04065s (16.6614GiB/s) | 20260506 00:56:49.277 | INFO | Analyzing chunk frames = [7682560,7683072) | 20260506 00:56:49.277 | INFO | MemInfo: { | 20260506 00:56:49.277 | INFO | "gpu" : " 17 GB", | 20260506 00:56:49.277 | INFO | "huge" : "", | 20260506 00:56:49.277 | INFO | "huge_shared" : " 27 GB", | 20260506 00:56:49.277 | INFO | "malloc" : " 14 GB", | 20260506 00:56:49.277 | INFO | "pinned" : " 29 GB" | 20260506 00:56:49.277 | INFO | } | 20260506 00:56:49.277 | INFO | Perf report for the last 5.13736 seconds: | 20260506 00:56:49.277 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:56:49.277 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:49.277 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:49.277 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:49.277 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 00:56:49.277 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:49.277 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:49.277 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:56:49.277 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:49.277 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:49.278 | INFO | Uploaded 36GiB to GPU in 2.07366s (17.3606GiB/s) | 20260506 00:56:49.278 | INFO | Downloaded 34GiB to GPU in 2.03222s (16.7305GiB/s) | 20260506 00:56:54.415 | INFO | Analyzing chunk frames = [7683072,7683584) | 20260506 00:56:54.415 | INFO | MemInfo: { | 20260506 00:56:54.415 | INFO | "gpu" : " 17 GB", | 20260506 00:56:54.415 | INFO | "huge" : "", | 20260506 00:56:54.415 | INFO | "huge_shared" : " 27 GB", | 20260506 00:56:54.415 | INFO | "malloc" : " 14 GB", | 20260506 00:56:54.415 | INFO | "pinned" : " 29 GB" | 20260506 00:56:54.415 | INFO | } | 20260506 00:56:54.415 | INFO | Perf report for the last 5.13749 seconds: | 20260506 00:56:54.415 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:56:54.415 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:54.415 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:54.415 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:54.415 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260506 00:56:54.415 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:54.415 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:54.415 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 00:56:54.415 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:54.415 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:54.415 | INFO | Uploaded 36GiB to GPU in 2.04086s (17.6396GiB/s) | 20260506 00:56:54.415 | INFO | Downloaded 34GiB to GPU in 2.0537s (16.5555GiB/s) | 20260506 00:56:59.545 | INFO | Analyzing chunk frames = [7683584,7684096) | 20260506 00:56:59.545 | INFO | MemInfo: { | 20260506 00:56:59.545 | INFO | "gpu" : " 17 GB", | 20260506 00:56:59.545 | INFO | "huge" : "", | 20260506 00:56:59.545 | INFO | "huge_shared" : " 27 GB", | 20260506 00:56:59.545 | INFO | "malloc" : " 14 GB", | 20260506 00:56:59.546 | INFO | "pinned" : " 29 GB" | 20260506 00:56:59.546 | INFO | } | 20260506 00:56:59.546 | INFO | Perf report for the last 5.13084 seconds: | 20260506 00:56:59.546 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:56:59.546 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:59.546 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:59.546 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:59.546 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 00:56:59.546 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:59.546 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:59.546 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 00:56:59.546 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:59.546 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:56:59.546 | INFO | Uploaded 36GiB to GPU in 2.13928s (16.8281GiB/s) | 20260506 00:56:59.546 | INFO | Downloaded 34GiB to GPU in 2.01967s (16.8344GiB/s) | 20260506 00:56:59.546 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 00:56:59.546 | INFO | Total mem usage: 15.3 GiB | 20260506 00:56:59.546 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 00:56:59.546 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.12%) | 20260506 00:56:59.546 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 00:56:59.546 | INFO | High Water Breakdown: | 20260506 00:56:59.546 | INFO | 60.9% : 8.55 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 00:56:59.546 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 00:56:59.546 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 00:56:59.546 | INFO | ------------------------------------------ | 20260506 00:56:59.546 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:56:59.546 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 00:56:59.546 | INFO | Total mem usage: 26.5 GiB | 20260506 00:56:59.546 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 00:56:59.546 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 00:56:59.546 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 00:56:59.546 | INFO | High Water Breakdown: | 20260506 00:56:59.546 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:56:59.546 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:56:59.546 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:56:59.546 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:56:59.546 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:56:59.546 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:56:59.547 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 00:56:59.547 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 00:56:59.547 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:56:59.547 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 00:56:59.547 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 00:56:59.547 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 00:56:59.547 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 00:56:59.547 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 00:56:59.547 | INFO | ------------------------------------------ | 20260506 00:56:59.547 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:56:59.547 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 00:56:59.547 | INFO | Total mem usage: 25.6 GiB | 20260506 00:56:59.547 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 00:56:59.547 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 00:56:59.547 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 00:56:59.547 | INFO | High Water Breakdown: | 20260506 00:56:59.547 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 00:56:59.547 | INFO | ------------------------------------------ | 20260506 00:56:59.547 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:56:59.547 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 00:56:59.547 | INFO | Total mem usage: 15.7 GiB | 20260506 00:56:59.547 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 00:56:59.547 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 00:56:59.547 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 00:56:59.547 | INFO | High Water Breakdown: | 20260506 00:56:59.547 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 00:56:59.547 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 00:56:59.547 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 00:56:59.547 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 00:56:59.547 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 00:56:59.547 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 00:56:59.547 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 00:56:59.547 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 00:56:59.547 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 00:56:59.547 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:56:59.547 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 00:56:59.547 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:56:59.547 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 00:56:59.547 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 00:56:59.547 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 00:56:59.547 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 00:56:59.547 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:56:59.547 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 00:56:59.547 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:56:59.547 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:56:59.547 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:56:59.547 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 00:56:59.547 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 00:56:59.547 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:56:59.547 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 00:56:59.547 | INFO | ------------------------------------------ | 20260506 00:56:59.547 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:56:59.547 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 00:56:59.547 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:56:59.547 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:56:59.547 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.39e+01, 1.31e+01 / 8.78e-01 | 20260506 00:56:59.547 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.05e-01, 1.05e-02 / 5.38e-03 | 20260506 00:56:59.547 | INFO | PerfReport: 11.85%, Baseline -- 3.71e+00 / 4.47e+00, 5.70e-01 / 2.70e-01 | 20260506 00:56:59.547 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.51e+00, 1.88e-01 / 1.21e-01 | 20260506 00:56:59.547 | INFO | PerfReport: 16.01%, DME -- 4.01e+01 / 4.42e+01, 4.00e+00 / 1.19e+00 | 20260506 00:56:59.547 | INFO | PerfReport: 57.34%, FrameLabeling -- 1.79e+01 / 1.94e+01, 1.47e+00 / 1.07e+00 | 20260506 00:56:59.547 | INFO | PerfReport: 7.09%, PulseAccumulating -- 2.22e+00 / 2.73e+00, 4.24e-01 / 2.63e-01 | 20260506 00:56:59.547 | INFO | PerfReport: 3.07%, Metrics -- 9.60e-01 / 1.24e+00, 2.58e-01 / 1.11e-01 | 20260506 00:56:59.547 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.32e-01 / 3.76e-01, 5.53e-02 / 2.27e-02 | 20260506 00:56:59.547 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 00:56:59.547 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:56:59.547 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:56:59.547 | INFO | PerfReport: Full Execution -- 9.82e+01 / -nan, 2.45e+01 / 0.00e+00 | 20260506 00:56:59.547 | INFO | PerfReport: 22.30%, Upload -- 2.19e+01 / 0.00e+00, 7.49e+00 / 0.00e+00 | 20260506 00:56:59.547 | INFO | PerfReport: 21.81%, Download -- 2.14e+01 / 0.00e+00, 7.54e+00 / 0.00e+00 | 20260506 00:56:59.547 | INFO | PerfReport: 31.87%, Compute -- 3.13e+01 / 0.00e+00, 1.31e+01 / 0.00e+00 | 20260506 00:56:59.547 | INFO | PerfReport: 2.11%, LockUpload -- 2.07e+00 / 0.00e+00, 7.86e+00 / 0.00e+00 | 20260506 00:56:59.547 | INFO | PerfReport: 1.32%, LockDownload -- 1.29e+00 / 0.00e+00, 2.92e+00 / 0.00e+00 | 20260506 00:56:59.547 | INFO | PerfReport: 20.57%, LockCompute -- 2.02e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260506 00:56:59.547 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.98e-02 / 0.00e+00 | 20260506 00:56:59.672 | INFO | Starting a round of Baz IO | 20260506 00:57:02.814 | INFO | Populating RTMetrics report | 20260506 00:57:02.815 | INFO | RTMetrics report populated | 20260506 00:57:02.815 | INFO | Opening tmp RTMetrics json file | 20260506 00:57:02.815 | INFO | Serializing RTMetrics to json | 20260506 00:57:02.817 | INFO | Serialization done, swapping tmp file now... | 20260506 00:57:02.817 | INFO | RTMetrics done | 20260506 00:57:02.896 | INFO | Finished a round of Baz IO: Wrote 1.842 GiB in 3.224 seconds | 20260506 00:57:02.896 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.34% / 0.00% / 78.66% | 20260506 00:57:02.896 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0867844% / 0% / 8.34465e-05% | 20260506 00:57:02.896 | INFO | Baz IO overall: | 20260506 00:57:02.896 | INFO | BytesWritten : 4350454722504 | 20260506 00:57:02.896 | INFO | NumEvents : 2046066766518 | 20260506 00:57:04.690 | INFO | Analyzing chunk frames = [7684096,7684608) | 20260506 00:57:04.690 | INFO | MemInfo: { | 20260506 00:57:04.690 | INFO | "gpu" : " 17 GB", | 20260506 00:57:04.690 | INFO | "huge" : "", | 20260506 00:57:04.690 | INFO | "huge_shared" : " 27 GB", | 20260506 00:57:04.690 | INFO | "malloc" : " 14 GB", | 20260506 00:57:04.690 | INFO | "pinned" : " 29 GB" | 20260506 00:57:04.690 | INFO | } | 20260506 00:57:04.690 | INFO | Perf report for the last 5.1442 seconds: | 20260506 00:57:04.690 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:57:04.690 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:04.690 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:04.690 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:04.690 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260506 00:57:04.690 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:04.690 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260506 00:57:04.690 | INFO | PRE_HQ : 59.8%, 1.8, 54.5ms, 0 | 20260506 00:57:04.690 | INFO | BAZWRITER : 62.0%, 3.5, 110.7ms, 0 | 20260506 00:57:04.690 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:57:04.691 | INFO | Uploaded 36GiB to GPU in 2.16709s (16.6121GiB/s) | 20260506 00:57:04.691 | INFO | Downloaded 38GiB to GPU in 2.23786s (16.9805GiB/s) | 20260506 00:57:09.836 | INFO | Analyzing chunk frames = [7684608,7685120) | 20260506 00:57:09.837 | INFO | MemInfo: { | 20260506 00:57:09.837 | INFO | "gpu" : " 17 GB", | 20260506 00:57:09.837 | INFO | "huge" : "", | 20260506 00:57:09.837 | INFO | "huge_shared" : " 27 GB", | 20260506 00:57:09.837 | INFO | "malloc" : " 14 GB", | 20260506 00:57:09.837 | INFO | "pinned" : " 29 GB" | 20260506 00:57:09.837 | INFO | } | 20260506 00:57:09.837 | INFO | Perf report for the last 5.14692 seconds: | 20260506 00:57:09.837 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:57:09.837 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:09.837 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:09.837 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:09.837 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 00:57:09.837 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:09.837 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:09.837 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:57:09.837 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:09.837 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:09.837 | INFO | Uploaded 36GiB to GPU in 1.98084s (18.1742GiB/s) | 20260506 00:57:09.837 | INFO | Downloaded 34GiB to GPU in 1.99443s (17.0474GiB/s) | 20260506 00:57:14.975 | INFO | Analyzing chunk frames = [7685120,7685632) | 20260506 00:57:14.975 | INFO | MemInfo: { | 20260506 00:57:14.975 | INFO | "gpu" : " 17 GB", | 20260506 00:57:14.975 | INFO | "huge" : "", | 20260506 00:57:14.975 | INFO | "huge_shared" : " 27 GB", | 20260506 00:57:14.975 | INFO | "malloc" : " 14 GB", | 20260506 00:57:14.975 | INFO | "pinned" : " 29 GB" | 20260506 00:57:14.975 | INFO | } | 20260506 00:57:14.975 | INFO | Perf report for the last 5.13848 seconds: | 20260506 00:57:14.975 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:57:14.975 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:14.975 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:14.975 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:14.975 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 00:57:14.975 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:14.975 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:14.975 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:57:14.975 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:14.975 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:14.975 | INFO | Uploaded 36GiB to GPU in 2.07254s (17.37GiB/s) | 20260506 00:57:14.976 | INFO | Downloaded 34GiB to GPU in 2.01926s (16.8379GiB/s) | 20260506 00:57:20.110 | INFO | Analyzing chunk frames = [7685632,7686144) | 20260506 00:57:20.110 | INFO | MemInfo: { | 20260506 00:57:20.110 | INFO | "gpu" : " 17 GB", | 20260506 00:57:20.110 | INFO | "huge" : "", | 20260506 00:57:20.110 | INFO | "huge_shared" : " 27 GB", | 20260506 00:57:20.110 | INFO | "malloc" : " 14 GB", | 20260506 00:57:20.110 | INFO | "pinned" : " 29 GB" | 20260506 00:57:20.110 | INFO | } | 20260506 00:57:20.110 | INFO | Perf report for the last 5.13497 seconds: | 20260506 00:57:20.110 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:57:20.110 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:20.110 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:20.110 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:20.110 | INFO | ANALYSIS : 61.6%, 3.0, 95.5ms, 0 | 20260506 00:57:20.110 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:20.110 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:20.110 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:57:20.110 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:20.110 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:20.110 | INFO | Uploaded 36GiB to GPU in 2.14483s (16.7846GiB/s) | 20260506 00:57:20.111 | INFO | Downloaded 34GiB to GPU in 2.04946s (16.5897GiB/s) | 20260506 00:57:25.244 | INFO | Analyzing chunk frames = [7686144,7686656) | 20260506 00:57:25.244 | INFO | MemInfo: { | 20260506 00:57:25.244 | INFO | "gpu" : " 17 GB", | 20260506 00:57:25.244 | INFO | "huge" : "", | 20260506 00:57:25.244 | INFO | "huge_shared" : " 27 GB", | 20260506 00:57:25.244 | INFO | "malloc" : " 14 GB", | 20260506 00:57:25.244 | INFO | "pinned" : " 29 GB" | 20260506 00:57:25.244 | INFO | } | 20260506 00:57:25.244 | INFO | Perf report for the last 5.13356 seconds: | 20260506 00:57:25.244 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:57:25.244 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:25.244 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:25.244 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:25.244 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 00:57:25.244 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:25.244 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:25.244 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:57:25.244 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:25.244 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:25.244 | INFO | Uploaded 36GiB to GPU in 2.08991s (17.2257GiB/s) | 20260506 00:57:25.244 | INFO | Downloaded 34GiB to GPU in 2.04659s (16.613GiB/s) | 20260506 00:57:30.389 | INFO | Analyzing chunk frames = [7686656,7687168) | 20260506 00:57:30.389 | INFO | MemInfo: { | 20260506 00:57:30.389 | INFO | "gpu" : " 17 GB", | 20260506 00:57:30.389 | INFO | "huge" : "", | 20260506 00:57:30.389 | INFO | "huge_shared" : " 27 GB", | 20260506 00:57:30.389 | INFO | "malloc" : " 14 GB", | 20260506 00:57:30.389 | INFO | "pinned" : " 29 GB" | 20260506 00:57:30.390 | INFO | } | 20260506 00:57:30.390 | INFO | Perf report for the last 5.14575 seconds: | 20260506 00:57:30.390 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:57:30.390 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:30.390 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:30.390 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:30.390 | INFO | ANALYSIS : 61.2%, 3.0, 95.7ms, 0 | 20260506 00:57:30.390 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:30.390 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:30.390 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 00:57:30.390 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:30.390 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:30.390 | INFO | Uploaded 36GiB to GPU in 2.08832s (17.2388GiB/s) | 20260506 00:57:30.390 | INFO | Downloaded 34GiB to GPU in 2.04418s (16.6326GiB/s) | 20260506 00:57:35.527 | INFO | Analyzing chunk frames = [7687168,7687680) | 20260506 00:57:35.527 | INFO | MemInfo: { | 20260506 00:57:35.527 | INFO | "gpu" : " 17 GB", | 20260506 00:57:35.527 | INFO | "huge" : "", | 20260506 00:57:35.527 | INFO | "huge_shared" : " 27 GB", | 20260506 00:57:35.527 | INFO | "malloc" : " 14 GB", | 20260506 00:57:35.527 | INFO | "pinned" : " 29 GB" | 20260506 00:57:35.527 | INFO | } | 20260506 00:57:35.528 | INFO | Perf report for the last 5.13791 seconds: | 20260506 00:57:35.528 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:57:35.528 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:35.528 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:35.528 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:35.528 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260506 00:57:35.528 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:35.528 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:35.528 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 00:57:35.528 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:35.528 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:35.528 | INFO | Uploaded 36GiB to GPU in 2.06844s (17.4044GiB/s) | 20260506 00:57:35.528 | INFO | Downloaded 34GiB to GPU in 2.06876s (16.435GiB/s) | 20260506 00:57:40.668 | INFO | Analyzing chunk frames = [7687680,7688192) | 20260506 00:57:40.668 | INFO | MemInfo: { | 20260506 00:57:40.668 | INFO | "gpu" : " 17 GB", | 20260506 00:57:40.668 | INFO | "huge" : "", | 20260506 00:57:40.668 | INFO | "huge_shared" : " 27 GB", | 20260506 00:57:40.668 | INFO | "malloc" : " 14 GB", | 20260506 00:57:40.668 | INFO | "pinned" : " 29 GB" | 20260506 00:57:40.668 | INFO | } | 20260506 00:57:40.668 | INFO | Perf report for the last 5.1407 seconds: | 20260506 00:57:40.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:57:40.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:40.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:40.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:40.668 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 00:57:40.668 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:40.668 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:40.668 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 00:57:40.668 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:40.668 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:40.668 | INFO | Uploaded 36GiB to GPU in 2.12965s (16.9042GiB/s) | 20260506 00:57:40.668 | INFO | Downloaded 34GiB to GPU in 2.02139s (16.8201GiB/s) | 20260506 00:57:40.786 | INFO | Starting a round of Baz IO | 20260506 00:57:43.945 | INFO | Populating RTMetrics report | 20260506 00:57:43.945 | INFO | RTMetrics report populated | 20260506 00:57:43.945 | INFO | Opening tmp RTMetrics json file | 20260506 00:57:43.946 | INFO | Serializing RTMetrics to json | 20260506 00:57:43.947 | INFO | Serialization done, swapping tmp file now... | 20260506 00:57:43.947 | INFO | RTMetrics done | 20260506 00:57:44.028 | INFO | Finished a round of Baz IO: Wrote 1.843 GiB in 3.241 seconds | 20260506 00:57:44.028 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.34% / 0.00% / 78.66% | 20260506 00:57:44.028 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0855048% / 3.97364e-06% / 7.94729e-05% | 20260506 00:57:44.028 | INFO | Baz IO overall: | 20260506 00:57:44.028 | INFO | BytesWritten : 4352433101383 | 20260506 00:57:44.028 | INFO | NumEvents : 2046970429691 | 20260506 00:57:45.807 | INFO | Analyzing chunk frames = [7688192,7688704) | 20260506 00:57:45.807 | INFO | MemInfo: { | 20260506 00:57:45.807 | INFO | "gpu" : " 17 GB", | 20260506 00:57:45.807 | INFO | "huge" : "", | 20260506 00:57:45.807 | INFO | "huge_shared" : " 27 GB", | 20260506 00:57:45.807 | INFO | "malloc" : " 14 GB", | 20260506 00:57:45.807 | INFO | "pinned" : " 29 GB" | 20260506 00:57:45.807 | INFO | } | 20260506 00:57:45.807 | INFO | Perf report for the last 5.13908 seconds: | 20260506 00:57:45.807 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:57:45.807 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:45.807 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:45.807 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:45.807 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260506 00:57:45.807 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:45.807 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260506 00:57:45.807 | INFO | PRE_HQ : 59.9%, 1.7, 54.0ms, 0 | 20260506 00:57:45.807 | INFO | BAZWRITER : 62.2%, 3.4, 109.8ms, 0 | 20260506 00:57:45.807 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:57:45.808 | INFO | Uploaded 36GiB to GPU in 2.18484s (16.4772GiB/s) | 20260506 00:57:45.808 | INFO | Downloaded 38GiB to GPU in 2.24609s (16.9183GiB/s) | 20260506 00:57:50.949 | INFO | Analyzing chunk frames = [7688704,7689216) | 20260506 00:57:50.949 | INFO | MemInfo: { | 20260506 00:57:50.949 | INFO | "gpu" : " 17 GB", | 20260506 00:57:50.949 | INFO | "huge" : "", | 20260506 00:57:50.950 | INFO | "huge_shared" : " 27 GB", | 20260506 00:57:50.950 | INFO | "malloc" : " 14 GB", | 20260506 00:57:50.950 | INFO | "pinned" : " 29 GB" | 20260506 00:57:50.950 | INFO | } | 20260506 00:57:50.950 | INFO | Perf report for the last 5.14232 seconds: | 20260506 00:57:50.950 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:57:50.950 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:50.950 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:50.950 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:50.950 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260506 00:57:50.950 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:50.950 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:50.950 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:57:50.950 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:50.950 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:50.950 | INFO | Uploaded 36GiB to GPU in 2.11228s (17.0432GiB/s) | 20260506 00:57:50.950 | INFO | Downloaded 34GiB to GPU in 2.04492s (16.6266GiB/s) | 20260506 00:57:56.088 | INFO | Analyzing chunk frames = [7689216,7689728) | 20260506 00:57:56.088 | INFO | MemInfo: { | 20260506 00:57:56.088 | INFO | "gpu" : " 17 GB", | 20260506 00:57:56.088 | INFO | "huge" : "", | 20260506 00:57:56.088 | INFO | "huge_shared" : " 27 GB", | 20260506 00:57:56.088 | INFO | "malloc" : " 14 GB", | 20260506 00:57:56.088 | INFO | "pinned" : " 29 GB" | 20260506 00:57:56.088 | INFO | } | 20260506 00:57:56.088 | INFO | Perf report for the last 5.13825 seconds: | 20260506 00:57:56.088 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:57:56.088 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:56.088 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:56.088 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:56.088 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 00:57:56.088 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:56.088 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:56.088 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:57:56.088 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:56.088 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:57:56.088 | INFO | Uploaded 36GiB to GPU in 2.08811s (17.2404GiB/s) | 20260506 00:57:56.088 | INFO | Downloaded 34GiB to GPU in 2.02543s (16.7866GiB/s) | 20260506 00:58:01.225 | INFO | Analyzing chunk frames = [7689728,7690240) | 20260506 00:58:01.225 | INFO | MemInfo: { | 20260506 00:58:01.225 | INFO | "gpu" : " 17 GB", | 20260506 00:58:01.225 | INFO | "huge" : "", | 20260506 00:58:01.225 | INFO | "huge_shared" : " 27 GB", | 20260506 00:58:01.225 | INFO | "malloc" : " 14 GB", | 20260506 00:58:01.226 | INFO | "pinned" : " 29 GB" | 20260506 00:58:01.226 | INFO | } | 20260506 00:58:01.226 | INFO | Perf report for the last 5.1377 seconds: | 20260506 00:58:01.226 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:58:01.226 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:01.226 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:01.226 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:01.226 | INFO | ANALYSIS : 61.6%, 3.0, 95.7ms, 0 | 20260506 00:58:01.226 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:01.226 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:01.226 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 00:58:01.226 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:01.226 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:01.226 | INFO | Uploaded 36GiB to GPU in 2.14269s (16.8013GiB/s) | 20260506 00:58:01.226 | INFO | Downloaded 34GiB to GPU in 2.03696s (16.6915GiB/s) | 20260506 00:58:06.361 | INFO | Analyzing chunk frames = [7690240,7690752) | 20260506 00:58:06.361 | INFO | MemInfo: { | 20260506 00:58:06.361 | INFO | "gpu" : " 17 GB", | 20260506 00:58:06.361 | INFO | "huge" : "", | 20260506 00:58:06.361 | INFO | "huge_shared" : " 27 GB", | 20260506 00:58:06.361 | INFO | "malloc" : " 14 GB", | 20260506 00:58:06.361 | INFO | "pinned" : " 29 GB" | 20260506 00:58:06.361 | INFO | } | 20260506 00:58:06.361 | INFO | Perf report for the last 5.13584 seconds: | 20260506 00:58:06.361 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:58:06.361 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:06.361 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:06.362 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:06.362 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 00:58:06.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:06.362 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:06.362 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:58:06.362 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:06.362 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:06.362 | INFO | Uploaded 36GiB to GPU in 2.09739s (17.1642GiB/s) | 20260506 00:58:06.362 | INFO | Downloaded 34GiB to GPU in 2.04958s (16.5888GiB/s) | 20260506 00:58:11.498 | INFO | Analyzing chunk frames = [7690752,7691264) | 20260506 00:58:11.498 | INFO | MemInfo: { | 20260506 00:58:11.498 | INFO | "gpu" : " 17 GB", | 20260506 00:58:11.498 | INFO | "huge" : "", | 20260506 00:58:11.498 | INFO | "huge_shared" : " 27 GB", | 20260506 00:58:11.498 | INFO | "malloc" : " 14 GB", | 20260506 00:58:11.498 | INFO | "pinned" : " 29 GB" | 20260506 00:58:11.498 | INFO | } | 20260506 00:58:11.498 | INFO | Perf report for the last 5.13637 seconds: | 20260506 00:58:11.498 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:58:11.498 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:11.498 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:11.498 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:11.498 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260506 00:58:11.498 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:11.498 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:11.498 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 00:58:11.498 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:11.498 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:11.498 | INFO | Uploaded 36GiB to GPU in 2.07047s (17.3874GiB/s) | 20260506 00:58:11.498 | INFO | Downloaded 34GiB to GPU in 2.03322s (16.7222GiB/s) | 20260506 00:58:16.648 | INFO | Analyzing chunk frames = [7691264,7691776) | 20260506 00:58:16.648 | INFO | MemInfo: { | 20260506 00:58:16.648 | INFO | "gpu" : " 17 GB", | 20260506 00:58:16.648 | INFO | "huge" : "", | 20260506 00:58:16.648 | INFO | "huge_shared" : " 27 GB", | 20260506 00:58:16.648 | INFO | "malloc" : " 14 GB", | 20260506 00:58:16.648 | INFO | "pinned" : " 29 GB" | 20260506 00:58:16.648 | INFO | } | 20260506 00:58:16.648 | INFO | Perf report for the last 5.15021 seconds: | 20260506 00:58:16.648 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:58:16.648 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:16.648 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:16.648 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:16.648 | INFO | ANALYSIS : 62.0%, 3.0, 97.0ms, 0 | 20260506 00:58:16.648 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:16.648 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:16.648 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 00:58:16.648 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:16.648 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:16.648 | INFO | Uploaded 36GiB to GPU in 2.02829s (17.7489GiB/s) | 20260506 00:58:16.648 | INFO | Downloaded 34GiB to GPU in 2.03715s (16.69GiB/s) | 20260506 00:58:21.784 | INFO | Analyzing chunk frames = [7691776,7692288) | 20260506 00:58:21.784 | INFO | MemInfo: { | 20260506 00:58:21.784 | INFO | "gpu" : " 17 GB", | 20260506 00:58:21.784 | INFO | "huge" : "", | 20260506 00:58:21.784 | INFO | "huge_shared" : " 27 GB", | 20260506 00:58:21.784 | INFO | "malloc" : " 14 GB", | 20260506 00:58:21.784 | INFO | "pinned" : " 29 GB" | 20260506 00:58:21.784 | INFO | } | 20260506 00:58:21.784 | INFO | Perf report for the last 5.13617 seconds: | 20260506 00:58:21.784 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:58:21.784 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:21.784 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:21.784 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:21.784 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 00:58:21.784 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:21.784 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:21.784 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:58:21.784 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:21.784 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:21.785 | INFO | Uploaded 36GiB to GPU in 2.03076s (17.7273GiB/s) | 20260506 00:58:21.785 | INFO | Downloaded 34GiB to GPU in 2.01835s (16.8454GiB/s) | 20260506 00:58:21.785 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 00:58:21.785 | INFO | Total mem usage: 15.3 GiB | 20260506 00:58:21.785 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 00:58:21.785 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.15%) | 20260506 00:58:21.785 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 00:58:21.785 | INFO | High Water Breakdown: | 20260506 00:58:21.785 | INFO | 60.9% : 8.55 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 00:58:21.785 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 00:58:21.785 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 00:58:21.785 | INFO | ------------------------------------------ | 20260506 00:58:21.785 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:58:21.785 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 00:58:21.785 | INFO | Total mem usage: 26.5 GiB | 20260506 00:58:21.785 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 00:58:21.785 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 00:58:21.785 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 00:58:21.785 | INFO | High Water Breakdown: | 20260506 00:58:21.785 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:58:21.785 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:58:21.785 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:58:21.785 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:58:21.785 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:58:21.785 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:58:21.785 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 00:58:21.785 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 00:58:21.785 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:58:21.785 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 00:58:21.785 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 00:58:21.785 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 00:58:21.785 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 00:58:21.785 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 00:58:21.785 | INFO | ------------------------------------------ | 20260506 00:58:21.785 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:58:21.785 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 00:58:21.785 | INFO | Total mem usage: 25.6 GiB | 20260506 00:58:21.785 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 00:58:21.785 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 00:58:21.785 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 00:58:21.785 | INFO | High Water Breakdown: | 20260506 00:58:21.785 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 00:58:21.785 | INFO | ------------------------------------------ | 20260506 00:58:21.785 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:58:21.785 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 00:58:21.785 | INFO | Total mem usage: 15.7 GiB | 20260506 00:58:21.785 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 00:58:21.785 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 00:58:21.785 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 00:58:21.785 | INFO | High Water Breakdown: | 20260506 00:58:21.785 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 00:58:21.785 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 00:58:21.785 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 00:58:21.785 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 00:58:21.785 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 00:58:21.785 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 00:58:21.785 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 00:58:21.785 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 00:58:21.785 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 00:58:21.785 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:58:21.785 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 00:58:21.785 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:58:21.785 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 00:58:21.785 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 00:58:21.785 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 00:58:21.785 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 00:58:21.785 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:58:21.785 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 00:58:21.785 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:58:21.785 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:58:21.785 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:58:21.785 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 00:58:21.785 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 00:58:21.785 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:58:21.785 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 00:58:21.785 | INFO | ------------------------------------------ | 20260506 00:58:21.785 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:58:21.786 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 00:58:21.786 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:58:21.786 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:58:21.786 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.41e+01, 1.34e+01 / 8.49e-01 | 20260506 00:58:21.786 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.05e-01, 1.04e-02 / 7.01e-03 | 20260506 00:58:21.786 | INFO | PerfReport: 11.83%, Baseline -- 3.70e+00 / 4.51e+00, 5.72e-01 / 1.83e-01 | 20260506 00:58:21.786 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.53e+00, 1.92e-01 / 8.66e-02 | 20260506 00:58:21.786 | INFO | PerfReport: 16.03%, DME -- 4.01e+01 / 4.42e+01, 3.92e+00 / 1.02e+00 | 20260506 00:58:21.786 | INFO | PerfReport: 57.33%, FrameLabeling -- 1.79e+01 / 1.94e+01, 1.45e+00 / 9.47e-01 | 20260506 00:58:21.786 | INFO | PerfReport: 7.10%, PulseAccumulating -- 2.22e+00 / 2.75e+00, 4.24e-01 / 2.00e-01 | 20260506 00:58:21.786 | INFO | PerfReport: 3.07%, Metrics -- 9.59e-01 / 1.25e+00, 2.59e-01 / 1.12e-01 | 20260506 00:58:21.786 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.31e-01 / 3.89e-01, 5.64e-02 / 2.66e-02 | 20260506 00:58:21.786 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 00:58:21.786 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:58:21.786 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:58:21.786 | INFO | PerfReport: Full Execution -- 9.81e+01 / -nan, 2.43e+01 / 0.00e+00 | 20260506 00:58:21.786 | INFO | PerfReport: 22.23%, Upload -- 2.18e+01 / 0.00e+00, 7.36e+00 / 0.00e+00 | 20260506 00:58:21.786 | INFO | PerfReport: 21.88%, Download -- 2.15e+01 / 0.00e+00, 7.61e+00 / 0.00e+00 | 20260506 00:58:21.786 | INFO | PerfReport: 31.89%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 00:58:21.786 | INFO | PerfReport: 2.08%, LockUpload -- 2.04e+00 / 0.00e+00, 7.90e+00 / 0.00e+00 | 20260506 00:58:21.786 | INFO | PerfReport: 1.32%, LockDownload -- 1.29e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260506 00:58:21.786 | INFO | PerfReport: 20.58%, LockCompute -- 2.02e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 00:58:21.786 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.95e-02 / 0.00e+00 | 20260506 00:58:21.910 | INFO | Starting a round of Baz IO | 20260506 00:58:25.047 | INFO | Populating RTMetrics report | 20260506 00:58:25.048 | INFO | RTMetrics report populated | 20260506 00:58:25.048 | INFO | Opening tmp RTMetrics json file | 20260506 00:58:25.048 | INFO | Serializing RTMetrics to json | 20260506 00:58:25.050 | INFO | Serialization done, swapping tmp file now... | 20260506 00:58:25.050 | INFO | RTMetrics done | 20260506 00:58:25.125 | INFO | Finished a round of Baz IO: Wrote 1.841 GiB in 3.214 seconds | 20260506 00:58:25.125 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.34% / 0.00% / 78.66% | 20260506 00:58:25.125 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0860572% / 3.97364e-06% / 7.54992e-05% | 20260506 00:58:25.125 | INFO | Baz IO overall: | 20260506 00:58:25.125 | INFO | BytesWritten : 4354409323362 | 20260506 00:58:25.125 | INFO | NumEvents : 2047872554411 | 20260506 00:58:26.925 | INFO | Analyzing chunk frames = [7692288,7692800) | 20260506 00:58:26.925 | INFO | MemInfo: { | 20260506 00:58:26.925 | INFO | "gpu" : " 17 GB", | 20260506 00:58:26.925 | INFO | "huge" : "", | 20260506 00:58:26.925 | INFO | "huge_shared" : " 27 GB", | 20260506 00:58:26.925 | INFO | "malloc" : " 14 GB", | 20260506 00:58:26.925 | INFO | "pinned" : " 29 GB" | 20260506 00:58:26.925 | INFO | } | 20260506 00:58:26.925 | INFO | Perf report for the last 5.14091 seconds: | 20260506 00:58:26.925 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:58:26.925 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:26.925 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:26.925 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:26.925 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260506 00:58:26.925 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:26.925 | INFO | RT_METRICS : 43.0%, 1.0, 22.6ms, 0 | 20260506 00:58:26.925 | INFO | PRE_HQ : 59.6%, 1.7, 53.9ms, 0 | 20260506 00:58:26.925 | INFO | BAZWRITER : 61.8%, 3.5, 109.9ms, 0 | 20260506 00:58:26.925 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:58:26.926 | INFO | Uploaded 36GiB to GPU in 2.17998s (16.5139GiB/s) | 20260506 00:58:26.926 | INFO | Downloaded 38GiB to GPU in 2.23984s (16.9655GiB/s) | 20260506 00:58:32.060 | INFO | Analyzing chunk frames = [7692800,7693312) | 20260506 00:58:32.060 | INFO | MemInfo: { | 20260506 00:58:32.060 | INFO | "gpu" : " 17 GB", | 20260506 00:58:32.060 | INFO | "huge" : "", | 20260506 00:58:32.060 | INFO | "huge_shared" : " 27 GB", | 20260506 00:58:32.060 | INFO | "malloc" : " 14 GB", | 20260506 00:58:32.060 | INFO | "pinned" : " 29 GB" | 20260506 00:58:32.060 | INFO | } | 20260506 00:58:32.060 | INFO | Perf report for the last 5.13507 seconds: | 20260506 00:58:32.060 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:58:32.060 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:32.060 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:32.060 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:32.060 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 00:58:32.060 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:32.060 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:32.060 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:58:32.060 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:32.060 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:32.061 | INFO | Uploaded 36GiB to GPU in 2.12552s (16.937GiB/s) | 20260506 00:58:32.061 | INFO | Downloaded 34GiB to GPU in 2.04654s (16.6134GiB/s) | 20260506 00:58:37.200 | INFO | Analyzing chunk frames = [7693312,7693824) | 20260506 00:58:37.200 | INFO | MemInfo: { | 20260506 00:58:37.200 | INFO | "gpu" : " 17 GB", | 20260506 00:58:37.200 | INFO | "huge" : "", | 20260506 00:58:37.200 | INFO | "huge_shared" : " 27 GB", | 20260506 00:58:37.200 | INFO | "malloc" : " 14 GB", | 20260506 00:58:37.200 | INFO | "pinned" : " 29 GB" | 20260506 00:58:37.200 | INFO | } | 20260506 00:58:37.200 | INFO | Perf report for the last 5.14016 seconds: | 20260506 00:58:37.200 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:58:37.200 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:37.200 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:37.200 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:37.200 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 00:58:37.201 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:37.201 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:37.201 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:58:37.201 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:37.201 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:37.201 | INFO | Uploaded 36GiB to GPU in 2.09615s (17.1743GiB/s) | 20260506 00:58:37.201 | INFO | Downloaded 34GiB to GPU in 2.03093s (16.7411GiB/s) | 20260506 00:58:42.338 | INFO | Analyzing chunk frames = [7693824,7694336) | 20260506 00:58:42.338 | INFO | MemInfo: { | 20260506 00:58:42.338 | INFO | "gpu" : " 17 GB", | 20260506 00:58:42.338 | INFO | "huge" : "", | 20260506 00:58:42.338 | INFO | "huge_shared" : " 27 GB", | 20260506 00:58:42.338 | INFO | "malloc" : " 14 GB", | 20260506 00:58:42.338 | INFO | "pinned" : " 29 GB" | 20260506 00:58:42.338 | INFO | } | 20260506 00:58:42.338 | INFO | Perf report for the last 5.13801 seconds: | 20260506 00:58:42.338 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:58:42.338 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:42.338 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:42.339 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:42.339 | INFO | ANALYSIS : 61.5%, 3.0, 95.5ms, 0 | 20260506 00:58:42.339 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:42.339 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:42.339 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:58:42.339 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:42.339 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:42.339 | INFO | Uploaded 36GiB to GPU in 2.12577s (16.9351GiB/s) | 20260506 00:58:42.339 | INFO | Downloaded 34GiB to GPU in 2.01458s (16.8769GiB/s) | 20260506 00:58:47.480 | INFO | Analyzing chunk frames = [7694336,7694848) | 20260506 00:58:47.480 | INFO | MemInfo: { | 20260506 00:58:47.480 | INFO | "gpu" : " 17 GB", | 20260506 00:58:47.480 | INFO | "huge" : "", | 20260506 00:58:47.480 | INFO | "huge_shared" : " 27 GB", | 20260506 00:58:47.481 | INFO | "malloc" : " 14 GB", | 20260506 00:58:47.481 | INFO | "pinned" : " 29 GB" | 20260506 00:58:47.481 | INFO | } | 20260506 00:58:47.481 | INFO | Perf report for the last 5.14224 seconds: | 20260506 00:58:47.481 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:58:47.481 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:47.481 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:47.481 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:47.481 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 00:58:47.481 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:47.481 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:47.481 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 00:58:47.481 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:47.481 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:47.481 | INFO | Uploaded 36GiB to GPU in 2.08792s (17.242GiB/s) | 20260506 00:58:47.481 | INFO | Downloaded 34GiB to GPU in 2.04638s (16.6147GiB/s) | 20260506 00:58:52.621 | INFO | Analyzing chunk frames = [7694848,7695360) | 20260506 00:58:52.621 | INFO | MemInfo: { | 20260506 00:58:52.621 | INFO | "gpu" : " 17 GB", | 20260506 00:58:52.621 | INFO | "huge" : "", | 20260506 00:58:52.621 | INFO | "huge_shared" : " 27 GB", | 20260506 00:58:52.621 | INFO | "malloc" : " 14 GB", | 20260506 00:58:52.621 | INFO | "pinned" : " 29 GB" | 20260506 00:58:52.621 | INFO | } | 20260506 00:58:52.621 | INFO | Perf report for the last 5.14008 seconds: | 20260506 00:58:52.621 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:58:52.621 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:52.621 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:52.621 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:52.621 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 00:58:52.621 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:52.621 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:52.621 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 00:58:52.621 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:52.621 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:52.621 | INFO | Uploaded 36GiB to GPU in 2.0672s (17.4148GiB/s) | 20260506 00:58:52.621 | INFO | Downloaded 34GiB to GPU in 2.04015s (16.6654GiB/s) | 20260506 00:58:57.754 | INFO | Analyzing chunk frames = [7695360,7695872) | 20260506 00:58:57.754 | INFO | MemInfo: { | 20260506 00:58:57.754 | INFO | "gpu" : " 17 GB", | 20260506 00:58:57.754 | INFO | "huge" : "", | 20260506 00:58:57.754 | INFO | "huge_shared" : " 27 GB", | 20260506 00:58:57.754 | INFO | "malloc" : " 14 GB", | 20260506 00:58:57.754 | INFO | "pinned" : " 29 GB" | 20260506 00:58:57.754 | INFO | } | 20260506 00:58:57.754 | INFO | Perf report for the last 5.1333 seconds: | 20260506 00:58:57.754 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:58:57.754 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:57.754 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:57.754 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:57.754 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 00:58:57.754 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:57.754 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:57.754 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 00:58:57.754 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:57.754 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:58:57.754 | INFO | Uploaded 36GiB to GPU in 2.06333s (17.4475GiB/s) | 20260506 00:58:57.754 | INFO | Downloaded 34GiB to GPU in 2.07571s (16.3799GiB/s) | 20260506 00:59:02.899 | INFO | Analyzing chunk frames = [7695872,7696384) | 20260506 00:59:02.899 | INFO | MemInfo: { | 20260506 00:59:02.899 | INFO | "gpu" : " 17 GB", | 20260506 00:59:02.899 | INFO | "huge" : "", | 20260506 00:59:02.899 | INFO | "huge_shared" : " 27 GB", | 20260506 00:59:02.899 | INFO | "malloc" : " 14 GB", | 20260506 00:59:02.899 | INFO | "pinned" : " 29 GB" | 20260506 00:59:02.899 | INFO | } | 20260506 00:59:02.899 | INFO | Perf report for the last 5.14501 seconds: | 20260506 00:59:02.899 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:59:02.899 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:02.899 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:02.899 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:02.899 | INFO | ANALYSIS : 61.9%, 3.0, 96.8ms, 0 | 20260506 00:59:02.899 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:02.899 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:02.899 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 00:59:02.899 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:02.899 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:02.899 | INFO | Uploaded 36GiB to GPU in 2.13524s (16.8599GiB/s) | 20260506 00:59:02.899 | INFO | Downloaded 34GiB to GPU in 2.02543s (16.7866GiB/s) | 20260506 00:59:03.030 | INFO | Starting a round of Baz IO | 20260506 00:59:06.160 | INFO | Populating RTMetrics report | 20260506 00:59:06.161 | INFO | RTMetrics report populated | 20260506 00:59:06.161 | INFO | Opening tmp RTMetrics json file | 20260506 00:59:06.161 | INFO | Serializing RTMetrics to json | 20260506 00:59:06.163 | INFO | Serialization done, swapping tmp file now... | 20260506 00:59:06.163 | INFO | RTMetrics done | 20260506 00:59:06.235 | INFO | Finished a round of Baz IO: Wrote 1.841 GiB in 3.205 seconds | 20260506 00:59:06.235 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.34% / 0.00% / 78.66% | 20260506 00:59:06.236 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.085942% / 3.97364e-06% / 7.15256e-05% | 20260506 00:59:06.236 | INFO | Baz IO overall: | 20260506 00:59:06.236 | INFO | BytesWritten : 4356386424633 | 20260506 00:59:06.236 | INFO | NumEvents : 2048775422789 | 20260506 00:59:08.037 | INFO | Analyzing chunk frames = [7696384,7696896) | 20260506 00:59:08.037 | INFO | MemInfo: { | 20260506 00:59:08.037 | INFO | "gpu" : " 17 GB", | 20260506 00:59:08.037 | INFO | "huge" : "", | 20260506 00:59:08.037 | INFO | "huge_shared" : " 27 GB", | 20260506 00:59:08.038 | INFO | "malloc" : " 14 GB", | 20260506 00:59:08.038 | INFO | "pinned" : " 29 GB" | 20260506 00:59:08.038 | INFO | } | 20260506 00:59:08.038 | INFO | Perf report for the last 5.13848 seconds: | 20260506 00:59:08.038 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:59:08.038 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:08.038 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:08.038 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:08.038 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260506 00:59:08.038 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:08.038 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260506 00:59:08.038 | INFO | PRE_HQ : 59.7%, 1.7, 53.9ms, 0 | 20260506 00:59:08.038 | INFO | BAZWRITER : 62.3%, 3.5, 111.4ms, 0 | 20260506 00:59:08.038 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:59:08.038 | INFO | Uploaded 36GiB to GPU in 2.17961s (16.5167GiB/s) | 20260506 00:59:08.038 | INFO | Downloaded 38GiB to GPU in 2.23836s (16.9767GiB/s) | 20260506 00:59:13.174 | INFO | Analyzing chunk frames = [7696896,7697408) | 20260506 00:59:13.174 | INFO | MemInfo: { | 20260506 00:59:13.174 | INFO | "gpu" : " 17 GB", | 20260506 00:59:13.174 | INFO | "huge" : "", | 20260506 00:59:13.174 | INFO | "huge_shared" : " 27 GB", | 20260506 00:59:13.174 | INFO | "malloc" : " 14 GB", | 20260506 00:59:13.174 | INFO | "pinned" : " 29 GB" | 20260506 00:59:13.174 | INFO | } | 20260506 00:59:13.174 | INFO | Perf report for the last 5.13641 seconds: | 20260506 00:59:13.174 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:59:13.174 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:13.174 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:13.174 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:13.174 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 00:59:13.174 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:13.174 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:13.174 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 00:59:13.174 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:13.174 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:13.174 | INFO | Uploaded 36GiB to GPU in 2.11906s (16.9887GiB/s) | 20260506 00:59:13.174 | INFO | Downloaded 34GiB to GPU in 2.03884s (16.6761GiB/s) | 20260506 00:59:18.314 | INFO | Analyzing chunk frames = [7697408,7697920) | 20260506 00:59:18.314 | INFO | MemInfo: { | 20260506 00:59:18.314 | INFO | "gpu" : " 17 GB", | 20260506 00:59:18.314 | INFO | "huge" : "", | 20260506 00:59:18.314 | INFO | "huge_shared" : " 27 GB", | 20260506 00:59:18.314 | INFO | "malloc" : " 14 GB", | 20260506 00:59:18.314 | INFO | "pinned" : " 29 GB" | 20260506 00:59:18.314 | INFO | } | 20260506 00:59:18.314 | INFO | Perf report for the last 5.13983 seconds: | 20260506 00:59:18.314 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:59:18.314 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:18.314 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:18.314 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:18.314 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 00:59:18.314 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:18.314 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:18.314 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 00:59:18.314 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:18.314 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:18.314 | INFO | Uploaded 36GiB to GPU in 2.09753s (17.1631GiB/s) | 20260506 00:59:18.314 | INFO | Downloaded 34GiB to GPU in 2.02797s (16.7656GiB/s) | 20260506 00:59:23.455 | INFO | Analyzing chunk frames = [7697920,7698432) | 20260506 00:59:23.455 | INFO | MemInfo: { | 20260506 00:59:23.455 | INFO | "gpu" : " 17 GB", | 20260506 00:59:23.455 | INFO | "huge" : "", | 20260506 00:59:23.455 | INFO | "huge_shared" : " 27 GB", | 20260506 00:59:23.455 | INFO | "malloc" : " 14 GB", | 20260506 00:59:23.455 | INFO | "pinned" : " 29 GB" | 20260506 00:59:23.455 | INFO | } | 20260506 00:59:23.455 | INFO | Perf report for the last 5.14152 seconds: | 20260506 00:59:23.455 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:59:23.455 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:23.455 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:23.455 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:23.455 | INFO | ANALYSIS : 61.7%, 3.0, 95.8ms, 0 | 20260506 00:59:23.455 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:23.456 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:23.456 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 00:59:23.456 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:23.456 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:23.456 | INFO | Uploaded 36GiB to GPU in 2.14632s (16.7729GiB/s) | 20260506 00:59:23.456 | INFO | Downloaded 34GiB to GPU in 2.021s (16.8233GiB/s) | 20260506 00:59:28.602 | INFO | Analyzing chunk frames = [7698432,7698944) | 20260506 00:59:28.602 | INFO | MemInfo: { | 20260506 00:59:28.602 | INFO | "gpu" : " 17 GB", | 20260506 00:59:28.602 | INFO | "huge" : "", | 20260506 00:59:28.602 | INFO | "huge_shared" : " 27 GB", | 20260506 00:59:28.602 | INFO | "malloc" : " 14 GB", | 20260506 00:59:28.602 | INFO | "pinned" : " 29 GB" | 20260506 00:59:28.602 | INFO | } | 20260506 00:59:28.602 | INFO | Perf report for the last 5.14703 seconds: | 20260506 00:59:28.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:59:28.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:28.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:28.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:28.603 | INFO | ANALYSIS : 60.6%, 3.0, 94.7ms, 0 | 20260506 00:59:28.603 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:28.603 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:28.603 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 00:59:28.603 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:28.603 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:28.603 | INFO | Uploaded 36GiB to GPU in 2.06481s (17.435GiB/s) | 20260506 00:59:28.603 | INFO | Downloaded 34GiB to GPU in 2.02648s (16.7779GiB/s) | 20260506 00:59:33.734 | INFO | Analyzing chunk frames = [7698944,7699456) | 20260506 00:59:33.734 | INFO | MemInfo: { | 20260506 00:59:33.734 | INFO | "gpu" : " 17 GB", | 20260506 00:59:33.734 | INFO | "huge" : "", | 20260506 00:59:33.734 | INFO | "huge_shared" : " 27 GB", | 20260506 00:59:33.734 | INFO | "malloc" : " 14 GB", | 20260506 00:59:33.734 | INFO | "pinned" : " 29 GB" | 20260506 00:59:33.734 | INFO | } | 20260506 00:59:33.734 | INFO | Perf report for the last 5.13146 seconds: | 20260506 00:59:33.734 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:59:33.734 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:33.734 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:33.734 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:33.734 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 00:59:33.734 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:33.734 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:33.734 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:59:33.734 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:33.734 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:33.734 | INFO | Uploaded 36GiB to GPU in 2.10672s (17.0882GiB/s) | 20260506 00:59:33.734 | INFO | Downloaded 34GiB to GPU in 2.03204s (16.7319GiB/s) | 20260506 00:59:38.867 | INFO | Analyzing chunk frames = [7699456,7699968) | 20260506 00:59:38.867 | INFO | MemInfo: { | 20260506 00:59:38.867 | INFO | "gpu" : " 17 GB", | 20260506 00:59:38.867 | INFO | "huge" : "", | 20260506 00:59:38.867 | INFO | "huge_shared" : " 27 GB", | 20260506 00:59:38.867 | INFO | "malloc" : " 14 GB", | 20260506 00:59:38.867 | INFO | "pinned" : " 29 GB" | 20260506 00:59:38.867 | INFO | } | 20260506 00:59:38.867 | INFO | Perf report for the last 5.1335 seconds: | 20260506 00:59:38.867 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:59:38.867 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:38.867 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:38.867 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:38.867 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260506 00:59:38.868 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:38.868 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:38.868 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 00:59:38.868 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:38.868 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:38.868 | INFO | Uploaded 36GiB to GPU in 2.06425s (17.4397GiB/s) | 20260506 00:59:38.868 | INFO | Downloaded 34GiB to GPU in 2.06823s (16.4392GiB/s) | 20260506 00:59:44.008 | INFO | Analyzing chunk frames = [7699968,7700480) | 20260506 00:59:44.008 | INFO | MemInfo: { | 20260506 00:59:44.008 | INFO | "gpu" : " 17 GB", | 20260506 00:59:44.008 | INFO | "huge" : "", | 20260506 00:59:44.008 | INFO | "huge_shared" : " 27 GB", | 20260506 00:59:44.008 | INFO | "malloc" : " 14 GB", | 20260506 00:59:44.008 | INFO | "pinned" : " 29 GB" | 20260506 00:59:44.008 | INFO | } | 20260506 00:59:44.008 | INFO | Perf report for the last 5.14097 seconds: | 20260506 00:59:44.008 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:59:44.008 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:44.008 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:44.008 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:44.008 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260506 00:59:44.009 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:44.009 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:44.009 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 00:59:44.009 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:44.009 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:44.009 | INFO | Uploaded 36GiB to GPU in 2.11418s (17.0279GiB/s) | 20260506 00:59:44.009 | INFO | Downloaded 34GiB to GPU in 2.00422s (16.9642GiB/s) | 20260506 00:59:44.009 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 00:59:44.009 | INFO | Total mem usage: 15.3 GiB | 20260506 00:59:44.009 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 00:59:44.009 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.16%) | 20260506 00:59:44.009 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 00:59:44.009 | INFO | High Water Breakdown: | 20260506 00:59:44.009 | INFO | 60.9% : 8.55 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 00:59:44.009 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 00:59:44.009 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 00:59:44.009 | INFO | ------------------------------------------ | 20260506 00:59:44.009 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:59:44.009 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 00:59:44.009 | INFO | Total mem usage: 26.5 GiB | 20260506 00:59:44.009 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 00:59:44.009 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 00:59:44.009 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260506 00:59:44.009 | INFO | High Water Breakdown: | 20260506 00:59:44.009 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:59:44.009 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:59:44.009 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:59:44.009 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:59:44.009 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:59:44.009 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:59:44.009 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 00:59:44.009 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 00:59:44.009 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:59:44.009 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 00:59:44.009 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 00:59:44.009 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 00:59:44.009 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 00:59:44.009 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 00:59:44.009 | INFO | ------------------------------------------ | 20260506 00:59:44.009 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:59:44.009 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 00:59:44.011 | INFO | Total mem usage: 25.6 GiB | 20260506 00:59:44.011 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 00:59:44.011 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 00:59:44.011 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 00:59:44.011 | INFO | High Water Breakdown: | 20260506 00:59:44.011 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 00:59:44.011 | INFO | ------------------------------------------ | 20260506 00:59:44.011 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:59:44.011 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 00:59:44.011 | INFO | Total mem usage: 15.7 GiB | 20260506 00:59:44.011 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 00:59:44.011 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 00:59:44.011 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 00:59:44.011 | INFO | High Water Breakdown: | 20260506 00:59:44.011 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 00:59:44.011 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 00:59:44.011 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 00:59:44.011 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 00:59:44.011 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 00:59:44.011 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 00:59:44.011 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 00:59:44.011 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 00:59:44.011 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 00:59:44.011 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 00:59:44.011 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 00:59:44.011 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 00:59:44.011 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 00:59:44.011 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 00:59:44.011 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 00:59:44.011 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 00:59:44.011 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 00:59:44.011 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 00:59:44.011 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 00:59:44.011 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 00:59:44.011 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 00:59:44.011 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 00:59:44.011 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 00:59:44.011 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 00:59:44.011 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 00:59:44.011 | INFO | ------------------------------------------ | 20260506 00:59:44.011 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 00:59:44.011 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 00:59:44.011 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:59:44.011 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:59:44.011 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.41e+01, 1.34e+01 / 8.90e-01 | 20260506 00:59:44.011 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.04e-01, 1.09e-02 / 3.07e-03 | 20260506 00:59:44.011 | INFO | PerfReport: 11.82%, Baseline -- 3.70e+00 / 4.47e+00, 5.65e-01 / 2.39e-01 | 20260506 00:59:44.011 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.50e+00, 1.88e-01 / 1.14e-01 | 20260506 00:59:44.011 | INFO | PerfReport: 15.99%, DME -- 4.01e+01 / 4.40e+01, 3.91e+00 / 1.13e+00 | 20260506 00:59:44.011 | INFO | PerfReport: 57.46%, FrameLabeling -- 1.80e+01 / 1.97e+01, 1.56e+00 / 1.24e+00 | 20260506 00:59:44.011 | INFO | PerfReport: 7.05%, PulseAccumulating -- 2.21e+00 / 2.72e+00, 4.19e-01 / 2.71e-01 | 20260506 00:59:44.011 | INFO | PerfReport: 3.05%, Metrics -- 9.56e-01 / 1.23e+00, 2.59e-01 / 7.91e-02 | 20260506 00:59:44.011 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.31e-01 / 3.82e-01, 5.69e-02 / 2.85e-02 | 20260506 00:59:44.011 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 00:59:44.011 | INFO | PerfReport: Gathered 1536 samples. | 20260506 00:59:44.011 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 00:59:44.011 | INFO | PerfReport: Full Execution -- 9.82e+01 / -nan, 2.45e+01 / 0.00e+00 | 20260506 00:59:44.011 | INFO | PerfReport: 22.39%, Upload -- 2.20e+01 / 0.00e+00, 7.55e+00 / 0.00e+00 | 20260506 00:59:44.011 | INFO | PerfReport: 21.86%, Download -- 2.15e+01 / 0.00e+00, 7.57e+00 / 0.00e+00 | 20260506 00:59:44.011 | INFO | PerfReport: 31.90%, Compute -- 3.13e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 00:59:44.011 | INFO | PerfReport: 2.10%, LockUpload -- 2.06e+00 / 0.00e+00, 7.91e+00 / 0.00e+00 | 20260506 00:59:44.011 | INFO | PerfReport: 1.28%, LockDownload -- 1.26e+00 / 0.00e+00, 2.83e+00 / 0.00e+00 | 20260506 00:59:44.011 | INFO | PerfReport: 20.46%, LockCompute -- 2.01e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 00:59:44.011 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.83e-02 / 0.00e+00 | 20260506 00:59:44.131 | INFO | Starting a round of Baz IO | 20260506 00:59:47.274 | INFO | Populating RTMetrics report | 20260506 00:59:47.274 | INFO | RTMetrics report populated | 20260506 00:59:47.274 | INFO | Opening tmp RTMetrics json file | 20260506 00:59:47.275 | INFO | Serializing RTMetrics to json | 20260506 00:59:47.278 | INFO | Serialization done, swapping tmp file now... | 20260506 00:59:47.280 | INFO | RTMetrics done | 20260506 00:59:47.352 | INFO | Finished a round of Baz IO: Wrote 1.839 GiB in 3.221 seconds | 20260506 00:59:47.353 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.34% / 0.00% / 78.66% | 20260506 00:59:47.353 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0851393% / 0% / 6.35783e-05% | 20260506 00:59:47.353 | INFO | Baz IO overall: | 20260506 00:59:47.353 | INFO | BytesWritten : 4358360652679 | 20260506 00:59:47.353 | INFO | NumEvents : 2049676289072 | 20260506 00:59:49.150 | INFO | Analyzing chunk frames = [7700480,7700992) | 20260506 00:59:49.150 | INFO | MemInfo: { | 20260506 00:59:49.150 | INFO | "gpu" : " 17 GB", | 20260506 00:59:49.150 | INFO | "huge" : "", | 20260506 00:59:49.150 | INFO | "huge_shared" : " 27 GB", | 20260506 00:59:49.150 | INFO | "malloc" : " 14 GB", | 20260506 00:59:49.150 | INFO | "pinned" : " 29 GB" | 20260506 00:59:49.150 | INFO | } | 20260506 00:59:49.150 | INFO | Perf report for the last 5.14204 seconds: | 20260506 00:59:49.151 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:59:49.151 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:49.151 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:49.151 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:49.151 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260506 00:59:49.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:49.151 | INFO | RT_METRICS : 43.2%, 1.0, 22.6ms, 0 | 20260506 00:59:49.151 | INFO | PRE_HQ : 59.8%, 1.7, 53.9ms, 0 | 20260506 00:59:49.151 | INFO | BAZWRITER : 61.9%, 3.5, 112.1ms, 0 | 20260506 00:59:49.151 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 00:59:49.151 | INFO | Uploaded 36GiB to GPU in 2.16179s (16.6529GiB/s) | 20260506 00:59:49.151 | INFO | Downloaded 38GiB to GPU in 2.23294s (17.018GiB/s) | 20260506 00:59:54.296 | INFO | Analyzing chunk frames = [7700992,7701504) | 20260506 00:59:54.296 | INFO | MemInfo: { | 20260506 00:59:54.296 | INFO | "gpu" : " 17 GB", | 20260506 00:59:54.296 | INFO | "huge" : "", | 20260506 00:59:54.296 | INFO | "huge_shared" : " 27 GB", | 20260506 00:59:54.296 | INFO | "malloc" : " 14 GB", | 20260506 00:59:54.296 | INFO | "pinned" : " 29 GB" | 20260506 00:59:54.296 | INFO | } | 20260506 00:59:54.296 | INFO | Perf report for the last 5.14586 seconds: | 20260506 00:59:54.296 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:59:54.296 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:54.296 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:54.296 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:54.296 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 00:59:54.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:54.296 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:54.296 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 00:59:54.296 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:54.297 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:54.297 | INFO | Uploaded 36GiB to GPU in 2.10985s (17.0628GiB/s) | 20260506 00:59:54.297 | INFO | Downloaded 34GiB to GPU in 2.04311s (16.6413GiB/s) | 20260506 00:59:59.431 | INFO | Analyzing chunk frames = [7701504,7702016) | 20260506 00:59:59.431 | INFO | MemInfo: { | 20260506 00:59:59.431 | INFO | "gpu" : " 17 GB", | 20260506 00:59:59.431 | INFO | "huge" : "", | 20260506 00:59:59.432 | INFO | "huge_shared" : " 27 GB", | 20260506 00:59:59.432 | INFO | "malloc" : " 14 GB", | 20260506 00:59:59.432 | INFO | "pinned" : " 29 GB" | 20260506 00:59:59.432 | INFO | } | 20260506 00:59:59.432 | INFO | Perf report for the last 5.13541 seconds: | 20260506 00:59:59.432 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 00:59:59.432 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:59.432 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:59.432 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:59.432 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 00:59:59.432 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:59.432 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:59.432 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 00:59:59.432 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:59.432 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 00:59:59.432 | INFO | Uploaded 36GiB to GPU in 2.08189s (17.292GiB/s) | 20260506 00:59:59.432 | INFO | Downloaded 34GiB to GPU in 2.01977s (16.8336GiB/s) | 20260506 01:00:04.567 | INFO | Analyzing chunk frames = [7702016,7702528) | 20260506 01:00:04.567 | INFO | MemInfo: { | 20260506 01:00:04.567 | INFO | "gpu" : " 17 GB", | 20260506 01:00:04.567 | INFO | "huge" : "", | 20260506 01:00:04.567 | INFO | "huge_shared" : " 27 GB", | 20260506 01:00:04.567 | INFO | "malloc" : " 14 GB", | 20260506 01:00:04.567 | INFO | "pinned" : " 29 GB" | 20260506 01:00:04.567 | INFO | } | 20260506 01:00:04.567 | INFO | Perf report for the last 5.13499 seconds: | 20260506 01:00:04.567 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:00:04.567 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:04.567 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:04.567 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:04.567 | INFO | ANALYSIS : 61.6%, 3.0, 95.6ms, 0 | 20260506 01:00:04.567 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:04.567 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:04.567 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 01:00:04.567 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:04.567 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:04.567 | INFO | Uploaded 36GiB to GPU in 2.13868s (16.8328GiB/s) | 20260506 01:00:04.567 | INFO | Downloaded 34GiB to GPU in 2.01679s (16.8585GiB/s) | 20260506 01:00:09.707 | INFO | Analyzing chunk frames = [7702528,7703040) | 20260506 01:00:09.707 | INFO | MemInfo: { | 20260506 01:00:09.707 | INFO | "gpu" : " 17 GB", | 20260506 01:00:09.707 | INFO | "huge" : "", | 20260506 01:00:09.707 | INFO | "huge_shared" : " 27 GB", | 20260506 01:00:09.707 | INFO | "malloc" : " 14 GB", | 20260506 01:00:09.707 | INFO | "pinned" : " 29 GB" | 20260506 01:00:09.707 | INFO | } | 20260506 01:00:09.707 | INFO | Perf report for the last 5.14022 seconds: | 20260506 01:00:09.707 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:00:09.707 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:09.707 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:09.707 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:09.707 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 01:00:09.707 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:09.707 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:09.707 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 01:00:09.707 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:09.707 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:09.707 | INFO | Uploaded 36GiB to GPU in 2.08917s (17.2317GiB/s) | 20260506 01:00:09.707 | INFO | Downloaded 34GiB to GPU in 2.04528s (16.6236GiB/s) | 20260506 01:00:14.850 | INFO | Analyzing chunk frames = [7703040,7703552) | 20260506 01:00:14.850 | INFO | MemInfo: { | 20260506 01:00:14.850 | INFO | "gpu" : " 17 GB", | 20260506 01:00:14.850 | INFO | "huge" : "", | 20260506 01:00:14.850 | INFO | "huge_shared" : " 27 GB", | 20260506 01:00:14.850 | INFO | "malloc" : " 14 GB", | 20260506 01:00:14.850 | INFO | "pinned" : " 29 GB" | 20260506 01:00:14.850 | INFO | } | 20260506 01:00:14.850 | INFO | Perf report for the last 5.143 seconds: | 20260506 01:00:14.850 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:00:14.850 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:14.850 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:14.850 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:14.850 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 01:00:14.850 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:14.850 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:14.850 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:00:14.850 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:14.850 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:14.850 | INFO | Uploaded 36GiB to GPU in 2.06684s (17.4179GiB/s) | 20260506 01:00:14.850 | INFO | Downloaded 34GiB to GPU in 2.03361s (16.7191GiB/s) | 20260506 01:00:19.988 | INFO | Analyzing chunk frames = [7703552,7704064) | 20260506 01:00:19.988 | INFO | MemInfo: { | 20260506 01:00:19.988 | INFO | "gpu" : " 17 GB", | 20260506 01:00:19.988 | INFO | "huge" : "", | 20260506 01:00:19.988 | INFO | "huge_shared" : " 27 GB", | 20260506 01:00:19.988 | INFO | "malloc" : " 14 GB", | 20260506 01:00:19.988 | INFO | "pinned" : " 29 GB" | 20260506 01:00:19.988 | INFO | } | 20260506 01:00:19.988 | INFO | Perf report for the last 5.13804 seconds: | 20260506 01:00:19.988 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:00:19.988 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:19.988 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:19.988 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:19.988 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260506 01:00:19.988 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:19.988 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:19.988 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 01:00:19.988 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:19.988 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:19.988 | INFO | Uploaded 36GiB to GPU in 2.06776s (17.4101GiB/s) | 20260506 01:00:19.988 | INFO | Downloaded 34GiB to GPU in 2.0731s (16.4006GiB/s) | 20260506 01:00:25.125 | INFO | Analyzing chunk frames = [7704064,7704576) | 20260506 01:00:25.125 | INFO | MemInfo: { | 20260506 01:00:25.125 | INFO | "gpu" : " 17 GB", | 20260506 01:00:25.125 | INFO | "huge" : "", | 20260506 01:00:25.125 | INFO | "huge_shared" : " 27 GB", | 20260506 01:00:25.125 | INFO | "malloc" : " 14 GB", | 20260506 01:00:25.125 | INFO | "pinned" : " 29 GB" | 20260506 01:00:25.125 | INFO | } | 20260506 01:00:25.125 | INFO | Perf report for the last 5.13732 seconds: | 20260506 01:00:25.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:00:25.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:25.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:25.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:25.125 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260506 01:00:25.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:25.125 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:25.126 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:00:25.126 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:25.126 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:25.126 | INFO | Uploaded 36GiB to GPU in 2.12657s (16.9287GiB/s) | 20260506 01:00:25.126 | INFO | Downloaded 34GiB to GPU in 2.01287s (16.8913GiB/s) | 20260506 01:00:25.245 | INFO | Starting a round of Baz IO | 20260506 01:00:28.391 | INFO | Populating RTMetrics report | 20260506 01:00:28.391 | INFO | RTMetrics report populated | 20260506 01:00:28.391 | INFO | Opening tmp RTMetrics json file | 20260506 01:00:28.392 | INFO | Serializing RTMetrics to json | 20260506 01:00:28.393 | INFO | Serialization done, swapping tmp file now... | 20260506 01:00:28.393 | INFO | RTMetrics done | 20260506 01:00:28.469 | INFO | Finished a round of Baz IO: Wrote 1.839 GiB in 3.223 seconds | 20260506 01:00:28.469 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.34% / 0.00% / 78.66% | 20260506 01:00:28.469 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0851432% / 0% / 7.94729e-05% | 20260506 01:00:28.469 | INFO | Baz IO overall: | 20260506 01:00:28.469 | INFO | BytesWritten : 4360335601005 | 20260506 01:00:28.469 | INFO | NumEvents : 2050577489435 | 20260506 01:00:30.270 | INFO | Analyzing chunk frames = [7704576,7705088) | 20260506 01:00:30.270 | INFO | MemInfo: { | 20260506 01:00:30.270 | INFO | "gpu" : " 17 GB", | 20260506 01:00:30.270 | INFO | "huge" : "", | 20260506 01:00:30.270 | INFO | "huge_shared" : " 27 GB", | 20260506 01:00:30.270 | INFO | "malloc" : " 14 GB", | 20260506 01:00:30.270 | INFO | "pinned" : " 29 GB" | 20260506 01:00:30.270 | INFO | } | 20260506 01:00:30.270 | INFO | Perf report for the last 5.14448 seconds: | 20260506 01:00:30.270 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:00:30.270 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:30.270 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:30.270 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:30.270 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260506 01:00:30.270 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:30.270 | INFO | RT_METRICS : 43.0%, 1.0, 22.5ms, 0 | 20260506 01:00:30.270 | INFO | PRE_HQ : 59.8%, 1.7, 53.6ms, 0 | 20260506 01:00:30.270 | INFO | BAZWRITER : 61.8%, 3.5, 111.3ms, 0 | 20260506 01:00:30.270 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:00:30.270 | INFO | Uploaded 36GiB to GPU in 2.17222s (16.5729GiB/s) | 20260506 01:00:30.270 | INFO | Downloaded 38GiB to GPU in 2.23891s (16.9725GiB/s) | 20260506 01:00:35.410 | INFO | Analyzing chunk frames = [7705088,7705600) | 20260506 01:00:35.410 | INFO | MemInfo: { | 20260506 01:00:35.410 | INFO | "gpu" : " 17 GB", | 20260506 01:00:35.410 | INFO | "huge" : "", | 20260506 01:00:35.410 | INFO | "huge_shared" : " 27 GB", | 20260506 01:00:35.410 | INFO | "malloc" : " 14 GB", | 20260506 01:00:35.410 | INFO | "pinned" : " 29 GB" | 20260506 01:00:35.410 | INFO | } | 20260506 01:00:35.411 | INFO | Perf report for the last 5.14064 seconds: | 20260506 01:00:35.411 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:00:35.411 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:35.411 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:35.411 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:35.411 | INFO | ANALYSIS : 60.9%, 3.0, 95.0ms, 0 | 20260506 01:00:35.411 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:35.411 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:35.411 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 01:00:35.411 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:35.411 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:35.411 | INFO | Uploaded 36GiB to GPU in 1.9499s (18.4625GiB/s) | 20260506 01:00:35.411 | INFO | Downloaded 34GiB to GPU in 1.99281s (17.0613GiB/s) | 20260506 01:00:40.549 | INFO | Analyzing chunk frames = [7705600,7706112) | 20260506 01:00:40.549 | INFO | MemInfo: { | 20260506 01:00:40.549 | INFO | "gpu" : " 17 GB", | 20260506 01:00:40.549 | INFO | "huge" : "", | 20260506 01:00:40.549 | INFO | "huge_shared" : " 27 GB", | 20260506 01:00:40.549 | INFO | "malloc" : " 14 GB", | 20260506 01:00:40.549 | INFO | "pinned" : " 29 GB" | 20260506 01:00:40.549 | INFO | } | 20260506 01:00:40.549 | INFO | Perf report for the last 5.13872 seconds: | 20260506 01:00:40.549 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:00:40.549 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:40.549 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:40.549 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:40.549 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 01:00:40.549 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:40.549 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:40.549 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 01:00:40.549 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:40.549 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:40.550 | INFO | Uploaded 36GiB to GPU in 2.0823s (17.2886GiB/s) | 20260506 01:00:40.550 | INFO | Downloaded 34GiB to GPU in 2.02292s (16.8074GiB/s) | 20260506 01:00:45.682 | INFO | Analyzing chunk frames = [7706112,7706624) | 20260506 01:00:45.682 | INFO | MemInfo: { | 20260506 01:00:45.682 | INFO | "gpu" : " 17 GB", | 20260506 01:00:45.682 | INFO | "huge" : "", | 20260506 01:00:45.682 | INFO | "huge_shared" : " 27 GB", | 20260506 01:00:45.682 | INFO | "malloc" : " 14 GB", | 20260506 01:00:45.682 | INFO | "pinned" : " 29 GB" | 20260506 01:00:45.682 | INFO | } | 20260506 01:00:45.683 | INFO | Perf report for the last 5.13329 seconds: | 20260506 01:00:45.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:00:45.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:45.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:45.683 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:45.683 | INFO | ANALYSIS : 61.3%, 3.0, 95.1ms, 0 | 20260506 01:00:45.683 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:45.683 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:45.683 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:00:45.683 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:45.683 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:45.683 | INFO | Uploaded 36GiB to GPU in 2.08392s (17.2751GiB/s) | 20260506 01:00:45.683 | INFO | Downloaded 34GiB to GPU in 2.02545s (16.7864GiB/s) | 20260506 01:00:50.816 | INFO | Analyzing chunk frames = [7706624,7707136) | 20260506 01:00:50.816 | INFO | MemInfo: { | 20260506 01:00:50.816 | INFO | "gpu" : " 17 GB", | 20260506 01:00:50.816 | INFO | "huge" : "", | 20260506 01:00:50.816 | INFO | "huge_shared" : " 27 GB", | 20260506 01:00:50.816 | INFO | "malloc" : " 14 GB", | 20260506 01:00:50.816 | INFO | "pinned" : " 29 GB" | 20260506 01:00:50.816 | INFO | } | 20260506 01:00:50.816 | INFO | Perf report for the last 5.13366 seconds: | 20260506 01:00:50.816 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:00:50.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:50.816 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:50.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:50.816 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 01:00:50.816 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:50.816 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:50.816 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 01:00:50.816 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:50.816 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:50.816 | INFO | Uploaded 36GiB to GPU in 2.09561s (17.1787GiB/s) | 20260506 01:00:50.816 | INFO | Downloaded 34GiB to GPU in 2.05394s (16.5535GiB/s) | 20260506 01:00:55.966 | INFO | Analyzing chunk frames = [7707136,7707648) | 20260506 01:00:55.966 | INFO | MemInfo: { | 20260506 01:00:55.966 | INFO | "gpu" : " 17 GB", | 20260506 01:00:55.966 | INFO | "huge" : "", | 20260506 01:00:55.966 | INFO | "huge_shared" : " 27 GB", | 20260506 01:00:55.966 | INFO | "malloc" : " 14 GB", | 20260506 01:00:55.966 | INFO | "pinned" : " 29 GB" | 20260506 01:00:55.966 | INFO | } | 20260506 01:00:55.966 | INFO | Perf report for the last 5.14961 seconds: | 20260506 01:00:55.966 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:00:55.966 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:55.966 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:55.966 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:55.966 | INFO | ANALYSIS : 61.1%, 3.0, 95.6ms, 0 | 20260506 01:00:55.966 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:55.966 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:55.966 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 01:00:55.966 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:55.966 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:00:55.966 | INFO | Uploaded 36GiB to GPU in 2.07018s (17.3898GiB/s) | 20260506 01:00:55.966 | INFO | Downloaded 34GiB to GPU in 2.0359s (16.7002GiB/s) | 20260506 01:01:01.110 | INFO | Analyzing chunk frames = [7707648,7708160) | 20260506 01:01:01.110 | INFO | MemInfo: { | 20260506 01:01:01.110 | INFO | "gpu" : " 17 GB", | 20260506 01:01:01.110 | INFO | "huge" : "", | 20260506 01:01:01.110 | INFO | "huge_shared" : " 27 GB", | 20260506 01:01:01.110 | INFO | "malloc" : " 14 GB", | 20260506 01:01:01.110 | INFO | "pinned" : " 29 GB" | 20260506 01:01:01.110 | INFO | } | 20260506 01:01:01.110 | INFO | Perf report for the last 5.14451 seconds: | 20260506 01:01:01.110 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:01:01.110 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:01.110 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:01.110 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:01.110 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260506 01:01:01.110 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:01.110 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:01.110 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 01:01:01.111 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:01.111 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:01.111 | INFO | Uploaded 36GiB to GPU in 2.0799s (17.3085GiB/s) | 20260506 01:01:01.111 | INFO | Downloaded 34GiB to GPU in 2.07154s (16.4129GiB/s) | 20260506 01:01:06.247 | INFO | Analyzing chunk frames = [7708160,7708672) | 20260506 01:01:06.247 | INFO | MemInfo: { | 20260506 01:01:06.247 | INFO | "gpu" : " 17 GB", | 20260506 01:01:06.247 | INFO | "huge" : "", | 20260506 01:01:06.247 | INFO | "huge_shared" : " 27 GB", | 20260506 01:01:06.247 | INFO | "malloc" : " 14 GB", | 20260506 01:01:06.247 | INFO | "pinned" : " 29 GB" | 20260506 01:01:06.247 | INFO | } | 20260506 01:01:06.247 | INFO | Perf report for the last 5.13694 seconds: | 20260506 01:01:06.247 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:01:06.247 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:06.247 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:06.247 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:06.247 | INFO | ANALYSIS : 61.5%, 3.0, 96.0ms, 0 | 20260506 01:01:06.247 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:06.247 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:06.247 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 01:01:06.247 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:06.247 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:06.248 | INFO | Uploaded 36GiB to GPU in 2.04189s (17.6307GiB/s) | 20260506 01:01:06.248 | INFO | Downloaded 34GiB to GPU in 2.02371s (16.8008GiB/s) | 20260506 01:01:06.248 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 01:01:06.248 | INFO | Total mem usage: 15.3 GiB | 20260506 01:01:06.248 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 01:01:06.248 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.17%) | 20260506 01:01:06.248 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 01:01:06.248 | INFO | High Water Breakdown: | 20260506 01:01:06.248 | INFO | 60.9% : 8.55 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 01:01:06.248 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 01:01:06.248 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 01:01:06.248 | INFO | ------------------------------------------ | 20260506 01:01:06.248 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:01:06.248 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 01:01:06.248 | INFO | Total mem usage: 26.5 GiB | 20260506 01:01:06.248 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 01:01:06.248 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 01:01:06.248 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260506 01:01:06.248 | INFO | High Water Breakdown: | 20260506 01:01:06.248 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:01:06.248 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:01:06.248 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:01:06.248 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:01:06.248 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:01:06.248 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:01:06.248 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 01:01:06.248 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 01:01:06.248 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:01:06.248 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 01:01:06.248 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 01:01:06.248 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 01:01:06.248 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 01:01:06.248 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 01:01:06.248 | INFO | ------------------------------------------ | 20260506 01:01:06.248 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:01:06.248 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 01:01:06.248 | INFO | Total mem usage: 25.6 GiB | 20260506 01:01:06.248 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 01:01:06.248 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 01:01:06.248 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 01:01:06.248 | INFO | High Water Breakdown: | 20260506 01:01:06.248 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 01:01:06.248 | INFO | ------------------------------------------ | 20260506 01:01:06.248 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:01:06.248 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 01:01:06.248 | INFO | Total mem usage: 15.7 GiB | 20260506 01:01:06.248 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 01:01:06.248 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 01:01:06.248 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 01:01:06.248 | INFO | High Water Breakdown: | 20260506 01:01:06.248 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 01:01:06.248 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 01:01:06.248 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 01:01:06.249 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 01:01:06.249 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 01:01:06.249 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 01:01:06.249 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 01:01:06.249 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 01:01:06.249 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 01:01:06.249 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:01:06.249 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 01:01:06.249 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:01:06.249 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 01:01:06.249 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 01:01:06.249 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 01:01:06.249 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 01:01:06.249 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:01:06.249 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 01:01:06.249 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:01:06.249 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:01:06.249 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:01:06.249 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 01:01:06.249 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 01:01:06.249 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:01:06.249 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 01:01:06.249 | INFO | ------------------------------------------ | 20260506 01:01:06.249 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:01:06.249 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 01:01:06.249 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:01:06.249 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:01:06.249 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.40e+01, 1.35e+01 / 7.27e-01 | 20260506 01:01:06.249 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.04e-01, 1.06e-02 / 5.17e-03 | 20260506 01:01:06.249 | INFO | PerfReport: 11.85%, Baseline -- 3.71e+00 / 4.52e+00, 5.76e-01 / 1.02e-01 | 20260506 01:01:06.249 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.51e+00, 1.91e-01 / 8.37e-02 | 20260506 01:01:06.249 | INFO | PerfReport: 16.00%, DME -- 4.01e+01 / 4.40e+01, 3.86e+00 / 9.52e-01 | 20260506 01:01:06.249 | INFO | PerfReport: 57.41%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.51e+00 / 9.70e-01 | 20260506 01:01:06.249 | INFO | PerfReport: 7.06%, PulseAccumulating -- 2.21e+00 / 2.73e+00, 4.19e-01 / 1.95e-01 | 20260506 01:01:06.249 | INFO | PerfReport: 3.05%, Metrics -- 9.57e-01 / 1.23e+00, 2.57e-01 / 9.82e-02 | 20260506 01:01:06.249 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.31e-01 / 3.81e-01, 5.61e-02 / 2.95e-02 | 20260506 01:01:06.249 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 01:01:06.249 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:01:06.249 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:01:06.249 | INFO | PerfReport: Full Execution -- 9.80e+01 / -nan, 2.44e+01 / 0.00e+00 | 20260506 01:01:06.249 | INFO | PerfReport: 22.19%, Upload -- 2.18e+01 / 0.00e+00, 7.38e+00 / 0.00e+00 | 20260506 01:01:06.249 | INFO | PerfReport: 21.88%, Download -- 2.14e+01 / 0.00e+00, 7.56e+00 / 0.00e+00 | 20260506 01:01:06.249 | INFO | PerfReport: 31.96%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 01:01:06.249 | INFO | PerfReport: 2.06%, LockUpload -- 2.02e+00 / 0.00e+00, 7.83e+00 / 0.00e+00 | 20260506 01:01:06.249 | INFO | PerfReport: 1.29%, LockDownload -- 1.27e+00 / 0.00e+00, 2.87e+00 / 0.00e+00 | 20260506 01:01:06.249 | INFO | PerfReport: 20.60%, LockCompute -- 2.02e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 01:01:06.249 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.48e-02 / 0.00e+00 | 20260506 01:01:06.371 | INFO | Starting a round of Baz IO | 20260506 01:01:09.510 | INFO | Populating RTMetrics report | 20260506 01:01:09.511 | INFO | RTMetrics report populated | 20260506 01:01:09.511 | INFO | Opening tmp RTMetrics json file | 20260506 01:01:09.512 | INFO | Serializing RTMetrics to json | 20260506 01:01:09.513 | INFO | Serialization done, swapping tmp file now... | 20260506 01:01:09.514 | INFO | RTMetrics done | 20260506 01:01:09.590 | INFO | Finished a round of Baz IO: Wrote 1.841 GiB in 3.219 seconds | 20260506 01:01:09.590 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.34% / 0.00% / 78.66% | 20260506 01:01:09.590 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0848929% / 0% / 6.35783e-05% | 20260506 01:01:09.590 | INFO | Baz IO overall: | 20260506 01:01:09.591 | INFO | BytesWritten : 4362311833613 | 20260506 01:01:09.591 | INFO | NumEvents : 2051479671667 | 20260506 01:01:11.382 | INFO | Analyzing chunk frames = [7708672,7709184) | 20260506 01:01:11.382 | INFO | MemInfo: { | 20260506 01:01:11.382 | INFO | "gpu" : " 17 GB", | 20260506 01:01:11.382 | INFO | "huge" : "", | 20260506 01:01:11.382 | INFO | "huge_shared" : " 27 GB", | 20260506 01:01:11.382 | INFO | "malloc" : " 14 GB", | 20260506 01:01:11.382 | INFO | "pinned" : " 29 GB" | 20260506 01:01:11.382 | INFO | } | 20260506 01:01:11.382 | INFO | Perf report for the last 5.13435 seconds: | 20260506 01:01:11.382 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:01:11.382 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:11.382 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:11.382 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:11.382 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260506 01:01:11.382 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:11.382 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260506 01:01:11.382 | INFO | PRE_HQ : 59.7%, 1.7, 53.8ms, 0 | 20260506 01:01:11.382 | INFO | BAZWRITER : 61.9%, 3.5, 110.6ms, 0 | 20260506 01:01:11.382 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:01:11.382 | INFO | Uploaded 36GiB to GPU in 2.18332s (16.4886GiB/s) | 20260506 01:01:11.382 | INFO | Downloaded 38GiB to GPU in 2.24097s (16.957GiB/s) | 20260506 01:01:16.517 | INFO | Analyzing chunk frames = [7709184,7709696) | 20260506 01:01:16.517 | INFO | MemInfo: { | 20260506 01:01:16.517 | INFO | "gpu" : " 17 GB", | 20260506 01:01:16.517 | INFO | "huge" : "", | 20260506 01:01:16.517 | INFO | "huge_shared" : " 27 GB", | 20260506 01:01:16.517 | INFO | "malloc" : " 14 GB", | 20260506 01:01:16.517 | INFO | "pinned" : " 29 GB" | 20260506 01:01:16.517 | INFO | } | 20260506 01:01:16.517 | INFO | Perf report for the last 5.13544 seconds: | 20260506 01:01:16.517 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:01:16.517 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:16.517 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:16.517 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:16.517 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 01:01:16.517 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:16.517 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:16.517 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:01:16.517 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:16.517 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:16.517 | INFO | Uploaded 36GiB to GPU in 2.09991s (17.1436GiB/s) | 20260506 01:01:16.517 | INFO | Downloaded 34GiB to GPU in 2.03784s (16.6843GiB/s) | 20260506 01:01:21.659 | INFO | Analyzing chunk frames = [7709696,7710208) | 20260506 01:01:21.659 | INFO | MemInfo: { | 20260506 01:01:21.659 | INFO | "gpu" : " 17 GB", | 20260506 01:01:21.659 | INFO | "huge" : "", | 20260506 01:01:21.659 | INFO | "huge_shared" : " 27 GB", | 20260506 01:01:21.659 | INFO | "malloc" : " 14 GB", | 20260506 01:01:21.659 | INFO | "pinned" : " 29 GB" | 20260506 01:01:21.659 | INFO | } | 20260506 01:01:21.659 | INFO | Perf report for the last 5.14196 seconds: | 20260506 01:01:21.659 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:01:21.659 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:21.659 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:21.659 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:21.659 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 01:01:21.659 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:21.659 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:21.659 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:01:21.659 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:21.659 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:21.659 | INFO | Uploaded 36GiB to GPU in 2.08223s (17.2892GiB/s) | 20260506 01:01:21.659 | INFO | Downloaded 34GiB to GPU in 2.01458s (16.877GiB/s) | 20260506 01:01:26.803 | INFO | Analyzing chunk frames = [7710208,7710720) | 20260506 01:01:26.803 | INFO | MemInfo: { | 20260506 01:01:26.803 | INFO | "gpu" : " 17 GB", | 20260506 01:01:26.803 | INFO | "huge" : "", | 20260506 01:01:26.803 | INFO | "huge_shared" : " 27 GB", | 20260506 01:01:26.803 | INFO | "malloc" : " 14 GB", | 20260506 01:01:26.803 | INFO | "pinned" : " 29 GB" | 20260506 01:01:26.803 | INFO | } | 20260506 01:01:26.803 | INFO | Perf report for the last 5.14412 seconds: | 20260506 01:01:26.803 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:01:26.803 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:26.803 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:26.803 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:26.803 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 01:01:26.803 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:26.803 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:26.803 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:01:26.803 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:26.803 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:26.804 | INFO | Uploaded 36GiB to GPU in 2.15374s (16.7151GiB/s) | 20260506 01:01:26.804 | INFO | Downloaded 34GiB to GPU in 2.05968s (16.5074GiB/s) | 20260506 01:01:31.936 | INFO | Analyzing chunk frames = [7710720,7711232) | 20260506 01:01:31.936 | INFO | MemInfo: { | 20260506 01:01:31.936 | INFO | "gpu" : " 17 GB", | 20260506 01:01:31.936 | INFO | "huge" : "", | 20260506 01:01:31.937 | INFO | "huge_shared" : " 27 GB", | 20260506 01:01:31.937 | INFO | "malloc" : " 14 GB", | 20260506 01:01:31.937 | INFO | "pinned" : " 29 GB" | 20260506 01:01:31.937 | INFO | } | 20260506 01:01:31.937 | INFO | Perf report for the last 5.1335 seconds: | 20260506 01:01:31.937 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:01:31.937 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:31.937 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:31.937 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:31.937 | INFO | ANALYSIS : 60.9%, 3.0, 95.0ms, 0 | 20260506 01:01:31.937 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:31.937 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:31.937 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 01:01:31.937 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:31.937 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:31.937 | INFO | Uploaded 36GiB to GPU in 2.0706s (17.3863GiB/s) | 20260506 01:01:31.937 | INFO | Downloaded 34GiB to GPU in 2.0306s (16.7438GiB/s) | 20260506 01:01:37.076 | INFO | Analyzing chunk frames = [7711232,7711744) | 20260506 01:01:37.076 | INFO | MemInfo: { | 20260506 01:01:37.076 | INFO | "gpu" : " 17 GB", | 20260506 01:01:37.076 | INFO | "huge" : "", | 20260506 01:01:37.076 | INFO | "huge_shared" : " 27 GB", | 20260506 01:01:37.076 | INFO | "malloc" : " 14 GB", | 20260506 01:01:37.076 | INFO | "pinned" : " 29 GB" | 20260506 01:01:37.076 | INFO | } | 20260506 01:01:37.077 | INFO | Perf report for the last 5.13976 seconds: | 20260506 01:01:37.077 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:01:37.077 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:37.077 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:37.077 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:37.077 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 01:01:37.077 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:37.077 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:37.077 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:01:37.077 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:37.077 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:37.077 | INFO | Uploaded 36GiB to GPU in 2.0792s (17.3144GiB/s) | 20260506 01:01:37.077 | INFO | Downloaded 34GiB to GPU in 2.0382s (16.6814GiB/s) | 20260506 01:01:42.223 | INFO | Analyzing chunk frames = [7711744,7712256) | 20260506 01:01:42.223 | INFO | MemInfo: { | 20260506 01:01:42.223 | INFO | "gpu" : " 17 GB", | 20260506 01:01:42.223 | INFO | "huge" : "", | 20260506 01:01:42.223 | INFO | "huge_shared" : " 27 GB", | 20260506 01:01:42.223 | INFO | "malloc" : " 14 GB", | 20260506 01:01:42.223 | INFO | "pinned" : " 29 GB" | 20260506 01:01:42.223 | INFO | } | 20260506 01:01:42.223 | INFO | Perf report for the last 5.14625 seconds: | 20260506 01:01:42.223 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:01:42.223 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:42.223 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:42.223 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:42.223 | INFO | ANALYSIS : 62.1%, 3.0, 97.1ms, 0 | 20260506 01:01:42.223 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:42.223 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:42.223 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 01:01:42.223 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:42.223 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:42.223 | INFO | Uploaded 36GiB to GPU in 2.03639s (17.6784GiB/s) | 20260506 01:01:42.223 | INFO | Downloaded 34GiB to GPU in 2.05385s (16.5543GiB/s) | 20260506 01:01:47.361 | INFO | Analyzing chunk frames = [7712256,7712768) | 20260506 01:01:47.361 | INFO | MemInfo: { | 20260506 01:01:47.361 | INFO | "gpu" : " 17 GB", | 20260506 01:01:47.361 | INFO | "huge" : "", | 20260506 01:01:47.361 | INFO | "huge_shared" : " 27 GB", | 20260506 01:01:47.361 | INFO | "malloc" : " 14 GB", | 20260506 01:01:47.361 | INFO | "pinned" : " 29 GB" | 20260506 01:01:47.361 | INFO | } | 20260506 01:01:47.361 | INFO | Perf report for the last 5.1384 seconds: | 20260506 01:01:47.361 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:01:47.361 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:47.361 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:47.361 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:47.361 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 01:01:47.361 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:47.361 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:47.361 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 01:01:47.361 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:47.361 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:47.362 | INFO | Uploaded 36GiB to GPU in 2.12667s (16.9279GiB/s) | 20260506 01:01:47.362 | INFO | Downloaded 34GiB to GPU in 2.02797s (16.7655GiB/s) | 20260506 01:01:47.485 | INFO | Starting a round of Baz IO | 20260506 01:01:50.620 | INFO | Populating RTMetrics report | 20260506 01:01:50.620 | INFO | RTMetrics report populated | 20260506 01:01:50.620 | INFO | Opening tmp RTMetrics json file | 20260506 01:01:50.621 | INFO | Serializing RTMetrics to json | 20260506 01:01:50.623 | INFO | Serialization done, swapping tmp file now... | 20260506 01:01:50.626 | INFO | RTMetrics done | 20260506 01:01:50.700 | INFO | Finished a round of Baz IO: Wrote 1.841 GiB in 3.214 seconds | 20260506 01:01:50.700 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.34% / 0.00% / 78.66% | 20260506 01:01:50.700 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0852942% / 0% / 8.34465e-05% | 20260506 01:01:50.700 | INFO | Baz IO overall: | 20260506 01:01:50.700 | INFO | BytesWritten : 4364288794264 | 20260506 01:01:50.700 | INFO | NumEvents : 2052382394381 | 20260506 01:01:52.494 | INFO | Analyzing chunk frames = [7712768,7713280) | 20260506 01:01:52.494 | INFO | MemInfo: { | 20260506 01:01:52.494 | INFO | "gpu" : " 17 GB", | 20260506 01:01:52.494 | INFO | "huge" : "", | 20260506 01:01:52.494 | INFO | "huge_shared" : " 27 GB", | 20260506 01:01:52.494 | INFO | "malloc" : " 14 GB", | 20260506 01:01:52.494 | INFO | "pinned" : " 29 GB" | 20260506 01:01:52.494 | INFO | } | 20260506 01:01:52.494 | INFO | Perf report for the last 5.13289 seconds: | 20260506 01:01:52.494 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:01:52.494 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:52.494 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:52.494 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:52.494 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260506 01:01:52.494 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:52.494 | INFO | RT_METRICS : 43.2%, 1.0, 22.6ms, 0 | 20260506 01:01:52.494 | INFO | PRE_HQ : 59.7%, 1.7, 53.6ms, 0 | 20260506 01:01:52.494 | INFO | BAZWRITER : 61.9%, 3.5, 112.3ms, 0 | 20260506 01:01:52.494 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 01:01:52.494 | INFO | Uploaded 36GiB to GPU in 2.18218s (16.4973GiB/s) | 20260506 01:01:52.495 | INFO | Downloaded 38GiB to GPU in 2.23856s (16.9752GiB/s) | 20260506 01:01:57.632 | INFO | Analyzing chunk frames = [7713280,7713792) | 20260506 01:01:57.632 | INFO | MemInfo: { | 20260506 01:01:57.632 | INFO | "gpu" : " 17 GB", | 20260506 01:01:57.632 | INFO | "huge" : "", | 20260506 01:01:57.632 | INFO | "huge_shared" : " 27 GB", | 20260506 01:01:57.632 | INFO | "malloc" : " 14 GB", | 20260506 01:01:57.632 | INFO | "pinned" : " 29 GB" | 20260506 01:01:57.632 | INFO | } | 20260506 01:01:57.632 | INFO | Perf report for the last 5.13815 seconds: | 20260506 01:01:57.632 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:01:57.632 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:57.632 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:57.632 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:57.632 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 01:01:57.632 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:57.632 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:57.632 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 01:01:57.632 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:57.632 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:01:57.632 | INFO | Uploaded 36GiB to GPU in 2.1117s (17.0479GiB/s) | 20260506 01:01:57.633 | INFO | Downloaded 34GiB to GPU in 2.04172s (16.6527GiB/s) | 20260506 01:02:02.772 | INFO | Analyzing chunk frames = [7713792,7714304) | 20260506 01:02:02.772 | INFO | MemInfo: { | 20260506 01:02:02.772 | INFO | "gpu" : " 17 GB", | 20260506 01:02:02.772 | INFO | "huge" : "", | 20260506 01:02:02.772 | INFO | "huge_shared" : " 27 GB", | 20260506 01:02:02.772 | INFO | "malloc" : " 14 GB", | 20260506 01:02:02.772 | INFO | "pinned" : " 29 GB" | 20260506 01:02:02.772 | INFO | } | 20260506 01:02:02.772 | INFO | Perf report for the last 5.13959 seconds: | 20260506 01:02:02.772 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:02:02.772 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:02.772 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:02.772 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:02.772 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 01:02:02.772 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:02.772 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:02.772 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:02:02.772 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:02.772 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:02.772 | INFO | Uploaded 36GiB to GPU in 2.0863s (17.2555GiB/s) | 20260506 01:02:02.772 | INFO | Downloaded 34GiB to GPU in 2.03073s (16.7427GiB/s) | 20260506 01:02:07.914 | INFO | Analyzing chunk frames = [7714304,7714816) | 20260506 01:02:07.914 | INFO | MemInfo: { | 20260506 01:02:07.914 | INFO | "gpu" : " 17 GB", | 20260506 01:02:07.914 | INFO | "huge" : "", | 20260506 01:02:07.914 | INFO | "huge_shared" : " 27 GB", | 20260506 01:02:07.914 | INFO | "malloc" : " 14 GB", | 20260506 01:02:07.914 | INFO | "pinned" : " 29 GB" | 20260506 01:02:07.914 | INFO | } | 20260506 01:02:07.914 | INFO | Perf report for the last 5.14236 seconds: | 20260506 01:02:07.914 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:02:07.914 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:07.914 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:07.914 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:07.914 | INFO | ANALYSIS : 61.5%, 3.0, 95.5ms, 0 | 20260506 01:02:07.914 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:07.914 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:07.914 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:02:07.914 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:07.914 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:07.915 | INFO | Uploaded 36GiB to GPU in 2.13584s (16.8552GiB/s) | 20260506 01:02:07.915 | INFO | Downloaded 34GiB to GPU in 2.02849s (16.7612GiB/s) | 20260506 01:02:13.059 | INFO | Analyzing chunk frames = [7714816,7715328) | 20260506 01:02:13.059 | INFO | MemInfo: { | 20260506 01:02:13.059 | INFO | "gpu" : " 17 GB", | 20260506 01:02:13.059 | INFO | "huge" : "", | 20260506 01:02:13.059 | INFO | "huge_shared" : " 27 GB", | 20260506 01:02:13.059 | INFO | "malloc" : " 14 GB", | 20260506 01:02:13.059 | INFO | "pinned" : " 29 GB" | 20260506 01:02:13.059 | INFO | } | 20260506 01:02:13.059 | INFO | Perf report for the last 5.14492 seconds: | 20260506 01:02:13.059 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:02:13.059 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:13.059 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:13.059 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:13.059 | INFO | ANALYSIS : 60.7%, 3.0, 94.9ms, 0 | 20260506 01:02:13.059 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:13.059 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:13.059 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 01:02:13.059 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:13.059 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:13.059 | INFO | Uploaded 36GiB to GPU in 2.05956s (17.4795GiB/s) | 20260506 01:02:13.059 | INFO | Downloaded 34GiB to GPU in 2.03071s (16.7429GiB/s) | 20260506 01:02:18.195 | INFO | Analyzing chunk frames = [7715328,7715840) | 20260506 01:02:18.195 | INFO | MemInfo: { | 20260506 01:02:18.195 | INFO | "gpu" : " 17 GB", | 20260506 01:02:18.195 | INFO | "huge" : "", | 20260506 01:02:18.195 | INFO | "huge_shared" : " 27 GB", | 20260506 01:02:18.195 | INFO | "malloc" : " 14 GB", | 20260506 01:02:18.195 | INFO | "pinned" : " 29 GB" | 20260506 01:02:18.195 | INFO | } | 20260506 01:02:18.195 | INFO | Perf report for the last 5.13579 seconds: | 20260506 01:02:18.195 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:02:18.195 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:18.195 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:18.195 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:18.195 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 01:02:18.195 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:18.195 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:18.195 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 01:02:18.195 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:18.195 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:18.195 | INFO | Uploaded 36GiB to GPU in 2.07409s (17.357GiB/s) | 20260506 01:02:18.195 | INFO | Downloaded 34GiB to GPU in 2.04264s (16.6451GiB/s) | 20260506 01:02:23.329 | INFO | Analyzing chunk frames = [7715840,7716352) | 20260506 01:02:23.329 | INFO | MemInfo: { | 20260506 01:02:23.329 | INFO | "gpu" : " 17 GB", | 20260506 01:02:23.329 | INFO | "huge" : "", | 20260506 01:02:23.329 | INFO | "huge_shared" : " 27 GB", | 20260506 01:02:23.329 | INFO | "malloc" : " 14 GB", | 20260506 01:02:23.329 | INFO | "pinned" : " 29 GB" | 20260506 01:02:23.329 | INFO | } | 20260506 01:02:23.329 | INFO | Perf report for the last 5.13424 seconds: | 20260506 01:02:23.329 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:02:23.329 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:23.329 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:23.329 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:23.329 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260506 01:02:23.329 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:23.329 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:23.329 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 01:02:23.329 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:23.329 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:23.329 | INFO | Uploaded 36GiB to GPU in 2.07345s (17.3623GiB/s) | 20260506 01:02:23.330 | INFO | Downloaded 34GiB to GPU in 2.06963s (16.4281GiB/s) | 20260506 01:02:28.467 | INFO | Analyzing chunk frames = [7716352,7716864) | 20260506 01:02:28.467 | INFO | MemInfo: { | 20260506 01:02:28.467 | INFO | "gpu" : " 17 GB", | 20260506 01:02:28.467 | INFO | "huge" : "", | 20260506 01:02:28.467 | INFO | "huge_shared" : " 27 GB", | 20260506 01:02:28.467 | INFO | "malloc" : " 14 GB", | 20260506 01:02:28.467 | INFO | "pinned" : " 29 GB" | 20260506 01:02:28.467 | INFO | } | 20260506 01:02:28.467 | INFO | Perf report for the last 5.13806 seconds: | 20260506 01:02:28.467 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:02:28.467 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:28.467 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:28.467 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:28.467 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260506 01:02:28.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:28.467 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:28.467 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:02:28.467 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:28.467 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:28.468 | INFO | Uploaded 36GiB to GPU in 2.1152s (17.0196GiB/s) | 20260506 01:02:28.468 | INFO | Downloaded 34GiB to GPU in 2.00782s (16.9338GiB/s) | 20260506 01:02:28.468 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 01:02:28.468 | INFO | Total mem usage: 15.3 GiB | 20260506 01:02:28.468 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 01:02:28.468 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.15%) | 20260506 01:02:28.468 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 01:02:28.468 | INFO | High Water Breakdown: | 20260506 01:02:28.468 | INFO | 60.9% : 8.55 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 01:02:28.468 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 01:02:28.468 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 01:02:28.468 | INFO | ------------------------------------------ | 20260506 01:02:28.468 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:02:28.468 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 01:02:28.468 | INFO | Total mem usage: 26.5 GiB | 20260506 01:02:28.468 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 01:02:28.468 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 01:02:28.468 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260506 01:02:28.468 | INFO | High Water Breakdown: | 20260506 01:02:28.468 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:02:28.468 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:02:28.468 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:02:28.468 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:02:28.468 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:02:28.468 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:02:28.468 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 01:02:28.468 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 01:02:28.468 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:02:28.468 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 01:02:28.468 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 01:02:28.468 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 01:02:28.468 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 01:02:28.468 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 01:02:28.468 | INFO | ------------------------------------------ | 20260506 01:02:28.468 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:02:28.468 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 01:02:28.468 | INFO | Total mem usage: 25.6 GiB | 20260506 01:02:28.468 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 01:02:28.468 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 01:02:28.468 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 01:02:28.468 | INFO | High Water Breakdown: | 20260506 01:02:28.468 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 01:02:28.468 | INFO | ------------------------------------------ | 20260506 01:02:28.468 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:02:28.468 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 01:02:28.468 | INFO | Total mem usage: 15.7 GiB | 20260506 01:02:28.468 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 01:02:28.468 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 01:02:28.468 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 01:02:28.468 | INFO | High Water Breakdown: | 20260506 01:02:28.468 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 01:02:28.468 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 01:02:28.468 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 01:02:28.468 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 01:02:28.469 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 01:02:28.469 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 01:02:28.469 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 01:02:28.469 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 01:02:28.469 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 01:02:28.469 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:02:28.469 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 01:02:28.469 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:02:28.469 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 01:02:28.469 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 01:02:28.469 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 01:02:28.469 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 01:02:28.469 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:02:28.469 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 01:02:28.469 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:02:28.469 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:02:28.469 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:02:28.469 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 01:02:28.469 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 01:02:28.469 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:02:28.469 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 01:02:28.469 | INFO | ------------------------------------------ | 20260506 01:02:28.469 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:02:28.469 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 01:02:28.469 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:02:28.469 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:02:28.469 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.40e+01, 1.32e+01 / 9.01e-01 | 20260506 01:02:28.469 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.05e-01, 1.05e-02 / 4.71e-03 | 20260506 01:02:28.469 | INFO | PerfReport: 11.80%, Baseline -- 3.70e+00 / 4.49e+00, 5.62e-01 / 2.00e-01 | 20260506 01:02:28.469 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.48e+00, 1.87e-01 / 1.19e-01 | 20260506 01:02:28.469 | INFO | PerfReport: 16.00%, DME -- 4.01e+01 / 4.40e+01, 3.81e+00 / 1.18e+00 | 20260506 01:02:28.469 | INFO | PerfReport: 57.46%, FrameLabeling -- 1.80e+01 / 1.96e+01, 1.53e+00 / 1.23e+00 | 20260506 01:02:28.469 | INFO | PerfReport: 7.06%, PulseAccumulating -- 2.21e+00 / 2.71e+00, 4.19e-01 / 2.77e-01 | 20260506 01:02:28.469 | INFO | PerfReport: 3.06%, Metrics -- 9.57e-01 / 1.22e+00, 2.58e-01 / 1.39e-01 | 20260506 01:02:28.469 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.31e-01 / 3.83e-01, 5.59e-02 / 2.63e-02 | 20260506 01:02:28.469 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 01:02:28.469 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:02:28.469 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:02:28.469 | INFO | PerfReport: Full Execution -- 9.81e+01 / -nan, 2.41e+01 / 0.00e+00 | 20260506 01:02:28.469 | INFO | PerfReport: 22.34%, Upload -- 2.19e+01 / 0.00e+00, 7.51e+00 / 0.00e+00 | 20260506 01:02:28.469 | INFO | PerfReport: 21.89%, Download -- 2.15e+01 / 0.00e+00, 7.61e+00 / 0.00e+00 | 20260506 01:02:28.469 | INFO | PerfReport: 31.92%, Compute -- 3.13e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 01:02:28.469 | INFO | PerfReport: 2.07%, LockUpload -- 2.03e+00 / 0.00e+00, 7.89e+00 / 0.00e+00 | 20260506 01:02:28.469 | INFO | PerfReport: 1.27%, LockDownload -- 1.25e+00 / 0.00e+00, 2.83e+00 / 0.00e+00 | 20260506 01:02:28.469 | INFO | PerfReport: 20.49%, LockCompute -- 2.01e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 01:02:28.469 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.14e-02 / 0.00e+00 | 20260506 01:02:28.585 | INFO | Starting a round of Baz IO | 20260506 01:02:31.730 | INFO | Populating RTMetrics report | 20260506 01:02:31.731 | INFO | RTMetrics report populated | 20260506 01:02:31.731 | INFO | Opening tmp RTMetrics json file | 20260506 01:02:31.733 | INFO | Serializing RTMetrics to json | 20260506 01:02:31.735 | INFO | Serialization done, swapping tmp file now... | 20260506 01:02:31.735 | INFO | RTMetrics done | 20260506 01:02:31.807 | INFO | Finished a round of Baz IO: Wrote 1.84 GiB in 3.222 seconds | 20260506 01:02:31.807 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.34% / 0.00% / 78.66% | 20260506 01:02:31.807 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0852744% / 0% / 7.94729e-05% | 20260506 01:02:31.807 | INFO | Baz IO overall: | 20260506 01:02:31.807 | INFO | BytesWritten : 4366264026743 | 20260506 01:02:31.807 | INFO | NumEvents : 2053284006010 | 20260506 01:02:33.607 | INFO | Analyzing chunk frames = [7716864,7717376) | 20260506 01:02:33.607 | INFO | MemInfo: { | 20260506 01:02:33.607 | INFO | "gpu" : " 17 GB", | 20260506 01:02:33.607 | INFO | "huge" : "", | 20260506 01:02:33.607 | INFO | "huge_shared" : " 27 GB", | 20260506 01:02:33.608 | INFO | "malloc" : " 14 GB", | 20260506 01:02:33.608 | INFO | "pinned" : " 29 GB" | 20260506 01:02:33.608 | INFO | } | 20260506 01:02:33.608 | INFO | Perf report for the last 5.14033 seconds: | 20260506 01:02:33.608 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:02:33.608 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:33.608 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:33.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:33.608 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260506 01:02:33.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:33.608 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260506 01:02:33.608 | INFO | PRE_HQ : 59.8%, 1.7, 54.1ms, 0 | 20260506 01:02:33.608 | INFO | BAZWRITER : 61.7%, 3.5, 112.3ms, 0 | 20260506 01:02:33.608 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:02:33.608 | INFO | Uploaded 36GiB to GPU in 2.16649s (16.6168GiB/s) | 20260506 01:02:33.608 | INFO | Downloaded 38GiB to GPU in 2.24172s (16.9513GiB/s) | 20260506 01:02:38.751 | INFO | Analyzing chunk frames = [7717376,7717888) | 20260506 01:02:38.751 | INFO | MemInfo: { | 20260506 01:02:38.751 | INFO | "gpu" : " 17 GB", | 20260506 01:02:38.751 | INFO | "huge" : "", | 20260506 01:02:38.751 | INFO | "huge_shared" : " 27 GB", | 20260506 01:02:38.751 | INFO | "malloc" : " 14 GB", | 20260506 01:02:38.751 | INFO | "pinned" : " 29 GB" | 20260506 01:02:38.751 | INFO | } | 20260506 01:02:38.751 | INFO | Perf report for the last 5.1436 seconds: | 20260506 01:02:38.751 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:02:38.751 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:38.751 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:38.751 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:38.751 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260506 01:02:38.751 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:38.751 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:38.751 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 01:02:38.751 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:38.751 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:38.752 | INFO | Uploaded 36GiB to GPU in 2.12166s (16.9678GiB/s) | 20260506 01:02:38.752 | INFO | Downloaded 34GiB to GPU in 2.04484s (16.6272GiB/s) | 20260506 01:02:43.887 | INFO | Analyzing chunk frames = [7717888,7718400) | 20260506 01:02:43.887 | INFO | MemInfo: { | 20260506 01:02:43.887 | INFO | "gpu" : " 17 GB", | 20260506 01:02:43.887 | INFO | "huge" : "", | 20260506 01:02:43.887 | INFO | "huge_shared" : " 27 GB", | 20260506 01:02:43.887 | INFO | "malloc" : " 14 GB", | 20260506 01:02:43.887 | INFO | "pinned" : " 29 GB" | 20260506 01:02:43.887 | INFO | } | 20260506 01:02:43.887 | INFO | Perf report for the last 5.13549 seconds: | 20260506 01:02:43.887 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:02:43.887 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:43.887 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:43.887 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:43.887 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 01:02:43.887 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:43.887 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:43.887 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 01:02:43.887 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:43.887 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:43.887 | INFO | Uploaded 36GiB to GPU in 2.08524s (17.2642GiB/s) | 20260506 01:02:43.887 | INFO | Downloaded 34GiB to GPU in 2.02263s (16.8098GiB/s) | 20260506 01:02:49.026 | INFO | Analyzing chunk frames = [7718400,7718912) | 20260506 01:02:49.026 | INFO | MemInfo: { | 20260506 01:02:49.026 | INFO | "gpu" : " 17 GB", | 20260506 01:02:49.026 | INFO | "huge" : "", | 20260506 01:02:49.026 | INFO | "huge_shared" : " 27 GB", | 20260506 01:02:49.026 | INFO | "malloc" : " 14 GB", | 20260506 01:02:49.026 | INFO | "pinned" : " 29 GB" | 20260506 01:02:49.026 | INFO | } | 20260506 01:02:49.026 | INFO | Perf report for the last 5.1397 seconds: | 20260506 01:02:49.026 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:02:49.026 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:49.026 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:49.027 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:49.027 | INFO | ANALYSIS : 61.6%, 3.0, 95.7ms, 0 | 20260506 01:02:49.027 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:49.027 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:49.027 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 01:02:49.027 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:49.027 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:49.027 | INFO | Uploaded 36GiB to GPU in 2.13318s (16.8762GiB/s) | 20260506 01:02:49.027 | INFO | Downloaded 34GiB to GPU in 2.01654s (16.8606GiB/s) | 20260506 01:02:54.173 | INFO | Analyzing chunk frames = [7718912,7719424) | 20260506 01:02:54.173 | INFO | MemInfo: { | 20260506 01:02:54.173 | INFO | "gpu" : " 17 GB", | 20260506 01:02:54.173 | INFO | "huge" : "", | 20260506 01:02:54.173 | INFO | "huge_shared" : " 27 GB", | 20260506 01:02:54.173 | INFO | "malloc" : " 14 GB", | 20260506 01:02:54.173 | INFO | "pinned" : " 29 GB" | 20260506 01:02:54.173 | INFO | } | 20260506 01:02:54.173 | INFO | Perf report for the last 5.14654 seconds: | 20260506 01:02:54.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:02:54.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:54.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:54.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:54.173 | INFO | ANALYSIS : 60.9%, 3.0, 95.3ms, 0 | 20260506 01:02:54.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:54.173 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:54.173 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:02:54.173 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:54.173 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:54.173 | INFO | Uploaded 36GiB to GPU in 2.09035s (17.222GiB/s) | 20260506 01:02:54.173 | INFO | Downloaded 34GiB to GPU in 2.05157s (16.5726GiB/s) | 20260506 01:02:59.311 | INFO | Analyzing chunk frames = [7719424,7719936) | 20260506 01:02:59.311 | INFO | MemInfo: { | 20260506 01:02:59.311 | INFO | "gpu" : " 17 GB", | 20260506 01:02:59.311 | INFO | "huge" : "", | 20260506 01:02:59.311 | INFO | "huge_shared" : " 27 GB", | 20260506 01:02:59.311 | INFO | "malloc" : " 14 GB", | 20260506 01:02:59.311 | INFO | "pinned" : " 29 GB" | 20260506 01:02:59.311 | INFO | } | 20260506 01:02:59.311 | INFO | Perf report for the last 5.13805 seconds: | 20260506 01:02:59.311 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:02:59.311 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:59.311 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:59.311 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:59.311 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 01:02:59.311 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:59.311 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:59.311 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:02:59.311 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:59.311 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:02:59.311 | INFO | Uploaded 36GiB to GPU in 2.06912s (17.3987GiB/s) | 20260506 01:02:59.311 | INFO | Downloaded 34GiB to GPU in 2.03226s (16.7302GiB/s) | 20260506 01:03:04.441 | INFO | Analyzing chunk frames = [7719936,7720448) | 20260506 01:03:04.441 | INFO | MemInfo: { | 20260506 01:03:04.441 | INFO | "gpu" : " 17 GB", | 20260506 01:03:04.441 | INFO | "huge" : "", | 20260506 01:03:04.441 | INFO | "huge_shared" : " 27 GB", | 20260506 01:03:04.441 | INFO | "malloc" : " 14 GB", | 20260506 01:03:04.441 | INFO | "pinned" : " 29 GB" | 20260506 01:03:04.441 | INFO | } | 20260506 01:03:04.441 | INFO | Perf report for the last 5.13025 seconds: | 20260506 01:03:04.441 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:03:04.441 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:04.441 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:04.441 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:04.441 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260506 01:03:04.441 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:04.441 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:04.441 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 01:03:04.442 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:04.442 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:04.442 | INFO | Uploaded 36GiB to GPU in 2.14205s (16.8063GiB/s) | 20260506 01:03:04.442 | INFO | Downloaded 34GiB to GPU in 2.08844s (16.2801GiB/s) | 20260506 01:03:09.583 | INFO | Analyzing chunk frames = [7720448,7720960) | 20260506 01:03:09.583 | INFO | MemInfo: { | 20260506 01:03:09.583 | INFO | "gpu" : " 17 GB", | 20260506 01:03:09.583 | INFO | "huge" : "", | 20260506 01:03:09.583 | INFO | "huge_shared" : " 27 GB", | 20260506 01:03:09.583 | INFO | "malloc" : " 14 GB", | 20260506 01:03:09.583 | INFO | "pinned" : " 29 GB" | 20260506 01:03:09.583 | INFO | } | 20260506 01:03:09.583 | INFO | Perf report for the last 5.14164 seconds: | 20260506 01:03:09.583 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:03:09.583 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:09.583 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:09.583 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:09.583 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260506 01:03:09.583 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:09.583 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:09.583 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:03:09.583 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:09.583 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:09.583 | INFO | Uploaded 36GiB to GPU in 2.11062s (17.0566GiB/s) | 20260506 01:03:09.583 | INFO | Downloaded 34GiB to GPU in 2.00496s (16.9579GiB/s) | 20260506 01:03:09.702 | INFO | Starting a round of Baz IO | 20260506 01:03:12.844 | INFO | Populating RTMetrics report | 20260506 01:03:12.844 | INFO | RTMetrics report populated | 20260506 01:03:12.844 | INFO | Opening tmp RTMetrics json file | 20260506 01:03:12.846 | INFO | Serializing RTMetrics to json | 20260506 01:03:12.848 | INFO | Serialization done, swapping tmp file now... | 20260506 01:03:12.849 | INFO | RTMetrics done | 20260506 01:03:12.922 | INFO | Finished a round of Baz IO: Wrote 1.837 GiB in 3.22 seconds | 20260506 01:03:12.922 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.34% / 0.00% / 78.66% | 20260506 01:03:12.922 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0844598% / 0% / 6.75519e-05% | 20260506 01:03:12.922 | INFO | Baz IO overall: | 20260506 01:03:12.922 | INFO | BytesWritten : 4368236622927 | 20260506 01:03:12.922 | INFO | NumEvents : 2054183556255 | 20260506 01:03:14.726 | INFO | Analyzing chunk frames = [7720960,7721472) | 20260506 01:03:14.726 | INFO | MemInfo: { | 20260506 01:03:14.726 | INFO | "gpu" : " 17 GB", | 20260506 01:03:14.726 | INFO | "huge" : "", | 20260506 01:03:14.726 | INFO | "huge_shared" : " 27 GB", | 20260506 01:03:14.726 | INFO | "malloc" : " 14 GB", | 20260506 01:03:14.726 | INFO | "pinned" : " 29 GB" | 20260506 01:03:14.726 | INFO | } | 20260506 01:03:14.726 | INFO | Perf report for the last 5.14298 seconds: | 20260506 01:03:14.726 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:03:14.726 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:14.726 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:14.726 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:14.726 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260506 01:03:14.726 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:14.726 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260506 01:03:14.726 | INFO | PRE_HQ : 59.5%, 1.7, 53.6ms, 0 | 20260506 01:03:14.726 | INFO | BAZWRITER : 61.8%, 3.5, 111.3ms, 0 | 20260506 01:03:14.726 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:03:14.726 | INFO | Uploaded 36GiB to GPU in 2.17045s (16.5864GiB/s) | 20260506 01:03:14.726 | INFO | Downloaded 38GiB to GPU in 2.23799s (16.9795GiB/s) | 20260506 01:03:19.868 | INFO | Analyzing chunk frames = [7721472,7721984) | 20260506 01:03:19.868 | INFO | MemInfo: { | 20260506 01:03:19.868 | INFO | "gpu" : " 17 GB", | 20260506 01:03:19.868 | INFO | "huge" : "", | 20260506 01:03:19.868 | INFO | "huge_shared" : " 27 GB", | 20260506 01:03:19.868 | INFO | "malloc" : " 14 GB", | 20260506 01:03:19.868 | INFO | "pinned" : " 29 GB" | 20260506 01:03:19.868 | INFO | } | 20260506 01:03:19.868 | INFO | Perf report for the last 5.14223 seconds: | 20260506 01:03:19.868 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:03:19.868 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:19.868 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:19.868 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:19.868 | INFO | ANALYSIS : 60.9%, 3.0, 95.1ms, 0 | 20260506 01:03:19.868 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:19.868 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:19.868 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:03:19.868 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:19.868 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:19.868 | INFO | Uploaded 36GiB to GPU in 1.97609s (18.2178GiB/s) | 20260506 01:03:19.869 | INFO | Downloaded 34GiB to GPU in 1.98092s (17.1637GiB/s) | 20260506 01:03:25.005 | INFO | Analyzing chunk frames = [7721984,7722496) | 20260506 01:03:25.005 | INFO | MemInfo: { | 20260506 01:03:25.005 | INFO | "gpu" : " 17 GB", | 20260506 01:03:25.005 | INFO | "huge" : "", | 20260506 01:03:25.005 | INFO | "huge_shared" : " 27 GB", | 20260506 01:03:25.005 | INFO | "malloc" : " 14 GB", | 20260506 01:03:25.005 | INFO | "pinned" : " 29 GB" | 20260506 01:03:25.006 | INFO | } | 20260506 01:03:25.006 | INFO | Perf report for the last 5.13731 seconds: | 20260506 01:03:25.006 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:03:25.006 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:25.006 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:25.006 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:25.006 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 01:03:25.006 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:25.006 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:25.006 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:03:25.006 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:25.006 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:25.006 | INFO | Uploaded 36GiB to GPU in 2.0784s (17.3211GiB/s) | 20260506 01:03:25.006 | INFO | Downloaded 34GiB to GPU in 2.02015s (16.8304GiB/s) | 20260506 01:03:30.141 | INFO | Analyzing chunk frames = [7722496,7723008) | 20260506 01:03:30.141 | INFO | MemInfo: { | 20260506 01:03:30.141 | INFO | "gpu" : " 17 GB", | 20260506 01:03:30.141 | INFO | "huge" : "", | 20260506 01:03:30.142 | INFO | "huge_shared" : " 27 GB", | 20260506 01:03:30.142 | INFO | "malloc" : " 14 GB", | 20260506 01:03:30.142 | INFO | "pinned" : " 29 GB" | 20260506 01:03:30.142 | INFO | } | 20260506 01:03:30.142 | INFO | Perf report for the last 5.13607 seconds: | 20260506 01:03:30.142 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:03:30.142 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:30.142 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:30.142 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:30.142 | INFO | ANALYSIS : 61.6%, 3.0, 95.6ms, 0 | 20260506 01:03:30.142 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:30.142 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:30.142 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 01:03:30.142 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:30.142 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:30.142 | INFO | Uploaded 36GiB to GPU in 2.12896s (16.9097GiB/s) | 20260506 01:03:30.142 | INFO | Downloaded 34GiB to GPU in 2.02116s (16.822GiB/s) | 20260506 01:03:35.284 | INFO | Analyzing chunk frames = [7723008,7723520) | 20260506 01:03:35.284 | INFO | MemInfo: { | 20260506 01:03:35.284 | INFO | "gpu" : " 17 GB", | 20260506 01:03:35.284 | INFO | "huge" : "", | 20260506 01:03:35.284 | INFO | "huge_shared" : " 27 GB", | 20260506 01:03:35.284 | INFO | "malloc" : " 14 GB", | 20260506 01:03:35.284 | INFO | "pinned" : " 29 GB" | 20260506 01:03:35.284 | INFO | } | 20260506 01:03:35.285 | INFO | Perf report for the last 5.14289 seconds: | 20260506 01:03:35.285 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:03:35.285 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:35.285 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:35.285 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:35.285 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 01:03:35.285 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:35.285 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:35.285 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:03:35.285 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:35.285 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:35.285 | INFO | Uploaded 36GiB to GPU in 2.10226s (17.1244GiB/s) | 20260506 01:03:35.285 | INFO | Downloaded 34GiB to GPU in 2.05165s (16.5721GiB/s) | 20260506 01:03:40.422 | INFO | Analyzing chunk frames = [7723520,7724032) | 20260506 01:03:40.422 | INFO | MemInfo: { | 20260506 01:03:40.422 | INFO | "gpu" : " 17 GB", | 20260506 01:03:40.422 | INFO | "huge" : "", | 20260506 01:03:40.422 | INFO | "huge_shared" : " 27 GB", | 20260506 01:03:40.422 | INFO | "malloc" : " 14 GB", | 20260506 01:03:40.422 | INFO | "pinned" : " 29 GB" | 20260506 01:03:40.422 | INFO | } | 20260506 01:03:40.422 | INFO | Perf report for the last 5.13738 seconds: | 20260506 01:03:40.422 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:03:40.422 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:40.422 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:40.422 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:40.422 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 01:03:40.422 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:40.422 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:40.422 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 01:03:40.422 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:40.422 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:40.422 | INFO | Uploaded 36GiB to GPU in 2.07552s (17.345GiB/s) | 20260506 01:03:40.422 | INFO | Downloaded 34GiB to GPU in 2.04347s (16.6383GiB/s) | 20260506 01:03:45.561 | INFO | Analyzing chunk frames = [7724032,7724544) | 20260506 01:03:45.561 | INFO | MemInfo: { | 20260506 01:03:45.561 | INFO | "gpu" : " 17 GB", | 20260506 01:03:45.561 | INFO | "huge" : "", | 20260506 01:03:45.561 | INFO | "huge_shared" : " 27 GB", | 20260506 01:03:45.561 | INFO | "malloc" : " 14 GB", | 20260506 01:03:45.561 | INFO | "pinned" : " 29 GB" | 20260506 01:03:45.561 | INFO | } | 20260506 01:03:45.561 | INFO | Perf report for the last 5.13909 seconds: | 20260506 01:03:45.561 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:03:45.561 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:45.561 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:45.561 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:45.561 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260506 01:03:45.561 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:45.561 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:45.561 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 01:03:45.561 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:45.561 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:45.561 | INFO | Uploaded 36GiB to GPU in 2.06204s (17.4584GiB/s) | 20260506 01:03:45.561 | INFO | Downloaded 34GiB to GPU in 2.06833s (16.4384GiB/s) | 20260506 01:03:50.700 | INFO | Analyzing chunk frames = [7724544,7725056) | 20260506 01:03:50.701 | INFO | MemInfo: { | 20260506 01:03:50.701 | INFO | "gpu" : " 17 GB", | 20260506 01:03:50.701 | INFO | "huge" : "", | 20260506 01:03:50.701 | INFO | "huge_shared" : " 27 GB", | 20260506 01:03:50.701 | INFO | "malloc" : " 14 GB", | 20260506 01:03:50.701 | INFO | "pinned" : " 29 GB" | 20260506 01:03:50.701 | INFO | } | 20260506 01:03:50.701 | INFO | Perf report for the last 5.13977 seconds: | 20260506 01:03:50.701 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:03:50.701 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:50.701 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:50.701 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:50.701 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260506 01:03:50.701 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:50.701 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:50.701 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 01:03:50.701 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:50.701 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:50.701 | INFO | Uploaded 36GiB to GPU in 2.12846s (16.9137GiB/s) | 20260506 01:03:50.701 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 01:03:50.701 | INFO | Total mem usage: 15.3 GiB | 20260506 01:03:50.701 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 01:03:50.701 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.19%) | 20260506 01:03:50.702 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 01:03:50.702 | INFO | High Water Breakdown: | 20260506 01:03:50.701 | INFO | Downloaded 34GiB to GPU in 2.02228s (16.8127GiB/s) | 20260506 01:03:50.702 | INFO | 60.9% : 8.54 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 01:03:50.702 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 01:03:50.702 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 01:03:50.702 | INFO | ------------------------------------------ | 20260506 01:03:50.702 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:03:50.702 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 01:03:50.702 | INFO | Total mem usage: 26.5 GiB | 20260506 01:03:50.702 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 01:03:50.702 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 01:03:50.702 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260506 01:03:50.702 | INFO | High Water Breakdown: | 20260506 01:03:50.702 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:03:50.702 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:03:50.702 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:03:50.702 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:03:50.702 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:03:50.702 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:03:50.702 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 01:03:50.702 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 01:03:50.702 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:03:50.702 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 01:03:50.702 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 01:03:50.702 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 01:03:50.702 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 01:03:50.702 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 01:03:50.702 | INFO | ------------------------------------------ | 20260506 01:03:50.702 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:03:50.702 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 01:03:50.702 | INFO | Total mem usage: 25.6 GiB | 20260506 01:03:50.702 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 01:03:50.702 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 01:03:50.702 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 01:03:50.702 | INFO | High Water Breakdown: | 20260506 01:03:50.702 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 01:03:50.702 | INFO | ------------------------------------------ | 20260506 01:03:50.702 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:03:50.702 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 01:03:50.702 | INFO | Total mem usage: 15.7 GiB | 20260506 01:03:50.702 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 01:03:50.702 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 01:03:50.702 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 01:03:50.702 | INFO | High Water Breakdown: | 20260506 01:03:50.702 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 01:03:50.702 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 01:03:50.702 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 01:03:50.702 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 01:03:50.702 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 01:03:50.702 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 01:03:50.702 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 01:03:50.702 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 01:03:50.702 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 01:03:50.702 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:03:50.702 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 01:03:50.702 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:03:50.702 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 01:03:50.702 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 01:03:50.702 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 01:03:50.702 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 01:03:50.702 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:03:50.702 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 01:03:50.702 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:03:50.702 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:03:50.702 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:03:50.702 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 01:03:50.702 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 01:03:50.702 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:03:50.702 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 01:03:50.702 | INFO | ------------------------------------------ | 20260506 01:03:50.702 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:03:50.702 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 01:03:50.702 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:03:50.702 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:03:50.702 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.37e+01, 1.35e+01 / 8.15e-01 | 20260506 01:03:50.702 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.05e-01, 1.04e-02 / 6.20e-03 | 20260506 01:03:50.702 | INFO | PerfReport: 11.84%, Baseline -- 3.70e+00 / 4.53e+00, 5.66e-01 / 1.97e-01 | 20260506 01:03:50.702 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.53e+00, 1.91e-01 / 9.33e-02 | 20260506 01:03:50.702 | INFO | PerfReport: 15.99%, DME -- 4.00e+01 / 4.38e+01, 3.76e+00 / 8.17e-01 | 20260506 01:03:50.703 | INFO | PerfReport: 57.37%, FrameLabeling -- 1.79e+01 / 1.94e+01, 1.45e+00 / 1.09e+00 | 20260506 01:03:50.703 | INFO | PerfReport: 7.10%, PulseAccumulating -- 2.22e+00 / 2.73e+00, 4.23e-01 / 2.41e-01 | 20260506 01:03:50.703 | INFO | PerfReport: 3.07%, Metrics -- 9.58e-01 / 1.24e+00, 2.58e-01 / 1.13e-01 | 20260506 01:03:50.703 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.31e-01 / 3.87e-01, 5.57e-02 / 3.10e-02 | 20260506 01:03:50.703 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 01:03:50.703 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:03:50.703 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:03:50.703 | INFO | PerfReport: Full Execution -- 9.81e+01 / -nan, 2.45e+01 / 0.00e+00 | 20260506 01:03:50.703 | INFO | PerfReport: 22.32%, Upload -- 2.19e+01 / 0.00e+00, 7.48e+00 / 0.00e+00 | 20260506 01:03:50.703 | INFO | PerfReport: 21.87%, Download -- 2.15e+01 / 0.00e+00, 7.59e+00 / 0.00e+00 | 20260506 01:03:50.703 | INFO | PerfReport: 31.86%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 01:03:50.703 | INFO | PerfReport: 2.10%, LockUpload -- 2.06e+00 / 0.00e+00, 7.86e+00 / 0.00e+00 | 20260506 01:03:50.703 | INFO | PerfReport: 1.33%, LockDownload -- 1.31e+00 / 0.00e+00, 2.89e+00 / 0.00e+00 | 20260506 01:03:50.703 | INFO | PerfReport: 20.50%, LockCompute -- 2.01e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260506 01:03:50.703 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.59e-02 / 0.00e+00 | 20260506 01:03:50.821 | INFO | Starting a round of Baz IO | 20260506 01:03:53.967 | INFO | Populating RTMetrics report | 20260506 01:03:53.968 | INFO | RTMetrics report populated | 20260506 01:03:53.968 | INFO | Opening tmp RTMetrics json file | 20260506 01:03:53.969 | INFO | Serializing RTMetrics to json | 20260506 01:03:53.970 | INFO | Serialization done, swapping tmp file now... | 20260506 01:03:53.970 | INFO | RTMetrics done | 20260506 01:03:54.046 | INFO | Finished a round of Baz IO: Wrote 1.836 GiB in 3.224 seconds | 20260506 01:03:54.046 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.34% / 0.00% / 78.66% | 20260506 01:03:54.046 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0843008% / 3.97364e-06% / 8.74201e-05% | 20260506 01:03:54.046 | INFO | Baz IO overall: | 20260506 01:03:54.046 | INFO | BytesWritten : 4370208294437 | 20260506 01:03:54.046 | INFO | NumEvents : 2055082230291 | 20260506 01:03:55.837 | INFO | Analyzing chunk frames = [7725056,7725568) | 20260506 01:03:55.837 | INFO | MemInfo: { | 20260506 01:03:55.837 | INFO | "gpu" : " 17 GB", | 20260506 01:03:55.837 | INFO | "huge" : "", | 20260506 01:03:55.837 | INFO | "huge_shared" : " 27 GB", | 20260506 01:03:55.837 | INFO | "malloc" : " 14 GB", | 20260506 01:03:55.837 | INFO | "pinned" : " 29 GB" | 20260506 01:03:55.837 | INFO | } | 20260506 01:03:55.837 | INFO | Perf report for the last 5.13599 seconds: | 20260506 01:03:55.837 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:03:55.837 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:55.837 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:55.837 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:55.837 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260506 01:03:55.837 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:03:55.837 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260506 01:03:55.837 | INFO | PRE_HQ : 59.8%, 1.7, 54.2ms, 0 | 20260506 01:03:55.837 | INFO | BAZWRITER : 61.9%, 3.5, 110.8ms, 0 | 20260506 01:03:55.837 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:03:55.837 | INFO | Uploaded 36GiB to GPU in 2.16072s (16.6611GiB/s) | 20260506 01:03:55.837 | INFO | Downloaded 38GiB to GPU in 2.23258s (17.0207GiB/s) | 20260506 01:04:00.982 | INFO | Analyzing chunk frames = [7725568,7726080) | 20260506 01:04:00.982 | INFO | MemInfo: { | 20260506 01:04:00.982 | INFO | "gpu" : " 17 GB", | 20260506 01:04:00.982 | INFO | "huge" : "", | 20260506 01:04:00.982 | INFO | "huge_shared" : " 27 GB", | 20260506 01:04:00.982 | INFO | "malloc" : " 14 GB", | 20260506 01:04:00.982 | INFO | "pinned" : " 29 GB" | 20260506 01:04:00.982 | INFO | } | 20260506 01:04:00.982 | INFO | Perf report for the last 5.14544 seconds: | 20260506 01:04:00.982 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:04:00.982 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:00.982 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:00.982 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:00.982 | INFO | ANALYSIS : 61.6%, 3.0, 96.3ms, 0 | 20260506 01:04:00.982 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:00.982 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:00.982 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 01:04:00.982 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:00.982 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:00.983 | INFO | Uploaded 36GiB to GPU in 2.11223s (17.0436GiB/s) | 20260506 01:04:00.983 | INFO | Downloaded 34GiB to GPU in 2.04068s (16.6611GiB/s) | 20260506 01:04:06.122 | INFO | Analyzing chunk frames = [7726080,7726592) | 20260506 01:04:06.122 | INFO | MemInfo: { | 20260506 01:04:06.122 | INFO | "gpu" : " 17 GB", | 20260506 01:04:06.122 | INFO | "huge" : "", | 20260506 01:04:06.122 | INFO | "huge_shared" : " 27 GB", | 20260506 01:04:06.123 | INFO | "malloc" : " 14 GB", | 20260506 01:04:06.123 | INFO | "pinned" : " 29 GB" | 20260506 01:04:06.123 | INFO | } | 20260506 01:04:06.123 | INFO | Perf report for the last 5.14036 seconds: | 20260506 01:04:06.123 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:04:06.123 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:06.123 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:06.123 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:06.123 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 01:04:06.123 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:06.123 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:06.123 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:04:06.123 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:06.123 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:06.123 | INFO | Uploaded 36GiB to GPU in 2.09361s (17.1952GiB/s) | 20260506 01:04:06.123 | INFO | Downloaded 34GiB to GPU in 2.02421s (16.7967GiB/s) | 20260506 01:04:11.262 | INFO | Analyzing chunk frames = [7726592,7727104) | 20260506 01:04:11.262 | INFO | MemInfo: { | 20260506 01:04:11.262 | INFO | "gpu" : " 17 GB", | 20260506 01:04:11.262 | INFO | "huge" : "", | 20260506 01:04:11.262 | INFO | "huge_shared" : " 27 GB", | 20260506 01:04:11.262 | INFO | "malloc" : " 14 GB", | 20260506 01:04:11.262 | INFO | "pinned" : " 29 GB" | 20260506 01:04:11.262 | INFO | } | 20260506 01:04:11.262 | INFO | Perf report for the last 5.13897 seconds: | 20260506 01:04:11.262 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:04:11.262 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:11.262 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:11.262 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:11.262 | INFO | ANALYSIS : 61.3%, 3.0, 95.2ms, 0 | 20260506 01:04:11.262 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:11.262 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:11.262 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 01:04:11.262 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:11.262 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:11.262 | INFO | Uploaded 36GiB to GPU in 2.10902s (17.0696GiB/s) | 20260506 01:04:11.262 | INFO | Downloaded 34GiB to GPU in 2.03172s (16.7346GiB/s) | 20260506 01:04:16.395 | INFO | Analyzing chunk frames = [7727104,7727616) | 20260506 01:04:16.395 | INFO | MemInfo: { | 20260506 01:04:16.395 | INFO | "gpu" : " 17 GB", | 20260506 01:04:16.396 | INFO | "huge" : "", | 20260506 01:04:16.396 | INFO | "huge_shared" : " 27 GB", | 20260506 01:04:16.396 | INFO | "malloc" : " 14 GB", | 20260506 01:04:16.396 | INFO | "pinned" : " 29 GB" | 20260506 01:04:16.396 | INFO | } | 20260506 01:04:16.396 | INFO | Perf report for the last 5.13405 seconds: | 20260506 01:04:16.396 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:04:16.396 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:16.396 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:16.396 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:16.396 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 01:04:16.396 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:16.396 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:16.396 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 01:04:16.396 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:16.396 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:16.396 | INFO | Uploaded 36GiB to GPU in 2.11002s (17.0614GiB/s) | 20260506 01:04:16.396 | INFO | Downloaded 34GiB to GPU in 2.04673s (16.6118GiB/s) | 20260506 01:04:21.539 | INFO | Analyzing chunk frames = [7727616,7728128) | 20260506 01:04:21.539 | INFO | MemInfo: { | 20260506 01:04:21.539 | INFO | "gpu" : " 17 GB", | 20260506 01:04:21.539 | INFO | "huge" : "", | 20260506 01:04:21.539 | INFO | "huge_shared" : " 27 GB", | 20260506 01:04:21.539 | INFO | "malloc" : " 14 GB", | 20260506 01:04:21.539 | INFO | "pinned" : " 29 GB" | 20260506 01:04:21.539 | INFO | } | 20260506 01:04:21.539 | INFO | Perf report for the last 5.14296 seconds: | 20260506 01:04:21.539 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:04:21.539 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:21.539 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:21.539 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:21.539 | INFO | ANALYSIS : 61.2%, 3.0, 95.6ms, 0 | 20260506 01:04:21.539 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:21.539 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:21.539 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 01:04:21.539 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:21.539 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:21.539 | INFO | Uploaded 36GiB to GPU in 2.0893s (17.2306GiB/s) | 20260506 01:04:21.539 | INFO | Downloaded 34GiB to GPU in 2.04722s (16.6079GiB/s) | 20260506 01:04:26.683 | INFO | Analyzing chunk frames = [7728128,7728640) | 20260506 01:04:26.683 | INFO | MemInfo: { | 20260506 01:04:26.683 | INFO | "gpu" : " 17 GB", | 20260506 01:04:26.683 | INFO | "huge" : "", | 20260506 01:04:26.683 | INFO | "huge_shared" : " 27 GB", | 20260506 01:04:26.683 | INFO | "malloc" : " 14 GB", | 20260506 01:04:26.683 | INFO | "pinned" : " 29 GB" | 20260506 01:04:26.683 | INFO | } | 20260506 01:04:26.683 | INFO | Perf report for the last 5.14427 seconds: | 20260506 01:04:26.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:04:26.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:26.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:26.683 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:26.683 | INFO | ANALYSIS : 62.1%, 3.0, 97.1ms, 0 | 20260506 01:04:26.683 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:26.683 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:26.683 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 01:04:26.683 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:26.683 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:26.683 | INFO | Uploaded 36GiB to GPU in 2.03473s (17.6928GiB/s) | 20260506 01:04:26.683 | INFO | Downloaded 34GiB to GPU in 2.047s (16.6096GiB/s) | 20260506 01:04:31.820 | INFO | Analyzing chunk frames = [7728640,7729152) | 20260506 01:04:31.820 | INFO | MemInfo: { | 20260506 01:04:31.820 | INFO | "gpu" : " 17 GB", | 20260506 01:04:31.820 | INFO | "huge" : "", | 20260506 01:04:31.820 | INFO | "huge_shared" : " 27 GB", | 20260506 01:04:31.820 | INFO | "malloc" : " 14 GB", | 20260506 01:04:31.820 | INFO | "pinned" : " 29 GB" | 20260506 01:04:31.820 | INFO | } | 20260506 01:04:31.820 | INFO | Perf report for the last 5.13748 seconds: | 20260506 01:04:31.820 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:04:31.820 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:31.820 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:31.820 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:31.821 | INFO | ANALYSIS : 61.4%, 3.0, 95.9ms, 0 | 20260506 01:04:31.821 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:31.821 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:31.821 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:04:31.821 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:31.821 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:31.821 | INFO | Uploaded 36GiB to GPU in 2.04526s (17.6017GiB/s) | 20260506 01:04:31.821 | INFO | Downloaded 34GiB to GPU in 2.01622s (16.8632GiB/s) | 20260506 01:04:31.946 | INFO | Starting a round of Baz IO | 20260506 01:04:35.085 | INFO | Populating RTMetrics report | 20260506 01:04:35.085 | INFO | RTMetrics report populated | 20260506 01:04:35.085 | INFO | Opening tmp RTMetrics json file | 20260506 01:04:35.086 | INFO | Serializing RTMetrics to json | 20260506 01:04:35.087 | INFO | Serialization done, swapping tmp file now... | 20260506 01:04:35.088 | INFO | RTMetrics done | 20260506 01:04:35.163 | INFO | Finished a round of Baz IO: Wrote 1.839 GiB in 3.216 seconds | 20260506 01:04:35.163 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.34% / 0.00% / 78.66% | 20260506 01:04:35.163 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0840743% / 0% / 7.15256e-05% | 20260506 01:04:35.163 | INFO | Baz IO overall: | 20260506 01:04:35.163 | INFO | BytesWritten : 4372183351237 | 20260506 01:04:35.163 | INFO | NumEvents : 2055983516791 | 20260506 01:04:36.956 | INFO | Analyzing chunk frames = [7729152,7729664) | 20260506 01:04:36.956 | INFO | MemInfo: { | 20260506 01:04:36.956 | INFO | "gpu" : " 17 GB", | 20260506 01:04:36.957 | INFO | "huge" : "", | 20260506 01:04:36.957 | INFO | "huge_shared" : " 27 GB", | 20260506 01:04:36.957 | INFO | "malloc" : " 14 GB", | 20260506 01:04:36.957 | INFO | "pinned" : " 29 GB" | 20260506 01:04:36.957 | INFO | } | 20260506 01:04:36.957 | INFO | Perf report for the last 5.13627 seconds: | 20260506 01:04:36.957 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:04:36.957 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:36.957 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:36.957 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:36.957 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260506 01:04:36.957 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:36.957 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260506 01:04:36.957 | INFO | PRE_HQ : 59.6%, 1.7, 54.0ms, 0 | 20260506 01:04:36.957 | INFO | BAZWRITER : 61.9%, 3.5, 110.8ms, 0 | 20260506 01:04:36.957 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:04:36.957 | INFO | Uploaded 36GiB to GPU in 2.18652s (16.4645GiB/s) | 20260506 01:04:36.957 | INFO | Downloaded 38GiB to GPU in 2.24137s (16.9539GiB/s) | 20260506 01:04:42.089 | INFO | Analyzing chunk frames = [7729664,7730176) | 20260506 01:04:42.089 | INFO | MemInfo: { | 20260506 01:04:42.089 | INFO | "gpu" : " 17 GB", | 20260506 01:04:42.089 | INFO | "huge" : "", | 20260506 01:04:42.089 | INFO | "huge_shared" : " 27 GB", | 20260506 01:04:42.089 | INFO | "malloc" : " 14 GB", | 20260506 01:04:42.090 | INFO | "pinned" : " 29 GB" | 20260506 01:04:42.090 | INFO | } | 20260506 01:04:42.090 | INFO | Perf report for the last 5.13287 seconds: | 20260506 01:04:42.090 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:04:42.090 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:42.090 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:42.090 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:42.090 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 01:04:42.090 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:42.090 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:42.090 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:04:42.090 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:42.090 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:42.090 | INFO | Uploaded 36GiB to GPU in 2.11974s (16.9832GiB/s) | 20260506 01:04:42.090 | INFO | Downloaded 34GiB to GPU in 2.04389s (16.635GiB/s) | 20260506 01:04:47.238 | INFO | Analyzing chunk frames = [7730176,7730688) | 20260506 01:04:47.238 | INFO | MemInfo: { | 20260506 01:04:47.238 | INFO | "gpu" : " 17 GB", | 20260506 01:04:47.238 | INFO | "huge" : "", | 20260506 01:04:47.238 | INFO | "huge_shared" : " 27 GB", | 20260506 01:04:47.238 | INFO | "malloc" : " 14 GB", | 20260506 01:04:47.238 | INFO | "pinned" : " 29 GB" | 20260506 01:04:47.238 | INFO | } | 20260506 01:04:47.238 | INFO | Perf report for the last 5.14872 seconds: | 20260506 01:04:47.238 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:04:47.238 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:47.238 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:47.238 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:47.238 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 01:04:47.238 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:47.238 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:47.238 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:04:47.238 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:47.238 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:47.239 | INFO | Uploaded 36GiB to GPU in 2.08723s (17.2477GiB/s) | 20260506 01:04:47.239 | INFO | Downloaded 34GiB to GPU in 2.02536s (16.7872GiB/s) | 20260506 01:04:52.385 | INFO | Analyzing chunk frames = [7730688,7731200) | 20260506 01:04:52.385 | INFO | MemInfo: { | 20260506 01:04:52.385 | INFO | "gpu" : " 17 GB", | 20260506 01:04:52.385 | INFO | "huge" : "", | 20260506 01:04:52.385 | INFO | "huge_shared" : " 27 GB", | 20260506 01:04:52.385 | INFO | "malloc" : " 14 GB", | 20260506 01:04:52.385 | INFO | "pinned" : " 29 GB" | 20260506 01:04:52.385 | INFO | } | 20260506 01:04:52.385 | INFO | Perf report for the last 5.14707 seconds: | 20260506 01:04:52.385 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:04:52.385 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:52.385 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:52.385 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:52.385 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 01:04:52.386 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:52.386 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:52.386 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 01:04:52.386 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:52.386 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:52.386 | INFO | Uploaded 36GiB to GPU in 2.15004s (16.7439GiB/s) | 20260506 01:04:52.386 | INFO | Downloaded 34GiB to GPU in 2.0557s (16.5394GiB/s) | 20260506 01:04:57.511 | INFO | Analyzing chunk frames = [7731200,7731712) | 20260506 01:04:57.511 | INFO | MemInfo: { | 20260506 01:04:57.511 | INFO | "gpu" : " 17 GB", | 20260506 01:04:57.511 | INFO | "huge" : "", | 20260506 01:04:57.511 | INFO | "huge_shared" : " 27 GB", | 20260506 01:04:57.511 | INFO | "malloc" : " 14 GB", | 20260506 01:04:57.511 | INFO | "pinned" : " 29 GB" | 20260506 01:04:57.511 | INFO | } | 20260506 01:04:57.511 | INFO | Perf report for the last 5.12538 seconds: | 20260506 01:04:57.511 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:04:57.511 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:57.511 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:57.511 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:57.511 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260506 01:04:57.511 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:57.511 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:57.511 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 01:04:57.511 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:57.511 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:04:57.511 | INFO | Uploaded 36GiB to GPU in 2.07678s (17.3346GiB/s) | 20260506 01:04:57.511 | INFO | Downloaded 34GiB to GPU in 2.03346s (16.7203GiB/s) | 20260506 01:05:02.649 | INFO | Analyzing chunk frames = [7731712,7732224) | 20260506 01:05:02.649 | INFO | MemInfo: { | 20260506 01:05:02.649 | INFO | "gpu" : " 17 GB", | 20260506 01:05:02.649 | INFO | "huge" : "", | 20260506 01:05:02.649 | INFO | "huge_shared" : " 27 GB", | 20260506 01:05:02.649 | INFO | "malloc" : " 14 GB", | 20260506 01:05:02.649 | INFO | "pinned" : " 29 GB" | 20260506 01:05:02.649 | INFO | } | 20260506 01:05:02.649 | INFO | Perf report for the last 5.13866 seconds: | 20260506 01:05:02.649 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:05:02.649 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:02.650 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:02.650 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:02.650 | INFO | ANALYSIS : 61.2%, 3.0, 95.6ms, 0 | 20260506 01:05:02.650 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:02.650 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:02.650 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:05:02.650 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:02.650 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:02.650 | INFO | Uploaded 36GiB to GPU in 2.06216s (17.4574GiB/s) | 20260506 01:05:02.650 | INFO | Downloaded 34GiB to GPU in 2.03393s (16.7164GiB/s) | 20260506 01:05:07.789 | INFO | Analyzing chunk frames = [7732224,7732736) | 20260506 01:05:07.789 | INFO | MemInfo: { | 20260506 01:05:07.789 | INFO | "gpu" : " 17 GB", | 20260506 01:05:07.789 | INFO | "huge" : "", | 20260506 01:05:07.789 | INFO | "huge_shared" : " 27 GB", | 20260506 01:05:07.789 | INFO | "malloc" : " 14 GB", | 20260506 01:05:07.789 | INFO | "pinned" : " 29 GB" | 20260506 01:05:07.789 | INFO | } | 20260506 01:05:07.790 | INFO | Perf report for the last 5.14009 seconds: | 20260506 01:05:07.790 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:05:07.790 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:07.790 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:07.790 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:07.790 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260506 01:05:07.790 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:07.790 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:07.790 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 01:05:07.790 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:07.790 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:07.790 | INFO | Uploaded 36GiB to GPU in 2.10135s (17.1318GiB/s) | 20260506 01:05:07.790 | INFO | Downloaded 34GiB to GPU in 2.08674s (16.2934GiB/s) | 20260506 01:05:12.932 | INFO | Analyzing chunk frames = [7732736,7733248) | 20260506 01:05:12.932 | INFO | MemInfo: { | 20260506 01:05:12.932 | INFO | "gpu" : " 17 GB", | 20260506 01:05:12.932 | INFO | "huge" : "", | 20260506 01:05:12.932 | INFO | "huge_shared" : " 27 GB", | 20260506 01:05:12.932 | INFO | "malloc" : " 14 GB", | 20260506 01:05:12.932 | INFO | "pinned" : " 29 GB" | 20260506 01:05:12.932 | INFO | } | 20260506 01:05:12.932 | INFO | Perf report for the last 5.14277 seconds: | 20260506 01:05:12.932 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:05:12.932 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:12.932 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:12.932 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:12.932 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 01:05:12.932 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:12.932 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:12.932 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:05:12.932 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:12.933 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:12.933 | INFO | Uploaded 36GiB to GPU in 2.13117s (16.8921GiB/s) | 20260506 01:05:12.933 | INFO | Downloaded 34GiB to GPU in 2.02807s (16.7647GiB/s) | 20260506 01:05:12.933 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 01:05:12.933 | INFO | Total mem usage: 15.3 GiB | 20260506 01:05:12.933 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 01:05:12.933 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.18%) | 20260506 01:05:12.933 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 01:05:12.933 | INFO | High Water Breakdown: | 20260506 01:05:12.933 | INFO | 60.9% : 8.55 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 01:05:12.933 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 01:05:12.933 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 01:05:12.933 | INFO | ------------------------------------------ | 20260506 01:05:12.933 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:05:12.933 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 01:05:12.933 | INFO | Total mem usage: 26.5 GiB | 20260506 01:05:12.933 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 01:05:12.933 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 01:05:12.933 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 01:05:12.933 | INFO | High Water Breakdown: | 20260506 01:05:12.933 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:05:12.933 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:05:12.933 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:05:12.933 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:05:12.933 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:05:12.933 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:05:12.933 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 01:05:12.933 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 01:05:12.933 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:05:12.933 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 01:05:12.933 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 01:05:12.933 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 01:05:12.933 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 01:05:12.933 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 01:05:12.933 | INFO | ------------------------------------------ | 20260506 01:05:12.933 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:05:12.933 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 01:05:12.933 | INFO | Total mem usage: 25.6 GiB | 20260506 01:05:12.933 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 01:05:12.933 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 01:05:12.933 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 01:05:12.933 | INFO | High Water Breakdown: | 20260506 01:05:12.933 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 01:05:12.933 | INFO | ------------------------------------------ | 20260506 01:05:12.933 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:05:12.933 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 01:05:12.933 | INFO | Total mem usage: 15.7 GiB | 20260506 01:05:12.933 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 01:05:12.933 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 01:05:12.933 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 01:05:12.933 | INFO | High Water Breakdown: | 20260506 01:05:12.933 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 01:05:12.933 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 01:05:12.933 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 01:05:12.933 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 01:05:12.933 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 01:05:12.934 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 01:05:12.934 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 01:05:12.934 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 01:05:12.934 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 01:05:12.934 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:05:12.934 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 01:05:12.934 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:05:12.934 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 01:05:12.934 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 01:05:12.934 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 01:05:12.934 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 01:05:12.934 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:05:12.934 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 01:05:12.934 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:05:12.934 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:05:12.934 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:05:12.934 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 01:05:12.934 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 01:05:12.934 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:05:12.934 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 01:05:12.934 | INFO | ------------------------------------------ | 20260506 01:05:12.934 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:05:12.934 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 01:05:12.934 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:05:12.934 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:05:12.934 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.40e+01, 1.35e+01 / 9.38e-01 | 20260506 01:05:12.934 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.04e-01, 1.03e-02 / 6.04e-03 | 20260506 01:05:12.934 | INFO | PerfReport: 11.82%, Baseline -- 3.70e+00 / 4.48e+00, 5.64e-01 / 2.20e-01 | 20260506 01:05:12.934 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.51e+00, 1.88e-01 / 8.84e-02 | 20260506 01:05:12.934 | INFO | PerfReport: 15.99%, DME -- 4.01e+01 / 4.40e+01, 3.71e+00 / 1.10e+00 | 20260506 01:05:12.934 | INFO | PerfReport: 57.45%, FrameLabeling -- 1.80e+01 / 1.96e+01, 1.53e+00 / 1.11e+00 | 20260506 01:05:12.934 | INFO | PerfReport: 7.06%, PulseAccumulating -- 2.21e+00 / 2.71e+00, 4.20e-01 / 2.39e-01 | 20260506 01:05:12.934 | INFO | PerfReport: 3.05%, Metrics -- 9.56e-01 / 1.23e+00, 2.60e-01 / 1.18e-01 | 20260506 01:05:12.934 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.32e-01 / 3.80e-01, 5.62e-02 / 2.51e-02 | 20260506 01:05:12.934 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 01:05:12.934 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:05:12.934 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:05:12.934 | INFO | PerfReport: Full Execution -- 9.81e+01 / -nan, 2.44e+01 / 0.00e+00 | 20260506 01:05:12.934 | INFO | PerfReport: 22.34%, Upload -- 2.19e+01 / 0.00e+00, 7.47e+00 / 0.00e+00 | 20260506 01:05:12.934 | INFO | PerfReport: 21.92%, Download -- 2.15e+01 / 0.00e+00, 7.63e+00 / 0.00e+00 | 20260506 01:05:12.934 | INFO | PerfReport: 31.92%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 01:05:12.934 | INFO | PerfReport: 2.05%, LockUpload -- 2.01e+00 / 0.00e+00, 7.90e+00 / 0.00e+00 | 20260506 01:05:12.934 | INFO | PerfReport: 1.29%, LockDownload -- 1.27e+00 / 0.00e+00, 2.83e+00 / 0.00e+00 | 20260506 01:05:12.934 | INFO | PerfReport: 20.45%, LockCompute -- 2.01e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 01:05:12.934 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 2.64e-02 / 0.00e+00 | 20260506 01:05:13.071 | INFO | Starting a round of Baz IO | 20260506 01:05:16.194 | INFO | Populating RTMetrics report | 20260506 01:05:16.194 | INFO | RTMetrics report populated | 20260506 01:05:16.194 | INFO | Opening tmp RTMetrics json file | 20260506 01:05:16.195 | INFO | Serializing RTMetrics to json | 20260506 01:05:16.196 | INFO | Serialization done, swapping tmp file now... | 20260506 01:05:16.197 | INFO | RTMetrics done | 20260506 01:05:16.268 | INFO | Finished a round of Baz IO: Wrote 1.837 GiB in 3.197 seconds | 20260506 01:05:16.268 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.33% / 0.00% / 78.66% | 20260506 01:05:16.268 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0847141% / 0% / 8.34465e-05% | 20260506 01:05:16.268 | INFO | Baz IO overall: | 20260506 01:05:16.268 | INFO | BytesWritten : 4374155568845 | 20260506 01:05:16.268 | INFO | NumEvents : 2056882727878 | 20260506 01:05:18.068 | INFO | Analyzing chunk frames = [7733248,7733760) | 20260506 01:05:18.068 | INFO | MemInfo: { | 20260506 01:05:18.068 | INFO | "gpu" : " 17 GB", | 20260506 01:05:18.068 | INFO | "huge" : "", | 20260506 01:05:18.068 | INFO | "huge_shared" : " 27 GB", | 20260506 01:05:18.068 | INFO | "malloc" : " 14 GB", | 20260506 01:05:18.068 | INFO | "pinned" : " 29 GB" | 20260506 01:05:18.068 | INFO | } | 20260506 01:05:18.068 | INFO | Perf report for the last 5.136 seconds: | 20260506 01:05:18.069 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:05:18.069 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:18.069 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:18.069 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:18.069 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260506 01:05:18.069 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:18.069 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260506 01:05:18.069 | INFO | PRE_HQ : 59.7%, 1.7, 53.8ms, 0 | 20260506 01:05:18.069 | INFO | BAZWRITER : 62.2%, 3.4, 110.2ms, 0 | 20260506 01:05:18.069 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:05:18.069 | INFO | Uploaded 36GiB to GPU in 2.18814s (16.4524GiB/s) | 20260506 01:05:18.069 | INFO | Downloaded 38GiB to GPU in 2.24725s (16.9096GiB/s) | 20260506 01:05:23.206 | INFO | Analyzing chunk frames = [7733760,7734272) | 20260506 01:05:23.206 | INFO | MemInfo: { | 20260506 01:05:23.206 | INFO | "gpu" : " 17 GB", | 20260506 01:05:23.206 | INFO | "huge" : "", | 20260506 01:05:23.206 | INFO | "huge_shared" : " 27 GB", | 20260506 01:05:23.206 | INFO | "malloc" : " 14 GB", | 20260506 01:05:23.206 | INFO | "pinned" : " 29 GB" | 20260506 01:05:23.206 | INFO | } | 20260506 01:05:23.206 | INFO | Perf report for the last 5.13793 seconds: | 20260506 01:05:23.206 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:05:23.206 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:23.206 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:23.206 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:23.206 | INFO | ANALYSIS : 61.4%, 3.0, 95.9ms, 0 | 20260506 01:05:23.206 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:23.206 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:23.206 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 01:05:23.206 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:23.206 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:23.207 | INFO | Uploaded 36GiB to GPU in 2.02262s (17.7987GiB/s) | 20260506 01:05:23.207 | INFO | Downloaded 34GiB to GPU in 2.00671s (16.9432GiB/s) | 20260506 01:05:28.347 | INFO | Analyzing chunk frames = [7734272,7734784) | 20260506 01:05:28.347 | INFO | MemInfo: { | 20260506 01:05:28.347 | INFO | "gpu" : " 17 GB", | 20260506 01:05:28.348 | INFO | "huge" : "", | 20260506 01:05:28.348 | INFO | "huge_shared" : " 27 GB", | 20260506 01:05:28.348 | INFO | "malloc" : " 14 GB", | 20260506 01:05:28.348 | INFO | "pinned" : " 29 GB" | 20260506 01:05:28.348 | INFO | } | 20260506 01:05:28.348 | INFO | Perf report for the last 5.14135 seconds: | 20260506 01:05:28.348 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:05:28.348 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:28.348 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:28.348 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:28.348 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 01:05:28.348 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:28.348 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:28.348 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 01:05:28.348 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:28.348 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:28.348 | INFO | Uploaded 36GiB to GPU in 2.11243s (17.042GiB/s) | 20260506 01:05:28.348 | INFO | Downloaded 34GiB to GPU in 2.02553s (16.7857GiB/s) | 20260506 01:05:33.490 | INFO | Analyzing chunk frames = [7734784,7735296) | 20260506 01:05:33.490 | INFO | MemInfo: { | 20260506 01:05:33.490 | INFO | "gpu" : " 17 GB", | 20260506 01:05:33.490 | INFO | "huge" : "", | 20260506 01:05:33.490 | INFO | "huge_shared" : " 27 GB", | 20260506 01:05:33.490 | INFO | "malloc" : " 14 GB", | 20260506 01:05:33.490 | INFO | "pinned" : " 29 GB" | 20260506 01:05:33.490 | INFO | } | 20260506 01:05:33.490 | INFO | Perf report for the last 5.14228 seconds: | 20260506 01:05:33.490 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:05:33.490 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:33.490 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:33.490 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:33.490 | INFO | ANALYSIS : 61.6%, 3.0, 95.7ms, 0 | 20260506 01:05:33.490 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:33.490 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:33.490 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 01:05:33.490 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:33.490 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:33.490 | INFO | Uploaded 36GiB to GPU in 2.13777s (16.84GiB/s) | 20260506 01:05:33.490 | INFO | Downloaded 34GiB to GPU in 2.02114s (16.8222GiB/s) | 20260506 01:05:38.633 | INFO | Analyzing chunk frames = [7735296,7735808) | 20260506 01:05:38.633 | INFO | MemInfo: { | 20260506 01:05:38.633 | INFO | "gpu" : " 17 GB", | 20260506 01:05:38.633 | INFO | "huge" : "", | 20260506 01:05:38.633 | INFO | "huge_shared" : " 27 GB", | 20260506 01:05:38.633 | INFO | "malloc" : " 14 GB", | 20260506 01:05:38.633 | INFO | "pinned" : " 29 GB" | 20260506 01:05:38.633 | INFO | } | 20260506 01:05:38.633 | INFO | Perf report for the last 5.14331 seconds: | 20260506 01:05:38.633 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:05:38.633 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:38.633 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:38.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:38.633 | INFO | ANALYSIS : 60.8%, 3.0, 94.9ms, 0 | 20260506 01:05:38.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:38.633 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:38.633 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:05:38.633 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:38.633 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:38.634 | INFO | Uploaded 36GiB to GPU in 2.06186s (17.46GiB/s) | 20260506 01:05:38.634 | INFO | Downloaded 34GiB to GPU in 2.0247s (16.7926GiB/s) | 20260506 01:05:43.770 | INFO | Analyzing chunk frames = [7735808,7736320) | 20260506 01:05:43.770 | INFO | MemInfo: { | 20260506 01:05:43.770 | INFO | "gpu" : " 17 GB", | 20260506 01:05:43.770 | INFO | "huge" : "", | 20260506 01:05:43.771 | INFO | "huge_shared" : " 27 GB", | 20260506 01:05:43.771 | INFO | "malloc" : " 14 GB", | 20260506 01:05:43.771 | INFO | "pinned" : " 29 GB" | 20260506 01:05:43.771 | INFO | } | 20260506 01:05:43.771 | INFO | Perf report for the last 5.13728 seconds: | 20260506 01:05:43.771 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:05:43.771 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:43.771 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:43.771 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:43.771 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 01:05:43.771 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:43.771 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:43.771 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 01:05:43.771 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:43.771 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:43.771 | INFO | Uploaded 36GiB to GPU in 2.07649s (17.3369GiB/s) | 20260506 01:05:43.771 | INFO | Downloaded 34GiB to GPU in 2.03419s (16.7143GiB/s) | 20260506 01:05:48.902 | INFO | Analyzing chunk frames = [7736320,7736832) | 20260506 01:05:48.902 | INFO | MemInfo: { | 20260506 01:05:48.902 | INFO | "gpu" : " 17 GB", | 20260506 01:05:48.902 | INFO | "huge" : "", | 20260506 01:05:48.902 | INFO | "huge_shared" : " 27 GB", | 20260506 01:05:48.902 | INFO | "malloc" : " 14 GB", | 20260506 01:05:48.902 | INFO | "pinned" : " 29 GB" | 20260506 01:05:48.902 | INFO | } | 20260506 01:05:48.902 | INFO | Perf report for the last 5.13147 seconds: | 20260506 01:05:48.902 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:05:48.902 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:48.902 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:48.902 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:48.902 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 01:05:48.902 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:48.902 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:48.902 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 01:05:48.902 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:48.902 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:48.902 | INFO | Uploaded 36GiB to GPU in 2.09675s (17.1695GiB/s) | 20260506 01:05:48.902 | INFO | Downloaded 34GiB to GPU in 2.08446s (16.3112GiB/s) | 20260506 01:05:54.042 | INFO | Analyzing chunk frames = [7736832,7737344) | 20260506 01:05:54.042 | INFO | MemInfo: { | 20260506 01:05:54.042 | INFO | "gpu" : " 17 GB", | 20260506 01:05:54.042 | INFO | "huge" : "", | 20260506 01:05:54.042 | INFO | "huge_shared" : " 27 GB", | 20260506 01:05:54.042 | INFO | "malloc" : " 14 GB", | 20260506 01:05:54.042 | INFO | "pinned" : " 29 GB" | 20260506 01:05:54.042 | INFO | } | 20260506 01:05:54.042 | INFO | Perf report for the last 5.13989 seconds: | 20260506 01:05:54.042 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:05:54.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:54.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:54.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:54.042 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260506 01:05:54.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:54.042 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:54.042 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:05:54.042 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:54.042 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:54.042 | INFO | Uploaded 36GiB to GPU in 2.11337s (17.0344GiB/s) | 20260506 01:05:54.042 | INFO | Downloaded 34GiB to GPU in 2.01063s (16.9101GiB/s) | 20260506 01:05:54.161 | INFO | Starting a round of Baz IO | 20260506 01:05:57.307 | INFO | Populating RTMetrics report | 20260506 01:05:57.307 | INFO | RTMetrics report populated | 20260506 01:05:57.307 | INFO | Opening tmp RTMetrics json file | 20260506 01:05:57.307 | INFO | Serializing RTMetrics to json | 20260506 01:05:57.309 | INFO | Serialization done, swapping tmp file now... | 20260506 01:05:57.309 | INFO | RTMetrics done | 20260506 01:05:57.385 | INFO | Finished a round of Baz IO: Wrote 1.836 GiB in 3.224 seconds | 20260506 01:05:57.385 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.33% / 0.00% / 78.67% | 20260506 01:05:57.385 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0842333% / 3.97364e-06% / 8.74201e-05% | 20260506 01:05:57.385 | INFO | Baz IO overall: | 20260506 01:05:57.385 | INFO | BytesWritten : 4376126717624 | 20260506 01:05:57.385 | INFO | NumEvents : 2057781320368 | 20260506 01:05:59.188 | INFO | Analyzing chunk frames = [7737344,7737856) | 20260506 01:05:59.188 | INFO | MemInfo: { | 20260506 01:05:59.188 | INFO | "gpu" : " 17 GB", | 20260506 01:05:59.188 | INFO | "huge" : "", | 20260506 01:05:59.188 | INFO | "huge_shared" : " 27 GB", | 20260506 01:05:59.188 | INFO | "malloc" : " 14 GB", | 20260506 01:05:59.188 | INFO | "pinned" : " 29 GB" | 20260506 01:05:59.188 | INFO | } | 20260506 01:05:59.188 | INFO | Perf report for the last 5.14637 seconds: | 20260506 01:05:59.188 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:05:59.188 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:59.188 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:59.188 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:59.188 | INFO | ANALYSIS : 62.4%, 3.0, 97.6ms, 0 | 20260506 01:05:59.188 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:05:59.188 | INFO | RT_METRICS : 43.0%, 1.0, 22.6ms, 0 | 20260506 01:05:59.188 | INFO | PRE_HQ : 59.8%, 1.7, 53.7ms, 0 | 20260506 01:05:59.188 | INFO | BAZWRITER : 61.8%, 3.5, 110.0ms, 0 | 20260506 01:05:59.188 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:05:59.189 | INFO | Uploaded 36GiB to GPU in 2.17708s (16.5359GiB/s) | 20260506 01:05:59.189 | INFO | Downloaded 38GiB to GPU in 2.24727s (16.9094GiB/s) | 20260506 01:06:04.337 | INFO | Analyzing chunk frames = [7737856,7738368) | 20260506 01:06:04.337 | INFO | MemInfo: { | 20260506 01:06:04.337 | INFO | "gpu" : " 17 GB", | 20260506 01:06:04.337 | INFO | "huge" : "", | 20260506 01:06:04.337 | INFO | "huge_shared" : " 27 GB", | 20260506 01:06:04.337 | INFO | "malloc" : " 14 GB", | 20260506 01:06:04.337 | INFO | "pinned" : " 29 GB" | 20260506 01:06:04.337 | INFO | } | 20260506 01:06:04.337 | INFO | Perf report for the last 5.14879 seconds: | 20260506 01:06:04.337 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:06:04.337 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:04.337 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:04.337 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:04.337 | INFO | ANALYSIS : 60.9%, 3.0, 95.3ms, 0 | 20260506 01:06:04.337 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:04.337 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:04.337 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 01:06:04.337 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:04.337 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:04.338 | INFO | Uploaded 36GiB to GPU in 1.96125s (18.3556GiB/s) | 20260506 01:06:04.338 | INFO | Downloaded 34GiB to GPU in 1.99591s (17.0349GiB/s) | 20260506 01:06:09.462 | INFO | Analyzing chunk frames = [7738368,7738880) | 20260506 01:06:09.462 | INFO | MemInfo: { | 20260506 01:06:09.462 | INFO | "gpu" : " 17 GB", | 20260506 01:06:09.462 | INFO | "huge" : "", | 20260506 01:06:09.462 | INFO | "huge_shared" : " 27 GB", | 20260506 01:06:09.462 | INFO | "malloc" : " 14 GB", | 20260506 01:06:09.462 | INFO | "pinned" : " 29 GB" | 20260506 01:06:09.462 | INFO | } | 20260506 01:06:09.463 | INFO | Perf report for the last 5.12537 seconds: | 20260506 01:06:09.463 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:06:09.463 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:09.463 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:09.463 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:09.463 | INFO | ANALYSIS : 61.3%, 3.0, 95.2ms, 0 | 20260506 01:06:09.463 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:09.463 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:09.463 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 01:06:09.463 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:09.463 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:09.463 | INFO | Uploaded 36GiB to GPU in 2.08325s (17.2807GiB/s) | 20260506 01:06:09.463 | INFO | Downloaded 34GiB to GPU in 2.02953s (16.7526GiB/s) | 20260506 01:06:14.602 | INFO | Analyzing chunk frames = [7738880,7739392) | 20260506 01:06:14.602 | INFO | MemInfo: { | 20260506 01:06:14.602 | INFO | "gpu" : " 17 GB", | 20260506 01:06:14.602 | INFO | "huge" : "", | 20260506 01:06:14.602 | INFO | "huge_shared" : " 27 GB", | 20260506 01:06:14.603 | INFO | "malloc" : " 14 GB", | 20260506 01:06:14.603 | INFO | "pinned" : " 29 GB" | 20260506 01:06:14.603 | INFO | } | 20260506 01:06:14.603 | INFO | Perf report for the last 5.14005 seconds: | 20260506 01:06:14.603 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:06:14.603 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:14.603 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:14.603 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:14.603 | INFO | ANALYSIS : 61.7%, 3.0, 95.8ms, 0 | 20260506 01:06:14.603 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:14.603 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:14.603 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:06:14.603 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:14.603 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:14.603 | INFO | Uploaded 36GiB to GPU in 2.13958s (16.8257GiB/s) | 20260506 01:06:14.603 | INFO | Downloaded 34GiB to GPU in 2.0279s (16.7661GiB/s) | 20260506 01:06:19.744 | INFO | Analyzing chunk frames = [7739392,7739904) | 20260506 01:06:19.744 | INFO | MemInfo: { | 20260506 01:06:19.744 | INFO | "gpu" : " 17 GB", | 20260506 01:06:19.744 | INFO | "huge" : "", | 20260506 01:06:19.744 | INFO | "huge_shared" : " 27 GB", | 20260506 01:06:19.744 | INFO | "malloc" : " 14 GB", | 20260506 01:06:19.744 | INFO | "pinned" : " 29 GB" | 20260506 01:06:19.744 | INFO | } | 20260506 01:06:19.744 | INFO | Perf report for the last 5.14166 seconds: | 20260506 01:06:19.744 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:06:19.744 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:19.744 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:19.744 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:19.744 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 01:06:19.744 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:19.744 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:19.744 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:06:19.744 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:19.744 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:19.745 | INFO | Uploaded 36GiB to GPU in 2.08274s (17.2849GiB/s) | 20260506 01:06:19.745 | INFO | Downloaded 34GiB to GPU in 2.04274s (16.6443GiB/s) | 20260506 01:06:24.882 | INFO | Analyzing chunk frames = [7739904,7740416) | 20260506 01:06:24.882 | INFO | MemInfo: { | 20260506 01:06:24.882 | INFO | "gpu" : " 17 GB", | 20260506 01:06:24.882 | INFO | "huge" : "", | 20260506 01:06:24.882 | INFO | "huge_shared" : " 27 GB", | 20260506 01:06:24.882 | INFO | "malloc" : " 14 GB", | 20260506 01:06:24.882 | INFO | "pinned" : " 29 GB" | 20260506 01:06:24.882 | INFO | } | 20260506 01:06:24.882 | INFO | Perf report for the last 5.13747 seconds: | 20260506 01:06:24.882 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:06:24.882 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:24.882 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:24.882 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:24.882 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 01:06:24.882 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:24.882 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:24.882 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:06:24.882 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:24.882 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:24.882 | INFO | Uploaded 36GiB to GPU in 2.06331s (17.4477GiB/s) | 20260506 01:06:24.882 | INFO | Downloaded 34GiB to GPU in 2.03464s (16.7105GiB/s) | 20260506 01:06:30.015 | INFO | Analyzing chunk frames = [7740416,7740928) | 20260506 01:06:30.015 | INFO | MemInfo: { | 20260506 01:06:30.015 | INFO | "gpu" : " 17 GB", | 20260506 01:06:30.015 | INFO | "huge" : "", | 20260506 01:06:30.015 | INFO | "huge_shared" : " 27 GB", | 20260506 01:06:30.015 | INFO | "malloc" : " 14 GB", | 20260506 01:06:30.015 | INFO | "pinned" : " 29 GB" | 20260506 01:06:30.015 | INFO | } | 20260506 01:06:30.015 | INFO | Perf report for the last 5.13353 seconds: | 20260506 01:06:30.015 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:06:30.015 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:30.015 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:30.015 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:30.015 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260506 01:06:30.015 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:30.015 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:30.015 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 01:06:30.015 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:30.015 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:30.016 | INFO | Uploaded 36GiB to GPU in 2.11012s (17.0607GiB/s) | 20260506 01:06:30.016 | INFO | Downloaded 34GiB to GPU in 2.08484s (16.3082GiB/s) | 20260506 01:06:35.158 | INFO | Analyzing chunk frames = [7740928,7741440) | 20260506 01:06:35.158 | INFO | MemInfo: { | 20260506 01:06:35.158 | INFO | "gpu" : " 17 GB", | 20260506 01:06:35.158 | INFO | "huge" : "", | 20260506 01:06:35.158 | INFO | "huge_shared" : " 27 GB", | 20260506 01:06:35.158 | INFO | "malloc" : " 14 GB", | 20260506 01:06:35.158 | INFO | "pinned" : " 29 GB" | 20260506 01:06:35.158 | INFO | } | 20260506 01:06:35.158 | INFO | Perf report for the last 5.14301 seconds: | 20260506 01:06:35.158 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:06:35.158 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:35.158 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:35.158 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:35.158 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260506 01:06:35.158 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:35.158 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:35.158 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:06:35.158 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:35.159 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:35.159 | INFO | Uploaded 36GiB to GPU in 2.11163s (17.0484GiB/s) | 20260506 01:06:35.159 | INFO | Downloaded 34GiB to GPU in 2.00707s (16.9401GiB/s) | 20260506 01:06:35.159 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 01:06:35.159 | INFO | Total mem usage: 15.3 GiB | 20260506 01:06:35.159 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 01:06:35.159 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.18%) | 20260506 01:06:35.159 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 01:06:35.159 | INFO | High Water Breakdown: | 20260506 01:06:35.159 | INFO | 60.9% : 8.55 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 01:06:35.159 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 01:06:35.159 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 01:06:35.159 | INFO | ------------------------------------------ | 20260506 01:06:35.159 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:06:35.159 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 01:06:35.159 | INFO | Total mem usage: 26.5 GiB | 20260506 01:06:35.159 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 01:06:35.159 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 01:06:35.159 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260506 01:06:35.159 | INFO | High Water Breakdown: | 20260506 01:06:35.159 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:06:35.159 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:06:35.159 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:06:35.159 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:06:35.159 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:06:35.159 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:06:35.159 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 01:06:35.159 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 01:06:35.159 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:06:35.159 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 01:06:35.159 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 01:06:35.159 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 01:06:35.159 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 01:06:35.159 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 01:06:35.159 | INFO | ------------------------------------------ | 20260506 01:06:35.159 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:06:35.159 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 01:06:35.159 | INFO | Total mem usage: 25.6 GiB | 20260506 01:06:35.159 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 01:06:35.159 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 01:06:35.159 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 01:06:35.159 | INFO | High Water Breakdown: | 20260506 01:06:35.159 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 01:06:35.159 | INFO | ------------------------------------------ | 20260506 01:06:35.159 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:06:35.159 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 01:06:35.159 | INFO | Total mem usage: 15.7 GiB | 20260506 01:06:35.159 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 01:06:35.159 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 01:06:35.159 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 01:06:35.159 | INFO | High Water Breakdown: | 20260506 01:06:35.159 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 01:06:35.159 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 01:06:35.159 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 01:06:35.159 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 01:06:35.160 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 01:06:35.160 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 01:06:35.160 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 01:06:35.160 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 01:06:35.160 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 01:06:35.160 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:06:35.160 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 01:06:35.160 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:06:35.160 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 01:06:35.160 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 01:06:35.160 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 01:06:35.160 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 01:06:35.160 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:06:35.160 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 01:06:35.160 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:06:35.160 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:06:35.160 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:06:35.160 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 01:06:35.160 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 01:06:35.160 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:06:35.160 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 01:06:35.160 | INFO | ------------------------------------------ | 20260506 01:06:35.160 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:06:35.160 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 01:06:35.160 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:06:35.160 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:06:35.160 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.39e+01, 1.35e+01 / 9.09e-01 | 20260506 01:06:35.160 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.04e-01, 1.03e-02 / 4.55e-03 | 20260506 01:06:35.160 | INFO | PerfReport: 11.83%, Baseline -- 3.71e+00 / 4.49e+00, 5.73e-01 / 1.99e-01 | 20260506 01:06:35.160 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.51e+00, 1.90e-01 / 1.01e-01 | 20260506 01:06:35.160 | INFO | PerfReport: 15.97%, DME -- 4.00e+01 / 4.39e+01, 3.76e+00 / 9.38e-01 | 20260506 01:06:35.160 | INFO | PerfReport: 57.48%, FrameLabeling -- 1.80e+01 / 1.97e+01, 1.58e+00 / 1.39e+00 | 20260506 01:06:35.160 | INFO | PerfReport: 7.04%, PulseAccumulating -- 2.21e+00 / 2.70e+00, 4.18e-01 / 2.82e-01 | 20260506 01:06:35.160 | INFO | PerfReport: 3.04%, Metrics -- 9.54e-01 / 1.22e+00, 2.56e-01 / 1.09e-01 | 20260506 01:06:35.160 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.31e-01 / 3.78e-01, 5.53e-02 / 2.37e-02 | 20260506 01:06:35.160 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 01:06:35.160 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:06:35.160 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:06:35.160 | INFO | PerfReport: Full Execution -- 9.81e+01 / -nan, 2.45e+01 / 0.00e+00 | 20260506 01:06:35.160 | INFO | PerfReport: 22.26%, Upload -- 2.18e+01 / 0.00e+00, 7.43e+00 / 0.00e+00 | 20260506 01:06:35.160 | INFO | PerfReport: 21.86%, Download -- 2.14e+01 / 0.00e+00, 7.56e+00 / 0.00e+00 | 20260506 01:06:35.160 | INFO | PerfReport: 31.95%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 01:06:35.160 | INFO | PerfReport: 2.10%, LockUpload -- 2.06e+00 / 0.00e+00, 7.90e+00 / 0.00e+00 | 20260506 01:06:35.160 | INFO | PerfReport: 1.27%, LockDownload -- 1.25e+00 / 0.00e+00, 2.82e+00 / 0.00e+00 | 20260506 01:06:35.160 | INFO | PerfReport: 20.54%, LockCompute -- 2.01e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 01:06:35.160 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.81e-02 / 0.00e+00 | 20260506 01:06:35.281 | INFO | Starting a round of Baz IO | 20260506 01:06:38.426 | INFO | Populating RTMetrics report | 20260506 01:06:38.426 | INFO | RTMetrics report populated | 20260506 01:06:38.426 | INFO | Opening tmp RTMetrics json file | 20260506 01:06:38.427 | INFO | Serializing RTMetrics to json | 20260506 01:06:38.428 | INFO | Serialization done, swapping tmp file now... | 20260506 01:06:38.429 | INFO | RTMetrics done | 20260506 01:06:38.506 | INFO | Finished a round of Baz IO: Wrote 1.835 GiB in 3.224 seconds | 20260506 01:06:38.506 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.33% / 0.00% / 78.67% | 20260506 01:06:38.506 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0839313% / 0% / 5.5631e-05% | 20260506 01:06:38.506 | INFO | Baz IO overall: | 20260506 01:06:38.506 | INFO | BytesWritten : 4378096735892 | 20260506 01:06:38.506 | INFO | NumEvents : 2058679674916 | 20260506 01:06:40.299 | INFO | Analyzing chunk frames = [7741440,7741952) | 20260506 01:06:40.299 | INFO | MemInfo: { | 20260506 01:06:40.299 | INFO | "gpu" : " 17 GB", | 20260506 01:06:40.299 | INFO | "huge" : "", | 20260506 01:06:40.299 | INFO | "huge_shared" : " 27 GB", | 20260506 01:06:40.299 | INFO | "malloc" : " 14 GB", | 20260506 01:06:40.299 | INFO | "pinned" : " 29 GB" | 20260506 01:06:40.299 | INFO | } | 20260506 01:06:40.299 | INFO | Perf report for the last 5.14089 seconds: | 20260506 01:06:40.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:06:40.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:40.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:40.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:40.299 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260506 01:06:40.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:40.300 | INFO | RT_METRICS : 43.0%, 1.0, 22.5ms, 0 | 20260506 01:06:40.300 | INFO | PRE_HQ : 60.0%, 1.7, 53.9ms, 0 | 20260506 01:06:40.300 | INFO | BAZWRITER : 61.9%, 3.5, 110.7ms, 0 | 20260506 01:06:40.300 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:06:40.300 | INFO | Uploaded 36GiB to GPU in 2.17027s (16.5878GiB/s) | 20260506 01:06:40.300 | INFO | Downloaded 38GiB to GPU in 2.24195s (16.9495GiB/s) | 20260506 01:06:45.444 | INFO | Analyzing chunk frames = [7741952,7742464) | 20260506 01:06:45.444 | INFO | MemInfo: { | 20260506 01:06:45.444 | INFO | "gpu" : " 17 GB", | 20260506 01:06:45.444 | INFO | "huge" : "", | 20260506 01:06:45.444 | INFO | "huge_shared" : " 27 GB", | 20260506 01:06:45.444 | INFO | "malloc" : " 14 GB", | 20260506 01:06:45.444 | INFO | "pinned" : " 29 GB" | 20260506 01:06:45.444 | INFO | } | 20260506 01:06:45.444 | INFO | Perf report for the last 5.14485 seconds: | 20260506 01:06:45.444 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:06:45.444 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:45.444 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:45.444 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:45.444 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260506 01:06:45.444 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:45.444 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:45.444 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 01:06:45.444 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:45.444 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:45.444 | INFO | Uploaded 36GiB to GPU in 2.10895s (17.0701GiB/s) | 20260506 01:06:45.444 | INFO | Downloaded 34GiB to GPU in 2.03851s (16.6788GiB/s) | 20260506 01:06:50.586 | INFO | Analyzing chunk frames = [7742464,7742976) | 20260506 01:06:50.586 | INFO | MemInfo: { | 20260506 01:06:50.586 | INFO | "gpu" : " 17 GB", | 20260506 01:06:50.586 | INFO | "huge" : "", | 20260506 01:06:50.586 | INFO | "huge_shared" : " 27 GB", | 20260506 01:06:50.586 | INFO | "malloc" : " 14 GB", | 20260506 01:06:50.586 | INFO | "pinned" : " 29 GB" | 20260506 01:06:50.586 | INFO | } | 20260506 01:06:50.586 | INFO | Perf report for the last 5.14226 seconds: | 20260506 01:06:50.586 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:06:50.586 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:50.587 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:50.587 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:50.587 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 01:06:50.587 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:50.587 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:50.587 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:06:50.587 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:50.587 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:50.587 | INFO | Uploaded 36GiB to GPU in 2.09269s (17.2027GiB/s) | 20260506 01:06:50.587 | INFO | Downloaded 34GiB to GPU in 2.02763s (16.7683GiB/s) | 20260506 01:06:55.720 | INFO | Analyzing chunk frames = [7742976,7743488) | 20260506 01:06:55.720 | INFO | MemInfo: { | 20260506 01:06:55.720 | INFO | "gpu" : " 17 GB", | 20260506 01:06:55.720 | INFO | "huge" : "", | 20260506 01:06:55.720 | INFO | "huge_shared" : " 27 GB", | 20260506 01:06:55.720 | INFO | "malloc" : " 14 GB", | 20260506 01:06:55.720 | INFO | "pinned" : " 29 GB" | 20260506 01:06:55.720 | INFO | } | 20260506 01:06:55.720 | INFO | Perf report for the last 5.1335 seconds: | 20260506 01:06:55.720 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:06:55.720 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:55.720 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:55.720 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:55.720 | INFO | ANALYSIS : 61.2%, 3.0, 95.0ms, 0 | 20260506 01:06:55.720 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:55.720 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:55.720 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:06:55.720 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:55.720 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:06:55.720 | INFO | Uploaded 36GiB to GPU in 2.09929s (17.1487GiB/s) | 20260506 01:06:55.720 | INFO | Downloaded 34GiB to GPU in 2.01358s (16.8853GiB/s) | 20260506 01:07:00.851 | INFO | Analyzing chunk frames = [7743488,7744000) | 20260506 01:07:00.851 | INFO | MemInfo: { | 20260506 01:07:00.851 | INFO | "gpu" : " 17 GB", | 20260506 01:07:00.851 | INFO | "huge" : "", | 20260506 01:07:00.851 | INFO | "huge_shared" : " 27 GB", | 20260506 01:07:00.851 | INFO | "malloc" : " 14 GB", | 20260506 01:07:00.851 | INFO | "pinned" : " 29 GB" | 20260506 01:07:00.851 | INFO | } | 20260506 01:07:00.851 | INFO | Perf report for the last 5.13134 seconds: | 20260506 01:07:00.851 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:07:00.851 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:00.851 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:00.851 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:00.851 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 01:07:00.851 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:00.851 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:00.851 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 01:07:00.851 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:00.851 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:00.852 | INFO | Uploaded 36GiB to GPU in 2.1159s (17.014GiB/s) | 20260506 01:07:00.852 | INFO | Downloaded 34GiB to GPU in 2.05386s (16.5542GiB/s) | 20260506 01:07:05.995 | INFO | Analyzing chunk frames = [7744000,7744512) | 20260506 01:07:05.995 | INFO | MemInfo: { | 20260506 01:07:05.995 | INFO | "gpu" : " 17 GB", | 20260506 01:07:05.995 | INFO | "huge" : "", | 20260506 01:07:05.995 | INFO | "huge_shared" : " 27 GB", | 20260506 01:07:05.995 | INFO | "malloc" : " 14 GB", | 20260506 01:07:05.995 | INFO | "pinned" : " 29 GB" | 20260506 01:07:05.995 | INFO | } | 20260506 01:07:05.995 | INFO | Perf report for the last 5.1439 seconds: | 20260506 01:07:05.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:07:05.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:05.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:05.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:05.995 | INFO | ANALYSIS : 61.3%, 3.0, 95.8ms, 0 | 20260506 01:07:05.995 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:05.995 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:05.995 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 01:07:05.995 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:05.995 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:05.995 | INFO | Uploaded 36GiB to GPU in 2.07373s (17.3601GiB/s) | 20260506 01:07:05.996 | INFO | Downloaded 34GiB to GPU in 2.0383s (16.6806GiB/s) | 20260506 01:07:11.140 | INFO | Analyzing chunk frames = [7744512,7745024) | 20260506 01:07:11.140 | INFO | MemInfo: { | 20260506 01:07:11.140 | INFO | "gpu" : " 17 GB", | 20260506 01:07:11.140 | INFO | "huge" : "", | 20260506 01:07:11.140 | INFO | "huge_shared" : " 27 GB", | 20260506 01:07:11.140 | INFO | "malloc" : " 14 GB", | 20260506 01:07:11.140 | INFO | "pinned" : " 29 GB" | 20260506 01:07:11.140 | INFO | } | 20260506 01:07:11.140 | INFO | Perf report for the last 5.14497 seconds: | 20260506 01:07:11.140 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:07:11.140 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:11.140 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:11.140 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:11.140 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260506 01:07:11.140 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:11.140 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:11.140 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 01:07:11.140 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:11.140 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:11.140 | INFO | Uploaded 36GiB to GPU in 2.03066s (17.7283GiB/s) | 20260506 01:07:11.140 | INFO | Downloaded 34GiB to GPU in 2.06379s (16.4745GiB/s) | 20260506 01:07:16.279 | INFO | Analyzing chunk frames = [7745024,7745536) | 20260506 01:07:16.279 | INFO | MemInfo: { | 20260506 01:07:16.279 | INFO | "gpu" : " 17 GB", | 20260506 01:07:16.279 | INFO | "huge" : "", | 20260506 01:07:16.279 | INFO | "huge_shared" : " 27 GB", | 20260506 01:07:16.279 | INFO | "malloc" : " 14 GB", | 20260506 01:07:16.279 | INFO | "pinned" : " 29 GB" | 20260506 01:07:16.279 | INFO | } | 20260506 01:07:16.279 | INFO | Perf report for the last 5.13908 seconds: | 20260506 01:07:16.279 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:07:16.279 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:16.279 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:16.279 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:16.279 | INFO | ANALYSIS : 61.6%, 3.0, 96.2ms, 0 | 20260506 01:07:16.279 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:16.279 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:16.279 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 01:07:16.279 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:16.279 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:16.279 | INFO | Uploaded 36GiB to GPU in 2.06803s (17.4079GiB/s) | 20260506 01:07:16.279 | INFO | Downloaded 34GiB to GPU in 2.02469s (16.7927GiB/s) | 20260506 01:07:16.397 | INFO | Starting a round of Baz IO | 20260506 01:07:19.543 | INFO | Populating RTMetrics report | 20260506 01:07:19.544 | INFO | RTMetrics report populated | 20260506 01:07:19.544 | INFO | Opening tmp RTMetrics json file | 20260506 01:07:19.544 | INFO | Serializing RTMetrics to json | 20260506 01:07:19.545 | INFO | Serialization done, swapping tmp file now... | 20260506 01:07:19.546 | INFO | RTMetrics done | 20260506 01:07:19.621 | INFO | Finished a round of Baz IO: Wrote 1.836 GiB in 3.224 seconds | 20260506 01:07:19.621 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.33% / 0.00% / 78.67% | 20260506 01:07:19.621 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0839869% / 3.97364e-06% / 9.53674e-05% | 20260506 01:07:19.621 | INFO | Baz IO overall: | 20260506 01:07:19.621 | INFO | BytesWritten : 4380067690627 | 20260506 01:07:19.621 | INFO | NumEvents : 2059579009964 | 20260506 01:07:21.414 | INFO | Analyzing chunk frames = [7745536,7746048) | 20260506 01:07:21.414 | INFO | MemInfo: { | 20260506 01:07:21.414 | INFO | "gpu" : " 17 GB", | 20260506 01:07:21.415 | INFO | "huge" : "", | 20260506 01:07:21.415 | INFO | "huge_shared" : " 27 GB", | 20260506 01:07:21.415 | INFO | "malloc" : " 14 GB", | 20260506 01:07:21.415 | INFO | "pinned" : " 29 GB" | 20260506 01:07:21.415 | INFO | } | 20260506 01:07:21.415 | INFO | Perf report for the last 5.13551 seconds: | 20260506 01:07:21.415 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:07:21.415 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:21.415 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:21.415 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:21.415 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260506 01:07:21.415 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:21.415 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260506 01:07:21.415 | INFO | PRE_HQ : 59.9%, 1.7, 54.0ms, 0 | 20260506 01:07:21.415 | INFO | BAZWRITER : 61.9%, 3.5, 110.4ms, 0 | 20260506 01:07:21.415 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:07:21.415 | INFO | Uploaded 36GiB to GPU in 2.17638s (16.5412GiB/s) | 20260506 01:07:21.416 | INFO | Downloaded 38GiB to GPU in 2.2411s (16.956GiB/s) | 20260506 01:07:26.555 | INFO | Analyzing chunk frames = [7746048,7746560) | 20260506 01:07:26.555 | INFO | MemInfo: { | 20260506 01:07:26.555 | INFO | "gpu" : " 17 GB", | 20260506 01:07:26.555 | INFO | "huge" : "", | 20260506 01:07:26.555 | INFO | "huge_shared" : " 27 GB", | 20260506 01:07:26.555 | INFO | "malloc" : " 14 GB", | 20260506 01:07:26.555 | INFO | "pinned" : " 29 GB" | 20260506 01:07:26.555 | INFO | } | 20260506 01:07:26.555 | INFO | Perf report for the last 5.14026 seconds: | 20260506 01:07:26.555 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:07:26.555 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:26.555 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:26.555 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:26.555 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260506 01:07:26.555 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:26.555 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:26.555 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:07:26.555 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:26.555 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:26.555 | INFO | Uploaded 36GiB to GPU in 2.1074s (17.0827GiB/s) | 20260506 01:07:26.555 | INFO | Downloaded 34GiB to GPU in 2.04047s (16.6629GiB/s) | 20260506 01:07:31.696 | INFO | Analyzing chunk frames = [7746560,7747072) | 20260506 01:07:31.696 | INFO | MemInfo: { | 20260506 01:07:31.696 | INFO | "gpu" : " 17 GB", | 20260506 01:07:31.696 | INFO | "huge" : "", | 20260506 01:07:31.696 | INFO | "huge_shared" : " 27 GB", | 20260506 01:07:31.696 | INFO | "malloc" : " 14 GB", | 20260506 01:07:31.696 | INFO | "pinned" : " 29 GB" | 20260506 01:07:31.696 | INFO | } | 20260506 01:07:31.696 | INFO | Perf report for the last 5.14117 seconds: | 20260506 01:07:31.696 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:07:31.696 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:31.696 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:31.696 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:31.696 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 01:07:31.696 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:31.696 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:31.696 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 01:07:31.696 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:31.696 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:31.696 | INFO | Uploaded 36GiB to GPU in 2.08747s (17.2457GiB/s) | 20260506 01:07:31.697 | INFO | Downloaded 34GiB to GPU in 2.02493s (16.7907GiB/s) | 20260506 01:07:36.827 | INFO | Analyzing chunk frames = [7747072,7747584) | 20260506 01:07:36.828 | INFO | MemInfo: { | 20260506 01:07:36.828 | INFO | "gpu" : " 17 GB", | 20260506 01:07:36.828 | INFO | "huge" : "", | 20260506 01:07:36.828 | INFO | "huge_shared" : " 27 GB", | 20260506 01:07:36.828 | INFO | "malloc" : " 14 GB", | 20260506 01:07:36.828 | INFO | "pinned" : " 29 GB" | 20260506 01:07:36.828 | INFO | } | 20260506 01:07:36.828 | INFO | Perf report for the last 5.13162 seconds: | 20260506 01:07:36.828 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:07:36.828 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:36.828 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:36.828 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:36.828 | INFO | ANALYSIS : 61.6%, 3.0, 95.6ms, 0 | 20260506 01:07:36.828 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:36.828 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:36.828 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260506 01:07:36.828 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:36.828 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:36.828 | INFO | Uploaded 36GiB to GPU in 2.14981s (16.7457GiB/s) | 20260506 01:07:36.828 | INFO | Downloaded 34GiB to GPU in 2.06081s (16.4984GiB/s) | 20260506 01:07:41.967 | INFO | Analyzing chunk frames = [7747584,7748096) | 20260506 01:07:41.967 | INFO | MemInfo: { | 20260506 01:07:41.967 | INFO | "gpu" : " 17 GB", | 20260506 01:07:41.967 | INFO | "huge" : "", | 20260506 01:07:41.967 | INFO | "huge_shared" : " 27 GB", | 20260506 01:07:41.967 | INFO | "malloc" : " 14 GB", | 20260506 01:07:41.967 | INFO | "pinned" : " 29 GB" | 20260506 01:07:41.967 | INFO | } | 20260506 01:07:41.967 | INFO | Perf report for the last 5.13952 seconds: | 20260506 01:07:41.967 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:07:41.967 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:41.967 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:41.967 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:41.967 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 01:07:41.967 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:41.967 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:41.967 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 01:07:41.967 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:41.968 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:41.968 | INFO | Uploaded 36GiB to GPU in 2.10495s (17.1026GiB/s) | 20260506 01:07:41.968 | INFO | Downloaded 34GiB to GPU in 2.05204s (16.5689GiB/s) | 20260506 01:07:47.108 | INFO | Analyzing chunk frames = [7748096,7748608) | 20260506 01:07:47.108 | INFO | MemInfo: { | 20260506 01:07:47.108 | INFO | "gpu" : " 17 GB", | 20260506 01:07:47.108 | INFO | "huge" : "", | 20260506 01:07:47.108 | INFO | "huge_shared" : " 27 GB", | 20260506 01:07:47.108 | INFO | "malloc" : " 14 GB", | 20260506 01:07:47.108 | INFO | "pinned" : " 29 GB" | 20260506 01:07:47.108 | INFO | } | 20260506 01:07:47.108 | INFO | Perf report for the last 5.1408 seconds: | 20260506 01:07:47.108 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:07:47.108 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:47.108 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:47.108 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:47.108 | INFO | ANALYSIS : 61.2%, 3.0, 95.6ms, 0 | 20260506 01:07:47.108 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:47.108 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:47.108 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 01:07:47.108 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:47.108 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:47.108 | INFO | Uploaded 36GiB to GPU in 2.07397s (17.358GiB/s) | 20260506 01:07:47.108 | INFO | Downloaded 34GiB to GPU in 2.04468s (16.6285GiB/s) | 20260506 01:07:52.252 | INFO | Analyzing chunk frames = [7748608,7749120) | 20260506 01:07:52.252 | INFO | MemInfo: { | 20260506 01:07:52.252 | INFO | "gpu" : " 17 GB", | 20260506 01:07:52.252 | INFO | "huge" : "", | 20260506 01:07:52.252 | INFO | "huge_shared" : " 27 GB", | 20260506 01:07:52.253 | INFO | "malloc" : " 14 GB", | 20260506 01:07:52.253 | INFO | "pinned" : " 29 GB" | 20260506 01:07:52.253 | INFO | } | 20260506 01:07:52.253 | INFO | Perf report for the last 5.1445 seconds: | 20260506 01:07:52.253 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:07:52.253 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:52.253 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:52.253 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:52.253 | INFO | ANALYSIS : 62.1%, 3.0, 97.1ms, 0 | 20260506 01:07:52.253 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:52.253 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:52.253 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 01:07:52.253 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:52.253 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:52.253 | INFO | Uploaded 36GiB to GPU in 2.0598s (17.4774GiB/s) | 20260506 01:07:52.253 | INFO | Downloaded 34GiB to GPU in 2.06107s (16.4963GiB/s) | 20260506 01:07:57.393 | INFO | Analyzing chunk frames = [7749120,7749632) | 20260506 01:07:57.394 | INFO | MemInfo: { | 20260506 01:07:57.394 | INFO | "gpu" : " 17 GB", | 20260506 01:07:57.394 | INFO | "huge" : "", | 20260506 01:07:57.394 | INFO | "huge_shared" : " 27 GB", | 20260506 01:07:57.394 | INFO | "malloc" : " 14 GB", | 20260506 01:07:57.394 | INFO | "pinned" : " 29 GB" | 20260506 01:07:57.394 | INFO | } | 20260506 01:07:57.394 | INFO | Perf report for the last 5.14113 seconds: | 20260506 01:07:57.394 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:07:57.394 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:57.394 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:57.394 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:57.394 | INFO | ANALYSIS : 61.5%, 3.0, 96.0ms, 0 | 20260506 01:07:57.394 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:57.394 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:57.394 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 01:07:57.394 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:57.394 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:07:57.394 | INFO | Uploaded 36GiB to GPU in 2.05815s (17.4914GiB/s) | 20260506 01:07:57.394 | INFO | Downloaded 34GiB to GPU in 2.02093s (16.8239GiB/s) | 20260506 01:07:57.394 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 01:07:57.394 | INFO | Total mem usage: 15.3 GiB | 20260506 01:07:57.394 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 01:07:57.394 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.17%) | 20260506 01:07:57.394 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 01:07:57.394 | INFO | High Water Breakdown: | 20260506 01:07:57.394 | INFO | 60.9% : 8.55 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 01:07:57.394 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 01:07:57.394 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 01:07:57.394 | INFO | ------------------------------------------ | 20260506 01:07:57.394 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:07:57.394 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 01:07:57.394 | INFO | Total mem usage: 26.5 GiB | 20260506 01:07:57.394 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 01:07:57.394 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 01:07:57.394 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260506 01:07:57.394 | INFO | High Water Breakdown: | 20260506 01:07:57.395 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:07:57.395 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:07:57.395 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:07:57.395 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:07:57.395 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:07:57.395 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:07:57.395 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 01:07:57.395 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 01:07:57.395 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:07:57.395 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 01:07:57.395 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 01:07:57.395 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 01:07:57.395 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 01:07:57.395 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 01:07:57.395 | INFO | ------------------------------------------ | 20260506 01:07:57.395 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:07:57.395 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 01:07:57.395 | INFO | Total mem usage: 25.6 GiB | 20260506 01:07:57.395 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 01:07:57.395 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 01:07:57.395 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 01:07:57.395 | INFO | High Water Breakdown: | 20260506 01:07:57.395 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 01:07:57.395 | INFO | ------------------------------------------ | 20260506 01:07:57.395 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:07:57.395 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 01:07:57.395 | INFO | Total mem usage: 15.7 GiB | 20260506 01:07:57.395 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 01:07:57.395 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 01:07:57.395 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 01:07:57.395 | INFO | High Water Breakdown: | 20260506 01:07:57.395 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 01:07:57.395 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 01:07:57.395 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 01:07:57.395 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 01:07:57.395 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 01:07:57.395 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 01:07:57.395 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 01:07:57.395 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 01:07:57.395 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 01:07:57.395 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:07:57.395 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 01:07:57.395 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:07:57.395 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 01:07:57.395 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 01:07:57.395 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 01:07:57.395 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 01:07:57.395 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:07:57.395 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 01:07:57.395 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:07:57.395 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:07:57.395 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:07:57.395 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 01:07:57.395 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 01:07:57.395 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:07:57.395 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 01:07:57.395 | INFO | ------------------------------------------ | 20260506 01:07:57.395 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:07:57.395 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 01:07:57.395 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:07:57.395 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:07:57.395 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.40e+01, 1.35e+01 / 8.34e-01 | 20260506 01:07:57.395 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.04e-01, 1.04e-02 / 3.99e-03 | 20260506 01:07:57.395 | INFO | PerfReport: 11.82%, Baseline -- 3.70e+00 / 4.49e+00, 5.67e-01 / 1.86e-01 | 20260506 01:07:57.395 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.51e+00, 1.91e-01 / 9.87e-02 | 20260506 01:07:57.395 | INFO | PerfReport: 16.01%, DME -- 4.01e+01 / 4.40e+01, 3.87e+00 / 9.43e-01 | 20260506 01:07:57.395 | INFO | PerfReport: 57.41%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.50e+00 / 1.04e+00 | 20260506 01:07:57.395 | INFO | PerfReport: 7.07%, PulseAccumulating -- 2.21e+00 / 2.73e+00, 4.21e-01 / 2.44e-01 | 20260506 01:07:57.395 | INFO | PerfReport: 3.05%, Metrics -- 9.55e-01 / 1.23e+00, 2.56e-01 / 1.21e-01 | 20260506 01:07:57.395 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.31e-01 / 3.86e-01, 5.64e-02 / 3.00e-02 | 20260506 01:07:57.395 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 01:07:57.395 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:07:57.395 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:07:57.395 | INFO | PerfReport: Full Execution -- 9.81e+01 / -nan, 2.44e+01 / 0.00e+00 | 20260506 01:07:57.395 | INFO | PerfReport: 22.29%, Upload -- 2.19e+01 / 0.00e+00, 7.45e+00 / 0.00e+00 | 20260506 01:07:57.395 | INFO | PerfReport: 21.94%, Download -- 2.15e+01 / 0.00e+00, 7.61e+00 / 0.00e+00 | 20260506 01:07:57.395 | INFO | PerfReport: 31.92%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 01:07:57.395 | INFO | PerfReport: 2.04%, LockUpload -- 2.01e+00 / 0.00e+00, 7.80e+00 / 0.00e+00 | 20260506 01:07:57.395 | INFO | PerfReport: 1.32%, LockDownload -- 1.30e+00 / 0.00e+00, 2.92e+00 / 0.00e+00 | 20260506 01:07:57.395 | INFO | PerfReport: 20.47%, LockCompute -- 2.01e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 01:07:57.395 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.50e-02 / 0.00e+00 | 20260506 01:07:57.512 | INFO | Starting a round of Baz IO | 20260506 01:08:00.667 | INFO | Populating RTMetrics report | 20260506 01:08:00.668 | INFO | RTMetrics report populated | 20260506 01:08:00.668 | INFO | Opening tmp RTMetrics json file | 20260506 01:08:00.668 | INFO | Serializing RTMetrics to json | 20260506 01:08:00.670 | INFO | Serialization done, swapping tmp file now... | 20260506 01:08:00.670 | INFO | RTMetrics done | 20260506 01:08:00.738 | INFO | Finished a round of Baz IO: Wrote 1.836 GiB in 3.226 seconds | 20260506 01:08:00.738 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.33% / 0.00% / 78.67% | 20260506 01:08:00.738 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0836253% / 0% / 9.13938e-05% | 20260506 01:08:00.738 | INFO | Baz IO overall: | 20260506 01:08:00.738 | INFO | BytesWritten : 4382039150621 | 20260506 01:08:00.738 | INFO | NumEvents : 2060478809300 | 20260506 01:08:02.534 | INFO | Analyzing chunk frames = [7749632,7750144) | 20260506 01:08:02.534 | INFO | MemInfo: { | 20260506 01:08:02.534 | INFO | "gpu" : " 17 GB", | 20260506 01:08:02.534 | INFO | "huge" : "", | 20260506 01:08:02.534 | INFO | "huge_shared" : " 27 GB", | 20260506 01:08:02.534 | INFO | "malloc" : " 14 GB", | 20260506 01:08:02.534 | INFO | "pinned" : " 29 GB" | 20260506 01:08:02.534 | INFO | } | 20260506 01:08:02.534 | INFO | Perf report for the last 5.14038 seconds: | 20260506 01:08:02.534 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:08:02.534 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:02.534 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:02.534 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:02.534 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260506 01:08:02.534 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:02.534 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260506 01:08:02.534 | INFO | PRE_HQ : 59.9%, 1.7, 54.2ms, 0 | 20260506 01:08:02.534 | INFO | BAZWRITER : 61.9%, 3.5, 110.5ms, 0 | 20260506 01:08:02.534 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:08:02.535 | INFO | Uploaded 36GiB to GPU in 2.18824s (16.4516GiB/s) | 20260506 01:08:02.535 | INFO | Downloaded 38GiB to GPU in 2.2483s (16.9017GiB/s) | 20260506 01:08:07.664 | INFO | Analyzing chunk frames = [7750144,7750656) | 20260506 01:08:07.664 | INFO | MemInfo: { | 20260506 01:08:07.664 | INFO | "gpu" : " 17 GB", | 20260506 01:08:07.664 | INFO | "huge" : "", | 20260506 01:08:07.664 | INFO | "huge_shared" : " 27 GB", | 20260506 01:08:07.664 | INFO | "malloc" : " 14 GB", | 20260506 01:08:07.664 | INFO | "pinned" : " 29 GB" | 20260506 01:08:07.665 | INFO | } | 20260506 01:08:07.665 | INFO | Perf report for the last 5.13046 seconds: | 20260506 01:08:07.665 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:08:07.665 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:07.665 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:07.665 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:07.665 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 01:08:07.665 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:07.665 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:07.665 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:08:07.665 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:07.665 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:07.665 | INFO | Uploaded 36GiB to GPU in 2.12241s (16.9619GiB/s) | 20260506 01:08:07.665 | INFO | Downloaded 34GiB to GPU in 2.0422s (16.6487GiB/s) | 20260506 01:08:12.806 | INFO | Analyzing chunk frames = [7750656,7751168) | 20260506 01:08:12.806 | INFO | MemInfo: { | 20260506 01:08:12.806 | INFO | "gpu" : " 17 GB", | 20260506 01:08:12.806 | INFO | "huge" : "", | 20260506 01:08:12.806 | INFO | "huge_shared" : " 27 GB", | 20260506 01:08:12.806 | INFO | "malloc" : " 14 GB", | 20260506 01:08:12.806 | INFO | "pinned" : " 29 GB" | 20260506 01:08:12.806 | INFO | } | 20260506 01:08:12.806 | INFO | Perf report for the last 5.14145 seconds: | 20260506 01:08:12.806 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:08:12.806 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:12.806 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:12.806 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:12.806 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 01:08:12.806 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:12.806 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:12.806 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 01:08:12.806 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:12.806 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:12.806 | INFO | Uploaded 36GiB to GPU in 2.08418s (17.273GiB/s) | 20260506 01:08:12.806 | INFO | Downloaded 34GiB to GPU in 2.02729s (16.7712GiB/s) | 20260506 01:08:17.951 | INFO | Analyzing chunk frames = [7751168,7751680) | 20260506 01:08:17.951 | INFO | MemInfo: { | 20260506 01:08:17.951 | INFO | "gpu" : " 17 GB", | 20260506 01:08:17.951 | INFO | "huge" : "", | 20260506 01:08:17.951 | INFO | "huge_shared" : " 27 GB", | 20260506 01:08:17.951 | INFO | "malloc" : " 14 GB", | 20260506 01:08:17.951 | INFO | "pinned" : " 29 GB" | 20260506 01:08:17.951 | INFO | } | 20260506 01:08:17.951 | INFO | Perf report for the last 5.1453 seconds: | 20260506 01:08:17.951 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:08:17.951 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:17.951 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:17.952 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:17.952 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260506 01:08:17.952 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:17.952 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:17.952 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:08:17.952 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:17.952 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:17.952 | INFO | Uploaded 36GiB to GPU in 2.15429s (16.7108GiB/s) | 20260506 01:08:17.952 | INFO | Downloaded 34GiB to GPU in 2.05466s (16.5477GiB/s) | 20260506 01:08:23.092 | INFO | Analyzing chunk frames = [7751680,7752192) | 20260506 01:08:23.092 | INFO | MemInfo: { | 20260506 01:08:23.092 | INFO | "gpu" : " 17 GB", | 20260506 01:08:23.092 | INFO | "huge" : "", | 20260506 01:08:23.092 | INFO | "huge_shared" : " 27 GB", | 20260506 01:08:23.092 | INFO | "malloc" : " 14 GB", | 20260506 01:08:23.092 | INFO | "pinned" : " 29 GB" | 20260506 01:08:23.092 | INFO | } | 20260506 01:08:23.092 | INFO | Perf report for the last 5.14081 seconds: | 20260506 01:08:23.092 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:08:23.092 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:23.092 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:23.092 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:23.092 | INFO | ANALYSIS : 60.7%, 3.0, 94.9ms, 0 | 20260506 01:08:23.092 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:23.092 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:23.092 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 01:08:23.092 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:23.092 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:23.092 | INFO | Uploaded 36GiB to GPU in 2.07896s (17.3164GiB/s) | 20260506 01:08:23.092 | INFO | Downloaded 34GiB to GPU in 2.03407s (16.7153GiB/s) | 20260506 01:08:28.225 | INFO | Analyzing chunk frames = [7752192,7752704) | 20260506 01:08:28.225 | INFO | MemInfo: { | 20260506 01:08:28.225 | INFO | "gpu" : " 17 GB", | 20260506 01:08:28.225 | INFO | "huge" : "", | 20260506 01:08:28.225 | INFO | "huge_shared" : " 27 GB", | 20260506 01:08:28.225 | INFO | "malloc" : " 14 GB", | 20260506 01:08:28.225 | INFO | "pinned" : " 29 GB" | 20260506 01:08:28.225 | INFO | } | 20260506 01:08:28.226 | INFO | Perf report for the last 5.13323 seconds: | 20260506 01:08:28.226 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:08:28.226 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:28.226 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:28.226 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:28.226 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 01:08:28.226 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:28.226 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:28.226 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 01:08:28.226 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:28.226 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:28.226 | INFO | Uploaded 36GiB to GPU in 2.10082s (17.1361GiB/s) | 20260506 01:08:28.226 | INFO | Downloaded 34GiB to GPU in 2.03984s (16.668GiB/s) | 20260506 01:08:33.362 | INFO | Analyzing chunk frames = [7752704,7753216) | 20260506 01:08:33.362 | INFO | MemInfo: { | 20260506 01:08:33.362 | INFO | "gpu" : " 17 GB", | 20260506 01:08:33.362 | INFO | "huge" : "", | 20260506 01:08:33.362 | INFO | "huge_shared" : " 27 GB", | 20260506 01:08:33.362 | INFO | "malloc" : " 14 GB", | 20260506 01:08:33.362 | INFO | "pinned" : " 29 GB" | 20260506 01:08:33.362 | INFO | } | 20260506 01:08:33.362 | INFO | Perf report for the last 5.13648 seconds: | 20260506 01:08:33.362 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:08:33.362 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:33.362 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:33.362 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:33.362 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260506 01:08:33.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:33.362 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:33.362 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 01:08:33.362 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:33.362 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:33.362 | INFO | Uploaded 36GiB to GPU in 2.10765s (17.0807GiB/s) | 20260506 01:08:33.363 | INFO | Downloaded 34GiB to GPU in 2.06932s (16.4305GiB/s) | 20260506 01:08:38.507 | INFO | Analyzing chunk frames = [7753216,7753728) | 20260506 01:08:38.508 | INFO | MemInfo: { | 20260506 01:08:38.508 | INFO | "gpu" : " 17 GB", | 20260506 01:08:38.508 | INFO | "huge" : "", | 20260506 01:08:38.508 | INFO | "huge_shared" : " 27 GB", | 20260506 01:08:38.508 | INFO | "malloc" : " 14 GB", | 20260506 01:08:38.508 | INFO | "pinned" : " 29 GB" | 20260506 01:08:38.508 | INFO | } | 20260506 01:08:38.508 | INFO | Perf report for the last 5.14569 seconds: | 20260506 01:08:38.508 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:08:38.508 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:38.508 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:38.508 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:38.508 | INFO | ANALYSIS : 61.8%, 3.0, 96.6ms, 0 | 20260506 01:08:38.508 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:38.508 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:38.508 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 01:08:38.508 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:38.508 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:38.508 | INFO | Uploaded 36GiB to GPU in 2.13667s (16.8487GiB/s) | 20260506 01:08:38.508 | INFO | Downloaded 34GiB to GPU in 2.01731s (16.8541GiB/s) | 20260506 01:08:38.633 | INFO | Starting a round of Baz IO | 20260506 01:08:41.766 | INFO | Populating RTMetrics report | 20260506 01:08:41.767 | INFO | RTMetrics report populated | 20260506 01:08:41.767 | INFO | Opening tmp RTMetrics json file | 20260506 01:08:41.768 | INFO | Serializing RTMetrics to json | 20260506 01:08:41.770 | INFO | Serialization done, swapping tmp file now... | 20260506 01:08:41.771 | INFO | RTMetrics done | 20260506 01:08:41.849 | INFO | Finished a round of Baz IO: Wrote 1.835 GiB in 3.216 seconds | 20260506 01:08:41.849 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.33% / 0.00% / 78.67% | 20260506 01:08:41.849 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.083669% / 0% / 7.54992e-05% | 20260506 01:08:41.849 | INFO | Baz IO overall: | 20260506 01:08:41.849 | INFO | BytesWritten : 4384009208294 | 20260506 01:08:41.849 | INFO | NumEvents : 2061377610427 | 20260506 01:08:43.649 | INFO | Analyzing chunk frames = [7753728,7754240) | 20260506 01:08:43.649 | INFO | MemInfo: { | 20260506 01:08:43.649 | INFO | "gpu" : " 17 GB", | 20260506 01:08:43.649 | INFO | "huge" : "", | 20260506 01:08:43.649 | INFO | "huge_shared" : " 27 GB", | 20260506 01:08:43.649 | INFO | "malloc" : " 14 GB", | 20260506 01:08:43.649 | INFO | "pinned" : " 29 GB" | 20260506 01:08:43.649 | INFO | } | 20260506 01:08:43.649 | INFO | Perf report for the last 5.14109 seconds: | 20260506 01:08:43.649 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:08:43.649 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:43.649 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:43.649 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:43.649 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260506 01:08:43.649 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:43.649 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260506 01:08:43.649 | INFO | PRE_HQ : 59.8%, 1.7, 54.1ms, 0 | 20260506 01:08:43.649 | INFO | BAZWRITER : 61.8%, 3.5, 110.4ms, 0 | 20260506 01:08:43.649 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:08:43.649 | INFO | Uploaded 36GiB to GPU in 2.1828s (16.4925GiB/s) | 20260506 01:08:43.649 | INFO | Downloaded 38GiB to GPU in 2.23818s (16.9781GiB/s) | 20260506 01:08:48.782 | INFO | Analyzing chunk frames = [7754240,7754752) | 20260506 01:08:48.782 | INFO | MemInfo: { | 20260506 01:08:48.782 | INFO | "gpu" : " 17 GB", | 20260506 01:08:48.782 | INFO | "huge" : "", | 20260506 01:08:48.782 | INFO | "huge_shared" : " 27 GB", | 20260506 01:08:48.782 | INFO | "malloc" : " 14 GB", | 20260506 01:08:48.782 | INFO | "pinned" : " 29 GB" | 20260506 01:08:48.782 | INFO | } | 20260506 01:08:48.782 | INFO | Perf report for the last 5.13281 seconds: | 20260506 01:08:48.782 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:08:48.782 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:48.782 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:48.782 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:48.782 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 01:08:48.782 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:48.782 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:48.782 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:08:48.782 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:48.782 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:48.782 | INFO | Uploaded 36GiB to GPU in 1.99477s (18.0472GiB/s) | 20260506 01:08:48.782 | INFO | Downloaded 34GiB to GPU in 1.99255s (17.0636GiB/s) | 20260506 01:08:53.919 | INFO | Analyzing chunk frames = [7754752,7755264) | 20260506 01:08:53.919 | INFO | MemInfo: { | 20260506 01:08:53.919 | INFO | "gpu" : " 17 GB", | 20260506 01:08:53.919 | INFO | "huge" : "", | 20260506 01:08:53.919 | INFO | "huge_shared" : " 27 GB", | 20260506 01:08:53.919 | INFO | "malloc" : " 14 GB", | 20260506 01:08:53.919 | INFO | "pinned" : " 29 GB" | 20260506 01:08:53.919 | INFO | } | 20260506 01:08:53.919 | INFO | Perf report for the last 5.13769 seconds: | 20260506 01:08:53.920 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:08:53.920 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:53.920 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:53.920 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:53.920 | INFO | ANALYSIS : 61.4%, 3.0, 95.4ms, 0 | 20260506 01:08:53.920 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:53.920 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:53.920 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:08:53.920 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:53.920 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:53.920 | INFO | Uploaded 36GiB to GPU in 2.08059s (17.3028GiB/s) | 20260506 01:08:53.920 | INFO | Downloaded 34GiB to GPU in 2.01493s (16.874GiB/s) | 20260506 01:08:59.060 | INFO | Analyzing chunk frames = [7755264,7755776) | 20260506 01:08:59.060 | INFO | MemInfo: { | 20260506 01:08:59.060 | INFO | "gpu" : " 17 GB", | 20260506 01:08:59.060 | INFO | "huge" : "", | 20260506 01:08:59.060 | INFO | "huge_shared" : " 27 GB", | 20260506 01:08:59.060 | INFO | "malloc" : " 14 GB", | 20260506 01:08:59.060 | INFO | "pinned" : " 29 GB" | 20260506 01:08:59.060 | INFO | } | 20260506 01:08:59.061 | INFO | Perf report for the last 5.14121 seconds: | 20260506 01:08:59.061 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:08:59.061 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:59.061 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:59.061 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:59.061 | INFO | ANALYSIS : 61.6%, 3.0, 95.7ms, 0 | 20260506 01:08:59.061 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:59.061 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:59.061 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 01:08:59.061 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:59.061 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:08:59.061 | INFO | Uploaded 36GiB to GPU in 2.14284s (16.8001GiB/s) | 20260506 01:08:59.061 | INFO | Downloaded 34GiB to GPU in 2.02443s (16.7949GiB/s) | 20260506 01:09:04.208 | INFO | Analyzing chunk frames = [7755776,7756288) | 20260506 01:09:04.208 | INFO | MemInfo: { | 20260506 01:09:04.208 | INFO | "gpu" : " 17 GB", | 20260506 01:09:04.208 | INFO | "huge" : "", | 20260506 01:09:04.208 | INFO | "huge_shared" : " 27 GB", | 20260506 01:09:04.208 | INFO | "malloc" : " 14 GB", | 20260506 01:09:04.208 | INFO | "pinned" : " 29 GB" | 20260506 01:09:04.208 | INFO | } | 20260506 01:09:04.208 | INFO | Perf report for the last 5.14783 seconds: | 20260506 01:09:04.209 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:09:04.209 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:04.209 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:04.209 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:04.209 | INFO | ANALYSIS : 61.0%, 3.0, 95.4ms, 0 | 20260506 01:09:04.209 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:04.209 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:04.209 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 01:09:04.209 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:04.209 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:04.209 | INFO | Uploaded 36GiB to GPU in 2.12369s (16.9516GiB/s) | 20260506 01:09:04.209 | INFO | Downloaded 34GiB to GPU in 2.05013s (16.5843GiB/s) | 20260506 01:09:09.344 | INFO | Analyzing chunk frames = [7756288,7756800) | 20260506 01:09:09.344 | INFO | MemInfo: { | 20260506 01:09:09.344 | INFO | "gpu" : " 17 GB", | 20260506 01:09:09.344 | INFO | "huge" : "", | 20260506 01:09:09.344 | INFO | "huge_shared" : " 27 GB", | 20260506 01:09:09.344 | INFO | "malloc" : " 14 GB", | 20260506 01:09:09.344 | INFO | "pinned" : " 29 GB" | 20260506 01:09:09.344 | INFO | } | 20260506 01:09:09.344 | INFO | Perf report for the last 5.13593 seconds: | 20260506 01:09:09.344 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:09:09.344 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:09.344 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:09.344 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:09.344 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 01:09:09.344 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:09.344 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:09.344 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:09:09.344 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:09.344 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:09.345 | INFO | Uploaded 36GiB to GPU in 2.08154s (17.2948GiB/s) | 20260506 01:09:09.345 | INFO | Downloaded 34GiB to GPU in 2.0337s (16.7183GiB/s) | 20260506 01:09:14.474 | INFO | Analyzing chunk frames = [7756800,7757312) | 20260506 01:09:14.474 | INFO | MemInfo: { | 20260506 01:09:14.474 | INFO | "gpu" : " 17 GB", | 20260506 01:09:14.474 | INFO | "huge" : "", | 20260506 01:09:14.474 | INFO | "huge_shared" : " 27 GB", | 20260506 01:09:14.474 | INFO | "malloc" : " 14 GB", | 20260506 01:09:14.474 | INFO | "pinned" : " 29 GB" | 20260506 01:09:14.474 | INFO | } | 20260506 01:09:14.474 | INFO | Perf report for the last 5.12999 seconds: | 20260506 01:09:14.475 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:09:14.475 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:14.475 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:14.475 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:14.475 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 01:09:14.475 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:14.475 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:14.475 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260506 01:09:14.475 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:14.475 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:14.475 | INFO | Uploaded 36GiB to GPU in 2.08827s (17.2391GiB/s) | 20260506 01:09:14.475 | INFO | Downloaded 34GiB to GPU in 2.07769s (16.3644GiB/s) | 20260506 01:09:19.618 | INFO | Analyzing chunk frames = [7757312,7757824) | 20260506 01:09:19.618 | INFO | MemInfo: { | 20260506 01:09:19.618 | INFO | "gpu" : " 17 GB", | 20260506 01:09:19.618 | INFO | "huge" : "", | 20260506 01:09:19.618 | INFO | "huge_shared" : " 27 GB", | 20260506 01:09:19.618 | INFO | "malloc" : " 14 GB", | 20260506 01:09:19.618 | INFO | "pinned" : " 29 GB" | 20260506 01:09:19.618 | INFO | } | 20260506 01:09:19.618 | INFO | Perf report for the last 5.14366 seconds: | 20260506 01:09:19.618 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:09:19.618 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:19.618 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:19.618 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:19.618 | INFO | ANALYSIS : 61.9%, 3.0, 96.8ms, 0 | 20260506 01:09:19.618 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:19.618 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:19.618 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:09:19.618 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:19.618 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:19.618 | INFO | Uploaded 36GiB to GPU in 2.11762s (17.0002GiB/s) | 20260506 01:09:19.618 | INFO | Downloaded 34GiB to GPU in 2.00324s (16.9725GiB/s) | 20260506 01:09:19.618 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 01:09:19.618 | INFO | Total mem usage: 15.3 GiB | 20260506 01:09:19.619 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 01:09:19.619 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.17%) | 20260506 01:09:19.619 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 01:09:19.619 | INFO | High Water Breakdown: | 20260506 01:09:19.619 | INFO | 60.9% : 8.55 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 01:09:19.619 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 01:09:19.619 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 01:09:19.619 | INFO | ------------------------------------------ | 20260506 01:09:19.619 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:09:19.619 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 01:09:19.619 | INFO | Total mem usage: 26.5 GiB | 20260506 01:09:19.619 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 01:09:19.619 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 01:09:19.619 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 01:09:19.619 | INFO | High Water Breakdown: | 20260506 01:09:19.619 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:09:19.619 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:09:19.619 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:09:19.619 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:09:19.619 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:09:19.619 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:09:19.619 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 01:09:19.619 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 01:09:19.619 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:09:19.619 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 01:09:19.619 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 01:09:19.619 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 01:09:19.619 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 01:09:19.619 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 01:09:19.619 | INFO | ------------------------------------------ | 20260506 01:09:19.619 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:09:19.619 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 01:09:19.619 | INFO | Total mem usage: 25.6 GiB | 20260506 01:09:19.619 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 01:09:19.619 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 01:09:19.619 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 01:09:19.619 | INFO | High Water Breakdown: | 20260506 01:09:19.619 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 01:09:19.619 | INFO | ------------------------------------------ | 20260506 01:09:19.619 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:09:19.619 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 01:09:19.619 | INFO | Total mem usage: 15.7 GiB | 20260506 01:09:19.619 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 01:09:19.619 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 01:09:19.619 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 01:09:19.619 | INFO | High Water Breakdown: | 20260506 01:09:19.619 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 01:09:19.619 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 01:09:19.619 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 01:09:19.619 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 01:09:19.619 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 01:09:19.619 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 01:09:19.619 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 01:09:19.619 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 01:09:19.619 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 01:09:19.619 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:09:19.619 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 01:09:19.619 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:09:19.619 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 01:09:19.619 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 01:09:19.619 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 01:09:19.619 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 01:09:19.619 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:09:19.619 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 01:09:19.620 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:09:19.620 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:09:19.620 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:09:19.620 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 01:09:19.620 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 01:09:19.620 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:09:19.620 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 01:09:19.620 | INFO | ------------------------------------------ | 20260506 01:09:19.620 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:09:19.620 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 01:09:19.620 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:09:19.620 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:09:19.620 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.39e+01, 1.33e+01 / 7.74e-01 | 20260506 01:09:19.620 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.06e-01, 1.04e-02 / 5.30e-03 | 20260506 01:09:19.620 | INFO | PerfReport: 11.83%, Baseline -- 3.70e+00 / 4.50e+00, 5.68e-01 / 1.80e-01 | 20260506 01:09:19.620 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.51e+00, 1.89e-01 / 9.64e-02 | 20260506 01:09:19.620 | INFO | PerfReport: 15.96%, DME -- 4.00e+01 / 4.38e+01, 3.74e+00 / 8.64e-01 | 20260506 01:09:19.620 | INFO | PerfReport: 57.46%, FrameLabeling -- 1.80e+01 / 1.96e+01, 1.52e+00 / 1.19e+00 | 20260506 01:09:19.620 | INFO | PerfReport: 7.06%, PulseAccumulating -- 2.21e+00 / 2.70e+00, 4.19e-01 / 2.61e-01 | 20260506 01:09:19.620 | INFO | PerfReport: 3.05%, Metrics -- 9.54e-01 / 1.21e+00, 2.54e-01 / 8.62e-02 | 20260506 01:09:19.620 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.31e-01 / 3.81e-01, 5.56e-02 / 2.85e-02 | 20260506 01:09:19.620 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 01:09:19.620 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:09:19.620 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:09:19.620 | INFO | PerfReport: Full Execution -- 9.81e+01 / -nan, 2.45e+01 / 0.00e+00 | 20260506 01:09:19.620 | INFO | PerfReport: 22.41%, Upload -- 2.20e+01 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260506 01:09:19.620 | INFO | PerfReport: 21.87%, Download -- 2.15e+01 / 0.00e+00, 7.55e+00 / 0.00e+00 | 20260506 01:09:19.620 | INFO | PerfReport: 31.90%, Compute -- 3.13e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 01:09:19.620 | INFO | PerfReport: 2.11%, LockUpload -- 2.07e+00 / 0.00e+00, 7.94e+00 / 0.00e+00 | 20260506 01:09:19.620 | INFO | PerfReport: 1.28%, LockDownload -- 1.25e+00 / 0.00e+00, 2.85e+00 / 0.00e+00 | 20260506 01:09:19.620 | INFO | PerfReport: 20.41%, LockCompute -- 2.00e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 01:09:19.620 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.61e-02 / 0.00e+00 | 20260506 01:09:19.744 | INFO | Starting a round of Baz IO | 20260506 01:09:22.879 | INFO | Populating RTMetrics report | 20260506 01:09:22.879 | INFO | RTMetrics report populated | 20260506 01:09:22.879 | INFO | Opening tmp RTMetrics json file | 20260506 01:09:22.880 | INFO | Serializing RTMetrics to json | 20260506 01:09:22.881 | INFO | Serialization done, swapping tmp file now... | 20260506 01:09:22.882 | INFO | RTMetrics done | 20260506 01:09:22.955 | INFO | Finished a round of Baz IO: Wrote 1.834 GiB in 3.211 seconds | 20260506 01:09:22.955 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.33% / 0.00% / 78.67% | 20260506 01:09:22.955 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0835419% / 0% / 0.000111262% | 20260506 01:09:22.955 | INFO | Baz IO overall: | 20260506 01:09:22.955 | INFO | BytesWritten : 4385978863578 | 20260506 01:09:22.955 | INFO | NumEvents : 2062276271537 | 20260506 01:09:24.761 | INFO | Analyzing chunk frames = [7757824,7758336) | 20260506 01:09:24.761 | INFO | MemInfo: { | 20260506 01:09:24.761 | INFO | "gpu" : " 17 GB", | 20260506 01:09:24.761 | INFO | "huge" : "", | 20260506 01:09:24.761 | INFO | "huge_shared" : " 27 GB", | 20260506 01:09:24.761 | INFO | "malloc" : " 14 GB", | 20260506 01:09:24.761 | INFO | "pinned" : " 29 GB" | 20260506 01:09:24.761 | INFO | } | 20260506 01:09:24.761 | INFO | Perf report for the last 5.14258 seconds: | 20260506 01:09:24.761 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:09:24.761 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:24.761 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:24.761 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:24.761 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260506 01:09:24.761 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:24.761 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260506 01:09:24.761 | INFO | PRE_HQ : 59.5%, 1.7, 53.5ms, 0 | 20260506 01:09:24.761 | INFO | BAZWRITER : 61.7%, 3.5, 110.0ms, 0 | 20260506 01:09:24.761 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:09:24.761 | INFO | Uploaded 36GiB to GPU in 2.19026s (16.4364GiB/s) | 20260506 01:09:24.761 | INFO | Downloaded 38GiB to GPU in 2.2452s (16.925GiB/s) | 20260506 01:09:29.908 | INFO | Analyzing chunk frames = [7758336,7758848) | 20260506 01:09:29.908 | INFO | MemInfo: { | 20260506 01:09:29.908 | INFO | "gpu" : " 17 GB", | 20260506 01:09:29.908 | INFO | "huge" : "", | 20260506 01:09:29.908 | INFO | "huge_shared" : " 27 GB", | 20260506 01:09:29.908 | INFO | "malloc" : " 14 GB", | 20260506 01:09:29.908 | INFO | "pinned" : " 29 GB" | 20260506 01:09:29.908 | INFO | } | 20260506 01:09:29.908 | INFO | Perf report for the last 5.14707 seconds: | 20260506 01:09:29.908 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:09:29.908 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:29.908 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:29.908 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:29.908 | INFO | ANALYSIS : 60.9%, 3.0, 95.1ms, 0 | 20260506 01:09:29.908 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:29.908 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:29.908 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 01:09:29.908 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:29.908 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:29.908 | INFO | Uploaded 36GiB to GPU in 1.95398s (18.4239GiB/s) | 20260506 01:09:29.908 | INFO | Downloaded 34GiB to GPU in 1.98295s (17.1462GiB/s) | 20260506 01:09:35.039 | INFO | Analyzing chunk frames = [7758848,7759360) | 20260506 01:09:35.039 | INFO | MemInfo: { | 20260506 01:09:35.039 | INFO | "gpu" : " 17 GB", | 20260506 01:09:35.039 | INFO | "huge" : "", | 20260506 01:09:35.039 | INFO | "huge_shared" : " 27 GB", | 20260506 01:09:35.039 | INFO | "malloc" : " 14 GB", | 20260506 01:09:35.039 | INFO | "pinned" : " 29 GB" | 20260506 01:09:35.039 | INFO | } | 20260506 01:09:35.039 | INFO | Perf report for the last 5.13125 seconds: | 20260506 01:09:35.039 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:09:35.039 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:35.039 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:35.039 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:35.039 | INFO | ANALYSIS : 61.3%, 3.0, 95.2ms, 0 | 20260506 01:09:35.039 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:35.039 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:35.039 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:09:35.039 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:35.039 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:35.039 | INFO | Uploaded 36GiB to GPU in 2.1062s (17.0924GiB/s) | 20260506 01:09:35.039 | INFO | Downloaded 34GiB to GPU in 2.04014s (16.6656GiB/s) | 20260506 01:09:40.176 | INFO | Analyzing chunk frames = [7759360,7759872) | 20260506 01:09:40.176 | INFO | MemInfo: { | 20260506 01:09:40.176 | INFO | "gpu" : " 17 GB", | 20260506 01:09:40.176 | INFO | "huge" : "", | 20260506 01:09:40.176 | INFO | "huge_shared" : " 27 GB", | 20260506 01:09:40.176 | INFO | "malloc" : " 14 GB", | 20260506 01:09:40.176 | INFO | "pinned" : " 29 GB" | 20260506 01:09:40.176 | INFO | } | 20260506 01:09:40.176 | INFO | Perf report for the last 5.13726 seconds: | 20260506 01:09:40.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:09:40.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:40.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:40.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:40.176 | INFO | ANALYSIS : 61.6%, 3.0, 95.6ms, 0 | 20260506 01:09:40.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:40.176 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:40.177 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:09:40.177 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:40.177 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:40.177 | INFO | Uploaded 36GiB to GPU in 2.14896s (16.7523GiB/s) | 20260506 01:09:40.177 | INFO | Downloaded 34GiB to GPU in 2.04519s (16.6244GiB/s) | 20260506 01:09:45.315 | INFO | Analyzing chunk frames = [7759872,7760384) | 20260506 01:09:45.315 | INFO | MemInfo: { | 20260506 01:09:45.315 | INFO | "gpu" : " 17 GB", | 20260506 01:09:45.315 | INFO | "huge" : "", | 20260506 01:09:45.315 | INFO | "huge_shared" : " 27 GB", | 20260506 01:09:45.315 | INFO | "malloc" : " 14 GB", | 20260506 01:09:45.315 | INFO | "pinned" : " 29 GB" | 20260506 01:09:45.315 | INFO | } | 20260506 01:09:45.315 | INFO | Perf report for the last 5.13916 seconds: | 20260506 01:09:45.315 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:09:45.315 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:45.315 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:45.315 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:45.315 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 01:09:45.315 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:45.315 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:45.315 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 01:09:45.315 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:45.315 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:45.316 | INFO | Uploaded 36GiB to GPU in 2.09552s (17.1795GiB/s) | 20260506 01:09:45.316 | INFO | Downloaded 34GiB to GPU in 2.05342s (16.5577GiB/s) | 20260506 01:09:50.455 | INFO | Analyzing chunk frames = [7760384,7760896) | 20260506 01:09:50.455 | INFO | MemInfo: { | 20260506 01:09:50.455 | INFO | "gpu" : " 17 GB", | 20260506 01:09:50.455 | INFO | "huge" : "", | 20260506 01:09:50.455 | INFO | "huge_shared" : " 27 GB", | 20260506 01:09:50.455 | INFO | "malloc" : " 14 GB", | 20260506 01:09:50.455 | INFO | "pinned" : " 29 GB" | 20260506 01:09:50.455 | INFO | } | 20260506 01:09:50.455 | INFO | Perf report for the last 5.13971 seconds: | 20260506 01:09:50.455 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:09:50.455 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:50.455 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:50.455 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:50.455 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 01:09:50.455 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:50.455 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:50.455 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 01:09:50.455 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:50.455 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:50.455 | INFO | Uploaded 36GiB to GPU in 2.06617s (17.4235GiB/s) | 20260506 01:09:50.455 | INFO | Downloaded 34GiB to GPU in 2.03562s (16.7026GiB/s) | 20260506 01:09:55.596 | INFO | Analyzing chunk frames = [7760896,7761408) | 20260506 01:09:55.596 | INFO | MemInfo: { | 20260506 01:09:55.596 | INFO | "gpu" : " 17 GB", | 20260506 01:09:55.596 | INFO | "huge" : "", | 20260506 01:09:55.596 | INFO | "huge_shared" : " 27 GB", | 20260506 01:09:55.596 | INFO | "malloc" : " 14 GB", | 20260506 01:09:55.596 | INFO | "pinned" : " 29 GB" | 20260506 01:09:55.596 | INFO | } | 20260506 01:09:55.596 | INFO | Perf report for the last 5.1407 seconds: | 20260506 01:09:55.596 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:09:55.596 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:55.596 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:55.596 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:55.596 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260506 01:09:55.596 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:55.596 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:55.596 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 01:09:55.596 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:55.596 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:09:55.596 | INFO | Uploaded 36GiB to GPU in 2.06629s (17.4225GiB/s) | 20260506 01:09:55.596 | INFO | Downloaded 34GiB to GPU in 2.07717s (16.3684GiB/s) | 20260506 01:10:00.735 | INFO | Analyzing chunk frames = [7761408,7761920) | 20260506 01:10:00.735 | INFO | MemInfo: { | 20260506 01:10:00.735 | INFO | "gpu" : " 17 GB", | 20260506 01:10:00.735 | INFO | "huge" : "", | 20260506 01:10:00.735 | INFO | "huge_shared" : " 27 GB", | 20260506 01:10:00.735 | INFO | "malloc" : " 14 GB", | 20260506 01:10:00.735 | INFO | "pinned" : " 29 GB" | 20260506 01:10:00.735 | INFO | } | 20260506 01:10:00.735 | INFO | Perf report for the last 5.1392 seconds: | 20260506 01:10:00.735 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:10:00.735 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:00.735 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:00.735 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:00.735 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260506 01:10:00.735 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:00.735 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:00.735 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 01:10:00.735 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:00.735 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:00.735 | INFO | Uploaded 36GiB to GPU in 2.14006s (16.8219GiB/s) | 20260506 01:10:00.735 | INFO | Downloaded 34GiB to GPU in 2.02781s (16.7668GiB/s) | 20260506 01:10:00.852 | INFO | Starting a round of Baz IO | 20260506 01:10:03.999 | INFO | Populating RTMetrics report | 20260506 01:10:03.999 | INFO | RTMetrics report populated | 20260506 01:10:03.999 | INFO | Opening tmp RTMetrics json file | 20260506 01:10:04.000 | INFO | Serializing RTMetrics to json | 20260506 01:10:04.001 | INFO | Serialization done, swapping tmp file now... | 20260506 01:10:04.001 | INFO | RTMetrics done | 20260506 01:10:04.073 | INFO | Finished a round of Baz IO: Wrote 1.833 GiB in 3.221 seconds | 20260506 01:10:04.073 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.33% / 0.00% / 78.67% | 20260506 01:10:04.073 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0831008% / 0% / 0.000107288% | 20260506 01:10:04.073 | INFO | Baz IO overall: | 20260506 01:10:04.073 | INFO | BytesWritten : 4387946781497 | 20260506 01:10:04.073 | INFO | NumEvents : 2063173634704 | 20260506 01:10:05.867 | INFO | Analyzing chunk frames = [7761920,7762432) | 20260506 01:10:05.867 | INFO | MemInfo: { | 20260506 01:10:05.867 | INFO | "gpu" : " 17 GB", | 20260506 01:10:05.867 | INFO | "huge" : "", | 20260506 01:10:05.867 | INFO | "huge_shared" : " 27 GB", | 20260506 01:10:05.867 | INFO | "malloc" : " 14 GB", | 20260506 01:10:05.867 | INFO | "pinned" : " 29 GB" | 20260506 01:10:05.867 | INFO | } | 20260506 01:10:05.867 | INFO | Perf report for the last 5.13227 seconds: | 20260506 01:10:05.867 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:10:05.867 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:05.867 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:05.867 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:05.867 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 01:10:05.867 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:05.867 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260506 01:10:05.867 | INFO | PRE_HQ : 59.8%, 1.7, 53.8ms, 0 | 20260506 01:10:05.867 | INFO | BAZWRITER : 61.9%, 3.5, 110.0ms, 0 | 20260506 01:10:05.867 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:10:05.868 | INFO | Uploaded 36GiB to GPU in 2.17238s (16.5717GiB/s) | 20260506 01:10:05.868 | INFO | Downloaded 38GiB to GPU in 2.2405s (16.9605GiB/s) | 20260506 01:10:11.013 | INFO | Analyzing chunk frames = [7762432,7762944) | 20260506 01:10:11.013 | INFO | MemInfo: { | 20260506 01:10:11.013 | INFO | "gpu" : " 17 GB", | 20260506 01:10:11.013 | INFO | "huge" : "", | 20260506 01:10:11.013 | INFO | "huge_shared" : " 27 GB", | 20260506 01:10:11.013 | INFO | "malloc" : " 14 GB", | 20260506 01:10:11.013 | INFO | "pinned" : " 29 GB" | 20260506 01:10:11.013 | INFO | } | 20260506 01:10:11.013 | INFO | Perf report for the last 5.14551 seconds: | 20260506 01:10:11.013 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:10:11.013 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:11.013 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:11.013 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:11.013 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260506 01:10:11.013 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:11.013 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:11.013 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 01:10:11.013 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:11.013 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:11.013 | INFO | Uploaded 36GiB to GPU in 2.11309s (17.0367GiB/s) | 20260506 01:10:11.013 | INFO | Downloaded 34GiB to GPU in 2.0428s (16.6438GiB/s) | 20260506 01:10:16.158 | INFO | Analyzing chunk frames = [7762944,7763456) | 20260506 01:10:16.158 | INFO | MemInfo: { | 20260506 01:10:16.158 | INFO | "gpu" : " 17 GB", | 20260506 01:10:16.158 | INFO | "huge" : "", | 20260506 01:10:16.158 | INFO | "huge_shared" : " 27 GB", | 20260506 01:10:16.158 | INFO | "malloc" : " 14 GB", | 20260506 01:10:16.158 | INFO | "pinned" : " 29 GB" | 20260506 01:10:16.158 | INFO | } | 20260506 01:10:16.158 | INFO | Perf report for the last 5.14499 seconds: | 20260506 01:10:16.158 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:10:16.158 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:16.158 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:16.158 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:16.158 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 01:10:16.158 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:16.158 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:16.158 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:10:16.158 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:16.158 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:16.158 | INFO | Uploaded 36GiB to GPU in 2.08412s (17.2735GiB/s) | 20260506 01:10:16.158 | INFO | Downloaded 34GiB to GPU in 2.01657s (16.8603GiB/s) | 20260506 01:10:21.290 | INFO | Analyzing chunk frames = [7763456,7763968) | 20260506 01:10:21.290 | INFO | MemInfo: { | 20260506 01:10:21.290 | INFO | "gpu" : " 17 GB", | 20260506 01:10:21.290 | INFO | "huge" : "", | 20260506 01:10:21.290 | INFO | "huge_shared" : " 27 GB", | 20260506 01:10:21.290 | INFO | "malloc" : " 14 GB", | 20260506 01:10:21.290 | INFO | "pinned" : " 29 GB" | 20260506 01:10:21.290 | INFO | } | 20260506 01:10:21.290 | INFO | Perf report for the last 5.13187 seconds: | 20260506 01:10:21.290 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:10:21.290 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:21.290 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:21.290 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:21.290 | INFO | ANALYSIS : 61.3%, 3.0, 95.0ms, 0 | 20260506 01:10:21.290 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:21.290 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:21.290 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 01:10:21.290 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:21.290 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:21.290 | INFO | Uploaded 36GiB to GPU in 2.09661s (17.1705GiB/s) | 20260506 01:10:21.290 | INFO | Downloaded 34GiB to GPU in 2.02781s (16.7668GiB/s) | 20260506 01:10:26.425 | INFO | Analyzing chunk frames = [7763968,7764480) | 20260506 01:10:26.425 | INFO | MemInfo: { | 20260506 01:10:26.425 | INFO | "gpu" : " 17 GB", | 20260506 01:10:26.425 | INFO | "huge" : "", | 20260506 01:10:26.425 | INFO | "huge_shared" : " 27 GB", | 20260506 01:10:26.425 | INFO | "malloc" : " 14 GB", | 20260506 01:10:26.425 | INFO | "pinned" : " 29 GB" | 20260506 01:10:26.425 | INFO | } | 20260506 01:10:26.425 | INFO | Perf report for the last 5.13565 seconds: | 20260506 01:10:26.425 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:10:26.425 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:26.425 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:26.425 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:26.425 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 01:10:26.425 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:26.425 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:26.425 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:10:26.425 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:26.425 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:26.426 | INFO | Uploaded 36GiB to GPU in 2.08406s (17.274GiB/s) | 20260506 01:10:26.426 | INFO | Downloaded 34GiB to GPU in 2.0505s (16.5813GiB/s) | 20260506 01:10:31.572 | INFO | Analyzing chunk frames = [7764480,7764992) | 20260506 01:10:31.572 | INFO | MemInfo: { | 20260506 01:10:31.572 | INFO | "gpu" : " 17 GB", | 20260506 01:10:31.572 | INFO | "huge" : "", | 20260506 01:10:31.572 | INFO | "huge_shared" : " 27 GB", | 20260506 01:10:31.572 | INFO | "malloc" : " 14 GB", | 20260506 01:10:31.572 | INFO | "pinned" : " 29 GB" | 20260506 01:10:31.572 | INFO | } | 20260506 01:10:31.572 | INFO | Perf report for the last 5.14678 seconds: | 20260506 01:10:31.572 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:10:31.572 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:31.572 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:31.572 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:31.572 | INFO | ANALYSIS : 61.3%, 3.0, 95.8ms, 0 | 20260506 01:10:31.572 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:31.572 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:31.572 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 01:10:31.572 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:31.572 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:31.572 | INFO | Uploaded 36GiB to GPU in 2.07981s (17.3092GiB/s) | 20260506 01:10:31.572 | INFO | Downloaded 34GiB to GPU in 2.04294s (16.6426GiB/s) | 20260506 01:10:36.718 | INFO | Analyzing chunk frames = [7764992,7765504) | 20260506 01:10:36.718 | INFO | MemInfo: { | 20260506 01:10:36.718 | INFO | "gpu" : " 17 GB", | 20260506 01:10:36.718 | INFO | "huge" : "", | 20260506 01:10:36.718 | INFO | "huge_shared" : " 27 GB", | 20260506 01:10:36.718 | INFO | "malloc" : " 14 GB", | 20260506 01:10:36.718 | INFO | "pinned" : " 29 GB" | 20260506 01:10:36.718 | INFO | } | 20260506 01:10:36.718 | INFO | Perf report for the last 5.14569 seconds: | 20260506 01:10:36.718 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:10:36.718 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:36.718 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:36.718 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:36.718 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260506 01:10:36.718 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:36.718 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:36.718 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 01:10:36.718 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:36.718 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:36.718 | INFO | Uploaded 36GiB to GPU in 2.08787s (17.2425GiB/s) | 20260506 01:10:36.718 | INFO | Downloaded 34GiB to GPU in 2.0955s (16.2252GiB/s) | 20260506 01:10:41.855 | INFO | Analyzing chunk frames = [7765504,7766016) | 20260506 01:10:41.855 | INFO | MemInfo: { | 20260506 01:10:41.855 | INFO | "gpu" : " 17 GB", | 20260506 01:10:41.855 | INFO | "huge" : "", | 20260506 01:10:41.855 | INFO | "huge_shared" : " 27 GB", | 20260506 01:10:41.855 | INFO | "malloc" : " 14 GB", | 20260506 01:10:41.855 | INFO | "pinned" : " 29 GB" | 20260506 01:10:41.855 | INFO | } | 20260506 01:10:41.855 | INFO | Perf report for the last 5.13707 seconds: | 20260506 01:10:41.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:10:41.855 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:41.855 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:41.855 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:41.855 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260506 01:10:41.855 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:41.855 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:41.855 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 01:10:41.855 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:41.855 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:41.855 | INFO | Uploaded 36GiB to GPU in 2.02287s (17.7965GiB/s) | 20260506 01:10:41.855 | INFO | Downloaded 34GiB to GPU in 2.01434s (16.8789GiB/s) | 20260506 01:10:41.855 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 01:10:41.855 | INFO | Total mem usage: 15.3 GiB | 20260506 01:10:41.855 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 01:10:41.855 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.18%) | 20260506 01:10:41.855 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 01:10:41.856 | INFO | High Water Breakdown: | 20260506 01:10:41.856 | INFO | 60.9% : 8.55 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 01:10:41.856 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 01:10:41.856 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 01:10:41.856 | INFO | ------------------------------------------ | 20260506 01:10:41.856 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:10:41.856 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 01:10:41.856 | INFO | Total mem usage: 26.5 GiB | 20260506 01:10:41.856 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 01:10:41.856 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 01:10:41.856 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260506 01:10:41.856 | INFO | High Water Breakdown: | 20260506 01:10:41.856 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:10:41.856 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:10:41.856 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:10:41.856 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:10:41.856 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:10:41.856 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:10:41.856 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 01:10:41.856 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 01:10:41.856 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:10:41.856 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 01:10:41.856 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 01:10:41.856 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 01:10:41.856 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 01:10:41.856 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 01:10:41.856 | INFO | ------------------------------------------ | 20260506 01:10:41.856 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:10:41.856 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 01:10:41.856 | INFO | Total mem usage: 25.6 GiB | 20260506 01:10:41.856 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 01:10:41.856 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 01:10:41.856 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 01:10:41.856 | INFO | High Water Breakdown: | 20260506 01:10:41.856 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 01:10:41.856 | INFO | ------------------------------------------ | 20260506 01:10:41.856 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:10:41.856 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 01:10:41.856 | INFO | Total mem usage: 15.7 GiB | 20260506 01:10:41.856 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 01:10:41.856 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 01:10:41.856 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 01:10:41.856 | INFO | High Water Breakdown: | 20260506 01:10:41.856 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 01:10:41.856 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 01:10:41.856 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 01:10:41.856 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 01:10:41.856 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 01:10:41.856 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 01:10:41.856 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 01:10:41.856 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 01:10:41.856 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 01:10:41.856 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:10:41.856 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 01:10:41.856 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:10:41.856 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 01:10:41.856 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 01:10:41.856 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 01:10:41.856 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 01:10:41.856 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:10:41.856 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 01:10:41.856 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:10:41.856 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:10:41.856 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:10:41.856 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 01:10:41.856 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 01:10:41.856 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:10:41.856 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 01:10:41.856 | INFO | ------------------------------------------ | 20260506 01:10:41.856 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:10:41.856 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 01:10:41.856 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:10:41.856 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:10:41.856 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.38e+01, 1.34e+01 / 7.73e-01 | 20260506 01:10:41.856 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.05e-01, 1.03e-02 / 4.54e-03 | 20260506 01:10:41.856 | INFO | PerfReport: 11.84%, Baseline -- 3.70e+00 / 4.50e+00, 5.73e-01 / 2.13e-01 | 20260506 01:10:41.856 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.52e+00, 1.91e-01 / 1.02e-01 | 20260506 01:10:41.856 | INFO | PerfReport: 15.98%, DME -- 4.00e+01 / 4.38e+01, 3.86e+00 / 9.49e-01 | 20260506 01:10:41.856 | INFO | PerfReport: 57.42%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.49e+00 / 1.25e+00 | 20260506 01:10:41.856 | INFO | PerfReport: 7.07%, PulseAccumulating -- 2.21e+00 / 2.72e+00, 4.19e-01 / 2.39e-01 | 20260506 01:10:41.856 | INFO | PerfReport: 3.06%, Metrics -- 9.56e-01 / 1.23e+00, 2.57e-01 / 1.08e-01 | 20260506 01:10:41.856 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.30e-01 / 3.82e-01, 5.63e-02 / 2.81e-02 | 20260506 01:10:41.856 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 01:10:41.856 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:10:41.856 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:10:41.856 | INFO | PerfReport: Full Execution -- 9.80e+01 / -nan, 2.43e+01 / 0.00e+00 | 20260506 01:10:41.856 | INFO | PerfReport: 22.26%, Upload -- 2.18e+01 / 0.00e+00, 7.38e+00 / 0.00e+00 | 20260506 01:10:41.856 | INFO | PerfReport: 21.95%, Download -- 2.15e+01 / 0.00e+00, 7.59e+00 / 0.00e+00 | 20260506 01:10:41.856 | INFO | PerfReport: 31.92%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 01:10:41.856 | INFO | PerfReport: 2.05%, LockUpload -- 2.01e+00 / 0.00e+00, 7.81e+00 / 0.00e+00 | 20260506 01:10:41.856 | INFO | PerfReport: 1.32%, LockDownload -- 1.30e+00 / 0.00e+00, 2.87e+00 / 0.00e+00 | 20260506 01:10:41.857 | INFO | PerfReport: 20.49%, LockCompute -- 2.01e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 01:10:41.857 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.81e-02 / 0.00e+00 | 20260506 01:10:41.981 | INFO | Starting a round of Baz IO | 20260506 01:10:45.120 | INFO | Populating RTMetrics report | 20260506 01:10:45.120 | INFO | RTMetrics report populated | 20260506 01:10:45.120 | INFO | Opening tmp RTMetrics json file | 20260506 01:10:45.121 | INFO | Serializing RTMetrics to json | 20260506 01:10:45.122 | INFO | Serialization done, swapping tmp file now... | 20260506 01:10:45.122 | INFO | RTMetrics done | 20260506 01:10:45.192 | INFO | Finished a round of Baz IO: Wrote 1.835 GiB in 3.211 seconds | 20260506 01:10:45.192 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.33% / 0.00% / 78.67% | 20260506 01:10:45.192 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0835101% / 0% / 7.94729e-05% | 20260506 01:10:45.192 | INFO | Baz IO overall: | 20260506 01:10:45.192 | INFO | BytesWritten : 4389917466014 | 20260506 01:10:45.192 | INFO | NumEvents : 2064073268676 | 20260506 01:10:46.990 | INFO | Analyzing chunk frames = [7766016,7766528) | 20260506 01:10:46.990 | INFO | MemInfo: { | 20260506 01:10:46.990 | INFO | "gpu" : " 17 GB", | 20260506 01:10:46.990 | INFO | "huge" : "", | 20260506 01:10:46.990 | INFO | "huge_shared" : " 27 GB", | 20260506 01:10:46.990 | INFO | "malloc" : " 14 GB", | 20260506 01:10:46.990 | INFO | "pinned" : " 29 GB" | 20260506 01:10:46.990 | INFO | } | 20260506 01:10:46.990 | INFO | Perf report for the last 5.13502 seconds: | 20260506 01:10:46.990 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:10:46.990 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:46.990 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:46.990 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:46.990 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260506 01:10:46.990 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:46.990 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260506 01:10:46.990 | INFO | PRE_HQ : 59.6%, 1.7, 53.8ms, 0 | 20260506 01:10:46.990 | INFO | BAZWRITER : 61.8%, 3.5, 109.8ms, 0 | 20260506 01:10:46.990 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:10:46.990 | INFO | Uploaded 36GiB to GPU in 2.182s (16.4987GiB/s) | 20260506 01:10:46.990 | INFO | Downloaded 38GiB to GPU in 2.2399s (16.9651GiB/s) | 20260506 01:10:52.125 | INFO | Analyzing chunk frames = [7766528,7767040) | 20260506 01:10:52.125 | INFO | MemInfo: { | 20260506 01:10:52.125 | INFO | "gpu" : " 17 GB", | 20260506 01:10:52.125 | INFO | "huge" : "", | 20260506 01:10:52.125 | INFO | "huge_shared" : " 27 GB", | 20260506 01:10:52.125 | INFO | "malloc" : " 14 GB", | 20260506 01:10:52.125 | INFO | "pinned" : " 29 GB" | 20260506 01:10:52.125 | INFO | } | 20260506 01:10:52.125 | INFO | Perf report for the last 5.1348 seconds: | 20260506 01:10:52.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:10:52.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:52.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:52.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:52.125 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 01:10:52.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:52.125 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:52.125 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:10:52.125 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:52.125 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:52.125 | INFO | Uploaded 36GiB to GPU in 1.9922s (18.0705GiB/s) | 20260506 01:10:52.125 | INFO | Downloaded 34GiB to GPU in 1.9939s (17.052GiB/s) | 20260506 01:10:57.269 | INFO | Analyzing chunk frames = [7767040,7767552) | 20260506 01:10:57.269 | INFO | MemInfo: { | 20260506 01:10:57.269 | INFO | "gpu" : " 17 GB", | 20260506 01:10:57.269 | INFO | "huge" : "", | 20260506 01:10:57.269 | INFO | "huge_shared" : " 27 GB", | 20260506 01:10:57.269 | INFO | "malloc" : " 14 GB", | 20260506 01:10:57.269 | INFO | "pinned" : " 29 GB" | 20260506 01:10:57.269 | INFO | } | 20260506 01:10:57.269 | INFO | Perf report for the last 5.14458 seconds: | 20260506 01:10:57.269 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:10:57.269 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:57.269 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:57.269 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:57.269 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 01:10:57.269 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:57.269 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:57.269 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 01:10:57.269 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:57.269 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:10:57.270 | INFO | Uploaded 36GiB to GPU in 2.075s (17.3494GiB/s) | 20260506 01:10:57.270 | INFO | Downloaded 34GiB to GPU in 2.02245s (16.8113GiB/s) | 20260506 01:11:02.407 | INFO | Analyzing chunk frames = [7767552,7768064) | 20260506 01:11:02.407 | INFO | MemInfo: { | 20260506 01:11:02.407 | INFO | "gpu" : " 17 GB", | 20260506 01:11:02.407 | INFO | "huge" : "", | 20260506 01:11:02.407 | INFO | "huge_shared" : " 27 GB", | 20260506 01:11:02.407 | INFO | "malloc" : " 14 GB", | 20260506 01:11:02.407 | INFO | "pinned" : " 29 GB" | 20260506 01:11:02.407 | INFO | } | 20260506 01:11:02.407 | INFO | Perf report for the last 5.13756 seconds: | 20260506 01:11:02.407 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:11:02.407 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:02.407 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:02.407 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:02.407 | INFO | ANALYSIS : 61.5%, 3.0, 95.5ms, 0 | 20260506 01:11:02.407 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:02.407 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:02.407 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 01:11:02.407 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:02.407 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:02.407 | INFO | Uploaded 36GiB to GPU in 2.15203s (16.7284GiB/s) | 20260506 01:11:02.407 | INFO | Downloaded 34GiB to GPU in 2.05632s (16.5344GiB/s) | 20260506 01:11:07.547 | INFO | Analyzing chunk frames = [7768064,7768576) | 20260506 01:11:07.547 | INFO | MemInfo: { | 20260506 01:11:07.547 | INFO | "gpu" : " 17 GB", | 20260506 01:11:07.547 | INFO | "huge" : "", | 20260506 01:11:07.547 | INFO | "huge_shared" : " 27 GB", | 20260506 01:11:07.547 | INFO | "malloc" : " 14 GB", | 20260506 01:11:07.547 | INFO | "pinned" : " 29 GB" | 20260506 01:11:07.547 | INFO | } | 20260506 01:11:07.547 | INFO | Perf report for the last 5.14025 seconds: | 20260506 01:11:07.547 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:11:07.547 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:07.547 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:07.547 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:07.547 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 01:11:07.547 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:07.547 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:07.547 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:11:07.547 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:07.547 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:07.548 | INFO | Uploaded 36GiB to GPU in 2.07982s (17.3092GiB/s) | 20260506 01:11:07.548 | INFO | Downloaded 34GiB to GPU in 2.04464s (16.6289GiB/s) | 20260506 01:11:12.684 | INFO | Analyzing chunk frames = [7768576,7769088) | 20260506 01:11:12.684 | INFO | MemInfo: { | 20260506 01:11:12.684 | INFO | "gpu" : " 17 GB", | 20260506 01:11:12.684 | INFO | "huge" : "", | 20260506 01:11:12.684 | INFO | "huge_shared" : " 27 GB", | 20260506 01:11:12.684 | INFO | "malloc" : " 14 GB", | 20260506 01:11:12.684 | INFO | "pinned" : " 29 GB" | 20260506 01:11:12.684 | INFO | } | 20260506 01:11:12.684 | INFO | Perf report for the last 5.13688 seconds: | 20260506 01:11:12.684 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:11:12.684 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:12.684 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:12.684 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:12.684 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260506 01:11:12.684 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:12.684 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:12.684 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 01:11:12.684 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:12.684 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:12.685 | INFO | Uploaded 36GiB to GPU in 2.06911s (17.3988GiB/s) | 20260506 01:11:12.685 | INFO | Downloaded 34GiB to GPU in 2.03608s (16.6987GiB/s) | 20260506 01:11:17.823 | INFO | Analyzing chunk frames = [7769088,7769600) | 20260506 01:11:17.823 | INFO | MemInfo: { | 20260506 01:11:17.823 | INFO | "gpu" : " 17 GB", | 20260506 01:11:17.823 | INFO | "huge" : "", | 20260506 01:11:17.823 | INFO | "huge_shared" : " 27 GB", | 20260506 01:11:17.823 | INFO | "malloc" : " 14 GB", | 20260506 01:11:17.823 | INFO | "pinned" : " 29 GB" | 20260506 01:11:17.823 | INFO | } | 20260506 01:11:17.823 | INFO | Perf report for the last 5.13927 seconds: | 20260506 01:11:17.823 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:11:17.823 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:17.823 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:17.823 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:17.823 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260506 01:11:17.823 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:17.823 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:17.823 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 01:11:17.823 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:17.824 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:17.824 | INFO | Uploaded 36GiB to GPU in 2.036s (17.6818GiB/s) | 20260506 01:11:17.824 | INFO | Downloaded 34GiB to GPU in 2.04174s (16.6525GiB/s) | 20260506 01:11:22.966 | INFO | Analyzing chunk frames = [7769600,7770112) | 20260506 01:11:22.966 | INFO | MemInfo: { | 20260506 01:11:22.966 | INFO | "gpu" : " 17 GB", | 20260506 01:11:22.966 | INFO | "huge" : "", | 20260506 01:11:22.966 | INFO | "huge_shared" : " 27 GB", | 20260506 01:11:22.966 | INFO | "malloc" : " 14 GB", | 20260506 01:11:22.966 | INFO | "pinned" : " 29 GB" | 20260506 01:11:22.966 | INFO | } | 20260506 01:11:22.966 | INFO | Perf report for the last 5.14233 seconds: | 20260506 01:11:22.966 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:11:22.966 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:22.966 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:22.966 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:22.966 | INFO | ANALYSIS : 61.5%, 3.0, 96.1ms, 0 | 20260506 01:11:22.966 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:22.966 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:22.966 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 01:11:22.966 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:22.966 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:22.966 | INFO | Uploaded 36GiB to GPU in 2.06158s (17.4623GiB/s) | 20260506 01:11:22.966 | INFO | Downloaded 34GiB to GPU in 2.01798s (16.8485GiB/s) | 20260506 01:11:23.092 | INFO | Starting a round of Baz IO | 20260506 01:11:26.226 | INFO | Populating RTMetrics report | 20260506 01:11:26.226 | INFO | RTMetrics report populated | 20260506 01:11:26.226 | INFO | Opening tmp RTMetrics json file | 20260506 01:11:26.226 | INFO | Serializing RTMetrics to json | 20260506 01:11:26.228 | INFO | Serialization done, swapping tmp file now... | 20260506 01:11:26.228 | INFO | RTMetrics done | 20260506 01:11:26.300 | INFO | Finished a round of Baz IO: Wrote 1.833 GiB in 3.208 seconds | 20260506 01:11:26.300 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.33% / 0.00% / 78.67% | 20260506 01:11:26.300 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0831525% / 3.97364e-06% / 9.93411e-05% | 20260506 01:11:26.300 | INFO | Baz IO overall: | 20260506 01:11:26.300 | INFO | BytesWritten : 4391886136464 | 20260506 01:11:26.300 | INFO | NumEvents : 2064971690419 | 20260506 01:11:28.105 | INFO | Analyzing chunk frames = [7770112,7770624) | 20260506 01:11:28.105 | INFO | MemInfo: { | 20260506 01:11:28.105 | INFO | "gpu" : " 17 GB", | 20260506 01:11:28.105 | INFO | "huge" : "", | 20260506 01:11:28.105 | INFO | "huge_shared" : " 27 GB", | 20260506 01:11:28.105 | INFO | "malloc" : " 14 GB", | 20260506 01:11:28.105 | INFO | "pinned" : " 29 GB" | 20260506 01:11:28.105 | INFO | } | 20260506 01:11:28.105 | INFO | Perf report for the last 5.13961 seconds: | 20260506 01:11:28.105 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:11:28.105 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:28.105 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:28.105 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:28.105 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260506 01:11:28.105 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:28.105 | INFO | RT_METRICS : 43.2%, 1.0, 22.6ms, 0 | 20260506 01:11:28.105 | INFO | PRE_HQ : 59.9%, 1.7, 54.1ms, 0 | 20260506 01:11:28.106 | INFO | BAZWRITER : 61.7%, 3.5, 110.6ms, 0 | 20260506 01:11:28.106 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:11:28.106 | INFO | Uploaded 36GiB to GPU in 2.18288s (16.492GiB/s) | 20260506 01:11:28.106 | INFO | Downloaded 38GiB to GPU in 2.24284s (16.9428GiB/s) | 20260506 01:11:33.237 | INFO | Analyzing chunk frames = [7770624,7771136) | 20260506 01:11:33.237 | INFO | MemInfo: { | 20260506 01:11:33.237 | INFO | "gpu" : " 17 GB", | 20260506 01:11:33.237 | INFO | "huge" : "", | 20260506 01:11:33.237 | INFO | "huge_shared" : " 27 GB", | 20260506 01:11:33.237 | INFO | "malloc" : " 14 GB", | 20260506 01:11:33.237 | INFO | "pinned" : " 29 GB" | 20260506 01:11:33.237 | INFO | } | 20260506 01:11:33.237 | INFO | Perf report for the last 5.13127 seconds: | 20260506 01:11:33.237 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:11:33.237 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:33.237 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:33.237 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:33.237 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260506 01:11:33.237 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:33.237 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:33.237 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:11:33.237 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:33.237 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:33.237 | INFO | Uploaded 36GiB to GPU in 1.98649s (18.1224GiB/s) | 20260506 01:11:33.237 | INFO | Downloaded 34GiB to GPU in 1.98329s (17.1432GiB/s) | 20260506 01:11:38.376 | INFO | Analyzing chunk frames = [7771136,7771648) | 20260506 01:11:38.376 | INFO | MemInfo: { | 20260506 01:11:38.376 | INFO | "gpu" : " 17 GB", | 20260506 01:11:38.376 | INFO | "huge" : "", | 20260506 01:11:38.376 | INFO | "huge_shared" : " 27 GB", | 20260506 01:11:38.376 | INFO | "malloc" : " 14 GB", | 20260506 01:11:38.376 | INFO | "pinned" : " 29 GB" | 20260506 01:11:38.376 | INFO | } | 20260506 01:11:38.376 | INFO | Perf report for the last 5.13979 seconds: | 20260506 01:11:38.376 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:11:38.376 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:38.376 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:38.376 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:38.377 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 01:11:38.377 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:38.377 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:38.377 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:11:38.377 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:38.377 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:38.377 | INFO | Uploaded 36GiB to GPU in 2.09535s (17.1809GiB/s) | 20260506 01:11:38.377 | INFO | Downloaded 34GiB to GPU in 2.03237s (16.7292GiB/s) | 20260506 01:11:43.523 | INFO | Analyzing chunk frames = [7771648,7772160) | 20260506 01:11:43.523 | INFO | MemInfo: { | 20260506 01:11:43.523 | INFO | "gpu" : " 17 GB", | 20260506 01:11:43.523 | INFO | "huge" : "", | 20260506 01:11:43.523 | INFO | "huge_shared" : " 27 GB", | 20260506 01:11:43.523 | INFO | "malloc" : " 14 GB", | 20260506 01:11:43.523 | INFO | "pinned" : " 29 GB" | 20260506 01:11:43.523 | INFO | } | 20260506 01:11:43.523 | INFO | Perf report for the last 5.14692 seconds: | 20260506 01:11:43.523 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:11:43.523 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:43.523 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:43.523 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:43.523 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 01:11:43.523 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:43.523 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:43.523 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 01:11:43.523 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:43.523 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:43.524 | INFO | Uploaded 36GiB to GPU in 2.13659s (16.8493GiB/s) | 20260506 01:11:43.524 | INFO | Downloaded 34GiB to GPU in 2.01073s (16.9093GiB/s) | 20260506 01:11:48.666 | INFO | Analyzing chunk frames = [7772160,7772672) | 20260506 01:11:48.666 | INFO | MemInfo: { | 20260506 01:11:48.666 | INFO | "gpu" : " 17 GB", | 20260506 01:11:48.666 | INFO | "huge" : "", | 20260506 01:11:48.666 | INFO | "huge_shared" : " 27 GB", | 20260506 01:11:48.666 | INFO | "malloc" : " 14 GB", | 20260506 01:11:48.666 | INFO | "pinned" : " 29 GB" | 20260506 01:11:48.666 | INFO | } | 20260506 01:11:48.666 | INFO | Perf report for the last 5.14259 seconds: | 20260506 01:11:48.666 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:11:48.666 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:48.666 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:48.666 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:48.666 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 01:11:48.666 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:48.666 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:48.666 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 01:11:48.666 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:48.666 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:48.666 | INFO | Uploaded 36GiB to GPU in 2.10664s (17.0888GiB/s) | 20260506 01:11:48.666 | INFO | Downloaded 34GiB to GPU in 2.04866s (16.5962GiB/s) | 20260506 01:11:53.801 | INFO | Analyzing chunk frames = [7772672,7773184) | 20260506 01:11:53.801 | INFO | MemInfo: { | 20260506 01:11:53.801 | INFO | "gpu" : " 17 GB", | 20260506 01:11:53.801 | INFO | "huge" : "", | 20260506 01:11:53.801 | INFO | "huge_shared" : " 27 GB", | 20260506 01:11:53.801 | INFO | "malloc" : " 14 GB", | 20260506 01:11:53.801 | INFO | "pinned" : " 29 GB" | 20260506 01:11:53.801 | INFO | } | 20260506 01:11:53.801 | INFO | Perf report for the last 5.13526 seconds: | 20260506 01:11:53.801 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:11:53.801 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:53.801 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:53.801 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:53.801 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 01:11:53.801 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:53.801 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:53.801 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 01:11:53.801 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:53.801 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:53.802 | INFO | Uploaded 36GiB to GPU in 2.08143s (17.2958GiB/s) | 20260506 01:11:53.802 | INFO | Downloaded 34GiB to GPU in 2.0381s (16.6822GiB/s) | 20260506 01:11:58.942 | INFO | Analyzing chunk frames = [7773184,7773696) | 20260506 01:11:58.942 | INFO | MemInfo: { | 20260506 01:11:58.942 | INFO | "gpu" : " 17 GB", | 20260506 01:11:58.942 | INFO | "huge" : "", | 20260506 01:11:58.942 | INFO | "huge_shared" : " 27 GB", | 20260506 01:11:58.942 | INFO | "malloc" : " 14 GB", | 20260506 01:11:58.942 | INFO | "pinned" : " 29 GB" | 20260506 01:11:58.942 | INFO | } | 20260506 01:11:58.942 | INFO | Perf report for the last 5.14086 seconds: | 20260506 01:11:58.942 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:11:58.942 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:58.942 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:58.942 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:58.942 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260506 01:11:58.942 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:58.942 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:58.942 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 01:11:58.942 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:58.942 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:11:58.942 | INFO | Uploaded 36GiB to GPU in 2.03456s (17.6942GiB/s) | 20260506 01:11:58.942 | INFO | Downloaded 34GiB to GPU in 2.05024s (16.5834GiB/s) | 20260506 01:12:04.081 | INFO | Analyzing chunk frames = [7773696,7774208) | 20260506 01:12:04.081 | INFO | MemInfo: { | 20260506 01:12:04.081 | INFO | "gpu" : " 17 GB", | 20260506 01:12:04.081 | INFO | "huge" : "", | 20260506 01:12:04.081 | INFO | "huge_shared" : " 27 GB", | 20260506 01:12:04.081 | INFO | "malloc" : " 14 GB", | 20260506 01:12:04.081 | INFO | "pinned" : " 29 GB" | 20260506 01:12:04.081 | INFO | } | 20260506 01:12:04.081 | INFO | Perf report for the last 5.13916 seconds: | 20260506 01:12:04.081 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:12:04.081 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:04.081 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:04.081 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:04.081 | INFO | ANALYSIS : 61.5%, 3.0, 96.1ms, 0 | 20260506 01:12:04.081 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:04.081 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:04.081 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 01:12:04.081 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:04.081 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:04.082 | INFO | Uploaded 36GiB to GPU in 2.06341s (17.4469GiB/s) | 20260506 01:12:04.082 | INFO | Downloaded 34GiB to GPU in 2.02361s (16.8016GiB/s) | 20260506 01:12:04.082 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 01:12:04.082 | INFO | Total mem usage: 15.3 GiB | 20260506 01:12:04.082 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 01:12:04.082 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.18%) | 20260506 01:12:04.082 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 01:12:04.082 | INFO | High Water Breakdown: | 20260506 01:12:04.082 | INFO | 60.9% : 8.55 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 01:12:04.082 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 01:12:04.082 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 01:12:04.082 | INFO | ------------------------------------------ | 20260506 01:12:04.082 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:12:04.082 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 01:12:04.082 | INFO | Total mem usage: 26.5 GiB | 20260506 01:12:04.082 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 01:12:04.082 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 01:12:04.082 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260506 01:12:04.082 | INFO | High Water Breakdown: | 20260506 01:12:04.082 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:12:04.082 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:12:04.082 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:12:04.082 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:12:04.082 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:12:04.082 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:12:04.082 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 01:12:04.082 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 01:12:04.082 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:12:04.082 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 01:12:04.082 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 01:12:04.082 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 01:12:04.082 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 01:12:04.082 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 01:12:04.082 | INFO | ------------------------------------------ | 20260506 01:12:04.082 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:12:04.082 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 01:12:04.082 | INFO | Total mem usage: 25.6 GiB | 20260506 01:12:04.082 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 01:12:04.082 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 01:12:04.082 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 01:12:04.082 | INFO | High Water Breakdown: | 20260506 01:12:04.082 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 01:12:04.082 | INFO | ------------------------------------------ | 20260506 01:12:04.082 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:12:04.082 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 01:12:04.082 | INFO | Total mem usage: 15.7 GiB | 20260506 01:12:04.082 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 01:12:04.082 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 01:12:04.082 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 01:12:04.082 | INFO | High Water Breakdown: | 20260506 01:12:04.082 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 01:12:04.082 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 01:12:04.082 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 01:12:04.082 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 01:12:04.082 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 01:12:04.082 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 01:12:04.082 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 01:12:04.082 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 01:12:04.082 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 01:12:04.082 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:12:04.082 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 01:12:04.082 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:12:04.082 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 01:12:04.082 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 01:12:04.082 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 01:12:04.082 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 01:12:04.082 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:12:04.082 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 01:12:04.082 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:12:04.082 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:12:04.082 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:12:04.082 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 01:12:04.082 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 01:12:04.082 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:12:04.082 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 01:12:04.082 | INFO | ------------------------------------------ | 20260506 01:12:04.082 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:12:04.083 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 01:12:04.083 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:12:04.083 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:12:04.083 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.41e+01, 1.36e+01 / 9.98e-01 | 20260506 01:12:04.083 | INFO | PerfReport: 0.29%, AggregateStats -- 9.24e-02 / 1.05e-01, 1.04e-02 / 4.84e-03 | 20260506 01:12:04.083 | INFO | PerfReport: 11.83%, Baseline -- 3.71e+00 / 4.48e+00, 5.79e-01 / 1.68e-01 | 20260506 01:12:04.083 | INFO | PerfReport: 3.30%, Histogram -- 1.04e+00 / 1.52e+00, 1.96e-01 / 9.68e-02 | 20260506 01:12:04.083 | INFO | PerfReport: 16.01%, DME -- 4.02e+01 / 4.44e+01, 4.12e+00 / 1.13e+00 | 20260506 01:12:04.083 | INFO | PerfReport: 57.44%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.60e+00 / 1.04e+00 | 20260506 01:12:04.083 | INFO | PerfReport: 7.03%, PulseAccumulating -- 2.21e+00 / 2.73e+00, 4.18e-01 / 2.47e-01 | 20260506 01:12:04.083 | INFO | PerfReport: 3.04%, Metrics -- 9.53e-01 / 1.21e+00, 2.55e-01 / 1.01e-01 | 20260506 01:12:04.083 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.30e-01 / 3.75e-01, 5.55e-02 / 2.52e-02 | 20260506 01:12:04.083 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 01:12:04.083 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:12:04.083 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:12:04.083 | INFO | PerfReport: Full Execution -- 9.80e+01 / -nan, 2.44e+01 / 0.00e+00 | 20260506 01:12:04.083 | INFO | PerfReport: 22.15%, Upload -- 2.17e+01 / 0.00e+00, 7.25e+00 / 0.00e+00 | 20260506 01:12:04.083 | INFO | PerfReport: 21.85%, Download -- 2.14e+01 / 0.00e+00, 7.56e+00 / 0.00e+00 | 20260506 01:12:04.083 | INFO | PerfReport: 32.03%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 01:12:04.083 | INFO | PerfReport: 2.02%, LockUpload -- 1.98e+00 / 0.00e+00, 7.78e+00 / 0.00e+00 | 20260506 01:12:04.083 | INFO | PerfReport: 1.31%, LockDownload -- 1.29e+00 / 0.00e+00, 2.96e+00 / 0.00e+00 | 20260506 01:12:04.083 | INFO | PerfReport: 20.62%, LockCompute -- 2.02e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 01:12:04.083 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.14e-02 / 0.00e+00 | 20260506 01:12:04.207 | INFO | Starting a round of Baz IO | 20260506 01:12:07.342 | INFO | Populating RTMetrics report | 20260506 01:12:07.342 | INFO | RTMetrics report populated | 20260506 01:12:07.342 | INFO | Opening tmp RTMetrics json file | 20260506 01:12:07.343 | INFO | Serializing RTMetrics to json | 20260506 01:12:07.344 | INFO | Serialization done, swapping tmp file now... | 20260506 01:12:07.344 | INFO | RTMetrics done | 20260506 01:12:07.418 | INFO | Finished a round of Baz IO: Wrote 1.834 GiB in 3.211 seconds | 20260506 01:12:07.419 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.33% / 0.00% / 78.67% | 20260506 01:12:07.419 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0828981% / 0% / 6.35783e-05% | 20260506 01:12:07.419 | INFO | Baz IO overall: | 20260506 01:12:07.419 | INFO | BytesWritten : 4393854910307 | 20260506 01:12:07.419 | INFO | NumEvents : 2065870212751 | 20260506 01:12:09.216 | INFO | Analyzing chunk frames = [7774208,7774720) | 20260506 01:12:09.216 | INFO | MemInfo: { | 20260506 01:12:09.216 | INFO | "gpu" : " 17 GB", | 20260506 01:12:09.216 | INFO | "huge" : "", | 20260506 01:12:09.216 | INFO | "huge_shared" : " 27 GB", | 20260506 01:12:09.216 | INFO | "malloc" : " 14 GB", | 20260506 01:12:09.216 | INFO | "pinned" : " 29 GB" | 20260506 01:12:09.216 | INFO | } | 20260506 01:12:09.216 | INFO | Perf report for the last 5.13435 seconds: | 20260506 01:12:09.216 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:12:09.216 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:09.216 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:09.216 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:09.216 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260506 01:12:09.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:09.216 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260506 01:12:09.216 | INFO | PRE_HQ : 59.4%, 1.7, 53.7ms, 0 | 20260506 01:12:09.216 | INFO | BAZWRITER : 61.8%, 3.5, 111.0ms, 0 | 20260506 01:12:09.216 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:12:09.216 | INFO | Uploaded 36GiB to GPU in 2.14797s (16.76GiB/s) | 20260506 01:12:09.216 | INFO | Downloaded 38GiB to GPU in 2.23254s (17.021GiB/s) | 20260506 01:12:14.364 | INFO | Analyzing chunk frames = [7774720,7775232) | 20260506 01:12:14.364 | INFO | MemInfo: { | 20260506 01:12:14.364 | INFO | "gpu" : " 17 GB", | 20260506 01:12:14.364 | INFO | "huge" : "", | 20260506 01:12:14.364 | INFO | "huge_shared" : " 27 GB", | 20260506 01:12:14.364 | INFO | "malloc" : " 14 GB", | 20260506 01:12:14.364 | INFO | "pinned" : " 29 GB" | 20260506 01:12:14.364 | INFO | } | 20260506 01:12:14.364 | INFO | Perf report for the last 5.14828 seconds: | 20260506 01:12:14.364 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:12:14.364 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:14.364 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:14.364 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:14.364 | INFO | ANALYSIS : 61.5%, 3.0, 96.2ms, 0 | 20260506 01:12:14.364 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:14.364 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:14.364 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 01:12:14.364 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:14.364 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:14.364 | INFO | Uploaded 36GiB to GPU in 2.11703s (17.0049GiB/s) | 20260506 01:12:14.364 | INFO | Downloaded 34GiB to GPU in 2.04085s (16.6598GiB/s) | 20260506 01:12:19.498 | INFO | Analyzing chunk frames = [7775232,7775744) | 20260506 01:12:19.498 | INFO | MemInfo: { | 20260506 01:12:19.498 | INFO | "gpu" : " 17 GB", | 20260506 01:12:19.498 | INFO | "huge" : "", | 20260506 01:12:19.498 | INFO | "huge_shared" : " 27 GB", | 20260506 01:12:19.498 | INFO | "malloc" : " 14 GB", | 20260506 01:12:19.498 | INFO | "pinned" : " 29 GB" | 20260506 01:12:19.498 | INFO | } | 20260506 01:12:19.498 | INFO | Perf report for the last 5.13398 seconds: | 20260506 01:12:19.498 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:12:19.498 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:19.498 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:19.498 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:19.498 | INFO | ANALYSIS : 61.4%, 3.0, 95.4ms, 0 | 20260506 01:12:19.498 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:19.498 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:19.498 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:12:19.498 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:19.498 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:19.498 | INFO | Uploaded 36GiB to GPU in 2.10043s (17.1393GiB/s) | 20260506 01:12:19.498 | INFO | Downloaded 34GiB to GPU in 2.02429s (16.796GiB/s) | 20260506 01:12:24.632 | INFO | Analyzing chunk frames = [7775744,7776256) | 20260506 01:12:24.632 | INFO | MemInfo: { | 20260506 01:12:24.632 | INFO | "gpu" : " 17 GB", | 20260506 01:12:24.632 | INFO | "huge" : "", | 20260506 01:12:24.632 | INFO | "huge_shared" : " 27 GB", | 20260506 01:12:24.632 | INFO | "malloc" : " 14 GB", | 20260506 01:12:24.632 | INFO | "pinned" : " 29 GB" | 20260506 01:12:24.632 | INFO | } | 20260506 01:12:24.632 | INFO | Perf report for the last 5.13405 seconds: | 20260506 01:12:24.632 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:12:24.632 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:24.632 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:24.632 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:24.632 | INFO | ANALYSIS : 61.7%, 3.0, 95.7ms, 0 | 20260506 01:12:24.632 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:24.632 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:24.632 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 01:12:24.632 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:24.632 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:24.632 | INFO | Uploaded 36GiB to GPU in 2.14017s (16.8211GiB/s) | 20260506 01:12:24.632 | INFO | Downloaded 34GiB to GPU in 2.01196s (16.8989GiB/s) | 20260506 01:12:29.777 | INFO | Analyzing chunk frames = [7776256,7776768) | 20260506 01:12:29.777 | INFO | MemInfo: { | 20260506 01:12:29.777 | INFO | "gpu" : " 17 GB", | 20260506 01:12:29.777 | INFO | "huge" : "", | 20260506 01:12:29.777 | INFO | "huge_shared" : " 27 GB", | 20260506 01:12:29.777 | INFO | "malloc" : " 14 GB", | 20260506 01:12:29.777 | INFO | "pinned" : " 29 GB" | 20260506 01:12:29.777 | INFO | } | 20260506 01:12:29.777 | INFO | Perf report for the last 5.14507 seconds: | 20260506 01:12:29.777 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:12:29.777 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:29.777 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:29.777 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:29.777 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 01:12:29.777 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:29.777 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:29.777 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 01:12:29.777 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:29.777 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:29.777 | INFO | Uploaded 36GiB to GPU in 2.10879s (17.0714GiB/s) | 20260506 01:12:29.778 | INFO | Downloaded 34GiB to GPU in 2.05214s (16.5681GiB/s) | 20260506 01:12:34.918 | INFO | Analyzing chunk frames = [7776768,7777280) | 20260506 01:12:34.918 | INFO | MemInfo: { | 20260506 01:12:34.918 | INFO | "gpu" : " 17 GB", | 20260506 01:12:34.918 | INFO | "huge" : "", | 20260506 01:12:34.918 | INFO | "huge_shared" : " 27 GB", | 20260506 01:12:34.918 | INFO | "malloc" : " 14 GB", | 20260506 01:12:34.918 | INFO | "pinned" : " 29 GB" | 20260506 01:12:34.918 | INFO | } | 20260506 01:12:34.918 | INFO | Perf report for the last 5.14115 seconds: | 20260506 01:12:34.918 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:12:34.918 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:34.918 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:34.918 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:34.918 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 01:12:34.918 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:34.918 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:34.918 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:12:34.918 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:34.918 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:34.918 | INFO | Uploaded 36GiB to GPU in 2.07237s (17.3714GiB/s) | 20260506 01:12:34.918 | INFO | Downloaded 34GiB to GPU in 2.03468s (16.7102GiB/s) | 20260506 01:12:40.052 | INFO | Analyzing chunk frames = [7777280,7777792) | 20260506 01:12:40.052 | INFO | MemInfo: { | 20260506 01:12:40.052 | INFO | "gpu" : " 17 GB", | 20260506 01:12:40.052 | INFO | "huge" : "", | 20260506 01:12:40.052 | INFO | "huge_shared" : " 27 GB", | 20260506 01:12:40.052 | INFO | "malloc" : " 14 GB", | 20260506 01:12:40.052 | INFO | "pinned" : " 29 GB" | 20260506 01:12:40.052 | INFO | } | 20260506 01:12:40.052 | INFO | Perf report for the last 5.13343 seconds: | 20260506 01:12:40.052 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:12:40.052 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:40.052 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:40.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:40.052 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 01:12:40.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:40.052 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:40.052 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 01:12:40.052 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:40.052 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:40.052 | INFO | Uploaded 36GiB to GPU in 2.03807s (17.6638GiB/s) | 20260506 01:12:40.052 | INFO | Downloaded 34GiB to GPU in 2.06681s (16.4504GiB/s) | 20260506 01:12:45.184 | INFO | Analyzing chunk frames = [7777792,7778304) | 20260506 01:12:45.184 | INFO | MemInfo: { | 20260506 01:12:45.184 | INFO | "gpu" : " 17 GB", | 20260506 01:12:45.184 | INFO | "huge" : "", | 20260506 01:12:45.184 | INFO | "huge_shared" : " 27 GB", | 20260506 01:12:45.184 | INFO | "malloc" : " 14 GB", | 20260506 01:12:45.184 | INFO | "pinned" : " 29 GB" | 20260506 01:12:45.184 | INFO | } | 20260506 01:12:45.184 | INFO | Perf report for the last 5.13221 seconds: | 20260506 01:12:45.184 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:12:45.184 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:45.184 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:45.184 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:45.184 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 01:12:45.184 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:45.184 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:45.184 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260506 01:12:45.184 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:45.184 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:45.184 | INFO | Uploaded 36GiB to GPU in 2.11679s (17.0069GiB/s) | 20260506 01:12:45.184 | INFO | Downloaded 34GiB to GPU in 2.01032s (16.9128GiB/s) | 20260506 01:12:45.308 | INFO | Starting a round of Baz IO | 20260506 01:12:48.453 | INFO | Populating RTMetrics report | 20260506 01:12:48.453 | INFO | RTMetrics report populated | 20260506 01:12:48.453 | INFO | Opening tmp RTMetrics json file | 20260506 01:12:48.454 | INFO | Serializing RTMetrics to json | 20260506 01:12:48.455 | INFO | Serialization done, swapping tmp file now... | 20260506 01:12:48.456 | INFO | RTMetrics done | 20260506 01:12:48.529 | INFO | Finished a round of Baz IO: Wrote 1.831 GiB in 3.22 seconds | 20260506 01:12:48.529 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.33% / 0.00% / 78.67% | 20260506 01:12:48.529 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0825326% / 0% / 9.13938e-05% | 20260506 01:12:48.529 | INFO | Baz IO overall: | 20260506 01:12:48.529 | INFO | BytesWritten : 4395820790743 | 20260506 01:12:48.529 | INFO | NumEvents : 2066766807858 | 20260506 01:12:50.329 | INFO | Analyzing chunk frames = [7778304,7778816) | 20260506 01:12:50.329 | INFO | MemInfo: { | 20260506 01:12:50.329 | INFO | "gpu" : " 17 GB", | 20260506 01:12:50.329 | INFO | "huge" : "", | 20260506 01:12:50.329 | INFO | "huge_shared" : " 27 GB", | 20260506 01:12:50.329 | INFO | "malloc" : " 14 GB", | 20260506 01:12:50.329 | INFO | "pinned" : " 29 GB" | 20260506 01:12:50.329 | INFO | } | 20260506 01:12:50.330 | INFO | Perf report for the last 5.14557 seconds: | 20260506 01:12:50.330 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:12:50.330 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:50.330 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:50.330 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:50.330 | INFO | ANALYSIS : 62.5%, 3.0, 97.7ms, 0 | 20260506 01:12:50.330 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:50.330 | INFO | RT_METRICS : 43.0%, 1.0, 22.6ms, 0 | 20260506 01:12:50.330 | INFO | PRE_HQ : 59.6%, 1.7, 53.8ms, 0 | 20260506 01:12:50.330 | INFO | BAZWRITER : 61.8%, 3.5, 110.5ms, 0 | 20260506 01:12:50.330 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:12:50.330 | INFO | Uploaded 36GiB to GPU in 2.17036s (16.5871GiB/s) | 20260506 01:12:50.330 | INFO | Downloaded 38GiB to GPU in 2.23657s (16.9903GiB/s) | 20260506 01:12:55.477 | INFO | Analyzing chunk frames = [7778816,7779328) | 20260506 01:12:55.477 | INFO | MemInfo: { | 20260506 01:12:55.477 | INFO | "gpu" : " 17 GB", | 20260506 01:12:55.477 | INFO | "huge" : "", | 20260506 01:12:55.477 | INFO | "huge_shared" : " 27 GB", | 20260506 01:12:55.477 | INFO | "malloc" : " 14 GB", | 20260506 01:12:55.477 | INFO | "pinned" : " 29 GB" | 20260506 01:12:55.477 | INFO | } | 20260506 01:12:55.477 | INFO | Perf report for the last 5.1477 seconds: | 20260506 01:12:55.477 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:12:55.477 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:55.477 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:55.477 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:55.477 | INFO | ANALYSIS : 61.3%, 3.0, 95.9ms, 0 | 20260506 01:12:55.477 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:55.477 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:55.477 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:12:55.477 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:55.477 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:12:55.477 | INFO | Uploaded 36GiB to GPU in 2.02727s (17.7578GiB/s) | 20260506 01:12:55.477 | INFO | Downloaded 34GiB to GPU in 2.00951s (16.9195GiB/s) | 20260506 01:13:00.614 | INFO | Analyzing chunk frames = [7779328,7779840) | 20260506 01:13:00.614 | INFO | MemInfo: { | 20260506 01:13:00.614 | INFO | "gpu" : " 17 GB", | 20260506 01:13:00.614 | INFO | "huge" : "", | 20260506 01:13:00.614 | INFO | "huge_shared" : " 27 GB", | 20260506 01:13:00.614 | INFO | "malloc" : " 14 GB", | 20260506 01:13:00.614 | INFO | "pinned" : " 29 GB" | 20260506 01:13:00.614 | INFO | } | 20260506 01:13:00.614 | INFO | Perf report for the last 5.13652 seconds: | 20260506 01:13:00.614 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:13:00.614 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:00.614 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:00.614 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:00.614 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 01:13:00.614 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:00.614 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:00.614 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:13:00.614 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:00.614 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:00.614 | INFO | Uploaded 36GiB to GPU in 2.09111s (17.2158GiB/s) | 20260506 01:13:00.614 | INFO | Downloaded 34GiB to GPU in 2.02569s (16.7844GiB/s) | 20260506 01:13:05.753 | INFO | Analyzing chunk frames = [7779840,7780352) | 20260506 01:13:05.753 | INFO | MemInfo: { | 20260506 01:13:05.753 | INFO | "gpu" : " 17 GB", | 20260506 01:13:05.753 | INFO | "huge" : "", | 20260506 01:13:05.753 | INFO | "huge_shared" : " 27 GB", | 20260506 01:13:05.753 | INFO | "malloc" : " 14 GB", | 20260506 01:13:05.753 | INFO | "pinned" : " 29 GB" | 20260506 01:13:05.753 | INFO | } | 20260506 01:13:05.753 | INFO | Perf report for the last 5.13941 seconds: | 20260506 01:13:05.753 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:13:05.753 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:05.753 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:05.753 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:05.753 | INFO | ANALYSIS : 61.5%, 3.0, 95.5ms, 0 | 20260506 01:13:05.753 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:05.753 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:05.753 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 01:13:05.753 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:05.753 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:05.753 | INFO | Uploaded 36GiB to GPU in 2.1297s (16.9038GiB/s) | 20260506 01:13:05.754 | INFO | Downloaded 34GiB to GPU in 2.01788s (16.8493GiB/s) | 20260506 01:13:10.895 | INFO | Analyzing chunk frames = [7780352,7780864) | 20260506 01:13:10.895 | INFO | MemInfo: { | 20260506 01:13:10.895 | INFO | "gpu" : " 17 GB", | 20260506 01:13:10.895 | INFO | "huge" : "", | 20260506 01:13:10.895 | INFO | "huge_shared" : " 27 GB", | 20260506 01:13:10.895 | INFO | "malloc" : " 14 GB", | 20260506 01:13:10.895 | INFO | "pinned" : " 29 GB" | 20260506 01:13:10.895 | INFO | } | 20260506 01:13:10.895 | INFO | Perf report for the last 5.14162 seconds: | 20260506 01:13:10.895 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:13:10.895 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:10.895 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:10.895 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:10.895 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 01:13:10.895 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:10.895 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:10.895 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:13:10.895 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:10.895 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:10.895 | INFO | Uploaded 36GiB to GPU in 2.08325s (17.2807GiB/s) | 20260506 01:13:10.895 | INFO | Downloaded 34GiB to GPU in 2.04487s (16.6269GiB/s) | 20260506 01:13:16.033 | INFO | Analyzing chunk frames = [7780864,7781376) | 20260506 01:13:16.033 | INFO | MemInfo: { | 20260506 01:13:16.033 | INFO | "gpu" : " 17 GB", | 20260506 01:13:16.033 | INFO | "huge" : "", | 20260506 01:13:16.033 | INFO | "huge_shared" : " 27 GB", | 20260506 01:13:16.033 | INFO | "malloc" : " 14 GB", | 20260506 01:13:16.033 | INFO | "pinned" : " 29 GB" | 20260506 01:13:16.033 | INFO | } | 20260506 01:13:16.033 | INFO | Perf report for the last 5.13791 seconds: | 20260506 01:13:16.033 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:13:16.033 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:16.033 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:16.033 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:16.033 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 01:13:16.033 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:16.033 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:16.033 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 01:13:16.033 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:16.033 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:16.033 | INFO | Uploaded 36GiB to GPU in 2.06s (17.4757GiB/s) | 20260506 01:13:16.033 | INFO | Downloaded 34GiB to GPU in 2.02751s (16.7693GiB/s) | 20260506 01:13:21.168 | INFO | Analyzing chunk frames = [7781376,7781888) | 20260506 01:13:21.168 | INFO | MemInfo: { | 20260506 01:13:21.168 | INFO | "gpu" : " 17 GB", | 20260506 01:13:21.168 | INFO | "huge" : "", | 20260506 01:13:21.168 | INFO | "huge_shared" : " 27 GB", | 20260506 01:13:21.168 | INFO | "malloc" : " 14 GB", | 20260506 01:13:21.168 | INFO | "pinned" : " 29 GB" | 20260506 01:13:21.168 | INFO | } | 20260506 01:13:21.168 | INFO | Perf report for the last 5.13512 seconds: | 20260506 01:13:21.168 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:13:21.168 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:21.168 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:21.168 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:21.168 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260506 01:13:21.168 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:21.168 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:21.168 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 01:13:21.168 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:21.168 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:21.168 | INFO | Uploaded 36GiB to GPU in 2.07185s (17.3758GiB/s) | 20260506 01:13:21.168 | INFO | Downloaded 34GiB to GPU in 2.08248s (16.3267GiB/s) | 20260506 01:13:26.309 | INFO | Analyzing chunk frames = [7781888,7782400) | 20260506 01:13:26.309 | INFO | MemInfo: { | 20260506 01:13:26.309 | INFO | "gpu" : " 17 GB", | 20260506 01:13:26.309 | INFO | "huge" : "", | 20260506 01:13:26.309 | INFO | "huge_shared" : " 27 GB", | 20260506 01:13:26.309 | INFO | "malloc" : " 14 GB", | 20260506 01:13:26.309 | INFO | "pinned" : " 29 GB" | 20260506 01:13:26.309 | INFO | } | 20260506 01:13:26.309 | INFO | Perf report for the last 5.1416 seconds: | 20260506 01:13:26.309 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:13:26.309 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:26.309 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:26.310 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:26.310 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260506 01:13:26.310 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:26.310 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:26.310 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 01:13:26.310 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:26.310 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:26.310 | INFO | Uploaded 36GiB to GPU in 2.1313s (16.8911GiB/s) | 20260506 01:13:26.310 | INFO | Downloaded 34GiB to GPU in 2.01643s (16.8615GiB/s) | 20260506 01:13:26.310 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 01:13:26.310 | INFO | Total mem usage: 15.3 GiB | 20260506 01:13:26.310 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 01:13:26.310 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.16%) | 20260506 01:13:26.310 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 01:13:26.310 | INFO | High Water Breakdown: | 20260506 01:13:26.310 | INFO | 60.9% : 8.55 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 01:13:26.310 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 01:13:26.310 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 01:13:26.310 | INFO | ------------------------------------------ | 20260506 01:13:26.310 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:13:26.310 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 01:13:26.310 | INFO | Total mem usage: 26.5 GiB | 20260506 01:13:26.310 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 01:13:26.310 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 01:13:26.310 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 01:13:26.310 | INFO | High Water Breakdown: | 20260506 01:13:26.310 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:13:26.310 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:13:26.310 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:13:26.310 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:13:26.310 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:13:26.310 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:13:26.310 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 01:13:26.310 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 01:13:26.310 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:13:26.310 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 01:13:26.310 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 01:13:26.310 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 01:13:26.310 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 01:13:26.310 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 01:13:26.310 | INFO | ------------------------------------------ | 20260506 01:13:26.310 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:13:26.310 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 01:13:26.310 | INFO | Total mem usage: 25.6 GiB | 20260506 01:13:26.310 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 01:13:26.310 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 01:13:26.310 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 01:13:26.310 | INFO | High Water Breakdown: | 20260506 01:13:26.310 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 01:13:26.310 | INFO | ------------------------------------------ | 20260506 01:13:26.310 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:13:26.310 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 01:13:26.310 | INFO | Total mem usage: 15.7 GiB | 20260506 01:13:26.310 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 01:13:26.310 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 01:13:26.310 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 01:13:26.310 | INFO | High Water Breakdown: | 20260506 01:13:26.310 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 01:13:26.310 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 01:13:26.310 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 01:13:26.310 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 01:13:26.310 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 01:13:26.310 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 01:13:26.310 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 01:13:26.310 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 01:13:26.310 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 01:13:26.310 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:13:26.310 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 01:13:26.310 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:13:26.310 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 01:13:26.310 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 01:13:26.310 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 01:13:26.310 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 01:13:26.310 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:13:26.310 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 01:13:26.310 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:13:26.310 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:13:26.310 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:13:26.310 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 01:13:26.310 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 01:13:26.310 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:13:26.310 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 01:13:26.310 | INFO | ------------------------------------------ | 20260506 01:13:26.311 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:13:26.311 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 01:13:26.311 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:13:26.311 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:13:26.311 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.39e+01, 1.35e+01 / 8.87e-01 | 20260506 01:13:26.311 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.04e-01, 1.09e-02 / 3.64e-03 | 20260506 01:13:26.311 | INFO | PerfReport: 11.85%, Baseline -- 3.71e+00 / 4.45e+00, 5.70e-01 / 2.58e-01 | 20260506 01:13:26.311 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.50e+00, 1.89e-01 / 1.04e-01 | 20260506 01:13:26.311 | INFO | PerfReport: 16.02%, DME -- 4.01e+01 / 4.41e+01, 3.87e+00 / 1.12e+00 | 20260506 01:13:26.311 | INFO | PerfReport: 57.36%, FrameLabeling -- 1.79e+01 / 1.94e+01, 1.48e+00 / 1.01e+00 | 20260506 01:13:26.311 | INFO | PerfReport: 7.07%, PulseAccumulating -- 2.21e+00 / 2.74e+00, 4.19e-01 / 2.24e-01 | 20260506 01:13:26.311 | INFO | PerfReport: 3.05%, Metrics -- 9.55e-01 / 1.23e+00, 2.58e-01 / 1.18e-01 | 20260506 01:13:26.311 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.30e-01 / 3.81e-01, 5.62e-02 / 2.88e-02 | 20260506 01:13:26.311 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 01:13:26.311 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:13:26.311 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:13:26.311 | INFO | PerfReport: Full Execution -- 9.81e+01 / -nan, 2.44e+01 / 0.00e+00 | 20260506 01:13:26.311 | INFO | PerfReport: 22.29%, Upload -- 2.19e+01 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260506 01:13:26.311 | INFO | PerfReport: 21.85%, Download -- 2.14e+01 / 0.00e+00, 7.55e+00 / 0.00e+00 | 20260506 01:13:26.311 | INFO | PerfReport: 31.87%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 01:13:26.311 | INFO | PerfReport: 2.10%, LockUpload -- 2.06e+00 / 0.00e+00, 7.81e+00 / 0.00e+00 | 20260506 01:13:26.311 | INFO | PerfReport: 1.35%, LockDownload -- 1.33e+00 / 0.00e+00, 2.98e+00 / 0.00e+00 | 20260506 01:13:26.311 | INFO | PerfReport: 20.51%, LockCompute -- 2.01e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260506 01:13:26.311 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.96e-02 / 0.00e+00 | 20260506 01:13:26.428 | INFO | Starting a round of Baz IO | 20260506 01:13:29.570 | INFO | Populating RTMetrics report | 20260506 01:13:29.571 | INFO | RTMetrics report populated | 20260506 01:13:29.571 | INFO | Opening tmp RTMetrics json file | 20260506 01:13:29.571 | INFO | Serializing RTMetrics to json | 20260506 01:13:29.573 | INFO | Serialization done, swapping tmp file now... | 20260506 01:13:29.573 | INFO | RTMetrics done | 20260506 01:13:29.645 | INFO | Finished a round of Baz IO: Wrote 1.831 GiB in 3.217 seconds | 20260506 01:13:29.645 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.33% / 0.00% / 78.67% | 20260506 01:13:29.645 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0827909% / 0% / 8.34465e-05% | 20260506 01:13:29.645 | INFO | Baz IO overall: | 20260506 01:13:29.645 | INFO | BytesWritten : 4397787131534 | 20260506 01:13:29.645 | INFO | NumEvents : 2067663754784 | 20260506 01:13:31.444 | INFO | Analyzing chunk frames = [7782400,7782912) | 20260506 01:13:31.444 | INFO | MemInfo: { | 20260506 01:13:31.444 | INFO | "gpu" : " 17 GB", | 20260506 01:13:31.444 | INFO | "huge" : "", | 20260506 01:13:31.444 | INFO | "huge_shared" : " 27 GB", | 20260506 01:13:31.444 | INFO | "malloc" : " 14 GB", | 20260506 01:13:31.444 | INFO | "pinned" : " 29 GB" | 20260506 01:13:31.444 | INFO | } | 20260506 01:13:31.444 | INFO | Perf report for the last 5.13419 seconds: | 20260506 01:13:31.444 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:13:31.444 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:31.444 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:31.444 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:31.444 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260506 01:13:31.444 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:31.444 | INFO | RT_METRICS : 43.2%, 1.0, 22.6ms, 0 | 20260506 01:13:31.444 | INFO | PRE_HQ : 59.6%, 1.7, 53.8ms, 0 | 20260506 01:13:31.444 | INFO | BAZWRITER : 61.7%, 3.5, 110.2ms, 0 | 20260506 01:13:31.444 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:13:31.444 | INFO | Uploaded 36GiB to GPU in 2.18338s (16.4882GiB/s) | 20260506 01:13:31.444 | INFO | Downloaded 38GiB to GPU in 2.24747s (16.9079GiB/s) | 20260506 01:13:36.590 | INFO | Analyzing chunk frames = [7782912,7783424) | 20260506 01:13:36.590 | INFO | MemInfo: { | 20260506 01:13:36.590 | INFO | "gpu" : " 17 GB", | 20260506 01:13:36.590 | INFO | "huge" : "", | 20260506 01:13:36.590 | INFO | "huge_shared" : " 27 GB", | 20260506 01:13:36.590 | INFO | "malloc" : " 14 GB", | 20260506 01:13:36.590 | INFO | "pinned" : " 29 GB" | 20260506 01:13:36.590 | INFO | } | 20260506 01:13:36.591 | INFO | Perf report for the last 5.14684 seconds: | 20260506 01:13:36.591 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:13:36.591 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:36.591 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:36.591 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:36.591 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260506 01:13:36.591 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:36.591 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:36.591 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 01:13:36.591 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:36.591 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:36.591 | INFO | Uploaded 36GiB to GPU in 2.10336s (17.1154GiB/s) | 20260506 01:13:36.591 | INFO | Downloaded 34GiB to GPU in 2.04276s (16.6442GiB/s) | 20260506 01:13:41.732 | INFO | Analyzing chunk frames = [7783424,7783936) | 20260506 01:13:41.732 | INFO | MemInfo: { | 20260506 01:13:41.732 | INFO | "gpu" : " 17 GB", | 20260506 01:13:41.732 | INFO | "huge" : "", | 20260506 01:13:41.732 | INFO | "huge_shared" : " 27 GB", | 20260506 01:13:41.732 | INFO | "malloc" : " 14 GB", | 20260506 01:13:41.732 | INFO | "pinned" : " 29 GB" | 20260506 01:13:41.732 | INFO | } | 20260506 01:13:41.732 | INFO | Perf report for the last 5.1415 seconds: | 20260506 01:13:41.732 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:13:41.732 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:41.732 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:41.732 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:41.732 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 01:13:41.732 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:41.732 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:41.732 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 01:13:41.732 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:41.732 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:41.732 | INFO | Uploaded 36GiB to GPU in 2.08368s (17.2771GiB/s) | 20260506 01:13:41.732 | INFO | Downloaded 34GiB to GPU in 2.03059s (16.7439GiB/s) | 20260506 01:13:46.865 | INFO | Analyzing chunk frames = [7783936,7784448) | 20260506 01:13:46.865 | INFO | MemInfo: { | 20260506 01:13:46.865 | INFO | "gpu" : " 17 GB", | 20260506 01:13:46.865 | INFO | "huge" : "", | 20260506 01:13:46.865 | INFO | "huge_shared" : " 27 GB", | 20260506 01:13:46.865 | INFO | "malloc" : " 14 GB", | 20260506 01:13:46.865 | INFO | "pinned" : " 29 GB" | 20260506 01:13:46.865 | INFO | } | 20260506 01:13:46.866 | INFO | Perf report for the last 5.13346 seconds: | 20260506 01:13:46.866 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:13:46.866 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:46.866 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:46.866 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:46.866 | INFO | ANALYSIS : 61.3%, 3.0, 95.2ms, 0 | 20260506 01:13:46.866 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:46.866 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:46.866 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 01:13:46.866 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:46.866 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:46.866 | INFO | Uploaded 36GiB to GPU in 2.09116s (17.2153GiB/s) | 20260506 01:13:46.866 | INFO | Downloaded 34GiB to GPU in 2.02679s (16.7753GiB/s) | 20260506 01:13:52.002 | INFO | Analyzing chunk frames = [7784448,7784960) | 20260506 01:13:52.002 | INFO | MemInfo: { | 20260506 01:13:52.002 | INFO | "gpu" : " 17 GB", | 20260506 01:13:52.002 | INFO | "huge" : "", | 20260506 01:13:52.002 | INFO | "huge_shared" : " 27 GB", | 20260506 01:13:52.002 | INFO | "malloc" : " 14 GB", | 20260506 01:13:52.002 | INFO | "pinned" : " 29 GB" | 20260506 01:13:52.002 | INFO | } | 20260506 01:13:52.002 | INFO | Perf report for the last 5.13668 seconds: | 20260506 01:13:52.002 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:13:52.002 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:52.002 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:52.002 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:52.002 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 01:13:52.002 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:52.002 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:52.002 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 01:13:52.002 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:52.002 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:52.002 | INFO | Uploaded 36GiB to GPU in 2.08984s (17.2262GiB/s) | 20260506 01:13:52.002 | INFO | Downloaded 34GiB to GPU in 2.04255s (16.6458GiB/s) | 20260506 01:13:57.140 | INFO | Analyzing chunk frames = [7784960,7785472) | 20260506 01:13:57.140 | INFO | MemInfo: { | 20260506 01:13:57.140 | INFO | "gpu" : " 17 GB", | 20260506 01:13:57.140 | INFO | "huge" : "", | 20260506 01:13:57.140 | INFO | "huge_shared" : " 27 GB", | 20260506 01:13:57.140 | INFO | "malloc" : " 14 GB", | 20260506 01:13:57.140 | INFO | "pinned" : " 29 GB" | 20260506 01:13:57.140 | INFO | } | 20260506 01:13:57.140 | INFO | Perf report for the last 5.1378 seconds: | 20260506 01:13:57.140 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:13:57.140 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:57.140 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:57.140 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:57.140 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260506 01:13:57.140 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:57.140 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:57.140 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 01:13:57.140 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:57.140 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:13:57.140 | INFO | Uploaded 36GiB to GPU in 2.08304s (17.2824GiB/s) | 20260506 01:13:57.140 | INFO | Downloaded 34GiB to GPU in 2.04689s (16.6106GiB/s) | 20260506 01:14:02.284 | INFO | Analyzing chunk frames = [7785472,7785984) | 20260506 01:14:02.284 | INFO | MemInfo: { | 20260506 01:14:02.284 | INFO | "gpu" : " 17 GB", | 20260506 01:14:02.284 | INFO | "huge" : "", | 20260506 01:14:02.284 | INFO | "huge_shared" : " 27 GB", | 20260506 01:14:02.284 | INFO | "malloc" : " 14 GB", | 20260506 01:14:02.284 | INFO | "pinned" : " 29 GB" | 20260506 01:14:02.284 | INFO | } | 20260506 01:14:02.284 | INFO | Perf report for the last 5.14435 seconds: | 20260506 01:14:02.284 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:14:02.284 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:02.284 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:02.284 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:02.285 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260506 01:14:02.285 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:02.285 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:02.285 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 01:14:02.285 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:02.285 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:02.285 | INFO | Uploaded 36GiB to GPU in 2.06598s (17.4251GiB/s) | 20260506 01:14:02.285 | INFO | Downloaded 34GiB to GPU in 2.07556s (16.3811GiB/s) | 20260506 01:14:07.432 | INFO | Analyzing chunk frames = [7785984,7786496) | 20260506 01:14:07.432 | INFO | MemInfo: { | 20260506 01:14:07.432 | INFO | "gpu" : " 17 GB", | 20260506 01:14:07.432 | INFO | "huge" : "", | 20260506 01:14:07.432 | INFO | "huge_shared" : " 27 GB", | 20260506 01:14:07.432 | INFO | "malloc" : " 14 GB", | 20260506 01:14:07.432 | INFO | "pinned" : " 29 GB" | 20260506 01:14:07.432 | INFO | } | 20260506 01:14:07.432 | INFO | Perf report for the last 5.14767 seconds: | 20260506 01:14:07.432 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:14:07.432 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:07.432 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:07.432 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:07.432 | INFO | ANALYSIS : 61.8%, 3.0, 96.6ms, 0 | 20260506 01:14:07.432 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:07.432 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:07.432 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 01:14:07.432 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:07.432 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:07.432 | INFO | Uploaded 36GiB to GPU in 2.1317s (16.8879GiB/s) | 20260506 01:14:07.432 | INFO | Downloaded 34GiB to GPU in 2.02657s (16.7771GiB/s) | 20260506 01:14:07.559 | INFO | Starting a round of Baz IO | 20260506 01:14:10.694 | INFO | Populating RTMetrics report | 20260506 01:14:10.694 | INFO | RTMetrics report populated | 20260506 01:14:10.694 | INFO | Opening tmp RTMetrics json file | 20260506 01:14:10.695 | INFO | Serializing RTMetrics to json | 20260506 01:14:10.696 | INFO | Serialization done, swapping tmp file now... | 20260506 01:14:10.697 | INFO | RTMetrics done | 20260506 01:14:10.777 | INFO | Finished a round of Baz IO: Wrote 1.83 GiB in 3.217 seconds | 20260506 01:14:10.777 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.33% / 0.00% / 78.67% | 20260506 01:14:10.777 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0820835% / 0% / 7.94729e-05% | 20260506 01:14:10.777 | INFO | Baz IO overall: | 20260506 01:14:10.777 | INFO | BytesWritten : 4399752003766 | 20260506 01:14:10.777 | INFO | NumEvents : 2068559684943 | 20260506 01:14:12.565 | INFO | Analyzing chunk frames = [7786496,7787008) | 20260506 01:14:12.565 | INFO | MemInfo: { | 20260506 01:14:12.565 | INFO | "gpu" : " 17 GB", | 20260506 01:14:12.565 | INFO | "huge" : "", | 20260506 01:14:12.565 | INFO | "huge_shared" : " 27 GB", | 20260506 01:14:12.565 | INFO | "malloc" : " 14 GB", | 20260506 01:14:12.565 | INFO | "pinned" : " 29 GB" | 20260506 01:14:12.565 | INFO | } | 20260506 01:14:12.565 | INFO | Perf report for the last 5.13298 seconds: | 20260506 01:14:12.565 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:14:12.565 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:12.565 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:12.565 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:12.565 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260506 01:14:12.565 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:12.565 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260506 01:14:12.565 | INFO | PRE_HQ : 59.3%, 1.8, 53.9ms, 0 | 20260506 01:14:12.565 | INFO | BAZWRITER : 61.8%, 3.5, 110.1ms, 0 | 20260506 01:14:12.565 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:14:12.565 | INFO | Uploaded 36GiB to GPU in 2.11025s (17.0596GiB/s) | 20260506 01:14:12.565 | INFO | Downloaded 38GiB to GPU in 2.22021s (17.1155GiB/s) | 20260506 01:14:17.698 | INFO | Analyzing chunk frames = [7787008,7787520) | 20260506 01:14:17.698 | INFO | MemInfo: { | 20260506 01:14:17.698 | INFO | "gpu" : " 17 GB", | 20260506 01:14:17.698 | INFO | "huge" : "", | 20260506 01:14:17.698 | INFO | "huge_shared" : " 27 GB", | 20260506 01:14:17.698 | INFO | "malloc" : " 14 GB", | 20260506 01:14:17.698 | INFO | "pinned" : " 29 GB" | 20260506 01:14:17.698 | INFO | } | 20260506 01:14:17.698 | INFO | Perf report for the last 5.13265 seconds: | 20260506 01:14:17.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:14:17.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:17.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:17.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:17.698 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 01:14:17.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:17.698 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:17.698 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 01:14:17.698 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:17.698 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:17.698 | INFO | Uploaded 36GiB to GPU in 2.12215s (16.9639GiB/s) | 20260506 01:14:17.698 | INFO | Downloaded 34GiB to GPU in 2.04763s (16.6046GiB/s) | 20260506 01:14:22.843 | INFO | Analyzing chunk frames = [7787520,7788032) | 20260506 01:14:22.843 | INFO | MemInfo: { | 20260506 01:14:22.843 | INFO | "gpu" : " 17 GB", | 20260506 01:14:22.843 | INFO | "huge" : "", | 20260506 01:14:22.843 | INFO | "huge_shared" : " 27 GB", | 20260506 01:14:22.843 | INFO | "malloc" : " 14 GB", | 20260506 01:14:22.843 | INFO | "pinned" : " 29 GB" | 20260506 01:14:22.843 | INFO | } | 20260506 01:14:22.843 | INFO | Perf report for the last 5.14574 seconds: | 20260506 01:14:22.844 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:14:22.844 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:22.844 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:22.844 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:22.844 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 01:14:22.844 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:22.844 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:22.844 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 01:14:22.844 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:22.844 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:22.844 | INFO | Uploaded 36GiB to GPU in 2.14672s (16.7698GiB/s) | 20260506 01:14:22.844 | INFO | Downloaded 34GiB to GPU in 2.0534s (16.5579GiB/s) | 20260506 01:14:27.988 | INFO | Analyzing chunk frames = [7788032,7788544) | 20260506 01:14:27.988 | INFO | MemInfo: { | 20260506 01:14:27.988 | INFO | "gpu" : " 17 GB", | 20260506 01:14:27.988 | INFO | "huge" : "", | 20260506 01:14:27.988 | INFO | "huge_shared" : " 27 GB", | 20260506 01:14:27.988 | INFO | "malloc" : " 14 GB", | 20260506 01:14:27.988 | INFO | "pinned" : " 29 GB" | 20260506 01:14:27.988 | INFO | } | 20260506 01:14:27.989 | INFO | Perf report for the last 5.14504 seconds: | 20260506 01:14:27.989 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:14:27.989 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:27.989 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:27.989 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:27.989 | INFO | ANALYSIS : 61.2%, 3.0, 95.1ms, 0 | 20260506 01:14:27.989 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:27.989 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:27.989 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:14:27.989 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:27.989 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:27.989 | INFO | Uploaded 36GiB to GPU in 2.10907s (17.0692GiB/s) | 20260506 01:14:27.989 | INFO | Downloaded 34GiB to GPU in 2.0247s (16.7926GiB/s) | 20260506 01:14:33.118 | INFO | Analyzing chunk frames = [7788544,7789056) | 20260506 01:14:33.118 | INFO | MemInfo: { | 20260506 01:14:33.118 | INFO | "gpu" : " 17 GB", | 20260506 01:14:33.118 | INFO | "huge" : "", | 20260506 01:14:33.118 | INFO | "huge_shared" : " 27 GB", | 20260506 01:14:33.118 | INFO | "malloc" : " 14 GB", | 20260506 01:14:33.118 | INFO | "pinned" : " 29 GB" | 20260506 01:14:33.118 | INFO | } | 20260506 01:14:33.118 | INFO | Perf report for the last 5.1293 seconds: | 20260506 01:14:33.118 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:14:33.118 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:33.118 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:33.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:33.118 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260506 01:14:33.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:33.118 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:33.118 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 01:14:33.118 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:33.118 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:33.119 | INFO | Uploaded 36GiB to GPU in 2.06975s (17.3934GiB/s) | 20260506 01:14:33.119 | INFO | Downloaded 34GiB to GPU in 2.03148s (16.7365GiB/s) | 20260506 01:14:38.264 | INFO | Analyzing chunk frames = [7789056,7789568) | 20260506 01:14:38.264 | INFO | MemInfo: { | 20260506 01:14:38.264 | INFO | "gpu" : " 17 GB", | 20260506 01:14:38.264 | INFO | "huge" : "", | 20260506 01:14:38.264 | INFO | "huge_shared" : " 27 GB", | 20260506 01:14:38.264 | INFO | "malloc" : " 14 GB", | 20260506 01:14:38.264 | INFO | "pinned" : " 29 GB" | 20260506 01:14:38.264 | INFO | } | 20260506 01:14:38.264 | INFO | Perf report for the last 5.14582 seconds: | 20260506 01:14:38.264 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:14:38.264 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:38.264 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:38.264 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:38.264 | INFO | ANALYSIS : 61.1%, 3.0, 95.5ms, 0 | 20260506 01:14:38.264 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:38.264 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:38.264 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 01:14:38.264 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:38.264 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:38.264 | INFO | Uploaded 36GiB to GPU in 2.06668s (17.4193GiB/s) | 20260506 01:14:38.264 | INFO | Downloaded 34GiB to GPU in 2.04144s (16.6549GiB/s) | 20260506 01:14:43.394 | INFO | Analyzing chunk frames = [7789568,7790080) | 20260506 01:14:43.394 | INFO | MemInfo: { | 20260506 01:14:43.394 | INFO | "gpu" : " 17 GB", | 20260506 01:14:43.394 | INFO | "huge" : "", | 20260506 01:14:43.394 | INFO | "huge_shared" : " 27 GB", | 20260506 01:14:43.394 | INFO | "malloc" : " 14 GB", | 20260506 01:14:43.394 | INFO | "pinned" : " 29 GB" | 20260506 01:14:43.394 | INFO | } | 20260506 01:14:43.394 | INFO | Perf report for the last 5.13081 seconds: | 20260506 01:14:43.394 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:14:43.394 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:43.395 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:43.395 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:43.395 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 01:14:43.395 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:43.395 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:43.395 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 01:14:43.395 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:43.395 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:43.395 | INFO | Uploaded 36GiB to GPU in 2.02087s (17.8141GiB/s) | 20260506 01:14:43.395 | INFO | Downloaded 34GiB to GPU in 2.05214s (16.5681GiB/s) | 20260506 01:14:48.540 | INFO | Analyzing chunk frames = [7790080,7790592) | 20260506 01:14:48.540 | INFO | MemInfo: { | 20260506 01:14:48.540 | INFO | "gpu" : " 17 GB", | 20260506 01:14:48.540 | INFO | "huge" : "", | 20260506 01:14:48.540 | INFO | "huge_shared" : " 27 GB", | 20260506 01:14:48.540 | INFO | "malloc" : " 14 GB", | 20260506 01:14:48.540 | INFO | "pinned" : " 29 GB" | 20260506 01:14:48.540 | INFO | } | 20260506 01:14:48.540 | INFO | Perf report for the last 5.14588 seconds: | 20260506 01:14:48.540 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:14:48.540 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:48.540 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:48.540 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:48.540 | INFO | ANALYSIS : 61.8%, 3.0, 96.6ms, 0 | 20260506 01:14:48.540 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:48.540 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:48.540 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:14:48.540 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:48.540 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:48.541 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 01:14:48.541 | INFO | Total mem usage: 15.3 GiB | 20260506 01:14:48.541 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 01:14:48.541 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.2%) | 20260506 01:14:48.541 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 01:14:48.541 | INFO | High Water Breakdown: | 20260506 01:14:48.541 | INFO | 60.9% : 8.54 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 01:14:48.541 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 01:14:48.541 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 01:14:48.541 | INFO | ------------------------------------------ | 20260506 01:14:48.541 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:14:48.541 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 01:14:48.541 | INFO | Total mem usage: 26.5 GiB | 20260506 01:14:48.541 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 01:14:48.541 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 01:14:48.541 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 01:14:48.541 | INFO | High Water Breakdown: | 20260506 01:14:48.541 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:14:48.541 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:14:48.541 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:14:48.541 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:14:48.541 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:14:48.541 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:14:48.541 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 01:14:48.541 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 01:14:48.541 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:14:48.541 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 01:14:48.541 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 01:14:48.541 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 01:14:48.541 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 01:14:48.541 | INFO | Uploaded 36GiB to GPU in 2.13063s (16.8964GiB/s) | 20260506 01:14:48.541 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 01:14:48.541 | INFO | Downloaded 34GiB to GPU in 2.03024s (16.7468GiB/s) | 20260506 01:14:48.541 | INFO | ------------------------------------------ | 20260506 01:14:48.541 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:14:48.541 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 01:14:48.541 | INFO | Total mem usage: 25.6 GiB | 20260506 01:14:48.541 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 01:14:48.541 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 01:14:48.541 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 01:14:48.541 | INFO | High Water Breakdown: | 20260506 01:14:48.541 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 01:14:48.541 | INFO | ------------------------------------------ | 20260506 01:14:48.541 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:14:48.541 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 01:14:48.541 | INFO | Total mem usage: 15.7 GiB | 20260506 01:14:48.541 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 01:14:48.541 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 01:14:48.541 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 01:14:48.541 | INFO | High Water Breakdown: | 20260506 01:14:48.541 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 01:14:48.541 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 01:14:48.541 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 01:14:48.541 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 01:14:48.541 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 01:14:48.541 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 01:14:48.541 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 01:14:48.541 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 01:14:48.541 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 01:14:48.541 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:14:48.541 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 01:14:48.541 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:14:48.541 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 01:14:48.541 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 01:14:48.541 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 01:14:48.541 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 01:14:48.541 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:14:48.541 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 01:14:48.541 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:14:48.541 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:14:48.541 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:14:48.541 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 01:14:48.541 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 01:14:48.541 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:14:48.541 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 01:14:48.541 | INFO | ------------------------------------------ | 20260506 01:14:48.542 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:14:48.542 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 01:14:48.542 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:14:48.542 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:14:48.542 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.40e+01, 1.35e+01 / 8.45e-01 | 20260506 01:14:48.542 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.05e-01, 1.04e-02 / 3.32e-03 | 20260506 01:14:48.542 | INFO | PerfReport: 11.84%, Baseline -- 3.71e+00 / 4.51e+00, 5.72e-01 / 1.94e-01 | 20260506 01:14:48.542 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.51e+00, 1.88e-01 / 8.36e-02 | 20260506 01:14:48.542 | INFO | PerfReport: 16.01%, DME -- 4.01e+01 / 4.41e+01, 3.91e+00 / 1.01e+00 | 20260506 01:14:48.542 | INFO | PerfReport: 57.43%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.52e+00 / 8.57e-01 | 20260506 01:14:48.542 | INFO | PerfReport: 7.05%, PulseAccumulating -- 2.21e+00 / 2.77e+00, 4.20e-01 / 1.48e-01 | 20260506 01:14:48.542 | INFO | PerfReport: 3.04%, Metrics -- 9.51e-01 / 1.24e+00, 2.57e-01 / 1.29e-01 | 20260506 01:14:48.542 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.30e-01 / 3.86e-01, 5.61e-02 / 2.97e-02 | 20260506 01:14:48.542 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 01:14:48.542 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:14:48.542 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:14:48.542 | INFO | PerfReport: Full Execution -- 9.81e+01 / -nan, 2.43e+01 / 0.00e+00 | 20260506 01:14:48.542 | INFO | PerfReport: 22.31%, Upload -- 2.19e+01 / 0.00e+00, 7.49e+00 / 0.00e+00 | 20260506 01:14:48.542 | INFO | PerfReport: 21.93%, Download -- 2.15e+01 / 0.00e+00, 7.65e+00 / 0.00e+00 | 20260506 01:14:48.542 | INFO | PerfReport: 31.91%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 01:14:48.542 | INFO | PerfReport: 2.05%, LockUpload -- 2.01e+00 / 0.00e+00, 7.88e+00 / 0.00e+00 | 20260506 01:14:48.542 | INFO | PerfReport: 1.32%, LockDownload -- 1.29e+00 / 0.00e+00, 2.96e+00 / 0.00e+00 | 20260506 01:14:48.542 | INFO | PerfReport: 20.47%, LockCompute -- 2.01e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 01:14:48.542 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 2.31e-02 / 0.00e+00 | 20260506 01:14:48.658 | INFO | Starting a round of Baz IO | 20260506 01:14:51.809 | INFO | Populating RTMetrics report | 20260506 01:14:51.809 | INFO | RTMetrics report populated | 20260506 01:14:51.809 | INFO | Opening tmp RTMetrics json file | 20260506 01:14:51.810 | INFO | Serializing RTMetrics to json | 20260506 01:14:51.812 | INFO | Serialization done, swapping tmp file now... | 20260506 01:14:51.812 | INFO | RTMetrics done | 20260506 01:14:51.885 | INFO | Finished a round of Baz IO: Wrote 1.829 GiB in 3.227 seconds | 20260506 01:14:51.885 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.33% / 0.00% / 78.67% | 20260506 01:14:51.885 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.082322% / 0% / 9.13938e-05% | 20260506 01:14:51.885 | INFO | Baz IO overall: | 20260506 01:14:51.885 | INFO | BytesWritten : 4401715839333 | 20260506 01:14:51.885 | INFO | NumEvents : 2069454746225 | 20260506 01:14:53.680 | INFO | Analyzing chunk frames = [7790592,7791104) | 20260506 01:14:53.680 | INFO | MemInfo: { | 20260506 01:14:53.680 | INFO | "gpu" : " 17 GB", | 20260506 01:14:53.680 | INFO | "huge" : "", | 20260506 01:14:53.680 | INFO | "huge_shared" : " 27 GB", | 20260506 01:14:53.680 | INFO | "malloc" : " 14 GB", | 20260506 01:14:53.680 | INFO | "pinned" : " 29 GB" | 20260506 01:14:53.680 | INFO | } | 20260506 01:14:53.680 | INFO | Perf report for the last 5.1393 seconds: | 20260506 01:14:53.680 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:14:53.680 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:53.680 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:53.680 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:53.680 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260506 01:14:53.680 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:53.680 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260506 01:14:53.680 | INFO | PRE_HQ : 59.6%, 1.7, 53.6ms, 0 | 20260506 01:14:53.680 | INFO | BAZWRITER : 61.9%, 3.4, 109.3ms, 0 | 20260506 01:14:53.680 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:14:53.680 | INFO | Uploaded 36GiB to GPU in 2.18365s (16.4862GiB/s) | 20260506 01:14:53.680 | INFO | Downloaded 38GiB to GPU in 2.2388s (16.9734GiB/s) | 20260506 01:14:58.810 | INFO | Analyzing chunk frames = [7791104,7791616) | 20260506 01:14:58.810 | INFO | MemInfo: { | 20260506 01:14:58.810 | INFO | "gpu" : " 17 GB", | 20260506 01:14:58.810 | INFO | "huge" : "", | 20260506 01:14:58.810 | INFO | "huge_shared" : " 27 GB", | 20260506 01:14:58.810 | INFO | "malloc" : " 14 GB", | 20260506 01:14:58.810 | INFO | "pinned" : " 29 GB" | 20260506 01:14:58.810 | INFO | } | 20260506 01:14:58.811 | INFO | Perf report for the last 5.13082 seconds: | 20260506 01:14:58.811 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:14:58.811 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:58.811 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:58.811 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:58.811 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 01:14:58.811 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:58.811 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:58.811 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 01:14:58.811 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:58.811 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:14:58.811 | INFO | Uploaded 36GiB to GPU in 2.11607s (17.0126GiB/s) | 20260506 01:14:58.811 | INFO | Downloaded 34GiB to GPU in 2.04022s (16.6649GiB/s) | 20260506 01:15:03.952 | INFO | Analyzing chunk frames = [7791616,7792128) | 20260506 01:15:03.952 | INFO | MemInfo: { | 20260506 01:15:03.952 | INFO | "gpu" : " 17 GB", | 20260506 01:15:03.952 | INFO | "huge" : "", | 20260506 01:15:03.952 | INFO | "huge_shared" : " 27 GB", | 20260506 01:15:03.952 | INFO | "malloc" : " 14 GB", | 20260506 01:15:03.952 | INFO | "pinned" : " 29 GB" | 20260506 01:15:03.952 | INFO | } | 20260506 01:15:03.952 | INFO | Perf report for the last 5.14151 seconds: | 20260506 01:15:03.952 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:15:03.952 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:03.952 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:03.952 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:03.952 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 01:15:03.952 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:03.952 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:03.952 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:15:03.952 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:03.952 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:03.952 | INFO | Uploaded 36GiB to GPU in 2.1429s (16.7996GiB/s) | 20260506 01:15:03.952 | INFO | Downloaded 34GiB to GPU in 2.05109s (16.5765GiB/s) | 20260506 01:15:09.095 | INFO | Analyzing chunk frames = [7792128,7792640) | 20260506 01:15:09.095 | INFO | MemInfo: { | 20260506 01:15:09.095 | INFO | "gpu" : " 17 GB", | 20260506 01:15:09.095 | INFO | "huge" : "", | 20260506 01:15:09.095 | INFO | "huge_shared" : " 27 GB", | 20260506 01:15:09.095 | INFO | "malloc" : " 14 GB", | 20260506 01:15:09.095 | INFO | "pinned" : " 29 GB" | 20260506 01:15:09.095 | INFO | } | 20260506 01:15:09.095 | INFO | Perf report for the last 5.14309 seconds: | 20260506 01:15:09.095 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:15:09.095 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:09.095 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:09.095 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:09.095 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260506 01:15:09.095 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:09.095 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:09.095 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 01:15:09.095 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:09.095 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:09.095 | INFO | Uploaded 36GiB to GPU in 2.13852s (16.834GiB/s) | 20260506 01:15:09.095 | INFO | Downloaded 34GiB to GPU in 2.02655s (16.7773GiB/s) | 20260506 01:15:14.237 | INFO | Analyzing chunk frames = [7792640,7793152) | 20260506 01:15:14.237 | INFO | MemInfo: { | 20260506 01:15:14.237 | INFO | "gpu" : " 17 GB", | 20260506 01:15:14.237 | INFO | "huge" : "", | 20260506 01:15:14.237 | INFO | "huge_shared" : " 27 GB", | 20260506 01:15:14.237 | INFO | "malloc" : " 14 GB", | 20260506 01:15:14.237 | INFO | "pinned" : " 29 GB" | 20260506 01:15:14.237 | INFO | } | 20260506 01:15:14.237 | INFO | Perf report for the last 5.14162 seconds: | 20260506 01:15:14.237 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:15:14.237 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:14.237 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:14.237 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:14.237 | INFO | ANALYSIS : 60.8%, 3.0, 94.9ms, 0 | 20260506 01:15:14.237 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:14.237 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:14.237 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 01:15:14.237 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:14.237 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:14.237 | INFO | Uploaded 36GiB to GPU in 2.0896s (17.2281GiB/s) | 20260506 01:15:14.237 | INFO | Downloaded 34GiB to GPU in 2.02777s (16.7671GiB/s) | 20260506 01:15:19.373 | INFO | Analyzing chunk frames = [7793152,7793664) | 20260506 01:15:19.373 | INFO | MemInfo: { | 20260506 01:15:19.373 | INFO | "gpu" : " 17 GB", | 20260506 01:15:19.373 | INFO | "huge" : "", | 20260506 01:15:19.373 | INFO | "huge_shared" : " 27 GB", | 20260506 01:15:19.373 | INFO | "malloc" : " 14 GB", | 20260506 01:15:19.373 | INFO | "pinned" : " 29 GB" | 20260506 01:15:19.373 | INFO | } | 20260506 01:15:19.373 | INFO | Perf report for the last 5.13609 seconds: | 20260506 01:15:19.373 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:15:19.373 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:19.373 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:19.373 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:19.373 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 01:15:19.373 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:19.373 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:19.373 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 01:15:19.373 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:19.373 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:19.373 | INFO | Uploaded 36GiB to GPU in 2.08172s (17.2934GiB/s) | 20260506 01:15:19.373 | INFO | Downloaded 34GiB to GPU in 2.03516s (16.7063GiB/s) | 20260506 01:15:24.510 | INFO | Analyzing chunk frames = [7793664,7794176) | 20260506 01:15:24.510 | INFO | MemInfo: { | 20260506 01:15:24.510 | INFO | "gpu" : " 17 GB", | 20260506 01:15:24.510 | INFO | "huge" : "", | 20260506 01:15:24.510 | INFO | "huge_shared" : " 27 GB", | 20260506 01:15:24.510 | INFO | "malloc" : " 14 GB", | 20260506 01:15:24.510 | INFO | "pinned" : " 29 GB" | 20260506 01:15:24.510 | INFO | } | 20260506 01:15:24.510 | INFO | Perf report for the last 5.13756 seconds: | 20260506 01:15:24.510 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:15:24.510 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:24.510 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:24.510 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:24.510 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260506 01:15:24.510 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:24.511 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:24.511 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 01:15:24.511 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:24.511 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:24.511 | INFO | Uploaded 36GiB to GPU in 2.09329s (17.1978GiB/s) | 20260506 01:15:24.511 | INFO | Downloaded 34GiB to GPU in 2.0933s (16.2423GiB/s) | 20260506 01:15:29.649 | INFO | Analyzing chunk frames = [7794176,7794688) | 20260506 01:15:29.649 | INFO | MemInfo: { | 20260506 01:15:29.649 | INFO | "gpu" : " 17 GB", | 20260506 01:15:29.649 | INFO | "huge" : "", | 20260506 01:15:29.649 | INFO | "huge_shared" : " 27 GB", | 20260506 01:15:29.649 | INFO | "malloc" : " 14 GB", | 20260506 01:15:29.649 | INFO | "pinned" : " 29 GB" | 20260506 01:15:29.649 | INFO | } | 20260506 01:15:29.649 | INFO | Perf report for the last 5.13827 seconds: | 20260506 01:15:29.649 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:15:29.649 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:29.649 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:29.649 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:29.649 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260506 01:15:29.649 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:29.649 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:29.649 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:15:29.649 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:29.649 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:29.649 | INFO | Uploaded 36GiB to GPU in 2.11641s (17.0099GiB/s) | 20260506 01:15:29.649 | INFO | Downloaded 34GiB to GPU in 2.01304s (16.8899GiB/s) | 20260506 01:15:29.768 | INFO | Starting a round of Baz IO | 20260506 01:15:32.915 | INFO | Populating RTMetrics report | 20260506 01:15:32.916 | INFO | RTMetrics report populated | 20260506 01:15:32.916 | INFO | Opening tmp RTMetrics json file | 20260506 01:15:32.916 | INFO | Serializing RTMetrics to json | 20260506 01:15:32.917 | INFO | Serialization done, swapping tmp file now... | 20260506 01:15:32.918 | INFO | RTMetrics done | 20260506 01:15:32.991 | INFO | Finished a round of Baz IO: Wrote 1.832 GiB in 3.223 seconds | 20260506 01:15:32.991 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.33% / 0.00% / 78.67% | 20260506 01:15:32.991 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0821352% / 0% / 8.74201e-05% | 20260506 01:15:32.991 | INFO | Baz IO overall: | 20260506 01:15:32.991 | INFO | BytesWritten : 4403682524885 | 20260506 01:15:32.991 | INFO | NumEvents : 2070351983918 | 20260506 01:15:34.791 | INFO | Analyzing chunk frames = [7794688,7795200) | 20260506 01:15:34.791 | INFO | MemInfo: { | 20260506 01:15:34.791 | INFO | "gpu" : " 17 GB", | 20260506 01:15:34.791 | INFO | "huge" : "", | 20260506 01:15:34.791 | INFO | "huge_shared" : " 27 GB", | 20260506 01:15:34.791 | INFO | "malloc" : " 14 GB", | 20260506 01:15:34.791 | INFO | "pinned" : " 29 GB" | 20260506 01:15:34.791 | INFO | } | 20260506 01:15:34.791 | INFO | Perf report for the last 5.14223 seconds: | 20260506 01:15:34.791 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:15:34.791 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:34.791 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:34.791 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:34.791 | INFO | ANALYSIS : 62.5%, 3.0, 97.7ms, 0 | 20260506 01:15:34.791 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:34.791 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260506 01:15:34.791 | INFO | PRE_HQ : 59.7%, 1.7, 53.6ms, 0 | 20260506 01:15:34.791 | INFO | BAZWRITER : 61.8%, 3.5, 110.3ms, 0 | 20260506 01:15:34.791 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:15:34.791 | INFO | Uploaded 36GiB to GPU in 2.17439s (16.5563GiB/s) | 20260506 01:15:34.791 | INFO | Downloaded 38GiB to GPU in 2.242s (16.9492GiB/s) | 20260506 01:15:39.941 | INFO | Analyzing chunk frames = [7795200,7795712) | 20260506 01:15:39.941 | INFO | MemInfo: { | 20260506 01:15:39.941 | INFO | "gpu" : " 17 GB", | 20260506 01:15:39.941 | INFO | "huge" : "", | 20260506 01:15:39.941 | INFO | "huge_shared" : " 27 GB", | 20260506 01:15:39.941 | INFO | "malloc" : " 14 GB", | 20260506 01:15:39.941 | INFO | "pinned" : " 29 GB" | 20260506 01:15:39.941 | INFO | } | 20260506 01:15:39.941 | INFO | Perf report for the last 5.15031 seconds: | 20260506 01:15:39.941 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:15:39.941 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:39.941 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:39.941 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:39.941 | INFO | ANALYSIS : 60.9%, 3.0, 95.3ms, 0 | 20260506 01:15:39.941 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:39.941 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:39.941 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 01:15:39.941 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:39.941 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:39.942 | INFO | Uploaded 36GiB to GPU in 1.97346s (18.2421GiB/s) | 20260506 01:15:39.942 | INFO | Downloaded 34GiB to GPU in 1.99305s (17.0593GiB/s) | 20260506 01:15:45.073 | INFO | Analyzing chunk frames = [7795712,7796224) | 20260506 01:15:45.073 | INFO | MemInfo: { | 20260506 01:15:45.073 | INFO | "gpu" : " 17 GB", | 20260506 01:15:45.074 | INFO | "huge" : "", | 20260506 01:15:45.074 | INFO | "huge_shared" : " 27 GB", | 20260506 01:15:45.074 | INFO | "malloc" : " 14 GB", | 20260506 01:15:45.074 | INFO | "pinned" : " 29 GB" | 20260506 01:15:45.074 | INFO | } | 20260506 01:15:45.074 | INFO | Perf report for the last 5.1323 seconds: | 20260506 01:15:45.074 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:15:45.074 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:45.074 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:45.074 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:45.074 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 01:15:45.074 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:45.074 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:45.074 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:15:45.074 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:45.074 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:45.074 | INFO | Uploaded 36GiB to GPU in 2.0998s (17.1445GiB/s) | 20260506 01:15:45.074 | INFO | Downloaded 34GiB to GPU in 2.03126s (16.7384GiB/s) | 20260506 01:15:50.210 | INFO | Analyzing chunk frames = [7796224,7796736) | 20260506 01:15:50.210 | INFO | MemInfo: { | 20260506 01:15:50.210 | INFO | "gpu" : " 17 GB", | 20260506 01:15:50.210 | INFO | "huge" : "", | 20260506 01:15:50.210 | INFO | "huge_shared" : " 27 GB", | 20260506 01:15:50.210 | INFO | "malloc" : " 14 GB", | 20260506 01:15:50.210 | INFO | "pinned" : " 29 GB" | 20260506 01:15:50.210 | INFO | } | 20260506 01:15:50.210 | INFO | Perf report for the last 5.13657 seconds: | 20260506 01:15:50.210 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:15:50.210 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:50.210 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:50.210 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:50.210 | INFO | ANALYSIS : 61.6%, 3.0, 95.6ms, 0 | 20260506 01:15:50.210 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:50.210 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:50.210 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:15:50.210 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:50.210 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:50.210 | INFO | Uploaded 36GiB to GPU in 2.14568s (16.7779GiB/s) | 20260506 01:15:50.210 | INFO | Downloaded 34GiB to GPU in 2.05616s (16.5357GiB/s) | 20260506 01:15:55.350 | INFO | Analyzing chunk frames = [7796736,7797248) | 20260506 01:15:55.350 | INFO | MemInfo: { | 20260506 01:15:55.350 | INFO | "gpu" : " 17 GB", | 20260506 01:15:55.350 | INFO | "huge" : "", | 20260506 01:15:55.350 | INFO | "huge_shared" : " 27 GB", | 20260506 01:15:55.350 | INFO | "malloc" : " 14 GB", | 20260506 01:15:55.350 | INFO | "pinned" : " 29 GB" | 20260506 01:15:55.350 | INFO | } | 20260506 01:15:55.350 | INFO | Perf report for the last 5.1396 seconds: | 20260506 01:15:55.350 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:15:55.350 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:55.350 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:55.350 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:55.350 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 01:15:55.350 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:55.350 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:55.350 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 01:15:55.350 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:55.350 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:15:55.350 | INFO | Uploaded 36GiB to GPU in 2.11326s (17.0353GiB/s) | 20260506 01:15:55.350 | INFO | Downloaded 34GiB to GPU in 2.04406s (16.6336GiB/s) | 20260506 01:16:00.488 | INFO | Analyzing chunk frames = [7797248,7797760) | 20260506 01:16:00.488 | INFO | MemInfo: { | 20260506 01:16:00.488 | INFO | "gpu" : " 17 GB", | 20260506 01:16:00.488 | INFO | "huge" : "", | 20260506 01:16:00.488 | INFO | "huge_shared" : " 27 GB", | 20260506 01:16:00.488 | INFO | "malloc" : " 14 GB", | 20260506 01:16:00.488 | INFO | "pinned" : " 29 GB" | 20260506 01:16:00.488 | INFO | } | 20260506 01:16:00.488 | INFO | Perf report for the last 5.13835 seconds: | 20260506 01:16:00.488 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:16:00.488 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:00.488 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:00.488 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:00.488 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 01:16:00.488 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:00.488 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:00.488 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 01:16:00.488 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:00.488 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:00.488 | INFO | Uploaded 36GiB to GPU in 2.0762s (17.3393GiB/s) | 20260506 01:16:00.488 | INFO | Downloaded 34GiB to GPU in 2.03096s (16.7408GiB/s) | 20260506 01:16:05.626 | INFO | Analyzing chunk frames = [7797760,7798272) | 20260506 01:16:05.626 | INFO | MemInfo: { | 20260506 01:16:05.626 | INFO | "gpu" : " 17 GB", | 20260506 01:16:05.626 | INFO | "huge" : "", | 20260506 01:16:05.626 | INFO | "huge_shared" : " 27 GB", | 20260506 01:16:05.626 | INFO | "malloc" : " 14 GB", | 20260506 01:16:05.626 | INFO | "pinned" : " 29 GB" | 20260506 01:16:05.626 | INFO | } | 20260506 01:16:05.626 | INFO | Perf report for the last 5.13751 seconds: | 20260506 01:16:05.626 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:16:05.626 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:05.626 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:05.626 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:05.626 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260506 01:16:05.626 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:05.626 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:05.626 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 01:16:05.626 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:05.626 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:05.626 | INFO | Uploaded 36GiB to GPU in 2.08121s (17.2976GiB/s) | 20260506 01:16:05.626 | INFO | Downloaded 34GiB to GPU in 2.07428s (16.3912GiB/s) | 20260506 01:16:10.766 | INFO | Analyzing chunk frames = [7798272,7798784) | 20260506 01:16:10.766 | INFO | MemInfo: { | 20260506 01:16:10.766 | INFO | "gpu" : " 17 GB", | 20260506 01:16:10.766 | INFO | "huge" : "", | 20260506 01:16:10.766 | INFO | "huge_shared" : " 27 GB", | 20260506 01:16:10.766 | INFO | "malloc" : " 14 GB", | 20260506 01:16:10.766 | INFO | "pinned" : " 29 GB" | 20260506 01:16:10.766 | INFO | } | 20260506 01:16:10.766 | INFO | Perf report for the last 5.14024 seconds: | 20260506 01:16:10.766 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:16:10.766 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:10.766 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:10.766 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:10.766 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260506 01:16:10.766 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:10.766 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:10.766 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 01:16:10.766 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:10.766 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:10.766 | INFO | Uploaded 36GiB to GPU in 2.1137s (17.0318GiB/s) | 20260506 01:16:10.766 | INFO | Downloaded 34GiB to GPU in 2.0063s (16.9466GiB/s) | 20260506 01:16:10.766 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 01:16:10.766 | INFO | Total mem usage: 15.3 GiB | 20260506 01:16:10.766 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 01:16:10.767 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.2%) | 20260506 01:16:10.767 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 01:16:10.767 | INFO | High Water Breakdown: | 20260506 01:16:10.767 | INFO | 60.9% : 8.54 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 01:16:10.767 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 01:16:10.767 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 01:16:10.767 | INFO | ------------------------------------------ | 20260506 01:16:10.767 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:16:10.767 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 01:16:10.767 | INFO | Total mem usage: 26.5 GiB | 20260506 01:16:10.767 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 01:16:10.767 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 01:16:10.767 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260506 01:16:10.767 | INFO | High Water Breakdown: | 20260506 01:16:10.767 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:16:10.767 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:16:10.767 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:16:10.767 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:16:10.767 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:16:10.767 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:16:10.767 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 01:16:10.767 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 01:16:10.767 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:16:10.767 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 01:16:10.767 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 01:16:10.767 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 01:16:10.767 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 01:16:10.767 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 01:16:10.767 | INFO | ------------------------------------------ | 20260506 01:16:10.767 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:16:10.767 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 01:16:10.767 | INFO | Total mem usage: 25.6 GiB | 20260506 01:16:10.767 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 01:16:10.767 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 01:16:10.767 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 01:16:10.767 | INFO | High Water Breakdown: | 20260506 01:16:10.767 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 01:16:10.767 | INFO | ------------------------------------------ | 20260506 01:16:10.767 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:16:10.767 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 01:16:10.767 | INFO | Total mem usage: 15.7 GiB | 20260506 01:16:10.767 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 01:16:10.767 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 01:16:10.767 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 01:16:10.767 | INFO | High Water Breakdown: | 20260506 01:16:10.767 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 01:16:10.767 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 01:16:10.767 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 01:16:10.767 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 01:16:10.767 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 01:16:10.767 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 01:16:10.767 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 01:16:10.767 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 01:16:10.767 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 01:16:10.767 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:16:10.767 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 01:16:10.767 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:16:10.767 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 01:16:10.767 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 01:16:10.767 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 01:16:10.767 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 01:16:10.767 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:16:10.767 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 01:16:10.767 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:16:10.767 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:16:10.767 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:16:10.767 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 01:16:10.767 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 01:16:10.767 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:16:10.767 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 01:16:10.767 | INFO | ------------------------------------------ | 20260506 01:16:10.767 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:16:10.767 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 01:16:10.767 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:16:10.767 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:16:10.767 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.37e+01, 1.35e+01 / 7.32e-01 | 20260506 01:16:10.767 | INFO | PerfReport: 0.29%, AggregateStats -- 9.18e-02 / 1.03e-01, 1.04e-02 / 3.85e-03 | 20260506 01:16:10.767 | INFO | PerfReport: 11.82%, Baseline -- 3.69e+00 / 4.51e+00, 5.63e-01 / 1.80e-01 | 20260506 01:16:10.767 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.52e+00, 1.89e-01 / 8.73e-02 | 20260506 01:16:10.767 | INFO | PerfReport: 15.99%, DME -- 4.00e+01 / 4.38e+01, 3.76e+00 / 9.93e-01 | 20260506 01:16:10.767 | INFO | PerfReport: 57.41%, FrameLabeling -- 1.79e+01 / 1.95e+01, 1.46e+00 / 1.16e+00 | 20260506 01:16:10.767 | INFO | PerfReport: 7.09%, PulseAccumulating -- 2.22e+00 / 2.73e+00, 4.23e-01 / 2.47e-01 | 20260506 01:16:10.767 | INFO | PerfReport: 3.06%, Metrics -- 9.55e-01 / 1.23e+00, 2.58e-01 / 1.15e-01 | 20260506 01:16:10.767 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.30e-01 / 3.87e-01, 5.57e-02 / 2.92e-02 | 20260506 01:16:10.767 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 01:16:10.767 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:16:10.767 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:16:10.767 | INFO | PerfReport: Full Execution -- 9.81e+01 / -nan, 2.46e+01 / 0.00e+00 | 20260506 01:16:10.767 | INFO | PerfReport: 22.38%, Upload -- 2.20e+01 / 0.00e+00, 7.53e+00 / 0.00e+00 | 20260506 01:16:10.767 | INFO | PerfReport: 21.89%, Download -- 2.15e+01 / 0.00e+00, 7.62e+00 / 0.00e+00 | 20260506 01:16:10.767 | INFO | PerfReport: 31.84%, Compute -- 3.12e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 01:16:10.767 | INFO | PerfReport: 2.13%, LockUpload -- 2.09e+00 / 0.00e+00, 7.97e+00 / 0.00e+00 | 20260506 01:16:10.767 | INFO | PerfReport: 1.30%, LockDownload -- 1.28e+00 / 0.00e+00, 2.85e+00 / 0.00e+00 | 20260506 01:16:10.767 | INFO | PerfReport: 20.44%, LockCompute -- 2.01e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260506 01:16:10.767 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.50e-02 / 0.00e+00 | 20260506 01:16:10.886 | INFO | Starting a round of Baz IO | 20260506 01:16:14.031 | INFO | Populating RTMetrics report | 20260506 01:16:14.031 | INFO | RTMetrics report populated | 20260506 01:16:14.031 | INFO | Opening tmp RTMetrics json file | 20260506 01:16:14.033 | INFO | Serializing RTMetrics to json | 20260506 01:16:14.034 | INFO | Serialization done, swapping tmp file now... | 20260506 01:16:14.034 | INFO | RTMetrics done | 20260506 01:16:14.112 | INFO | Finished a round of Baz IO: Wrote 1.831 GiB in 3.226 seconds | 20260506 01:16:14.112 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.33% / 0.00% / 78.67% | 20260506 01:16:14.112 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0828942% / 0% / 7.15256e-05% | 20260506 01:16:14.112 | INFO | Baz IO overall: | 20260506 01:16:14.112 | INFO | BytesWritten : 4405648965338 | 20260506 01:16:14.112 | INFO | NumEvents : 2071248919417 | 20260506 01:16:15.908 | INFO | Analyzing chunk frames = [7798784,7799296) | 20260506 01:16:15.908 | INFO | MemInfo: { | 20260506 01:16:15.908 | INFO | "gpu" : " 17 GB", | 20260506 01:16:15.908 | INFO | "huge" : "", | 20260506 01:16:15.908 | INFO | "huge_shared" : " 27 GB", | 20260506 01:16:15.908 | INFO | "malloc" : " 14 GB", | 20260506 01:16:15.908 | INFO | "pinned" : " 29 GB" | 20260506 01:16:15.908 | INFO | } | 20260506 01:16:15.908 | INFO | Perf report for the last 5.14225 seconds: | 20260506 01:16:15.908 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:16:15.908 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:15.908 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:15.908 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:15.908 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260506 01:16:15.908 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:15.908 | INFO | RT_METRICS : 43.0%, 1.0, 22.6ms, 0 | 20260506 01:16:15.908 | INFO | PRE_HQ : 59.9%, 1.7, 54.3ms, 0 | 20260506 01:16:15.908 | INFO | BAZWRITER : 61.8%, 3.4, 109.6ms, 0 | 20260506 01:16:15.908 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:16:15.909 | INFO | Uploaded 36GiB to GPU in 2.17499s (16.5518GiB/s) | 20260506 01:16:15.909 | INFO | Downloaded 38GiB to GPU in 2.2399s (16.965GiB/s) | 20260506 01:16:21.051 | INFO | Analyzing chunk frames = [7799296,7799808) | 20260506 01:16:21.051 | INFO | MemInfo: { | 20260506 01:16:21.051 | INFO | "gpu" : " 17 GB", | 20260506 01:16:21.051 | INFO | "huge" : "", | 20260506 01:16:21.051 | INFO | "huge_shared" : " 27 GB", | 20260506 01:16:21.051 | INFO | "malloc" : " 14 GB", | 20260506 01:16:21.051 | INFO | "pinned" : " 29 GB" | 20260506 01:16:21.051 | INFO | } | 20260506 01:16:21.051 | INFO | Perf report for the last 5.14268 seconds: | 20260506 01:16:21.051 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:16:21.051 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:21.051 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:21.051 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:21.051 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260506 01:16:21.051 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:21.051 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:21.051 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 01:16:21.051 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:21.051 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:21.051 | INFO | Uploaded 36GiB to GPU in 2.11411s (17.0284GiB/s) | 20260506 01:16:21.051 | INFO | Downloaded 34GiB to GPU in 2.04048s (16.6628GiB/s) | 20260506 01:16:26.187 | INFO | Analyzing chunk frames = [7799808,7800320) | 20260506 01:16:26.187 | INFO | MemInfo: { | 20260506 01:16:26.187 | INFO | "gpu" : " 17 GB", | 20260506 01:16:26.187 | INFO | "huge" : "", | 20260506 01:16:26.187 | INFO | "huge_shared" : " 27 GB", | 20260506 01:16:26.187 | INFO | "malloc" : " 14 GB", | 20260506 01:16:26.187 | INFO | "pinned" : " 29 GB" | 20260506 01:16:26.187 | INFO | } | 20260506 01:16:26.187 | INFO | Perf report for the last 5.13597 seconds: | 20260506 01:16:26.187 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:16:26.187 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:26.187 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:26.187 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:26.187 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 01:16:26.187 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:26.187 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:26.187 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 01:16:26.187 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:26.187 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:26.187 | INFO | Uploaded 36GiB to GPU in 2.09368s (17.1946GiB/s) | 20260506 01:16:26.187 | INFO | Downloaded 34GiB to GPU in 2.02348s (16.8027GiB/s) | 20260506 01:16:31.321 | INFO | Analyzing chunk frames = [7800320,7800832) | 20260506 01:16:31.321 | INFO | MemInfo: { | 20260506 01:16:31.321 | INFO | "gpu" : " 17 GB", | 20260506 01:16:31.321 | INFO | "huge" : "", | 20260506 01:16:31.321 | INFO | "huge_shared" : " 27 GB", | 20260506 01:16:31.321 | INFO | "malloc" : " 14 GB", | 20260506 01:16:31.321 | INFO | "pinned" : " 29 GB" | 20260506 01:16:31.321 | INFO | } | 20260506 01:16:31.321 | INFO | Perf report for the last 5.13441 seconds: | 20260506 01:16:31.321 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:16:31.321 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:31.321 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:31.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:31.321 | INFO | ANALYSIS : 61.6%, 3.0, 95.7ms, 0 | 20260506 01:16:31.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:31.321 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:31.321 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 01:16:31.321 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:31.321 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:31.322 | INFO | Uploaded 36GiB to GPU in 2.15117s (16.7351GiB/s) | 20260506 01:16:31.322 | INFO | Downloaded 34GiB to GPU in 2.05027s (16.5832GiB/s) | 20260506 01:16:36.457 | INFO | Analyzing chunk frames = [7800832,7801344) | 20260506 01:16:36.457 | INFO | MemInfo: { | 20260506 01:16:36.457 | INFO | "gpu" : " 17 GB", | 20260506 01:16:36.457 | INFO | "huge" : "", | 20260506 01:16:36.457 | INFO | "huge_shared" : " 27 GB", | 20260506 01:16:36.457 | INFO | "malloc" : " 14 GB", | 20260506 01:16:36.457 | INFO | "pinned" : " 29 GB" | 20260506 01:16:36.457 | INFO | } | 20260506 01:16:36.457 | INFO | Perf report for the last 5.13582 seconds: | 20260506 01:16:36.457 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:16:36.457 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:36.457 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:36.457 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:36.457 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 01:16:36.457 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:36.457 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:36.457 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 01:16:36.457 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:36.457 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:36.457 | INFO | Uploaded 36GiB to GPU in 2.10391s (17.111GiB/s) | 20260506 01:16:36.457 | INFO | Downloaded 34GiB to GPU in 2.05115s (16.5761GiB/s) | 20260506 01:16:41.604 | INFO | Analyzing chunk frames = [7801344,7801856) | 20260506 01:16:41.604 | INFO | MemInfo: { | 20260506 01:16:41.604 | INFO | "gpu" : " 17 GB", | 20260506 01:16:41.604 | INFO | "huge" : "", | 20260506 01:16:41.604 | INFO | "huge_shared" : " 27 GB", | 20260506 01:16:41.604 | INFO | "malloc" : " 14 GB", | 20260506 01:16:41.604 | INFO | "pinned" : " 29 GB" | 20260506 01:16:41.604 | INFO | } | 20260506 01:16:41.604 | INFO | Perf report for the last 5.14657 seconds: | 20260506 01:16:41.604 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:16:41.604 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:41.604 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:41.604 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:41.604 | INFO | ANALYSIS : 61.2%, 3.0, 95.7ms, 0 | 20260506 01:16:41.604 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:41.604 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:41.604 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:16:41.604 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:41.604 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:41.604 | INFO | Uploaded 36GiB to GPU in 2.07629s (17.3386GiB/s) | 20260506 01:16:41.604 | INFO | Downloaded 34GiB to GPU in 2.04431s (16.6316GiB/s) | 20260506 01:16:46.743 | INFO | Analyzing chunk frames = [7801856,7802368) | 20260506 01:16:46.743 | INFO | MemInfo: { | 20260506 01:16:46.743 | INFO | "gpu" : " 17 GB", | 20260506 01:16:46.743 | INFO | "huge" : "", | 20260506 01:16:46.743 | INFO | "huge_shared" : " 27 GB", | 20260506 01:16:46.743 | INFO | "malloc" : " 14 GB", | 20260506 01:16:46.743 | INFO | "pinned" : " 29 GB" | 20260506 01:16:46.743 | INFO | } | 20260506 01:16:46.743 | INFO | Perf report for the last 5.13955 seconds: | 20260506 01:16:46.743 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:16:46.743 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:46.743 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:46.743 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:46.743 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260506 01:16:46.743 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:46.743 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:46.743 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 01:16:46.743 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:46.743 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:46.744 | INFO | Uploaded 36GiB to GPU in 2.06808s (17.4075GiB/s) | 20260506 01:16:46.744 | INFO | Downloaded 34GiB to GPU in 2.07893s (16.3546GiB/s) | 20260506 01:16:51.892 | INFO | Analyzing chunk frames = [7802368,7802880) | 20260506 01:16:51.892 | INFO | MemInfo: { | 20260506 01:16:51.892 | INFO | "gpu" : " 17 GB", | 20260506 01:16:51.892 | INFO | "huge" : "", | 20260506 01:16:51.892 | INFO | "huge_shared" : " 27 GB", | 20260506 01:16:51.892 | INFO | "malloc" : " 14 GB", | 20260506 01:16:51.892 | INFO | "pinned" : " 29 GB" | 20260506 01:16:51.892 | INFO | } | 20260506 01:16:51.892 | INFO | Perf report for the last 5.14869 seconds: | 20260506 01:16:51.892 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:16:51.892 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:51.892 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:51.892 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:51.892 | INFO | ANALYSIS : 61.8%, 3.0, 96.6ms, 0 | 20260506 01:16:51.892 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:51.892 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:51.892 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:16:51.892 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:51.892 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:51.892 | INFO | Uploaded 36GiB to GPU in 2.12783s (16.9186GiB/s) | 20260506 01:16:51.892 | INFO | Downloaded 34GiB to GPU in 2.02631s (16.7792GiB/s) | 20260506 01:16:52.018 | INFO | Starting a round of Baz IO | 20260506 01:16:55.146 | INFO | Populating RTMetrics report | 20260506 01:16:55.147 | INFO | RTMetrics report populated | 20260506 01:16:55.147 | INFO | Opening tmp RTMetrics json file | 20260506 01:16:55.147 | INFO | Serializing RTMetrics to json | 20260506 01:16:55.149 | INFO | Serialization done, swapping tmp file now... | 20260506 01:16:55.149 | INFO | RTMetrics done | 20260506 01:16:55.223 | INFO | Finished a round of Baz IO: Wrote 1.832 GiB in 3.205 seconds | 20260506 01:16:55.223 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.32% / 0.00% / 78.67% | 20260506 01:16:55.223 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0830332% / 0% / 7.15256e-05% | 20260506 01:16:55.223 | INFO | Baz IO overall: | 20260506 01:16:55.223 | INFO | BytesWritten : 4407616081723 | 20260506 01:16:55.223 | INFO | NumEvents : 2072146123035 | 20260506 01:16:57.024 | INFO | Analyzing chunk frames = [7802880,7803392) | 20260506 01:16:57.024 | INFO | MemInfo: { | 20260506 01:16:57.024 | INFO | "gpu" : " 17 GB", | 20260506 01:16:57.024 | INFO | "huge" : "", | 20260506 01:16:57.024 | INFO | "huge_shared" : " 27 GB", | 20260506 01:16:57.024 | INFO | "malloc" : " 14 GB", | 20260506 01:16:57.024 | INFO | "pinned" : " 29 GB" | 20260506 01:16:57.024 | INFO | } | 20260506 01:16:57.024 | INFO | Perf report for the last 5.13223 seconds: | 20260506 01:16:57.024 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:16:57.024 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:57.024 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:57.024 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:57.024 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260506 01:16:57.024 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:16:57.024 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260506 01:16:57.024 | INFO | PRE_HQ : 59.3%, 1.7, 53.5ms, 0 | 20260506 01:16:57.024 | INFO | BAZWRITER : 61.7%, 3.5, 110.5ms, 0 | 20260506 01:16:57.024 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:16:57.025 | INFO | Uploaded 36GiB to GPU in 2.14314s (16.7978GiB/s) | 20260506 01:16:57.025 | INFO | Downloaded 38GiB to GPU in 2.23107s (17.0322GiB/s) | 20260506 01:17:02.163 | INFO | Analyzing chunk frames = [7803392,7803904) | 20260506 01:17:02.163 | INFO | MemInfo: { | 20260506 01:17:02.163 | INFO | "gpu" : " 17 GB", | 20260506 01:17:02.163 | INFO | "huge" : "", | 20260506 01:17:02.163 | INFO | "huge_shared" : " 27 GB", | 20260506 01:17:02.163 | INFO | "malloc" : " 14 GB", | 20260506 01:17:02.163 | INFO | "pinned" : " 29 GB" | 20260506 01:17:02.163 | INFO | } | 20260506 01:17:02.164 | INFO | Perf report for the last 5.13924 seconds: | 20260506 01:17:02.164 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:17:02.164 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:02.164 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:02.164 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:02.164 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 01:17:02.164 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:02.164 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:02.164 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 01:17:02.164 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:02.164 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:02.164 | INFO | Uploaded 36GiB to GPU in 2.11226s (17.0434GiB/s) | 20260506 01:17:02.164 | INFO | Downloaded 34GiB to GPU in 2.03926s (16.6727GiB/s) | 20260506 01:17:07.302 | INFO | Analyzing chunk frames = [7803904,7804416) | 20260506 01:17:07.302 | INFO | MemInfo: { | 20260506 01:17:07.302 | INFO | "gpu" : " 17 GB", | 20260506 01:17:07.302 | INFO | "huge" : "", | 20260506 01:17:07.302 | INFO | "huge_shared" : " 27 GB", | 20260506 01:17:07.302 | INFO | "malloc" : " 14 GB", | 20260506 01:17:07.302 | INFO | "pinned" : " 29 GB" | 20260506 01:17:07.302 | INFO | } | 20260506 01:17:07.302 | INFO | Perf report for the last 5.13834 seconds: | 20260506 01:17:07.302 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:17:07.302 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:07.302 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:07.302 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:07.302 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 01:17:07.302 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:07.302 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:07.302 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 01:17:07.302 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:07.302 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:07.302 | INFO | Uploaded 36GiB to GPU in 2.10053s (17.1385GiB/s) | 20260506 01:17:07.302 | INFO | Downloaded 34GiB to GPU in 2.04314s (16.641GiB/s) | 20260506 01:17:12.438 | INFO | Analyzing chunk frames = [7804416,7804928) | 20260506 01:17:12.438 | INFO | MemInfo: { | 20260506 01:17:12.438 | INFO | "gpu" : " 17 GB", | 20260506 01:17:12.438 | INFO | "huge" : "", | 20260506 01:17:12.438 | INFO | "huge_shared" : " 27 GB", | 20260506 01:17:12.438 | INFO | "malloc" : " 14 GB", | 20260506 01:17:12.438 | INFO | "pinned" : " 29 GB" | 20260506 01:17:12.438 | INFO | } | 20260506 01:17:12.439 | INFO | Perf report for the last 5.1367 seconds: | 20260506 01:17:12.439 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:17:12.439 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:12.439 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:12.439 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:12.439 | INFO | ANALYSIS : 61.6%, 3.0, 95.6ms, 0 | 20260506 01:17:12.439 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:12.439 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:12.439 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:17:12.439 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:12.439 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:12.439 | INFO | Uploaded 36GiB to GPU in 2.14584s (16.7767GiB/s) | 20260506 01:17:12.439 | INFO | Downloaded 34GiB to GPU in 2.05151s (16.5732GiB/s) | 20260506 01:17:17.579 | INFO | Analyzing chunk frames = [7804928,7805440) | 20260506 01:17:17.579 | INFO | MemInfo: { | 20260506 01:17:17.579 | INFO | "gpu" : " 17 GB", | 20260506 01:17:17.579 | INFO | "huge" : "", | 20260506 01:17:17.579 | INFO | "huge_shared" : " 27 GB", | 20260506 01:17:17.580 | INFO | "malloc" : " 14 GB", | 20260506 01:17:17.580 | INFO | "pinned" : " 29 GB" | 20260506 01:17:17.580 | INFO | } | 20260506 01:17:17.580 | INFO | Perf report for the last 5.14103 seconds: | 20260506 01:17:17.580 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:17:17.580 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:17.580 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:17.580 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:17.580 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 01:17:17.580 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:17.580 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:17.580 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 01:17:17.580 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:17.580 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:17.580 | INFO | Uploaded 36GiB to GPU in 2.09354s (17.1958GiB/s) | 20260506 01:17:17.580 | INFO | Downloaded 34GiB to GPU in 2.05237s (16.5662GiB/s) | 20260506 01:17:22.716 | INFO | Analyzing chunk frames = [7805440,7805952) | 20260506 01:17:22.716 | INFO | MemInfo: { | 20260506 01:17:22.716 | INFO | "gpu" : " 17 GB", | 20260506 01:17:22.716 | INFO | "huge" : "", | 20260506 01:17:22.716 | INFO | "huge_shared" : " 27 GB", | 20260506 01:17:22.716 | INFO | "malloc" : " 14 GB", | 20260506 01:17:22.716 | INFO | "pinned" : " 29 GB" | 20260506 01:17:22.716 | INFO | } | 20260506 01:17:22.716 | INFO | Perf report for the last 5.13618 seconds: | 20260506 01:17:22.716 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:17:22.716 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:22.716 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:22.716 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:22.716 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260506 01:17:22.716 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:22.716 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:22.716 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 01:17:22.716 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:22.716 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:22.716 | INFO | Uploaded 36GiB to GPU in 2.06969s (17.3939GiB/s) | 20260506 01:17:22.716 | INFO | Downloaded 34GiB to GPU in 2.04322s (16.6404GiB/s) | 20260506 01:17:27.860 | INFO | Analyzing chunk frames = [7805952,7806464) | 20260506 01:17:27.860 | INFO | MemInfo: { | 20260506 01:17:27.860 | INFO | "gpu" : " 17 GB", | 20260506 01:17:27.860 | INFO | "huge" : "", | 20260506 01:17:27.860 | INFO | "huge_shared" : " 27 GB", | 20260506 01:17:27.860 | INFO | "malloc" : " 14 GB", | 20260506 01:17:27.860 | INFO | "pinned" : " 29 GB" | 20260506 01:17:27.860 | INFO | } | 20260506 01:17:27.860 | INFO | Perf report for the last 5.14467 seconds: | 20260506 01:17:27.860 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:17:27.860 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:27.860 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:27.860 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:27.861 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260506 01:17:27.861 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:27.861 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:27.861 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 01:17:27.861 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:27.861 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:27.861 | INFO | Uploaded 36GiB to GPU in 2.06243s (17.4551GiB/s) | 20260506 01:17:27.861 | INFO | Downloaded 34GiB to GPU in 2.06949s (16.4291GiB/s) | 20260506 01:17:33.005 | INFO | Analyzing chunk frames = [7806464,7806976) | 20260506 01:17:33.005 | INFO | MemInfo: { | 20260506 01:17:33.005 | INFO | "gpu" : " 17 GB", | 20260506 01:17:33.005 | INFO | "huge" : "", | 20260506 01:17:33.005 | INFO | "huge_shared" : " 27 GB", | 20260506 01:17:33.005 | INFO | "malloc" : " 14 GB", | 20260506 01:17:33.005 | INFO | "pinned" : " 29 GB" | 20260506 01:17:33.005 | INFO | } | 20260506 01:17:33.005 | INFO | Perf report for the last 5.14463 seconds: | 20260506 01:17:33.005 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:17:33.005 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:33.005 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:33.005 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:33.005 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 01:17:33.005 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:33.005 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:33.005 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 01:17:33.005 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:33.005 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:33.005 | INFO | Uploaded 36GiB to GPU in 2.12414s (16.948GiB/s) | 20260506 01:17:33.005 | INFO | Downloaded 34GiB to GPU in 2.0272s (16.7719GiB/s) | 20260506 01:17:33.005 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 01:17:33.005 | INFO | Total mem usage: 15.3 GiB | 20260506 01:17:33.005 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 01:17:33.005 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.17%) | 20260506 01:17:33.005 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 01:17:33.005 | INFO | High Water Breakdown: | 20260506 01:17:33.005 | INFO | 60.9% : 8.55 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 01:17:33.005 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 01:17:33.005 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 01:17:33.005 | INFO | ------------------------------------------ | 20260506 01:17:33.006 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:17:33.006 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 01:17:33.006 | INFO | Total mem usage: 26.5 GiB | 20260506 01:17:33.006 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 01:17:33.006 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 01:17:33.006 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 01:17:33.006 | INFO | High Water Breakdown: | 20260506 01:17:33.006 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:17:33.006 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:17:33.006 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:17:33.006 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:17:33.006 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:17:33.006 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:17:33.006 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 01:17:33.006 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 01:17:33.006 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:17:33.006 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 01:17:33.006 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 01:17:33.006 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 01:17:33.006 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 01:17:33.006 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 01:17:33.006 | INFO | ------------------------------------------ | 20260506 01:17:33.006 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:17:33.006 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 01:17:33.006 | INFO | Total mem usage: 25.6 GiB | 20260506 01:17:33.006 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 01:17:33.006 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 01:17:33.006 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 01:17:33.006 | INFO | High Water Breakdown: | 20260506 01:17:33.006 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 01:17:33.006 | INFO | ------------------------------------------ | 20260506 01:17:33.006 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:17:33.006 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 01:17:33.006 | INFO | Total mem usage: 15.7 GiB | 20260506 01:17:33.006 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 01:17:33.006 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 01:17:33.006 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 01:17:33.006 | INFO | High Water Breakdown: | 20260506 01:17:33.006 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 01:17:33.006 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 01:17:33.006 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 01:17:33.006 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 01:17:33.006 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 01:17:33.006 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 01:17:33.006 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 01:17:33.006 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 01:17:33.006 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 01:17:33.006 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:17:33.006 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 01:17:33.006 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:17:33.006 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 01:17:33.006 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 01:17:33.006 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 01:17:33.006 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 01:17:33.006 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:17:33.006 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 01:17:33.006 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:17:33.006 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:17:33.006 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:17:33.006 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 01:17:33.006 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 01:17:33.006 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:17:33.006 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 01:17:33.006 | INFO | ------------------------------------------ | 20260506 01:17:33.006 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:17:33.006 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 01:17:33.006 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:17:33.006 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:17:33.006 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.37e+01, 1.33e+01 / 7.34e-01 | 20260506 01:17:33.006 | INFO | PerfReport: 0.29%, AggregateStats -- 9.18e-02 / 1.03e-01, 1.05e-02 / 3.47e-03 | 20260506 01:17:33.006 | INFO | PerfReport: 11.79%, Baseline -- 3.68e+00 / 4.45e+00, 5.54e-01 / 2.38e-01 | 20260506 01:17:33.006 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.49e+00, 1.85e-01 / 9.89e-02 | 20260506 01:17:33.006 | INFO | PerfReport: 16.02%, DME -- 4.00e+01 / 4.38e+01, 3.71e+00 / 9.77e-01 | 20260506 01:17:33.006 | INFO | PerfReport: 57.41%, FrameLabeling -- 1.79e+01 / 1.95e+01, 1.44e+00 / 9.70e-01 | 20260506 01:17:33.006 | INFO | PerfReport: 7.10%, PulseAccumulating -- 2.22e+00 / 2.73e+00, 4.21e-01 / 2.18e-01 | 20260506 01:17:33.006 | INFO | PerfReport: 3.06%, Metrics -- 9.55e-01 / 1.23e+00, 2.61e-01 / 1.68e-01 | 20260506 01:17:33.006 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.30e-01 / 3.90e-01, 5.62e-02 / 3.72e-02 | 20260506 01:17:33.006 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 01:17:33.006 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:17:33.006 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:17:33.006 | INFO | PerfReport: Full Execution -- 9.82e+01 / -nan, 2.43e+01 / 0.00e+00 | 20260506 01:17:33.006 | INFO | PerfReport: 22.39%, Upload -- 2.20e+01 / 0.00e+00, 7.56e+00 / 0.00e+00 | 20260506 01:17:33.006 | INFO | PerfReport: 21.96%, Download -- 2.16e+01 / 0.00e+00, 7.67e+00 / 0.00e+00 | 20260506 01:17:33.006 | INFO | PerfReport: 31.81%, Compute -- 3.12e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 01:17:33.006 | INFO | PerfReport: 2.06%, LockUpload -- 2.02e+00 / 0.00e+00, 7.80e+00 / 0.00e+00 | 20260506 01:17:33.006 | INFO | PerfReport: 1.36%, LockDownload -- 1.34e+00 / 0.00e+00, 2.93e+00 / 0.00e+00 | 20260506 01:17:33.006 | INFO | PerfReport: 20.40%, LockCompute -- 2.00e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 01:17:33.006 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.07e-02 / 0.00e+00 | 20260506 01:17:33.131 | INFO | Starting a round of Baz IO | 20260506 01:17:36.264 | INFO | Populating RTMetrics report | 20260506 01:17:36.264 | INFO | RTMetrics report populated | 20260506 01:17:36.264 | INFO | Opening tmp RTMetrics json file | 20260506 01:17:36.265 | INFO | Serializing RTMetrics to json | 20260506 01:17:36.266 | INFO | Serialization done, swapping tmp file now... | 20260506 01:17:36.267 | INFO | RTMetrics done | 20260506 01:17:36.341 | INFO | Finished a round of Baz IO: Wrote 1.83 GiB in 3.209 seconds | 20260506 01:17:36.341 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.32% / 0.00% / 78.68% | 20260506 01:17:36.341 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0820955% / 0% / 5.5631e-05% | 20260506 01:17:36.341 | INFO | Baz IO overall: | 20260506 01:17:36.341 | INFO | BytesWritten : 4409580602601 | 20260506 01:17:36.341 | INFO | NumEvents : 2073041559249 | 20260506 01:17:38.134 | INFO | Analyzing chunk frames = [7806976,7807488) | 20260506 01:17:38.134 | INFO | MemInfo: { | 20260506 01:17:38.134 | INFO | "gpu" : " 17 GB", | 20260506 01:17:38.134 | INFO | "huge" : "", | 20260506 01:17:38.134 | INFO | "huge_shared" : " 27 GB", | 20260506 01:17:38.134 | INFO | "malloc" : " 14 GB", | 20260506 01:17:38.134 | INFO | "pinned" : " 29 GB" | 20260506 01:17:38.134 | INFO | } | 20260506 01:17:38.134 | INFO | Perf report for the last 5.12854 seconds: | 20260506 01:17:38.134 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:17:38.134 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:38.134 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:38.134 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:38.134 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260506 01:17:38.134 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:38.134 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260506 01:17:38.134 | INFO | PRE_HQ : 59.6%, 1.7, 53.7ms, 0 | 20260506 01:17:38.134 | INFO | BAZWRITER : 61.9%, 3.5, 109.5ms, 0 | 20260506 01:17:38.134 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:17:38.134 | INFO | Uploaded 36GiB to GPU in 2.14069s (16.817GiB/s) | 20260506 01:17:38.134 | INFO | Downloaded 38GiB to GPU in 2.22732s (17.0608GiB/s) | 20260506 01:17:43.269 | INFO | Analyzing chunk frames = [7807488,7808000) | 20260506 01:17:43.269 | INFO | MemInfo: { | 20260506 01:17:43.269 | INFO | "gpu" : " 17 GB", | 20260506 01:17:43.269 | INFO | "huge" : "", | 20260506 01:17:43.269 | INFO | "huge_shared" : " 27 GB", | 20260506 01:17:43.269 | INFO | "malloc" : " 14 GB", | 20260506 01:17:43.269 | INFO | "pinned" : " 29 GB" | 20260506 01:17:43.269 | INFO | } | 20260506 01:17:43.269 | INFO | Perf report for the last 5.13523 seconds: | 20260506 01:17:43.269 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:17:43.269 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:43.269 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:43.269 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:43.269 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 01:17:43.269 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:43.269 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:43.269 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 01:17:43.269 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:43.269 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:43.269 | INFO | Uploaded 36GiB to GPU in 2.11417s (17.0279GiB/s) | 20260506 01:17:43.269 | INFO | Downloaded 34GiB to GPU in 2.04087s (16.6596GiB/s) | 20260506 01:17:48.414 | INFO | Analyzing chunk frames = [7808000,7808512) | 20260506 01:17:48.414 | INFO | MemInfo: { | 20260506 01:17:48.414 | INFO | "gpu" : " 17 GB", | 20260506 01:17:48.414 | INFO | "huge" : "", | 20260506 01:17:48.414 | INFO | "huge_shared" : " 27 GB", | 20260506 01:17:48.414 | INFO | "malloc" : " 14 GB", | 20260506 01:17:48.414 | INFO | "pinned" : " 29 GB" | 20260506 01:17:48.414 | INFO | } | 20260506 01:17:48.414 | INFO | Perf report for the last 5.14536 seconds: | 20260506 01:17:48.414 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:17:48.414 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:48.414 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:48.414 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:48.414 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 01:17:48.414 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:48.414 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:48.414 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:17:48.414 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:48.414 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:48.415 | INFO | Uploaded 36GiB to GPU in 2.15141s (16.7332GiB/s) | 20260506 01:17:48.415 | INFO | Downloaded 34GiB to GPU in 2.05886s (16.514GiB/s) | 20260506 01:17:53.556 | INFO | Analyzing chunk frames = [7808512,7809024) | 20260506 01:17:53.556 | INFO | MemInfo: { | 20260506 01:17:53.556 | INFO | "gpu" : " 17 GB", | 20260506 01:17:53.556 | INFO | "huge" : "", | 20260506 01:17:53.556 | INFO | "huge_shared" : " 27 GB", | 20260506 01:17:53.556 | INFO | "malloc" : " 14 GB", | 20260506 01:17:53.556 | INFO | "pinned" : " 29 GB" | 20260506 01:17:53.556 | INFO | } | 20260506 01:17:53.556 | INFO | Perf report for the last 5.14212 seconds: | 20260506 01:17:53.556 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:17:53.556 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:53.556 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:53.556 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:53.556 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260506 01:17:53.556 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:53.557 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:53.557 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:17:53.557 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:53.557 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:53.557 | INFO | Uploaded 36GiB to GPU in 2.15143s (16.7331GiB/s) | 20260506 01:17:53.557 | INFO | Downloaded 34GiB to GPU in 2.04905s (16.5931GiB/s) | 20260506 01:17:58.697 | INFO | Analyzing chunk frames = [7809024,7809536) | 20260506 01:17:58.697 | INFO | MemInfo: { | 20260506 01:17:58.697 | INFO | "gpu" : " 17 GB", | 20260506 01:17:58.697 | INFO | "huge" : "", | 20260506 01:17:58.697 | INFO | "huge_shared" : " 27 GB", | 20260506 01:17:58.697 | INFO | "malloc" : " 14 GB", | 20260506 01:17:58.697 | INFO | "pinned" : " 29 GB" | 20260506 01:17:58.697 | INFO | } | 20260506 01:17:58.697 | INFO | Perf report for the last 5.14026 seconds: | 20260506 01:17:58.697 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:17:58.697 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:58.697 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:58.697 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:58.697 | INFO | ANALYSIS : 60.8%, 3.0, 94.9ms, 0 | 20260506 01:17:58.697 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:58.697 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:58.697 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:17:58.697 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:58.697 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:17:58.697 | INFO | Uploaded 36GiB to GPU in 2.09051s (17.2207GiB/s) | 20260506 01:17:58.697 | INFO | Downloaded 34GiB to GPU in 2.03677s (16.6931GiB/s) | 20260506 01:18:03.839 | INFO | Analyzing chunk frames = [7809536,7810048) | 20260506 01:18:03.839 | INFO | MemInfo: { | 20260506 01:18:03.839 | INFO | "gpu" : " 17 GB", | 20260506 01:18:03.839 | INFO | "huge" : "", | 20260506 01:18:03.839 | INFO | "huge_shared" : " 27 GB", | 20260506 01:18:03.839 | INFO | "malloc" : " 14 GB", | 20260506 01:18:03.839 | INFO | "pinned" : " 29 GB" | 20260506 01:18:03.839 | INFO | } | 20260506 01:18:03.840 | INFO | Perf report for the last 5.14283 seconds: | 20260506 01:18:03.840 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:18:03.840 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:03.840 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:03.840 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:03.840 | INFO | ANALYSIS : 61.1%, 3.0, 95.5ms, 0 | 20260506 01:18:03.840 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:03.840 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:03.840 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 01:18:03.840 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:03.840 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:03.840 | INFO | Uploaded 36GiB to GPU in 2.07579s (17.3428GiB/s) | 20260506 01:18:03.840 | INFO | Downloaded 34GiB to GPU in 2.04426s (16.632GiB/s) | 20260506 01:18:08.972 | INFO | Analyzing chunk frames = [7810048,7810560) | 20260506 01:18:08.972 | INFO | MemInfo: { | 20260506 01:18:08.972 | INFO | "gpu" : " 17 GB", | 20260506 01:18:08.972 | INFO | "huge" : "", | 20260506 01:18:08.972 | INFO | "huge_shared" : " 27 GB", | 20260506 01:18:08.972 | INFO | "malloc" : " 14 GB", | 20260506 01:18:08.972 | INFO | "pinned" : " 29 GB" | 20260506 01:18:08.972 | INFO | } | 20260506 01:18:08.972 | INFO | Perf report for the last 5.13294 seconds: | 20260506 01:18:08.972 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:18:08.972 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:08.972 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:08.972 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:08.972 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 01:18:08.973 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:08.973 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:08.973 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 01:18:08.973 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:08.973 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:08.973 | INFO | Uploaded 36GiB to GPU in 2.09487s (17.1848GiB/s) | 20260506 01:18:08.973 | INFO | Downloaded 34GiB to GPU in 2.06704s (16.4486GiB/s) | 20260506 01:18:14.111 | INFO | Analyzing chunk frames = [7810560,7811072) | 20260506 01:18:14.111 | INFO | MemInfo: { | 20260506 01:18:14.111 | INFO | "gpu" : " 17 GB", | 20260506 01:18:14.111 | INFO | "huge" : "", | 20260506 01:18:14.111 | INFO | "huge_shared" : " 27 GB", | 20260506 01:18:14.111 | INFO | "malloc" : " 14 GB", | 20260506 01:18:14.111 | INFO | "pinned" : " 29 GB" | 20260506 01:18:14.111 | INFO | } | 20260506 01:18:14.111 | INFO | Perf report for the last 5.13895 seconds: | 20260506 01:18:14.111 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:18:14.111 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:14.111 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:14.111 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:14.111 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260506 01:18:14.112 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:14.112 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:14.112 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:18:14.112 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:14.112 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:14.112 | INFO | Uploaded 36GiB to GPU in 2.13114s (16.8924GiB/s) | 20260506 01:18:14.112 | INFO | Downloaded 34GiB to GPU in 2.02796s (16.7656GiB/s) | 20260506 01:18:14.235 | INFO | Starting a round of Baz IO | 20260506 01:18:17.375 | INFO | Populating RTMetrics report | 20260506 01:18:17.375 | INFO | RTMetrics report populated | 20260506 01:18:17.375 | INFO | Opening tmp RTMetrics json file | 20260506 01:18:17.377 | INFO | Serializing RTMetrics to json | 20260506 01:18:17.378 | INFO | Serialization done, swapping tmp file now... | 20260506 01:18:17.378 | INFO | RTMetrics done | 20260506 01:18:17.449 | INFO | Finished a round of Baz IO: Wrote 1.828 GiB in 3.214 seconds | 20260506 01:18:17.449 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.32% / 0.00% / 78.68% | 20260506 01:18:17.449 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.081416% / 0% / 3.97364e-05% | 20260506 01:18:17.449 | INFO | Baz IO overall: | 20260506 01:18:17.449 | INFO | BytesWritten : 4411543234443 | 20260506 01:18:17.449 | INFO | NumEvents : 2073935956391 | 20260506 01:18:19.258 | INFO | Analyzing chunk frames = [7811072,7811584) | 20260506 01:18:19.258 | INFO | MemInfo: { | 20260506 01:18:19.258 | INFO | "gpu" : " 17 GB", | 20260506 01:18:19.258 | INFO | "huge" : "", | 20260506 01:18:19.258 | INFO | "huge_shared" : " 27 GB", | 20260506 01:18:19.258 | INFO | "malloc" : " 14 GB", | 20260506 01:18:19.258 | INFO | "pinned" : " 29 GB" | 20260506 01:18:19.258 | INFO | } | 20260506 01:18:19.258 | INFO | Perf report for the last 5.14642 seconds: | 20260506 01:18:19.258 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:18:19.258 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:19.258 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:19.258 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:19.258 | INFO | ANALYSIS : 62.4%, 3.0, 97.6ms, 0 | 20260506 01:18:19.258 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:19.258 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260506 01:18:19.258 | INFO | PRE_HQ : 59.6%, 1.7, 53.7ms, 0 | 20260506 01:18:19.258 | INFO | BAZWRITER : 61.5%, 3.5, 110.0ms, 0 | 20260506 01:18:19.258 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:18:19.258 | INFO | Uploaded 36GiB to GPU in 2.18446s (16.4801GiB/s) | 20260506 01:18:19.258 | INFO | Downloaded 38GiB to GPU in 2.24195s (16.9495GiB/s) | 20260506 01:18:24.392 | INFO | Analyzing chunk frames = [7811584,7812096) | 20260506 01:18:24.392 | INFO | MemInfo: { | 20260506 01:18:24.392 | INFO | "gpu" : " 17 GB", | 20260506 01:18:24.392 | INFO | "huge" : "", | 20260506 01:18:24.392 | INFO | "huge_shared" : " 27 GB", | 20260506 01:18:24.392 | INFO | "malloc" : " 14 GB", | 20260506 01:18:24.392 | INFO | "pinned" : " 29 GB" | 20260506 01:18:24.392 | INFO | } | 20260506 01:18:24.392 | INFO | Perf report for the last 5.13407 seconds: | 20260506 01:18:24.392 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:18:24.392 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:24.392 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:24.392 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:24.392 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260506 01:18:24.392 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:24.392 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:24.392 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 01:18:24.392 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:24.392 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:24.392 | INFO | Uploaded 36GiB to GPU in 1.95489s (18.4154GiB/s) | 20260506 01:18:24.392 | INFO | Downloaded 34GiB to GPU in 1.98286s (17.147GiB/s) | 20260506 01:18:29.528 | INFO | Analyzing chunk frames = [7812096,7812608) | 20260506 01:18:29.528 | INFO | MemInfo: { | 20260506 01:18:29.528 | INFO | "gpu" : " 17 GB", | 20260506 01:18:29.528 | INFO | "huge" : "", | 20260506 01:18:29.528 | INFO | "huge_shared" : " 27 GB", | 20260506 01:18:29.529 | INFO | "malloc" : " 14 GB", | 20260506 01:18:29.529 | INFO | "pinned" : " 29 GB" | 20260506 01:18:29.529 | INFO | } | 20260506 01:18:29.529 | INFO | Perf report for the last 5.13665 seconds: | 20260506 01:18:29.529 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:18:29.529 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:29.529 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:29.529 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:29.529 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 01:18:29.529 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:29.529 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:29.529 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:18:29.529 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:29.529 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:29.529 | INFO | Uploaded 36GiB to GPU in 2.07738s (17.3295GiB/s) | 20260506 01:18:29.529 | INFO | Downloaded 34GiB to GPU in 2.01844s (16.8447GiB/s) | 20260506 01:18:34.669 | INFO | Analyzing chunk frames = [7812608,7813120) | 20260506 01:18:34.669 | INFO | MemInfo: { | 20260506 01:18:34.669 | INFO | "gpu" : " 17 GB", | 20260506 01:18:34.669 | INFO | "huge" : "", | 20260506 01:18:34.670 | INFO | "huge_shared" : " 27 GB", | 20260506 01:18:34.670 | INFO | "malloc" : " 14 GB", | 20260506 01:18:34.670 | INFO | "pinned" : " 29 GB" | 20260506 01:18:34.670 | INFO | } | 20260506 01:18:34.670 | INFO | Perf report for the last 5.14096 seconds: | 20260506 01:18:34.670 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:18:34.670 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:34.670 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:34.670 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:34.670 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260506 01:18:34.670 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:34.670 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:34.670 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 01:18:34.670 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:34.670 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:34.670 | INFO | Uploaded 36GiB to GPU in 2.13464s (16.8647GiB/s) | 20260506 01:18:34.670 | INFO | Downloaded 34GiB to GPU in 2.01968s (16.8343GiB/s) | 20260506 01:18:39.811 | INFO | Analyzing chunk frames = [7813120,7813632) | 20260506 01:18:39.811 | INFO | MemInfo: { | 20260506 01:18:39.811 | INFO | "gpu" : " 17 GB", | 20260506 01:18:39.811 | INFO | "huge" : "", | 20260506 01:18:39.811 | INFO | "huge_shared" : " 27 GB", | 20260506 01:18:39.811 | INFO | "malloc" : " 14 GB", | 20260506 01:18:39.811 | INFO | "pinned" : " 29 GB" | 20260506 01:18:39.811 | INFO | } | 20260506 01:18:39.811 | INFO | Perf report for the last 5.14124 seconds: | 20260506 01:18:39.811 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:18:39.811 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:39.811 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:39.811 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:39.811 | INFO | ANALYSIS : 60.7%, 3.0, 94.8ms, 0 | 20260506 01:18:39.811 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:39.811 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:39.811 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 01:18:39.811 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:39.811 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:39.811 | INFO | Uploaded 36GiB to GPU in 2.07111s (17.382GiB/s) | 20260506 01:18:39.811 | INFO | Downloaded 34GiB to GPU in 2.03247s (16.7284GiB/s) | 20260506 01:18:44.947 | INFO | Analyzing chunk frames = [7813632,7814144) | 20260506 01:18:44.947 | INFO | MemInfo: { | 20260506 01:18:44.947 | INFO | "gpu" : " 17 GB", | 20260506 01:18:44.947 | INFO | "huge" : "", | 20260506 01:18:44.947 | INFO | "huge_shared" : " 27 GB", | 20260506 01:18:44.947 | INFO | "malloc" : " 14 GB", | 20260506 01:18:44.947 | INFO | "pinned" : " 29 GB" | 20260506 01:18:44.947 | INFO | } | 20260506 01:18:44.947 | INFO | Perf report for the last 5.13588 seconds: | 20260506 01:18:44.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:18:44.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:44.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:44.947 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:44.947 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 01:18:44.947 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:44.947 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:44.947 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 01:18:44.947 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:44.947 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:44.947 | INFO | Uploaded 36GiB to GPU in 2.07561s (17.3443GiB/s) | 20260506 01:18:44.947 | INFO | Downloaded 34GiB to GPU in 2.03714s (16.6901GiB/s) | 20260506 01:18:50.084 | INFO | Analyzing chunk frames = [7814144,7814656) | 20260506 01:18:50.084 | INFO | MemInfo: { | 20260506 01:18:50.084 | INFO | "gpu" : " 17 GB", | 20260506 01:18:50.084 | INFO | "huge" : "", | 20260506 01:18:50.084 | INFO | "huge_shared" : " 27 GB", | 20260506 01:18:50.084 | INFO | "malloc" : " 14 GB", | 20260506 01:18:50.084 | INFO | "pinned" : " 29 GB" | 20260506 01:18:50.084 | INFO | } | 20260506 01:18:50.084 | INFO | Perf report for the last 5.13741 seconds: | 20260506 01:18:50.084 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:18:50.084 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:50.084 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:50.084 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:50.084 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260506 01:18:50.084 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:50.084 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:50.084 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 01:18:50.084 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:50.084 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:50.084 | INFO | Uploaded 36GiB to GPU in 2.09335s (17.1973GiB/s) | 20260506 01:18:50.084 | INFO | Downloaded 34GiB to GPU in 2.0856s (16.3023GiB/s) | 20260506 01:18:55.224 | INFO | Analyzing chunk frames = [7814656,7815168) | 20260506 01:18:55.224 | INFO | MemInfo: { | 20260506 01:18:55.224 | INFO | "gpu" : " 17 GB", | 20260506 01:18:55.224 | INFO | "huge" : "", | 20260506 01:18:55.224 | INFO | "huge_shared" : " 27 GB", | 20260506 01:18:55.224 | INFO | "malloc" : " 14 GB", | 20260506 01:18:55.224 | INFO | "pinned" : " 29 GB" | 20260506 01:18:55.224 | INFO | } | 20260506 01:18:55.224 | INFO | Perf report for the last 5.13999 seconds: | 20260506 01:18:55.224 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:18:55.224 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:55.224 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:55.224 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:55.224 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 01:18:55.224 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:55.224 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:55.224 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:18:55.224 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:55.224 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:18:55.224 | INFO | Uploaded 36GiB to GPU in 2.10757s (17.0813GiB/s) | 20260506 01:18:55.224 | INFO | Downloaded 34GiB to GPU in 2.01018s (16.9139GiB/s) | 20260506 01:18:55.224 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 01:18:55.224 | INFO | Total mem usage: 15.3 GiB | 20260506 01:18:55.225 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 01:18:55.225 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.17%) | 20260506 01:18:55.225 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 01:18:55.225 | INFO | High Water Breakdown: | 20260506 01:18:55.225 | INFO | 60.9% : 8.55 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 01:18:55.225 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 01:18:55.225 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 01:18:55.225 | INFO | ------------------------------------------ | 20260506 01:18:55.225 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:18:55.225 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 01:18:55.225 | INFO | Total mem usage: 26.5 GiB | 20260506 01:18:55.225 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 01:18:55.225 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 01:18:55.225 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 01:18:55.225 | INFO | High Water Breakdown: | 20260506 01:18:55.225 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:18:55.225 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:18:55.225 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:18:55.225 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:18:55.225 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:18:55.225 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:18:55.225 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 01:18:55.225 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 01:18:55.225 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:18:55.225 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 01:18:55.225 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 01:18:55.225 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 01:18:55.225 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 01:18:55.225 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 01:18:55.225 | INFO | ------------------------------------------ | 20260506 01:18:55.225 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:18:55.225 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 01:18:55.225 | INFO | Total mem usage: 25.6 GiB | 20260506 01:18:55.225 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 01:18:55.225 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 01:18:55.225 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 01:18:55.225 | INFO | High Water Breakdown: | 20260506 01:18:55.225 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 01:18:55.225 | INFO | ------------------------------------------ | 20260506 01:18:55.225 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:18:55.225 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 01:18:55.225 | INFO | Total mem usage: 15.7 GiB | 20260506 01:18:55.225 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 01:18:55.225 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 01:18:55.225 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 01:18:55.225 | INFO | High Water Breakdown: | 20260506 01:18:55.225 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 01:18:55.225 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 01:18:55.225 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 01:18:55.225 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 01:18:55.225 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 01:18:55.225 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 01:18:55.225 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 01:18:55.225 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 01:18:55.225 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 01:18:55.225 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:18:55.225 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 01:18:55.225 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:18:55.225 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 01:18:55.225 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 01:18:55.225 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 01:18:55.225 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 01:18:55.225 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:18:55.225 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 01:18:55.225 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:18:55.225 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:18:55.225 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:18:55.225 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 01:18:55.225 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 01:18:55.225 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:18:55.225 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 01:18:55.225 | INFO | ------------------------------------------ | 20260506 01:18:55.225 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:18:55.225 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 01:18:55.225 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:18:55.225 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:18:55.225 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.36e+01, 1.35e+01 / 5.51e-01 | 20260506 01:18:55.225 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.05e-01, 1.07e-02 / 5.23e-03 | 20260506 01:18:55.225 | INFO | PerfReport: 11.81%, Baseline -- 3.69e+00 / 4.47e+00, 5.62e-01 / 2.59e-01 | 20260506 01:18:55.225 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.51e+00, 1.88e-01 / 9.86e-02 | 20260506 01:18:55.225 | INFO | PerfReport: 15.97%, DME -- 3.99e+01 / 4.36e+01, 3.72e+00 / 6.85e-01 | 20260506 01:18:55.225 | INFO | PerfReport: 57.48%, FrameLabeling -- 1.80e+01 / 1.96e+01, 1.54e+00 / 1.07e+00 | 20260506 01:18:55.225 | INFO | PerfReport: 7.07%, PulseAccumulating -- 2.21e+00 / 2.71e+00, 4.17e-01 / 2.57e-01 | 20260506 01:18:55.225 | INFO | PerfReport: 3.04%, Metrics -- 9.52e-01 / 1.22e+00, 2.53e-01 / 1.23e-01 | 20260506 01:18:55.225 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.30e-01 / 3.83e-01, 5.54e-02 / 2.90e-02 | 20260506 01:18:55.225 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 01:18:55.225 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:18:55.225 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:18:55.225 | INFO | PerfReport: Full Execution -- 9.80e+01 / -nan, 2.44e+01 / 0.00e+00 | 20260506 01:18:55.225 | INFO | PerfReport: 22.34%, Upload -- 2.19e+01 / 0.00e+00, 7.44e+00 / 0.00e+00 | 20260506 01:18:55.225 | INFO | PerfReport: 21.90%, Download -- 2.15e+01 / 0.00e+00, 7.59e+00 / 0.00e+00 | 20260506 01:18:55.225 | INFO | PerfReport: 31.90%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 01:18:55.225 | INFO | PerfReport: 2.09%, LockUpload -- 2.05e+00 / 0.00e+00, 7.93e+00 / 0.00e+00 | 20260506 01:18:55.225 | INFO | PerfReport: 1.29%, LockDownload -- 1.26e+00 / 0.00e+00, 2.85e+00 / 0.00e+00 | 20260506 01:18:55.225 | INFO | PerfReport: 20.46%, LockCompute -- 2.01e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 01:18:55.225 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.25e-02 / 0.00e+00 | 20260506 01:18:55.341 | INFO | Starting a round of Baz IO | 20260506 01:18:58.489 | INFO | Populating RTMetrics report | 20260506 01:18:58.490 | INFO | RTMetrics report populated | 20260506 01:18:58.490 | INFO | Opening tmp RTMetrics json file | 20260506 01:18:58.490 | INFO | Serializing RTMetrics to json | 20260506 01:18:58.491 | INFO | Serialization done, swapping tmp file now... | 20260506 01:18:58.492 | INFO | RTMetrics done | 20260506 01:18:58.565 | INFO | Finished a round of Baz IO: Wrote 1.829 GiB in 3.224 seconds | 20260506 01:18:58.565 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.32% / 0.00% / 78.68% | 20260506 01:18:58.565 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0813246% / 3.97364e-06% / 7.15256e-05% | 20260506 01:18:58.566 | INFO | Baz IO overall: | 20260506 01:18:58.566 | INFO | BytesWritten : 4413507421659 | 20260506 01:18:58.566 | INFO | NumEvents : 2074831817712 | 20260506 01:19:00.367 | INFO | Analyzing chunk frames = [7815168,7815680) | 20260506 01:19:00.367 | INFO | MemInfo: { | 20260506 01:19:00.367 | INFO | "gpu" : " 17 GB", | 20260506 01:19:00.367 | INFO | "huge" : "", | 20260506 01:19:00.367 | INFO | "huge_shared" : " 27 GB", | 20260506 01:19:00.367 | INFO | "malloc" : " 14 GB", | 20260506 01:19:00.367 | INFO | "pinned" : " 29 GB" | 20260506 01:19:00.367 | INFO | } | 20260506 01:19:00.367 | INFO | Perf report for the last 5.14276 seconds: | 20260506 01:19:00.367 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:19:00.367 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:00.367 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:00.367 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:00.367 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260506 01:19:00.367 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:00.367 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260506 01:19:00.367 | INFO | PRE_HQ : 59.8%, 1.7, 53.7ms, 0 | 20260506 01:19:00.367 | INFO | BAZWRITER : 61.8%, 3.5, 109.7ms, 0 | 20260506 01:19:00.367 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:19:00.367 | INFO | Uploaded 36GiB to GPU in 2.16862s (16.6005GiB/s) | 20260506 01:19:00.367 | INFO | Downloaded 38GiB to GPU in 2.24504s (16.9262GiB/s) | 20260506 01:19:05.509 | INFO | Analyzing chunk frames = [7815680,7816192) | 20260506 01:19:05.509 | INFO | MemInfo: { | 20260506 01:19:05.509 | INFO | "gpu" : " 17 GB", | 20260506 01:19:05.509 | INFO | "huge" : "", | 20260506 01:19:05.509 | INFO | "huge_shared" : " 27 GB", | 20260506 01:19:05.509 | INFO | "malloc" : " 14 GB", | 20260506 01:19:05.509 | INFO | "pinned" : " 29 GB" | 20260506 01:19:05.509 | INFO | } | 20260506 01:19:05.509 | INFO | Perf report for the last 5.14195 seconds: | 20260506 01:19:05.509 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:19:05.509 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:05.509 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:05.509 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:05.509 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 01:19:05.509 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:05.509 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:05.509 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 01:19:05.509 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:05.509 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:05.509 | INFO | Uploaded 36GiB to GPU in 2.00388s (17.9651GiB/s) | 20260506 01:19:05.509 | INFO | Downloaded 34GiB to GPU in 1.99499s (17.0427GiB/s) | 20260506 01:19:10.644 | INFO | Analyzing chunk frames = [7816192,7816704) | 20260506 01:19:10.644 | INFO | MemInfo: { | 20260506 01:19:10.644 | INFO | "gpu" : " 17 GB", | 20260506 01:19:10.644 | INFO | "huge" : "", | 20260506 01:19:10.644 | INFO | "huge_shared" : " 27 GB", | 20260506 01:19:10.644 | INFO | "malloc" : " 14 GB", | 20260506 01:19:10.644 | INFO | "pinned" : " 29 GB" | 20260506 01:19:10.644 | INFO | } | 20260506 01:19:10.644 | INFO | Perf report for the last 5.13502 seconds: | 20260506 01:19:10.644 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:19:10.644 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:10.644 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:10.644 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:10.644 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 01:19:10.644 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:10.644 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:10.644 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:19:10.644 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:10.644 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:10.644 | INFO | Uploaded 36GiB to GPU in 2.09011s (17.2239GiB/s) | 20260506 01:19:10.644 | INFO | Downloaded 34GiB to GPU in 2.02871s (16.7594GiB/s) | 20260506 01:19:15.780 | INFO | Analyzing chunk frames = [7816704,7817216) | 20260506 01:19:15.780 | INFO | MemInfo: { | 20260506 01:19:15.780 | INFO | "gpu" : " 17 GB", | 20260506 01:19:15.780 | INFO | "huge" : "", | 20260506 01:19:15.780 | INFO | "huge_shared" : " 27 GB", | 20260506 01:19:15.780 | INFO | "malloc" : " 14 GB", | 20260506 01:19:15.780 | INFO | "pinned" : " 29 GB" | 20260506 01:19:15.780 | INFO | } | 20260506 01:19:15.780 | INFO | Perf report for the last 5.13592 seconds: | 20260506 01:19:15.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:19:15.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:15.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:15.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:15.780 | INFO | ANALYSIS : 61.6%, 3.0, 95.6ms, 0 | 20260506 01:19:15.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:15.780 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:15.780 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 01:19:15.780 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:15.780 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:15.780 | INFO | Uploaded 36GiB to GPU in 2.13403s (16.8695GiB/s) | 20260506 01:19:15.780 | INFO | Downloaded 34GiB to GPU in 2.02006s (16.8312GiB/s) | 20260506 01:19:20.923 | INFO | Analyzing chunk frames = [7817216,7817728) | 20260506 01:19:20.923 | INFO | MemInfo: { | 20260506 01:19:20.923 | INFO | "gpu" : " 17 GB", | 20260506 01:19:20.923 | INFO | "huge" : "", | 20260506 01:19:20.923 | INFO | "huge_shared" : " 27 GB", | 20260506 01:19:20.923 | INFO | "malloc" : " 14 GB", | 20260506 01:19:20.923 | INFO | "pinned" : " 29 GB" | 20260506 01:19:20.923 | INFO | } | 20260506 01:19:20.923 | INFO | Perf report for the last 5.14336 seconds: | 20260506 01:19:20.923 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:19:20.923 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:20.923 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:20.923 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:20.923 | INFO | ANALYSIS : 61.1%, 3.0, 95.5ms, 0 | 20260506 01:19:20.923 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:20.923 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:20.923 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:19:20.923 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:20.923 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:20.923 | INFO | Uploaded 36GiB to GPU in 2.10576s (17.096GiB/s) | 20260506 01:19:20.924 | INFO | Downloaded 34GiB to GPU in 2.05463s (16.548GiB/s) | 20260506 01:19:26.066 | INFO | Analyzing chunk frames = [7817728,7818240) | 20260506 01:19:26.066 | INFO | MemInfo: { | 20260506 01:19:26.066 | INFO | "gpu" : " 17 GB", | 20260506 01:19:26.066 | INFO | "huge" : "", | 20260506 01:19:26.066 | INFO | "huge_shared" : " 27 GB", | 20260506 01:19:26.066 | INFO | "malloc" : " 14 GB", | 20260506 01:19:26.066 | INFO | "pinned" : " 29 GB" | 20260506 01:19:26.066 | INFO | } | 20260506 01:19:26.066 | INFO | Perf report for the last 5.14297 seconds: | 20260506 01:19:26.066 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:19:26.066 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:26.066 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:26.066 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:26.066 | INFO | ANALYSIS : 61.1%, 3.0, 95.5ms, 0 | 20260506 01:19:26.066 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:26.066 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:26.066 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 01:19:26.066 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:26.066 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:26.067 | INFO | Uploaded 36GiB to GPU in 2.0772s (17.331GiB/s) | 20260506 01:19:26.067 | INFO | Downloaded 34GiB to GPU in 2.04257s (16.6457GiB/s) | 20260506 01:19:31.213 | INFO | Analyzing chunk frames = [7818240,7818752) | 20260506 01:19:31.213 | INFO | MemInfo: { | 20260506 01:19:31.213 | INFO | "gpu" : " 17 GB", | 20260506 01:19:31.213 | INFO | "huge" : "", | 20260506 01:19:31.213 | INFO | "huge_shared" : " 27 GB", | 20260506 01:19:31.213 | INFO | "malloc" : " 14 GB", | 20260506 01:19:31.213 | INFO | "pinned" : " 29 GB" | 20260506 01:19:31.213 | INFO | } | 20260506 01:19:31.213 | INFO | Perf report for the last 5.14676 seconds: | 20260506 01:19:31.213 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:19:31.213 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:31.213 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:31.213 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:31.213 | INFO | ANALYSIS : 62.0%, 3.0, 97.0ms, 0 | 20260506 01:19:31.213 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:31.213 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:31.213 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 01:19:31.213 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:31.213 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:31.213 | INFO | Uploaded 36GiB to GPU in 2.06691s (17.4173GiB/s) | 20260506 01:19:31.213 | INFO | Downloaded 34GiB to GPU in 2.07361s (16.3965GiB/s) | 20260506 01:19:36.344 | INFO | Analyzing chunk frames = [7818752,7819264) | 20260506 01:19:36.344 | INFO | MemInfo: { | 20260506 01:19:36.344 | INFO | "gpu" : " 17 GB", | 20260506 01:19:36.344 | INFO | "huge" : "", | 20260506 01:19:36.344 | INFO | "huge_shared" : " 27 GB", | 20260506 01:19:36.344 | INFO | "malloc" : " 14 GB", | 20260506 01:19:36.344 | INFO | "pinned" : " 29 GB" | 20260506 01:19:36.344 | INFO | } | 20260506 01:19:36.344 | INFO | Perf report for the last 5.13107 seconds: | 20260506 01:19:36.344 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:19:36.344 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:36.344 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:36.344 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:36.344 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 01:19:36.344 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:36.344 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:36.344 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 01:19:36.344 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:36.344 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:36.344 | INFO | Uploaded 36GiB to GPU in 2.03853s (17.6598GiB/s) | 20260506 01:19:36.344 | INFO | Downloaded 34GiB to GPU in 2.01544s (16.8698GiB/s) | 20260506 01:19:36.461 | INFO | Starting a round of Baz IO | 20260506 01:19:39.602 | INFO | Populating RTMetrics report | 20260506 01:19:39.603 | INFO | RTMetrics report populated | 20260506 01:19:39.603 | INFO | Opening tmp RTMetrics json file | 20260506 01:19:39.609 | INFO | Serializing RTMetrics to json | 20260506 01:19:39.610 | INFO | Serialization done, swapping tmp file now... | 20260506 01:19:39.611 | INFO | RTMetrics done | 20260506 01:19:39.688 | INFO | Finished a round of Baz IO: Wrote 1.828 GiB in 3.226 seconds | 20260506 01:19:39.688 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.32% / 0.00% / 78.68% | 20260506 01:19:39.688 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0813286% / 0% / 5.5631e-05% | 20260506 01:19:39.688 | INFO | Baz IO overall: | 20260506 01:19:39.688 | INFO | BytesWritten : 4415469808500 | 20260506 01:19:39.688 | INFO | NumEvents : 2075726219263 | 20260506 01:19:41.479 | INFO | Analyzing chunk frames = [7819264,7819776) | 20260506 01:19:41.479 | INFO | MemInfo: { | 20260506 01:19:41.479 | INFO | "gpu" : " 17 GB", | 20260506 01:19:41.479 | INFO | "huge" : "", | 20260506 01:19:41.479 | INFO | "huge_shared" : " 27 GB", | 20260506 01:19:41.479 | INFO | "malloc" : " 14 GB", | 20260506 01:19:41.479 | INFO | "pinned" : " 29 GB" | 20260506 01:19:41.479 | INFO | } | 20260506 01:19:41.479 | INFO | Perf report for the last 5.13471 seconds: | 20260506 01:19:41.479 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:19:41.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:41.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:41.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:41.479 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260506 01:19:41.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:41.479 | INFO | RT_METRICS : 43.2%, 1.0, 22.6ms, 0 | 20260506 01:19:41.479 | INFO | PRE_HQ : 60.0%, 1.7, 53.9ms, 0 | 20260506 01:19:41.479 | INFO | BAZWRITER : 61.9%, 3.5, 109.7ms, 0 | 20260506 01:19:41.479 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 01:19:41.480 | INFO | Uploaded 36GiB to GPU in 2.16997s (16.5901GiB/s) | 20260506 01:19:41.480 | INFO | Downloaded 38GiB to GPU in 2.24314s (16.9405GiB/s) | 20260506 01:19:46.620 | INFO | Analyzing chunk frames = [7819776,7820288) | 20260506 01:19:46.620 | INFO | MemInfo: { | 20260506 01:19:46.620 | INFO | "gpu" : " 17 GB", | 20260506 01:19:46.620 | INFO | "huge" : "", | 20260506 01:19:46.620 | INFO | "huge_shared" : " 27 GB", | 20260506 01:19:46.620 | INFO | "malloc" : " 14 GB", | 20260506 01:19:46.620 | INFO | "pinned" : " 29 GB" | 20260506 01:19:46.620 | INFO | } | 20260506 01:19:46.620 | INFO | Perf report for the last 5.14165 seconds: | 20260506 01:19:46.620 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:19:46.620 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:46.620 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:46.620 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:46.620 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260506 01:19:46.620 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:46.620 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:46.621 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 01:19:46.621 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:46.621 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:46.621 | INFO | Uploaded 36GiB to GPU in 2.12035s (16.9783GiB/s) | 20260506 01:19:46.621 | INFO | Downloaded 34GiB to GPU in 2.04236s (16.6474GiB/s) | 20260506 01:19:51.760 | INFO | Analyzing chunk frames = [7820288,7820800) | 20260506 01:19:51.760 | INFO | MemInfo: { | 20260506 01:19:51.760 | INFO | "gpu" : " 17 GB", | 20260506 01:19:51.760 | INFO | "huge" : "", | 20260506 01:19:51.760 | INFO | "huge_shared" : " 27 GB", | 20260506 01:19:51.760 | INFO | "malloc" : " 14 GB", | 20260506 01:19:51.760 | INFO | "pinned" : " 29 GB" | 20260506 01:19:51.760 | INFO | } | 20260506 01:19:51.760 | INFO | Perf report for the last 5.13949 seconds: | 20260506 01:19:51.760 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:19:51.760 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:51.760 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:51.760 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:51.760 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 01:19:51.760 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:51.760 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:51.760 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:19:51.760 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:51.760 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:51.760 | INFO | Uploaded 36GiB to GPU in 2.07694s (17.3332GiB/s) | 20260506 01:19:51.760 | INFO | Downloaded 34GiB to GPU in 2.01416s (16.8805GiB/s) | 20260506 01:19:56.897 | INFO | Analyzing chunk frames = [7820800,7821312) | 20260506 01:19:56.897 | INFO | MemInfo: { | 20260506 01:19:56.897 | INFO | "gpu" : " 17 GB", | 20260506 01:19:56.897 | INFO | "huge" : "", | 20260506 01:19:56.897 | INFO | "huge_shared" : " 27 GB", | 20260506 01:19:56.898 | INFO | "malloc" : " 14 GB", | 20260506 01:19:56.898 | INFO | "pinned" : " 29 GB" | 20260506 01:19:56.898 | INFO | } | 20260506 01:19:56.898 | INFO | Perf report for the last 5.13762 seconds: | 20260506 01:19:56.898 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:19:56.898 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:56.898 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:56.898 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:56.898 | INFO | ANALYSIS : 61.6%, 3.0, 95.6ms, 0 | 20260506 01:19:56.898 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:56.898 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:56.898 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 01:19:56.898 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:56.898 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:19:56.898 | INFO | Uploaded 36GiB to GPU in 2.14198s (16.8069GiB/s) | 20260506 01:19:56.898 | INFO | Downloaded 34GiB to GPU in 2.01957s (16.8353GiB/s) | 20260506 01:20:02.038 | INFO | Analyzing chunk frames = [7821312,7821824) | 20260506 01:20:02.038 | INFO | MemInfo: { | 20260506 01:20:02.038 | INFO | "gpu" : " 17 GB", | 20260506 01:20:02.038 | INFO | "huge" : "", | 20260506 01:20:02.038 | INFO | "huge_shared" : " 27 GB", | 20260506 01:20:02.038 | INFO | "malloc" : " 14 GB", | 20260506 01:20:02.038 | INFO | "pinned" : " 29 GB" | 20260506 01:20:02.038 | INFO | } | 20260506 01:20:02.038 | INFO | Perf report for the last 5.1402 seconds: | 20260506 01:20:02.038 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:20:02.038 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:02.038 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:02.038 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:02.038 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 01:20:02.038 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:02.038 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:02.038 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 01:20:02.038 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:02.038 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:02.038 | INFO | Uploaded 36GiB to GPU in 2.0926s (17.2035GiB/s) | 20260506 01:20:02.038 | INFO | Downloaded 34GiB to GPU in 2.05297s (16.5614GiB/s) | 20260506 01:20:07.180 | INFO | Analyzing chunk frames = [7821824,7822336) | 20260506 01:20:07.180 | INFO | MemInfo: { | 20260506 01:20:07.180 | INFO | "gpu" : " 17 GB", | 20260506 01:20:07.180 | INFO | "huge" : "", | 20260506 01:20:07.180 | INFO | "huge_shared" : " 27 GB", | 20260506 01:20:07.180 | INFO | "malloc" : " 14 GB", | 20260506 01:20:07.181 | INFO | "pinned" : " 29 GB" | 20260506 01:20:07.181 | INFO | } | 20260506 01:20:07.181 | INFO | Perf report for the last 5.14278 seconds: | 20260506 01:20:07.181 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:20:07.181 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:07.181 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:07.181 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:07.181 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260506 01:20:07.181 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:07.181 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:07.181 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:20:07.181 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:07.181 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:07.181 | INFO | Uploaded 36GiB to GPU in 2.07399s (17.3579GiB/s) | 20260506 01:20:07.181 | INFO | Downloaded 34GiB to GPU in 2.0368s (16.6929GiB/s) | 20260506 01:20:12.323 | INFO | Analyzing chunk frames = [7822336,7822848) | 20260506 01:20:12.323 | INFO | MemInfo: { | 20260506 01:20:12.323 | INFO | "gpu" : " 17 GB", | 20260506 01:20:12.323 | INFO | "huge" : "", | 20260506 01:20:12.323 | INFO | "huge_shared" : " 27 GB", | 20260506 01:20:12.323 | INFO | "malloc" : " 14 GB", | 20260506 01:20:12.323 | INFO | "pinned" : " 29 GB" | 20260506 01:20:12.323 | INFO | } | 20260506 01:20:12.323 | INFO | Perf report for the last 5.14251 seconds: | 20260506 01:20:12.323 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:20:12.323 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:12.323 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:12.323 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:12.323 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260506 01:20:12.323 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:12.323 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:12.323 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 01:20:12.323 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:12.323 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:12.323 | INFO | Uploaded 36GiB to GPU in 2.03474s (17.6927GiB/s) | 20260506 01:20:12.323 | INFO | Downloaded 34GiB to GPU in 2.05639s (16.5338GiB/s) | 20260506 01:20:17.456 | INFO | Analyzing chunk frames = [7822848,7823360) | 20260506 01:20:17.456 | INFO | MemInfo: { | 20260506 01:20:17.456 | INFO | "gpu" : " 17 GB", | 20260506 01:20:17.456 | INFO | "huge" : "", | 20260506 01:20:17.456 | INFO | "huge_shared" : " 27 GB", | 20260506 01:20:17.456 | INFO | "malloc" : " 14 GB", | 20260506 01:20:17.456 | INFO | "pinned" : " 29 GB" | 20260506 01:20:17.456 | INFO | } | 20260506 01:20:17.456 | INFO | Perf report for the last 5.13274 seconds: | 20260506 01:20:17.456 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:20:17.456 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:17.456 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:17.456 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:17.456 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 01:20:17.456 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:17.456 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:17.456 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 01:20:17.456 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:17.456 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:17.456 | INFO | Uploaded 36GiB to GPU in 2.04032s (17.6443GiB/s) | 20260506 01:20:17.456 | INFO | Downloaded 34GiB to GPU in 2.02013s (16.8306GiB/s) | 20260506 01:20:17.456 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 01:20:17.456 | INFO | Total mem usage: 15.3 GiB | 20260506 01:20:17.456 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 01:20:17.456 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.21%) | 20260506 01:20:17.456 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 01:20:17.456 | INFO | High Water Breakdown: | 20260506 01:20:17.456 | INFO | 60.9% : 8.54 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 01:20:17.456 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 01:20:17.456 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 01:20:17.456 | INFO | ------------------------------------------ | 20260506 01:20:17.457 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:20:17.457 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 01:20:17.457 | INFO | Total mem usage: 26.5 GiB | 20260506 01:20:17.457 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 01:20:17.457 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 01:20:17.457 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 01:20:17.457 | INFO | High Water Breakdown: | 20260506 01:20:17.457 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:20:17.457 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:20:17.457 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:20:17.457 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:20:17.457 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:20:17.457 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:20:17.457 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 01:20:17.457 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 01:20:17.457 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:20:17.457 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 01:20:17.457 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 01:20:17.457 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 01:20:17.457 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 01:20:17.457 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 01:20:17.457 | INFO | ------------------------------------------ | 20260506 01:20:17.457 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:20:17.457 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 01:20:17.457 | INFO | Total mem usage: 25.6 GiB | 20260506 01:20:17.457 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 01:20:17.457 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 01:20:17.457 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 01:20:17.457 | INFO | High Water Breakdown: | 20260506 01:20:17.457 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 01:20:17.457 | INFO | ------------------------------------------ | 20260506 01:20:17.457 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:20:17.457 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 01:20:17.457 | INFO | Total mem usage: 15.7 GiB | 20260506 01:20:17.457 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 01:20:17.457 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 01:20:17.457 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 01:20:17.457 | INFO | High Water Breakdown: | 20260506 01:20:17.457 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 01:20:17.457 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 01:20:17.457 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 01:20:17.457 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 01:20:17.457 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 01:20:17.457 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 01:20:17.457 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 01:20:17.457 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 01:20:17.457 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 01:20:17.457 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:20:17.457 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 01:20:17.457 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:20:17.457 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 01:20:17.457 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 01:20:17.457 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 01:20:17.457 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 01:20:17.457 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:20:17.457 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 01:20:17.457 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:20:17.457 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:20:17.457 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:20:17.457 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 01:20:17.457 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 01:20:17.457 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:20:17.457 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 01:20:17.457 | INFO | ------------------------------------------ | 20260506 01:20:17.457 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:20:17.457 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 01:20:17.457 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:20:17.457 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:20:17.457 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.39e+01, 1.35e+01 / 8.93e-01 | 20260506 01:20:17.457 | INFO | PerfReport: 0.29%, AggregateStats -- 9.17e-02 / 1.04e-01, 1.09e-02 / 3.63e-03 | 20260506 01:20:17.457 | INFO | PerfReport: 11.86%, Baseline -- 3.71e+00 / 4.54e+00, 5.76e-01 / 1.41e-01 | 20260506 01:20:17.457 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.53e+00, 1.92e-01 / 1.01e-01 | 20260506 01:20:17.457 | INFO | PerfReport: 16.02%, DME -- 4.01e+01 / 4.40e+01, 3.76e+00 / 1.08e+00 | 20260506 01:20:17.457 | INFO | PerfReport: 57.37%, FrameLabeling -- 1.79e+01 / 1.94e+01, 1.47e+00 / 1.11e+00 | 20260506 01:20:17.457 | INFO | PerfReport: 7.07%, PulseAccumulating -- 2.21e+00 / 2.74e+00, 4.20e-01 / 2.31e-01 | 20260506 01:20:17.457 | INFO | PerfReport: 3.05%, Metrics -- 9.55e-01 / 1.23e+00, 2.58e-01 / 9.79e-02 | 20260506 01:20:17.457 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.30e-01 / 3.75e-01, 5.51e-02 / 2.29e-02 | 20260506 01:20:17.457 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 01:20:17.457 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:20:17.457 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:20:17.457 | INFO | PerfReport: Full Execution -- 9.80e+01 / -nan, 2.43e+01 / 0.00e+00 | 20260506 01:20:17.457 | INFO | PerfReport: 22.22%, Upload -- 2.18e+01 / 0.00e+00, 7.34e+00 / 0.00e+00 | 20260506 01:20:17.457 | INFO | PerfReport: 21.90%, Download -- 2.15e+01 / 0.00e+00, 7.59e+00 / 0.00e+00 | 20260506 01:20:17.457 | INFO | PerfReport: 31.93%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 01:20:17.457 | INFO | PerfReport: 2.05%, LockUpload -- 2.01e+00 / 0.00e+00, 7.73e+00 / 0.00e+00 | 20260506 01:20:17.457 | INFO | PerfReport: 1.35%, LockDownload -- 1.32e+00 / 0.00e+00, 2.96e+00 / 0.00e+00 | 20260506 01:20:17.457 | INFO | PerfReport: 20.54%, LockCompute -- 2.01e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 01:20:17.457 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.96e-02 / 0.00e+00 | 20260506 01:20:17.582 | INFO | Starting a round of Baz IO | 20260506 01:20:20.714 | INFO | Populating RTMetrics report | 20260506 01:20:20.714 | INFO | RTMetrics report populated | 20260506 01:20:20.714 | INFO | Opening tmp RTMetrics json file | 20260506 01:20:20.715 | INFO | Serializing RTMetrics to json | 20260506 01:20:20.716 | INFO | Serialization done, swapping tmp file now... | 20260506 01:20:20.716 | INFO | RTMetrics done | 20260506 01:20:20.793 | INFO | Finished a round of Baz IO: Wrote 1.828 GiB in 3.211 seconds | 20260506 01:20:20.793 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.32% / 0.00% / 78.68% | 20260506 01:20:20.793 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0807206% / 0% / 5.96046e-05% | 20260506 01:20:20.793 | INFO | Baz IO overall: | 20260506 01:20:20.793 | INFO | BytesWritten : 4417432307062 | 20260506 01:20:20.793 | INFO | NumEvents : 2076620717148 | 20260506 01:20:22.590 | INFO | Analyzing chunk frames = [7823360,7823872) | 20260506 01:20:22.590 | INFO | MemInfo: { | 20260506 01:20:22.590 | INFO | "gpu" : " 17 GB", | 20260506 01:20:22.590 | INFO | "huge" : "", | 20260506 01:20:22.590 | INFO | "huge_shared" : " 27 GB", | 20260506 01:20:22.590 | INFO | "malloc" : " 14 GB", | 20260506 01:20:22.590 | INFO | "pinned" : " 29 GB" | 20260506 01:20:22.590 | INFO | } | 20260506 01:20:22.590 | INFO | Perf report for the last 5.1345 seconds: | 20260506 01:20:22.590 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:20:22.590 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:22.590 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:22.590 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:22.590 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260506 01:20:22.590 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:22.591 | INFO | RT_METRICS : 43.0%, 1.0, 22.5ms, 0 | 20260506 01:20:22.591 | INFO | PRE_HQ : 59.5%, 1.7, 53.5ms, 0 | 20260506 01:20:22.591 | INFO | BAZWRITER : 61.8%, 3.4, 109.3ms, 0 | 20260506 01:20:22.591 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:20:22.591 | INFO | Uploaded 36GiB to GPU in 2.1805s (16.5099GiB/s) | 20260506 01:20:22.591 | INFO | Downloaded 38GiB to GPU in 2.24113s (16.9558GiB/s) | 20260506 01:20:27.734 | INFO | Analyzing chunk frames = [7823872,7824384) | 20260506 01:20:27.734 | INFO | MemInfo: { | 20260506 01:20:27.734 | INFO | "gpu" : " 17 GB", | 20260506 01:20:27.734 | INFO | "huge" : "", | 20260506 01:20:27.734 | INFO | "huge_shared" : " 27 GB", | 20260506 01:20:27.734 | INFO | "malloc" : " 14 GB", | 20260506 01:20:27.734 | INFO | "pinned" : " 29 GB" | 20260506 01:20:27.734 | INFO | } | 20260506 01:20:27.734 | INFO | Perf report for the last 5.14361 seconds: | 20260506 01:20:27.734 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:20:27.734 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:27.734 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:27.734 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:27.734 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260506 01:20:27.734 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:27.734 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:27.734 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 01:20:27.734 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:27.734 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:27.734 | INFO | Uploaded 36GiB to GPU in 2.09893s (17.1516GiB/s) | 20260506 01:20:27.734 | INFO | Downloaded 34GiB to GPU in 2.03923s (16.673GiB/s) | 20260506 01:20:32.876 | INFO | Analyzing chunk frames = [7824384,7824896) | 20260506 01:20:32.876 | INFO | MemInfo: { | 20260506 01:20:32.876 | INFO | "gpu" : " 17 GB", | 20260506 01:20:32.876 | INFO | "huge" : "", | 20260506 01:20:32.876 | INFO | "huge_shared" : " 27 GB", | 20260506 01:20:32.876 | INFO | "malloc" : " 14 GB", | 20260506 01:20:32.876 | INFO | "pinned" : " 29 GB" | 20260506 01:20:32.876 | INFO | } | 20260506 01:20:32.876 | INFO | Perf report for the last 5.14167 seconds: | 20260506 01:20:32.876 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:20:32.876 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:32.876 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:32.876 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:32.876 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 01:20:32.876 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:32.876 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:32.876 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:20:32.876 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:32.876 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:32.876 | INFO | Uploaded 36GiB to GPU in 2.09123s (17.2148GiB/s) | 20260506 01:20:32.876 | INFO | Downloaded 34GiB to GPU in 2.02666s (16.7764GiB/s) | 20260506 01:20:38.016 | INFO | Analyzing chunk frames = [7824896,7825408) | 20260506 01:20:38.017 | INFO | MemInfo: { | 20260506 01:20:38.017 | INFO | "gpu" : " 17 GB", | 20260506 01:20:38.017 | INFO | "huge" : "", | 20260506 01:20:38.017 | INFO | "huge_shared" : " 27 GB", | 20260506 01:20:38.017 | INFO | "malloc" : " 14 GB", | 20260506 01:20:38.017 | INFO | "pinned" : " 29 GB" | 20260506 01:20:38.017 | INFO | } | 20260506 01:20:38.017 | INFO | Perf report for the last 5.14095 seconds: | 20260506 01:20:38.017 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:20:38.017 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:38.017 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:38.017 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:38.017 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260506 01:20:38.017 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:38.017 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:38.017 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:20:38.017 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:38.017 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:38.017 | INFO | Uploaded 36GiB to GPU in 2.14269s (16.8013GiB/s) | 20260506 01:20:38.017 | INFO | Downloaded 34GiB to GPU in 2.03401s (16.7157GiB/s) | 20260506 01:20:43.165 | INFO | Analyzing chunk frames = [7825408,7825920) | 20260506 01:20:43.165 | INFO | MemInfo: { | 20260506 01:20:43.165 | INFO | "gpu" : " 17 GB", | 20260506 01:20:43.165 | INFO | "huge" : "", | 20260506 01:20:43.165 | INFO | "huge_shared" : " 27 GB", | 20260506 01:20:43.165 | INFO | "malloc" : " 14 GB", | 20260506 01:20:43.165 | INFO | "pinned" : " 29 GB" | 20260506 01:20:43.165 | INFO | } | 20260506 01:20:43.165 | INFO | Perf report for the last 5.14885 seconds: | 20260506 01:20:43.165 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:20:43.166 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:43.166 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:43.166 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:43.166 | INFO | ANALYSIS : 60.9%, 3.0, 95.2ms, 0 | 20260506 01:20:43.166 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:43.166 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:43.166 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 01:20:43.166 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:43.166 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:43.166 | INFO | Uploaded 36GiB to GPU in 2.10335s (17.1156GiB/s) | 20260506 01:20:43.166 | INFO | Downloaded 34GiB to GPU in 2.04684s (16.611GiB/s) | 20260506 01:20:48.295 | INFO | Analyzing chunk frames = [7825920,7826432) | 20260506 01:20:48.295 | INFO | MemInfo: { | 20260506 01:20:48.295 | INFO | "gpu" : " 17 GB", | 20260506 01:20:48.295 | INFO | "huge" : "", | 20260506 01:20:48.295 | INFO | "huge_shared" : " 27 GB", | 20260506 01:20:48.295 | INFO | "malloc" : " 14 GB", | 20260506 01:20:48.295 | INFO | "pinned" : " 29 GB" | 20260506 01:20:48.295 | INFO | } | 20260506 01:20:48.295 | INFO | Perf report for the last 5.12931 seconds: | 20260506 01:20:48.295 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:20:48.295 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:48.295 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:48.295 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:48.295 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 01:20:48.295 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:48.295 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:48.295 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:20:48.295 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:48.295 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:48.295 | INFO | Uploaded 36GiB to GPU in 2.07843s (17.3207GiB/s) | 20260506 01:20:48.295 | INFO | Downloaded 34GiB to GPU in 2.03543s (16.7041GiB/s) | 20260506 01:20:53.433 | INFO | Analyzing chunk frames = [7826432,7826944) | 20260506 01:20:53.433 | INFO | MemInfo: { | 20260506 01:20:53.433 | INFO | "gpu" : " 17 GB", | 20260506 01:20:53.433 | INFO | "huge" : "", | 20260506 01:20:53.433 | INFO | "huge_shared" : " 27 GB", | 20260506 01:20:53.433 | INFO | "malloc" : " 14 GB", | 20260506 01:20:53.433 | INFO | "pinned" : " 29 GB" | 20260506 01:20:53.433 | INFO | } | 20260506 01:20:53.433 | INFO | Perf report for the last 5.13826 seconds: | 20260506 01:20:53.433 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:20:53.433 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:53.433 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:53.433 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:53.433 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260506 01:20:53.433 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:53.433 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:53.433 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 01:20:53.433 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:53.433 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:53.433 | INFO | Uploaded 36GiB to GPU in 2.07329s (17.3637GiB/s) | 20260506 01:20:53.433 | INFO | Downloaded 34GiB to GPU in 2.07671s (16.3721GiB/s) | 20260506 01:20:58.574 | INFO | Analyzing chunk frames = [7826944,7827456) | 20260506 01:20:58.574 | INFO | MemInfo: { | 20260506 01:20:58.574 | INFO | "gpu" : " 17 GB", | 20260506 01:20:58.574 | INFO | "huge" : "", | 20260506 01:20:58.574 | INFO | "huge_shared" : " 27 GB", | 20260506 01:20:58.574 | INFO | "malloc" : " 14 GB", | 20260506 01:20:58.574 | INFO | "pinned" : " 29 GB" | 20260506 01:20:58.574 | INFO | } | 20260506 01:20:58.574 | INFO | Perf report for the last 5.14094 seconds: | 20260506 01:20:58.574 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:20:58.574 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:58.574 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:58.574 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:58.574 | INFO | ANALYSIS : 61.6%, 3.0, 96.2ms, 0 | 20260506 01:20:58.574 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:58.574 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:58.574 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 01:20:58.574 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:58.574 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:20:58.574 | INFO | Uploaded 36GiB to GPU in 2.0717s (17.3771GiB/s) | 20260506 01:20:58.574 | INFO | Downloaded 34GiB to GPU in 2.02565s (16.7848GiB/s) | 20260506 01:20:58.700 | INFO | Starting a round of Baz IO | 20260506 01:21:01.835 | INFO | Populating RTMetrics report | 20260506 01:21:01.836 | INFO | RTMetrics report populated | 20260506 01:21:01.836 | INFO | Opening tmp RTMetrics json file | 20260506 01:21:01.836 | INFO | Serializing RTMetrics to json | 20260506 01:21:01.837 | INFO | Serialization done, swapping tmp file now... | 20260506 01:21:01.838 | INFO | RTMetrics done | 20260506 01:21:01.915 | INFO | Finished a round of Baz IO: Wrote 1.83 GiB in 3.215 seconds | 20260506 01:21:01.915 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.32% / 0.00% / 78.68% | 20260506 01:21:01.915 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0814716% / 0% / 6.75519e-05% | 20260506 01:21:01.915 | INFO | Baz IO overall: | 20260506 01:21:01.915 | INFO | BytesWritten : 4419397112187 | 20260506 01:21:01.915 | INFO | NumEvents : 2077516779680 | 20260506 01:21:03.709 | INFO | Analyzing chunk frames = [7827456,7827968) | 20260506 01:21:03.709 | INFO | MemInfo: { | 20260506 01:21:03.709 | INFO | "gpu" : " 17 GB", | 20260506 01:21:03.709 | INFO | "huge" : "", | 20260506 01:21:03.709 | INFO | "huge_shared" : " 27 GB", | 20260506 01:21:03.709 | INFO | "malloc" : " 14 GB", | 20260506 01:21:03.709 | INFO | "pinned" : " 29 GB" | 20260506 01:21:03.709 | INFO | } | 20260506 01:21:03.709 | INFO | Perf report for the last 5.13494 seconds: | 20260506 01:21:03.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:21:03.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:03.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:03.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:03.709 | INFO | ANALYSIS : 62.5%, 3.0, 96.5ms, 0 | 20260506 01:21:03.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:03.709 | INFO | RT_METRICS : 43.1%, 1.0, 22.3ms, 0 | 20260506 01:21:03.709 | INFO | PRE_HQ : 59.6%, 1.7, 53.3ms, 0 | 20260506 01:21:03.709 | INFO | BAZWRITER : 61.9%, 3.5, 109.3ms, 0 | 20260506 01:21:03.709 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:21:03.709 | INFO | Uploaded 36GiB to GPU in 2.17886s (16.5224GiB/s) | 20260506 01:21:03.709 | INFO | Downloaded 38GiB to GPU in 2.24145s (16.9533GiB/s) | 20260506 01:21:08.850 | INFO | Analyzing chunk frames = [7827968,7828480) | 20260506 01:21:08.850 | INFO | MemInfo: { | 20260506 01:21:08.850 | INFO | "gpu" : " 17 GB", | 20260506 01:21:08.850 | INFO | "huge" : "", | 20260506 01:21:08.850 | INFO | "huge_shared" : " 27 GB", | 20260506 01:21:08.850 | INFO | "malloc" : " 14 GB", | 20260506 01:21:08.850 | INFO | "pinned" : " 29 GB" | 20260506 01:21:08.850 | INFO | } | 20260506 01:21:08.850 | INFO | Perf report for the last 5.14115 seconds: | 20260506 01:21:08.850 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:21:08.850 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:08.850 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:08.850 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:08.850 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 01:21:08.850 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:08.850 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:08.850 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:21:08.850 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:08.850 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:08.850 | INFO | Uploaded 36GiB to GPU in 1.96359s (18.3337GiB/s) | 20260506 01:21:08.850 | INFO | Downloaded 34GiB to GPU in 1.99484s (17.044GiB/s) | 20260506 01:21:13.990 | INFO | Analyzing chunk frames = [7828480,7828992) | 20260506 01:21:13.990 | INFO | MemInfo: { | 20260506 01:21:13.990 | INFO | "gpu" : " 17 GB", | 20260506 01:21:13.990 | INFO | "huge" : "", | 20260506 01:21:13.990 | INFO | "huge_shared" : " 27 GB", | 20260506 01:21:13.990 | INFO | "malloc" : " 14 GB", | 20260506 01:21:13.990 | INFO | "pinned" : " 29 GB" | 20260506 01:21:13.990 | INFO | } | 20260506 01:21:13.990 | INFO | Perf report for the last 5.13954 seconds: | 20260506 01:21:13.990 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:21:13.990 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:13.990 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:13.990 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:13.990 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260506 01:21:13.990 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:13.990 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:13.990 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:21:13.990 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:13.990 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:13.990 | INFO | Uploaded 36GiB to GPU in 2.09092s (17.2173GiB/s) | 20260506 01:21:13.990 | INFO | Downloaded 34GiB to GPU in 2.02659s (16.777GiB/s) | 20260506 01:21:19.132 | INFO | Analyzing chunk frames = [7828992,7829504) | 20260506 01:21:19.132 | INFO | MemInfo: { | 20260506 01:21:19.132 | INFO | "gpu" : " 17 GB", | 20260506 01:21:19.132 | INFO | "huge" : "", | 20260506 01:21:19.132 | INFO | "huge_shared" : " 27 GB", | 20260506 01:21:19.132 | INFO | "malloc" : " 14 GB", | 20260506 01:21:19.132 | INFO | "pinned" : " 29 GB" | 20260506 01:21:19.132 | INFO | } | 20260506 01:21:19.132 | INFO | Perf report for the last 5.14199 seconds: | 20260506 01:21:19.132 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:21:19.132 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:19.132 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:19.132 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:19.132 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260506 01:21:19.132 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:19.132 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:19.132 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 01:21:19.132 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:19.132 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:19.132 | INFO | Uploaded 36GiB to GPU in 2.14985s (16.7453GiB/s) | 20260506 01:21:19.132 | INFO | Downloaded 34GiB to GPU in 2.04637s (16.6148GiB/s) | 20260506 01:21:24.274 | INFO | Analyzing chunk frames = [7829504,7830016) | 20260506 01:21:24.274 | INFO | MemInfo: { | 20260506 01:21:24.274 | INFO | "gpu" : " 17 GB", | 20260506 01:21:24.274 | INFO | "huge" : "", | 20260506 01:21:24.274 | INFO | "huge_shared" : " 27 GB", | 20260506 01:21:24.274 | INFO | "malloc" : " 14 GB", | 20260506 01:21:24.275 | INFO | "pinned" : " 29 GB" | 20260506 01:21:24.275 | INFO | } | 20260506 01:21:24.275 | INFO | Perf report for the last 5.14286 seconds: | 20260506 01:21:24.275 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:21:24.275 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:24.275 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:24.275 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:24.275 | INFO | ANALYSIS : 60.7%, 3.0, 94.8ms, 0 | 20260506 01:21:24.275 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:24.275 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:24.275 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:21:24.275 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:24.275 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:24.275 | INFO | Uploaded 36GiB to GPU in 2.08103s (17.2991GiB/s) | 20260506 01:21:24.275 | INFO | Downloaded 34GiB to GPU in 2.02875s (16.7591GiB/s) | 20260506 01:21:29.409 | INFO | Analyzing chunk frames = [7830016,7830528) | 20260506 01:21:29.409 | INFO | MemInfo: { | 20260506 01:21:29.409 | INFO | "gpu" : " 17 GB", | 20260506 01:21:29.409 | INFO | "huge" : "", | 20260506 01:21:29.409 | INFO | "huge_shared" : " 27 GB", | 20260506 01:21:29.409 | INFO | "malloc" : " 14 GB", | 20260506 01:21:29.409 | INFO | "pinned" : " 29 GB" | 20260506 01:21:29.409 | INFO | } | 20260506 01:21:29.409 | INFO | Perf report for the last 5.13459 seconds: | 20260506 01:21:29.409 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:21:29.409 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:29.409 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:29.409 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:29.409 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 01:21:29.409 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:29.409 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:29.409 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 01:21:29.409 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:29.409 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:29.410 | INFO | Uploaded 36GiB to GPU in 2.10038s (17.1398GiB/s) | 20260506 01:21:29.410 | INFO | Downloaded 34GiB to GPU in 2.04361s (16.6372GiB/s) | 20260506 01:21:34.539 | INFO | Analyzing chunk frames = [7830528,7831040) | 20260506 01:21:34.539 | INFO | MemInfo: { | 20260506 01:21:34.539 | INFO | "gpu" : " 17 GB", | 20260506 01:21:34.539 | INFO | "huge" : "", | 20260506 01:21:34.539 | INFO | "huge_shared" : " 27 GB", | 20260506 01:21:34.539 | INFO | "malloc" : " 14 GB", | 20260506 01:21:34.539 | INFO | "pinned" : " 29 GB" | 20260506 01:21:34.539 | INFO | } | 20260506 01:21:34.539 | INFO | Perf report for the last 5.12996 seconds: | 20260506 01:21:34.539 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:21:34.539 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:34.539 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:34.539 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:34.539 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260506 01:21:34.539 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:34.539 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:34.539 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 01:21:34.539 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:34.539 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:34.539 | INFO | Uploaded 36GiB to GPU in 2.0693s (17.3972GiB/s) | 20260506 01:21:34.539 | INFO | Downloaded 34GiB to GPU in 2.07728s (16.3675GiB/s) | 20260506 01:21:39.685 | INFO | Analyzing chunk frames = [7831040,7831552) | 20260506 01:21:39.685 | INFO | MemInfo: { | 20260506 01:21:39.685 | INFO | "gpu" : " 17 GB", | 20260506 01:21:39.685 | INFO | "huge" : "", | 20260506 01:21:39.685 | INFO | "huge_shared" : " 27 GB", | 20260506 01:21:39.685 | INFO | "malloc" : " 14 GB", | 20260506 01:21:39.685 | INFO | "pinned" : " 29 GB" | 20260506 01:21:39.685 | INFO | } | 20260506 01:21:39.685 | INFO | Perf report for the last 5.14602 seconds: | 20260506 01:21:39.685 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:21:39.685 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:39.685 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:39.685 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:39.685 | INFO | ANALYSIS : 61.9%, 3.0, 96.8ms, 0 | 20260506 01:21:39.685 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:39.685 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:39.685 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 01:21:39.685 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:39.685 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:39.685 | INFO | Uploaded 36GiB to GPU in 2.11775s (16.9992GiB/s) | 20260506 01:21:39.685 | INFO | Downloaded 34GiB to GPU in 2.01253s (16.8942GiB/s) | 20260506 01:21:39.686 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 01:21:39.686 | INFO | Total mem usage: 15.3 GiB | 20260506 01:21:39.686 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 01:21:39.686 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.2%) | 20260506 01:21:39.686 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 01:21:39.686 | INFO | High Water Breakdown: | 20260506 01:21:39.686 | INFO | 60.9% : 8.54 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 01:21:39.686 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 01:21:39.686 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 01:21:39.686 | INFO | ------------------------------------------ | 20260506 01:21:39.686 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:21:39.686 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 01:21:39.686 | INFO | Total mem usage: 26.5 GiB | 20260506 01:21:39.686 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 01:21:39.686 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 01:21:39.686 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260506 01:21:39.686 | INFO | High Water Breakdown: | 20260506 01:21:39.686 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:21:39.686 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:21:39.686 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:21:39.686 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:21:39.686 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:21:39.686 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:21:39.686 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 01:21:39.686 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 01:21:39.686 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:21:39.686 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 01:21:39.686 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 01:21:39.686 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 01:21:39.686 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 01:21:39.686 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 01:21:39.686 | INFO | ------------------------------------------ | 20260506 01:21:39.686 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:21:39.686 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 01:21:39.686 | INFO | Total mem usage: 25.6 GiB | 20260506 01:21:39.686 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 01:21:39.686 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 01:21:39.686 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 01:21:39.686 | INFO | High Water Breakdown: | 20260506 01:21:39.686 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 01:21:39.686 | INFO | ------------------------------------------ | 20260506 01:21:39.686 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:21:39.686 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 01:21:39.686 | INFO | Total mem usage: 15.7 GiB | 20260506 01:21:39.686 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 01:21:39.686 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 01:21:39.686 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 01:21:39.686 | INFO | High Water Breakdown: | 20260506 01:21:39.686 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 01:21:39.686 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 01:21:39.686 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 01:21:39.686 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 01:21:39.686 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 01:21:39.686 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 01:21:39.686 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 01:21:39.686 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 01:21:39.686 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 01:21:39.686 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:21:39.686 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 01:21:39.686 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:21:39.686 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 01:21:39.686 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 01:21:39.686 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 01:21:39.686 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 01:21:39.686 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:21:39.686 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 01:21:39.686 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:21:39.686 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:21:39.686 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:21:39.686 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 01:21:39.686 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 01:21:39.686 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:21:39.686 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 01:21:39.686 | INFO | ------------------------------------------ | 20260506 01:21:39.686 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:21:39.686 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 01:21:39.686 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:21:39.686 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:21:39.686 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.40e+01, 1.32e+01 / 8.92e-01 | 20260506 01:21:39.686 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.05e-01, 1.07e-02 / 6.55e-03 | 20260506 01:21:39.686 | INFO | PerfReport: 11.82%, Baseline -- 3.70e+00 / 4.49e+00, 5.68e-01 / 2.22e-01 | 20260506 01:21:39.686 | INFO | PerfReport: 3.29%, Histogram -- 1.03e+00 / 1.53e+00, 1.91e-01 / 7.82e-02 | 20260506 01:21:39.686 | INFO | PerfReport: 15.98%, DME -- 4.00e+01 / 4.39e+01, 3.86e+00 / 1.17e+00 | 20260506 01:21:39.686 | INFO | PerfReport: 57.46%, FrameLabeling -- 1.80e+01 / 1.97e+01, 1.50e+00 / 1.21e+00 | 20260506 01:21:39.686 | INFO | PerfReport: 7.05%, PulseAccumulating -- 2.21e+00 / 2.70e+00, 4.19e-01 / 2.59e-01 | 20260506 01:21:39.686 | INFO | PerfReport: 3.04%, Metrics -- 9.52e-01 / 1.21e+00, 2.56e-01 / 1.00e-01 | 20260506 01:21:39.687 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.30e-01 / 3.80e-01, 5.57e-02 / 2.76e-02 | 20260506 01:21:39.687 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 01:21:39.687 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:21:39.687 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:21:39.687 | INFO | PerfReport: Full Execution -- 9.81e+01 / -nan, 2.43e+01 / 0.00e+00 | 20260506 01:21:39.687 | INFO | PerfReport: 22.30%, Upload -- 2.19e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260506 01:21:39.687 | INFO | PerfReport: 21.91%, Download -- 2.15e+01 / 0.00e+00, 7.54e+00 / 0.00e+00 | 20260506 01:21:39.687 | INFO | PerfReport: 31.91%, Compute -- 3.13e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260506 01:21:39.687 | INFO | PerfReport: 2.08%, LockUpload -- 2.04e+00 / 0.00e+00, 7.89e+00 / 0.00e+00 | 20260506 01:21:39.687 | INFO | PerfReport: 1.32%, LockDownload -- 1.29e+00 / 0.00e+00, 2.92e+00 / 0.00e+00 | 20260506 01:21:39.687 | INFO | PerfReport: 20.46%, LockCompute -- 2.01e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260506 01:21:39.687 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.67e-02 / 0.00e+00 | 20260506 01:21:39.811 | INFO | Starting a round of Baz IO | 20260506 01:21:42.949 | INFO | Populating RTMetrics report | 20260506 01:21:42.949 | INFO | RTMetrics report populated | 20260506 01:21:42.949 | INFO | Opening tmp RTMetrics json file | 20260506 01:21:42.950 | INFO | Serializing RTMetrics to json | 20260506 01:21:42.952 | INFO | Serialization done, swapping tmp file now... | 20260506 01:21:42.952 | INFO | RTMetrics done | 20260506 01:21:43.025 | INFO | Finished a round of Baz IO: Wrote 1.827 GiB in 3.213 seconds | 20260506 01:21:43.025 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.32% / 0.00% / 78.68% | 20260506 01:21:43.025 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0814795% / 0% / 7.15256e-05% | 20260506 01:21:43.025 | INFO | Baz IO overall: | 20260506 01:21:43.025 | INFO | BytesWritten : 4421359264606 | 20260506 01:21:43.025 | INFO | NumEvents : 2078411138861 | 20260506 01:21:44.834 | INFO | Analyzing chunk frames = [7831552,7832064) | 20260506 01:21:44.834 | INFO | MemInfo: { | 20260506 01:21:44.834 | INFO | "gpu" : " 17 GB", | 20260506 01:21:44.834 | INFO | "huge" : "", | 20260506 01:21:44.834 | INFO | "huge_shared" : " 27 GB", | 20260506 01:21:44.834 | INFO | "malloc" : " 14 GB", | 20260506 01:21:44.834 | INFO | "pinned" : " 29 GB" | 20260506 01:21:44.834 | INFO | } | 20260506 01:21:44.834 | INFO | Perf report for the last 5.14904 seconds: | 20260506 01:21:44.834 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:21:44.834 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:44.834 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:44.834 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:44.835 | INFO | ANALYSIS : 62.4%, 3.0, 97.6ms, 0 | 20260506 01:21:44.835 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:44.835 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260506 01:21:44.835 | INFO | PRE_HQ : 59.5%, 1.7, 53.8ms, 0 | 20260506 01:21:44.835 | INFO | BAZWRITER : 61.7%, 3.4, 109.5ms, 0 | 20260506 01:21:44.835 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:21:44.835 | INFO | Uploaded 36GiB to GPU in 2.18272s (16.4932GiB/s) | 20260506 01:21:44.835 | INFO | Downloaded 38GiB to GPU in 2.2327s (17.0197GiB/s) | 20260506 01:21:49.969 | INFO | Analyzing chunk frames = [7832064,7832576) | 20260506 01:21:49.969 | INFO | MemInfo: { | 20260506 01:21:49.969 | INFO | "gpu" : " 17 GB", | 20260506 01:21:49.969 | INFO | "huge" : "", | 20260506 01:21:49.969 | INFO | "huge_shared" : " 27 GB", | 20260506 01:21:49.969 | INFO | "malloc" : " 14 GB", | 20260506 01:21:49.969 | INFO | "pinned" : " 29 GB" | 20260506 01:21:49.969 | INFO | } | 20260506 01:21:49.969 | INFO | Perf report for the last 5.13453 seconds: | 20260506 01:21:49.969 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:21:49.969 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:49.969 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:49.969 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:49.969 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260506 01:21:49.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:49.969 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:49.969 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 01:21:49.969 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:49.969 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:49.969 | INFO | Uploaded 36GiB to GPU in 1.94148s (18.5426GiB/s) | 20260506 01:21:49.969 | INFO | Downloaded 34GiB to GPU in 1.98519s (17.1269GiB/s) | 20260506 01:21:55.103 | INFO | Analyzing chunk frames = [7832576,7833088) | 20260506 01:21:55.103 | INFO | MemInfo: { | 20260506 01:21:55.103 | INFO | "gpu" : " 17 GB", | 20260506 01:21:55.103 | INFO | "huge" : "", | 20260506 01:21:55.103 | INFO | "huge_shared" : " 27 GB", | 20260506 01:21:55.103 | INFO | "malloc" : " 14 GB", | 20260506 01:21:55.103 | INFO | "pinned" : " 29 GB" | 20260506 01:21:55.103 | INFO | } | 20260506 01:21:55.103 | INFO | Perf report for the last 5.13456 seconds: | 20260506 01:21:55.103 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:21:55.103 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:55.103 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:55.103 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:55.103 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 01:21:55.103 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:55.103 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:55.103 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:21:55.103 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:55.103 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:21:55.104 | INFO | Uploaded 36GiB to GPU in 2.08049s (17.3036GiB/s) | 20260506 01:21:55.104 | INFO | Downloaded 34GiB to GPU in 2.02261s (16.81GiB/s) | 20260506 01:22:00.244 | INFO | Analyzing chunk frames = [7833088,7833600) | 20260506 01:22:00.244 | INFO | MemInfo: { | 20260506 01:22:00.244 | INFO | "gpu" : " 17 GB", | 20260506 01:22:00.244 | INFO | "huge" : "", | 20260506 01:22:00.244 | INFO | "huge_shared" : " 27 GB", | 20260506 01:22:00.244 | INFO | "malloc" : " 14 GB", | 20260506 01:22:00.244 | INFO | "pinned" : " 29 GB" | 20260506 01:22:00.244 | INFO | } | 20260506 01:22:00.244 | INFO | Perf report for the last 5.14103 seconds: | 20260506 01:22:00.245 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:22:00.245 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:00.245 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:00.245 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:00.245 | INFO | ANALYSIS : 61.6%, 3.0, 95.7ms, 0 | 20260506 01:22:00.245 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:00.245 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:00.245 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 01:22:00.245 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:00.245 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:00.245 | INFO | Uploaded 36GiB to GPU in 2.13935s (16.8276GiB/s) | 20260506 01:22:00.245 | INFO | Downloaded 34GiB to GPU in 2.02102s (16.8232GiB/s) | 20260506 01:22:05.389 | INFO | Analyzing chunk frames = [7833600,7834112) | 20260506 01:22:05.389 | INFO | MemInfo: { | 20260506 01:22:05.389 | INFO | "gpu" : " 17 GB", | 20260506 01:22:05.389 | INFO | "huge" : "", | 20260506 01:22:05.389 | INFO | "huge_shared" : " 27 GB", | 20260506 01:22:05.389 | INFO | "malloc" : " 14 GB", | 20260506 01:22:05.389 | INFO | "pinned" : " 29 GB" | 20260506 01:22:05.389 | INFO | } | 20260506 01:22:05.389 | INFO | Perf report for the last 5.14431 seconds: | 20260506 01:22:05.389 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:22:05.389 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:05.389 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:05.389 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:05.389 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 01:22:05.389 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:05.389 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:05.389 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 01:22:05.389 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:05.389 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:05.389 | INFO | Uploaded 36GiB to GPU in 2.08814s (17.2402GiB/s) | 20260506 01:22:05.389 | INFO | Downloaded 34GiB to GPU in 2.0391s (16.6741GiB/s) | 20260506 01:22:10.525 | INFO | Analyzing chunk frames = [7834112,7834624) | 20260506 01:22:10.525 | INFO | MemInfo: { | 20260506 01:22:10.525 | INFO | "gpu" : " 17 GB", | 20260506 01:22:10.525 | INFO | "huge" : "", | 20260506 01:22:10.525 | INFO | "huge_shared" : " 27 GB", | 20260506 01:22:10.525 | INFO | "malloc" : " 14 GB", | 20260506 01:22:10.525 | INFO | "pinned" : " 29 GB" | 20260506 01:22:10.525 | INFO | } | 20260506 01:22:10.525 | INFO | Perf report for the last 5.13637 seconds: | 20260506 01:22:10.525 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:22:10.525 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:10.525 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:10.525 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:10.525 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 01:22:10.525 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:10.525 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:10.525 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 01:22:10.525 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:10.525 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:10.526 | INFO | Uploaded 36GiB to GPU in 2.11322s (17.0356GiB/s) | 20260506 01:22:10.526 | INFO | Downloaded 34GiB to GPU in 2.04s (16.6667GiB/s) | 20260506 01:22:15.662 | INFO | Analyzing chunk frames = [7834624,7835136) | 20260506 01:22:15.662 | INFO | MemInfo: { | 20260506 01:22:15.662 | INFO | "gpu" : " 17 GB", | 20260506 01:22:15.662 | INFO | "huge" : "", | 20260506 01:22:15.662 | INFO | "huge_shared" : " 27 GB", | 20260506 01:22:15.662 | INFO | "malloc" : " 14 GB", | 20260506 01:22:15.662 | INFO | "pinned" : " 29 GB" | 20260506 01:22:15.662 | INFO | } | 20260506 01:22:15.662 | INFO | Perf report for the last 5.13672 seconds: | 20260506 01:22:15.662 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:22:15.662 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:15.662 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:15.662 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:15.662 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260506 01:22:15.662 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:15.662 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:15.662 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 01:22:15.662 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:15.662 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:15.662 | INFO | Uploaded 36GiB to GPU in 2.03545s (17.6865GiB/s) | 20260506 01:22:15.662 | INFO | Downloaded 34GiB to GPU in 2.06507s (16.4643GiB/s) | 20260506 01:22:20.801 | INFO | Analyzing chunk frames = [7835136,7835648) | 20260506 01:22:20.801 | INFO | MemInfo: { | 20260506 01:22:20.801 | INFO | "gpu" : " 17 GB", | 20260506 01:22:20.801 | INFO | "huge" : "", | 20260506 01:22:20.801 | INFO | "huge_shared" : " 27 GB", | 20260506 01:22:20.801 | INFO | "malloc" : " 14 GB", | 20260506 01:22:20.801 | INFO | "pinned" : " 29 GB" | 20260506 01:22:20.801 | INFO | } | 20260506 01:22:20.801 | INFO | Perf report for the last 5.13935 seconds: | 20260506 01:22:20.801 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:22:20.801 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:20.801 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:20.801 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:20.801 | INFO | ANALYSIS : 61.8%, 3.0, 96.6ms, 0 | 20260506 01:22:20.801 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:20.801 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:20.801 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 01:22:20.801 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:20.801 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:20.801 | INFO | Uploaded 36GiB to GPU in 2.11648s (17.0093GiB/s) | 20260506 01:22:20.802 | INFO | Downloaded 34GiB to GPU in 2.03344s (16.7204GiB/s) | 20260506 01:22:20.920 | INFO | Starting a round of Baz IO | 20260506 01:22:24.063 | INFO | Populating RTMetrics report | 20260506 01:22:24.063 | INFO | RTMetrics report populated | 20260506 01:22:24.063 | INFO | Opening tmp RTMetrics json file | 20260506 01:22:24.064 | INFO | Serializing RTMetrics to json | 20260506 01:22:24.065 | INFO | Serialization done, swapping tmp file now... | 20260506 01:22:24.065 | INFO | RTMetrics done | 20260506 01:22:24.144 | INFO | Finished a round of Baz IO: Wrote 1.827 GiB in 3.224 seconds | 20260506 01:22:24.144 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.32% / 0.00% / 78.68% | 20260506 01:22:24.144 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0812848% / 3.97364e-06% / 6.75519e-05% | 20260506 01:22:24.144 | INFO | Baz IO overall: | 20260506 01:22:24.144 | INFO | BytesWritten : 4423321260599 | 20260506 01:22:24.144 | INFO | NumEvents : 2079305445097 | 20260506 01:22:25.942 | INFO | Analyzing chunk frames = [7835648,7836160) | 20260506 01:22:25.942 | INFO | MemInfo: { | 20260506 01:22:25.942 | INFO | "gpu" : " 17 GB", | 20260506 01:22:25.942 | INFO | "huge" : "", | 20260506 01:22:25.942 | INFO | "huge_shared" : " 27 GB", | 20260506 01:22:25.942 | INFO | "malloc" : " 14 GB", | 20260506 01:22:25.942 | INFO | "pinned" : " 29 GB" | 20260506 01:22:25.942 | INFO | } | 20260506 01:22:25.942 | INFO | Perf report for the last 5.14078 seconds: | 20260506 01:22:25.942 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:22:25.942 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:25.942 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:25.942 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:25.942 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260506 01:22:25.942 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:25.942 | INFO | RT_METRICS : 43.0%, 1.0, 22.5ms, 0 | 20260506 01:22:25.942 | INFO | PRE_HQ : 59.8%, 1.7, 54.0ms, 0 | 20260506 01:22:25.942 | INFO | BAZWRITER : 61.9%, 3.5, 109.9ms, 0 | 20260506 01:22:25.942 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:22:25.942 | INFO | Uploaded 36GiB to GPU in 2.17065s (16.5849GiB/s) | 20260506 01:22:25.942 | INFO | Downloaded 38GiB to GPU in 2.24152s (16.9528GiB/s) | 20260506 01:22:31.080 | INFO | Analyzing chunk frames = [7836160,7836672) | 20260506 01:22:31.080 | INFO | MemInfo: { | 20260506 01:22:31.080 | INFO | "gpu" : " 17 GB", | 20260506 01:22:31.080 | INFO | "huge" : "", | 20260506 01:22:31.080 | INFO | "huge_shared" : " 27 GB", | 20260506 01:22:31.080 | INFO | "malloc" : " 14 GB", | 20260506 01:22:31.080 | INFO | "pinned" : " 29 GB" | 20260506 01:22:31.080 | INFO | } | 20260506 01:22:31.080 | INFO | Perf report for the last 5.13804 seconds: | 20260506 01:22:31.080 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:22:31.080 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:31.080 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:31.080 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:31.080 | INFO | ANALYSIS : 60.9%, 3.0, 95.1ms, 0 | 20260506 01:22:31.080 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:31.080 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:31.080 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:22:31.080 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:31.080 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:31.081 | INFO | Uploaded 36GiB to GPU in 1.98471s (18.1387GiB/s) | 20260506 01:22:31.081 | INFO | Downloaded 34GiB to GPU in 1.98811s (17.1017GiB/s) | 20260506 01:22:36.221 | INFO | Analyzing chunk frames = [7836672,7837184) | 20260506 01:22:36.221 | INFO | MemInfo: { | 20260506 01:22:36.221 | INFO | "gpu" : " 17 GB", | 20260506 01:22:36.221 | INFO | "huge" : "", | 20260506 01:22:36.221 | INFO | "huge_shared" : " 27 GB", | 20260506 01:22:36.221 | INFO | "malloc" : " 14 GB", | 20260506 01:22:36.221 | INFO | "pinned" : " 29 GB" | 20260506 01:22:36.221 | INFO | } | 20260506 01:22:36.221 | INFO | Perf report for the last 5.14124 seconds: | 20260506 01:22:36.221 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:22:36.221 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:36.222 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:36.222 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:36.222 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 01:22:36.222 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:36.222 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:36.222 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:22:36.222 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:36.222 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:36.222 | INFO | Uploaded 36GiB to GPU in 2.07799s (17.3244GiB/s) | 20260506 01:22:36.222 | INFO | Downloaded 34GiB to GPU in 2.01938s (16.8368GiB/s) | 20260506 01:22:41.357 | INFO | Analyzing chunk frames = [7837184,7837696) | 20260506 01:22:41.357 | INFO | MemInfo: { | 20260506 01:22:41.357 | INFO | "gpu" : " 17 GB", | 20260506 01:22:41.357 | INFO | "huge" : "", | 20260506 01:22:41.357 | INFO | "huge_shared" : " 27 GB", | 20260506 01:22:41.357 | INFO | "malloc" : " 14 GB", | 20260506 01:22:41.357 | INFO | "pinned" : " 29 GB" | 20260506 01:22:41.358 | INFO | } | 20260506 01:22:41.358 | INFO | Perf report for the last 5.13622 seconds: | 20260506 01:22:41.358 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:22:41.358 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:41.358 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:41.358 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:41.358 | INFO | ANALYSIS : 61.6%, 3.0, 95.6ms, 0 | 20260506 01:22:41.358 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:41.358 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:41.358 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 01:22:41.358 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:41.358 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:41.358 | INFO | Uploaded 36GiB to GPU in 2.14481s (16.7847GiB/s) | 20260506 01:22:41.358 | INFO | Downloaded 34GiB to GPU in 2.05494s (16.5455GiB/s) | 20260506 01:22:46.494 | INFO | Analyzing chunk frames = [7837696,7838208) | 20260506 01:22:46.494 | INFO | MemInfo: { | 20260506 01:22:46.494 | INFO | "gpu" : " 17 GB", | 20260506 01:22:46.494 | INFO | "huge" : "", | 20260506 01:22:46.494 | INFO | "huge_shared" : " 27 GB", | 20260506 01:22:46.494 | INFO | "malloc" : " 14 GB", | 20260506 01:22:46.494 | INFO | "pinned" : " 29 GB" | 20260506 01:22:46.494 | INFO | } | 20260506 01:22:46.494 | INFO | Perf report for the last 5.13679 seconds: | 20260506 01:22:46.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:22:46.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:46.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:46.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:46.495 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 01:22:46.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:46.495 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:46.495 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 01:22:46.495 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:46.495 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:46.495 | INFO | Uploaded 36GiB to GPU in 2.10943s (17.0662GiB/s) | 20260506 01:22:46.495 | INFO | Downloaded 34GiB to GPU in 2.0499s (16.5861GiB/s) | 20260506 01:22:51.634 | INFO | Analyzing chunk frames = [7838208,7838720) | 20260506 01:22:51.634 | INFO | MemInfo: { | 20260506 01:22:51.634 | INFO | "gpu" : " 17 GB", | 20260506 01:22:51.634 | INFO | "huge" : "", | 20260506 01:22:51.634 | INFO | "huge_shared" : " 27 GB", | 20260506 01:22:51.634 | INFO | "malloc" : " 14 GB", | 20260506 01:22:51.634 | INFO | "pinned" : " 29 GB" | 20260506 01:22:51.634 | INFO | } | 20260506 01:22:51.634 | INFO | Perf report for the last 5.13982 seconds: | 20260506 01:22:51.634 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:22:51.634 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:51.634 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:51.634 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:51.634 | INFO | ANALYSIS : 61.3%, 3.0, 95.8ms, 0 | 20260506 01:22:51.634 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:51.634 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:51.634 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 01:22:51.634 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:51.634 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:51.635 | INFO | Uploaded 36GiB to GPU in 2.07126s (17.3807GiB/s) | 20260506 01:22:51.635 | INFO | Downloaded 34GiB to GPU in 2.03595s (16.6998GiB/s) | 20260506 01:22:56.782 | INFO | Analyzing chunk frames = [7838720,7839232) | 20260506 01:22:56.782 | INFO | MemInfo: { | 20260506 01:22:56.782 | INFO | "gpu" : " 17 GB", | 20260506 01:22:56.782 | INFO | "huge" : "", | 20260506 01:22:56.782 | INFO | "huge_shared" : " 27 GB", | 20260506 01:22:56.782 | INFO | "malloc" : " 14 GB", | 20260506 01:22:56.782 | INFO | "pinned" : " 29 GB" | 20260506 01:22:56.782 | INFO | } | 20260506 01:22:56.782 | INFO | Perf report for the last 5.14798 seconds: | 20260506 01:22:56.782 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:22:56.782 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:56.782 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:56.782 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:56.782 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260506 01:22:56.782 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:56.782 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:56.782 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 01:22:56.783 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:56.783 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:22:56.783 | INFO | Uploaded 36GiB to GPU in 2.06784s (17.4095GiB/s) | 20260506 01:22:56.783 | INFO | Downloaded 34GiB to GPU in 2.06284s (16.4822GiB/s) | 20260506 01:23:01.920 | INFO | Analyzing chunk frames = [7839232,7839744) | 20260506 01:23:01.920 | INFO | MemInfo: { | 20260506 01:23:01.920 | INFO | "gpu" : " 17 GB", | 20260506 01:23:01.920 | INFO | "huge" : "", | 20260506 01:23:01.920 | INFO | "huge_shared" : " 27 GB", | 20260506 01:23:01.920 | INFO | "malloc" : " 14 GB", | 20260506 01:23:01.920 | INFO | "pinned" : " 29 GB" | 20260506 01:23:01.920 | INFO | } | 20260506 01:23:01.920 | INFO | Perf report for the last 5.13761 seconds: | 20260506 01:23:01.920 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:23:01.920 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:01.920 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:01.920 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:01.920 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260506 01:23:01.920 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:01.920 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:01.920 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 01:23:01.920 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:01.920 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:01.920 | INFO | Uploaded 36GiB to GPU in 2.03487s (17.6915GiB/s) | 20260506 01:23:01.920 | INFO | Downloaded 34GiB to GPU in 2.0235s (16.8025GiB/s) | 20260506 01:23:01.920 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 01:23:01.920 | INFO | Total mem usage: 15.3 GiB | 20260506 01:23:01.920 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 01:23:01.920 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.19%) | 20260506 01:23:01.920 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 01:23:01.920 | INFO | High Water Breakdown: | 20260506 01:23:01.920 | INFO | 60.9% : 8.54 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 01:23:01.920 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 01:23:01.920 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 01:23:01.920 | INFO | ------------------------------------------ | 20260506 01:23:01.920 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:23:01.920 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 01:23:01.921 | INFO | Total mem usage: 26.5 GiB | 20260506 01:23:01.921 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 01:23:01.921 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 01:23:01.921 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260506 01:23:01.921 | INFO | High Water Breakdown: | 20260506 01:23:01.921 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:23:01.921 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:23:01.921 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:23:01.921 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:23:01.921 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:23:01.921 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:23:01.921 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 01:23:01.921 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 01:23:01.921 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:23:01.921 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 01:23:01.921 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 01:23:01.921 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 01:23:01.921 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 01:23:01.921 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 01:23:01.921 | INFO | ------------------------------------------ | 20260506 01:23:01.921 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:23:01.921 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 01:23:01.921 | INFO | Total mem usage: 25.6 GiB | 20260506 01:23:01.921 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 01:23:01.921 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 01:23:01.921 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 01:23:01.921 | INFO | High Water Breakdown: | 20260506 01:23:01.921 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 01:23:01.921 | INFO | ------------------------------------------ | 20260506 01:23:01.921 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:23:01.921 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 01:23:01.921 | INFO | Total mem usage: 15.7 GiB | 20260506 01:23:01.921 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 01:23:01.921 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 01:23:01.921 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 01:23:01.921 | INFO | High Water Breakdown: | 20260506 01:23:01.921 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 01:23:01.921 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 01:23:01.921 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 01:23:01.921 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 01:23:01.921 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 01:23:01.921 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 01:23:01.921 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 01:23:01.921 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 01:23:01.921 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 01:23:01.921 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:23:01.921 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 01:23:01.921 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:23:01.921 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 01:23:01.921 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 01:23:01.921 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 01:23:01.921 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 01:23:01.921 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:23:01.921 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 01:23:01.921 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:23:01.921 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:23:01.921 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:23:01.921 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 01:23:01.921 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 01:23:01.921 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:23:01.921 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 01:23:01.921 | INFO | ------------------------------------------ | 20260506 01:23:01.921 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:23:01.921 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 01:23:01.921 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:23:01.921 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:23:01.921 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.39e+01, 1.35e+01 / 6.75e-01 | 20260506 01:23:01.921 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.04e-01, 1.02e-02 / 3.76e-03 | 20260506 01:23:01.921 | INFO | PerfReport: 11.84%, Baseline -- 3.71e+00 / 4.46e+00, 5.73e-01 / 2.17e-01 | 20260506 01:23:01.921 | INFO | PerfReport: 3.29%, Histogram -- 1.03e+00 / 1.52e+00, 1.93e-01 / 9.41e-02 | 20260506 01:23:01.921 | INFO | PerfReport: 15.98%, DME -- 4.00e+01 / 4.40e+01, 3.90e+00 / 7.77e-01 | 20260506 01:23:01.921 | INFO | PerfReport: 57.46%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.57e+00 / 1.11e+00 | 20260506 01:23:01.921 | INFO | PerfReport: 7.05%, PulseAccumulating -- 2.21e+00 / 2.72e+00, 4.15e-01 / 2.22e-01 | 20260506 01:23:01.921 | INFO | PerfReport: 3.04%, Metrics -- 9.51e-01 / 1.21e+00, 2.52e-01 / 9.65e-02 | 20260506 01:23:01.921 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.30e-01 / 3.76e-01, 5.55e-02 / 2.11e-02 | 20260506 01:23:01.921 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 01:23:01.921 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:23:01.921 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:23:01.922 | INFO | PerfReport: Full Execution -- 9.79e+01 / -nan, 2.42e+01 / 0.00e+00 | 20260506 01:23:01.922 | INFO | PerfReport: 22.18%, Upload -- 2.17e+01 / 0.00e+00, 7.34e+00 / 0.00e+00 | 20260506 01:23:01.922 | INFO | PerfReport: 21.89%, Download -- 2.14e+01 / 0.00e+00, 7.56e+00 / 0.00e+00 | 20260506 01:23:01.922 | INFO | PerfReport: 32.00%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 01:23:01.922 | INFO | PerfReport: 2.05%, LockUpload -- 2.01e+00 / 0.00e+00, 7.78e+00 / 0.00e+00 | 20260506 01:23:01.922 | INFO | PerfReport: 1.28%, LockDownload -- 1.25e+00 / 0.00e+00, 2.85e+00 / 0.00e+00 | 20260506 01:23:01.922 | INFO | PerfReport: 20.59%, LockCompute -- 2.02e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 01:23:01.922 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 2.28e-02 / 0.00e+00 | 20260506 01:23:02.045 | INFO | Starting a round of Baz IO | 20260506 01:23:05.179 | INFO | Populating RTMetrics report | 20260506 01:23:05.180 | INFO | RTMetrics report populated | 20260506 01:23:05.180 | INFO | Opening tmp RTMetrics json file | 20260506 01:23:05.181 | INFO | Serializing RTMetrics to json | 20260506 01:23:05.182 | INFO | Serialization done, swapping tmp file now... | 20260506 01:23:05.182 | INFO | RTMetrics done | 20260506 01:23:05.258 | INFO | Finished a round of Baz IO: Wrote 1.827 GiB in 3.212 seconds | 20260506 01:23:05.258 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.32% / 0.00% / 78.68% | 20260506 01:23:05.258 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0809034% / 0% / 4.76837e-05% | 20260506 01:23:05.258 | INFO | Baz IO overall: | 20260506 01:23:05.258 | INFO | BytesWritten : 4425283491053 | 20260506 01:23:05.258 | INFO | NumEvents : 2080199789527 | 20260506 01:23:07.052 | INFO | Analyzing chunk frames = [7839744,7840256) | 20260506 01:23:07.052 | INFO | MemInfo: { | 20260506 01:23:07.052 | INFO | "gpu" : " 17 GB", | 20260506 01:23:07.052 | INFO | "huge" : "", | 20260506 01:23:07.052 | INFO | "huge_shared" : " 27 GB", | 20260506 01:23:07.052 | INFO | "malloc" : " 14 GB", | 20260506 01:23:07.052 | INFO | "pinned" : " 29 GB" | 20260506 01:23:07.052 | INFO | } | 20260506 01:23:07.052 | INFO | Perf report for the last 5.13267 seconds: | 20260506 01:23:07.052 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:23:07.052 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:07.052 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:07.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:07.052 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260506 01:23:07.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:07.053 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260506 01:23:07.053 | INFO | PRE_HQ : 59.8%, 1.7, 53.8ms, 0 | 20260506 01:23:07.053 | INFO | BAZWRITER : 61.8%, 3.5, 110.6ms, 0 | 20260506 01:23:07.053 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:23:07.053 | INFO | Uploaded 36GiB to GPU in 2.18252s (16.4947GiB/s) | 20260506 01:23:07.053 | INFO | Downloaded 38GiB to GPU in 2.24198s (16.9493GiB/s) | 20260506 01:23:12.195 | INFO | Analyzing chunk frames = [7840256,7840768) | 20260506 01:23:12.195 | INFO | MemInfo: { | 20260506 01:23:12.195 | INFO | "gpu" : " 17 GB", | 20260506 01:23:12.195 | INFO | "huge" : "", | 20260506 01:23:12.195 | INFO | "huge_shared" : " 27 GB", | 20260506 01:23:12.195 | INFO | "malloc" : " 14 GB", | 20260506 01:23:12.195 | INFO | "pinned" : " 29 GB" | 20260506 01:23:12.195 | INFO | } | 20260506 01:23:12.195 | INFO | Perf report for the last 5.14294 seconds: | 20260506 01:23:12.196 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:23:12.196 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:12.196 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:12.196 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:12.196 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 01:23:12.196 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:12.196 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:12.196 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:23:12.196 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:12.196 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:12.196 | INFO | Uploaded 36GiB to GPU in 2.11124s (17.0516GiB/s) | 20260506 01:23:12.196 | INFO | Downloaded 34GiB to GPU in 2.04117s (16.6571GiB/s) | 20260506 01:23:17.338 | INFO | Analyzing chunk frames = [7840768,7841280) | 20260506 01:23:17.338 | INFO | MemInfo: { | 20260506 01:23:17.338 | INFO | "gpu" : " 17 GB", | 20260506 01:23:17.338 | INFO | "huge" : "", | 20260506 01:23:17.338 | INFO | "huge_shared" : " 27 GB", | 20260506 01:23:17.338 | INFO | "malloc" : " 14 GB", | 20260506 01:23:17.338 | INFO | "pinned" : " 29 GB" | 20260506 01:23:17.338 | INFO | } | 20260506 01:23:17.338 | INFO | Perf report for the last 5.14239 seconds: | 20260506 01:23:17.338 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:23:17.338 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:17.338 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:17.338 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:17.338 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 01:23:17.338 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:17.338 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:17.338 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 01:23:17.338 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:17.338 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:17.338 | INFO | Uploaded 36GiB to GPU in 2.08829s (17.239GiB/s) | 20260506 01:23:17.338 | INFO | Downloaded 34GiB to GPU in 2.0236s (16.8018GiB/s) | 20260506 01:23:22.470 | INFO | Analyzing chunk frames = [7841280,7841792) | 20260506 01:23:22.470 | INFO | MemInfo: { | 20260506 01:23:22.470 | INFO | "gpu" : " 17 GB", | 20260506 01:23:22.470 | INFO | "huge" : "", | 20260506 01:23:22.470 | INFO | "huge_shared" : " 27 GB", | 20260506 01:23:22.470 | INFO | "malloc" : " 14 GB", | 20260506 01:23:22.470 | INFO | "pinned" : " 29 GB" | 20260506 01:23:22.470 | INFO | } | 20260506 01:23:22.470 | INFO | Perf report for the last 5.13252 seconds: | 20260506 01:23:22.471 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:23:22.471 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:22.471 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:22.471 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:22.471 | INFO | ANALYSIS : 61.6%, 3.0, 95.6ms, 0 | 20260506 01:23:22.471 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:22.471 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:22.471 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 01:23:22.471 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:22.471 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:22.471 | INFO | Uploaded 36GiB to GPU in 2.15765s (16.6848GiB/s) | 20260506 01:23:22.471 | INFO | Downloaded 34GiB to GPU in 2.06053s (16.5006GiB/s) | 20260506 01:23:27.610 | INFO | Analyzing chunk frames = [7841792,7842304) | 20260506 01:23:27.610 | INFO | MemInfo: { | 20260506 01:23:27.610 | INFO | "gpu" : " 17 GB", | 20260506 01:23:27.610 | INFO | "huge" : "", | 20260506 01:23:27.610 | INFO | "huge_shared" : " 27 GB", | 20260506 01:23:27.610 | INFO | "malloc" : " 14 GB", | 20260506 01:23:27.610 | INFO | "pinned" : " 29 GB" | 20260506 01:23:27.610 | INFO | } | 20260506 01:23:27.610 | INFO | Perf report for the last 5.13937 seconds: | 20260506 01:23:27.610 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:23:27.610 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:27.610 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:27.610 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:27.610 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 01:23:27.610 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:27.610 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:27.610 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:23:27.610 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:27.610 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:27.610 | INFO | Uploaded 36GiB to GPU in 2.09427s (17.1897GiB/s) | 20260506 01:23:27.610 | INFO | Downloaded 34GiB to GPU in 2.04399s (16.6341GiB/s) | 20260506 01:23:32.751 | INFO | Analyzing chunk frames = [7842304,7842816) | 20260506 01:23:32.751 | INFO | MemInfo: { | 20260506 01:23:32.751 | INFO | "gpu" : " 17 GB", | 20260506 01:23:32.751 | INFO | "huge" : "", | 20260506 01:23:32.751 | INFO | "huge_shared" : " 27 GB", | 20260506 01:23:32.751 | INFO | "malloc" : " 14 GB", | 20260506 01:23:32.751 | INFO | "pinned" : " 29 GB" | 20260506 01:23:32.751 | INFO | } | 20260506 01:23:32.751 | INFO | Perf report for the last 5.14129 seconds: | 20260506 01:23:32.751 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:23:32.751 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:32.751 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:32.751 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:32.751 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260506 01:23:32.751 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:32.751 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:32.751 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 01:23:32.751 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:32.751 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:32.751 | INFO | Uploaded 36GiB to GPU in 2.0768s (17.3344GiB/s) | 20260506 01:23:32.752 | INFO | Downloaded 34GiB to GPU in 2.04371s (16.6364GiB/s) | 20260506 01:23:37.896 | INFO | Analyzing chunk frames = [7842816,7843328) | 20260506 01:23:37.896 | INFO | MemInfo: { | 20260506 01:23:37.896 | INFO | "gpu" : " 17 GB", | 20260506 01:23:37.896 | INFO | "huge" : "", | 20260506 01:23:37.896 | INFO | "huge_shared" : " 27 GB", | 20260506 01:23:37.896 | INFO | "malloc" : " 14 GB", | 20260506 01:23:37.896 | INFO | "pinned" : " 29 GB" | 20260506 01:23:37.896 | INFO | } | 20260506 01:23:37.896 | INFO | Perf report for the last 5.14481 seconds: | 20260506 01:23:37.896 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:23:37.896 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:37.896 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:37.896 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:37.896 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260506 01:23:37.896 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:37.896 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:37.896 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 01:23:37.896 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:37.896 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:37.896 | INFO | Uploaded 36GiB to GPU in 2.02969s (17.7367GiB/s) | 20260506 01:23:37.896 | INFO | Downloaded 34GiB to GPU in 2.05309s (16.5604GiB/s) | 20260506 01:23:43.029 | INFO | Analyzing chunk frames = [7843328,7843840) | 20260506 01:23:43.029 | INFO | MemInfo: { | 20260506 01:23:43.029 | INFO | "gpu" : " 17 GB", | 20260506 01:23:43.029 | INFO | "huge" : "", | 20260506 01:23:43.029 | INFO | "huge_shared" : " 27 GB", | 20260506 01:23:43.029 | INFO | "malloc" : " 14 GB", | 20260506 01:23:43.029 | INFO | "pinned" : " 29 GB" | 20260506 01:23:43.029 | INFO | } | 20260506 01:23:43.029 | INFO | Perf report for the last 5.13343 seconds: | 20260506 01:23:43.029 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:23:43.029 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:43.029 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:43.030 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:43.030 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 01:23:43.030 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:43.030 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:43.030 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 01:23:43.030 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:43.030 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:43.030 | INFO | Uploaded 36GiB to GPU in 2.06825s (17.406GiB/s) | 20260506 01:23:43.030 | INFO | Downloaded 34GiB to GPU in 2.02809s (16.7645GiB/s) | 20260506 01:23:43.148 | INFO | Starting a round of Baz IO | 20260506 01:23:46.293 | INFO | Populating RTMetrics report | 20260506 01:23:46.294 | INFO | RTMetrics report populated | 20260506 01:23:46.294 | INFO | Opening tmp RTMetrics json file | 20260506 01:23:46.295 | INFO | Serializing RTMetrics to json | 20260506 01:23:46.297 | INFO | Serialization done, swapping tmp file now... | 20260506 01:23:46.298 | INFO | RTMetrics done | 20260506 01:23:46.372 | INFO | Finished a round of Baz IO: Wrote 1.826 GiB in 3.224 seconds | 20260506 01:23:46.373 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.32% / 0.00% / 78.68% | 20260506 01:23:46.373 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0805577% / 0% / 5.96046e-05% | 20260506 01:23:46.373 | INFO | Baz IO overall: | 20260506 01:23:46.373 | INFO | BytesWritten : 4427244292915 | 20260506 01:23:46.373 | INFO | NumEvents : 2081093375506 | 20260506 01:23:48.166 | INFO | Analyzing chunk frames = [7843840,7844352) | 20260506 01:23:48.166 | INFO | MemInfo: { | 20260506 01:23:48.166 | INFO | "gpu" : " 17 GB", | 20260506 01:23:48.166 | INFO | "huge" : "", | 20260506 01:23:48.166 | INFO | "huge_shared" : " 27 GB", | 20260506 01:23:48.166 | INFO | "malloc" : " 14 GB", | 20260506 01:23:48.166 | INFO | "pinned" : " 29 GB" | 20260506 01:23:48.166 | INFO | } | 20260506 01:23:48.166 | INFO | Perf report for the last 5.13696 seconds: | 20260506 01:23:48.166 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:23:48.166 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:48.166 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:48.166 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:48.167 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260506 01:23:48.167 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:48.167 | INFO | RT_METRICS : 43.2%, 1.0, 22.6ms, 0 | 20260506 01:23:48.167 | INFO | PRE_HQ : 59.8%, 1.7, 54.1ms, 0 | 20260506 01:23:48.167 | INFO | BAZWRITER : 61.8%, 3.5, 110.5ms, 0 | 20260506 01:23:48.167 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:23:48.167 | INFO | Uploaded 36GiB to GPU in 2.18243s (16.4953GiB/s) | 20260506 01:23:48.167 | INFO | Downloaded 38GiB to GPU in 2.24086s (16.9578GiB/s) | 20260506 01:23:53.305 | INFO | Analyzing chunk frames = [7844352,7844864) | 20260506 01:23:53.305 | INFO | MemInfo: { | 20260506 01:23:53.305 | INFO | "gpu" : " 17 GB", | 20260506 01:23:53.305 | INFO | "huge" : "", | 20260506 01:23:53.305 | INFO | "huge_shared" : " 27 GB", | 20260506 01:23:53.305 | INFO | "malloc" : " 14 GB", | 20260506 01:23:53.305 | INFO | "pinned" : " 29 GB" | 20260506 01:23:53.305 | INFO | } | 20260506 01:23:53.305 | INFO | Perf report for the last 5.1387 seconds: | 20260506 01:23:53.305 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:23:53.305 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:53.305 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:53.305 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:53.305 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260506 01:23:53.305 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:53.305 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:53.305 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 01:23:53.305 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:53.305 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:53.305 | INFO | Uploaded 36GiB to GPU in 2.08981s (17.2264GiB/s) | 20260506 01:23:53.305 | INFO | Downloaded 34GiB to GPU in 2.03006s (16.7483GiB/s) | 20260506 01:23:58.444 | INFO | Analyzing chunk frames = [7844864,7845376) | 20260506 01:23:58.444 | INFO | MemInfo: { | 20260506 01:23:58.444 | INFO | "gpu" : " 17 GB", | 20260506 01:23:58.444 | INFO | "huge" : "", | 20260506 01:23:58.444 | INFO | "huge_shared" : " 27 GB", | 20260506 01:23:58.444 | INFO | "malloc" : " 14 GB", | 20260506 01:23:58.444 | INFO | "pinned" : " 29 GB" | 20260506 01:23:58.444 | INFO | } | 20260506 01:23:58.444 | INFO | Perf report for the last 5.13946 seconds: | 20260506 01:23:58.445 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:23:58.445 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:58.445 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:58.445 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:58.445 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 01:23:58.445 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:58.445 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:58.445 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 01:23:58.445 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:58.445 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:23:58.445 | INFO | Uploaded 36GiB to GPU in 2.09676s (17.1694GiB/s) | 20260506 01:23:58.445 | INFO | Downloaded 34GiB to GPU in 2.03423s (16.7139GiB/s) | 20260506 01:24:03.594 | INFO | Analyzing chunk frames = [7845376,7845888) | 20260506 01:24:03.594 | INFO | MemInfo: { | 20260506 01:24:03.594 | INFO | "gpu" : " 17 GB", | 20260506 01:24:03.594 | INFO | "huge" : "", | 20260506 01:24:03.594 | INFO | "huge_shared" : " 27 GB", | 20260506 01:24:03.594 | INFO | "malloc" : " 14 GB", | 20260506 01:24:03.594 | INFO | "pinned" : " 29 GB" | 20260506 01:24:03.594 | INFO | } | 20260506 01:24:03.594 | INFO | Perf report for the last 5.14917 seconds: | 20260506 01:24:03.594 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:24:03.594 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:03.594 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:03.594 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:03.594 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260506 01:24:03.594 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:03.594 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:03.594 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 01:24:03.594 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:03.594 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:03.594 | INFO | Uploaded 36GiB to GPU in 2.14052s (16.8184GiB/s) | 20260506 01:24:03.594 | INFO | Downloaded 34GiB to GPU in 2.01963s (16.8347GiB/s) | 20260506 01:24:08.733 | INFO | Analyzing chunk frames = [7845888,7846400) | 20260506 01:24:08.733 | INFO | MemInfo: { | 20260506 01:24:08.733 | INFO | "gpu" : " 17 GB", | 20260506 01:24:08.734 | INFO | "huge" : "", | 20260506 01:24:08.734 | INFO | "huge_shared" : " 27 GB", | 20260506 01:24:08.734 | INFO | "malloc" : " 14 GB", | 20260506 01:24:08.734 | INFO | "pinned" : " 29 GB" | 20260506 01:24:08.734 | INFO | } | 20260506 01:24:08.734 | INFO | Perf report for the last 5.14012 seconds: | 20260506 01:24:08.734 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:24:08.734 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:08.734 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:08.734 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:08.734 | INFO | ANALYSIS : 60.7%, 3.0, 94.8ms, 0 | 20260506 01:24:08.734 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:08.734 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:08.734 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:24:08.734 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:08.734 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:08.734 | INFO | Uploaded 36GiB to GPU in 2.0939s (17.1928GiB/s) | 20260506 01:24:08.734 | INFO | Downloaded 34GiB to GPU in 2.03342s (16.7206GiB/s) | 20260506 01:24:13.867 | INFO | Analyzing chunk frames = [7846400,7846912) | 20260506 01:24:13.867 | INFO | MemInfo: { | 20260506 01:24:13.867 | INFO | "gpu" : " 17 GB", | 20260506 01:24:13.867 | INFO | "huge" : "", | 20260506 01:24:13.867 | INFO | "huge_shared" : " 27 GB", | 20260506 01:24:13.867 | INFO | "malloc" : " 14 GB", | 20260506 01:24:13.867 | INFO | "pinned" : " 29 GB" | 20260506 01:24:13.867 | INFO | } | 20260506 01:24:13.867 | INFO | Perf report for the last 5.13298 seconds: | 20260506 01:24:13.867 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:24:13.867 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:13.867 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:13.867 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:13.867 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 01:24:13.867 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:13.867 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:13.867 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 01:24:13.867 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:13.867 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:13.867 | INFO | Uploaded 36GiB to GPU in 2.06554s (17.4289GiB/s) | 20260506 01:24:13.867 | INFO | Downloaded 34GiB to GPU in 2.0306s (16.7438GiB/s) | 20260506 01:24:19.005 | INFO | Analyzing chunk frames = [7846912,7847424) | 20260506 01:24:19.005 | INFO | MemInfo: { | 20260506 01:24:19.005 | INFO | "gpu" : " 17 GB", | 20260506 01:24:19.005 | INFO | "huge" : "", | 20260506 01:24:19.005 | INFO | "huge_shared" : " 27 GB", | 20260506 01:24:19.005 | INFO | "malloc" : " 14 GB", | 20260506 01:24:19.005 | INFO | "pinned" : " 29 GB" | 20260506 01:24:19.005 | INFO | } | 20260506 01:24:19.005 | INFO | Perf report for the last 5.13845 seconds: | 20260506 01:24:19.005 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:24:19.005 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:19.005 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:19.005 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:19.005 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260506 01:24:19.005 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:19.005 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:19.005 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 01:24:19.005 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:19.005 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:19.005 | INFO | Uploaded 36GiB to GPU in 2.08176s (17.2931GiB/s) | 20260506 01:24:19.006 | INFO | Downloaded 34GiB to GPU in 2.0884s (16.2804GiB/s) | 20260506 01:24:24.149 | INFO | Analyzing chunk frames = [7847424,7847936) | 20260506 01:24:24.149 | INFO | MemInfo: { | 20260506 01:24:24.149 | INFO | "gpu" : " 17 GB", | 20260506 01:24:24.149 | INFO | "huge" : "", | 20260506 01:24:24.149 | INFO | "huge_shared" : " 27 GB", | 20260506 01:24:24.149 | INFO | "malloc" : " 14 GB", | 20260506 01:24:24.149 | INFO | "pinned" : " 29 GB" | 20260506 01:24:24.150 | INFO | } | 20260506 01:24:24.150 | INFO | Perf report for the last 5.14436 seconds: | 20260506 01:24:24.150 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:24:24.150 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:24.150 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:24.150 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:24.150 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260506 01:24:24.150 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:24.150 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:24.150 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 01:24:24.150 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:24.150 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:24.150 | INFO | Uploaded 36GiB to GPU in 2.12391s (16.9498GiB/s) | 20260506 01:24:24.150 | INFO | Downloaded 34GiB to GPU in 2.01256s (16.8939GiB/s) | 20260506 01:24:24.150 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 01:24:24.150 | INFO | Total mem usage: 15.3 GiB | 20260506 01:24:24.150 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 01:24:24.150 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.19%) | 20260506 01:24:24.150 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 01:24:24.150 | INFO | High Water Breakdown: | 20260506 01:24:24.150 | INFO | 60.9% : 8.54 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 01:24:24.150 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 01:24:24.150 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 01:24:24.150 | INFO | ------------------------------------------ | 20260506 01:24:24.150 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:24:24.150 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 01:24:24.150 | INFO | Total mem usage: 26.5 GiB | 20260506 01:24:24.150 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 01:24:24.150 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 01:24:24.150 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260506 01:24:24.150 | INFO | High Water Breakdown: | 20260506 01:24:24.150 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:24:24.150 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:24:24.150 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:24:24.150 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:24:24.150 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:24:24.150 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:24:24.150 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 01:24:24.150 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 01:24:24.150 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:24:24.150 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 01:24:24.150 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 01:24:24.150 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 01:24:24.150 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 01:24:24.150 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 01:24:24.150 | INFO | ------------------------------------------ | 20260506 01:24:24.150 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:24:24.150 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 01:24:24.150 | INFO | Total mem usage: 25.6 GiB | 20260506 01:24:24.150 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 01:24:24.151 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 01:24:24.151 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 01:24:24.151 | INFO | High Water Breakdown: | 20260506 01:24:24.151 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 01:24:24.151 | INFO | ------------------------------------------ | 20260506 01:24:24.151 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:24:24.151 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 01:24:24.151 | INFO | Total mem usage: 15.7 GiB | 20260506 01:24:24.151 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 01:24:24.151 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 01:24:24.151 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 01:24:24.151 | INFO | High Water Breakdown: | 20260506 01:24:24.151 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 01:24:24.151 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 01:24:24.151 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 01:24:24.151 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 01:24:24.151 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 01:24:24.151 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 01:24:24.151 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 01:24:24.151 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 01:24:24.151 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 01:24:24.151 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:24:24.151 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 01:24:24.151 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:24:24.151 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 01:24:24.151 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 01:24:24.151 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 01:24:24.151 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 01:24:24.151 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:24:24.151 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 01:24:24.151 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:24:24.151 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:24:24.151 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:24:24.151 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 01:24:24.151 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 01:24:24.151 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:24:24.151 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 01:24:24.151 | INFO | ------------------------------------------ | 20260506 01:24:24.151 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:24:24.151 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 01:24:24.151 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:24:24.151 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:24:24.151 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.39e+01, 1.35e+01 / 8.72e-01 | 20260506 01:24:24.151 | INFO | PerfReport: 0.30%, AggregateStats -- 9.23e-02 / 1.06e-01, 1.10e-02 / 6.84e-03 | 20260506 01:24:24.151 | INFO | PerfReport: 11.84%, Baseline -- 3.70e+00 / 4.53e+00, 5.68e-01 / 1.61e-01 | 20260506 01:24:24.151 | INFO | PerfReport: 3.28%, Histogram -- 1.02e+00 / 1.51e+00, 1.87e-01 / 9.26e-02 | 20260506 01:24:24.151 | INFO | PerfReport: 16.01%, DME -- 4.00e+01 / 4.39e+01, 3.90e+00 / 9.82e-01 | 20260506 01:24:24.151 | INFO | PerfReport: 57.41%, FrameLabeling -- 1.79e+01 / 1.95e+01, 1.47e+00 / 1.07e+00 | 20260506 01:24:24.151 | INFO | PerfReport: 7.07%, PulseAccumulating -- 2.21e+00 / 2.71e+00, 4.19e-01 / 2.48e-01 | 20260506 01:24:24.151 | INFO | PerfReport: 3.04%, Metrics -- 9.52e-01 / 1.20e+00, 2.54e-01 / 1.04e-01 | 20260506 01:24:24.151 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.29e-01 / 3.80e-01, 5.57e-02 / 2.61e-02 | 20260506 01:24:24.151 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 01:24:24.151 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:24:24.151 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:24:24.151 | INFO | PerfReport: Full Execution -- 9.81e+01 / -nan, 2.43e+01 / 0.00e+00 | 20260506 01:24:24.151 | INFO | PerfReport: 22.35%, Upload -- 2.19e+01 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260506 01:24:24.151 | INFO | PerfReport: 21.92%, Download -- 2.15e+01 / 0.00e+00, 7.60e+00 / 0.00e+00 | 20260506 01:24:24.151 | INFO | PerfReport: 31.87%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 01:24:24.151 | INFO | PerfReport: 2.09%, LockUpload -- 2.05e+00 / 0.00e+00, 7.93e+00 / 0.00e+00 | 20260506 01:24:24.151 | INFO | PerfReport: 1.31%, LockDownload -- 1.29e+00 / 0.00e+00, 2.86e+00 / 0.00e+00 | 20260506 01:24:24.151 | INFO | PerfReport: 20.44%, LockCompute -- 2.01e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 01:24:24.151 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 2.03e-02 / 0.00e+00 | 20260506 01:24:24.275 | INFO | Starting a round of Baz IO | 20260506 01:24:27.411 | INFO | Populating RTMetrics report | 20260506 01:24:27.412 | INFO | RTMetrics report populated | 20260506 01:24:27.412 | INFO | Opening tmp RTMetrics json file | 20260506 01:24:27.412 | INFO | Serializing RTMetrics to json | 20260506 01:24:27.414 | INFO | Serialization done, swapping tmp file now... | 20260506 01:24:27.414 | INFO | RTMetrics done | 20260506 01:24:27.487 | INFO | Finished a round of Baz IO: Wrote 1.825 GiB in 3.211 seconds | 20260506 01:24:27.487 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.32% / 0.00% / 78.68% | 20260506 01:24:27.487 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.08051% / 0% / 7.54992e-05% | 20260506 01:24:27.487 | INFO | Baz IO overall: | 20260506 01:24:27.487 | INFO | BytesWritten : 4429203766938 | 20260506 01:24:27.487 | INFO | NumEvents : 2081986238048 | 20260506 01:24:29.285 | INFO | Analyzing chunk frames = [7847936,7848448) | 20260506 01:24:29.285 | INFO | MemInfo: { | 20260506 01:24:29.285 | INFO | "gpu" : " 17 GB", | 20260506 01:24:29.285 | INFO | "huge" : "", | 20260506 01:24:29.285 | INFO | "huge_shared" : " 27 GB", | 20260506 01:24:29.285 | INFO | "malloc" : " 14 GB", | 20260506 01:24:29.285 | INFO | "pinned" : " 29 GB" | 20260506 01:24:29.285 | INFO | } | 20260506 01:24:29.285 | INFO | Perf report for the last 5.13555 seconds: | 20260506 01:24:29.285 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:24:29.285 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:29.285 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:29.285 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:29.285 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260506 01:24:29.285 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:29.285 | INFO | RT_METRICS : 43.0%, 1.0, 22.6ms, 0 | 20260506 01:24:29.285 | INFO | PRE_HQ : 59.6%, 1.7, 53.7ms, 0 | 20260506 01:24:29.285 | INFO | BAZWRITER : 61.8%, 3.5, 110.2ms, 0 | 20260506 01:24:29.285 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:24:29.286 | INFO | Uploaded 36GiB to GPU in 2.18781s (16.4548GiB/s) | 20260506 01:24:29.286 | INFO | Downloaded 38GiB to GPU in 2.24511s (16.9257GiB/s) | 20260506 01:24:34.425 | INFO | Analyzing chunk frames = [7848448,7848960) | 20260506 01:24:34.425 | INFO | MemInfo: { | 20260506 01:24:34.425 | INFO | "gpu" : " 17 GB", | 20260506 01:24:34.425 | INFO | "huge" : "", | 20260506 01:24:34.425 | INFO | "huge_shared" : " 27 GB", | 20260506 01:24:34.425 | INFO | "malloc" : " 14 GB", | 20260506 01:24:34.425 | INFO | "pinned" : " 29 GB" | 20260506 01:24:34.425 | INFO | } | 20260506 01:24:34.425 | INFO | Perf report for the last 5.14003 seconds: | 20260506 01:24:34.425 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:24:34.425 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:34.425 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:34.425 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:34.425 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260506 01:24:34.425 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:34.425 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:34.425 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 01:24:34.425 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:34.425 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:34.426 | INFO | Uploaded 36GiB to GPU in 2.11556s (17.0168GiB/s) | 20260506 01:24:34.426 | INFO | Downloaded 34GiB to GPU in 2.0392s (16.6732GiB/s) | 20260506 01:24:39.567 | INFO | Analyzing chunk frames = [7848960,7849472) | 20260506 01:24:39.567 | INFO | MemInfo: { | 20260506 01:24:39.567 | INFO | "gpu" : " 17 GB", | 20260506 01:24:39.567 | INFO | "huge" : "", | 20260506 01:24:39.567 | INFO | "huge_shared" : " 27 GB", | 20260506 01:24:39.567 | INFO | "malloc" : " 14 GB", | 20260506 01:24:39.567 | INFO | "pinned" : " 29 GB" | 20260506 01:24:39.567 | INFO | } | 20260506 01:24:39.567 | INFO | Perf report for the last 5.1422 seconds: | 20260506 01:24:39.568 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:24:39.568 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:39.568 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:39.568 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:39.568 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 01:24:39.568 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:39.568 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:39.568 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:24:39.568 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:39.568 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:39.568 | INFO | Uploaded 36GiB to GPU in 2.08282s (17.2843GiB/s) | 20260506 01:24:39.568 | INFO | Downloaded 34GiB to GPU in 2.01937s (16.837GiB/s) | 20260506 01:24:44.707 | INFO | Analyzing chunk frames = [7849472,7849984) | 20260506 01:24:44.707 | INFO | MemInfo: { | 20260506 01:24:44.707 | INFO | "gpu" : " 17 GB", | 20260506 01:24:44.707 | INFO | "huge" : "", | 20260506 01:24:44.707 | INFO | "huge_shared" : " 27 GB", | 20260506 01:24:44.707 | INFO | "malloc" : " 14 GB", | 20260506 01:24:44.707 | INFO | "pinned" : " 29 GB" | 20260506 01:24:44.707 | INFO | } | 20260506 01:24:44.708 | INFO | Perf report for the last 5.14009 seconds: | 20260506 01:24:44.708 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:24:44.708 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:44.708 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:44.708 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:44.708 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260506 01:24:44.708 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:44.708 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:44.708 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:24:44.708 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:44.708 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:44.708 | INFO | Uploaded 36GiB to GPU in 2.15268s (16.7233GiB/s) | 20260506 01:24:44.708 | INFO | Downloaded 34GiB to GPU in 2.05661s (16.5321GiB/s) | 20260506 01:24:49.844 | INFO | Analyzing chunk frames = [7849984,7850496) | 20260506 01:24:49.844 | INFO | MemInfo: { | 20260506 01:24:49.844 | INFO | "gpu" : " 17 GB", | 20260506 01:24:49.844 | INFO | "huge" : "", | 20260506 01:24:49.844 | INFO | "huge_shared" : " 27 GB", | 20260506 01:24:49.844 | INFO | "malloc" : " 14 GB", | 20260506 01:24:49.844 | INFO | "pinned" : " 29 GB" | 20260506 01:24:49.844 | INFO | } | 20260506 01:24:49.844 | INFO | Perf report for the last 5.13634 seconds: | 20260506 01:24:49.844 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:24:49.844 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:49.844 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:49.844 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:49.844 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260506 01:24:49.844 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:49.844 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:49.844 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 01:24:49.844 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:49.844 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:49.844 | INFO | Uploaded 36GiB to GPU in 2.10175s (17.1286GiB/s) | 20260506 01:24:49.844 | INFO | Downloaded 34GiB to GPU in 2.03123s (16.7386GiB/s) | 20260506 01:24:54.980 | INFO | Analyzing chunk frames = [7850496,7851008) | 20260506 01:24:54.980 | INFO | MemInfo: { | 20260506 01:24:54.980 | INFO | "gpu" : " 17 GB", | 20260506 01:24:54.980 | INFO | "huge" : "", | 20260506 01:24:54.980 | INFO | "huge_shared" : " 27 GB", | 20260506 01:24:54.980 | INFO | "malloc" : " 14 GB", | 20260506 01:24:54.980 | INFO | "pinned" : " 29 GB" | 20260506 01:24:54.980 | INFO | } | 20260506 01:24:54.980 | INFO | Perf report for the last 5.13579 seconds: | 20260506 01:24:54.980 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:24:54.980 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:54.980 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:54.980 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:54.980 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 01:24:54.980 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:54.980 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:54.980 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:24:54.980 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:54.980 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:24:54.980 | INFO | Uploaded 36GiB to GPU in 2.07474s (17.3515GiB/s) | 20260506 01:24:54.980 | INFO | Downloaded 34GiB to GPU in 2.0393s (16.6724GiB/s) | 20260506 01:25:00.117 | INFO | Analyzing chunk frames = [7851008,7851520) | 20260506 01:25:00.117 | INFO | MemInfo: { | 20260506 01:25:00.117 | INFO | "gpu" : " 17 GB", | 20260506 01:25:00.117 | INFO | "huge" : "", | 20260506 01:25:00.117 | INFO | "huge_shared" : " 27 GB", | 20260506 01:25:00.117 | INFO | "malloc" : " 14 GB", | 20260506 01:25:00.117 | INFO | "pinned" : " 29 GB" | 20260506 01:25:00.117 | INFO | } | 20260506 01:25:00.117 | INFO | Perf report for the last 5.13725 seconds: | 20260506 01:25:00.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:25:00.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:00.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:00.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:00.117 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260506 01:25:00.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:00.117 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:00.117 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 01:25:00.117 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:00.117 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:00.117 | INFO | Uploaded 36GiB to GPU in 2.1232s (16.9555GiB/s) | 20260506 01:25:00.117 | INFO | Downloaded 34GiB to GPU in 2.10889s (16.1223GiB/s) | 20260506 01:25:05.257 | INFO | Analyzing chunk frames = [7851520,7852032) | 20260506 01:25:05.257 | INFO | MemInfo: { | 20260506 01:25:05.257 | INFO | "gpu" : " 17 GB", | 20260506 01:25:05.257 | INFO | "huge" : "", | 20260506 01:25:05.257 | INFO | "huge_shared" : " 27 GB", | 20260506 01:25:05.257 | INFO | "malloc" : " 14 GB", | 20260506 01:25:05.257 | INFO | "pinned" : " 29 GB" | 20260506 01:25:05.257 | INFO | } | 20260506 01:25:05.257 | INFO | Perf report for the last 5.14051 seconds: | 20260506 01:25:05.257 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:25:05.257 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:05.257 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:05.257 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:05.258 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260506 01:25:05.258 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:05.258 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:05.258 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 01:25:05.258 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:05.258 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:05.258 | INFO | Uploaded 36GiB to GPU in 2.11534s (17.0185GiB/s) | 20260506 01:25:05.258 | INFO | Downloaded 34GiB to GPU in 1.99827s (17.0147GiB/s) | 20260506 01:25:05.378 | INFO | Starting a round of Baz IO | 20260506 01:25:08.523 | INFO | Populating RTMetrics report | 20260506 01:25:08.523 | INFO | RTMetrics report populated | 20260506 01:25:08.523 | INFO | Opening tmp RTMetrics json file | 20260506 01:25:08.524 | INFO | Serializing RTMetrics to json | 20260506 01:25:08.526 | INFO | Serialization done, swapping tmp file now... | 20260506 01:25:08.526 | INFO | RTMetrics done | 20260506 01:25:08.599 | INFO | Finished a round of Baz IO: Wrote 1.824 GiB in 3.221 seconds | 20260506 01:25:08.599 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.32% / 0.00% / 78.68% | 20260506 01:25:08.599 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.080212% / 0% / 0.000103315% | 20260506 01:25:08.599 | INFO | Baz IO overall: | 20260506 01:25:08.599 | INFO | BytesWritten : 4431162259480 | 20260506 01:25:08.599 | INFO | NumEvents : 2082878352754 | 20260506 01:25:10.398 | INFO | Analyzing chunk frames = [7852032,7852544) | 20260506 01:25:10.398 | INFO | MemInfo: { | 20260506 01:25:10.398 | INFO | "gpu" : " 17 GB", | 20260506 01:25:10.398 | INFO | "huge" : "", | 20260506 01:25:10.398 | INFO | "huge_shared" : " 27 GB", | 20260506 01:25:10.398 | INFO | "malloc" : " 14 GB", | 20260506 01:25:10.398 | INFO | "pinned" : " 29 GB" | 20260506 01:25:10.398 | INFO | } | 20260506 01:25:10.398 | INFO | Perf report for the last 5.14063 seconds: | 20260506 01:25:10.398 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:25:10.398 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:10.398 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:10.398 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:10.398 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260506 01:25:10.398 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:10.398 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260506 01:25:10.398 | INFO | PRE_HQ : 59.7%, 1.7, 53.6ms, 0 | 20260506 01:25:10.398 | INFO | BAZWRITER : 61.8%, 3.4, 109.5ms, 0 | 20260506 01:25:10.398 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:25:10.398 | INFO | Uploaded 36GiB to GPU in 2.17706s (16.536GiB/s) | 20260506 01:25:10.398 | INFO | Downloaded 38GiB to GPU in 2.24413s (16.9331GiB/s) | 20260506 01:25:15.547 | INFO | Analyzing chunk frames = [7852544,7853056) | 20260506 01:25:15.547 | INFO | MemInfo: { | 20260506 01:25:15.547 | INFO | "gpu" : " 17 GB", | 20260506 01:25:15.547 | INFO | "huge" : "", | 20260506 01:25:15.547 | INFO | "huge_shared" : " 27 GB", | 20260506 01:25:15.547 | INFO | "malloc" : " 14 GB", | 20260506 01:25:15.547 | INFO | "pinned" : " 29 GB" | 20260506 01:25:15.547 | INFO | } | 20260506 01:25:15.547 | INFO | Perf report for the last 5.14903 seconds: | 20260506 01:25:15.547 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:25:15.547 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:15.547 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:15.547 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:15.547 | INFO | ANALYSIS : 60.9%, 3.0, 95.3ms, 0 | 20260506 01:25:15.547 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:15.547 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:15.547 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 01:25:15.547 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:15.547 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:15.547 | INFO | Uploaded 36GiB to GPU in 1.99369s (18.0569GiB/s) | 20260506 01:25:15.547 | INFO | Downloaded 34GiB to GPU in 1.98701s (17.1112GiB/s) | 20260506 01:25:20.680 | INFO | Analyzing chunk frames = [7853056,7853568) | 20260506 01:25:20.680 | INFO | MemInfo: { | 20260506 01:25:20.680 | INFO | "gpu" : " 17 GB", | 20260506 01:25:20.680 | INFO | "huge" : "", | 20260506 01:25:20.680 | INFO | "huge_shared" : " 27 GB", | 20260506 01:25:20.680 | INFO | "malloc" : " 14 GB", | 20260506 01:25:20.680 | INFO | "pinned" : " 29 GB" | 20260506 01:25:20.680 | INFO | } | 20260506 01:25:20.680 | INFO | Perf report for the last 5.13336 seconds: | 20260506 01:25:20.680 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:25:20.680 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:20.680 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:20.680 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:20.681 | INFO | ANALYSIS : 61.3%, 3.0, 95.2ms, 0 | 20260506 01:25:20.681 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:20.681 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:20.681 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:25:20.681 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:20.681 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:20.681 | INFO | Uploaded 36GiB to GPU in 2.0916s (17.2117GiB/s) | 20260506 01:25:20.681 | INFO | Downloaded 34GiB to GPU in 2.03206s (16.7318GiB/s) | 20260506 01:25:25.810 | INFO | Analyzing chunk frames = [7853568,7854080) | 20260506 01:25:25.810 | INFO | MemInfo: { | 20260506 01:25:25.810 | INFO | "gpu" : " 17 GB", | 20260506 01:25:25.810 | INFO | "huge" : "", | 20260506 01:25:25.810 | INFO | "huge_shared" : " 27 GB", | 20260506 01:25:25.810 | INFO | "malloc" : " 14 GB", | 20260506 01:25:25.810 | INFO | "pinned" : " 29 GB" | 20260506 01:25:25.810 | INFO | } | 20260506 01:25:25.811 | INFO | Perf report for the last 5.13017 seconds: | 20260506 01:25:25.811 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:25:25.811 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:25.811 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:25.811 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:25.811 | INFO | ANALYSIS : 61.6%, 3.0, 95.5ms, 0 | 20260506 01:25:25.811 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:25.811 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:25.811 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 01:25:25.811 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:25.811 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:25.811 | INFO | Uploaded 36GiB to GPU in 2.14526s (16.7812GiB/s) | 20260506 01:25:25.811 | INFO | Downloaded 34GiB to GPU in 2.04715s (16.6085GiB/s) | 20260506 01:25:30.956 | INFO | Analyzing chunk frames = [7854080,7854592) | 20260506 01:25:30.956 | INFO | MemInfo: { | 20260506 01:25:30.956 | INFO | "gpu" : " 17 GB", | 20260506 01:25:30.956 | INFO | "huge" : "", | 20260506 01:25:30.956 | INFO | "huge_shared" : " 27 GB", | 20260506 01:25:30.956 | INFO | "malloc" : " 14 GB", | 20260506 01:25:30.956 | INFO | "pinned" : " 29 GB" | 20260506 01:25:30.956 | INFO | } | 20260506 01:25:30.957 | INFO | Perf report for the last 5.14584 seconds: | 20260506 01:25:30.957 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:25:30.957 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:30.957 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:30.957 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:30.957 | INFO | ANALYSIS : 61.0%, 3.0, 95.4ms, 0 | 20260506 01:25:30.957 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:30.957 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:30.957 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:25:30.957 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:30.957 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:30.957 | INFO | Uploaded 36GiB to GPU in 2.09338s (17.1971GiB/s) | 20260506 01:25:30.957 | INFO | Downloaded 34GiB to GPU in 2.04531s (16.6234GiB/s) | 20260506 01:25:36.097 | INFO | Analyzing chunk frames = [7854592,7855104) | 20260506 01:25:36.097 | INFO | MemInfo: { | 20260506 01:25:36.097 | INFO | "gpu" : " 17 GB", | 20260506 01:25:36.097 | INFO | "huge" : "", | 20260506 01:25:36.097 | INFO | "huge_shared" : " 27 GB", | 20260506 01:25:36.097 | INFO | "malloc" : " 14 GB", | 20260506 01:25:36.097 | INFO | "pinned" : " 29 GB" | 20260506 01:25:36.097 | INFO | } | 20260506 01:25:36.097 | INFO | Perf report for the last 5.1403 seconds: | 20260506 01:25:36.097 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:25:36.097 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:36.097 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:36.097 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:36.097 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 01:25:36.097 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:36.097 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:36.097 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 01:25:36.097 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:36.097 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:36.097 | INFO | Uploaded 36GiB to GPU in 2.07092s (17.3836GiB/s) | 20260506 01:25:36.097 | INFO | Downloaded 34GiB to GPU in 2.04204s (16.65GiB/s) | 20260506 01:25:41.242 | INFO | Analyzing chunk frames = [7855104,7855616) | 20260506 01:25:41.242 | INFO | MemInfo: { | 20260506 01:25:41.242 | INFO | "gpu" : " 17 GB", | 20260506 01:25:41.242 | INFO | "huge" : "", | 20260506 01:25:41.242 | INFO | "huge_shared" : " 27 GB", | 20260506 01:25:41.242 | INFO | "malloc" : " 14 GB", | 20260506 01:25:41.242 | INFO | "pinned" : " 29 GB" | 20260506 01:25:41.242 | INFO | } | 20260506 01:25:41.242 | INFO | Perf report for the last 5.14472 seconds: | 20260506 01:25:41.242 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:25:41.242 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:41.242 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:41.242 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:41.242 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260506 01:25:41.242 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:41.242 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:41.242 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 01:25:41.242 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:41.242 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:41.242 | INFO | Uploaded 36GiB to GPU in 2.02433s (17.7837GiB/s) | 20260506 01:25:41.242 | INFO | Downloaded 34GiB to GPU in 2.03597s (16.6996GiB/s) | 20260506 01:25:46.377 | INFO | Analyzing chunk frames = [7855616,7856128) | 20260506 01:25:46.377 | INFO | MemInfo: { | 20260506 01:25:46.377 | INFO | "gpu" : " 17 GB", | 20260506 01:25:46.377 | INFO | "huge" : "", | 20260506 01:25:46.377 | INFO | "huge_shared" : " 27 GB", | 20260506 01:25:46.377 | INFO | "malloc" : " 14 GB", | 20260506 01:25:46.377 | INFO | "pinned" : " 29 GB" | 20260506 01:25:46.377 | INFO | } | 20260506 01:25:46.377 | INFO | Perf report for the last 5.13546 seconds: | 20260506 01:25:46.377 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:25:46.377 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:46.377 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:46.377 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:46.377 | INFO | ANALYSIS : 61.5%, 3.0, 96.0ms, 0 | 20260506 01:25:46.377 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:46.377 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:46.377 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:25:46.377 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:46.377 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:46.377 | INFO | Uploaded 36GiB to GPU in 2.06471s (17.4358GiB/s) | 20260506 01:25:46.377 | INFO | Downloaded 34GiB to GPU in 2.01814s (16.8472GiB/s) | 20260506 01:25:46.378 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 01:25:46.378 | INFO | Total mem usage: 15.3 GiB | 20260506 01:25:46.378 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 01:25:46.378 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.2%) | 20260506 01:25:46.378 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 01:25:46.378 | INFO | High Water Breakdown: | 20260506 01:25:46.378 | INFO | 60.9% : 8.54 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 01:25:46.378 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 01:25:46.378 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 01:25:46.378 | INFO | ------------------------------------------ | 20260506 01:25:46.378 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:25:46.378 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 01:25:46.378 | INFO | Total mem usage: 26.5 GiB | 20260506 01:25:46.378 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 01:25:46.378 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 01:25:46.378 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260506 01:25:46.378 | INFO | High Water Breakdown: | 20260506 01:25:46.378 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:25:46.378 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:25:46.378 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:25:46.378 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:25:46.378 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:25:46.378 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:25:46.378 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 01:25:46.378 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 01:25:46.378 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:25:46.378 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 01:25:46.378 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 01:25:46.378 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 01:25:46.378 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 01:25:46.378 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 01:25:46.378 | INFO | ------------------------------------------ | 20260506 01:25:46.378 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:25:46.378 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 01:25:46.378 | INFO | Total mem usage: 25.6 GiB | 20260506 01:25:46.378 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 01:25:46.378 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 01:25:46.378 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 01:25:46.378 | INFO | High Water Breakdown: | 20260506 01:25:46.378 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 01:25:46.378 | INFO | ------------------------------------------ | 20260506 01:25:46.378 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:25:46.378 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 01:25:46.378 | INFO | Total mem usage: 15.7 GiB | 20260506 01:25:46.378 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 01:25:46.378 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 01:25:46.378 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 01:25:46.378 | INFO | High Water Breakdown: | 20260506 01:25:46.378 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 01:25:46.378 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 01:25:46.378 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 01:25:46.378 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 01:25:46.378 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 01:25:46.378 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 01:25:46.378 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 01:25:46.378 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 01:25:46.378 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 01:25:46.378 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:25:46.378 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 01:25:46.378 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:25:46.378 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 01:25:46.378 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 01:25:46.378 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 01:25:46.378 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 01:25:46.378 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:25:46.378 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 01:25:46.378 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:25:46.378 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:25:46.378 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:25:46.378 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 01:25:46.378 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 01:25:46.378 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:25:46.378 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 01:25:46.378 | INFO | ------------------------------------------ | 20260506 01:25:46.378 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:25:46.379 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 01:25:46.379 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:25:46.379 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:25:46.379 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.40e+01, 1.34e+01 / 6.68e-01 | 20260506 01:25:46.379 | INFO | PerfReport: 0.30%, AggregateStats -- 9.23e-02 / 1.05e-01, 1.09e-02 / 4.10e-03 | 20260506 01:25:46.379 | INFO | PerfReport: 11.84%, Baseline -- 3.71e+00 / 4.53e+00, 5.70e-01 / 1.43e-01 | 20260506 01:25:46.379 | INFO | PerfReport: 3.29%, Histogram -- 1.03e+00 / 1.53e+00, 1.89e-01 / 9.24e-02 | 20260506 01:25:46.379 | INFO | PerfReport: 16.01%, DME -- 4.01e+01 / 4.40e+01, 3.82e+00 / 7.84e-01 | 20260506 01:25:46.379 | INFO | PerfReport: 57.40%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.48e+00 / 1.08e+00 | 20260506 01:25:46.379 | INFO | PerfReport: 7.07%, PulseAccumulating -- 2.21e+00 / 2.72e+00, 4.17e-01 / 2.46e-01 | 20260506 01:25:46.379 | INFO | PerfReport: 3.04%, Metrics -- 9.51e-01 / 1.21e+00, 2.52e-01 / 9.91e-02 | 20260506 01:25:46.379 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.29e-01 / 3.81e-01, 5.50e-02 / 2.48e-02 | 20260506 01:25:46.379 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 01:25:46.379 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:25:46.379 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:25:46.379 | INFO | PerfReport: Full Execution -- 9.80e+01 / -nan, 2.43e+01 / 0.00e+00 | 20260506 01:25:46.379 | INFO | PerfReport: 22.32%, Upload -- 2.19e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260506 01:25:46.379 | INFO | PerfReport: 21.91%, Download -- 2.15e+01 / 0.00e+00, 7.57e+00 / 0.00e+00 | 20260506 01:25:46.379 | INFO | PerfReport: 31.91%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 01:25:46.379 | INFO | PerfReport: 2.06%, LockUpload -- 2.02e+00 / 0.00e+00, 7.82e+00 / 0.00e+00 | 20260506 01:25:46.379 | INFO | PerfReport: 1.32%, LockDownload -- 1.30e+00 / 0.00e+00, 2.87e+00 / 0.00e+00 | 20260506 01:25:46.379 | INFO | PerfReport: 20.46%, LockCompute -- 2.01e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 01:25:46.379 | INFO | PerfReport: 0.02%, Misc -- 1.49e-02 / 0.00e+00, 2.69e-02 / 0.00e+00 | 20260506 01:25:46.495 | INFO | Starting a round of Baz IO | 20260506 01:25:49.646 | INFO | Populating RTMetrics report | 20260506 01:25:49.646 | INFO | RTMetrics report populated | 20260506 01:25:49.646 | INFO | Opening tmp RTMetrics json file | 20260506 01:25:49.647 | INFO | Serializing RTMetrics to json | 20260506 01:25:49.649 | INFO | Serialization done, swapping tmp file now... | 20260506 01:25:49.649 | INFO | RTMetrics done | 20260506 01:25:49.717 | INFO | Finished a round of Baz IO: Wrote 1.825 GiB in 3.222 seconds | 20260506 01:25:49.717 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.32% / 0.00% / 78.68% | 20260506 01:25:49.717 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0805259% / 3.97364e-06% / 6.35783e-05% | 20260506 01:25:49.717 | INFO | Baz IO overall: | 20260506 01:25:49.717 | INFO | BytesWritten : 4433121666157 | 20260506 01:25:49.717 | INFO | NumEvents : 2083771147701 | 20260506 01:25:51.517 | INFO | Analyzing chunk frames = [7856128,7856640) | 20260506 01:25:51.517 | INFO | MemInfo: { | 20260506 01:25:51.517 | INFO | "gpu" : " 17 GB", | 20260506 01:25:51.517 | INFO | "huge" : "", | 20260506 01:25:51.517 | INFO | "huge_shared" : " 27 GB", | 20260506 01:25:51.517 | INFO | "malloc" : " 14 GB", | 20260506 01:25:51.518 | INFO | "pinned" : " 29 GB" | 20260506 01:25:51.518 | INFO | } | 20260506 01:25:51.518 | INFO | Perf report for the last 5.14056 seconds: | 20260506 01:25:51.518 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:25:51.518 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:51.518 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:51.518 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:51.518 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260506 01:25:51.518 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:51.518 | INFO | RT_METRICS : 43.0%, 1.0, 22.6ms, 0 | 20260506 01:25:51.518 | INFO | PRE_HQ : 59.6%, 1.7, 53.6ms, 0 | 20260506 01:25:51.518 | INFO | BAZWRITER : 61.8%, 3.5, 109.7ms, 0 | 20260506 01:25:51.518 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:25:51.518 | INFO | Uploaded 36GiB to GPU in 2.17742s (16.5333GiB/s) | 20260506 01:25:51.518 | INFO | Downloaded 38GiB to GPU in 2.23558s (16.9978GiB/s) | 20260506 01:25:56.660 | INFO | Analyzing chunk frames = [7856640,7857152) | 20260506 01:25:56.660 | INFO | MemInfo: { | 20260506 01:25:56.660 | INFO | "gpu" : " 17 GB", | 20260506 01:25:56.660 | INFO | "huge" : "", | 20260506 01:25:56.660 | INFO | "huge_shared" : " 27 GB", | 20260506 01:25:56.660 | INFO | "malloc" : " 14 GB", | 20260506 01:25:56.660 | INFO | "pinned" : " 29 GB" | 20260506 01:25:56.660 | INFO | } | 20260506 01:25:56.660 | INFO | Perf report for the last 5.14265 seconds: | 20260506 01:25:56.660 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:25:56.660 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:56.660 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:56.660 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:56.661 | INFO | ANALYSIS : 60.7%, 3.0, 94.8ms, 0 | 20260506 01:25:56.661 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:56.661 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:56.661 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 01:25:56.661 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:56.661 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:25:56.661 | INFO | Uploaded 36GiB to GPU in 1.97216s (18.2541GiB/s) | 20260506 01:25:56.661 | INFO | Downloaded 34GiB to GPU in 1.98296s (17.1461GiB/s) | 20260506 01:26:01.794 | INFO | Analyzing chunk frames = [7857152,7857664) | 20260506 01:26:01.794 | INFO | MemInfo: { | 20260506 01:26:01.794 | INFO | "gpu" : " 17 GB", | 20260506 01:26:01.794 | INFO | "huge" : "", | 20260506 01:26:01.794 | INFO | "huge_shared" : " 27 GB", | 20260506 01:26:01.794 | INFO | "malloc" : " 14 GB", | 20260506 01:26:01.794 | INFO | "pinned" : " 29 GB" | 20260506 01:26:01.794 | INFO | } | 20260506 01:26:01.794 | INFO | Perf report for the last 5.13363 seconds: | 20260506 01:26:01.794 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:26:01.794 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:01.794 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:01.794 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:01.794 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 01:26:01.794 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:01.794 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:01.794 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 01:26:01.794 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:01.794 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:01.794 | INFO | Uploaded 36GiB to GPU in 2.10213s (17.1255GiB/s) | 20260506 01:26:01.794 | INFO | Downloaded 34GiB to GPU in 2.0423s (16.6479GiB/s) | 20260506 01:26:06.928 | INFO | Analyzing chunk frames = [7857664,7858176) | 20260506 01:26:06.928 | INFO | MemInfo: { | 20260506 01:26:06.928 | INFO | "gpu" : " 17 GB", | 20260506 01:26:06.928 | INFO | "huge" : "", | 20260506 01:26:06.928 | INFO | "huge_shared" : " 27 GB", | 20260506 01:26:06.928 | INFO | "malloc" : " 14 GB", | 20260506 01:26:06.929 | INFO | "pinned" : " 29 GB" | 20260506 01:26:06.929 | INFO | } | 20260506 01:26:06.929 | INFO | Perf report for the last 5.13464 seconds: | 20260506 01:26:06.929 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:26:06.929 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:06.929 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:06.929 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:06.929 | INFO | ANALYSIS : 61.6%, 3.0, 95.6ms, 0 | 20260506 01:26:06.929 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:06.929 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:06.929 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 01:26:06.929 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:06.929 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:06.929 | INFO | Uploaded 36GiB to GPU in 2.13466s (16.8645GiB/s) | 20260506 01:26:06.929 | INFO | Downloaded 34GiB to GPU in 2.02018s (16.8302GiB/s) | 20260506 01:26:12.064 | INFO | Analyzing chunk frames = [7858176,7858688) | 20260506 01:26:12.064 | INFO | MemInfo: { | 20260506 01:26:12.064 | INFO | "gpu" : " 17 GB", | 20260506 01:26:12.064 | INFO | "huge" : "", | 20260506 01:26:12.064 | INFO | "huge_shared" : " 27 GB", | 20260506 01:26:12.064 | INFO | "malloc" : " 14 GB", | 20260506 01:26:12.064 | INFO | "pinned" : " 29 GB" | 20260506 01:26:12.065 | INFO | } | 20260506 01:26:12.065 | INFO | Perf report for the last 5.13608 seconds: | 20260506 01:26:12.065 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:26:12.065 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:12.065 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:12.065 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:12.065 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 01:26:12.065 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:12.065 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:12.065 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 01:26:12.065 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:12.065 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:12.065 | INFO | Uploaded 36GiB to GPU in 2.09457s (17.1873GiB/s) | 20260506 01:26:12.065 | INFO | Downloaded 34GiB to GPU in 2.04913s (16.5924GiB/s) | 20260506 01:26:17.215 | INFO | Analyzing chunk frames = [7858688,7859200) | 20260506 01:26:17.215 | INFO | MemInfo: { | 20260506 01:26:17.215 | INFO | "gpu" : " 17 GB", | 20260506 01:26:17.215 | INFO | "huge" : "", | 20260506 01:26:17.215 | INFO | "huge_shared" : " 27 GB", | 20260506 01:26:17.215 | INFO | "malloc" : " 14 GB", | 20260506 01:26:17.215 | INFO | "pinned" : " 29 GB" | 20260506 01:26:17.215 | INFO | } | 20260506 01:26:17.215 | INFO | Perf report for the last 5.15051 seconds: | 20260506 01:26:17.215 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:26:17.215 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:17.215 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:17.215 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:17.215 | INFO | ANALYSIS : 61.1%, 3.0, 95.6ms, 0 | 20260506 01:26:17.215 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:17.215 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:17.215 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 01:26:17.215 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:17.215 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:17.216 | INFO | Uploaded 36GiB to GPU in 2.06754s (17.412GiB/s) | 20260506 01:26:17.216 | INFO | Downloaded 34GiB to GPU in 2.03988s (16.6677GiB/s) | 20260506 01:26:22.355 | INFO | Analyzing chunk frames = [7859200,7859712) | 20260506 01:26:22.355 | INFO | MemInfo: { | 20260506 01:26:22.355 | INFO | "gpu" : " 17 GB", | 20260506 01:26:22.355 | INFO | "huge" : "", | 20260506 01:26:22.355 | INFO | "huge_shared" : " 27 GB", | 20260506 01:26:22.355 | INFO | "malloc" : " 14 GB", | 20260506 01:26:22.355 | INFO | "pinned" : " 29 GB" | 20260506 01:26:22.355 | INFO | } | 20260506 01:26:22.355 | INFO | Perf report for the last 5.14023 seconds: | 20260506 01:26:22.356 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:26:22.356 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:22.356 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:22.356 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:22.356 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260506 01:26:22.356 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:22.356 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:22.356 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260506 01:26:22.356 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:22.356 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:22.356 | INFO | Uploaded 36GiB to GPU in 2.03056s (17.7291GiB/s) | 20260506 01:26:22.356 | INFO | Downloaded 34GiB to GPU in 2.05141s (16.574GiB/s) | 20260506 01:26:27.496 | INFO | Analyzing chunk frames = [7859712,7860224) | 20260506 01:26:27.496 | INFO | MemInfo: { | 20260506 01:26:27.496 | INFO | "gpu" : " 17 GB", | 20260506 01:26:27.496 | INFO | "huge" : "", | 20260506 01:26:27.496 | INFO | "huge_shared" : " 27 GB", | 20260506 01:26:27.496 | INFO | "malloc" : " 14 GB", | 20260506 01:26:27.496 | INFO | "pinned" : " 29 GB" | 20260506 01:26:27.496 | INFO | } | 20260506 01:26:27.496 | INFO | Perf report for the last 5.14047 seconds: | 20260506 01:26:27.496 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:26:27.496 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:27.496 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:27.496 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:27.496 | INFO | ANALYSIS : 61.4%, 3.0, 95.9ms, 0 | 20260506 01:26:27.496 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:27.496 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:27.496 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 01:26:27.496 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:27.496 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:27.496 | INFO | Uploaded 36GiB to GPU in 2.03846s (17.6604GiB/s) | 20260506 01:26:27.496 | INFO | Downloaded 34GiB to GPU in 2.02153s (16.8189GiB/s) | 20260506 01:26:27.622 | INFO | Starting a round of Baz IO | 20260506 01:26:30.759 | INFO | Populating RTMetrics report | 20260506 01:26:30.759 | INFO | RTMetrics report populated | 20260506 01:26:30.759 | INFO | Opening tmp RTMetrics json file | 20260506 01:26:30.760 | INFO | Serializing RTMetrics to json | 20260506 01:26:30.761 | INFO | Serialization done, swapping tmp file now... | 20260506 01:26:30.761 | INFO | RTMetrics done | 20260506 01:26:30.838 | INFO | Finished a round of Baz IO: Wrote 1.824 GiB in 3.216 seconds | 20260506 01:26:30.838 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.32% / 0.00% / 78.68% | 20260506 01:26:30.838 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0800172% / 0% / 8.34465e-05% | 20260506 01:26:30.838 | INFO | Baz IO overall: | 20260506 01:26:30.838 | INFO | BytesWritten : 4435080215545 | 20260506 01:26:30.838 | INFO | NumEvents : 2084663524979 | 20260506 01:26:32.628 | INFO | Analyzing chunk frames = [7860224,7860736) | 20260506 01:26:32.628 | INFO | MemInfo: { | 20260506 01:26:32.628 | INFO | "gpu" : " 17 GB", | 20260506 01:26:32.628 | INFO | "huge" : "", | 20260506 01:26:32.628 | INFO | "huge_shared" : " 27 GB", | 20260506 01:26:32.628 | INFO | "malloc" : " 14 GB", | 20260506 01:26:32.628 | INFO | "pinned" : " 29 GB" | 20260506 01:26:32.628 | INFO | } | 20260506 01:26:32.628 | INFO | Perf report for the last 5.13184 seconds: | 20260506 01:26:32.628 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:26:32.628 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:32.628 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:32.628 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:32.628 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260506 01:26:32.628 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:32.628 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260506 01:26:32.628 | INFO | PRE_HQ : 59.8%, 1.7, 53.9ms, 0 | 20260506 01:26:32.628 | INFO | BAZWRITER : 61.9%, 3.5, 110.3ms, 0 | 20260506 01:26:32.628 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:26:32.628 | INFO | Uploaded 36GiB to GPU in 2.18062s (16.5091GiB/s) | 20260506 01:26:32.628 | INFO | Downloaded 38GiB to GPU in 2.24187s (16.9502GiB/s) | 20260506 01:26:37.766 | INFO | Analyzing chunk frames = [7860736,7861248) | 20260506 01:26:37.766 | INFO | MemInfo: { | 20260506 01:26:37.766 | INFO | "gpu" : " 17 GB", | 20260506 01:26:37.766 | INFO | "huge" : "", | 20260506 01:26:37.766 | INFO | "huge_shared" : " 27 GB", | 20260506 01:26:37.766 | INFO | "malloc" : " 14 GB", | 20260506 01:26:37.766 | INFO | "pinned" : " 29 GB" | 20260506 01:26:37.766 | INFO | } | 20260506 01:26:37.766 | INFO | Perf report for the last 5.13816 seconds: | 20260506 01:26:37.766 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:26:37.766 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:37.766 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:37.766 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:37.766 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 01:26:37.766 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:37.766 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:37.766 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 01:26:37.766 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:37.766 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:37.766 | INFO | Uploaded 36GiB to GPU in 2.01618s (17.8556GiB/s) | 20260506 01:26:37.766 | INFO | Downloaded 34GiB to GPU in 2.01227s (16.8963GiB/s) | 20260506 01:26:42.906 | INFO | Analyzing chunk frames = [7861248,7861760) | 20260506 01:26:42.906 | INFO | MemInfo: { | 20260506 01:26:42.906 | INFO | "gpu" : " 17 GB", | 20260506 01:26:42.906 | INFO | "huge" : "", | 20260506 01:26:42.906 | INFO | "huge_shared" : " 27 GB", | 20260506 01:26:42.906 | INFO | "malloc" : " 14 GB", | 20260506 01:26:42.906 | INFO | "pinned" : " 29 GB" | 20260506 01:26:42.906 | INFO | } | 20260506 01:26:42.906 | INFO | Perf report for the last 5.1398 seconds: | 20260506 01:26:42.906 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:26:42.906 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:42.906 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:42.906 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:42.906 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 01:26:42.906 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:42.906 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:42.906 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 01:26:42.906 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:42.906 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:42.906 | INFO | Uploaded 36GiB to GPU in 2.09357s (17.1955GiB/s) | 20260506 01:26:42.906 | INFO | Downloaded 34GiB to GPU in 2.026s (16.7818GiB/s) | 20260506 01:26:48.048 | INFO | Analyzing chunk frames = [7861760,7862272) | 20260506 01:26:48.048 | INFO | MemInfo: { | 20260506 01:26:48.048 | INFO | "gpu" : " 17 GB", | 20260506 01:26:48.048 | INFO | "huge" : "", | 20260506 01:26:48.048 | INFO | "huge_shared" : " 27 GB", | 20260506 01:26:48.048 | INFO | "malloc" : " 14 GB", | 20260506 01:26:48.048 | INFO | "pinned" : " 29 GB" | 20260506 01:26:48.048 | INFO | } | 20260506 01:26:48.048 | INFO | Perf report for the last 5.14261 seconds: | 20260506 01:26:48.048 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:26:48.048 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:48.048 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:48.048 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:48.048 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260506 01:26:48.048 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:48.048 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:48.049 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 01:26:48.049 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:48.049 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:48.049 | INFO | Uploaded 36GiB to GPU in 2.14892s (16.7526GiB/s) | 20260506 01:26:48.049 | INFO | Downloaded 34GiB to GPU in 2.04416s (16.6327GiB/s) | 20260506 01:26:53.193 | INFO | Analyzing chunk frames = [7862272,7862784) | 20260506 01:26:53.193 | INFO | MemInfo: { | 20260506 01:26:53.193 | INFO | "gpu" : " 17 GB", | 20260506 01:26:53.193 | INFO | "huge" : "", | 20260506 01:26:53.193 | INFO | "huge_shared" : " 27 GB", | 20260506 01:26:53.193 | INFO | "malloc" : " 14 GB", | 20260506 01:26:53.194 | INFO | "pinned" : " 29 GB" | 20260506 01:26:53.194 | INFO | } | 20260506 01:26:53.194 | INFO | Perf report for the last 5.14517 seconds: | 20260506 01:26:53.194 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:26:53.194 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:53.194 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:53.194 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:53.194 | INFO | ANALYSIS : 60.9%, 3.0, 95.2ms, 0 | 20260506 01:26:53.194 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:53.194 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:53.194 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:26:53.194 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:53.194 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:53.194 | INFO | Uploaded 36GiB to GPU in 2.09829s (17.1568GiB/s) | 20260506 01:26:53.194 | INFO | Downloaded 34GiB to GPU in 2.04223s (16.6485GiB/s) | 20260506 01:26:58.324 | INFO | Analyzing chunk frames = [7862784,7863296) | 20260506 01:26:58.324 | INFO | MemInfo: { | 20260506 01:26:58.324 | INFO | "gpu" : " 17 GB", | 20260506 01:26:58.324 | INFO | "huge" : "", | 20260506 01:26:58.324 | INFO | "huge_shared" : " 27 GB", | 20260506 01:26:58.324 | INFO | "malloc" : " 14 GB", | 20260506 01:26:58.324 | INFO | "pinned" : " 29 GB" | 20260506 01:26:58.324 | INFO | } | 20260506 01:26:58.324 | INFO | Perf report for the last 5.13071 seconds: | 20260506 01:26:58.324 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:26:58.324 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:58.324 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:58.324 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:58.324 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 01:26:58.324 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:58.324 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:58.324 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:26:58.324 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:58.324 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:26:58.325 | INFO | Uploaded 36GiB to GPU in 2.07078s (17.3847GiB/s) | 20260506 01:26:58.325 | INFO | Downloaded 34GiB to GPU in 2.03545s (16.704GiB/s) | 20260506 01:27:03.466 | INFO | Analyzing chunk frames = [7863296,7863808) | 20260506 01:27:03.466 | INFO | MemInfo: { | 20260506 01:27:03.466 | INFO | "gpu" : " 17 GB", | 20260506 01:27:03.466 | INFO | "huge" : "", | 20260506 01:27:03.466 | INFO | "huge_shared" : " 27 GB", | 20260506 01:27:03.466 | INFO | "malloc" : " 14 GB", | 20260506 01:27:03.466 | INFO | "pinned" : " 29 GB" | 20260506 01:27:03.466 | INFO | } | 20260506 01:27:03.466 | INFO | Perf report for the last 5.14211 seconds: | 20260506 01:27:03.466 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:27:03.466 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:03.466 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:03.466 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:03.466 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260506 01:27:03.466 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:03.466 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:03.466 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 01:27:03.466 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:03.466 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:03.467 | INFO | Uploaded 36GiB to GPU in 2.02708s (17.7595GiB/s) | 20260506 01:27:03.467 | INFO | Downloaded 34GiB to GPU in 2.04047s (16.6628GiB/s) | 20260506 01:27:08.608 | INFO | Analyzing chunk frames = [7863808,7864320) | 20260506 01:27:08.608 | INFO | MemInfo: { | 20260506 01:27:08.608 | INFO | "gpu" : " 17 GB", | 20260506 01:27:08.608 | INFO | "huge" : "", | 20260506 01:27:08.608 | INFO | "huge_shared" : " 27 GB", | 20260506 01:27:08.608 | INFO | "malloc" : " 14 GB", | 20260506 01:27:08.608 | INFO | "pinned" : " 29 GB" | 20260506 01:27:08.608 | INFO | } | 20260506 01:27:08.608 | INFO | Perf report for the last 5.14153 seconds: | 20260506 01:27:08.608 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:27:08.608 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:08.608 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:08.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:08.608 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260506 01:27:08.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:08.608 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:08.608 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:27:08.608 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:08.608 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:08.608 | INFO | Uploaded 36GiB to GPU in 2.13753s (16.8418GiB/s) | 20260506 01:27:08.608 | INFO | Downloaded 34GiB to GPU in 2.02746s (16.7697GiB/s) | 20260506 01:27:08.608 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 01:27:08.608 | INFO | Total mem usage: 15.3 GiB | 20260506 01:27:08.608 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 01:27:08.608 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.2%) | 20260506 01:27:08.608 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 01:27:08.608 | INFO | High Water Breakdown: | 20260506 01:27:08.608 | INFO | 60.9% : 8.54 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 01:27:08.608 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 01:27:08.608 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 01:27:08.608 | INFO | ------------------------------------------ | 20260506 01:27:08.608 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:27:08.609 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 01:27:08.609 | INFO | Total mem usage: 26.5 GiB | 20260506 01:27:08.609 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 01:27:08.609 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 01:27:08.609 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260506 01:27:08.609 | INFO | High Water Breakdown: | 20260506 01:27:08.609 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:27:08.609 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:27:08.609 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:27:08.609 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:27:08.609 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:27:08.609 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:27:08.609 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 01:27:08.609 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 01:27:08.609 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:27:08.609 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 01:27:08.609 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 01:27:08.609 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 01:27:08.609 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 01:27:08.609 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 01:27:08.609 | INFO | ------------------------------------------ | 20260506 01:27:08.609 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:27:08.609 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 01:27:08.609 | INFO | Total mem usage: 25.6 GiB | 20260506 01:27:08.609 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 01:27:08.609 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 01:27:08.609 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 01:27:08.609 | INFO | High Water Breakdown: | 20260506 01:27:08.609 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 01:27:08.609 | INFO | ------------------------------------------ | 20260506 01:27:08.609 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:27:08.609 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 01:27:08.609 | INFO | Total mem usage: 15.7 GiB | 20260506 01:27:08.609 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 01:27:08.609 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 01:27:08.609 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 01:27:08.609 | INFO | High Water Breakdown: | 20260506 01:27:08.609 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 01:27:08.609 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 01:27:08.609 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 01:27:08.609 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 01:27:08.609 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 01:27:08.609 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 01:27:08.609 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 01:27:08.609 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 01:27:08.609 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 01:27:08.609 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:27:08.609 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 01:27:08.609 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:27:08.609 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 01:27:08.609 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 01:27:08.609 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 01:27:08.609 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 01:27:08.609 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:27:08.609 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 01:27:08.609 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:27:08.609 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:27:08.609 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:27:08.609 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 01:27:08.609 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 01:27:08.609 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:27:08.609 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 01:27:08.609 | INFO | ------------------------------------------ | 20260506 01:27:08.609 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:27:08.609 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 01:27:08.609 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:27:08.609 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:27:08.609 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.39e+01, 1.34e+01 / 8.89e-01 | 20260506 01:27:08.609 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.04e-01, 1.14e-02 / 4.45e-03 | 20260506 01:27:08.609 | INFO | PerfReport: 11.85%, Baseline -- 3.71e+00 / 4.52e+00, 5.73e-01 / 1.50e-01 | 20260506 01:27:08.609 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.51e+00, 1.90e-01 / 1.01e-01 | 20260506 01:27:08.609 | INFO | PerfReport: 16.03%, DME -- 4.01e+01 / 4.42e+01, 3.80e+00 / 1.14e+00 | 20260506 01:27:08.609 | INFO | PerfReport: 57.37%, FrameLabeling -- 1.79e+01 / 1.93e+01, 1.46e+00 / 9.89e-01 | 20260506 01:27:08.609 | INFO | PerfReport: 7.08%, PulseAccumulating -- 2.21e+00 / 2.73e+00, 4.24e-01 / 2.41e-01 | 20260506 01:27:08.609 | INFO | PerfReport: 3.04%, Metrics -- 9.52e-01 / 1.20e+00, 2.54e-01 / 8.89e-02 | 20260506 01:27:08.609 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.29e-01 / 3.78e-01, 5.55e-02 / 2.63e-02 | 20260506 01:27:08.609 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 01:27:08.609 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:27:08.609 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:27:08.610 | INFO | PerfReport: Full Execution -- 9.79e+01 / -nan, 2.42e+01 / 0.00e+00 | 20260506 01:27:08.610 | INFO | PerfReport: 22.20%, Upload -- 2.17e+01 / 0.00e+00, 7.33e+00 / 0.00e+00 | 20260506 01:27:08.610 | INFO | PerfReport: 21.88%, Download -- 2.14e+01 / 0.00e+00, 7.54e+00 / 0.00e+00 | 20260506 01:27:08.610 | INFO | PerfReport: 31.94%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 01:27:08.610 | INFO | PerfReport: 2.06%, LockUpload -- 2.02e+00 / 0.00e+00, 7.80e+00 / 0.00e+00 | 20260506 01:27:08.610 | INFO | PerfReport: 1.31%, LockDownload -- 1.29e+00 / 0.00e+00, 2.92e+00 / 0.00e+00 | 20260506 01:27:08.610 | INFO | PerfReport: 20.60%, LockCompute -- 2.02e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 01:27:08.610 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.23e-02 / 0.00e+00 | 20260506 01:27:08.733 | INFO | Starting a round of Baz IO | 20260506 01:27:11.873 | INFO | Populating RTMetrics report | 20260506 01:27:11.873 | INFO | RTMetrics report populated | 20260506 01:27:11.873 | INFO | Opening tmp RTMetrics json file | 20260506 01:27:11.874 | INFO | Serializing RTMetrics to json | 20260506 01:27:11.876 | INFO | Serialization done, swapping tmp file now... | 20260506 01:27:11.876 | INFO | RTMetrics done | 20260506 01:27:11.949 | INFO | Finished a round of Baz IO: Wrote 1.825 GiB in 3.215 seconds | 20260506 01:27:11.949 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.32% / 0.00% / 78.68% | 20260506 01:27:11.949 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0801722% / 0% / 0.000115236% | 20260506 01:27:11.949 | INFO | Baz IO overall: | 20260506 01:27:11.950 | INFO | BytesWritten : 4437039577931 | 20260506 01:27:11.950 | INFO | NumEvents : 2085556668221 | 20260506 01:27:13.742 | INFO | Analyzing chunk frames = [7864320,7864832) | 20260506 01:27:13.742 | INFO | MemInfo: { | 20260506 01:27:13.742 | INFO | "gpu" : " 17 GB", | 20260506 01:27:13.742 | INFO | "huge" : "", | 20260506 01:27:13.742 | INFO | "huge_shared" : " 27 GB", | 20260506 01:27:13.742 | INFO | "malloc" : " 14 GB", | 20260506 01:27:13.742 | INFO | "pinned" : " 29 GB" | 20260506 01:27:13.742 | INFO | } | 20260506 01:27:13.742 | INFO | Perf report for the last 5.13387 seconds: | 20260506 01:27:13.742 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:27:13.742 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:13.742 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:13.742 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:13.742 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260506 01:27:13.742 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:13.742 | INFO | RT_METRICS : 43.0%, 1.0, 22.5ms, 0 | 20260506 01:27:13.742 | INFO | PRE_HQ : 59.6%, 1.7, 53.5ms, 0 | 20260506 01:27:13.742 | INFO | BAZWRITER : 61.6%, 3.5, 109.9ms, 0 | 20260506 01:27:13.742 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:27:13.742 | INFO | Uploaded 36GiB to GPU in 2.1795s (16.5175GiB/s) | 20260506 01:27:13.742 | INFO | Downloaded 38GiB to GPU in 2.23273s (17.0196GiB/s) | 20260506 01:27:18.881 | INFO | Analyzing chunk frames = [7864832,7865344) | 20260506 01:27:18.881 | INFO | MemInfo: { | 20260506 01:27:18.881 | INFO | "gpu" : " 17 GB", | 20260506 01:27:18.881 | INFO | "huge" : "", | 20260506 01:27:18.881 | INFO | "huge_shared" : " 27 GB", | 20260506 01:27:18.881 | INFO | "malloc" : " 14 GB", | 20260506 01:27:18.881 | INFO | "pinned" : " 29 GB" | 20260506 01:27:18.881 | INFO | } | 20260506 01:27:18.881 | INFO | Perf report for the last 5.13908 seconds: | 20260506 01:27:18.881 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:27:18.881 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:18.881 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:18.881 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:18.881 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260506 01:27:18.881 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:18.881 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:18.881 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 01:27:18.881 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:18.881 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:18.881 | INFO | Uploaded 36GiB to GPU in 2.11194s (17.0459GiB/s) | 20260506 01:27:18.881 | INFO | Downloaded 34GiB to GPU in 2.04625s (16.6157GiB/s) | 20260506 01:27:24.020 | INFO | Analyzing chunk frames = [7865344,7865856) | 20260506 01:27:24.020 | INFO | MemInfo: { | 20260506 01:27:24.020 | INFO | "gpu" : " 17 GB", | 20260506 01:27:24.020 | INFO | "huge" : "", | 20260506 01:27:24.020 | INFO | "huge_shared" : " 27 GB", | 20260506 01:27:24.020 | INFO | "malloc" : " 14 GB", | 20260506 01:27:24.020 | INFO | "pinned" : " 29 GB" | 20260506 01:27:24.020 | INFO | } | 20260506 01:27:24.020 | INFO | Perf report for the last 5.13906 seconds: | 20260506 01:27:24.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:27:24.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:24.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:24.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:24.020 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 01:27:24.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:24.020 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:24.020 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 01:27:24.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:24.020 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:24.020 | INFO | Uploaded 36GiB to GPU in 2.08326s (17.2806GiB/s) | 20260506 01:27:24.020 | INFO | Downloaded 34GiB to GPU in 2.01758s (16.8519GiB/s) | 20260506 01:27:29.166 | INFO | Analyzing chunk frames = [7865856,7866368) | 20260506 01:27:29.166 | INFO | MemInfo: { | 20260506 01:27:29.166 | INFO | "gpu" : " 17 GB", | 20260506 01:27:29.166 | INFO | "huge" : "", | 20260506 01:27:29.166 | INFO | "huge_shared" : " 27 GB", | 20260506 01:27:29.166 | INFO | "malloc" : " 14 GB", | 20260506 01:27:29.166 | INFO | "pinned" : " 29 GB" | 20260506 01:27:29.166 | INFO | } | 20260506 01:27:29.167 | INFO | Perf report for the last 5.14655 seconds: | 20260506 01:27:29.167 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:27:29.167 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:29.167 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:29.167 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:29.167 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260506 01:27:29.167 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:29.167 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:29.167 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:27:29.167 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:29.167 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:29.167 | INFO | Uploaded 36GiB to GPU in 2.1527s (16.7232GiB/s) | 20260506 01:27:29.167 | INFO | Downloaded 34GiB to GPU in 2.04639s (16.6146GiB/s) | 20260506 01:27:34.306 | INFO | Analyzing chunk frames = [7866368,7866880) | 20260506 01:27:34.306 | INFO | MemInfo: { | 20260506 01:27:34.306 | INFO | "gpu" : " 17 GB", | 20260506 01:27:34.306 | INFO | "huge" : "", | 20260506 01:27:34.306 | INFO | "huge_shared" : " 27 GB", | 20260506 01:27:34.306 | INFO | "malloc" : " 14 GB", | 20260506 01:27:34.307 | INFO | "pinned" : " 29 GB" | 20260506 01:27:34.307 | INFO | } | 20260506 01:27:34.307 | INFO | Perf report for the last 5.13998 seconds: | 20260506 01:27:34.307 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:27:34.307 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:34.307 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:34.307 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:34.307 | INFO | ANALYSIS : 60.6%, 3.0, 94.7ms, 0 | 20260506 01:27:34.307 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:34.307 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:34.307 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:27:34.307 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:34.307 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:34.307 | INFO | Uploaded 36GiB to GPU in 2.0675s (17.4123GiB/s) | 20260506 01:27:34.307 | INFO | Downloaded 34GiB to GPU in 2.0291s (16.7562GiB/s) | 20260506 01:27:39.441 | INFO | Analyzing chunk frames = [7866880,7867392) | 20260506 01:27:39.441 | INFO | MemInfo: { | 20260506 01:27:39.441 | INFO | "gpu" : " 17 GB", | 20260506 01:27:39.441 | INFO | "huge" : "", | 20260506 01:27:39.441 | INFO | "huge_shared" : " 27 GB", | 20260506 01:27:39.441 | INFO | "malloc" : " 14 GB", | 20260506 01:27:39.441 | INFO | "pinned" : " 29 GB" | 20260506 01:27:39.441 | INFO | } | 20260506 01:27:39.441 | INFO | Perf report for the last 5.13468 seconds: | 20260506 01:27:39.441 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:27:39.441 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:39.441 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:39.441 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:39.441 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 01:27:39.441 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:39.441 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:39.441 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 01:27:39.441 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:39.441 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:39.442 | INFO | Uploaded 36GiB to GPU in 2.07447s (17.3539GiB/s) | 20260506 01:27:39.442 | INFO | Downloaded 34GiB to GPU in 2.03719s (16.6897GiB/s) | 20260506 01:27:44.575 | INFO | Analyzing chunk frames = [7867392,7867904) | 20260506 01:27:44.575 | INFO | MemInfo: { | 20260506 01:27:44.575 | INFO | "gpu" : " 17 GB", | 20260506 01:27:44.575 | INFO | "huge" : "", | 20260506 01:27:44.575 | INFO | "huge_shared" : " 27 GB", | 20260506 01:27:44.575 | INFO | "malloc" : " 14 GB", | 20260506 01:27:44.575 | INFO | "pinned" : " 29 GB" | 20260506 01:27:44.575 | INFO | } | 20260506 01:27:44.575 | INFO | Perf report for the last 5.13353 seconds: | 20260506 01:27:44.575 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:27:44.575 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:44.575 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:44.575 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:44.575 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 01:27:44.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:44.575 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:44.575 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 01:27:44.575 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:44.575 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:44.575 | INFO | Uploaded 36GiB to GPU in 2.09996s (17.1432GiB/s) | 20260506 01:27:44.575 | INFO | Downloaded 34GiB to GPU in 2.07764s (16.3647GiB/s) | 20260506 01:27:49.725 | INFO | Analyzing chunk frames = [7867904,7868416) | 20260506 01:27:49.725 | INFO | MemInfo: { | 20260506 01:27:49.725 | INFO | "gpu" : " 17 GB", | 20260506 01:27:49.725 | INFO | "huge" : "", | 20260506 01:27:49.725 | INFO | "huge_shared" : " 27 GB", | 20260506 01:27:49.725 | INFO | "malloc" : " 14 GB", | 20260506 01:27:49.725 | INFO | "pinned" : " 29 GB" | 20260506 01:27:49.725 | INFO | } | 20260506 01:27:49.725 | INFO | Perf report for the last 5.1504 seconds: | 20260506 01:27:49.725 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:27:49.725 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:49.725 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:49.725 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:49.725 | INFO | ANALYSIS : 61.9%, 3.0, 96.8ms, 0 | 20260506 01:27:49.725 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:49.725 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:49.725 | INFO | PRE_HQ : 26.8%, 1.0, 14.0ms, 0 | 20260506 01:27:49.725 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:49.725 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:49.725 | INFO | Uploaded 36GiB to GPU in 2.11609s (17.0125GiB/s) | 20260506 01:27:49.726 | INFO | Downloaded 34GiB to GPU in 2.01524s (16.8714GiB/s) | 20260506 01:27:49.851 | INFO | Starting a round of Baz IO | 20260506 01:27:52.978 | INFO | Populating RTMetrics report | 20260506 01:27:52.979 | INFO | RTMetrics report populated | 20260506 01:27:52.979 | INFO | Opening tmp RTMetrics json file | 20260506 01:27:52.980 | INFO | Serializing RTMetrics to json | 20260506 01:27:52.982 | INFO | Serialization done, swapping tmp file now... | 20260506 01:27:52.982 | INFO | RTMetrics done | 20260506 01:27:53.063 | INFO | Finished a round of Baz IO: Wrote 1.823 GiB in 3.211 seconds | 20260506 01:27:53.063 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.32% / 0.00% / 78.68% | 20260506 01:27:53.063 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0801563% / 0% / 0.000103315% | 20260506 01:27:53.063 | INFO | Baz IO overall: | 20260506 01:27:53.063 | INFO | BytesWritten : 4438997481782 | 20260506 01:27:53.063 | INFO | NumEvents : 2086448610998 | 20260506 01:27:54.861 | INFO | Analyzing chunk frames = [7868416,7868928) | 20260506 01:27:54.861 | INFO | MemInfo: { | 20260506 01:27:54.861 | INFO | "gpu" : " 17 GB", | 20260506 01:27:54.861 | INFO | "huge" : "", | 20260506 01:27:54.861 | INFO | "huge_shared" : " 27 GB", | 20260506 01:27:54.861 | INFO | "malloc" : " 14 GB", | 20260506 01:27:54.861 | INFO | "pinned" : " 29 GB" | 20260506 01:27:54.861 | INFO | } | 20260506 01:27:54.861 | INFO | Perf report for the last 5.13554 seconds: | 20260506 01:27:54.861 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:27:54.861 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:54.861 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:54.861 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:54.861 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260506 01:27:54.861 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:54.861 | INFO | RT_METRICS : 43.0%, 1.0, 22.5ms, 0 | 20260506 01:27:54.861 | INFO | PRE_HQ : 59.2%, 1.7, 53.7ms, 0 | 20260506 01:27:54.861 | INFO | BAZWRITER : 61.7%, 3.5, 111.0ms, 0 | 20260506 01:27:54.861 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:27:54.861 | INFO | Uploaded 36GiB to GPU in 2.10293s (17.119GiB/s) | 20260506 01:27:54.861 | INFO | Downloaded 38GiB to GPU in 2.22065s (17.1121GiB/s) | 20260506 01:27:59.998 | INFO | Analyzing chunk frames = [7868928,7869440) | 20260506 01:27:59.998 | INFO | MemInfo: { | 20260506 01:27:59.998 | INFO | "gpu" : " 17 GB", | 20260506 01:27:59.998 | INFO | "huge" : "", | 20260506 01:27:59.998 | INFO | "huge_shared" : " 27 GB", | 20260506 01:27:59.998 | INFO | "malloc" : " 14 GB", | 20260506 01:27:59.998 | INFO | "pinned" : " 29 GB" | 20260506 01:27:59.998 | INFO | } | 20260506 01:27:59.998 | INFO | Perf report for the last 5.1372 seconds: | 20260506 01:27:59.998 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:27:59.998 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:59.998 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:59.998 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:59.998 | INFO | ANALYSIS : 60.9%, 3.0, 95.1ms, 0 | 20260506 01:27:59.998 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:59.998 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:59.998 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 01:27:59.998 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:59.998 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:27:59.998 | INFO | Uploaded 36GiB to GPU in 1.95409s (18.4229GiB/s) | 20260506 01:27:59.998 | INFO | Downloaded 34GiB to GPU in 1.98057s (17.1668GiB/s) | 20260506 01:28:05.138 | INFO | Analyzing chunk frames = [7869440,7869952) | 20260506 01:28:05.138 | INFO | MemInfo: { | 20260506 01:28:05.138 | INFO | "gpu" : " 17 GB", | 20260506 01:28:05.138 | INFO | "huge" : "", | 20260506 01:28:05.138 | INFO | "huge_shared" : " 27 GB", | 20260506 01:28:05.138 | INFO | "malloc" : " 14 GB", | 20260506 01:28:05.138 | INFO | "pinned" : " 29 GB" | 20260506 01:28:05.138 | INFO | } | 20260506 01:28:05.138 | INFO | Perf report for the last 5.14049 seconds: | 20260506 01:28:05.138 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:28:05.138 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:05.139 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:05.139 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:05.139 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 01:28:05.139 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:05.139 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:05.139 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:28:05.139 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:05.139 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:05.139 | INFO | Uploaded 36GiB to GPU in 2.08487s (17.2673GiB/s) | 20260506 01:28:05.139 | INFO | Downloaded 34GiB to GPU in 2.01672s (16.859GiB/s) | 20260506 01:28:10.277 | INFO | Analyzing chunk frames = [7869952,7870464) | 20260506 01:28:10.277 | INFO | MemInfo: { | 20260506 01:28:10.277 | INFO | "gpu" : " 17 GB", | 20260506 01:28:10.277 | INFO | "huge" : "", | 20260506 01:28:10.277 | INFO | "huge_shared" : " 27 GB", | 20260506 01:28:10.277 | INFO | "malloc" : " 14 GB", | 20260506 01:28:10.277 | INFO | "pinned" : " 29 GB" | 20260506 01:28:10.277 | INFO | } | 20260506 01:28:10.277 | INFO | Perf report for the last 5.13856 seconds: | 20260506 01:28:10.277 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:28:10.277 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:10.277 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:10.277 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:10.277 | INFO | ANALYSIS : 61.6%, 3.0, 95.7ms, 0 | 20260506 01:28:10.277 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:10.277 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:10.277 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260506 01:28:10.277 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:10.277 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:10.277 | INFO | Uploaded 36GiB to GPU in 2.1605s (16.6628GiB/s) | 20260506 01:28:10.277 | INFO | Downloaded 34GiB to GPU in 2.05114s (16.5761GiB/s) | 20260506 01:28:15.415 | INFO | Analyzing chunk frames = [7870464,7870976) | 20260506 01:28:15.415 | INFO | MemInfo: { | 20260506 01:28:15.415 | INFO | "gpu" : " 17 GB", | 20260506 01:28:15.415 | INFO | "huge" : "", | 20260506 01:28:15.415 | INFO | "huge_shared" : " 27 GB", | 20260506 01:28:15.415 | INFO | "malloc" : " 14 GB", | 20260506 01:28:15.415 | INFO | "pinned" : " 29 GB" | 20260506 01:28:15.415 | INFO | } | 20260506 01:28:15.415 | INFO | Perf report for the last 5.13802 seconds: | 20260506 01:28:15.415 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:28:15.415 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:15.415 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:15.415 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:15.415 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 01:28:15.415 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:15.415 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:15.415 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 01:28:15.415 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:15.415 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:15.415 | INFO | Uploaded 36GiB to GPU in 2.09966s (17.1456GiB/s) | 20260506 01:28:15.415 | INFO | Downloaded 34GiB to GPU in 2.04039s (16.6635GiB/s) | 20260506 01:28:20.558 | INFO | Analyzing chunk frames = [7870976,7871488) | 20260506 01:28:20.558 | INFO | MemInfo: { | 20260506 01:28:20.558 | INFO | "gpu" : " 17 GB", | 20260506 01:28:20.558 | INFO | "huge" : "", | 20260506 01:28:20.558 | INFO | "huge_shared" : " 27 GB", | 20260506 01:28:20.558 | INFO | "malloc" : " 14 GB", | 20260506 01:28:20.558 | INFO | "pinned" : " 29 GB" | 20260506 01:28:20.558 | INFO | } | 20260506 01:28:20.558 | INFO | Perf report for the last 5.14341 seconds: | 20260506 01:28:20.558 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:28:20.559 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:20.559 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:20.559 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:20.559 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 01:28:20.559 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:20.559 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:20.559 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:28:20.559 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:20.559 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:20.559 | INFO | Uploaded 36GiB to GPU in 2.06318s (17.4488GiB/s) | 20260506 01:28:20.559 | INFO | Downloaded 34GiB to GPU in 2.02927s (16.7548GiB/s) | 20260506 01:28:25.696 | INFO | Analyzing chunk frames = [7871488,7872000) | 20260506 01:28:25.696 | INFO | MemInfo: { | 20260506 01:28:25.696 | INFO | "gpu" : " 17 GB", | 20260506 01:28:25.696 | INFO | "huge" : "", | 20260506 01:28:25.696 | INFO | "huge_shared" : " 27 GB", | 20260506 01:28:25.696 | INFO | "malloc" : " 14 GB", | 20260506 01:28:25.696 | INFO | "pinned" : " 29 GB" | 20260506 01:28:25.696 | INFO | } | 20260506 01:28:25.696 | INFO | Perf report for the last 5.13784 seconds: | 20260506 01:28:25.696 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:28:25.696 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:25.696 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:25.696 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:25.696 | INFO | ANALYSIS : 62.6%, 3.0, 97.8ms, 0 | 20260506 01:28:25.696 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:25.696 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:25.696 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 01:28:25.696 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:25.696 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:25.697 | INFO | Uploaded 36GiB to GPU in 2.13039s (16.8983GiB/s) | 20260506 01:28:25.697 | INFO | Downloaded 34GiB to GPU in 2.09949s (16.1944GiB/s) | 20260506 01:28:30.834 | INFO | Analyzing chunk frames = [7872000,7872512) | 20260506 01:28:30.834 | INFO | MemInfo: { | 20260506 01:28:30.834 | INFO | "gpu" : " 17 GB", | 20260506 01:28:30.834 | INFO | "huge" : "", | 20260506 01:28:30.834 | INFO | "huge_shared" : " 27 GB", | 20260506 01:28:30.834 | INFO | "malloc" : " 14 GB", | 20260506 01:28:30.834 | INFO | "pinned" : " 29 GB" | 20260506 01:28:30.834 | INFO | } | 20260506 01:28:30.834 | INFO | Perf report for the last 5.13803 seconds: | 20260506 01:28:30.834 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:28:30.834 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:30.834 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:30.834 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:30.834 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260506 01:28:30.834 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:30.834 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:30.834 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:28:30.834 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:30.835 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:30.835 | INFO | Uploaded 36GiB to GPU in 2.13859s (16.8336GiB/s) | 20260506 01:28:30.835 | INFO | Downloaded 34GiB to GPU in 2.02181s (16.8166GiB/s) | 20260506 01:28:30.835 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 01:28:30.835 | INFO | Total mem usage: 15.3 GiB | 20260506 01:28:30.835 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 01:28:30.835 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.19%) | 20260506 01:28:30.835 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 01:28:30.835 | INFO | High Water Breakdown: | 20260506 01:28:30.835 | INFO | 60.9% : 8.54 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 01:28:30.835 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 01:28:30.835 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 01:28:30.835 | INFO | ------------------------------------------ | 20260506 01:28:30.835 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:28:30.835 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 01:28:30.835 | INFO | Total mem usage: 26.5 GiB | 20260506 01:28:30.835 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 01:28:30.835 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 01:28:30.835 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 01:28:30.835 | INFO | High Water Breakdown: | 20260506 01:28:30.835 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:28:30.835 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:28:30.835 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:28:30.835 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:28:30.835 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:28:30.835 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:28:30.835 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 01:28:30.835 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 01:28:30.835 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:28:30.835 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 01:28:30.835 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 01:28:30.835 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 01:28:30.835 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 01:28:30.835 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 01:28:30.835 | INFO | ------------------------------------------ | 20260506 01:28:30.835 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:28:30.835 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 01:28:30.835 | INFO | Total mem usage: 25.6 GiB | 20260506 01:28:30.835 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 01:28:30.835 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260506 01:28:30.835 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260506 01:28:30.835 | INFO | High Water Breakdown: | 20260506 01:28:30.835 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260506 01:28:30.835 | INFO | ------------------------------------------ | 20260506 01:28:30.835 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:28:30.835 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 01:28:30.835 | INFO | Total mem usage: 15.7 GiB | 20260506 01:28:30.835 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 01:28:30.835 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 01:28:30.835 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 01:28:30.835 | INFO | High Water Breakdown: | 20260506 01:28:30.835 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 01:28:30.835 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 01:28:30.835 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 01:28:30.835 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 01:28:30.835 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 01:28:30.835 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 01:28:30.835 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 01:28:30.835 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 01:28:30.835 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 01:28:30.835 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:28:30.835 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 01:28:30.835 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:28:30.835 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 01:28:30.835 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 01:28:30.835 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 01:28:30.835 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 01:28:30.835 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:28:30.836 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 01:28:30.836 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:28:30.836 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:28:30.836 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:28:30.836 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 01:28:30.836 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 01:28:30.836 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:28:30.836 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 01:28:30.836 | INFO | ------------------------------------------ | 20260506 01:28:30.836 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:28:30.836 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 01:28:30.836 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:28:30.836 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:28:30.836 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.36e+01, 1.31e+01 / 7.40e-01 | 20260506 01:28:30.836 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.05e-01, 1.06e-02 / 7.00e-03 | 20260506 01:28:30.836 | INFO | PerfReport: 11.81%, Baseline -- 3.69e+00 / 4.46e+00, 5.69e-01 / 2.94e-01 | 20260506 01:28:30.836 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.50e+00, 1.89e-01 / 1.15e-01 | 20260506 01:28:30.836 | INFO | PerfReport: 15.98%, DME -- 4.00e+01 / 4.37e+01, 3.84e+00 / 8.13e-01 | 20260506 01:28:30.836 | INFO | PerfReport: 57.49%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.54e+00 / 1.07e+00 | 20260506 01:28:30.836 | INFO | PerfReport: 7.05%, PulseAccumulating -- 2.21e+00 / 2.73e+00, 4.20e-01 / 2.36e-01 | 20260506 01:28:30.836 | INFO | PerfReport: 3.04%, Metrics -- 9.51e-01 / 1.24e+00, 2.56e-01 / 1.27e-01 | 20260506 01:28:30.836 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.29e-01 / 3.83e-01, 5.55e-02 / 2.72e-02 | 20260506 01:28:30.836 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 01:28:30.836 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:28:30.836 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:28:30.836 | INFO | PerfReport: Full Execution -- 9.81e+01 / -nan, 2.45e+01 / 0.00e+00 | 20260506 01:28:30.836 | INFO | PerfReport: 22.32%, Upload -- 2.19e+01 / 0.00e+00, 7.47e+00 / 0.00e+00 | 20260506 01:28:30.836 | INFO | PerfReport: 21.88%, Download -- 2.15e+01 / 0.00e+00, 7.64e+00 / 0.00e+00 | 20260506 01:28:30.836 | INFO | PerfReport: 31.89%, Compute -- 3.13e+01 / 0.00e+00, 1.31e+01 / 0.00e+00 | 20260506 01:28:30.836 | INFO | PerfReport: 2.09%, LockUpload -- 2.05e+00 / 0.00e+00, 7.95e+00 / 0.00e+00 | 20260506 01:28:30.836 | INFO | PerfReport: 1.30%, LockDownload -- 1.27e+00 / 0.00e+00, 2.92e+00 / 0.00e+00 | 20260506 01:28:30.836 | INFO | PerfReport: 20.50%, LockCompute -- 2.01e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 01:28:30.836 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.52e-02 / 0.00e+00 | 20260506 01:28:30.952 | INFO | Starting a round of Baz IO | 20260506 01:28:34.097 | INFO | Populating RTMetrics report | 20260506 01:28:34.098 | INFO | RTMetrics report populated | 20260506 01:28:34.098 | INFO | Opening tmp RTMetrics json file | 20260506 01:28:34.099 | INFO | Serializing RTMetrics to json | 20260506 01:28:34.100 | INFO | Serialization done, swapping tmp file now... | 20260506 01:28:34.100 | INFO | RTMetrics done | 20260506 01:28:34.169 | INFO | Finished a round of Baz IO: Wrote 1.824 GiB in 3.217 seconds | 20260506 01:28:34.169 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.31% / 0.00% / 78.68% | 20260506 01:28:34.169 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0801841% / 0% / 7.94729e-05% | 20260506 01:28:34.169 | INFO | Baz IO overall: | 20260506 01:28:34.169 | INFO | BytesWritten : 4440956084463 | 20260506 01:28:34.169 | INFO | NumEvents : 2087341304056 | 20260506 01:28:35.973 | INFO | Analyzing chunk frames = [7872512,7873024) | 20260506 01:28:35.973 | INFO | MemInfo: { | 20260506 01:28:35.973 | INFO | "gpu" : " 17 GB", | 20260506 01:28:35.973 | INFO | "huge" : "", | 20260506 01:28:35.973 | INFO | "huge_shared" : " 27 GB", | 20260506 01:28:35.973 | INFO | "malloc" : " 14 GB", | 20260506 01:28:35.973 | INFO | "pinned" : " 29 GB" | 20260506 01:28:35.973 | INFO | } | 20260506 01:28:35.973 | INFO | Perf report for the last 5.13888 seconds: | 20260506 01:28:35.973 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:28:35.973 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:35.973 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:35.973 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:35.973 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260506 01:28:35.973 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:35.973 | INFO | RT_METRICS : 43.0%, 1.0, 22.6ms, 0 | 20260506 01:28:35.973 | INFO | PRE_HQ : 59.7%, 1.7, 53.6ms, 0 | 20260506 01:28:35.973 | INFO | BAZWRITER : 61.7%, 3.5, 110.4ms, 0 | 20260506 01:28:35.973 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:28:35.974 | INFO | Uploaded 36GiB to GPU in 2.18148s (16.5026GiB/s) | 20260506 01:28:35.974 | INFO | Downloaded 38GiB to GPU in 2.24329s (16.9394GiB/s) | 20260506 01:28:41.116 | INFO | Analyzing chunk frames = [7873024,7873536) | 20260506 01:28:41.116 | INFO | MemInfo: { | 20260506 01:28:41.116 | INFO | "gpu" : " 17 GB", | 20260506 01:28:41.116 | INFO | "huge" : "", | 20260506 01:28:41.116 | INFO | "huge_shared" : " 27 GB", | 20260506 01:28:41.116 | INFO | "malloc" : " 14 GB", | 20260506 01:28:41.116 | INFO | "pinned" : " 29 GB" | 20260506 01:28:41.116 | INFO | } | 20260506 01:28:41.116 | INFO | Perf report for the last 5.14301 seconds: | 20260506 01:28:41.116 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:28:41.116 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:41.116 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:41.116 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:41.116 | INFO | ANALYSIS : 60.8%, 3.0, 95.0ms, 0 | 20260506 01:28:41.116 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:41.116 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:41.116 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 01:28:41.116 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:41.116 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:41.117 | INFO | Uploaded 36GiB to GPU in 1.97562s (18.2221GiB/s) | 20260506 01:28:41.117 | INFO | Downloaded 34GiB to GPU in 1.98173s (17.1567GiB/s) | 20260506 01:28:46.253 | INFO | Analyzing chunk frames = [7873536,7874048) | 20260506 01:28:46.253 | INFO | MemInfo: { | 20260506 01:28:46.253 | INFO | "gpu" : " 17 GB", | 20260506 01:28:46.253 | INFO | "huge" : "", | 20260506 01:28:46.253 | INFO | "huge_shared" : " 27 GB", | 20260506 01:28:46.253 | INFO | "malloc" : " 14 GB", | 20260506 01:28:46.253 | INFO | "pinned" : " 29 GB" | 20260506 01:28:46.253 | INFO | } | 20260506 01:28:46.253 | INFO | Perf report for the last 5.13717 seconds: | 20260506 01:28:46.253 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:28:46.253 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:46.253 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:46.253 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:46.254 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 01:28:46.254 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:46.254 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:46.254 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 01:28:46.254 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:46.254 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:46.254 | INFO | Uploaded 36GiB to GPU in 2.08538s (17.263GiB/s) | 20260506 01:28:46.254 | INFO | Downloaded 34GiB to GPU in 2.02568s (16.7845GiB/s) | 20260506 01:28:51.385 | INFO | Analyzing chunk frames = [7874048,7874560) | 20260506 01:28:51.385 | INFO | MemInfo: { | 20260506 01:28:51.385 | INFO | "gpu" : " 17 GB", | 20260506 01:28:51.385 | INFO | "huge" : "", | 20260506 01:28:51.385 | INFO | "huge_shared" : " 27 GB", | 20260506 01:28:51.385 | INFO | "malloc" : " 14 GB", | 20260506 01:28:51.385 | INFO | "pinned" : " 29 GB" | 20260506 01:28:51.385 | INFO | } | 20260506 01:28:51.386 | INFO | Perf report for the last 5.13206 seconds: | 20260506 01:28:51.386 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:28:51.386 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:51.386 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:51.386 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:51.386 | INFO | ANALYSIS : 61.6%, 3.0, 95.5ms, 0 | 20260506 01:28:51.386 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:51.386 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:51.386 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 01:28:51.386 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:51.386 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:51.386 | INFO | Uploaded 36GiB to GPU in 2.12997s (16.9017GiB/s) | 20260506 01:28:51.386 | INFO | Downloaded 34GiB to GPU in 2.01063s (16.9102GiB/s) | 20260506 01:28:56.530 | INFO | Analyzing chunk frames = [7874560,7875072) | 20260506 01:28:56.530 | INFO | MemInfo: { | 20260506 01:28:56.530 | INFO | "gpu" : " 17 GB", | 20260506 01:28:56.530 | INFO | "huge" : "", | 20260506 01:28:56.530 | INFO | "huge_shared" : " 27 GB", | 20260506 01:28:56.530 | INFO | "malloc" : " 14 GB", | 20260506 01:28:56.530 | INFO | "pinned" : " 29 GB" | 20260506 01:28:56.530 | INFO | } | 20260506 01:28:56.530 | INFO | Perf report for the last 5.14437 seconds: | 20260506 01:28:56.530 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:28:56.530 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:56.530 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:56.530 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:56.530 | INFO | ANALYSIS : 61.1%, 3.0, 95.5ms, 0 | 20260506 01:28:56.530 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:56.530 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:56.530 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:28:56.530 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:56.530 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:28:56.530 | INFO | Uploaded 36GiB to GPU in 2.10749s (17.0819GiB/s) | 20260506 01:28:56.530 | INFO | Downloaded 34GiB to GPU in 2.05633s (16.5343GiB/s) | 20260506 01:29:01.679 | INFO | Analyzing chunk frames = [7875072,7875584) | 20260506 01:29:01.679 | INFO | MemInfo: { | 20260506 01:29:01.679 | INFO | "gpu" : " 17 GB", | 20260506 01:29:01.679 | INFO | "huge" : "", | 20260506 01:29:01.679 | INFO | "huge_shared" : " 27 GB", | 20260506 01:29:01.679 | INFO | "malloc" : " 14 GB", | 20260506 01:29:01.679 | INFO | "pinned" : " 29 GB" | 20260506 01:29:01.680 | INFO | } | 20260506 01:29:01.680 | INFO | Perf report for the last 5.14974 seconds: | 20260506 01:29:01.680 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:29:01.680 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:01.680 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:01.680 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:01.680 | INFO | ANALYSIS : 61.0%, 3.0, 95.5ms, 0 | 20260506 01:29:01.680 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:01.680 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:01.680 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:29:01.680 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:01.680 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:01.680 | INFO | Uploaded 36GiB to GPU in 2.06396s (17.4422GiB/s) | 20260506 01:29:01.680 | INFO | Downloaded 34GiB to GPU in 2.04023s (16.6648GiB/s) | 20260506 01:29:06.810 | INFO | Analyzing chunk frames = [7875584,7876096) | 20260506 01:29:06.810 | INFO | MemInfo: { | 20260506 01:29:06.810 | INFO | "gpu" : " 17 GB", | 20260506 01:29:06.810 | INFO | "huge" : "", | 20260506 01:29:06.810 | INFO | "huge_shared" : " 27 GB", | 20260506 01:29:06.810 | INFO | "malloc" : " 14 GB", | 20260506 01:29:06.810 | INFO | "pinned" : " 29 GB" | 20260506 01:29:06.810 | INFO | } | 20260506 01:29:06.810 | INFO | Perf report for the last 5.13041 seconds: | 20260506 01:29:06.810 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:29:06.810 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:06.810 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:06.810 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:06.810 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 01:29:06.810 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:06.810 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:06.810 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 01:29:06.810 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:06.810 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:06.811 | INFO | Uploaded 36GiB to GPU in 2.06227s (17.4565GiB/s) | 20260506 01:29:06.811 | INFO | Downloaded 34GiB to GPU in 2.07078s (16.4189GiB/s) | 20260506 01:29:11.948 | INFO | Analyzing chunk frames = [7876096,7876608) | 20260506 01:29:11.948 | INFO | MemInfo: { | 20260506 01:29:11.948 | INFO | "gpu" : " 17 GB", | 20260506 01:29:11.948 | INFO | "huge" : "", | 20260506 01:29:11.948 | INFO | "huge_shared" : " 27 GB", | 20260506 01:29:11.948 | INFO | "malloc" : " 14 GB", | 20260506 01:29:11.948 | INFO | "pinned" : " 29 GB" | 20260506 01:29:11.948 | INFO | } | 20260506 01:29:11.948 | INFO | Perf report for the last 5.13821 seconds: | 20260506 01:29:11.948 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:29:11.948 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:11.948 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:11.948 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:11.948 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 01:29:11.948 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:11.948 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:11.948 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 01:29:11.948 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:11.948 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:11.949 | INFO | Uploaded 36GiB to GPU in 2.12965s (16.9042GiB/s) | 20260506 01:29:11.949 | INFO | Downloaded 34GiB to GPU in 2.0203s (16.8292GiB/s) | 20260506 01:29:12.067 | INFO | Starting a round of Baz IO | 20260506 01:29:15.211 | INFO | Populating RTMetrics report | 20260506 01:29:15.211 | INFO | RTMetrics report populated | 20260506 01:29:15.211 | INFO | Opening tmp RTMetrics json file | 20260506 01:29:15.212 | INFO | Serializing RTMetrics to json | 20260506 01:29:15.213 | INFO | Serialization done, swapping tmp file now... | 20260506 01:29:15.214 | INFO | RTMetrics done | 20260506 01:29:15.287 | INFO | Finished a round of Baz IO: Wrote 1.822 GiB in 3.219 seconds | 20260506 01:29:15.287 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.31% / 0.00% / 78.69% | 20260506 01:29:15.287 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0799855% / 0% / 9.13938e-05% | 20260506 01:29:15.287 | INFO | Baz IO overall: | 20260506 01:29:15.287 | INFO | BytesWritten : 4442912795044 | 20260506 01:29:15.287 | INFO | NumEvents : 2088232915435 | 20260506 01:29:17.086 | INFO | Analyzing chunk frames = [7876608,7877120) | 20260506 01:29:17.086 | INFO | MemInfo: { | 20260506 01:29:17.086 | INFO | "gpu" : " 17 GB", | 20260506 01:29:17.086 | INFO | "huge" : "", | 20260506 01:29:17.086 | INFO | "huge_shared" : " 27 GB", | 20260506 01:29:17.086 | INFO | "malloc" : " 14 GB", | 20260506 01:29:17.086 | INFO | "pinned" : " 29 GB" | 20260506 01:29:17.086 | INFO | } | 20260506 01:29:17.086 | INFO | Perf report for the last 5.13772 seconds: | 20260506 01:29:17.086 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:29:17.086 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:17.086 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:17.086 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:17.086 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260506 01:29:17.086 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:17.086 | INFO | RT_METRICS : 43.0%, 1.0, 22.6ms, 0 | 20260506 01:29:17.086 | INFO | PRE_HQ : 59.8%, 1.7, 53.7ms, 0 | 20260506 01:29:17.086 | INFO | BAZWRITER : 61.8%, 3.4, 109.2ms, 0 | 20260506 01:29:17.086 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:29:17.086 | INFO | Uploaded 36GiB to GPU in 2.17026s (16.5879GiB/s) | 20260506 01:29:17.086 | INFO | Downloaded 38GiB to GPU in 2.2401s (16.9636GiB/s) | 20260506 01:29:22.226 | INFO | Analyzing chunk frames = [7877120,7877632) | 20260506 01:29:22.226 | INFO | MemInfo: { | 20260506 01:29:22.226 | INFO | "gpu" : " 17 GB", | 20260506 01:29:22.226 | INFO | "huge" : "", | 20260506 01:29:22.226 | INFO | "huge_shared" : " 27 GB", | 20260506 01:29:22.227 | INFO | "malloc" : " 14 GB", | 20260506 01:29:22.227 | INFO | "pinned" : " 29 GB" | 20260506 01:29:22.227 | INFO | } | 20260506 01:29:22.227 | INFO | Perf report for the last 5.14049 seconds: | 20260506 01:29:22.227 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:29:22.227 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:22.227 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:22.227 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:22.227 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 01:29:22.227 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:22.227 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:22.227 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 01:29:22.227 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:22.227 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:22.227 | INFO | Uploaded 36GiB to GPU in 1.96117s (18.3563GiB/s) | 20260506 01:29:22.227 | INFO | Downloaded 34GiB to GPU in 1.99562s (17.0373GiB/s) | 20260506 01:29:27.367 | INFO | Analyzing chunk frames = [7877632,7878144) | 20260506 01:29:27.367 | INFO | MemInfo: { | 20260506 01:29:27.367 | INFO | "gpu" : " 17 GB", | 20260506 01:29:27.367 | INFO | "huge" : "", | 20260506 01:29:27.367 | INFO | "huge_shared" : " 27 GB", | 20260506 01:29:27.367 | INFO | "malloc" : " 14 GB", | 20260506 01:29:27.367 | INFO | "pinned" : " 29 GB" | 20260506 01:29:27.367 | INFO | } | 20260506 01:29:27.367 | INFO | Perf report for the last 5.14029 seconds: | 20260506 01:29:27.367 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:29:27.367 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:27.367 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:27.367 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:27.367 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 01:29:27.367 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:27.367 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:27.367 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:29:27.367 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:27.367 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:27.367 | INFO | Uploaded 36GiB to GPU in 2.08764s (17.2443GiB/s) | 20260506 01:29:27.367 | INFO | Downloaded 34GiB to GPU in 2.02086s (16.8245GiB/s) | 20260506 01:29:32.506 | INFO | Analyzing chunk frames = [7878144,7878656) | 20260506 01:29:32.506 | INFO | MemInfo: { | 20260506 01:29:32.506 | INFO | "gpu" : " 17 GB", | 20260506 01:29:32.506 | INFO | "huge" : "", | 20260506 01:29:32.506 | INFO | "huge_shared" : " 27 GB", | 20260506 01:29:32.506 | INFO | "malloc" : " 14 GB", | 20260506 01:29:32.506 | INFO | "pinned" : " 29 GB" | 20260506 01:29:32.506 | INFO | } | 20260506 01:29:32.506 | INFO | Perf report for the last 5.1391 seconds: | 20260506 01:29:32.506 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:29:32.506 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:32.506 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:32.506 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:32.506 | INFO | ANALYSIS : 61.5%, 3.0, 95.5ms, 0 | 20260506 01:29:32.506 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:32.506 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:32.506 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 01:29:32.506 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:32.506 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:32.506 | INFO | Uploaded 36GiB to GPU in 2.1499s (16.7449GiB/s) | 20260506 01:29:32.506 | INFO | Downloaded 34GiB to GPU in 2.05287s (16.5621GiB/s) | 20260506 01:29:37.649 | INFO | Analyzing chunk frames = [7878656,7879168) | 20260506 01:29:37.649 | INFO | MemInfo: { | 20260506 01:29:37.649 | INFO | "gpu" : " 17 GB", | 20260506 01:29:37.649 | INFO | "huge" : "", | 20260506 01:29:37.649 | INFO | "huge_shared" : " 27 GB", | 20260506 01:29:37.649 | INFO | "malloc" : " 14 GB", | 20260506 01:29:37.649 | INFO | "pinned" : " 29 GB" | 20260506 01:29:37.649 | INFO | } | 20260506 01:29:37.649 | INFO | Perf report for the last 5.14286 seconds: | 20260506 01:29:37.649 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:29:37.649 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:37.649 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:37.649 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:37.649 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 01:29:37.649 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:37.649 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:37.649 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:29:37.649 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:37.649 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:37.649 | INFO | Uploaded 36GiB to GPU in 2.09848s (17.1553GiB/s) | 20260506 01:29:37.649 | INFO | Downloaded 34GiB to GPU in 2.04989s (16.5863GiB/s) | 20260506 01:29:42.789 | INFO | Analyzing chunk frames = [7879168,7879680) | 20260506 01:29:42.789 | INFO | MemInfo: { | 20260506 01:29:42.789 | INFO | "gpu" : " 17 GB", | 20260506 01:29:42.789 | INFO | "huge" : "", | 20260506 01:29:42.789 | INFO | "huge_shared" : " 27 GB", | 20260506 01:29:42.789 | INFO | "malloc" : " 14 GB", | 20260506 01:29:42.789 | INFO | "pinned" : " 29 GB" | 20260506 01:29:42.789 | INFO | } | 20260506 01:29:42.790 | INFO | Perf report for the last 5.14065 seconds: | 20260506 01:29:42.790 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:29:42.790 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:42.790 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:42.790 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:42.790 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 01:29:42.790 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:42.790 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:42.790 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 01:29:42.790 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:42.790 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:42.790 | INFO | Uploaded 36GiB to GPU in 2.07321s (17.3644GiB/s) | 20260506 01:29:42.790 | INFO | Downloaded 34GiB to GPU in 2.04865s (16.5963GiB/s) | 20260506 01:29:47.926 | INFO | Analyzing chunk frames = [7879680,7880192) | 20260506 01:29:47.926 | INFO | MemInfo: { | 20260506 01:29:47.926 | INFO | "gpu" : " 17 GB", | 20260506 01:29:47.926 | INFO | "huge" : "", | 20260506 01:29:47.926 | INFO | "huge_shared" : " 27 GB", | 20260506 01:29:47.926 | INFO | "malloc" : " 14 GB", | 20260506 01:29:47.926 | INFO | "pinned" : " 29 GB" | 20260506 01:29:47.926 | INFO | } | 20260506 01:29:47.926 | INFO | Perf report for the last 5.1366 seconds: | 20260506 01:29:47.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:29:47.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:47.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:47.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:47.926 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260506 01:29:47.926 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:47.926 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:47.926 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 01:29:47.926 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:47.926 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:47.926 | INFO | Uploaded 36GiB to GPU in 2.09472s (17.186GiB/s) | 20260506 01:29:47.926 | INFO | Downloaded 34GiB to GPU in 2.0876s (16.2866GiB/s) | 20260506 01:29:53.063 | INFO | Analyzing chunk frames = [7880192,7880704) | 20260506 01:29:53.063 | INFO | MemInfo: { | 20260506 01:29:53.063 | INFO | "gpu" : " 17 GB", | 20260506 01:29:53.063 | INFO | "huge" : "", | 20260506 01:29:53.063 | INFO | "huge_shared" : " 27 GB", | 20260506 01:29:53.063 | INFO | "malloc" : " 14 GB", | 20260506 01:29:53.063 | INFO | "pinned" : " 29 GB" | 20260506 01:29:53.063 | INFO | } | 20260506 01:29:53.063 | INFO | Perf report for the last 5.13726 seconds: | 20260506 01:29:53.063 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:29:53.063 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:53.063 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:53.063 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:53.063 | INFO | ANALYSIS : 61.4%, 3.0, 95.9ms, 0 | 20260506 01:29:53.063 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:53.063 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:53.063 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 01:29:53.064 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:53.064 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:53.064 | INFO | Uploaded 36GiB to GPU in 2.04734s (17.5838GiB/s) | 20260506 01:29:53.064 | INFO | Downloaded 34GiB to GPU in 2.02171s (16.8174GiB/s) | 20260506 01:29:53.064 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 01:29:53.064 | INFO | Total mem usage: 15.3 GiB | 20260506 01:29:53.064 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 01:29:53.064 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.19%) | 20260506 01:29:53.064 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 01:29:53.064 | INFO | High Water Breakdown: | 20260506 01:29:53.064 | INFO | 60.9% : 8.54 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 01:29:53.064 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 01:29:53.064 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 01:29:53.064 | INFO | ------------------------------------------ | 20260506 01:29:53.064 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:29:53.064 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 01:29:53.064 | INFO | Total mem usage: 26.5 GiB | 20260506 01:29:53.064 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 01:29:53.064 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 01:29:53.064 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260506 01:29:53.064 | INFO | High Water Breakdown: | 20260506 01:29:53.064 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:29:53.064 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:29:53.064 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:29:53.064 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:29:53.064 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:29:53.064 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:29:53.064 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 01:29:53.064 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 01:29:53.064 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:29:53.064 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 01:29:53.064 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 01:29:53.064 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 01:29:53.064 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 01:29:53.064 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 01:29:53.064 | INFO | ------------------------------------------ | 20260506 01:29:53.064 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:29:53.064 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 01:29:53.064 | INFO | Total mem usage: 25.6 GiB | 20260506 01:29:53.064 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 01:29:53.064 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 01:29:53.064 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 01:29:53.064 | INFO | High Water Breakdown: | 20260506 01:29:53.064 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 01:29:53.064 | INFO | ------------------------------------------ | 20260506 01:29:53.064 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:29:53.064 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 01:29:53.064 | INFO | Total mem usage: 15.7 GiB | 20260506 01:29:53.064 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 01:29:53.064 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 01:29:53.064 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 01:29:53.064 | INFO | High Water Breakdown: | 20260506 01:29:53.064 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 01:29:53.064 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 01:29:53.064 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 01:29:53.064 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 01:29:53.064 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 01:29:53.064 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 01:29:53.064 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 01:29:53.064 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 01:29:53.064 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 01:29:53.064 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:29:53.064 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 01:29:53.064 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:29:53.064 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 01:29:53.064 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 01:29:53.064 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 01:29:53.064 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 01:29:53.064 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:29:53.065 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 01:29:53.065 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:29:53.065 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:29:53.065 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:29:53.065 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 01:29:53.065 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 01:29:53.065 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:29:53.065 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 01:29:53.065 | INFO | ------------------------------------------ | 20260506 01:29:53.065 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:29:53.065 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 01:29:53.065 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:29:53.065 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:29:53.065 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.37e+01, 1.33e+01 / 6.04e-01 | 20260506 01:29:53.065 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.05e-01, 1.02e-02 / 4.11e-03 | 20260506 01:29:53.065 | INFO | PerfReport: 11.84%, Baseline -- 3.71e+00 / 4.53e+00, 5.74e-01 / 1.83e-01 | 20260506 01:29:53.065 | INFO | PerfReport: 3.29%, Histogram -- 1.03e+00 / 1.54e+00, 1.94e-01 / 5.06e-02 | 20260506 01:29:53.065 | INFO | PerfReport: 15.98%, DME -- 4.00e+01 / 4.38e+01, 3.68e+00 / 7.27e-01 | 20260506 01:29:53.065 | INFO | PerfReport: 57.42%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.50e+00 / 1.04e+00 | 20260506 01:29:53.065 | INFO | PerfReport: 7.06%, PulseAccumulating -- 2.21e+00 / 2.72e+00, 4.21e-01 / 2.15e-01 | 20260506 01:29:53.065 | INFO | PerfReport: 3.05%, Metrics -- 9.53e-01 / 1.23e+00, 2.55e-01 / 9.76e-02 | 20260506 01:29:53.065 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.29e-01 / 3.79e-01, 5.55e-02 / 2.30e-02 | 20260506 01:29:53.065 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 01:29:53.065 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:29:53.065 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:29:53.065 | INFO | PerfReport: Full Execution -- 9.79e+01 / -nan, 2.43e+01 / 0.00e+00 | 20260506 01:29:53.065 | INFO | PerfReport: 22.21%, Upload -- 2.18e+01 / 0.00e+00, 7.35e+00 / 0.00e+00 | 20260506 01:29:53.065 | INFO | PerfReport: 21.91%, Download -- 2.15e+01 / 0.00e+00, 7.59e+00 / 0.00e+00 | 20260506 01:29:53.065 | INFO | PerfReport: 31.95%, Compute -- 3.13e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 01:29:53.065 | INFO | PerfReport: 2.05%, LockUpload -- 2.01e+00 / 0.00e+00, 7.77e+00 / 0.00e+00 | 20260506 01:29:53.065 | INFO | PerfReport: 1.34%, LockDownload -- 1.31e+00 / 0.00e+00, 2.93e+00 / 0.00e+00 | 20260506 01:29:53.065 | INFO | PerfReport: 20.52%, LockCompute -- 2.01e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 01:29:53.065 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.71e-02 / 0.00e+00 | 20260506 01:29:53.181 | INFO | Starting a round of Baz IO | 20260506 01:29:56.334 | INFO | Populating RTMetrics report | 20260506 01:29:56.334 | INFO | RTMetrics report populated | 20260506 01:29:56.334 | INFO | Opening tmp RTMetrics json file | 20260506 01:29:56.335 | INFO | Serializing RTMetrics to json | 20260506 01:29:56.336 | INFO | Serialization done, swapping tmp file now... | 20260506 01:29:56.337 | INFO | RTMetrics done | 20260506 01:29:56.411 | INFO | Finished a round of Baz IO: Wrote 1.824 GiB in 3.23 seconds | 20260506 01:29:56.411 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.31% / 0.00% / 78.69% | 20260506 01:29:56.411 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0799775% / 0% / 7.54992e-05% | 20260506 01:29:56.411 | INFO | Baz IO overall: | 20260506 01:29:56.411 | INFO | BytesWritten : 4444871718811 | 20260506 01:29:56.411 | INFO | NumEvents : 2089126122307 | 20260506 01:29:58.205 | INFO | Analyzing chunk frames = [7880704,7881216) | 20260506 01:29:58.205 | INFO | MemInfo: { | 20260506 01:29:58.205 | INFO | "gpu" : " 17 GB", | 20260506 01:29:58.205 | INFO | "huge" : "", | 20260506 01:29:58.205 | INFO | "huge_shared" : " 27 GB", | 20260506 01:29:58.205 | INFO | "malloc" : " 14 GB", | 20260506 01:29:58.205 | INFO | "pinned" : " 29 GB" | 20260506 01:29:58.205 | INFO | } | 20260506 01:29:58.205 | INFO | Perf report for the last 5.14193 seconds: | 20260506 01:29:58.205 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:29:58.205 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:58.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:58.206 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:58.206 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260506 01:29:58.206 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:29:58.206 | INFO | RT_METRICS : 43.0%, 1.0, 22.6ms, 0 | 20260506 01:29:58.206 | INFO | PRE_HQ : 59.8%, 1.7, 54.0ms, 0 | 20260506 01:29:58.206 | INFO | BAZWRITER : 61.8%, 3.4, 109.7ms, 0 | 20260506 01:29:58.206 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:29:58.206 | INFO | Uploaded 36GiB to GPU in 2.18362s (16.4864GiB/s) | 20260506 01:29:58.206 | INFO | Downloaded 38GiB to GPU in 2.23808s (16.9789GiB/s) | 20260506 01:30:03.337 | INFO | Analyzing chunk frames = [7881216,7881728) | 20260506 01:30:03.337 | INFO | MemInfo: { | 20260506 01:30:03.337 | INFO | "gpu" : " 17 GB", | 20260506 01:30:03.337 | INFO | "huge" : "", | 20260506 01:30:03.337 | INFO | "huge_shared" : " 27 GB", | 20260506 01:30:03.337 | INFO | "malloc" : " 14 GB", | 20260506 01:30:03.337 | INFO | "pinned" : " 29 GB" | 20260506 01:30:03.337 | INFO | } | 20260506 01:30:03.337 | INFO | Perf report for the last 5.13193 seconds: | 20260506 01:30:03.337 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:30:03.337 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:03.337 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:03.337 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:03.337 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 01:30:03.337 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:03.337 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:03.337 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:30:03.337 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:03.337 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:03.338 | INFO | Uploaded 36GiB to GPU in 2.10629s (17.0917GiB/s) | 20260506 01:30:03.338 | INFO | Downloaded 34GiB to GPU in 2.0422s (16.6487GiB/s) | 20260506 01:30:08.485 | INFO | Analyzing chunk frames = [7881728,7882240) | 20260506 01:30:08.485 | INFO | MemInfo: { | 20260506 01:30:08.485 | INFO | "gpu" : " 17 GB", | 20260506 01:30:08.485 | INFO | "huge" : "", | 20260506 01:30:08.485 | INFO | "huge_shared" : " 27 GB", | 20260506 01:30:08.485 | INFO | "malloc" : " 14 GB", | 20260506 01:30:08.485 | INFO | "pinned" : " 29 GB" | 20260506 01:30:08.485 | INFO | } | 20260506 01:30:08.485 | INFO | Perf report for the last 5.14776 seconds: | 20260506 01:30:08.485 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:30:08.485 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:08.485 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:08.485 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:08.485 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 01:30:08.485 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:08.485 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:08.485 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:30:08.485 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:08.485 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:08.485 | INFO | Uploaded 36GiB to GPU in 2.08218s (17.2896GiB/s) | 20260506 01:30:08.485 | INFO | Downloaded 34GiB to GPU in 2.01889s (16.8409GiB/s) | 20260506 01:30:13.631 | INFO | Analyzing chunk frames = [7882240,7882752) | 20260506 01:30:13.631 | INFO | MemInfo: { | 20260506 01:30:13.631 | INFO | "gpu" : " 17 GB", | 20260506 01:30:13.631 | INFO | "huge" : "", | 20260506 01:30:13.631 | INFO | "huge_shared" : " 27 GB", | 20260506 01:30:13.631 | INFO | "malloc" : " 14 GB", | 20260506 01:30:13.631 | INFO | "pinned" : " 29 GB" | 20260506 01:30:13.631 | INFO | } | 20260506 01:30:13.631 | INFO | Perf report for the last 5.14589 seconds: | 20260506 01:30:13.631 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:30:13.631 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:13.631 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:13.631 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:13.631 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260506 01:30:13.631 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:13.631 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:13.631 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:30:13.631 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:13.631 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:13.631 | INFO | Uploaded 36GiB to GPU in 2.14414s (16.7899GiB/s) | 20260506 01:30:13.631 | INFO | Downloaded 34GiB to GPU in 2.05287s (16.5622GiB/s) | 20260506 01:30:18.763 | INFO | Analyzing chunk frames = [7882752,7883264) | 20260506 01:30:18.763 | INFO | MemInfo: { | 20260506 01:30:18.763 | INFO | "gpu" : " 17 GB", | 20260506 01:30:18.763 | INFO | "huge" : "", | 20260506 01:30:18.763 | INFO | "huge_shared" : " 27 GB", | 20260506 01:30:18.763 | INFO | "malloc" : " 14 GB", | 20260506 01:30:18.763 | INFO | "pinned" : " 29 GB" | 20260506 01:30:18.763 | INFO | } | 20260506 01:30:18.764 | INFO | Perf report for the last 5.13255 seconds: | 20260506 01:30:18.764 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:30:18.764 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:18.764 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:18.764 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:18.764 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260506 01:30:18.764 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:18.764 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:18.764 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:30:18.764 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:18.764 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:18.764 | INFO | Uploaded 36GiB to GPU in 2.06941s (17.3963GiB/s) | 20260506 01:30:18.764 | INFO | Downloaded 34GiB to GPU in 2.02378s (16.8003GiB/s) | 20260506 01:30:23.894 | INFO | Analyzing chunk frames = [7883264,7883776) | 20260506 01:30:23.894 | INFO | MemInfo: { | 20260506 01:30:23.894 | INFO | "gpu" : " 17 GB", | 20260506 01:30:23.894 | INFO | "huge" : "", | 20260506 01:30:23.894 | INFO | "huge_shared" : " 27 GB", | 20260506 01:30:23.894 | INFO | "malloc" : " 14 GB", | 20260506 01:30:23.894 | INFO | "pinned" : " 29 GB" | 20260506 01:30:23.894 | INFO | } | 20260506 01:30:23.895 | INFO | Perf report for the last 5.13098 seconds: | 20260506 01:30:23.895 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:30:23.895 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:23.895 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:23.895 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:23.895 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 01:30:23.895 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:23.895 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:23.895 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:30:23.895 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:23.895 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:23.895 | INFO | Uploaded 36GiB to GPU in 2.06847s (17.4041GiB/s) | 20260506 01:30:23.895 | INFO | Downloaded 34GiB to GPU in 2.03631s (16.6969GiB/s) | 20260506 01:30:29.036 | INFO | Analyzing chunk frames = [7883776,7884288) | 20260506 01:30:29.036 | INFO | MemInfo: { | 20260506 01:30:29.036 | INFO | "gpu" : " 17 GB", | 20260506 01:30:29.036 | INFO | "huge" : "", | 20260506 01:30:29.036 | INFO | "huge_shared" : " 27 GB", | 20260506 01:30:29.036 | INFO | "malloc" : " 14 GB", | 20260506 01:30:29.036 | INFO | "pinned" : " 29 GB" | 20260506 01:30:29.036 | INFO | } | 20260506 01:30:29.036 | INFO | Perf report for the last 5.14161 seconds: | 20260506 01:30:29.036 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:30:29.036 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:29.036 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:29.036 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:29.036 | INFO | ANALYSIS : 62.5%, 3.0, 97.7ms, 0 | 20260506 01:30:29.036 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:29.036 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:29.036 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 01:30:29.036 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:29.036 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:29.036 | INFO | Uploaded 36GiB to GPU in 2.14198s (16.8069GiB/s) | 20260506 01:30:29.036 | INFO | Downloaded 34GiB to GPU in 2.10211s (16.1742GiB/s) | 20260506 01:30:34.182 | INFO | Analyzing chunk frames = [7884288,7884800) | 20260506 01:30:34.182 | INFO | MemInfo: { | 20260506 01:30:34.182 | INFO | "gpu" : " 17 GB", | 20260506 01:30:34.182 | INFO | "huge" : "", | 20260506 01:30:34.182 | INFO | "huge_shared" : " 27 GB", | 20260506 01:30:34.182 | INFO | "malloc" : " 14 GB", | 20260506 01:30:34.182 | INFO | "pinned" : " 29 GB" | 20260506 01:30:34.182 | INFO | } | 20260506 01:30:34.182 | INFO | Perf report for the last 5.14608 seconds: | 20260506 01:30:34.182 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:30:34.182 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:34.182 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:34.182 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:34.182 | INFO | ANALYSIS : 61.8%, 3.0, 96.6ms, 0 | 20260506 01:30:34.182 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:34.182 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:34.182 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:30:34.182 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:34.182 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:34.183 | INFO | Uploaded 36GiB to GPU in 2.12693s (16.9258GiB/s) | 20260506 01:30:34.183 | INFO | Downloaded 34GiB to GPU in 2.02673s (16.7758GiB/s) | 20260506 01:30:34.318 | INFO | Starting a round of Baz IO | 20260506 01:30:37.433 | INFO | Populating RTMetrics report | 20260506 01:30:37.434 | INFO | RTMetrics report populated | 20260506 01:30:37.434 | INFO | Opening tmp RTMetrics json file | 20260506 01:30:37.434 | INFO | Serializing RTMetrics to json | 20260506 01:30:37.436 | INFO | Serialization done, swapping tmp file now... | 20260506 01:30:37.437 | INFO | RTMetrics done | 20260506 01:30:37.508 | INFO | Finished a round of Baz IO: Wrote 1.823 GiB in 3.189 seconds | 20260506 01:30:37.508 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.31% / 0.00% / 78.69% | 20260506 01:30:37.508 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0800769% / 3.97364e-06% / 9.13938e-05% | 20260506 01:30:37.508 | INFO | Baz IO overall: | 20260506 01:30:37.508 | INFO | BytesWritten : 4446828692986 | 20260506 01:30:37.508 | INFO | NumEvents : 2090018008766 | 20260506 01:30:39.316 | INFO | Analyzing chunk frames = [7884800,7885312) | 20260506 01:30:39.316 | INFO | MemInfo: { | 20260506 01:30:39.316 | INFO | "gpu" : " 17 GB", | 20260506 01:30:39.316 | INFO | "huge" : "", | 20260506 01:30:39.316 | INFO | "huge_shared" : " 27 GB", | 20260506 01:30:39.316 | INFO | "malloc" : " 14 GB", | 20260506 01:30:39.316 | INFO | "pinned" : " 29 GB" | 20260506 01:30:39.316 | INFO | } | 20260506 01:30:39.316 | INFO | Perf report for the last 5.13422 seconds: | 20260506 01:30:39.316 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:30:39.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:39.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:39.317 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:39.317 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260506 01:30:39.317 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:39.317 | INFO | RT_METRICS : 43.0%, 1.0, 22.5ms, 0 | 20260506 01:30:39.317 | INFO | PRE_HQ : 59.2%, 1.8, 53.9ms, 0 | 20260506 01:30:39.317 | INFO | BAZWRITER : 61.3%, 3.5, 109.9ms, 0 | 20260506 01:30:39.317 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:30:39.317 | INFO | Uploaded 36GiB to GPU in 2.11467s (17.024GiB/s) | 20260506 01:30:39.317 | INFO | Downloaded 38GiB to GPU in 2.22081s (17.1109GiB/s) | 20260506 01:30:44.458 | INFO | Analyzing chunk frames = [7885312,7885824) | 20260506 01:30:44.458 | INFO | MemInfo: { | 20260506 01:30:44.458 | INFO | "gpu" : " 17 GB", | 20260506 01:30:44.458 | INFO | "huge" : "", | 20260506 01:30:44.458 | INFO | "huge_shared" : " 27 GB", | 20260506 01:30:44.458 | INFO | "malloc" : " 14 GB", | 20260506 01:30:44.458 | INFO | "pinned" : " 29 GB" | 20260506 01:30:44.458 | INFO | } | 20260506 01:30:44.458 | INFO | Perf report for the last 5.14151 seconds: | 20260506 01:30:44.458 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:30:44.458 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:44.458 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:44.458 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:44.458 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260506 01:30:44.458 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:44.458 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:44.458 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 01:30:44.458 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:44.458 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:44.458 | INFO | Uploaded 36GiB to GPU in 2.11216s (17.0442GiB/s) | 20260506 01:30:44.458 | INFO | Downloaded 34GiB to GPU in 2.04272s (16.6445GiB/s) | 20260506 01:30:49.598 | INFO | Analyzing chunk frames = [7885824,7886336) | 20260506 01:30:49.598 | INFO | MemInfo: { | 20260506 01:30:49.598 | INFO | "gpu" : " 17 GB", | 20260506 01:30:49.598 | INFO | "huge" : "", | 20260506 01:30:49.598 | INFO | "huge_shared" : " 27 GB", | 20260506 01:30:49.598 | INFO | "malloc" : " 14 GB", | 20260506 01:30:49.598 | INFO | "pinned" : " 29 GB" | 20260506 01:30:49.598 | INFO | } | 20260506 01:30:49.598 | INFO | Perf report for the last 5.14003 seconds: | 20260506 01:30:49.598 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:30:49.598 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:49.598 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:49.598 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:49.598 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 01:30:49.598 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:49.598 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:49.598 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 01:30:49.598 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:49.598 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:49.598 | INFO | Uploaded 36GiB to GPU in 2.06025s (17.4736GiB/s) | 20260506 01:30:49.598 | INFO | Downloaded 34GiB to GPU in 2.02111s (16.8224GiB/s) | 20260506 01:30:54.738 | INFO | Analyzing chunk frames = [7886336,7886848) | 20260506 01:30:54.738 | INFO | MemInfo: { | 20260506 01:30:54.738 | INFO | "gpu" : " 17 GB", | 20260506 01:30:54.738 | INFO | "huge" : "", | 20260506 01:30:54.738 | INFO | "huge_shared" : " 27 GB", | 20260506 01:30:54.738 | INFO | "malloc" : " 14 GB", | 20260506 01:30:54.738 | INFO | "pinned" : " 29 GB" | 20260506 01:30:54.738 | INFO | } | 20260506 01:30:54.738 | INFO | Perf report for the last 5.13999 seconds: | 20260506 01:30:54.738 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:30:54.738 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:54.738 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:54.738 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:54.738 | INFO | ANALYSIS : 61.6%, 3.0, 95.6ms, 0 | 20260506 01:30:54.738 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:54.738 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:54.738 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 01:30:54.738 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:54.738 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:54.738 | INFO | Uploaded 36GiB to GPU in 2.13446s (16.8661GiB/s) | 20260506 01:30:54.738 | INFO | Downloaded 34GiB to GPU in 2.01841s (16.845GiB/s) | 20260506 01:30:59.877 | INFO | Analyzing chunk frames = [7886848,7887360) | 20260506 01:30:59.877 | INFO | MemInfo: { | 20260506 01:30:59.877 | INFO | "gpu" : " 17 GB", | 20260506 01:30:59.877 | INFO | "huge" : "", | 20260506 01:30:59.877 | INFO | "huge_shared" : " 27 GB", | 20260506 01:30:59.877 | INFO | "malloc" : " 14 GB", | 20260506 01:30:59.877 | INFO | "pinned" : " 29 GB" | 20260506 01:30:59.877 | INFO | } | 20260506 01:30:59.877 | INFO | Perf report for the last 5.13898 seconds: | 20260506 01:30:59.877 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:30:59.877 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:59.877 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:59.877 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:59.877 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 01:30:59.877 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:59.877 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:59.877 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:30:59.877 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:59.877 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:30:59.877 | INFO | Uploaded 36GiB to GPU in 2.06545s (17.4296GiB/s) | 20260506 01:30:59.877 | INFO | Downloaded 34GiB to GPU in 2.02834s (16.7625GiB/s) | 20260506 01:31:05.011 | INFO | Analyzing chunk frames = [7887360,7887872) | 20260506 01:31:05.011 | INFO | MemInfo: { | 20260506 01:31:05.011 | INFO | "gpu" : " 17 GB", | 20260506 01:31:05.011 | INFO | "huge" : "", | 20260506 01:31:05.011 | INFO | "huge_shared" : " 27 GB", | 20260506 01:31:05.012 | INFO | "malloc" : " 14 GB", | 20260506 01:31:05.012 | INFO | "pinned" : " 29 GB" | 20260506 01:31:05.012 | INFO | } | 20260506 01:31:05.012 | INFO | Perf report for the last 5.13462 seconds: | 20260506 01:31:05.012 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:31:05.012 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:05.012 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:05.012 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:05.012 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 01:31:05.012 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:05.012 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:05.012 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:31:05.012 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:05.012 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:05.012 | INFO | Uploaded 36GiB to GPU in 2.06573s (17.4273GiB/s) | 20260506 01:31:05.012 | INFO | Downloaded 34GiB to GPU in 2.0356s (16.7027GiB/s) | 20260506 01:31:10.158 | INFO | Analyzing chunk frames = [7887872,7888384) | 20260506 01:31:10.158 | INFO | MemInfo: { | 20260506 01:31:10.158 | INFO | "gpu" : " 17 GB", | 20260506 01:31:10.158 | INFO | "huge" : "", | 20260506 01:31:10.158 | INFO | "huge_shared" : " 27 GB", | 20260506 01:31:10.158 | INFO | "malloc" : " 14 GB", | 20260506 01:31:10.158 | INFO | "pinned" : " 29 GB" | 20260506 01:31:10.158 | INFO | } | 20260506 01:31:10.158 | INFO | Perf report for the last 5.14645 seconds: | 20260506 01:31:10.158 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:31:10.158 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:10.158 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:10.158 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:10.158 | INFO | ANALYSIS : 62.2%, 3.0, 97.3ms, 0 | 20260506 01:31:10.158 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:10.158 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:10.158 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 01:31:10.158 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:10.158 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:10.158 | INFO | Uploaded 36GiB to GPU in 2.10261s (17.1216GiB/s) | 20260506 01:31:10.158 | INFO | Downloaded 34GiB to GPU in 2.07722s (16.368GiB/s) | 20260506 01:31:15.292 | INFO | Analyzing chunk frames = [7888384,7888896) | 20260506 01:31:15.292 | INFO | MemInfo: { | 20260506 01:31:15.292 | INFO | "gpu" : " 17 GB", | 20260506 01:31:15.292 | INFO | "huge" : "", | 20260506 01:31:15.292 | INFO | "huge_shared" : " 27 GB", | 20260506 01:31:15.292 | INFO | "malloc" : " 14 GB", | 20260506 01:31:15.292 | INFO | "pinned" : " 29 GB" | 20260506 01:31:15.292 | INFO | } | 20260506 01:31:15.292 | INFO | Perf report for the last 5.13422 seconds: | 20260506 01:31:15.292 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:31:15.292 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:15.292 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:15.292 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:15.292 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260506 01:31:15.292 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:15.292 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:15.292 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 01:31:15.292 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:15.292 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:15.293 | INFO | Uploaded 36GiB to GPU in 2.12806s (16.9169GiB/s) | 20260506 01:31:15.293 | INFO | Downloaded 34GiB to GPU in 2.02968s (16.7514GiB/s) | 20260506 01:31:15.293 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 01:31:15.293 | INFO | Total mem usage: 15.3 GiB | 20260506 01:31:15.293 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 01:31:15.293 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.16%) | 20260506 01:31:15.293 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 01:31:15.293 | INFO | High Water Breakdown: | 20260506 01:31:15.293 | INFO | 60.9% : 8.55 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 01:31:15.293 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 01:31:15.293 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 01:31:15.293 | INFO | ------------------------------------------ | 20260506 01:31:15.293 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:31:15.293 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 01:31:15.293 | INFO | Total mem usage: 26.5 GiB | 20260506 01:31:15.293 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 01:31:15.293 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 01:31:15.293 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 01:31:15.293 | INFO | High Water Breakdown: | 20260506 01:31:15.293 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:31:15.293 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:31:15.293 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:31:15.293 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:31:15.293 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:31:15.293 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:31:15.293 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 01:31:15.293 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 01:31:15.293 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:31:15.293 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 01:31:15.293 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 01:31:15.293 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 01:31:15.293 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 01:31:15.293 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 01:31:15.293 | INFO | ------------------------------------------ | 20260506 01:31:15.293 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:31:15.293 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 01:31:15.293 | INFO | Total mem usage: 25.6 GiB | 20260506 01:31:15.293 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 01:31:15.293 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 01:31:15.293 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 01:31:15.293 | INFO | High Water Breakdown: | 20260506 01:31:15.293 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 01:31:15.293 | INFO | ------------------------------------------ | 20260506 01:31:15.293 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:31:15.293 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 01:31:15.293 | INFO | Total mem usage: 15.7 GiB | 20260506 01:31:15.293 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 01:31:15.293 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 01:31:15.293 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 01:31:15.293 | INFO | High Water Breakdown: | 20260506 01:31:15.293 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 01:31:15.293 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 01:31:15.293 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 01:31:15.293 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 01:31:15.293 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 01:31:15.293 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 01:31:15.293 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 01:31:15.293 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 01:31:15.293 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 01:31:15.293 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:31:15.293 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 01:31:15.293 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:31:15.293 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 01:31:15.293 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 01:31:15.293 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 01:31:15.293 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 01:31:15.293 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:31:15.293 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 01:31:15.293 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:31:15.293 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:31:15.293 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:31:15.293 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 01:31:15.293 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 01:31:15.293 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:31:15.293 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 01:31:15.293 | INFO | ------------------------------------------ | 20260506 01:31:15.293 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:31:15.294 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 01:31:15.294 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:31:15.294 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:31:15.294 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.37e+01, 1.33e+01 / 6.04e-01 | 20260506 01:31:15.294 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.04e-01, 1.03e-02 / 3.45e-03 | 20260506 01:31:15.294 | INFO | PerfReport: 11.82%, Baseline -- 3.70e+00 / 4.50e+00, 5.58e-01 / 2.30e-01 | 20260506 01:31:15.294 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.52e+00, 1.90e-01 / 9.95e-02 | 20260506 01:31:15.294 | INFO | PerfReport: 15.99%, DME -- 4.00e+01 / 4.38e+01, 3.78e+00 / 7.98e-01 | 20260506 01:31:15.294 | INFO | PerfReport: 57.45%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.51e+00 / 9.54e-01 | 20260506 01:31:15.294 | INFO | PerfReport: 7.06%, PulseAccumulating -- 2.21e+00 / 2.73e+00, 4.19e-01 / 2.18e-01 | 20260506 01:31:15.294 | INFO | PerfReport: 3.04%, Metrics -- 9.50e-01 / 1.21e+00, 2.54e-01 / 1.34e-01 | 20260506 01:31:15.294 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.29e-01 / 3.80e-01, 5.52e-02 / 3.24e-02 | 20260506 01:31:15.294 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 01:31:15.294 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:31:15.294 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:31:15.294 | INFO | PerfReport: Full Execution -- 9.81e+01 / -nan, 2.44e+01 / 0.00e+00 | 20260506 01:31:15.294 | INFO | PerfReport: 22.37%, Upload -- 2.19e+01 / 0.00e+00, 7.49e+00 / 0.00e+00 | 20260506 01:31:15.294 | INFO | PerfReport: 21.91%, Download -- 2.15e+01 / 0.00e+00, 7.59e+00 / 0.00e+00 | 20260506 01:31:15.294 | INFO | PerfReport: 31.88%, Compute -- 3.13e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 01:31:15.294 | INFO | PerfReport: 2.08%, LockUpload -- 2.04e+00 / 0.00e+00, 7.93e+00 / 0.00e+00 | 20260506 01:31:15.294 | INFO | PerfReport: 1.32%, LockDownload -- 1.29e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260506 01:31:15.294 | INFO | PerfReport: 20.43%, LockCompute -- 2.00e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 01:31:15.294 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 2.15e-02 / 0.00e+00 | 20260506 01:31:15.418 | INFO | Starting a round of Baz IO | 20260506 01:31:18.554 | INFO | Populating RTMetrics report | 20260506 01:31:18.554 | INFO | RTMetrics report populated | 20260506 01:31:18.554 | INFO | Opening tmp RTMetrics json file | 20260506 01:31:18.555 | INFO | Serializing RTMetrics to json | 20260506 01:31:18.557 | INFO | Serialization done, swapping tmp file now... | 20260506 01:31:18.557 | INFO | RTMetrics done | 20260506 01:31:18.628 | INFO | Finished a round of Baz IO: Wrote 1.822 GiB in 3.21 seconds | 20260506 01:31:18.628 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.31% / 0.00% / 78.69% | 20260506 01:31:18.628 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0801245% / 3.97364e-06% / 5.16574e-05% | 20260506 01:31:18.628 | INFO | Baz IO overall: | 20260506 01:31:18.628 | INFO | BytesWritten : 4448784528510 | 20260506 01:31:18.628 | INFO | NumEvents : 2090909261229 | 20260506 01:31:20.436 | INFO | Analyzing chunk frames = [7888896,7889408) | 20260506 01:31:20.436 | INFO | MemInfo: { | 20260506 01:31:20.436 | INFO | "gpu" : " 17 GB", | 20260506 01:31:20.436 | INFO | "huge" : "", | 20260506 01:31:20.436 | INFO | "huge_shared" : " 27 GB", | 20260506 01:31:20.436 | INFO | "malloc" : " 14 GB", | 20260506 01:31:20.436 | INFO | "pinned" : " 29 GB" | 20260506 01:31:20.436 | INFO | } | 20260506 01:31:20.436 | INFO | Perf report for the last 5.14329 seconds: | 20260506 01:31:20.436 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:31:20.436 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:20.436 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:20.436 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:20.436 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260506 01:31:20.436 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:20.436 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260506 01:31:20.436 | INFO | PRE_HQ : 59.6%, 1.7, 53.7ms, 0 | 20260506 01:31:20.436 | INFO | BAZWRITER : 61.7%, 3.4, 108.9ms, 0 | 20260506 01:31:20.436 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:31:20.436 | INFO | Uploaded 36GiB to GPU in 2.1818s (16.5001GiB/s) | 20260506 01:31:20.436 | INFO | Downloaded 38GiB to GPU in 2.2447s (16.9287GiB/s) | 20260506 01:31:25.575 | INFO | Analyzing chunk frames = [7889408,7889920) | 20260506 01:31:25.575 | INFO | MemInfo: { | 20260506 01:31:25.575 | INFO | "gpu" : " 17 GB", | 20260506 01:31:25.575 | INFO | "huge" : "", | 20260506 01:31:25.575 | INFO | "huge_shared" : " 27 GB", | 20260506 01:31:25.575 | INFO | "malloc" : " 14 GB", | 20260506 01:31:25.575 | INFO | "pinned" : " 29 GB" | 20260506 01:31:25.575 | INFO | } | 20260506 01:31:25.575 | INFO | Perf report for the last 5.13975 seconds: | 20260506 01:31:25.576 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:31:25.576 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:25.576 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:25.576 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:25.576 | INFO | ANALYSIS : 60.9%, 3.0, 95.0ms, 0 | 20260506 01:31:25.576 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:25.576 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:25.576 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 01:31:25.576 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:25.576 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:25.576 | INFO | Uploaded 36GiB to GPU in 1.96645s (18.3071GiB/s) | 20260506 01:31:25.576 | INFO | Downloaded 34GiB to GPU in 1.97948s (17.1762GiB/s) | 20260506 01:31:30.711 | INFO | Analyzing chunk frames = [7889920,7890432) | 20260506 01:31:30.711 | INFO | MemInfo: { | 20260506 01:31:30.711 | INFO | "gpu" : " 17 GB", | 20260506 01:31:30.711 | INFO | "huge" : "", | 20260506 01:31:30.711 | INFO | "huge_shared" : " 27 GB", | 20260506 01:31:30.711 | INFO | "malloc" : " 14 GB", | 20260506 01:31:30.711 | INFO | "pinned" : " 29 GB" | 20260506 01:31:30.711 | INFO | } | 20260506 01:31:30.711 | INFO | Perf report for the last 5.13599 seconds: | 20260506 01:31:30.711 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:31:30.711 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:30.711 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:30.711 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:30.712 | INFO | ANALYSIS : 61.3%, 3.0, 95.2ms, 0 | 20260506 01:31:30.712 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:30.712 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:30.712 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 01:31:30.712 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:30.712 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:30.712 | INFO | Uploaded 36GiB to GPU in 2.08587s (17.259GiB/s) | 20260506 01:31:30.712 | INFO | Downloaded 34GiB to GPU in 2.01918s (16.8385GiB/s) | 20260506 01:31:35.846 | INFO | Analyzing chunk frames = [7890432,7890944) | 20260506 01:31:35.846 | INFO | MemInfo: { | 20260506 01:31:35.846 | INFO | "gpu" : " 17 GB", | 20260506 01:31:35.846 | INFO | "huge" : "", | 20260506 01:31:35.846 | INFO | "huge_shared" : " 27 GB", | 20260506 01:31:35.846 | INFO | "malloc" : " 14 GB", | 20260506 01:31:35.846 | INFO | "pinned" : " 29 GB" | 20260506 01:31:35.846 | INFO | } | 20260506 01:31:35.846 | INFO | Perf report for the last 5.13439 seconds: | 20260506 01:31:35.846 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:31:35.846 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:35.846 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:35.846 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:35.846 | INFO | ANALYSIS : 61.7%, 3.0, 95.7ms, 0 | 20260506 01:31:35.846 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:35.846 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:35.846 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:31:35.846 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:35.846 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:35.846 | INFO | Uploaded 36GiB to GPU in 2.13889s (16.8312GiB/s) | 20260506 01:31:35.846 | INFO | Downloaded 34GiB to GPU in 2.02752s (16.7693GiB/s) | 20260506 01:31:40.988 | INFO | Analyzing chunk frames = [7890944,7891456) | 20260506 01:31:40.988 | INFO | MemInfo: { | 20260506 01:31:40.988 | INFO | "gpu" : " 17 GB", | 20260506 01:31:40.988 | INFO | "huge" : "", | 20260506 01:31:40.989 | INFO | "huge_shared" : " 27 GB", | 20260506 01:31:40.989 | INFO | "malloc" : " 14 GB", | 20260506 01:31:40.989 | INFO | "pinned" : " 29 GB" | 20260506 01:31:40.989 | INFO | } | 20260506 01:31:40.989 | INFO | Perf report for the last 5.14288 seconds: | 20260506 01:31:40.989 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:31:40.989 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:40.989 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:40.989 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:40.989 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 01:31:40.989 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:40.989 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:40.989 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 01:31:40.989 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:40.989 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:40.989 | INFO | Uploaded 36GiB to GPU in 2.09746s (17.1636GiB/s) | 20260506 01:31:40.989 | INFO | Downloaded 34GiB to GPU in 2.04648s (16.6139GiB/s) | 20260506 01:31:46.132 | INFO | Analyzing chunk frames = [7891456,7891968) | 20260506 01:31:46.132 | INFO | MemInfo: { | 20260506 01:31:46.132 | INFO | "gpu" : " 17 GB", | 20260506 01:31:46.132 | INFO | "huge" : "", | 20260506 01:31:46.133 | INFO | "huge_shared" : " 27 GB", | 20260506 01:31:46.133 | INFO | "malloc" : " 14 GB", | 20260506 01:31:46.133 | INFO | "pinned" : " 29 GB" | 20260506 01:31:46.133 | INFO | } | 20260506 01:31:46.133 | INFO | Perf report for the last 5.14405 seconds: | 20260506 01:31:46.133 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:31:46.133 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:46.133 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:46.133 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:46.133 | INFO | ANALYSIS : 61.2%, 3.0, 95.6ms, 0 | 20260506 01:31:46.133 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:46.133 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:46.133 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:31:46.133 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:46.133 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:46.133 | INFO | Uploaded 36GiB to GPU in 2.07119s (17.3813GiB/s) | 20260506 01:31:46.133 | INFO | Downloaded 34GiB to GPU in 2.03315s (16.7228GiB/s) | 20260506 01:31:51.270 | INFO | Analyzing chunk frames = [7891968,7892480) | 20260506 01:31:51.270 | INFO | MemInfo: { | 20260506 01:31:51.270 | INFO | "gpu" : " 17 GB", | 20260506 01:31:51.270 | INFO | "huge" : "", | 20260506 01:31:51.270 | INFO | "huge_shared" : " 27 GB", | 20260506 01:31:51.270 | INFO | "malloc" : " 14 GB", | 20260506 01:31:51.270 | INFO | "pinned" : " 29 GB" | 20260506 01:31:51.270 | INFO | } | 20260506 01:31:51.270 | INFO | Perf report for the last 5.13704 seconds: | 20260506 01:31:51.270 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:31:51.270 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:51.270 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:51.270 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:51.270 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260506 01:31:51.270 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:51.270 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:51.270 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 01:31:51.270 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:51.270 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:51.270 | INFO | Uploaded 36GiB to GPU in 2.06913s (17.3986GiB/s) | 20260506 01:31:51.270 | INFO | Downloaded 34GiB to GPU in 2.07234s (16.4066GiB/s) | 20260506 01:31:56.406 | INFO | Analyzing chunk frames = [7892480,7892992) | 20260506 01:31:56.406 | INFO | MemInfo: { | 20260506 01:31:56.406 | INFO | "gpu" : " 17 GB", | 20260506 01:31:56.406 | INFO | "huge" : "", | 20260506 01:31:56.406 | INFO | "huge_shared" : " 27 GB", | 20260506 01:31:56.406 | INFO | "malloc" : " 14 GB", | 20260506 01:31:56.406 | INFO | "pinned" : " 29 GB" | 20260506 01:31:56.406 | INFO | } | 20260506 01:31:56.406 | INFO | Perf report for the last 5.13632 seconds: | 20260506 01:31:56.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:31:56.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:56.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:56.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:56.406 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260506 01:31:56.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:56.406 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:56.406 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:31:56.406 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:56.406 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:31:56.406 | INFO | Uploaded 36GiB to GPU in 2.13764s (16.841GiB/s) | 20260506 01:31:56.406 | INFO | Downloaded 34GiB to GPU in 2.02658s (16.777GiB/s) | 20260506 01:31:56.524 | INFO | Starting a round of Baz IO | 20260506 01:31:59.669 | INFO | Populating RTMetrics report | 20260506 01:31:59.670 | INFO | RTMetrics report populated | 20260506 01:31:59.670 | INFO | Opening tmp RTMetrics json file | 20260506 01:31:59.670 | INFO | Serializing RTMetrics to json | 20260506 01:31:59.672 | INFO | Serialization done, swapping tmp file now... | 20260506 01:31:59.672 | INFO | RTMetrics done | 20260506 01:31:59.746 | INFO | Finished a round of Baz IO: Wrote 1.823 GiB in 3.221 seconds | 20260506 01:31:59.746 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.31% / 0.00% / 78.69% | 20260506 01:31:59.746 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0795722% / 3.97364e-06% / 7.54992e-05% | 20260506 01:31:59.746 | INFO | Baz IO overall: | 20260506 01:31:59.746 | INFO | BytesWritten : 4450741751489 | 20260506 01:31:59.746 | INFO | NumEvents : 2091801414359 | 20260506 01:32:01.550 | INFO | Analyzing chunk frames = [7892992,7893504) | 20260506 01:32:01.550 | INFO | MemInfo: { | 20260506 01:32:01.550 | INFO | "gpu" : " 17 GB", | 20260506 01:32:01.550 | INFO | "huge" : "", | 20260506 01:32:01.550 | INFO | "huge_shared" : " 27 GB", | 20260506 01:32:01.550 | INFO | "malloc" : " 14 GB", | 20260506 01:32:01.550 | INFO | "pinned" : " 29 GB" | 20260506 01:32:01.550 | INFO | } | 20260506 01:32:01.550 | INFO | Perf report for the last 5.14425 seconds: | 20260506 01:32:01.550 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:32:01.550 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:01.550 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:01.550 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:01.550 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260506 01:32:01.550 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:01.550 | INFO | RT_METRICS : 43.0%, 1.0, 22.6ms, 0 | 20260506 01:32:01.551 | INFO | PRE_HQ : 59.6%, 1.7, 53.6ms, 0 | 20260506 01:32:01.551 | INFO | BAZWRITER : 61.7%, 3.4, 109.0ms, 0 | 20260506 01:32:01.551 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:32:01.551 | INFO | Uploaded 36GiB to GPU in 2.18139s (16.5033GiB/s) | 20260506 01:32:01.551 | INFO | Downloaded 38GiB to GPU in 2.2442s (16.9325GiB/s) | 20260506 01:32:06.688 | INFO | Analyzing chunk frames = [7893504,7894016) | 20260506 01:32:06.688 | INFO | MemInfo: { | 20260506 01:32:06.688 | INFO | "gpu" : " 17 GB", | 20260506 01:32:06.688 | INFO | "huge" : "", | 20260506 01:32:06.688 | INFO | "huge_shared" : " 27 GB", | 20260506 01:32:06.688 | INFO | "malloc" : " 14 GB", | 20260506 01:32:06.688 | INFO | "pinned" : " 29 GB" | 20260506 01:32:06.688 | INFO | } | 20260506 01:32:06.688 | INFO | Perf report for the last 5.13759 seconds: | 20260506 01:32:06.688 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:32:06.688 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:06.688 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:06.688 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:06.688 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260506 01:32:06.688 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:06.688 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:06.688 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 01:32:06.688 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:06.688 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:06.688 | INFO | Uploaded 36GiB to GPU in 1.9469s (18.4909GiB/s) | 20260506 01:32:06.688 | INFO | Downloaded 34GiB to GPU in 1.97757s (17.1928GiB/s) | 20260506 01:32:11.828 | INFO | Analyzing chunk frames = [7894016,7894528) | 20260506 01:32:11.828 | INFO | MemInfo: { | 20260506 01:32:11.828 | INFO | "gpu" : " 17 GB", | 20260506 01:32:11.828 | INFO | "huge" : "", | 20260506 01:32:11.828 | INFO | "huge_shared" : " 27 GB", | 20260506 01:32:11.829 | INFO | "malloc" : " 14 GB", | 20260506 01:32:11.829 | INFO | "pinned" : " 29 GB" | 20260506 01:32:11.829 | INFO | } | 20260506 01:32:11.829 | INFO | Perf report for the last 5.14064 seconds: | 20260506 01:32:11.829 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:32:11.829 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:11.829 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:11.829 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:11.829 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 01:32:11.829 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:11.829 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:11.829 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 01:32:11.829 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:11.829 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:11.829 | INFO | Uploaded 36GiB to GPU in 2.08956s (17.2285GiB/s) | 20260506 01:32:11.829 | INFO | Downloaded 34GiB to GPU in 2.01991s (16.8325GiB/s) | 20260506 01:32:16.964 | INFO | Analyzing chunk frames = [7894528,7895040) | 20260506 01:32:16.964 | INFO | MemInfo: { | 20260506 01:32:16.964 | INFO | "gpu" : " 17 GB", | 20260506 01:32:16.964 | INFO | "huge" : "", | 20260506 01:32:16.964 | INFO | "huge_shared" : " 27 GB", | 20260506 01:32:16.964 | INFO | "malloc" : " 14 GB", | 20260506 01:32:16.964 | INFO | "pinned" : " 29 GB" | 20260506 01:32:16.964 | INFO | } | 20260506 01:32:16.964 | INFO | Perf report for the last 5.13538 seconds: | 20260506 01:32:16.964 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:32:16.964 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:16.964 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:16.964 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:16.964 | INFO | ANALYSIS : 61.5%, 3.0, 95.5ms, 0 | 20260506 01:32:16.964 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:16.964 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:16.964 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:32:16.964 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:16.964 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:16.964 | INFO | Uploaded 36GiB to GPU in 2.14883s (16.7533GiB/s) | 20260506 01:32:16.964 | INFO | Downloaded 34GiB to GPU in 2.05597s (16.5372GiB/s) | 20260506 01:32:22.109 | INFO | Analyzing chunk frames = [7895040,7895552) | 20260506 01:32:22.109 | INFO | MemInfo: { | 20260506 01:32:22.109 | INFO | "gpu" : " 17 GB", | 20260506 01:32:22.109 | INFO | "huge" : "", | 20260506 01:32:22.109 | INFO | "huge_shared" : " 27 GB", | 20260506 01:32:22.109 | INFO | "malloc" : " 14 GB", | 20260506 01:32:22.109 | INFO | "pinned" : " 29 GB" | 20260506 01:32:22.109 | INFO | } | 20260506 01:32:22.109 | INFO | Perf report for the last 5.14512 seconds: | 20260506 01:32:22.109 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:32:22.109 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:22.109 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:22.109 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:22.109 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 01:32:22.109 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:22.109 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:22.109 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 01:32:22.109 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:22.109 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:22.109 | INFO | Uploaded 36GiB to GPU in 2.09592s (17.1762GiB/s) | 20260506 01:32:22.109 | INFO | Downloaded 34GiB to GPU in 2.04739s (16.6065GiB/s) | 20260506 01:32:27.249 | INFO | Analyzing chunk frames = [7895552,7896064) | 20260506 01:32:27.249 | INFO | MemInfo: { | 20260506 01:32:27.249 | INFO | "gpu" : " 17 GB", | 20260506 01:32:27.249 | INFO | "huge" : "", | 20260506 01:32:27.249 | INFO | "huge_shared" : " 27 GB", | 20260506 01:32:27.249 | INFO | "malloc" : " 14 GB", | 20260506 01:32:27.250 | INFO | "pinned" : " 29 GB" | 20260506 01:32:27.250 | INFO | } | 20260506 01:32:27.250 | INFO | Perf report for the last 5.1404 seconds: | 20260506 01:32:27.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:32:27.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:27.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:27.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:27.250 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 01:32:27.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:27.250 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:27.250 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:32:27.250 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:27.250 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:27.250 | INFO | Uploaded 36GiB to GPU in 2.0926s (17.2035GiB/s) | 20260506 01:32:27.250 | INFO | Downloaded 34GiB to GPU in 2.03545s (16.704GiB/s) | 20260506 01:32:32.387 | INFO | Analyzing chunk frames = [7896064,7896576) | 20260506 01:32:32.387 | INFO | MemInfo: { | 20260506 01:32:32.387 | INFO | "gpu" : " 17 GB", | 20260506 01:32:32.387 | INFO | "huge" : "", | 20260506 01:32:32.387 | INFO | "huge_shared" : " 27 GB", | 20260506 01:32:32.387 | INFO | "malloc" : " 14 GB", | 20260506 01:32:32.387 | INFO | "pinned" : " 29 GB" | 20260506 01:32:32.387 | INFO | } | 20260506 01:32:32.388 | INFO | Perf report for the last 5.13792 seconds: | 20260506 01:32:32.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:32:32.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:32.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:32.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:32.388 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260506 01:32:32.388 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:32.388 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:32.388 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 01:32:32.388 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:32.388 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:32.388 | INFO | Uploaded 36GiB to GPU in 2.04395s (17.6129GiB/s) | 20260506 01:32:32.388 | INFO | Downloaded 34GiB to GPU in 2.06306s (16.4804GiB/s) | 20260506 01:32:37.521 | INFO | Analyzing chunk frames = [7896576,7897088) | 20260506 01:32:37.521 | INFO | MemInfo: { | 20260506 01:32:37.521 | INFO | "gpu" : " 17 GB", | 20260506 01:32:37.521 | INFO | "huge" : "", | 20260506 01:32:37.521 | INFO | "huge_shared" : " 27 GB", | 20260506 01:32:37.521 | INFO | "malloc" : " 14 GB", | 20260506 01:32:37.521 | INFO | "pinned" : " 29 GB" | 20260506 01:32:37.521 | INFO | } | 20260506 01:32:37.521 | INFO | Perf report for the last 5.13355 seconds: | 20260506 01:32:37.521 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:32:37.521 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:37.521 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:37.521 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:37.521 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 01:32:37.521 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:37.521 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:37.521 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 01:32:37.521 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:37.521 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:37.521 | INFO | Uploaded 36GiB to GPU in 2.03128s (17.7228GiB/s) | 20260506 01:32:37.521 | INFO | Downloaded 34GiB to GPU in 2.01435s (16.8789GiB/s) | 20260506 01:32:37.521 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 01:32:37.521 | INFO | Total mem usage: 15.3 GiB | 20260506 01:32:37.521 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 01:32:37.521 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.19%) | 20260506 01:32:37.521 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 01:32:37.521 | INFO | High Water Breakdown: | 20260506 01:32:37.521 | INFO | 60.9% : 8.54 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 01:32:37.521 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 01:32:37.521 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 01:32:37.521 | INFO | ------------------------------------------ | 20260506 01:32:37.521 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:32:37.522 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 01:32:37.522 | INFO | Total mem usage: 26.5 GiB | 20260506 01:32:37.522 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 01:32:37.522 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 01:32:37.522 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 01:32:37.522 | INFO | High Water Breakdown: | 20260506 01:32:37.522 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:32:37.522 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:32:37.522 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:32:37.522 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:32:37.522 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:32:37.522 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:32:37.522 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 01:32:37.522 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 01:32:37.522 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:32:37.522 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 01:32:37.522 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 01:32:37.522 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 01:32:37.522 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 01:32:37.522 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 01:32:37.522 | INFO | ------------------------------------------ | 20260506 01:32:37.522 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:32:37.522 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 01:32:37.522 | INFO | Total mem usage: 25.6 GiB | 20260506 01:32:37.522 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 01:32:37.522 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 01:32:37.522 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 01:32:37.522 | INFO | High Water Breakdown: | 20260506 01:32:37.522 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 01:32:37.522 | INFO | ------------------------------------------ | 20260506 01:32:37.522 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:32:37.522 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 01:32:37.522 | INFO | Total mem usage: 15.7 GiB | 20260506 01:32:37.522 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 01:32:37.522 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 01:32:37.522 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 01:32:37.522 | INFO | High Water Breakdown: | 20260506 01:32:37.522 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 01:32:37.522 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 01:32:37.522 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 01:32:37.522 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 01:32:37.522 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 01:32:37.522 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 01:32:37.522 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 01:32:37.522 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 01:32:37.522 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 01:32:37.522 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:32:37.522 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 01:32:37.522 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:32:37.522 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 01:32:37.522 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 01:32:37.522 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 01:32:37.522 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 01:32:37.522 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:32:37.522 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 01:32:37.522 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:32:37.522 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:32:37.522 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:32:37.522 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 01:32:37.522 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 01:32:37.522 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:32:37.522 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 01:32:37.522 | INFO | ------------------------------------------ | 20260506 01:32:37.522 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:32:37.522 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 01:32:37.522 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:32:37.522 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:32:37.522 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.37e+01, 1.35e+01 / 7.89e-01 | 20260506 01:32:37.522 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.05e-01, 1.03e-02 / 4.71e-03 | 20260506 01:32:37.522 | INFO | PerfReport: 11.84%, Baseline -- 3.70e+00 / 4.48e+00, 5.67e-01 / 2.67e-01 | 20260506 01:32:37.522 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.50e+00, 1.88e-01 / 1.17e-01 | 20260506 01:32:37.522 | INFO | PerfReport: 16.01%, DME -- 4.00e+01 / 4.40e+01, 3.85e+00 / 1.01e+00 | 20260506 01:32:37.522 | INFO | PerfReport: 57.39%, FrameLabeling -- 1.79e+01 / 1.93e+01, 1.46e+00 / 1.08e+00 | 20260506 01:32:37.522 | INFO | PerfReport: 7.08%, PulseAccumulating -- 2.21e+00 / 2.73e+00, 4.22e-01 / 2.55e-01 | 20260506 01:32:37.522 | INFO | PerfReport: 3.04%, Metrics -- 9.52e-01 / 1.23e+00, 2.55e-01 / 9.76e-02 | 20260506 01:32:37.522 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.29e-01 / 3.80e-01, 5.50e-02 / 2.66e-02 | 20260506 01:32:37.522 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 01:32:37.522 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:32:37.522 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:32:37.522 | INFO | PerfReport: Full Execution -- 9.79e+01 / -nan, 2.42e+01 / 0.00e+00 | 20260506 01:32:37.522 | INFO | PerfReport: 22.20%, Upload -- 2.17e+01 / 0.00e+00, 7.34e+00 / 0.00e+00 | 20260506 01:32:37.522 | INFO | PerfReport: 21.89%, Download -- 2.14e+01 / 0.00e+00, 7.56e+00 / 0.00e+00 | 20260506 01:32:37.522 | INFO | PerfReport: 31.95%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 01:32:37.522 | INFO | PerfReport: 2.06%, LockUpload -- 2.02e+00 / 0.00e+00, 7.78e+00 / 0.00e+00 | 20260506 01:32:37.522 | INFO | PerfReport: 1.32%, LockDownload -- 1.29e+00 / 0.00e+00, 2.94e+00 / 0.00e+00 | 20260506 01:32:37.522 | INFO | PerfReport: 20.57%, LockCompute -- 2.01e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 01:32:37.522 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.77e-02 / 0.00e+00 | 20260506 01:32:37.638 | INFO | Starting a round of Baz IO | 20260506 01:32:40.781 | INFO | Populating RTMetrics report | 20260506 01:32:40.781 | INFO | RTMetrics report populated | 20260506 01:32:40.781 | INFO | Opening tmp RTMetrics json file | 20260506 01:32:40.782 | INFO | Serializing RTMetrics to json | 20260506 01:32:40.784 | INFO | Serialization done, swapping tmp file now... | 20260506 01:32:40.784 | INFO | RTMetrics done | 20260506 01:32:40.855 | INFO | Finished a round of Baz IO: Wrote 1.823 GiB in 3.216 seconds | 20260506 01:32:40.855 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.31% / 0.00% / 78.69% | 20260506 01:32:40.855 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0796795% / 3.97364e-06% / 0.000111262% | 20260506 01:32:40.855 | INFO | Baz IO overall: | 20260506 01:32:40.855 | INFO | BytesWritten : 4452699192604 | 20260506 01:32:40.855 | INFO | NumEvents : 2092693491438 | 20260506 01:32:42.664 | INFO | Analyzing chunk frames = [7897088,7897600) | 20260506 01:32:42.664 | INFO | MemInfo: { | 20260506 01:32:42.664 | INFO | "gpu" : " 17 GB", | 20260506 01:32:42.664 | INFO | "huge" : "", | 20260506 01:32:42.664 | INFO | "huge_shared" : " 27 GB", | 20260506 01:32:42.664 | INFO | "malloc" : " 14 GB", | 20260506 01:32:42.664 | INFO | "pinned" : " 29 GB" | 20260506 01:32:42.664 | INFO | } | 20260506 01:32:42.664 | INFO | Perf report for the last 5.1429 seconds: | 20260506 01:32:42.664 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:32:42.664 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:42.664 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:42.664 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:42.664 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260506 01:32:42.664 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:42.664 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260506 01:32:42.664 | INFO | PRE_HQ : 59.7%, 1.7, 53.6ms, 0 | 20260506 01:32:42.664 | INFO | BAZWRITER : 61.7%, 3.4, 109.2ms, 0 | 20260506 01:32:42.664 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:32:42.664 | INFO | Uploaded 36GiB to GPU in 2.17068s (16.5846GiB/s) | 20260506 01:32:42.664 | INFO | Downloaded 38GiB to GPU in 2.24188s (16.9501GiB/s) | 20260506 01:32:47.803 | INFO | Analyzing chunk frames = [7897600,7898112) | 20260506 01:32:47.803 | INFO | MemInfo: { | 20260506 01:32:47.803 | INFO | "gpu" : " 17 GB", | 20260506 01:32:47.803 | INFO | "huge" : "", | 20260506 01:32:47.803 | INFO | "huge_shared" : " 27 GB", | 20260506 01:32:47.803 | INFO | "malloc" : " 14 GB", | 20260506 01:32:47.803 | INFO | "pinned" : " 29 GB" | 20260506 01:32:47.803 | INFO | } | 20260506 01:32:47.803 | INFO | Perf report for the last 5.13871 seconds: | 20260506 01:32:47.803 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:32:47.803 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:47.803 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:47.803 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:47.803 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 01:32:47.803 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:47.803 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:47.803 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 01:32:47.803 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:47.803 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:47.803 | INFO | Uploaded 36GiB to GPU in 1.99394s (18.0547GiB/s) | 20260506 01:32:47.803 | INFO | Downloaded 34GiB to GPU in 1.98903s (17.0938GiB/s) | 20260506 01:32:52.943 | INFO | Analyzing chunk frames = [7898112,7898624) | 20260506 01:32:52.943 | INFO | MemInfo: { | 20260506 01:32:52.943 | INFO | "gpu" : " 17 GB", | 20260506 01:32:52.943 | INFO | "huge" : "", | 20260506 01:32:52.943 | INFO | "huge_shared" : " 27 GB", | 20260506 01:32:52.943 | INFO | "malloc" : " 14 GB", | 20260506 01:32:52.943 | INFO | "pinned" : " 29 GB" | 20260506 01:32:52.943 | INFO | } | 20260506 01:32:52.943 | INFO | Perf report for the last 5.14003 seconds: | 20260506 01:32:52.943 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:32:52.943 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:52.943 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:52.943 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:52.943 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 01:32:52.943 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:52.943 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:52.943 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:32:52.943 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:52.943 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:52.943 | INFO | Uploaded 36GiB to GPU in 2.07949s (17.312GiB/s) | 20260506 01:32:52.943 | INFO | Downloaded 34GiB to GPU in 2.01213s (16.8975GiB/s) | 20260506 01:32:58.085 | INFO | Analyzing chunk frames = [7898624,7899136) | 20260506 01:32:58.085 | INFO | MemInfo: { | 20260506 01:32:58.085 | INFO | "gpu" : " 17 GB", | 20260506 01:32:58.085 | INFO | "huge" : "", | 20260506 01:32:58.085 | INFO | "huge_shared" : " 27 GB", | 20260506 01:32:58.085 | INFO | "malloc" : " 14 GB", | 20260506 01:32:58.085 | INFO | "pinned" : " 29 GB" | 20260506 01:32:58.085 | INFO | } | 20260506 01:32:58.085 | INFO | Perf report for the last 5.142 seconds: | 20260506 01:32:58.085 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:32:58.085 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:58.085 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:58.085 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:58.085 | INFO | ANALYSIS : 61.5%, 3.0, 95.5ms, 0 | 20260506 01:32:58.085 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:58.085 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:58.085 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:32:58.085 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:58.085 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:32:58.085 | INFO | Uploaded 36GiB to GPU in 2.14686s (16.7687GiB/s) | 20260506 01:32:58.085 | INFO | Downloaded 34GiB to GPU in 2.05443s (16.5496GiB/s) | 20260506 01:33:03.218 | INFO | Analyzing chunk frames = [7899136,7899648) | 20260506 01:33:03.218 | INFO | MemInfo: { | 20260506 01:33:03.218 | INFO | "gpu" : " 17 GB", | 20260506 01:33:03.218 | INFO | "huge" : "", | 20260506 01:33:03.218 | INFO | "huge_shared" : " 27 GB", | 20260506 01:33:03.218 | INFO | "malloc" : " 14 GB", | 20260506 01:33:03.218 | INFO | "pinned" : " 29 GB" | 20260506 01:33:03.219 | INFO | } | 20260506 01:33:03.219 | INFO | Perf report for the last 5.13377 seconds: | 20260506 01:33:03.219 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:33:03.219 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:03.219 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:03.219 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:03.219 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260506 01:33:03.219 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:03.219 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:03.219 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 01:33:03.219 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:03.219 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:03.219 | INFO | Uploaded 36GiB to GPU in 2.05949s (17.48GiB/s) | 20260506 01:33:03.219 | INFO | Downloaded 34GiB to GPU in 2.02959s (16.7521GiB/s) | 20260506 01:33:08.359 | INFO | Analyzing chunk frames = [7899648,7900160) | 20260506 01:33:08.359 | INFO | MemInfo: { | 20260506 01:33:08.359 | INFO | "gpu" : " 17 GB", | 20260506 01:33:08.359 | INFO | "huge" : "", | 20260506 01:33:08.359 | INFO | "huge_shared" : " 27 GB", | 20260506 01:33:08.359 | INFO | "malloc" : " 14 GB", | 20260506 01:33:08.359 | INFO | "pinned" : " 29 GB" | 20260506 01:33:08.359 | INFO | } | 20260506 01:33:08.359 | INFO | Perf report for the last 5.14072 seconds: | 20260506 01:33:08.359 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:33:08.359 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:08.359 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:08.359 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:08.359 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 01:33:08.359 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:08.359 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:08.359 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 01:33:08.359 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:08.359 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:08.360 | INFO | Uploaded 36GiB to GPU in 2.06994s (17.3918GiB/s) | 20260506 01:33:08.360 | INFO | Downloaded 34GiB to GPU in 2.04049s (16.6627GiB/s) | 20260506 01:33:13.500 | INFO | Analyzing chunk frames = [7900160,7900672) | 20260506 01:33:13.500 | INFO | MemInfo: { | 20260506 01:33:13.500 | INFO | "gpu" : " 17 GB", | 20260506 01:33:13.500 | INFO | "huge" : "", | 20260506 01:33:13.500 | INFO | "huge_shared" : " 27 GB", | 20260506 01:33:13.500 | INFO | "malloc" : " 14 GB", | 20260506 01:33:13.500 | INFO | "pinned" : " 29 GB" | 20260506 01:33:13.500 | INFO | } | 20260506 01:33:13.500 | INFO | Perf report for the last 5.14115 seconds: | 20260506 01:33:13.500 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:33:13.500 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:13.500 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:13.500 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:13.501 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260506 01:33:13.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:13.501 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:13.501 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 01:33:13.501 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:13.501 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:13.501 | INFO | Uploaded 36GiB to GPU in 2.02401s (17.7865GiB/s) | 20260506 01:33:13.501 | INFO | Downloaded 34GiB to GPU in 2.04713s (16.6086GiB/s) | 20260506 01:33:18.639 | INFO | Analyzing chunk frames = [7900672,7901184) | 20260506 01:33:18.639 | INFO | MemInfo: { | 20260506 01:33:18.639 | INFO | "gpu" : " 17 GB", | 20260506 01:33:18.639 | INFO | "huge" : "", | 20260506 01:33:18.639 | INFO | "huge_shared" : " 27 GB", | 20260506 01:33:18.639 | INFO | "malloc" : " 14 GB", | 20260506 01:33:18.639 | INFO | "pinned" : " 29 GB" | 20260506 01:33:18.639 | INFO | } | 20260506 01:33:18.639 | INFO | Perf report for the last 5.13866 seconds: | 20260506 01:33:18.639 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:33:18.639 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:18.639 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:18.639 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:18.639 | INFO | ANALYSIS : 61.4%, 3.0, 95.9ms, 0 | 20260506 01:33:18.639 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:18.639 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:18.639 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:33:18.639 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:18.639 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:18.639 | INFO | Uploaded 36GiB to GPU in 2.04426s (17.6103GiB/s) | 20260506 01:33:18.639 | INFO | Downloaded 34GiB to GPU in 2.02484s (16.7915GiB/s) | 20260506 01:33:18.757 | INFO | Starting a round of Baz IO | 20260506 01:33:21.902 | INFO | Populating RTMetrics report | 20260506 01:33:21.902 | INFO | RTMetrics report populated | 20260506 01:33:21.902 | INFO | Opening tmp RTMetrics json file | 20260506 01:33:21.903 | INFO | Serializing RTMetrics to json | 20260506 01:33:21.905 | INFO | Serialization done, swapping tmp file now... | 20260506 01:33:21.905 | INFO | RTMetrics done | 20260506 01:33:21.981 | INFO | Finished a round of Baz IO: Wrote 1.823 GiB in 3.224 seconds | 20260506 01:33:21.981 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.31% / 0.00% / 78.69% | 20260506 01:33:21.981 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0790834% / 3.97364e-06% / 7.15256e-05% | 20260506 01:33:21.981 | INFO | Baz IO overall: | 20260506 01:33:21.981 | INFO | BytesWritten : 4454656499374 | 20260506 01:33:21.981 | INFO | NumEvents : 2093585452026 | 20260506 01:33:23.779 | INFO | Analyzing chunk frames = [7901184,7901696) | 20260506 01:33:23.779 | INFO | MemInfo: { | 20260506 01:33:23.779 | INFO | "gpu" : " 17 GB", | 20260506 01:33:23.779 | INFO | "huge" : "", | 20260506 01:33:23.779 | INFO | "huge_shared" : " 27 GB", | 20260506 01:33:23.779 | INFO | "malloc" : " 14 GB", | 20260506 01:33:23.779 | INFO | "pinned" : " 29 GB" | 20260506 01:33:23.779 | INFO | } | 20260506 01:33:23.779 | INFO | Perf report for the last 5.13977 seconds: | 20260506 01:33:23.779 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:33:23.779 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:23.779 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:23.779 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:23.779 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260506 01:33:23.779 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:23.779 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260506 01:33:23.779 | INFO | PRE_HQ : 59.7%, 1.7, 53.6ms, 0 | 20260506 01:33:23.779 | INFO | BAZWRITER : 61.8%, 3.4, 109.2ms, 0 | 20260506 01:33:23.779 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:33:23.779 | INFO | Uploaded 36GiB to GPU in 2.16988s (16.5907GiB/s) | 20260506 01:33:23.779 | INFO | Downloaded 38GiB to GPU in 2.23871s (16.974GiB/s) | 20260506 01:33:28.915 | INFO | Analyzing chunk frames = [7901696,7902208) | 20260506 01:33:28.915 | INFO | MemInfo: { | 20260506 01:33:28.915 | INFO | "gpu" : " 17 GB", | 20260506 01:33:28.915 | INFO | "huge" : "", | 20260506 01:33:28.915 | INFO | "huge_shared" : " 27 GB", | 20260506 01:33:28.915 | INFO | "malloc" : " 14 GB", | 20260506 01:33:28.915 | INFO | "pinned" : " 29 GB" | 20260506 01:33:28.915 | INFO | } | 20260506 01:33:28.915 | INFO | Perf report for the last 5.13659 seconds: | 20260506 01:33:28.915 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:33:28.915 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:28.916 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:28.916 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:28.916 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260506 01:33:28.916 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:28.916 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:28.916 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:33:28.916 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:28.916 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:28.916 | INFO | Uploaded 36GiB to GPU in 2.11996s (16.9815GiB/s) | 20260506 01:33:28.916 | INFO | Downloaded 34GiB to GPU in 2.0431s (16.6413GiB/s) | 20260506 01:33:34.058 | INFO | Analyzing chunk frames = [7902208,7902720) | 20260506 01:33:34.058 | INFO | MemInfo: { | 20260506 01:33:34.058 | INFO | "gpu" : " 17 GB", | 20260506 01:33:34.058 | INFO | "huge" : "", | 20260506 01:33:34.058 | INFO | "huge_shared" : " 27 GB", | 20260506 01:33:34.058 | INFO | "malloc" : " 14 GB", | 20260506 01:33:34.058 | INFO | "pinned" : " 29 GB" | 20260506 01:33:34.058 | INFO | } | 20260506 01:33:34.058 | INFO | Perf report for the last 5.14296 seconds: | 20260506 01:33:34.058 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:33:34.058 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:34.058 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:34.058 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:34.058 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 01:33:34.059 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:34.059 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:34.059 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 01:33:34.059 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:34.059 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:34.059 | INFO | Uploaded 36GiB to GPU in 2.09239s (17.2052GiB/s) | 20260506 01:33:34.059 | INFO | Downloaded 34GiB to GPU in 2.02667s (16.7763GiB/s) | 20260506 01:33:39.200 | INFO | Analyzing chunk frames = [7902720,7903232) | 20260506 01:33:39.200 | INFO | MemInfo: { | 20260506 01:33:39.200 | INFO | "gpu" : " 17 GB", | 20260506 01:33:39.200 | INFO | "huge" : "", | 20260506 01:33:39.201 | INFO | "huge_shared" : " 27 GB", | 20260506 01:33:39.201 | INFO | "malloc" : " 14 GB", | 20260506 01:33:39.201 | INFO | "pinned" : " 29 GB" | 20260506 01:33:39.201 | INFO | } | 20260506 01:33:39.201 | INFO | Perf report for the last 5.14213 seconds: | 20260506 01:33:39.201 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:33:39.201 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:39.201 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:39.201 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:39.201 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260506 01:33:39.201 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:39.201 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:39.201 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:33:39.201 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:39.201 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:39.201 | INFO | Uploaded 36GiB to GPU in 2.09779s (17.1609GiB/s) | 20260506 01:33:39.201 | INFO | Downloaded 34GiB to GPU in 2.01594s (16.8656GiB/s) | 20260506 01:33:44.334 | INFO | Analyzing chunk frames = [7903232,7903744) | 20260506 01:33:44.334 | INFO | MemInfo: { | 20260506 01:33:44.334 | INFO | "gpu" : " 17 GB", | 20260506 01:33:44.334 | INFO | "huge" : "", | 20260506 01:33:44.334 | INFO | "huge_shared" : " 27 GB", | 20260506 01:33:44.334 | INFO | "malloc" : " 14 GB", | 20260506 01:33:44.334 | INFO | "pinned" : " 29 GB" | 20260506 01:33:44.334 | INFO | } | 20260506 01:33:44.334 | INFO | Perf report for the last 5.13323 seconds: | 20260506 01:33:44.334 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:33:44.334 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:44.334 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:44.334 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:44.334 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 01:33:44.334 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:44.334 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:44.334 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 01:33:44.334 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:44.334 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:44.334 | INFO | Uploaded 36GiB to GPU in 2.10934s (17.067GiB/s) | 20260506 01:33:44.334 | INFO | Downloaded 34GiB to GPU in 2.03294s (16.7246GiB/s) | 20260506 01:33:49.469 | INFO | Analyzing chunk frames = [7903744,7904256) | 20260506 01:33:49.469 | INFO | MemInfo: { | 20260506 01:33:49.469 | INFO | "gpu" : " 17 GB", | 20260506 01:33:49.469 | INFO | "huge" : "", | 20260506 01:33:49.469 | INFO | "huge_shared" : " 27 GB", | 20260506 01:33:49.469 | INFO | "malloc" : " 14 GB", | 20260506 01:33:49.469 | INFO | "pinned" : " 29 GB" | 20260506 01:33:49.469 | INFO | } | 20260506 01:33:49.469 | INFO | Perf report for the last 5.13504 seconds: | 20260506 01:33:49.469 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:33:49.469 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:49.469 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:49.469 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:49.469 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 01:33:49.469 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:49.469 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:49.469 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:33:49.469 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:49.469 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:49.469 | INFO | Uploaded 36GiB to GPU in 2.05485s (17.5195GiB/s) | 20260506 01:33:49.469 | INFO | Downloaded 34GiB to GPU in 2.04021s (16.665GiB/s) | 20260506 01:33:54.614 | INFO | Analyzing chunk frames = [7904256,7904768) | 20260506 01:33:54.614 | INFO | MemInfo: { | 20260506 01:33:54.614 | INFO | "gpu" : " 17 GB", | 20260506 01:33:54.614 | INFO | "huge" : "", | 20260506 01:33:54.614 | INFO | "huge_shared" : " 27 GB", | 20260506 01:33:54.614 | INFO | "malloc" : " 14 GB", | 20260506 01:33:54.614 | INFO | "pinned" : " 29 GB" | 20260506 01:33:54.614 | INFO | } | 20260506 01:33:54.614 | INFO | Perf report for the last 5.14526 seconds: | 20260506 01:33:54.614 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:33:54.614 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:54.614 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:54.614 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:54.614 | INFO | ANALYSIS : 62.3%, 3.0, 97.4ms, 0 | 20260506 01:33:54.614 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:54.614 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:54.614 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 01:33:54.614 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:54.614 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:54.614 | INFO | Uploaded 36GiB to GPU in 2.11832s (16.9946GiB/s) | 20260506 01:33:54.615 | INFO | Downloaded 34GiB to GPU in 2.11065s (16.1088GiB/s) | 20260506 01:33:59.757 | INFO | Analyzing chunk frames = [7904768,7905280) | 20260506 01:33:59.757 | INFO | MemInfo: { | 20260506 01:33:59.757 | INFO | "gpu" : " 17 GB", | 20260506 01:33:59.757 | INFO | "huge" : "", | 20260506 01:33:59.757 | INFO | "huge_shared" : " 27 GB", | 20260506 01:33:59.757 | INFO | "malloc" : " 14 GB", | 20260506 01:33:59.757 | INFO | "pinned" : " 29 GB" | 20260506 01:33:59.757 | INFO | } | 20260506 01:33:59.757 | INFO | Perf report for the last 5.14301 seconds: | 20260506 01:33:59.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:33:59.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:59.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:59.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:59.757 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260506 01:33:59.757 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:59.757 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:59.757 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 01:33:59.757 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:59.757 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:33:59.758 | INFO | Uploaded 36GiB to GPU in 2.1311s (16.8927GiB/s) | 20260506 01:33:59.758 | INFO | Downloaded 34GiB to GPU in 2.02096s (16.8237GiB/s) | 20260506 01:33:59.758 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 01:33:59.758 | INFO | Total mem usage: 15.3 GiB | 20260506 01:33:59.758 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 01:33:59.758 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.2%) | 20260506 01:33:59.758 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 01:33:59.758 | INFO | High Water Breakdown: | 20260506 01:33:59.758 | INFO | 60.9% : 8.54 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 01:33:59.758 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 01:33:59.758 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 01:33:59.758 | INFO | ------------------------------------------ | 20260506 01:33:59.758 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:33:59.758 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 01:33:59.758 | INFO | Total mem usage: 26.5 GiB | 20260506 01:33:59.758 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 01:33:59.758 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 01:33:59.758 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260506 01:33:59.758 | INFO | High Water Breakdown: | 20260506 01:33:59.758 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:33:59.758 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:33:59.758 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:33:59.758 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:33:59.758 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:33:59.758 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:33:59.758 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 01:33:59.758 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 01:33:59.758 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:33:59.758 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 01:33:59.758 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 01:33:59.758 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 01:33:59.758 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 01:33:59.758 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 01:33:59.758 | INFO | ------------------------------------------ | 20260506 01:33:59.758 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:33:59.758 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 01:33:59.758 | INFO | Total mem usage: 25.6 GiB | 20260506 01:33:59.758 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 01:33:59.759 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 01:33:59.759 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 01:33:59.759 | INFO | High Water Breakdown: | 20260506 01:33:59.759 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 01:33:59.759 | INFO | ------------------------------------------ | 20260506 01:33:59.759 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:33:59.759 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 01:33:59.759 | INFO | Total mem usage: 15.7 GiB | 20260506 01:33:59.759 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 01:33:59.759 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 01:33:59.759 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 01:33:59.759 | INFO | High Water Breakdown: | 20260506 01:33:59.759 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 01:33:59.759 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 01:33:59.759 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 01:33:59.759 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 01:33:59.759 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 01:33:59.759 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 01:33:59.759 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 01:33:59.759 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 01:33:59.759 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 01:33:59.759 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:33:59.759 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 01:33:59.759 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:33:59.759 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 01:33:59.759 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 01:33:59.759 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 01:33:59.759 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 01:33:59.759 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:33:59.759 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 01:33:59.759 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:33:59.759 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:33:59.759 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:33:59.759 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 01:33:59.759 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 01:33:59.759 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:33:59.759 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 01:33:59.759 | INFO | ------------------------------------------ | 20260506 01:33:59.759 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:33:59.759 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 01:33:59.759 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:33:59.759 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:33:59.759 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.40e+01, 1.33e+01 / 6.19e-01 | 20260506 01:33:59.759 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.05e-01, 1.11e-02 / 3.82e-03 | 20260506 01:33:59.759 | INFO | PerfReport: 11.83%, Baseline -- 3.71e+00 / 4.49e+00, 5.74e-01 / 1.32e-01 | 20260506 01:33:59.759 | INFO | PerfReport: 3.30%, Histogram -- 1.03e+00 / 1.52e+00, 1.95e-01 / 8.52e-02 | 20260506 01:33:59.759 | INFO | PerfReport: 15.98%, DME -- 4.01e+01 / 4.41e+01, 3.88e+00 / 9.37e-01 | 20260506 01:33:59.759 | INFO | PerfReport: 57.49%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.55e+00 / 1.05e+00 | 20260506 01:33:59.759 | INFO | PerfReport: 7.01%, PulseAccumulating -- 2.20e+00 / 2.72e+00, 4.21e-01 / 2.25e-01 | 20260506 01:33:59.759 | INFO | PerfReport: 3.03%, Metrics -- 9.49e-01 / 1.23e+00, 2.57e-01 / 1.06e-01 | 20260506 01:33:59.759 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.29e-01 / 3.81e-01, 5.60e-02 / 2.56e-02 | 20260506 01:33:59.759 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 01:33:59.759 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:33:59.759 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:33:59.759 | INFO | PerfReport: Full Execution -- 9.79e+01 / -nan, 2.42e+01 / 0.00e+00 | 20260506 01:33:59.759 | INFO | PerfReport: 22.26%, Upload -- 2.18e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260506 01:33:59.759 | INFO | PerfReport: 21.92%, Download -- 2.15e+01 / 0.00e+00, 7.59e+00 / 0.00e+00 | 20260506 01:33:59.759 | INFO | PerfReport: 31.99%, Compute -- 3.13e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 01:33:59.759 | INFO | PerfReport: 2.06%, LockUpload -- 2.02e+00 / 0.00e+00, 7.87e+00 / 0.00e+00 | 20260506 01:33:59.759 | INFO | PerfReport: 1.28%, LockDownload -- 1.26e+00 / 0.00e+00, 2.87e+00 / 0.00e+00 | 20260506 01:33:59.759 | INFO | PerfReport: 20.48%, LockCompute -- 2.01e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260506 01:33:59.759 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.87e-02 / 0.00e+00 | 20260506 01:33:59.883 | INFO | Starting a round of Baz IO | 20260506 01:34:03.016 | INFO | Populating RTMetrics report | 20260506 01:34:03.017 | INFO | RTMetrics report populated | 20260506 01:34:03.017 | INFO | Opening tmp RTMetrics json file | 20260506 01:34:03.017 | INFO | Serializing RTMetrics to json | 20260506 01:34:03.019 | INFO | Serialization done, swapping tmp file now... | 20260506 01:34:03.019 | INFO | RTMetrics done | 20260506 01:34:03.092 | INFO | Finished a round of Baz IO: Wrote 1.823 GiB in 3.208 seconds | 20260506 01:34:03.092 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.31% / 0.00% / 78.69% | 20260506 01:34:03.092 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0795325% / 0% / 9.53674e-05% | 20260506 01:34:03.092 | INFO | Baz IO overall: | 20260506 01:34:03.092 | INFO | BytesWritten : 4456613754703 | 20260506 01:34:03.092 | INFO | NumEvents : 2094477627395 | 20260506 01:34:04.896 | INFO | Analyzing chunk frames = [7905280,7905792) | 20260506 01:34:04.896 | INFO | MemInfo: { | 20260506 01:34:04.896 | INFO | "gpu" : " 17 GB", | 20260506 01:34:04.896 | INFO | "huge" : "", | 20260506 01:34:04.896 | INFO | "huge_shared" : " 27 GB", | 20260506 01:34:04.896 | INFO | "malloc" : " 14 GB", | 20260506 01:34:04.896 | INFO | "pinned" : " 29 GB" | 20260506 01:34:04.896 | INFO | } | 20260506 01:34:04.896 | INFO | Perf report for the last 5.13888 seconds: | 20260506 01:34:04.896 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:34:04.896 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:04.896 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:04.896 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:04.896 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260506 01:34:04.896 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:04.896 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260506 01:34:04.896 | INFO | PRE_HQ : 59.4%, 1.7, 53.7ms, 0 | 20260506 01:34:04.896 | INFO | BAZWRITER : 61.7%, 3.5, 109.9ms, 0 | 20260506 01:34:04.896 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:34:04.896 | INFO | Uploaded 36GiB to GPU in 2.14917s (16.7506GiB/s) | 20260506 01:34:04.896 | INFO | Downloaded 38GiB to GPU in 2.23239s (17.0221GiB/s) | 20260506 01:34:10.029 | INFO | Analyzing chunk frames = [7905792,7906304) | 20260506 01:34:10.029 | INFO | MemInfo: { | 20260506 01:34:10.029 | INFO | "gpu" : " 17 GB", | 20260506 01:34:10.029 | INFO | "huge" : "", | 20260506 01:34:10.029 | INFO | "huge_shared" : " 27 GB", | 20260506 01:34:10.029 | INFO | "malloc" : " 14 GB", | 20260506 01:34:10.029 | INFO | "pinned" : " 29 GB" | 20260506 01:34:10.029 | INFO | } | 20260506 01:34:10.029 | INFO | Perf report for the last 5.13294 seconds: | 20260506 01:34:10.029 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:34:10.029 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:10.029 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:10.029 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:10.029 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260506 01:34:10.029 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:10.029 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:10.029 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 01:34:10.029 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:10.029 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:10.029 | INFO | Uploaded 36GiB to GPU in 1.97651s (18.2139GiB/s) | 20260506 01:34:10.029 | INFO | Downloaded 34GiB to GPU in 1.97872s (17.1828GiB/s) | 20260506 01:34:15.169 | INFO | Analyzing chunk frames = [7906304,7906816) | 20260506 01:34:15.169 | INFO | MemInfo: { | 20260506 01:34:15.169 | INFO | "gpu" : " 17 GB", | 20260506 01:34:15.169 | INFO | "huge" : "", | 20260506 01:34:15.169 | INFO | "huge_shared" : " 27 GB", | 20260506 01:34:15.169 | INFO | "malloc" : " 14 GB", | 20260506 01:34:15.169 | INFO | "pinned" : " 29 GB" | 20260506 01:34:15.169 | INFO | } | 20260506 01:34:15.169 | INFO | Perf report for the last 5.14028 seconds: | 20260506 01:34:15.169 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:34:15.169 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:15.169 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:15.169 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:15.169 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 01:34:15.169 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:15.169 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:15.169 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:34:15.169 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:15.169 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:15.170 | INFO | Uploaded 36GiB to GPU in 2.08084s (17.3007GiB/s) | 20260506 01:34:15.170 | INFO | Downloaded 34GiB to GPU in 2.01297s (16.8905GiB/s) | 20260506 01:34:20.309 | INFO | Analyzing chunk frames = [7906816,7907328) | 20260506 01:34:20.309 | INFO | MemInfo: { | 20260506 01:34:20.309 | INFO | "gpu" : " 17 GB", | 20260506 01:34:20.309 | INFO | "huge" : "", | 20260506 01:34:20.309 | INFO | "huge_shared" : " 27 GB", | 20260506 01:34:20.309 | INFO | "malloc" : " 14 GB", | 20260506 01:34:20.309 | INFO | "pinned" : " 29 GB" | 20260506 01:34:20.309 | INFO | } | 20260506 01:34:20.309 | INFO | Perf report for the last 5.1401 seconds: | 20260506 01:34:20.309 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:34:20.309 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:20.310 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:20.310 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:20.310 | INFO | ANALYSIS : 61.6%, 3.0, 95.6ms, 0 | 20260506 01:34:20.310 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:20.310 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:20.310 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:34:20.310 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:20.310 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:20.310 | INFO | Uploaded 36GiB to GPU in 2.13736s (16.8432GiB/s) | 20260506 01:34:20.310 | INFO | Downloaded 34GiB to GPU in 2.01504s (16.8731GiB/s) | 20260506 01:34:25.448 | INFO | Analyzing chunk frames = [7907328,7907840) | 20260506 01:34:25.448 | INFO | MemInfo: { | 20260506 01:34:25.448 | INFO | "gpu" : " 17 GB", | 20260506 01:34:25.448 | INFO | "huge" : "", | 20260506 01:34:25.448 | INFO | "huge_shared" : " 27 GB", | 20260506 01:34:25.448 | INFO | "malloc" : " 14 GB", | 20260506 01:34:25.448 | INFO | "pinned" : " 29 GB" | 20260506 01:34:25.448 | INFO | } | 20260506 01:34:25.448 | INFO | Perf report for the last 5.13857 seconds: | 20260506 01:34:25.448 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:34:25.448 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:25.448 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:25.448 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:25.448 | INFO | ANALYSIS : 60.8%, 3.0, 94.9ms, 0 | 20260506 01:34:25.448 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:25.448 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:25.448 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:34:25.448 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:25.448 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:25.448 | INFO | Uploaded 36GiB to GPU in 2.06613s (17.4239GiB/s) | 20260506 01:34:25.448 | INFO | Downloaded 34GiB to GPU in 2.0289s (16.7578GiB/s) | 20260506 01:34:30.588 | INFO | Analyzing chunk frames = [7907840,7908352) | 20260506 01:34:30.588 | INFO | MemInfo: { | 20260506 01:34:30.588 | INFO | "gpu" : " 17 GB", | 20260506 01:34:30.588 | INFO | "huge" : "", | 20260506 01:34:30.588 | INFO | "huge_shared" : " 27 GB", | 20260506 01:34:30.588 | INFO | "malloc" : " 14 GB", | 20260506 01:34:30.588 | INFO | "pinned" : " 29 GB" | 20260506 01:34:30.588 | INFO | } | 20260506 01:34:30.588 | INFO | Perf report for the last 5.14011 seconds: | 20260506 01:34:30.588 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:34:30.588 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:30.588 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:30.588 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:30.588 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 01:34:30.588 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:30.588 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:30.588 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 01:34:30.588 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:30.588 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:30.589 | INFO | Uploaded 36GiB to GPU in 2.05499s (17.5183GiB/s) | 20260506 01:34:30.589 | INFO | Downloaded 34GiB to GPU in 2.02303s (16.8065GiB/s) | 20260506 01:34:35.728 | INFO | Analyzing chunk frames = [7908352,7908864) | 20260506 01:34:35.728 | INFO | MemInfo: { | 20260506 01:34:35.728 | INFO | "gpu" : " 17 GB", | 20260506 01:34:35.728 | INFO | "huge" : "", | 20260506 01:34:35.728 | INFO | "huge_shared" : " 27 GB", | 20260506 01:34:35.728 | INFO | "malloc" : " 14 GB", | 20260506 01:34:35.728 | INFO | "pinned" : " 29 GB" | 20260506 01:34:35.728 | INFO | } | 20260506 01:34:35.728 | INFO | Perf report for the last 5.14008 seconds: | 20260506 01:34:35.728 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:34:35.728 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:35.728 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:35.728 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:35.728 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260506 01:34:35.728 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:35.728 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:35.728 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 01:34:35.728 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:35.728 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:35.729 | INFO | Uploaded 36GiB to GPU in 2.06657s (17.4202GiB/s) | 20260506 01:34:35.729 | INFO | Downloaded 34GiB to GPU in 2.0692s (16.4315GiB/s) | 20260506 01:34:40.867 | INFO | Analyzing chunk frames = [7908864,7909376) | 20260506 01:34:40.867 | INFO | MemInfo: { | 20260506 01:34:40.867 | INFO | "gpu" : " 17 GB", | 20260506 01:34:40.867 | INFO | "huge" : "", | 20260506 01:34:40.867 | INFO | "huge_shared" : " 27 GB", | 20260506 01:34:40.867 | INFO | "malloc" : " 14 GB", | 20260506 01:34:40.867 | INFO | "pinned" : " 29 GB" | 20260506 01:34:40.867 | INFO | } | 20260506 01:34:40.867 | INFO | Perf report for the last 5.13872 seconds: | 20260506 01:34:40.867 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:34:40.867 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:40.867 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:40.867 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:40.867 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260506 01:34:40.867 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:40.867 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:40.867 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 01:34:40.867 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:40.867 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:40.867 | INFO | Uploaded 36GiB to GPU in 2.11183s (17.0468GiB/s) | 20260506 01:34:40.867 | INFO | Downloaded 34GiB to GPU in 2.00576s (16.9512GiB/s) | 20260506 01:34:40.993 | INFO | Starting a round of Baz IO | 20260506 01:34:44.128 | INFO | Populating RTMetrics report | 20260506 01:34:44.128 | INFO | RTMetrics report populated | 20260506 01:34:44.128 | INFO | Opening tmp RTMetrics json file | 20260506 01:34:44.130 | INFO | Serializing RTMetrics to json | 20260506 01:34:44.132 | INFO | Serialization done, swapping tmp file now... | 20260506 01:34:44.133 | INFO | RTMetrics done | 20260506 01:34:44.212 | INFO | Finished a round of Baz IO: Wrote 1.821 GiB in 3.219 seconds | 20260506 01:34:44.212 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.31% / 0.00% / 78.69% | 20260506 01:34:44.212 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0789841% / 3.97364e-06% / 7.94729e-05% | 20260506 01:34:44.212 | INFO | Baz IO overall: | 20260506 01:34:44.212 | INFO | BytesWritten : 4458568733120 | 20260506 01:34:44.212 | INFO | NumEvents : 2095368236347 | 20260506 01:34:46.010 | INFO | Analyzing chunk frames = [7909376,7909888) | 20260506 01:34:46.010 | INFO | MemInfo: { | 20260506 01:34:46.010 | INFO | "gpu" : " 17 GB", | 20260506 01:34:46.010 | INFO | "huge" : "", | 20260506 01:34:46.010 | INFO | "huge_shared" : " 27 GB", | 20260506 01:34:46.010 | INFO | "malloc" : " 14 GB", | 20260506 01:34:46.010 | INFO | "pinned" : " 29 GB" | 20260506 01:34:46.010 | INFO | } | 20260506 01:34:46.010 | INFO | Perf report for the last 5.14334 seconds: | 20260506 01:34:46.010 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:34:46.010 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:46.010 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:46.010 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:46.010 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260506 01:34:46.010 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:46.010 | INFO | RT_METRICS : 43.0%, 1.0, 22.6ms, 0 | 20260506 01:34:46.010 | INFO | PRE_HQ : 59.4%, 1.7, 53.8ms, 0 | 20260506 01:34:46.011 | INFO | BAZWRITER : 61.8%, 3.5, 110.0ms, 0 | 20260506 01:34:46.011 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:34:46.011 | INFO | Uploaded 36GiB to GPU in 2.19033s (16.4359GiB/s) | 20260506 01:34:46.011 | INFO | Downloaded 38GiB to GPU in 2.24458s (16.9296GiB/s) | 20260506 01:34:51.153 | INFO | Analyzing chunk frames = [7909888,7910400) | 20260506 01:34:51.153 | INFO | MemInfo: { | 20260506 01:34:51.153 | INFO | "gpu" : " 17 GB", | 20260506 01:34:51.153 | INFO | "huge" : "", | 20260506 01:34:51.153 | INFO | "huge_shared" : " 27 GB", | 20260506 01:34:51.153 | INFO | "malloc" : " 14 GB", | 20260506 01:34:51.153 | INFO | "pinned" : " 29 GB" | 20260506 01:34:51.153 | INFO | } | 20260506 01:34:51.153 | INFO | Perf report for the last 5.14264 seconds: | 20260506 01:34:51.153 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:34:51.153 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:51.153 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:51.153 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:51.153 | INFO | ANALYSIS : 60.6%, 3.0, 94.7ms, 0 | 20260506 01:34:51.153 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:51.153 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:51.153 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 01:34:51.153 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:51.153 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:51.153 | INFO | Uploaded 36GiB to GPU in 1.94049s (18.552GiB/s) | 20260506 01:34:51.153 | INFO | Downloaded 34GiB to GPU in 1.98597s (17.1201GiB/s) | 20260506 01:34:56.282 | INFO | Analyzing chunk frames = [7910400,7910912) | 20260506 01:34:56.282 | INFO | MemInfo: { | 20260506 01:34:56.282 | INFO | "gpu" : " 17 GB", | 20260506 01:34:56.282 | INFO | "huge" : "", | 20260506 01:34:56.282 | INFO | "huge_shared" : " 27 GB", | 20260506 01:34:56.282 | INFO | "malloc" : " 14 GB", | 20260506 01:34:56.282 | INFO | "pinned" : " 29 GB" | 20260506 01:34:56.282 | INFO | } | 20260506 01:34:56.282 | INFO | Perf report for the last 5.12875 seconds: | 20260506 01:34:56.282 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:34:56.282 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:56.282 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:56.282 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:56.282 | INFO | ANALYSIS : 61.3%, 3.0, 95.2ms, 0 | 20260506 01:34:56.282 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:56.282 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:56.282 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:34:56.282 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:56.282 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:34:56.282 | INFO | Uploaded 36GiB to GPU in 2.09853s (17.1549GiB/s) | 20260506 01:34:56.282 | INFO | Downloaded 34GiB to GPU in 2.03934s (16.672GiB/s) | 20260506 01:35:01.420 | INFO | Analyzing chunk frames = [7910912,7911424) | 20260506 01:35:01.420 | INFO | MemInfo: { | 20260506 01:35:01.421 | INFO | "gpu" : " 17 GB", | 20260506 01:35:01.421 | INFO | "huge" : "", | 20260506 01:35:01.421 | INFO | "huge_shared" : " 27 GB", | 20260506 01:35:01.421 | INFO | "malloc" : " 14 GB", | 20260506 01:35:01.421 | INFO | "pinned" : " 29 GB" | 20260506 01:35:01.421 | INFO | } | 20260506 01:35:01.421 | INFO | Perf report for the last 5.1388 seconds: | 20260506 01:35:01.421 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:35:01.421 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:01.421 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:01.421 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:01.421 | INFO | ANALYSIS : 61.6%, 3.0, 95.7ms, 0 | 20260506 01:35:01.421 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:01.421 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:01.421 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 01:35:01.421 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:01.421 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:01.421 | INFO | Uploaded 36GiB to GPU in 2.13906s (16.8298GiB/s) | 20260506 01:35:01.421 | INFO | Downloaded 34GiB to GPU in 2.01788s (16.8494GiB/s) | 20260506 01:35:06.569 | INFO | Analyzing chunk frames = [7911424,7911936) | 20260506 01:35:06.569 | INFO | MemInfo: { | 20260506 01:35:06.569 | INFO | "gpu" : " 17 GB", | 20260506 01:35:06.569 | INFO | "huge" : "", | 20260506 01:35:06.569 | INFO | "huge_shared" : " 27 GB", | 20260506 01:35:06.569 | INFO | "malloc" : " 14 GB", | 20260506 01:35:06.569 | INFO | "pinned" : " 29 GB" | 20260506 01:35:06.569 | INFO | } | 20260506 01:35:06.569 | INFO | Perf report for the last 5.14865 seconds: | 20260506 01:35:06.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:35:06.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:06.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:06.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:06.569 | INFO | ANALYSIS : 60.9%, 3.0, 95.3ms, 0 | 20260506 01:35:06.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:06.569 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:06.569 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:35:06.569 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:06.569 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:06.570 | INFO | Uploaded 36GiB to GPU in 2.08919s (17.2315GiB/s) | 20260506 01:35:06.570 | INFO | Downloaded 34GiB to GPU in 2.0508s (16.5789GiB/s) | 20260506 01:35:11.707 | INFO | Analyzing chunk frames = [7911936,7912448) | 20260506 01:35:11.707 | INFO | MemInfo: { | 20260506 01:35:11.707 | INFO | "gpu" : " 17 GB", | 20260506 01:35:11.707 | INFO | "huge" : "", | 20260506 01:35:11.707 | INFO | "huge_shared" : " 27 GB", | 20260506 01:35:11.707 | INFO | "malloc" : " 14 GB", | 20260506 01:35:11.707 | INFO | "pinned" : " 29 GB" | 20260506 01:35:11.707 | INFO | } | 20260506 01:35:11.707 | INFO | Perf report for the last 5.13818 seconds: | 20260506 01:35:11.707 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:35:11.707 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:11.707 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:11.707 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:11.707 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 01:35:11.707 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:11.708 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:11.708 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 01:35:11.708 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:11.708 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:11.708 | INFO | Uploaded 36GiB to GPU in 2.07526s (17.3472GiB/s) | 20260506 01:35:11.708 | INFO | Downloaded 34GiB to GPU in 2.03015s (16.7475GiB/s) | 20260506 01:35:16.850 | INFO | Analyzing chunk frames = [7912448,7912960) | 20260506 01:35:16.850 | INFO | MemInfo: { | 20260506 01:35:16.850 | INFO | "gpu" : " 17 GB", | 20260506 01:35:16.850 | INFO | "huge" : "", | 20260506 01:35:16.850 | INFO | "huge_shared" : " 27 GB", | 20260506 01:35:16.850 | INFO | "malloc" : " 14 GB", | 20260506 01:35:16.850 | INFO | "pinned" : " 29 GB" | 20260506 01:35:16.850 | INFO | } | 20260506 01:35:16.850 | INFO | Perf report for the last 5.14289 seconds: | 20260506 01:35:16.850 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:35:16.850 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:16.850 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:16.850 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:16.850 | INFO | ANALYSIS : 61.8%, 3.0, 96.6ms, 0 | 20260506 01:35:16.850 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:16.850 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:16.850 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 01:35:16.850 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:16.850 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:16.851 | INFO | Uploaded 36GiB to GPU in 2.00574s (17.9485GiB/s) | 20260506 01:35:16.851 | INFO | Downloaded 34GiB to GPU in 2.04422s (16.6323GiB/s) | 20260506 01:35:21.987 | INFO | Analyzing chunk frames = [7912960,7913472) | 20260506 01:35:21.987 | INFO | MemInfo: { | 20260506 01:35:21.987 | INFO | "gpu" : " 17 GB", | 20260506 01:35:21.987 | INFO | "huge" : "", | 20260506 01:35:21.987 | INFO | "huge_shared" : " 27 GB", | 20260506 01:35:21.987 | INFO | "malloc" : " 14 GB", | 20260506 01:35:21.987 | INFO | "pinned" : " 29 GB" | 20260506 01:35:21.987 | INFO | } | 20260506 01:35:21.987 | INFO | Perf report for the last 5.13675 seconds: | 20260506 01:35:21.987 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:35:21.987 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:21.987 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:21.987 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:21.987 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260506 01:35:21.987 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:21.987 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:21.987 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:35:21.987 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:21.987 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:21.987 | INFO | Uploaded 36GiB to GPU in 2.07505s (17.3489GiB/s) | 20260506 01:35:21.987 | INFO | Downloaded 34GiB to GPU in 2.01867s (16.8428GiB/s) | 20260506 01:35:21.988 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 01:35:21.988 | INFO | Total mem usage: 15.3 GiB | 20260506 01:35:21.988 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 01:35:21.988 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.2%) | 20260506 01:35:21.988 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 01:35:21.988 | INFO | High Water Breakdown: | 20260506 01:35:21.988 | INFO | 60.9% : 8.54 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 01:35:21.988 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 01:35:21.988 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 01:35:21.988 | INFO | ------------------------------------------ | 20260506 01:35:21.988 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:35:21.988 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 01:35:21.988 | INFO | Total mem usage: 26.5 GiB | 20260506 01:35:21.988 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 01:35:21.988 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 01:35:21.988 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 01:35:21.988 | INFO | High Water Breakdown: | 20260506 01:35:21.988 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:35:21.988 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:35:21.988 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:35:21.988 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:35:21.988 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:35:21.988 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:35:21.988 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 01:35:21.988 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 01:35:21.988 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:35:21.988 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 01:35:21.988 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 01:35:21.988 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 01:35:21.988 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 01:35:21.988 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 01:35:21.988 | INFO | ------------------------------------------ | 20260506 01:35:21.988 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:35:21.988 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 01:35:21.988 | INFO | Total mem usage: 25.6 GiB | 20260506 01:35:21.988 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 01:35:21.988 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 01:35:21.988 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 01:35:21.988 | INFO | High Water Breakdown: | 20260506 01:35:21.988 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 01:35:21.988 | INFO | ------------------------------------------ | 20260506 01:35:21.988 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:35:21.988 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 01:35:21.988 | INFO | Total mem usage: 15.7 GiB | 20260506 01:35:21.988 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 01:35:21.988 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 01:35:21.988 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 01:35:21.988 | INFO | High Water Breakdown: | 20260506 01:35:21.988 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 01:35:21.988 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 01:35:21.988 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 01:35:21.988 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 01:35:21.988 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 01:35:21.988 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 01:35:21.988 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 01:35:21.988 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 01:35:21.988 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 01:35:21.988 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:35:21.988 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 01:35:21.988 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:35:21.988 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 01:35:21.988 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 01:35:21.988 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 01:35:21.988 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 01:35:21.988 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:35:21.988 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 01:35:21.988 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:35:21.988 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:35:21.988 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:35:21.988 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 01:35:21.989 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 01:35:21.989 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:35:21.989 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 01:35:21.989 | INFO | ------------------------------------------ | 20260506 01:35:21.989 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:35:21.989 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 01:35:21.989 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:35:21.989 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:35:21.989 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.38e+01, 1.33e+01 / 8.61e-01 | 20260506 01:35:21.989 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.04e-01, 1.02e-02 / 4.20e-03 | 20260506 01:35:21.989 | INFO | PerfReport: 11.86%, Baseline -- 3.71e+00 / 4.47e+00, 5.81e-01 / 2.53e-01 | 20260506 01:35:21.989 | INFO | PerfReport: 3.29%, Histogram -- 1.03e+00 / 1.52e+00, 1.95e-01 / 9.84e-02 | 20260506 01:35:21.989 | INFO | PerfReport: 16.01%, DME -- 4.01e+01 / 4.41e+01, 3.82e+00 / 1.07e+00 | 20260506 01:35:21.989 | INFO | PerfReport: 57.42%, FrameLabeling -- 1.80e+01 / 1.93e+01, 1.54e+00 / 9.89e-01 | 20260506 01:35:21.989 | INFO | PerfReport: 7.04%, PulseAccumulating -- 2.21e+00 / 2.72e+00, 4.19e-01 / 2.30e-01 | 20260506 01:35:21.989 | INFO | PerfReport: 3.03%, Metrics -- 9.50e-01 / 1.22e+00, 2.56e-01 / 1.37e-01 | 20260506 01:35:21.989 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.28e-01 / 3.76e-01, 5.51e-02 / 3.19e-02 | 20260506 01:35:21.989 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 01:35:21.989 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:35:21.989 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:35:21.989 | INFO | PerfReport: Full Execution -- 9.79e+01 / -nan, 2.45e+01 / 0.00e+00 | 20260506 01:35:21.989 | INFO | PerfReport: 22.12%, Upload -- 2.17e+01 / 0.00e+00, 7.22e+00 / 0.00e+00 | 20260506 01:35:21.989 | INFO | PerfReport: 21.82%, Download -- 2.14e+01 / 0.00e+00, 7.49e+00 / 0.00e+00 | 20260506 01:35:21.989 | INFO | PerfReport: 32.01%, Compute -- 3.13e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 01:35:21.989 | INFO | PerfReport: 2.07%, LockUpload -- 2.02e+00 / 0.00e+00, 7.82e+00 / 0.00e+00 | 20260506 01:35:21.989 | INFO | PerfReport: 1.31%, LockDownload -- 1.28e+00 / 0.00e+00, 2.96e+00 / 0.00e+00 | 20260506 01:35:21.989 | INFO | PerfReport: 20.66%, LockCompute -- 2.02e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 01:35:21.989 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.26e-02 / 0.00e+00 | 20260506 01:35:22.113 | INFO | Starting a round of Baz IO | 20260506 01:35:25.249 | INFO | Populating RTMetrics report | 20260506 01:35:25.249 | INFO | RTMetrics report populated | 20260506 01:35:25.249 | INFO | Opening tmp RTMetrics json file | 20260506 01:35:25.250 | INFO | Serializing RTMetrics to json | 20260506 01:35:25.251 | INFO | Serialization done, swapping tmp file now... | 20260506 01:35:25.252 | INFO | RTMetrics done | 20260506 01:35:25.326 | INFO | Finished a round of Baz IO: Wrote 1.821 GiB in 3.213 seconds | 20260506 01:35:25.326 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.31% / 0.00% / 78.69% | 20260506 01:35:25.326 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0795841% / 0% / 7.54992e-05% | 20260506 01:35:25.326 | INFO | Baz IO overall: | 20260506 01:35:25.326 | INFO | BytesWritten : 4460523962733 | 20260506 01:35:25.326 | INFO | NumEvents : 2096258818967 | 20260506 01:35:27.123 | INFO | Analyzing chunk frames = [7913472,7913984) | 20260506 01:35:27.123 | INFO | MemInfo: { | 20260506 01:35:27.123 | INFO | "gpu" : " 17 GB", | 20260506 01:35:27.123 | INFO | "huge" : "", | 20260506 01:35:27.123 | INFO | "huge_shared" : " 27 GB", | 20260506 01:35:27.123 | INFO | "malloc" : " 14 GB", | 20260506 01:35:27.123 | INFO | "pinned" : " 29 GB" | 20260506 01:35:27.123 | INFO | } | 20260506 01:35:27.123 | INFO | Perf report for the last 5.1362 seconds: | 20260506 01:35:27.123 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:35:27.123 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:27.123 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:27.123 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:27.124 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260506 01:35:27.124 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:27.124 | INFO | RT_METRICS : 43.0%, 1.0, 22.5ms, 0 | 20260506 01:35:27.124 | INFO | PRE_HQ : 59.6%, 1.7, 53.6ms, 0 | 20260506 01:35:27.124 | INFO | BAZWRITER : 61.7%, 3.5, 109.4ms, 0 | 20260506 01:35:27.124 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:35:27.124 | INFO | Uploaded 36GiB to GPU in 2.17622s (16.5425GiB/s) | 20260506 01:35:27.124 | INFO | Downloaded 38GiB to GPU in 2.23905s (16.9715GiB/s) | 20260506 01:35:32.259 | INFO | Analyzing chunk frames = [7913984,7914496) | 20260506 01:35:32.259 | INFO | MemInfo: { | 20260506 01:35:32.259 | INFO | "gpu" : " 17 GB", | 20260506 01:35:32.259 | INFO | "huge" : "", | 20260506 01:35:32.259 | INFO | "huge_shared" : " 27 GB", | 20260506 01:35:32.259 | INFO | "malloc" : " 14 GB", | 20260506 01:35:32.259 | INFO | "pinned" : " 29 GB" | 20260506 01:35:32.259 | INFO | } | 20260506 01:35:32.259 | INFO | Perf report for the last 5.13556 seconds: | 20260506 01:35:32.259 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:35:32.259 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:32.259 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:32.259 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:32.259 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260506 01:35:32.259 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:32.259 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:32.259 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 01:35:32.259 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:32.259 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:32.259 | INFO | Uploaded 36GiB to GPU in 1.96093s (18.3586GiB/s) | 20260506 01:35:32.259 | INFO | Downloaded 34GiB to GPU in 1.98651s (17.1155GiB/s) | 20260506 01:35:37.400 | INFO | Analyzing chunk frames = [7914496,7915008) | 20260506 01:35:37.400 | INFO | MemInfo: { | 20260506 01:35:37.400 | INFO | "gpu" : " 17 GB", | 20260506 01:35:37.400 | INFO | "huge" : "", | 20260506 01:35:37.400 | INFO | "huge_shared" : " 27 GB", | 20260506 01:35:37.400 | INFO | "malloc" : " 14 GB", | 20260506 01:35:37.400 | INFO | "pinned" : " 29 GB" | 20260506 01:35:37.400 | INFO | } | 20260506 01:35:37.400 | INFO | Perf report for the last 5.14129 seconds: | 20260506 01:35:37.400 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:35:37.400 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:37.400 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:37.400 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:37.400 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 01:35:37.400 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:37.400 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:37.400 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:35:37.400 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:37.400 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:37.400 | INFO | Uploaded 36GiB to GPU in 2.07946s (17.3122GiB/s) | 20260506 01:35:37.401 | INFO | Downloaded 34GiB to GPU in 2.02005s (16.8312GiB/s) | 20260506 01:35:42.537 | INFO | Analyzing chunk frames = [7915008,7915520) | 20260506 01:35:42.537 | INFO | MemInfo: { | 20260506 01:35:42.537 | INFO | "gpu" : " 17 GB", | 20260506 01:35:42.537 | INFO | "huge" : "", | 20260506 01:35:42.537 | INFO | "huge_shared" : " 27 GB", | 20260506 01:35:42.537 | INFO | "malloc" : " 14 GB", | 20260506 01:35:42.537 | INFO | "pinned" : " 29 GB" | 20260506 01:35:42.537 | INFO | } | 20260506 01:35:42.537 | INFO | Perf report for the last 5.13659 seconds: | 20260506 01:35:42.537 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:35:42.537 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:42.537 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:42.537 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:42.537 | INFO | ANALYSIS : 61.6%, 3.0, 95.6ms, 0 | 20260506 01:35:42.537 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:42.537 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:42.537 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:35:42.537 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:42.537 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:42.537 | INFO | Uploaded 36GiB to GPU in 2.14245s (16.8032GiB/s) | 20260506 01:35:42.537 | INFO | Downloaded 34GiB to GPU in 2.02677s (16.7755GiB/s) | 20260506 01:35:47.675 | INFO | Analyzing chunk frames = [7915520,7916032) | 20260506 01:35:47.675 | INFO | MemInfo: { | 20260506 01:35:47.675 | INFO | "gpu" : " 17 GB", | 20260506 01:35:47.675 | INFO | "huge" : "", | 20260506 01:35:47.675 | INFO | "huge_shared" : " 27 GB", | 20260506 01:35:47.675 | INFO | "malloc" : " 14 GB", | 20260506 01:35:47.675 | INFO | "pinned" : " 29 GB" | 20260506 01:35:47.675 | INFO | } | 20260506 01:35:47.675 | INFO | Perf report for the last 5.13827 seconds: | 20260506 01:35:47.675 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:35:47.675 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:47.675 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:47.675 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:47.675 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 01:35:47.675 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:47.675 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:47.675 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:35:47.675 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:47.675 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:47.675 | INFO | Uploaded 36GiB to GPU in 2.08681s (17.2512GiB/s) | 20260506 01:35:47.675 | INFO | Downloaded 34GiB to GPU in 2.04669s (16.6122GiB/s) | 20260506 01:35:52.825 | INFO | Analyzing chunk frames = [7916032,7916544) | 20260506 01:35:52.825 | INFO | MemInfo: { | 20260506 01:35:52.825 | INFO | "gpu" : " 17 GB", | 20260506 01:35:52.825 | INFO | "huge" : "", | 20260506 01:35:52.825 | INFO | "huge_shared" : " 27 GB", | 20260506 01:35:52.825 | INFO | "malloc" : " 14 GB", | 20260506 01:35:52.825 | INFO | "pinned" : " 29 GB" | 20260506 01:35:52.825 | INFO | } | 20260506 01:35:52.825 | INFO | Perf report for the last 5.15007 seconds: | 20260506 01:35:52.825 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:35:52.825 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:52.825 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:52.825 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:52.825 | INFO | ANALYSIS : 61.1%, 3.0, 95.6ms, 0 | 20260506 01:35:52.825 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:52.825 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:52.825 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 01:35:52.825 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:52.825 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:52.825 | INFO | Uploaded 36GiB to GPU in 2.0778s (17.326GiB/s) | 20260506 01:35:52.825 | INFO | Downloaded 34GiB to GPU in 2.04206s (16.6499GiB/s) | 20260506 01:35:57.966 | INFO | Analyzing chunk frames = [7916544,7917056) | 20260506 01:35:57.966 | INFO | MemInfo: { | 20260506 01:35:57.966 | INFO | "gpu" : " 17 GB", | 20260506 01:35:57.966 | INFO | "huge" : "", | 20260506 01:35:57.966 | INFO | "huge_shared" : " 27 GB", | 20260506 01:35:57.966 | INFO | "malloc" : " 14 GB", | 20260506 01:35:57.966 | INFO | "pinned" : " 29 GB" | 20260506 01:35:57.966 | INFO | } | 20260506 01:35:57.967 | INFO | Perf report for the last 5.14133 seconds: | 20260506 01:35:57.967 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:35:57.967 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:57.967 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:57.967 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:57.967 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260506 01:35:57.967 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:57.967 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:57.967 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 01:35:57.967 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:57.967 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:35:57.967 | INFO | Uploaded 36GiB to GPU in 2.03126s (17.723GiB/s) | 20260506 01:35:57.967 | INFO | Downloaded 34GiB to GPU in 2.06052s (16.5007GiB/s) | 20260506 01:36:03.102 | INFO | Analyzing chunk frames = [7917056,7917568) | 20260506 01:36:03.102 | INFO | MemInfo: { | 20260506 01:36:03.102 | INFO | "gpu" : " 17 GB", | 20260506 01:36:03.102 | INFO | "huge" : "", | 20260506 01:36:03.102 | INFO | "huge_shared" : " 27 GB", | 20260506 01:36:03.102 | INFO | "malloc" : " 14 GB", | 20260506 01:36:03.103 | INFO | "pinned" : " 29 GB" | 20260506 01:36:03.103 | INFO | } | 20260506 01:36:03.103 | INFO | Perf report for the last 5.13607 seconds: | 20260506 01:36:03.103 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:36:03.103 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:03.103 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:03.103 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:03.103 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 01:36:03.103 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:03.103 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:03.103 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 01:36:03.103 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:03.103 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:03.103 | INFO | Uploaded 36GiB to GPU in 2.03536s (17.6873GiB/s) | 20260506 01:36:03.103 | INFO | Downloaded 34GiB to GPU in 2.02424s (16.7965GiB/s) | 20260506 01:36:03.237 | INFO | Starting a round of Baz IO | 20260506 01:36:06.365 | INFO | Populating RTMetrics report | 20260506 01:36:06.366 | INFO | RTMetrics report populated | 20260506 01:36:06.366 | INFO | Opening tmp RTMetrics json file | 20260506 01:36:06.366 | INFO | Serializing RTMetrics to json | 20260506 01:36:06.368 | INFO | Serialization done, swapping tmp file now... | 20260506 01:36:06.368 | INFO | RTMetrics done | 20260506 01:36:06.441 | INFO | Finished a round of Baz IO: Wrote 1.822 GiB in 3.204 seconds | 20260506 01:36:06.441 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.31% / 0.00% / 78.69% | 20260506 01:36:06.441 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0794689% / 3.97364e-06% / 5.5631e-05% | 20260506 01:36:06.441 | INFO | Baz IO overall: | 20260506 01:36:06.441 | INFO | BytesWritten : 4462480560534 | 20260506 01:36:06.441 | INFO | NumEvents : 2097150907297 | 20260506 01:36:08.229 | INFO | Analyzing chunk frames = [7917568,7918080) | 20260506 01:36:08.229 | INFO | MemInfo: { | 20260506 01:36:08.229 | INFO | "gpu" : " 17 GB", | 20260506 01:36:08.229 | INFO | "huge" : "", | 20260506 01:36:08.229 | INFO | "huge_shared" : " 27 GB", | 20260506 01:36:08.229 | INFO | "malloc" : " 14 GB", | 20260506 01:36:08.229 | INFO | "pinned" : " 29 GB" | 20260506 01:36:08.229 | INFO | } | 20260506 01:36:08.229 | INFO | Perf report for the last 5.12653 seconds: | 20260506 01:36:08.229 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:36:08.229 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:08.229 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:08.229 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:08.229 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 01:36:08.229 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:08.229 | INFO | RT_METRICS : 43.1%, 1.0, 22.5ms, 0 | 20260506 01:36:08.229 | INFO | PRE_HQ : 59.9%, 1.7, 54.2ms, 0 | 20260506 01:36:08.229 | INFO | BAZWRITER : 61.9%, 3.5, 110.2ms, 0 | 20260506 01:36:08.229 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:36:08.229 | INFO | Uploaded 36GiB to GPU in 2.18336s (16.4884GiB/s) | 20260506 01:36:08.229 | INFO | Downloaded 38GiB to GPU in 2.24137s (16.9539GiB/s) | 20260506 01:36:13.371 | INFO | Analyzing chunk frames = [7918080,7918592) | 20260506 01:36:13.371 | INFO | MemInfo: { | 20260506 01:36:13.371 | INFO | "gpu" : " 17 GB", | 20260506 01:36:13.371 | INFO | "huge" : "", | 20260506 01:36:13.371 | INFO | "huge_shared" : " 27 GB", | 20260506 01:36:13.371 | INFO | "malloc" : " 14 GB", | 20260506 01:36:13.371 | INFO | "pinned" : " 29 GB" | 20260506 01:36:13.371 | INFO | } | 20260506 01:36:13.371 | INFO | Perf report for the last 5.14191 seconds: | 20260506 01:36:13.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:36:13.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:13.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:13.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:13.371 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260506 01:36:13.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:13.371 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:13.371 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 01:36:13.371 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:13.371 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:13.371 | INFO | Uploaded 36GiB to GPU in 2.09567s (17.1782GiB/s) | 20260506 01:36:13.371 | INFO | Downloaded 34GiB to GPU in 2.03257s (16.7276GiB/s) | 20260506 01:36:18.523 | INFO | Analyzing chunk frames = [7918592,7919104) | 20260506 01:36:18.523 | INFO | MemInfo: { | 20260506 01:36:18.523 | INFO | "gpu" : " 17 GB", | 20260506 01:36:18.523 | INFO | "huge" : "", | 20260506 01:36:18.523 | INFO | "huge_shared" : " 27 GB", | 20260506 01:36:18.523 | INFO | "malloc" : " 14 GB", | 20260506 01:36:18.523 | INFO | "pinned" : " 29 GB" | 20260506 01:36:18.523 | INFO | } | 20260506 01:36:18.523 | INFO | Perf report for the last 5.15237 seconds: | 20260506 01:36:18.523 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:36:18.523 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:18.523 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:18.523 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:18.523 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 01:36:18.523 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:18.524 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:18.524 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:36:18.524 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:18.524 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:18.524 | INFO | Uploaded 36GiB to GPU in 2.07929s (17.3136GiB/s) | 20260506 01:36:18.524 | INFO | Downloaded 34GiB to GPU in 2.02128s (16.821GiB/s) | 20260506 01:36:23.661 | INFO | Analyzing chunk frames = [7919104,7919616) | 20260506 01:36:23.661 | INFO | MemInfo: { | 20260506 01:36:23.661 | INFO | "gpu" : " 17 GB", | 20260506 01:36:23.661 | INFO | "huge" : "", | 20260506 01:36:23.661 | INFO | "huge_shared" : " 27 GB", | 20260506 01:36:23.661 | INFO | "malloc" : " 14 GB", | 20260506 01:36:23.661 | INFO | "pinned" : " 29 GB" | 20260506 01:36:23.661 | INFO | } | 20260506 01:36:23.661 | INFO | Perf report for the last 5.138 seconds: | 20260506 01:36:23.661 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:36:23.661 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:23.661 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:23.661 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:23.661 | INFO | ANALYSIS : 61.0%, 3.0, 94.7ms, 0 | 20260506 01:36:23.662 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:23.662 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:23.662 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:36:23.662 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:23.662 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:23.662 | INFO | Uploaded 36GiB to GPU in 2.07115s (17.3816GiB/s) | 20260506 01:36:23.662 | INFO | Downloaded 34GiB to GPU in 2.00029s (16.9976GiB/s) | 20260506 01:36:28.796 | INFO | Analyzing chunk frames = [7919616,7920128) | 20260506 01:36:28.796 | INFO | MemInfo: { | 20260506 01:36:28.796 | INFO | "gpu" : " 17 GB", | 20260506 01:36:28.796 | INFO | "huge" : "", | 20260506 01:36:28.796 | INFO | "huge_shared" : " 27 GB", | 20260506 01:36:28.796 | INFO | "malloc" : " 14 GB", | 20260506 01:36:28.796 | INFO | "pinned" : " 29 GB" | 20260506 01:36:28.796 | INFO | } | 20260506 01:36:28.796 | INFO | Perf report for the last 5.13459 seconds: | 20260506 01:36:28.796 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:36:28.796 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:28.796 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:28.796 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:28.796 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260506 01:36:28.796 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:28.796 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:28.796 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 01:36:28.796 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:28.796 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:28.796 | INFO | Uploaded 36GiB to GPU in 2.08857s (17.2367GiB/s) | 20260506 01:36:28.796 | INFO | Downloaded 34GiB to GPU in 2.03131s (16.738GiB/s) | 20260506 01:36:33.936 | INFO | Analyzing chunk frames = [7920128,7920640) | 20260506 01:36:33.936 | INFO | MemInfo: { | 20260506 01:36:33.936 | INFO | "gpu" : " 17 GB", | 20260506 01:36:33.936 | INFO | "huge" : "", | 20260506 01:36:33.936 | INFO | "huge_shared" : " 27 GB", | 20260506 01:36:33.936 | INFO | "malloc" : " 14 GB", | 20260506 01:36:33.936 | INFO | "pinned" : " 29 GB" | 20260506 01:36:33.936 | INFO | } | 20260506 01:36:33.936 | INFO | Perf report for the last 5.14035 seconds: | 20260506 01:36:33.936 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:36:33.936 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:33.936 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:33.936 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:33.936 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 01:36:33.937 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:33.937 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:33.937 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 01:36:33.937 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:33.937 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:33.937 | INFO | Uploaded 36GiB to GPU in 2.07513s (17.3483GiB/s) | 20260506 01:36:33.937 | INFO | Downloaded 34GiB to GPU in 2.04452s (16.6298GiB/s) | 20260506 01:36:39.073 | INFO | Analyzing chunk frames = [7920640,7921152) | 20260506 01:36:39.073 | INFO | MemInfo: { | 20260506 01:36:39.073 | INFO | "gpu" : " 17 GB", | 20260506 01:36:39.073 | INFO | "huge" : "", | 20260506 01:36:39.073 | INFO | "huge_shared" : " 27 GB", | 20260506 01:36:39.073 | INFO | "malloc" : " 14 GB", | 20260506 01:36:39.073 | INFO | "pinned" : " 29 GB" | 20260506 01:36:39.073 | INFO | } | 20260506 01:36:39.073 | INFO | Perf report for the last 5.13681 seconds: | 20260506 01:36:39.073 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:36:39.073 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:39.073 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:39.073 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:39.073 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260506 01:36:39.073 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:39.073 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:39.073 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 01:36:39.073 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:39.073 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:39.074 | INFO | Uploaded 36GiB to GPU in 2.06341s (17.4469GiB/s) | 20260506 01:36:39.074 | INFO | Downloaded 34GiB to GPU in 2.07333s (16.3987GiB/s) | 20260506 01:36:44.211 | INFO | Analyzing chunk frames = [7921152,7921664) | 20260506 01:36:44.211 | INFO | MemInfo: { | 20260506 01:36:44.211 | INFO | "gpu" : " 17 GB", | 20260506 01:36:44.211 | INFO | "huge" : "", | 20260506 01:36:44.211 | INFO | "huge_shared" : " 27 GB", | 20260506 01:36:44.211 | INFO | "malloc" : " 14 GB", | 20260506 01:36:44.211 | INFO | "pinned" : " 29 GB" | 20260506 01:36:44.211 | INFO | } | 20260506 01:36:44.211 | INFO | Perf report for the last 5.13801 seconds: | 20260506 01:36:44.211 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:36:44.211 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:44.211 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:44.211 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:44.211 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260506 01:36:44.211 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:44.211 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:44.211 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:36:44.211 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:44.211 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:44.211 | INFO | Uploaded 36GiB to GPU in 2.13278s (16.8794GiB/s) | 20260506 01:36:44.212 | INFO | Downloaded 34GiB to GPU in 2.03509s (16.7068GiB/s) | 20260506 01:36:44.212 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 01:36:44.212 | INFO | Total mem usage: 15.3 GiB | 20260506 01:36:44.212 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 01:36:44.212 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.21%) | 20260506 01:36:44.212 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 01:36:44.212 | INFO | High Water Breakdown: | 20260506 01:36:44.212 | INFO | 60.9% : 8.54 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 01:36:44.212 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 01:36:44.212 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 01:36:44.212 | INFO | ------------------------------------------ | 20260506 01:36:44.212 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:36:44.212 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 01:36:44.212 | INFO | Total mem usage: 26.5 GiB | 20260506 01:36:44.212 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 01:36:44.212 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 01:36:44.212 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260506 01:36:44.212 | INFO | High Water Breakdown: | 20260506 01:36:44.212 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:36:44.212 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:36:44.212 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:36:44.212 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:36:44.212 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:36:44.212 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:36:44.212 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 01:36:44.212 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 01:36:44.212 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:36:44.212 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 01:36:44.212 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 01:36:44.212 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 01:36:44.212 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 01:36:44.212 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 01:36:44.212 | INFO | ------------------------------------------ | 20260506 01:36:44.212 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:36:44.213 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 01:36:44.213 | INFO | Total mem usage: 25.6 GiB | 20260506 01:36:44.213 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 01:36:44.213 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 01:36:44.213 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 01:36:44.213 | INFO | High Water Breakdown: | 20260506 01:36:44.213 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 01:36:44.213 | INFO | ------------------------------------------ | 20260506 01:36:44.213 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:36:44.213 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 01:36:44.213 | INFO | Total mem usage: 15.7 GiB | 20260506 01:36:44.213 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 01:36:44.213 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 01:36:44.213 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 01:36:44.213 | INFO | High Water Breakdown: | 20260506 01:36:44.213 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 01:36:44.213 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 01:36:44.213 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 01:36:44.213 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 01:36:44.213 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 01:36:44.213 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 01:36:44.213 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 01:36:44.213 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 01:36:44.213 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 01:36:44.213 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:36:44.213 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 01:36:44.213 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:36:44.213 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 01:36:44.213 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 01:36:44.213 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 01:36:44.213 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 01:36:44.213 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:36:44.213 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 01:36:44.213 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:36:44.213 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:36:44.213 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:36:44.213 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 01:36:44.214 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 01:36:44.214 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:36:44.214 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 01:36:44.214 | INFO | ------------------------------------------ | 20260506 01:36:44.214 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:36:44.214 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 01:36:44.214 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:36:44.214 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:36:44.214 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.40e+01, 1.35e+01 / 8.57e-01 | 20260506 01:36:44.214 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.04e-01, 1.06e-02 / 4.18e-03 | 20260506 01:36:44.214 | INFO | PerfReport: 11.84%, Baseline -- 3.71e+00 / 4.51e+00, 5.72e-01 / 1.88e-01 | 20260506 01:36:44.214 | INFO | PerfReport: 3.29%, Histogram -- 1.03e+00 / 1.50e+00, 1.90e-01 / 1.17e-01 | 20260506 01:36:44.214 | INFO | PerfReport: 16.02%, DME -- 4.01e+01 / 4.41e+01, 3.98e+00 / 1.09e+00 | 20260506 01:36:44.214 | INFO | PerfReport: 57.39%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.48e+00 / 1.08e+00 | 20260506 01:36:44.214 | INFO | PerfReport: 7.06%, PulseAccumulating -- 2.21e+00 / 2.72e+00, 4.19e-01 / 2.66e-01 | 20260506 01:36:44.214 | INFO | PerfReport: 3.04%, Metrics -- 9.52e-01 / 1.21e+00, 2.56e-01 / 9.94e-02 | 20260506 01:36:44.214 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.29e-01 / 3.79e-01, 5.53e-02 / 2.65e-02 | 20260506 01:36:44.214 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 01:36:44.214 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:36:44.214 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:36:44.214 | INFO | PerfReport: Full Execution -- 9.79e+01 / -nan, 2.41e+01 / 0.00e+00 | 20260506 01:36:44.214 | INFO | PerfReport: 22.19%, Upload -- 2.17e+01 / 0.00e+00, 7.33e+00 / 0.00e+00 | 20260506 01:36:44.214 | INFO | PerfReport: 21.89%, Download -- 2.14e+01 / 0.00e+00, 7.57e+00 / 0.00e+00 | 20260506 01:36:44.214 | INFO | PerfReport: 31.95%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 01:36:44.214 | INFO | PerfReport: 2.06%, LockUpload -- 2.02e+00 / 0.00e+00, 7.82e+00 / 0.00e+00 | 20260506 01:36:44.214 | INFO | PerfReport: 1.32%, LockDownload -- 1.29e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260506 01:36:44.214 | INFO | PerfReport: 20.58%, LockCompute -- 2.02e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 01:36:44.214 | INFO | PerfReport: 0.01%, Misc -- 1.26e-02 / 0.00e+00, 1.48e-02 / 0.00e+00 | 20260506 01:36:44.329 | INFO | Starting a round of Baz IO | 20260506 01:36:47.476 | INFO | Populating RTMetrics report | 20260506 01:36:47.476 | INFO | RTMetrics report populated | 20260506 01:36:47.476 | INFO | Opening tmp RTMetrics json file | 20260506 01:36:47.478 | INFO | Serializing RTMetrics to json | 20260506 01:36:47.480 | INFO | Serialization done, swapping tmp file now... | 20260506 01:36:47.480 | INFO | RTMetrics done | 20260506 01:36:47.559 | INFO | Finished a round of Baz IO: Wrote 1.821 GiB in 3.23 seconds | 20260506 01:36:47.559 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.31% / 0.00% / 78.69% | 20260506 01:36:47.559 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0791351% / 0% / 7.54992e-05% | 20260506 01:36:47.559 | INFO | Baz IO overall: | 20260506 01:36:47.559 | INFO | BytesWritten : 4464436058020 | 20260506 01:36:47.559 | INFO | NumEvents : 2098042349560 | 20260506 01:36:49.348 | INFO | Analyzing chunk frames = [7921664,7922176) | 20260506 01:36:49.348 | INFO | MemInfo: { | 20260506 01:36:49.348 | INFO | "gpu" : " 17 GB", | 20260506 01:36:49.348 | INFO | "huge" : "", | 20260506 01:36:49.348 | INFO | "huge_shared" : " 27 GB", | 20260506 01:36:49.348 | INFO | "malloc" : " 14 GB", | 20260506 01:36:49.348 | INFO | "pinned" : " 29 GB" | 20260506 01:36:49.348 | INFO | } | 20260506 01:36:49.348 | INFO | Perf report for the last 5.13719 seconds: | 20260506 01:36:49.348 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:36:49.348 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:49.349 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:49.349 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:49.349 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260506 01:36:49.349 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:49.349 | INFO | RT_METRICS : 43.2%, 1.0, 22.6ms, 0 | 20260506 01:36:49.349 | INFO | PRE_HQ : 59.7%, 1.7, 53.8ms, 0 | 20260506 01:36:49.349 | INFO | BAZWRITER : 61.9%, 3.5, 110.9ms, 0 | 20260506 01:36:49.349 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:36:49.349 | INFO | Uploaded 36GiB to GPU in 2.18031s (16.5114GiB/s) | 20260506 01:36:49.349 | INFO | Downloaded 38GiB to GPU in 2.23961s (16.9672GiB/s) | 20260506 01:36:54.492 | INFO | Analyzing chunk frames = [7922176,7922688) | 20260506 01:36:54.492 | INFO | MemInfo: { | 20260506 01:36:54.492 | INFO | "gpu" : " 17 GB", | 20260506 01:36:54.492 | INFO | "huge" : "", | 20260506 01:36:54.492 | INFO | "huge_shared" : " 27 GB", | 20260506 01:36:54.492 | INFO | "malloc" : " 14 GB", | 20260506 01:36:54.492 | INFO | "pinned" : " 29 GB" | 20260506 01:36:54.492 | INFO | } | 20260506 01:36:54.492 | INFO | Perf report for the last 5.14355 seconds: | 20260506 01:36:54.492 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:36:54.492 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:54.492 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:54.492 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:54.492 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260506 01:36:54.492 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:54.492 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:54.492 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 01:36:54.492 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:54.492 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:54.492 | INFO | Uploaded 36GiB to GPU in 2.11948s (16.9853GiB/s) | 20260506 01:36:54.492 | INFO | Downloaded 34GiB to GPU in 2.04538s (16.6228GiB/s) | 20260506 01:36:59.627 | INFO | Analyzing chunk frames = [7922688,7923200) | 20260506 01:36:59.627 | INFO | MemInfo: { | 20260506 01:36:59.627 | INFO | "gpu" : " 17 GB", | 20260506 01:36:59.627 | INFO | "huge" : "", | 20260506 01:36:59.627 | INFO | "huge_shared" : " 27 GB", | 20260506 01:36:59.627 | INFO | "malloc" : " 14 GB", | 20260506 01:36:59.627 | INFO | "pinned" : " 29 GB" | 20260506 01:36:59.627 | INFO | } | 20260506 01:36:59.627 | INFO | Perf report for the last 5.13528 seconds: | 20260506 01:36:59.627 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:36:59.627 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:59.627 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:59.627 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:59.627 | INFO | ANALYSIS : 61.4%, 3.0, 95.4ms, 0 | 20260506 01:36:59.627 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:59.627 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:59.627 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 01:36:59.627 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:59.627 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:36:59.628 | INFO | Uploaded 36GiB to GPU in 2.09511s (17.1828GiB/s) | 20260506 01:36:59.628 | INFO | Downloaded 34GiB to GPU in 2.02366s (16.8013GiB/s) | 20260506 01:37:04.776 | INFO | Analyzing chunk frames = [7923200,7923712) | 20260506 01:37:04.776 | INFO | MemInfo: { | 20260506 01:37:04.776 | INFO | "gpu" : " 17 GB", | 20260506 01:37:04.776 | INFO | "huge" : "", | 20260506 01:37:04.776 | INFO | "huge_shared" : " 27 GB", | 20260506 01:37:04.776 | INFO | "malloc" : " 14 GB", | 20260506 01:37:04.776 | INFO | "pinned" : " 29 GB" | 20260506 01:37:04.776 | INFO | } | 20260506 01:37:04.776 | INFO | Perf report for the last 5.14861 seconds: | 20260506 01:37:04.776 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:37:04.776 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:04.776 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:04.776 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:04.776 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 01:37:04.776 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:04.776 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:04.776 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:37:04.776 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:04.776 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:04.776 | INFO | Uploaded 36GiB to GPU in 2.15072s (16.7386GiB/s) | 20260506 01:37:04.776 | INFO | Downloaded 34GiB to GPU in 2.04818s (16.6001GiB/s) | 20260506 01:37:09.913 | INFO | Analyzing chunk frames = [7923712,7924224) | 20260506 01:37:09.913 | INFO | MemInfo: { | 20260506 01:37:09.913 | INFO | "gpu" : " 17 GB", | 20260506 01:37:09.913 | INFO | "huge" : "", | 20260506 01:37:09.913 | INFO | "huge_shared" : " 27 GB", | 20260506 01:37:09.913 | INFO | "malloc" : " 14 GB", | 20260506 01:37:09.913 | INFO | "pinned" : " 29 GB" | 20260506 01:37:09.913 | INFO | } | 20260506 01:37:09.913 | INFO | Perf report for the last 5.13695 seconds: | 20260506 01:37:09.913 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:37:09.913 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:09.913 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:09.913 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:09.913 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260506 01:37:09.913 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:09.913 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:09.913 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 01:37:09.913 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:09.913 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:09.913 | INFO | Uploaded 36GiB to GPU in 2.06562s (17.4282GiB/s) | 20260506 01:37:09.913 | INFO | Downloaded 34GiB to GPU in 2.03076s (16.7425GiB/s) | 20260506 01:37:15.047 | INFO | Analyzing chunk frames = [7924224,7924736) | 20260506 01:37:15.047 | INFO | MemInfo: { | 20260506 01:37:15.047 | INFO | "gpu" : " 17 GB", | 20260506 01:37:15.047 | INFO | "huge" : "", | 20260506 01:37:15.047 | INFO | "huge_shared" : " 27 GB", | 20260506 01:37:15.047 | INFO | "malloc" : " 14 GB", | 20260506 01:37:15.047 | INFO | "pinned" : " 29 GB" | 20260506 01:37:15.047 | INFO | } | 20260506 01:37:15.047 | INFO | Perf report for the last 5.1345 seconds: | 20260506 01:37:15.047 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:37:15.047 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:15.047 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:15.047 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:15.047 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 01:37:15.047 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:15.047 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:15.047 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:37:15.047 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:15.048 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:15.048 | INFO | Uploaded 36GiB to GPU in 2.07193s (17.3751GiB/s) | 20260506 01:37:15.048 | INFO | Downloaded 34GiB to GPU in 2.03673s (16.6934GiB/s) | 20260506 01:37:20.186 | INFO | Analyzing chunk frames = [7924736,7925248) | 20260506 01:37:20.186 | INFO | MemInfo: { | 20260506 01:37:20.186 | INFO | "gpu" : " 17 GB", | 20260506 01:37:20.186 | INFO | "huge" : "", | 20260506 01:37:20.186 | INFO | "huge_shared" : " 27 GB", | 20260506 01:37:20.186 | INFO | "malloc" : " 14 GB", | 20260506 01:37:20.186 | INFO | "pinned" : " 29 GB" | 20260506 01:37:20.186 | INFO | } | 20260506 01:37:20.186 | INFO | Perf report for the last 5.13886 seconds: | 20260506 01:37:20.186 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:37:20.186 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:20.186 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:20.186 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:20.186 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260506 01:37:20.186 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:20.186 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:20.186 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 01:37:20.186 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:20.186 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:20.186 | INFO | Uploaded 36GiB to GPU in 2.14554s (16.779GiB/s) | 20260506 01:37:20.187 | INFO | Downloaded 34GiB to GPU in 2.08368s (16.3173GiB/s) | 20260506 01:37:25.325 | INFO | Analyzing chunk frames = [7925248,7925760) | 20260506 01:37:25.325 | INFO | MemInfo: { | 20260506 01:37:25.325 | INFO | "gpu" : " 17 GB", | 20260506 01:37:25.325 | INFO | "huge" : "", | 20260506 01:37:25.325 | INFO | "huge_shared" : " 27 GB", | 20260506 01:37:25.325 | INFO | "malloc" : " 14 GB", | 20260506 01:37:25.325 | INFO | "pinned" : " 29 GB" | 20260506 01:37:25.325 | INFO | } | 20260506 01:37:25.325 | INFO | Perf report for the last 5.13844 seconds: | 20260506 01:37:25.325 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:37:25.325 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:25.325 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:25.325 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:25.325 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260506 01:37:25.325 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:25.325 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:25.325 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:37:25.325 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:25.325 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:25.325 | INFO | Uploaded 36GiB to GPU in 2.13518s (16.8604GiB/s) | 20260506 01:37:25.325 | INFO | Downloaded 34GiB to GPU in 2.02003s (16.8314GiB/s) | 20260506 01:37:25.443 | INFO | Starting a round of Baz IO | 20260506 01:37:28.592 | INFO | Populating RTMetrics report | 20260506 01:37:28.592 | INFO | RTMetrics report populated | 20260506 01:37:28.592 | INFO | Opening tmp RTMetrics json file | 20260506 01:37:28.592 | INFO | Serializing RTMetrics to json | 20260506 01:37:28.594 | INFO | Serialization done, swapping tmp file now... | 20260506 01:37:28.594 | INFO | RTMetrics done | 20260506 01:37:28.665 | INFO | Finished a round of Baz IO: Wrote 1.82 GiB in 3.222 seconds | 20260506 01:37:28.665 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.31% / 0.00% / 78.69% | 20260506 01:37:28.665 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0786901% / 0% / 8.34465e-05% | 20260506 01:37:28.665 | INFO | Baz IO overall: | 20260506 01:37:28.665 | INFO | BytesWritten : 4466389765763 | 20260506 01:37:28.665 | INFO | NumEvents : 2098932474479 | 20260506 01:37:30.473 | INFO | Analyzing chunk frames = [7925760,7926272) | 20260506 01:37:30.473 | INFO | MemInfo: { | 20260506 01:37:30.473 | INFO | "gpu" : " 17 GB", | 20260506 01:37:30.473 | INFO | "huge" : "", | 20260506 01:37:30.473 | INFO | "huge_shared" : " 27 GB", | 20260506 01:37:30.473 | INFO | "malloc" : " 14 GB", | 20260506 01:37:30.473 | INFO | "pinned" : " 29 GB" | 20260506 01:37:30.473 | INFO | } | 20260506 01:37:30.473 | INFO | Perf report for the last 5.14837 seconds: | 20260506 01:37:30.473 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:37:30.473 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:30.473 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:30.473 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:30.473 | INFO | ANALYSIS : 62.4%, 3.0, 97.7ms, 0 | 20260506 01:37:30.473 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:30.473 | INFO | RT_METRICS : 42.8%, 1.0, 22.5ms, 0 | 20260506 01:37:30.473 | INFO | PRE_HQ : 59.3%, 1.7, 53.4ms, 0 | 20260506 01:37:30.473 | INFO | BAZWRITER : 61.6%, 3.4, 109.1ms, 0 | 20260506 01:37:30.473 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:37:30.473 | INFO | Uploaded 36GiB to GPU in 2.14557s (16.7788GiB/s) | 20260506 01:37:30.473 | INFO | Downloaded 38GiB to GPU in 2.22987s (17.0414GiB/s) | 20260506 01:37:35.609 | INFO | Analyzing chunk frames = [7926272,7926784) | 20260506 01:37:35.609 | INFO | MemInfo: { | 20260506 01:37:35.609 | INFO | "gpu" : " 17 GB", | 20260506 01:37:35.609 | INFO | "huge" : "", | 20260506 01:37:35.609 | INFO | "huge_shared" : " 27 GB", | 20260506 01:37:35.609 | INFO | "malloc" : " 14 GB", | 20260506 01:37:35.610 | INFO | "pinned" : " 29 GB" | 20260506 01:37:35.610 | INFO | } | 20260506 01:37:35.610 | INFO | Perf report for the last 5.13643 seconds: | 20260506 01:37:35.610 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:37:35.610 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:35.610 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:35.610 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:35.610 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260506 01:37:35.610 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:35.610 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:35.610 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 01:37:35.610 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:35.610 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:35.610 | INFO | Uploaded 36GiB to GPU in 1.95258s (18.4371GiB/s) | 20260506 01:37:35.610 | INFO | Downloaded 34GiB to GPU in 1.9794s (17.1769GiB/s) | 20260506 01:37:40.743 | INFO | Analyzing chunk frames = [7926784,7927296) | 20260506 01:37:40.743 | INFO | MemInfo: { | 20260506 01:37:40.743 | INFO | "gpu" : " 17 GB", | 20260506 01:37:40.743 | INFO | "huge" : "", | 20260506 01:37:40.743 | INFO | "huge_shared" : " 27 GB", | 20260506 01:37:40.743 | INFO | "malloc" : " 14 GB", | 20260506 01:37:40.743 | INFO | "pinned" : " 29 GB" | 20260506 01:37:40.743 | INFO | } | 20260506 01:37:40.743 | INFO | Perf report for the last 5.13377 seconds: | 20260506 01:37:40.743 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:37:40.743 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:40.743 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:40.743 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:40.743 | INFO | ANALYSIS : 61.3%, 3.0, 95.2ms, 0 | 20260506 01:37:40.743 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:40.743 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:40.743 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 01:37:40.743 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:40.743 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:40.744 | INFO | Uploaded 36GiB to GPU in 2.08734s (17.2468GiB/s) | 20260506 01:37:40.744 | INFO | Downloaded 34GiB to GPU in 2.02265s (16.8097GiB/s) | 20260506 01:37:45.884 | INFO | Analyzing chunk frames = [7927296,7927808) | 20260506 01:37:45.884 | INFO | MemInfo: { | 20260506 01:37:45.884 | INFO | "gpu" : " 17 GB", | 20260506 01:37:45.884 | INFO | "huge" : "", | 20260506 01:37:45.884 | INFO | "huge_shared" : " 27 GB", | 20260506 01:37:45.884 | INFO | "malloc" : " 14 GB", | 20260506 01:37:45.884 | INFO | "pinned" : " 29 GB" | 20260506 01:37:45.884 | INFO | } | 20260506 01:37:45.884 | INFO | Perf report for the last 5.14107 seconds: | 20260506 01:37:45.884 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:37:45.884 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:45.884 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:45.884 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:45.884 | INFO | ANALYSIS : 61.6%, 3.0, 95.7ms, 0 | 20260506 01:37:45.884 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:45.884 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:45.885 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 01:37:45.885 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:45.885 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:45.885 | INFO | Uploaded 36GiB to GPU in 2.13485s (16.863GiB/s) | 20260506 01:37:45.885 | INFO | Downloaded 34GiB to GPU in 2.02559s (16.7852GiB/s) | 20260506 01:37:51.027 | INFO | Analyzing chunk frames = [7927808,7928320) | 20260506 01:37:51.027 | INFO | MemInfo: { | 20260506 01:37:51.027 | INFO | "gpu" : " 17 GB", | 20260506 01:37:51.027 | INFO | "huge" : "", | 20260506 01:37:51.027 | INFO | "huge_shared" : " 27 GB", | 20260506 01:37:51.027 | INFO | "malloc" : " 14 GB", | 20260506 01:37:51.027 | INFO | "pinned" : " 29 GB" | 20260506 01:37:51.027 | INFO | } | 20260506 01:37:51.027 | INFO | Perf report for the last 5.14304 seconds: | 20260506 01:37:51.027 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:37:51.027 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:51.027 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:51.027 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:51.028 | INFO | ANALYSIS : 60.6%, 3.0, 94.7ms, 0 | 20260506 01:37:51.028 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:51.028 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:51.028 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 01:37:51.028 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:51.028 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:51.028 | INFO | Uploaded 36GiB to GPU in 2.0822s (17.2894GiB/s) | 20260506 01:37:51.028 | INFO | Downloaded 34GiB to GPU in 2.02692s (16.7742GiB/s) | 20260506 01:37:56.164 | INFO | Analyzing chunk frames = [7928320,7928832) | 20260506 01:37:56.164 | INFO | MemInfo: { | 20260506 01:37:56.164 | INFO | "gpu" : " 17 GB", | 20260506 01:37:56.164 | INFO | "huge" : "", | 20260506 01:37:56.164 | INFO | "huge_shared" : " 27 GB", | 20260506 01:37:56.164 | INFO | "malloc" : " 14 GB", | 20260506 01:37:56.164 | INFO | "pinned" : " 29 GB" | 20260506 01:37:56.164 | INFO | } | 20260506 01:37:56.164 | INFO | Perf report for the last 5.13665 seconds: | 20260506 01:37:56.164 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:37:56.164 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:56.164 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:56.164 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:56.164 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 01:37:56.164 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:56.164 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:56.164 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:37:56.164 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:56.164 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:37:56.165 | INFO | Uploaded 36GiB to GPU in 2.08726s (17.2475GiB/s) | 20260506 01:37:56.165 | INFO | Downloaded 34GiB to GPU in 2.03932s (16.6722GiB/s) | 20260506 01:38:01.301 | INFO | Analyzing chunk frames = [7928832,7929344) | 20260506 01:38:01.301 | INFO | MemInfo: { | 20260506 01:38:01.301 | INFO | "gpu" : " 17 GB", | 20260506 01:38:01.301 | INFO | "huge" : "", | 20260506 01:38:01.301 | INFO | "huge_shared" : " 27 GB", | 20260506 01:38:01.301 | INFO | "malloc" : " 14 GB", | 20260506 01:38:01.302 | INFO | "pinned" : " 29 GB" | 20260506 01:38:01.302 | INFO | } | 20260506 01:38:01.302 | INFO | Perf report for the last 5.13749 seconds: | 20260506 01:38:01.302 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:38:01.302 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:01.302 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:01.302 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:01.302 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260506 01:38:01.302 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:01.302 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:01.302 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 01:38:01.302 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:01.302 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:01.302 | INFO | Uploaded 36GiB to GPU in 2.11248s (17.0416GiB/s) | 20260506 01:38:01.302 | INFO | Downloaded 34GiB to GPU in 2.09179s (16.254GiB/s) | 20260506 01:38:06.438 | INFO | Analyzing chunk frames = [7929344,7929856) | 20260506 01:38:06.438 | INFO | MemInfo: { | 20260506 01:38:06.438 | INFO | "gpu" : " 17 GB", | 20260506 01:38:06.438 | INFO | "huge" : "", | 20260506 01:38:06.438 | INFO | "huge_shared" : " 27 GB", | 20260506 01:38:06.439 | INFO | "malloc" : " 14 GB", | 20260506 01:38:06.439 | INFO | "pinned" : " 29 GB" | 20260506 01:38:06.439 | INFO | } | 20260506 01:38:06.439 | INFO | Perf report for the last 5.13707 seconds: | 20260506 01:38:06.439 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:38:06.439 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:06.439 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:06.439 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:06.439 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260506 01:38:06.439 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:06.439 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:06.439 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:38:06.439 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:06.439 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:06.439 | INFO | Uploaded 36GiB to GPU in 2.12517s (16.9398GiB/s) | 20260506 01:38:06.439 | INFO | Downloaded 34GiB to GPU in 2.01477s (16.8754GiB/s) | 20260506 01:38:06.439 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 01:38:06.439 | INFO | Total mem usage: 15.3 GiB | 20260506 01:38:06.439 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 01:38:06.439 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.19%) | 20260506 01:38:06.439 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 01:38:06.439 | INFO | High Water Breakdown: | 20260506 01:38:06.439 | INFO | 60.9% : 8.54 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 01:38:06.439 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 01:38:06.439 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 01:38:06.439 | INFO | ------------------------------------------ | 20260506 01:38:06.439 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:38:06.439 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 01:38:06.439 | INFO | Total mem usage: 26.5 GiB | 20260506 01:38:06.439 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 01:38:06.439 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 01:38:06.439 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 01:38:06.439 | INFO | High Water Breakdown: | 20260506 01:38:06.439 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:38:06.439 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:38:06.439 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:38:06.439 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:38:06.440 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:38:06.440 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:38:06.440 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 01:38:06.440 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 01:38:06.440 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:38:06.440 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 01:38:06.440 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 01:38:06.440 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 01:38:06.440 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 01:38:06.440 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 01:38:06.440 | INFO | ------------------------------------------ | 20260506 01:38:06.440 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:38:06.440 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 01:38:06.440 | INFO | Total mem usage: 25.6 GiB | 20260506 01:38:06.440 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 01:38:06.440 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 01:38:06.440 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 01:38:06.440 | INFO | High Water Breakdown: | 20260506 01:38:06.440 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 01:38:06.440 | INFO | ------------------------------------------ | 20260506 01:38:06.440 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:38:06.440 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 01:38:06.440 | INFO | Total mem usage: 15.7 GiB | 20260506 01:38:06.440 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 01:38:06.440 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 01:38:06.440 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 01:38:06.440 | INFO | High Water Breakdown: | 20260506 01:38:06.440 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 01:38:06.440 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 01:38:06.440 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 01:38:06.440 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 01:38:06.440 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 01:38:06.440 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 01:38:06.440 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 01:38:06.440 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 01:38:06.440 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 01:38:06.440 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:38:06.440 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 01:38:06.440 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:38:06.440 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 01:38:06.440 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 01:38:06.440 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 01:38:06.440 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 01:38:06.440 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:38:06.440 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 01:38:06.440 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:38:06.440 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:38:06.440 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:38:06.440 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 01:38:06.440 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 01:38:06.440 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:38:06.440 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 01:38:06.440 | INFO | ------------------------------------------ | 20260506 01:38:06.440 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:38:06.440 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 01:38:06.440 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:38:06.440 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:38:06.440 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.37e+01, 1.35e+01 / 5.80e-01 | 20260506 01:38:06.440 | INFO | PerfReport: 0.29%, AggregateStats -- 9.18e-02 / 1.05e-01, 1.03e-02 / 4.33e-03 | 20260506 01:38:06.440 | INFO | PerfReport: 11.82%, Baseline -- 3.70e+00 / 4.51e+00, 5.67e-01 / 2.41e-01 | 20260506 01:38:06.440 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.51e+00, 1.89e-01 / 1.06e-01 | 20260506 01:38:06.440 | INFO | PerfReport: 15.97%, DME -- 3.99e+01 / 4.36e+01, 3.71e+00 / 7.16e-01 | 20260506 01:38:06.440 | INFO | PerfReport: 57.49%, FrameLabeling -- 1.80e+01 / 1.97e+01, 1.50e+00 / 1.08e+00 | 20260506 01:38:06.440 | INFO | PerfReport: 7.06%, PulseAccumulating -- 2.21e+00 / 2.69e+00, 4.17e-01 / 2.67e-01 | 20260506 01:38:06.440 | INFO | PerfReport: 3.04%, Metrics -- 9.49e-01 / 1.21e+00, 2.54e-01 / 1.04e-01 | 20260506 01:38:06.440 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.29e-01 / 3.85e-01, 5.54e-02 / 3.25e-02 | 20260506 01:38:06.440 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 01:38:06.440 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:38:06.440 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:38:06.440 | INFO | PerfReport: Full Execution -- 9.80e+01 / -nan, 2.42e+01 / 0.00e+00 | 20260506 01:38:06.440 | INFO | PerfReport: 22.37%, Upload -- 2.19e+01 / 0.00e+00, 7.49e+00 / 0.00e+00 | 20260506 01:38:06.440 | INFO | PerfReport: 21.89%, Download -- 2.15e+01 / 0.00e+00, 7.61e+00 / 0.00e+00 | 20260506 01:38:06.440 | INFO | PerfReport: 31.88%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 01:38:06.440 | INFO | PerfReport: 2.14%, LockUpload -- 2.09e+00 / 0.00e+00, 8.07e+00 / 0.00e+00 | 20260506 01:38:06.440 | INFO | PerfReport: 1.27%, LockDownload -- 1.24e+00 / 0.00e+00, 2.80e+00 / 0.00e+00 | 20260506 01:38:06.440 | INFO | PerfReport: 20.44%, LockCompute -- 2.00e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 01:38:06.440 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.96e-02 / 0.00e+00 | 20260506 01:38:06.565 | INFO | Starting a round of Baz IO | 20260506 01:38:09.696 | INFO | Populating RTMetrics report | 20260506 01:38:09.696 | INFO | RTMetrics report populated | 20260506 01:38:09.696 | INFO | Opening tmp RTMetrics json file | 20260506 01:38:09.698 | INFO | Serializing RTMetrics to json | 20260506 01:38:09.700 | INFO | Serialization done, swapping tmp file now... | 20260506 01:38:09.700 | INFO | RTMetrics done | 20260506 01:38:09.773 | INFO | Finished a round of Baz IO: Wrote 1.819 GiB in 3.208 seconds | 20260506 01:38:09.773 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.31% / 0.00% / 78.69% | 20260506 01:38:09.773 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0785708% / 0% / 9.53674e-05% | 20260506 01:38:09.773 | INFO | Baz IO overall: | 20260506 01:38:09.773 | INFO | BytesWritten : 4468342470211 | 20260506 01:38:09.773 | INFO | NumEvents : 2099821744721 | 20260506 01:38:11.584 | INFO | Analyzing chunk frames = [7929856,7930368) | 20260506 01:38:11.584 | INFO | MemInfo: { | 20260506 01:38:11.584 | INFO | "gpu" : " 17 GB", | 20260506 01:38:11.584 | INFO | "huge" : "", | 20260506 01:38:11.584 | INFO | "huge_shared" : " 27 GB", | 20260506 01:38:11.584 | INFO | "malloc" : " 14 GB", | 20260506 01:38:11.584 | INFO | "pinned" : " 29 GB" | 20260506 01:38:11.584 | INFO | } | 20260506 01:38:11.584 | INFO | Perf report for the last 5.14502 seconds: | 20260506 01:38:11.584 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:38:11.584 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:11.584 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:11.584 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:11.584 | INFO | ANALYSIS : 62.3%, 3.0, 97.4ms, 0 | 20260506 01:38:11.584 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:11.584 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260506 01:38:11.584 | INFO | PRE_HQ : 59.5%, 1.7, 53.7ms, 0 | 20260506 01:38:11.584 | INFO | BAZWRITER : 61.6%, 3.5, 110.6ms, 0 | 20260506 01:38:11.584 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:38:11.584 | INFO | Uploaded 36GiB to GPU in 2.18616s (16.4673GiB/s) | 20260506 01:38:11.584 | INFO | Downloaded 38GiB to GPU in 2.23742s (16.9839GiB/s) | 20260506 01:38:16.723 | INFO | Analyzing chunk frames = [7930368,7930880) | 20260506 01:38:16.723 | INFO | MemInfo: { | 20260506 01:38:16.723 | INFO | "gpu" : " 17 GB", | 20260506 01:38:16.723 | INFO | "huge" : "", | 20260506 01:38:16.723 | INFO | "huge_shared" : " 27 GB", | 20260506 01:38:16.723 | INFO | "malloc" : " 14 GB", | 20260506 01:38:16.723 | INFO | "pinned" : " 29 GB" | 20260506 01:38:16.723 | INFO | } | 20260506 01:38:16.723 | INFO | Perf report for the last 5.13895 seconds: | 20260506 01:38:16.723 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:38:16.723 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:16.723 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:16.723 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:16.723 | INFO | ANALYSIS : 60.8%, 3.0, 94.9ms, 0 | 20260506 01:38:16.723 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:16.723 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:16.723 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:38:16.723 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:16.723 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:16.723 | INFO | Uploaded 36GiB to GPU in 1.94844s (18.4763GiB/s) | 20260506 01:38:16.723 | INFO | Downloaded 34GiB to GPU in 1.97663s (17.201GiB/s) | 20260506 01:38:21.856 | INFO | Analyzing chunk frames = [7930880,7931392) | 20260506 01:38:21.856 | INFO | MemInfo: { | 20260506 01:38:21.856 | INFO | "gpu" : " 17 GB", | 20260506 01:38:21.856 | INFO | "huge" : "", | 20260506 01:38:21.856 | INFO | "huge_shared" : " 27 GB", | 20260506 01:38:21.856 | INFO | "malloc" : " 14 GB", | 20260506 01:38:21.856 | INFO | "pinned" : " 29 GB" | 20260506 01:38:21.856 | INFO | } | 20260506 01:38:21.857 | INFO | Perf report for the last 5.13378 seconds: | 20260506 01:38:21.857 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:38:21.857 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:21.857 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:21.857 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:21.857 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 01:38:21.857 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:21.857 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:21.857 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 01:38:21.857 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:21.857 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:21.857 | INFO | Uploaded 36GiB to GPU in 2.09001s (17.2248GiB/s) | 20260506 01:38:21.857 | INFO | Downloaded 34GiB to GPU in 2.02297s (16.8069GiB/s) | 20260506 01:38:26.996 | INFO | Analyzing chunk frames = [7931392,7931904) | 20260506 01:38:26.996 | INFO | MemInfo: { | 20260506 01:38:26.996 | INFO | "gpu" : " 17 GB", | 20260506 01:38:26.996 | INFO | "huge" : "", | 20260506 01:38:26.996 | INFO | "huge_shared" : " 27 GB", | 20260506 01:38:26.996 | INFO | "malloc" : " 14 GB", | 20260506 01:38:26.996 | INFO | "pinned" : " 29 GB" | 20260506 01:38:26.996 | INFO | } | 20260506 01:38:26.996 | INFO | Perf report for the last 5.13983 seconds: | 20260506 01:38:26.996 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:38:26.996 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:26.996 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:26.996 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:26.996 | INFO | ANALYSIS : 61.6%, 3.0, 95.6ms, 0 | 20260506 01:38:26.996 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:26.996 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:26.996 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:38:26.997 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:26.997 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:26.997 | INFO | Uploaded 36GiB to GPU in 2.14172s (16.8089GiB/s) | 20260506 01:38:26.997 | INFO | Downloaded 34GiB to GPU in 2.02218s (16.8136GiB/s) | 20260506 01:38:32.142 | INFO | Analyzing chunk frames = [7931904,7932416) | 20260506 01:38:32.142 | INFO | MemInfo: { | 20260506 01:38:32.142 | INFO | "gpu" : " 17 GB", | 20260506 01:38:32.142 | INFO | "huge" : "", | 20260506 01:38:32.142 | INFO | "huge_shared" : " 27 GB", | 20260506 01:38:32.142 | INFO | "malloc" : " 14 GB", | 20260506 01:38:32.142 | INFO | "pinned" : " 29 GB" | 20260506 01:38:32.142 | INFO | } | 20260506 01:38:32.142 | INFO | Perf report for the last 5.14559 seconds: | 20260506 01:38:32.142 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:38:32.142 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:32.142 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:32.142 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:32.142 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 01:38:32.142 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:32.142 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:32.142 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:38:32.142 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:32.142 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:32.142 | INFO | Uploaded 36GiB to GPU in 2.08869s (17.2357GiB/s) | 20260506 01:38:32.142 | INFO | Downloaded 34GiB to GPU in 2.04532s (16.6234GiB/s) | 20260506 01:38:37.280 | INFO | Analyzing chunk frames = [7932416,7932928) | 20260506 01:38:37.281 | INFO | MemInfo: { | 20260506 01:38:37.281 | INFO | "gpu" : " 17 GB", | 20260506 01:38:37.281 | INFO | "huge" : "", | 20260506 01:38:37.281 | INFO | "huge_shared" : " 27 GB", | 20260506 01:38:37.281 | INFO | "malloc" : " 14 GB", | 20260506 01:38:37.281 | INFO | "pinned" : " 29 GB" | 20260506 01:38:37.281 | INFO | } | 20260506 01:38:37.281 | INFO | Perf report for the last 5.13879 seconds: | 20260506 01:38:37.281 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:38:37.281 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:37.281 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:37.281 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:37.281 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 01:38:37.281 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:37.281 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:37.281 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 01:38:37.281 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:37.281 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:37.281 | INFO | Uploaded 36GiB to GPU in 2.09618s (17.1741GiB/s) | 20260506 01:38:37.281 | INFO | Downloaded 34GiB to GPU in 2.03404s (16.7155GiB/s) | 20260506 01:38:42.422 | INFO | Analyzing chunk frames = [7932928,7933440) | 20260506 01:38:42.422 | INFO | MemInfo: { | 20260506 01:38:42.422 | INFO | "gpu" : " 17 GB", | 20260506 01:38:42.422 | INFO | "huge" : "", | 20260506 01:38:42.422 | INFO | "huge_shared" : " 27 GB", | 20260506 01:38:42.422 | INFO | "malloc" : " 14 GB", | 20260506 01:38:42.422 | INFO | "pinned" : " 29 GB" | 20260506 01:38:42.422 | INFO | } | 20260506 01:38:42.422 | INFO | Perf report for the last 5.14138 seconds: | 20260506 01:38:42.422 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:38:42.422 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:42.422 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:42.422 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:42.422 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260506 01:38:42.422 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:42.422 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:42.422 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 01:38:42.422 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:42.422 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:42.422 | INFO | Uploaded 36GiB to GPU in 2.03502s (17.6902GiB/s) | 20260506 01:38:42.422 | INFO | Downloaded 34GiB to GPU in 2.06544s (16.4614GiB/s) | 20260506 01:38:47.558 | INFO | Analyzing chunk frames = [7933440,7933952) | 20260506 01:38:47.558 | INFO | MemInfo: { | 20260506 01:38:47.558 | INFO | "gpu" : " 17 GB", | 20260506 01:38:47.558 | INFO | "huge" : "", | 20260506 01:38:47.558 | INFO | "huge_shared" : " 27 GB", | 20260506 01:38:47.559 | INFO | "malloc" : " 14 GB", | 20260506 01:38:47.559 | INFO | "pinned" : " 29 GB" | 20260506 01:38:47.559 | INFO | } | 20260506 01:38:47.559 | INFO | Perf report for the last 5.13648 seconds: | 20260506 01:38:47.559 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:38:47.559 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:47.559 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:47.559 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:47.559 | INFO | ANALYSIS : 61.5%, 3.0, 96.0ms, 0 | 20260506 01:38:47.559 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:47.559 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:47.559 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 01:38:47.559 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:47.559 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:47.559 | INFO | Uploaded 36GiB to GPU in 2.05941s (17.4807GiB/s) | 20260506 01:38:47.559 | INFO | Downloaded 34GiB to GPU in 2.02672s (16.7759GiB/s) | 20260506 01:38:47.676 | INFO | Starting a round of Baz IO | 20260506 01:38:50.820 | INFO | Populating RTMetrics report | 20260506 01:38:50.820 | INFO | RTMetrics report populated | 20260506 01:38:50.820 | INFO | Opening tmp RTMetrics json file | 20260506 01:38:50.821 | INFO | Serializing RTMetrics to json | 20260506 01:38:50.822 | INFO | Serialization done, swapping tmp file now... | 20260506 01:38:50.823 | INFO | RTMetrics done | 20260506 01:38:50.895 | INFO | Finished a round of Baz IO: Wrote 1.818 GiB in 3.218 seconds | 20260506 01:38:50.895 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.31% / 0.00% / 78.69% | 20260506 01:38:50.895 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0778675% / 0% / 7.94729e-05% | 20260506 01:38:50.895 | INFO | Baz IO overall: | 20260506 01:38:50.895 | INFO | BytesWritten : 4470294650503 | 20260506 01:38:50.895 | INFO | NumEvents : 2100710577507 | 20260506 01:38:52.691 | INFO | Analyzing chunk frames = [7933952,7934464) | 20260506 01:38:52.691 | INFO | MemInfo: { | 20260506 01:38:52.691 | INFO | "gpu" : " 17 GB", | 20260506 01:38:52.691 | INFO | "huge" : "", | 20260506 01:38:52.691 | INFO | "huge_shared" : " 27 GB", | 20260506 01:38:52.691 | INFO | "malloc" : " 14 GB", | 20260506 01:38:52.691 | INFO | "pinned" : " 29 GB" | 20260506 01:38:52.691 | INFO | } | 20260506 01:38:52.691 | INFO | Perf report for the last 5.13241 seconds: | 20260506 01:38:52.691 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:38:52.691 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:52.691 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:52.691 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:52.691 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260506 01:38:52.691 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:52.691 | INFO | RT_METRICS : 43.2%, 1.0, 22.6ms, 0 | 20260506 01:38:52.691 | INFO | PRE_HQ : 59.6%, 1.7, 53.6ms, 0 | 20260506 01:38:52.691 | INFO | BAZWRITER : 61.8%, 3.5, 109.8ms, 0 | 20260506 01:38:52.691 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:38:52.691 | INFO | Uploaded 36GiB to GPU in 2.17709s (16.5359GiB/s) | 20260506 01:38:52.692 | INFO | Downloaded 38GiB to GPU in 2.23699s (16.9871GiB/s) | 20260506 01:38:57.840 | INFO | Analyzing chunk frames = [7934464,7934976) | 20260506 01:38:57.840 | INFO | MemInfo: { | 20260506 01:38:57.840 | INFO | "gpu" : " 17 GB", | 20260506 01:38:57.840 | INFO | "huge" : "", | 20260506 01:38:57.840 | INFO | "huge_shared" : " 27 GB", | 20260506 01:38:57.840 | INFO | "malloc" : " 14 GB", | 20260506 01:38:57.840 | INFO | "pinned" : " 29 GB" | 20260506 01:38:57.840 | INFO | } | 20260506 01:38:57.840 | INFO | Perf report for the last 5.14901 seconds: | 20260506 01:38:57.840 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:38:57.840 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:57.840 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:57.840 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:57.840 | INFO | ANALYSIS : 61.0%, 3.0, 95.4ms, 0 | 20260506 01:38:57.840 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:57.840 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:57.840 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 01:38:57.840 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:57.840 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:38:57.840 | INFO | Uploaded 36GiB to GPU in 1.97117s (18.2633GiB/s) | 20260506 01:38:57.840 | INFO | Downloaded 34GiB to GPU in 1.99642s (17.0305GiB/s) | 20260506 01:39:02.983 | INFO | Analyzing chunk frames = [7934976,7935488) | 20260506 01:39:02.983 | INFO | MemInfo: { | 20260506 01:39:02.983 | INFO | "gpu" : " 17 GB", | 20260506 01:39:02.983 | INFO | "huge" : "", | 20260506 01:39:02.984 | INFO | "huge_shared" : " 27 GB", | 20260506 01:39:02.984 | INFO | "malloc" : " 14 GB", | 20260506 01:39:02.984 | INFO | "pinned" : " 29 GB" | 20260506 01:39:02.984 | INFO | } | 20260506 01:39:02.984 | INFO | Perf report for the last 5.14349 seconds: | 20260506 01:39:02.984 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:39:02.984 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:02.984 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:02.984 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:02.984 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 01:39:02.984 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:02.984 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:02.984 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 01:39:02.984 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:02.984 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:02.984 | INFO | Uploaded 36GiB to GPU in 2.08052s (17.3034GiB/s) | 20260506 01:39:02.984 | INFO | Downloaded 34GiB to GPU in 2.01471s (16.8759GiB/s) | 20260506 01:39:08.115 | INFO | Analyzing chunk frames = [7935488,7936000) | 20260506 01:39:08.115 | INFO | MemInfo: { | 20260506 01:39:08.115 | INFO | "gpu" : " 17 GB", | 20260506 01:39:08.115 | INFO | "huge" : "", | 20260506 01:39:08.115 | INFO | "huge_shared" : " 27 GB", | 20260506 01:39:08.115 | INFO | "malloc" : " 14 GB", | 20260506 01:39:08.115 | INFO | "pinned" : " 29 GB" | 20260506 01:39:08.115 | INFO | } | 20260506 01:39:08.115 | INFO | Perf report for the last 5.13145 seconds: | 20260506 01:39:08.115 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:39:08.115 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:08.115 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:08.115 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:08.115 | INFO | ANALYSIS : 61.2%, 3.0, 95.0ms, 0 | 20260506 01:39:08.115 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:08.115 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:08.115 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:39:08.115 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:08.115 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:08.115 | INFO | Uploaded 36GiB to GPU in 2.09657s (17.1709GiB/s) | 20260506 01:39:08.115 | INFO | Downloaded 34GiB to GPU in 2.02092s (16.824GiB/s) | 20260506 01:39:13.251 | INFO | Analyzing chunk frames = [7936000,7936512) | 20260506 01:39:13.251 | INFO | MemInfo: { | 20260506 01:39:13.251 | INFO | "gpu" : " 17 GB", | 20260506 01:39:13.251 | INFO | "huge" : "", | 20260506 01:39:13.251 | INFO | "huge_shared" : " 27 GB", | 20260506 01:39:13.251 | INFO | "malloc" : " 14 GB", | 20260506 01:39:13.251 | INFO | "pinned" : " 29 GB" | 20260506 01:39:13.251 | INFO | } | 20260506 01:39:13.251 | INFO | Perf report for the last 5.13605 seconds: | 20260506 01:39:13.251 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:39:13.251 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:13.251 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:13.251 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:13.251 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 01:39:13.251 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:13.251 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:13.251 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 01:39:13.251 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:13.251 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:13.251 | INFO | Uploaded 36GiB to GPU in 2.09446s (17.1882GiB/s) | 20260506 01:39:13.251 | INFO | Downloaded 34GiB to GPU in 2.04188s (16.6513GiB/s) | 20260506 01:39:18.391 | INFO | Analyzing chunk frames = [7936512,7937024) | 20260506 01:39:18.391 | INFO | MemInfo: { | 20260506 01:39:18.391 | INFO | "gpu" : " 17 GB", | 20260506 01:39:18.391 | INFO | "huge" : "", | 20260506 01:39:18.391 | INFO | "huge_shared" : " 27 GB", | 20260506 01:39:18.392 | INFO | "malloc" : " 14 GB", | 20260506 01:39:18.392 | INFO | "pinned" : " 29 GB" | 20260506 01:39:18.392 | INFO | } | 20260506 01:39:18.392 | INFO | Perf report for the last 5.14052 seconds: | 20260506 01:39:18.392 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:39:18.392 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:18.392 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:18.392 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:18.392 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 01:39:18.392 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:18.392 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:18.392 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:39:18.392 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:18.392 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:18.392 | INFO | Uploaded 36GiB to GPU in 2.07572s (17.3434GiB/s) | 20260506 01:39:18.392 | INFO | Downloaded 34GiB to GPU in 2.04489s (16.6268GiB/s) | 20260506 01:39:23.533 | INFO | Analyzing chunk frames = [7937024,7937536) | 20260506 01:39:23.533 | INFO | MemInfo: { | 20260506 01:39:23.533 | INFO | "gpu" : " 17 GB", | 20260506 01:39:23.533 | INFO | "huge" : "", | 20260506 01:39:23.533 | INFO | "huge_shared" : " 27 GB", | 20260506 01:39:23.533 | INFO | "malloc" : " 14 GB", | 20260506 01:39:23.533 | INFO | "pinned" : " 29 GB" | 20260506 01:39:23.533 | INFO | } | 20260506 01:39:23.533 | INFO | Perf report for the last 5.14156 seconds: | 20260506 01:39:23.533 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:39:23.533 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:23.533 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:23.533 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:23.533 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260506 01:39:23.533 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:23.533 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:23.533 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 01:39:23.533 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:23.533 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:23.534 | INFO | Uploaded 36GiB to GPU in 2.09326s (17.198GiB/s) | 20260506 01:39:23.534 | INFO | Downloaded 34GiB to GPU in 2.07375s (16.3954GiB/s) | 20260506 01:39:28.671 | INFO | Analyzing chunk frames = [7937536,7938048) | 20260506 01:39:28.671 | INFO | MemInfo: { | 20260506 01:39:28.671 | INFO | "gpu" : " 17 GB", | 20260506 01:39:28.672 | INFO | "huge" : "", | 20260506 01:39:28.672 | INFO | "huge_shared" : " 27 GB", | 20260506 01:39:28.672 | INFO | "malloc" : " 14 GB", | 20260506 01:39:28.672 | INFO | "pinned" : " 29 GB" | 20260506 01:39:28.672 | INFO | } | 20260506 01:39:28.672 | INFO | Perf report for the last 5.13839 seconds: | 20260506 01:39:28.672 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:39:28.672 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:28.672 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:28.672 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:28.672 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 01:39:28.672 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:28.672 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:28.672 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 01:39:28.672 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:28.672 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:28.672 | INFO | Uploaded 36GiB to GPU in 2.0458s (17.597GiB/s) | 20260506 01:39:28.672 | INFO | Downloaded 34GiB to GPU in 2.02268s (16.8094GiB/s) | 20260506 01:39:28.672 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 01:39:28.672 | INFO | Total mem usage: 15.3 GiB | 20260506 01:39:28.672 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 01:39:28.672 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.2%) | 20260506 01:39:28.672 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 01:39:28.672 | INFO | High Water Breakdown: | 20260506 01:39:28.672 | INFO | 60.9% : 8.54 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 01:39:28.672 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 01:39:28.672 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 01:39:28.672 | INFO | ------------------------------------------ | 20260506 01:39:28.672 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:39:28.672 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 01:39:28.672 | INFO | Total mem usage: 26.5 GiB | 20260506 01:39:28.672 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 01:39:28.672 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 01:39:28.672 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260506 01:39:28.672 | INFO | High Water Breakdown: | 20260506 01:39:28.672 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:39:28.672 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:39:28.672 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:39:28.672 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:39:28.672 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:39:28.672 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:39:28.672 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 01:39:28.672 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 01:39:28.672 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:39:28.672 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 01:39:28.672 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 01:39:28.672 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 01:39:28.673 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 01:39:28.673 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 01:39:28.673 | INFO | ------------------------------------------ | 20260506 01:39:28.673 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:39:28.673 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 01:39:28.673 | INFO | Total mem usage: 25.6 GiB | 20260506 01:39:28.673 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 01:39:28.673 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 01:39:28.673 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 01:39:28.673 | INFO | High Water Breakdown: | 20260506 01:39:28.673 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 01:39:28.673 | INFO | ------------------------------------------ | 20260506 01:39:28.673 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:39:28.673 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 01:39:28.673 | INFO | Total mem usage: 15.7 GiB | 20260506 01:39:28.673 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 01:39:28.673 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 01:39:28.673 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 01:39:28.673 | INFO | High Water Breakdown: | 20260506 01:39:28.673 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 01:39:28.673 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 01:39:28.673 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 01:39:28.673 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 01:39:28.673 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 01:39:28.673 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 01:39:28.673 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 01:39:28.673 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 01:39:28.673 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 01:39:28.673 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:39:28.673 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 01:39:28.673 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:39:28.673 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 01:39:28.673 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 01:39:28.673 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 01:39:28.673 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 01:39:28.673 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:39:28.673 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 01:39:28.673 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:39:28.673 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:39:28.673 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:39:28.673 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 01:39:28.673 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 01:39:28.673 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:39:28.673 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 01:39:28.673 | INFO | ------------------------------------------ | 20260506 01:39:28.673 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:39:28.673 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 01:39:28.673 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:39:28.673 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:39:28.673 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.39e+01, 1.35e+01 / 7.79e-01 | 20260506 01:39:28.673 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.04e-01, 1.03e-02 / 4.15e-03 | 20260506 01:39:28.673 | INFO | PerfReport: 11.86%, Baseline -- 3.71e+00 / 4.49e+00, 5.79e-01 / 2.16e-01 | 20260506 01:39:28.673 | INFO | PerfReport: 3.30%, Histogram -- 1.03e+00 / 1.53e+00, 1.95e-01 / 8.54e-02 | 20260506 01:39:28.673 | INFO | PerfReport: 15.99%, DME -- 4.01e+01 / 4.41e+01, 3.76e+00 / 1.04e+00 | 20260506 01:39:28.673 | INFO | PerfReport: 57.43%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.52e+00 / 1.12e+00 | 20260506 01:39:28.673 | INFO | PerfReport: 7.04%, PulseAccumulating -- 2.20e+00 / 2.71e+00, 4.16e-01 / 2.59e-01 | 20260506 01:39:28.673 | INFO | PerfReport: 3.03%, Metrics -- 9.50e-01 / 1.23e+00, 2.56e-01 / 1.15e-01 | 20260506 01:39:28.673 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.28e-01 / 3.74e-01, 5.53e-02 / 2.26e-02 | 20260506 01:39:28.673 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 01:39:28.673 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:39:28.673 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:39:28.673 | INFO | PerfReport: Full Execution -- 9.78e+01 / -nan, 2.41e+01 / 0.00e+00 | 20260506 01:39:28.673 | INFO | PerfReport: 22.15%, Upload -- 2.17e+01 / 0.00e+00, 7.22e+00 / 0.00e+00 | 20260506 01:39:28.673 | INFO | PerfReport: 21.89%, Download -- 2.14e+01 / 0.00e+00, 7.54e+00 / 0.00e+00 | 20260506 01:39:28.673 | INFO | PerfReport: 32.02%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 01:39:28.673 | INFO | PerfReport: 2.04%, LockUpload -- 2.00e+00 / 0.00e+00, 7.71e+00 / 0.00e+00 | 20260506 01:39:28.673 | INFO | PerfReport: 1.29%, LockDownload -- 1.26e+00 / 0.00e+00, 2.89e+00 / 0.00e+00 | 20260506 01:39:28.673 | INFO | PerfReport: 20.60%, LockCompute -- 2.01e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260506 01:39:28.673 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 2.16e-02 / 0.00e+00 | 20260506 01:39:28.789 | INFO | Starting a round of Baz IO | 20260506 01:39:31.938 | INFO | Populating RTMetrics report | 20260506 01:39:31.938 | INFO | RTMetrics report populated | 20260506 01:39:31.938 | INFO | Opening tmp RTMetrics json file | 20260506 01:39:31.939 | INFO | Serializing RTMetrics to json | 20260506 01:39:31.940 | INFO | Serialization done, swapping tmp file now... | 20260506 01:39:31.941 | INFO | RTMetrics done | 20260506 01:39:32.013 | INFO | Finished a round of Baz IO: Wrote 1.817 GiB in 3.223 seconds | 20260506 01:39:32.013 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.31% / 0.00% / 78.69% | 20260506 01:39:32.013 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0782331% / 0% / 7.94729e-05% | 20260506 01:39:32.013 | INFO | Baz IO overall: | 20260506 01:39:32.013 | INFO | BytesWritten : 4472245696640 | 20260506 01:39:32.013 | INFO | NumEvents : 2101598979394 | 20260506 01:39:33.806 | INFO | Analyzing chunk frames = [7938048,7938560) | 20260506 01:39:33.806 | INFO | MemInfo: { | 20260506 01:39:33.806 | INFO | "gpu" : " 17 GB", | 20260506 01:39:33.806 | INFO | "huge" : "", | 20260506 01:39:33.806 | INFO | "huge_shared" : " 27 GB", | 20260506 01:39:33.806 | INFO | "malloc" : " 14 GB", | 20260506 01:39:33.806 | INFO | "pinned" : " 29 GB" | 20260506 01:39:33.806 | INFO | } | 20260506 01:39:33.806 | INFO | Perf report for the last 5.13455 seconds: | 20260506 01:39:33.806 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:39:33.806 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:33.806 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:33.806 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:33.806 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260506 01:39:33.806 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:33.806 | INFO | RT_METRICS : 43.2%, 1.0, 22.6ms, 0 | 20260506 01:39:33.806 | INFO | PRE_HQ : 59.7%, 1.7, 53.9ms, 0 | 20260506 01:39:33.806 | INFO | BAZWRITER : 61.8%, 3.5, 109.5ms, 0 | 20260506 01:39:33.806 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:39:33.807 | INFO | Uploaded 36GiB to GPU in 2.16148s (16.6553GiB/s) | 20260506 01:39:33.807 | INFO | Downloaded 38GiB to GPU in 2.23923s (16.9701GiB/s) | 20260506 01:39:38.946 | INFO | Analyzing chunk frames = [7938560,7939072) | 20260506 01:39:38.946 | INFO | MemInfo: { | 20260506 01:39:38.946 | INFO | "gpu" : " 17 GB", | 20260506 01:39:38.946 | INFO | "huge" : "", | 20260506 01:39:38.946 | INFO | "huge_shared" : " 27 GB", | 20260506 01:39:38.946 | INFO | "malloc" : " 14 GB", | 20260506 01:39:38.946 | INFO | "pinned" : " 29 GB" | 20260506 01:39:38.946 | INFO | } | 20260506 01:39:38.946 | INFO | Perf report for the last 5.13987 seconds: | 20260506 01:39:38.946 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:39:38.946 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:38.946 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:38.946 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:38.946 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260506 01:39:38.946 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:38.946 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:38.946 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 01:39:38.946 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:38.946 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:38.946 | INFO | Uploaded 36GiB to GPU in 2.10948s (17.0658GiB/s) | 20260506 01:39:38.946 | INFO | Downloaded 34GiB to GPU in 2.04374s (16.6362GiB/s) | 20260506 01:39:44.095 | INFO | Analyzing chunk frames = [7939072,7939584) | 20260506 01:39:44.095 | INFO | MemInfo: { | 20260506 01:39:44.095 | INFO | "gpu" : " 17 GB", | 20260506 01:39:44.095 | INFO | "huge" : "", | 20260506 01:39:44.095 | INFO | "huge_shared" : " 27 GB", | 20260506 01:39:44.095 | INFO | "malloc" : " 14 GB", | 20260506 01:39:44.095 | INFO | "pinned" : " 29 GB" | 20260506 01:39:44.095 | INFO | } | 20260506 01:39:44.096 | INFO | Perf report for the last 5.14945 seconds: | 20260506 01:39:44.096 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:39:44.096 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:44.096 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:44.096 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:44.096 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 01:39:44.096 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:44.096 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:44.096 | INFO | PRE_HQ : 26.4%, 1.0, 13.8ms, 0 | 20260506 01:39:44.096 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:44.096 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:44.096 | INFO | Uploaded 36GiB to GPU in 2.07583s (17.3425GiB/s) | 20260506 01:39:44.096 | INFO | Downloaded 34GiB to GPU in 2.01435s (16.8789GiB/s) | 20260506 01:39:49.238 | INFO | Analyzing chunk frames = [7939584,7940096) | 20260506 01:39:49.238 | INFO | MemInfo: { | 20260506 01:39:49.238 | INFO | "gpu" : " 17 GB", | 20260506 01:39:49.238 | INFO | "huge" : "", | 20260506 01:39:49.238 | INFO | "huge_shared" : " 27 GB", | 20260506 01:39:49.238 | INFO | "malloc" : " 14 GB", | 20260506 01:39:49.238 | INFO | "pinned" : " 29 GB" | 20260506 01:39:49.238 | INFO | } | 20260506 01:39:49.238 | INFO | Perf report for the last 5.14243 seconds: | 20260506 01:39:49.238 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:39:49.238 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:49.238 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:49.238 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:49.238 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260506 01:39:49.238 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:49.238 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:49.238 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:39:49.238 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:49.238 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:49.238 | INFO | Uploaded 36GiB to GPU in 2.08554s (17.2617GiB/s) | 20260506 01:39:49.238 | INFO | Downloaded 34GiB to GPU in 2.01746s (16.8529GiB/s) | 20260506 01:39:54.371 | INFO | Analyzing chunk frames = [7940096,7940608) | 20260506 01:39:54.371 | INFO | MemInfo: { | 20260506 01:39:54.371 | INFO | "gpu" : " 17 GB", | 20260506 01:39:54.371 | INFO | "huge" : "", | 20260506 01:39:54.371 | INFO | "huge_shared" : " 27 GB", | 20260506 01:39:54.371 | INFO | "malloc" : " 14 GB", | 20260506 01:39:54.371 | INFO | "pinned" : " 29 GB" | 20260506 01:39:54.371 | INFO | } | 20260506 01:39:54.372 | INFO | Perf report for the last 5.13356 seconds: | 20260506 01:39:54.372 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:39:54.372 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:54.372 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:54.372 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:54.372 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260506 01:39:54.372 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:54.372 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:54.372 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:39:54.372 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:54.372 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:54.372 | INFO | Uploaded 36GiB to GPU in 2.09101s (17.2165GiB/s) | 20260506 01:39:54.372 | INFO | Downloaded 34GiB to GPU in 2.02681s (16.7751GiB/s) | 20260506 01:39:59.507 | INFO | Analyzing chunk frames = [7940608,7941120) | 20260506 01:39:59.507 | INFO | MemInfo: { | 20260506 01:39:59.507 | INFO | "gpu" : " 17 GB", | 20260506 01:39:59.507 | INFO | "huge" : "", | 20260506 01:39:59.508 | INFO | "huge_shared" : " 27 GB", | 20260506 01:39:59.508 | INFO | "malloc" : " 14 GB", | 20260506 01:39:59.508 | INFO | "pinned" : " 29 GB" | 20260506 01:39:59.508 | INFO | } | 20260506 01:39:59.508 | INFO | Perf report for the last 5.13607 seconds: | 20260506 01:39:59.508 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:39:59.508 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:59.508 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:59.508 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:59.508 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 01:39:59.508 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:59.508 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:59.508 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:39:59.508 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:59.508 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:39:59.508 | INFO | Uploaded 36GiB to GPU in 2.08664s (17.2526GiB/s) | 20260506 01:39:59.508 | INFO | Downloaded 34GiB to GPU in 2.04161s (16.6535GiB/s) | 20260506 01:40:04.646 | INFO | Analyzing chunk frames = [7941120,7941632) | 20260506 01:40:04.646 | INFO | MemInfo: { | 20260506 01:40:04.646 | INFO | "gpu" : " 17 GB", | 20260506 01:40:04.646 | INFO | "huge" : "", | 20260506 01:40:04.646 | INFO | "huge_shared" : " 27 GB", | 20260506 01:40:04.646 | INFO | "malloc" : " 14 GB", | 20260506 01:40:04.646 | INFO | "pinned" : " 29 GB" | 20260506 01:40:04.646 | INFO | } | 20260506 01:40:04.646 | INFO | Perf report for the last 5.13832 seconds: | 20260506 01:40:04.646 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:40:04.646 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:04.646 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:04.646 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:04.646 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260506 01:40:04.646 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:04.646 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:04.646 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 01:40:04.646 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:04.646 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:04.646 | INFO | Uploaded 36GiB to GPU in 2.09535s (17.1809GiB/s) | 20260506 01:40:04.646 | INFO | Downloaded 34GiB to GPU in 2.08888s (16.2767GiB/s) | 20260506 01:40:09.790 | INFO | Analyzing chunk frames = [7941632,7942144) | 20260506 01:40:09.790 | INFO | MemInfo: { | 20260506 01:40:09.790 | INFO | "gpu" : " 17 GB", | 20260506 01:40:09.790 | INFO | "huge" : "", | 20260506 01:40:09.790 | INFO | "huge_shared" : " 27 GB", | 20260506 01:40:09.790 | INFO | "malloc" : " 14 GB", | 20260506 01:40:09.790 | INFO | "pinned" : " 29 GB" | 20260506 01:40:09.790 | INFO | } | 20260506 01:40:09.790 | INFO | Perf report for the last 5.14429 seconds: | 20260506 01:40:09.790 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:40:09.790 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:09.790 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:09.790 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:09.790 | INFO | ANALYSIS : 61.3%, 3.0, 95.9ms, 0 | 20260506 01:40:09.790 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:09.790 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:09.790 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 01:40:09.790 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:09.790 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:09.790 | INFO | Uploaded 36GiB to GPU in 2.05151s (17.5481GiB/s) | 20260506 01:40:09.791 | INFO | Downloaded 34GiB to GPU in 2.02154s (16.8189GiB/s) | 20260506 01:40:09.916 | INFO | Starting a round of Baz IO | 20260506 01:40:13.047 | INFO | Populating RTMetrics report | 20260506 01:40:13.047 | INFO | RTMetrics report populated | 20260506 01:40:13.047 | INFO | Opening tmp RTMetrics json file | 20260506 01:40:13.048 | INFO | Serializing RTMetrics to json | 20260506 01:40:13.049 | INFO | Serialization done, swapping tmp file now... | 20260506 01:40:13.050 | INFO | RTMetrics done | 20260506 01:40:13.123 | INFO | Finished a round of Baz IO: Wrote 1.817 GiB in 3.207 seconds | 20260506 01:40:13.123 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.30% / 0.00% / 78.69% | 20260506 01:40:13.123 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0777404% / 3.97364e-06% / 6.35783e-05% | 20260506 01:40:13.123 | INFO | Baz IO overall: | 20260506 01:40:13.123 | INFO | BytesWritten : 4474196569300 | 20260506 01:40:13.123 | INFO | NumEvents : 2102487495372 | 20260506 01:40:14.938 | INFO | Analyzing chunk frames = [7942144,7942656) | 20260506 01:40:14.938 | INFO | MemInfo: { | 20260506 01:40:14.938 | INFO | "gpu" : " 17 GB", | 20260506 01:40:14.938 | INFO | "huge" : "", | 20260506 01:40:14.938 | INFO | "huge_shared" : " 27 GB", | 20260506 01:40:14.938 | INFO | "malloc" : " 14 GB", | 20260506 01:40:14.938 | INFO | "pinned" : " 29 GB" | 20260506 01:40:14.938 | INFO | } | 20260506 01:40:14.938 | INFO | Perf report for the last 5.14784 seconds: | 20260506 01:40:14.938 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:40:14.938 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:14.938 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:14.938 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:14.938 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260506 01:40:14.938 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:14.938 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260506 01:40:14.938 | INFO | PRE_HQ : 59.5%, 1.7, 53.7ms, 0 | 20260506 01:40:14.938 | INFO | BAZWRITER : 61.6%, 3.5, 110.2ms, 0 | 20260506 01:40:14.938 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:40:14.938 | INFO | Uploaded 36GiB to GPU in 2.17973s (16.5158GiB/s) | 20260506 01:40:14.938 | INFO | Downloaded 38GiB to GPU in 2.23619s (16.9932GiB/s) | 20260506 01:40:20.068 | INFO | Analyzing chunk frames = [7942656,7943168) | 20260506 01:40:20.068 | INFO | MemInfo: { | 20260506 01:40:20.068 | INFO | "gpu" : " 17 GB", | 20260506 01:40:20.068 | INFO | "huge" : "", | 20260506 01:40:20.068 | INFO | "huge_shared" : " 27 GB", | 20260506 01:40:20.068 | INFO | "malloc" : " 14 GB", | 20260506 01:40:20.068 | INFO | "pinned" : " 29 GB" | 20260506 01:40:20.068 | INFO | } | 20260506 01:40:20.068 | INFO | Perf report for the last 5.12964 seconds: | 20260506 01:40:20.068 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:40:20.068 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:20.068 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:20.068 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:20.068 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260506 01:40:20.068 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:20.068 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:20.068 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 01:40:20.068 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:20.068 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:20.068 | INFO | Uploaded 36GiB to GPU in 1.95786s (18.3874GiB/s) | 20260506 01:40:20.068 | INFO | Downloaded 34GiB to GPU in 1.98629s (17.1173GiB/s) | 20260506 01:40:25.200 | INFO | Analyzing chunk frames = [7943168,7943680) | 20260506 01:40:25.200 | INFO | MemInfo: { | 20260506 01:40:25.200 | INFO | "gpu" : " 17 GB", | 20260506 01:40:25.200 | INFO | "huge" : "", | 20260506 01:40:25.200 | INFO | "huge_shared" : " 27 GB", | 20260506 01:40:25.200 | INFO | "malloc" : " 14 GB", | 20260506 01:40:25.200 | INFO | "pinned" : " 29 GB" | 20260506 01:40:25.200 | INFO | } | 20260506 01:40:25.200 | INFO | Perf report for the last 5.1325 seconds: | 20260506 01:40:25.200 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:40:25.200 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:25.200 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:25.200 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:25.200 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 01:40:25.200 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:25.200 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:25.200 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:40:25.200 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:25.200 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:25.201 | INFO | Uploaded 36GiB to GPU in 2.08209s (17.2903GiB/s) | 20260506 01:40:25.201 | INFO | Downloaded 34GiB to GPU in 2.01962s (16.8348GiB/s) | 20260506 01:40:30.348 | INFO | Analyzing chunk frames = [7943680,7944192) | 20260506 01:40:30.348 | INFO | MemInfo: { | 20260506 01:40:30.348 | INFO | "gpu" : " 17 GB", | 20260506 01:40:30.348 | INFO | "huge" : "", | 20260506 01:40:30.348 | INFO | "huge_shared" : " 27 GB", | 20260506 01:40:30.348 | INFO | "malloc" : " 14 GB", | 20260506 01:40:30.348 | INFO | "pinned" : " 29 GB" | 20260506 01:40:30.348 | INFO | } | 20260506 01:40:30.348 | INFO | Perf report for the last 5.14815 seconds: | 20260506 01:40:30.348 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:40:30.348 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:30.348 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:30.349 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:30.349 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260506 01:40:30.349 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:30.349 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:30.349 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:40:30.349 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:30.349 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:30.349 | INFO | Uploaded 36GiB to GPU in 2.14637s (16.7725GiB/s) | 20260506 01:40:30.349 | INFO | Downloaded 34GiB to GPU in 2.04387s (16.6351GiB/s) | 20260506 01:40:35.484 | INFO | Analyzing chunk frames = [7944192,7944704) | 20260506 01:40:35.484 | INFO | MemInfo: { | 20260506 01:40:35.484 | INFO | "gpu" : " 17 GB", | 20260506 01:40:35.484 | INFO | "huge" : "", | 20260506 01:40:35.484 | INFO | "huge_shared" : " 27 GB", | 20260506 01:40:35.484 | INFO | "malloc" : " 14 GB", | 20260506 01:40:35.484 | INFO | "pinned" : " 29 GB" | 20260506 01:40:35.484 | INFO | } | 20260506 01:40:35.484 | INFO | Perf report for the last 5.13558 seconds: | 20260506 01:40:35.484 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:40:35.484 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:35.484 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:35.484 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:35.484 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260506 01:40:35.484 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:35.484 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:35.484 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 01:40:35.484 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:35.484 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:35.484 | INFO | Uploaded 36GiB to GPU in 2.09776s (17.1611GiB/s) | 20260506 01:40:35.484 | INFO | Downloaded 34GiB to GPU in 2.03804s (16.6827GiB/s) | 20260506 01:40:40.620 | INFO | Analyzing chunk frames = [7944704,7945216) | 20260506 01:40:40.620 | INFO | MemInfo: { | 20260506 01:40:40.620 | INFO | "gpu" : " 17 GB", | 20260506 01:40:40.620 | INFO | "huge" : "", | 20260506 01:40:40.620 | INFO | "huge_shared" : " 27 GB", | 20260506 01:40:40.620 | INFO | "malloc" : " 14 GB", | 20260506 01:40:40.620 | INFO | "pinned" : " 29 GB" | 20260506 01:40:40.620 | INFO | } | 20260506 01:40:40.620 | INFO | Perf report for the last 5.13576 seconds: | 20260506 01:40:40.620 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:40:40.620 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:40.620 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:40.620 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:40.620 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 01:40:40.620 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:40.620 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:40.620 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 01:40:40.620 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:40.620 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:40.620 | INFO | Uploaded 36GiB to GPU in 2.06516s (17.4321GiB/s) | 20260506 01:40:40.620 | INFO | Downloaded 34GiB to GPU in 2.03301s (16.724GiB/s) | 20260506 01:40:45.757 | INFO | Analyzing chunk frames = [7945216,7945728) | 20260506 01:40:45.757 | INFO | MemInfo: { | 20260506 01:40:45.757 | INFO | "gpu" : " 17 GB", | 20260506 01:40:45.757 | INFO | "huge" : "", | 20260506 01:40:45.757 | INFO | "huge_shared" : " 27 GB", | 20260506 01:40:45.757 | INFO | "malloc" : " 14 GB", | 20260506 01:40:45.757 | INFO | "pinned" : " 29 GB" | 20260506 01:40:45.757 | INFO | } | 20260506 01:40:45.757 | INFO | Perf report for the last 5.13764 seconds: | 20260506 01:40:45.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:40:45.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:45.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:45.758 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:45.758 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260506 01:40:45.758 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:45.758 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:45.758 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 01:40:45.758 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:45.758 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:45.758 | INFO | Uploaded 36GiB to GPU in 2.09562s (17.1787GiB/s) | 20260506 01:40:45.758 | INFO | Downloaded 34GiB to GPU in 2.07354s (16.3971GiB/s) | 20260506 01:40:50.900 | INFO | Analyzing chunk frames = [7945728,7946240) | 20260506 01:40:50.900 | INFO | MemInfo: { | 20260506 01:40:50.900 | INFO | "gpu" : " 17 GB", | 20260506 01:40:50.900 | INFO | "huge" : "", | 20260506 01:40:50.900 | INFO | "huge_shared" : " 27 GB", | 20260506 01:40:50.900 | INFO | "malloc" : " 14 GB", | 20260506 01:40:50.900 | INFO | "pinned" : " 29 GB" | 20260506 01:40:50.900 | INFO | } | 20260506 01:40:50.900 | INFO | Perf report for the last 5.1425 seconds: | 20260506 01:40:50.900 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:40:50.900 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:50.900 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:50.900 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:50.900 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260506 01:40:50.900 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:50.900 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:50.900 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 01:40:50.900 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:50.900 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:50.900 | INFO | Uploaded 36GiB to GPU in 2.12945s (16.9057GiB/s) | 20260506 01:40:50.900 | INFO | Downloaded 34GiB to GPU in 2.01754s (16.8522GiB/s) | 20260506 01:40:50.900 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 01:40:50.900 | INFO | Total mem usage: 15.3 GiB | 20260506 01:40:50.900 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 01:40:50.900 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.18%) | 20260506 01:40:50.900 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 01:40:50.900 | INFO | High Water Breakdown: | 20260506 01:40:50.900 | INFO | 60.9% : 8.55 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 01:40:50.900 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 01:40:50.900 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 01:40:50.900 | INFO | ------------------------------------------ | 20260506 01:40:50.900 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:40:50.900 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 01:40:50.900 | INFO | Total mem usage: 26.5 GiB | 20260506 01:40:50.900 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 01:40:50.901 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 01:40:50.901 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260506 01:40:50.901 | INFO | High Water Breakdown: | 20260506 01:40:50.901 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:40:50.901 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:40:50.901 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:40:50.901 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:40:50.901 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:40:50.901 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:40:50.901 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 01:40:50.901 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 01:40:50.901 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:40:50.901 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 01:40:50.901 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 01:40:50.901 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 01:40:50.901 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 01:40:50.901 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 01:40:50.901 | INFO | ------------------------------------------ | 20260506 01:40:50.901 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:40:50.901 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 01:40:50.901 | INFO | Total mem usage: 25.6 GiB | 20260506 01:40:50.901 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 01:40:50.901 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 01:40:50.901 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 01:40:50.901 | INFO | High Water Breakdown: | 20260506 01:40:50.901 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 01:40:50.901 | INFO | ------------------------------------------ | 20260506 01:40:50.901 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:40:50.901 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 01:40:50.901 | INFO | Total mem usage: 15.7 GiB | 20260506 01:40:50.901 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 01:40:50.901 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 01:40:50.901 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 01:40:50.901 | INFO | High Water Breakdown: | 20260506 01:40:50.901 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 01:40:50.901 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 01:40:50.901 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 01:40:50.901 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 01:40:50.901 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 01:40:50.901 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 01:40:50.901 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 01:40:50.901 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 01:40:50.901 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 01:40:50.901 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:40:50.901 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 01:40:50.901 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:40:50.901 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 01:40:50.901 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 01:40:50.901 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 01:40:50.901 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 01:40:50.901 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:40:50.901 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 01:40:50.901 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:40:50.901 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:40:50.901 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:40:50.901 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 01:40:50.901 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 01:40:50.901 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:40:50.901 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 01:40:50.901 | INFO | ------------------------------------------ | 20260506 01:40:50.901 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:40:50.901 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 01:40:50.901 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:40:50.901 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:40:50.901 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.38e+01, 1.33e+01 / 6.89e-01 | 20260506 01:40:50.901 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.04e-01, 1.01e-02 / 2.89e-03 | 20260506 01:40:50.901 | INFO | PerfReport: 11.85%, Baseline -- 3.71e+00 / 4.50e+00, 5.80e-01 / 2.01e-01 | 20260506 01:40:50.901 | INFO | PerfReport: 3.29%, Histogram -- 1.03e+00 / 1.53e+00, 1.94e-01 / 7.43e-02 | 20260506 01:40:50.901 | INFO | PerfReport: 15.99%, DME -- 4.00e+01 / 4.39e+01, 3.85e+00 / 8.29e-01 | 20260506 01:40:50.901 | INFO | PerfReport: 57.44%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.51e+00 / 1.10e+00 | 20260506 01:40:50.901 | INFO | PerfReport: 7.04%, PulseAccumulating -- 2.20e+00 / 2.71e+00, 4.19e-01 / 2.39e-01 | 20260506 01:40:50.901 | INFO | PerfReport: 3.04%, Metrics -- 9.51e-01 / 1.24e+00, 2.56e-01 / 1.33e-01 | 20260506 01:40:50.901 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.29e-01 / 3.81e-01, 5.61e-02 / 3.14e-02 | 20260506 01:40:50.901 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 01:40:50.901 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:40:50.901 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:40:50.901 | INFO | PerfReport: Full Execution -- 9.79e+01 / -nan, 2.43e+01 / 0.00e+00 | 20260506 01:40:50.901 | INFO | PerfReport: 22.28%, Upload -- 2.18e+01 / 0.00e+00, 7.35e+00 / 0.00e+00 | 20260506 01:40:50.901 | INFO | PerfReport: 21.90%, Download -- 2.14e+01 / 0.00e+00, 7.59e+00 / 0.00e+00 | 20260506 01:40:50.901 | INFO | PerfReport: 31.96%, Compute -- 3.13e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 01:40:50.901 | INFO | PerfReport: 2.07%, LockUpload -- 2.03e+00 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260506 01:40:50.901 | INFO | PerfReport: 1.30%, LockDownload -- 1.27e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260506 01:40:50.901 | INFO | PerfReport: 20.47%, LockCompute -- 2.00e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260506 01:40:50.901 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.67e-02 / 0.00e+00 | 20260506 01:40:51.018 | INFO | Starting a round of Baz IO | 20260506 01:40:54.165 | INFO | Populating RTMetrics report | 20260506 01:40:54.165 | INFO | RTMetrics report populated | 20260506 01:40:54.165 | INFO | Opening tmp RTMetrics json file | 20260506 01:40:54.166 | INFO | Serializing RTMetrics to json | 20260506 01:40:54.167 | INFO | Serialization done, swapping tmp file now... | 20260506 01:40:54.167 | INFO | RTMetrics done | 20260506 01:40:54.240 | INFO | Finished a round of Baz IO: Wrote 1.818 GiB in 3.221 seconds | 20260506 01:40:54.240 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.30% / 0.00% / 78.70% | 20260506 01:40:54.240 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0782132% / 0% / 8.34465e-05% | 20260506 01:40:54.240 | INFO | Baz IO overall: | 20260506 01:40:54.240 | INFO | BytesWritten : 4476148991665 | 20260506 01:40:54.240 | INFO | NumEvents : 2103377084016 | 20260506 01:40:56.044 | INFO | Analyzing chunk frames = [7946240,7946752) | 20260506 01:40:56.044 | INFO | MemInfo: { | 20260506 01:40:56.044 | INFO | "gpu" : " 17 GB", | 20260506 01:40:56.044 | INFO | "huge" : "", | 20260506 01:40:56.044 | INFO | "huge_shared" : " 27 GB", | 20260506 01:40:56.044 | INFO | "malloc" : " 14 GB", | 20260506 01:40:56.044 | INFO | "pinned" : " 29 GB" | 20260506 01:40:56.044 | INFO | } | 20260506 01:40:56.045 | INFO | Perf report for the last 5.14454 seconds: | 20260506 01:40:56.045 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:40:56.045 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:56.045 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:56.045 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:56.045 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260506 01:40:56.045 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:40:56.045 | INFO | RT_METRICS : 42.8%, 1.0, 22.5ms, 0 | 20260506 01:40:56.045 | INFO | PRE_HQ : 59.6%, 1.7, 53.6ms, 0 | 20260506 01:40:56.045 | INFO | BAZWRITER : 61.7%, 3.5, 110.0ms, 0 | 20260506 01:40:56.045 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:40:56.045 | INFO | Uploaded 36GiB to GPU in 2.17946s (16.5179GiB/s) | 20260506 01:40:56.045 | INFO | Downloaded 38GiB to GPU in 2.2356s (16.9977GiB/s) | 20260506 01:41:01.188 | INFO | Analyzing chunk frames = [7946752,7947264) | 20260506 01:41:01.188 | INFO | MemInfo: { | 20260506 01:41:01.188 | INFO | "gpu" : " 17 GB", | 20260506 01:41:01.188 | INFO | "huge" : "", | 20260506 01:41:01.188 | INFO | "huge_shared" : " 27 GB", | 20260506 01:41:01.188 | INFO | "malloc" : " 14 GB", | 20260506 01:41:01.188 | INFO | "pinned" : " 29 GB" | 20260506 01:41:01.188 | INFO | } | 20260506 01:41:01.188 | INFO | Perf report for the last 5.14359 seconds: | 20260506 01:41:01.188 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:41:01.188 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:01.188 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:01.188 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:01.188 | INFO | ANALYSIS : 60.8%, 3.0, 95.0ms, 0 | 20260506 01:41:01.188 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:01.188 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:01.188 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 01:41:01.188 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:01.188 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:01.188 | INFO | Uploaded 36GiB to GPU in 1.97179s (18.2575GiB/s) | 20260506 01:41:01.188 | INFO | Downloaded 34GiB to GPU in 1.97955s (17.1756GiB/s) | 20260506 01:41:06.320 | INFO | Analyzing chunk frames = [7947264,7947776) | 20260506 01:41:06.320 | INFO | MemInfo: { | 20260506 01:41:06.320 | INFO | "gpu" : " 17 GB", | 20260506 01:41:06.320 | INFO | "huge" : "", | 20260506 01:41:06.320 | INFO | "huge_shared" : " 27 GB", | 20260506 01:41:06.320 | INFO | "malloc" : " 14 GB", | 20260506 01:41:06.321 | INFO | "pinned" : " 29 GB" | 20260506 01:41:06.321 | INFO | } | 20260506 01:41:06.321 | INFO | Perf report for the last 5.13243 seconds: | 20260506 01:41:06.321 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:41:06.321 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:06.321 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:06.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:06.321 | INFO | ANALYSIS : 61.2%, 3.0, 95.1ms, 0 | 20260506 01:41:06.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:06.321 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:06.321 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 01:41:06.321 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:06.321 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:06.321 | INFO | Uploaded 36GiB to GPU in 2.08767s (17.2441GiB/s) | 20260506 01:41:06.321 | INFO | Downloaded 34GiB to GPU in 2.04212s (16.6494GiB/s) | 20260506 01:41:11.457 | INFO | Analyzing chunk frames = [7947776,7948288) | 20260506 01:41:11.457 | INFO | MemInfo: { | 20260506 01:41:11.457 | INFO | "gpu" : " 17 GB", | 20260506 01:41:11.457 | INFO | "huge" : "", | 20260506 01:41:11.457 | INFO | "huge_shared" : " 27 GB", | 20260506 01:41:11.457 | INFO | "malloc" : " 14 GB", | 20260506 01:41:11.457 | INFO | "pinned" : " 29 GB" | 20260506 01:41:11.457 | INFO | } | 20260506 01:41:11.457 | INFO | Perf report for the last 5.13653 seconds: | 20260506 01:41:11.457 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:41:11.457 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:11.457 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:11.457 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:11.457 | INFO | ANALYSIS : 61.6%, 3.0, 95.6ms, 0 | 20260506 01:41:11.457 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:11.457 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:11.457 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 01:41:11.457 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:11.457 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:11.457 | INFO | Uploaded 36GiB to GPU in 2.12891s (16.9101GiB/s) | 20260506 01:41:11.458 | INFO | Downloaded 34GiB to GPU in 2.01547s (16.8695GiB/s) | 20260506 01:41:16.597 | INFO | Analyzing chunk frames = [7948288,7948800) | 20260506 01:41:16.597 | INFO | MemInfo: { | 20260506 01:41:16.597 | INFO | "gpu" : " 17 GB", | 20260506 01:41:16.597 | INFO | "huge" : "", | 20260506 01:41:16.597 | INFO | "huge_shared" : " 27 GB", | 20260506 01:41:16.597 | INFO | "malloc" : " 14 GB", | 20260506 01:41:16.597 | INFO | "pinned" : " 29 GB" | 20260506 01:41:16.597 | INFO | } | 20260506 01:41:16.597 | INFO | Perf report for the last 5.13988 seconds: | 20260506 01:41:16.597 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:41:16.597 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:16.597 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:16.597 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:16.597 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 01:41:16.597 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:16.597 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:16.597 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:41:16.597 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:16.597 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:16.597 | INFO | Uploaded 36GiB to GPU in 2.09132s (17.214GiB/s) | 20260506 01:41:16.597 | INFO | Downloaded 34GiB to GPU in 2.05055s (16.5809GiB/s) | 20260506 01:41:21.743 | INFO | Analyzing chunk frames = [7948800,7949312) | 20260506 01:41:21.743 | INFO | MemInfo: { | 20260506 01:41:21.743 | INFO | "gpu" : " 17 GB", | 20260506 01:41:21.743 | INFO | "huge" : "", | 20260506 01:41:21.743 | INFO | "huge_shared" : " 27 GB", | 20260506 01:41:21.743 | INFO | "malloc" : " 14 GB", | 20260506 01:41:21.743 | INFO | "pinned" : " 29 GB" | 20260506 01:41:21.743 | INFO | } | 20260506 01:41:21.743 | INFO | Perf report for the last 5.1464 seconds: | 20260506 01:41:21.743 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:41:21.743 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:21.743 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:21.743 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:21.744 | INFO | ANALYSIS : 60.9%, 3.0, 95.2ms, 0 | 20260506 01:41:21.744 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:21.744 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:21.744 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 01:41:21.744 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:21.744 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:21.744 | INFO | Uploaded 36GiB to GPU in 2.06219s (17.4571GiB/s) | 20260506 01:41:21.744 | INFO | Downloaded 34GiB to GPU in 2.03438s (16.7127GiB/s) | 20260506 01:41:26.889 | INFO | Analyzing chunk frames = [7949312,7949824) | 20260506 01:41:26.889 | INFO | MemInfo: { | 20260506 01:41:26.889 | INFO | "gpu" : " 17 GB", | 20260506 01:41:26.889 | INFO | "huge" : "", | 20260506 01:41:26.889 | INFO | "huge_shared" : " 27 GB", | 20260506 01:41:26.889 | INFO | "malloc" : " 14 GB", | 20260506 01:41:26.889 | INFO | "pinned" : " 29 GB" | 20260506 01:41:26.889 | INFO | } | 20260506 01:41:26.889 | INFO | Perf report for the last 5.1458 seconds: | 20260506 01:41:26.889 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:41:26.889 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:26.889 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:26.889 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:26.889 | INFO | ANALYSIS : 61.8%, 3.0, 96.6ms, 0 | 20260506 01:41:26.889 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:26.889 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:26.889 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 01:41:26.889 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:26.889 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:26.890 | INFO | Uploaded 36GiB to GPU in 2.02195s (17.8046GiB/s) | 20260506 01:41:26.890 | INFO | Downloaded 34GiB to GPU in 2.05376s (16.555GiB/s) | 20260506 01:41:32.012 | INFO | Analyzing chunk frames = [7949824,7950336) | 20260506 01:41:32.012 | INFO | MemInfo: { | 20260506 01:41:32.012 | INFO | "gpu" : " 17 GB", | 20260506 01:41:32.012 | INFO | "huge" : "", | 20260506 01:41:32.012 | INFO | "huge_shared" : " 27 GB", | 20260506 01:41:32.012 | INFO | "malloc" : " 14 GB", | 20260506 01:41:32.012 | INFO | "pinned" : " 29 GB" | 20260506 01:41:32.012 | INFO | } | 20260506 01:41:32.012 | INFO | Perf report for the last 5.12305 seconds: | 20260506 01:41:32.012 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:41:32.012 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:32.012 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:32.012 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:32.012 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 01:41:32.012 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:32.012 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:32.012 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 01:41:32.012 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:32.012 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:32.013 | INFO | Uploaded 36GiB to GPU in 2.0287s (17.7454GiB/s) | 20260506 01:41:32.013 | INFO | Downloaded 34GiB to GPU in 2.02236s (16.812GiB/s) | 20260506 01:41:32.135 | INFO | Starting a round of Baz IO | 20260506 01:41:35.274 | INFO | Populating RTMetrics report | 20260506 01:41:35.275 | INFO | RTMetrics report populated | 20260506 01:41:35.275 | INFO | Opening tmp RTMetrics json file | 20260506 01:41:35.275 | INFO | Serializing RTMetrics to json | 20260506 01:41:35.276 | INFO | Serialization done, swapping tmp file now... | 20260506 01:41:35.277 | INFO | RTMetrics done | 20260506 01:41:35.349 | INFO | Finished a round of Baz IO: Wrote 1.815 GiB in 3.214 seconds | 20260506 01:41:35.349 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.30% / 0.00% / 78.70% | 20260506 01:41:35.349 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.077494% / 3.97364e-06% / 5.5631e-05% | 20260506 01:41:35.349 | INFO | Baz IO overall: | 20260506 01:41:35.349 | INFO | BytesWritten : 4478098316585 | 20260506 01:41:35.349 | INFO | NumEvents : 2104264516678 | 20260506 01:41:37.159 | INFO | Analyzing chunk frames = [7950336,7950848) | 20260506 01:41:37.159 | INFO | MemInfo: { | 20260506 01:41:37.159 | INFO | "gpu" : " 17 GB", | 20260506 01:41:37.159 | INFO | "huge" : "", | 20260506 01:41:37.159 | INFO | "huge_shared" : " 27 GB", | 20260506 01:41:37.159 | INFO | "malloc" : " 14 GB", | 20260506 01:41:37.159 | INFO | "pinned" : " 29 GB" | 20260506 01:41:37.159 | INFO | } | 20260506 01:41:37.159 | INFO | Perf report for the last 5.14646 seconds: | 20260506 01:41:37.159 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:41:37.159 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:37.159 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:37.159 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:37.159 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260506 01:41:37.159 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:37.159 | INFO | RT_METRICS : 42.8%, 1.0, 22.5ms, 0 | 20260506 01:41:37.159 | INFO | PRE_HQ : 59.8%, 1.7, 53.8ms, 0 | 20260506 01:41:37.159 | INFO | BAZWRITER : 61.7%, 3.5, 110.6ms, 0 | 20260506 01:41:37.159 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:41:37.159 | INFO | Uploaded 36GiB to GPU in 2.17759s (16.532GiB/s) | 20260506 01:41:37.159 | INFO | Downloaded 38GiB to GPU in 2.2422s (16.9477GiB/s) | 20260506 01:41:42.300 | INFO | Analyzing chunk frames = [7950848,7951360) | 20260506 01:41:42.300 | INFO | MemInfo: { | 20260506 01:41:42.300 | INFO | "gpu" : " 17 GB", | 20260506 01:41:42.300 | INFO | "huge" : "", | 20260506 01:41:42.300 | INFO | "huge_shared" : " 27 GB", | 20260506 01:41:42.300 | INFO | "malloc" : " 14 GB", | 20260506 01:41:42.300 | INFO | "pinned" : " 29 GB" | 20260506 01:41:42.300 | INFO | } | 20260506 01:41:42.300 | INFO | Perf report for the last 5.14135 seconds: | 20260506 01:41:42.300 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:41:42.300 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:42.300 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:42.300 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:42.300 | INFO | ANALYSIS : 60.9%, 3.0, 95.0ms, 0 | 20260506 01:41:42.300 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:42.300 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:42.300 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 01:41:42.300 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:42.300 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:42.300 | INFO | Uploaded 36GiB to GPU in 1.97574s (18.221GiB/s) | 20260506 01:41:42.301 | INFO | Downloaded 34GiB to GPU in 1.98309s (17.145GiB/s) | 20260506 01:41:47.433 | INFO | Analyzing chunk frames = [7951360,7951872) | 20260506 01:41:47.433 | INFO | MemInfo: { | 20260506 01:41:47.433 | INFO | "gpu" : " 17 GB", | 20260506 01:41:47.433 | INFO | "huge" : "", | 20260506 01:41:47.433 | INFO | "huge_shared" : " 27 GB", | 20260506 01:41:47.433 | INFO | "malloc" : " 14 GB", | 20260506 01:41:47.433 | INFO | "pinned" : " 29 GB" | 20260506 01:41:47.433 | INFO | } | 20260506 01:41:47.433 | INFO | Perf report for the last 5.13334 seconds: | 20260506 01:41:47.433 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:41:47.433 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:47.433 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:47.434 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:47.434 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 01:41:47.434 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:47.434 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:47.434 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:41:47.434 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:47.434 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:47.434 | INFO | Uploaded 36GiB to GPU in 2.09732s (17.1648GiB/s) | 20260506 01:41:47.434 | INFO | Downloaded 34GiB to GPU in 2.03743s (16.6877GiB/s) | 20260506 01:41:52.569 | INFO | Analyzing chunk frames = [7951872,7952384) | 20260506 01:41:52.569 | INFO | MemInfo: { | 20260506 01:41:52.569 | INFO | "gpu" : " 17 GB", | 20260506 01:41:52.569 | INFO | "huge" : "", | 20260506 01:41:52.569 | INFO | "huge_shared" : " 27 GB", | 20260506 01:41:52.569 | INFO | "malloc" : " 14 GB", | 20260506 01:41:52.569 | INFO | "pinned" : " 29 GB" | 20260506 01:41:52.569 | INFO | } | 20260506 01:41:52.569 | INFO | Perf report for the last 5.13573 seconds: | 20260506 01:41:52.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:41:52.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:52.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:52.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:52.569 | INFO | ANALYSIS : 61.6%, 3.0, 95.6ms, 0 | 20260506 01:41:52.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:52.569 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:52.569 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 01:41:52.569 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:52.569 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:52.570 | INFO | Uploaded 36GiB to GPU in 2.14002s (16.8222GiB/s) | 20260506 01:41:52.570 | INFO | Downloaded 34GiB to GPU in 2.02713s (16.7725GiB/s) | 20260506 01:41:57.711 | INFO | Analyzing chunk frames = [7952384,7952896) | 20260506 01:41:57.711 | INFO | MemInfo: { | 20260506 01:41:57.711 | INFO | "gpu" : " 17 GB", | 20260506 01:41:57.711 | INFO | "huge" : "", | 20260506 01:41:57.711 | INFO | "huge_shared" : " 27 GB", | 20260506 01:41:57.711 | INFO | "malloc" : " 14 GB", | 20260506 01:41:57.711 | INFO | "pinned" : " 29 GB" | 20260506 01:41:57.711 | INFO | } | 20260506 01:41:57.711 | INFO | Perf report for the last 5.1417 seconds: | 20260506 01:41:57.711 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:41:57.711 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:57.711 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:57.711 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:57.711 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 01:41:57.711 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:57.711 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:57.711 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:41:57.711 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:57.711 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:41:57.711 | INFO | Uploaded 36GiB to GPU in 2.09979s (17.1446GiB/s) | 20260506 01:41:57.711 | INFO | Downloaded 34GiB to GPU in 2.04374s (16.6361GiB/s) | 20260506 01:42:02.855 | INFO | Analyzing chunk frames = [7952896,7953408) | 20260506 01:42:02.855 | INFO | MemInfo: { | 20260506 01:42:02.855 | INFO | "gpu" : " 17 GB", | 20260506 01:42:02.855 | INFO | "huge" : "", | 20260506 01:42:02.855 | INFO | "huge_shared" : " 27 GB", | 20260506 01:42:02.855 | INFO | "malloc" : " 14 GB", | 20260506 01:42:02.855 | INFO | "pinned" : " 29 GB" | 20260506 01:42:02.855 | INFO | } | 20260506 01:42:02.855 | INFO | Perf report for the last 5.14443 seconds: | 20260506 01:42:02.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:42:02.855 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:02.855 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:02.855 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:02.855 | INFO | ANALYSIS : 61.2%, 3.0, 95.7ms, 0 | 20260506 01:42:02.855 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:02.855 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:02.856 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 01:42:02.856 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:02.856 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:02.856 | INFO | Uploaded 36GiB to GPU in 2.07651s (17.3368GiB/s) | 20260506 01:42:02.856 | INFO | Downloaded 34GiB to GPU in 2.03182s (16.7337GiB/s) | 20260506 01:42:07.991 | INFO | Analyzing chunk frames = [7953408,7953920) | 20260506 01:42:07.991 | INFO | MemInfo: { | 20260506 01:42:07.991 | INFO | "gpu" : " 17 GB", | 20260506 01:42:07.991 | INFO | "huge" : "", | 20260506 01:42:07.991 | INFO | "huge_shared" : " 27 GB", | 20260506 01:42:07.991 | INFO | "malloc" : " 14 GB", | 20260506 01:42:07.991 | INFO | "pinned" : " 29 GB" | 20260506 01:42:07.991 | INFO | } | 20260506 01:42:07.991 | INFO | Perf report for the last 5.136 seconds: | 20260506 01:42:07.991 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:42:07.991 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:07.991 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:07.991 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:07.991 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260506 01:42:07.991 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:07.992 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:07.992 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 01:42:07.992 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:07.992 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:07.992 | INFO | Uploaded 36GiB to GPU in 2.02472s (17.7803GiB/s) | 20260506 01:42:07.992 | INFO | Downloaded 34GiB to GPU in 2.05087s (16.5784GiB/s) | 20260506 01:42:13.134 | INFO | Analyzing chunk frames = [7953920,7954432) | 20260506 01:42:13.134 | INFO | MemInfo: { | 20260506 01:42:13.134 | INFO | "gpu" : " 17 GB", | 20260506 01:42:13.134 | INFO | "huge" : "", | 20260506 01:42:13.134 | INFO | "huge_shared" : " 27 GB", | 20260506 01:42:13.134 | INFO | "malloc" : " 14 GB", | 20260506 01:42:13.134 | INFO | "pinned" : " 29 GB" | 20260506 01:42:13.134 | INFO | } | 20260506 01:42:13.134 | INFO | Perf report for the last 5.14284 seconds: | 20260506 01:42:13.134 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:42:13.134 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:13.134 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:13.134 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:13.134 | INFO | ANALYSIS : 61.5%, 3.0, 96.1ms, 0 | 20260506 01:42:13.134 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:13.134 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:13.134 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 01:42:13.134 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:13.134 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:13.135 | INFO | Uploaded 36GiB to GPU in 2.06063s (17.4704GiB/s) | 20260506 01:42:13.135 | INFO | Downloaded 34GiB to GPU in 2.02099s (16.8234GiB/s) | 20260506 01:42:13.135 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 01:42:13.135 | INFO | Total mem usage: 15.3 GiB | 20260506 01:42:13.135 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 01:42:13.135 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.21%) | 20260506 01:42:13.135 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 01:42:13.135 | INFO | High Water Breakdown: | 20260506 01:42:13.135 | INFO | 60.9% : 8.54 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 01:42:13.135 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 01:42:13.135 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 01:42:13.135 | INFO | ------------------------------------------ | 20260506 01:42:13.135 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:42:13.135 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 01:42:13.135 | INFO | Total mem usage: 26.5 GiB | 20260506 01:42:13.135 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 01:42:13.135 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 01:42:13.135 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 01:42:13.135 | INFO | High Water Breakdown: | 20260506 01:42:13.135 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:42:13.135 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:42:13.135 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:42:13.135 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:42:13.135 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:42:13.135 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:42:13.135 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 01:42:13.135 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 01:42:13.135 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:42:13.135 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 01:42:13.135 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 01:42:13.135 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 01:42:13.135 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 01:42:13.135 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 01:42:13.135 | INFO | ------------------------------------------ | 20260506 01:42:13.135 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:42:13.135 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 01:42:13.135 | INFO | Total mem usage: 25.6 GiB | 20260506 01:42:13.135 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 01:42:13.135 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 01:42:13.135 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 01:42:13.135 | INFO | High Water Breakdown: | 20260506 01:42:13.135 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 01:42:13.135 | INFO | ------------------------------------------ | 20260506 01:42:13.135 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:42:13.135 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 01:42:13.135 | INFO | Total mem usage: 15.7 GiB | 20260506 01:42:13.135 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 01:42:13.135 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 01:42:13.135 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 01:42:13.135 | INFO | High Water Breakdown: | 20260506 01:42:13.135 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 01:42:13.135 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 01:42:13.135 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 01:42:13.135 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 01:42:13.135 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 01:42:13.135 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 01:42:13.135 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 01:42:13.135 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 01:42:13.135 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 01:42:13.135 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:42:13.135 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 01:42:13.136 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:42:13.136 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 01:42:13.136 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 01:42:13.136 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 01:42:13.136 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 01:42:13.136 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:42:13.136 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 01:42:13.136 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:42:13.136 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:42:13.136 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:42:13.136 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 01:42:13.136 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 01:42:13.136 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:42:13.136 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 01:42:13.136 | INFO | ------------------------------------------ | 20260506 01:42:13.136 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:42:13.136 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 01:42:13.136 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:42:13.136 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:42:13.136 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.40e+01, 1.35e+01 / 9.86e-01 | 20260506 01:42:13.136 | INFO | PerfReport: 0.29%, AggregateStats -- 9.17e-02 / 1.03e-01, 9.89e-03 / 3.94e-03 | 20260506 01:42:13.136 | INFO | PerfReport: 11.88%, Baseline -- 3.72e+00 / 4.54e+00, 5.82e-01 / 2.01e-01 | 20260506 01:42:13.136 | INFO | PerfReport: 3.30%, Histogram -- 1.03e+00 / 1.54e+00, 1.95e-01 / 8.37e-02 | 20260506 01:42:13.136 | INFO | PerfReport: 16.03%, DME -- 4.01e+01 / 4.43e+01, 3.91e+00 / 1.26e+00 | 20260506 01:42:13.136 | INFO | PerfReport: 57.37%, FrameLabeling -- 1.80e+01 / 1.92e+01, 1.49e+00 / 1.12e+00 | 20260506 01:42:13.136 | INFO | PerfReport: 7.05%, PulseAccumulating -- 2.21e+00 / 2.74e+00, 4.18e-01 / 2.57e-01 | 20260506 01:42:13.136 | INFO | PerfReport: 3.03%, Metrics -- 9.48e-01 / 1.21e+00, 2.52e-01 / 8.55e-02 | 20260506 01:42:13.136 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.28e-01 / 3.72e-01, 5.46e-02 / 2.13e-02 | 20260506 01:42:13.136 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 01:42:13.136 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:42:13.136 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:42:13.136 | INFO | PerfReport: Full Execution -- 9.78e+01 / -nan, 2.41e+01 / 0.00e+00 | 20260506 01:42:13.136 | INFO | PerfReport: 22.11%, Upload -- 2.16e+01 / 0.00e+00, 7.22e+00 / 0.00e+00 | 20260506 01:42:13.136 | INFO | PerfReport: 21.88%, Download -- 2.14e+01 / 0.00e+00, 7.54e+00 / 0.00e+00 | 20260506 01:42:13.136 | INFO | PerfReport: 31.99%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 01:42:13.136 | INFO | PerfReport: 2.03%, LockUpload -- 1.98e+00 / 0.00e+00, 7.74e+00 / 0.00e+00 | 20260506 01:42:13.136 | INFO | PerfReport: 1.32%, LockDownload -- 1.29e+00 / 0.00e+00, 2.95e+00 / 0.00e+00 | 20260506 01:42:13.136 | INFO | PerfReport: 20.66%, LockCompute -- 2.02e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260506 01:42:13.136 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.77e-02 / 0.00e+00 | 20260506 01:42:13.252 | INFO | Starting a round of Baz IO | 20260506 01:42:16.395 | INFO | Populating RTMetrics report | 20260506 01:42:16.395 | INFO | RTMetrics report populated | 20260506 01:42:16.395 | INFO | Opening tmp RTMetrics json file | 20260506 01:42:16.396 | INFO | Serializing RTMetrics to json | 20260506 01:42:16.397 | INFO | Serialization done, swapping tmp file now... | 20260506 01:42:16.397 | INFO | RTMetrics done | 20260506 01:42:16.473 | INFO | Finished a round of Baz IO: Wrote 1.815 GiB in 3.22 seconds | 20260506 01:42:16.473 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.30% / 0.00% / 78.70% | 20260506 01:42:16.473 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0771244% / 3.97364e-06% / 6.75519e-05% | 20260506 01:42:16.473 | INFO | Baz IO overall: | 20260506 01:42:16.473 | INFO | BytesWritten : 4480047147233 | 20260506 01:42:16.474 | INFO | NumEvents : 2105151845521 | 20260506 01:42:18.269 | INFO | Analyzing chunk frames = [7954432,7954944) | 20260506 01:42:18.269 | INFO | MemInfo: { | 20260506 01:42:18.269 | INFO | "gpu" : " 17 GB", | 20260506 01:42:18.269 | INFO | "huge" : "", | 20260506 01:42:18.269 | INFO | "huge_shared" : " 27 GB", | 20260506 01:42:18.269 | INFO | "malloc" : " 14 GB", | 20260506 01:42:18.269 | INFO | "pinned" : " 29 GB" | 20260506 01:42:18.269 | INFO | } | 20260506 01:42:18.269 | INFO | Perf report for the last 5.13494 seconds: | 20260506 01:42:18.269 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:42:18.269 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:18.269 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:18.269 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:18.269 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260506 01:42:18.269 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:18.269 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260506 01:42:18.269 | INFO | PRE_HQ : 59.6%, 1.7, 53.7ms, 0 | 20260506 01:42:18.269 | INFO | BAZWRITER : 61.8%, 3.5, 111.2ms, 0 | 20260506 01:42:18.269 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:42:18.269 | INFO | Uploaded 36GiB to GPU in 2.14134s (16.8119GiB/s) | 20260506 01:42:18.269 | INFO | Downloaded 38GiB to GPU in 2.23626s (16.9927GiB/s) | 20260506 01:42:23.409 | INFO | Analyzing chunk frames = [7954944,7955456) | 20260506 01:42:23.409 | INFO | MemInfo: { | 20260506 01:42:23.409 | INFO | "gpu" : " 17 GB", | 20260506 01:42:23.409 | INFO | "huge" : "", | 20260506 01:42:23.409 | INFO | "huge_shared" : " 27 GB", | 20260506 01:42:23.409 | INFO | "malloc" : " 14 GB", | 20260506 01:42:23.409 | INFO | "pinned" : " 29 GB" | 20260506 01:42:23.409 | INFO | } | 20260506 01:42:23.410 | INFO | Perf report for the last 5.14036 seconds: | 20260506 01:42:23.410 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:42:23.410 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:23.410 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:23.410 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:23.410 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260506 01:42:23.410 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:23.410 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:23.410 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 01:42:23.410 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:23.410 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:23.410 | INFO | Uploaded 36GiB to GPU in 2.1079s (17.0786GiB/s) | 20260506 01:42:23.410 | INFO | Downloaded 34GiB to GPU in 2.0365s (16.6954GiB/s) | 20260506 01:42:28.548 | INFO | Analyzing chunk frames = [7955456,7955968) | 20260506 01:42:28.548 | INFO | MemInfo: { | 20260506 01:42:28.548 | INFO | "gpu" : " 17 GB", | 20260506 01:42:28.548 | INFO | "huge" : "", | 20260506 01:42:28.548 | INFO | "huge_shared" : " 27 GB", | 20260506 01:42:28.548 | INFO | "malloc" : " 14 GB", | 20260506 01:42:28.548 | INFO | "pinned" : " 29 GB" | 20260506 01:42:28.548 | INFO | } | 20260506 01:42:28.548 | INFO | Perf report for the last 5.13888 seconds: | 20260506 01:42:28.548 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:42:28.548 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:28.548 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:28.549 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:28.549 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 01:42:28.549 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:28.549 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:28.549 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 01:42:28.549 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:28.549 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:28.549 | INFO | Uploaded 36GiB to GPU in 2.07903s (17.3158GiB/s) | 20260506 01:42:28.549 | INFO | Downloaded 34GiB to GPU in 2.02179s (16.8168GiB/s) | 20260506 01:42:33.691 | INFO | Analyzing chunk frames = [7955968,7956480) | 20260506 01:42:33.691 | INFO | MemInfo: { | 20260506 01:42:33.691 | INFO | "gpu" : " 17 GB", | 20260506 01:42:33.691 | INFO | "huge" : "", | 20260506 01:42:33.691 | INFO | "huge_shared" : " 27 GB", | 20260506 01:42:33.691 | INFO | "malloc" : " 14 GB", | 20260506 01:42:33.691 | INFO | "pinned" : " 29 GB" | 20260506 01:42:33.691 | INFO | } | 20260506 01:42:33.691 | INFO | Perf report for the last 5.14237 seconds: | 20260506 01:42:33.691 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:42:33.691 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:33.691 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:33.691 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:33.691 | INFO | ANALYSIS : 61.3%, 3.0, 95.2ms, 0 | 20260506 01:42:33.691 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:33.691 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:33.691 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:42:33.691 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:33.691 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:33.691 | INFO | Uploaded 36GiB to GPU in 2.10818s (17.0763GiB/s) | 20260506 01:42:33.691 | INFO | Downloaded 34GiB to GPU in 2.01939s (16.8367GiB/s) | 20260506 01:42:38.833 | INFO | Analyzing chunk frames = [7956480,7956992) | 20260506 01:42:38.833 | INFO | MemInfo: { | 20260506 01:42:38.833 | INFO | "gpu" : " 17 GB", | 20260506 01:42:38.833 | INFO | "huge" : "", | 20260506 01:42:38.833 | INFO | "huge_shared" : " 27 GB", | 20260506 01:42:38.833 | INFO | "malloc" : " 14 GB", | 20260506 01:42:38.833 | INFO | "pinned" : " 29 GB" | 20260506 01:42:38.833 | INFO | } | 20260506 01:42:38.833 | INFO | Perf report for the last 5.14246 seconds: | 20260506 01:42:38.833 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:42:38.833 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:38.833 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:38.833 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:38.833 | INFO | ANALYSIS : 60.8%, 3.0, 94.9ms, 0 | 20260506 01:42:38.833 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:38.833 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:38.833 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:42:38.833 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:38.833 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:38.834 | INFO | Uploaded 36GiB to GPU in 2.07166s (17.3774GiB/s) | 20260506 01:42:38.834 | INFO | Downloaded 34GiB to GPU in 2.03095s (16.7409GiB/s) | 20260506 01:42:43.964 | INFO | Analyzing chunk frames = [7956992,7957504) | 20260506 01:42:43.964 | INFO | MemInfo: { | 20260506 01:42:43.964 | INFO | "gpu" : " 17 GB", | 20260506 01:42:43.964 | INFO | "huge" : "", | 20260506 01:42:43.964 | INFO | "huge_shared" : " 27 GB", | 20260506 01:42:43.964 | INFO | "malloc" : " 14 GB", | 20260506 01:42:43.964 | INFO | "pinned" : " 29 GB" | 20260506 01:42:43.964 | INFO | } | 20260506 01:42:43.965 | INFO | Perf report for the last 5.13132 seconds: | 20260506 01:42:43.965 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:42:43.965 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:43.965 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:43.965 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:43.965 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 01:42:43.965 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:43.965 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:43.965 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 01:42:43.965 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:43.965 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:43.965 | INFO | Uploaded 36GiB to GPU in 2.0738s (17.3595GiB/s) | 20260506 01:42:43.965 | INFO | Downloaded 34GiB to GPU in 2.02972s (16.7511GiB/s) | 20260506 01:42:49.109 | INFO | Analyzing chunk frames = [7957504,7958016) | 20260506 01:42:49.109 | INFO | MemInfo: { | 20260506 01:42:49.109 | INFO | "gpu" : " 17 GB", | 20260506 01:42:49.109 | INFO | "huge" : "", | 20260506 01:42:49.109 | INFO | "huge_shared" : " 27 GB", | 20260506 01:42:49.109 | INFO | "malloc" : " 14 GB", | 20260506 01:42:49.109 | INFO | "pinned" : " 29 GB" | 20260506 01:42:49.109 | INFO | } | 20260506 01:42:49.109 | INFO | Perf report for the last 5.14447 seconds: | 20260506 01:42:49.109 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:42:49.109 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:49.109 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:49.109 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:49.109 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260506 01:42:49.109 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:49.109 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:49.109 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 01:42:49.109 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:49.109 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:49.109 | INFO | Uploaded 36GiB to GPU in 2.10424s (17.1083GiB/s) | 20260506 01:42:49.109 | INFO | Downloaded 34GiB to GPU in 2.07504s (16.3852GiB/s) | 20260506 01:42:54.250 | INFO | Analyzing chunk frames = [7958016,7958528) | 20260506 01:42:54.250 | INFO | MemInfo: { | 20260506 01:42:54.250 | INFO | "gpu" : " 17 GB", | 20260506 01:42:54.250 | INFO | "huge" : "", | 20260506 01:42:54.250 | INFO | "huge_shared" : " 27 GB", | 20260506 01:42:54.250 | INFO | "malloc" : " 14 GB", | 20260506 01:42:54.250 | INFO | "pinned" : " 29 GB" | 20260506 01:42:54.250 | INFO | } | 20260506 01:42:54.250 | INFO | Perf report for the last 5.14119 seconds: | 20260506 01:42:54.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:42:54.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:54.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:54.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:54.250 | INFO | ANALYSIS : 61.5%, 3.0, 96.1ms, 0 | 20260506 01:42:54.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:54.250 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:54.250 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:42:54.250 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:54.250 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:54.251 | INFO | Uploaded 36GiB to GPU in 2.06519s (17.4318GiB/s) | 20260506 01:42:54.251 | INFO | Downloaded 34GiB to GPU in 2.02072s (16.8257GiB/s) | 20260506 01:42:54.375 | INFO | Starting a round of Baz IO | 20260506 01:42:57.515 | INFO | Populating RTMetrics report | 20260506 01:42:57.515 | INFO | RTMetrics report populated | 20260506 01:42:57.515 | INFO | Opening tmp RTMetrics json file | 20260506 01:42:57.516 | INFO | Serializing RTMetrics to json | 20260506 01:42:57.518 | INFO | Serialization done, swapping tmp file now... | 20260506 01:42:57.520 | INFO | RTMetrics done | 20260506 01:42:57.607 | INFO | Finished a round of Baz IO: Wrote 1.823 GiB in 3.231 seconds | 20260506 01:42:57.607 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.30% / 0.00% / 78.70% | 20260506 01:42:57.607 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0772278% / 0% / 6.35783e-05% | 20260506 01:42:57.607 | INFO | Baz IO overall: | 20260506 01:42:57.607 | INFO | BytesWritten : 4482004737616 | 20260506 01:42:57.607 | INFO | NumEvents : 2106041608380 | 20260506 01:42:59.382 | INFO | Analyzing chunk frames = [7958528,7959040) | 20260506 01:42:59.382 | INFO | MemInfo: { | 20260506 01:42:59.382 | INFO | "gpu" : " 17 GB", | 20260506 01:42:59.382 | INFO | "huge" : "", | 20260506 01:42:59.382 | INFO | "huge_shared" : " 27 GB", | 20260506 01:42:59.382 | INFO | "malloc" : " 14 GB", | 20260506 01:42:59.382 | INFO | "pinned" : " 29 GB" | 20260506 01:42:59.382 | INFO | } | 20260506 01:42:59.382 | INFO | Perf report for the last 5.13171 seconds: | 20260506 01:42:59.382 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:42:59.382 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:59.382 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:59.382 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:59.382 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260506 01:42:59.382 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:42:59.382 | INFO | RT_METRICS : 42.8%, 1.0, 22.4ms, 0 | 20260506 01:42:59.382 | INFO | PRE_HQ : 59.6%, 1.7, 53.5ms, 0 | 20260506 01:42:59.382 | INFO | BAZWRITER : 62.2%, 3.5, 112.1ms, 0 | 20260506 01:42:59.382 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 01:42:59.382 | INFO | Uploaded 36GiB to GPU in 2.18506s (16.4756GiB/s) | 20260506 01:42:59.382 | INFO | Downloaded 38GiB to GPU in 2.23213s (17.0241GiB/s) | 20260506 01:43:04.519 | INFO | Analyzing chunk frames = [7959040,7959552) | 20260506 01:43:04.519 | INFO | MemInfo: { | 20260506 01:43:04.519 | INFO | "gpu" : " 17 GB", | 20260506 01:43:04.519 | INFO | "huge" : "", | 20260506 01:43:04.519 | INFO | "huge_shared" : " 27 GB", | 20260506 01:43:04.519 | INFO | "malloc" : " 14 GB", | 20260506 01:43:04.519 | INFO | "pinned" : " 29 GB" | 20260506 01:43:04.519 | INFO | } | 20260506 01:43:04.519 | INFO | Perf report for the last 5.13697 seconds: | 20260506 01:43:04.519 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:43:04.519 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:04.519 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:04.519 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:04.519 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260506 01:43:04.519 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:04.519 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:04.519 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 01:43:04.519 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:04.519 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:04.519 | INFO | Uploaded 36GiB to GPU in 2.10831s (17.0753GiB/s) | 20260506 01:43:04.519 | INFO | Downloaded 34GiB to GPU in 2.04099s (16.6586GiB/s) | 20260506 01:43:09.662 | INFO | Analyzing chunk frames = [7959552,7960064) | 20260506 01:43:09.662 | INFO | MemInfo: { | 20260506 01:43:09.662 | INFO | "gpu" : " 17 GB", | 20260506 01:43:09.662 | INFO | "huge" : "", | 20260506 01:43:09.662 | INFO | "huge_shared" : " 27 GB", | 20260506 01:43:09.662 | INFO | "malloc" : " 14 GB", | 20260506 01:43:09.662 | INFO | "pinned" : " 29 GB" | 20260506 01:43:09.662 | INFO | } | 20260506 01:43:09.662 | INFO | Perf report for the last 5.14328 seconds: | 20260506 01:43:09.662 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:43:09.662 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:09.662 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:09.662 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:09.662 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 01:43:09.662 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:09.663 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:09.663 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 01:43:09.663 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:09.663 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:09.663 | INFO | Uploaded 36GiB to GPU in 2.10046s (17.1391GiB/s) | 20260506 01:43:09.663 | INFO | Downloaded 34GiB to GPU in 2.03394s (16.7163GiB/s) | 20260506 01:43:14.806 | INFO | Analyzing chunk frames = [7960064,7960576) | 20260506 01:43:14.806 | INFO | MemInfo: { | 20260506 01:43:14.806 | INFO | "gpu" : " 17 GB", | 20260506 01:43:14.806 | INFO | "huge" : "", | 20260506 01:43:14.806 | INFO | "huge_shared" : " 27 GB", | 20260506 01:43:14.806 | INFO | "malloc" : " 14 GB", | 20260506 01:43:14.806 | INFO | "pinned" : " 29 GB" | 20260506 01:43:14.806 | INFO | } | 20260506 01:43:14.806 | INFO | Perf report for the last 5.14394 seconds: | 20260506 01:43:14.806 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:43:14.806 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:14.806 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:14.806 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:14.806 | INFO | ANALYSIS : 61.2%, 3.0, 95.1ms, 0 | 20260506 01:43:14.806 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:14.806 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:14.806 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 01:43:14.806 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:14.806 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:14.807 | INFO | Uploaded 36GiB to GPU in 2.09225s (17.2063GiB/s) | 20260506 01:43:14.807 | INFO | Downloaded 34GiB to GPU in 2.0306s (16.7439GiB/s) | 20260506 01:43:19.942 | INFO | Analyzing chunk frames = [7960576,7961088) | 20260506 01:43:19.942 | INFO | MemInfo: { | 20260506 01:43:19.942 | INFO | "gpu" : " 17 GB", | 20260506 01:43:19.942 | INFO | "huge" : "", | 20260506 01:43:19.942 | INFO | "huge_shared" : " 27 GB", | 20260506 01:43:19.942 | INFO | "malloc" : " 14 GB", | 20260506 01:43:19.942 | INFO | "pinned" : " 29 GB" | 20260506 01:43:19.942 | INFO | } | 20260506 01:43:19.942 | INFO | Perf report for the last 5.13593 seconds: | 20260506 01:43:19.942 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:43:19.942 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:19.942 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:19.942 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:19.942 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 01:43:19.942 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:19.942 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:19.942 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 01:43:19.942 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:19.942 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:19.942 | INFO | Uploaded 36GiB to GPU in 2.0872s (17.248GiB/s) | 20260506 01:43:19.943 | INFO | Downloaded 34GiB to GPU in 2.02593s (16.7824GiB/s) | 20260506 01:43:25.076 | INFO | Analyzing chunk frames = [7961088,7961600) | 20260506 01:43:25.076 | INFO | MemInfo: { | 20260506 01:43:25.076 | INFO | "gpu" : " 17 GB", | 20260506 01:43:25.076 | INFO | "huge" : "", | 20260506 01:43:25.076 | INFO | "huge_shared" : " 27 GB", | 20260506 01:43:25.076 | INFO | "malloc" : " 14 GB", | 20260506 01:43:25.076 | INFO | "pinned" : " 29 GB" | 20260506 01:43:25.076 | INFO | } | 20260506 01:43:25.076 | INFO | Perf report for the last 5.13406 seconds: | 20260506 01:43:25.076 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:43:25.076 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:25.076 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:25.076 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:25.076 | INFO | ANALYSIS : 61.1%, 3.0, 94.3ms, 0 | 20260506 01:43:25.076 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:25.076 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:25.076 | INFO | PRE_HQ : 26.2%, 1.0, 13.5ms, 0 | 20260506 01:43:25.076 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:25.076 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:25.077 | INFO | Uploaded 36GiB to GPU in 2.07311s (17.3652GiB/s) | 20260506 01:43:25.077 | INFO | Downloaded 34GiB to GPU in 2.04063s (16.6615GiB/s) | 20260506 01:43:30.218 | INFO | Analyzing chunk frames = [7961600,7962112) | 20260506 01:43:30.219 | INFO | MemInfo: { | 20260506 01:43:30.219 | INFO | "gpu" : " 17 GB", | 20260506 01:43:30.219 | INFO | "huge" : "", | 20260506 01:43:30.219 | INFO | "huge_shared" : " 27 GB", | 20260506 01:43:30.219 | INFO | "malloc" : " 14 GB", | 20260506 01:43:30.219 | INFO | "pinned" : " 29 GB" | 20260506 01:43:30.219 | INFO | } | 20260506 01:43:30.219 | INFO | Perf report for the last 5.14244 seconds: | 20260506 01:43:30.219 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:43:30.219 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:30.219 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:30.219 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:30.219 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260506 01:43:30.219 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:30.219 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:30.219 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 01:43:30.219 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:30.219 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:30.219 | INFO | Uploaded 36GiB to GPU in 2.03449s (17.6948GiB/s) | 20260506 01:43:30.219 | INFO | Downloaded 34GiB to GPU in 2.03683s (16.6926GiB/s) | 20260506 01:43:35.363 | INFO | Analyzing chunk frames = [7962112,7962624) | 20260506 01:43:35.363 | INFO | MemInfo: { | 20260506 01:43:35.363 | INFO | "gpu" : " 17 GB", | 20260506 01:43:35.363 | INFO | "huge" : "", | 20260506 01:43:35.363 | INFO | "huge_shared" : " 27 GB", | 20260506 01:43:35.363 | INFO | "malloc" : " 14 GB", | 20260506 01:43:35.363 | INFO | "pinned" : " 29 GB" | 20260506 01:43:35.363 | INFO | } | 20260506 01:43:35.363 | INFO | Perf report for the last 5.14422 seconds: | 20260506 01:43:35.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:43:35.363 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:35.363 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:35.363 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:35.363 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260506 01:43:35.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:35.363 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:35.363 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 01:43:35.363 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:35.363 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:35.363 | INFO | Uploaded 36GiB to GPU in 2.10735s (17.0831GiB/s) | 20260506 01:43:35.363 | INFO | Downloaded 34GiB to GPU in 2.03717s (16.6899GiB/s) | 20260506 01:43:35.363 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 01:43:35.363 | INFO | Total mem usage: 15.3 GiB | 20260506 01:43:35.363 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 01:43:35.363 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.2%) | 20260506 01:43:35.363 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 01:43:35.363 | INFO | High Water Breakdown: | 20260506 01:43:35.363 | INFO | 60.9% : 8.54 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 01:43:35.363 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 01:43:35.363 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 01:43:35.363 | INFO | ------------------------------------------ | 20260506 01:43:35.363 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:43:35.363 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 01:43:35.363 | INFO | Total mem usage: 26.5 GiB | 20260506 01:43:35.363 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 01:43:35.363 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 01:43:35.363 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260506 01:43:35.363 | INFO | High Water Breakdown: | 20260506 01:43:35.363 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:43:35.363 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:43:35.363 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:43:35.363 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:43:35.363 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:43:35.363 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:43:35.364 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 01:43:35.364 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 01:43:35.364 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:43:35.364 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 01:43:35.364 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 01:43:35.364 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 01:43:35.364 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 01:43:35.364 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 01:43:35.364 | INFO | ------------------------------------------ | 20260506 01:43:35.364 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:43:35.364 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 01:43:35.364 | INFO | Total mem usage: 25.6 GiB | 20260506 01:43:35.364 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 01:43:35.364 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 01:43:35.364 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 01:43:35.364 | INFO | High Water Breakdown: | 20260506 01:43:35.364 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 01:43:35.364 | INFO | ------------------------------------------ | 20260506 01:43:35.364 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:43:35.364 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 01:43:35.364 | INFO | Total mem usage: 15.7 GiB | 20260506 01:43:35.364 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 01:43:35.364 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 01:43:35.364 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 01:43:35.364 | INFO | High Water Breakdown: | 20260506 01:43:35.364 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 01:43:35.364 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 01:43:35.364 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 01:43:35.364 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 01:43:35.364 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 01:43:35.364 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 01:43:35.364 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 01:43:35.364 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 01:43:35.364 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 01:43:35.364 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:43:35.364 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 01:43:35.364 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:43:35.364 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 01:43:35.364 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 01:43:35.364 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 01:43:35.364 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 01:43:35.364 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:43:35.364 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 01:43:35.364 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:43:35.364 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:43:35.364 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:43:35.364 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 01:43:35.364 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 01:43:35.364 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:43:35.364 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 01:43:35.364 | INFO | ------------------------------------------ | 20260506 01:43:35.364 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:43:35.364 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 01:43:35.364 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:43:35.364 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:43:35.364 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.42e+01, 1.35e+01 / 9.13e-01 | 20260506 01:43:35.364 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.04e-01, 1.09e-02 / 4.42e-03 | 20260506 01:43:35.364 | INFO | PerfReport: 11.81%, Baseline -- 3.71e+00 / 4.48e+00, 5.77e-01 / 2.20e-01 | 20260506 01:43:35.364 | INFO | PerfReport: 3.29%, Histogram -- 1.03e+00 / 1.52e+00, 1.96e-01 / 8.69e-02 | 20260506 01:43:35.364 | INFO | PerfReport: 15.98%, DME -- 4.01e+01 / 4.41e+01, 3.78e+00 / 1.00e+00 | 20260506 01:43:35.364 | INFO | PerfReport: 57.61%, FrameLabeling -- 1.81e+01 / 1.97e+01, 1.68e+00 / 1.32e+00 | 20260506 01:43:35.364 | INFO | PerfReport: 6.96%, PulseAccumulating -- 2.19e+00 / 2.70e+00, 4.07e-01 / 2.68e-01 | 20260506 01:43:35.364 | INFO | PerfReport: 3.01%, Metrics -- 9.46e-01 / 1.24e+00, 2.56e-01 / 1.21e-01 | 20260506 01:43:35.364 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.28e-01 / 3.79e-01, 5.62e-02 / 2.82e-02 | 20260506 01:43:35.364 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 01:43:35.364 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:43:35.364 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:43:35.364 | INFO | PerfReport: Full Execution -- 9.80e+01 / -nan, 2.43e+01 / 0.00e+00 | 20260506 01:43:35.364 | INFO | PerfReport: 22.28%, Upload -- 2.18e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260506 01:43:35.364 | INFO | PerfReport: 21.89%, Download -- 2.15e+01 / 0.00e+00, 7.60e+00 / 0.00e+00 | 20260506 01:43:35.364 | INFO | PerfReport: 32.05%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 01:43:35.364 | INFO | PerfReport: 2.05%, LockUpload -- 2.01e+00 / 0.00e+00, 7.87e+00 / 0.00e+00 | 20260506 01:43:35.364 | INFO | PerfReport: 1.22%, LockDownload -- 1.20e+00 / 0.00e+00, 2.82e+00 / 0.00e+00 | 20260506 01:43:35.364 | INFO | PerfReport: 20.49%, LockCompute -- 2.01e+01 / 0.00e+00, 1.78e+01 / 0.00e+00 | 20260506 01:43:35.364 | INFO | PerfReport: 0.01%, Misc -- 1.27e-02 / 0.00e+00, 1.54e-02 / 0.00e+00 | 20260506 01:43:35.495 | INFO | Starting a round of Baz IO | 20260506 01:43:38.616 | INFO | Populating RTMetrics report | 20260506 01:43:38.616 | INFO | RTMetrics report populated | 20260506 01:43:38.616 | INFO | Opening tmp RTMetrics json file | 20260506 01:43:38.618 | INFO | Serializing RTMetrics to json | 20260506 01:43:38.619 | INFO | Serialization done, swapping tmp file now... | 20260506 01:43:38.620 | INFO | RTMetrics done | 20260506 01:43:38.698 | INFO | Finished a round of Baz IO: Wrote 1.821 GiB in 3.203 seconds | 20260506 01:43:38.698 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.30% / 0.00% / 78.70% | 20260506 01:43:38.698 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0769138% / 3.97364e-06% / 8.74201e-05% | 20260506 01:43:38.698 | INFO | Baz IO overall: | 20260506 01:43:38.698 | INFO | BytesWritten : 4483960117540 | 20260506 01:43:38.698 | INFO | NumEvents : 2106929904760 | 20260506 01:43:40.506 | INFO | Analyzing chunk frames = [7962624,7963136) | 20260506 01:43:40.506 | INFO | MemInfo: { | 20260506 01:43:40.506 | INFO | "gpu" : " 17 GB", | 20260506 01:43:40.506 | INFO | "huge" : "", | 20260506 01:43:40.506 | INFO | "huge_shared" : " 27 GB", | 20260506 01:43:40.506 | INFO | "malloc" : " 14 GB", | 20260506 01:43:40.506 | INFO | "pinned" : " 29 GB" | 20260506 01:43:40.507 | INFO | } | 20260506 01:43:40.507 | INFO | Perf report for the last 5.1436 seconds: | 20260506 01:43:40.507 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:43:40.507 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:40.507 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:40.507 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:40.507 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260506 01:43:40.507 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:40.507 | INFO | RT_METRICS : 42.8%, 1.0, 22.5ms, 0 | 20260506 01:43:40.507 | INFO | PRE_HQ : 59.2%, 1.7, 53.6ms, 0 | 20260506 01:43:40.507 | INFO | BAZWRITER : 61.5%, 3.5, 112.1ms, 0 | 20260506 01:43:40.507 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:43:40.507 | INFO | Uploaded 36GiB to GPU in 2.14101s (16.8145GiB/s) | 20260506 01:43:40.507 | INFO | Downloaded 38GiB to GPU in 2.22275s (17.0959GiB/s) | 20260506 01:43:45.644 | INFO | Analyzing chunk frames = [7963136,7963648) | 20260506 01:43:45.644 | INFO | MemInfo: { | 20260506 01:43:45.644 | INFO | "gpu" : " 17 GB", | 20260506 01:43:45.644 | INFO | "huge" : "", | 20260506 01:43:45.644 | INFO | "huge_shared" : " 27 GB", | 20260506 01:43:45.644 | INFO | "malloc" : " 14 GB", | 20260506 01:43:45.644 | INFO | "pinned" : " 29 GB" | 20260506 01:43:45.644 | INFO | } | 20260506 01:43:45.644 | INFO | Perf report for the last 5.13764 seconds: | 20260506 01:43:45.644 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:43:45.644 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:45.644 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:45.644 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:45.644 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 01:43:45.644 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:45.644 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:45.644 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 01:43:45.644 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:45.644 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:45.644 | INFO | Uploaded 36GiB to GPU in 1.96426s (18.3275GiB/s) | 20260506 01:43:45.645 | INFO | Downloaded 34GiB to GPU in 1.97464s (17.2183GiB/s) | 20260506 01:43:50.779 | INFO | Analyzing chunk frames = [7963648,7964160) | 20260506 01:43:50.779 | INFO | MemInfo: { | 20260506 01:43:50.779 | INFO | "gpu" : " 17 GB", | 20260506 01:43:50.779 | INFO | "huge" : "", | 20260506 01:43:50.779 | INFO | "huge_shared" : " 27 GB", | 20260506 01:43:50.779 | INFO | "malloc" : " 14 GB", | 20260506 01:43:50.779 | INFO | "pinned" : " 29 GB" | 20260506 01:43:50.779 | INFO | } | 20260506 01:43:50.779 | INFO | Perf report for the last 5.1346 seconds: | 20260506 01:43:50.779 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:43:50.779 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:50.779 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:50.779 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:50.779 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 01:43:50.779 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:50.779 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:50.779 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 01:43:50.779 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:50.779 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:50.779 | INFO | Uploaded 36GiB to GPU in 2.09348s (17.1962GiB/s) | 20260506 01:43:50.779 | INFO | Downloaded 34GiB to GPU in 2.02519s (16.7885GiB/s) | 20260506 01:43:55.918 | INFO | Analyzing chunk frames = [7964160,7964672) | 20260506 01:43:55.918 | INFO | MemInfo: { | 20260506 01:43:55.918 | INFO | "gpu" : " 17 GB", | 20260506 01:43:55.918 | INFO | "huge" : "", | 20260506 01:43:55.918 | INFO | "huge_shared" : " 27 GB", | 20260506 01:43:55.918 | INFO | "malloc" : " 14 GB", | 20260506 01:43:55.918 | INFO | "pinned" : " 29 GB" | 20260506 01:43:55.918 | INFO | } | 20260506 01:43:55.918 | INFO | Perf report for the last 5.13931 seconds: | 20260506 01:43:55.918 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:43:55.918 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:55.918 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:55.918 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:55.918 | INFO | ANALYSIS : 61.5%, 3.0, 95.5ms, 0 | 20260506 01:43:55.918 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:55.918 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:55.918 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:43:55.918 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:55.918 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:43:55.919 | INFO | Uploaded 36GiB to GPU in 2.13733s (16.8435GiB/s) | 20260506 01:43:55.919 | INFO | Downloaded 34GiB to GPU in 2.02118s (16.8218GiB/s) | 20260506 01:44:01.058 | INFO | Analyzing chunk frames = [7964672,7965184) | 20260506 01:44:01.058 | INFO | MemInfo: { | 20260506 01:44:01.058 | INFO | "gpu" : " 17 GB", | 20260506 01:44:01.058 | INFO | "huge" : "", | 20260506 01:44:01.058 | INFO | "huge_shared" : " 27 GB", | 20260506 01:44:01.058 | INFO | "malloc" : " 14 GB", | 20260506 01:44:01.058 | INFO | "pinned" : " 29 GB" | 20260506 01:44:01.058 | INFO | } | 20260506 01:44:01.058 | INFO | Perf report for the last 5.14026 seconds: | 20260506 01:44:01.058 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:44:01.058 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:01.058 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:01.059 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:01.059 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 01:44:01.059 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:01.059 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:01.059 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:44:01.059 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:01.059 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:01.059 | INFO | Uploaded 36GiB to GPU in 2.07619s (17.3394GiB/s) | 20260506 01:44:01.059 | INFO | Downloaded 34GiB to GPU in 2.03074s (16.7426GiB/s) | 20260506 01:44:06.199 | INFO | Analyzing chunk frames = [7965184,7965696) | 20260506 01:44:06.199 | INFO | MemInfo: { | 20260506 01:44:06.199 | INFO | "gpu" : " 17 GB", | 20260506 01:44:06.199 | INFO | "huge" : "", | 20260506 01:44:06.199 | INFO | "huge_shared" : " 27 GB", | 20260506 01:44:06.200 | INFO | "malloc" : " 14 GB", | 20260506 01:44:06.200 | INFO | "pinned" : " 29 GB" | 20260506 01:44:06.200 | INFO | } | 20260506 01:44:06.200 | INFO | Perf report for the last 5.14114 seconds: | 20260506 01:44:06.200 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:44:06.200 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:06.200 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:06.200 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:06.200 | INFO | ANALYSIS : 61.1%, 3.0, 95.5ms, 0 | 20260506 01:44:06.200 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:06.200 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:06.200 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:44:06.200 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:06.200 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:06.200 | INFO | Uploaded 36GiB to GPU in 2.07604s (17.3407GiB/s) | 20260506 01:44:06.200 | INFO | Downloaded 34GiB to GPU in 2.04308s (16.6415GiB/s) | 20260506 01:44:11.336 | INFO | Analyzing chunk frames = [7965696,7966208) | 20260506 01:44:11.336 | INFO | MemInfo: { | 20260506 01:44:11.336 | INFO | "gpu" : " 17 GB", | 20260506 01:44:11.336 | INFO | "huge" : "", | 20260506 01:44:11.336 | INFO | "huge_shared" : " 27 GB", | 20260506 01:44:11.337 | INFO | "malloc" : " 14 GB", | 20260506 01:44:11.337 | INFO | "pinned" : " 29 GB" | 20260506 01:44:11.337 | INFO | } | 20260506 01:44:11.337 | INFO | Perf report for the last 5.13706 seconds: | 20260506 01:44:11.337 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:44:11.337 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:11.337 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:11.337 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:11.337 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260506 01:44:11.337 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:11.337 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:11.337 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 01:44:11.337 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:11.337 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:11.337 | INFO | Uploaded 36GiB to GPU in 2.03125s (17.7231GiB/s) | 20260506 01:44:11.337 | INFO | Downloaded 34GiB to GPU in 2.05532s (16.5424GiB/s) | 20260506 01:44:16.474 | INFO | Analyzing chunk frames = [7966208,7966720) | 20260506 01:44:16.474 | INFO | MemInfo: { | 20260506 01:44:16.474 | INFO | "gpu" : " 17 GB", | 20260506 01:44:16.474 | INFO | "huge" : "", | 20260506 01:44:16.474 | INFO | "huge_shared" : " 27 GB", | 20260506 01:44:16.474 | INFO | "malloc" : " 14 GB", | 20260506 01:44:16.474 | INFO | "pinned" : " 29 GB" | 20260506 01:44:16.474 | INFO | } | 20260506 01:44:16.474 | INFO | Perf report for the last 5.13729 seconds: | 20260506 01:44:16.474 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:44:16.474 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:16.474 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:16.474 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:16.474 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260506 01:44:16.474 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:16.474 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:16.474 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:44:16.474 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:16.474 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:16.474 | INFO | Uploaded 36GiB to GPU in 2.14246s (16.8031GiB/s) | 20260506 01:44:16.474 | INFO | Downloaded 34GiB to GPU in 2.02475s (16.7922GiB/s) | 20260506 01:44:16.600 | INFO | Starting a round of Baz IO | 20260506 01:44:19.727 | INFO | Populating RTMetrics report | 20260506 01:44:19.728 | INFO | RTMetrics report populated | 20260506 01:44:19.728 | INFO | Opening tmp RTMetrics json file | 20260506 01:44:19.728 | INFO | Serializing RTMetrics to json | 20260506 01:44:19.730 | INFO | Serialization done, swapping tmp file now... | 20260506 01:44:19.730 | INFO | RTMetrics done | 20260506 01:44:19.808 | INFO | Finished a round of Baz IO: Wrote 1.82 GiB in 3.208 seconds | 20260506 01:44:19.808 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.30% / 0.00% / 78.70% | 20260506 01:44:19.808 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0777245% / 0% / 5.96046e-05% | 20260506 01:44:19.808 | INFO | Baz IO overall: | 20260506 01:44:19.808 | INFO | BytesWritten : 4485914101405 | 20260506 01:44:19.809 | INFO | NumEvents : 2107819021697 | 20260506 01:44:21.617 | INFO | Analyzing chunk frames = [7966720,7967232) | 20260506 01:44:21.617 | INFO | MemInfo: { | 20260506 01:44:21.617 | INFO | "gpu" : " 17 GB", | 20260506 01:44:21.617 | INFO | "huge" : "", | 20260506 01:44:21.617 | INFO | "huge_shared" : " 27 GB", | 20260506 01:44:21.617 | INFO | "malloc" : " 14 GB", | 20260506 01:44:21.617 | INFO | "pinned" : " 29 GB" | 20260506 01:44:21.617 | INFO | } | 20260506 01:44:21.617 | INFO | Perf report for the last 5.14299 seconds: | 20260506 01:44:21.617 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:44:21.617 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:21.617 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:21.617 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:21.617 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260506 01:44:21.617 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:21.617 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260506 01:44:21.617 | INFO | PRE_HQ : 59.4%, 1.7, 53.5ms, 0 | 20260506 01:44:21.617 | INFO | BAZWRITER : 61.7%, 3.5, 111.1ms, 0 | 20260506 01:44:21.617 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:44:21.617 | INFO | Uploaded 36GiB to GPU in 2.17864s (16.5241GiB/s) | 20260506 01:44:21.617 | INFO | Downloaded 38GiB to GPU in 2.23592s (16.9953GiB/s) | 20260506 01:44:26.754 | INFO | Analyzing chunk frames = [7967232,7967744) | 20260506 01:44:26.754 | INFO | MemInfo: { | 20260506 01:44:26.754 | INFO | "gpu" : " 17 GB", | 20260506 01:44:26.754 | INFO | "huge" : "", | 20260506 01:44:26.754 | INFO | "huge_shared" : " 27 GB", | 20260506 01:44:26.754 | INFO | "malloc" : " 14 GB", | 20260506 01:44:26.754 | INFO | "pinned" : " 29 GB" | 20260506 01:44:26.754 | INFO | } | 20260506 01:44:26.754 | INFO | Perf report for the last 5.13716 seconds: | 20260506 01:44:26.754 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:44:26.754 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:26.754 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:26.754 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:26.754 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260506 01:44:26.754 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:26.754 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:26.754 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 01:44:26.754 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:26.754 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:26.754 | INFO | Uploaded 36GiB to GPU in 1.962s (18.3486GiB/s) | 20260506 01:44:26.754 | INFO | Downloaded 34GiB to GPU in 1.97716s (17.1964GiB/s) | 20260506 01:44:31.891 | INFO | Analyzing chunk frames = [7967744,7968256) | 20260506 01:44:31.891 | INFO | MemInfo: { | 20260506 01:44:31.891 | INFO | "gpu" : " 17 GB", | 20260506 01:44:31.891 | INFO | "huge" : "", | 20260506 01:44:31.891 | INFO | "huge_shared" : " 27 GB", | 20260506 01:44:31.891 | INFO | "malloc" : " 14 GB", | 20260506 01:44:31.891 | INFO | "pinned" : " 29 GB" | 20260506 01:44:31.891 | INFO | } | 20260506 01:44:31.891 | INFO | Perf report for the last 5.1367 seconds: | 20260506 01:44:31.891 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:44:31.891 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:31.891 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:31.891 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:31.891 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 01:44:31.891 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:31.891 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:31.891 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 01:44:31.891 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:31.891 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:31.891 | INFO | Uploaded 36GiB to GPU in 2.08621s (17.2562GiB/s) | 20260506 01:44:31.891 | INFO | Downloaded 34GiB to GPU in 2.02465s (16.793GiB/s) | 20260506 01:44:37.026 | INFO | Analyzing chunk frames = [7968256,7968768) | 20260506 01:44:37.026 | INFO | MemInfo: { | 20260506 01:44:37.026 | INFO | "gpu" : " 17 GB", | 20260506 01:44:37.026 | INFO | "huge" : "", | 20260506 01:44:37.026 | INFO | "huge_shared" : " 27 GB", | 20260506 01:44:37.026 | INFO | "malloc" : " 14 GB", | 20260506 01:44:37.026 | INFO | "pinned" : " 29 GB" | 20260506 01:44:37.026 | INFO | } | 20260506 01:44:37.026 | INFO | Perf report for the last 5.13549 seconds: | 20260506 01:44:37.026 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:44:37.026 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:37.026 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:37.026 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:37.026 | INFO | ANALYSIS : 61.7%, 3.0, 95.7ms, 0 | 20260506 01:44:37.026 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:37.026 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:37.026 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:44:37.026 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:37.026 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:37.027 | INFO | Uploaded 36GiB to GPU in 2.14272s (16.8011GiB/s) | 20260506 01:44:37.027 | INFO | Downloaded 34GiB to GPU in 2.03159s (16.7357GiB/s) | 20260506 01:44:42.169 | INFO | Analyzing chunk frames = [7968768,7969280) | 20260506 01:44:42.169 | INFO | MemInfo: { | 20260506 01:44:42.169 | INFO | "gpu" : " 17 GB", | 20260506 01:44:42.169 | INFO | "huge" : "", | 20260506 01:44:42.169 | INFO | "huge_shared" : " 27 GB", | 20260506 01:44:42.169 | INFO | "malloc" : " 14 GB", | 20260506 01:44:42.169 | INFO | "pinned" : " 29 GB" | 20260506 01:44:42.169 | INFO | } | 20260506 01:44:42.169 | INFO | Perf report for the last 5.14294 seconds: | 20260506 01:44:42.169 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:44:42.169 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:42.169 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:42.169 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:42.169 | INFO | ANALYSIS : 61.1%, 3.0, 95.5ms, 0 | 20260506 01:44:42.169 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:42.169 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:42.169 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 01:44:42.169 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:42.169 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:42.170 | INFO | Uploaded 36GiB to GPU in 2.11583s (17.0146GiB/s) | 20260506 01:44:42.170 | INFO | Downloaded 34GiB to GPU in 2.05494s (16.5455GiB/s) | 20260506 01:44:47.321 | INFO | Analyzing chunk frames = [7969280,7969792) | 20260506 01:44:47.321 | INFO | MemInfo: { | 20260506 01:44:47.321 | INFO | "gpu" : " 17 GB", | 20260506 01:44:47.321 | INFO | "huge" : "", | 20260506 01:44:47.321 | INFO | "huge_shared" : " 27 GB", | 20260506 01:44:47.321 | INFO | "malloc" : " 14 GB", | 20260506 01:44:47.321 | INFO | "pinned" : " 29 GB" | 20260506 01:44:47.321 | INFO | } | 20260506 01:44:47.321 | INFO | Perf report for the last 5.15167 seconds: | 20260506 01:44:47.321 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:44:47.321 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:47.321 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:47.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:47.321 | INFO | ANALYSIS : 61.0%, 3.0, 95.5ms, 0 | 20260506 01:44:47.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:47.321 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:47.321 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:44:47.321 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:47.321 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:47.321 | INFO | Uploaded 36GiB to GPU in 2.07669s (17.3353GiB/s) | 20260506 01:44:47.322 | INFO | Downloaded 34GiB to GPU in 2.04532s (16.6233GiB/s) | 20260506 01:44:52.461 | INFO | Analyzing chunk frames = [7969792,7970304) | 20260506 01:44:52.461 | INFO | MemInfo: { | 20260506 01:44:52.461 | INFO | "gpu" : " 17 GB", | 20260506 01:44:52.461 | INFO | "huge" : "", | 20260506 01:44:52.461 | INFO | "huge_shared" : " 27 GB", | 20260506 01:44:52.461 | INFO | "malloc" : " 14 GB", | 20260506 01:44:52.461 | INFO | "pinned" : " 29 GB" | 20260506 01:44:52.461 | INFO | } | 20260506 01:44:52.461 | INFO | Perf report for the last 5.14038 seconds: | 20260506 01:44:52.461 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:44:52.461 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:52.461 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:52.461 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:52.461 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260506 01:44:52.461 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:52.462 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:52.462 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 01:44:52.462 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:52.462 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:52.462 | INFO | Uploaded 36GiB to GPU in 2.06616s (17.4237GiB/s) | 20260506 01:44:52.462 | INFO | Downloaded 34GiB to GPU in 2.05585s (16.5382GiB/s) | 20260506 01:44:57.588 | INFO | Analyzing chunk frames = [7970304,7970816) | 20260506 01:44:57.588 | INFO | MemInfo: { | 20260506 01:44:57.588 | INFO | "gpu" : " 17 GB", | 20260506 01:44:57.588 | INFO | "huge" : "", | 20260506 01:44:57.588 | INFO | "huge_shared" : " 27 GB", | 20260506 01:44:57.588 | INFO | "malloc" : " 14 GB", | 20260506 01:44:57.588 | INFO | "pinned" : " 29 GB" | 20260506 01:44:57.588 | INFO | } | 20260506 01:44:57.588 | INFO | Perf report for the last 5.12634 seconds: | 20260506 01:44:57.588 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:44:57.588 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:57.588 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:57.588 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:57.588 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 01:44:57.588 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:57.588 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:57.588 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260506 01:44:57.588 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:57.588 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:44:57.588 | INFO | Uploaded 36GiB to GPU in 2.06254s (17.4542GiB/s) | 20260506 01:44:57.588 | INFO | Downloaded 34GiB to GPU in 2.0241s (16.7976GiB/s) | 20260506 01:44:57.588 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 01:44:57.588 | INFO | Total mem usage: 15.3 GiB | 20260506 01:44:57.588 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 01:44:57.588 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.19%) | 20260506 01:44:57.588 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 01:44:57.588 | INFO | High Water Breakdown: | 20260506 01:44:57.588 | INFO | 60.9% : 8.54 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 01:44:57.588 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 01:44:57.588 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 01:44:57.588 | INFO | ------------------------------------------ | 20260506 01:44:57.588 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:44:57.588 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 01:44:57.588 | INFO | Total mem usage: 26.5 GiB | 20260506 01:44:57.588 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 01:44:57.588 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 01:44:57.588 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 01:44:57.588 | INFO | High Water Breakdown: | 20260506 01:44:57.588 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:44:57.588 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:44:57.589 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:44:57.589 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:44:57.589 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:44:57.589 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:44:57.589 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 01:44:57.589 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 01:44:57.589 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:44:57.589 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 01:44:57.589 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 01:44:57.589 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 01:44:57.589 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 01:44:57.589 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 01:44:57.589 | INFO | ------------------------------------------ | 20260506 01:44:57.589 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:44:57.589 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 01:44:57.589 | INFO | Total mem usage: 25.6 GiB | 20260506 01:44:57.589 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 01:44:57.589 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 01:44:57.589 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 01:44:57.589 | INFO | High Water Breakdown: | 20260506 01:44:57.589 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 01:44:57.589 | INFO | ------------------------------------------ | 20260506 01:44:57.589 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:44:57.589 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 01:44:57.589 | INFO | Total mem usage: 15.7 GiB | 20260506 01:44:57.589 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 01:44:57.589 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 01:44:57.589 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 01:44:57.589 | INFO | High Water Breakdown: | 20260506 01:44:57.589 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 01:44:57.589 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 01:44:57.589 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 01:44:57.589 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 01:44:57.589 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 01:44:57.589 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 01:44:57.589 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 01:44:57.589 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 01:44:57.589 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 01:44:57.589 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:44:57.589 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 01:44:57.589 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:44:57.589 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 01:44:57.589 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 01:44:57.589 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 01:44:57.589 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 01:44:57.589 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:44:57.589 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 01:44:57.589 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:44:57.589 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:44:57.589 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:44:57.589 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 01:44:57.589 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 01:44:57.589 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:44:57.589 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 01:44:57.589 | INFO | ------------------------------------------ | 20260506 01:44:57.589 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:44:57.589 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 01:44:57.589 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:44:57.589 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:44:57.589 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.36e+01, 1.34e+01 / 7.65e-01 | 20260506 01:44:57.589 | INFO | PerfReport: 0.29%, AggregateStats -- 9.18e-02 / 1.04e-01, 1.12e-02 / 3.09e-03 | 20260506 01:44:57.589 | INFO | PerfReport: 11.85%, Baseline -- 3.71e+00 / 4.49e+00, 5.74e-01 / 2.73e-01 | 20260506 01:44:57.589 | INFO | PerfReport: 3.29%, Histogram -- 1.03e+00 / 1.54e+00, 1.93e-01 / 9.47e-02 | 20260506 01:44:57.589 | INFO | PerfReport: 16.00%, DME -- 4.00e+01 / 4.38e+01, 3.74e+00 / 8.81e-01 | 20260506 01:44:57.589 | INFO | PerfReport: 57.39%, FrameLabeling -- 1.79e+01 / 1.93e+01, 1.47e+00 / 1.12e+00 | 20260506 01:44:57.589 | INFO | PerfReport: 7.07%, PulseAccumulating -- 2.21e+00 / 2.73e+00, 4.17e-01 / 2.53e-01 | 20260506 01:44:57.589 | INFO | PerfReport: 3.04%, Metrics -- 9.51e-01 / 1.23e+00, 2.52e-01 / 1.15e-01 | 20260506 01:44:57.590 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.28e-01 / 3.76e-01, 5.47e-02 / 2.94e-02 | 20260506 01:44:57.590 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 01:44:57.590 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:44:57.590 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:44:57.590 | INFO | PerfReport: Full Execution -- 9.78e+01 / -nan, 2.42e+01 / 0.00e+00 | 20260506 01:44:57.590 | INFO | PerfReport: 22.20%, Upload -- 2.17e+01 / 0.00e+00, 7.22e+00 / 0.00e+00 | 20260506 01:44:57.590 | INFO | PerfReport: 21.87%, Download -- 2.14e+01 / 0.00e+00, 7.52e+00 / 0.00e+00 | 20260506 01:44:57.590 | INFO | PerfReport: 31.98%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 01:44:57.590 | INFO | PerfReport: 2.05%, LockUpload -- 2.01e+00 / 0.00e+00, 7.76e+00 / 0.00e+00 | 20260506 01:44:57.590 | INFO | PerfReport: 1.29%, LockDownload -- 1.27e+00 / 0.00e+00, 2.92e+00 / 0.00e+00 | 20260506 01:44:57.590 | INFO | PerfReport: 20.59%, LockCompute -- 2.01e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260506 01:44:57.590 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.89e-02 / 0.00e+00 | 20260506 01:44:57.705 | INFO | Starting a round of Baz IO | 20260506 01:45:00.854 | INFO | Populating RTMetrics report | 20260506 01:45:00.855 | INFO | RTMetrics report populated | 20260506 01:45:00.855 | INFO | Opening tmp RTMetrics json file | 20260506 01:45:00.855 | INFO | Serializing RTMetrics to json | 20260506 01:45:00.856 | INFO | Serialization done, swapping tmp file now... | 20260506 01:45:00.857 | INFO | RTMetrics done | 20260506 01:45:00.939 | INFO | Finished a round of Baz IO: Wrote 1.818 GiB in 3.233 seconds | 20260506 01:45:00.939 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.30% / 0.00% / 78.70% | 20260506 01:45:00.939 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0772158% / 0% / 9.93411e-05% | 20260506 01:45:00.939 | INFO | Baz IO overall: | 20260506 01:45:00.939 | INFO | BytesWritten : 4487866252192 | 20260506 01:45:00.939 | INFO | NumEvents : 2108707747474 | 20260506 01:45:02.727 | INFO | Analyzing chunk frames = [7970816,7971328) | 20260506 01:45:02.727 | INFO | MemInfo: { | 20260506 01:45:02.727 | INFO | "gpu" : " 17 GB", | 20260506 01:45:02.727 | INFO | "huge" : "", | 20260506 01:45:02.727 | INFO | "huge_shared" : " 27 GB", | 20260506 01:45:02.727 | INFO | "malloc" : " 14 GB", | 20260506 01:45:02.727 | INFO | "pinned" : " 29 GB" | 20260506 01:45:02.727 | INFO | } | 20260506 01:45:02.727 | INFO | Perf report for the last 5.13907 seconds: | 20260506 01:45:02.727 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:45:02.727 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:02.727 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:02.727 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:02.727 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260506 01:45:02.727 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:02.727 | INFO | RT_METRICS : 43.0%, 1.0, 22.6ms, 0 | 20260506 01:45:02.727 | INFO | PRE_HQ : 59.6%, 1.7, 53.9ms, 0 | 20260506 01:45:02.727 | INFO | BAZWRITER : 61.9%, 3.5, 112.2ms, 0 | 20260506 01:45:02.727 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:45:02.727 | INFO | Uploaded 36GiB to GPU in 2.1834s (16.488GiB/s) | 20260506 01:45:02.727 | INFO | Downloaded 38GiB to GPU in 2.24125s (16.9548GiB/s) | 20260506 01:45:07.871 | INFO | Analyzing chunk frames = [7971328,7971840) | 20260506 01:45:07.871 | INFO | MemInfo: { | 20260506 01:45:07.871 | INFO | "gpu" : " 17 GB", | 20260506 01:45:07.871 | INFO | "huge" : "", | 20260506 01:45:07.871 | INFO | "huge_shared" : " 27 GB", | 20260506 01:45:07.871 | INFO | "malloc" : " 14 GB", | 20260506 01:45:07.871 | INFO | "pinned" : " 29 GB" | 20260506 01:45:07.871 | INFO | } | 20260506 01:45:07.871 | INFO | Perf report for the last 5.14394 seconds: | 20260506 01:45:07.871 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:45:07.871 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:07.871 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:07.871 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:07.871 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 01:45:07.871 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:07.871 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:07.871 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 01:45:07.871 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:07.871 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:07.871 | INFO | Uploaded 36GiB to GPU in 1.9832s (18.1525GiB/s) | 20260506 01:45:07.871 | INFO | Downloaded 34GiB to GPU in 1.99116s (17.0754GiB/s) | 20260506 01:45:13.009 | INFO | Analyzing chunk frames = [7971840,7972352) | 20260506 01:45:13.009 | INFO | MemInfo: { | 20260506 01:45:13.009 | INFO | "gpu" : " 17 GB", | 20260506 01:45:13.009 | INFO | "huge" : "", | 20260506 01:45:13.009 | INFO | "huge_shared" : " 27 GB", | 20260506 01:45:13.009 | INFO | "malloc" : " 14 GB", | 20260506 01:45:13.009 | INFO | "pinned" : " 29 GB" | 20260506 01:45:13.009 | INFO | } | 20260506 01:45:13.009 | INFO | Perf report for the last 5.13848 seconds: | 20260506 01:45:13.009 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:45:13.009 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:13.009 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:13.009 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:13.009 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 01:45:13.009 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:13.009 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:13.009 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 01:45:13.009 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:13.009 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:13.010 | INFO | Uploaded 36GiB to GPU in 2.08722s (17.2479GiB/s) | 20260506 01:45:13.010 | INFO | Downloaded 34GiB to GPU in 2.02443s (16.7948GiB/s) | 20260506 01:45:18.150 | INFO | Analyzing chunk frames = [7972352,7972864) | 20260506 01:45:18.150 | INFO | MemInfo: { | 20260506 01:45:18.150 | INFO | "gpu" : " 17 GB", | 20260506 01:45:18.150 | INFO | "huge" : "", | 20260506 01:45:18.150 | INFO | "huge_shared" : " 27 GB", | 20260506 01:45:18.150 | INFO | "malloc" : " 14 GB", | 20260506 01:45:18.150 | INFO | "pinned" : " 29 GB" | 20260506 01:45:18.150 | INFO | } | 20260506 01:45:18.151 | INFO | Perf report for the last 5.14141 seconds: | 20260506 01:45:18.151 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:45:18.151 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:18.151 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:18.151 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:18.151 | INFO | ANALYSIS : 61.5%, 3.0, 95.5ms, 0 | 20260506 01:45:18.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:18.151 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:18.151 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 01:45:18.151 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:18.151 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:18.151 | INFO | Uploaded 36GiB to GPU in 2.15492s (16.7059GiB/s) | 20260506 01:45:18.151 | INFO | Downloaded 34GiB to GPU in 2.05879s (16.5146GiB/s) | 20260506 01:45:23.291 | INFO | Analyzing chunk frames = [7972864,7973376) | 20260506 01:45:23.291 | INFO | MemInfo: { | 20260506 01:45:23.291 | INFO | "gpu" : " 17 GB", | 20260506 01:45:23.291 | INFO | "huge" : "", | 20260506 01:45:23.292 | INFO | "huge_shared" : " 27 GB", | 20260506 01:45:23.292 | INFO | "malloc" : " 14 GB", | 20260506 01:45:23.292 | INFO | "pinned" : " 29 GB" | 20260506 01:45:23.292 | INFO | } | 20260506 01:45:23.292 | INFO | Perf report for the last 5.141 seconds: | 20260506 01:45:23.292 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:45:23.292 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:23.292 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:23.292 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:23.292 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 01:45:23.292 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:23.292 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:23.292 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 01:45:23.292 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:23.292 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:23.292 | INFO | Uploaded 36GiB to GPU in 2.11529s (17.019GiB/s) | 20260506 01:45:23.292 | INFO | Downloaded 34GiB to GPU in 2.04813s (16.6005GiB/s) | 20260506 01:45:28.431 | INFO | Analyzing chunk frames = [7973376,7973888) | 20260506 01:45:28.431 | INFO | MemInfo: { | 20260506 01:45:28.431 | INFO | "gpu" : " 17 GB", | 20260506 01:45:28.431 | INFO | "huge" : "", | 20260506 01:45:28.431 | INFO | "huge_shared" : " 27 GB", | 20260506 01:45:28.431 | INFO | "malloc" : " 14 GB", | 20260506 01:45:28.432 | INFO | "pinned" : " 29 GB" | 20260506 01:45:28.432 | INFO | } | 20260506 01:45:28.432 | INFO | Perf report for the last 5.13989 seconds: | 20260506 01:45:28.432 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:45:28.432 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:28.432 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:28.432 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:28.432 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 01:45:28.432 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:28.432 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:28.432 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 01:45:28.432 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:28.432 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:28.432 | INFO | Uploaded 36GiB to GPU in 2.06s (17.4757GiB/s) | 20260506 01:45:28.432 | INFO | Downloaded 34GiB to GPU in 2.03166s (16.7351GiB/s) | 20260506 01:45:33.568 | INFO | Analyzing chunk frames = [7973888,7974400) | 20260506 01:45:33.568 | INFO | MemInfo: { | 20260506 01:45:33.568 | INFO | "gpu" : " 17 GB", | 20260506 01:45:33.568 | INFO | "huge" : "", | 20260506 01:45:33.568 | INFO | "huge_shared" : " 27 GB", | 20260506 01:45:33.569 | INFO | "malloc" : " 14 GB", | 20260506 01:45:33.569 | INFO | "pinned" : " 29 GB" | 20260506 01:45:33.569 | INFO | } | 20260506 01:45:33.569 | INFO | Perf report for the last 5.13709 seconds: | 20260506 01:45:33.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:45:33.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:33.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:33.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:33.569 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260506 01:45:33.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:33.569 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:33.569 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 01:45:33.569 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:33.569 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:33.569 | INFO | Uploaded 36GiB to GPU in 2.02872s (17.7452GiB/s) | 20260506 01:45:33.569 | INFO | Downloaded 34GiB to GPU in 2.05599s (16.537GiB/s) | 20260506 01:45:38.699 | INFO | Analyzing chunk frames = [7974400,7974912) | 20260506 01:45:38.699 | INFO | MemInfo: { | 20260506 01:45:38.699 | INFO | "gpu" : " 17 GB", | 20260506 01:45:38.699 | INFO | "huge" : "", | 20260506 01:45:38.699 | INFO | "huge_shared" : " 27 GB", | 20260506 01:45:38.699 | INFO | "malloc" : " 14 GB", | 20260506 01:45:38.699 | INFO | "pinned" : " 29 GB" | 20260506 01:45:38.699 | INFO | } | 20260506 01:45:38.699 | INFO | Perf report for the last 5.13058 seconds: | 20260506 01:45:38.699 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:45:38.699 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:38.699 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:38.699 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:38.699 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 01:45:38.699 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:38.699 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:38.699 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 01:45:38.699 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:38.699 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:38.700 | INFO | Uploaded 36GiB to GPU in 2.04133s (17.6355GiB/s) | 20260506 01:45:38.700 | INFO | Downloaded 34GiB to GPU in 2.01504s (16.8731GiB/s) | 20260506 01:45:38.817 | INFO | Starting a round of Baz IO | 20260506 01:45:41.960 | INFO | Populating RTMetrics report | 20260506 01:45:41.960 | INFO | RTMetrics report populated | 20260506 01:45:41.960 | INFO | Opening tmp RTMetrics json file | 20260506 01:45:41.961 | INFO | Serializing RTMetrics to json | 20260506 01:45:41.962 | INFO | Serialization done, swapping tmp file now... | 20260506 01:45:41.962 | INFO | RTMetrics done | 20260506 01:45:42.045 | INFO | Finished a round of Baz IO: Wrote 1.817 GiB in 3.228 seconds | 20260506 01:45:42.046 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.30% / 0.00% / 78.70% | 20260506 01:45:42.046 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0773072% / 0% / 0.000103315% | 20260506 01:45:42.046 | INFO | Baz IO overall: | 20260506 01:45:42.046 | INFO | BytesWritten : 4489817483157 | 20260506 01:45:42.046 | INFO | NumEvents : 2109596182729 | 20260506 01:45:43.838 | INFO | Analyzing chunk frames = [7974912,7975424) | 20260506 01:45:43.838 | INFO | MemInfo: { | 20260506 01:45:43.838 | INFO | "gpu" : " 17 GB", | 20260506 01:45:43.838 | INFO | "huge" : "", | 20260506 01:45:43.838 | INFO | "huge_shared" : " 27 GB", | 20260506 01:45:43.838 | INFO | "malloc" : " 14 GB", | 20260506 01:45:43.838 | INFO | "pinned" : " 29 GB" | 20260506 01:45:43.838 | INFO | } | 20260506 01:45:43.838 | INFO | Perf report for the last 5.13894 seconds: | 20260506 01:45:43.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:45:43.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:43.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:43.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:43.838 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260506 01:45:43.838 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:43.838 | INFO | RT_METRICS : 43.0%, 1.0, 22.5ms, 0 | 20260506 01:45:43.838 | INFO | PRE_HQ : 59.4%, 1.7, 53.8ms, 0 | 20260506 01:45:43.838 | INFO | BAZWRITER : 61.7%, 3.5, 111.6ms, 0 | 20260506 01:45:43.838 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:45:43.838 | INFO | Uploaded 36GiB to GPU in 2.14561s (16.7785GiB/s) | 20260506 01:45:43.838 | INFO | Downloaded 38GiB to GPU in 2.23888s (16.9727GiB/s) | 20260506 01:45:48.977 | INFO | Analyzing chunk frames = [7975424,7975936) | 20260506 01:45:48.977 | INFO | MemInfo: { | 20260506 01:45:48.977 | INFO | "gpu" : " 17 GB", | 20260506 01:45:48.977 | INFO | "huge" : "", | 20260506 01:45:48.978 | INFO | "huge_shared" : " 27 GB", | 20260506 01:45:48.978 | INFO | "malloc" : " 14 GB", | 20260506 01:45:48.978 | INFO | "pinned" : " 29 GB" | 20260506 01:45:48.978 | INFO | } | 20260506 01:45:48.978 | INFO | Perf report for the last 5.13944 seconds: | 20260506 01:45:48.978 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:45:48.978 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:48.978 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:48.978 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:48.978 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260506 01:45:48.978 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:48.978 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:48.978 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:45:48.978 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:48.978 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:48.978 | INFO | Uploaded 36GiB to GPU in 2.0959s (17.1764GiB/s) | 20260506 01:45:48.978 | INFO | Downloaded 34GiB to GPU in 2.03109s (16.7397GiB/s) | 20260506 01:45:54.125 | INFO | Analyzing chunk frames = [7975936,7976448) | 20260506 01:45:54.126 | INFO | MemInfo: { | 20260506 01:45:54.126 | INFO | "gpu" : " 17 GB", | 20260506 01:45:54.126 | INFO | "huge" : "", | 20260506 01:45:54.126 | INFO | "huge_shared" : " 27 GB", | 20260506 01:45:54.126 | INFO | "malloc" : " 14 GB", | 20260506 01:45:54.126 | INFO | "pinned" : " 29 GB" | 20260506 01:45:54.126 | INFO | } | 20260506 01:45:54.126 | INFO | Perf report for the last 5.14805 seconds: | 20260506 01:45:54.126 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:45:54.126 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:54.126 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:54.126 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:54.126 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 01:45:54.126 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:54.126 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:54.126 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 01:45:54.126 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:54.126 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:54.126 | INFO | Uploaded 36GiB to GPU in 2.07911s (17.3151GiB/s) | 20260506 01:45:54.126 | INFO | Downloaded 34GiB to GPU in 2.01605s (16.8646GiB/s) | 20260506 01:45:59.270 | INFO | Analyzing chunk frames = [7976448,7976960) | 20260506 01:45:59.270 | INFO | MemInfo: { | 20260506 01:45:59.270 | INFO | "gpu" : " 17 GB", | 20260506 01:45:59.270 | INFO | "huge" : "", | 20260506 01:45:59.270 | INFO | "huge_shared" : " 27 GB", | 20260506 01:45:59.270 | INFO | "malloc" : " 14 GB", | 20260506 01:45:59.270 | INFO | "pinned" : " 29 GB" | 20260506 01:45:59.270 | INFO | } | 20260506 01:45:59.270 | INFO | Perf report for the last 5.14466 seconds: | 20260506 01:45:59.270 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:45:59.270 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:59.270 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:59.270 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:59.271 | INFO | ANALYSIS : 61.0%, 3.0, 94.8ms, 0 | 20260506 01:45:59.271 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:59.271 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:59.271 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:45:59.271 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:59.271 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:45:59.271 | INFO | Uploaded 36GiB to GPU in 2.08288s (17.2837GiB/s) | 20260506 01:45:59.271 | INFO | Downloaded 34GiB to GPU in 2.0066s (16.9441GiB/s) | 20260506 01:46:04.406 | INFO | Analyzing chunk frames = [7976960,7977472) | 20260506 01:46:04.406 | INFO | MemInfo: { | 20260506 01:46:04.406 | INFO | "gpu" : " 17 GB", | 20260506 01:46:04.406 | INFO | "huge" : "", | 20260506 01:46:04.406 | INFO | "huge_shared" : " 27 GB", | 20260506 01:46:04.406 | INFO | "malloc" : " 14 GB", | 20260506 01:46:04.406 | INFO | "pinned" : " 29 GB" | 20260506 01:46:04.406 | INFO | } | 20260506 01:46:04.406 | INFO | Perf report for the last 5.1354 seconds: | 20260506 01:46:04.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:46:04.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:04.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:04.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:04.406 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 01:46:04.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:04.406 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:04.406 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260506 01:46:04.406 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:04.406 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:04.406 | INFO | Uploaded 36GiB to GPU in 2.07348s (17.3621GiB/s) | 20260506 01:46:04.406 | INFO | Downloaded 34GiB to GPU in 2.02623s (16.78GiB/s) | 20260506 01:46:09.539 | INFO | Analyzing chunk frames = [7977472,7977984) | 20260506 01:46:09.539 | INFO | MemInfo: { | 20260506 01:46:09.539 | INFO | "gpu" : " 17 GB", | 20260506 01:46:09.539 | INFO | "huge" : "", | 20260506 01:46:09.539 | INFO | "huge_shared" : " 27 GB", | 20260506 01:46:09.539 | INFO | "malloc" : " 14 GB", | 20260506 01:46:09.539 | INFO | "pinned" : " 29 GB" | 20260506 01:46:09.539 | INFO | } | 20260506 01:46:09.539 | INFO | Perf report for the last 5.13321 seconds: | 20260506 01:46:09.539 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:46:09.539 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:09.539 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:09.539 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:09.539 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 01:46:09.539 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:09.539 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:09.539 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 01:46:09.539 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:09.539 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:09.539 | INFO | Uploaded 36GiB to GPU in 2.07308s (17.3655GiB/s) | 20260506 01:46:09.540 | INFO | Downloaded 34GiB to GPU in 2.03989s (16.6676GiB/s) | 20260506 01:46:14.682 | INFO | Analyzing chunk frames = [7977984,7978496) | 20260506 01:46:14.682 | INFO | MemInfo: { | 20260506 01:46:14.682 | INFO | "gpu" : " 17 GB", | 20260506 01:46:14.682 | INFO | "huge" : "", | 20260506 01:46:14.682 | INFO | "huge_shared" : " 27 GB", | 20260506 01:46:14.682 | INFO | "malloc" : " 14 GB", | 20260506 01:46:14.682 | INFO | "pinned" : " 29 GB" | 20260506 01:46:14.682 | INFO | } | 20260506 01:46:14.682 | INFO | Perf report for the last 5.14276 seconds: | 20260506 01:46:14.682 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:46:14.682 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:14.682 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:14.682 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:14.682 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260506 01:46:14.682 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:14.682 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:14.682 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260506 01:46:14.682 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:14.682 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:14.682 | INFO | Uploaded 36GiB to GPU in 2.1102s (17.06GiB/s) | 20260506 01:46:14.682 | INFO | Downloaded 34GiB to GPU in 2.08298s (16.3228GiB/s) | 20260506 01:46:19.826 | INFO | Analyzing chunk frames = [7978496,7979008) | 20260506 01:46:19.826 | INFO | MemInfo: { | 20260506 01:46:19.826 | INFO | "gpu" : " 17 GB", | 20260506 01:46:19.826 | INFO | "huge" : "", | 20260506 01:46:19.826 | INFO | "huge_shared" : " 27 GB", | 20260506 01:46:19.826 | INFO | "malloc" : " 14 GB", | 20260506 01:46:19.826 | INFO | "pinned" : " 29 GB" | 20260506 01:46:19.826 | INFO | } | 20260506 01:46:19.826 | INFO | Perf report for the last 5.14407 seconds: | 20260506 01:46:19.826 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:46:19.826 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:19.826 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:19.826 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:19.826 | INFO | ANALYSIS : 61.3%, 3.0, 95.8ms, 0 | 20260506 01:46:19.826 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:19.826 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:19.826 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 01:46:19.826 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:19.826 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:19.826 | INFO | Uploaded 36GiB to GPU in 2.04567s (17.5982GiB/s) | 20260506 01:46:19.826 | INFO | Downloaded 34GiB to GPU in 2.0168s (16.8584GiB/s) | 20260506 01:46:19.826 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 01:46:19.826 | INFO | Total mem usage: 15.3 GiB | 20260506 01:46:19.826 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 01:46:19.826 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.18%) | 20260506 01:46:19.826 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 01:46:19.827 | INFO | High Water Breakdown: | 20260506 01:46:19.827 | INFO | 60.9% : 8.55 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 01:46:19.827 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 01:46:19.827 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 01:46:19.827 | INFO | ------------------------------------------ | 20260506 01:46:19.827 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:46:19.827 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 01:46:19.827 | INFO | Total mem usage: 26.5 GiB | 20260506 01:46:19.827 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 01:46:19.827 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 01:46:19.827 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260506 01:46:19.827 | INFO | High Water Breakdown: | 20260506 01:46:19.827 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:46:19.827 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:46:19.827 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:46:19.827 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:46:19.827 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:46:19.827 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:46:19.827 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 01:46:19.827 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 01:46:19.827 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:46:19.827 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 01:46:19.827 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 01:46:19.827 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 01:46:19.827 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 01:46:19.827 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 01:46:19.827 | INFO | ------------------------------------------ | 20260506 01:46:19.827 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:46:19.827 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 01:46:19.827 | INFO | Total mem usage: 25.6 GiB | 20260506 01:46:19.827 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 01:46:19.827 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 01:46:19.827 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 01:46:19.827 | INFO | High Water Breakdown: | 20260506 01:46:19.827 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 01:46:19.827 | INFO | ------------------------------------------ | 20260506 01:46:19.827 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:46:19.827 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 01:46:19.827 | INFO | Total mem usage: 15.7 GiB | 20260506 01:46:19.827 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 01:46:19.827 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 01:46:19.827 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 01:46:19.827 | INFO | High Water Breakdown: | 20260506 01:46:19.827 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 01:46:19.827 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 01:46:19.827 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 01:46:19.827 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 01:46:19.827 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 01:46:19.827 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 01:46:19.827 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 01:46:19.827 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 01:46:19.827 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 01:46:19.827 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:46:19.827 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 01:46:19.827 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:46:19.827 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 01:46:19.827 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 01:46:19.827 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 01:46:19.827 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 01:46:19.827 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:46:19.827 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 01:46:19.827 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:46:19.827 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:46:19.827 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:46:19.827 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 01:46:19.827 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 01:46:19.827 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:46:19.827 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 01:46:19.827 | INFO | ------------------------------------------ | 20260506 01:46:19.827 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:46:19.828 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 01:46:19.828 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:46:19.828 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:46:19.828 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.39e+01, 1.34e+01 / 7.23e-01 | 20260506 01:46:19.828 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.04e-01, 1.02e-02 / 3.40e-03 | 20260506 01:46:19.828 | INFO | PerfReport: 11.86%, Baseline -- 3.71e+00 / 4.55e+00, 5.82e-01 / 1.65e-01 | 20260506 01:46:19.828 | INFO | PerfReport: 3.30%, Histogram -- 1.03e+00 / 1.53e+00, 1.93e-01 / 6.31e-02 | 20260506 01:46:19.828 | INFO | PerfReport: 15.98%, DME -- 4.00e+01 / 4.40e+01, 3.84e+00 / 9.95e-01 | 20260506 01:46:19.828 | INFO | PerfReport: 57.44%, FrameLabeling -- 1.80e+01 / 1.93e+01, 1.51e+00 / 8.15e-01 | 20260506 01:46:19.828 | INFO | PerfReport: 7.03%, PulseAccumulating -- 2.20e+00 / 2.75e+00, 4.17e-01 / 1.67e-01 | 20260506 01:46:19.828 | INFO | PerfReport: 3.04%, Metrics -- 9.51e-01 / 1.23e+00, 2.55e-01 / 1.04e-01 | 20260506 01:46:19.828 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.28e-01 / 3.77e-01, 5.46e-02 / 2.29e-02 | 20260506 01:46:19.828 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 01:46:19.828 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:46:19.828 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:46:19.828 | INFO | PerfReport: Full Execution -- 9.79e+01 / -nan, 2.42e+01 / 0.00e+00 | 20260506 01:46:19.828 | INFO | PerfReport: 22.19%, Upload -- 2.17e+01 / 0.00e+00, 7.29e+00 / 0.00e+00 | 20260506 01:46:19.828 | INFO | PerfReport: 21.90%, Download -- 2.14e+01 / 0.00e+00, 7.57e+00 / 0.00e+00 | 20260506 01:46:19.828 | INFO | PerfReport: 31.98%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 01:46:19.828 | INFO | PerfReport: 2.05%, LockUpload -- 2.01e+00 / 0.00e+00, 7.89e+00 / 0.00e+00 | 20260506 01:46:19.828 | INFO | PerfReport: 1.29%, LockDownload -- 1.26e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260506 01:46:19.828 | INFO | PerfReport: 20.58%, LockCompute -- 2.01e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260506 01:46:19.828 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.92e-02 / 0.00e+00 | 20260506 01:46:19.952 | INFO | Starting a round of Baz IO | 20260506 01:46:23.083 | INFO | Populating RTMetrics report | 20260506 01:46:23.084 | INFO | RTMetrics report populated | 20260506 01:46:23.084 | INFO | Opening tmp RTMetrics json file | 20260506 01:46:23.084 | INFO | Serializing RTMetrics to json | 20260506 01:46:23.086 | INFO | Serialization done, swapping tmp file now... | 20260506 01:46:23.086 | INFO | RTMetrics done | 20260506 01:46:23.163 | INFO | Finished a round of Baz IO: Wrote 1.817 GiB in 3.21 seconds | 20260506 01:46:23.163 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.30% / 0.00% / 78.70% | 20260506 01:46:23.163 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0769456% / 3.97364e-06% / 8.34465e-05% | 20260506 01:46:23.163 | INFO | Baz IO overall: | 20260506 01:46:23.163 | INFO | BytesWritten : 4491768821324 | 20260506 01:46:23.163 | INFO | NumEvents : 2110484892114 | 20260506 01:46:24.959 | INFO | Analyzing chunk frames = [7979008,7979520) | 20260506 01:46:24.959 | INFO | MemInfo: { | 20260506 01:46:24.959 | INFO | "gpu" : " 17 GB", | 20260506 01:46:24.959 | INFO | "huge" : "", | 20260506 01:46:24.959 | INFO | "huge_shared" : " 27 GB", | 20260506 01:46:24.959 | INFO | "malloc" : " 14 GB", | 20260506 01:46:24.959 | INFO | "pinned" : " 29 GB" | 20260506 01:46:24.959 | INFO | } | 20260506 01:46:24.959 | INFO | Perf report for the last 5.13338 seconds: | 20260506 01:46:24.959 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:46:24.959 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:24.959 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:24.959 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:24.959 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260506 01:46:24.959 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:24.959 | INFO | RT_METRICS : 42.9%, 1.0, 22.4ms, 0 | 20260506 01:46:24.960 | INFO | PRE_HQ : 59.3%, 1.7, 53.6ms, 0 | 20260506 01:46:24.960 | INFO | BAZWRITER : 61.7%, 3.5, 111.2ms, 0 | 20260506 01:46:24.960 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:46:24.960 | INFO | Uploaded 36GiB to GPU in 2.10127s (17.1325GiB/s) | 20260506 01:46:24.960 | INFO | Downloaded 38GiB to GPU in 2.22469s (17.081GiB/s) | 20260506 01:46:30.098 | INFO | Analyzing chunk frames = [7979520,7980032) | 20260506 01:46:30.098 | INFO | MemInfo: { | 20260506 01:46:30.098 | INFO | "gpu" : " 17 GB", | 20260506 01:46:30.098 | INFO | "huge" : "", | 20260506 01:46:30.098 | INFO | "huge_shared" : " 27 GB", | 20260506 01:46:30.098 | INFO | "malloc" : " 14 GB", | 20260506 01:46:30.098 | INFO | "pinned" : " 29 GB" | 20260506 01:46:30.098 | INFO | } | 20260506 01:46:30.098 | INFO | Perf report for the last 5.13858 seconds: | 20260506 01:46:30.098 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:46:30.098 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:30.098 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:30.098 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:30.098 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260506 01:46:30.098 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:30.098 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:30.098 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 01:46:30.098 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:30.098 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:30.098 | INFO | Uploaded 36GiB to GPU in 1.94452s (18.5136GiB/s) | 20260506 01:46:30.098 | INFO | Downloaded 34GiB to GPU in 1.98024s (17.1696GiB/s) | 20260506 01:46:35.242 | INFO | Analyzing chunk frames = [7980032,7980544) | 20260506 01:46:35.242 | INFO | MemInfo: { | 20260506 01:46:35.242 | INFO | "gpu" : " 17 GB", | 20260506 01:46:35.242 | INFO | "huge" : "", | 20260506 01:46:35.242 | INFO | "huge_shared" : " 27 GB", | 20260506 01:46:35.242 | INFO | "malloc" : " 14 GB", | 20260506 01:46:35.242 | INFO | "pinned" : " 29 GB" | 20260506 01:46:35.242 | INFO | } | 20260506 01:46:35.242 | INFO | Perf report for the last 5.14407 seconds: | 20260506 01:46:35.242 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:46:35.242 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:35.242 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:35.242 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:35.242 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 01:46:35.242 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:35.242 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:35.242 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 01:46:35.242 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:35.242 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:35.242 | INFO | Uploaded 36GiB to GPU in 2.06615s (17.4237GiB/s) | 20260506 01:46:35.242 | INFO | Downloaded 34GiB to GPU in 2.01413s (16.8807GiB/s) | 20260506 01:46:40.378 | INFO | Analyzing chunk frames = [7980544,7981056) | 20260506 01:46:40.378 | INFO | MemInfo: { | 20260506 01:46:40.378 | INFO | "gpu" : " 17 GB", | 20260506 01:46:40.378 | INFO | "huge" : "", | 20260506 01:46:40.378 | INFO | "huge_shared" : " 27 GB", | 20260506 01:46:40.378 | INFO | "malloc" : " 14 GB", | 20260506 01:46:40.378 | INFO | "pinned" : " 29 GB" | 20260506 01:46:40.378 | INFO | } | 20260506 01:46:40.378 | INFO | Perf report for the last 5.13621 seconds: | 20260506 01:46:40.378 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:46:40.378 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:40.378 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:40.378 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:40.378 | INFO | ANALYSIS : 61.3%, 3.0, 95.1ms, 0 | 20260506 01:46:40.378 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:40.378 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:40.378 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:46:40.378 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:40.378 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:40.379 | INFO | Uploaded 36GiB to GPU in 2.10464s (17.1051GiB/s) | 20260506 01:46:40.379 | INFO | Downloaded 34GiB to GPU in 2.0244s (16.7951GiB/s) | 20260506 01:46:45.515 | INFO | Analyzing chunk frames = [7981056,7981568) | 20260506 01:46:45.515 | INFO | MemInfo: { | 20260506 01:46:45.515 | INFO | "gpu" : " 17 GB", | 20260506 01:46:45.515 | INFO | "huge" : "", | 20260506 01:46:45.515 | INFO | "huge_shared" : " 27 GB", | 20260506 01:46:45.515 | INFO | "malloc" : " 14 GB", | 20260506 01:46:45.515 | INFO | "pinned" : " 29 GB" | 20260506 01:46:45.515 | INFO | } | 20260506 01:46:45.515 | INFO | Perf report for the last 5.13678 seconds: | 20260506 01:46:45.515 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:46:45.515 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:45.515 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:45.515 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:45.515 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 01:46:45.515 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:45.515 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:45.515 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 01:46:45.515 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:45.515 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:45.515 | INFO | Uploaded 36GiB to GPU in 2.0841s (17.2737GiB/s) | 20260506 01:46:45.515 | INFO | Downloaded 34GiB to GPU in 2.02574s (16.784GiB/s) | 20260506 01:46:50.654 | INFO | Analyzing chunk frames = [7981568,7982080) | 20260506 01:46:50.654 | INFO | MemInfo: { | 20260506 01:46:50.654 | INFO | "gpu" : " 17 GB", | 20260506 01:46:50.654 | INFO | "huge" : "", | 20260506 01:46:50.654 | INFO | "huge_shared" : " 27 GB", | 20260506 01:46:50.654 | INFO | "malloc" : " 14 GB", | 20260506 01:46:50.654 | INFO | "pinned" : " 29 GB" | 20260506 01:46:50.654 | INFO | } | 20260506 01:46:50.654 | INFO | Perf report for the last 5.13912 seconds: | 20260506 01:46:50.654 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:46:50.654 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:50.654 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:50.654 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:50.654 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 01:46:50.654 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:50.654 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:50.654 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:46:50.654 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:50.654 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:50.654 | INFO | Uploaded 36GiB to GPU in 2.05717s (17.4997GiB/s) | 20260506 01:46:50.654 | INFO | Downloaded 34GiB to GPU in 2.03997s (16.6669GiB/s) | 20260506 01:46:55.793 | INFO | Analyzing chunk frames = [7982080,7982592) | 20260506 01:46:55.793 | INFO | MemInfo: { | 20260506 01:46:55.793 | INFO | "gpu" : " 17 GB", | 20260506 01:46:55.793 | INFO | "huge" : "", | 20260506 01:46:55.793 | INFO | "huge_shared" : " 27 GB", | 20260506 01:46:55.793 | INFO | "malloc" : " 14 GB", | 20260506 01:46:55.793 | INFO | "pinned" : " 29 GB" | 20260506 01:46:55.793 | INFO | } | 20260506 01:46:55.793 | INFO | Perf report for the last 5.13868 seconds: | 20260506 01:46:55.793 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:46:55.793 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:55.793 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:55.793 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:55.793 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260506 01:46:55.793 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:55.793 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:55.793 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 01:46:55.793 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:55.793 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:46:55.793 | INFO | Uploaded 36GiB to GPU in 2.02943s (17.739GiB/s) | 20260506 01:46:55.793 | INFO | Downloaded 34GiB to GPU in 2.03889s (16.6758GiB/s) | 20260506 01:47:00.933 | INFO | Analyzing chunk frames = [7982592,7983104) | 20260506 01:47:00.933 | INFO | MemInfo: { | 20260506 01:47:00.933 | INFO | "gpu" : " 17 GB", | 20260506 01:47:00.933 | INFO | "huge" : "", | 20260506 01:47:00.933 | INFO | "huge_shared" : " 27 GB", | 20260506 01:47:00.933 | INFO | "malloc" : " 14 GB", | 20260506 01:47:00.933 | INFO | "pinned" : " 29 GB" | 20260506 01:47:00.933 | INFO | } | 20260506 01:47:00.933 | INFO | Perf report for the last 5.1406 seconds: | 20260506 01:47:00.933 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:47:00.933 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:00.933 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:00.933 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:00.934 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 01:47:00.934 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:00.934 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:00.934 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 01:47:00.934 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:00.934 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:00.934 | INFO | Uploaded 36GiB to GPU in 2.10487s (17.1032GiB/s) | 20260506 01:47:00.934 | INFO | Downloaded 34GiB to GPU in 2.03086s (16.7417GiB/s) | 20260506 01:47:01.060 | INFO | Starting a round of Baz IO | 20260506 01:47:04.193 | INFO | Populating RTMetrics report | 20260506 01:47:04.194 | INFO | RTMetrics report populated | 20260506 01:47:04.194 | INFO | Opening tmp RTMetrics json file | 20260506 01:47:04.195 | INFO | Serializing RTMetrics to json | 20260506 01:47:04.197 | INFO | Serialization done, swapping tmp file now... | 20260506 01:47:04.197 | INFO | RTMetrics done | 20260506 01:47:04.280 | INFO | Finished a round of Baz IO: Wrote 1.817 GiB in 3.219 seconds | 20260506 01:47:04.280 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.30% / 0.00% / 78.70% | 20260506 01:47:04.280 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0771364% / 0% / 9.13938e-05% | 20260506 01:47:04.280 | INFO | Baz IO overall: | 20260506 01:47:04.280 | INFO | BytesWritten : 4493719273858 | 20260506 01:47:04.280 | INFO | NumEvents : 2111373131944 | 20260506 01:47:06.080 | INFO | Analyzing chunk frames = [7983104,7983616) | 20260506 01:47:06.080 | INFO | MemInfo: { | 20260506 01:47:06.080 | INFO | "gpu" : " 17 GB", | 20260506 01:47:06.080 | INFO | "huge" : "", | 20260506 01:47:06.080 | INFO | "huge_shared" : " 27 GB", | 20260506 01:47:06.080 | INFO | "malloc" : " 14 GB", | 20260506 01:47:06.080 | INFO | "pinned" : " 29 GB" | 20260506 01:47:06.080 | INFO | } | 20260506 01:47:06.080 | INFO | Perf report for the last 5.14698 seconds: | 20260506 01:47:06.080 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:47:06.080 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:06.080 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:06.080 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:06.081 | INFO | ANALYSIS : 62.3%, 3.0, 97.5ms, 0 | 20260506 01:47:06.081 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:06.081 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260506 01:47:06.081 | INFO | PRE_HQ : 59.5%, 1.7, 53.7ms, 0 | 20260506 01:47:06.081 | INFO | BAZWRITER : 61.8%, 3.5, 111.4ms, 0 | 20260506 01:47:06.081 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:47:06.081 | INFO | Uploaded 36GiB to GPU in 2.18056s (16.5095GiB/s) | 20260506 01:47:06.081 | INFO | Downloaded 38GiB to GPU in 2.23856s (16.9752GiB/s) | 20260506 01:47:11.215 | INFO | Analyzing chunk frames = [7983616,7984128) | 20260506 01:47:11.215 | INFO | MemInfo: { | 20260506 01:47:11.215 | INFO | "gpu" : " 17 GB", | 20260506 01:47:11.215 | INFO | "huge" : "", | 20260506 01:47:11.215 | INFO | "huge_shared" : " 27 GB", | 20260506 01:47:11.216 | INFO | "malloc" : " 14 GB", | 20260506 01:47:11.216 | INFO | "pinned" : " 29 GB" | 20260506 01:47:11.216 | INFO | } | 20260506 01:47:11.216 | INFO | Perf report for the last 5.13518 seconds: | 20260506 01:47:11.216 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:47:11.216 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:11.216 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:11.216 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:11.216 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 01:47:11.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:11.216 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:11.216 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:47:11.216 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:11.216 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:11.216 | INFO | Uploaded 36GiB to GPU in 1.94209s (18.5367GiB/s) | 20260506 01:47:11.216 | INFO | Downloaded 34GiB to GPU in 1.97702s (17.1976GiB/s) | 20260506 01:47:16.345 | INFO | Analyzing chunk frames = [7984128,7984640) | 20260506 01:47:16.345 | INFO | MemInfo: { | 20260506 01:47:16.345 | INFO | "gpu" : " 17 GB", | 20260506 01:47:16.345 | INFO | "huge" : "", | 20260506 01:47:16.345 | INFO | "huge_shared" : " 27 GB", | 20260506 01:47:16.345 | INFO | "malloc" : " 14 GB", | 20260506 01:47:16.345 | INFO | "pinned" : " 29 GB" | 20260506 01:47:16.345 | INFO | } | 20260506 01:47:16.345 | INFO | Perf report for the last 5.1297 seconds: | 20260506 01:47:16.345 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:47:16.345 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:16.345 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:16.345 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:16.345 | INFO | ANALYSIS : 61.4%, 3.0, 95.4ms, 0 | 20260506 01:47:16.345 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:16.345 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:16.345 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:47:16.345 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:16.345 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:16.346 | INFO | Uploaded 36GiB to GPU in 2.09222s (17.2066GiB/s) | 20260506 01:47:16.346 | INFO | Downloaded 34GiB to GPU in 2.02164s (16.818GiB/s) | 20260506 01:47:21.492 | INFO | Analyzing chunk frames = [7984640,7985152) | 20260506 01:47:21.492 | INFO | MemInfo: { | 20260506 01:47:21.492 | INFO | "gpu" : " 17 GB", | 20260506 01:47:21.492 | INFO | "huge" : "", | 20260506 01:47:21.492 | INFO | "huge_shared" : " 27 GB", | 20260506 01:47:21.492 | INFO | "malloc" : " 14 GB", | 20260506 01:47:21.492 | INFO | "pinned" : " 29 GB" | 20260506 01:47:21.492 | INFO | } | 20260506 01:47:21.492 | INFO | Perf report for the last 5.14675 seconds: | 20260506 01:47:21.492 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:47:21.492 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:21.492 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:21.492 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:21.492 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 01:47:21.492 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:21.492 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:21.492 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 01:47:21.492 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:21.492 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:21.492 | INFO | Uploaded 36GiB to GPU in 2.13244s (16.8821GiB/s) | 20260506 01:47:21.492 | INFO | Downloaded 34GiB to GPU in 2.02512s (16.7891GiB/s) | 20260506 01:47:26.632 | INFO | Analyzing chunk frames = [7985152,7985664) | 20260506 01:47:26.633 | INFO | MemInfo: { | 20260506 01:47:26.633 | INFO | "gpu" : " 17 GB", | 20260506 01:47:26.633 | INFO | "huge" : "", | 20260506 01:47:26.633 | INFO | "huge_shared" : " 27 GB", | 20260506 01:47:26.633 | INFO | "malloc" : " 14 GB", | 20260506 01:47:26.633 | INFO | "pinned" : " 29 GB" | 20260506 01:47:26.633 | INFO | } | 20260506 01:47:26.633 | INFO | Perf report for the last 5.14064 seconds: | 20260506 01:47:26.633 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:47:26.633 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:26.633 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:26.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:26.633 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 01:47:26.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:26.633 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:26.633 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 01:47:26.633 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:26.633 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:26.633 | INFO | Uploaded 36GiB to GPU in 2.07467s (17.3522GiB/s) | 20260506 01:47:26.633 | INFO | Downloaded 34GiB to GPU in 2.03188s (16.7332GiB/s) | 20260506 01:47:31.780 | INFO | Analyzing chunk frames = [7985664,7986176) | 20260506 01:47:31.780 | INFO | MemInfo: { | 20260506 01:47:31.780 | INFO | "gpu" : " 17 GB", | 20260506 01:47:31.780 | INFO | "huge" : "", | 20260506 01:47:31.780 | INFO | "huge_shared" : " 27 GB", | 20260506 01:47:31.780 | INFO | "malloc" : " 14 GB", | 20260506 01:47:31.780 | INFO | "pinned" : " 29 GB" | 20260506 01:47:31.780 | INFO | } | 20260506 01:47:31.780 | INFO | Perf report for the last 5.14708 seconds: | 20260506 01:47:31.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:47:31.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:31.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:31.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:31.780 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 01:47:31.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:31.780 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:31.780 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:47:31.780 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:31.780 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:31.780 | INFO | Uploaded 36GiB to GPU in 2.09223s (17.2065GiB/s) | 20260506 01:47:31.780 | INFO | Downloaded 34GiB to GPU in 2.03479s (16.7093GiB/s) | 20260506 01:47:36.909 | INFO | Analyzing chunk frames = [7986176,7986688) | 20260506 01:47:36.909 | INFO | MemInfo: { | 20260506 01:47:36.909 | INFO | "gpu" : " 17 GB", | 20260506 01:47:36.909 | INFO | "huge" : "", | 20260506 01:47:36.909 | INFO | "huge_shared" : " 27 GB", | 20260506 01:47:36.910 | INFO | "malloc" : " 14 GB", | 20260506 01:47:36.910 | INFO | "pinned" : " 29 GB" | 20260506 01:47:36.910 | INFO | } | 20260506 01:47:36.910 | INFO | Perf report for the last 5.13012 seconds: | 20260506 01:47:36.910 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:47:36.910 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:36.910 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:36.910 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:36.910 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 01:47:36.910 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:36.910 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:36.910 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 01:47:36.910 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:36.910 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:36.910 | INFO | Uploaded 36GiB to GPU in 2.022s (17.8042GiB/s) | 20260506 01:47:36.910 | INFO | Downloaded 34GiB to GPU in 2.05818s (16.5194GiB/s) | 20260506 01:47:42.049 | INFO | Analyzing chunk frames = [7986688,7987200) | 20260506 01:47:42.049 | INFO | MemInfo: { | 20260506 01:47:42.049 | INFO | "gpu" : " 17 GB", | 20260506 01:47:42.049 | INFO | "huge" : "", | 20260506 01:47:42.049 | INFO | "huge_shared" : " 27 GB", | 20260506 01:47:42.049 | INFO | "malloc" : " 14 GB", | 20260506 01:47:42.049 | INFO | "pinned" : " 29 GB" | 20260506 01:47:42.049 | INFO | } | 20260506 01:47:42.049 | INFO | Perf report for the last 5.13913 seconds: | 20260506 01:47:42.049 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:47:42.049 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:42.049 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:42.049 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:42.049 | INFO | ANALYSIS : 61.8%, 3.0, 96.6ms, 0 | 20260506 01:47:42.049 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:42.049 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:42.049 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 01:47:42.049 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:42.049 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:42.049 | INFO | Uploaded 36GiB to GPU in 2.12891s (16.91GiB/s) | 20260506 01:47:42.049 | INFO | Downloaded 34GiB to GPU in 2.02881s (16.7586GiB/s) | 20260506 01:47:42.050 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 01:47:42.050 | INFO | Total mem usage: 15.3 GiB | 20260506 01:47:42.050 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 01:47:42.050 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.2%) | 20260506 01:47:42.050 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 01:47:42.050 | INFO | High Water Breakdown: | 20260506 01:47:42.050 | INFO | 60.9% : 8.54 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 01:47:42.050 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 01:47:42.050 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 01:47:42.050 | INFO | ------------------------------------------ | 20260506 01:47:42.050 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:47:42.050 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 01:47:42.050 | INFO | Total mem usage: 26.5 GiB | 20260506 01:47:42.050 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 01:47:42.050 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 01:47:42.050 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 01:47:42.050 | INFO | High Water Breakdown: | 20260506 01:47:42.050 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:47:42.050 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:47:42.050 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:47:42.050 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:47:42.050 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:47:42.050 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:47:42.050 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 01:47:42.050 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 01:47:42.050 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:47:42.050 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 01:47:42.050 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 01:47:42.050 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 01:47:42.050 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 01:47:42.050 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 01:47:42.050 | INFO | ------------------------------------------ | 20260506 01:47:42.050 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:47:42.050 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 01:47:42.050 | INFO | Total mem usage: 25.6 GiB | 20260506 01:47:42.050 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 01:47:42.050 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 01:47:42.051 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 01:47:42.051 | INFO | High Water Breakdown: | 20260506 01:47:42.051 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 01:47:42.051 | INFO | ------------------------------------------ | 20260506 01:47:42.051 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:47:42.051 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 01:47:42.051 | INFO | Total mem usage: 15.7 GiB | 20260506 01:47:42.051 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 01:47:42.051 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 01:47:42.051 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 01:47:42.051 | INFO | High Water Breakdown: | 20260506 01:47:42.051 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 01:47:42.051 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 01:47:42.051 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 01:47:42.051 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 01:47:42.051 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 01:47:42.051 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 01:47:42.051 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 01:47:42.051 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 01:47:42.051 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 01:47:42.051 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:47:42.051 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 01:47:42.051 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:47:42.051 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 01:47:42.051 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 01:47:42.051 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 01:47:42.051 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 01:47:42.051 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:47:42.051 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 01:47:42.051 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:47:42.051 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:47:42.051 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:47:42.051 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 01:47:42.051 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 01:47:42.051 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:47:42.051 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 01:47:42.051 | INFO | ------------------------------------------ | 20260506 01:47:42.051 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:47:42.051 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 01:47:42.051 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:47:42.051 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:47:42.051 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.41e+01, 1.33e+01 / 9.90e-01 | 20260506 01:47:42.051 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.06e-01, 1.05e-02 / 1.35e-02 | 20260506 01:47:42.051 | INFO | PerfReport: 11.86%, Baseline -- 3.72e+00 / 4.48e+00, 5.84e-01 / 2.34e-01 | 20260506 01:47:42.051 | INFO | PerfReport: 3.31%, Histogram -- 1.04e+00 / 1.52e+00, 1.97e-01 / 9.89e-02 | 20260506 01:47:42.051 | INFO | PerfReport: 15.99%, DME -- 4.01e+01 / 4.43e+01, 3.98e+00 / 1.12e+00 | 20260506 01:47:42.051 | INFO | PerfReport: 57.49%, FrameLabeling -- 1.80e+01 / 1.93e+01, 1.61e+00 / 1.24e+00 | 20260506 01:47:42.051 | INFO | PerfReport: 6.99%, PulseAccumulating -- 2.19e+00 / 2.76e+00, 4.12e-01 / 2.07e-01 | 20260506 01:47:42.051 | INFO | PerfReport: 3.01%, Metrics -- 9.45e-01 / 1.23e+00, 2.54e-01 / 1.00e-01 | 20260506 01:47:42.051 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.28e-01 / 3.80e-01, 5.53e-02 / 2.65e-02 | 20260506 01:47:42.051 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 01:47:42.051 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:47:42.051 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:47:42.051 | INFO | PerfReport: Full Execution -- 9.78e+01 / -nan, 2.42e+01 / 0.00e+00 | 20260506 01:47:42.051 | INFO | PerfReport: 22.08%, Upload -- 2.16e+01 / 0.00e+00, 7.12e+00 / 0.00e+00 | 20260506 01:47:42.051 | INFO | PerfReport: 21.84%, Download -- 2.14e+01 / 0.00e+00, 7.48e+00 / 0.00e+00 | 20260506 01:47:42.051 | INFO | PerfReport: 32.09%, Compute -- 3.14e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 01:47:42.051 | INFO | PerfReport: 2.03%, LockUpload -- 1.99e+00 / 0.00e+00, 7.82e+00 / 0.00e+00 | 20260506 01:47:42.051 | INFO | PerfReport: 1.28%, LockDownload -- 1.25e+00 / 0.00e+00, 3.00e+00 / 0.00e+00 | 20260506 01:47:42.051 | INFO | PerfReport: 20.66%, LockCompute -- 2.02e+01 / 0.00e+00, 1.78e+01 / 0.00e+00 | 20260506 01:47:42.051 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.91e-02 / 0.00e+00 | 20260506 01:47:42.173 | INFO | Starting a round of Baz IO | 20260506 01:47:45.308 | INFO | Populating RTMetrics report | 20260506 01:47:45.308 | INFO | RTMetrics report populated | 20260506 01:47:45.308 | INFO | Opening tmp RTMetrics json file | 20260506 01:47:45.309 | INFO | Serializing RTMetrics to json | 20260506 01:47:45.310 | INFO | Serialization done, swapping tmp file now... | 20260506 01:47:45.311 | INFO | RTMetrics done | 20260506 01:47:45.391 | INFO | Finished a round of Baz IO: Wrote 1.816 GiB in 3.217 seconds | 20260506 01:47:45.391 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.30% / 0.00% / 78.70% | 20260506 01:47:45.391 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.077196% / 0% / 0.000115236% | 20260506 01:47:45.391 | INFO | Baz IO overall: | 20260506 01:47:45.391 | INFO | BytesWritten : 4495668817549 | 20260506 01:47:45.391 | INFO | NumEvents : 2112260944982 | 20260506 01:47:47.190 | INFO | Analyzing chunk frames = [7987200,7987712) | 20260506 01:47:47.190 | INFO | MemInfo: { | 20260506 01:47:47.191 | INFO | "gpu" : " 17 GB", | 20260506 01:47:47.191 | INFO | "huge" : "", | 20260506 01:47:47.191 | INFO | "huge_shared" : " 27 GB", | 20260506 01:47:47.191 | INFO | "malloc" : " 14 GB", | 20260506 01:47:47.191 | INFO | "pinned" : " 29 GB" | 20260506 01:47:47.191 | INFO | } | 20260506 01:47:47.191 | INFO | Perf report for the last 5.14139 seconds: | 20260506 01:47:47.191 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:47:47.191 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:47.191 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:47.191 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:47.191 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260506 01:47:47.191 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:47.191 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260506 01:47:47.191 | INFO | PRE_HQ : 59.4%, 1.7, 53.7ms, 0 | 20260506 01:47:47.191 | INFO | BAZWRITER : 61.8%, 3.5, 112.0ms, 0 | 20260506 01:47:47.191 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:47:47.191 | INFO | Uploaded 36GiB to GPU in 2.18905s (16.4455GiB/s) | 20260506 01:47:47.191 | INFO | Downloaded 38GiB to GPU in 2.24168s (16.9516GiB/s) | 20260506 01:47:52.329 | INFO | Analyzing chunk frames = [7987712,7988224) | 20260506 01:47:52.329 | INFO | MemInfo: { | 20260506 01:47:52.329 | INFO | "gpu" : " 17 GB", | 20260506 01:47:52.329 | INFO | "huge" : "", | 20260506 01:47:52.329 | INFO | "huge_shared" : " 27 GB", | 20260506 01:47:52.329 | INFO | "malloc" : " 14 GB", | 20260506 01:47:52.329 | INFO | "pinned" : " 29 GB" | 20260506 01:47:52.329 | INFO | } | 20260506 01:47:52.330 | INFO | Perf report for the last 5.13898 seconds: | 20260506 01:47:52.330 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:47:52.330 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:52.330 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:52.330 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:52.330 | INFO | ANALYSIS : 60.9%, 3.0, 95.0ms, 0 | 20260506 01:47:52.330 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:52.330 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:52.330 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 01:47:52.330 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:52.330 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:52.330 | INFO | Uploaded 36GiB to GPU in 1.98711s (18.1168GiB/s) | 20260506 01:47:52.330 | INFO | Downloaded 34GiB to GPU in 1.97863s (17.1836GiB/s) | 20260506 01:47:57.467 | INFO | Analyzing chunk frames = [7988224,7988736) | 20260506 01:47:57.467 | INFO | MemInfo: { | 20260506 01:47:57.467 | INFO | "gpu" : " 17 GB", | 20260506 01:47:57.467 | INFO | "huge" : "", | 20260506 01:47:57.467 | INFO | "huge_shared" : " 27 GB", | 20260506 01:47:57.467 | INFO | "malloc" : " 14 GB", | 20260506 01:47:57.467 | INFO | "pinned" : " 29 GB" | 20260506 01:47:57.467 | INFO | } | 20260506 01:47:57.468 | INFO | Perf report for the last 5.13798 seconds: | 20260506 01:47:57.468 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:47:57.468 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:57.468 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:57.468 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:57.468 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 01:47:57.468 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:57.468 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:57.468 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 01:47:57.468 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:57.468 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:47:57.468 | INFO | Uploaded 36GiB to GPU in 2.09537s (17.1807GiB/s) | 20260506 01:47:57.468 | INFO | Downloaded 34GiB to GPU in 2.02108s (16.8227GiB/s) | 20260506 01:48:02.603 | INFO | Analyzing chunk frames = [7988736,7989248) | 20260506 01:48:02.603 | INFO | MemInfo: { | 20260506 01:48:02.603 | INFO | "gpu" : " 17 GB", | 20260506 01:48:02.603 | INFO | "huge" : "", | 20260506 01:48:02.603 | INFO | "huge_shared" : " 27 GB", | 20260506 01:48:02.603 | INFO | "malloc" : " 14 GB", | 20260506 01:48:02.603 | INFO | "pinned" : " 29 GB" | 20260506 01:48:02.603 | INFO | } | 20260506 01:48:02.603 | INFO | Perf report for the last 5.13583 seconds: | 20260506 01:48:02.603 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:48:02.603 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:02.603 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:02.603 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:02.603 | INFO | ANALYSIS : 61.6%, 3.0, 95.6ms, 0 | 20260506 01:48:02.604 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:02.604 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:02.604 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 01:48:02.604 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:02.604 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:02.604 | INFO | Uploaded 36GiB to GPU in 2.14036s (16.8196GiB/s) | 20260506 01:48:02.604 | INFO | Downloaded 34GiB to GPU in 2.03643s (16.6959GiB/s) | 20260506 01:48:07.744 | INFO | Analyzing chunk frames = [7989248,7989760) | 20260506 01:48:07.744 | INFO | MemInfo: { | 20260506 01:48:07.744 | INFO | "gpu" : " 17 GB", | 20260506 01:48:07.744 | INFO | "huge" : "", | 20260506 01:48:07.744 | INFO | "huge_shared" : " 27 GB", | 20260506 01:48:07.744 | INFO | "malloc" : " 14 GB", | 20260506 01:48:07.744 | INFO | "pinned" : " 29 GB" | 20260506 01:48:07.744 | INFO | } | 20260506 01:48:07.744 | INFO | Perf report for the last 5.14052 seconds: | 20260506 01:48:07.744 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:48:07.744 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:07.744 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:07.744 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:07.744 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 01:48:07.744 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:07.744 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:07.744 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 01:48:07.744 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:07.744 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:07.744 | INFO | Uploaded 36GiB to GPU in 2.08489s (17.2671GiB/s) | 20260506 01:48:07.744 | INFO | Downloaded 34GiB to GPU in 2.04504s (16.6256GiB/s) | 20260506 01:48:12.887 | INFO | Analyzing chunk frames = [7989760,7990272) | 20260506 01:48:12.887 | INFO | MemInfo: { | 20260506 01:48:12.887 | INFO | "gpu" : " 17 GB", | 20260506 01:48:12.887 | INFO | "huge" : "", | 20260506 01:48:12.887 | INFO | "huge_shared" : " 27 GB", | 20260506 01:48:12.887 | INFO | "malloc" : " 14 GB", | 20260506 01:48:12.887 | INFO | "pinned" : " 29 GB" | 20260506 01:48:12.887 | INFO | } | 20260506 01:48:12.887 | INFO | Perf report for the last 5.14316 seconds: | 20260506 01:48:12.887 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:48:12.887 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:12.887 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:12.887 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:12.887 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 01:48:12.887 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:12.887 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:12.887 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 01:48:12.887 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:12.887 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:12.888 | INFO | Uploaded 36GiB to GPU in 2.06561s (17.4283GiB/s) | 20260506 01:48:12.888 | INFO | Downloaded 34GiB to GPU in 2.03095s (16.7409GiB/s) | 20260506 01:48:18.028 | INFO | Analyzing chunk frames = [7990272,7990784) | 20260506 01:48:18.028 | INFO | MemInfo: { | 20260506 01:48:18.028 | INFO | "gpu" : " 17 GB", | 20260506 01:48:18.028 | INFO | "huge" : "", | 20260506 01:48:18.028 | INFO | "huge_shared" : " 27 GB", | 20260506 01:48:18.028 | INFO | "malloc" : " 14 GB", | 20260506 01:48:18.028 | INFO | "pinned" : " 29 GB" | 20260506 01:48:18.028 | INFO | } | 20260506 01:48:18.028 | INFO | Perf report for the last 5.14118 seconds: | 20260506 01:48:18.028 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:48:18.028 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:18.028 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:18.029 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:18.029 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260506 01:48:18.029 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:18.029 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:18.029 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 01:48:18.029 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:18.029 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:18.029 | INFO | Uploaded 36GiB to GPU in 2.04724s (17.5846GiB/s) | 20260506 01:48:18.029 | INFO | Downloaded 34GiB to GPU in 2.05791s (16.5216GiB/s) | 20260506 01:48:23.165 | INFO | Analyzing chunk frames = [7990784,7991296) | 20260506 01:48:23.165 | INFO | MemInfo: { | 20260506 01:48:23.165 | INFO | "gpu" : " 17 GB", | 20260506 01:48:23.165 | INFO | "huge" : "", | 20260506 01:48:23.165 | INFO | "huge_shared" : " 27 GB", | 20260506 01:48:23.165 | INFO | "malloc" : " 14 GB", | 20260506 01:48:23.165 | INFO | "pinned" : " 29 GB" | 20260506 01:48:23.166 | INFO | } | 20260506 01:48:23.166 | INFO | Perf report for the last 5.13727 seconds: | 20260506 01:48:23.166 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:48:23.166 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:23.166 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:23.166 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:23.166 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 01:48:23.166 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:23.166 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:23.166 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:48:23.166 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:23.166 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:23.166 | INFO | Uploaded 36GiB to GPU in 2.0538s (17.5285GiB/s) | 20260506 01:48:23.166 | INFO | Downloaded 34GiB to GPU in 2.01373s (16.8841GiB/s) | 20260506 01:48:23.283 | INFO | Starting a round of Baz IO | 20260506 01:48:26.430 | INFO | Populating RTMetrics report | 20260506 01:48:26.430 | INFO | RTMetrics report populated | 20260506 01:48:26.430 | INFO | Opening tmp RTMetrics json file | 20260506 01:48:26.431 | INFO | Serializing RTMetrics to json | 20260506 01:48:26.432 | INFO | Serialization done, swapping tmp file now... | 20260506 01:48:26.432 | INFO | RTMetrics done | 20260506 01:48:26.507 | INFO | Finished a round of Baz IO: Wrote 1.816 GiB in 3.223 seconds | 20260506 01:48:26.507 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.30% / 0.00% / 78.70% | 20260506 01:48:26.507 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0770688% / 0% / 9.53674e-05% | 20260506 01:48:26.507 | INFO | Baz IO overall: | 20260506 01:48:26.507 | INFO | BytesWritten : 4497618274101 | 20260506 01:48:26.507 | INFO | NumEvents : 2113148878513 | 20260506 01:48:28.298 | INFO | Analyzing chunk frames = [7991296,7991808) | 20260506 01:48:28.298 | INFO | MemInfo: { | 20260506 01:48:28.298 | INFO | "gpu" : " 17 GB", | 20260506 01:48:28.298 | INFO | "huge" : "", | 20260506 01:48:28.298 | INFO | "huge_shared" : " 27 GB", | 20260506 01:48:28.298 | INFO | "malloc" : " 14 GB", | 20260506 01:48:28.298 | INFO | "pinned" : " 29 GB" | 20260506 01:48:28.298 | INFO | } | 20260506 01:48:28.298 | INFO | Perf report for the last 5.13268 seconds: | 20260506 01:48:28.298 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:48:28.298 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:28.298 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:28.298 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:28.298 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260506 01:48:28.298 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:28.298 | INFO | RT_METRICS : 43.2%, 1.0, 22.6ms, 0 | 20260506 01:48:28.298 | INFO | PRE_HQ : 59.8%, 1.7, 53.8ms, 0 | 20260506 01:48:28.298 | INFO | BAZWRITER : 61.9%, 3.5, 111.8ms, 0 | 20260506 01:48:28.298 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:48:28.299 | INFO | Uploaded 36GiB to GPU in 2.18251s (16.4948GiB/s) | 20260506 01:48:28.299 | INFO | Downloaded 38GiB to GPU in 2.24459s (16.9296GiB/s) | 20260506 01:48:33.445 | INFO | Analyzing chunk frames = [7991808,7992320) | 20260506 01:48:33.445 | INFO | MemInfo: { | 20260506 01:48:33.445 | INFO | "gpu" : " 17 GB", | 20260506 01:48:33.445 | INFO | "huge" : "", | 20260506 01:48:33.445 | INFO | "huge_shared" : " 27 GB", | 20260506 01:48:33.445 | INFO | "malloc" : " 14 GB", | 20260506 01:48:33.445 | INFO | "pinned" : " 29 GB" | 20260506 01:48:33.445 | INFO | } | 20260506 01:48:33.445 | INFO | Perf report for the last 5.14679 seconds: | 20260506 01:48:33.445 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:48:33.445 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:33.445 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:33.445 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:33.445 | INFO | ANALYSIS : 61.3%, 3.0, 95.8ms, 0 | 20260506 01:48:33.445 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:33.445 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:33.445 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 01:48:33.445 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:33.445 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:33.445 | INFO | Uploaded 36GiB to GPU in 2.02404s (17.7862GiB/s) | 20260506 01:48:33.445 | INFO | Downloaded 34GiB to GPU in 2.00852s (16.9279GiB/s) | 20260506 01:48:38.589 | INFO | Analyzing chunk frames = [7992320,7992832) | 20260506 01:48:38.589 | INFO | MemInfo: { | 20260506 01:48:38.589 | INFO | "gpu" : " 17 GB", | 20260506 01:48:38.589 | INFO | "huge" : "", | 20260506 01:48:38.589 | INFO | "huge_shared" : " 27 GB", | 20260506 01:48:38.589 | INFO | "malloc" : " 14 GB", | 20260506 01:48:38.589 | INFO | "pinned" : " 29 GB" | 20260506 01:48:38.589 | INFO | } | 20260506 01:48:38.589 | INFO | Perf report for the last 5.14386 seconds: | 20260506 01:48:38.589 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:48:38.589 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:38.589 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:38.589 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:38.589 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 01:48:38.589 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:38.589 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:38.589 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 01:48:38.589 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:38.589 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:38.589 | INFO | Uploaded 36GiB to GPU in 2.09534s (17.181GiB/s) | 20260506 01:48:38.589 | INFO | Downloaded 34GiB to GPU in 2.03303s (16.7238GiB/s) | 20260506 01:48:43.727 | INFO | Analyzing chunk frames = [7992832,7993344) | 20260506 01:48:43.727 | INFO | MemInfo: { | 20260506 01:48:43.727 | INFO | "gpu" : " 17 GB", | 20260506 01:48:43.727 | INFO | "huge" : "", | 20260506 01:48:43.727 | INFO | "huge_shared" : " 27 GB", | 20260506 01:48:43.727 | INFO | "malloc" : " 14 GB", | 20260506 01:48:43.727 | INFO | "pinned" : " 29 GB" | 20260506 01:48:43.727 | INFO | } | 20260506 01:48:43.727 | INFO | Perf report for the last 5.1381 seconds: | 20260506 01:48:43.727 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:48:43.727 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:43.727 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:43.727 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:43.727 | INFO | ANALYSIS : 61.2%, 3.0, 95.0ms, 0 | 20260506 01:48:43.727 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:43.727 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:43.727 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 01:48:43.727 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:43.727 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:43.727 | INFO | Uploaded 36GiB to GPU in 2.08499s (17.2663GiB/s) | 20260506 01:48:43.727 | INFO | Downloaded 34GiB to GPU in 2.017s (16.8567GiB/s) | 20260506 01:48:48.863 | INFO | Analyzing chunk frames = [7993344,7993856) | 20260506 01:48:48.863 | INFO | MemInfo: { | 20260506 01:48:48.863 | INFO | "gpu" : " 17 GB", | 20260506 01:48:48.863 | INFO | "huge" : "", | 20260506 01:48:48.863 | INFO | "huge_shared" : " 27 GB", | 20260506 01:48:48.863 | INFO | "malloc" : " 14 GB", | 20260506 01:48:48.863 | INFO | "pinned" : " 29 GB" | 20260506 01:48:48.863 | INFO | } | 20260506 01:48:48.864 | INFO | Perf report for the last 5.13645 seconds: | 20260506 01:48:48.864 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:48:48.864 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:48.864 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:48.864 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:48.864 | INFO | ANALYSIS : 60.7%, 3.0, 94.8ms, 0 | 20260506 01:48:48.864 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:48.864 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:48.864 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:48:48.864 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:48.864 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:48.864 | INFO | Uploaded 36GiB to GPU in 2.09811s (17.1583GiB/s) | 20260506 01:48:48.864 | INFO | Downloaded 34GiB to GPU in 2.0316s (16.7356GiB/s) | 20260506 01:48:54.000 | INFO | Analyzing chunk frames = [7993856,7994368) | 20260506 01:48:54.000 | INFO | MemInfo: { | 20260506 01:48:54.000 | INFO | "gpu" : " 17 GB", | 20260506 01:48:54.000 | INFO | "huge" : "", | 20260506 01:48:54.000 | INFO | "huge_shared" : " 27 GB", | 20260506 01:48:54.000 | INFO | "malloc" : " 14 GB", | 20260506 01:48:54.000 | INFO | "pinned" : " 29 GB" | 20260506 01:48:54.000 | INFO | } | 20260506 01:48:54.000 | INFO | Perf report for the last 5.13678 seconds: | 20260506 01:48:54.000 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:48:54.000 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:54.001 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:54.001 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:54.001 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 01:48:54.001 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:54.001 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:54.001 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:48:54.001 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:54.001 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:54.001 | INFO | Uploaded 36GiB to GPU in 2.07102s (17.3827GiB/s) | 20260506 01:48:54.001 | INFO | Downloaded 34GiB to GPU in 2.03376s (16.7178GiB/s) | 20260506 01:48:59.142 | INFO | Analyzing chunk frames = [7994368,7994880) | 20260506 01:48:59.142 | INFO | MemInfo: { | 20260506 01:48:59.142 | INFO | "gpu" : " 17 GB", | 20260506 01:48:59.142 | INFO | "huge" : "", | 20260506 01:48:59.142 | INFO | "huge_shared" : " 27 GB", | 20260506 01:48:59.142 | INFO | "malloc" : " 14 GB", | 20260506 01:48:59.142 | INFO | "pinned" : " 29 GB" | 20260506 01:48:59.142 | INFO | } | 20260506 01:48:59.142 | INFO | Perf report for the last 5.14211 seconds: | 20260506 01:48:59.142 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:48:59.142 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:59.142 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:59.142 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:59.143 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260506 01:48:59.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:59.143 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:59.143 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 01:48:59.143 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:59.143 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:48:59.143 | INFO | Uploaded 36GiB to GPU in 2.05693s (17.5019GiB/s) | 20260506 01:48:59.143 | INFO | Downloaded 34GiB to GPU in 2.06645s (16.4533GiB/s) | 20260506 01:49:04.288 | INFO | Analyzing chunk frames = [7994880,7995392) | 20260506 01:49:04.288 | INFO | MemInfo: { | 20260506 01:49:04.288 | INFO | "gpu" : " 17 GB", | 20260506 01:49:04.288 | INFO | "huge" : "", | 20260506 01:49:04.288 | INFO | "huge_shared" : " 27 GB", | 20260506 01:49:04.288 | INFO | "malloc" : " 14 GB", | 20260506 01:49:04.288 | INFO | "pinned" : " 29 GB" | 20260506 01:49:04.288 | INFO | } | 20260506 01:49:04.288 | INFO | Perf report for the last 5.14541 seconds: | 20260506 01:49:04.288 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:49:04.288 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:04.288 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:04.288 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:04.288 | INFO | ANALYSIS : 61.3%, 3.0, 95.8ms, 0 | 20260506 01:49:04.288 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:04.288 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:04.288 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:49:04.288 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:04.288 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:04.288 | INFO | Uploaded 36GiB to GPU in 2.03222s (17.7146GiB/s) | 20260506 01:49:04.288 | INFO | Downloaded 34GiB to GPU in 2.01464s (16.8765GiB/s) | 20260506 01:49:04.288 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 01:49:04.288 | INFO | Total mem usage: 15.3 GiB | 20260506 01:49:04.288 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 01:49:04.288 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.21%) | 20260506 01:49:04.288 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 01:49:04.288 | INFO | High Water Breakdown: | 20260506 01:49:04.288 | INFO | 60.9% : 8.54 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 01:49:04.288 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 01:49:04.288 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 01:49:04.288 | INFO | ------------------------------------------ | 20260506 01:49:04.288 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:49:04.288 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 01:49:04.288 | INFO | Total mem usage: 26.5 GiB | 20260506 01:49:04.288 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 01:49:04.289 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 01:49:04.289 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 01:49:04.289 | INFO | High Water Breakdown: | 20260506 01:49:04.289 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:49:04.289 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:49:04.289 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:49:04.289 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:49:04.289 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:49:04.289 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:49:04.289 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 01:49:04.289 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 01:49:04.289 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:49:04.289 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 01:49:04.289 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 01:49:04.289 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 01:49:04.289 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 01:49:04.289 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 01:49:04.289 | INFO | ------------------------------------------ | 20260506 01:49:04.289 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:49:04.289 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 01:49:04.289 | INFO | Total mem usage: 25.6 GiB | 20260506 01:49:04.289 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 01:49:04.289 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 01:49:04.289 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 01:49:04.289 | INFO | High Water Breakdown: | 20260506 01:49:04.289 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 01:49:04.289 | INFO | ------------------------------------------ | 20260506 01:49:04.289 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:49:04.289 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 01:49:04.289 | INFO | Total mem usage: 15.7 GiB | 20260506 01:49:04.289 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 01:49:04.289 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 01:49:04.289 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 01:49:04.289 | INFO | High Water Breakdown: | 20260506 01:49:04.289 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 01:49:04.289 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 01:49:04.289 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 01:49:04.289 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 01:49:04.289 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 01:49:04.289 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 01:49:04.289 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 01:49:04.289 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 01:49:04.289 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 01:49:04.289 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:49:04.289 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 01:49:04.289 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:49:04.289 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 01:49:04.289 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 01:49:04.289 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 01:49:04.289 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 01:49:04.289 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:49:04.289 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 01:49:04.289 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:49:04.289 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:49:04.289 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:49:04.289 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 01:49:04.289 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 01:49:04.289 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:49:04.289 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 01:49:04.289 | INFO | ------------------------------------------ | 20260506 01:49:04.289 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:49:04.289 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 01:49:04.289 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:49:04.289 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:49:04.289 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.39e+01, 1.33e+01 / 8.25e-01 | 20260506 01:49:04.289 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.04e-01, 1.07e-02 / 4.97e-03 | 20260506 01:49:04.289 | INFO | PerfReport: 11.85%, Baseline -- 3.71e+00 / 4.49e+00, 5.82e-01 / 2.41e-01 | 20260506 01:49:04.289 | INFO | PerfReport: 3.30%, Histogram -- 1.03e+00 / 1.53e+00, 1.95e-01 / 6.39e-02 | 20260506 01:49:04.289 | INFO | PerfReport: 16.00%, DME -- 4.01e+01 / 4.41e+01, 3.85e+00 / 1.00e+00 | 20260506 01:49:04.289 | INFO | PerfReport: 57.45%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.53e+00 / 1.05e+00 | 20260506 01:49:04.289 | INFO | PerfReport: 7.03%, PulseAccumulating -- 2.20e+00 / 2.71e+00, 4.19e-01 / 2.17e-01 | 20260506 01:49:04.289 | INFO | PerfReport: 3.03%, Metrics -- 9.47e-01 / 1.21e+00, 2.53e-01 / 8.94e-02 | 20260506 01:49:04.289 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.28e-01 / 3.74e-01, 5.46e-02 / 2.43e-02 | 20260506 01:49:04.289 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 01:49:04.289 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:49:04.289 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:49:04.289 | INFO | PerfReport: Full Execution -- 9.78e+01 / -nan, 2.42e+01 / 0.00e+00 | 20260506 01:49:04.289 | INFO | PerfReport: 22.18%, Upload -- 2.17e+01 / 0.00e+00, 7.30e+00 / 0.00e+00 | 20260506 01:49:04.289 | INFO | PerfReport: 21.89%, Download -- 2.14e+01 / 0.00e+00, 7.57e+00 / 0.00e+00 | 20260506 01:49:04.289 | INFO | PerfReport: 32.01%, Compute -- 3.13e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 01:49:04.289 | INFO | PerfReport: 2.03%, LockUpload -- 1.98e+00 / 0.00e+00, 7.75e+00 / 0.00e+00 | 20260506 01:49:04.290 | INFO | PerfReport: 1.29%, LockDownload -- 1.26e+00 / 0.00e+00, 2.92e+00 / 0.00e+00 | 20260506 01:49:04.290 | INFO | PerfReport: 20.59%, LockCompute -- 2.01e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 01:49:04.290 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.59e-02 / 0.00e+00 | 20260506 01:49:04.414 | INFO | Starting a round of Baz IO | 20260506 01:49:07.543 | INFO | Populating RTMetrics report | 20260506 01:49:07.543 | INFO | RTMetrics report populated | 20260506 01:49:07.543 | INFO | Opening tmp RTMetrics json file | 20260506 01:49:07.544 | INFO | Serializing RTMetrics to json | 20260506 01:49:07.545 | INFO | Serialization done, swapping tmp file now... | 20260506 01:49:07.545 | INFO | RTMetrics done | 20260506 01:49:07.624 | INFO | Finished a round of Baz IO: Wrote 1.815 GiB in 3.21 seconds | 20260506 01:49:07.624 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.30% / 0.00% / 78.70% | 20260506 01:49:07.624 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.077184% / 0% / 9.53674e-05% | 20260506 01:49:07.624 | INFO | Baz IO overall: | 20260506 01:49:07.624 | INFO | BytesWritten : 4499566627030 | 20260506 01:49:07.624 | INFO | NumEvents : 2114036077398 | 20260506 01:49:09.420 | INFO | Analyzing chunk frames = [7995392,7995904) | 20260506 01:49:09.420 | INFO | MemInfo: { | 20260506 01:49:09.420 | INFO | "gpu" : " 17 GB", | 20260506 01:49:09.420 | INFO | "huge" : "", | 20260506 01:49:09.420 | INFO | "huge_shared" : " 27 GB", | 20260506 01:49:09.420 | INFO | "malloc" : " 14 GB", | 20260506 01:49:09.420 | INFO | "pinned" : " 29 GB" | 20260506 01:49:09.420 | INFO | } | 20260506 01:49:09.420 | INFO | Perf report for the last 5.1319 seconds: | 20260506 01:49:09.420 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:49:09.420 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:09.420 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:09.420 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:09.420 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 01:49:09.420 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:09.420 | INFO | RT_METRICS : 43.1%, 1.0, 22.5ms, 0 | 20260506 01:49:09.420 | INFO | PRE_HQ : 59.2%, 1.8, 53.9ms, 0 | 20260506 01:49:09.420 | INFO | BAZWRITER : 61.7%, 3.5, 112.2ms, 0 | 20260506 01:49:09.420 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:49:09.420 | INFO | Uploaded 36GiB to GPU in 2.11865s (16.992GiB/s) | 20260506 01:49:09.420 | INFO | Downloaded 38GiB to GPU in 2.22308s (17.0934GiB/s) | 20260506 01:49:14.551 | INFO | Analyzing chunk frames = [7995904,7996416) | 20260506 01:49:14.551 | INFO | MemInfo: { | 20260506 01:49:14.551 | INFO | "gpu" : " 17 GB", | 20260506 01:49:14.551 | INFO | "huge" : "", | 20260506 01:49:14.551 | INFO | "huge_shared" : " 27 GB", | 20260506 01:49:14.551 | INFO | "malloc" : " 14 GB", | 20260506 01:49:14.551 | INFO | "pinned" : " 29 GB" | 20260506 01:49:14.551 | INFO | } | 20260506 01:49:14.551 | INFO | Perf report for the last 5.13099 seconds: | 20260506 01:49:14.551 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:49:14.551 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:14.551 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:14.551 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:14.551 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 01:49:14.551 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:14.551 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:14.551 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 01:49:14.551 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:14.551 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:14.551 | INFO | Uploaded 36GiB to GPU in 2.12805s (16.9169GiB/s) | 20260506 01:49:14.551 | INFO | Downloaded 34GiB to GPU in 2.04735s (16.6069GiB/s) | 20260506 01:49:19.696 | INFO | Analyzing chunk frames = [7996416,7996928) | 20260506 01:49:19.696 | INFO | MemInfo: { | 20260506 01:49:19.696 | INFO | "gpu" : " 17 GB", | 20260506 01:49:19.696 | INFO | "huge" : "", | 20260506 01:49:19.696 | INFO | "huge_shared" : " 27 GB", | 20260506 01:49:19.696 | INFO | "malloc" : " 14 GB", | 20260506 01:49:19.696 | INFO | "pinned" : " 29 GB" | 20260506 01:49:19.696 | INFO | } | 20260506 01:49:19.696 | INFO | Perf report for the last 5.14554 seconds: | 20260506 01:49:19.696 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:49:19.696 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:19.696 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:19.696 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:19.696 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 01:49:19.696 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:19.696 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:19.696 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:49:19.696 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:19.696 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:19.697 | INFO | Uploaded 36GiB to GPU in 2.14187s (16.8077GiB/s) | 20260506 01:49:19.697 | INFO | Downloaded 34GiB to GPU in 2.05488s (16.546GiB/s) | 20260506 01:49:24.843 | INFO | Analyzing chunk frames = [7996928,7997440) | 20260506 01:49:24.843 | INFO | MemInfo: { | 20260506 01:49:24.843 | INFO | "gpu" : " 17 GB", | 20260506 01:49:24.843 | INFO | "huge" : "", | 20260506 01:49:24.843 | INFO | "huge_shared" : " 27 GB", | 20260506 01:49:24.843 | INFO | "malloc" : " 14 GB", | 20260506 01:49:24.843 | INFO | "pinned" : " 29 GB" | 20260506 01:49:24.843 | INFO | } | 20260506 01:49:24.843 | INFO | Perf report for the last 5.14689 seconds: | 20260506 01:49:24.843 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:49:24.843 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:24.843 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:24.843 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:24.843 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260506 01:49:24.843 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:24.843 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:24.843 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:49:24.843 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:24.843 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:24.843 | INFO | Uploaded 36GiB to GPU in 2.09649s (17.1716GiB/s) | 20260506 01:49:24.844 | INFO | Downloaded 34GiB to GPU in 2.0287s (16.7595GiB/s) | 20260506 01:49:29.978 | INFO | Analyzing chunk frames = [7997440,7997952) | 20260506 01:49:29.978 | INFO | MemInfo: { | 20260506 01:49:29.978 | INFO | "gpu" : " 17 GB", | 20260506 01:49:29.978 | INFO | "huge" : "", | 20260506 01:49:29.978 | INFO | "huge_shared" : " 27 GB", | 20260506 01:49:29.978 | INFO | "malloc" : " 14 GB", | 20260506 01:49:29.978 | INFO | "pinned" : " 29 GB" | 20260506 01:49:29.978 | INFO | } | 20260506 01:49:29.978 | INFO | Perf report for the last 5.13497 seconds: | 20260506 01:49:29.978 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:49:29.978 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:29.978 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:29.978 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:29.978 | INFO | ANALYSIS : 60.8%, 3.0, 94.9ms, 0 | 20260506 01:49:29.978 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:29.978 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:29.978 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 01:49:29.978 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:29.978 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:29.979 | INFO | Uploaded 36GiB to GPU in 2.10566s (17.0968GiB/s) | 20260506 01:49:29.979 | INFO | Downloaded 34GiB to GPU in 2.03197s (16.7325GiB/s) | 20260506 01:49:35.111 | INFO | Analyzing chunk frames = [7997952,7998464) | 20260506 01:49:35.111 | INFO | MemInfo: { | 20260506 01:49:35.111 | INFO | "gpu" : " 17 GB", | 20260506 01:49:35.111 | INFO | "huge" : "", | 20260506 01:49:35.111 | INFO | "huge_shared" : " 27 GB", | 20260506 01:49:35.111 | INFO | "malloc" : " 14 GB", | 20260506 01:49:35.111 | INFO | "pinned" : " 29 GB" | 20260506 01:49:35.111 | INFO | } | 20260506 01:49:35.111 | INFO | Perf report for the last 5.13261 seconds: | 20260506 01:49:35.111 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:49:35.111 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:35.111 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:35.111 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:35.111 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 01:49:35.111 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:35.111 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:35.111 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 01:49:35.111 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:35.111 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:35.111 | INFO | Uploaded 36GiB to GPU in 2.07986s (17.3089GiB/s) | 20260506 01:49:35.111 | INFO | Downloaded 34GiB to GPU in 2.02612s (16.7808GiB/s) | 20260506 01:49:40.253 | INFO | Analyzing chunk frames = [7998464,7998976) | 20260506 01:49:40.253 | INFO | MemInfo: { | 20260506 01:49:40.253 | INFO | "gpu" : " 17 GB", | 20260506 01:49:40.253 | INFO | "huge" : "", | 20260506 01:49:40.253 | INFO | "huge_shared" : " 27 GB", | 20260506 01:49:40.253 | INFO | "malloc" : " 14 GB", | 20260506 01:49:40.253 | INFO | "pinned" : " 29 GB" | 20260506 01:49:40.253 | INFO | } | 20260506 01:49:40.253 | INFO | Perf report for the last 5.14244 seconds: | 20260506 01:49:40.253 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:49:40.253 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:40.253 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:40.253 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:40.253 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260506 01:49:40.253 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:40.253 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:40.253 | INFO | PRE_HQ : 27.2%, 1.0, 14.2ms, 0 | 20260506 01:49:40.253 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:40.253 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:40.254 | INFO | Uploaded 36GiB to GPU in 2.01329s (17.8811GiB/s) | 20260506 01:49:40.254 | INFO | Downloaded 34GiB to GPU in 2.0366s (16.6945GiB/s) | 20260506 01:49:45.393 | INFO | Analyzing chunk frames = [7998976,7999488) | 20260506 01:49:45.393 | INFO | MemInfo: { | 20260506 01:49:45.393 | INFO | "gpu" : " 17 GB", | 20260506 01:49:45.393 | INFO | "huge" : "", | 20260506 01:49:45.393 | INFO | "huge_shared" : " 27 GB", | 20260506 01:49:45.393 | INFO | "malloc" : " 14 GB", | 20260506 01:49:45.393 | INFO | "pinned" : " 29 GB" | 20260506 01:49:45.393 | INFO | } | 20260506 01:49:45.393 | INFO | Perf report for the last 5.13994 seconds: | 20260506 01:49:45.393 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:49:45.393 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:45.393 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:45.393 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:45.393 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 01:49:45.393 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:45.393 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:45.393 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:49:45.393 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:45.393 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:45.394 | INFO | Uploaded 36GiB to GPU in 2.12766s (16.92GiB/s) | 20260506 01:49:45.394 | INFO | Downloaded 34GiB to GPU in 2.03168s (16.7349GiB/s) | 20260506 01:49:45.519 | INFO | Starting a round of Baz IO | 20260506 01:49:48.657 | INFO | Populating RTMetrics report | 20260506 01:49:48.657 | INFO | RTMetrics report populated | 20260506 01:49:48.657 | INFO | Opening tmp RTMetrics json file | 20260506 01:49:48.658 | INFO | Serializing RTMetrics to json | 20260506 01:49:48.659 | INFO | Serialization done, swapping tmp file now... | 20260506 01:49:48.660 | INFO | RTMetrics done | 20260506 01:49:48.739 | INFO | Finished a round of Baz IO: Wrote 1.814 GiB in 3.219 seconds | 20260506 01:49:48.739 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.30% / 0.00% / 78.70% | 20260506 01:49:48.739 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0768582% / 3.97364e-06% / 5.16574e-05% | 20260506 01:49:48.739 | INFO | Baz IO overall: | 20260506 01:49:48.739 | INFO | BytesWritten : 4501514892370 | 20260506 01:49:48.739 | INFO | NumEvents : 2114923270512 | 20260506 01:49:50.538 | INFO | Analyzing chunk frames = [7999488,8000000) | 20260506 01:49:50.538 | INFO | MemInfo: { | 20260506 01:49:50.538 | INFO | "gpu" : " 17 GB", | 20260506 01:49:50.538 | INFO | "huge" : "", | 20260506 01:49:50.538 | INFO | "huge_shared" : " 27 GB", | 20260506 01:49:50.538 | INFO | "malloc" : " 14 GB", | 20260506 01:49:50.538 | INFO | "pinned" : " 29 GB" | 20260506 01:49:50.538 | INFO | } | 20260506 01:49:50.538 | INFO | Perf report for the last 5.14503 seconds: | 20260506 01:49:50.538 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:49:50.538 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:50.538 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:50.538 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:50.538 | INFO | ANALYSIS : 62.4%, 3.0, 97.6ms, 0 | 20260506 01:49:50.538 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:50.538 | INFO | RT_METRICS : 42.8%, 1.0, 22.5ms, 0 | 20260506 01:49:50.538 | INFO | PRE_HQ : 59.4%, 1.7, 53.3ms, 0 | 20260506 01:49:50.538 | INFO | BAZWRITER : 61.8%, 3.5, 111.4ms, 0 | 20260506 01:49:50.538 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:49:50.539 | INFO | Uploaded 36GiB to GPU in 2.18465s (16.4786GiB/s) | 20260506 01:49:50.539 | INFO | Downloaded 38GiB to GPU in 2.24048s (16.9606GiB/s) | 20260506 01:49:55.677 | INFO | Analyzing chunk frames = [8000000,8000512) | 20260506 01:49:55.677 | INFO | MemInfo: { | 20260506 01:49:55.677 | INFO | "gpu" : " 17 GB", | 20260506 01:49:55.677 | INFO | "huge" : "", | 20260506 01:49:55.677 | INFO | "huge_shared" : " 27 GB", | 20260506 01:49:55.677 | INFO | "malloc" : " 14 GB", | 20260506 01:49:55.677 | INFO | "pinned" : " 29 GB" | 20260506 01:49:55.677 | INFO | } | 20260506 01:49:55.677 | INFO | Perf report for the last 5.13856 seconds: | 20260506 01:49:55.677 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:49:55.677 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:55.677 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:55.677 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:55.677 | INFO | ANALYSIS : 60.8%, 3.0, 94.9ms, 0 | 20260506 01:49:55.677 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:55.677 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:55.677 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 01:49:55.677 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:55.677 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:49:55.677 | INFO | Uploaded 36GiB to GPU in 1.96906s (18.2828GiB/s) | 20260506 01:49:55.677 | INFO | Downloaded 34GiB to GPU in 1.98274s (17.148GiB/s) | 20260506 01:50:00.816 | INFO | Analyzing chunk frames = [8000512,8001024) | 20260506 01:50:00.816 | INFO | MemInfo: { | 20260506 01:50:00.816 | INFO | "gpu" : " 17 GB", | 20260506 01:50:00.816 | INFO | "huge" : "", | 20260506 01:50:00.816 | INFO | "huge_shared" : " 27 GB", | 20260506 01:50:00.816 | INFO | "malloc" : " 14 GB", | 20260506 01:50:00.816 | INFO | "pinned" : " 29 GB" | 20260506 01:50:00.816 | INFO | } | 20260506 01:50:00.816 | INFO | Perf report for the last 5.13892 seconds: | 20260506 01:50:00.816 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:50:00.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:00.816 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:00.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:00.816 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 01:50:00.816 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:00.816 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:00.816 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 01:50:00.816 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:00.816 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:00.816 | INFO | Uploaded 36GiB to GPU in 2.09283s (17.2016GiB/s) | 20260506 01:50:00.816 | INFO | Downloaded 34GiB to GPU in 2.0259s (16.7827GiB/s) | 20260506 01:50:05.953 | INFO | Analyzing chunk frames = [8001024,8001536) | 20260506 01:50:05.953 | INFO | MemInfo: { | 20260506 01:50:05.953 | INFO | "gpu" : " 17 GB", | 20260506 01:50:05.953 | INFO | "huge" : "", | 20260506 01:50:05.953 | INFO | "huge_shared" : " 27 GB", | 20260506 01:50:05.953 | INFO | "malloc" : " 14 GB", | 20260506 01:50:05.953 | INFO | "pinned" : " 29 GB" | 20260506 01:50:05.953 | INFO | } | 20260506 01:50:05.953 | INFO | Perf report for the last 5.13735 seconds: | 20260506 01:50:05.953 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:50:05.953 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:05.953 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:05.953 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:05.953 | INFO | ANALYSIS : 61.2%, 3.0, 95.0ms, 0 | 20260506 01:50:05.953 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:05.953 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:05.953 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:50:05.953 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:05.953 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:05.953 | INFO | Uploaded 36GiB to GPU in 2.09296s (17.2005GiB/s) | 20260506 01:50:05.954 | INFO | Downloaded 34GiB to GPU in 2.02052s (16.8273GiB/s) | 20260506 01:50:11.092 | INFO | Analyzing chunk frames = [8001536,8002048) | 20260506 01:50:11.092 | INFO | MemInfo: { | 20260506 01:50:11.092 | INFO | "gpu" : " 17 GB", | 20260506 01:50:11.092 | INFO | "huge" : "", | 20260506 01:50:11.092 | INFO | "huge_shared" : " 27 GB", | 20260506 01:50:11.092 | INFO | "malloc" : " 14 GB", | 20260506 01:50:11.092 | INFO | "pinned" : " 29 GB" | 20260506 01:50:11.093 | INFO | } | 20260506 01:50:11.093 | INFO | Perf report for the last 5.13958 seconds: | 20260506 01:50:11.093 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:50:11.093 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:11.093 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:11.093 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:11.093 | INFO | ANALYSIS : 60.8%, 3.0, 94.9ms, 0 | 20260506 01:50:11.093 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:11.093 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:11.093 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 01:50:11.093 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:11.093 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:11.093 | INFO | Uploaded 36GiB to GPU in 2.10346s (17.1147GiB/s) | 20260506 01:50:11.093 | INFO | Downloaded 34GiB to GPU in 2.03513s (16.7066GiB/s) | 20260506 01:50:16.233 | INFO | Analyzing chunk frames = [8002048,8002560) | 20260506 01:50:16.233 | INFO | MemInfo: { | 20260506 01:50:16.233 | INFO | "gpu" : " 17 GB", | 20260506 01:50:16.233 | INFO | "huge" : "", | 20260506 01:50:16.233 | INFO | "huge_shared" : " 27 GB", | 20260506 01:50:16.233 | INFO | "malloc" : " 14 GB", | 20260506 01:50:16.233 | INFO | "pinned" : " 29 GB" | 20260506 01:50:16.234 | INFO | } | 20260506 01:50:16.234 | INFO | Perf report for the last 5.14099 seconds: | 20260506 01:50:16.234 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:50:16.234 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:16.234 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:16.234 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:16.234 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 01:50:16.234 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:16.234 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:16.234 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 01:50:16.234 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:16.234 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:16.234 | INFO | Uploaded 36GiB to GPU in 2.07975s (17.3098GiB/s) | 20260506 01:50:16.234 | INFO | Downloaded 34GiB to GPU in 2.02803s (16.7651GiB/s) | 20260506 01:50:21.369 | INFO | Analyzing chunk frames = [8002560,8003072) | 20260506 01:50:21.369 | INFO | MemInfo: { | 20260506 01:50:21.369 | INFO | "gpu" : " 17 GB", | 20260506 01:50:21.369 | INFO | "huge" : "", | 20260506 01:50:21.369 | INFO | "huge_shared" : " 27 GB", | 20260506 01:50:21.369 | INFO | "malloc" : " 14 GB", | 20260506 01:50:21.369 | INFO | "pinned" : " 29 GB" | 20260506 01:50:21.369 | INFO | } | 20260506 01:50:21.369 | INFO | Perf report for the last 5.13519 seconds: | 20260506 01:50:21.369 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:50:21.369 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:21.369 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:21.369 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:21.369 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260506 01:50:21.369 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:21.369 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:21.369 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 01:50:21.369 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:21.369 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:21.369 | INFO | Uploaded 36GiB to GPU in 2.02891s (17.7435GiB/s) | 20260506 01:50:21.369 | INFO | Downloaded 34GiB to GPU in 2.04532s (16.6233GiB/s) | 20260506 01:50:26.509 | INFO | Analyzing chunk frames = [8003072,8003584) | 20260506 01:50:26.509 | INFO | MemInfo: { | 20260506 01:50:26.509 | INFO | "gpu" : " 17 GB", | 20260506 01:50:26.509 | INFO | "huge" : "", | 20260506 01:50:26.509 | INFO | "huge_shared" : " 27 GB", | 20260506 01:50:26.509 | INFO | "malloc" : " 14 GB", | 20260506 01:50:26.509 | INFO | "pinned" : " 29 GB" | 20260506 01:50:26.509 | INFO | } | 20260506 01:50:26.509 | INFO | Perf report for the last 5.13976 seconds: | 20260506 01:50:26.509 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:50:26.509 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:26.509 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:26.509 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:26.509 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260506 01:50:26.509 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:26.509 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:26.509 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 01:50:26.509 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:26.509 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:26.509 | INFO | Uploaded 36GiB to GPU in 2.13426s (16.8676GiB/s) | 20260506 01:50:26.509 | INFO | Downloaded 34GiB to GPU in 2.01729s (16.8543GiB/s) | 20260506 01:50:26.509 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 01:50:26.509 | INFO | Total mem usage: 15.3 GiB | 20260506 01:50:26.509 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 01:50:26.509 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.17%) | 20260506 01:50:26.509 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 01:50:26.509 | INFO | High Water Breakdown: | 20260506 01:50:26.509 | INFO | 60.9% : 8.55 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 01:50:26.509 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 01:50:26.509 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 01:50:26.509 | INFO | ------------------------------------------ | 20260506 01:50:26.509 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:50:26.509 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 01:50:26.510 | INFO | Total mem usage: 26.5 GiB | 20260506 01:50:26.510 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 01:50:26.510 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 01:50:26.510 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 01:50:26.510 | INFO | High Water Breakdown: | 20260506 01:50:26.510 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:50:26.510 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:50:26.510 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:50:26.510 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:50:26.510 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:50:26.510 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:50:26.510 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 01:50:26.510 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 01:50:26.510 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:50:26.510 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 01:50:26.510 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 01:50:26.510 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 01:50:26.510 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 01:50:26.510 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 01:50:26.510 | INFO | ------------------------------------------ | 20260506 01:50:26.510 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:50:26.510 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 01:50:26.510 | INFO | Total mem usage: 25.6 GiB | 20260506 01:50:26.510 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 01:50:26.510 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 01:50:26.510 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 01:50:26.510 | INFO | High Water Breakdown: | 20260506 01:50:26.510 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 01:50:26.510 | INFO | ------------------------------------------ | 20260506 01:50:26.510 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:50:26.510 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 01:50:26.510 | INFO | Total mem usage: 15.7 GiB | 20260506 01:50:26.510 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 01:50:26.510 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 01:50:26.510 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 01:50:26.510 | INFO | High Water Breakdown: | 20260506 01:50:26.510 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 01:50:26.510 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 01:50:26.510 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 01:50:26.510 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 01:50:26.510 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 01:50:26.510 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 01:50:26.510 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 01:50:26.510 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 01:50:26.510 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 01:50:26.510 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:50:26.510 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 01:50:26.510 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:50:26.510 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 01:50:26.510 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 01:50:26.510 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 01:50:26.510 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 01:50:26.510 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:50:26.510 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 01:50:26.510 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:50:26.510 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:50:26.510 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:50:26.510 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 01:50:26.510 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 01:50:26.510 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:50:26.510 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 01:50:26.510 | INFO | ------------------------------------------ | 20260506 01:50:26.510 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:50:26.510 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 01:50:26.510 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:50:26.510 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:50:26.510 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.40e+01, 1.34e+01 / 9.28e-01 | 20260506 01:50:26.510 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.04e-01, 1.11e-02 / 4.57e-03 | 20260506 01:50:26.510 | INFO | PerfReport: 11.83%, Baseline -- 3.70e+00 / 4.48e+00, 5.69e-01 / 2.27e-01 | 20260506 01:50:26.510 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.53e+00, 1.90e-01 / 9.73e-02 | 20260506 01:50:26.510 | INFO | PerfReport: 16.01%, DME -- 4.01e+01 / 4.42e+01, 3.97e+00 / 1.14e+00 | 20260506 01:50:26.510 | INFO | PerfReport: 57.48%, FrameLabeling -- 1.80e+01 / 1.93e+01, 1.52e+00 / 1.07e+00 | 20260506 01:50:26.510 | INFO | PerfReport: 7.04%, PulseAccumulating -- 2.20e+00 / 2.77e+00, 4.18e-01 / 2.27e-01 | 20260506 01:50:26.510 | INFO | PerfReport: 3.02%, Metrics -- 9.45e-01 / 1.23e+00, 2.52e-01 / 1.15e-01 | 20260506 01:50:26.510 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.28e-01 / 3.81e-01, 5.51e-02 / 2.39e-02 | 20260506 01:50:26.510 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 01:50:26.510 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:50:26.510 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:50:26.510 | INFO | PerfReport: Full Execution -- 9.79e+01 / -nan, 2.44e+01 / 0.00e+00 | 20260506 01:50:26.510 | INFO | PerfReport: 22.28%, Upload -- 2.18e+01 / 0.00e+00, 7.31e+00 / 0.00e+00 | 20260506 01:50:26.510 | INFO | PerfReport: 21.86%, Download -- 2.14e+01 / 0.00e+00, 7.53e+00 / 0.00e+00 | 20260506 01:50:26.510 | INFO | PerfReport: 31.96%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 01:50:26.511 | INFO | PerfReport: 2.09%, LockUpload -- 2.04e+00 / 0.00e+00, 7.95e+00 / 0.00e+00 | 20260506 01:50:26.511 | INFO | PerfReport: 1.29%, LockDownload -- 1.26e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260506 01:50:26.511 | INFO | PerfReport: 20.51%, LockCompute -- 2.01e+01 / 0.00e+00, 1.78e+01 / 0.00e+00 | 20260506 01:50:26.511 | INFO | PerfReport: 0.02%, Misc -- 1.47e-02 / 0.00e+00, 2.65e-02 / 0.00e+00 | 20260506 01:50:26.635 | INFO | Starting a round of Baz IO | 20260506 01:50:29.767 | INFO | Populating RTMetrics report | 20260506 01:50:29.768 | INFO | RTMetrics report populated | 20260506 01:50:29.768 | INFO | Opening tmp RTMetrics json file | 20260506 01:50:29.768 | INFO | Serializing RTMetrics to json | 20260506 01:50:29.770 | INFO | Serialization done, swapping tmp file now... | 20260506 01:50:29.770 | INFO | RTMetrics done | 20260506 01:50:29.849 | INFO | Finished a round of Baz IO: Wrote 1.812 GiB in 3.214 seconds | 20260506 01:50:29.849 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.30% / 0.00% / 78.70% | 20260506 01:50:29.849 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0767708% / 0% / 8.34465e-05% | 20260506 01:50:29.849 | INFO | Baz IO overall: | 20260506 01:50:29.849 | INFO | BytesWritten : 4503460450495 | 20260506 01:50:29.849 | INFO | NumEvents : 2115808906410 | 20260506 01:50:31.656 | INFO | Analyzing chunk frames = [8003584,8004096) | 20260506 01:50:31.656 | INFO | MemInfo: { | 20260506 01:50:31.656 | INFO | "gpu" : " 17 GB", | 20260506 01:50:31.656 | INFO | "huge" : "", | 20260506 01:50:31.656 | INFO | "huge_shared" : " 27 GB", | 20260506 01:50:31.656 | INFO | "malloc" : " 14 GB", | 20260506 01:50:31.656 | INFO | "pinned" : " 29 GB" | 20260506 01:50:31.656 | INFO | } | 20260506 01:50:31.656 | INFO | Perf report for the last 5.14733 seconds: | 20260506 01:50:31.656 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:50:31.656 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:31.656 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:31.656 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:31.656 | INFO | ANALYSIS : 62.3%, 3.0, 97.4ms, 0 | 20260506 01:50:31.656 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:31.656 | INFO | RT_METRICS : 43.1%, 1.0, 22.7ms, 0 | 20260506 01:50:31.656 | INFO | PRE_HQ : 59.7%, 1.7, 54.1ms, 0 | 20260506 01:50:31.656 | INFO | BAZWRITER : 61.7%, 3.5, 111.8ms, 0 | 20260506 01:50:31.656 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:50:31.656 | INFO | Uploaded 36GiB to GPU in 2.18611s (16.4676GiB/s) | 20260506 01:50:31.656 | INFO | Downloaded 38GiB to GPU in 2.24397s (16.9343GiB/s) | 20260506 01:50:36.790 | INFO | Analyzing chunk frames = [8004096,8004608) | 20260506 01:50:36.790 | INFO | MemInfo: { | 20260506 01:50:36.790 | INFO | "gpu" : " 17 GB", | 20260506 01:50:36.790 | INFO | "huge" : "", | 20260506 01:50:36.790 | INFO | "huge_shared" : " 27 GB", | 20260506 01:50:36.790 | INFO | "malloc" : " 14 GB", | 20260506 01:50:36.790 | INFO | "pinned" : " 29 GB" | 20260506 01:50:36.790 | INFO | } | 20260506 01:50:36.790 | INFO | Perf report for the last 5.13392 seconds: | 20260506 01:50:36.790 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:50:36.790 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:36.790 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:36.790 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:36.790 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260506 01:50:36.790 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:36.790 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:36.790 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 01:50:36.790 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:36.790 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:36.790 | INFO | Uploaded 36GiB to GPU in 1.95424s (18.4215GiB/s) | 20260506 01:50:36.790 | INFO | Downloaded 34GiB to GPU in 1.98796s (17.1029GiB/s) | 20260506 01:50:41.922 | INFO | Analyzing chunk frames = [8004608,8005120) | 20260506 01:50:41.922 | INFO | MemInfo: { | 20260506 01:50:41.922 | INFO | "gpu" : " 17 GB", | 20260506 01:50:41.922 | INFO | "huge" : "", | 20260506 01:50:41.922 | INFO | "huge_shared" : " 27 GB", | 20260506 01:50:41.922 | INFO | "malloc" : " 14 GB", | 20260506 01:50:41.922 | INFO | "pinned" : " 29 GB" | 20260506 01:50:41.922 | INFO | } | 20260506 01:50:41.922 | INFO | Perf report for the last 5.13234 seconds: | 20260506 01:50:41.922 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:50:41.922 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:41.922 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:41.922 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:41.922 | INFO | ANALYSIS : 61.3%, 3.0, 95.2ms, 0 | 20260506 01:50:41.922 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:41.922 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:41.922 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 01:50:41.923 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:41.923 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:41.923 | INFO | Uploaded 36GiB to GPU in 2.08824s (17.2394GiB/s) | 20260506 01:50:41.923 | INFO | Downloaded 34GiB to GPU in 2.02468s (16.7928GiB/s) | 20260506 01:50:47.065 | INFO | Analyzing chunk frames = [8005120,8005632) | 20260506 01:50:47.065 | INFO | MemInfo: { | 20260506 01:50:47.065 | INFO | "gpu" : " 17 GB", | 20260506 01:50:47.065 | INFO | "huge" : "", | 20260506 01:50:47.065 | INFO | "huge_shared" : " 27 GB", | 20260506 01:50:47.065 | INFO | "malloc" : " 14 GB", | 20260506 01:50:47.065 | INFO | "pinned" : " 29 GB" | 20260506 01:50:47.065 | INFO | } | 20260506 01:50:47.065 | INFO | Perf report for the last 5.14263 seconds: | 20260506 01:50:47.065 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:50:47.065 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:47.065 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:47.065 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:47.065 | INFO | ANALYSIS : 61.5%, 3.0, 95.5ms, 0 | 20260506 01:50:47.065 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:47.065 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:47.065 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:50:47.065 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:47.065 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:47.065 | INFO | Uploaded 36GiB to GPU in 2.13836s (16.8353GiB/s) | 20260506 01:50:47.065 | INFO | Downloaded 34GiB to GPU in 2.02631s (16.7793GiB/s) | 20260506 01:50:52.205 | INFO | Analyzing chunk frames = [8005632,8006144) | 20260506 01:50:52.205 | INFO | MemInfo: { | 20260506 01:50:52.205 | INFO | "gpu" : " 17 GB", | 20260506 01:50:52.205 | INFO | "huge" : "", | 20260506 01:50:52.205 | INFO | "huge_shared" : " 27 GB", | 20260506 01:50:52.205 | INFO | "malloc" : " 14 GB", | 20260506 01:50:52.205 | INFO | "pinned" : " 29 GB" | 20260506 01:50:52.205 | INFO | } | 20260506 01:50:52.205 | INFO | Perf report for the last 5.13987 seconds: | 20260506 01:50:52.205 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:50:52.205 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:52.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:52.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:52.205 | INFO | ANALYSIS : 60.7%, 3.0, 94.8ms, 0 | 20260506 01:50:52.205 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:52.205 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:52.205 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 01:50:52.205 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:52.205 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:52.205 | INFO | Uploaded 36GiB to GPU in 2.06377s (17.4438GiB/s) | 20260506 01:50:52.205 | INFO | Downloaded 34GiB to GPU in 2.03239s (16.7291GiB/s) | 20260506 01:50:57.342 | INFO | Analyzing chunk frames = [8006144,8006656) | 20260506 01:50:57.342 | INFO | MemInfo: { | 20260506 01:50:57.342 | INFO | "gpu" : " 17 GB", | 20260506 01:50:57.342 | INFO | "huge" : "", | 20260506 01:50:57.342 | INFO | "huge_shared" : " 27 GB", | 20260506 01:50:57.342 | INFO | "malloc" : " 14 GB", | 20260506 01:50:57.342 | INFO | "pinned" : " 29 GB" | 20260506 01:50:57.342 | INFO | } | 20260506 01:50:57.342 | INFO | Perf report for the last 5.13738 seconds: | 20260506 01:50:57.342 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:50:57.342 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:57.342 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:57.342 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:57.342 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 01:50:57.342 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:57.343 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:57.343 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 01:50:57.343 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:57.343 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:50:57.343 | INFO | Uploaded 36GiB to GPU in 2.06886s (17.4009GiB/s) | 20260506 01:50:57.343 | INFO | Downloaded 34GiB to GPU in 2.03237s (16.7292GiB/s) | 20260506 01:51:02.492 | INFO | Analyzing chunk frames = [8006656,8007168) | 20260506 01:51:02.492 | INFO | MemInfo: { | 20260506 01:51:02.492 | INFO | "gpu" : " 17 GB", | 20260506 01:51:02.492 | INFO | "huge" : "", | 20260506 01:51:02.492 | INFO | "huge_shared" : " 27 GB", | 20260506 01:51:02.492 | INFO | "malloc" : " 14 GB", | 20260506 01:51:02.492 | INFO | "pinned" : " 29 GB" | 20260506 01:51:02.493 | INFO | } | 20260506 01:51:02.493 | INFO | Perf report for the last 5.15048 seconds: | 20260506 01:51:02.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:51:02.493 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:02.493 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:02.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:02.493 | INFO | ANALYSIS : 62.1%, 3.0, 97.2ms, 0 | 20260506 01:51:02.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:02.493 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:02.493 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260506 01:51:02.493 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:02.493 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:02.493 | INFO | Uploaded 36GiB to GPU in 2.13366s (16.8724GiB/s) | 20260506 01:51:02.493 | INFO | Downloaded 34GiB to GPU in 2.08577s (16.3009GiB/s) | 20260506 01:51:07.626 | INFO | Analyzing chunk frames = [8007168,8007680) | 20260506 01:51:07.626 | INFO | MemInfo: { | 20260506 01:51:07.626 | INFO | "gpu" : " 17 GB", | 20260506 01:51:07.626 | INFO | "huge" : "", | 20260506 01:51:07.626 | INFO | "huge_shared" : " 27 GB", | 20260506 01:51:07.626 | INFO | "malloc" : " 14 GB", | 20260506 01:51:07.626 | INFO | "pinned" : " 29 GB" | 20260506 01:51:07.626 | INFO | } | 20260506 01:51:07.626 | INFO | Perf report for the last 5.13311 seconds: | 20260506 01:51:07.626 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:51:07.626 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:07.626 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:07.626 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:07.626 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 01:51:07.626 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:07.626 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:07.626 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 01:51:07.626 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:07.626 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:07.627 | INFO | Uploaded 36GiB to GPU in 2.02404s (17.7862GiB/s) | 20260506 01:51:07.627 | INFO | Downloaded 34GiB to GPU in 2.01613s (16.864GiB/s) | 20260506 01:51:07.750 | INFO | Starting a round of Baz IO | 20260506 01:51:10.883 | INFO | Populating RTMetrics report | 20260506 01:51:10.883 | INFO | RTMetrics report populated | 20260506 01:51:10.883 | INFO | Opening tmp RTMetrics json file | 20260506 01:51:10.884 | INFO | Serializing RTMetrics to json | 20260506 01:51:10.885 | INFO | Serialization done, swapping tmp file now... | 20260506 01:51:10.886 | INFO | RTMetrics done | 20260506 01:51:10.967 | INFO | Finished a round of Baz IO: Wrote 1.811 GiB in 3.217 seconds | 20260506 01:51:10.967 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.30% / 0.00% / 78.70% | 20260506 01:51:10.967 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0763655% / 0% / 9.53674e-05% | 20260506 01:51:10.967 | INFO | Baz IO overall: | 20260506 01:51:10.967 | INFO | BytesWritten : 4505405265044 | 20260506 01:51:10.967 | INFO | NumEvents : 2116694159178 | 20260506 01:51:12.764 | INFO | Analyzing chunk frames = [8007680,8008192) | 20260506 01:51:12.764 | INFO | MemInfo: { | 20260506 01:51:12.764 | INFO | "gpu" : " 17 GB", | 20260506 01:51:12.764 | INFO | "huge" : "", | 20260506 01:51:12.764 | INFO | "huge_shared" : " 27 GB", | 20260506 01:51:12.764 | INFO | "malloc" : " 14 GB", | 20260506 01:51:12.764 | INFO | "pinned" : " 29 GB" | 20260506 01:51:12.764 | INFO | } | 20260506 01:51:12.764 | INFO | Perf report for the last 5.13859 seconds: | 20260506 01:51:12.764 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:51:12.764 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:12.764 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:12.764 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:12.765 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260506 01:51:12.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:12.765 | INFO | RT_METRICS : 43.2%, 1.0, 22.6ms, 0 | 20260506 01:51:12.765 | INFO | PRE_HQ : 59.6%, 1.7, 54.0ms, 0 | 20260506 01:51:12.765 | INFO | BAZWRITER : 61.8%, 3.5, 111.7ms, 0 | 20260506 01:51:12.765 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:51:12.765 | INFO | Uploaded 36GiB to GPU in 2.14651s (16.7714GiB/s) | 20260506 01:51:12.765 | INFO | Downloaded 38GiB to GPU in 2.2227s (17.0963GiB/s) | 20260506 01:51:17.902 | INFO | Analyzing chunk frames = [8008192,8008704) | 20260506 01:51:17.902 | INFO | MemInfo: { | 20260506 01:51:17.902 | INFO | "gpu" : " 17 GB", | 20260506 01:51:17.902 | INFO | "huge" : "", | 20260506 01:51:17.902 | INFO | "huge_shared" : " 27 GB", | 20260506 01:51:17.902 | INFO | "malloc" : " 14 GB", | 20260506 01:51:17.902 | INFO | "pinned" : " 29 GB" | 20260506 01:51:17.902 | INFO | } | 20260506 01:51:17.902 | INFO | Perf report for the last 5.13766 seconds: | 20260506 01:51:17.902 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:51:17.902 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:17.902 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:17.902 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:17.902 | INFO | ANALYSIS : 60.8%, 3.0, 94.9ms, 0 | 20260506 01:51:17.902 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:17.902 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:17.902 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 01:51:17.902 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:17.902 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:17.902 | INFO | Uploaded 36GiB to GPU in 1.94978s (18.4637GiB/s) | 20260506 01:51:17.902 | INFO | Downloaded 34GiB to GPU in 1.98314s (17.1446GiB/s) | 20260506 01:51:23.040 | INFO | Analyzing chunk frames = [8008704,8009216) | 20260506 01:51:23.040 | INFO | MemInfo: { | 20260506 01:51:23.040 | INFO | "gpu" : " 17 GB", | 20260506 01:51:23.040 | INFO | "huge" : "", | 20260506 01:51:23.040 | INFO | "huge_shared" : " 27 GB", | 20260506 01:51:23.040 | INFO | "malloc" : " 14 GB", | 20260506 01:51:23.040 | INFO | "pinned" : " 29 GB" | 20260506 01:51:23.040 | INFO | } | 20260506 01:51:23.040 | INFO | Perf report for the last 5.13827 seconds: | 20260506 01:51:23.040 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:51:23.040 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:23.040 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:23.041 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:23.041 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 01:51:23.041 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:23.041 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:23.041 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:51:23.041 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:23.041 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:23.041 | INFO | Uploaded 36GiB to GPU in 2.09986s (17.144GiB/s) | 20260506 01:51:23.041 | INFO | Downloaded 34GiB to GPU in 2.04071s (16.6609GiB/s) | 20260506 01:51:28.183 | INFO | Analyzing chunk frames = [8009216,8009728) | 20260506 01:51:28.183 | INFO | MemInfo: { | 20260506 01:51:28.183 | INFO | "gpu" : " 17 GB", | 20260506 01:51:28.183 | INFO | "huge" : "", | 20260506 01:51:28.183 | INFO | "huge_shared" : " 27 GB", | 20260506 01:51:28.183 | INFO | "malloc" : " 14 GB", | 20260506 01:51:28.184 | INFO | "pinned" : " 29 GB" | 20260506 01:51:28.184 | INFO | } | 20260506 01:51:28.184 | INFO | Perf report for the last 5.14319 seconds: | 20260506 01:51:28.184 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:51:28.184 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:28.184 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:28.184 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:28.184 | INFO | ANALYSIS : 61.4%, 3.0, 95.4ms, 0 | 20260506 01:51:28.184 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:28.184 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:28.184 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 01:51:28.184 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:28.184 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:28.184 | INFO | Uploaded 36GiB to GPU in 2.12735s (16.9225GiB/s) | 20260506 01:51:28.184 | INFO | Downloaded 34GiB to GPU in 2.00694s (16.9412GiB/s) | 20260506 01:51:33.317 | INFO | Analyzing chunk frames = [8009728,8010240) | 20260506 01:51:33.317 | INFO | MemInfo: { | 20260506 01:51:33.317 | INFO | "gpu" : " 17 GB", | 20260506 01:51:33.317 | INFO | "huge" : "", | 20260506 01:51:33.317 | INFO | "huge_shared" : " 27 GB", | 20260506 01:51:33.317 | INFO | "malloc" : " 14 GB", | 20260506 01:51:33.317 | INFO | "pinned" : " 29 GB" | 20260506 01:51:33.317 | INFO | } | 20260506 01:51:33.317 | INFO | Perf report for the last 5.13356 seconds: | 20260506 01:51:33.317 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:51:33.317 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:33.317 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:33.317 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:33.317 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 01:51:33.317 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:33.318 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:33.318 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:51:33.318 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:33.318 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:33.318 | INFO | Uploaded 36GiB to GPU in 2.10593s (17.0946GiB/s) | 20260506 01:51:33.318 | INFO | Downloaded 34GiB to GPU in 2.04879s (16.5951GiB/s) | 20260506 01:51:38.464 | INFO | Analyzing chunk frames = [8010240,8010752) | 20260506 01:51:38.464 | INFO | MemInfo: { | 20260506 01:51:38.464 | INFO | "gpu" : " 17 GB", | 20260506 01:51:38.464 | INFO | "huge" : "", | 20260506 01:51:38.464 | INFO | "huge_shared" : " 27 GB", | 20260506 01:51:38.464 | INFO | "malloc" : " 14 GB", | 20260506 01:51:38.464 | INFO | "pinned" : " 29 GB" | 20260506 01:51:38.464 | INFO | } | 20260506 01:51:38.465 | INFO | Perf report for the last 5.14732 seconds: | 20260506 01:51:38.465 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:51:38.465 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:38.465 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:38.465 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:38.465 | INFO | ANALYSIS : 61.0%, 3.0, 95.4ms, 0 | 20260506 01:51:38.465 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:38.465 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:38.465 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 01:51:38.465 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:38.465 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:38.465 | INFO | Uploaded 36GiB to GPU in 2.0637s (17.4444GiB/s) | 20260506 01:51:38.465 | INFO | Downloaded 34GiB to GPU in 2.03579s (16.7011GiB/s) | 20260506 01:51:43.608 | INFO | Analyzing chunk frames = [8010752,8011264) | 20260506 01:51:43.608 | INFO | MemInfo: { | 20260506 01:51:43.608 | INFO | "gpu" : " 17 GB", | 20260506 01:51:43.608 | INFO | "huge" : "", | 20260506 01:51:43.608 | INFO | "huge_shared" : " 27 GB", | 20260506 01:51:43.608 | INFO | "malloc" : " 14 GB", | 20260506 01:51:43.608 | INFO | "pinned" : " 29 GB" | 20260506 01:51:43.609 | INFO | } | 20260506 01:51:43.609 | INFO | Perf report for the last 5.14416 seconds: | 20260506 01:51:43.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:51:43.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:43.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:43.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:43.609 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260506 01:51:43.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:43.609 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:43.609 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 01:51:43.609 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:43.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:43.609 | INFO | Uploaded 36GiB to GPU in 2.04181s (17.6314GiB/s) | 20260506 01:51:43.609 | INFO | Downloaded 34GiB to GPU in 2.06674s (16.451GiB/s) | 20260506 01:51:48.736 | INFO | Analyzing chunk frames = [8011264,8011776) | 20260506 01:51:48.736 | INFO | MemInfo: { | 20260506 01:51:48.736 | INFO | "gpu" : " 17 GB", | 20260506 01:51:48.736 | INFO | "huge" : "", | 20260506 01:51:48.736 | INFO | "huge_shared" : " 27 GB", | 20260506 01:51:48.736 | INFO | "malloc" : " 14 GB", | 20260506 01:51:48.736 | INFO | "pinned" : " 29 GB" | 20260506 01:51:48.736 | INFO | } | 20260506 01:51:48.736 | INFO | Perf report for the last 5.12742 seconds: | 20260506 01:51:48.736 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:51:48.736 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:48.736 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:48.736 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:48.736 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 01:51:48.736 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:48.736 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:48.736 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 01:51:48.736 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:48.736 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:48.737 | INFO | Uploaded 36GiB to GPU in 2.045s (17.6039GiB/s) | 20260506 01:51:48.737 | INFO | Downloaded 34GiB to GPU in 2.01551s (16.8692GiB/s) | 20260506 01:51:48.737 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 01:51:48.737 | INFO | Total mem usage: 15.3 GiB | 20260506 01:51:48.737 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 01:51:48.737 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.19%) | 20260506 01:51:48.737 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 01:51:48.737 | INFO | High Water Breakdown: | 20260506 01:51:48.737 | INFO | 60.9% : 8.54 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 01:51:48.737 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 01:51:48.737 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 01:51:48.737 | INFO | ------------------------------------------ | 20260506 01:51:48.737 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:51:48.737 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 01:51:48.737 | INFO | Total mem usage: 26.5 GiB | 20260506 01:51:48.737 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 01:51:48.737 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 01:51:48.737 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 01:51:48.737 | INFO | High Water Breakdown: | 20260506 01:51:48.737 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:51:48.737 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:51:48.737 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:51:48.737 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:51:48.737 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:51:48.737 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:51:48.737 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 01:51:48.737 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 01:51:48.737 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:51:48.737 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 01:51:48.737 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 01:51:48.737 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 01:51:48.737 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 01:51:48.737 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 01:51:48.737 | INFO | ------------------------------------------ | 20260506 01:51:48.737 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:51:48.737 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 01:51:48.737 | INFO | Total mem usage: 25.6 GiB | 20260506 01:51:48.737 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 01:51:48.737 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 01:51:48.737 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 01:51:48.737 | INFO | High Water Breakdown: | 20260506 01:51:48.737 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 01:51:48.737 | INFO | ------------------------------------------ | 20260506 01:51:48.737 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:51:48.737 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 01:51:48.737 | INFO | Total mem usage: 15.7 GiB | 20260506 01:51:48.737 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 01:51:48.737 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 01:51:48.737 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 01:51:48.737 | INFO | High Water Breakdown: | 20260506 01:51:48.737 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 01:51:48.737 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 01:51:48.737 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 01:51:48.737 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 01:51:48.738 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 01:51:48.738 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 01:51:48.738 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 01:51:48.738 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 01:51:48.738 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 01:51:48.738 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:51:48.738 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 01:51:48.738 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:51:48.738 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 01:51:48.738 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 01:51:48.738 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 01:51:48.738 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 01:51:48.738 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:51:48.738 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 01:51:48.738 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:51:48.738 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:51:48.738 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:51:48.738 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 01:51:48.738 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 01:51:48.738 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:51:48.738 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 01:51:48.738 | INFO | ------------------------------------------ | 20260506 01:51:48.738 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:51:48.738 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 01:51:48.738 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:51:48.738 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:51:48.738 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.38e+01, 1.34e+01 / 6.77e-01 | 20260506 01:51:48.738 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.05e-01, 1.02e-02 / 3.72e-03 | 20260506 01:51:48.738 | INFO | PerfReport: 11.87%, Baseline -- 3.71e+00 / 4.51e+00, 5.82e-01 / 2.02e-01 | 20260506 01:51:48.738 | INFO | PerfReport: 3.29%, Histogram -- 1.03e+00 / 1.51e+00, 1.94e-01 / 8.66e-02 | 20260506 01:51:48.738 | INFO | PerfReport: 16.00%, DME -- 4.01e+01 / 4.41e+01, 3.89e+00 / 9.52e-01 | 20260506 01:51:48.738 | INFO | PerfReport: 57.41%, FrameLabeling -- 1.80e+01 / 1.92e+01, 1.53e+00 / 9.97e-01 | 20260506 01:51:48.738 | INFO | PerfReport: 7.05%, PulseAccumulating -- 2.21e+00 / 2.73e+00, 4.21e-01 / 2.27e-01 | 20260506 01:51:48.738 | INFO | PerfReport: 3.03%, Metrics -- 9.49e-01 / 1.21e+00, 2.53e-01 / 1.07e-01 | 20260506 01:51:48.738 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.28e-01 / 3.75e-01, 5.48e-02 / 2.70e-02 | 20260506 01:51:48.738 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 01:51:48.738 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:51:48.738 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:51:48.738 | INFO | PerfReport: Full Execution -- 9.77e+01 / -nan, 2.42e+01 / 0.00e+00 | 20260506 01:51:48.738 | INFO | PerfReport: 22.14%, Upload -- 2.16e+01 / 0.00e+00, 7.24e+00 / 0.00e+00 | 20260506 01:51:48.738 | INFO | PerfReport: 21.90%, Download -- 2.14e+01 / 0.00e+00, 7.52e+00 / 0.00e+00 | 20260506 01:51:48.738 | INFO | PerfReport: 32.03%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 01:51:48.739 | INFO | PerfReport: 2.01%, LockUpload -- 1.97e+00 / 0.00e+00, 7.74e+00 / 0.00e+00 | 20260506 01:51:48.739 | INFO | PerfReport: 1.31%, LockDownload -- 1.28e+00 / 0.00e+00, 2.89e+00 / 0.00e+00 | 20260506 01:51:48.739 | INFO | PerfReport: 20.60%, LockCompute -- 2.01e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260506 01:51:48.739 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.54e-02 / 0.00e+00 | 20260506 01:51:48.854 | INFO | Starting a round of Baz IO | 20260506 01:51:51.996 | INFO | Populating RTMetrics report | 20260506 01:51:51.996 | INFO | RTMetrics report populated | 20260506 01:51:51.996 | INFO | Opening tmp RTMetrics json file | 20260506 01:51:51.997 | INFO | Serializing RTMetrics to json | 20260506 01:51:51.998 | INFO | Serialization done, swapping tmp file now... | 20260506 01:51:51.999 | INFO | RTMetrics done | 20260506 01:51:52.087 | INFO | Finished a round of Baz IO: Wrote 1.811 GiB in 3.232 seconds | 20260506 01:51:52.087 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.29% / 0.00% / 78.70% | 20260506 01:51:52.087 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0763575% / 0% / 6.75519e-05% | 20260506 01:51:52.087 | INFO | Baz IO overall: | 20260506 01:51:52.087 | INFO | BytesWritten : 4507350078989 | 20260506 01:51:52.087 | INFO | NumEvents : 2117579847144 | 20260506 01:51:53.868 | INFO | Analyzing chunk frames = [8011776,8012288) | 20260506 01:51:53.868 | INFO | MemInfo: { | 20260506 01:51:53.868 | INFO | "gpu" : " 17 GB", | 20260506 01:51:53.868 | INFO | "huge" : "", | 20260506 01:51:53.868 | INFO | "huge_shared" : " 27 GB", | 20260506 01:51:53.868 | INFO | "malloc" : " 14 GB", | 20260506 01:51:53.868 | INFO | "pinned" : " 29 GB" | 20260506 01:51:53.868 | INFO | } | 20260506 01:51:53.868 | INFO | Perf report for the last 5.13219 seconds: | 20260506 01:51:53.868 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:51:53.868 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:53.868 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:53.869 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:53.869 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260506 01:51:53.869 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:53.869 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260506 01:51:53.869 | INFO | PRE_HQ : 59.7%, 1.7, 53.3ms, 0 | 20260506 01:51:53.869 | INFO | BAZWRITER : 62.1%, 3.5, 111.8ms, 0 | 20260506 01:51:53.869 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:51:53.869 | INFO | Uploaded 36GiB to GPU in 2.17919s (16.5199GiB/s) | 20260506 01:51:53.869 | INFO | Downloaded 38GiB to GPU in 2.24595s (16.9193GiB/s) | 20260506 01:51:59.011 | INFO | Analyzing chunk frames = [8012288,8012800) | 20260506 01:51:59.011 | INFO | MemInfo: { | 20260506 01:51:59.011 | INFO | "gpu" : " 17 GB", | 20260506 01:51:59.011 | INFO | "huge" : "", | 20260506 01:51:59.011 | INFO | "huge_shared" : " 27 GB", | 20260506 01:51:59.011 | INFO | "malloc" : " 14 GB", | 20260506 01:51:59.011 | INFO | "pinned" : " 29 GB" | 20260506 01:51:59.011 | INFO | } | 20260506 01:51:59.011 | INFO | Perf report for the last 5.14303 seconds: | 20260506 01:51:59.011 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:51:59.011 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:59.011 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:59.011 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:59.011 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260506 01:51:59.011 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:59.011 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:59.011 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 01:51:59.011 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:59.011 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:51:59.012 | INFO | Uploaded 36GiB to GPU in 2.09069s (17.2192GiB/s) | 20260506 01:51:59.012 | INFO | Downloaded 34GiB to GPU in 2.03101s (16.7405GiB/s) | 20260506 01:52:04.159 | INFO | Analyzing chunk frames = [8012800,8013312) | 20260506 01:52:04.159 | INFO | MemInfo: { | 20260506 01:52:04.159 | INFO | "gpu" : " 17 GB", | 20260506 01:52:04.159 | INFO | "huge" : "", | 20260506 01:52:04.159 | INFO | "huge_shared" : " 27 GB", | 20260506 01:52:04.159 | INFO | "malloc" : " 14 GB", | 20260506 01:52:04.159 | INFO | "pinned" : " 29 GB" | 20260506 01:52:04.159 | INFO | } | 20260506 01:52:04.159 | INFO | Perf report for the last 5.14779 seconds: | 20260506 01:52:04.159 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:52:04.159 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:04.159 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:04.159 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:04.159 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260506 01:52:04.159 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:04.159 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:04.159 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 01:52:04.159 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:04.159 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:04.159 | INFO | Uploaded 36GiB to GPU in 2.08564s (17.2609GiB/s) | 20260506 01:52:04.159 | INFO | Downloaded 34GiB to GPU in 2.02524s (16.7882GiB/s) | 20260506 01:52:09.298 | INFO | Analyzing chunk frames = [8013312,8013824) | 20260506 01:52:09.298 | INFO | MemInfo: { | 20260506 01:52:09.298 | INFO | "gpu" : " 17 GB", | 20260506 01:52:09.298 | INFO | "huge" : "", | 20260506 01:52:09.298 | INFO | "huge_shared" : " 27 GB", | 20260506 01:52:09.298 | INFO | "malloc" : " 14 GB", | 20260506 01:52:09.298 | INFO | "pinned" : " 29 GB" | 20260506 01:52:09.298 | INFO | } | 20260506 01:52:09.298 | INFO | Perf report for the last 5.13878 seconds: | 20260506 01:52:09.298 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:52:09.298 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:09.298 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:09.298 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:09.298 | INFO | ANALYSIS : 61.0%, 3.0, 94.7ms, 0 | 20260506 01:52:09.298 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:09.298 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:09.298 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 01:52:09.298 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:09.298 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:09.298 | INFO | Uploaded 36GiB to GPU in 2.08843s (17.2379GiB/s) | 20260506 01:52:09.298 | INFO | Downloaded 34GiB to GPU in 2.00928s (16.9215GiB/s) | 20260506 01:52:14.438 | INFO | Analyzing chunk frames = [8013824,8014336) | 20260506 01:52:14.438 | INFO | MemInfo: { | 20260506 01:52:14.438 | INFO | "gpu" : " 17 GB", | 20260506 01:52:14.438 | INFO | "huge" : "", | 20260506 01:52:14.438 | INFO | "huge_shared" : " 27 GB", | 20260506 01:52:14.438 | INFO | "malloc" : " 14 GB", | 20260506 01:52:14.438 | INFO | "pinned" : " 29 GB" | 20260506 01:52:14.438 | INFO | } | 20260506 01:52:14.438 | INFO | Perf report for the last 5.14018 seconds: | 20260506 01:52:14.438 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:52:14.438 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:14.438 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:14.438 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:14.438 | INFO | ANALYSIS : 60.7%, 3.0, 94.8ms, 0 | 20260506 01:52:14.438 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:14.438 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:14.438 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 01:52:14.439 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:14.439 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:14.439 | INFO | Uploaded 36GiB to GPU in 2.06497s (17.4337GiB/s) | 20260506 01:52:14.439 | INFO | Downloaded 34GiB to GPU in 2.03097s (16.7407GiB/s) | 20260506 01:52:19.575 | INFO | Analyzing chunk frames = [8014336,8014848) | 20260506 01:52:19.575 | INFO | MemInfo: { | 20260506 01:52:19.575 | INFO | "gpu" : " 17 GB", | 20260506 01:52:19.575 | INFO | "huge" : "", | 20260506 01:52:19.575 | INFO | "huge_shared" : " 27 GB", | 20260506 01:52:19.575 | INFO | "malloc" : " 14 GB", | 20260506 01:52:19.575 | INFO | "pinned" : " 29 GB" | 20260506 01:52:19.575 | INFO | } | 20260506 01:52:19.575 | INFO | Perf report for the last 5.13688 seconds: | 20260506 01:52:19.575 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:52:19.575 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:19.575 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:19.575 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:19.575 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 01:52:19.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:19.575 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:19.575 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 01:52:19.575 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:19.575 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:19.575 | INFO | Uploaded 36GiB to GPU in 2.07756s (17.328GiB/s) | 20260506 01:52:19.575 | INFO | Downloaded 34GiB to GPU in 2.04602s (16.6176GiB/s) | 20260506 01:52:24.716 | INFO | Analyzing chunk frames = [8014848,8015360) | 20260506 01:52:24.716 | INFO | MemInfo: { | 20260506 01:52:24.716 | INFO | "gpu" : " 17 GB", | 20260506 01:52:24.716 | INFO | "huge" : "", | 20260506 01:52:24.716 | INFO | "huge_shared" : " 27 GB", | 20260506 01:52:24.716 | INFO | "malloc" : " 14 GB", | 20260506 01:52:24.716 | INFO | "pinned" : " 29 GB" | 20260506 01:52:24.717 | INFO | } | 20260506 01:52:24.717 | INFO | Perf report for the last 5.14149 seconds: | 20260506 01:52:24.717 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:52:24.717 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:24.717 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:24.717 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:24.717 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260506 01:52:24.717 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:24.717 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:24.717 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 01:52:24.717 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:24.717 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:24.717 | INFO | Uploaded 36GiB to GPU in 2.0657s (17.4275GiB/s) | 20260506 01:52:24.717 | INFO | Downloaded 34GiB to GPU in 2.07s (16.4252GiB/s) | 20260506 01:52:29.853 | INFO | Analyzing chunk frames = [8015360,8015872) | 20260506 01:52:29.853 | INFO | MemInfo: { | 20260506 01:52:29.853 | INFO | "gpu" : " 17 GB", | 20260506 01:52:29.853 | INFO | "huge" : "", | 20260506 01:52:29.853 | INFO | "huge_shared" : " 27 GB", | 20260506 01:52:29.853 | INFO | "malloc" : " 14 GB", | 20260506 01:52:29.853 | INFO | "pinned" : " 29 GB" | 20260506 01:52:29.853 | INFO | } | 20260506 01:52:29.853 | INFO | Perf report for the last 5.13629 seconds: | 20260506 01:52:29.853 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:52:29.853 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:29.853 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:29.853 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:29.853 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260506 01:52:29.853 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:29.853 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:29.853 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 01:52:29.853 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:29.853 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:29.853 | INFO | Uploaded 36GiB to GPU in 2.03888s (17.6568GiB/s) | 20260506 01:52:29.853 | INFO | Downloaded 34GiB to GPU in 2.02142s (16.8199GiB/s) | 20260506 01:52:29.979 | INFO | Starting a round of Baz IO | 20260506 01:52:33.107 | INFO | Populating RTMetrics report | 20260506 01:52:33.107 | INFO | RTMetrics report populated | 20260506 01:52:33.107 | INFO | Opening tmp RTMetrics json file | 20260506 01:52:33.108 | INFO | Serializing RTMetrics to json | 20260506 01:52:33.109 | INFO | Serialization done, swapping tmp file now... | 20260506 01:52:33.110 | INFO | RTMetrics done | 20260506 01:52:33.190 | INFO | Finished a round of Baz IO: Wrote 1.81 GiB in 3.21 seconds | 20260506 01:52:33.190 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.29% / 0.00% / 78.71% | 20260506 01:52:33.190 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0761588% / 0% / 8.74201e-05% | 20260506 01:52:33.190 | INFO | Baz IO overall: | 20260506 01:52:33.190 | INFO | BytesWritten : 4509293898280 | 20260506 01:52:33.190 | INFO | NumEvents : 2118465109750 | 20260506 01:52:34.991 | INFO | Analyzing chunk frames = [8015872,8016384) | 20260506 01:52:34.991 | INFO | MemInfo: { | 20260506 01:52:34.991 | INFO | "gpu" : " 17 GB", | 20260506 01:52:34.991 | INFO | "huge" : "", | 20260506 01:52:34.991 | INFO | "huge_shared" : " 27 GB", | 20260506 01:52:34.991 | INFO | "malloc" : " 14 GB", | 20260506 01:52:34.991 | INFO | "pinned" : " 29 GB" | 20260506 01:52:34.991 | INFO | } | 20260506 01:52:34.991 | INFO | Perf report for the last 5.13833 seconds: | 20260506 01:52:34.991 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:52:34.991 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:34.991 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:34.991 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:34.991 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260506 01:52:34.991 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:34.991 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260506 01:52:34.991 | INFO | PRE_HQ : 59.5%, 1.7, 53.5ms, 0 | 20260506 01:52:34.991 | INFO | BAZWRITER : 61.8%, 3.5, 111.6ms, 0 | 20260506 01:52:34.991 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:52:34.991 | INFO | Uploaded 36GiB to GPU in 2.18137s (16.5034GiB/s) | 20260506 01:52:34.992 | INFO | Downloaded 38GiB to GPU in 2.23826s (16.9775GiB/s) | 20260506 01:52:40.134 | INFO | Analyzing chunk frames = [8016384,8016896) | 20260506 01:52:40.134 | INFO | MemInfo: { | 20260506 01:52:40.134 | INFO | "gpu" : " 17 GB", | 20260506 01:52:40.134 | INFO | "huge" : "", | 20260506 01:52:40.134 | INFO | "huge_shared" : " 27 GB", | 20260506 01:52:40.134 | INFO | "malloc" : " 14 GB", | 20260506 01:52:40.134 | INFO | "pinned" : " 29 GB" | 20260506 01:52:40.134 | INFO | } | 20260506 01:52:40.134 | INFO | Perf report for the last 5.14269 seconds: | 20260506 01:52:40.134 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:52:40.134 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:40.134 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:40.134 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:40.134 | INFO | ANALYSIS : 60.9%, 3.0, 95.0ms, 0 | 20260506 01:52:40.134 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:40.134 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:40.134 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 01:52:40.134 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:40.134 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:40.134 | INFO | Uploaded 36GiB to GPU in 1.96755s (18.2969GiB/s) | 20260506 01:52:40.134 | INFO | Downloaded 34GiB to GPU in 1.98284s (17.1471GiB/s) | 20260506 01:52:45.271 | INFO | Analyzing chunk frames = [8016896,8017408) | 20260506 01:52:45.271 | INFO | MemInfo: { | 20260506 01:52:45.271 | INFO | "gpu" : " 17 GB", | 20260506 01:52:45.271 | INFO | "huge" : "", | 20260506 01:52:45.271 | INFO | "huge_shared" : " 27 GB", | 20260506 01:52:45.271 | INFO | "malloc" : " 14 GB", | 20260506 01:52:45.271 | INFO | "pinned" : " 29 GB" | 20260506 01:52:45.271 | INFO | } | 20260506 01:52:45.271 | INFO | Perf report for the last 5.13685 seconds: | 20260506 01:52:45.271 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:52:45.271 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:45.271 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:45.271 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:45.271 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 01:52:45.271 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:45.271 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:45.271 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:52:45.271 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:45.271 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:45.271 | INFO | Uploaded 36GiB to GPU in 2.09165s (17.2113GiB/s) | 20260506 01:52:45.271 | INFO | Downloaded 34GiB to GPU in 2.02488s (16.7911GiB/s) | 20260506 01:52:50.415 | INFO | Analyzing chunk frames = [8017408,8017920) | 20260506 01:52:50.415 | INFO | MemInfo: { | 20260506 01:52:50.415 | INFO | "gpu" : " 17 GB", | 20260506 01:52:50.415 | INFO | "huge" : "", | 20260506 01:52:50.415 | INFO | "huge_shared" : " 27 GB", | 20260506 01:52:50.415 | INFO | "malloc" : " 14 GB", | 20260506 01:52:50.415 | INFO | "pinned" : " 29 GB" | 20260506 01:52:50.415 | INFO | } | 20260506 01:52:50.415 | INFO | Perf report for the last 5.1445 seconds: | 20260506 01:52:50.415 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:52:50.415 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:50.415 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:50.415 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:50.415 | INFO | ANALYSIS : 61.0%, 3.0, 94.8ms, 0 | 20260506 01:52:50.415 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:50.415 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:50.415 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 01:52:50.415 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:50.415 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:50.416 | INFO | Uploaded 36GiB to GPU in 2.07878s (17.3179GiB/s) | 20260506 01:52:50.416 | INFO | Downloaded 34GiB to GPU in 2.01335s (16.8872GiB/s) | 20260506 01:52:55.553 | INFO | Analyzing chunk frames = [8017920,8018432) | 20260506 01:52:55.553 | INFO | MemInfo: { | 20260506 01:52:55.553 | INFO | "gpu" : " 17 GB", | 20260506 01:52:55.553 | INFO | "huge" : "", | 20260506 01:52:55.553 | INFO | "huge_shared" : " 27 GB", | 20260506 01:52:55.553 | INFO | "malloc" : " 14 GB", | 20260506 01:52:55.553 | INFO | "pinned" : " 29 GB" | 20260506 01:52:55.553 | INFO | } | 20260506 01:52:55.553 | INFO | Perf report for the last 5.13817 seconds: | 20260506 01:52:55.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:52:55.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:55.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:55.554 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:55.554 | INFO | ANALYSIS : 60.7%, 3.0, 94.8ms, 0 | 20260506 01:52:55.554 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:55.554 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:55.554 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:52:55.554 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:55.554 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:52:55.554 | INFO | Uploaded 36GiB to GPU in 2.08429s (17.2721GiB/s) | 20260506 01:52:55.554 | INFO | Downloaded 34GiB to GPU in 2.03351s (16.7199GiB/s) | 20260506 01:53:00.689 | INFO | Analyzing chunk frames = [8018432,8018944) | 20260506 01:53:00.689 | INFO | MemInfo: { | 20260506 01:53:00.689 | INFO | "gpu" : " 17 GB", | 20260506 01:53:00.689 | INFO | "huge" : "", | 20260506 01:53:00.689 | INFO | "huge_shared" : " 27 GB", | 20260506 01:53:00.689 | INFO | "malloc" : " 14 GB", | 20260506 01:53:00.689 | INFO | "pinned" : " 29 GB" | 20260506 01:53:00.689 | INFO | } | 20260506 01:53:00.689 | INFO | Perf report for the last 5.13593 seconds: | 20260506 01:53:00.690 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:53:00.690 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:00.690 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:00.690 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:00.690 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 01:53:00.690 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:00.690 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:00.690 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:53:00.690 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:00.690 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:00.690 | INFO | Uploaded 36GiB to GPU in 2.10107s (17.1341GiB/s) | 20260506 01:53:00.690 | INFO | Downloaded 34GiB to GPU in 2.03732s (16.6886GiB/s) | 20260506 01:53:05.827 | INFO | Analyzing chunk frames = [8018944,8019456) | 20260506 01:53:05.827 | INFO | MemInfo: { | 20260506 01:53:05.827 | INFO | "gpu" : " 17 GB", | 20260506 01:53:05.827 | INFO | "huge" : "", | 20260506 01:53:05.827 | INFO | "huge_shared" : " 27 GB", | 20260506 01:53:05.827 | INFO | "malloc" : " 14 GB", | 20260506 01:53:05.827 | INFO | "pinned" : " 29 GB" | 20260506 01:53:05.827 | INFO | } | 20260506 01:53:05.827 | INFO | Perf report for the last 5.1374 seconds: | 20260506 01:53:05.827 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:53:05.827 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:05.827 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:05.827 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:05.827 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 01:53:05.827 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:05.827 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:05.827 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260506 01:53:05.827 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:05.827 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:05.827 | INFO | Uploaded 36GiB to GPU in 2.0497s (17.5635GiB/s) | 20260506 01:53:05.827 | INFO | Downloaded 34GiB to GPU in 2.06783s (16.4424GiB/s) | 20260506 01:53:10.966 | INFO | Analyzing chunk frames = [8019456,8019968) | 20260506 01:53:10.966 | INFO | MemInfo: { | 20260506 01:53:10.966 | INFO | "gpu" : " 17 GB", | 20260506 01:53:10.966 | INFO | "huge" : "", | 20260506 01:53:10.966 | INFO | "huge_shared" : " 27 GB", | 20260506 01:53:10.966 | INFO | "malloc" : " 14 GB", | 20260506 01:53:10.966 | INFO | "pinned" : " 29 GB" | 20260506 01:53:10.966 | INFO | } | 20260506 01:53:10.966 | INFO | Perf report for the last 5.13945 seconds: | 20260506 01:53:10.966 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:53:10.966 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:10.966 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:10.966 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:10.966 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 01:53:10.966 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:10.966 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:10.966 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 01:53:10.966 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:10.966 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:10.967 | INFO | Uploaded 36GiB to GPU in 2.13686s (16.8472GiB/s) | 20260506 01:53:10.967 | INFO | Downloaded 34GiB to GPU in 2.03383s (16.7173GiB/s) | 20260506 01:53:10.967 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 01:53:10.967 | INFO | Total mem usage: 15.3 GiB | 20260506 01:53:10.967 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 01:53:10.967 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.21%) | 20260506 01:53:10.967 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 01:53:10.967 | INFO | High Water Breakdown: | 20260506 01:53:10.967 | INFO | 60.9% : 8.54 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 01:53:10.967 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 01:53:10.967 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 01:53:10.967 | INFO | ------------------------------------------ | 20260506 01:53:10.967 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:53:10.967 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 01:53:10.967 | INFO | Total mem usage: 26.5 GiB | 20260506 01:53:10.967 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 01:53:10.967 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 01:53:10.967 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260506 01:53:10.967 | INFO | High Water Breakdown: | 20260506 01:53:10.967 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:53:10.967 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:53:10.967 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:53:10.967 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:53:10.967 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:53:10.967 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:53:10.967 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 01:53:10.967 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 01:53:10.967 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:53:10.967 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 01:53:10.967 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 01:53:10.967 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 01:53:10.967 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 01:53:10.967 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 01:53:10.967 | INFO | ------------------------------------------ | 20260506 01:53:10.967 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:53:10.967 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 01:53:10.967 | INFO | Total mem usage: 25.6 GiB | 20260506 01:53:10.967 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 01:53:10.967 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 01:53:10.967 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 01:53:10.967 | INFO | High Water Breakdown: | 20260506 01:53:10.967 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 01:53:10.968 | INFO | ------------------------------------------ | 20260506 01:53:10.968 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:53:10.968 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 01:53:10.968 | INFO | Total mem usage: 15.7 GiB | 20260506 01:53:10.968 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 01:53:10.968 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 01:53:10.968 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 01:53:10.968 | INFO | High Water Breakdown: | 20260506 01:53:10.968 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 01:53:10.968 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 01:53:10.968 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 01:53:10.968 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 01:53:10.968 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 01:53:10.968 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 01:53:10.968 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 01:53:10.968 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 01:53:10.968 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 01:53:10.968 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:53:10.968 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 01:53:10.968 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:53:10.968 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 01:53:10.968 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 01:53:10.968 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 01:53:10.968 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 01:53:10.968 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:53:10.968 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 01:53:10.968 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:53:10.968 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:53:10.968 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:53:10.968 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 01:53:10.968 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 01:53:10.968 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:53:10.968 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 01:53:10.968 | INFO | ------------------------------------------ | 20260506 01:53:10.968 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:53:10.968 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 01:53:10.968 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:53:10.968 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:53:10.968 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.39e+01, 1.32e+01 / 7.66e-01 | 20260506 01:53:10.968 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.04e-01, 1.08e-02 / 6.20e-03 | 20260506 01:53:10.968 | INFO | PerfReport: 11.85%, Baseline -- 3.71e+00 / 4.49e+00, 5.82e-01 / 2.35e-01 | 20260506 01:53:10.968 | INFO | PerfReport: 3.29%, Histogram -- 1.03e+00 / 1.52e+00, 1.92e-01 / 9.68e-02 | 20260506 01:53:10.968 | INFO | PerfReport: 16.00%, DME -- 4.01e+01 / 4.40e+01, 3.81e+00 / 9.97e-01 | 20260506 01:53:10.968 | INFO | PerfReport: 57.47%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.52e+00 / 1.17e+00 | 20260506 01:53:10.968 | INFO | PerfReport: 7.02%, PulseAccumulating -- 2.20e+00 / 2.71e+00, 4.18e-01 / 2.73e-01 | 20260506 01:53:10.968 | INFO | PerfReport: 3.03%, Metrics -- 9.47e-01 / 1.22e+00, 2.53e-01 / 1.11e-01 | 20260506 01:53:10.968 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.28e-01 / 3.83e-01, 5.56e-02 / 3.22e-02 | 20260506 01:53:10.968 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 01:53:10.968 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:53:10.968 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:53:10.968 | INFO | PerfReport: Full Execution -- 9.78e+01 / -nan, 2.41e+01 / 0.00e+00 | 20260506 01:53:10.968 | INFO | PerfReport: 22.23%, Upload -- 2.17e+01 / 0.00e+00, 7.31e+00 / 0.00e+00 | 20260506 01:53:10.968 | INFO | PerfReport: 21.91%, Download -- 2.14e+01 / 0.00e+00, 7.57e+00 / 0.00e+00 | 20260506 01:53:10.968 | INFO | PerfReport: 32.01%, Compute -- 3.13e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260506 01:53:10.968 | INFO | PerfReport: 2.06%, LockUpload -- 2.01e+00 / 0.00e+00, 7.92e+00 / 0.00e+00 | 20260506 01:53:10.968 | INFO | PerfReport: 1.29%, LockDownload -- 1.26e+00 / 0.00e+00, 2.87e+00 / 0.00e+00 | 20260506 01:53:10.968 | INFO | PerfReport: 20.50%, LockCompute -- 2.00e+01 / 0.00e+00, 1.77e+01 / 0.00e+00 | 20260506 01:53:10.968 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 2.00e-02 / 0.00e+00 | 20260506 01:53:11.084 | INFO | Starting a round of Baz IO | 20260506 01:53:14.230 | INFO | Populating RTMetrics report | 20260506 01:53:14.230 | INFO | RTMetrics report populated | 20260506 01:53:14.230 | INFO | Opening tmp RTMetrics json file | 20260506 01:53:14.231 | INFO | Serializing RTMetrics to json | 20260506 01:53:14.232 | INFO | Serialization done, swapping tmp file now... | 20260506 01:53:14.232 | INFO | RTMetrics done | 20260506 01:53:14.312 | INFO | Finished a round of Baz IO: Wrote 1.809 GiB in 3.228 seconds | 20260506 01:53:14.312 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.29% / 0.00% / 78.71% | 20260506 01:53:14.312 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0757933% / 0% / 6.35783e-05% | 20260506 01:53:14.312 | INFO | Baz IO overall: | 20260506 01:53:14.312 | INFO | BytesWritten : 4511236409252 | 20260506 01:53:14.312 | INFO | NumEvents : 2119349052523 | 20260506 01:53:16.114 | INFO | Analyzing chunk frames = [8019968,8020480) | 20260506 01:53:16.114 | INFO | MemInfo: { | 20260506 01:53:16.114 | INFO | "gpu" : " 17 GB", | 20260506 01:53:16.114 | INFO | "huge" : "", | 20260506 01:53:16.114 | INFO | "huge_shared" : " 27 GB", | 20260506 01:53:16.114 | INFO | "malloc" : " 14 GB", | 20260506 01:53:16.114 | INFO | "pinned" : " 29 GB" | 20260506 01:53:16.114 | INFO | } | 20260506 01:53:16.114 | INFO | Perf report for the last 5.14762 seconds: | 20260506 01:53:16.114 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:53:16.114 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:16.114 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:16.114 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:16.114 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260506 01:53:16.114 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:16.114 | INFO | RT_METRICS : 43.0%, 1.0, 22.6ms, 0 | 20260506 01:53:16.114 | INFO | PRE_HQ : 59.7%, 1.7, 54.0ms, 0 | 20260506 01:53:16.114 | INFO | BAZWRITER : 61.8%, 3.5, 112.8ms, 0 | 20260506 01:53:16.114 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:53:16.114 | INFO | Uploaded 36GiB to GPU in 2.18577s (16.4701GiB/s) | 20260506 01:53:16.115 | INFO | Downloaded 38GiB to GPU in 2.24186s (16.9502GiB/s) | 20260506 01:53:21.248 | INFO | Analyzing chunk frames = [8020480,8020992) | 20260506 01:53:21.248 | INFO | MemInfo: { | 20260506 01:53:21.248 | INFO | "gpu" : " 17 GB", | 20260506 01:53:21.248 | INFO | "huge" : "", | 20260506 01:53:21.248 | INFO | "huge_shared" : " 27 GB", | 20260506 01:53:21.248 | INFO | "malloc" : " 14 GB", | 20260506 01:53:21.248 | INFO | "pinned" : " 29 GB" | 20260506 01:53:21.248 | INFO | } | 20260506 01:53:21.248 | INFO | Perf report for the last 5.13436 seconds: | 20260506 01:53:21.248 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:53:21.248 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:21.248 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:21.248 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:21.248 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260506 01:53:21.248 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:21.248 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:21.248 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 01:53:21.248 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:21.249 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:21.249 | INFO | Uploaded 36GiB to GPU in 1.95072s (18.4547GiB/s) | 20260506 01:53:21.249 | INFO | Downloaded 34GiB to GPU in 1.97934s (17.1774GiB/s) | 20260506 01:53:26.385 | INFO | Analyzing chunk frames = [8020992,8021504) | 20260506 01:53:26.385 | INFO | MemInfo: { | 20260506 01:53:26.385 | INFO | "gpu" : " 17 GB", | 20260506 01:53:26.385 | INFO | "huge" : "", | 20260506 01:53:26.385 | INFO | "huge_shared" : " 27 GB", | 20260506 01:53:26.385 | INFO | "malloc" : " 14 GB", | 20260506 01:53:26.385 | INFO | "pinned" : " 29 GB" | 20260506 01:53:26.385 | INFO | } | 20260506 01:53:26.385 | INFO | Perf report for the last 5.13688 seconds: | 20260506 01:53:26.385 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:53:26.385 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:26.385 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:26.385 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:26.385 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 01:53:26.385 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:26.385 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:26.385 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:53:26.385 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:26.385 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:26.386 | INFO | Uploaded 36GiB to GPU in 2.09309s (17.1995GiB/s) | 20260506 01:53:26.386 | INFO | Downloaded 34GiB to GPU in 2.02475s (16.7922GiB/s) | 20260506 01:53:31.525 | INFO | Analyzing chunk frames = [8021504,8022016) | 20260506 01:53:31.525 | INFO | MemInfo: { | 20260506 01:53:31.525 | INFO | "gpu" : " 17 GB", | 20260506 01:53:31.525 | INFO | "huge" : "", | 20260506 01:53:31.525 | INFO | "huge_shared" : " 27 GB", | 20260506 01:53:31.525 | INFO | "malloc" : " 14 GB", | 20260506 01:53:31.526 | INFO | "pinned" : " 29 GB" | 20260506 01:53:31.526 | INFO | } | 20260506 01:53:31.526 | INFO | Perf report for the last 5.14048 seconds: | 20260506 01:53:31.526 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:53:31.526 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:31.526 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:31.526 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:31.526 | INFO | ANALYSIS : 61.5%, 3.0, 95.5ms, 0 | 20260506 01:53:31.526 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:31.526 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:31.526 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:53:31.526 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:31.526 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:31.526 | INFO | Uploaded 36GiB to GPU in 2.13932s (16.8278GiB/s) | 20260506 01:53:31.526 | INFO | Downloaded 34GiB to GPU in 2.03313s (16.723GiB/s) | 20260506 01:53:36.665 | INFO | Analyzing chunk frames = [8022016,8022528) | 20260506 01:53:36.665 | INFO | MemInfo: { | 20260506 01:53:36.665 | INFO | "gpu" : " 17 GB", | 20260506 01:53:36.665 | INFO | "huge" : "", | 20260506 01:53:36.665 | INFO | "huge_shared" : " 27 GB", | 20260506 01:53:36.665 | INFO | "malloc" : " 14 GB", | 20260506 01:53:36.665 | INFO | "pinned" : " 29 GB" | 20260506 01:53:36.665 | INFO | } | 20260506 01:53:36.665 | INFO | Perf report for the last 5.13948 seconds: | 20260506 01:53:36.665 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:53:36.665 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:36.665 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:36.665 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:36.665 | INFO | ANALYSIS : 60.7%, 3.0, 94.8ms, 0 | 20260506 01:53:36.665 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:36.665 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:36.665 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 01:53:36.665 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:36.665 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:36.665 | INFO | Uploaded 36GiB to GPU in 2.08197s (17.2913GiB/s) | 20260506 01:53:36.666 | INFO | Downloaded 34GiB to GPU in 2.03417s (16.7144GiB/s) | 20260506 01:53:41.810 | INFO | Analyzing chunk frames = [8022528,8023040) | 20260506 01:53:41.810 | INFO | MemInfo: { | 20260506 01:53:41.810 | INFO | "gpu" : " 17 GB", | 20260506 01:53:41.810 | INFO | "huge" : "", | 20260506 01:53:41.810 | INFO | "huge_shared" : " 27 GB", | 20260506 01:53:41.811 | INFO | "malloc" : " 14 GB", | 20260506 01:53:41.811 | INFO | "pinned" : " 29 GB" | 20260506 01:53:41.811 | INFO | } | 20260506 01:53:41.811 | INFO | Perf report for the last 5.14551 seconds: | 20260506 01:53:41.811 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:53:41.811 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:41.811 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:41.811 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:41.811 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 01:53:41.811 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:41.811 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:41.811 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:53:41.811 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:41.811 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:41.811 | INFO | Uploaded 36GiB to GPU in 2.09419s (17.1904GiB/s) | 20260506 01:53:41.811 | INFO | Downloaded 34GiB to GPU in 2.0326s (16.7273GiB/s) | 20260506 01:53:46.943 | INFO | Analyzing chunk frames = [8023040,8023552) | 20260506 01:53:46.943 | INFO | MemInfo: { | 20260506 01:53:46.943 | INFO | "gpu" : " 17 GB", | 20260506 01:53:46.943 | INFO | "huge" : "", | 20260506 01:53:46.943 | INFO | "huge_shared" : " 27 GB", | 20260506 01:53:46.943 | INFO | "malloc" : " 14 GB", | 20260506 01:53:46.943 | INFO | "pinned" : " 29 GB" | 20260506 01:53:46.944 | INFO | } | 20260506 01:53:46.944 | INFO | Perf report for the last 5.13291 seconds: | 20260506 01:53:46.944 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:53:46.944 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:46.944 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:46.944 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:46.944 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 01:53:46.944 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:46.944 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:46.944 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 01:53:46.944 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:46.944 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:46.944 | INFO | Uploaded 36GiB to GPU in 2.05993s (17.4764GiB/s) | 20260506 01:53:46.944 | INFO | Downloaded 34GiB to GPU in 2.08022s (16.3444GiB/s) | 20260506 01:53:52.085 | INFO | Analyzing chunk frames = [8023552,8024064) | 20260506 01:53:52.085 | INFO | MemInfo: { | 20260506 01:53:52.085 | INFO | "gpu" : " 17 GB", | 20260506 01:53:52.085 | INFO | "huge" : "", | 20260506 01:53:52.085 | INFO | "huge_shared" : " 27 GB", | 20260506 01:53:52.085 | INFO | "malloc" : " 14 GB", | 20260506 01:53:52.085 | INFO | "pinned" : " 29 GB" | 20260506 01:53:52.085 | INFO | } | 20260506 01:53:52.085 | INFO | Perf report for the last 5.14127 seconds: | 20260506 01:53:52.085 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:53:52.085 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:52.085 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:52.085 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:52.085 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260506 01:53:52.085 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:52.085 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:52.085 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 01:53:52.085 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:52.085 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:52.085 | INFO | Uploaded 36GiB to GPU in 2.14357s (16.7944GiB/s) | 20260506 01:53:52.085 | INFO | Downloaded 34GiB to GPU in 2.03504s (16.7073GiB/s) | 20260506 01:53:52.211 | INFO | Starting a round of Baz IO | 20260506 01:53:55.348 | INFO | Populating RTMetrics report | 20260506 01:53:55.348 | INFO | RTMetrics report populated | 20260506 01:53:55.348 | INFO | Opening tmp RTMetrics json file | 20260506 01:53:55.348 | INFO | Serializing RTMetrics to json | 20260506 01:53:55.350 | INFO | Serialization done, swapping tmp file now... | 20260506 01:53:55.351 | INFO | RTMetrics done | 20260506 01:53:55.429 | INFO | Finished a round of Baz IO: Wrote 1.812 GiB in 3.217 seconds | 20260506 01:53:55.429 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.29% / 0.00% / 78.71% | 20260506 01:53:55.429 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0759085% / 0% / 7.94729e-05% | 20260506 01:53:55.429 | INFO | Baz IO overall: | 20260506 01:53:55.429 | INFO | BytesWritten : 4513181861772 | 20260506 01:53:55.429 | INFO | NumEvents : 2120235495703 | 20260506 01:53:57.227 | INFO | Analyzing chunk frames = [8024064,8024576) | 20260506 01:53:57.227 | INFO | MemInfo: { | 20260506 01:53:57.227 | INFO | "gpu" : " 17 GB", | 20260506 01:53:57.227 | INFO | "huge" : "", | 20260506 01:53:57.227 | INFO | "huge_shared" : " 27 GB", | 20260506 01:53:57.227 | INFO | "malloc" : " 14 GB", | 20260506 01:53:57.227 | INFO | "pinned" : " 29 GB" | 20260506 01:53:57.227 | INFO | } | 20260506 01:53:57.227 | INFO | Perf report for the last 5.14252 seconds: | 20260506 01:53:57.227 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:53:57.227 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:57.227 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:57.227 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:57.227 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260506 01:53:57.228 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:53:57.228 | INFO | RT_METRICS : 43.0%, 1.0, 22.5ms, 0 | 20260506 01:53:57.228 | INFO | PRE_HQ : 59.7%, 1.7, 53.9ms, 0 | 20260506 01:53:57.228 | INFO | BAZWRITER : 61.7%, 3.5, 112.4ms, 0 | 20260506 01:53:57.228 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:53:57.228 | INFO | Uploaded 36GiB to GPU in 2.18954s (16.4418GiB/s) | 20260506 01:53:57.228 | INFO | Downloaded 38GiB to GPU in 2.24807s (16.9034GiB/s) | 20260506 01:54:02.364 | INFO | Analyzing chunk frames = [8024576,8025088) | 20260506 01:54:02.364 | INFO | MemInfo: { | 20260506 01:54:02.364 | INFO | "gpu" : " 17 GB", | 20260506 01:54:02.364 | INFO | "huge" : "", | 20260506 01:54:02.364 | INFO | "huge_shared" : " 27 GB", | 20260506 01:54:02.364 | INFO | "malloc" : " 14 GB", | 20260506 01:54:02.364 | INFO | "pinned" : " 29 GB" | 20260506 01:54:02.364 | INFO | } | 20260506 01:54:02.364 | INFO | Perf report for the last 5.13658 seconds: | 20260506 01:54:02.364 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:54:02.364 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:02.364 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:02.364 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:02.364 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260506 01:54:02.364 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:02.364 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:02.364 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:54:02.364 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:02.364 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:02.364 | INFO | Uploaded 36GiB to GPU in 1.94325s (18.5256GiB/s) | 20260506 01:54:02.364 | INFO | Downloaded 34GiB to GPU in 1.98649s (17.1156GiB/s) | 20260506 01:54:07.501 | INFO | Analyzing chunk frames = [8025088,8025600) | 20260506 01:54:07.502 | INFO | MemInfo: { | 20260506 01:54:07.502 | INFO | "gpu" : " 17 GB", | 20260506 01:54:07.502 | INFO | "huge" : "", | 20260506 01:54:07.502 | INFO | "huge_shared" : " 27 GB", | 20260506 01:54:07.502 | INFO | "malloc" : " 14 GB", | 20260506 01:54:07.502 | INFO | "pinned" : " 29 GB" | 20260506 01:54:07.502 | INFO | } | 20260506 01:54:07.502 | INFO | Perf report for the last 5.13771 seconds: | 20260506 01:54:07.502 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:54:07.502 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:07.502 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:07.502 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:07.502 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 01:54:07.502 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:07.502 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:07.502 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:54:07.502 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:07.502 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:07.502 | INFO | Uploaded 36GiB to GPU in 2.09418s (17.1905GiB/s) | 20260506 01:54:07.502 | INFO | Downloaded 34GiB to GPU in 2.02475s (16.7922GiB/s) | 20260506 01:54:12.646 | INFO | Analyzing chunk frames = [8025600,8026112) | 20260506 01:54:12.646 | INFO | MemInfo: { | 20260506 01:54:12.646 | INFO | "gpu" : " 17 GB", | 20260506 01:54:12.646 | INFO | "huge" : "", | 20260506 01:54:12.646 | INFO | "huge_shared" : " 27 GB", | 20260506 01:54:12.646 | INFO | "malloc" : " 14 GB", | 20260506 01:54:12.646 | INFO | "pinned" : " 29 GB" | 20260506 01:54:12.646 | INFO | } | 20260506 01:54:12.646 | INFO | Perf report for the last 5.14467 seconds: | 20260506 01:54:12.646 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:54:12.646 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:12.646 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:12.646 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:12.646 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 01:54:12.647 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:12.647 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:12.647 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:54:12.647 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:12.647 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:12.647 | INFO | Uploaded 36GiB to GPU in 2.15354s (16.7166GiB/s) | 20260506 01:54:12.647 | INFO | Downloaded 34GiB to GPU in 2.05298s (16.5613GiB/s) | 20260506 01:54:17.780 | INFO | Analyzing chunk frames = [8026112,8026624) | 20260506 01:54:17.780 | INFO | MemInfo: { | 20260506 01:54:17.780 | INFO | "gpu" : " 17 GB", | 20260506 01:54:17.780 | INFO | "huge" : "", | 20260506 01:54:17.780 | INFO | "huge_shared" : " 27 GB", | 20260506 01:54:17.780 | INFO | "malloc" : " 14 GB", | 20260506 01:54:17.780 | INFO | "pinned" : " 29 GB" | 20260506 01:54:17.780 | INFO | } | 20260506 01:54:17.780 | INFO | Perf report for the last 5.13369 seconds: | 20260506 01:54:17.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:54:17.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:17.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:17.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:17.780 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260506 01:54:17.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:17.780 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:17.780 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:54:17.780 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:17.780 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:17.780 | INFO | Uploaded 36GiB to GPU in 2.08828s (17.239GiB/s) | 20260506 01:54:17.780 | INFO | Downloaded 34GiB to GPU in 2.03222s (16.7305GiB/s) | 20260506 01:54:22.921 | INFO | Analyzing chunk frames = [8026624,8027136) | 20260506 01:54:22.921 | INFO | MemInfo: { | 20260506 01:54:22.921 | INFO | "gpu" : " 17 GB", | 20260506 01:54:22.921 | INFO | "huge" : "", | 20260506 01:54:22.921 | INFO | "huge_shared" : " 27 GB", | 20260506 01:54:22.921 | INFO | "malloc" : " 14 GB", | 20260506 01:54:22.922 | INFO | "pinned" : " 29 GB" | 20260506 01:54:22.922 | INFO | } | 20260506 01:54:22.922 | INFO | Perf report for the last 5.14144 seconds: | 20260506 01:54:22.922 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:54:22.922 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:22.922 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:22.922 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:22.922 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 01:54:22.922 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:22.922 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:22.922 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 01:54:22.922 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:22.922 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:22.922 | INFO | Uploaded 36GiB to GPU in 2.06088s (17.4682GiB/s) | 20260506 01:54:22.922 | INFO | Downloaded 34GiB to GPU in 2.034s (16.7158GiB/s) | 20260506 01:54:28.062 | INFO | Analyzing chunk frames = [8027136,8027648) | 20260506 01:54:28.062 | INFO | MemInfo: { | 20260506 01:54:28.062 | INFO | "gpu" : " 17 GB", | 20260506 01:54:28.062 | INFO | "huge" : "", | 20260506 01:54:28.062 | INFO | "huge_shared" : " 27 GB", | 20260506 01:54:28.062 | INFO | "malloc" : " 14 GB", | 20260506 01:54:28.063 | INFO | "pinned" : " 29 GB" | 20260506 01:54:28.063 | INFO | } | 20260506 01:54:28.063 | INFO | Perf report for the last 5.14107 seconds: | 20260506 01:54:28.063 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:54:28.063 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:28.063 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:28.063 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:28.063 | INFO | ANALYSIS : 61.8%, 3.0, 96.6ms, 0 | 20260506 01:54:28.063 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:28.063 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:28.063 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 01:54:28.063 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:28.063 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:28.063 | INFO | Uploaded 36GiB to GPU in 2.0227s (17.798GiB/s) | 20260506 01:54:28.063 | INFO | Downloaded 34GiB to GPU in 2.04639s (16.6147GiB/s) | 20260506 01:54:33.197 | INFO | Analyzing chunk frames = [8027648,8028160) | 20260506 01:54:33.197 | INFO | MemInfo: { | 20260506 01:54:33.197 | INFO | "gpu" : " 17 GB", | 20260506 01:54:33.197 | INFO | "huge" : "", | 20260506 01:54:33.197 | INFO | "huge_shared" : " 27 GB", | 20260506 01:54:33.197 | INFO | "malloc" : " 14 GB", | 20260506 01:54:33.197 | INFO | "pinned" : " 29 GB" | 20260506 01:54:33.197 | INFO | } | 20260506 01:54:33.197 | INFO | Perf report for the last 5.13467 seconds: | 20260506 01:54:33.197 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:54:33.197 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:33.197 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:33.197 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:33.197 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 01:54:33.197 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:33.197 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:33.197 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 01:54:33.197 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:33.198 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:33.198 | INFO | Uploaded 36GiB to GPU in 2.03429s (17.6966GiB/s) | 20260506 01:54:33.198 | INFO | Downloaded 34GiB to GPU in 2.01349s (16.8861GiB/s) | 20260506 01:54:33.198 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 01:54:33.198 | INFO | Total mem usage: 15.3 GiB | 20260506 01:54:33.198 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 01:54:33.198 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -8.87%) | 20260506 01:54:33.198 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 01:54:33.198 | INFO | High Water Breakdown: | 20260506 01:54:33.198 | INFO | 61% : 8.58 GiB ( 90.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 01:54:33.198 | INFO | 38.9% : 5.48 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 01:54:33.198 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 01:54:33.198 | INFO | ------------------------------------------ | 20260506 01:54:33.198 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:54:33.198 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 01:54:33.198 | INFO | Total mem usage: 26.5 GiB | 20260506 01:54:33.198 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 01:54:33.198 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 01:54:33.198 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260506 01:54:33.198 | INFO | High Water Breakdown: | 20260506 01:54:33.198 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:54:33.198 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:54:33.198 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:54:33.198 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:54:33.198 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:54:33.198 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:54:33.198 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 01:54:33.198 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 01:54:33.198 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:54:33.198 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 01:54:33.198 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 01:54:33.198 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 01:54:33.198 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 01:54:33.198 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 01:54:33.198 | INFO | ------------------------------------------ | 20260506 01:54:33.198 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:54:33.198 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 01:54:33.198 | INFO | Total mem usage: 25.6 GiB | 20260506 01:54:33.198 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 01:54:33.199 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 01:54:33.199 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 01:54:33.199 | INFO | High Water Breakdown: | 20260506 01:54:33.199 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 01:54:33.199 | INFO | ------------------------------------------ | 20260506 01:54:33.199 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:54:33.199 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 01:54:33.199 | INFO | Total mem usage: 15.7 GiB | 20260506 01:54:33.199 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 01:54:33.199 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 01:54:33.199 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 01:54:33.199 | INFO | High Water Breakdown: | 20260506 01:54:33.199 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 01:54:33.199 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 01:54:33.199 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 01:54:33.199 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 01:54:33.199 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 01:54:33.199 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 01:54:33.199 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 01:54:33.199 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 01:54:33.199 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 01:54:33.199 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:54:33.199 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 01:54:33.199 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:54:33.199 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 01:54:33.199 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 01:54:33.199 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 01:54:33.199 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 01:54:33.199 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:54:33.199 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 01:54:33.199 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:54:33.199 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:54:33.199 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:54:33.199 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 01:54:33.199 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 01:54:33.199 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:54:33.199 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 01:54:33.199 | INFO | ------------------------------------------ | 20260506 01:54:33.199 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:54:33.199 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 01:54:33.199 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:54:33.199 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:54:33.199 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.38e+01, 1.34e+01 / 8.95e-01 | 20260506 01:54:33.199 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.05e-01, 1.05e-02 / 3.35e-03 | 20260506 01:54:33.199 | INFO | PerfReport: 11.86%, Baseline -- 3.71e+00 / 4.51e+00, 5.78e-01 / 2.06e-01 | 20260506 01:54:33.199 | INFO | PerfReport: 3.29%, Histogram -- 1.03e+00 / 1.51e+00, 1.91e-01 / 1.04e-01 | 20260506 01:54:33.199 | INFO | PerfReport: 16.01%, DME -- 4.00e+01 / 4.40e+01, 3.77e+00 / 1.07e+00 | 20260506 01:54:33.199 | INFO | PerfReport: 57.40%, FrameLabeling -- 1.79e+01 / 1.94e+01, 1.45e+00 / 1.17e+00 | 20260506 01:54:33.199 | INFO | PerfReport: 7.06%, PulseAccumulating -- 2.21e+00 / 2.71e+00, 4.19e-01 / 2.47e-01 | 20260506 01:54:33.199 | INFO | PerfReport: 3.03%, Metrics -- 9.48e-01 / 1.22e+00, 2.52e-01 / 1.17e-01 | 20260506 01:54:33.199 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.27e-01 / 3.76e-01, 5.44e-02 / 2.44e-02 | 20260506 01:54:33.199 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 01:54:33.199 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:54:33.199 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:54:33.199 | INFO | PerfReport: Full Execution -- 9.78e+01 / -nan, 2.42e+01 / 0.00e+00 | 20260506 01:54:33.199 | INFO | PerfReport: 22.20%, Upload -- 2.17e+01 / 0.00e+00, 7.26e+00 / 0.00e+00 | 20260506 01:54:33.199 | INFO | PerfReport: 21.91%, Download -- 2.14e+01 / 0.00e+00, 7.57e+00 / 0.00e+00 | 20260506 01:54:33.199 | INFO | PerfReport: 31.96%, Compute -- 3.12e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 01:54:33.199 | INFO | PerfReport: 2.08%, LockUpload -- 2.03e+00 / 0.00e+00, 7.90e+00 / 0.00e+00 | 20260506 01:54:33.199 | INFO | PerfReport: 1.31%, LockDownload -- 1.28e+00 / 0.00e+00, 2.92e+00 / 0.00e+00 | 20260506 01:54:33.199 | INFO | PerfReport: 20.53%, LockCompute -- 2.01e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260506 01:54:33.199 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.69e-02 / 0.00e+00 | 20260506 01:54:33.322 | INFO | Starting a round of Baz IO | 20260506 01:54:36.460 | INFO | Populating RTMetrics report | 20260506 01:54:36.461 | INFO | RTMetrics report populated | 20260506 01:54:36.461 | INFO | Opening tmp RTMetrics json file | 20260506 01:54:36.464 | INFO | Serializing RTMetrics to json | 20260506 01:54:36.468 | INFO | Serialization done, swapping tmp file now... | 20260506 01:54:36.470 | INFO | RTMetrics done | 20260506 01:54:36.542 | INFO | Finished a round of Baz IO: Wrote 1.81 GiB in 3.22 seconds | 20260506 01:54:36.542 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.29% / 0.00% / 78.71% | 20260506 01:54:36.542 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.075678% / 0% / 8.34465e-05% | 20260506 01:54:36.542 | INFO | Baz IO overall: | 20260506 01:54:36.543 | INFO | BytesWritten : 4515125442417 | 20260506 01:54:36.543 | INFO | NumEvents : 2121120256862 | 20260506 01:54:38.334 | INFO | Analyzing chunk frames = [8028160,8028672) | 20260506 01:54:38.334 | INFO | MemInfo: { | 20260506 01:54:38.334 | INFO | "gpu" : " 17 GB", | 20260506 01:54:38.334 | INFO | "huge" : "", | 20260506 01:54:38.334 | INFO | "huge_shared" : " 27 GB", | 20260506 01:54:38.334 | INFO | "malloc" : " 14 GB", | 20260506 01:54:38.334 | INFO | "pinned" : " 29 GB" | 20260506 01:54:38.334 | INFO | } | 20260506 01:54:38.334 | INFO | Perf report for the last 5.13687 seconds: | 20260506 01:54:38.334 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:54:38.334 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:38.334 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:38.334 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:38.334 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260506 01:54:38.334 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:38.334 | INFO | RT_METRICS : 43.0%, 1.0, 22.5ms, 0 | 20260506 01:54:38.334 | INFO | PRE_HQ : 59.8%, 1.7, 53.6ms, 0 | 20260506 01:54:38.334 | INFO | BAZWRITER : 61.9%, 3.5, 112.0ms, 0 | 20260506 01:54:38.334 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 01:54:38.335 | INFO | Uploaded 36GiB to GPU in 2.18611s (16.4676GiB/s) | 20260506 01:54:38.335 | INFO | Downloaded 38GiB to GPU in 2.2368s (16.9886GiB/s) | 20260506 01:54:43.475 | INFO | Analyzing chunk frames = [8028672,8029184) | 20260506 01:54:43.475 | INFO | MemInfo: { | 20260506 01:54:43.475 | INFO | "gpu" : " 17 GB", | 20260506 01:54:43.475 | INFO | "huge" : "", | 20260506 01:54:43.475 | INFO | "huge_shared" : " 27 GB", | 20260506 01:54:43.475 | INFO | "malloc" : " 14 GB", | 20260506 01:54:43.475 | INFO | "pinned" : " 29 GB" | 20260506 01:54:43.475 | INFO | } | 20260506 01:54:43.475 | INFO | Perf report for the last 5.14088 seconds: | 20260506 01:54:43.475 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:54:43.475 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:43.475 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:43.475 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:43.475 | INFO | ANALYSIS : 60.9%, 3.0, 95.1ms, 0 | 20260506 01:54:43.475 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:43.475 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:43.475 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 01:54:43.475 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:43.475 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:43.475 | INFO | Uploaded 36GiB to GPU in 1.96745s (18.2978GiB/s) | 20260506 01:54:43.475 | INFO | Downloaded 34GiB to GPU in 1.98249s (17.1502GiB/s) | 20260506 01:54:48.620 | INFO | Analyzing chunk frames = [8029184,8029696) | 20260506 01:54:48.620 | INFO | MemInfo: { | 20260506 01:54:48.620 | INFO | "gpu" : " 17 GB", | 20260506 01:54:48.620 | INFO | "huge" : "", | 20260506 01:54:48.620 | INFO | "huge_shared" : " 27 GB", | 20260506 01:54:48.620 | INFO | "malloc" : " 14 GB", | 20260506 01:54:48.620 | INFO | "pinned" : " 29 GB" | 20260506 01:54:48.620 | INFO | } | 20260506 01:54:48.620 | INFO | Perf report for the last 5.14529 seconds: | 20260506 01:54:48.620 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:54:48.620 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:48.620 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:48.620 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:48.620 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 01:54:48.621 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:48.621 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:48.621 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:54:48.621 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:48.621 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:48.621 | INFO | Uploaded 36GiB to GPU in 2.08845s (17.2376GiB/s) | 20260506 01:54:48.621 | INFO | Downloaded 34GiB to GPU in 2.03216s (16.731GiB/s) | 20260506 01:54:53.759 | INFO | Analyzing chunk frames = [8029696,8030208) | 20260506 01:54:53.759 | INFO | MemInfo: { | 20260506 01:54:53.759 | INFO | "gpu" : " 17 GB", | 20260506 01:54:53.759 | INFO | "huge" : "", | 20260506 01:54:53.759 | INFO | "huge_shared" : " 27 GB", | 20260506 01:54:53.759 | INFO | "malloc" : " 14 GB", | 20260506 01:54:53.759 | INFO | "pinned" : " 29 GB" | 20260506 01:54:53.759 | INFO | } | 20260506 01:54:53.759 | INFO | Perf report for the last 5.13878 seconds: | 20260506 01:54:53.759 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:54:53.759 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:53.759 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:53.759 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:53.759 | INFO | ANALYSIS : 61.5%, 3.0, 95.5ms, 0 | 20260506 01:54:53.759 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:53.759 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:53.759 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:54:53.759 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:53.759 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:53.760 | INFO | Uploaded 36GiB to GPU in 2.15529s (16.7031GiB/s) | 20260506 01:54:53.760 | INFO | Downloaded 34GiB to GPU in 2.05957s (16.5083GiB/s) | 20260506 01:54:58.893 | INFO | Analyzing chunk frames = [8030208,8030720) | 20260506 01:54:58.893 | INFO | MemInfo: { | 20260506 01:54:58.893 | INFO | "gpu" : " 17 GB", | 20260506 01:54:58.893 | INFO | "huge" : "", | 20260506 01:54:58.893 | INFO | "huge_shared" : " 27 GB", | 20260506 01:54:58.893 | INFO | "malloc" : " 14 GB", | 20260506 01:54:58.893 | INFO | "pinned" : " 29 GB" | 20260506 01:54:58.893 | INFO | } | 20260506 01:54:58.893 | INFO | Perf report for the last 5.13353 seconds: | 20260506 01:54:58.893 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:54:58.893 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:58.893 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:58.893 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:58.893 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260506 01:54:58.893 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:58.893 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:58.893 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:54:58.893 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:58.893 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:54:58.893 | INFO | Uploaded 36GiB to GPU in 2.07495s (17.3498GiB/s) | 20260506 01:54:58.893 | INFO | Downloaded 34GiB to GPU in 2.03023s (16.7469GiB/s) | 20260506 01:55:04.037 | INFO | Analyzing chunk frames = [8030720,8031232) | 20260506 01:55:04.037 | INFO | MemInfo: { | 20260506 01:55:04.037 | INFO | "gpu" : " 17 GB", | 20260506 01:55:04.037 | INFO | "huge" : "", | 20260506 01:55:04.037 | INFO | "huge_shared" : " 27 GB", | 20260506 01:55:04.037 | INFO | "malloc" : " 14 GB", | 20260506 01:55:04.037 | INFO | "pinned" : " 29 GB" | 20260506 01:55:04.037 | INFO | } | 20260506 01:55:04.037 | INFO | Perf report for the last 5.14417 seconds: | 20260506 01:55:04.037 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:55:04.037 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:04.037 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:04.037 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:04.037 | INFO | ANALYSIS : 61.1%, 3.0, 95.5ms, 0 | 20260506 01:55:04.037 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:04.037 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:04.037 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 01:55:04.037 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:04.037 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:04.037 | INFO | Uploaded 36GiB to GPU in 2.0652s (17.4318GiB/s) | 20260506 01:55:04.037 | INFO | Downloaded 34GiB to GPU in 2.03524s (16.7056GiB/s) | 20260506 01:55:09.177 | INFO | Analyzing chunk frames = [8031232,8031744) | 20260506 01:55:09.177 | INFO | MemInfo: { | 20260506 01:55:09.177 | INFO | "gpu" : " 17 GB", | 20260506 01:55:09.177 | INFO | "huge" : "", | 20260506 01:55:09.177 | INFO | "huge_shared" : " 27 GB", | 20260506 01:55:09.177 | INFO | "malloc" : " 14 GB", | 20260506 01:55:09.177 | INFO | "pinned" : " 29 GB" | 20260506 01:55:09.178 | INFO | } | 20260506 01:55:09.178 | INFO | Perf report for the last 5.14068 seconds: | 20260506 01:55:09.178 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:55:09.178 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:09.178 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:09.178 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:09.178 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260506 01:55:09.178 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:09.178 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:09.178 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 01:55:09.178 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:09.178 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:09.178 | INFO | Uploaded 36GiB to GPU in 2.03866s (17.6587GiB/s) | 20260506 01:55:09.178 | INFO | Downloaded 34GiB to GPU in 2.07414s (16.3924GiB/s) | 20260506 01:55:14.312 | INFO | Analyzing chunk frames = [8031744,8032256) | 20260506 01:55:14.312 | INFO | MemInfo: { | 20260506 01:55:14.312 | INFO | "gpu" : " 17 GB", | 20260506 01:55:14.312 | INFO | "huge" : "", | 20260506 01:55:14.312 | INFO | "huge_shared" : " 27 GB", | 20260506 01:55:14.313 | INFO | "malloc" : " 14 GB", | 20260506 01:55:14.313 | INFO | "pinned" : " 29 GB" | 20260506 01:55:14.313 | INFO | } | 20260506 01:55:14.313 | INFO | Perf report for the last 5.13501 seconds: | 20260506 01:55:14.313 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:55:14.313 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:14.313 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:14.313 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:14.313 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 01:55:14.313 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:14.313 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:14.313 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 01:55:14.313 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:14.313 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:14.313 | INFO | Uploaded 36GiB to GPU in 2.03363s (17.7023GiB/s) | 20260506 01:55:14.313 | INFO | Downloaded 34GiB to GPU in 2.01588s (16.8661GiB/s) | 20260506 01:55:14.439 | INFO | Starting a round of Baz IO | 20260506 01:55:17.572 | INFO | Populating RTMetrics report | 20260506 01:55:17.572 | INFO | RTMetrics report populated | 20260506 01:55:17.572 | INFO | Opening tmp RTMetrics json file | 20260506 01:55:17.573 | INFO | Serializing RTMetrics to json | 20260506 01:55:17.576 | INFO | Serialization done, swapping tmp file now... | 20260506 01:55:17.579 | INFO | RTMetrics done | 20260506 01:55:17.658 | INFO | Finished a round of Baz IO: Wrote 1.809 GiB in 3.219 seconds | 20260506 01:55:17.658 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.29% / 0.00% / 78.71% | 20260506 01:55:17.658 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0751972% / 0% / 6.35783e-05% | 20260506 01:55:17.658 | INFO | Baz IO overall: | 20260506 01:55:17.658 | INFO | BytesWritten : 4517067969190 | 20260506 01:55:17.658 | INFO | NumEvents : 2122004334328 | 20260506 01:55:19.451 | INFO | Analyzing chunk frames = [8032256,8032768) | 20260506 01:55:19.451 | INFO | MemInfo: { | 20260506 01:55:19.451 | INFO | "gpu" : " 17 GB", | 20260506 01:55:19.451 | INFO | "huge" : "", | 20260506 01:55:19.451 | INFO | "huge_shared" : " 27 GB", | 20260506 01:55:19.451 | INFO | "malloc" : " 14 GB", | 20260506 01:55:19.451 | INFO | "pinned" : " 29 GB" | 20260506 01:55:19.451 | INFO | } | 20260506 01:55:19.451 | INFO | Perf report for the last 5.13862 seconds: | 20260506 01:55:19.451 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:55:19.451 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:19.451 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:19.451 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:19.451 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260506 01:55:19.451 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:19.451 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260506 01:55:19.451 | INFO | PRE_HQ : 59.6%, 1.7, 53.6ms, 0 | 20260506 01:55:19.451 | INFO | BAZWRITER : 61.9%, 3.5, 112.2ms, 0 | 20260506 01:55:19.451 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 01:55:19.452 | INFO | Uploaded 36GiB to GPU in 2.19055s (16.4342GiB/s) | 20260506 01:55:19.452 | INFO | Downloaded 38GiB to GPU in 2.24607s (16.9185GiB/s) | 20260506 01:55:24.600 | INFO | Analyzing chunk frames = [8032768,8033280) | 20260506 01:55:24.600 | INFO | MemInfo: { | 20260506 01:55:24.600 | INFO | "gpu" : " 17 GB", | 20260506 01:55:24.600 | INFO | "huge" : "", | 20260506 01:55:24.601 | INFO | "huge_shared" : " 27 GB", | 20260506 01:55:24.601 | INFO | "malloc" : " 14 GB", | 20260506 01:55:24.601 | INFO | "pinned" : " 29 GB" | 20260506 01:55:24.601 | INFO | } | 20260506 01:55:24.601 | INFO | Perf report for the last 5.14943 seconds: | 20260506 01:55:24.601 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:55:24.601 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:24.601 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:24.601 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:24.601 | INFO | ANALYSIS : 61.2%, 3.0, 95.8ms, 0 | 20260506 01:55:24.601 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:24.601 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:24.601 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 01:55:24.601 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:24.601 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:24.601 | INFO | Uploaded 36GiB to GPU in 2.02679s (17.7621GiB/s) | 20260506 01:55:24.601 | INFO | Downloaded 34GiB to GPU in 2.00556s (16.9529GiB/s) | 20260506 01:55:29.731 | INFO | Analyzing chunk frames = [8033280,8033792) | 20260506 01:55:29.731 | INFO | MemInfo: { | 20260506 01:55:29.731 | INFO | "gpu" : " 17 GB", | 20260506 01:55:29.731 | INFO | "huge" : "", | 20260506 01:55:29.731 | INFO | "huge_shared" : " 27 GB", | 20260506 01:55:29.731 | INFO | "malloc" : " 14 GB", | 20260506 01:55:29.731 | INFO | "pinned" : " 29 GB" | 20260506 01:55:29.731 | INFO | } | 20260506 01:55:29.731 | INFO | Perf report for the last 5.13046 seconds: | 20260506 01:55:29.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:55:29.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:29.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:29.731 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:29.731 | INFO | ANALYSIS : 61.2%, 3.0, 95.1ms, 0 | 20260506 01:55:29.731 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:29.731 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:29.731 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:55:29.731 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:29.731 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:29.732 | INFO | Uploaded 36GiB to GPU in 2.09794s (17.1597GiB/s) | 20260506 01:55:29.732 | INFO | Downloaded 34GiB to GPU in 2.03688s (16.6922GiB/s) | 20260506 01:55:34.873 | INFO | Analyzing chunk frames = [8033792,8034304) | 20260506 01:55:34.873 | INFO | MemInfo: { | 20260506 01:55:34.873 | INFO | "gpu" : " 17 GB", | 20260506 01:55:34.873 | INFO | "huge" : "", | 20260506 01:55:34.874 | INFO | "huge_shared" : " 27 GB", | 20260506 01:55:34.874 | INFO | "malloc" : " 14 GB", | 20260506 01:55:34.874 | INFO | "pinned" : " 29 GB" | 20260506 01:55:34.874 | INFO | } | 20260506 01:55:34.874 | INFO | Perf report for the last 5.14253 seconds: | 20260506 01:55:34.874 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:55:34.874 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:34.874 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:34.874 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:34.874 | INFO | ANALYSIS : 61.2%, 3.0, 95.1ms, 0 | 20260506 01:55:34.874 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:34.874 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:34.874 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:55:34.874 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:34.874 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:34.874 | INFO | Uploaded 36GiB to GPU in 2.08973s (17.2271GiB/s) | 20260506 01:55:34.874 | INFO | Downloaded 34GiB to GPU in 2.02771s (16.7677GiB/s) | 20260506 01:55:40.007 | INFO | Analyzing chunk frames = [8034304,8034816) | 20260506 01:55:40.007 | INFO | MemInfo: { | 20260506 01:55:40.007 | INFO | "gpu" : " 17 GB", | 20260506 01:55:40.007 | INFO | "huge" : "", | 20260506 01:55:40.007 | INFO | "huge_shared" : " 27 GB", | 20260506 01:55:40.007 | INFO | "malloc" : " 14 GB", | 20260506 01:55:40.007 | INFO | "pinned" : " 29 GB" | 20260506 01:55:40.007 | INFO | } | 20260506 01:55:40.007 | INFO | Perf report for the last 5.13319 seconds: | 20260506 01:55:40.007 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:55:40.007 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:40.007 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:40.007 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:40.007 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260506 01:55:40.007 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:40.007 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:40.007 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 01:55:40.007 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:40.007 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:40.007 | INFO | Uploaded 36GiB to GPU in 2.09082s (17.2182GiB/s) | 20260506 01:55:40.007 | INFO | Downloaded 34GiB to GPU in 2.03242s (16.7288GiB/s) | 20260506 01:55:45.143 | INFO | Analyzing chunk frames = [8034816,8035328) | 20260506 01:55:45.143 | INFO | MemInfo: { | 20260506 01:55:45.143 | INFO | "gpu" : " 17 GB", | 20260506 01:55:45.143 | INFO | "huge" : "", | 20260506 01:55:45.143 | INFO | "huge_shared" : " 27 GB", | 20260506 01:55:45.143 | INFO | "malloc" : " 14 GB", | 20260506 01:55:45.143 | INFO | "pinned" : " 29 GB" | 20260506 01:55:45.143 | INFO | } | 20260506 01:55:45.143 | INFO | Perf report for the last 5.13638 seconds: | 20260506 01:55:45.143 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:55:45.143 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:45.143 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:45.143 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:45.143 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 01:55:45.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:45.143 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:45.143 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 01:55:45.144 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:45.144 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:45.144 | INFO | Uploaded 36GiB to GPU in 2.07911s (17.3151GiB/s) | 20260506 01:55:45.144 | INFO | Downloaded 34GiB to GPU in 2.0375s (16.6871GiB/s) | 20260506 01:55:50.285 | INFO | Analyzing chunk frames = [8035328,8035840) | 20260506 01:55:50.285 | INFO | MemInfo: { | 20260506 01:55:50.285 | INFO | "gpu" : " 17 GB", | 20260506 01:55:50.285 | INFO | "huge" : "", | 20260506 01:55:50.285 | INFO | "huge_shared" : " 27 GB", | 20260506 01:55:50.285 | INFO | "malloc" : " 14 GB", | 20260506 01:55:50.285 | INFO | "pinned" : " 29 GB" | 20260506 01:55:50.285 | INFO | } | 20260506 01:55:50.285 | INFO | Perf report for the last 5.14163 seconds: | 20260506 01:55:50.285 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:55:50.285 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:50.285 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:50.285 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:50.285 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260506 01:55:50.285 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:50.285 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:50.285 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 01:55:50.285 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:50.285 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:50.285 | INFO | Uploaded 36GiB to GPU in 2.03073s (17.7276GiB/s) | 20260506 01:55:50.286 | INFO | Downloaded 34GiB to GPU in 2.06735s (16.4462GiB/s) | 20260506 01:55:55.427 | INFO | Analyzing chunk frames = [8035840,8036352) | 20260506 01:55:55.427 | INFO | MemInfo: { | 20260506 01:55:55.427 | INFO | "gpu" : " 17 GB", | 20260506 01:55:55.427 | INFO | "huge" : "", | 20260506 01:55:55.427 | INFO | "huge_shared" : " 27 GB", | 20260506 01:55:55.427 | INFO | "malloc" : " 14 GB", | 20260506 01:55:55.427 | INFO | "pinned" : " 29 GB" | 20260506 01:55:55.427 | INFO | } | 20260506 01:55:55.427 | INFO | Perf report for the last 5.14208 seconds: | 20260506 01:55:55.427 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:55:55.427 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:55.427 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:55.427 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:55.427 | INFO | ANALYSIS : 61.4%, 3.0, 96.0ms, 0 | 20260506 01:55:55.427 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:55.427 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:55.427 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:55:55.427 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:55.427 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:55:55.427 | INFO | Uploaded 36GiB to GPU in 2.065s (17.4334GiB/s) | 20260506 01:55:55.427 | INFO | Downloaded 34GiB to GPU in 2.02609s (16.7811GiB/s) | 20260506 01:55:55.428 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 01:55:55.428 | INFO | Total mem usage: 15.3 GiB | 20260506 01:55:55.428 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 01:55:55.428 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.21%) | 20260506 01:55:55.428 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 01:55:55.428 | INFO | High Water Breakdown: | 20260506 01:55:55.428 | INFO | 60.9% : 8.54 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 01:55:55.428 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 01:55:55.428 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 01:55:55.428 | INFO | ------------------------------------------ | 20260506 01:55:55.428 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:55:55.428 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 01:55:55.428 | INFO | Total mem usage: 26.5 GiB | 20260506 01:55:55.428 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 01:55:55.428 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 01:55:55.428 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260506 01:55:55.428 | INFO | High Water Breakdown: | 20260506 01:55:55.428 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:55:55.428 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:55:55.428 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:55:55.428 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:55:55.428 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:55:55.428 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:55:55.428 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 01:55:55.428 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 01:55:55.428 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:55:55.428 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 01:55:55.428 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 01:55:55.428 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 01:55:55.428 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 01:55:55.428 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 01:55:55.428 | INFO | ------------------------------------------ | 20260506 01:55:55.428 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:55:55.428 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 01:55:55.428 | INFO | Total mem usage: 25.6 GiB | 20260506 01:55:55.428 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 01:55:55.428 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 01:55:55.428 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 01:55:55.428 | INFO | High Water Breakdown: | 20260506 01:55:55.428 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 01:55:55.428 | INFO | ------------------------------------------ | 20260506 01:55:55.428 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:55:55.428 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 01:55:55.428 | INFO | Total mem usage: 15.7 GiB | 20260506 01:55:55.428 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 01:55:55.428 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 01:55:55.428 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 01:55:55.428 | INFO | High Water Breakdown: | 20260506 01:55:55.428 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 01:55:55.428 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 01:55:55.428 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 01:55:55.428 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 01:55:55.428 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 01:55:55.428 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 01:55:55.428 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 01:55:55.428 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 01:55:55.428 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 01:55:55.428 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:55:55.428 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 01:55:55.428 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:55:55.428 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 01:55:55.428 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 01:55:55.428 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 01:55:55.428 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 01:55:55.428 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:55:55.428 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 01:55:55.428 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:55:55.429 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:55:55.429 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:55:55.429 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 01:55:55.429 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 01:55:55.429 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:55:55.429 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 01:55:55.429 | INFO | ------------------------------------------ | 20260506 01:55:55.429 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:55:55.429 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 01:55:55.429 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:55:55.429 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:55:55.429 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.40e+01, 1.34e+01 / 8.39e-01 | 20260506 01:55:55.429 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.05e-01, 1.10e-02 / 4.03e-03 | 20260506 01:55:55.429 | INFO | PerfReport: 11.83%, Baseline -- 3.71e+00 / 4.47e+00, 5.80e-01 / 2.63e-01 | 20260506 01:55:55.429 | INFO | PerfReport: 3.29%, Histogram -- 1.03e+00 / 1.50e+00, 1.91e-01 / 1.10e-01 | 20260506 01:55:55.429 | INFO | PerfReport: 15.99%, DME -- 4.01e+01 / 4.42e+01, 3.99e+00 / 1.19e+00 | 20260506 01:55:55.429 | INFO | PerfReport: 57.55%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.61e+00 / 1.22e+00 | 20260506 01:55:55.429 | INFO | PerfReport: 6.99%, PulseAccumulating -- 2.19e+00 / 2.69e+00, 4.14e-01 / 2.69e-01 | 20260506 01:55:55.429 | INFO | PerfReport: 3.01%, Metrics -- 9.44e-01 / 1.20e+00, 2.52e-01 / 1.05e-01 | 20260506 01:55:55.429 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.27e-01 / 3.71e-01, 5.49e-02 / 2.40e-02 | 20260506 01:55:55.429 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 01:55:55.429 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:55:55.429 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:55:55.429 | INFO | PerfReport: Full Execution -- 9.78e+01 / -nan, 2.41e+01 / 0.00e+00 | 20260506 01:55:55.429 | INFO | PerfReport: 22.16%, Upload -- 2.17e+01 / 0.00e+00, 7.24e+00 / 0.00e+00 | 20260506 01:55:55.429 | INFO | PerfReport: 21.94%, Download -- 2.15e+01 / 0.00e+00, 7.60e+00 / 0.00e+00 | 20260506 01:55:55.429 | INFO | PerfReport: 32.06%, Compute -- 3.14e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 01:55:55.429 | INFO | PerfReport: 1.99%, LockUpload -- 1.95e+00 / 0.00e+00, 7.80e+00 / 0.00e+00 | 20260506 01:55:55.429 | INFO | PerfReport: 1.28%, LockDownload -- 1.25e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260506 01:55:55.429 | INFO | PerfReport: 20.56%, LockCompute -- 2.01e+01 / 0.00e+00, 1.75e+01 / 0.00e+00 | 20260506 01:55:55.429 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.70e-02 / 0.00e+00 | 20260506 01:55:55.553 | INFO | Starting a round of Baz IO | 20260506 01:55:58.686 | INFO | Populating RTMetrics report | 20260506 01:55:58.686 | INFO | RTMetrics report populated | 20260506 01:55:58.686 | INFO | Opening tmp RTMetrics json file | 20260506 01:55:58.687 | INFO | Serializing RTMetrics to json | 20260506 01:55:58.689 | INFO | Serialization done, swapping tmp file now... | 20260506 01:55:58.690 | INFO | RTMetrics done | 20260506 01:55:58.762 | INFO | Finished a round of Baz IO: Wrote 1.809 GiB in 3.208 seconds | 20260506 01:55:58.762 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.29% / 0.00% / 78.71% | 20260506 01:55:58.762 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0751813% / 3.97364e-06% / 9.93411e-05% | 20260506 01:55:58.762 | INFO | Baz IO overall: | 20260506 01:55:58.762 | INFO | BytesWritten : 4519009923466 | 20260506 01:55:58.762 | INFO | NumEvents : 2122888143887 | 20260506 01:56:00.570 | INFO | Analyzing chunk frames = [8036352,8036864) | 20260506 01:56:00.570 | INFO | MemInfo: { | 20260506 01:56:00.570 | INFO | "gpu" : " 17 GB", | 20260506 01:56:00.570 | INFO | "huge" : "", | 20260506 01:56:00.570 | INFO | "huge_shared" : " 27 GB", | 20260506 01:56:00.570 | INFO | "malloc" : " 14 GB", | 20260506 01:56:00.570 | INFO | "pinned" : " 29 GB" | 20260506 01:56:00.570 | INFO | } | 20260506 01:56:00.570 | INFO | Perf report for the last 5.14295 seconds: | 20260506 01:56:00.570 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:56:00.570 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:00.570 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:00.570 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:00.570 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260506 01:56:00.570 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:00.570 | INFO | RT_METRICS : 42.8%, 1.0, 22.5ms, 0 | 20260506 01:56:00.570 | INFO | PRE_HQ : 59.3%, 1.7, 53.5ms, 0 | 20260506 01:56:00.570 | INFO | BAZWRITER : 61.7%, 3.5, 112.3ms, 0 | 20260506 01:56:00.570 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:56:00.570 | INFO | Uploaded 36GiB to GPU in 2.1515s (16.7325GiB/s) | 20260506 01:56:00.570 | INFO | Downloaded 38GiB to GPU in 2.23968s (16.9667GiB/s) | 20260506 01:56:05.710 | INFO | Analyzing chunk frames = [8036864,8037376) | 20260506 01:56:05.710 | INFO | MemInfo: { | 20260506 01:56:05.710 | INFO | "gpu" : " 17 GB", | 20260506 01:56:05.710 | INFO | "huge" : "", | 20260506 01:56:05.710 | INFO | "huge_shared" : " 27 GB", | 20260506 01:56:05.710 | INFO | "malloc" : " 14 GB", | 20260506 01:56:05.710 | INFO | "pinned" : " 29 GB" | 20260506 01:56:05.711 | INFO | } | 20260506 01:56:05.711 | INFO | Perf report for the last 5.14051 seconds: | 20260506 01:56:05.711 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:56:05.711 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:05.711 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:05.711 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:05.711 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 01:56:05.711 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:05.711 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:05.711 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 01:56:05.711 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:05.711 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:05.711 | INFO | Uploaded 36GiB to GPU in 1.97403s (18.2368GiB/s) | 20260506 01:56:05.711 | INFO | Downloaded 34GiB to GPU in 1.9881s (17.1018GiB/s) | 20260506 01:56:10.846 | INFO | Analyzing chunk frames = [8037376,8037888) | 20260506 01:56:10.846 | INFO | MemInfo: { | 20260506 01:56:10.846 | INFO | "gpu" : " 17 GB", | 20260506 01:56:10.846 | INFO | "huge" : "", | 20260506 01:56:10.846 | INFO | "huge_shared" : " 27 GB", | 20260506 01:56:10.846 | INFO | "malloc" : " 14 GB", | 20260506 01:56:10.846 | INFO | "pinned" : " 29 GB" | 20260506 01:56:10.846 | INFO | } | 20260506 01:56:10.846 | INFO | Perf report for the last 5.1358 seconds: | 20260506 01:56:10.846 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:56:10.846 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:10.846 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:10.846 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:10.846 | INFO | ANALYSIS : 61.3%, 3.0, 95.2ms, 0 | 20260506 01:56:10.846 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:10.846 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:10.846 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 01:56:10.846 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:10.846 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:10.847 | INFO | Uploaded 36GiB to GPU in 2.0795s (17.3119GiB/s) | 20260506 01:56:10.847 | INFO | Downloaded 34GiB to GPU in 2.02163s (16.8181GiB/s) | 20260506 01:56:15.987 | INFO | Analyzing chunk frames = [8037888,8038400) | 20260506 01:56:15.987 | INFO | MemInfo: { | 20260506 01:56:15.987 | INFO | "gpu" : " 17 GB", | 20260506 01:56:15.987 | INFO | "huge" : "", | 20260506 01:56:15.987 | INFO | "huge_shared" : " 27 GB", | 20260506 01:56:15.987 | INFO | "malloc" : " 14 GB", | 20260506 01:56:15.987 | INFO | "pinned" : " 29 GB" | 20260506 01:56:15.987 | INFO | } | 20260506 01:56:15.987 | INFO | Perf report for the last 5.14104 seconds: | 20260506 01:56:15.987 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:56:15.987 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:15.987 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:15.987 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:15.987 | INFO | ANALYSIS : 61.2%, 3.0, 95.0ms, 0 | 20260506 01:56:15.988 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:15.988 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:15.988 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:56:15.988 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:15.988 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:15.988 | INFO | Uploaded 36GiB to GPU in 2.09282s (17.2017GiB/s) | 20260506 01:56:15.988 | INFO | Downloaded 34GiB to GPU in 2.03079s (16.7423GiB/s) | 20260506 01:56:21.127 | INFO | Analyzing chunk frames = [8038400,8038912) | 20260506 01:56:21.127 | INFO | MemInfo: { | 20260506 01:56:21.127 | INFO | "gpu" : " 17 GB", | 20260506 01:56:21.127 | INFO | "huge" : "", | 20260506 01:56:21.127 | INFO | "huge_shared" : " 27 GB", | 20260506 01:56:21.127 | INFO | "malloc" : " 14 GB", | 20260506 01:56:21.127 | INFO | "pinned" : " 29 GB" | 20260506 01:56:21.127 | INFO | } | 20260506 01:56:21.127 | INFO | Perf report for the last 5.13941 seconds: | 20260506 01:56:21.127 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:56:21.127 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:21.127 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:21.127 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:21.127 | INFO | ANALYSIS : 60.7%, 3.0, 94.8ms, 0 | 20260506 01:56:21.127 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:21.127 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:21.127 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 01:56:21.127 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:21.127 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:21.127 | INFO | Uploaded 36GiB to GPU in 2.10221s (17.1248GiB/s) | 20260506 01:56:21.127 | INFO | Downloaded 34GiB to GPU in 2.03629s (16.697GiB/s) | 20260506 01:56:26.265 | INFO | Analyzing chunk frames = [8038912,8039424) | 20260506 01:56:26.265 | INFO | MemInfo: { | 20260506 01:56:26.265 | INFO | "gpu" : " 17 GB", | 20260506 01:56:26.265 | INFO | "huge" : "", | 20260506 01:56:26.265 | INFO | "huge_shared" : " 27 GB", | 20260506 01:56:26.265 | INFO | "malloc" : " 14 GB", | 20260506 01:56:26.265 | INFO | "pinned" : " 29 GB" | 20260506 01:56:26.265 | INFO | } | 20260506 01:56:26.265 | INFO | Perf report for the last 5.13815 seconds: | 20260506 01:56:26.265 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:56:26.265 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:26.265 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:26.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:26.265 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 01:56:26.265 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:26.265 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:26.265 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 01:56:26.265 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:26.265 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:26.265 | INFO | Uploaded 36GiB to GPU in 2.08979s (17.2266GiB/s) | 20260506 01:56:26.265 | INFO | Downloaded 34GiB to GPU in 2.03454s (16.7114GiB/s) | 20260506 01:56:31.398 | INFO | Analyzing chunk frames = [8039424,8039936) | 20260506 01:56:31.398 | INFO | MemInfo: { | 20260506 01:56:31.398 | INFO | "gpu" : " 17 GB", | 20260506 01:56:31.398 | INFO | "huge" : "", | 20260506 01:56:31.398 | INFO | "huge_shared" : " 27 GB", | 20260506 01:56:31.399 | INFO | "malloc" : " 14 GB", | 20260506 01:56:31.399 | INFO | "pinned" : " 29 GB" | 20260506 01:56:31.399 | INFO | } | 20260506 01:56:31.399 | INFO | Perf report for the last 5.13364 seconds: | 20260506 01:56:31.399 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:56:31.399 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:31.399 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:31.399 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:31.399 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260506 01:56:31.399 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:31.399 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:31.399 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 01:56:31.399 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:31.399 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:31.399 | INFO | Uploaded 36GiB to GPU in 2.12802s (16.9171GiB/s) | 20260506 01:56:31.399 | INFO | Downloaded 34GiB to GPU in 2.12022s (16.036GiB/s) | 20260506 01:56:36.547 | INFO | Analyzing chunk frames = [8039936,8040448) | 20260506 01:56:36.547 | INFO | MemInfo: { | 20260506 01:56:36.547 | INFO | "gpu" : " 17 GB", | 20260506 01:56:36.547 | INFO | "huge" : "", | 20260506 01:56:36.547 | INFO | "huge_shared" : " 27 GB", | 20260506 01:56:36.547 | INFO | "malloc" : " 14 GB", | 20260506 01:56:36.547 | INFO | "pinned" : " 29 GB" | 20260506 01:56:36.547 | INFO | } | 20260506 01:56:36.547 | INFO | Perf report for the last 5.14813 seconds: | 20260506 01:56:36.547 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:56:36.547 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:36.547 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:36.547 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:36.547 | INFO | ANALYSIS : 61.6%, 3.0, 96.3ms, 0 | 20260506 01:56:36.547 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:36.547 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:36.547 | INFO | PRE_HQ : 26.4%, 1.0, 13.8ms, 0 | 20260506 01:56:36.547 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:36.547 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:36.547 | INFO | Uploaded 36GiB to GPU in 2.07847s (17.3204GiB/s) | 20260506 01:56:36.547 | INFO | Downloaded 34GiB to GPU in 2.02656s (16.7772GiB/s) | 20260506 01:56:36.672 | INFO | Starting a round of Baz IO | 20260506 01:56:39.804 | INFO | Populating RTMetrics report | 20260506 01:56:39.805 | INFO | RTMetrics report populated | 20260506 01:56:39.805 | INFO | Opening tmp RTMetrics json file | 20260506 01:56:39.805 | INFO | Serializing RTMetrics to json | 20260506 01:56:39.807 | INFO | Serialization done, swapping tmp file now... | 20260506 01:56:39.807 | INFO | RTMetrics done | 20260506 01:56:39.882 | INFO | Finished a round of Baz IO: Wrote 1.807 GiB in 3.21 seconds | 20260506 01:56:39.882 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.29% / 0.00% / 78.71% | 20260506 01:56:39.882 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0750979% / 0% / 0.000115236% | 20260506 01:56:39.882 | INFO | Baz IO overall: | 20260506 01:56:39.882 | INFO | BytesWritten : 4520950332518 | 20260506 01:56:39.882 | INFO | NumEvents : 2123770746077 | 20260506 01:56:41.685 | INFO | Analyzing chunk frames = [8040448,8040960) | 20260506 01:56:41.685 | INFO | MemInfo: { | 20260506 01:56:41.685 | INFO | "gpu" : " 17 GB", | 20260506 01:56:41.685 | INFO | "huge" : "", | 20260506 01:56:41.685 | INFO | "huge_shared" : " 27 GB", | 20260506 01:56:41.685 | INFO | "malloc" : " 14 GB", | 20260506 01:56:41.685 | INFO | "pinned" : " 29 GB" | 20260506 01:56:41.685 | INFO | } | 20260506 01:56:41.685 | INFO | Perf report for the last 5.13838 seconds: | 20260506 01:56:41.685 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:56:41.685 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:41.685 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:41.685 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:41.685 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260506 01:56:41.685 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:41.685 | INFO | RT_METRICS : 43.0%, 1.0, 22.5ms, 0 | 20260506 01:56:41.685 | INFO | PRE_HQ : 59.7%, 1.7, 53.9ms, 0 | 20260506 01:56:41.685 | INFO | BAZWRITER : 61.8%, 3.5, 110.6ms, 0 | 20260506 01:56:41.685 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:56:41.685 | INFO | Uploaded 36GiB to GPU in 2.18071s (16.5084GiB/s) | 20260506 01:56:41.686 | INFO | Downloaded 38GiB to GPU in 2.24612s (16.9181GiB/s) | 20260506 01:56:46.821 | INFO | Analyzing chunk frames = [8040960,8041472) | 20260506 01:56:46.821 | INFO | MemInfo: { | 20260506 01:56:46.821 | INFO | "gpu" : " 17 GB", | 20260506 01:56:46.821 | INFO | "huge" : "", | 20260506 01:56:46.821 | INFO | "huge_shared" : " 27 GB", | 20260506 01:56:46.821 | INFO | "malloc" : " 14 GB", | 20260506 01:56:46.821 | INFO | "pinned" : " 29 GB" | 20260506 01:56:46.821 | INFO | } | 20260506 01:56:46.821 | INFO | Perf report for the last 5.13586 seconds: | 20260506 01:56:46.821 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:56:46.821 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:46.821 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:46.821 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:46.821 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 01:56:46.821 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:46.821 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:46.821 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 01:56:46.821 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:46.821 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:46.822 | INFO | Uploaded 36GiB to GPU in 1.93898s (18.5664GiB/s) | 20260506 01:56:46.822 | INFO | Downloaded 34GiB to GPU in 1.9788s (17.1821GiB/s) | 20260506 01:56:51.962 | INFO | Analyzing chunk frames = [8041472,8041984) | 20260506 01:56:51.962 | INFO | MemInfo: { | 20260506 01:56:51.962 | INFO | "gpu" : " 17 GB", | 20260506 01:56:51.962 | INFO | "huge" : "", | 20260506 01:56:51.962 | INFO | "huge_shared" : " 27 GB", | 20260506 01:56:51.962 | INFO | "malloc" : " 14 GB", | 20260506 01:56:51.962 | INFO | "pinned" : " 29 GB" | 20260506 01:56:51.962 | INFO | } | 20260506 01:56:51.962 | INFO | Perf report for the last 5.14075 seconds: | 20260506 01:56:51.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:56:51.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:51.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:51.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:51.962 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 01:56:51.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:51.962 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:51.962 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 01:56:51.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:51.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:51.962 | INFO | Uploaded 36GiB to GPU in 2.08909s (17.2324GiB/s) | 20260506 01:56:51.962 | INFO | Downloaded 34GiB to GPU in 2.02755s (16.769GiB/s) | 20260506 01:56:57.096 | INFO | Analyzing chunk frames = [8041984,8042496) | 20260506 01:56:57.096 | INFO | MemInfo: { | 20260506 01:56:57.096 | INFO | "gpu" : " 17 GB", | 20260506 01:56:57.096 | INFO | "huge" : "", | 20260506 01:56:57.096 | INFO | "huge_shared" : " 27 GB", | 20260506 01:56:57.096 | INFO | "malloc" : " 14 GB", | 20260506 01:56:57.096 | INFO | "pinned" : " 29 GB" | 20260506 01:56:57.096 | INFO | } | 20260506 01:56:57.096 | INFO | Perf report for the last 5.13446 seconds: | 20260506 01:56:57.096 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:56:57.096 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:57.096 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:57.096 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:57.096 | INFO | ANALYSIS : 61.5%, 3.0, 95.5ms, 0 | 20260506 01:56:57.096 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:57.096 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:57.096 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:56:57.096 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:57.096 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:56:57.097 | INFO | Uploaded 36GiB to GPU in 2.15241s (16.7254GiB/s) | 20260506 01:56:57.097 | INFO | Downloaded 34GiB to GPU in 2.0596s (16.5081GiB/s) | 20260506 01:57:02.238 | INFO | Analyzing chunk frames = [8042496,8043008) | 20260506 01:57:02.238 | INFO | MemInfo: { | 20260506 01:57:02.238 | INFO | "gpu" : " 17 GB", | 20260506 01:57:02.238 | INFO | "huge" : "", | 20260506 01:57:02.238 | INFO | "huge_shared" : " 27 GB", | 20260506 01:57:02.238 | INFO | "malloc" : " 14 GB", | 20260506 01:57:02.238 | INFO | "pinned" : " 29 GB" | 20260506 01:57:02.238 | INFO | } | 20260506 01:57:02.238 | INFO | Perf report for the last 5.14151 seconds: | 20260506 01:57:02.238 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:57:02.238 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:02.238 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:02.238 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:02.238 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 01:57:02.238 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:02.238 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:02.238 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 01:57:02.238 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:02.238 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:02.238 | INFO | Uploaded 36GiB to GPU in 2.10749s (17.082GiB/s) | 20260506 01:57:02.238 | INFO | Downloaded 34GiB to GPU in 2.04616s (16.6165GiB/s) | 20260506 01:57:07.384 | INFO | Analyzing chunk frames = [8043008,8043520) | 20260506 01:57:07.384 | INFO | MemInfo: { | 20260506 01:57:07.384 | INFO | "gpu" : " 17 GB", | 20260506 01:57:07.384 | INFO | "huge" : "", | 20260506 01:57:07.384 | INFO | "huge_shared" : " 27 GB", | 20260506 01:57:07.384 | INFO | "malloc" : " 14 GB", | 20260506 01:57:07.384 | INFO | "pinned" : " 29 GB" | 20260506 01:57:07.384 | INFO | } | 20260506 01:57:07.384 | INFO | Perf report for the last 5.14633 seconds: | 20260506 01:57:07.384 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:57:07.384 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:07.384 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:07.384 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:07.384 | INFO | ANALYSIS : 60.9%, 3.0, 95.2ms, 0 | 20260506 01:57:07.384 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:07.384 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:07.384 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 01:57:07.384 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:07.384 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:07.385 | INFO | Uploaded 36GiB to GPU in 2.089s (17.2331GiB/s) | 20260506 01:57:07.385 | INFO | Downloaded 34GiB to GPU in 2.03664s (16.6942GiB/s) | 20260506 01:57:12.519 | INFO | Analyzing chunk frames = [8043520,8044032) | 20260506 01:57:12.519 | INFO | MemInfo: { | 20260506 01:57:12.519 | INFO | "gpu" : " 17 GB", | 20260506 01:57:12.519 | INFO | "huge" : "", | 20260506 01:57:12.519 | INFO | "huge_shared" : " 27 GB", | 20260506 01:57:12.519 | INFO | "malloc" : " 14 GB", | 20260506 01:57:12.519 | INFO | "pinned" : " 29 GB" | 20260506 01:57:12.519 | INFO | } | 20260506 01:57:12.519 | INFO | Perf report for the last 5.13517 seconds: | 20260506 01:57:12.519 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:57:12.519 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:12.519 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:12.519 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:12.519 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 01:57:12.519 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:12.519 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:12.519 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 01:57:12.519 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:12.519 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:12.520 | INFO | Uploaded 36GiB to GPU in 2.0729s (17.367GiB/s) | 20260506 01:57:12.520 | INFO | Downloaded 34GiB to GPU in 2.07914s (16.3529GiB/s) | 20260506 01:57:17.655 | INFO | Analyzing chunk frames = [8044032,8044544) | 20260506 01:57:17.655 | INFO | MemInfo: { | 20260506 01:57:17.655 | INFO | "gpu" : " 17 GB", | 20260506 01:57:17.655 | INFO | "huge" : "", | 20260506 01:57:17.655 | INFO | "huge_shared" : " 27 GB", | 20260506 01:57:17.655 | INFO | "malloc" : " 14 GB", | 20260506 01:57:17.655 | INFO | "pinned" : " 29 GB" | 20260506 01:57:17.655 | INFO | } | 20260506 01:57:17.655 | INFO | Perf report for the last 5.13613 seconds: | 20260506 01:57:17.655 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:57:17.655 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:17.655 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:17.656 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:17.656 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 01:57:17.656 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:17.656 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:17.656 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 01:57:17.656 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:17.656 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:17.656 | INFO | Uploaded 36GiB to GPU in 2.12896s (16.9097GiB/s) | 20260506 01:57:17.656 | INFO | Downloaded 34GiB to GPU in 2.03007s (16.7482GiB/s) | 20260506 01:57:17.656 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 01:57:17.656 | INFO | Total mem usage: 15.3 GiB | 20260506 01:57:17.656 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 01:57:17.656 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.19%) | 20260506 01:57:17.656 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 01:57:17.656 | INFO | High Water Breakdown: | 20260506 01:57:17.656 | INFO | 60.9% : 8.54 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 01:57:17.656 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 01:57:17.656 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 01:57:17.656 | INFO | ------------------------------------------ | 20260506 01:57:17.656 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:57:17.656 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 01:57:17.656 | INFO | Total mem usage: 26.5 GiB | 20260506 01:57:17.656 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 01:57:17.656 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 01:57:17.656 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260506 01:57:17.656 | INFO | High Water Breakdown: | 20260506 01:57:17.656 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:57:17.656 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:57:17.656 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:57:17.656 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:57:17.656 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:57:17.656 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:57:17.656 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 01:57:17.656 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 01:57:17.656 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:57:17.656 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 01:57:17.656 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 01:57:17.656 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 01:57:17.656 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 01:57:17.656 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 01:57:17.656 | INFO | ------------------------------------------ | 20260506 01:57:17.657 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:57:17.657 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 01:57:17.657 | INFO | Total mem usage: 25.6 GiB | 20260506 01:57:17.657 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 01:57:17.657 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 01:57:17.657 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 01:57:17.657 | INFO | High Water Breakdown: | 20260506 01:57:17.657 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 01:57:17.657 | INFO | ------------------------------------------ | 20260506 01:57:17.657 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:57:17.657 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 01:57:17.657 | INFO | Total mem usage: 15.7 GiB | 20260506 01:57:17.657 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 01:57:17.657 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 01:57:17.657 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 01:57:17.657 | INFO | High Water Breakdown: | 20260506 01:57:17.657 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 01:57:17.657 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 01:57:17.657 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 01:57:17.657 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 01:57:17.657 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 01:57:17.657 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 01:57:17.657 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 01:57:17.657 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 01:57:17.657 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 01:57:17.657 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:57:17.657 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 01:57:17.657 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:57:17.657 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 01:57:17.657 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 01:57:17.657 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 01:57:17.657 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 01:57:17.657 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:57:17.657 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 01:57:17.657 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:57:17.657 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:57:17.657 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:57:17.657 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 01:57:17.657 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 01:57:17.657 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:57:17.657 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 01:57:17.657 | INFO | ------------------------------------------ | 20260506 01:57:17.657 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:57:17.657 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 01:57:17.657 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:57:17.657 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:57:17.657 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.37e+01, 1.34e+01 / 7.42e-01 | 20260506 01:57:17.657 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.05e-01, 1.04e-02 / 4.65e-03 | 20260506 01:57:17.657 | INFO | PerfReport: 11.83%, Baseline -- 3.70e+00 / 4.50e+00, 5.69e-01 / 2.42e-01 | 20260506 01:57:17.657 | INFO | PerfReport: 3.29%, Histogram -- 1.03e+00 / 1.51e+00, 1.91e-01 / 9.99e-02 | 20260506 01:57:17.657 | INFO | PerfReport: 15.95%, DME -- 3.99e+01 / 4.38e+01, 3.78e+00 / 8.36e-01 | 20260506 01:57:17.657 | INFO | PerfReport: 57.54%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.56e+00 / 1.21e+00 | 20260506 01:57:17.657 | INFO | PerfReport: 7.02%, PulseAccumulating -- 2.20e+00 / 2.70e+00, 4.13e-01 / 2.59e-01 | 20260506 01:57:17.657 | INFO | PerfReport: 3.02%, Metrics -- 9.44e-01 / 1.22e+00, 2.52e-01 / 1.14e-01 | 20260506 01:57:17.657 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.27e-01 / 3.75e-01, 5.43e-02 / 2.30e-02 | 20260506 01:57:17.657 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 01:57:17.657 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:57:17.657 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:57:17.657 | INFO | PerfReport: Full Execution -- 9.78e+01 / -nan, 2.42e+01 / 0.00e+00 | 20260506 01:57:17.657 | INFO | PerfReport: 22.26%, Upload -- 2.18e+01 / 0.00e+00, 7.31e+00 / 0.00e+00 | 20260506 01:57:17.657 | INFO | PerfReport: 21.96%, Download -- 2.15e+01 / 0.00e+00, 7.61e+00 / 0.00e+00 | 20260506 01:57:17.657 | INFO | PerfReport: 31.97%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 01:57:17.657 | INFO | PerfReport: 2.03%, LockUpload -- 1.99e+00 / 0.00e+00, 7.82e+00 / 0.00e+00 | 20260506 01:57:17.657 | INFO | PerfReport: 1.34%, LockDownload -- 1.31e+00 / 0.00e+00, 2.95e+00 / 0.00e+00 | 20260506 01:57:17.657 | INFO | PerfReport: 20.42%, LockCompute -- 2.00e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 01:57:17.657 | INFO | PerfReport: 0.01%, Misc -- 1.46e-02 / 0.00e+00, 2.85e-02 / 0.00e+00 | 20260506 01:57:17.773 | INFO | Starting a round of Baz IO | 20260506 01:57:20.917 | INFO | Populating RTMetrics report | 20260506 01:57:20.917 | INFO | RTMetrics report populated | 20260506 01:57:20.917 | INFO | Opening tmp RTMetrics json file | 20260506 01:57:20.918 | INFO | Serializing RTMetrics to json | 20260506 01:57:20.919 | INFO | Serialization done, swapping tmp file now... | 20260506 01:57:20.920 | INFO | RTMetrics done | 20260506 01:57:20.992 | INFO | Finished a round of Baz IO: Wrote 1.807 GiB in 3.218 seconds | 20260506 01:57:20.992 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.29% / 0.00% / 78.71% | 20260506 01:57:20.992 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0752807% / 0% / 0.000111262% | 20260506 01:57:20.992 | INFO | Baz IO overall: | 20260506 01:57:20.992 | INFO | BytesWritten : 4522890975277 | 20260506 01:57:20.992 | INFO | NumEvents : 2124653638536 | 20260506 01:57:22.800 | INFO | Analyzing chunk frames = [8044544,8045056) | 20260506 01:57:22.800 | INFO | MemInfo: { | 20260506 01:57:22.800 | INFO | "gpu" : " 17 GB", | 20260506 01:57:22.800 | INFO | "huge" : "", | 20260506 01:57:22.800 | INFO | "huge_shared" : " 27 GB", | 20260506 01:57:22.800 | INFO | "malloc" : " 14 GB", | 20260506 01:57:22.800 | INFO | "pinned" : " 29 GB" | 20260506 01:57:22.800 | INFO | } | 20260506 01:57:22.800 | INFO | Perf report for the last 5.14474 seconds: | 20260506 01:57:22.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:57:22.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:22.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:22.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:22.800 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260506 01:57:22.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:22.800 | INFO | RT_METRICS : 42.8%, 1.0, 22.4ms, 0 | 20260506 01:57:22.800 | INFO | PRE_HQ : 59.5%, 1.7, 53.4ms, 0 | 20260506 01:57:22.800 | INFO | BAZWRITER : 61.7%, 3.5, 110.2ms, 0 | 20260506 01:57:22.801 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:57:22.801 | INFO | Uploaded 36GiB to GPU in 2.17746s (16.533GiB/s) | 20260506 01:57:22.801 | INFO | Downloaded 38GiB to GPU in 2.24412s (16.9332GiB/s) | 20260506 01:57:27.940 | INFO | Analyzing chunk frames = [8045056,8045568) | 20260506 01:57:27.940 | INFO | MemInfo: { | 20260506 01:57:27.940 | INFO | "gpu" : " 17 GB", | 20260506 01:57:27.940 | INFO | "huge" : "", | 20260506 01:57:27.940 | INFO | "huge_shared" : " 27 GB", | 20260506 01:57:27.940 | INFO | "malloc" : " 14 GB", | 20260506 01:57:27.940 | INFO | "pinned" : " 29 GB" | 20260506 01:57:27.940 | INFO | } | 20260506 01:57:27.940 | INFO | Perf report for the last 5.13943 seconds: | 20260506 01:57:27.940 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:57:27.940 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:27.940 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:27.940 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:27.940 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 01:57:27.940 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:27.940 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:27.940 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 01:57:27.940 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:27.940 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:27.940 | INFO | Uploaded 36GiB to GPU in 1.95729s (18.3928GiB/s) | 20260506 01:57:27.940 | INFO | Downloaded 34GiB to GPU in 1.97549s (17.2109GiB/s) | 20260506 01:57:33.075 | INFO | Analyzing chunk frames = [8045568,8046080) | 20260506 01:57:33.075 | INFO | MemInfo: { | 20260506 01:57:33.075 | INFO | "gpu" : " 17 GB", | 20260506 01:57:33.075 | INFO | "huge" : "", | 20260506 01:57:33.075 | INFO | "huge_shared" : " 27 GB", | 20260506 01:57:33.075 | INFO | "malloc" : " 14 GB", | 20260506 01:57:33.075 | INFO | "pinned" : " 29 GB" | 20260506 01:57:33.075 | INFO | } | 20260506 01:57:33.075 | INFO | Perf report for the last 5.13576 seconds: | 20260506 01:57:33.075 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:57:33.075 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:33.075 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:33.076 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:33.076 | INFO | ANALYSIS : 61.2%, 3.0, 95.1ms, 0 | 20260506 01:57:33.076 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:33.076 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:33.076 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 01:57:33.076 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:33.076 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:33.076 | INFO | Uploaded 36GiB to GPU in 2.09873s (17.1533GiB/s) | 20260506 01:57:33.076 | INFO | Downloaded 34GiB to GPU in 2.04276s (16.6442GiB/s) | 20260506 01:57:38.214 | INFO | Analyzing chunk frames = [8046080,8046592) | 20260506 01:57:38.214 | INFO | MemInfo: { | 20260506 01:57:38.214 | INFO | "gpu" : " 17 GB", | 20260506 01:57:38.214 | INFO | "huge" : "", | 20260506 01:57:38.214 | INFO | "huge_shared" : " 27 GB", | 20260506 01:57:38.214 | INFO | "malloc" : " 14 GB", | 20260506 01:57:38.214 | INFO | "pinned" : " 29 GB" | 20260506 01:57:38.214 | INFO | } | 20260506 01:57:38.214 | INFO | Perf report for the last 5.13899 seconds: | 20260506 01:57:38.214 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:57:38.214 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:38.214 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:38.214 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:38.214 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260506 01:57:38.214 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:38.214 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:38.214 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:57:38.214 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:38.214 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:38.215 | INFO | Uploaded 36GiB to GPU in 2.13541s (16.8586GiB/s) | 20260506 01:57:38.215 | INFO | Downloaded 34GiB to GPU in 2.02475s (16.7922GiB/s) | 20260506 01:57:43.353 | INFO | Analyzing chunk frames = [8046592,8047104) | 20260506 01:57:43.353 | INFO | MemInfo: { | 20260506 01:57:43.353 | INFO | "gpu" : " 17 GB", | 20260506 01:57:43.353 | INFO | "huge" : "", | 20260506 01:57:43.353 | INFO | "huge_shared" : " 27 GB", | 20260506 01:57:43.353 | INFO | "malloc" : " 14 GB", | 20260506 01:57:43.353 | INFO | "pinned" : " 29 GB" | 20260506 01:57:43.353 | INFO | } | 20260506 01:57:43.354 | INFO | Perf report for the last 5.13906 seconds: | 20260506 01:57:43.354 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:57:43.354 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:43.354 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:43.354 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:43.354 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 01:57:43.354 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:43.354 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:43.354 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:57:43.354 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:43.354 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:43.354 | INFO | Uploaded 36GiB to GPU in 2.09506s (17.1833GiB/s) | 20260506 01:57:43.354 | INFO | Downloaded 34GiB to GPU in 2.04554s (16.6215GiB/s) | 20260506 01:57:48.498 | INFO | Analyzing chunk frames = [8047104,8047616) | 20260506 01:57:48.498 | INFO | MemInfo: { | 20260506 01:57:48.498 | INFO | "gpu" : " 17 GB", | 20260506 01:57:48.498 | INFO | "huge" : "", | 20260506 01:57:48.498 | INFO | "huge_shared" : " 27 GB", | 20260506 01:57:48.498 | INFO | "malloc" : " 14 GB", | 20260506 01:57:48.498 | INFO | "pinned" : " 29 GB" | 20260506 01:57:48.499 | INFO | } | 20260506 01:57:48.499 | INFO | Perf report for the last 5.14514 seconds: | 20260506 01:57:48.499 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:57:48.499 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:48.499 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:48.499 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:48.499 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 01:57:48.499 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:48.499 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:48.499 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 01:57:48.499 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:48.499 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:48.499 | INFO | Uploaded 36GiB to GPU in 2.08304s (17.2824GiB/s) | 20260506 01:57:48.499 | INFO | Downloaded 34GiB to GPU in 2.03159s (16.7357GiB/s) | 20260506 01:57:53.635 | INFO | Analyzing chunk frames = [8047616,8048128) | 20260506 01:57:53.635 | INFO | MemInfo: { | 20260506 01:57:53.635 | INFO | "gpu" : " 17 GB", | 20260506 01:57:53.635 | INFO | "huge" : "", | 20260506 01:57:53.635 | INFO | "huge_shared" : " 27 GB", | 20260506 01:57:53.635 | INFO | "malloc" : " 14 GB", | 20260506 01:57:53.635 | INFO | "pinned" : " 29 GB" | 20260506 01:57:53.635 | INFO | } | 20260506 01:57:53.635 | INFO | Perf report for the last 5.13658 seconds: | 20260506 01:57:53.635 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:57:53.635 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:53.635 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:53.635 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:53.635 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260506 01:57:53.635 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:53.635 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:53.635 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 01:57:53.635 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:53.635 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:53.636 | INFO | Uploaded 36GiB to GPU in 2.13082s (16.8949GiB/s) | 20260506 01:57:53.636 | INFO | Downloaded 34GiB to GPU in 2.10114s (16.1817GiB/s) | 20260506 01:57:58.771 | INFO | Analyzing chunk frames = [8048128,8048640) | 20260506 01:57:58.771 | INFO | MemInfo: { | 20260506 01:57:58.771 | INFO | "gpu" : " 17 GB", | 20260506 01:57:58.771 | INFO | "huge" : "", | 20260506 01:57:58.771 | INFO | "huge_shared" : " 27 GB", | 20260506 01:57:58.771 | INFO | "malloc" : " 14 GB", | 20260506 01:57:58.771 | INFO | "pinned" : " 29 GB" | 20260506 01:57:58.771 | INFO | } | 20260506 01:57:58.771 | INFO | Perf report for the last 5.13603 seconds: | 20260506 01:57:58.771 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:57:58.771 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:58.771 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:58.771 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:58.771 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 01:57:58.771 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:58.771 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:58.771 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 01:57:58.771 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:58.771 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:57:58.772 | INFO | Uploaded 36GiB to GPU in 2.04031s (17.6444GiB/s) | 20260506 01:57:58.772 | INFO | Downloaded 34GiB to GPU in 2.0123s (16.8961GiB/s) | 20260506 01:57:58.890 | INFO | Starting a round of Baz IO | 20260506 01:58:02.032 | INFO | Populating RTMetrics report | 20260506 01:58:02.032 | INFO | RTMetrics report populated | 20260506 01:58:02.032 | INFO | Opening tmp RTMetrics json file | 20260506 01:58:02.033 | INFO | Serializing RTMetrics to json | 20260506 01:58:02.036 | INFO | Serialization done, swapping tmp file now... | 20260506 01:58:02.038 | INFO | RTMetrics done | 20260506 01:58:02.111 | INFO | Finished a round of Baz IO: Wrote 1.807 GiB in 3.221 seconds | 20260506 01:58:02.111 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.29% / 0.00% / 78.71% | 20260506 01:58:02.111 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0751615% / 0% / 8.34465e-05% | 20260506 01:58:02.111 | INFO | Baz IO overall: | 20260506 01:58:02.111 | INFO | BytesWritten : 4524831423510 | 20260506 01:58:02.111 | INFO | NumEvents : 2125536530750 | 20260506 01:58:03.917 | INFO | Analyzing chunk frames = [8048640,8049152) | 20260506 01:58:03.917 | INFO | MemInfo: { | 20260506 01:58:03.917 | INFO | "gpu" : " 17 GB", | 20260506 01:58:03.918 | INFO | "huge" : "", | 20260506 01:58:03.918 | INFO | "huge_shared" : " 27 GB", | 20260506 01:58:03.918 | INFO | "malloc" : " 14 GB", | 20260506 01:58:03.918 | INFO | "pinned" : " 29 GB" | 20260506 01:58:03.918 | INFO | } | 20260506 01:58:03.918 | INFO | Perf report for the last 5.14641 seconds: | 20260506 01:58:03.918 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:58:03.918 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:03.918 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:03.918 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:03.918 | INFO | ANALYSIS : 62.3%, 3.0, 97.4ms, 0 | 20260506 01:58:03.918 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:03.918 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260506 01:58:03.918 | INFO | PRE_HQ : 59.5%, 1.7, 53.7ms, 0 | 20260506 01:58:03.918 | INFO | BAZWRITER : 61.7%, 3.5, 112.3ms, 0 | 20260506 01:58:03.918 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 01:58:03.918 | INFO | Uploaded 36GiB to GPU in 2.17071s (16.5845GiB/s) | 20260506 01:58:03.918 | INFO | Downloaded 38GiB to GPU in 2.2401s (16.9635GiB/s) | 20260506 01:58:09.052 | INFO | Analyzing chunk frames = [8049152,8049664) | 20260506 01:58:09.052 | INFO | MemInfo: { | 20260506 01:58:09.052 | INFO | "gpu" : " 17 GB", | 20260506 01:58:09.052 | INFO | "huge" : "", | 20260506 01:58:09.052 | INFO | "huge_shared" : " 27 GB", | 20260506 01:58:09.052 | INFO | "malloc" : " 14 GB", | 20260506 01:58:09.052 | INFO | "pinned" : " 29 GB" | 20260506 01:58:09.052 | INFO | } | 20260506 01:58:09.052 | INFO | Perf report for the last 5.13423 seconds: | 20260506 01:58:09.052 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:58:09.052 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:09.052 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:09.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:09.052 | INFO | ANALYSIS : 60.9%, 3.0, 95.0ms, 0 | 20260506 01:58:09.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:09.052 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:09.052 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 01:58:09.052 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:09.052 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:09.053 | INFO | Uploaded 36GiB to GPU in 1.9785s (18.1956GiB/s) | 20260506 01:58:09.053 | INFO | Downloaded 34GiB to GPU in 1.9881s (17.1017GiB/s) | 20260506 01:58:14.193 | INFO | Analyzing chunk frames = [8049664,8050176) | 20260506 01:58:14.193 | INFO | MemInfo: { | 20260506 01:58:14.193 | INFO | "gpu" : " 17 GB", | 20260506 01:58:14.193 | INFO | "huge" : "", | 20260506 01:58:14.193 | INFO | "huge_shared" : " 27 GB", | 20260506 01:58:14.193 | INFO | "malloc" : " 14 GB", | 20260506 01:58:14.193 | INFO | "pinned" : " 29 GB" | 20260506 01:58:14.193 | INFO | } | 20260506 01:58:14.193 | INFO | Perf report for the last 5.14081 seconds: | 20260506 01:58:14.193 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:58:14.193 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:14.193 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:14.193 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:14.193 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 01:58:14.193 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:14.193 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:14.193 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 01:58:14.193 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:14.193 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:14.193 | INFO | Uploaded 36GiB to GPU in 2.08691s (17.2504GiB/s) | 20260506 01:58:14.193 | INFO | Downloaded 34GiB to GPU in 2.035s (16.7076GiB/s) | 20260506 01:58:19.331 | INFO | Analyzing chunk frames = [8050176,8050688) | 20260506 01:58:19.331 | INFO | MemInfo: { | 20260506 01:58:19.331 | INFO | "gpu" : " 17 GB", | 20260506 01:58:19.331 | INFO | "huge" : "", | 20260506 01:58:19.331 | INFO | "huge_shared" : " 27 GB", | 20260506 01:58:19.331 | INFO | "malloc" : " 14 GB", | 20260506 01:58:19.331 | INFO | "pinned" : " 29 GB" | 20260506 01:58:19.331 | INFO | } | 20260506 01:58:19.332 | INFO | Perf report for the last 5.13871 seconds: | 20260506 01:58:19.332 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:58:19.332 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:19.332 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:19.332 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:19.332 | INFO | ANALYSIS : 61.1%, 3.0, 94.9ms, 0 | 20260506 01:58:19.332 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:19.332 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:19.332 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:58:19.332 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:19.332 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:19.332 | INFO | Uploaded 36GiB to GPU in 2.09115s (17.2154GiB/s) | 20260506 01:58:19.332 | INFO | Downloaded 34GiB to GPU in 2.01513s (16.8724GiB/s) | 20260506 01:58:24.467 | INFO | Analyzing chunk frames = [8050688,8051200) | 20260506 01:58:24.467 | INFO | MemInfo: { | 20260506 01:58:24.467 | INFO | "gpu" : " 17 GB", | 20260506 01:58:24.467 | INFO | "huge" : "", | 20260506 01:58:24.467 | INFO | "huge_shared" : " 27 GB", | 20260506 01:58:24.467 | INFO | "malloc" : " 14 GB", | 20260506 01:58:24.467 | INFO | "pinned" : " 29 GB" | 20260506 01:58:24.467 | INFO | } | 20260506 01:58:24.467 | INFO | Perf report for the last 5.13596 seconds: | 20260506 01:58:24.467 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:58:24.467 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:24.467 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:24.467 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:24.467 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260506 01:58:24.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:24.467 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:24.468 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 01:58:24.468 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:24.468 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:24.468 | INFO | Uploaded 36GiB to GPU in 2.09362s (17.1951GiB/s) | 20260506 01:58:24.468 | INFO | Downloaded 34GiB to GPU in 2.01827s (16.8461GiB/s) | 20260506 01:58:29.605 | INFO | Analyzing chunk frames = [8051200,8051712) | 20260506 01:58:29.605 | INFO | MemInfo: { | 20260506 01:58:29.605 | INFO | "gpu" : " 17 GB", | 20260506 01:58:29.605 | INFO | "huge" : "", | 20260506 01:58:29.605 | INFO | "huge_shared" : " 27 GB", | 20260506 01:58:29.605 | INFO | "malloc" : " 14 GB", | 20260506 01:58:29.605 | INFO | "pinned" : " 29 GB" | 20260506 01:58:29.605 | INFO | } | 20260506 01:58:29.605 | INFO | Perf report for the last 5.13801 seconds: | 20260506 01:58:29.605 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:58:29.605 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:29.605 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:29.606 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:29.606 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 01:58:29.606 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:29.606 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:29.606 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 01:58:29.606 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:29.606 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:29.606 | INFO | Uploaded 36GiB to GPU in 2.0686s (17.4031GiB/s) | 20260506 01:58:29.606 | INFO | Downloaded 34GiB to GPU in 2.03955s (16.6703GiB/s) | 20260506 01:58:34.746 | INFO | Analyzing chunk frames = [8051712,8052224) | 20260506 01:58:34.746 | INFO | MemInfo: { | 20260506 01:58:34.746 | INFO | "gpu" : " 17 GB", | 20260506 01:58:34.747 | INFO | "huge" : "", | 20260506 01:58:34.747 | INFO | "huge_shared" : " 27 GB", | 20260506 01:58:34.747 | INFO | "malloc" : " 14 GB", | 20260506 01:58:34.747 | INFO | "pinned" : " 29 GB" | 20260506 01:58:34.747 | INFO | } | 20260506 01:58:34.747 | INFO | Perf report for the last 5.14105 seconds: | 20260506 01:58:34.747 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:58:34.747 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:34.747 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:34.747 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:34.747 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260506 01:58:34.747 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:34.747 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:34.747 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 01:58:34.747 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:34.747 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:34.747 | INFO | Uploaded 36GiB to GPU in 2.03032s (17.7312GiB/s) | 20260506 01:58:34.747 | INFO | Downloaded 34GiB to GPU in 2.05212s (16.5682GiB/s) | 20260506 01:58:39.890 | INFO | Analyzing chunk frames = [8052224,8052736) | 20260506 01:58:39.890 | INFO | MemInfo: { | 20260506 01:58:39.890 | INFO | "gpu" : " 17 GB", | 20260506 01:58:39.890 | INFO | "huge" : "", | 20260506 01:58:39.890 | INFO | "huge_shared" : " 27 GB", | 20260506 01:58:39.890 | INFO | "malloc" : " 14 GB", | 20260506 01:58:39.890 | INFO | "pinned" : " 29 GB" | 20260506 01:58:39.890 | INFO | } | 20260506 01:58:39.890 | INFO | Perf report for the last 5.14316 seconds: | 20260506 01:58:39.890 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:58:39.890 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:39.890 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:39.890 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:39.890 | INFO | ANALYSIS : 61.5%, 3.0, 96.1ms, 0 | 20260506 01:58:39.890 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:39.890 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:39.890 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:58:39.890 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:39.890 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:39.890 | INFO | Uploaded 36GiB to GPU in 2.07264s (17.3692GiB/s) | 20260506 01:58:39.890 | INFO | Downloaded 34GiB to GPU in 2.02914s (16.7559GiB/s) | 20260506 01:58:39.890 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 01:58:39.890 | INFO | Total mem usage: 15.3 GiB | 20260506 01:58:39.890 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 01:58:39.890 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.21%) | 20260506 01:58:39.890 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 01:58:39.890 | INFO | High Water Breakdown: | 20260506 01:58:39.890 | INFO | 60.9% : 8.54 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 01:58:39.890 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 01:58:39.890 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 01:58:39.890 | INFO | ------------------------------------------ | 20260506 01:58:39.890 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:58:39.891 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 01:58:39.891 | INFO | Total mem usage: 26.5 GiB | 20260506 01:58:39.891 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 01:58:39.891 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 01:58:39.891 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260506 01:58:39.891 | INFO | High Water Breakdown: | 20260506 01:58:39.891 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:58:39.891 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:58:39.891 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:58:39.891 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:58:39.891 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:58:39.891 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:58:39.891 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 01:58:39.891 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 01:58:39.891 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:58:39.891 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 01:58:39.891 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 01:58:39.891 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 01:58:39.891 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 01:58:39.891 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 01:58:39.891 | INFO | ------------------------------------------ | 20260506 01:58:39.891 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:58:39.891 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 01:58:39.891 | INFO | Total mem usage: 25.6 GiB | 20260506 01:58:39.891 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 01:58:39.891 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 01:58:39.891 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 01:58:39.891 | INFO | High Water Breakdown: | 20260506 01:58:39.891 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 01:58:39.891 | INFO | ------------------------------------------ | 20260506 01:58:39.891 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:58:39.891 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 01:58:39.891 | INFO | Total mem usage: 15.7 GiB | 20260506 01:58:39.891 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 01:58:39.891 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 01:58:39.891 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 01:58:39.891 | INFO | High Water Breakdown: | 20260506 01:58:39.891 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 01:58:39.891 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 01:58:39.891 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 01:58:39.891 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 01:58:39.891 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 01:58:39.891 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 01:58:39.891 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 01:58:39.891 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 01:58:39.891 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 01:58:39.891 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 01:58:39.891 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 01:58:39.891 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 01:58:39.891 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 01:58:39.891 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 01:58:39.892 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 01:58:39.892 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 01:58:39.892 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 01:58:39.892 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 01:58:39.892 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 01:58:39.892 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 01:58:39.892 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 01:58:39.892 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 01:58:39.892 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 01:58:39.892 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 01:58:39.892 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 01:58:39.892 | INFO | ------------------------------------------ | 20260506 01:58:39.892 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 01:58:39.892 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 01:58:39.892 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:58:39.892 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:58:39.892 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.39e+01, 1.34e+01 / 9.19e-01 | 20260506 01:58:39.892 | INFO | PerfReport: 0.29%, AggregateStats -- 9.17e-02 / 1.04e-01, 1.02e-02 / 3.28e-03 | 20260506 01:58:39.892 | INFO | PerfReport: 11.86%, Baseline -- 3.71e+00 / 4.50e+00, 5.80e-01 / 2.44e-01 | 20260506 01:58:39.892 | INFO | PerfReport: 3.29%, Histogram -- 1.03e+00 / 1.52e+00, 1.94e-01 / 1.03e-01 | 20260506 01:58:39.892 | INFO | PerfReport: 15.99%, DME -- 4.00e+01 / 4.41e+01, 3.99e+00 / 1.12e+00 | 20260506 01:58:39.892 | INFO | PerfReport: 57.44%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.50e+00 / 1.20e+00 | 20260506 01:58:39.892 | INFO | PerfReport: 7.04%, PulseAccumulating -- 2.20e+00 / 2.72e+00, 4.16e-01 / 2.49e-01 | 20260506 01:58:39.892 | INFO | PerfReport: 3.03%, Metrics -- 9.47e-01 / 1.22e+00, 2.53e-01 / 1.23e-01 | 20260506 01:58:39.892 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.27e-01 / 3.73e-01, 5.41e-02 / 2.47e-02 | 20260506 01:58:39.892 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 01:58:39.892 | INFO | PerfReport: Gathered 1536 samples. | 20260506 01:58:39.892 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 01:58:39.892 | INFO | PerfReport: Full Execution -- 9.77e+01 / -nan, 2.42e+01 / 0.00e+00 | 20260506 01:58:39.892 | INFO | PerfReport: 22.19%, Upload -- 2.17e+01 / 0.00e+00, 7.26e+00 / 0.00e+00 | 20260506 01:58:39.892 | INFO | PerfReport: 21.91%, Download -- 2.14e+01 / 0.00e+00, 7.55e+00 / 0.00e+00 | 20260506 01:58:39.892 | INFO | PerfReport: 32.02%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 01:58:39.892 | INFO | PerfReport: 2.03%, LockUpload -- 1.99e+00 / 0.00e+00, 7.77e+00 / 0.00e+00 | 20260506 01:58:39.892 | INFO | PerfReport: 1.30%, LockDownload -- 1.27e+00 / 0.00e+00, 2.90e+00 / 0.00e+00 | 20260506 01:58:39.892 | INFO | PerfReport: 20.53%, LockCompute -- 2.01e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260506 01:58:39.892 | INFO | PerfReport: 0.02%, Misc -- 1.57e-02 / 0.00e+00, 3.50e-02 / 0.00e+00 | 20260506 01:58:40.015 | INFO | Starting a round of Baz IO | 20260506 01:58:43.152 | INFO | Populating RTMetrics report | 20260506 01:58:43.152 | INFO | RTMetrics report populated | 20260506 01:58:43.152 | INFO | Opening tmp RTMetrics json file | 20260506 01:58:43.153 | INFO | Serializing RTMetrics to json | 20260506 01:58:43.154 | INFO | Serialization done, swapping tmp file now... | 20260506 01:58:43.155 | INFO | RTMetrics done | 20260506 01:58:43.233 | INFO | Finished a round of Baz IO: Wrote 1.808 GiB in 3.217 seconds | 20260506 01:58:43.233 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.29% / 0.00% / 78.71% | 20260506 01:58:43.233 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0751177% / 0% / 6.75519e-05% | 20260506 01:58:43.233 | INFO | Baz IO overall: | 20260506 01:58:43.233 | INFO | BytesWritten : 4526772794650 | 20260506 01:58:43.233 | INFO | NumEvents : 2126420051016 | 20260506 01:58:45.026 | INFO | Analyzing chunk frames = [8052736,8053248) | 20260506 01:58:45.026 | INFO | MemInfo: { | 20260506 01:58:45.026 | INFO | "gpu" : " 17 GB", | 20260506 01:58:45.026 | INFO | "huge" : "", | 20260506 01:58:45.026 | INFO | "huge_shared" : " 27 GB", | 20260506 01:58:45.026 | INFO | "malloc" : " 14 GB", | 20260506 01:58:45.026 | INFO | "pinned" : " 29 GB" | 20260506 01:58:45.026 | INFO | } | 20260506 01:58:45.026 | INFO | Perf report for the last 5.13607 seconds: | 20260506 01:58:45.026 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:58:45.026 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:45.026 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:45.026 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:45.026 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260506 01:58:45.026 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:45.026 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260506 01:58:45.026 | INFO | PRE_HQ : 59.5%, 1.7, 53.6ms, 0 | 20260506 01:58:45.026 | INFO | BAZWRITER : 61.9%, 3.5, 111.7ms, 0 | 20260506 01:58:45.026 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:58:45.026 | INFO | Uploaded 36GiB to GPU in 2.18976s (16.4401GiB/s) | 20260506 01:58:45.027 | INFO | Downloaded 38GiB to GPU in 2.23881s (16.9733GiB/s) | 20260506 01:58:50.168 | INFO | Analyzing chunk frames = [8053248,8053760) | 20260506 01:58:50.168 | INFO | MemInfo: { | 20260506 01:58:50.168 | INFO | "gpu" : " 17 GB", | 20260506 01:58:50.168 | INFO | "huge" : "", | 20260506 01:58:50.168 | INFO | "huge_shared" : " 27 GB", | 20260506 01:58:50.168 | INFO | "malloc" : " 14 GB", | 20260506 01:58:50.168 | INFO | "pinned" : " 29 GB" | 20260506 01:58:50.168 | INFO | } | 20260506 01:58:50.168 | INFO | Perf report for the last 5.14216 seconds: | 20260506 01:58:50.168 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:58:50.168 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:50.168 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:50.168 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:50.168 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 01:58:50.168 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:50.168 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:50.168 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 01:58:50.168 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:50.168 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:50.168 | INFO | Uploaded 36GiB to GPU in 1.94803s (18.4802GiB/s) | 20260506 01:58:50.168 | INFO | Downloaded 34GiB to GPU in 1.99006s (17.0849GiB/s) | 20260506 01:58:55.311 | INFO | Analyzing chunk frames = [8053760,8054272) | 20260506 01:58:55.311 | INFO | MemInfo: { | 20260506 01:58:55.311 | INFO | "gpu" : " 17 GB", | 20260506 01:58:55.311 | INFO | "huge" : "", | 20260506 01:58:55.311 | INFO | "huge_shared" : " 27 GB", | 20260506 01:58:55.311 | INFO | "malloc" : " 14 GB", | 20260506 01:58:55.311 | INFO | "pinned" : " 29 GB" | 20260506 01:58:55.311 | INFO | } | 20260506 01:58:55.311 | INFO | Perf report for the last 5.14316 seconds: | 20260506 01:58:55.311 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:58:55.311 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:55.311 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:55.311 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:55.311 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 01:58:55.311 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:55.311 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:55.311 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 01:58:55.311 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:55.311 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:58:55.312 | INFO | Uploaded 36GiB to GPU in 2.09659s (17.1707GiB/s) | 20260506 01:58:55.312 | INFO | Downloaded 34GiB to GPU in 2.03171s (16.7347GiB/s) | 20260506 01:59:00.449 | INFO | Analyzing chunk frames = [8054272,8054784) | 20260506 01:59:00.449 | INFO | MemInfo: { | 20260506 01:59:00.449 | INFO | "gpu" : " 17 GB", | 20260506 01:59:00.449 | INFO | "huge" : "", | 20260506 01:59:00.449 | INFO | "huge_shared" : " 27 GB", | 20260506 01:59:00.449 | INFO | "malloc" : " 14 GB", | 20260506 01:59:00.450 | INFO | "pinned" : " 29 GB" | 20260506 01:59:00.450 | INFO | } | 20260506 01:59:00.450 | INFO | Perf report for the last 5.13839 seconds: | 20260506 01:59:00.450 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:59:00.450 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:00.450 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:00.450 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:00.450 | INFO | ANALYSIS : 61.0%, 3.0, 94.7ms, 0 | 20260506 01:59:00.450 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:00.450 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:00.450 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 01:59:00.450 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:00.450 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:00.450 | INFO | Uploaded 36GiB to GPU in 2.07842s (17.3209GiB/s) | 20260506 01:59:00.450 | INFO | Downloaded 34GiB to GPU in 2.00506s (16.9571GiB/s) | 20260506 01:59:05.578 | INFO | Analyzing chunk frames = [8054784,8055296) | 20260506 01:59:05.578 | INFO | MemInfo: { | 20260506 01:59:05.578 | INFO | "gpu" : " 17 GB", | 20260506 01:59:05.578 | INFO | "huge" : "", | 20260506 01:59:05.578 | INFO | "huge_shared" : " 27 GB", | 20260506 01:59:05.578 | INFO | "malloc" : " 14 GB", | 20260506 01:59:05.578 | INFO | "pinned" : " 29 GB" | 20260506 01:59:05.578 | INFO | } | 20260506 01:59:05.578 | INFO | Perf report for the last 5.12827 seconds: | 20260506 01:59:05.578 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:59:05.578 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:05.578 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:05.578 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:05.578 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 01:59:05.578 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:05.578 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:05.578 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:59:05.578 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:05.578 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:05.578 | INFO | Uploaded 36GiB to GPU in 2.0873s (17.2472GiB/s) | 20260506 01:59:05.578 | INFO | Downloaded 34GiB to GPU in 2.02643s (16.7783GiB/s) | 20260506 01:59:10.722 | INFO | Analyzing chunk frames = [8055296,8055808) | 20260506 01:59:10.722 | INFO | MemInfo: { | 20260506 01:59:10.722 | INFO | "gpu" : " 17 GB", | 20260506 01:59:10.722 | INFO | "huge" : "", | 20260506 01:59:10.722 | INFO | "huge_shared" : " 27 GB", | 20260506 01:59:10.722 | INFO | "malloc" : " 14 GB", | 20260506 01:59:10.722 | INFO | "pinned" : " 29 GB" | 20260506 01:59:10.722 | INFO | } | 20260506 01:59:10.722 | INFO | Perf report for the last 5.14451 seconds: | 20260506 01:59:10.722 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:59:10.722 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:10.722 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:10.722 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:10.722 | INFO | ANALYSIS : 61.0%, 3.0, 95.4ms, 0 | 20260506 01:59:10.722 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:10.722 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:10.722 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:59:10.722 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:10.722 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:10.723 | INFO | Uploaded 36GiB to GPU in 2.07276s (17.3681GiB/s) | 20260506 01:59:10.723 | INFO | Downloaded 34GiB to GPU in 2.0415s (16.6544GiB/s) | 20260506 01:59:15.870 | INFO | Analyzing chunk frames = [8055808,8056320) | 20260506 01:59:15.870 | INFO | MemInfo: { | 20260506 01:59:15.870 | INFO | "gpu" : " 17 GB", | 20260506 01:59:15.870 | INFO | "huge" : "", | 20260506 01:59:15.870 | INFO | "huge_shared" : " 27 GB", | 20260506 01:59:15.870 | INFO | "malloc" : " 14 GB", | 20260506 01:59:15.870 | INFO | "pinned" : " 29 GB" | 20260506 01:59:15.870 | INFO | } | 20260506 01:59:15.870 | INFO | Perf report for the last 5.14783 seconds: | 20260506 01:59:15.870 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:59:15.870 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:15.870 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:15.870 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:15.870 | INFO | ANALYSIS : 61.9%, 3.0, 96.8ms, 0 | 20260506 01:59:15.870 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:15.870 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:15.870 | INFO | PRE_HQ : 27.2%, 1.0, 14.2ms, 0 | 20260506 01:59:15.870 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:15.871 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:15.871 | INFO | Uploaded 36GiB to GPU in 2.0381s (17.6635GiB/s) | 20260506 01:59:15.871 | INFO | Downloaded 34GiB to GPU in 2.03889s (16.6758GiB/s) | 20260506 01:59:21.000 | INFO | Analyzing chunk frames = [8056320,8056832) | 20260506 01:59:21.000 | INFO | MemInfo: { | 20260506 01:59:21.000 | INFO | "gpu" : " 17 GB", | 20260506 01:59:21.000 | INFO | "huge" : "", | 20260506 01:59:21.000 | INFO | "huge_shared" : " 27 GB", | 20260506 01:59:21.000 | INFO | "malloc" : " 14 GB", | 20260506 01:59:21.000 | INFO | "pinned" : " 29 GB" | 20260506 01:59:21.000 | INFO | } | 20260506 01:59:21.001 | INFO | Perf report for the last 5.13031 seconds: | 20260506 01:59:21.001 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:59:21.001 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:21.001 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:21.001 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:21.001 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 01:59:21.001 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:21.001 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:21.001 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:59:21.001 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:21.001 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:21.001 | INFO | Uploaded 36GiB to GPU in 2.04448s (17.6084GiB/s) | 20260506 01:59:21.001 | INFO | Downloaded 34GiB to GPU in 2.01762s (16.8515GiB/s) | 20260506 01:59:21.127 | INFO | Starting a round of Baz IO | 20260506 01:59:24.265 | INFO | Populating RTMetrics report | 20260506 01:59:24.265 | INFO | RTMetrics report populated | 20260506 01:59:24.265 | INFO | Opening tmp RTMetrics json file | 20260506 01:59:24.266 | INFO | Serializing RTMetrics to json | 20260506 01:59:24.267 | INFO | Serialization done, swapping tmp file now... | 20260506 01:59:24.267 | INFO | RTMetrics done | 20260506 01:59:24.341 | INFO | Finished a round of Baz IO: Wrote 1.808 GiB in 3.214 seconds | 20260506 01:59:24.341 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.29% / 0.00% / 78.71% | 20260506 01:59:24.341 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0749667% / 3.97364e-06% / 7.15256e-05% | 20260506 01:59:24.341 | INFO | Baz IO overall: | 20260506 01:59:24.341 | INFO | BytesWritten : 4528714240515 | 20260506 01:59:24.341 | INFO | NumEvents : 2127303508005 | 20260506 01:59:26.147 | INFO | Analyzing chunk frames = [8056832,8057344) | 20260506 01:59:26.147 | INFO | MemInfo: { | 20260506 01:59:26.147 | INFO | "gpu" : " 17 GB", | 20260506 01:59:26.147 | INFO | "huge" : "", | 20260506 01:59:26.147 | INFO | "huge_shared" : " 27 GB", | 20260506 01:59:26.147 | INFO | "malloc" : " 14 GB", | 20260506 01:59:26.147 | INFO | "pinned" : " 29 GB" | 20260506 01:59:26.147 | INFO | } | 20260506 01:59:26.147 | INFO | Perf report for the last 5.14633 seconds: | 20260506 01:59:26.147 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:59:26.147 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:26.147 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:26.147 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:26.147 | INFO | ANALYSIS : 62.4%, 3.0, 97.6ms, 0 | 20260506 01:59:26.147 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:26.147 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260506 01:59:26.147 | INFO | PRE_HQ : 59.4%, 1.7, 53.4ms, 0 | 20260506 01:59:26.147 | INFO | BAZWRITER : 61.7%, 3.5, 111.3ms, 0 | 20260506 01:59:26.147 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 01:59:26.147 | INFO | Uploaded 36GiB to GPU in 2.18507s (16.4755GiB/s) | 20260506 01:59:26.147 | INFO | Downloaded 38GiB to GPU in 2.23936s (16.9691GiB/s) | 20260506 01:59:31.281 | INFO | Analyzing chunk frames = [8057344,8057856) | 20260506 01:59:31.281 | INFO | MemInfo: { | 20260506 01:59:31.281 | INFO | "gpu" : " 17 GB", | 20260506 01:59:31.281 | INFO | "huge" : "", | 20260506 01:59:31.281 | INFO | "huge_shared" : " 27 GB", | 20260506 01:59:31.281 | INFO | "malloc" : " 14 GB", | 20260506 01:59:31.281 | INFO | "pinned" : " 29 GB" | 20260506 01:59:31.281 | INFO | } | 20260506 01:59:31.282 | INFO | Perf report for the last 5.13465 seconds: | 20260506 01:59:31.282 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:59:31.282 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:31.282 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:31.282 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:31.282 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 01:59:31.282 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:31.282 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:31.282 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:59:31.282 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:31.282 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:31.282 | INFO | Uploaded 36GiB to GPU in 1.92961s (18.6567GiB/s) | 20260506 01:59:31.282 | INFO | Downloaded 34GiB to GPU in 1.97542s (17.2115GiB/s) | 20260506 01:59:36.416 | INFO | Analyzing chunk frames = [8057856,8058368) | 20260506 01:59:36.416 | INFO | MemInfo: { | 20260506 01:59:36.416 | INFO | "gpu" : " 17 GB", | 20260506 01:59:36.416 | INFO | "huge" : "", | 20260506 01:59:36.416 | INFO | "huge_shared" : " 27 GB", | 20260506 01:59:36.416 | INFO | "malloc" : " 14 GB", | 20260506 01:59:36.416 | INFO | "pinned" : " 29 GB" | 20260506 01:59:36.416 | INFO | } | 20260506 01:59:36.416 | INFO | Perf report for the last 5.13421 seconds: | 20260506 01:59:36.416 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:59:36.416 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:36.416 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:36.416 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:36.416 | INFO | ANALYSIS : 61.3%, 3.0, 95.2ms, 0 | 20260506 01:59:36.416 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:36.416 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:36.416 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:59:36.416 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:36.416 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:36.416 | INFO | Uploaded 36GiB to GPU in 2.08412s (17.2735GiB/s) | 20260506 01:59:36.416 | INFO | Downloaded 34GiB to GPU in 2.02228s (16.8127GiB/s) | 20260506 01:59:41.556 | INFO | Analyzing chunk frames = [8058368,8058880) | 20260506 01:59:41.556 | INFO | MemInfo: { | 20260506 01:59:41.556 | INFO | "gpu" : " 17 GB", | 20260506 01:59:41.556 | INFO | "huge" : "", | 20260506 01:59:41.556 | INFO | "huge_shared" : " 27 GB", | 20260506 01:59:41.556 | INFO | "malloc" : " 14 GB", | 20260506 01:59:41.556 | INFO | "pinned" : " 29 GB" | 20260506 01:59:41.556 | INFO | } | 20260506 01:59:41.556 | INFO | Perf report for the last 5.14021 seconds: | 20260506 01:59:41.556 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:59:41.556 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:41.556 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:41.556 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:41.556 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260506 01:59:41.556 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:41.556 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:41.556 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 01:59:41.556 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:41.556 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:41.556 | INFO | Uploaded 36GiB to GPU in 2.14225s (16.8047GiB/s) | 20260506 01:59:41.556 | INFO | Downloaded 34GiB to GPU in 2.02053s (16.8273GiB/s) | 20260506 01:59:46.701 | INFO | Analyzing chunk frames = [8058880,8059392) | 20260506 01:59:46.701 | INFO | MemInfo: { | 20260506 01:59:46.701 | INFO | "gpu" : " 17 GB", | 20260506 01:59:46.701 | INFO | "huge" : "", | 20260506 01:59:46.701 | INFO | "huge_shared" : " 27 GB", | 20260506 01:59:46.701 | INFO | "malloc" : " 14 GB", | 20260506 01:59:46.701 | INFO | "pinned" : " 29 GB" | 20260506 01:59:46.701 | INFO | } | 20260506 01:59:46.701 | INFO | Perf report for the last 5.14531 seconds: | 20260506 01:59:46.701 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:59:46.701 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:46.701 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:46.701 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:46.701 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 01:59:46.701 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:46.702 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:46.702 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 01:59:46.702 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:46.702 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:46.702 | INFO | Uploaded 36GiB to GPU in 2.11693s (17.0058GiB/s) | 20260506 01:59:46.702 | INFO | Downloaded 34GiB to GPU in 2.04729s (16.6073GiB/s) | 20260506 01:59:51.843 | INFO | Analyzing chunk frames = [8059392,8059904) | 20260506 01:59:51.843 | INFO | MemInfo: { | 20260506 01:59:51.843 | INFO | "gpu" : " 17 GB", | 20260506 01:59:51.843 | INFO | "huge" : "", | 20260506 01:59:51.843 | INFO | "huge_shared" : " 27 GB", | 20260506 01:59:51.843 | INFO | "malloc" : " 14 GB", | 20260506 01:59:51.843 | INFO | "pinned" : " 29 GB" | 20260506 01:59:51.843 | INFO | } | 20260506 01:59:51.843 | INFO | Perf report for the last 5.14147 seconds: | 20260506 01:59:51.843 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:59:51.843 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:51.843 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:51.843 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:51.843 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 01:59:51.843 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:51.843 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:51.843 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 01:59:51.843 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:51.843 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:51.843 | INFO | Uploaded 36GiB to GPU in 2.09233s (17.2057GiB/s) | 20260506 01:59:51.843 | INFO | Downloaded 34GiB to GPU in 2.03537s (16.7046GiB/s) | 20260506 01:59:56.976 | INFO | Analyzing chunk frames = [8059904,8060416) | 20260506 01:59:56.976 | INFO | MemInfo: { | 20260506 01:59:56.976 | INFO | "gpu" : " 17 GB", | 20260506 01:59:56.976 | INFO | "huge" : "", | 20260506 01:59:56.976 | INFO | "huge_shared" : " 27 GB", | 20260506 01:59:56.976 | INFO | "malloc" : " 14 GB", | 20260506 01:59:56.976 | INFO | "pinned" : " 29 GB" | 20260506 01:59:56.976 | INFO | } | 20260506 01:59:56.976 | INFO | Perf report for the last 5.13354 seconds: | 20260506 01:59:56.976 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 01:59:56.976 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:56.977 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:56.977 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:56.977 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260506 01:59:56.977 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:56.977 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:56.977 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260506 01:59:56.977 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:56.977 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 01:59:56.977 | INFO | Uploaded 36GiB to GPU in 2.01993s (17.8224GiB/s) | 20260506 01:59:56.977 | INFO | Downloaded 34GiB to GPU in 2.04863s (16.5965GiB/s) | 20260506 02:00:02.123 | INFO | Analyzing chunk frames = [8060416,8060928) | 20260506 02:00:02.123 | INFO | MemInfo: { | 20260506 02:00:02.123 | INFO | "gpu" : " 17 GB", | 20260506 02:00:02.123 | INFO | "huge" : "", | 20260506 02:00:02.123 | INFO | "huge_shared" : " 27 GB", | 20260506 02:00:02.123 | INFO | "malloc" : " 14 GB", | 20260506 02:00:02.123 | INFO | "pinned" : " 29 GB" | 20260506 02:00:02.123 | INFO | } | 20260506 02:00:02.123 | INFO | Perf report for the last 5.14693 seconds: | 20260506 02:00:02.123 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:00:02.123 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:02.123 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:02.123 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:02.123 | INFO | ANALYSIS : 61.7%, 3.0, 96.5ms, 0 | 20260506 02:00:02.123 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:02.123 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:02.123 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 02:00:02.124 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:02.124 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:02.124 | INFO | Uploaded 36GiB to GPU in 2.13157s (16.889GiB/s) | 20260506 02:00:02.124 | INFO | Downloaded 34GiB to GPU in 2.03025s (16.7467GiB/s) | 20260506 02:00:02.124 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 02:00:02.124 | INFO | Total mem usage: 15.3 GiB | 20260506 02:00:02.124 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 02:00:02.124 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.2%) | 20260506 02:00:02.124 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 02:00:02.124 | INFO | High Water Breakdown: | 20260506 02:00:02.124 | INFO | 60.9% : 8.54 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 02:00:02.124 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 02:00:02.124 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 02:00:02.124 | INFO | ------------------------------------------ | 20260506 02:00:02.124 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:00:02.124 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 02:00:02.124 | INFO | Total mem usage: 26.5 GiB | 20260506 02:00:02.124 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 02:00:02.124 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 02:00:02.124 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260506 02:00:02.124 | INFO | High Water Breakdown: | 20260506 02:00:02.124 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:00:02.124 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:00:02.124 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:00:02.124 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:00:02.124 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:00:02.124 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:00:02.124 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 02:00:02.124 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 02:00:02.124 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:00:02.124 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 02:00:02.124 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 02:00:02.124 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 02:00:02.124 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 02:00:02.124 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 02:00:02.124 | INFO | ------------------------------------------ | 20260506 02:00:02.124 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:00:02.124 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 02:00:02.124 | INFO | Total mem usage: 25.6 GiB | 20260506 02:00:02.124 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 02:00:02.124 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 02:00:02.124 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 02:00:02.124 | INFO | High Water Breakdown: | 20260506 02:00:02.124 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 02:00:02.124 | INFO | ------------------------------------------ | 20260506 02:00:02.124 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:00:02.124 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 02:00:02.124 | INFO | Total mem usage: 15.7 GiB | 20260506 02:00:02.124 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 02:00:02.124 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 02:00:02.124 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 02:00:02.124 | INFO | High Water Breakdown: | 20260506 02:00:02.124 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 02:00:02.124 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 02:00:02.124 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 02:00:02.124 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 02:00:02.124 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 02:00:02.124 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 02:00:02.124 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 02:00:02.124 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 02:00:02.124 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 02:00:02.124 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:00:02.124 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 02:00:02.124 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:00:02.124 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 02:00:02.124 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 02:00:02.124 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 02:00:02.124 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 02:00:02.125 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:00:02.125 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 02:00:02.125 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:00:02.125 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:00:02.125 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:00:02.125 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 02:00:02.125 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 02:00:02.125 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:00:02.125 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 02:00:02.125 | INFO | ------------------------------------------ | 20260506 02:00:02.125 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:00:02.125 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 02:00:02.125 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:00:02.125 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:00:02.125 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.37e+01, 1.34e+01 / 6.87e-01 | 20260506 02:00:02.125 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.05e-01, 1.02e-02 / 6.58e-03 | 20260506 02:00:02.125 | INFO | PerfReport: 11.85%, Baseline -- 3.71e+00 / 4.49e+00, 5.76e-01 / 2.10e-01 | 20260506 02:00:02.125 | INFO | PerfReport: 3.29%, Histogram -- 1.03e+00 / 1.51e+00, 1.92e-01 / 9.49e-02 | 20260506 02:00:02.125 | INFO | PerfReport: 16.01%, DME -- 4.00e+01 / 4.40e+01, 3.81e+00 / 8.38e-01 | 20260506 02:00:02.125 | INFO | PerfReport: 57.42%, FrameLabeling -- 1.80e+01 / 1.92e+01, 1.50e+00 / 9.13e-01 | 20260506 02:00:02.125 | INFO | PerfReport: 7.06%, PulseAccumulating -- 2.21e+00 / 2.73e+00, 4.17e-01 / 2.19e-01 | 20260506 02:00:02.125 | INFO | PerfReport: 3.02%, Metrics -- 9.45e-01 / 1.20e+00, 2.49e-01 / 1.06e-01 | 20260506 02:00:02.125 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.27e-01 / 3.74e-01, 5.38e-02 / 2.37e-02 | 20260506 02:00:02.125 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 02:00:02.125 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:00:02.125 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:00:02.125 | INFO | PerfReport: Full Execution -- 9.77e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260506 02:00:02.125 | INFO | PerfReport: 22.15%, Upload -- 2.17e+01 / 0.00e+00, 7.23e+00 / 0.00e+00 | 20260506 02:00:02.125 | INFO | PerfReport: 21.86%, Download -- 2.14e+01 / 0.00e+00, 7.51e+00 / 0.00e+00 | 20260506 02:00:02.125 | INFO | PerfReport: 31.99%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 02:00:02.125 | INFO | PerfReport: 2.07%, LockUpload -- 2.02e+00 / 0.00e+00, 7.90e+00 / 0.00e+00 | 20260506 02:00:02.125 | INFO | PerfReport: 1.29%, LockDownload -- 1.26e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260506 02:00:02.125 | INFO | PerfReport: 20.62%, LockCompute -- 2.02e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260506 02:00:02.125 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 2.47e-02 / 0.00e+00 | 20260506 02:00:02.249 | INFO | Starting a round of Baz IO | 20260506 02:00:05.374 | INFO | Populating RTMetrics report | 20260506 02:00:05.374 | INFO | RTMetrics report populated | 20260506 02:00:05.374 | INFO | Opening tmp RTMetrics json file | 20260506 02:00:05.375 | INFO | Serializing RTMetrics to json | 20260506 02:00:05.377 | INFO | Serialization done, swapping tmp file now... | 20260506 02:00:05.377 | INFO | RTMetrics done | 20260506 02:00:05.458 | INFO | Finished a round of Baz IO: Wrote 1.809 GiB in 3.209 seconds | 20260506 02:00:05.459 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.29% / 0.00% / 78.71% | 20260506 02:00:05.459 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0750343% / 0% / 8.74201e-05% | 20260506 02:00:05.459 | INFO | Baz IO overall: | 20260506 02:00:05.459 | INFO | BytesWritten : 4530656595867 | 20260506 02:00:05.459 | INFO | NumEvents : 2128187847348 | 20260506 02:00:07.261 | INFO | Analyzing chunk frames = [8060928,8061440) | 20260506 02:00:07.261 | INFO | MemInfo: { | 20260506 02:00:07.261 | INFO | "gpu" : " 17 GB", | 20260506 02:00:07.261 | INFO | "huge" : "", | 20260506 02:00:07.261 | INFO | "huge_shared" : " 27 GB", | 20260506 02:00:07.261 | INFO | "malloc" : " 14 GB", | 20260506 02:00:07.261 | INFO | "pinned" : " 29 GB" | 20260506 02:00:07.261 | INFO | } | 20260506 02:00:07.261 | INFO | Perf report for the last 5.13759 seconds: | 20260506 02:00:07.261 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:00:07.261 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:07.261 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:07.261 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:07.261 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260506 02:00:07.261 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:07.261 | INFO | RT_METRICS : 42.8%, 1.0, 22.4ms, 0 | 20260506 02:00:07.261 | INFO | PRE_HQ : 59.3%, 1.7, 53.6ms, 0 | 20260506 02:00:07.261 | INFO | BAZWRITER : 61.6%, 3.5, 111.7ms, 0 | 20260506 02:00:07.261 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:00:07.261 | INFO | Uploaded 36GiB to GPU in 2.14449s (16.7872GiB/s) | 20260506 02:00:07.261 | INFO | Downloaded 38GiB to GPU in 2.22537s (17.0758GiB/s) | 20260506 02:00:12.396 | INFO | Analyzing chunk frames = [8061440,8061952) | 20260506 02:00:12.396 | INFO | MemInfo: { | 20260506 02:00:12.396 | INFO | "gpu" : " 17 GB", | 20260506 02:00:12.396 | INFO | "huge" : "", | 20260506 02:00:12.396 | INFO | "huge_shared" : " 27 GB", | 20260506 02:00:12.396 | INFO | "malloc" : " 14 GB", | 20260506 02:00:12.396 | INFO | "pinned" : " 29 GB" | 20260506 02:00:12.396 | INFO | } | 20260506 02:00:12.396 | INFO | Perf report for the last 5.13547 seconds: | 20260506 02:00:12.396 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:00:12.396 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:12.396 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:12.396 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:12.396 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260506 02:00:12.396 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:12.396 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:12.396 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 02:00:12.396 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:12.396 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:12.397 | INFO | Uploaded 36GiB to GPU in 1.93928s (18.5636GiB/s) | 20260506 02:00:12.397 | INFO | Downloaded 34GiB to GPU in 1.98606s (17.1194GiB/s) | 20260506 02:00:17.533 | INFO | Analyzing chunk frames = [8061952,8062464) | 20260506 02:00:17.533 | INFO | MemInfo: { | 20260506 02:00:17.533 | INFO | "gpu" : " 17 GB", | 20260506 02:00:17.533 | INFO | "huge" : "", | 20260506 02:00:17.533 | INFO | "huge_shared" : " 27 GB", | 20260506 02:00:17.533 | INFO | "malloc" : " 14 GB", | 20260506 02:00:17.533 | INFO | "pinned" : " 29 GB" | 20260506 02:00:17.533 | INFO | } | 20260506 02:00:17.533 | INFO | Perf report for the last 5.13663 seconds: | 20260506 02:00:17.533 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:00:17.533 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:17.533 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:17.533 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:17.533 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 02:00:17.533 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:17.533 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:17.533 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:00:17.533 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:17.533 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:17.533 | INFO | Uploaded 36GiB to GPU in 2.08865s (17.2361GiB/s) | 20260506 02:00:17.533 | INFO | Downloaded 34GiB to GPU in 2.01951s (16.8358GiB/s) | 20260506 02:00:22.669 | INFO | Analyzing chunk frames = [8062464,8062976) | 20260506 02:00:22.669 | INFO | MemInfo: { | 20260506 02:00:22.669 | INFO | "gpu" : " 17 GB", | 20260506 02:00:22.669 | INFO | "huge" : "", | 20260506 02:00:22.669 | INFO | "huge_shared" : " 27 GB", | 20260506 02:00:22.669 | INFO | "malloc" : " 14 GB", | 20260506 02:00:22.669 | INFO | "pinned" : " 29 GB" | 20260506 02:00:22.669 | INFO | } | 20260506 02:00:22.669 | INFO | Perf report for the last 5.13641 seconds: | 20260506 02:00:22.669 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:00:22.669 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:22.669 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:22.669 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:22.669 | INFO | ANALYSIS : 61.5%, 3.0, 95.5ms, 0 | 20260506 02:00:22.669 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:22.669 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:22.670 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 02:00:22.670 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:22.670 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:22.670 | INFO | Uploaded 36GiB to GPU in 2.13409s (16.869GiB/s) | 20260506 02:00:22.670 | INFO | Downloaded 34GiB to GPU in 2.0113s (16.9045GiB/s) | 20260506 02:00:27.816 | INFO | Analyzing chunk frames = [8062976,8063488) | 20260506 02:00:27.816 | INFO | MemInfo: { | 20260506 02:00:27.816 | INFO | "gpu" : " 17 GB", | 20260506 02:00:27.816 | INFO | "huge" : "", | 20260506 02:00:27.816 | INFO | "huge_shared" : " 27 GB", | 20260506 02:00:27.816 | INFO | "malloc" : " 14 GB", | 20260506 02:00:27.816 | INFO | "pinned" : " 29 GB" | 20260506 02:00:27.816 | INFO | } | 20260506 02:00:27.816 | INFO | Perf report for the last 5.14673 seconds: | 20260506 02:00:27.816 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:00:27.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:27.816 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:27.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:27.816 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 02:00:27.816 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:27.816 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:27.816 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:00:27.816 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:27.816 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:27.816 | INFO | Uploaded 36GiB to GPU in 2.08808s (17.2407GiB/s) | 20260506 02:00:27.816 | INFO | Downloaded 34GiB to GPU in 2.04688s (16.6106GiB/s) | 20260506 02:00:32.960 | INFO | Analyzing chunk frames = [8063488,8064000) | 20260506 02:00:32.960 | INFO | MemInfo: { | 20260506 02:00:32.960 | INFO | "gpu" : " 17 GB", | 20260506 02:00:32.960 | INFO | "huge" : "", | 20260506 02:00:32.960 | INFO | "huge_shared" : " 27 GB", | 20260506 02:00:32.960 | INFO | "malloc" : " 14 GB", | 20260506 02:00:32.960 | INFO | "pinned" : " 29 GB" | 20260506 02:00:32.960 | INFO | } | 20260506 02:00:32.960 | INFO | Perf report for the last 5.14365 seconds: | 20260506 02:00:32.960 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:00:32.960 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:32.960 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:32.960 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:32.960 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 02:00:32.960 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:32.960 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:32.960 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 02:00:32.960 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:32.960 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:32.960 | INFO | Uploaded 36GiB to GPU in 2.10045s (17.1392GiB/s) | 20260506 02:00:32.960 | INFO | Downloaded 34GiB to GPU in 2.03796s (16.6834GiB/s) | 20260506 02:00:38.092 | INFO | Analyzing chunk frames = [8064000,8064512) | 20260506 02:00:38.092 | INFO | MemInfo: { | 20260506 02:00:38.092 | INFO | "gpu" : " 17 GB", | 20260506 02:00:38.092 | INFO | "huge" : "", | 20260506 02:00:38.092 | INFO | "huge_shared" : " 27 GB", | 20260506 02:00:38.092 | INFO | "malloc" : " 14 GB", | 20260506 02:00:38.092 | INFO | "pinned" : " 29 GB" | 20260506 02:00:38.092 | INFO | } | 20260506 02:00:38.092 | INFO | Perf report for the last 5.13248 seconds: | 20260506 02:00:38.092 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:00:38.092 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:38.092 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:38.092 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:38.092 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260506 02:00:38.092 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:38.092 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:38.092 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260506 02:00:38.092 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:38.092 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:38.093 | INFO | Uploaded 36GiB to GPU in 2.09318s (17.1987GiB/s) | 20260506 02:00:38.093 | INFO | Downloaded 34GiB to GPU in 2.06358s (16.4762GiB/s) | 20260506 02:00:43.230 | INFO | Analyzing chunk frames = [8064512,8065024) | 20260506 02:00:43.230 | INFO | MemInfo: { | 20260506 02:00:43.230 | INFO | "gpu" : " 17 GB", | 20260506 02:00:43.230 | INFO | "huge" : "", | 20260506 02:00:43.230 | INFO | "huge_shared" : " 27 GB", | 20260506 02:00:43.230 | INFO | "malloc" : " 14 GB", | 20260506 02:00:43.230 | INFO | "pinned" : " 29 GB" | 20260506 02:00:43.230 | INFO | } | 20260506 02:00:43.230 | INFO | Perf report for the last 5.13757 seconds: | 20260506 02:00:43.230 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:00:43.230 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:43.230 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:43.230 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:43.230 | INFO | ANALYSIS : 61.4%, 3.0, 95.9ms, 0 | 20260506 02:00:43.230 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:43.230 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:43.230 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 02:00:43.230 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:43.230 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:43.230 | INFO | Uploaded 36GiB to GPU in 2.04863s (17.5727GiB/s) | 20260506 02:00:43.230 | INFO | Downloaded 34GiB to GPU in 2.02213s (16.8139GiB/s) | 20260506 02:00:43.348 | INFO | Starting a round of Baz IO | 20260506 02:00:46.494 | INFO | Populating RTMetrics report | 20260506 02:00:46.494 | INFO | RTMetrics report populated | 20260506 02:00:46.494 | INFO | Opening tmp RTMetrics json file | 20260506 02:00:46.497 | INFO | Serializing RTMetrics to json | 20260506 02:00:46.498 | INFO | Serialization done, swapping tmp file now... | 20260506 02:00:46.498 | INFO | RTMetrics done | 20260506 02:00:46.570 | INFO | Finished a round of Baz IO: Wrote 1.809 GiB in 3.222 seconds | 20260506 02:00:46.570 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.29% / 0.00% / 78.71% | 20260506 02:00:46.570 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.074772% / 0% / 8.74201e-05% | 20260506 02:00:46.570 | INFO | Baz IO overall: | 20260506 02:00:46.570 | INFO | BytesWritten : 4532598753426 | 20260506 02:00:46.570 | INFO | NumEvents : 2129072180384 | 20260506 02:00:48.368 | INFO | Analyzing chunk frames = [8065024,8065536) | 20260506 02:00:48.368 | INFO | MemInfo: { | 20260506 02:00:48.368 | INFO | "gpu" : " 17 GB", | 20260506 02:00:48.368 | INFO | "huge" : "", | 20260506 02:00:48.368 | INFO | "huge_shared" : " 27 GB", | 20260506 02:00:48.368 | INFO | "malloc" : " 14 GB", | 20260506 02:00:48.368 | INFO | "pinned" : " 29 GB" | 20260506 02:00:48.368 | INFO | } | 20260506 02:00:48.368 | INFO | Perf report for the last 5.1382 seconds: | 20260506 02:00:48.368 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:00:48.368 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:48.368 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:48.368 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:48.368 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260506 02:00:48.368 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:48.368 | INFO | RT_METRICS : 43.0%, 1.0, 22.5ms, 0 | 20260506 02:00:48.368 | INFO | PRE_HQ : 59.7%, 1.7, 54.1ms, 0 | 20260506 02:00:48.368 | INFO | BAZWRITER : 61.7%, 3.5, 111.1ms, 0 | 20260506 02:00:48.368 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:00:48.368 | INFO | Uploaded 36GiB to GPU in 2.18242s (16.4954GiB/s) | 20260506 02:00:48.368 | INFO | Downloaded 38GiB to GPU in 2.24412s (16.9331GiB/s) | 20260506 02:00:53.509 | INFO | Analyzing chunk frames = [8065536,8066048) | 20260506 02:00:53.509 | INFO | MemInfo: { | 20260506 02:00:53.509 | INFO | "gpu" : " 17 GB", | 20260506 02:00:53.509 | INFO | "huge" : "", | 20260506 02:00:53.509 | INFO | "huge_shared" : " 27 GB", | 20260506 02:00:53.509 | INFO | "malloc" : " 14 GB", | 20260506 02:00:53.509 | INFO | "pinned" : " 29 GB" | 20260506 02:00:53.509 | INFO | } | 20260506 02:00:53.509 | INFO | Perf report for the last 5.14116 seconds: | 20260506 02:00:53.509 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:00:53.509 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:53.509 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:53.509 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:53.509 | INFO | ANALYSIS : 60.8%, 3.0, 94.9ms, 0 | 20260506 02:00:53.509 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:53.509 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:53.509 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:00:53.509 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:53.509 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:53.510 | INFO | Uploaded 36GiB to GPU in 1.98976s (18.0926GiB/s) | 20260506 02:00:53.510 | INFO | Downloaded 34GiB to GPU in 1.98338s (17.1424GiB/s) | 20260506 02:00:58.652 | INFO | Analyzing chunk frames = [8066048,8066560) | 20260506 02:00:58.652 | INFO | MemInfo: { | 20260506 02:00:58.652 | INFO | "gpu" : " 17 GB", | 20260506 02:00:58.652 | INFO | "huge" : "", | 20260506 02:00:58.652 | INFO | "huge_shared" : " 27 GB", | 20260506 02:00:58.652 | INFO | "malloc" : " 14 GB", | 20260506 02:00:58.652 | INFO | "pinned" : " 29 GB" | 20260506 02:00:58.652 | INFO | } | 20260506 02:00:58.652 | INFO | Perf report for the last 5.14264 seconds: | 20260506 02:00:58.652 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:00:58.652 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:58.652 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:58.652 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:58.652 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 02:00:58.652 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:58.652 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:58.652 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:00:58.652 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:58.652 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:00:58.652 | INFO | Uploaded 36GiB to GPU in 2.07571s (17.3435GiB/s) | 20260506 02:00:58.652 | INFO | Downloaded 34GiB to GPU in 2.02177s (16.817GiB/s) | 20260506 02:01:03.792 | INFO | Analyzing chunk frames = [8066560,8067072) | 20260506 02:01:03.792 | INFO | MemInfo: { | 20260506 02:01:03.792 | INFO | "gpu" : " 17 GB", | 20260506 02:01:03.792 | INFO | "huge" : "", | 20260506 02:01:03.792 | INFO | "huge_shared" : " 27 GB", | 20260506 02:01:03.792 | INFO | "malloc" : " 14 GB", | 20260506 02:01:03.792 | INFO | "pinned" : " 29 GB" | 20260506 02:01:03.792 | INFO | } | 20260506 02:01:03.792 | INFO | Perf report for the last 5.13977 seconds: | 20260506 02:01:03.792 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:01:03.792 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:03.792 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:03.792 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:03.792 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260506 02:01:03.792 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:03.792 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:03.792 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 02:01:03.792 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:03.792 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:03.792 | INFO | Uploaded 36GiB to GPU in 2.15263s (16.7237GiB/s) | 20260506 02:01:03.792 | INFO | Downloaded 34GiB to GPU in 2.06111s (16.496GiB/s) | 20260506 02:01:08.928 | INFO | Analyzing chunk frames = [8067072,8067584) | 20260506 02:01:08.928 | INFO | MemInfo: { | 20260506 02:01:08.928 | INFO | "gpu" : " 17 GB", | 20260506 02:01:08.928 | INFO | "huge" : "", | 20260506 02:01:08.928 | INFO | "huge_shared" : " 27 GB", | 20260506 02:01:08.928 | INFO | "malloc" : " 14 GB", | 20260506 02:01:08.928 | INFO | "pinned" : " 29 GB" | 20260506 02:01:08.928 | INFO | } | 20260506 02:01:08.928 | INFO | Perf report for the last 5.13608 seconds: | 20260506 02:01:08.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:01:08.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:08.928 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:08.928 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:08.928 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260506 02:01:08.928 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:08.928 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:08.928 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 02:01:08.928 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:08.928 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:08.928 | INFO | Uploaded 36GiB to GPU in 2.10013s (17.1418GiB/s) | 20260506 02:01:08.928 | INFO | Downloaded 34GiB to GPU in 2.04s (16.6667GiB/s) | 20260506 02:01:14.073 | INFO | Analyzing chunk frames = [8067584,8068096) | 20260506 02:01:14.073 | INFO | MemInfo: { | 20260506 02:01:14.073 | INFO | "gpu" : " 17 GB", | 20260506 02:01:14.073 | INFO | "huge" : "", | 20260506 02:01:14.073 | INFO | "huge_shared" : " 27 GB", | 20260506 02:01:14.073 | INFO | "malloc" : " 14 GB", | 20260506 02:01:14.073 | INFO | "pinned" : " 29 GB" | 20260506 02:01:14.073 | INFO | } | 20260506 02:01:14.073 | INFO | Perf report for the last 5.14508 seconds: | 20260506 02:01:14.073 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:01:14.073 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:14.073 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:14.073 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:14.073 | INFO | ANALYSIS : 61.0%, 3.0, 95.4ms, 0 | 20260506 02:01:14.073 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:14.073 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:14.073 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:01:14.073 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:14.073 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:14.073 | INFO | Uploaded 36GiB to GPU in 2.07534s (17.3466GiB/s) | 20260506 02:01:14.073 | INFO | Downloaded 34GiB to GPU in 2.04169s (16.6529GiB/s) | 20260506 02:01:19.208 | INFO | Analyzing chunk frames = [8068096,8068608) | 20260506 02:01:19.208 | INFO | MemInfo: { | 20260506 02:01:19.208 | INFO | "gpu" : " 17 GB", | 20260506 02:01:19.208 | INFO | "huge" : "", | 20260506 02:01:19.208 | INFO | "huge_shared" : " 27 GB", | 20260506 02:01:19.208 | INFO | "malloc" : " 14 GB", | 20260506 02:01:19.208 | INFO | "pinned" : " 29 GB" | 20260506 02:01:19.208 | INFO | } | 20260506 02:01:19.209 | INFO | Perf report for the last 5.1356 seconds: | 20260506 02:01:19.209 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:01:19.209 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:19.209 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:19.209 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:19.209 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 02:01:19.209 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:19.209 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:19.209 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 02:01:19.209 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:19.209 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:19.209 | INFO | Uploaded 36GiB to GPU in 2.03731s (17.6704GiB/s) | 20260506 02:01:19.209 | INFO | Downloaded 34GiB to GPU in 2.05083s (16.5786GiB/s) | 20260506 02:01:24.343 | INFO | Analyzing chunk frames = [8068608,8069120) | 20260506 02:01:24.343 | INFO | MemInfo: { | 20260506 02:01:24.343 | INFO | "gpu" : " 17 GB", | 20260506 02:01:24.343 | INFO | "huge" : "", | 20260506 02:01:24.343 | INFO | "huge_shared" : " 27 GB", | 20260506 02:01:24.343 | INFO | "malloc" : " 14 GB", | 20260506 02:01:24.343 | INFO | "pinned" : " 29 GB" | 20260506 02:01:24.343 | INFO | } | 20260506 02:01:24.343 | INFO | Perf report for the last 5.13408 seconds: | 20260506 02:01:24.343 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:01:24.343 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:24.343 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:24.343 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:24.343 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260506 02:01:24.343 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:24.343 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:24.343 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 02:01:24.343 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:24.343 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:24.343 | INFO | Uploaded 36GiB to GPU in 2.0315s (17.7209GiB/s) | 20260506 02:01:24.343 | INFO | Downloaded 34GiB to GPU in 2.02094s (16.8239GiB/s) | 20260506 02:01:24.343 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 02:01:24.343 | INFO | Total mem usage: 15.3 GiB | 20260506 02:01:24.343 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 02:01:24.343 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.19%) | 20260506 02:01:24.343 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 02:01:24.343 | INFO | High Water Breakdown: | 20260506 02:01:24.343 | INFO | 60.9% : 8.54 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 02:01:24.343 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 02:01:24.343 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 02:01:24.343 | INFO | ------------------------------------------ | 20260506 02:01:24.343 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:01:24.343 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 02:01:24.343 | INFO | Total mem usage: 26.5 GiB | 20260506 02:01:24.343 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 02:01:24.343 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 02:01:24.343 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 02:01:24.343 | INFO | High Water Breakdown: | 20260506 02:01:24.343 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:01:24.343 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:01:24.343 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:01:24.343 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:01:24.343 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:01:24.343 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:01:24.343 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 02:01:24.343 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 02:01:24.343 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:01:24.344 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 02:01:24.344 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 02:01:24.344 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 02:01:24.344 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 02:01:24.344 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 02:01:24.344 | INFO | ------------------------------------------ | 20260506 02:01:24.344 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:01:24.344 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 02:01:24.344 | INFO | Total mem usage: 25.6 GiB | 20260506 02:01:24.344 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 02:01:24.344 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 02:01:24.344 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 02:01:24.344 | INFO | High Water Breakdown: | 20260506 02:01:24.344 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 02:01:24.344 | INFO | ------------------------------------------ | 20260506 02:01:24.344 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:01:24.344 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 02:01:24.344 | INFO | Total mem usage: 15.7 GiB | 20260506 02:01:24.344 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 02:01:24.344 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 02:01:24.344 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 02:01:24.344 | INFO | High Water Breakdown: | 20260506 02:01:24.344 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 02:01:24.344 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 02:01:24.344 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 02:01:24.344 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 02:01:24.344 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 02:01:24.344 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 02:01:24.344 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 02:01:24.344 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 02:01:24.344 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 02:01:24.344 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:01:24.344 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 02:01:24.344 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:01:24.344 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 02:01:24.344 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 02:01:24.344 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 02:01:24.344 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 02:01:24.344 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:01:24.344 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 02:01:24.344 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:01:24.344 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:01:24.344 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:01:24.344 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 02:01:24.344 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 02:01:24.344 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:01:24.344 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 02:01:24.344 | INFO | ------------------------------------------ | 20260506 02:01:24.344 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:01:24.344 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 02:01:24.344 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:01:24.344 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:01:24.344 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.37e+01, 1.35e+01 / 7.36e-01 | 20260506 02:01:24.344 | INFO | PerfReport: 0.30%, AggregateStats -- 9.24e-02 / 1.04e-01, 1.07e-02 / 5.64e-03 | 20260506 02:01:24.344 | INFO | PerfReport: 11.86%, Baseline -- 3.71e+00 / 4.49e+00, 5.74e-01 / 2.57e-01 | 20260506 02:01:24.344 | INFO | PerfReport: 3.30%, Histogram -- 1.03e+00 / 1.53e+00, 1.93e-01 / 7.88e-02 | 20260506 02:01:24.344 | INFO | PerfReport: 16.00%, DME -- 4.00e+01 / 4.40e+01, 3.87e+00 / 8.68e-01 | 20260506 02:01:24.344 | INFO | PerfReport: 57.40%, FrameLabeling -- 1.79e+01 / 1.92e+01, 1.49e+00 / 8.85e-01 | 20260506 02:01:24.344 | INFO | PerfReport: 7.06%, PulseAccumulating -- 2.21e+00 / 2.74e+00, 4.19e-01 / 2.07e-01 | 20260506 02:01:24.344 | INFO | PerfReport: 3.03%, Metrics -- 9.48e-01 / 1.21e+00, 2.52e-01 / 1.02e-01 | 20260506 02:01:24.344 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.27e-01 / 3.74e-01, 5.45e-02 / 3.04e-02 | 20260506 02:01:24.344 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 02:01:24.344 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:01:24.344 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:01:24.344 | INFO | PerfReport: Full Execution -- 9.77e+01 / -nan, 2.42e+01 / 0.00e+00 | 20260506 02:01:24.344 | INFO | PerfReport: 22.17%, Upload -- 2.17e+01 / 0.00e+00, 7.23e+00 / 0.00e+00 | 20260506 02:01:24.344 | INFO | PerfReport: 21.90%, Download -- 2.14e+01 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260506 02:01:24.344 | INFO | PerfReport: 32.00%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 02:01:24.344 | INFO | PerfReport: 2.01%, LockUpload -- 1.97e+00 / 0.00e+00, 7.65e+00 / 0.00e+00 | 20260506 02:01:24.344 | INFO | PerfReport: 1.31%, LockDownload -- 1.28e+00 / 0.00e+00, 2.94e+00 / 0.00e+00 | 20260506 02:01:24.344 | INFO | PerfReport: 20.60%, LockCompute -- 2.01e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260506 02:01:24.344 | INFO | PerfReport: 0.02%, Misc -- 1.49e-02 / 0.00e+00, 3.02e-02 / 0.00e+00 | 20260506 02:01:24.460 | INFO | Starting a round of Baz IO | 20260506 02:01:27.603 | INFO | Populating RTMetrics report | 20260506 02:01:27.603 | INFO | RTMetrics report populated | 20260506 02:01:27.603 | INFO | Opening tmp RTMetrics json file | 20260506 02:01:27.604 | INFO | Serializing RTMetrics to json | 20260506 02:01:27.605 | INFO | Serialization done, swapping tmp file now... | 20260506 02:01:27.605 | INFO | RTMetrics done | 20260506 02:01:27.683 | INFO | Finished a round of Baz IO: Wrote 1.807 GiB in 3.223 seconds | 20260506 02:01:27.683 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.29% / 0.00% / 78.71% | 20260506 02:01:27.683 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0745654% / 0% / 6.35783e-05% | 20260506 02:01:27.683 | INFO | Baz IO overall: | 20260506 02:01:27.683 | INFO | BytesWritten : 4534538783434 | 20260506 02:01:27.683 | INFO | NumEvents : 2129955269510 | 20260506 02:01:29.484 | INFO | Analyzing chunk frames = [8069120,8069632) | 20260506 02:01:29.484 | INFO | MemInfo: { | 20260506 02:01:29.484 | INFO | "gpu" : " 17 GB", | 20260506 02:01:29.484 | INFO | "huge" : "", | 20260506 02:01:29.484 | INFO | "huge_shared" : " 27 GB", | 20260506 02:01:29.484 | INFO | "malloc" : " 14 GB", | 20260506 02:01:29.484 | INFO | "pinned" : " 29 GB" | 20260506 02:01:29.484 | INFO | } | 20260506 02:01:29.484 | INFO | Perf report for the last 5.14117 seconds: | 20260506 02:01:29.484 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:01:29.484 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:29.484 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:29.484 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:29.484 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260506 02:01:29.484 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:29.484 | INFO | RT_METRICS : 42.8%, 1.0, 22.4ms, 0 | 20260506 02:01:29.484 | INFO | PRE_HQ : 59.5%, 1.7, 53.3ms, 0 | 20260506 02:01:29.484 | INFO | BAZWRITER : 61.8%, 3.5, 111.7ms, 0 | 20260506 02:01:29.484 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:01:29.484 | INFO | Uploaded 36GiB to GPU in 2.18943s (16.4426GiB/s) | 20260506 02:01:29.484 | INFO | Downloaded 38GiB to GPU in 2.24759s (16.907GiB/s) | 20260506 02:01:34.624 | INFO | Analyzing chunk frames = [8069632,8070144) | 20260506 02:01:34.624 | INFO | MemInfo: { | 20260506 02:01:34.624 | INFO | "gpu" : " 17 GB", | 20260506 02:01:34.624 | INFO | "huge" : "", | 20260506 02:01:34.624 | INFO | "huge_shared" : " 27 GB", | 20260506 02:01:34.624 | INFO | "malloc" : " 14 GB", | 20260506 02:01:34.624 | INFO | "pinned" : " 29 GB" | 20260506 02:01:34.624 | INFO | } | 20260506 02:01:34.624 | INFO | Perf report for the last 5.14029 seconds: | 20260506 02:01:34.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:01:34.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:34.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:34.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:34.624 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260506 02:01:34.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:34.624 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:34.624 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 02:01:34.624 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:34.624 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:34.625 | INFO | Uploaded 36GiB to GPU in 2.12092s (16.9738GiB/s) | 20260506 02:01:34.625 | INFO | Downloaded 34GiB to GPU in 2.04078s (16.6603GiB/s) | 20260506 02:01:39.767 | INFO | Analyzing chunk frames = [8070144,8070656) | 20260506 02:01:39.767 | INFO | MemInfo: { | 20260506 02:01:39.767 | INFO | "gpu" : " 17 GB", | 20260506 02:01:39.767 | INFO | "huge" : "", | 20260506 02:01:39.767 | INFO | "huge_shared" : " 27 GB", | 20260506 02:01:39.767 | INFO | "malloc" : " 14 GB", | 20260506 02:01:39.767 | INFO | "pinned" : " 29 GB" | 20260506 02:01:39.767 | INFO | } | 20260506 02:01:39.767 | INFO | Perf report for the last 5.14293 seconds: | 20260506 02:01:39.767 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:01:39.767 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:39.767 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:39.767 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:39.767 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 02:01:39.767 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:39.767 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:39.767 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:01:39.767 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:39.767 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:39.767 | INFO | Uploaded 36GiB to GPU in 2.08823s (17.2395GiB/s) | 20260506 02:01:39.768 | INFO | Downloaded 34GiB to GPU in 2.02916s (16.7557GiB/s) | 20260506 02:01:44.908 | INFO | Analyzing chunk frames = [8070656,8071168) | 20260506 02:01:44.908 | INFO | MemInfo: { | 20260506 02:01:44.908 | INFO | "gpu" : " 17 GB", | 20260506 02:01:44.908 | INFO | "huge" : "", | 20260506 02:01:44.908 | INFO | "huge_shared" : " 27 GB", | 20260506 02:01:44.908 | INFO | "malloc" : " 14 GB", | 20260506 02:01:44.908 | INFO | "pinned" : " 29 GB" | 20260506 02:01:44.908 | INFO | } | 20260506 02:01:44.908 | INFO | Perf report for the last 5.14121 seconds: | 20260506 02:01:44.908 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:01:44.908 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:44.908 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:44.908 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:44.908 | INFO | ANALYSIS : 61.0%, 3.0, 94.8ms, 0 | 20260506 02:01:44.908 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:44.908 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:44.908 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:01:44.908 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:44.908 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:44.909 | INFO | Uploaded 36GiB to GPU in 2.08129s (17.297GiB/s) | 20260506 02:01:44.909 | INFO | Downloaded 34GiB to GPU in 2.00617s (16.9477GiB/s) | 20260506 02:01:50.042 | INFO | Analyzing chunk frames = [8071168,8071680) | 20260506 02:01:50.042 | INFO | MemInfo: { | 20260506 02:01:50.042 | INFO | "gpu" : " 17 GB", | 20260506 02:01:50.042 | INFO | "huge" : "", | 20260506 02:01:50.042 | INFO | "huge_shared" : " 27 GB", | 20260506 02:01:50.042 | INFO | "malloc" : " 14 GB", | 20260506 02:01:50.042 | INFO | "pinned" : " 29 GB" | 20260506 02:01:50.042 | INFO | } | 20260506 02:01:50.042 | INFO | Perf report for the last 5.13398 seconds: | 20260506 02:01:50.042 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:01:50.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:50.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:50.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:50.042 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260506 02:01:50.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:50.042 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:50.042 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:01:50.042 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:50.042 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:50.043 | INFO | Uploaded 36GiB to GPU in 2.09545s (17.1801GiB/s) | 20260506 02:01:50.043 | INFO | Downloaded 34GiB to GPU in 2.02765s (16.7682GiB/s) | 20260506 02:01:55.179 | INFO | Analyzing chunk frames = [8071680,8072192) | 20260506 02:01:55.179 | INFO | MemInfo: { | 20260506 02:01:55.179 | INFO | "gpu" : " 17 GB", | 20260506 02:01:55.179 | INFO | "huge" : "", | 20260506 02:01:55.179 | INFO | "huge_shared" : " 27 GB", | 20260506 02:01:55.179 | INFO | "malloc" : " 14 GB", | 20260506 02:01:55.179 | INFO | "pinned" : " 29 GB" | 20260506 02:01:55.179 | INFO | } | 20260506 02:01:55.179 | INFO | Perf report for the last 5.13692 seconds: | 20260506 02:01:55.179 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:01:55.179 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:55.179 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:55.179 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:55.179 | INFO | ANALYSIS : 61.2%, 3.0, 95.6ms, 0 | 20260506 02:01:55.179 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:55.179 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:55.179 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 02:01:55.179 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:55.179 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:01:55.179 | INFO | Uploaded 36GiB to GPU in 2.07306s (17.3656GiB/s) | 20260506 02:01:55.179 | INFO | Downloaded 34GiB to GPU in 2.03618s (16.6979GiB/s) | 20260506 02:02:00.319 | INFO | Analyzing chunk frames = [8072192,8072704) | 20260506 02:02:00.319 | INFO | MemInfo: { | 20260506 02:02:00.319 | INFO | "gpu" : " 17 GB", | 20260506 02:02:00.319 | INFO | "huge" : "", | 20260506 02:02:00.319 | INFO | "huge_shared" : " 27 GB", | 20260506 02:02:00.319 | INFO | "malloc" : " 14 GB", | 20260506 02:02:00.319 | INFO | "pinned" : " 29 GB" | 20260506 02:02:00.319 | INFO | } | 20260506 02:02:00.319 | INFO | Perf report for the last 5.13995 seconds: | 20260506 02:02:00.319 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:02:00.319 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:00.319 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:00.319 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:00.319 | INFO | ANALYSIS : 62.0%, 3.0, 95.8ms, 0 | 20260506 02:02:00.319 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:00.319 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:00.319 | INFO | PRE_HQ : 27.0%, 1.0, 13.9ms, 0 | 20260506 02:02:00.319 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:00.319 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:00.319 | INFO | Uploaded 36GiB to GPU in 2.0623s (17.4562GiB/s) | 20260506 02:02:00.319 | INFO | Downloaded 34GiB to GPU in 2.07464s (16.3884GiB/s) | 20260506 02:02:05.465 | INFO | Analyzing chunk frames = [8072704,8073216) | 20260506 02:02:05.465 | INFO | MemInfo: { | 20260506 02:02:05.465 | INFO | "gpu" : " 17 GB", | 20260506 02:02:05.465 | INFO | "huge" : "", | 20260506 02:02:05.465 | INFO | "huge_shared" : " 27 GB", | 20260506 02:02:05.465 | INFO | "malloc" : " 14 GB", | 20260506 02:02:05.465 | INFO | "pinned" : " 29 GB" | 20260506 02:02:05.465 | INFO | } | 20260506 02:02:05.465 | INFO | Perf report for the last 5.14581 seconds: | 20260506 02:02:05.465 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:02:05.465 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:05.465 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:05.465 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:05.465 | INFO | ANALYSIS : 61.4%, 3.0, 96.0ms, 0 | 20260506 02:02:05.465 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:05.465 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:05.465 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 02:02:05.465 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:05.465 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:05.465 | INFO | Uploaded 36GiB to GPU in 2.06417s (17.4404GiB/s) | 20260506 02:02:05.465 | INFO | Downloaded 34GiB to GPU in 2.01385s (16.8831GiB/s) | 20260506 02:02:05.594 | INFO | Starting a round of Baz IO | 20260506 02:02:08.711 | INFO | Populating RTMetrics report | 20260506 02:02:08.711 | INFO | RTMetrics report populated | 20260506 02:02:08.711 | INFO | Opening tmp RTMetrics json file | 20260506 02:02:08.713 | INFO | Serializing RTMetrics to json | 20260506 02:02:08.714 | INFO | Serialization done, swapping tmp file now... | 20260506 02:02:08.715 | INFO | RTMetrics done | 20260506 02:02:08.793 | INFO | Finished a round of Baz IO: Wrote 1.806 GiB in 3.199 seconds | 20260506 02:02:08.793 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.29% / 0.00% / 78.71% | 20260506 02:02:08.793 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0751376% / 0% / 7.54992e-05% | 20260506 02:02:08.793 | INFO | Baz IO overall: | 20260506 02:02:08.793 | INFO | BytesWritten : 4536477843612 | 20260506 02:02:08.793 | INFO | NumEvents : 2130837536036 | 20260506 02:02:10.600 | INFO | Analyzing chunk frames = [8073216,8073728) | 20260506 02:02:10.600 | INFO | MemInfo: { | 20260506 02:02:10.600 | INFO | "gpu" : " 17 GB", | 20260506 02:02:10.600 | INFO | "huge" : "", | 20260506 02:02:10.600 | INFO | "huge_shared" : " 27 GB", | 20260506 02:02:10.600 | INFO | "malloc" : " 14 GB", | 20260506 02:02:10.600 | INFO | "pinned" : " 29 GB" | 20260506 02:02:10.600 | INFO | } | 20260506 02:02:10.601 | INFO | Perf report for the last 5.13556 seconds: | 20260506 02:02:10.601 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:02:10.601 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:10.601 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:10.601 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:10.601 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260506 02:02:10.601 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:10.601 | INFO | RT_METRICS : 43.2%, 1.0, 22.6ms, 0 | 20260506 02:02:10.601 | INFO | PRE_HQ : 59.5%, 1.8, 54.1ms, 0 | 20260506 02:02:10.601 | INFO | BAZWRITER : 61.5%, 3.5, 110.6ms, 0 | 20260506 02:02:10.601 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:02:10.601 | INFO | Uploaded 36GiB to GPU in 2.09178s (17.2102GiB/s) | 20260506 02:02:10.601 | INFO | Downloaded 38GiB to GPU in 2.22173s (17.1038GiB/s) | 20260506 02:02:15.737 | INFO | Analyzing chunk frames = [8073728,8074240) | 20260506 02:02:15.737 | INFO | MemInfo: { | 20260506 02:02:15.737 | INFO | "gpu" : " 17 GB", | 20260506 02:02:15.737 | INFO | "huge" : "", | 20260506 02:02:15.737 | INFO | "huge_shared" : " 27 GB", | 20260506 02:02:15.737 | INFO | "malloc" : " 14 GB", | 20260506 02:02:15.737 | INFO | "pinned" : " 29 GB" | 20260506 02:02:15.737 | INFO | } | 20260506 02:02:15.737 | INFO | Perf report for the last 5.13669 seconds: | 20260506 02:02:15.737 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:02:15.737 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:15.737 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:15.737 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:15.737 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260506 02:02:15.737 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:15.737 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:15.737 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:02:15.737 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:15.737 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:15.737 | INFO | Uploaded 36GiB to GPU in 1.97211s (18.2546GiB/s) | 20260506 02:02:15.737 | INFO | Downloaded 34GiB to GPU in 1.98059s (17.1666GiB/s) | 20260506 02:02:20.881 | INFO | Analyzing chunk frames = [8074240,8074752) | 20260506 02:02:20.881 | INFO | MemInfo: { | 20260506 02:02:20.881 | INFO | "gpu" : " 17 GB", | 20260506 02:02:20.881 | INFO | "huge" : "", | 20260506 02:02:20.881 | INFO | "huge_shared" : " 27 GB", | 20260506 02:02:20.881 | INFO | "malloc" : " 14 GB", | 20260506 02:02:20.881 | INFO | "pinned" : " 29 GB" | 20260506 02:02:20.881 | INFO | } | 20260506 02:02:20.881 | INFO | Perf report for the last 5.14351 seconds: | 20260506 02:02:20.881 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:02:20.881 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:20.881 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:20.881 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:20.881 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 02:02:20.881 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:20.881 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:20.881 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:02:20.881 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:20.881 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:20.881 | INFO | Uploaded 36GiB to GPU in 2.07888s (17.317GiB/s) | 20260506 02:02:20.881 | INFO | Downloaded 34GiB to GPU in 2.02284s (16.808GiB/s) | 20260506 02:02:26.019 | INFO | Analyzing chunk frames = [8074752,8075264) | 20260506 02:02:26.019 | INFO | MemInfo: { | 20260506 02:02:26.019 | INFO | "gpu" : " 17 GB", | 20260506 02:02:26.019 | INFO | "huge" : "", | 20260506 02:02:26.019 | INFO | "huge_shared" : " 27 GB", | 20260506 02:02:26.019 | INFO | "malloc" : " 14 GB", | 20260506 02:02:26.019 | INFO | "pinned" : " 29 GB" | 20260506 02:02:26.019 | INFO | } | 20260506 02:02:26.019 | INFO | Perf report for the last 5.13861 seconds: | 20260506 02:02:26.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:02:26.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:26.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:26.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:26.020 | INFO | ANALYSIS : 61.2%, 3.0, 95.0ms, 0 | 20260506 02:02:26.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:26.020 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:26.020 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 02:02:26.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:26.020 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:26.020 | INFO | Uploaded 36GiB to GPU in 2.1121s (17.0446GiB/s) | 20260506 02:02:26.020 | INFO | Downloaded 34GiB to GPU in 2.02194s (16.8155GiB/s) | 20260506 02:02:31.158 | INFO | Analyzing chunk frames = [8075264,8075776) | 20260506 02:02:31.158 | INFO | MemInfo: { | 20260506 02:02:31.158 | INFO | "gpu" : " 17 GB", | 20260506 02:02:31.158 | INFO | "huge" : "", | 20260506 02:02:31.158 | INFO | "huge_shared" : " 27 GB", | 20260506 02:02:31.158 | INFO | "malloc" : " 14 GB", | 20260506 02:02:31.158 | INFO | "pinned" : " 29 GB" | 20260506 02:02:31.158 | INFO | } | 20260506 02:02:31.158 | INFO | Perf report for the last 5.13865 seconds: | 20260506 02:02:31.158 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:02:31.158 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:31.158 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:31.158 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:31.158 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260506 02:02:31.158 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:31.158 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:31.158 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:02:31.158 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:31.158 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:31.158 | INFO | Uploaded 36GiB to GPU in 2.09052s (17.2206GiB/s) | 20260506 02:02:31.158 | INFO | Downloaded 34GiB to GPU in 2.03026s (16.7466GiB/s) | 20260506 02:02:36.296 | INFO | Analyzing chunk frames = [8075776,8076288) | 20260506 02:02:36.296 | INFO | MemInfo: { | 20260506 02:02:36.296 | INFO | "gpu" : " 17 GB", | 20260506 02:02:36.296 | INFO | "huge" : "", | 20260506 02:02:36.296 | INFO | "huge_shared" : " 27 GB", | 20260506 02:02:36.296 | INFO | "malloc" : " 14 GB", | 20260506 02:02:36.296 | INFO | "pinned" : " 29 GB" | 20260506 02:02:36.296 | INFO | } | 20260506 02:02:36.296 | INFO | Perf report for the last 5.1381 seconds: | 20260506 02:02:36.296 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:02:36.296 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:36.296 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:36.296 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:36.296 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 02:02:36.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:36.296 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:36.296 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:02:36.296 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:36.296 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:36.297 | INFO | Uploaded 36GiB to GPU in 2.09019s (17.2233GiB/s) | 20260506 02:02:36.297 | INFO | Downloaded 34GiB to GPU in 2.03897s (16.6751GiB/s) | 20260506 02:02:41.435 | INFO | Analyzing chunk frames = [8076288,8076800) | 20260506 02:02:41.435 | INFO | MemInfo: { | 20260506 02:02:41.435 | INFO | "gpu" : " 17 GB", | 20260506 02:02:41.435 | INFO | "huge" : "", | 20260506 02:02:41.435 | INFO | "huge_shared" : " 27 GB", | 20260506 02:02:41.435 | INFO | "malloc" : " 14 GB", | 20260506 02:02:41.435 | INFO | "pinned" : " 29 GB" | 20260506 02:02:41.435 | INFO | } | 20260506 02:02:41.435 | INFO | Perf report for the last 5.13866 seconds: | 20260506 02:02:41.435 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:02:41.435 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:41.435 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:41.435 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:41.435 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260506 02:02:41.435 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:41.435 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:41.435 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 02:02:41.435 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:41.435 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:41.435 | INFO | Uploaded 36GiB to GPU in 2.11627s (17.0111GiB/s) | 20260506 02:02:41.435 | INFO | Downloaded 34GiB to GPU in 2.11681s (16.0619GiB/s) | 20260506 02:02:46.576 | INFO | Analyzing chunk frames = [8076800,8077312) | 20260506 02:02:46.576 | INFO | MemInfo: { | 20260506 02:02:46.576 | INFO | "gpu" : " 17 GB", | 20260506 02:02:46.576 | INFO | "huge" : "", | 20260506 02:02:46.576 | INFO | "huge_shared" : " 27 GB", | 20260506 02:02:46.576 | INFO | "malloc" : " 14 GB", | 20260506 02:02:46.576 | INFO | "pinned" : " 29 GB" | 20260506 02:02:46.576 | INFO | } | 20260506 02:02:46.576 | INFO | Perf report for the last 5.14104 seconds: | 20260506 02:02:46.576 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:02:46.576 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:46.576 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:46.576 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:46.576 | INFO | ANALYSIS : 61.5%, 3.0, 96.0ms, 0 | 20260506 02:02:46.576 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:46.576 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:46.576 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 02:02:46.576 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:46.576 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:46.576 | INFO | Uploaded 36GiB to GPU in 2.0543s (17.5242GiB/s) | 20260506 02:02:46.576 | INFO | Downloaded 34GiB to GPU in 2.02609s (16.7811GiB/s) | 20260506 02:02:46.576 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 02:02:46.576 | INFO | Total mem usage: 15.3 GiB | 20260506 02:02:46.576 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 02:02:46.576 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.21%) | 20260506 02:02:46.576 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 02:02:46.576 | INFO | High Water Breakdown: | 20260506 02:02:46.576 | INFO | 60.9% : 8.54 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 02:02:46.576 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 02:02:46.576 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 02:02:46.576 | INFO | ------------------------------------------ | 20260506 02:02:46.576 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:02:46.576 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 02:02:46.576 | INFO | Total mem usage: 26.5 GiB | 20260506 02:02:46.576 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 02:02:46.577 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 02:02:46.577 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 02:02:46.577 | INFO | High Water Breakdown: | 20260506 02:02:46.577 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:02:46.577 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:02:46.577 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:02:46.577 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:02:46.577 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:02:46.577 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:02:46.577 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 02:02:46.577 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 02:02:46.577 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:02:46.577 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 02:02:46.577 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 02:02:46.577 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 02:02:46.577 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 02:02:46.577 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 02:02:46.577 | INFO | ------------------------------------------ | 20260506 02:02:46.577 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:02:46.577 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 02:02:46.577 | INFO | Total mem usage: 25.6 GiB | 20260506 02:02:46.577 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 02:02:46.577 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 02:02:46.577 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 02:02:46.577 | INFO | High Water Breakdown: | 20260506 02:02:46.577 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 02:02:46.577 | INFO | ------------------------------------------ | 20260506 02:02:46.577 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:02:46.577 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 02:02:46.577 | INFO | Total mem usage: 15.7 GiB | 20260506 02:02:46.577 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 02:02:46.577 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 02:02:46.577 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 02:02:46.577 | INFO | High Water Breakdown: | 20260506 02:02:46.577 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 02:02:46.577 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 02:02:46.577 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 02:02:46.577 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 02:02:46.577 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 02:02:46.577 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 02:02:46.577 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 02:02:46.577 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 02:02:46.577 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 02:02:46.577 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:02:46.577 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 02:02:46.577 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:02:46.577 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 02:02:46.577 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 02:02:46.577 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 02:02:46.577 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 02:02:46.577 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:02:46.577 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 02:02:46.577 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:02:46.577 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:02:46.577 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:02:46.577 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 02:02:46.577 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 02:02:46.577 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:02:46.577 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 02:02:46.577 | INFO | ------------------------------------------ | 20260506 02:02:46.577 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:02:46.577 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 02:02:46.577 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:02:46.577 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:02:46.577 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.38e+01, 1.33e+01 / 6.77e-01 | 20260506 02:02:46.577 | INFO | PerfReport: 0.29%, AggregateStats -- 9.17e-02 / 1.04e-01, 1.03e-02 / 2.89e-03 | 20260506 02:02:46.577 | INFO | PerfReport: 11.85%, Baseline -- 3.71e+00 / 4.49e+00, 5.82e-01 / 2.20e-01 | 20260506 02:02:46.577 | INFO | PerfReport: 3.30%, Histogram -- 1.03e+00 / 1.52e+00, 1.95e-01 / 8.34e-02 | 20260506 02:02:46.577 | INFO | PerfReport: 15.95%, DME -- 4.00e+01 / 4.39e+01, 3.79e+00 / 9.60e-01 | 20260506 02:02:46.577 | INFO | PerfReport: 57.53%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.57e+00 / 1.05e+00 | 20260506 02:02:46.577 | INFO | PerfReport: 7.00%, PulseAccumulating -- 2.19e+00 / 2.73e+00, 4.13e-01 / 2.06e-01 | 20260506 02:02:46.577 | INFO | PerfReport: 3.02%, Metrics -- 9.45e-01 / 1.23e+00, 2.53e-01 / 9.64e-02 | 20260506 02:02:46.577 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.27e-01 / 3.76e-01, 5.47e-02 / 2.53e-02 | 20260506 02:02:46.577 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 02:02:46.577 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:02:46.577 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:02:46.577 | INFO | PerfReport: Full Execution -- 9.78e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260506 02:02:46.577 | INFO | PerfReport: 22.23%, Upload -- 2.17e+01 / 0.00e+00, 7.28e+00 / 0.00e+00 | 20260506 02:02:46.577 | INFO | PerfReport: 21.93%, Download -- 2.14e+01 / 0.00e+00, 7.61e+00 / 0.00e+00 | 20260506 02:02:46.577 | INFO | PerfReport: 32.04%, Compute -- 3.13e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 02:02:46.577 | INFO | PerfReport: 2.03%, LockUpload -- 1.98e+00 / 0.00e+00, 7.89e+00 / 0.00e+00 | 20260506 02:02:46.578 | INFO | PerfReport: 1.31%, LockDownload -- 1.28e+00 / 0.00e+00, 2.95e+00 / 0.00e+00 | 20260506 02:02:46.578 | INFO | PerfReport: 20.45%, LockCompute -- 2.00e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260506 02:02:46.578 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.78e-02 / 0.00e+00 | 20260506 02:02:46.713 | INFO | Starting a round of Baz IO | 20260506 02:02:49.826 | INFO | Populating RTMetrics report | 20260506 02:02:49.826 | INFO | RTMetrics report populated | 20260506 02:02:49.826 | INFO | Opening tmp RTMetrics json file | 20260506 02:02:49.827 | INFO | Serializing RTMetrics to json | 20260506 02:02:49.829 | INFO | Serialization done, swapping tmp file now... | 20260506 02:02:49.829 | INFO | RTMetrics done | 20260506 02:02:49.909 | INFO | Finished a round of Baz IO: Wrote 1.806 GiB in 3.196 seconds | 20260506 02:02:49.909 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.29% / 0.00% / 78.71% | 20260506 02:02:49.909 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0745734% / 7.94729e-06% / 7.54992e-05% | 20260506 02:02:49.909 | INFO | Baz IO overall: | 20260506 02:02:49.909 | INFO | BytesWritten : 4538417310544 | 20260506 02:02:49.909 | INFO | NumEvents : 2131720341618 | 20260506 02:02:51.718 | INFO | Analyzing chunk frames = [8077312,8077824) | 20260506 02:02:51.718 | INFO | MemInfo: { | 20260506 02:02:51.718 | INFO | "gpu" : " 17 GB", | 20260506 02:02:51.718 | INFO | "huge" : "", | 20260506 02:02:51.718 | INFO | "huge_shared" : " 27 GB", | 20260506 02:02:51.718 | INFO | "malloc" : " 14 GB", | 20260506 02:02:51.718 | INFO | "pinned" : " 29 GB" | 20260506 02:02:51.718 | INFO | } | 20260506 02:02:51.718 | INFO | Perf report for the last 5.14253 seconds: | 20260506 02:02:51.719 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:02:51.719 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:51.719 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:51.719 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:51.719 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260506 02:02:51.719 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:51.719 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260506 02:02:51.719 | INFO | PRE_HQ : 59.2%, 1.7, 53.8ms, 0 | 20260506 02:02:51.719 | INFO | BAZWRITER : 61.5%, 3.5, 111.4ms, 0 | 20260506 02:02:51.719 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:02:51.719 | INFO | Uploaded 36GiB to GPU in 2.11263s (17.0404GiB/s) | 20260506 02:02:51.719 | INFO | Downloaded 38GiB to GPU in 2.22033s (17.1146GiB/s) | 20260506 02:02:56.855 | INFO | Analyzing chunk frames = [8077824,8078336) | 20260506 02:02:56.855 | INFO | MemInfo: { | 20260506 02:02:56.855 | INFO | "gpu" : " 17 GB", | 20260506 02:02:56.855 | INFO | "huge" : "", | 20260506 02:02:56.855 | INFO | "huge_shared" : " 27 GB", | 20260506 02:02:56.855 | INFO | "malloc" : " 14 GB", | 20260506 02:02:56.855 | INFO | "pinned" : " 29 GB" | 20260506 02:02:56.855 | INFO | } | 20260506 02:02:56.855 | INFO | Perf report for the last 5.13646 seconds: | 20260506 02:02:56.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:02:56.855 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:56.855 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:56.855 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:56.855 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260506 02:02:56.855 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:56.855 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:56.855 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:02:56.855 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:56.855 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:02:56.856 | INFO | Uploaded 36GiB to GPU in 1.96607s (18.3106GiB/s) | 20260506 02:02:56.856 | INFO | Downloaded 34GiB to GPU in 1.97397s (17.2242GiB/s) | 20260506 02:03:01.987 | INFO | Analyzing chunk frames = [8078336,8078848) | 20260506 02:03:01.987 | INFO | MemInfo: { | 20260506 02:03:01.987 | INFO | "gpu" : " 17 GB", | 20260506 02:03:01.987 | INFO | "huge" : "", | 20260506 02:03:01.987 | INFO | "huge_shared" : " 27 GB", | 20260506 02:03:01.987 | INFO | "malloc" : " 14 GB", | 20260506 02:03:01.987 | INFO | "pinned" : " 29 GB" | 20260506 02:03:01.987 | INFO | } | 20260506 02:03:01.987 | INFO | Perf report for the last 5.13257 seconds: | 20260506 02:03:01.987 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:03:01.987 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:01.987 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:01.987 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:01.987 | INFO | ANALYSIS : 61.3%, 3.0, 95.2ms, 0 | 20260506 02:03:01.987 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:01.988 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:01.988 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 02:03:01.988 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:01.988 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:01.988 | INFO | Uploaded 36GiB to GPU in 2.07235s (17.3716GiB/s) | 20260506 02:03:01.988 | INFO | Downloaded 34GiB to GPU in 2.02424s (16.7964GiB/s) | 20260506 02:03:07.127 | INFO | Analyzing chunk frames = [8078848,8079360) | 20260506 02:03:07.127 | INFO | MemInfo: { | 20260506 02:03:07.127 | INFO | "gpu" : " 17 GB", | 20260506 02:03:07.127 | INFO | "huge" : "", | 20260506 02:03:07.127 | INFO | "huge_shared" : " 27 GB", | 20260506 02:03:07.127 | INFO | "malloc" : " 14 GB", | 20260506 02:03:07.127 | INFO | "pinned" : " 29 GB" | 20260506 02:03:07.127 | INFO | } | 20260506 02:03:07.127 | INFO | Perf report for the last 5.13971 seconds: | 20260506 02:03:07.127 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:03:07.127 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:07.127 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:07.127 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:07.127 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260506 02:03:07.127 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:07.128 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:07.128 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 02:03:07.128 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:07.128 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:07.128 | INFO | Uploaded 36GiB to GPU in 2.1367s (16.8484GiB/s) | 20260506 02:03:07.128 | INFO | Downloaded 34GiB to GPU in 2.02832s (16.7626GiB/s) | 20260506 02:03:12.277 | INFO | Analyzing chunk frames = [8079360,8079872) | 20260506 02:03:12.277 | INFO | MemInfo: { | 20260506 02:03:12.277 | INFO | "gpu" : " 17 GB", | 20260506 02:03:12.277 | INFO | "huge" : "", | 20260506 02:03:12.277 | INFO | "huge_shared" : " 27 GB", | 20260506 02:03:12.277 | INFO | "malloc" : " 14 GB", | 20260506 02:03:12.277 | INFO | "pinned" : " 29 GB" | 20260506 02:03:12.277 | INFO | } | 20260506 02:03:12.277 | INFO | Perf report for the last 5.15015 seconds: | 20260506 02:03:12.278 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:03:12.278 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:12.278 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:12.278 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:12.278 | INFO | ANALYSIS : 60.8%, 3.0, 95.1ms, 0 | 20260506 02:03:12.278 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:12.278 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:12.278 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 02:03:12.278 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:12.278 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:12.278 | INFO | Uploaded 36GiB to GPU in 2.08472s (17.2685GiB/s) | 20260506 02:03:12.278 | INFO | Downloaded 34GiB to GPU in 2.04884s (16.5948GiB/s) | 20260506 02:03:17.411 | INFO | Analyzing chunk frames = [8079872,8080384) | 20260506 02:03:17.411 | INFO | MemInfo: { | 20260506 02:03:17.411 | INFO | "gpu" : " 17 GB", | 20260506 02:03:17.411 | INFO | "huge" : "", | 20260506 02:03:17.411 | INFO | "huge_shared" : " 27 GB", | 20260506 02:03:17.411 | INFO | "malloc" : " 14 GB", | 20260506 02:03:17.411 | INFO | "pinned" : " 29 GB" | 20260506 02:03:17.411 | INFO | } | 20260506 02:03:17.411 | INFO | Perf report for the last 5.13369 seconds: | 20260506 02:03:17.411 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:03:17.411 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:17.411 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:17.411 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:17.411 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260506 02:03:17.411 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:17.411 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:17.411 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 02:03:17.411 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:17.411 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:17.412 | INFO | Uploaded 36GiB to GPU in 2.04302s (17.621GiB/s) | 20260506 02:03:17.412 | INFO | Downloaded 34GiB to GPU in 2.02518s (16.7887GiB/s) | 20260506 02:03:22.550 | INFO | Analyzing chunk frames = [8080384,8080896) | 20260506 02:03:22.550 | INFO | MemInfo: { | 20260506 02:03:22.550 | INFO | "gpu" : " 17 GB", | 20260506 02:03:22.550 | INFO | "huge" : "", | 20260506 02:03:22.550 | INFO | "huge_shared" : " 27 GB", | 20260506 02:03:22.550 | INFO | "malloc" : " 14 GB", | 20260506 02:03:22.550 | INFO | "pinned" : " 29 GB" | 20260506 02:03:22.550 | INFO | } | 20260506 02:03:22.550 | INFO | Perf report for the last 5.13911 seconds: | 20260506 02:03:22.550 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:03:22.550 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:22.550 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:22.550 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:22.550 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260506 02:03:22.550 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:22.550 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:22.551 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 02:03:22.551 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:22.551 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:22.551 | INFO | Uploaded 36GiB to GPU in 2.03212s (17.7155GiB/s) | 20260506 02:03:22.551 | INFO | Downloaded 34GiB to GPU in 2.04733s (16.607GiB/s) | 20260506 02:03:27.691 | INFO | Analyzing chunk frames = [8080896,8081408) | 20260506 02:03:27.691 | INFO | MemInfo: { | 20260506 02:03:27.691 | INFO | "gpu" : " 17 GB", | 20260506 02:03:27.691 | INFO | "huge" : "", | 20260506 02:03:27.691 | INFO | "huge_shared" : " 27 GB", | 20260506 02:03:27.691 | INFO | "malloc" : " 14 GB", | 20260506 02:03:27.691 | INFO | "pinned" : " 29 GB" | 20260506 02:03:27.691 | INFO | } | 20260506 02:03:27.691 | INFO | Perf report for the last 5.14112 seconds: | 20260506 02:03:27.691 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:03:27.692 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:27.692 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:27.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:27.692 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260506 02:03:27.692 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:27.692 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:27.692 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 02:03:27.692 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:27.692 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:27.692 | INFO | Uploaded 36GiB to GPU in 2.13529s (16.8595GiB/s) | 20260506 02:03:27.692 | INFO | Downloaded 34GiB to GPU in 2.03665s (16.6941GiB/s) | 20260506 02:03:27.817 | INFO | Starting a round of Baz IO | 20260506 02:03:30.951 | INFO | Populating RTMetrics report | 20260506 02:03:30.951 | INFO | RTMetrics report populated | 20260506 02:03:30.951 | INFO | Opening tmp RTMetrics json file | 20260506 02:03:30.952 | INFO | Serializing RTMetrics to json | 20260506 02:03:30.954 | INFO | Serialization done, swapping tmp file now... | 20260506 02:03:30.954 | INFO | RTMetrics done | 20260506 02:03:31.030 | INFO | Finished a round of Baz IO: Wrote 1.806 GiB in 3.213 seconds | 20260506 02:03:31.031 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.29% / 0.00% / 78.71% | 20260506 02:03:31.031 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.074176% / 3.97364e-06% / 8.34465e-05% | 20260506 02:03:31.031 | INFO | Baz IO overall: | 20260506 02:03:31.031 | INFO | BytesWritten : 4540356909851 | 20260506 02:03:31.031 | INFO | NumEvents : 2132603475358 | 20260506 02:03:32.829 | INFO | Analyzing chunk frames = [8081408,8081920) | 20260506 02:03:32.829 | INFO | MemInfo: { | 20260506 02:03:32.829 | INFO | "gpu" : " 17 GB", | 20260506 02:03:32.829 | INFO | "huge" : "", | 20260506 02:03:32.829 | INFO | "huge_shared" : " 27 GB", | 20260506 02:03:32.829 | INFO | "malloc" : " 14 GB", | 20260506 02:03:32.829 | INFO | "pinned" : " 29 GB" | 20260506 02:03:32.829 | INFO | } | 20260506 02:03:32.829 | INFO | Perf report for the last 5.13792 seconds: | 20260506 02:03:32.829 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:03:32.829 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:32.829 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:32.829 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:32.829 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260506 02:03:32.829 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:32.829 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260506 02:03:32.829 | INFO | PRE_HQ : 59.6%, 1.7, 54.0ms, 0 | 20260506 02:03:32.829 | INFO | BAZWRITER : 61.8%, 3.5, 111.2ms, 0 | 20260506 02:03:32.829 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:03:32.829 | INFO | Uploaded 36GiB to GPU in 2.14615s (16.7742GiB/s) | 20260506 02:03:32.829 | INFO | Downloaded 38GiB to GPU in 2.23214s (17.024GiB/s) | 20260506 02:03:37.968 | INFO | Analyzing chunk frames = [8081920,8082432) | 20260506 02:03:37.968 | INFO | MemInfo: { | 20260506 02:03:37.968 | INFO | "gpu" : " 17 GB", | 20260506 02:03:37.968 | INFO | "huge" : "", | 20260506 02:03:37.968 | INFO | "huge_shared" : " 27 GB", | 20260506 02:03:37.968 | INFO | "malloc" : " 14 GB", | 20260506 02:03:37.968 | INFO | "pinned" : " 29 GB" | 20260506 02:03:37.968 | INFO | } | 20260506 02:03:37.968 | INFO | Perf report for the last 5.13896 seconds: | 20260506 02:03:37.968 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:03:37.968 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:37.968 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:37.968 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:37.968 | INFO | ANALYSIS : 60.7%, 3.0, 94.8ms, 0 | 20260506 02:03:37.968 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:37.968 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:37.969 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:03:37.969 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:37.969 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:37.969 | INFO | Uploaded 36GiB to GPU in 1.93933s (18.5631GiB/s) | 20260506 02:03:37.969 | INFO | Downloaded 34GiB to GPU in 1.97911s (17.1794GiB/s) | 20260506 02:03:43.112 | INFO | Analyzing chunk frames = [8082432,8082944) | 20260506 02:03:43.112 | INFO | MemInfo: { | 20260506 02:03:43.112 | INFO | "gpu" : " 17 GB", | 20260506 02:03:43.112 | INFO | "huge" : "", | 20260506 02:03:43.112 | INFO | "huge_shared" : " 27 GB", | 20260506 02:03:43.112 | INFO | "malloc" : " 14 GB", | 20260506 02:03:43.112 | INFO | "pinned" : " 29 GB" | 20260506 02:03:43.112 | INFO | } | 20260506 02:03:43.112 | INFO | Perf report for the last 5.14387 seconds: | 20260506 02:03:43.112 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:03:43.112 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:43.112 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:43.112 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:43.112 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 02:03:43.112 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:43.112 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:43.112 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:03:43.112 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:43.112 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:43.113 | INFO | Uploaded 36GiB to GPU in 2.09059s (17.22GiB/s) | 20260506 02:03:43.113 | INFO | Downloaded 34GiB to GPU in 2.02083s (16.8248GiB/s) | 20260506 02:03:48.251 | INFO | Analyzing chunk frames = [8082944,8083456) | 20260506 02:03:48.251 | INFO | MemInfo: { | 20260506 02:03:48.251 | INFO | "gpu" : " 17 GB", | 20260506 02:03:48.251 | INFO | "huge" : "", | 20260506 02:03:48.251 | INFO | "huge_shared" : " 27 GB", | 20260506 02:03:48.251 | INFO | "malloc" : " 14 GB", | 20260506 02:03:48.251 | INFO | "pinned" : " 29 GB" | 20260506 02:03:48.251 | INFO | } | 20260506 02:03:48.251 | INFO | Perf report for the last 5.13914 seconds: | 20260506 02:03:48.251 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:03:48.251 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:48.251 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:48.251 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:48.251 | INFO | ANALYSIS : 61.1%, 3.0, 94.9ms, 0 | 20260506 02:03:48.251 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:48.252 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:48.252 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 02:03:48.252 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:48.252 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:48.252 | INFO | Uploaded 36GiB to GPU in 2.08391s (17.2752GiB/s) | 20260506 02:03:48.252 | INFO | Downloaded 34GiB to GPU in 2.0139s (16.8826GiB/s) | 20260506 02:03:53.387 | INFO | Analyzing chunk frames = [8083456,8083968) | 20260506 02:03:53.387 | INFO | MemInfo: { | 20260506 02:03:53.387 | INFO | "gpu" : " 17 GB", | 20260506 02:03:53.387 | INFO | "huge" : "", | 20260506 02:03:53.387 | INFO | "huge_shared" : " 27 GB", | 20260506 02:03:53.387 | INFO | "malloc" : " 14 GB", | 20260506 02:03:53.387 | INFO | "pinned" : " 29 GB" | 20260506 02:03:53.387 | INFO | } | 20260506 02:03:53.387 | INFO | Perf report for the last 5.13596 seconds: | 20260506 02:03:53.387 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:03:53.387 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:53.387 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:53.387 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:53.387 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260506 02:03:53.387 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:53.388 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:53.388 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:03:53.388 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:53.388 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:53.388 | INFO | Uploaded 36GiB to GPU in 2.07061s (17.3862GiB/s) | 20260506 02:03:53.388 | INFO | Downloaded 34GiB to GPU in 2.02328s (16.8044GiB/s) | 20260506 02:03:58.527 | INFO | Analyzing chunk frames = [8083968,8084480) | 20260506 02:03:58.527 | INFO | MemInfo: { | 20260506 02:03:58.527 | INFO | "gpu" : " 17 GB", | 20260506 02:03:58.527 | INFO | "huge" : "", | 20260506 02:03:58.527 | INFO | "huge_shared" : " 27 GB", | 20260506 02:03:58.527 | INFO | "malloc" : " 14 GB", | 20260506 02:03:58.527 | INFO | "pinned" : " 29 GB" | 20260506 02:03:58.527 | INFO | } | 20260506 02:03:58.527 | INFO | Perf report for the last 5.13977 seconds: | 20260506 02:03:58.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:03:58.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:58.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:58.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:58.527 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 02:03:58.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:58.527 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:58.527 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:03:58.527 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:58.527 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:03:58.527 | INFO | Uploaded 36GiB to GPU in 2.09288s (17.2012GiB/s) | 20260506 02:03:58.527 | INFO | Downloaded 34GiB to GPU in 2.03318s (16.7225GiB/s) | 20260506 02:04:03.668 | INFO | Analyzing chunk frames = [8084480,8084992) | 20260506 02:04:03.668 | INFO | MemInfo: { | 20260506 02:04:03.668 | INFO | "gpu" : " 17 GB", | 20260506 02:04:03.668 | INFO | "huge" : "", | 20260506 02:04:03.668 | INFO | "huge_shared" : " 27 GB", | 20260506 02:04:03.668 | INFO | "malloc" : " 14 GB", | 20260506 02:04:03.668 | INFO | "pinned" : " 29 GB" | 20260506 02:04:03.668 | INFO | } | 20260506 02:04:03.668 | INFO | Perf report for the last 5.14131 seconds: | 20260506 02:04:03.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:04:03.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:03.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:03.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:03.668 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 02:04:03.668 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:03.668 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:03.668 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 02:04:03.668 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:03.669 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:03.669 | INFO | Uploaded 36GiB to GPU in 2.07096s (17.3833GiB/s) | 20260506 02:04:03.669 | INFO | Downloaded 34GiB to GPU in 2.08177s (16.3323GiB/s) | 20260506 02:04:08.802 | INFO | Analyzing chunk frames = [8084992,8085504) | 20260506 02:04:08.802 | INFO | MemInfo: { | 20260506 02:04:08.802 | INFO | "gpu" : " 17 GB", | 20260506 02:04:08.802 | INFO | "huge" : "", | 20260506 02:04:08.802 | INFO | "huge_shared" : " 27 GB", | 20260506 02:04:08.802 | INFO | "malloc" : " 14 GB", | 20260506 02:04:08.802 | INFO | "pinned" : " 29 GB" | 20260506 02:04:08.802 | INFO | } | 20260506 02:04:08.802 | INFO | Perf report for the last 5.13368 seconds: | 20260506 02:04:08.802 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:04:08.802 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:08.802 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:08.802 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:08.802 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 02:04:08.802 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:08.802 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:08.802 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 02:04:08.802 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:08.802 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:08.802 | INFO | Uploaded 36GiB to GPU in 2.04068s (17.6411GiB/s) | 20260506 02:04:08.802 | INFO | Downloaded 34GiB to GPU in 2.02001s (16.8316GiB/s) | 20260506 02:04:08.803 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 02:04:08.803 | INFO | Total mem usage: 15.3 GiB | 20260506 02:04:08.803 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 02:04:08.803 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.17%) | 20260506 02:04:08.803 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 02:04:08.803 | INFO | High Water Breakdown: | 20260506 02:04:08.803 | INFO | 60.9% : 8.55 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 02:04:08.803 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 02:04:08.803 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 02:04:08.803 | INFO | ------------------------------------------ | 20260506 02:04:08.803 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:04:08.803 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 02:04:08.803 | INFO | Total mem usage: 26.5 GiB | 20260506 02:04:08.803 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 02:04:08.803 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 02:04:08.803 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 02:04:08.803 | INFO | High Water Breakdown: | 20260506 02:04:08.803 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:04:08.803 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:04:08.803 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:04:08.803 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:04:08.803 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:04:08.803 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:04:08.803 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 02:04:08.803 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 02:04:08.803 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:04:08.803 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 02:04:08.803 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 02:04:08.803 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 02:04:08.803 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 02:04:08.803 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 02:04:08.803 | INFO | ------------------------------------------ | 20260506 02:04:08.803 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:04:08.803 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 02:04:08.803 | INFO | Total mem usage: 25.6 GiB | 20260506 02:04:08.803 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 02:04:08.803 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 02:04:08.803 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 02:04:08.803 | INFO | High Water Breakdown: | 20260506 02:04:08.803 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 02:04:08.803 | INFO | ------------------------------------------ | 20260506 02:04:08.803 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:04:08.803 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 02:04:08.803 | INFO | Total mem usage: 15.7 GiB | 20260506 02:04:08.803 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 02:04:08.803 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 02:04:08.803 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 02:04:08.803 | INFO | High Water Breakdown: | 20260506 02:04:08.803 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 02:04:08.803 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 02:04:08.803 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 02:04:08.803 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 02:04:08.803 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 02:04:08.803 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 02:04:08.803 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 02:04:08.803 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 02:04:08.804 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 02:04:08.804 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:04:08.804 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 02:04:08.804 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:04:08.804 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 02:04:08.804 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 02:04:08.804 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 02:04:08.804 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 02:04:08.804 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:04:08.804 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 02:04:08.804 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:04:08.804 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:04:08.804 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:04:08.804 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 02:04:08.804 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 02:04:08.804 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:04:08.804 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 02:04:08.804 | INFO | ------------------------------------------ | 20260506 02:04:08.804 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:04:08.804 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 02:04:08.804 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:04:08.804 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:04:08.804 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.37e+01, 1.36e+01 / 5.65e-01 | 20260506 02:04:08.804 | INFO | PerfReport: 0.29%, AggregateStats -- 9.17e-02 / 1.04e-01, 1.06e-02 / 2.52e-03 | 20260506 02:04:08.804 | INFO | PerfReport: 11.86%, Baseline -- 3.71e+00 / 4.48e+00, 5.86e-01 / 2.67e-01 | 20260506 02:04:08.804 | INFO | PerfReport: 3.30%, Histogram -- 1.03e+00 / 1.53e+00, 1.96e-01 / 8.10e-02 | 20260506 02:04:08.804 | INFO | PerfReport: 16.02%, DME -- 4.01e+01 / 4.42e+01, 3.76e+00 / 7.68e-01 | 20260506 02:04:08.804 | INFO | PerfReport: 57.43%, FrameLabeling -- 1.80e+01 / 1.91e+01, 1.56e+00 / 8.02e-01 | 20260506 02:04:08.804 | INFO | PerfReport: 7.04%, PulseAccumulating -- 2.20e+00 / 2.76e+00, 4.35e-01 / 1.92e-01 | 20260506 02:04:08.804 | INFO | PerfReport: 3.02%, Metrics -- 9.45e-01 / 1.24e+00, 2.55e-01 / 9.30e-02 | 20260506 02:04:08.804 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.27e-01 / 3.85e-01, 5.51e-02 / 3.28e-02 | 20260506 02:04:08.804 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 02:04:08.804 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:04:08.804 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:04:08.804 | INFO | PerfReport: Full Execution -- 9.77e+01 / -nan, 2.41e+01 / 0.00e+00 | 20260506 02:04:08.804 | INFO | PerfReport: 22.07%, Upload -- 2.16e+01 / 0.00e+00, 7.16e+00 / 0.00e+00 | 20260506 02:04:08.804 | INFO | PerfReport: 21.86%, Download -- 2.14e+01 / 0.00e+00, 7.56e+00 / 0.00e+00 | 20260506 02:04:08.804 | INFO | PerfReport: 32.06%, Compute -- 3.13e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 02:04:08.804 | INFO | PerfReport: 2.05%, LockUpload -- 2.01e+00 / 0.00e+00, 7.86e+00 / 0.00e+00 | 20260506 02:04:08.804 | INFO | PerfReport: 1.28%, LockDownload -- 1.25e+00 / 0.00e+00, 2.96e+00 / 0.00e+00 | 20260506 02:04:08.804 | INFO | PerfReport: 20.67%, LockCompute -- 2.02e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260506 02:04:08.804 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.61e-02 / 0.00e+00 | 20260506 02:04:08.922 | INFO | Starting a round of Baz IO | 20260506 02:04:12.070 | INFO | Populating RTMetrics report | 20260506 02:04:12.071 | INFO | RTMetrics report populated | 20260506 02:04:12.071 | INFO | Opening tmp RTMetrics json file | 20260506 02:04:12.072 | INFO | Serializing RTMetrics to json | 20260506 02:04:12.073 | INFO | Serialization done, swapping tmp file now... | 20260506 02:04:12.073 | INFO | RTMetrics done | 20260506 02:04:12.151 | INFO | Finished a round of Baz IO: Wrote 1.804 GiB in 3.228 seconds | 20260506 02:04:12.151 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.29% / 0.00% / 78.71% | 20260506 02:04:12.151 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0739535% / 0% / 7.54992e-05% | 20260506 02:04:12.151 | INFO | Baz IO overall: | 20260506 02:04:12.151 | INFO | BytesWritten : 4542293676092 | 20260506 02:04:12.151 | INFO | NumEvents : 2133484523042 | 20260506 02:04:13.940 | INFO | Analyzing chunk frames = [8085504,8086016) | 20260506 02:04:13.940 | INFO | MemInfo: { | 20260506 02:04:13.940 | INFO | "gpu" : " 17 GB", | 20260506 02:04:13.940 | INFO | "huge" : "", | 20260506 02:04:13.940 | INFO | "huge_shared" : " 27 GB", | 20260506 02:04:13.940 | INFO | "malloc" : " 14 GB", | 20260506 02:04:13.940 | INFO | "pinned" : " 29 GB" | 20260506 02:04:13.941 | INFO | } | 20260506 02:04:13.941 | INFO | Perf report for the last 5.13851 seconds: | 20260506 02:04:13.941 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:04:13.941 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:13.941 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:13.941 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:13.941 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260506 02:04:13.941 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:13.941 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260506 02:04:13.941 | INFO | PRE_HQ : 60.0%, 1.7, 53.9ms, 0 | 20260506 02:04:13.941 | INFO | BAZWRITER : 61.9%, 3.5, 110.5ms, 0 | 20260506 02:04:13.941 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:04:13.941 | INFO | Uploaded 36GiB to GPU in 2.16834s (16.6025GiB/s) | 20260506 02:04:13.941 | INFO | Downloaded 38GiB to GPU in 2.23758s (16.9826GiB/s) | 20260506 02:04:19.087 | INFO | Analyzing chunk frames = [8086016,8086528) | 20260506 02:04:19.087 | INFO | MemInfo: { | 20260506 02:04:19.087 | INFO | "gpu" : " 17 GB", | 20260506 02:04:19.087 | INFO | "huge" : "", | 20260506 02:04:19.087 | INFO | "huge_shared" : " 27 GB", | 20260506 02:04:19.087 | INFO | "malloc" : " 14 GB", | 20260506 02:04:19.087 | INFO | "pinned" : " 29 GB" | 20260506 02:04:19.087 | INFO | } | 20260506 02:04:19.087 | INFO | Perf report for the last 5.14684 seconds: | 20260506 02:04:19.087 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:04:19.087 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:19.087 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:19.087 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:19.087 | INFO | ANALYSIS : 60.9%, 3.0, 95.2ms, 0 | 20260506 02:04:19.087 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:19.087 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:19.087 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 02:04:19.087 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:19.087 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:19.088 | INFO | Uploaded 36GiB to GPU in 1.97041s (18.2703GiB/s) | 20260506 02:04:19.088 | INFO | Downloaded 34GiB to GPU in 1.99448s (17.047GiB/s) | 20260506 02:04:24.234 | INFO | Analyzing chunk frames = [8086528,8087040) | 20260506 02:04:24.234 | INFO | MemInfo: { | 20260506 02:04:24.234 | INFO | "gpu" : " 17 GB", | 20260506 02:04:24.234 | INFO | "huge" : "", | 20260506 02:04:24.234 | INFO | "huge_shared" : " 27 GB", | 20260506 02:04:24.234 | INFO | "malloc" : " 14 GB", | 20260506 02:04:24.234 | INFO | "pinned" : " 29 GB" | 20260506 02:04:24.235 | INFO | } | 20260506 02:04:24.235 | INFO | Perf report for the last 5.14725 seconds: | 20260506 02:04:24.235 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:04:24.235 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:24.235 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:24.235 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:24.235 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 02:04:24.235 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:24.235 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:24.235 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:04:24.235 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:24.235 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:24.235 | INFO | Uploaded 36GiB to GPU in 2.08989s (17.2258GiB/s) | 20260506 02:04:24.235 | INFO | Downloaded 34GiB to GPU in 2.0373s (16.6887GiB/s) | 20260506 02:04:29.362 | INFO | Analyzing chunk frames = [8087040,8087552) | 20260506 02:04:29.362 | INFO | MemInfo: { | 20260506 02:04:29.362 | INFO | "gpu" : " 17 GB", | 20260506 02:04:29.362 | INFO | "huge" : "", | 20260506 02:04:29.362 | INFO | "huge_shared" : " 27 GB", | 20260506 02:04:29.362 | INFO | "malloc" : " 14 GB", | 20260506 02:04:29.362 | INFO | "pinned" : " 29 GB" | 20260506 02:04:29.362 | INFO | } | 20260506 02:04:29.362 | INFO | Perf report for the last 5.12754 seconds: | 20260506 02:04:29.362 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:04:29.362 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:29.362 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:29.362 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:29.362 | INFO | ANALYSIS : 61.1%, 3.0, 94.7ms, 0 | 20260506 02:04:29.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:29.362 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:29.362 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 02:04:29.362 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:29.362 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:29.362 | INFO | Uploaded 36GiB to GPU in 2.07512s (17.3484GiB/s) | 20260506 02:04:29.363 | INFO | Downloaded 34GiB to GPU in 2.00796s (16.9326GiB/s) | 20260506 02:04:34.500 | INFO | Analyzing chunk frames = [8087552,8088064) | 20260506 02:04:34.500 | INFO | MemInfo: { | 20260506 02:04:34.500 | INFO | "gpu" : " 17 GB", | 20260506 02:04:34.500 | INFO | "huge" : "", | 20260506 02:04:34.500 | INFO | "huge_shared" : " 27 GB", | 20260506 02:04:34.500 | INFO | "malloc" : " 14 GB", | 20260506 02:04:34.500 | INFO | "pinned" : " 29 GB" | 20260506 02:04:34.500 | INFO | } | 20260506 02:04:34.500 | INFO | Perf report for the last 5.13786 seconds: | 20260506 02:04:34.500 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:04:34.500 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:34.500 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:34.500 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:34.500 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 02:04:34.500 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:34.500 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:34.500 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 02:04:34.500 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:34.500 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:34.500 | INFO | Uploaded 36GiB to GPU in 2.11872s (16.9914GiB/s) | 20260506 02:04:34.500 | INFO | Downloaded 34GiB to GPU in 2.05071s (16.5796GiB/s) | 20260506 02:04:39.641 | INFO | Analyzing chunk frames = [8088064,8088576) | 20260506 02:04:39.641 | INFO | MemInfo: { | 20260506 02:04:39.641 | INFO | "gpu" : " 17 GB", | 20260506 02:04:39.641 | INFO | "huge" : "", | 20260506 02:04:39.641 | INFO | "huge_shared" : " 27 GB", | 20260506 02:04:39.641 | INFO | "malloc" : " 14 GB", | 20260506 02:04:39.641 | INFO | "pinned" : " 29 GB" | 20260506 02:04:39.641 | INFO | } | 20260506 02:04:39.641 | INFO | Perf report for the last 5.14096 seconds: | 20260506 02:04:39.641 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:04:39.641 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:39.641 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:39.641 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:39.641 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 02:04:39.641 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:39.641 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:39.641 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 02:04:39.641 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:39.641 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:39.641 | INFO | Uploaded 36GiB to GPU in 2.06612s (17.424GiB/s) | 20260506 02:04:39.642 | INFO | Downloaded 34GiB to GPU in 2.032s (16.7323GiB/s) | 20260506 02:04:44.784 | INFO | Analyzing chunk frames = [8088576,8089088) | 20260506 02:04:44.784 | INFO | MemInfo: { | 20260506 02:04:44.784 | INFO | "gpu" : " 17 GB", | 20260506 02:04:44.784 | INFO | "huge" : "", | 20260506 02:04:44.784 | INFO | "huge_shared" : " 27 GB", | 20260506 02:04:44.784 | INFO | "malloc" : " 14 GB", | 20260506 02:04:44.784 | INFO | "pinned" : " 29 GB" | 20260506 02:04:44.784 | INFO | } | 20260506 02:04:44.784 | INFO | Perf report for the last 5.14277 seconds: | 20260506 02:04:44.784 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:04:44.784 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:44.784 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:44.784 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:44.784 | INFO | ANALYSIS : 61.9%, 3.0, 96.8ms, 0 | 20260506 02:04:44.784 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:44.784 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:44.784 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 02:04:44.784 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:44.784 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:44.784 | INFO | Uploaded 36GiB to GPU in 2.02551s (17.7733GiB/s) | 20260506 02:04:44.784 | INFO | Downloaded 34GiB to GPU in 2.0573s (16.5265GiB/s) | 20260506 02:04:49.921 | INFO | Analyzing chunk frames = [8089088,8089600) | 20260506 02:04:49.921 | INFO | MemInfo: { | 20260506 02:04:49.921 | INFO | "gpu" : " 17 GB", | 20260506 02:04:49.921 | INFO | "huge" : "", | 20260506 02:04:49.921 | INFO | "huge_shared" : " 27 GB", | 20260506 02:04:49.921 | INFO | "malloc" : " 14 GB", | 20260506 02:04:49.921 | INFO | "pinned" : " 29 GB" | 20260506 02:04:49.921 | INFO | } | 20260506 02:04:49.921 | INFO | Perf report for the last 5.13686 seconds: | 20260506 02:04:49.921 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:04:49.921 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:49.921 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:49.921 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:49.921 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260506 02:04:49.921 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:49.921 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:49.921 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 02:04:49.921 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:49.921 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:49.921 | INFO | Uploaded 36GiB to GPU in 2.0425s (17.6254GiB/s) | 20260506 02:04:49.921 | INFO | Downloaded 34GiB to GPU in 2.02342s (16.8033GiB/s) | 20260506 02:04:50.045 | INFO | Starting a round of Baz IO | 20260506 02:04:53.183 | INFO | Populating RTMetrics report | 20260506 02:04:53.184 | INFO | RTMetrics report populated | 20260506 02:04:53.184 | INFO | Opening tmp RTMetrics json file | 20260506 02:04:53.185 | INFO | Serializing RTMetrics to json | 20260506 02:04:53.187 | INFO | Serialization done, swapping tmp file now... | 20260506 02:04:53.188 | INFO | RTMetrics done | 20260506 02:04:53.261 | INFO | Finished a round of Baz IO: Wrote 1.804 GiB in 3.216 seconds | 20260506 02:04:53.261 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.28% / 0.00% / 78.71% | 20260506 02:04:53.261 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0742912% / 7.94729e-06% / 8.74201e-05% | 20260506 02:04:53.261 | INFO | Baz IO overall: | 20260506 02:04:53.261 | INFO | BytesWritten : 4544230371663 | 20260506 02:04:53.261 | INFO | NumEvents : 2134365663169 | 20260506 02:04:55.062 | INFO | Analyzing chunk frames = [8089600,8090112) | 20260506 02:04:55.062 | INFO | MemInfo: { | 20260506 02:04:55.062 | INFO | "gpu" : " 17 GB", | 20260506 02:04:55.062 | INFO | "huge" : "", | 20260506 02:04:55.062 | INFO | "huge_shared" : " 27 GB", | 20260506 02:04:55.062 | INFO | "malloc" : " 14 GB", | 20260506 02:04:55.062 | INFO | "pinned" : " 29 GB" | 20260506 02:04:55.062 | INFO | } | 20260506 02:04:55.062 | INFO | Perf report for the last 5.14138 seconds: | 20260506 02:04:55.062 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:04:55.062 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:55.062 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:55.062 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:55.062 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260506 02:04:55.062 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:04:55.062 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260506 02:04:55.062 | INFO | PRE_HQ : 59.8%, 1.7, 53.8ms, 0 | 20260506 02:04:55.062 | INFO | BAZWRITER : 61.8%, 3.5, 111.3ms, 0 | 20260506 02:04:55.062 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:04:55.062 | INFO | Uploaded 36GiB to GPU in 2.18715s (16.4598GiB/s) | 20260506 02:04:55.063 | INFO | Downloaded 38GiB to GPU in 2.23766s (16.982GiB/s) | 20260506 02:05:00.200 | INFO | Analyzing chunk frames = [8090112,8090624) | 20260506 02:05:00.200 | INFO | MemInfo: { | 20260506 02:05:00.200 | INFO | "gpu" : " 17 GB", | 20260506 02:05:00.200 | INFO | "huge" : "", | 20260506 02:05:00.200 | INFO | "huge_shared" : " 27 GB", | 20260506 02:05:00.200 | INFO | "malloc" : " 14 GB", | 20260506 02:05:00.200 | INFO | "pinned" : " 29 GB" | 20260506 02:05:00.200 | INFO | } | 20260506 02:05:00.200 | INFO | Perf report for the last 5.13807 seconds: | 20260506 02:05:00.200 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:05:00.200 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:00.200 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:00.200 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:00.200 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 02:05:00.200 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:00.200 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:00.200 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:05:00.200 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:00.200 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:00.201 | INFO | Uploaded 36GiB to GPU in 1.97224s (18.2534GiB/s) | 20260506 02:05:00.201 | INFO | Downloaded 34GiB to GPU in 1.99204s (17.0679GiB/s) | 20260506 02:05:05.341 | INFO | Analyzing chunk frames = [8090624,8091136) | 20260506 02:05:05.341 | INFO | MemInfo: { | 20260506 02:05:05.341 | INFO | "gpu" : " 17 GB", | 20260506 02:05:05.341 | INFO | "huge" : "", | 20260506 02:05:05.341 | INFO | "huge_shared" : " 27 GB", | 20260506 02:05:05.341 | INFO | "malloc" : " 14 GB", | 20260506 02:05:05.341 | INFO | "pinned" : " 29 GB" | 20260506 02:05:05.341 | INFO | } | 20260506 02:05:05.341 | INFO | Perf report for the last 5.14091 seconds: | 20260506 02:05:05.341 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:05:05.341 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:05.341 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:05.341 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:05.341 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 02:05:05.341 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:05.341 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:05.341 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 02:05:05.341 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:05.341 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:05.341 | INFO | Uploaded 36GiB to GPU in 2.0897s (17.2273GiB/s) | 20260506 02:05:05.341 | INFO | Downloaded 34GiB to GPU in 2.02028s (16.8293GiB/s) | 20260506 02:05:10.482 | INFO | Analyzing chunk frames = [8091136,8091648) | 20260506 02:05:10.482 | INFO | MemInfo: { | 20260506 02:05:10.482 | INFO | "gpu" : " 17 GB", | 20260506 02:05:10.482 | INFO | "huge" : "", | 20260506 02:05:10.482 | INFO | "huge_shared" : " 27 GB", | 20260506 02:05:10.482 | INFO | "malloc" : " 14 GB", | 20260506 02:05:10.482 | INFO | "pinned" : " 29 GB" | 20260506 02:05:10.482 | INFO | } | 20260506 02:05:10.482 | INFO | Perf report for the last 5.14077 seconds: | 20260506 02:05:10.482 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:05:10.482 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:10.482 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:10.482 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:10.482 | INFO | ANALYSIS : 61.0%, 3.0, 94.7ms, 0 | 20260506 02:05:10.482 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:10.482 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:10.482 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 02:05:10.482 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:10.482 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:10.482 | INFO | Uploaded 36GiB to GPU in 2.08772s (17.2437GiB/s) | 20260506 02:05:10.482 | INFO | Downloaded 34GiB to GPU in 2.00569s (16.9518GiB/s) | 20260506 02:05:15.617 | INFO | Analyzing chunk frames = [8091648,8092160) | 20260506 02:05:15.617 | INFO | MemInfo: { | 20260506 02:05:15.617 | INFO | "gpu" : " 17 GB", | 20260506 02:05:15.617 | INFO | "huge" : "", | 20260506 02:05:15.617 | INFO | "huge_shared" : " 27 GB", | 20260506 02:05:15.617 | INFO | "malloc" : " 14 GB", | 20260506 02:05:15.617 | INFO | "pinned" : " 29 GB" | 20260506 02:05:15.617 | INFO | } | 20260506 02:05:15.617 | INFO | Perf report for the last 5.13497 seconds: | 20260506 02:05:15.617 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:05:15.617 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:15.617 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:15.617 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:15.617 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260506 02:05:15.617 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:15.617 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:15.617 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260506 02:05:15.617 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:15.617 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:15.617 | INFO | Uploaded 36GiB to GPU in 2.10692s (17.0866GiB/s) | 20260506 02:05:15.617 | INFO | Downloaded 34GiB to GPU in 2.02798s (16.7654GiB/s) | 20260506 02:05:20.754 | INFO | Analyzing chunk frames = [8092160,8092672) | 20260506 02:05:20.754 | INFO | MemInfo: { | 20260506 02:05:20.754 | INFO | "gpu" : " 17 GB", | 20260506 02:05:20.754 | INFO | "huge" : "", | 20260506 02:05:20.754 | INFO | "huge_shared" : " 27 GB", | 20260506 02:05:20.754 | INFO | "malloc" : " 14 GB", | 20260506 02:05:20.755 | INFO | "pinned" : " 29 GB" | 20260506 02:05:20.755 | INFO | } | 20260506 02:05:20.755 | INFO | Perf report for the last 5.13779 seconds: | 20260506 02:05:20.755 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:05:20.755 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:20.755 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:20.755 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:20.755 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 02:05:20.755 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:20.755 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:20.755 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 02:05:20.755 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:20.755 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:20.755 | INFO | Uploaded 36GiB to GPU in 2.08394s (17.2749GiB/s) | 20260506 02:05:20.755 | INFO | Downloaded 34GiB to GPU in 2.03507s (16.707GiB/s) | 20260506 02:05:25.894 | INFO | Analyzing chunk frames = [8092672,8093184) | 20260506 02:05:25.894 | INFO | MemInfo: { | 20260506 02:05:25.894 | INFO | "gpu" : " 17 GB", | 20260506 02:05:25.894 | INFO | "huge" : "", | 20260506 02:05:25.894 | INFO | "huge_shared" : " 27 GB", | 20260506 02:05:25.894 | INFO | "malloc" : " 14 GB", | 20260506 02:05:25.894 | INFO | "pinned" : " 29 GB" | 20260506 02:05:25.894 | INFO | } | 20260506 02:05:25.894 | INFO | Perf report for the last 5.13942 seconds: | 20260506 02:05:25.894 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:05:25.894 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:25.894 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:25.894 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:25.894 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260506 02:05:25.894 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:25.894 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:25.894 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 02:05:25.894 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:25.894 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:25.894 | INFO | Uploaded 36GiB to GPU in 2.06809s (17.4074GiB/s) | 20260506 02:05:25.894 | INFO | Downloaded 34GiB to GPU in 2.0765s (16.3737GiB/s) | 20260506 02:05:31.043 | INFO | Analyzing chunk frames = [8093184,8093696) | 20260506 02:05:31.043 | INFO | MemInfo: { | 20260506 02:05:31.043 | INFO | "gpu" : " 17 GB", | 20260506 02:05:31.043 | INFO | "huge" : "", | 20260506 02:05:31.043 | INFO | "huge_shared" : " 27 GB", | 20260506 02:05:31.043 | INFO | "malloc" : " 14 GB", | 20260506 02:05:31.043 | INFO | "pinned" : " 29 GB" | 20260506 02:05:31.043 | INFO | } | 20260506 02:05:31.043 | INFO | Perf report for the last 5.14872 seconds: | 20260506 02:05:31.043 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:05:31.043 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:31.043 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:31.043 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:31.043 | INFO | ANALYSIS : 61.7%, 3.0, 96.5ms, 0 | 20260506 02:05:31.043 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:31.043 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:31.043 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 02:05:31.043 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:31.043 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:31.043 | INFO | Uploaded 36GiB to GPU in 2.1318s (16.8872GiB/s) | 20260506 02:05:31.043 | INFO | Downloaded 34GiB to GPU in 2.02648s (16.7779GiB/s) | 20260506 02:05:31.043 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 02:05:31.043 | INFO | Total mem usage: 15.3 GiB | 20260506 02:05:31.043 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 02:05:31.043 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.21%) | 20260506 02:05:31.043 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 02:05:31.043 | INFO | High Water Breakdown: | 20260506 02:05:31.043 | INFO | 60.9% : 8.54 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 02:05:31.043 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 02:05:31.043 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 02:05:31.043 | INFO | ------------------------------------------ | 20260506 02:05:31.043 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:05:31.043 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 02:05:31.043 | INFO | Total mem usage: 26.5 GiB | 20260506 02:05:31.043 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 02:05:31.043 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 02:05:31.043 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 02:05:31.044 | INFO | High Water Breakdown: | 20260506 02:05:31.044 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:05:31.044 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:05:31.044 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:05:31.044 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:05:31.044 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:05:31.044 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:05:31.044 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 02:05:31.044 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 02:05:31.044 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:05:31.044 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 02:05:31.044 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 02:05:31.044 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 02:05:31.044 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 02:05:31.044 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 02:05:31.044 | INFO | ------------------------------------------ | 20260506 02:05:31.044 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:05:31.044 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 02:05:31.044 | INFO | Total mem usage: 25.6 GiB | 20260506 02:05:31.044 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 02:05:31.044 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 02:05:31.044 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 02:05:31.044 | INFO | High Water Breakdown: | 20260506 02:05:31.044 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 02:05:31.044 | INFO | ------------------------------------------ | 20260506 02:05:31.044 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:05:31.044 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 02:05:31.044 | INFO | Total mem usage: 15.7 GiB | 20260506 02:05:31.044 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 02:05:31.044 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 02:05:31.044 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 02:05:31.044 | INFO | High Water Breakdown: | 20260506 02:05:31.044 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 02:05:31.044 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 02:05:31.044 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 02:05:31.044 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 02:05:31.044 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 02:05:31.044 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 02:05:31.044 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 02:05:31.044 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 02:05:31.044 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 02:05:31.044 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:05:31.044 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 02:05:31.044 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:05:31.044 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 02:05:31.044 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 02:05:31.044 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 02:05:31.044 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 02:05:31.044 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:05:31.044 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 02:05:31.044 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:05:31.044 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:05:31.044 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:05:31.044 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 02:05:31.044 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 02:05:31.044 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:05:31.044 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 02:05:31.044 | INFO | ------------------------------------------ | 20260506 02:05:31.044 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:05:31.044 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 02:05:31.044 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:05:31.044 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:05:31.044 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.38e+01, 1.33e+01 / 8.83e-01 | 20260506 02:05:31.044 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.04e-01, 1.01e-02 / 5.81e-03 | 20260506 02:05:31.044 | INFO | PerfReport: 11.87%, Baseline -- 3.72e+00 / 4.53e+00, 5.81e-01 / 1.38e-01 | 20260506 02:05:31.044 | INFO | PerfReport: 3.29%, Histogram -- 1.03e+00 / 1.51e+00, 1.92e-01 / 9.31e-02 | 20260506 02:05:31.044 | INFO | PerfReport: 16.00%, DME -- 4.01e+01 / 4.41e+01, 3.99e+00 / 1.08e+00 | 20260506 02:05:31.044 | INFO | PerfReport: 57.45%, FrameLabeling -- 1.80e+01 / 1.92e+01, 1.52e+00 / 9.23e-01 | 20260506 02:05:31.044 | INFO | PerfReport: 7.03%, PulseAccumulating -- 2.20e+00 / 2.74e+00, 4.18e-01 / 2.14e-01 | 20260506 02:05:31.044 | INFO | PerfReport: 3.01%, Metrics -- 9.43e-01 / 1.22e+00, 2.47e-01 / 1.04e-01 | 20260506 02:05:31.044 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.27e-01 / 3.74e-01, 5.40e-02 / 2.55e-02 | 20260506 02:05:31.044 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 02:05:31.044 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:05:31.044 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:05:31.044 | INFO | PerfReport: Full Execution -- 9.78e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260506 02:05:31.044 | INFO | PerfReport: 22.17%, Upload -- 2.17e+01 / 0.00e+00, 7.25e+00 / 0.00e+00 | 20260506 02:05:31.044 | INFO | PerfReport: 21.89%, Download -- 2.14e+01 / 0.00e+00, 7.55e+00 / 0.00e+00 | 20260506 02:05:31.045 | INFO | PerfReport: 32.01%, Compute -- 3.13e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 02:05:31.045 | INFO | PerfReport: 2.04%, LockUpload -- 1.99e+00 / 0.00e+00, 7.87e+00 / 0.00e+00 | 20260506 02:05:31.045 | INFO | PerfReport: 1.30%, LockDownload -- 1.27e+00 / 0.00e+00, 2.89e+00 / 0.00e+00 | 20260506 02:05:31.045 | INFO | PerfReport: 20.58%, LockCompute -- 2.01e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260506 02:05:31.045 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 2.16e-02 / 0.00e+00 | 20260506 02:05:31.169 | INFO | Starting a round of Baz IO | 20260506 02:05:34.288 | INFO | Populating RTMetrics report | 20260506 02:05:34.288 | INFO | RTMetrics report populated | 20260506 02:05:34.288 | INFO | Opening tmp RTMetrics json file | 20260506 02:05:34.290 | INFO | Serializing RTMetrics to json | 20260506 02:05:34.292 | INFO | Serialization done, swapping tmp file now... | 20260506 02:05:34.293 | INFO | RTMetrics done | 20260506 02:05:34.363 | INFO | Finished a round of Baz IO: Wrote 1.803 GiB in 3.194 seconds | 20260506 02:05:34.363 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.28% / 0.00% / 78.72% | 20260506 02:05:34.363 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0736515% / 0% / 9.93411e-05% | 20260506 02:05:34.363 | INFO | Baz IO overall: | 20260506 02:05:34.363 | INFO | BytesWritten : 4546165929577 | 20260506 02:05:34.363 | INFO | NumEvents : 2135245971804 | 20260506 02:05:36.180 | INFO | Analyzing chunk frames = [8093696,8094208) | 20260506 02:05:36.180 | INFO | MemInfo: { | 20260506 02:05:36.180 | INFO | "gpu" : " 17 GB", | 20260506 02:05:36.180 | INFO | "huge" : "", | 20260506 02:05:36.180 | INFO | "huge_shared" : " 27 GB", | 20260506 02:05:36.180 | INFO | "malloc" : " 14 GB", | 20260506 02:05:36.180 | INFO | "pinned" : " 29 GB" | 20260506 02:05:36.180 | INFO | } | 20260506 02:05:36.180 | INFO | Perf report for the last 5.13693 seconds: | 20260506 02:05:36.180 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:05:36.180 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:36.180 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:36.180 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:36.180 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260506 02:05:36.180 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:36.180 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260506 02:05:36.180 | INFO | PRE_HQ : 59.1%, 1.8, 53.8ms, 0 | 20260506 02:05:36.180 | INFO | BAZWRITER : 61.3%, 3.5, 111.2ms, 0 | 20260506 02:05:36.180 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:05:36.180 | INFO | Uploaded 36GiB to GPU in 2.10404s (17.11GiB/s) | 20260506 02:05:36.180 | INFO | Downloaded 38GiB to GPU in 2.21943s (17.1215GiB/s) | 20260506 02:05:41.312 | INFO | Analyzing chunk frames = [8094208,8094720) | 20260506 02:05:41.312 | INFO | MemInfo: { | 20260506 02:05:41.312 | INFO | "gpu" : " 17 GB", | 20260506 02:05:41.312 | INFO | "huge" : "", | 20260506 02:05:41.312 | INFO | "huge_shared" : " 27 GB", | 20260506 02:05:41.312 | INFO | "malloc" : " 14 GB", | 20260506 02:05:41.312 | INFO | "pinned" : " 29 GB" | 20260506 02:05:41.312 | INFO | } | 20260506 02:05:41.312 | INFO | Perf report for the last 5.13206 seconds: | 20260506 02:05:41.312 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:05:41.312 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:41.312 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:41.312 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:41.312 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260506 02:05:41.312 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:41.312 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:41.312 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:05:41.312 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:41.312 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:41.312 | INFO | Uploaded 36GiB to GPU in 1.9774s (18.2057GiB/s) | 20260506 02:05:41.312 | INFO | Downloaded 34GiB to GPU in 1.98506s (17.128GiB/s) | 20260506 02:05:46.453 | INFO | Analyzing chunk frames = [8094720,8095232) | 20260506 02:05:46.453 | INFO | MemInfo: { | 20260506 02:05:46.453 | INFO | "gpu" : " 17 GB", | 20260506 02:05:46.453 | INFO | "huge" : "", | 20260506 02:05:46.453 | INFO | "huge_shared" : " 27 GB", | 20260506 02:05:46.453 | INFO | "malloc" : " 14 GB", | 20260506 02:05:46.453 | INFO | "pinned" : " 29 GB" | 20260506 02:05:46.453 | INFO | } | 20260506 02:05:46.453 | INFO | Perf report for the last 5.14149 seconds: | 20260506 02:05:46.453 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:05:46.453 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:46.453 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:46.453 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:46.453 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 02:05:46.453 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:46.453 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:46.453 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 02:05:46.453 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:46.454 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:46.454 | INFO | Uploaded 36GiB to GPU in 2.08467s (17.2689GiB/s) | 20260506 02:05:46.454 | INFO | Downloaded 34GiB to GPU in 2.02309s (16.8059GiB/s) | 20260506 02:05:51.593 | INFO | Analyzing chunk frames = [8095232,8095744) | 20260506 02:05:51.593 | INFO | MemInfo: { | 20260506 02:05:51.593 | INFO | "gpu" : " 17 GB", | 20260506 02:05:51.593 | INFO | "huge" : "", | 20260506 02:05:51.593 | INFO | "huge_shared" : " 27 GB", | 20260506 02:05:51.593 | INFO | "malloc" : " 14 GB", | 20260506 02:05:51.593 | INFO | "pinned" : " 29 GB" | 20260506 02:05:51.594 | INFO | } | 20260506 02:05:51.594 | INFO | Perf report for the last 5.14027 seconds: | 20260506 02:05:51.594 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:05:51.594 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:51.594 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:51.594 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:51.594 | INFO | ANALYSIS : 61.4%, 3.0, 95.3ms, 0 | 20260506 02:05:51.594 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:51.594 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:51.594 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:05:51.594 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:51.594 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:51.594 | INFO | Uploaded 36GiB to GPU in 2.11107s (17.0529GiB/s) | 20260506 02:05:51.594 | INFO | Downloaded 34GiB to GPU in 2.03528s (16.7053GiB/s) | 20260506 02:05:56.730 | INFO | Analyzing chunk frames = [8095744,8096256) | 20260506 02:05:56.730 | INFO | MemInfo: { | 20260506 02:05:56.730 | INFO | "gpu" : " 17 GB", | 20260506 02:05:56.730 | INFO | "huge" : "", | 20260506 02:05:56.730 | INFO | "huge_shared" : " 27 GB", | 20260506 02:05:56.730 | INFO | "malloc" : " 14 GB", | 20260506 02:05:56.730 | INFO | "pinned" : " 29 GB" | 20260506 02:05:56.730 | INFO | } | 20260506 02:05:56.730 | INFO | Perf report for the last 5.13636 seconds: | 20260506 02:05:56.730 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:05:56.730 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:56.730 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:56.730 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:56.730 | INFO | ANALYSIS : 60.7%, 3.0, 94.8ms, 0 | 20260506 02:05:56.730 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:56.730 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:56.730 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 02:05:56.730 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:56.730 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:05:56.730 | INFO | Uploaded 36GiB to GPU in 2.09681s (17.1689GiB/s) | 20260506 02:05:56.730 | INFO | Downloaded 34GiB to GPU in 2.03104s (16.7402GiB/s) | 20260506 02:06:01.873 | INFO | Analyzing chunk frames = [8096256,8096768) | 20260506 02:06:01.873 | INFO | MemInfo: { | 20260506 02:06:01.873 | INFO | "gpu" : " 17 GB", | 20260506 02:06:01.873 | INFO | "huge" : "", | 20260506 02:06:01.873 | INFO | "huge_shared" : " 27 GB", | 20260506 02:06:01.873 | INFO | "malloc" : " 14 GB", | 20260506 02:06:01.873 | INFO | "pinned" : " 29 GB" | 20260506 02:06:01.873 | INFO | } | 20260506 02:06:01.873 | INFO | Perf report for the last 5.14334 seconds: | 20260506 02:06:01.873 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:06:01.873 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:01.873 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:01.873 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:01.873 | INFO | ANALYSIS : 61.0%, 3.0, 95.4ms, 0 | 20260506 02:06:01.873 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:01.873 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:01.873 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:06:01.873 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:01.874 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:01.874 | INFO | Uploaded 36GiB to GPU in 2.08415s (17.2732GiB/s) | 20260506 02:06:01.874 | INFO | Downloaded 34GiB to GPU in 2.03548s (16.7037GiB/s) | 20260506 02:06:07.013 | INFO | Analyzing chunk frames = [8096768,8097280) | 20260506 02:06:07.013 | INFO | MemInfo: { | 20260506 02:06:07.013 | INFO | "gpu" : " 17 GB", | 20260506 02:06:07.013 | INFO | "huge" : "", | 20260506 02:06:07.013 | INFO | "huge_shared" : " 27 GB", | 20260506 02:06:07.013 | INFO | "malloc" : " 14 GB", | 20260506 02:06:07.013 | INFO | "pinned" : " 29 GB" | 20260506 02:06:07.013 | INFO | } | 20260506 02:06:07.013 | INFO | Perf report for the last 5.13945 seconds: | 20260506 02:06:07.013 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:06:07.013 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:07.013 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:07.013 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:07.013 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260506 02:06:07.013 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:07.013 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:07.013 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 02:06:07.013 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:07.013 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:07.013 | INFO | Uploaded 36GiB to GPU in 2.06583s (17.4264GiB/s) | 20260506 02:06:07.013 | INFO | Downloaded 34GiB to GPU in 2.07916s (16.3528GiB/s) | 20260506 02:06:12.153 | INFO | Analyzing chunk frames = [8097280,8097792) | 20260506 02:06:12.153 | INFO | MemInfo: { | 20260506 02:06:12.153 | INFO | "gpu" : " 17 GB", | 20260506 02:06:12.153 | INFO | "huge" : "", | 20260506 02:06:12.153 | INFO | "huge_shared" : " 27 GB", | 20260506 02:06:12.153 | INFO | "malloc" : " 14 GB", | 20260506 02:06:12.153 | INFO | "pinned" : " 29 GB" | 20260506 02:06:12.153 | INFO | } | 20260506 02:06:12.153 | INFO | Perf report for the last 5.14063 seconds: | 20260506 02:06:12.154 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:06:12.154 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:12.154 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:12.154 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:12.154 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260506 02:06:12.154 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:12.154 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:12.154 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 02:06:12.154 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:12.154 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:12.154 | INFO | Uploaded 36GiB to GPU in 2.13222s (16.8838GiB/s) | 20260506 02:06:12.154 | INFO | Downloaded 34GiB to GPU in 2.02452s (16.7941GiB/s) | 20260506 02:06:12.279 | INFO | Starting a round of Baz IO | 20260506 02:06:15.408 | INFO | Populating RTMetrics report | 20260506 02:06:15.408 | INFO | RTMetrics report populated | 20260506 02:06:15.408 | INFO | Opening tmp RTMetrics json file | 20260506 02:06:15.408 | INFO | Serializing RTMetrics to json | 20260506 02:06:15.409 | INFO | Serialization done, swapping tmp file now... | 20260506 02:06:15.410 | INFO | RTMetrics done | 20260506 02:06:15.485 | INFO | Finished a round of Baz IO: Wrote 1.803 GiB in 3.205 seconds | 20260506 02:06:15.485 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.28% / 0.00% / 78.72% | 20260506 02:06:15.485 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0739018% / 0% / 9.53674e-05% | 20260506 02:06:15.485 | INFO | Baz IO overall: | 20260506 02:06:15.485 | INFO | BytesWritten : 4548101523524 | 20260506 02:06:15.485 | INFO | NumEvents : 2136126570352 | 20260506 02:06:17.291 | INFO | Analyzing chunk frames = [8097792,8098304) | 20260506 02:06:17.291 | INFO | MemInfo: { | 20260506 02:06:17.291 | INFO | "gpu" : " 17 GB", | 20260506 02:06:17.291 | INFO | "huge" : "", | 20260506 02:06:17.291 | INFO | "huge_shared" : " 27 GB", | 20260506 02:06:17.291 | INFO | "malloc" : " 14 GB", | 20260506 02:06:17.291 | INFO | "pinned" : " 29 GB" | 20260506 02:06:17.291 | INFO | } | 20260506 02:06:17.292 | INFO | Perf report for the last 5.13847 seconds: | 20260506 02:06:17.292 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:06:17.292 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:17.292 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:17.292 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:17.292 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260506 02:06:17.292 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:17.292 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260506 02:06:17.292 | INFO | PRE_HQ : 59.3%, 1.7, 53.5ms, 0 | 20260506 02:06:17.292 | INFO | BAZWRITER : 61.6%, 3.5, 110.9ms, 0 | 20260506 02:06:17.292 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:06:17.292 | INFO | Uploaded 36GiB to GPU in 2.10406s (17.1098GiB/s) | 20260506 02:06:17.292 | INFO | Downloaded 38GiB to GPU in 2.22728s (17.0612GiB/s) | 20260506 02:06:22.430 | INFO | Analyzing chunk frames = [8098304,8098816) | 20260506 02:06:22.430 | INFO | MemInfo: { | 20260506 02:06:22.430 | INFO | "gpu" : " 17 GB", | 20260506 02:06:22.430 | INFO | "huge" : "", | 20260506 02:06:22.430 | INFO | "huge_shared" : " 27 GB", | 20260506 02:06:22.430 | INFO | "malloc" : " 14 GB", | 20260506 02:06:22.430 | INFO | "pinned" : " 29 GB" | 20260506 02:06:22.430 | INFO | } | 20260506 02:06:22.430 | INFO | Perf report for the last 5.13788 seconds: | 20260506 02:06:22.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:06:22.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:22.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:22.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:22.430 | INFO | ANALYSIS : 60.8%, 3.0, 94.9ms, 0 | 20260506 02:06:22.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:22.430 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:22.430 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:06:22.430 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:22.430 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:22.430 | INFO | Uploaded 36GiB to GPU in 1.92424s (18.7087GiB/s) | 20260506 02:06:22.430 | INFO | Downloaded 34GiB to GPU in 1.97583s (17.208GiB/s) | 20260506 02:06:27.564 | INFO | Analyzing chunk frames = [8098816,8099328) | 20260506 02:06:27.564 | INFO | MemInfo: { | 20260506 02:06:27.564 | INFO | "gpu" : " 17 GB", | 20260506 02:06:27.564 | INFO | "huge" : "", | 20260506 02:06:27.564 | INFO | "huge_shared" : " 27 GB", | 20260506 02:06:27.564 | INFO | "malloc" : " 14 GB", | 20260506 02:06:27.564 | INFO | "pinned" : " 29 GB" | 20260506 02:06:27.564 | INFO | } | 20260506 02:06:27.564 | INFO | Perf report for the last 5.13388 seconds: | 20260506 02:06:27.564 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:06:27.564 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:27.564 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:27.564 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:27.564 | INFO | ANALYSIS : 61.2%, 3.0, 95.0ms, 0 | 20260506 02:06:27.564 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:27.564 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:27.564 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 02:06:27.564 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:27.564 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:27.564 | INFO | Uploaded 36GiB to GPU in 2.07934s (17.3132GiB/s) | 20260506 02:06:27.564 | INFO | Downloaded 34GiB to GPU in 2.0226s (16.81GiB/s) | 20260506 02:06:32.702 | INFO | Analyzing chunk frames = [8099328,8099840) | 20260506 02:06:32.702 | INFO | MemInfo: { | 20260506 02:06:32.702 | INFO | "gpu" : " 17 GB", | 20260506 02:06:32.702 | INFO | "huge" : "", | 20260506 02:06:32.702 | INFO | "huge_shared" : " 27 GB", | 20260506 02:06:32.702 | INFO | "malloc" : " 14 GB", | 20260506 02:06:32.702 | INFO | "pinned" : " 29 GB" | 20260506 02:06:32.702 | INFO | } | 20260506 02:06:32.703 | INFO | Perf report for the last 5.1388 seconds: | 20260506 02:06:32.703 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:06:32.703 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:32.703 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:32.703 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:32.703 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260506 02:06:32.703 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:32.703 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:32.703 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:06:32.703 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:32.703 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:32.703 | INFO | Uploaded 36GiB to GPU in 2.13552s (16.8577GiB/s) | 20260506 02:06:32.703 | INFO | Downloaded 34GiB to GPU in 2.0216s (16.8184GiB/s) | 20260506 02:06:37.850 | INFO | Analyzing chunk frames = [8099840,8100352) | 20260506 02:06:37.850 | INFO | MemInfo: { | 20260506 02:06:37.850 | INFO | "gpu" : " 17 GB", | 20260506 02:06:37.850 | INFO | "huge" : "", | 20260506 02:06:37.850 | INFO | "huge_shared" : " 27 GB", | 20260506 02:06:37.850 | INFO | "malloc" : " 14 GB", | 20260506 02:06:37.850 | INFO | "pinned" : " 29 GB" | 20260506 02:06:37.850 | INFO | } | 20260506 02:06:37.850 | INFO | Perf report for the last 5.14728 seconds: | 20260506 02:06:37.850 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:06:37.850 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:37.850 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:37.850 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:37.850 | INFO | ANALYSIS : 60.9%, 3.0, 95.2ms, 0 | 20260506 02:06:37.850 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:37.850 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:37.850 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 02:06:37.850 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:37.850 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:37.850 | INFO | Uploaded 36GiB to GPU in 2.09032s (17.2222GiB/s) | 20260506 02:06:37.850 | INFO | Downloaded 34GiB to GPU in 2.04591s (16.6185GiB/s) | 20260506 02:06:42.991 | INFO | Analyzing chunk frames = [8100352,8100864) | 20260506 02:06:42.991 | INFO | MemInfo: { | 20260506 02:06:42.991 | INFO | "gpu" : " 17 GB", | 20260506 02:06:42.991 | INFO | "huge" : "", | 20260506 02:06:42.991 | INFO | "huge_shared" : " 27 GB", | 20260506 02:06:42.991 | INFO | "malloc" : " 14 GB", | 20260506 02:06:42.991 | INFO | "pinned" : " 29 GB" | 20260506 02:06:42.991 | INFO | } | 20260506 02:06:42.991 | INFO | Perf report for the last 5.14136 seconds: | 20260506 02:06:42.991 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:06:42.991 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:42.991 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:42.991 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:42.991 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 02:06:42.991 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:42.991 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:42.991 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:06:42.991 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:42.991 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:42.992 | INFO | Uploaded 36GiB to GPU in 2.11007s (17.0611GiB/s) | 20260506 02:06:42.992 | INFO | Downloaded 34GiB to GPU in 2.04757s (16.6051GiB/s) | 20260506 02:06:48.128 | INFO | Analyzing chunk frames = [8100864,8101376) | 20260506 02:06:48.128 | INFO | MemInfo: { | 20260506 02:06:48.128 | INFO | "gpu" : " 17 GB", | 20260506 02:06:48.128 | INFO | "huge" : "", | 20260506 02:06:48.128 | INFO | "huge_shared" : " 27 GB", | 20260506 02:06:48.128 | INFO | "malloc" : " 14 GB", | 20260506 02:06:48.128 | INFO | "pinned" : " 29 GB" | 20260506 02:06:48.128 | INFO | } | 20260506 02:06:48.128 | INFO | Perf report for the last 5.13675 seconds: | 20260506 02:06:48.128 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:06:48.128 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:48.128 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:48.128 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:48.128 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 02:06:48.128 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:48.128 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:48.128 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 02:06:48.128 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:48.128 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:48.128 | INFO | Uploaded 36GiB to GPU in 2.06258s (17.4538GiB/s) | 20260506 02:06:48.128 | INFO | Downloaded 34GiB to GPU in 2.07115s (16.416GiB/s) | 20260506 02:06:53.264 | INFO | Analyzing chunk frames = [8101376,8101888) | 20260506 02:06:53.264 | INFO | MemInfo: { | 20260506 02:06:53.264 | INFO | "gpu" : " 17 GB", | 20260506 02:06:53.264 | INFO | "huge" : "", | 20260506 02:06:53.264 | INFO | "huge_shared" : " 27 GB", | 20260506 02:06:53.264 | INFO | "malloc" : " 14 GB", | 20260506 02:06:53.264 | INFO | "pinned" : " 29 GB" | 20260506 02:06:53.264 | INFO | } | 20260506 02:06:53.264 | INFO | Perf report for the last 5.13578 seconds: | 20260506 02:06:53.264 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:06:53.264 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:53.264 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:53.264 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:53.264 | INFO | ANALYSIS : 61.5%, 3.0, 96.0ms, 0 | 20260506 02:06:53.264 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:53.264 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:53.264 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 02:06:53.264 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:53.264 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:53.264 | INFO | Uploaded 36GiB to GPU in 2.06848s (17.4041GiB/s) | 20260506 02:06:53.264 | INFO | Downloaded 34GiB to GPU in 2.02815s (16.764GiB/s) | 20260506 02:06:53.264 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 02:06:53.264 | INFO | Total mem usage: 15.3 GiB | 20260506 02:06:53.264 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 02:06:53.264 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.19%) | 20260506 02:06:53.264 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 02:06:53.264 | INFO | High Water Breakdown: | 20260506 02:06:53.264 | INFO | 60.9% : 8.54 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 02:06:53.264 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 02:06:53.264 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 02:06:53.264 | INFO | ------------------------------------------ | 20260506 02:06:53.264 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:06:53.264 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 02:06:53.264 | INFO | Total mem usage: 26.5 GiB | 20260506 02:06:53.265 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 02:06:53.265 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 02:06:53.265 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 02:06:53.265 | INFO | High Water Breakdown: | 20260506 02:06:53.265 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:06:53.265 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:06:53.265 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:06:53.265 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:06:53.265 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:06:53.265 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:06:53.265 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 02:06:53.265 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 02:06:53.265 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:06:53.265 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 02:06:53.265 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 02:06:53.265 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 02:06:53.265 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 02:06:53.265 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 02:06:53.265 | INFO | ------------------------------------------ | 20260506 02:06:53.265 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:06:53.265 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 02:06:53.265 | INFO | Total mem usage: 25.6 GiB | 20260506 02:06:53.265 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 02:06:53.265 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 02:06:53.265 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 02:06:53.265 | INFO | High Water Breakdown: | 20260506 02:06:53.265 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 02:06:53.265 | INFO | ------------------------------------------ | 20260506 02:06:53.265 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:06:53.265 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 02:06:53.265 | INFO | Total mem usage: 15.7 GiB | 20260506 02:06:53.265 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 02:06:53.265 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 02:06:53.265 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 02:06:53.265 | INFO | High Water Breakdown: | 20260506 02:06:53.265 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 02:06:53.265 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 02:06:53.265 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 02:06:53.265 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 02:06:53.265 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 02:06:53.265 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 02:06:53.265 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 02:06:53.265 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 02:06:53.265 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 02:06:53.265 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:06:53.265 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 02:06:53.265 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:06:53.265 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 02:06:53.265 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 02:06:53.265 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 02:06:53.265 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 02:06:53.265 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:06:53.265 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 02:06:53.265 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:06:53.265 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:06:53.265 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:06:53.265 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 02:06:53.265 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 02:06:53.265 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:06:53.265 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 02:06:53.265 | INFO | ------------------------------------------ | 20260506 02:06:53.265 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:06:53.265 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 02:06:53.265 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:06:53.265 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:06:53.265 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.38e+01, 1.31e+01 / 7.27e-01 | 20260506 02:06:53.265 | INFO | PerfReport: 0.29%, AggregateStats -- 9.18e-02 / 1.04e-01, 9.95e-03 / 4.39e-03 | 20260506 02:06:53.265 | INFO | PerfReport: 11.86%, Baseline -- 3.72e+00 / 4.49e+00, 5.79e-01 / 2.59e-01 | 20260506 02:06:53.265 | INFO | PerfReport: 3.31%, Histogram -- 1.04e+00 / 1.54e+00, 1.96e-01 / 9.07e-02 | 20260506 02:06:53.265 | INFO | PerfReport: 15.95%, DME -- 4.00e+01 / 4.39e+01, 3.86e+00 / 9.69e-01 | 20260506 02:06:53.265 | INFO | PerfReport: 57.50%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.58e+00 / 1.12e+00 | 20260506 02:06:53.265 | INFO | PerfReport: 7.02%, PulseAccumulating -- 2.20e+00 / 2.75e+00, 4.12e-01 / 2.00e-01 | 20260506 02:06:53.265 | INFO | PerfReport: 3.02%, Metrics -- 9.47e-01 / 1.25e+00, 2.56e-01 / 1.34e-01 | 20260506 02:06:53.265 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.27e-01 / 3.73e-01, 5.50e-02 / 2.35e-02 | 20260506 02:06:53.265 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 02:06:53.265 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:06:53.265 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:06:53.265 | INFO | PerfReport: Full Execution -- 9.78e+01 / -nan, 2.44e+01 / 0.00e+00 | 20260506 02:06:53.265 | INFO | PerfReport: 22.13%, Upload -- 2.16e+01 / 0.00e+00, 7.20e+00 / 0.00e+00 | 20260506 02:06:53.265 | INFO | PerfReport: 21.89%, Download -- 2.14e+01 / 0.00e+00, 7.51e+00 / 0.00e+00 | 20260506 02:06:53.265 | INFO | PerfReport: 32.03%, Compute -- 3.13e+01 / 0.00e+00, 1.31e+01 / 0.00e+00 | 20260506 02:06:53.265 | INFO | PerfReport: 2.02%, LockUpload -- 1.98e+00 / 0.00e+00, 7.81e+00 / 0.00e+00 | 20260506 02:06:53.265 | INFO | PerfReport: 1.34%, LockDownload -- 1.31e+00 / 0.00e+00, 3.03e+00 / 0.00e+00 | 20260506 02:06:53.265 | INFO | PerfReport: 20.58%, LockCompute -- 2.01e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260506 02:06:53.265 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.64e-02 / 0.00e+00 | 20260506 02:06:53.382 | INFO | Starting a round of Baz IO | 20260506 02:06:56.525 | INFO | Populating RTMetrics report | 20260506 02:06:56.525 | INFO | RTMetrics report populated | 20260506 02:06:56.525 | INFO | Opening tmp RTMetrics json file | 20260506 02:06:56.526 | INFO | Serializing RTMetrics to json | 20260506 02:06:56.527 | INFO | Serialization done, swapping tmp file now... | 20260506 02:06:56.527 | INFO | RTMetrics done | 20260506 02:06:56.600 | INFO | Finished a round of Baz IO: Wrote 1.806 GiB in 3.217 seconds | 20260506 02:06:56.600 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.28% / 0.00% / 78.72% | 20260506 02:06:56.600 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0736833% / 7.94729e-06% / 9.13938e-05% | 20260506 02:06:56.600 | INFO | Baz IO overall: | 20260506 02:06:56.600 | INFO | BytesWritten : 4550040290191 | 20260506 02:06:56.600 | INFO | NumEvents : 2137009283202 | 20260506 02:06:58.404 | INFO | Analyzing chunk frames = [8101888,8102400) | 20260506 02:06:58.404 | INFO | MemInfo: { | 20260506 02:06:58.404 | INFO | "gpu" : " 17 GB", | 20260506 02:06:58.404 | INFO | "huge" : "", | 20260506 02:06:58.404 | INFO | "huge_shared" : " 27 GB", | 20260506 02:06:58.404 | INFO | "malloc" : " 14 GB", | 20260506 02:06:58.404 | INFO | "pinned" : " 29 GB" | 20260506 02:06:58.404 | INFO | } | 20260506 02:06:58.404 | INFO | Perf report for the last 5.14046 seconds: | 20260506 02:06:58.404 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:06:58.404 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:58.404 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:58.404 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:58.404 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260506 02:06:58.404 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:06:58.405 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260506 02:06:58.405 | INFO | PRE_HQ : 59.5%, 1.7, 53.4ms, 0 | 20260506 02:06:58.405 | INFO | BAZWRITER : 61.7%, 3.5, 109.7ms, 0 | 20260506 02:06:58.405 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:06:58.405 | INFO | Uploaded 36GiB to GPU in 2.18599s (16.4685GiB/s) | 20260506 02:06:58.405 | INFO | Downloaded 38GiB to GPU in 2.24006s (16.9638GiB/s) | 20260506 02:07:03.542 | INFO | Analyzing chunk frames = [8102400,8102912) | 20260506 02:07:03.542 | INFO | MemInfo: { | 20260506 02:07:03.542 | INFO | "gpu" : " 17 GB", | 20260506 02:07:03.542 | INFO | "huge" : "", | 20260506 02:07:03.542 | INFO | "huge_shared" : " 27 GB", | 20260506 02:07:03.542 | INFO | "malloc" : " 14 GB", | 20260506 02:07:03.542 | INFO | "pinned" : " 29 GB" | 20260506 02:07:03.542 | INFO | } | 20260506 02:07:03.542 | INFO | Perf report for the last 5.13814 seconds: | 20260506 02:07:03.542 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:07:03.542 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:03.542 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:03.542 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:03.542 | INFO | ANALYSIS : 60.7%, 3.0, 94.8ms, 0 | 20260506 02:07:03.542 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:03.542 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:03.542 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 02:07:03.542 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:03.542 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:03.543 | INFO | Uploaded 36GiB to GPU in 1.97452s (18.2323GiB/s) | 20260506 02:07:03.543 | INFO | Downloaded 34GiB to GPU in 1.98394s (17.1376GiB/s) | 20260506 02:07:08.679 | INFO | Analyzing chunk frames = [8102912,8103424) | 20260506 02:07:08.679 | INFO | MemInfo: { | 20260506 02:07:08.679 | INFO | "gpu" : " 17 GB", | 20260506 02:07:08.679 | INFO | "huge" : "", | 20260506 02:07:08.679 | INFO | "huge_shared" : " 27 GB", | 20260506 02:07:08.679 | INFO | "malloc" : " 14 GB", | 20260506 02:07:08.679 | INFO | "pinned" : " 29 GB" | 20260506 02:07:08.679 | INFO | } | 20260506 02:07:08.679 | INFO | Perf report for the last 5.13678 seconds: | 20260506 02:07:08.679 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:07:08.679 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:08.679 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:08.679 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:08.679 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 02:07:08.679 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:08.679 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:08.679 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 02:07:08.679 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:08.679 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:08.680 | INFO | Uploaded 36GiB to GPU in 2.08407s (17.2739GiB/s) | 20260506 02:07:08.680 | INFO | Downloaded 34GiB to GPU in 2.02299s (16.8068GiB/s) | 20260506 02:07:13.822 | INFO | Analyzing chunk frames = [8103424,8103936) | 20260506 02:07:13.822 | INFO | MemInfo: { | 20260506 02:07:13.822 | INFO | "gpu" : " 17 GB", | 20260506 02:07:13.822 | INFO | "huge" : "", | 20260506 02:07:13.822 | INFO | "huge_shared" : " 27 GB", | 20260506 02:07:13.822 | INFO | "malloc" : " 14 GB", | 20260506 02:07:13.822 | INFO | "pinned" : " 29 GB" | 20260506 02:07:13.822 | INFO | } | 20260506 02:07:13.822 | INFO | Perf report for the last 5.14305 seconds: | 20260506 02:07:13.822 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:07:13.822 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:13.822 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:13.822 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:13.822 | INFO | ANALYSIS : 61.4%, 3.0, 95.4ms, 0 | 20260506 02:07:13.822 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:13.822 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:13.822 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:07:13.822 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:13.822 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:13.823 | INFO | Uploaded 36GiB to GPU in 2.14927s (16.7498GiB/s) | 20260506 02:07:13.823 | INFO | Downloaded 34GiB to GPU in 2.05898s (16.513GiB/s) | 20260506 02:07:18.960 | INFO | Analyzing chunk frames = [8103936,8104448) | 20260506 02:07:18.960 | INFO | MemInfo: { | 20260506 02:07:18.960 | INFO | "gpu" : " 17 GB", | 20260506 02:07:18.960 | INFO | "huge" : "", | 20260506 02:07:18.960 | INFO | "huge_shared" : " 27 GB", | 20260506 02:07:18.960 | INFO | "malloc" : " 14 GB", | 20260506 02:07:18.960 | INFO | "pinned" : " 29 GB" | 20260506 02:07:18.960 | INFO | } | 20260506 02:07:18.960 | INFO | Perf report for the last 5.13777 seconds: | 20260506 02:07:18.960 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:07:18.960 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:18.960 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:18.960 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:18.960 | INFO | ANALYSIS : 60.7%, 3.0, 94.8ms, 0 | 20260506 02:07:18.960 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:18.960 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:18.960 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:07:18.960 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:18.960 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:18.960 | INFO | Uploaded 36GiB to GPU in 2.07757s (17.3279GiB/s) | 20260506 02:07:18.960 | INFO | Downloaded 34GiB to GPU in 2.03134s (16.7377GiB/s) | 20260506 02:07:24.104 | INFO | Analyzing chunk frames = [8104448,8104960) | 20260506 02:07:24.104 | INFO | MemInfo: { | 20260506 02:07:24.104 | INFO | "gpu" : " 17 GB", | 20260506 02:07:24.104 | INFO | "huge" : "", | 20260506 02:07:24.104 | INFO | "huge_shared" : " 27 GB", | 20260506 02:07:24.104 | INFO | "malloc" : " 14 GB", | 20260506 02:07:24.104 | INFO | "pinned" : " 29 GB" | 20260506 02:07:24.104 | INFO | } | 20260506 02:07:24.104 | INFO | Perf report for the last 5.14411 seconds: | 20260506 02:07:24.104 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:07:24.104 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:24.104 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:24.104 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:24.104 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 02:07:24.104 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:24.104 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:24.104 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:07:24.104 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:24.104 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:24.104 | INFO | Uploaded 36GiB to GPU in 2.07208s (17.3738GiB/s) | 20260506 02:07:24.105 | INFO | Downloaded 34GiB to GPU in 2.0422s (16.6487GiB/s) | 20260506 02:07:29.241 | INFO | Analyzing chunk frames = [8104960,8105472) | 20260506 02:07:29.241 | INFO | MemInfo: { | 20260506 02:07:29.241 | INFO | "gpu" : " 17 GB", | 20260506 02:07:29.241 | INFO | "huge" : "", | 20260506 02:07:29.241 | INFO | "huge_shared" : " 27 GB", | 20260506 02:07:29.241 | INFO | "malloc" : " 14 GB", | 20260506 02:07:29.241 | INFO | "pinned" : " 29 GB" | 20260506 02:07:29.242 | INFO | } | 20260506 02:07:29.242 | INFO | Perf report for the last 5.13742 seconds: | 20260506 02:07:29.242 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:07:29.242 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:29.242 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:29.242 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:29.242 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260506 02:07:29.242 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:29.242 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:29.242 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 02:07:29.242 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:29.242 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:29.242 | INFO | Uploaded 36GiB to GPU in 2.07017s (17.3899GiB/s) | 20260506 02:07:29.242 | INFO | Downloaded 34GiB to GPU in 2.06977s (16.4269GiB/s) | 20260506 02:07:34.380 | INFO | Analyzing chunk frames = [8105472,8105984) | 20260506 02:07:34.380 | INFO | MemInfo: { | 20260506 02:07:34.380 | INFO | "gpu" : " 17 GB", | 20260506 02:07:34.380 | INFO | "huge" : "", | 20260506 02:07:34.380 | INFO | "huge_shared" : " 27 GB", | 20260506 02:07:34.380 | INFO | "malloc" : " 14 GB", | 20260506 02:07:34.380 | INFO | "pinned" : " 29 GB" | 20260506 02:07:34.380 | INFO | } | 20260506 02:07:34.380 | INFO | Perf report for the last 5.13842 seconds: | 20260506 02:07:34.380 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:07:34.380 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:34.380 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:34.380 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:34.380 | INFO | ANALYSIS : 61.5%, 3.0, 96.0ms, 0 | 20260506 02:07:34.380 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:34.380 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:34.380 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 02:07:34.380 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:34.380 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:34.380 | INFO | Uploaded 36GiB to GPU in 2.06573s (17.4273GiB/s) | 20260506 02:07:34.380 | INFO | Downloaded 34GiB to GPU in 2.02591s (16.7826GiB/s) | 20260506 02:07:34.498 | INFO | Starting a round of Baz IO | 20260506 02:07:37.640 | INFO | Populating RTMetrics report | 20260506 02:07:37.640 | INFO | RTMetrics report populated | 20260506 02:07:37.640 | INFO | Opening tmp RTMetrics json file | 20260506 02:07:37.641 | INFO | Serializing RTMetrics to json | 20260506 02:07:37.642 | INFO | Serialization done, swapping tmp file now... | 20260506 02:07:37.642 | INFO | RTMetrics done | 20260506 02:07:37.717 | INFO | Finished a round of Baz IO: Wrote 1.803 GiB in 3.219 seconds | 20260506 02:07:37.717 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.28% / 0.00% / 78.72% | 20260506 02:07:37.717 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0733654% / 3.97364e-06% / 5.96046e-05% | 20260506 02:07:37.717 | INFO | Baz IO overall: | 20260506 02:07:37.717 | INFO | BytesWritten : 4551976751349 | 20260506 02:07:37.717 | INFO | NumEvents : 2137890371770 | 20260506 02:07:39.514 | INFO | Analyzing chunk frames = [8105984,8106496) | 20260506 02:07:39.514 | INFO | MemInfo: { | 20260506 02:07:39.514 | INFO | "gpu" : " 17 GB", | 20260506 02:07:39.514 | INFO | "huge" : "", | 20260506 02:07:39.514 | INFO | "huge_shared" : " 27 GB", | 20260506 02:07:39.514 | INFO | "malloc" : " 14 GB", | 20260506 02:07:39.514 | INFO | "pinned" : " 29 GB" | 20260506 02:07:39.514 | INFO | } | 20260506 02:07:39.514 | INFO | Perf report for the last 5.13407 seconds: | 20260506 02:07:39.514 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:07:39.514 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:39.514 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:39.514 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:39.514 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260506 02:07:39.514 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:39.514 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260506 02:07:39.514 | INFO | PRE_HQ : 59.5%, 1.7, 53.4ms, 0 | 20260506 02:07:39.514 | INFO | BAZWRITER : 61.7%, 3.5, 109.8ms, 0 | 20260506 02:07:39.514 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:07:39.514 | INFO | Uploaded 36GiB to GPU in 2.187s (16.4609GiB/s) | 20260506 02:07:39.514 | INFO | Downloaded 38GiB to GPU in 2.24095s (16.9571GiB/s) | 20260506 02:07:44.656 | INFO | Analyzing chunk frames = [8106496,8107008) | 20260506 02:07:44.656 | INFO | MemInfo: { | 20260506 02:07:44.656 | INFO | "gpu" : " 17 GB", | 20260506 02:07:44.656 | INFO | "huge" : "", | 20260506 02:07:44.656 | INFO | "huge_shared" : " 27 GB", | 20260506 02:07:44.656 | INFO | "malloc" : " 14 GB", | 20260506 02:07:44.656 | INFO | "pinned" : " 29 GB" | 20260506 02:07:44.656 | INFO | } | 20260506 02:07:44.656 | INFO | Perf report for the last 5.14236 seconds: | 20260506 02:07:44.656 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:07:44.656 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:44.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:44.657 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:44.657 | INFO | ANALYSIS : 61.6%, 3.0, 96.2ms, 0 | 20260506 02:07:44.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:44.657 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:44.657 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:07:44.657 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:44.657 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:44.657 | INFO | Uploaded 36GiB to GPU in 2.12118s (16.9717GiB/s) | 20260506 02:07:44.657 | INFO | Downloaded 34GiB to GPU in 2.04416s (16.6328GiB/s) | 20260506 02:07:49.807 | INFO | Analyzing chunk frames = [8107008,8107520) | 20260506 02:07:49.807 | INFO | MemInfo: { | 20260506 02:07:49.807 | INFO | "gpu" : " 17 GB", | 20260506 02:07:49.807 | INFO | "huge" : "", | 20260506 02:07:49.807 | INFO | "huge_shared" : " 27 GB", | 20260506 02:07:49.807 | INFO | "malloc" : " 14 GB", | 20260506 02:07:49.807 | INFO | "pinned" : " 29 GB" | 20260506 02:07:49.807 | INFO | } | 20260506 02:07:49.807 | INFO | Perf report for the last 5.15033 seconds: | 20260506 02:07:49.807 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:07:49.807 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:49.807 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:49.807 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:49.807 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 02:07:49.807 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:49.807 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:49.807 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:07:49.807 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:49.807 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:49.807 | INFO | Uploaded 36GiB to GPU in 2.11312s (17.0364GiB/s) | 20260506 02:07:49.807 | INFO | Downloaded 34GiB to GPU in 2.04227s (16.6482GiB/s) | 20260506 02:07:54.939 | INFO | Analyzing chunk frames = [8107520,8108032) | 20260506 02:07:54.939 | INFO | MemInfo: { | 20260506 02:07:54.939 | INFO | "gpu" : " 17 GB", | 20260506 02:07:54.939 | INFO | "huge" : "", | 20260506 02:07:54.939 | INFO | "huge_shared" : " 27 GB", | 20260506 02:07:54.939 | INFO | "malloc" : " 14 GB", | 20260506 02:07:54.939 | INFO | "pinned" : " 29 GB" | 20260506 02:07:54.939 | INFO | } | 20260506 02:07:54.939 | INFO | Perf report for the last 5.13257 seconds: | 20260506 02:07:54.939 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:07:54.939 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:54.939 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:54.939 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:54.939 | INFO | ANALYSIS : 61.0%, 3.0, 94.6ms, 0 | 20260506 02:07:54.940 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:54.940 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:54.940 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:07:54.940 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:54.940 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:07:54.940 | INFO | Uploaded 36GiB to GPU in 2.09298s (17.2004GiB/s) | 20260506 02:07:54.940 | INFO | Downloaded 34GiB to GPU in 2.01197s (16.8988GiB/s) | 20260506 02:08:00.072 | INFO | Analyzing chunk frames = [8108032,8108544) | 20260506 02:08:00.072 | INFO | MemInfo: { | 20260506 02:08:00.072 | INFO | "gpu" : " 17 GB", | 20260506 02:08:00.072 | INFO | "huge" : "", | 20260506 02:08:00.072 | INFO | "huge_shared" : " 27 GB", | 20260506 02:08:00.072 | INFO | "malloc" : " 14 GB", | 20260506 02:08:00.072 | INFO | "pinned" : " 29 GB" | 20260506 02:08:00.072 | INFO | } | 20260506 02:08:00.072 | INFO | Perf report for the last 5.1328 seconds: | 20260506 02:08:00.072 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:08:00.072 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:00.072 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:00.072 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:00.072 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260506 02:08:00.072 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:00.072 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:00.072 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 02:08:00.072 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:00.072 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:00.072 | INFO | Uploaded 36GiB to GPU in 2.09884s (17.1523GiB/s) | 20260506 02:08:00.072 | INFO | Downloaded 34GiB to GPU in 2.03094s (16.741GiB/s) | 20260506 02:08:05.213 | INFO | Analyzing chunk frames = [8108544,8109056) | 20260506 02:08:05.213 | INFO | MemInfo: { | 20260506 02:08:05.213 | INFO | "gpu" : " 17 GB", | 20260506 02:08:05.213 | INFO | "huge" : "", | 20260506 02:08:05.213 | INFO | "huge_shared" : " 27 GB", | 20260506 02:08:05.213 | INFO | "malloc" : " 14 GB", | 20260506 02:08:05.213 | INFO | "pinned" : " 29 GB" | 20260506 02:08:05.213 | INFO | } | 20260506 02:08:05.213 | INFO | Perf report for the last 5.14094 seconds: | 20260506 02:08:05.213 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:08:05.213 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:05.213 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:05.213 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:05.213 | INFO | ANALYSIS : 61.4%, 3.0, 95.9ms, 0 | 20260506 02:08:05.213 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:05.213 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:05.213 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:08:05.213 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:05.213 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:05.213 | INFO | Uploaded 36GiB to GPU in 2.07965s (17.3106GiB/s) | 20260506 02:08:05.213 | INFO | Downloaded 34GiB to GPU in 2.0385s (16.6789GiB/s) | 20260506 02:08:10.355 | INFO | Analyzing chunk frames = [8109056,8109568) | 20260506 02:08:10.355 | INFO | MemInfo: { | 20260506 02:08:10.355 | INFO | "gpu" : " 17 GB", | 20260506 02:08:10.355 | INFO | "huge" : "", | 20260506 02:08:10.355 | INFO | "huge_shared" : " 27 GB", | 20260506 02:08:10.355 | INFO | "malloc" : " 14 GB", | 20260506 02:08:10.355 | INFO | "pinned" : " 29 GB" | 20260506 02:08:10.355 | INFO | } | 20260506 02:08:10.355 | INFO | Perf report for the last 5.14208 seconds: | 20260506 02:08:10.355 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:08:10.355 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:10.355 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:10.355 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:10.355 | INFO | ANALYSIS : 62.3%, 3.0, 97.4ms, 0 | 20260506 02:08:10.355 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:10.355 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:10.355 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 02:08:10.355 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:10.355 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:10.355 | INFO | Uploaded 36GiB to GPU in 2.13492s (16.8625GiB/s) | 20260506 02:08:10.356 | INFO | Downloaded 34GiB to GPU in 2.08902s (16.2756GiB/s) | 20260506 02:08:15.496 | INFO | Analyzing chunk frames = [8109568,8110080) | 20260506 02:08:15.496 | INFO | MemInfo: { | 20260506 02:08:15.496 | INFO | "gpu" : " 17 GB", | 20260506 02:08:15.496 | INFO | "huge" : "", | 20260506 02:08:15.496 | INFO | "huge_shared" : " 27 GB", | 20260506 02:08:15.496 | INFO | "malloc" : " 14 GB", | 20260506 02:08:15.496 | INFO | "pinned" : " 29 GB" | 20260506 02:08:15.496 | INFO | } | 20260506 02:08:15.496 | INFO | Perf report for the last 5.14086 seconds: | 20260506 02:08:15.496 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:08:15.496 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:15.496 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:15.496 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:15.496 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 02:08:15.496 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:15.496 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:15.496 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:08:15.496 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:15.496 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:15.496 | INFO | Uploaded 36GiB to GPU in 2.13655s (16.8496GiB/s) | 20260506 02:08:15.496 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 02:08:15.496 | INFO | Total mem usage: 15.3 GiB | 20260506 02:08:15.496 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 02:08:15.496 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.19%) | 20260506 02:08:15.496 | INFO | Downloaded 34GiB to GPU in 2.02761s (16.7685GiB/s) | 20260506 02:08:15.496 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 02:08:15.496 | INFO | High Water Breakdown: | 20260506 02:08:15.496 | INFO | 60.9% : 8.54 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 02:08:15.497 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 02:08:15.497 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 02:08:15.497 | INFO | ------------------------------------------ | 20260506 02:08:15.497 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:08:15.497 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 02:08:15.497 | INFO | Total mem usage: 26.5 GiB | 20260506 02:08:15.497 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 02:08:15.497 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 02:08:15.497 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260506 02:08:15.497 | INFO | High Water Breakdown: | 20260506 02:08:15.497 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:08:15.497 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:08:15.497 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:08:15.497 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:08:15.497 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:08:15.497 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:08:15.497 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 02:08:15.497 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 02:08:15.497 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:08:15.497 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 02:08:15.497 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 02:08:15.497 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 02:08:15.497 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 02:08:15.497 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 02:08:15.497 | INFO | ------------------------------------------ | 20260506 02:08:15.497 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:08:15.497 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 02:08:15.497 | INFO | Total mem usage: 25.6 GiB | 20260506 02:08:15.497 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 02:08:15.497 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 02:08:15.497 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 02:08:15.497 | INFO | High Water Breakdown: | 20260506 02:08:15.497 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 02:08:15.497 | INFO | ------------------------------------------ | 20260506 02:08:15.497 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:08:15.497 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 02:08:15.497 | INFO | Total mem usage: 15.7 GiB | 20260506 02:08:15.497 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 02:08:15.497 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 02:08:15.497 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 02:08:15.497 | INFO | High Water Breakdown: | 20260506 02:08:15.497 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 02:08:15.497 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 02:08:15.497 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 02:08:15.497 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 02:08:15.497 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 02:08:15.497 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 02:08:15.497 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 02:08:15.497 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 02:08:15.497 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 02:08:15.497 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:08:15.497 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 02:08:15.497 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:08:15.497 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 02:08:15.497 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 02:08:15.497 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 02:08:15.497 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 02:08:15.497 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:08:15.497 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 02:08:15.497 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:08:15.497 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:08:15.497 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:08:15.497 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 02:08:15.497 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 02:08:15.497 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:08:15.497 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 02:08:15.497 | INFO | ------------------------------------------ | 20260506 02:08:15.497 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:08:15.497 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 02:08:15.497 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:08:15.497 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:08:15.497 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.39e+01, 1.36e+01 / 6.44e-01 | 20260506 02:08:15.497 | INFO | PerfReport: 0.30%, AggregateStats -- 9.22e-02 / 1.05e-01, 1.06e-02 / 4.22e-03 | 20260506 02:08:15.498 | INFO | PerfReport: 11.83%, Baseline -- 3.70e+00 / 4.50e+00, 5.65e-01 / 1.78e-01 | 20260506 02:08:15.498 | INFO | PerfReport: 3.29%, Histogram -- 1.03e+00 / 1.53e+00, 1.92e-01 / 8.25e-02 | 20260506 02:08:15.498 | INFO | PerfReport: 15.99%, DME -- 4.00e+01 / 4.38e+01, 3.74e+00 / 8.38e-01 | 20260506 02:08:15.498 | INFO | PerfReport: 57.48%, FrameLabeling -- 1.80e+01 / 1.97e+01, 1.48e+00 / 1.10e+00 | 20260506 02:08:15.498 | INFO | PerfReport: 7.04%, PulseAccumulating -- 2.20e+00 / 2.68e+00, 4.14e-01 / 2.47e-01 | 20260506 02:08:15.498 | INFO | PerfReport: 3.03%, Metrics -- 9.45e-01 / 1.20e+00, 2.54e-01 / 1.11e-01 | 20260506 02:08:15.498 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.27e-01 / 3.78e-01, 5.46e-02 / 2.63e-02 | 20260506 02:08:15.498 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 02:08:15.498 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:08:15.498 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:08:15.498 | INFO | PerfReport: Full Execution -- 9.79e+01 / -nan, 2.42e+01 / 0.00e+00 | 20260506 02:08:15.498 | INFO | PerfReport: 22.38%, Upload -- 2.19e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260506 02:08:15.498 | INFO | PerfReport: 21.95%, Download -- 2.15e+01 / 0.00e+00, 7.63e+00 / 0.00e+00 | 20260506 02:08:15.498 | INFO | PerfReport: 31.92%, Compute -- 3.12e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 02:08:15.498 | INFO | PerfReport: 2.08%, LockUpload -- 2.03e+00 / 0.00e+00, 7.97e+00 / 0.00e+00 | 20260506 02:08:15.498 | INFO | PerfReport: 1.31%, LockDownload -- 1.28e+00 / 0.00e+00, 2.84e+00 / 0.00e+00 | 20260506 02:08:15.498 | INFO | PerfReport: 20.35%, LockCompute -- 1.99e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260506 02:08:15.498 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.86e-02 / 0.00e+00 | 20260506 02:08:15.612 | INFO | Starting a round of Baz IO | 20260506 02:08:18.757 | INFO | Populating RTMetrics report | 20260506 02:08:18.757 | INFO | RTMetrics report populated | 20260506 02:08:18.757 | INFO | Opening tmp RTMetrics json file | 20260506 02:08:18.758 | INFO | Serializing RTMetrics to json | 20260506 02:08:18.759 | INFO | Serialization done, swapping tmp file now... | 20260506 02:08:18.760 | INFO | RTMetrics done | 20260506 02:08:18.832 | INFO | Finished a round of Baz IO: Wrote 1.802 GiB in 3.219 seconds | 20260506 02:08:18.832 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.28% / 0.00% / 78.72% | 20260506 02:08:18.832 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0733852% / 0% / 6.75519e-05% | 20260506 02:08:18.832 | INFO | Baz IO overall: | 20260506 02:08:18.832 | INFO | BytesWritten : 4553912095281 | 20260506 02:08:18.832 | INFO | NumEvents : 2138770866624 | 20260506 02:08:20.628 | INFO | Analyzing chunk frames = [8110080,8110592) | 20260506 02:08:20.628 | INFO | MemInfo: { | 20260506 02:08:20.628 | INFO | "gpu" : " 17 GB", | 20260506 02:08:20.628 | INFO | "huge" : "", | 20260506 02:08:20.628 | INFO | "huge_shared" : " 27 GB", | 20260506 02:08:20.628 | INFO | "malloc" : " 14 GB", | 20260506 02:08:20.628 | INFO | "pinned" : " 29 GB" | 20260506 02:08:20.628 | INFO | } | 20260506 02:08:20.628 | INFO | Perf report for the last 5.13192 seconds: | 20260506 02:08:20.628 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:08:20.628 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:20.628 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:20.628 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:20.628 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260506 02:08:20.628 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:20.628 | INFO | RT_METRICS : 42.8%, 1.0, 22.4ms, 0 | 20260506 02:08:20.628 | INFO | PRE_HQ : 59.6%, 1.7, 53.1ms, 0 | 20260506 02:08:20.628 | INFO | BAZWRITER : 61.8%, 3.5, 110.1ms, 0 | 20260506 02:08:20.628 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:08:20.628 | INFO | Uploaded 36GiB to GPU in 2.18097s (16.5064GiB/s) | 20260506 02:08:20.628 | INFO | Downloaded 38GiB to GPU in 2.24565s (16.9216GiB/s) | 20260506 02:08:25.772 | INFO | Analyzing chunk frames = [8110592,8111104) | 20260506 02:08:25.772 | INFO | MemInfo: { | 20260506 02:08:25.772 | INFO | "gpu" : " 17 GB", | 20260506 02:08:25.772 | INFO | "huge" : "", | 20260506 02:08:25.772 | INFO | "huge_shared" : " 27 GB", | 20260506 02:08:25.772 | INFO | "malloc" : " 14 GB", | 20260506 02:08:25.772 | INFO | "pinned" : " 29 GB" | 20260506 02:08:25.772 | INFO | } | 20260506 02:08:25.772 | INFO | Perf report for the last 5.14428 seconds: | 20260506 02:08:25.772 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:08:25.772 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:25.772 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:25.772 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:25.772 | INFO | ANALYSIS : 61.6%, 3.0, 96.3ms, 0 | 20260506 02:08:25.772 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:25.772 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:25.772 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 02:08:25.773 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:25.773 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:25.773 | INFO | Uploaded 36GiB to GPU in 2.11954s (16.9848GiB/s) | 20260506 02:08:25.773 | INFO | Downloaded 34GiB to GPU in 2.03976s (16.6686GiB/s) | 20260506 02:08:30.914 | INFO | Analyzing chunk frames = [8111104,8111616) | 20260506 02:08:30.914 | INFO | MemInfo: { | 20260506 02:08:30.914 | INFO | "gpu" : " 17 GB", | 20260506 02:08:30.914 | INFO | "huge" : "", | 20260506 02:08:30.914 | INFO | "huge_shared" : " 27 GB", | 20260506 02:08:30.914 | INFO | "malloc" : " 14 GB", | 20260506 02:08:30.914 | INFO | "pinned" : " 29 GB" | 20260506 02:08:30.914 | INFO | } | 20260506 02:08:30.914 | INFO | Perf report for the last 5.14134 seconds: | 20260506 02:08:30.914 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:08:30.914 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:30.914 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:30.914 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:30.914 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 02:08:30.914 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:30.914 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:30.914 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 02:08:30.914 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:30.914 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:30.914 | INFO | Uploaded 36GiB to GPU in 2.0898s (17.2265GiB/s) | 20260506 02:08:30.914 | INFO | Downloaded 34GiB to GPU in 2.02667s (16.7763GiB/s) | 20260506 02:08:36.053 | INFO | Analyzing chunk frames = [8111616,8112128) | 20260506 02:08:36.053 | INFO | MemInfo: { | 20260506 02:08:36.053 | INFO | "gpu" : " 17 GB", | 20260506 02:08:36.053 | INFO | "huge" : "", | 20260506 02:08:36.053 | INFO | "huge_shared" : " 27 GB", | 20260506 02:08:36.053 | INFO | "malloc" : " 14 GB", | 20260506 02:08:36.053 | INFO | "pinned" : " 29 GB" | 20260506 02:08:36.053 | INFO | } | 20260506 02:08:36.053 | INFO | Perf report for the last 5.13968 seconds: | 20260506 02:08:36.053 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:08:36.053 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:36.053 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:36.053 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:36.053 | INFO | ANALYSIS : 61.1%, 3.0, 94.8ms, 0 | 20260506 02:08:36.053 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:36.053 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:36.053 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 02:08:36.053 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:36.053 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:36.054 | INFO | Uploaded 36GiB to GPU in 2.07932s (17.3134GiB/s) | 20260506 02:08:36.054 | INFO | Downloaded 34GiB to GPU in 2.00479s (16.9594GiB/s) | 20260506 02:08:41.191 | INFO | Analyzing chunk frames = [8112128,8112640) | 20260506 02:08:41.191 | INFO | MemInfo: { | 20260506 02:08:41.191 | INFO | "gpu" : " 17 GB", | 20260506 02:08:41.191 | INFO | "huge" : "", | 20260506 02:08:41.191 | INFO | "huge_shared" : " 27 GB", | 20260506 02:08:41.191 | INFO | "malloc" : " 14 GB", | 20260506 02:08:41.191 | INFO | "pinned" : " 29 GB" | 20260506 02:08:41.191 | INFO | } | 20260506 02:08:41.191 | INFO | Perf report for the last 5.1374 seconds: | 20260506 02:08:41.191 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:08:41.191 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:41.191 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:41.191 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:41.191 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 02:08:41.191 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:41.191 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:41.191 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 02:08:41.191 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:41.191 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:41.191 | INFO | Uploaded 36GiB to GPU in 2.07938s (17.3129GiB/s) | 20260506 02:08:41.191 | INFO | Downloaded 34GiB to GPU in 2.02651s (16.7776GiB/s) | 20260506 02:08:46.330 | INFO | Analyzing chunk frames = [8112640,8113152) | 20260506 02:08:46.330 | INFO | MemInfo: { | 20260506 02:08:46.330 | INFO | "gpu" : " 17 GB", | 20260506 02:08:46.330 | INFO | "huge" : "", | 20260506 02:08:46.330 | INFO | "huge_shared" : " 27 GB", | 20260506 02:08:46.330 | INFO | "malloc" : " 14 GB", | 20260506 02:08:46.330 | INFO | "pinned" : " 29 GB" | 20260506 02:08:46.330 | INFO | } | 20260506 02:08:46.330 | INFO | Perf report for the last 5.13932 seconds: | 20260506 02:08:46.330 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:08:46.330 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:46.330 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:46.330 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:46.330 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 02:08:46.330 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:46.330 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:46.330 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 02:08:46.330 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:46.330 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:46.330 | INFO | Uploaded 36GiB to GPU in 2.0719s (17.3753GiB/s) | 20260506 02:08:46.330 | INFO | Downloaded 34GiB to GPU in 2.03504s (16.7073GiB/s) | 20260506 02:08:51.470 | INFO | Analyzing chunk frames = [8113152,8113664) | 20260506 02:08:51.470 | INFO | MemInfo: { | 20260506 02:08:51.470 | INFO | "gpu" : " 17 GB", | 20260506 02:08:51.470 | INFO | "huge" : "", | 20260506 02:08:51.471 | INFO | "huge_shared" : " 27 GB", | 20260506 02:08:51.471 | INFO | "malloc" : " 14 GB", | 20260506 02:08:51.471 | INFO | "pinned" : " 29 GB" | 20260506 02:08:51.471 | INFO | } | 20260506 02:08:51.471 | INFO | Perf report for the last 5.14048 seconds: | 20260506 02:08:51.471 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:08:51.471 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:51.471 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:51.471 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:51.471 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260506 02:08:51.471 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:51.471 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:51.471 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 02:08:51.471 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:51.471 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:51.471 | INFO | Uploaded 36GiB to GPU in 2.06535s (17.4305GiB/s) | 20260506 02:08:51.471 | INFO | Downloaded 34GiB to GPU in 2.06971s (16.4274GiB/s) | 20260506 02:08:56.613 | INFO | Analyzing chunk frames = [8113664,8114176) | 20260506 02:08:56.613 | INFO | MemInfo: { | 20260506 02:08:56.613 | INFO | "gpu" : " 17 GB", | 20260506 02:08:56.613 | INFO | "huge" : "", | 20260506 02:08:56.613 | INFO | "huge_shared" : " 27 GB", | 20260506 02:08:56.613 | INFO | "malloc" : " 14 GB", | 20260506 02:08:56.614 | INFO | "pinned" : " 29 GB" | 20260506 02:08:56.614 | INFO | } | 20260506 02:08:56.614 | INFO | Perf report for the last 5.14291 seconds: | 20260506 02:08:56.614 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:08:56.614 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:56.614 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:56.614 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:56.614 | INFO | ANALYSIS : 61.4%, 3.0, 96.0ms, 0 | 20260506 02:08:56.614 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:56.614 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:56.614 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:08:56.614 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:56.614 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:08:56.614 | INFO | Uploaded 36GiB to GPU in 2.06606s (17.4245GiB/s) | 20260506 02:08:56.614 | INFO | Downloaded 34GiB to GPU in 2.02654s (16.7774GiB/s) | 20260506 02:08:56.737 | INFO | Starting a round of Baz IO | 20260506 02:08:59.863 | INFO | Populating RTMetrics report | 20260506 02:08:59.863 | INFO | RTMetrics report populated | 20260506 02:08:59.863 | INFO | Opening tmp RTMetrics json file | 20260506 02:08:59.863 | INFO | Serializing RTMetrics to json | 20260506 02:08:59.865 | INFO | Serialization done, swapping tmp file now... | 20260506 02:08:59.865 | INFO | RTMetrics done | 20260506 02:08:59.938 | INFO | Finished a round of Baz IO: Wrote 1.803 GiB in 3.2 seconds | 20260506 02:08:59.938 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.28% / 0.00% / 78.72% | 20260506 02:08:59.938 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0731508% / 0% / 7.94729e-05% | 20260506 02:08:59.938 | INFO | Baz IO overall: | 20260506 02:08:59.938 | INFO | BytesWritten : 4555847692403 | 20260506 02:08:59.938 | INFO | NumEvents : 2139651184876 | 20260506 02:09:01.757 | INFO | Analyzing chunk frames = [8114176,8114688) | 20260506 02:09:01.757 | INFO | MemInfo: { | 20260506 02:09:01.757 | INFO | "gpu" : " 17 GB", | 20260506 02:09:01.757 | INFO | "huge" : "", | 20260506 02:09:01.757 | INFO | "huge_shared" : " 27 GB", | 20260506 02:09:01.757 | INFO | "malloc" : " 14 GB", | 20260506 02:09:01.757 | INFO | "pinned" : " 29 GB" | 20260506 02:09:01.757 | INFO | } | 20260506 02:09:01.757 | INFO | Perf report for the last 5.14321 seconds: | 20260506 02:09:01.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:09:01.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:01.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:01.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:01.757 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260506 02:09:01.757 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:01.757 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260506 02:09:01.757 | INFO | PRE_HQ : 59.0%, 1.8, 53.8ms, 0 | 20260506 02:09:01.757 | INFO | BAZWRITER : 61.4%, 3.5, 109.4ms, 0 | 20260506 02:09:01.757 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:09:01.757 | INFO | Uploaded 36GiB to GPU in 2.08947s (17.2293GiB/s) | 20260506 02:09:01.757 | INFO | Downloaded 38GiB to GPU in 2.21639s (17.145GiB/s) | 20260506 02:09:06.886 | INFO | Analyzing chunk frames = [8114688,8115200) | 20260506 02:09:06.886 | INFO | MemInfo: { | 20260506 02:09:06.886 | INFO | "gpu" : " 17 GB", | 20260506 02:09:06.887 | INFO | "huge" : "", | 20260506 02:09:06.887 | INFO | "huge_shared" : " 27 GB", | 20260506 02:09:06.887 | INFO | "malloc" : " 14 GB", | 20260506 02:09:06.887 | INFO | "pinned" : " 29 GB" | 20260506 02:09:06.887 | INFO | } | 20260506 02:09:06.887 | INFO | Perf report for the last 5.12986 seconds: | 20260506 02:09:06.887 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:09:06.887 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:06.887 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:06.887 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:06.887 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260506 02:09:06.887 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:06.887 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:06.887 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:09:06.887 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:06.887 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:06.887 | INFO | Uploaded 36GiB to GPU in 1.96022s (18.3653GiB/s) | 20260506 02:09:06.887 | INFO | Downloaded 34GiB to GPU in 1.99167s (17.0711GiB/s) | 20260506 02:09:12.023 | INFO | Analyzing chunk frames = [8115200,8115712) | 20260506 02:09:12.023 | INFO | MemInfo: { | 20260506 02:09:12.023 | INFO | "gpu" : " 17 GB", | 20260506 02:09:12.023 | INFO | "huge" : "", | 20260506 02:09:12.023 | INFO | "huge_shared" : " 27 GB", | 20260506 02:09:12.024 | INFO | "malloc" : " 14 GB", | 20260506 02:09:12.024 | INFO | "pinned" : " 29 GB" | 20260506 02:09:12.024 | INFO | } | 20260506 02:09:12.024 | INFO | Perf report for the last 5.13692 seconds: | 20260506 02:09:12.024 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:09:12.024 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:12.024 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:12.024 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:12.024 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 02:09:12.024 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:12.024 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:12.024 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:09:12.024 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:12.024 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:12.024 | INFO | Uploaded 36GiB to GPU in 2.1s (17.1429GiB/s) | 20260506 02:09:12.024 | INFO | Downloaded 34GiB to GPU in 2.02612s (16.7808GiB/s) | 20260506 02:09:17.168 | INFO | Analyzing chunk frames = [8115712,8116224) | 20260506 02:09:17.168 | INFO | MemInfo: { | 20260506 02:09:17.168 | INFO | "gpu" : " 17 GB", | 20260506 02:09:17.168 | INFO | "huge" : "", | 20260506 02:09:17.168 | INFO | "huge_shared" : " 27 GB", | 20260506 02:09:17.168 | INFO | "malloc" : " 14 GB", | 20260506 02:09:17.168 | INFO | "pinned" : " 29 GB" | 20260506 02:09:17.168 | INFO | } | 20260506 02:09:17.168 | INFO | Perf report for the last 5.14481 seconds: | 20260506 02:09:17.168 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:09:17.168 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:17.168 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:17.168 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:17.168 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 02:09:17.168 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:17.168 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:17.168 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 02:09:17.168 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:17.169 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:17.169 | INFO | Uploaded 36GiB to GPU in 2.1418s (16.8083GiB/s) | 20260506 02:09:17.169 | INFO | Downloaded 34GiB to GPU in 2.02451s (16.7942GiB/s) | 20260506 02:09:22.311 | INFO | Analyzing chunk frames = [8116224,8116736) | 20260506 02:09:22.311 | INFO | MemInfo: { | 20260506 02:09:22.311 | INFO | "gpu" : " 17 GB", | 20260506 02:09:22.311 | INFO | "huge" : "", | 20260506 02:09:22.311 | INFO | "huge_shared" : " 27 GB", | 20260506 02:09:22.311 | INFO | "malloc" : " 14 GB", | 20260506 02:09:22.311 | INFO | "pinned" : " 29 GB" | 20260506 02:09:22.311 | INFO | } | 20260506 02:09:22.311 | INFO | Perf report for the last 5.14276 seconds: | 20260506 02:09:22.311 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:09:22.311 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:22.311 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:22.311 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:22.311 | INFO | ANALYSIS : 60.6%, 3.0, 94.7ms, 0 | 20260506 02:09:22.311 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:22.311 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:22.311 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 02:09:22.311 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:22.311 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:22.311 | INFO | Uploaded 36GiB to GPU in 2.07441s (17.3544GiB/s) | 20260506 02:09:22.312 | INFO | Downloaded 34GiB to GPU in 2.02772s (16.7676GiB/s) | 20260506 02:09:27.452 | INFO | Analyzing chunk frames = [8116736,8117248) | 20260506 02:09:27.452 | INFO | MemInfo: { | 20260506 02:09:27.452 | INFO | "gpu" : " 17 GB", | 20260506 02:09:27.452 | INFO | "huge" : "", | 20260506 02:09:27.452 | INFO | "huge_shared" : " 27 GB", | 20260506 02:09:27.452 | INFO | "malloc" : " 14 GB", | 20260506 02:09:27.452 | INFO | "pinned" : " 29 GB" | 20260506 02:09:27.452 | INFO | } | 20260506 02:09:27.452 | INFO | Perf report for the last 5.14082 seconds: | 20260506 02:09:27.452 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:09:27.452 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:27.452 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:27.452 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:27.452 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 02:09:27.452 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:27.452 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:27.452 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:09:27.452 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:27.452 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:27.452 | INFO | Uploaded 36GiB to GPU in 2.09274s (17.2024GiB/s) | 20260506 02:09:27.452 | INFO | Downloaded 34GiB to GPU in 2.03865s (16.6777GiB/s) | 20260506 02:09:32.585 | INFO | Analyzing chunk frames = [8117248,8117760) | 20260506 02:09:32.585 | INFO | MemInfo: { | 20260506 02:09:32.585 | INFO | "gpu" : " 17 GB", | 20260506 02:09:32.585 | INFO | "huge" : "", | 20260506 02:09:32.585 | INFO | "huge_shared" : " 27 GB", | 20260506 02:09:32.585 | INFO | "malloc" : " 14 GB", | 20260506 02:09:32.585 | INFO | "pinned" : " 29 GB" | 20260506 02:09:32.585 | INFO | } | 20260506 02:09:32.585 | INFO | Perf report for the last 5.13318 seconds: | 20260506 02:09:32.585 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:09:32.585 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:32.585 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:32.585 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:32.585 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260506 02:09:32.585 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:32.585 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:32.585 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260506 02:09:32.585 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:32.585 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:32.586 | INFO | Uploaded 36GiB to GPU in 2.09217s (17.207GiB/s) | 20260506 02:09:32.586 | INFO | Downloaded 34GiB to GPU in 2.09672s (16.2158GiB/s) | 20260506 02:09:37.723 | INFO | Analyzing chunk frames = [8117760,8118272) | 20260506 02:09:37.723 | INFO | MemInfo: { | 20260506 02:09:37.723 | INFO | "gpu" : " 17 GB", | 20260506 02:09:37.723 | INFO | "huge" : "", | 20260506 02:09:37.723 | INFO | "huge_shared" : " 27 GB", | 20260506 02:09:37.723 | INFO | "malloc" : " 14 GB", | 20260506 02:09:37.723 | INFO | "pinned" : " 29 GB" | 20260506 02:09:37.723 | INFO | } | 20260506 02:09:37.723 | INFO | Perf report for the last 5.13758 seconds: | 20260506 02:09:37.723 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:09:37.723 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:37.723 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:37.723 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:37.723 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 02:09:37.723 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:37.723 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:37.723 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:09:37.723 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:37.723 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:37.723 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 02:09:37.723 | INFO | Total mem usage: 15.3 GiB | 20260506 02:09:37.723 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 02:09:37.723 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.2%) | 20260506 02:09:37.723 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 02:09:37.723 | INFO | High Water Breakdown: | 20260506 02:09:37.723 | INFO | 60.9% : 8.54 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 02:09:37.723 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 02:09:37.723 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 02:09:37.723 | INFO | ------------------------------------------ | 20260506 02:09:37.723 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:09:37.723 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 02:09:37.723 | INFO | Total mem usage: 26.5 GiB | 20260506 02:09:37.723 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 02:09:37.723 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 02:09:37.723 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 02:09:37.723 | INFO | High Water Breakdown: | 20260506 02:09:37.723 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:09:37.723 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:09:37.723 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:09:37.723 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:09:37.723 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:09:37.723 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:09:37.723 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 02:09:37.723 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 02:09:37.723 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:09:37.723 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 02:09:37.724 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 02:09:37.724 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 02:09:37.724 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 02:09:37.724 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 02:09:37.724 | INFO | ------------------------------------------ | 20260506 02:09:37.724 | INFO | Uploaded 36GiB to GPU in 2.05183s (17.5453GiB/s) | 20260506 02:09:37.724 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:09:37.724 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 02:09:37.724 | INFO | Total mem usage: 25.6 GiB | 20260506 02:09:37.724 | INFO | Downloaded 34GiB to GPU in 2.02384s (16.7998GiB/s) | 20260506 02:09:37.724 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 02:09:37.724 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 02:09:37.724 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 02:09:37.724 | INFO | High Water Breakdown: | 20260506 02:09:37.724 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 02:09:37.724 | INFO | ------------------------------------------ | 20260506 02:09:37.724 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:09:37.724 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 02:09:37.724 | INFO | Total mem usage: 15.7 GiB | 20260506 02:09:37.724 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 02:09:37.724 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 02:09:37.724 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 02:09:37.724 | INFO | High Water Breakdown: | 20260506 02:09:37.724 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 02:09:37.724 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 02:09:37.724 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 02:09:37.724 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 02:09:37.724 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 02:09:37.724 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 02:09:37.724 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 02:09:37.724 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 02:09:37.724 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 02:09:37.724 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:09:37.724 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 02:09:37.724 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:09:37.724 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 02:09:37.724 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 02:09:37.724 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 02:09:37.724 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 02:09:37.724 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:09:37.724 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 02:09:37.724 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:09:37.724 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:09:37.724 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:09:37.724 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 02:09:37.724 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 02:09:37.724 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:09:37.724 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 02:09:37.724 | INFO | ------------------------------------------ | 20260506 02:09:37.724 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:09:37.724 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 02:09:37.724 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:09:37.724 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:09:37.724 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.38e+01, 1.35e+01 / 7.98e-01 | 20260506 02:09:37.724 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.04e-01, 1.04e-02 / 3.29e-03 | 20260506 02:09:37.724 | INFO | PerfReport: 11.86%, Baseline -- 3.71e+00 / 4.52e+00, 5.81e-01 / 2.48e-01 | 20260506 02:09:37.724 | INFO | PerfReport: 3.30%, Histogram -- 1.03e+00 / 1.54e+00, 1.96e-01 / 8.77e-02 | 20260506 02:09:37.724 | INFO | PerfReport: 15.97%, DME -- 4.00e+01 / 4.39e+01, 3.83e+00 / 9.86e-01 | 20260506 02:09:37.724 | INFO | PerfReport: 57.49%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.52e+00 / 1.22e+00 | 20260506 02:09:37.724 | INFO | PerfReport: 7.02%, PulseAccumulating -- 2.20e+00 / 2.74e+00, 4.16e-01 / 2.31e-01 | 20260506 02:09:37.724 | INFO | PerfReport: 3.02%, Metrics -- 9.45e-01 / 1.22e+00, 2.54e-01 / 9.71e-02 | 20260506 02:09:37.724 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.27e-01 / 3.80e-01, 5.47e-02 / 2.87e-02 | 20260506 02:09:37.724 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 02:09:37.724 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:09:37.724 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:09:37.724 | INFO | PerfReport: Full Execution -- 9.78e+01 / -nan, 2.42e+01 / 0.00e+00 | 20260506 02:09:37.724 | INFO | PerfReport: 22.21%, Upload -- 2.17e+01 / 0.00e+00, 7.25e+00 / 0.00e+00 | 20260506 02:09:37.724 | INFO | PerfReport: 21.92%, Download -- 2.14e+01 / 0.00e+00, 7.62e+00 / 0.00e+00 | 20260506 02:09:37.724 | INFO | PerfReport: 32.01%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 02:09:37.724 | INFO | PerfReport: 2.05%, LockUpload -- 2.00e+00 / 0.00e+00, 7.92e+00 / 0.00e+00 | 20260506 02:09:37.724 | INFO | PerfReport: 1.32%, LockDownload -- 1.29e+00 / 0.00e+00, 3.00e+00 / 0.00e+00 | 20260506 02:09:37.724 | INFO | PerfReport: 20.47%, LockCompute -- 2.00e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260506 02:09:37.724 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 2.12e-02 / 0.00e+00 | 20260506 02:09:37.849 | INFO | Starting a round of Baz IO | 20260506 02:09:40.973 | INFO | Populating RTMetrics report | 20260506 02:09:40.973 | INFO | RTMetrics report populated | 20260506 02:09:40.973 | INFO | Opening tmp RTMetrics json file | 20260506 02:09:40.974 | INFO | Serializing RTMetrics to json | 20260506 02:09:40.975 | INFO | Serialization done, swapping tmp file now... | 20260506 02:09:40.976 | INFO | RTMetrics done | 20260506 02:09:41.048 | INFO | Finished a round of Baz IO: Wrote 1.805 GiB in 3.199 seconds | 20260506 02:09:41.049 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.28% / 0.00% / 78.72% | 20260506 02:09:41.049 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0734647% / 0% / 5.96046e-05% | 20260506 02:09:41.049 | INFO | Baz IO overall: | 20260506 02:09:41.049 | INFO | BytesWritten : 4557786189640 | 20260506 02:09:41.049 | INFO | NumEvents : 2140533895895 | 20260506 02:09:42.864 | INFO | Analyzing chunk frames = [8118272,8118784) | 20260506 02:09:42.864 | INFO | MemInfo: { | 20260506 02:09:42.864 | INFO | "gpu" : " 17 GB", | 20260506 02:09:42.864 | INFO | "huge" : "", | 20260506 02:09:42.864 | INFO | "huge_shared" : " 27 GB", | 20260506 02:09:42.864 | INFO | "malloc" : " 14 GB", | 20260506 02:09:42.864 | INFO | "pinned" : " 29 GB" | 20260506 02:09:42.864 | INFO | } | 20260506 02:09:42.864 | INFO | Perf report for the last 5.14103 seconds: | 20260506 02:09:42.864 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:09:42.864 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:42.864 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:42.864 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:42.864 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260506 02:09:42.864 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:42.864 | INFO | RT_METRICS : 42.8%, 1.0, 22.4ms, 0 | 20260506 02:09:42.864 | INFO | PRE_HQ : 59.1%, 1.7, 53.2ms, 0 | 20260506 02:09:42.864 | INFO | BAZWRITER : 61.4%, 3.5, 109.9ms, 0 | 20260506 02:09:42.864 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:09:42.864 | INFO | Uploaded 36GiB to GPU in 2.09673s (17.1696GiB/s) | 20260506 02:09:42.864 | INFO | Downloaded 38GiB to GPU in 2.23001s (17.0403GiB/s) | 20260506 02:09:48.003 | INFO | Analyzing chunk frames = [8118784,8119296) | 20260506 02:09:48.003 | INFO | MemInfo: { | 20260506 02:09:48.003 | INFO | "gpu" : " 17 GB", | 20260506 02:09:48.003 | INFO | "huge" : "", | 20260506 02:09:48.004 | INFO | "huge_shared" : " 27 GB", | 20260506 02:09:48.004 | INFO | "malloc" : " 14 GB", | 20260506 02:09:48.004 | INFO | "pinned" : " 29 GB" | 20260506 02:09:48.004 | INFO | } | 20260506 02:09:48.004 | INFO | Perf report for the last 5.13977 seconds: | 20260506 02:09:48.004 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:09:48.004 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:48.004 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:48.004 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:48.004 | INFO | ANALYSIS : 60.7%, 3.0, 94.8ms, 0 | 20260506 02:09:48.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:48.004 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:48.004 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:09:48.004 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:48.004 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:48.004 | INFO | Uploaded 36GiB to GPU in 1.95945s (18.3725GiB/s) | 20260506 02:09:48.004 | INFO | Downloaded 34GiB to GPU in 1.98745s (17.1074GiB/s) | 20260506 02:09:53.140 | INFO | Analyzing chunk frames = [8119296,8119808) | 20260506 02:09:53.140 | INFO | MemInfo: { | 20260506 02:09:53.140 | INFO | "gpu" : " 17 GB", | 20260506 02:09:53.140 | INFO | "huge" : "", | 20260506 02:09:53.140 | INFO | "huge_shared" : " 27 GB", | 20260506 02:09:53.140 | INFO | "malloc" : " 14 GB", | 20260506 02:09:53.140 | INFO | "pinned" : " 29 GB" | 20260506 02:09:53.140 | INFO | } | 20260506 02:09:53.140 | INFO | Perf report for the last 5.13627 seconds: | 20260506 02:09:53.140 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:09:53.140 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:53.140 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:53.140 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:53.140 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 02:09:53.140 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:53.140 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:53.140 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 02:09:53.140 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:53.140 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:53.140 | INFO | Uploaded 36GiB to GPU in 2.09067s (17.2194GiB/s) | 20260506 02:09:53.140 | INFO | Downloaded 34GiB to GPU in 2.02695s (16.774GiB/s) | 20260506 02:09:58.276 | INFO | Analyzing chunk frames = [8119808,8120320) | 20260506 02:09:58.276 | INFO | MemInfo: { | 20260506 02:09:58.276 | INFO | "gpu" : " 17 GB", | 20260506 02:09:58.276 | INFO | "huge" : "", | 20260506 02:09:58.276 | INFO | "huge_shared" : " 27 GB", | 20260506 02:09:58.277 | INFO | "malloc" : " 14 GB", | 20260506 02:09:58.277 | INFO | "pinned" : " 29 GB" | 20260506 02:09:58.277 | INFO | } | 20260506 02:09:58.277 | INFO | Perf report for the last 5.13669 seconds: | 20260506 02:09:58.277 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:09:58.277 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:58.277 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:58.277 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:58.277 | INFO | ANALYSIS : 61.6%, 3.0, 95.6ms, 0 | 20260506 02:09:58.277 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:58.277 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:58.277 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 02:09:58.277 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:58.277 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:09:58.277 | INFO | Uploaded 36GiB to GPU in 2.13353s (16.8735GiB/s) | 20260506 02:09:58.277 | INFO | Downloaded 34GiB to GPU in 2.01769s (16.8509GiB/s) | 20260506 02:10:03.429 | INFO | Analyzing chunk frames = [8120320,8120832) | 20260506 02:10:03.429 | INFO | MemInfo: { | 20260506 02:10:03.429 | INFO | "gpu" : " 17 GB", | 20260506 02:10:03.429 | INFO | "huge" : "", | 20260506 02:10:03.429 | INFO | "huge_shared" : " 27 GB", | 20260506 02:10:03.429 | INFO | "malloc" : " 14 GB", | 20260506 02:10:03.429 | INFO | "pinned" : " 29 GB" | 20260506 02:10:03.429 | INFO | } | 20260506 02:10:03.429 | INFO | Perf report for the last 5.15256 seconds: | 20260506 02:10:03.429 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:10:03.429 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:03.429 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:03.429 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:03.429 | INFO | ANALYSIS : 60.8%, 3.0, 95.1ms, 0 | 20260506 02:10:03.429 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:03.429 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:03.429 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 02:10:03.429 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:03.429 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:03.430 | INFO | Uploaded 36GiB to GPU in 2.09541s (17.1804GiB/s) | 20260506 02:10:03.430 | INFO | Downloaded 34GiB to GPU in 2.05212s (16.5682GiB/s) | 20260506 02:10:08.564 | INFO | Analyzing chunk frames = [8120832,8121344) | 20260506 02:10:08.564 | INFO | MemInfo: { | 20260506 02:10:08.564 | INFO | "gpu" : " 17 GB", | 20260506 02:10:08.564 | INFO | "huge" : "", | 20260506 02:10:08.564 | INFO | "huge_shared" : " 27 GB", | 20260506 02:10:08.564 | INFO | "malloc" : " 14 GB", | 20260506 02:10:08.564 | INFO | "pinned" : " 29 GB" | 20260506 02:10:08.564 | INFO | } | 20260506 02:10:08.564 | INFO | Perf report for the last 5.13516 seconds: | 20260506 02:10:08.564 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:10:08.564 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:08.564 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:08.564 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:08.564 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260506 02:10:08.564 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:08.564 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:08.564 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 02:10:08.564 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:08.564 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:08.565 | INFO | Uploaded 36GiB to GPU in 2.05058s (17.556GiB/s) | 20260506 02:10:08.565 | INFO | Downloaded 34GiB to GPU in 2.02922s (16.7552GiB/s) | 20260506 02:10:13.699 | INFO | Analyzing chunk frames = [8121344,8121856) | 20260506 02:10:13.699 | INFO | MemInfo: { | 20260506 02:10:13.699 | INFO | "gpu" : " 17 GB", | 20260506 02:10:13.699 | INFO | "huge" : "", | 20260506 02:10:13.700 | INFO | "huge_shared" : " 27 GB", | 20260506 02:10:13.700 | INFO | "malloc" : " 14 GB", | 20260506 02:10:13.700 | INFO | "pinned" : " 29 GB" | 20260506 02:10:13.700 | INFO | } | 20260506 02:10:13.700 | INFO | Perf report for the last 5.13522 seconds: | 20260506 02:10:13.700 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:10:13.700 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:13.700 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:13.700 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:13.700 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260506 02:10:13.700 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:13.700 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:13.700 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 02:10:13.700 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:13.700 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:13.700 | INFO | Uploaded 36GiB to GPU in 2.08894s (17.2336GiB/s) | 20260506 02:10:13.700 | INFO | Downloaded 34GiB to GPU in 2.06421s (16.4712GiB/s) | 20260506 02:10:18.835 | INFO | Analyzing chunk frames = [8121856,8122368) | 20260506 02:10:18.835 | INFO | MemInfo: { | 20260506 02:10:18.835 | INFO | "gpu" : " 17 GB", | 20260506 02:10:18.835 | INFO | "huge" : "", | 20260506 02:10:18.835 | INFO | "huge_shared" : " 27 GB", | 20260506 02:10:18.835 | INFO | "malloc" : " 14 GB", | 20260506 02:10:18.835 | INFO | "pinned" : " 29 GB" | 20260506 02:10:18.835 | INFO | } | 20260506 02:10:18.835 | INFO | Perf report for the last 5.13546 seconds: | 20260506 02:10:18.835 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:10:18.835 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:18.835 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:18.835 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:18.835 | INFO | ANALYSIS : 61.5%, 3.0, 94.9ms, 0 | 20260506 02:10:18.835 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:18.835 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:18.835 | INFO | PRE_HQ : 26.6%, 1.0, 13.7ms, 0 | 20260506 02:10:18.835 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:18.835 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:18.836 | INFO | Uploaded 36GiB to GPU in 2.05022s (17.5591GiB/s) | 20260506 02:10:18.836 | INFO | Downloaded 34GiB to GPU in 2.02391s (16.7992GiB/s) | 20260506 02:10:18.960 | INFO | Starting a round of Baz IO | 20260506 02:10:22.095 | INFO | Populating RTMetrics report | 20260506 02:10:22.095 | INFO | RTMetrics report populated | 20260506 02:10:22.095 | INFO | Opening tmp RTMetrics json file | 20260506 02:10:22.096 | INFO | Serializing RTMetrics to json | 20260506 02:10:22.097 | INFO | Serialization done, swapping tmp file now... | 20260506 02:10:22.098 | INFO | RTMetrics done | 20260506 02:10:22.170 | INFO | Finished a round of Baz IO: Wrote 1.804 GiB in 3.209 seconds | 20260506 02:10:22.170 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.28% / 0.00% / 78.72% | 20260506 02:10:22.170 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.073421% / 0% / 7.54992e-05% | 20260506 02:10:22.170 | INFO | Baz IO overall: | 20260506 02:10:22.170 | INFO | BytesWritten : 4559723024563 | 20260506 02:10:22.170 | INFO | NumEvents : 2141415199232 | 20260506 02:10:23.973 | INFO | Analyzing chunk frames = [8122368,8122880) | 20260506 02:10:23.973 | INFO | MemInfo: { | 20260506 02:10:23.973 | INFO | "gpu" : " 17 GB", | 20260506 02:10:23.973 | INFO | "huge" : "", | 20260506 02:10:23.973 | INFO | "huge_shared" : " 27 GB", | 20260506 02:10:23.973 | INFO | "malloc" : " 14 GB", | 20260506 02:10:23.973 | INFO | "pinned" : " 29 GB" | 20260506 02:10:23.973 | INFO | } | 20260506 02:10:23.973 | INFO | Perf report for the last 5.13792 seconds: | 20260506 02:10:23.973 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:10:23.973 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:23.973 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:23.973 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:23.973 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260506 02:10:23.973 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:23.973 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260506 02:10:23.973 | INFO | PRE_HQ : 59.5%, 1.7, 53.7ms, 0 | 20260506 02:10:23.973 | INFO | BAZWRITER : 61.7%, 3.5, 110.1ms, 0 | 20260506 02:10:23.973 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:10:23.973 | INFO | Uploaded 36GiB to GPU in 2.18835s (16.4508GiB/s) | 20260506 02:10:23.973 | INFO | Downloaded 38GiB to GPU in 2.24069s (16.9591GiB/s) | 20260506 02:10:29.117 | INFO | Analyzing chunk frames = [8122880,8123392) | 20260506 02:10:29.117 | INFO | MemInfo: { | 20260506 02:10:29.117 | INFO | "gpu" : " 17 GB", | 20260506 02:10:29.117 | INFO | "huge" : "", | 20260506 02:10:29.117 | INFO | "huge_shared" : " 27 GB", | 20260506 02:10:29.117 | INFO | "malloc" : " 14 GB", | 20260506 02:10:29.117 | INFO | "pinned" : " 29 GB" | 20260506 02:10:29.117 | INFO | } | 20260506 02:10:29.118 | INFO | Perf report for the last 5.14458 seconds: | 20260506 02:10:29.118 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:10:29.118 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:29.118 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:29.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:29.118 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 02:10:29.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:29.118 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:29.118 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 02:10:29.118 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:29.118 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:29.118 | INFO | Uploaded 36GiB to GPU in 1.97839s (18.1966GiB/s) | 20260506 02:10:29.118 | INFO | Downloaded 34GiB to GPU in 1.98816s (17.1012GiB/s) | 20260506 02:10:34.259 | INFO | Analyzing chunk frames = [8123392,8123904) | 20260506 02:10:34.259 | INFO | MemInfo: { | 20260506 02:10:34.259 | INFO | "gpu" : " 17 GB", | 20260506 02:10:34.259 | INFO | "huge" : "", | 20260506 02:10:34.259 | INFO | "huge_shared" : " 27 GB", | 20260506 02:10:34.259 | INFO | "malloc" : " 14 GB", | 20260506 02:10:34.259 | INFO | "pinned" : " 29 GB" | 20260506 02:10:34.259 | INFO | } | 20260506 02:10:34.259 | INFO | Perf report for the last 5.14174 seconds: | 20260506 02:10:34.259 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:10:34.260 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:34.260 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:34.260 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:34.260 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 02:10:34.260 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:34.260 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:34.260 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 02:10:34.260 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:34.260 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:34.260 | INFO | Uploaded 36GiB to GPU in 2.0943s (17.1895GiB/s) | 20260506 02:10:34.260 | INFO | Downloaded 34GiB to GPU in 2.02466s (16.7929GiB/s) | 20260506 02:10:39.401 | INFO | Analyzing chunk frames = [8123904,8124416) | 20260506 02:10:39.401 | INFO | MemInfo: { | 20260506 02:10:39.401 | INFO | "gpu" : " 17 GB", | 20260506 02:10:39.401 | INFO | "huge" : "", | 20260506 02:10:39.401 | INFO | "huge_shared" : " 27 GB", | 20260506 02:10:39.402 | INFO | "malloc" : " 14 GB", | 20260506 02:10:39.402 | INFO | "pinned" : " 29 GB" | 20260506 02:10:39.402 | INFO | } | 20260506 02:10:39.402 | INFO | Perf report for the last 5.14218 seconds: | 20260506 02:10:39.402 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:10:39.402 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:39.402 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:39.402 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:39.402 | INFO | ANALYSIS : 61.2%, 3.0, 95.1ms, 0 | 20260506 02:10:39.402 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:39.402 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:39.402 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 02:10:39.402 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:39.402 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:39.402 | INFO | Uploaded 36GiB to GPU in 2.09755s (17.1629GiB/s) | 20260506 02:10:39.402 | INFO | Downloaded 34GiB to GPU in 2.02878s (16.7589GiB/s) | 20260506 02:10:44.537 | INFO | Analyzing chunk frames = [8124416,8124928) | 20260506 02:10:44.537 | INFO | MemInfo: { | 20260506 02:10:44.537 | INFO | "gpu" : " 17 GB", | 20260506 02:10:44.537 | INFO | "huge" : "", | 20260506 02:10:44.537 | INFO | "huge_shared" : " 27 GB", | 20260506 02:10:44.537 | INFO | "malloc" : " 14 GB", | 20260506 02:10:44.537 | INFO | "pinned" : " 29 GB" | 20260506 02:10:44.537 | INFO | } | 20260506 02:10:44.537 | INFO | Perf report for the last 5.13582 seconds: | 20260506 02:10:44.538 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:10:44.538 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:44.538 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:44.538 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:44.538 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260506 02:10:44.538 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:44.538 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:44.538 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:10:44.538 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:44.538 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:44.538 | INFO | Uploaded 36GiB to GPU in 2.06635s (17.4221GiB/s) | 20260506 02:10:44.538 | INFO | Downloaded 34GiB to GPU in 2.02868s (16.7596GiB/s) | 20260506 02:10:49.676 | INFO | Analyzing chunk frames = [8124928,8125440) | 20260506 02:10:49.676 | INFO | MemInfo: { | 20260506 02:10:49.676 | INFO | "gpu" : " 17 GB", | 20260506 02:10:49.676 | INFO | "huge" : "", | 20260506 02:10:49.676 | INFO | "huge_shared" : " 27 GB", | 20260506 02:10:49.676 | INFO | "malloc" : " 14 GB", | 20260506 02:10:49.676 | INFO | "pinned" : " 29 GB" | 20260506 02:10:49.676 | INFO | } | 20260506 02:10:49.676 | INFO | Perf report for the last 5.13906 seconds: | 20260506 02:10:49.677 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:10:49.677 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:49.677 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:49.677 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:49.677 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 02:10:49.677 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:49.677 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:49.677 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 02:10:49.677 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:49.677 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:49.677 | INFO | Uploaded 36GiB to GPU in 2.06886s (17.4009GiB/s) | 20260506 02:10:49.677 | INFO | Downloaded 34GiB to GPU in 2.03435s (16.713GiB/s) | 20260506 02:10:54.813 | INFO | Analyzing chunk frames = [8125440,8125952) | 20260506 02:10:54.813 | INFO | MemInfo: { | 20260506 02:10:54.813 | INFO | "gpu" : " 17 GB", | 20260506 02:10:54.813 | INFO | "huge" : "", | 20260506 02:10:54.813 | INFO | "huge_shared" : " 27 GB", | 20260506 02:10:54.813 | INFO | "malloc" : " 14 GB", | 20260506 02:10:54.813 | INFO | "pinned" : " 29 GB" | 20260506 02:10:54.813 | INFO | } | 20260506 02:10:54.813 | INFO | Perf report for the last 5.13618 seconds: | 20260506 02:10:54.813 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:10:54.813 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:54.813 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:54.813 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:54.813 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260506 02:10:54.813 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:54.813 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:54.813 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 02:10:54.813 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:54.813 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:54.813 | INFO | Uploaded 36GiB to GPU in 2.03254s (17.7118GiB/s) | 20260506 02:10:54.813 | INFO | Downloaded 34GiB to GPU in 2.05824s (16.519GiB/s) | 20260506 02:10:59.951 | INFO | Analyzing chunk frames = [8125952,8126464) | 20260506 02:10:59.951 | INFO | MemInfo: { | 20260506 02:10:59.951 | INFO | "gpu" : " 17 GB", | 20260506 02:10:59.951 | INFO | "huge" : "", | 20260506 02:10:59.951 | INFO | "huge_shared" : " 27 GB", | 20260506 02:10:59.951 | INFO | "malloc" : " 14 GB", | 20260506 02:10:59.951 | INFO | "pinned" : " 29 GB" | 20260506 02:10:59.951 | INFO | } | 20260506 02:10:59.951 | INFO | Perf report for the last 5.13834 seconds: | 20260506 02:10:59.951 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:10:59.951 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:59.951 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:59.951 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:59.951 | INFO | ANALYSIS : 61.4%, 3.0, 95.9ms, 0 | 20260506 02:10:59.951 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:59.951 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:59.951 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 02:10:59.951 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:59.951 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:10:59.951 | INFO | Uploaded 36GiB to GPU in 2.0578s (17.4944GiB/s) | 20260506 02:10:59.951 | INFO | Downloaded 34GiB to GPU in 2.02785s (16.7665GiB/s) | 20260506 02:10:59.951 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 02:10:59.951 | INFO | Total mem usage: 15.3 GiB | 20260506 02:10:59.951 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 02:10:59.951 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.21%) | 20260506 02:10:59.951 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 02:10:59.951 | INFO | High Water Breakdown: | 20260506 02:10:59.951 | INFO | 60.9% : 8.54 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 02:10:59.951 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 02:10:59.951 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 02:10:59.951 | INFO | ------------------------------------------ | 20260506 02:10:59.952 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:10:59.952 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 02:10:59.952 | INFO | Total mem usage: 26.5 GiB | 20260506 02:10:59.952 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 02:10:59.952 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 02:10:59.952 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 02:10:59.952 | INFO | High Water Breakdown: | 20260506 02:10:59.952 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:10:59.952 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:10:59.952 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:10:59.952 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:10:59.952 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:10:59.952 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:10:59.952 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 02:10:59.952 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 02:10:59.952 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:10:59.952 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 02:10:59.952 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 02:10:59.952 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 02:10:59.952 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 02:10:59.952 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 02:10:59.952 | INFO | ------------------------------------------ | 20260506 02:10:59.952 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:10:59.952 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 02:10:59.952 | INFO | Total mem usage: 25.6 GiB | 20260506 02:10:59.952 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 02:10:59.952 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 02:10:59.952 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 02:10:59.952 | INFO | High Water Breakdown: | 20260506 02:10:59.952 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 02:10:59.952 | INFO | ------------------------------------------ | 20260506 02:10:59.952 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:10:59.952 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 02:10:59.952 | INFO | Total mem usage: 15.7 GiB | 20260506 02:10:59.952 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 02:10:59.952 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 02:10:59.952 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 02:10:59.952 | INFO | High Water Breakdown: | 20260506 02:10:59.952 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 02:10:59.952 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 02:10:59.952 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 02:10:59.952 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 02:10:59.952 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 02:10:59.952 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 02:10:59.952 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 02:10:59.952 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 02:10:59.952 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 02:10:59.952 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:10:59.952 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 02:10:59.952 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:10:59.952 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 02:10:59.952 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 02:10:59.952 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 02:10:59.952 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 02:10:59.952 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:10:59.952 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 02:10:59.952 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:10:59.952 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:10:59.952 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:10:59.952 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 02:10:59.952 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 02:10:59.952 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:10:59.952 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 02:10:59.952 | INFO | ------------------------------------------ | 20260506 02:10:59.952 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:10:59.952 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 02:10:59.952 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:10:59.952 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:10:59.952 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.40e+01, 1.35e+01 / 8.84e-01 | 20260506 02:10:59.952 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.04e-01, 1.01e-02 / 3.89e-03 | 20260506 02:10:59.952 | INFO | PerfReport: 11.85%, Baseline -- 3.71e+00 / 4.54e+00, 5.85e-01 / 1.80e-01 | 20260506 02:10:59.952 | INFO | PerfReport: 3.30%, Histogram -- 1.04e+00 / 1.53e+00, 1.98e-01 / 7.74e-02 | 20260506 02:10:59.952 | INFO | PerfReport: 15.99%, DME -- 4.01e+01 / 4.42e+01, 3.93e+00 / 1.05e+00 | 20260506 02:10:59.952 | INFO | PerfReport: 57.50%, FrameLabeling -- 1.80e+01 / 1.92e+01, 1.59e+00 / 9.83e-01 | 20260506 02:10:59.952 | INFO | PerfReport: 7.00%, PulseAccumulating -- 2.19e+00 / 2.75e+00, 4.14e-01 / 1.88e-01 | 20260506 02:10:59.952 | INFO | PerfReport: 3.02%, Metrics -- 9.45e-01 / 1.21e+00, 2.53e-01 / 1.16e-01 | 20260506 02:10:59.952 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.27e-01 / 3.72e-01, 5.45e-02 / 3.24e-02 | 20260506 02:10:59.952 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 02:10:59.952 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:10:59.952 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:10:59.952 | INFO | PerfReport: Full Execution -- 9.77e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260506 02:10:59.953 | INFO | PerfReport: 22.10%, Upload -- 2.16e+01 / 0.00e+00, 7.09e+00 / 0.00e+00 | 20260506 02:10:59.953 | INFO | PerfReport: 21.91%, Download -- 2.14e+01 / 0.00e+00, 7.57e+00 / 0.00e+00 | 20260506 02:10:59.953 | INFO | PerfReport: 32.10%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 02:10:59.953 | INFO | PerfReport: 1.98%, LockUpload -- 1.94e+00 / 0.00e+00, 7.68e+00 / 0.00e+00 | 20260506 02:10:59.953 | INFO | PerfReport: 1.29%, LockDownload -- 1.26e+00 / 0.00e+00, 2.96e+00 / 0.00e+00 | 20260506 02:10:59.953 | INFO | PerfReport: 20.61%, LockCompute -- 2.01e+01 / 0.00e+00, 1.77e+01 / 0.00e+00 | 20260506 02:10:59.953 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 2.16e-02 / 0.00e+00 | 20260506 02:11:00.077 | INFO | Starting a round of Baz IO | 20260506 02:11:03.211 | INFO | Populating RTMetrics report | 20260506 02:11:03.211 | INFO | RTMetrics report populated | 20260506 02:11:03.211 | INFO | Opening tmp RTMetrics json file | 20260506 02:11:03.212 | INFO | Serializing RTMetrics to json | 20260506 02:11:03.213 | INFO | Serialization done, swapping tmp file now... | 20260506 02:11:03.213 | INFO | RTMetrics done | 20260506 02:11:03.287 | INFO | Finished a round of Baz IO: Wrote 1.803 GiB in 3.21 seconds | 20260506 02:11:03.287 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.28% / 0.00% / 78.72% | 20260506 02:11:03.287 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.072972% / 7.94729e-06% / 7.54992e-05% | 20260506 02:11:03.287 | INFO | Baz IO overall: | 20260506 02:11:03.287 | INFO | BytesWritten : 4561658896217 | 20260506 02:11:03.287 | INFO | NumEvents : 2142295886791 | 20260506 02:11:05.091 | INFO | Analyzing chunk frames = [8126464,8126976) | 20260506 02:11:05.091 | INFO | MemInfo: { | 20260506 02:11:05.091 | INFO | "gpu" : " 17 GB", | 20260506 02:11:05.091 | INFO | "huge" : "", | 20260506 02:11:05.091 | INFO | "huge_shared" : " 27 GB", | 20260506 02:11:05.091 | INFO | "malloc" : " 14 GB", | 20260506 02:11:05.091 | INFO | "pinned" : " 29 GB" | 20260506 02:11:05.091 | INFO | } | 20260506 02:11:05.091 | INFO | Perf report for the last 5.13981 seconds: | 20260506 02:11:05.091 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:11:05.091 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:05.091 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:05.091 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:05.091 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260506 02:11:05.091 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:05.091 | INFO | RT_METRICS : 42.8%, 1.0, 22.5ms, 0 | 20260506 02:11:05.091 | INFO | PRE_HQ : 59.5%, 1.7, 53.5ms, 0 | 20260506 02:11:05.091 | INFO | BAZWRITER : 61.7%, 3.5, 109.9ms, 0 | 20260506 02:11:05.091 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:11:05.091 | INFO | Uploaded 36GiB to GPU in 2.17834s (16.5264GiB/s) | 20260506 02:11:05.091 | INFO | Downloaded 38GiB to GPU in 2.23981s (16.9657GiB/s) | 20260506 02:11:10.230 | INFO | Analyzing chunk frames = [8126976,8127488) | 20260506 02:11:10.230 | INFO | MemInfo: { | 20260506 02:11:10.230 | INFO | "gpu" : " 17 GB", | 20260506 02:11:10.230 | INFO | "huge" : "", | 20260506 02:11:10.230 | INFO | "huge_shared" : " 27 GB", | 20260506 02:11:10.230 | INFO | "malloc" : " 14 GB", | 20260506 02:11:10.230 | INFO | "pinned" : " 29 GB" | 20260506 02:11:10.230 | INFO | } | 20260506 02:11:10.230 | INFO | Perf report for the last 5.13874 seconds: | 20260506 02:11:10.230 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:11:10.230 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:10.230 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:10.230 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:10.230 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 02:11:10.230 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:10.230 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:10.230 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 02:11:10.230 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:10.230 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:10.230 | INFO | Uploaded 36GiB to GPU in 2.02485s (17.7791GiB/s) | 20260506 02:11:10.230 | INFO | Downloaded 34GiB to GPU in 1.98862s (17.0973GiB/s) | 20260506 02:11:15.382 | INFO | Analyzing chunk frames = [8127488,8128000) | 20260506 02:11:15.382 | INFO | MemInfo: { | 20260506 02:11:15.382 | INFO | "gpu" : " 17 GB", | 20260506 02:11:15.382 | INFO | "huge" : "", | 20260506 02:11:15.382 | INFO | "huge_shared" : " 27 GB", | 20260506 02:11:15.382 | INFO | "malloc" : " 14 GB", | 20260506 02:11:15.382 | INFO | "pinned" : " 29 GB" | 20260506 02:11:15.382 | INFO | } | 20260506 02:11:15.382 | INFO | Perf report for the last 5.15288 seconds: | 20260506 02:11:15.383 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:11:15.383 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:15.383 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:15.383 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:15.383 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 02:11:15.383 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:15.383 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:15.383 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:11:15.383 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:15.383 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:15.383 | INFO | Uploaded 36GiB to GPU in 2.09088s (17.2176GiB/s) | 20260506 02:11:15.383 | INFO | Downloaded 34GiB to GPU in 2.02591s (16.7826GiB/s) | 20260506 02:11:20.517 | INFO | Analyzing chunk frames = [8128000,8128512) | 20260506 02:11:20.517 | INFO | MemInfo: { | 20260506 02:11:20.517 | INFO | "gpu" : " 17 GB", | 20260506 02:11:20.517 | INFO | "huge" : "", | 20260506 02:11:20.517 | INFO | "huge_shared" : " 27 GB", | 20260506 02:11:20.517 | INFO | "malloc" : " 14 GB", | 20260506 02:11:20.517 | INFO | "pinned" : " 29 GB" | 20260506 02:11:20.517 | INFO | } | 20260506 02:11:20.517 | INFO | Perf report for the last 5.13455 seconds: | 20260506 02:11:20.517 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:11:20.517 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:20.517 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:20.517 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:20.517 | INFO | ANALYSIS : 61.0%, 3.0, 94.7ms, 0 | 20260506 02:11:20.517 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:20.517 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:20.517 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 02:11:20.517 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:20.517 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:20.517 | INFO | Uploaded 36GiB to GPU in 2.09284s (17.2015GiB/s) | 20260506 02:11:20.517 | INFO | Downloaded 34GiB to GPU in 2.01186s (16.8998GiB/s) | 20260506 02:11:25.646 | INFO | Analyzing chunk frames = [8128512,8129024) | 20260506 02:11:25.646 | INFO | MemInfo: { | 20260506 02:11:25.646 | INFO | "gpu" : " 17 GB", | 20260506 02:11:25.646 | INFO | "huge" : "", | 20260506 02:11:25.646 | INFO | "huge_shared" : " 27 GB", | 20260506 02:11:25.646 | INFO | "malloc" : " 14 GB", | 20260506 02:11:25.646 | INFO | "pinned" : " 29 GB" | 20260506 02:11:25.646 | INFO | } | 20260506 02:11:25.646 | INFO | Perf report for the last 5.12871 seconds: | 20260506 02:11:25.646 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:11:25.646 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:25.646 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:25.646 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:25.646 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260506 02:11:25.646 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:25.646 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:25.646 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:11:25.646 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:25.646 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:25.646 | INFO | Uploaded 36GiB to GPU in 2.09675s (17.1694GiB/s) | 20260506 02:11:25.646 | INFO | Downloaded 34GiB to GPU in 2.03654s (16.695GiB/s) | 20260506 02:11:30.786 | INFO | Analyzing chunk frames = [8129024,8129536) | 20260506 02:11:30.786 | INFO | MemInfo: { | 20260506 02:11:30.786 | INFO | "gpu" : " 17 GB", | 20260506 02:11:30.786 | INFO | "huge" : "", | 20260506 02:11:30.786 | INFO | "huge_shared" : " 27 GB", | 20260506 02:11:30.786 | INFO | "malloc" : " 14 GB", | 20260506 02:11:30.786 | INFO | "pinned" : " 29 GB" | 20260506 02:11:30.786 | INFO | } | 20260506 02:11:30.786 | INFO | Perf report for the last 5.14018 seconds: | 20260506 02:11:30.786 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:11:30.786 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:30.786 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:30.786 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:30.786 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 02:11:30.786 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:30.786 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:30.786 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 02:11:30.786 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:30.786 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:30.786 | INFO | Uploaded 36GiB to GPU in 2.07404s (17.3574GiB/s) | 20260506 02:11:30.786 | INFO | Downloaded 34GiB to GPU in 2.04162s (16.6534GiB/s) | 20260506 02:11:35.929 | INFO | Analyzing chunk frames = [8129536,8130048) | 20260506 02:11:35.929 | INFO | MemInfo: { | 20260506 02:11:35.929 | INFO | "gpu" : " 17 GB", | 20260506 02:11:35.929 | INFO | "huge" : "", | 20260506 02:11:35.929 | INFO | "huge_shared" : " 27 GB", | 20260506 02:11:35.929 | INFO | "malloc" : " 14 GB", | 20260506 02:11:35.929 | INFO | "pinned" : " 29 GB" | 20260506 02:11:35.929 | INFO | } | 20260506 02:11:35.929 | INFO | Perf report for the last 5.14277 seconds: | 20260506 02:11:35.929 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:11:35.929 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:35.929 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:35.929 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:35.929 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260506 02:11:35.929 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:35.929 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:35.929 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 02:11:35.929 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:35.929 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:35.929 | INFO | Uploaded 36GiB to GPU in 2.10648s (17.0901GiB/s) | 20260506 02:11:35.929 | INFO | Downloaded 34GiB to GPU in 2.08212s (16.3295GiB/s) | 20260506 02:11:41.073 | INFO | Analyzing chunk frames = [8130048,8130560) | 20260506 02:11:41.073 | INFO | MemInfo: { | 20260506 02:11:41.073 | INFO | "gpu" : " 17 GB", | 20260506 02:11:41.073 | INFO | "huge" : "", | 20260506 02:11:41.073 | INFO | "huge_shared" : " 27 GB", | 20260506 02:11:41.073 | INFO | "malloc" : " 14 GB", | 20260506 02:11:41.073 | INFO | "pinned" : " 29 GB" | 20260506 02:11:41.073 | INFO | } | 20260506 02:11:41.073 | INFO | Perf report for the last 5.14473 seconds: | 20260506 02:11:41.073 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:11:41.073 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:41.073 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:41.073 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:41.073 | INFO | ANALYSIS : 61.2%, 3.0, 95.7ms, 0 | 20260506 02:11:41.074 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:41.074 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:41.074 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:11:41.074 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:41.074 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:41.074 | INFO | Uploaded 36GiB to GPU in 2.04261s (17.6245GiB/s) | 20260506 02:11:41.074 | INFO | Downloaded 34GiB to GPU in 2.01565s (16.868GiB/s) | 20260506 02:11:41.199 | INFO | Starting a round of Baz IO | 20260506 02:11:44.327 | INFO | Populating RTMetrics report | 20260506 02:11:44.327 | INFO | RTMetrics report populated | 20260506 02:11:44.327 | INFO | Opening tmp RTMetrics json file | 20260506 02:11:44.328 | INFO | Serializing RTMetrics to json | 20260506 02:11:44.329 | INFO | Serialization done, swapping tmp file now... | 20260506 02:11:44.330 | INFO | RTMetrics done | 20260506 02:11:44.402 | INFO | Finished a round of Baz IO: Wrote 1.802 GiB in 3.203 seconds | 20260506 02:11:44.402 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.28% / 0.00% / 78.72% | 20260506 02:11:44.402 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0734409% / 0% / 9.13938e-05% | 20260506 02:11:44.402 | INFO | Baz IO overall: | 20260506 02:11:44.402 | INFO | BytesWritten : 4563593264570 | 20260506 02:11:44.402 | INFO | NumEvents : 2143175546179 | 20260506 02:11:46.205 | INFO | Analyzing chunk frames = [8130560,8131072) | 20260506 02:11:46.205 | INFO | MemInfo: { | 20260506 02:11:46.205 | INFO | "gpu" : " 17 GB", | 20260506 02:11:46.205 | INFO | "huge" : "", | 20260506 02:11:46.206 | INFO | "huge_shared" : " 27 GB", | 20260506 02:11:46.206 | INFO | "malloc" : " 14 GB", | 20260506 02:11:46.206 | INFO | "pinned" : " 29 GB" | 20260506 02:11:46.206 | INFO | } | 20260506 02:11:46.206 | INFO | Perf report for the last 5.13225 seconds: | 20260506 02:11:46.206 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:11:46.206 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:46.206 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:46.206 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:46.206 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 02:11:46.206 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:46.206 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260506 02:11:46.206 | INFO | PRE_HQ : 59.4%, 1.8, 53.9ms, 0 | 20260506 02:11:46.206 | INFO | BAZWRITER : 61.6%, 3.5, 109.9ms, 0 | 20260506 02:11:46.206 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:11:46.206 | INFO | Uploaded 36GiB to GPU in 2.13418s (16.8683GiB/s) | 20260506 02:11:46.206 | INFO | Downloaded 38GiB to GPU in 2.23325s (17.0156GiB/s) | 20260506 02:11:51.348 | INFO | Analyzing chunk frames = [8131072,8131584) | 20260506 02:11:51.348 | INFO | MemInfo: { | 20260506 02:11:51.348 | INFO | "gpu" : " 17 GB", | 20260506 02:11:51.348 | INFO | "huge" : "", | 20260506 02:11:51.348 | INFO | "huge_shared" : " 27 GB", | 20260506 02:11:51.348 | INFO | "malloc" : " 14 GB", | 20260506 02:11:51.348 | INFO | "pinned" : " 29 GB" | 20260506 02:11:51.348 | INFO | } | 20260506 02:11:51.348 | INFO | Perf report for the last 5.14195 seconds: | 20260506 02:11:51.348 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:11:51.348 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:51.348 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:51.348 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:51.348 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 02:11:51.348 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:51.348 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:51.348 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 02:11:51.348 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:51.348 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:51.348 | INFO | Uploaded 36GiB to GPU in 1.97461s (18.2314GiB/s) | 20260506 02:11:51.348 | INFO | Downloaded 34GiB to GPU in 1.99228s (17.0659GiB/s) | 20260506 02:11:56.487 | INFO | Analyzing chunk frames = [8131584,8132096) | 20260506 02:11:56.487 | INFO | MemInfo: { | 20260506 02:11:56.487 | INFO | "gpu" : " 17 GB", | 20260506 02:11:56.487 | INFO | "huge" : "", | 20260506 02:11:56.487 | INFO | "huge_shared" : " 27 GB", | 20260506 02:11:56.488 | INFO | "malloc" : " 14 GB", | 20260506 02:11:56.488 | INFO | "pinned" : " 29 GB" | 20260506 02:11:56.488 | INFO | } | 20260506 02:11:56.488 | INFO | Perf report for the last 5.13987 seconds: | 20260506 02:11:56.488 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:11:56.488 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:56.488 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:56.488 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:56.488 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 02:11:56.488 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:56.488 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:56.488 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:11:56.488 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:56.488 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:11:56.488 | INFO | Uploaded 36GiB to GPU in 2.07913s (17.3149GiB/s) | 20260506 02:11:56.488 | INFO | Downloaded 34GiB to GPU in 2.01968s (16.8344GiB/s) | 20260506 02:12:01.624 | INFO | Analyzing chunk frames = [8132096,8132608) | 20260506 02:12:01.624 | INFO | MemInfo: { | 20260506 02:12:01.624 | INFO | "gpu" : " 17 GB", | 20260506 02:12:01.624 | INFO | "huge" : "", | 20260506 02:12:01.624 | INFO | "huge_shared" : " 27 GB", | 20260506 02:12:01.624 | INFO | "malloc" : " 14 GB", | 20260506 02:12:01.624 | INFO | "pinned" : " 29 GB" | 20260506 02:12:01.624 | INFO | } | 20260506 02:12:01.624 | INFO | Perf report for the last 5.13649 seconds: | 20260506 02:12:01.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:12:01.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:01.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:01.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:01.624 | INFO | ANALYSIS : 61.3%, 3.0, 95.1ms, 0 | 20260506 02:12:01.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:01.624 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:01.624 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 02:12:01.624 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:01.624 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:01.624 | INFO | Uploaded 36GiB to GPU in 2.09591s (17.1763GiB/s) | 20260506 02:12:01.624 | INFO | Downloaded 34GiB to GPU in 2.03282s (16.7255GiB/s) | 20260506 02:12:06.763 | INFO | Analyzing chunk frames = [8132608,8133120) | 20260506 02:12:06.763 | INFO | MemInfo: { | 20260506 02:12:06.763 | INFO | "gpu" : " 17 GB", | 20260506 02:12:06.763 | INFO | "huge" : "", | 20260506 02:12:06.763 | INFO | "huge_shared" : " 27 GB", | 20260506 02:12:06.763 | INFO | "malloc" : " 14 GB", | 20260506 02:12:06.763 | INFO | "pinned" : " 29 GB" | 20260506 02:12:06.763 | INFO | } | 20260506 02:12:06.763 | INFO | Perf report for the last 5.13861 seconds: | 20260506 02:12:06.763 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:12:06.763 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:06.763 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:06.763 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:06.763 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260506 02:12:06.763 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:06.763 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:06.763 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:12:06.763 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:06.763 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:06.763 | INFO | Uploaded 36GiB to GPU in 2.06774s (17.4103GiB/s) | 20260506 02:12:06.763 | INFO | Downloaded 34GiB to GPU in 2.03067s (16.7433GiB/s) | 20260506 02:12:11.902 | INFO | Analyzing chunk frames = [8133120,8133632) | 20260506 02:12:11.902 | INFO | MemInfo: { | 20260506 02:12:11.902 | INFO | "gpu" : " 17 GB", | 20260506 02:12:11.902 | INFO | "huge" : "", | 20260506 02:12:11.902 | INFO | "huge_shared" : " 27 GB", | 20260506 02:12:11.902 | INFO | "malloc" : " 14 GB", | 20260506 02:12:11.902 | INFO | "pinned" : " 29 GB" | 20260506 02:12:11.902 | INFO | } | 20260506 02:12:11.902 | INFO | Perf report for the last 5.13937 seconds: | 20260506 02:12:11.902 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:12:11.902 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:11.902 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:11.902 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:11.902 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 02:12:11.902 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:11.902 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:11.902 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 02:12:11.902 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:11.902 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:11.902 | INFO | Uploaded 36GiB to GPU in 2.06677s (17.4185GiB/s) | 20260506 02:12:11.902 | INFO | Downloaded 34GiB to GPU in 2.03775s (16.6851GiB/s) | 20260506 02:12:17.038 | INFO | Analyzing chunk frames = [8133632,8134144) | 20260506 02:12:17.038 | INFO | MemInfo: { | 20260506 02:12:17.038 | INFO | "gpu" : " 17 GB", | 20260506 02:12:17.038 | INFO | "huge" : "", | 20260506 02:12:17.038 | INFO | "huge_shared" : " 27 GB", | 20260506 02:12:17.038 | INFO | "malloc" : " 14 GB", | 20260506 02:12:17.038 | INFO | "pinned" : " 29 GB" | 20260506 02:12:17.038 | INFO | } | 20260506 02:12:17.039 | INFO | Perf report for the last 5.13641 seconds: | 20260506 02:12:17.039 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:12:17.039 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:17.039 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:17.039 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:17.039 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260506 02:12:17.039 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:17.039 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:17.039 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 02:12:17.039 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:17.039 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:17.039 | INFO | Uploaded 36GiB to GPU in 2.10645s (17.0903GiB/s) | 20260506 02:12:17.039 | INFO | Downloaded 34GiB to GPU in 2.08617s (16.2978GiB/s) | 20260506 02:12:22.183 | INFO | Analyzing chunk frames = [8134144,8134656) | 20260506 02:12:22.183 | INFO | MemInfo: { | 20260506 02:12:22.183 | INFO | "gpu" : " 17 GB", | 20260506 02:12:22.183 | INFO | "huge" : "", | 20260506 02:12:22.183 | INFO | "huge_shared" : " 27 GB", | 20260506 02:12:22.183 | INFO | "malloc" : " 14 GB", | 20260506 02:12:22.183 | INFO | "pinned" : " 29 GB" | 20260506 02:12:22.184 | INFO | } | 20260506 02:12:22.184 | INFO | Perf report for the last 5.14507 seconds: | 20260506 02:12:22.184 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:12:22.184 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:22.184 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:22.184 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:22.184 | INFO | ANALYSIS : 61.5%, 3.0, 96.1ms, 0 | 20260506 02:12:22.184 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:22.184 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:22.184 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 02:12:22.184 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:22.184 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:22.184 | INFO | Uploaded 36GiB to GPU in 2.06154s (17.4627GiB/s) | 20260506 02:12:22.184 | INFO | Downloaded 34GiB to GPU in 2.02229s (16.8126GiB/s) | 20260506 02:12:22.184 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 02:12:22.184 | INFO | Total mem usage: 15.3 GiB | 20260506 02:12:22.184 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 02:12:22.184 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.18%) | 20260506 02:12:22.184 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 02:12:22.184 | INFO | High Water Breakdown: | 20260506 02:12:22.184 | INFO | 60.9% : 8.55 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 02:12:22.184 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 02:12:22.184 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 02:12:22.184 | INFO | ------------------------------------------ | 20260506 02:12:22.184 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:12:22.184 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 02:12:22.184 | INFO | Total mem usage: 26.5 GiB | 20260506 02:12:22.184 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 02:12:22.184 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 02:12:22.184 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 02:12:22.184 | INFO | High Water Breakdown: | 20260506 02:12:22.184 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:12:22.184 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:12:22.184 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:12:22.184 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:12:22.184 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:12:22.184 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:12:22.184 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 02:12:22.184 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 02:12:22.184 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:12:22.184 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 02:12:22.184 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 02:12:22.184 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 02:12:22.184 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 02:12:22.184 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 02:12:22.184 | INFO | ------------------------------------------ | 20260506 02:12:22.184 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:12:22.184 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 02:12:22.184 | INFO | Total mem usage: 25.6 GiB | 20260506 02:12:22.185 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 02:12:22.185 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 02:12:22.185 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 02:12:22.185 | INFO | High Water Breakdown: | 20260506 02:12:22.185 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 02:12:22.185 | INFO | ------------------------------------------ | 20260506 02:12:22.185 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:12:22.185 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 02:12:22.185 | INFO | Total mem usage: 15.7 GiB | 20260506 02:12:22.185 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 02:12:22.185 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 02:12:22.185 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 02:12:22.185 | INFO | High Water Breakdown: | 20260506 02:12:22.185 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 02:12:22.185 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 02:12:22.185 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 02:12:22.185 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 02:12:22.185 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 02:12:22.185 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 02:12:22.185 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 02:12:22.185 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 02:12:22.185 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 02:12:22.185 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:12:22.185 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 02:12:22.185 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:12:22.185 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 02:12:22.185 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 02:12:22.185 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 02:12:22.185 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 02:12:22.185 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:12:22.185 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 02:12:22.185 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:12:22.185 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:12:22.185 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:12:22.185 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 02:12:22.185 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 02:12:22.185 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:12:22.185 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 02:12:22.185 | INFO | ------------------------------------------ | 20260506 02:12:22.185 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:12:22.185 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 02:12:22.185 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:12:22.185 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:12:22.185 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.39e+01, 1.34e+01 / 8.73e-01 | 20260506 02:12:22.185 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.05e-01, 1.05e-02 / 2.75e-03 | 20260506 02:12:22.185 | INFO | PerfReport: 11.86%, Baseline -- 3.72e+00 / 4.44e+00, 5.86e-01 / 2.55e-01 | 20260506 02:12:22.185 | INFO | PerfReport: 3.31%, Histogram -- 1.04e+00 / 1.50e+00, 1.96e-01 / 1.05e-01 | 20260506 02:12:22.185 | INFO | PerfReport: 15.97%, DME -- 4.00e+01 / 4.41e+01, 3.95e+00 / 1.12e+00 | 20260506 02:12:22.185 | INFO | PerfReport: 57.53%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.60e+00 / 1.03e+00 | 20260506 02:12:22.185 | INFO | PerfReport: 6.98%, PulseAccumulating -- 2.19e+00 / 2.68e+00, 4.13e-01 / 2.30e-01 | 20260506 02:12:22.185 | INFO | PerfReport: 3.01%, Metrics -- 9.45e-01 / 1.21e+00, 2.50e-01 / 1.06e-01 | 20260506 02:12:22.185 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.27e-01 / 3.76e-01, 5.42e-02 / 2.83e-02 | 20260506 02:12:22.185 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 02:12:22.185 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:12:22.185 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:12:22.185 | INFO | PerfReport: Full Execution -- 9.77e+01 / -nan, 2.43e+01 / 0.00e+00 | 20260506 02:12:22.185 | INFO | PerfReport: 22.18%, Upload -- 2.17e+01 / 0.00e+00, 7.24e+00 / 0.00e+00 | 20260506 02:12:22.185 | INFO | PerfReport: 21.92%, Download -- 2.14e+01 / 0.00e+00, 7.58e+00 / 0.00e+00 | 20260506 02:12:22.185 | INFO | PerfReport: 32.08%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 02:12:22.185 | INFO | PerfReport: 2.01%, LockUpload -- 1.96e+00 / 0.00e+00, 7.90e+00 / 0.00e+00 | 20260506 02:12:22.185 | INFO | PerfReport: 1.28%, LockDownload -- 1.25e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260506 02:12:22.185 | INFO | PerfReport: 20.52%, LockCompute -- 2.01e+01 / 0.00e+00, 1.77e+01 / 0.00e+00 | 20260506 02:12:22.185 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.68e-02 / 0.00e+00 | 20260506 02:12:22.309 | INFO | Starting a round of Baz IO | 20260506 02:12:25.436 | INFO | Populating RTMetrics report | 20260506 02:12:25.437 | INFO | RTMetrics report populated | 20260506 02:12:25.437 | INFO | Opening tmp RTMetrics json file | 20260506 02:12:25.437 | INFO | Serializing RTMetrics to json | 20260506 02:12:25.439 | INFO | Serialization done, swapping tmp file now... | 20260506 02:12:25.439 | INFO | RTMetrics done | 20260506 02:12:25.518 | INFO | Finished a round of Baz IO: Wrote 1.801 GiB in 3.209 seconds | 20260506 02:12:25.519 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.28% / 0.00% / 78.72% | 20260506 02:12:25.519 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0728965% / 3.97364e-06% / 9.53674e-05% | 20260506 02:12:25.519 | INFO | Baz IO overall: | 20260506 02:12:25.519 | INFO | BytesWritten : 4565526895823 | 20260506 02:12:25.519 | INFO | NumEvents : 2144054679656 | 20260506 02:12:27.328 | INFO | Analyzing chunk frames = [8134656,8135168) | 20260506 02:12:27.328 | INFO | MemInfo: { | 20260506 02:12:27.328 | INFO | "gpu" : " 17 GB", | 20260506 02:12:27.328 | INFO | "huge" : "", | 20260506 02:12:27.328 | INFO | "huge_shared" : " 27 GB", | 20260506 02:12:27.328 | INFO | "malloc" : " 14 GB", | 20260506 02:12:27.328 | INFO | "pinned" : " 29 GB" | 20260506 02:12:27.328 | INFO | } | 20260506 02:12:27.328 | INFO | Perf report for the last 5.14449 seconds: | 20260506 02:12:27.328 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:12:27.328 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:27.328 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:27.328 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:27.328 | INFO | ANALYSIS : 62.2%, 3.0, 97.3ms, 0 | 20260506 02:12:27.328 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:27.328 | INFO | RT_METRICS : 42.8%, 1.0, 22.5ms, 0 | 20260506 02:12:27.328 | INFO | PRE_HQ : 59.1%, 1.8, 53.7ms, 0 | 20260506 02:12:27.328 | INFO | BAZWRITER : 61.5%, 3.5, 110.2ms, 0 | 20260506 02:12:27.328 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:12:27.328 | INFO | Uploaded 36GiB to GPU in 2.08224s (17.2891GiB/s) | 20260506 02:12:27.328 | INFO | Downloaded 38GiB to GPU in 2.21665s (17.143GiB/s) | 20260506 02:12:32.459 | INFO | Analyzing chunk frames = [8135168,8135680) | 20260506 02:12:32.459 | INFO | MemInfo: { | 20260506 02:12:32.459 | INFO | "gpu" : " 17 GB", | 20260506 02:12:32.459 | INFO | "huge" : "", | 20260506 02:12:32.459 | INFO | "huge_shared" : " 27 GB", | 20260506 02:12:32.459 | INFO | "malloc" : " 14 GB", | 20260506 02:12:32.459 | INFO | "pinned" : " 29 GB" | 20260506 02:12:32.459 | INFO | } | 20260506 02:12:32.459 | INFO | Perf report for the last 5.13104 seconds: | 20260506 02:12:32.459 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:12:32.459 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:32.459 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:32.459 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:32.459 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260506 02:12:32.459 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:32.459 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:32.459 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 02:12:32.459 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:32.459 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:32.459 | INFO | Uploaded 36GiB to GPU in 1.96518s (18.3189GiB/s) | 20260506 02:12:32.459 | INFO | Downloaded 34GiB to GPU in 1.97423s (17.2219GiB/s) | 20260506 02:12:37.596 | INFO | Analyzing chunk frames = [8135680,8136192) | 20260506 02:12:37.596 | INFO | MemInfo: { | 20260506 02:12:37.596 | INFO | "gpu" : " 17 GB", | 20260506 02:12:37.596 | INFO | "huge" : "", | 20260506 02:12:37.596 | INFO | "huge_shared" : " 27 GB", | 20260506 02:12:37.597 | INFO | "malloc" : " 14 GB", | 20260506 02:12:37.597 | INFO | "pinned" : " 29 GB" | 20260506 02:12:37.597 | INFO | } | 20260506 02:12:37.597 | INFO | Perf report for the last 5.13747 seconds: | 20260506 02:12:37.597 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:12:37.597 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:37.597 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:37.597 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:37.597 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 02:12:37.597 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:37.597 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:37.597 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 02:12:37.597 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:37.597 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:37.597 | INFO | Uploaded 36GiB to GPU in 2.08734s (17.2469GiB/s) | 20260506 02:12:37.597 | INFO | Downloaded 34GiB to GPU in 2.01985s (16.8329GiB/s) | 20260506 02:12:42.744 | INFO | Analyzing chunk frames = [8136192,8136704) | 20260506 02:12:42.744 | INFO | MemInfo: { | 20260506 02:12:42.744 | INFO | "gpu" : " 17 GB", | 20260506 02:12:42.744 | INFO | "huge" : "", | 20260506 02:12:42.744 | INFO | "huge_shared" : " 27 GB", | 20260506 02:12:42.744 | INFO | "malloc" : " 14 GB", | 20260506 02:12:42.744 | INFO | "pinned" : " 29 GB" | 20260506 02:12:42.744 | INFO | } | 20260506 02:12:42.744 | INFO | Perf report for the last 5.14729 seconds: | 20260506 02:12:42.744 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:12:42.744 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:42.744 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:42.744 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:42.744 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 02:12:42.744 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:42.744 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:42.744 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:12:42.744 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:42.744 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:42.744 | INFO | Uploaded 36GiB to GPU in 2.14393s (16.7916GiB/s) | 20260506 02:12:42.744 | INFO | Downloaded 34GiB to GPU in 2.02403s (16.7982GiB/s) | 20260506 02:12:47.882 | INFO | Analyzing chunk frames = [8136704,8137216) | 20260506 02:12:47.882 | INFO | MemInfo: { | 20260506 02:12:47.882 | INFO | "gpu" : " 17 GB", | 20260506 02:12:47.882 | INFO | "huge" : "", | 20260506 02:12:47.882 | INFO | "huge_shared" : " 27 GB", | 20260506 02:12:47.882 | INFO | "malloc" : " 14 GB", | 20260506 02:12:47.882 | INFO | "pinned" : " 29 GB" | 20260506 02:12:47.882 | INFO | } | 20260506 02:12:47.882 | INFO | Perf report for the last 5.13834 seconds: | 20260506 02:12:47.882 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:12:47.882 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:47.882 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:47.882 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:47.882 | INFO | ANALYSIS : 60.7%, 3.0, 94.8ms, 0 | 20260506 02:12:47.882 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:47.882 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:47.882 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 02:12:47.882 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:47.882 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:47.883 | INFO | Uploaded 36GiB to GPU in 2.12422s (16.9474GiB/s) | 20260506 02:12:47.883 | INFO | Downloaded 34GiB to GPU in 2.03581s (16.7009GiB/s) | 20260506 02:12:53.025 | INFO | Analyzing chunk frames = [8137216,8137728) | 20260506 02:12:53.025 | INFO | MemInfo: { | 20260506 02:12:53.025 | INFO | "gpu" : " 17 GB", | 20260506 02:12:53.025 | INFO | "huge" : "", | 20260506 02:12:53.025 | INFO | "huge_shared" : " 27 GB", | 20260506 02:12:53.025 | INFO | "malloc" : " 14 GB", | 20260506 02:12:53.025 | INFO | "pinned" : " 29 GB" | 20260506 02:12:53.025 | INFO | } | 20260506 02:12:53.025 | INFO | Perf report for the last 5.14307 seconds: | 20260506 02:12:53.025 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:12:53.025 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:53.025 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:53.025 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:53.025 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 02:12:53.025 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:53.025 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:53.025 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 02:12:53.025 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:53.025 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:53.026 | INFO | Uploaded 36GiB to GPU in 2.09192s (17.2091GiB/s) | 20260506 02:12:53.026 | INFO | Downloaded 34GiB to GPU in 2.0329s (16.7248GiB/s) | 20260506 02:12:58.160 | INFO | Analyzing chunk frames = [8137728,8138240) | 20260506 02:12:58.160 | INFO | MemInfo: { | 20260506 02:12:58.160 | INFO | "gpu" : " 17 GB", | 20260506 02:12:58.160 | INFO | "huge" : "", | 20260506 02:12:58.160 | INFO | "huge_shared" : " 27 GB", | 20260506 02:12:58.160 | INFO | "malloc" : " 14 GB", | 20260506 02:12:58.160 | INFO | "pinned" : " 29 GB" | 20260506 02:12:58.160 | INFO | } | 20260506 02:12:58.160 | INFO | Perf report for the last 5.13476 seconds: | 20260506 02:12:58.160 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:12:58.160 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:58.160 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:58.160 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:58.160 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260506 02:12:58.160 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:58.160 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:58.160 | INFO | PRE_HQ : 27.5%, 1.0, 14.2ms, 0 | 20260506 02:12:58.160 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:58.160 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:12:58.160 | INFO | Uploaded 36GiB to GPU in 2.06298s (17.4505GiB/s) | 20260506 02:12:58.160 | INFO | Downloaded 34GiB to GPU in 2.07926s (16.352GiB/s) | 20260506 02:13:03.297 | INFO | Analyzing chunk frames = [8138240,8138752) | 20260506 02:13:03.297 | INFO | MemInfo: { | 20260506 02:13:03.297 | INFO | "gpu" : " 17 GB", | 20260506 02:13:03.297 | INFO | "huge" : "", | 20260506 02:13:03.297 | INFO | "huge_shared" : " 27 GB", | 20260506 02:13:03.297 | INFO | "malloc" : " 14 GB", | 20260506 02:13:03.297 | INFO | "pinned" : " 29 GB" | 20260506 02:13:03.297 | INFO | } | 20260506 02:13:03.297 | INFO | Perf report for the last 5.1366 seconds: | 20260506 02:13:03.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:13:03.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:03.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:03.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:03.297 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260506 02:13:03.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:03.297 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:03.297 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 02:13:03.297 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:03.297 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:03.297 | INFO | Uploaded 36GiB to GPU in 2.07157s (17.3781GiB/s) | 20260506 02:13:03.297 | INFO | Downloaded 34GiB to GPU in 2.029s (16.757GiB/s) | 20260506 02:13:03.423 | INFO | Starting a round of Baz IO | 20260506 02:13:06.558 | INFO | Populating RTMetrics report | 20260506 02:13:06.558 | INFO | RTMetrics report populated | 20260506 02:13:06.558 | INFO | Opening tmp RTMetrics json file | 20260506 02:13:06.558 | INFO | Serializing RTMetrics to json | 20260506 02:13:06.561 | INFO | Serialization done, swapping tmp file now... | 20260506 02:13:06.561 | INFO | RTMetrics done | 20260506 02:13:06.638 | INFO | Finished a round of Baz IO: Wrote 1.801 GiB in 3.215 seconds | 20260506 02:13:06.638 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.28% / 0.00% / 78.72% | 20260506 02:13:06.638 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0727971% / 0% / 6.75519e-05% | 20260506 02:13:06.638 | INFO | Baz IO overall: | 20260506 02:13:06.638 | INFO | BytesWritten : 4567461154328 | 20260506 02:13:06.638 | INFO | NumEvents : 2144934895205 | 20260506 02:13:08.438 | INFO | Analyzing chunk frames = [8138752,8139264) | 20260506 02:13:08.438 | INFO | MemInfo: { | 20260506 02:13:08.438 | INFO | "gpu" : " 17 GB", | 20260506 02:13:08.438 | INFO | "huge" : "", | 20260506 02:13:08.438 | INFO | "huge_shared" : " 27 GB", | 20260506 02:13:08.438 | INFO | "malloc" : " 14 GB", | 20260506 02:13:08.438 | INFO | "pinned" : " 29 GB" | 20260506 02:13:08.438 | INFO | } | 20260506 02:13:08.438 | INFO | Perf report for the last 5.14133 seconds: | 20260506 02:13:08.438 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:13:08.438 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:08.438 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:08.438 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:08.438 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260506 02:13:08.438 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:08.438 | INFO | RT_METRICS : 43.0%, 1.0, 22.5ms, 0 | 20260506 02:13:08.438 | INFO | PRE_HQ : 59.6%, 1.7, 53.8ms, 0 | 20260506 02:13:08.438 | INFO | BAZWRITER : 61.8%, 3.5, 111.1ms, 0 | 20260506 02:13:08.438 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:13:08.438 | INFO | Uploaded 36GiB to GPU in 2.18418s (16.4822GiB/s) | 20260506 02:13:08.438 | INFO | Downloaded 38GiB to GPU in 2.24395s (16.9344GiB/s) | 20260506 02:13:13.586 | INFO | Analyzing chunk frames = [8139264,8139776) | 20260506 02:13:13.586 | INFO | MemInfo: { | 20260506 02:13:13.586 | INFO | "gpu" : " 17 GB", | 20260506 02:13:13.586 | INFO | "huge" : "", | 20260506 02:13:13.586 | INFO | "huge_shared" : " 27 GB", | 20260506 02:13:13.586 | INFO | "malloc" : " 14 GB", | 20260506 02:13:13.586 | INFO | "pinned" : " 29 GB" | 20260506 02:13:13.586 | INFO | } | 20260506 02:13:13.586 | INFO | Perf report for the last 5.14774 seconds: | 20260506 02:13:13.586 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:13:13.586 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:13.586 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:13.586 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:13.586 | INFO | ANALYSIS : 60.7%, 3.0, 94.9ms, 0 | 20260506 02:13:13.586 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:13.586 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:13.586 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:13:13.586 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:13.586 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:13.586 | INFO | Uploaded 36GiB to GPU in 1.94295s (18.5285GiB/s) | 20260506 02:13:13.586 | INFO | Downloaded 34GiB to GPU in 1.97742s (17.1941GiB/s) | 20260506 02:13:18.715 | INFO | Analyzing chunk frames = [8139776,8140288) | 20260506 02:13:18.715 | INFO | MemInfo: { | 20260506 02:13:18.715 | INFO | "gpu" : " 17 GB", | 20260506 02:13:18.715 | INFO | "huge" : "", | 20260506 02:13:18.715 | INFO | "huge_shared" : " 27 GB", | 20260506 02:13:18.715 | INFO | "malloc" : " 14 GB", | 20260506 02:13:18.715 | INFO | "pinned" : " 29 GB" | 20260506 02:13:18.715 | INFO | } | 20260506 02:13:18.715 | INFO | Perf report for the last 5.12922 seconds: | 20260506 02:13:18.715 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:13:18.715 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:18.715 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:18.715 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:18.715 | INFO | ANALYSIS : 61.2%, 3.0, 95.0ms, 0 | 20260506 02:13:18.715 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:18.715 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:18.715 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 02:13:18.715 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:18.715 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:18.715 | INFO | Uploaded 36GiB to GPU in 2.09012s (17.2239GiB/s) | 20260506 02:13:18.715 | INFO | Downloaded 34GiB to GPU in 2.03917s (16.6735GiB/s) | 20260506 02:13:23.852 | INFO | Analyzing chunk frames = [8140288,8140800) | 20260506 02:13:23.852 | INFO | MemInfo: { | 20260506 02:13:23.852 | INFO | "gpu" : " 17 GB", | 20260506 02:13:23.852 | INFO | "huge" : "", | 20260506 02:13:23.852 | INFO | "huge_shared" : " 27 GB", | 20260506 02:13:23.852 | INFO | "malloc" : " 14 GB", | 20260506 02:13:23.852 | INFO | "pinned" : " 29 GB" | 20260506 02:13:23.852 | INFO | } | 20260506 02:13:23.852 | INFO | Perf report for the last 5.1374 seconds: | 20260506 02:13:23.853 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:13:23.853 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:23.853 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:23.853 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:23.853 | INFO | ANALYSIS : 61.5%, 3.0, 95.5ms, 0 | 20260506 02:13:23.853 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:23.853 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:23.853 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 02:13:23.853 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:23.853 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:23.853 | INFO | Uploaded 36GiB to GPU in 2.14782s (16.7612GiB/s) | 20260506 02:13:23.853 | INFO | Downloaded 34GiB to GPU in 2.04753s (16.6054GiB/s) | 20260506 02:13:28.994 | INFO | Analyzing chunk frames = [8140800,8141312) | 20260506 02:13:28.994 | INFO | MemInfo: { | 20260506 02:13:28.994 | INFO | "gpu" : " 17 GB", | 20260506 02:13:28.994 | INFO | "huge" : "", | 20260506 02:13:28.994 | INFO | "huge_shared" : " 27 GB", | 20260506 02:13:28.994 | INFO | "malloc" : " 14 GB", | 20260506 02:13:28.994 | INFO | "pinned" : " 29 GB" | 20260506 02:13:28.994 | INFO | } | 20260506 02:13:28.994 | INFO | Perf report for the last 5.14142 seconds: | 20260506 02:13:28.994 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:13:28.994 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:28.994 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:28.994 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:28.994 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 02:13:28.994 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:28.994 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:28.994 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:13:28.994 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:28.994 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:28.994 | INFO | Uploaded 36GiB to GPU in 2.09244s (17.2048GiB/s) | 20260506 02:13:28.994 | INFO | Downloaded 34GiB to GPU in 2.05071s (16.5797GiB/s) | 20260506 02:13:34.136 | INFO | Analyzing chunk frames = [8141312,8141824) | 20260506 02:13:34.136 | INFO | MemInfo: { | 20260506 02:13:34.136 | INFO | "gpu" : " 17 GB", | 20260506 02:13:34.136 | INFO | "huge" : "", | 20260506 02:13:34.136 | INFO | "huge_shared" : " 27 GB", | 20260506 02:13:34.136 | INFO | "malloc" : " 14 GB", | 20260506 02:13:34.136 | INFO | "pinned" : " 29 GB" | 20260506 02:13:34.136 | INFO | } | 20260506 02:13:34.136 | INFO | Perf report for the last 5.14224 seconds: | 20260506 02:13:34.136 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:13:34.136 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:34.136 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:34.136 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:34.136 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 02:13:34.136 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:34.136 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:34.136 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:13:34.136 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:34.136 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:34.136 | INFO | Uploaded 36GiB to GPU in 2.10591s (17.0947GiB/s) | 20260506 02:13:34.137 | INFO | Downloaded 34GiB to GPU in 2.04382s (16.6355GiB/s) | 20260506 02:13:39.275 | INFO | Analyzing chunk frames = [8141824,8142336) | 20260506 02:13:39.275 | INFO | MemInfo: { | 20260506 02:13:39.275 | INFO | "gpu" : " 17 GB", | 20260506 02:13:39.275 | INFO | "huge" : "", | 20260506 02:13:39.275 | INFO | "huge_shared" : " 27 GB", | 20260506 02:13:39.275 | INFO | "malloc" : " 14 GB", | 20260506 02:13:39.275 | INFO | "pinned" : " 29 GB" | 20260506 02:13:39.275 | INFO | } | 20260506 02:13:39.275 | INFO | Perf report for the last 5.13856 seconds: | 20260506 02:13:39.275 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:13:39.275 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:39.275 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:39.275 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:39.275 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260506 02:13:39.275 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:39.275 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:39.275 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 02:13:39.275 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:39.275 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:39.275 | INFO | Uploaded 36GiB to GPU in 2.06248s (17.4547GiB/s) | 20260506 02:13:39.275 | INFO | Downloaded 34GiB to GPU in 2.0791s (16.3533GiB/s) | 20260506 02:13:44.408 | INFO | Analyzing chunk frames = [8142336,8142848) | 20260506 02:13:44.408 | INFO | MemInfo: { | 20260506 02:13:44.408 | INFO | "gpu" : " 17 GB", | 20260506 02:13:44.408 | INFO | "huge" : "", | 20260506 02:13:44.408 | INFO | "huge_shared" : " 27 GB", | 20260506 02:13:44.408 | INFO | "malloc" : " 14 GB", | 20260506 02:13:44.408 | INFO | "pinned" : " 29 GB" | 20260506 02:13:44.408 | INFO | } | 20260506 02:13:44.408 | INFO | Perf report for the last 5.13346 seconds: | 20260506 02:13:44.408 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:13:44.408 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:44.408 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:44.408 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:44.408 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 02:13:44.408 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:44.408 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:44.408 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 02:13:44.408 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:44.408 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:44.409 | INFO | Uploaded 36GiB to GPU in 2.06529s (17.431GiB/s) | 20260506 02:13:44.409 | INFO | Downloaded 34GiB to GPU in 2.0229s (16.8076GiB/s) | 20260506 02:13:44.409 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 02:13:44.409 | INFO | Total mem usage: 15.3 GiB | 20260506 02:13:44.409 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 02:13:44.409 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.2%) | 20260506 02:13:44.409 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 02:13:44.409 | INFO | High Water Breakdown: | 20260506 02:13:44.409 | INFO | 60.9% : 8.54 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 02:13:44.409 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 02:13:44.409 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 02:13:44.409 | INFO | ------------------------------------------ | 20260506 02:13:44.409 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:13:44.409 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 02:13:44.409 | INFO | Total mem usage: 26.5 GiB | 20260506 02:13:44.409 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 02:13:44.409 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 02:13:44.409 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 02:13:44.409 | INFO | High Water Breakdown: | 20260506 02:13:44.409 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:13:44.409 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:13:44.409 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:13:44.409 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:13:44.409 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:13:44.409 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:13:44.409 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 02:13:44.409 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 02:13:44.409 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:13:44.409 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 02:13:44.409 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 02:13:44.409 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 02:13:44.409 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 02:13:44.409 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 02:13:44.409 | INFO | ------------------------------------------ | 20260506 02:13:44.409 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:13:44.409 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 02:13:44.409 | INFO | Total mem usage: 25.6 GiB | 20260506 02:13:44.409 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 02:13:44.409 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 02:13:44.409 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 02:13:44.409 | INFO | High Water Breakdown: | 20260506 02:13:44.409 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 02:13:44.409 | INFO | ------------------------------------------ | 20260506 02:13:44.409 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:13:44.409 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 02:13:44.409 | INFO | Total mem usage: 15.7 GiB | 20260506 02:13:44.409 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 02:13:44.409 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 02:13:44.409 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 02:13:44.409 | INFO | High Water Breakdown: | 20260506 02:13:44.409 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 02:13:44.409 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 02:13:44.409 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 02:13:44.410 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 02:13:44.410 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 02:13:44.410 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 02:13:44.410 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 02:13:44.410 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 02:13:44.410 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 02:13:44.410 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:13:44.410 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 02:13:44.410 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:13:44.410 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 02:13:44.410 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 02:13:44.410 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 02:13:44.410 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 02:13:44.410 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:13:44.410 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 02:13:44.410 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:13:44.410 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:13:44.410 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:13:44.410 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 02:13:44.410 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 02:13:44.410 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:13:44.410 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 02:13:44.410 | INFO | ------------------------------------------ | 20260506 02:13:44.410 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:13:44.410 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 02:13:44.410 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:13:44.410 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:13:44.410 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.36e+01, 1.32e+01 / 7.71e-01 | 20260506 02:13:44.410 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.03e-01, 1.08e-02 / 3.38e-03 | 20260506 02:13:44.410 | INFO | PerfReport: 11.86%, Baseline -- 3.71e+00 / 4.50e+00, 5.79e-01 / 2.78e-01 | 20260506 02:13:44.410 | INFO | PerfReport: 3.30%, Histogram -- 1.03e+00 / 1.54e+00, 1.92e-01 / 7.78e-02 | 20260506 02:13:44.410 | INFO | PerfReport: 15.97%, DME -- 3.99e+01 / 4.39e+01, 3.75e+00 / 9.63e-01 | 20260506 02:13:44.410 | INFO | PerfReport: 57.46%, FrameLabeling -- 1.80e+01 / 1.93e+01, 1.52e+00 / 9.79e-01 | 20260506 02:13:44.410 | INFO | PerfReport: 7.04%, PulseAccumulating -- 2.20e+00 / 2.72e+00, 4.14e-01 / 2.24e-01 | 20260506 02:13:44.410 | INFO | PerfReport: 3.03%, Metrics -- 9.47e-01 / 1.22e+00, 2.54e-01 / 9.94e-02 | 20260506 02:13:44.410 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.26e-01 / 3.73e-01, 5.38e-02 / 2.55e-02 | 20260506 02:13:44.410 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 02:13:44.410 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:13:44.410 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:13:44.410 | INFO | PerfReport: Full Execution -- 9.77e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260506 02:13:44.410 | INFO | PerfReport: 22.20%, Upload -- 2.17e+01 / 0.00e+00, 7.25e+00 / 0.00e+00 | 20260506 02:13:44.410 | INFO | PerfReport: 21.93%, Download -- 2.14e+01 / 0.00e+00, 7.58e+00 / 0.00e+00 | 20260506 02:13:44.410 | INFO | PerfReport: 31.99%, Compute -- 3.13e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 02:13:44.410 | INFO | PerfReport: 2.05%, LockUpload -- 2.00e+00 / 0.00e+00, 7.81e+00 / 0.00e+00 | 20260506 02:13:44.410 | INFO | PerfReport: 1.33%, LockDownload -- 1.30e+00 / 0.00e+00, 3.02e+00 / 0.00e+00 | 20260506 02:13:44.410 | INFO | PerfReport: 20.49%, LockCompute -- 2.00e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260506 02:13:44.410 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 2.13e-02 / 0.00e+00 | 20260506 02:13:44.525 | INFO | Starting a round of Baz IO | 20260506 02:13:47.670 | INFO | Populating RTMetrics report | 20260506 02:13:47.671 | INFO | RTMetrics report populated | 20260506 02:13:47.671 | INFO | Opening tmp RTMetrics json file | 20260506 02:13:47.671 | INFO | Serializing RTMetrics to json | 20260506 02:13:47.673 | INFO | Serialization done, swapping tmp file now... | 20260506 02:13:47.673 | INFO | RTMetrics done | 20260506 02:13:47.748 | INFO | Finished a round of Baz IO: Wrote 1.801 GiB in 3.222 seconds | 20260506 02:13:47.748 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.28% / 0.00% / 78.72% | 20260506 02:13:47.748 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0725985% / 3.97364e-06% / 7.15256e-05% | 20260506 02:13:47.748 | INFO | Baz IO overall: | 20260506 02:13:47.748 | INFO | BytesWritten : 4569394644183 | 20260506 02:13:47.748 | INFO | NumEvents : 2145814822842 | 20260506 02:13:49.550 | INFO | Analyzing chunk frames = [8142848,8143360) | 20260506 02:13:49.550 | INFO | MemInfo: { | 20260506 02:13:49.550 | INFO | "gpu" : " 17 GB", | 20260506 02:13:49.550 | INFO | "huge" : "", | 20260506 02:13:49.550 | INFO | "huge_shared" : " 27 GB", | 20260506 02:13:49.550 | INFO | "malloc" : " 14 GB", | 20260506 02:13:49.550 | INFO | "pinned" : " 29 GB" | 20260506 02:13:49.550 | INFO | } | 20260506 02:13:49.550 | INFO | Perf report for the last 5.14193 seconds: | 20260506 02:13:49.550 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:13:49.550 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:49.550 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:49.550 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:49.550 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260506 02:13:49.550 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:49.550 | INFO | RT_METRICS : 42.8%, 1.0, 22.5ms, 0 | 20260506 02:13:49.551 | INFO | PRE_HQ : 59.6%, 1.7, 53.7ms, 0 | 20260506 02:13:49.551 | INFO | BAZWRITER : 61.8%, 3.4, 109.2ms, 0 | 20260506 02:13:49.551 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:13:49.551 | INFO | Uploaded 36GiB to GPU in 2.17435s (16.5567GiB/s) | 20260506 02:13:49.551 | INFO | Downloaded 38GiB to GPU in 2.24498s (16.9267GiB/s) | 20260506 02:13:54.698 | INFO | Analyzing chunk frames = [8143360,8143872) | 20260506 02:13:54.698 | INFO | MemInfo: { | 20260506 02:13:54.698 | INFO | "gpu" : " 17 GB", | 20260506 02:13:54.698 | INFO | "huge" : "", | 20260506 02:13:54.698 | INFO | "huge_shared" : " 27 GB", | 20260506 02:13:54.698 | INFO | "malloc" : " 14 GB", | 20260506 02:13:54.698 | INFO | "pinned" : " 29 GB" | 20260506 02:13:54.698 | INFO | } | 20260506 02:13:54.699 | INFO | Perf report for the last 5.1484 seconds: | 20260506 02:13:54.699 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:13:54.699 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:54.699 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:54.699 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:54.699 | INFO | ANALYSIS : 60.8%, 3.0, 95.1ms, 0 | 20260506 02:13:54.699 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:54.699 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:54.699 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 02:13:54.699 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:54.699 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:54.699 | INFO | Uploaded 36GiB to GPU in 1.96775s (18.2951GiB/s) | 20260506 02:13:54.699 | INFO | Downloaded 34GiB to GPU in 1.98676s (17.1133GiB/s) | 20260506 02:13:59.837 | INFO | Analyzing chunk frames = [8143872,8144384) | 20260506 02:13:59.837 | INFO | MemInfo: { | 20260506 02:13:59.837 | INFO | "gpu" : " 17 GB", | 20260506 02:13:59.837 | INFO | "huge" : "", | 20260506 02:13:59.837 | INFO | "huge_shared" : " 27 GB", | 20260506 02:13:59.837 | INFO | "malloc" : " 14 GB", | 20260506 02:13:59.837 | INFO | "pinned" : " 29 GB" | 20260506 02:13:59.837 | INFO | } | 20260506 02:13:59.838 | INFO | Perf report for the last 5.13895 seconds: | 20260506 02:13:59.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:13:59.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:59.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:59.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:59.838 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 02:13:59.838 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:59.838 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:59.838 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:13:59.838 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:59.838 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:13:59.838 | INFO | Uploaded 36GiB to GPU in 2.11664s (17.0081GiB/s) | 20260506 02:13:59.838 | INFO | Downloaded 34GiB to GPU in 2.04554s (16.6215GiB/s) | 20260506 02:14:04.969 | INFO | Analyzing chunk frames = [8144384,8144896) | 20260506 02:14:04.969 | INFO | MemInfo: { | 20260506 02:14:04.969 | INFO | "gpu" : " 17 GB", | 20260506 02:14:04.969 | INFO | "huge" : "", | 20260506 02:14:04.969 | INFO | "huge_shared" : " 27 GB", | 20260506 02:14:04.969 | INFO | "malloc" : " 14 GB", | 20260506 02:14:04.969 | INFO | "pinned" : " 29 GB" | 20260506 02:14:04.969 | INFO | } | 20260506 02:14:04.969 | INFO | Perf report for the last 5.13162 seconds: | 20260506 02:14:04.969 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:14:04.969 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:04.969 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:04.969 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:04.969 | INFO | ANALYSIS : 61.1%, 3.0, 94.7ms, 0 | 20260506 02:14:04.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:04.969 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:04.969 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 02:14:04.969 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:04.969 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:04.970 | INFO | Uploaded 36GiB to GPU in 2.09435s (17.1891GiB/s) | 20260506 02:14:04.970 | INFO | Downloaded 34GiB to GPU in 2.00772s (16.9347GiB/s) | 20260506 02:14:10.107 | INFO | Analyzing chunk frames = [8144896,8145408) | 20260506 02:14:10.107 | INFO | MemInfo: { | 20260506 02:14:10.107 | INFO | "gpu" : " 17 GB", | 20260506 02:14:10.107 | INFO | "huge" : "", | 20260506 02:14:10.107 | INFO | "huge_shared" : " 27 GB", | 20260506 02:14:10.107 | INFO | "malloc" : " 14 GB", | 20260506 02:14:10.107 | INFO | "pinned" : " 29 GB" | 20260506 02:14:10.107 | INFO | } | 20260506 02:14:10.107 | INFO | Perf report for the last 5.13822 seconds: | 20260506 02:14:10.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:14:10.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:10.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:10.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:10.107 | INFO | ANALYSIS : 60.7%, 3.0, 94.8ms, 0 | 20260506 02:14:10.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:10.107 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:10.107 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 02:14:10.107 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:10.108 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:10.108 | INFO | Uploaded 36GiB to GPU in 2.08948s (17.2292GiB/s) | 20260506 02:14:10.108 | INFO | Downloaded 34GiB to GPU in 2.02677s (16.7754GiB/s) | 20260506 02:14:15.250 | INFO | Analyzing chunk frames = [8145408,8145920) | 20260506 02:14:15.250 | INFO | MemInfo: { | 20260506 02:14:15.250 | INFO | "gpu" : " 17 GB", | 20260506 02:14:15.250 | INFO | "huge" : "", | 20260506 02:14:15.250 | INFO | "huge_shared" : " 27 GB", | 20260506 02:14:15.250 | INFO | "malloc" : " 14 GB", | 20260506 02:14:15.250 | INFO | "pinned" : " 29 GB" | 20260506 02:14:15.250 | INFO | } | 20260506 02:14:15.250 | INFO | Perf report for the last 5.14305 seconds: | 20260506 02:14:15.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:14:15.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:15.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:15.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:15.251 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 02:14:15.251 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:15.251 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:15.251 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 02:14:15.251 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:15.251 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:15.251 | INFO | Uploaded 36GiB to GPU in 2.07979s (17.3094GiB/s) | 20260506 02:14:15.251 | INFO | Downloaded 34GiB to GPU in 2.03668s (16.6938GiB/s) | 20260506 02:14:20.387 | INFO | Analyzing chunk frames = [8145920,8146432) | 20260506 02:14:20.387 | INFO | MemInfo: { | 20260506 02:14:20.387 | INFO | "gpu" : " 17 GB", | 20260506 02:14:20.387 | INFO | "huge" : "", | 20260506 02:14:20.387 | INFO | "huge_shared" : " 27 GB", | 20260506 02:14:20.387 | INFO | "malloc" : " 14 GB", | 20260506 02:14:20.387 | INFO | "pinned" : " 29 GB" | 20260506 02:14:20.387 | INFO | } | 20260506 02:14:20.387 | INFO | Perf report for the last 5.13651 seconds: | 20260506 02:14:20.387 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:14:20.387 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:20.387 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:20.387 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:20.387 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260506 02:14:20.387 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:20.387 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:20.387 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 02:14:20.387 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:20.387 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:20.387 | INFO | Uploaded 36GiB to GPU in 2.08228s (17.2887GiB/s) | 20260506 02:14:20.387 | INFO | Downloaded 34GiB to GPU in 2.08531s (16.3045GiB/s) | 20260506 02:14:25.531 | INFO | Analyzing chunk frames = [8146432,8146944) | 20260506 02:14:25.531 | INFO | MemInfo: { | 20260506 02:14:25.531 | INFO | "gpu" : " 17 GB", | 20260506 02:14:25.531 | INFO | "huge" : "", | 20260506 02:14:25.531 | INFO | "huge_shared" : " 27 GB", | 20260506 02:14:25.531 | INFO | "malloc" : " 14 GB", | 20260506 02:14:25.531 | INFO | "pinned" : " 29 GB" | 20260506 02:14:25.531 | INFO | } | 20260506 02:14:25.531 | INFO | Perf report for the last 5.14423 seconds: | 20260506 02:14:25.531 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:14:25.531 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:25.531 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:25.531 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:25.531 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260506 02:14:25.531 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:25.531 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:25.532 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:14:25.532 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:25.532 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:25.532 | INFO | Uploaded 36GiB to GPU in 2.13192s (16.8862GiB/s) | 20260506 02:14:25.532 | INFO | Downloaded 34GiB to GPU in 2.0272s (16.7719GiB/s) | 20260506 02:14:25.657 | INFO | Starting a round of Baz IO | 20260506 02:14:28.789 | INFO | Populating RTMetrics report | 20260506 02:14:28.790 | INFO | RTMetrics report populated | 20260506 02:14:28.795 | INFO | Opening tmp RTMetrics json file | 20260506 02:14:28.795 | INFO | Serializing RTMetrics to json | 20260506 02:14:28.796 | INFO | Serialization done, swapping tmp file now... | 20260506 02:14:28.798 | INFO | RTMetrics done | 20260506 02:14:28.866 | INFO | Finished a round of Baz IO: Wrote 1.799 GiB in 3.209 seconds | 20260506 02:14:28.866 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.28% / 0.00% / 78.72% | 20260506 02:14:28.867 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0721494% / 0% / 8.34465e-05% | 20260506 02:14:28.867 | INFO | Baz IO overall: | 20260506 02:14:28.867 | INFO | BytesWritten : 4571326549314 | 20260506 02:14:28.867 | INFO | NumEvents : 2146693534401 | 20260506 02:14:30.667 | INFO | Analyzing chunk frames = [8146944,8147456) | 20260506 02:14:30.667 | INFO | MemInfo: { | 20260506 02:14:30.667 | INFO | "gpu" : " 17 GB", | 20260506 02:14:30.667 | INFO | "huge" : "", | 20260506 02:14:30.667 | INFO | "huge_shared" : " 27 GB", | 20260506 02:14:30.668 | INFO | "malloc" : " 14 GB", | 20260506 02:14:30.668 | INFO | "pinned" : " 29 GB" | 20260506 02:14:30.668 | INFO | } | 20260506 02:14:30.668 | INFO | Perf report for the last 5.13644 seconds: | 20260506 02:14:30.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:14:30.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:30.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:30.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:30.668 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260506 02:14:30.668 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:30.668 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260506 02:14:30.668 | INFO | PRE_HQ : 59.6%, 1.7, 53.5ms, 0 | 20260506 02:14:30.668 | INFO | BAZWRITER : 61.7%, 3.8, 121.2ms, 0 | 20260506 02:14:30.668 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:14:30.668 | INFO | Uploaded 36GiB to GPU in 2.19262s (16.4187GiB/s) | 20260506 02:14:30.668 | INFO | Downloaded 38GiB to GPU in 2.24839s (16.901GiB/s) | 20260506 02:14:35.803 | INFO | Analyzing chunk frames = [8147456,8147968) | 20260506 02:14:35.803 | INFO | MemInfo: { | 20260506 02:14:35.803 | INFO | "gpu" : " 17 GB", | 20260506 02:14:35.803 | INFO | "huge" : "", | 20260506 02:14:35.803 | INFO | "huge_shared" : " 27 GB", | 20260506 02:14:35.803 | INFO | "malloc" : " 14 GB", | 20260506 02:14:35.804 | INFO | "pinned" : " 29 GB" | 20260506 02:14:35.804 | INFO | } | 20260506 02:14:35.804 | INFO | Perf report for the last 5.13588 seconds: | 20260506 02:14:35.804 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:14:35.804 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:35.804 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:35.804 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:35.804 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260506 02:14:35.804 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:35.804 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:35.804 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 02:14:35.804 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:35.804 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:35.804 | INFO | Uploaded 36GiB to GPU in 1.98752s (18.113GiB/s) | 20260506 02:14:35.804 | INFO | Downloaded 34GiB to GPU in 1.98607s (17.1192GiB/s) | 20260506 02:14:40.944 | INFO | Analyzing chunk frames = [8147968,8148480) | 20260506 02:14:40.944 | INFO | MemInfo: { | 20260506 02:14:40.944 | INFO | "gpu" : " 17 GB", | 20260506 02:14:40.944 | INFO | "huge" : "", | 20260506 02:14:40.944 | INFO | "huge_shared" : " 27 GB", | 20260506 02:14:40.944 | INFO | "malloc" : " 14 GB", | 20260506 02:14:40.944 | INFO | "pinned" : " 29 GB" | 20260506 02:14:40.944 | INFO | } | 20260506 02:14:40.944 | INFO | Perf report for the last 5.14065 seconds: | 20260506 02:14:40.944 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:14:40.944 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:40.944 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:40.944 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:40.944 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 02:14:40.944 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:40.944 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:40.944 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:14:40.944 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:40.945 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:40.945 | INFO | Uploaded 36GiB to GPU in 2.08567s (17.2606GiB/s) | 20260506 02:14:40.945 | INFO | Downloaded 34GiB to GPU in 2.02077s (16.8253GiB/s) | 20260506 02:14:46.088 | INFO | Analyzing chunk frames = [8148480,8148992) | 20260506 02:14:46.088 | INFO | MemInfo: { | 20260506 02:14:46.088 | INFO | "gpu" : " 17 GB", | 20260506 02:14:46.088 | INFO | "huge" : "", | 20260506 02:14:46.088 | INFO | "huge_shared" : " 27 GB", | 20260506 02:14:46.088 | INFO | "malloc" : " 14 GB", | 20260506 02:14:46.088 | INFO | "pinned" : " 29 GB" | 20260506 02:14:46.088 | INFO | } | 20260506 02:14:46.088 | INFO | Perf report for the last 5.14408 seconds: | 20260506 02:14:46.088 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:14:46.088 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:46.088 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:46.088 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:46.088 | INFO | ANALYSIS : 61.0%, 3.0, 94.8ms, 0 | 20260506 02:14:46.088 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:46.088 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:46.088 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:14:46.088 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:46.089 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:46.089 | INFO | Uploaded 36GiB to GPU in 2.07695s (17.3331GiB/s) | 20260506 02:14:46.089 | INFO | Downloaded 34GiB to GPU in 2.00784s (16.9337GiB/s) | 20260506 02:14:51.224 | INFO | Analyzing chunk frames = [8148992,8149504) | 20260506 02:14:51.224 | INFO | MemInfo: { | 20260506 02:14:51.224 | INFO | "gpu" : " 17 GB", | 20260506 02:14:51.224 | INFO | "huge" : "", | 20260506 02:14:51.224 | INFO | "huge_shared" : " 27 GB", | 20260506 02:14:51.224 | INFO | "malloc" : " 14 GB", | 20260506 02:14:51.224 | INFO | "pinned" : " 29 GB" | 20260506 02:14:51.224 | INFO | } | 20260506 02:14:51.224 | INFO | Perf report for the last 5.13526 seconds: | 20260506 02:14:51.224 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:14:51.224 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:51.224 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:51.224 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:51.224 | INFO | ANALYSIS : 60.8%, 3.0, 94.9ms, 0 | 20260506 02:14:51.224 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:51.224 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:51.224 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:14:51.224 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:51.224 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:51.224 | INFO | Uploaded 36GiB to GPU in 2.10247s (17.1227GiB/s) | 20260506 02:14:51.224 | INFO | Downloaded 34GiB to GPU in 2.03375s (16.7179GiB/s) | 20260506 02:14:56.361 | INFO | Analyzing chunk frames = [8149504,8150016) | 20260506 02:14:56.361 | INFO | MemInfo: { | 20260506 02:14:56.361 | INFO | "gpu" : " 17 GB", | 20260506 02:14:56.361 | INFO | "huge" : "", | 20260506 02:14:56.361 | INFO | "huge_shared" : " 27 GB", | 20260506 02:14:56.361 | INFO | "malloc" : " 14 GB", | 20260506 02:14:56.361 | INFO | "pinned" : " 29 GB" | 20260506 02:14:56.361 | INFO | } | 20260506 02:14:56.361 | INFO | Perf report for the last 5.13733 seconds: | 20260506 02:14:56.361 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:14:56.361 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:56.361 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:56.361 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:56.361 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 02:14:56.361 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:56.361 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:56.361 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:14:56.361 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:56.361 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:14:56.361 | INFO | Uploaded 36GiB to GPU in 2.06694s (17.417GiB/s) | 20260506 02:14:56.361 | INFO | Downloaded 34GiB to GPU in 2.03767s (16.6857GiB/s) | 20260506 02:15:01.503 | INFO | Analyzing chunk frames = [8150016,8150528) | 20260506 02:15:01.503 | INFO | MemInfo: { | 20260506 02:15:01.504 | INFO | "gpu" : " 17 GB", | 20260506 02:15:01.504 | INFO | "huge" : "", | 20260506 02:15:01.504 | INFO | "huge_shared" : " 27 GB", | 20260506 02:15:01.504 | INFO | "malloc" : " 14 GB", | 20260506 02:15:01.504 | INFO | "pinned" : " 29 GB" | 20260506 02:15:01.504 | INFO | } | 20260506 02:15:01.504 | INFO | Perf report for the last 5.14282 seconds: | 20260506 02:15:01.504 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:15:01.504 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:01.504 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:01.504 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:01.504 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260506 02:15:01.504 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:01.504 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:01.504 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 02:15:01.504 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:01.504 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:01.504 | INFO | Uploaded 36GiB to GPU in 2.0384s (17.6609GiB/s) | 20260506 02:15:01.504 | INFO | Downloaded 34GiB to GPU in 2.03792s (16.6837GiB/s) | 20260506 02:15:06.652 | INFO | Analyzing chunk frames = [8150528,8151040) | 20260506 02:15:06.652 | INFO | MemInfo: { | 20260506 02:15:06.652 | INFO | "gpu" : " 17 GB", | 20260506 02:15:06.652 | INFO | "huge" : "", | 20260506 02:15:06.652 | INFO | "huge_shared" : " 27 GB", | 20260506 02:15:06.652 | INFO | "malloc" : " 14 GB", | 20260506 02:15:06.652 | INFO | "pinned" : " 29 GB" | 20260506 02:15:06.652 | INFO | } | 20260506 02:15:06.652 | INFO | Perf report for the last 5.14784 seconds: | 20260506 02:15:06.652 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:15:06.652 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:06.652 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:06.652 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:06.652 | INFO | ANALYSIS : 61.4%, 3.0, 96.0ms, 0 | 20260506 02:15:06.652 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:06.652 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:06.652 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:15:06.652 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:06.652 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:06.652 | INFO | Uploaded 36GiB to GPU in 2.06512s (17.4324GiB/s) | 20260506 02:15:06.652 | INFO | Downloaded 34GiB to GPU in 2.03095s (16.7409GiB/s) | 20260506 02:15:06.652 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 02:15:06.652 | INFO | Total mem usage: 15.3 GiB | 20260506 02:15:06.652 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 02:15:06.652 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -3.54%) | 20260506 02:15:06.652 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 02:15:06.652 | INFO | High Water Breakdown: | 20260506 02:15:06.652 | INFO | 61.7% : 9.13 GiB ( 96% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 02:15:06.652 | INFO | 38.2% : 5.65 GiB ( 85% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 02:15:06.652 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 02:15:06.652 | INFO | ------------------------------------------ | 20260506 02:15:06.652 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:15:06.652 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 02:15:06.652 | INFO | Total mem usage: 26.5 GiB | 20260506 02:15:06.652 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 02:15:06.652 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 02:15:06.652 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260506 02:15:06.652 | INFO | High Water Breakdown: | 20260506 02:15:06.652 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:15:06.652 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:15:06.652 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:15:06.652 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:15:06.652 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:15:06.652 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:15:06.652 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 02:15:06.652 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 02:15:06.652 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:15:06.652 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 02:15:06.652 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 02:15:06.652 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 02:15:06.652 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 02:15:06.652 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 02:15:06.652 | INFO | ------------------------------------------ | 20260506 02:15:06.652 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:15:06.652 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 02:15:06.652 | INFO | Total mem usage: 25.6 GiB | 20260506 02:15:06.653 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 02:15:06.653 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 02:15:06.653 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 02:15:06.653 | INFO | High Water Breakdown: | 20260506 02:15:06.653 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 02:15:06.653 | INFO | ------------------------------------------ | 20260506 02:15:06.653 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:15:06.653 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 02:15:06.653 | INFO | Total mem usage: 15.7 GiB | 20260506 02:15:06.653 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 02:15:06.653 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 02:15:06.653 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 02:15:06.653 | INFO | High Water Breakdown: | 20260506 02:15:06.653 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 02:15:06.653 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 02:15:06.653 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 02:15:06.653 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 02:15:06.653 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 02:15:06.653 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 02:15:06.653 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 02:15:06.653 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 02:15:06.653 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 02:15:06.653 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:15:06.653 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 02:15:06.653 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:15:06.653 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 02:15:06.653 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 02:15:06.653 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 02:15:06.653 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 02:15:06.653 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:15:06.653 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 02:15:06.653 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:15:06.653 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:15:06.653 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:15:06.653 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 02:15:06.653 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 02:15:06.653 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:15:06.653 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 02:15:06.653 | INFO | ------------------------------------------ | 20260506 02:15:06.653 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:15:06.653 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 02:15:06.653 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:15:06.653 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:15:06.653 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.39e+01, 1.34e+01 / 6.99e-01 | 20260506 02:15:06.653 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.04e-01, 1.01e-02 / 3.77e-03 | 20260506 02:15:06.653 | INFO | PerfReport: 11.86%, Baseline -- 3.71e+00 / 4.51e+00, 5.81e-01 / 1.13e-01 | 20260506 02:15:06.653 | INFO | PerfReport: 3.30%, Histogram -- 1.03e+00 / 1.52e+00, 1.95e-01 / 9.21e-02 | 20260506 02:15:06.653 | INFO | PerfReport: 16.01%, DME -- 4.01e+01 / 4.41e+01, 3.88e+00 / 1.13e+00 | 20260506 02:15:06.653 | INFO | PerfReport: 57.45%, FrameLabeling -- 1.80e+01 / 1.93e+01, 1.53e+00 / 9.72e-01 | 20260506 02:15:06.653 | INFO | PerfReport: 7.03%, PulseAccumulating -- 2.20e+00 / 2.74e+00, 4.19e-01 / 1.93e-01 | 20260506 02:15:06.653 | INFO | PerfReport: 3.01%, Metrics -- 9.42e-01 / 1.20e+00, 2.52e-01 / 1.08e-01 | 20260506 02:15:06.653 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.26e-01 / 3.75e-01, 5.37e-02 / 3.08e-02 | 20260506 02:15:06.653 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 02:15:06.653 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:15:06.653 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:15:06.653 | INFO | PerfReport: Full Execution -- 9.77e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260506 02:15:06.653 | INFO | PerfReport: 22.23%, Upload -- 2.17e+01 / 0.00e+00, 7.28e+00 / 0.00e+00 | 20260506 02:15:06.653 | INFO | PerfReport: 21.90%, Download -- 2.14e+01 / 0.00e+00, 7.55e+00 / 0.00e+00 | 20260506 02:15:06.653 | INFO | PerfReport: 32.03%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 02:15:06.653 | INFO | PerfReport: 2.05%, LockUpload -- 2.00e+00 / 0.00e+00, 7.90e+00 / 0.00e+00 | 20260506 02:15:06.653 | INFO | PerfReport: 1.31%, LockDownload -- 1.28e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260506 02:15:06.653 | INFO | PerfReport: 20.47%, LockCompute -- 2.00e+01 / 0.00e+00, 1.78e+01 / 0.00e+00 | 20260506 02:15:06.653 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.89e-02 / 0.00e+00 | 20260506 02:15:06.774 | INFO | Starting a round of Baz IO | 20260506 02:15:09.894 | INFO | Populating RTMetrics report | 20260506 02:15:09.894 | INFO | RTMetrics report populated | 20260506 02:15:09.894 | INFO | Opening tmp RTMetrics json file | 20260506 02:15:09.895 | INFO | Serializing RTMetrics to json | 20260506 02:15:09.896 | INFO | Serialization done, swapping tmp file now... | 20260506 02:15:09.897 | INFO | RTMetrics done | 20260506 02:15:09.969 | INFO | Finished a round of Baz IO: Wrote 1.798 GiB in 3.195 seconds | 20260506 02:15:09.969 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.28% / 0.00% / 78.72% | 20260506 02:15:09.969 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0721931% / 0% / 8.34465e-05% | 20260506 02:15:09.969 | INFO | Baz IO overall: | 20260506 02:15:09.969 | INFO | BytesWritten : 4573257495306 | 20260506 02:15:09.969 | INFO | NumEvents : 2147571635068 | 20260506 02:15:11.793 | INFO | Analyzing chunk frames = [8151040,8151552) | 20260506 02:15:11.793 | INFO | MemInfo: { | 20260506 02:15:11.793 | INFO | "gpu" : " 17 GB", | 20260506 02:15:11.793 | INFO | "huge" : "", | 20260506 02:15:11.793 | INFO | "huge_shared" : " 27 GB", | 20260506 02:15:11.793 | INFO | "malloc" : " 14 GB", | 20260506 02:15:11.793 | INFO | "pinned" : " 29 GB" | 20260506 02:15:11.793 | INFO | } | 20260506 02:15:11.793 | INFO | Perf report for the last 5.14174 seconds: | 20260506 02:15:11.793 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:15:11.793 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:11.793 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:11.793 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:11.793 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 02:15:11.793 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:11.794 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260506 02:15:11.794 | INFO | PRE_HQ : 59.2%, 1.7, 53.7ms, 0 | 20260506 02:15:11.794 | INFO | BAZWRITER : 61.3%, 3.5, 109.6ms, 0 | 20260506 02:15:11.794 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:15:11.794 | INFO | Uploaded 36GiB to GPU in 2.08299s (17.2829GiB/s) | 20260506 02:15:11.794 | INFO | Downloaded 38GiB to GPU in 2.22208s (17.1011GiB/s) | 20260506 02:15:16.921 | INFO | Analyzing chunk frames = [8151552,8152064) | 20260506 02:15:16.921 | INFO | MemInfo: { | 20260506 02:15:16.921 | INFO | "gpu" : " 17 GB", | 20260506 02:15:16.921 | INFO | "huge" : "", | 20260506 02:15:16.921 | INFO | "huge_shared" : " 27 GB", | 20260506 02:15:16.921 | INFO | "malloc" : " 14 GB", | 20260506 02:15:16.921 | INFO | "pinned" : " 29 GB" | 20260506 02:15:16.921 | INFO | } | 20260506 02:15:16.921 | INFO | Perf report for the last 5.12748 seconds: | 20260506 02:15:16.921 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:15:16.921 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:16.921 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:16.921 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:16.921 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260506 02:15:16.921 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:16.921 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:16.921 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:15:16.921 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:16.921 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:16.921 | INFO | Uploaded 36GiB to GPU in 1.95796s (18.3865GiB/s) | 20260506 02:15:16.921 | INFO | Downloaded 34GiB to GPU in 1.9909s (17.0777GiB/s) | 20260506 02:15:22.059 | INFO | Analyzing chunk frames = [8152064,8152576) | 20260506 02:15:22.059 | INFO | MemInfo: { | 20260506 02:15:22.059 | INFO | "gpu" : " 17 GB", | 20260506 02:15:22.059 | INFO | "huge" : "", | 20260506 02:15:22.059 | INFO | "huge_shared" : " 27 GB", | 20260506 02:15:22.059 | INFO | "malloc" : " 14 GB", | 20260506 02:15:22.059 | INFO | "pinned" : " 29 GB" | 20260506 02:15:22.059 | INFO | } | 20260506 02:15:22.060 | INFO | Perf report for the last 5.13864 seconds: | 20260506 02:15:22.060 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:15:22.060 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:22.060 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:22.060 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:22.060 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 02:15:22.060 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:22.060 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:22.060 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 02:15:22.060 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:22.060 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:22.060 | INFO | Uploaded 36GiB to GPU in 2.09288s (17.2012GiB/s) | 20260506 02:15:22.060 | INFO | Downloaded 34GiB to GPU in 2.02378s (16.8003GiB/s) | 20260506 02:15:27.199 | INFO | Analyzing chunk frames = [8152576,8153088) | 20260506 02:15:27.199 | INFO | MemInfo: { | 20260506 02:15:27.199 | INFO | "gpu" : " 17 GB", | 20260506 02:15:27.199 | INFO | "huge" : "", | 20260506 02:15:27.199 | INFO | "huge_shared" : " 27 GB", | 20260506 02:15:27.199 | INFO | "malloc" : " 14 GB", | 20260506 02:15:27.199 | INFO | "pinned" : " 29 GB" | 20260506 02:15:27.199 | INFO | } | 20260506 02:15:27.199 | INFO | Perf report for the last 5.13957 seconds: | 20260506 02:15:27.199 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:15:27.199 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:27.199 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:27.199 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:27.199 | INFO | ANALYSIS : 61.2%, 3.0, 95.1ms, 0 | 20260506 02:15:27.199 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:27.199 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:27.199 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 02:15:27.199 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:27.199 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:27.199 | INFO | Uploaded 36GiB to GPU in 2.09893s (17.1516GiB/s) | 20260506 02:15:27.199 | INFO | Downloaded 34GiB to GPU in 2.02527s (16.7879GiB/s) | 20260506 02:15:32.340 | INFO | Analyzing chunk frames = [8153088,8153600) | 20260506 02:15:32.341 | INFO | MemInfo: { | 20260506 02:15:32.341 | INFO | "gpu" : " 17 GB", | 20260506 02:15:32.341 | INFO | "huge" : "", | 20260506 02:15:32.341 | INFO | "huge_shared" : " 27 GB", | 20260506 02:15:32.341 | INFO | "malloc" : " 14 GB", | 20260506 02:15:32.341 | INFO | "pinned" : " 29 GB" | 20260506 02:15:32.341 | INFO | } | 20260506 02:15:32.341 | INFO | Perf report for the last 5.1416 seconds: | 20260506 02:15:32.341 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:15:32.341 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:32.341 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:32.341 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:32.341 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260506 02:15:32.341 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:32.341 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:32.341 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 02:15:32.341 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:32.341 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:32.341 | INFO | Uploaded 36GiB to GPU in 2.0794s (17.3127GiB/s) | 20260506 02:15:32.341 | INFO | Downloaded 34GiB to GPU in 2.03595s (16.6998GiB/s) | 20260506 02:15:37.478 | INFO | Analyzing chunk frames = [8153600,8154112) | 20260506 02:15:37.478 | INFO | MemInfo: { | 20260506 02:15:37.478 | INFO | "gpu" : " 17 GB", | 20260506 02:15:37.478 | INFO | "huge" : "", | 20260506 02:15:37.478 | INFO | "huge_shared" : " 27 GB", | 20260506 02:15:37.479 | INFO | "malloc" : " 14 GB", | 20260506 02:15:37.479 | INFO | "pinned" : " 29 GB" | 20260506 02:15:37.479 | INFO | } | 20260506 02:15:37.479 | INFO | Perf report for the last 5.1379 seconds: | 20260506 02:15:37.479 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:15:37.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:37.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:37.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:37.479 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 02:15:37.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:37.479 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:37.479 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:15:37.479 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:37.479 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:37.479 | INFO | Uploaded 36GiB to GPU in 2.06966s (17.3941GiB/s) | 20260506 02:15:37.479 | INFO | Downloaded 34GiB to GPU in 2.03874s (16.6769GiB/s) | 20260506 02:15:42.614 | INFO | Analyzing chunk frames = [8154112,8154624) | 20260506 02:15:42.614 | INFO | MemInfo: { | 20260506 02:15:42.614 | INFO | "gpu" : " 17 GB", | 20260506 02:15:42.614 | INFO | "huge" : "", | 20260506 02:15:42.614 | INFO | "huge_shared" : " 27 GB", | 20260506 02:15:42.614 | INFO | "malloc" : " 14 GB", | 20260506 02:15:42.614 | INFO | "pinned" : " 29 GB" | 20260506 02:15:42.614 | INFO | } | 20260506 02:15:42.614 | INFO | Perf report for the last 5.13569 seconds: | 20260506 02:15:42.614 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:15:42.614 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:42.614 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:42.614 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:42.614 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260506 02:15:42.614 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:42.614 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:42.614 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 02:15:42.614 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:42.615 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:42.615 | INFO | Uploaded 36GiB to GPU in 2.03371s (17.7017GiB/s) | 20260506 02:15:42.615 | INFO | Downloaded 34GiB to GPU in 2.04749s (16.6057GiB/s) | 20260506 02:15:47.760 | INFO | Analyzing chunk frames = [8154624,8155136) | 20260506 02:15:47.760 | INFO | MemInfo: { | 20260506 02:15:47.760 | INFO | "gpu" : " 17 GB", | 20260506 02:15:47.760 | INFO | "huge" : "", | 20260506 02:15:47.760 | INFO | "huge_shared" : " 27 GB", | 20260506 02:15:47.760 | INFO | "malloc" : " 14 GB", | 20260506 02:15:47.760 | INFO | "pinned" : " 29 GB" | 20260506 02:15:47.760 | INFO | } | 20260506 02:15:47.760 | INFO | Perf report for the last 5.1456 seconds: | 20260506 02:15:47.760 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:15:47.760 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:47.760 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:47.760 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:47.760 | INFO | ANALYSIS : 61.9%, 3.0, 96.8ms, 0 | 20260506 02:15:47.760 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:47.760 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:47.760 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 02:15:47.760 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:47.760 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:47.760 | INFO | Uploaded 36GiB to GPU in 2.13086s (16.8946GiB/s) | 20260506 02:15:47.760 | INFO | Downloaded 34GiB to GPU in 2.02833s (16.7625GiB/s) | 20260506 02:15:47.885 | INFO | Starting a round of Baz IO | 20260506 02:15:51.011 | INFO | Populating RTMetrics report | 20260506 02:15:51.011 | INFO | RTMetrics report populated | 20260506 02:15:51.011 | INFO | Opening tmp RTMetrics json file | 20260506 02:15:51.012 | INFO | Serializing RTMetrics to json | 20260506 02:15:51.013 | INFO | Serialization done, swapping tmp file now... | 20260506 02:15:51.013 | INFO | RTMetrics done | 20260506 02:15:51.086 | INFO | Finished a round of Baz IO: Wrote 1.799 GiB in 3.201 seconds | 20260506 02:15:51.086 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.28% / 0.00% / 78.72% | 20260506 02:15:51.086 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0724912% / 0% / 9.13938e-05% | 20260506 02:15:51.086 | INFO | Baz IO overall: | 20260506 02:15:51.086 | INFO | BytesWritten : 4575189041743 | 20260506 02:15:51.087 | INFO | NumEvents : 2148450419082 | 20260506 02:15:52.900 | INFO | Analyzing chunk frames = [8155136,8155648) | 20260506 02:15:52.900 | INFO | MemInfo: { | 20260506 02:15:52.900 | INFO | "gpu" : " 17 GB", | 20260506 02:15:52.900 | INFO | "huge" : "", | 20260506 02:15:52.900 | INFO | "huge_shared" : " 27 GB", | 20260506 02:15:52.900 | INFO | "malloc" : " 14 GB", | 20260506 02:15:52.900 | INFO | "pinned" : " 29 GB" | 20260506 02:15:52.900 | INFO | } | 20260506 02:15:52.900 | INFO | Perf report for the last 5.13998 seconds: | 20260506 02:15:52.900 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:15:52.900 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:52.900 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:52.900 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:52.900 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260506 02:15:52.900 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:52.900 | INFO | RT_METRICS : 42.8%, 1.0, 22.5ms, 0 | 20260506 02:15:52.900 | INFO | PRE_HQ : 59.2%, 1.7, 53.5ms, 0 | 20260506 02:15:52.900 | INFO | BAZWRITER : 61.4%, 3.5, 110.2ms, 0 | 20260506 02:15:52.900 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:15:52.900 | INFO | Uploaded 36GiB to GPU in 2.09745s (17.1637GiB/s) | 20260506 02:15:52.900 | INFO | Downloaded 38GiB to GPU in 2.21975s (17.1191GiB/s) | 20260506 02:15:58.033 | INFO | Analyzing chunk frames = [8155648,8156160) | 20260506 02:15:58.033 | INFO | MemInfo: { | 20260506 02:15:58.033 | INFO | "gpu" : " 17 GB", | 20260506 02:15:58.033 | INFO | "huge" : "", | 20260506 02:15:58.033 | INFO | "huge_shared" : " 27 GB", | 20260506 02:15:58.033 | INFO | "malloc" : " 14 GB", | 20260506 02:15:58.033 | INFO | "pinned" : " 29 GB" | 20260506 02:15:58.033 | INFO | } | 20260506 02:15:58.033 | INFO | Perf report for the last 5.1335 seconds: | 20260506 02:15:58.033 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:15:58.033 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:58.033 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:58.033 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:58.033 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 02:15:58.033 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:58.034 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:58.034 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:15:58.034 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:58.034 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:15:58.034 | INFO | Uploaded 36GiB to GPU in 1.96148s (18.3535GiB/s) | 20260506 02:15:58.034 | INFO | Downloaded 34GiB to GPU in 1.9826s (17.1492GiB/s) | 20260506 02:16:03.171 | INFO | Analyzing chunk frames = [8156160,8156672) | 20260506 02:16:03.171 | INFO | MemInfo: { | 20260506 02:16:03.171 | INFO | "gpu" : " 17 GB", | 20260506 02:16:03.171 | INFO | "huge" : "", | 20260506 02:16:03.171 | INFO | "huge_shared" : " 27 GB", | 20260506 02:16:03.171 | INFO | "malloc" : " 14 GB", | 20260506 02:16:03.171 | INFO | "pinned" : " 29 GB" | 20260506 02:16:03.171 | INFO | } | 20260506 02:16:03.171 | INFO | Perf report for the last 5.13715 seconds: | 20260506 02:16:03.171 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:16:03.171 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:03.171 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:03.171 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:03.171 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 02:16:03.171 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:03.171 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:03.171 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:16:03.171 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:03.171 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:03.171 | INFO | Uploaded 36GiB to GPU in 2.07776s (17.3263GiB/s) | 20260506 02:16:03.171 | INFO | Downloaded 34GiB to GPU in 2.0197s (16.8342GiB/s) | 20260506 02:16:08.311 | INFO | Analyzing chunk frames = [8156672,8157184) | 20260506 02:16:08.311 | INFO | MemInfo: { | 20260506 02:16:08.311 | INFO | "gpu" : " 17 GB", | 20260506 02:16:08.311 | INFO | "huge" : "", | 20260506 02:16:08.311 | INFO | "huge_shared" : " 27 GB", | 20260506 02:16:08.311 | INFO | "malloc" : " 14 GB", | 20260506 02:16:08.311 | INFO | "pinned" : " 29 GB" | 20260506 02:16:08.311 | INFO | } | 20260506 02:16:08.312 | INFO | Perf report for the last 5.14113 seconds: | 20260506 02:16:08.312 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:16:08.312 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:08.312 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:08.312 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:08.312 | INFO | ANALYSIS : 61.6%, 3.0, 95.7ms, 0 | 20260506 02:16:08.312 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:08.312 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:08.312 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 02:16:08.312 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:08.312 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:08.312 | INFO | Uploaded 36GiB to GPU in 2.14215s (16.8056GiB/s) | 20260506 02:16:08.312 | INFO | Downloaded 34GiB to GPU in 2.02346s (16.8029GiB/s) | 20260506 02:16:13.452 | INFO | Analyzing chunk frames = [8157184,8157696) | 20260506 02:16:13.452 | INFO | MemInfo: { | 20260506 02:16:13.452 | INFO | "gpu" : " 17 GB", | 20260506 02:16:13.452 | INFO | "huge" : "", | 20260506 02:16:13.452 | INFO | "huge_shared" : " 27 GB", | 20260506 02:16:13.452 | INFO | "malloc" : " 14 GB", | 20260506 02:16:13.452 | INFO | "pinned" : " 29 GB" | 20260506 02:16:13.452 | INFO | } | 20260506 02:16:13.452 | INFO | Perf report for the last 5.14022 seconds: | 20260506 02:16:13.452 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:16:13.452 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:13.452 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:13.452 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:13.452 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 02:16:13.452 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:13.452 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:13.452 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:16:13.452 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:13.452 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:13.452 | INFO | Uploaded 36GiB to GPU in 2.12451s (16.9451GiB/s) | 20260506 02:16:13.452 | INFO | Downloaded 34GiB to GPU in 2.0545s (16.5491GiB/s) | 20260506 02:16:18.600 | INFO | Analyzing chunk frames = [8157696,8158208) | 20260506 02:16:18.600 | INFO | MemInfo: { | 20260506 02:16:18.600 | INFO | "gpu" : " 17 GB", | 20260506 02:16:18.600 | INFO | "huge" : "", | 20260506 02:16:18.600 | INFO | "huge_shared" : " 27 GB", | 20260506 02:16:18.600 | INFO | "malloc" : " 14 GB", | 20260506 02:16:18.600 | INFO | "pinned" : " 29 GB" | 20260506 02:16:18.600 | INFO | } | 20260506 02:16:18.600 | INFO | Perf report for the last 5.14849 seconds: | 20260506 02:16:18.600 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:16:18.600 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:18.600 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:18.600 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:18.600 | INFO | ANALYSIS : 60.9%, 3.0, 95.3ms, 0 | 20260506 02:16:18.601 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:18.601 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:18.601 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:16:18.601 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:18.601 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:18.601 | INFO | Uploaded 36GiB to GPU in 2.0734s (17.3628GiB/s) | 20260506 02:16:18.601 | INFO | Downloaded 34GiB to GPU in 2.02792s (16.766GiB/s) | 20260506 02:16:23.745 | INFO | Analyzing chunk frames = [8158208,8158720) | 20260506 02:16:23.745 | INFO | MemInfo: { | 20260506 02:16:23.745 | INFO | "gpu" : " 17 GB", | 20260506 02:16:23.745 | INFO | "huge" : "", | 20260506 02:16:23.745 | INFO | "huge_shared" : " 27 GB", | 20260506 02:16:23.745 | INFO | "malloc" : " 14 GB", | 20260506 02:16:23.745 | INFO | "pinned" : " 29 GB" | 20260506 02:16:23.745 | INFO | } | 20260506 02:16:23.746 | INFO | Perf report for the last 5.14505 seconds: | 20260506 02:16:23.746 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:16:23.746 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:23.746 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:23.746 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:23.746 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260506 02:16:23.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:23.746 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:23.746 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 02:16:23.746 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:23.746 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:23.746 | INFO | Uploaded 36GiB to GPU in 2.10868s (17.0723GiB/s) | 20260506 02:16:23.746 | INFO | Downloaded 34GiB to GPU in 2.09493s (16.2297GiB/s) | 20260506 02:16:28.871 | INFO | Analyzing chunk frames = [8158720,8159232) | 20260506 02:16:28.871 | INFO | MemInfo: { | 20260506 02:16:28.871 | INFO | "gpu" : " 17 GB", | 20260506 02:16:28.871 | INFO | "huge" : "", | 20260506 02:16:28.871 | INFO | "huge_shared" : " 27 GB", | 20260506 02:16:28.871 | INFO | "malloc" : " 14 GB", | 20260506 02:16:28.871 | INFO | "pinned" : " 29 GB" | 20260506 02:16:28.871 | INFO | } | 20260506 02:16:28.871 | INFO | Perf report for the last 5.12535 seconds: | 20260506 02:16:28.871 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:16:28.871 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:28.871 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:28.871 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:28.871 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 02:16:28.871 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:28.871 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:28.871 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:16:28.871 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:28.871 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:28.871 | INFO | Uploaded 36GiB to GPU in 2.06645s (17.4212GiB/s) | 20260506 02:16:28.871 | INFO | Downloaded 34GiB to GPU in 2.03804s (16.6827GiB/s) | 20260506 02:16:28.872 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 02:16:28.872 | INFO | Total mem usage: 15.3 GiB | 20260506 02:16:28.872 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 02:16:28.872 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.17%) | 20260506 02:16:28.872 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 02:16:28.872 | INFO | High Water Breakdown: | 20260506 02:16:28.872 | INFO | 60.9% : 8.55 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 02:16:28.872 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 02:16:28.872 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 02:16:28.872 | INFO | ------------------------------------------ | 20260506 02:16:28.872 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:16:28.872 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 02:16:28.872 | INFO | Total mem usage: 26.5 GiB | 20260506 02:16:28.872 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 02:16:28.872 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 02:16:28.872 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 02:16:28.872 | INFO | High Water Breakdown: | 20260506 02:16:28.872 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:16:28.872 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:16:28.872 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:16:28.872 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:16:28.872 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:16:28.872 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:16:28.872 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 02:16:28.872 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 02:16:28.872 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:16:28.872 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 02:16:28.872 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 02:16:28.872 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 02:16:28.872 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 02:16:28.872 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 02:16:28.872 | INFO | ------------------------------------------ | 20260506 02:16:28.872 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:16:28.872 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 02:16:28.872 | INFO | Total mem usage: 25.6 GiB | 20260506 02:16:28.872 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 02:16:28.872 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 02:16:28.872 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 02:16:28.872 | INFO | High Water Breakdown: | 20260506 02:16:28.872 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 02:16:28.872 | INFO | ------------------------------------------ | 20260506 02:16:28.872 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:16:28.872 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 02:16:28.872 | INFO | Total mem usage: 15.7 GiB | 20260506 02:16:28.872 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 02:16:28.872 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 02:16:28.872 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 02:16:28.872 | INFO | High Water Breakdown: | 20260506 02:16:28.872 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 02:16:28.872 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 02:16:28.872 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 02:16:28.873 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 02:16:28.873 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 02:16:28.873 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 02:16:28.873 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 02:16:28.873 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 02:16:28.873 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 02:16:28.873 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:16:28.873 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 02:16:28.873 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:16:28.873 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 02:16:28.873 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 02:16:28.873 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 02:16:28.873 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 02:16:28.873 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:16:28.873 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 02:16:28.873 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:16:28.873 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:16:28.873 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:16:28.873 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 02:16:28.873 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 02:16:28.873 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:16:28.873 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 02:16:28.873 | INFO | ------------------------------------------ | 20260506 02:16:28.873 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:16:28.873 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 02:16:28.873 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:16:28.873 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:16:28.873 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.38e+01, 1.34e+01 / 7.65e-01 | 20260506 02:16:28.873 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.04e-01, 1.09e-02 / 3.02e-03 | 20260506 02:16:28.873 | INFO | PerfReport: 11.90%, Baseline -- 3.72e+00 / 4.52e+00, 5.88e-01 / 2.17e-01 | 20260506 02:16:28.873 | INFO | PerfReport: 3.30%, Histogram -- 1.03e+00 / 1.53e+00, 1.95e-01 / 8.81e-02 | 20260506 02:16:28.873 | INFO | PerfReport: 16.00%, DME -- 4.00e+01 / 4.41e+01, 3.98e+00 / 1.04e+00 | 20260506 02:16:28.873 | INFO | PerfReport: 57.40%, FrameLabeling -- 1.79e+01 / 1.91e+01, 1.49e+00 / 8.85e-01 | 20260506 02:16:28.873 | INFO | PerfReport: 7.04%, PulseAccumulating -- 2.20e+00 / 2.79e+00, 4.17e-01 / 1.62e-01 | 20260506 02:16:28.873 | INFO | PerfReport: 3.02%, Metrics -- 9.45e-01 / 1.24e+00, 2.53e-01 / 1.20e-01 | 20260506 02:16:28.873 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.26e-01 / 3.77e-01, 5.37e-02 / 2.07e-02 | 20260506 02:16:28.873 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 02:16:28.873 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:16:28.873 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:16:28.873 | INFO | PerfReport: Full Execution -- 9.77e+01 / -nan, 2.42e+01 / 0.00e+00 | 20260506 02:16:28.873 | INFO | PerfReport: 22.12%, Upload -- 2.16e+01 / 0.00e+00, 7.16e+00 / 0.00e+00 | 20260506 02:16:28.873 | INFO | PerfReport: 21.91%, Download -- 2.14e+01 / 0.00e+00, 7.58e+00 / 0.00e+00 | 20260506 02:16:28.873 | INFO | PerfReport: 32.00%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 02:16:28.873 | INFO | PerfReport: 2.02%, LockUpload -- 1.97e+00 / 0.00e+00, 7.81e+00 / 0.00e+00 | 20260506 02:16:28.873 | INFO | PerfReport: 1.35%, LockDownload -- 1.32e+00 / 0.00e+00, 3.06e+00 / 0.00e+00 | 20260506 02:16:28.873 | INFO | PerfReport: 20.59%, LockCompute -- 2.01e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 02:16:28.873 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.50e-02 / 0.00e+00 | 20260506 02:16:28.996 | INFO | Starting a round of Baz IO | 20260506 02:16:32.129 | INFO | Populating RTMetrics report | 20260506 02:16:32.130 | INFO | RTMetrics report populated | 20260506 02:16:32.130 | INFO | Opening tmp RTMetrics json file | 20260506 02:16:32.130 | INFO | Serializing RTMetrics to json | 20260506 02:16:32.132 | INFO | Serialization done, swapping tmp file now... | 20260506 02:16:32.132 | INFO | RTMetrics done | 20260506 02:16:32.204 | INFO | Finished a round of Baz IO: Wrote 1.798 GiB in 3.207 seconds | 20260506 02:16:32.204 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.28% / 0.00% / 78.72% | 20260506 02:16:32.204 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0721852% / 0% / 9.93411e-05% | 20260506 02:16:32.204 | INFO | Baz IO overall: | 20260506 02:16:32.204 | INFO | BytesWritten : 4577119781773 | 20260506 02:16:32.204 | INFO | NumEvents : 2149328460977 | 20260506 02:16:34.015 | INFO | Analyzing chunk frames = [8159232,8159744) | 20260506 02:16:34.015 | INFO | MemInfo: { | 20260506 02:16:34.015 | INFO | "gpu" : " 17 GB", | 20260506 02:16:34.015 | INFO | "huge" : "", | 20260506 02:16:34.015 | INFO | "huge_shared" : " 27 GB", | 20260506 02:16:34.015 | INFO | "malloc" : " 14 GB", | 20260506 02:16:34.015 | INFO | "pinned" : " 29 GB" | 20260506 02:16:34.015 | INFO | } | 20260506 02:16:34.015 | INFO | Perf report for the last 5.14375 seconds: | 20260506 02:16:34.015 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:16:34.015 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:34.015 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:34.015 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:34.015 | INFO | ANALYSIS : 62.3%, 3.0, 97.4ms, 0 | 20260506 02:16:34.015 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:34.015 | INFO | RT_METRICS : 43.0%, 1.0, 22.6ms, 0 | 20260506 02:16:34.015 | INFO | PRE_HQ : 59.5%, 1.7, 53.4ms, 0 | 20260506 02:16:34.015 | INFO | BAZWRITER : 61.6%, 3.5, 109.9ms, 0 | 20260506 02:16:34.015 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:16:34.015 | INFO | Uploaded 36GiB to GPU in 2.17844s (16.5256GiB/s) | 20260506 02:16:34.015 | INFO | Downloaded 38GiB to GPU in 2.23853s (16.9755GiB/s) | 20260506 02:16:39.150 | INFO | Analyzing chunk frames = [8159744,8160256) | 20260506 02:16:39.150 | INFO | MemInfo: { | 20260506 02:16:39.150 | INFO | "gpu" : " 17 GB", | 20260506 02:16:39.150 | INFO | "huge" : "", | 20260506 02:16:39.150 | INFO | "huge_shared" : " 27 GB", | 20260506 02:16:39.150 | INFO | "malloc" : " 14 GB", | 20260506 02:16:39.150 | INFO | "pinned" : " 29 GB" | 20260506 02:16:39.150 | INFO | } | 20260506 02:16:39.150 | INFO | Perf report for the last 5.13536 seconds: | 20260506 02:16:39.150 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:16:39.150 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:39.150 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:39.150 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:39.150 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 02:16:39.150 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:39.150 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:39.150 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 02:16:39.150 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:39.150 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:39.150 | INFO | Uploaded 36GiB to GPU in 1.95239s (18.439GiB/s) | 20260506 02:16:39.150 | INFO | Downloaded 34GiB to GPU in 1.98161s (17.1578GiB/s) | 20260506 02:16:44.288 | INFO | Analyzing chunk frames = [8160256,8160768) | 20260506 02:16:44.288 | INFO | MemInfo: { | 20260506 02:16:44.288 | INFO | "gpu" : " 17 GB", | 20260506 02:16:44.288 | INFO | "huge" : "", | 20260506 02:16:44.288 | INFO | "huge_shared" : " 27 GB", | 20260506 02:16:44.288 | INFO | "malloc" : " 14 GB", | 20260506 02:16:44.289 | INFO | "pinned" : " 29 GB" | 20260506 02:16:44.289 | INFO | } | 20260506 02:16:44.289 | INFO | Perf report for the last 5.13862 seconds: | 20260506 02:16:44.289 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:16:44.289 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:44.289 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:44.289 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:44.289 | INFO | ANALYSIS : 61.2%, 3.0, 95.1ms, 0 | 20260506 02:16:44.289 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:44.289 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:44.289 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:16:44.289 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:44.289 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:44.289 | INFO | Uploaded 36GiB to GPU in 2.08544s (17.2626GiB/s) | 20260506 02:16:44.289 | INFO | Downloaded 34GiB to GPU in 2.03111s (16.7396GiB/s) | 20260506 02:16:49.428 | INFO | Analyzing chunk frames = [8160768,8161280) | 20260506 02:16:49.428 | INFO | MemInfo: { | 20260506 02:16:49.428 | INFO | "gpu" : " 17 GB", | 20260506 02:16:49.428 | INFO | "huge" : "", | 20260506 02:16:49.428 | INFO | "huge_shared" : " 27 GB", | 20260506 02:16:49.428 | INFO | "malloc" : " 14 GB", | 20260506 02:16:49.428 | INFO | "pinned" : " 29 GB" | 20260506 02:16:49.428 | INFO | } | 20260506 02:16:49.428 | INFO | Perf report for the last 5.13962 seconds: | 20260506 02:16:49.428 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:16:49.428 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:49.428 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:49.428 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:49.428 | INFO | ANALYSIS : 61.6%, 3.0, 95.6ms, 0 | 20260506 02:16:49.428 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:49.428 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:49.428 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 02:16:49.428 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:49.428 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:49.429 | INFO | Uploaded 36GiB to GPU in 2.13523s (16.86GiB/s) | 20260506 02:16:49.429 | INFO | Downloaded 34GiB to GPU in 2.01587s (16.8662GiB/s) | 20260506 02:16:54.567 | INFO | Analyzing chunk frames = [8161280,8161792) | 20260506 02:16:54.567 | INFO | MemInfo: { | 20260506 02:16:54.567 | INFO | "gpu" : " 17 GB", | 20260506 02:16:54.567 | INFO | "huge" : "", | 20260506 02:16:54.567 | INFO | "huge_shared" : " 27 GB", | 20260506 02:16:54.567 | INFO | "malloc" : " 14 GB", | 20260506 02:16:54.567 | INFO | "pinned" : " 29 GB" | 20260506 02:16:54.567 | INFO | } | 20260506 02:16:54.567 | INFO | Perf report for the last 5.13889 seconds: | 20260506 02:16:54.567 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:16:54.567 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:54.567 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:54.567 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:54.567 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 02:16:54.567 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:54.567 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:54.567 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:16:54.567 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:54.567 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:54.567 | INFO | Uploaded 36GiB to GPU in 2.08388s (17.2755GiB/s) | 20260506 02:16:54.567 | INFO | Downloaded 34GiB to GPU in 2.04545s (16.6223GiB/s) | 20260506 02:16:59.713 | INFO | Analyzing chunk frames = [8161792,8162304) | 20260506 02:16:59.713 | INFO | MemInfo: { | 20260506 02:16:59.713 | INFO | "gpu" : " 17 GB", | 20260506 02:16:59.713 | INFO | "huge" : "", | 20260506 02:16:59.713 | INFO | "huge_shared" : " 27 GB", | 20260506 02:16:59.713 | INFO | "malloc" : " 14 GB", | 20260506 02:16:59.713 | INFO | "pinned" : " 29 GB" | 20260506 02:16:59.713 | INFO | } | 20260506 02:16:59.713 | INFO | Perf report for the last 5.1459 seconds: | 20260506 02:16:59.713 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:16:59.713 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:59.713 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:59.713 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:59.713 | INFO | ANALYSIS : 61.0%, 3.0, 95.4ms, 0 | 20260506 02:16:59.713 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:59.713 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:59.713 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 02:16:59.713 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:59.713 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:16:59.713 | INFO | Uploaded 36GiB to GPU in 2.07547s (17.3454GiB/s) | 20260506 02:16:59.713 | INFO | Downloaded 34GiB to GPU in 2.03149s (16.7365GiB/s) | 20260506 02:17:04.853 | INFO | Analyzing chunk frames = [8162304,8162816) | 20260506 02:17:04.853 | INFO | MemInfo: { | 20260506 02:17:04.853 | INFO | "gpu" : " 17 GB", | 20260506 02:17:04.853 | INFO | "huge" : "", | 20260506 02:17:04.853 | INFO | "huge_shared" : " 27 GB", | 20260506 02:17:04.853 | INFO | "malloc" : " 14 GB", | 20260506 02:17:04.853 | INFO | "pinned" : " 29 GB" | 20260506 02:17:04.853 | INFO | } | 20260506 02:17:04.853 | INFO | Perf report for the last 5.1399 seconds: | 20260506 02:17:04.853 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:17:04.853 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:04.853 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:04.853 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:04.853 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260506 02:17:04.853 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:04.853 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:04.853 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 02:17:04.853 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:04.853 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:04.853 | INFO | Uploaded 36GiB to GPU in 2.01208s (17.8919GiB/s) | 20260506 02:17:04.853 | INFO | Downloaded 34GiB to GPU in 2.04721s (16.6079GiB/s) | 20260506 02:17:09.983 | INFO | Analyzing chunk frames = [8162816,8163328) | 20260506 02:17:09.983 | INFO | MemInfo: { | 20260506 02:17:09.983 | INFO | "gpu" : " 17 GB", | 20260506 02:17:09.983 | INFO | "huge" : "", | 20260506 02:17:09.983 | INFO | "huge_shared" : " 27 GB", | 20260506 02:17:09.983 | INFO | "malloc" : " 14 GB", | 20260506 02:17:09.983 | INFO | "pinned" : " 29 GB" | 20260506 02:17:09.983 | INFO | } | 20260506 02:17:09.983 | INFO | Perf report for the last 5.12978 seconds: | 20260506 02:17:09.983 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:17:09.983 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:09.983 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:09.983 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:09.983 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 02:17:09.983 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:09.983 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:09.983 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260506 02:17:09.983 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:09.983 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:09.983 | INFO | Uploaded 36GiB to GPU in 2.04899s (17.5697GiB/s) | 20260506 02:17:09.983 | INFO | Downloaded 34GiB to GPU in 2.02385s (16.7996GiB/s) | 20260506 02:17:10.100 | INFO | Starting a round of Baz IO | 20260506 02:17:13.245 | INFO | Populating RTMetrics report | 20260506 02:17:13.245 | INFO | RTMetrics report populated | 20260506 02:17:13.245 | INFO | Opening tmp RTMetrics json file | 20260506 02:17:13.246 | INFO | Serializing RTMetrics to json | 20260506 02:17:13.247 | INFO | Serialization done, swapping tmp file now... | 20260506 02:17:13.248 | INFO | RTMetrics done | 20260506 02:17:13.319 | INFO | Finished a round of Baz IO: Wrote 1.798 GiB in 3.219 seconds | 20260506 02:17:13.319 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.28% / 0.00% / 78.72% | 20260506 02:17:13.320 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0718911% / 0% / 9.13938e-05% | 20260506 02:17:13.320 | INFO | Baz IO overall: | 20260506 02:17:13.320 | INFO | BytesWritten : 4579050387078 | 20260506 02:17:13.320 | INFO | NumEvents : 2150206835285 | 20260506 02:17:15.119 | INFO | Analyzing chunk frames = [8163328,8163840) | 20260506 02:17:15.119 | INFO | MemInfo: { | 20260506 02:17:15.119 | INFO | "gpu" : " 17 GB", | 20260506 02:17:15.119 | INFO | "huge" : "", | 20260506 02:17:15.119 | INFO | "huge_shared" : " 27 GB", | 20260506 02:17:15.119 | INFO | "malloc" : " 14 GB", | 20260506 02:17:15.119 | INFO | "pinned" : " 29 GB" | 20260506 02:17:15.119 | INFO | } | 20260506 02:17:15.119 | INFO | Perf report for the last 5.13653 seconds: | 20260506 02:17:15.119 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:17:15.119 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:15.119 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:15.119 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:15.119 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260506 02:17:15.119 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:15.119 | INFO | RT_METRICS : 43.0%, 1.0, 22.6ms, 0 | 20260506 02:17:15.119 | INFO | PRE_HQ : 59.7%, 1.7, 53.8ms, 0 | 20260506 02:17:15.120 | INFO | BAZWRITER : 61.8%, 3.5, 110.4ms, 0 | 20260506 02:17:15.120 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 02:17:15.120 | INFO | Uploaded 36GiB to GPU in 2.17576s (16.546GiB/s) | 20260506 02:17:15.120 | INFO | Downloaded 38GiB to GPU in 2.23952s (16.9679GiB/s) | 20260506 02:17:20.263 | INFO | Analyzing chunk frames = [8163840,8164352) | 20260506 02:17:20.263 | INFO | MemInfo: { | 20260506 02:17:20.263 | INFO | "gpu" : " 17 GB", | 20260506 02:17:20.263 | INFO | "huge" : "", | 20260506 02:17:20.263 | INFO | "huge_shared" : " 27 GB", | 20260506 02:17:20.263 | INFO | "malloc" : " 14 GB", | 20260506 02:17:20.263 | INFO | "pinned" : " 29 GB" | 20260506 02:17:20.263 | INFO | } | 20260506 02:17:20.263 | INFO | Perf report for the last 5.14403 seconds: | 20260506 02:17:20.263 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:17:20.263 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:20.263 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:20.263 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:20.263 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260506 02:17:20.263 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:20.264 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:20.264 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 02:17:20.264 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:20.264 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:20.264 | INFO | Uploaded 36GiB to GPU in 2.11209s (17.0447GiB/s) | 20260506 02:17:20.264 | INFO | Downloaded 34GiB to GPU in 2.04346s (16.6384GiB/s) | 20260506 02:17:25.410 | INFO | Analyzing chunk frames = [8164352,8164864) | 20260506 02:17:25.410 | INFO | MemInfo: { | 20260506 02:17:25.410 | INFO | "gpu" : " 17 GB", | 20260506 02:17:25.410 | INFO | "huge" : "", | 20260506 02:17:25.410 | INFO | "huge_shared" : " 27 GB", | 20260506 02:17:25.410 | INFO | "malloc" : " 14 GB", | 20260506 02:17:25.410 | INFO | "pinned" : " 29 GB" | 20260506 02:17:25.410 | INFO | } | 20260506 02:17:25.410 | INFO | Perf report for the last 5.14654 seconds: | 20260506 02:17:25.410 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:17:25.410 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:25.410 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:25.410 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:25.410 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 02:17:25.410 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:25.410 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:25.410 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 02:17:25.410 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:25.410 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:25.410 | INFO | Uploaded 36GiB to GPU in 2.09462s (17.1869GiB/s) | 20260506 02:17:25.410 | INFO | Downloaded 34GiB to GPU in 2.02803s (16.7651GiB/s) | 20260506 02:17:30.550 | INFO | Analyzing chunk frames = [8164864,8165376) | 20260506 02:17:30.550 | INFO | MemInfo: { | 20260506 02:17:30.550 | INFO | "gpu" : " 17 GB", | 20260506 02:17:30.550 | INFO | "huge" : "", | 20260506 02:17:30.550 | INFO | "huge_shared" : " 27 GB", | 20260506 02:17:30.550 | INFO | "malloc" : " 14 GB", | 20260506 02:17:30.550 | INFO | "pinned" : " 29 GB" | 20260506 02:17:30.550 | INFO | } | 20260506 02:17:30.551 | INFO | Perf report for the last 5.14061 seconds: | 20260506 02:17:30.551 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:17:30.551 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:30.551 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:30.551 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:30.551 | INFO | ANALYSIS : 61.0%, 3.0, 94.7ms, 0 | 20260506 02:17:30.551 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:30.551 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:30.551 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 02:17:30.551 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:30.551 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:30.551 | INFO | Uploaded 36GiB to GPU in 2.08779s (17.2431GiB/s) | 20260506 02:17:30.551 | INFO | Downloaded 34GiB to GPU in 2.00906s (16.9234GiB/s) | 20260506 02:17:35.692 | INFO | Analyzing chunk frames = [8165376,8165888) | 20260506 02:17:35.692 | INFO | MemInfo: { | 20260506 02:17:35.692 | INFO | "gpu" : " 17 GB", | 20260506 02:17:35.692 | INFO | "huge" : "", | 20260506 02:17:35.692 | INFO | "huge_shared" : " 27 GB", | 20260506 02:17:35.692 | INFO | "malloc" : " 14 GB", | 20260506 02:17:35.692 | INFO | "pinned" : " 29 GB" | 20260506 02:17:35.693 | INFO | } | 20260506 02:17:35.693 | INFO | Perf report for the last 5.14219 seconds: | 20260506 02:17:35.693 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:17:35.693 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:35.693 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:35.693 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:35.693 | INFO | ANALYSIS : 60.7%, 3.0, 94.9ms, 0 | 20260506 02:17:35.693 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:35.693 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:35.693 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 02:17:35.693 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:35.693 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:35.693 | INFO | Uploaded 36GiB to GPU in 2.08167s (17.2938GiB/s) | 20260506 02:17:35.693 | INFO | Downloaded 34GiB to GPU in 2.03653s (16.6951GiB/s) | 20260506 02:17:40.825 | INFO | Analyzing chunk frames = [8165888,8166400) | 20260506 02:17:40.825 | INFO | MemInfo: { | 20260506 02:17:40.825 | INFO | "gpu" : " 17 GB", | 20260506 02:17:40.825 | INFO | "huge" : "", | 20260506 02:17:40.825 | INFO | "huge_shared" : " 27 GB", | 20260506 02:17:40.825 | INFO | "malloc" : " 14 GB", | 20260506 02:17:40.826 | INFO | "pinned" : " 29 GB" | 20260506 02:17:40.826 | INFO | } | 20260506 02:17:40.826 | INFO | Perf report for the last 5.13282 seconds: | 20260506 02:17:40.826 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:17:40.826 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:40.826 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:40.826 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:40.826 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 02:17:40.826 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:40.826 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:40.826 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:17:40.826 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:40.826 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:40.826 | INFO | Uploaded 36GiB to GPU in 2.08597s (17.2582GiB/s) | 20260506 02:17:40.826 | INFO | Downloaded 34GiB to GPU in 2.0386s (16.6781GiB/s) | 20260506 02:17:45.965 | INFO | Analyzing chunk frames = [8166400,8166912) | 20260506 02:17:45.965 | INFO | MemInfo: { | 20260506 02:17:45.965 | INFO | "gpu" : " 17 GB", | 20260506 02:17:45.965 | INFO | "huge" : "", | 20260506 02:17:45.965 | INFO | "huge_shared" : " 27 GB", | 20260506 02:17:45.965 | INFO | "malloc" : " 14 GB", | 20260506 02:17:45.965 | INFO | "pinned" : " 29 GB" | 20260506 02:17:45.965 | INFO | } | 20260506 02:17:45.965 | INFO | Perf report for the last 5.13948 seconds: | 20260506 02:17:45.965 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:17:45.965 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:45.965 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:45.965 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:45.965 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260506 02:17:45.965 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:45.965 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:45.965 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 02:17:45.965 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:45.965 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:45.965 | INFO | Uploaded 36GiB to GPU in 2.10395s (17.1107GiB/s) | 20260506 02:17:45.965 | INFO | Downloaded 34GiB to GPU in 2.09187s (16.2534GiB/s) | 20260506 02:17:51.104 | INFO | Analyzing chunk frames = [8166912,8167424) | 20260506 02:17:51.104 | INFO | MemInfo: { | 20260506 02:17:51.104 | INFO | "gpu" : " 17 GB", | 20260506 02:17:51.104 | INFO | "huge" : "", | 20260506 02:17:51.104 | INFO | "huge_shared" : " 27 GB", | 20260506 02:17:51.104 | INFO | "malloc" : " 14 GB", | 20260506 02:17:51.104 | INFO | "pinned" : " 29 GB" | 20260506 02:17:51.104 | INFO | } | 20260506 02:17:51.104 | INFO | Perf report for the last 5.13912 seconds: | 20260506 02:17:51.104 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:17:51.104 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:51.104 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:51.104 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:51.104 | INFO | ANALYSIS : 61.3%, 3.0, 95.8ms, 0 | 20260506 02:17:51.104 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:51.104 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:51.104 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 02:17:51.104 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:51.104 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:51.104 | INFO | Uploaded 36GiB to GPU in 2.05221s (17.5421GiB/s) | 20260506 02:17:51.105 | INFO | Downloaded 34GiB to GPU in 2.02246s (16.8112GiB/s) | 20260506 02:17:51.105 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 02:17:51.105 | INFO | Total mem usage: 15.3 GiB | 20260506 02:17:51.105 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 02:17:51.105 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.19%) | 20260506 02:17:51.105 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 02:17:51.105 | INFO | High Water Breakdown: | 20260506 02:17:51.105 | INFO | 60.9% : 8.54 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 02:17:51.105 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 02:17:51.105 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 02:17:51.105 | INFO | ------------------------------------------ | 20260506 02:17:51.105 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:17:51.105 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 02:17:51.105 | INFO | Total mem usage: 26.5 GiB | 20260506 02:17:51.105 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 02:17:51.105 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 02:17:51.105 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 02:17:51.105 | INFO | High Water Breakdown: | 20260506 02:17:51.105 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:17:51.105 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:17:51.105 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:17:51.105 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:17:51.105 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:17:51.105 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:17:51.105 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 02:17:51.105 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 02:17:51.105 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:17:51.105 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 02:17:51.105 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 02:17:51.105 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 02:17:51.105 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 02:17:51.105 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 02:17:51.105 | INFO | ------------------------------------------ | 20260506 02:17:51.105 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:17:51.105 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 02:17:51.105 | INFO | Total mem usage: 25.6 GiB | 20260506 02:17:51.105 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 02:17:51.105 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 02:17:51.105 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 02:17:51.105 | INFO | High Water Breakdown: | 20260506 02:17:51.105 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 02:17:51.105 | INFO | ------------------------------------------ | 20260506 02:17:51.105 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:17:51.105 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 02:17:51.105 | INFO | Total mem usage: 15.7 GiB | 20260506 02:17:51.105 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 02:17:51.105 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 02:17:51.105 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 02:17:51.105 | INFO | High Water Breakdown: | 20260506 02:17:51.105 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 02:17:51.105 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 02:17:51.105 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 02:17:51.105 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 02:17:51.105 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 02:17:51.105 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 02:17:51.105 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 02:17:51.105 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 02:17:51.106 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 02:17:51.106 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:17:51.106 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 02:17:51.106 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:17:51.106 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 02:17:51.106 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 02:17:51.106 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 02:17:51.106 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 02:17:51.106 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:17:51.106 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 02:17:51.106 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:17:51.106 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:17:51.106 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:17:51.106 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 02:17:51.106 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 02:17:51.106 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:17:51.106 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 02:17:51.106 | INFO | ------------------------------------------ | 20260506 02:17:51.106 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:17:51.106 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 02:17:51.106 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:17:51.106 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:17:51.106 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.40e+01, 1.35e+01 / 6.86e-01 | 20260506 02:17:51.106 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.07e-01, 1.06e-02 / 1.04e-02 | 20260506 02:17:51.106 | INFO | PerfReport: 11.87%, Baseline -- 3.71e+00 / 4.53e+00, 5.75e-01 / 1.59e-01 | 20260506 02:17:51.106 | INFO | PerfReport: 3.29%, Histogram -- 1.03e+00 / 1.53e+00, 1.93e-01 / 7.64e-02 | 20260506 02:17:51.106 | INFO | PerfReport: 16.05%, DME -- 4.01e+01 / 4.41e+01, 3.87e+00 / 9.41e-01 | 20260506 02:17:51.106 | INFO | PerfReport: 57.35%, FrameLabeling -- 1.79e+01 / 1.94e+01, 1.41e+00 / 9.20e-01 | 20260506 02:17:51.106 | INFO | PerfReport: 7.07%, PulseAccumulating -- 2.21e+00 / 2.73e+00, 4.21e-01 / 2.08e-01 | 20260506 02:17:51.106 | INFO | PerfReport: 3.03%, Metrics -- 9.46e-01 / 1.21e+00, 2.53e-01 / 8.36e-02 | 20260506 02:17:51.106 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.26e-01 / 3.74e-01, 5.41e-02 / 2.82e-02 | 20260506 02:17:51.106 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 02:17:51.106 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:17:51.106 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:17:51.106 | INFO | PerfReport: Full Execution -- 9.78e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260506 02:17:51.106 | INFO | PerfReport: 22.22%, Upload -- 2.17e+01 / 0.00e+00, 7.33e+00 / 0.00e+00 | 20260506 02:17:51.106 | INFO | PerfReport: 21.93%, Download -- 2.14e+01 / 0.00e+00, 7.57e+00 / 0.00e+00 | 20260506 02:17:51.106 | INFO | PerfReport: 31.94%, Compute -- 3.12e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 02:17:51.106 | INFO | PerfReport: 2.03%, LockUpload -- 1.98e+00 / 0.00e+00, 7.77e+00 / 0.00e+00 | 20260506 02:17:51.106 | INFO | PerfReport: 1.36%, LockDownload -- 1.33e+00 / 0.00e+00, 2.95e+00 / 0.00e+00 | 20260506 02:17:51.106 | INFO | PerfReport: 20.51%, LockCompute -- 2.01e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260506 02:17:51.106 | INFO | PerfReport: 0.01%, Misc -- 1.22e-02 / 0.00e+00, 1.20e-02 / 0.00e+00 | 20260506 02:17:51.230 | INFO | Starting a round of Baz IO | 20260506 02:17:54.367 | INFO | Populating RTMetrics report | 20260506 02:17:54.367 | INFO | RTMetrics report populated | 20260506 02:17:54.367 | INFO | Opening tmp RTMetrics json file | 20260506 02:17:54.368 | INFO | Serializing RTMetrics to json | 20260506 02:17:54.369 | INFO | Serialization done, swapping tmp file now... | 20260506 02:17:54.369 | INFO | RTMetrics done | 20260506 02:17:54.441 | INFO | Finished a round of Baz IO: Wrote 1.797 GiB in 3.211 seconds | 20260506 02:17:54.441 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.27% / 0.00% / 78.72% | 20260506 02:17:54.441 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0715534% / 3.97364e-06% / 7.94729e-05% | 20260506 02:17:54.441 | INFO | Baz IO overall: | 20260506 02:17:54.441 | INFO | BytesWritten : 4580980250676 | 20260506 02:17:54.441 | INFO | NumEvents : 2151084692824 | 20260506 02:17:56.239 | INFO | Analyzing chunk frames = [8167424,8167936) | 20260506 02:17:56.239 | INFO | MemInfo: { | 20260506 02:17:56.239 | INFO | "gpu" : " 17 GB", | 20260506 02:17:56.239 | INFO | "huge" : "", | 20260506 02:17:56.239 | INFO | "huge_shared" : " 27 GB", | 20260506 02:17:56.239 | INFO | "malloc" : " 14 GB", | 20260506 02:17:56.239 | INFO | "pinned" : " 29 GB" | 20260506 02:17:56.239 | INFO | } | 20260506 02:17:56.240 | INFO | Perf report for the last 5.13532 seconds: | 20260506 02:17:56.240 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:17:56.240 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:56.240 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:56.240 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:56.240 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260506 02:17:56.240 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:17:56.240 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260506 02:17:56.240 | INFO | PRE_HQ : 59.6%, 1.7, 53.4ms, 0 | 20260506 02:17:56.240 | INFO | BAZWRITER : 61.8%, 3.5, 109.7ms, 0 | 20260506 02:17:56.240 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:17:56.240 | INFO | Uploaded 36GiB to GPU in 2.18576s (16.4703GiB/s) | 20260506 02:17:56.240 | INFO | Downloaded 38GiB to GPU in 2.24329s (16.9394GiB/s) | 20260506 02:18:01.380 | INFO | Analyzing chunk frames = [8167936,8168448) | 20260506 02:18:01.380 | INFO | MemInfo: { | 20260506 02:18:01.380 | INFO | "gpu" : " 17 GB", | 20260506 02:18:01.380 | INFO | "huge" : "", | 20260506 02:18:01.380 | INFO | "huge_shared" : " 27 GB", | 20260506 02:18:01.380 | INFO | "malloc" : " 14 GB", | 20260506 02:18:01.380 | INFO | "pinned" : " 29 GB" | 20260506 02:18:01.380 | INFO | } | 20260506 02:18:01.380 | INFO | Perf report for the last 5.14052 seconds: | 20260506 02:18:01.380 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:18:01.380 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:01.380 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:01.380 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:01.380 | INFO | ANALYSIS : 60.8%, 3.0, 94.9ms, 0 | 20260506 02:18:01.380 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:01.380 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:01.380 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 02:18:01.380 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:01.380 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:01.380 | INFO | Uploaded 36GiB to GPU in 1.97278s (18.2484GiB/s) | 20260506 02:18:01.380 | INFO | Downloaded 34GiB to GPU in 1.98786s (17.1038GiB/s) | 20260506 02:18:06.523 | INFO | Analyzing chunk frames = [8168448,8168960) | 20260506 02:18:06.523 | INFO | MemInfo: { | 20260506 02:18:06.523 | INFO | "gpu" : " 17 GB", | 20260506 02:18:06.523 | INFO | "huge" : "", | 20260506 02:18:06.523 | INFO | "huge_shared" : " 27 GB", | 20260506 02:18:06.523 | INFO | "malloc" : " 14 GB", | 20260506 02:18:06.523 | INFO | "pinned" : " 29 GB" | 20260506 02:18:06.523 | INFO | } | 20260506 02:18:06.523 | INFO | Perf report for the last 5.14292 seconds: | 20260506 02:18:06.523 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:18:06.523 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:06.523 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:06.523 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:06.523 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 02:18:06.523 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:06.523 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:06.523 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:18:06.523 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:06.523 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:06.523 | INFO | Uploaded 36GiB to GPU in 2.07596s (17.3413GiB/s) | 20260506 02:18:06.523 | INFO | Downloaded 34GiB to GPU in 2.01632s (16.8624GiB/s) | 20260506 02:18:11.664 | INFO | Analyzing chunk frames = [8168960,8169472) | 20260506 02:18:11.664 | INFO | MemInfo: { | 20260506 02:18:11.664 | INFO | "gpu" : " 17 GB", | 20260506 02:18:11.664 | INFO | "huge" : "", | 20260506 02:18:11.664 | INFO | "huge_shared" : " 27 GB", | 20260506 02:18:11.664 | INFO | "malloc" : " 14 GB", | 20260506 02:18:11.664 | INFO | "pinned" : " 29 GB" | 20260506 02:18:11.664 | INFO | } | 20260506 02:18:11.664 | INFO | Perf report for the last 5.14085 seconds: | 20260506 02:18:11.664 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:18:11.664 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:11.664 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:11.664 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:11.664 | INFO | ANALYSIS : 60.9%, 3.0, 94.6ms, 0 | 20260506 02:18:11.664 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:11.664 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:11.664 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:18:11.664 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:11.664 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:11.664 | INFO | Uploaded 36GiB to GPU in 2.06355s (17.4457GiB/s) | 20260506 02:18:11.664 | INFO | Downloaded 34GiB to GPU in 2.00443s (16.9624GiB/s) | 20260506 02:18:16.796 | INFO | Analyzing chunk frames = [8169472,8169984) | 20260506 02:18:16.796 | INFO | MemInfo: { | 20260506 02:18:16.796 | INFO | "gpu" : " 17 GB", | 20260506 02:18:16.796 | INFO | "huge" : "", | 20260506 02:18:16.796 | INFO | "huge_shared" : " 27 GB", | 20260506 02:18:16.796 | INFO | "malloc" : " 14 GB", | 20260506 02:18:16.796 | INFO | "pinned" : " 29 GB" | 20260506 02:18:16.796 | INFO | } | 20260506 02:18:16.797 | INFO | Perf report for the last 5.13274 seconds: | 20260506 02:18:16.797 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:18:16.797 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:16.797 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:16.797 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:16.797 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260506 02:18:16.797 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:16.797 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:16.797 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 02:18:16.797 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:16.797 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:16.797 | INFO | Uploaded 36GiB to GPU in 2.10244s (17.123GiB/s) | 20260506 02:18:16.797 | INFO | Downloaded 34GiB to GPU in 2.03096s (16.7409GiB/s) | 20260506 02:18:21.932 | INFO | Analyzing chunk frames = [8169984,8170496) | 20260506 02:18:21.932 | INFO | MemInfo: { | 20260506 02:18:21.932 | INFO | "gpu" : " 17 GB", | 20260506 02:18:21.932 | INFO | "huge" : "", | 20260506 02:18:21.932 | INFO | "huge_shared" : " 27 GB", | 20260506 02:18:21.932 | INFO | "malloc" : " 14 GB", | 20260506 02:18:21.932 | INFO | "pinned" : " 29 GB" | 20260506 02:18:21.932 | INFO | } | 20260506 02:18:21.932 | INFO | Perf report for the last 5.13549 seconds: | 20260506 02:18:21.932 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:18:21.932 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:21.932 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:21.932 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:21.932 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 02:18:21.932 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:21.932 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:21.932 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 02:18:21.932 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:21.932 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:21.932 | INFO | Uploaded 36GiB to GPU in 2.06999s (17.3914GiB/s) | 20260506 02:18:21.932 | INFO | Downloaded 34GiB to GPU in 2.04112s (16.6575GiB/s) | 20260506 02:18:27.074 | INFO | Analyzing chunk frames = [8170496,8171008) | 20260506 02:18:27.074 | INFO | MemInfo: { | 20260506 02:18:27.074 | INFO | "gpu" : " 17 GB", | 20260506 02:18:27.074 | INFO | "huge" : "", | 20260506 02:18:27.074 | INFO | "huge_shared" : " 27 GB", | 20260506 02:18:27.074 | INFO | "malloc" : " 14 GB", | 20260506 02:18:27.074 | INFO | "pinned" : " 29 GB" | 20260506 02:18:27.074 | INFO | } | 20260506 02:18:27.074 | INFO | Perf report for the last 5.14224 seconds: | 20260506 02:18:27.074 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:18:27.074 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:27.074 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:27.074 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:27.074 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260506 02:18:27.074 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:27.074 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:27.074 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260506 02:18:27.075 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:27.075 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:27.075 | INFO | Uploaded 36GiB to GPU in 2.06799s (17.4082GiB/s) | 20260506 02:18:27.075 | INFO | Downloaded 34GiB to GPU in 2.0659s (16.4577GiB/s) | 20260506 02:18:32.215 | INFO | Analyzing chunk frames = [8171008,8171520) | 20260506 02:18:32.215 | INFO | MemInfo: { | 20260506 02:18:32.215 | INFO | "gpu" : " 17 GB", | 20260506 02:18:32.215 | INFO | "huge" : "", | 20260506 02:18:32.215 | INFO | "huge_shared" : " 27 GB", | 20260506 02:18:32.215 | INFO | "malloc" : " 14 GB", | 20260506 02:18:32.216 | INFO | "pinned" : " 29 GB" | 20260506 02:18:32.216 | INFO | } | 20260506 02:18:32.216 | INFO | Perf report for the last 5.14137 seconds: | 20260506 02:18:32.216 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:18:32.216 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:32.216 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:32.216 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:32.216 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260506 02:18:32.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:32.216 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:32.216 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 02:18:32.216 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:32.216 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:32.216 | INFO | Uploaded 36GiB to GPU in 2.12948s (16.9055GiB/s) | 20260506 02:18:32.216 | INFO | Downloaded 34GiB to GPU in 2.02236s (16.8121GiB/s) | 20260506 02:18:32.342 | INFO | Starting a round of Baz IO | 20260506 02:18:35.471 | INFO | Populating RTMetrics report | 20260506 02:18:35.472 | INFO | RTMetrics report populated | 20260506 02:18:35.472 | INFO | Opening tmp RTMetrics json file | 20260506 02:18:35.472 | INFO | Serializing RTMetrics to json | 20260506 02:18:35.473 | INFO | Serialization done, swapping tmp file now... | 20260506 02:18:35.474 | INFO | RTMetrics done | 20260506 02:18:35.549 | INFO | Finished a round of Baz IO: Wrote 1.798 GiB in 3.207 seconds | 20260506 02:18:35.549 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.27% / 0.00% / 78.73% | 20260506 02:18:35.549 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0717322% / 0% / 0.000111262% | 20260506 02:18:35.549 | INFO | Baz IO overall: | 20260506 02:18:35.549 | INFO | BytesWritten : 4582911305952 | 20260506 02:18:35.549 | INFO | NumEvents : 2151963168584 | 20260506 02:18:37.359 | INFO | Analyzing chunk frames = [8171520,8172032) | 20260506 02:18:37.359 | INFO | MemInfo: { | 20260506 02:18:37.359 | INFO | "gpu" : " 17 GB", | 20260506 02:18:37.359 | INFO | "huge" : "", | 20260506 02:18:37.359 | INFO | "huge_shared" : " 27 GB", | 20260506 02:18:37.359 | INFO | "malloc" : " 14 GB", | 20260506 02:18:37.359 | INFO | "pinned" : " 29 GB" | 20260506 02:18:37.359 | INFO | } | 20260506 02:18:37.359 | INFO | Perf report for the last 5.1433 seconds: | 20260506 02:18:37.359 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:18:37.359 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:37.359 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:37.359 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:37.359 | INFO | ANALYSIS : 62.3%, 3.0, 97.4ms, 0 | 20260506 02:18:37.359 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:37.359 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260506 02:18:37.359 | INFO | PRE_HQ : 59.5%, 1.7, 53.4ms, 0 | 20260506 02:18:37.359 | INFO | BAZWRITER : 61.7%, 3.4, 108.4ms, 0 | 20260506 02:18:37.359 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:18:37.359 | INFO | Uploaded 36GiB to GPU in 2.18051s (16.5099GiB/s) | 20260506 02:18:37.359 | INFO | Downloaded 38GiB to GPU in 2.24061s (16.9596GiB/s) | 20260506 02:18:42.497 | INFO | Analyzing chunk frames = [8172032,8172544) | 20260506 02:18:42.497 | INFO | MemInfo: { | 20260506 02:18:42.497 | INFO | "gpu" : " 17 GB", | 20260506 02:18:42.497 | INFO | "huge" : "", | 20260506 02:18:42.497 | INFO | "huge_shared" : " 27 GB", | 20260506 02:18:42.497 | INFO | "malloc" : " 14 GB", | 20260506 02:18:42.498 | INFO | "pinned" : " 29 GB" | 20260506 02:18:42.498 | INFO | } | 20260506 02:18:42.498 | INFO | Perf report for the last 5.13864 seconds: | 20260506 02:18:42.498 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:18:42.498 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:42.498 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:42.498 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:42.498 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 02:18:42.498 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:42.498 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:42.498 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 02:18:42.498 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:42.498 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:42.498 | INFO | Uploaded 36GiB to GPU in 1.92785s (18.6736GiB/s) | 20260506 02:18:42.498 | INFO | Downloaded 34GiB to GPU in 1.97586s (17.2077GiB/s) | 20260506 02:18:47.636 | INFO | Analyzing chunk frames = [8172544,8173056) | 20260506 02:18:47.636 | INFO | MemInfo: { | 20260506 02:18:47.636 | INFO | "gpu" : " 17 GB", | 20260506 02:18:47.636 | INFO | "huge" : "", | 20260506 02:18:47.636 | INFO | "huge_shared" : " 27 GB", | 20260506 02:18:47.636 | INFO | "malloc" : " 14 GB", | 20260506 02:18:47.636 | INFO | "pinned" : " 29 GB" | 20260506 02:18:47.636 | INFO | } | 20260506 02:18:47.637 | INFO | Perf report for the last 5.13878 seconds: | 20260506 02:18:47.637 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:18:47.637 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:47.637 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:47.637 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:47.637 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 02:18:47.637 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:47.637 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:47.637 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 02:18:47.637 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:47.637 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:47.637 | INFO | Uploaded 36GiB to GPU in 2.08748s (17.2457GiB/s) | 20260506 02:18:47.637 | INFO | Downloaded 34GiB to GPU in 2.01511s (16.8725GiB/s) | 20260506 02:18:52.775 | INFO | Analyzing chunk frames = [8173056,8173568) | 20260506 02:18:52.775 | INFO | MemInfo: { | 20260506 02:18:52.775 | INFO | "gpu" : " 17 GB", | 20260506 02:18:52.775 | INFO | "huge" : "", | 20260506 02:18:52.775 | INFO | "huge_shared" : " 27 GB", | 20260506 02:18:52.775 | INFO | "malloc" : " 14 GB", | 20260506 02:18:52.775 | INFO | "pinned" : " 29 GB" | 20260506 02:18:52.775 | INFO | } | 20260506 02:18:52.775 | INFO | Perf report for the last 5.13826 seconds: | 20260506 02:18:52.775 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:18:52.775 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:52.775 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:52.775 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:52.775 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260506 02:18:52.775 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:52.775 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:52.775 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 02:18:52.775 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:52.775 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:52.775 | INFO | Uploaded 36GiB to GPU in 2.1516s (16.7318GiB/s) | 20260506 02:18:52.775 | INFO | Downloaded 34GiB to GPU in 2.05478s (16.5468GiB/s) | 20260506 02:18:57.915 | INFO | Analyzing chunk frames = [8173568,8174080) | 20260506 02:18:57.915 | INFO | MemInfo: { | 20260506 02:18:57.915 | INFO | "gpu" : " 17 GB", | 20260506 02:18:57.915 | INFO | "huge" : "", | 20260506 02:18:57.915 | INFO | "huge_shared" : " 27 GB", | 20260506 02:18:57.915 | INFO | "malloc" : " 14 GB", | 20260506 02:18:57.915 | INFO | "pinned" : " 29 GB" | 20260506 02:18:57.915 | INFO | } | 20260506 02:18:57.915 | INFO | Perf report for the last 5.13998 seconds: | 20260506 02:18:57.915 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:18:57.915 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:57.915 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:57.915 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:57.915 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260506 02:18:57.915 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:57.915 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:57.915 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 02:18:57.915 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:57.915 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:18:57.915 | INFO | Uploaded 36GiB to GPU in 2.10073s (17.1369GiB/s) | 20260506 02:18:57.915 | INFO | Downloaded 34GiB to GPU in 2.03419s (16.7143GiB/s) | 20260506 02:19:03.059 | INFO | Analyzing chunk frames = [8174080,8174592) | 20260506 02:19:03.059 | INFO | MemInfo: { | 20260506 02:19:03.059 | INFO | "gpu" : " 17 GB", | 20260506 02:19:03.059 | INFO | "huge" : "", | 20260506 02:19:03.059 | INFO | "huge_shared" : " 27 GB", | 20260506 02:19:03.059 | INFO | "malloc" : " 14 GB", | 20260506 02:19:03.059 | INFO | "pinned" : " 29 GB" | 20260506 02:19:03.059 | INFO | } | 20260506 02:19:03.059 | INFO | Perf report for the last 5.14463 seconds: | 20260506 02:19:03.059 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:19:03.059 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:03.059 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:03.059 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:03.060 | INFO | ANALYSIS : 61.0%, 3.0, 95.4ms, 0 | 20260506 02:19:03.060 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:03.060 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:03.060 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 02:19:03.060 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:03.060 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:03.060 | INFO | Uploaded 36GiB to GPU in 2.10118s (17.1332GiB/s) | 20260506 02:19:03.060 | INFO | Downloaded 34GiB to GPU in 2.04256s (16.6458GiB/s) | 20260506 02:19:08.192 | INFO | Analyzing chunk frames = [8174592,8175104) | 20260506 02:19:08.192 | INFO | MemInfo: { | 20260506 02:19:08.192 | INFO | "gpu" : " 17 GB", | 20260506 02:19:08.192 | INFO | "huge" : "", | 20260506 02:19:08.192 | INFO | "huge_shared" : " 27 GB", | 20260506 02:19:08.192 | INFO | "malloc" : " 14 GB", | 20260506 02:19:08.192 | INFO | "pinned" : " 29 GB" | 20260506 02:19:08.192 | INFO | } | 20260506 02:19:08.192 | INFO | Perf report for the last 5.13218 seconds: | 20260506 02:19:08.192 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:19:08.192 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:08.192 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:08.192 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:08.192 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260506 02:19:08.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:08.192 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:08.192 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 02:19:08.192 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:08.192 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:08.192 | INFO | Uploaded 36GiB to GPU in 2.05948s (17.4801GiB/s) | 20260506 02:19:08.192 | INFO | Downloaded 34GiB to GPU in 2.07643s (16.3743GiB/s) | 20260506 02:19:13.330 | INFO | Analyzing chunk frames = [8175104,8175616) | 20260506 02:19:13.330 | INFO | MemInfo: { | 20260506 02:19:13.330 | INFO | "gpu" : " 17 GB", | 20260506 02:19:13.330 | INFO | "huge" : "", | 20260506 02:19:13.330 | INFO | "huge_shared" : " 27 GB", | 20260506 02:19:13.330 | INFO | "malloc" : " 14 GB", | 20260506 02:19:13.330 | INFO | "pinned" : " 29 GB" | 20260506 02:19:13.330 | INFO | } | 20260506 02:19:13.331 | INFO | Perf report for the last 5.13898 seconds: | 20260506 02:19:13.331 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:19:13.331 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:13.331 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:13.331 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:13.331 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260506 02:19:13.331 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:13.331 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:13.331 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:19:13.331 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:13.331 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:13.331 | INFO | Uploaded 36GiB to GPU in 2.13595s (16.8543GiB/s) | 20260506 02:19:13.331 | INFO | Downloaded 34GiB to GPU in 2.02879s (16.7588GiB/s) | 20260506 02:19:13.331 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 02:19:13.331 | INFO | Total mem usage: 15.3 GiB | 20260506 02:19:13.331 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 02:19:13.331 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.2%) | 20260506 02:19:13.331 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 02:19:13.331 | INFO | High Water Breakdown: | 20260506 02:19:13.331 | INFO | 60.9% : 8.54 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 02:19:13.331 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 02:19:13.331 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 02:19:13.331 | INFO | ------------------------------------------ | 20260506 02:19:13.331 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:19:13.331 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 02:19:13.331 | INFO | Total mem usage: 26.5 GiB | 20260506 02:19:13.331 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 02:19:13.331 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 02:19:13.331 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260506 02:19:13.331 | INFO | High Water Breakdown: | 20260506 02:19:13.331 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:19:13.331 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:19:13.331 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:19:13.331 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:19:13.331 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:19:13.331 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:19:13.331 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 02:19:13.331 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 02:19:13.331 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:19:13.331 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 02:19:13.331 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 02:19:13.331 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 02:19:13.331 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 02:19:13.331 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 02:19:13.331 | INFO | ------------------------------------------ | 20260506 02:19:13.332 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:19:13.332 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 02:19:13.332 | INFO | Total mem usage: 25.6 GiB | 20260506 02:19:13.332 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 02:19:13.332 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 02:19:13.332 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 02:19:13.332 | INFO | High Water Breakdown: | 20260506 02:19:13.332 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 02:19:13.332 | INFO | ------------------------------------------ | 20260506 02:19:13.332 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:19:13.332 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 02:19:13.332 | INFO | Total mem usage: 15.7 GiB | 20260506 02:19:13.332 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 02:19:13.332 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 02:19:13.332 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 02:19:13.332 | INFO | High Water Breakdown: | 20260506 02:19:13.332 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 02:19:13.332 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 02:19:13.332 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 02:19:13.332 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 02:19:13.332 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 02:19:13.332 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 02:19:13.332 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 02:19:13.332 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 02:19:13.332 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 02:19:13.332 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:19:13.332 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 02:19:13.332 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:19:13.332 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 02:19:13.332 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 02:19:13.332 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 02:19:13.332 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 02:19:13.332 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:19:13.332 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 02:19:13.332 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:19:13.332 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:19:13.332 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:19:13.332 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 02:19:13.332 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 02:19:13.332 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:19:13.332 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 02:19:13.332 | INFO | ------------------------------------------ | 20260506 02:19:13.332 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:19:13.332 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 02:19:13.332 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:19:13.332 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:19:13.332 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.37e+01, 1.33e+01 / 7.84e-01 | 20260506 02:19:13.332 | INFO | PerfReport: 0.29%, AggregateStats -- 9.17e-02 / 1.05e-01, 1.03e-02 / 4.93e-03 | 20260506 02:19:13.332 | INFO | PerfReport: 11.86%, Baseline -- 3.71e+00 / 4.51e+00, 5.74e-01 / 2.29e-01 | 20260506 02:19:13.332 | INFO | PerfReport: 3.29%, Histogram -- 1.03e+00 / 1.51e+00, 1.89e-01 / 1.16e-01 | 20260506 02:19:13.332 | INFO | PerfReport: 16.01%, DME -- 4.00e+01 / 4.40e+01, 3.85e+00 / 1.00e+00 | 20260506 02:19:13.332 | INFO | PerfReport: 57.44%, FrameLabeling -- 1.79e+01 / 1.94e+01, 1.49e+00 / 1.15e+00 | 20260506 02:19:13.332 | INFO | PerfReport: 7.04%, PulseAccumulating -- 2.20e+00 / 2.70e+00, 4.17e-01 / 2.69e-01 | 20260506 02:19:13.332 | INFO | PerfReport: 3.02%, Metrics -- 9.43e-01 / 1.19e+00, 2.49e-01 / 1.11e-01 | 20260506 02:19:13.333 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.26e-01 / 3.69e-01, 5.32e-02 / 2.72e-02 | 20260506 02:19:13.333 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 02:19:13.333 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:19:13.333 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:19:13.333 | INFO | PerfReport: Full Execution -- 9.78e+01 / -nan, 2.42e+01 / 0.00e+00 | 20260506 02:19:13.333 | INFO | PerfReport: 22.25%, Upload -- 2.18e+01 / 0.00e+00, 7.34e+00 / 0.00e+00 | 20260506 02:19:13.333 | INFO | PerfReport: 21.90%, Download -- 2.14e+01 / 0.00e+00, 7.56e+00 / 0.00e+00 | 20260506 02:19:13.333 | INFO | PerfReport: 31.96%, Compute -- 3.12e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 02:19:13.333 | INFO | PerfReport: 2.07%, LockUpload -- 2.02e+00 / 0.00e+00, 7.95e+00 / 0.00e+00 | 20260506 02:19:13.333 | INFO | PerfReport: 1.29%, LockDownload -- 1.26e+00 / 0.00e+00, 2.82e+00 / 0.00e+00 | 20260506 02:19:13.333 | INFO | PerfReport: 20.52%, LockCompute -- 2.01e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 02:19:13.333 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.81e-02 / 0.00e+00 | 20260506 02:19:13.456 | INFO | Starting a round of Baz IO | 20260506 02:19:16.588 | INFO | Populating RTMetrics report | 20260506 02:19:16.588 | INFO | RTMetrics report populated | 20260506 02:19:16.588 | INFO | Opening tmp RTMetrics json file | 20260506 02:19:16.589 | INFO | Serializing RTMetrics to json | 20260506 02:19:16.590 | INFO | Serialization done, swapping tmp file now... | 20260506 02:19:16.591 | INFO | RTMetrics done | 20260506 02:19:16.661 | INFO | Finished a round of Baz IO: Wrote 1.796 GiB in 3.204 seconds | 20260506 02:19:16.661 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.27% / 0.00% / 78.73% | 20260506 02:19:16.661 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0712156% / 0% / 8.34465e-05% | 20260506 02:19:16.661 | INFO | Baz IO overall: | 20260506 02:19:16.661 | INFO | BytesWritten : 4584840021211 | 20260506 02:19:16.661 | INFO | NumEvents : 2152840343006 | 20260506 02:19:18.475 | INFO | Analyzing chunk frames = [8175616,8176128) | 20260506 02:19:18.475 | INFO | MemInfo: { | 20260506 02:19:18.475 | INFO | "gpu" : " 17 GB", | 20260506 02:19:18.475 | INFO | "huge" : "", | 20260506 02:19:18.475 | INFO | "huge_shared" : " 27 GB", | 20260506 02:19:18.475 | INFO | "malloc" : " 14 GB", | 20260506 02:19:18.475 | INFO | "pinned" : " 29 GB" | 20260506 02:19:18.475 | INFO | } | 20260506 02:19:18.475 | INFO | Perf report for the last 5.1442 seconds: | 20260506 02:19:18.475 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:19:18.475 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:18.475 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:18.475 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:18.475 | INFO | ANALYSIS : 62.3%, 3.0, 97.4ms, 0 | 20260506 02:19:18.475 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:18.475 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260506 02:19:18.475 | INFO | PRE_HQ : 59.5%, 1.7, 53.5ms, 0 | 20260506 02:19:18.475 | INFO | BAZWRITER : 61.6%, 3.4, 107.8ms, 0 | 20260506 02:19:18.475 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:19:18.475 | INFO | Uploaded 36GiB to GPU in 2.18433s (16.481GiB/s) | 20260506 02:19:18.475 | INFO | Downloaded 38GiB to GPU in 2.24572s (16.9211GiB/s) | 20260506 02:19:23.612 | INFO | Analyzing chunk frames = [8176128,8176640) | 20260506 02:19:23.612 | INFO | MemInfo: { | 20260506 02:19:23.612 | INFO | "gpu" : " 17 GB", | 20260506 02:19:23.612 | INFO | "huge" : "", | 20260506 02:19:23.612 | INFO | "huge_shared" : " 27 GB", | 20260506 02:19:23.612 | INFO | "malloc" : " 14 GB", | 20260506 02:19:23.612 | INFO | "pinned" : " 29 GB" | 20260506 02:19:23.612 | INFO | } | 20260506 02:19:23.612 | INFO | Perf report for the last 5.13733 seconds: | 20260506 02:19:23.612 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:19:23.612 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:23.612 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:23.612 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:23.612 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 02:19:23.612 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:23.612 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:23.612 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 02:19:23.612 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:23.612 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:23.612 | INFO | Uploaded 36GiB to GPU in 1.93251s (18.6287GiB/s) | 20260506 02:19:23.612 | INFO | Downloaded 34GiB to GPU in 1.97461s (17.2186GiB/s) | 20260506 02:19:28.746 | INFO | Analyzing chunk frames = [8176640,8177152) | 20260506 02:19:28.746 | INFO | MemInfo: { | 20260506 02:19:28.746 | INFO | "gpu" : " 17 GB", | 20260506 02:19:28.746 | INFO | "huge" : "", | 20260506 02:19:28.747 | INFO | "huge_shared" : " 27 GB", | 20260506 02:19:28.747 | INFO | "malloc" : " 14 GB", | 20260506 02:19:28.747 | INFO | "pinned" : " 29 GB" | 20260506 02:19:28.747 | INFO | } | 20260506 02:19:28.747 | INFO | Perf report for the last 5.1346 seconds: | 20260506 02:19:28.747 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:19:28.747 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:28.747 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:28.747 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:28.747 | INFO | ANALYSIS : 61.2%, 3.0, 95.1ms, 0 | 20260506 02:19:28.747 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:28.747 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:28.747 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 02:19:28.747 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:28.747 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:28.747 | INFO | Uploaded 36GiB to GPU in 2.08719s (17.248GiB/s) | 20260506 02:19:28.747 | INFO | Downloaded 34GiB to GPU in 2.03367s (16.7185GiB/s) | 20260506 02:19:33.883 | INFO | Analyzing chunk frames = [8177152,8177664) | 20260506 02:19:33.883 | INFO | MemInfo: { | 20260506 02:19:33.883 | INFO | "gpu" : " 17 GB", | 20260506 02:19:33.883 | INFO | "huge" : "", | 20260506 02:19:33.883 | INFO | "huge_shared" : " 27 GB", | 20260506 02:19:33.883 | INFO | "malloc" : " 14 GB", | 20260506 02:19:33.883 | INFO | "pinned" : " 29 GB" | 20260506 02:19:33.883 | INFO | } | 20260506 02:19:33.883 | INFO | Perf report for the last 5.13628 seconds: | 20260506 02:19:33.883 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:19:33.883 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:33.883 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:33.883 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:33.883 | INFO | ANALYSIS : 61.6%, 3.0, 95.7ms, 0 | 20260506 02:19:33.883 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:33.883 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:33.883 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:19:33.883 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:33.883 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:33.883 | INFO | Uploaded 36GiB to GPU in 2.13572s (16.8561GiB/s) | 20260506 02:19:33.883 | INFO | Downloaded 34GiB to GPU in 2.01662s (16.8599GiB/s) | 20260506 02:19:39.029 | INFO | Analyzing chunk frames = [8177664,8178176) | 20260506 02:19:39.029 | INFO | MemInfo: { | 20260506 02:19:39.029 | INFO | "gpu" : " 17 GB", | 20260506 02:19:39.029 | INFO | "huge" : "", | 20260506 02:19:39.029 | INFO | "huge_shared" : " 27 GB", | 20260506 02:19:39.029 | INFO | "malloc" : " 14 GB", | 20260506 02:19:39.029 | INFO | "pinned" : " 29 GB" | 20260506 02:19:39.029 | INFO | } | 20260506 02:19:39.029 | INFO | Perf report for the last 5.14613 seconds: | 20260506 02:19:39.029 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:19:39.029 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:39.029 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:39.029 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:39.029 | INFO | ANALYSIS : 60.9%, 3.0, 95.2ms, 0 | 20260506 02:19:39.029 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:39.029 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:39.029 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 02:19:39.029 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:39.029 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:39.029 | INFO | Uploaded 36GiB to GPU in 2.10841s (17.0745GiB/s) | 20260506 02:19:39.029 | INFO | Downloaded 34GiB to GPU in 2.04337s (16.6392GiB/s) | 20260506 02:19:44.171 | INFO | Analyzing chunk frames = [8178176,8178688) | 20260506 02:19:44.171 | INFO | MemInfo: { | 20260506 02:19:44.171 | INFO | "gpu" : " 17 GB", | 20260506 02:19:44.171 | INFO | "huge" : "", | 20260506 02:19:44.171 | INFO | "huge_shared" : " 27 GB", | 20260506 02:19:44.171 | INFO | "malloc" : " 14 GB", | 20260506 02:19:44.171 | INFO | "pinned" : " 29 GB" | 20260506 02:19:44.171 | INFO | } | 20260506 02:19:44.171 | INFO | Perf report for the last 5.14173 seconds: | 20260506 02:19:44.171 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:19:44.171 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:44.171 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:44.171 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:44.171 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 02:19:44.171 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:44.171 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:44.171 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:19:44.171 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:44.171 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:44.171 | INFO | Uploaded 36GiB to GPU in 2.10738s (17.0828GiB/s) | 20260506 02:19:44.171 | INFO | Downloaded 34GiB to GPU in 2.04331s (16.6396GiB/s) | 20260506 02:19:49.307 | INFO | Analyzing chunk frames = [8178688,8179200) | 20260506 02:19:49.307 | INFO | MemInfo: { | 20260506 02:19:49.307 | INFO | "gpu" : " 17 GB", | 20260506 02:19:49.307 | INFO | "huge" : "", | 20260506 02:19:49.307 | INFO | "huge_shared" : " 27 GB", | 20260506 02:19:49.307 | INFO | "malloc" : " 14 GB", | 20260506 02:19:49.307 | INFO | "pinned" : " 29 GB" | 20260506 02:19:49.307 | INFO | } | 20260506 02:19:49.307 | INFO | Perf report for the last 5.13594 seconds: | 20260506 02:19:49.307 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:19:49.307 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:49.307 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:49.307 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:49.307 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260506 02:19:49.307 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:49.307 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:49.307 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 02:19:49.307 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:49.307 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:49.307 | INFO | Uploaded 36GiB to GPU in 2.09727s (17.1652GiB/s) | 20260506 02:19:49.307 | INFO | Downloaded 34GiB to GPU in 2.08937s (16.2728GiB/s) | 20260506 02:19:54.443 | INFO | Analyzing chunk frames = [8179200,8179712) | 20260506 02:19:54.443 | INFO | MemInfo: { | 20260506 02:19:54.443 | INFO | "gpu" : " 17 GB", | 20260506 02:19:54.443 | INFO | "huge" : "", | 20260506 02:19:54.443 | INFO | "huge_shared" : " 27 GB", | 20260506 02:19:54.443 | INFO | "malloc" : " 14 GB", | 20260506 02:19:54.443 | INFO | "pinned" : " 29 GB" | 20260506 02:19:54.443 | INFO | } | 20260506 02:19:54.444 | INFO | Perf report for the last 5.13662 seconds: | 20260506 02:19:54.444 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:19:54.444 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:54.444 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:54.444 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:54.444 | INFO | ANALYSIS : 61.4%, 3.0, 95.9ms, 0 | 20260506 02:19:54.444 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:54.444 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:54.444 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:19:54.444 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:54.444 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:54.444 | INFO | Uploaded 36GiB to GPU in 2.02968s (17.7367GiB/s) | 20260506 02:19:54.444 | INFO | Downloaded 34GiB to GPU in 2.01705s (16.8563GiB/s) | 20260506 02:19:54.561 | INFO | Starting a round of Baz IO | 20260506 02:19:57.704 | INFO | Populating RTMetrics report | 20260506 02:19:57.704 | INFO | RTMetrics report populated | 20260506 02:19:57.704 | INFO | Opening tmp RTMetrics json file | 20260506 02:19:57.705 | INFO | Serializing RTMetrics to json | 20260506 02:19:57.706 | INFO | Serialization done, swapping tmp file now... | 20260506 02:19:57.707 | INFO | RTMetrics done | 20260506 02:19:57.776 | INFO | Finished a round of Baz IO: Wrote 1.797 GiB in 3.215 seconds | 20260506 02:19:57.776 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.27% / 0.00% / 78.73% | 20260506 02:19:57.776 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0718474% / 0% / 9.13938e-05% | 20260506 02:19:57.777 | INFO | Baz IO overall: | 20260506 02:19:57.777 | INFO | BytesWritten : 4586769824425 | 20260506 02:19:57.777 | INFO | NumEvents : 2153718054723 | 20260506 02:19:59.585 | INFO | Analyzing chunk frames = [8179712,8180224) | 20260506 02:19:59.585 | INFO | MemInfo: { | 20260506 02:19:59.585 | INFO | "gpu" : " 17 GB", | 20260506 02:19:59.585 | INFO | "huge" : "", | 20260506 02:19:59.585 | INFO | "huge_shared" : " 27 GB", | 20260506 02:19:59.585 | INFO | "malloc" : " 14 GB", | 20260506 02:19:59.585 | INFO | "pinned" : " 29 GB" | 20260506 02:19:59.585 | INFO | } | 20260506 02:19:59.585 | INFO | Perf report for the last 5.14176 seconds: | 20260506 02:19:59.585 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:19:59.585 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:59.585 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:59.585 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:59.585 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260506 02:19:59.585 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:19:59.585 | INFO | RT_METRICS : 42.8%, 1.0, 22.4ms, 0 | 20260506 02:19:59.585 | INFO | PRE_HQ : 59.6%, 1.7, 53.4ms, 0 | 20260506 02:19:59.585 | INFO | BAZWRITER : 61.6%, 3.4, 107.9ms, 0 | 20260506 02:19:59.585 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:19:59.586 | INFO | Uploaded 36GiB to GPU in 2.19367s (16.4108GiB/s) | 20260506 02:19:59.586 | INFO | Downloaded 38GiB to GPU in 2.25416s (16.8578GiB/s) | 20260506 02:20:04.728 | INFO | Analyzing chunk frames = [8180224,8180736) | 20260506 02:20:04.728 | INFO | MemInfo: { | 20260506 02:20:04.728 | INFO | "gpu" : " 17 GB", | 20260506 02:20:04.728 | INFO | "huge" : "", | 20260506 02:20:04.728 | INFO | "huge_shared" : " 27 GB", | 20260506 02:20:04.728 | INFO | "malloc" : " 14 GB", | 20260506 02:20:04.728 | INFO | "pinned" : " 29 GB" | 20260506 02:20:04.728 | INFO | } | 20260506 02:20:04.728 | INFO | Perf report for the last 5.14274 seconds: | 20260506 02:20:04.728 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:20:04.728 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:04.728 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:04.728 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:04.728 | INFO | ANALYSIS : 60.7%, 3.0, 94.8ms, 0 | 20260506 02:20:04.728 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:04.728 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:04.728 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 02:20:04.728 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:04.728 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:04.728 | INFO | Uploaded 36GiB to GPU in 1.94793s (18.4812GiB/s) | 20260506 02:20:04.728 | INFO | Downloaded 34GiB to GPU in 1.97348s (17.2284GiB/s) | 20260506 02:20:09.869 | INFO | Analyzing chunk frames = [8180736,8181248) | 20260506 02:20:09.869 | INFO | MemInfo: { | 20260506 02:20:09.869 | INFO | "gpu" : " 17 GB", | 20260506 02:20:09.869 | INFO | "huge" : "", | 20260506 02:20:09.869 | INFO | "huge_shared" : " 27 GB", | 20260506 02:20:09.869 | INFO | "malloc" : " 14 GB", | 20260506 02:20:09.869 | INFO | "pinned" : " 29 GB" | 20260506 02:20:09.869 | INFO | } | 20260506 02:20:09.869 | INFO | Perf report for the last 5.14099 seconds: | 20260506 02:20:09.869 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:20:09.869 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:09.869 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:09.869 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:09.869 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 02:20:09.869 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:09.869 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:09.869 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 02:20:09.869 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:09.869 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:09.869 | INFO | Uploaded 36GiB to GPU in 2.10541s (17.0988GiB/s) | 20260506 02:20:09.869 | INFO | Downloaded 34GiB to GPU in 2.04036s (16.6637GiB/s) | 20260506 02:20:15.010 | INFO | Analyzing chunk frames = [8181248,8181760) | 20260506 02:20:15.010 | INFO | MemInfo: { | 20260506 02:20:15.010 | INFO | "gpu" : " 17 GB", | 20260506 02:20:15.010 | INFO | "huge" : "", | 20260506 02:20:15.010 | INFO | "huge_shared" : " 27 GB", | 20260506 02:20:15.010 | INFO | "malloc" : " 14 GB", | 20260506 02:20:15.010 | INFO | "pinned" : " 29 GB" | 20260506 02:20:15.010 | INFO | } | 20260506 02:20:15.010 | INFO | Perf report for the last 5.14125 seconds: | 20260506 02:20:15.010 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:20:15.010 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:15.010 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:15.010 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:15.010 | INFO | ANALYSIS : 61.1%, 3.0, 94.9ms, 0 | 20260506 02:20:15.010 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:15.010 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:15.010 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 02:20:15.010 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:15.010 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:15.011 | INFO | Uploaded 36GiB to GPU in 2.08373s (17.2767GiB/s) | 20260506 02:20:15.011 | INFO | Downloaded 34GiB to GPU in 2.01756s (16.8521GiB/s) | 20260506 02:20:20.144 | INFO | Analyzing chunk frames = [8181760,8182272) | 20260506 02:20:20.144 | INFO | MemInfo: { | 20260506 02:20:20.144 | INFO | "gpu" : " 17 GB", | 20260506 02:20:20.144 | INFO | "huge" : "", | 20260506 02:20:20.144 | INFO | "huge_shared" : " 27 GB", | 20260506 02:20:20.144 | INFO | "malloc" : " 14 GB", | 20260506 02:20:20.144 | INFO | "pinned" : " 29 GB" | 20260506 02:20:20.144 | INFO | } | 20260506 02:20:20.144 | INFO | Perf report for the last 5.13362 seconds: | 20260506 02:20:20.144 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:20:20.144 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:20.144 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:20.144 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:20.144 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 02:20:20.144 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:20.144 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:20.144 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:20:20.144 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:20.144 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:20.144 | INFO | Uploaded 36GiB to GPU in 2.09691s (17.1681GiB/s) | 20260506 02:20:20.144 | INFO | Downloaded 34GiB to GPU in 2.02809s (16.7645GiB/s) | 20260506 02:20:25.282 | INFO | Analyzing chunk frames = [8182272,8182784) | 20260506 02:20:25.282 | INFO | MemInfo: { | 20260506 02:20:25.282 | INFO | "gpu" : " 17 GB", | 20260506 02:20:25.282 | INFO | "huge" : "", | 20260506 02:20:25.282 | INFO | "huge_shared" : " 27 GB", | 20260506 02:20:25.282 | INFO | "malloc" : " 14 GB", | 20260506 02:20:25.283 | INFO | "pinned" : " 29 GB" | 20260506 02:20:25.283 | INFO | } | 20260506 02:20:25.283 | INFO | Perf report for the last 5.13862 seconds: | 20260506 02:20:25.283 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:20:25.283 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:25.283 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:25.283 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:25.283 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 02:20:25.283 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:25.283 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:25.283 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 02:20:25.283 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:25.283 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:25.283 | INFO | Uploaded 36GiB to GPU in 2.09278s (17.202GiB/s) | 20260506 02:20:25.283 | INFO | Downloaded 34GiB to GPU in 2.04188s (16.6513GiB/s) | 20260506 02:20:30.420 | INFO | Analyzing chunk frames = [8182784,8183296) | 20260506 02:20:30.420 | INFO | MemInfo: { | 20260506 02:20:30.420 | INFO | "gpu" : " 17 GB", | 20260506 02:20:30.420 | INFO | "huge" : "", | 20260506 02:20:30.420 | INFO | "huge_shared" : " 27 GB", | 20260506 02:20:30.420 | INFO | "malloc" : " 14 GB", | 20260506 02:20:30.420 | INFO | "pinned" : " 29 GB" | 20260506 02:20:30.420 | INFO | } | 20260506 02:20:30.421 | INFO | Perf report for the last 5.1379 seconds: | 20260506 02:20:30.421 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:20:30.421 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:30.421 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:30.421 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:30.421 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260506 02:20:30.421 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:30.421 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:30.421 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 02:20:30.421 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:30.421 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:30.421 | INFO | Uploaded 36GiB to GPU in 2.04208s (17.6291GiB/s) | 20260506 02:20:30.421 | INFO | Downloaded 34GiB to GPU in 2.04931s (16.5909GiB/s) | 20260506 02:20:35.560 | INFO | Analyzing chunk frames = [8183296,8183808) | 20260506 02:20:35.560 | INFO | MemInfo: { | 20260506 02:20:35.560 | INFO | "gpu" : " 17 GB", | 20260506 02:20:35.560 | INFO | "huge" : "", | 20260506 02:20:35.560 | INFO | "huge_shared" : " 27 GB", | 20260506 02:20:35.560 | INFO | "malloc" : " 14 GB", | 20260506 02:20:35.560 | INFO | "pinned" : " 29 GB" | 20260506 02:20:35.560 | INFO | } | 20260506 02:20:35.560 | INFO | Perf report for the last 5.13928 seconds: | 20260506 02:20:35.560 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:20:35.560 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:35.560 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:35.560 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:35.560 | INFO | ANALYSIS : 61.5%, 3.0, 96.0ms, 0 | 20260506 02:20:35.560 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:35.560 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:35.560 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 02:20:35.560 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:35.560 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:35.560 | INFO | Uploaded 36GiB to GPU in 2.04706s (17.5862GiB/s) | 20260506 02:20:35.560 | INFO | Downloaded 34GiB to GPU in 2.02773s (16.7675GiB/s) | 20260506 02:20:35.561 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 02:20:35.561 | INFO | Total mem usage: 15.3 GiB | 20260506 02:20:35.561 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 02:20:35.561 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.2%) | 20260506 02:20:35.561 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 02:20:35.561 | INFO | High Water Breakdown: | 20260506 02:20:35.561 | INFO | 60.9% : 8.54 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 02:20:35.561 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 02:20:35.561 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 02:20:35.561 | INFO | ------------------------------------------ | 20260506 02:20:35.561 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:20:35.561 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 02:20:35.561 | INFO | Total mem usage: 26.5 GiB | 20260506 02:20:35.561 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 02:20:35.561 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 02:20:35.561 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260506 02:20:35.561 | INFO | High Water Breakdown: | 20260506 02:20:35.561 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:20:35.561 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:20:35.561 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:20:35.561 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:20:35.561 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:20:35.561 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:20:35.561 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 02:20:35.561 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 02:20:35.561 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:20:35.561 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 02:20:35.561 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 02:20:35.561 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 02:20:35.561 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 02:20:35.561 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 02:20:35.561 | INFO | ------------------------------------------ | 20260506 02:20:35.561 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:20:35.561 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 02:20:35.561 | INFO | Total mem usage: 25.6 GiB | 20260506 02:20:35.561 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 02:20:35.561 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 02:20:35.561 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 02:20:35.561 | INFO | High Water Breakdown: | 20260506 02:20:35.561 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 02:20:35.561 | INFO | ------------------------------------------ | 20260506 02:20:35.561 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:20:35.561 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 02:20:35.561 | INFO | Total mem usage: 15.7 GiB | 20260506 02:20:35.561 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 02:20:35.561 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 02:20:35.561 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 02:20:35.561 | INFO | High Water Breakdown: | 20260506 02:20:35.561 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 02:20:35.561 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 02:20:35.561 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 02:20:35.561 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 02:20:35.561 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 02:20:35.561 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 02:20:35.561 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 02:20:35.561 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 02:20:35.561 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 02:20:35.561 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:20:35.561 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 02:20:35.561 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:20:35.561 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 02:20:35.561 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 02:20:35.561 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 02:20:35.561 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 02:20:35.561 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:20:35.561 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 02:20:35.561 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:20:35.561 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:20:35.561 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:20:35.561 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 02:20:35.561 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 02:20:35.561 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:20:35.561 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 02:20:35.561 | INFO | ------------------------------------------ | 20260506 02:20:35.561 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:20:35.561 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 02:20:35.561 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:20:35.561 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:20:35.561 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.37e+01, 1.34e+01 / 6.81e-01 | 20260506 02:20:35.562 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.05e-01, 1.14e-02 / 4.62e-03 | 20260506 02:20:35.562 | INFO | PerfReport: 11.87%, Baseline -- 3.71e+00 / 4.50e+00, 5.75e-01 / 1.97e-01 | 20260506 02:20:35.562 | INFO | PerfReport: 3.30%, Histogram -- 1.03e+00 / 1.53e+00, 1.93e-01 / 7.85e-02 | 20260506 02:20:35.562 | INFO | PerfReport: 16.00%, DME -- 4.00e+01 / 4.40e+01, 3.87e+00 / 9.36e-01 | 20260506 02:20:35.562 | INFO | PerfReport: 57.43%, FrameLabeling -- 1.79e+01 / 1.93e+01, 1.48e+00 / 8.53e-01 | 20260506 02:20:35.562 | INFO | PerfReport: 7.05%, PulseAccumulating -- 2.20e+00 / 2.72e+00, 4.17e-01 / 2.05e-01 | 20260506 02:20:35.562 | INFO | PerfReport: 3.02%, Metrics -- 9.44e-01 / 1.20e+00, 2.50e-01 / 9.94e-02 | 20260506 02:20:35.562 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.25e-01 / 3.71e-01, 5.39e-02 / 2.37e-02 | 20260506 02:20:35.562 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 02:20:35.562 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:20:35.562 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:20:35.562 | INFO | PerfReport: Full Execution -- 9.77e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260506 02:20:35.562 | INFO | PerfReport: 22.19%, Upload -- 2.17e+01 / 0.00e+00, 7.23e+00 / 0.00e+00 | 20260506 02:20:35.562 | INFO | PerfReport: 21.93%, Download -- 2.14e+01 / 0.00e+00, 7.49e+00 / 0.00e+00 | 20260506 02:20:35.562 | INFO | PerfReport: 32.00%, Compute -- 3.12e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 02:20:35.562 | INFO | PerfReport: 2.02%, LockUpload -- 1.97e+00 / 0.00e+00, 7.83e+00 / 0.00e+00 | 20260506 02:20:35.562 | INFO | PerfReport: 1.33%, LockDownload -- 1.30e+00 / 0.00e+00, 2.93e+00 / 0.00e+00 | 20260506 02:20:35.562 | INFO | PerfReport: 20.51%, LockCompute -- 2.00e+01 / 0.00e+00, 1.77e+01 / 0.00e+00 | 20260506 02:20:35.562 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.31e-02 / 0.00e+00 | 20260506 02:20:35.686 | INFO | Starting a round of Baz IO | 20260506 02:20:38.820 | INFO | Populating RTMetrics report | 20260506 02:20:38.820 | INFO | RTMetrics report populated | 20260506 02:20:38.820 | INFO | Opening tmp RTMetrics json file | 20260506 02:20:38.821 | INFO | Serializing RTMetrics to json | 20260506 02:20:38.822 | INFO | Serialization done, swapping tmp file now... | 20260506 02:20:38.823 | INFO | RTMetrics done | 20260506 02:20:38.894 | INFO | Finished a round of Baz IO: Wrote 1.797 GiB in 3.207 seconds | 20260506 02:20:38.894 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.27% / 0.00% / 78.73% | 20260506 02:20:38.894 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0720263% / 0% / 9.53674e-05% | 20260506 02:20:38.894 | INFO | Baz IO overall: | 20260506 02:20:38.894 | INFO | BytesWritten : 4588699415235 | 20260506 02:20:38.894 | INFO | NumEvents : 2154595498181 | 20260506 02:20:40.699 | INFO | Analyzing chunk frames = [8183808,8184320) | 20260506 02:20:40.699 | INFO | MemInfo: { | 20260506 02:20:40.699 | INFO | "gpu" : " 17 GB", | 20260506 02:20:40.699 | INFO | "huge" : "", | 20260506 02:20:40.699 | INFO | "huge_shared" : " 27 GB", | 20260506 02:20:40.699 | INFO | "malloc" : " 14 GB", | 20260506 02:20:40.699 | INFO | "pinned" : " 29 GB" | 20260506 02:20:40.699 | INFO | } | 20260506 02:20:40.699 | INFO | Perf report for the last 5.13936 seconds: | 20260506 02:20:40.699 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:20:40.699 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:40.699 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:40.699 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:40.699 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260506 02:20:40.699 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:40.699 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260506 02:20:40.699 | INFO | PRE_HQ : 59.5%, 1.7, 53.3ms, 0 | 20260506 02:20:40.699 | INFO | BAZWRITER : 61.7%, 3.4, 108.1ms, 0 | 20260506 02:20:40.699 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:20:40.700 | INFO | Uploaded 36GiB to GPU in 2.18404s (16.4832GiB/s) | 20260506 02:20:40.700 | INFO | Downloaded 38GiB to GPU in 2.23841s (16.9763GiB/s) | 20260506 02:20:45.839 | INFO | Analyzing chunk frames = [8184320,8184832) | 20260506 02:20:45.839 | INFO | MemInfo: { | 20260506 02:20:45.839 | INFO | "gpu" : " 17 GB", | 20260506 02:20:45.839 | INFO | "huge" : "", | 20260506 02:20:45.839 | INFO | "huge_shared" : " 27 GB", | 20260506 02:20:45.839 | INFO | "malloc" : " 14 GB", | 20260506 02:20:45.839 | INFO | "pinned" : " 29 GB" | 20260506 02:20:45.839 | INFO | } | 20260506 02:20:45.839 | INFO | Perf report for the last 5.13978 seconds: | 20260506 02:20:45.839 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:20:45.839 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:45.839 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:45.839 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:45.839 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 02:20:45.839 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:45.839 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:45.839 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 02:20:45.839 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:45.839 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:45.839 | INFO | Uploaded 36GiB to GPU in 2.13159s (16.8888GiB/s) | 20260506 02:20:45.839 | INFO | Downloaded 34GiB to GPU in 2.04788s (16.6025GiB/s) | 20260506 02:20:50.983 | INFO | Analyzing chunk frames = [8184832,8185344) | 20260506 02:20:50.983 | INFO | MemInfo: { | 20260506 02:20:50.983 | INFO | "gpu" : " 17 GB", | 20260506 02:20:50.983 | INFO | "huge" : "", | 20260506 02:20:50.983 | INFO | "huge_shared" : " 27 GB", | 20260506 02:20:50.983 | INFO | "malloc" : " 14 GB", | 20260506 02:20:50.983 | INFO | "pinned" : " 29 GB" | 20260506 02:20:50.983 | INFO | } | 20260506 02:20:50.983 | INFO | Perf report for the last 5.14434 seconds: | 20260506 02:20:50.983 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:20:50.983 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:50.983 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:50.983 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:50.983 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 02:20:50.983 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:50.983 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:50.983 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 02:20:50.983 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:50.983 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:50.984 | INFO | Uploaded 36GiB to GPU in 2.076s (17.3411GiB/s) | 20260506 02:20:50.984 | INFO | Downloaded 34GiB to GPU in 2.01969s (16.8343GiB/s) | 20260506 02:20:56.121 | INFO | Analyzing chunk frames = [8185344,8185856) | 20260506 02:20:56.121 | INFO | MemInfo: { | 20260506 02:20:56.121 | INFO | "gpu" : " 17 GB", | 20260506 02:20:56.121 | INFO | "huge" : "", | 20260506 02:20:56.121 | INFO | "huge_shared" : " 27 GB", | 20260506 02:20:56.121 | INFO | "malloc" : " 14 GB", | 20260506 02:20:56.121 | INFO | "pinned" : " 29 GB" | 20260506 02:20:56.121 | INFO | } | 20260506 02:20:56.121 | INFO | Perf report for the last 5.13806 seconds: | 20260506 02:20:56.121 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:20:56.121 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:56.121 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:56.121 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:56.121 | INFO | ANALYSIS : 61.0%, 3.0, 94.7ms, 0 | 20260506 02:20:56.121 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:56.121 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:56.121 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 02:20:56.121 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:56.122 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:20:56.122 | INFO | Uploaded 36GiB to GPU in 2.08094s (17.2999GiB/s) | 20260506 02:20:56.122 | INFO | Downloaded 34GiB to GPU in 2.00772s (16.9346GiB/s) | 20260506 02:21:01.252 | INFO | Analyzing chunk frames = [8185856,8186368) | 20260506 02:21:01.252 | INFO | MemInfo: { | 20260506 02:21:01.252 | INFO | "gpu" : " 17 GB", | 20260506 02:21:01.252 | INFO | "huge" : "", | 20260506 02:21:01.252 | INFO | "huge_shared" : " 27 GB", | 20260506 02:21:01.252 | INFO | "malloc" : " 14 GB", | 20260506 02:21:01.253 | INFO | "pinned" : " 29 GB" | 20260506 02:21:01.253 | INFO | } | 20260506 02:21:01.253 | INFO | Perf report for the last 5.13129 seconds: | 20260506 02:21:01.253 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:21:01.253 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:01.253 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:01.253 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:01.253 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260506 02:21:01.253 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:01.253 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:01.253 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 02:21:01.253 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:01.253 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:01.253 | INFO | Uploaded 36GiB to GPU in 2.07426s (17.3556GiB/s) | 20260506 02:21:01.253 | INFO | Downloaded 34GiB to GPU in 2.03389s (16.7168GiB/s) | 20260506 02:21:06.393 | INFO | Analyzing chunk frames = [8186368,8186880) | 20260506 02:21:06.393 | INFO | MemInfo: { | 20260506 02:21:06.393 | INFO | "gpu" : " 17 GB", | 20260506 02:21:06.393 | INFO | "huge" : "", | 20260506 02:21:06.393 | INFO | "huge_shared" : " 27 GB", | 20260506 02:21:06.393 | INFO | "malloc" : " 14 GB", | 20260506 02:21:06.393 | INFO | "pinned" : " 29 GB" | 20260506 02:21:06.393 | INFO | } | 20260506 02:21:06.393 | INFO | Perf report for the last 5.14023 seconds: | 20260506 02:21:06.393 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:21:06.393 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:06.393 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:06.393 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:06.393 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 02:21:06.393 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:06.393 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:06.393 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 02:21:06.393 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:06.393 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:06.393 | INFO | Uploaded 36GiB to GPU in 2.06797s (17.4083GiB/s) | 20260506 02:21:06.393 | INFO | Downloaded 34GiB to GPU in 2.03694s (16.6917GiB/s) | 20260506 02:21:11.537 | INFO | Analyzing chunk frames = [8186880,8187392) | 20260506 02:21:11.537 | INFO | MemInfo: { | 20260506 02:21:11.537 | INFO | "gpu" : " 17 GB", | 20260506 02:21:11.537 | INFO | "huge" : "", | 20260506 02:21:11.537 | INFO | "huge_shared" : " 27 GB", | 20260506 02:21:11.537 | INFO | "malloc" : " 14 GB", | 20260506 02:21:11.537 | INFO | "pinned" : " 29 GB" | 20260506 02:21:11.537 | INFO | } | 20260506 02:21:11.537 | INFO | Perf report for the last 5.14407 seconds: | 20260506 02:21:11.537 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:21:11.537 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:11.537 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:11.537 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:11.537 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260506 02:21:11.537 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:11.537 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:11.537 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 02:21:11.537 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:11.537 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:11.537 | INFO | Uploaded 36GiB to GPU in 2.06139s (17.464GiB/s) | 20260506 02:21:11.537 | INFO | Downloaded 34GiB to GPU in 2.0753s (16.3831GiB/s) | 20260506 02:21:16.683 | INFO | Analyzing chunk frames = [8187392,8187904) | 20260506 02:21:16.683 | INFO | MemInfo: { | 20260506 02:21:16.683 | INFO | "gpu" : " 17 GB", | 20260506 02:21:16.683 | INFO | "huge" : "", | 20260506 02:21:16.683 | INFO | "huge_shared" : " 27 GB", | 20260506 02:21:16.683 | INFO | "malloc" : " 14 GB", | 20260506 02:21:16.683 | INFO | "pinned" : " 29 GB" | 20260506 02:21:16.683 | INFO | } | 20260506 02:21:16.683 | INFO | Perf report for the last 5.14638 seconds: | 20260506 02:21:16.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:21:16.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:16.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:16.683 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:16.683 | INFO | ANALYSIS : 61.4%, 3.0, 95.9ms, 0 | 20260506 02:21:16.683 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:16.684 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:16.684 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:21:16.684 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:16.684 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:16.684 | INFO | Uploaded 36GiB to GPU in 2.0536s (17.5302GiB/s) | 20260506 02:21:16.684 | INFO | Downloaded 34GiB to GPU in 2.02867s (16.7598GiB/s) | 20260506 02:21:16.808 | INFO | Starting a round of Baz IO | 20260506 02:21:19.933 | INFO | Populating RTMetrics report | 20260506 02:21:19.933 | INFO | RTMetrics report populated | 20260506 02:21:19.933 | INFO | Opening tmp RTMetrics json file | 20260506 02:21:19.934 | INFO | Serializing RTMetrics to json | 20260506 02:21:19.935 | INFO | Serialization done, swapping tmp file now... | 20260506 02:21:19.935 | INFO | RTMetrics done | 20260506 02:21:20.007 | INFO | Finished a round of Baz IO: Wrote 1.796 GiB in 3.198 seconds | 20260506 02:21:20.007 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.27% / 0.00% / 78.73% | 20260506 02:21:20.007 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0716647% / 0% / 9.13938e-05% | 20260506 02:21:20.008 | INFO | Baz IO overall: | 20260506 02:21:20.008 | INFO | BytesWritten : 4590628309640 | 20260506 02:21:20.008 | INFO | NumEvents : 2155472676484 | 20260506 02:21:21.819 | INFO | Analyzing chunk frames = [8187904,8188416) | 20260506 02:21:21.819 | INFO | MemInfo: { | 20260506 02:21:21.819 | INFO | "gpu" : " 17 GB", | 20260506 02:21:21.819 | INFO | "huge" : "", | 20260506 02:21:21.819 | INFO | "huge_shared" : " 27 GB", | 20260506 02:21:21.819 | INFO | "malloc" : " 14 GB", | 20260506 02:21:21.819 | INFO | "pinned" : " 29 GB" | 20260506 02:21:21.819 | INFO | } | 20260506 02:21:21.819 | INFO | Perf report for the last 5.13579 seconds: | 20260506 02:21:21.819 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:21:21.819 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:21.819 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:21.819 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:21.819 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260506 02:21:21.819 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:21.819 | INFO | RT_METRICS : 42.8%, 1.0, 22.4ms, 0 | 20260506 02:21:21.819 | INFO | PRE_HQ : 59.1%, 1.7, 53.2ms, 0 | 20260506 02:21:21.819 | INFO | BAZWRITER : 61.4%, 3.5, 109.3ms, 0 | 20260506 02:21:21.819 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:21:21.819 | INFO | Uploaded 36GiB to GPU in 2.1037s (17.1127GiB/s) | 20260506 02:21:21.820 | INFO | Downloaded 38GiB to GPU in 2.2268s (17.0649GiB/s) | 20260506 02:21:26.961 | INFO | Analyzing chunk frames = [8188416,8188928) | 20260506 02:21:26.961 | INFO | MemInfo: { | 20260506 02:21:26.961 | INFO | "gpu" : " 17 GB", | 20260506 02:21:26.961 | INFO | "huge" : "", | 20260506 02:21:26.961 | INFO | "huge_shared" : " 27 GB", | 20260506 02:21:26.961 | INFO | "malloc" : " 14 GB", | 20260506 02:21:26.961 | INFO | "pinned" : " 29 GB" | 20260506 02:21:26.961 | INFO | } | 20260506 02:21:26.961 | INFO | Perf report for the last 5.14222 seconds: | 20260506 02:21:26.961 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:21:26.961 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:26.961 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:26.961 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:26.962 | INFO | ANALYSIS : 60.7%, 3.0, 94.8ms, 0 | 20260506 02:21:26.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:26.962 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:26.962 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 02:21:26.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:26.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:26.962 | INFO | Uploaded 36GiB to GPU in 1.95434s (18.4205GiB/s) | 20260506 02:21:26.962 | INFO | Downloaded 34GiB to GPU in 1.97886s (17.1816GiB/s) | 20260506 02:21:32.096 | INFO | Analyzing chunk frames = [8188928,8189440) | 20260506 02:21:32.096 | INFO | MemInfo: { | 20260506 02:21:32.096 | INFO | "gpu" : " 17 GB", | 20260506 02:21:32.096 | INFO | "huge" : "", | 20260506 02:21:32.096 | INFO | "huge_shared" : " 27 GB", | 20260506 02:21:32.096 | INFO | "malloc" : " 14 GB", | 20260506 02:21:32.096 | INFO | "pinned" : " 29 GB" | 20260506 02:21:32.097 | INFO | } | 20260506 02:21:32.097 | INFO | Perf report for the last 5.13526 seconds: | 20260506 02:21:32.097 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:21:32.097 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:32.097 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:32.097 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:32.097 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 02:21:32.097 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:32.097 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:32.097 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 02:21:32.097 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:32.097 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:32.097 | INFO | Uploaded 36GiB to GPU in 2.106s (17.094GiB/s) | 20260506 02:21:32.097 | INFO | Downloaded 34GiB to GPU in 2.04357s (16.6376GiB/s) | 20260506 02:21:37.231 | INFO | Analyzing chunk frames = [8189440,8189952) | 20260506 02:21:37.231 | INFO | MemInfo: { | 20260506 02:21:37.231 | INFO | "gpu" : " 17 GB", | 20260506 02:21:37.231 | INFO | "huge" : "", | 20260506 02:21:37.231 | INFO | "huge_shared" : " 27 GB", | 20260506 02:21:37.231 | INFO | "malloc" : " 14 GB", | 20260506 02:21:37.231 | INFO | "pinned" : " 29 GB" | 20260506 02:21:37.232 | INFO | } | 20260506 02:21:37.232 | INFO | Perf report for the last 5.13494 seconds: | 20260506 02:21:37.232 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:21:37.232 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:37.232 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:37.232 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:37.232 | INFO | ANALYSIS : 61.0%, 3.0, 94.7ms, 0 | 20260506 02:21:37.232 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:37.232 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:37.232 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 02:21:37.232 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:37.232 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:37.232 | INFO | Uploaded 36GiB to GPU in 2.07946s (17.3122GiB/s) | 20260506 02:21:37.232 | INFO | Downloaded 34GiB to GPU in 2.00786s (16.9335GiB/s) | 20260506 02:21:42.373 | INFO | Analyzing chunk frames = [8189952,8190464) | 20260506 02:21:42.373 | INFO | MemInfo: { | 20260506 02:21:42.373 | INFO | "gpu" : " 17 GB", | 20260506 02:21:42.373 | INFO | "huge" : "", | 20260506 02:21:42.373 | INFO | "huge_shared" : " 27 GB", | 20260506 02:21:42.373 | INFO | "malloc" : " 14 GB", | 20260506 02:21:42.373 | INFO | "pinned" : " 29 GB" | 20260506 02:21:42.374 | INFO | } | 20260506 02:21:42.374 | INFO | Perf report for the last 5.14203 seconds: | 20260506 02:21:42.374 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:21:42.374 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:42.374 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:42.374 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:42.374 | INFO | ANALYSIS : 60.6%, 3.0, 94.7ms, 0 | 20260506 02:21:42.374 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:42.374 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:42.374 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:21:42.374 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:42.374 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:42.374 | INFO | Uploaded 36GiB to GPU in 2.07972s (17.31GiB/s) | 20260506 02:21:42.374 | INFO | Downloaded 34GiB to GPU in 2.02692s (16.7742GiB/s) | 20260506 02:21:47.511 | INFO | Analyzing chunk frames = [8190464,8190976) | 20260506 02:21:47.511 | INFO | MemInfo: { | 20260506 02:21:47.511 | INFO | "gpu" : " 17 GB", | 20260506 02:21:47.511 | INFO | "huge" : "", | 20260506 02:21:47.511 | INFO | "huge_shared" : " 27 GB", | 20260506 02:21:47.511 | INFO | "malloc" : " 14 GB", | 20260506 02:21:47.511 | INFO | "pinned" : " 29 GB" | 20260506 02:21:47.511 | INFO | } | 20260506 02:21:47.511 | INFO | Perf report for the last 5.13763 seconds: | 20260506 02:21:47.512 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:21:47.512 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:47.512 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:47.512 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:47.512 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 02:21:47.512 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:47.512 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:47.512 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 02:21:47.512 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:47.512 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:47.512 | INFO | Uploaded 36GiB to GPU in 2.08809s (17.2406GiB/s) | 20260506 02:21:47.512 | INFO | Downloaded 34GiB to GPU in 2.04119s (16.6569GiB/s) | 20260506 02:21:52.649 | INFO | Analyzing chunk frames = [8190976,8191488) | 20260506 02:21:52.649 | INFO | MemInfo: { | 20260506 02:21:52.649 | INFO | "gpu" : " 17 GB", | 20260506 02:21:52.649 | INFO | "huge" : "", | 20260506 02:21:52.649 | INFO | "huge_shared" : " 27 GB", | 20260506 02:21:52.649 | INFO | "malloc" : " 14 GB", | 20260506 02:21:52.649 | INFO | "pinned" : " 29 GB" | 20260506 02:21:52.649 | INFO | } | 20260506 02:21:52.649 | INFO | Perf report for the last 5.13754 seconds: | 20260506 02:21:52.649 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:21:52.649 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:52.649 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:52.649 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:52.649 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260506 02:21:52.649 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:52.649 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:52.649 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 02:21:52.649 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:52.649 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:52.649 | INFO | Uploaded 36GiB to GPU in 2.10547s (17.0984GiB/s) | 20260506 02:21:52.649 | INFO | Downloaded 34GiB to GPU in 2.09044s (16.2645GiB/s) | 20260506 02:21:57.796 | INFO | Analyzing chunk frames = [8191488,8192000) | 20260506 02:21:57.796 | INFO | MemInfo: { | 20260506 02:21:57.796 | INFO | "gpu" : " 17 GB", | 20260506 02:21:57.796 | INFO | "huge" : "", | 20260506 02:21:57.796 | INFO | "huge_shared" : " 27 GB", | 20260506 02:21:57.796 | INFO | "malloc" : " 14 GB", | 20260506 02:21:57.796 | INFO | "pinned" : " 29 GB" | 20260506 02:21:57.796 | INFO | } | 20260506 02:21:57.797 | INFO | Perf report for the last 5.14763 seconds: | 20260506 02:21:57.797 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:21:57.797 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:57.797 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:57.797 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:57.797 | INFO | ANALYSIS : 61.9%, 3.0, 96.8ms, 0 | 20260506 02:21:57.797 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:57.797 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:57.797 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 02:21:57.797 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:57.797 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:21:57.797 | INFO | Uploaded 36GiB to GPU in 2.1522s (16.7271GiB/s) | 20260506 02:21:57.797 | INFO | Downloaded 34GiB to GPU in 2.04282s (16.6437GiB/s) | 20260506 02:21:57.797 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 02:21:57.797 | INFO | Total mem usage: 15.3 GiB | 20260506 02:21:57.797 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 02:21:57.797 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.2%) | 20260506 02:21:57.797 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 02:21:57.797 | INFO | High Water Breakdown: | 20260506 02:21:57.797 | INFO | 60.9% : 8.54 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 02:21:57.797 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 02:21:57.797 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 02:21:57.797 | INFO | ------------------------------------------ | 20260506 02:21:57.797 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:21:57.797 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 02:21:57.797 | INFO | Total mem usage: 26.5 GiB | 20260506 02:21:57.797 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 02:21:57.797 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 02:21:57.797 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 02:21:57.797 | INFO | High Water Breakdown: | 20260506 02:21:57.797 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:21:57.797 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:21:57.797 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:21:57.797 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:21:57.797 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:21:57.797 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:21:57.797 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 02:21:57.797 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 02:21:57.797 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:21:57.797 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 02:21:57.797 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 02:21:57.797 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 02:21:57.797 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 02:21:57.797 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 02:21:57.797 | INFO | ------------------------------------------ | 20260506 02:21:57.797 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:21:57.797 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 02:21:57.797 | INFO | Total mem usage: 25.6 GiB | 20260506 02:21:57.798 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 02:21:57.798 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 02:21:57.798 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 02:21:57.798 | INFO | High Water Breakdown: | 20260506 02:21:57.798 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 02:21:57.798 | INFO | ------------------------------------------ | 20260506 02:21:57.798 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:21:57.798 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 02:21:57.798 | INFO | Total mem usage: 15.7 GiB | 20260506 02:21:57.798 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 02:21:57.798 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 02:21:57.798 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 02:21:57.798 | INFO | High Water Breakdown: | 20260506 02:21:57.798 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 02:21:57.798 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 02:21:57.798 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 02:21:57.798 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 02:21:57.798 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 02:21:57.798 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 02:21:57.798 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 02:21:57.798 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 02:21:57.798 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 02:21:57.798 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:21:57.798 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 02:21:57.798 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:21:57.798 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 02:21:57.798 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 02:21:57.798 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 02:21:57.798 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 02:21:57.798 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:21:57.798 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 02:21:57.798 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:21:57.798 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:21:57.798 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:21:57.798 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 02:21:57.798 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 02:21:57.798 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:21:57.798 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 02:21:57.798 | INFO | ------------------------------------------ | 20260506 02:21:57.798 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:21:57.798 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 02:21:57.798 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:21:57.798 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:21:57.798 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.39e+01, 1.33e+01 / 8.11e-01 | 20260506 02:21:57.798 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.04e-01, 1.03e-02 / 3.14e-03 | 20260506 02:21:57.798 | INFO | PerfReport: 11.85%, Baseline -- 3.71e+00 / 4.48e+00, 5.78e-01 / 2.30e-01 | 20260506 02:21:57.798 | INFO | PerfReport: 3.30%, Histogram -- 1.03e+00 / 1.52e+00, 1.94e-01 / 9.61e-02 | 20260506 02:21:57.798 | INFO | PerfReport: 15.96%, DME -- 4.00e+01 / 4.39e+01, 3.87e+00 / 1.01e+00 | 20260506 02:21:57.798 | INFO | PerfReport: 57.58%, FrameLabeling -- 1.80e+01 / 1.97e+01, 1.60e+00 / 1.34e+00 | 20260506 02:21:57.798 | INFO | PerfReport: 6.98%, PulseAccumulating -- 2.19e+00 / 2.67e+00, 4.09e-01 / 2.72e-01 | 20260506 02:21:57.798 | INFO | PerfReport: 3.00%, Metrics -- 9.40e-01 / 1.22e+00, 2.48e-01 / 1.12e-01 | 20260506 02:21:57.798 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.26e-01 / 3.76e-01, 5.41e-02 / 2.83e-02 | 20260506 02:21:57.798 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 02:21:57.798 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:21:57.798 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:21:57.798 | INFO | PerfReport: Full Execution -- 9.78e+01 / -nan, 2.42e+01 / 0.00e+00 | 20260506 02:21:57.798 | INFO | PerfReport: 22.24%, Upload -- 2.17e+01 / 0.00e+00, 7.32e+00 / 0.00e+00 | 20260506 02:21:57.798 | INFO | PerfReport: 21.94%, Download -- 2.15e+01 / 0.00e+00, 7.54e+00 / 0.00e+00 | 20260506 02:21:57.798 | INFO | PerfReport: 32.04%, Compute -- 3.13e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 02:21:57.798 | INFO | PerfReport: 2.04%, LockUpload -- 1.99e+00 / 0.00e+00, 7.90e+00 / 0.00e+00 | 20260506 02:21:57.798 | INFO | PerfReport: 1.28%, LockDownload -- 1.25e+00 / 0.00e+00, 2.90e+00 / 0.00e+00 | 20260506 02:21:57.798 | INFO | PerfReport: 20.45%, LockCompute -- 2.00e+01 / 0.00e+00, 1.76e+01 / 0.00e+00 | 20260506 02:21:57.798 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.50e-02 / 0.00e+00 | 20260506 02:21:57.922 | INFO | Starting a round of Baz IO | 20260506 02:22:01.046 | INFO | Populating RTMetrics report | 20260506 02:22:01.047 | INFO | RTMetrics report populated | 20260506 02:22:01.047 | INFO | Opening tmp RTMetrics json file | 20260506 02:22:01.048 | INFO | Serializing RTMetrics to json | 20260506 02:22:01.049 | INFO | Serialization done, swapping tmp file now... | 20260506 02:22:01.049 | INFO | RTMetrics done | 20260506 02:22:01.119 | INFO | Finished a round of Baz IO: Wrote 1.796 GiB in 3.197 seconds | 20260506 02:22:01.119 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.27% / 0.00% / 78.73% | 20260506 02:22:01.120 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0710368% / 3.97364e-06% / 9.13938e-05% | 20260506 02:22:01.120 | INFO | Baz IO overall: | 20260506 02:22:01.120 | INFO | BytesWritten : 4592557271683 | 20260506 02:22:01.120 | INFO | NumEvents : 2156349822906 | 20260506 02:22:02.935 | INFO | Analyzing chunk frames = [8192000,8192512) | 20260506 02:22:02.935 | INFO | MemInfo: { | 20260506 02:22:02.935 | INFO | "gpu" : " 17 GB", | 20260506 02:22:02.935 | INFO | "huge" : "", | 20260506 02:22:02.935 | INFO | "huge_shared" : " 27 GB", | 20260506 02:22:02.935 | INFO | "malloc" : " 14 GB", | 20260506 02:22:02.935 | INFO | "pinned" : " 29 GB" | 20260506 02:22:02.935 | INFO | } | 20260506 02:22:02.935 | INFO | Perf report for the last 5.13823 seconds: | 20260506 02:22:02.935 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:22:02.935 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:02.935 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:02.935 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:02.935 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260506 02:22:02.935 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:02.935 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260506 02:22:02.935 | INFO | PRE_HQ : 59.2%, 1.7, 53.7ms, 0 | 20260506 02:22:02.935 | INFO | BAZWRITER : 61.4%, 3.4, 107.6ms, 0 | 20260506 02:22:02.935 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:22:02.935 | INFO | Uploaded 36GiB to GPU in 2.12702s (16.9251GiB/s) | 20260506 02:22:02.935 | INFO | Downloaded 38GiB to GPU in 2.22856s (17.0514GiB/s) | 20260506 02:22:08.069 | INFO | Analyzing chunk frames = [8192512,8193024) | 20260506 02:22:08.069 | INFO | MemInfo: { | 20260506 02:22:08.069 | INFO | "gpu" : " 17 GB", | 20260506 02:22:08.069 | INFO | "huge" : "", | 20260506 02:22:08.069 | INFO | "huge_shared" : " 27 GB", | 20260506 02:22:08.069 | INFO | "malloc" : " 14 GB", | 20260506 02:22:08.069 | INFO | "pinned" : " 29 GB" | 20260506 02:22:08.069 | INFO | } | 20260506 02:22:08.069 | INFO | Perf report for the last 5.13404 seconds: | 20260506 02:22:08.069 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:22:08.069 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:08.069 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:08.069 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:08.069 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 02:22:08.069 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:08.069 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:08.069 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 02:22:08.069 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:08.069 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:08.069 | INFO | Uploaded 36GiB to GPU in 1.93472s (18.6074GiB/s) | 20260506 02:22:08.069 | INFO | Downloaded 34GiB to GPU in 1.97344s (17.2288GiB/s) | 20260506 02:22:13.204 | INFO | Analyzing chunk frames = [8193024,8193536) | 20260506 02:22:13.204 | INFO | MemInfo: { | 20260506 02:22:13.204 | INFO | "gpu" : " 17 GB", | 20260506 02:22:13.204 | INFO | "huge" : "", | 20260506 02:22:13.204 | INFO | "huge_shared" : " 27 GB", | 20260506 02:22:13.204 | INFO | "malloc" : " 14 GB", | 20260506 02:22:13.204 | INFO | "pinned" : " 29 GB" | 20260506 02:22:13.204 | INFO | } | 20260506 02:22:13.205 | INFO | Perf report for the last 5.13569 seconds: | 20260506 02:22:13.205 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:22:13.205 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:13.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:13.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:13.205 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 02:22:13.205 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:13.205 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:13.205 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 02:22:13.205 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:13.205 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:13.205 | INFO | Uploaded 36GiB to GPU in 2.08601s (17.2578GiB/s) | 20260506 02:22:13.205 | INFO | Downloaded 34GiB to GPU in 2.02462s (16.7933GiB/s) | 20260506 02:22:18.343 | INFO | Analyzing chunk frames = [8193536,8194048) | 20260506 02:22:18.343 | INFO | MemInfo: { | 20260506 02:22:18.343 | INFO | "gpu" : " 17 GB", | 20260506 02:22:18.343 | INFO | "huge" : "", | 20260506 02:22:18.343 | INFO | "huge_shared" : " 27 GB", | 20260506 02:22:18.343 | INFO | "malloc" : " 14 GB", | 20260506 02:22:18.343 | INFO | "pinned" : " 29 GB" | 20260506 02:22:18.343 | INFO | } | 20260506 02:22:18.343 | INFO | Perf report for the last 5.13887 seconds: | 20260506 02:22:18.343 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:22:18.343 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:18.343 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:18.343 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:18.343 | INFO | ANALYSIS : 61.5%, 3.0, 95.5ms, 0 | 20260506 02:22:18.344 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:18.344 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:18.344 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 02:22:18.344 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:18.344 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:18.344 | INFO | Uploaded 36GiB to GPU in 2.13511s (16.861GiB/s) | 20260506 02:22:18.344 | INFO | Downloaded 34GiB to GPU in 2.01552s (16.8691GiB/s) | 20260506 02:22:23.488 | INFO | Analyzing chunk frames = [8194048,8194560) | 20260506 02:22:23.488 | INFO | MemInfo: { | 20260506 02:22:23.488 | INFO | "gpu" : " 17 GB", | 20260506 02:22:23.488 | INFO | "huge" : "", | 20260506 02:22:23.488 | INFO | "huge_shared" : " 27 GB", | 20260506 02:22:23.488 | INFO | "malloc" : " 14 GB", | 20260506 02:22:23.488 | INFO | "pinned" : " 29 GB" | 20260506 02:22:23.488 | INFO | } | 20260506 02:22:23.488 | INFO | Perf report for the last 5.14489 seconds: | 20260506 02:22:23.488 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:22:23.488 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:23.489 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:23.489 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:23.489 | INFO | ANALYSIS : 60.5%, 3.0, 94.6ms, 0 | 20260506 02:22:23.489 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:23.489 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:23.489 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 02:22:23.489 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:23.489 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:23.489 | INFO | Uploaded 36GiB to GPU in 2.07473s (17.3517GiB/s) | 20260506 02:22:23.489 | INFO | Downloaded 34GiB to GPU in 2.02929s (16.7546GiB/s) | 20260506 02:22:28.634 | INFO | Analyzing chunk frames = [8194560,8195072) | 20260506 02:22:28.634 | INFO | MemInfo: { | 20260506 02:22:28.634 | INFO | "gpu" : " 17 GB", | 20260506 02:22:28.634 | INFO | "huge" : "", | 20260506 02:22:28.634 | INFO | "huge_shared" : " 27 GB", | 20260506 02:22:28.634 | INFO | "malloc" : " 14 GB", | 20260506 02:22:28.634 | INFO | "pinned" : " 29 GB" | 20260506 02:22:28.634 | INFO | } | 20260506 02:22:28.634 | INFO | Perf report for the last 5.1457 seconds: | 20260506 02:22:28.634 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:22:28.634 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:28.634 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:28.634 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:28.634 | INFO | ANALYSIS : 61.1%, 3.0, 95.5ms, 0 | 20260506 02:22:28.634 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:28.634 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:28.634 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:22:28.635 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:28.635 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:28.635 | INFO | Uploaded 36GiB to GPU in 2.1079s (17.0786GiB/s) | 20260506 02:22:28.635 | INFO | Downloaded 34GiB to GPU in 2.0412s (16.6569GiB/s) | 20260506 02:22:33.768 | INFO | Analyzing chunk frames = [8195072,8195584) | 20260506 02:22:33.768 | INFO | MemInfo: { | 20260506 02:22:33.768 | INFO | "gpu" : " 17 GB", | 20260506 02:22:33.768 | INFO | "huge" : "", | 20260506 02:22:33.768 | INFO | "huge_shared" : " 27 GB", | 20260506 02:22:33.768 | INFO | "malloc" : " 14 GB", | 20260506 02:22:33.768 | INFO | "pinned" : " 29 GB" | 20260506 02:22:33.768 | INFO | } | 20260506 02:22:33.768 | INFO | Perf report for the last 5.13378 seconds: | 20260506 02:22:33.768 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:22:33.768 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:33.768 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:33.768 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:33.768 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260506 02:22:33.768 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:33.768 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:33.768 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 02:22:33.768 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:33.768 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:33.768 | INFO | Uploaded 36GiB to GPU in 2.10296s (17.1187GiB/s) | 20260506 02:22:33.768 | INFO | Downloaded 34GiB to GPU in 2.09227s (16.2503GiB/s) | 20260506 02:22:38.903 | INFO | Analyzing chunk frames = [8195584,8196096) | 20260506 02:22:38.903 | INFO | MemInfo: { | 20260506 02:22:38.903 | INFO | "gpu" : " 17 GB", | 20260506 02:22:38.903 | INFO | "huge" : "", | 20260506 02:22:38.903 | INFO | "huge_shared" : " 27 GB", | 20260506 02:22:38.903 | INFO | "malloc" : " 14 GB", | 20260506 02:22:38.903 | INFO | "pinned" : " 29 GB" | 20260506 02:22:38.903 | INFO | } | 20260506 02:22:38.903 | INFO | Perf report for the last 5.13536 seconds: | 20260506 02:22:38.903 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:22:38.903 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:38.903 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:38.903 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:38.903 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260506 02:22:38.903 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:38.903 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:38.904 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 02:22:38.904 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:38.904 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:38.904 | INFO | Uploaded 36GiB to GPU in 2.13676s (16.8479GiB/s) | 20260506 02:22:38.904 | INFO | Downloaded 34GiB to GPU in 2.02963s (16.7518GiB/s) | 20260506 02:22:39.030 | INFO | Starting a round of Baz IO | 20260506 02:22:42.163 | INFO | Populating RTMetrics report | 20260506 02:22:42.163 | INFO | RTMetrics report populated | 20260506 02:22:42.163 | INFO | Opening tmp RTMetrics json file | 20260506 02:22:42.164 | INFO | Serializing RTMetrics to json | 20260506 02:22:42.165 | INFO | Serialization done, swapping tmp file now... | 20260506 02:22:42.165 | INFO | RTMetrics done | 20260506 02:22:42.239 | INFO | Finished a round of Baz IO: Wrote 1.795 GiB in 3.208 seconds | 20260506 02:22:42.239 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.27% / 0.00% / 78.73% | 20260506 02:22:42.239 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0709136% / 3.97364e-06% / 0.000135104% | 20260506 02:22:42.239 | INFO | Baz IO overall: | 20260506 02:22:42.239 | INFO | BytesWritten : 4594484246098 | 20260506 02:22:42.239 | INFO | NumEvents : 2157225571985 | 20260506 02:22:44.051 | INFO | Analyzing chunk frames = [8196096,8196608) | 20260506 02:22:44.051 | INFO | MemInfo: { | 20260506 02:22:44.051 | INFO | "gpu" : " 17 GB", | 20260506 02:22:44.051 | INFO | "huge" : "", | 20260506 02:22:44.051 | INFO | "huge_shared" : " 27 GB", | 20260506 02:22:44.051 | INFO | "malloc" : " 14 GB", | 20260506 02:22:44.051 | INFO | "pinned" : " 29 GB" | 20260506 02:22:44.052 | INFO | } | 20260506 02:22:44.052 | INFO | Perf report for the last 5.14838 seconds: | 20260506 02:22:44.052 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:22:44.052 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:44.052 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:44.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:44.052 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260506 02:22:44.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:44.052 | INFO | RT_METRICS : 42.6%, 1.0, 22.4ms, 0 | 20260506 02:22:44.052 | INFO | PRE_HQ : 59.4%, 1.7, 53.6ms, 0 | 20260506 02:22:44.052 | INFO | BAZWRITER : 61.6%, 3.7, 117.9ms, 0 | 20260506 02:22:44.052 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:22:44.052 | INFO | Uploaded 36GiB to GPU in 2.19681s (16.3874GiB/s) | 20260506 02:22:44.052 | INFO | Downloaded 38GiB to GPU in 2.24647s (16.9155GiB/s) | 20260506 02:22:49.185 | INFO | Analyzing chunk frames = [8196608,8197120) | 20260506 02:22:49.185 | INFO | MemInfo: { | 20260506 02:22:49.185 | INFO | "gpu" : " 17 GB", | 20260506 02:22:49.185 | INFO | "huge" : "", | 20260506 02:22:49.185 | INFO | "huge_shared" : " 27 GB", | 20260506 02:22:49.185 | INFO | "malloc" : " 14 GB", | 20260506 02:22:49.185 | INFO | "pinned" : " 29 GB" | 20260506 02:22:49.185 | INFO | } | 20260506 02:22:49.185 | INFO | Perf report for the last 5.13387 seconds: | 20260506 02:22:49.185 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:22:49.185 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:49.185 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:49.185 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:49.186 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260506 02:22:49.186 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:49.186 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:49.186 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:22:49.186 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:49.186 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:49.186 | INFO | Uploaded 36GiB to GPU in 1.98722s (18.1158GiB/s) | 20260506 02:22:49.186 | INFO | Downloaded 34GiB to GPU in 1.98276s (17.1478GiB/s) | 20260506 02:22:54.324 | INFO | Analyzing chunk frames = [8197120,8197632) | 20260506 02:22:54.324 | INFO | MemInfo: { | 20260506 02:22:54.324 | INFO | "gpu" : " 17 GB", | 20260506 02:22:54.324 | INFO | "huge" : "", | 20260506 02:22:54.324 | INFO | "huge_shared" : " 27 GB", | 20260506 02:22:54.324 | INFO | "malloc" : " 14 GB", | 20260506 02:22:54.324 | INFO | "pinned" : " 29 GB" | 20260506 02:22:54.324 | INFO | } | 20260506 02:22:54.325 | INFO | Perf report for the last 5.13911 seconds: | 20260506 02:22:54.325 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:22:54.325 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:54.325 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:54.325 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:54.325 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 02:22:54.325 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:54.325 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:54.325 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 02:22:54.325 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:54.325 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:54.325 | INFO | Uploaded 36GiB to GPU in 2.10043s (17.1393GiB/s) | 20260506 02:22:54.325 | INFO | Downloaded 34GiB to GPU in 2.0422s (16.6487GiB/s) | 20260506 02:22:59.462 | INFO | Analyzing chunk frames = [8197632,8198144) | 20260506 02:22:59.462 | INFO | MemInfo: { | 20260506 02:22:59.462 | INFO | "gpu" : " 17 GB", | 20260506 02:22:59.462 | INFO | "huge" : "", | 20260506 02:22:59.462 | INFO | "huge_shared" : " 27 GB", | 20260506 02:22:59.462 | INFO | "malloc" : " 14 GB", | 20260506 02:22:59.462 | INFO | "pinned" : " 29 GB" | 20260506 02:22:59.462 | INFO | } | 20260506 02:22:59.463 | INFO | Perf report for the last 5.13804 seconds: | 20260506 02:22:59.463 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:22:59.463 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:59.463 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:59.463 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:59.463 | INFO | ANALYSIS : 61.5%, 3.0, 95.5ms, 0 | 20260506 02:22:59.463 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:59.463 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:59.463 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 02:22:59.463 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:59.463 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:22:59.463 | INFO | Uploaded 36GiB to GPU in 2.14812s (16.7588GiB/s) | 20260506 02:22:59.463 | INFO | Downloaded 34GiB to GPU in 2.05815s (16.5197GiB/s) | 20260506 02:23:04.606 | INFO | Analyzing chunk frames = [8198144,8198656) | 20260506 02:23:04.606 | INFO | MemInfo: { | 20260506 02:23:04.606 | INFO | "gpu" : " 17 GB", | 20260506 02:23:04.606 | INFO | "huge" : "", | 20260506 02:23:04.606 | INFO | "huge_shared" : " 27 GB", | 20260506 02:23:04.606 | INFO | "malloc" : " 14 GB", | 20260506 02:23:04.606 | INFO | "pinned" : " 29 GB" | 20260506 02:23:04.606 | INFO | } | 20260506 02:23:04.606 | INFO | Perf report for the last 5.14342 seconds: | 20260506 02:23:04.606 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:23:04.606 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:04.606 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:04.606 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:04.606 | INFO | ANALYSIS : 60.7%, 3.0, 94.8ms, 0 | 20260506 02:23:04.606 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:04.606 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:04.606 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 02:23:04.606 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:04.606 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:04.606 | INFO | Uploaded 36GiB to GPU in 2.0809s (17.3002GiB/s) | 20260506 02:23:04.606 | INFO | Downloaded 34GiB to GPU in 2.03301s (16.724GiB/s) | 20260506 02:23:09.745 | INFO | Analyzing chunk frames = [8198656,8199168) | 20260506 02:23:09.745 | INFO | MemInfo: { | 20260506 02:23:09.745 | INFO | "gpu" : " 17 GB", | 20260506 02:23:09.745 | INFO | "huge" : "", | 20260506 02:23:09.745 | INFO | "huge_shared" : " 27 GB", | 20260506 02:23:09.745 | INFO | "malloc" : " 14 GB", | 20260506 02:23:09.745 | INFO | "pinned" : " 29 GB" | 20260506 02:23:09.745 | INFO | } | 20260506 02:23:09.745 | INFO | Perf report for the last 5.1394 seconds: | 20260506 02:23:09.746 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:23:09.746 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:09.746 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:09.746 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:09.746 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 02:23:09.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:09.746 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:09.746 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 02:23:09.746 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:09.746 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:09.746 | INFO | Uploaded 36GiB to GPU in 2.09795s (17.1596GiB/s) | 20260506 02:23:09.746 | INFO | Downloaded 34GiB to GPU in 2.03802s (16.6828GiB/s) | 20260506 02:23:14.881 | INFO | Analyzing chunk frames = [8199168,8199680) | 20260506 02:23:14.881 | INFO | MemInfo: { | 20260506 02:23:14.881 | INFO | "gpu" : " 17 GB", | 20260506 02:23:14.881 | INFO | "huge" : "", | 20260506 02:23:14.881 | INFO | "huge_shared" : " 27 GB", | 20260506 02:23:14.881 | INFO | "malloc" : " 14 GB", | 20260506 02:23:14.881 | INFO | "pinned" : " 29 GB" | 20260506 02:23:14.881 | INFO | } | 20260506 02:23:14.881 | INFO | Perf report for the last 5.13567 seconds: | 20260506 02:23:14.881 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:23:14.881 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:14.881 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:14.881 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:14.881 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 02:23:14.881 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:14.881 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:14.881 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 02:23:14.881 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:14.881 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:14.881 | INFO | Uploaded 36GiB to GPU in 2.10344s (17.1148GiB/s) | 20260506 02:23:14.881 | INFO | Downloaded 34GiB to GPU in 2.07486s (16.3866GiB/s) | 20260506 02:23:20.019 | INFO | Analyzing chunk frames = [8199680,8200192) | 20260506 02:23:20.019 | INFO | MemInfo: { | 20260506 02:23:20.019 | INFO | "gpu" : " 17 GB", | 20260506 02:23:20.019 | INFO | "huge" : "", | 20260506 02:23:20.019 | INFO | "huge_shared" : " 27 GB", | 20260506 02:23:20.019 | INFO | "malloc" : " 14 GB", | 20260506 02:23:20.019 | INFO | "pinned" : " 29 GB" | 20260506 02:23:20.019 | INFO | } | 20260506 02:23:20.019 | INFO | Perf report for the last 5.13813 seconds: | 20260506 02:23:20.019 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:23:20.019 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:20.019 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:20.019 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:20.019 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260506 02:23:20.019 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:20.019 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:20.019 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 02:23:20.019 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:20.019 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:20.020 | INFO | Uploaded 36GiB to GPU in 2.06024s (17.4737GiB/s) | 20260506 02:23:20.020 | INFO | Downloaded 34GiB to GPU in 2.02241s (16.8116GiB/s) | 20260506 02:23:20.020 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 02:23:20.020 | INFO | Total mem usage: 15.3 GiB | 20260506 02:23:20.020 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 02:23:20.020 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -5.84%) | 20260506 02:23:20.020 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 02:23:20.020 | INFO | High Water Breakdown: | 20260506 02:23:20.020 | INFO | 61.4% : 8.89 GiB ( 93.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 02:23:20.020 | INFO | 38.5% : 5.57 GiB ( 83.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 02:23:20.020 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 02:23:20.020 | INFO | ------------------------------------------ | 20260506 02:23:20.020 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:23:20.020 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 02:23:20.020 | INFO | Total mem usage: 26.5 GiB | 20260506 02:23:20.020 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 02:23:20.020 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 02:23:20.020 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 02:23:20.020 | INFO | High Water Breakdown: | 20260506 02:23:20.020 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:23:20.020 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:23:20.020 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:23:20.020 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:23:20.020 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:23:20.020 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:23:20.020 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 02:23:20.020 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 02:23:20.020 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:23:20.020 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 02:23:20.020 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 02:23:20.020 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 02:23:20.020 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 02:23:20.020 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 02:23:20.020 | INFO | ------------------------------------------ | 20260506 02:23:20.020 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:23:20.020 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 02:23:20.020 | INFO | Total mem usage: 25.6 GiB | 20260506 02:23:20.020 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 02:23:20.020 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 02:23:20.020 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 02:23:20.020 | INFO | High Water Breakdown: | 20260506 02:23:20.020 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 02:23:20.020 | INFO | ------------------------------------------ | 20260506 02:23:20.020 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:23:20.020 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 02:23:20.020 | INFO | Total mem usage: 15.7 GiB | 20260506 02:23:20.020 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 02:23:20.020 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 02:23:20.021 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 02:23:20.021 | INFO | High Water Breakdown: | 20260506 02:23:20.021 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 02:23:20.021 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 02:23:20.021 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 02:23:20.021 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 02:23:20.021 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 02:23:20.021 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 02:23:20.021 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 02:23:20.021 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 02:23:20.021 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 02:23:20.021 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:23:20.021 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 02:23:20.021 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:23:20.021 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 02:23:20.021 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 02:23:20.021 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 02:23:20.021 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 02:23:20.021 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:23:20.021 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 02:23:20.021 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:23:20.021 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:23:20.021 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:23:20.021 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 02:23:20.021 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 02:23:20.021 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:23:20.021 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 02:23:20.021 | INFO | ------------------------------------------ | 20260506 02:23:20.021 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:23:20.021 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 02:23:20.021 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:23:20.021 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:23:20.021 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.36e+01, 1.35e+01 / 6.36e-01 | 20260506 02:23:20.021 | INFO | PerfReport: 0.29%, AggregateStats -- 9.18e-02 / 1.04e-01, 1.05e-02 / 2.86e-03 | 20260506 02:23:20.021 | INFO | PerfReport: 11.85%, Baseline -- 3.70e+00 / 4.48e+00, 5.73e-01 / 2.27e-01 | 20260506 02:23:20.021 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.53e+00, 1.91e-01 / 7.71e-02 | 20260506 02:23:20.021 | INFO | PerfReport: 15.98%, DME -- 3.99e+01 / 4.38e+01, 3.76e+00 / 8.77e-01 | 20260506 02:23:20.021 | INFO | PerfReport: 57.49%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.50e+00 / 1.00e+00 | 20260506 02:23:20.021 | INFO | PerfReport: 7.04%, PulseAccumulating -- 2.20e+00 / 2.68e+00, 4.27e-01 / 2.18e-01 | 20260506 02:23:20.021 | INFO | PerfReport: 3.02%, Metrics -- 9.42e-01 / 1.21e+00, 2.53e-01 / 1.13e-01 | 20260506 02:23:20.021 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.25e-01 / 3.75e-01, 5.43e-02 / 2.51e-02 | 20260506 02:23:20.021 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 02:23:20.021 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:23:20.021 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:23:20.021 | INFO | PerfReport: Full Execution -- 9.77e+01 / -nan, 2.42e+01 / 0.00e+00 | 20260506 02:23:20.021 | INFO | PerfReport: 22.30%, Upload -- 2.18e+01 / 0.00e+00, 7.33e+00 / 0.00e+00 | 20260506 02:23:20.021 | INFO | PerfReport: 21.94%, Download -- 2.14e+01 / 0.00e+00, 7.56e+00 / 0.00e+00 | 20260506 02:23:20.021 | INFO | PerfReport: 31.97%, Compute -- 3.12e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 02:23:20.021 | INFO | PerfReport: 2.04%, LockUpload -- 2.00e+00 / 0.00e+00, 7.76e+00 / 0.00e+00 | 20260506 02:23:20.021 | INFO | PerfReport: 1.29%, LockDownload -- 1.26e+00 / 0.00e+00, 2.90e+00 / 0.00e+00 | 20260506 02:23:20.021 | INFO | PerfReport: 20.43%, LockCompute -- 2.00e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 02:23:20.021 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.39e-02 / 0.00e+00 | 20260506 02:23:20.144 | INFO | Starting a round of Baz IO | 20260506 02:23:23.278 | INFO | Populating RTMetrics report | 20260506 02:23:23.278 | INFO | RTMetrics report populated | 20260506 02:23:23.278 | INFO | Opening tmp RTMetrics json file | 20260506 02:23:23.279 | INFO | Serializing RTMetrics to json | 20260506 02:23:23.280 | INFO | Serialization done, swapping tmp file now... | 20260506 02:23:23.281 | INFO | RTMetrics done | 20260506 02:23:23.354 | INFO | Finished a round of Baz IO: Wrote 1.795 GiB in 3.21 seconds | 20260506 02:23:23.354 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.27% / 0.00% / 78.73% | 20260506 02:23:23.354 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0711481% / 3.97364e-06% / 0.000107288% | 20260506 02:23:23.354 | INFO | Baz IO overall: | 20260506 02:23:23.354 | INFO | BytesWritten : 4596411295097 | 20260506 02:23:23.354 | INFO | NumEvents : 2158101626994 | 20260506 02:23:25.158 | INFO | Analyzing chunk frames = [8200192,8200704) | 20260506 02:23:25.158 | INFO | MemInfo: { | 20260506 02:23:25.158 | INFO | "gpu" : " 17 GB", | 20260506 02:23:25.159 | INFO | "huge" : "", | 20260506 02:23:25.159 | INFO | "huge_shared" : " 27 GB", | 20260506 02:23:25.159 | INFO | "malloc" : " 14 GB", | 20260506 02:23:25.159 | INFO | "pinned" : " 29 GB" | 20260506 02:23:25.159 | INFO | } | 20260506 02:23:25.159 | INFO | Perf report for the last 5.13938 seconds: | 20260506 02:23:25.159 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:23:25.159 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:25.159 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:25.159 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:25.159 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260506 02:23:25.159 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:25.159 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260506 02:23:25.159 | INFO | PRE_HQ : 59.7%, 1.7, 53.5ms, 0 | 20260506 02:23:25.159 | INFO | BAZWRITER : 61.7%, 3.4, 108.3ms, 0 | 20260506 02:23:25.159 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:23:25.159 | INFO | Uploaded 36GiB to GPU in 2.18765s (16.456GiB/s) | 20260506 02:23:25.159 | INFO | Downloaded 38GiB to GPU in 2.24158s (16.9523GiB/s) | 20260506 02:23:30.297 | INFO | Analyzing chunk frames = [8200704,8201216) | 20260506 02:23:30.297 | INFO | MemInfo: { | 20260506 02:23:30.297 | INFO | "gpu" : " 17 GB", | 20260506 02:23:30.297 | INFO | "huge" : "", | 20260506 02:23:30.297 | INFO | "huge_shared" : " 27 GB", | 20260506 02:23:30.297 | INFO | "malloc" : " 14 GB", | 20260506 02:23:30.297 | INFO | "pinned" : " 29 GB" | 20260506 02:23:30.297 | INFO | } | 20260506 02:23:30.297 | INFO | Perf report for the last 5.13867 seconds: | 20260506 02:23:30.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:23:30.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:30.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:30.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:30.297 | INFO | ANALYSIS : 60.9%, 3.0, 95.1ms, 0 | 20260506 02:23:30.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:30.297 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:30.297 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 02:23:30.297 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:30.298 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:30.298 | INFO | Uploaded 36GiB to GPU in 1.98914s (18.0983GiB/s) | 20260506 02:23:30.298 | INFO | Downloaded 34GiB to GPU in 1.99095s (17.0773GiB/s) | 20260506 02:23:35.443 | INFO | Analyzing chunk frames = [8201216,8201728) | 20260506 02:23:35.443 | INFO | MemInfo: { | 20260506 02:23:35.443 | INFO | "gpu" : " 17 GB", | 20260506 02:23:35.443 | INFO | "huge" : "", | 20260506 02:23:35.443 | INFO | "huge_shared" : " 27 GB", | 20260506 02:23:35.443 | INFO | "malloc" : " 14 GB", | 20260506 02:23:35.443 | INFO | "pinned" : " 29 GB" | 20260506 02:23:35.443 | INFO | } | 20260506 02:23:35.443 | INFO | Perf report for the last 5.14584 seconds: | 20260506 02:23:35.443 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:23:35.443 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:35.443 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:35.443 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:35.443 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 02:23:35.443 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:35.443 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:35.443 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 02:23:35.443 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:35.443 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:35.444 | INFO | Uploaded 36GiB to GPU in 2.09788s (17.1602GiB/s) | 20260506 02:23:35.444 | INFO | Downloaded 34GiB to GPU in 2.04023s (16.6648GiB/s) | 20260506 02:23:40.582 | INFO | Analyzing chunk frames = [8201728,8202240) | 20260506 02:23:40.582 | INFO | MemInfo: { | 20260506 02:23:40.582 | INFO | "gpu" : " 17 GB", | 20260506 02:23:40.582 | INFO | "huge" : "", | 20260506 02:23:40.582 | INFO | "huge_shared" : " 27 GB", | 20260506 02:23:40.582 | INFO | "malloc" : " 14 GB", | 20260506 02:23:40.582 | INFO | "pinned" : " 29 GB" | 20260506 02:23:40.583 | INFO | } | 20260506 02:23:40.583 | INFO | Perf report for the last 5.13939 seconds: | 20260506 02:23:40.583 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:23:40.583 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:40.583 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:40.583 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:40.583 | INFO | ANALYSIS : 61.1%, 3.0, 94.9ms, 0 | 20260506 02:23:40.583 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:40.583 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:40.583 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 02:23:40.583 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:40.583 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:40.583 | INFO | Uploaded 36GiB to GPU in 2.07823s (17.3225GiB/s) | 20260506 02:23:40.583 | INFO | Downloaded 34GiB to GPU in 2.01226s (16.8965GiB/s) | 20260506 02:23:45.714 | INFO | Analyzing chunk frames = [8202240,8202752) | 20260506 02:23:45.714 | INFO | MemInfo: { | 20260506 02:23:45.714 | INFO | "gpu" : " 17 GB", | 20260506 02:23:45.714 | INFO | "huge" : "", | 20260506 02:23:45.714 | INFO | "huge_shared" : " 27 GB", | 20260506 02:23:45.714 | INFO | "malloc" : " 14 GB", | 20260506 02:23:45.714 | INFO | "pinned" : " 29 GB" | 20260506 02:23:45.714 | INFO | } | 20260506 02:23:45.715 | INFO | Perf report for the last 5.13188 seconds: | 20260506 02:23:45.715 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:23:45.715 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:45.715 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:45.715 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:45.715 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260506 02:23:45.715 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:45.715 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:45.715 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:23:45.715 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:45.715 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:45.715 | INFO | Uploaded 36GiB to GPU in 2.11127s (17.0514GiB/s) | 20260506 02:23:45.715 | INFO | Downloaded 34GiB to GPU in 2.03244s (16.7286GiB/s) | 20260506 02:23:50.855 | INFO | Analyzing chunk frames = [8202752,8203264) | 20260506 02:23:50.855 | INFO | MemInfo: { | 20260506 02:23:50.855 | INFO | "gpu" : " 17 GB", | 20260506 02:23:50.855 | INFO | "huge" : "", | 20260506 02:23:50.855 | INFO | "huge_shared" : " 27 GB", | 20260506 02:23:50.855 | INFO | "malloc" : " 14 GB", | 20260506 02:23:50.855 | INFO | "pinned" : " 29 GB" | 20260506 02:23:50.855 | INFO | } | 20260506 02:23:50.855 | INFO | Perf report for the last 5.14095 seconds: | 20260506 02:23:50.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:23:50.856 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:50.856 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:50.856 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:50.856 | INFO | ANALYSIS : 61.1%, 3.0, 95.5ms, 0 | 20260506 02:23:50.856 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:50.856 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:50.856 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:23:50.856 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:50.856 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:50.856 | INFO | Uploaded 36GiB to GPU in 2.07198s (17.3747GiB/s) | 20260506 02:23:50.856 | INFO | Downloaded 34GiB to GPU in 2.04284s (16.6435GiB/s) | 20260506 02:23:56.003 | INFO | Analyzing chunk frames = [8203264,8203776) | 20260506 02:23:56.003 | INFO | MemInfo: { | 20260506 02:23:56.003 | INFO | "gpu" : " 17 GB", | 20260506 02:23:56.003 | INFO | "huge" : "", | 20260506 02:23:56.003 | INFO | "huge_shared" : " 27 GB", | 20260506 02:23:56.003 | INFO | "malloc" : " 14 GB", | 20260506 02:23:56.003 | INFO | "pinned" : " 29 GB" | 20260506 02:23:56.003 | INFO | } | 20260506 02:23:56.003 | INFO | Perf report for the last 5.14796 seconds: | 20260506 02:23:56.003 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:23:56.003 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:56.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:56.004 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:56.004 | INFO | ANALYSIS : 61.8%, 3.0, 96.7ms, 0 | 20260506 02:23:56.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:56.004 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:56.004 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 02:23:56.004 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:56.004 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:23:56.004 | INFO | Uploaded 36GiB to GPU in 2.06672s (17.4189GiB/s) | 20260506 02:23:56.004 | INFO | Downloaded 34GiB to GPU in 2.0689s (16.4338GiB/s) | 20260506 02:24:01.139 | INFO | Analyzing chunk frames = [8203776,8204288) | 20260506 02:24:01.139 | INFO | MemInfo: { | 20260506 02:24:01.139 | INFO | "gpu" : " 17 GB", | 20260506 02:24:01.139 | INFO | "huge" : "", | 20260506 02:24:01.139 | INFO | "huge_shared" : " 27 GB", | 20260506 02:24:01.139 | INFO | "malloc" : " 14 GB", | 20260506 02:24:01.139 | INFO | "pinned" : " 29 GB" | 20260506 02:24:01.139 | INFO | } | 20260506 02:24:01.139 | INFO | Perf report for the last 5.13537 seconds: | 20260506 02:24:01.139 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:24:01.139 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:01.139 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:01.139 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:01.139 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 02:24:01.139 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:01.139 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:01.139 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 02:24:01.139 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:01.139 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:01.139 | INFO | Uploaded 36GiB to GPU in 2.0376s (17.6679GiB/s) | 20260506 02:24:01.139 | INFO | Downloaded 34GiB to GPU in 2.02681s (16.7752GiB/s) | 20260506 02:24:01.264 | INFO | Starting a round of Baz IO | 20260506 02:24:04.396 | INFO | Populating RTMetrics report | 20260506 02:24:04.396 | INFO | RTMetrics report populated | 20260506 02:24:04.396 | INFO | Opening tmp RTMetrics json file | 20260506 02:24:04.396 | INFO | Serializing RTMetrics to json | 20260506 02:24:04.398 | INFO | Serialization done, swapping tmp file now... | 20260506 02:24:04.398 | INFO | RTMetrics done | 20260506 02:24:04.467 | INFO | Finished a round of Baz IO: Wrote 1.793 GiB in 3.203 seconds | 20260506 02:24:04.467 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.27% / 0.00% / 78.73% | 20260506 02:24:04.468 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0706156% / 0% / 0.000107288% | 20260506 02:24:04.468 | INFO | Baz IO overall: | 20260506 02:24:04.468 | INFO | BytesWritten : 4598336814585 | 20260506 02:24:04.468 | INFO | NumEvents : 2158976532591 | 20260506 02:24:06.276 | INFO | Analyzing chunk frames = [8204288,8204800) | 20260506 02:24:06.276 | INFO | MemInfo: { | 20260506 02:24:06.276 | INFO | "gpu" : " 17 GB", | 20260506 02:24:06.276 | INFO | "huge" : "", | 20260506 02:24:06.276 | INFO | "huge_shared" : " 27 GB", | 20260506 02:24:06.276 | INFO | "malloc" : " 14 GB", | 20260506 02:24:06.276 | INFO | "pinned" : " 29 GB" | 20260506 02:24:06.276 | INFO | } | 20260506 02:24:06.276 | INFO | Perf report for the last 5.13738 seconds: | 20260506 02:24:06.276 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:24:06.276 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:06.276 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:06.276 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:06.276 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260506 02:24:06.276 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:06.276 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260506 02:24:06.276 | INFO | PRE_HQ : 59.1%, 1.7, 53.4ms, 0 | 20260506 02:24:06.276 | INFO | BAZWRITER : 61.5%, 3.4, 107.0ms, 0 | 20260506 02:24:06.276 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:24:06.277 | INFO | Uploaded 36GiB to GPU in 2.13015s (16.9002GiB/s) | 20260506 02:24:06.277 | INFO | Downloaded 38GiB to GPU in 2.23309s (17.0168GiB/s) | 20260506 02:24:11.414 | INFO | Analyzing chunk frames = [8204800,8205312) | 20260506 02:24:11.414 | INFO | MemInfo: { | 20260506 02:24:11.414 | INFO | "gpu" : " 17 GB", | 20260506 02:24:11.414 | INFO | "huge" : "", | 20260506 02:24:11.414 | INFO | "huge_shared" : " 27 GB", | 20260506 02:24:11.414 | INFO | "malloc" : " 14 GB", | 20260506 02:24:11.414 | INFO | "pinned" : " 29 GB" | 20260506 02:24:11.414 | INFO | } | 20260506 02:24:11.414 | INFO | Perf report for the last 5.13749 seconds: | 20260506 02:24:11.414 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:24:11.414 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:11.414 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:11.414 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:11.414 | INFO | ANALYSIS : 60.9%, 3.0, 94.2ms, 0 | 20260506 02:24:11.414 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:11.414 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:11.414 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 02:24:11.414 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:11.414 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:11.414 | INFO | Uploaded 36GiB to GPU in 1.9641s (18.329GiB/s) | 20260506 02:24:11.414 | INFO | Downloaded 34GiB to GPU in 1.98479s (17.1303GiB/s) | 20260506 02:24:16.552 | INFO | Analyzing chunk frames = [8205312,8205824) | 20260506 02:24:16.552 | INFO | MemInfo: { | 20260506 02:24:16.552 | INFO | "gpu" : " 17 GB", | 20260506 02:24:16.552 | INFO | "huge" : "", | 20260506 02:24:16.552 | INFO | "huge_shared" : " 27 GB", | 20260506 02:24:16.552 | INFO | "malloc" : " 14 GB", | 20260506 02:24:16.552 | INFO | "pinned" : " 29 GB" | 20260506 02:24:16.552 | INFO | } | 20260506 02:24:16.553 | INFO | Perf report for the last 5.13888 seconds: | 20260506 02:24:16.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:24:16.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:16.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:16.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:16.553 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 02:24:16.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:16.553 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:16.553 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:24:16.553 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:16.553 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:16.553 | INFO | Uploaded 36GiB to GPU in 2.09398s (17.1921GiB/s) | 20260506 02:24:16.553 | INFO | Downloaded 34GiB to GPU in 2.0246s (16.7934GiB/s) | 20260506 02:24:21.696 | INFO | Analyzing chunk frames = [8205824,8206336) | 20260506 02:24:21.696 | INFO | MemInfo: { | 20260506 02:24:21.696 | INFO | "gpu" : " 17 GB", | 20260506 02:24:21.696 | INFO | "huge" : "", | 20260506 02:24:21.696 | INFO | "huge_shared" : " 27 GB", | 20260506 02:24:21.696 | INFO | "malloc" : " 14 GB", | 20260506 02:24:21.696 | INFO | "pinned" : " 29 GB" | 20260506 02:24:21.696 | INFO | } | 20260506 02:24:21.696 | INFO | Perf report for the last 5.14356 seconds: | 20260506 02:24:21.696 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:24:21.696 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:21.696 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:21.696 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:21.696 | INFO | ANALYSIS : 61.0%, 3.0, 94.7ms, 0 | 20260506 02:24:21.696 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:21.696 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:21.696 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:24:21.696 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:21.696 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:21.697 | INFO | Uploaded 36GiB to GPU in 2.07476s (17.3514GiB/s) | 20260506 02:24:21.697 | INFO | Downloaded 34GiB to GPU in 2.01621s (16.8633GiB/s) | 20260506 02:24:26.831 | INFO | Analyzing chunk frames = [8206336,8206848) | 20260506 02:24:26.831 | INFO | MemInfo: { | 20260506 02:24:26.831 | INFO | "gpu" : " 17 GB", | 20260506 02:24:26.831 | INFO | "huge" : "", | 20260506 02:24:26.831 | INFO | "huge_shared" : " 27 GB", | 20260506 02:24:26.831 | INFO | "malloc" : " 14 GB", | 20260506 02:24:26.831 | INFO | "pinned" : " 29 GB" | 20260506 02:24:26.831 | INFO | } | 20260506 02:24:26.831 | INFO | Perf report for the last 5.13489 seconds: | 20260506 02:24:26.831 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:24:26.831 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:26.831 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:26.831 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:26.831 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260506 02:24:26.831 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:26.831 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:26.831 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 02:24:26.831 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:26.831 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:26.831 | INFO | Uploaded 36GiB to GPU in 2.08502s (17.2661GiB/s) | 20260506 02:24:26.831 | INFO | Downloaded 34GiB to GPU in 2.04089s (16.6594GiB/s) | 20260506 02:24:31.975 | INFO | Analyzing chunk frames = [8206848,8207360) | 20260506 02:24:31.975 | INFO | MemInfo: { | 20260506 02:24:31.975 | INFO | "gpu" : " 17 GB", | 20260506 02:24:31.975 | INFO | "huge" : "", | 20260506 02:24:31.975 | INFO | "huge_shared" : " 27 GB", | 20260506 02:24:31.975 | INFO | "malloc" : " 14 GB", | 20260506 02:24:31.975 | INFO | "pinned" : " 29 GB" | 20260506 02:24:31.975 | INFO | } | 20260506 02:24:31.975 | INFO | Perf report for the last 5.14374 seconds: | 20260506 02:24:31.975 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:24:31.975 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:31.975 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:31.975 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:31.975 | INFO | ANALYSIS : 60.9%, 3.0, 95.2ms, 0 | 20260506 02:24:31.975 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:31.975 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:31.975 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:24:31.975 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:31.975 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:31.975 | INFO | Uploaded 36GiB to GPU in 2.0708s (17.3846GiB/s) | 20260506 02:24:31.975 | INFO | Downloaded 34GiB to GPU in 2.036s (16.6994GiB/s) | 20260506 02:24:37.115 | INFO | Analyzing chunk frames = [8207360,8207872) | 20260506 02:24:37.115 | INFO | MemInfo: { | 20260506 02:24:37.115 | INFO | "gpu" : " 17 GB", | 20260506 02:24:37.115 | INFO | "huge" : "", | 20260506 02:24:37.115 | INFO | "huge_shared" : " 27 GB", | 20260506 02:24:37.115 | INFO | "malloc" : " 14 GB", | 20260506 02:24:37.115 | INFO | "pinned" : " 29 GB" | 20260506 02:24:37.115 | INFO | } | 20260506 02:24:37.115 | INFO | Perf report for the last 5.1404 seconds: | 20260506 02:24:37.115 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:24:37.115 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:37.115 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:37.115 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:37.115 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260506 02:24:37.115 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:37.116 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:37.116 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 02:24:37.116 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:37.116 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:37.116 | INFO | Uploaded 36GiB to GPU in 2.13956s (16.8259GiB/s) | 20260506 02:24:37.116 | INFO | Downloaded 34GiB to GPU in 2.08836s (16.2807GiB/s) | 20260506 02:24:42.251 | INFO | Analyzing chunk frames = [8207872,8208384) | 20260506 02:24:42.251 | INFO | MemInfo: { | 20260506 02:24:42.251 | INFO | "gpu" : " 17 GB", | 20260506 02:24:42.251 | INFO | "huge" : "", | 20260506 02:24:42.251 | INFO | "huge_shared" : " 27 GB", | 20260506 02:24:42.251 | INFO | "malloc" : " 14 GB", | 20260506 02:24:42.251 | INFO | "pinned" : " 29 GB" | 20260506 02:24:42.251 | INFO | } | 20260506 02:24:42.251 | INFO | Perf report for the last 5.13584 seconds: | 20260506 02:24:42.251 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:24:42.251 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:42.251 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:42.251 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:42.251 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260506 02:24:42.251 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:42.251 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:42.251 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 02:24:42.251 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:42.251 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:42.251 | INFO | Uploaded 36GiB to GPU in 2.03246s (17.7125GiB/s) | 20260506 02:24:42.251 | INFO | Downloaded 34GiB to GPU in 2.02126s (16.8212GiB/s) | 20260506 02:24:42.252 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 02:24:42.252 | INFO | Total mem usage: 15.3 GiB | 20260506 02:24:42.252 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 02:24:42.252 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.2%) | 20260506 02:24:42.252 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 02:24:42.252 | INFO | High Water Breakdown: | 20260506 02:24:42.252 | INFO | 60.9% : 8.54 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 02:24:42.252 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 02:24:42.252 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 02:24:42.252 | INFO | ------------------------------------------ | 20260506 02:24:42.252 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:24:42.252 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 02:24:42.252 | INFO | Total mem usage: 26.5 GiB | 20260506 02:24:42.252 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 02:24:42.252 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 02:24:42.252 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 02:24:42.252 | INFO | High Water Breakdown: | 20260506 02:24:42.252 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:24:42.252 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:24:42.252 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:24:42.252 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:24:42.252 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:24:42.252 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:24:42.252 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 02:24:42.252 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 02:24:42.252 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:24:42.252 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 02:24:42.252 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 02:24:42.252 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 02:24:42.252 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 02:24:42.252 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 02:24:42.252 | INFO | ------------------------------------------ | 20260506 02:24:42.252 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:24:42.252 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 02:24:42.252 | INFO | Total mem usage: 25.6 GiB | 20260506 02:24:42.252 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 02:24:42.252 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 02:24:42.252 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 02:24:42.252 | INFO | High Water Breakdown: | 20260506 02:24:42.252 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 02:24:42.252 | INFO | ------------------------------------------ | 20260506 02:24:42.252 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:24:42.252 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 02:24:42.252 | INFO | Total mem usage: 15.7 GiB | 20260506 02:24:42.252 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 02:24:42.252 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 02:24:42.252 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 02:24:42.252 | INFO | High Water Breakdown: | 20260506 02:24:42.252 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 02:24:42.252 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 02:24:42.252 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 02:24:42.252 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 02:24:42.252 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 02:24:42.252 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 02:24:42.252 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 02:24:42.252 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 02:24:42.252 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 02:24:42.252 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:24:42.252 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 02:24:42.252 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:24:42.252 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 02:24:42.252 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 02:24:42.252 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 02:24:42.252 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 02:24:42.252 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:24:42.252 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 02:24:42.252 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:24:42.252 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:24:42.252 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:24:42.252 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 02:24:42.252 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 02:24:42.252 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:24:42.252 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 02:24:42.252 | INFO | ------------------------------------------ | 20260506 02:24:42.252 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:24:42.253 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 02:24:42.253 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:24:42.253 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:24:42.253 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.36e+01, 1.32e+01 / 6.74e-01 | 20260506 02:24:42.253 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.05e-01, 1.05e-02 / 4.07e-03 | 20260506 02:24:42.253 | INFO | PerfReport: 11.87%, Baseline -- 3.71e+00 / 4.50e+00, 5.79e-01 / 2.55e-01 | 20260506 02:24:42.253 | INFO | PerfReport: 3.30%, Histogram -- 1.03e+00 / 1.53e+00, 1.95e-01 / 8.11e-02 | 20260506 02:24:42.253 | INFO | PerfReport: 15.98%, DME -- 4.00e+01 / 4.39e+01, 3.82e+00 / 8.16e-01 | 20260506 02:24:42.253 | INFO | PerfReport: 57.51%, FrameLabeling -- 1.80e+01 / 1.93e+01, 1.54e+00 / 1.04e+00 | 20260506 02:24:42.253 | INFO | PerfReport: 7.00%, PulseAccumulating -- 2.19e+00 / 2.72e+00, 4.13e-01 / 2.21e-01 | 20260506 02:24:42.253 | INFO | PerfReport: 3.00%, Metrics -- 9.40e-01 / 1.20e+00, 2.52e-01 / 1.23e-01 | 20260506 02:24:42.253 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.25e-01 / 3.72e-01, 5.40e-02 / 2.55e-02 | 20260506 02:24:42.253 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 02:24:42.253 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:24:42.253 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:24:42.253 | INFO | PerfReport: Full Execution -- 9.76e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260506 02:24:42.253 | INFO | PerfReport: 22.17%, Upload -- 2.16e+01 / 0.00e+00, 7.14e+00 / 0.00e+00 | 20260506 02:24:42.253 | INFO | PerfReport: 21.95%, Download -- 2.14e+01 / 0.00e+00, 7.49e+00 / 0.00e+00 | 20260506 02:24:42.253 | INFO | PerfReport: 32.06%, Compute -- 3.13e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 02:24:42.253 | INFO | PerfReport: 2.01%, LockUpload -- 1.96e+00 / 0.00e+00, 7.81e+00 / 0.00e+00 | 20260506 02:24:42.253 | INFO | PerfReport: 1.33%, LockDownload -- 1.30e+00 / 0.00e+00, 2.94e+00 / 0.00e+00 | 20260506 02:24:42.253 | INFO | PerfReport: 20.48%, LockCompute -- 2.00e+01 / 0.00e+00, 1.75e+01 / 0.00e+00 | 20260506 02:24:42.253 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.82e-02 / 0.00e+00 | 20260506 02:24:42.377 | INFO | Starting a round of Baz IO | 20260506 02:24:45.491 | INFO | Populating RTMetrics report | 20260506 02:24:45.491 | INFO | RTMetrics report populated | 20260506 02:24:45.491 | INFO | Opening tmp RTMetrics json file | 20260506 02:24:45.492 | INFO | Serializing RTMetrics to json | 20260506 02:24:45.493 | INFO | Serialization done, swapping tmp file now... | 20260506 02:24:45.494 | INFO | RTMetrics done | 20260506 02:24:45.573 | INFO | Finished a round of Baz IO: Wrote 1.793 GiB in 3.196 seconds | 20260506 02:24:45.573 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.27% / 0.00% / 78.73% | 20260506 02:24:45.573 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0705004% / 0% / 8.34465e-05% | 20260506 02:24:45.573 | INFO | Baz IO overall: | 20260506 02:24:45.573 | INFO | BytesWritten : 4600262109782 | 20260506 02:24:45.573 | INFO | NumEvents : 2159851326126 | 20260506 02:24:47.391 | INFO | Analyzing chunk frames = [8208384,8208896) | 20260506 02:24:47.391 | INFO | MemInfo: { | 20260506 02:24:47.391 | INFO | "gpu" : " 17 GB", | 20260506 02:24:47.391 | INFO | "huge" : "", | 20260506 02:24:47.391 | INFO | "huge_shared" : " 27 GB", | 20260506 02:24:47.391 | INFO | "malloc" : " 14 GB", | 20260506 02:24:47.391 | INFO | "pinned" : " 29 GB" | 20260506 02:24:47.391 | INFO | } | 20260506 02:24:47.392 | INFO | Perf report for the last 5.14039 seconds: | 20260506 02:24:47.392 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:24:47.392 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:47.392 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:47.392 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:47.392 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260506 02:24:47.392 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:47.392 | INFO | RT_METRICS : 43.0%, 1.0, 22.5ms, 0 | 20260506 02:24:47.392 | INFO | PRE_HQ : 59.4%, 1.8, 54.0ms, 0 | 20260506 02:24:47.392 | INFO | BAZWRITER : 61.3%, 3.5, 109.1ms, 0 | 20260506 02:24:47.392 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:24:47.392 | INFO | Uploaded 36GiB to GPU in 2.08466s (17.269GiB/s) | 20260506 02:24:47.392 | INFO | Downloaded 38GiB to GPU in 2.22041s (17.114GiB/s) | 20260506 02:24:52.529 | INFO | Analyzing chunk frames = [8208896,8209408) | 20260506 02:24:52.529 | INFO | MemInfo: { | 20260506 02:24:52.529 | INFO | "gpu" : " 17 GB", | 20260506 02:24:52.529 | INFO | "huge" : "", | 20260506 02:24:52.529 | INFO | "huge_shared" : " 27 GB", | 20260506 02:24:52.529 | INFO | "malloc" : " 14 GB", | 20260506 02:24:52.529 | INFO | "pinned" : " 29 GB" | 20260506 02:24:52.529 | INFO | } | 20260506 02:24:52.530 | INFO | Perf report for the last 5.13798 seconds: | 20260506 02:24:52.530 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:24:52.530 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:52.530 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:52.530 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:52.530 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260506 02:24:52.530 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:52.530 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:52.530 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:24:52.530 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:52.530 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:52.530 | INFO | Uploaded 36GiB to GPU in 1.93868s (18.5693GiB/s) | 20260506 02:24:52.530 | INFO | Downloaded 34GiB to GPU in 1.97717s (17.1963GiB/s) | 20260506 02:24:57.668 | INFO | Analyzing chunk frames = [8209408,8209920) | 20260506 02:24:57.668 | INFO | MemInfo: { | 20260506 02:24:57.668 | INFO | "gpu" : " 17 GB", | 20260506 02:24:57.668 | INFO | "huge" : "", | 20260506 02:24:57.668 | INFO | "huge_shared" : " 27 GB", | 20260506 02:24:57.668 | INFO | "malloc" : " 14 GB", | 20260506 02:24:57.668 | INFO | "pinned" : " 29 GB" | 20260506 02:24:57.668 | INFO | } | 20260506 02:24:57.668 | INFO | Perf report for the last 5.13827 seconds: | 20260506 02:24:57.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:24:57.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:57.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:57.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:57.668 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 02:24:57.668 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:57.668 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:57.668 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 02:24:57.668 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:57.668 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:24:57.668 | INFO | Uploaded 36GiB to GPU in 2.09307s (17.1996GiB/s) | 20260506 02:24:57.668 | INFO | Downloaded 34GiB to GPU in 2.0198s (16.8333GiB/s) | 20260506 02:25:02.807 | INFO | Analyzing chunk frames = [8209920,8210432) | 20260506 02:25:02.807 | INFO | MemInfo: { | 20260506 02:25:02.807 | INFO | "gpu" : " 17 GB", | 20260506 02:25:02.807 | INFO | "huge" : "", | 20260506 02:25:02.807 | INFO | "huge_shared" : " 27 GB", | 20260506 02:25:02.807 | INFO | "malloc" : " 14 GB", | 20260506 02:25:02.807 | INFO | "pinned" : " 29 GB" | 20260506 02:25:02.807 | INFO | } | 20260506 02:25:02.807 | INFO | Perf report for the last 5.13893 seconds: | 20260506 02:25:02.807 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:25:02.807 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:02.807 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:02.807 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:02.807 | INFO | ANALYSIS : 61.2%, 3.0, 95.0ms, 0 | 20260506 02:25:02.807 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:02.807 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:02.807 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 02:25:02.807 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:02.807 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:02.807 | INFO | Uploaded 36GiB to GPU in 2.10525s (17.1001GiB/s) | 20260506 02:25:02.807 | INFO | Downloaded 34GiB to GPU in 2.03174s (16.7344GiB/s) | 20260506 02:25:07.955 | INFO | Analyzing chunk frames = [8210432,8210944) | 20260506 02:25:07.955 | INFO | MemInfo: { | 20260506 02:25:07.955 | INFO | "gpu" : " 17 GB", | 20260506 02:25:07.955 | INFO | "huge" : "", | 20260506 02:25:07.955 | INFO | "huge_shared" : " 27 GB", | 20260506 02:25:07.955 | INFO | "malloc" : " 14 GB", | 20260506 02:25:07.955 | INFO | "pinned" : " 29 GB" | 20260506 02:25:07.955 | INFO | } | 20260506 02:25:07.955 | INFO | Perf report for the last 5.14827 seconds: | 20260506 02:25:07.955 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:25:07.955 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:07.955 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:07.955 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:07.955 | INFO | ANALYSIS : 60.5%, 3.0, 94.7ms, 0 | 20260506 02:25:07.955 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:07.955 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:07.955 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 02:25:07.955 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:07.955 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:07.955 | INFO | Uploaded 36GiB to GPU in 2.08711s (17.2488GiB/s) | 20260506 02:25:07.955 | INFO | Downloaded 34GiB to GPU in 2.03319s (16.7225GiB/s) | 20260506 02:25:13.090 | INFO | Analyzing chunk frames = [8210944,8211456) | 20260506 02:25:13.090 | INFO | MemInfo: { | 20260506 02:25:13.090 | INFO | "gpu" : " 17 GB", | 20260506 02:25:13.090 | INFO | "huge" : "", | 20260506 02:25:13.090 | INFO | "huge_shared" : " 27 GB", | 20260506 02:25:13.090 | INFO | "malloc" : " 14 GB", | 20260506 02:25:13.090 | INFO | "pinned" : " 29 GB" | 20260506 02:25:13.090 | INFO | } | 20260506 02:25:13.091 | INFO | Perf report for the last 5.1355 seconds: | 20260506 02:25:13.091 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:25:13.091 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:13.091 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:13.091 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:13.091 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 02:25:13.091 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:13.091 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:13.091 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 02:25:13.091 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:13.091 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:13.091 | INFO | Uploaded 36GiB to GPU in 2.04823s (17.5762GiB/s) | 20260506 02:25:13.091 | INFO | Downloaded 34GiB to GPU in 2.02628s (16.7795GiB/s) | 20260506 02:25:18.223 | INFO | Analyzing chunk frames = [8211456,8211968) | 20260506 02:25:18.223 | INFO | MemInfo: { | 20260506 02:25:18.223 | INFO | "gpu" : " 17 GB", | 20260506 02:25:18.223 | INFO | "huge" : "", | 20260506 02:25:18.223 | INFO | "huge_shared" : " 27 GB", | 20260506 02:25:18.223 | INFO | "malloc" : " 14 GB", | 20260506 02:25:18.223 | INFO | "pinned" : " 29 GB" | 20260506 02:25:18.223 | INFO | } | 20260506 02:25:18.223 | INFO | Perf report for the last 5.13243 seconds: | 20260506 02:25:18.223 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:25:18.223 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:18.223 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:18.223 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:18.223 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260506 02:25:18.223 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:18.223 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:18.223 | INFO | PRE_HQ : 27.7%, 1.0, 14.3ms, 0 | 20260506 02:25:18.223 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:18.223 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:18.223 | INFO | Uploaded 36GiB to GPU in 2.04017s (17.6456GiB/s) | 20260506 02:25:18.223 | INFO | Downloaded 34GiB to GPU in 2.04615s (16.6166GiB/s) | 20260506 02:25:23.366 | INFO | Analyzing chunk frames = [8211968,8212480) | 20260506 02:25:23.366 | INFO | MemInfo: { | 20260506 02:25:23.366 | INFO | "gpu" : " 17 GB", | 20260506 02:25:23.366 | INFO | "huge" : "", | 20260506 02:25:23.366 | INFO | "huge_shared" : " 27 GB", | 20260506 02:25:23.366 | INFO | "malloc" : " 14 GB", | 20260506 02:25:23.366 | INFO | "pinned" : " 29 GB" | 20260506 02:25:23.367 | INFO | } | 20260506 02:25:23.367 | INFO | Perf report for the last 5.14373 seconds: | 20260506 02:25:23.367 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:25:23.367 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:23.367 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:23.367 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:23.367 | INFO | ANALYSIS : 61.7%, 3.0, 96.5ms, 0 | 20260506 02:25:23.367 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:23.367 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:23.367 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:25:23.367 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:23.367 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:23.367 | INFO | Uploaded 36GiB to GPU in 2.13263s (16.8805GiB/s) | 20260506 02:25:23.367 | INFO | Downloaded 34GiB to GPU in 2.02844s (16.7616GiB/s) | 20260506 02:25:23.492 | INFO | Starting a round of Baz IO | 20260506 02:25:26.614 | INFO | Populating RTMetrics report | 20260506 02:25:26.614 | INFO | RTMetrics report populated | 20260506 02:25:26.614 | INFO | Opening tmp RTMetrics json file | 20260506 02:25:26.615 | INFO | Serializing RTMetrics to json | 20260506 02:25:26.616 | INFO | Serialization done, swapping tmp file now... | 20260506 02:25:26.617 | INFO | RTMetrics done | 20260506 02:25:26.686 | INFO | Finished a round of Baz IO: Wrote 1.794 GiB in 3.194 seconds | 20260506 02:25:26.686 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.27% / 0.00% / 78.73% | 20260506 02:25:26.686 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0707467% / 0% / 0.000115236% | 20260506 02:25:26.686 | INFO | Baz IO overall: | 20260506 02:25:26.686 | INFO | BytesWritten : 4602188281204 | 20260506 02:25:26.686 | INFO | NumEvents : 2160727167316 | 20260506 02:25:28.506 | INFO | Analyzing chunk frames = [8212480,8212992) | 20260506 02:25:28.506 | INFO | MemInfo: { | 20260506 02:25:28.506 | INFO | "gpu" : " 17 GB", | 20260506 02:25:28.506 | INFO | "huge" : "", | 20260506 02:25:28.506 | INFO | "huge_shared" : " 27 GB", | 20260506 02:25:28.506 | INFO | "malloc" : " 14 GB", | 20260506 02:25:28.506 | INFO | "pinned" : " 29 GB" | 20260506 02:25:28.506 | INFO | } | 20260506 02:25:28.506 | INFO | Perf report for the last 5.13949 seconds: | 20260506 02:25:28.506 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:25:28.506 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:28.506 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:28.506 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:28.506 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260506 02:25:28.506 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:28.506 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260506 02:25:28.506 | INFO | PRE_HQ : 59.0%, 1.7, 53.5ms, 0 | 20260506 02:25:28.506 | INFO | BAZWRITER : 61.2%, 3.4, 107.6ms, 0 | 20260506 02:25:28.506 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:25:28.507 | INFO | Uploaded 36GiB to GPU in 2.09236s (17.2055GiB/s) | 20260506 02:25:28.507 | INFO | Downloaded 38GiB to GPU in 2.22854s (17.0515GiB/s) | 20260506 02:25:33.642 | INFO | Analyzing chunk frames = [8212992,8213504) | 20260506 02:25:33.642 | INFO | MemInfo: { | 20260506 02:25:33.642 | INFO | "gpu" : " 17 GB", | 20260506 02:25:33.642 | INFO | "huge" : "", | 20260506 02:25:33.642 | INFO | "huge_shared" : " 27 GB", | 20260506 02:25:33.642 | INFO | "malloc" : " 14 GB", | 20260506 02:25:33.642 | INFO | "pinned" : " 29 GB" | 20260506 02:25:33.642 | INFO | } | 20260506 02:25:33.643 | INFO | Perf report for the last 5.13644 seconds: | 20260506 02:25:33.643 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:25:33.643 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:33.643 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:33.643 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:33.643 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260506 02:25:33.643 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:33.643 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:33.643 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 02:25:33.643 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:33.643 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:33.643 | INFO | Uploaded 36GiB to GPU in 1.9414s (18.5434GiB/s) | 20260506 02:25:33.643 | INFO | Downloaded 34GiB to GPU in 1.97286s (17.2339GiB/s) | 20260506 02:25:38.782 | INFO | Analyzing chunk frames = [8213504,8214016) | 20260506 02:25:38.782 | INFO | MemInfo: { | 20260506 02:25:38.782 | INFO | "gpu" : " 17 GB", | 20260506 02:25:38.782 | INFO | "huge" : "", | 20260506 02:25:38.782 | INFO | "huge_shared" : " 27 GB", | 20260506 02:25:38.782 | INFO | "malloc" : " 14 GB", | 20260506 02:25:38.782 | INFO | "pinned" : " 29 GB" | 20260506 02:25:38.782 | INFO | } | 20260506 02:25:38.782 | INFO | Perf report for the last 5.13977 seconds: | 20260506 02:25:38.782 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:25:38.782 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:38.782 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:38.783 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:38.783 | INFO | ANALYSIS : 61.2%, 3.0, 95.1ms, 0 | 20260506 02:25:38.783 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:38.783 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:38.783 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:25:38.783 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:38.783 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:38.783 | INFO | Uploaded 36GiB to GPU in 2.09282s (17.2017GiB/s) | 20260506 02:25:38.783 | INFO | Downloaded 34GiB to GPU in 2.02245s (16.8113GiB/s) | 20260506 02:25:43.924 | INFO | Analyzing chunk frames = [8214016,8214528) | 20260506 02:25:43.924 | INFO | MemInfo: { | 20260506 02:25:43.924 | INFO | "gpu" : " 17 GB", | 20260506 02:25:43.924 | INFO | "huge" : "", | 20260506 02:25:43.924 | INFO | "huge_shared" : " 27 GB", | 20260506 02:25:43.924 | INFO | "malloc" : " 14 GB", | 20260506 02:25:43.925 | INFO | "pinned" : " 29 GB" | 20260506 02:25:43.925 | INFO | } | 20260506 02:25:43.925 | INFO | Perf report for the last 5.14216 seconds: | 20260506 02:25:43.925 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:25:43.925 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:43.925 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:43.925 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:43.925 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260506 02:25:43.925 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:43.925 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:43.925 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 02:25:43.925 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:43.925 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:43.925 | INFO | Uploaded 36GiB to GPU in 2.14597s (16.7756GiB/s) | 20260506 02:25:43.925 | INFO | Downloaded 34GiB to GPU in 2.02225s (16.813GiB/s) | 20260506 02:25:49.067 | INFO | Analyzing chunk frames = [8214528,8215040) | 20260506 02:25:49.067 | INFO | MemInfo: { | 20260506 02:25:49.067 | INFO | "gpu" : " 17 GB", | 20260506 02:25:49.067 | INFO | "huge" : "", | 20260506 02:25:49.067 | INFO | "huge_shared" : " 27 GB", | 20260506 02:25:49.067 | INFO | "malloc" : " 14 GB", | 20260506 02:25:49.067 | INFO | "pinned" : " 29 GB" | 20260506 02:25:49.067 | INFO | } | 20260506 02:25:49.067 | INFO | Perf report for the last 5.14251 seconds: | 20260506 02:25:49.067 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:25:49.067 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:49.067 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:49.067 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:49.067 | INFO | ANALYSIS : 60.5%, 3.0, 94.5ms, 0 | 20260506 02:25:49.067 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:49.067 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:49.067 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 02:25:49.067 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:49.067 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:49.068 | INFO | Uploaded 36GiB to GPU in 2.06883s (17.4011GiB/s) | 20260506 02:25:49.068 | INFO | Downloaded 34GiB to GPU in 2.02477s (16.792GiB/s) | 20260506 02:25:54.204 | INFO | Analyzing chunk frames = [8215040,8215552) | 20260506 02:25:54.204 | INFO | MemInfo: { | 20260506 02:25:54.204 | INFO | "gpu" : " 17 GB", | 20260506 02:25:54.204 | INFO | "huge" : "", | 20260506 02:25:54.204 | INFO | "huge_shared" : " 27 GB", | 20260506 02:25:54.204 | INFO | "malloc" : " 14 GB", | 20260506 02:25:54.204 | INFO | "pinned" : " 29 GB" | 20260506 02:25:54.204 | INFO | } | 20260506 02:25:54.204 | INFO | Perf report for the last 5.13688 seconds: | 20260506 02:25:54.204 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:25:54.204 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:54.204 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:54.204 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:54.204 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 02:25:54.204 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:54.204 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:54.204 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:25:54.204 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:54.204 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:54.204 | INFO | Uploaded 36GiB to GPU in 2.09606s (17.1751GiB/s) | 20260506 02:25:54.204 | INFO | Downloaded 34GiB to GPU in 2.03958s (16.6701GiB/s) | 20260506 02:25:59.339 | INFO | Analyzing chunk frames = [8215552,8216064) | 20260506 02:25:59.339 | INFO | MemInfo: { | 20260506 02:25:59.339 | INFO | "gpu" : " 17 GB", | 20260506 02:25:59.339 | INFO | "huge" : "", | 20260506 02:25:59.339 | INFO | "huge_shared" : " 27 GB", | 20260506 02:25:59.339 | INFO | "malloc" : " 14 GB", | 20260506 02:25:59.339 | INFO | "pinned" : " 29 GB" | 20260506 02:25:59.339 | INFO | } | 20260506 02:25:59.339 | INFO | Perf report for the last 5.13507 seconds: | 20260506 02:25:59.339 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:25:59.339 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:59.339 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:59.339 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:59.339 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260506 02:25:59.339 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:59.339 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:59.339 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 02:25:59.339 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:59.339 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:25:59.339 | INFO | Uploaded 36GiB to GPU in 2.08431s (17.2719GiB/s) | 20260506 02:25:59.340 | INFO | Downloaded 34GiB to GPU in 2.06009s (16.5041GiB/s) | 20260506 02:26:04.475 | INFO | Analyzing chunk frames = [8216064,8216576) | 20260506 02:26:04.475 | INFO | MemInfo: { | 20260506 02:26:04.475 | INFO | "gpu" : " 17 GB", | 20260506 02:26:04.475 | INFO | "huge" : "", | 20260506 02:26:04.475 | INFO | "huge_shared" : " 27 GB", | 20260506 02:26:04.475 | INFO | "malloc" : " 14 GB", | 20260506 02:26:04.475 | INFO | "pinned" : " 29 GB" | 20260506 02:26:04.475 | INFO | } | 20260506 02:26:04.475 | INFO | Perf report for the last 5.13606 seconds: | 20260506 02:26:04.475 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:26:04.475 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:04.475 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:04.475 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:04.475 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 02:26:04.475 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:04.475 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:04.475 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:26:04.475 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:04.475 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:04.475 | INFO | Uploaded 36GiB to GPU in 2.13369s (16.8722GiB/s) | 20260506 02:26:04.476 | INFO | Downloaded 34GiB to GPU in 2.03562s (16.7025GiB/s) | 20260506 02:26:04.476 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 02:26:04.476 | INFO | Total mem usage: 15.3 GiB | 20260506 02:26:04.476 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 02:26:04.476 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.21%) | 20260506 02:26:04.476 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 02:26:04.476 | INFO | High Water Breakdown: | 20260506 02:26:04.476 | INFO | 60.9% : 8.54 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 02:26:04.476 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 02:26:04.476 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 02:26:04.476 | INFO | ------------------------------------------ | 20260506 02:26:04.476 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:26:04.476 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 02:26:04.476 | INFO | Total mem usage: 26.5 GiB | 20260506 02:26:04.476 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 02:26:04.476 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 02:26:04.476 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 02:26:04.476 | INFO | High Water Breakdown: | 20260506 02:26:04.476 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:26:04.476 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:26:04.476 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:26:04.476 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:26:04.476 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:26:04.476 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:26:04.476 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 02:26:04.476 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 02:26:04.476 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:26:04.476 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 02:26:04.476 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 02:26:04.476 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 02:26:04.476 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 02:26:04.476 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 02:26:04.476 | INFO | ------------------------------------------ | 20260506 02:26:04.476 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:26:04.476 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 02:26:04.476 | INFO | Total mem usage: 25.6 GiB | 20260506 02:26:04.476 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 02:26:04.476 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 02:26:04.476 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 02:26:04.476 | INFO | High Water Breakdown: | 20260506 02:26:04.476 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 02:26:04.476 | INFO | ------------------------------------------ | 20260506 02:26:04.476 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:26:04.476 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 02:26:04.476 | INFO | Total mem usage: 15.7 GiB | 20260506 02:26:04.476 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 02:26:04.476 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 02:26:04.476 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 02:26:04.476 | INFO | High Water Breakdown: | 20260506 02:26:04.476 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 02:26:04.476 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 02:26:04.476 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 02:26:04.477 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 02:26:04.477 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 02:26:04.477 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 02:26:04.477 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 02:26:04.477 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 02:26:04.477 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 02:26:04.477 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:26:04.477 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 02:26:04.477 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:26:04.477 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 02:26:04.477 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 02:26:04.477 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 02:26:04.477 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 02:26:04.477 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:26:04.477 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 02:26:04.477 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:26:04.477 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:26:04.477 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:26:04.477 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 02:26:04.477 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 02:26:04.477 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:26:04.477 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 02:26:04.477 | INFO | ------------------------------------------ | 20260506 02:26:04.477 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:26:04.477 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 02:26:04.477 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:26:04.477 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:26:04.477 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.36e+01, 1.33e+01 / 8.06e-01 | 20260506 02:26:04.477 | INFO | PerfReport: 0.29%, AggregateStats -- 9.18e-02 / 1.04e-01, 1.01e-02 / 3.63e-03 | 20260506 02:26:04.477 | INFO | PerfReport: 11.88%, Baseline -- 3.72e+00 / 4.51e+00, 5.85e-01 / 2.47e-01 | 20260506 02:26:04.477 | INFO | PerfReport: 3.30%, Histogram -- 1.03e+00 / 1.53e+00, 1.96e-01 / 8.96e-02 | 20260506 02:26:04.477 | INFO | PerfReport: 15.98%, DME -- 4.00e+01 / 4.40e+01, 3.79e+00 / 8.72e-01 | 20260506 02:26:04.477 | INFO | PerfReport: 57.48%, FrameLabeling -- 1.80e+01 / 1.91e+01, 1.55e+00 / 9.48e-01 | 20260506 02:26:04.477 | INFO | PerfReport: 7.01%, PulseAccumulating -- 2.19e+00 / 2.76e+00, 4.11e-01 / 1.99e-01 | 20260506 02:26:04.477 | INFO | PerfReport: 3.02%, Metrics -- 9.44e-01 / 1.23e+00, 2.56e-01 / 1.32e-01 | 20260506 02:26:04.477 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.26e-01 / 3.79e-01, 5.45e-02 / 2.95e-02 | 20260506 02:26:04.477 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 02:26:04.477 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:26:04.477 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:26:04.477 | INFO | PerfReport: Full Execution -- 9.76e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260506 02:26:04.477 | INFO | PerfReport: 22.14%, Upload -- 2.16e+01 / 0.00e+00, 7.05e+00 / 0.00e+00 | 20260506 02:26:04.477 | INFO | PerfReport: 21.88%, Download -- 2.13e+01 / 0.00e+00, 7.53e+00 / 0.00e+00 | 20260506 02:26:04.477 | INFO | PerfReport: 32.07%, Compute -- 3.13e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 02:26:04.477 | INFO | PerfReport: 2.02%, LockUpload -- 1.97e+00 / 0.00e+00, 7.79e+00 / 0.00e+00 | 20260506 02:26:04.477 | INFO | PerfReport: 1.31%, LockDownload -- 1.28e+00 / 0.00e+00, 3.04e+00 / 0.00e+00 | 20260506 02:26:04.477 | INFO | PerfReport: 20.56%, LockCompute -- 2.01e+01 / 0.00e+00, 1.77e+01 / 0.00e+00 | 20260506 02:26:04.477 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.64e-02 / 0.00e+00 | 20260506 02:26:04.600 | INFO | Starting a round of Baz IO | 20260506 02:26:07.731 | INFO | Populating RTMetrics report | 20260506 02:26:07.732 | INFO | RTMetrics report populated | 20260506 02:26:07.732 | INFO | Opening tmp RTMetrics json file | 20260506 02:26:07.732 | INFO | Serializing RTMetrics to json | 20260506 02:26:07.734 | INFO | Serialization done, swapping tmp file now... | 20260506 02:26:07.734 | INFO | RTMetrics done | 20260506 02:26:07.804 | INFO | Finished a round of Baz IO: Wrote 1.793 GiB in 3.204 seconds | 20260506 02:26:07.804 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.27% / 0.00% / 78.73% | 20260506 02:26:07.804 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0702818% / 0% / 7.15256e-05% | 20260506 02:26:07.804 | INFO | Baz IO overall: | 20260506 02:26:07.804 | INFO | BytesWritten : 4604113722328 | 20260506 02:26:07.804 | INFO | NumEvents : 2161602189695 | 20260506 02:26:09.618 | INFO | Analyzing chunk frames = [8216576,8217088) | 20260506 02:26:09.618 | INFO | MemInfo: { | 20260506 02:26:09.618 | INFO | "gpu" : " 17 GB", | 20260506 02:26:09.618 | INFO | "huge" : "", | 20260506 02:26:09.618 | INFO | "huge_shared" : " 27 GB", | 20260506 02:26:09.618 | INFO | "malloc" : " 14 GB", | 20260506 02:26:09.618 | INFO | "pinned" : " 29 GB" | 20260506 02:26:09.618 | INFO | } | 20260506 02:26:09.618 | INFO | Perf report for the last 5.14268 seconds: | 20260506 02:26:09.618 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:26:09.618 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:09.618 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:09.618 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:09.618 | INFO | ANALYSIS : 62.3%, 3.0, 97.4ms, 0 | 20260506 02:26:09.618 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:09.618 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260506 02:26:09.618 | INFO | PRE_HQ : 59.5%, 1.7, 53.6ms, 0 | 20260506 02:26:09.618 | INFO | BAZWRITER : 61.5%, 3.4, 107.8ms, 0 | 20260506 02:26:09.618 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:26:09.618 | INFO | Uploaded 36GiB to GPU in 2.18183s (16.4999GiB/s) | 20260506 02:26:09.618 | INFO | Downloaded 38GiB to GPU in 2.23607s (16.9941GiB/s) | 20260506 02:26:14.760 | INFO | Analyzing chunk frames = [8217088,8217600) | 20260506 02:26:14.760 | INFO | MemInfo: { | 20260506 02:26:14.760 | INFO | "gpu" : " 17 GB", | 20260506 02:26:14.760 | INFO | "huge" : "", | 20260506 02:26:14.760 | INFO | "huge_shared" : " 27 GB", | 20260506 02:26:14.760 | INFO | "malloc" : " 14 GB", | 20260506 02:26:14.760 | INFO | "pinned" : " 29 GB" | 20260506 02:26:14.760 | INFO | } | 20260506 02:26:14.760 | INFO | Perf report for the last 5.14243 seconds: | 20260506 02:26:14.760 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:26:14.760 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:14.760 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:14.760 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:14.760 | INFO | ANALYSIS : 60.7%, 3.0, 94.8ms, 0 | 20260506 02:26:14.760 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:14.760 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:14.760 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:26:14.760 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:14.760 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:14.761 | INFO | Uploaded 36GiB to GPU in 1.96134s (18.3548GiB/s) | 20260506 02:26:14.761 | INFO | Downloaded 34GiB to GPU in 1.97751s (17.1933GiB/s) | 20260506 02:26:19.903 | INFO | Analyzing chunk frames = [8217600,8218112) | 20260506 02:26:19.903 | INFO | MemInfo: { | 20260506 02:26:19.903 | INFO | "gpu" : " 17 GB", | 20260506 02:26:19.903 | INFO | "huge" : "", | 20260506 02:26:19.903 | INFO | "huge_shared" : " 27 GB", | 20260506 02:26:19.903 | INFO | "malloc" : " 14 GB", | 20260506 02:26:19.903 | INFO | "pinned" : " 29 GB" | 20260506 02:26:19.903 | INFO | } | 20260506 02:26:19.903 | INFO | Perf report for the last 5.14293 seconds: | 20260506 02:26:19.903 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:26:19.903 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:19.903 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:19.903 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:19.903 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 02:26:19.903 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:19.903 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:19.903 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:26:19.903 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:19.903 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:19.904 | INFO | Uploaded 36GiB to GPU in 2.10224s (17.1246GiB/s) | 20260506 02:26:19.904 | INFO | Downloaded 34GiB to GPU in 2.03855s (16.6785GiB/s) | 20260506 02:26:25.040 | INFO | Analyzing chunk frames = [8218112,8218624) | 20260506 02:26:25.040 | INFO | MemInfo: { | 20260506 02:26:25.040 | INFO | "gpu" : " 17 GB", | 20260506 02:26:25.040 | INFO | "huge" : "", | 20260506 02:26:25.040 | INFO | "huge_shared" : " 27 GB", | 20260506 02:26:25.040 | INFO | "malloc" : " 14 GB", | 20260506 02:26:25.040 | INFO | "pinned" : " 29 GB" | 20260506 02:26:25.040 | INFO | } | 20260506 02:26:25.040 | INFO | Perf report for the last 5.13717 seconds: | 20260506 02:26:25.040 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:26:25.040 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:25.041 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:25.041 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:25.041 | INFO | ANALYSIS : 61.5%, 3.0, 95.5ms, 0 | 20260506 02:26:25.041 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:25.041 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:25.041 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:26:25.041 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:25.041 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:25.041 | INFO | Uploaded 36GiB to GPU in 2.15403s (16.7129GiB/s) | 20260506 02:26:25.041 | INFO | Downloaded 34GiB to GPU in 2.05771s (16.5232GiB/s) | 20260506 02:26:30.176 | INFO | Analyzing chunk frames = [8218624,8219136) | 20260506 02:26:30.176 | INFO | MemInfo: { | 20260506 02:26:30.176 | INFO | "gpu" : " 17 GB", | 20260506 02:26:30.176 | INFO | "huge" : "", | 20260506 02:26:30.176 | INFO | "huge_shared" : " 27 GB", | 20260506 02:26:30.176 | INFO | "malloc" : " 14 GB", | 20260506 02:26:30.176 | INFO | "pinned" : " 29 GB" | 20260506 02:26:30.176 | INFO | } | 20260506 02:26:30.176 | INFO | Perf report for the last 5.13571 seconds: | 20260506 02:26:30.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:26:30.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:30.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:30.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:30.176 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 02:26:30.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:30.176 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:30.176 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 02:26:30.176 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:30.176 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:30.176 | INFO | Uploaded 36GiB to GPU in 2.08791s (17.2421GiB/s) | 20260506 02:26:30.177 | INFO | Downloaded 34GiB to GPU in 2.0332s (16.7224GiB/s) | 20260506 02:26:35.327 | INFO | Analyzing chunk frames = [8219136,8219648) | 20260506 02:26:35.327 | INFO | MemInfo: { | 20260506 02:26:35.327 | INFO | "gpu" : " 17 GB", | 20260506 02:26:35.327 | INFO | "huge" : "", | 20260506 02:26:35.327 | INFO | "huge_shared" : " 27 GB", | 20260506 02:26:35.327 | INFO | "malloc" : " 14 GB", | 20260506 02:26:35.327 | INFO | "pinned" : " 29 GB" | 20260506 02:26:35.327 | INFO | } | 20260506 02:26:35.327 | INFO | Perf report for the last 5.15101 seconds: | 20260506 02:26:35.327 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:26:35.327 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:35.327 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:35.327 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:35.327 | INFO | ANALYSIS : 60.9%, 3.0, 95.3ms, 0 | 20260506 02:26:35.327 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:35.327 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:35.327 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 02:26:35.327 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:35.327 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:35.328 | INFO | Uploaded 36GiB to GPU in 2.08969s (17.2275GiB/s) | 20260506 02:26:35.328 | INFO | Downloaded 34GiB to GPU in 2.03825s (16.681GiB/s) | 20260506 02:26:40.456 | INFO | Analyzing chunk frames = [8219648,8220160) | 20260506 02:26:40.456 | INFO | MemInfo: { | 20260506 02:26:40.456 | INFO | "gpu" : " 17 GB", | 20260506 02:26:40.456 | INFO | "huge" : "", | 20260506 02:26:40.456 | INFO | "huge_shared" : " 27 GB", | 20260506 02:26:40.456 | INFO | "malloc" : " 14 GB", | 20260506 02:26:40.456 | INFO | "pinned" : " 29 GB" | 20260506 02:26:40.456 | INFO | } | 20260506 02:26:40.456 | INFO | Perf report for the last 5.12905 seconds: | 20260506 02:26:40.456 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:26:40.456 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:40.456 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:40.456 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:40.456 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 02:26:40.456 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:40.456 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:40.456 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 02:26:40.456 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:40.456 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:40.457 | INFO | Uploaded 36GiB to GPU in 2.07088s (17.3839GiB/s) | 20260506 02:26:40.457 | INFO | Downloaded 34GiB to GPU in 2.05757s (16.5243GiB/s) | 20260506 02:26:45.592 | INFO | Analyzing chunk frames = [8220160,8220672) | 20260506 02:26:45.592 | INFO | MemInfo: { | 20260506 02:26:45.592 | INFO | "gpu" : " 17 GB", | 20260506 02:26:45.592 | INFO | "huge" : "", | 20260506 02:26:45.592 | INFO | "huge_shared" : " 27 GB", | 20260506 02:26:45.592 | INFO | "malloc" : " 14 GB", | 20260506 02:26:45.592 | INFO | "pinned" : " 29 GB" | 20260506 02:26:45.592 | INFO | } | 20260506 02:26:45.592 | INFO | Perf report for the last 5.13606 seconds: | 20260506 02:26:45.592 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:26:45.592 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:45.592 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:45.592 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:45.592 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 02:26:45.592 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:45.592 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:45.592 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260506 02:26:45.593 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:45.593 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:45.593 | INFO | Uploaded 36GiB to GPU in 2.04066s (17.6413GiB/s) | 20260506 02:26:45.593 | INFO | Downloaded 34GiB to GPU in 2.01767s (16.8511GiB/s) | 20260506 02:26:45.718 | INFO | Starting a round of Baz IO | 20260506 02:26:48.849 | INFO | Populating RTMetrics report | 20260506 02:26:48.850 | INFO | RTMetrics report populated | 20260506 02:26:48.850 | INFO | Opening tmp RTMetrics json file | 20260506 02:26:48.850 | INFO | Serializing RTMetrics to json | 20260506 02:26:48.852 | INFO | Serialization done, swapping tmp file now... | 20260506 02:26:48.852 | INFO | RTMetrics done | 20260506 02:26:48.926 | INFO | Finished a round of Baz IO: Wrote 1.793 GiB in 3.208 seconds | 20260506 02:26:48.926 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.27% / 0.00% / 78.73% | 20260506 02:26:48.926 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0704924% / 0% / 6.75519e-05% | 20260506 02:26:48.926 | INFO | Baz IO overall: | 20260506 02:26:48.926 | INFO | BytesWritten : 4606038771424 | 20260506 02:26:48.926 | INFO | NumEvents : 2162477030894 | 20260506 02:26:50.734 | INFO | Analyzing chunk frames = [8220672,8221184) | 20260506 02:26:50.734 | INFO | MemInfo: { | 20260506 02:26:50.734 | INFO | "gpu" : " 17 GB", | 20260506 02:26:50.734 | INFO | "huge" : "", | 20260506 02:26:50.734 | INFO | "huge_shared" : " 27 GB", | 20260506 02:26:50.734 | INFO | "malloc" : " 14 GB", | 20260506 02:26:50.734 | INFO | "pinned" : " 29 GB" | 20260506 02:26:50.734 | INFO | } | 20260506 02:26:50.734 | INFO | Perf report for the last 5.14158 seconds: | 20260506 02:26:50.734 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:26:50.734 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:50.734 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:50.734 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:50.734 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260506 02:26:50.734 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:50.734 | INFO | RT_METRICS : 42.8%, 1.0, 22.5ms, 0 | 20260506 02:26:50.734 | INFO | PRE_HQ : 59.6%, 1.7, 53.6ms, 0 | 20260506 02:26:50.734 | INFO | BAZWRITER : 61.6%, 3.4, 108.4ms, 0 | 20260506 02:26:50.734 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:26:50.734 | INFO | Uploaded 36GiB to GPU in 2.18285s (16.4922GiB/s) | 20260506 02:26:50.734 | INFO | Downloaded 38GiB to GPU in 2.24102s (16.9565GiB/s) | 20260506 02:26:55.872 | INFO | Analyzing chunk frames = [8221184,8221696) | 20260506 02:26:55.872 | INFO | MemInfo: { | 20260506 02:26:55.872 | INFO | "gpu" : " 17 GB", | 20260506 02:26:55.872 | INFO | "huge" : "", | 20260506 02:26:55.872 | INFO | "huge_shared" : " 27 GB", | 20260506 02:26:55.872 | INFO | "malloc" : " 14 GB", | 20260506 02:26:55.872 | INFO | "pinned" : " 29 GB" | 20260506 02:26:55.872 | INFO | } | 20260506 02:26:55.872 | INFO | Perf report for the last 5.13799 seconds: | 20260506 02:26:55.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:26:55.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:55.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:55.872 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:55.872 | INFO | ANALYSIS : 60.9%, 3.0, 95.1ms, 0 | 20260506 02:26:55.872 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:55.872 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:55.872 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:26:55.872 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:55.872 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:26:55.872 | INFO | Uploaded 36GiB to GPU in 1.95806s (18.3855GiB/s) | 20260506 02:26:55.872 | INFO | Downloaded 34GiB to GPU in 1.9842s (17.1354GiB/s) | 20260506 02:27:01.018 | INFO | Analyzing chunk frames = [8221696,8222208) | 20260506 02:27:01.018 | INFO | MemInfo: { | 20260506 02:27:01.018 | INFO | "gpu" : " 17 GB", | 20260506 02:27:01.018 | INFO | "huge" : "", | 20260506 02:27:01.018 | INFO | "huge_shared" : " 27 GB", | 20260506 02:27:01.018 | INFO | "malloc" : " 14 GB", | 20260506 02:27:01.018 | INFO | "pinned" : " 29 GB" | 20260506 02:27:01.018 | INFO | } | 20260506 02:27:01.019 | INFO | Perf report for the last 5.14657 seconds: | 20260506 02:27:01.019 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:27:01.019 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:01.019 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:01.019 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:01.019 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260506 02:27:01.019 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:01.019 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:01.019 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 02:27:01.019 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:01.019 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:01.019 | INFO | Uploaded 36GiB to GPU in 2.09036s (17.222GiB/s) | 20260506 02:27:01.019 | INFO | Downloaded 34GiB to GPU in 2.02882s (16.7585GiB/s) | 20260506 02:27:06.158 | INFO | Analyzing chunk frames = [8222208,8222720) | 20260506 02:27:06.158 | INFO | MemInfo: { | 20260506 02:27:06.158 | INFO | "gpu" : " 17 GB", | 20260506 02:27:06.158 | INFO | "huge" : "", | 20260506 02:27:06.158 | INFO | "huge_shared" : " 27 GB", | 20260506 02:27:06.158 | INFO | "malloc" : " 14 GB", | 20260506 02:27:06.158 | INFO | "pinned" : " 29 GB" | 20260506 02:27:06.158 | INFO | } | 20260506 02:27:06.158 | INFO | Perf report for the last 5.1395 seconds: | 20260506 02:27:06.158 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:27:06.158 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:06.158 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:06.158 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:06.158 | INFO | ANALYSIS : 60.9%, 3.0, 94.6ms, 0 | 20260506 02:27:06.158 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:06.158 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:06.158 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:27:06.158 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:06.158 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:06.158 | INFO | Uploaded 36GiB to GPU in 2.06858s (17.4032GiB/s) | 20260506 02:27:06.158 | INFO | Downloaded 34GiB to GPU in 1.99788s (17.018GiB/s) | 20260506 02:27:11.288 | INFO | Analyzing chunk frames = [8222720,8223232) | 20260506 02:27:11.288 | INFO | MemInfo: { | 20260506 02:27:11.288 | INFO | "gpu" : " 17 GB", | 20260506 02:27:11.288 | INFO | "huge" : "", | 20260506 02:27:11.288 | INFO | "huge_shared" : " 27 GB", | 20260506 02:27:11.288 | INFO | "malloc" : " 14 GB", | 20260506 02:27:11.288 | INFO | "pinned" : " 29 GB" | 20260506 02:27:11.288 | INFO | } | 20260506 02:27:11.288 | INFO | Perf report for the last 5.13042 seconds: | 20260506 02:27:11.288 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:27:11.288 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:11.288 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:11.288 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:11.289 | INFO | ANALYSIS : 60.9%, 3.0, 95.0ms, 0 | 20260506 02:27:11.289 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:11.289 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:11.289 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 02:27:11.289 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:11.289 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:11.289 | INFO | Uploaded 36GiB to GPU in 2.11466s (17.024GiB/s) | 20260506 02:27:11.289 | INFO | Downloaded 34GiB to GPU in 2.04039s (16.6635GiB/s) | 20260506 02:27:16.427 | INFO | Analyzing chunk frames = [8223232,8223744) | 20260506 02:27:16.427 | INFO | MemInfo: { | 20260506 02:27:16.427 | INFO | "gpu" : " 17 GB", | 20260506 02:27:16.427 | INFO | "huge" : "", | 20260506 02:27:16.427 | INFO | "huge_shared" : " 27 GB", | 20260506 02:27:16.427 | INFO | "malloc" : " 14 GB", | 20260506 02:27:16.427 | INFO | "pinned" : " 29 GB" | 20260506 02:27:16.427 | INFO | } | 20260506 02:27:16.427 | INFO | Perf report for the last 5.13849 seconds: | 20260506 02:27:16.427 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:27:16.427 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:16.427 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:16.427 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:16.427 | INFO | ANALYSIS : 61.2%, 3.0, 95.6ms, 0 | 20260506 02:27:16.427 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:16.427 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:16.427 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 02:27:16.427 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:16.427 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:16.427 | INFO | Uploaded 36GiB to GPU in 2.07632s (17.3384GiB/s) | 20260506 02:27:16.427 | INFO | Downloaded 34GiB to GPU in 2.05092s (16.5779GiB/s) | 20260506 02:27:21.570 | INFO | Analyzing chunk frames = [8223744,8224256) | 20260506 02:27:21.570 | INFO | MemInfo: { | 20260506 02:27:21.570 | INFO | "gpu" : " 17 GB", | 20260506 02:27:21.570 | INFO | "huge" : "", | 20260506 02:27:21.570 | INFO | "huge_shared" : " 27 GB", | 20260506 02:27:21.570 | INFO | "malloc" : " 14 GB", | 20260506 02:27:21.570 | INFO | "pinned" : " 29 GB" | 20260506 02:27:21.570 | INFO | } | 20260506 02:27:21.570 | INFO | Perf report for the last 5.1433 seconds: | 20260506 02:27:21.570 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:27:21.570 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:21.570 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:21.570 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:21.570 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260506 02:27:21.570 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:21.570 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:21.570 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 02:27:21.570 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:21.570 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:21.570 | INFO | Uploaded 36GiB to GPU in 2.06562s (17.4281GiB/s) | 20260506 02:27:21.571 | INFO | Downloaded 34GiB to GPU in 2.07913s (16.353GiB/s) | 20260506 02:27:26.712 | INFO | Analyzing chunk frames = [8224256,8224768) | 20260506 02:27:26.712 | INFO | MemInfo: { | 20260506 02:27:26.712 | INFO | "gpu" : " 17 GB", | 20260506 02:27:26.712 | INFO | "huge" : "", | 20260506 02:27:26.712 | INFO | "huge_shared" : " 27 GB", | 20260506 02:27:26.712 | INFO | "malloc" : " 14 GB", | 20260506 02:27:26.712 | INFO | "pinned" : " 29 GB" | 20260506 02:27:26.712 | INFO | } | 20260506 02:27:26.712 | INFO | Perf report for the last 5.14172 seconds: | 20260506 02:27:26.712 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:27:26.712 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:26.712 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:26.712 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:26.712 | INFO | ANALYSIS : 61.3%, 3.0, 95.8ms, 0 | 20260506 02:27:26.712 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:26.712 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:26.712 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 02:27:26.712 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:26.712 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:26.712 | INFO | Uploaded 36GiB to GPU in 2.03768s (17.6672GiB/s) | 20260506 02:27:26.712 | INFO | Downloaded 34GiB to GPU in 2.0237s (16.8009GiB/s) | 20260506 02:27:26.712 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 02:27:26.712 | INFO | Total mem usage: 15.3 GiB | 20260506 02:27:26.712 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 02:27:26.712 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.19%) | 20260506 02:27:26.712 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 02:27:26.712 | INFO | High Water Breakdown: | 20260506 02:27:26.712 | INFO | 60.9% : 8.54 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 02:27:26.712 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 02:27:26.712 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 02:27:26.712 | INFO | ------------------------------------------ | 20260506 02:27:26.712 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:27:26.713 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 02:27:26.713 | INFO | Total mem usage: 26.5 GiB | 20260506 02:27:26.713 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 02:27:26.713 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 02:27:26.713 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 02:27:26.713 | INFO | High Water Breakdown: | 20260506 02:27:26.713 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:27:26.713 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:27:26.713 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:27:26.713 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:27:26.713 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:27:26.713 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:27:26.713 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 02:27:26.713 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 02:27:26.713 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:27:26.713 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 02:27:26.713 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 02:27:26.713 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 02:27:26.713 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 02:27:26.713 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 02:27:26.713 | INFO | ------------------------------------------ | 20260506 02:27:26.713 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:27:26.713 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 02:27:26.713 | INFO | Total mem usage: 25.6 GiB | 20260506 02:27:26.713 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 02:27:26.713 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 02:27:26.713 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 02:27:26.713 | INFO | High Water Breakdown: | 20260506 02:27:26.713 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 02:27:26.713 | INFO | ------------------------------------------ | 20260506 02:27:26.713 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:27:26.713 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 02:27:26.713 | INFO | Total mem usage: 15.7 GiB | 20260506 02:27:26.713 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 02:27:26.713 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 02:27:26.713 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 02:27:26.713 | INFO | High Water Breakdown: | 20260506 02:27:26.713 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 02:27:26.713 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 02:27:26.713 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 02:27:26.713 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 02:27:26.713 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 02:27:26.713 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 02:27:26.713 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 02:27:26.713 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 02:27:26.713 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 02:27:26.713 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:27:26.713 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 02:27:26.713 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:27:26.713 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 02:27:26.713 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 02:27:26.713 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 02:27:26.713 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 02:27:26.713 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:27:26.713 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 02:27:26.713 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:27:26.713 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:27:26.713 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:27:26.713 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 02:27:26.713 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 02:27:26.713 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:27:26.713 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 02:27:26.713 | INFO | ------------------------------------------ | 20260506 02:27:26.713 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:27:26.713 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 02:27:26.713 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:27:26.713 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:27:26.713 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.38e+01, 1.35e+01 / 7.81e-01 | 20260506 02:27:26.713 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.04e-01, 1.14e-02 / 3.47e-03 | 20260506 02:27:26.713 | INFO | PerfReport: 11.85%, Baseline -- 3.70e+00 / 4.53e+00, 5.75e-01 / 1.93e-01 | 20260506 02:27:26.713 | INFO | PerfReport: 3.30%, Histogram -- 1.03e+00 / 1.54e+00, 1.95e-01 / 7.08e-02 | 20260506 02:27:26.713 | INFO | PerfReport: 16.02%, DME -- 4.01e+01 / 4.40e+01, 3.88e+00 / 9.20e-01 | 20260506 02:27:26.713 | INFO | PerfReport: 57.47%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.50e+00 / 1.25e+00 | 20260506 02:27:26.713 | INFO | PerfReport: 7.02%, PulseAccumulating -- 2.20e+00 / 2.67e+00, 4.14e-01 / 2.83e-01 | 20260506 02:27:26.713 | INFO | PerfReport: 3.01%, Metrics -- 9.42e-01 / 1.20e+00, 2.52e-01 / 1.11e-01 | 20260506 02:27:26.713 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.26e-01 / 3.74e-01, 5.36e-02 / 2.62e-02 | 20260506 02:27:26.714 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 02:27:26.714 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:27:26.714 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:27:26.714 | INFO | PerfReport: Full Execution -- 9.76e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260506 02:27:26.714 | INFO | PerfReport: 22.19%, Upload -- 2.17e+01 / 0.00e+00, 7.21e+00 / 0.00e+00 | 20260506 02:27:26.714 | INFO | PerfReport: 21.94%, Download -- 2.14e+01 / 0.00e+00, 7.57e+00 / 0.00e+00 | 20260506 02:27:26.714 | INFO | PerfReport: 32.02%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 02:27:26.714 | INFO | PerfReport: 2.02%, LockUpload -- 1.97e+00 / 0.00e+00, 7.87e+00 / 0.00e+00 | 20260506 02:27:26.714 | INFO | PerfReport: 1.29%, LockDownload -- 1.26e+00 / 0.00e+00, 2.86e+00 / 0.00e+00 | 20260506 02:27:26.714 | INFO | PerfReport: 20.53%, LockCompute -- 2.00e+01 / 0.00e+00, 1.77e+01 / 0.00e+00 | 20260506 02:27:26.714 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.60e-02 / 0.00e+00 | 20260506 02:27:26.837 | INFO | Starting a round of Baz IO | 20260506 02:27:29.971 | INFO | Populating RTMetrics report | 20260506 02:27:29.972 | INFO | RTMetrics report populated | 20260506 02:27:29.972 | INFO | Opening tmp RTMetrics json file | 20260506 02:27:29.972 | INFO | Serializing RTMetrics to json | 20260506 02:27:29.973 | INFO | Serialization done, swapping tmp file now... | 20260506 02:27:29.974 | INFO | RTMetrics done | 20260506 02:27:30.047 | INFO | Finished a round of Baz IO: Wrote 1.794 GiB in 3.209 seconds | 20260506 02:27:30.047 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.27% / 0.00% / 78.73% | 20260506 02:27:30.047 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0707904% / 7.94729e-06% / 9.93411e-05% | 20260506 02:27:30.047 | INFO | Baz IO overall: | 20260506 02:27:30.047 | INFO | BytesWritten : 4607965101948 | 20260506 02:27:30.047 | INFO | NumEvents : 2163353137146 | 20260506 02:27:31.851 | INFO | Analyzing chunk frames = [8224768,8225280) | 20260506 02:27:31.851 | INFO | MemInfo: { | 20260506 02:27:31.851 | INFO | "gpu" : " 17 GB", | 20260506 02:27:31.851 | INFO | "huge" : "", | 20260506 02:27:31.851 | INFO | "huge_shared" : " 27 GB", | 20260506 02:27:31.851 | INFO | "malloc" : " 14 GB", | 20260506 02:27:31.851 | INFO | "pinned" : " 29 GB" | 20260506 02:27:31.851 | INFO | } | 20260506 02:27:31.851 | INFO | Perf report for the last 5.13908 seconds: | 20260506 02:27:31.851 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:27:31.851 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:31.851 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:31.851 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:31.851 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260506 02:27:31.851 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:31.851 | INFO | RT_METRICS : 42.8%, 1.0, 22.5ms, 0 | 20260506 02:27:31.851 | INFO | PRE_HQ : 59.9%, 1.7, 53.8ms, 0 | 20260506 02:27:31.851 | INFO | BAZWRITER : 61.7%, 3.4, 108.6ms, 0 | 20260506 02:27:31.851 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:27:31.851 | INFO | Uploaded 36GiB to GPU in 2.18093s (16.5067GiB/s) | 20260506 02:27:31.852 | INFO | Downloaded 38GiB to GPU in 2.2429s (16.9423GiB/s) | 20260506 02:27:36.989 | INFO | Analyzing chunk frames = [8225280,8225792) | 20260506 02:27:36.989 | INFO | MemInfo: { | 20260506 02:27:36.989 | INFO | "gpu" : " 17 GB", | 20260506 02:27:36.989 | INFO | "huge" : "", | 20260506 02:27:36.989 | INFO | "huge_shared" : " 27 GB", | 20260506 02:27:36.989 | INFO | "malloc" : " 14 GB", | 20260506 02:27:36.989 | INFO | "pinned" : " 29 GB" | 20260506 02:27:36.989 | INFO | } | 20260506 02:27:36.989 | INFO | Perf report for the last 5.13828 seconds: | 20260506 02:27:36.989 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:27:36.989 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:36.989 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:36.989 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:36.989 | INFO | ANALYSIS : 60.8%, 3.0, 94.9ms, 0 | 20260506 02:27:36.989 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:36.989 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:36.989 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 02:27:36.990 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:36.990 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:36.990 | INFO | Uploaded 36GiB to GPU in 1.95118s (18.4503GiB/s) | 20260506 02:27:36.990 | INFO | Downloaded 34GiB to GPU in 1.98565s (17.1228GiB/s) | 20260506 02:27:42.129 | INFO | Analyzing chunk frames = [8225792,8226304) | 20260506 02:27:42.129 | INFO | MemInfo: { | 20260506 02:27:42.129 | INFO | "gpu" : " 17 GB", | 20260506 02:27:42.129 | INFO | "huge" : "", | 20260506 02:27:42.129 | INFO | "huge_shared" : " 27 GB", | 20260506 02:27:42.129 | INFO | "malloc" : " 14 GB", | 20260506 02:27:42.129 | INFO | "pinned" : " 29 GB" | 20260506 02:27:42.129 | INFO | } | 20260506 02:27:42.129 | INFO | Perf report for the last 5.13966 seconds: | 20260506 02:27:42.129 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:27:42.129 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:42.129 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:42.129 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:42.129 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 02:27:42.129 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:42.129 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:42.129 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:27:42.129 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:42.129 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:42.129 | INFO | Uploaded 36GiB to GPU in 2.07846s (17.3205GiB/s) | 20260506 02:27:42.129 | INFO | Downloaded 34GiB to GPU in 2.02224s (16.8131GiB/s) | 20260506 02:27:47.269 | INFO | Analyzing chunk frames = [8226304,8226816) | 20260506 02:27:47.269 | INFO | MemInfo: { | 20260506 02:27:47.269 | INFO | "gpu" : " 17 GB", | 20260506 02:27:47.269 | INFO | "huge" : "", | 20260506 02:27:47.269 | INFO | "huge_shared" : " 27 GB", | 20260506 02:27:47.269 | INFO | "malloc" : " 14 GB", | 20260506 02:27:47.269 | INFO | "pinned" : " 29 GB" | 20260506 02:27:47.269 | INFO | } | 20260506 02:27:47.269 | INFO | Perf report for the last 5.14011 seconds: | 20260506 02:27:47.269 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:27:47.269 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:47.269 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:47.269 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:47.269 | INFO | ANALYSIS : 61.0%, 3.0, 94.8ms, 0 | 20260506 02:27:47.269 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:47.269 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:47.270 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:27:47.270 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:47.270 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:47.270 | INFO | Uploaded 36GiB to GPU in 2.07397s (17.358GiB/s) | 20260506 02:27:47.270 | INFO | Downloaded 34GiB to GPU in 2.00945s (16.9201GiB/s) | 20260506 02:27:52.404 | INFO | Analyzing chunk frames = [8226816,8227328) | 20260506 02:27:52.404 | INFO | MemInfo: { | 20260506 02:27:52.404 | INFO | "gpu" : " 17 GB", | 20260506 02:27:52.404 | INFO | "huge" : "", | 20260506 02:27:52.404 | INFO | "huge_shared" : " 27 GB", | 20260506 02:27:52.404 | INFO | "malloc" : " 14 GB", | 20260506 02:27:52.404 | INFO | "pinned" : " 29 GB" | 20260506 02:27:52.404 | INFO | } | 20260506 02:27:52.404 | INFO | Perf report for the last 5.13467 seconds: | 20260506 02:27:52.404 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:27:52.404 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:52.404 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:52.404 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:52.404 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260506 02:27:52.404 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:52.404 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:52.404 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 02:27:52.404 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:52.404 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:52.404 | INFO | Uploaded 36GiB to GPU in 2.0851s (17.2654GiB/s) | 20260506 02:27:52.404 | INFO | Downloaded 34GiB to GPU in 2.02199s (16.8151GiB/s) | 20260506 02:27:57.540 | INFO | Analyzing chunk frames = [8227328,8227840) | 20260506 02:27:57.540 | INFO | MemInfo: { | 20260506 02:27:57.540 | INFO | "gpu" : " 17 GB", | 20260506 02:27:57.540 | INFO | "huge" : "", | 20260506 02:27:57.540 | INFO | "huge_shared" : " 27 GB", | 20260506 02:27:57.540 | INFO | "malloc" : " 14 GB", | 20260506 02:27:57.540 | INFO | "pinned" : " 29 GB" | 20260506 02:27:57.540 | INFO | } | 20260506 02:27:57.540 | INFO | Perf report for the last 5.13659 seconds: | 20260506 02:27:57.540 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:27:57.540 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:57.540 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:57.541 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:57.541 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 02:27:57.541 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:57.541 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:57.541 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:27:57.541 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:57.541 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:27:57.541 | INFO | Uploaded 36GiB to GPU in 2.07065s (17.3858GiB/s) | 20260506 02:27:57.541 | INFO | Downloaded 34GiB to GPU in 2.03829s (16.6807GiB/s) | 20260506 02:28:02.683 | INFO | Analyzing chunk frames = [8227840,8228352) | 20260506 02:28:02.683 | INFO | MemInfo: { | 20260506 02:28:02.683 | INFO | "gpu" : " 17 GB", | 20260506 02:28:02.683 | INFO | "huge" : "", | 20260506 02:28:02.683 | INFO | "huge_shared" : " 27 GB", | 20260506 02:28:02.683 | INFO | "malloc" : " 14 GB", | 20260506 02:28:02.683 | INFO | "pinned" : " 29 GB" | 20260506 02:28:02.683 | INFO | } | 20260506 02:28:02.683 | INFO | Perf report for the last 5.14296 seconds: | 20260506 02:28:02.684 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:28:02.684 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:02.684 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:02.684 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:02.684 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260506 02:28:02.684 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:02.684 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:02.684 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 02:28:02.684 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:02.684 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:02.684 | INFO | Uploaded 36GiB to GPU in 2.06606s (17.4245GiB/s) | 20260506 02:28:02.684 | INFO | Downloaded 34GiB to GPU in 2.06807s (16.4405GiB/s) | 20260506 02:28:07.828 | INFO | Analyzing chunk frames = [8228352,8228864) | 20260506 02:28:07.828 | INFO | MemInfo: { | 20260506 02:28:07.828 | INFO | "gpu" : " 17 GB", | 20260506 02:28:07.828 | INFO | "huge" : "", | 20260506 02:28:07.828 | INFO | "huge_shared" : " 27 GB", | 20260506 02:28:07.828 | INFO | "malloc" : " 14 GB", | 20260506 02:28:07.828 | INFO | "pinned" : " 29 GB" | 20260506 02:28:07.828 | INFO | } | 20260506 02:28:07.828 | INFO | Perf report for the last 5.1444 seconds: | 20260506 02:28:07.828 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:28:07.828 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:07.828 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:07.828 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:07.828 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260506 02:28:07.828 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:07.828 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:07.828 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 02:28:07.828 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:07.828 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:07.828 | INFO | Uploaded 36GiB to GPU in 2.04183s (17.6313GiB/s) | 20260506 02:28:07.828 | INFO | Downloaded 34GiB to GPU in 2.01244s (16.8949GiB/s) | 20260506 02:28:07.953 | INFO | Starting a round of Baz IO | 20260506 02:28:11.084 | INFO | Populating RTMetrics report | 20260506 02:28:11.084 | INFO | RTMetrics report populated | 20260506 02:28:11.084 | INFO | Opening tmp RTMetrics json file | 20260506 02:28:11.084 | INFO | Serializing RTMetrics to json | 20260506 02:28:11.086 | INFO | Serialization done, swapping tmp file now... | 20260506 02:28:11.086 | INFO | RTMetrics done | 20260506 02:28:11.157 | INFO | Finished a round of Baz IO: Wrote 1.793 GiB in 3.203 seconds | 20260506 02:28:11.157 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.27% / 0.00% / 78.73% | 20260506 02:28:11.157 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0706951% / 0% / 0.000103315% | 20260506 02:28:11.157 | INFO | Baz IO overall: | 20260506 02:28:11.157 | INFO | BytesWritten : 4609890413586 | 20260506 02:28:11.157 | INFO | NumEvents : 2164228497981 | 20260506 02:28:12.969 | INFO | Analyzing chunk frames = [8228864,8229376) | 20260506 02:28:12.969 | INFO | MemInfo: { | 20260506 02:28:12.969 | INFO | "gpu" : " 17 GB", | 20260506 02:28:12.969 | INFO | "huge" : "", | 20260506 02:28:12.969 | INFO | "huge_shared" : " 27 GB", | 20260506 02:28:12.969 | INFO | "malloc" : " 14 GB", | 20260506 02:28:12.969 | INFO | "pinned" : " 29 GB" | 20260506 02:28:12.970 | INFO | } | 20260506 02:28:12.970 | INFO | Perf report for the last 5.14185 seconds: | 20260506 02:28:12.970 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:28:12.970 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:12.970 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:12.970 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:12.970 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260506 02:28:12.970 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:12.970 | INFO | RT_METRICS : 42.8%, 1.0, 22.5ms, 0 | 20260506 02:28:12.970 | INFO | PRE_HQ : 59.1%, 1.7, 53.4ms, 0 | 20260506 02:28:12.970 | INFO | BAZWRITER : 61.5%, 3.4, 108.2ms, 0 | 20260506 02:28:12.970 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:28:12.970 | INFO | Uploaded 36GiB to GPU in 2.11211s (17.0445GiB/s) | 20260506 02:28:12.970 | INFO | Downloaded 38GiB to GPU in 2.2364s (16.9916GiB/s) | 20260506 02:28:18.105 | INFO | Analyzing chunk frames = [8229376,8229888) | 20260506 02:28:18.105 | INFO | MemInfo: { | 20260506 02:28:18.105 | INFO | "gpu" : " 17 GB", | 20260506 02:28:18.105 | INFO | "huge" : "", | 20260506 02:28:18.105 | INFO | "huge_shared" : " 27 GB", | 20260506 02:28:18.105 | INFO | "malloc" : " 14 GB", | 20260506 02:28:18.105 | INFO | "pinned" : " 29 GB" | 20260506 02:28:18.105 | INFO | } | 20260506 02:28:18.105 | INFO | Perf report for the last 5.13569 seconds: | 20260506 02:28:18.105 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:28:18.105 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:18.105 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:18.105 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:18.105 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260506 02:28:18.106 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:18.106 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:18.106 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:28:18.106 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:18.106 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:18.106 | INFO | Uploaded 36GiB to GPU in 1.9579s (18.387GiB/s) | 20260506 02:28:18.106 | INFO | Downloaded 34GiB to GPU in 1.9857s (17.1224GiB/s) | 20260506 02:28:23.236 | INFO | Analyzing chunk frames = [8229888,8230400) | 20260506 02:28:23.236 | INFO | MemInfo: { | 20260506 02:28:23.236 | INFO | "gpu" : " 17 GB", | 20260506 02:28:23.236 | INFO | "huge" : "", | 20260506 02:28:23.236 | INFO | "huge_shared" : " 27 GB", | 20260506 02:28:23.236 | INFO | "malloc" : " 14 GB", | 20260506 02:28:23.236 | INFO | "pinned" : " 29 GB" | 20260506 02:28:23.236 | INFO | } | 20260506 02:28:23.236 | INFO | Perf report for the last 5.13101 seconds: | 20260506 02:28:23.236 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:28:23.236 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:23.236 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:23.237 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:23.237 | INFO | ANALYSIS : 61.3%, 3.0, 95.2ms, 0 | 20260506 02:28:23.237 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:23.237 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:23.237 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 02:28:23.237 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:23.237 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:23.237 | INFO | Uploaded 36GiB to GPU in 2.092s (17.2084GiB/s) | 20260506 02:28:23.237 | INFO | Downloaded 34GiB to GPU in 2.0254s (16.7868GiB/s) | 20260506 02:28:28.379 | INFO | Analyzing chunk frames = [8230400,8230912) | 20260506 02:28:28.379 | INFO | MemInfo: { | 20260506 02:28:28.379 | INFO | "gpu" : " 17 GB", | 20260506 02:28:28.379 | INFO | "huge" : "", | 20260506 02:28:28.379 | INFO | "huge_shared" : " 27 GB", | 20260506 02:28:28.379 | INFO | "malloc" : " 14 GB", | 20260506 02:28:28.379 | INFO | "pinned" : " 29 GB" | 20260506 02:28:28.379 | INFO | } | 20260506 02:28:28.379 | INFO | Perf report for the last 5.14237 seconds: | 20260506 02:28:28.379 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:28:28.379 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:28.379 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:28.379 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:28.379 | INFO | ANALYSIS : 61.5%, 3.0, 95.5ms, 0 | 20260506 02:28:28.379 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:28.379 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:28.379 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 02:28:28.379 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:28.379 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:28.379 | INFO | Uploaded 36GiB to GPU in 2.14668s (16.7701GiB/s) | 20260506 02:28:28.379 | INFO | Downloaded 34GiB to GPU in 2.02987s (16.7498GiB/s) | 20260506 02:28:33.519 | INFO | Analyzing chunk frames = [8230912,8231424) | 20260506 02:28:33.519 | INFO | MemInfo: { | 20260506 02:28:33.519 | INFO | "gpu" : " 17 GB", | 20260506 02:28:33.519 | INFO | "huge" : "", | 20260506 02:28:33.519 | INFO | "huge_shared" : " 27 GB", | 20260506 02:28:33.519 | INFO | "malloc" : " 14 GB", | 20260506 02:28:33.519 | INFO | "pinned" : " 29 GB" | 20260506 02:28:33.519 | INFO | } | 20260506 02:28:33.519 | INFO | Perf report for the last 5.1403 seconds: | 20260506 02:28:33.519 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:28:33.519 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:33.519 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:33.519 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:33.519 | INFO | ANALYSIS : 60.8%, 3.0, 94.9ms, 0 | 20260506 02:28:33.519 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:33.519 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:33.519 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 02:28:33.519 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:33.519 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:33.520 | INFO | Uploaded 36GiB to GPU in 2.0862s (17.2562GiB/s) | 20260506 02:28:33.520 | INFO | Downloaded 34GiB to GPU in 2.03523s (16.7058GiB/s) | 20260506 02:28:38.664 | INFO | Analyzing chunk frames = [8231424,8231936) | 20260506 02:28:38.664 | INFO | MemInfo: { | 20260506 02:28:38.664 | INFO | "gpu" : " 17 GB", | 20260506 02:28:38.664 | INFO | "huge" : "", | 20260506 02:28:38.664 | INFO | "huge_shared" : " 27 GB", | 20260506 02:28:38.664 | INFO | "malloc" : " 14 GB", | 20260506 02:28:38.664 | INFO | "pinned" : " 29 GB" | 20260506 02:28:38.664 | INFO | } | 20260506 02:28:38.664 | INFO | Perf report for the last 5.14494 seconds: | 20260506 02:28:38.664 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:28:38.664 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:38.664 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:38.664 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:38.664 | INFO | ANALYSIS : 61.0%, 3.0, 95.4ms, 0 | 20260506 02:28:38.664 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:38.664 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:38.664 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 02:28:38.664 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:38.664 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:38.665 | INFO | Uploaded 36GiB to GPU in 2.11311s (17.0365GiB/s) | 20260506 02:28:38.665 | INFO | Downloaded 34GiB to GPU in 2.04264s (16.6452GiB/s) | 20260506 02:28:43.800 | INFO | Analyzing chunk frames = [8231936,8232448) | 20260506 02:28:43.800 | INFO | MemInfo: { | 20260506 02:28:43.800 | INFO | "gpu" : " 17 GB", | 20260506 02:28:43.800 | INFO | "huge" : "", | 20260506 02:28:43.800 | INFO | "huge_shared" : " 27 GB", | 20260506 02:28:43.800 | INFO | "malloc" : " 14 GB", | 20260506 02:28:43.800 | INFO | "pinned" : " 29 GB" | 20260506 02:28:43.800 | INFO | } | 20260506 02:28:43.800 | INFO | Perf report for the last 5.13602 seconds: | 20260506 02:28:43.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:28:43.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:43.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:43.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:43.800 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260506 02:28:43.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:43.800 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:43.800 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 02:28:43.800 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:43.800 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:43.800 | INFO | Uploaded 36GiB to GPU in 2.13606s (16.8535GiB/s) | 20260506 02:28:43.801 | INFO | Downloaded 34GiB to GPU in 2.09085s (16.2613GiB/s) | 20260506 02:28:48.942 | INFO | Analyzing chunk frames = [8232448,8232960) | 20260506 02:28:48.942 | INFO | MemInfo: { | 20260506 02:28:48.942 | INFO | "gpu" : " 17 GB", | 20260506 02:28:48.942 | INFO | "huge" : "", | 20260506 02:28:48.942 | INFO | "huge_shared" : " 27 GB", | 20260506 02:28:48.942 | INFO | "malloc" : " 14 GB", | 20260506 02:28:48.942 | INFO | "pinned" : " 29 GB" | 20260506 02:28:48.942 | INFO | } | 20260506 02:28:48.942 | INFO | Perf report for the last 5.14211 seconds: | 20260506 02:28:48.942 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:28:48.942 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:48.942 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:48.942 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:48.942 | INFO | ANALYSIS : 61.4%, 3.0, 95.9ms, 0 | 20260506 02:28:48.942 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:48.942 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:48.942 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:28:48.942 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:48.942 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:48.943 | INFO | Uploaded 36GiB to GPU in 2.05483s (17.5197GiB/s) | 20260506 02:28:48.943 | INFO | Downloaded 34GiB to GPU in 2.01808s (16.8477GiB/s) | 20260506 02:28:48.943 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 02:28:48.943 | INFO | Total mem usage: 15.3 GiB | 20260506 02:28:48.943 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 02:28:48.943 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.19%) | 20260506 02:28:48.943 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 02:28:48.943 | INFO | High Water Breakdown: | 20260506 02:28:48.943 | INFO | 60.9% : 8.54 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 02:28:48.943 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 02:28:48.943 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 02:28:48.943 | INFO | ------------------------------------------ | 20260506 02:28:48.943 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:28:48.943 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 02:28:48.943 | INFO | Total mem usage: 26.5 GiB | 20260506 02:28:48.943 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 02:28:48.943 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 02:28:48.943 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 02:28:48.943 | INFO | High Water Breakdown: | 20260506 02:28:48.943 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:28:48.943 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:28:48.943 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:28:48.943 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:28:48.943 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:28:48.943 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:28:48.943 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 02:28:48.943 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 02:28:48.943 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:28:48.943 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 02:28:48.943 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 02:28:48.943 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 02:28:48.943 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 02:28:48.943 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 02:28:48.943 | INFO | ------------------------------------------ | 20260506 02:28:48.943 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:28:48.943 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 02:28:48.943 | INFO | Total mem usage: 25.6 GiB | 20260506 02:28:48.943 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 02:28:48.943 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 02:28:48.943 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 02:28:48.943 | INFO | High Water Breakdown: | 20260506 02:28:48.943 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 02:28:48.943 | INFO | ------------------------------------------ | 20260506 02:28:48.944 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:28:48.944 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 02:28:48.944 | INFO | Total mem usage: 15.7 GiB | 20260506 02:28:48.944 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 02:28:48.944 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 02:28:48.944 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 02:28:48.944 | INFO | High Water Breakdown: | 20260506 02:28:48.944 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 02:28:48.944 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 02:28:48.944 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 02:28:48.944 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 02:28:48.944 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 02:28:48.944 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 02:28:48.944 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 02:28:48.944 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 02:28:48.944 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 02:28:48.944 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:28:48.944 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 02:28:48.944 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:28:48.944 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 02:28:48.944 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 02:28:48.944 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 02:28:48.944 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 02:28:48.944 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:28:48.944 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 02:28:48.944 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:28:48.944 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:28:48.944 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:28:48.944 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 02:28:48.944 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 02:28:48.944 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:28:48.944 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 02:28:48.944 | INFO | ------------------------------------------ | 20260506 02:28:48.944 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:28:48.944 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 02:28:48.944 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:28:48.944 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:28:48.944 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.37e+01, 1.35e+01 / 6.87e-01 | 20260506 02:28:48.944 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.05e-01, 1.02e-02 / 4.95e-03 | 20260506 02:28:48.944 | INFO | PerfReport: 11.86%, Baseline -- 3.71e+00 / 4.48e+00, 5.82e-01 / 2.61e-01 | 20260506 02:28:48.944 | INFO | PerfReport: 3.30%, Histogram -- 1.03e+00 / 1.51e+00, 1.94e-01 / 9.56e-02 | 20260506 02:28:48.944 | INFO | PerfReport: 15.97%, DME -- 4.00e+01 / 4.39e+01, 3.81e+00 / 9.27e-01 | 20260506 02:28:48.944 | INFO | PerfReport: 57.52%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.55e+00 / 8.60e-01 | 20260506 02:28:48.944 | INFO | PerfReport: 7.00%, PulseAccumulating -- 2.19e+00 / 2.70e+00, 4.14e-01 / 2.23e-01 | 20260506 02:28:48.944 | INFO | PerfReport: 3.01%, Metrics -- 9.42e-01 / 1.20e+00, 2.54e-01 / 1.03e-01 | 20260506 02:28:48.944 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.26e-01 / 3.74e-01, 5.38e-02 / 2.59e-02 | 20260506 02:28:48.944 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 02:28:48.944 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:28:48.944 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:28:48.944 | INFO | PerfReport: Full Execution -- 9.77e+01 / -nan, 2.42e+01 / 0.00e+00 | 20260506 02:28:48.944 | INFO | PerfReport: 22.16%, Upload -- 2.16e+01 / 0.00e+00, 7.12e+00 / 0.00e+00 | 20260506 02:28:48.944 | INFO | PerfReport: 21.91%, Download -- 2.14e+01 / 0.00e+00, 7.55e+00 / 0.00e+00 | 20260506 02:28:48.944 | INFO | PerfReport: 32.04%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 02:28:48.944 | INFO | PerfReport: 2.03%, LockUpload -- 1.98e+00 / 0.00e+00, 7.87e+00 / 0.00e+00 | 20260506 02:28:48.944 | INFO | PerfReport: 1.29%, LockDownload -- 1.26e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260506 02:28:48.944 | INFO | PerfReport: 20.55%, LockCompute -- 2.01e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260506 02:28:48.944 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.53e-02 / 0.00e+00 | 20260506 02:28:49.067 | INFO | Starting a round of Baz IO | 20260506 02:28:52.191 | INFO | Populating RTMetrics report | 20260506 02:28:52.192 | INFO | RTMetrics report populated | 20260506 02:28:52.192 | INFO | Opening tmp RTMetrics json file | 20260506 02:28:52.193 | INFO | Serializing RTMetrics to json | 20260506 02:28:52.194 | INFO | Serialization done, swapping tmp file now... | 20260506 02:28:52.194 | INFO | RTMetrics done | 20260506 02:28:52.265 | INFO | Finished a round of Baz IO: Wrote 1.793 GiB in 3.198 seconds | 20260506 02:28:52.265 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.27% / 0.00% / 78.73% | 20260506 02:28:52.265 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0704527% / 0% / 9.13938e-05% | 20260506 02:28:52.265 | INFO | Baz IO overall: | 20260506 02:28:52.265 | INFO | BytesWritten : 4611815473239 | 20260506 02:28:52.265 | INFO | NumEvents : 2165103512501 | 20260506 02:28:54.078 | INFO | Analyzing chunk frames = [8232960,8233472) | 20260506 02:28:54.078 | INFO | MemInfo: { | 20260506 02:28:54.078 | INFO | "gpu" : " 17 GB", | 20260506 02:28:54.078 | INFO | "huge" : "", | 20260506 02:28:54.078 | INFO | "huge_shared" : " 27 GB", | 20260506 02:28:54.078 | INFO | "malloc" : " 14 GB", | 20260506 02:28:54.078 | INFO | "pinned" : " 29 GB" | 20260506 02:28:54.078 | INFO | } | 20260506 02:28:54.078 | INFO | Perf report for the last 5.13573 seconds: | 20260506 02:28:54.078 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:28:54.078 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:54.078 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:54.078 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:54.078 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260506 02:28:54.078 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:54.078 | INFO | RT_METRICS : 42.8%, 1.0, 22.4ms, 0 | 20260506 02:28:54.078 | INFO | PRE_HQ : 59.2%, 1.7, 53.4ms, 0 | 20260506 02:28:54.078 | INFO | BAZWRITER : 61.4%, 3.4, 108.1ms, 0 | 20260506 02:28:54.078 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:28:54.079 | INFO | Uploaded 36GiB to GPU in 2.11198s (17.0456GiB/s) | 20260506 02:28:54.079 | INFO | Downloaded 38GiB to GPU in 2.2284s (17.0526GiB/s) | 20260506 02:28:59.221 | INFO | Analyzing chunk frames = [8233472,8233984) | 20260506 02:28:59.221 | INFO | MemInfo: { | 20260506 02:28:59.221 | INFO | "gpu" : " 17 GB", | 20260506 02:28:59.221 | INFO | "huge" : "", | 20260506 02:28:59.221 | INFO | "huge_shared" : " 27 GB", | 20260506 02:28:59.221 | INFO | "malloc" : " 14 GB", | 20260506 02:28:59.221 | INFO | "pinned" : " 29 GB" | 20260506 02:28:59.221 | INFO | } | 20260506 02:28:59.221 | INFO | Perf report for the last 5.14326 seconds: | 20260506 02:28:59.221 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:28:59.221 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:59.221 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:59.221 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:59.222 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 02:28:59.222 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:59.222 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:59.222 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 02:28:59.222 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:59.222 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:28:59.222 | INFO | Uploaded 36GiB to GPU in 1.94829s (18.4778GiB/s) | 20260506 02:28:59.222 | INFO | Downloaded 34GiB to GPU in 1.97426s (17.2217GiB/s) | 20260506 02:29:04.358 | INFO | Analyzing chunk frames = [8233984,8234496) | 20260506 02:29:04.358 | INFO | MemInfo: { | 20260506 02:29:04.358 | INFO | "gpu" : " 17 GB", | 20260506 02:29:04.358 | INFO | "huge" : "", | 20260506 02:29:04.358 | INFO | "huge_shared" : " 27 GB", | 20260506 02:29:04.358 | INFO | "malloc" : " 14 GB", | 20260506 02:29:04.358 | INFO | "pinned" : " 29 GB" | 20260506 02:29:04.358 | INFO | } | 20260506 02:29:04.358 | INFO | Perf report for the last 5.13664 seconds: | 20260506 02:29:04.358 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:29:04.358 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:04.358 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:04.358 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:04.358 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 02:29:04.358 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:04.358 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:04.358 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:29:04.358 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:04.358 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:04.358 | INFO | Uploaded 36GiB to GPU in 2.11018s (17.0601GiB/s) | 20260506 02:29:04.358 | INFO | Downloaded 34GiB to GPU in 2.04303s (16.642GiB/s) | 20260506 02:29:09.495 | INFO | Analyzing chunk frames = [8234496,8235008) | 20260506 02:29:09.495 | INFO | MemInfo: { | 20260506 02:29:09.495 | INFO | "gpu" : " 17 GB", | 20260506 02:29:09.495 | INFO | "huge" : "", | 20260506 02:29:09.495 | INFO | "huge_shared" : " 27 GB", | 20260506 02:29:09.495 | INFO | "malloc" : " 14 GB", | 20260506 02:29:09.495 | INFO | "pinned" : " 29 GB" | 20260506 02:29:09.495 | INFO | } | 20260506 02:29:09.495 | INFO | Perf report for the last 5.1372 seconds: | 20260506 02:29:09.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:29:09.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:09.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:09.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:09.495 | INFO | ANALYSIS : 61.2%, 3.0, 95.0ms, 0 | 20260506 02:29:09.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:09.495 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:09.495 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 02:29:09.495 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:09.495 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:09.495 | INFO | Uploaded 36GiB to GPU in 2.10053s (17.1385GiB/s) | 20260506 02:29:09.495 | INFO | Downloaded 34GiB to GPU in 2.03189s (16.7332GiB/s) | 20260506 02:29:14.640 | INFO | Analyzing chunk frames = [8235008,8235520) | 20260506 02:29:14.640 | INFO | MemInfo: { | 20260506 02:29:14.640 | INFO | "gpu" : " 17 GB", | 20260506 02:29:14.640 | INFO | "huge" : "", | 20260506 02:29:14.640 | INFO | "huge_shared" : " 27 GB", | 20260506 02:29:14.640 | INFO | "malloc" : " 14 GB", | 20260506 02:29:14.640 | INFO | "pinned" : " 29 GB" | 20260506 02:29:14.640 | INFO | } | 20260506 02:29:14.640 | INFO | Perf report for the last 5.1449 seconds: | 20260506 02:29:14.640 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:29:14.640 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:14.640 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:14.640 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:14.640 | INFO | ANALYSIS : 60.6%, 3.0, 94.7ms, 0 | 20260506 02:29:14.640 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:14.640 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:14.640 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 02:29:14.640 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:14.640 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:14.640 | INFO | Uploaded 36GiB to GPU in 2.07802s (17.3242GiB/s) | 20260506 02:29:14.640 | INFO | Downloaded 34GiB to GPU in 2.0291s (16.7562GiB/s) | 20260506 02:29:19.777 | INFO | Analyzing chunk frames = [8235520,8236032) | 20260506 02:29:19.777 | INFO | MemInfo: { | 20260506 02:29:19.777 | INFO | "gpu" : " 17 GB", | 20260506 02:29:19.777 | INFO | "huge" : "", | 20260506 02:29:19.777 | INFO | "huge_shared" : " 27 GB", | 20260506 02:29:19.777 | INFO | "malloc" : " 14 GB", | 20260506 02:29:19.777 | INFO | "pinned" : " 29 GB" | 20260506 02:29:19.777 | INFO | } | 20260506 02:29:19.777 | INFO | Perf report for the last 5.13694 seconds: | 20260506 02:29:19.777 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:29:19.777 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:19.777 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:19.777 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:19.777 | INFO | ANALYSIS : 60.7%, 3.0, 94.8ms, 0 | 20260506 02:29:19.777 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:19.777 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:19.777 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 02:29:19.777 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:19.777 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:19.777 | INFO | Uploaded 36GiB to GPU in 2.05625s (17.5076GiB/s) | 20260506 02:29:19.777 | INFO | Downloaded 34GiB to GPU in 2.02427s (16.7961GiB/s) | 20260506 02:29:24.916 | INFO | Analyzing chunk frames = [8236032,8236544) | 20260506 02:29:24.916 | INFO | MemInfo: { | 20260506 02:29:24.916 | INFO | "gpu" : " 17 GB", | 20260506 02:29:24.916 | INFO | "huge" : "", | 20260506 02:29:24.916 | INFO | "huge_shared" : " 27 GB", | 20260506 02:29:24.916 | INFO | "malloc" : " 14 GB", | 20260506 02:29:24.916 | INFO | "pinned" : " 29 GB" | 20260506 02:29:24.916 | INFO | } | 20260506 02:29:24.916 | INFO | Perf report for the last 5.13913 seconds: | 20260506 02:29:24.916 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:29:24.916 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:24.916 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:24.916 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:24.916 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 02:29:24.916 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:24.916 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:24.916 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 02:29:24.916 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:24.916 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:24.916 | INFO | Uploaded 36GiB to GPU in 2.02647s (17.7649GiB/s) | 20260506 02:29:24.916 | INFO | Downloaded 34GiB to GPU in 2.05466s (16.5477GiB/s) | 20260506 02:29:30.051 | INFO | Analyzing chunk frames = [8236544,8237056) | 20260506 02:29:30.051 | INFO | MemInfo: { | 20260506 02:29:30.051 | INFO | "gpu" : " 17 GB", | 20260506 02:29:30.051 | INFO | "huge" : "", | 20260506 02:29:30.051 | INFO | "huge_shared" : " 27 GB", | 20260506 02:29:30.051 | INFO | "malloc" : " 14 GB", | 20260506 02:29:30.051 | INFO | "pinned" : " 29 GB" | 20260506 02:29:30.051 | INFO | } | 20260506 02:29:30.051 | INFO | Perf report for the last 5.13465 seconds: | 20260506 02:29:30.051 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:29:30.051 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:30.051 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:30.051 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:30.051 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 02:29:30.051 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:30.051 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:30.051 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 02:29:30.051 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:30.051 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:30.051 | INFO | Uploaded 36GiB to GPU in 2.13514s (16.8607GiB/s) | 20260506 02:29:30.051 | INFO | Downloaded 34GiB to GPU in 2.03201s (16.7322GiB/s) | 20260506 02:29:30.168 | INFO | Starting a round of Baz IO | 20260506 02:29:33.307 | INFO | Populating RTMetrics report | 20260506 02:29:33.307 | INFO | RTMetrics report populated | 20260506 02:29:33.307 | INFO | Opening tmp RTMetrics json file | 20260506 02:29:33.308 | INFO | Serializing RTMetrics to json | 20260506 02:29:33.309 | INFO | Serialization done, swapping tmp file now... | 20260506 02:29:33.310 | INFO | RTMetrics done | 20260506 02:29:33.382 | INFO | Finished a round of Baz IO: Wrote 1.791 GiB in 3.214 seconds | 20260506 02:29:33.382 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.27% / 0.00% / 78.73% | 20260506 02:29:33.382 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0697056% / 0% / 9.93411e-05% | 20260506 02:29:33.382 | INFO | Baz IO overall: | 20260506 02:29:33.382 | INFO | BytesWritten : 4613738569486 | 20260506 02:29:33.382 | INFO | NumEvents : 2165977481457 | 20260506 02:29:35.188 | INFO | Analyzing chunk frames = [8237056,8237568) | 20260506 02:29:35.188 | INFO | MemInfo: { | 20260506 02:29:35.188 | INFO | "gpu" : " 17 GB", | 20260506 02:29:35.188 | INFO | "huge" : "", | 20260506 02:29:35.188 | INFO | "huge_shared" : " 27 GB", | 20260506 02:29:35.188 | INFO | "malloc" : " 14 GB", | 20260506 02:29:35.188 | INFO | "pinned" : " 29 GB" | 20260506 02:29:35.188 | INFO | } | 20260506 02:29:35.188 | INFO | Perf report for the last 5.13718 seconds: | 20260506 02:29:35.188 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:29:35.188 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:35.188 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:35.188 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:35.188 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260506 02:29:35.188 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:35.188 | INFO | RT_METRICS : 42.8%, 1.0, 22.4ms, 0 | 20260506 02:29:35.188 | INFO | PRE_HQ : 59.6%, 1.7, 53.2ms, 0 | 20260506 02:29:35.188 | INFO | BAZWRITER : 61.6%, 3.4, 107.7ms, 0 | 20260506 02:29:35.188 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:29:35.188 | INFO | Uploaded 36GiB to GPU in 2.17024s (16.588GiB/s) | 20260506 02:29:35.188 | INFO | Downloaded 38GiB to GPU in 2.24328s (16.9395GiB/s) | 20260506 02:29:40.331 | INFO | Analyzing chunk frames = [8237568,8238080) | 20260506 02:29:40.331 | INFO | MemInfo: { | 20260506 02:29:40.331 | INFO | "gpu" : " 17 GB", | 20260506 02:29:40.331 | INFO | "huge" : "", | 20260506 02:29:40.331 | INFO | "huge_shared" : " 27 GB", | 20260506 02:29:40.331 | INFO | "malloc" : " 14 GB", | 20260506 02:29:40.331 | INFO | "pinned" : " 29 GB" | 20260506 02:29:40.331 | INFO | } | 20260506 02:29:40.331 | INFO | Perf report for the last 5.14319 seconds: | 20260506 02:29:40.331 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:29:40.331 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:40.331 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:40.331 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:40.331 | INFO | ANALYSIS : 60.9%, 3.0, 95.2ms, 0 | 20260506 02:29:40.331 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:40.331 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:40.331 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 02:29:40.331 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:40.331 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:40.331 | INFO | Uploaded 36GiB to GPU in 2.00216s (17.9806GiB/s) | 20260506 02:29:40.332 | INFO | Downloaded 34GiB to GPU in 1.99369s (17.0538GiB/s) | 20260506 02:29:45.476 | INFO | Analyzing chunk frames = [8238080,8238592) | 20260506 02:29:45.476 | INFO | MemInfo: { | 20260506 02:29:45.476 | INFO | "gpu" : " 17 GB", | 20260506 02:29:45.476 | INFO | "huge" : "", | 20260506 02:29:45.476 | INFO | "huge_shared" : " 27 GB", | 20260506 02:29:45.476 | INFO | "malloc" : " 14 GB", | 20260506 02:29:45.476 | INFO | "pinned" : " 29 GB" | 20260506 02:29:45.476 | INFO | } | 20260506 02:29:45.476 | INFO | Perf report for the last 5.14492 seconds: | 20260506 02:29:45.476 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:29:45.476 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:45.476 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:45.476 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:45.476 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260506 02:29:45.476 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:45.476 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:45.476 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 02:29:45.476 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:45.476 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:45.476 | INFO | Uploaded 36GiB to GPU in 2.08156s (17.2947GiB/s) | 20260506 02:29:45.476 | INFO | Downloaded 34GiB to GPU in 2.02133s (16.8206GiB/s) | 20260506 02:29:50.611 | INFO | Analyzing chunk frames = [8238592,8239104) | 20260506 02:29:50.611 | INFO | MemInfo: { | 20260506 02:29:50.611 | INFO | "gpu" : " 17 GB", | 20260506 02:29:50.611 | INFO | "huge" : "", | 20260506 02:29:50.611 | INFO | "huge_shared" : " 27 GB", | 20260506 02:29:50.611 | INFO | "malloc" : " 14 GB", | 20260506 02:29:50.611 | INFO | "pinned" : " 29 GB" | 20260506 02:29:50.611 | INFO | } | 20260506 02:29:50.611 | INFO | Perf report for the last 5.13504 seconds: | 20260506 02:29:50.611 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:29:50.611 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:50.611 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:50.611 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:50.611 | INFO | ANALYSIS : 61.0%, 3.0, 94.6ms, 0 | 20260506 02:29:50.611 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:50.611 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:50.611 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:29:50.611 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:50.611 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:50.611 | INFO | Uploaded 36GiB to GPU in 2.07872s (17.3183GiB/s) | 20260506 02:29:50.611 | INFO | Downloaded 34GiB to GPU in 2.00692s (16.9414GiB/s) | 20260506 02:29:55.747 | INFO | Analyzing chunk frames = [8239104,8239616) | 20260506 02:29:55.747 | INFO | MemInfo: { | 20260506 02:29:55.747 | INFO | "gpu" : " 17 GB", | 20260506 02:29:55.747 | INFO | "huge" : "", | 20260506 02:29:55.748 | INFO | "huge_shared" : " 27 GB", | 20260506 02:29:55.748 | INFO | "malloc" : " 14 GB", | 20260506 02:29:55.748 | INFO | "pinned" : " 29 GB" | 20260506 02:29:55.748 | INFO | } | 20260506 02:29:55.748 | INFO | Perf report for the last 5.13643 seconds: | 20260506 02:29:55.748 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:29:55.748 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:55.748 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:55.748 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:55.748 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260506 02:29:55.748 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:55.748 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:55.748 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 02:29:55.748 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:55.748 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:29:55.748 | INFO | Uploaded 36GiB to GPU in 2.06815s (17.4069GiB/s) | 20260506 02:29:55.748 | INFO | Downloaded 34GiB to GPU in 2.02863s (16.7601GiB/s) | 20260506 02:30:00.894 | INFO | Analyzing chunk frames = [8239616,8240128) | 20260506 02:30:00.894 | INFO | MemInfo: { | 20260506 02:30:00.894 | INFO | "gpu" : " 17 GB", | 20260506 02:30:00.894 | INFO | "huge" : "", | 20260506 02:30:00.894 | INFO | "huge_shared" : " 27 GB", | 20260506 02:30:00.894 | INFO | "malloc" : " 14 GB", | 20260506 02:30:00.894 | INFO | "pinned" : " 29 GB" | 20260506 02:30:00.894 | INFO | } | 20260506 02:30:00.894 | INFO | Perf report for the last 5.14681 seconds: | 20260506 02:30:00.894 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:30:00.894 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:00.894 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:00.894 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:00.894 | INFO | ANALYSIS : 61.1%, 3.0, 95.5ms, 0 | 20260506 02:30:00.894 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:00.895 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:00.895 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:30:00.895 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:00.895 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:00.895 | INFO | Uploaded 36GiB to GPU in 2.07353s (17.3617GiB/s) | 20260506 02:30:00.895 | INFO | Downloaded 34GiB to GPU in 2.04228s (16.648GiB/s) | 20260506 02:30:06.028 | INFO | Analyzing chunk frames = [8240128,8240640) | 20260506 02:30:06.028 | INFO | MemInfo: { | 20260506 02:30:06.028 | INFO | "gpu" : " 17 GB", | 20260506 02:30:06.028 | INFO | "huge" : "", | 20260506 02:30:06.028 | INFO | "huge_shared" : " 27 GB", | 20260506 02:30:06.028 | INFO | "malloc" : " 14 GB", | 20260506 02:30:06.028 | INFO | "pinned" : " 29 GB" | 20260506 02:30:06.028 | INFO | } | 20260506 02:30:06.028 | INFO | Perf report for the last 5.13374 seconds: | 20260506 02:30:06.028 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:30:06.028 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:06.028 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:06.028 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:06.028 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260506 02:30:06.028 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:06.028 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:06.028 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 02:30:06.028 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:06.028 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:06.029 | INFO | Uploaded 36GiB to GPU in 2.1135s (17.0334GiB/s) | 20260506 02:30:06.029 | INFO | Downloaded 34GiB to GPU in 2.09999s (16.1905GiB/s) | 20260506 02:30:11.166 | INFO | Analyzing chunk frames = [8240640,8241152) | 20260506 02:30:11.166 | INFO | MemInfo: { | 20260506 02:30:11.166 | INFO | "gpu" : " 17 GB", | 20260506 02:30:11.166 | INFO | "huge" : "", | 20260506 02:30:11.166 | INFO | "huge_shared" : " 27 GB", | 20260506 02:30:11.166 | INFO | "malloc" : " 14 GB", | 20260506 02:30:11.166 | INFO | "pinned" : " 29 GB" | 20260506 02:30:11.166 | INFO | } | 20260506 02:30:11.167 | INFO | Perf report for the last 5.13831 seconds: | 20260506 02:30:11.167 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:30:11.167 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:11.167 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:11.167 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:11.167 | INFO | ANALYSIS : 61.5%, 3.0, 96.0ms, 0 | 20260506 02:30:11.167 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:11.167 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:11.167 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 02:30:11.167 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:11.167 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:11.167 | INFO | Uploaded 36GiB to GPU in 2.06427s (17.4396GiB/s) | 20260506 02:30:11.167 | INFO | Downloaded 34GiB to GPU in 2.02363s (16.8015GiB/s) | 20260506 02:30:11.167 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 02:30:11.167 | INFO | Total mem usage: 15.3 GiB | 20260506 02:30:11.167 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 02:30:11.167 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.2%) | 20260506 02:30:11.167 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 02:30:11.167 | INFO | High Water Breakdown: | 20260506 02:30:11.167 | INFO | 60.9% : 8.54 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 02:30:11.167 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 02:30:11.167 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 02:30:11.167 | INFO | ------------------------------------------ | 20260506 02:30:11.167 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:30:11.167 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 02:30:11.167 | INFO | Total mem usage: 26.5 GiB | 20260506 02:30:11.167 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 02:30:11.167 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 02:30:11.167 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 02:30:11.167 | INFO | High Water Breakdown: | 20260506 02:30:11.167 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:30:11.167 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:30:11.167 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:30:11.167 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:30:11.167 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:30:11.167 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:30:11.167 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 02:30:11.167 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 02:30:11.167 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:30:11.167 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 02:30:11.167 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 02:30:11.167 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 02:30:11.167 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 02:30:11.167 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 02:30:11.167 | INFO | ------------------------------------------ | 20260506 02:30:11.167 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:30:11.167 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 02:30:11.167 | INFO | Total mem usage: 25.6 GiB | 20260506 02:30:11.167 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 02:30:11.167 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 02:30:11.167 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 02:30:11.167 | INFO | High Water Breakdown: | 20260506 02:30:11.167 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 02:30:11.167 | INFO | ------------------------------------------ | 20260506 02:30:11.167 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:30:11.167 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 02:30:11.167 | INFO | Total mem usage: 15.7 GiB | 20260506 02:30:11.167 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 02:30:11.168 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 02:30:11.168 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 02:30:11.168 | INFO | High Water Breakdown: | 20260506 02:30:11.168 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 02:30:11.168 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 02:30:11.168 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 02:30:11.168 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 02:30:11.168 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 02:30:11.168 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 02:30:11.168 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 02:30:11.168 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 02:30:11.168 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 02:30:11.168 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:30:11.168 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 02:30:11.168 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:30:11.168 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 02:30:11.168 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 02:30:11.168 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 02:30:11.168 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 02:30:11.168 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:30:11.168 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 02:30:11.168 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:30:11.168 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:30:11.168 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:30:11.168 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 02:30:11.168 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 02:30:11.168 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:30:11.168 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 02:30:11.168 | INFO | ------------------------------------------ | 20260506 02:30:11.168 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:30:11.168 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 02:30:11.168 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:30:11.168 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:30:11.168 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.38e+01, 1.34e+01 / 6.41e-01 | 20260506 02:30:11.168 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.05e-01, 1.04e-02 / 3.97e-03 | 20260506 02:30:11.168 | INFO | PerfReport: 11.85%, Baseline -- 3.71e+00 / 4.48e+00, 5.82e-01 / 2.17e-01 | 20260506 02:30:11.168 | INFO | PerfReport: 3.31%, Histogram -- 1.04e+00 / 1.54e+00, 1.96e-01 / 7.76e-02 | 20260506 02:30:11.168 | INFO | PerfReport: 15.99%, DME -- 4.01e+01 / 4.41e+01, 3.93e+00 / 8.38e-01 | 20260506 02:30:11.168 | INFO | PerfReport: 57.52%, FrameLabeling -- 1.80e+01 / 1.93e+01, 1.58e+00 / 1.06e+00 | 20260506 02:30:11.168 | INFO | PerfReport: 6.99%, PulseAccumulating -- 2.19e+00 / 2.74e+00, 4.13e-01 / 2.06e-01 | 20260506 02:30:11.168 | INFO | PerfReport: 3.00%, Metrics -- 9.41e-01 / 1.23e+00, 2.55e-01 / 9.78e-02 | 20260506 02:30:11.168 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.25e-01 / 3.74e-01, 5.46e-02 / 2.41e-02 | 20260506 02:30:11.168 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 02:30:11.168 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:30:11.168 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:30:11.168 | INFO | PerfReport: Full Execution -- 9.76e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260506 02:30:11.168 | INFO | PerfReport: 22.16%, Upload -- 2.16e+01 / 0.00e+00, 7.15e+00 / 0.00e+00 | 20260506 02:30:11.168 | INFO | PerfReport: 21.93%, Download -- 2.14e+01 / 0.00e+00, 7.58e+00 / 0.00e+00 | 20260506 02:30:11.168 | INFO | PerfReport: 32.10%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 02:30:11.168 | INFO | PerfReport: 1.99%, LockUpload -- 1.94e+00 / 0.00e+00, 7.79e+00 / 0.00e+00 | 20260506 02:30:11.168 | INFO | PerfReport: 1.30%, LockDownload -- 1.27e+00 / 0.00e+00, 2.97e+00 / 0.00e+00 | 20260506 02:30:11.168 | INFO | PerfReport: 20.51%, LockCompute -- 2.00e+01 / 0.00e+00, 1.75e+01 / 0.00e+00 | 20260506 02:30:11.168 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.99e-02 / 0.00e+00 | 20260506 02:30:11.292 | INFO | Starting a round of Baz IO | 20260506 02:30:14.423 | INFO | Populating RTMetrics report | 20260506 02:30:14.423 | INFO | RTMetrics report populated | 20260506 02:30:14.423 | INFO | Opening tmp RTMetrics json file | 20260506 02:30:14.424 | INFO | Serializing RTMetrics to json | 20260506 02:30:14.425 | INFO | Serialization done, swapping tmp file now... | 20260506 02:30:14.426 | INFO | RTMetrics done | 20260506 02:30:14.500 | INFO | Finished a round of Baz IO: Wrote 1.792 GiB in 3.207 seconds | 20260506 02:30:14.500 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.27% / 0.00% / 78.73% | 20260506 02:30:14.500 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0695666% / 7.94729e-06% / 7.94729e-05% | 20260506 02:30:14.500 | INFO | Baz IO overall: | 20260506 02:30:14.500 | INFO | BytesWritten : 4615663008310 | 20260506 02:30:14.500 | INFO | NumEvents : 2166852404000 | 20260506 02:30:16.316 | INFO | Analyzing chunk frames = [8241152,8241664) | 20260506 02:30:16.316 | INFO | MemInfo: { | 20260506 02:30:16.316 | INFO | "gpu" : " 17 GB", | 20260506 02:30:16.316 | INFO | "huge" : "", | 20260506 02:30:16.316 | INFO | "huge_shared" : " 27 GB", | 20260506 02:30:16.316 | INFO | "malloc" : " 14 GB", | 20260506 02:30:16.316 | INFO | "pinned" : " 29 GB" | 20260506 02:30:16.316 | INFO | } | 20260506 02:30:16.316 | INFO | Perf report for the last 5.1493 seconds: | 20260506 02:30:16.316 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:30:16.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:16.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:16.316 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:16.316 | INFO | ANALYSIS : 62.2%, 3.0, 97.4ms, 0 | 20260506 02:30:16.316 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:16.316 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260506 02:30:16.316 | INFO | PRE_HQ : 59.2%, 1.7, 53.8ms, 0 | 20260506 02:30:16.316 | INFO | BAZWRITER : 61.5%, 3.5, 109.6ms, 0 | 20260506 02:30:16.316 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:30:16.316 | INFO | Uploaded 36GiB to GPU in 2.14874s (16.754GiB/s) | 20260506 02:30:16.316 | INFO | Downloaded 38GiB to GPU in 2.22689s (17.0641GiB/s) | 20260506 02:30:21.450 | INFO | Analyzing chunk frames = [8241664,8242176) | 20260506 02:30:21.450 | INFO | MemInfo: { | 20260506 02:30:21.450 | INFO | "gpu" : " 17 GB", | 20260506 02:30:21.451 | INFO | "huge" : "", | 20260506 02:30:21.451 | INFO | "huge_shared" : " 27 GB", | 20260506 02:30:21.451 | INFO | "malloc" : " 14 GB", | 20260506 02:30:21.451 | INFO | "pinned" : " 29 GB" | 20260506 02:30:21.451 | INFO | } | 20260506 02:30:21.451 | INFO | Perf report for the last 5.13484 seconds: | 20260506 02:30:21.451 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:30:21.451 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:21.451 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:21.451 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:21.451 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 02:30:21.451 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:21.451 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:21.451 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 02:30:21.451 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:21.451 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:21.451 | INFO | Uploaded 36GiB to GPU in 1.91905s (18.7593GiB/s) | 20260506 02:30:21.451 | INFO | Downloaded 34GiB to GPU in 1.97612s (17.2054GiB/s) | 20260506 02:30:26.589 | INFO | Analyzing chunk frames = [8242176,8242688) | 20260506 02:30:26.589 | INFO | MemInfo: { | 20260506 02:30:26.589 | INFO | "gpu" : " 17 GB", | 20260506 02:30:26.589 | INFO | "huge" : "", | 20260506 02:30:26.589 | INFO | "huge_shared" : " 27 GB", | 20260506 02:30:26.590 | INFO | "malloc" : " 14 GB", | 20260506 02:30:26.590 | INFO | "pinned" : " 29 GB" | 20260506 02:30:26.590 | INFO | } | 20260506 02:30:26.590 | INFO | Perf report for the last 5.13893 seconds: | 20260506 02:30:26.590 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:30:26.590 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:26.590 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:26.590 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:26.590 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260506 02:30:26.590 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:26.590 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:26.590 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 02:30:26.590 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:26.590 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:26.590 | INFO | Uploaded 36GiB to GPU in 2.04827s (17.5758GiB/s) | 20260506 02:30:26.590 | INFO | Downloaded 34GiB to GPU in 2.00772s (16.9346GiB/s) | 20260506 02:30:31.725 | INFO | Analyzing chunk frames = [8242688,8243200) | 20260506 02:30:31.725 | INFO | MemInfo: { | 20260506 02:30:31.725 | INFO | "gpu" : " 17 GB", | 20260506 02:30:31.725 | INFO | "huge" : "", | 20260506 02:30:31.726 | INFO | "huge_shared" : " 27 GB", | 20260506 02:30:31.726 | INFO | "malloc" : " 14 GB", | 20260506 02:30:31.726 | INFO | "pinned" : " 29 GB" | 20260506 02:30:31.726 | INFO | } | 20260506 02:30:31.726 | INFO | Perf report for the last 5.13603 seconds: | 20260506 02:30:31.726 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:30:31.726 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:31.726 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:31.726 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:31.726 | INFO | ANALYSIS : 61.0%, 3.0, 94.7ms, 0 | 20260506 02:30:31.726 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:31.726 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:31.726 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:30:31.726 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:31.726 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:31.726 | INFO | Uploaded 36GiB to GPU in 2.08016s (17.3064GiB/s) | 20260506 02:30:31.726 | INFO | Downloaded 34GiB to GPU in 2.00524s (16.9556GiB/s) | 20260506 02:30:36.861 | INFO | Analyzing chunk frames = [8243200,8243712) | 20260506 02:30:36.861 | INFO | MemInfo: { | 20260506 02:30:36.861 | INFO | "gpu" : " 17 GB", | 20260506 02:30:36.861 | INFO | "huge" : "", | 20260506 02:30:36.861 | INFO | "huge_shared" : " 27 GB", | 20260506 02:30:36.861 | INFO | "malloc" : " 14 GB", | 20260506 02:30:36.861 | INFO | "pinned" : " 29 GB" | 20260506 02:30:36.861 | INFO | } | 20260506 02:30:36.861 | INFO | Perf report for the last 5.13577 seconds: | 20260506 02:30:36.861 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:30:36.861 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:36.861 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:36.862 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:36.862 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 02:30:36.862 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:36.862 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:36.862 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 02:30:36.862 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:36.862 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:36.862 | INFO | Uploaded 36GiB to GPU in 2.08054s (17.3032GiB/s) | 20260506 02:30:36.862 | INFO | Downloaded 34GiB to GPU in 2.03615s (16.6982GiB/s) | 20260506 02:30:42.000 | INFO | Analyzing chunk frames = [8243712,8244224) | 20260506 02:30:42.000 | INFO | MemInfo: { | 20260506 02:30:42.000 | INFO | "gpu" : " 17 GB", | 20260506 02:30:42.000 | INFO | "huge" : "", | 20260506 02:30:42.000 | INFO | "huge_shared" : " 27 GB", | 20260506 02:30:42.000 | INFO | "malloc" : " 14 GB", | 20260506 02:30:42.000 | INFO | "pinned" : " 29 GB" | 20260506 02:30:42.000 | INFO | } | 20260506 02:30:42.000 | INFO | Perf report for the last 5.13841 seconds: | 20260506 02:30:42.000 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:30:42.000 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:42.000 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:42.000 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:42.000 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260506 02:30:42.000 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:42.000 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:42.000 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 02:30:42.000 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:42.000 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:42.000 | INFO | Uploaded 36GiB to GPU in 2.11003s (17.0613GiB/s) | 20260506 02:30:42.000 | INFO | Downloaded 34GiB to GPU in 2.04275s (16.6442GiB/s) | 20260506 02:30:47.143 | INFO | Analyzing chunk frames = [8244224,8244736) | 20260506 02:30:47.143 | INFO | MemInfo: { | 20260506 02:30:47.143 | INFO | "gpu" : " 17 GB", | 20260506 02:30:47.143 | INFO | "huge" : "", | 20260506 02:30:47.143 | INFO | "huge_shared" : " 27 GB", | 20260506 02:30:47.143 | INFO | "malloc" : " 14 GB", | 20260506 02:30:47.143 | INFO | "pinned" : " 29 GB" | 20260506 02:30:47.143 | INFO | } | 20260506 02:30:47.143 | INFO | Perf report for the last 5.14321 seconds: | 20260506 02:30:47.143 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:30:47.143 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:47.143 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:47.143 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:47.143 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260506 02:30:47.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:47.143 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:47.143 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 02:30:47.143 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:47.143 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:47.143 | INFO | Uploaded 36GiB to GPU in 2.0953s (17.1813GiB/s) | 20260506 02:30:47.143 | INFO | Downloaded 34GiB to GPU in 2.08026s (16.3441GiB/s) | 20260506 02:30:52.290 | INFO | Analyzing chunk frames = [8244736,8245248) | 20260506 02:30:52.290 | INFO | MemInfo: { | 20260506 02:30:52.290 | INFO | "gpu" : " 17 GB", | 20260506 02:30:52.290 | INFO | "huge" : "", | 20260506 02:30:52.290 | INFO | "huge_shared" : " 27 GB", | 20260506 02:30:52.290 | INFO | "malloc" : " 14 GB", | 20260506 02:30:52.290 | INFO | "pinned" : " 29 GB" | 20260506 02:30:52.290 | INFO | } | 20260506 02:30:52.290 | INFO | Perf report for the last 5.14716 seconds: | 20260506 02:30:52.290 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:30:52.290 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:52.290 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:52.290 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:52.290 | INFO | ANALYSIS : 61.3%, 3.0, 95.8ms, 0 | 20260506 02:30:52.290 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:52.290 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:52.290 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 02:30:52.290 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:52.290 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:52.291 | INFO | Uploaded 36GiB to GPU in 2.03188s (17.7175GiB/s) | 20260506 02:30:52.291 | INFO | Downloaded 34GiB to GPU in 2.02087s (16.8245GiB/s) | 20260506 02:30:52.414 | INFO | Starting a round of Baz IO | 20260506 02:30:55.538 | INFO | Populating RTMetrics report | 20260506 02:30:55.538 | INFO | RTMetrics report populated | 20260506 02:30:55.538 | INFO | Opening tmp RTMetrics json file | 20260506 02:30:55.539 | INFO | Serializing RTMetrics to json | 20260506 02:30:55.540 | INFO | Serialization done, swapping tmp file now... | 20260506 02:30:55.541 | INFO | RTMetrics done | 20260506 02:30:55.610 | INFO | Finished a round of Baz IO: Wrote 1.791 GiB in 3.195 seconds | 20260506 02:30:55.610 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.26% / 0.00% / 78.73% | 20260506 02:30:55.610 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0699083% / 0% / 9.93411e-05% | 20260506 02:30:55.610 | INFO | Baz IO overall: | 20260506 02:30:55.610 | INFO | BytesWritten : 4617586349526 | 20260506 02:30:55.610 | INFO | NumEvents : 2167726434818 | 20260506 02:30:57.429 | INFO | Analyzing chunk frames = [8245248,8245760) | 20260506 02:30:57.429 | INFO | MemInfo: { | 20260506 02:30:57.429 | INFO | "gpu" : " 17 GB", | 20260506 02:30:57.429 | INFO | "huge" : "", | 20260506 02:30:57.429 | INFO | "huge_shared" : " 27 GB", | 20260506 02:30:57.429 | INFO | "malloc" : " 14 GB", | 20260506 02:30:57.429 | INFO | "pinned" : " 29 GB" | 20260506 02:30:57.429 | INFO | } | 20260506 02:30:57.429 | INFO | Perf report for the last 5.13884 seconds: | 20260506 02:30:57.429 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:30:57.429 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:57.429 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:57.429 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:57.429 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260506 02:30:57.429 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:30:57.429 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260506 02:30:57.429 | INFO | PRE_HQ : 59.2%, 1.7, 53.4ms, 0 | 20260506 02:30:57.429 | INFO | BAZWRITER : 61.3%, 3.4, 107.5ms, 0 | 20260506 02:30:57.429 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:30:57.429 | INFO | Uploaded 36GiB to GPU in 2.09819s (17.1577GiB/s) | 20260506 02:30:57.429 | INFO | Downloaded 38GiB to GPU in 2.23296s (17.0177GiB/s) | 20260506 02:31:02.558 | INFO | Analyzing chunk frames = [8245760,8246272) | 20260506 02:31:02.558 | INFO | MemInfo: { | 20260506 02:31:02.558 | INFO | "gpu" : " 17 GB", | 20260506 02:31:02.558 | INFO | "huge" : "", | 20260506 02:31:02.558 | INFO | "huge_shared" : " 27 GB", | 20260506 02:31:02.558 | INFO | "malloc" : " 14 GB", | 20260506 02:31:02.558 | INFO | "pinned" : " 29 GB" | 20260506 02:31:02.558 | INFO | } | 20260506 02:31:02.558 | INFO | Perf report for the last 5.12882 seconds: | 20260506 02:31:02.558 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:31:02.558 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:02.558 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:02.558 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:02.558 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260506 02:31:02.558 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:02.558 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:02.558 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:31:02.558 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:02.558 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:02.558 | INFO | Uploaded 36GiB to GPU in 1.96965s (18.2774GiB/s) | 20260506 02:31:02.558 | INFO | Downloaded 34GiB to GPU in 1.9858s (17.1216GiB/s) | 20260506 02:31:07.698 | INFO | Analyzing chunk frames = [8246272,8246784) | 20260506 02:31:07.698 | INFO | MemInfo: { | 20260506 02:31:07.698 | INFO | "gpu" : " 17 GB", | 20260506 02:31:07.698 | INFO | "huge" : "", | 20260506 02:31:07.698 | INFO | "huge_shared" : " 27 GB", | 20260506 02:31:07.698 | INFO | "malloc" : " 14 GB", | 20260506 02:31:07.698 | INFO | "pinned" : " 29 GB" | 20260506 02:31:07.698 | INFO | } | 20260506 02:31:07.698 | INFO | Perf report for the last 5.14018 seconds: | 20260506 02:31:07.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:31:07.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:07.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:07.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:07.698 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 02:31:07.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:07.698 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:07.698 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 02:31:07.698 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:07.698 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:07.698 | INFO | Uploaded 36GiB to GPU in 2.07865s (17.3189GiB/s) | 20260506 02:31:07.698 | INFO | Downloaded 34GiB to GPU in 2.02027s (16.8294GiB/s) | 20260506 02:31:12.842 | INFO | Analyzing chunk frames = [8246784,8247296) | 20260506 02:31:12.842 | INFO | MemInfo: { | 20260506 02:31:12.842 | INFO | "gpu" : " 17 GB", | 20260506 02:31:12.842 | INFO | "huge" : "", | 20260506 02:31:12.842 | INFO | "huge_shared" : " 27 GB", | 20260506 02:31:12.842 | INFO | "malloc" : " 14 GB", | 20260506 02:31:12.842 | INFO | "pinned" : " 29 GB" | 20260506 02:31:12.842 | INFO | } | 20260506 02:31:12.842 | INFO | Perf report for the last 5.14425 seconds: | 20260506 02:31:12.842 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:31:12.842 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:12.842 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:12.842 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:12.842 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260506 02:31:12.843 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:12.843 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:12.843 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 02:31:12.843 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:12.843 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:12.843 | INFO | Uploaded 36GiB to GPU in 2.0989s (17.1518GiB/s) | 20260506 02:31:12.843 | INFO | Downloaded 34GiB to GPU in 2.02367s (16.8011GiB/s) | 20260506 02:31:17.984 | INFO | Analyzing chunk frames = [8247296,8247808) | 20260506 02:31:17.984 | INFO | MemInfo: { | 20260506 02:31:17.984 | INFO | "gpu" : " 17 GB", | 20260506 02:31:17.984 | INFO | "huge" : "", | 20260506 02:31:17.984 | INFO | "huge_shared" : " 27 GB", | 20260506 02:31:17.984 | INFO | "malloc" : " 14 GB", | 20260506 02:31:17.984 | INFO | "pinned" : " 29 GB" | 20260506 02:31:17.984 | INFO | } | 20260506 02:31:17.984 | INFO | Perf report for the last 5.1416 seconds: | 20260506 02:31:17.984 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:31:17.984 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:17.984 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:17.984 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:17.984 | INFO | ANALYSIS : 60.6%, 3.0, 94.7ms, 0 | 20260506 02:31:17.984 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:17.984 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:17.984 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:31:17.984 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:17.984 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:17.984 | INFO | Uploaded 36GiB to GPU in 2.08868s (17.2358GiB/s) | 20260506 02:31:17.984 | INFO | Downloaded 34GiB to GPU in 2.0308s (16.7422GiB/s) | 20260506 02:31:23.122 | INFO | Analyzing chunk frames = [8247808,8248320) | 20260506 02:31:23.122 | INFO | MemInfo: { | 20260506 02:31:23.122 | INFO | "gpu" : " 17 GB", | 20260506 02:31:23.122 | INFO | "huge" : "", | 20260506 02:31:23.122 | INFO | "huge_shared" : " 27 GB", | 20260506 02:31:23.122 | INFO | "malloc" : " 14 GB", | 20260506 02:31:23.122 | INFO | "pinned" : " 29 GB" | 20260506 02:31:23.122 | INFO | } | 20260506 02:31:23.122 | INFO | Perf report for the last 5.13773 seconds: | 20260506 02:31:23.122 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:31:23.122 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:23.122 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:23.122 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:23.122 | INFO | ANALYSIS : 60.9%, 3.0, 95.1ms, 0 | 20260506 02:31:23.122 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:23.122 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:23.122 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 02:31:23.122 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:23.122 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:23.122 | INFO | Uploaded 36GiB to GPU in 2.08773s (17.2436GiB/s) | 20260506 02:31:23.122 | INFO | Downloaded 34GiB to GPU in 2.04056s (16.6621GiB/s) | 20260506 02:31:28.261 | INFO | Analyzing chunk frames = [8248320,8248832) | 20260506 02:31:28.261 | INFO | MemInfo: { | 20260506 02:31:28.261 | INFO | "gpu" : " 17 GB", | 20260506 02:31:28.261 | INFO | "huge" : "", | 20260506 02:31:28.261 | INFO | "huge_shared" : " 27 GB", | 20260506 02:31:28.261 | INFO | "malloc" : " 14 GB", | 20260506 02:31:28.261 | INFO | "pinned" : " 29 GB" | 20260506 02:31:28.261 | INFO | } | 20260506 02:31:28.261 | INFO | Perf report for the last 5.13963 seconds: | 20260506 02:31:28.261 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:31:28.261 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:28.261 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:28.262 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:28.262 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260506 02:31:28.262 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:28.262 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:28.262 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 02:31:28.262 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:28.262 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:28.262 | INFO | Uploaded 36GiB to GPU in 2.08171s (17.2935GiB/s) | 20260506 02:31:28.262 | INFO | Downloaded 34GiB to GPU in 2.07462s (16.3885GiB/s) | 20260506 02:31:33.401 | INFO | Analyzing chunk frames = [8248832,8249344) | 20260506 02:31:33.401 | INFO | MemInfo: { | 20260506 02:31:33.401 | INFO | "gpu" : " 17 GB", | 20260506 02:31:33.401 | INFO | "huge" : "", | 20260506 02:31:33.401 | INFO | "huge_shared" : " 27 GB", | 20260506 02:31:33.401 | INFO | "malloc" : " 14 GB", | 20260506 02:31:33.401 | INFO | "pinned" : " 29 GB" | 20260506 02:31:33.401 | INFO | } | 20260506 02:31:33.401 | INFO | Perf report for the last 5.13993 seconds: | 20260506 02:31:33.401 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:31:33.401 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:33.401 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:33.401 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:33.401 | INFO | ANALYSIS : 61.5%, 3.0, 96.1ms, 0 | 20260506 02:31:33.401 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:33.401 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:33.402 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 02:31:33.402 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:33.402 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:33.402 | INFO | Uploaded 36GiB to GPU in 2.07401s (17.3576GiB/s) | 20260506 02:31:33.402 | INFO | Downloaded 34GiB to GPU in 2.02601s (16.7818GiB/s) | 20260506 02:31:33.402 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 02:31:33.402 | INFO | Total mem usage: 15.3 GiB | 20260506 02:31:33.402 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 02:31:33.402 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.17%) | 20260506 02:31:33.402 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 02:31:33.402 | INFO | High Water Breakdown: | 20260506 02:31:33.402 | INFO | 60.9% : 8.55 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 02:31:33.402 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 02:31:33.402 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 02:31:33.402 | INFO | ------------------------------------------ | 20260506 02:31:33.402 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:31:33.402 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 02:31:33.402 | INFO | Total mem usage: 26.5 GiB | 20260506 02:31:33.402 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 02:31:33.402 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 02:31:33.402 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 02:31:33.402 | INFO | High Water Breakdown: | 20260506 02:31:33.402 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:31:33.402 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:31:33.402 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:31:33.402 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:31:33.402 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:31:33.402 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:31:33.402 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 02:31:33.402 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 02:31:33.402 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:31:33.402 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 02:31:33.402 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 02:31:33.402 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 02:31:33.402 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 02:31:33.402 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 02:31:33.402 | INFO | ------------------------------------------ | 20260506 02:31:33.402 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:31:33.402 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 02:31:33.402 | INFO | Total mem usage: 25.6 GiB | 20260506 02:31:33.402 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 02:31:33.402 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 02:31:33.402 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 02:31:33.402 | INFO | High Water Breakdown: | 20260506 02:31:33.402 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 02:31:33.402 | INFO | ------------------------------------------ | 20260506 02:31:33.402 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:31:33.402 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 02:31:33.402 | INFO | Total mem usage: 15.7 GiB | 20260506 02:31:33.402 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 02:31:33.402 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 02:31:33.402 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 02:31:33.402 | INFO | High Water Breakdown: | 20260506 02:31:33.402 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 02:31:33.402 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 02:31:33.402 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 02:31:33.402 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 02:31:33.402 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 02:31:33.402 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 02:31:33.402 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 02:31:33.402 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 02:31:33.402 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 02:31:33.402 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:31:33.402 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 02:31:33.402 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:31:33.402 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 02:31:33.403 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 02:31:33.403 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 02:31:33.403 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 02:31:33.403 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:31:33.403 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 02:31:33.403 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:31:33.403 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:31:33.403 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:31:33.403 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 02:31:33.403 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 02:31:33.403 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:31:33.403 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 02:31:33.403 | INFO | ------------------------------------------ | 20260506 02:31:33.403 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:31:33.403 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 02:31:33.403 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:31:33.403 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:31:33.403 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.37e+01, 1.35e+01 / 6.64e-01 | 20260506 02:31:33.403 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.05e-01, 1.03e-02 / 5.67e-03 | 20260506 02:31:33.403 | INFO | PerfReport: 11.86%, Baseline -- 3.72e+00 / 4.44e+00, 5.93e-01 / 3.21e-01 | 20260506 02:31:33.403 | INFO | PerfReport: 3.31%, Histogram -- 1.04e+00 / 1.51e+00, 1.99e-01 / 1.15e-01 | 20260506 02:31:33.403 | INFO | PerfReport: 15.94%, DME -- 4.00e+01 / 4.40e+01, 3.73e+00 / 8.74e-01 | 20260506 02:31:33.403 | INFO | PerfReport: 57.60%, FrameLabeling -- 1.81e+01 / 1.94e+01, 1.66e+00 / 1.14e+00 | 20260506 02:31:33.403 | INFO | PerfReport: 6.95%, PulseAccumulating -- 2.18e+00 / 2.68e+00, 4.10e-01 / 2.67e-01 | 20260506 02:31:33.403 | INFO | PerfReport: 3.00%, Metrics -- 9.40e-01 / 1.22e+00, 2.55e-01 / 1.02e-01 | 20260506 02:31:33.403 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.25e-01 / 3.75e-01, 5.44e-02 / 2.90e-02 | 20260506 02:31:33.403 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 02:31:33.403 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:31:33.403 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:31:33.403 | INFO | PerfReport: Full Execution -- 9.76e+01 / -nan, 2.41e+01 / 0.00e+00 | 20260506 02:31:33.403 | INFO | PerfReport: 22.08%, Upload -- 2.15e+01 / 0.00e+00, 7.08e+00 / 0.00e+00 | 20260506 02:31:33.403 | INFO | PerfReport: 21.90%, Download -- 2.14e+01 / 0.00e+00, 7.58e+00 / 0.00e+00 | 20260506 02:31:33.403 | INFO | PerfReport: 32.15%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 02:31:33.403 | INFO | PerfReport: 2.00%, LockUpload -- 1.95e+00 / 0.00e+00, 7.87e+00 / 0.00e+00 | 20260506 02:31:33.403 | INFO | PerfReport: 1.26%, LockDownload -- 1.23e+00 / 0.00e+00, 2.93e+00 / 0.00e+00 | 20260506 02:31:33.403 | INFO | PerfReport: 20.60%, LockCompute -- 2.01e+01 / 0.00e+00, 1.78e+01 / 0.00e+00 | 20260506 02:31:33.403 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.36e-02 / 0.00e+00 | 20260506 02:31:33.527 | INFO | Starting a round of Baz IO | 20260506 02:31:36.657 | INFO | Populating RTMetrics report | 20260506 02:31:36.657 | INFO | RTMetrics report populated | 20260506 02:31:36.657 | INFO | Opening tmp RTMetrics json file | 20260506 02:31:36.658 | INFO | Serializing RTMetrics to json | 20260506 02:31:36.659 | INFO | Serialization done, swapping tmp file now... | 20260506 02:31:36.660 | INFO | RTMetrics done | 20260506 02:31:36.734 | INFO | Finished a round of Baz IO: Wrote 1.791 GiB in 3.207 seconds | 20260506 02:31:36.734 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.26% / 0.00% / 78.74% | 20260506 02:31:36.734 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0693123% / 3.97364e-06% / 7.15256e-05% | 20260506 02:31:36.734 | INFO | Baz IO overall: | 20260506 02:31:36.734 | INFO | BytesWritten : 4619509033071 | 20260506 02:31:36.734 | INFO | NumEvents : 2168600029818 | 20260506 02:31:38.540 | INFO | Analyzing chunk frames = [8249344,8249856) | 20260506 02:31:38.540 | INFO | MemInfo: { | 20260506 02:31:38.540 | INFO | "gpu" : " 17 GB", | 20260506 02:31:38.540 | INFO | "huge" : "", | 20260506 02:31:38.540 | INFO | "huge_shared" : " 27 GB", | 20260506 02:31:38.540 | INFO | "malloc" : " 14 GB", | 20260506 02:31:38.540 | INFO | "pinned" : " 29 GB" | 20260506 02:31:38.540 | INFO | } | 20260506 02:31:38.540 | INFO | Perf report for the last 5.13866 seconds: | 20260506 02:31:38.540 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:31:38.540 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:38.540 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:38.540 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:38.540 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260506 02:31:38.540 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:38.540 | INFO | RT_METRICS : 42.8%, 1.0, 22.4ms, 0 | 20260506 02:31:38.540 | INFO | PRE_HQ : 59.1%, 1.7, 53.3ms, 0 | 20260506 02:31:38.540 | INFO | BAZWRITER : 61.4%, 3.4, 107.0ms, 0 | 20260506 02:31:38.540 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:31:38.540 | INFO | Uploaded 36GiB to GPU in 2.10998s (17.0618GiB/s) | 20260506 02:31:38.540 | INFO | Downloaded 38GiB to GPU in 2.22395s (17.0867GiB/s) | 20260506 02:31:43.674 | INFO | Analyzing chunk frames = [8249856,8250368) | 20260506 02:31:43.674 | INFO | MemInfo: { | 20260506 02:31:43.674 | INFO | "gpu" : " 17 GB", | 20260506 02:31:43.674 | INFO | "huge" : "", | 20260506 02:31:43.674 | INFO | "huge_shared" : " 27 GB", | 20260506 02:31:43.675 | INFO | "malloc" : " 14 GB", | 20260506 02:31:43.675 | INFO | "pinned" : " 29 GB" | 20260506 02:31:43.675 | INFO | } | 20260506 02:31:43.675 | INFO | Perf report for the last 5.13462 seconds: | 20260506 02:31:43.675 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:31:43.675 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:43.675 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:43.675 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:43.675 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260506 02:31:43.675 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:43.675 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:43.675 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 02:31:43.675 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:43.675 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:43.675 | INFO | Uploaded 36GiB to GPU in 2.01808s (17.8387GiB/s) | 20260506 02:31:43.675 | INFO | Downloaded 34GiB to GPU in 1.99377s (17.0531GiB/s) | 20260506 02:31:48.811 | INFO | Analyzing chunk frames = [8250368,8250880) | 20260506 02:31:48.811 | INFO | MemInfo: { | 20260506 02:31:48.811 | INFO | "gpu" : " 17 GB", | 20260506 02:31:48.811 | INFO | "huge" : "", | 20260506 02:31:48.811 | INFO | "huge_shared" : " 27 GB", | 20260506 02:31:48.811 | INFO | "malloc" : " 14 GB", | 20260506 02:31:48.811 | INFO | "pinned" : " 29 GB" | 20260506 02:31:48.811 | INFO | } | 20260506 02:31:48.812 | INFO | Perf report for the last 5.13684 seconds: | 20260506 02:31:48.812 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:31:48.812 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:48.812 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:48.812 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:48.812 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 02:31:48.812 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:48.812 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:48.812 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:31:48.812 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:48.812 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:48.812 | INFO | Uploaded 36GiB to GPU in 2.08726s (17.2475GiB/s) | 20260506 02:31:48.812 | INFO | Downloaded 34GiB to GPU in 2.0238s (16.8001GiB/s) | 20260506 02:31:53.958 | INFO | Analyzing chunk frames = [8250880,8251392) | 20260506 02:31:53.958 | INFO | MemInfo: { | 20260506 02:31:53.958 | INFO | "gpu" : " 17 GB", | 20260506 02:31:53.958 | INFO | "huge" : "", | 20260506 02:31:53.958 | INFO | "huge_shared" : " 27 GB", | 20260506 02:31:53.958 | INFO | "malloc" : " 14 GB", | 20260506 02:31:53.958 | INFO | "pinned" : " 29 GB" | 20260506 02:31:53.958 | INFO | } | 20260506 02:31:53.959 | INFO | Perf report for the last 5.14701 seconds: | 20260506 02:31:53.959 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:31:53.959 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:53.959 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:53.959 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:53.959 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 02:31:53.959 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:53.959 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:53.959 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:31:53.959 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:53.959 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:53.959 | INFO | Uploaded 36GiB to GPU in 2.14568s (16.7779GiB/s) | 20260506 02:31:53.959 | INFO | Downloaded 34GiB to GPU in 2.03882s (16.6763GiB/s) | 20260506 02:31:59.097 | INFO | Analyzing chunk frames = [8251392,8251904) | 20260506 02:31:59.097 | INFO | MemInfo: { | 20260506 02:31:59.097 | INFO | "gpu" : " 17 GB", | 20260506 02:31:59.097 | INFO | "huge" : "", | 20260506 02:31:59.097 | INFO | "huge_shared" : " 27 GB", | 20260506 02:31:59.097 | INFO | "malloc" : " 14 GB", | 20260506 02:31:59.097 | INFO | "pinned" : " 29 GB" | 20260506 02:31:59.097 | INFO | } | 20260506 02:31:59.097 | INFO | Perf report for the last 5.1388 seconds: | 20260506 02:31:59.097 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:31:59.097 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:59.097 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:59.097 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:59.097 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 02:31:59.097 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:59.097 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:59.097 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:31:59.098 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:59.098 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:31:59.098 | INFO | Uploaded 36GiB to GPU in 2.09942s (17.1476GiB/s) | 20260506 02:31:59.098 | INFO | Downloaded 34GiB to GPU in 2.02811s (16.7644GiB/s) | 20260506 02:32:04.241 | INFO | Analyzing chunk frames = [8251904,8252416) | 20260506 02:32:04.241 | INFO | MemInfo: { | 20260506 02:32:04.241 | INFO | "gpu" : " 17 GB", | 20260506 02:32:04.241 | INFO | "huge" : "", | 20260506 02:32:04.241 | INFO | "huge_shared" : " 27 GB", | 20260506 02:32:04.241 | INFO | "malloc" : " 14 GB", | 20260506 02:32:04.241 | INFO | "pinned" : " 29 GB" | 20260506 02:32:04.241 | INFO | } | 20260506 02:32:04.241 | INFO | Perf report for the last 5.14331 seconds: | 20260506 02:32:04.241 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:32:04.241 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:04.241 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:04.241 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:04.241 | INFO | ANALYSIS : 60.5%, 3.0, 94.6ms, 0 | 20260506 02:32:04.241 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:04.241 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:04.241 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 02:32:04.241 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:04.241 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:04.241 | INFO | Uploaded 36GiB to GPU in 2.01068s (17.9044GiB/s) | 20260506 02:32:04.241 | INFO | Downloaded 34GiB to GPU in 2.01696s (16.857GiB/s) | 20260506 02:32:09.377 | INFO | Analyzing chunk frames = [8252416,8252928) | 20260506 02:32:09.377 | INFO | MemInfo: { | 20260506 02:32:09.377 | INFO | "gpu" : " 17 GB", | 20260506 02:32:09.377 | INFO | "huge" : "", | 20260506 02:32:09.377 | INFO | "huge_shared" : " 27 GB", | 20260506 02:32:09.377 | INFO | "malloc" : " 14 GB", | 20260506 02:32:09.377 | INFO | "pinned" : " 29 GB" | 20260506 02:32:09.377 | INFO | } | 20260506 02:32:09.377 | INFO | Perf report for the last 5.13653 seconds: | 20260506 02:32:09.377 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:32:09.377 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:09.377 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:09.377 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:09.377 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 02:32:09.377 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:09.377 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:09.377 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 02:32:09.377 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:09.377 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:09.377 | INFO | Uploaded 36GiB to GPU in 2.04681s (17.5883GiB/s) | 20260506 02:32:09.378 | INFO | Downloaded 34GiB to GPU in 2.0621s (16.488GiB/s) | 20260506 02:32:14.508 | INFO | Analyzing chunk frames = [8252928,8253440) | 20260506 02:32:14.508 | INFO | MemInfo: { | 20260506 02:32:14.508 | INFO | "gpu" : " 17 GB", | 20260506 02:32:14.508 | INFO | "huge" : "", | 20260506 02:32:14.508 | INFO | "huge_shared" : " 27 GB", | 20260506 02:32:14.508 | INFO | "malloc" : " 14 GB", | 20260506 02:32:14.508 | INFO | "pinned" : " 29 GB" | 20260506 02:32:14.508 | INFO | } | 20260506 02:32:14.508 | INFO | Perf report for the last 5.13036 seconds: | 20260506 02:32:14.508 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:32:14.508 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:14.508 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:14.508 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:14.508 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 02:32:14.508 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:14.508 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:14.508 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 02:32:14.508 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:14.508 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:14.508 | INFO | Uploaded 36GiB to GPU in 2.05467s (17.521GiB/s) | 20260506 02:32:14.508 | INFO | Downloaded 34GiB to GPU in 2.02684s (16.7748GiB/s) | 20260506 02:32:14.625 | INFO | Starting a round of Baz IO | 20260506 02:32:17.772 | INFO | Populating RTMetrics report | 20260506 02:32:17.773 | INFO | RTMetrics report populated | 20260506 02:32:17.773 | INFO | Opening tmp RTMetrics json file | 20260506 02:32:17.775 | INFO | Serializing RTMetrics to json | 20260506 02:32:17.776 | INFO | Serialization done, swapping tmp file now... | 20260506 02:32:17.776 | INFO | RTMetrics done | 20260506 02:32:18.062 | INFO | Finished a round of Baz IO: Wrote 1.792 GiB in 3.436 seconds | 20260506 02:32:18.062 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.26% / 0.00% / 78.74% | 20260506 02:32:18.062 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0695507% / 3.97364e-06% / 9.13938e-05% | 20260506 02:32:18.062 | INFO | Baz IO overall: | 20260506 02:32:18.062 | INFO | BytesWritten : 4621433516983 | 20260506 02:32:18.063 | INFO | NumEvents : 2169475263149 | 20260506 02:32:19.652 | INFO | Analyzing chunk frames = [8253440,8253952) | 20260506 02:32:19.652 | INFO | MemInfo: { | 20260506 02:32:19.652 | INFO | "gpu" : " 17 GB", | 20260506 02:32:19.652 | INFO | "huge" : "", | 20260506 02:32:19.652 | INFO | "huge_shared" : " 27 GB", | 20260506 02:32:19.652 | INFO | "malloc" : " 14 GB", | 20260506 02:32:19.652 | INFO | "pinned" : " 29 GB" | 20260506 02:32:19.652 | INFO | } | 20260506 02:32:19.652 | INFO | Perf report for the last 5.14444 seconds: | 20260506 02:32:19.652 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:32:19.652 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:19.652 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:19.652 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:19.652 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260506 02:32:19.652 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:19.652 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260506 02:32:19.652 | INFO | PRE_HQ : 59.6%, 1.7, 53.5ms, 0 | 20260506 02:32:19.652 | INFO | BAZWRITER : 65.9%, 3.4, 116.5ms, 0 | 20260506 02:32:19.652 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:32:19.652 | INFO | Uploaded 36GiB to GPU in 2.19086s (16.4319GiB/s) | 20260506 02:32:19.652 | INFO | Downloaded 38GiB to GPU in 2.25028s (16.8868GiB/s) | 20260506 02:32:24.792 | INFO | Analyzing chunk frames = [8253952,8254464) | 20260506 02:32:24.792 | INFO | MemInfo: { | 20260506 02:32:24.792 | INFO | "gpu" : " 17 GB", | 20260506 02:32:24.792 | INFO | "huge" : "", | 20260506 02:32:24.792 | INFO | "huge_shared" : " 27 GB", | 20260506 02:32:24.792 | INFO | "malloc" : " 14 GB", | 20260506 02:32:24.792 | INFO | "pinned" : " 29 GB" | 20260506 02:32:24.792 | INFO | } | 20260506 02:32:24.793 | INFO | Perf report for the last 5.14042 seconds: | 20260506 02:32:24.793 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:32:24.793 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:24.793 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:24.793 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:24.793 | INFO | ANALYSIS : 60.6%, 3.0, 94.7ms, 0 | 20260506 02:32:24.793 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:24.793 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:24.793 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 02:32:24.793 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:24.793 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:24.793 | INFO | Uploaded 36GiB to GPU in 1.9345s (18.6095GiB/s) | 20260506 02:32:24.793 | INFO | Downloaded 34GiB to GPU in 1.9806s (17.1665GiB/s) | 20260506 02:32:29.934 | INFO | Analyzing chunk frames = [8254464,8254976) | 20260506 02:32:29.934 | INFO | MemInfo: { | 20260506 02:32:29.934 | INFO | "gpu" : " 17 GB", | 20260506 02:32:29.934 | INFO | "huge" : "", | 20260506 02:32:29.934 | INFO | "huge_shared" : " 27 GB", | 20260506 02:32:29.934 | INFO | "malloc" : " 14 GB", | 20260506 02:32:29.934 | INFO | "pinned" : " 29 GB" | 20260506 02:32:29.934 | INFO | } | 20260506 02:32:29.934 | INFO | Perf report for the last 5.14126 seconds: | 20260506 02:32:29.934 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:32:29.934 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:29.934 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:29.934 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:29.934 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 02:32:29.934 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:29.934 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:29.934 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:32:29.934 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:29.934 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:29.934 | INFO | Uploaded 36GiB to GPU in 2.08103s (17.2991GiB/s) | 20260506 02:32:29.934 | INFO | Downloaded 34GiB to GPU in 2.02629s (16.7794GiB/s) | 20260506 02:32:35.069 | INFO | Analyzing chunk frames = [8254976,8255488) | 20260506 02:32:35.069 | INFO | MemInfo: { | 20260506 02:32:35.069 | INFO | "gpu" : " 17 GB", | 20260506 02:32:35.069 | INFO | "huge" : "", | 20260506 02:32:35.069 | INFO | "huge_shared" : " 27 GB", | 20260506 02:32:35.069 | INFO | "malloc" : " 14 GB", | 20260506 02:32:35.069 | INFO | "pinned" : " 29 GB" | 20260506 02:32:35.069 | INFO | } | 20260506 02:32:35.069 | INFO | Perf report for the last 5.13511 seconds: | 20260506 02:32:35.069 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:32:35.069 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:35.069 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:35.069 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:35.069 | INFO | ANALYSIS : 61.0%, 3.0, 94.7ms, 0 | 20260506 02:32:35.069 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:35.069 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:35.069 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:32:35.069 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:35.069 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:35.069 | INFO | Uploaded 36GiB to GPU in 2.07432s (17.3551GiB/s) | 20260506 02:32:35.069 | INFO | Downloaded 34GiB to GPU in 2.00577s (16.9511GiB/s) | 20260506 02:32:40.213 | INFO | Analyzing chunk frames = [8255488,8256000) | 20260506 02:32:40.213 | INFO | MemInfo: { | 20260506 02:32:40.213 | INFO | "gpu" : " 17 GB", | 20260506 02:32:40.213 | INFO | "huge" : "", | 20260506 02:32:40.213 | INFO | "huge_shared" : " 27 GB", | 20260506 02:32:40.213 | INFO | "malloc" : " 14 GB", | 20260506 02:32:40.213 | INFO | "pinned" : " 29 GB" | 20260506 02:32:40.213 | INFO | } | 20260506 02:32:40.213 | INFO | Perf report for the last 5.14395 seconds: | 20260506 02:32:40.213 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:32:40.213 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:40.213 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:40.213 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:40.213 | INFO | ANALYSIS : 60.7%, 3.0, 94.9ms, 0 | 20260506 02:32:40.213 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:40.213 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:40.213 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 02:32:40.213 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:40.213 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:40.213 | INFO | Uploaded 36GiB to GPU in 2.08526s (17.264GiB/s) | 20260506 02:32:40.213 | INFO | Downloaded 34GiB to GPU in 2.0354s (16.7044GiB/s) | 20260506 02:32:45.351 | INFO | Analyzing chunk frames = [8256000,8256512) | 20260506 02:32:45.351 | INFO | MemInfo: { | 20260506 02:32:45.351 | INFO | "gpu" : " 17 GB", | 20260506 02:32:45.351 | INFO | "huge" : "", | 20260506 02:32:45.351 | INFO | "huge_shared" : " 27 GB", | 20260506 02:32:45.351 | INFO | "malloc" : " 14 GB", | 20260506 02:32:45.351 | INFO | "pinned" : " 29 GB" | 20260506 02:32:45.351 | INFO | } | 20260506 02:32:45.351 | INFO | Perf report for the last 5.13851 seconds: | 20260506 02:32:45.351 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:32:45.351 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:45.351 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:45.351 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:45.351 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 02:32:45.351 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:45.352 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:45.352 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 02:32:45.352 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:45.352 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:45.352 | INFO | Uploaded 36GiB to GPU in 2.09285s (17.2014GiB/s) | 20260506 02:32:45.352 | INFO | Downloaded 34GiB to GPU in 2.03754s (16.6868GiB/s) | 20260506 02:32:50.485 | INFO | Analyzing chunk frames = [8256512,8257024) | 20260506 02:32:50.485 | INFO | MemInfo: { | 20260506 02:32:50.485 | INFO | "gpu" : " 17 GB", | 20260506 02:32:50.485 | INFO | "huge" : "", | 20260506 02:32:50.485 | INFO | "huge_shared" : " 27 GB", | 20260506 02:32:50.485 | INFO | "malloc" : " 14 GB", | 20260506 02:32:50.486 | INFO | "pinned" : " 29 GB" | 20260506 02:32:50.486 | INFO | } | 20260506 02:32:50.486 | INFO | Perf report for the last 5.13416 seconds: | 20260506 02:32:50.486 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:32:50.486 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:50.486 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:50.486 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:50.486 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 02:32:50.486 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:50.486 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:50.486 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 02:32:50.486 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:50.486 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:50.486 | INFO | Uploaded 36GiB to GPU in 2.08065s (17.3023GiB/s) | 20260506 02:32:50.486 | INFO | Downloaded 34GiB to GPU in 2.06421s (16.4712GiB/s) | 20260506 02:32:55.628 | INFO | Analyzing chunk frames = [8257024,8257536) | 20260506 02:32:55.628 | INFO | MemInfo: { | 20260506 02:32:55.628 | INFO | "gpu" : " 17 GB", | 20260506 02:32:55.628 | INFO | "huge" : "", | 20260506 02:32:55.628 | INFO | "huge_shared" : " 27 GB", | 20260506 02:32:55.628 | INFO | "malloc" : " 14 GB", | 20260506 02:32:55.628 | INFO | "pinned" : " 29 GB" | 20260506 02:32:55.628 | INFO | } | 20260506 02:32:55.628 | INFO | Perf report for the last 5.14282 seconds: | 20260506 02:32:55.628 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:32:55.628 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:55.628 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:55.628 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:55.628 | INFO | ANALYSIS : 61.4%, 3.0, 96.0ms, 0 | 20260506 02:32:55.628 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:55.628 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:55.629 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 02:32:55.629 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:55.629 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:32:55.629 | INFO | Uploaded 36GiB to GPU in 2.06754s (17.412GiB/s) | 20260506 02:32:55.629 | INFO | Downloaded 34GiB to GPU in 2.02475s (16.7922GiB/s) | 20260506 02:32:55.629 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 02:32:55.629 | INFO | Total mem usage: 15.3 GiB | 20260506 02:32:55.629 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 02:32:55.629 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.2%) | 20260506 02:32:55.629 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 02:32:55.629 | INFO | High Water Breakdown: | 20260506 02:32:55.629 | INFO | 60.9% : 8.54 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 02:32:55.629 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 02:32:55.629 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 02:32:55.629 | INFO | ------------------------------------------ | 20260506 02:32:55.629 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:32:55.629 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 02:32:55.629 | INFO | Total mem usage: 26.5 GiB | 20260506 02:32:55.629 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 02:32:55.629 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 02:32:55.629 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 02:32:55.629 | INFO | High Water Breakdown: | 20260506 02:32:55.629 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:32:55.629 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:32:55.629 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:32:55.629 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:32:55.629 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:32:55.629 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:32:55.629 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 02:32:55.629 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 02:32:55.629 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:32:55.629 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 02:32:55.629 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 02:32:55.629 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 02:32:55.629 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 02:32:55.629 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 02:32:55.629 | INFO | ------------------------------------------ | 20260506 02:32:55.629 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:32:55.629 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 02:32:55.629 | INFO | Total mem usage: 25.6 GiB | 20260506 02:32:55.629 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 02:32:55.629 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 02:32:55.629 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 02:32:55.629 | INFO | High Water Breakdown: | 20260506 02:32:55.629 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 02:32:55.629 | INFO | ------------------------------------------ | 20260506 02:32:55.629 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:32:55.629 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 02:32:55.629 | INFO | Total mem usage: 15.7 GiB | 20260506 02:32:55.629 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 02:32:55.629 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 02:32:55.629 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 02:32:55.629 | INFO | High Water Breakdown: | 20260506 02:32:55.629 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 02:32:55.629 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 02:32:55.629 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 02:32:55.629 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 02:32:55.629 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 02:32:55.629 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 02:32:55.629 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 02:32:55.629 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 02:32:55.629 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 02:32:55.629 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:32:55.629 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 02:32:55.629 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:32:55.629 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 02:32:55.629 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 02:32:55.629 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 02:32:55.629 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 02:32:55.629 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:32:55.629 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 02:32:55.630 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:32:55.630 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:32:55.630 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:32:55.630 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 02:32:55.630 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 02:32:55.630 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:32:55.630 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 02:32:55.630 | INFO | ------------------------------------------ | 20260506 02:32:55.630 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:32:55.630 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 02:32:55.630 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:32:55.630 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:32:55.630 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.39e+01, 1.31e+01 / 7.79e-01 | 20260506 02:32:55.630 | INFO | PerfReport: 0.29%, AggregateStats -- 9.18e-02 / 1.03e-01, 1.08e-02 / 2.92e-03 | 20260506 02:32:55.630 | INFO | PerfReport: 11.85%, Baseline -- 3.71e+00 / 4.53e+00, 5.83e-01 / 2.09e-01 | 20260506 02:32:55.630 | INFO | PerfReport: 3.30%, Histogram -- 1.03e+00 / 1.54e+00, 1.95e-01 / 8.18e-02 | 20260506 02:32:55.630 | INFO | PerfReport: 16.01%, DME -- 4.01e+01 / 4.42e+01, 3.87e+00 / 1.05e+00 | 20260506 02:32:55.630 | INFO | PerfReport: 57.49%, FrameLabeling -- 1.80e+01 / 1.92e+01, 1.56e+00 / 9.10e-01 | 20260506 02:32:55.630 | INFO | PerfReport: 7.01%, PulseAccumulating -- 2.19e+00 / 2.74e+00, 4.16e-01 / 2.00e-01 | 20260506 02:32:55.630 | INFO | PerfReport: 3.01%, Metrics -- 9.42e-01 / 1.20e+00, 2.51e-01 / 8.11e-02 | 20260506 02:32:55.630 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.25e-01 / 3.71e-01, 5.36e-02 / 2.36e-02 | 20260506 02:32:55.630 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 02:32:55.630 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:32:55.630 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:32:55.630 | INFO | PerfReport: Full Execution -- 9.76e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260506 02:32:55.630 | INFO | PerfReport: 22.14%, Upload -- 2.16e+01 / 0.00e+00, 7.11e+00 / 0.00e+00 | 20260506 02:32:55.630 | INFO | PerfReport: 21.91%, Download -- 2.14e+01 / 0.00e+00, 7.58e+00 / 0.00e+00 | 20260506 02:32:55.630 | INFO | PerfReport: 32.08%, Compute -- 3.13e+01 / 0.00e+00, 1.31e+01 / 0.00e+00 | 20260506 02:32:55.630 | INFO | PerfReport: 2.00%, LockUpload -- 1.95e+00 / 0.00e+00, 7.79e+00 / 0.00e+00 | 20260506 02:32:55.630 | INFO | PerfReport: 1.30%, LockDownload -- 1.27e+00 / 0.00e+00, 2.98e+00 / 0.00e+00 | 20260506 02:32:55.630 | INFO | PerfReport: 20.55%, LockCompute -- 2.01e+01 / 0.00e+00, 1.76e+01 / 0.00e+00 | 20260506 02:32:55.630 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.96e-02 / 0.00e+00 | 20260506 02:32:55.754 | INFO | Starting a round of Baz IO | 20260506 02:32:58.895 | INFO | Populating RTMetrics report | 20260506 02:32:58.896 | INFO | RTMetrics report populated | 20260506 02:32:58.896 | INFO | Opening tmp RTMetrics json file | 20260506 02:32:58.897 | INFO | Serializing RTMetrics to json | 20260506 02:32:58.900 | INFO | Serialization done, swapping tmp file now... | 20260506 02:32:58.901 | INFO | RTMetrics done | 20260506 02:32:58.995 | INFO | Finished a round of Baz IO: Wrote 1.792 GiB in 3.24 seconds | 20260506 02:32:58.995 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.26% / 0.00% / 78.74% | 20260506 02:32:58.995 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0696699% / 3.97364e-06% / 9.13938e-05% | 20260506 02:32:58.995 | INFO | Baz IO overall: | 20260506 02:32:58.995 | INFO | BytesWritten : 4623357998109 | 20260506 02:32:58.995 | INFO | NumEvents : 2170350564018 | 20260506 02:33:00.765 | INFO | Analyzing chunk frames = [8257536,8258048) | 20260506 02:33:00.765 | INFO | MemInfo: { | 20260506 02:33:00.765 | INFO | "gpu" : " 17 GB", | 20260506 02:33:00.765 | INFO | "huge" : "", | 20260506 02:33:00.765 | INFO | "huge_shared" : " 27 GB", | 20260506 02:33:00.765 | INFO | "malloc" : " 14 GB", | 20260506 02:33:00.765 | INFO | "pinned" : " 29 GB" | 20260506 02:33:00.765 | INFO | } | 20260506 02:33:00.765 | INFO | Perf report for the last 5.13648 seconds: | 20260506 02:33:00.765 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:33:00.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:00.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:00.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:00.765 | INFO | ANALYSIS : 62.5%, 3.0, 96.8ms, 0 | 20260506 02:33:00.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:00.765 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260506 02:33:00.765 | INFO | PRE_HQ : 59.8%, 1.7, 53.5ms, 0 | 20260506 02:33:00.765 | INFO | BAZWRITER : 62.3%, 4.0, 127.6ms, 0 | 20260506 02:33:00.765 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:33:00.765 | INFO | Uploaded 36GiB to GPU in 2.1955s (16.3972GiB/s) | 20260506 02:33:00.765 | INFO | Downloaded 38GiB to GPU in 2.25045s (16.8855GiB/s) | 20260506 02:33:05.909 | INFO | Analyzing chunk frames = [8258048,8258560) | 20260506 02:33:05.909 | INFO | MemInfo: { | 20260506 02:33:05.909 | INFO | "gpu" : " 17 GB", | 20260506 02:33:05.909 | INFO | "huge" : "", | 20260506 02:33:05.909 | INFO | "huge_shared" : " 27 GB", | 20260506 02:33:05.909 | INFO | "malloc" : " 14 GB", | 20260506 02:33:05.909 | INFO | "pinned" : " 29 GB" | 20260506 02:33:05.909 | INFO | } | 20260506 02:33:05.909 | INFO | Perf report for the last 5.14402 seconds: | 20260506 02:33:05.909 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:33:05.909 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:05.909 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:05.909 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:05.909 | INFO | ANALYSIS : 60.7%, 3.0, 94.9ms, 0 | 20260506 02:33:05.909 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:05.909 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:05.909 | INFO | PRE_HQ : 26.4%, 1.0, 13.8ms, 0 | 20260506 02:33:05.909 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:05.909 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:05.909 | INFO | Uploaded 36GiB to GPU in 1.98755s (18.1127GiB/s) | 20260506 02:33:05.909 | INFO | Downloaded 34GiB to GPU in 1.97777s (17.191GiB/s) | 20260506 02:33:11.050 | INFO | Analyzing chunk frames = [8258560,8259072) | 20260506 02:33:11.050 | INFO | MemInfo: { | 20260506 02:33:11.050 | INFO | "gpu" : " 17 GB", | 20260506 02:33:11.050 | INFO | "huge" : "", | 20260506 02:33:11.050 | INFO | "huge_shared" : " 27 GB", | 20260506 02:33:11.050 | INFO | "malloc" : " 14 GB", | 20260506 02:33:11.050 | INFO | "pinned" : " 29 GB" | 20260506 02:33:11.050 | INFO | } | 20260506 02:33:11.050 | INFO | Perf report for the last 5.14093 seconds: | 20260506 02:33:11.050 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:33:11.050 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:11.050 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:11.050 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:11.050 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260506 02:33:11.050 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:11.050 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:11.050 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 02:33:11.050 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:11.050 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:11.050 | INFO | Uploaded 36GiB to GPU in 2.09668s (17.17GiB/s) | 20260506 02:33:11.050 | INFO | Downloaded 34GiB to GPU in 2.04215s (16.6491GiB/s) | 20260506 02:33:16.185 | INFO | Analyzing chunk frames = [8259072,8259584) | 20260506 02:33:16.185 | INFO | MemInfo: { | 20260506 02:33:16.185 | INFO | "gpu" : " 17 GB", | 20260506 02:33:16.185 | INFO | "huge" : "", | 20260506 02:33:16.185 | INFO | "huge_shared" : " 27 GB", | 20260506 02:33:16.185 | INFO | "malloc" : " 14 GB", | 20260506 02:33:16.185 | INFO | "pinned" : " 29 GB" | 20260506 02:33:16.185 | INFO | } | 20260506 02:33:16.185 | INFO | Perf report for the last 5.13532 seconds: | 20260506 02:33:16.185 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:33:16.185 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:16.185 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:16.185 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:16.185 | INFO | ANALYSIS : 61.1%, 3.0, 94.7ms, 0 | 20260506 02:33:16.185 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:16.185 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:16.185 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:33:16.185 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:16.185 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:16.185 | INFO | Uploaded 36GiB to GPU in 2.08633s (17.2552GiB/s) | 20260506 02:33:16.185 | INFO | Downloaded 34GiB to GPU in 2.00616s (16.9478GiB/s) | 20260506 02:33:21.322 | INFO | Analyzing chunk frames = [8259584,8260096) | 20260506 02:33:21.322 | INFO | MemInfo: { | 20260506 02:33:21.322 | INFO | "gpu" : " 17 GB", | 20260506 02:33:21.322 | INFO | "huge" : "", | 20260506 02:33:21.322 | INFO | "huge_shared" : " 27 GB", | 20260506 02:33:21.322 | INFO | "malloc" : " 14 GB", | 20260506 02:33:21.322 | INFO | "pinned" : " 29 GB" | 20260506 02:33:21.322 | INFO | } | 20260506 02:33:21.322 | INFO | Perf report for the last 5.13712 seconds: | 20260506 02:33:21.322 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:33:21.322 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:21.322 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:21.322 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:21.322 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260506 02:33:21.322 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:21.322 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:21.322 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:33:21.322 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:21.322 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:21.323 | INFO | Uploaded 36GiB to GPU in 2.06893s (17.4003GiB/s) | 20260506 02:33:21.323 | INFO | Downloaded 34GiB to GPU in 2.03026s (16.7466GiB/s) | 20260506 02:33:26.462 | INFO | Analyzing chunk frames = [8260096,8260608) | 20260506 02:33:26.462 | INFO | MemInfo: { | 20260506 02:33:26.462 | INFO | "gpu" : " 17 GB", | 20260506 02:33:26.462 | INFO | "huge" : "", | 20260506 02:33:26.462 | INFO | "huge_shared" : " 27 GB", | 20260506 02:33:26.462 | INFO | "malloc" : " 14 GB", | 20260506 02:33:26.462 | INFO | "pinned" : " 29 GB" | 20260506 02:33:26.462 | INFO | } | 20260506 02:33:26.462 | INFO | Perf report for the last 5.14016 seconds: | 20260506 02:33:26.463 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:33:26.463 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:26.463 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:26.463 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:26.463 | INFO | ANALYSIS : 61.1%, 3.0, 95.5ms, 0 | 20260506 02:33:26.463 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:26.463 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:26.463 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 02:33:26.463 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:26.463 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:26.463 | INFO | Uploaded 36GiB to GPU in 2.0722s (17.3729GiB/s) | 20260506 02:33:26.463 | INFO | Downloaded 34GiB to GPU in 2.04456s (16.6295GiB/s) | 20260506 02:33:31.604 | INFO | Analyzing chunk frames = [8260608,8261120) | 20260506 02:33:31.604 | INFO | MemInfo: { | 20260506 02:33:31.604 | INFO | "gpu" : " 17 GB", | 20260506 02:33:31.604 | INFO | "huge" : "", | 20260506 02:33:31.604 | INFO | "huge_shared" : " 27 GB", | 20260506 02:33:31.604 | INFO | "malloc" : " 14 GB", | 20260506 02:33:31.604 | INFO | "pinned" : " 29 GB" | 20260506 02:33:31.604 | INFO | } | 20260506 02:33:31.604 | INFO | Perf report for the last 5.14135 seconds: | 20260506 02:33:31.604 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:33:31.604 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:31.604 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:31.604 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:31.604 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260506 02:33:31.604 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:31.604 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:31.604 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 02:33:31.604 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:31.604 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:31.604 | INFO | Uploaded 36GiB to GPU in 2.06269s (17.4529GiB/s) | 20260506 02:33:31.604 | INFO | Downloaded 34GiB to GPU in 2.08398s (16.315GiB/s) | 20260506 02:33:36.743 | INFO | Analyzing chunk frames = [8261120,8261632) | 20260506 02:33:36.743 | INFO | MemInfo: { | 20260506 02:33:36.743 | INFO | "gpu" : " 17 GB", | 20260506 02:33:36.743 | INFO | "huge" : "", | 20260506 02:33:36.743 | INFO | "huge_shared" : " 27 GB", | 20260506 02:33:36.743 | INFO | "malloc" : " 14 GB", | 20260506 02:33:36.743 | INFO | "pinned" : " 29 GB" | 20260506 02:33:36.743 | INFO | } | 20260506 02:33:36.743 | INFO | Perf report for the last 5.13916 seconds: | 20260506 02:33:36.743 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:33:36.743 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:36.743 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:36.743 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:36.743 | INFO | ANALYSIS : 61.3%, 3.0, 95.8ms, 0 | 20260506 02:33:36.743 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:36.743 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:36.743 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 02:33:36.743 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:36.743 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:36.743 | INFO | Uploaded 36GiB to GPU in 2.0414s (17.6349GiB/s) | 20260506 02:33:36.743 | INFO | Downloaded 34GiB to GPU in 2.02454s (16.794GiB/s) | 20260506 02:33:36.868 | INFO | Starting a round of Baz IO | 20260506 02:33:40.002 | INFO | Populating RTMetrics report | 20260506 02:33:40.003 | INFO | RTMetrics report populated | 20260506 02:33:40.003 | INFO | Opening tmp RTMetrics json file | 20260506 02:33:40.006 | INFO | Serializing RTMetrics to json | 20260506 02:33:40.008 | INFO | Serialization done, swapping tmp file now... | 20260506 02:33:40.008 | INFO | RTMetrics done | 20260506 02:33:40.092 | INFO | Finished a round of Baz IO: Wrote 1.79 GiB in 3.223 seconds | 20260506 02:33:40.092 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.26% / 0.00% / 78.74% | 20260506 02:33:40.092 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0693242% / 0% / 0.000103315% | 20260506 02:33:40.092 | INFO | Baz IO overall: | 20260506 02:33:40.092 | INFO | BytesWritten : 4625280184129 | 20260506 02:33:40.092 | INFO | NumEvents : 2171224230079 | 20260506 02:33:41.882 | INFO | Analyzing chunk frames = [8261632,8262144) | 20260506 02:33:41.882 | INFO | MemInfo: { | 20260506 02:33:41.882 | INFO | "gpu" : " 17 GB", | 20260506 02:33:41.882 | INFO | "huge" : "", | 20260506 02:33:41.882 | INFO | "huge_shared" : " 27 GB", | 20260506 02:33:41.882 | INFO | "malloc" : " 14 GB", | 20260506 02:33:41.882 | INFO | "pinned" : " 29 GB" | 20260506 02:33:41.882 | INFO | } | 20260506 02:33:41.882 | INFO | Perf report for the last 5.13938 seconds: | 20260506 02:33:41.882 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:33:41.882 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:41.882 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:41.882 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:41.882 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260506 02:33:41.883 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:41.883 | INFO | RT_METRICS : 42.8%, 1.0, 22.4ms, 0 | 20260506 02:33:41.883 | INFO | PRE_HQ : 59.5%, 1.7, 53.3ms, 0 | 20260506 02:33:41.883 | INFO | BAZWRITER : 62.0%, 3.5, 112.5ms, 0 | 20260506 02:33:41.883 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:33:41.883 | INFO | Uploaded 36GiB to GPU in 2.1882s (16.4519GiB/s) | 20260506 02:33:41.883 | INFO | Downloaded 38GiB to GPU in 2.24494s (16.927GiB/s) | 20260506 02:33:47.019 | INFO | Analyzing chunk frames = [8262144,8262656) | 20260506 02:33:47.019 | INFO | MemInfo: { | 20260506 02:33:47.019 | INFO | "gpu" : " 17 GB", | 20260506 02:33:47.019 | INFO | "huge" : "", | 20260506 02:33:47.019 | INFO | "huge_shared" : " 27 GB", | 20260506 02:33:47.019 | INFO | "malloc" : " 14 GB", | 20260506 02:33:47.019 | INFO | "pinned" : " 29 GB" | 20260506 02:33:47.019 | INFO | } | 20260506 02:33:47.019 | INFO | Perf report for the last 5.13623 seconds: | 20260506 02:33:47.019 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:33:47.019 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:47.019 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:47.019 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:47.019 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260506 02:33:47.019 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:47.019 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:47.019 | INFO | PRE_HQ : 26.8%, 1.0, 14.0ms, 0 | 20260506 02:33:47.019 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:47.019 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:47.019 | INFO | Uploaded 36GiB to GPU in 1.96611s (18.3103GiB/s) | 20260506 02:33:47.019 | INFO | Downloaded 34GiB to GPU in 1.97974s (17.174GiB/s) | 20260506 02:33:52.163 | INFO | Analyzing chunk frames = [8262656,8263168) | 20260506 02:33:52.163 | INFO | MemInfo: { | 20260506 02:33:52.163 | INFO | "gpu" : " 17 GB", | 20260506 02:33:52.163 | INFO | "huge" : "", | 20260506 02:33:52.163 | INFO | "huge_shared" : " 27 GB", | 20260506 02:33:52.163 | INFO | "malloc" : " 14 GB", | 20260506 02:33:52.163 | INFO | "pinned" : " 29 GB" | 20260506 02:33:52.163 | INFO | } | 20260506 02:33:52.163 | INFO | Perf report for the last 5.14468 seconds: | 20260506 02:33:52.163 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:33:52.163 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:52.163 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:52.163 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:52.163 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260506 02:33:52.163 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:52.163 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:52.163 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 02:33:52.163 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:52.164 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:52.164 | INFO | Uploaded 36GiB to GPU in 2.09407s (17.1914GiB/s) | 20260506 02:33:52.164 | INFO | Downloaded 34GiB to GPU in 2.0246s (16.7934GiB/s) | 20260506 02:33:57.302 | INFO | Analyzing chunk frames = [8263168,8263680) | 20260506 02:33:57.302 | INFO | MemInfo: { | 20260506 02:33:57.302 | INFO | "gpu" : " 17 GB", | 20260506 02:33:57.302 | INFO | "huge" : "", | 20260506 02:33:57.302 | INFO | "huge_shared" : " 27 GB", | 20260506 02:33:57.302 | INFO | "malloc" : " 14 GB", | 20260506 02:33:57.302 | INFO | "pinned" : " 29 GB" | 20260506 02:33:57.302 | INFO | } | 20260506 02:33:57.302 | INFO | Perf report for the last 5.13854 seconds: | 20260506 02:33:57.302 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:33:57.302 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:57.302 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:57.302 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:57.302 | INFO | ANALYSIS : 60.9%, 3.0, 94.6ms, 0 | 20260506 02:33:57.302 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:57.302 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:57.302 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 02:33:57.302 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:57.302 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:33:57.302 | INFO | Uploaded 36GiB to GPU in 2.07711s (17.3318GiB/s) | 20260506 02:33:57.302 | INFO | Downloaded 34GiB to GPU in 2.00474s (16.9598GiB/s) | 20260506 02:34:02.441 | INFO | Analyzing chunk frames = [8263680,8264192) | 20260506 02:34:02.441 | INFO | MemInfo: { | 20260506 02:34:02.441 | INFO | "gpu" : " 17 GB", | 20260506 02:34:02.441 | INFO | "huge" : "", | 20260506 02:34:02.441 | INFO | "huge_shared" : " 27 GB", | 20260506 02:34:02.441 | INFO | "malloc" : " 14 GB", | 20260506 02:34:02.441 | INFO | "pinned" : " 29 GB" | 20260506 02:34:02.441 | INFO | } | 20260506 02:34:02.441 | INFO | Perf report for the last 5.13909 seconds: | 20260506 02:34:02.441 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:34:02.441 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:02.441 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:02.441 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:02.441 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 02:34:02.441 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:02.441 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:02.441 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 02:34:02.441 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:02.441 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:02.441 | INFO | Uploaded 36GiB to GPU in 2.10865s (17.0725GiB/s) | 20260506 02:34:02.441 | INFO | Downloaded 34GiB to GPU in 2.03298s (16.7242GiB/s) | 20260506 02:34:07.582 | INFO | Analyzing chunk frames = [8264192,8264704) | 20260506 02:34:07.582 | INFO | MemInfo: { | 20260506 02:34:07.582 | INFO | "gpu" : " 17 GB", | 20260506 02:34:07.582 | INFO | "huge" : "", | 20260506 02:34:07.582 | INFO | "huge_shared" : " 27 GB", | 20260506 02:34:07.582 | INFO | "malloc" : " 14 GB", | 20260506 02:34:07.582 | INFO | "pinned" : " 29 GB" | 20260506 02:34:07.582 | INFO | } | 20260506 02:34:07.582 | INFO | Perf report for the last 5.1409 seconds: | 20260506 02:34:07.582 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:34:07.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:07.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:07.582 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:07.582 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 02:34:07.582 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:07.582 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:07.582 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 02:34:07.582 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:07.582 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:07.582 | INFO | Uploaded 36GiB to GPU in 2.0874s (17.2463GiB/s) | 20260506 02:34:07.582 | INFO | Downloaded 34GiB to GPU in 2.03622s (16.6976GiB/s) | 20260506 02:34:12.717 | INFO | Analyzing chunk frames = [8264704,8265216) | 20260506 02:34:12.717 | INFO | MemInfo: { | 20260506 02:34:12.717 | INFO | "gpu" : " 17 GB", | 20260506 02:34:12.717 | INFO | "huge" : "", | 20260506 02:34:12.717 | INFO | "huge_shared" : " 27 GB", | 20260506 02:34:12.717 | INFO | "malloc" : " 14 GB", | 20260506 02:34:12.717 | INFO | "pinned" : " 29 GB" | 20260506 02:34:12.717 | INFO | } | 20260506 02:34:12.717 | INFO | Perf report for the last 5.13486 seconds: | 20260506 02:34:12.717 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:34:12.717 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:12.717 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:12.717 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:12.717 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 02:34:12.717 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:12.717 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:12.717 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 02:34:12.717 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:12.717 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:12.717 | INFO | Uploaded 36GiB to GPU in 2.02883s (17.7442GiB/s) | 20260506 02:34:12.717 | INFO | Downloaded 34GiB to GPU in 2.04637s (16.6148GiB/s) | 20260506 02:34:17.863 | INFO | Analyzing chunk frames = [8265216,8265728) | 20260506 02:34:17.863 | INFO | MemInfo: { | 20260506 02:34:17.863 | INFO | "gpu" : " 17 GB", | 20260506 02:34:17.863 | INFO | "huge" : "", | 20260506 02:34:17.863 | INFO | "huge_shared" : " 27 GB", | 20260506 02:34:17.863 | INFO | "malloc" : " 14 GB", | 20260506 02:34:17.863 | INFO | "pinned" : " 29 GB" | 20260506 02:34:17.863 | INFO | } | 20260506 02:34:17.863 | INFO | Perf report for the last 5.14597 seconds: | 20260506 02:34:17.863 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:34:17.863 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:17.863 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:17.863 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:17.863 | INFO | ANALYSIS : 61.3%, 3.0, 95.9ms, 0 | 20260506 02:34:17.863 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:17.863 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:17.863 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:34:17.863 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:17.863 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:17.863 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 02:34:17.863 | INFO | Total mem usage: 15.3 GiB | 20260506 02:34:17.863 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 02:34:17.863 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -3.54%) | 20260506 02:34:17.863 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 02:34:17.863 | INFO | High Water Breakdown: | 20260506 02:34:17.863 | INFO | 61.7% : 9.13 GiB ( 96% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 02:34:17.863 | INFO | 38.2% : 5.65 GiB ( 85% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 02:34:17.863 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 02:34:17.863 | INFO | ------------------------------------------ | 20260506 02:34:17.863 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:34:17.863 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 02:34:17.863 | INFO | Total mem usage: 26.5 GiB | 20260506 02:34:17.863 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 02:34:17.863 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 02:34:17.863 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260506 02:34:17.863 | INFO | High Water Breakdown: | 20260506 02:34:17.863 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:34:17.863 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:34:17.863 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:34:17.863 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:34:17.863 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:34:17.863 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:34:17.863 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 02:34:17.863 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 02:34:17.863 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:34:17.863 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 02:34:17.863 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 02:34:17.863 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 02:34:17.863 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 02:34:17.863 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 02:34:17.863 | INFO | ------------------------------------------ | 20260506 02:34:17.863 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:34:17.863 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 02:34:17.863 | INFO | Total mem usage: 25.6 GiB | 20260506 02:34:17.863 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 02:34:17.863 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 02:34:17.863 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 02:34:17.863 | INFO | Uploaded 36GiB to GPU in 2.04539s (17.6005GiB/s) | 20260506 02:34:17.864 | INFO | High Water Breakdown: | 20260506 02:34:17.864 | INFO | Downloaded 34GiB to GPU in 2.01645s (16.8613GiB/s) | 20260506 02:34:17.864 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 02:34:17.864 | INFO | ------------------------------------------ | 20260506 02:34:17.864 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:34:17.864 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 02:34:17.864 | INFO | Total mem usage: 15.7 GiB | 20260506 02:34:17.864 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 02:34:17.864 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 02:34:17.864 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 02:34:17.864 | INFO | High Water Breakdown: | 20260506 02:34:17.864 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 02:34:17.864 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 02:34:17.864 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 02:34:17.864 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 02:34:17.864 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 02:34:17.864 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 02:34:17.864 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 02:34:17.864 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 02:34:17.864 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 02:34:17.864 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:34:17.864 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 02:34:17.864 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:34:17.864 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 02:34:17.864 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 02:34:17.864 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 02:34:17.864 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 02:34:17.864 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:34:17.864 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 02:34:17.864 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:34:17.864 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:34:17.864 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:34:17.864 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 02:34:17.864 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 02:34:17.864 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:34:17.864 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 02:34:17.864 | INFO | ------------------------------------------ | 20260506 02:34:17.864 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:34:17.864 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 02:34:17.864 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:34:17.864 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:34:17.864 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.39e+01, 1.36e+01 / 7.76e-01 | 20260506 02:34:17.864 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.04e-01, 1.04e-02 / 3.85e-03 | 20260506 02:34:17.864 | INFO | PerfReport: 11.88%, Baseline -- 3.72e+00 / 4.50e+00, 5.87e-01 / 1.87e-01 | 20260506 02:34:17.864 | INFO | PerfReport: 3.30%, Histogram -- 1.03e+00 / 1.51e+00, 1.95e-01 / 9.57e-02 | 20260506 02:34:17.864 | INFO | PerfReport: 16.01%, DME -- 4.01e+01 / 4.41e+01, 3.87e+00 / 1.02e+00 | 20260506 02:34:17.864 | INFO | PerfReport: 57.47%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.52e+00 / 1.09e+00 | 20260506 02:34:17.864 | INFO | PerfReport: 7.01%, PulseAccumulating -- 2.19e+00 / 2.70e+00, 4.17e-01 / 2.36e-01 | 20260506 02:34:17.864 | INFO | PerfReport: 3.00%, Metrics -- 9.40e-01 / 1.19e+00, 2.49e-01 / 1.04e-01 | 20260506 02:34:17.864 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.25e-01 / 3.73e-01, 5.42e-02 / 2.59e-02 | 20260506 02:34:17.864 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 02:34:17.865 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:34:17.865 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:34:17.865 | INFO | PerfReport: Full Execution -- 9.75e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260506 02:34:17.865 | INFO | PerfReport: 22.17%, Upload -- 2.16e+01 / 0.00e+00, 7.17e+00 / 0.00e+00 | 20260506 02:34:17.865 | INFO | PerfReport: 21.93%, Download -- 2.14e+01 / 0.00e+00, 7.52e+00 / 0.00e+00 | 20260506 02:34:17.865 | INFO | PerfReport: 32.08%, Compute -- 3.13e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 02:34:17.865 | INFO | PerfReport: 2.01%, LockUpload -- 1.96e+00 / 0.00e+00, 7.87e+00 / 0.00e+00 | 20260506 02:34:17.865 | INFO | PerfReport: 1.31%, LockDownload -- 1.28e+00 / 0.00e+00, 2.94e+00 / 0.00e+00 | 20260506 02:34:17.865 | INFO | PerfReport: 20.47%, LockCompute -- 2.00e+01 / 0.00e+00, 1.77e+01 / 0.00e+00 | 20260506 02:34:17.865 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 2.71e-02 / 0.00e+00 | 20260506 02:34:17.991 | INFO | Starting a round of Baz IO | 20260506 02:34:21.105 | INFO | Populating RTMetrics report | 20260506 02:34:21.106 | INFO | RTMetrics report populated | 20260506 02:34:21.106 | INFO | Opening tmp RTMetrics json file | 20260506 02:34:21.152 | INFO | Serializing RTMetrics to json | 20260506 02:34:21.208 | INFO | Serialization done, swapping tmp file now... | 20260506 02:34:21.226 | INFO | RTMetrics done | 20260506 02:34:21.324 | INFO | Finished a round of Baz IO: Wrote 1.789 GiB in 3.333 seconds | 20260506 02:34:21.324 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.26% / 0.00% / 78.74% | 20260506 02:34:21.324 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0697851% / 0% / 9.93411e-05% | 20260506 02:34:21.324 | INFO | Baz IO overall: | 20260506 02:34:21.324 | INFO | BytesWritten : 4627201089036 | 20260506 02:34:21.324 | INFO | NumEvents : 2172096742304 | 20260506 02:34:23.002 | INFO | Analyzing chunk frames = [8265728,8266240) | 20260506 02:34:23.002 | INFO | MemInfo: { | 20260506 02:34:23.002 | INFO | "gpu" : " 17 GB", | 20260506 02:34:23.002 | INFO | "huge" : "", | 20260506 02:34:23.002 | INFO | "huge_shared" : " 27 GB", | 20260506 02:34:23.002 | INFO | "malloc" : " 14 GB", | 20260506 02:34:23.002 | INFO | "pinned" : " 29 GB" | 20260506 02:34:23.002 | INFO | } | 20260506 02:34:23.002 | INFO | Perf report for the last 5.13907 seconds: | 20260506 02:34:23.002 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:34:23.002 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:23.002 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:23.002 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:23.002 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 02:34:23.002 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:23.002 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260506 02:34:23.002 | INFO | PRE_HQ : 59.1%, 1.7, 53.1ms, 0 | 20260506 02:34:23.002 | INFO | BAZWRITER : 64.1%, 3.7, 123.2ms, 0 | 20260506 02:34:23.002 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260506 02:34:23.002 | INFO | Uploaded 36GiB to GPU in 2.09784s (17.1605GiB/s) | 20260506 02:34:23.003 | INFO | Downloaded 38GiB to GPU in 2.23941s (16.9688GiB/s) | 20260506 02:34:28.133 | INFO | Analyzing chunk frames = [8266240,8266752) | 20260506 02:34:28.133 | INFO | MemInfo: { | 20260506 02:34:28.133 | INFO | "gpu" : " 17 GB", | 20260506 02:34:28.133 | INFO | "huge" : "", | 20260506 02:34:28.133 | INFO | "huge_shared" : " 27 GB", | 20260506 02:34:28.133 | INFO | "malloc" : " 14 GB", | 20260506 02:34:28.133 | INFO | "pinned" : " 29 GB" | 20260506 02:34:28.133 | INFO | } | 20260506 02:34:28.133 | INFO | Perf report for the last 5.13124 seconds: | 20260506 02:34:28.133 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:34:28.133 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:28.133 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:28.133 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:28.133 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260506 02:34:28.133 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:28.133 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:28.133 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 02:34:28.133 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:28.133 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:28.133 | INFO | Uploaded 36GiB to GPU in 1.93672s (18.5881GiB/s) | 20260506 02:34:28.133 | INFO | Downloaded 34GiB to GPU in 1.98251s (17.15GiB/s) | 20260506 02:34:33.270 | INFO | Analyzing chunk frames = [8266752,8267264) | 20260506 02:34:33.270 | INFO | MemInfo: { | 20260506 02:34:33.270 | INFO | "gpu" : " 17 GB", | 20260506 02:34:33.270 | INFO | "huge" : "", | 20260506 02:34:33.270 | INFO | "huge_shared" : " 27 GB", | 20260506 02:34:33.270 | INFO | "malloc" : " 14 GB", | 20260506 02:34:33.270 | INFO | "pinned" : " 29 GB" | 20260506 02:34:33.270 | INFO | } | 20260506 02:34:33.271 | INFO | Perf report for the last 5.13743 seconds: | 20260506 02:34:33.271 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:34:33.271 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:33.271 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:33.271 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:33.271 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 02:34:33.271 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:33.271 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:33.271 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:34:33.271 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:33.271 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:33.271 | INFO | Uploaded 36GiB to GPU in 2.08612s (17.2569GiB/s) | 20260506 02:34:33.271 | INFO | Downloaded 34GiB to GPU in 2.02143s (16.8198GiB/s) | 20260506 02:34:38.422 | INFO | Analyzing chunk frames = [8267264,8267776) | 20260506 02:34:38.422 | INFO | MemInfo: { | 20260506 02:34:38.422 | INFO | "gpu" : " 17 GB", | 20260506 02:34:38.422 | INFO | "huge" : "", | 20260506 02:34:38.422 | INFO | "huge_shared" : " 27 GB", | 20260506 02:34:38.422 | INFO | "malloc" : " 14 GB", | 20260506 02:34:38.422 | INFO | "pinned" : " 29 GB" | 20260506 02:34:38.422 | INFO | } | 20260506 02:34:38.422 | INFO | Perf report for the last 5.15113 seconds: | 20260506 02:34:38.422 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:34:38.422 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:38.422 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:38.422 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:38.422 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 02:34:38.422 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:38.422 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:38.422 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 02:34:38.422 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:38.422 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:38.422 | INFO | Uploaded 36GiB to GPU in 2.15164s (16.7315GiB/s) | 20260506 02:34:38.422 | INFO | Downloaded 34GiB to GPU in 2.05476s (16.5469GiB/s) | 20260506 02:34:43.556 | INFO | Analyzing chunk frames = [8267776,8268288) | 20260506 02:34:43.556 | INFO | MemInfo: { | 20260506 02:34:43.556 | INFO | "gpu" : " 17 GB", | 20260506 02:34:43.556 | INFO | "huge" : "", | 20260506 02:34:43.556 | INFO | "huge_shared" : " 27 GB", | 20260506 02:34:43.556 | INFO | "malloc" : " 14 GB", | 20260506 02:34:43.556 | INFO | "pinned" : " 29 GB" | 20260506 02:34:43.556 | INFO | } | 20260506 02:34:43.556 | INFO | Perf report for the last 5.13443 seconds: | 20260506 02:34:43.556 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:34:43.556 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:43.556 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:43.556 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:43.556 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260506 02:34:43.556 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:43.556 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:43.556 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 02:34:43.556 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:43.556 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:43.556 | INFO | Uploaded 36GiB to GPU in 2.09968s (17.1455GiB/s) | 20260506 02:34:43.556 | INFO | Downloaded 34GiB to GPU in 2.02872s (16.7593GiB/s) | 20260506 02:34:48.696 | INFO | Analyzing chunk frames = [8268288,8268800) | 20260506 02:34:48.696 | INFO | MemInfo: { | 20260506 02:34:48.696 | INFO | "gpu" : " 17 GB", | 20260506 02:34:48.696 | INFO | "huge" : "", | 20260506 02:34:48.696 | INFO | "huge_shared" : " 27 GB", | 20260506 02:34:48.696 | INFO | "malloc" : " 14 GB", | 20260506 02:34:48.696 | INFO | "pinned" : " 29 GB" | 20260506 02:34:48.696 | INFO | } | 20260506 02:34:48.696 | INFO | Perf report for the last 5.13967 seconds: | 20260506 02:34:48.696 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:34:48.696 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:48.696 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:48.696 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:48.696 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 02:34:48.696 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:48.696 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:48.696 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 02:34:48.696 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:48.696 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:48.696 | INFO | Uploaded 36GiB to GPU in 2.11529s (17.0189GiB/s) | 20260506 02:34:48.696 | INFO | Downloaded 34GiB to GPU in 2.04781s (16.6031GiB/s) | 20260506 02:34:53.831 | INFO | Analyzing chunk frames = [8268800,8269312) | 20260506 02:34:53.831 | INFO | MemInfo: { | 20260506 02:34:53.831 | INFO | "gpu" : " 17 GB", | 20260506 02:34:53.831 | INFO | "huge" : "", | 20260506 02:34:53.831 | INFO | "huge_shared" : " 27 GB", | 20260506 02:34:53.831 | INFO | "malloc" : " 14 GB", | 20260506 02:34:53.831 | INFO | "pinned" : " 29 GB" | 20260506 02:34:53.831 | INFO | } | 20260506 02:34:53.831 | INFO | Perf report for the last 5.13522 seconds: | 20260506 02:34:53.831 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:34:53.831 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:53.831 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:53.831 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:53.831 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 02:34:53.831 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:53.831 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:53.831 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 02:34:53.831 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:53.831 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:53.831 | INFO | Uploaded 36GiB to GPU in 2.07871s (17.3185GiB/s) | 20260506 02:34:53.831 | INFO | Downloaded 34GiB to GPU in 2.07302s (16.4012GiB/s) | 20260506 02:34:58.974 | INFO | Analyzing chunk frames = [8269312,8269824) | 20260506 02:34:58.975 | INFO | MemInfo: { | 20260506 02:34:58.975 | INFO | "gpu" : " 17 GB", | 20260506 02:34:58.975 | INFO | "huge" : "", | 20260506 02:34:58.975 | INFO | "huge_shared" : " 27 GB", | 20260506 02:34:58.975 | INFO | "malloc" : " 14 GB", | 20260506 02:34:58.975 | INFO | "pinned" : " 29 GB" | 20260506 02:34:58.975 | INFO | } | 20260506 02:34:58.975 | INFO | Perf report for the last 5.14363 seconds: | 20260506 02:34:58.975 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:34:58.975 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:58.975 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:58.975 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:58.975 | INFO | ANALYSIS : 61.4%, 3.0, 95.9ms, 0 | 20260506 02:34:58.975 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:58.975 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:58.975 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:34:58.975 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:58.975 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:34:58.975 | INFO | Uploaded 36GiB to GPU in 2.0537s (17.5293GiB/s) | 20260506 02:34:58.975 | INFO | Downloaded 34GiB to GPU in 2.01934s (16.8372GiB/s) | 20260506 02:34:59.100 | INFO | Starting a round of Baz IO | 20260506 02:35:02.229 | INFO | Populating RTMetrics report | 20260506 02:35:02.229 | INFO | RTMetrics report populated | 20260506 02:35:02.229 | INFO | Opening tmp RTMetrics json file | 20260506 02:35:02.230 | INFO | Serializing RTMetrics to json | 20260506 02:35:02.231 | INFO | Serialization done, swapping tmp file now... | 20260506 02:35:02.231 | INFO | RTMetrics done | 20260506 02:35:02.305 | INFO | Finished a round of Baz IO: Wrote 1.791 GiB in 3.204 seconds | 20260506 02:35:02.305 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.26% / 0.00% / 78.74% | 20260506 02:35:02.305 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0695427% / 3.97364e-06% / 6.75519e-05% | 20260506 02:35:02.305 | INFO | Baz IO overall: | 20260506 02:35:02.305 | INFO | BytesWritten : 4629124242679 | 20260506 02:35:02.305 | INFO | NumEvents : 2172971447880 | 20260506 02:35:04.114 | INFO | Analyzing chunk frames = [8269824,8270336) | 20260506 02:35:04.114 | INFO | MemInfo: { | 20260506 02:35:04.114 | INFO | "gpu" : " 17 GB", | 20260506 02:35:04.114 | INFO | "huge" : "", | 20260506 02:35:04.114 | INFO | "huge_shared" : " 27 GB", | 20260506 02:35:04.114 | INFO | "malloc" : " 14 GB", | 20260506 02:35:04.114 | INFO | "pinned" : " 29 GB" | 20260506 02:35:04.114 | INFO | } | 20260506 02:35:04.114 | INFO | Perf report for the last 5.13976 seconds: | 20260506 02:35:04.114 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:35:04.114 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:04.114 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:04.114 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:04.115 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260506 02:35:04.115 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:04.115 | INFO | RT_METRICS : 42.8%, 1.0, 22.4ms, 0 | 20260506 02:35:04.115 | INFO | PRE_HQ : 59.2%, 1.7, 53.3ms, 0 | 20260506 02:35:04.115 | INFO | BAZWRITER : 61.5%, 3.8, 121.3ms, 0 | 20260506 02:35:04.115 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:35:04.115 | INFO | Uploaded 36GiB to GPU in 2.10963s (17.0646GiB/s) | 20260506 02:35:04.115 | INFO | Downloaded 38GiB to GPU in 2.22253s (17.0977GiB/s) | 20260506 02:35:09.251 | INFO | Analyzing chunk frames = [8270336,8270848) | 20260506 02:35:09.251 | INFO | MemInfo: { | 20260506 02:35:09.251 | INFO | "gpu" : " 17 GB", | 20260506 02:35:09.251 | INFO | "huge" : "", | 20260506 02:35:09.251 | INFO | "huge_shared" : " 27 GB", | 20260506 02:35:09.251 | INFO | "malloc" : " 14 GB", | 20260506 02:35:09.251 | INFO | "pinned" : " 29 GB" | 20260506 02:35:09.251 | INFO | } | 20260506 02:35:09.252 | INFO | Perf report for the last 5.13718 seconds: | 20260506 02:35:09.252 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:35:09.252 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:09.252 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:09.252 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:09.252 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260506 02:35:09.252 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:09.252 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:09.252 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 02:35:09.252 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:09.252 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:09.252 | INFO | Uploaded 36GiB to GPU in 2.00795s (17.9288GiB/s) | 20260506 02:35:09.252 | INFO | Downloaded 34GiB to GPU in 1.9894s (17.0906GiB/s) | 20260506 02:35:14.390 | INFO | Analyzing chunk frames = [8270848,8271360) | 20260506 02:35:14.390 | INFO | MemInfo: { | 20260506 02:35:14.390 | INFO | "gpu" : " 17 GB", | 20260506 02:35:14.390 | INFO | "huge" : "", | 20260506 02:35:14.390 | INFO | "huge_shared" : " 27 GB", | 20260506 02:35:14.390 | INFO | "malloc" : " 14 GB", | 20260506 02:35:14.390 | INFO | "pinned" : " 29 GB" | 20260506 02:35:14.390 | INFO | } | 20260506 02:35:14.390 | INFO | Perf report for the last 5.13839 seconds: | 20260506 02:35:14.390 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:35:14.390 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:14.390 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:14.390 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:14.390 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 02:35:14.390 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:14.390 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:14.390 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 02:35:14.390 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:14.390 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:14.390 | INFO | Uploaded 36GiB to GPU in 2.0907s (17.2191GiB/s) | 20260506 02:35:14.390 | INFO | Downloaded 34GiB to GPU in 2.03061s (16.7437GiB/s) | 20260506 02:35:19.529 | INFO | Analyzing chunk frames = [8271360,8271872) | 20260506 02:35:19.529 | INFO | MemInfo: { | 20260506 02:35:19.529 | INFO | "gpu" : " 17 GB", | 20260506 02:35:19.529 | INFO | "huge" : "", | 20260506 02:35:19.529 | INFO | "huge_shared" : " 27 GB", | 20260506 02:35:19.529 | INFO | "malloc" : " 14 GB", | 20260506 02:35:19.529 | INFO | "pinned" : " 29 GB" | 20260506 02:35:19.529 | INFO | } | 20260506 02:35:19.529 | INFO | Perf report for the last 5.13904 seconds: | 20260506 02:35:19.529 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:35:19.529 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:19.529 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:19.529 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:19.529 | INFO | ANALYSIS : 61.5%, 3.0, 95.5ms, 0 | 20260506 02:35:19.529 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:19.529 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:19.529 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:35:19.529 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:19.529 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:19.529 | INFO | Uploaded 36GiB to GPU in 2.14643s (16.7721GiB/s) | 20260506 02:35:19.529 | INFO | Downloaded 34GiB to GPU in 2.05208s (16.5685GiB/s) | 20260506 02:35:24.673 | INFO | Analyzing chunk frames = [8271872,8272384) | 20260506 02:35:24.673 | INFO | MemInfo: { | 20260506 02:35:24.673 | INFO | "gpu" : " 17 GB", | 20260506 02:35:24.673 | INFO | "huge" : "", | 20260506 02:35:24.673 | INFO | "huge_shared" : " 27 GB", | 20260506 02:35:24.673 | INFO | "malloc" : " 14 GB", | 20260506 02:35:24.673 | INFO | "pinned" : " 29 GB" | 20260506 02:35:24.673 | INFO | } | 20260506 02:35:24.673 | INFO | Perf report for the last 5.14433 seconds: | 20260506 02:35:24.673 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:35:24.673 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:24.673 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:24.673 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:24.673 | INFO | ANALYSIS : 60.6%, 3.0, 94.7ms, 0 | 20260506 02:35:24.673 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:24.673 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:24.673 | INFO | PRE_HQ : 26.8%, 1.0, 14.0ms, 0 | 20260506 02:35:24.673 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:24.673 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:24.674 | INFO | Uploaded 36GiB to GPU in 2.07139s (17.3796GiB/s) | 20260506 02:35:24.674 | INFO | Downloaded 34GiB to GPU in 2.02221s (16.8133GiB/s) | 20260506 02:35:29.814 | INFO | Analyzing chunk frames = [8272384,8272896) | 20260506 02:35:29.814 | INFO | MemInfo: { | 20260506 02:35:29.814 | INFO | "gpu" : " 17 GB", | 20260506 02:35:29.814 | INFO | "huge" : "", | 20260506 02:35:29.814 | INFO | "huge_shared" : " 27 GB", | 20260506 02:35:29.814 | INFO | "malloc" : " 14 GB", | 20260506 02:35:29.814 | INFO | "pinned" : " 29 GB" | 20260506 02:35:29.814 | INFO | } | 20260506 02:35:29.814 | INFO | Perf report for the last 5.14068 seconds: | 20260506 02:35:29.814 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:35:29.814 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:29.814 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:29.814 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:29.814 | INFO | ANALYSIS : 60.7%, 3.0, 94.8ms, 0 | 20260506 02:35:29.814 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:29.814 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:29.814 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:35:29.814 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:29.814 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:29.814 | INFO | Uploaded 36GiB to GPU in 2.03994s (17.6476GiB/s) | 20260506 02:35:29.814 | INFO | Downloaded 34GiB to GPU in 2.02528s (16.7878GiB/s) | 20260506 02:35:34.952 | INFO | Analyzing chunk frames = [8272896,8273408) | 20260506 02:35:34.952 | INFO | MemInfo: { | 20260506 02:35:34.952 | INFO | "gpu" : " 17 GB", | 20260506 02:35:34.952 | INFO | "huge" : "", | 20260506 02:35:34.952 | INFO | "huge_shared" : " 27 GB", | 20260506 02:35:34.952 | INFO | "malloc" : " 14 GB", | 20260506 02:35:34.952 | INFO | "pinned" : " 29 GB" | 20260506 02:35:34.952 | INFO | } | 20260506 02:35:34.952 | INFO | Perf report for the last 5.13752 seconds: | 20260506 02:35:34.952 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:35:34.952 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:34.952 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:34.952 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:34.952 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 02:35:34.952 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:34.952 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:34.952 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 02:35:34.952 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:34.952 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:34.952 | INFO | Uploaded 36GiB to GPU in 2.0653s (17.4308GiB/s) | 20260506 02:35:34.952 | INFO | Downloaded 34GiB to GPU in 2.08409s (16.3141GiB/s) | 20260506 02:35:40.082 | INFO | Analyzing chunk frames = [8273408,8273920) | 20260506 02:35:40.082 | INFO | MemInfo: { | 20260506 02:35:40.082 | INFO | "gpu" : " 17 GB", | 20260506 02:35:40.082 | INFO | "huge" : "", | 20260506 02:35:40.082 | INFO | "huge_shared" : " 27 GB", | 20260506 02:35:40.082 | INFO | "malloc" : " 14 GB", | 20260506 02:35:40.082 | INFO | "pinned" : " 29 GB" | 20260506 02:35:40.082 | INFO | } | 20260506 02:35:40.082 | INFO | Perf report for the last 5.13076 seconds: | 20260506 02:35:40.082 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:35:40.082 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:40.082 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:40.082 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:40.083 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 02:35:40.083 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:40.083 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:40.083 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 02:35:40.083 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:40.083 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:40.083 | INFO | Uploaded 36GiB to GPU in 2.03975s (17.6492GiB/s) | 20260506 02:35:40.083 | INFO | Downloaded 34GiB to GPU in 2.01725s (16.8546GiB/s) | 20260506 02:35:40.083 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 02:35:40.083 | INFO | Total mem usage: 15.3 GiB | 20260506 02:35:40.083 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 5.28%) | 20260506 02:35:40.083 | INFO | Recent High Water Marks Sum: 14.7 GiB (Savings: -4.49%) | 20260506 02:35:40.083 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 02:35:40.083 | INFO | High Water Breakdown: | 20260506 02:35:40.083 | INFO | 61.5% : 9.02 GiB ( 94.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 02:35:40.083 | INFO | 38.3% : 5.62 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 02:35:40.083 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 02:35:40.083 | INFO | ------------------------------------------ | 20260506 02:35:40.083 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:35:40.083 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 02:35:40.083 | INFO | Total mem usage: 26.5 GiB | 20260506 02:35:40.083 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 02:35:40.083 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 02:35:40.083 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 02:35:40.083 | INFO | High Water Breakdown: | 20260506 02:35:40.083 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:35:40.083 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:35:40.083 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:35:40.083 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:35:40.083 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:35:40.083 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:35:40.083 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 02:35:40.083 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 02:35:40.083 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:35:40.083 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 02:35:40.083 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 02:35:40.083 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 02:35:40.083 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 02:35:40.083 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 02:35:40.083 | INFO | ------------------------------------------ | 20260506 02:35:40.083 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:35:40.083 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 02:35:40.083 | INFO | Total mem usage: 25.6 GiB | 20260506 02:35:40.083 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 02:35:40.083 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 02:35:40.083 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 02:35:40.083 | INFO | High Water Breakdown: | 20260506 02:35:40.084 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 02:35:40.084 | INFO | ------------------------------------------ | 20260506 02:35:40.084 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:35:40.084 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 02:35:40.084 | INFO | Total mem usage: 15.7 GiB | 20260506 02:35:40.084 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 02:35:40.084 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 02:35:40.084 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 02:35:40.084 | INFO | High Water Breakdown: | 20260506 02:35:40.084 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 02:35:40.084 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 02:35:40.084 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 02:35:40.084 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 02:35:40.084 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 02:35:40.084 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 02:35:40.084 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 02:35:40.084 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 02:35:40.084 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 02:35:40.084 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:35:40.084 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 02:35:40.084 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:35:40.084 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 02:35:40.084 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 02:35:40.084 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 02:35:40.084 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 02:35:40.084 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:35:40.084 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 02:35:40.084 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:35:40.084 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:35:40.084 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:35:40.084 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 02:35:40.084 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 02:35:40.084 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:35:40.084 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 02:35:40.084 | INFO | ------------------------------------------ | 20260506 02:35:40.084 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:35:40.084 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 02:35:40.084 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:35:40.084 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:35:40.084 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.37e+01, 1.33e+01 / 7.61e-01 | 20260506 02:35:40.084 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.05e-01, 1.33e-02 / 5.63e-03 | 20260506 02:35:40.084 | INFO | PerfReport: 11.86%, Baseline -- 3.71e+00 / 4.48e+00, 5.81e-01 / 2.92e-01 | 20260506 02:35:40.084 | INFO | PerfReport: 3.31%, Histogram -- 1.04e+00 / 1.53e+00, 1.98e-01 / 9.41e-02 | 20260506 02:35:40.084 | INFO | PerfReport: 15.98%, DME -- 4.00e+01 / 4.39e+01, 3.83e+00 / 1.01e+00 | 20260506 02:35:40.084 | INFO | PerfReport: 57.49%, FrameLabeling -- 1.80e+01 / 1.93e+01, 1.54e+00 / 1.13e+00 | 20260506 02:35:40.084 | INFO | PerfReport: 7.02%, PulseAccumulating -- 2.19e+00 / 2.74e+00, 4.15e-01 / 1.86e-01 | 20260506 02:35:40.084 | INFO | PerfReport: 3.01%, Metrics -- 9.42e-01 / 1.22e+00, 2.52e-01 / 8.74e-02 | 20260506 02:35:40.084 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.25e-01 / 3.76e-01, 5.36e-02 / 2.90e-02 | 20260506 02:35:40.084 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 02:35:40.084 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:35:40.084 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:35:40.084 | INFO | PerfReport: Full Execution -- 9.76e+01 / -nan, 2.41e+01 / 0.00e+00 | 20260506 02:35:40.084 | INFO | PerfReport: 22.14%, Upload -- 2.16e+01 / 0.00e+00, 7.11e+00 / 0.00e+00 | 20260506 02:35:40.084 | INFO | PerfReport: 21.96%, Download -- 2.14e+01 / 0.00e+00, 7.59e+00 / 0.00e+00 | 20260506 02:35:40.084 | INFO | PerfReport: 32.06%, Compute -- 3.13e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 02:35:40.084 | INFO | PerfReport: 1.99%, LockUpload -- 1.94e+00 / 0.00e+00, 7.81e+00 / 0.00e+00 | 20260506 02:35:40.084 | INFO | PerfReport: 1.32%, LockDownload -- 1.29e+00 / 0.00e+00, 3.05e+00 / 0.00e+00 | 20260506 02:35:40.084 | INFO | PerfReport: 20.52%, LockCompute -- 2.00e+01 / 0.00e+00, 1.78e+01 / 0.00e+00 | 20260506 02:35:40.084 | INFO | PerfReport: 0.02%, Misc -- 1.49e-02 / 0.00e+00, 3.70e-02 / 0.00e+00 | 20260506 02:35:40.200 | INFO | Starting a round of Baz IO | 20260506 02:35:43.349 | INFO | Populating RTMetrics report | 20260506 02:35:43.349 | INFO | RTMetrics report populated | 20260506 02:35:43.349 | INFO | Opening tmp RTMetrics json file | 20260506 02:35:43.352 | INFO | Serializing RTMetrics to json | 20260506 02:35:43.354 | INFO | Serialization done, swapping tmp file now... | 20260506 02:35:43.355 | INFO | RTMetrics done | 20260506 02:35:43.432 | INFO | Finished a round of Baz IO: Wrote 1.789 GiB in 3.232 seconds | 20260506 02:35:43.432 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.26% / 0.00% / 78.74% | 20260506 02:35:43.432 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.069507% / 0% / 7.15256e-05% | 20260506 02:35:43.432 | INFO | Baz IO overall: | 20260506 02:35:43.432 | INFO | BytesWritten : 4631045276644 | 20260506 02:35:43.432 | INFO | NumEvents : 2173844427109 | 20260506 02:35:45.227 | INFO | Analyzing chunk frames = [8273920,8274432) | 20260506 02:35:45.227 | INFO | MemInfo: { | 20260506 02:35:45.227 | INFO | "gpu" : " 17 GB", | 20260506 02:35:45.227 | INFO | "huge" : "", | 20260506 02:35:45.227 | INFO | "huge_shared" : " 27 GB", | 20260506 02:35:45.227 | INFO | "malloc" : " 14 GB", | 20260506 02:35:45.227 | INFO | "pinned" : " 29 GB" | 20260506 02:35:45.227 | INFO | } | 20260506 02:35:45.227 | INFO | Perf report for the last 5.14496 seconds: | 20260506 02:35:45.228 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:35:45.228 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:45.228 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:45.228 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:45.228 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260506 02:35:45.228 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:45.228 | INFO | RT_METRICS : 42.8%, 1.0, 22.5ms, 0 | 20260506 02:35:45.228 | INFO | PRE_HQ : 59.5%, 1.7, 53.5ms, 0 | 20260506 02:35:45.228 | INFO | BAZWRITER : 61.9%, 4.2, 134.2ms, 0 | 20260506 02:35:45.228 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:35:45.228 | INFO | Uploaded 36GiB to GPU in 2.18758s (16.4565GiB/s) | 20260506 02:35:45.228 | INFO | Downloaded 38GiB to GPU in 2.25142s (16.8782GiB/s) | 20260506 02:35:50.372 | INFO | Analyzing chunk frames = [8274432,8274944) | 20260506 02:35:50.372 | INFO | MemInfo: { | 20260506 02:35:50.372 | INFO | "gpu" : " 17 GB", | 20260506 02:35:50.372 | INFO | "huge" : "", | 20260506 02:35:50.372 | INFO | "huge_shared" : " 27 GB", | 20260506 02:35:50.372 | INFO | "malloc" : " 14 GB", | 20260506 02:35:50.372 | INFO | "pinned" : " 29 GB" | 20260506 02:35:50.373 | INFO | } | 20260506 02:35:50.373 | INFO | Perf report for the last 5.1452 seconds: | 20260506 02:35:50.373 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:35:50.373 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:50.373 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:50.373 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:50.373 | INFO | ANALYSIS : 60.7%, 3.0, 94.8ms, 0 | 20260506 02:35:50.373 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:50.373 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:50.373 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 02:35:50.373 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:50.373 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:50.373 | INFO | Uploaded 36GiB to GPU in 1.96933s (18.2803GiB/s) | 20260506 02:35:50.373 | INFO | Downloaded 34GiB to GPU in 1.9754s (17.2117GiB/s) | 20260506 02:35:55.506 | INFO | Analyzing chunk frames = [8274944,8275456) | 20260506 02:35:55.506 | INFO | MemInfo: { | 20260506 02:35:55.506 | INFO | "gpu" : " 17 GB", | 20260506 02:35:55.506 | INFO | "huge" : "", | 20260506 02:35:55.506 | INFO | "huge_shared" : " 27 GB", | 20260506 02:35:55.506 | INFO | "malloc" : " 14 GB", | 20260506 02:35:55.507 | INFO | "pinned" : " 29 GB" | 20260506 02:35:55.507 | INFO | } | 20260506 02:35:55.507 | INFO | Perf report for the last 5.13397 seconds: | 20260506 02:35:55.507 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:35:55.507 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:55.507 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:55.507 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:55.507 | INFO | ANALYSIS : 61.2%, 3.0, 95.1ms, 0 | 20260506 02:35:55.507 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:55.507 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:55.507 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:35:55.507 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:55.507 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:35:55.507 | INFO | Uploaded 36GiB to GPU in 2.09345s (17.1965GiB/s) | 20260506 02:35:55.507 | INFO | Downloaded 34GiB to GPU in 2.03601s (16.6994GiB/s) | 20260506 02:36:00.647 | INFO | Analyzing chunk frames = [8275456,8275968) | 20260506 02:36:00.647 | INFO | MemInfo: { | 20260506 02:36:00.647 | INFO | "gpu" : " 17 GB", | 20260506 02:36:00.647 | INFO | "huge" : "", | 20260506 02:36:00.647 | INFO | "huge_shared" : " 27 GB", | 20260506 02:36:00.647 | INFO | "malloc" : " 14 GB", | 20260506 02:36:00.647 | INFO | "pinned" : " 29 GB" | 20260506 02:36:00.647 | INFO | } | 20260506 02:36:00.647 | INFO | Perf report for the last 5.14018 seconds: | 20260506 02:36:00.647 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:36:00.647 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:00.647 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:00.647 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:00.647 | INFO | ANALYSIS : 61.5%, 3.0, 95.5ms, 0 | 20260506 02:36:00.647 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:00.647 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:00.647 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 02:36:00.647 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:00.647 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:00.647 | INFO | Uploaded 36GiB to GPU in 2.15152s (16.7323GiB/s) | 20260506 02:36:00.647 | INFO | Downloaded 34GiB to GPU in 2.06144s (16.4934GiB/s) | 20260506 02:36:05.786 | INFO | Analyzing chunk frames = [8275968,8276480) | 20260506 02:36:05.786 | INFO | MemInfo: { | 20260506 02:36:05.786 | INFO | "gpu" : " 17 GB", | 20260506 02:36:05.787 | INFO | "huge" : "", | 20260506 02:36:05.787 | INFO | "huge_shared" : " 27 GB", | 20260506 02:36:05.787 | INFO | "malloc" : " 14 GB", | 20260506 02:36:05.787 | INFO | "pinned" : " 29 GB" | 20260506 02:36:05.787 | INFO | } | 20260506 02:36:05.787 | INFO | Perf report for the last 5.13995 seconds: | 20260506 02:36:05.787 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:36:05.787 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:05.787 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:05.787 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:05.787 | INFO | ANALYSIS : 60.6%, 3.0, 94.7ms, 0 | 20260506 02:36:05.787 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:05.787 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:05.787 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 02:36:05.787 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:05.787 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:05.787 | INFO | Uploaded 36GiB to GPU in 2.08203s (17.2908GiB/s) | 20260506 02:36:05.787 | INFO | Downloaded 34GiB to GPU in 2.0294s (16.7537GiB/s) | 20260506 02:36:10.926 | INFO | Analyzing chunk frames = [8276480,8276992) | 20260506 02:36:10.926 | INFO | MemInfo: { | 20260506 02:36:10.926 | INFO | "gpu" : " 17 GB", | 20260506 02:36:10.926 | INFO | "huge" : "", | 20260506 02:36:10.926 | INFO | "huge_shared" : " 27 GB", | 20260506 02:36:10.926 | INFO | "malloc" : " 14 GB", | 20260506 02:36:10.926 | INFO | "pinned" : " 29 GB" | 20260506 02:36:10.926 | INFO | } | 20260506 02:36:10.926 | INFO | Perf report for the last 5.13903 seconds: | 20260506 02:36:10.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:36:10.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:10.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:10.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:10.926 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 02:36:10.926 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:10.926 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:10.926 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:36:10.926 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:10.926 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:10.926 | INFO | Uploaded 36GiB to GPU in 2.09751s (17.1632GiB/s) | 20260506 02:36:10.926 | INFO | Downloaded 34GiB to GPU in 2.04264s (16.6451GiB/s) | 20260506 02:36:16.062 | INFO | Analyzing chunk frames = [8276992,8277504) | 20260506 02:36:16.062 | INFO | MemInfo: { | 20260506 02:36:16.062 | INFO | "gpu" : " 17 GB", | 20260506 02:36:16.062 | INFO | "huge" : "", | 20260506 02:36:16.062 | INFO | "huge_shared" : " 27 GB", | 20260506 02:36:16.062 | INFO | "malloc" : " 14 GB", | 20260506 02:36:16.062 | INFO | "pinned" : " 29 GB" | 20260506 02:36:16.062 | INFO | } | 20260506 02:36:16.063 | INFO | Perf report for the last 5.1367 seconds: | 20260506 02:36:16.063 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:36:16.063 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:16.063 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:16.063 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:16.063 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260506 02:36:16.063 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:16.063 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:16.063 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 02:36:16.063 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:16.063 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:16.063 | INFO | Uploaded 36GiB to GPU in 2.09076s (17.2186GiB/s) | 20260506 02:36:16.063 | INFO | Downloaded 34GiB to GPU in 2.05689s (16.5298GiB/s) | 20260506 02:36:21.200 | INFO | Analyzing chunk frames = [8277504,8278016) | 20260506 02:36:21.200 | INFO | MemInfo: { | 20260506 02:36:21.200 | INFO | "gpu" : " 17 GB", | 20260506 02:36:21.200 | INFO | "huge" : "", | 20260506 02:36:21.201 | INFO | "huge_shared" : " 27 GB", | 20260506 02:36:21.201 | INFO | "malloc" : " 14 GB", | 20260506 02:36:21.201 | INFO | "pinned" : " 29 GB" | 20260506 02:36:21.201 | INFO | } | 20260506 02:36:21.201 | INFO | Perf report for the last 5.1382 seconds: | 20260506 02:36:21.201 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:36:21.201 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:21.201 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:21.201 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:21.201 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260506 02:36:21.201 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:21.201 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:21.201 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 02:36:21.201 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:21.201 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:21.201 | INFO | Uploaded 36GiB to GPU in 2.03617s (17.6803GiB/s) | 20260506 02:36:21.201 | INFO | Downloaded 34GiB to GPU in 2.01345s (16.8865GiB/s) | 20260506 02:36:21.328 | INFO | Starting a round of Baz IO | 20260506 02:36:24.456 | INFO | Populating RTMetrics report | 20260506 02:36:24.456 | INFO | RTMetrics report populated | 20260506 02:36:24.456 | INFO | Opening tmp RTMetrics json file | 20260506 02:36:24.493 | INFO | Serializing RTMetrics to json | 20260506 02:36:24.495 | INFO | Serialization done, swapping tmp file now... | 20260506 02:36:24.496 | INFO | RTMetrics done | 20260506 02:36:24.904 | INFO | Finished a round of Baz IO: Wrote 1.789 GiB in 3.575 seconds | 20260506 02:36:24.904 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.26% / 0.00% / 78.74% | 20260506 02:36:24.904 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0691017% / 0% / 0.000107288% | 20260506 02:36:24.904 | INFO | Baz IO overall: | 20260506 02:36:24.904 | INFO | BytesWritten : 4632965783988 | 20260506 02:36:24.904 | INFO | NumEvents : 2174717257864 | 20260506 02:36:26.337 | INFO | Analyzing chunk frames = [8278016,8278528) | 20260506 02:36:26.337 | INFO | MemInfo: { | 20260506 02:36:26.337 | INFO | "gpu" : " 17 GB", | 20260506 02:36:26.337 | INFO | "huge" : "", | 20260506 02:36:26.337 | INFO | "huge_shared" : " 27 GB", | 20260506 02:36:26.338 | INFO | "malloc" : " 14 GB", | 20260506 02:36:26.338 | INFO | "pinned" : " 29 GB" | 20260506 02:36:26.338 | INFO | } | 20260506 02:36:26.338 | INFO | Perf report for the last 5.13681 seconds: | 20260506 02:36:26.338 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:36:26.338 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:26.338 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:26.338 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:26.338 | INFO | ANALYSIS : 62.4%, 3.0, 96.9ms, 0 | 20260506 02:36:26.338 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:26.338 | INFO | RT_METRICS : 43.4%, 1.0, 22.8ms, 0 | 20260506 02:36:26.338 | INFO | PRE_HQ : 59.5%, 1.7, 53.0ms, 0 | 20260506 02:36:26.338 | INFO | BAZWRITER : 68.8%, 3.9, 136.7ms, 0 | 20260506 02:36:26.338 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260506 02:36:26.338 | INFO | Uploaded 36GiB to GPU in 2.19133s (16.4283GiB/s) | 20260506 02:36:26.338 | INFO | Downloaded 38GiB to GPU in 2.25173s (16.8759GiB/s) | 20260506 02:36:31.479 | INFO | Analyzing chunk frames = [8278528,8279040) | 20260506 02:36:31.479 | INFO | MemInfo: { | 20260506 02:36:31.479 | INFO | "gpu" : " 17 GB", | 20260506 02:36:31.479 | INFO | "huge" : "", | 20260506 02:36:31.479 | INFO | "huge_shared" : " 27 GB", | 20260506 02:36:31.479 | INFO | "malloc" : " 14 GB", | 20260506 02:36:31.479 | INFO | "pinned" : " 29 GB" | 20260506 02:36:31.479 | INFO | } | 20260506 02:36:31.479 | INFO | Perf report for the last 5.14156 seconds: | 20260506 02:36:31.479 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:36:31.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:31.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:31.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:31.479 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260506 02:36:31.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:31.479 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:31.479 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 02:36:31.479 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:31.479 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:31.479 | INFO | Uploaded 36GiB to GPU in 2.11752s (17.001GiB/s) | 20260506 02:36:31.479 | INFO | Downloaded 34GiB to GPU in 2.04308s (16.6416GiB/s) | 20260506 02:36:36.632 | INFO | Analyzing chunk frames = [8279040,8279552) | 20260506 02:36:36.632 | INFO | MemInfo: { | 20260506 02:36:36.632 | INFO | "gpu" : " 17 GB", | 20260506 02:36:36.632 | INFO | "huge" : "", | 20260506 02:36:36.632 | INFO | "huge_shared" : " 27 GB", | 20260506 02:36:36.632 | INFO | "malloc" : " 14 GB", | 20260506 02:36:36.633 | INFO | "pinned" : " 29 GB" | 20260506 02:36:36.633 | INFO | } | 20260506 02:36:36.633 | INFO | Perf report for the last 5.15352 seconds: | 20260506 02:36:36.633 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:36:36.633 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:36.633 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:36.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:36.633 | INFO | ANALYSIS : 60.9%, 3.0, 95.0ms, 0 | 20260506 02:36:36.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:36.633 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:36.633 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 02:36:36.633 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:36.633 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:36.633 | INFO | Uploaded 36GiB to GPU in 2.09098s (17.2168GiB/s) | 20260506 02:36:36.633 | INFO | Downloaded 34GiB to GPU in 2.03804s (16.6827GiB/s) | 20260506 02:36:41.763 | INFO | Analyzing chunk frames = [8279552,8280064) | 20260506 02:36:41.763 | INFO | MemInfo: { | 20260506 02:36:41.763 | INFO | "gpu" : " 17 GB", | 20260506 02:36:41.763 | INFO | "huge" : "", | 20260506 02:36:41.763 | INFO | "huge_shared" : " 27 GB", | 20260506 02:36:41.763 | INFO | "malloc" : " 14 GB", | 20260506 02:36:41.763 | INFO | "pinned" : " 29 GB" | 20260506 02:36:41.763 | INFO | } | 20260506 02:36:41.763 | INFO | Perf report for the last 5.13069 seconds: | 20260506 02:36:41.763 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:36:41.763 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:41.763 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:41.763 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:41.763 | INFO | ANALYSIS : 61.0%, 3.0, 94.6ms, 0 | 20260506 02:36:41.763 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:41.764 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:41.764 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 02:36:41.764 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:41.764 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:41.764 | INFO | Uploaded 36GiB to GPU in 2.10683s (17.0873GiB/s) | 20260506 02:36:41.764 | INFO | Downloaded 34GiB to GPU in 2.04019s (16.6651GiB/s) | 20260506 02:36:46.898 | INFO | Analyzing chunk frames = [8280064,8280576) | 20260506 02:36:46.898 | INFO | MemInfo: { | 20260506 02:36:46.899 | INFO | "gpu" : " 17 GB", | 20260506 02:36:46.899 | INFO | "huge" : "", | 20260506 02:36:46.899 | INFO | "huge_shared" : " 27 GB", | 20260506 02:36:46.899 | INFO | "malloc" : " 14 GB", | 20260506 02:36:46.899 | INFO | "pinned" : " 29 GB" | 20260506 02:36:46.899 | INFO | } | 20260506 02:36:46.899 | INFO | Perf report for the last 5.13517 seconds: | 20260506 02:36:46.899 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:36:46.899 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:46.899 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:46.899 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:46.899 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 02:36:46.899 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:46.899 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:46.899 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:36:46.899 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:46.899 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:46.899 | INFO | Uploaded 36GiB to GPU in 2.09816s (17.1579GiB/s) | 20260506 02:36:46.899 | INFO | Downloaded 34GiB to GPU in 2.0343s (16.7134GiB/s) | 20260506 02:36:52.035 | INFO | Analyzing chunk frames = [8280576,8281088) | 20260506 02:36:52.035 | INFO | MemInfo: { | 20260506 02:36:52.035 | INFO | "gpu" : " 17 GB", | 20260506 02:36:52.035 | INFO | "huge" : "", | 20260506 02:36:52.035 | INFO | "huge_shared" : " 27 GB", | 20260506 02:36:52.035 | INFO | "malloc" : " 14 GB", | 20260506 02:36:52.035 | INFO | "pinned" : " 29 GB" | 20260506 02:36:52.035 | INFO | } | 20260506 02:36:52.035 | INFO | Perf report for the last 5.1368 seconds: | 20260506 02:36:52.036 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:36:52.036 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:52.036 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:52.036 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:52.036 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 02:36:52.036 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:52.036 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:52.036 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:36:52.036 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:52.036 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:52.036 | INFO | Uploaded 36GiB to GPU in 2.06767s (17.4109GiB/s) | 20260506 02:36:52.036 | INFO | Downloaded 34GiB to GPU in 2.03886s (16.676GiB/s) | 20260506 02:36:57.180 | INFO | Analyzing chunk frames = [8281088,8281600) | 20260506 02:36:57.180 | INFO | MemInfo: { | 20260506 02:36:57.180 | INFO | "gpu" : " 17 GB", | 20260506 02:36:57.180 | INFO | "huge" : "", | 20260506 02:36:57.180 | INFO | "huge_shared" : " 27 GB", | 20260506 02:36:57.180 | INFO | "malloc" : " 14 GB", | 20260506 02:36:57.180 | INFO | "pinned" : " 29 GB" | 20260506 02:36:57.180 | INFO | } | 20260506 02:36:57.180 | INFO | Perf report for the last 5.14488 seconds: | 20260506 02:36:57.180 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:36:57.180 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:57.180 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:57.180 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:57.180 | INFO | ANALYSIS : 62.1%, 3.0, 97.1ms, 0 | 20260506 02:36:57.180 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:57.180 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:57.180 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 02:36:57.180 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:57.180 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:36:57.181 | INFO | Uploaded 36GiB to GPU in 2.106s (17.094GiB/s) | 20260506 02:36:57.181 | INFO | Downloaded 34GiB to GPU in 2.09004s (16.2676GiB/s) | 20260506 02:37:02.326 | INFO | Analyzing chunk frames = [8281600,8282112) | 20260506 02:37:02.326 | INFO | MemInfo: { | 20260506 02:37:02.326 | INFO | "gpu" : " 17 GB", | 20260506 02:37:02.326 | INFO | "huge" : "", | 20260506 02:37:02.326 | INFO | "huge_shared" : " 27 GB", | 20260506 02:37:02.326 | INFO | "malloc" : " 14 GB", | 20260506 02:37:02.326 | INFO | "pinned" : " 29 GB" | 20260506 02:37:02.326 | INFO | } | 20260506 02:37:02.326 | INFO | Perf report for the last 5.14612 seconds: | 20260506 02:37:02.327 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:37:02.327 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:02.327 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:02.327 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:02.327 | INFO | ANALYSIS : 61.2%, 3.0, 95.7ms, 0 | 20260506 02:37:02.327 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:02.327 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:02.327 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 02:37:02.327 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:02.327 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:02.327 | INFO | Uploaded 36GiB to GPU in 2.04659s (17.5903GiB/s) | 20260506 02:37:02.327 | INFO | Downloaded 34GiB to GPU in 2.01654s (16.8606GiB/s) | 20260506 02:37:02.327 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 02:37:02.327 | INFO | Total mem usage: 15.4 GiB | 20260506 02:37:02.327 | INFO | All Time High Water Marks sum: 16.3 GiB (Savings: 5.2%) | 20260506 02:37:02.327 | INFO | Recent High Water Marks Sum: 15.4 GiB (Savings: 0.0249%) | 20260506 02:37:02.327 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 02:37:02.327 | INFO | High Water Breakdown: | 20260506 02:37:02.327 | INFO | 62.3% : 9.62 GiB ( 100% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 02:37:02.327 | INFO | 37.6% : 5.8 GiB ( 87.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 02:37:02.327 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 02:37:02.327 | INFO | ------------------------------------------ | 20260506 02:37:02.327 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:37:02.327 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 02:37:02.327 | INFO | Total mem usage: 26.5 GiB | 20260506 02:37:02.327 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 02:37:02.327 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 02:37:02.327 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260506 02:37:02.327 | INFO | High Water Breakdown: | 20260506 02:37:02.327 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:37:02.327 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:37:02.327 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:37:02.327 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:37:02.327 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:37:02.327 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:37:02.327 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 02:37:02.327 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 02:37:02.327 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:37:02.327 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 02:37:02.327 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 02:37:02.327 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 02:37:02.327 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 02:37:02.327 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 02:37:02.327 | INFO | ------------------------------------------ | 20260506 02:37:02.327 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:37:02.327 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 02:37:02.327 | INFO | Total mem usage: 25.6 GiB | 20260506 02:37:02.327 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 02:37:02.327 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 02:37:02.327 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 02:37:02.327 | INFO | High Water Breakdown: | 20260506 02:37:02.327 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 02:37:02.328 | INFO | ------------------------------------------ | 20260506 02:37:02.328 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:37:02.328 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 02:37:02.328 | INFO | Total mem usage: 15.7 GiB | 20260506 02:37:02.328 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 02:37:02.328 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 02:37:02.328 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 02:37:02.328 | INFO | High Water Breakdown: | 20260506 02:37:02.328 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 02:37:02.328 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 02:37:02.328 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 02:37:02.328 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 02:37:02.328 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 02:37:02.328 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 02:37:02.328 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 02:37:02.328 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 02:37:02.328 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 02:37:02.328 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:37:02.328 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 02:37:02.328 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:37:02.328 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 02:37:02.328 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 02:37:02.328 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 02:37:02.328 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 02:37:02.328 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:37:02.328 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 02:37:02.328 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:37:02.328 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:37:02.328 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:37:02.328 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 02:37:02.328 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 02:37:02.328 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:37:02.328 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 02:37:02.328 | INFO | ------------------------------------------ | 20260506 02:37:02.328 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:37:02.328 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 02:37:02.328 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:37:02.328 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:37:02.328 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.36e+01, 1.34e+01 / 5.95e-01 | 20260506 02:37:02.328 | INFO | PerfReport: 0.29%, AggregateStats -- 9.18e-02 / 1.05e-01, 1.03e-02 / 4.85e-03 | 20260506 02:37:02.328 | INFO | PerfReport: 11.84%, Baseline -- 3.70e+00 / 4.51e+00, 5.72e-01 / 1.94e-01 | 20260506 02:37:02.328 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.51e+00, 1.90e-01 / 9.64e-02 | 20260506 02:37:02.328 | INFO | PerfReport: 15.98%, DME -- 3.99e+01 / 4.37e+01, 3.77e+00 / 7.32e-01 | 20260506 02:37:02.328 | INFO | PerfReport: 57.51%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.49e+00 / 1.14e+00 | 20260506 02:37:02.328 | INFO | PerfReport: 7.04%, PulseAccumulating -- 2.20e+00 / 2.71e+00, 4.19e-01 / 2.37e-01 | 20260506 02:37:02.328 | INFO | PerfReport: 3.01%, Metrics -- 9.41e-01 / 1.20e+00, 2.52e-01 / 1.16e-01 | 20260506 02:37:02.328 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.25e-01 / 3.75e-01, 5.33e-02 / 2.86e-02 | 20260506 02:37:02.328 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 02:37:02.328 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:37:02.328 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:37:02.328 | INFO | PerfReport: Full Execution -- 9.77e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260506 02:37:02.328 | INFO | PerfReport: 22.35%, Upload -- 2.18e+01 / 0.00e+00, 7.35e+00 / 0.00e+00 | 20260506 02:37:02.328 | INFO | PerfReport: 22.01%, Download -- 2.15e+01 / 0.00e+00, 7.62e+00 / 0.00e+00 | 20260506 02:37:02.328 | INFO | PerfReport: 31.99%, Compute -- 3.12e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 02:37:02.328 | INFO | PerfReport: 2.00%, LockUpload -- 1.96e+00 / 0.00e+00, 7.85e+00 / 0.00e+00 | 20260506 02:37:02.328 | INFO | PerfReport: 1.28%, LockDownload -- 1.25e+00 / 0.00e+00, 2.84e+00 / 0.00e+00 | 20260506 02:37:02.328 | INFO | PerfReport: 20.35%, LockCompute -- 1.99e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260506 02:37:02.328 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 2.08e-02 / 0.00e+00 | 20260506 02:37:02.454 | INFO | Starting a round of Baz IO | 20260506 02:37:05.578 | INFO | Populating RTMetrics report | 20260506 02:37:05.578 | INFO | RTMetrics report populated | 20260506 02:37:05.578 | INFO | Opening tmp RTMetrics json file | 20260506 02:37:05.669 | INFO | Serializing RTMetrics to json | 20260506 02:37:05.697 | INFO | Serialization done, swapping tmp file now... | 20260506 02:37:05.697 | INFO | RTMetrics done | 20260506 02:37:05.728 | INFO | Finished a round of Baz IO: Wrote 1.789 GiB in 3.274 seconds | 20260506 02:37:05.728 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.26% / 0.00% / 78.74% | 20260506 02:37:05.728 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0691414% / 0% / 9.13938e-05% | 20260506 02:37:05.728 | INFO | Baz IO overall: | 20260506 02:37:05.728 | INFO | BytesWritten : 4634887107541 | 20260506 02:37:05.729 | INFO | NumEvents : 2175590784025 | 20260506 02:37:07.461 | INFO | Analyzing chunk frames = [8282112,8282624) | 20260506 02:37:07.461 | INFO | MemInfo: { | 20260506 02:37:07.461 | INFO | "gpu" : " 17 GB", | 20260506 02:37:07.461 | INFO | "huge" : "", | 20260506 02:37:07.461 | INFO | "huge_shared" : " 27 GB", | 20260506 02:37:07.461 | INFO | "malloc" : " 14 GB", | 20260506 02:37:07.461 | INFO | "pinned" : " 29 GB" | 20260506 02:37:07.461 | INFO | } | 20260506 02:37:07.461 | INFO | Perf report for the last 5.13461 seconds: | 20260506 02:37:07.461 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:37:07.461 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:07.461 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:07.461 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:07.461 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 02:37:07.461 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:07.461 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260506 02:37:07.461 | INFO | PRE_HQ : 59.1%, 1.8, 53.9ms, 0 | 20260506 02:37:07.461 | INFO | BAZWRITER : 62.9%, 3.8, 121.3ms, 0 | 20260506 02:37:07.461 | INFO | BAZ_SAMPLING : 2.3%, 1.0, 1.2ms, 0 | 20260506 02:37:07.461 | INFO | Uploaded 36GiB to GPU in 2.12691s (16.926GiB/s) | 20260506 02:37:07.461 | INFO | Downloaded 38GiB to GPU in 2.23526s (17.0002GiB/s) | 20260506 02:37:12.596 | INFO | Analyzing chunk frames = [8282624,8283136) | 20260506 02:37:12.596 | INFO | MemInfo: { | 20260506 02:37:12.596 | INFO | "gpu" : " 17 GB", | 20260506 02:37:12.596 | INFO | "huge" : "", | 20260506 02:37:12.596 | INFO | "huge_shared" : " 27 GB", | 20260506 02:37:12.596 | INFO | "malloc" : " 14 GB", | 20260506 02:37:12.596 | INFO | "pinned" : " 29 GB" | 20260506 02:37:12.596 | INFO | } | 20260506 02:37:12.596 | INFO | Perf report for the last 5.13513 seconds: | 20260506 02:37:12.596 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:37:12.596 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:12.596 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:12.596 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:12.596 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 02:37:12.596 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:12.596 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:12.596 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:37:12.596 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:12.596 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:12.596 | INFO | Uploaded 36GiB to GPU in 1.93157s (18.6377GiB/s) | 20260506 02:37:12.597 | INFO | Downloaded 34GiB to GPU in 1.97622s (17.2045GiB/s) | 20260506 02:37:17.735 | INFO | Analyzing chunk frames = [8283136,8283648) | 20260506 02:37:17.735 | INFO | MemInfo: { | 20260506 02:37:17.735 | INFO | "gpu" : " 17 GB", | 20260506 02:37:17.735 | INFO | "huge" : "", | 20260506 02:37:17.735 | INFO | "huge_shared" : " 27 GB", | 20260506 02:37:17.735 | INFO | "malloc" : " 14 GB", | 20260506 02:37:17.735 | INFO | "pinned" : " 29 GB" | 20260506 02:37:17.735 | INFO | } | 20260506 02:37:17.735 | INFO | Perf report for the last 5.13912 seconds: | 20260506 02:37:17.735 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:37:17.735 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:17.735 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:17.735 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:17.735 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 02:37:17.735 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:17.735 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:17.735 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 02:37:17.735 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:17.735 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:17.736 | INFO | Uploaded 36GiB to GPU in 2.09183s (17.2098GiB/s) | 20260506 02:37:17.736 | INFO | Downloaded 34GiB to GPU in 2.02429s (16.796GiB/s) | 20260506 02:37:22.877 | INFO | Analyzing chunk frames = [8283648,8284160) | 20260506 02:37:22.877 | INFO | MemInfo: { | 20260506 02:37:22.877 | INFO | "gpu" : " 17 GB", | 20260506 02:37:22.877 | INFO | "huge" : "", | 20260506 02:37:22.877 | INFO | "huge_shared" : " 27 GB", | 20260506 02:37:22.878 | INFO | "malloc" : " 14 GB", | 20260506 02:37:22.878 | INFO | "pinned" : " 29 GB" | 20260506 02:37:22.878 | INFO | } | 20260506 02:37:22.878 | INFO | Perf report for the last 5.14238 seconds: | 20260506 02:37:22.878 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:37:22.878 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:22.878 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:22.878 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:22.878 | INFO | ANALYSIS : 61.0%, 3.0, 94.8ms, 0 | 20260506 02:37:22.878 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:22.878 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:22.878 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 02:37:22.878 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:22.878 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:22.878 | INFO | Uploaded 36GiB to GPU in 2.0784s (17.321GiB/s) | 20260506 02:37:22.878 | INFO | Downloaded 34GiB to GPU in 2.01578s (16.8669GiB/s) | 20260506 02:37:28.011 | INFO | Analyzing chunk frames = [8284160,8284672) | 20260506 02:37:28.011 | INFO | MemInfo: { | 20260506 02:37:28.011 | INFO | "gpu" : " 17 GB", | 20260506 02:37:28.011 | INFO | "huge" : "", | 20260506 02:37:28.011 | INFO | "huge_shared" : " 27 GB", | 20260506 02:37:28.011 | INFO | "malloc" : " 14 GB", | 20260506 02:37:28.011 | INFO | "pinned" : " 29 GB" | 20260506 02:37:28.011 | INFO | } | 20260506 02:37:28.011 | INFO | Perf report for the last 5.13313 seconds: | 20260506 02:37:28.011 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:37:28.011 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:28.011 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:28.011 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:28.011 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260506 02:37:28.011 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:28.011 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:28.011 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 02:37:28.011 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:28.011 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:28.011 | INFO | Uploaded 36GiB to GPU in 2.10803s (17.0776GiB/s) | 20260506 02:37:28.011 | INFO | Downloaded 34GiB to GPU in 2.03839s (16.6798GiB/s) | 20260506 02:37:33.148 | INFO | Analyzing chunk frames = [8284672,8285184) | 20260506 02:37:33.148 | INFO | MemInfo: { | 20260506 02:37:33.148 | INFO | "gpu" : " 17 GB", | 20260506 02:37:33.148 | INFO | "huge" : "", | 20260506 02:37:33.148 | INFO | "huge_shared" : " 27 GB", | 20260506 02:37:33.148 | INFO | "malloc" : " 14 GB", | 20260506 02:37:33.148 | INFO | "pinned" : " 29 GB" | 20260506 02:37:33.148 | INFO | } | 20260506 02:37:33.148 | INFO | Perf report for the last 5.137 seconds: | 20260506 02:37:33.148 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:37:33.148 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:33.148 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:33.148 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:33.148 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 02:37:33.148 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:33.148 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:33.148 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 02:37:33.148 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:33.148 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:33.148 | INFO | Uploaded 36GiB to GPU in 2.06188s (17.4598GiB/s) | 20260506 02:37:33.148 | INFO | Downloaded 34GiB to GPU in 2.03708s (16.6906GiB/s) | 20260506 02:37:38.292 | INFO | Analyzing chunk frames = [8285184,8285696) | 20260506 02:37:38.292 | INFO | MemInfo: { | 20260506 02:37:38.292 | INFO | "gpu" : " 17 GB", | 20260506 02:37:38.292 | INFO | "huge" : "", | 20260506 02:37:38.292 | INFO | "huge_shared" : " 27 GB", | 20260506 02:37:38.292 | INFO | "malloc" : " 14 GB", | 20260506 02:37:38.292 | INFO | "pinned" : " 29 GB" | 20260506 02:37:38.292 | INFO | } | 20260506 02:37:38.292 | INFO | Perf report for the last 5.14454 seconds: | 20260506 02:37:38.292 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:37:38.292 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:38.292 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:38.292 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:38.292 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260506 02:37:38.292 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:38.292 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:38.292 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 02:37:38.292 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:38.292 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:38.293 | INFO | Uploaded 36GiB to GPU in 2.07718s (17.3312GiB/s) | 20260506 02:37:38.293 | INFO | Downloaded 34GiB to GPU in 2.07433s (16.3908GiB/s) | 20260506 02:37:43.439 | INFO | Analyzing chunk frames = [8285696,8286208) | 20260506 02:37:43.439 | INFO | MemInfo: { | 20260506 02:37:43.439 | INFO | "gpu" : " 17 GB", | 20260506 02:37:43.439 | INFO | "huge" : "", | 20260506 02:37:43.439 | INFO | "huge_shared" : " 27 GB", | 20260506 02:37:43.439 | INFO | "malloc" : " 14 GB", | 20260506 02:37:43.439 | INFO | "pinned" : " 29 GB" | 20260506 02:37:43.439 | INFO | } | 20260506 02:37:43.439 | INFO | Perf report for the last 5.14694 seconds: | 20260506 02:37:43.439 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:37:43.439 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:43.439 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:43.439 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:43.439 | INFO | ANALYSIS : 61.3%, 3.0, 95.9ms, 0 | 20260506 02:37:43.439 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:43.439 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:43.439 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:37:43.439 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:43.439 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:43.440 | INFO | Uploaded 36GiB to GPU in 2.04212s (17.6287GiB/s) | 20260506 02:37:43.440 | INFO | Downloaded 34GiB to GPU in 2.02462s (16.7933GiB/s) | 20260506 02:37:43.562 | INFO | Starting a round of Baz IO | 20260506 02:37:46.687 | INFO | Populating RTMetrics report | 20260506 02:37:46.687 | INFO | RTMetrics report populated | 20260506 02:37:46.687 | INFO | Opening tmp RTMetrics json file | 20260506 02:37:46.687 | INFO | Serializing RTMetrics to json | 20260506 02:37:46.689 | INFO | Serialization done, swapping tmp file now... | 20260506 02:37:46.689 | INFO | RTMetrics done | 20260506 02:37:46.772 | INFO | Finished a round of Baz IO: Wrote 1.788 GiB in 3.209 seconds | 20260506 02:37:46.772 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.26% / 0.00% / 78.74% | 20260506 02:37:46.772 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0689785% / 0% / 7.94729e-05% | 20260506 02:37:46.772 | INFO | Baz IO overall: | 20260506 02:37:46.772 | INFO | BytesWritten : 4636807109356 | 20260506 02:37:46.772 | INFO | NumEvents : 2176463492543 | 20260506 02:37:48.580 | INFO | Analyzing chunk frames = [8286208,8286720) | 20260506 02:37:48.580 | INFO | MemInfo: { | 20260506 02:37:48.580 | INFO | "gpu" : " 17 GB", | 20260506 02:37:48.580 | INFO | "huge" : "", | 20260506 02:37:48.580 | INFO | "huge_shared" : " 27 GB", | 20260506 02:37:48.580 | INFO | "malloc" : " 14 GB", | 20260506 02:37:48.580 | INFO | "pinned" : " 29 GB" | 20260506 02:37:48.580 | INFO | } | 20260506 02:37:48.580 | INFO | Perf report for the last 5.14087 seconds: | 20260506 02:37:48.580 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:37:48.580 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:48.580 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:48.580 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:48.580 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260506 02:37:48.580 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:48.580 | INFO | RT_METRICS : 42.8%, 1.0, 22.5ms, 0 | 20260506 02:37:48.580 | INFO | PRE_HQ : 59.1%, 1.7, 53.5ms, 0 | 20260506 02:37:48.580 | INFO | BAZWRITER : 61.6%, 3.6, 112.9ms, 0 | 20260506 02:37:48.580 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:37:48.581 | INFO | Uploaded 36GiB to GPU in 2.09477s (17.1856GiB/s) | 20260506 02:37:48.581 | INFO | Downloaded 38GiB to GPU in 2.22966s (17.0429GiB/s) | 20260506 02:37:53.711 | INFO | Analyzing chunk frames = [8286720,8287232) | 20260506 02:37:53.711 | INFO | MemInfo: { | 20260506 02:37:53.711 | INFO | "gpu" : " 17 GB", | 20260506 02:37:53.711 | INFO | "huge" : "", | 20260506 02:37:53.711 | INFO | "huge_shared" : " 27 GB", | 20260506 02:37:53.711 | INFO | "malloc" : " 14 GB", | 20260506 02:37:53.711 | INFO | "pinned" : " 29 GB" | 20260506 02:37:53.711 | INFO | } | 20260506 02:37:53.712 | INFO | Perf report for the last 5.13125 seconds: | 20260506 02:37:53.712 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:37:53.712 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:53.712 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:53.712 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:53.712 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260506 02:37:53.712 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:53.712 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:53.712 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 02:37:53.712 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:53.712 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:53.712 | INFO | Uploaded 36GiB to GPU in 2.01731s (17.8455GiB/s) | 20260506 02:37:53.712 | INFO | Downloaded 34GiB to GPU in 1.99066s (17.0798GiB/s) | 20260506 02:37:58.848 | INFO | Analyzing chunk frames = [8287232,8287744) | 20260506 02:37:58.848 | INFO | MemInfo: { | 20260506 02:37:58.848 | INFO | "gpu" : " 17 GB", | 20260506 02:37:58.848 | INFO | "huge" : "", | 20260506 02:37:58.848 | INFO | "huge_shared" : " 27 GB", | 20260506 02:37:58.848 | INFO | "malloc" : " 14 GB", | 20260506 02:37:58.848 | INFO | "pinned" : " 29 GB" | 20260506 02:37:58.848 | INFO | } | 20260506 02:37:58.848 | INFO | Perf report for the last 5.13675 seconds: | 20260506 02:37:58.848 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:37:58.848 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:58.848 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:58.848 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:58.848 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 02:37:58.848 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:58.848 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:58.848 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:37:58.848 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:58.848 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:37:58.848 | INFO | Uploaded 36GiB to GPU in 2.10387s (17.1113GiB/s) | 20260506 02:37:58.848 | INFO | Downloaded 34GiB to GPU in 2.03689s (16.6921GiB/s) | 20260506 02:38:03.989 | INFO | Analyzing chunk frames = [8287744,8288256) | 20260506 02:38:03.989 | INFO | MemInfo: { | 20260506 02:38:03.989 | INFO | "gpu" : " 17 GB", | 20260506 02:38:03.989 | INFO | "huge" : "", | 20260506 02:38:03.989 | INFO | "huge_shared" : " 27 GB", | 20260506 02:38:03.990 | INFO | "malloc" : " 14 GB", | 20260506 02:38:03.990 | INFO | "pinned" : " 29 GB" | 20260506 02:38:03.990 | INFO | } | 20260506 02:38:03.990 | INFO | Perf report for the last 5.14128 seconds: | 20260506 02:38:03.990 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:38:03.990 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:03.990 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:03.990 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:03.990 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 02:38:03.990 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:03.990 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:03.990 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:38:03.990 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:03.990 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:03.990 | INFO | Uploaded 36GiB to GPU in 2.15116s (16.7352GiB/s) | 20260506 02:38:03.990 | INFO | Downloaded 34GiB to GPU in 2.05954s (16.5086GiB/s) | 20260506 02:38:09.126 | INFO | Analyzing chunk frames = [8288256,8288768) | 20260506 02:38:09.126 | INFO | MemInfo: { | 20260506 02:38:09.126 | INFO | "gpu" : " 17 GB", | 20260506 02:38:09.126 | INFO | "huge" : "", | 20260506 02:38:09.126 | INFO | "huge_shared" : " 27 GB", | 20260506 02:38:09.126 | INFO | "malloc" : " 14 GB", | 20260506 02:38:09.126 | INFO | "pinned" : " 29 GB" | 20260506 02:38:09.127 | INFO | } | 20260506 02:38:09.127 | INFO | Perf report for the last 5.13703 seconds: | 20260506 02:38:09.127 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:38:09.127 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:09.127 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:09.127 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:09.127 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 02:38:09.127 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:09.127 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:09.127 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 02:38:09.127 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:09.127 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:09.127 | INFO | Uploaded 36GiB to GPU in 2.07928s (17.3137GiB/s) | 20260506 02:38:09.127 | INFO | Downloaded 34GiB to GPU in 2.02833s (16.7626GiB/s) | 20260506 02:38:14.274 | INFO | Analyzing chunk frames = [8288768,8289280) | 20260506 02:38:14.274 | INFO | MemInfo: { | 20260506 02:38:14.274 | INFO | "gpu" : " 17 GB", | 20260506 02:38:14.274 | INFO | "huge" : "", | 20260506 02:38:14.274 | INFO | "huge_shared" : " 27 GB", | 20260506 02:38:14.274 | INFO | "malloc" : " 14 GB", | 20260506 02:38:14.274 | INFO | "pinned" : " 29 GB" | 20260506 02:38:14.274 | INFO | } | 20260506 02:38:14.274 | INFO | Perf report for the last 5.14742 seconds: | 20260506 02:38:14.274 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:38:14.274 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:14.274 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:14.274 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:14.274 | INFO | ANALYSIS : 60.8%, 3.0, 95.1ms, 0 | 20260506 02:38:14.274 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:14.274 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:14.274 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:38:14.274 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:14.274 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:14.274 | INFO | Uploaded 36GiB to GPU in 2.10991s (17.0623GiB/s) | 20260506 02:38:14.274 | INFO | Downloaded 34GiB to GPU in 2.04068s (16.6611GiB/s) | 20260506 02:38:19.413 | INFO | Analyzing chunk frames = [8289280,8289792) | 20260506 02:38:19.413 | INFO | MemInfo: { | 20260506 02:38:19.413 | INFO | "gpu" : " 17 GB", | 20260506 02:38:19.413 | INFO | "huge" : "", | 20260506 02:38:19.413 | INFO | "huge_shared" : " 27 GB", | 20260506 02:38:19.413 | INFO | "malloc" : " 14 GB", | 20260506 02:38:19.413 | INFO | "pinned" : " 29 GB" | 20260506 02:38:19.413 | INFO | } | 20260506 02:38:19.413 | INFO | Perf report for the last 5.13912 seconds: | 20260506 02:38:19.413 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:38:19.413 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:19.413 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:19.413 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:19.413 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260506 02:38:19.413 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:19.413 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:19.413 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 02:38:19.414 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:19.414 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:19.414 | INFO | Uploaded 36GiB to GPU in 2.12077s (16.9749GiB/s) | 20260506 02:38:19.414 | INFO | Downloaded 34GiB to GPU in 2.0906s (16.2632GiB/s) | 20260506 02:38:24.545 | INFO | Analyzing chunk frames = [8289792,8290304) | 20260506 02:38:24.545 | INFO | MemInfo: { | 20260506 02:38:24.545 | INFO | "gpu" : " 17 GB", | 20260506 02:38:24.545 | INFO | "huge" : "", | 20260506 02:38:24.545 | INFO | "huge_shared" : " 27 GB", | 20260506 02:38:24.545 | INFO | "malloc" : " 14 GB", | 20260506 02:38:24.545 | INFO | "pinned" : " 29 GB" | 20260506 02:38:24.545 | INFO | } | 20260506 02:38:24.545 | INFO | Perf report for the last 5.13181 seconds: | 20260506 02:38:24.545 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:38:24.545 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:24.545 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:24.545 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:24.545 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 02:38:24.545 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:24.545 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:24.545 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 02:38:24.545 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:24.545 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:24.545 | INFO | Uploaded 36GiB to GPU in 2.06281s (17.4519GiB/s) | 20260506 02:38:24.545 | INFO | Downloaded 34GiB to GPU in 2.02259s (16.8101GiB/s) | 20260506 02:38:24.545 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 02:38:24.545 | INFO | Total mem usage: 15.4 GiB | 20260506 02:38:24.545 | INFO | All Time High Water Marks sum: 16.3 GiB (Savings: 5.2%) | 20260506 02:38:24.545 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.57%) | 20260506 02:38:24.545 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 02:38:24.545 | INFO | High Water Breakdown: | 20260506 02:38:24.545 | INFO | 61% : 8.67 GiB ( 90.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 02:38:24.545 | INFO | 38.8% : 5.52 GiB ( 83.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 02:38:24.545 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 02:38:24.545 | INFO | ------------------------------------------ | 20260506 02:38:24.545 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:38:24.545 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 02:38:24.545 | INFO | Total mem usage: 26.5 GiB | 20260506 02:38:24.546 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 02:38:24.546 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 02:38:24.546 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 02:38:24.546 | INFO | High Water Breakdown: | 20260506 02:38:24.546 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:38:24.546 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:38:24.546 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:38:24.546 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:38:24.546 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:38:24.546 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:38:24.546 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 02:38:24.546 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 02:38:24.546 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:38:24.546 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 02:38:24.546 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 02:38:24.546 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 02:38:24.546 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 02:38:24.546 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 02:38:24.546 | INFO | ------------------------------------------ | 20260506 02:38:24.546 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:38:24.546 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 02:38:24.546 | INFO | Total mem usage: 25.6 GiB | 20260506 02:38:24.546 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 02:38:24.546 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 02:38:24.546 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 02:38:24.546 | INFO | High Water Breakdown: | 20260506 02:38:24.546 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 02:38:24.546 | INFO | ------------------------------------------ | 20260506 02:38:24.546 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:38:24.546 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 02:38:24.546 | INFO | Total mem usage: 15.7 GiB | 20260506 02:38:24.546 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 02:38:24.546 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 02:38:24.546 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 02:38:24.546 | INFO | High Water Breakdown: | 20260506 02:38:24.546 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 02:38:24.546 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 02:38:24.546 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 02:38:24.546 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 02:38:24.546 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 02:38:24.546 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 02:38:24.546 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 02:38:24.546 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 02:38:24.546 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 02:38:24.546 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:38:24.546 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 02:38:24.546 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:38:24.546 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 02:38:24.546 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 02:38:24.546 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 02:38:24.546 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 02:38:24.546 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:38:24.546 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 02:38:24.546 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:38:24.546 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:38:24.546 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:38:24.546 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 02:38:24.546 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 02:38:24.546 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:38:24.546 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 02:38:24.546 | INFO | ------------------------------------------ | 20260506 02:38:24.546 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:38:24.546 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 02:38:24.546 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:38:24.546 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:38:24.546 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.37e+01, 1.34e+01 / 7.59e-01 | 20260506 02:38:24.546 | INFO | PerfReport: 0.29%, AggregateStats -- 9.17e-02 / 1.04e-01, 1.02e-02 / 4.01e-03 | 20260506 02:38:24.546 | INFO | PerfReport: 11.86%, Baseline -- 3.71e+00 / 4.49e+00, 5.84e-01 / 2.85e-01 | 20260506 02:38:24.546 | INFO | PerfReport: 3.30%, Histogram -- 1.03e+00 / 1.52e+00, 1.94e-01 / 9.91e-02 | 20260506 02:38:24.546 | INFO | PerfReport: 15.99%, DME -- 4.00e+01 / 4.39e+01, 3.86e+00 / 8.90e-01 | 20260506 02:38:24.546 | INFO | PerfReport: 57.49%, FrameLabeling -- 1.80e+01 / 1.93e+01, 1.52e+00 / 1.03e+00 | 20260506 02:38:24.546 | INFO | PerfReport: 7.02%, PulseAccumulating -- 2.19e+00 / 2.74e+00, 4.15e-01 / 2.02e-01 | 20260506 02:38:24.546 | INFO | PerfReport: 3.01%, Metrics -- 9.42e-01 / 1.22e+00, 2.54e-01 / 1.13e-01 | 20260506 02:38:24.546 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.25e-01 / 3.75e-01, 5.38e-02 / 2.57e-02 | 20260506 02:38:24.546 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 02:38:24.546 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:38:24.546 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:38:24.546 | INFO | PerfReport: Full Execution -- 9.76e+01 / -nan, 2.42e+01 / 0.00e+00 | 20260506 02:38:24.546 | INFO | PerfReport: 22.18%, Upload -- 2.17e+01 / 0.00e+00, 7.20e+00 / 0.00e+00 | 20260506 02:38:24.546 | INFO | PerfReport: 21.96%, Download -- 2.14e+01 / 0.00e+00, 7.56e+00 / 0.00e+00 | 20260506 02:38:24.546 | INFO | PerfReport: 32.02%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 02:38:24.547 | INFO | PerfReport: 2.00%, LockUpload -- 1.95e+00 / 0.00e+00, 7.81e+00 / 0.00e+00 | 20260506 02:38:24.547 | INFO | PerfReport: 1.33%, LockDownload -- 1.30e+00 / 0.00e+00, 2.98e+00 / 0.00e+00 | 20260506 02:38:24.547 | INFO | PerfReport: 20.50%, LockCompute -- 2.00e+01 / 0.00e+00, 1.77e+01 / 0.00e+00 | 20260506 02:38:24.547 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.72e-02 / 0.00e+00 | 20260506 02:38:24.670 | INFO | Starting a round of Baz IO | 20260506 02:38:27.806 | INFO | Populating RTMetrics report | 20260506 02:38:27.806 | INFO | RTMetrics report populated | 20260506 02:38:27.806 | INFO | Opening tmp RTMetrics json file | 20260506 02:38:27.807 | INFO | Serializing RTMetrics to json | 20260506 02:38:27.808 | INFO | Serialization done, swapping tmp file now... | 20260506 02:38:27.809 | INFO | RTMetrics done | 20260506 02:38:27.891 | INFO | Finished a round of Baz IO: Wrote 1.787 GiB in 3.221 seconds | 20260506 02:38:27.891 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.26% / 0.00% / 78.74% | 20260506 02:38:27.891 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0686049% / 0% / 7.54992e-05% | 20260506 02:38:27.891 | INFO | Baz IO overall: | 20260506 02:38:27.891 | INFO | BytesWritten : 4638725927709 | 20260506 02:38:27.891 | INFO | NumEvents : 2177335550108 | 20260506 02:38:29.686 | INFO | Analyzing chunk frames = [8290304,8290816) | 20260506 02:38:29.686 | INFO | MemInfo: { | 20260506 02:38:29.686 | INFO | "gpu" : " 17 GB", | 20260506 02:38:29.686 | INFO | "huge" : "", | 20260506 02:38:29.686 | INFO | "huge_shared" : " 27 GB", | 20260506 02:38:29.686 | INFO | "malloc" : " 14 GB", | 20260506 02:38:29.686 | INFO | "pinned" : " 29 GB" | 20260506 02:38:29.686 | INFO | } | 20260506 02:38:29.686 | INFO | Perf report for the last 5.14067 seconds: | 20260506 02:38:29.686 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:38:29.686 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:29.686 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:29.686 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:29.686 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260506 02:38:29.686 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:29.686 | INFO | RT_METRICS : 42.8%, 1.0, 22.5ms, 0 | 20260506 02:38:29.686 | INFO | PRE_HQ : 59.5%, 1.7, 53.5ms, 0 | 20260506 02:38:29.686 | INFO | BAZWRITER : 61.6%, 3.4, 109.2ms, 0 | 20260506 02:38:29.686 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:38:29.686 | INFO | Uploaded 36GiB to GPU in 2.17937s (16.5185GiB/s) | 20260506 02:38:29.686 | INFO | Downloaded 38GiB to GPU in 2.23488s (17.0032GiB/s) | 20260506 02:38:34.825 | INFO | Analyzing chunk frames = [8290816,8291328) | 20260506 02:38:34.825 | INFO | MemInfo: { | 20260506 02:38:34.825 | INFO | "gpu" : " 17 GB", | 20260506 02:38:34.825 | INFO | "huge" : "", | 20260506 02:38:34.825 | INFO | "huge_shared" : " 27 GB", | 20260506 02:38:34.825 | INFO | "malloc" : " 14 GB", | 20260506 02:38:34.825 | INFO | "pinned" : " 29 GB" | 20260506 02:38:34.825 | INFO | } | 20260506 02:38:34.825 | INFO | Perf report for the last 5.13898 seconds: | 20260506 02:38:34.825 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:38:34.825 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:34.825 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:34.825 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:34.825 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260506 02:38:34.825 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:34.825 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:34.825 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 02:38:34.825 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:34.825 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:34.825 | INFO | Uploaded 36GiB to GPU in 1.96968s (18.2771GiB/s) | 20260506 02:38:34.825 | INFO | Downloaded 34GiB to GPU in 1.97587s (17.2076GiB/s) | 20260506 02:38:39.962 | INFO | Analyzing chunk frames = [8291328,8291840) | 20260506 02:38:39.962 | INFO | MemInfo: { | 20260506 02:38:39.962 | INFO | "gpu" : " 17 GB", | 20260506 02:38:39.962 | INFO | "huge" : "", | 20260506 02:38:39.962 | INFO | "huge_shared" : " 27 GB", | 20260506 02:38:39.962 | INFO | "malloc" : " 14 GB", | 20260506 02:38:39.962 | INFO | "pinned" : " 29 GB" | 20260506 02:38:39.962 | INFO | } | 20260506 02:38:39.962 | INFO | Perf report for the last 5.13759 seconds: | 20260506 02:38:39.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:38:39.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:39.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:39.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:39.962 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 02:38:39.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:39.962 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:39.962 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:38:39.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:39.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:39.963 | INFO | Uploaded 36GiB to GPU in 2.0901s (17.224GiB/s) | 20260506 02:38:39.963 | INFO | Downloaded 34GiB to GPU in 2.02989s (16.7496GiB/s) | 20260506 02:38:45.097 | INFO | Analyzing chunk frames = [8291840,8292352) | 20260506 02:38:45.097 | INFO | MemInfo: { | 20260506 02:38:45.097 | INFO | "gpu" : " 17 GB", | 20260506 02:38:45.097 | INFO | "huge" : "", | 20260506 02:38:45.097 | INFO | "huge_shared" : " 27 GB", | 20260506 02:38:45.097 | INFO | "malloc" : " 14 GB", | 20260506 02:38:45.097 | INFO | "pinned" : " 29 GB" | 20260506 02:38:45.097 | INFO | } | 20260506 02:38:45.097 | INFO | Perf report for the last 5.13489 seconds: | 20260506 02:38:45.097 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:38:45.097 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:45.097 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:45.097 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:45.097 | INFO | ANALYSIS : 61.6%, 3.0, 95.5ms, 0 | 20260506 02:38:45.097 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:45.097 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:45.097 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 02:38:45.097 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:45.097 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:45.098 | INFO | Uploaded 36GiB to GPU in 2.15698s (16.69GiB/s) | 20260506 02:38:45.098 | INFO | Downloaded 34GiB to GPU in 2.05759s (16.5242GiB/s) | 20260506 02:38:50.243 | INFO | Analyzing chunk frames = [8292352,8292864) | 20260506 02:38:50.243 | INFO | MemInfo: { | 20260506 02:38:50.243 | INFO | "gpu" : " 17 GB", | 20260506 02:38:50.243 | INFO | "huge" : "", | 20260506 02:38:50.243 | INFO | "huge_shared" : " 27 GB", | 20260506 02:38:50.243 | INFO | "malloc" : " 14 GB", | 20260506 02:38:50.243 | INFO | "pinned" : " 29 GB" | 20260506 02:38:50.243 | INFO | } | 20260506 02:38:50.243 | INFO | Perf report for the last 5.14543 seconds: | 20260506 02:38:50.243 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:38:50.243 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:50.243 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:50.243 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:50.243 | INFO | ANALYSIS : 60.9%, 3.0, 95.2ms, 0 | 20260506 02:38:50.243 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:50.243 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:50.243 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 02:38:50.243 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:50.243 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:50.243 | INFO | Uploaded 36GiB to GPU in 2.09902s (17.1508GiB/s) | 20260506 02:38:50.243 | INFO | Downloaded 34GiB to GPU in 2.04901s (16.5934GiB/s) | 20260506 02:38:55.388 | INFO | Analyzing chunk frames = [8292864,8293376) | 20260506 02:38:55.388 | INFO | MemInfo: { | 20260506 02:38:55.388 | INFO | "gpu" : " 17 GB", | 20260506 02:38:55.388 | INFO | "huge" : "", | 20260506 02:38:55.388 | INFO | "huge_shared" : " 27 GB", | 20260506 02:38:55.388 | INFO | "malloc" : " 14 GB", | 20260506 02:38:55.388 | INFO | "pinned" : " 29 GB" | 20260506 02:38:55.388 | INFO | } | 20260506 02:38:55.388 | INFO | Perf report for the last 5.14574 seconds: | 20260506 02:38:55.389 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:38:55.389 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:55.389 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:55.389 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:55.389 | INFO | ANALYSIS : 61.1%, 3.0, 95.6ms, 0 | 20260506 02:38:55.389 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:55.389 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:55.389 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:38:55.389 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:55.389 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:38:55.389 | INFO | Uploaded 36GiB to GPU in 2.11517s (17.0199GiB/s) | 20260506 02:38:55.389 | INFO | Downloaded 34GiB to GPU in 2.0446s (16.6291GiB/s) | 20260506 02:39:00.531 | INFO | Analyzing chunk frames = [8293376,8293888) | 20260506 02:39:00.531 | INFO | MemInfo: { | 20260506 02:39:00.531 | INFO | "gpu" : " 17 GB", | 20260506 02:39:00.531 | INFO | "huge" : "", | 20260506 02:39:00.531 | INFO | "huge_shared" : " 27 GB", | 20260506 02:39:00.531 | INFO | "malloc" : " 14 GB", | 20260506 02:39:00.531 | INFO | "pinned" : " 29 GB" | 20260506 02:39:00.531 | INFO | } | 20260506 02:39:00.531 | INFO | Perf report for the last 5.1424 seconds: | 20260506 02:39:00.531 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:39:00.531 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:00.531 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:00.531 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:00.531 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 02:39:00.531 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:00.531 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:00.531 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 02:39:00.531 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:00.531 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:00.531 | INFO | Uploaded 36GiB to GPU in 2.09549s (17.1798GiB/s) | 20260506 02:39:00.531 | INFO | Downloaded 34GiB to GPU in 2.05997s (16.5051GiB/s) | 20260506 02:39:05.661 | INFO | Analyzing chunk frames = [8293888,8294400) | 20260506 02:39:05.661 | INFO | MemInfo: { | 20260506 02:39:05.661 | INFO | "gpu" : " 17 GB", | 20260506 02:39:05.661 | INFO | "huge" : "", | 20260506 02:39:05.661 | INFO | "huge_shared" : " 27 GB", | 20260506 02:39:05.661 | INFO | "malloc" : " 14 GB", | 20260506 02:39:05.661 | INFO | "pinned" : " 29 GB" | 20260506 02:39:05.661 | INFO | } | 20260506 02:39:05.662 | INFO | Perf report for the last 5.13076 seconds: | 20260506 02:39:05.662 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:39:05.662 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:05.662 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:05.662 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:05.662 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 02:39:05.662 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:05.662 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:05.662 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 02:39:05.662 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:05.662 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:05.662 | INFO | Uploaded 36GiB to GPU in 2.05113s (17.5513GiB/s) | 20260506 02:39:05.662 | INFO | Downloaded 34GiB to GPU in 2.03653s (16.695GiB/s) | 20260506 02:39:05.787 | INFO | Starting a round of Baz IO | 20260506 02:39:08.922 | INFO | Populating RTMetrics report | 20260506 02:39:08.922 | INFO | RTMetrics report populated | 20260506 02:39:08.922 | INFO | Opening tmp RTMetrics json file | 20260506 02:39:08.924 | INFO | Serializing RTMetrics to json | 20260506 02:39:08.926 | INFO | Serialization done, swapping tmp file now... | 20260506 02:39:08.927 | INFO | RTMetrics done | 20260506 02:39:08.999 | INFO | Finished a round of Baz IO: Wrote 1.788 GiB in 3.211 seconds | 20260506 02:39:08.999 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.26% / 0.00% / 78.74% | 20260506 02:39:08.999 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0686765% / 0% / 7.94729e-05% | 20260506 02:39:08.999 | INFO | Baz IO overall: | 20260506 02:39:08.999 | INFO | BytesWritten : 4640645603777 | 20260506 02:39:08.999 | INFO | NumEvents : 2178208202993 | 20260506 02:39:10.799 | INFO | Analyzing chunk frames = [8294400,8294912) | 20260506 02:39:10.799 | INFO | MemInfo: { | 20260506 02:39:10.799 | INFO | "gpu" : " 17 GB", | 20260506 02:39:10.799 | INFO | "huge" : "", | 20260506 02:39:10.799 | INFO | "huge_shared" : " 27 GB", | 20260506 02:39:10.799 | INFO | "malloc" : " 14 GB", | 20260506 02:39:10.799 | INFO | "pinned" : " 29 GB" | 20260506 02:39:10.799 | INFO | } | 20260506 02:39:10.799 | INFO | Perf report for the last 5.13757 seconds: | 20260506 02:39:10.799 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:39:10.799 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:10.799 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:10.799 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:10.799 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260506 02:39:10.799 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:10.799 | INFO | RT_METRICS : 42.8%, 1.0, 22.4ms, 0 | 20260506 02:39:10.799 | INFO | PRE_HQ : 59.4%, 1.7, 53.3ms, 0 | 20260506 02:39:10.799 | INFO | BAZWRITER : 61.8%, 3.5, 110.5ms, 0 | 20260506 02:39:10.799 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:39:10.800 | INFO | Uploaded 36GiB to GPU in 2.14599s (16.7754GiB/s) | 20260506 02:39:10.800 | INFO | Downloaded 38GiB to GPU in 2.23434s (17.0072GiB/s) | 20260506 02:39:15.942 | INFO | Analyzing chunk frames = [8294912,8295424) | 20260506 02:39:15.942 | INFO | MemInfo: { | 20260506 02:39:15.942 | INFO | "gpu" : " 17 GB", | 20260506 02:39:15.942 | INFO | "huge" : "", | 20260506 02:39:15.942 | INFO | "huge_shared" : " 27 GB", | 20260506 02:39:15.942 | INFO | "malloc" : " 14 GB", | 20260506 02:39:15.942 | INFO | "pinned" : " 29 GB" | 20260506 02:39:15.942 | INFO | } | 20260506 02:39:15.942 | INFO | Perf report for the last 5.1433 seconds: | 20260506 02:39:15.942 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:39:15.942 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:15.942 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:15.943 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:15.943 | INFO | ANALYSIS : 60.7%, 3.0, 94.8ms, 0 | 20260506 02:39:15.943 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:15.943 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:15.943 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 02:39:15.943 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:15.943 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:15.943 | INFO | Uploaded 36GiB to GPU in 1.94944s (18.4668GiB/s) | 20260506 02:39:15.943 | INFO | Downloaded 34GiB to GPU in 1.97508s (17.2145GiB/s) | 20260506 02:39:21.084 | INFO | Analyzing chunk frames = [8295424,8295936) | 20260506 02:39:21.084 | INFO | MemInfo: { | 20260506 02:39:21.084 | INFO | "gpu" : " 17 GB", | 20260506 02:39:21.084 | INFO | "huge" : "", | 20260506 02:39:21.084 | INFO | "huge_shared" : " 27 GB", | 20260506 02:39:21.084 | INFO | "malloc" : " 14 GB", | 20260506 02:39:21.084 | INFO | "pinned" : " 29 GB" | 20260506 02:39:21.084 | INFO | } | 20260506 02:39:21.084 | INFO | Perf report for the last 5.14179 seconds: | 20260506 02:39:21.084 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:39:21.084 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:21.084 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:21.084 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:21.084 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 02:39:21.084 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:21.084 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:21.084 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 02:39:21.085 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:21.085 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:21.085 | INFO | Uploaded 36GiB to GPU in 2.09698s (17.1675GiB/s) | 20260506 02:39:21.085 | INFO | Downloaded 34GiB to GPU in 2.03874s (16.677GiB/s) | 20260506 02:39:26.217 | INFO | Analyzing chunk frames = [8295936,8296448) | 20260506 02:39:26.217 | INFO | MemInfo: { | 20260506 02:39:26.217 | INFO | "gpu" : " 17 GB", | 20260506 02:39:26.217 | INFO | "huge" : "", | 20260506 02:39:26.217 | INFO | "huge_shared" : " 27 GB", | 20260506 02:39:26.217 | INFO | "malloc" : " 14 GB", | 20260506 02:39:26.217 | INFO | "pinned" : " 29 GB" | 20260506 02:39:26.217 | INFO | } | 20260506 02:39:26.217 | INFO | Perf report for the last 5.1326 seconds: | 20260506 02:39:26.217 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:39:26.217 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:26.217 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:26.217 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:26.217 | INFO | ANALYSIS : 61.1%, 3.0, 94.7ms, 0 | 20260506 02:39:26.217 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:26.217 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:26.217 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:39:26.217 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:26.217 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:26.217 | INFO | Uploaded 36GiB to GPU in 2.0761s (17.3402GiB/s) | 20260506 02:39:26.217 | INFO | Downloaded 34GiB to GPU in 2.00265s (16.9775GiB/s) | 20260506 02:39:31.358 | INFO | Analyzing chunk frames = [8296448,8296960) | 20260506 02:39:31.358 | INFO | MemInfo: { | 20260506 02:39:31.358 | INFO | "gpu" : " 17 GB", | 20260506 02:39:31.358 | INFO | "huge" : "", | 20260506 02:39:31.358 | INFO | "huge_shared" : " 27 GB", | 20260506 02:39:31.358 | INFO | "malloc" : " 14 GB", | 20260506 02:39:31.358 | INFO | "pinned" : " 29 GB" | 20260506 02:39:31.358 | INFO | } | 20260506 02:39:31.358 | INFO | Perf report for the last 5.14136 seconds: | 20260506 02:39:31.358 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:39:31.358 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:31.358 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:31.358 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:31.358 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 02:39:31.358 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:31.358 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:31.358 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:39:31.358 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:31.358 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:31.359 | INFO | Uploaded 36GiB to GPU in 2.10852s (17.0736GiB/s) | 20260506 02:39:31.359 | INFO | Downloaded 34GiB to GPU in 2.04889s (16.5944GiB/s) | 20260506 02:39:36.499 | INFO | Analyzing chunk frames = [8296960,8297472) | 20260506 02:39:36.499 | INFO | MemInfo: { | 20260506 02:39:36.499 | INFO | "gpu" : " 17 GB", | 20260506 02:39:36.499 | INFO | "huge" : "", | 20260506 02:39:36.499 | INFO | "huge_shared" : " 27 GB", | 20260506 02:39:36.499 | INFO | "malloc" : " 14 GB", | 20260506 02:39:36.499 | INFO | "pinned" : " 29 GB" | 20260506 02:39:36.499 | INFO | } | 20260506 02:39:36.499 | INFO | Perf report for the last 5.14097 seconds: | 20260506 02:39:36.499 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:39:36.499 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:36.499 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:36.499 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:36.499 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 02:39:36.499 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:36.499 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:36.500 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 02:39:36.500 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:36.500 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:36.500 | INFO | Uploaded 36GiB to GPU in 2.06946s (17.3958GiB/s) | 20260506 02:39:36.500 | INFO | Downloaded 34GiB to GPU in 2.04238s (16.6472GiB/s) | 20260506 02:39:41.640 | INFO | Analyzing chunk frames = [8297472,8297984) | 20260506 02:39:41.640 | INFO | MemInfo: { | 20260506 02:39:41.640 | INFO | "gpu" : " 17 GB", | 20260506 02:39:41.640 | INFO | "huge" : "", | 20260506 02:39:41.640 | INFO | "huge_shared" : " 27 GB", | 20260506 02:39:41.640 | INFO | "malloc" : " 14 GB", | 20260506 02:39:41.640 | INFO | "pinned" : " 29 GB" | 20260506 02:39:41.640 | INFO | } | 20260506 02:39:41.640 | INFO | Perf report for the last 5.14099 seconds: | 20260506 02:39:41.640 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:39:41.640 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:41.640 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:41.640 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:41.640 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260506 02:39:41.640 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:41.640 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:41.641 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 02:39:41.641 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:41.641 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:41.641 | INFO | Uploaded 36GiB to GPU in 2.09107s (17.2161GiB/s) | 20260506 02:39:41.641 | INFO | Downloaded 34GiB to GPU in 2.06849s (16.4371GiB/s) | 20260506 02:39:46.775 | INFO | Analyzing chunk frames = [8297984,8298496) | 20260506 02:39:46.775 | INFO | MemInfo: { | 20260506 02:39:46.775 | INFO | "gpu" : " 17 GB", | 20260506 02:39:46.775 | INFO | "huge" : "", | 20260506 02:39:46.775 | INFO | "huge_shared" : " 27 GB", | 20260506 02:39:46.775 | INFO | "malloc" : " 14 GB", | 20260506 02:39:46.775 | INFO | "pinned" : " 29 GB" | 20260506 02:39:46.775 | INFO | } | 20260506 02:39:46.775 | INFO | Perf report for the last 5.13502 seconds: | 20260506 02:39:46.775 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:39:46.775 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:46.775 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:46.775 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:46.775 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 02:39:46.775 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:46.775 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:46.775 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 02:39:46.775 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:46.775 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:46.776 | INFO | Uploaded 36GiB to GPU in 2.06143s (17.4636GiB/s) | 20260506 02:39:46.776 | INFO | Downloaded 34GiB to GPU in 2.03504s (16.7073GiB/s) | 20260506 02:39:46.776 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 02:39:46.776 | INFO | Total mem usage: 15.4 GiB | 20260506 02:39:46.776 | INFO | All Time High Water Marks sum: 16.3 GiB (Savings: 5.2%) | 20260506 02:39:46.776 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.97%) | 20260506 02:39:46.776 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 02:39:46.776 | INFO | High Water Breakdown: | 20260506 02:39:46.776 | INFO | 60.9% : 8.54 GiB ( 88.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 02:39:46.776 | INFO | 39% : 5.47 GiB ( 82.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 02:39:46.776 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 02:39:46.776 | INFO | ------------------------------------------ | 20260506 02:39:46.776 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:39:46.776 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 02:39:46.776 | INFO | Total mem usage: 26.5 GiB | 20260506 02:39:46.776 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 02:39:46.776 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 02:39:46.776 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260506 02:39:46.776 | INFO | High Water Breakdown: | 20260506 02:39:46.776 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:39:46.776 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:39:46.776 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:39:46.776 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:39:46.776 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:39:46.776 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:39:46.776 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 02:39:46.776 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 02:39:46.776 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:39:46.776 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 02:39:46.776 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 02:39:46.776 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 02:39:46.776 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 02:39:46.776 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 02:39:46.776 | INFO | ------------------------------------------ | 20260506 02:39:46.776 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:39:46.776 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 02:39:46.776 | INFO | Total mem usage: 25.6 GiB | 20260506 02:39:46.776 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 02:39:46.776 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 02:39:46.776 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 02:39:46.776 | INFO | High Water Breakdown: | 20260506 02:39:46.776 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 02:39:46.776 | INFO | ------------------------------------------ | 20260506 02:39:46.776 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:39:46.776 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 02:39:46.776 | INFO | Total mem usage: 15.7 GiB | 20260506 02:39:46.776 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 02:39:46.776 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 02:39:46.776 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 02:39:46.776 | INFO | High Water Breakdown: | 20260506 02:39:46.776 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 02:39:46.776 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 02:39:46.776 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 02:39:46.776 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 02:39:46.776 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 02:39:46.776 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 02:39:46.776 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 02:39:46.776 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 02:39:46.776 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 02:39:46.776 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:39:46.776 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 02:39:46.776 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:39:46.776 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 02:39:46.776 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 02:39:46.776 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 02:39:46.776 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 02:39:46.776 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:39:46.776 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 02:39:46.776 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:39:46.776 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:39:46.776 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:39:46.776 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 02:39:46.776 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 02:39:46.776 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:39:46.776 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 02:39:46.776 | INFO | ------------------------------------------ | 20260506 02:39:46.776 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:39:46.777 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 02:39:46.777 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:39:46.777 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:39:46.777 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.36e+01, 1.34e+01 / 6.23e-01 | 20260506 02:39:46.777 | INFO | PerfReport: 0.30%, AggregateStats -- 9.23e-02 / 1.04e-01, 1.12e-02 / 4.02e-03 | 20260506 02:39:46.777 | INFO | PerfReport: 11.85%, Baseline -- 3.70e+00 / 4.50e+00, 5.74e-01 / 2.84e-01 | 20260506 02:39:46.777 | INFO | PerfReport: 3.29%, Histogram -- 1.03e+00 / 1.53e+00, 1.91e-01 / 1.04e-01 | 20260506 02:39:46.777 | INFO | PerfReport: 15.99%, DME -- 4.00e+01 / 4.39e+01, 3.88e+00 / 7.31e-01 | 20260506 02:39:46.777 | INFO | PerfReport: 57.48%, FrameLabeling -- 1.80e+01 / 1.93e+01, 1.50e+00 / 1.12e+00 | 20260506 02:39:46.777 | INFO | PerfReport: 7.04%, PulseAccumulating -- 2.20e+00 / 2.71e+00, 4.16e-01 / 2.57e-01 | 20260506 02:39:46.777 | INFO | PerfReport: 3.01%, Metrics -- 9.41e-01 / 1.21e+00, 2.49e-01 / 1.25e-01 | 20260506 02:39:46.777 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.24e-01 / 3.69e-01, 5.32e-02 / 3.01e-02 | 20260506 02:39:46.777 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 02:39:46.777 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:39:46.777 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:39:46.777 | INFO | PerfReport: Full Execution -- 9.77e+01 / -nan, 2.41e+01 / 0.00e+00 | 20260506 02:39:46.777 | INFO | PerfReport: 22.23%, Upload -- 2.17e+01 / 0.00e+00, 7.28e+00 / 0.00e+00 | 20260506 02:39:46.777 | INFO | PerfReport: 21.95%, Download -- 2.14e+01 / 0.00e+00, 7.59e+00 / 0.00e+00 | 20260506 02:39:46.777 | INFO | PerfReport: 31.99%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 02:39:46.777 | INFO | PerfReport: 1.98%, LockUpload -- 1.94e+00 / 0.00e+00, 7.60e+00 / 0.00e+00 | 20260506 02:39:46.777 | INFO | PerfReport: 1.29%, LockDownload -- 1.26e+00 / 0.00e+00, 2.85e+00 / 0.00e+00 | 20260506 02:39:46.777 | INFO | PerfReport: 20.54%, LockCompute -- 2.01e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260506 02:39:46.777 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.63e-02 / 0.00e+00 | 20260506 02:39:46.901 | INFO | Starting a round of Baz IO | 20260506 02:39:50.028 | INFO | Populating RTMetrics report | 20260506 02:39:50.029 | INFO | RTMetrics report populated | 20260506 02:39:50.029 | INFO | Opening tmp RTMetrics json file | 20260506 02:39:50.029 | INFO | Serializing RTMetrics to json | 20260506 02:39:50.031 | INFO | Serialization done, swapping tmp file now... | 20260506 02:39:50.031 | INFO | RTMetrics done | 20260506 02:39:50.110 | INFO | Finished a round of Baz IO: Wrote 1.786 GiB in 3.209 seconds | 20260506 02:39:50.110 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.26% / 0.00% / 78.74% | 20260506 02:39:50.110 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0679056% / 3.97364e-06% / 8.34465e-05% | 20260506 02:39:50.110 | INFO | Baz IO overall: | 20260506 02:39:50.110 | INFO | BytesWritten : 4642563806937 | 20260506 02:39:50.110 | INFO | NumEvents : 2179080013946 | 20260506 02:39:51.911 | INFO | Analyzing chunk frames = [8298496,8299008) | 20260506 02:39:51.911 | INFO | MemInfo: { | 20260506 02:39:51.911 | INFO | "gpu" : " 17 GB", | 20260506 02:39:51.911 | INFO | "huge" : "", | 20260506 02:39:51.911 | INFO | "huge_shared" : " 27 GB", | 20260506 02:39:51.911 | INFO | "malloc" : " 14 GB", | 20260506 02:39:51.911 | INFO | "pinned" : " 29 GB" | 20260506 02:39:51.911 | INFO | } | 20260506 02:39:51.911 | INFO | Perf report for the last 5.13544 seconds: | 20260506 02:39:51.911 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:39:51.911 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:51.911 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:51.911 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:51.911 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260506 02:39:51.911 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:51.911 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260506 02:39:51.911 | INFO | PRE_HQ : 59.5%, 1.7, 53.5ms, 0 | 20260506 02:39:51.911 | INFO | BAZWRITER : 61.8%, 3.5, 110.1ms, 0 | 20260506 02:39:51.911 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:39:51.911 | INFO | Uploaded 36GiB to GPU in 2.18161s (16.5016GiB/s) | 20260506 02:39:51.911 | INFO | Downloaded 38GiB to GPU in 2.24302s (16.9415GiB/s) | 20260506 02:39:57.051 | INFO | Analyzing chunk frames = [8299008,8299520) | 20260506 02:39:57.051 | INFO | MemInfo: { | 20260506 02:39:57.051 | INFO | "gpu" : " 17 GB", | 20260506 02:39:57.051 | INFO | "huge" : "", | 20260506 02:39:57.051 | INFO | "huge_shared" : " 27 GB", | 20260506 02:39:57.051 | INFO | "malloc" : " 14 GB", | 20260506 02:39:57.051 | INFO | "pinned" : " 29 GB" | 20260506 02:39:57.051 | INFO | } | 20260506 02:39:57.051 | INFO | Perf report for the last 5.14048 seconds: | 20260506 02:39:57.051 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:39:57.051 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:57.051 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:57.051 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:57.051 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 02:39:57.051 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:57.051 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:57.051 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 02:39:57.051 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:57.051 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:39:57.051 | INFO | Uploaded 36GiB to GPU in 1.97141s (18.261GiB/s) | 20260506 02:39:57.051 | INFO | Downloaded 34GiB to GPU in 1.9939s (17.052GiB/s) | 20260506 02:40:02.197 | INFO | Analyzing chunk frames = [8299520,8300032) | 20260506 02:40:02.197 | INFO | MemInfo: { | 20260506 02:40:02.197 | INFO | "gpu" : " 17 GB", | 20260506 02:40:02.197 | INFO | "huge" : "", | 20260506 02:40:02.197 | INFO | "huge_shared" : " 27 GB", | 20260506 02:40:02.197 | INFO | "malloc" : " 14 GB", | 20260506 02:40:02.197 | INFO | "pinned" : " 29 GB" | 20260506 02:40:02.197 | INFO | } | 20260506 02:40:02.197 | INFO | Perf report for the last 5.1461 seconds: | 20260506 02:40:02.197 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:40:02.197 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:02.197 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:02.197 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:02.197 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 02:40:02.197 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:02.197 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:02.197 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 02:40:02.197 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:02.198 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:02.198 | INFO | Uploaded 36GiB to GPU in 2.09571s (17.178GiB/s) | 20260506 02:40:02.198 | INFO | Downloaded 34GiB to GPU in 2.01986s (16.8328GiB/s) | 20260506 02:40:07.335 | INFO | Analyzing chunk frames = [8300032,8300544) | 20260506 02:40:07.335 | INFO | MemInfo: { | 20260506 02:40:07.335 | INFO | "gpu" : " 17 GB", | 20260506 02:40:07.335 | INFO | "huge" : "", | 20260506 02:40:07.335 | INFO | "huge_shared" : " 27 GB", | 20260506 02:40:07.335 | INFO | "malloc" : " 14 GB", | 20260506 02:40:07.335 | INFO | "pinned" : " 29 GB" | 20260506 02:40:07.335 | INFO | } | 20260506 02:40:07.335 | INFO | Perf report for the last 5.13787 seconds: | 20260506 02:40:07.335 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:40:07.335 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:07.335 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:07.335 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:07.335 | INFO | ANALYSIS : 61.0%, 3.0, 94.7ms, 0 | 20260506 02:40:07.335 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:07.335 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:07.335 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:40:07.335 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:07.335 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:07.335 | INFO | Uploaded 36GiB to GPU in 2.07243s (17.3709GiB/s) | 20260506 02:40:07.336 | INFO | Downloaded 34GiB to GPU in 2.00221s (16.9812GiB/s) | 20260506 02:40:12.478 | INFO | Analyzing chunk frames = [8300544,8301056) | 20260506 02:40:12.478 | INFO | MemInfo: { | 20260506 02:40:12.478 | INFO | "gpu" : " 17 GB", | 20260506 02:40:12.478 | INFO | "huge" : "", | 20260506 02:40:12.478 | INFO | "huge_shared" : " 27 GB", | 20260506 02:40:12.478 | INFO | "malloc" : " 14 GB", | 20260506 02:40:12.478 | INFO | "pinned" : " 29 GB" | 20260506 02:40:12.478 | INFO | } | 20260506 02:40:12.478 | INFO | Perf report for the last 5.14242 seconds: | 20260506 02:40:12.478 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:40:12.478 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:12.478 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:12.478 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:12.478 | INFO | ANALYSIS : 60.7%, 3.0, 94.8ms, 0 | 20260506 02:40:12.478 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:12.478 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:12.478 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 02:40:12.478 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:12.478 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:12.478 | INFO | Uploaded 36GiB to GPU in 2.10407s (17.1097GiB/s) | 20260506 02:40:12.478 | INFO | Downloaded 34GiB to GPU in 2.03372s (16.7181GiB/s) | 20260506 02:40:17.616 | INFO | Analyzing chunk frames = [8301056,8301568) | 20260506 02:40:17.617 | INFO | MemInfo: { | 20260506 02:40:17.617 | INFO | "gpu" : " 17 GB", | 20260506 02:40:17.617 | INFO | "huge" : "", | 20260506 02:40:17.617 | INFO | "huge_shared" : " 27 GB", | 20260506 02:40:17.617 | INFO | "malloc" : " 14 GB", | 20260506 02:40:17.617 | INFO | "pinned" : " 29 GB" | 20260506 02:40:17.617 | INFO | } | 20260506 02:40:17.617 | INFO | Perf report for the last 5.13914 seconds: | 20260506 02:40:17.617 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:40:17.617 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:17.617 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:17.617 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:17.617 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 02:40:17.617 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:17.617 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:17.617 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 02:40:17.617 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:17.617 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:17.617 | INFO | Uploaded 36GiB to GPU in 2.1046s (17.1054GiB/s) | 20260506 02:40:17.617 | INFO | Downloaded 34GiB to GPU in 2.0373s (16.6888GiB/s) | 20260506 02:40:22.753 | INFO | Analyzing chunk frames = [8301568,8302080) | 20260506 02:40:22.753 | INFO | MemInfo: { | 20260506 02:40:22.753 | INFO | "gpu" : " 17 GB", | 20260506 02:40:22.753 | INFO | "huge" : "", | 20260506 02:40:22.753 | INFO | "huge_shared" : " 27 GB", | 20260506 02:40:22.753 | INFO | "malloc" : " 14 GB", | 20260506 02:40:22.753 | INFO | "pinned" : " 29 GB" | 20260506 02:40:22.753 | INFO | } | 20260506 02:40:22.753 | INFO | Perf report for the last 5.13602 seconds: | 20260506 02:40:22.753 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:40:22.753 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:22.753 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:22.753 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:22.753 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260506 02:40:22.753 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:22.753 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:22.753 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 02:40:22.753 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:22.753 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:22.753 | INFO | Uploaded 36GiB to GPU in 2.10957s (17.0651GiB/s) | 20260506 02:40:22.753 | INFO | Downloaded 34GiB to GPU in 2.07224s (16.4074GiB/s) | 20260506 02:40:27.892 | INFO | Analyzing chunk frames = [8302080,8302592) | 20260506 02:40:27.892 | INFO | MemInfo: { | 20260506 02:40:27.892 | INFO | "gpu" : " 17 GB", | 20260506 02:40:27.892 | INFO | "huge" : "", | 20260506 02:40:27.892 | INFO | "huge_shared" : " 27 GB", | 20260506 02:40:27.892 | INFO | "malloc" : " 14 GB", | 20260506 02:40:27.892 | INFO | "pinned" : " 29 GB" | 20260506 02:40:27.892 | INFO | } | 20260506 02:40:27.892 | INFO | Perf report for the last 5.13947 seconds: | 20260506 02:40:27.892 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:40:27.893 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:27.893 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:27.893 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:27.893 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260506 02:40:27.893 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:27.893 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:27.893 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 02:40:27.893 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:27.893 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:27.893 | INFO | Uploaded 36GiB to GPU in 2.03058s (17.729GiB/s) | 20260506 02:40:27.893 | INFO | Downloaded 34GiB to GPU in 2.01657s (16.8603GiB/s) | 20260506 02:40:28.019 | INFO | Starting a round of Baz IO | 20260506 02:40:31.143 | INFO | Populating RTMetrics report | 20260506 02:40:31.143 | INFO | RTMetrics report populated | 20260506 02:40:31.143 | INFO | Opening tmp RTMetrics json file | 20260506 02:40:31.144 | INFO | Serializing RTMetrics to json | 20260506 02:40:31.145 | INFO | Serialization done, swapping tmp file now... | 20260506 02:40:31.146 | INFO | RTMetrics done | 20260506 02:40:31.215 | INFO | Finished a round of Baz IO: Wrote 1.786 GiB in 3.195 seconds | 20260506 02:40:31.215 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.26% / 0.00% / 78.74% | 20260506 02:40:31.215 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0681718% / 0% / 0.000111262% | 20260506 02:40:31.215 | INFO | Baz IO overall: | 20260506 02:40:31.215 | INFO | BytesWritten : 4644482020312 | 20260506 02:40:31.215 | INFO | NumEvents : 2179951770400 | 20260506 02:40:33.036 | INFO | Analyzing chunk frames = [8302592,8303104) | 20260506 02:40:33.036 | INFO | MemInfo: { | 20260506 02:40:33.036 | INFO | "gpu" : " 17 GB", | 20260506 02:40:33.036 | INFO | "huge" : "", | 20260506 02:40:33.036 | INFO | "huge_shared" : " 27 GB", | 20260506 02:40:33.036 | INFO | "malloc" : " 14 GB", | 20260506 02:40:33.036 | INFO | "pinned" : " 29 GB" | 20260506 02:40:33.036 | INFO | } | 20260506 02:40:33.037 | INFO | Perf report for the last 5.1442 seconds: | 20260506 02:40:33.037 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:40:33.037 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:33.037 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:33.037 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:33.037 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260506 02:40:33.037 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:33.037 | INFO | RT_METRICS : 42.8%, 1.0, 22.5ms, 0 | 20260506 02:40:33.037 | INFO | PRE_HQ : 59.1%, 1.7, 53.5ms, 0 | 20260506 02:40:33.037 | INFO | BAZWRITER : 61.3%, 3.4, 108.6ms, 0 | 20260506 02:40:33.037 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:40:33.037 | INFO | Uploaded 36GiB to GPU in 2.10335s (17.1156GiB/s) | 20260506 02:40:33.037 | INFO | Downloaded 38GiB to GPU in 2.21925s (17.1229GiB/s) | 20260506 02:40:38.172 | INFO | Analyzing chunk frames = [8303104,8303616) | 20260506 02:40:38.172 | INFO | MemInfo: { | 20260506 02:40:38.172 | INFO | "gpu" : " 17 GB", | 20260506 02:40:38.172 | INFO | "huge" : "", | 20260506 02:40:38.172 | INFO | "huge_shared" : " 27 GB", | 20260506 02:40:38.172 | INFO | "malloc" : " 14 GB", | 20260506 02:40:38.172 | INFO | "pinned" : " 29 GB" | 20260506 02:40:38.172 | INFO | } | 20260506 02:40:38.172 | INFO | Perf report for the last 5.13506 seconds: | 20260506 02:40:38.172 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:40:38.172 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:38.172 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:38.172 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:38.172 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 02:40:38.172 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:38.172 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:38.172 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 02:40:38.172 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:38.172 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:38.172 | INFO | Uploaded 36GiB to GPU in 1.96851s (18.2879GiB/s) | 20260506 02:40:38.172 | INFO | Downloaded 34GiB to GPU in 1.99356s (17.0549GiB/s) | 20260506 02:40:43.307 | INFO | Analyzing chunk frames = [8303616,8304128) | 20260506 02:40:43.307 | INFO | MemInfo: { | 20260506 02:40:43.307 | INFO | "gpu" : " 17 GB", | 20260506 02:40:43.307 | INFO | "huge" : "", | 20260506 02:40:43.307 | INFO | "huge_shared" : " 27 GB", | 20260506 02:40:43.307 | INFO | "malloc" : " 14 GB", | 20260506 02:40:43.307 | INFO | "pinned" : " 29 GB" | 20260506 02:40:43.307 | INFO | } | 20260506 02:40:43.307 | INFO | Perf report for the last 5.13546 seconds: | 20260506 02:40:43.307 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:40:43.307 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:43.307 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:43.307 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:43.307 | INFO | ANALYSIS : 61.3%, 3.0, 95.2ms, 0 | 20260506 02:40:43.307 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:43.307 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:43.307 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:40:43.307 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:43.307 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:43.308 | INFO | Uploaded 36GiB to GPU in 2.09315s (17.199GiB/s) | 20260506 02:40:43.308 | INFO | Downloaded 34GiB to GPU in 2.02582s (16.7833GiB/s) | 20260506 02:40:48.455 | INFO | Analyzing chunk frames = [8304128,8304640) | 20260506 02:40:48.455 | INFO | MemInfo: { | 20260506 02:40:48.455 | INFO | "gpu" : " 17 GB", | 20260506 02:40:48.455 | INFO | "huge" : "", | 20260506 02:40:48.455 | INFO | "huge_shared" : " 27 GB", | 20260506 02:40:48.455 | INFO | "malloc" : " 14 GB", | 20260506 02:40:48.455 | INFO | "pinned" : " 29 GB" | 20260506 02:40:48.455 | INFO | } | 20260506 02:40:48.455 | INFO | Perf report for the last 5.14808 seconds: | 20260506 02:40:48.455 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:40:48.455 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:48.455 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:48.455 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:48.455 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260506 02:40:48.455 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:48.455 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:48.455 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 02:40:48.455 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:48.455 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:48.455 | INFO | Uploaded 36GiB to GPU in 2.10095s (17.1351GiB/s) | 20260506 02:40:48.455 | INFO | Downloaded 34GiB to GPU in 2.02758s (16.7688GiB/s) | 20260506 02:40:53.592 | INFO | Analyzing chunk frames = [8304640,8305152) | 20260506 02:40:53.592 | INFO | MemInfo: { | 20260506 02:40:53.592 | INFO | "gpu" : " 17 GB", | 20260506 02:40:53.592 | INFO | "huge" : "", | 20260506 02:40:53.592 | INFO | "huge_shared" : " 27 GB", | 20260506 02:40:53.592 | INFO | "malloc" : " 14 GB", | 20260506 02:40:53.592 | INFO | "pinned" : " 29 GB" | 20260506 02:40:53.592 | INFO | } | 20260506 02:40:53.592 | INFO | Perf report for the last 5.13678 seconds: | 20260506 02:40:53.592 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:40:53.592 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:53.592 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:53.592 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:53.592 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260506 02:40:53.592 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:53.592 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:53.592 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 02:40:53.592 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:53.592 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:53.592 | INFO | Uploaded 36GiB to GPU in 2.11721s (17.0035GiB/s) | 20260506 02:40:53.592 | INFO | Downloaded 34GiB to GPU in 2.03758s (16.6865GiB/s) | 20260506 02:40:58.722 | INFO | Analyzing chunk frames = [8305152,8305664) | 20260506 02:40:58.722 | INFO | MemInfo: { | 20260506 02:40:58.722 | INFO | "gpu" : " 17 GB", | 20260506 02:40:58.722 | INFO | "huge" : "", | 20260506 02:40:58.722 | INFO | "huge_shared" : " 27 GB", | 20260506 02:40:58.722 | INFO | "malloc" : " 14 GB", | 20260506 02:40:58.722 | INFO | "pinned" : " 29 GB" | 20260506 02:40:58.722 | INFO | } | 20260506 02:40:58.722 | INFO | Perf report for the last 5.1301 seconds: | 20260506 02:40:58.722 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:40:58.722 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:58.722 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:58.722 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:58.722 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260506 02:40:58.722 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:58.722 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:58.722 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 02:40:58.722 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:58.722 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:40:58.722 | INFO | Uploaded 36GiB to GPU in 2.05208s (17.5432GiB/s) | 20260506 02:40:58.722 | INFO | Downloaded 34GiB to GPU in 2.02646s (16.778GiB/s) | 20260506 02:41:03.862 | INFO | Analyzing chunk frames = [8305664,8306176) | 20260506 02:41:03.862 | INFO | MemInfo: { | 20260506 02:41:03.862 | INFO | "gpu" : " 17 GB", | 20260506 02:41:03.862 | INFO | "huge" : "", | 20260506 02:41:03.862 | INFO | "huge_shared" : " 27 GB", | 20260506 02:41:03.862 | INFO | "malloc" : " 14 GB", | 20260506 02:41:03.862 | INFO | "pinned" : " 29 GB" | 20260506 02:41:03.862 | INFO | } | 20260506 02:41:03.862 | INFO | Perf report for the last 5.14031 seconds: | 20260506 02:41:03.863 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:41:03.863 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:03.863 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:03.863 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:03.863 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260506 02:41:03.863 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:03.863 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:03.863 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 02:41:03.863 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:03.863 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:03.863 | INFO | Uploaded 36GiB to GPU in 2.10212s (17.1256GiB/s) | 20260506 02:41:03.863 | INFO | Downloaded 34GiB to GPU in 2.07963s (16.3491GiB/s) | 20260506 02:41:09.003 | INFO | Analyzing chunk frames = [8306176,8306688) | 20260506 02:41:09.004 | INFO | MemInfo: { | 20260506 02:41:09.004 | INFO | "gpu" : " 17 GB", | 20260506 02:41:09.004 | INFO | "huge" : "", | 20260506 02:41:09.004 | INFO | "huge_shared" : " 27 GB", | 20260506 02:41:09.004 | INFO | "malloc" : " 14 GB", | 20260506 02:41:09.004 | INFO | "pinned" : " 29 GB" | 20260506 02:41:09.004 | INFO | } | 20260506 02:41:09.004 | INFO | Perf report for the last 5.14116 seconds: | 20260506 02:41:09.004 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:41:09.004 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:09.004 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:09.004 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:09.004 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260506 02:41:09.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:09.004 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:09.004 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 02:41:09.004 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:09.004 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:09.004 | INFO | Uploaded 36GiB to GPU in 2.12733s (16.9226GiB/s) | 20260506 02:41:09.004 | INFO | Downloaded 34GiB to GPU in 2.02603s (16.7816GiB/s) | 20260506 02:41:09.004 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 02:41:09.004 | INFO | Total mem usage: 15.4 GiB | 20260506 02:41:09.004 | INFO | All Time High Water Marks sum: 16.3 GiB (Savings: 5.2%) | 20260506 02:41:09.004 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.96%) | 20260506 02:41:09.004 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 02:41:09.004 | INFO | High Water Breakdown: | 20260506 02:41:09.004 | INFO | 60.9% : 8.55 GiB ( 88.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 02:41:09.004 | INFO | 39% : 5.47 GiB ( 82.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 02:41:09.004 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 02:41:09.004 | INFO | ------------------------------------------ | 20260506 02:41:09.004 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:41:09.004 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 02:41:09.004 | INFO | Total mem usage: 26.5 GiB | 20260506 02:41:09.004 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 02:41:09.004 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 02:41:09.004 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 02:41:09.004 | INFO | High Water Breakdown: | 20260506 02:41:09.004 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:41:09.004 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:41:09.004 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:41:09.004 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:41:09.004 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:41:09.005 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:41:09.005 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 02:41:09.005 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 02:41:09.005 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:41:09.005 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 02:41:09.005 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 02:41:09.005 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 02:41:09.005 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 02:41:09.005 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 02:41:09.005 | INFO | ------------------------------------------ | 20260506 02:41:09.005 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:41:09.005 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 02:41:09.005 | INFO | Total mem usage: 25.6 GiB | 20260506 02:41:09.005 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 02:41:09.005 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 02:41:09.005 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 02:41:09.005 | INFO | High Water Breakdown: | 20260506 02:41:09.005 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 02:41:09.005 | INFO | ------------------------------------------ | 20260506 02:41:09.005 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:41:09.005 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 02:41:09.005 | INFO | Total mem usage: 15.7 GiB | 20260506 02:41:09.005 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 02:41:09.005 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 02:41:09.005 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 02:41:09.005 | INFO | High Water Breakdown: | 20260506 02:41:09.005 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 02:41:09.005 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 02:41:09.005 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 02:41:09.005 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 02:41:09.005 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 02:41:09.005 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 02:41:09.005 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 02:41:09.005 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 02:41:09.005 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 02:41:09.005 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:41:09.005 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 02:41:09.005 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:41:09.005 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 02:41:09.005 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 02:41:09.005 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 02:41:09.005 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 02:41:09.005 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:41:09.005 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 02:41:09.005 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:41:09.005 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:41:09.005 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:41:09.005 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 02:41:09.005 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 02:41:09.005 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:41:09.005 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 02:41:09.005 | INFO | ------------------------------------------ | 20260506 02:41:09.005 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:41:09.005 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 02:41:09.005 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:41:09.005 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:41:09.005 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.39e+01, 1.32e+01 / 6.89e-01 | 20260506 02:41:09.005 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.04e-01, 1.14e-02 / 3.79e-03 | 20260506 02:41:09.005 | INFO | PerfReport: 11.85%, Baseline -- 3.71e+00 / 4.52e+00, 5.81e-01 / 2.47e-01 | 20260506 02:41:09.005 | INFO | PerfReport: 3.30%, Histogram -- 1.03e+00 / 1.52e+00, 1.94e-01 / 1.00e-01 | 20260506 02:41:09.005 | INFO | PerfReport: 16.00%, DME -- 4.01e+01 / 4.40e+01, 3.75e+00 / 8.89e-01 | 20260506 02:41:09.005 | INFO | PerfReport: 57.54%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.58e+00 / 1.03e+00 | 20260506 02:41:09.005 | INFO | PerfReport: 6.99%, PulseAccumulating -- 2.19e+00 / 2.72e+00, 4.16e-01 / 2.25e-01 | 20260506 02:41:09.005 | INFO | PerfReport: 3.00%, Metrics -- 9.39e-01 / 1.21e+00, 2.53e-01 / 1.28e-01 | 20260506 02:41:09.005 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.25e-01 / 3.70e-01, 5.40e-02 / 2.88e-02 | 20260506 02:41:09.005 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 02:41:09.005 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:41:09.005 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:41:09.005 | INFO | PerfReport: Full Execution -- 9.76e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260506 02:41:09.005 | INFO | PerfReport: 22.23%, Upload -- 2.17e+01 / 0.00e+00, 7.23e+00 / 0.00e+00 | 20260506 02:41:09.005 | INFO | PerfReport: 21.91%, Download -- 2.14e+01 / 0.00e+00, 7.55e+00 / 0.00e+00 | 20260506 02:41:09.005 | INFO | PerfReport: 32.07%, Compute -- 3.13e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260506 02:41:09.005 | INFO | PerfReport: 2.01%, LockUpload -- 1.97e+00 / 0.00e+00, 7.87e+00 / 0.00e+00 | 20260506 02:41:09.005 | INFO | PerfReport: 1.26%, LockDownload -- 1.23e+00 / 0.00e+00, 2.85e+00 / 0.00e+00 | 20260506 02:41:09.005 | INFO | PerfReport: 20.49%, LockCompute -- 2.00e+01 / 0.00e+00, 1.77e+01 / 0.00e+00 | 20260506 02:41:09.005 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.63e-02 / 0.00e+00 | 20260506 02:41:09.130 | INFO | Starting a round of Baz IO | 20260506 02:41:12.261 | INFO | Populating RTMetrics report | 20260506 02:41:12.261 | INFO | RTMetrics report populated | 20260506 02:41:12.261 | INFO | Opening tmp RTMetrics json file | 20260506 02:41:12.262 | INFO | Serializing RTMetrics to json | 20260506 02:41:12.264 | INFO | Serialization done, swapping tmp file now... | 20260506 02:41:12.264 | INFO | RTMetrics done | 20260506 02:41:12.344 | INFO | Finished a round of Baz IO: Wrote 1.787 GiB in 3.213 seconds | 20260506 02:41:12.344 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.26% / 0.00% / 78.74% | 20260506 02:41:12.344 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0679692% / 0% / 7.54992e-05% | 20260506 02:41:12.344 | INFO | Baz IO overall: | 20260506 02:41:12.344 | INFO | BytesWritten : 4646401010709 | 20260506 02:41:12.344 | INFO | NumEvents : 2180824227772 | 20260506 02:41:14.148 | INFO | Analyzing chunk frames = [8306688,8307200) | 20260506 02:41:14.148 | INFO | MemInfo: { | 20260506 02:41:14.148 | INFO | "gpu" : " 17 GB", | 20260506 02:41:14.148 | INFO | "huge" : "", | 20260506 02:41:14.148 | INFO | "huge_shared" : " 27 GB", | 20260506 02:41:14.148 | INFO | "malloc" : " 14 GB", | 20260506 02:41:14.148 | INFO | "pinned" : " 29 GB" | 20260506 02:41:14.148 | INFO | } | 20260506 02:41:14.148 | INFO | Perf report for the last 5.1445 seconds: | 20260506 02:41:14.148 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:41:14.148 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:14.148 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:14.148 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:14.148 | INFO | ANALYSIS : 62.3%, 3.0, 97.4ms, 0 | 20260506 02:41:14.148 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:14.148 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260506 02:41:14.148 | INFO | PRE_HQ : 59.4%, 1.7, 53.2ms, 0 | 20260506 02:41:14.148 | INFO | BAZWRITER : 61.8%, 3.4, 109.3ms, 0 | 20260506 02:41:14.148 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:41:14.148 | INFO | Uploaded 36GiB to GPU in 2.18305s (16.4907GiB/s) | 20260506 02:41:14.148 | INFO | Downloaded 38GiB to GPU in 2.24655s (16.9148GiB/s) | 20260506 02:41:19.292 | INFO | Analyzing chunk frames = [8307200,8307712) | 20260506 02:41:19.292 | INFO | MemInfo: { | 20260506 02:41:19.292 | INFO | "gpu" : " 17 GB", | 20260506 02:41:19.292 | INFO | "huge" : "", | 20260506 02:41:19.292 | INFO | "huge_shared" : " 27 GB", | 20260506 02:41:19.292 | INFO | "malloc" : " 14 GB", | 20260506 02:41:19.292 | INFO | "pinned" : " 29 GB" | 20260506 02:41:19.292 | INFO | } | 20260506 02:41:19.292 | INFO | Perf report for the last 5.14401 seconds: | 20260506 02:41:19.292 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:41:19.292 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:19.292 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:19.292 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:19.292 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260506 02:41:19.292 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:19.292 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:19.292 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 02:41:19.292 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:19.292 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:19.292 | INFO | Uploaded 36GiB to GPU in 1.93787s (18.5771GiB/s) | 20260506 02:41:19.293 | INFO | Downloaded 34GiB to GPU in 1.98657s (17.1149GiB/s) | 20260506 02:41:24.429 | INFO | Analyzing chunk frames = [8307712,8308224) | 20260506 02:41:24.429 | INFO | MemInfo: { | 20260506 02:41:24.429 | INFO | "gpu" : " 17 GB", | 20260506 02:41:24.429 | INFO | "huge" : "", | 20260506 02:41:24.429 | INFO | "huge_shared" : " 27 GB", | 20260506 02:41:24.429 | INFO | "malloc" : " 14 GB", | 20260506 02:41:24.429 | INFO | "pinned" : " 29 GB" | 20260506 02:41:24.429 | INFO | } | 20260506 02:41:24.429 | INFO | Perf report for the last 5.13668 seconds: | 20260506 02:41:24.429 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:41:24.429 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:24.429 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:24.429 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:24.429 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260506 02:41:24.429 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:24.429 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:24.429 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 02:41:24.429 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:24.429 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:24.429 | INFO | Uploaded 36GiB to GPU in 2.11044s (17.058GiB/s) | 20260506 02:41:24.429 | INFO | Downloaded 34GiB to GPU in 2.04173s (16.6525GiB/s) | 20260506 02:41:29.564 | INFO | Analyzing chunk frames = [8308224,8308736) | 20260506 02:41:29.564 | INFO | MemInfo: { | 20260506 02:41:29.564 | INFO | "gpu" : " 17 GB", | 20260506 02:41:29.564 | INFO | "huge" : "", | 20260506 02:41:29.564 | INFO | "huge_shared" : " 27 GB", | 20260506 02:41:29.564 | INFO | "malloc" : " 14 GB", | 20260506 02:41:29.564 | INFO | "pinned" : " 29 GB" | 20260506 02:41:29.564 | INFO | } | 20260506 02:41:29.565 | INFO | Perf report for the last 5.13573 seconds: | 20260506 02:41:29.565 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:41:29.565 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:29.565 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:29.565 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:29.565 | INFO | ANALYSIS : 61.2%, 3.0, 95.0ms, 0 | 20260506 02:41:29.565 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:29.565 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:29.565 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:41:29.565 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:29.565 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:29.565 | INFO | Uploaded 36GiB to GPU in 2.09126s (17.2145GiB/s) | 20260506 02:41:29.565 | INFO | Downloaded 34GiB to GPU in 2.02423s (16.7965GiB/s) | 20260506 02:41:34.704 | INFO | Analyzing chunk frames = [8308736,8309248) | 20260506 02:41:34.704 | INFO | MemInfo: { | 20260506 02:41:34.704 | INFO | "gpu" : " 17 GB", | 20260506 02:41:34.704 | INFO | "huge" : "", | 20260506 02:41:34.704 | INFO | "huge_shared" : " 27 GB", | 20260506 02:41:34.704 | INFO | "malloc" : " 14 GB", | 20260506 02:41:34.704 | INFO | "pinned" : " 29 GB" | 20260506 02:41:34.704 | INFO | } | 20260506 02:41:34.704 | INFO | Perf report for the last 5.13906 seconds: | 20260506 02:41:34.704 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:41:34.704 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:34.704 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:34.704 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:34.704 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 02:41:34.704 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:34.704 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:34.704 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 02:41:34.704 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:34.704 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:34.704 | INFO | Uploaded 36GiB to GPU in 2.08893s (17.2337GiB/s) | 20260506 02:41:34.704 | INFO | Downloaded 34GiB to GPU in 2.02835s (16.7624GiB/s) | 20260506 02:41:39.842 | INFO | Analyzing chunk frames = [8309248,8309760) | 20260506 02:41:39.842 | INFO | MemInfo: { | 20260506 02:41:39.842 | INFO | "gpu" : " 17 GB", | 20260506 02:41:39.842 | INFO | "huge" : "", | 20260506 02:41:39.842 | INFO | "huge_shared" : " 27 GB", | 20260506 02:41:39.842 | INFO | "malloc" : " 14 GB", | 20260506 02:41:39.842 | INFO | "pinned" : " 29 GB" | 20260506 02:41:39.842 | INFO | } | 20260506 02:41:39.842 | INFO | Perf report for the last 5.13818 seconds: | 20260506 02:41:39.842 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:41:39.842 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:39.842 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:39.842 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:39.842 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 02:41:39.842 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:39.842 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:39.842 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:41:39.842 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:39.842 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:39.842 | INFO | Uploaded 36GiB to GPU in 2.09576s (17.1775GiB/s) | 20260506 02:41:39.842 | INFO | Downloaded 34GiB to GPU in 2.03659s (16.6946GiB/s) | 20260506 02:41:44.979 | INFO | Analyzing chunk frames = [8309760,8310272) | 20260506 02:41:44.979 | INFO | MemInfo: { | 20260506 02:41:44.979 | INFO | "gpu" : " 17 GB", | 20260506 02:41:44.979 | INFO | "huge" : "", | 20260506 02:41:44.979 | INFO | "huge_shared" : " 27 GB", | 20260506 02:41:44.979 | INFO | "malloc" : " 14 GB", | 20260506 02:41:44.979 | INFO | "pinned" : " 29 GB" | 20260506 02:41:44.979 | INFO | } | 20260506 02:41:44.979 | INFO | Perf report for the last 5.13757 seconds: | 20260506 02:41:44.979 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:41:44.979 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:44.979 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:44.979 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:44.979 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 02:41:44.979 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:44.979 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:44.979 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260506 02:41:44.980 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:44.980 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:44.980 | INFO | Uploaded 36GiB to GPU in 2.02938s (17.7394GiB/s) | 20260506 02:41:44.980 | INFO | Downloaded 34GiB to GPU in 2.04646s (16.614GiB/s) | 20260506 02:41:50.118 | INFO | Analyzing chunk frames = [8310272,8310784) | 20260506 02:41:50.118 | INFO | MemInfo: { | 20260506 02:41:50.118 | INFO | "gpu" : " 17 GB", | 20260506 02:41:50.118 | INFO | "huge" : "", | 20260506 02:41:50.118 | INFO | "huge_shared" : " 27 GB", | 20260506 02:41:50.118 | INFO | "malloc" : " 14 GB", | 20260506 02:41:50.118 | INFO | "pinned" : " 29 GB" | 20260506 02:41:50.118 | INFO | } | 20260506 02:41:50.118 | INFO | Perf report for the last 5.13886 seconds: | 20260506 02:41:50.118 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:41:50.118 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:50.118 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:50.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:50.118 | INFO | ANALYSIS : 61.8%, 3.0, 95.4ms, 0 | 20260506 02:41:50.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:50.118 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:50.118 | INFO | PRE_HQ : 26.4%, 1.0, 13.6ms, 0 | 20260506 02:41:50.118 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:50.118 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:50.118 | INFO | Uploaded 36GiB to GPU in 2.12779s (16.919GiB/s) | 20260506 02:41:50.119 | INFO | Downloaded 34GiB to GPU in 2.02836s (16.7623GiB/s) | 20260506 02:41:50.246 | INFO | Starting a round of Baz IO | 20260506 02:41:53.373 | INFO | Populating RTMetrics report | 20260506 02:41:53.373 | INFO | RTMetrics report populated | 20260506 02:41:53.373 | INFO | Opening tmp RTMetrics json file | 20260506 02:41:53.376 | INFO | Serializing RTMetrics to json | 20260506 02:41:53.377 | INFO | Serialization done, swapping tmp file now... | 20260506 02:41:53.377 | INFO | RTMetrics done | 20260506 02:41:53.453 | INFO | Finished a round of Baz IO: Wrote 1.785 GiB in 3.207 seconds | 20260506 02:41:53.453 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.26% / 0.00% / 78.74% | 20260506 02:41:53.453 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0677943% / 7.94729e-06% / 6.75519e-05% | 20260506 02:41:53.453 | INFO | Baz IO overall: | 20260506 02:41:53.453 | INFO | BytesWritten : 4648317264716 | 20260506 02:41:53.453 | INFO | NumEvents : 2181694766584 | 20260506 02:41:55.262 | INFO | Analyzing chunk frames = [8310784,8311296) | 20260506 02:41:55.262 | INFO | MemInfo: { | 20260506 02:41:55.262 | INFO | "gpu" : " 17 GB", | 20260506 02:41:55.262 | INFO | "huge" : "", | 20260506 02:41:55.262 | INFO | "huge_shared" : " 27 GB", | 20260506 02:41:55.262 | INFO | "malloc" : " 14 GB", | 20260506 02:41:55.262 | INFO | "pinned" : " 29 GB" | 20260506 02:41:55.262 | INFO | } | 20260506 02:41:55.263 | INFO | Perf report for the last 5.14425 seconds: | 20260506 02:41:55.263 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:41:55.263 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:55.263 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:55.263 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:55.263 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260506 02:41:55.263 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:41:55.263 | INFO | RT_METRICS : 43.0%, 1.0, 22.6ms, 0 | 20260506 02:41:55.263 | INFO | PRE_HQ : 59.5%, 1.7, 53.6ms, 0 | 20260506 02:41:55.263 | INFO | BAZWRITER : 61.7%, 3.6, 115.3ms, 0 | 20260506 02:41:55.263 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 02:41:55.263 | INFO | Uploaded 36GiB to GPU in 2.17882s (16.5227GiB/s) | 20260506 02:41:55.263 | INFO | Downloaded 38GiB to GPU in 2.24078s (16.9584GiB/s) | 20260506 02:42:00.410 | INFO | Analyzing chunk frames = [8311296,8311808) | 20260506 02:42:00.410 | INFO | MemInfo: { | 20260506 02:42:00.410 | INFO | "gpu" : " 17 GB", | 20260506 02:42:00.410 | INFO | "huge" : "", | 20260506 02:42:00.410 | INFO | "huge_shared" : " 27 GB", | 20260506 02:42:00.410 | INFO | "malloc" : " 14 GB", | 20260506 02:42:00.410 | INFO | "pinned" : " 29 GB" | 20260506 02:42:00.410 | INFO | } | 20260506 02:42:00.410 | INFO | Perf report for the last 5.14766 seconds: | 20260506 02:42:00.410 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:42:00.410 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:00.410 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:00.410 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:00.410 | INFO | ANALYSIS : 60.5%, 3.0, 94.6ms, 0 | 20260506 02:42:00.410 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:00.410 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:00.410 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 02:42:00.410 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:00.410 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:00.410 | INFO | Uploaded 36GiB to GPU in 1.94685s (18.4914GiB/s) | 20260506 02:42:00.410 | INFO | Downloaded 34GiB to GPU in 1.98128s (17.1606GiB/s) | 20260506 02:42:05.536 | INFO | Analyzing chunk frames = [8311808,8312320) | 20260506 02:42:05.536 | INFO | MemInfo: { | 20260506 02:42:05.536 | INFO | "gpu" : " 17 GB", | 20260506 02:42:05.536 | INFO | "huge" : "", | 20260506 02:42:05.536 | INFO | "huge_shared" : " 27 GB", | 20260506 02:42:05.536 | INFO | "malloc" : " 14 GB", | 20260506 02:42:05.536 | INFO | "pinned" : " 29 GB" | 20260506 02:42:05.536 | INFO | } | 20260506 02:42:05.536 | INFO | Perf report for the last 5.12623 seconds: | 20260506 02:42:05.536 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:42:05.536 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:05.536 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:05.536 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:05.536 | INFO | ANALYSIS : 60.5%, 3.0, 93.9ms, 0 | 20260506 02:42:05.536 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:05.537 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:05.537 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 02:42:05.537 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:05.537 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:05.537 | INFO | Uploaded 36GiB to GPU in 1.97593s (18.2193GiB/s) | 20260506 02:42:05.537 | INFO | Downloaded 34GiB to GPU in 1.97894s (17.181GiB/s) | 20260506 02:42:10.671 | INFO | Analyzing chunk frames = [8312320,8312832) | 20260506 02:42:10.671 | INFO | MemInfo: { | 20260506 02:42:10.671 | INFO | "gpu" : " 17 GB", | 20260506 02:42:10.671 | INFO | "huge" : "", | 20260506 02:42:10.671 | INFO | "huge_shared" : " 27 GB", | 20260506 02:42:10.671 | INFO | "malloc" : " 14 GB", | 20260506 02:42:10.671 | INFO | "pinned" : " 29 GB" | 20260506 02:42:10.672 | INFO | } | 20260506 02:42:10.672 | INFO | Perf report for the last 5.13517 seconds: | 20260506 02:42:10.672 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:42:10.672 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:10.672 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:10.672 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:10.672 | INFO | ANALYSIS : 61.2%, 3.0, 94.9ms, 0 | 20260506 02:42:10.672 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:10.672 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:10.672 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 02:42:10.672 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:10.672 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:10.672 | INFO | Uploaded 36GiB to GPU in 2.09788s (17.1601GiB/s) | 20260506 02:42:10.672 | INFO | Downloaded 34GiB to GPU in 2.02257s (16.8103GiB/s) | 20260506 02:42:15.817 | INFO | Analyzing chunk frames = [8312832,8313344) | 20260506 02:42:15.817 | INFO | MemInfo: { | 20260506 02:42:15.817 | INFO | "gpu" : " 17 GB", | 20260506 02:42:15.817 | INFO | "huge" : "", | 20260506 02:42:15.817 | INFO | "huge_shared" : " 27 GB", | 20260506 02:42:15.817 | INFO | "malloc" : " 14 GB", | 20260506 02:42:15.817 | INFO | "pinned" : " 29 GB" | 20260506 02:42:15.817 | INFO | } | 20260506 02:42:15.817 | INFO | Perf report for the last 5.14511 seconds: | 20260506 02:42:15.817 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:42:15.817 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:15.817 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:15.817 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:15.817 | INFO | ANALYSIS : 60.9%, 3.0, 95.2ms, 0 | 20260506 02:42:15.817 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:15.817 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:15.817 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:42:15.817 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:15.817 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:15.817 | INFO | Uploaded 36GiB to GPU in 2.08906s (17.2326GiB/s) | 20260506 02:42:15.817 | INFO | Downloaded 34GiB to GPU in 2.05277s (16.563GiB/s) | 20260506 02:42:20.961 | INFO | Analyzing chunk frames = [8313344,8313856) | 20260506 02:42:20.961 | INFO | MemInfo: { | 20260506 02:42:20.961 | INFO | "gpu" : " 17 GB", | 20260506 02:42:20.961 | INFO | "huge" : "", | 20260506 02:42:20.961 | INFO | "huge_shared" : " 27 GB", | 20260506 02:42:20.961 | INFO | "malloc" : " 14 GB", | 20260506 02:42:20.961 | INFO | "pinned" : " 29 GB" | 20260506 02:42:20.961 | INFO | } | 20260506 02:42:20.961 | INFO | Perf report for the last 5.14408 seconds: | 20260506 02:42:20.961 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:42:20.961 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:20.961 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:20.961 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:20.961 | INFO | ANALYSIS : 60.9%, 3.0, 95.2ms, 0 | 20260506 02:42:20.961 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:20.961 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:20.961 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:42:20.961 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:20.961 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:20.961 | INFO | Uploaded 36GiB to GPU in 2.09121s (17.2149GiB/s) | 20260506 02:42:20.961 | INFO | Downloaded 34GiB to GPU in 2.03604s (16.6991GiB/s) | 20260506 02:42:26.099 | INFO | Analyzing chunk frames = [8313856,8314368) | 20260506 02:42:26.099 | INFO | MemInfo: { | 20260506 02:42:26.099 | INFO | "gpu" : " 17 GB", | 20260506 02:42:26.099 | INFO | "huge" : "", | 20260506 02:42:26.099 | INFO | "huge_shared" : " 27 GB", | 20260506 02:42:26.099 | INFO | "malloc" : " 14 GB", | 20260506 02:42:26.099 | INFO | "pinned" : " 29 GB" | 20260506 02:42:26.099 | INFO | } | 20260506 02:42:26.099 | INFO | Perf report for the last 5.13829 seconds: | 20260506 02:42:26.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:42:26.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:26.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:26.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:26.099 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260506 02:42:26.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:26.099 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:26.099 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 02:42:26.099 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:26.099 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:26.099 | INFO | Uploaded 36GiB to GPU in 2.10239s (17.1233GiB/s) | 20260506 02:42:26.099 | INFO | Downloaded 34GiB to GPU in 2.08616s (16.2979GiB/s) | 20260506 02:42:31.241 | INFO | Analyzing chunk frames = [8314368,8314880) | 20260506 02:42:31.241 | INFO | MemInfo: { | 20260506 02:42:31.241 | INFO | "gpu" : " 17 GB", | 20260506 02:42:31.241 | INFO | "huge" : "", | 20260506 02:42:31.241 | INFO | "huge_shared" : " 27 GB", | 20260506 02:42:31.241 | INFO | "malloc" : " 14 GB", | 20260506 02:42:31.241 | INFO | "pinned" : " 29 GB" | 20260506 02:42:31.241 | INFO | } | 20260506 02:42:31.241 | INFO | Perf report for the last 5.14191 seconds: | 20260506 02:42:31.241 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:42:31.241 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:31.241 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:31.241 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:31.241 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260506 02:42:31.241 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:31.241 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:31.241 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 02:42:31.241 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:31.241 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:31.241 | INFO | Uploaded 36GiB to GPU in 2.03796s (17.6648GiB/s) | 20260506 02:42:31.241 | INFO | Downloaded 34GiB to GPU in 2.02199s (16.8151GiB/s) | 20260506 02:42:31.241 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 02:42:31.241 | INFO | Total mem usage: 15.4 GiB | 20260506 02:42:31.241 | INFO | All Time High Water Marks sum: 16.3 GiB (Savings: 5.2%) | 20260506 02:42:31.241 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.95%) | 20260506 02:42:31.241 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 02:42:31.241 | INFO | High Water Breakdown: | 20260506 02:42:31.241 | INFO | 60.9% : 8.55 GiB ( 88.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 02:42:31.241 | INFO | 39% : 5.47 GiB ( 82.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 02:42:31.241 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 02:42:31.241 | INFO | ------------------------------------------ | 20260506 02:42:31.242 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:42:31.242 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 02:42:31.242 | INFO | Total mem usage: 26.5 GiB | 20260506 02:42:31.242 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 02:42:31.242 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 02:42:31.242 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 02:42:31.242 | INFO | High Water Breakdown: | 20260506 02:42:31.242 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:42:31.242 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:42:31.242 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:42:31.242 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:42:31.242 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:42:31.242 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:42:31.242 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 02:42:31.242 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 02:42:31.242 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:42:31.242 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 02:42:31.242 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 02:42:31.242 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 02:42:31.242 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 02:42:31.242 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 02:42:31.242 | INFO | ------------------------------------------ | 20260506 02:42:31.242 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:42:31.242 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 02:42:31.242 | INFO | Total mem usage: 25.6 GiB | 20260506 02:42:31.242 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 02:42:31.242 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 02:42:31.242 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 02:42:31.242 | INFO | High Water Breakdown: | 20260506 02:42:31.242 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 02:42:31.242 | INFO | ------------------------------------------ | 20260506 02:42:31.242 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:42:31.242 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 02:42:31.242 | INFO | Total mem usage: 15.7 GiB | 20260506 02:42:31.242 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 02:42:31.242 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 02:42:31.242 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 02:42:31.242 | INFO | High Water Breakdown: | 20260506 02:42:31.242 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 02:42:31.242 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 02:42:31.242 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 02:42:31.242 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 02:42:31.242 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 02:42:31.242 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 02:42:31.242 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 02:42:31.242 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 02:42:31.242 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 02:42:31.242 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:42:31.242 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 02:42:31.242 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:42:31.242 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 02:42:31.242 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 02:42:31.242 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 02:42:31.242 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 02:42:31.242 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:42:31.242 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 02:42:31.242 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:42:31.242 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:42:31.242 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:42:31.242 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 02:42:31.242 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 02:42:31.242 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:42:31.242 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 02:42:31.242 | INFO | ------------------------------------------ | 20260506 02:42:31.242 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:42:31.242 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 02:42:31.242 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:42:31.242 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:42:31.242 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.39e+01, 1.34e+01 / 8.15e-01 | 20260506 02:42:31.242 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.04e-01, 1.01e-02 / 3.74e-03 | 20260506 02:42:31.242 | INFO | PerfReport: 11.86%, Baseline -- 3.71e+00 / 4.53e+00, 5.77e-01 / 2.00e-01 | 20260506 02:42:31.242 | INFO | PerfReport: 3.30%, Histogram -- 1.03e+00 / 1.52e+00, 1.94e-01 / 1.06e-01 | 20260506 02:42:31.242 | INFO | PerfReport: 15.98%, DME -- 4.00e+01 / 4.40e+01, 3.86e+00 / 8.35e-01 | 20260506 02:42:31.242 | INFO | PerfReport: 57.56%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.59e+00 / 1.33e+00 | 20260506 02:42:31.242 | INFO | PerfReport: 6.98%, PulseAccumulating -- 2.18e+00 / 2.69e+00, 4.08e-01 / 2.82e-01 | 20260506 02:42:31.242 | INFO | PerfReport: 2.99%, Metrics -- 9.37e-01 / 1.21e+00, 2.51e-01 / 1.34e-01 | 20260506 02:42:31.242 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.24e-01 / 3.69e-01, 5.40e-02 / 2.98e-02 | 20260506 02:42:31.242 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 02:42:31.242 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:42:31.242 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:42:31.242 | INFO | PerfReport: Full Execution -- 9.75e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260506 02:42:31.242 | INFO | PerfReport: 22.16%, Upload -- 2.16e+01 / 0.00e+00, 7.12e+00 / 0.00e+00 | 20260506 02:42:31.242 | INFO | PerfReport: 21.94%, Download -- 2.14e+01 / 0.00e+00, 7.48e+00 / 0.00e+00 | 20260506 02:42:31.242 | INFO | PerfReport: 32.11%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 02:42:31.242 | INFO | PerfReport: 1.97%, LockUpload -- 1.92e+00 / 0.00e+00, 7.67e+00 / 0.00e+00 | 20260506 02:42:31.242 | INFO | PerfReport: 1.28%, LockDownload -- 1.25e+00 / 0.00e+00, 2.82e+00 / 0.00e+00 | 20260506 02:42:31.242 | INFO | PerfReport: 20.54%, LockCompute -- 2.00e+01 / 0.00e+00, 1.76e+01 / 0.00e+00 | 20260506 02:42:31.242 | INFO | PerfReport: 0.01%, Misc -- 1.23e-02 / 0.00e+00, 1.30e-02 / 0.00e+00 | 20260506 02:42:31.367 | INFO | Starting a round of Baz IO | 20260506 02:42:34.498 | INFO | Populating RTMetrics report | 20260506 02:42:34.498 | INFO | RTMetrics report populated | 20260506 02:42:34.498 | INFO | Opening tmp RTMetrics json file | 20260506 02:42:34.500 | INFO | Serializing RTMetrics to json | 20260506 02:42:34.502 | INFO | Serialization done, swapping tmp file now... | 20260506 02:42:34.502 | INFO | RTMetrics done | 20260506 02:42:34.590 | INFO | Finished a round of Baz IO: Wrote 1.788 GiB in 3.223 seconds | 20260506 02:42:34.590 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.26% / 0.00% / 78.74% | 20260506 02:42:34.590 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0677864% / 0% / 8.74201e-05% | 20260506 02:42:34.590 | INFO | Baz IO overall: | 20260506 02:42:34.590 | INFO | BytesWritten : 4650236731411 | 20260506 02:42:34.590 | INFO | NumEvents : 2182566035902 | 20260506 02:42:36.373 | INFO | Analyzing chunk frames = [8314880,8315392) | 20260506 02:42:36.373 | INFO | MemInfo: { | 20260506 02:42:36.373 | INFO | "gpu" : " 17 GB", | 20260506 02:42:36.373 | INFO | "huge" : "", | 20260506 02:42:36.373 | INFO | "huge_shared" : " 27 GB", | 20260506 02:42:36.373 | INFO | "malloc" : " 14 GB", | 20260506 02:42:36.373 | INFO | "pinned" : " 29 GB" | 20260506 02:42:36.373 | INFO | } | 20260506 02:42:36.373 | INFO | Perf report for the last 5.1319 seconds: | 20260506 02:42:36.373 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:42:36.373 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:36.373 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:36.373 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:36.373 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260506 02:42:36.373 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:36.373 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260506 02:42:36.373 | INFO | PRE_HQ : 59.6%, 1.7, 53.2ms, 0 | 20260506 02:42:36.373 | INFO | BAZWRITER : 62.1%, 3.6, 113.5ms, 0 | 20260506 02:42:36.373 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:42:36.373 | INFO | Uploaded 36GiB to GPU in 2.18189s (16.4994GiB/s) | 20260506 02:42:36.373 | INFO | Downloaded 38GiB to GPU in 2.23844s (16.9761GiB/s) | 20260506 02:42:41.515 | INFO | Analyzing chunk frames = [8315392,8315904) | 20260506 02:42:41.515 | INFO | MemInfo: { | 20260506 02:42:41.515 | INFO | "gpu" : " 17 GB", | 20260506 02:42:41.515 | INFO | "huge" : "", | 20260506 02:42:41.515 | INFO | "huge_shared" : " 27 GB", | 20260506 02:42:41.515 | INFO | "malloc" : " 14 GB", | 20260506 02:42:41.515 | INFO | "pinned" : " 29 GB" | 20260506 02:42:41.515 | INFO | } | 20260506 02:42:41.515 | INFO | Perf report for the last 5.14251 seconds: | 20260506 02:42:41.515 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:42:41.515 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:41.515 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:41.515 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:41.516 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 02:42:41.516 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:41.516 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:41.516 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 02:42:41.516 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:41.516 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:41.516 | INFO | Uploaded 36GiB to GPU in 1.96149s (18.3534GiB/s) | 20260506 02:42:41.516 | INFO | Downloaded 34GiB to GPU in 1.98974s (17.0877GiB/s) | 20260506 02:42:46.652 | INFO | Analyzing chunk frames = [8315904,8316416) | 20260506 02:42:46.652 | INFO | MemInfo: { | 20260506 02:42:46.652 | INFO | "gpu" : " 17 GB", | 20260506 02:42:46.652 | INFO | "huge" : "", | 20260506 02:42:46.652 | INFO | "huge_shared" : " 27 GB", | 20260506 02:42:46.652 | INFO | "malloc" : " 14 GB", | 20260506 02:42:46.652 | INFO | "pinned" : " 29 GB" | 20260506 02:42:46.652 | INFO | } | 20260506 02:42:46.652 | INFO | Perf report for the last 5.13698 seconds: | 20260506 02:42:46.653 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:42:46.653 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:46.653 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:46.653 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:46.653 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 02:42:46.653 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:46.653 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:46.653 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:42:46.653 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:46.653 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:46.653 | INFO | Uploaded 36GiB to GPU in 2.10724s (17.084GiB/s) | 20260506 02:42:46.653 | INFO | Downloaded 34GiB to GPU in 2.04273s (16.6444GiB/s) | 20260506 02:42:51.790 | INFO | Analyzing chunk frames = [8316416,8316928) | 20260506 02:42:51.790 | INFO | MemInfo: { | 20260506 02:42:51.790 | INFO | "gpu" : " 17 GB", | 20260506 02:42:51.790 | INFO | "huge" : "", | 20260506 02:42:51.790 | INFO | "huge_shared" : " 27 GB", | 20260506 02:42:51.790 | INFO | "malloc" : " 14 GB", | 20260506 02:42:51.790 | INFO | "pinned" : " 29 GB" | 20260506 02:42:51.790 | INFO | } | 20260506 02:42:51.790 | INFO | Perf report for the last 5.13742 seconds: | 20260506 02:42:51.790 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:42:51.790 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:51.790 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:51.790 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:51.790 | INFO | ANALYSIS : 61.0%, 3.0, 94.6ms, 0 | 20260506 02:42:51.790 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:51.790 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:51.790 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:42:51.790 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:51.790 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:51.790 | INFO | Uploaded 36GiB to GPU in 2.08574s (17.2601GiB/s) | 20260506 02:42:51.790 | INFO | Downloaded 34GiB to GPU in 2.0079s (16.9331GiB/s) | 20260506 02:42:56.934 | INFO | Analyzing chunk frames = [8316928,8317440) | 20260506 02:42:56.934 | INFO | MemInfo: { | 20260506 02:42:56.934 | INFO | "gpu" : " 17 GB", | 20260506 02:42:56.934 | INFO | "huge" : "", | 20260506 02:42:56.934 | INFO | "huge_shared" : " 27 GB", | 20260506 02:42:56.934 | INFO | "malloc" : " 14 GB", | 20260506 02:42:56.934 | INFO | "pinned" : " 29 GB" | 20260506 02:42:56.934 | INFO | } | 20260506 02:42:56.934 | INFO | Perf report for the last 5.14451 seconds: | 20260506 02:42:56.934 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:42:56.934 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:56.934 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:56.934 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:56.935 | INFO | ANALYSIS : 60.6%, 3.0, 94.7ms, 0 | 20260506 02:42:56.935 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:56.935 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:56.935 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:42:56.935 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:56.935 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:42:56.935 | INFO | Uploaded 36GiB to GPU in 2.07841s (17.3209GiB/s) | 20260506 02:42:56.935 | INFO | Downloaded 34GiB to GPU in 2.02673s (16.7758GiB/s) | 20260506 02:43:02.069 | INFO | Analyzing chunk frames = [8317440,8317952) | 20260506 02:43:02.069 | INFO | MemInfo: { | 20260506 02:43:02.069 | INFO | "gpu" : " 17 GB", | 20260506 02:43:02.069 | INFO | "huge" : "", | 20260506 02:43:02.069 | INFO | "huge_shared" : " 27 GB", | 20260506 02:43:02.069 | INFO | "malloc" : " 14 GB", | 20260506 02:43:02.069 | INFO | "pinned" : " 29 GB" | 20260506 02:43:02.070 | INFO | } | 20260506 02:43:02.070 | INFO | Perf report for the last 5.13519 seconds: | 20260506 02:43:02.070 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:43:02.070 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:02.070 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:02.070 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:02.070 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 02:43:02.070 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:02.070 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:02.070 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:43:02.070 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:02.070 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:02.070 | INFO | Uploaded 36GiB to GPU in 2.07383s (17.3591GiB/s) | 20260506 02:43:02.070 | INFO | Downloaded 34GiB to GPU in 2.03713s (16.6901GiB/s) | 20260506 02:43:07.214 | INFO | Analyzing chunk frames = [8317952,8318464) | 20260506 02:43:07.214 | INFO | MemInfo: { | 20260506 02:43:07.214 | INFO | "gpu" : " 17 GB", | 20260506 02:43:07.214 | INFO | "huge" : "", | 20260506 02:43:07.214 | INFO | "huge_shared" : " 27 GB", | 20260506 02:43:07.214 | INFO | "malloc" : " 14 GB", | 20260506 02:43:07.214 | INFO | "pinned" : " 29 GB" | 20260506 02:43:07.214 | INFO | } | 20260506 02:43:07.214 | INFO | Perf report for the last 5.14429 seconds: | 20260506 02:43:07.214 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:43:07.214 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:07.214 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:07.214 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:07.214 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260506 02:43:07.214 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:07.214 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:07.214 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 02:43:07.214 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:07.214 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:07.214 | INFO | Uploaded 36GiB to GPU in 2.14167s (16.8093GiB/s) | 20260506 02:43:07.214 | INFO | Downloaded 34GiB to GPU in 2.09948s (16.1945GiB/s) | 20260506 02:43:12.358 | INFO | Analyzing chunk frames = [8318464,8318976) | 20260506 02:43:12.358 | INFO | MemInfo: { | 20260506 02:43:12.358 | INFO | "gpu" : " 17 GB", | 20260506 02:43:12.358 | INFO | "huge" : "", | 20260506 02:43:12.358 | INFO | "huge_shared" : " 27 GB", | 20260506 02:43:12.358 | INFO | "malloc" : " 14 GB", | 20260506 02:43:12.358 | INFO | "pinned" : " 29 GB" | 20260506 02:43:12.358 | INFO | } | 20260506 02:43:12.358 | INFO | Perf report for the last 5.14446 seconds: | 20260506 02:43:12.358 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:43:12.358 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:12.358 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:12.358 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:12.358 | INFO | ANALYSIS : 61.3%, 3.0, 95.8ms, 0 | 20260506 02:43:12.358 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:12.359 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:12.359 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 02:43:12.359 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:12.359 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:12.359 | INFO | Uploaded 36GiB to GPU in 2.03295s (17.7082GiB/s) | 20260506 02:43:12.359 | INFO | Downloaded 34GiB to GPU in 2.01977s (16.8336GiB/s) | 20260506 02:43:12.483 | INFO | Starting a round of Baz IO | 20260506 02:43:15.604 | INFO | Populating RTMetrics report | 20260506 02:43:15.604 | INFO | RTMetrics report populated | 20260506 02:43:15.604 | INFO | Opening tmp RTMetrics json file | 20260506 02:43:15.605 | INFO | Serializing RTMetrics to json | 20260506 02:43:15.607 | INFO | Serialization done, swapping tmp file now... | 20260506 02:43:15.608 | INFO | RTMetrics done | 20260506 02:43:15.702 | INFO | Finished a round of Baz IO: Wrote 1.794 GiB in 3.218 seconds | 20260506 02:43:15.702 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.26% / 0.00% / 78.74% | 20260506 02:43:15.702 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0662367% / 0% / 8.74201e-05% | 20260506 02:43:15.702 | INFO | Baz IO overall: | 20260506 02:43:15.702 | INFO | BytesWritten : 4652162918987 | 20260506 02:43:15.702 | INFO | NumEvents : 2183436986202 | 20260506 02:43:17.485 | INFO | Analyzing chunk frames = [8318976,8319488) | 20260506 02:43:17.485 | INFO | MemInfo: { | 20260506 02:43:17.485 | INFO | "gpu" : " 17 GB", | 20260506 02:43:17.485 | INFO | "huge" : "", | 20260506 02:43:17.485 | INFO | "huge_shared" : " 27 GB", | 20260506 02:43:17.485 | INFO | "malloc" : " 14 GB", | 20260506 02:43:17.485 | INFO | "pinned" : " 29 GB" | 20260506 02:43:17.485 | INFO | } | 20260506 02:43:17.485 | INFO | Perf report for the last 5.12675 seconds: | 20260506 02:43:17.485 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:43:17.485 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:17.485 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:17.485 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:17.485 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 02:43:17.485 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:17.485 | INFO | RT_METRICS : 42.9%, 1.0, 22.4ms, 0 | 20260506 02:43:17.485 | INFO | PRE_HQ : 59.2%, 1.7, 53.6ms, 0 | 20260506 02:43:17.485 | INFO | BAZWRITER : 62.0%, 3.9, 124.5ms, 0 | 20260506 02:43:17.485 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:43:17.486 | INFO | Uploaded 36GiB to GPU in 2.10205s (17.1261GiB/s) | 20260506 02:43:17.486 | INFO | Downloaded 38GiB to GPU in 2.22468s (17.0811GiB/s) | 20260506 02:43:22.625 | INFO | Analyzing chunk frames = [8319488,8320000) | 20260506 02:43:22.625 | INFO | MemInfo: { | 20260506 02:43:22.625 | INFO | "gpu" : " 17 GB", | 20260506 02:43:22.625 | INFO | "huge" : "", | 20260506 02:43:22.625 | INFO | "huge_shared" : " 27 GB", | 20260506 02:43:22.625 | INFO | "malloc" : " 14 GB", | 20260506 02:43:22.625 | INFO | "pinned" : " 29 GB" | 20260506 02:43:22.625 | INFO | } | 20260506 02:43:22.625 | INFO | Perf report for the last 5.14009 seconds: | 20260506 02:43:22.625 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:43:22.625 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:22.625 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:22.625 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:22.625 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260506 02:43:22.625 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:22.625 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:22.625 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 02:43:22.625 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:22.625 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:22.626 | INFO | Uploaded 36GiB to GPU in 2.12428s (16.9469GiB/s) | 20260506 02:43:22.626 | INFO | Downloaded 34GiB to GPU in 2.04331s (16.6397GiB/s) | 20260506 02:43:27.766 | INFO | Analyzing chunk frames = [8320000,8320512) | 20260506 02:43:27.766 | INFO | MemInfo: { | 20260506 02:43:27.766 | INFO | "gpu" : " 17 GB", | 20260506 02:43:27.766 | INFO | "huge" : "", | 20260506 02:43:27.766 | INFO | "huge_shared" : " 27 GB", | 20260506 02:43:27.766 | INFO | "malloc" : " 14 GB", | 20260506 02:43:27.766 | INFO | "pinned" : " 29 GB" | 20260506 02:43:27.766 | INFO | } | 20260506 02:43:27.766 | INFO | Perf report for the last 5.1408 seconds: | 20260506 02:43:27.766 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:43:27.766 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:27.766 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:27.766 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:27.766 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 02:43:27.766 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:27.766 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:27.766 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 02:43:27.766 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:27.766 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:27.766 | INFO | Uploaded 36GiB to GPU in 2.08899s (17.2332GiB/s) | 20260506 02:43:27.766 | INFO | Downloaded 34GiB to GPU in 2.0245s (16.7942GiB/s) | 20260506 02:43:32.913 | INFO | Analyzing chunk frames = [8320512,8321024) | 20260506 02:43:32.913 | INFO | MemInfo: { | 20260506 02:43:32.913 | INFO | "gpu" : " 17 GB", | 20260506 02:43:32.913 | INFO | "huge" : "", | 20260506 02:43:32.913 | INFO | "huge_shared" : " 27 GB", | 20260506 02:43:32.913 | INFO | "malloc" : " 14 GB", | 20260506 02:43:32.913 | INFO | "pinned" : " 29 GB" | 20260506 02:43:32.913 | INFO | } | 20260506 02:43:32.913 | INFO | Perf report for the last 5.14722 seconds: | 20260506 02:43:32.913 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:43:32.913 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:32.913 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:32.913 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:32.913 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 02:43:32.913 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:32.913 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:32.913 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 02:43:32.913 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:32.913 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:32.914 | INFO | Uploaded 36GiB to GPU in 2.10375s (17.1123GiB/s) | 20260506 02:43:32.914 | INFO | Downloaded 34GiB to GPU in 2.02539s (16.7869GiB/s) | 20260506 02:43:38.046 | INFO | Analyzing chunk frames = [8321024,8321536) | 20260506 02:43:38.046 | INFO | MemInfo: { | 20260506 02:43:38.046 | INFO | "gpu" : " 17 GB", | 20260506 02:43:38.046 | INFO | "huge" : "", | 20260506 02:43:38.046 | INFO | "huge_shared" : " 27 GB", | 20260506 02:43:38.046 | INFO | "malloc" : " 14 GB", | 20260506 02:43:38.046 | INFO | "pinned" : " 29 GB" | 20260506 02:43:38.046 | INFO | } | 20260506 02:43:38.046 | INFO | Perf report for the last 5.13271 seconds: | 20260506 02:43:38.046 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:43:38.046 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:38.046 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:38.046 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:38.046 | INFO | ANALYSIS : 60.8%, 3.0, 94.9ms, 0 | 20260506 02:43:38.046 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:38.046 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:38.046 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 02:43:38.046 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:38.046 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:38.046 | INFO | Uploaded 36GiB to GPU in 2.11603s (17.013GiB/s) | 20260506 02:43:38.046 | INFO | Downloaded 34GiB to GPU in 2.0397s (16.6691GiB/s) | 20260506 02:43:43.188 | INFO | Analyzing chunk frames = [8321536,8322048) | 20260506 02:43:43.188 | INFO | MemInfo: { | 20260506 02:43:43.188 | INFO | "gpu" : " 17 GB", | 20260506 02:43:43.188 | INFO | "huge" : "", | 20260506 02:43:43.188 | INFO | "huge_shared" : " 27 GB", | 20260506 02:43:43.188 | INFO | "malloc" : " 14 GB", | 20260506 02:43:43.188 | INFO | "pinned" : " 29 GB" | 20260506 02:43:43.188 | INFO | } | 20260506 02:43:43.188 | INFO | Perf report for the last 5.14182 seconds: | 20260506 02:43:43.188 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:43:43.188 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:43.188 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:43.188 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:43.188 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 02:43:43.188 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:43.188 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:43.188 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:43:43.188 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:43.188 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:43.188 | INFO | Uploaded 36GiB to GPU in 2.07113s (17.3818GiB/s) | 20260506 02:43:43.188 | INFO | Downloaded 34GiB to GPU in 2.03525s (16.7056GiB/s) | 20260506 02:43:48.324 | INFO | Analyzing chunk frames = [8322048,8322560) | 20260506 02:43:48.324 | INFO | MemInfo: { | 20260506 02:43:48.324 | INFO | "gpu" : " 17 GB", | 20260506 02:43:48.324 | INFO | "huge" : "", | 20260506 02:43:48.324 | INFO | "huge_shared" : " 27 GB", | 20260506 02:43:48.324 | INFO | "malloc" : " 14 GB", | 20260506 02:43:48.324 | INFO | "pinned" : " 29 GB" | 20260506 02:43:48.324 | INFO | } | 20260506 02:43:48.324 | INFO | Perf report for the last 5.13608 seconds: | 20260506 02:43:48.324 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:43:48.324 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:48.324 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:48.324 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:48.324 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260506 02:43:48.324 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:48.324 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:48.324 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 02:43:48.324 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:48.324 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:48.324 | INFO | Uploaded 36GiB to GPU in 2.09408s (17.1913GiB/s) | 20260506 02:43:48.324 | INFO | Downloaded 34GiB to GPU in 2.08082s (16.3397GiB/s) | 20260506 02:43:53.462 | INFO | Analyzing chunk frames = [8322560,8323072) | 20260506 02:43:53.462 | INFO | MemInfo: { | 20260506 02:43:53.462 | INFO | "gpu" : " 17 GB", | 20260506 02:43:53.462 | INFO | "huge" : "", | 20260506 02:43:53.462 | INFO | "huge_shared" : " 27 GB", | 20260506 02:43:53.462 | INFO | "malloc" : " 14 GB", | 20260506 02:43:53.462 | INFO | "pinned" : " 29 GB" | 20260506 02:43:53.462 | INFO | } | 20260506 02:43:53.463 | INFO | Perf report for the last 5.13855 seconds: | 20260506 02:43:53.463 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:43:53.463 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:53.463 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:53.463 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:53.463 | INFO | ANALYSIS : 61.4%, 3.0, 95.9ms, 0 | 20260506 02:43:53.463 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:53.463 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:53.463 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:43:53.463 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:53.463 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:53.463 | INFO | Uploaded 36GiB to GPU in 2.05865s (17.4872GiB/s) | 20260506 02:43:53.463 | INFO | Downloaded 34GiB to GPU in 2.02611s (16.7809GiB/s) | 20260506 02:43:53.463 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 02:43:53.463 | INFO | Total mem usage: 15.4 GiB | 20260506 02:43:53.463 | INFO | All Time High Water Marks sum: 16.3 GiB (Savings: 5.2%) | 20260506 02:43:53.463 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -7.38%) | 20260506 02:43:53.463 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 02:43:53.463 | INFO | High Water Breakdown: | 20260506 02:43:53.463 | INFO | 61.3% : 8.81 GiB ( 91.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 02:43:53.463 | INFO | 38.6% : 5.55 GiB ( 83.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 02:43:53.463 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 02:43:53.463 | INFO | ------------------------------------------ | 20260506 02:43:53.463 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:43:53.463 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 02:43:53.463 | INFO | Total mem usage: 26.5 GiB | 20260506 02:43:53.463 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 02:43:53.463 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 02:43:53.463 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 02:43:53.463 | INFO | High Water Breakdown: | 20260506 02:43:53.463 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:43:53.463 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:43:53.463 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:43:53.463 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:43:53.463 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:43:53.463 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:43:53.463 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 02:43:53.463 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 02:43:53.463 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:43:53.463 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 02:43:53.463 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 02:43:53.463 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 02:43:53.463 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 02:43:53.463 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 02:43:53.463 | INFO | ------------------------------------------ | 20260506 02:43:53.463 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:43:53.463 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 02:43:53.463 | INFO | Total mem usage: 25.6 GiB | 20260506 02:43:53.464 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 02:43:53.464 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 02:43:53.464 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 02:43:53.464 | INFO | High Water Breakdown: | 20260506 02:43:53.464 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 02:43:53.464 | INFO | ------------------------------------------ | 20260506 02:43:53.464 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:43:53.464 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 02:43:53.464 | INFO | Total mem usage: 15.7 GiB | 20260506 02:43:53.464 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 02:43:53.464 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 02:43:53.464 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 02:43:53.464 | INFO | High Water Breakdown: | 20260506 02:43:53.464 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 02:43:53.464 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 02:43:53.464 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 02:43:53.464 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 02:43:53.464 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 02:43:53.464 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 02:43:53.464 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 02:43:53.464 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 02:43:53.464 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 02:43:53.464 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:43:53.464 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 02:43:53.464 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:43:53.464 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 02:43:53.464 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 02:43:53.464 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 02:43:53.464 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 02:43:53.464 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:43:53.464 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 02:43:53.464 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:43:53.464 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:43:53.464 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:43:53.464 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 02:43:53.464 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 02:43:53.464 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:43:53.464 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 02:43:53.464 | INFO | ------------------------------------------ | 20260506 02:43:53.464 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:43:53.464 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 02:43:53.464 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:43:53.464 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:43:53.464 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.39e+01, 1.35e+01 / 6.83e-01 | 20260506 02:43:53.464 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.06e-01, 1.05e-02 / 5.51e-03 | 20260506 02:43:53.464 | INFO | PerfReport: 11.83%, Baseline -- 3.70e+00 / 4.46e+00, 5.69e-01 / 2.40e-01 | 20260506 02:43:53.464 | INFO | PerfReport: 3.29%, Histogram -- 1.03e+00 / 1.52e+00, 1.93e-01 / 9.09e-02 | 20260506 02:43:53.464 | INFO | PerfReport: 16.00%, DME -- 4.01e+01 / 4.39e+01, 3.71e+00 / 8.66e-01 | 20260506 02:43:53.464 | INFO | PerfReport: 57.59%, FrameLabeling -- 1.80e+01 / 1.97e+01, 1.58e+00 / 1.14e+00 | 20260506 02:43:53.464 | INFO | PerfReport: 6.97%, PulseAccumulating -- 2.18e+00 / 2.69e+00, 4.08e-01 / 2.42e-01 | 20260506 02:43:53.464 | INFO | PerfReport: 2.99%, Metrics -- 9.35e-01 / 1.20e+00, 2.52e-01 / 1.22e-01 | 20260506 02:43:53.464 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.23e-01 / 3.75e-01, 5.47e-02 / 2.96e-02 | 20260506 02:43:53.464 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 02:43:53.464 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:43:53.464 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:43:53.464 | INFO | PerfReport: Full Execution -- 9.78e+01 / -nan, 2.41e+01 / 0.00e+00 | 20260506 02:43:53.464 | INFO | PerfReport: 22.26%, Upload -- 2.18e+01 / 0.00e+00, 7.23e+00 / 0.00e+00 | 20260506 02:43:53.464 | INFO | PerfReport: 21.95%, Download -- 2.15e+01 / 0.00e+00, 7.54e+00 / 0.00e+00 | 20260506 02:43:53.464 | INFO | PerfReport: 32.03%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 02:43:53.464 | INFO | PerfReport: 2.01%, LockUpload -- 1.96e+00 / 0.00e+00, 7.78e+00 / 0.00e+00 | 20260506 02:43:53.464 | INFO | PerfReport: 1.28%, LockDownload -- 1.25e+00 / 0.00e+00, 2.85e+00 / 0.00e+00 | 20260506 02:43:53.464 | INFO | PerfReport: 20.46%, LockCompute -- 2.00e+01 / 0.00e+00, 1.77e+01 / 0.00e+00 | 20260506 02:43:53.464 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.58e-02 / 0.00e+00 | 20260506 02:43:53.589 | INFO | Starting a round of Baz IO | 20260506 02:43:56.709 | INFO | Populating RTMetrics report | 20260506 02:43:56.709 | INFO | RTMetrics report populated | 20260506 02:43:56.709 | INFO | Opening tmp RTMetrics json file | 20260506 02:43:56.711 | INFO | Serializing RTMetrics to json | 20260506 02:43:56.713 | INFO | Serialization done, swapping tmp file now... | 20260506 02:43:56.713 | INFO | RTMetrics done | 20260506 02:43:56.797 | INFO | Finished a round of Baz IO: Wrote 1.792 GiB in 3.208 seconds | 20260506 02:43:56.797 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.26% / 0.00% / 78.74% | 20260506 02:43:56.797 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0668208% / 0% / 7.15256e-05% | 20260506 02:43:56.797 | INFO | Baz IO overall: | 20260506 02:43:56.797 | INFO | BytesWritten : 4654086871351 | 20260506 02:43:56.797 | INFO | NumEvents : 2184307802264 | 20260506 02:43:58.607 | INFO | Analyzing chunk frames = [8323072,8323584) | 20260506 02:43:58.607 | INFO | MemInfo: { | 20260506 02:43:58.607 | INFO | "gpu" : " 17 GB", | 20260506 02:43:58.607 | INFO | "huge" : "", | 20260506 02:43:58.607 | INFO | "huge_shared" : " 27 GB", | 20260506 02:43:58.607 | INFO | "malloc" : " 14 GB", | 20260506 02:43:58.607 | INFO | "pinned" : " 29 GB" | 20260506 02:43:58.607 | INFO | } | 20260506 02:43:58.608 | INFO | Perf report for the last 5.14497 seconds: | 20260506 02:43:58.608 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:43:58.608 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:58.608 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:58.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:58.608 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260506 02:43:58.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:43:58.608 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260506 02:43:58.608 | INFO | PRE_HQ : 59.0%, 1.7, 53.4ms, 0 | 20260506 02:43:58.608 | INFO | BAZWRITER : 61.5%, 3.6, 114.5ms, 0 | 20260506 02:43:58.608 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:43:58.608 | INFO | Uploaded 36GiB to GPU in 2.09831s (17.1566GiB/s) | 20260506 02:43:58.608 | INFO | Downloaded 38GiB to GPU in 2.22188s (17.1027GiB/s) | 20260506 02:44:03.743 | INFO | Analyzing chunk frames = [8323584,8324096) | 20260506 02:44:03.743 | INFO | MemInfo: { | 20260506 02:44:03.743 | INFO | "gpu" : " 17 GB", | 20260506 02:44:03.743 | INFO | "huge" : "", | 20260506 02:44:03.743 | INFO | "huge_shared" : " 27 GB", | 20260506 02:44:03.743 | INFO | "malloc" : " 14 GB", | 20260506 02:44:03.743 | INFO | "pinned" : " 29 GB" | 20260506 02:44:03.743 | INFO | } | 20260506 02:44:03.743 | INFO | Perf report for the last 5.1355 seconds: | 20260506 02:44:03.743 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:44:03.743 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:03.743 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:03.743 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:03.743 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 02:44:03.743 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:03.743 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:03.743 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:44:03.743 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:03.743 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:03.743 | INFO | Uploaded 36GiB to GPU in 1.95461s (18.418GiB/s) | 20260506 02:44:03.743 | INFO | Downloaded 34GiB to GPU in 1.98187s (17.1556GiB/s) | 20260506 02:44:08.885 | INFO | Analyzing chunk frames = [8324096,8324608) | 20260506 02:44:08.885 | INFO | MemInfo: { | 20260506 02:44:08.885 | INFO | "gpu" : " 17 GB", | 20260506 02:44:08.885 | INFO | "huge" : "", | 20260506 02:44:08.885 | INFO | "huge_shared" : " 27 GB", | 20260506 02:44:08.885 | INFO | "malloc" : " 14 GB", | 20260506 02:44:08.885 | INFO | "pinned" : " 29 GB" | 20260506 02:44:08.885 | INFO | } | 20260506 02:44:08.885 | INFO | Perf report for the last 5.142 seconds: | 20260506 02:44:08.885 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:44:08.885 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:08.885 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:08.885 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:08.885 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 02:44:08.885 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:08.885 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:08.885 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 02:44:08.885 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:08.885 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:08.885 | INFO | Uploaded 36GiB to GPU in 2.09243s (17.2049GiB/s) | 20260506 02:44:08.885 | INFO | Downloaded 34GiB to GPU in 2.0363s (16.6969GiB/s) | 20260506 02:44:14.026 | INFO | Analyzing chunk frames = [8324608,8325120) | 20260506 02:44:14.026 | INFO | MemInfo: { | 20260506 02:44:14.026 | INFO | "gpu" : " 17 GB", | 20260506 02:44:14.026 | INFO | "huge" : "", | 20260506 02:44:14.026 | INFO | "huge_shared" : " 27 GB", | 20260506 02:44:14.026 | INFO | "malloc" : " 14 GB", | 20260506 02:44:14.026 | INFO | "pinned" : " 29 GB" | 20260506 02:44:14.027 | INFO | } | 20260506 02:44:14.027 | INFO | Perf report for the last 5.1424 seconds: | 20260506 02:44:14.027 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:44:14.027 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:14.027 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:14.027 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:14.027 | INFO | ANALYSIS : 61.0%, 3.0, 94.7ms, 0 | 20260506 02:44:14.027 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:14.027 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:14.028 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 02:44:14.028 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:14.028 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:14.028 | INFO | Uploaded 36GiB to GPU in 2.07496s (17.3497GiB/s) | 20260506 02:44:14.028 | INFO | Downloaded 34GiB to GPU in 2.01405s (16.8814GiB/s) | 20260506 02:44:19.163 | INFO | Analyzing chunk frames = [8325120,8325632) | 20260506 02:44:19.163 | INFO | MemInfo: { | 20260506 02:44:19.163 | INFO | "gpu" : " 17 GB", | 20260506 02:44:19.163 | INFO | "huge" : "", | 20260506 02:44:19.163 | INFO | "huge_shared" : " 27 GB", | 20260506 02:44:19.163 | INFO | "malloc" : " 14 GB", | 20260506 02:44:19.163 | INFO | "pinned" : " 29 GB" | 20260506 02:44:19.163 | INFO | } | 20260506 02:44:19.163 | INFO | Perf report for the last 5.13591 seconds: | 20260506 02:44:19.163 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:44:19.163 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:19.163 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:19.163 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:19.163 | INFO | ANALYSIS : 60.8%, 3.0, 94.9ms, 0 | 20260506 02:44:19.163 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:19.163 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:19.164 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 02:44:19.164 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:19.164 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:19.164 | INFO | Uploaded 36GiB to GPU in 2.12233s (16.9625GiB/s) | 20260506 02:44:19.164 | INFO | Downloaded 34GiB to GPU in 2.03992s (16.6673GiB/s) | 20260506 02:44:24.303 | INFO | Analyzing chunk frames = [8325632,8326144) | 20260506 02:44:24.303 | INFO | MemInfo: { | 20260506 02:44:24.303 | INFO | "gpu" : " 17 GB", | 20260506 02:44:24.303 | INFO | "huge" : "", | 20260506 02:44:24.303 | INFO | "huge_shared" : " 27 GB", | 20260506 02:44:24.303 | INFO | "malloc" : " 14 GB", | 20260506 02:44:24.303 | INFO | "pinned" : " 29 GB" | 20260506 02:44:24.304 | INFO | } | 20260506 02:44:24.304 | INFO | Perf report for the last 5.1403 seconds: | 20260506 02:44:24.304 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:44:24.304 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:24.304 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:24.304 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:24.304 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 02:44:24.304 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:24.304 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:24.304 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 02:44:24.304 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:24.304 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:24.304 | INFO | Uploaded 36GiB to GPU in 2.11835s (16.9944GiB/s) | 20260506 02:44:24.304 | INFO | Downloaded 34GiB to GPU in 2.04198s (16.6505GiB/s) | 20260506 02:44:29.440 | INFO | Analyzing chunk frames = [8326144,8326656) | 20260506 02:44:29.440 | INFO | MemInfo: { | 20260506 02:44:29.440 | INFO | "gpu" : " 17 GB", | 20260506 02:44:29.440 | INFO | "huge" : "", | 20260506 02:44:29.440 | INFO | "huge_shared" : " 27 GB", | 20260506 02:44:29.440 | INFO | "malloc" : " 14 GB", | 20260506 02:44:29.440 | INFO | "pinned" : " 29 GB" | 20260506 02:44:29.440 | INFO | } | 20260506 02:44:29.440 | INFO | Perf report for the last 5.13609 seconds: | 20260506 02:44:29.440 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:44:29.440 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:29.440 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:29.440 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:29.440 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 02:44:29.440 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:29.440 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:29.440 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 02:44:29.440 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:29.440 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:29.440 | INFO | Uploaded 36GiB to GPU in 2.05005s (17.5606GiB/s) | 20260506 02:44:29.440 | INFO | Downloaded 34GiB to GPU in 2.07199s (16.4094GiB/s) | 20260506 02:44:34.579 | INFO | Analyzing chunk frames = [8326656,8327168) | 20260506 02:44:34.579 | INFO | MemInfo: { | 20260506 02:44:34.579 | INFO | "gpu" : " 17 GB", | 20260506 02:44:34.579 | INFO | "huge" : "", | 20260506 02:44:34.579 | INFO | "huge_shared" : " 27 GB", | 20260506 02:44:34.579 | INFO | "malloc" : " 14 GB", | 20260506 02:44:34.579 | INFO | "pinned" : " 29 GB" | 20260506 02:44:34.579 | INFO | } | 20260506 02:44:34.579 | INFO | Perf report for the last 5.13936 seconds: | 20260506 02:44:34.579 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:44:34.579 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:34.579 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:34.579 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:34.579 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260506 02:44:34.579 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:34.579 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:34.579 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:44:34.579 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:34.579 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:34.579 | INFO | Uploaded 36GiB to GPU in 2.09395s (17.1924GiB/s) | 20260506 02:44:34.579 | INFO | Downloaded 34GiB to GPU in 2.03738s (16.6881GiB/s) | 20260506 02:44:34.705 | INFO | Starting a round of Baz IO | 20260506 02:44:37.839 | INFO | Populating RTMetrics report | 20260506 02:44:37.839 | INFO | RTMetrics report populated | 20260506 02:44:37.840 | INFO | Opening tmp RTMetrics json file | 20260506 02:44:37.842 | INFO | Serializing RTMetrics to json | 20260506 02:44:37.843 | INFO | Serialization done, swapping tmp file now... | 20260506 02:44:37.843 | INFO | RTMetrics done | 20260506 02:44:37.915 | INFO | Finished a round of Baz IO: Wrote 1.784 GiB in 3.209 seconds | 20260506 02:44:37.915 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.25% / 0.00% / 78.74% | 20260506 02:44:37.915 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0659188% / 0% / 8.34465e-05% | 20260506 02:44:37.915 | INFO | Baz IO overall: | 20260506 02:44:37.915 | INFO | BytesWritten : 4656002357691 | 20260506 02:44:37.915 | INFO | NumEvents : 2185175454530 | 20260506 02:44:39.718 | INFO | Analyzing chunk frames = [8327168,8327680) | 20260506 02:44:39.718 | INFO | MemInfo: { | 20260506 02:44:39.718 | INFO | "gpu" : " 17 GB", | 20260506 02:44:39.718 | INFO | "huge" : "", | 20260506 02:44:39.718 | INFO | "huge_shared" : " 27 GB", | 20260506 02:44:39.718 | INFO | "malloc" : " 14 GB", | 20260506 02:44:39.718 | INFO | "pinned" : " 29 GB" | 20260506 02:44:39.718 | INFO | } | 20260506 02:44:39.719 | INFO | Perf report for the last 5.13936 seconds: | 20260506 02:44:39.719 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:44:39.719 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:39.719 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:39.719 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:39.719 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260506 02:44:39.719 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:39.719 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260506 02:44:39.719 | INFO | PRE_HQ : 59.5%, 1.7, 53.4ms, 0 | 20260506 02:44:39.719 | INFO | BAZWRITER : 61.7%, 3.5, 112.0ms, 0 | 20260506 02:44:39.719 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:44:39.719 | INFO | Uploaded 36GiB to GPU in 2.18196s (16.4989GiB/s) | 20260506 02:44:39.719 | INFO | Downloaded 38GiB to GPU in 2.24173s (16.9512GiB/s) | 20260506 02:44:44.865 | INFO | Analyzing chunk frames = [8327680,8328192) | 20260506 02:44:44.865 | INFO | MemInfo: { | 20260506 02:44:44.865 | INFO | "gpu" : " 17 GB", | 20260506 02:44:44.865 | INFO | "huge" : "", | 20260506 02:44:44.865 | INFO | "huge_shared" : " 27 GB", | 20260506 02:44:44.865 | INFO | "malloc" : " 14 GB", | 20260506 02:44:44.865 | INFO | "pinned" : " 29 GB" | 20260506 02:44:44.865 | INFO | } | 20260506 02:44:44.865 | INFO | Perf report for the last 5.14659 seconds: | 20260506 02:44:44.865 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:44:44.865 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:44.865 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:44.865 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:44.865 | INFO | ANALYSIS : 60.6%, 3.0, 94.8ms, 0 | 20260506 02:44:44.865 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:44.865 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:44.865 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 02:44:44.865 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:44.865 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:44.865 | INFO | Uploaded 36GiB to GPU in 1.93524s (18.6024GiB/s) | 20260506 02:44:44.865 | INFO | Downloaded 34GiB to GPU in 1.97877s (17.1824GiB/s) | 20260506 02:44:49.998 | INFO | Analyzing chunk frames = [8328192,8328704) | 20260506 02:44:49.998 | INFO | MemInfo: { | 20260506 02:44:49.998 | INFO | "gpu" : " 17 GB", | 20260506 02:44:49.998 | INFO | "huge" : "", | 20260506 02:44:49.998 | INFO | "huge_shared" : " 27 GB", | 20260506 02:44:49.998 | INFO | "malloc" : " 14 GB", | 20260506 02:44:49.998 | INFO | "pinned" : " 29 GB" | 20260506 02:44:49.999 | INFO | } | 20260506 02:44:49.999 | INFO | Perf report for the last 5.13341 seconds: | 20260506 02:44:49.999 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:44:49.999 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:49.999 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:49.999 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:49.999 | INFO | ANALYSIS : 61.2%, 3.0, 95.1ms, 0 | 20260506 02:44:49.999 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:49.999 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:49.999 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 02:44:49.999 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:49.999 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:49.999 | INFO | Uploaded 36GiB to GPU in 2.10651s (17.0899GiB/s) | 20260506 02:44:49.999 | INFO | Downloaded 34GiB to GPU in 2.04058s (16.6619GiB/s) | 20260506 02:44:55.136 | INFO | Analyzing chunk frames = [8328704,8329216) | 20260506 02:44:55.136 | INFO | MemInfo: { | 20260506 02:44:55.136 | INFO | "gpu" : " 17 GB", | 20260506 02:44:55.136 | INFO | "huge" : "", | 20260506 02:44:55.136 | INFO | "huge_shared" : " 27 GB", | 20260506 02:44:55.136 | INFO | "malloc" : " 14 GB", | 20260506 02:44:55.136 | INFO | "pinned" : " 29 GB" | 20260506 02:44:55.136 | INFO | } | 20260506 02:44:55.136 | INFO | Perf report for the last 5.13782 seconds: | 20260506 02:44:55.137 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:44:55.137 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:55.137 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:55.137 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:55.137 | INFO | ANALYSIS : 61.3%, 3.0, 95.2ms, 0 | 20260506 02:44:55.137 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:55.137 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:55.137 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 02:44:55.137 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:55.137 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:44:55.137 | INFO | Uploaded 36GiB to GPU in 2.09895s (17.1514GiB/s) | 20260506 02:44:55.137 | INFO | Downloaded 34GiB to GPU in 2.03298s (16.7242GiB/s) | 20260506 02:45:00.276 | INFO | Analyzing chunk frames = [8329216,8329728) | 20260506 02:45:00.277 | INFO | MemInfo: { | 20260506 02:45:00.277 | INFO | "gpu" : " 17 GB", | 20260506 02:45:00.277 | INFO | "huge" : "", | 20260506 02:45:00.277 | INFO | "huge_shared" : " 27 GB", | 20260506 02:45:00.277 | INFO | "malloc" : " 14 GB", | 20260506 02:45:00.277 | INFO | "pinned" : " 29 GB" | 20260506 02:45:00.277 | INFO | } | 20260506 02:45:00.277 | INFO | Perf report for the last 5.14032 seconds: | 20260506 02:45:00.277 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:45:00.277 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:00.277 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:00.277 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:00.277 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260506 02:45:00.277 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:00.277 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:00.277 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 02:45:00.277 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:00.277 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:00.277 | INFO | Uploaded 36GiB to GPU in 2.07699s (17.3328GiB/s) | 20260506 02:45:00.277 | INFO | Downloaded 34GiB to GPU in 2.03474s (16.7097GiB/s) | 20260506 02:45:05.421 | INFO | Analyzing chunk frames = [8329728,8330240) | 20260506 02:45:05.421 | INFO | MemInfo: { | 20260506 02:45:05.421 | INFO | "gpu" : " 17 GB", | 20260506 02:45:05.421 | INFO | "huge" : "", | 20260506 02:45:05.421 | INFO | "huge_shared" : " 27 GB", | 20260506 02:45:05.421 | INFO | "malloc" : " 14 GB", | 20260506 02:45:05.421 | INFO | "pinned" : " 29 GB" | 20260506 02:45:05.422 | INFO | } | 20260506 02:45:05.422 | INFO | Perf report for the last 5.14493 seconds: | 20260506 02:45:05.422 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:45:05.422 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:05.422 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:05.422 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:05.422 | INFO | ANALYSIS : 61.0%, 3.0, 95.4ms, 0 | 20260506 02:45:05.422 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:05.422 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:05.422 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 02:45:05.422 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:05.422 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:05.422 | INFO | Uploaded 36GiB to GPU in 2.11369s (17.0318GiB/s) | 20260506 02:45:05.422 | INFO | Downloaded 34GiB to GPU in 2.05138s (16.5742GiB/s) | 20260506 02:45:10.562 | INFO | Analyzing chunk frames = [8330240,8330752) | 20260506 02:45:10.562 | INFO | MemInfo: { | 20260506 02:45:10.562 | INFO | "gpu" : " 17 GB", | 20260506 02:45:10.562 | INFO | "huge" : "", | 20260506 02:45:10.562 | INFO | "huge_shared" : " 27 GB", | 20260506 02:45:10.562 | INFO | "malloc" : " 14 GB", | 20260506 02:45:10.562 | INFO | "pinned" : " 29 GB" | 20260506 02:45:10.562 | INFO | } | 20260506 02:45:10.562 | INFO | Perf report for the last 5.14014 seconds: | 20260506 02:45:10.562 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:45:10.562 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:10.562 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:10.562 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:10.562 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260506 02:45:10.562 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:10.562 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:10.562 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 02:45:10.562 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:10.562 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:10.562 | INFO | Uploaded 36GiB to GPU in 2.08251s (17.2869GiB/s) | 20260506 02:45:10.562 | INFO | Downloaded 34GiB to GPU in 2.06917s (16.4317GiB/s) | 20260506 02:45:15.691 | INFO | Analyzing chunk frames = [8330752,8331264) | 20260506 02:45:15.691 | INFO | MemInfo: { | 20260506 02:45:15.691 | INFO | "gpu" : " 17 GB", | 20260506 02:45:15.691 | INFO | "huge" : "", | 20260506 02:45:15.691 | INFO | "huge_shared" : " 27 GB", | 20260506 02:45:15.691 | INFO | "malloc" : " 14 GB", | 20260506 02:45:15.692 | INFO | "pinned" : " 29 GB" | 20260506 02:45:15.692 | INFO | } | 20260506 02:45:15.692 | INFO | Perf report for the last 5.1299 seconds: | 20260506 02:45:15.692 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:45:15.692 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:15.692 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:15.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:15.692 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 02:45:15.692 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:15.692 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:15.692 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 02:45:15.692 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:15.692 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:15.692 | INFO | Uploaded 36GiB to GPU in 2.06583s (17.4264GiB/s) | 20260506 02:45:15.692 | INFO | Downloaded 34GiB to GPU in 2.03893s (16.6754GiB/s) | 20260506 02:45:15.692 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 02:45:15.692 | INFO | Total mem usage: 15.4 GiB | 20260506 02:45:15.692 | INFO | All Time High Water Marks sum: 16.3 GiB (Savings: 5.2%) | 20260506 02:45:15.692 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.97%) | 20260506 02:45:15.692 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 02:45:15.692 | INFO | High Water Breakdown: | 20260506 02:45:15.692 | INFO | 60.9% : 8.54 GiB ( 88.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 02:45:15.692 | INFO | 39% : 5.47 GiB ( 82.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 02:45:15.692 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 02:45:15.692 | INFO | ------------------------------------------ | 20260506 02:45:15.692 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:45:15.692 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 02:45:15.692 | INFO | Total mem usage: 26.5 GiB | 20260506 02:45:15.692 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 02:45:15.692 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 02:45:15.692 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 02:45:15.692 | INFO | High Water Breakdown: | 20260506 02:45:15.692 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:45:15.692 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:45:15.692 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:45:15.692 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:45:15.692 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:45:15.692 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:45:15.692 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 02:45:15.692 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 02:45:15.693 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:45:15.693 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 02:45:15.693 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 02:45:15.693 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 02:45:15.693 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 02:45:15.693 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 02:45:15.693 | INFO | ------------------------------------------ | 20260506 02:45:15.693 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:45:15.693 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 02:45:15.693 | INFO | Total mem usage: 25.6 GiB | 20260506 02:45:15.693 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 02:45:15.693 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 02:45:15.693 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 02:45:15.693 | INFO | High Water Breakdown: | 20260506 02:45:15.693 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 02:45:15.693 | INFO | ------------------------------------------ | 20260506 02:45:15.693 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:45:15.693 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 02:45:15.693 | INFO | Total mem usage: 15.7 GiB | 20260506 02:45:15.693 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 02:45:15.693 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 02:45:15.693 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 02:45:15.693 | INFO | High Water Breakdown: | 20260506 02:45:15.693 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 02:45:15.693 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 02:45:15.693 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 02:45:15.693 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 02:45:15.693 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 02:45:15.693 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 02:45:15.693 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 02:45:15.693 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 02:45:15.693 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 02:45:15.693 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:45:15.693 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 02:45:15.693 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:45:15.693 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 02:45:15.693 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 02:45:15.693 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 02:45:15.693 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 02:45:15.693 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:45:15.693 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 02:45:15.693 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:45:15.693 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:45:15.693 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:45:15.693 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 02:45:15.693 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 02:45:15.693 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:45:15.693 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 02:45:15.693 | INFO | ------------------------------------------ | 20260506 02:45:15.693 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:45:15.693 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 02:45:15.693 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:45:15.693 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:45:15.693 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.39e+01, 1.33e+01 / 8.88e-01 | 20260506 02:45:15.693 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.06e-01, 1.03e-02 / 3.98e-03 | 20260506 02:45:15.693 | INFO | PerfReport: 11.85%, Baseline -- 3.71e+00 / 4.50e+00, 5.82e-01 / 2.17e-01 | 20260506 02:45:15.693 | INFO | PerfReport: 3.29%, Histogram -- 1.03e+00 / 1.52e+00, 1.93e-01 / 8.83e-02 | 20260506 02:45:15.693 | INFO | PerfReport: 15.98%, DME -- 4.00e+01 / 4.39e+01, 3.81e+00 / 9.13e-01 | 20260506 02:45:15.693 | INFO | PerfReport: 57.60%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.61e+00 / 1.18e+00 | 20260506 02:45:15.693 | INFO | PerfReport: 6.97%, PulseAccumulating -- 2.18e+00 / 2.69e+00, 4.10e-01 / 2.34e-01 | 20260506 02:45:15.693 | INFO | PerfReport: 2.98%, Metrics -- 9.34e-01 / 1.21e+00, 2.53e-01 / 1.15e-01 | 20260506 02:45:15.693 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.23e-01 / 3.72e-01, 5.55e-02 / 2.53e-02 | 20260506 02:45:15.693 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 02:45:15.693 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:45:15.693 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:45:15.693 | INFO | PerfReport: Full Execution -- 9.76e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260506 02:45:15.693 | INFO | PerfReport: 22.20%, Upload -- 2.17e+01 / 0.00e+00, 7.14e+00 / 0.00e+00 | 20260506 02:45:15.693 | INFO | PerfReport: 21.98%, Download -- 2.14e+01 / 0.00e+00, 7.57e+00 / 0.00e+00 | 20260506 02:45:15.693 | INFO | PerfReport: 32.10%, Compute -- 3.13e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 02:45:15.693 | INFO | PerfReport: 1.98%, LockUpload -- 1.93e+00 / 0.00e+00, 7.74e+00 / 0.00e+00 | 20260506 02:45:15.693 | INFO | PerfReport: 1.27%, LockDownload -- 1.24e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260506 02:45:15.693 | INFO | PerfReport: 20.47%, LockCompute -- 2.00e+01 / 0.00e+00, 1.77e+01 / 0.00e+00 | 20260506 02:45:15.693 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.08e-02 / 0.00e+00 | 20260506 02:45:15.809 | INFO | Starting a round of Baz IO | 20260506 02:45:18.952 | INFO | Populating RTMetrics report | 20260506 02:45:18.953 | INFO | RTMetrics report populated | 20260506 02:45:18.953 | INFO | Opening tmp RTMetrics json file | 20260506 02:45:18.954 | INFO | Serializing RTMetrics to json | 20260506 02:45:18.955 | INFO | Serialization done, swapping tmp file now... | 20260506 02:45:18.956 | INFO | RTMetrics done | 20260506 02:45:19.045 | INFO | Finished a round of Baz IO: Wrote 1.784 GiB in 3.235 seconds | 20260506 02:45:19.045 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.25% / 0.00% / 78.75% | 20260506 02:45:19.046 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0664949% / 3.97364e-06% / 6.75519e-05% | 20260506 02:45:19.046 | INFO | Baz IO overall: | 20260506 02:45:19.046 | INFO | BytesWritten : 4657917860881 | 20260506 02:45:19.046 | INFO | NumEvents : 2186044443813 | 20260506 02:45:20.837 | INFO | Analyzing chunk frames = [8331264,8331776) | 20260506 02:45:20.837 | INFO | MemInfo: { | 20260506 02:45:20.837 | INFO | "gpu" : " 17 GB", | 20260506 02:45:20.837 | INFO | "huge" : "", | 20260506 02:45:20.837 | INFO | "huge_shared" : " 27 GB", | 20260506 02:45:20.837 | INFO | "malloc" : " 14 GB", | 20260506 02:45:20.837 | INFO | "pinned" : " 29 GB" | 20260506 02:45:20.837 | INFO | } | 20260506 02:45:20.837 | INFO | Perf report for the last 5.14506 seconds: | 20260506 02:45:20.837 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:45:20.837 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:20.837 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:20.837 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:20.837 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260506 02:45:20.837 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:20.837 | INFO | RT_METRICS : 42.6%, 1.0, 22.4ms, 0 | 20260506 02:45:20.837 | INFO | PRE_HQ : 59.6%, 1.7, 53.6ms, 0 | 20260506 02:45:20.837 | INFO | BAZWRITER : 62.0%, 3.5, 112.9ms, 0 | 20260506 02:45:20.837 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:45:20.837 | INFO | Uploaded 36GiB to GPU in 2.18654s (16.4644GiB/s) | 20260506 02:45:20.837 | INFO | Downloaded 38GiB to GPU in 2.24797s (16.9041GiB/s) | 20260506 02:45:25.977 | INFO | Analyzing chunk frames = [8331776,8332288) | 20260506 02:45:25.977 | INFO | MemInfo: { | 20260506 02:45:25.977 | INFO | "gpu" : " 17 GB", | 20260506 02:45:25.977 | INFO | "huge" : "", | 20260506 02:45:25.977 | INFO | "huge_shared" : " 27 GB", | 20260506 02:45:25.977 | INFO | "malloc" : " 14 GB", | 20260506 02:45:25.977 | INFO | "pinned" : " 29 GB" | 20260506 02:45:25.977 | INFO | } | 20260506 02:45:25.977 | INFO | Perf report for the last 5.14047 seconds: | 20260506 02:45:25.977 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:45:25.977 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:25.977 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:25.977 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:25.977 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260506 02:45:25.978 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:25.978 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:25.978 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 02:45:25.978 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:25.978 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:25.978 | INFO | Uploaded 36GiB to GPU in 1.95032s (18.4585GiB/s) | 20260506 02:45:25.978 | INFO | Downloaded 34GiB to GPU in 1.9824s (17.1509GiB/s) | 20260506 02:45:31.110 | INFO | Analyzing chunk frames = [8332288,8332800) | 20260506 02:45:31.110 | INFO | MemInfo: { | 20260506 02:45:31.110 | INFO | "gpu" : " 17 GB", | 20260506 02:45:31.110 | INFO | "huge" : "", | 20260506 02:45:31.110 | INFO | "huge_shared" : " 27 GB", | 20260506 02:45:31.110 | INFO | "malloc" : " 14 GB", | 20260506 02:45:31.110 | INFO | "pinned" : " 29 GB" | 20260506 02:45:31.110 | INFO | } | 20260506 02:45:31.111 | INFO | Perf report for the last 5.13333 seconds: | 20260506 02:45:31.111 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:45:31.111 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:31.111 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:31.111 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:31.111 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260506 02:45:31.111 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:31.111 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:31.111 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 02:45:31.111 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:31.111 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:31.111 | INFO | Uploaded 36GiB to GPU in 1.983s (18.1543GiB/s) | 20260506 02:45:31.111 | INFO | Downloaded 34GiB to GPU in 1.98479s (17.1302GiB/s) | 20260506 02:45:36.247 | INFO | Analyzing chunk frames = [8332800,8333312) | 20260506 02:45:36.247 | INFO | MemInfo: { | 20260506 02:45:36.247 | INFO | "gpu" : " 17 GB", | 20260506 02:45:36.247 | INFO | "huge" : "", | 20260506 02:45:36.247 | INFO | "huge_shared" : " 27 GB", | 20260506 02:45:36.247 | INFO | "malloc" : " 14 GB", | 20260506 02:45:36.247 | INFO | "pinned" : " 29 GB" | 20260506 02:45:36.247 | INFO | } | 20260506 02:45:36.247 | INFO | Perf report for the last 5.13678 seconds: | 20260506 02:45:36.247 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:45:36.247 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:36.247 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:36.247 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:36.247 | INFO | ANALYSIS : 61.1%, 3.0, 94.8ms, 0 | 20260506 02:45:36.248 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:36.248 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:36.248 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:45:36.248 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:36.248 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:36.248 | INFO | Uploaded 36GiB to GPU in 2.09946s (17.1472GiB/s) | 20260506 02:45:36.248 | INFO | Downloaded 34GiB to GPU in 2.01988s (16.8327GiB/s) | 20260506 02:45:41.396 | INFO | Analyzing chunk frames = [8333312,8333824) | 20260506 02:45:41.396 | INFO | MemInfo: { | 20260506 02:45:41.396 | INFO | "gpu" : " 17 GB", | 20260506 02:45:41.396 | INFO | "huge" : "", | 20260506 02:45:41.396 | INFO | "huge_shared" : " 27 GB", | 20260506 02:45:41.396 | INFO | "malloc" : " 14 GB", | 20260506 02:45:41.396 | INFO | "pinned" : " 29 GB" | 20260506 02:45:41.396 | INFO | } | 20260506 02:45:41.396 | INFO | Perf report for the last 5.14897 seconds: | 20260506 02:45:41.396 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:45:41.396 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:41.396 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:41.396 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:41.396 | INFO | ANALYSIS : 60.8%, 3.0, 95.1ms, 0 | 20260506 02:45:41.396 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:41.397 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:41.397 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 02:45:41.397 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:41.397 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:41.397 | INFO | Uploaded 36GiB to GPU in 2.0928s (17.2018GiB/s) | 20260506 02:45:41.397 | INFO | Downloaded 34GiB to GPU in 2.03656s (16.6948GiB/s) | 20260506 02:45:46.532 | INFO | Analyzing chunk frames = [8333824,8334336) | 20260506 02:45:46.532 | INFO | MemInfo: { | 20260506 02:45:46.532 | INFO | "gpu" : " 17 GB", | 20260506 02:45:46.532 | INFO | "huge" : "", | 20260506 02:45:46.532 | INFO | "huge_shared" : " 27 GB", | 20260506 02:45:46.532 | INFO | "malloc" : " 14 GB", | 20260506 02:45:46.532 | INFO | "pinned" : " 29 GB" | 20260506 02:45:46.532 | INFO | } | 20260506 02:45:46.532 | INFO | Perf report for the last 5.13567 seconds: | 20260506 02:45:46.532 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:45:46.532 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:46.532 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:46.532 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:46.532 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 02:45:46.532 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:46.532 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:46.532 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:45:46.532 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:46.532 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:46.532 | INFO | Uploaded 36GiB to GPU in 2.0904s (17.2215GiB/s) | 20260506 02:45:46.532 | INFO | Downloaded 34GiB to GPU in 2.03845s (16.6793GiB/s) | 20260506 02:45:51.672 | INFO | Analyzing chunk frames = [8334336,8334848) | 20260506 02:45:51.672 | INFO | MemInfo: { | 20260506 02:45:51.672 | INFO | "gpu" : " 17 GB", | 20260506 02:45:51.672 | INFO | "huge" : "", | 20260506 02:45:51.672 | INFO | "huge_shared" : " 27 GB", | 20260506 02:45:51.672 | INFO | "malloc" : " 14 GB", | 20260506 02:45:51.672 | INFO | "pinned" : " 29 GB" | 20260506 02:45:51.672 | INFO | } | 20260506 02:45:51.672 | INFO | Perf report for the last 5.14029 seconds: | 20260506 02:45:51.672 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:45:51.672 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:51.672 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:51.672 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:51.672 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260506 02:45:51.672 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:51.672 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:51.673 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 02:45:51.673 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:51.673 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:51.673 | INFO | Uploaded 36GiB to GPU in 2.08962s (17.228GiB/s) | 20260506 02:45:51.673 | INFO | Downloaded 34GiB to GPU in 2.05865s (16.5156GiB/s) | 20260506 02:45:56.809 | INFO | Analyzing chunk frames = [8334848,8335360) | 20260506 02:45:56.809 | INFO | MemInfo: { | 20260506 02:45:56.809 | INFO | "gpu" : " 17 GB", | 20260506 02:45:56.809 | INFO | "huge" : "", | 20260506 02:45:56.809 | INFO | "huge_shared" : " 27 GB", | 20260506 02:45:56.809 | INFO | "malloc" : " 14 GB", | 20260506 02:45:56.809 | INFO | "pinned" : " 29 GB" | 20260506 02:45:56.809 | INFO | } | 20260506 02:45:56.809 | INFO | Perf report for the last 5.13707 seconds: | 20260506 02:45:56.809 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:45:56.809 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:56.810 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:56.810 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:56.810 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 02:45:56.810 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:56.810 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:56.810 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:45:56.810 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:56.810 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:45:56.810 | INFO | Uploaded 36GiB to GPU in 2.02662s (17.7635GiB/s) | 20260506 02:45:56.810 | INFO | Downloaded 34GiB to GPU in 2.01773s (16.8506GiB/s) | 20260506 02:45:56.934 | INFO | Starting a round of Baz IO | 20260506 02:46:00.061 | INFO | Populating RTMetrics report | 20260506 02:46:00.061 | INFO | RTMetrics report populated | 20260506 02:46:00.061 | INFO | Opening tmp RTMetrics json file | 20260506 02:46:00.064 | INFO | Serializing RTMetrics to json | 20260506 02:46:00.066 | INFO | Serialization done, swapping tmp file now... | 20260506 02:46:00.067 | INFO | RTMetrics done | 20260506 02:46:00.146 | INFO | Finished a round of Baz IO: Wrote 1.782 GiB in 3.212 seconds | 20260506 02:46:00.146 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.25% / 0.00% / 78.75% | 20260506 02:46:00.146 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0665863% / 3.97364e-06% / 7.94729e-05% | 20260506 02:46:00.146 | INFO | Baz IO overall: | 20260506 02:46:00.146 | INFO | BytesWritten : 4659831778270 | 20260506 02:46:00.146 | INFO | NumEvents : 2186913053896 | 20260506 02:46:01.945 | INFO | Analyzing chunk frames = [8335360,8335872) | 20260506 02:46:01.945 | INFO | MemInfo: { | 20260506 02:46:01.945 | INFO | "gpu" : " 17 GB", | 20260506 02:46:01.945 | INFO | "huge" : "", | 20260506 02:46:01.945 | INFO | "huge_shared" : " 27 GB", | 20260506 02:46:01.945 | INFO | "malloc" : " 14 GB", | 20260506 02:46:01.945 | INFO | "pinned" : " 29 GB" | 20260506 02:46:01.945 | INFO | } | 20260506 02:46:01.945 | INFO | Perf report for the last 5.13581 seconds: | 20260506 02:46:01.945 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:46:01.945 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:01.945 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:01.945 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:01.945 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260506 02:46:01.945 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:01.945 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260506 02:46:01.945 | INFO | PRE_HQ : 59.1%, 1.7, 53.6ms, 0 | 20260506 02:46:01.945 | INFO | BAZWRITER : 61.7%, 3.5, 111.7ms, 0 | 20260506 02:46:01.945 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:46:01.946 | INFO | Uploaded 36GiB to GPU in 2.11715s (17.004GiB/s) | 20260506 02:46:01.946 | INFO | Downloaded 38GiB to GPU in 2.22121s (17.1078GiB/s) | 20260506 02:46:07.089 | INFO | Analyzing chunk frames = [8335872,8336384) | 20260506 02:46:07.089 | INFO | MemInfo: { | 20260506 02:46:07.089 | INFO | "gpu" : " 17 GB", | 20260506 02:46:07.089 | INFO | "huge" : "", | 20260506 02:46:07.089 | INFO | "huge_shared" : " 27 GB", | 20260506 02:46:07.089 | INFO | "malloc" : " 14 GB", | 20260506 02:46:07.089 | INFO | "pinned" : " 29 GB" | 20260506 02:46:07.089 | INFO | } | 20260506 02:46:07.089 | INFO | Perf report for the last 5.14379 seconds: | 20260506 02:46:07.089 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:46:07.089 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:07.089 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:07.089 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:07.089 | INFO | ANALYSIS : 60.8%, 3.0, 95.0ms, 0 | 20260506 02:46:07.089 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:07.089 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:07.089 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 02:46:07.089 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:07.089 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:07.089 | INFO | Uploaded 36GiB to GPU in 1.97349s (18.2418GiB/s) | 20260506 02:46:07.089 | INFO | Downloaded 34GiB to GPU in 1.97933s (17.1775GiB/s) | 20260506 02:46:12.228 | INFO | Analyzing chunk frames = [8336384,8336896) | 20260506 02:46:12.228 | INFO | MemInfo: { | 20260506 02:46:12.228 | INFO | "gpu" : " 17 GB", | 20260506 02:46:12.228 | INFO | "huge" : "", | 20260506 02:46:12.228 | INFO | "huge_shared" : " 27 GB", | 20260506 02:46:12.228 | INFO | "malloc" : " 14 GB", | 20260506 02:46:12.228 | INFO | "pinned" : " 29 GB" | 20260506 02:46:12.228 | INFO | } | 20260506 02:46:12.228 | INFO | Perf report for the last 5.13908 seconds: | 20260506 02:46:12.228 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:46:12.228 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:12.228 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:12.228 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:12.228 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 02:46:12.228 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:12.228 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:12.228 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 02:46:12.228 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:12.228 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:12.228 | INFO | Uploaded 36GiB to GPU in 2.09894s (17.1515GiB/s) | 20260506 02:46:12.228 | INFO | Downloaded 34GiB to GPU in 2.03709s (16.6905GiB/s) | 20260506 02:46:17.370 | INFO | Analyzing chunk frames = [8336896,8337408) | 20260506 02:46:17.370 | INFO | MemInfo: { | 20260506 02:46:17.370 | INFO | "gpu" : " 17 GB", | 20260506 02:46:17.370 | INFO | "huge" : "", | 20260506 02:46:17.370 | INFO | "huge_shared" : " 27 GB", | 20260506 02:46:17.370 | INFO | "malloc" : " 14 GB", | 20260506 02:46:17.370 | INFO | "pinned" : " 29 GB" | 20260506 02:46:17.370 | INFO | } | 20260506 02:46:17.370 | INFO | Perf report for the last 5.14191 seconds: | 20260506 02:46:17.370 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:46:17.370 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:17.370 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:17.370 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:17.370 | INFO | ANALYSIS : 60.9%, 3.0, 94.6ms, 0 | 20260506 02:46:17.370 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:17.370 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:17.370 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:46:17.370 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:17.370 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:17.370 | INFO | Uploaded 36GiB to GPU in 2.08107s (17.2988GiB/s) | 20260506 02:46:17.370 | INFO | Downloaded 34GiB to GPU in 2.0046s (16.961GiB/s) | 20260506 02:46:22.508 | INFO | Analyzing chunk frames = [8337408,8337920) | 20260506 02:46:22.508 | INFO | MemInfo: { | 20260506 02:46:22.508 | INFO | "gpu" : " 17 GB", | 20260506 02:46:22.508 | INFO | "huge" : "", | 20260506 02:46:22.508 | INFO | "huge_shared" : " 27 GB", | 20260506 02:46:22.508 | INFO | "malloc" : " 14 GB", | 20260506 02:46:22.508 | INFO | "pinned" : " 29 GB" | 20260506 02:46:22.508 | INFO | } | 20260506 02:46:22.508 | INFO | Perf report for the last 5.13836 seconds: | 20260506 02:46:22.509 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:46:22.509 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:22.509 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:22.509 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:22.509 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 02:46:22.509 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:22.509 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:22.509 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 02:46:22.509 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:22.509 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:22.509 | INFO | Uploaded 36GiB to GPU in 2.08737s (17.2466GiB/s) | 20260506 02:46:22.509 | INFO | Downloaded 34GiB to GPU in 2.03669s (16.6937GiB/s) | 20260506 02:46:27.647 | INFO | Analyzing chunk frames = [8337920,8338432) | 20260506 02:46:27.647 | INFO | MemInfo: { | 20260506 02:46:27.647 | INFO | "gpu" : " 17 GB", | 20260506 02:46:27.647 | INFO | "huge" : "", | 20260506 02:46:27.647 | INFO | "huge_shared" : " 27 GB", | 20260506 02:46:27.647 | INFO | "malloc" : " 14 GB", | 20260506 02:46:27.647 | INFO | "pinned" : " 29 GB" | 20260506 02:46:27.647 | INFO | } | 20260506 02:46:27.647 | INFO | Perf report for the last 5.13831 seconds: | 20260506 02:46:27.647 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:46:27.647 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:27.647 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:27.647 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:27.647 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 02:46:27.647 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:27.647 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:27.647 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 02:46:27.647 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:27.647 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:27.647 | INFO | Uploaded 36GiB to GPU in 2.05952s (17.4798GiB/s) | 20260506 02:46:27.647 | INFO | Downloaded 34GiB to GPU in 2.03456s (16.7112GiB/s) | 20260506 02:46:32.788 | INFO | Analyzing chunk frames = [8338432,8338944) | 20260506 02:46:32.788 | INFO | MemInfo: { | 20260506 02:46:32.788 | INFO | "gpu" : " 17 GB", | 20260506 02:46:32.788 | INFO | "huge" : "", | 20260506 02:46:32.788 | INFO | "huge_shared" : " 27 GB", | 20260506 02:46:32.788 | INFO | "malloc" : " 14 GB", | 20260506 02:46:32.788 | INFO | "pinned" : " 29 GB" | 20260506 02:46:32.788 | INFO | } | 20260506 02:46:32.788 | INFO | Perf report for the last 5.14125 seconds: | 20260506 02:46:32.788 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:46:32.788 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:32.788 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:32.788 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:32.788 | INFO | ANALYSIS : 61.8%, 3.0, 96.6ms, 0 | 20260506 02:46:32.788 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:32.788 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:32.788 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 02:46:32.788 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:32.788 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:32.788 | INFO | Uploaded 36GiB to GPU in 2.02869s (17.7454GiB/s) | 20260506 02:46:32.788 | INFO | Downloaded 34GiB to GPU in 2.04909s (16.5927GiB/s) | 20260506 02:46:37.930 | INFO | Analyzing chunk frames = [8338944,8339456) | 20260506 02:46:37.930 | INFO | MemInfo: { | 20260506 02:46:37.930 | INFO | "gpu" : " 17 GB", | 20260506 02:46:37.930 | INFO | "huge" : "", | 20260506 02:46:37.930 | INFO | "huge_shared" : " 27 GB", | 20260506 02:46:37.930 | INFO | "malloc" : " 14 GB", | 20260506 02:46:37.930 | INFO | "pinned" : " 29 GB" | 20260506 02:46:37.930 | INFO | } | 20260506 02:46:37.930 | INFO | Perf report for the last 5.14231 seconds: | 20260506 02:46:37.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:46:37.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:37.930 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:37.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:37.930 | INFO | ANALYSIS : 61.2%, 3.0, 95.7ms, 0 | 20260506 02:46:37.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:37.930 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:37.930 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 02:46:37.930 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:37.931 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:37.931 | INFO | Uploaded 36GiB to GPU in 2.03558s (17.6853GiB/s) | 20260506 02:46:37.931 | INFO | Downloaded 34GiB to GPU in 2.01452s (16.8774GiB/s) | 20260506 02:46:37.931 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 02:46:37.931 | INFO | Total mem usage: 15.4 GiB | 20260506 02:46:37.931 | INFO | All Time High Water Marks sum: 16.3 GiB (Savings: 5.2%) | 20260506 02:46:37.931 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.99%) | 20260506 02:46:37.931 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 02:46:37.931 | INFO | High Water Breakdown: | 20260506 02:46:37.931 | INFO | 60.9% : 8.54 GiB ( 88.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 02:46:37.931 | INFO | 39% : 5.47 GiB ( 82.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 02:46:37.931 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 02:46:37.931 | INFO | ------------------------------------------ | 20260506 02:46:37.931 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:46:37.931 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 02:46:37.931 | INFO | Total mem usage: 26.5 GiB | 20260506 02:46:37.931 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 02:46:37.931 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 02:46:37.931 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 02:46:37.931 | INFO | High Water Breakdown: | 20260506 02:46:37.931 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:46:37.931 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:46:37.931 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:46:37.931 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:46:37.931 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:46:37.931 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:46:37.931 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 02:46:37.931 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 02:46:37.931 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:46:37.931 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 02:46:37.931 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 02:46:37.931 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 02:46:37.931 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 02:46:37.931 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 02:46:37.931 | INFO | ------------------------------------------ | 20260506 02:46:37.931 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:46:37.931 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 02:46:37.931 | INFO | Total mem usage: 25.6 GiB | 20260506 02:46:37.931 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 02:46:37.931 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 02:46:37.931 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 02:46:37.931 | INFO | High Water Breakdown: | 20260506 02:46:37.931 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 02:46:37.931 | INFO | ------------------------------------------ | 20260506 02:46:37.931 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:46:37.931 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 02:46:37.931 | INFO | Total mem usage: 15.7 GiB | 20260506 02:46:37.931 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 02:46:37.931 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 02:46:37.931 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 02:46:37.931 | INFO | High Water Breakdown: | 20260506 02:46:37.931 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 02:46:37.931 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 02:46:37.931 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 02:46:37.931 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 02:46:37.931 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 02:46:37.931 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 02:46:37.931 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 02:46:37.931 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 02:46:37.931 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 02:46:37.931 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:46:37.932 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 02:46:37.932 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:46:37.932 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 02:46:37.932 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 02:46:37.932 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 02:46:37.932 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 02:46:37.932 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:46:37.932 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 02:46:37.932 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:46:37.932 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:46:37.932 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:46:37.932 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 02:46:37.932 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 02:46:37.932 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:46:37.932 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 02:46:37.932 | INFO | ------------------------------------------ | 20260506 02:46:37.932 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:46:37.932 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 02:46:37.932 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:46:37.932 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:46:37.932 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.37e+01, 1.34e+01 / 7.95e-01 | 20260506 02:46:37.932 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.05e-01, 1.03e-02 / 3.11e-03 | 20260506 02:46:37.932 | INFO | PerfReport: 11.90%, Baseline -- 3.73e+00 / 4.52e+00, 5.94e-01 / 1.82e-01 | 20260506 02:46:37.932 | INFO | PerfReport: 3.32%, Histogram -- 1.04e+00 / 1.53e+00, 2.00e-01 / 7.27e-02 | 20260506 02:46:37.932 | INFO | PerfReport: 16.00%, DME -- 4.01e+01 / 4.41e+01, 3.92e+00 / 8.98e-01 | 20260506 02:46:37.932 | INFO | PerfReport: 57.49%, FrameLabeling -- 1.80e+01 / 1.92e+01, 1.58e+00 / 9.60e-01 | 20260506 02:46:37.932 | INFO | PerfReport: 6.97%, PulseAccumulating -- 2.18e+00 / 2.72e+00, 4.12e-01 / 1.81e-01 | 20260506 02:46:37.932 | INFO | PerfReport: 3.00%, Metrics -- 9.38e-01 / 1.21e+00, 2.52e-01 / 1.09e-01 | 20260506 02:46:37.932 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.23e-01 / 3.64e-01, 5.40e-02 / 2.22e-02 | 20260506 02:46:37.932 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 02:46:37.932 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:46:37.932 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:46:37.932 | INFO | PerfReport: Full Execution -- 9.75e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260506 02:46:37.932 | INFO | PerfReport: 22.04%, Upload -- 2.15e+01 / 0.00e+00, 7.06e+00 / 0.00e+00 | 20260506 02:46:37.932 | INFO | PerfReport: 21.89%, Download -- 2.13e+01 / 0.00e+00, 7.55e+00 / 0.00e+00 | 20260506 02:46:37.932 | INFO | PerfReport: 32.14%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 02:46:37.932 | INFO | PerfReport: 1.95%, LockUpload -- 1.90e+00 / 0.00e+00, 7.70e+00 / 0.00e+00 | 20260506 02:46:37.932 | INFO | PerfReport: 1.30%, LockDownload -- 1.26e+00 / 0.00e+00, 3.01e+00 / 0.00e+00 | 20260506 02:46:37.932 | INFO | PerfReport: 20.67%, LockCompute -- 2.01e+01 / 0.00e+00, 1.78e+01 / 0.00e+00 | 20260506 02:46:37.932 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.55e-02 / 0.00e+00 | 20260506 02:46:38.051 | INFO | Starting a round of Baz IO | 20260506 02:46:41.177 | INFO | Populating RTMetrics report | 20260506 02:46:41.177 | INFO | RTMetrics report populated | 20260506 02:46:41.177 | INFO | Opening tmp RTMetrics json file | 20260506 02:46:41.178 | INFO | Serializing RTMetrics to json | 20260506 02:46:41.179 | INFO | Serialization done, swapping tmp file now... | 20260506 02:46:41.179 | INFO | RTMetrics done | 20260506 02:46:41.247 | INFO | Finished a round of Baz IO: Wrote 1.781 GiB in 3.195 seconds | 20260506 02:46:41.247 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.25% / 0.00% / 78.75% | 20260506 02:46:41.247 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0664473% / 0% / 6.75519e-05% | 20260506 02:46:41.247 | INFO | Baz IO overall: | 20260506 02:46:41.247 | INFO | BytesWritten : 4661744404702 | 20260506 02:46:41.247 | INFO | NumEvents : 2187781085279 | 20260506 02:46:43.060 | INFO | Analyzing chunk frames = [8339456,8339968) | 20260506 02:46:43.060 | INFO | MemInfo: { | 20260506 02:46:43.060 | INFO | "gpu" : " 17 GB", | 20260506 02:46:43.060 | INFO | "huge" : "", | 20260506 02:46:43.060 | INFO | "huge_shared" : " 27 GB", | 20260506 02:46:43.060 | INFO | "malloc" : " 14 GB", | 20260506 02:46:43.060 | INFO | "pinned" : " 29 GB" | 20260506 02:46:43.060 | INFO | } | 20260506 02:46:43.060 | INFO | Perf report for the last 5.12936 seconds: | 20260506 02:46:43.060 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:46:43.060 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:43.060 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:43.060 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:43.060 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260506 02:46:43.060 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:43.060 | INFO | RT_METRICS : 42.8%, 1.0, 22.4ms, 0 | 20260506 02:46:43.060 | INFO | PRE_HQ : 59.2%, 1.7, 53.2ms, 0 | 20260506 02:46:43.060 | INFO | BAZWRITER : 61.5%, 3.5, 109.3ms, 0 | 20260506 02:46:43.060 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:46:43.060 | INFO | Uploaded 36GiB to GPU in 2.13235s (16.8828GiB/s) | 20260506 02:46:43.060 | INFO | Downloaded 38GiB to GPU in 2.23038s (17.0375GiB/s) | 20260506 02:46:48.201 | INFO | Analyzing chunk frames = [8339968,8340480) | 20260506 02:46:48.201 | INFO | MemInfo: { | 20260506 02:46:48.201 | INFO | "gpu" : " 17 GB", | 20260506 02:46:48.201 | INFO | "huge" : "", | 20260506 02:46:48.201 | INFO | "huge_shared" : " 27 GB", | 20260506 02:46:48.201 | INFO | "malloc" : " 14 GB", | 20260506 02:46:48.201 | INFO | "pinned" : " 29 GB" | 20260506 02:46:48.201 | INFO | } | 20260506 02:46:48.201 | INFO | Perf report for the last 5.14165 seconds: | 20260506 02:46:48.201 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:46:48.201 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:48.201 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:48.201 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:48.202 | INFO | ANALYSIS : 60.8%, 3.0, 95.0ms, 0 | 20260506 02:46:48.202 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:48.202 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:48.202 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 02:46:48.202 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:48.202 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:48.202 | INFO | Uploaded 36GiB to GPU in 1.96972s (18.2767GiB/s) | 20260506 02:46:48.202 | INFO | Downloaded 34GiB to GPU in 1.98064s (17.1662GiB/s) | 20260506 02:46:53.346 | INFO | Analyzing chunk frames = [8340480,8340992) | 20260506 02:46:53.346 | INFO | MemInfo: { | 20260506 02:46:53.346 | INFO | "gpu" : " 17 GB", | 20260506 02:46:53.346 | INFO | "huge" : "", | 20260506 02:46:53.346 | INFO | "huge_shared" : " 27 GB", | 20260506 02:46:53.346 | INFO | "malloc" : " 14 GB", | 20260506 02:46:53.346 | INFO | "pinned" : " 29 GB" | 20260506 02:46:53.346 | INFO | } | 20260506 02:46:53.347 | INFO | Perf report for the last 5.14516 seconds: | 20260506 02:46:53.347 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:46:53.347 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:53.347 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:53.347 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:53.347 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 02:46:53.347 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:53.347 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:53.347 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:46:53.347 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:53.347 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:53.347 | INFO | Uploaded 36GiB to GPU in 2.09075s (17.2187GiB/s) | 20260506 02:46:53.347 | INFO | Downloaded 34GiB to GPU in 2.02755s (16.769GiB/s) | 20260506 02:46:58.488 | INFO | Analyzing chunk frames = [8340992,8341504) | 20260506 02:46:58.488 | INFO | MemInfo: { | 20260506 02:46:58.488 | INFO | "gpu" : " 17 GB", | 20260506 02:46:58.488 | INFO | "huge" : "", | 20260506 02:46:58.488 | INFO | "huge_shared" : " 27 GB", | 20260506 02:46:58.488 | INFO | "malloc" : " 14 GB", | 20260506 02:46:58.488 | INFO | "pinned" : " 29 GB" | 20260506 02:46:58.488 | INFO | } | 20260506 02:46:58.488 | INFO | Perf report for the last 5.14116 seconds: | 20260506 02:46:58.488 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:46:58.488 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:58.488 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:58.488 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:58.488 | INFO | ANALYSIS : 60.9%, 3.0, 94.6ms, 0 | 20260506 02:46:58.488 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:58.488 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:58.488 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 02:46:58.488 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:58.488 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:46:58.488 | INFO | Uploaded 36GiB to GPU in 2.08771s (17.2438GiB/s) | 20260506 02:46:58.488 | INFO | Downloaded 34GiB to GPU in 2.00561s (16.9524GiB/s) | 20260506 02:47:03.625 | INFO | Analyzing chunk frames = [8341504,8342016) | 20260506 02:47:03.625 | INFO | MemInfo: { | 20260506 02:47:03.625 | INFO | "gpu" : " 17 GB", | 20260506 02:47:03.625 | INFO | "huge" : "", | 20260506 02:47:03.625 | INFO | "huge_shared" : " 27 GB", | 20260506 02:47:03.625 | INFO | "malloc" : " 14 GB", | 20260506 02:47:03.625 | INFO | "pinned" : " 29 GB" | 20260506 02:47:03.625 | INFO | } | 20260506 02:47:03.625 | INFO | Perf report for the last 5.13703 seconds: | 20260506 02:47:03.625 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:47:03.625 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:03.625 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:03.625 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:03.625 | INFO | ANALYSIS : 60.8%, 3.0, 94.9ms, 0 | 20260506 02:47:03.625 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:03.625 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:03.625 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 02:47:03.625 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:03.625 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:03.625 | INFO | Uploaded 36GiB to GPU in 2.12162s (16.9682GiB/s) | 20260506 02:47:03.625 | INFO | Downloaded 34GiB to GPU in 2.03951s (16.6707GiB/s) | 20260506 02:47:08.764 | INFO | Analyzing chunk frames = [8342016,8342528) | 20260506 02:47:08.764 | INFO | MemInfo: { | 20260506 02:47:08.764 | INFO | "gpu" : " 17 GB", | 20260506 02:47:08.764 | INFO | "huge" : "", | 20260506 02:47:08.764 | INFO | "huge_shared" : " 27 GB", | 20260506 02:47:08.764 | INFO | "malloc" : " 14 GB", | 20260506 02:47:08.764 | INFO | "pinned" : " 29 GB" | 20260506 02:47:08.764 | INFO | } | 20260506 02:47:08.764 | INFO | Perf report for the last 5.13914 seconds: | 20260506 02:47:08.764 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:47:08.764 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:08.764 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:08.764 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:08.764 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 02:47:08.764 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:08.764 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:08.764 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 02:47:08.764 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:08.764 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:08.764 | INFO | Uploaded 36GiB to GPU in 2.10614s (17.0929GiB/s) | 20260506 02:47:08.764 | INFO | Downloaded 34GiB to GPU in 2.03542s (16.7042GiB/s) | 20260506 02:47:13.898 | INFO | Analyzing chunk frames = [8342528,8343040) | 20260506 02:47:13.898 | INFO | MemInfo: { | 20260506 02:47:13.898 | INFO | "gpu" : " 17 GB", | 20260506 02:47:13.898 | INFO | "huge" : "", | 20260506 02:47:13.898 | INFO | "huge_shared" : " 27 GB", | 20260506 02:47:13.898 | INFO | "malloc" : " 14 GB", | 20260506 02:47:13.898 | INFO | "pinned" : " 29 GB" | 20260506 02:47:13.898 | INFO | } | 20260506 02:47:13.898 | INFO | Perf report for the last 5.13427 seconds: | 20260506 02:47:13.898 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:47:13.898 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:13.898 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:13.898 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:13.898 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 02:47:13.898 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:13.898 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:13.898 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 02:47:13.898 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:13.898 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:13.898 | INFO | Uploaded 36GiB to GPU in 2.00013s (17.9988GiB/s) | 20260506 02:47:13.898 | INFO | Downloaded 34GiB to GPU in 2.0459s (16.6186GiB/s) | 20260506 02:47:19.038 | INFO | Analyzing chunk frames = [8343040,8343552) | 20260506 02:47:19.038 | INFO | MemInfo: { | 20260506 02:47:19.038 | INFO | "gpu" : " 17 GB", | 20260506 02:47:19.038 | INFO | "huge" : "", | 20260506 02:47:19.038 | INFO | "huge_shared" : " 27 GB", | 20260506 02:47:19.038 | INFO | "malloc" : " 14 GB", | 20260506 02:47:19.038 | INFO | "pinned" : " 29 GB" | 20260506 02:47:19.038 | INFO | } | 20260506 02:47:19.038 | INFO | Perf report for the last 5.13966 seconds: | 20260506 02:47:19.038 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:47:19.038 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:19.038 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:19.038 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:19.038 | INFO | ANALYSIS : 61.4%, 3.0, 95.9ms, 0 | 20260506 02:47:19.038 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:19.038 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:19.038 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 02:47:19.038 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:19.038 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:19.038 | INFO | Uploaded 36GiB to GPU in 2.07393s (17.3584GiB/s) | 20260506 02:47:19.038 | INFO | Downloaded 34GiB to GPU in 2.02478s (16.792GiB/s) | 20260506 02:47:19.163 | INFO | Starting a round of Baz IO | 20260506 02:47:22.288 | INFO | Populating RTMetrics report | 20260506 02:47:22.289 | INFO | RTMetrics report populated | 20260506 02:47:22.289 | INFO | Opening tmp RTMetrics json file | 20260506 02:47:22.289 | INFO | Serializing RTMetrics to json | 20260506 02:47:22.292 | INFO | Serialization done, swapping tmp file now... | 20260506 02:47:22.293 | INFO | RTMetrics done | 20260506 02:47:22.377 | INFO | Finished a round of Baz IO: Wrote 1.779 GiB in 3.214 seconds | 20260506 02:47:22.378 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.25% / 0.00% / 78.75% | 20260506 02:47:22.378 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0662565% / 3.97364e-06% / 7.94729e-05% | 20260506 02:47:22.378 | INFO | Baz IO overall: | 20260506 02:47:22.378 | INFO | BytesWritten : 4663654458505 | 20260506 02:47:22.378 | INFO | NumEvents : 2188647487807 | 20260506 02:47:24.178 | INFO | Analyzing chunk frames = [8343552,8344064) | 20260506 02:47:24.178 | INFO | MemInfo: { | 20260506 02:47:24.178 | INFO | "gpu" : " 17 GB", | 20260506 02:47:24.178 | INFO | "huge" : "", | 20260506 02:47:24.178 | INFO | "huge_shared" : " 27 GB", | 20260506 02:47:24.178 | INFO | "malloc" : " 14 GB", | 20260506 02:47:24.178 | INFO | "pinned" : " 29 GB" | 20260506 02:47:24.178 | INFO | } | 20260506 02:47:24.178 | INFO | Perf report for the last 5.14011 seconds: | 20260506 02:47:24.178 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:47:24.178 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:24.178 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:24.178 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:24.178 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260506 02:47:24.178 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:24.178 | INFO | RT_METRICS : 42.6%, 1.0, 22.4ms, 0 | 20260506 02:47:24.178 | INFO | PRE_HQ : 59.2%, 1.7, 53.5ms, 0 | 20260506 02:47:24.178 | INFO | BAZWRITER : 61.7%, 3.6, 113.6ms, 0 | 20260506 02:47:24.178 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:47:24.178 | INFO | Uploaded 36GiB to GPU in 2.11279s (17.0391GiB/s) | 20260506 02:47:24.178 | INFO | Downloaded 38GiB to GPU in 2.23354s (17.0133GiB/s) | 20260506 02:47:29.319 | INFO | Analyzing chunk frames = [8344064,8344576) | 20260506 02:47:29.319 | INFO | MemInfo: { | 20260506 02:47:29.319 | INFO | "gpu" : " 17 GB", | 20260506 02:47:29.319 | INFO | "huge" : "", | 20260506 02:47:29.319 | INFO | "huge_shared" : " 27 GB", | 20260506 02:47:29.319 | INFO | "malloc" : " 14 GB", | 20260506 02:47:29.319 | INFO | "pinned" : " 29 GB" | 20260506 02:47:29.319 | INFO | } | 20260506 02:47:29.319 | INFO | Perf report for the last 5.14142 seconds: | 20260506 02:47:29.319 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:47:29.319 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:29.319 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:29.320 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:29.320 | INFO | ANALYSIS : 60.7%, 3.0, 94.8ms, 0 | 20260506 02:47:29.320 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:29.320 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:29.320 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 02:47:29.320 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:29.320 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:29.320 | INFO | Uploaded 36GiB to GPU in 1.93129s (18.6404GiB/s) | 20260506 02:47:29.320 | INFO | Downloaded 34GiB to GPU in 1.97856s (17.1842GiB/s) | 20260506 02:47:34.455 | INFO | Analyzing chunk frames = [8344576,8345088) | 20260506 02:47:34.455 | INFO | MemInfo: { | 20260506 02:47:34.455 | INFO | "gpu" : " 17 GB", | 20260506 02:47:34.455 | INFO | "huge" : "", | 20260506 02:47:34.455 | INFO | "huge_shared" : " 27 GB", | 20260506 02:47:34.455 | INFO | "malloc" : " 14 GB", | 20260506 02:47:34.455 | INFO | "pinned" : " 29 GB" | 20260506 02:47:34.455 | INFO | } | 20260506 02:47:34.456 | INFO | Perf report for the last 5.13609 seconds: | 20260506 02:47:34.456 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:47:34.456 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:34.456 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:34.456 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:34.456 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 02:47:34.456 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:34.456 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:34.456 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:47:34.456 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:34.456 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:34.456 | INFO | Uploaded 36GiB to GPU in 2.09033s (17.2222GiB/s) | 20260506 02:47:34.456 | INFO | Downloaded 34GiB to GPU in 2.02644s (16.7782GiB/s) | 20260506 02:47:39.600 | INFO | Analyzing chunk frames = [8345088,8345600) | 20260506 02:47:39.600 | INFO | MemInfo: { | 20260506 02:47:39.600 | INFO | "gpu" : " 17 GB", | 20260506 02:47:39.600 | INFO | "huge" : "", | 20260506 02:47:39.600 | INFO | "huge_shared" : " 27 GB", | 20260506 02:47:39.600 | INFO | "malloc" : " 14 GB", | 20260506 02:47:39.600 | INFO | "pinned" : " 29 GB" | 20260506 02:47:39.600 | INFO | } | 20260506 02:47:39.600 | INFO | Perf report for the last 5.14485 seconds: | 20260506 02:47:39.600 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:47:39.600 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:39.600 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:39.600 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:39.601 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 02:47:39.601 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:39.601 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:39.601 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:47:39.601 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:39.601 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:39.601 | INFO | Uploaded 36GiB to GPU in 2.1424s (16.8036GiB/s) | 20260506 02:47:39.601 | INFO | Downloaded 34GiB to GPU in 2.0572s (16.5273GiB/s) | 20260506 02:47:44.740 | INFO | Analyzing chunk frames = [8345600,8346112) | 20260506 02:47:44.740 | INFO | MemInfo: { | 20260506 02:47:44.740 | INFO | "gpu" : " 17 GB", | 20260506 02:47:44.740 | INFO | "huge" : "", | 20260506 02:47:44.740 | INFO | "huge_shared" : " 27 GB", | 20260506 02:47:44.740 | INFO | "malloc" : " 14 GB", | 20260506 02:47:44.740 | INFO | "pinned" : " 29 GB" | 20260506 02:47:44.740 | INFO | } | 20260506 02:47:44.740 | INFO | Perf report for the last 5.13943 seconds: | 20260506 02:47:44.740 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:47:44.740 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:44.740 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:44.740 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:44.740 | INFO | ANALYSIS : 60.8%, 3.0, 95.0ms, 0 | 20260506 02:47:44.740 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:44.740 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:44.740 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:47:44.740 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:44.740 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:44.740 | INFO | Uploaded 36GiB to GPU in 2.12105s (16.9727GiB/s) | 20260506 02:47:44.740 | INFO | Downloaded 34GiB to GPU in 2.0445s (16.63GiB/s) | 20260506 02:47:49.875 | INFO | Analyzing chunk frames = [8346112,8346624) | 20260506 02:47:49.875 | INFO | MemInfo: { | 20260506 02:47:49.875 | INFO | "gpu" : " 17 GB", | 20260506 02:47:49.875 | INFO | "huge" : "", | 20260506 02:47:49.875 | INFO | "huge_shared" : " 27 GB", | 20260506 02:47:49.875 | INFO | "malloc" : " 14 GB", | 20260506 02:47:49.875 | INFO | "pinned" : " 29 GB" | 20260506 02:47:49.875 | INFO | } | 20260506 02:47:49.875 | INFO | Perf report for the last 5.13551 seconds: | 20260506 02:47:49.875 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:47:49.875 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:49.875 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:49.875 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:49.875 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260506 02:47:49.875 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:49.876 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:49.876 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 02:47:49.876 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:49.876 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:49.876 | INFO | Uploaded 36GiB to GPU in 2.0571s (17.5004GiB/s) | 20260506 02:47:49.876 | INFO | Downloaded 34GiB to GPU in 2.03673s (16.6934GiB/s) | 20260506 02:47:55.013 | INFO | Analyzing chunk frames = [8346624,8347136) | 20260506 02:47:55.013 | INFO | MemInfo: { | 20260506 02:47:55.013 | INFO | "gpu" : " 17 GB", | 20260506 02:47:55.013 | INFO | "huge" : "", | 20260506 02:47:55.013 | INFO | "huge_shared" : " 27 GB", | 20260506 02:47:55.013 | INFO | "malloc" : " 14 GB", | 20260506 02:47:55.013 | INFO | "pinned" : " 29 GB" | 20260506 02:47:55.013 | INFO | } | 20260506 02:47:55.014 | INFO | Perf report for the last 5.13812 seconds: | 20260506 02:47:55.014 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:47:55.014 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:55.014 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:55.014 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:55.014 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 02:47:55.014 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:55.014 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:55.014 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 02:47:55.014 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:55.014 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:47:55.014 | INFO | Uploaded 36GiB to GPU in 2.02957s (17.7378GiB/s) | 20260506 02:47:55.014 | INFO | Downloaded 34GiB to GPU in 2.04855s (16.5971GiB/s) | 20260506 02:48:00.156 | INFO | Analyzing chunk frames = [8347136,8347648) | 20260506 02:48:00.156 | INFO | MemInfo: { | 20260506 02:48:00.156 | INFO | "gpu" : " 17 GB", | 20260506 02:48:00.156 | INFO | "huge" : "", | 20260506 02:48:00.156 | INFO | "huge_shared" : " 27 GB", | 20260506 02:48:00.156 | INFO | "malloc" : " 14 GB", | 20260506 02:48:00.156 | INFO | "pinned" : " 29 GB" | 20260506 02:48:00.156 | INFO | } | 20260506 02:48:00.156 | INFO | Perf report for the last 5.14265 seconds: | 20260506 02:48:00.156 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:48:00.156 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:00.156 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:00.156 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:00.156 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260506 02:48:00.156 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:00.156 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:00.156 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 02:48:00.156 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:00.156 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:00.156 | INFO | Uploaded 36GiB to GPU in 2.10281s (17.12GiB/s) | 20260506 02:48:00.157 | INFO | Downloaded 34GiB to GPU in 2.04061s (16.6617GiB/s) | 20260506 02:48:00.157 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 02:48:00.157 | INFO | Total mem usage: 15.4 GiB | 20260506 02:48:00.157 | INFO | All Time High Water Marks sum: 16.3 GiB (Savings: 5.2%) | 20260506 02:48:00.157 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.99%) | 20260506 02:48:00.157 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 02:48:00.157 | INFO | High Water Breakdown: | 20260506 02:48:00.157 | INFO | 60.9% : 8.54 GiB ( 88.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 02:48:00.157 | INFO | 39% : 5.47 GiB ( 82.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 02:48:00.157 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 02:48:00.157 | INFO | ------------------------------------------ | 20260506 02:48:00.157 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:48:00.157 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 02:48:00.157 | INFO | Total mem usage: 26.5 GiB | 20260506 02:48:00.157 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 02:48:00.157 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 02:48:00.157 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 02:48:00.157 | INFO | High Water Breakdown: | 20260506 02:48:00.157 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:48:00.157 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:48:00.157 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:48:00.157 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:48:00.157 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:48:00.157 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:48:00.157 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 02:48:00.157 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 02:48:00.157 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:48:00.157 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 02:48:00.157 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 02:48:00.157 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 02:48:00.157 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 02:48:00.157 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 02:48:00.157 | INFO | ------------------------------------------ | 20260506 02:48:00.157 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:48:00.157 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 02:48:00.157 | INFO | Total mem usage: 25.6 GiB | 20260506 02:48:00.157 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 02:48:00.157 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260506 02:48:00.157 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260506 02:48:00.157 | INFO | High Water Breakdown: | 20260506 02:48:00.157 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260506 02:48:00.157 | INFO | ------------------------------------------ | 20260506 02:48:00.157 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:48:00.157 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 02:48:00.157 | INFO | Total mem usage: 15.7 GiB | 20260506 02:48:00.157 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 02:48:00.157 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 02:48:00.157 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 02:48:00.157 | INFO | High Water Breakdown: | 20260506 02:48:00.157 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 02:48:00.157 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 02:48:00.157 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 02:48:00.157 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 02:48:00.157 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 02:48:00.157 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 02:48:00.157 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 02:48:00.157 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 02:48:00.157 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 02:48:00.157 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:48:00.157 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 02:48:00.157 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:48:00.157 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 02:48:00.157 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 02:48:00.157 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 02:48:00.157 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 02:48:00.157 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:48:00.157 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 02:48:00.158 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:48:00.158 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:48:00.158 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:48:00.158 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 02:48:00.158 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 02:48:00.158 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:48:00.158 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 02:48:00.158 | INFO | ------------------------------------------ | 20260506 02:48:00.158 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:48:00.158 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 02:48:00.158 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:48:00.158 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:48:00.158 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.37e+01, 1.34e+01 / 8.69e-01 | 20260506 02:48:00.158 | INFO | PerfReport: 0.29%, AggregateStats -- 9.18e-02 / 1.04e-01, 9.94e-03 / 3.43e-03 | 20260506 02:48:00.158 | INFO | PerfReport: 11.85%, Baseline -- 3.71e+00 / 4.50e+00, 5.82e-01 / 3.43e-01 | 20260506 02:48:00.158 | INFO | PerfReport: 3.30%, Histogram -- 1.03e+00 / 1.53e+00, 1.94e-01 / 1.09e-01 | 20260506 02:48:00.158 | INFO | PerfReport: 16.04%, DME -- 4.01e+01 / 4.41e+01, 3.89e+00 / 9.59e-01 | 20260506 02:48:00.158 | INFO | PerfReport: 57.48%, FrameLabeling -- 1.80e+01 / 1.91e+01, 1.53e+00 / 1.09e+00 | 20260506 02:48:00.158 | INFO | PerfReport: 7.00%, PulseAccumulating -- 2.19e+00 / 2.75e+00, 4.13e-01 / 2.11e-01 | 20260506 02:48:00.158 | INFO | PerfReport: 2.99%, Metrics -- 9.36e-01 / 1.21e+00, 2.52e-01 / 1.11e-01 | 20260506 02:48:00.158 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.23e-01 / 3.71e-01, 5.42e-02 / 2.47e-02 | 20260506 02:48:00.158 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 02:48:00.158 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:48:00.158 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:48:00.158 | INFO | PerfReport: Full Execution -- 9.76e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260506 02:48:00.158 | INFO | PerfReport: 22.13%, Upload -- 2.16e+01 / 0.00e+00, 7.08e+00 / 0.00e+00 | 20260506 02:48:00.158 | INFO | PerfReport: 21.92%, Download -- 2.14e+01 / 0.00e+00, 7.61e+00 / 0.00e+00 | 20260506 02:48:00.158 | INFO | PerfReport: 32.05%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 02:48:00.158 | INFO | PerfReport: 2.04%, LockUpload -- 1.99e+00 / 0.00e+00, 7.92e+00 / 0.00e+00 | 20260506 02:48:00.158 | INFO | PerfReport: 1.35%, LockDownload -- 1.32e+00 / 0.00e+00, 3.09e+00 / 0.00e+00 | 20260506 02:48:00.158 | INFO | PerfReport: 20.50%, LockCompute -- 2.00e+01 / 0.00e+00, 1.75e+01 / 0.00e+00 | 20260506 02:48:00.158 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 2.32e-02 / 0.00e+00 | 20260506 02:48:00.281 | INFO | Starting a round of Baz IO | 20260506 02:48:03.410 | INFO | Populating RTMetrics report | 20260506 02:48:03.410 | INFO | RTMetrics report populated | 20260506 02:48:03.410 | INFO | Opening tmp RTMetrics json file | 20260506 02:48:03.411 | INFO | Serializing RTMetrics to json | 20260506 02:48:03.412 | INFO | Serialization done, swapping tmp file now... | 20260506 02:48:03.412 | INFO | RTMetrics done | 20260506 02:48:03.482 | INFO | Finished a round of Baz IO: Wrote 1.779 GiB in 3.2 seconds | 20260506 02:48:03.482 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.25% / 0.00% / 78.75% | 20260506 02:48:03.482 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0662565% / 0% / 7.54992e-05% | 20260506 02:48:03.482 | INFO | Baz IO overall: | 20260506 02:48:03.482 | INFO | BytesWritten : 4665565094563 | 20260506 02:48:03.482 | INFO | NumEvents : 2189514545513 | 20260506 02:48:05.297 | INFO | Analyzing chunk frames = [8347648,8348160) | 20260506 02:48:05.297 | INFO | MemInfo: { | 20260506 02:48:05.297 | INFO | "gpu" : " 17 GB", | 20260506 02:48:05.297 | INFO | "huge" : "", | 20260506 02:48:05.297 | INFO | "huge_shared" : " 27 GB", | 20260506 02:48:05.297 | INFO | "malloc" : " 14 GB", | 20260506 02:48:05.297 | INFO | "pinned" : " 29 GB" | 20260506 02:48:05.297 | INFO | } | 20260506 02:48:05.297 | INFO | Perf report for the last 5.14091 seconds: | 20260506 02:48:05.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:48:05.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:05.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:05.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:05.297 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260506 02:48:05.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:05.297 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260506 02:48:05.297 | INFO | PRE_HQ : 59.1%, 1.7, 53.6ms, 0 | 20260506 02:48:05.297 | INFO | BAZWRITER : 61.4%, 3.4, 108.6ms, 0 | 20260506 02:48:05.297 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:48:05.298 | INFO | Uploaded 36GiB to GPU in 2.10051s (17.1387GiB/s) | 20260506 02:48:05.298 | INFO | Downloaded 38GiB to GPU in 2.22176s (17.1036GiB/s) | 20260506 02:48:10.438 | INFO | Analyzing chunk frames = [8348160,8348672) | 20260506 02:48:10.438 | INFO | MemInfo: { | 20260506 02:48:10.438 | INFO | "gpu" : " 17 GB", | 20260506 02:48:10.438 | INFO | "huge" : "", | 20260506 02:48:10.438 | INFO | "huge_shared" : " 27 GB", | 20260506 02:48:10.438 | INFO | "malloc" : " 14 GB", | 20260506 02:48:10.438 | INFO | "pinned" : " 29 GB" | 20260506 02:48:10.438 | INFO | } | 20260506 02:48:10.438 | INFO | Perf report for the last 5.14085 seconds: | 20260506 02:48:10.438 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:48:10.438 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:10.438 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:10.438 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:10.438 | INFO | ANALYSIS : 60.6%, 3.0, 94.7ms, 0 | 20260506 02:48:10.438 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:10.438 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:10.438 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 02:48:10.438 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:10.438 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:10.438 | INFO | Uploaded 36GiB to GPU in 1.98108s (18.1719GiB/s) | 20260506 02:48:10.438 | INFO | Downloaded 34GiB to GPU in 1.98353s (17.1412GiB/s) | 20260506 02:48:15.573 | INFO | Analyzing chunk frames = [8348672,8349184) | 20260506 02:48:15.573 | INFO | MemInfo: { | 20260506 02:48:15.573 | INFO | "gpu" : " 17 GB", | 20260506 02:48:15.573 | INFO | "huge" : "", | 20260506 02:48:15.573 | INFO | "huge_shared" : " 27 GB", | 20260506 02:48:15.573 | INFO | "malloc" : " 14 GB", | 20260506 02:48:15.573 | INFO | "pinned" : " 29 GB" | 20260506 02:48:15.574 | INFO | } | 20260506 02:48:15.574 | INFO | Perf report for the last 5.13574 seconds: | 20260506 02:48:15.574 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:48:15.574 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:15.574 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:15.574 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:15.574 | INFO | ANALYSIS : 60.7%, 3.0, 94.3ms, 0 | 20260506 02:48:15.574 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:15.574 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:15.574 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:48:15.574 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:15.574 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:15.574 | INFO | Uploaded 36GiB to GPU in 2.00898s (17.9196GiB/s) | 20260506 02:48:15.574 | INFO | Downloaded 34GiB to GPU in 1.98631s (17.1172GiB/s) | 20260506 02:48:20.715 | INFO | Analyzing chunk frames = [8349184,8349696) | 20260506 02:48:20.715 | INFO | MemInfo: { | 20260506 02:48:20.715 | INFO | "gpu" : " 17 GB", | 20260506 02:48:20.715 | INFO | "huge" : "", | 20260506 02:48:20.715 | INFO | "huge_shared" : " 27 GB", | 20260506 02:48:20.715 | INFO | "malloc" : " 14 GB", | 20260506 02:48:20.715 | INFO | "pinned" : " 29 GB" | 20260506 02:48:20.715 | INFO | } | 20260506 02:48:20.715 | INFO | Perf report for the last 5.14175 seconds: | 20260506 02:48:20.716 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:48:20.716 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:20.716 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:20.716 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:20.716 | INFO | ANALYSIS : 61.2%, 3.0, 95.1ms, 0 | 20260506 02:48:20.716 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:20.716 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:20.716 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 02:48:20.716 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:20.716 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:20.716 | INFO | Uploaded 36GiB to GPU in 2.10342s (17.115GiB/s) | 20260506 02:48:20.716 | INFO | Downloaded 34GiB to GPU in 2.02143s (16.8198GiB/s) | 20260506 02:48:25.848 | INFO | Analyzing chunk frames = [8349696,8350208) | 20260506 02:48:25.848 | INFO | MemInfo: { | 20260506 02:48:25.848 | INFO | "gpu" : " 17 GB", | 20260506 02:48:25.848 | INFO | "huge" : "", | 20260506 02:48:25.848 | INFO | "huge_shared" : " 27 GB", | 20260506 02:48:25.848 | INFO | "malloc" : " 14 GB", | 20260506 02:48:25.848 | INFO | "pinned" : " 29 GB" | 20260506 02:48:25.848 | INFO | } | 20260506 02:48:25.848 | INFO | Perf report for the last 5.13286 seconds: | 20260506 02:48:25.848 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:48:25.848 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:25.848 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:25.848 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:25.848 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260506 02:48:25.848 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:25.848 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:25.848 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 02:48:25.848 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:25.848 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:25.849 | INFO | Uploaded 36GiB to GPU in 2.07618s (17.3395GiB/s) | 20260506 02:48:25.849 | INFO | Downloaded 34GiB to GPU in 2.02807s (16.7647GiB/s) | 20260506 02:48:30.994 | INFO | Analyzing chunk frames = [8350208,8350720) | 20260506 02:48:30.994 | INFO | MemInfo: { | 20260506 02:48:30.994 | INFO | "gpu" : " 17 GB", | 20260506 02:48:30.994 | INFO | "huge" : "", | 20260506 02:48:30.994 | INFO | "huge_shared" : " 27 GB", | 20260506 02:48:30.994 | INFO | "malloc" : " 14 GB", | 20260506 02:48:30.994 | INFO | "pinned" : " 29 GB" | 20260506 02:48:30.994 | INFO | } | 20260506 02:48:30.995 | INFO | Perf report for the last 5.14617 seconds: | 20260506 02:48:30.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:48:30.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:30.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:30.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:30.995 | INFO | ANALYSIS : 60.9%, 3.0, 95.2ms, 0 | 20260506 02:48:30.995 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:30.995 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:30.995 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 02:48:30.995 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:30.995 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:30.995 | INFO | Uploaded 36GiB to GPU in 2.096s (17.1756GiB/s) | 20260506 02:48:30.995 | INFO | Downloaded 34GiB to GPU in 2.04067s (16.6612GiB/s) | 20260506 02:48:36.132 | INFO | Analyzing chunk frames = [8350720,8351232) | 20260506 02:48:36.132 | INFO | MemInfo: { | 20260506 02:48:36.132 | INFO | "gpu" : " 17 GB", | 20260506 02:48:36.132 | INFO | "huge" : "", | 20260506 02:48:36.132 | INFO | "huge_shared" : " 27 GB", | 20260506 02:48:36.132 | INFO | "malloc" : " 14 GB", | 20260506 02:48:36.132 | INFO | "pinned" : " 29 GB" | 20260506 02:48:36.132 | INFO | } | 20260506 02:48:36.132 | INFO | Perf report for the last 5.13753 seconds: | 20260506 02:48:36.132 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:48:36.132 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:36.132 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:36.132 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:36.132 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260506 02:48:36.132 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:36.132 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:36.132 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 02:48:36.132 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:36.132 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:36.132 | INFO | Uploaded 36GiB to GPU in 2.09651s (17.1714GiB/s) | 20260506 02:48:36.132 | INFO | Downloaded 34GiB to GPU in 2.07171s (16.4116GiB/s) | 20260506 02:48:41.269 | INFO | Analyzing chunk frames = [8351232,8351744) | 20260506 02:48:41.269 | INFO | MemInfo: { | 20260506 02:48:41.269 | INFO | "gpu" : " 17 GB", | 20260506 02:48:41.269 | INFO | "huge" : "", | 20260506 02:48:41.269 | INFO | "huge_shared" : " 27 GB", | 20260506 02:48:41.269 | INFO | "malloc" : " 14 GB", | 20260506 02:48:41.269 | INFO | "pinned" : " 29 GB" | 20260506 02:48:41.269 | INFO | } | 20260506 02:48:41.269 | INFO | Perf report for the last 5.13701 seconds: | 20260506 02:48:41.269 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:48:41.269 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:41.269 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:41.269 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:41.269 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260506 02:48:41.269 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:41.269 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:41.269 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 02:48:41.269 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:41.269 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:41.269 | INFO | Uploaded 36GiB to GPU in 2.07032s (17.3886GiB/s) | 20260506 02:48:41.269 | INFO | Downloaded 34GiB to GPU in 2.02317s (16.8053GiB/s) | 20260506 02:48:41.395 | INFO | Starting a round of Baz IO | 20260506 02:48:44.528 | INFO | Populating RTMetrics report | 20260506 02:48:44.528 | INFO | RTMetrics report populated | 20260506 02:48:44.528 | INFO | Opening tmp RTMetrics json file | 20260506 02:48:44.529 | INFO | Serializing RTMetrics to json | 20260506 02:48:44.530 | INFO | Serialization done, swapping tmp file now... | 20260506 02:48:44.530 | INFO | RTMetrics done | 20260506 02:48:44.603 | INFO | Finished a round of Baz IO: Wrote 1.781 GiB in 3.208 seconds | 20260506 02:48:44.603 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.25% / 0.00% / 78.75% | 20260506 02:48:44.603 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0661731% / 3.97364e-06% / 7.54992e-05% | 20260506 02:48:44.603 | INFO | Baz IO overall: | 20260506 02:48:44.603 | INFO | BytesWritten : 4667477252938 | 20260506 02:48:44.603 | INFO | NumEvents : 2190383036671 | 20260506 02:48:46.406 | INFO | Analyzing chunk frames = [8351744,8352256) | 20260506 02:48:46.406 | INFO | MemInfo: { | 20260506 02:48:46.406 | INFO | "gpu" : " 17 GB", | 20260506 02:48:46.406 | INFO | "huge" : "", | 20260506 02:48:46.406 | INFO | "huge_shared" : " 27 GB", | 20260506 02:48:46.406 | INFO | "malloc" : " 14 GB", | 20260506 02:48:46.406 | INFO | "pinned" : " 29 GB" | 20260506 02:48:46.406 | INFO | } | 20260506 02:48:46.406 | INFO | Perf report for the last 5.13693 seconds: | 20260506 02:48:46.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:48:46.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:46.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:46.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:46.406 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260506 02:48:46.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:46.406 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260506 02:48:46.406 | INFO | PRE_HQ : 59.5%, 1.7, 53.1ms, 0 | 20260506 02:48:46.406 | INFO | BAZWRITER : 61.7%, 3.5, 110.0ms, 0 | 20260506 02:48:46.406 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:48:46.406 | INFO | Uploaded 36GiB to GPU in 2.17999s (16.5138GiB/s) | 20260506 02:48:46.407 | INFO | Downloaded 38GiB to GPU in 2.24079s (16.9583GiB/s) | 20260506 02:48:51.549 | INFO | Analyzing chunk frames = [8352256,8352768) | 20260506 02:48:51.550 | INFO | MemInfo: { | 20260506 02:48:51.550 | INFO | "gpu" : " 17 GB", | 20260506 02:48:51.550 | INFO | "huge" : "", | 20260506 02:48:51.550 | INFO | "huge_shared" : " 27 GB", | 20260506 02:48:51.550 | INFO | "malloc" : " 14 GB", | 20260506 02:48:51.550 | INFO | "pinned" : " 29 GB" | 20260506 02:48:51.550 | INFO | } | 20260506 02:48:51.550 | INFO | Perf report for the last 5.14372 seconds: | 20260506 02:48:51.550 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:48:51.550 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:51.550 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:51.550 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:51.550 | INFO | ANALYSIS : 60.6%, 3.0, 94.7ms, 0 | 20260506 02:48:51.550 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:51.550 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:51.550 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 02:48:51.550 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:51.550 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:51.550 | INFO | Uploaded 36GiB to GPU in 1.97209s (18.2547GiB/s) | 20260506 02:48:51.550 | INFO | Downloaded 34GiB to GPU in 1.97649s (17.2022GiB/s) | 20260506 02:48:56.688 | INFO | Analyzing chunk frames = [8352768,8353280) | 20260506 02:48:56.688 | INFO | MemInfo: { | 20260506 02:48:56.688 | INFO | "gpu" : " 17 GB", | 20260506 02:48:56.688 | INFO | "huge" : "", | 20260506 02:48:56.688 | INFO | "huge_shared" : " 27 GB", | 20260506 02:48:56.688 | INFO | "malloc" : " 14 GB", | 20260506 02:48:56.689 | INFO | "pinned" : " 29 GB" | 20260506 02:48:56.689 | INFO | } | 20260506 02:48:56.689 | INFO | Perf report for the last 5.13884 seconds: | 20260506 02:48:56.689 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:48:56.689 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:56.689 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:56.689 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:56.689 | INFO | ANALYSIS : 61.2%, 3.0, 95.1ms, 0 | 20260506 02:48:56.689 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:56.689 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:56.689 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 02:48:56.689 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:56.689 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:48:56.689 | INFO | Uploaded 36GiB to GPU in 2.11052s (17.0574GiB/s) | 20260506 02:48:56.689 | INFO | Downloaded 34GiB to GPU in 2.04214s (16.6492GiB/s) | 20260506 02:49:01.823 | INFO | Analyzing chunk frames = [8353280,8353792) | 20260506 02:49:01.823 | INFO | MemInfo: { | 20260506 02:49:01.823 | INFO | "gpu" : " 17 GB", | 20260506 02:49:01.823 | INFO | "huge" : "", | 20260506 02:49:01.823 | INFO | "huge_shared" : " 27 GB", | 20260506 02:49:01.823 | INFO | "malloc" : " 14 GB", | 20260506 02:49:01.824 | INFO | "pinned" : " 29 GB" | 20260506 02:49:01.824 | INFO | } | 20260506 02:49:01.824 | INFO | Perf report for the last 5.13506 seconds: | 20260506 02:49:01.824 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:49:01.824 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:01.824 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:01.824 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:01.824 | INFO | ANALYSIS : 61.0%, 3.0, 94.7ms, 0 | 20260506 02:49:01.824 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:01.824 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:01.824 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 02:49:01.824 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:01.824 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:01.824 | INFO | Uploaded 36GiB to GPU in 2.08584s (17.2592GiB/s) | 20260506 02:49:01.824 | INFO | Downloaded 34GiB to GPU in 1.99995s (17.0004GiB/s) | 20260506 02:49:06.963 | INFO | Analyzing chunk frames = [8353792,8354304) | 20260506 02:49:06.963 | INFO | MemInfo: { | 20260506 02:49:06.963 | INFO | "gpu" : " 17 GB", | 20260506 02:49:06.963 | INFO | "huge" : "", | 20260506 02:49:06.963 | INFO | "huge_shared" : " 27 GB", | 20260506 02:49:06.963 | INFO | "malloc" : " 14 GB", | 20260506 02:49:06.963 | INFO | "pinned" : " 29 GB" | 20260506 02:49:06.963 | INFO | } | 20260506 02:49:06.963 | INFO | Perf report for the last 5.1397 seconds: | 20260506 02:49:06.963 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:49:06.963 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:06.963 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:06.963 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:06.963 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 02:49:06.964 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:06.964 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:06.964 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 02:49:06.964 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:06.964 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:06.964 | INFO | Uploaded 36GiB to GPU in 2.11549s (17.0173GiB/s) | 20260506 02:49:06.964 | INFO | Downloaded 34GiB to GPU in 2.0432s (16.6405GiB/s) | 20260506 02:49:12.106 | INFO | Analyzing chunk frames = [8354304,8354816) | 20260506 02:49:12.106 | INFO | MemInfo: { | 20260506 02:49:12.106 | INFO | "gpu" : " 17 GB", | 20260506 02:49:12.106 | INFO | "huge" : "", | 20260506 02:49:12.106 | INFO | "huge_shared" : " 27 GB", | 20260506 02:49:12.106 | INFO | "malloc" : " 14 GB", | 20260506 02:49:12.106 | INFO | "pinned" : " 29 GB" | 20260506 02:49:12.106 | INFO | } | 20260506 02:49:12.106 | INFO | Perf report for the last 5.14257 seconds: | 20260506 02:49:12.106 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:49:12.106 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:12.106 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:12.106 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:12.106 | INFO | ANALYSIS : 60.9%, 3.0, 95.2ms, 0 | 20260506 02:49:12.106 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:12.106 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:12.106 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 02:49:12.106 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:12.106 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:12.106 | INFO | Uploaded 36GiB to GPU in 2.09579s (17.1773GiB/s) | 20260506 02:49:12.106 | INFO | Downloaded 34GiB to GPU in 2.03465s (16.7105GiB/s) | 20260506 02:49:17.253 | INFO | Analyzing chunk frames = [8354816,8355328) | 20260506 02:49:17.253 | INFO | MemInfo: { | 20260506 02:49:17.253 | INFO | "gpu" : " 17 GB", | 20260506 02:49:17.253 | INFO | "huge" : "", | 20260506 02:49:17.253 | INFO | "huge_shared" : " 27 GB", | 20260506 02:49:17.253 | INFO | "malloc" : " 14 GB", | 20260506 02:49:17.253 | INFO | "pinned" : " 29 GB" | 20260506 02:49:17.253 | INFO | } | 20260506 02:49:17.253 | INFO | Perf report for the last 5.14702 seconds: | 20260506 02:49:17.253 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:49:17.253 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:17.253 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:17.253 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:17.253 | INFO | ANALYSIS : 61.7%, 3.0, 96.5ms, 0 | 20260506 02:49:17.253 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:17.253 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:17.253 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 02:49:17.253 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:17.253 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:17.253 | INFO | Uploaded 36GiB to GPU in 2.08209s (17.2904GiB/s) | 20260506 02:49:17.253 | INFO | Downloaded 34GiB to GPU in 2.06413s (16.4718GiB/s) | 20260506 02:49:22.382 | INFO | Analyzing chunk frames = [8355328,8355840) | 20260506 02:49:22.382 | INFO | MemInfo: { | 20260506 02:49:22.382 | INFO | "gpu" : " 17 GB", | 20260506 02:49:22.382 | INFO | "huge" : "", | 20260506 02:49:22.382 | INFO | "huge_shared" : " 27 GB", | 20260506 02:49:22.382 | INFO | "malloc" : " 14 GB", | 20260506 02:49:22.382 | INFO | "pinned" : " 29 GB" | 20260506 02:49:22.382 | INFO | } | 20260506 02:49:22.382 | INFO | Perf report for the last 5.12939 seconds: | 20260506 02:49:22.383 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:49:22.383 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:22.383 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:22.383 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:22.383 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 02:49:22.383 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:22.383 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:22.383 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 02:49:22.383 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:22.383 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:22.383 | INFO | Uploaded 36GiB to GPU in 2.03755s (17.6682GiB/s) | 20260506 02:49:22.383 | INFO | Downloaded 34GiB to GPU in 2.03236s (16.7294GiB/s) | 20260506 02:49:22.383 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 02:49:22.383 | INFO | Total mem usage: 15.4 GiB | 20260506 02:49:22.383 | INFO | All Time High Water Marks sum: 16.3 GiB (Savings: 5.2%) | 20260506 02:49:22.383 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.98%) | 20260506 02:49:22.383 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 02:49:22.383 | INFO | High Water Breakdown: | 20260506 02:49:22.383 | INFO | 60.9% : 8.54 GiB ( 88.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 02:49:22.383 | INFO | 39% : 5.47 GiB ( 82.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 02:49:22.383 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 02:49:22.383 | INFO | ------------------------------------------ | 20260506 02:49:22.383 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:49:22.383 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 02:49:22.383 | INFO | Total mem usage: 26.5 GiB | 20260506 02:49:22.383 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 02:49:22.383 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 02:49:22.383 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 02:49:22.383 | INFO | High Water Breakdown: | 20260506 02:49:22.383 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:49:22.383 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:49:22.383 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:49:22.383 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:49:22.383 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:49:22.383 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:49:22.383 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 02:49:22.383 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 02:49:22.383 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:49:22.383 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 02:49:22.383 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 02:49:22.383 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 02:49:22.383 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 02:49:22.383 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 02:49:22.383 | INFO | ------------------------------------------ | 20260506 02:49:22.383 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:49:22.383 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 02:49:22.383 | INFO | Total mem usage: 25.6 GiB | 20260506 02:49:22.383 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 02:49:22.383 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 02:49:22.383 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 02:49:22.383 | INFO | High Water Breakdown: | 20260506 02:49:22.383 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 02:49:22.383 | INFO | ------------------------------------------ | 20260506 02:49:22.384 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:49:22.384 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 02:49:22.384 | INFO | Total mem usage: 15.7 GiB | 20260506 02:49:22.384 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 02:49:22.384 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 02:49:22.384 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 02:49:22.384 | INFO | High Water Breakdown: | 20260506 02:49:22.384 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 02:49:22.384 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 02:49:22.384 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 02:49:22.384 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 02:49:22.384 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 02:49:22.384 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 02:49:22.384 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 02:49:22.384 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 02:49:22.384 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 02:49:22.384 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:49:22.384 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 02:49:22.384 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:49:22.384 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 02:49:22.384 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 02:49:22.384 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 02:49:22.384 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 02:49:22.384 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:49:22.384 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 02:49:22.384 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:49:22.384 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:49:22.384 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:49:22.384 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 02:49:22.384 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 02:49:22.384 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:49:22.384 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 02:49:22.384 | INFO | ------------------------------------------ | 20260506 02:49:22.384 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:49:22.384 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 02:49:22.384 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:49:22.384 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:49:22.384 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.38e+01, 1.33e+01 / 8.30e-01 | 20260506 02:49:22.384 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.04e-01, 1.04e-02 / 3.43e-03 | 20260506 02:49:22.384 | INFO | PerfReport: 11.86%, Baseline -- 3.71e+00 / 4.52e+00, 5.82e-01 / 2.43e-01 | 20260506 02:49:22.384 | INFO | PerfReport: 3.31%, Histogram -- 1.03e+00 / 1.53e+00, 1.95e-01 / 8.92e-02 | 20260506 02:49:22.384 | INFO | PerfReport: 15.98%, DME -- 4.00e+01 / 4.39e+01, 3.84e+00 / 1.00e+00 | 20260506 02:49:22.384 | INFO | PerfReport: 57.55%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.56e+00 / 1.06e+00 | 20260506 02:49:22.384 | INFO | PerfReport: 6.99%, PulseAccumulating -- 2.19e+00 / 2.71e+00, 4.13e-01 / 2.29e-01 | 20260506 02:49:22.384 | INFO | PerfReport: 2.99%, Metrics -- 9.36e-01 / 1.21e+00, 2.49e-01 / 1.23e-01 | 20260506 02:49:22.384 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.23e-01 / 3.66e-01, 5.48e-02 / 3.03e-02 | 20260506 02:49:22.384 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 02:49:22.384 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:49:22.384 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:49:22.384 | INFO | PerfReport: Full Execution -- 9.75e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260506 02:49:22.384 | INFO | PerfReport: 22.18%, Upload -- 2.16e+01 / 0.00e+00, 7.10e+00 / 0.00e+00 | 20260506 02:49:22.384 | INFO | PerfReport: 21.91%, Download -- 2.14e+01 / 0.00e+00, 7.51e+00 / 0.00e+00 | 20260506 02:49:22.384 | INFO | PerfReport: 32.10%, Compute -- 3.13e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 02:49:22.384 | INFO | PerfReport: 1.97%, LockUpload -- 1.92e+00 / 0.00e+00, 7.57e+00 / 0.00e+00 | 20260506 02:49:22.384 | INFO | PerfReport: 1.27%, LockDownload -- 1.24e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260506 02:49:22.384 | INFO | PerfReport: 20.54%, LockCompute -- 2.00e+01 / 0.00e+00, 1.76e+01 / 0.00e+00 | 20260506 02:49:22.384 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.77e-02 / 0.00e+00 | 20260506 02:49:22.508 | INFO | Starting a round of Baz IO | 20260506 02:49:25.634 | INFO | Populating RTMetrics report | 20260506 02:49:25.634 | INFO | RTMetrics report populated | 20260506 02:49:25.634 | INFO | Opening tmp RTMetrics json file | 20260506 02:49:25.635 | INFO | Serializing RTMetrics to json | 20260506 02:49:25.636 | INFO | Serialization done, swapping tmp file now... | 20260506 02:49:25.636 | INFO | RTMetrics done | 20260506 02:49:25.711 | INFO | Finished a round of Baz IO: Wrote 1.779 GiB in 3.203 seconds | 20260506 02:49:25.711 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.25% / 0.00% / 78.75% | 20260506 02:49:25.711 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0664075% / 0% / 9.13938e-05% | 20260506 02:49:25.711 | INFO | Baz IO overall: | 20260506 02:49:25.711 | INFO | BytesWritten : 4669386942396 | 20260506 02:49:25.711 | INFO | NumEvents : 2191249685674 | 20260506 02:49:27.520 | INFO | Analyzing chunk frames = [8355840,8356352) | 20260506 02:49:27.520 | INFO | MemInfo: { | 20260506 02:49:27.520 | INFO | "gpu" : " 17 GB", | 20260506 02:49:27.520 | INFO | "huge" : "", | 20260506 02:49:27.520 | INFO | "huge_shared" : " 27 GB", | 20260506 02:49:27.520 | INFO | "malloc" : " 14 GB", | 20260506 02:49:27.520 | INFO | "pinned" : " 29 GB" | 20260506 02:49:27.520 | INFO | } | 20260506 02:49:27.520 | INFO | Perf report for the last 5.13797 seconds: | 20260506 02:49:27.521 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:49:27.521 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:27.521 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:27.521 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:27.521 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260506 02:49:27.521 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:27.521 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260506 02:49:27.521 | INFO | PRE_HQ : 59.1%, 1.7, 53.2ms, 0 | 20260506 02:49:27.521 | INFO | BAZWRITER : 61.5%, 3.4, 108.9ms, 0 | 20260506 02:49:27.521 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:49:27.521 | INFO | Uploaded 36GiB to GPU in 2.10874s (17.0718GiB/s) | 20260506 02:49:27.521 | INFO | Downloaded 38GiB to GPU in 2.21553s (17.1517GiB/s) | 20260506 02:49:32.661 | INFO | Analyzing chunk frames = [8356352,8356864) | 20260506 02:49:32.661 | INFO | MemInfo: { | 20260506 02:49:32.661 | INFO | "gpu" : " 17 GB", | 20260506 02:49:32.661 | INFO | "huge" : "", | 20260506 02:49:32.661 | INFO | "huge_shared" : " 27 GB", | 20260506 02:49:32.661 | INFO | "malloc" : " 14 GB", | 20260506 02:49:32.661 | INFO | "pinned" : " 29 GB" | 20260506 02:49:32.661 | INFO | } | 20260506 02:49:32.661 | INFO | Perf report for the last 5.14098 seconds: | 20260506 02:49:32.661 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:49:32.661 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:32.661 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:32.661 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:32.661 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260506 02:49:32.662 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:32.662 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:32.662 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 02:49:32.662 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:32.662 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:32.662 | INFO | Uploaded 36GiB to GPU in 2.12127s (16.9709GiB/s) | 20260506 02:49:32.662 | INFO | Downloaded 34GiB to GPU in 2.0416s (16.6536GiB/s) | 20260506 02:49:37.800 | INFO | Analyzing chunk frames = [8356864,8357376) | 20260506 02:49:37.800 | INFO | MemInfo: { | 20260506 02:49:37.800 | INFO | "gpu" : " 17 GB", | 20260506 02:49:37.800 | INFO | "huge" : "", | 20260506 02:49:37.800 | INFO | "huge_shared" : " 27 GB", | 20260506 02:49:37.800 | INFO | "malloc" : " 14 GB", | 20260506 02:49:37.800 | INFO | "pinned" : " 29 GB" | 20260506 02:49:37.800 | INFO | } | 20260506 02:49:37.800 | INFO | Perf report for the last 5.13868 seconds: | 20260506 02:49:37.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:49:37.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:37.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:37.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:37.800 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 02:49:37.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:37.800 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:37.800 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 02:49:37.800 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:37.800 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:37.800 | INFO | Uploaded 36GiB to GPU in 2.09551s (17.1796GiB/s) | 20260506 02:49:37.800 | INFO | Downloaded 34GiB to GPU in 2.02093s (16.8239GiB/s) | 20260506 02:49:42.940 | INFO | Analyzing chunk frames = [8357376,8357888) | 20260506 02:49:42.940 | INFO | MemInfo: { | 20260506 02:49:42.940 | INFO | "gpu" : " 17 GB", | 20260506 02:49:42.940 | INFO | "huge" : "", | 20260506 02:49:42.940 | INFO | "huge_shared" : " 27 GB", | 20260506 02:49:42.940 | INFO | "malloc" : " 14 GB", | 20260506 02:49:42.940 | INFO | "pinned" : " 29 GB" | 20260506 02:49:42.940 | INFO | } | 20260506 02:49:42.940 | INFO | Perf report for the last 5.13983 seconds: | 20260506 02:49:42.940 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:49:42.940 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:42.940 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:42.940 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:42.940 | INFO | ANALYSIS : 61.0%, 3.0, 94.7ms, 0 | 20260506 02:49:42.940 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:42.940 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:42.940 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:49:42.940 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:42.940 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:42.940 | INFO | Uploaded 36GiB to GPU in 2.07825s (17.3223GiB/s) | 20260506 02:49:42.940 | INFO | Downloaded 34GiB to GPU in 2.00719s (16.9391GiB/s) | 20260506 02:49:48.083 | INFO | Analyzing chunk frames = [8357888,8358400) | 20260506 02:49:48.083 | INFO | MemInfo: { | 20260506 02:49:48.083 | INFO | "gpu" : " 17 GB", | 20260506 02:49:48.083 | INFO | "huge" : "", | 20260506 02:49:48.083 | INFO | "huge_shared" : " 27 GB", | 20260506 02:49:48.083 | INFO | "malloc" : " 14 GB", | 20260506 02:49:48.083 | INFO | "pinned" : " 29 GB" | 20260506 02:49:48.083 | INFO | } | 20260506 02:49:48.084 | INFO | Perf report for the last 5.14357 seconds: | 20260506 02:49:48.084 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:49:48.084 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:48.084 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:48.084 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:48.084 | INFO | ANALYSIS : 60.7%, 3.0, 94.8ms, 0 | 20260506 02:49:48.084 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:48.084 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:48.084 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 02:49:48.084 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:48.084 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:48.084 | INFO | Uploaded 36GiB to GPU in 2.09293s (17.2008GiB/s) | 20260506 02:49:48.084 | INFO | Downloaded 34GiB to GPU in 2.04176s (16.6523GiB/s) | 20260506 02:49:53.221 | INFO | Analyzing chunk frames = [8358400,8358912) | 20260506 02:49:53.221 | INFO | MemInfo: { | 20260506 02:49:53.221 | INFO | "gpu" : " 17 GB", | 20260506 02:49:53.221 | INFO | "huge" : "", | 20260506 02:49:53.221 | INFO | "huge_shared" : " 27 GB", | 20260506 02:49:53.221 | INFO | "malloc" : " 14 GB", | 20260506 02:49:53.221 | INFO | "pinned" : " 29 GB" | 20260506 02:49:53.221 | INFO | } | 20260506 02:49:53.221 | INFO | Perf report for the last 5.13753 seconds: | 20260506 02:49:53.221 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:49:53.221 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:53.221 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:53.221 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:53.221 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 02:49:53.221 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:53.221 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:53.221 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 02:49:53.221 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:53.221 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:53.221 | INFO | Uploaded 36GiB to GPU in 2.09952s (17.1468GiB/s) | 20260506 02:49:53.221 | INFO | Downloaded 34GiB to GPU in 2.03591s (16.7002GiB/s) | 20260506 02:49:58.359 | INFO | Analyzing chunk frames = [8358912,8359424) | 20260506 02:49:58.359 | INFO | MemInfo: { | 20260506 02:49:58.359 | INFO | "gpu" : " 17 GB", | 20260506 02:49:58.359 | INFO | "huge" : "", | 20260506 02:49:58.359 | INFO | "huge_shared" : " 27 GB", | 20260506 02:49:58.359 | INFO | "malloc" : " 14 GB", | 20260506 02:49:58.359 | INFO | "pinned" : " 29 GB" | 20260506 02:49:58.359 | INFO | } | 20260506 02:49:58.359 | INFO | Perf report for the last 5.13825 seconds: | 20260506 02:49:58.359 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:49:58.359 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:58.359 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:58.359 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:58.359 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260506 02:49:58.359 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:58.359 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:58.359 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 02:49:58.359 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:58.360 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:49:58.360 | INFO | Uploaded 36GiB to GPU in 2.04735s (17.5837GiB/s) | 20260506 02:49:58.360 | INFO | Downloaded 34GiB to GPU in 2.07572s (16.3799GiB/s) | 20260506 02:50:03.498 | INFO | Analyzing chunk frames = [8359424,8359936) | 20260506 02:50:03.498 | INFO | MemInfo: { | 20260506 02:50:03.498 | INFO | "gpu" : " 17 GB", | 20260506 02:50:03.498 | INFO | "huge" : "", | 20260506 02:50:03.498 | INFO | "huge_shared" : " 27 GB", | 20260506 02:50:03.498 | INFO | "malloc" : " 14 GB", | 20260506 02:50:03.498 | INFO | "pinned" : " 29 GB" | 20260506 02:50:03.498 | INFO | } | 20260506 02:50:03.498 | INFO | Perf report for the last 5.13908 seconds: | 20260506 02:50:03.498 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:50:03.498 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:03.498 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:03.498 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:03.498 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 02:50:03.498 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:03.499 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:03.499 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 02:50:03.499 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:03.499 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:03.499 | INFO | Uploaded 36GiB to GPU in 2.04189s (17.6307GiB/s) | 20260506 02:50:03.499 | INFO | Downloaded 34GiB to GPU in 2.01165s (16.9016GiB/s) | 20260506 02:50:03.624 | INFO | Starting a round of Baz IO | 20260506 02:50:06.747 | INFO | Populating RTMetrics report | 20260506 02:50:06.748 | INFO | RTMetrics report populated | 20260506 02:50:06.748 | INFO | Opening tmp RTMetrics json file | 20260506 02:50:06.748 | INFO | Serializing RTMetrics to json | 20260506 02:50:06.749 | INFO | Serialization done, swapping tmp file now... | 20260506 02:50:06.750 | INFO | RTMetrics done | 20260506 02:50:06.821 | INFO | Finished a round of Baz IO: Wrote 1.78 GiB in 3.197 seconds | 20260506 02:50:06.822 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.25% / 0.00% / 78.75% | 20260506 02:50:06.822 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0664552% / 0% / 5.16574e-05% | 20260506 02:50:06.822 | INFO | Baz IO overall: | 20260506 02:50:06.822 | INFO | BytesWritten : 4671297884650 | 20260506 02:50:06.822 | INFO | NumEvents : 2192117536918 | 20260506 02:50:08.640 | INFO | Analyzing chunk frames = [8359936,8360448) | 20260506 02:50:08.640 | INFO | MemInfo: { | 20260506 02:50:08.640 | INFO | "gpu" : " 17 GB", | 20260506 02:50:08.640 | INFO | "huge" : "", | 20260506 02:50:08.640 | INFO | "huge_shared" : " 27 GB", | 20260506 02:50:08.640 | INFO | "malloc" : " 14 GB", | 20260506 02:50:08.640 | INFO | "pinned" : " 29 GB" | 20260506 02:50:08.640 | INFO | } | 20260506 02:50:08.640 | INFO | Perf report for the last 5.14142 seconds: | 20260506 02:50:08.640 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:50:08.640 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:08.640 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:08.640 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:08.640 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260506 02:50:08.640 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:08.640 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260506 02:50:08.640 | INFO | PRE_HQ : 58.9%, 1.7, 53.3ms, 0 | 20260506 02:50:08.640 | INFO | BAZWRITER : 61.4%, 3.7, 117.6ms, 0 | 20260506 02:50:08.640 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:50:08.640 | INFO | Uploaded 36GiB to GPU in 2.09567s (17.1783GiB/s) | 20260506 02:50:08.640 | INFO | Downloaded 38GiB to GPU in 2.21632s (17.1455GiB/s) | 20260506 02:50:13.773 | INFO | Analyzing chunk frames = [8360448,8360960) | 20260506 02:50:13.773 | INFO | MemInfo: { | 20260506 02:50:13.773 | INFO | "gpu" : " 17 GB", | 20260506 02:50:13.773 | INFO | "huge" : "", | 20260506 02:50:13.773 | INFO | "huge_shared" : " 27 GB", | 20260506 02:50:13.773 | INFO | "malloc" : " 14 GB", | 20260506 02:50:13.773 | INFO | "pinned" : " 29 GB" | 20260506 02:50:13.773 | INFO | } | 20260506 02:50:13.773 | INFO | Perf report for the last 5.13325 seconds: | 20260506 02:50:13.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:50:13.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:13.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:13.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:13.773 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260506 02:50:13.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:13.773 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:13.773 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 02:50:13.773 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:13.773 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:13.773 | INFO | Uploaded 36GiB to GPU in 1.9264s (18.6877GiB/s) | 20260506 02:50:13.773 | INFO | Downloaded 34GiB to GPU in 1.97973s (17.174GiB/s) | 20260506 02:50:18.916 | INFO | Analyzing chunk frames = [8360960,8361472) | 20260506 02:50:18.916 | INFO | MemInfo: { | 20260506 02:50:18.916 | INFO | "gpu" : " 17 GB", | 20260506 02:50:18.916 | INFO | "huge" : "", | 20260506 02:50:18.916 | INFO | "huge_shared" : " 27 GB", | 20260506 02:50:18.916 | INFO | "malloc" : " 14 GB", | 20260506 02:50:18.916 | INFO | "pinned" : " 29 GB" | 20260506 02:50:18.916 | INFO | } | 20260506 02:50:18.916 | INFO | Perf report for the last 5.14334 seconds: | 20260506 02:50:18.917 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:50:18.917 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:18.917 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:18.917 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:18.917 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 02:50:18.917 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:18.917 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:18.917 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:50:18.917 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:18.917 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:18.917 | INFO | Uploaded 36GiB to GPU in 2.07977s (17.3096GiB/s) | 20260506 02:50:18.917 | INFO | Downloaded 34GiB to GPU in 2.01563s (16.8682GiB/s) | 20260506 02:50:24.060 | INFO | Analyzing chunk frames = [8361472,8361984) | 20260506 02:50:24.060 | INFO | MemInfo: { | 20260506 02:50:24.060 | INFO | "gpu" : " 17 GB", | 20260506 02:50:24.060 | INFO | "huge" : "", | 20260506 02:50:24.060 | INFO | "huge_shared" : " 27 GB", | 20260506 02:50:24.060 | INFO | "malloc" : " 14 GB", | 20260506 02:50:24.060 | INFO | "pinned" : " 29 GB" | 20260506 02:50:24.060 | INFO | } | 20260506 02:50:24.060 | INFO | Perf report for the last 5.14354 seconds: | 20260506 02:50:24.060 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:50:24.060 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:24.060 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:24.060 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:24.060 | INFO | ANALYSIS : 61.0%, 3.0, 94.8ms, 0 | 20260506 02:50:24.060 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:24.060 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:24.060 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 02:50:24.060 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:24.060 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:24.060 | INFO | Uploaded 36GiB to GPU in 2.07286s (17.3673GiB/s) | 20260506 02:50:24.060 | INFO | Downloaded 34GiB to GPU in 2.00385s (16.9674GiB/s) | 20260506 02:50:29.203 | INFO | Analyzing chunk frames = [8361984,8362496) | 20260506 02:50:29.203 | INFO | MemInfo: { | 20260506 02:50:29.203 | INFO | "gpu" : " 17 GB", | 20260506 02:50:29.203 | INFO | "huge" : "", | 20260506 02:50:29.203 | INFO | "huge_shared" : " 27 GB", | 20260506 02:50:29.203 | INFO | "malloc" : " 14 GB", | 20260506 02:50:29.203 | INFO | "pinned" : " 29 GB" | 20260506 02:50:29.203 | INFO | } | 20260506 02:50:29.203 | INFO | Perf report for the last 5.14346 seconds: | 20260506 02:50:29.204 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:50:29.204 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:29.204 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:29.204 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:29.204 | INFO | ANALYSIS : 60.7%, 3.0, 94.9ms, 0 | 20260506 02:50:29.204 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:29.204 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:29.204 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:50:29.204 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:29.204 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:29.204 | INFO | Uploaded 36GiB to GPU in 2.12473s (16.9433GiB/s) | 20260506 02:50:29.204 | INFO | Downloaded 34GiB to GPU in 2.03738s (16.6881GiB/s) | 20260506 02:50:34.334 | INFO | Analyzing chunk frames = [8362496,8363008) | 20260506 02:50:34.334 | INFO | MemInfo: { | 20260506 02:50:34.334 | INFO | "gpu" : " 17 GB", | 20260506 02:50:34.334 | INFO | "huge" : "", | 20260506 02:50:34.334 | INFO | "huge_shared" : " 27 GB", | 20260506 02:50:34.334 | INFO | "malloc" : " 14 GB", | 20260506 02:50:34.334 | INFO | "pinned" : " 29 GB" | 20260506 02:50:34.334 | INFO | } | 20260506 02:50:34.334 | INFO | Perf report for the last 5.13075 seconds: | 20260506 02:50:34.334 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:50:34.334 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:34.334 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:34.334 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:34.334 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260506 02:50:34.334 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:34.334 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:34.335 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:50:34.335 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:34.335 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:34.335 | INFO | Uploaded 36GiB to GPU in 2.05589s (17.5107GiB/s) | 20260506 02:50:34.335 | INFO | Downloaded 34GiB to GPU in 2.02259s (16.8102GiB/s) | 20260506 02:50:39.472 | INFO | Analyzing chunk frames = [8363008,8363520) | 20260506 02:50:39.472 | INFO | MemInfo: { | 20260506 02:50:39.472 | INFO | "gpu" : " 17 GB", | 20260506 02:50:39.472 | INFO | "huge" : "", | 20260506 02:50:39.472 | INFO | "huge_shared" : " 27 GB", | 20260506 02:50:39.472 | INFO | "malloc" : " 14 GB", | 20260506 02:50:39.472 | INFO | "pinned" : " 29 GB" | 20260506 02:50:39.472 | INFO | } | 20260506 02:50:39.472 | INFO | Perf report for the last 5.13744 seconds: | 20260506 02:50:39.472 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:50:39.472 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:39.472 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:39.472 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:39.472 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260506 02:50:39.472 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:39.472 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:39.472 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 02:50:39.472 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:39.472 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:39.472 | INFO | Uploaded 36GiB to GPU in 2.04945s (17.5656GiB/s) | 20260506 02:50:39.472 | INFO | Downloaded 34GiB to GPU in 2.0503s (16.5829GiB/s) | 20260506 02:50:44.614 | INFO | Analyzing chunk frames = [8363520,8364032) | 20260506 02:50:44.614 | INFO | MemInfo: { | 20260506 02:50:44.614 | INFO | "gpu" : " 17 GB", | 20260506 02:50:44.614 | INFO | "huge" : "", | 20260506 02:50:44.614 | INFO | "huge_shared" : " 27 GB", | 20260506 02:50:44.614 | INFO | "malloc" : " 14 GB", | 20260506 02:50:44.614 | INFO | "pinned" : " 29 GB" | 20260506 02:50:44.614 | INFO | } | 20260506 02:50:44.614 | INFO | Perf report for the last 5.14257 seconds: | 20260506 02:50:44.614 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:50:44.614 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:44.614 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:44.614 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:44.614 | INFO | ANALYSIS : 61.8%, 3.0, 96.6ms, 0 | 20260506 02:50:44.614 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:44.614 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:44.614 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:50:44.614 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:44.614 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:44.615 | INFO | Uploaded 36GiB to GPU in 2.13602s (16.8538GiB/s) | 20260506 02:50:44.615 | INFO | Downloaded 34GiB to GPU in 2.03012s (16.7478GiB/s) | 20260506 02:50:44.615 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 02:50:44.615 | INFO | Total mem usage: 15.4 GiB | 20260506 02:50:44.615 | INFO | All Time High Water Marks sum: 16.3 GiB (Savings: 5.2%) | 20260506 02:50:44.615 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -6.56%) | 20260506 02:50:44.615 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 02:50:44.615 | INFO | High Water Breakdown: | 20260506 02:50:44.615 | INFO | 61.4% : 8.89 GiB ( 92.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 02:50:44.615 | INFO | 38.5% : 5.57 GiB ( 83.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 02:50:44.615 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 02:50:44.615 | INFO | ------------------------------------------ | 20260506 02:50:44.615 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:50:44.615 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 02:50:44.615 | INFO | Total mem usage: 26.5 GiB | 20260506 02:50:44.615 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 02:50:44.615 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 02:50:44.615 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 02:50:44.615 | INFO | High Water Breakdown: | 20260506 02:50:44.615 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:50:44.615 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:50:44.615 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:50:44.615 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:50:44.615 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:50:44.615 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:50:44.615 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 02:50:44.615 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 02:50:44.615 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:50:44.615 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 02:50:44.615 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 02:50:44.615 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 02:50:44.615 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 02:50:44.616 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 02:50:44.616 | INFO | ------------------------------------------ | 20260506 02:50:44.616 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:50:44.616 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 02:50:44.616 | INFO | Total mem usage: 25.6 GiB | 20260506 02:50:44.616 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 02:50:44.616 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 02:50:44.616 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 02:50:44.616 | INFO | High Water Breakdown: | 20260506 02:50:44.616 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 02:50:44.616 | INFO | ------------------------------------------ | 20260506 02:50:44.616 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:50:44.616 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 02:50:44.616 | INFO | Total mem usage: 15.7 GiB | 20260506 02:50:44.616 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 02:50:44.616 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 02:50:44.616 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 02:50:44.616 | INFO | High Water Breakdown: | 20260506 02:50:44.616 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 02:50:44.616 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 02:50:44.616 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 02:50:44.616 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 02:50:44.616 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 02:50:44.616 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 02:50:44.616 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 02:50:44.616 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 02:50:44.616 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 02:50:44.616 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:50:44.616 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 02:50:44.616 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:50:44.616 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 02:50:44.616 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 02:50:44.616 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 02:50:44.616 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 02:50:44.616 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:50:44.616 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 02:50:44.616 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:50:44.616 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:50:44.616 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:50:44.616 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 02:50:44.616 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 02:50:44.616 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:50:44.616 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 02:50:44.616 | INFO | ------------------------------------------ | 20260506 02:50:44.616 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:50:44.616 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 02:50:44.616 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:50:44.616 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:50:44.616 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.38e+01, 1.34e+01 / 9.31e-01 | 20260506 02:50:44.616 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.06e-01, 1.22e-02 / 2.19e-02 | 20260506 02:50:44.616 | INFO | PerfReport: 11.88%, Baseline -- 3.72e+00 / 4.50e+00, 5.87e-01 / 2.41e-01 | 20260506 02:50:44.616 | INFO | PerfReport: 3.30%, Histogram -- 1.03e+00 / 1.51e+00, 1.95e-01 / 1.13e-01 | 20260506 02:50:44.616 | INFO | PerfReport: 16.05%, DME -- 4.02e+01 / 4.43e+01, 3.81e+00 / 1.20e+00 | 20260506 02:50:44.616 | INFO | PerfReport: 57.45%, FrameLabeling -- 1.80e+01 / 1.91e+01, 1.51e+00 / 7.67e-01 | 20260506 02:50:44.616 | INFO | PerfReport: 7.00%, PulseAccumulating -- 2.19e+00 / 2.78e+00, 4.15e-01 / 1.39e-01 | 20260506 02:50:44.616 | INFO | PerfReport: 3.00%, Metrics -- 9.37e-01 / 1.21e+00, 2.53e-01 / 1.25e-01 | 20260506 02:50:44.616 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.24e-01 / 3.77e-01, 5.54e-02 / 3.17e-02 | 20260506 02:50:44.616 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 02:50:44.616 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:50:44.616 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:50:44.616 | INFO | PerfReport: Full Execution -- 9.76e+01 / -nan, 2.41e+01 / 0.00e+00 | 20260506 02:50:44.616 | INFO | PerfReport: 22.15%, Upload -- 2.16e+01 / 0.00e+00, 7.12e+00 / 0.00e+00 | 20260506 02:50:44.616 | INFO | PerfReport: 21.88%, Download -- 2.14e+01 / 0.00e+00, 7.57e+00 / 0.00e+00 | 20260506 02:50:44.616 | INFO | PerfReport: 32.04%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 02:50:44.616 | INFO | PerfReport: 2.05%, LockUpload -- 2.00e+00 / 0.00e+00, 7.94e+00 / 0.00e+00 | 20260506 02:50:44.616 | INFO | PerfReport: 1.34%, LockDownload -- 1.31e+00 / 0.00e+00, 3.05e+00 / 0.00e+00 | 20260506 02:50:44.616 | INFO | PerfReport: 20.53%, LockCompute -- 2.00e+01 / 0.00e+00, 1.77e+01 / 0.00e+00 | 20260506 02:50:44.616 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.82e-02 / 0.00e+00 | 20260506 02:50:44.739 | INFO | Starting a round of Baz IO | 20260506 02:50:47.872 | INFO | Populating RTMetrics report | 20260506 02:50:47.872 | INFO | RTMetrics report populated | 20260506 02:50:47.872 | INFO | Opening tmp RTMetrics json file | 20260506 02:50:47.873 | INFO | Serializing RTMetrics to json | 20260506 02:50:47.874 | INFO | Serialization done, swapping tmp file now... | 20260506 02:50:47.875 | INFO | RTMetrics done | 20260506 02:50:47.946 | INFO | Finished a round of Baz IO: Wrote 1.78 GiB in 3.207 seconds | 20260506 02:50:47.946 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.25% / 0.00% / 78.75% | 20260506 02:50:47.946 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0663916% / 0% / 8.34465e-05% | 20260506 02:50:47.946 | INFO | Baz IO overall: | 20260506 02:50:47.946 | INFO | BytesWritten : 4673209160213 | 20260506 02:50:47.946 | INFO | NumEvents : 2192985444663 | 20260506 02:50:49.753 | INFO | Analyzing chunk frames = [8364032,8364544) | 20260506 02:50:49.754 | INFO | MemInfo: { | 20260506 02:50:49.754 | INFO | "gpu" : " 17 GB", | 20260506 02:50:49.754 | INFO | "huge" : "", | 20260506 02:50:49.754 | INFO | "huge_shared" : " 27 GB", | 20260506 02:50:49.754 | INFO | "malloc" : " 14 GB", | 20260506 02:50:49.754 | INFO | "pinned" : " 29 GB" | 20260506 02:50:49.754 | INFO | } | 20260506 02:50:49.754 | INFO | Perf report for the last 5.13927 seconds: | 20260506 02:50:49.754 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:50:49.754 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:49.754 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:49.754 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:49.754 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260506 02:50:49.754 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:49.754 | INFO | RT_METRICS : 42.6%, 1.0, 22.4ms, 0 | 20260506 02:50:49.754 | INFO | PRE_HQ : 59.1%, 1.7, 53.2ms, 0 | 20260506 02:50:49.754 | INFO | BAZWRITER : 61.6%, 3.4, 108.2ms, 0 | 20260506 02:50:49.754 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:50:49.754 | INFO | Uploaded 36GiB to GPU in 2.11711s (17.0043GiB/s) | 20260506 02:50:49.754 | INFO | Downloaded 38GiB to GPU in 2.22696s (17.0636GiB/s) | 20260506 02:50:54.894 | INFO | Analyzing chunk frames = [8364544,8365056) | 20260506 02:50:54.894 | INFO | MemInfo: { | 20260506 02:50:54.894 | INFO | "gpu" : " 17 GB", | 20260506 02:50:54.894 | INFO | "huge" : "", | 20260506 02:50:54.894 | INFO | "huge_shared" : " 27 GB", | 20260506 02:50:54.894 | INFO | "malloc" : " 14 GB", | 20260506 02:50:54.894 | INFO | "pinned" : " 29 GB" | 20260506 02:50:54.894 | INFO | } | 20260506 02:50:54.894 | INFO | Perf report for the last 5.14014 seconds: | 20260506 02:50:54.894 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:50:54.894 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:54.894 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:54.894 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:54.894 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260506 02:50:54.894 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:54.894 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:54.894 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 02:50:54.894 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:54.894 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:50:54.894 | INFO | Uploaded 36GiB to GPU in 1.9381s (18.5749GiB/s) | 20260506 02:50:54.894 | INFO | Downloaded 34GiB to GPU in 1.9835s (17.1414GiB/s) | 20260506 02:51:00.031 | INFO | Analyzing chunk frames = [8365056,8365568) | 20260506 02:51:00.031 | INFO | MemInfo: { | 20260506 02:51:00.031 | INFO | "gpu" : " 17 GB", | 20260506 02:51:00.031 | INFO | "huge" : "", | 20260506 02:51:00.031 | INFO | "huge_shared" : " 27 GB", | 20260506 02:51:00.031 | INFO | "malloc" : " 14 GB", | 20260506 02:51:00.032 | INFO | "pinned" : " 29 GB" | 20260506 02:51:00.032 | INFO | } | 20260506 02:51:00.032 | INFO | Perf report for the last 5.13786 seconds: | 20260506 02:51:00.032 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:51:00.032 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:00.032 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:00.032 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:00.032 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260506 02:51:00.032 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:00.032 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:00.032 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 02:51:00.032 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:00.032 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:00.032 | INFO | Uploaded 36GiB to GPU in 2.09978s (17.1446GiB/s) | 20260506 02:51:00.032 | INFO | Downloaded 34GiB to GPU in 2.04001s (16.6666GiB/s) | 20260506 02:51:05.172 | INFO | Analyzing chunk frames = [8365568,8366080) | 20260506 02:51:05.172 | INFO | MemInfo: { | 20260506 02:51:05.172 | INFO | "gpu" : " 17 GB", | 20260506 02:51:05.172 | INFO | "huge" : "", | 20260506 02:51:05.172 | INFO | "huge_shared" : " 27 GB", | 20260506 02:51:05.172 | INFO | "malloc" : " 14 GB", | 20260506 02:51:05.172 | INFO | "pinned" : " 29 GB" | 20260506 02:51:05.172 | INFO | } | 20260506 02:51:05.172 | INFO | Perf report for the last 5.14063 seconds: | 20260506 02:51:05.172 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:51:05.172 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:05.172 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:05.172 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:05.172 | INFO | ANALYSIS : 61.5%, 3.0, 95.5ms, 0 | 20260506 02:51:05.172 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:05.172 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:05.172 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:51:05.172 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:05.172 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:05.173 | INFO | Uploaded 36GiB to GPU in 2.14922s (16.7503GiB/s) | 20260506 02:51:05.173 | INFO | Downloaded 34GiB to GPU in 2.04587s (16.6189GiB/s) | 20260506 02:51:10.312 | INFO | Analyzing chunk frames = [8366080,8366592) | 20260506 02:51:10.312 | INFO | MemInfo: { | 20260506 02:51:10.312 | INFO | "gpu" : " 17 GB", | 20260506 02:51:10.312 | INFO | "huge" : "", | 20260506 02:51:10.312 | INFO | "huge_shared" : " 27 GB", | 20260506 02:51:10.312 | INFO | "malloc" : " 14 GB", | 20260506 02:51:10.312 | INFO | "pinned" : " 29 GB" | 20260506 02:51:10.312 | INFO | } | 20260506 02:51:10.312 | INFO | Perf report for the last 5.13975 seconds: | 20260506 02:51:10.312 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:51:10.312 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:10.312 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:10.312 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:10.312 | INFO | ANALYSIS : 60.8%, 3.0, 94.9ms, 0 | 20260506 02:51:10.312 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:10.312 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:10.312 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:51:10.312 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:10.312 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:10.312 | INFO | Uploaded 36GiB to GPU in 2.12s (16.9811GiB/s) | 20260506 02:51:10.312 | INFO | Downloaded 34GiB to GPU in 2.03806s (16.6825GiB/s) | 20260506 02:51:15.449 | INFO | Analyzing chunk frames = [8366592,8367104) | 20260506 02:51:15.449 | INFO | MemInfo: { | 20260506 02:51:15.449 | INFO | "gpu" : " 17 GB", | 20260506 02:51:15.449 | INFO | "huge" : "", | 20260506 02:51:15.449 | INFO | "huge_shared" : " 27 GB", | 20260506 02:51:15.449 | INFO | "malloc" : " 14 GB", | 20260506 02:51:15.449 | INFO | "pinned" : " 29 GB" | 20260506 02:51:15.449 | INFO | } | 20260506 02:51:15.449 | INFO | Perf report for the last 5.1369 seconds: | 20260506 02:51:15.449 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:51:15.449 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:15.449 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:15.449 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:15.449 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 02:51:15.449 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:15.449 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:15.449 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:51:15.449 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:15.449 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:15.449 | INFO | Uploaded 36GiB to GPU in 2.11447s (17.0255GiB/s) | 20260506 02:51:15.449 | INFO | Downloaded 34GiB to GPU in 2.05192s (16.5699GiB/s) | 20260506 02:51:20.588 | INFO | Analyzing chunk frames = [8367104,8367616) | 20260506 02:51:20.588 | INFO | MemInfo: { | 20260506 02:51:20.588 | INFO | "gpu" : " 17 GB", | 20260506 02:51:20.588 | INFO | "huge" : "", | 20260506 02:51:20.588 | INFO | "huge_shared" : " 27 GB", | 20260506 02:51:20.588 | INFO | "malloc" : " 14 GB", | 20260506 02:51:20.588 | INFO | "pinned" : " 29 GB" | 20260506 02:51:20.588 | INFO | } | 20260506 02:51:20.588 | INFO | Perf report for the last 5.139 seconds: | 20260506 02:51:20.588 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:51:20.588 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:20.588 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:20.588 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:20.588 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260506 02:51:20.588 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:20.588 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:20.588 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 02:51:20.588 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:20.588 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:20.588 | INFO | Uploaded 36GiB to GPU in 2.08676s (17.2516GiB/s) | 20260506 02:51:20.588 | INFO | Downloaded 34GiB to GPU in 2.09529s (16.2268GiB/s) | 20260506 02:51:25.727 | INFO | Analyzing chunk frames = [8367616,8368128) | 20260506 02:51:25.727 | INFO | MemInfo: { | 20260506 02:51:25.727 | INFO | "gpu" : " 17 GB", | 20260506 02:51:25.727 | INFO | "huge" : "", | 20260506 02:51:25.727 | INFO | "huge_shared" : " 27 GB", | 20260506 02:51:25.727 | INFO | "malloc" : " 14 GB", | 20260506 02:51:25.727 | INFO | "pinned" : " 29 GB" | 20260506 02:51:25.727 | INFO | } | 20260506 02:51:25.727 | INFO | Perf report for the last 5.13894 seconds: | 20260506 02:51:25.727 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:51:25.727 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:25.727 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:25.727 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:25.727 | INFO | ANALYSIS : 61.4%, 3.0, 95.9ms, 0 | 20260506 02:51:25.727 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:25.727 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:25.727 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 02:51:25.727 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:25.727 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:25.727 | INFO | Uploaded 36GiB to GPU in 2.05111s (17.5515GiB/s) | 20260506 02:51:25.727 | INFO | Downloaded 34GiB to GPU in 2.01665s (16.8597GiB/s) | 20260506 02:51:25.853 | INFO | Starting a round of Baz IO | 20260506 02:51:28.986 | INFO | Populating RTMetrics report | 20260506 02:51:28.986 | INFO | RTMetrics report populated | 20260506 02:51:28.986 | INFO | Opening tmp RTMetrics json file | 20260506 02:51:28.986 | INFO | Serializing RTMetrics to json | 20260506 02:51:28.988 | INFO | Serialization done, swapping tmp file now... | 20260506 02:51:28.988 | INFO | RTMetrics done | 20260506 02:51:29.065 | INFO | Finished a round of Baz IO: Wrote 1.778 GiB in 3.211 seconds | 20260506 02:51:29.065 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.25% / 0.00% / 78.75% | 20260506 02:51:29.065 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0653585% / 3.97364e-06% / 9.53674e-05% | 20260506 02:51:29.065 | INFO | Baz IO overall: | 20260506 02:51:29.065 | INFO | BytesWritten : 4675117871172 | 20260506 02:51:29.065 | INFO | NumEvents : 2193851453201 | 20260506 02:51:30.873 | INFO | Analyzing chunk frames = [8368128,8368640) | 20260506 02:51:30.873 | INFO | MemInfo: { | 20260506 02:51:30.873 | INFO | "gpu" : " 17 GB", | 20260506 02:51:30.873 | INFO | "huge" : "", | 20260506 02:51:30.873 | INFO | "huge_shared" : " 27 GB", | 20260506 02:51:30.873 | INFO | "malloc" : " 14 GB", | 20260506 02:51:30.873 | INFO | "pinned" : " 29 GB" | 20260506 02:51:30.873 | INFO | } | 20260506 02:51:30.873 | INFO | Perf report for the last 5.14587 seconds: | 20260506 02:51:30.873 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:51:30.873 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:30.873 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:30.873 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:30.873 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260506 02:51:30.873 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:30.873 | INFO | RT_METRICS : 42.6%, 1.0, 22.4ms, 0 | 20260506 02:51:30.873 | INFO | PRE_HQ : 59.5%, 1.7, 53.6ms, 0 | 20260506 02:51:30.873 | INFO | BAZWRITER : 61.7%, 3.5, 110.6ms, 0 | 20260506 02:51:30.873 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:51:30.873 | INFO | Uploaded 36GiB to GPU in 2.1789s (16.5221GiB/s) | 20260506 02:51:30.873 | INFO | Downloaded 38GiB to GPU in 2.2384s (16.9764GiB/s) | 20260506 02:51:36.008 | INFO | Analyzing chunk frames = [8368640,8369152) | 20260506 02:51:36.009 | INFO | MemInfo: { | 20260506 02:51:36.009 | INFO | "gpu" : " 17 GB", | 20260506 02:51:36.009 | INFO | "huge" : "", | 20260506 02:51:36.009 | INFO | "huge_shared" : " 27 GB", | 20260506 02:51:36.009 | INFO | "malloc" : " 14 GB", | 20260506 02:51:36.009 | INFO | "pinned" : " 29 GB" | 20260506 02:51:36.009 | INFO | } | 20260506 02:51:36.009 | INFO | Perf report for the last 5.136 seconds: | 20260506 02:51:36.009 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:51:36.009 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:36.009 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:36.009 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:36.009 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 02:51:36.009 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:36.009 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:36.009 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 02:51:36.009 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:36.009 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:36.009 | INFO | Uploaded 36GiB to GPU in 1.98577s (18.129GiB/s) | 20260506 02:51:36.009 | INFO | Downloaded 34GiB to GPU in 1.99005s (17.085GiB/s) | 20260506 02:51:41.149 | INFO | Analyzing chunk frames = [8369152,8369664) | 20260506 02:51:41.149 | INFO | MemInfo: { | 20260506 02:51:41.149 | INFO | "gpu" : " 17 GB", | 20260506 02:51:41.149 | INFO | "huge" : "", | 20260506 02:51:41.149 | INFO | "huge_shared" : " 27 GB", | 20260506 02:51:41.149 | INFO | "malloc" : " 14 GB", | 20260506 02:51:41.149 | INFO | "pinned" : " 29 GB" | 20260506 02:51:41.149 | INFO | } | 20260506 02:51:41.149 | INFO | Perf report for the last 5.14019 seconds: | 20260506 02:51:41.149 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:51:41.149 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:41.149 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:41.149 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:41.149 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 02:51:41.149 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:41.149 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:41.149 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 02:51:41.149 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:41.149 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:41.149 | INFO | Uploaded 36GiB to GPU in 2.10804s (17.0775GiB/s) | 20260506 02:51:41.149 | INFO | Downloaded 34GiB to GPU in 2.04491s (16.6266GiB/s) | 20260506 02:51:46.284 | INFO | Analyzing chunk frames = [8369664,8370176) | 20260506 02:51:46.284 | INFO | MemInfo: { | 20260506 02:51:46.284 | INFO | "gpu" : " 17 GB", | 20260506 02:51:46.284 | INFO | "huge" : "", | 20260506 02:51:46.284 | INFO | "huge_shared" : " 27 GB", | 20260506 02:51:46.284 | INFO | "malloc" : " 14 GB", | 20260506 02:51:46.284 | INFO | "pinned" : " 29 GB" | 20260506 02:51:46.284 | INFO | } | 20260506 02:51:46.284 | INFO | Perf report for the last 5.13508 seconds: | 20260506 02:51:46.284 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:51:46.284 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:46.284 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:46.284 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:46.284 | INFO | ANALYSIS : 61.1%, 3.0, 94.8ms, 0 | 20260506 02:51:46.284 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:46.284 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:46.284 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 02:51:46.284 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:46.285 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:46.285 | INFO | Uploaded 36GiB to GPU in 2.07978s (17.3095GiB/s) | 20260506 02:51:46.285 | INFO | Downloaded 34GiB to GPU in 2.01377s (16.8838GiB/s) | 20260506 02:51:51.422 | INFO | Analyzing chunk frames = [8370176,8370688) | 20260506 02:51:51.422 | INFO | MemInfo: { | 20260506 02:51:51.422 | INFO | "gpu" : " 17 GB", | 20260506 02:51:51.422 | INFO | "huge" : "", | 20260506 02:51:51.422 | INFO | "huge_shared" : " 27 GB", | 20260506 02:51:51.422 | INFO | "malloc" : " 14 GB", | 20260506 02:51:51.422 | INFO | "pinned" : " 29 GB" | 20260506 02:51:51.422 | INFO | } | 20260506 02:51:51.422 | INFO | Perf report for the last 5.13826 seconds: | 20260506 02:51:51.422 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:51:51.422 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:51.422 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:51.422 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:51.422 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260506 02:51:51.422 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:51.422 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:51.422 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 02:51:51.422 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:51.422 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:51.423 | INFO | Uploaded 36GiB to GPU in 2.08256s (17.2864GiB/s) | 20260506 02:51:51.423 | INFO | Downloaded 34GiB to GPU in 2.03659s (16.6946GiB/s) | 20260506 02:51:56.566 | INFO | Analyzing chunk frames = [8370688,8371200) | 20260506 02:51:56.566 | INFO | MemInfo: { | 20260506 02:51:56.566 | INFO | "gpu" : " 17 GB", | 20260506 02:51:56.566 | INFO | "huge" : "", | 20260506 02:51:56.566 | INFO | "huge_shared" : " 27 GB", | 20260506 02:51:56.566 | INFO | "malloc" : " 14 GB", | 20260506 02:51:56.566 | INFO | "pinned" : " 29 GB" | 20260506 02:51:56.566 | INFO | } | 20260506 02:51:56.566 | INFO | Perf report for the last 5.14368 seconds: | 20260506 02:51:56.566 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:51:56.566 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:56.566 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:56.566 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:56.566 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 02:51:56.566 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:56.566 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:56.566 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 02:51:56.566 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:56.566 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:51:56.566 | INFO | Uploaded 36GiB to GPU in 2.10212s (17.1256GiB/s) | 20260506 02:51:56.566 | INFO | Downloaded 34GiB to GPU in 2.04219s (16.6488GiB/s) | 20260506 02:52:01.710 | INFO | Analyzing chunk frames = [8371200,8371712) | 20260506 02:52:01.710 | INFO | MemInfo: { | 20260506 02:52:01.710 | INFO | "gpu" : " 17 GB", | 20260506 02:52:01.710 | INFO | "huge" : "", | 20260506 02:52:01.710 | INFO | "huge_shared" : " 27 GB", | 20260506 02:52:01.710 | INFO | "malloc" : " 14 GB", | 20260506 02:52:01.710 | INFO | "pinned" : " 29 GB" | 20260506 02:52:01.710 | INFO | } | 20260506 02:52:01.710 | INFO | Perf report for the last 5.14442 seconds: | 20260506 02:52:01.710 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:52:01.710 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:01.710 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:01.710 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:01.711 | INFO | ANALYSIS : 61.8%, 3.0, 96.6ms, 0 | 20260506 02:52:01.711 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:01.711 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:01.711 | INFO | PRE_HQ : 27.0%, 1.0, 14.1ms, 0 | 20260506 02:52:01.711 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:01.711 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:01.711 | INFO | Uploaded 36GiB to GPU in 2.01959s (17.8254GiB/s) | 20260506 02:52:01.711 | INFO | Downloaded 34GiB to GPU in 2.05915s (16.5117GiB/s) | 20260506 02:52:06.841 | INFO | Analyzing chunk frames = [8371712,8372224) | 20260506 02:52:06.841 | INFO | MemInfo: { | 20260506 02:52:06.841 | INFO | "gpu" : " 17 GB", | 20260506 02:52:06.841 | INFO | "huge" : "", | 20260506 02:52:06.841 | INFO | "huge_shared" : " 27 GB", | 20260506 02:52:06.841 | INFO | "malloc" : " 14 GB", | 20260506 02:52:06.841 | INFO | "pinned" : " 29 GB" | 20260506 02:52:06.841 | INFO | } | 20260506 02:52:06.841 | INFO | Perf report for the last 5.13099 seconds: | 20260506 02:52:06.841 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:52:06.841 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:06.841 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:06.842 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:06.842 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 02:52:06.842 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:06.842 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:06.842 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260506 02:52:06.842 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:06.842 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:06.842 | INFO | Uploaded 36GiB to GPU in 2.04608s (17.5946GiB/s) | 20260506 02:52:06.842 | INFO | Downloaded 34GiB to GPU in 2.01706s (16.8562GiB/s) | 20260506 02:52:06.842 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 02:52:06.842 | INFO | Total mem usage: 15.4 GiB | 20260506 02:52:06.842 | INFO | All Time High Water Marks sum: 16.3 GiB (Savings: 5.2%) | 20260506 02:52:06.842 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.98%) | 20260506 02:52:06.842 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 02:52:06.842 | INFO | High Water Breakdown: | 20260506 02:52:06.842 | INFO | 60.9% : 8.54 GiB ( 88.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 02:52:06.842 | INFO | 39% : 5.47 GiB ( 82.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 02:52:06.842 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 02:52:06.842 | INFO | ------------------------------------------ | 20260506 02:52:06.842 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:52:06.842 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 02:52:06.842 | INFO | Total mem usage: 26.5 GiB | 20260506 02:52:06.842 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 02:52:06.842 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 02:52:06.842 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 02:52:06.842 | INFO | High Water Breakdown: | 20260506 02:52:06.842 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:52:06.842 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:52:06.842 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:52:06.842 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:52:06.842 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:52:06.842 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:52:06.842 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 02:52:06.842 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 02:52:06.842 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:52:06.842 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 02:52:06.842 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 02:52:06.842 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 02:52:06.842 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 02:52:06.842 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 02:52:06.842 | INFO | ------------------------------------------ | 20260506 02:52:06.842 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:52:06.842 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 02:52:06.842 | INFO | Total mem usage: 25.6 GiB | 20260506 02:52:06.842 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 02:52:06.842 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260506 02:52:06.842 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260506 02:52:06.842 | INFO | High Water Breakdown: | 20260506 02:52:06.842 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260506 02:52:06.842 | INFO | ------------------------------------------ | 20260506 02:52:06.842 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:52:06.842 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 02:52:06.842 | INFO | Total mem usage: 15.7 GiB | 20260506 02:52:06.842 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 02:52:06.842 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 02:52:06.842 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 02:52:06.842 | INFO | High Water Breakdown: | 20260506 02:52:06.842 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 02:52:06.842 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 02:52:06.842 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 02:52:06.842 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 02:52:06.842 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 02:52:06.842 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 02:52:06.842 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 02:52:06.842 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 02:52:06.842 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 02:52:06.842 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:52:06.842 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 02:52:06.842 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:52:06.842 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 02:52:06.842 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 02:52:06.842 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 02:52:06.842 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 02:52:06.842 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:52:06.842 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 02:52:06.842 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:52:06.842 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:52:06.842 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:52:06.842 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 02:52:06.842 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 02:52:06.842 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:52:06.843 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 02:52:06.843 | INFO | ------------------------------------------ | 20260506 02:52:06.843 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:52:06.843 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 02:52:06.843 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:52:06.843 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:52:06.843 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.37e+01, 1.32e+01 / 7.78e-01 | 20260506 02:52:06.843 | INFO | PerfReport: 0.29%, AggregateStats -- 9.17e-02 / 1.03e-01, 1.00e-02 / 2.79e-03 | 20260506 02:52:06.843 | INFO | PerfReport: 11.84%, Baseline -- 3.70e+00 / 4.48e+00, 5.76e-01 / 2.37e-01 | 20260506 02:52:06.843 | INFO | PerfReport: 3.29%, Histogram -- 1.03e+00 / 1.52e+00, 1.91e-01 / 7.24e-02 | 20260506 02:52:06.843 | INFO | PerfReport: 16.00%, DME -- 4.00e+01 / 4.40e+01, 3.78e+00 / 9.61e-01 | 20260506 02:52:06.843 | INFO | PerfReport: 57.53%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.51e+00 / 9.26e-01 | 20260506 02:52:06.843 | INFO | PerfReport: 7.01%, PulseAccumulating -- 2.19e+00 / 2.71e+00, 4.17e-01 / 2.09e-01 | 20260506 02:52:06.843 | INFO | PerfReport: 2.99%, Metrics -- 9.34e-01 / 1.20e+00, 2.46e-01 / 7.72e-02 | 20260506 02:52:06.843 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.22e-01 / 3.68e-01, 5.49e-02 / 2.09e-02 | 20260506 02:52:06.843 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 02:52:06.843 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:52:06.843 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:52:06.843 | INFO | PerfReport: Full Execution -- 9.76e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260506 02:52:06.843 | INFO | PerfReport: 22.21%, Upload -- 2.17e+01 / 0.00e+00, 7.20e+00 / 0.00e+00 | 20260506 02:52:06.843 | INFO | PerfReport: 21.98%, Download -- 2.14e+01 / 0.00e+00, 7.60e+00 / 0.00e+00 | 20260506 02:52:06.843 | INFO | PerfReport: 32.03%, Compute -- 3.12e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260506 02:52:06.843 | INFO | PerfReport: 2.00%, LockUpload -- 1.95e+00 / 0.00e+00, 7.85e+00 / 0.00e+00 | 20260506 02:52:06.843 | INFO | PerfReport: 1.34%, LockDownload -- 1.31e+00 / 0.00e+00, 2.99e+00 / 0.00e+00 | 20260506 02:52:06.843 | INFO | PerfReport: 20.43%, LockCompute -- 1.99e+01 / 0.00e+00, 1.77e+01 / 0.00e+00 | 20260506 02:52:06.843 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.66e-02 / 0.00e+00 | 20260506 02:52:06.967 | INFO | Starting a round of Baz IO | 20260506 02:52:10.096 | INFO | Populating RTMetrics report | 20260506 02:52:10.096 | INFO | RTMetrics report populated | 20260506 02:52:10.096 | INFO | Opening tmp RTMetrics json file | 20260506 02:52:10.097 | INFO | Serializing RTMetrics to json | 20260506 02:52:10.098 | INFO | Serialization done, swapping tmp file now... | 20260506 02:52:10.098 | INFO | RTMetrics done | 20260506 02:52:10.173 | INFO | Finished a round of Baz IO: Wrote 1.776 GiB in 3.206 seconds | 20260506 02:52:10.173 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.25% / 0.00% / 78.75% | 20260506 02:52:10.173 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0652313% / 0% / 8.34465e-05% | 20260506 02:52:10.173 | INFO | Baz IO overall: | 20260506 02:52:10.173 | INFO | BytesWritten : 4677025260423 | 20260506 02:52:10.173 | INFO | NumEvents : 2194716725714 | 20260506 02:52:11.980 | INFO | Analyzing chunk frames = [8372224,8372736) | 20260506 02:52:11.980 | INFO | MemInfo: { | 20260506 02:52:11.980 | INFO | "gpu" : " 17 GB", | 20260506 02:52:11.980 | INFO | "huge" : "", | 20260506 02:52:11.980 | INFO | "huge_shared" : " 27 GB", | 20260506 02:52:11.980 | INFO | "malloc" : " 14 GB", | 20260506 02:52:11.980 | INFO | "pinned" : " 29 GB" | 20260506 02:52:11.980 | INFO | } | 20260506 02:52:11.980 | INFO | Perf report for the last 5.13885 seconds: | 20260506 02:52:11.980 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:52:11.980 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:11.980 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:11.980 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:11.980 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260506 02:52:11.980 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:11.980 | INFO | RT_METRICS : 42.8%, 1.0, 22.4ms, 0 | 20260506 02:52:11.980 | INFO | PRE_HQ : 59.3%, 1.7, 53.8ms, 0 | 20260506 02:52:11.980 | INFO | BAZWRITER : 61.5%, 3.5, 111.5ms, 0 | 20260506 02:52:11.980 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:52:11.981 | INFO | Uploaded 36GiB to GPU in 2.11407s (17.0287GiB/s) | 20260506 02:52:11.981 | INFO | Downloaded 38GiB to GPU in 2.23356s (17.0132GiB/s) | 20260506 02:52:17.124 | INFO | Analyzing chunk frames = [8372736,8373248) | 20260506 02:52:17.124 | INFO | MemInfo: { | 20260506 02:52:17.124 | INFO | "gpu" : " 17 GB", | 20260506 02:52:17.124 | INFO | "huge" : "", | 20260506 02:52:17.124 | INFO | "huge_shared" : " 27 GB", | 20260506 02:52:17.124 | INFO | "malloc" : " 14 GB", | 20260506 02:52:17.124 | INFO | "pinned" : " 29 GB" | 20260506 02:52:17.124 | INFO | } | 20260506 02:52:17.124 | INFO | Perf report for the last 5.14396 seconds: | 20260506 02:52:17.124 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:52:17.124 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:17.124 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:17.124 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:17.124 | INFO | ANALYSIS : 60.6%, 3.0, 94.7ms, 0 | 20260506 02:52:17.124 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:17.124 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:17.124 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:52:17.124 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:17.124 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:17.125 | INFO | Uploaded 36GiB to GPU in 1.97327s (18.2438GiB/s) | 20260506 02:52:17.125 | INFO | Downloaded 34GiB to GPU in 1.98377s (17.1391GiB/s) | 20260506 02:52:22.263 | INFO | Analyzing chunk frames = [8373248,8373760) | 20260506 02:52:22.263 | INFO | MemInfo: { | 20260506 02:52:22.263 | INFO | "gpu" : " 17 GB", | 20260506 02:52:22.263 | INFO | "huge" : "", | 20260506 02:52:22.263 | INFO | "huge_shared" : " 27 GB", | 20260506 02:52:22.263 | INFO | "malloc" : " 14 GB", | 20260506 02:52:22.263 | INFO | "pinned" : " 29 GB" | 20260506 02:52:22.263 | INFO | } | 20260506 02:52:22.263 | INFO | Perf report for the last 5.13856 seconds: | 20260506 02:52:22.263 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:52:22.263 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:22.263 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:22.263 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:22.263 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 02:52:22.263 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:22.263 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:22.263 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:52:22.263 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:22.263 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:22.263 | INFO | Uploaded 36GiB to GPU in 2.10232s (17.1239GiB/s) | 20260506 02:52:22.263 | INFO | Downloaded 34GiB to GPU in 2.03885s (16.6761GiB/s) | 20260506 02:52:27.397 | INFO | Analyzing chunk frames = [8373760,8374272) | 20260506 02:52:27.397 | INFO | MemInfo: { | 20260506 02:52:27.397 | INFO | "gpu" : " 17 GB", | 20260506 02:52:27.397 | INFO | "huge" : "", | 20260506 02:52:27.397 | INFO | "huge_shared" : " 27 GB", | 20260506 02:52:27.397 | INFO | "malloc" : " 14 GB", | 20260506 02:52:27.397 | INFO | "pinned" : " 29 GB" | 20260506 02:52:27.397 | INFO | } | 20260506 02:52:27.398 | INFO | Perf report for the last 5.13478 seconds: | 20260506 02:52:27.398 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:52:27.398 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:27.398 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:27.398 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:27.398 | INFO | ANALYSIS : 61.0%, 3.0, 94.6ms, 0 | 20260506 02:52:27.398 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:27.398 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:27.398 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 02:52:27.398 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:27.398 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:27.398 | INFO | Uploaded 36GiB to GPU in 2.08602s (17.2578GiB/s) | 20260506 02:52:27.398 | INFO | Downloaded 34GiB to GPU in 2.00527s (16.9554GiB/s) | 20260506 02:52:32.537 | INFO | Analyzing chunk frames = [8374272,8374784) | 20260506 02:52:32.537 | INFO | MemInfo: { | 20260506 02:52:32.537 | INFO | "gpu" : " 17 GB", | 20260506 02:52:32.537 | INFO | "huge" : "", | 20260506 02:52:32.537 | INFO | "huge_shared" : " 27 GB", | 20260506 02:52:32.537 | INFO | "malloc" : " 14 GB", | 20260506 02:52:32.537 | INFO | "pinned" : " 29 GB" | 20260506 02:52:32.537 | INFO | } | 20260506 02:52:32.537 | INFO | Perf report for the last 5.13979 seconds: | 20260506 02:52:32.537 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:52:32.537 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:32.537 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:32.537 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:32.538 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260506 02:52:32.538 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:32.538 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:32.538 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 02:52:32.538 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:32.538 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:32.538 | INFO | Uploaded 36GiB to GPU in 2.07848s (17.3203GiB/s) | 20260506 02:52:32.538 | INFO | Downloaded 34GiB to GPU in 2.03367s (16.7186GiB/s) | 20260506 02:52:37.681 | INFO | Analyzing chunk frames = [8374784,8375296) | 20260506 02:52:37.681 | INFO | MemInfo: { | 20260506 02:52:37.681 | INFO | "gpu" : " 17 GB", | 20260506 02:52:37.681 | INFO | "huge" : "", | 20260506 02:52:37.681 | INFO | "huge_shared" : " 27 GB", | 20260506 02:52:37.681 | INFO | "malloc" : " 14 GB", | 20260506 02:52:37.681 | INFO | "pinned" : " 29 GB" | 20260506 02:52:37.681 | INFO | } | 20260506 02:52:37.681 | INFO | Perf report for the last 5.14402 seconds: | 20260506 02:52:37.682 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:52:37.682 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:37.682 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:37.682 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:37.682 | INFO | ANALYSIS : 60.9%, 3.0, 95.2ms, 0 | 20260506 02:52:37.682 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:37.682 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:37.682 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:52:37.682 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:37.682 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:37.682 | INFO | Uploaded 36GiB to GPU in 2.05907s (17.4836GiB/s) | 20260506 02:52:37.682 | INFO | Downloaded 34GiB to GPU in 2.03346s (16.7203GiB/s) | 20260506 02:52:42.824 | INFO | Analyzing chunk frames = [8375296,8375808) | 20260506 02:52:42.824 | INFO | MemInfo: { | 20260506 02:52:42.824 | INFO | "gpu" : " 17 GB", | 20260506 02:52:42.824 | INFO | "huge" : "", | 20260506 02:52:42.824 | INFO | "huge_shared" : " 27 GB", | 20260506 02:52:42.824 | INFO | "malloc" : " 14 GB", | 20260506 02:52:42.824 | INFO | "pinned" : " 29 GB" | 20260506 02:52:42.824 | INFO | } | 20260506 02:52:42.824 | INFO | Perf report for the last 5.1424 seconds: | 20260506 02:52:42.824 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:52:42.824 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:42.824 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:42.824 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:42.824 | INFO | ANALYSIS : 61.8%, 3.0, 96.6ms, 0 | 20260506 02:52:42.824 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:42.824 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:42.824 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 02:52:42.824 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:42.824 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:42.824 | INFO | Uploaded 36GiB to GPU in 2.05217s (17.5424GiB/s) | 20260506 02:52:42.824 | INFO | Downloaded 34GiB to GPU in 2.05191s (16.5699GiB/s) | 20260506 02:52:47.964 | INFO | Analyzing chunk frames = [8375808,8376320) | 20260506 02:52:47.964 | INFO | MemInfo: { | 20260506 02:52:47.964 | INFO | "gpu" : " 17 GB", | 20260506 02:52:47.964 | INFO | "huge" : "", | 20260506 02:52:47.964 | INFO | "huge_shared" : " 27 GB", | 20260506 02:52:47.964 | INFO | "malloc" : " 14 GB", | 20260506 02:52:47.964 | INFO | "pinned" : " 29 GB" | 20260506 02:52:47.964 | INFO | } | 20260506 02:52:47.964 | INFO | Perf report for the last 5.14015 seconds: | 20260506 02:52:47.964 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:52:47.964 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:47.964 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:47.964 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:47.964 | INFO | ANALYSIS : 61.3%, 3.0, 95.8ms, 0 | 20260506 02:52:47.964 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:47.964 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:47.964 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 02:52:47.964 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:47.964 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:47.964 | INFO | Uploaded 36GiB to GPU in 2.06105s (17.4668GiB/s) | 20260506 02:52:47.964 | INFO | Downloaded 34GiB to GPU in 2.03958s (16.6701GiB/s) | 20260506 02:52:48.089 | INFO | Starting a round of Baz IO | 20260506 02:52:51.212 | INFO | Populating RTMetrics report | 20260506 02:52:51.212 | INFO | RTMetrics report populated | 20260506 02:52:51.212 | INFO | Opening tmp RTMetrics json file | 20260506 02:52:51.212 | INFO | Serializing RTMetrics to json | 20260506 02:52:51.213 | INFO | Serialization done, swapping tmp file now... | 20260506 02:52:51.214 | INFO | RTMetrics done | 20260506 02:52:51.286 | INFO | Finished a round of Baz IO: Wrote 1.778 GiB in 3.197 seconds | 20260506 02:52:51.286 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.25% / 0.00% / 78.75% | 20260506 02:52:51.286 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0655413% / 0% / 7.94729e-05% | 20260506 02:52:51.286 | INFO | Baz IO overall: | 20260506 02:52:51.286 | INFO | BytesWritten : 4678934133844 | 20260506 02:52:51.286 | INFO | NumEvents : 2195583129127 | 20260506 02:52:53.099 | INFO | Analyzing chunk frames = [8376320,8376832) | 20260506 02:52:53.099 | INFO | MemInfo: { | 20260506 02:52:53.099 | INFO | "gpu" : " 17 GB", | 20260506 02:52:53.099 | INFO | "huge" : "", | 20260506 02:52:53.099 | INFO | "huge_shared" : " 27 GB", | 20260506 02:52:53.099 | INFO | "malloc" : " 14 GB", | 20260506 02:52:53.099 | INFO | "pinned" : " 29 GB" | 20260506 02:52:53.099 | INFO | } | 20260506 02:52:53.099 | INFO | Perf report for the last 5.13504 seconds: | 20260506 02:52:53.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:52:53.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:53.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:53.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:53.099 | INFO | ANALYSIS : 62.2%, 3.0, 96.2ms, 0 | 20260506 02:52:53.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:53.099 | INFO | RT_METRICS : 42.6%, 1.0, 22.3ms, 0 | 20260506 02:52:53.099 | INFO | PRE_HQ : 59.3%, 1.7, 53.1ms, 0 | 20260506 02:52:53.099 | INFO | BAZWRITER : 61.4%, 4.1, 128.0ms, 0 | 20260506 02:52:53.099 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:52:53.099 | INFO | Uploaded 36GiB to GPU in 2.12658s (16.9286GiB/s) | 20260506 02:52:53.099 | INFO | Downloaded 38GiB to GPU in 2.25404s (16.8586GiB/s) | 20260506 02:52:58.233 | INFO | Analyzing chunk frames = [8376832,8377344) | 20260506 02:52:58.233 | INFO | MemInfo: { | 20260506 02:52:58.233 | INFO | "gpu" : " 17 GB", | 20260506 02:52:58.233 | INFO | "huge" : "", | 20260506 02:52:58.233 | INFO | "huge_shared" : " 27 GB", | 20260506 02:52:58.234 | INFO | "malloc" : " 14 GB", | 20260506 02:52:58.234 | INFO | "pinned" : " 29 GB" | 20260506 02:52:58.234 | INFO | } | 20260506 02:52:58.234 | INFO | Perf report for the last 5.13453 seconds: | 20260506 02:52:58.234 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:52:58.234 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:58.234 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:58.234 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:58.234 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260506 02:52:58.234 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:58.234 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:58.234 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 02:52:58.234 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:58.234 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:52:58.234 | INFO | Uploaded 36GiB to GPU in 1.93405s (18.6138GiB/s) | 20260506 02:52:58.234 | INFO | Downloaded 34GiB to GPU in 1.97552s (17.2107GiB/s) | 20260506 02:53:03.377 | INFO | Analyzing chunk frames = [8377344,8377856) | 20260506 02:53:03.377 | INFO | MemInfo: { | 20260506 02:53:03.377 | INFO | "gpu" : " 17 GB", | 20260506 02:53:03.377 | INFO | "huge" : "", | 20260506 02:53:03.377 | INFO | "huge_shared" : " 27 GB", | 20260506 02:53:03.377 | INFO | "malloc" : " 14 GB", | 20260506 02:53:03.377 | INFO | "pinned" : " 29 GB" | 20260506 02:53:03.377 | INFO | } | 20260506 02:53:03.377 | INFO | Perf report for the last 5.14347 seconds: | 20260506 02:53:03.377 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:53:03.377 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:03.377 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:03.377 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:03.377 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 02:53:03.377 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:03.377 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:03.377 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 02:53:03.377 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:03.377 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:03.377 | INFO | Uploaded 36GiB to GPU in 2.09513s (17.1827GiB/s) | 20260506 02:53:03.377 | INFO | Downloaded 34GiB to GPU in 2.02348s (16.8027GiB/s) | 20260506 02:53:08.519 | INFO | Analyzing chunk frames = [8377856,8378368) | 20260506 02:53:08.519 | INFO | MemInfo: { | 20260506 02:53:08.519 | INFO | "gpu" : " 17 GB", | 20260506 02:53:08.519 | INFO | "huge" : "", | 20260506 02:53:08.519 | INFO | "huge_shared" : " 27 GB", | 20260506 02:53:08.519 | INFO | "malloc" : " 14 GB", | 20260506 02:53:08.519 | INFO | "pinned" : " 29 GB" | 20260506 02:53:08.519 | INFO | } | 20260506 02:53:08.520 | INFO | Perf report for the last 5.14235 seconds: | 20260506 02:53:08.520 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:53:08.520 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:08.520 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:08.520 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:08.520 | INFO | ANALYSIS : 61.0%, 3.0, 94.7ms, 0 | 20260506 02:53:08.520 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:08.520 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:08.520 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 02:53:08.520 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:08.520 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:08.520 | INFO | Uploaded 36GiB to GPU in 2.09002s (17.2247GiB/s) | 20260506 02:53:08.520 | INFO | Downloaded 34GiB to GPU in 2.01054s (16.9109GiB/s) | 20260506 02:53:13.658 | INFO | Analyzing chunk frames = [8378368,8378880) | 20260506 02:53:13.658 | INFO | MemInfo: { | 20260506 02:53:13.658 | INFO | "gpu" : " 17 GB", | 20260506 02:53:13.658 | INFO | "huge" : "", | 20260506 02:53:13.658 | INFO | "huge_shared" : " 27 GB", | 20260506 02:53:13.659 | INFO | "malloc" : " 14 GB", | 20260506 02:53:13.659 | INFO | "pinned" : " 29 GB" | 20260506 02:53:13.659 | INFO | } | 20260506 02:53:13.659 | INFO | Perf report for the last 5.13919 seconds: | 20260506 02:53:13.659 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:53:13.659 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:13.659 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:13.659 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:13.659 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260506 02:53:13.659 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:13.659 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:13.659 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:53:13.659 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:13.659 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:13.659 | INFO | Uploaded 36GiB to GPU in 2.10239s (17.1234GiB/s) | 20260506 02:53:13.659 | INFO | Downloaded 34GiB to GPU in 2.03335s (16.7212GiB/s) | 20260506 02:53:18.792 | INFO | Analyzing chunk frames = [8378880,8379392) | 20260506 02:53:18.792 | INFO | MemInfo: { | 20260506 02:53:18.792 | INFO | "gpu" : " 17 GB", | 20260506 02:53:18.792 | INFO | "huge" : "", | 20260506 02:53:18.792 | INFO | "huge_shared" : " 27 GB", | 20260506 02:53:18.793 | INFO | "malloc" : " 14 GB", | 20260506 02:53:18.793 | INFO | "pinned" : " 29 GB" | 20260506 02:53:18.793 | INFO | } | 20260506 02:53:18.793 | INFO | Perf report for the last 5.13398 seconds: | 20260506 02:53:18.793 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:53:18.793 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:18.793 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:18.793 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:18.793 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 02:53:18.793 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:18.793 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:18.793 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 02:53:18.793 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:18.793 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:18.793 | INFO | Uploaded 36GiB to GPU in 2.09131s (17.2141GiB/s) | 20260506 02:53:18.793 | INFO | Downloaded 34GiB to GPU in 2.03624s (16.6974GiB/s) | 20260506 02:53:23.934 | INFO | Analyzing chunk frames = [8379392,8379904) | 20260506 02:53:23.934 | INFO | MemInfo: { | 20260506 02:53:23.934 | INFO | "gpu" : " 17 GB", | 20260506 02:53:23.934 | INFO | "huge" : "", | 20260506 02:53:23.934 | INFO | "huge_shared" : " 27 GB", | 20260506 02:53:23.934 | INFO | "malloc" : " 14 GB", | 20260506 02:53:23.934 | INFO | "pinned" : " 29 GB" | 20260506 02:53:23.934 | INFO | } | 20260506 02:53:23.934 | INFO | Perf report for the last 5.14117 seconds: | 20260506 02:53:23.934 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:53:23.934 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:23.934 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:23.934 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:23.934 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260506 02:53:23.934 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:23.934 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:23.934 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 02:53:23.934 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:23.934 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:23.934 | INFO | Uploaded 36GiB to GPU in 2.03013s (17.7328GiB/s) | 20260506 02:53:23.934 | INFO | Downloaded 34GiB to GPU in 2.04547s (16.6221GiB/s) | 20260506 02:53:29.076 | INFO | Analyzing chunk frames = [8379904,8380416) | 20260506 02:53:29.076 | INFO | MemInfo: { | 20260506 02:53:29.076 | INFO | "gpu" : " 17 GB", | 20260506 02:53:29.076 | INFO | "huge" : "", | 20260506 02:53:29.076 | INFO | "huge_shared" : " 27 GB", | 20260506 02:53:29.076 | INFO | "malloc" : " 14 GB", | 20260506 02:53:29.076 | INFO | "pinned" : " 29 GB" | 20260506 02:53:29.076 | INFO | } | 20260506 02:53:29.076 | INFO | Perf report for the last 5.14246 seconds: | 20260506 02:53:29.076 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:53:29.076 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:29.076 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:29.076 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:29.076 | INFO | ANALYSIS : 61.2%, 3.0, 95.6ms, 0 | 20260506 02:53:29.076 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:29.076 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:29.076 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 02:53:29.076 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:29.077 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:29.077 | INFO | Uploaded 36GiB to GPU in 2.03899s (17.6558GiB/s) | 20260506 02:53:29.077 | INFO | Downloaded 34GiB to GPU in 2.01542s (16.87GiB/s) | 20260506 02:53:29.077 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 02:53:29.077 | INFO | Total mem usage: 15.4 GiB | 20260506 02:53:29.077 | INFO | All Time High Water Marks sum: 16.3 GiB (Savings: 5.2%) | 20260506 02:53:29.077 | INFO | Recent High Water Marks Sum: 15.2 GiB (Savings: -1.38%) | 20260506 02:53:29.077 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 02:53:29.077 | INFO | High Water Breakdown: | 20260506 02:53:29.077 | INFO | 62.1% : 9.45 GiB ( 98.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 02:53:29.077 | INFO | 37.8% : 5.75 GiB ( 86.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 02:53:29.077 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 02:53:29.077 | INFO | ------------------------------------------ | 20260506 02:53:29.077 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:53:29.077 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 02:53:29.077 | INFO | Total mem usage: 26.5 GiB | 20260506 02:53:29.077 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 02:53:29.077 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 02:53:29.077 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 02:53:29.077 | INFO | High Water Breakdown: | 20260506 02:53:29.077 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:53:29.077 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:53:29.077 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:53:29.077 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:53:29.077 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:53:29.077 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:53:29.077 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 02:53:29.077 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 02:53:29.077 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:53:29.077 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 02:53:29.077 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 02:53:29.077 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 02:53:29.077 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 02:53:29.077 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 02:53:29.077 | INFO | ------------------------------------------ | 20260506 02:53:29.077 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:53:29.077 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 02:53:29.077 | INFO | Total mem usage: 25.6 GiB | 20260506 02:53:29.077 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 02:53:29.077 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 02:53:29.077 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 02:53:29.077 | INFO | High Water Breakdown: | 20260506 02:53:29.077 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 02:53:29.077 | INFO | ------------------------------------------ | 20260506 02:53:29.077 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:53:29.077 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 02:53:29.077 | INFO | Total mem usage: 15.7 GiB | 20260506 02:53:29.077 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 02:53:29.077 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 02:53:29.077 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 02:53:29.077 | INFO | High Water Breakdown: | 20260506 02:53:29.077 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 02:53:29.077 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 02:53:29.077 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 02:53:29.077 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 02:53:29.077 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 02:53:29.077 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 02:53:29.077 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 02:53:29.077 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 02:53:29.077 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 02:53:29.077 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:53:29.077 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 02:53:29.077 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:53:29.077 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 02:53:29.077 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 02:53:29.077 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 02:53:29.078 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 02:53:29.078 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:53:29.078 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 02:53:29.078 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:53:29.078 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:53:29.078 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:53:29.078 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 02:53:29.078 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 02:53:29.078 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:53:29.078 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 02:53:29.078 | INFO | ------------------------------------------ | 20260506 02:53:29.078 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:53:29.078 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 02:53:29.078 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:53:29.078 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:53:29.078 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.37e+01, 1.32e+01 / 7.35e-01 | 20260506 02:53:29.078 | INFO | PerfReport: 0.29%, AggregateStats -- 9.18e-02 / 1.05e-01, 9.94e-03 / 3.15e-03 | 20260506 02:53:29.078 | INFO | PerfReport: 11.90%, Baseline -- 3.72e+00 / 4.52e+00, 5.88e-01 / 2.55e-01 | 20260506 02:53:29.078 | INFO | PerfReport: 3.31%, Histogram -- 1.04e+00 / 1.54e+00, 1.97e-01 / 7.67e-02 | 20260506 02:53:29.078 | INFO | PerfReport: 15.99%, DME -- 4.00e+01 / 4.41e+01, 3.75e+00 / 9.13e-01 | 20260506 02:53:29.078 | INFO | PerfReport: 57.46%, FrameLabeling -- 1.80e+01 / 1.91e+01, 1.50e+00 / 8.99e-01 | 20260506 02:53:29.078 | INFO | PerfReport: 7.01%, PulseAccumulating -- 2.19e+00 / 2.76e+00, 4.16e-01 / 1.83e-01 | 20260506 02:53:29.078 | INFO | PerfReport: 3.00%, Metrics -- 9.38e-01 / 1.23e+00, 2.52e-01 / 1.23e-01 | 20260506 02:53:29.078 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.23e-01 / 3.70e-01, 5.48e-02 / 2.62e-02 | 20260506 02:53:29.078 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 02:53:29.078 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:53:29.078 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:53:29.078 | INFO | PerfReport: Full Execution -- 9.74e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260506 02:53:29.078 | INFO | PerfReport: 22.08%, Upload -- 2.15e+01 / 0.00e+00, 7.06e+00 / 0.00e+00 | 20260506 02:53:29.078 | INFO | PerfReport: 21.93%, Download -- 2.14e+01 / 0.00e+00, 7.55e+00 / 0.00e+00 | 20260506 02:53:29.078 | INFO | PerfReport: 32.11%, Compute -- 3.13e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260506 02:53:29.078 | INFO | PerfReport: 1.99%, LockUpload -- 1.94e+00 / 0.00e+00, 7.87e+00 / 0.00e+00 | 20260506 02:53:29.078 | INFO | PerfReport: 1.32%, LockDownload -- 1.28e+00 / 0.00e+00, 3.00e+00 / 0.00e+00 | 20260506 02:53:29.078 | INFO | PerfReport: 20.56%, LockCompute -- 2.00e+01 / 0.00e+00, 1.74e+01 / 0.00e+00 | 20260506 02:53:29.078 | INFO | PerfReport: 0.01%, Misc -- 1.26e-02 / 0.00e+00, 1.59e-02 / 0.00e+00 | 20260506 02:53:29.200 | INFO | Starting a round of Baz IO | 20260506 02:53:32.326 | INFO | Populating RTMetrics report | 20260506 02:53:32.326 | INFO | RTMetrics report populated | 20260506 02:53:32.326 | INFO | Opening tmp RTMetrics json file | 20260506 02:53:32.327 | INFO | Serializing RTMetrics to json | 20260506 02:53:32.328 | INFO | Serialization done, swapping tmp file now... | 20260506 02:53:32.329 | INFO | RTMetrics done | 20260506 02:53:32.409 | INFO | Finished a round of Baz IO: Wrote 1.779 GiB in 3.208 seconds | 20260506 02:53:32.409 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.25% / 0.00% / 78.75% | 20260506 02:53:32.409 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0658552% / 0% / 7.54992e-05% | 20260506 02:53:32.409 | INFO | Baz IO overall: | 20260506 02:53:32.409 | INFO | BytesWritten : 4680844390705 | 20260506 02:53:32.409 | INFO | NumEvents : 2196450271074 | 20260506 02:53:34.216 | INFO | Analyzing chunk frames = [8380416,8380928) | 20260506 02:53:34.216 | INFO | MemInfo: { | 20260506 02:53:34.216 | INFO | "gpu" : " 17 GB", | 20260506 02:53:34.216 | INFO | "huge" : "", | 20260506 02:53:34.216 | INFO | "huge_shared" : " 27 GB", | 20260506 02:53:34.216 | INFO | "malloc" : " 14 GB", | 20260506 02:53:34.216 | INFO | "pinned" : " 29 GB" | 20260506 02:53:34.216 | INFO | } | 20260506 02:53:34.216 | INFO | Perf report for the last 5.13962 seconds: | 20260506 02:53:34.216 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:53:34.216 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:34.216 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:34.216 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:34.216 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260506 02:53:34.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:34.216 | INFO | RT_METRICS : 42.6%, 1.0, 22.4ms, 0 | 20260506 02:53:34.216 | INFO | PRE_HQ : 59.1%, 1.7, 53.2ms, 0 | 20260506 02:53:34.216 | INFO | BAZWRITER : 61.6%, 3.5, 111.1ms, 0 | 20260506 02:53:34.216 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:53:34.216 | INFO | Uploaded 36GiB to GPU in 2.10643s (17.0905GiB/s) | 20260506 02:53:34.216 | INFO | Downloaded 38GiB to GPU in 2.23214s (17.024GiB/s) | 20260506 02:53:39.347 | INFO | Analyzing chunk frames = [8380928,8381440) | 20260506 02:53:39.347 | INFO | MemInfo: { | 20260506 02:53:39.347 | INFO | "gpu" : " 17 GB", | 20260506 02:53:39.347 | INFO | "huge" : "", | 20260506 02:53:39.347 | INFO | "huge_shared" : " 27 GB", | 20260506 02:53:39.347 | INFO | "malloc" : " 14 GB", | 20260506 02:53:39.347 | INFO | "pinned" : " 29 GB" | 20260506 02:53:39.347 | INFO | } | 20260506 02:53:39.347 | INFO | Perf report for the last 5.13146 seconds: | 20260506 02:53:39.347 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:53:39.347 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:39.347 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:39.347 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:39.347 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 02:53:39.348 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:39.348 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:39.348 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 02:53:39.348 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:39.348 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:39.348 | INFO | Uploaded 36GiB to GPU in 1.94277s (18.5303GiB/s) | 20260506 02:53:39.348 | INFO | Downloaded 34GiB to GPU in 1.97385s (17.2252GiB/s) | 20260506 02:53:44.488 | INFO | Analyzing chunk frames = [8381440,8381952) | 20260506 02:53:44.488 | INFO | MemInfo: { | 20260506 02:53:44.488 | INFO | "gpu" : " 17 GB", | 20260506 02:53:44.488 | INFO | "huge" : "", | 20260506 02:53:44.488 | INFO | "huge_shared" : " 27 GB", | 20260506 02:53:44.488 | INFO | "malloc" : " 14 GB", | 20260506 02:53:44.488 | INFO | "pinned" : " 29 GB" | 20260506 02:53:44.488 | INFO | } | 20260506 02:53:44.488 | INFO | Perf report for the last 5.14034 seconds: | 20260506 02:53:44.488 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:53:44.488 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:44.488 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:44.488 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:44.488 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 02:53:44.488 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:44.488 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:44.488 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 02:53:44.488 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:44.488 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:44.488 | INFO | Uploaded 36GiB to GPU in 2.09059s (17.22GiB/s) | 20260506 02:53:44.488 | INFO | Downloaded 34GiB to GPU in 2.02147s (16.8195GiB/s) | 20260506 02:53:49.631 | INFO | Analyzing chunk frames = [8381952,8382464) | 20260506 02:53:49.631 | INFO | MemInfo: { | 20260506 02:53:49.631 | INFO | "gpu" : " 17 GB", | 20260506 02:53:49.631 | INFO | "huge" : "", | 20260506 02:53:49.631 | INFO | "huge_shared" : " 27 GB", | 20260506 02:53:49.631 | INFO | "malloc" : " 14 GB", | 20260506 02:53:49.631 | INFO | "pinned" : " 29 GB" | 20260506 02:53:49.631 | INFO | } | 20260506 02:53:49.631 | INFO | Perf report for the last 5.14347 seconds: | 20260506 02:53:49.631 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:53:49.631 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:49.631 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:49.631 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:49.631 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 02:53:49.631 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:49.631 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:49.631 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 02:53:49.631 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:49.631 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:49.632 | INFO | Uploaded 36GiB to GPU in 2.15643s (16.6942GiB/s) | 20260506 02:53:49.632 | INFO | Downloaded 34GiB to GPU in 2.05341s (16.5579GiB/s) | 20260506 02:53:54.772 | INFO | Analyzing chunk frames = [8382464,8382976) | 20260506 02:53:54.772 | INFO | MemInfo: { | 20260506 02:53:54.772 | INFO | "gpu" : " 17 GB", | 20260506 02:53:54.772 | INFO | "huge" : "", | 20260506 02:53:54.772 | INFO | "huge_shared" : " 27 GB", | 20260506 02:53:54.773 | INFO | "malloc" : " 14 GB", | 20260506 02:53:54.773 | INFO | "pinned" : " 29 GB" | 20260506 02:53:54.773 | INFO | } | 20260506 02:53:54.773 | INFO | Perf report for the last 5.14133 seconds: | 20260506 02:53:54.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:53:54.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:54.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:54.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:54.773 | INFO | ANALYSIS : 60.7%, 3.0, 94.8ms, 0 | 20260506 02:53:54.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:54.773 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:54.773 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 02:53:54.773 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:54.773 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:54.773 | INFO | Uploaded 36GiB to GPU in 2.10873s (17.0719GiB/s) | 20260506 02:53:54.773 | INFO | Downloaded 34GiB to GPU in 2.03311s (16.7231GiB/s) | 20260506 02:53:59.912 | INFO | Analyzing chunk frames = [8382976,8383488) | 20260506 02:53:59.912 | INFO | MemInfo: { | 20260506 02:53:59.912 | INFO | "gpu" : " 17 GB", | 20260506 02:53:59.912 | INFO | "huge" : "", | 20260506 02:53:59.912 | INFO | "huge_shared" : " 27 GB", | 20260506 02:53:59.912 | INFO | "malloc" : " 14 GB", | 20260506 02:53:59.912 | INFO | "pinned" : " 29 GB" | 20260506 02:53:59.912 | INFO | } | 20260506 02:53:59.912 | INFO | Perf report for the last 5.13976 seconds: | 20260506 02:53:59.912 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:53:59.912 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:59.912 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:59.912 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:59.912 | INFO | ANALYSIS : 60.8%, 3.0, 94.9ms, 0 | 20260506 02:53:59.912 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:59.912 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:59.913 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:53:59.913 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:59.913 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:53:59.913 | INFO | Uploaded 36GiB to GPU in 2.05306s (17.5348GiB/s) | 20260506 02:53:59.913 | INFO | Downloaded 34GiB to GPU in 2.02699s (16.7736GiB/s) | 20260506 02:54:05.046 | INFO | Analyzing chunk frames = [8383488,8384000) | 20260506 02:54:05.046 | INFO | MemInfo: { | 20260506 02:54:05.046 | INFO | "gpu" : " 17 GB", | 20260506 02:54:05.046 | INFO | "huge" : "", | 20260506 02:54:05.046 | INFO | "huge_shared" : " 27 GB", | 20260506 02:54:05.047 | INFO | "malloc" : " 14 GB", | 20260506 02:54:05.047 | INFO | "pinned" : " 29 GB" | 20260506 02:54:05.047 | INFO | } | 20260506 02:54:05.047 | INFO | Perf report for the last 5.13424 seconds: | 20260506 02:54:05.047 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:54:05.047 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:05.047 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:05.047 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:05.047 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 02:54:05.047 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:05.047 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:05.047 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 02:54:05.047 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:05.047 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:05.047 | INFO | Uploaded 36GiB to GPU in 2.07323s (17.3642GiB/s) | 20260506 02:54:05.047 | INFO | Downloaded 34GiB to GPU in 2.0529s (16.5619GiB/s) | 20260506 02:54:10.189 | INFO | Analyzing chunk frames = [8384000,8384512) | 20260506 02:54:10.189 | INFO | MemInfo: { | 20260506 02:54:10.189 | INFO | "gpu" : " 17 GB", | 20260506 02:54:10.189 | INFO | "huge" : "", | 20260506 02:54:10.190 | INFO | "huge_shared" : " 27 GB", | 20260506 02:54:10.190 | INFO | "malloc" : " 14 GB", | 20260506 02:54:10.190 | INFO | "pinned" : " 29 GB" | 20260506 02:54:10.190 | INFO | } | 20260506 02:54:10.190 | INFO | Perf report for the last 5.14309 seconds: | 20260506 02:54:10.190 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:54:10.190 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:10.190 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:10.190 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:10.190 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260506 02:54:10.190 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:10.190 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:10.190 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 02:54:10.190 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:10.190 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:10.190 | INFO | Uploaded 36GiB to GPU in 2.13335s (16.8749GiB/s) | 20260506 02:54:10.190 | INFO | Downloaded 34GiB to GPU in 2.02747s (16.7697GiB/s) | 20260506 02:54:10.312 | INFO | Starting a round of Baz IO | 20260506 02:54:13.422 | INFO | Populating RTMetrics report | 20260506 02:54:13.422 | INFO | RTMetrics report populated | 20260506 02:54:13.422 | INFO | Opening tmp RTMetrics json file | 20260506 02:54:13.423 | INFO | Serializing RTMetrics to json | 20260506 02:54:13.424 | INFO | Serialization done, swapping tmp file now... | 20260506 02:54:13.425 | INFO | RTMetrics done | 20260506 02:54:13.500 | INFO | Finished a round of Baz IO: Wrote 1.779 GiB in 3.187 seconds | 20260506 02:54:13.500 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.25% / 0.00% / 78.75% | 20260506 02:54:13.500 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0654538% / 0% / 6.75519e-05% | 20260506 02:54:13.500 | INFO | Baz IO overall: | 20260506 02:54:13.500 | INFO | BytesWritten : 4682754372890 | 20260506 02:54:13.500 | INFO | NumEvents : 2197317417439 | 20260506 02:54:15.328 | INFO | Analyzing chunk frames = [8384512,8385024) | 20260506 02:54:15.328 | INFO | MemInfo: { | 20260506 02:54:15.328 | INFO | "gpu" : " 17 GB", | 20260506 02:54:15.328 | INFO | "huge" : "", | 20260506 02:54:15.328 | INFO | "huge_shared" : " 27 GB", | 20260506 02:54:15.328 | INFO | "malloc" : " 14 GB", | 20260506 02:54:15.328 | INFO | "pinned" : " 29 GB" | 20260506 02:54:15.328 | INFO | } | 20260506 02:54:15.328 | INFO | Perf report for the last 5.13836 seconds: | 20260506 02:54:15.328 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:54:15.328 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:15.328 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:15.328 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:15.328 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260506 02:54:15.328 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:15.328 | INFO | RT_METRICS : 42.8%, 1.0, 22.5ms, 0 | 20260506 02:54:15.328 | INFO | PRE_HQ : 59.0%, 1.7, 53.5ms, 0 | 20260506 02:54:15.328 | INFO | BAZWRITER : 61.1%, 3.5, 109.5ms, 0 | 20260506 02:54:15.328 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:54:15.328 | INFO | Uploaded 36GiB to GPU in 2.09835s (17.1564GiB/s) | 20260506 02:54:15.329 | INFO | Downloaded 38GiB to GPU in 2.22775s (17.0576GiB/s) | 20260506 02:54:20.469 | INFO | Analyzing chunk frames = [8385024,8385536) | 20260506 02:54:20.469 | INFO | MemInfo: { | 20260506 02:54:20.469 | INFO | "gpu" : " 17 GB", | 20260506 02:54:20.469 | INFO | "huge" : "", | 20260506 02:54:20.469 | INFO | "huge_shared" : " 27 GB", | 20260506 02:54:20.469 | INFO | "malloc" : " 14 GB", | 20260506 02:54:20.469 | INFO | "pinned" : " 29 GB" | 20260506 02:54:20.469 | INFO | } | 20260506 02:54:20.470 | INFO | Perf report for the last 5.1414 seconds: | 20260506 02:54:20.470 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:54:20.470 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:20.470 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:20.470 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:20.470 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260506 02:54:20.470 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:20.470 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:20.470 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 02:54:20.470 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:20.470 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:20.470 | INFO | Uploaded 36GiB to GPU in 1.94723s (18.4878GiB/s) | 20260506 02:54:20.470 | INFO | Downloaded 34GiB to GPU in 1.98233s (17.1515GiB/s) | 20260506 02:54:25.607 | INFO | Analyzing chunk frames = [8385536,8386048) | 20260506 02:54:25.608 | INFO | MemInfo: { | 20260506 02:54:25.608 | INFO | "gpu" : " 17 GB", | 20260506 02:54:25.608 | INFO | "huge" : "", | 20260506 02:54:25.608 | INFO | "huge_shared" : " 27 GB", | 20260506 02:54:25.608 | INFO | "malloc" : " 14 GB", | 20260506 02:54:25.608 | INFO | "pinned" : " 29 GB" | 20260506 02:54:25.608 | INFO | } | 20260506 02:54:25.608 | INFO | Perf report for the last 5.1382 seconds: | 20260506 02:54:25.608 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:54:25.608 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:25.608 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:25.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:25.608 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 02:54:25.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:25.608 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:25.608 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 02:54:25.608 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:25.608 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:25.608 | INFO | Uploaded 36GiB to GPU in 2.1261s (16.9324GiB/s) | 20260506 02:54:25.608 | INFO | Downloaded 34GiB to GPU in 2.04728s (16.6074GiB/s) | 20260506 02:54:30.743 | INFO | Analyzing chunk frames = [8386048,8386560) | 20260506 02:54:30.743 | INFO | MemInfo: { | 20260506 02:54:30.743 | INFO | "gpu" : " 17 GB", | 20260506 02:54:30.743 | INFO | "huge" : "", | 20260506 02:54:30.743 | INFO | "huge_shared" : " 27 GB", | 20260506 02:54:30.743 | INFO | "malloc" : " 14 GB", | 20260506 02:54:30.743 | INFO | "pinned" : " 29 GB" | 20260506 02:54:30.743 | INFO | } | 20260506 02:54:30.743 | INFO | Perf report for the last 5.13541 seconds: | 20260506 02:54:30.743 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:54:30.743 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:30.743 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:30.743 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:30.743 | INFO | ANALYSIS : 61.1%, 3.0, 94.9ms, 0 | 20260506 02:54:30.743 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:30.743 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:30.743 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:54:30.743 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:30.743 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:30.743 | INFO | Uploaded 36GiB to GPU in 2.08775s (17.2435GiB/s) | 20260506 02:54:30.744 | INFO | Downloaded 34GiB to GPU in 2.01551s (16.8692GiB/s) | 20260506 02:54:35.886 | INFO | Analyzing chunk frames = [8386560,8387072) | 20260506 02:54:35.886 | INFO | MemInfo: { | 20260506 02:54:35.886 | INFO | "gpu" : " 17 GB", | 20260506 02:54:35.886 | INFO | "huge" : "", | 20260506 02:54:35.886 | INFO | "huge_shared" : " 27 GB", | 20260506 02:54:35.886 | INFO | "malloc" : " 14 GB", | 20260506 02:54:35.886 | INFO | "pinned" : " 29 GB" | 20260506 02:54:35.886 | INFO | } | 20260506 02:54:35.886 | INFO | Perf report for the last 5.1428 seconds: | 20260506 02:54:35.886 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:54:35.886 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:35.886 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:35.886 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:35.886 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260506 02:54:35.886 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:35.886 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:35.886 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 02:54:35.886 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:35.886 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:35.886 | INFO | Uploaded 36GiB to GPU in 2.09826s (17.157GiB/s) | 20260506 02:54:35.886 | INFO | Downloaded 34GiB to GPU in 2.0335s (16.7199GiB/s) | 20260506 02:54:41.030 | INFO | Analyzing chunk frames = [8387072,8387584) | 20260506 02:54:41.030 | INFO | MemInfo: { | 20260506 02:54:41.030 | INFO | "gpu" : " 17 GB", | 20260506 02:54:41.030 | INFO | "huge" : "", | 20260506 02:54:41.030 | INFO | "huge_shared" : " 27 GB", | 20260506 02:54:41.030 | INFO | "malloc" : " 14 GB", | 20260506 02:54:41.030 | INFO | "pinned" : " 29 GB" | 20260506 02:54:41.030 | INFO | } | 20260506 02:54:41.030 | INFO | Perf report for the last 5.14384 seconds: | 20260506 02:54:41.030 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:54:41.030 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:41.030 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:41.030 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:41.030 | INFO | ANALYSIS : 60.7%, 3.0, 94.8ms, 0 | 20260506 02:54:41.030 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:41.030 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:41.030 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 02:54:41.030 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:41.030 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:41.030 | INFO | Uploaded 36GiB to GPU in 2.03587s (17.6829GiB/s) | 20260506 02:54:41.030 | INFO | Downloaded 34GiB to GPU in 2.02421s (16.7967GiB/s) | 20260506 02:54:46.162 | INFO | Analyzing chunk frames = [8387584,8388096) | 20260506 02:54:46.162 | INFO | MemInfo: { | 20260506 02:54:46.162 | INFO | "gpu" : " 17 GB", | 20260506 02:54:46.162 | INFO | "huge" : "", | 20260506 02:54:46.162 | INFO | "huge_shared" : " 27 GB", | 20260506 02:54:46.163 | INFO | "malloc" : " 14 GB", | 20260506 02:54:46.163 | INFO | "pinned" : " 29 GB" | 20260506 02:54:46.163 | INFO | } | 20260506 02:54:46.163 | INFO | Perf report for the last 5.13281 seconds: | 20260506 02:54:46.163 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:54:46.163 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:46.163 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:46.163 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:46.163 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 02:54:46.163 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:46.163 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:46.163 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260506 02:54:46.163 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:46.163 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:46.163 | INFO | Uploaded 36GiB to GPU in 2.08262s (17.2859GiB/s) | 20260506 02:54:46.163 | INFO | Downloaded 34GiB to GPU in 2.06317s (16.4795GiB/s) | 20260506 02:54:51.297 | INFO | Analyzing chunk frames = [8388096,8388608) | 20260506 02:54:51.297 | INFO | MemInfo: { | 20260506 02:54:51.297 | INFO | "gpu" : " 17 GB", | 20260506 02:54:51.297 | INFO | "huge" : "", | 20260506 02:54:51.297 | INFO | "huge_shared" : " 27 GB", | 20260506 02:54:51.297 | INFO | "malloc" : " 14 GB", | 20260506 02:54:51.297 | INFO | "pinned" : " 29 GB" | 20260506 02:54:51.297 | INFO | } | 20260506 02:54:51.297 | INFO | Perf report for the last 5.13438 seconds: | 20260506 02:54:51.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:54:51.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:51.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:51.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:51.297 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 02:54:51.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:51.297 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:51.297 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 02:54:51.297 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:51.297 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:51.297 | INFO | Uploaded 36GiB to GPU in 2.04186s (17.631GiB/s) | 20260506 02:54:51.297 | INFO | Downloaded 34GiB to GPU in 2.01587s (16.8662GiB/s) | 20260506 02:54:51.298 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 02:54:51.298 | INFO | Total mem usage: 15.4 GiB | 20260506 02:54:51.298 | INFO | All Time High Water Marks sum: 16.3 GiB (Savings: 5.2%) | 20260506 02:54:51.298 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.97%) | 20260506 02:54:51.298 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 02:54:51.298 | INFO | High Water Breakdown: | 20260506 02:54:51.298 | INFO | 60.9% : 8.54 GiB ( 88.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 02:54:51.298 | INFO | 39% : 5.47 GiB ( 82.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 02:54:51.298 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 02:54:51.298 | INFO | ------------------------------------------ | 20260506 02:54:51.298 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:54:51.298 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 02:54:51.298 | INFO | Total mem usage: 26.5 GiB | 20260506 02:54:51.298 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 02:54:51.298 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 02:54:51.298 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 02:54:51.298 | INFO | High Water Breakdown: | 20260506 02:54:51.298 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:54:51.298 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:54:51.298 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:54:51.298 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:54:51.298 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:54:51.298 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:54:51.298 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 02:54:51.298 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 02:54:51.298 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:54:51.298 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 02:54:51.298 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 02:54:51.298 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 02:54:51.298 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 02:54:51.298 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 02:54:51.298 | INFO | ------------------------------------------ | 20260506 02:54:51.298 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:54:51.298 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 02:54:51.298 | INFO | Total mem usage: 25.6 GiB | 20260506 02:54:51.298 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 02:54:51.298 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 02:54:51.298 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 02:54:51.298 | INFO | High Water Breakdown: | 20260506 02:54:51.298 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 02:54:51.298 | INFO | ------------------------------------------ | 20260506 02:54:51.298 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:54:51.298 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 02:54:51.298 | INFO | Total mem usage: 15.7 GiB | 20260506 02:54:51.298 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 02:54:51.298 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 02:54:51.298 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 02:54:51.298 | INFO | High Water Breakdown: | 20260506 02:54:51.298 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 02:54:51.298 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 02:54:51.298 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 02:54:51.298 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 02:54:51.298 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 02:54:51.298 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 02:54:51.298 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 02:54:51.298 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 02:54:51.298 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 02:54:51.298 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:54:51.298 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 02:54:51.298 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:54:51.298 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 02:54:51.298 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 02:54:51.298 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 02:54:51.298 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 02:54:51.298 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:54:51.298 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 02:54:51.298 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:54:51.298 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:54:51.298 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:54:51.298 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 02:54:51.298 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 02:54:51.298 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:54:51.298 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 02:54:51.298 | INFO | ------------------------------------------ | 20260506 02:54:51.298 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:54:51.298 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 02:54:51.298 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:54:51.298 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:54:51.298 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.37e+01, 1.32e+01 / 7.13e-01 | 20260506 02:54:51.298 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.04e-01, 1.02e-02 / 3.92e-03 | 20260506 02:54:51.298 | INFO | PerfReport: 11.85%, Baseline -- 3.71e+00 / 4.52e+00, 5.86e-01 / 2.47e-01 | 20260506 02:54:51.298 | INFO | PerfReport: 3.31%, Histogram -- 1.03e+00 / 1.53e+00, 1.96e-01 / 9.60e-02 | 20260506 02:54:51.298 | INFO | PerfReport: 15.98%, DME -- 4.00e+01 / 4.40e+01, 3.82e+00 / 7.03e-01 | 20260506 02:54:51.298 | INFO | PerfReport: 57.54%, FrameLabeling -- 1.80e+01 / 1.91e+01, 1.57e+00 / 9.14e-01 | 20260506 02:54:51.298 | INFO | PerfReport: 6.99%, PulseAccumulating -- 2.19e+00 / 2.75e+00, 4.17e-01 / 1.57e-01 | 20260506 02:54:51.298 | INFO | PerfReport: 3.00%, Metrics -- 9.38e-01 / 1.22e+00, 2.55e-01 / 1.40e-01 | 20260506 02:54:51.298 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.23e-01 / 3.71e-01, 5.48e-02 / 3.03e-02 | 20260506 02:54:51.298 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 02:54:51.298 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:54:51.298 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:54:51.298 | INFO | PerfReport: Full Execution -- 9.75e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260506 02:54:51.298 | INFO | PerfReport: 22.16%, Upload -- 2.16e+01 / 0.00e+00, 7.12e+00 / 0.00e+00 | 20260506 02:54:51.299 | INFO | PerfReport: 21.93%, Download -- 2.14e+01 / 0.00e+00, 7.64e+00 / 0.00e+00 | 20260506 02:54:51.299 | INFO | PerfReport: 32.11%, Compute -- 3.13e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260506 02:54:51.299 | INFO | PerfReport: 1.97%, LockUpload -- 1.92e+00 / 0.00e+00, 7.75e+00 / 0.00e+00 | 20260506 02:54:51.299 | INFO | PerfReport: 1.26%, LockDownload -- 1.23e+00 / 0.00e+00, 2.89e+00 / 0.00e+00 | 20260506 02:54:51.299 | INFO | PerfReport: 20.55%, LockCompute -- 2.00e+01 / 0.00e+00, 1.75e+01 / 0.00e+00 | 20260506 02:54:51.299 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.50e-02 / 0.00e+00 | 20260506 02:54:51.415 | INFO | Starting a round of Baz IO | 20260506 02:54:54.552 | INFO | Populating RTMetrics report | 20260506 02:54:54.552 | INFO | RTMetrics report populated | 20260506 02:54:54.552 | INFO | Opening tmp RTMetrics json file | 20260506 02:54:54.553 | INFO | Serializing RTMetrics to json | 20260506 02:54:54.555 | INFO | Serialization done, swapping tmp file now... | 20260506 02:54:54.556 | INFO | RTMetrics done | 20260506 02:54:54.627 | INFO | Finished a round of Baz IO: Wrote 1.777 GiB in 3.212 seconds | 20260506 02:54:54.627 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.25% / 0.00% / 78.75% | 20260506 02:54:54.627 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0646949% / 0% / 7.54992e-05% | 20260506 02:54:54.627 | INFO | Baz IO overall: | 20260506 02:54:54.627 | INFO | BytesWritten : 4684662879419 | 20260506 02:54:54.627 | INFO | NumEvents : 2198183958948 | 20260506 02:54:56.438 | INFO | Analyzing chunk frames = [8388608,8389120) | 20260506 02:54:56.438 | INFO | MemInfo: { | 20260506 02:54:56.438 | INFO | "gpu" : " 17 GB", | 20260506 02:54:56.438 | INFO | "huge" : "", | 20260506 02:54:56.438 | INFO | "huge_shared" : " 27 GB", | 20260506 02:54:56.438 | INFO | "malloc" : " 14 GB", | 20260506 02:54:56.438 | INFO | "pinned" : " 29 GB" | 20260506 02:54:56.438 | INFO | } | 20260506 02:54:56.438 | INFO | Perf report for the last 5.14067 seconds: | 20260506 02:54:56.438 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:54:56.438 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:56.438 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:56.438 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:56.438 | INFO | ANALYSIS : 62.3%, 3.0, 96.7ms, 0 | 20260506 02:54:56.438 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:54:56.438 | INFO | RT_METRICS : 43.0%, 1.0, 22.5ms, 0 | 20260506 02:54:56.438 | INFO | PRE_HQ : 59.6%, 1.7, 53.4ms, 0 | 20260506 02:54:56.438 | INFO | BAZWRITER : 61.6%, 3.9, 123.8ms, 0 | 20260506 02:54:56.438 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:54:56.438 | INFO | Uploaded 36GiB to GPU in 2.19135s (16.4282GiB/s) | 20260506 02:54:56.438 | INFO | Downloaded 38GiB to GPU in 2.25413s (16.8579GiB/s) | 20260506 02:55:01.585 | INFO | Analyzing chunk frames = [8389120,8389632) | 20260506 02:55:01.585 | INFO | MemInfo: { | 20260506 02:55:01.585 | INFO | "gpu" : " 17 GB", | 20260506 02:55:01.585 | INFO | "huge" : "", | 20260506 02:55:01.585 | INFO | "huge_shared" : " 27 GB", | 20260506 02:55:01.585 | INFO | "malloc" : " 14 GB", | 20260506 02:55:01.585 | INFO | "pinned" : " 29 GB" | 20260506 02:55:01.585 | INFO | } | 20260506 02:55:01.585 | INFO | Perf report for the last 5.14772 seconds: | 20260506 02:55:01.585 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:55:01.585 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:01.585 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:01.586 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:01.586 | INFO | ANALYSIS : 60.7%, 3.0, 94.9ms, 0 | 20260506 02:55:01.586 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:01.586 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:01.586 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 02:55:01.586 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:01.586 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:01.586 | INFO | Uploaded 36GiB to GPU in 1.9749s (18.2288GiB/s) | 20260506 02:55:01.586 | INFO | Downloaded 34GiB to GPU in 1.98462s (17.1317GiB/s) | 20260506 02:55:06.723 | INFO | Analyzing chunk frames = [8389632,8390144) | 20260506 02:55:06.723 | INFO | MemInfo: { | 20260506 02:55:06.723 | INFO | "gpu" : " 17 GB", | 20260506 02:55:06.723 | INFO | "huge" : "", | 20260506 02:55:06.723 | INFO | "huge_shared" : " 27 GB", | 20260506 02:55:06.724 | INFO | "malloc" : " 14 GB", | 20260506 02:55:06.724 | INFO | "pinned" : " 29 GB" | 20260506 02:55:06.724 | INFO | } | 20260506 02:55:06.724 | INFO | Perf report for the last 5.13814 seconds: | 20260506 02:55:06.724 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:55:06.724 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:06.724 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:06.724 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:06.724 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260506 02:55:06.724 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:06.724 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:06.724 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:55:06.724 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:06.724 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:06.724 | INFO | Uploaded 36GiB to GPU in 2.0062s (17.9443GiB/s) | 20260506 02:55:06.724 | INFO | Downloaded 34GiB to GPU in 1.99077s (17.0788GiB/s) | 20260506 02:55:11.867 | INFO | Analyzing chunk frames = [8390144,8390656) | 20260506 02:55:11.867 | INFO | MemInfo: { | 20260506 02:55:11.867 | INFO | "gpu" : " 17 GB", | 20260506 02:55:11.867 | INFO | "huge" : "", | 20260506 02:55:11.867 | INFO | "huge_shared" : " 27 GB", | 20260506 02:55:11.867 | INFO | "malloc" : " 14 GB", | 20260506 02:55:11.867 | INFO | "pinned" : " 29 GB" | 20260506 02:55:11.867 | INFO | } | 20260506 02:55:11.867 | INFO | Perf report for the last 5.14378 seconds: | 20260506 02:55:11.867 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:55:11.867 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:11.867 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:11.867 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:11.867 | INFO | ANALYSIS : 61.0%, 3.0, 94.7ms, 0 | 20260506 02:55:11.867 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:11.867 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:11.868 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 02:55:11.868 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:11.868 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:11.868 | INFO | Uploaded 36GiB to GPU in 2.09816s (17.1579GiB/s) | 20260506 02:55:11.868 | INFO | Downloaded 34GiB to GPU in 2.01162s (16.9018GiB/s) | 20260506 02:55:16.995 | INFO | Analyzing chunk frames = [8390656,8391168) | 20260506 02:55:16.995 | INFO | MemInfo: { | 20260506 02:55:16.995 | INFO | "gpu" : " 17 GB", | 20260506 02:55:16.995 | INFO | "huge" : "", | 20260506 02:55:16.995 | INFO | "huge_shared" : " 27 GB", | 20260506 02:55:16.995 | INFO | "malloc" : " 14 GB", | 20260506 02:55:16.995 | INFO | "pinned" : " 29 GB" | 20260506 02:55:16.995 | INFO | } | 20260506 02:55:16.995 | INFO | Perf report for the last 5.12748 seconds: | 20260506 02:55:16.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:55:16.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:16.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:16.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:16.995 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260506 02:55:16.995 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:16.995 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:16.995 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 02:55:16.995 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:16.995 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:16.995 | INFO | Uploaded 36GiB to GPU in 2.11157s (17.0489GiB/s) | 20260506 02:55:16.995 | INFO | Downloaded 34GiB to GPU in 2.03322s (16.7222GiB/s) | 20260506 02:55:22.139 | INFO | Analyzing chunk frames = [8391168,8391680) | 20260506 02:55:22.139 | INFO | MemInfo: { | 20260506 02:55:22.139 | INFO | "gpu" : " 17 GB", | 20260506 02:55:22.139 | INFO | "huge" : "", | 20260506 02:55:22.139 | INFO | "huge_shared" : " 27 GB", | 20260506 02:55:22.139 | INFO | "malloc" : " 14 GB", | 20260506 02:55:22.140 | INFO | "pinned" : " 29 GB" | 20260506 02:55:22.140 | INFO | } | 20260506 02:55:22.140 | INFO | Perf report for the last 5.1447 seconds: | 20260506 02:55:22.140 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:55:22.140 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:22.140 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:22.140 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:22.140 | INFO | ANALYSIS : 60.9%, 3.0, 95.2ms, 0 | 20260506 02:55:22.140 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:22.140 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:22.140 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 02:55:22.140 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:22.140 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:22.140 | INFO | Uploaded 36GiB to GPU in 2.06724s (17.4146GiB/s) | 20260506 02:55:22.140 | INFO | Downloaded 34GiB to GPU in 2.03269s (16.7266GiB/s) | 20260506 02:55:27.283 | INFO | Analyzing chunk frames = [8391680,8392192) | 20260506 02:55:27.283 | INFO | MemInfo: { | 20260506 02:55:27.283 | INFO | "gpu" : " 17 GB", | 20260506 02:55:27.283 | INFO | "huge" : "", | 20260506 02:55:27.283 | INFO | "huge_shared" : " 27 GB", | 20260506 02:55:27.283 | INFO | "malloc" : " 14 GB", | 20260506 02:55:27.283 | INFO | "pinned" : " 29 GB" | 20260506 02:55:27.283 | INFO | } | 20260506 02:55:27.283 | INFO | Perf report for the last 5.14342 seconds: | 20260506 02:55:27.283 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:55:27.283 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:27.283 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:27.283 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:27.283 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260506 02:55:27.283 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:27.283 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:27.283 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260506 02:55:27.283 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:27.283 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:27.283 | INFO | Uploaded 36GiB to GPU in 2.11787s (16.9982GiB/s) | 20260506 02:55:27.283 | INFO | Downloaded 34GiB to GPU in 2.10339s (16.1644GiB/s) | 20260506 02:55:32.413 | INFO | Analyzing chunk frames = [8392192,8392704) | 20260506 02:55:32.413 | INFO | MemInfo: { | 20260506 02:55:32.413 | INFO | "gpu" : " 17 GB", | 20260506 02:55:32.413 | INFO | "huge" : "", | 20260506 02:55:32.413 | INFO | "huge_shared" : " 27 GB", | 20260506 02:55:32.414 | INFO | "malloc" : " 14 GB", | 20260506 02:55:32.414 | INFO | "pinned" : " 29 GB" | 20260506 02:55:32.414 | INFO | } | 20260506 02:55:32.414 | INFO | Perf report for the last 5.13058 seconds: | 20260506 02:55:32.414 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:55:32.414 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:32.414 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:32.414 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:32.414 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 02:55:32.414 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:32.414 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:32.414 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:55:32.414 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:32.414 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:32.414 | INFO | Uploaded 36GiB to GPU in 2.07037s (17.3882GiB/s) | 20260506 02:55:32.414 | INFO | Downloaded 34GiB to GPU in 2.02551s (16.7859GiB/s) | 20260506 02:55:32.539 | INFO | Starting a round of Baz IO | 20260506 02:55:35.673 | INFO | Populating RTMetrics report | 20260506 02:55:35.673 | INFO | RTMetrics report populated | 20260506 02:55:35.673 | INFO | Opening tmp RTMetrics json file | 20260506 02:55:35.674 | INFO | Serializing RTMetrics to json | 20260506 02:55:35.675 | INFO | Serialization done, swapping tmp file now... | 20260506 02:55:35.675 | INFO | RTMetrics done | 20260506 02:55:35.746 | INFO | Finished a round of Baz IO: Wrote 1.777 GiB in 3.206 seconds | 20260506 02:55:35.746 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.25% / 0.00% / 78.75% | 20260506 02:55:35.746 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0654578% / 0% / 7.15256e-05% | 20260506 02:55:35.746 | INFO | Baz IO overall: | 20260506 02:55:35.746 | INFO | BytesWritten : 4686571154164 | 20260506 02:55:35.747 | INFO | NumEvents : 2199050017262 | 20260506 02:55:37.554 | INFO | Analyzing chunk frames = [8392704,8393216) | 20260506 02:55:37.554 | INFO | MemInfo: { | 20260506 02:55:37.554 | INFO | "gpu" : " 17 GB", | 20260506 02:55:37.554 | INFO | "huge" : "", | 20260506 02:55:37.554 | INFO | "huge_shared" : " 27 GB", | 20260506 02:55:37.554 | INFO | "malloc" : " 14 GB", | 20260506 02:55:37.554 | INFO | "pinned" : " 29 GB" | 20260506 02:55:37.554 | INFO | } | 20260506 02:55:37.554 | INFO | Perf report for the last 5.14007 seconds: | 20260506 02:55:37.554 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:55:37.554 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:37.554 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:37.554 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:37.554 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260506 02:55:37.554 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:37.554 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260506 02:55:37.554 | INFO | PRE_HQ : 59.4%, 1.7, 53.2ms, 0 | 20260506 02:55:37.554 | INFO | BAZWRITER : 61.6%, 3.5, 109.3ms, 0 | 20260506 02:55:37.554 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:55:37.554 | INFO | Uploaded 36GiB to GPU in 2.17931s (16.519GiB/s) | 20260506 02:55:37.554 | INFO | Downloaded 38GiB to GPU in 2.23724s (16.9852GiB/s) | 20260506 02:55:42.700 | INFO | Analyzing chunk frames = [8393216,8393728) | 20260506 02:55:42.700 | INFO | MemInfo: { | 20260506 02:55:42.700 | INFO | "gpu" : " 17 GB", | 20260506 02:55:42.700 | INFO | "huge" : "", | 20260506 02:55:42.700 | INFO | "huge_shared" : " 27 GB", | 20260506 02:55:42.700 | INFO | "malloc" : " 14 GB", | 20260506 02:55:42.700 | INFO | "pinned" : " 29 GB" | 20260506 02:55:42.700 | INFO | } | 20260506 02:55:42.700 | INFO | Perf report for the last 5.14662 seconds: | 20260506 02:55:42.700 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:55:42.700 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:42.700 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:42.700 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:42.700 | INFO | ANALYSIS : 60.8%, 3.0, 95.1ms, 0 | 20260506 02:55:42.700 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:42.700 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:42.700 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 02:55:42.700 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:42.701 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:42.701 | INFO | Uploaded 36GiB to GPU in 1.96045s (18.3631GiB/s) | 20260506 02:55:42.701 | INFO | Downloaded 34GiB to GPU in 1.99362s (17.0544GiB/s) | 20260506 02:55:47.840 | INFO | Analyzing chunk frames = [8393728,8394240) | 20260506 02:55:47.840 | INFO | MemInfo: { | 20260506 02:55:47.840 | INFO | "gpu" : " 17 GB", | 20260506 02:55:47.840 | INFO | "huge" : "", | 20260506 02:55:47.840 | INFO | "huge_shared" : " 27 GB", | 20260506 02:55:47.840 | INFO | "malloc" : " 14 GB", | 20260506 02:55:47.840 | INFO | "pinned" : " 29 GB" | 20260506 02:55:47.840 | INFO | } | 20260506 02:55:47.840 | INFO | Perf report for the last 5.13945 seconds: | 20260506 02:55:47.840 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:55:47.840 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:47.840 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:47.840 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:47.840 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 02:55:47.840 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:47.840 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:47.840 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:55:47.840 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:47.840 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:47.840 | INFO | Uploaded 36GiB to GPU in 2.10609s (17.0933GiB/s) | 20260506 02:55:47.840 | INFO | Downloaded 34GiB to GPU in 2.03718s (16.6897GiB/s) | 20260506 02:55:52.975 | INFO | Analyzing chunk frames = [8394240,8394752) | 20260506 02:55:52.975 | INFO | MemInfo: { | 20260506 02:55:52.975 | INFO | "gpu" : " 17 GB", | 20260506 02:55:52.975 | INFO | "huge" : "", | 20260506 02:55:52.975 | INFO | "huge_shared" : " 27 GB", | 20260506 02:55:52.975 | INFO | "malloc" : " 14 GB", | 20260506 02:55:52.975 | INFO | "pinned" : " 29 GB" | 20260506 02:55:52.975 | INFO | } | 20260506 02:55:52.975 | INFO | Perf report for the last 5.13551 seconds: | 20260506 02:55:52.975 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:55:52.975 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:52.975 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:52.975 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:52.975 | INFO | ANALYSIS : 61.0%, 3.0, 94.6ms, 0 | 20260506 02:55:52.975 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:52.975 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:52.975 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 02:55:52.976 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:52.976 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:52.976 | INFO | Uploaded 36GiB to GPU in 2.08272s (17.2851GiB/s) | 20260506 02:55:52.976 | INFO | Downloaded 34GiB to GPU in 2.01325s (16.8881GiB/s) | 20260506 02:55:58.113 | INFO | Analyzing chunk frames = [8394752,8395264) | 20260506 02:55:58.113 | INFO | MemInfo: { | 20260506 02:55:58.113 | INFO | "gpu" : " 17 GB", | 20260506 02:55:58.113 | INFO | "huge" : "", | 20260506 02:55:58.113 | INFO | "huge_shared" : " 27 GB", | 20260506 02:55:58.113 | INFO | "malloc" : " 14 GB", | 20260506 02:55:58.113 | INFO | "pinned" : " 29 GB" | 20260506 02:55:58.113 | INFO | } | 20260506 02:55:58.113 | INFO | Perf report for the last 5.13749 seconds: | 20260506 02:55:58.113 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:55:58.113 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:58.113 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:58.113 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:58.113 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 02:55:58.113 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:58.113 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:58.113 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 02:55:58.113 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:58.113 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:55:58.113 | INFO | Uploaded 36GiB to GPU in 2.08341s (17.2794GiB/s) | 20260506 02:55:58.113 | INFO | Downloaded 34GiB to GPU in 2.03714s (16.6901GiB/s) | 20260506 02:56:03.250 | INFO | Analyzing chunk frames = [8395264,8395776) | 20260506 02:56:03.250 | INFO | MemInfo: { | 20260506 02:56:03.250 | INFO | "gpu" : " 17 GB", | 20260506 02:56:03.250 | INFO | "huge" : "", | 20260506 02:56:03.250 | INFO | "huge_shared" : " 27 GB", | 20260506 02:56:03.250 | INFO | "malloc" : " 14 GB", | 20260506 02:56:03.250 | INFO | "pinned" : " 29 GB" | 20260506 02:56:03.250 | INFO | } | 20260506 02:56:03.250 | INFO | Perf report for the last 5.13703 seconds: | 20260506 02:56:03.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:56:03.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:03.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:03.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:03.250 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 02:56:03.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:03.250 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:03.250 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:56:03.250 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:03.250 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:03.250 | INFO | Uploaded 36GiB to GPU in 2.07047s (17.3874GiB/s) | 20260506 02:56:03.250 | INFO | Downloaded 34GiB to GPU in 2.0383s (16.6806GiB/s) | 20260506 02:56:08.390 | INFO | Analyzing chunk frames = [8395776,8396288) | 20260506 02:56:08.390 | INFO | MemInfo: { | 20260506 02:56:08.390 | INFO | "gpu" : " 17 GB", | 20260506 02:56:08.390 | INFO | "huge" : "", | 20260506 02:56:08.390 | INFO | "huge_shared" : " 27 GB", | 20260506 02:56:08.390 | INFO | "malloc" : " 14 GB", | 20260506 02:56:08.390 | INFO | "pinned" : " 29 GB" | 20260506 02:56:08.390 | INFO | } | 20260506 02:56:08.390 | INFO | Perf report for the last 5.14004 seconds: | 20260506 02:56:08.390 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:56:08.390 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:08.390 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:08.390 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:08.390 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260506 02:56:08.390 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:08.390 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:08.390 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 02:56:08.390 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:08.390 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:08.390 | INFO | Uploaded 36GiB to GPU in 2.10642s (17.0906GiB/s) | 20260506 02:56:08.390 | INFO | Downloaded 34GiB to GPU in 2.08244s (16.327GiB/s) | 20260506 02:56:13.540 | INFO | Analyzing chunk frames = [8396288,8396800) | 20260506 02:56:13.540 | INFO | MemInfo: { | 20260506 02:56:13.540 | INFO | "gpu" : " 17 GB", | 20260506 02:56:13.540 | INFO | "huge" : "", | 20260506 02:56:13.540 | INFO | "huge_shared" : " 27 GB", | 20260506 02:56:13.540 | INFO | "malloc" : " 14 GB", | 20260506 02:56:13.540 | INFO | "pinned" : " 29 GB" | 20260506 02:56:13.540 | INFO | } | 20260506 02:56:13.540 | INFO | Perf report for the last 5.15038 seconds: | 20260506 02:56:13.540 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:56:13.540 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:13.540 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:13.540 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:13.540 | INFO | ANALYSIS : 61.2%, 3.0, 95.7ms, 0 | 20260506 02:56:13.540 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:13.540 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:13.540 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:56:13.540 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:13.541 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:13.541 | INFO | Uploaded 36GiB to GPU in 2.03039s (17.7306GiB/s) | 20260506 02:56:13.541 | INFO | Downloaded 34GiB to GPU in 2.02129s (16.8209GiB/s) | 20260506 02:56:13.541 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 02:56:13.541 | INFO | Total mem usage: 15.4 GiB | 20260506 02:56:13.541 | INFO | All Time High Water Marks sum: 16.3 GiB (Savings: 5.2%) | 20260506 02:56:13.541 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -4.32%) | 20260506 02:56:13.541 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 02:56:13.541 | INFO | High Water Breakdown: | 20260506 02:56:13.541 | INFO | 61.7% : 9.13 GiB ( 94.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 02:56:13.541 | INFO | 38.2% : 5.65 GiB ( 85% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 02:56:13.541 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 02:56:13.541 | INFO | ------------------------------------------ | 20260506 02:56:13.541 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:56:13.541 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 02:56:13.541 | INFO | Total mem usage: 26.5 GiB | 20260506 02:56:13.541 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 02:56:13.541 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 02:56:13.541 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260506 02:56:13.541 | INFO | High Water Breakdown: | 20260506 02:56:13.541 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:56:13.541 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:56:13.541 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:56:13.541 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:56:13.541 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:56:13.541 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:56:13.541 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 02:56:13.541 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 02:56:13.541 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:56:13.541 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 02:56:13.541 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 02:56:13.541 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 02:56:13.541 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 02:56:13.541 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 02:56:13.541 | INFO | ------------------------------------------ | 20260506 02:56:13.541 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:56:13.541 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 02:56:13.541 | INFO | Total mem usage: 25.6 GiB | 20260506 02:56:13.541 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 02:56:13.541 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 02:56:13.541 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 02:56:13.541 | INFO | High Water Breakdown: | 20260506 02:56:13.541 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 02:56:13.541 | INFO | ------------------------------------------ | 20260506 02:56:13.541 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:56:13.541 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 02:56:13.541 | INFO | Total mem usage: 15.7 GiB | 20260506 02:56:13.541 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 02:56:13.541 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 02:56:13.541 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 02:56:13.541 | INFO | High Water Breakdown: | 20260506 02:56:13.541 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 02:56:13.541 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 02:56:13.541 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 02:56:13.541 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 02:56:13.541 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 02:56:13.541 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 02:56:13.541 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 02:56:13.541 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 02:56:13.541 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 02:56:13.541 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:56:13.541 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 02:56:13.541 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:56:13.541 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 02:56:13.541 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 02:56:13.542 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 02:56:13.542 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 02:56:13.542 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:56:13.542 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 02:56:13.542 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:56:13.542 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:56:13.542 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:56:13.542 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 02:56:13.542 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 02:56:13.542 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:56:13.542 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 02:56:13.542 | INFO | ------------------------------------------ | 20260506 02:56:13.542 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:56:13.542 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 02:56:13.542 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:56:13.542 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:56:13.542 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.37e+01, 1.34e+01 / 5.84e-01 | 20260506 02:56:13.542 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.05e-01, 1.07e-02 / 4.12e-03 | 20260506 02:56:13.542 | INFO | PerfReport: 11.87%, Baseline -- 3.71e+00 / 4.53e+00, 5.84e-01 / 2.24e-01 | 20260506 02:56:13.542 | INFO | PerfReport: 3.30%, Histogram -- 1.03e+00 / 1.52e+00, 1.95e-01 / 1.04e-01 | 20260506 02:56:13.542 | INFO | PerfReport: 15.98%, DME -- 4.00e+01 / 4.38e+01, 3.78e+00 / 7.97e-01 | 20260506 02:56:13.542 | INFO | PerfReport: 57.54%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.54e+00 / 1.22e+00 | 20260506 02:56:13.542 | INFO | PerfReport: 6.99%, PulseAccumulating -- 2.19e+00 / 2.67e+00, 4.15e-01 / 2.82e-01 | 20260506 02:56:13.542 | INFO | PerfReport: 2.99%, Metrics -- 9.34e-01 / 1.19e+00, 2.52e-01 / 1.06e-01 | 20260506 02:56:13.542 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.23e-01 / 3.65e-01, 5.50e-02 / 2.03e-02 | 20260506 02:56:13.542 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 02:56:13.542 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:56:13.542 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:56:13.542 | INFO | PerfReport: Full Execution -- 9.75e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260506 02:56:13.542 | INFO | PerfReport: 22.21%, Upload -- 2.17e+01 / 0.00e+00, 7.23e+00 / 0.00e+00 | 20260506 02:56:13.542 | INFO | PerfReport: 21.97%, Download -- 2.14e+01 / 0.00e+00, 7.61e+00 / 0.00e+00 | 20260506 02:56:13.542 | INFO | PerfReport: 32.07%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 02:56:13.542 | INFO | PerfReport: 2.02%, LockUpload -- 1.97e+00 / 0.00e+00, 7.86e+00 / 0.00e+00 | 20260506 02:56:13.542 | INFO | PerfReport: 1.29%, LockDownload -- 1.25e+00 / 0.00e+00, 2.83e+00 / 0.00e+00 | 20260506 02:56:13.542 | INFO | PerfReport: 20.44%, LockCompute -- 1.99e+01 / 0.00e+00, 1.77e+01 / 0.00e+00 | 20260506 02:56:13.542 | INFO | PerfReport: 0.01%, Misc -- 1.27e-02 / 0.00e+00, 1.36e-02 / 0.00e+00 | 20260506 02:56:13.662 | INFO | Starting a round of Baz IO | 20260506 02:56:16.783 | INFO | Populating RTMetrics report | 20260506 02:56:16.784 | INFO | RTMetrics report populated | 20260506 02:56:16.784 | INFO | Opening tmp RTMetrics json file | 20260506 02:56:16.785 | INFO | Serializing RTMetrics to json | 20260506 02:56:16.786 | INFO | Serialization done, swapping tmp file now... | 20260506 02:56:16.786 | INFO | RTMetrics done | 20260506 02:56:16.858 | INFO | Finished a round of Baz IO: Wrote 1.775 GiB in 3.196 seconds | 20260506 02:56:16.858 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.25% / 0.00% / 78.75% | 20260506 02:56:16.858 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0644922% / 0% / 8.74201e-05% | 20260506 02:56:16.858 | INFO | Baz IO overall: | 20260506 02:56:16.858 | INFO | BytesWritten : 4688476770077 | 20260506 02:56:16.858 | INFO | NumEvents : 2199914326684 | 20260506 02:56:18.675 | INFO | Analyzing chunk frames = [8396800,8397312) | 20260506 02:56:18.675 | INFO | MemInfo: { | 20260506 02:56:18.675 | INFO | "gpu" : " 17 GB", | 20260506 02:56:18.675 | INFO | "huge" : "", | 20260506 02:56:18.675 | INFO | "huge_shared" : " 27 GB", | 20260506 02:56:18.675 | INFO | "malloc" : " 14 GB", | 20260506 02:56:18.675 | INFO | "pinned" : " 29 GB" | 20260506 02:56:18.675 | INFO | } | 20260506 02:56:18.675 | INFO | Perf report for the last 5.13501 seconds: | 20260506 02:56:18.675 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:56:18.675 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:18.675 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:18.675 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:18.675 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260506 02:56:18.676 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:18.676 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260506 02:56:18.676 | INFO | PRE_HQ : 59.1%, 1.7, 53.5ms, 0 | 20260506 02:56:18.676 | INFO | BAZWRITER : 61.4%, 3.5, 108.9ms, 0 | 20260506 02:56:18.676 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:56:18.676 | INFO | Uploaded 36GiB to GPU in 2.11676s (17.0071GiB/s) | 20260506 02:56:18.676 | INFO | Downloaded 38GiB to GPU in 2.22722s (17.0617GiB/s) | 20260506 02:56:23.814 | INFO | Analyzing chunk frames = [8397312,8397824) | 20260506 02:56:23.814 | INFO | MemInfo: { | 20260506 02:56:23.814 | INFO | "gpu" : " 17 GB", | 20260506 02:56:23.814 | INFO | "huge" : "", | 20260506 02:56:23.814 | INFO | "huge_shared" : " 27 GB", | 20260506 02:56:23.814 | INFO | "malloc" : " 14 GB", | 20260506 02:56:23.814 | INFO | "pinned" : " 29 GB" | 20260506 02:56:23.814 | INFO | } | 20260506 02:56:23.814 | INFO | Perf report for the last 5.13856 seconds: | 20260506 02:56:23.814 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:56:23.814 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:23.814 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:23.814 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:23.814 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260506 02:56:23.814 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:23.814 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:23.814 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 02:56:23.814 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:23.814 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:23.814 | INFO | Uploaded 36GiB to GPU in 1.97114s (18.2635GiB/s) | 20260506 02:56:23.814 | INFO | Downloaded 34GiB to GPU in 1.98665s (17.1143GiB/s) | 20260506 02:56:28.952 | INFO | Analyzing chunk frames = [8397824,8398336) | 20260506 02:56:28.952 | INFO | MemInfo: { | 20260506 02:56:28.952 | INFO | "gpu" : " 17 GB", | 20260506 02:56:28.952 | INFO | "huge" : "", | 20260506 02:56:28.952 | INFO | "huge_shared" : " 27 GB", | 20260506 02:56:28.952 | INFO | "malloc" : " 14 GB", | 20260506 02:56:28.952 | INFO | "pinned" : " 29 GB" | 20260506 02:56:28.952 | INFO | } | 20260506 02:56:28.952 | INFO | Perf report for the last 5.13817 seconds: | 20260506 02:56:28.952 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:56:28.952 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:28.952 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:28.952 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:28.952 | INFO | ANALYSIS : 61.2%, 3.0, 95.1ms, 0 | 20260506 02:56:28.952 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:28.952 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:28.952 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:56:28.952 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:28.952 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:28.952 | INFO | Uploaded 36GiB to GPU in 2.0807s (17.3019GiB/s) | 20260506 02:56:28.952 | INFO | Downloaded 34GiB to GPU in 2.0236s (16.8017GiB/s) | 20260506 02:56:34.089 | INFO | Analyzing chunk frames = [8398336,8398848) | 20260506 02:56:34.089 | INFO | MemInfo: { | 20260506 02:56:34.089 | INFO | "gpu" : " 17 GB", | 20260506 02:56:34.089 | INFO | "huge" : "", | 20260506 02:56:34.089 | INFO | "huge_shared" : " 27 GB", | 20260506 02:56:34.089 | INFO | "malloc" : " 14 GB", | 20260506 02:56:34.089 | INFO | "pinned" : " 29 GB" | 20260506 02:56:34.089 | INFO | } | 20260506 02:56:34.089 | INFO | Perf report for the last 5.13708 seconds: | 20260506 02:56:34.089 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:56:34.089 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:34.089 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:34.089 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:34.089 | INFO | ANALYSIS : 61.0%, 3.0, 94.7ms, 0 | 20260506 02:56:34.089 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:34.089 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:34.089 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 02:56:34.089 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:34.089 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:34.090 | INFO | Uploaded 36GiB to GPU in 2.0896s (17.2282GiB/s) | 20260506 02:56:34.090 | INFO | Downloaded 34GiB to GPU in 2.00919s (16.9223GiB/s) | 20260506 02:56:39.231 | INFO | Analyzing chunk frames = [8398848,8399360) | 20260506 02:56:39.231 | INFO | MemInfo: { | 20260506 02:56:39.231 | INFO | "gpu" : " 17 GB", | 20260506 02:56:39.231 | INFO | "huge" : "", | 20260506 02:56:39.231 | INFO | "huge_shared" : " 27 GB", | 20260506 02:56:39.231 | INFO | "malloc" : " 14 GB", | 20260506 02:56:39.231 | INFO | "pinned" : " 29 GB" | 20260506 02:56:39.231 | INFO | } | 20260506 02:56:39.232 | INFO | Perf report for the last 5.14233 seconds: | 20260506 02:56:39.232 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:56:39.232 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:39.232 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:39.232 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:39.232 | INFO | ANALYSIS : 60.7%, 3.0, 94.8ms, 0 | 20260506 02:56:39.232 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:39.232 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:39.232 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:56:39.232 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:39.232 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:39.232 | INFO | Uploaded 36GiB to GPU in 2.1064s (17.0908GiB/s) | 20260506 02:56:39.232 | INFO | Downloaded 34GiB to GPU in 2.03079s (16.7423GiB/s) | 20260506 02:56:44.378 | INFO | Analyzing chunk frames = [8399360,8399872) | 20260506 02:56:44.378 | INFO | MemInfo: { | 20260506 02:56:44.378 | INFO | "gpu" : " 17 GB", | 20260506 02:56:44.378 | INFO | "huge" : "", | 20260506 02:56:44.378 | INFO | "huge_shared" : " 27 GB", | 20260506 02:56:44.378 | INFO | "malloc" : " 14 GB", | 20260506 02:56:44.378 | INFO | "pinned" : " 29 GB" | 20260506 02:56:44.378 | INFO | } | 20260506 02:56:44.378 | INFO | Perf report for the last 5.14687 seconds: | 20260506 02:56:44.378 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:56:44.378 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:44.378 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:44.378 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:44.379 | INFO | ANALYSIS : 60.9%, 3.0, 95.2ms, 0 | 20260506 02:56:44.379 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:44.379 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:44.379 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:56:44.379 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:44.379 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:44.379 | INFO | Uploaded 36GiB to GPU in 2.08666s (17.2525GiB/s) | 20260506 02:56:44.379 | INFO | Downloaded 34GiB to GPU in 2.0395s (16.6708GiB/s) | 20260506 02:56:49.505 | INFO | Analyzing chunk frames = [8399872,8400384) | 20260506 02:56:49.505 | INFO | MemInfo: { | 20260506 02:56:49.505 | INFO | "gpu" : " 17 GB", | 20260506 02:56:49.505 | INFO | "huge" : "", | 20260506 02:56:49.505 | INFO | "huge_shared" : " 27 GB", | 20260506 02:56:49.505 | INFO | "malloc" : " 14 GB", | 20260506 02:56:49.505 | INFO | "pinned" : " 29 GB" | 20260506 02:56:49.505 | INFO | } | 20260506 02:56:49.505 | INFO | Perf report for the last 5.12648 seconds: | 20260506 02:56:49.505 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:56:49.505 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:49.505 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:49.505 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:49.505 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 02:56:49.505 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:49.505 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:49.505 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 02:56:49.505 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:49.505 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:49.505 | INFO | Uploaded 36GiB to GPU in 2.08223s (17.2892GiB/s) | 20260506 02:56:49.505 | INFO | Downloaded 34GiB to GPU in 2.06874s (16.4351GiB/s) | 20260506 02:56:54.648 | INFO | Analyzing chunk frames = [8400384,8400896) | 20260506 02:56:54.648 | INFO | MemInfo: { | 20260506 02:56:54.648 | INFO | "gpu" : " 17 GB", | 20260506 02:56:54.648 | INFO | "huge" : "", | 20260506 02:56:54.648 | INFO | "huge_shared" : " 27 GB", | 20260506 02:56:54.649 | INFO | "malloc" : " 14 GB", | 20260506 02:56:54.649 | INFO | "pinned" : " 29 GB" | 20260506 02:56:54.649 | INFO | } | 20260506 02:56:54.649 | INFO | Perf report for the last 5.14366 seconds: | 20260506 02:56:54.649 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:56:54.649 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:54.649 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:54.649 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:54.649 | INFO | ANALYSIS : 61.6%, 3.0, 96.3ms, 0 | 20260506 02:56:54.649 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:54.649 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:54.649 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:56:54.649 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:54.649 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:54.649 | INFO | Uploaded 36GiB to GPU in 2.12337s (16.9542GiB/s) | 20260506 02:56:54.649 | INFO | Downloaded 34GiB to GPU in 2.02885s (16.7583GiB/s) | 20260506 02:56:54.773 | INFO | Starting a round of Baz IO | 20260506 02:56:57.899 | INFO | Populating RTMetrics report | 20260506 02:56:57.900 | INFO | RTMetrics report populated | 20260506 02:56:57.900 | INFO | Opening tmp RTMetrics json file | 20260506 02:56:57.900 | INFO | Serializing RTMetrics to json | 20260506 02:56:57.902 | INFO | Serialization done, swapping tmp file now... | 20260506 02:56:57.902 | INFO | RTMetrics done | 20260506 02:56:57.986 | INFO | Finished a round of Baz IO: Wrote 1.775 GiB in 3.212 seconds | 20260506 02:56:57.986 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.25% / 0.00% / 78.75% | 20260506 02:56:57.986 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0649134% / 0% / 9.93411e-05% | 20260506 02:56:57.986 | INFO | Baz IO overall: | 20260506 02:56:57.986 | INFO | BytesWritten : 4690382406468 | 20260506 02:56:57.986 | INFO | NumEvents : 2200778918532 | 20260506 02:56:59.792 | INFO | Analyzing chunk frames = [8400896,8401408) | 20260506 02:56:59.792 | INFO | MemInfo: { | 20260506 02:56:59.792 | INFO | "gpu" : " 17 GB", | 20260506 02:56:59.792 | INFO | "huge" : "", | 20260506 02:56:59.792 | INFO | "huge_shared" : " 27 GB", | 20260506 02:56:59.792 | INFO | "malloc" : " 14 GB", | 20260506 02:56:59.792 | INFO | "pinned" : " 29 GB" | 20260506 02:56:59.792 | INFO | } | 20260506 02:56:59.792 | INFO | Perf report for the last 5.14366 seconds: | 20260506 02:56:59.792 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:56:59.792 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:59.792 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:59.792 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:59.793 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260506 02:56:59.793 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:56:59.793 | INFO | RT_METRICS : 42.8%, 1.0, 22.5ms, 0 | 20260506 02:56:59.793 | INFO | PRE_HQ : 59.2%, 1.7, 53.5ms, 0 | 20260506 02:56:59.793 | INFO | BAZWRITER : 61.6%, 3.6, 114.5ms, 0 | 20260506 02:56:59.793 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:56:59.793 | INFO | Uploaded 36GiB to GPU in 2.11526s (17.0192GiB/s) | 20260506 02:56:59.793 | INFO | Downloaded 38GiB to GPU in 2.23415s (17.0087GiB/s) | 20260506 02:57:04.926 | INFO | Analyzing chunk frames = [8401408,8401920) | 20260506 02:57:04.926 | INFO | MemInfo: { | 20260506 02:57:04.926 | INFO | "gpu" : " 17 GB", | 20260506 02:57:04.926 | INFO | "huge" : "", | 20260506 02:57:04.926 | INFO | "huge_shared" : " 27 GB", | 20260506 02:57:04.927 | INFO | "malloc" : " 14 GB", | 20260506 02:57:04.927 | INFO | "pinned" : " 29 GB" | 20260506 02:57:04.927 | INFO | } | 20260506 02:57:04.927 | INFO | Perf report for the last 5.13435 seconds: | 20260506 02:57:04.927 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:57:04.927 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:04.927 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:04.927 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:04.927 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 02:57:04.927 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:04.927 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:04.927 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 02:57:04.927 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:04.927 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:04.927 | INFO | Uploaded 36GiB to GPU in 2.01517s (17.8645GiB/s) | 20260506 02:57:04.927 | INFO | Downloaded 34GiB to GPU in 1.9871s (17.1104GiB/s) | 20260506 02:57:10.062 | INFO | Analyzing chunk frames = [8401920,8402432) | 20260506 02:57:10.062 | INFO | MemInfo: { | 20260506 02:57:10.062 | INFO | "gpu" : " 17 GB", | 20260506 02:57:10.062 | INFO | "huge" : "", | 20260506 02:57:10.062 | INFO | "huge_shared" : " 27 GB", | 20260506 02:57:10.062 | INFO | "malloc" : " 14 GB", | 20260506 02:57:10.062 | INFO | "pinned" : " 29 GB" | 20260506 02:57:10.062 | INFO | } | 20260506 02:57:10.062 | INFO | Perf report for the last 5.13549 seconds: | 20260506 02:57:10.062 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:57:10.062 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:10.062 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:10.062 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:10.062 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260506 02:57:10.062 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:10.062 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:10.062 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:57:10.062 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:10.062 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:10.062 | INFO | Uploaded 36GiB to GPU in 2.09648s (17.1716GiB/s) | 20260506 02:57:10.062 | INFO | Downloaded 34GiB to GPU in 2.03746s (16.6875GiB/s) | 20260506 02:57:15.207 | INFO | Analyzing chunk frames = [8402432,8402944) | 20260506 02:57:15.207 | INFO | MemInfo: { | 20260506 02:57:15.207 | INFO | "gpu" : " 17 GB", | 20260506 02:57:15.207 | INFO | "huge" : "", | 20260506 02:57:15.207 | INFO | "huge_shared" : " 27 GB", | 20260506 02:57:15.207 | INFO | "malloc" : " 14 GB", | 20260506 02:57:15.207 | INFO | "pinned" : " 29 GB" | 20260506 02:57:15.207 | INFO | } | 20260506 02:57:15.207 | INFO | Perf report for the last 5.14515 seconds: | 20260506 02:57:15.207 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:57:15.207 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:15.207 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:15.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:15.207 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 02:57:15.207 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:15.207 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:15.207 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 02:57:15.207 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:15.207 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:15.208 | INFO | Uploaded 36GiB to GPU in 2.13202s (16.8854GiB/s) | 20260506 02:57:15.208 | INFO | Downloaded 34GiB to GPU in 2.03361s (16.719GiB/s) | 20260506 02:57:20.346 | INFO | Analyzing chunk frames = [8402944,8403456) | 20260506 02:57:20.346 | INFO | MemInfo: { | 20260506 02:57:20.346 | INFO | "gpu" : " 17 GB", | 20260506 02:57:20.346 | INFO | "huge" : "", | 20260506 02:57:20.346 | INFO | "huge_shared" : " 27 GB", | 20260506 02:57:20.346 | INFO | "malloc" : " 14 GB", | 20260506 02:57:20.346 | INFO | "pinned" : " 29 GB" | 20260506 02:57:20.347 | INFO | } | 20260506 02:57:20.347 | INFO | Perf report for the last 5.13934 seconds: | 20260506 02:57:20.347 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:57:20.347 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:20.347 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:20.347 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:20.347 | INFO | ANALYSIS : 60.8%, 3.0, 94.9ms, 0 | 20260506 02:57:20.347 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:20.347 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:20.347 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 02:57:20.347 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:20.347 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:20.347 | INFO | Uploaded 36GiB to GPU in 2.1124s (17.0422GiB/s) | 20260506 02:57:20.347 | INFO | Downloaded 34GiB to GPU in 2.04023s (16.6648GiB/s) | 20260506 02:57:25.490 | INFO | Analyzing chunk frames = [8403456,8403968) | 20260506 02:57:25.490 | INFO | MemInfo: { | 20260506 02:57:25.490 | INFO | "gpu" : " 17 GB", | 20260506 02:57:25.490 | INFO | "huge" : "", | 20260506 02:57:25.490 | INFO | "huge_shared" : " 27 GB", | 20260506 02:57:25.491 | INFO | "malloc" : " 14 GB", | 20260506 02:57:25.491 | INFO | "pinned" : " 29 GB" | 20260506 02:57:25.491 | INFO | } | 20260506 02:57:25.491 | INFO | Perf report for the last 5.144 seconds: | 20260506 02:57:25.491 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:57:25.491 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:25.491 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:25.491 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:25.491 | INFO | ANALYSIS : 60.7%, 3.0, 94.8ms, 0 | 20260506 02:57:25.491 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:25.491 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:25.491 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 02:57:25.491 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:25.491 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:25.491 | INFO | Uploaded 36GiB to GPU in 2.05525s (17.5162GiB/s) | 20260506 02:57:25.491 | INFO | Downloaded 34GiB to GPU in 2.02347s (16.8028GiB/s) | 20260506 02:57:30.626 | INFO | Analyzing chunk frames = [8403968,8404480) | 20260506 02:57:30.626 | INFO | MemInfo: { | 20260506 02:57:30.626 | INFO | "gpu" : " 17 GB", | 20260506 02:57:30.626 | INFO | "huge" : "", | 20260506 02:57:30.626 | INFO | "huge_shared" : " 27 GB", | 20260506 02:57:30.626 | INFO | "malloc" : " 14 GB", | 20260506 02:57:30.626 | INFO | "pinned" : " 29 GB" | 20260506 02:57:30.627 | INFO | } | 20260506 02:57:30.627 | INFO | Perf report for the last 5.13592 seconds: | 20260506 02:57:30.627 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:57:30.627 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:30.627 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:30.627 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:30.627 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260506 02:57:30.627 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:30.627 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:30.627 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 02:57:30.627 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:30.627 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:30.627 | INFO | Uploaded 36GiB to GPU in 2.08143s (17.2958GiB/s) | 20260506 02:57:30.627 | INFO | Downloaded 34GiB to GPU in 2.06606s (16.4564GiB/s) | 20260506 02:57:35.759 | INFO | Analyzing chunk frames = [8404480,8404992) | 20260506 02:57:35.759 | INFO | MemInfo: { | 20260506 02:57:35.759 | INFO | "gpu" : " 17 GB", | 20260506 02:57:35.759 | INFO | "huge" : "", | 20260506 02:57:35.759 | INFO | "huge_shared" : " 27 GB", | 20260506 02:57:35.759 | INFO | "malloc" : " 14 GB", | 20260506 02:57:35.759 | INFO | "pinned" : " 29 GB" | 20260506 02:57:35.759 | INFO | } | 20260506 02:57:35.759 | INFO | Perf report for the last 5.13239 seconds: | 20260506 02:57:35.759 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:57:35.759 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:35.759 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:35.759 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:35.759 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 02:57:35.759 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:35.759 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:35.759 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 02:57:35.759 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:35.759 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:35.759 | INFO | Uploaded 36GiB to GPU in 2.02345s (17.7914GiB/s) | 20260506 02:57:35.759 | INFO | Downloaded 34GiB to GPU in 2.0158s (16.8668GiB/s) | 20260506 02:57:35.760 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 02:57:35.760 | INFO | Total mem usage: 15.4 GiB | 20260506 02:57:35.760 | INFO | All Time High Water Marks sum: 16.3 GiB (Savings: 5.2%) | 20260506 02:57:35.760 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -8.44%) | 20260506 02:57:35.760 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 02:57:35.760 | INFO | High Water Breakdown: | 20260506 02:57:35.760 | INFO | 61.1% : 8.69 GiB ( 90.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 02:57:35.760 | INFO | 38.8% : 5.52 GiB ( 83.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 02:57:35.760 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 02:57:35.760 | INFO | ------------------------------------------ | 20260506 02:57:35.760 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:57:35.760 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 02:57:35.760 | INFO | Total mem usage: 26.5 GiB | 20260506 02:57:35.760 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 02:57:35.760 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 02:57:35.760 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 02:57:35.760 | INFO | High Water Breakdown: | 20260506 02:57:35.760 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:57:35.760 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:57:35.760 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:57:35.760 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:57:35.760 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:57:35.760 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:57:35.760 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 02:57:35.760 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 02:57:35.760 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:57:35.760 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 02:57:35.760 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 02:57:35.760 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 02:57:35.760 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 02:57:35.760 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 02:57:35.760 | INFO | ------------------------------------------ | 20260506 02:57:35.760 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:57:35.760 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 02:57:35.760 | INFO | Total mem usage: 25.6 GiB | 20260506 02:57:35.760 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 02:57:35.760 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 02:57:35.760 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 02:57:35.760 | INFO | High Water Breakdown: | 20260506 02:57:35.760 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 02:57:35.760 | INFO | ------------------------------------------ | 20260506 02:57:35.760 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:57:35.760 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 02:57:35.760 | INFO | Total mem usage: 15.7 GiB | 20260506 02:57:35.760 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 02:57:35.760 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 02:57:35.760 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 02:57:35.760 | INFO | High Water Breakdown: | 20260506 02:57:35.760 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 02:57:35.760 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 02:57:35.760 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 02:57:35.760 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 02:57:35.760 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 02:57:35.760 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 02:57:35.760 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 02:57:35.760 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 02:57:35.760 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 02:57:35.760 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:57:35.760 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 02:57:35.760 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:57:35.760 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 02:57:35.760 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 02:57:35.760 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 02:57:35.760 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 02:57:35.760 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:57:35.760 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 02:57:35.760 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:57:35.760 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:57:35.760 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:57:35.760 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 02:57:35.760 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 02:57:35.760 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:57:35.760 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 02:57:35.760 | INFO | ------------------------------------------ | 20260506 02:57:35.760 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:57:35.760 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 02:57:35.760 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:57:35.760 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:57:35.760 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.36e+01, 1.34e+01 / 8.51e-01 | 20260506 02:57:35.760 | INFO | PerfReport: 0.29%, AggregateStats -- 9.17e-02 / 1.04e-01, 1.01e-02 / 2.72e-03 | 20260506 02:57:35.760 | INFO | PerfReport: 11.88%, Baseline -- 3.71e+00 / 4.53e+00, 5.84e-01 / 2.66e-01 | 20260506 02:57:35.760 | INFO | PerfReport: 3.30%, Histogram -- 1.03e+00 / 1.53e+00, 1.94e-01 / 8.17e-02 | 20260506 02:57:35.760 | INFO | PerfReport: 16.00%, DME -- 4.00e+01 / 4.38e+01, 3.73e+00 / 8.31e-01 | 20260506 02:57:35.761 | INFO | PerfReport: 57.47%, FrameLabeling -- 1.79e+01 / 1.93e+01, 1.50e+00 / 1.04e+00 | 20260506 02:57:35.761 | INFO | PerfReport: 7.02%, PulseAccumulating -- 2.19e+00 / 2.74e+00, 4.18e-01 / 2.11e-01 | 20260506 02:57:35.761 | INFO | PerfReport: 2.99%, Metrics -- 9.35e-01 / 1.20e+00, 2.51e-01 / 1.06e-01 | 20260506 02:57:35.761 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.23e-01 / 3.69e-01, 5.46e-02 / 2.93e-02 | 20260506 02:57:35.761 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 02:57:35.761 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:57:35.761 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:57:35.761 | INFO | PerfReport: Full Execution -- 9.75e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260506 02:57:35.761 | INFO | PerfReport: 22.22%, Upload -- 2.17e+01 / 0.00e+00, 7.27e+00 / 0.00e+00 | 20260506 02:57:35.761 | INFO | PerfReport: 21.93%, Download -- 2.14e+01 / 0.00e+00, 7.62e+00 / 0.00e+00 | 20260506 02:57:35.761 | INFO | PerfReport: 32.02%, Compute -- 3.12e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 02:57:35.761 | INFO | PerfReport: 2.00%, LockUpload -- 1.95e+00 / 0.00e+00, 7.81e+00 / 0.00e+00 | 20260506 02:57:35.761 | INFO | PerfReport: 1.29%, LockDownload -- 1.26e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260506 02:57:35.761 | INFO | PerfReport: 20.52%, LockCompute -- 2.00e+01 / 0.00e+00, 1.78e+01 / 0.00e+00 | 20260506 02:57:35.761 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.83e-02 / 0.00e+00 | 20260506 02:57:35.884 | INFO | Starting a round of Baz IO | 20260506 02:57:39.001 | INFO | Populating RTMetrics report | 20260506 02:57:39.001 | INFO | RTMetrics report populated | 20260506 02:57:39.001 | INFO | Opening tmp RTMetrics json file | 20260506 02:57:39.003 | INFO | Serializing RTMetrics to json | 20260506 02:57:39.004 | INFO | Serialization done, swapping tmp file now... | 20260506 02:57:39.004 | INFO | RTMetrics done | 20260506 02:57:39.089 | INFO | Finished a round of Baz IO: Wrote 1.774 GiB in 3.204 seconds | 20260506 02:57:39.089 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.25% / 0.00% / 78.75% | 20260506 02:57:39.089 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0650326% / 0% / 5.96046e-05% | 20260506 02:57:39.089 | INFO | Baz IO overall: | 20260506 02:57:39.089 | INFO | BytesWritten : 4692286985896 | 20260506 02:57:39.089 | INFO | NumEvents : 2201642624460 | 20260506 02:57:40.902 | INFO | Analyzing chunk frames = [8404992,8405504) | 20260506 02:57:40.902 | INFO | MemInfo: { | 20260506 02:57:40.902 | INFO | "gpu" : " 17 GB", | 20260506 02:57:40.902 | INFO | "huge" : "", | 20260506 02:57:40.902 | INFO | "huge_shared" : " 27 GB", | 20260506 02:57:40.902 | INFO | "malloc" : " 14 GB", | 20260506 02:57:40.902 | INFO | "pinned" : " 29 GB" | 20260506 02:57:40.902 | INFO | } | 20260506 02:57:40.903 | INFO | Perf report for the last 5.14364 seconds: | 20260506 02:57:40.903 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:57:40.903 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:40.903 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:40.903 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:40.903 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260506 02:57:40.903 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:40.903 | INFO | RT_METRICS : 42.6%, 1.0, 22.4ms, 0 | 20260506 02:57:40.903 | INFO | PRE_HQ : 59.0%, 1.7, 53.4ms, 0 | 20260506 02:57:40.903 | INFO | BAZWRITER : 61.5%, 3.6, 114.2ms, 0 | 20260506 02:57:40.903 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:57:40.903 | INFO | Uploaded 36GiB to GPU in 2.09406s (17.1915GiB/s) | 20260506 02:57:40.903 | INFO | Downloaded 38GiB to GPU in 2.22671s (17.0655GiB/s) | 20260506 02:57:46.042 | INFO | Analyzing chunk frames = [8405504,8406016) | 20260506 02:57:46.042 | INFO | MemInfo: { | 20260506 02:57:46.042 | INFO | "gpu" : " 17 GB", | 20260506 02:57:46.042 | INFO | "huge" : "", | 20260506 02:57:46.042 | INFO | "huge_shared" : " 27 GB", | 20260506 02:57:46.042 | INFO | "malloc" : " 14 GB", | 20260506 02:57:46.042 | INFO | "pinned" : " 29 GB" | 20260506 02:57:46.042 | INFO | } | 20260506 02:57:46.042 | INFO | Perf report for the last 5.13931 seconds: | 20260506 02:57:46.042 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:57:46.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:46.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:46.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:46.042 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260506 02:57:46.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:46.042 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:46.042 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 02:57:46.042 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:46.042 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:46.042 | INFO | Uploaded 36GiB to GPU in 1.9552s (18.4124GiB/s) | 20260506 02:57:46.042 | INFO | Downloaded 34GiB to GPU in 1.98731s (17.1086GiB/s) | 20260506 02:57:51.181 | INFO | Analyzing chunk frames = [8406016,8406528) | 20260506 02:57:51.181 | INFO | MemInfo: { | 20260506 02:57:51.181 | INFO | "gpu" : " 17 GB", | 20260506 02:57:51.181 | INFO | "huge" : "", | 20260506 02:57:51.181 | INFO | "huge_shared" : " 27 GB", | 20260506 02:57:51.181 | INFO | "malloc" : " 14 GB", | 20260506 02:57:51.181 | INFO | "pinned" : " 29 GB" | 20260506 02:57:51.181 | INFO | } | 20260506 02:57:51.181 | INFO | Perf report for the last 5.1394 seconds: | 20260506 02:57:51.181 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:57:51.181 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:51.181 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:51.181 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:51.181 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260506 02:57:51.181 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:51.182 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:51.182 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 02:57:51.182 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:51.182 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:51.182 | INFO | Uploaded 36GiB to GPU in 2.11214s (17.0444GiB/s) | 20260506 02:57:51.182 | INFO | Downloaded 34GiB to GPU in 2.04361s (16.6372GiB/s) | 20260506 02:57:56.326 | INFO | Analyzing chunk frames = [8406528,8407040) | 20260506 02:57:56.326 | INFO | MemInfo: { | 20260506 02:57:56.326 | INFO | "gpu" : " 17 GB", | 20260506 02:57:56.326 | INFO | "huge" : "", | 20260506 02:57:56.326 | INFO | "huge_shared" : " 27 GB", | 20260506 02:57:56.326 | INFO | "malloc" : " 14 GB", | 20260506 02:57:56.326 | INFO | "pinned" : " 29 GB" | 20260506 02:57:56.327 | INFO | } | 20260506 02:57:56.327 | INFO | Perf report for the last 5.14515 seconds: | 20260506 02:57:56.327 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:57:56.327 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:56.327 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:56.327 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:56.327 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 02:57:56.327 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:56.327 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:56.327 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:57:56.327 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:56.327 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:57:56.327 | INFO | Uploaded 36GiB to GPU in 2.12918s (16.9079GiB/s) | 20260506 02:57:56.327 | INFO | Downloaded 34GiB to GPU in 2.0375s (16.6871GiB/s) | 20260506 02:58:01.458 | INFO | Analyzing chunk frames = [8407040,8407552) | 20260506 02:58:01.458 | INFO | MemInfo: { | 20260506 02:58:01.458 | INFO | "gpu" : " 17 GB", | 20260506 02:58:01.458 | INFO | "huge" : "", | 20260506 02:58:01.458 | INFO | "huge_shared" : " 27 GB", | 20260506 02:58:01.458 | INFO | "malloc" : " 14 GB", | 20260506 02:58:01.458 | INFO | "pinned" : " 29 GB" | 20260506 02:58:01.458 | INFO | } | 20260506 02:58:01.458 | INFO | Perf report for the last 5.13141 seconds: | 20260506 02:58:01.458 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:58:01.458 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:01.458 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:01.458 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:01.458 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260506 02:58:01.458 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:01.458 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:01.458 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 02:58:01.458 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:01.458 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:01.458 | INFO | Uploaded 36GiB to GPU in 2.11461s (17.0244GiB/s) | 20260506 02:58:01.459 | INFO | Downloaded 34GiB to GPU in 2.03375s (16.7179GiB/s) | 20260506 02:58:06.602 | INFO | Analyzing chunk frames = [8407552,8408064) | 20260506 02:58:06.602 | INFO | MemInfo: { | 20260506 02:58:06.602 | INFO | "gpu" : " 17 GB", | 20260506 02:58:06.602 | INFO | "huge" : "", | 20260506 02:58:06.602 | INFO | "huge_shared" : " 27 GB", | 20260506 02:58:06.602 | INFO | "malloc" : " 14 GB", | 20260506 02:58:06.602 | INFO | "pinned" : " 29 GB" | 20260506 02:58:06.602 | INFO | } | 20260506 02:58:06.602 | INFO | Perf report for the last 5.14407 seconds: | 20260506 02:58:06.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:58:06.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:06.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:06.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:06.602 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 02:58:06.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:06.602 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:06.602 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:58:06.602 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:06.602 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:06.602 | INFO | Uploaded 36GiB to GPU in 2.10557s (17.0975GiB/s) | 20260506 02:58:06.602 | INFO | Downloaded 34GiB to GPU in 2.03955s (16.6703GiB/s) | 20260506 02:58:11.736 | INFO | Analyzing chunk frames = [8408064,8408576) | 20260506 02:58:11.736 | INFO | MemInfo: { | 20260506 02:58:11.736 | INFO | "gpu" : " 17 GB", | 20260506 02:58:11.736 | INFO | "huge" : "", | 20260506 02:58:11.736 | INFO | "huge_shared" : " 27 GB", | 20260506 02:58:11.736 | INFO | "malloc" : " 14 GB", | 20260506 02:58:11.736 | INFO | "pinned" : " 29 GB" | 20260506 02:58:11.736 | INFO | } | 20260506 02:58:11.736 | INFO | Perf report for the last 5.13399 seconds: | 20260506 02:58:11.736 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:58:11.736 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:11.736 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:11.736 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:11.736 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 02:58:11.736 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:11.736 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:11.736 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 02:58:11.736 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:11.736 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:11.736 | INFO | Uploaded 36GiB to GPU in 2.08889s (17.2341GiB/s) | 20260506 02:58:11.736 | INFO | Downloaded 34GiB to GPU in 2.06495s (16.4653GiB/s) | 20260506 02:58:16.874 | INFO | Analyzing chunk frames = [8408576,8409088) | 20260506 02:58:16.874 | INFO | MemInfo: { | 20260506 02:58:16.874 | INFO | "gpu" : " 17 GB", | 20260506 02:58:16.874 | INFO | "huge" : "", | 20260506 02:58:16.874 | INFO | "huge_shared" : " 27 GB", | 20260506 02:58:16.874 | INFO | "malloc" : " 14 GB", | 20260506 02:58:16.875 | INFO | "pinned" : " 29 GB" | 20260506 02:58:16.875 | INFO | } | 20260506 02:58:16.875 | INFO | Perf report for the last 5.13852 seconds: | 20260506 02:58:16.875 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:58:16.875 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:16.875 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:16.875 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:16.875 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260506 02:58:16.875 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:16.875 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:16.875 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 02:58:16.875 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:16.875 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:16.875 | INFO | Uploaded 36GiB to GPU in 2.058s (17.4927GiB/s) | 20260506 02:58:16.875 | INFO | Downloaded 34GiB to GPU in 2.02238s (16.8118GiB/s) | 20260506 02:58:17.000 | INFO | Starting a round of Baz IO | 20260506 02:58:20.129 | INFO | Populating RTMetrics report | 20260506 02:58:20.129 | INFO | RTMetrics report populated | 20260506 02:58:20.129 | INFO | Opening tmp RTMetrics json file | 20260506 02:58:20.130 | INFO | Serializing RTMetrics to json | 20260506 02:58:20.131 | INFO | Serialization done, swapping tmp file now... | 20260506 02:58:20.132 | INFO | RTMetrics done | 20260506 02:58:20.217 | INFO | Finished a round of Baz IO: Wrote 1.775 GiB in 3.216 seconds | 20260506 02:58:20.217 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.24% / 0.00% / 78.75% | 20260506 02:58:20.217 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.064671% / 7.94729e-06% / 5.96046e-05% | 20260506 02:58:20.217 | INFO | Baz IO overall: | 20260506 02:58:20.217 | INFO | BytesWritten : 4694193211525 | 20260506 02:58:20.217 | INFO | NumEvents : 2202507941531 | 20260506 02:58:22.013 | INFO | Analyzing chunk frames = [8409088,8409600) | 20260506 02:58:22.013 | INFO | MemInfo: { | 20260506 02:58:22.013 | INFO | "gpu" : " 17 GB", | 20260506 02:58:22.013 | INFO | "huge" : "", | 20260506 02:58:22.013 | INFO | "huge_shared" : " 27 GB", | 20260506 02:58:22.013 | INFO | "malloc" : " 14 GB", | 20260506 02:58:22.013 | INFO | "pinned" : " 29 GB" | 20260506 02:58:22.013 | INFO | } | 20260506 02:58:22.013 | INFO | Perf report for the last 5.13866 seconds: | 20260506 02:58:22.013 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:58:22.013 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:22.013 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:22.013 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:22.013 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260506 02:58:22.013 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:22.013 | INFO | RT_METRICS : 42.6%, 1.0, 22.4ms, 0 | 20260506 02:58:22.013 | INFO | PRE_HQ : 59.5%, 1.7, 53.4ms, 0 | 20260506 02:58:22.013 | INFO | BAZWRITER : 61.9%, 3.5, 112.8ms, 0 | 20260506 02:58:22.013 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 02:58:22.014 | INFO | Uploaded 36GiB to GPU in 2.18218s (16.4972GiB/s) | 20260506 02:58:22.014 | INFO | Downloaded 38GiB to GPU in 2.24171s (16.9513GiB/s) | 20260506 02:58:27.159 | INFO | Analyzing chunk frames = [8409600,8410112) | 20260506 02:58:27.159 | INFO | MemInfo: { | 20260506 02:58:27.159 | INFO | "gpu" : " 17 GB", | 20260506 02:58:27.159 | INFO | "huge" : "", | 20260506 02:58:27.159 | INFO | "huge_shared" : " 27 GB", | 20260506 02:58:27.159 | INFO | "malloc" : " 14 GB", | 20260506 02:58:27.159 | INFO | "pinned" : " 29 GB" | 20260506 02:58:27.159 | INFO | } | 20260506 02:58:27.159 | INFO | Perf report for the last 5.1461 seconds: | 20260506 02:58:27.159 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:58:27.159 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:27.159 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:27.159 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:27.159 | INFO | ANALYSIS : 60.7%, 3.0, 94.9ms, 0 | 20260506 02:58:27.159 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:27.159 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:27.159 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 02:58:27.160 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:27.160 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:27.160 | INFO | Uploaded 36GiB to GPU in 1.96339s (18.3356GiB/s) | 20260506 02:58:27.160 | INFO | Downloaded 34GiB to GPU in 1.98019s (17.1701GiB/s) | 20260506 02:58:32.299 | INFO | Analyzing chunk frames = [8410112,8410624) | 20260506 02:58:32.299 | INFO | MemInfo: { | 20260506 02:58:32.299 | INFO | "gpu" : " 17 GB", | 20260506 02:58:32.299 | INFO | "huge" : "", | 20260506 02:58:32.299 | INFO | "huge_shared" : " 27 GB", | 20260506 02:58:32.300 | INFO | "malloc" : " 14 GB", | 20260506 02:58:32.300 | INFO | "pinned" : " 29 GB" | 20260506 02:58:32.300 | INFO | } | 20260506 02:58:32.300 | INFO | Perf report for the last 5.1402 seconds: | 20260506 02:58:32.300 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:58:32.300 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:32.300 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:32.300 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:32.300 | INFO | ANALYSIS : 61.1%, 3.0, 94.4ms, 0 | 20260506 02:58:32.300 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:32.300 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:32.300 | INFO | PRE_HQ : 26.1%, 1.0, 13.4ms, 0 | 20260506 02:58:32.300 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:32.300 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:32.300 | INFO | Uploaded 36GiB to GPU in 2.11172s (17.0477GiB/s) | 20260506 02:58:32.300 | INFO | Downloaded 34GiB to GPU in 2.03671s (16.6936GiB/s) | 20260506 02:58:37.438 | INFO | Analyzing chunk frames = [8410624,8411136) | 20260506 02:58:37.438 | INFO | MemInfo: { | 20260506 02:58:37.438 | INFO | "gpu" : " 17 GB", | 20260506 02:58:37.438 | INFO | "huge" : "", | 20260506 02:58:37.439 | INFO | "huge_shared" : " 27 GB", | 20260506 02:58:37.439 | INFO | "malloc" : " 14 GB", | 20260506 02:58:37.439 | INFO | "pinned" : " 29 GB" | 20260506 02:58:37.439 | INFO | } | 20260506 02:58:37.439 | INFO | Perf report for the last 5.13913 seconds: | 20260506 02:58:37.439 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:58:37.439 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:37.439 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:37.439 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:37.439 | INFO | ANALYSIS : 61.0%, 3.0, 94.7ms, 0 | 20260506 02:58:37.439 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:37.439 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:37.439 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:58:37.439 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:37.439 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:37.439 | INFO | Uploaded 36GiB to GPU in 2.0875s (17.2455GiB/s) | 20260506 02:58:37.439 | INFO | Downloaded 34GiB to GPU in 2.00237s (16.9799GiB/s) | 20260506 02:58:42.573 | INFO | Analyzing chunk frames = [8411136,8411648) | 20260506 02:58:42.573 | INFO | MemInfo: { | 20260506 02:58:42.573 | INFO | "gpu" : " 17 GB", | 20260506 02:58:42.573 | INFO | "huge" : "", | 20260506 02:58:42.573 | INFO | "huge_shared" : " 27 GB", | 20260506 02:58:42.573 | INFO | "malloc" : " 14 GB", | 20260506 02:58:42.573 | INFO | "pinned" : " 29 GB" | 20260506 02:58:42.573 | INFO | } | 20260506 02:58:42.573 | INFO | Perf report for the last 5.13437 seconds: | 20260506 02:58:42.573 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:58:42.573 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:42.573 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:42.573 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:42.573 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 02:58:42.573 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:42.573 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:42.573 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 02:58:42.573 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:42.573 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:42.573 | INFO | Uploaded 36GiB to GPU in 2.10935s (17.0669GiB/s) | 20260506 02:58:42.573 | INFO | Downloaded 34GiB to GPU in 2.03141s (16.7371GiB/s) | 20260506 02:58:47.716 | INFO | Analyzing chunk frames = [8411648,8412160) | 20260506 02:58:47.716 | INFO | MemInfo: { | 20260506 02:58:47.716 | INFO | "gpu" : " 17 GB", | 20260506 02:58:47.716 | INFO | "huge" : "", | 20260506 02:58:47.716 | INFO | "huge_shared" : " 27 GB", | 20260506 02:58:47.717 | INFO | "malloc" : " 14 GB", | 20260506 02:58:47.717 | INFO | "pinned" : " 29 GB" | 20260506 02:58:47.717 | INFO | } | 20260506 02:58:47.717 | INFO | Perf report for the last 5.14355 seconds: | 20260506 02:58:47.717 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:58:47.717 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:47.717 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:47.717 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:47.717 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 02:58:47.717 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:47.717 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:47.717 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 02:58:47.717 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:47.717 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:47.717 | INFO | Uploaded 36GiB to GPU in 2.09548s (17.1798GiB/s) | 20260506 02:58:47.717 | INFO | Downloaded 34GiB to GPU in 2.03577s (16.7013GiB/s) | 20260506 02:58:52.854 | INFO | Analyzing chunk frames = [8412160,8412672) | 20260506 02:58:52.854 | INFO | MemInfo: { | 20260506 02:58:52.854 | INFO | "gpu" : " 17 GB", | 20260506 02:58:52.854 | INFO | "huge" : "", | 20260506 02:58:52.854 | INFO | "huge_shared" : " 27 GB", | 20260506 02:58:52.854 | INFO | "malloc" : " 14 GB", | 20260506 02:58:52.854 | INFO | "pinned" : " 29 GB" | 20260506 02:58:52.854 | INFO | } | 20260506 02:58:52.855 | INFO | Perf report for the last 5.13785 seconds: | 20260506 02:58:52.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:58:52.855 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:52.855 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:52.855 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:52.855 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 02:58:52.855 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:52.855 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:52.855 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 02:58:52.855 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:52.855 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:52.855 | INFO | Uploaded 36GiB to GPU in 2.07857s (17.3196GiB/s) | 20260506 02:58:52.855 | INFO | Downloaded 34GiB to GPU in 2.06332s (16.4783GiB/s) | 20260506 02:58:57.989 | INFO | Analyzing chunk frames = [8412672,8413184) | 20260506 02:58:57.989 | INFO | MemInfo: { | 20260506 02:58:57.989 | INFO | "gpu" : " 17 GB", | 20260506 02:58:57.989 | INFO | "huge" : "", | 20260506 02:58:57.989 | INFO | "huge_shared" : " 27 GB", | 20260506 02:58:57.989 | INFO | "malloc" : " 14 GB", | 20260506 02:58:57.989 | INFO | "pinned" : " 29 GB" | 20260506 02:58:57.989 | INFO | } | 20260506 02:58:57.989 | INFO | Perf report for the last 5.13468 seconds: | 20260506 02:58:57.989 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:58:57.989 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:57.989 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:57.989 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:57.989 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260506 02:58:57.989 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:57.989 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:57.989 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 02:58:57.989 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:57.989 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:58:57.990 | INFO | Uploaded 36GiB to GPU in 2.05426s (17.5246GiB/s) | 20260506 02:58:57.990 | INFO | Downloaded 34GiB to GPU in 2.03416s (16.7145GiB/s) | 20260506 02:58:57.990 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 02:58:57.990 | INFO | Total mem usage: 15.4 GiB | 20260506 02:58:57.990 | INFO | All Time High Water Marks sum: 16.3 GiB (Savings: 5.2%) | 20260506 02:58:57.990 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -9.16%) | 20260506 02:58:57.990 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 02:58:57.990 | INFO | High Water Breakdown: | 20260506 02:58:57.990 | INFO | 61% : 8.62 GiB ( 89.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 02:58:57.990 | INFO | 38.9% : 5.5 GiB ( 82.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 02:58:57.990 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 02:58:57.990 | INFO | ------------------------------------------ | 20260506 02:58:57.990 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:58:57.990 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 02:58:57.990 | INFO | Total mem usage: 26.5 GiB | 20260506 02:58:57.990 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 02:58:57.990 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 02:58:57.990 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 02:58:57.990 | INFO | High Water Breakdown: | 20260506 02:58:57.990 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:58:57.990 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:58:57.990 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:58:57.990 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:58:57.990 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:58:57.990 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:58:57.990 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 02:58:57.990 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 02:58:57.990 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:58:57.990 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 02:58:57.990 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 02:58:57.990 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 02:58:57.990 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 02:58:57.990 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 02:58:57.990 | INFO | ------------------------------------------ | 20260506 02:58:57.990 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:58:57.990 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 02:58:57.990 | INFO | Total mem usage: 25.6 GiB | 20260506 02:58:57.990 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 02:58:57.990 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 02:58:57.990 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 02:58:57.990 | INFO | High Water Breakdown: | 20260506 02:58:57.990 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 02:58:57.990 | INFO | ------------------------------------------ | 20260506 02:58:57.990 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:58:57.990 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 02:58:57.990 | INFO | Total mem usage: 15.7 GiB | 20260506 02:58:57.990 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 02:58:57.990 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 02:58:57.990 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 02:58:57.990 | INFO | High Water Breakdown: | 20260506 02:58:57.990 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 02:58:57.990 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 02:58:57.990 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 02:58:57.990 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 02:58:57.990 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 02:58:57.990 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 02:58:57.990 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 02:58:57.990 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 02:58:57.990 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 02:58:57.990 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 02:58:57.990 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 02:58:57.990 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 02:58:57.990 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 02:58:57.990 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 02:58:57.990 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 02:58:57.990 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 02:58:57.990 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 02:58:57.990 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 02:58:57.990 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 02:58:57.990 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 02:58:57.990 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 02:58:57.990 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 02:58:57.990 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 02:58:57.990 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 02:58:57.990 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 02:58:57.990 | INFO | ------------------------------------------ | 20260506 02:58:57.990 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 02:58:57.990 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 02:58:57.990 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:58:57.990 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:58:57.990 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.38e+01, 1.32e+01 / 6.85e-01 | 20260506 02:58:57.990 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.04e-01, 1.07e-02 / 4.39e-03 | 20260506 02:58:57.990 | INFO | PerfReport: 11.85%, Baseline -- 3.71e+00 / 4.52e+00, 5.80e-01 / 2.17e-01 | 20260506 02:58:57.990 | INFO | PerfReport: 3.31%, Histogram -- 1.04e+00 / 1.52e+00, 1.96e-01 / 8.76e-02 | 20260506 02:58:57.990 | INFO | PerfReport: 15.97%, DME -- 4.00e+01 / 4.39e+01, 3.87e+00 / 7.62e-01 | 20260506 02:58:57.990 | INFO | PerfReport: 57.58%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.59e+00 / 1.06e+00 | 20260506 02:58:57.990 | INFO | PerfReport: 6.98%, PulseAccumulating -- 2.18e+00 / 2.71e+00, 4.15e-01 / 2.13e-01 | 20260506 02:58:57.990 | INFO | PerfReport: 2.99%, Metrics -- 9.34e-01 / 1.21e+00, 2.51e-01 / 1.27e-01 | 20260506 02:58:57.990 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.23e-01 / 3.68e-01, 5.51e-02 / 3.18e-02 | 20260506 02:58:57.990 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 02:58:57.990 | INFO | PerfReport: Gathered 1536 samples. | 20260506 02:58:57.990 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 02:58:57.991 | INFO | PerfReport: Full Execution -- 9.75e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260506 02:58:57.991 | INFO | PerfReport: 22.26%, Upload -- 2.17e+01 / 0.00e+00, 7.25e+00 / 0.00e+00 | 20260506 02:58:57.991 | INFO | PerfReport: 21.95%, Download -- 2.14e+01 / 0.00e+00, 7.63e+00 / 0.00e+00 | 20260506 02:58:57.991 | INFO | PerfReport: 32.09%, Compute -- 3.13e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260506 02:58:57.991 | INFO | PerfReport: 1.97%, LockUpload -- 1.92e+00 / 0.00e+00, 7.71e+00 / 0.00e+00 | 20260506 02:58:57.991 | INFO | PerfReport: 1.24%, LockDownload -- 1.21e+00 / 0.00e+00, 2.85e+00 / 0.00e+00 | 20260506 02:58:57.991 | INFO | PerfReport: 20.47%, LockCompute -- 2.00e+01 / 0.00e+00, 1.75e+01 / 0.00e+00 | 20260506 02:58:57.991 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.53e-02 / 0.00e+00 | 20260506 02:58:58.115 | INFO | Starting a round of Baz IO | 20260506 02:59:01.249 | INFO | Populating RTMetrics report | 20260506 02:59:01.249 | INFO | RTMetrics report populated | 20260506 02:59:01.249 | INFO | Opening tmp RTMetrics json file | 20260506 02:59:01.254 | INFO | Serializing RTMetrics to json | 20260506 02:59:01.256 | INFO | Serialization done, swapping tmp file now... | 20260506 02:59:01.256 | INFO | RTMetrics done | 20260506 02:59:01.335 | INFO | Finished a round of Baz IO: Wrote 1.776 GiB in 3.22 seconds | 20260506 02:59:01.336 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.24% / 0.00% / 78.76% | 20260506 02:59:01.336 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0655095% / 0% / 5.5631e-05% | 20260506 02:59:01.336 | INFO | Baz IO overall: | 20260506 02:59:01.336 | INFO | BytesWritten : 4696099929181 | 20260506 02:59:01.336 | INFO | NumEvents : 2203373816375 | 20260506 02:59:03.129 | INFO | Analyzing chunk frames = [8413184,8413696) | 20260506 02:59:03.129 | INFO | MemInfo: { | 20260506 02:59:03.129 | INFO | "gpu" : " 17 GB", | 20260506 02:59:03.129 | INFO | "huge" : "", | 20260506 02:59:03.129 | INFO | "huge_shared" : " 27 GB", | 20260506 02:59:03.129 | INFO | "malloc" : " 14 GB", | 20260506 02:59:03.129 | INFO | "pinned" : " 29 GB" | 20260506 02:59:03.129 | INFO | } | 20260506 02:59:03.129 | INFO | Perf report for the last 5.13997 seconds: | 20260506 02:59:03.129 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:59:03.129 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:03.129 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:03.129 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:03.129 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260506 02:59:03.129 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:03.129 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260506 02:59:03.129 | INFO | PRE_HQ : 59.5%, 1.7, 53.3ms, 0 | 20260506 02:59:03.129 | INFO | BAZWRITER : 61.9%, 3.6, 113.6ms, 0 | 20260506 02:59:03.129 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 02:59:03.130 | INFO | Uploaded 36GiB to GPU in 2.18341s (16.488GiB/s) | 20260506 02:59:03.130 | INFO | Downloaded 38GiB to GPU in 2.23685s (16.9882GiB/s) | 20260506 02:59:08.272 | INFO | Analyzing chunk frames = [8413696,8414208) | 20260506 02:59:08.272 | INFO | MemInfo: { | 20260506 02:59:08.272 | INFO | "gpu" : " 17 GB", | 20260506 02:59:08.272 | INFO | "huge" : "", | 20260506 02:59:08.272 | INFO | "huge_shared" : " 27 GB", | 20260506 02:59:08.272 | INFO | "malloc" : " 14 GB", | 20260506 02:59:08.272 | INFO | "pinned" : " 29 GB" | 20260506 02:59:08.272 | INFO | } | 20260506 02:59:08.272 | INFO | Perf report for the last 5.14301 seconds: | 20260506 02:59:08.272 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:59:08.272 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:08.272 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:08.272 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:08.272 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 02:59:08.272 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:08.272 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:08.272 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 02:59:08.272 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:08.272 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:08.273 | INFO | Uploaded 36GiB to GPU in 1.98055s (18.1768GiB/s) | 20260506 02:59:08.273 | INFO | Downloaded 34GiB to GPU in 1.99295s (17.0602GiB/s) | 20260506 02:59:13.408 | INFO | Analyzing chunk frames = [8414208,8414720) | 20260506 02:59:13.408 | INFO | MemInfo: { | 20260506 02:59:13.408 | INFO | "gpu" : " 17 GB", | 20260506 02:59:13.408 | INFO | "huge" : "", | 20260506 02:59:13.408 | INFO | "huge_shared" : " 27 GB", | 20260506 02:59:13.409 | INFO | "malloc" : " 14 GB", | 20260506 02:59:13.409 | INFO | "pinned" : " 29 GB" | 20260506 02:59:13.409 | INFO | } | 20260506 02:59:13.409 | INFO | Perf report for the last 5.13644 seconds: | 20260506 02:59:13.409 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:59:13.409 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:13.409 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:13.409 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:13.409 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 02:59:13.409 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:13.409 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:13.409 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 02:59:13.409 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:13.409 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:13.409 | INFO | Uploaded 36GiB to GPU in 2.09001s (17.2248GiB/s) | 20260506 02:59:13.409 | INFO | Downloaded 34GiB to GPU in 2.02958s (16.7522GiB/s) | 20260506 02:59:18.556 | INFO | Analyzing chunk frames = [8414720,8415232) | 20260506 02:59:18.556 | INFO | MemInfo: { | 20260506 02:59:18.556 | INFO | "gpu" : " 17 GB", | 20260506 02:59:18.556 | INFO | "huge" : "", | 20260506 02:59:18.556 | INFO | "huge_shared" : " 27 GB", | 20260506 02:59:18.556 | INFO | "malloc" : " 14 GB", | 20260506 02:59:18.556 | INFO | "pinned" : " 29 GB" | 20260506 02:59:18.556 | INFO | } | 20260506 02:59:18.556 | INFO | Perf report for the last 5.14727 seconds: | 20260506 02:59:18.556 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:59:18.556 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:18.556 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:18.556 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:18.556 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260506 02:59:18.556 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:18.556 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:18.556 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:59:18.556 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:18.556 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:18.556 | INFO | Uploaded 36GiB to GPU in 2.08694s (17.2501GiB/s) | 20260506 02:59:18.556 | INFO | Downloaded 34GiB to GPU in 2.01106s (16.9065GiB/s) | 20260506 02:59:23.689 | INFO | Analyzing chunk frames = [8415232,8415744) | 20260506 02:59:23.689 | INFO | MemInfo: { | 20260506 02:59:23.689 | INFO | "gpu" : " 17 GB", | 20260506 02:59:23.689 | INFO | "huge" : "", | 20260506 02:59:23.689 | INFO | "huge_shared" : " 27 GB", | 20260506 02:59:23.689 | INFO | "malloc" : " 14 GB", | 20260506 02:59:23.689 | INFO | "pinned" : " 29 GB" | 20260506 02:59:23.689 | INFO | } | 20260506 02:59:23.689 | INFO | Perf report for the last 5.13343 seconds: | 20260506 02:59:23.689 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:59:23.689 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:23.689 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:23.689 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:23.689 | INFO | ANALYSIS : 60.8%, 3.0, 94.9ms, 0 | 20260506 02:59:23.689 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:23.690 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:23.690 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:59:23.690 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:23.690 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:23.690 | INFO | Uploaded 36GiB to GPU in 2.10684s (17.0872GiB/s) | 20260506 02:59:23.690 | INFO | Downloaded 34GiB to GPU in 2.04118s (16.657GiB/s) | 20260506 02:59:28.821 | INFO | Analyzing chunk frames = [8415744,8416256) | 20260506 02:59:28.821 | INFO | MemInfo: { | 20260506 02:59:28.821 | INFO | "gpu" : " 17 GB", | 20260506 02:59:28.821 | INFO | "huge" : "", | 20260506 02:59:28.822 | INFO | "huge_shared" : " 27 GB", | 20260506 02:59:28.822 | INFO | "malloc" : " 14 GB", | 20260506 02:59:28.822 | INFO | "pinned" : " 29 GB" | 20260506 02:59:28.822 | INFO | } | 20260506 02:59:28.822 | INFO | Perf report for the last 5.13224 seconds: | 20260506 02:59:28.822 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:59:28.822 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:28.822 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:28.822 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:28.822 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 02:59:28.822 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:28.822 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:28.822 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:59:28.822 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:28.822 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:28.822 | INFO | Uploaded 36GiB to GPU in 2.111s (17.0536GiB/s) | 20260506 02:59:28.822 | INFO | Downloaded 34GiB to GPU in 2.04232s (16.6477GiB/s) | 20260506 02:59:33.962 | INFO | Analyzing chunk frames = [8416256,8416768) | 20260506 02:59:33.962 | INFO | MemInfo: { | 20260506 02:59:33.962 | INFO | "gpu" : " 17 GB", | 20260506 02:59:33.962 | INFO | "huge" : "", | 20260506 02:59:33.962 | INFO | "huge_shared" : " 27 GB", | 20260506 02:59:33.962 | INFO | "malloc" : " 14 GB", | 20260506 02:59:33.962 | INFO | "pinned" : " 29 GB" | 20260506 02:59:33.963 | INFO | } | 20260506 02:59:33.963 | INFO | Perf report for the last 5.14089 seconds: | 20260506 02:59:33.963 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:59:33.963 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:33.963 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:33.963 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:33.963 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260506 02:59:33.963 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:33.963 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:33.963 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 02:59:33.963 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:33.963 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:33.963 | INFO | Uploaded 36GiB to GPU in 2.03232s (17.7137GiB/s) | 20260506 02:59:33.963 | INFO | Downloaded 34GiB to GPU in 2.05735s (16.5261GiB/s) | 20260506 02:59:39.113 | INFO | Analyzing chunk frames = [8416768,8417280) | 20260506 02:59:39.113 | INFO | MemInfo: { | 20260506 02:59:39.113 | INFO | "gpu" : " 17 GB", | 20260506 02:59:39.113 | INFO | "huge" : "", | 20260506 02:59:39.113 | INFO | "huge_shared" : " 27 GB", | 20260506 02:59:39.113 | INFO | "malloc" : " 14 GB", | 20260506 02:59:39.113 | INFO | "pinned" : " 29 GB" | 20260506 02:59:39.113 | INFO | } | 20260506 02:59:39.113 | INFO | Perf report for the last 5.15041 seconds: | 20260506 02:59:39.113 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:59:39.113 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:39.113 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:39.113 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:39.113 | INFO | ANALYSIS : 61.1%, 3.0, 95.6ms, 0 | 20260506 02:59:39.113 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:39.113 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:39.113 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 02:59:39.113 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:39.113 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:39.113 | INFO | Uploaded 36GiB to GPU in 2.04136s (17.6353GiB/s) | 20260506 02:59:39.113 | INFO | Downloaded 34GiB to GPU in 2.02134s (16.8205GiB/s) | 20260506 02:59:39.240 | INFO | Starting a round of Baz IO | 20260506 02:59:42.358 | INFO | Populating RTMetrics report | 20260506 02:59:42.359 | INFO | RTMetrics report populated | 20260506 02:59:42.359 | INFO | Opening tmp RTMetrics json file | 20260506 02:59:42.360 | INFO | Serializing RTMetrics to json | 20260506 02:59:42.361 | INFO | Serialization done, swapping tmp file now... | 20260506 02:59:42.361 | INFO | RTMetrics done | 20260506 02:59:42.439 | INFO | Finished a round of Baz IO: Wrote 1.775 GiB in 3.199 seconds | 20260506 02:59:42.439 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.24% / 0.00% / 78.76% | 20260506 02:59:42.439 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0650605% / 0% / 8.74201e-05% | 20260506 02:59:42.439 | INFO | Baz IO overall: | 20260506 02:59:42.439 | INFO | BytesWritten : 4698005817870 | 20260506 02:59:42.439 | INFO | NumEvents : 2204238912857 | 20260506 02:59:44.247 | INFO | Analyzing chunk frames = [8417280,8417792) | 20260506 02:59:44.247 | INFO | MemInfo: { | 20260506 02:59:44.247 | INFO | "gpu" : " 17 GB", | 20260506 02:59:44.247 | INFO | "huge" : "", | 20260506 02:59:44.247 | INFO | "huge_shared" : " 27 GB", | 20260506 02:59:44.248 | INFO | "malloc" : " 14 GB", | 20260506 02:59:44.248 | INFO | "pinned" : " 29 GB" | 20260506 02:59:44.248 | INFO | } | 20260506 02:59:44.248 | INFO | Perf report for the last 5.13466 seconds: | 20260506 02:59:44.248 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:59:44.248 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:44.248 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:44.248 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:44.248 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260506 02:59:44.248 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:44.248 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260506 02:59:44.248 | INFO | PRE_HQ : 59.1%, 1.7, 53.3ms, 0 | 20260506 02:59:44.248 | INFO | BAZWRITER : 61.6%, 3.6, 112.4ms, 0 | 20260506 02:59:44.248 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 02:59:44.248 | INFO | Uploaded 36GiB to GPU in 2.0898s (17.2265GiB/s) | 20260506 02:59:44.248 | INFO | Downloaded 38GiB to GPU in 2.22746s (17.0598GiB/s) | 20260506 02:59:49.386 | INFO | Analyzing chunk frames = [8417792,8418304) | 20260506 02:59:49.386 | INFO | MemInfo: { | 20260506 02:59:49.386 | INFO | "gpu" : " 17 GB", | 20260506 02:59:49.386 | INFO | "huge" : "", | 20260506 02:59:49.386 | INFO | "huge_shared" : " 27 GB", | 20260506 02:59:49.386 | INFO | "malloc" : " 14 GB", | 20260506 02:59:49.386 | INFO | "pinned" : " 29 GB" | 20260506 02:59:49.386 | INFO | } | 20260506 02:59:49.386 | INFO | Perf report for the last 5.1382 seconds: | 20260506 02:59:49.386 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:59:49.386 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:49.386 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:49.386 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:49.386 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 02:59:49.386 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:49.386 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:49.386 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 02:59:49.386 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:49.386 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:49.386 | INFO | Uploaded 36GiB to GPU in 1.94844s (18.4764GiB/s) | 20260506 02:59:49.386 | INFO | Downloaded 34GiB to GPU in 1.97922s (17.1785GiB/s) | 20260506 02:59:54.526 | INFO | Analyzing chunk frames = [8418304,8418816) | 20260506 02:59:54.526 | INFO | MemInfo: { | 20260506 02:59:54.526 | INFO | "gpu" : " 17 GB", | 20260506 02:59:54.526 | INFO | "huge" : "", | 20260506 02:59:54.526 | INFO | "huge_shared" : " 27 GB", | 20260506 02:59:54.527 | INFO | "malloc" : " 14 GB", | 20260506 02:59:54.527 | INFO | "pinned" : " 29 GB" | 20260506 02:59:54.527 | INFO | } | 20260506 02:59:54.527 | INFO | Perf report for the last 5.14082 seconds: | 20260506 02:59:54.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:59:54.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:54.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:54.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:54.527 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 02:59:54.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:54.527 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:54.527 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 02:59:54.527 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:54.527 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:54.527 | INFO | Uploaded 36GiB to GPU in 2.09002s (17.2247GiB/s) | 20260506 02:59:54.527 | INFO | Downloaded 34GiB to GPU in 2.02623s (16.7799GiB/s) | 20260506 02:59:59.668 | INFO | Analyzing chunk frames = [8418816,8419328) | 20260506 02:59:59.668 | INFO | MemInfo: { | 20260506 02:59:59.668 | INFO | "gpu" : " 17 GB", | 20260506 02:59:59.668 | INFO | "huge" : "", | 20260506 02:59:59.668 | INFO | "huge_shared" : " 27 GB", | 20260506 02:59:59.668 | INFO | "malloc" : " 14 GB", | 20260506 02:59:59.668 | INFO | "pinned" : " 29 GB" | 20260506 02:59:59.668 | INFO | } | 20260506 02:59:59.668 | INFO | Perf report for the last 5.14167 seconds: | 20260506 02:59:59.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 02:59:59.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:59.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:59.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:59.668 | INFO | ANALYSIS : 60.9%, 3.0, 94.6ms, 0 | 20260506 02:59:59.668 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:59.668 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:59.669 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 02:59:59.669 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:59.669 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 02:59:59.669 | INFO | Uploaded 36GiB to GPU in 2.08492s (17.2668GiB/s) | 20260506 02:59:59.669 | INFO | Downloaded 34GiB to GPU in 2.00527s (16.9553GiB/s) | 20260506 03:00:04.802 | INFO | Analyzing chunk frames = [8419328,8419840) | 20260506 03:00:04.802 | INFO | MemInfo: { | 20260506 03:00:04.802 | INFO | "gpu" : " 17 GB", | 20260506 03:00:04.802 | INFO | "huge" : "", | 20260506 03:00:04.802 | INFO | "huge_shared" : " 27 GB", | 20260506 03:00:04.802 | INFO | "malloc" : " 14 GB", | 20260506 03:00:04.802 | INFO | "pinned" : " 29 GB" | 20260506 03:00:04.802 | INFO | } | 20260506 03:00:04.803 | INFO | Perf report for the last 5.13413 seconds: | 20260506 03:00:04.803 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:00:04.803 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:04.803 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:04.803 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:04.803 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260506 03:00:04.803 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:04.803 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:04.803 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:00:04.803 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:04.803 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:04.803 | INFO | Uploaded 36GiB to GPU in 2.11952s (16.985GiB/s) | 20260506 03:00:04.803 | INFO | Downloaded 34GiB to GPU in 2.0429s (16.643GiB/s) | 20260506 03:00:09.939 | INFO | Analyzing chunk frames = [8419840,8420352) | 20260506 03:00:09.939 | INFO | MemInfo: { | 20260506 03:00:09.939 | INFO | "gpu" : " 17 GB", | 20260506 03:00:09.939 | INFO | "huge" : "", | 20260506 03:00:09.939 | INFO | "huge_shared" : " 27 GB", | 20260506 03:00:09.939 | INFO | "malloc" : " 14 GB", | 20260506 03:00:09.939 | INFO | "pinned" : " 29 GB" | 20260506 03:00:09.939 | INFO | } | 20260506 03:00:09.940 | INFO | Perf report for the last 5.13696 seconds: | 20260506 03:00:09.940 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:00:09.940 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:09.940 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:09.940 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:09.940 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 03:00:09.940 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:09.940 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:09.940 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 03:00:09.940 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:09.940 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:09.940 | INFO | Uploaded 36GiB to GPU in 2.10437s (17.1073GiB/s) | 20260506 03:00:09.940 | INFO | Downloaded 34GiB to GPU in 2.03985s (16.6679GiB/s) | 20260506 03:00:15.082 | INFO | Analyzing chunk frames = [8420352,8420864) | 20260506 03:00:15.082 | INFO | MemInfo: { | 20260506 03:00:15.082 | INFO | "gpu" : " 17 GB", | 20260506 03:00:15.082 | INFO | "huge" : "", | 20260506 03:00:15.082 | INFO | "huge_shared" : " 27 GB", | 20260506 03:00:15.082 | INFO | "malloc" : " 14 GB", | 20260506 03:00:15.082 | INFO | "pinned" : " 29 GB" | 20260506 03:00:15.082 | INFO | } | 20260506 03:00:15.082 | INFO | Perf report for the last 5.14254 seconds: | 20260506 03:00:15.082 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:00:15.082 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:15.082 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:15.082 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:15.082 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260506 03:00:15.082 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:15.082 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:15.082 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 03:00:15.082 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:15.082 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:15.082 | INFO | Uploaded 36GiB to GPU in 2.1015s (17.1306GiB/s) | 20260506 03:00:15.082 | INFO | Downloaded 34GiB to GPU in 2.08409s (16.3141GiB/s) | 20260506 03:00:20.223 | INFO | Analyzing chunk frames = [8420864,8421376) | 20260506 03:00:20.223 | INFO | MemInfo: { | 20260506 03:00:20.223 | INFO | "gpu" : " 17 GB", | 20260506 03:00:20.223 | INFO | "huge" : "", | 20260506 03:00:20.223 | INFO | "huge_shared" : " 27 GB", | 20260506 03:00:20.223 | INFO | "malloc" : " 14 GB", | 20260506 03:00:20.223 | INFO | "pinned" : " 29 GB" | 20260506 03:00:20.223 | INFO | } | 20260506 03:00:20.223 | INFO | Perf report for the last 5.14096 seconds: | 20260506 03:00:20.223 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:00:20.223 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:20.223 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:20.223 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:20.223 | INFO | ANALYSIS : 61.3%, 3.0, 95.8ms, 0 | 20260506 03:00:20.223 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:20.223 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:20.223 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 03:00:20.223 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:20.223 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:20.223 | INFO | Uploaded 36GiB to GPU in 2.04298s (17.6213GiB/s) | 20260506 03:00:20.223 | INFO | Downloaded 34GiB to GPU in 2.01632s (16.8624GiB/s) | 20260506 03:00:20.224 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 03:00:20.224 | INFO | Total mem usage: 15.4 GiB | 20260506 03:00:20.224 | INFO | All Time High Water Marks sum: 16.3 GiB (Savings: 5.2%) | 20260506 03:00:20.224 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -9.37%) | 20260506 03:00:20.224 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 03:00:20.224 | INFO | High Water Breakdown: | 20260506 03:00:20.224 | INFO | 60.9% : 8.6 GiB ( 89.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 03:00:20.224 | INFO | 38.9% : 5.5 GiB ( 82.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 03:00:20.224 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 03:00:20.224 | INFO | ------------------------------------------ | 20260506 03:00:20.224 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:00:20.224 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 03:00:20.224 | INFO | Total mem usage: 26.5 GiB | 20260506 03:00:20.224 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 03:00:20.224 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 03:00:20.224 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 03:00:20.224 | INFO | High Water Breakdown: | 20260506 03:00:20.224 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 03:00:20.224 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 03:00:20.224 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 03:00:20.224 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 03:00:20.224 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 03:00:20.224 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 03:00:20.224 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 03:00:20.224 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 03:00:20.224 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 03:00:20.224 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 03:00:20.224 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 03:00:20.224 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 03:00:20.224 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 03:00:20.224 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 03:00:20.224 | INFO | ------------------------------------------ | 20260506 03:00:20.224 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:00:20.224 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 03:00:20.224 | INFO | Total mem usage: 25.6 GiB | 20260506 03:00:20.224 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 03:00:20.224 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 03:00:20.224 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 03:00:20.224 | INFO | High Water Breakdown: | 20260506 03:00:20.224 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 03:00:20.224 | INFO | ------------------------------------------ | 20260506 03:00:20.224 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:00:20.224 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 03:00:20.224 | INFO | Total mem usage: 15.7 GiB | 20260506 03:00:20.224 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 03:00:20.224 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 03:00:20.224 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 03:00:20.224 | INFO | High Water Breakdown: | 20260506 03:00:20.224 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 03:00:20.224 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 03:00:20.224 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 03:00:20.224 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 03:00:20.224 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 03:00:20.224 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 03:00:20.224 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 03:00:20.224 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 03:00:20.224 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 03:00:20.224 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 03:00:20.224 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 03:00:20.224 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 03:00:20.224 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 03:00:20.224 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 03:00:20.224 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 03:00:20.224 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 03:00:20.224 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 03:00:20.224 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 03:00:20.224 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 03:00:20.224 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 03:00:20.224 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 03:00:20.224 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 03:00:20.224 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 03:00:20.224 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 03:00:20.224 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 03:00:20.224 | INFO | ------------------------------------------ | 20260506 03:00:20.224 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:00:20.224 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 03:00:20.224 | INFO | PerfReport: Gathered 1536 samples. | 20260506 03:00:20.224 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 03:00:20.224 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.39e+01, 1.33e+01 / 8.34e-01 | 20260506 03:00:20.224 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.04e-01, 1.06e-02 / 4.09e-03 | 20260506 03:00:20.224 | INFO | PerfReport: 11.85%, Baseline -- 3.71e+00 / 4.48e+00, 5.77e-01 / 2.58e-01 | 20260506 03:00:20.224 | INFO | PerfReport: 3.30%, Histogram -- 1.03e+00 / 1.52e+00, 1.92e-01 / 8.48e-02 | 20260506 03:00:20.224 | INFO | PerfReport: 16.01%, DME -- 4.00e+01 / 4.41e+01, 3.87e+00 / 1.12e+00 | 20260506 03:00:20.224 | INFO | PerfReport: 57.53%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.55e+00 / 7.44e-01 | 20260506 03:00:20.224 | INFO | PerfReport: 7.00%, PulseAccumulating -- 2.19e+00 / 2.73e+00, 4.15e-01 / 1.94e-01 | 20260506 03:00:20.224 | INFO | PerfReport: 2.99%, Metrics -- 9.34e-01 / 1.20e+00, 2.47e-01 / 1.17e-01 | 20260506 03:00:20.224 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.22e-01 / 3.68e-01, 5.39e-02 / 2.38e-02 | 20260506 03:00:20.224 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 03:00:20.224 | INFO | PerfReport: Gathered 1536 samples. | 20260506 03:00:20.224 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 03:00:20.224 | INFO | PerfReport: Full Execution -- 9.76e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260506 03:00:20.224 | INFO | PerfReport: 22.15%, Upload -- 2.16e+01 / 0.00e+00, 7.21e+00 / 0.00e+00 | 20260506 03:00:20.224 | INFO | PerfReport: 21.91%, Download -- 2.14e+01 / 0.00e+00, 7.59e+00 / 0.00e+00 | 20260506 03:00:20.224 | INFO | PerfReport: 32.04%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 03:00:20.225 | INFO | PerfReport: 2.03%, LockUpload -- 1.98e+00 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260506 03:00:20.225 | INFO | PerfReport: 1.31%, LockDownload -- 1.27e+00 / 0.00e+00, 2.95e+00 / 0.00e+00 | 20260506 03:00:20.225 | INFO | PerfReport: 20.55%, LockCompute -- 2.01e+01 / 0.00e+00, 1.78e+01 / 0.00e+00 | 20260506 03:00:20.225 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.70e-02 / 0.00e+00 | 20260506 03:00:20.351 | INFO | Starting a round of Baz IO | 20260506 03:00:23.468 | INFO | Populating RTMetrics report | 20260506 03:00:23.468 | INFO | RTMetrics report populated | 20260506 03:00:23.468 | INFO | Opening tmp RTMetrics json file | 20260506 03:00:23.469 | INFO | Serializing RTMetrics to json | 20260506 03:00:23.471 | INFO | Serialization done, swapping tmp file now... | 20260506 03:00:23.473 | INFO | RTMetrics done | 20260506 03:00:23.558 | INFO | Finished a round of Baz IO: Wrote 1.775 GiB in 3.207 seconds | 20260506 03:00:23.558 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.24% / 0.00% / 78.76% | 20260506 03:00:23.558 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0644962% / 3.97364e-06% / 8.34465e-05% | 20260506 03:00:23.558 | INFO | Baz IO overall: | 20260506 03:00:23.558 | INFO | BytesWritten : 4699911587156 | 20260506 03:00:23.558 | INFO | NumEvents : 2205104245023 | 20260506 03:00:25.366 | INFO | Analyzing chunk frames = [8421376,8421888) | 20260506 03:00:25.366 | INFO | MemInfo: { | 20260506 03:00:25.366 | INFO | "gpu" : " 17 GB", | 20260506 03:00:25.366 | INFO | "huge" : "", | 20260506 03:00:25.366 | INFO | "huge_shared" : " 27 GB", | 20260506 03:00:25.366 | INFO | "malloc" : " 14 GB", | 20260506 03:00:25.366 | INFO | "pinned" : " 29 GB" | 20260506 03:00:25.366 | INFO | } | 20260506 03:00:25.366 | INFO | Perf report for the last 5.14279 seconds: | 20260506 03:00:25.366 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:00:25.366 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:25.366 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:25.366 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:25.366 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260506 03:00:25.366 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:25.366 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260506 03:00:25.366 | INFO | PRE_HQ : 59.0%, 1.7, 53.2ms, 0 | 20260506 03:00:25.366 | INFO | BAZWRITER : 61.6%, 3.6, 113.5ms, 0 | 20260506 03:00:25.366 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 03:00:25.366 | INFO | Uploaded 36GiB to GPU in 2.11473s (17.0234GiB/s) | 20260506 03:00:25.366 | INFO | Downloaded 38GiB to GPU in 2.22411s (17.0855GiB/s) | 20260506 03:00:30.502 | INFO | Analyzing chunk frames = [8421888,8422400) | 20260506 03:00:30.502 | INFO | MemInfo: { | 20260506 03:00:30.502 | INFO | "gpu" : " 17 GB", | 20260506 03:00:30.502 | INFO | "huge" : "", | 20260506 03:00:30.502 | INFO | "huge_shared" : " 27 GB", | 20260506 03:00:30.502 | INFO | "malloc" : " 14 GB", | 20260506 03:00:30.502 | INFO | "pinned" : " 29 GB" | 20260506 03:00:30.502 | INFO | } | 20260506 03:00:30.503 | INFO | Perf report for the last 5.13669 seconds: | 20260506 03:00:30.503 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:00:30.503 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:30.503 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:30.503 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:30.503 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 03:00:30.503 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:30.503 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:30.503 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 03:00:30.503 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:30.503 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:30.503 | INFO | Uploaded 36GiB to GPU in 2.02191s (17.805GiB/s) | 20260506 03:00:30.503 | INFO | Downloaded 34GiB to GPU in 2.00096s (16.9918GiB/s) | 20260506 03:00:35.638 | INFO | Analyzing chunk frames = [8422400,8422912) | 20260506 03:00:35.638 | INFO | MemInfo: { | 20260506 03:00:35.638 | INFO | "gpu" : " 17 GB", | 20260506 03:00:35.638 | INFO | "huge" : "", | 20260506 03:00:35.638 | INFO | "huge_shared" : " 27 GB", | 20260506 03:00:35.638 | INFO | "malloc" : " 14 GB", | 20260506 03:00:35.638 | INFO | "pinned" : " 29 GB" | 20260506 03:00:35.638 | INFO | } | 20260506 03:00:35.638 | INFO | Perf report for the last 5.13586 seconds: | 20260506 03:00:35.638 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:00:35.638 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:35.638 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:35.638 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:35.638 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 03:00:35.639 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:35.639 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:35.639 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 03:00:35.639 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:35.639 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:35.639 | INFO | Uploaded 36GiB to GPU in 2.1176s (17.0004GiB/s) | 20260506 03:00:35.639 | INFO | Downloaded 34GiB to GPU in 2.04078s (16.6603GiB/s) | 20260506 03:00:40.774 | INFO | Analyzing chunk frames = [8422912,8423424) | 20260506 03:00:40.774 | INFO | MemInfo: { | 20260506 03:00:40.774 | INFO | "gpu" : " 17 GB", | 20260506 03:00:40.774 | INFO | "huge" : "", | 20260506 03:00:40.774 | INFO | "huge_shared" : " 27 GB", | 20260506 03:00:40.774 | INFO | "malloc" : " 14 GB", | 20260506 03:00:40.774 | INFO | "pinned" : " 29 GB" | 20260506 03:00:40.774 | INFO | } | 20260506 03:00:40.775 | INFO | Perf report for the last 5.13613 seconds: | 20260506 03:00:40.775 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:00:40.775 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:40.775 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:40.775 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:40.775 | INFO | ANALYSIS : 61.5%, 3.0, 95.5ms, 0 | 20260506 03:00:40.775 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:40.775 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:40.775 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:00:40.775 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:40.775 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:40.775 | INFO | Uploaded 36GiB to GPU in 2.14846s (16.7562GiB/s) | 20260506 03:00:40.775 | INFO | Downloaded 34GiB to GPU in 2.04704s (16.6094GiB/s) | 20260506 03:00:45.916 | INFO | Analyzing chunk frames = [8423424,8423936) | 20260506 03:00:45.916 | INFO | MemInfo: { | 20260506 03:00:45.916 | INFO | "gpu" : " 17 GB", | 20260506 03:00:45.916 | INFO | "huge" : "", | 20260506 03:00:45.916 | INFO | "huge_shared" : " 27 GB", | 20260506 03:00:45.916 | INFO | "malloc" : " 14 GB", | 20260506 03:00:45.916 | INFO | "pinned" : " 29 GB" | 20260506 03:00:45.916 | INFO | } | 20260506 03:00:45.916 | INFO | Perf report for the last 5.14186 seconds: | 20260506 03:00:45.916 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:00:45.916 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:45.916 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:45.917 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:45.917 | INFO | ANALYSIS : 60.6%, 3.0, 94.7ms, 0 | 20260506 03:00:45.917 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:45.917 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:45.917 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 03:00:45.917 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:45.917 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:45.917 | INFO | Uploaded 36GiB to GPU in 2.09517s (17.1824GiB/s) | 20260506 03:00:45.917 | INFO | Downloaded 34GiB to GPU in 2.0336s (16.7191GiB/s) | 20260506 03:00:51.062 | INFO | Analyzing chunk frames = [8423936,8424448) | 20260506 03:00:51.062 | INFO | MemInfo: { | 20260506 03:00:51.062 | INFO | "gpu" : " 17 GB", | 20260506 03:00:51.062 | INFO | "huge" : "", | 20260506 03:00:51.062 | INFO | "huge_shared" : " 27 GB", | 20260506 03:00:51.062 | INFO | "malloc" : " 14 GB", | 20260506 03:00:51.062 | INFO | "pinned" : " 29 GB" | 20260506 03:00:51.062 | INFO | } | 20260506 03:00:51.062 | INFO | Perf report for the last 5.14565 seconds: | 20260506 03:00:51.062 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:00:51.062 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:51.062 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:51.062 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:51.062 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 03:00:51.062 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:51.062 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:51.062 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 03:00:51.062 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:51.062 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:51.062 | INFO | Uploaded 36GiB to GPU in 2.09629s (17.1732GiB/s) | 20260506 03:00:51.062 | INFO | Downloaded 34GiB to GPU in 2.03698s (16.6914GiB/s) | 20260506 03:00:56.198 | INFO | Analyzing chunk frames = [8424448,8424960) | 20260506 03:00:56.198 | INFO | MemInfo: { | 20260506 03:00:56.198 | INFO | "gpu" : " 17 GB", | 20260506 03:00:56.198 | INFO | "huge" : "", | 20260506 03:00:56.198 | INFO | "huge_shared" : " 27 GB", | 20260506 03:00:56.198 | INFO | "malloc" : " 14 GB", | 20260506 03:00:56.198 | INFO | "pinned" : " 29 GB" | 20260506 03:00:56.198 | INFO | } | 20260506 03:00:56.198 | INFO | Perf report for the last 5.13625 seconds: | 20260506 03:00:56.198 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:00:56.198 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:56.198 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:56.198 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:56.198 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260506 03:00:56.198 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:56.198 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:56.198 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 03:00:56.199 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:56.199 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:00:56.199 | INFO | Uploaded 36GiB to GPU in 2.09606s (17.1751GiB/s) | 20260506 03:00:56.199 | INFO | Downloaded 34GiB to GPU in 2.0723s (16.4069GiB/s) | 20260506 03:01:01.334 | INFO | Analyzing chunk frames = [8424960,8425472) | 20260506 03:01:01.334 | INFO | MemInfo: { | 20260506 03:01:01.334 | INFO | "gpu" : " 17 GB", | 20260506 03:01:01.334 | INFO | "huge" : "", | 20260506 03:01:01.334 | INFO | "huge_shared" : " 27 GB", | 20260506 03:01:01.334 | INFO | "malloc" : " 14 GB", | 20260506 03:01:01.334 | INFO | "pinned" : " 29 GB" | 20260506 03:01:01.334 | INFO | } | 20260506 03:01:01.334 | INFO | Perf report for the last 5.13578 seconds: | 20260506 03:01:01.334 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:01:01.334 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:01.334 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:01.334 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:01.334 | INFO | ANALYSIS : 61.4%, 3.0, 94.8ms, 0 | 20260506 03:01:01.334 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:01.334 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:01.334 | INFO | PRE_HQ : 26.4%, 1.0, 13.6ms, 0 | 20260506 03:01:01.334 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:01.334 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:01.334 | INFO | Uploaded 36GiB to GPU in 2.05402s (17.5266GiB/s) | 20260506 03:01:01.334 | INFO | Downloaded 34GiB to GPU in 2.02477s (16.7921GiB/s) | 20260506 03:01:01.452 | INFO | Starting a round of Baz IO | 20260506 03:01:04.588 | INFO | Populating RTMetrics report | 20260506 03:01:04.588 | INFO | RTMetrics report populated | 20260506 03:01:04.588 | INFO | Opening tmp RTMetrics json file | 20260506 03:01:04.612 | INFO | Serializing RTMetrics to json | 20260506 03:01:04.682 | INFO | Serialization done, swapping tmp file now... | 20260506 03:01:04.703 | INFO | RTMetrics done | 20260506 03:01:04.716 | INFO | Finished a round of Baz IO: Wrote 1.775 GiB in 3.263 seconds | 20260506 03:01:04.716 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.24% / 0.00% / 78.76% | 20260506 03:01:04.716 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0647823% / 0% / 9.13938e-05% | 20260506 03:01:04.716 | INFO | Baz IO overall: | 20260506 03:01:04.716 | INFO | BytesWritten : 4701817019518 | 20260506 03:01:04.716 | INFO | NumEvents : 2205969374113 | 20260506 03:01:06.481 | INFO | Analyzing chunk frames = [8425472,8425984) | 20260506 03:01:06.481 | INFO | MemInfo: { | 20260506 03:01:06.481 | INFO | "gpu" : " 17 GB", | 20260506 03:01:06.481 | INFO | "huge" : "", | 20260506 03:01:06.481 | INFO | "huge_shared" : " 27 GB", | 20260506 03:01:06.481 | INFO | "malloc" : " 14 GB", | 20260506 03:01:06.481 | INFO | "pinned" : " 29 GB" | 20260506 03:01:06.481 | INFO | } | 20260506 03:01:06.481 | INFO | Perf report for the last 5.14723 seconds: | 20260506 03:01:06.481 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:01:06.481 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:06.481 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:06.481 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:06.481 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260506 03:01:06.481 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:06.482 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260506 03:01:06.482 | INFO | PRE_HQ : 59.4%, 1.7, 53.4ms, 0 | 20260506 03:01:06.482 | INFO | BAZWRITER : 62.5%, 3.9, 125.7ms, 0 | 20260506 03:01:06.482 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260506 03:01:06.482 | INFO | Uploaded 36GiB to GPU in 2.18673s (16.4629GiB/s) | 20260506 03:01:06.482 | INFO | Downloaded 38GiB to GPU in 2.25191s (16.8745GiB/s) | 20260506 03:01:11.615 | INFO | Analyzing chunk frames = [8425984,8426496) | 20260506 03:01:11.615 | INFO | MemInfo: { | 20260506 03:01:11.615 | INFO | "gpu" : " 17 GB", | 20260506 03:01:11.615 | INFO | "huge" : "", | 20260506 03:01:11.615 | INFO | "huge_shared" : " 27 GB", | 20260506 03:01:11.615 | INFO | "malloc" : " 14 GB", | 20260506 03:01:11.615 | INFO | "pinned" : " 29 GB" | 20260506 03:01:11.615 | INFO | } | 20260506 03:01:11.615 | INFO | Perf report for the last 5.13371 seconds: | 20260506 03:01:11.615 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:01:11.615 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:11.615 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:11.615 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:11.615 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260506 03:01:11.615 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:11.615 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:11.615 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 03:01:11.615 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:11.615 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:11.615 | INFO | Uploaded 36GiB to GPU in 1.95892s (18.3774GiB/s) | 20260506 03:01:11.615 | INFO | Downloaded 34GiB to GPU in 1.98496s (17.1288GiB/s) | 20260506 03:01:16.752 | INFO | Analyzing chunk frames = [8426496,8427008) | 20260506 03:01:16.752 | INFO | MemInfo: { | 20260506 03:01:16.752 | INFO | "gpu" : " 17 GB", | 20260506 03:01:16.752 | INFO | "huge" : "", | 20260506 03:01:16.752 | INFO | "huge_shared" : " 27 GB", | 20260506 03:01:16.752 | INFO | "malloc" : " 14 GB", | 20260506 03:01:16.752 | INFO | "pinned" : " 29 GB" | 20260506 03:01:16.752 | INFO | } | 20260506 03:01:16.752 | INFO | Perf report for the last 5.13674 seconds: | 20260506 03:01:16.752 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:01:16.752 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:16.752 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:16.752 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:16.752 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260506 03:01:16.752 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:16.752 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:16.752 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 03:01:16.752 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:16.752 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:16.752 | INFO | Uploaded 36GiB to GPU in 2.11751s (17.0011GiB/s) | 20260506 03:01:16.752 | INFO | Downloaded 34GiB to GPU in 2.04251s (16.6462GiB/s) | 20260506 03:01:21.893 | INFO | Analyzing chunk frames = [8427008,8427520) | 20260506 03:01:21.893 | INFO | MemInfo: { | 20260506 03:01:21.893 | INFO | "gpu" : " 17 GB", | 20260506 03:01:21.893 | INFO | "huge" : "", | 20260506 03:01:21.893 | INFO | "huge_shared" : " 27 GB", | 20260506 03:01:21.893 | INFO | "malloc" : " 14 GB", | 20260506 03:01:21.894 | INFO | "pinned" : " 29 GB" | 20260506 03:01:21.894 | INFO | } | 20260506 03:01:21.894 | INFO | Perf report for the last 5.14185 seconds: | 20260506 03:01:21.894 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:01:21.894 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:21.894 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:21.894 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:21.894 | INFO | ANALYSIS : 61.2%, 3.0, 95.1ms, 0 | 20260506 03:01:21.894 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:21.894 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:21.894 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:01:21.894 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:21.894 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:21.894 | INFO | Uploaded 36GiB to GPU in 2.09341s (17.1968GiB/s) | 20260506 03:01:21.894 | INFO | Downloaded 34GiB to GPU in 2.02191s (16.8158GiB/s) | 20260506 03:01:27.035 | INFO | Analyzing chunk frames = [8427520,8428032) | 20260506 03:01:27.035 | INFO | MemInfo: { | 20260506 03:01:27.035 | INFO | "gpu" : " 17 GB", | 20260506 03:01:27.035 | INFO | "huge" : "", | 20260506 03:01:27.035 | INFO | "huge_shared" : " 27 GB", | 20260506 03:01:27.035 | INFO | "malloc" : " 14 GB", | 20260506 03:01:27.035 | INFO | "pinned" : " 29 GB" | 20260506 03:01:27.035 | INFO | } | 20260506 03:01:27.035 | INFO | Perf report for the last 5.14147 seconds: | 20260506 03:01:27.035 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:01:27.035 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:27.035 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:27.035 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:27.035 | INFO | ANALYSIS : 60.5%, 3.0, 94.5ms, 0 | 20260506 03:01:27.035 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:27.035 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:27.035 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 03:01:27.035 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:27.035 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:27.035 | INFO | Uploaded 36GiB to GPU in 2.08506s (17.2657GiB/s) | 20260506 03:01:27.036 | INFO | Downloaded 34GiB to GPU in 2.02852s (16.761GiB/s) | 20260506 03:01:32.175 | INFO | Analyzing chunk frames = [8428032,8428544) | 20260506 03:01:32.175 | INFO | MemInfo: { | 20260506 03:01:32.175 | INFO | "gpu" : " 17 GB", | 20260506 03:01:32.175 | INFO | "huge" : "", | 20260506 03:01:32.175 | INFO | "huge_shared" : " 27 GB", | 20260506 03:01:32.175 | INFO | "malloc" : " 14 GB", | 20260506 03:01:32.175 | INFO | "pinned" : " 29 GB" | 20260506 03:01:32.175 | INFO | } | 20260506 03:01:32.175 | INFO | Perf report for the last 5.14023 seconds: | 20260506 03:01:32.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:01:32.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:32.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:32.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:32.176 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260506 03:01:32.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:32.176 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:32.176 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 03:01:32.176 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:32.176 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:32.176 | INFO | Uploaded 36GiB to GPU in 2.0566s (17.5046GiB/s) | 20260506 03:01:32.176 | INFO | Downloaded 34GiB to GPU in 2.02813s (16.7642GiB/s) | 20260506 03:01:37.316 | INFO | Analyzing chunk frames = [8428544,8429056) | 20260506 03:01:37.316 | INFO | MemInfo: { | 20260506 03:01:37.317 | INFO | "gpu" : " 17 GB", | 20260506 03:01:37.317 | INFO | "huge" : "", | 20260506 03:01:37.317 | INFO | "huge_shared" : " 27 GB", | 20260506 03:01:37.317 | INFO | "malloc" : " 14 GB", | 20260506 03:01:37.317 | INFO | "pinned" : " 29 GB" | 20260506 03:01:37.317 | INFO | } | 20260506 03:01:37.317 | INFO | Perf report for the last 5.14113 seconds: | 20260506 03:01:37.317 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:01:37.317 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:37.317 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:37.317 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:37.317 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 03:01:37.317 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:37.317 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:37.317 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 03:01:37.317 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:37.317 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:37.317 | INFO | Uploaded 36GiB to GPU in 2.09088s (17.2177GiB/s) | 20260506 03:01:37.317 | INFO | Downloaded 34GiB to GPU in 2.0629s (16.4816GiB/s) | 20260506 03:01:42.449 | INFO | Analyzing chunk frames = [8429056,8429568) | 20260506 03:01:42.449 | INFO | MemInfo: { | 20260506 03:01:42.449 | INFO | "gpu" : " 17 GB", | 20260506 03:01:42.449 | INFO | "huge" : "", | 20260506 03:01:42.449 | INFO | "huge_shared" : " 27 GB", | 20260506 03:01:42.449 | INFO | "malloc" : " 14 GB", | 20260506 03:01:42.449 | INFO | "pinned" : " 29 GB" | 20260506 03:01:42.449 | INFO | } | 20260506 03:01:42.449 | INFO | Perf report for the last 5.13222 seconds: | 20260506 03:01:42.449 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:01:42.449 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:42.449 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:42.449 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:42.449 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 03:01:42.449 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:42.449 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:42.449 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:01:42.449 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:42.449 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:42.449 | INFO | Uploaded 36GiB to GPU in 2.04202s (17.6296GiB/s) | 20260506 03:01:42.449 | INFO | Downloaded 34GiB to GPU in 2.01752s (16.8524GiB/s) | 20260506 03:01:42.449 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 03:01:42.449 | INFO | Total mem usage: 15.4 GiB | 20260506 03:01:42.449 | INFO | All Time High Water Marks sum: 16.3 GiB (Savings: 5.2%) | 20260506 03:01:42.449 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -5.99%) | 20260506 03:01:42.449 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 03:01:42.449 | INFO | High Water Breakdown: | 20260506 03:01:42.449 | INFO | 61.4% : 8.95 GiB ( 93% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 03:01:42.449 | INFO | 38.4% : 5.6 GiB ( 84.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 03:01:42.449 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 03:01:42.450 | INFO | ------------------------------------------ | 20260506 03:01:42.450 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:01:42.450 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 03:01:42.450 | INFO | Total mem usage: 26.5 GiB | 20260506 03:01:42.450 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 03:01:42.450 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 03:01:42.450 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 03:01:42.450 | INFO | High Water Breakdown: | 20260506 03:01:42.450 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 03:01:42.450 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 03:01:42.450 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 03:01:42.450 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 03:01:42.450 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 03:01:42.450 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 03:01:42.450 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 03:01:42.450 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 03:01:42.450 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 03:01:42.450 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 03:01:42.450 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 03:01:42.450 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 03:01:42.450 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 03:01:42.450 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 03:01:42.450 | INFO | ------------------------------------------ | 20260506 03:01:42.450 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:01:42.450 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 03:01:42.450 | INFO | Total mem usage: 25.6 GiB | 20260506 03:01:42.450 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 03:01:42.450 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 03:01:42.450 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 03:01:42.450 | INFO | High Water Breakdown: | 20260506 03:01:42.450 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 03:01:42.450 | INFO | ------------------------------------------ | 20260506 03:01:42.450 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:01:42.450 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 03:01:42.450 | INFO | Total mem usage: 15.7 GiB | 20260506 03:01:42.450 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 03:01:42.450 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 03:01:42.450 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 03:01:42.450 | INFO | High Water Breakdown: | 20260506 03:01:42.450 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 03:01:42.450 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 03:01:42.450 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 03:01:42.450 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 03:01:42.450 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 03:01:42.450 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 03:01:42.450 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 03:01:42.450 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 03:01:42.450 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 03:01:42.450 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 03:01:42.450 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 03:01:42.450 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 03:01:42.450 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 03:01:42.450 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 03:01:42.450 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 03:01:42.450 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 03:01:42.450 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 03:01:42.450 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 03:01:42.450 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 03:01:42.450 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 03:01:42.450 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 03:01:42.450 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 03:01:42.450 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 03:01:42.450 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 03:01:42.450 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 03:01:42.450 | INFO | ------------------------------------------ | 20260506 03:01:42.450 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:01:42.450 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 03:01:42.450 | INFO | PerfReport: Gathered 1536 samples. | 20260506 03:01:42.450 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 03:01:42.450 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.37e+01, 1.34e+01 / 7.88e-01 | 20260506 03:01:42.450 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.04e-01, 1.01e-02 / 3.39e-03 | 20260506 03:01:42.450 | INFO | PerfReport: 11.87%, Baseline -- 3.71e+00 / 4.53e+00, 5.94e-01 / 2.45e-01 | 20260506 03:01:42.450 | INFO | PerfReport: 3.30%, Histogram -- 1.03e+00 / 1.53e+00, 1.94e-01 / 8.71e-02 | 20260506 03:01:42.450 | INFO | PerfReport: 15.98%, DME -- 4.00e+01 / 4.38e+01, 3.80e+00 / 8.88e-01 | 20260506 03:01:42.450 | INFO | PerfReport: 57.53%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.52e+00 / 1.16e+00 | 20260506 03:01:42.450 | INFO | PerfReport: 7.00%, PulseAccumulating -- 2.19e+00 / 2.71e+00, 4.16e-01 / 2.39e-01 | 20260506 03:01:42.450 | INFO | PerfReport: 2.99%, Metrics -- 9.36e-01 / 1.21e+00, 2.53e-01 / 1.04e-01 | 20260506 03:01:42.451 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.22e-01 / 3.70e-01, 5.46e-02 / 2.76e-02 | 20260506 03:01:42.451 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 03:01:42.451 | INFO | PerfReport: Gathered 1536 samples. | 20260506 03:01:42.451 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 03:01:42.451 | INFO | PerfReport: Full Execution -- 9.75e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260506 03:01:42.451 | INFO | PerfReport: 22.29%, Upload -- 2.17e+01 / 0.00e+00, 7.19e+00 / 0.00e+00 | 20260506 03:01:42.451 | INFO | PerfReport: 21.99%, Download -- 2.14e+01 / 0.00e+00, 7.55e+00 / 0.00e+00 | 20260506 03:01:42.451 | INFO | PerfReport: 32.08%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 03:01:42.451 | INFO | PerfReport: 1.96%, LockUpload -- 1.91e+00 / 0.00e+00, 7.72e+00 / 0.00e+00 | 20260506 03:01:42.451 | INFO | PerfReport: 1.28%, LockDownload -- 1.25e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260506 03:01:42.451 | INFO | PerfReport: 20.38%, LockCompute -- 1.99e+01 / 0.00e+00, 1.76e+01 / 0.00e+00 | 20260506 03:01:42.451 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.72e-02 / 0.00e+00 | 20260506 03:01:42.575 | INFO | Starting a round of Baz IO | 20260506 03:01:45.703 | INFO | Populating RTMetrics report | 20260506 03:01:45.703 | INFO | RTMetrics report populated | 20260506 03:01:45.703 | INFO | Opening tmp RTMetrics json file | 20260506 03:01:45.704 | INFO | Serializing RTMetrics to json | 20260506 03:01:45.705 | INFO | Serialization done, swapping tmp file now... | 20260506 03:01:45.706 | INFO | RTMetrics done | 20260506 03:01:45.789 | INFO | Finished a round of Baz IO: Wrote 1.774 GiB in 3.213 seconds | 20260506 03:01:45.789 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.24% / 0.00% / 78.76% | 20260506 03:01:45.789 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0648459% / 0% / 8.34465e-05% | 20260506 03:01:45.789 | INFO | Baz IO overall: | 20260506 03:01:45.789 | INFO | BytesWritten : 4703722369182 | 20260506 03:01:45.789 | INFO | NumEvents : 2206834446848 | 20260506 03:01:47.586 | INFO | Analyzing chunk frames = [8429568,8430080) | 20260506 03:01:47.586 | INFO | MemInfo: { | 20260506 03:01:47.586 | INFO | "gpu" : " 17 GB", | 20260506 03:01:47.586 | INFO | "huge" : "", | 20260506 03:01:47.586 | INFO | "huge_shared" : " 27 GB", | 20260506 03:01:47.586 | INFO | "malloc" : " 14 GB", | 20260506 03:01:47.586 | INFO | "pinned" : " 29 GB" | 20260506 03:01:47.586 | INFO | } | 20260506 03:01:47.586 | INFO | Perf report for the last 5.13703 seconds: | 20260506 03:01:47.586 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:01:47.586 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:47.586 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:47.586 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:47.586 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260506 03:01:47.586 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:47.586 | INFO | RT_METRICS : 42.8%, 1.0, 22.4ms, 0 | 20260506 03:01:47.586 | INFO | PRE_HQ : 59.4%, 1.7, 53.5ms, 0 | 20260506 03:01:47.586 | INFO | BAZWRITER : 61.7%, 3.6, 112.8ms, 0 | 20260506 03:01:47.586 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 03:01:47.586 | INFO | Uploaded 36GiB to GPU in 2.18488s (16.4769GiB/s) | 20260506 03:01:47.586 | INFO | Downloaded 38GiB to GPU in 2.24226s (16.9472GiB/s) | 20260506 03:01:52.725 | INFO | Analyzing chunk frames = [8430080,8430592) | 20260506 03:01:52.725 | INFO | MemInfo: { | 20260506 03:01:52.725 | INFO | "gpu" : " 17 GB", | 20260506 03:01:52.725 | INFO | "huge" : "", | 20260506 03:01:52.725 | INFO | "huge_shared" : " 27 GB", | 20260506 03:01:52.725 | INFO | "malloc" : " 14 GB", | 20260506 03:01:52.725 | INFO | "pinned" : " 29 GB" | 20260506 03:01:52.725 | INFO | } | 20260506 03:01:52.725 | INFO | Perf report for the last 5.13933 seconds: | 20260506 03:01:52.725 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:01:52.725 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:52.725 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:52.725 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:52.725 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260506 03:01:52.725 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:52.725 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:52.725 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 03:01:52.725 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:52.725 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:52.726 | INFO | Uploaded 36GiB to GPU in 2.12086s (16.9743GiB/s) | 20260506 03:01:52.726 | INFO | Downloaded 34GiB to GPU in 2.0413s (16.656GiB/s) | 20260506 03:01:57.870 | INFO | Analyzing chunk frames = [8430592,8431104) | 20260506 03:01:57.870 | INFO | MemInfo: { | 20260506 03:01:57.870 | INFO | "gpu" : " 17 GB", | 20260506 03:01:57.870 | INFO | "huge" : "", | 20260506 03:01:57.870 | INFO | "huge_shared" : " 27 GB", | 20260506 03:01:57.870 | INFO | "malloc" : " 14 GB", | 20260506 03:01:57.870 | INFO | "pinned" : " 29 GB" | 20260506 03:01:57.870 | INFO | } | 20260506 03:01:57.870 | INFO | Perf report for the last 5.14465 seconds: | 20260506 03:01:57.870 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:01:57.870 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:57.870 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:57.870 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:57.870 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 03:01:57.870 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:57.870 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:57.870 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:01:57.870 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:57.870 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:01:57.870 | INFO | Uploaded 36GiB to GPU in 2.10956s (17.0652GiB/s) | 20260506 03:01:57.870 | INFO | Downloaded 34GiB to GPU in 2.04526s (16.6238GiB/s) | 20260506 03:02:03.009 | INFO | Analyzing chunk frames = [8431104,8431616) | 20260506 03:02:03.009 | INFO | MemInfo: { | 20260506 03:02:03.009 | INFO | "gpu" : " 17 GB", | 20260506 03:02:03.010 | INFO | "huge" : "", | 20260506 03:02:03.010 | INFO | "huge_shared" : " 27 GB", | 20260506 03:02:03.010 | INFO | "malloc" : " 14 GB", | 20260506 03:02:03.010 | INFO | "pinned" : " 29 GB" | 20260506 03:02:03.010 | INFO | } | 20260506 03:02:03.010 | INFO | Perf report for the last 5.13979 seconds: | 20260506 03:02:03.010 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:02:03.010 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:03.010 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:03.010 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:03.010 | INFO | ANALYSIS : 61.0%, 3.0, 94.8ms, 0 | 20260506 03:02:03.010 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:03.010 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:03.010 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 03:02:03.010 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:03.010 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:03.010 | INFO | Uploaded 36GiB to GPU in 2.09628s (17.1733GiB/s) | 20260506 03:02:03.010 | INFO | Downloaded 34GiB to GPU in 2.00795s (16.9327GiB/s) | 20260506 03:02:08.147 | INFO | Analyzing chunk frames = [8431616,8432128) | 20260506 03:02:08.147 | INFO | MemInfo: { | 20260506 03:02:08.147 | INFO | "gpu" : " 17 GB", | 20260506 03:02:08.147 | INFO | "huge" : "", | 20260506 03:02:08.148 | INFO | "huge_shared" : " 27 GB", | 20260506 03:02:08.148 | INFO | "malloc" : " 14 GB", | 20260506 03:02:08.148 | INFO | "pinned" : " 29 GB" | 20260506 03:02:08.148 | INFO | } | 20260506 03:02:08.148 | INFO | Perf report for the last 5.13803 seconds: | 20260506 03:02:08.148 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:02:08.148 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:08.148 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:08.148 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:08.148 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260506 03:02:08.148 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:08.148 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:08.148 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 03:02:08.148 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:08.148 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:08.148 | INFO | Uploaded 36GiB to GPU in 2.09402s (17.1918GiB/s) | 20260506 03:02:08.148 | INFO | Downloaded 34GiB to GPU in 2.02888s (16.758GiB/s) | 20260506 03:02:13.287 | INFO | Analyzing chunk frames = [8432128,8432640) | 20260506 03:02:13.287 | INFO | MemInfo: { | 20260506 03:02:13.287 | INFO | "gpu" : " 17 GB", | 20260506 03:02:13.287 | INFO | "huge" : "", | 20260506 03:02:13.288 | INFO | "huge_shared" : " 27 GB", | 20260506 03:02:13.288 | INFO | "malloc" : " 14 GB", | 20260506 03:02:13.288 | INFO | "pinned" : " 29 GB" | 20260506 03:02:13.288 | INFO | } | 20260506 03:02:13.288 | INFO | Perf report for the last 5.1399 seconds: | 20260506 03:02:13.288 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:02:13.288 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:13.288 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:13.288 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:13.288 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 03:02:13.288 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:13.288 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:13.288 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:02:13.288 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:13.288 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:13.288 | INFO | Uploaded 36GiB to GPU in 2.09134s (17.2138GiB/s) | 20260506 03:02:13.288 | INFO | Downloaded 34GiB to GPU in 2.03421s (16.7141GiB/s) | 20260506 03:02:18.427 | INFO | Analyzing chunk frames = [8432640,8433152) | 20260506 03:02:18.427 | INFO | MemInfo: { | 20260506 03:02:18.427 | INFO | "gpu" : " 17 GB", | 20260506 03:02:18.427 | INFO | "huge" : "", | 20260506 03:02:18.427 | INFO | "huge_shared" : " 27 GB", | 20260506 03:02:18.427 | INFO | "malloc" : " 14 GB", | 20260506 03:02:18.427 | INFO | "pinned" : " 29 GB" | 20260506 03:02:18.427 | INFO | } | 20260506 03:02:18.427 | INFO | Perf report for the last 5.13978 seconds: | 20260506 03:02:18.427 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:02:18.427 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:18.427 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:18.427 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:18.428 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 03:02:18.428 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:18.428 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:18.428 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 03:02:18.428 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:18.428 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:18.428 | INFO | Uploaded 36GiB to GPU in 2.01767s (17.8423GiB/s) | 20260506 03:02:18.428 | INFO | Downloaded 34GiB to GPU in 2.04772s (16.6039GiB/s) | 20260506 03:02:23.569 | INFO | Analyzing chunk frames = [8433152,8433664) | 20260506 03:02:23.569 | INFO | MemInfo: { | 20260506 03:02:23.569 | INFO | "gpu" : " 17 GB", | 20260506 03:02:23.569 | INFO | "huge" : "", | 20260506 03:02:23.569 | INFO | "huge_shared" : " 27 GB", | 20260506 03:02:23.569 | INFO | "malloc" : " 14 GB", | 20260506 03:02:23.569 | INFO | "pinned" : " 29 GB" | 20260506 03:02:23.569 | INFO | } | 20260506 03:02:23.569 | INFO | Perf report for the last 5.14169 seconds: | 20260506 03:02:23.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:02:23.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:23.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:23.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:23.569 | INFO | ANALYSIS : 61.3%, 3.0, 95.8ms, 0 | 20260506 03:02:23.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:23.569 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:23.569 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 03:02:23.569 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:23.569 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:23.569 | INFO | Uploaded 36GiB to GPU in 2.06537s (17.4303GiB/s) | 20260506 03:02:23.569 | INFO | Downloaded 34GiB to GPU in 2.02972s (16.7511GiB/s) | 20260506 03:02:23.695 | INFO | Starting a round of Baz IO | 20260506 03:02:26.824 | INFO | Populating RTMetrics report | 20260506 03:02:26.825 | INFO | RTMetrics report populated | 20260506 03:02:26.825 | INFO | Opening tmp RTMetrics json file | 20260506 03:02:26.832 | INFO | Serializing RTMetrics to json | 20260506 03:02:26.833 | INFO | Serialization done, swapping tmp file now... | 20260506 03:02:26.834 | INFO | RTMetrics done | 20260506 03:02:26.926 | INFO | Finished a round of Baz IO: Wrote 1.773 GiB in 3.23 seconds | 20260506 03:02:26.926 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.24% / 0.00% / 78.76% | 20260506 03:02:26.926 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.064822% / 3.97364e-06% / 8.34465e-05% | 20260506 03:02:26.926 | INFO | Baz IO overall: | 20260506 03:02:26.926 | INFO | BytesWritten : 4705626288376 | 20260506 03:02:26.926 | INFO | NumEvents : 2207698515335 | 20260506 03:02:28.706 | INFO | Analyzing chunk frames = [8433664,8434176) | 20260506 03:02:28.706 | INFO | MemInfo: { | 20260506 03:02:28.706 | INFO | "gpu" : " 17 GB", | 20260506 03:02:28.706 | INFO | "huge" : "", | 20260506 03:02:28.706 | INFO | "huge_shared" : " 27 GB", | 20260506 03:02:28.706 | INFO | "malloc" : " 14 GB", | 20260506 03:02:28.706 | INFO | "pinned" : " 29 GB" | 20260506 03:02:28.706 | INFO | } | 20260506 03:02:28.706 | INFO | Perf report for the last 5.13732 seconds: | 20260506 03:02:28.706 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:02:28.706 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:28.706 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:28.706 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:28.707 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260506 03:02:28.707 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:28.707 | INFO | RT_METRICS : 42.8%, 1.0, 22.4ms, 0 | 20260506 03:02:28.707 | INFO | PRE_HQ : 59.1%, 1.7, 53.3ms, 0 | 20260506 03:02:28.707 | INFO | BAZWRITER : 62.2%, 3.6, 113.9ms, 0 | 20260506 03:02:28.707 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 03:02:28.707 | INFO | Uploaded 36GiB to GPU in 2.1165s (17.0092GiB/s) | 20260506 03:02:28.707 | INFO | Downloaded 38GiB to GPU in 2.22112s (17.1085GiB/s) | 20260506 03:02:33.850 | INFO | Analyzing chunk frames = [8434176,8434688) | 20260506 03:02:33.850 | INFO | MemInfo: { | 20260506 03:02:33.850 | INFO | "gpu" : " 17 GB", | 20260506 03:02:33.850 | INFO | "huge" : "", | 20260506 03:02:33.850 | INFO | "huge_shared" : " 27 GB", | 20260506 03:02:33.850 | INFO | "malloc" : " 14 GB", | 20260506 03:02:33.850 | INFO | "pinned" : " 29 GB" | 20260506 03:02:33.850 | INFO | } | 20260506 03:02:33.850 | INFO | Perf report for the last 5.14354 seconds: | 20260506 03:02:33.850 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:02:33.850 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:33.850 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:33.850 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:33.850 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260506 03:02:33.850 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:33.850 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:33.850 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 03:02:33.850 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:33.850 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:33.850 | INFO | Uploaded 36GiB to GPU in 1.95214s (18.4413GiB/s) | 20260506 03:02:33.850 | INFO | Downloaded 34GiB to GPU in 1.97407s (17.2233GiB/s) | 20260506 03:02:38.984 | INFO | Analyzing chunk frames = [8434688,8435200) | 20260506 03:02:38.984 | INFO | MemInfo: { | 20260506 03:02:38.984 | INFO | "gpu" : " 17 GB", | 20260506 03:02:38.984 | INFO | "huge" : "", | 20260506 03:02:38.984 | INFO | "huge_shared" : " 27 GB", | 20260506 03:02:38.984 | INFO | "malloc" : " 14 GB", | 20260506 03:02:38.984 | INFO | "pinned" : " 29 GB" | 20260506 03:02:38.984 | INFO | } | 20260506 03:02:38.984 | INFO | Perf report for the last 5.1344 seconds: | 20260506 03:02:38.984 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:02:38.984 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:38.984 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:38.984 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:38.985 | INFO | ANALYSIS : 61.2%, 3.0, 95.1ms, 0 | 20260506 03:02:38.985 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:38.985 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:38.985 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:02:38.985 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:38.985 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:38.985 | INFO | Uploaded 36GiB to GPU in 2.11208s (17.0448GiB/s) | 20260506 03:02:38.985 | INFO | Downloaded 34GiB to GPU in 2.04542s (16.6225GiB/s) | 20260506 03:02:44.126 | INFO | Analyzing chunk frames = [8435200,8435712) | 20260506 03:02:44.126 | INFO | MemInfo: { | 20260506 03:02:44.126 | INFO | "gpu" : " 17 GB", | 20260506 03:02:44.126 | INFO | "huge" : "", | 20260506 03:02:44.126 | INFO | "huge_shared" : " 27 GB", | 20260506 03:02:44.126 | INFO | "malloc" : " 14 GB", | 20260506 03:02:44.126 | INFO | "pinned" : " 29 GB" | 20260506 03:02:44.127 | INFO | } | 20260506 03:02:44.127 | INFO | Perf report for the last 5.14211 seconds: | 20260506 03:02:44.127 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:02:44.127 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:44.127 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:44.127 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:44.127 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260506 03:02:44.127 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:44.127 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:44.127 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:02:44.127 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:44.127 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:44.127 | INFO | Uploaded 36GiB to GPU in 2.0932s (17.1986GiB/s) | 20260506 03:02:44.127 | INFO | Downloaded 34GiB to GPU in 2.01012s (16.9144GiB/s) | 20260506 03:02:49.260 | INFO | Analyzing chunk frames = [8435712,8436224) | 20260506 03:02:49.260 | INFO | MemInfo: { | 20260506 03:02:49.260 | INFO | "gpu" : " 17 GB", | 20260506 03:02:49.260 | INFO | "huge" : "", | 20260506 03:02:49.260 | INFO | "huge_shared" : " 27 GB", | 20260506 03:02:49.260 | INFO | "malloc" : " 14 GB", | 20260506 03:02:49.260 | INFO | "pinned" : " 29 GB" | 20260506 03:02:49.260 | INFO | } | 20260506 03:02:49.260 | INFO | Perf report for the last 5.13338 seconds: | 20260506 03:02:49.260 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:02:49.260 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:49.260 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:49.260 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:49.260 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260506 03:02:49.260 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:49.260 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:49.260 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:02:49.260 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:49.260 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:49.260 | INFO | Uploaded 36GiB to GPU in 2.1018s (17.1281GiB/s) | 20260506 03:02:49.260 | INFO | Downloaded 34GiB to GPU in 2.03847s (16.6792GiB/s) | 20260506 03:02:54.396 | INFO | Analyzing chunk frames = [8436224,8436736) | 20260506 03:02:54.396 | INFO | MemInfo: { | 20260506 03:02:54.396 | INFO | "gpu" : " 17 GB", | 20260506 03:02:54.396 | INFO | "huge" : "", | 20260506 03:02:54.396 | INFO | "huge_shared" : " 27 GB", | 20260506 03:02:54.396 | INFO | "malloc" : " 14 GB", | 20260506 03:02:54.396 | INFO | "pinned" : " 29 GB" | 20260506 03:02:54.396 | INFO | } | 20260506 03:02:54.396 | INFO | Perf report for the last 5.13595 seconds: | 20260506 03:02:54.396 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:02:54.396 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:54.396 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:54.396 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:54.396 | INFO | ANALYSIS : 60.9%, 3.0, 95.0ms, 0 | 20260506 03:02:54.396 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:54.396 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:54.396 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 03:02:54.396 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:54.396 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:54.396 | INFO | Uploaded 36GiB to GPU in 2.09394s (17.1925GiB/s) | 20260506 03:02:54.396 | INFO | Downloaded 34GiB to GPU in 2.0384s (16.6797GiB/s) | 20260506 03:02:59.541 | INFO | Analyzing chunk frames = [8436736,8437248) | 20260506 03:02:59.541 | INFO | MemInfo: { | 20260506 03:02:59.541 | INFO | "gpu" : " 17 GB", | 20260506 03:02:59.541 | INFO | "huge" : "", | 20260506 03:02:59.541 | INFO | "huge_shared" : " 27 GB", | 20260506 03:02:59.542 | INFO | "malloc" : " 14 GB", | 20260506 03:02:59.542 | INFO | "pinned" : " 29 GB" | 20260506 03:02:59.542 | INFO | } | 20260506 03:02:59.542 | INFO | Perf report for the last 5.14566 seconds: | 20260506 03:02:59.542 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:02:59.542 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:59.542 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:59.542 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:59.542 | INFO | ANALYSIS : 62.3%, 3.0, 97.4ms, 0 | 20260506 03:02:59.542 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:59.542 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:59.542 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 03:02:59.542 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:59.542 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:02:59.542 | INFO | Uploaded 36GiB to GPU in 2.117s (17.0052GiB/s) | 20260506 03:02:59.542 | INFO | Downloaded 34GiB to GPU in 2.07679s (16.3714GiB/s) | 20260506 03:03:04.683 | INFO | Analyzing chunk frames = [8437248,8437760) | 20260506 03:03:04.683 | INFO | MemInfo: { | 20260506 03:03:04.683 | INFO | "gpu" : " 17 GB", | 20260506 03:03:04.683 | INFO | "huge" : "", | 20260506 03:03:04.683 | INFO | "huge_shared" : " 27 GB", | 20260506 03:03:04.683 | INFO | "malloc" : " 14 GB", | 20260506 03:03:04.683 | INFO | "pinned" : " 29 GB" | 20260506 03:03:04.683 | INFO | } | 20260506 03:03:04.683 | INFO | Perf report for the last 5.14139 seconds: | 20260506 03:03:04.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:03:04.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:04.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:04.683 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:04.683 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260506 03:03:04.683 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:04.683 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:04.683 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 03:03:04.683 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:04.683 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:04.683 | INFO | Uploaded 36GiB to GPU in 2.04013s (17.6459GiB/s) | 20260506 03:03:04.683 | INFO | Downloaded 34GiB to GPU in 2.01834s (16.8455GiB/s) | 20260506 03:03:04.683 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 03:03:04.683 | INFO | Total mem usage: 15.4 GiB | 20260506 03:03:04.683 | INFO | All Time High Water Marks sum: 16.3 GiB (Savings: 5.2%) | 20260506 03:03:04.683 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -9.27%) | 20260506 03:03:04.683 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 03:03:04.683 | INFO | High Water Breakdown: | 20260506 03:03:04.683 | INFO | 60.9% : 8.61 GiB ( 89.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 03:03:04.683 | INFO | 38.9% : 5.5 GiB ( 82.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 03:03:04.683 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 03:03:04.684 | INFO | ------------------------------------------ | 20260506 03:03:04.684 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:03:04.684 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 03:03:04.684 | INFO | Total mem usage: 26.5 GiB | 20260506 03:03:04.684 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 03:03:04.684 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 03:03:04.684 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 03:03:04.684 | INFO | High Water Breakdown: | 20260506 03:03:04.684 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 03:03:04.684 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 03:03:04.684 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 03:03:04.684 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 03:03:04.684 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 03:03:04.684 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 03:03:04.684 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 03:03:04.684 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 03:03:04.684 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 03:03:04.684 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 03:03:04.684 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 03:03:04.684 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 03:03:04.684 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 03:03:04.684 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 03:03:04.684 | INFO | ------------------------------------------ | 20260506 03:03:04.684 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:03:04.684 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 03:03:04.684 | INFO | Total mem usage: 25.6 GiB | 20260506 03:03:04.684 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 03:03:04.684 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 03:03:04.684 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 03:03:04.684 | INFO | High Water Breakdown: | 20260506 03:03:04.684 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 03:03:04.684 | INFO | ------------------------------------------ | 20260506 03:03:04.684 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:03:04.684 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 03:03:04.684 | INFO | Total mem usage: 15.7 GiB | 20260506 03:03:04.684 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 03:03:04.684 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 03:03:04.684 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 03:03:04.684 | INFO | High Water Breakdown: | 20260506 03:03:04.684 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 03:03:04.684 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 03:03:04.684 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 03:03:04.684 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 03:03:04.684 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 03:03:04.684 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 03:03:04.684 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 03:03:04.684 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 03:03:04.684 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 03:03:04.684 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 03:03:04.684 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 03:03:04.684 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 03:03:04.684 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 03:03:04.684 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 03:03:04.684 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 03:03:04.684 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 03:03:04.684 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 03:03:04.684 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 03:03:04.684 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 03:03:04.684 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 03:03:04.684 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 03:03:04.684 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 03:03:04.684 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 03:03:04.684 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 03:03:04.684 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 03:03:04.684 | INFO | ------------------------------------------ | 20260506 03:03:04.684 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:03:04.684 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 03:03:04.684 | INFO | PerfReport: Gathered 1536 samples. | 20260506 03:03:04.684 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 03:03:04.684 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.38e+01, 1.35e+01 / 8.60e-01 | 20260506 03:03:04.684 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.05e-01, 1.07e-02 / 3.14e-03 | 20260506 03:03:04.684 | INFO | PerfReport: 11.85%, Baseline -- 3.70e+00 / 4.52e+00, 5.73e-01 / 2.02e-01 | 20260506 03:03:04.684 | INFO | PerfReport: 3.29%, Histogram -- 1.03e+00 / 1.53e+00, 1.93e-01 / 9.51e-02 | 20260506 03:03:04.684 | INFO | PerfReport: 16.00%, DME -- 4.00e+01 / 4.39e+01, 3.88e+00 / 9.87e-01 | 20260506 03:03:04.684 | INFO | PerfReport: 57.52%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.50e+00 / 1.13e+00 | 20260506 03:03:04.684 | INFO | PerfReport: 7.03%, PulseAccumulating -- 2.19e+00 / 2.72e+00, 4.18e-01 / 2.55e-01 | 20260506 03:03:04.684 | INFO | PerfReport: 2.99%, Metrics -- 9.33e-01 / 1.20e+00, 2.49e-01 / 1.17e-01 | 20260506 03:03:04.684 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.22e-01 / 3.71e-01, 5.42e-02 / 3.31e-02 | 20260506 03:03:04.684 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 03:03:04.684 | INFO | PerfReport: Gathered 1536 samples. | 20260506 03:03:04.684 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 03:03:04.684 | INFO | PerfReport: Full Execution -- 9.76e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260506 03:03:04.684 | INFO | PerfReport: 22.28%, Upload -- 2.17e+01 / 0.00e+00, 7.28e+00 / 0.00e+00 | 20260506 03:03:04.684 | INFO | PerfReport: 21.94%, Download -- 2.14e+01 / 0.00e+00, 7.62e+00 / 0.00e+00 | 20260506 03:03:04.684 | INFO | PerfReport: 31.99%, Compute -- 3.12e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 03:03:04.684 | INFO | PerfReport: 2.04%, LockUpload -- 1.99e+00 / 0.00e+00, 7.90e+00 / 0.00e+00 | 20260506 03:03:04.684 | INFO | PerfReport: 1.32%, LockDownload -- 1.29e+00 / 0.00e+00, 2.92e+00 / 0.00e+00 | 20260506 03:03:04.684 | INFO | PerfReport: 20.41%, LockCompute -- 1.99e+01 / 0.00e+00, 1.74e+01 / 0.00e+00 | 20260506 03:03:04.684 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.72e-02 / 0.00e+00 | 20260506 03:03:04.805 | INFO | Starting a round of Baz IO | 20260506 03:03:07.925 | INFO | Populating RTMetrics report | 20260506 03:03:07.925 | INFO | RTMetrics report populated | 20260506 03:03:07.925 | INFO | Opening tmp RTMetrics json file | 20260506 03:03:07.927 | INFO | Serializing RTMetrics to json | 20260506 03:03:07.930 | INFO | Serialization done, swapping tmp file now... | 20260506 03:03:07.933 | INFO | RTMetrics done | 20260506 03:03:08.008 | INFO | Finished a round of Baz IO: Wrote 1.772 GiB in 3.203 seconds | 20260506 03:03:08.008 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.24% / 0.00% / 78.76% | 20260506 03:03:08.008 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.06392% / 0% / 7.94729e-05% | 20260506 03:03:08.008 | INFO | Baz IO overall: | 20260506 03:03:08.008 | INFO | BytesWritten : 4707528891643 | 20260506 03:03:08.008 | INFO | NumEvents : 2208561807533 | 20260506 03:03:09.823 | INFO | Analyzing chunk frames = [8437760,8438272) | 20260506 03:03:09.823 | INFO | MemInfo: { | 20260506 03:03:09.823 | INFO | "gpu" : " 17 GB", | 20260506 03:03:09.823 | INFO | "huge" : "", | 20260506 03:03:09.823 | INFO | "huge_shared" : " 27 GB", | 20260506 03:03:09.823 | INFO | "malloc" : " 14 GB", | 20260506 03:03:09.823 | INFO | "pinned" : " 29 GB" | 20260506 03:03:09.823 | INFO | } | 20260506 03:03:09.823 | INFO | Perf report for the last 5.14044 seconds: | 20260506 03:03:09.824 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:03:09.824 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:09.824 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:09.824 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:09.824 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 03:03:09.824 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:09.824 | INFO | RT_METRICS : 42.8%, 1.0, 22.4ms, 0 | 20260506 03:03:09.824 | INFO | PRE_HQ : 59.1%, 1.7, 53.2ms, 0 | 20260506 03:03:09.824 | INFO | BAZWRITER : 61.5%, 3.8, 118.9ms, 0 | 20260506 03:03:09.824 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 03:03:09.824 | INFO | Uploaded 36GiB to GPU in 2.10724s (17.0839GiB/s) | 20260506 03:03:09.824 | INFO | Downloaded 38GiB to GPU in 2.23214s (17.024GiB/s) | 20260506 03:03:14.957 | INFO | Analyzing chunk frames = [8438272,8438784) | 20260506 03:03:14.957 | INFO | MemInfo: { | 20260506 03:03:14.957 | INFO | "gpu" : " 17 GB", | 20260506 03:03:14.957 | INFO | "huge" : "", | 20260506 03:03:14.957 | INFO | "huge_shared" : " 27 GB", | 20260506 03:03:14.957 | INFO | "malloc" : " 14 GB", | 20260506 03:03:14.957 | INFO | "pinned" : " 29 GB" | 20260506 03:03:14.957 | INFO | } | 20260506 03:03:14.958 | INFO | Perf report for the last 5.13405 seconds: | 20260506 03:03:14.958 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:03:14.958 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:14.958 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:14.958 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:14.958 | INFO | ANALYSIS : 60.9%, 3.0, 95.0ms, 0 | 20260506 03:03:14.958 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:14.958 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:14.958 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 03:03:14.958 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:14.958 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:14.958 | INFO | Uploaded 36GiB to GPU in 2.00932s (17.9165GiB/s) | 20260506 03:03:14.958 | INFO | Downloaded 34GiB to GPU in 1.99879s (17.0103GiB/s) | 20260506 03:03:20.093 | INFO | Analyzing chunk frames = [8438784,8439296) | 20260506 03:03:20.093 | INFO | MemInfo: { | 20260506 03:03:20.093 | INFO | "gpu" : " 17 GB", | 20260506 03:03:20.093 | INFO | "huge" : "", | 20260506 03:03:20.093 | INFO | "huge_shared" : " 27 GB", | 20260506 03:03:20.093 | INFO | "malloc" : " 14 GB", | 20260506 03:03:20.093 | INFO | "pinned" : " 29 GB" | 20260506 03:03:20.093 | INFO | } | 20260506 03:03:20.093 | INFO | Perf report for the last 5.13533 seconds: | 20260506 03:03:20.093 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:03:20.093 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:20.093 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:20.093 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:20.093 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 03:03:20.093 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:20.093 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:20.093 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 03:03:20.093 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:20.093 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:20.093 | INFO | Uploaded 36GiB to GPU in 2.08835s (17.2385GiB/s) | 20260506 03:03:20.093 | INFO | Downloaded 34GiB to GPU in 2.02208s (16.8144GiB/s) | 20260506 03:03:25.236 | INFO | Analyzing chunk frames = [8439296,8439808) | 20260506 03:03:25.236 | INFO | MemInfo: { | 20260506 03:03:25.236 | INFO | "gpu" : " 17 GB", | 20260506 03:03:25.236 | INFO | "huge" : "", | 20260506 03:03:25.236 | INFO | "huge_shared" : " 27 GB", | 20260506 03:03:25.236 | INFO | "malloc" : " 14 GB", | 20260506 03:03:25.236 | INFO | "pinned" : " 29 GB" | 20260506 03:03:25.236 | INFO | } | 20260506 03:03:25.236 | INFO | Perf report for the last 5.14359 seconds: | 20260506 03:03:25.236 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:03:25.236 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:25.236 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:25.237 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:25.237 | INFO | ANALYSIS : 61.0%, 3.0, 94.8ms, 0 | 20260506 03:03:25.237 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:25.237 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:25.237 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 03:03:25.237 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:25.237 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:25.237 | INFO | Uploaded 36GiB to GPU in 2.07835s (17.3214GiB/s) | 20260506 03:03:25.237 | INFO | Downloaded 34GiB to GPU in 2.01205s (16.8982GiB/s) | 20260506 03:03:30.377 | INFO | Analyzing chunk frames = [8439808,8440320) | 20260506 03:03:30.377 | INFO | MemInfo: { | 20260506 03:03:30.377 | INFO | "gpu" : " 17 GB", | 20260506 03:03:30.377 | INFO | "huge" : "", | 20260506 03:03:30.377 | INFO | "huge_shared" : " 27 GB", | 20260506 03:03:30.377 | INFO | "malloc" : " 14 GB", | 20260506 03:03:30.377 | INFO | "pinned" : " 29 GB" | 20260506 03:03:30.377 | INFO | } | 20260506 03:03:30.378 | INFO | Perf report for the last 5.14103 seconds: | 20260506 03:03:30.378 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:03:30.378 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:30.378 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:30.378 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:30.378 | INFO | ANALYSIS : 60.7%, 3.0, 94.8ms, 0 | 20260506 03:03:30.378 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:30.378 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:30.378 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 03:03:30.378 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:30.378 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:30.378 | INFO | Uploaded 36GiB to GPU in 2.11223s (17.0436GiB/s) | 20260506 03:03:30.378 | INFO | Downloaded 34GiB to GPU in 2.03717s (16.6898GiB/s) | 20260506 03:03:35.520 | INFO | Analyzing chunk frames = [8440320,8440832) | 20260506 03:03:35.520 | INFO | MemInfo: { | 20260506 03:03:35.520 | INFO | "gpu" : " 17 GB", | 20260506 03:03:35.520 | INFO | "huge" : "", | 20260506 03:03:35.520 | INFO | "huge_shared" : " 27 GB", | 20260506 03:03:35.520 | INFO | "malloc" : " 14 GB", | 20260506 03:03:35.520 | INFO | "pinned" : " 29 GB" | 20260506 03:03:35.520 | INFO | } | 20260506 03:03:35.520 | INFO | Perf report for the last 5.14294 seconds: | 20260506 03:03:35.520 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:03:35.520 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:35.520 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:35.520 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:35.520 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 03:03:35.520 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:35.521 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:35.521 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 03:03:35.521 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:35.521 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:35.521 | INFO | Uploaded 36GiB to GPU in 2.10777s (17.0796GiB/s) | 20260506 03:03:35.521 | INFO | Downloaded 34GiB to GPU in 2.03931s (16.6723GiB/s) | 20260506 03:03:40.659 | INFO | Analyzing chunk frames = [8440832,8441344) | 20260506 03:03:40.659 | INFO | MemInfo: { | 20260506 03:03:40.659 | INFO | "gpu" : " 17 GB", | 20260506 03:03:40.659 | INFO | "huge" : "", | 20260506 03:03:40.659 | INFO | "huge_shared" : " 27 GB", | 20260506 03:03:40.659 | INFO | "malloc" : " 14 GB", | 20260506 03:03:40.659 | INFO | "pinned" : " 29 GB" | 20260506 03:03:40.659 | INFO | } | 20260506 03:03:40.659 | INFO | Perf report for the last 5.13854 seconds: | 20260506 03:03:40.659 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:03:40.659 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:40.659 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:40.659 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:40.659 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260506 03:03:40.659 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:40.659 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:40.659 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 03:03:40.659 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:40.659 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:40.659 | INFO | Uploaded 36GiB to GPU in 2.04138s (17.6352GiB/s) | 20260506 03:03:40.659 | INFO | Downloaded 34GiB to GPU in 2.06244s (16.4853GiB/s) | 20260506 03:03:45.801 | INFO | Analyzing chunk frames = [8441344,8441856) | 20260506 03:03:45.801 | INFO | MemInfo: { | 20260506 03:03:45.801 | INFO | "gpu" : " 17 GB", | 20260506 03:03:45.801 | INFO | "huge" : "", | 20260506 03:03:45.801 | INFO | "huge_shared" : " 27 GB", | 20260506 03:03:45.801 | INFO | "malloc" : " 14 GB", | 20260506 03:03:45.801 | INFO | "pinned" : " 29 GB" | 20260506 03:03:45.801 | INFO | } | 20260506 03:03:45.802 | INFO | Perf report for the last 5.14249 seconds: | 20260506 03:03:45.802 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:03:45.802 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:45.802 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:45.802 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:45.802 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260506 03:03:45.802 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:45.802 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:45.802 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 03:03:45.802 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:45.802 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:45.802 | INFO | Uploaded 36GiB to GPU in 2.05402s (17.5266GiB/s) | 20260506 03:03:45.802 | INFO | Downloaded 34GiB to GPU in 2.02547s (16.7862GiB/s) | 20260506 03:03:45.923 | INFO | Starting a round of Baz IO | 20260506 03:03:49.049 | INFO | Populating RTMetrics report | 20260506 03:03:49.049 | INFO | RTMetrics report populated | 20260506 03:03:49.049 | INFO | Opening tmp RTMetrics json file | 20260506 03:03:49.050 | INFO | Serializing RTMetrics to json | 20260506 03:03:49.051 | INFO | Serialization done, swapping tmp file now... | 20260506 03:03:49.052 | INFO | RTMetrics done | 20260506 03:03:49.131 | INFO | Finished a round of Baz IO: Wrote 1.772 GiB in 3.207 seconds | 20260506 03:03:49.131 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.24% / 0.00% / 78.76% | 20260506 03:03:49.131 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0640551% / 0% / 0.000103315% | 20260506 03:03:49.131 | INFO | Baz IO overall: | 20260506 03:03:49.131 | INFO | BytesWritten : 4709431777671 | 20260506 03:03:49.131 | INFO | NumEvents : 2209425273412 | 20260506 03:03:50.937 | INFO | Analyzing chunk frames = [8441856,8442368) | 20260506 03:03:50.937 | INFO | MemInfo: { | 20260506 03:03:50.937 | INFO | "gpu" : " 17 GB", | 20260506 03:03:50.937 | INFO | "huge" : "", | 20260506 03:03:50.937 | INFO | "huge_shared" : " 27 GB", | 20260506 03:03:50.937 | INFO | "malloc" : " 14 GB", | 20260506 03:03:50.937 | INFO | "pinned" : " 29 GB" | 20260506 03:03:50.937 | INFO | } | 20260506 03:03:50.937 | INFO | Perf report for the last 5.13551 seconds: | 20260506 03:03:50.937 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:03:50.937 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:50.937 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:50.937 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:50.937 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260506 03:03:50.937 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:50.937 | INFO | RT_METRICS : 42.8%, 1.0, 22.4ms, 0 | 20260506 03:03:50.937 | INFO | PRE_HQ : 59.2%, 1.7, 53.4ms, 0 | 20260506 03:03:50.937 | INFO | BAZWRITER : 61.7%, 3.5, 112.1ms, 0 | 20260506 03:03:50.937 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 03:03:50.937 | INFO | Uploaded 36GiB to GPU in 2.12538s (16.9382GiB/s) | 20260506 03:03:50.937 | INFO | Downloaded 38GiB to GPU in 2.22888s (17.049GiB/s) | 20260506 03:03:56.072 | INFO | Analyzing chunk frames = [8442368,8442880) | 20260506 03:03:56.072 | INFO | MemInfo: { | 20260506 03:03:56.072 | INFO | "gpu" : " 17 GB", | 20260506 03:03:56.072 | INFO | "huge" : "", | 20260506 03:03:56.072 | INFO | "huge_shared" : " 27 GB", | 20260506 03:03:56.072 | INFO | "malloc" : " 14 GB", | 20260506 03:03:56.072 | INFO | "pinned" : " 29 GB" | 20260506 03:03:56.072 | INFO | } | 20260506 03:03:56.072 | INFO | Perf report for the last 5.13535 seconds: | 20260506 03:03:56.072 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:03:56.072 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:56.072 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:56.072 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:56.072 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 03:03:56.072 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:56.072 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:56.073 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 03:03:56.073 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:56.073 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:03:56.073 | INFO | Uploaded 36GiB to GPU in 2.02312s (17.7943GiB/s) | 20260506 03:03:56.073 | INFO | Downloaded 34GiB to GPU in 1.99353s (17.0552GiB/s) | 20260506 03:04:01.210 | INFO | Analyzing chunk frames = [8442880,8443392) | 20260506 03:04:01.210 | INFO | MemInfo: { | 20260506 03:04:01.210 | INFO | "gpu" : " 17 GB", | 20260506 03:04:01.210 | INFO | "huge" : "", | 20260506 03:04:01.210 | INFO | "huge_shared" : " 27 GB", | 20260506 03:04:01.210 | INFO | "malloc" : " 14 GB", | 20260506 03:04:01.210 | INFO | "pinned" : " 29 GB" | 20260506 03:04:01.210 | INFO | } | 20260506 03:04:01.210 | INFO | Perf report for the last 5.13775 seconds: | 20260506 03:04:01.210 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:04:01.210 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:01.210 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:01.210 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:01.210 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 03:04:01.210 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:01.210 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:01.210 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 03:04:01.210 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:01.210 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:01.210 | INFO | Uploaded 36GiB to GPU in 2.10333s (17.1157GiB/s) | 20260506 03:04:01.210 | INFO | Downloaded 34GiB to GPU in 2.0305s (16.7446GiB/s) | 20260506 03:04:06.349 | INFO | Analyzing chunk frames = [8443392,8443904) | 20260506 03:04:06.349 | INFO | MemInfo: { | 20260506 03:04:06.350 | INFO | "gpu" : " 17 GB", | 20260506 03:04:06.350 | INFO | "huge" : "", | 20260506 03:04:06.350 | INFO | "huge_shared" : " 27 GB", | 20260506 03:04:06.350 | INFO | "malloc" : " 14 GB", | 20260506 03:04:06.350 | INFO | "pinned" : " 29 GB" | 20260506 03:04:06.350 | INFO | } | 20260506 03:04:06.350 | INFO | Perf report for the last 5.13954 seconds: | 20260506 03:04:06.350 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:04:06.350 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:06.350 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:06.350 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:06.350 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260506 03:04:06.350 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:06.350 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:06.350 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 03:04:06.350 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:06.350 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:06.350 | INFO | Uploaded 36GiB to GPU in 2.15097s (16.7366GiB/s) | 20260506 03:04:06.350 | INFO | Downloaded 34GiB to GPU in 2.05341s (16.5578GiB/s) | 20260506 03:04:11.497 | INFO | Analyzing chunk frames = [8443904,8444416) | 20260506 03:04:11.497 | INFO | MemInfo: { | 20260506 03:04:11.497 | INFO | "gpu" : " 17 GB", | 20260506 03:04:11.497 | INFO | "huge" : "", | 20260506 03:04:11.497 | INFO | "huge_shared" : " 27 GB", | 20260506 03:04:11.497 | INFO | "malloc" : " 14 GB", | 20260506 03:04:11.497 | INFO | "pinned" : " 29 GB" | 20260506 03:04:11.497 | INFO | } | 20260506 03:04:11.497 | INFO | Perf report for the last 5.14688 seconds: | 20260506 03:04:11.497 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:04:11.497 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:11.497 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:11.497 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:11.497 | INFO | ANALYSIS : 60.6%, 3.0, 94.7ms, 0 | 20260506 03:04:11.497 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:11.497 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:11.497 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 03:04:11.497 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:11.497 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:11.497 | INFO | Uploaded 36GiB to GPU in 2.06605s (17.4245GiB/s) | 20260506 03:04:11.497 | INFO | Downloaded 34GiB to GPU in 2.02815s (16.7641GiB/s) | 20260506 03:04:16.639 | INFO | Analyzing chunk frames = [8444416,8444928) | 20260506 03:04:16.639 | INFO | MemInfo: { | 20260506 03:04:16.639 | INFO | "gpu" : " 17 GB", | 20260506 03:04:16.639 | INFO | "huge" : "", | 20260506 03:04:16.639 | INFO | "huge_shared" : " 27 GB", | 20260506 03:04:16.639 | INFO | "malloc" : " 14 GB", | 20260506 03:04:16.639 | INFO | "pinned" : " 29 GB" | 20260506 03:04:16.639 | INFO | } | 20260506 03:04:16.640 | INFO | Perf report for the last 5.1429 seconds: | 20260506 03:04:16.640 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:04:16.640 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:16.640 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:16.640 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:16.640 | INFO | ANALYSIS : 60.7%, 3.0, 94.8ms, 0 | 20260506 03:04:16.640 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:16.640 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:16.640 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 03:04:16.640 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:16.640 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:16.640 | INFO | Uploaded 36GiB to GPU in 2.05497s (17.5185GiB/s) | 20260506 03:04:16.640 | INFO | Downloaded 34GiB to GPU in 2.02328s (16.8044GiB/s) | 20260506 03:04:21.774 | INFO | Analyzing chunk frames = [8444928,8445440) | 20260506 03:04:21.774 | INFO | MemInfo: { | 20260506 03:04:21.774 | INFO | "gpu" : " 17 GB", | 20260506 03:04:21.774 | INFO | "huge" : "", | 20260506 03:04:21.774 | INFO | "huge_shared" : " 27 GB", | 20260506 03:04:21.774 | INFO | "malloc" : " 14 GB", | 20260506 03:04:21.774 | INFO | "pinned" : " 29 GB" | 20260506 03:04:21.774 | INFO | } | 20260506 03:04:21.774 | INFO | Perf report for the last 5.1348 seconds: | 20260506 03:04:21.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:04:21.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:21.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:21.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:21.774 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 03:04:21.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:21.774 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:21.774 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 03:04:21.774 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:21.775 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:21.775 | INFO | Uploaded 36GiB to GPU in 2.07514s (17.3482GiB/s) | 20260506 03:04:21.775 | INFO | Downloaded 34GiB to GPU in 2.06179s (16.4905GiB/s) | 20260506 03:04:26.907 | INFO | Analyzing chunk frames = [8445440,8445952) | 20260506 03:04:26.907 | INFO | MemInfo: { | 20260506 03:04:26.907 | INFO | "gpu" : " 17 GB", | 20260506 03:04:26.907 | INFO | "huge" : "", | 20260506 03:04:26.907 | INFO | "huge_shared" : " 27 GB", | 20260506 03:04:26.908 | INFO | "malloc" : " 14 GB", | 20260506 03:04:26.908 | INFO | "pinned" : " 29 GB" | 20260506 03:04:26.908 | INFO | } | 20260506 03:04:26.908 | INFO | Perf report for the last 5.13355 seconds: | 20260506 03:04:26.908 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:04:26.908 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:26.908 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:26.908 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:26.908 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260506 03:04:26.908 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:26.908 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:26.908 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:04:26.908 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:26.908 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:26.908 | INFO | Uploaded 36GiB to GPU in 2.03405s (17.6986GiB/s) | 20260506 03:04:26.908 | INFO | Downloaded 34GiB to GPU in 2.01608s (16.8644GiB/s) | 20260506 03:04:26.908 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 03:04:26.908 | INFO | Total mem usage: 15.4 GiB | 20260506 03:04:26.908 | INFO | All Time High Water Marks sum: 16.3 GiB (Savings: 5.2%) | 20260506 03:04:26.908 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -9.06%) | 20260506 03:04:26.908 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 03:04:26.908 | INFO | High Water Breakdown: | 20260506 03:04:26.908 | INFO | 61% : 8.64 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 03:04:26.908 | INFO | 38.8% : 5.5 GiB ( 82.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 03:04:26.908 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 03:04:26.908 | INFO | ------------------------------------------ | 20260506 03:04:26.908 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:04:26.908 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 03:04:26.908 | INFO | Total mem usage: 26.5 GiB | 20260506 03:04:26.908 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 03:04:26.908 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 03:04:26.908 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 03:04:26.908 | INFO | High Water Breakdown: | 20260506 03:04:26.908 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 03:04:26.908 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 03:04:26.909 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 03:04:26.909 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 03:04:26.909 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 03:04:26.909 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 03:04:26.909 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 03:04:26.909 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 03:04:26.909 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 03:04:26.909 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 03:04:26.909 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 03:04:26.909 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 03:04:26.909 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 03:04:26.909 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 03:04:26.909 | INFO | ------------------------------------------ | 20260506 03:04:26.909 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:04:26.909 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 03:04:26.909 | INFO | Total mem usage: 25.6 GiB | 20260506 03:04:26.909 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 03:04:26.909 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 03:04:26.909 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 03:04:26.909 | INFO | High Water Breakdown: | 20260506 03:04:26.909 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 03:04:26.909 | INFO | ------------------------------------------ | 20260506 03:04:26.909 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:04:26.909 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 03:04:26.909 | INFO | Total mem usage: 15.7 GiB | 20260506 03:04:26.909 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 03:04:26.909 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 03:04:26.909 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 03:04:26.909 | INFO | High Water Breakdown: | 20260506 03:04:26.909 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 03:04:26.909 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 03:04:26.909 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 03:04:26.909 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 03:04:26.909 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 03:04:26.909 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 03:04:26.909 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 03:04:26.909 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 03:04:26.909 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 03:04:26.909 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 03:04:26.909 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 03:04:26.909 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 03:04:26.909 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 03:04:26.909 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 03:04:26.909 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 03:04:26.909 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 03:04:26.909 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 03:04:26.909 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 03:04:26.909 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 03:04:26.909 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 03:04:26.909 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 03:04:26.909 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 03:04:26.909 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 03:04:26.909 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 03:04:26.909 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 03:04:26.909 | INFO | ------------------------------------------ | 20260506 03:04:26.909 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:04:26.909 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 03:04:26.909 | INFO | PerfReport: Gathered 1536 samples. | 20260506 03:04:26.909 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 03:04:26.909 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.37e+01, 1.35e+01 / 7.91e-01 | 20260506 03:04:26.909 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.04e-01, 1.04e-02 / 2.40e-03 | 20260506 03:04:26.909 | INFO | PerfReport: 11.88%, Baseline -- 3.72e+00 / 4.52e+00, 5.82e-01 / 2.71e-01 | 20260506 03:04:26.909 | INFO | PerfReport: 3.30%, Histogram -- 1.03e+00 / 1.52e+00, 1.96e-01 / 1.01e-01 | 20260506 03:04:26.909 | INFO | PerfReport: 16.01%, DME -- 4.01e+01 / 4.40e+01, 3.88e+00 / 9.31e-01 | 20260506 03:04:26.909 | INFO | PerfReport: 57.50%, FrameLabeling -- 1.80e+01 / 1.92e+01, 1.52e+00 / 9.15e-01 | 20260506 03:04:26.909 | INFO | PerfReport: 6.99%, PulseAccumulating -- 2.19e+00 / 2.75e+00, 4.12e-01 / 2.06e-01 | 20260506 03:04:26.909 | INFO | PerfReport: 2.99%, Metrics -- 9.35e-01 / 1.19e+00, 2.46e-01 / 1.08e-01 | 20260506 03:04:26.909 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.22e-01 / 3.70e-01, 5.45e-02 / 2.55e-02 | 20260506 03:04:26.909 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 03:04:26.909 | INFO | PerfReport: Gathered 1536 samples. | 20260506 03:04:26.909 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 03:04:26.909 | INFO | PerfReport: Full Execution -- 9.75e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260506 03:04:26.909 | INFO | PerfReport: 22.19%, Upload -- 2.16e+01 / 0.00e+00, 7.11e+00 / 0.00e+00 | 20260506 03:04:26.909 | INFO | PerfReport: 21.94%, Download -- 2.14e+01 / 0.00e+00, 7.50e+00 / 0.00e+00 | 20260506 03:04:26.909 | INFO | PerfReport: 32.08%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 03:04:26.909 | INFO | PerfReport: 1.96%, LockUpload -- 1.92e+00 / 0.00e+00, 7.70e+00 / 0.00e+00 | 20260506 03:04:26.909 | INFO | PerfReport: 1.29%, LockDownload -- 1.25e+00 / 0.00e+00, 2.97e+00 / 0.00e+00 | 20260506 03:04:26.909 | INFO | PerfReport: 20.53%, LockCompute -- 2.00e+01 / 0.00e+00, 1.77e+01 / 0.00e+00 | 20260506 03:04:26.909 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 2.01e-02 / 0.00e+00 | 20260506 03:04:27.033 | INFO | Starting a round of Baz IO | 20260506 03:04:30.170 | INFO | Populating RTMetrics report | 20260506 03:04:30.170 | INFO | RTMetrics report populated | 20260506 03:04:30.170 | INFO | Opening tmp RTMetrics json file | 20260506 03:04:30.173 | INFO | Serializing RTMetrics to json | 20260506 03:04:30.177 | INFO | Serialization done, swapping tmp file now... | 20260506 03:04:30.179 | INFO | RTMetrics done | 20260506 03:04:30.264 | INFO | Finished a round of Baz IO: Wrote 1.772 GiB in 3.23 seconds | 20260506 03:04:30.264 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.24% / 0.00% / 78.76% | 20260506 03:04:30.264 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0647386% / 0% / 0.000119209% | 20260506 03:04:30.264 | INFO | Baz IO overall: | 20260506 03:04:30.264 | INFO | BytesWritten : 4711334818250 | 20260506 03:04:30.264 | INFO | NumEvents : 2210288514538 | 20260506 03:04:32.045 | INFO | Analyzing chunk frames = [8445952,8446464) | 20260506 03:04:32.045 | INFO | MemInfo: { | 20260506 03:04:32.045 | INFO | "gpu" : " 17 GB", | 20260506 03:04:32.045 | INFO | "huge" : "", | 20260506 03:04:32.045 | INFO | "huge_shared" : " 27 GB", | 20260506 03:04:32.045 | INFO | "malloc" : " 14 GB", | 20260506 03:04:32.045 | INFO | "pinned" : " 29 GB" | 20260506 03:04:32.045 | INFO | } | 20260506 03:04:32.045 | INFO | Perf report for the last 5.13718 seconds: | 20260506 03:04:32.045 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:04:32.045 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:32.045 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:32.045 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:32.045 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260506 03:04:32.045 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:32.045 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260506 03:04:32.045 | INFO | PRE_HQ : 59.5%, 1.7, 53.1ms, 0 | 20260506 03:04:32.045 | INFO | BAZWRITER : 62.2%, 3.5, 111.8ms, 0 | 20260506 03:04:32.045 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 03:04:32.045 | INFO | Uploaded 36GiB to GPU in 2.15053s (16.74GiB/s) | 20260506 03:04:32.045 | INFO | Downloaded 38GiB to GPU in 2.23705s (16.9866GiB/s) | 20260506 03:04:37.188 | INFO | Analyzing chunk frames = [8446464,8446976) | 20260506 03:04:37.188 | INFO | MemInfo: { | 20260506 03:04:37.188 | INFO | "gpu" : " 17 GB", | 20260506 03:04:37.188 | INFO | "huge" : "", | 20260506 03:04:37.188 | INFO | "huge_shared" : " 27 GB", | 20260506 03:04:37.188 | INFO | "malloc" : " 14 GB", | 20260506 03:04:37.188 | INFO | "pinned" : " 29 GB" | 20260506 03:04:37.188 | INFO | } | 20260506 03:04:37.188 | INFO | Perf report for the last 5.14278 seconds: | 20260506 03:04:37.188 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:04:37.188 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:37.188 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:37.188 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:37.188 | INFO | ANALYSIS : 60.7%, 3.0, 94.8ms, 0 | 20260506 03:04:37.188 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:37.188 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:37.188 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 03:04:37.188 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:37.188 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:37.188 | INFO | Uploaded 36GiB to GPU in 1.96622s (18.3092GiB/s) | 20260506 03:04:37.188 | INFO | Downloaded 34GiB to GPU in 1.9784s (17.1856GiB/s) | 20260506 03:04:42.329 | INFO | Analyzing chunk frames = [8446976,8447488) | 20260506 03:04:42.329 | INFO | MemInfo: { | 20260506 03:04:42.329 | INFO | "gpu" : " 17 GB", | 20260506 03:04:42.329 | INFO | "huge" : "", | 20260506 03:04:42.329 | INFO | "huge_shared" : " 27 GB", | 20260506 03:04:42.329 | INFO | "malloc" : " 14 GB", | 20260506 03:04:42.329 | INFO | "pinned" : " 29 GB" | 20260506 03:04:42.329 | INFO | } | 20260506 03:04:42.329 | INFO | Perf report for the last 5.14098 seconds: | 20260506 03:04:42.329 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:04:42.329 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:42.329 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:42.329 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:42.329 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260506 03:04:42.329 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:42.329 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:42.329 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 03:04:42.329 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:42.329 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:42.329 | INFO | Uploaded 36GiB to GPU in 2.00992s (17.9112GiB/s) | 20260506 03:04:42.329 | INFO | Downloaded 34GiB to GPU in 1.98365s (17.1401GiB/s) | 20260506 03:04:47.470 | INFO | Analyzing chunk frames = [8447488,8448000) | 20260506 03:04:47.470 | INFO | MemInfo: { | 20260506 03:04:47.470 | INFO | "gpu" : " 17 GB", | 20260506 03:04:47.470 | INFO | "huge" : "", | 20260506 03:04:47.470 | INFO | "huge_shared" : " 27 GB", | 20260506 03:04:47.470 | INFO | "malloc" : " 14 GB", | 20260506 03:04:47.470 | INFO | "pinned" : " 29 GB" | 20260506 03:04:47.470 | INFO | } | 20260506 03:04:47.470 | INFO | Perf report for the last 5.14113 seconds: | 20260506 03:04:47.470 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:04:47.470 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:47.470 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:47.470 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:47.470 | INFO | ANALYSIS : 61.0%, 3.0, 94.8ms, 0 | 20260506 03:04:47.470 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:47.470 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:47.470 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 03:04:47.470 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:47.470 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:47.470 | INFO | Uploaded 36GiB to GPU in 2.06742s (17.413GiB/s) | 20260506 03:04:47.470 | INFO | Downloaded 34GiB to GPU in 2.00787s (16.9334GiB/s) | 20260506 03:04:52.608 | INFO | Analyzing chunk frames = [8448000,8448512) | 20260506 03:04:52.608 | INFO | MemInfo: { | 20260506 03:04:52.608 | INFO | "gpu" : " 17 GB", | 20260506 03:04:52.608 | INFO | "huge" : "", | 20260506 03:04:52.608 | INFO | "huge_shared" : " 27 GB", | 20260506 03:04:52.608 | INFO | "malloc" : " 14 GB", | 20260506 03:04:52.608 | INFO | "pinned" : " 29 GB" | 20260506 03:04:52.608 | INFO | } | 20260506 03:04:52.608 | INFO | Perf report for the last 5.13793 seconds: | 20260506 03:04:52.608 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:04:52.608 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:52.608 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:52.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:52.608 | INFO | ANALYSIS : 60.5%, 3.0, 94.5ms, 0 | 20260506 03:04:52.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:52.608 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:52.608 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:04:52.608 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:52.608 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:52.608 | INFO | Uploaded 36GiB to GPU in 2.07835s (17.3214GiB/s) | 20260506 03:04:52.608 | INFO | Downloaded 34GiB to GPU in 2.0342s (16.7142GiB/s) | 20260506 03:04:57.749 | INFO | Analyzing chunk frames = [8448512,8449024) | 20260506 03:04:57.749 | INFO | MemInfo: { | 20260506 03:04:57.749 | INFO | "gpu" : " 17 GB", | 20260506 03:04:57.749 | INFO | "huge" : "", | 20260506 03:04:57.749 | INFO | "huge_shared" : " 27 GB", | 20260506 03:04:57.749 | INFO | "malloc" : " 14 GB", | 20260506 03:04:57.749 | INFO | "pinned" : " 29 GB" | 20260506 03:04:57.749 | INFO | } | 20260506 03:04:57.749 | INFO | Perf report for the last 5.14147 seconds: | 20260506 03:04:57.749 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:04:57.749 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:57.749 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:57.749 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:57.749 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 03:04:57.750 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:57.750 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:57.750 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 03:04:57.750 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:57.750 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:04:57.750 | INFO | Uploaded 36GiB to GPU in 2.08821s (17.2396GiB/s) | 20260506 03:04:57.750 | INFO | Downloaded 34GiB to GPU in 2.03176s (16.7343GiB/s) | 20260506 03:05:02.887 | INFO | Analyzing chunk frames = [8449024,8449536) | 20260506 03:05:02.887 | INFO | MemInfo: { | 20260506 03:05:02.887 | INFO | "gpu" : " 17 GB", | 20260506 03:05:02.887 | INFO | "huge" : "", | 20260506 03:05:02.887 | INFO | "huge_shared" : " 27 GB", | 20260506 03:05:02.887 | INFO | "malloc" : " 14 GB", | 20260506 03:05:02.887 | INFO | "pinned" : " 29 GB" | 20260506 03:05:02.887 | INFO | } | 20260506 03:05:02.887 | INFO | Perf report for the last 5.13779 seconds: | 20260506 03:05:02.887 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:05:02.887 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:02.887 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:02.887 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:02.887 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260506 03:05:02.887 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:02.887 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:02.887 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 03:05:02.887 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:02.887 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:02.888 | INFO | Uploaded 36GiB to GPU in 2.03089s (17.7262GiB/s) | 20260506 03:05:02.888 | INFO | Downloaded 34GiB to GPU in 2.05843s (16.5174GiB/s) | 20260506 03:05:08.024 | INFO | Analyzing chunk frames = [8449536,8450048) | 20260506 03:05:08.024 | INFO | MemInfo: { | 20260506 03:05:08.024 | INFO | "gpu" : " 17 GB", | 20260506 03:05:08.024 | INFO | "huge" : "", | 20260506 03:05:08.024 | INFO | "huge_shared" : " 27 GB", | 20260506 03:05:08.025 | INFO | "malloc" : " 14 GB", | 20260506 03:05:08.025 | INFO | "pinned" : " 29 GB" | 20260506 03:05:08.025 | INFO | } | 20260506 03:05:08.025 | INFO | Perf report for the last 5.13737 seconds: | 20260506 03:05:08.025 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:05:08.025 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:08.025 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:08.025 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:08.025 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 03:05:08.025 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:08.025 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:08.025 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 03:05:08.025 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:08.025 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:08.025 | INFO | Uploaded 36GiB to GPU in 2.03086s (17.7265GiB/s) | 20260506 03:05:08.025 | INFO | Downloaded 34GiB to GPU in 2.02217s (16.8136GiB/s) | 20260506 03:05:08.150 | INFO | Starting a round of Baz IO | 20260506 03:05:11.276 | INFO | Populating RTMetrics report | 20260506 03:05:11.276 | INFO | RTMetrics report populated | 20260506 03:05:11.276 | INFO | Opening tmp RTMetrics json file | 20260506 03:05:11.277 | INFO | Serializing RTMetrics to json | 20260506 03:05:11.278 | INFO | Serialization done, swapping tmp file now... | 20260506 03:05:11.278 | INFO | RTMetrics done | 20260506 03:05:11.360 | INFO | Finished a round of Baz IO: Wrote 1.771 GiB in 3.209 seconds | 20260506 03:05:11.360 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.24% / 0.00% / 78.76% | 20260506 03:05:11.360 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0636895% / 0% / 9.93411e-05% | 20260506 03:05:11.360 | INFO | Baz IO overall: | 20260506 03:05:11.360 | INFO | BytesWritten : 4713236169071 | 20260506 03:05:11.360 | INFO | NumEvents : 2211150680528 | 20260506 03:05:13.163 | INFO | Analyzing chunk frames = [8450048,8450560) | 20260506 03:05:13.163 | INFO | MemInfo: { | 20260506 03:05:13.163 | INFO | "gpu" : " 17 GB", | 20260506 03:05:13.163 | INFO | "huge" : "", | 20260506 03:05:13.163 | INFO | "huge_shared" : " 27 GB", | 20260506 03:05:13.163 | INFO | "malloc" : " 14 GB", | 20260506 03:05:13.163 | INFO | "pinned" : " 29 GB" | 20260506 03:05:13.163 | INFO | } | 20260506 03:05:13.163 | INFO | Perf report for the last 5.13812 seconds: | 20260506 03:05:13.163 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:05:13.163 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:13.163 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:13.163 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:13.163 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260506 03:05:13.163 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:13.163 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260506 03:05:13.163 | INFO | PRE_HQ : 59.0%, 1.7, 53.1ms, 0 | 20260506 03:05:13.163 | INFO | BAZWRITER : 61.6%, 3.5, 112.0ms, 0 | 20260506 03:05:13.163 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 03:05:13.163 | INFO | Uploaded 36GiB to GPU in 2.10576s (17.0959GiB/s) | 20260506 03:05:13.163 | INFO | Downloaded 38GiB to GPU in 2.22503s (17.0784GiB/s) | 20260506 03:05:18.300 | INFO | Analyzing chunk frames = [8450560,8451072) | 20260506 03:05:18.300 | INFO | MemInfo: { | 20260506 03:05:18.300 | INFO | "gpu" : " 17 GB", | 20260506 03:05:18.300 | INFO | "huge" : "", | 20260506 03:05:18.300 | INFO | "huge_shared" : " 27 GB", | 20260506 03:05:18.300 | INFO | "malloc" : " 14 GB", | 20260506 03:05:18.301 | INFO | "pinned" : " 29 GB" | 20260506 03:05:18.301 | INFO | } | 20260506 03:05:18.301 | INFO | Perf report for the last 5.13782 seconds: | 20260506 03:05:18.301 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:05:18.301 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:18.301 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:18.301 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:18.301 | INFO | ANALYSIS : 60.7%, 3.0, 94.8ms, 0 | 20260506 03:05:18.301 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:18.301 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:18.301 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 03:05:18.301 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:18.301 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:18.301 | INFO | Uploaded 36GiB to GPU in 1.94731s (18.4871GiB/s) | 20260506 03:05:18.301 | INFO | Downloaded 34GiB to GPU in 1.97879s (17.1822GiB/s) | 20260506 03:05:23.446 | INFO | Analyzing chunk frames = [8451072,8451584) | 20260506 03:05:23.446 | INFO | MemInfo: { | 20260506 03:05:23.446 | INFO | "gpu" : " 17 GB", | 20260506 03:05:23.446 | INFO | "huge" : "", | 20260506 03:05:23.446 | INFO | "huge_shared" : " 27 GB", | 20260506 03:05:23.447 | INFO | "malloc" : " 14 GB", | 20260506 03:05:23.447 | INFO | "pinned" : " 29 GB" | 20260506 03:05:23.447 | INFO | } | 20260506 03:05:23.447 | INFO | Perf report for the last 5.14608 seconds: | 20260506 03:05:23.447 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:05:23.447 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:23.447 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:23.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:23.447 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260506 03:05:23.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:23.447 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:23.447 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 03:05:23.447 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:23.447 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:23.447 | INFO | Uploaded 36GiB to GPU in 2.10577s (17.0959GiB/s) | 20260506 03:05:23.447 | INFO | Downloaded 34GiB to GPU in 2.03801s (16.6829GiB/s) | 20260506 03:05:28.586 | INFO | Analyzing chunk frames = [8451584,8452096) | 20260506 03:05:28.586 | INFO | MemInfo: { | 20260506 03:05:28.586 | INFO | "gpu" : " 17 GB", | 20260506 03:05:28.586 | INFO | "huge" : "", | 20260506 03:05:28.586 | INFO | "huge_shared" : " 27 GB", | 20260506 03:05:28.587 | INFO | "malloc" : " 14 GB", | 20260506 03:05:28.587 | INFO | "pinned" : " 29 GB" | 20260506 03:05:28.587 | INFO | } | 20260506 03:05:28.587 | INFO | Perf report for the last 5.14 seconds: | 20260506 03:05:28.587 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:05:28.587 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:28.587 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:28.587 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:28.587 | INFO | ANALYSIS : 60.9%, 3.0, 94.6ms, 0 | 20260506 03:05:28.587 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:28.587 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:28.587 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 03:05:28.587 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:28.587 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:28.587 | INFO | Uploaded 36GiB to GPU in 2.08219s (17.2895GiB/s) | 20260506 03:05:28.587 | INFO | Downloaded 34GiB to GPU in 1.99952s (17.0041GiB/s) | 20260506 03:05:33.720 | INFO | Analyzing chunk frames = [8452096,8452608) | 20260506 03:05:33.720 | INFO | MemInfo: { | 20260506 03:05:33.720 | INFO | "gpu" : " 17 GB", | 20260506 03:05:33.720 | INFO | "huge" : "", | 20260506 03:05:33.720 | INFO | "huge_shared" : " 27 GB", | 20260506 03:05:33.721 | INFO | "malloc" : " 14 GB", | 20260506 03:05:33.721 | INFO | "pinned" : " 29 GB" | 20260506 03:05:33.721 | INFO | } | 20260506 03:05:33.721 | INFO | Perf report for the last 5.13397 seconds: | 20260506 03:05:33.721 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:05:33.721 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:33.721 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:33.721 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:33.721 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 03:05:33.721 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:33.721 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:33.721 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 03:05:33.721 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:33.721 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:33.721 | INFO | Uploaded 36GiB to GPU in 2.11215s (17.0442GiB/s) | 20260506 03:05:33.721 | INFO | Downloaded 34GiB to GPU in 2.03971s (16.669GiB/s) | 20260506 03:05:38.860 | INFO | Analyzing chunk frames = [8452608,8453120) | 20260506 03:05:38.860 | INFO | MemInfo: { | 20260506 03:05:38.860 | INFO | "gpu" : " 17 GB", | 20260506 03:05:38.860 | INFO | "huge" : "", | 20260506 03:05:38.860 | INFO | "huge_shared" : " 27 GB", | 20260506 03:05:38.860 | INFO | "malloc" : " 14 GB", | 20260506 03:05:38.860 | INFO | "pinned" : " 29 GB" | 20260506 03:05:38.860 | INFO | } | 20260506 03:05:38.860 | INFO | Perf report for the last 5.13926 seconds: | 20260506 03:05:38.860 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:05:38.860 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:38.860 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:38.860 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:38.860 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 03:05:38.860 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:38.860 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:38.860 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:05:38.860 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:38.860 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:38.860 | INFO | Uploaded 36GiB to GPU in 2.07648s (17.3371GiB/s) | 20260506 03:05:38.860 | INFO | Downloaded 34GiB to GPU in 2.03525s (16.7056GiB/s) | 20260506 03:05:43.998 | INFO | Analyzing chunk frames = [8453120,8453632) | 20260506 03:05:43.998 | INFO | MemInfo: { | 20260506 03:05:43.998 | INFO | "gpu" : " 17 GB", | 20260506 03:05:43.998 | INFO | "huge" : "", | 20260506 03:05:43.998 | INFO | "huge_shared" : " 27 GB", | 20260506 03:05:43.998 | INFO | "malloc" : " 14 GB", | 20260506 03:05:43.998 | INFO | "pinned" : " 29 GB" | 20260506 03:05:43.998 | INFO | } | 20260506 03:05:43.998 | INFO | Perf report for the last 5.13814 seconds: | 20260506 03:05:43.998 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:05:43.998 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:43.998 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:43.998 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:43.998 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260506 03:05:43.998 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:43.998 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:43.998 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 03:05:43.998 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:43.998 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:43.998 | INFO | Uploaded 36GiB to GPU in 2.03497s (17.6907GiB/s) | 20260506 03:05:43.998 | INFO | Downloaded 34GiB to GPU in 2.05033s (16.5827GiB/s) | 20260506 03:05:49.140 | INFO | Analyzing chunk frames = [8453632,8454144) | 20260506 03:05:49.140 | INFO | MemInfo: { | 20260506 03:05:49.140 | INFO | "gpu" : " 17 GB", | 20260506 03:05:49.140 | INFO | "huge" : "", | 20260506 03:05:49.140 | INFO | "huge_shared" : " 27 GB", | 20260506 03:05:49.140 | INFO | "malloc" : " 14 GB", | 20260506 03:05:49.140 | INFO | "pinned" : " 29 GB" | 20260506 03:05:49.140 | INFO | } | 20260506 03:05:49.140 | INFO | Perf report for the last 5.14212 seconds: | 20260506 03:05:49.140 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:05:49.140 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:49.140 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:49.140 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:49.140 | INFO | ANALYSIS : 61.2%, 3.0, 95.6ms, 0 | 20260506 03:05:49.140 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:49.140 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:49.140 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 03:05:49.140 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:49.140 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:49.140 | INFO | Uploaded 36GiB to GPU in 2.06787s (17.4092GiB/s) | 20260506 03:05:49.141 | INFO | Downloaded 34GiB to GPU in 2.02283s (16.8082GiB/s) | 20260506 03:05:49.141 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 03:05:49.141 | INFO | Total mem usage: 15.4 GiB | 20260506 03:05:49.141 | INFO | All Time High Water Marks sum: 16.3 GiB (Savings: 5.2%) | 20260506 03:05:49.141 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -10%) | 20260506 03:05:49.141 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 03:05:49.141 | INFO | High Water Breakdown: | 20260506 03:05:49.141 | INFO | 60.9% : 8.54 GiB ( 88.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 03:05:49.141 | INFO | 39% : 5.47 GiB ( 82.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 03:05:49.141 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 03:05:49.141 | INFO | ------------------------------------------ | 20260506 03:05:49.141 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:05:49.141 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 03:05:49.141 | INFO | Total mem usage: 26.5 GiB | 20260506 03:05:49.141 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 03:05:49.141 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 03:05:49.141 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 03:05:49.141 | INFO | High Water Breakdown: | 20260506 03:05:49.141 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 03:05:49.141 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 03:05:49.141 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 03:05:49.141 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 03:05:49.141 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 03:05:49.141 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 03:05:49.141 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 03:05:49.141 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 03:05:49.141 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 03:05:49.141 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 03:05:49.141 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 03:05:49.141 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 03:05:49.141 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 03:05:49.141 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 03:05:49.141 | INFO | ------------------------------------------ | 20260506 03:05:49.141 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:05:49.141 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 03:05:49.141 | INFO | Total mem usage: 25.6 GiB | 20260506 03:05:49.141 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 03:05:49.141 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 03:05:49.141 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 03:05:49.141 | INFO | High Water Breakdown: | 20260506 03:05:49.141 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 03:05:49.141 | INFO | ------------------------------------------ | 20260506 03:05:49.141 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:05:49.141 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 03:05:49.141 | INFO | Total mem usage: 15.7 GiB | 20260506 03:05:49.142 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 03:05:49.142 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 03:05:49.142 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 03:05:49.142 | INFO | High Water Breakdown: | 20260506 03:05:49.142 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 03:05:49.142 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 03:05:49.142 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 03:05:49.142 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 03:05:49.142 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 03:05:49.142 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 03:05:49.142 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 03:05:49.142 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 03:05:49.142 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 03:05:49.142 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 03:05:49.142 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 03:05:49.142 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 03:05:49.142 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 03:05:49.142 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 03:05:49.142 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 03:05:49.142 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 03:05:49.142 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 03:05:49.142 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 03:05:49.142 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 03:05:49.142 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 03:05:49.142 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 03:05:49.142 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 03:05:49.142 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 03:05:49.142 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 03:05:49.142 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 03:05:49.142 | INFO | ------------------------------------------ | 20260506 03:05:49.142 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:05:49.142 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 03:05:49.142 | INFO | PerfReport: Gathered 1536 samples. | 20260506 03:05:49.142 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 03:05:49.142 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.38e+01, 1.32e+01 / 9.60e-01 | 20260506 03:05:49.142 | INFO | PerfReport: 0.29%, AggregateStats -- 9.18e-02 / 1.04e-01, 1.00e-02 / 3.28e-03 | 20260506 03:05:49.142 | INFO | PerfReport: 11.89%, Baseline -- 3.72e+00 / 4.48e+00, 5.91e-01 / 2.22e-01 | 20260506 03:05:49.142 | INFO | PerfReport: 3.32%, Histogram -- 1.04e+00 / 1.52e+00, 1.98e-01 / 9.01e-02 | 20260506 03:05:49.142 | INFO | PerfReport: 16.01%, DME -- 4.01e+01 / 4.43e+01, 4.00e+00 / 1.19e+00 | 20260506 03:05:49.142 | INFO | PerfReport: 57.50%, FrameLabeling -- 1.80e+01 / 1.90e+01, 1.56e+00 / 5.61e-01 | 20260506 03:05:49.142 | INFO | PerfReport: 6.98%, PulseAccumulating -- 2.18e+00 / 2.76e+00, 4.15e-01 / 1.43e-01 | 20260506 03:05:49.142 | INFO | PerfReport: 2.98%, Metrics -- 9.33e-01 / 1.21e+00, 2.47e-01 / 1.18e-01 | 20260506 03:05:49.142 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.22e-01 / 3.72e-01, 5.49e-02 / 2.56e-02 | 20260506 03:05:49.142 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 03:05:49.142 | INFO | PerfReport: Gathered 1536 samples. | 20260506 03:05:49.142 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 03:05:49.142 | INFO | PerfReport: Full Execution -- 9.74e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260506 03:05:49.142 | INFO | PerfReport: 22.02%, Upload -- 2.15e+01 / 0.00e+00, 6.99e+00 / 0.00e+00 | 20260506 03:05:49.142 | INFO | PerfReport: 21.88%, Download -- 2.13e+01 / 0.00e+00, 7.56e+00 / 0.00e+00 | 20260506 03:05:49.142 | INFO | PerfReport: 32.13%, Compute -- 3.13e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260506 03:05:49.142 | INFO | PerfReport: 2.00%, LockUpload -- 1.94e+00 / 0.00e+00, 7.81e+00 / 0.00e+00 | 20260506 03:05:49.142 | INFO | PerfReport: 1.33%, LockDownload -- 1.30e+00 / 0.00e+00, 3.08e+00 / 0.00e+00 | 20260506 03:05:49.142 | INFO | PerfReport: 20.63%, LockCompute -- 2.01e+01 / 0.00e+00, 1.76e+01 / 0.00e+00 | 20260506 03:05:49.142 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.51e-02 / 0.00e+00 | 20260506 03:05:49.263 | INFO | Starting a round of Baz IO | 20260506 03:05:52.388 | INFO | Populating RTMetrics report | 20260506 03:05:52.388 | INFO | RTMetrics report populated | 20260506 03:05:52.388 | INFO | Opening tmp RTMetrics json file | 20260506 03:05:52.391 | INFO | Serializing RTMetrics to json | 20260506 03:05:52.393 | INFO | Serialization done, swapping tmp file now... | 20260506 03:05:52.393 | INFO | RTMetrics done | 20260506 03:05:52.475 | INFO | Finished a round of Baz IO: Wrote 1.773 GiB in 3.211 seconds | 20260506 03:05:52.475 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.24% / 0.00% / 78.76% | 20260506 03:05:52.475 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0637293% / 0% / 9.93411e-05% | 20260506 03:05:52.475 | INFO | Baz IO overall: | 20260506 03:05:52.475 | INFO | BytesWritten : 4715139767310 | 20260506 03:05:52.475 | INFO | NumEvents : 2212014786086 | 20260506 03:05:54.280 | INFO | Analyzing chunk frames = [8454144,8454656) | 20260506 03:05:54.280 | INFO | MemInfo: { | 20260506 03:05:54.280 | INFO | "gpu" : " 17 GB", | 20260506 03:05:54.280 | INFO | "huge" : "", | 20260506 03:05:54.280 | INFO | "huge_shared" : " 27 GB", | 20260506 03:05:54.280 | INFO | "malloc" : " 14 GB", | 20260506 03:05:54.280 | INFO | "pinned" : " 29 GB" | 20260506 03:05:54.280 | INFO | } | 20260506 03:05:54.280 | INFO | Perf report for the last 5.1396 seconds: | 20260506 03:05:54.280 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:05:54.280 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:54.280 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:54.280 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:54.280 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260506 03:05:54.280 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:54.280 | INFO | RT_METRICS : 42.6%, 1.0, 22.4ms, 0 | 20260506 03:05:54.280 | INFO | PRE_HQ : 59.1%, 1.7, 53.2ms, 0 | 20260506 03:05:54.280 | INFO | BAZWRITER : 61.6%, 3.5, 112.1ms, 0 | 20260506 03:05:54.280 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 03:05:54.280 | INFO | Uploaded 36GiB to GPU in 2.12462s (16.9442GiB/s) | 20260506 03:05:54.280 | INFO | Downloaded 38GiB to GPU in 2.2276s (17.0587GiB/s) | 20260506 03:05:59.422 | INFO | Analyzing chunk frames = [8454656,8455168) | 20260506 03:05:59.422 | INFO | MemInfo: { | 20260506 03:05:59.422 | INFO | "gpu" : " 17 GB", | 20260506 03:05:59.422 | INFO | "huge" : "", | 20260506 03:05:59.422 | INFO | "huge_shared" : " 27 GB", | 20260506 03:05:59.422 | INFO | "malloc" : " 14 GB", | 20260506 03:05:59.422 | INFO | "pinned" : " 29 GB" | 20260506 03:05:59.422 | INFO | } | 20260506 03:05:59.422 | INFO | Perf report for the last 5.14222 seconds: | 20260506 03:05:59.422 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:05:59.422 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:59.422 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:59.422 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:59.422 | INFO | ANALYSIS : 60.5%, 3.0, 94.5ms, 0 | 20260506 03:05:59.422 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:59.422 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:59.422 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 03:05:59.422 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:59.422 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:05:59.422 | INFO | Uploaded 36GiB to GPU in 1.93463s (18.6082GiB/s) | 20260506 03:05:59.422 | INFO | Downloaded 34GiB to GPU in 1.98685s (17.1125GiB/s) | 20260506 03:06:04.557 | INFO | Analyzing chunk frames = [8455168,8455680) | 20260506 03:06:04.557 | INFO | MemInfo: { | 20260506 03:06:04.557 | INFO | "gpu" : " 17 GB", | 20260506 03:06:04.557 | INFO | "huge" : "", | 20260506 03:06:04.557 | INFO | "huge_shared" : " 27 GB", | 20260506 03:06:04.558 | INFO | "malloc" : " 14 GB", | 20260506 03:06:04.558 | INFO | "pinned" : " 29 GB" | 20260506 03:06:04.558 | INFO | } | 20260506 03:06:04.558 | INFO | Perf report for the last 5.13559 seconds: | 20260506 03:06:04.558 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:06:04.558 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:04.558 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:04.558 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:04.558 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260506 03:06:04.558 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:04.558 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:04.558 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 03:06:04.558 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:04.558 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:04.558 | INFO | Uploaded 36GiB to GPU in 2.08765s (17.2443GiB/s) | 20260506 03:06:04.558 | INFO | Downloaded 34GiB to GPU in 2.03827s (16.6808GiB/s) | 20260506 03:06:09.699 | INFO | Analyzing chunk frames = [8455680,8456192) | 20260506 03:06:09.699 | INFO | MemInfo: { | 20260506 03:06:09.699 | INFO | "gpu" : " 17 GB", | 20260506 03:06:09.699 | INFO | "huge" : "", | 20260506 03:06:09.699 | INFO | "huge_shared" : " 27 GB", | 20260506 03:06:09.700 | INFO | "malloc" : " 14 GB", | 20260506 03:06:09.700 | INFO | "pinned" : " 29 GB" | 20260506 03:06:09.700 | INFO | } | 20260506 03:06:09.700 | INFO | Perf report for the last 5.14203 seconds: | 20260506 03:06:09.700 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:06:09.700 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:09.700 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:09.700 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:09.700 | INFO | ANALYSIS : 61.0%, 3.0, 94.7ms, 0 | 20260506 03:06:09.700 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:09.700 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:09.700 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 03:06:09.700 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:09.700 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:09.700 | INFO | Uploaded 36GiB to GPU in 2.07514s (17.3483GiB/s) | 20260506 03:06:09.700 | INFO | Downloaded 34GiB to GPU in 2.01731s (16.8541GiB/s) | 20260506 03:06:14.842 | INFO | Analyzing chunk frames = [8456192,8456704) | 20260506 03:06:14.842 | INFO | MemInfo: { | 20260506 03:06:14.842 | INFO | "gpu" : " 17 GB", | 20260506 03:06:14.842 | INFO | "huge" : "", | 20260506 03:06:14.842 | INFO | "huge_shared" : " 27 GB", | 20260506 03:06:14.842 | INFO | "malloc" : " 14 GB", | 20260506 03:06:14.842 | INFO | "pinned" : " 29 GB" | 20260506 03:06:14.842 | INFO | } | 20260506 03:06:14.843 | INFO | Perf report for the last 5.1428 seconds: | 20260506 03:06:14.843 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:06:14.843 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:14.843 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:14.843 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:14.843 | INFO | ANALYSIS : 60.8%, 3.0, 94.9ms, 0 | 20260506 03:06:14.843 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:14.843 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:14.843 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:06:14.843 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:14.843 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:14.843 | INFO | Uploaded 36GiB to GPU in 2.10676s (17.0878GiB/s) | 20260506 03:06:14.843 | INFO | Downloaded 34GiB to GPU in 2.04078s (16.6603GiB/s) | 20260506 03:06:19.978 | INFO | Analyzing chunk frames = [8456704,8457216) | 20260506 03:06:19.978 | INFO | MemInfo: { | 20260506 03:06:19.978 | INFO | "gpu" : " 17 GB", | 20260506 03:06:19.978 | INFO | "huge" : "", | 20260506 03:06:19.978 | INFO | "huge_shared" : " 27 GB", | 20260506 03:06:19.978 | INFO | "malloc" : " 14 GB", | 20260506 03:06:19.978 | INFO | "pinned" : " 29 GB" | 20260506 03:06:19.978 | INFO | } | 20260506 03:06:19.978 | INFO | Perf report for the last 5.13532 seconds: | 20260506 03:06:19.978 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:06:19.978 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:19.978 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:19.978 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:19.978 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 03:06:19.978 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:19.978 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:19.978 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 03:06:19.978 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:19.978 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:19.978 | INFO | Uploaded 36GiB to GPU in 2.05631s (17.5071GiB/s) | 20260506 03:06:19.978 | INFO | Downloaded 34GiB to GPU in 2.01964s (16.8347GiB/s) | 20260506 03:06:25.112 | INFO | Analyzing chunk frames = [8457216,8457728) | 20260506 03:06:25.112 | INFO | MemInfo: { | 20260506 03:06:25.112 | INFO | "gpu" : " 17 GB", | 20260506 03:06:25.112 | INFO | "huge" : "", | 20260506 03:06:25.112 | INFO | "huge_shared" : " 27 GB", | 20260506 03:06:25.114 | INFO | "malloc" : " 14 GB", | 20260506 03:06:25.114 | INFO | "pinned" : " 29 GB" | 20260506 03:06:25.114 | INFO | } | 20260506 03:06:25.115 | INFO | Perf report for the last 5.13664 seconds: | 20260506 03:06:25.115 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:06:25.115 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:25.115 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:25.115 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:25.115 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260506 03:06:25.115 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:25.115 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:25.115 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 03:06:25.115 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:25.115 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:25.115 | INFO | Uploaded 36GiB to GPU in 2.02797s (17.7517GiB/s) | 20260506 03:06:25.115 | INFO | Downloaded 34GiB to GPU in 2.05605s (16.5366GiB/s) | 20260506 03:06:30.254 | INFO | Analyzing chunk frames = [8457728,8458240) | 20260506 03:06:30.254 | INFO | MemInfo: { | 20260506 03:06:30.254 | INFO | "gpu" : " 17 GB", | 20260506 03:06:30.254 | INFO | "huge" : "", | 20260506 03:06:30.254 | INFO | "huge_shared" : " 27 GB", | 20260506 03:06:30.254 | INFO | "malloc" : " 14 GB", | 20260506 03:06:30.254 | INFO | "pinned" : " 29 GB" | 20260506 03:06:30.254 | INFO | } | 20260506 03:06:30.254 | INFO | Perf report for the last 5.13966 seconds: | 20260506 03:06:30.254 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:06:30.254 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:30.254 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:30.254 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:30.254 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260506 03:06:30.254 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:30.254 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:30.254 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:06:30.254 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:30.254 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:30.254 | INFO | Uploaded 36GiB to GPU in 2.02658s (17.764GiB/s) | 20260506 03:06:30.255 | INFO | Downloaded 34GiB to GPU in 2.02252s (16.8107GiB/s) | 20260506 03:06:30.379 | INFO | Starting a round of Baz IO | 20260506 03:06:33.501 | INFO | Populating RTMetrics report | 20260506 03:06:33.501 | INFO | RTMetrics report populated | 20260506 03:06:33.501 | INFO | Opening tmp RTMetrics json file | 20260506 03:06:33.503 | INFO | Serializing RTMetrics to json | 20260506 03:06:33.504 | INFO | Serialization done, swapping tmp file now... | 20260506 03:06:33.505 | INFO | RTMetrics done | 20260506 03:06:33.584 | INFO | Finished a round of Baz IO: Wrote 1.772 GiB in 3.205 seconds | 20260506 03:06:33.584 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.24% / 0.00% / 78.76% | 20260506 03:06:33.584 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0635664% / 0% / 6.35783e-05% | 20260506 03:06:33.584 | INFO | Baz IO overall: | 20260506 03:06:33.584 | INFO | BytesWritten : 4717042251114 | 20260506 03:06:33.584 | INFO | NumEvents : 2212878274256 | 20260506 03:06:35.395 | INFO | Analyzing chunk frames = [8458240,8458752) | 20260506 03:06:35.395 | INFO | MemInfo: { | 20260506 03:06:35.395 | INFO | "gpu" : " 17 GB", | 20260506 03:06:35.395 | INFO | "huge" : "", | 20260506 03:06:35.395 | INFO | "huge_shared" : " 27 GB", | 20260506 03:06:35.395 | INFO | "malloc" : " 14 GB", | 20260506 03:06:35.395 | INFO | "pinned" : " 29 GB" | 20260506 03:06:35.395 | INFO | } | 20260506 03:06:35.395 | INFO | Perf report for the last 5.14061 seconds: | 20260506 03:06:35.395 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:06:35.395 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:35.395 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:35.395 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:35.395 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 03:06:35.395 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:35.395 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260506 03:06:35.395 | INFO | PRE_HQ : 59.1%, 1.7, 53.3ms, 0 | 20260506 03:06:35.395 | INFO | BAZWRITER : 61.6%, 3.6, 114.7ms, 0 | 20260506 03:06:35.395 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 03:06:35.395 | INFO | Uploaded 36GiB to GPU in 2.1292s (16.9078GiB/s) | 20260506 03:06:35.395 | INFO | Downloaded 38GiB to GPU in 2.22822s (17.054GiB/s) | 20260506 03:06:40.530 | INFO | Analyzing chunk frames = [8458752,8459264) | 20260506 03:06:40.530 | INFO | MemInfo: { | 20260506 03:06:40.530 | INFO | "gpu" : " 17 GB", | 20260506 03:06:40.530 | INFO | "huge" : "", | 20260506 03:06:40.530 | INFO | "huge_shared" : " 27 GB", | 20260506 03:06:40.530 | INFO | "malloc" : " 14 GB", | 20260506 03:06:40.530 | INFO | "pinned" : " 29 GB" | 20260506 03:06:40.530 | INFO | } | 20260506 03:06:40.530 | INFO | Perf report for the last 5.13536 seconds: | 20260506 03:06:40.530 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:06:40.530 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:40.530 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:40.530 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:40.530 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260506 03:06:40.530 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:40.530 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:40.530 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 03:06:40.530 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:40.530 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:40.531 | INFO | Uploaded 36GiB to GPU in 1.98481s (18.1377GiB/s) | 20260506 03:06:40.531 | INFO | Downloaded 34GiB to GPU in 1.99175s (17.0704GiB/s) | 20260506 03:06:45.667 | INFO | Analyzing chunk frames = [8459264,8459776) | 20260506 03:06:45.667 | INFO | MemInfo: { | 20260506 03:06:45.667 | INFO | "gpu" : " 17 GB", | 20260506 03:06:45.667 | INFO | "huge" : "", | 20260506 03:06:45.667 | INFO | "huge_shared" : " 27 GB", | 20260506 03:06:45.667 | INFO | "malloc" : " 14 GB", | 20260506 03:06:45.667 | INFO | "pinned" : " 29 GB" | 20260506 03:06:45.667 | INFO | } | 20260506 03:06:45.667 | INFO | Perf report for the last 5.13681 seconds: | 20260506 03:06:45.667 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:06:45.667 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:45.667 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:45.667 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:45.667 | INFO | ANALYSIS : 61.2%, 3.0, 95.1ms, 0 | 20260506 03:06:45.667 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:45.667 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:45.667 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 03:06:45.667 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:45.667 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:45.667 | INFO | Uploaded 36GiB to GPU in 2.08501s (17.2661GiB/s) | 20260506 03:06:45.667 | INFO | Downloaded 34GiB to GPU in 2.02116s (16.822GiB/s) | 20260506 03:06:50.813 | INFO | Analyzing chunk frames = [8459776,8460288) | 20260506 03:06:50.813 | INFO | MemInfo: { | 20260506 03:06:50.813 | INFO | "gpu" : " 17 GB", | 20260506 03:06:50.813 | INFO | "huge" : "", | 20260506 03:06:50.813 | INFO | "huge_shared" : " 27 GB", | 20260506 03:06:50.813 | INFO | "malloc" : " 14 GB", | 20260506 03:06:50.813 | INFO | "pinned" : " 29 GB" | 20260506 03:06:50.813 | INFO | } | 20260506 03:06:50.813 | INFO | Perf report for the last 5.1463 seconds: | 20260506 03:06:50.813 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:06:50.813 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:50.813 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:50.813 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:50.813 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 03:06:50.813 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:50.813 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:50.813 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 03:06:50.813 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:50.813 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:50.814 | INFO | Uploaded 36GiB to GPU in 2.14786s (16.7609GiB/s) | 20260506 03:06:50.814 | INFO | Downloaded 34GiB to GPU in 2.04504s (16.6256GiB/s) | 20260506 03:06:55.956 | INFO | Analyzing chunk frames = [8460288,8460800) | 20260506 03:06:55.956 | INFO | MemInfo: { | 20260506 03:06:55.956 | INFO | "gpu" : " 17 GB", | 20260506 03:06:55.956 | INFO | "huge" : "", | 20260506 03:06:55.956 | INFO | "huge_shared" : " 27 GB", | 20260506 03:06:55.956 | INFO | "malloc" : " 14 GB", | 20260506 03:06:55.956 | INFO | "pinned" : " 29 GB" | 20260506 03:06:55.956 | INFO | } | 20260506 03:06:55.956 | INFO | Perf report for the last 5.14295 seconds: | 20260506 03:06:55.956 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:06:55.956 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:55.956 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:55.956 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:55.956 | INFO | ANALYSIS : 60.7%, 3.0, 94.8ms, 0 | 20260506 03:06:55.956 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:55.956 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:55.956 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:06:55.956 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:55.956 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:06:55.957 | INFO | Uploaded 36GiB to GPU in 2.10964s (17.0645GiB/s) | 20260506 03:06:55.957 | INFO | Downloaded 34GiB to GPU in 2.03782s (16.6845GiB/s) | 20260506 03:07:01.091 | INFO | Analyzing chunk frames = [8460800,8461312) | 20260506 03:07:01.091 | INFO | MemInfo: { | 20260506 03:07:01.091 | INFO | "gpu" : " 17 GB", | 20260506 03:07:01.091 | INFO | "huge" : "", | 20260506 03:07:01.091 | INFO | "huge_shared" : " 27 GB", | 20260506 03:07:01.091 | INFO | "malloc" : " 14 GB", | 20260506 03:07:01.091 | INFO | "pinned" : " 29 GB" | 20260506 03:07:01.091 | INFO | } | 20260506 03:07:01.091 | INFO | Perf report for the last 5.13494 seconds: | 20260506 03:07:01.091 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:07:01.091 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:01.091 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:01.091 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:01.091 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260506 03:07:01.091 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:01.091 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:01.091 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 03:07:01.091 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:01.091 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:01.091 | INFO | Uploaded 36GiB to GPU in 2.04359s (17.6161GiB/s) | 20260506 03:07:01.092 | INFO | Downloaded 34GiB to GPU in 2.0324s (16.729GiB/s) | 20260506 03:07:06.227 | INFO | Analyzing chunk frames = [8461312,8461824) | 20260506 03:07:06.227 | INFO | MemInfo: { | 20260506 03:07:06.227 | INFO | "gpu" : " 17 GB", | 20260506 03:07:06.227 | INFO | "huge" : "", | 20260506 03:07:06.227 | INFO | "huge_shared" : " 27 GB", | 20260506 03:07:06.227 | INFO | "malloc" : " 14 GB", | 20260506 03:07:06.227 | INFO | "pinned" : " 29 GB" | 20260506 03:07:06.227 | INFO | } | 20260506 03:07:06.227 | INFO | Perf report for the last 5.13569 seconds: | 20260506 03:07:06.227 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:07:06.227 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:06.227 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:06.227 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:06.227 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260506 03:07:06.227 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:06.227 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:06.227 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 03:07:06.227 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:06.227 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:06.227 | INFO | Uploaded 36GiB to GPU in 2.06503s (17.4332GiB/s) | 20260506 03:07:06.227 | INFO | Downloaded 34GiB to GPU in 2.08609s (16.2984GiB/s) | 20260506 03:07:11.367 | INFO | Analyzing chunk frames = [8461824,8462336) | 20260506 03:07:11.367 | INFO | MemInfo: { | 20260506 03:07:11.367 | INFO | "gpu" : " 17 GB", | 20260506 03:07:11.367 | INFO | "huge" : "", | 20260506 03:07:11.367 | INFO | "huge_shared" : " 27 GB", | 20260506 03:07:11.367 | INFO | "malloc" : " 14 GB", | 20260506 03:07:11.367 | INFO | "pinned" : " 29 GB" | 20260506 03:07:11.367 | INFO | } | 20260506 03:07:11.367 | INFO | Perf report for the last 5.14035 seconds: | 20260506 03:07:11.367 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:07:11.367 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:11.367 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:11.367 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:11.367 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260506 03:07:11.367 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:11.367 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:11.367 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:07:11.367 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:11.367 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:11.368 | INFO | Uploaded 36GiB to GPU in 2.04146s (17.6344GiB/s) | 20260506 03:07:11.368 | INFO | Downloaded 34GiB to GPU in 2.02151s (16.8191GiB/s) | 20260506 03:07:11.368 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 03:07:11.368 | INFO | Total mem usage: 15.4 GiB | 20260506 03:07:11.368 | INFO | All Time High Water Marks sum: 16.3 GiB (Savings: 5.2%) | 20260506 03:07:11.368 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -9.33%) | 20260506 03:07:11.368 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 03:07:11.368 | INFO | High Water Breakdown: | 20260506 03:07:11.368 | INFO | 60.9% : 8.6 GiB ( 89.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 03:07:11.368 | INFO | 38.9% : 5.5 GiB ( 82.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 03:07:11.368 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 03:07:11.368 | INFO | ------------------------------------------ | 20260506 03:07:11.368 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:07:11.368 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 03:07:11.368 | INFO | Total mem usage: 26.5 GiB | 20260506 03:07:11.368 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 03:07:11.368 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 03:07:11.368 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 03:07:11.368 | INFO | High Water Breakdown: | 20260506 03:07:11.368 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 03:07:11.368 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 03:07:11.368 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 03:07:11.368 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 03:07:11.368 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 03:07:11.368 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 03:07:11.368 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 03:07:11.368 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 03:07:11.368 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 03:07:11.368 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 03:07:11.368 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 03:07:11.368 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 03:07:11.368 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 03:07:11.368 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 03:07:11.368 | INFO | ------------------------------------------ | 20260506 03:07:11.368 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:07:11.368 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 03:07:11.368 | INFO | Total mem usage: 25.6 GiB | 20260506 03:07:11.368 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 03:07:11.368 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 03:07:11.368 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 03:07:11.368 | INFO | High Water Breakdown: | 20260506 03:07:11.368 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 03:07:11.368 | INFO | ------------------------------------------ | 20260506 03:07:11.368 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:07:11.368 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 03:07:11.368 | INFO | Total mem usage: 15.7 GiB | 20260506 03:07:11.368 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 03:07:11.368 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 03:07:11.368 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 03:07:11.368 | INFO | High Water Breakdown: | 20260506 03:07:11.368 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 03:07:11.368 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 03:07:11.368 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 03:07:11.368 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 03:07:11.368 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 03:07:11.368 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 03:07:11.368 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 03:07:11.368 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 03:07:11.368 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 03:07:11.368 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 03:07:11.368 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 03:07:11.368 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 03:07:11.368 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 03:07:11.368 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 03:07:11.368 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 03:07:11.368 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 03:07:11.368 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 03:07:11.368 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 03:07:11.368 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 03:07:11.368 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 03:07:11.368 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 03:07:11.369 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 03:07:11.369 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 03:07:11.369 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 03:07:11.369 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 03:07:11.369 | INFO | ------------------------------------------ | 20260506 03:07:11.369 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:07:11.369 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 03:07:11.369 | INFO | PerfReport: Gathered 1536 samples. | 20260506 03:07:11.369 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 03:07:11.369 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.37e+01, 1.33e+01 / 6.20e-01 | 20260506 03:07:11.369 | INFO | PerfReport: 0.29%, AggregateStats -- 9.16e-02 / 1.05e-01, 1.06e-02 / 6.19e-03 | 20260506 03:07:11.369 | INFO | PerfReport: 11.88%, Baseline -- 3.71e+00 / 4.47e+00, 5.91e-01 / 2.88e-01 | 20260506 03:07:11.369 | INFO | PerfReport: 3.32%, Histogram -- 1.04e+00 / 1.54e+00, 1.99e-01 / 9.34e-02 | 20260506 03:07:11.369 | INFO | PerfReport: 16.01%, DME -- 4.00e+01 / 4.41e+01, 3.80e+00 / 8.11e-01 | 20260506 03:07:11.369 | INFO | PerfReport: 57.48%, FrameLabeling -- 1.80e+01 / 1.92e+01, 1.50e+00 / 9.19e-01 | 20260506 03:07:11.369 | INFO | PerfReport: 7.00%, PulseAccumulating -- 2.19e+00 / 2.75e+00, 4.16e-01 / 1.95e-01 | 20260506 03:07:11.369 | INFO | PerfReport: 2.99%, Metrics -- 9.34e-01 / 1.21e+00, 2.53e-01 / 1.05e-01 | 20260506 03:07:11.369 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.22e-01 / 3.75e-01, 5.48e-02 / 2.80e-02 | 20260506 03:07:11.369 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 03:07:11.369 | INFO | PerfReport: Gathered 1536 samples. | 20260506 03:07:11.369 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 03:07:11.369 | INFO | PerfReport: Full Execution -- 9.74e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260506 03:07:11.369 | INFO | PerfReport: 22.08%, Upload -- 2.15e+01 / 0.00e+00, 7.11e+00 / 0.00e+00 | 20260506 03:07:11.369 | INFO | PerfReport: 21.97%, Download -- 2.14e+01 / 0.00e+00, 7.58e+00 / 0.00e+00 | 20260506 03:07:11.369 | INFO | PerfReport: 32.09%, Compute -- 3.13e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 03:07:11.369 | INFO | PerfReport: 1.99%, LockUpload -- 1.94e+00 / 0.00e+00, 7.87e+00 / 0.00e+00 | 20260506 03:07:11.369 | INFO | PerfReport: 1.34%, LockDownload -- 1.30e+00 / 0.00e+00, 3.04e+00 / 0.00e+00 | 20260506 03:07:11.369 | INFO | PerfReport: 20.52%, LockCompute -- 2.00e+01 / 0.00e+00, 1.75e+01 / 0.00e+00 | 20260506 03:07:11.369 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.95e-02 / 0.00e+00 | 20260506 03:07:11.492 | INFO | Starting a round of Baz IO | 20260506 03:07:14.623 | INFO | Populating RTMetrics report | 20260506 03:07:14.623 | INFO | RTMetrics report populated | 20260506 03:07:14.623 | INFO | Opening tmp RTMetrics json file | 20260506 03:07:14.625 | INFO | Serializing RTMetrics to json | 20260506 03:07:14.626 | INFO | Serialization done, swapping tmp file now... | 20260506 03:07:14.627 | INFO | RTMetrics done | 20260506 03:07:14.705 | INFO | Finished a round of Baz IO: Wrote 1.771 GiB in 3.212 seconds | 20260506 03:07:14.705 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.24% / 0.00% / 78.76% | 20260506 03:07:14.705 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0637372% / 0% / 8.34465e-05% | 20260506 03:07:14.705 | INFO | Baz IO overall: | 20260506 03:07:14.705 | INFO | BytesWritten : 4718944355980 | 20260506 03:07:14.705 | INFO | NumEvents : 2213741375617 | 20260506 03:07:16.512 | INFO | Analyzing chunk frames = [8462336,8462848) | 20260506 03:07:16.512 | INFO | MemInfo: { | 20260506 03:07:16.512 | INFO | "gpu" : " 17 GB", | 20260506 03:07:16.512 | INFO | "huge" : "", | 20260506 03:07:16.512 | INFO | "huge_shared" : " 27 GB", | 20260506 03:07:16.512 | INFO | "malloc" : " 14 GB", | 20260506 03:07:16.512 | INFO | "pinned" : " 29 GB" | 20260506 03:07:16.512 | INFO | } | 20260506 03:07:16.512 | INFO | Perf report for the last 5.14458 seconds: | 20260506 03:07:16.512 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:07:16.512 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:16.512 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:16.512 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:16.512 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260506 03:07:16.512 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:16.512 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260506 03:07:16.512 | INFO | PRE_HQ : 58.9%, 1.7, 53.3ms, 0 | 20260506 03:07:16.512 | INFO | BAZWRITER : 61.6%, 3.6, 112.6ms, 0 | 20260506 03:07:16.512 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 03:07:16.512 | INFO | Uploaded 36GiB to GPU in 2.0981s (17.1584GiB/s) | 20260506 03:07:16.512 | INFO | Downloaded 38GiB to GPU in 2.22738s (17.0604GiB/s) | 20260506 03:07:21.648 | INFO | Analyzing chunk frames = [8462848,8463360) | 20260506 03:07:21.648 | INFO | MemInfo: { | 20260506 03:07:21.648 | INFO | "gpu" : " 17 GB", | 20260506 03:07:21.648 | INFO | "huge" : "", | 20260506 03:07:21.648 | INFO | "huge_shared" : " 27 GB", | 20260506 03:07:21.648 | INFO | "malloc" : " 14 GB", | 20260506 03:07:21.648 | INFO | "pinned" : " 29 GB" | 20260506 03:07:21.648 | INFO | } | 20260506 03:07:21.649 | INFO | Perf report for the last 5.1368 seconds: | 20260506 03:07:21.649 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:07:21.649 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:21.649 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:21.649 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:21.649 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260506 03:07:21.649 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:21.649 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:21.649 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 03:07:21.649 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:21.649 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:21.649 | INFO | Uploaded 36GiB to GPU in 1.98693s (18.1184GiB/s) | 20260506 03:07:21.649 | INFO | Downloaded 34GiB to GPU in 1.99093s (17.0775GiB/s) | 20260506 03:07:26.790 | INFO | Analyzing chunk frames = [8463360,8463872) | 20260506 03:07:26.790 | INFO | MemInfo: { | 20260506 03:07:26.790 | INFO | "gpu" : " 17 GB", | 20260506 03:07:26.790 | INFO | "huge" : "", | 20260506 03:07:26.790 | INFO | "huge_shared" : " 27 GB", | 20260506 03:07:26.790 | INFO | "malloc" : " 14 GB", | 20260506 03:07:26.790 | INFO | "pinned" : " 29 GB" | 20260506 03:07:26.790 | INFO | } | 20260506 03:07:26.790 | INFO | Perf report for the last 5.14173 seconds: | 20260506 03:07:26.790 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:07:26.790 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:26.790 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:26.790 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:26.790 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 03:07:26.791 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:26.791 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:26.791 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 03:07:26.791 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:26.791 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:26.791 | INFO | Uploaded 36GiB to GPU in 2.08821s (17.2396GiB/s) | 20260506 03:07:26.791 | INFO | Downloaded 34GiB to GPU in 2.01811s (16.8474GiB/s) | 20260506 03:07:31.929 | INFO | Analyzing chunk frames = [8463872,8464384) | 20260506 03:07:31.929 | INFO | MemInfo: { | 20260506 03:07:31.929 | INFO | "gpu" : " 17 GB", | 20260506 03:07:31.929 | INFO | "huge" : "", | 20260506 03:07:31.929 | INFO | "huge_shared" : " 27 GB", | 20260506 03:07:31.929 | INFO | "malloc" : " 14 GB", | 20260506 03:07:31.929 | INFO | "pinned" : " 29 GB" | 20260506 03:07:31.929 | INFO | } | 20260506 03:07:31.929 | INFO | Perf report for the last 5.13877 seconds: | 20260506 03:07:31.929 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:07:31.929 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:31.929 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:31.929 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:31.929 | INFO | ANALYSIS : 61.1%, 3.0, 94.9ms, 0 | 20260506 03:07:31.929 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:31.929 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:31.929 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 03:07:31.929 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:31.929 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:31.929 | INFO | Uploaded 36GiB to GPU in 2.10116s (17.1334GiB/s) | 20260506 03:07:31.929 | INFO | Downloaded 34GiB to GPU in 2.01754s (16.8522GiB/s) | 20260506 03:07:37.068 | INFO | Analyzing chunk frames = [8464384,8464896) | 20260506 03:07:37.068 | INFO | MemInfo: { | 20260506 03:07:37.068 | INFO | "gpu" : " 17 GB", | 20260506 03:07:37.068 | INFO | "huge" : "", | 20260506 03:07:37.068 | INFO | "huge_shared" : " 27 GB", | 20260506 03:07:37.068 | INFO | "malloc" : " 14 GB", | 20260506 03:07:37.068 | INFO | "pinned" : " 29 GB" | 20260506 03:07:37.068 | INFO | } | 20260506 03:07:37.068 | INFO | Perf report for the last 5.13906 seconds: | 20260506 03:07:37.068 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:07:37.068 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:37.068 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:37.068 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:37.068 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260506 03:07:37.068 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:37.068 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:37.068 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:07:37.068 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:37.068 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:37.069 | INFO | Uploaded 36GiB to GPU in 2.09557s (17.1791GiB/s) | 20260506 03:07:37.069 | INFO | Downloaded 34GiB to GPU in 2.03523s (16.7057GiB/s) | 20260506 03:07:42.213 | INFO | Analyzing chunk frames = [8464896,8465408) | 20260506 03:07:42.213 | INFO | MemInfo: { | 20260506 03:07:42.213 | INFO | "gpu" : " 17 GB", | 20260506 03:07:42.213 | INFO | "huge" : "", | 20260506 03:07:42.213 | INFO | "huge_shared" : " 27 GB", | 20260506 03:07:42.213 | INFO | "malloc" : " 14 GB", | 20260506 03:07:42.213 | INFO | "pinned" : " 29 GB" | 20260506 03:07:42.213 | INFO | } | 20260506 03:07:42.213 | INFO | Perf report for the last 5.14453 seconds: | 20260506 03:07:42.213 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:07:42.213 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:42.213 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:42.213 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:42.213 | INFO | ANALYSIS : 60.6%, 3.0, 94.7ms, 0 | 20260506 03:07:42.213 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:42.213 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:42.213 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 03:07:42.213 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:42.213 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:42.213 | INFO | Uploaded 36GiB to GPU in 2.04431s (17.6098GiB/s) | 20260506 03:07:42.213 | INFO | Downloaded 34GiB to GPU in 2.026s (16.7819GiB/s) | 20260506 03:07:47.347 | INFO | Analyzing chunk frames = [8465408,8465920) | 20260506 03:07:47.347 | INFO | MemInfo: { | 20260506 03:07:47.347 | INFO | "gpu" : " 17 GB", | 20260506 03:07:47.347 | INFO | "huge" : "", | 20260506 03:07:47.347 | INFO | "huge_shared" : " 27 GB", | 20260506 03:07:47.347 | INFO | "malloc" : " 14 GB", | 20260506 03:07:47.347 | INFO | "pinned" : " 29 GB" | 20260506 03:07:47.347 | INFO | } | 20260506 03:07:47.347 | INFO | Perf report for the last 5.13433 seconds: | 20260506 03:07:47.347 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:07:47.347 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:47.347 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:47.347 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:47.347 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 03:07:47.347 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:47.347 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:47.347 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 03:07:47.347 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:47.347 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:47.347 | INFO | Uploaded 36GiB to GPU in 2.0703s (17.3888GiB/s) | 20260506 03:07:47.347 | INFO | Downloaded 34GiB to GPU in 2.05425s (16.551GiB/s) | 20260506 03:07:52.477 | INFO | Analyzing chunk frames = [8465920,8466432) | 20260506 03:07:52.477 | INFO | MemInfo: { | 20260506 03:07:52.477 | INFO | "gpu" : " 17 GB", | 20260506 03:07:52.477 | INFO | "huge" : "", | 20260506 03:07:52.477 | INFO | "huge_shared" : " 27 GB", | 20260506 03:07:52.477 | INFO | "malloc" : " 14 GB", | 20260506 03:07:52.477 | INFO | "pinned" : " 29 GB" | 20260506 03:07:52.477 | INFO | } | 20260506 03:07:52.477 | INFO | Perf report for the last 5.13017 seconds: | 20260506 03:07:52.477 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:07:52.477 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:52.477 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:52.477 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:52.477 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 03:07:52.477 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:52.477 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:52.478 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 03:07:52.478 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:52.478 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:52.478 | INFO | Uploaded 36GiB to GPU in 2.04475s (17.6061GiB/s) | 20260506 03:07:52.478 | INFO | Downloaded 34GiB to GPU in 2.01544s (16.8698GiB/s) | 20260506 03:07:52.604 | INFO | Starting a round of Baz IO | 20260506 03:07:55.734 | INFO | Populating RTMetrics report | 20260506 03:07:55.735 | INFO | RTMetrics report populated | 20260506 03:07:55.735 | INFO | Opening tmp RTMetrics json file | 20260506 03:07:55.736 | INFO | Serializing RTMetrics to json | 20260506 03:07:55.737 | INFO | Serialization done, swapping tmp file now... | 20260506 03:07:55.738 | INFO | RTMetrics done | 20260506 03:07:55.848 | INFO | Finished a round of Baz IO: Wrote 1.77 GiB in 3.244 seconds | 20260506 03:07:55.848 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.24% / 0.00% / 78.76% | 20260506 03:07:55.848 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0632763% / 0% / 5.96046e-05% | 20260506 03:07:55.848 | INFO | Baz IO overall: | 20260506 03:07:55.848 | INFO | BytesWritten : 4720844355805 | 20260506 03:07:55.848 | INFO | NumEvents : 2214603171447 | 20260506 03:07:57.624 | INFO | Analyzing chunk frames = [8466432,8466944) | 20260506 03:07:57.624 | INFO | MemInfo: { | 20260506 03:07:57.624 | INFO | "gpu" : " 17 GB", | 20260506 03:07:57.624 | INFO | "huge" : "", | 20260506 03:07:57.624 | INFO | "huge_shared" : " 27 GB", | 20260506 03:07:57.624 | INFO | "malloc" : " 14 GB", | 20260506 03:07:57.624 | INFO | "pinned" : " 29 GB" | 20260506 03:07:57.624 | INFO | } | 20260506 03:07:57.624 | INFO | Perf report for the last 5.14679 seconds: | 20260506 03:07:57.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:07:57.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:57.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:57.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:57.624 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 03:07:57.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:07:57.624 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260506 03:07:57.624 | INFO | PRE_HQ : 59.3%, 1.7, 53.0ms, 0 | 20260506 03:07:57.624 | INFO | BAZWRITER : 62.3%, 3.8, 123.4ms, 0 | 20260506 03:07:57.624 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 03:07:57.624 | INFO | Uploaded 36GiB to GPU in 2.18495s (16.4763GiB/s) | 20260506 03:07:57.625 | INFO | Downloaded 38GiB to GPU in 2.24662s (16.9143GiB/s) | 20260506 03:08:02.766 | INFO | Analyzing chunk frames = [8466944,8467456) | 20260506 03:08:02.766 | INFO | MemInfo: { | 20260506 03:08:02.766 | INFO | "gpu" : " 17 GB", | 20260506 03:08:02.766 | INFO | "huge" : "", | 20260506 03:08:02.766 | INFO | "huge_shared" : " 27 GB", | 20260506 03:08:02.766 | INFO | "malloc" : " 14 GB", | 20260506 03:08:02.766 | INFO | "pinned" : " 29 GB" | 20260506 03:08:02.766 | INFO | } | 20260506 03:08:02.766 | INFO | Perf report for the last 5.1416 seconds: | 20260506 03:08:02.766 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:08:02.766 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:02.766 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:02.766 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:02.766 | INFO | ANALYSIS : 60.8%, 3.0, 94.9ms, 0 | 20260506 03:08:02.766 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:02.766 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:02.766 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 03:08:02.766 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:02.766 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:02.766 | INFO | Uploaded 36GiB to GPU in 1.97233s (18.2525GiB/s) | 20260506 03:08:02.766 | INFO | Downloaded 34GiB to GPU in 1.98291s (17.1465GiB/s) | 20260506 03:08:07.907 | INFO | Analyzing chunk frames = [8467456,8467968) | 20260506 03:08:07.907 | INFO | MemInfo: { | 20260506 03:08:07.907 | INFO | "gpu" : " 17 GB", | 20260506 03:08:07.907 | INFO | "huge" : "", | 20260506 03:08:07.907 | INFO | "huge_shared" : " 27 GB", | 20260506 03:08:07.907 | INFO | "malloc" : " 14 GB", | 20260506 03:08:07.907 | INFO | "pinned" : " 29 GB" | 20260506 03:08:07.907 | INFO | } | 20260506 03:08:07.907 | INFO | Perf report for the last 5.14107 seconds: | 20260506 03:08:07.907 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:08:07.907 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:07.907 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:07.907 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:07.907 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 03:08:07.907 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:07.907 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:07.907 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 03:08:07.907 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:07.907 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:07.907 | INFO | Uploaded 36GiB to GPU in 2.11117s (17.0521GiB/s) | 20260506 03:08:07.907 | INFO | Downloaded 34GiB to GPU in 2.04355s (16.6377GiB/s) | 20260506 03:08:13.046 | INFO | Analyzing chunk frames = [8467968,8468480) | 20260506 03:08:13.046 | INFO | MemInfo: { | 20260506 03:08:13.046 | INFO | "gpu" : " 17 GB", | 20260506 03:08:13.046 | INFO | "huge" : "", | 20260506 03:08:13.046 | INFO | "huge_shared" : " 27 GB", | 20260506 03:08:13.046 | INFO | "malloc" : " 14 GB", | 20260506 03:08:13.046 | INFO | "pinned" : " 29 GB" | 20260506 03:08:13.046 | INFO | } | 20260506 03:08:13.046 | INFO | Perf report for the last 5.13958 seconds: | 20260506 03:08:13.046 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:08:13.046 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:13.046 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:13.047 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:13.047 | INFO | ANALYSIS : 61.1%, 3.0, 94.9ms, 0 | 20260506 03:08:13.047 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:13.047 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:13.047 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 03:08:13.047 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:13.047 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:13.047 | INFO | Uploaded 36GiB to GPU in 2.10968s (17.0642GiB/s) | 20260506 03:08:13.047 | INFO | Downloaded 34GiB to GPU in 2.0142s (16.8801GiB/s) | 20260506 03:08:18.177 | INFO | Analyzing chunk frames = [8468480,8468992) | 20260506 03:08:18.177 | INFO | MemInfo: { | 20260506 03:08:18.177 | INFO | "gpu" : " 17 GB", | 20260506 03:08:18.177 | INFO | "huge" : "", | 20260506 03:08:18.177 | INFO | "huge_shared" : " 27 GB", | 20260506 03:08:18.178 | INFO | "malloc" : " 14 GB", | 20260506 03:08:18.178 | INFO | "pinned" : " 29 GB" | 20260506 03:08:18.178 | INFO | } | 20260506 03:08:18.178 | INFO | Perf report for the last 5.13117 seconds: | 20260506 03:08:18.178 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:08:18.178 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:18.178 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:18.178 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:18.178 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260506 03:08:18.178 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:18.178 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:18.178 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 03:08:18.178 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:18.178 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:18.178 | INFO | Uploaded 36GiB to GPU in 2.1161s (17.0124GiB/s) | 20260506 03:08:18.178 | INFO | Downloaded 34GiB to GPU in 2.03578s (16.7012GiB/s) | 20260506 03:08:23.321 | INFO | Analyzing chunk frames = [8468992,8469504) | 20260506 03:08:23.321 | INFO | MemInfo: { | 20260506 03:08:23.321 | INFO | "gpu" : " 17 GB", | 20260506 03:08:23.321 | INFO | "huge" : "", | 20260506 03:08:23.321 | INFO | "huge_shared" : " 27 GB", | 20260506 03:08:23.321 | INFO | "malloc" : " 14 GB", | 20260506 03:08:23.321 | INFO | "pinned" : " 29 GB" | 20260506 03:08:23.321 | INFO | } | 20260506 03:08:23.321 | INFO | Perf report for the last 5.14326 seconds: | 20260506 03:08:23.321 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:08:23.321 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:23.321 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:23.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:23.321 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 03:08:23.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:23.321 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:23.321 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 03:08:23.321 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:23.321 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:23.321 | INFO | Uploaded 36GiB to GPU in 2.06889s (17.4006GiB/s) | 20260506 03:08:23.321 | INFO | Downloaded 34GiB to GPU in 2.03481s (16.7092GiB/s) | 20260506 03:08:28.457 | INFO | Analyzing chunk frames = [8469504,8470016) | 20260506 03:08:28.457 | INFO | MemInfo: { | 20260506 03:08:28.457 | INFO | "gpu" : " 17 GB", | 20260506 03:08:28.457 | INFO | "huge" : "", | 20260506 03:08:28.457 | INFO | "huge_shared" : " 27 GB", | 20260506 03:08:28.457 | INFO | "malloc" : " 14 GB", | 20260506 03:08:28.457 | INFO | "pinned" : " 29 GB" | 20260506 03:08:28.457 | INFO | } | 20260506 03:08:28.458 | INFO | Perf report for the last 5.13662 seconds: | 20260506 03:08:28.458 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:08:28.458 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:28.458 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:28.458 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:28.458 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260506 03:08:28.458 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:28.458 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:28.458 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 03:08:28.458 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:28.458 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:28.458 | INFO | Uploaded 36GiB to GPU in 2.08424s (17.2724GiB/s) | 20260506 03:08:28.458 | INFO | Downloaded 34GiB to GPU in 2.06758s (16.4443GiB/s) | 20260506 03:08:33.602 | INFO | Analyzing chunk frames = [8470016,8470528) | 20260506 03:08:33.602 | INFO | MemInfo: { | 20260506 03:08:33.602 | INFO | "gpu" : " 17 GB", | 20260506 03:08:33.602 | INFO | "huge" : "", | 20260506 03:08:33.602 | INFO | "huge_shared" : " 27 GB", | 20260506 03:08:33.602 | INFO | "malloc" : " 14 GB", | 20260506 03:08:33.602 | INFO | "pinned" : " 29 GB" | 20260506 03:08:33.602 | INFO | } | 20260506 03:08:33.602 | INFO | Perf report for the last 5.14468 seconds: | 20260506 03:08:33.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:08:33.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:33.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:33.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:33.602 | INFO | ANALYSIS : 61.2%, 3.0, 95.6ms, 0 | 20260506 03:08:33.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:33.602 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:33.602 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 03:08:33.602 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:33.602 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:33.603 | INFO | Uploaded 36GiB to GPU in 2.02754s (17.7555GiB/s) | 20260506 03:08:33.603 | INFO | Downloaded 34GiB to GPU in 2.01064s (16.9101GiB/s) | 20260506 03:08:33.603 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 03:08:33.603 | INFO | Total mem usage: 15.4 GiB | 20260506 03:08:33.603 | INFO | All Time High Water Marks sum: 16.3 GiB (Savings: 5.2%) | 20260506 03:08:33.603 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -4.39%) | 20260506 03:08:33.603 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 03:08:33.603 | INFO | High Water Breakdown: | 20260506 03:08:33.603 | INFO | 61.7% : 9.12 GiB ( 94.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 03:08:33.603 | INFO | 38.2% : 5.65 GiB ( 85% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 03:08:33.603 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 03:08:33.603 | INFO | ------------------------------------------ | 20260506 03:08:33.603 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:08:33.603 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 03:08:33.603 | INFO | Total mem usage: 26.5 GiB | 20260506 03:08:33.603 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 03:08:33.603 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 03:08:33.603 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 03:08:33.603 | INFO | High Water Breakdown: | 20260506 03:08:33.603 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 03:08:33.603 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 03:08:33.603 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 03:08:33.603 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 03:08:33.603 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 03:08:33.603 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 03:08:33.603 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 03:08:33.603 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 03:08:33.603 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 03:08:33.603 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 03:08:33.603 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 03:08:33.603 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 03:08:33.603 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 03:08:33.603 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 03:08:33.603 | INFO | ------------------------------------------ | 20260506 03:08:33.603 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:08:33.603 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 03:08:33.603 | INFO | Total mem usage: 25.6 GiB | 20260506 03:08:33.603 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 03:08:33.603 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 03:08:33.603 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 03:08:33.603 | INFO | High Water Breakdown: | 20260506 03:08:33.603 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 03:08:33.603 | INFO | ------------------------------------------ | 20260506 03:08:33.603 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:08:33.603 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 03:08:33.603 | INFO | Total mem usage: 15.7 GiB | 20260506 03:08:33.603 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 03:08:33.603 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 03:08:33.603 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 03:08:33.603 | INFO | High Water Breakdown: | 20260506 03:08:33.603 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 03:08:33.603 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 03:08:33.603 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 03:08:33.603 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 03:08:33.603 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 03:08:33.603 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 03:08:33.603 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 03:08:33.603 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 03:08:33.603 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 03:08:33.603 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 03:08:33.603 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 03:08:33.603 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 03:08:33.603 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 03:08:33.603 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 03:08:33.603 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 03:08:33.603 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 03:08:33.603 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 03:08:33.603 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 03:08:33.603 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 03:08:33.603 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 03:08:33.603 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 03:08:33.603 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 03:08:33.603 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 03:08:33.603 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 03:08:33.603 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 03:08:33.603 | INFO | ------------------------------------------ | 20260506 03:08:33.603 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:08:33.604 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 03:08:33.604 | INFO | PerfReport: Gathered 1536 samples. | 20260506 03:08:33.604 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 03:08:33.604 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.38e+01, 1.34e+01 / 7.79e-01 | 20260506 03:08:33.604 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.05e-01, 1.05e-02 / 2.76e-03 | 20260506 03:08:33.604 | INFO | PerfReport: 11.86%, Baseline -- 3.71e+00 / 4.53e+00, 5.84e-01 / 2.62e-01 | 20260506 03:08:33.604 | INFO | PerfReport: 3.31%, Histogram -- 1.04e+00 / 1.53e+00, 1.96e-01 / 8.72e-02 | 20260506 03:08:33.604 | INFO | PerfReport: 16.00%, DME -- 4.01e+01 / 4.41e+01, 3.85e+00 / 8.09e-01 | 20260506 03:08:33.604 | INFO | PerfReport: 57.57%, FrameLabeling -- 1.80e+01 / 1.93e+01, 1.60e+00 / 1.18e+00 | 20260506 03:08:33.604 | INFO | PerfReport: 6.96%, PulseAccumulating -- 2.18e+00 / 2.69e+00, 4.13e-01 / 2.41e-01 | 20260506 03:08:33.604 | INFO | PerfReport: 2.97%, Metrics -- 9.31e-01 / 1.21e+00, 2.50e-01 / 1.35e-01 | 20260506 03:08:33.604 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.22e-01 / 3.69e-01, 5.48e-02 / 3.08e-02 | 20260506 03:08:33.604 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 03:08:33.604 | INFO | PerfReport: Gathered 1536 samples. | 20260506 03:08:33.604 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 03:08:33.604 | INFO | PerfReport: Full Execution -- 9.75e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260506 03:08:33.604 | INFO | PerfReport: 22.18%, Upload -- 2.16e+01 / 0.00e+00, 7.17e+00 / 0.00e+00 | 20260506 03:08:33.604 | INFO | PerfReport: 21.93%, Download -- 2.14e+01 / 0.00e+00, 7.58e+00 / 0.00e+00 | 20260506 03:08:33.604 | INFO | PerfReport: 32.13%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 03:08:33.604 | INFO | PerfReport: 1.97%, LockUpload -- 1.92e+00 / 0.00e+00, 7.73e+00 / 0.00e+00 | 20260506 03:08:33.604 | INFO | PerfReport: 1.26%, LockDownload -- 1.23e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260506 03:08:33.604 | INFO | PerfReport: 20.51%, LockCompute -- 2.00e+01 / 0.00e+00, 1.74e+01 / 0.00e+00 | 20260506 03:08:33.604 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.38e-02 / 0.00e+00 | 20260506 03:08:33.728 | INFO | Starting a round of Baz IO | 20260506 03:08:36.846 | INFO | Populating RTMetrics report | 20260506 03:08:36.847 | INFO | RTMetrics report populated | 20260506 03:08:36.847 | INFO | Opening tmp RTMetrics json file | 20260506 03:08:36.850 | INFO | Serializing RTMetrics to json | 20260506 03:08:36.855 | INFO | Serialization done, swapping tmp file now... | 20260506 03:08:36.862 | INFO | RTMetrics done | 20260506 03:08:36.938 | INFO | Finished a round of Baz IO: Wrote 1.77 GiB in 3.21 seconds | 20260506 03:08:36.938 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.24% / 0.00% / 78.76% | 20260506 03:08:36.938 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0634829% / 0% / 7.15256e-05% | 20260506 03:08:36.938 | INFO | Baz IO overall: | 20260506 03:08:36.938 | INFO | BytesWritten : 4722744344502 | 20260506 03:08:36.938 | INFO | NumEvents : 2215465017875 | 20260506 03:08:38.738 | INFO | Analyzing chunk frames = [8470528,8471040) | 20260506 03:08:38.738 | INFO | MemInfo: { | 20260506 03:08:38.738 | INFO | "gpu" : " 17 GB", | 20260506 03:08:38.738 | INFO | "huge" : "", | 20260506 03:08:38.738 | INFO | "huge_shared" : " 27 GB", | 20260506 03:08:38.738 | INFO | "malloc" : " 14 GB", | 20260506 03:08:38.738 | INFO | "pinned" : " 29 GB" | 20260506 03:08:38.738 | INFO | } | 20260506 03:08:38.738 | INFO | Perf report for the last 5.13536 seconds: | 20260506 03:08:38.738 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:08:38.738 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:38.738 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:38.738 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:38.738 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260506 03:08:38.738 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:38.738 | INFO | RT_METRICS : 43.0%, 1.0, 22.5ms, 0 | 20260506 03:08:38.738 | INFO | PRE_HQ : 59.3%, 1.8, 54.0ms, 0 | 20260506 03:08:38.738 | INFO | BAZWRITER : 61.5%, 3.6, 112.7ms, 0 | 20260506 03:08:38.738 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260506 03:08:38.738 | INFO | Uploaded 36GiB to GPU in 2.1069s (17.0867GiB/s) | 20260506 03:08:38.738 | INFO | Downloaded 38GiB to GPU in 2.22363s (17.0892GiB/s) | 20260506 03:08:43.879 | INFO | Analyzing chunk frames = [8471040,8471552) | 20260506 03:08:43.879 | INFO | MemInfo: { | 20260506 03:08:43.879 | INFO | "gpu" : " 17 GB", | 20260506 03:08:43.879 | INFO | "huge" : "", | 20260506 03:08:43.879 | INFO | "huge_shared" : " 27 GB", | 20260506 03:08:43.879 | INFO | "malloc" : " 14 GB", | 20260506 03:08:43.879 | INFO | "pinned" : " 29 GB" | 20260506 03:08:43.879 | INFO | } | 20260506 03:08:43.879 | INFO | Perf report for the last 5.14137 seconds: | 20260506 03:08:43.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:08:43.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:43.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:43.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:43.879 | INFO | ANALYSIS : 60.9%, 3.0, 95.0ms, 0 | 20260506 03:08:43.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:43.879 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:43.879 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 03:08:43.879 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:43.879 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:43.879 | INFO | Uploaded 36GiB to GPU in 1.97807s (18.1995GiB/s) | 20260506 03:08:43.879 | INFO | Downloaded 34GiB to GPU in 1.97909s (17.1796GiB/s) | 20260506 03:08:49.023 | INFO | Analyzing chunk frames = [8471552,8472064) | 20260506 03:08:49.023 | INFO | MemInfo: { | 20260506 03:08:49.023 | INFO | "gpu" : " 17 GB", | 20260506 03:08:49.023 | INFO | "huge" : "", | 20260506 03:08:49.023 | INFO | "huge_shared" : " 27 GB", | 20260506 03:08:49.023 | INFO | "malloc" : " 14 GB", | 20260506 03:08:49.023 | INFO | "pinned" : " 29 GB" | 20260506 03:08:49.023 | INFO | } | 20260506 03:08:49.023 | INFO | Perf report for the last 5.14444 seconds: | 20260506 03:08:49.023 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:08:49.023 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:49.023 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:49.023 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:49.024 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260506 03:08:49.024 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:49.024 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:49.024 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:08:49.024 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:49.024 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:49.024 | INFO | Uploaded 36GiB to GPU in 2.1125s (17.0414GiB/s) | 20260506 03:08:49.024 | INFO | Downloaded 34GiB to GPU in 2.04792s (16.6022GiB/s) | 20260506 03:08:54.161 | INFO | Analyzing chunk frames = [8472064,8472576) | 20260506 03:08:54.161 | INFO | MemInfo: { | 20260506 03:08:54.161 | INFO | "gpu" : " 17 GB", | 20260506 03:08:54.161 | INFO | "huge" : "", | 20260506 03:08:54.161 | INFO | "huge_shared" : " 27 GB", | 20260506 03:08:54.162 | INFO | "malloc" : " 14 GB", | 20260506 03:08:54.162 | INFO | "pinned" : " 29 GB" | 20260506 03:08:54.162 | INFO | } | 20260506 03:08:54.162 | INFO | Perf report for the last 5.13821 seconds: | 20260506 03:08:54.162 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:08:54.162 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:54.162 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:54.162 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:54.162 | INFO | ANALYSIS : 61.0%, 3.0, 94.6ms, 0 | 20260506 03:08:54.162 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:54.162 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:54.162 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 03:08:54.162 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:54.162 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:54.162 | INFO | Uploaded 36GiB to GPU in 2.08889s (17.2341GiB/s) | 20260506 03:08:54.162 | INFO | Downloaded 34GiB to GPU in 2.01303s (16.89GiB/s) | 20260506 03:08:59.296 | INFO | Analyzing chunk frames = [8472576,8473088) | 20260506 03:08:59.296 | INFO | MemInfo: { | 20260506 03:08:59.296 | INFO | "gpu" : " 17 GB", | 20260506 03:08:59.296 | INFO | "huge" : "", | 20260506 03:08:59.296 | INFO | "huge_shared" : " 27 GB", | 20260506 03:08:59.296 | INFO | "malloc" : " 14 GB", | 20260506 03:08:59.296 | INFO | "pinned" : " 29 GB" | 20260506 03:08:59.296 | INFO | } | 20260506 03:08:59.296 | INFO | Perf report for the last 5.13418 seconds: | 20260506 03:08:59.296 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:08:59.296 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:59.296 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:59.296 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:59.296 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260506 03:08:59.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:59.296 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:59.296 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 03:08:59.296 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:59.296 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:08:59.296 | INFO | Uploaded 36GiB to GPU in 2.1235s (16.9532GiB/s) | 20260506 03:08:59.296 | INFO | Downloaded 34GiB to GPU in 2.04374s (16.6361GiB/s) | 20260506 03:09:04.430 | INFO | Analyzing chunk frames = [8473088,8473600) | 20260506 03:09:04.430 | INFO | MemInfo: { | 20260506 03:09:04.430 | INFO | "gpu" : " 17 GB", | 20260506 03:09:04.430 | INFO | "huge" : "", | 20260506 03:09:04.430 | INFO | "huge_shared" : " 27 GB", | 20260506 03:09:04.430 | INFO | "malloc" : " 14 GB", | 20260506 03:09:04.430 | INFO | "pinned" : " 29 GB" | 20260506 03:09:04.430 | INFO | } | 20260506 03:09:04.430 | INFO | Perf report for the last 5.13401 seconds: | 20260506 03:09:04.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:09:04.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:04.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:04.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:04.430 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 03:09:04.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:04.430 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:04.430 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 03:09:04.430 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:04.430 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:04.430 | INFO | Uploaded 36GiB to GPU in 2.09775s (17.1612GiB/s) | 20260506 03:09:04.430 | INFO | Downloaded 34GiB to GPU in 2.03889s (16.6758GiB/s) | 20260506 03:09:09.575 | INFO | Analyzing chunk frames = [8473600,8474112) | 20260506 03:09:09.575 | INFO | MemInfo: { | 20260506 03:09:09.575 | INFO | "gpu" : " 17 GB", | 20260506 03:09:09.575 | INFO | "huge" : "", | 20260506 03:09:09.575 | INFO | "huge_shared" : " 27 GB", | 20260506 03:09:09.575 | INFO | "malloc" : " 14 GB", | 20260506 03:09:09.575 | INFO | "pinned" : " 29 GB" | 20260506 03:09:09.575 | INFO | } | 20260506 03:09:09.575 | INFO | Perf report for the last 5.1456 seconds: | 20260506 03:09:09.575 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:09:09.576 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:09.576 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:09.576 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:09.576 | INFO | ANALYSIS : 62.0%, 3.0, 97.0ms, 0 | 20260506 03:09:09.576 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:09.576 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:09.576 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 03:09:09.576 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:09.576 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:09.576 | INFO | Uploaded 36GiB to GPU in 2.07479s (17.3511GiB/s) | 20260506 03:09:09.576 | INFO | Downloaded 34GiB to GPU in 2.07093s (16.4177GiB/s) | 20260506 03:09:14.714 | INFO | Analyzing chunk frames = [8474112,8474624) | 20260506 03:09:14.714 | INFO | MemInfo: { | 20260506 03:09:14.714 | INFO | "gpu" : " 17 GB", | 20260506 03:09:14.714 | INFO | "huge" : "", | 20260506 03:09:14.714 | INFO | "huge_shared" : " 27 GB", | 20260506 03:09:14.714 | INFO | "malloc" : " 14 GB", | 20260506 03:09:14.714 | INFO | "pinned" : " 29 GB" | 20260506 03:09:14.714 | INFO | } | 20260506 03:09:14.714 | INFO | Perf report for the last 5.13896 seconds: | 20260506 03:09:14.714 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:09:14.714 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:14.714 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:14.715 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:14.715 | INFO | ANALYSIS : 61.2%, 3.0, 95.6ms, 0 | 20260506 03:09:14.715 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:14.715 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:14.715 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 03:09:14.715 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:14.715 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:14.715 | INFO | Uploaded 36GiB to GPU in 2.04767s (17.5809GiB/s) | 20260506 03:09:14.715 | INFO | Downloaded 34GiB to GPU in 2.02086s (16.8245GiB/s) | 20260506 03:09:14.838 | INFO | Starting a round of Baz IO | 20260506 03:09:17.962 | INFO | Populating RTMetrics report | 20260506 03:09:17.962 | INFO | RTMetrics report populated | 20260506 03:09:17.962 | INFO | Opening tmp RTMetrics json file | 20260506 03:09:17.965 | INFO | Serializing RTMetrics to json | 20260506 03:09:17.966 | INFO | Serialization done, swapping tmp file now... | 20260506 03:09:17.967 | INFO | RTMetrics done | 20260506 03:09:18.046 | INFO | Finished a round of Baz IO: Wrote 1.77 GiB in 3.208 seconds | 20260506 03:09:18.046 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.24% / 0.00% / 78.76% | 20260506 03:09:18.046 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0629822% / 0% / 9.53674e-05% | 20260506 03:09:18.046 | INFO | Baz IO overall: | 20260506 03:09:18.046 | INFO | BytesWritten : 4724644332173 | 20260506 03:09:18.046 | INFO | NumEvents : 2216326782744 | 20260506 03:09:19.858 | INFO | Analyzing chunk frames = [8474624,8475136) | 20260506 03:09:19.858 | INFO | MemInfo: { | 20260506 03:09:19.858 | INFO | "gpu" : " 17 GB", | 20260506 03:09:19.858 | INFO | "huge" : "", | 20260506 03:09:19.858 | INFO | "huge_shared" : " 27 GB", | 20260506 03:09:19.858 | INFO | "malloc" : " 14 GB", | 20260506 03:09:19.858 | INFO | "pinned" : " 29 GB" | 20260506 03:09:19.858 | INFO | } | 20260506 03:09:19.858 | INFO | Perf report for the last 5.14396 seconds: | 20260506 03:09:19.858 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:09:19.858 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:19.858 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:19.858 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:19.859 | INFO | ANALYSIS : 62.1%, 3.0, 97.1ms, 0 | 20260506 03:09:19.859 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:19.859 | INFO | RT_METRICS : 42.6%, 1.0, 22.4ms, 0 | 20260506 03:09:19.859 | INFO | PRE_HQ : 59.0%, 1.7, 53.2ms, 0 | 20260506 03:09:19.859 | INFO | BAZWRITER : 61.5%, 3.6, 114.3ms, 0 | 20260506 03:09:19.859 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 03:09:19.859 | INFO | Uploaded 36GiB to GPU in 2.14045s (16.8189GiB/s) | 20260506 03:09:19.859 | INFO | Downloaded 38GiB to GPU in 2.23247s (17.0215GiB/s) | 20260506 03:09:24.998 | INFO | Analyzing chunk frames = [8475136,8475648) | 20260506 03:09:24.998 | INFO | MemInfo: { | 20260506 03:09:24.998 | INFO | "gpu" : " 17 GB", | 20260506 03:09:24.998 | INFO | "huge" : "", | 20260506 03:09:24.998 | INFO | "huge_shared" : " 27 GB", | 20260506 03:09:24.998 | INFO | "malloc" : " 14 GB", | 20260506 03:09:24.998 | INFO | "pinned" : " 29 GB" | 20260506 03:09:24.998 | INFO | } | 20260506 03:09:24.998 | INFO | Perf report for the last 5.13954 seconds: | 20260506 03:09:24.998 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:09:24.998 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:24.998 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:24.998 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:24.998 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260506 03:09:24.998 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:24.998 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:24.998 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 03:09:24.998 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:24.998 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:24.998 | INFO | Uploaded 36GiB to GPU in 1.96114s (18.3566GiB/s) | 20260506 03:09:24.998 | INFO | Downloaded 34GiB to GPU in 1.99227s (17.0659GiB/s) | 20260506 03:09:30.130 | INFO | Analyzing chunk frames = [8475648,8476160) | 20260506 03:09:30.130 | INFO | MemInfo: { | 20260506 03:09:30.130 | INFO | "gpu" : " 17 GB", | 20260506 03:09:30.130 | INFO | "huge" : "", | 20260506 03:09:30.130 | INFO | "huge_shared" : " 27 GB", | 20260506 03:09:30.130 | INFO | "malloc" : " 14 GB", | 20260506 03:09:30.130 | INFO | "pinned" : " 29 GB" | 20260506 03:09:30.130 | INFO | } | 20260506 03:09:30.130 | INFO | Perf report for the last 5.13232 seconds: | 20260506 03:09:30.130 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:09:30.130 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:30.130 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:30.130 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:30.130 | INFO | ANALYSIS : 61.2%, 3.0, 95.1ms, 0 | 20260506 03:09:30.130 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:30.130 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:30.130 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:09:30.130 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:30.130 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:30.131 | INFO | Uploaded 36GiB to GPU in 2.09842s (17.1558GiB/s) | 20260506 03:09:30.131 | INFO | Downloaded 34GiB to GPU in 2.03992s (16.6673GiB/s) | 20260506 03:09:35.270 | INFO | Analyzing chunk frames = [8476160,8476672) | 20260506 03:09:35.270 | INFO | MemInfo: { | 20260506 03:09:35.270 | INFO | "gpu" : " 17 GB", | 20260506 03:09:35.270 | INFO | "huge" : "", | 20260506 03:09:35.270 | INFO | "huge_shared" : " 27 GB", | 20260506 03:09:35.270 | INFO | "malloc" : " 14 GB", | 20260506 03:09:35.271 | INFO | "pinned" : " 29 GB" | 20260506 03:09:35.271 | INFO | } | 20260506 03:09:35.271 | INFO | Perf report for the last 5.14023 seconds: | 20260506 03:09:35.271 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:09:35.271 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:35.271 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:35.271 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:35.271 | INFO | ANALYSIS : 61.1%, 3.0, 94.9ms, 0 | 20260506 03:09:35.271 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:35.271 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:35.271 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 03:09:35.271 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:35.271 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:35.271 | INFO | Uploaded 36GiB to GPU in 2.09472s (17.186GiB/s) | 20260506 03:09:35.271 | INFO | Downloaded 34GiB to GPU in 2.02188s (16.816GiB/s) | 20260506 03:09:40.409 | INFO | Analyzing chunk frames = [8476672,8477184) | 20260506 03:09:40.409 | INFO | MemInfo: { | 20260506 03:09:40.409 | INFO | "gpu" : " 17 GB", | 20260506 03:09:40.409 | INFO | "huge" : "", | 20260506 03:09:40.409 | INFO | "huge_shared" : " 27 GB", | 20260506 03:09:40.409 | INFO | "malloc" : " 14 GB", | 20260506 03:09:40.409 | INFO | "pinned" : " 29 GB" | 20260506 03:09:40.409 | INFO | } | 20260506 03:09:40.409 | INFO | Perf report for the last 5.13839 seconds: | 20260506 03:09:40.409 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:09:40.409 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:40.409 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:40.409 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:40.409 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260506 03:09:40.409 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:40.409 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:40.409 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 03:09:40.409 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:40.409 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:40.409 | INFO | Uploaded 36GiB to GPU in 2.10655s (17.0895GiB/s) | 20260506 03:09:40.409 | INFO | Downloaded 34GiB to GPU in 2.03096s (16.7409GiB/s) | 20260506 03:09:45.551 | INFO | Analyzing chunk frames = [8477184,8477696) | 20260506 03:09:45.551 | INFO | MemInfo: { | 20260506 03:09:45.551 | INFO | "gpu" : " 17 GB", | 20260506 03:09:45.551 | INFO | "huge" : "", | 20260506 03:09:45.551 | INFO | "huge_shared" : " 27 GB", | 20260506 03:09:45.551 | INFO | "malloc" : " 14 GB", | 20260506 03:09:45.551 | INFO | "pinned" : " 29 GB" | 20260506 03:09:45.551 | INFO | } | 20260506 03:09:45.551 | INFO | Perf report for the last 5.14229 seconds: | 20260506 03:09:45.551 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:09:45.551 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:45.551 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:45.551 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:45.551 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 03:09:45.551 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:45.551 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:45.551 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:09:45.551 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:45.551 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:45.552 | INFO | Uploaded 36GiB to GPU in 2.11684s (17.0065GiB/s) | 20260506 03:09:45.552 | INFO | Downloaded 34GiB to GPU in 2.04297s (16.6424GiB/s) | 20260506 03:09:50.690 | INFO | Analyzing chunk frames = [8477696,8478208) | 20260506 03:09:50.690 | INFO | MemInfo: { | 20260506 03:09:50.690 | INFO | "gpu" : " 17 GB", | 20260506 03:09:50.690 | INFO | "huge" : "", | 20260506 03:09:50.690 | INFO | "huge_shared" : " 27 GB", | 20260506 03:09:50.690 | INFO | "malloc" : " 14 GB", | 20260506 03:09:50.690 | INFO | "pinned" : " 29 GB" | 20260506 03:09:50.690 | INFO | } | 20260506 03:09:50.690 | INFO | Perf report for the last 5.1388 seconds: | 20260506 03:09:50.690 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:09:50.690 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:50.690 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:50.690 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:50.690 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260506 03:09:50.690 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:50.690 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:50.690 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 03:09:50.690 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:50.690 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:50.690 | INFO | Uploaded 36GiB to GPU in 2.06545s (17.4297GiB/s) | 20260506 03:09:50.690 | INFO | Downloaded 34GiB to GPU in 2.0723s (16.4069GiB/s) | 20260506 03:09:55.835 | INFO | Analyzing chunk frames = [8478208,8478720) | 20260506 03:09:55.835 | INFO | MemInfo: { | 20260506 03:09:55.835 | INFO | "gpu" : " 17 GB", | 20260506 03:09:55.835 | INFO | "huge" : "", | 20260506 03:09:55.835 | INFO | "huge_shared" : " 27 GB", | 20260506 03:09:55.835 | INFO | "malloc" : " 14 GB", | 20260506 03:09:55.835 | INFO | "pinned" : " 29 GB" | 20260506 03:09:55.835 | INFO | } | 20260506 03:09:55.835 | INFO | Perf report for the last 5.14529 seconds: | 20260506 03:09:55.835 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:09:55.835 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:55.835 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:55.835 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:55.835 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260506 03:09:55.835 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:55.835 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:55.835 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 03:09:55.836 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:55.836 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:09:55.836 | INFO | Uploaded 36GiB to GPU in 2.05087s (17.5536GiB/s) | 20260506 03:09:55.836 | INFO | Downloaded 34GiB to GPU in 2.01711s (16.8558GiB/s) | 20260506 03:09:55.836 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 03:09:55.836 | INFO | Total mem usage: 15.4 GiB | 20260506 03:09:55.836 | INFO | All Time High Water Marks sum: 16.3 GiB (Savings: 5.2%) | 20260506 03:09:55.836 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -9.18%) | 20260506 03:09:55.836 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 03:09:55.836 | INFO | High Water Breakdown: | 20260506 03:09:55.836 | INFO | 61% : 8.62 GiB ( 89.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 03:09:55.836 | INFO | 38.9% : 5.5 GiB ( 82.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 03:09:55.836 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 03:09:55.836 | INFO | ------------------------------------------ | 20260506 03:09:55.836 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:09:55.836 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 03:09:55.836 | INFO | Total mem usage: 26.5 GiB | 20260506 03:09:55.836 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 03:09:55.836 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 03:09:55.836 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 03:09:55.836 | INFO | High Water Breakdown: | 20260506 03:09:55.836 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 03:09:55.836 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 03:09:55.836 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 03:09:55.836 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 03:09:55.836 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 03:09:55.836 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 03:09:55.836 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 03:09:55.836 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 03:09:55.836 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 03:09:55.836 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 03:09:55.836 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 03:09:55.836 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 03:09:55.836 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 03:09:55.836 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 03:09:55.836 | INFO | ------------------------------------------ | 20260506 03:09:55.836 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:09:55.836 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 03:09:55.836 | INFO | Total mem usage: 25.6 GiB | 20260506 03:09:55.836 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 03:09:55.836 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 03:09:55.836 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 03:09:55.836 | INFO | High Water Breakdown: | 20260506 03:09:55.836 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 03:09:55.836 | INFO | ------------------------------------------ | 20260506 03:09:55.836 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:09:55.836 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 03:09:55.836 | INFO | Total mem usage: 15.7 GiB | 20260506 03:09:55.836 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 03:09:55.836 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 03:09:55.836 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 03:09:55.836 | INFO | High Water Breakdown: | 20260506 03:09:55.836 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 03:09:55.836 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 03:09:55.836 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 03:09:55.836 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 03:09:55.836 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 03:09:55.836 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 03:09:55.836 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 03:09:55.836 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 03:09:55.836 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 03:09:55.836 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 03:09:55.836 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 03:09:55.837 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 03:09:55.837 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 03:09:55.837 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 03:09:55.837 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 03:09:55.837 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 03:09:55.837 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 03:09:55.837 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 03:09:55.837 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 03:09:55.837 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 03:09:55.837 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 03:09:55.837 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 03:09:55.837 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 03:09:55.837 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 03:09:55.837 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 03:09:55.837 | INFO | ------------------------------------------ | 20260506 03:09:55.837 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:09:55.837 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 03:09:55.837 | INFO | PerfReport: Gathered 1536 samples. | 20260506 03:09:55.837 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 03:09:55.837 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.37e+01, 1.31e+01 / 7.52e-01 | 20260506 03:09:55.837 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.05e-01, 1.04e-02 / 2.74e-03 | 20260506 03:09:55.837 | INFO | PerfReport: 11.86%, Baseline -- 3.71e+00 / 4.47e+00, 5.80e-01 / 2.68e-01 | 20260506 03:09:55.837 | INFO | PerfReport: 3.30%, Histogram -- 1.03e+00 / 1.52e+00, 1.95e-01 / 9.37e-02 | 20260506 03:09:55.837 | INFO | PerfReport: 16.00%, DME -- 4.00e+01 / 4.39e+01, 3.89e+00 / 9.23e-01 | 20260506 03:09:55.837 | INFO | PerfReport: 57.53%, FrameLabeling -- 1.80e+01 / 1.93e+01, 1.52e+00 / 9.43e-01 | 20260506 03:09:55.837 | INFO | PerfReport: 6.99%, PulseAccumulating -- 2.19e+00 / 2.73e+00, 4.14e-01 / 1.93e-01 | 20260506 03:09:55.837 | INFO | PerfReport: 2.99%, Metrics -- 9.33e-01 / 1.21e+00, 2.51e-01 / 1.17e-01 | 20260506 03:09:55.837 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.22e-01 / 3.71e-01, 5.47e-02 / 2.47e-02 | 20260506 03:09:55.837 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 03:09:55.837 | INFO | PerfReport: Gathered 1536 samples. | 20260506 03:09:55.837 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 03:09:55.837 | INFO | PerfReport: Full Execution -- 9.75e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260506 03:09:55.837 | INFO | PerfReport: 22.21%, Upload -- 2.17e+01 / 0.00e+00, 7.16e+00 / 0.00e+00 | 20260506 03:09:55.837 | INFO | PerfReport: 21.96%, Download -- 2.14e+01 / 0.00e+00, 7.57e+00 / 0.00e+00 | 20260506 03:09:55.837 | INFO | PerfReport: 32.06%, Compute -- 3.13e+01 / 0.00e+00, 1.31e+01 / 0.00e+00 | 20260506 03:09:55.837 | INFO | PerfReport: 2.00%, LockUpload -- 1.95e+00 / 0.00e+00, 7.86e+00 / 0.00e+00 | 20260506 03:09:55.837 | INFO | PerfReport: 1.30%, LockDownload -- 1.27e+00 / 0.00e+00, 2.95e+00 / 0.00e+00 | 20260506 03:09:55.837 | INFO | PerfReport: 20.45%, LockCompute -- 1.99e+01 / 0.00e+00, 1.77e+01 / 0.00e+00 | 20260506 03:09:55.837 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.69e-02 / 0.00e+00 | 20260506 03:09:55.965 | INFO | Starting a round of Baz IO | 20260506 03:09:59.076 | INFO | Populating RTMetrics report | 20260506 03:09:59.076 | INFO | RTMetrics report populated | 20260506 03:09:59.076 | INFO | Opening tmp RTMetrics json file | 20260506 03:09:59.077 | INFO | Serializing RTMetrics to json | 20260506 03:09:59.079 | INFO | Serialization done, swapping tmp file now... | 20260506 03:09:59.082 | INFO | RTMetrics done | 20260506 03:09:59.165 | INFO | Finished a round of Baz IO: Wrote 1.77 GiB in 3.2 seconds | 20260506 03:09:59.165 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.24% / 0.00% / 78.76% | 20260506 03:09:59.165 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0632922% / 0% / 9.53674e-05% | 20260506 03:09:59.165 | INFO | Baz IO overall: | 20260506 03:09:59.165 | INFO | BytesWritten : 4726544697905 | 20260506 03:09:59.165 | INFO | NumEvents : 2217189144450 | 20260506 03:10:00.970 | INFO | Analyzing chunk frames = [8478720,8479232) | 20260506 03:10:00.970 | INFO | MemInfo: { | 20260506 03:10:00.970 | INFO | "gpu" : " 17 GB", | 20260506 03:10:00.970 | INFO | "huge" : "", | 20260506 03:10:00.970 | INFO | "huge_shared" : " 27 GB", | 20260506 03:10:00.970 | INFO | "malloc" : " 14 GB", | 20260506 03:10:00.970 | INFO | "pinned" : " 29 GB" | 20260506 03:10:00.970 | INFO | } | 20260506 03:10:00.970 | INFO | Perf report for the last 5.13487 seconds: | 20260506 03:10:00.970 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:10:00.970 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:00.970 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:00.970 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:00.970 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 03:10:00.970 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:00.970 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260506 03:10:00.970 | INFO | PRE_HQ : 59.4%, 1.7, 53.8ms, 0 | 20260506 03:10:00.970 | INFO | BAZWRITER : 61.5%, 3.6, 112.9ms, 0 | 20260506 03:10:00.971 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 03:10:00.971 | INFO | Uploaded 36GiB to GPU in 2.08224s (17.2891GiB/s) | 20260506 03:10:00.971 | INFO | Downloaded 38GiB to GPU in 2.22481s (17.0801GiB/s) | 20260506 03:10:06.108 | INFO | Analyzing chunk frames = [8479232,8479744) | 20260506 03:10:06.108 | INFO | MemInfo: { | 20260506 03:10:06.108 | INFO | "gpu" : " 17 GB", | 20260506 03:10:06.108 | INFO | "huge" : "", | 20260506 03:10:06.108 | INFO | "huge_shared" : " 27 GB", | 20260506 03:10:06.108 | INFO | "malloc" : " 14 GB", | 20260506 03:10:06.108 | INFO | "pinned" : " 29 GB" | 20260506 03:10:06.108 | INFO | } | 20260506 03:10:06.108 | INFO | Perf report for the last 5.1375 seconds: | 20260506 03:10:06.108 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:10:06.108 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:06.108 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:06.108 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:06.108 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260506 03:10:06.108 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:06.108 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:06.108 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 03:10:06.108 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:06.108 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:06.108 | INFO | Uploaded 36GiB to GPU in 1.96722s (18.2999GiB/s) | 20260506 03:10:06.108 | INFO | Downloaded 34GiB to GPU in 1.98445s (17.1332GiB/s) | 20260506 03:10:11.247 | INFO | Analyzing chunk frames = [8479744,8480256) | 20260506 03:10:11.247 | INFO | MemInfo: { | 20260506 03:10:11.247 | INFO | "gpu" : " 17 GB", | 20260506 03:10:11.247 | INFO | "huge" : "", | 20260506 03:10:11.247 | INFO | "huge_shared" : " 27 GB", | 20260506 03:10:11.248 | INFO | "malloc" : " 14 GB", | 20260506 03:10:11.248 | INFO | "pinned" : " 29 GB" | 20260506 03:10:11.248 | INFO | } | 20260506 03:10:11.248 | INFO | Perf report for the last 5.13983 seconds: | 20260506 03:10:11.248 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:10:11.248 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:11.248 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:11.248 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:11.248 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 03:10:11.248 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:11.248 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:11.248 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 03:10:11.248 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:11.248 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:11.248 | INFO | Uploaded 36GiB to GPU in 2.11579s (17.0149GiB/s) | 20260506 03:10:11.248 | INFO | Downloaded 34GiB to GPU in 2.04062s (16.6616GiB/s) | 20260506 03:10:16.379 | INFO | Analyzing chunk frames = [8480256,8480768) | 20260506 03:10:16.379 | INFO | MemInfo: { | 20260506 03:10:16.379 | INFO | "gpu" : " 17 GB", | 20260506 03:10:16.379 | INFO | "huge" : "", | 20260506 03:10:16.379 | INFO | "huge_shared" : " 27 GB", | 20260506 03:10:16.379 | INFO | "malloc" : " 14 GB", | 20260506 03:10:16.379 | INFO | "pinned" : " 29 GB" | 20260506 03:10:16.379 | INFO | } | 20260506 03:10:16.379 | INFO | Perf report for the last 5.13167 seconds: | 20260506 03:10:16.379 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:10:16.379 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:16.379 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:16.379 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:16.379 | INFO | ANALYSIS : 61.1%, 3.0, 94.8ms, 0 | 20260506 03:10:16.379 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:16.379 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:16.379 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 03:10:16.379 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:16.379 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:16.380 | INFO | Uploaded 36GiB to GPU in 2.08558s (17.2614GiB/s) | 20260506 03:10:16.380 | INFO | Downloaded 34GiB to GPU in 2.01358s (16.8853GiB/s) | 20260506 03:10:21.528 | INFO | Analyzing chunk frames = [8480768,8481280) | 20260506 03:10:21.528 | INFO | MemInfo: { | 20260506 03:10:21.528 | INFO | "gpu" : " 17 GB", | 20260506 03:10:21.528 | INFO | "huge" : "", | 20260506 03:10:21.528 | INFO | "huge_shared" : " 27 GB", | 20260506 03:10:21.528 | INFO | "malloc" : " 14 GB", | 20260506 03:10:21.528 | INFO | "pinned" : " 29 GB" | 20260506 03:10:21.528 | INFO | } | 20260506 03:10:21.528 | INFO | Perf report for the last 5.14889 seconds: | 20260506 03:10:21.528 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:10:21.528 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:21.528 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:21.528 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:21.528 | INFO | ANALYSIS : 60.5%, 3.0, 94.6ms, 0 | 20260506 03:10:21.528 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:21.528 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:21.528 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 03:10:21.528 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:21.528 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:21.529 | INFO | Uploaded 36GiB to GPU in 2.07291s (17.3669GiB/s) | 20260506 03:10:21.529 | INFO | Downloaded 34GiB to GPU in 2.03148s (16.7366GiB/s) | 20260506 03:10:26.669 | INFO | Analyzing chunk frames = [8481280,8481792) | 20260506 03:10:26.669 | INFO | MemInfo: { | 20260506 03:10:26.669 | INFO | "gpu" : " 17 GB", | 20260506 03:10:26.669 | INFO | "huge" : "", | 20260506 03:10:26.669 | INFO | "huge_shared" : " 27 GB", | 20260506 03:10:26.669 | INFO | "malloc" : " 14 GB", | 20260506 03:10:26.669 | INFO | "pinned" : " 29 GB" | 20260506 03:10:26.669 | INFO | } | 20260506 03:10:26.669 | INFO | Perf report for the last 5.14084 seconds: | 20260506 03:10:26.669 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:10:26.669 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:26.669 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:26.669 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:26.669 | INFO | ANALYSIS : 60.5%, 3.0, 94.4ms, 0 | 20260506 03:10:26.669 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:26.669 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:26.669 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:10:26.669 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:26.669 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:26.669 | INFO | Uploaded 36GiB to GPU in 2.05056s (17.5562GiB/s) | 20260506 03:10:26.669 | INFO | Downloaded 34GiB to GPU in 2.01545s (16.8697GiB/s) | 20260506 03:10:31.806 | INFO | Analyzing chunk frames = [8481792,8482304) | 20260506 03:10:31.806 | INFO | MemInfo: { | 20260506 03:10:31.806 | INFO | "gpu" : " 17 GB", | 20260506 03:10:31.806 | INFO | "huge" : "", | 20260506 03:10:31.806 | INFO | "huge_shared" : " 27 GB", | 20260506 03:10:31.806 | INFO | "malloc" : " 14 GB", | 20260506 03:10:31.806 | INFO | "pinned" : " 29 GB" | 20260506 03:10:31.806 | INFO | } | 20260506 03:10:31.806 | INFO | Perf report for the last 5.13685 seconds: | 20260506 03:10:31.806 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:10:31.806 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:31.806 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:31.806 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:31.806 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260506 03:10:31.806 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:31.806 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:31.806 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 03:10:31.806 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:31.806 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:31.806 | INFO | Uploaded 36GiB to GPU in 2.08266s (17.2856GiB/s) | 20260506 03:10:31.806 | INFO | Downloaded 34GiB to GPU in 2.06521s (16.4632GiB/s) | 20260506 03:10:36.941 | INFO | Analyzing chunk frames = [8482304,8482816) | 20260506 03:10:36.942 | INFO | MemInfo: { | 20260506 03:10:36.942 | INFO | "gpu" : " 17 GB", | 20260506 03:10:36.942 | INFO | "huge" : "", | 20260506 03:10:36.942 | INFO | "huge_shared" : " 27 GB", | 20260506 03:10:36.942 | INFO | "malloc" : " 14 GB", | 20260506 03:10:36.942 | INFO | "pinned" : " 29 GB" | 20260506 03:10:36.942 | INFO | } | 20260506 03:10:36.942 | INFO | Perf report for the last 5.1358 seconds: | 20260506 03:10:36.942 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:10:36.942 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:36.942 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:36.942 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:36.942 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260506 03:10:36.942 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:36.942 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:36.942 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 03:10:36.942 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:36.942 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:36.942 | INFO | Uploaded 36GiB to GPU in 2.03439s (17.6957GiB/s) | 20260506 03:10:36.942 | INFO | Downloaded 34GiB to GPU in 2.01736s (16.8537GiB/s) | 20260506 03:10:37.068 | INFO | Starting a round of Baz IO | 20260506 03:10:40.196 | INFO | Populating RTMetrics report | 20260506 03:10:40.196 | INFO | RTMetrics report populated | 20260506 03:10:40.196 | INFO | Opening tmp RTMetrics json file | 20260506 03:10:40.197 | INFO | Serializing RTMetrics to json | 20260506 03:10:40.198 | INFO | Serialization done, swapping tmp file now... | 20260506 03:10:40.198 | INFO | RTMetrics done | 20260506 03:10:40.276 | INFO | Finished a round of Baz IO: Wrote 1.769 GiB in 3.208 seconds | 20260506 03:10:40.276 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.24% / 0.00% / 78.76% | 20260506 03:10:40.276 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0635386% / 0% / 0.000107288% | 20260506 03:10:40.276 | INFO | Baz IO overall: | 20260506 03:10:40.276 | INFO | BytesWritten : 4728444661196 | 20260506 03:10:40.276 | INFO | NumEvents : 2218051219179 | 20260506 03:10:42.081 | INFO | Analyzing chunk frames = [8482816,8483328) | 20260506 03:10:42.082 | INFO | MemInfo: { | 20260506 03:10:42.082 | INFO | "gpu" : " 17 GB", | 20260506 03:10:42.082 | INFO | "huge" : "", | 20260506 03:10:42.082 | INFO | "huge_shared" : " 27 GB", | 20260506 03:10:42.082 | INFO | "malloc" : " 14 GB", | 20260506 03:10:42.082 | INFO | "pinned" : " 29 GB" | 20260506 03:10:42.082 | INFO | } | 20260506 03:10:42.082 | INFO | Perf report for the last 5.13997 seconds: | 20260506 03:10:42.082 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:10:42.082 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:42.082 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:42.082 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:42.082 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 03:10:42.082 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:42.082 | INFO | RT_METRICS : 42.6%, 1.0, 22.3ms, 0 | 20260506 03:10:42.082 | INFO | PRE_HQ : 59.4%, 1.7, 53.1ms, 0 | 20260506 03:10:42.082 | INFO | BAZWRITER : 61.7%, 3.9, 123.7ms, 0 | 20260506 03:10:42.082 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 03:10:42.082 | INFO | Uploaded 36GiB to GPU in 2.18582s (16.4698GiB/s) | 20260506 03:10:42.082 | INFO | Downloaded 38GiB to GPU in 2.24242s (16.946GiB/s) | 20260506 03:10:47.225 | INFO | Analyzing chunk frames = [8483328,8483840) | 20260506 03:10:47.225 | INFO | MemInfo: { | 20260506 03:10:47.225 | INFO | "gpu" : " 17 GB", | 20260506 03:10:47.225 | INFO | "huge" : "", | 20260506 03:10:47.225 | INFO | "huge_shared" : " 27 GB", | 20260506 03:10:47.225 | INFO | "malloc" : " 14 GB", | 20260506 03:10:47.225 | INFO | "pinned" : " 29 GB" | 20260506 03:10:47.225 | INFO | } | 20260506 03:10:47.225 | INFO | Perf report for the last 5.14361 seconds: | 20260506 03:10:47.225 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:10:47.225 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:47.225 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:47.225 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:47.225 | INFO | ANALYSIS : 60.5%, 3.0, 94.5ms, 0 | 20260506 03:10:47.225 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:47.225 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:47.225 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 03:10:47.225 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:47.226 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:47.226 | INFO | Uploaded 36GiB to GPU in 1.95415s (18.4224GiB/s) | 20260506 03:10:47.226 | INFO | Downloaded 34GiB to GPU in 1.97942s (17.1767GiB/s) | 20260506 03:10:52.362 | INFO | Analyzing chunk frames = [8483840,8484352) | 20260506 03:10:52.363 | INFO | MemInfo: { | 20260506 03:10:52.363 | INFO | "gpu" : " 17 GB", | 20260506 03:10:52.363 | INFO | "huge" : "", | 20260506 03:10:52.363 | INFO | "huge_shared" : " 27 GB", | 20260506 03:10:52.363 | INFO | "malloc" : " 14 GB", | 20260506 03:10:52.363 | INFO | "pinned" : " 29 GB" | 20260506 03:10:52.363 | INFO | } | 20260506 03:10:52.363 | INFO | Perf report for the last 5.13741 seconds: | 20260506 03:10:52.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:10:52.363 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:52.363 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:52.363 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:52.363 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260506 03:10:52.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:52.363 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:52.363 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 03:10:52.363 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:52.363 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:52.363 | INFO | Uploaded 36GiB to GPU in 2.0086s (17.9229GiB/s) | 20260506 03:10:52.363 | INFO | Downloaded 34GiB to GPU in 1.98438s (17.1338GiB/s) | 20260506 03:10:57.502 | INFO | Analyzing chunk frames = [8484352,8484864) | 20260506 03:10:57.502 | INFO | MemInfo: { | 20260506 03:10:57.502 | INFO | "gpu" : " 17 GB", | 20260506 03:10:57.502 | INFO | "huge" : "", | 20260506 03:10:57.502 | INFO | "huge_shared" : " 27 GB", | 20260506 03:10:57.503 | INFO | "malloc" : " 14 GB", | 20260506 03:10:57.503 | INFO | "pinned" : " 29 GB" | 20260506 03:10:57.503 | INFO | } | 20260506 03:10:57.503 | INFO | Perf report for the last 5.13986 seconds: | 20260506 03:10:57.503 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:10:57.503 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:57.503 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:57.503 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:57.503 | INFO | ANALYSIS : 61.0%, 3.0, 94.7ms, 0 | 20260506 03:10:57.503 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:57.503 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:57.503 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 03:10:57.503 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:57.503 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:10:57.503 | INFO | Uploaded 36GiB to GPU in 2.06196s (17.4592GiB/s) | 20260506 03:10:57.503 | INFO | Downloaded 34GiB to GPU in 2.00088s (16.9926GiB/s) | 20260506 03:11:02.638 | INFO | Analyzing chunk frames = [8484864,8485376) | 20260506 03:11:02.638 | INFO | MemInfo: { | 20260506 03:11:02.638 | INFO | "gpu" : " 17 GB", | 20260506 03:11:02.638 | INFO | "huge" : "", | 20260506 03:11:02.638 | INFO | "huge_shared" : " 27 GB", | 20260506 03:11:02.638 | INFO | "malloc" : " 14 GB", | 20260506 03:11:02.638 | INFO | "pinned" : " 29 GB" | 20260506 03:11:02.638 | INFO | } | 20260506 03:11:02.638 | INFO | Perf report for the last 5.13533 seconds: | 20260506 03:11:02.638 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:11:02.638 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:02.638 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:02.638 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:02.638 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260506 03:11:02.638 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:02.638 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:02.638 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 03:11:02.638 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:02.638 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:02.638 | INFO | Uploaded 36GiB to GPU in 2.1017s (17.129GiB/s) | 20260506 03:11:02.638 | INFO | Downloaded 34GiB to GPU in 2.03085s (16.7418GiB/s) | 20260506 03:11:07.788 | INFO | Analyzing chunk frames = [8485376,8485888) | 20260506 03:11:07.788 | INFO | MemInfo: { | 20260506 03:11:07.788 | INFO | "gpu" : " 17 GB", | 20260506 03:11:07.788 | INFO | "huge" : "", | 20260506 03:11:07.788 | INFO | "huge_shared" : " 27 GB", | 20260506 03:11:07.788 | INFO | "malloc" : " 14 GB", | 20260506 03:11:07.788 | INFO | "pinned" : " 29 GB" | 20260506 03:11:07.788 | INFO | } | 20260506 03:11:07.788 | INFO | Perf report for the last 5.15007 seconds: | 20260506 03:11:07.788 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:11:07.788 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:07.788 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:07.788 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:07.788 | INFO | ANALYSIS : 60.9%, 3.0, 95.3ms, 0 | 20260506 03:11:07.788 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:07.788 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:07.788 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 03:11:07.788 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:07.788 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:07.788 | INFO | Uploaded 36GiB to GPU in 2.12328s (16.9549GiB/s) | 20260506 03:11:07.788 | INFO | Downloaded 34GiB to GPU in 2.04861s (16.5966GiB/s) | 20260506 03:11:12.920 | INFO | Analyzing chunk frames = [8485888,8486400) | 20260506 03:11:12.920 | INFO | MemInfo: { | 20260506 03:11:12.920 | INFO | "gpu" : " 17 GB", | 20260506 03:11:12.920 | INFO | "huge" : "", | 20260506 03:11:12.920 | INFO | "huge_shared" : " 27 GB", | 20260506 03:11:12.920 | INFO | "malloc" : " 14 GB", | 20260506 03:11:12.921 | INFO | "pinned" : " 29 GB" | 20260506 03:11:12.921 | INFO | } | 20260506 03:11:12.921 | INFO | Perf report for the last 5.13251 seconds: | 20260506 03:11:12.921 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:11:12.921 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:12.921 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:12.921 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:12.921 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 03:11:12.921 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:12.921 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:12.921 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 03:11:12.921 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:12.921 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:12.921 | INFO | Uploaded 36GiB to GPU in 2.08666s (17.2524GiB/s) | 20260506 03:11:12.921 | INFO | Downloaded 34GiB to GPU in 2.07044s (16.4216GiB/s) | 20260506 03:11:18.055 | INFO | Analyzing chunk frames = [8486400,8486912) | 20260506 03:11:18.055 | INFO | MemInfo: { | 20260506 03:11:18.055 | INFO | "gpu" : " 17 GB", | 20260506 03:11:18.055 | INFO | "huge" : "", | 20260506 03:11:18.055 | INFO | "huge_shared" : " 27 GB", | 20260506 03:11:18.055 | INFO | "malloc" : " 14 GB", | 20260506 03:11:18.055 | INFO | "pinned" : " 29 GB" | 20260506 03:11:18.055 | INFO | } | 20260506 03:11:18.055 | INFO | Perf report for the last 5.13438 seconds: | 20260506 03:11:18.055 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:11:18.055 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:18.055 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:18.055 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:18.055 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 03:11:18.055 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:18.055 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:18.055 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 03:11:18.055 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:18.055 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:18.055 | INFO | Uploaded 36GiB to GPU in 2.04345s (17.6173GiB/s) | 20260506 03:11:18.055 | INFO | Downloaded 34GiB to GPU in 2.02597s (16.7821GiB/s) | 20260506 03:11:18.056 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 03:11:18.056 | INFO | Total mem usage: 15.4 GiB | 20260506 03:11:18.056 | INFO | All Time High Water Marks sum: 16.3 GiB (Savings: 5.2%) | 20260506 03:11:18.056 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -7.39%) | 20260506 03:11:18.056 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 03:11:18.056 | INFO | High Water Breakdown: | 20260506 03:11:18.056 | INFO | 61.3% : 8.81 GiB ( 91.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 03:11:18.056 | INFO | 38.6% : 5.55 GiB ( 83.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 03:11:18.056 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 03:11:18.056 | INFO | ------------------------------------------ | 20260506 03:11:18.056 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:11:18.056 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 03:11:18.056 | INFO | Total mem usage: 26.5 GiB | 20260506 03:11:18.056 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 03:11:18.056 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 03:11:18.056 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 03:11:18.056 | INFO | High Water Breakdown: | 20260506 03:11:18.056 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 03:11:18.056 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 03:11:18.056 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 03:11:18.056 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 03:11:18.056 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 03:11:18.056 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 03:11:18.056 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 03:11:18.056 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 03:11:18.056 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 03:11:18.056 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 03:11:18.056 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 03:11:18.056 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 03:11:18.056 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 03:11:18.056 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 03:11:18.056 | INFO | ------------------------------------------ | 20260506 03:11:18.056 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:11:18.056 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 03:11:18.056 | INFO | Total mem usage: 25.6 GiB | 20260506 03:11:18.056 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 03:11:18.056 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 03:11:18.056 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 03:11:18.056 | INFO | High Water Breakdown: | 20260506 03:11:18.056 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 03:11:18.056 | INFO | ------------------------------------------ | 20260506 03:11:18.056 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:11:18.056 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 03:11:18.056 | INFO | Total mem usage: 15.7 GiB | 20260506 03:11:18.056 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 03:11:18.056 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 03:11:18.056 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 03:11:18.056 | INFO | High Water Breakdown: | 20260506 03:11:18.056 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 03:11:18.056 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 03:11:18.056 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 03:11:18.056 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 03:11:18.056 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 03:11:18.056 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 03:11:18.056 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 03:11:18.056 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 03:11:18.056 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 03:11:18.056 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 03:11:18.056 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 03:11:18.056 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 03:11:18.056 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 03:11:18.056 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 03:11:18.056 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 03:11:18.056 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 03:11:18.056 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 03:11:18.056 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 03:11:18.056 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 03:11:18.056 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 03:11:18.056 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 03:11:18.056 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 03:11:18.056 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 03:11:18.056 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 03:11:18.056 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 03:11:18.056 | INFO | ------------------------------------------ | 20260506 03:11:18.057 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:11:18.057 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 03:11:18.057 | INFO | PerfReport: Gathered 1536 samples. | 20260506 03:11:18.057 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 03:11:18.057 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.37e+01, 1.35e+01 / 8.26e-01 | 20260506 03:11:18.057 | INFO | PerfReport: 0.29%, AggregateStats -- 9.18e-02 / 1.04e-01, 1.03e-02 / 2.36e-03 | 20260506 03:11:18.057 | INFO | PerfReport: 11.87%, Baseline -- 3.72e+00 / 4.55e+00, 5.90e-01 / 2.07e-01 | 20260506 03:11:18.057 | INFO | PerfReport: 3.32%, Histogram -- 1.04e+00 / 1.54e+00, 1.99e-01 / 7.41e-02 | 20260506 03:11:18.057 | INFO | PerfReport: 15.98%, DME -- 4.00e+01 / 4.40e+01, 3.88e+00 / 9.36e-01 | 20260506 03:11:18.057 | INFO | PerfReport: 57.57%, FrameLabeling -- 1.80e+01 / 1.93e+01, 1.59e+00 / 1.15e+00 | 20260506 03:11:18.057 | INFO | PerfReport: 6.96%, PulseAccumulating -- 2.18e+00 / 2.74e+00, 4.14e-01 / 2.18e-01 | 20260506 03:11:18.057 | INFO | PerfReport: 2.98%, Metrics -- 9.32e-01 / 1.19e+00, 2.52e-01 / 1.17e-01 | 20260506 03:11:18.057 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.22e-01 / 3.65e-01, 5.46e-02 / 2.73e-02 | 20260506 03:11:18.057 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 03:11:18.057 | INFO | PerfReport: Gathered 1536 samples. | 20260506 03:11:18.057 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 03:11:18.057 | INFO | PerfReport: Full Execution -- 9.73e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 03:11:18.057 | INFO | PerfReport: 22.11%, Upload -- 2.15e+01 / 0.00e+00, 7.03e+00 / 0.00e+00 | 20260506 03:11:18.057 | INFO | PerfReport: 21.93%, Download -- 2.13e+01 / 0.00e+00, 7.51e+00 / 0.00e+00 | 20260506 03:11:18.057 | INFO | PerfReport: 32.17%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 03:11:18.057 | INFO | PerfReport: 1.96%, LockUpload -- 1.90e+00 / 0.00e+00, 7.70e+00 / 0.00e+00 | 20260506 03:11:18.057 | INFO | PerfReport: 1.23%, LockDownload -- 1.20e+00 / 0.00e+00, 2.85e+00 / 0.00e+00 | 20260506 03:11:18.057 | INFO | PerfReport: 20.58%, LockCompute -- 2.00e+01 / 0.00e+00, 1.74e+01 / 0.00e+00 | 20260506 03:11:18.057 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.82e-02 / 0.00e+00 | 20260506 03:11:18.181 | INFO | Starting a round of Baz IO | 20260506 03:11:21.306 | INFO | Populating RTMetrics report | 20260506 03:11:21.306 | INFO | RTMetrics report populated | 20260506 03:11:21.306 | INFO | Opening tmp RTMetrics json file | 20260506 03:11:21.307 | INFO | Serializing RTMetrics to json | 20260506 03:11:21.308 | INFO | Serialization done, swapping tmp file now... | 20260506 03:11:21.308 | INFO | RTMetrics done | 20260506 03:11:21.398 | INFO | Finished a round of Baz IO: Wrote 1.768 GiB in 3.216 seconds | 20260506 03:11:21.398 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.24% / 0.00% / 78.76% | 20260506 03:11:21.398 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0630697% / 0% / 8.34465e-05% | 20260506 03:11:21.398 | INFO | Baz IO overall: | 20260506 03:11:21.398 | INFO | BytesWritten : 4730342956032 | 20260506 03:11:21.398 | INFO | NumEvents : 2218911965035 | 20260506 03:11:23.199 | INFO | Analyzing chunk frames = [8486912,8487424) | 20260506 03:11:23.199 | INFO | MemInfo: { | 20260506 03:11:23.199 | INFO | "gpu" : " 17 GB", | 20260506 03:11:23.199 | INFO | "huge" : "", | 20260506 03:11:23.199 | INFO | "huge_shared" : " 27 GB", | 20260506 03:11:23.199 | INFO | "malloc" : " 14 GB", | 20260506 03:11:23.199 | INFO | "pinned" : " 29 GB" | 20260506 03:11:23.199 | INFO | } | 20260506 03:11:23.199 | INFO | Perf report for the last 5.14435 seconds: | 20260506 03:11:23.199 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:11:23.199 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:23.200 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:23.200 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:23.200 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260506 03:11:23.200 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:23.200 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260506 03:11:23.200 | INFO | PRE_HQ : 58.9%, 1.7, 52.9ms, 0 | 20260506 03:11:23.200 | INFO | BAZWRITER : 61.7%, 3.9, 124.6ms, 0 | 20260506 03:11:23.200 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 03:11:23.200 | INFO | Uploaded 36GiB to GPU in 2.13206s (16.8851GiB/s) | 20260506 03:11:23.200 | INFO | Downloaded 38GiB to GPU in 2.23224s (17.0233GiB/s) | 20260506 03:11:28.338 | INFO | Analyzing chunk frames = [8487424,8487936) | 20260506 03:11:28.338 | INFO | MemInfo: { | 20260506 03:11:28.338 | INFO | "gpu" : " 17 GB", | 20260506 03:11:28.338 | INFO | "huge" : "", | 20260506 03:11:28.338 | INFO | "huge_shared" : " 27 GB", | 20260506 03:11:28.338 | INFO | "malloc" : " 14 GB", | 20260506 03:11:28.338 | INFO | "pinned" : " 29 GB" | 20260506 03:11:28.339 | INFO | } | 20260506 03:11:28.339 | INFO | Perf report for the last 5.13939 seconds: | 20260506 03:11:28.339 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:11:28.339 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:28.339 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:28.339 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:28.339 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 03:11:28.339 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:28.339 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:28.339 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 03:11:28.339 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:28.339 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:28.339 | INFO | Uploaded 36GiB to GPU in 1.96365s (18.3332GiB/s) | 20260506 03:11:28.339 | INFO | Downloaded 34GiB to GPU in 1.9771s (17.1969GiB/s) | 20260506 03:11:33.481 | INFO | Analyzing chunk frames = [8487936,8488448) | 20260506 03:11:33.481 | INFO | MemInfo: { | 20260506 03:11:33.481 | INFO | "gpu" : " 17 GB", | 20260506 03:11:33.481 | INFO | "huge" : "", | 20260506 03:11:33.481 | INFO | "huge_shared" : " 27 GB", | 20260506 03:11:33.481 | INFO | "malloc" : " 14 GB", | 20260506 03:11:33.481 | INFO | "pinned" : " 29 GB" | 20260506 03:11:33.481 | INFO | } | 20260506 03:11:33.481 | INFO | Perf report for the last 5.14253 seconds: | 20260506 03:11:33.481 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:11:33.481 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:33.481 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:33.481 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:33.481 | INFO | ANALYSIS : 60.3%, 3.0, 93.9ms, 0 | 20260506 03:11:33.481 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:33.481 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:33.481 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 03:11:33.481 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:33.481 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:33.482 | INFO | Uploaded 36GiB to GPU in 1.97732s (18.2064GiB/s) | 20260506 03:11:33.482 | INFO | Downloaded 34GiB to GPU in 1.98398s (17.1372GiB/s) | 20260506 03:11:38.620 | INFO | Analyzing chunk frames = [8488448,8488960) | 20260506 03:11:38.620 | INFO | MemInfo: { | 20260506 03:11:38.620 | INFO | "gpu" : " 17 GB", | 20260506 03:11:38.620 | INFO | "huge" : "", | 20260506 03:11:38.620 | INFO | "huge_shared" : " 27 GB", | 20260506 03:11:38.620 | INFO | "malloc" : " 14 GB", | 20260506 03:11:38.620 | INFO | "pinned" : " 29 GB" | 20260506 03:11:38.620 | INFO | } | 20260506 03:11:38.620 | INFO | Perf report for the last 5.13879 seconds: | 20260506 03:11:38.620 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:11:38.620 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:38.620 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:38.620 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:38.620 | INFO | ANALYSIS : 60.9%, 3.0, 94.6ms, 0 | 20260506 03:11:38.620 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:38.620 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:38.620 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 03:11:38.620 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:38.620 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:38.620 | INFO | Uploaded 36GiB to GPU in 2.09348s (17.1962GiB/s) | 20260506 03:11:38.620 | INFO | Downloaded 34GiB to GPU in 2.01179s (16.9004GiB/s) | 20260506 03:11:43.753 | INFO | Analyzing chunk frames = [8488960,8489472) | 20260506 03:11:43.753 | INFO | MemInfo: { | 20260506 03:11:43.753 | INFO | "gpu" : " 17 GB", | 20260506 03:11:43.753 | INFO | "huge" : "", | 20260506 03:11:43.753 | INFO | "huge_shared" : " 27 GB", | 20260506 03:11:43.753 | INFO | "malloc" : " 14 GB", | 20260506 03:11:43.753 | INFO | "pinned" : " 29 GB" | 20260506 03:11:43.753 | INFO | } | 20260506 03:11:43.753 | INFO | Perf report for the last 5.13279 seconds: | 20260506 03:11:43.753 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:11:43.753 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:43.753 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:43.753 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:43.753 | INFO | ANALYSIS : 60.9%, 3.0, 95.0ms, 0 | 20260506 03:11:43.753 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:43.753 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:43.753 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:11:43.753 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:43.753 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:43.753 | INFO | Uploaded 36GiB to GPU in 2.11967s (16.9837GiB/s) | 20260506 03:11:43.754 | INFO | Downloaded 34GiB to GPU in 2.04335s (16.6394GiB/s) | 20260506 03:11:48.895 | INFO | Analyzing chunk frames = [8489472,8489984) | 20260506 03:11:48.895 | INFO | MemInfo: { | 20260506 03:11:48.895 | INFO | "gpu" : " 17 GB", | 20260506 03:11:48.895 | INFO | "huge" : "", | 20260506 03:11:48.895 | INFO | "huge_shared" : " 27 GB", | 20260506 03:11:48.895 | INFO | "malloc" : " 14 GB", | 20260506 03:11:48.895 | INFO | "pinned" : " 29 GB" | 20260506 03:11:48.895 | INFO | } | 20260506 03:11:48.895 | INFO | Perf report for the last 5.14215 seconds: | 20260506 03:11:48.895 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:11:48.895 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:48.895 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:48.895 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:48.895 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 03:11:48.895 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:48.895 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:48.895 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 03:11:48.895 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:48.895 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:48.895 | INFO | Uploaded 36GiB to GPU in 2.06433s (17.4391GiB/s) | 20260506 03:11:48.895 | INFO | Downloaded 34GiB to GPU in 2.03932s (16.6722GiB/s) | 20260506 03:11:54.035 | INFO | Analyzing chunk frames = [8489984,8490496) | 20260506 03:11:54.035 | INFO | MemInfo: { | 20260506 03:11:54.035 | INFO | "gpu" : " 17 GB", | 20260506 03:11:54.036 | INFO | "huge" : "", | 20260506 03:11:54.036 | INFO | "huge_shared" : " 27 GB", | 20260506 03:11:54.036 | INFO | "malloc" : " 14 GB", | 20260506 03:11:54.036 | INFO | "pinned" : " 29 GB" | 20260506 03:11:54.036 | INFO | } | 20260506 03:11:54.036 | INFO | Perf report for the last 5.14065 seconds: | 20260506 03:11:54.036 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:11:54.036 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:54.036 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:54.036 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:54.036 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 03:11:54.036 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:54.036 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:54.036 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 03:11:54.036 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:54.036 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:54.036 | INFO | Uploaded 36GiB to GPU in 2.10811s (17.0769GiB/s) | 20260506 03:11:54.036 | INFO | Downloaded 34GiB to GPU in 2.09235s (16.2497GiB/s) | 20260506 03:11:59.179 | INFO | Analyzing chunk frames = [8490496,8491008) | 20260506 03:11:59.179 | INFO | MemInfo: { | 20260506 03:11:59.179 | INFO | "gpu" : " 17 GB", | 20260506 03:11:59.179 | INFO | "huge" : "", | 20260506 03:11:59.179 | INFO | "huge_shared" : " 27 GB", | 20260506 03:11:59.179 | INFO | "malloc" : " 14 GB", | 20260506 03:11:59.179 | INFO | "pinned" : " 29 GB" | 20260506 03:11:59.179 | INFO | } | 20260506 03:11:59.179 | INFO | Perf report for the last 5.14316 seconds: | 20260506 03:11:59.179 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:11:59.179 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:59.179 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:59.179 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:59.179 | INFO | ANALYSIS : 61.4%, 3.0, 95.9ms, 0 | 20260506 03:11:59.179 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:59.179 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:59.179 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 03:11:59.179 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:59.179 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:11:59.179 | INFO | Uploaded 36GiB to GPU in 2.05783s (17.4942GiB/s) | 20260506 03:11:59.179 | INFO | Downloaded 34GiB to GPU in 2.03191s (16.7331GiB/s) | 20260506 03:11:59.303 | INFO | Starting a round of Baz IO | 20260506 03:12:02.419 | INFO | Populating RTMetrics report | 20260506 03:12:02.419 | INFO | RTMetrics report populated | 20260506 03:12:02.419 | INFO | Opening tmp RTMetrics json file | 20260506 03:12:02.420 | INFO | Serializing RTMetrics to json | 20260506 03:12:02.422 | INFO | Serialization done, swapping tmp file now... | 20260506 03:12:02.422 | INFO | RTMetrics done | 20260506 03:12:02.503 | INFO | Finished a round of Baz IO: Wrote 1.768 GiB in 3.2 seconds | 20260506 03:12:02.504 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.23% / 0.00% / 78.76% | 20260506 03:12:02.504 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.063165% / 0% / 0.000115236% | 20260506 03:12:02.504 | INFO | Baz IO overall: | 20260506 03:12:02.504 | INFO | BytesWritten : 4732241547687 | 20260506 03:12:02.504 | INFO | NumEvents : 2219772974422 | 20260506 03:12:04.316 | INFO | Analyzing chunk frames = [8491008,8491520) | 20260506 03:12:04.316 | INFO | MemInfo: { | 20260506 03:12:04.316 | INFO | "gpu" : " 17 GB", | 20260506 03:12:04.316 | INFO | "huge" : "", | 20260506 03:12:04.316 | INFO | "huge_shared" : " 27 GB", | 20260506 03:12:04.316 | INFO | "malloc" : " 14 GB", | 20260506 03:12:04.316 | INFO | "pinned" : " 29 GB" | 20260506 03:12:04.316 | INFO | } | 20260506 03:12:04.316 | INFO | Perf report for the last 5.1374 seconds: | 20260506 03:12:04.316 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:12:04.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:04.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:04.316 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:04.316 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 03:12:04.316 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:04.316 | INFO | RT_METRICS : 42.8%, 1.0, 22.4ms, 0 | 20260506 03:12:04.316 | INFO | PRE_HQ : 59.1%, 1.7, 53.2ms, 0 | 20260506 03:12:04.316 | INFO | BAZWRITER : 61.5%, 3.5, 111.0ms, 0 | 20260506 03:12:04.316 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 03:12:04.317 | INFO | Uploaded 36GiB to GPU in 2.09324s (17.1982GiB/s) | 20260506 03:12:04.317 | INFO | Downloaded 38GiB to GPU in 2.22923s (17.0462GiB/s) | 20260506 03:12:09.448 | INFO | Analyzing chunk frames = [8491520,8492032) | 20260506 03:12:09.448 | INFO | MemInfo: { | 20260506 03:12:09.448 | INFO | "gpu" : " 17 GB", | 20260506 03:12:09.448 | INFO | "huge" : "", | 20260506 03:12:09.448 | INFO | "huge_shared" : " 27 GB", | 20260506 03:12:09.448 | INFO | "malloc" : " 14 GB", | 20260506 03:12:09.448 | INFO | "pinned" : " 29 GB" | 20260506 03:12:09.448 | INFO | } | 20260506 03:12:09.448 | INFO | Perf report for the last 5.13206 seconds: | 20260506 03:12:09.448 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:12:09.448 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:09.448 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:09.448 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:09.448 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 03:12:09.448 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:09.448 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:09.448 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 03:12:09.448 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:09.449 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:09.449 | INFO | Uploaded 36GiB to GPU in 1.95809s (18.3853GiB/s) | 20260506 03:12:09.449 | INFO | Downloaded 34GiB to GPU in 1.97968s (17.1745GiB/s) | 20260506 03:12:14.592 | INFO | Analyzing chunk frames = [8492032,8492544) | 20260506 03:12:14.592 | INFO | MemInfo: { | 20260506 03:12:14.592 | INFO | "gpu" : " 17 GB", | 20260506 03:12:14.592 | INFO | "huge" : "", | 20260506 03:12:14.592 | INFO | "huge_shared" : " 27 GB", | 20260506 03:12:14.592 | INFO | "malloc" : " 14 GB", | 20260506 03:12:14.592 | INFO | "pinned" : " 29 GB" | 20260506 03:12:14.592 | INFO | } | 20260506 03:12:14.592 | INFO | Perf report for the last 5.14362 seconds: | 20260506 03:12:14.592 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:12:14.592 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:14.592 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:14.592 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:14.592 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 03:12:14.592 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:14.592 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:14.592 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 03:12:14.592 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:14.592 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:14.592 | INFO | Uploaded 36GiB to GPU in 2.09226s (17.2063GiB/s) | 20260506 03:12:14.592 | INFO | Downloaded 34GiB to GPU in 2.02664s (16.7765GiB/s) | 20260506 03:12:19.732 | INFO | Analyzing chunk frames = [8492544,8493056) | 20260506 03:12:19.732 | INFO | MemInfo: { | 20260506 03:12:19.732 | INFO | "gpu" : " 17 GB", | 20260506 03:12:19.732 | INFO | "huge" : "", | 20260506 03:12:19.732 | INFO | "huge_shared" : " 27 GB", | 20260506 03:12:19.732 | INFO | "malloc" : " 14 GB", | 20260506 03:12:19.732 | INFO | "pinned" : " 29 GB" | 20260506 03:12:19.732 | INFO | } | 20260506 03:12:19.732 | INFO | Perf report for the last 5.14001 seconds: | 20260506 03:12:19.732 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:12:19.732 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:19.732 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:19.732 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:19.732 | INFO | ANALYSIS : 61.0%, 3.0, 94.7ms, 0 | 20260506 03:12:19.732 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:19.732 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:19.732 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 03:12:19.732 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:19.732 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:19.732 | INFO | Uploaded 36GiB to GPU in 2.08797s (17.2417GiB/s) | 20260506 03:12:19.732 | INFO | Downloaded 34GiB to GPU in 2.01849s (16.8443GiB/s) | 20260506 03:12:24.868 | INFO | Analyzing chunk frames = [8493056,8493568) | 20260506 03:12:24.868 | INFO | MemInfo: { | 20260506 03:12:24.868 | INFO | "gpu" : " 17 GB", | 20260506 03:12:24.868 | INFO | "huge" : "", | 20260506 03:12:24.868 | INFO | "huge_shared" : " 27 GB", | 20260506 03:12:24.868 | INFO | "malloc" : " 14 GB", | 20260506 03:12:24.868 | INFO | "pinned" : " 29 GB" | 20260506 03:12:24.868 | INFO | } | 20260506 03:12:24.868 | INFO | Perf report for the last 5.13634 seconds: | 20260506 03:12:24.868 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:12:24.868 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:24.868 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:24.868 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:24.868 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 03:12:24.869 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:24.869 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:24.869 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 03:12:24.869 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:24.869 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:24.869 | INFO | Uploaded 36GiB to GPU in 2.1085s (17.0738GiB/s) | 20260506 03:12:24.869 | INFO | Downloaded 34GiB to GPU in 2.03514s (16.7064GiB/s) | 20260506 03:12:30.008 | INFO | Analyzing chunk frames = [8493568,8494080) | 20260506 03:12:30.008 | INFO | MemInfo: { | 20260506 03:12:30.008 | INFO | "gpu" : " 17 GB", | 20260506 03:12:30.008 | INFO | "huge" : "", | 20260506 03:12:30.008 | INFO | "huge_shared" : " 27 GB", | 20260506 03:12:30.008 | INFO | "malloc" : " 14 GB", | 20260506 03:12:30.008 | INFO | "pinned" : " 29 GB" | 20260506 03:12:30.008 | INFO | } | 20260506 03:12:30.008 | INFO | Perf report for the last 5.13955 seconds: | 20260506 03:12:30.008 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:12:30.008 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:30.008 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:30.008 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:30.008 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 03:12:30.008 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:30.008 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:30.008 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 03:12:30.008 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:30.008 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:30.008 | INFO | Uploaded 36GiB to GPU in 2.08483s (17.2676GiB/s) | 20260506 03:12:30.008 | INFO | Downloaded 34GiB to GPU in 2.03528s (16.7053GiB/s) | 20260506 03:12:35.153 | INFO | Analyzing chunk frames = [8494080,8494592) | 20260506 03:12:35.153 | INFO | MemInfo: { | 20260506 03:12:35.153 | INFO | "gpu" : " 17 GB", | 20260506 03:12:35.153 | INFO | "huge" : "", | 20260506 03:12:35.153 | INFO | "huge_shared" : " 27 GB", | 20260506 03:12:35.153 | INFO | "malloc" : " 14 GB", | 20260506 03:12:35.153 | INFO | "pinned" : " 29 GB" | 20260506 03:12:35.153 | INFO | } | 20260506 03:12:35.153 | INFO | Perf report for the last 5.1454 seconds: | 20260506 03:12:35.153 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:12:35.153 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:35.153 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:35.153 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:35.153 | INFO | ANALYSIS : 62.0%, 3.0, 97.0ms, 0 | 20260506 03:12:35.153 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:35.153 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:35.153 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 03:12:35.153 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:35.153 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:35.154 | INFO | Uploaded 36GiB to GPU in 2.06195s (17.4592GiB/s) | 20260506 03:12:35.154 | INFO | Downloaded 34GiB to GPU in 2.08025s (16.3442GiB/s) | 20260506 03:12:40.294 | INFO | Analyzing chunk frames = [8494592,8495104) | 20260506 03:12:40.295 | INFO | MemInfo: { | 20260506 03:12:40.295 | INFO | "gpu" : " 17 GB", | 20260506 03:12:40.295 | INFO | "huge" : "", | 20260506 03:12:40.295 | INFO | "huge_shared" : " 27 GB", | 20260506 03:12:40.295 | INFO | "malloc" : " 14 GB", | 20260506 03:12:40.295 | INFO | "pinned" : " 29 GB" | 20260506 03:12:40.295 | INFO | } | 20260506 03:12:40.295 | INFO | Perf report for the last 5.14141 seconds: | 20260506 03:12:40.295 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:12:40.295 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:40.295 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:40.295 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:40.295 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260506 03:12:40.295 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:40.295 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:40.295 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 03:12:40.295 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:40.295 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:40.295 | INFO | Uploaded 36GiB to GPU in 2.04864s (17.5726GiB/s) | 20260506 03:12:40.295 | INFO | Downloaded 34GiB to GPU in 2.02867s (16.7597GiB/s) | 20260506 03:12:40.295 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 03:12:40.295 | INFO | Total mem usage: 15.4 GiB | 20260506 03:12:40.295 | INFO | All Time High Water Marks sum: 16.3 GiB (Savings: 5.2%) | 20260506 03:12:40.295 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -4.26%) | 20260506 03:12:40.295 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 03:12:40.295 | INFO | High Water Breakdown: | 20260506 03:12:40.295 | INFO | 61.7% : 9.13 GiB ( 94.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 03:12:40.295 | INFO | 38.2% : 5.66 GiB ( 85.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 03:12:40.295 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 03:12:40.295 | INFO | ------------------------------------------ | 20260506 03:12:40.295 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:12:40.295 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 03:12:40.295 | INFO | Total mem usage: 26.5 GiB | 20260506 03:12:40.295 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 03:12:40.295 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 03:12:40.295 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 03:12:40.295 | INFO | High Water Breakdown: | 20260506 03:12:40.295 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 03:12:40.295 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 03:12:40.295 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 03:12:40.295 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 03:12:40.295 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 03:12:40.295 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 03:12:40.296 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 03:12:40.296 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 03:12:40.296 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 03:12:40.296 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 03:12:40.296 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 03:12:40.296 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 03:12:40.296 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 03:12:40.296 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 03:12:40.296 | INFO | ------------------------------------------ | 20260506 03:12:40.296 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:12:40.296 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 03:12:40.296 | INFO | Total mem usage: 25.6 GiB | 20260506 03:12:40.296 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 03:12:40.296 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 03:12:40.296 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 03:12:40.296 | INFO | High Water Breakdown: | 20260506 03:12:40.296 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 03:12:40.296 | INFO | ------------------------------------------ | 20260506 03:12:40.296 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:12:40.296 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 03:12:40.296 | INFO | Total mem usage: 15.7 GiB | 20260506 03:12:40.296 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 03:12:40.296 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 03:12:40.296 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 03:12:40.296 | INFO | High Water Breakdown: | 20260506 03:12:40.296 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 03:12:40.296 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 03:12:40.296 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 03:12:40.296 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 03:12:40.296 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 03:12:40.296 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 03:12:40.296 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 03:12:40.296 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 03:12:40.296 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 03:12:40.296 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 03:12:40.296 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 03:12:40.296 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 03:12:40.296 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 03:12:40.296 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 03:12:40.296 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 03:12:40.296 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 03:12:40.296 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 03:12:40.296 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 03:12:40.296 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 03:12:40.296 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 03:12:40.296 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 03:12:40.296 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 03:12:40.296 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 03:12:40.296 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 03:12:40.296 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 03:12:40.296 | INFO | ------------------------------------------ | 20260506 03:12:40.296 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:12:40.296 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 03:12:40.296 | INFO | PerfReport: Gathered 1536 samples. | 20260506 03:12:40.296 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 03:12:40.296 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.37e+01, 1.35e+01 / 5.84e-01 | 20260506 03:12:40.296 | INFO | PerfReport: 0.29%, AggregateStats -- 9.16e-02 / 1.05e-01, 9.87e-03 / 4.28e-03 | 20260506 03:12:40.296 | INFO | PerfReport: 11.88%, Baseline -- 3.71e+00 / 4.47e+00, 5.86e-01 / 3.16e-01 | 20260506 03:12:40.296 | INFO | PerfReport: 3.30%, Histogram -- 1.03e+00 / 1.52e+00, 1.96e-01 / 1.07e-01 | 20260506 03:12:40.296 | INFO | PerfReport: 15.99%, DME -- 4.00e+01 / 4.39e+01, 3.77e+00 / 9.16e-01 | 20260506 03:12:40.296 | INFO | PerfReport: 57.52%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.53e+00 / 1.10e+00 | 20260506 03:12:40.296 | INFO | PerfReport: 7.00%, PulseAccumulating -- 2.19e+00 / 2.73e+00, 4.10e-01 / 2.20e-01 | 20260506 03:12:40.296 | INFO | PerfReport: 2.99%, Metrics -- 9.33e-01 / 1.22e+00, 2.51e-01 / 1.17e-01 | 20260506 03:12:40.296 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.21e-01 / 3.70e-01, 5.40e-02 / 2.48e-02 | 20260506 03:12:40.296 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 03:12:40.296 | INFO | PerfReport: Gathered 1536 samples. | 20260506 03:12:40.296 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 03:12:40.296 | INFO | PerfReport: Full Execution -- 9.74e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260506 03:12:40.296 | INFO | PerfReport: 22.09%, Upload -- 2.15e+01 / 0.00e+00, 7.03e+00 / 0.00e+00 | 20260506 03:12:40.296 | INFO | PerfReport: 21.95%, Download -- 2.14e+01 / 0.00e+00, 7.51e+00 / 0.00e+00 | 20260506 03:12:40.296 | INFO | PerfReport: 32.09%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 03:12:40.296 | INFO | PerfReport: 1.99%, LockUpload -- 1.94e+00 / 0.00e+00, 7.82e+00 / 0.00e+00 | 20260506 03:12:40.296 | INFO | PerfReport: 1.29%, LockDownload -- 1.26e+00 / 0.00e+00, 2.94e+00 / 0.00e+00 | 20260506 03:12:40.296 | INFO | PerfReport: 20.57%, LockCompute -- 2.00e+01 / 0.00e+00, 1.78e+01 / 0.00e+00 | 20260506 03:12:40.296 | INFO | PerfReport: 0.01%, Misc -- 1.46e-02 / 0.00e+00, 2.93e-02 / 0.00e+00 | 20260506 03:12:40.421 | INFO | Starting a round of Baz IO | 20260506 03:12:43.548 | INFO | Populating RTMetrics report | 20260506 03:12:43.548 | INFO | RTMetrics report populated | 20260506 03:12:43.548 | INFO | Opening tmp RTMetrics json file | 20260506 03:12:43.550 | INFO | Serializing RTMetrics to json | 20260506 03:12:43.552 | INFO | Serialization done, swapping tmp file now... | 20260506 03:12:43.552 | INFO | RTMetrics done | 20260506 03:12:43.639 | INFO | Finished a round of Baz IO: Wrote 1.766 GiB in 3.217 seconds | 20260506 03:12:43.639 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.23% / 0.00% / 78.77% | 20260506 03:12:43.639 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0634114% / 0% / 9.93411e-05% | 20260506 03:12:43.639 | INFO | Baz IO overall: | 20260506 03:12:43.639 | INFO | BytesWritten : 4734137844362 | 20260506 03:12:43.639 | INFO | NumEvents : 2220632457038 | 20260506 03:12:45.433 | INFO | Analyzing chunk frames = [8495104,8495616) | 20260506 03:12:45.433 | INFO | MemInfo: { | 20260506 03:12:45.433 | INFO | "gpu" : " 17 GB", | 20260506 03:12:45.433 | INFO | "huge" : "", | 20260506 03:12:45.433 | INFO | "huge_shared" : " 27 GB", | 20260506 03:12:45.433 | INFO | "malloc" : " 14 GB", | 20260506 03:12:45.433 | INFO | "pinned" : " 29 GB" | 20260506 03:12:45.433 | INFO | } | 20260506 03:12:45.433 | INFO | Perf report for the last 5.13786 seconds: | 20260506 03:12:45.433 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:12:45.433 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:45.433 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:45.433 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:45.433 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260506 03:12:45.433 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:45.433 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260506 03:12:45.433 | INFO | PRE_HQ : 59.1%, 1.7, 53.1ms, 0 | 20260506 03:12:45.433 | INFO | BAZWRITER : 61.9%, 3.5, 111.9ms, 0 | 20260506 03:12:45.433 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 03:12:45.433 | INFO | Uploaded 36GiB to GPU in 2.11811s (16.9963GiB/s) | 20260506 03:12:45.433 | INFO | Downloaded 38GiB to GPU in 2.22907s (17.0475GiB/s) | 20260506 03:12:50.566 | INFO | Analyzing chunk frames = [8495616,8496128) | 20260506 03:12:50.566 | INFO | MemInfo: { | 20260506 03:12:50.566 | INFO | "gpu" : " 17 GB", | 20260506 03:12:50.566 | INFO | "huge" : "", | 20260506 03:12:50.566 | INFO | "huge_shared" : " 27 GB", | 20260506 03:12:50.566 | INFO | "malloc" : " 14 GB", | 20260506 03:12:50.566 | INFO | "pinned" : " 29 GB" | 20260506 03:12:50.566 | INFO | } | 20260506 03:12:50.566 | INFO | Perf report for the last 5.13351 seconds: | 20260506 03:12:50.566 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:12:50.566 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:50.566 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:50.566 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:50.566 | INFO | ANALYSIS : 60.8%, 3.0, 94.9ms, 0 | 20260506 03:12:50.566 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:50.566 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:50.566 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 03:12:50.566 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:50.566 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:50.566 | INFO | Uploaded 36GiB to GPU in 2.05091s (17.5532GiB/s) | 20260506 03:12:50.566 | INFO | Downloaded 34GiB to GPU in 2.00052s (16.9956GiB/s) | 20260506 03:12:55.702 | INFO | Analyzing chunk frames = [8496128,8496640) | 20260506 03:12:55.702 | INFO | MemInfo: { | 20260506 03:12:55.702 | INFO | "gpu" : " 17 GB", | 20260506 03:12:55.702 | INFO | "huge" : "", | 20260506 03:12:55.702 | INFO | "huge_shared" : " 27 GB", | 20260506 03:12:55.702 | INFO | "malloc" : " 14 GB", | 20260506 03:12:55.702 | INFO | "pinned" : " 29 GB" | 20260506 03:12:55.702 | INFO | } | 20260506 03:12:55.702 | INFO | Perf report for the last 5.13617 seconds: | 20260506 03:12:55.702 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:12:55.702 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:55.702 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:55.702 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:55.702 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 03:12:55.702 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:55.702 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:55.702 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 03:12:55.703 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:55.703 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:12:55.703 | INFO | Uploaded 36GiB to GPU in 2.09712s (17.1664GiB/s) | 20260506 03:12:55.703 | INFO | Downloaded 34GiB to GPU in 2.028s (16.7653GiB/s) | 20260506 03:13:00.846 | INFO | Analyzing chunk frames = [8496640,8497152) | 20260506 03:13:00.846 | INFO | MemInfo: { | 20260506 03:13:00.846 | INFO | "gpu" : " 17 GB", | 20260506 03:13:00.846 | INFO | "huge" : "", | 20260506 03:13:00.846 | INFO | "huge_shared" : " 27 GB", | 20260506 03:13:00.846 | INFO | "malloc" : " 14 GB", | 20260506 03:13:00.846 | INFO | "pinned" : " 29 GB" | 20260506 03:13:00.846 | INFO | } | 20260506 03:13:00.846 | INFO | Perf report for the last 5.14413 seconds: | 20260506 03:13:00.846 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:13:00.846 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:00.846 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:00.846 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:00.846 | INFO | ANALYSIS : 61.0%, 3.0, 94.8ms, 0 | 20260506 03:13:00.847 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:00.847 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:00.847 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 03:13:00.847 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:00.847 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:00.847 | INFO | Uploaded 36GiB to GPU in 2.10232s (17.1239GiB/s) | 20260506 03:13:00.847 | INFO | Downloaded 34GiB to GPU in 2.02924s (16.755GiB/s) | 20260506 03:13:05.995 | INFO | Analyzing chunk frames = [8497152,8497664) | 20260506 03:13:05.995 | INFO | MemInfo: { | 20260506 03:13:05.995 | INFO | "gpu" : " 17 GB", | 20260506 03:13:05.995 | INFO | "huge" : "", | 20260506 03:13:05.995 | INFO | "huge_shared" : " 27 GB", | 20260506 03:13:05.995 | INFO | "malloc" : " 14 GB", | 20260506 03:13:05.996 | INFO | "pinned" : " 29 GB" | 20260506 03:13:05.996 | INFO | } | 20260506 03:13:05.996 | INFO | Perf report for the last 5.14917 seconds: | 20260506 03:13:05.996 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:13:05.996 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:05.996 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:05.996 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:05.996 | INFO | ANALYSIS : 60.5%, 3.0, 94.7ms, 0 | 20260506 03:13:05.996 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:05.996 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:05.996 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 03:13:05.996 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:05.996 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:05.996 | INFO | Uploaded 36GiB to GPU in 2.09419s (17.1904GiB/s) | 20260506 03:13:05.996 | INFO | Downloaded 34GiB to GPU in 2.02333s (16.804GiB/s) | 20260506 03:13:11.124 | INFO | Analyzing chunk frames = [8497664,8498176) | 20260506 03:13:11.124 | INFO | MemInfo: { | 20260506 03:13:11.124 | INFO | "gpu" : " 17 GB", | 20260506 03:13:11.124 | INFO | "huge" : "", | 20260506 03:13:11.125 | INFO | "huge_shared" : " 27 GB", | 20260506 03:13:11.125 | INFO | "malloc" : " 14 GB", | 20260506 03:13:11.125 | INFO | "pinned" : " 29 GB" | 20260506 03:13:11.125 | INFO | } | 20260506 03:13:11.125 | INFO | Perf report for the last 5.12907 seconds: | 20260506 03:13:11.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:13:11.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:11.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:11.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:11.125 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260506 03:13:11.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:11.125 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:11.125 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:13:11.125 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:11.125 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:11.125 | INFO | Uploaded 36GiB to GPU in 2.07392s (17.3584GiB/s) | 20260506 03:13:11.125 | INFO | Downloaded 34GiB to GPU in 2.02554s (16.7856GiB/s) | 20260506 03:13:16.260 | INFO | Analyzing chunk frames = [8498176,8498688) | 20260506 03:13:16.260 | INFO | MemInfo: { | 20260506 03:13:16.260 | INFO | "gpu" : " 17 GB", | 20260506 03:13:16.260 | INFO | "huge" : "", | 20260506 03:13:16.260 | INFO | "huge_shared" : " 27 GB", | 20260506 03:13:16.260 | INFO | "malloc" : " 14 GB", | 20260506 03:13:16.261 | INFO | "pinned" : " 29 GB" | 20260506 03:13:16.261 | INFO | } | 20260506 03:13:16.261 | INFO | Perf report for the last 5.13583 seconds: | 20260506 03:13:16.261 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:13:16.261 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:16.261 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:16.261 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:16.261 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260506 03:13:16.261 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:16.261 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:16.261 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 03:13:16.261 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:16.261 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:16.261 | INFO | Uploaded 36GiB to GPU in 2.06678s (17.4184GiB/s) | 20260506 03:13:16.261 | INFO | Downloaded 34GiB to GPU in 2.0739s (16.3942GiB/s) | 20260506 03:13:21.400 | INFO | Analyzing chunk frames = [8498688,8499200) | 20260506 03:13:21.400 | INFO | MemInfo: { | 20260506 03:13:21.400 | INFO | "gpu" : " 17 GB", | 20260506 03:13:21.400 | INFO | "huge" : "", | 20260506 03:13:21.400 | INFO | "huge_shared" : " 27 GB", | 20260506 03:13:21.401 | INFO | "malloc" : " 14 GB", | 20260506 03:13:21.401 | INFO | "pinned" : " 29 GB" | 20260506 03:13:21.401 | INFO | } | 20260506 03:13:21.401 | INFO | Perf report for the last 5.14006 seconds: | 20260506 03:13:21.401 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:13:21.401 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:21.401 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:21.401 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:21.401 | INFO | ANALYSIS : 61.5%, 3.0, 96.0ms, 0 | 20260506 03:13:21.401 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:21.401 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:21.401 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 03:13:21.401 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:21.401 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:21.401 | INFO | Uploaded 36GiB to GPU in 2.07692s (17.3333GiB/s) | 20260506 03:13:21.401 | INFO | Downloaded 34GiB to GPU in 2.02657s (16.7771GiB/s) | 20260506 03:13:21.526 | INFO | Starting a round of Baz IO | 20260506 03:13:24.646 | INFO | Populating RTMetrics report | 20260506 03:13:24.647 | INFO | RTMetrics report populated | 20260506 03:13:24.647 | INFO | Opening tmp RTMetrics json file | 20260506 03:13:24.648 | INFO | Serializing RTMetrics to json | 20260506 03:13:24.650 | INFO | Serialization done, swapping tmp file now... | 20260506 03:13:24.650 | INFO | RTMetrics done | 20260506 03:13:24.728 | INFO | Finished a round of Baz IO: Wrote 1.767 GiB in 3.202 seconds | 20260506 03:13:24.728 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.23% / 0.00% / 78.77% | 20260506 03:13:24.728 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0629981% / 0% / 5.96046e-05% | 20260506 03:13:24.728 | INFO | Baz IO overall: | 20260506 03:13:24.728 | INFO | BytesWritten : 4736034718206 | 20260506 03:13:24.728 | INFO | NumEvents : 2221492270800 | 20260506 03:13:26.545 | INFO | Analyzing chunk frames = [8499200,8499712) | 20260506 03:13:26.545 | INFO | MemInfo: { | 20260506 03:13:26.545 | INFO | "gpu" : " 17 GB", | 20260506 03:13:26.545 | INFO | "huge" : "", | 20260506 03:13:26.545 | INFO | "huge_shared" : " 27 GB", | 20260506 03:13:26.545 | INFO | "malloc" : " 14 GB", | 20260506 03:13:26.545 | INFO | "pinned" : " 29 GB" | 20260506 03:13:26.545 | INFO | } | 20260506 03:13:26.545 | INFO | Perf report for the last 5.14469 seconds: | 20260506 03:13:26.545 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:13:26.545 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:26.545 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:26.545 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:26.545 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260506 03:13:26.545 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:26.546 | INFO | RT_METRICS : 42.6%, 1.0, 22.4ms, 0 | 20260506 03:13:26.546 | INFO | PRE_HQ : 59.1%, 1.7, 53.7ms, 0 | 20260506 03:13:26.546 | INFO | BAZWRITER : 61.3%, 3.5, 111.4ms, 0 | 20260506 03:13:26.546 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 03:13:26.546 | INFO | Uploaded 36GiB to GPU in 2.10486s (17.1033GiB/s) | 20260506 03:13:26.546 | INFO | Downloaded 38GiB to GPU in 2.22948s (17.0444GiB/s) | 20260506 03:13:31.682 | INFO | Analyzing chunk frames = [8499712,8500224) | 20260506 03:13:31.682 | INFO | MemInfo: { | 20260506 03:13:31.682 | INFO | "gpu" : " 17 GB", | 20260506 03:13:31.682 | INFO | "huge" : "", | 20260506 03:13:31.682 | INFO | "huge_shared" : " 27 GB", | 20260506 03:13:31.682 | INFO | "malloc" : " 14 GB", | 20260506 03:13:31.682 | INFO | "pinned" : " 29 GB" | 20260506 03:13:31.682 | INFO | } | 20260506 03:13:31.682 | INFO | Perf report for the last 5.13698 seconds: | 20260506 03:13:31.682 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:13:31.682 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:31.682 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:31.682 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:31.682 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260506 03:13:31.682 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:31.683 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:31.683 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 03:13:31.683 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:31.683 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:31.683 | INFO | Uploaded 36GiB to GPU in 2.03825s (17.6622GiB/s) | 20260506 03:13:31.683 | INFO | Downloaded 34GiB to GPU in 1.9909s (17.0777GiB/s) | 20260506 03:13:36.816 | INFO | Analyzing chunk frames = [8500224,8500736) | 20260506 03:13:36.816 | INFO | MemInfo: { | 20260506 03:13:36.816 | INFO | "gpu" : " 17 GB", | 20260506 03:13:36.816 | INFO | "huge" : "", | 20260506 03:13:36.816 | INFO | "huge_shared" : " 27 GB", | 20260506 03:13:36.816 | INFO | "malloc" : " 14 GB", | 20260506 03:13:36.816 | INFO | "pinned" : " 29 GB" | 20260506 03:13:36.816 | INFO | } | 20260506 03:13:36.816 | INFO | Perf report for the last 5.13409 seconds: | 20260506 03:13:36.817 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:13:36.817 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:36.817 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:36.817 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:36.817 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 03:13:36.817 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:36.817 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:36.817 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:13:36.817 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:36.817 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:36.817 | INFO | Uploaded 36GiB to GPU in 2.10894s (17.0702GiB/s) | 20260506 03:13:36.817 | INFO | Downloaded 34GiB to GPU in 2.04295s (16.6426GiB/s) | 20260506 03:13:41.960 | INFO | Analyzing chunk frames = [8500736,8501248) | 20260506 03:13:41.960 | INFO | MemInfo: { | 20260506 03:13:41.960 | INFO | "gpu" : " 17 GB", | 20260506 03:13:41.960 | INFO | "huge" : "", | 20260506 03:13:41.960 | INFO | "huge_shared" : " 27 GB", | 20260506 03:13:41.960 | INFO | "malloc" : " 14 GB", | 20260506 03:13:41.960 | INFO | "pinned" : " 29 GB" | 20260506 03:13:41.961 | INFO | } | 20260506 03:13:41.961 | INFO | Perf report for the last 5.14421 seconds: | 20260506 03:13:41.961 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:13:41.961 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:41.961 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:41.961 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:41.961 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260506 03:13:41.961 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:41.961 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:41.961 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 03:13:41.961 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:41.961 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:41.961 | INFO | Uploaded 36GiB to GPU in 2.09305s (17.1998GiB/s) | 20260506 03:13:41.961 | INFO | Downloaded 34GiB to GPU in 2.03293s (16.7247GiB/s) | 20260506 03:13:47.107 | INFO | Analyzing chunk frames = [8501248,8501760) | 20260506 03:13:47.107 | INFO | MemInfo: { | 20260506 03:13:47.107 | INFO | "gpu" : " 17 GB", | 20260506 03:13:47.107 | INFO | "huge" : "", | 20260506 03:13:47.107 | INFO | "huge_shared" : " 27 GB", | 20260506 03:13:47.107 | INFO | "malloc" : " 14 GB", | 20260506 03:13:47.107 | INFO | "pinned" : " 29 GB" | 20260506 03:13:47.107 | INFO | } | 20260506 03:13:47.107 | INFO | Perf report for the last 5.14605 seconds: | 20260506 03:13:47.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:13:47.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:47.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:47.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:47.107 | INFO | ANALYSIS : 60.5%, 3.0, 94.6ms, 0 | 20260506 03:13:47.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:47.107 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:47.107 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 03:13:47.107 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:47.107 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:47.107 | INFO | Uploaded 36GiB to GPU in 2.06964s (17.3943GiB/s) | 20260506 03:13:47.107 | INFO | Downloaded 34GiB to GPU in 2.03256s (16.7277GiB/s) | 20260506 03:13:52.247 | INFO | Analyzing chunk frames = [8501760,8502272) | 20260506 03:13:52.247 | INFO | MemInfo: { | 20260506 03:13:52.247 | INFO | "gpu" : " 17 GB", | 20260506 03:13:52.247 | INFO | "huge" : "", | 20260506 03:13:52.247 | INFO | "huge_shared" : " 27 GB", | 20260506 03:13:52.247 | INFO | "malloc" : " 14 GB", | 20260506 03:13:52.247 | INFO | "pinned" : " 29 GB" | 20260506 03:13:52.247 | INFO | } | 20260506 03:13:52.247 | INFO | Perf report for the last 5.1401 seconds: | 20260506 03:13:52.247 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:13:52.247 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:52.247 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:52.247 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:52.247 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260506 03:13:52.247 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:52.247 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:52.247 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:13:52.247 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:52.247 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:52.247 | INFO | Uploaded 36GiB to GPU in 2.06135s (17.4643GiB/s) | 20260506 03:13:52.247 | INFO | Downloaded 34GiB to GPU in 2.03283s (16.7254GiB/s) | 20260506 03:13:57.382 | INFO | Analyzing chunk frames = [8502272,8502784) | 20260506 03:13:57.382 | INFO | MemInfo: { | 20260506 03:13:57.382 | INFO | "gpu" : " 17 GB", | 20260506 03:13:57.382 | INFO | "huge" : "", | 20260506 03:13:57.382 | INFO | "huge_shared" : " 27 GB", | 20260506 03:13:57.382 | INFO | "malloc" : " 14 GB", | 20260506 03:13:57.382 | INFO | "pinned" : " 29 GB" | 20260506 03:13:57.382 | INFO | } | 20260506 03:13:57.382 | INFO | Perf report for the last 5.13493 seconds: | 20260506 03:13:57.382 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:13:57.382 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:57.382 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:57.382 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:57.382 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 03:13:57.382 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:57.382 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:57.382 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260506 03:13:57.382 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:57.382 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:13:57.382 | INFO | Uploaded 36GiB to GPU in 2.06406s (17.4413GiB/s) | 20260506 03:13:57.382 | INFO | Downloaded 34GiB to GPU in 2.06032s (16.5023GiB/s) | 20260506 03:14:02.516 | INFO | Analyzing chunk frames = [8502784,8503296) | 20260506 03:14:02.516 | INFO | MemInfo: { | 20260506 03:14:02.516 | INFO | "gpu" : " 17 GB", | 20260506 03:14:02.516 | INFO | "huge" : "", | 20260506 03:14:02.516 | INFO | "huge_shared" : " 27 GB", | 20260506 03:14:02.516 | INFO | "malloc" : " 14 GB", | 20260506 03:14:02.516 | INFO | "pinned" : " 29 GB" | 20260506 03:14:02.516 | INFO | } | 20260506 03:14:02.516 | INFO | Perf report for the last 5.13425 seconds: | 20260506 03:14:02.516 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:14:02.516 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:02.516 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:02.516 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:02.516 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260506 03:14:02.516 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:02.516 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:02.516 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 03:14:02.516 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:02.516 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:02.516 | INFO | Uploaded 36GiB to GPU in 2.04198s (17.63GiB/s) | 20260506 03:14:02.516 | INFO | Downloaded 34GiB to GPU in 2.01716s (16.8553GiB/s) | 20260506 03:14:02.517 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 03:14:02.517 | INFO | Total mem usage: 15.4 GiB | 20260506 03:14:02.517 | INFO | All Time High Water Marks sum: 16.3 GiB (Savings: 5.2%) | 20260506 03:14:02.517 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -10%) | 20260506 03:14:02.517 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 03:14:02.517 | INFO | High Water Breakdown: | 20260506 03:14:02.517 | INFO | 60.9% : 8.54 GiB ( 88.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 03:14:02.517 | INFO | 39% : 5.47 GiB ( 82.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 03:14:02.517 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 03:14:02.517 | INFO | ------------------------------------------ | 20260506 03:14:02.517 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:14:02.517 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 03:14:02.517 | INFO | Total mem usage: 26.5 GiB | 20260506 03:14:02.517 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 03:14:02.517 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 03:14:02.517 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 03:14:02.517 | INFO | High Water Breakdown: | 20260506 03:14:02.517 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 03:14:02.517 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 03:14:02.517 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 03:14:02.517 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 03:14:02.517 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 03:14:02.517 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 03:14:02.517 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 03:14:02.517 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 03:14:02.517 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 03:14:02.517 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 03:14:02.517 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 03:14:02.517 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 03:14:02.517 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 03:14:02.517 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 03:14:02.517 | INFO | ------------------------------------------ | 20260506 03:14:02.517 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:14:02.517 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 03:14:02.517 | INFO | Total mem usage: 25.6 GiB | 20260506 03:14:02.517 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 03:14:02.517 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 03:14:02.517 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 03:14:02.517 | INFO | High Water Breakdown: | 20260506 03:14:02.517 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 03:14:02.517 | INFO | ------------------------------------------ | 20260506 03:14:02.517 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:14:02.517 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 03:14:02.517 | INFO | Total mem usage: 15.7 GiB | 20260506 03:14:02.517 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 03:14:02.517 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 03:14:02.517 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 03:14:02.517 | INFO | High Water Breakdown: | 20260506 03:14:02.517 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 03:14:02.517 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 03:14:02.517 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 03:14:02.517 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 03:14:02.517 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 03:14:02.517 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 03:14:02.517 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 03:14:02.517 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 03:14:02.517 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 03:14:02.517 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 03:14:02.517 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 03:14:02.517 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 03:14:02.517 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 03:14:02.517 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 03:14:02.517 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 03:14:02.517 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 03:14:02.517 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 03:14:02.517 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 03:14:02.517 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 03:14:02.517 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 03:14:02.517 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 03:14:02.517 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 03:14:02.518 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 03:14:02.518 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 03:14:02.518 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 03:14:02.518 | INFO | ------------------------------------------ | 20260506 03:14:02.518 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:14:02.518 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 03:14:02.518 | INFO | PerfReport: Gathered 1536 samples. | 20260506 03:14:02.518 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 03:14:02.518 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.37e+01, 1.33e+01 / 7.85e-01 | 20260506 03:14:02.518 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.05e-01, 1.03e-02 / 2.99e-03 | 20260506 03:14:02.518 | INFO | PerfReport: 11.86%, Baseline -- 3.72e+00 / 4.49e+00, 5.87e-01 / 3.16e-01 | 20260506 03:14:02.518 | INFO | PerfReport: 3.31%, Histogram -- 1.04e+00 / 1.52e+00, 1.96e-01 / 1.02e-01 | 20260506 03:14:02.518 | INFO | PerfReport: 15.95%, DME -- 4.00e+01 / 4.39e+01, 3.84e+00 / 8.06e-01 | 20260506 03:14:02.518 | INFO | PerfReport: 57.63%, FrameLabeling -- 1.80e+01 / 1.94e+01, 1.61e+00 / 1.20e+00 | 20260506 03:14:02.518 | INFO | PerfReport: 6.95%, PulseAccumulating -- 2.18e+00 / 2.69e+00, 4.10e-01 / 2.63e-01 | 20260506 03:14:02.518 | INFO | PerfReport: 2.98%, Metrics -- 9.32e-01 / 1.19e+00, 2.54e-01 / 1.08e-01 | 20260506 03:14:02.518 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.22e-01 / 3.77e-01, 5.52e-02 / 3.63e-02 | 20260506 03:14:02.518 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 03:14:02.518 | INFO | PerfReport: Gathered 1536 samples. | 20260506 03:14:02.518 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 03:14:02.518 | INFO | PerfReport: Full Execution -- 9.75e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260506 03:14:02.518 | INFO | PerfReport: 22.22%, Upload -- 2.17e+01 / 0.00e+00, 7.12e+00 / 0.00e+00 | 20260506 03:14:02.518 | INFO | PerfReport: 21.96%, Download -- 2.14e+01 / 0.00e+00, 7.59e+00 / 0.00e+00 | 20260506 03:14:02.518 | INFO | PerfReport: 32.14%, Compute -- 3.13e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 03:14:02.518 | INFO | PerfReport: 1.93%, LockUpload -- 1.88e+00 / 0.00e+00, 7.74e+00 / 0.00e+00 | 20260506 03:14:02.518 | INFO | PerfReport: 1.30%, LockDownload -- 1.26e+00 / 0.00e+00, 3.01e+00 / 0.00e+00 | 20260506 03:14:02.518 | INFO | PerfReport: 20.43%, LockCompute -- 1.99e+01 / 0.00e+00, 1.76e+01 / 0.00e+00 | 20260506 03:14:02.518 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.63e-02 / 0.00e+00 | 20260506 03:14:02.642 | INFO | Starting a round of Baz IO | 20260506 03:14:05.774 | INFO | Populating RTMetrics report | 20260506 03:14:05.775 | INFO | RTMetrics report populated | 20260506 03:14:05.775 | INFO | Opening tmp RTMetrics json file | 20260506 03:14:05.776 | INFO | Serializing RTMetrics to json | 20260506 03:14:05.777 | INFO | Serialization done, swapping tmp file now... | 20260506 03:14:05.777 | INFO | RTMetrics done | 20260506 03:14:05.846 | INFO | Finished a round of Baz IO: Wrote 1.767 GiB in 3.203 seconds | 20260506 03:14:05.846 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.23% / 0.00% / 78.77% | 20260506 03:14:05.846 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0626922% / 0% / 7.15256e-05% | 20260506 03:14:05.846 | INFO | Baz IO overall: | 20260506 03:14:05.846 | INFO | BytesWritten : 4737931821791 | 20260506 03:14:05.846 | INFO | NumEvents : 2222352218734 | 20260506 03:14:07.655 | INFO | Analyzing chunk frames = [8503296,8503808) | 20260506 03:14:07.655 | INFO | MemInfo: { | 20260506 03:14:07.655 | INFO | "gpu" : " 17 GB", | 20260506 03:14:07.655 | INFO | "huge" : "", | 20260506 03:14:07.655 | INFO | "huge_shared" : " 27 GB", | 20260506 03:14:07.655 | INFO | "malloc" : " 14 GB", | 20260506 03:14:07.655 | INFO | "pinned" : " 29 GB" | 20260506 03:14:07.655 | INFO | } | 20260506 03:14:07.655 | INFO | Perf report for the last 5.13878 seconds: | 20260506 03:14:07.655 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:14:07.655 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:07.655 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:07.655 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:07.655 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260506 03:14:07.655 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:07.655 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260506 03:14:07.655 | INFO | PRE_HQ : 59.5%, 1.7, 53.5ms, 0 | 20260506 03:14:07.655 | INFO | BAZWRITER : 61.6%, 3.5, 109.7ms, 0 | 20260506 03:14:07.655 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 03:14:07.655 | INFO | Uploaded 36GiB to GPU in 2.18521s (16.4744GiB/s) | 20260506 03:14:07.655 | INFO | Downloaded 38GiB to GPU in 2.24371s (16.9362GiB/s) | 20260506 03:14:12.797 | INFO | Analyzing chunk frames = [8503808,8504320) | 20260506 03:14:12.797 | INFO | MemInfo: { | 20260506 03:14:12.797 | INFO | "gpu" : " 17 GB", | 20260506 03:14:12.797 | INFO | "huge" : "", | 20260506 03:14:12.797 | INFO | "huge_shared" : " 27 GB", | 20260506 03:14:12.797 | INFO | "malloc" : " 14 GB", | 20260506 03:14:12.797 | INFO | "pinned" : " 29 GB" | 20260506 03:14:12.797 | INFO | } | 20260506 03:14:12.797 | INFO | Perf report for the last 5.14258 seconds: | 20260506 03:14:12.797 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:14:12.797 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:12.797 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:12.798 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:12.798 | INFO | ANALYSIS : 60.5%, 3.0, 94.6ms, 0 | 20260506 03:14:12.798 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:12.798 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:12.798 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 03:14:12.798 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:12.798 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:12.798 | INFO | Uploaded 36GiB to GPU in 1.94103s (18.5469GiB/s) | 20260506 03:14:12.798 | INFO | Downloaded 34GiB to GPU in 1.97903s (17.1801GiB/s) | 20260506 03:14:17.945 | INFO | Analyzing chunk frames = [8504320,8504832) | 20260506 03:14:17.945 | INFO | MemInfo: { | 20260506 03:14:17.945 | INFO | "gpu" : " 17 GB", | 20260506 03:14:17.945 | INFO | "huge" : "", | 20260506 03:14:17.945 | INFO | "huge_shared" : " 27 GB", | 20260506 03:14:17.945 | INFO | "malloc" : " 14 GB", | 20260506 03:14:17.945 | INFO | "pinned" : " 29 GB" | 20260506 03:14:17.945 | INFO | } | 20260506 03:14:17.945 | INFO | Perf report for the last 5.14779 seconds: | 20260506 03:14:17.945 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:14:17.945 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:17.945 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:17.945 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:17.945 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 03:14:17.945 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:17.945 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:17.945 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 03:14:17.945 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:17.945 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:17.945 | INFO | Uploaded 36GiB to GPU in 2.11475s (17.0233GiB/s) | 20260506 03:14:17.946 | INFO | Downloaded 34GiB to GPU in 2.04718s (16.6082GiB/s) | 20260506 03:14:23.076 | INFO | Analyzing chunk frames = [8504832,8505344) | 20260506 03:14:23.076 | INFO | MemInfo: { | 20260506 03:14:23.076 | INFO | "gpu" : " 17 GB", | 20260506 03:14:23.076 | INFO | "huge" : "", | 20260506 03:14:23.076 | INFO | "huge_shared" : " 27 GB", | 20260506 03:14:23.076 | INFO | "malloc" : " 14 GB", | 20260506 03:14:23.076 | INFO | "pinned" : " 29 GB" | 20260506 03:14:23.076 | INFO | } | 20260506 03:14:23.076 | INFO | Perf report for the last 5.13082 seconds: | 20260506 03:14:23.076 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:14:23.076 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:23.076 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:23.076 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:23.076 | INFO | ANALYSIS : 61.0%, 3.0, 94.6ms, 0 | 20260506 03:14:23.076 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:23.076 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:23.076 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 03:14:23.076 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:23.076 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:23.076 | INFO | Uploaded 36GiB to GPU in 2.07578s (17.3429GiB/s) | 20260506 03:14:23.076 | INFO | Downloaded 34GiB to GPU in 2.00323s (16.9726GiB/s) | 20260506 03:14:28.216 | INFO | Analyzing chunk frames = [8505344,8505856) | 20260506 03:14:28.216 | INFO | MemInfo: { | 20260506 03:14:28.216 | INFO | "gpu" : " 17 GB", | 20260506 03:14:28.216 | INFO | "huge" : "", | 20260506 03:14:28.216 | INFO | "huge_shared" : " 27 GB", | 20260506 03:14:28.216 | INFO | "malloc" : " 14 GB", | 20260506 03:14:28.216 | INFO | "pinned" : " 29 GB" | 20260506 03:14:28.216 | INFO | } | 20260506 03:14:28.216 | INFO | Perf report for the last 5.13969 seconds: | 20260506 03:14:28.216 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:14:28.216 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:28.216 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:28.216 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:28.216 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 03:14:28.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:28.216 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:28.216 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 03:14:28.216 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:28.216 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:28.216 | INFO | Uploaded 36GiB to GPU in 2.0777s (17.3268GiB/s) | 20260506 03:14:28.216 | INFO | Downloaded 34GiB to GPU in 2.03482s (16.7091GiB/s) | 20260506 03:14:33.355 | INFO | Analyzing chunk frames = [8505856,8506368) | 20260506 03:14:33.355 | INFO | MemInfo: { | 20260506 03:14:33.355 | INFO | "gpu" : " 17 GB", | 20260506 03:14:33.355 | INFO | "huge" : "", | 20260506 03:14:33.355 | INFO | "huge_shared" : " 27 GB", | 20260506 03:14:33.355 | INFO | "malloc" : " 14 GB", | 20260506 03:14:33.355 | INFO | "pinned" : " 29 GB" | 20260506 03:14:33.355 | INFO | } | 20260506 03:14:33.355 | INFO | Perf report for the last 5.13934 seconds: | 20260506 03:14:33.355 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:14:33.355 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:33.355 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:33.355 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:33.355 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 03:14:33.355 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:33.355 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:33.355 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:14:33.355 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:33.355 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:33.355 | INFO | Uploaded 36GiB to GPU in 2.08839s (17.2381GiB/s) | 20260506 03:14:33.356 | INFO | Downloaded 34GiB to GPU in 2.03839s (16.6798GiB/s) | 20260506 03:14:38.494 | INFO | Analyzing chunk frames = [8506368,8506880) | 20260506 03:14:38.494 | INFO | MemInfo: { | 20260506 03:14:38.494 | INFO | "gpu" : " 17 GB", | 20260506 03:14:38.494 | INFO | "huge" : "", | 20260506 03:14:38.494 | INFO | "huge_shared" : " 27 GB", | 20260506 03:14:38.494 | INFO | "malloc" : " 14 GB", | 20260506 03:14:38.494 | INFO | "pinned" : " 29 GB" | 20260506 03:14:38.494 | INFO | } | 20260506 03:14:38.494 | INFO | Perf report for the last 5.1393 seconds: | 20260506 03:14:38.494 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:14:38.494 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:38.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:38.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:38.495 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 03:14:38.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:38.495 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:38.495 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 03:14:38.495 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:38.495 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:38.495 | INFO | Uploaded 36GiB to GPU in 2.08298s (17.283GiB/s) | 20260506 03:14:38.495 | INFO | Downloaded 34GiB to GPU in 2.06737s (16.446GiB/s) | 20260506 03:14:43.633 | INFO | Analyzing chunk frames = [8506880,8507392) | 20260506 03:14:43.633 | INFO | MemInfo: { | 20260506 03:14:43.633 | INFO | "gpu" : " 17 GB", | 20260506 03:14:43.633 | INFO | "huge" : "", | 20260506 03:14:43.633 | INFO | "huge_shared" : " 27 GB", | 20260506 03:14:43.633 | INFO | "malloc" : " 14 GB", | 20260506 03:14:43.633 | INFO | "pinned" : " 29 GB" | 20260506 03:14:43.633 | INFO | } | 20260506 03:14:43.633 | INFO | Perf report for the last 5.13891 seconds: | 20260506 03:14:43.633 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:14:43.633 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:43.633 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:43.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:43.633 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260506 03:14:43.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:43.633 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:43.633 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 03:14:43.633 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:43.634 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:43.634 | INFO | Uploaded 36GiB to GPU in 2.035s (17.6904GiB/s) | 20260506 03:14:43.634 | INFO | Downloaded 34GiB to GPU in 2.02492s (16.7908GiB/s) | 20260506 03:14:43.759 | INFO | Starting a round of Baz IO | 20260506 03:14:46.881 | INFO | Populating RTMetrics report | 20260506 03:14:46.882 | INFO | RTMetrics report populated | 20260506 03:14:46.882 | INFO | Opening tmp RTMetrics json file | 20260506 03:14:46.883 | INFO | Serializing RTMetrics to json | 20260506 03:14:46.885 | INFO | Serialization done, swapping tmp file now... | 20260506 03:14:46.885 | INFO | RTMetrics done | 20260506 03:14:46.958 | INFO | Finished a round of Baz IO: Wrote 1.766 GiB in 3.199 seconds | 20260506 03:14:46.958 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.23% / 0.00% / 78.77% | 20260506 03:14:46.958 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0626405% / 3.97364e-06% / 8.74201e-05% | 20260506 03:14:46.958 | INFO | Baz IO overall: | 20260506 03:14:46.958 | INFO | BytesWritten : 4739828564075 | 20260506 03:14:46.958 | INFO | NumEvents : 2223211945472 | 20260506 03:14:48.770 | INFO | Analyzing chunk frames = [8507392,8507904) | 20260506 03:14:48.770 | INFO | MemInfo: { | 20260506 03:14:48.770 | INFO | "gpu" : " 17 GB", | 20260506 03:14:48.770 | INFO | "huge" : "", | 20260506 03:14:48.770 | INFO | "huge_shared" : " 27 GB", | 20260506 03:14:48.770 | INFO | "malloc" : " 14 GB", | 20260506 03:14:48.770 | INFO | "pinned" : " 29 GB" | 20260506 03:14:48.770 | INFO | } | 20260506 03:14:48.770 | INFO | Perf report for the last 5.13674 seconds: | 20260506 03:14:48.770 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:14:48.770 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:48.770 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:48.770 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:48.770 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260506 03:14:48.770 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:48.770 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260506 03:14:48.770 | INFO | PRE_HQ : 59.1%, 1.7, 53.5ms, 0 | 20260506 03:14:48.770 | INFO | BAZWRITER : 61.4%, 3.6, 112.7ms, 0 | 20260506 03:14:48.770 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 03:14:48.770 | INFO | Uploaded 36GiB to GPU in 2.10358s (17.1136GiB/s) | 20260506 03:14:48.770 | INFO | Downloaded 38GiB to GPU in 2.21811s (17.1317GiB/s) | 20260506 03:14:53.910 | INFO | Analyzing chunk frames = [8507904,8508416) | 20260506 03:14:53.910 | INFO | MemInfo: { | 20260506 03:14:53.910 | INFO | "gpu" : " 17 GB", | 20260506 03:14:53.910 | INFO | "huge" : "", | 20260506 03:14:53.910 | INFO | "huge_shared" : " 27 GB", | 20260506 03:14:53.910 | INFO | "malloc" : " 14 GB", | 20260506 03:14:53.910 | INFO | "pinned" : " 29 GB" | 20260506 03:14:53.910 | INFO | } | 20260506 03:14:53.910 | INFO | Perf report for the last 5.14031 seconds: | 20260506 03:14:53.910 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:14:53.910 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:53.910 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:53.910 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:53.910 | INFO | ANALYSIS : 60.9%, 3.0, 95.1ms, 0 | 20260506 03:14:53.910 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:53.911 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:53.911 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 03:14:53.911 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:53.911 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:53.911 | INFO | Uploaded 36GiB to GPU in 1.9767s (18.2121GiB/s) | 20260506 03:14:53.911 | INFO | Downloaded 34GiB to GPU in 1.99539s (17.0393GiB/s) | 20260506 03:14:59.054 | INFO | Analyzing chunk frames = [8508416,8508928) | 20260506 03:14:59.054 | INFO | MemInfo: { | 20260506 03:14:59.054 | INFO | "gpu" : " 17 GB", | 20260506 03:14:59.054 | INFO | "huge" : "", | 20260506 03:14:59.054 | INFO | "huge_shared" : " 27 GB", | 20260506 03:14:59.054 | INFO | "malloc" : " 14 GB", | 20260506 03:14:59.054 | INFO | "pinned" : " 29 GB" | 20260506 03:14:59.054 | INFO | } | 20260506 03:14:59.054 | INFO | Perf report for the last 5.1433 seconds: | 20260506 03:14:59.054 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:14:59.054 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:59.054 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:59.054 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:59.054 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260506 03:14:59.054 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:59.054 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:59.054 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:14:59.054 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:59.055 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:14:59.055 | INFO | Uploaded 36GiB to GPU in 2.09323s (17.1983GiB/s) | 20260506 03:14:59.055 | INFO | Downloaded 34GiB to GPU in 2.03881s (16.6764GiB/s) | 20260506 03:15:04.197 | INFO | Analyzing chunk frames = [8508928,8509440) | 20260506 03:15:04.197 | INFO | MemInfo: { | 20260506 03:15:04.197 | INFO | "gpu" : " 17 GB", | 20260506 03:15:04.197 | INFO | "huge" : "", | 20260506 03:15:04.197 | INFO | "huge_shared" : " 27 GB", | 20260506 03:15:04.197 | INFO | "malloc" : " 14 GB", | 20260506 03:15:04.197 | INFO | "pinned" : " 29 GB" | 20260506 03:15:04.197 | INFO | } | 20260506 03:15:04.197 | INFO | Perf report for the last 5.14355 seconds: | 20260506 03:15:04.197 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:15:04.197 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:04.197 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:04.197 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:04.197 | INFO | ANALYSIS : 60.9%, 3.0, 94.6ms, 0 | 20260506 03:15:04.197 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:04.197 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:04.197 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 03:15:04.197 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:04.197 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:04.198 | INFO | Uploaded 36GiB to GPU in 2.0881s (17.2406GiB/s) | 20260506 03:15:04.198 | INFO | Downloaded 34GiB to GPU in 2.01556s (16.8687GiB/s) | 20260506 03:15:09.329 | INFO | Analyzing chunk frames = [8509440,8509952) | 20260506 03:15:09.329 | INFO | MemInfo: { | 20260506 03:15:09.329 | INFO | "gpu" : " 17 GB", | 20260506 03:15:09.329 | INFO | "huge" : "", | 20260506 03:15:09.329 | INFO | "huge_shared" : " 27 GB", | 20260506 03:15:09.329 | INFO | "malloc" : " 14 GB", | 20260506 03:15:09.329 | INFO | "pinned" : " 29 GB" | 20260506 03:15:09.329 | INFO | } | 20260506 03:15:09.329 | INFO | Perf report for the last 5.13185 seconds: | 20260506 03:15:09.329 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:15:09.329 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:09.329 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:09.329 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:09.329 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260506 03:15:09.329 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:09.329 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:09.329 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 03:15:09.329 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:09.329 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:09.329 | INFO | Uploaded 36GiB to GPU in 2.11517s (17.0199GiB/s) | 20260506 03:15:09.329 | INFO | Downloaded 34GiB to GPU in 2.04177s (16.6522GiB/s) | 20260506 03:15:14.464 | INFO | Analyzing chunk frames = [8509952,8510464) | 20260506 03:15:14.464 | INFO | MemInfo: { | 20260506 03:15:14.464 | INFO | "gpu" : " 17 GB", | 20260506 03:15:14.464 | INFO | "huge" : "", | 20260506 03:15:14.464 | INFO | "huge_shared" : " 27 GB", | 20260506 03:15:14.464 | INFO | "malloc" : " 14 GB", | 20260506 03:15:14.464 | INFO | "pinned" : " 29 GB" | 20260506 03:15:14.464 | INFO | } | 20260506 03:15:14.464 | INFO | Perf report for the last 5.13471 seconds: | 20260506 03:15:14.464 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:15:14.464 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:14.464 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:14.464 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:14.464 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 03:15:14.464 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:14.464 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:14.464 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 03:15:14.464 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:14.464 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:14.464 | INFO | Uploaded 36GiB to GPU in 2.10307s (17.1179GiB/s) | 20260506 03:15:14.464 | INFO | Downloaded 34GiB to GPU in 2.04088s (16.6595GiB/s) | 20260506 03:15:19.614 | INFO | Analyzing chunk frames = [8510464,8510976) | 20260506 03:15:19.614 | INFO | MemInfo: { | 20260506 03:15:19.614 | INFO | "gpu" : " 17 GB", | 20260506 03:15:19.614 | INFO | "huge" : "", | 20260506 03:15:19.614 | INFO | "huge_shared" : " 27 GB", | 20260506 03:15:19.614 | INFO | "malloc" : " 14 GB", | 20260506 03:15:19.614 | INFO | "pinned" : " 29 GB" | 20260506 03:15:19.614 | INFO | } | 20260506 03:15:19.614 | INFO | Perf report for the last 5.15041 seconds: | 20260506 03:15:19.614 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:15:19.614 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:19.614 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:19.614 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:19.614 | INFO | ANALYSIS : 61.8%, 3.0, 96.7ms, 0 | 20260506 03:15:19.614 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:19.614 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:19.614 | INFO | PRE_HQ : 27.0%, 1.0, 14.1ms, 0 | 20260506 03:15:19.614 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:19.614 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:19.615 | INFO | Uploaded 36GiB to GPU in 2.03972s (17.6495GiB/s) | 20260506 03:15:19.615 | INFO | Downloaded 34GiB to GPU in 2.06219s (16.4874GiB/s) | 20260506 03:15:24.748 | INFO | Analyzing chunk frames = [8510976,8511488) | 20260506 03:15:24.748 | INFO | MemInfo: { | 20260506 03:15:24.748 | INFO | "gpu" : " 17 GB", | 20260506 03:15:24.748 | INFO | "huge" : "", | 20260506 03:15:24.748 | INFO | "huge_shared" : " 27 GB", | 20260506 03:15:24.748 | INFO | "malloc" : " 14 GB", | 20260506 03:15:24.748 | INFO | "pinned" : " 29 GB" | 20260506 03:15:24.748 | INFO | } | 20260506 03:15:24.748 | INFO | Perf report for the last 5.13368 seconds: | 20260506 03:15:24.748 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:15:24.748 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:24.748 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:24.748 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:24.748 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 03:15:24.748 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:24.748 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:24.748 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 03:15:24.748 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:24.748 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:24.748 | INFO | Uploaded 36GiB to GPU in 2.04954s (17.5649GiB/s) | 20260506 03:15:24.748 | INFO | Downloaded 34GiB to GPU in 2.03017s (16.7474GiB/s) | 20260506 03:15:24.749 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 03:15:24.749 | INFO | Total mem usage: 15.4 GiB | 20260506 03:15:24.749 | INFO | All Time High Water Marks sum: 16.3 GiB (Savings: 5.2%) | 20260506 03:15:24.749 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -9.32%) | 20260506 03:15:24.749 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 03:15:24.749 | INFO | High Water Breakdown: | 20260506 03:15:24.749 | INFO | 61% : 8.61 GiB ( 89.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 03:15:24.749 | INFO | 38.9% : 5.49 GiB ( 82.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 03:15:24.749 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 03:15:24.749 | INFO | ------------------------------------------ | 20260506 03:15:24.749 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:15:24.749 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 03:15:24.749 | INFO | Total mem usage: 26.5 GiB | 20260506 03:15:24.749 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 03:15:24.749 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 03:15:24.749 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 03:15:24.749 | INFO | High Water Breakdown: | 20260506 03:15:24.749 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 03:15:24.749 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 03:15:24.749 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 03:15:24.749 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 03:15:24.749 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 03:15:24.749 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 03:15:24.749 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 03:15:24.749 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 03:15:24.749 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 03:15:24.749 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 03:15:24.749 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 03:15:24.749 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 03:15:24.749 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 03:15:24.749 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 03:15:24.749 | INFO | ------------------------------------------ | 20260506 03:15:24.749 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:15:24.749 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 03:15:24.749 | INFO | Total mem usage: 25.6 GiB | 20260506 03:15:24.749 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 03:15:24.749 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 03:15:24.749 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 03:15:24.749 | INFO | High Water Breakdown: | 20260506 03:15:24.749 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 03:15:24.749 | INFO | ------------------------------------------ | 20260506 03:15:24.749 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:15:24.749 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 03:15:24.749 | INFO | Total mem usage: 15.7 GiB | 20260506 03:15:24.749 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 03:15:24.749 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 03:15:24.749 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 03:15:24.749 | INFO | High Water Breakdown: | 20260506 03:15:24.749 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 03:15:24.749 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 03:15:24.749 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 03:15:24.749 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 03:15:24.749 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 03:15:24.749 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 03:15:24.749 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 03:15:24.749 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 03:15:24.749 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 03:15:24.749 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 03:15:24.749 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 03:15:24.749 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 03:15:24.749 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 03:15:24.749 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 03:15:24.749 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 03:15:24.749 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 03:15:24.749 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 03:15:24.749 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 03:15:24.749 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 03:15:24.749 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 03:15:24.749 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 03:15:24.749 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 03:15:24.749 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 03:15:24.749 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 03:15:24.749 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 03:15:24.749 | INFO | ------------------------------------------ | 20260506 03:15:24.749 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:15:24.750 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 03:15:24.750 | INFO | PerfReport: Gathered 1536 samples. | 20260506 03:15:24.750 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 03:15:24.750 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.37e+01, 1.33e+01 / 8.40e-01 | 20260506 03:15:24.750 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.05e-01, 1.03e-02 / 3.09e-03 | 20260506 03:15:24.750 | INFO | PerfReport: 11.87%, Baseline -- 3.71e+00 / 4.52e+00, 5.79e-01 / 2.50e-01 | 20260506 03:15:24.750 | INFO | PerfReport: 3.30%, Histogram -- 1.03e+00 / 1.52e+00, 1.94e-01 / 9.25e-02 | 20260506 03:15:24.750 | INFO | PerfReport: 16.01%, DME -- 4.00e+01 / 4.40e+01, 3.88e+00 / 1.01e+00 | 20260506 03:15:24.750 | INFO | PerfReport: 57.49%, FrameLabeling -- 1.80e+01 / 1.92e+01, 1.50e+00 / 1.01e+00 | 20260506 03:15:24.750 | INFO | PerfReport: 7.01%, PulseAccumulating -- 2.19e+00 / 2.75e+00, 4.18e-01 / 2.07e-01 | 20260506 03:15:24.750 | INFO | PerfReport: 2.98%, Metrics -- 9.32e-01 / 1.18e+00, 2.48e-01 / 9.94e-02 | 20260506 03:15:24.750 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.21e-01 / 3.65e-01, 5.35e-02 / 2.74e-02 | 20260506 03:15:24.750 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 03:15:24.750 | INFO | PerfReport: Gathered 1536 samples. | 20260506 03:15:24.750 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 03:15:24.750 | INFO | PerfReport: Full Execution -- 9.75e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260506 03:15:24.750 | INFO | PerfReport: 22.15%, Upload -- 2.16e+01 / 0.00e+00, 7.20e+00 / 0.00e+00 | 20260506 03:15:24.750 | INFO | PerfReport: 21.96%, Download -- 2.14e+01 / 0.00e+00, 7.62e+00 / 0.00e+00 | 20260506 03:15:24.750 | INFO | PerfReport: 32.05%, Compute -- 3.12e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260506 03:15:24.750 | INFO | PerfReport: 1.97%, LockUpload -- 1.92e+00 / 0.00e+00, 7.78e+00 / 0.00e+00 | 20260506 03:15:24.750 | INFO | PerfReport: 1.33%, LockDownload -- 1.30e+00 / 0.00e+00, 2.99e+00 / 0.00e+00 | 20260506 03:15:24.750 | INFO | PerfReport: 20.54%, LockCompute -- 2.00e+01 / 0.00e+00, 1.76e+01 / 0.00e+00 | 20260506 03:15:24.750 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.74e-02 / 0.00e+00 | 20260506 03:15:24.872 | INFO | Starting a round of Baz IO | 20260506 03:15:27.994 | INFO | Populating RTMetrics report | 20260506 03:15:27.994 | INFO | RTMetrics report populated | 20260506 03:15:27.994 | INFO | Opening tmp RTMetrics json file | 20260506 03:15:27.996 | INFO | Serializing RTMetrics to json | 20260506 03:15:27.997 | INFO | Serialization done, swapping tmp file now... | 20260506 03:15:27.997 | INFO | RTMetrics done | 20260506 03:15:28.078 | INFO | Finished a round of Baz IO: Wrote 1.766 GiB in 3.206 seconds | 20260506 03:15:28.078 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.23% / 0.00% / 78.77% | 20260506 03:15:28.078 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0627915% / 0% / 0.000111262% | 20260506 03:15:28.078 | INFO | Baz IO overall: | 20260506 03:15:28.078 | INFO | BytesWritten : 4741724304118 | 20260506 03:15:28.078 | INFO | NumEvents : 2224071318429 | 20260506 03:15:29.890 | INFO | Analyzing chunk frames = [8511488,8512000) | 20260506 03:15:29.890 | INFO | MemInfo: { | 20260506 03:15:29.890 | INFO | "gpu" : " 17 GB", | 20260506 03:15:29.890 | INFO | "huge" : "", | 20260506 03:15:29.890 | INFO | "huge_shared" : " 27 GB", | 20260506 03:15:29.890 | INFO | "malloc" : " 14 GB", | 20260506 03:15:29.890 | INFO | "pinned" : " 29 GB" | 20260506 03:15:29.890 | INFO | } | 20260506 03:15:29.890 | INFO | Perf report for the last 5.14229 seconds: | 20260506 03:15:29.890 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:15:29.890 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:29.890 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:29.890 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:29.890 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260506 03:15:29.890 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:29.890 | INFO | RT_METRICS : 42.6%, 1.0, 22.4ms, 0 | 20260506 03:15:29.890 | INFO | PRE_HQ : 58.9%, 1.7, 53.1ms, 0 | 20260506 03:15:29.890 | INFO | BAZWRITER : 61.5%, 3.5, 112.3ms, 0 | 20260506 03:15:29.890 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 03:15:29.891 | INFO | Uploaded 36GiB to GPU in 2.0931s (17.1994GiB/s) | 20260506 03:15:29.891 | INFO | Downloaded 38GiB to GPU in 2.21964s (17.1199GiB/s) | 20260506 03:15:35.025 | INFO | Analyzing chunk frames = [8512000,8512512) | 20260506 03:15:35.025 | INFO | MemInfo: { | 20260506 03:15:35.025 | INFO | "gpu" : " 17 GB", | 20260506 03:15:35.025 | INFO | "huge" : "", | 20260506 03:15:35.025 | INFO | "huge_shared" : " 27 GB", | 20260506 03:15:35.025 | INFO | "malloc" : " 14 GB", | 20260506 03:15:35.025 | INFO | "pinned" : " 29 GB" | 20260506 03:15:35.025 | INFO | } | 20260506 03:15:35.025 | INFO | Perf report for the last 5.13491 seconds: | 20260506 03:15:35.025 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:15:35.025 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:35.025 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:35.025 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:35.025 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 03:15:35.025 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:35.025 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:35.025 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 03:15:35.025 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:35.025 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:35.026 | INFO | Uploaded 36GiB to GPU in 1.93773s (18.5784GiB/s) | 20260506 03:15:35.026 | INFO | Downloaded 34GiB to GPU in 1.97663s (17.201GiB/s) | 20260506 03:15:40.169 | INFO | Analyzing chunk frames = [8512512,8513024) | 20260506 03:15:40.169 | INFO | MemInfo: { | 20260506 03:15:40.169 | INFO | "gpu" : " 17 GB", | 20260506 03:15:40.169 | INFO | "huge" : "", | 20260506 03:15:40.169 | INFO | "huge_shared" : " 27 GB", | 20260506 03:15:40.169 | INFO | "malloc" : " 14 GB", | 20260506 03:15:40.169 | INFO | "pinned" : " 29 GB" | 20260506 03:15:40.169 | INFO | } | 20260506 03:15:40.169 | INFO | Perf report for the last 5.14341 seconds: | 20260506 03:15:40.169 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:15:40.169 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:40.169 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:40.169 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:40.169 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 03:15:40.169 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:40.169 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:40.169 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:15:40.169 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:40.169 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:40.169 | INFO | Uploaded 36GiB to GPU in 2.09666s (17.1702GiB/s) | 20260506 03:15:40.169 | INFO | Downloaded 34GiB to GPU in 2.03737s (16.6882GiB/s) | 20260506 03:15:45.305 | INFO | Analyzing chunk frames = [8513024,8513536) | 20260506 03:15:45.305 | INFO | MemInfo: { | 20260506 03:15:45.305 | INFO | "gpu" : " 17 GB", | 20260506 03:15:45.305 | INFO | "huge" : "", | 20260506 03:15:45.305 | INFO | "huge_shared" : " 27 GB", | 20260506 03:15:45.305 | INFO | "malloc" : " 14 GB", | 20260506 03:15:45.305 | INFO | "pinned" : " 29 GB" | 20260506 03:15:45.305 | INFO | } | 20260506 03:15:45.305 | INFO | Perf report for the last 5.13651 seconds: | 20260506 03:15:45.305 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:15:45.305 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:45.305 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:45.305 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:45.305 | INFO | ANALYSIS : 61.0%, 3.0, 94.7ms, 0 | 20260506 03:15:45.305 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:45.305 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:45.305 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 03:15:45.305 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:45.305 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:45.305 | INFO | Uploaded 36GiB to GPU in 2.0781s (17.3235GiB/s) | 20260506 03:15:45.306 | INFO | Downloaded 34GiB to GPU in 2.0035s (16.9703GiB/s) | 20260506 03:15:50.442 | INFO | Analyzing chunk frames = [8513536,8514048) | 20260506 03:15:50.442 | INFO | MemInfo: { | 20260506 03:15:50.442 | INFO | "gpu" : " 17 GB", | 20260506 03:15:50.442 | INFO | "huge" : "", | 20260506 03:15:50.442 | INFO | "huge_shared" : " 27 GB", | 20260506 03:15:50.442 | INFO | "malloc" : " 14 GB", | 20260506 03:15:50.442 | INFO | "pinned" : " 29 GB" | 20260506 03:15:50.442 | INFO | } | 20260506 03:15:50.442 | INFO | Perf report for the last 5.1371 seconds: | 20260506 03:15:50.442 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:15:50.442 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:50.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:50.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:50.442 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 03:15:50.442 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:50.442 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:50.442 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 03:15:50.442 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:50.442 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:50.443 | INFO | Uploaded 36GiB to GPU in 2.10053s (17.1385GiB/s) | 20260506 03:15:50.443 | INFO | Downloaded 34GiB to GPU in 2.03467s (16.7103GiB/s) | 20260506 03:15:55.582 | INFO | Analyzing chunk frames = [8514048,8514560) | 20260506 03:15:55.582 | INFO | MemInfo: { | 20260506 03:15:55.582 | INFO | "gpu" : " 17 GB", | 20260506 03:15:55.582 | INFO | "huge" : "", | 20260506 03:15:55.582 | INFO | "huge_shared" : " 27 GB", | 20260506 03:15:55.582 | INFO | "malloc" : " 14 GB", | 20260506 03:15:55.582 | INFO | "pinned" : " 29 GB" | 20260506 03:15:55.582 | INFO | } | 20260506 03:15:55.582 | INFO | Perf report for the last 5.14015 seconds: | 20260506 03:15:55.582 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:15:55.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:55.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:55.583 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:55.583 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 03:15:55.583 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:55.583 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:55.583 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 03:15:55.583 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:55.583 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:15:55.583 | INFO | Uploaded 36GiB to GPU in 2.09025s (17.2228GiB/s) | 20260506 03:15:55.583 | INFO | Downloaded 34GiB to GPU in 2.04155s (16.654GiB/s) | 20260506 03:16:00.719 | INFO | Analyzing chunk frames = [8514560,8515072) | 20260506 03:16:00.719 | INFO | MemInfo: { | 20260506 03:16:00.719 | INFO | "gpu" : " 17 GB", | 20260506 03:16:00.719 | INFO | "huge" : "", | 20260506 03:16:00.719 | INFO | "huge_shared" : " 27 GB", | 20260506 03:16:00.719 | INFO | "malloc" : " 14 GB", | 20260506 03:16:00.719 | INFO | "pinned" : " 29 GB" | 20260506 03:16:00.719 | INFO | } | 20260506 03:16:00.719 | INFO | Perf report for the last 5.13656 seconds: | 20260506 03:16:00.719 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:16:00.719 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:00.719 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:00.719 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:00.719 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260506 03:16:00.719 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:00.719 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:00.719 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 03:16:00.719 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:00.719 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:00.719 | INFO | Uploaded 36GiB to GPU in 2.11579s (17.0149GiB/s) | 20260506 03:16:00.719 | INFO | Downloaded 34GiB to GPU in 2.08482s (16.3084GiB/s) | 20260506 03:16:05.865 | INFO | Analyzing chunk frames = [8515072,8515584) | 20260506 03:16:05.865 | INFO | MemInfo: { | 20260506 03:16:05.865 | INFO | "gpu" : " 17 GB", | 20260506 03:16:05.865 | INFO | "huge" : "", | 20260506 03:16:05.865 | INFO | "huge_shared" : " 27 GB", | 20260506 03:16:05.865 | INFO | "malloc" : " 14 GB", | 20260506 03:16:05.865 | INFO | "pinned" : " 29 GB" | 20260506 03:16:05.865 | INFO | } | 20260506 03:16:05.865 | INFO | Perf report for the last 5.14599 seconds: | 20260506 03:16:05.865 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:16:05.865 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:05.865 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:05.865 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:05.865 | INFO | ANALYSIS : 61.2%, 3.0, 95.7ms, 0 | 20260506 03:16:05.865 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:05.865 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:05.865 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:16:05.865 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:05.865 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:05.865 | INFO | Uploaded 36GiB to GPU in 2.04942s (17.5659GiB/s) | 20260506 03:16:05.865 | INFO | Downloaded 34GiB to GPU in 2.01049s (16.9113GiB/s) | 20260506 03:16:05.992 | INFO | Starting a round of Baz IO | 20260506 03:16:09.111 | INFO | Populating RTMetrics report | 20260506 03:16:09.111 | INFO | RTMetrics report populated | 20260506 03:16:09.111 | INFO | Opening tmp RTMetrics json file | 20260506 03:16:09.113 | INFO | Serializing RTMetrics to json | 20260506 03:16:09.115 | INFO | Serialization done, swapping tmp file now... | 20260506 03:16:09.116 | INFO | RTMetrics done | 20260506 03:16:09.201 | INFO | Finished a round of Baz IO: Wrote 1.765 GiB in 3.208 seconds | 20260506 03:16:09.202 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.23% / 0.00% / 78.77% | 20260506 03:16:09.202 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0624537% / 0% / 9.13938e-05% | 20260506 03:16:09.202 | INFO | Baz IO overall: | 20260506 03:16:09.202 | INFO | BytesWritten : 4743619784941 | 20260506 03:16:09.202 | INFO | NumEvents : 2224930465576 | 20260506 03:16:11.005 | INFO | Analyzing chunk frames = [8515584,8516096) | 20260506 03:16:11.005 | INFO | MemInfo: { | 20260506 03:16:11.005 | INFO | "gpu" : " 17 GB", | 20260506 03:16:11.005 | INFO | "huge" : "", | 20260506 03:16:11.005 | INFO | "huge_shared" : " 27 GB", | 20260506 03:16:11.005 | INFO | "malloc" : " 14 GB", | 20260506 03:16:11.005 | INFO | "pinned" : " 29 GB" | 20260506 03:16:11.005 | INFO | } | 20260506 03:16:11.005 | INFO | Perf report for the last 5.13973 seconds: | 20260506 03:16:11.005 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:16:11.005 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:11.005 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:11.005 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:11.005 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260506 03:16:11.005 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:11.005 | INFO | RT_METRICS : 42.6%, 1.0, 22.3ms, 0 | 20260506 03:16:11.005 | INFO | PRE_HQ : 59.0%, 1.7, 53.0ms, 0 | 20260506 03:16:11.005 | INFO | BAZWRITER : 61.7%, 3.5, 109.9ms, 0 | 20260506 03:16:11.005 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 03:16:11.005 | INFO | Uploaded 36GiB to GPU in 2.10538s (17.0991GiB/s) | 20260506 03:16:11.005 | INFO | Downloaded 38GiB to GPU in 2.23217s (17.0238GiB/s) | 20260506 03:16:16.142 | INFO | Analyzing chunk frames = [8516096,8516608) | 20260506 03:16:16.142 | INFO | MemInfo: { | 20260506 03:16:16.142 | INFO | "gpu" : " 17 GB", | 20260506 03:16:16.142 | INFO | "huge" : "", | 20260506 03:16:16.142 | INFO | "huge_shared" : " 27 GB", | 20260506 03:16:16.142 | INFO | "malloc" : " 14 GB", | 20260506 03:16:16.143 | INFO | "pinned" : " 29 GB" | 20260506 03:16:16.143 | INFO | } | 20260506 03:16:16.143 | INFO | Perf report for the last 5.13779 seconds: | 20260506 03:16:16.143 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:16:16.143 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:16.143 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:16.143 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:16.143 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260506 03:16:16.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:16.143 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:16.143 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 03:16:16.143 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:16.143 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:16.143 | INFO | Uploaded 36GiB to GPU in 1.96422s (18.3279GiB/s) | 20260506 03:16:16.143 | INFO | Downloaded 34GiB to GPU in 1.99001s (17.0853GiB/s) | 20260506 03:16:21.278 | INFO | Analyzing chunk frames = [8516608,8517120) | 20260506 03:16:21.278 | INFO | MemInfo: { | 20260506 03:16:21.278 | INFO | "gpu" : " 17 GB", | 20260506 03:16:21.278 | INFO | "huge" : "", | 20260506 03:16:21.278 | INFO | "huge_shared" : " 27 GB", | 20260506 03:16:21.278 | INFO | "malloc" : " 14 GB", | 20260506 03:16:21.278 | INFO | "pinned" : " 29 GB" | 20260506 03:16:21.278 | INFO | } | 20260506 03:16:21.278 | INFO | Perf report for the last 5.13537 seconds: | 20260506 03:16:21.278 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:16:21.278 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:21.278 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:21.278 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:21.278 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260506 03:16:21.278 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:21.278 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:21.278 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 03:16:21.278 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:21.278 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:21.278 | INFO | Uploaded 36GiB to GPU in 2.08607s (17.2573GiB/s) | 20260506 03:16:21.278 | INFO | Downloaded 34GiB to GPU in 2.03787s (16.6841GiB/s) | 20260506 03:16:26.418 | INFO | Analyzing chunk frames = [8517120,8517632) | 20260506 03:16:26.418 | INFO | MemInfo: { | 20260506 03:16:26.418 | INFO | "gpu" : " 17 GB", | 20260506 03:16:26.418 | INFO | "huge" : "", | 20260506 03:16:26.418 | INFO | "huge_shared" : " 27 GB", | 20260506 03:16:26.418 | INFO | "malloc" : " 14 GB", | 20260506 03:16:26.418 | INFO | "pinned" : " 29 GB" | 20260506 03:16:26.419 | INFO | } | 20260506 03:16:26.419 | INFO | Perf report for the last 5.14058 seconds: | 20260506 03:16:26.419 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:16:26.419 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:26.419 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:26.419 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:26.419 | INFO | ANALYSIS : 61.2%, 3.0, 95.0ms, 0 | 20260506 03:16:26.419 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:26.419 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:26.419 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 03:16:26.419 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:26.419 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:26.419 | INFO | Uploaded 36GiB to GPU in 2.08628s (17.2556GiB/s) | 20260506 03:16:26.419 | INFO | Downloaded 34GiB to GPU in 2.02465s (16.7931GiB/s) | 20260506 03:16:31.568 | INFO | Analyzing chunk frames = [8517632,8518144) | 20260506 03:16:31.568 | INFO | MemInfo: { | 20260506 03:16:31.568 | INFO | "gpu" : " 17 GB", | 20260506 03:16:31.568 | INFO | "huge" : "", | 20260506 03:16:31.568 | INFO | "huge_shared" : " 27 GB", | 20260506 03:16:31.568 | INFO | "malloc" : " 14 GB", | 20260506 03:16:31.568 | INFO | "pinned" : " 29 GB" | 20260506 03:16:31.568 | INFO | } | 20260506 03:16:31.568 | INFO | Perf report for the last 5.14943 seconds: | 20260506 03:16:31.568 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:16:31.568 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:31.568 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:31.568 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:31.568 | INFO | ANALYSIS : 60.4%, 3.0, 94.5ms, 0 | 20260506 03:16:31.568 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:31.568 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:31.568 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 03:16:31.568 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:31.568 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:31.568 | INFO | Uploaded 36GiB to GPU in 2.08612s (17.2569GiB/s) | 20260506 03:16:31.568 | INFO | Downloaded 34GiB to GPU in 2.02539s (16.7869GiB/s) | 20260506 03:16:36.702 | INFO | Analyzing chunk frames = [8518144,8518656) | 20260506 03:16:36.702 | INFO | MemInfo: { | 20260506 03:16:36.702 | INFO | "gpu" : " 17 GB", | 20260506 03:16:36.702 | INFO | "huge" : "", | 20260506 03:16:36.702 | INFO | "huge_shared" : " 27 GB", | 20260506 03:16:36.702 | INFO | "malloc" : " 14 GB", | 20260506 03:16:36.703 | INFO | "pinned" : " 29 GB" | 20260506 03:16:36.703 | INFO | } | 20260506 03:16:36.703 | INFO | Perf report for the last 5.13484 seconds: | 20260506 03:16:36.703 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:16:36.703 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:36.703 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:36.703 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:36.703 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260506 03:16:36.703 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:36.703 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:36.703 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:16:36.703 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:36.703 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:36.703 | INFO | Uploaded 36GiB to GPU in 2.07842s (17.3208GiB/s) | 20260506 03:16:36.703 | INFO | Downloaded 34GiB to GPU in 2.03607s (16.6988GiB/s) | 20260506 03:16:41.841 | INFO | Analyzing chunk frames = [8518656,8519168) | 20260506 03:16:41.841 | INFO | MemInfo: { | 20260506 03:16:41.841 | INFO | "gpu" : " 17 GB", | 20260506 03:16:41.841 | INFO | "huge" : "", | 20260506 03:16:41.841 | INFO | "huge_shared" : " 27 GB", | 20260506 03:16:41.841 | INFO | "malloc" : " 14 GB", | 20260506 03:16:41.841 | INFO | "pinned" : " 29 GB" | 20260506 03:16:41.841 | INFO | } | 20260506 03:16:41.841 | INFO | Perf report for the last 5.13834 seconds: | 20260506 03:16:41.841 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:16:41.841 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:41.841 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:41.841 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:41.841 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260506 03:16:41.841 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:41.841 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:41.841 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 03:16:41.841 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:41.841 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:41.842 | INFO | Uploaded 36GiB to GPU in 2.09602s (17.1754GiB/s) | 20260506 03:16:41.842 | INFO | Downloaded 34GiB to GPU in 2.06693s (16.4495GiB/s) | 20260506 03:16:46.973 | INFO | Analyzing chunk frames = [8519168,8519680) | 20260506 03:16:46.973 | INFO | MemInfo: { | 20260506 03:16:46.973 | INFO | "gpu" : " 17 GB", | 20260506 03:16:46.973 | INFO | "huge" : "", | 20260506 03:16:46.973 | INFO | "huge_shared" : " 27 GB", | 20260506 03:16:46.973 | INFO | "malloc" : " 14 GB", | 20260506 03:16:46.974 | INFO | "pinned" : " 29 GB" | 20260506 03:16:46.974 | INFO | } | 20260506 03:16:46.974 | INFO | Perf report for the last 5.13238 seconds: | 20260506 03:16:46.974 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:16:46.974 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:46.974 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:46.974 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:46.974 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 03:16:46.974 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:46.974 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:46.974 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:16:46.974 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:46.974 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:46.974 | INFO | Uploaded 36GiB to GPU in 2.0412s (17.6366GiB/s) | 20260506 03:16:46.974 | INFO | Downloaded 34GiB to GPU in 2.01768s (16.851GiB/s) | 20260506 03:16:46.974 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 03:16:46.974 | INFO | Total mem usage: 15.4 GiB | 20260506 03:16:46.974 | INFO | All Time High Water Marks sum: 16.3 GiB (Savings: 5.2%) | 20260506 03:16:46.974 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -10%) | 20260506 03:16:46.974 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 03:16:46.974 | INFO | High Water Breakdown: | 20260506 03:16:46.974 | INFO | 60.9% : 8.54 GiB ( 88.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 03:16:46.974 | INFO | 39% : 5.47 GiB ( 82.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 03:16:46.974 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 03:16:46.974 | INFO | ------------------------------------------ | 20260506 03:16:46.974 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:16:46.974 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 03:16:46.974 | INFO | Total mem usage: 26.5 GiB | 20260506 03:16:46.974 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 03:16:46.974 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 03:16:46.974 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 03:16:46.974 | INFO | High Water Breakdown: | 20260506 03:16:46.974 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 03:16:46.974 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 03:16:46.974 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 03:16:46.974 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 03:16:46.975 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 03:16:46.975 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 03:16:46.975 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 03:16:46.975 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 03:16:46.975 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 03:16:46.975 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 03:16:46.975 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 03:16:46.975 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 03:16:46.975 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 03:16:46.975 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 03:16:46.975 | INFO | ------------------------------------------ | 20260506 03:16:46.975 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:16:46.975 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 03:16:46.975 | INFO | Total mem usage: 25.6 GiB | 20260506 03:16:46.975 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 03:16:46.975 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 03:16:46.975 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 03:16:46.975 | INFO | High Water Breakdown: | 20260506 03:16:46.975 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 03:16:46.975 | INFO | ------------------------------------------ | 20260506 03:16:46.975 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:16:46.975 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 03:16:46.975 | INFO | Total mem usage: 15.7 GiB | 20260506 03:16:46.975 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 03:16:46.975 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 03:16:46.975 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 03:16:46.975 | INFO | High Water Breakdown: | 20260506 03:16:46.975 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 03:16:46.975 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 03:16:46.975 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 03:16:46.975 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 03:16:46.975 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 03:16:46.975 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 03:16:46.975 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 03:16:46.975 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 03:16:46.975 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 03:16:46.975 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 03:16:46.975 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 03:16:46.975 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 03:16:46.975 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 03:16:46.975 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 03:16:46.975 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 03:16:46.975 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 03:16:46.975 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 03:16:46.975 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 03:16:46.975 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 03:16:46.975 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 03:16:46.975 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 03:16:46.975 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 03:16:46.975 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 03:16:46.975 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 03:16:46.975 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 03:16:46.975 | INFO | ------------------------------------------ | 20260506 03:16:46.975 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:16:46.975 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 03:16:46.975 | INFO | PerfReport: Gathered 1536 samples. | 20260506 03:16:46.975 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 03:16:46.975 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.37e+01, 1.33e+01 / 7.15e-01 | 20260506 03:16:46.975 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.03e-01, 1.09e-02 / 3.30e-03 | 20260506 03:16:46.975 | INFO | PerfReport: 11.87%, Baseline -- 3.71e+00 / 4.51e+00, 5.82e-01 / 2.06e-01 | 20260506 03:16:46.975 | INFO | PerfReport: 3.31%, Histogram -- 1.04e+00 / 1.54e+00, 1.97e-01 / 7.55e-02 | 20260506 03:16:46.975 | INFO | PerfReport: 15.97%, DME -- 4.00e+01 / 4.39e+01, 3.89e+00 / 9.18e-01 | 20260506 03:16:46.975 | INFO | PerfReport: 57.58%, FrameLabeling -- 1.80e+01 / 1.93e+01, 1.57e+00 / 9.57e-01 | 20260506 03:16:46.975 | INFO | PerfReport: 6.96%, PulseAccumulating -- 2.18e+00 / 2.73e+00, 4.06e-01 / 1.68e-01 | 20260506 03:16:46.975 | INFO | PerfReport: 2.98%, Metrics -- 9.32e-01 / 1.21e+00, 2.47e-01 / 9.57e-02 | 20260506 03:16:46.975 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.21e-01 / 3.66e-01, 5.39e-02 / 2.74e-02 | 20260506 03:16:46.975 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 03:16:46.975 | INFO | PerfReport: Gathered 1536 samples. | 20260506 03:16:46.975 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 03:16:46.975 | INFO | PerfReport: Full Execution -- 9.75e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260506 03:16:46.975 | INFO | PerfReport: 22.11%, Upload -- 2.16e+01 / 0.00e+00, 7.09e+00 / 0.00e+00 | 20260506 03:16:46.975 | INFO | PerfReport: 21.94%, Download -- 2.14e+01 / 0.00e+00, 7.50e+00 / 0.00e+00 | 20260506 03:16:46.975 | INFO | PerfReport: 32.10%, Compute -- 3.13e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 03:16:46.975 | INFO | PerfReport: 1.99%, LockUpload -- 1.94e+00 / 0.00e+00, 7.84e+00 / 0.00e+00 | 20260506 03:16:46.975 | INFO | PerfReport: 1.26%, LockDownload -- 1.23e+00 / 0.00e+00, 2.94e+00 / 0.00e+00 | 20260506 03:16:46.975 | INFO | PerfReport: 20.59%, LockCompute -- 2.01e+01 / 0.00e+00, 1.75e+01 / 0.00e+00 | 20260506 03:16:46.975 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 2.57e-02 / 0.00e+00 | 20260506 03:16:47.098 | INFO | Starting a round of Baz IO | 20260506 03:16:50.233 | INFO | Populating RTMetrics report | 20260506 03:16:50.234 | INFO | RTMetrics report populated | 20260506 03:16:50.234 | INFO | Opening tmp RTMetrics json file | 20260506 03:16:50.235 | INFO | Serializing RTMetrics to json | 20260506 03:16:50.236 | INFO | Serialization done, swapping tmp file now... | 20260506 03:16:50.237 | INFO | RTMetrics done | 20260506 03:16:50.316 | INFO | Finished a round of Baz IO: Wrote 1.764 GiB in 3.217 seconds | 20260506 03:16:50.316 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.23% / 0.00% / 78.77% | 20260506 03:16:50.316 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0621517% / 7.94729e-06% / 8.74201e-05% | 20260506 03:16:50.316 | INFO | Baz IO overall: | 20260506 03:16:50.316 | INFO | BytesWritten : 4745514358838 | 20260506 03:16:50.316 | INFO | NumEvents : 2225789109082 | 20260506 03:16:52.116 | INFO | Analyzing chunk frames = [8519680,8520192) | 20260506 03:16:52.116 | INFO | MemInfo: { | 20260506 03:16:52.116 | INFO | "gpu" : " 17 GB", | 20260506 03:16:52.116 | INFO | "huge" : "", | 20260506 03:16:52.116 | INFO | "huge_shared" : " 27 GB", | 20260506 03:16:52.116 | INFO | "malloc" : " 14 GB", | 20260506 03:16:52.116 | INFO | "pinned" : " 29 GB" | 20260506 03:16:52.116 | INFO | } | 20260506 03:16:52.116 | INFO | Perf report for the last 5.14248 seconds: | 20260506 03:16:52.116 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:16:52.116 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:52.116 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:52.116 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:52.116 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260506 03:16:52.116 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:52.116 | INFO | RT_METRICS : 42.6%, 1.0, 22.3ms, 0 | 20260506 03:16:52.116 | INFO | PRE_HQ : 59.3%, 1.7, 53.2ms, 0 | 20260506 03:16:52.116 | INFO | BAZWRITER : 61.8%, 3.5, 112.3ms, 0 | 20260506 03:16:52.116 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 03:16:52.116 | INFO | Uploaded 36GiB to GPU in 2.16668s (16.6153GiB/s) | 20260506 03:16:52.116 | INFO | Downloaded 38GiB to GPU in 2.24376s (16.9359GiB/s) | 20260506 03:16:57.255 | INFO | Analyzing chunk frames = [8520192,8520704) | 20260506 03:16:57.255 | INFO | MemInfo: { | 20260506 03:16:57.255 | INFO | "gpu" : " 17 GB", | 20260506 03:16:57.255 | INFO | "huge" : "", | 20260506 03:16:57.255 | INFO | "huge_shared" : " 27 GB", | 20260506 03:16:57.255 | INFO | "malloc" : " 14 GB", | 20260506 03:16:57.255 | INFO | "pinned" : " 29 GB" | 20260506 03:16:57.255 | INFO | } | 20260506 03:16:57.255 | INFO | Perf report for the last 5.13924 seconds: | 20260506 03:16:57.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:16:57.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:57.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:57.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:57.255 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260506 03:16:57.255 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:57.255 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:57.255 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 03:16:57.255 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:57.255 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:16:57.256 | INFO | Uploaded 36GiB to GPU in 1.93428s (18.6116GiB/s) | 20260506 03:16:57.256 | INFO | Downloaded 34GiB to GPU in 1.97561s (17.2099GiB/s) | 20260506 03:17:02.390 | INFO | Analyzing chunk frames = [8520704,8521216) | 20260506 03:17:02.390 | INFO | MemInfo: { | 20260506 03:17:02.390 | INFO | "gpu" : " 17 GB", | 20260506 03:17:02.390 | INFO | "huge" : "", | 20260506 03:17:02.390 | INFO | "huge_shared" : " 27 GB", | 20260506 03:17:02.390 | INFO | "malloc" : " 14 GB", | 20260506 03:17:02.390 | INFO | "pinned" : " 29 GB" | 20260506 03:17:02.390 | INFO | } | 20260506 03:17:02.390 | INFO | Perf report for the last 5.1347 seconds: | 20260506 03:17:02.390 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:17:02.390 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:02.390 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:02.390 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:02.390 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 03:17:02.390 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:02.390 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:02.390 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 03:17:02.390 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:02.390 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:02.390 | INFO | Uploaded 36GiB to GPU in 2.11118s (17.052GiB/s) | 20260506 03:17:02.390 | INFO | Downloaded 34GiB to GPU in 2.04112s (16.6576GiB/s) | 20260506 03:17:07.532 | INFO | Analyzing chunk frames = [8521216,8521728) | 20260506 03:17:07.532 | INFO | MemInfo: { | 20260506 03:17:07.532 | INFO | "gpu" : " 17 GB", | 20260506 03:17:07.532 | INFO | "huge" : "", | 20260506 03:17:07.532 | INFO | "huge_shared" : " 27 GB", | 20260506 03:17:07.532 | INFO | "malloc" : " 14 GB", | 20260506 03:17:07.532 | INFO | "pinned" : " 29 GB" | 20260506 03:17:07.532 | INFO | } | 20260506 03:17:07.532 | INFO | Perf report for the last 5.14232 seconds: | 20260506 03:17:07.532 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:17:07.532 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:07.532 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:07.532 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:07.532 | INFO | ANALYSIS : 61.1%, 3.0, 94.9ms, 0 | 20260506 03:17:07.532 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:07.532 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:07.532 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:17:07.532 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:07.533 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:07.533 | INFO | Uploaded 36GiB to GPU in 2.1001s (17.142GiB/s) | 20260506 03:17:07.533 | INFO | Downloaded 34GiB to GPU in 2.03219s (16.7307GiB/s) | 20260506 03:17:12.674 | INFO | Analyzing chunk frames = [8521728,8522240) | 20260506 03:17:12.674 | INFO | MemInfo: { | 20260506 03:17:12.674 | INFO | "gpu" : " 17 GB", | 20260506 03:17:12.674 | INFO | "huge" : "", | 20260506 03:17:12.674 | INFO | "huge_shared" : " 27 GB", | 20260506 03:17:12.674 | INFO | "malloc" : " 14 GB", | 20260506 03:17:12.674 | INFO | "pinned" : " 29 GB" | 20260506 03:17:12.674 | INFO | } | 20260506 03:17:12.674 | INFO | Perf report for the last 5.14167 seconds: | 20260506 03:17:12.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:17:12.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:12.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:12.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:12.674 | INFO | ANALYSIS : 60.7%, 3.0, 94.8ms, 0 | 20260506 03:17:12.674 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:12.674 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:12.674 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 03:17:12.674 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:12.674 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:12.674 | INFO | Uploaded 36GiB to GPU in 2.08158s (17.2946GiB/s) | 20260506 03:17:12.674 | INFO | Downloaded 34GiB to GPU in 2.0421s (16.6495GiB/s) | 20260506 03:17:17.815 | INFO | Analyzing chunk frames = [8522240,8522752) | 20260506 03:17:17.815 | INFO | MemInfo: { | 20260506 03:17:17.815 | INFO | "gpu" : " 17 GB", | 20260506 03:17:17.815 | INFO | "huge" : "", | 20260506 03:17:17.815 | INFO | "huge_shared" : " 27 GB", | 20260506 03:17:17.815 | INFO | "malloc" : " 14 GB", | 20260506 03:17:17.815 | INFO | "pinned" : " 29 GB" | 20260506 03:17:17.815 | INFO | } | 20260506 03:17:17.815 | INFO | Perf report for the last 5.14118 seconds: | 20260506 03:17:17.815 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:17:17.815 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:17.815 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:17.815 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:17.815 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260506 03:17:17.815 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:17.815 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:17.815 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:17:17.815 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:17.815 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:17.816 | INFO | Uploaded 36GiB to GPU in 2.05536s (17.5152GiB/s) | 20260506 03:17:17.816 | INFO | Downloaded 34GiB to GPU in 2.02554s (16.7856GiB/s) | 20260506 03:17:22.955 | INFO | Analyzing chunk frames = [8522752,8523264) | 20260506 03:17:22.955 | INFO | MemInfo: { | 20260506 03:17:22.955 | INFO | "gpu" : " 17 GB", | 20260506 03:17:22.955 | INFO | "huge" : "", | 20260506 03:17:22.955 | INFO | "huge_shared" : " 27 GB", | 20260506 03:17:22.955 | INFO | "malloc" : " 14 GB", | 20260506 03:17:22.955 | INFO | "pinned" : " 29 GB" | 20260506 03:17:22.955 | INFO | } | 20260506 03:17:22.955 | INFO | Perf report for the last 5.13955 seconds: | 20260506 03:17:22.955 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:17:22.955 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:22.955 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:22.955 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:22.955 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260506 03:17:22.955 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:22.955 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:22.955 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 03:17:22.955 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:22.955 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:22.955 | INFO | Uploaded 36GiB to GPU in 2.08205s (17.2907GiB/s) | 20260506 03:17:22.956 | INFO | Downloaded 34GiB to GPU in 2.05877s (16.5147GiB/s) | 20260506 03:17:28.088 | INFO | Analyzing chunk frames = [8523264,8523776) | 20260506 03:17:28.088 | INFO | MemInfo: { | 20260506 03:17:28.088 | INFO | "gpu" : " 17 GB", | 20260506 03:17:28.088 | INFO | "huge" : "", | 20260506 03:17:28.088 | INFO | "huge_shared" : " 27 GB", | 20260506 03:17:28.088 | INFO | "malloc" : " 14 GB", | 20260506 03:17:28.088 | INFO | "pinned" : " 29 GB" | 20260506 03:17:28.088 | INFO | } | 20260506 03:17:28.088 | INFO | Perf report for the last 5.1336 seconds: | 20260506 03:17:28.088 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:17:28.088 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:28.088 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:28.088 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:28.089 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 03:17:28.089 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:28.089 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:28.089 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:17:28.089 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:28.089 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:28.089 | INFO | Uploaded 36GiB to GPU in 2.05501s (17.5181GiB/s) | 20260506 03:17:28.089 | INFO | Downloaded 34GiB to GPU in 2.03016s (16.7474GiB/s) | 20260506 03:17:28.207 | INFO | Starting a round of Baz IO | 20260506 03:17:31.351 | INFO | Populating RTMetrics report | 20260506 03:17:31.351 | INFO | RTMetrics report populated | 20260506 03:17:31.351 | INFO | Opening tmp RTMetrics json file | 20260506 03:17:31.372 | INFO | Serializing RTMetrics to json | 20260506 03:17:31.399 | INFO | Serialization done, swapping tmp file now... | 20260506 03:17:31.402 | INFO | RTMetrics done | 20260506 03:17:31.563 | INFO | Finished a round of Baz IO: Wrote 1.766 GiB in 3.356 seconds | 20260506 03:17:31.563 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.23% / 0.00% / 78.77% | 20260506 03:17:31.563 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0620564% / 0% / 0.000111262% | 20260506 03:17:31.563 | INFO | Baz IO overall: | 20260506 03:17:31.563 | INFO | BytesWritten : 4747410355144 | 20260506 03:17:31.563 | INFO | NumEvents : 2226648653964 | 20260506 03:17:33.228 | INFO | Analyzing chunk frames = [8523776,8524288) | 20260506 03:17:33.228 | INFO | MemInfo: { | 20260506 03:17:33.228 | INFO | "gpu" : " 17 GB", | 20260506 03:17:33.228 | INFO | "huge" : "", | 20260506 03:17:33.228 | INFO | "huge_shared" : " 27 GB", | 20260506 03:17:33.228 | INFO | "malloc" : " 14 GB", | 20260506 03:17:33.228 | INFO | "pinned" : " 29 GB" | 20260506 03:17:33.228 | INFO | } | 20260506 03:17:33.228 | INFO | Perf report for the last 5.13982 seconds: | 20260506 03:17:33.228 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:17:33.228 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:33.228 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:33.228 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:33.228 | INFO | ANALYSIS : 62.5%, 3.0, 97.1ms, 0 | 20260506 03:17:33.228 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:33.228 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260506 03:17:33.228 | INFO | PRE_HQ : 59.8%, 1.7, 53.4ms, 0 | 20260506 03:17:33.228 | INFO | BAZWRITER : 64.4%, 3.8, 126.1ms, 0 | 20260506 03:17:33.228 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260506 03:17:33.229 | INFO | Uploaded 36GiB to GPU in 2.19842s (16.3754GiB/s) | 20260506 03:17:33.229 | INFO | Downloaded 38GiB to GPU in 2.25461s (16.8544GiB/s) | 20260506 03:17:38.373 | INFO | Analyzing chunk frames = [8524288,8524800) | 20260506 03:17:38.373 | INFO | MemInfo: { | 20260506 03:17:38.373 | INFO | "gpu" : " 17 GB", | 20260506 03:17:38.373 | INFO | "huge" : "", | 20260506 03:17:38.373 | INFO | "huge_shared" : " 27 GB", | 20260506 03:17:38.373 | INFO | "malloc" : " 14 GB", | 20260506 03:17:38.373 | INFO | "pinned" : " 29 GB" | 20260506 03:17:38.373 | INFO | } | 20260506 03:17:38.373 | INFO | Perf report for the last 5.1451 seconds: | 20260506 03:17:38.373 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:17:38.373 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:38.373 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:38.373 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:38.373 | INFO | ANALYSIS : 60.6%, 3.0, 94.7ms, 0 | 20260506 03:17:38.373 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:38.373 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:38.374 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:17:38.374 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:38.374 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:38.374 | INFO | Uploaded 36GiB to GPU in 1.97423s (18.2349GiB/s) | 20260506 03:17:38.374 | INFO | Downloaded 34GiB to GPU in 1.98192s (17.1551GiB/s) | 20260506 03:17:43.522 | INFO | Analyzing chunk frames = [8524800,8525312) | 20260506 03:17:43.522 | INFO | MemInfo: { | 20260506 03:17:43.522 | INFO | "gpu" : " 17 GB", | 20260506 03:17:43.522 | INFO | "huge" : "", | 20260506 03:17:43.522 | INFO | "huge_shared" : " 27 GB", | 20260506 03:17:43.523 | INFO | "malloc" : " 14 GB", | 20260506 03:17:43.523 | INFO | "pinned" : " 29 GB" | 20260506 03:17:43.523 | INFO | } | 20260506 03:17:43.523 | INFO | Perf report for the last 5.14931 seconds: | 20260506 03:17:43.523 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:17:43.523 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:43.523 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:43.523 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:43.523 | INFO | ANALYSIS : 60.5%, 3.0, 94.4ms, 0 | 20260506 03:17:43.523 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:43.523 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:43.523 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 03:17:43.523 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:43.523 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:43.523 | INFO | Uploaded 36GiB to GPU in 2.01109s (17.9007GiB/s) | 20260506 03:17:43.523 | INFO | Downloaded 34GiB to GPU in 1.98627s (17.1175GiB/s) | 20260506 03:17:48.652 | INFO | Analyzing chunk frames = [8525312,8525824) | 20260506 03:17:48.652 | INFO | MemInfo: { | 20260506 03:17:48.652 | INFO | "gpu" : " 17 GB", | 20260506 03:17:48.652 | INFO | "huge" : "", | 20260506 03:17:48.652 | INFO | "huge_shared" : " 27 GB", | 20260506 03:17:48.652 | INFO | "malloc" : " 14 GB", | 20260506 03:17:48.652 | INFO | "pinned" : " 29 GB" | 20260506 03:17:48.652 | INFO | } | 20260506 03:17:48.653 | INFO | Perf report for the last 5.12985 seconds: | 20260506 03:17:48.653 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:17:48.653 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:48.653 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:48.653 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:48.653 | INFO | ANALYSIS : 60.9%, 3.0, 94.5ms, 0 | 20260506 03:17:48.653 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:48.653 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:48.653 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 03:17:48.653 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:48.653 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:48.653 | INFO | Uploaded 36GiB to GPU in 2.0982s (17.1575GiB/s) | 20260506 03:17:48.653 | INFO | Downloaded 34GiB to GPU in 2.03028s (16.7464GiB/s) | 20260506 03:17:53.787 | INFO | Analyzing chunk frames = [8525824,8526336) | 20260506 03:17:53.787 | INFO | MemInfo: { | 20260506 03:17:53.787 | INFO | "gpu" : " 17 GB", | 20260506 03:17:53.787 | INFO | "huge" : "", | 20260506 03:17:53.787 | INFO | "huge_shared" : " 27 GB", | 20260506 03:17:53.787 | INFO | "malloc" : " 14 GB", | 20260506 03:17:53.787 | INFO | "pinned" : " 29 GB" | 20260506 03:17:53.787 | INFO | } | 20260506 03:17:53.787 | INFO | Perf report for the last 5.13488 seconds: | 20260506 03:17:53.787 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:17:53.787 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:53.787 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:53.788 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:53.788 | INFO | ANALYSIS : 60.8%, 3.0, 94.9ms, 0 | 20260506 03:17:53.788 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:53.788 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:53.788 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:17:53.788 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:53.788 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:53.788 | INFO | Uploaded 36GiB to GPU in 2.1121s (17.0447GiB/s) | 20260506 03:17:53.788 | INFO | Downloaded 34GiB to GPU in 2.04133s (16.6558GiB/s) | 20260506 03:17:58.928 | INFO | Analyzing chunk frames = [8526336,8526848) | 20260506 03:17:58.928 | INFO | MemInfo: { | 20260506 03:17:58.928 | INFO | "gpu" : " 17 GB", | 20260506 03:17:58.928 | INFO | "huge" : "", | 20260506 03:17:58.928 | INFO | "huge_shared" : " 27 GB", | 20260506 03:17:58.928 | INFO | "malloc" : " 14 GB", | 20260506 03:17:58.928 | INFO | "pinned" : " 29 GB" | 20260506 03:17:58.928 | INFO | } | 20260506 03:17:58.928 | INFO | Perf report for the last 5.14079 seconds: | 20260506 03:17:58.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:17:58.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:58.928 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:58.928 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:58.928 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 03:17:58.928 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:58.928 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:58.928 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 03:17:58.928 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:58.928 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:17:58.929 | INFO | Uploaded 36GiB to GPU in 2.08739s (17.2464GiB/s) | 20260506 03:17:58.929 | INFO | Downloaded 34GiB to GPU in 2.04071s (16.6609GiB/s) | 20260506 03:18:04.064 | INFO | Analyzing chunk frames = [8526848,8527360) | 20260506 03:18:04.064 | INFO | MemInfo: { | 20260506 03:18:04.064 | INFO | "gpu" : " 17 GB", | 20260506 03:18:04.064 | INFO | "huge" : "", | 20260506 03:18:04.064 | INFO | "huge_shared" : " 27 GB", | 20260506 03:18:04.064 | INFO | "malloc" : " 14 GB", | 20260506 03:18:04.064 | INFO | "pinned" : " 29 GB" | 20260506 03:18:04.064 | INFO | } | 20260506 03:18:04.064 | INFO | Perf report for the last 5.13559 seconds: | 20260506 03:18:04.064 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:18:04.064 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:04.064 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:04.064 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:04.064 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260506 03:18:04.064 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:04.064 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:04.064 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 03:18:04.064 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:04.064 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:04.064 | INFO | Uploaded 36GiB to GPU in 2.08295s (17.2831GiB/s) | 20260506 03:18:04.064 | INFO | Downloaded 34GiB to GPU in 2.06958s (16.4285GiB/s) | 20260506 03:18:09.206 | INFO | Analyzing chunk frames = [8527360,8527872) | 20260506 03:18:09.206 | INFO | MemInfo: { | 20260506 03:18:09.206 | INFO | "gpu" : " 17 GB", | 20260506 03:18:09.206 | INFO | "huge" : "", | 20260506 03:18:09.206 | INFO | "huge_shared" : " 27 GB", | 20260506 03:18:09.206 | INFO | "malloc" : " 14 GB", | 20260506 03:18:09.206 | INFO | "pinned" : " 29 GB" | 20260506 03:18:09.206 | INFO | } | 20260506 03:18:09.206 | INFO | Perf report for the last 5.14259 seconds: | 20260506 03:18:09.206 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:18:09.206 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:09.207 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:09.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:09.207 | INFO | ANALYSIS : 61.3%, 3.0, 95.8ms, 0 | 20260506 03:18:09.207 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:09.207 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:09.207 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 03:18:09.207 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:09.207 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:09.207 | INFO | Uploaded 36GiB to GPU in 2.05204s (17.5435GiB/s) | 20260506 03:18:09.207 | INFO | Downloaded 34GiB to GPU in 2.01846s (16.8445GiB/s) | 20260506 03:18:09.207 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 03:18:09.207 | INFO | Total mem usage: 15.4 GiB | 20260506 03:18:09.207 | INFO | All Time High Water Marks sum: 16.3 GiB (Savings: 5.2%) | 20260506 03:18:09.207 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -4.42%) | 20260506 03:18:09.207 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 03:18:09.207 | INFO | High Water Breakdown: | 20260506 03:18:09.207 | INFO | 61.7% : 9.11 GiB ( 94.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 03:18:09.207 | INFO | 38.2% : 5.65 GiB ( 85% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 03:18:09.207 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 03:18:09.207 | INFO | ------------------------------------------ | 20260506 03:18:09.207 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:18:09.207 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 03:18:09.207 | INFO | Total mem usage: 26.5 GiB | 20260506 03:18:09.207 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 03:18:09.207 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 03:18:09.207 | INFO | Unused Cache: 1.74 GiB (Waste: 6.58%) | 20260506 03:18:09.207 | INFO | High Water Breakdown: | 20260506 03:18:09.207 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 03:18:09.207 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 03:18:09.207 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 03:18:09.207 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 03:18:09.207 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 03:18:09.207 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 03:18:09.207 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 03:18:09.207 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 03:18:09.207 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 03:18:09.207 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 03:18:09.207 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 03:18:09.207 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 03:18:09.207 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 03:18:09.207 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 03:18:09.207 | INFO | ------------------------------------------ | 20260506 03:18:09.207 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:18:09.207 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 03:18:09.207 | INFO | Total mem usage: 25.6 GiB | 20260506 03:18:09.207 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 03:18:09.207 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 03:18:09.207 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 03:18:09.207 | INFO | High Water Breakdown: | 20260506 03:18:09.208 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 03:18:09.208 | INFO | ------------------------------------------ | 20260506 03:18:09.208 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:18:09.208 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 03:18:09.208 | INFO | Total mem usage: 15.7 GiB | 20260506 03:18:09.208 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 03:18:09.208 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 03:18:09.208 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 03:18:09.208 | INFO | High Water Breakdown: | 20260506 03:18:09.208 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 03:18:09.208 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 03:18:09.208 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 03:18:09.208 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 03:18:09.208 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 03:18:09.208 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 03:18:09.208 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 03:18:09.208 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 03:18:09.208 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 03:18:09.208 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 03:18:09.208 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 03:18:09.208 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 03:18:09.208 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 03:18:09.208 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 03:18:09.208 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 03:18:09.208 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 03:18:09.208 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 03:18:09.208 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 03:18:09.208 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 03:18:09.208 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 03:18:09.208 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 03:18:09.208 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 03:18:09.208 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 03:18:09.208 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 03:18:09.208 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 03:18:09.208 | INFO | ------------------------------------------ | 20260506 03:18:09.208 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:18:09.208 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 03:18:09.208 | INFO | PerfReport: Gathered 1536 samples. | 20260506 03:18:09.208 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 03:18:09.208 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.37e+01, 1.32e+01 / 6.92e-01 | 20260506 03:18:09.208 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.06e-01, 1.06e-02 / 4.72e-03 | 20260506 03:18:09.208 | INFO | PerfReport: 11.86%, Baseline -- 3.71e+00 / 4.54e+00, 5.83e-01 / 2.15e-01 | 20260506 03:18:09.208 | INFO | PerfReport: 3.30%, Histogram -- 1.03e+00 / 1.52e+00, 1.97e-01 / 1.01e-01 | 20260506 03:18:09.208 | INFO | PerfReport: 15.98%, DME -- 4.00e+01 / 4.39e+01, 3.79e+00 / 7.74e-01 | 20260506 03:18:09.208 | INFO | PerfReport: 57.61%, FrameLabeling -- 1.80e+01 / 1.95e+01, 1.60e+00 / 1.29e+00 | 20260506 03:18:09.208 | INFO | PerfReport: 6.95%, PulseAccumulating -- 2.18e+00 / 2.66e+00, 4.10e-01 / 2.89e-01 | 20260506 03:18:09.208 | INFO | PerfReport: 2.97%, Metrics -- 9.28e-01 / 1.18e+00, 2.50e-01 / 1.18e-01 | 20260506 03:18:09.208 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.21e-01 / 3.70e-01, 5.48e-02 / 3.55e-02 | 20260506 03:18:09.208 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 03:18:09.208 | INFO | PerfReport: Gathered 1536 samples. | 20260506 03:18:09.208 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 03:18:09.208 | INFO | PerfReport: Full Execution -- 9.74e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260506 03:18:09.208 | INFO | PerfReport: 22.19%, Upload -- 2.16e+01 / 0.00e+00, 7.11e+00 / 0.00e+00 | 20260506 03:18:09.208 | INFO | PerfReport: 21.97%, Download -- 2.14e+01 / 0.00e+00, 7.58e+00 / 0.00e+00 | 20260506 03:18:09.208 | INFO | PerfReport: 32.13%, Compute -- 3.13e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260506 03:18:09.208 | INFO | PerfReport: 1.95%, LockUpload -- 1.90e+00 / 0.00e+00, 7.75e+00 / 0.00e+00 | 20260506 03:18:09.208 | INFO | PerfReport: 1.26%, LockDownload -- 1.22e+00 / 0.00e+00, 2.83e+00 / 0.00e+00 | 20260506 03:18:09.208 | INFO | PerfReport: 20.50%, LockCompute -- 2.00e+01 / 0.00e+00, 1.73e+01 / 0.00e+00 | 20260506 03:18:09.208 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.33e-02 / 0.00e+00 | 20260506 03:18:09.331 | INFO | Starting a round of Baz IO | 20260506 03:18:12.458 | INFO | Populating RTMetrics report | 20260506 03:18:12.459 | INFO | RTMetrics report populated | 20260506 03:18:12.459 | INFO | Opening tmp RTMetrics json file | 20260506 03:18:12.707 | INFO | Serializing RTMetrics to json | 20260506 03:18:12.729 | INFO | Serialization done, swapping tmp file now... | 20260506 03:18:12.730 | INFO | RTMetrics done | 20260506 03:18:12.742 | INFO | Finished a round of Baz IO: Wrote 1.764 GiB in 3.41 seconds | 20260506 03:18:12.742 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.23% / 0.00% / 78.77% | 20260506 03:18:12.742 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0622352% / 0% / 0.000107288% | 20260506 03:18:12.742 | INFO | Baz IO overall: | 20260506 03:18:12.742 | INFO | BytesWritten : 4749304724912 | 20260506 03:18:12.743 | INFO | NumEvents : 2227507409914 | 20260506 03:18:14.349 | INFO | Analyzing chunk frames = [8527872,8528384) | 20260506 03:18:14.349 | INFO | MemInfo: { | 20260506 03:18:14.349 | INFO | "gpu" : " 17 GB", | 20260506 03:18:14.349 | INFO | "huge" : "", | 20260506 03:18:14.349 | INFO | "huge_shared" : " 27 GB", | 20260506 03:18:14.349 | INFO | "malloc" : " 14 GB", | 20260506 03:18:14.349 | INFO | "pinned" : " 29 GB" | 20260506 03:18:14.349 | INFO | } | 20260506 03:18:14.349 | INFO | Perf report for the last 5.14235 seconds: | 20260506 03:18:14.349 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:18:14.349 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:14.349 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:14.349 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:14.349 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260506 03:18:14.349 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:14.349 | INFO | RT_METRICS : 47.8%, 1.0, 25.1ms, 0 | 20260506 03:18:14.349 | INFO | PRE_HQ : 59.1%, 1.7, 53.6ms, 0 | 20260506 03:18:14.349 | INFO | BAZWRITER : 65.5%, 3.6, 121.9ms, 0 | 20260506 03:18:14.349 | INFO | BAZ_SAMPLING : 5.3%, 1.0, 2.7ms, 0 | 20260506 03:18:14.349 | INFO | Uploaded 36GiB to GPU in 2.14137s (16.8116GiB/s) | 20260506 03:18:14.349 | INFO | Downloaded 38GiB to GPU in 2.22491s (17.0794GiB/s) | 20260506 03:18:19.485 | INFO | Analyzing chunk frames = [8528384,8528896) | 20260506 03:18:19.485 | INFO | MemInfo: { | 20260506 03:18:19.485 | INFO | "gpu" : " 17 GB", | 20260506 03:18:19.485 | INFO | "huge" : "", | 20260506 03:18:19.485 | INFO | "huge_shared" : " 27 GB", | 20260506 03:18:19.485 | INFO | "malloc" : " 14 GB", | 20260506 03:18:19.486 | INFO | "pinned" : " 29 GB" | 20260506 03:18:19.486 | INFO | } | 20260506 03:18:19.486 | INFO | Perf report for the last 5.13675 seconds: | 20260506 03:18:19.486 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:18:19.486 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:19.486 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:19.486 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:19.486 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260506 03:18:19.486 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:19.486 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:19.486 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 03:18:19.486 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:19.486 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:19.486 | INFO | Uploaded 36GiB to GPU in 1.94019s (18.5549GiB/s) | 20260506 03:18:19.486 | INFO | Downloaded 34GiB to GPU in 1.97368s (17.2267GiB/s) | 20260506 03:18:24.626 | INFO | Analyzing chunk frames = [8528896,8529408) | 20260506 03:18:24.626 | INFO | MemInfo: { | 20260506 03:18:24.626 | INFO | "gpu" : " 17 GB", | 20260506 03:18:24.626 | INFO | "huge" : "", | 20260506 03:18:24.626 | INFO | "huge_shared" : " 27 GB", | 20260506 03:18:24.626 | INFO | "malloc" : " 14 GB", | 20260506 03:18:24.626 | INFO | "pinned" : " 29 GB" | 20260506 03:18:24.626 | INFO | } | 20260506 03:18:24.626 | INFO | Perf report for the last 5.14038 seconds: | 20260506 03:18:24.626 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:18:24.626 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:24.626 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:24.626 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:24.626 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260506 03:18:24.626 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:24.626 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:24.626 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:18:24.626 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:24.626 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:24.626 | INFO | Uploaded 36GiB to GPU in 2.11057s (17.057GiB/s) | 20260506 03:18:24.626 | INFO | Downloaded 34GiB to GPU in 2.03988s (16.6676GiB/s) | 20260506 03:18:29.766 | INFO | Analyzing chunk frames = [8529408,8529920) | 20260506 03:18:29.766 | INFO | MemInfo: { | 20260506 03:18:29.766 | INFO | "gpu" : " 17 GB", | 20260506 03:18:29.766 | INFO | "huge" : "", | 20260506 03:18:29.766 | INFO | "huge_shared" : " 27 GB", | 20260506 03:18:29.766 | INFO | "malloc" : " 14 GB", | 20260506 03:18:29.766 | INFO | "pinned" : " 29 GB" | 20260506 03:18:29.766 | INFO | } | 20260506 03:18:29.766 | INFO | Perf report for the last 5.14031 seconds: | 20260506 03:18:29.766 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:18:29.766 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:29.766 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:29.766 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:29.766 | INFO | ANALYSIS : 60.9%, 3.0, 94.6ms, 0 | 20260506 03:18:29.766 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:29.766 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:29.766 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 03:18:29.766 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:29.766 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:29.767 | INFO | Uploaded 36GiB to GPU in 2.09286s (17.2014GiB/s) | 20260506 03:18:29.767 | INFO | Downloaded 34GiB to GPU in 2.01156s (16.9023GiB/s) | 20260506 03:18:34.898 | INFO | Analyzing chunk frames = [8529920,8530432) | 20260506 03:18:34.898 | INFO | MemInfo: { | 20260506 03:18:34.898 | INFO | "gpu" : " 17 GB", | 20260506 03:18:34.898 | INFO | "huge" : "", | 20260506 03:18:34.898 | INFO | "huge_shared" : " 27 GB", | 20260506 03:18:34.898 | INFO | "malloc" : " 14 GB", | 20260506 03:18:34.898 | INFO | "pinned" : " 29 GB" | 20260506 03:18:34.898 | INFO | } | 20260506 03:18:34.898 | INFO | Perf report for the last 5.13181 seconds: | 20260506 03:18:34.898 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:18:34.898 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:34.898 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:34.898 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:34.898 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260506 03:18:34.898 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:34.898 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:34.898 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 03:18:34.898 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:34.898 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:34.898 | INFO | Uploaded 36GiB to GPU in 2.12206s (16.9646GiB/s) | 20260506 03:18:34.898 | INFO | Downloaded 34GiB to GPU in 2.04263s (16.6452GiB/s) | 20260506 03:18:40.036 | INFO | Analyzing chunk frames = [8530432,8530944) | 20260506 03:18:40.036 | INFO | MemInfo: { | 20260506 03:18:40.036 | INFO | "gpu" : " 17 GB", | 20260506 03:18:40.036 | INFO | "huge" : "", | 20260506 03:18:40.036 | INFO | "huge_shared" : " 27 GB", | 20260506 03:18:40.036 | INFO | "malloc" : " 14 GB", | 20260506 03:18:40.037 | INFO | "pinned" : " 29 GB" | 20260506 03:18:40.037 | INFO | } | 20260506 03:18:40.037 | INFO | Perf report for the last 5.13847 seconds: | 20260506 03:18:40.037 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:18:40.037 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:40.037 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:40.037 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:40.037 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 03:18:40.037 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:40.037 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:40.037 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 03:18:40.037 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:40.037 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:40.037 | INFO | Uploaded 36GiB to GPU in 2.07516s (17.348GiB/s) | 20260506 03:18:40.037 | INFO | Downloaded 34GiB to GPU in 2.03716s (16.6899GiB/s) | 20260506 03:18:45.183 | INFO | Analyzing chunk frames = [8530944,8531456) | 20260506 03:18:45.183 | INFO | MemInfo: { | 20260506 03:18:45.183 | INFO | "gpu" : " 17 GB", | 20260506 03:18:45.183 | INFO | "huge" : "", | 20260506 03:18:45.183 | INFO | "huge_shared" : " 27 GB", | 20260506 03:18:45.183 | INFO | "malloc" : " 14 GB", | 20260506 03:18:45.183 | INFO | "pinned" : " 29 GB" | 20260506 03:18:45.183 | INFO | } | 20260506 03:18:45.183 | INFO | Perf report for the last 5.14682 seconds: | 20260506 03:18:45.183 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:18:45.183 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:45.183 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:45.183 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:45.183 | INFO | ANALYSIS : 62.1%, 3.0, 97.2ms, 0 | 20260506 03:18:45.183 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:45.184 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:45.184 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 03:18:45.184 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:45.184 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:45.184 | INFO | Uploaded 36GiB to GPU in 2.04323s (17.6192GiB/s) | 20260506 03:18:45.184 | INFO | Downloaded 34GiB to GPU in 2.06961s (16.4282GiB/s) | 20260506 03:18:50.329 | INFO | Analyzing chunk frames = [8531456,8531968) | 20260506 03:18:50.329 | INFO | MemInfo: { | 20260506 03:18:50.329 | INFO | "gpu" : " 17 GB", | 20260506 03:18:50.329 | INFO | "huge" : "", | 20260506 03:18:50.329 | INFO | "huge_shared" : " 27 GB", | 20260506 03:18:50.329 | INFO | "malloc" : " 14 GB", | 20260506 03:18:50.329 | INFO | "pinned" : " 29 GB" | 20260506 03:18:50.329 | INFO | } | 20260506 03:18:50.329 | INFO | Perf report for the last 5.14546 seconds: | 20260506 03:18:50.329 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:18:50.329 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:50.329 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:50.329 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:50.329 | INFO | ANALYSIS : 61.2%, 3.0, 95.6ms, 0 | 20260506 03:18:50.329 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:50.329 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:50.329 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 03:18:50.329 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:50.329 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:50.329 | INFO | Uploaded 36GiB to GPU in 2.04851s (17.5737GiB/s) | 20260506 03:18:50.329 | INFO | Downloaded 34GiB to GPU in 2.02434s (16.7956GiB/s) | 20260506 03:18:50.460 | INFO | Starting a round of Baz IO | 20260506 03:18:53.575 | INFO | Populating RTMetrics report | 20260506 03:18:53.576 | INFO | RTMetrics report populated | 20260506 03:18:53.576 | INFO | Opening tmp RTMetrics json file | 20260506 03:18:53.577 | INFO | Serializing RTMetrics to json | 20260506 03:18:53.578 | INFO | Serialization done, swapping tmp file now... | 20260506 03:18:53.579 | INFO | RTMetrics done | 20260506 03:18:53.668 | INFO | Finished a round of Baz IO: Wrote 1.765 GiB in 3.208 seconds | 20260506 03:18:53.668 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.23% / 0.00% / 78.77% | 20260506 03:18:53.668 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.061969% / 0% / 6.75519e-05% | 20260506 03:18:53.668 | INFO | Baz IO overall: | 20260506 03:18:53.668 | INFO | BytesWritten : 4751199731045 | 20260506 03:18:53.668 | INFO | NumEvents : 2228366972350 | 20260506 03:18:55.470 | INFO | Analyzing chunk frames = [8531968,8532480) | 20260506 03:18:55.470 | INFO | MemInfo: { | 20260506 03:18:55.470 | INFO | "gpu" : " 17 GB", | 20260506 03:18:55.470 | INFO | "huge" : "", | 20260506 03:18:55.470 | INFO | "huge_shared" : " 27 GB", | 20260506 03:18:55.470 | INFO | "malloc" : " 14 GB", | 20260506 03:18:55.470 | INFO | "pinned" : " 29 GB" | 20260506 03:18:55.470 | INFO | } | 20260506 03:18:55.470 | INFO | Perf report for the last 5.14136 seconds: | 20260506 03:18:55.470 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:18:55.470 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:55.470 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:55.470 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:55.470 | INFO | ANALYSIS : 62.1%, 3.0, 97.1ms, 0 | 20260506 03:18:55.470 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:18:55.470 | INFO | RT_METRICS : 42.6%, 1.0, 22.4ms, 0 | 20260506 03:18:55.470 | INFO | PRE_HQ : 59.0%, 1.7, 53.2ms, 0 | 20260506 03:18:55.470 | INFO | BAZWRITER : 61.7%, 3.6, 112.6ms, 0 | 20260506 03:18:55.470 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 03:18:55.471 | INFO | Uploaded 36GiB to GPU in 2.09993s (17.1434GiB/s) | 20260506 03:18:55.471 | INFO | Downloaded 38GiB to GPU in 2.24774s (16.9059GiB/s) | 20260506 03:19:00.602 | INFO | Analyzing chunk frames = [8532480,8532992) | 20260506 03:19:00.602 | INFO | MemInfo: { | 20260506 03:19:00.602 | INFO | "gpu" : " 17 GB", | 20260506 03:19:00.602 | INFO | "huge" : "", | 20260506 03:19:00.602 | INFO | "huge_shared" : " 27 GB", | 20260506 03:19:00.602 | INFO | "malloc" : " 14 GB", | 20260506 03:19:00.602 | INFO | "pinned" : " 29 GB" | 20260506 03:19:00.602 | INFO | } | 20260506 03:19:00.602 | INFO | Perf report for the last 5.13166 seconds: | 20260506 03:19:00.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:19:00.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:00.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:00.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:00.602 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260506 03:19:00.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:00.602 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:00.602 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 03:19:00.602 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:00.602 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:00.602 | INFO | Uploaded 36GiB to GPU in 1.96794s (18.2932GiB/s) | 20260506 03:19:00.602 | INFO | Downloaded 34GiB to GPU in 1.98687s (17.1123GiB/s) | 20260506 03:19:05.737 | INFO | Analyzing chunk frames = [8532992,8533504) | 20260506 03:19:05.737 | INFO | MemInfo: { | 20260506 03:19:05.737 | INFO | "gpu" : " 17 GB", | 20260506 03:19:05.737 | INFO | "huge" : "", | 20260506 03:19:05.737 | INFO | "huge_shared" : " 27 GB", | 20260506 03:19:05.737 | INFO | "malloc" : " 14 GB", | 20260506 03:19:05.737 | INFO | "pinned" : " 29 GB" | 20260506 03:19:05.737 | INFO | } | 20260506 03:19:05.737 | INFO | Perf report for the last 5.1355 seconds: | 20260506 03:19:05.737 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:19:05.737 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:05.737 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:05.738 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:05.738 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 03:19:05.738 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:05.738 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:05.738 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 03:19:05.738 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:05.738 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:05.738 | INFO | Uploaded 36GiB to GPU in 2.11509s (17.0206GiB/s) | 20260506 03:19:05.738 | INFO | Downloaded 34GiB to GPU in 2.04221s (16.6486GiB/s) | 20260506 03:19:10.880 | INFO | Analyzing chunk frames = [8533504,8534016) | 20260506 03:19:10.880 | INFO | MemInfo: { | 20260506 03:19:10.880 | INFO | "gpu" : " 17 GB", | 20260506 03:19:10.880 | INFO | "huge" : "", | 20260506 03:19:10.880 | INFO | "huge_shared" : " 27 GB", | 20260506 03:19:10.880 | INFO | "malloc" : " 14 GB", | 20260506 03:19:10.880 | INFO | "pinned" : " 29 GB" | 20260506 03:19:10.880 | INFO | } | 20260506 03:19:10.880 | INFO | Perf report for the last 5.14261 seconds: | 20260506 03:19:10.880 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:19:10.880 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:10.880 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:10.880 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:10.880 | INFO | ANALYSIS : 61.2%, 3.0, 95.0ms, 0 | 20260506 03:19:10.880 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:10.880 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:10.880 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 03:19:10.880 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:10.880 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:10.880 | INFO | Uploaded 36GiB to GPU in 2.10013s (17.1418GiB/s) | 20260506 03:19:10.881 | INFO | Downloaded 34GiB to GPU in 2.01537s (16.8703GiB/s) | 20260506 03:19:16.019 | INFO | Analyzing chunk frames = [8534016,8534528) | 20260506 03:19:16.019 | INFO | MemInfo: { | 20260506 03:19:16.019 | INFO | "gpu" : " 17 GB", | 20260506 03:19:16.019 | INFO | "huge" : "", | 20260506 03:19:16.019 | INFO | "huge_shared" : " 27 GB", | 20260506 03:19:16.019 | INFO | "malloc" : " 14 GB", | 20260506 03:19:16.019 | INFO | "pinned" : " 29 GB" | 20260506 03:19:16.019 | INFO | } | 20260506 03:19:16.019 | INFO | Perf report for the last 5.13866 seconds: | 20260506 03:19:16.019 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:19:16.019 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:16.019 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:16.019 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:16.019 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 03:19:16.019 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:16.019 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:16.019 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 03:19:16.019 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:16.019 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:16.019 | INFO | Uploaded 36GiB to GPU in 2.11213s (17.0444GiB/s) | 20260506 03:19:16.019 | INFO | Downloaded 34GiB to GPU in 2.03266s (16.7268GiB/s) | 20260506 03:19:21.161 | INFO | Analyzing chunk frames = [8534528,8535040) | 20260506 03:19:21.161 | INFO | MemInfo: { | 20260506 03:19:21.161 | INFO | "gpu" : " 17 GB", | 20260506 03:19:21.161 | INFO | "huge" : "", | 20260506 03:19:21.161 | INFO | "huge_shared" : " 27 GB", | 20260506 03:19:21.161 | INFO | "malloc" : " 14 GB", | 20260506 03:19:21.162 | INFO | "pinned" : " 29 GB" | 20260506 03:19:21.162 | INFO | } | 20260506 03:19:21.162 | INFO | Perf report for the last 5.14289 seconds: | 20260506 03:19:21.162 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:19:21.162 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:21.162 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:21.162 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:21.162 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 03:19:21.162 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:21.162 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:21.162 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 03:19:21.162 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:21.162 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:21.162 | INFO | Uploaded 36GiB to GPU in 2.10212s (17.1256GiB/s) | 20260506 03:19:21.162 | INFO | Downloaded 34GiB to GPU in 2.04099s (16.6585GiB/s) | 20260506 03:19:26.298 | INFO | Analyzing chunk frames = [8535040,8535552) | 20260506 03:19:26.298 | INFO | MemInfo: { | 20260506 03:19:26.298 | INFO | "gpu" : " 17 GB", | 20260506 03:19:26.298 | INFO | "huge" : "", | 20260506 03:19:26.298 | INFO | "huge_shared" : " 27 GB", | 20260506 03:19:26.298 | INFO | "malloc" : " 14 GB", | 20260506 03:19:26.298 | INFO | "pinned" : " 29 GB" | 20260506 03:19:26.298 | INFO | } | 20260506 03:19:26.298 | INFO | Perf report for the last 5.13675 seconds: | 20260506 03:19:26.298 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:19:26.298 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:26.298 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:26.298 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:26.299 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260506 03:19:26.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:26.299 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:26.299 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 03:19:26.299 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:26.299 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:26.299 | INFO | Uploaded 36GiB to GPU in 2.11182s (17.0469GiB/s) | 20260506 03:19:26.299 | INFO | Downloaded 34GiB to GPU in 2.0905s (16.2641GiB/s) | 20260506 03:19:31.439 | INFO | Analyzing chunk frames = [8535552,8536064) | 20260506 03:19:31.439 | INFO | MemInfo: { | 20260506 03:19:31.439 | INFO | "gpu" : " 17 GB", | 20260506 03:19:31.439 | INFO | "huge" : "", | 20260506 03:19:31.439 | INFO | "huge_shared" : " 27 GB", | 20260506 03:19:31.439 | INFO | "malloc" : " 14 GB", | 20260506 03:19:31.439 | INFO | "pinned" : " 29 GB" | 20260506 03:19:31.439 | INFO | } | 20260506 03:19:31.439 | INFO | Perf report for the last 5.14054 seconds: | 20260506 03:19:31.439 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:19:31.439 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:31.439 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:31.439 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:31.439 | INFO | ANALYSIS : 61.4%, 3.0, 95.9ms, 0 | 20260506 03:19:31.439 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:31.439 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:31.439 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 03:19:31.439 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:31.439 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:31.439 | INFO | Uploaded 36GiB to GPU in 2.04964s (17.5641GiB/s) | 20260506 03:19:31.439 | INFO | Downloaded 34GiB to GPU in 2.02374s (16.8006GiB/s) | 20260506 03:19:31.439 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 03:19:31.439 | INFO | Total mem usage: 15.4 GiB | 20260506 03:19:31.439 | INFO | All Time High Water Marks sum: 16.3 GiB (Savings: 5.2%) | 20260506 03:19:31.439 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -9.42%) | 20260506 03:19:31.439 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 03:19:31.440 | INFO | High Water Breakdown: | 20260506 03:19:31.440 | INFO | 60.9% : 8.59 GiB ( 89.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 03:19:31.440 | INFO | 39% : 5.5 GiB ( 82.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 03:19:31.440 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 03:19:31.440 | INFO | ------------------------------------------ | 20260506 03:19:31.440 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:19:31.440 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 03:19:31.440 | INFO | Total mem usage: 26.5 GiB | 20260506 03:19:31.440 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 03:19:31.440 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 03:19:31.440 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 03:19:31.440 | INFO | High Water Breakdown: | 20260506 03:19:31.440 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 03:19:31.440 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 03:19:31.440 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 03:19:31.440 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 03:19:31.440 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 03:19:31.440 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 03:19:31.440 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 03:19:31.440 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 03:19:31.440 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 03:19:31.440 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 03:19:31.440 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 03:19:31.440 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 03:19:31.440 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 03:19:31.440 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 03:19:31.440 | INFO | ------------------------------------------ | 20260506 03:19:31.440 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:19:31.440 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 03:19:31.440 | INFO | Total mem usage: 25.6 GiB | 20260506 03:19:31.440 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 03:19:31.440 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260506 03:19:31.440 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260506 03:19:31.440 | INFO | High Water Breakdown: | 20260506 03:19:31.440 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260506 03:19:31.440 | INFO | ------------------------------------------ | 20260506 03:19:31.440 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:19:31.440 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 03:19:31.440 | INFO | Total mem usage: 15.7 GiB | 20260506 03:19:31.440 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 03:19:31.440 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 03:19:31.440 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 03:19:31.440 | INFO | High Water Breakdown: | 20260506 03:19:31.440 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 03:19:31.440 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 03:19:31.440 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 03:19:31.440 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 03:19:31.440 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 03:19:31.440 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 03:19:31.440 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 03:19:31.440 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 03:19:31.440 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 03:19:31.440 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 03:19:31.440 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 03:19:31.440 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 03:19:31.440 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 03:19:31.440 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 03:19:31.440 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 03:19:31.440 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 03:19:31.440 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 03:19:31.440 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 03:19:31.440 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 03:19:31.440 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 03:19:31.440 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 03:19:31.440 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 03:19:31.440 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 03:19:31.440 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 03:19:31.440 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 03:19:31.440 | INFO | ------------------------------------------ | 20260506 03:19:31.440 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:19:31.440 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 03:19:31.440 | INFO | PerfReport: Gathered 1536 samples. | 20260506 03:19:31.440 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 03:19:31.440 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.37e+01, 1.33e+01 / 8.79e-01 | 20260506 03:19:31.440 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.05e-01, 1.04e-02 / 5.28e-03 | 20260506 03:19:31.440 | INFO | PerfReport: 11.86%, Baseline -- 3.71e+00 / 4.47e+00, 5.81e-01 / 2.85e-01 | 20260506 03:19:31.440 | INFO | PerfReport: 3.30%, Histogram -- 1.03e+00 / 1.52e+00, 1.95e-01 / 1.14e-01 | 20260506 03:19:31.440 | INFO | PerfReport: 15.99%, DME -- 4.00e+01 / 4.40e+01, 3.86e+00 / 1.09e+00 | 20260506 03:19:31.440 | INFO | PerfReport: 57.57%, FrameLabeling -- 1.80e+01 / 1.93e+01, 1.54e+00 / 9.53e-01 | 20260506 03:19:31.440 | INFO | PerfReport: 6.98%, PulseAccumulating -- 2.18e+00 / 2.72e+00, 4.12e-01 / 2.01e-01 | 20260506 03:19:31.440 | INFO | PerfReport: 2.98%, Metrics -- 9.31e-01 / 1.19e+00, 2.52e-01 / 9.70e-02 | 20260506 03:19:31.440 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.21e-01 / 3.72e-01, 5.44e-02 / 2.86e-02 | 20260506 03:19:31.440 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 03:19:31.440 | INFO | PerfReport: Gathered 1536 samples. | 20260506 03:19:31.440 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 03:19:31.440 | INFO | PerfReport: Full Execution -- 9.76e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260506 03:19:31.440 | INFO | PerfReport: 22.17%, Upload -- 2.16e+01 / 0.00e+00, 7.19e+00 / 0.00e+00 | 20260506 03:19:31.440 | INFO | PerfReport: 21.95%, Download -- 2.14e+01 / 0.00e+00, 7.61e+00 / 0.00e+00 | 20260506 03:19:31.440 | INFO | PerfReport: 32.03%, Compute -- 3.13e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 03:19:31.440 | INFO | PerfReport: 2.00%, LockUpload -- 1.95e+00 / 0.00e+00, 7.83e+00 / 0.00e+00 | 20260506 03:19:31.440 | INFO | PerfReport: 1.35%, LockDownload -- 1.32e+00 / 0.00e+00, 3.05e+00 / 0.00e+00 | 20260506 03:19:31.440 | INFO | PerfReport: 20.48%, LockCompute -- 2.00e+01 / 0.00e+00, 1.78e+01 / 0.00e+00 | 20260506 03:19:31.440 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.81e-02 / 0.00e+00 | 20260506 03:19:31.561 | INFO | Starting a round of Baz IO | 20260506 03:19:34.679 | INFO | Populating RTMetrics report | 20260506 03:19:34.679 | INFO | RTMetrics report populated | 20260506 03:19:34.679 | INFO | Opening tmp RTMetrics json file | 20260506 03:19:34.680 | INFO | Serializing RTMetrics to json | 20260506 03:19:34.682 | INFO | Serialization done, swapping tmp file now... | 20260506 03:19:34.682 | INFO | RTMetrics done | 20260506 03:19:34.751 | INFO | Finished a round of Baz IO: Wrote 1.763 GiB in 3.189 seconds | 20260506 03:19:34.751 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.23% / 0.00% / 78.77% | 20260506 03:19:34.751 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0622869% / 0% / 4.76837e-05% | 20260506 03:19:34.751 | INFO | Baz IO overall: | 20260506 03:19:34.751 | INFO | BytesWritten : 4753093007122 | 20260506 03:19:34.751 | INFO | NumEvents : 2229225490024 | 20260506 03:19:36.576 | INFO | Analyzing chunk frames = [8536064,8536576) | 20260506 03:19:36.576 | INFO | MemInfo: { | 20260506 03:19:36.576 | INFO | "gpu" : " 17 GB", | 20260506 03:19:36.576 | INFO | "huge" : "", | 20260506 03:19:36.576 | INFO | "huge_shared" : " 27 GB", | 20260506 03:19:36.576 | INFO | "malloc" : " 14 GB", | 20260506 03:19:36.576 | INFO | "pinned" : " 29 GB" | 20260506 03:19:36.576 | INFO | } | 20260506 03:19:36.576 | INFO | Perf report for the last 5.13698 seconds: | 20260506 03:19:36.576 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:19:36.576 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:36.576 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:36.576 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:36.576 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 03:19:36.576 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:36.576 | INFO | RT_METRICS : 42.6%, 1.0, 22.3ms, 0 | 20260506 03:19:36.576 | INFO | PRE_HQ : 59.1%, 1.7, 53.3ms, 0 | 20260506 03:19:36.576 | INFO | BAZWRITER : 61.3%, 3.5, 110.5ms, 0 | 20260506 03:19:36.576 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 03:19:36.576 | INFO | Uploaded 36GiB to GPU in 2.09577s (17.1774GiB/s) | 20260506 03:19:36.576 | INFO | Downloaded 38GiB to GPU in 2.22595s (17.0713GiB/s) | 20260506 03:19:41.712 | INFO | Analyzing chunk frames = [8536576,8537088) | 20260506 03:19:41.712 | INFO | MemInfo: { | 20260506 03:19:41.712 | INFO | "gpu" : " 17 GB", | 20260506 03:19:41.712 | INFO | "huge" : "", | 20260506 03:19:41.712 | INFO | "huge_shared" : " 27 GB", | 20260506 03:19:41.712 | INFO | "malloc" : " 14 GB", | 20260506 03:19:41.712 | INFO | "pinned" : " 29 GB" | 20260506 03:19:41.712 | INFO | } | 20260506 03:19:41.712 | INFO | Perf report for the last 5.13588 seconds: | 20260506 03:19:41.712 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:19:41.712 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:41.712 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:41.712 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:41.712 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 03:19:41.712 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:41.712 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:41.712 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 03:19:41.712 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:41.712 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:41.712 | INFO | Uploaded 36GiB to GPU in 2.02201s (17.804GiB/s) | 20260506 03:19:41.712 | INFO | Downloaded 34GiB to GPU in 1.99663s (17.0287GiB/s) | 20260506 03:19:46.848 | INFO | Analyzing chunk frames = [8537088,8537600) | 20260506 03:19:46.848 | INFO | MemInfo: { | 20260506 03:19:46.848 | INFO | "gpu" : " 17 GB", | 20260506 03:19:46.848 | INFO | "huge" : "", | 20260506 03:19:46.848 | INFO | "huge_shared" : " 27 GB", | 20260506 03:19:46.848 | INFO | "malloc" : " 14 GB", | 20260506 03:19:46.848 | INFO | "pinned" : " 29 GB" | 20260506 03:19:46.848 | INFO | } | 20260506 03:19:46.848 | INFO | Perf report for the last 5.13651 seconds: | 20260506 03:19:46.848 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:19:46.848 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:46.848 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:46.848 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:46.848 | INFO | ANALYSIS : 61.2%, 3.0, 95.1ms, 0 | 20260506 03:19:46.848 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:46.849 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:46.849 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:19:46.849 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:46.849 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:46.849 | INFO | Uploaded 36GiB to GPU in 2.1105s (17.0575GiB/s) | 20260506 03:19:46.849 | INFO | Downloaded 34GiB to GPU in 2.04409s (16.6334GiB/s) | 20260506 03:19:51.988 | INFO | Analyzing chunk frames = [8537600,8538112) | 20260506 03:19:51.988 | INFO | MemInfo: { | 20260506 03:19:51.988 | INFO | "gpu" : " 17 GB", | 20260506 03:19:51.988 | INFO | "huge" : "", | 20260506 03:19:51.988 | INFO | "huge_shared" : " 27 GB", | 20260506 03:19:51.988 | INFO | "malloc" : " 14 GB", | 20260506 03:19:51.988 | INFO | "pinned" : " 29 GB" | 20260506 03:19:51.988 | INFO | } | 20260506 03:19:51.988 | INFO | Perf report for the last 5.13998 seconds: | 20260506 03:19:51.988 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:19:51.988 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:51.988 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:51.988 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:51.988 | INFO | ANALYSIS : 61.5%, 3.0, 95.5ms, 0 | 20260506 03:19:51.988 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:51.988 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:51.989 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:19:51.989 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:51.989 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:51.989 | INFO | Uploaded 36GiB to GPU in 2.15519s (16.7039GiB/s) | 20260506 03:19:51.989 | INFO | Downloaded 34GiB to GPU in 2.05336s (16.5583GiB/s) | 20260506 03:19:57.131 | INFO | Analyzing chunk frames = [8538112,8538624) | 20260506 03:19:57.131 | INFO | MemInfo: { | 20260506 03:19:57.131 | INFO | "gpu" : " 17 GB", | 20260506 03:19:57.131 | INFO | "huge" : "", | 20260506 03:19:57.131 | INFO | "huge_shared" : " 27 GB", | 20260506 03:19:57.131 | INFO | "malloc" : " 14 GB", | 20260506 03:19:57.131 | INFO | "pinned" : " 29 GB" | 20260506 03:19:57.131 | INFO | } | 20260506 03:19:57.131 | INFO | Perf report for the last 5.14309 seconds: | 20260506 03:19:57.131 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:19:57.131 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:57.131 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:57.132 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:57.132 | INFO | ANALYSIS : 60.6%, 3.0, 94.7ms, 0 | 20260506 03:19:57.132 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:57.132 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:57.132 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 03:19:57.132 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:57.132 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:19:57.132 | INFO | Uploaded 36GiB to GPU in 2.08259s (17.2862GiB/s) | 20260506 03:19:57.132 | INFO | Downloaded 34GiB to GPU in 2.03085s (16.7418GiB/s) | 20260506 03:20:02.281 | INFO | Analyzing chunk frames = [8538624,8539136) | 20260506 03:20:02.281 | INFO | MemInfo: { | 20260506 03:20:02.281 | INFO | "gpu" : " 17 GB", | 20260506 03:20:02.281 | INFO | "huge" : "", | 20260506 03:20:02.281 | INFO | "huge_shared" : " 27 GB", | 20260506 03:20:02.281 | INFO | "malloc" : " 14 GB", | 20260506 03:20:02.281 | INFO | "pinned" : " 29 GB" | 20260506 03:20:02.281 | INFO | } | 20260506 03:20:02.281 | INFO | Perf report for the last 5.14944 seconds: | 20260506 03:20:02.281 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:20:02.281 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:02.281 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:02.281 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:02.281 | INFO | ANALYSIS : 60.5%, 3.0, 94.7ms, 0 | 20260506 03:20:02.281 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:02.281 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:02.281 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 03:20:02.281 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:02.281 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:02.281 | INFO | Uploaded 36GiB to GPU in 2.05169s (17.5465GiB/s) | 20260506 03:20:02.281 | INFO | Downloaded 34GiB to GPU in 2.03039s (16.7456GiB/s) | 20260506 03:20:07.420 | INFO | Analyzing chunk frames = [8539136,8539648) | 20260506 03:20:07.420 | INFO | MemInfo: { | 20260506 03:20:07.420 | INFO | "gpu" : " 17 GB", | 20260506 03:20:07.420 | INFO | "huge" : "", | 20260506 03:20:07.420 | INFO | "huge_shared" : " 27 GB", | 20260506 03:20:07.420 | INFO | "malloc" : " 14 GB", | 20260506 03:20:07.420 | INFO | "pinned" : " 29 GB" | 20260506 03:20:07.420 | INFO | } | 20260506 03:20:07.420 | INFO | Perf report for the last 5.13891 seconds: | 20260506 03:20:07.420 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:20:07.420 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:07.420 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:07.420 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:07.420 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 03:20:07.420 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:07.420 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:07.420 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 03:20:07.420 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:07.420 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:07.420 | INFO | Uploaded 36GiB to GPU in 2.08279s (17.2845GiB/s) | 20260506 03:20:07.420 | INFO | Downloaded 34GiB to GPU in 2.05135s (16.5745GiB/s) | 20260506 03:20:12.552 | INFO | Analyzing chunk frames = [8539648,8540160) | 20260506 03:20:12.552 | INFO | MemInfo: { | 20260506 03:20:12.552 | INFO | "gpu" : " 17 GB", | 20260506 03:20:12.552 | INFO | "huge" : "", | 20260506 03:20:12.552 | INFO | "huge_shared" : " 27 GB", | 20260506 03:20:12.552 | INFO | "malloc" : " 14 GB", | 20260506 03:20:12.552 | INFO | "pinned" : " 29 GB" | 20260506 03:20:12.552 | INFO | } | 20260506 03:20:12.552 | INFO | Perf report for the last 5.1319 seconds: | 20260506 03:20:12.552 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:20:12.552 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:12.552 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:12.552 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:12.552 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 03:20:12.552 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:12.552 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:12.552 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:20:12.552 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:12.552 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:12.552 | INFO | Uploaded 36GiB to GPU in 2.06905s (17.3993GiB/s) | 20260506 03:20:12.552 | INFO | Downloaded 34GiB to GPU in 2.04483s (16.6273GiB/s) | 20260506 03:20:12.677 | INFO | Starting a round of Baz IO | 20260506 03:20:15.806 | INFO | Populating RTMetrics report | 20260506 03:20:15.807 | INFO | RTMetrics report populated | 20260506 03:20:15.807 | INFO | Opening tmp RTMetrics json file | 20260506 03:20:15.808 | INFO | Serializing RTMetrics to json | 20260506 03:20:15.810 | INFO | Serialization done, swapping tmp file now... | 20260506 03:20:15.811 | INFO | RTMetrics done | 20260506 03:20:15.898 | INFO | Finished a round of Baz IO: Wrote 1.764 GiB in 3.221 seconds | 20260506 03:20:15.898 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.23% / 0.00% / 78.77% | 20260506 03:20:15.898 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0624299% / 0% / 8.34465e-05% | 20260506 03:20:15.898 | INFO | Baz IO overall: | 20260506 03:20:15.898 | INFO | BytesWritten : 4754986993102 | 20260506 03:20:15.898 | INFO | NumEvents : 2230084576464 | 20260506 03:20:17.691 | INFO | Analyzing chunk frames = [8540160,8540672) | 20260506 03:20:17.691 | INFO | MemInfo: { | 20260506 03:20:17.691 | INFO | "gpu" : " 17 GB", | 20260506 03:20:17.691 | INFO | "huge" : "", | 20260506 03:20:17.691 | INFO | "huge_shared" : " 27 GB", | 20260506 03:20:17.691 | INFO | "malloc" : " 14 GB", | 20260506 03:20:17.691 | INFO | "pinned" : " 29 GB" | 20260506 03:20:17.691 | INFO | } | 20260506 03:20:17.691 | INFO | Perf report for the last 5.13963 seconds: | 20260506 03:20:17.691 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:20:17.691 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:17.691 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:17.691 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:17.691 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260506 03:20:17.691 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:17.691 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260506 03:20:17.691 | INFO | PRE_HQ : 59.0%, 1.7, 53.3ms, 0 | 20260506 03:20:17.691 | INFO | BAZWRITER : 61.8%, 3.5, 110.1ms, 0 | 20260506 03:20:17.691 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 03:20:17.692 | INFO | Uploaded 36GiB to GPU in 2.10805s (17.0774GiB/s) | 20260506 03:20:17.692 | INFO | Downloaded 38GiB to GPU in 2.21898s (17.125GiB/s) | 20260506 03:20:22.831 | INFO | Analyzing chunk frames = [8540672,8541184) | 20260506 03:20:22.831 | INFO | MemInfo: { | 20260506 03:20:22.831 | INFO | "gpu" : " 17 GB", | 20260506 03:20:22.831 | INFO | "huge" : "", | 20260506 03:20:22.831 | INFO | "huge_shared" : " 27 GB", | 20260506 03:20:22.831 | INFO | "malloc" : " 14 GB", | 20260506 03:20:22.831 | INFO | "pinned" : " 29 GB" | 20260506 03:20:22.831 | INFO | } | 20260506 03:20:22.831 | INFO | Perf report for the last 5.13931 seconds: | 20260506 03:20:22.831 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:20:22.831 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:22.831 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:22.831 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:22.831 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260506 03:20:22.831 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:22.831 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:22.831 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 03:20:22.831 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:22.831 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:22.831 | INFO | Uploaded 36GiB to GPU in 1.99106s (18.0808GiB/s) | 20260506 03:20:22.831 | INFO | Downloaded 34GiB to GPU in 1.99427s (17.0488GiB/s) | 20260506 03:20:27.977 | INFO | Analyzing chunk frames = [8541184,8541696) | 20260506 03:20:27.977 | INFO | MemInfo: { | 20260506 03:20:27.977 | INFO | "gpu" : " 17 GB", | 20260506 03:20:27.977 | INFO | "huge" : "", | 20260506 03:20:27.977 | INFO | "huge_shared" : " 27 GB", | 20260506 03:20:27.977 | INFO | "malloc" : " 14 GB", | 20260506 03:20:27.977 | INFO | "pinned" : " 29 GB" | 20260506 03:20:27.977 | INFO | } | 20260506 03:20:27.977 | INFO | Perf report for the last 5.14626 seconds: | 20260506 03:20:27.977 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:20:27.977 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:27.977 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:27.977 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:27.977 | INFO | ANALYSIS : 60.4%, 3.0, 94.1ms, 0 | 20260506 03:20:27.977 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:27.977 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:27.977 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 03:20:27.977 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:27.977 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:27.977 | INFO | Uploaded 36GiB to GPU in 1.98359s (18.1489GiB/s) | 20260506 03:20:27.977 | INFO | Downloaded 34GiB to GPU in 1.99317s (17.0583GiB/s) | 20260506 03:20:33.109 | INFO | Analyzing chunk frames = [8541696,8542208) | 20260506 03:20:33.109 | INFO | MemInfo: { | 20260506 03:20:33.109 | INFO | "gpu" : " 17 GB", | 20260506 03:20:33.109 | INFO | "huge" : "", | 20260506 03:20:33.109 | INFO | "huge_shared" : " 27 GB", | 20260506 03:20:33.109 | INFO | "malloc" : " 14 GB", | 20260506 03:20:33.109 | INFO | "pinned" : " 29 GB" | 20260506 03:20:33.109 | INFO | } | 20260506 03:20:33.109 | INFO | Perf report for the last 5.13202 seconds: | 20260506 03:20:33.109 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:20:33.109 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:33.109 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:33.109 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:33.109 | INFO | ANALYSIS : 61.0%, 3.0, 94.6ms, 0 | 20260506 03:20:33.109 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:33.109 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:33.109 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 03:20:33.109 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:33.109 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:33.109 | INFO | Uploaded 36GiB to GPU in 2.09596s (17.1759GiB/s) | 20260506 03:20:33.109 | INFO | Downloaded 34GiB to GPU in 2.01239s (16.8953GiB/s) | 20260506 03:20:38.245 | INFO | Analyzing chunk frames = [8542208,8542720) | 20260506 03:20:38.245 | INFO | MemInfo: { | 20260506 03:20:38.245 | INFO | "gpu" : " 17 GB", | 20260506 03:20:38.245 | INFO | "huge" : "", | 20260506 03:20:38.245 | INFO | "huge_shared" : " 27 GB", | 20260506 03:20:38.245 | INFO | "malloc" : " 14 GB", | 20260506 03:20:38.245 | INFO | "pinned" : " 29 GB" | 20260506 03:20:38.245 | INFO | } | 20260506 03:20:38.246 | INFO | Perf report for the last 5.13647 seconds: | 20260506 03:20:38.246 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:20:38.246 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:38.246 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:38.246 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:38.246 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 03:20:38.246 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:38.246 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:38.246 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:20:38.246 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:38.246 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:38.246 | INFO | Uploaded 36GiB to GPU in 2.07021s (17.3895GiB/s) | 20260506 03:20:38.246 | INFO | Downloaded 34GiB to GPU in 2.03503s (16.7073GiB/s) | 20260506 03:20:43.387 | INFO | Analyzing chunk frames = [8542720,8543232) | 20260506 03:20:43.387 | INFO | MemInfo: { | 20260506 03:20:43.387 | INFO | "gpu" : " 17 GB", | 20260506 03:20:43.387 | INFO | "huge" : "", | 20260506 03:20:43.387 | INFO | "huge_shared" : " 27 GB", | 20260506 03:20:43.387 | INFO | "malloc" : " 14 GB", | 20260506 03:20:43.387 | INFO | "pinned" : " 29 GB" | 20260506 03:20:43.387 | INFO | } | 20260506 03:20:43.387 | INFO | Perf report for the last 5.14158 seconds: | 20260506 03:20:43.387 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:20:43.387 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:43.387 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:43.387 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:43.387 | INFO | ANALYSIS : 60.9%, 3.0, 95.1ms, 0 | 20260506 03:20:43.387 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:43.387 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:43.387 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:20:43.387 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:43.387 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:43.387 | INFO | Uploaded 36GiB to GPU in 2.10006s (17.1424GiB/s) | 20260506 03:20:43.387 | INFO | Downloaded 34GiB to GPU in 2.0427s (16.6447GiB/s) | 20260506 03:20:48.528 | INFO | Analyzing chunk frames = [8543232,8543744) | 20260506 03:20:48.528 | INFO | MemInfo: { | 20260506 03:20:48.528 | INFO | "gpu" : " 17 GB", | 20260506 03:20:48.528 | INFO | "huge" : "", | 20260506 03:20:48.528 | INFO | "huge_shared" : " 27 GB", | 20260506 03:20:48.528 | INFO | "malloc" : " 14 GB", | 20260506 03:20:48.528 | INFO | "pinned" : " 29 GB" | 20260506 03:20:48.528 | INFO | } | 20260506 03:20:48.528 | INFO | Perf report for the last 5.14118 seconds: | 20260506 03:20:48.528 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:20:48.528 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:48.528 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:48.529 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:48.529 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 03:20:48.529 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:48.529 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:48.529 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260506 03:20:48.529 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:48.529 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:48.529 | INFO | Uploaded 36GiB to GPU in 2.06956s (17.395GiB/s) | 20260506 03:20:48.529 | INFO | Downloaded 34GiB to GPU in 2.05344s (16.5576GiB/s) | 20260506 03:20:53.661 | INFO | Analyzing chunk frames = [8543744,8544256) | 20260506 03:20:53.661 | INFO | MemInfo: { | 20260506 03:20:53.661 | INFO | "gpu" : " 17 GB", | 20260506 03:20:53.661 | INFO | "huge" : "", | 20260506 03:20:53.661 | INFO | "huge_shared" : " 27 GB", | 20260506 03:20:53.661 | INFO | "malloc" : " 14 GB", | 20260506 03:20:53.661 | INFO | "pinned" : " 29 GB" | 20260506 03:20:53.661 | INFO | } | 20260506 03:20:53.661 | INFO | Perf report for the last 5.13267 seconds: | 20260506 03:20:53.661 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:20:53.661 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:53.661 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:53.661 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:53.661 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 03:20:53.661 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:53.661 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:53.661 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 03:20:53.661 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:53.661 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:53.661 | INFO | Uploaded 36GiB to GPU in 2.03372s (17.7016GiB/s) | 20260506 03:20:53.661 | INFO | Downloaded 34GiB to GPU in 2.02444s (16.7948GiB/s) | 20260506 03:20:53.661 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 03:20:53.661 | INFO | Total mem usage: 15.4 GiB | 20260506 03:20:53.661 | INFO | All Time High Water Marks sum: 16.3 GiB (Savings: 5.2%) | 20260506 03:20:53.661 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -10%) | 20260506 03:20:53.661 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 03:20:53.662 | INFO | High Water Breakdown: | 20260506 03:20:53.662 | INFO | 60.9% : 8.54 GiB ( 88.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 03:20:53.662 | INFO | 39% : 5.47 GiB ( 82.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 03:20:53.662 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 03:20:53.662 | INFO | ------------------------------------------ | 20260506 03:20:53.662 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:20:53.662 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 03:20:53.662 | INFO | Total mem usage: 26.5 GiB | 20260506 03:20:53.662 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 03:20:53.662 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 03:20:53.662 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 03:20:53.662 | INFO | High Water Breakdown: | 20260506 03:20:53.662 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 03:20:53.662 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 03:20:53.662 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 03:20:53.662 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 03:20:53.662 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 03:20:53.662 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 03:20:53.662 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 03:20:53.662 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 03:20:53.662 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 03:20:53.662 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 03:20:53.662 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 03:20:53.662 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 03:20:53.662 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 03:20:53.662 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 03:20:53.662 | INFO | ------------------------------------------ | 20260506 03:20:53.662 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:20:53.662 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 03:20:53.662 | INFO | Total mem usage: 25.6 GiB | 20260506 03:20:53.662 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 03:20:53.662 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 03:20:53.662 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 03:20:53.662 | INFO | High Water Breakdown: | 20260506 03:20:53.662 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 03:20:53.662 | INFO | ------------------------------------------ | 20260506 03:20:53.662 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:20:53.662 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 03:20:53.662 | INFO | Total mem usage: 15.7 GiB | 20260506 03:20:53.662 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 03:20:53.662 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 03:20:53.662 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 03:20:53.662 | INFO | High Water Breakdown: | 20260506 03:20:53.662 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 03:20:53.662 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 03:20:53.662 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 03:20:53.662 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 03:20:53.662 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 03:20:53.662 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 03:20:53.662 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 03:20:53.662 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 03:20:53.662 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 03:20:53.662 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 03:20:53.662 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 03:20:53.662 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 03:20:53.662 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 03:20:53.662 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 03:20:53.662 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 03:20:53.662 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 03:20:53.662 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 03:20:53.662 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 03:20:53.662 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 03:20:53.662 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 03:20:53.662 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 03:20:53.662 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 03:20:53.662 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 03:20:53.662 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 03:20:53.662 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 03:20:53.662 | INFO | ------------------------------------------ | 20260506 03:20:53.662 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:20:53.662 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 03:20:53.662 | INFO | PerfReport: Gathered 1536 samples. | 20260506 03:20:53.662 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 03:20:53.662 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.36e+01, 1.35e+01 / 5.87e-01 | 20260506 03:20:53.662 | INFO | PerfReport: 0.29%, AggregateStats -- 9.16e-02 / 1.04e-01, 1.01e-02 / 3.02e-03 | 20260506 03:20:53.662 | INFO | PerfReport: 11.89%, Baseline -- 3.72e+00 / 4.55e+00, 5.90e-01 / 2.40e-01 | 20260506 03:20:53.662 | INFO | PerfReport: 3.31%, Histogram -- 1.03e+00 / 1.54e+00, 1.98e-01 / 7.91e-02 | 20260506 03:20:53.662 | INFO | PerfReport: 15.99%, DME -- 4.00e+01 / 4.39e+01, 3.69e+00 / 7.68e-01 | 20260506 03:20:53.662 | INFO | PerfReport: 57.53%, FrameLabeling -- 1.80e+01 / 1.92e+01, 1.54e+00 / 9.37e-01 | 20260506 03:20:53.662 | INFO | PerfReport: 6.99%, PulseAccumulating -- 2.18e+00 / 2.75e+00, 4.17e-01 / 1.82e-01 | 20260506 03:20:53.662 | INFO | PerfReport: 2.98%, Metrics -- 9.31e-01 / 1.20e+00, 2.51e-01 / 1.24e-01 | 20260506 03:20:53.662 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.21e-01 / 3.70e-01, 5.42e-02 / 3.14e-02 | 20260506 03:20:53.662 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 03:20:53.662 | INFO | PerfReport: Gathered 1536 samples. | 20260506 03:20:53.662 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 03:20:53.662 | INFO | PerfReport: Full Execution -- 9.74e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260506 03:20:53.662 | INFO | PerfReport: 22.15%, Upload -- 2.16e+01 / 0.00e+00, 7.03e+00 / 0.00e+00 | 20260506 03:20:53.662 | INFO | PerfReport: 21.97%, Download -- 2.14e+01 / 0.00e+00, 7.59e+00 / 0.00e+00 | 20260506 03:20:53.662 | INFO | PerfReport: 32.12%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 03:20:53.662 | INFO | PerfReport: 1.93%, LockUpload -- 1.88e+00 / 0.00e+00, 7.61e+00 / 0.00e+00 | 20260506 03:20:53.662 | INFO | PerfReport: 1.30%, LockDownload -- 1.27e+00 / 0.00e+00, 2.99e+00 / 0.00e+00 | 20260506 03:20:53.662 | INFO | PerfReport: 20.53%, LockCompute -- 2.00e+01 / 0.00e+00, 1.76e+01 / 0.00e+00 | 20260506 03:20:53.662 | INFO | PerfReport: 0.02%, Misc -- 1.47e-02 / 0.00e+00, 2.45e-02 / 0.00e+00 | 20260506 03:20:53.785 | INFO | Starting a round of Baz IO | 20260506 03:20:56.914 | INFO | Populating RTMetrics report | 20260506 03:20:56.914 | INFO | RTMetrics report populated | 20260506 03:20:56.914 | INFO | Opening tmp RTMetrics json file | 20260506 03:20:56.917 | INFO | Serializing RTMetrics to json | 20260506 03:20:56.918 | INFO | Serialization done, swapping tmp file now... | 20260506 03:20:56.919 | INFO | RTMetrics done | 20260506 03:20:56.992 | INFO | Finished a round of Baz IO: Wrote 1.765 GiB in 3.205 seconds | 20260506 03:20:56.992 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.23% / 0.00% / 78.77% | 20260506 03:20:56.992 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.062116% / 3.97364e-06% / 9.13938e-05% | 20260506 03:20:56.992 | INFO | Baz IO overall: | 20260506 03:20:56.992 | INFO | BytesWritten : 4756882273161 | 20260506 03:20:56.992 | INFO | NumEvents : 2230944735263 | 20260506 03:20:58.800 | INFO | Analyzing chunk frames = [8544256,8544768) | 20260506 03:20:58.800 | INFO | MemInfo: { | 20260506 03:20:58.800 | INFO | "gpu" : " 17 GB", | 20260506 03:20:58.800 | INFO | "huge" : "", | 20260506 03:20:58.800 | INFO | "huge_shared" : " 27 GB", | 20260506 03:20:58.800 | INFO | "malloc" : " 14 GB", | 20260506 03:20:58.800 | INFO | "pinned" : " 29 GB" | 20260506 03:20:58.800 | INFO | } | 20260506 03:20:58.800 | INFO | Perf report for the last 5.139 seconds: | 20260506 03:20:58.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:20:58.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:58.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:58.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:58.800 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260506 03:20:58.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:20:58.800 | INFO | RT_METRICS : 42.6%, 1.0, 22.4ms, 0 | 20260506 03:20:58.800 | INFO | PRE_HQ : 59.1%, 1.7, 53.3ms, 0 | 20260506 03:20:58.800 | INFO | BAZWRITER : 61.6%, 3.5, 110.5ms, 0 | 20260506 03:20:58.800 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 03:20:58.800 | INFO | Uploaded 36GiB to GPU in 2.11568s (17.0158GiB/s) | 20260506 03:20:58.800 | INFO | Downloaded 38GiB to GPU in 2.2348s (17.0038GiB/s) | 20260506 03:21:03.947 | INFO | Analyzing chunk frames = [8544768,8545280) | 20260506 03:21:03.947 | INFO | MemInfo: { | 20260506 03:21:03.947 | INFO | "gpu" : " 17 GB", | 20260506 03:21:03.947 | INFO | "huge" : "", | 20260506 03:21:03.947 | INFO | "huge_shared" : " 27 GB", | 20260506 03:21:03.947 | INFO | "malloc" : " 14 GB", | 20260506 03:21:03.947 | INFO | "pinned" : " 29 GB" | 20260506 03:21:03.947 | INFO | } | 20260506 03:21:03.947 | INFO | Perf report for the last 5.14702 seconds: | 20260506 03:21:03.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:21:03.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:03.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:03.947 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:03.947 | INFO | ANALYSIS : 60.5%, 3.0, 94.6ms, 0 | 20260506 03:21:03.947 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:03.947 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:03.947 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:21:03.947 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:03.947 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:03.947 | INFO | Uploaded 36GiB to GPU in 1.98568s (18.1298GiB/s) | 20260506 03:21:03.947 | INFO | Downloaded 34GiB to GPU in 1.98803s (17.1024GiB/s) | 20260506 03:21:09.086 | INFO | Analyzing chunk frames = [8545280,8545792) | 20260506 03:21:09.086 | INFO | MemInfo: { | 20260506 03:21:09.086 | INFO | "gpu" : " 17 GB", | 20260506 03:21:09.086 | INFO | "huge" : "", | 20260506 03:21:09.086 | INFO | "huge_shared" : " 27 GB", | 20260506 03:21:09.086 | INFO | "malloc" : " 14 GB", | 20260506 03:21:09.086 | INFO | "pinned" : " 29 GB" | 20260506 03:21:09.086 | INFO | } | 20260506 03:21:09.086 | INFO | Perf report for the last 5.13928 seconds: | 20260506 03:21:09.086 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:21:09.086 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:09.086 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:09.086 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:09.086 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260506 03:21:09.086 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:09.086 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:09.086 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 03:21:09.086 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:09.086 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:09.087 | INFO | Uploaded 36GiB to GPU in 2.04412s (17.6115GiB/s) | 20260506 03:21:09.087 | INFO | Downloaded 34GiB to GPU in 2.00922s (16.922GiB/s) | 20260506 03:21:14.226 | INFO | Analyzing chunk frames = [8545792,8546304) | 20260506 03:21:14.226 | INFO | MemInfo: { | 20260506 03:21:14.226 | INFO | "gpu" : " 17 GB", | 20260506 03:21:14.226 | INFO | "huge" : "", | 20260506 03:21:14.226 | INFO | "huge_shared" : " 27 GB", | 20260506 03:21:14.226 | INFO | "malloc" : " 14 GB", | 20260506 03:21:14.226 | INFO | "pinned" : " 29 GB" | 20260506 03:21:14.226 | INFO | } | 20260506 03:21:14.226 | INFO | Perf report for the last 5.13993 seconds: | 20260506 03:21:14.226 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:21:14.226 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:14.226 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:14.226 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:14.226 | INFO | ANALYSIS : 60.9%, 3.0, 94.6ms, 0 | 20260506 03:21:14.226 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:14.226 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:14.226 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 03:21:14.226 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:14.226 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:14.226 | INFO | Uploaded 36GiB to GPU in 2.09237s (17.2054GiB/s) | 20260506 03:21:14.227 | INFO | Downloaded 34GiB to GPU in 2.01729s (16.8543GiB/s) | 20260506 03:21:19.363 | INFO | Analyzing chunk frames = [8546304,8546816) | 20260506 03:21:19.363 | INFO | MemInfo: { | 20260506 03:21:19.363 | INFO | "gpu" : " 17 GB", | 20260506 03:21:19.363 | INFO | "huge" : "", | 20260506 03:21:19.363 | INFO | "huge_shared" : " 27 GB", | 20260506 03:21:19.363 | INFO | "malloc" : " 14 GB", | 20260506 03:21:19.363 | INFO | "pinned" : " 29 GB" | 20260506 03:21:19.363 | INFO | } | 20260506 03:21:19.363 | INFO | Perf report for the last 5.13658 seconds: | 20260506 03:21:19.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:21:19.363 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:19.363 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:19.363 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:19.363 | INFO | ANALYSIS : 60.8%, 3.0, 94.9ms, 0 | 20260506 03:21:19.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:19.363 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:19.363 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 03:21:19.363 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:19.363 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:19.363 | INFO | Uploaded 36GiB to GPU in 2.11044s (17.0581GiB/s) | 20260506 03:21:19.363 | INFO | Downloaded 34GiB to GPU in 2.0414s (16.6552GiB/s) | 20260506 03:21:24.511 | INFO | Analyzing chunk frames = [8546816,8547328) | 20260506 03:21:24.511 | INFO | MemInfo: { | 20260506 03:21:24.511 | INFO | "gpu" : " 17 GB", | 20260506 03:21:24.511 | INFO | "huge" : "", | 20260506 03:21:24.511 | INFO | "huge_shared" : " 27 GB", | 20260506 03:21:24.511 | INFO | "malloc" : " 14 GB", | 20260506 03:21:24.511 | INFO | "pinned" : " 29 GB" | 20260506 03:21:24.511 | INFO | } | 20260506 03:21:24.511 | INFO | Perf report for the last 5.1484 seconds: | 20260506 03:21:24.511 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:21:24.511 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:24.511 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:24.511 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:24.511 | INFO | ANALYSIS : 60.9%, 3.0, 95.2ms, 0 | 20260506 03:21:24.511 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:24.511 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:24.511 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 03:21:24.511 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:24.511 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:24.512 | INFO | Uploaded 36GiB to GPU in 2.10149s (17.1307GiB/s) | 20260506 03:21:24.512 | INFO | Downloaded 34GiB to GPU in 2.0351s (16.7068GiB/s) | 20260506 03:21:29.642 | INFO | Analyzing chunk frames = [8547328,8547840) | 20260506 03:21:29.642 | INFO | MemInfo: { | 20260506 03:21:29.642 | INFO | "gpu" : " 17 GB", | 20260506 03:21:29.642 | INFO | "huge" : "", | 20260506 03:21:29.642 | INFO | "huge_shared" : " 27 GB", | 20260506 03:21:29.642 | INFO | "malloc" : " 14 GB", | 20260506 03:21:29.642 | INFO | "pinned" : " 29 GB" | 20260506 03:21:29.642 | INFO | } | 20260506 03:21:29.642 | INFO | Perf report for the last 5.13112 seconds: | 20260506 03:21:29.642 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:21:29.642 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:29.642 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:29.642 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:29.642 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 03:21:29.642 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:29.643 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:29.643 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 03:21:29.643 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:29.643 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:29.643 | INFO | Uploaded 36GiB to GPU in 2.08172s (17.2934GiB/s) | 20260506 03:21:29.643 | INFO | Downloaded 34GiB to GPU in 2.0609s (16.4976GiB/s) | 20260506 03:21:34.783 | INFO | Analyzing chunk frames = [8547840,8548352) | 20260506 03:21:34.783 | INFO | MemInfo: { | 20260506 03:21:34.783 | INFO | "gpu" : " 17 GB", | 20260506 03:21:34.783 | INFO | "huge" : "", | 20260506 03:21:34.783 | INFO | "huge_shared" : " 27 GB", | 20260506 03:21:34.783 | INFO | "malloc" : " 14 GB", | 20260506 03:21:34.783 | INFO | "pinned" : " 29 GB" | 20260506 03:21:34.783 | INFO | } | 20260506 03:21:34.783 | INFO | Perf report for the last 5.14058 seconds: | 20260506 03:21:34.783 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:21:34.783 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:34.783 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:34.783 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:34.783 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260506 03:21:34.783 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:34.783 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:34.783 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:21:34.783 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:34.783 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:34.783 | INFO | Uploaded 36GiB to GPU in 2.07269s (17.3687GiB/s) | 20260506 03:21:34.783 | INFO | Downloaded 34GiB to GPU in 2.04108s (16.6579GiB/s) | 20260506 03:21:34.908 | INFO | Starting a round of Baz IO | 20260506 03:21:38.027 | INFO | Populating RTMetrics report | 20260506 03:21:38.027 | INFO | RTMetrics report populated | 20260506 03:21:38.027 | INFO | Opening tmp RTMetrics json file | 20260506 03:21:38.028 | INFO | Serializing RTMetrics to json | 20260506 03:21:38.029 | INFO | Serialization done, swapping tmp file now... | 20260506 03:21:38.030 | INFO | RTMetrics done | 20260506 03:21:38.101 | INFO | Finished a round of Baz IO: Wrote 1.763 GiB in 3.192 seconds | 20260506 03:21:38.101 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.23% / 0.00% / 78.77% | 20260506 03:21:38.101 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0618736% / 0% / 9.53674e-05% | 20260506 03:21:38.101 | INFO | Baz IO overall: | 20260506 03:21:38.101 | INFO | BytesWritten : 4758775294629 | 20260506 03:21:38.101 | INFO | NumEvents : 2231802794974 | 20260506 03:21:39.923 | INFO | Analyzing chunk frames = [8548352,8548864) | 20260506 03:21:39.923 | INFO | MemInfo: { | 20260506 03:21:39.923 | INFO | "gpu" : " 17 GB", | 20260506 03:21:39.923 | INFO | "huge" : "", | 20260506 03:21:39.923 | INFO | "huge_shared" : " 27 GB", | 20260506 03:21:39.923 | INFO | "malloc" : " 14 GB", | 20260506 03:21:39.923 | INFO | "pinned" : " 29 GB" | 20260506 03:21:39.923 | INFO | } | 20260506 03:21:39.923 | INFO | Perf report for the last 5.14033 seconds: | 20260506 03:21:39.923 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:21:39.923 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:39.923 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:39.923 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:39.923 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260506 03:21:39.923 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:39.923 | INFO | RT_METRICS : 42.6%, 1.0, 22.4ms, 0 | 20260506 03:21:39.923 | INFO | PRE_HQ : 59.0%, 1.7, 53.5ms, 0 | 20260506 03:21:39.923 | INFO | BAZWRITER : 61.3%, 3.5, 109.4ms, 0 | 20260506 03:21:39.924 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 03:21:39.924 | INFO | Uploaded 36GiB to GPU in 2.09475s (17.1858GiB/s) | 20260506 03:21:39.924 | INFO | Downloaded 38GiB to GPU in 2.22492s (17.0793GiB/s) | 20260506 03:21:45.057 | INFO | Analyzing chunk frames = [8548864,8549376) | 20260506 03:21:45.057 | INFO | MemInfo: { | 20260506 03:21:45.057 | INFO | "gpu" : " 17 GB", | 20260506 03:21:45.057 | INFO | "huge" : "", | 20260506 03:21:45.057 | INFO | "huge_shared" : " 27 GB", | 20260506 03:21:45.057 | INFO | "malloc" : " 14 GB", | 20260506 03:21:45.057 | INFO | "pinned" : " 29 GB" | 20260506 03:21:45.057 | INFO | } | 20260506 03:21:45.057 | INFO | Perf report for the last 5.13407 seconds: | 20260506 03:21:45.057 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:21:45.057 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:45.057 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:45.057 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:45.057 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260506 03:21:45.057 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:45.058 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:45.058 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 03:21:45.058 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:45.058 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:45.058 | INFO | Uploaded 36GiB to GPU in 1.94229s (18.5348GiB/s) | 20260506 03:21:45.058 | INFO | Downloaded 34GiB to GPU in 1.98936s (17.0909GiB/s) | 20260506 03:21:50.198 | INFO | Analyzing chunk frames = [8549376,8549888) | 20260506 03:21:50.198 | INFO | MemInfo: { | 20260506 03:21:50.198 | INFO | "gpu" : " 17 GB", | 20260506 03:21:50.198 | INFO | "huge" : "", | 20260506 03:21:50.198 | INFO | "huge_shared" : " 27 GB", | 20260506 03:21:50.198 | INFO | "malloc" : " 14 GB", | 20260506 03:21:50.198 | INFO | "pinned" : " 29 GB" | 20260506 03:21:50.198 | INFO | } | 20260506 03:21:50.198 | INFO | Perf report for the last 5.14045 seconds: | 20260506 03:21:50.198 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:21:50.198 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:50.198 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:50.198 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:50.198 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 03:21:50.198 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:50.198 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:50.198 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 03:21:50.198 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:50.198 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:50.198 | INFO | Uploaded 36GiB to GPU in 2.09354s (17.1958GiB/s) | 20260506 03:21:50.198 | INFO | Downloaded 34GiB to GPU in 2.02522s (16.7883GiB/s) | 20260506 03:21:55.339 | INFO | Analyzing chunk frames = [8549888,8550400) | 20260506 03:21:55.339 | INFO | MemInfo: { | 20260506 03:21:55.339 | INFO | "gpu" : " 17 GB", | 20260506 03:21:55.339 | INFO | "huge" : "", | 20260506 03:21:55.339 | INFO | "huge_shared" : " 27 GB", | 20260506 03:21:55.339 | INFO | "malloc" : " 14 GB", | 20260506 03:21:55.339 | INFO | "pinned" : " 29 GB" | 20260506 03:21:55.339 | INFO | } | 20260506 03:21:55.339 | INFO | Perf report for the last 5.14093 seconds: | 20260506 03:21:55.339 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:21:55.339 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:55.339 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:55.339 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:55.339 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260506 03:21:55.339 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:55.339 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:55.339 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 03:21:55.339 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:55.339 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:21:55.339 | INFO | Uploaded 36GiB to GPU in 2.09141s (17.2133GiB/s) | 20260506 03:21:55.339 | INFO | Downloaded 34GiB to GPU in 2.00456s (16.9613GiB/s) | 20260506 03:22:00.479 | INFO | Analyzing chunk frames = [8550400,8550912) | 20260506 03:22:00.479 | INFO | MemInfo: { | 20260506 03:22:00.479 | INFO | "gpu" : " 17 GB", | 20260506 03:22:00.479 | INFO | "huge" : "", | 20260506 03:22:00.479 | INFO | "huge_shared" : " 27 GB", | 20260506 03:22:00.479 | INFO | "malloc" : " 14 GB", | 20260506 03:22:00.479 | INFO | "pinned" : " 29 GB" | 20260506 03:22:00.479 | INFO | } | 20260506 03:22:00.479 | INFO | Perf report for the last 5.14036 seconds: | 20260506 03:22:00.479 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:22:00.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:00.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:00.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:00.479 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 03:22:00.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:00.479 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:00.479 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 03:22:00.479 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:00.479 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:00.480 | INFO | Uploaded 36GiB to GPU in 2.11423s (17.0274GiB/s) | 20260506 03:22:00.480 | INFO | Downloaded 34GiB to GPU in 2.03604s (16.6991GiB/s) | 20260506 03:22:05.611 | INFO | Analyzing chunk frames = [8550912,8551424) | 20260506 03:22:05.611 | INFO | MemInfo: { | 20260506 03:22:05.611 | INFO | "gpu" : " 17 GB", | 20260506 03:22:05.611 | INFO | "huge" : "", | 20260506 03:22:05.611 | INFO | "huge_shared" : " 27 GB", | 20260506 03:22:05.611 | INFO | "malloc" : " 14 GB", | 20260506 03:22:05.611 | INFO | "pinned" : " 29 GB" | 20260506 03:22:05.611 | INFO | } | 20260506 03:22:05.611 | INFO | Perf report for the last 5.1319 seconds: | 20260506 03:22:05.611 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:22:05.611 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:05.611 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:05.611 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:05.611 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 03:22:05.611 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:05.611 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:05.611 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 03:22:05.611 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:05.611 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:05.611 | INFO | Uploaded 36GiB to GPU in 2.11468s (17.0239GiB/s) | 20260506 03:22:05.611 | INFO | Downloaded 34GiB to GPU in 2.04849s (16.5976GiB/s) | 20260506 03:22:10.750 | INFO | Analyzing chunk frames = [8551424,8551936) | 20260506 03:22:10.750 | INFO | MemInfo: { | 20260506 03:22:10.750 | INFO | "gpu" : " 17 GB", | 20260506 03:22:10.750 | INFO | "huge" : "", | 20260506 03:22:10.750 | INFO | "huge_shared" : " 27 GB", | 20260506 03:22:10.750 | INFO | "malloc" : " 14 GB", | 20260506 03:22:10.750 | INFO | "pinned" : " 29 GB" | 20260506 03:22:10.750 | INFO | } | 20260506 03:22:10.750 | INFO | Perf report for the last 5.13897 seconds: | 20260506 03:22:10.750 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:22:10.750 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:10.750 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:10.750 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:10.750 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 03:22:10.750 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:10.751 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:10.751 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 03:22:10.751 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:10.751 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:10.751 | INFO | Uploaded 36GiB to GPU in 2.01955s (17.8258GiB/s) | 20260506 03:22:10.751 | INFO | Downloaded 34GiB to GPU in 2.05361s (16.5562GiB/s) | 20260506 03:22:15.894 | INFO | Analyzing chunk frames = [8551936,8552448) | 20260506 03:22:15.894 | INFO | MemInfo: { | 20260506 03:22:15.894 | INFO | "gpu" : " 17 GB", | 20260506 03:22:15.894 | INFO | "huge" : "", | 20260506 03:22:15.894 | INFO | "huge_shared" : " 27 GB", | 20260506 03:22:15.894 | INFO | "malloc" : " 14 GB", | 20260506 03:22:15.894 | INFO | "pinned" : " 29 GB" | 20260506 03:22:15.894 | INFO | } | 20260506 03:22:15.894 | INFO | Perf report for the last 5.14405 seconds: | 20260506 03:22:15.894 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:22:15.894 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:15.894 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:15.894 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:15.894 | INFO | ANALYSIS : 61.4%, 3.0, 96.0ms, 0 | 20260506 03:22:15.894 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:15.894 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:15.894 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 03:22:15.894 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:15.894 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:15.894 | INFO | Uploaded 36GiB to GPU in 2.07076s (17.3849GiB/s) | 20260506 03:22:15.894 | INFO | Downloaded 34GiB to GPU in 2.0271s (16.7727GiB/s) | 20260506 03:22:15.895 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 03:22:15.895 | INFO | Total mem usage: 15.4 GiB | 20260506 03:22:15.895 | INFO | All Time High Water Marks sum: 16.3 GiB (Savings: 5.2%) | 20260506 03:22:15.895 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -10%) | 20260506 03:22:15.895 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 03:22:15.895 | INFO | High Water Breakdown: | 20260506 03:22:15.895 | INFO | 60.9% : 8.54 GiB ( 88.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 03:22:15.895 | INFO | 39% : 5.47 GiB ( 82.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 03:22:15.895 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 03:22:15.895 | INFO | ------------------------------------------ | 20260506 03:22:15.895 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:22:15.895 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 03:22:15.895 | INFO | Total mem usage: 26.5 GiB | 20260506 03:22:15.895 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 03:22:15.895 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 03:22:15.895 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 03:22:15.895 | INFO | High Water Breakdown: | 20260506 03:22:15.895 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 03:22:15.895 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 03:22:15.895 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 03:22:15.895 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 03:22:15.895 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 03:22:15.895 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 03:22:15.895 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 03:22:15.895 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 03:22:15.895 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 03:22:15.895 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 03:22:15.895 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 03:22:15.895 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 03:22:15.895 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 03:22:15.895 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 03:22:15.895 | INFO | ------------------------------------------ | 20260506 03:22:15.895 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:22:15.895 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 03:22:15.895 | INFO | Total mem usage: 25.6 GiB | 20260506 03:22:15.895 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 03:22:15.895 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 03:22:15.895 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 03:22:15.895 | INFO | High Water Breakdown: | 20260506 03:22:15.895 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 03:22:15.895 | INFO | ------------------------------------------ | 20260506 03:22:15.895 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:22:15.895 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 03:22:15.895 | INFO | Total mem usage: 15.7 GiB | 20260506 03:22:15.895 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 03:22:15.895 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 03:22:15.895 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 03:22:15.895 | INFO | High Water Breakdown: | 20260506 03:22:15.895 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 03:22:15.895 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 03:22:15.895 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 03:22:15.895 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 03:22:15.895 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 03:22:15.895 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 03:22:15.895 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 03:22:15.895 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 03:22:15.895 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 03:22:15.895 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 03:22:15.895 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 03:22:15.895 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 03:22:15.895 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 03:22:15.895 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 03:22:15.895 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 03:22:15.895 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 03:22:15.895 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 03:22:15.895 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 03:22:15.895 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 03:22:15.895 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 03:22:15.895 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 03:22:15.895 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 03:22:15.895 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 03:22:15.895 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 03:22:15.895 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 03:22:15.895 | INFO | ------------------------------------------ | 20260506 03:22:15.895 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:22:15.895 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 03:22:15.895 | INFO | PerfReport: Gathered 1536 samples. | 20260506 03:22:15.895 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 03:22:15.895 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.36e+01, 1.35e+01 / 8.76e-01 | 20260506 03:22:15.895 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.04e-01, 1.02e-02 / 2.32e-03 | 20260506 03:22:15.895 | INFO | PerfReport: 11.90%, Baseline -- 3.72e+00 / 4.54e+00, 5.92e-01 / 2.30e-01 | 20260506 03:22:15.895 | INFO | PerfReport: 3.30%, Histogram -- 1.03e+00 / 1.54e+00, 1.96e-01 / 7.14e-02 | 20260506 03:22:15.895 | INFO | PerfReport: 16.00%, DME -- 4.00e+01 / 4.40e+01, 3.85e+00 / 9.22e-01 | 20260506 03:22:15.895 | INFO | PerfReport: 57.52%, FrameLabeling -- 1.80e+01 / 1.91e+01, 1.54e+00 / 7.61e-01 | 20260506 03:22:15.895 | INFO | PerfReport: 6.98%, PulseAccumulating -- 2.18e+00 / 2.71e+00, 4.11e-01 / 1.85e-01 | 20260506 03:22:15.895 | INFO | PerfReport: 2.98%, Metrics -- 9.33e-01 / 1.21e+00, 2.54e-01 / 8.48e-02 | 20260506 03:22:15.895 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.21e-01 / 3.69e-01, 5.44e-02 / 3.04e-02 | 20260506 03:22:15.896 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 03:22:15.896 | INFO | PerfReport: Gathered 1536 samples. | 20260506 03:22:15.896 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 03:22:15.896 | INFO | PerfReport: Full Execution -- 9.74e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260506 03:22:15.896 | INFO | PerfReport: 22.14%, Upload -- 2.16e+01 / 0.00e+00, 7.11e+00 / 0.00e+00 | 20260506 03:22:15.896 | INFO | PerfReport: 21.94%, Download -- 2.14e+01 / 0.00e+00, 7.59e+00 / 0.00e+00 | 20260506 03:22:15.896 | INFO | PerfReport: 32.09%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 03:22:15.896 | INFO | PerfReport: 2.01%, LockUpload -- 1.96e+00 / 0.00e+00, 7.85e+00 / 0.00e+00 | 20260506 03:22:15.896 | INFO | PerfReport: 1.29%, LockDownload -- 1.26e+00 / 0.00e+00, 2.96e+00 / 0.00e+00 | 20260506 03:22:15.896 | INFO | PerfReport: 20.51%, LockCompute -- 2.00e+01 / 0.00e+00, 1.76e+01 / 0.00e+00 | 20260506 03:22:15.896 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.87e-02 / 0.00e+00 | 20260506 03:22:16.018 | INFO | Starting a round of Baz IO | 20260506 03:22:19.136 | INFO | Populating RTMetrics report | 20260506 03:22:19.137 | INFO | RTMetrics report populated | 20260506 03:22:19.137 | INFO | Opening tmp RTMetrics json file | 20260506 03:22:19.141 | INFO | Serializing RTMetrics to json | 20260506 03:22:19.142 | INFO | Serialization done, swapping tmp file now... | 20260506 03:22:19.142 | INFO | RTMetrics done | 20260506 03:22:19.228 | INFO | Finished a round of Baz IO: Wrote 1.763 GiB in 3.209 seconds | 20260506 03:22:19.228 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.23% / 0.00% / 78.77% | 20260506 03:22:19.228 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.061659% / 0% / 8.74201e-05% | 20260506 03:22:19.228 | INFO | Baz IO overall: | 20260506 03:22:19.228 | INFO | BytesWritten : 4760667891423 | 20260506 03:22:19.228 | INFO | NumEvents : 2232660908642 | 20260506 03:22:21.039 | INFO | Analyzing chunk frames = [8552448,8552960) | 20260506 03:22:21.039 | INFO | MemInfo: { | 20260506 03:22:21.039 | INFO | "gpu" : " 17 GB", | 20260506 03:22:21.039 | INFO | "huge" : "", | 20260506 03:22:21.039 | INFO | "huge_shared" : " 27 GB", | 20260506 03:22:21.039 | INFO | "malloc" : " 14 GB", | 20260506 03:22:21.039 | INFO | "pinned" : " 29 GB" | 20260506 03:22:21.039 | INFO | } | 20260506 03:22:21.039 | INFO | Perf report for the last 5.14466 seconds: | 20260506 03:22:21.039 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:22:21.039 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:21.039 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:21.039 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:21.039 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260506 03:22:21.039 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:21.039 | INFO | RT_METRICS : 42.6%, 1.0, 22.4ms, 0 | 20260506 03:22:21.039 | INFO | PRE_HQ : 58.9%, 1.7, 53.2ms, 0 | 20260506 03:22:21.039 | INFO | BAZWRITER : 61.6%, 3.5, 111.6ms, 0 | 20260506 03:22:21.039 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 03:22:21.039 | INFO | Uploaded 36GiB to GPU in 2.11677s (17.0071GiB/s) | 20260506 03:22:21.039 | INFO | Downloaded 38GiB to GPU in 2.22244s (17.0983GiB/s) | 20260506 03:22:26.174 | INFO | Analyzing chunk frames = [8552960,8553472) | 20260506 03:22:26.175 | INFO | MemInfo: { | 20260506 03:22:26.175 | INFO | "gpu" : " 17 GB", | 20260506 03:22:26.175 | INFO | "huge" : "", | 20260506 03:22:26.175 | INFO | "huge_shared" : " 27 GB", | 20260506 03:22:26.175 | INFO | "malloc" : " 14 GB", | 20260506 03:22:26.175 | INFO | "pinned" : " 29 GB" | 20260506 03:22:26.175 | INFO | } | 20260506 03:22:26.175 | INFO | Perf report for the last 5.13594 seconds: | 20260506 03:22:26.175 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:22:26.175 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:26.175 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:26.175 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:26.175 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 03:22:26.175 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:26.175 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:26.175 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 03:22:26.175 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:26.175 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:26.175 | INFO | Uploaded 36GiB to GPU in 1.97748s (18.205GiB/s) | 20260506 03:22:26.175 | INFO | Downloaded 34GiB to GPU in 1.99098s (17.077GiB/s) | 20260506 03:22:31.310 | INFO | Analyzing chunk frames = [8553472,8553984) | 20260506 03:22:31.310 | INFO | MemInfo: { | 20260506 03:22:31.311 | INFO | "gpu" : " 17 GB", | 20260506 03:22:31.311 | INFO | "huge" : "", | 20260506 03:22:31.311 | INFO | "huge_shared" : " 27 GB", | 20260506 03:22:31.311 | INFO | "malloc" : " 14 GB", | 20260506 03:22:31.311 | INFO | "pinned" : " 29 GB" | 20260506 03:22:31.311 | INFO | } | 20260506 03:22:31.311 | INFO | Perf report for the last 5.13598 seconds: | 20260506 03:22:31.311 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:22:31.311 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:31.311 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:31.311 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:31.311 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 03:22:31.311 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:31.311 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:31.311 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:22:31.311 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:31.311 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:31.311 | INFO | Uploaded 36GiB to GPU in 2.13226s (16.8835GiB/s) | 20260506 03:22:31.311 | INFO | Downloaded 34GiB to GPU in 2.04941s (16.5901GiB/s) | 20260506 03:22:36.459 | INFO | Analyzing chunk frames = [8553984,8554496) | 20260506 03:22:36.459 | INFO | MemInfo: { | 20260506 03:22:36.459 | INFO | "gpu" : " 17 GB", | 20260506 03:22:36.459 | INFO | "huge" : "", | 20260506 03:22:36.459 | INFO | "huge_shared" : " 27 GB", | 20260506 03:22:36.459 | INFO | "malloc" : " 14 GB", | 20260506 03:22:36.459 | INFO | "pinned" : " 29 GB" | 20260506 03:22:36.459 | INFO | } | 20260506 03:22:36.459 | INFO | Perf report for the last 5.14821 seconds: | 20260506 03:22:36.459 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:22:36.459 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:36.459 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:36.459 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:36.459 | INFO | ANALYSIS : 61.0%, 3.0, 94.8ms, 0 | 20260506 03:22:36.459 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:36.459 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:36.459 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 03:22:36.459 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:36.459 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:36.459 | INFO | Uploaded 36GiB to GPU in 2.071s (17.3829GiB/s) | 20260506 03:22:36.459 | INFO | Downloaded 34GiB to GPU in 2.01247s (16.8947GiB/s) | 20260506 03:22:41.594 | INFO | Analyzing chunk frames = [8554496,8555008) | 20260506 03:22:41.594 | INFO | MemInfo: { | 20260506 03:22:41.594 | INFO | "gpu" : " 17 GB", | 20260506 03:22:41.594 | INFO | "huge" : "", | 20260506 03:22:41.594 | INFO | "huge_shared" : " 27 GB", | 20260506 03:22:41.594 | INFO | "malloc" : " 14 GB", | 20260506 03:22:41.594 | INFO | "pinned" : " 29 GB" | 20260506 03:22:41.594 | INFO | } | 20260506 03:22:41.594 | INFO | Perf report for the last 5.13523 seconds: | 20260506 03:22:41.594 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:22:41.594 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:41.594 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:41.594 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:41.594 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260506 03:22:41.594 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:41.594 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:41.594 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:22:41.594 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:41.594 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:41.594 | INFO | Uploaded 36GiB to GPU in 2.00132s (17.9881GiB/s) | 20260506 03:22:41.595 | INFO | Downloaded 34GiB to GPU in 1.98813s (17.1015GiB/s) | 20260506 03:22:46.730 | INFO | Analyzing chunk frames = [8555008,8555520) | 20260506 03:22:46.730 | INFO | MemInfo: { | 20260506 03:22:46.730 | INFO | "gpu" : " 17 GB", | 20260506 03:22:46.730 | INFO | "huge" : "", | 20260506 03:22:46.730 | INFO | "huge_shared" : " 27 GB", | 20260506 03:22:46.730 | INFO | "malloc" : " 14 GB", | 20260506 03:22:46.730 | INFO | "pinned" : " 29 GB" | 20260506 03:22:46.730 | INFO | } | 20260506 03:22:46.730 | INFO | Perf report for the last 5.13612 seconds: | 20260506 03:22:46.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:22:46.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:46.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:46.731 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:46.731 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260506 03:22:46.731 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:46.731 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:46.731 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 03:22:46.731 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:46.731 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:46.731 | INFO | Uploaded 36GiB to GPU in 2.06152s (17.4628GiB/s) | 20260506 03:22:46.731 | INFO | Downloaded 34GiB to GPU in 2.03227s (16.73GiB/s) | 20260506 03:22:51.871 | INFO | Analyzing chunk frames = [8555520,8556032) | 20260506 03:22:51.871 | INFO | MemInfo: { | 20260506 03:22:51.871 | INFO | "gpu" : " 17 GB", | 20260506 03:22:51.871 | INFO | "huge" : "", | 20260506 03:22:51.871 | INFO | "huge_shared" : " 27 GB", | 20260506 03:22:51.871 | INFO | "malloc" : " 14 GB", | 20260506 03:22:51.871 | INFO | "pinned" : " 29 GB" | 20260506 03:22:51.871 | INFO | } | 20260506 03:22:51.871 | INFO | Perf report for the last 5.14073 seconds: | 20260506 03:22:51.871 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:22:51.871 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:51.871 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:51.871 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:51.871 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260506 03:22:51.871 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:51.871 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:51.871 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 03:22:51.871 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:51.871 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:51.871 | INFO | Uploaded 36GiB to GPU in 2.04981s (17.5626GiB/s) | 20260506 03:22:51.871 | INFO | Downloaded 34GiB to GPU in 2.05829s (16.5186GiB/s) | 20260506 03:22:57.007 | INFO | Analyzing chunk frames = [8556032,8556544) | 20260506 03:22:57.007 | INFO | MemInfo: { | 20260506 03:22:57.007 | INFO | "gpu" : " 17 GB", | 20260506 03:22:57.007 | INFO | "huge" : "", | 20260506 03:22:57.007 | INFO | "huge_shared" : " 27 GB", | 20260506 03:22:57.007 | INFO | "malloc" : " 14 GB", | 20260506 03:22:57.007 | INFO | "pinned" : " 29 GB" | 20260506 03:22:57.007 | INFO | } | 20260506 03:22:57.007 | INFO | Perf report for the last 5.13635 seconds: | 20260506 03:22:57.007 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:22:57.007 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:57.007 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:57.008 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:57.008 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 03:22:57.008 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:57.008 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:57.008 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:22:57.008 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:57.008 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:22:57.008 | INFO | Uploaded 36GiB to GPU in 2.06219s (17.4572GiB/s) | 20260506 03:22:57.008 | INFO | Downloaded 34GiB to GPU in 2.03196s (16.7327GiB/s) | 20260506 03:22:57.133 | INFO | Starting a round of Baz IO | 20260506 03:23:00.255 | INFO | Populating RTMetrics report | 20260506 03:23:00.255 | INFO | RTMetrics report populated | 20260506 03:23:00.255 | INFO | Opening tmp RTMetrics json file | 20260506 03:23:00.273 | INFO | Serializing RTMetrics to json | 20260506 03:23:00.275 | INFO | Serialization done, swapping tmp file now... | 20260506 03:23:00.277 | INFO | RTMetrics done | 20260506 03:23:00.360 | INFO | Finished a round of Baz IO: Wrote 1.761 GiB in 3.226 seconds | 20260506 03:23:00.360 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.23% / 0.00% / 78.77% | 20260506 03:23:00.360 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0616272% / 0% / 7.15256e-05% | 20260506 03:23:00.360 | INFO | Baz IO overall: | 20260506 03:23:00.360 | INFO | BytesWritten : 4762558985720 | 20260506 03:23:00.360 | INFO | NumEvents : 2233517988557 | 20260506 03:23:02.150 | INFO | Analyzing chunk frames = [8556544,8557056) | 20260506 03:23:02.150 | INFO | MemInfo: { | 20260506 03:23:02.150 | INFO | "gpu" : " 17 GB", | 20260506 03:23:02.150 | INFO | "huge" : "", | 20260506 03:23:02.151 | INFO | "huge_shared" : " 27 GB", | 20260506 03:23:02.151 | INFO | "malloc" : " 14 GB", | 20260506 03:23:02.151 | INFO | "pinned" : " 29 GB" | 20260506 03:23:02.151 | INFO | } | 20260506 03:23:02.151 | INFO | Perf report for the last 5.14308 seconds: | 20260506 03:23:02.151 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:23:02.151 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:02.151 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:02.151 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:02.151 | INFO | ANALYSIS : 62.2%, 3.0, 96.5ms, 0 | 20260506 03:23:02.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:02.151 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260506 03:23:02.151 | INFO | PRE_HQ : 59.1%, 1.7, 53.4ms, 0 | 20260506 03:23:02.151 | INFO | BAZWRITER : 61.9%, 3.8, 122.1ms, 0 | 20260506 03:23:02.151 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260506 03:23:02.151 | INFO | Uploaded 36GiB to GPU in 2.11975s (16.9831GiB/s) | 20260506 03:23:02.151 | INFO | Downloaded 38GiB to GPU in 2.23743s (16.9838GiB/s) | 20260506 03:23:07.294 | INFO | Analyzing chunk frames = [8557056,8557568) | 20260506 03:23:07.294 | INFO | MemInfo: { | 20260506 03:23:07.294 | INFO | "gpu" : " 17 GB", | 20260506 03:23:07.294 | INFO | "huge" : "", | 20260506 03:23:07.294 | INFO | "huge_shared" : " 27 GB", | 20260506 03:23:07.294 | INFO | "malloc" : " 14 GB", | 20260506 03:23:07.294 | INFO | "pinned" : " 29 GB" | 20260506 03:23:07.294 | INFO | } | 20260506 03:23:07.294 | INFO | Perf report for the last 5.14378 seconds: | 20260506 03:23:07.294 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:23:07.294 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:07.294 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:07.294 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:07.294 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260506 03:23:07.294 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:07.294 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:07.294 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 03:23:07.294 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:07.294 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:07.295 | INFO | Uploaded 36GiB to GPU in 1.97065s (18.2681GiB/s) | 20260506 03:23:07.295 | INFO | Downloaded 34GiB to GPU in 1.99229s (17.0658GiB/s) | 20260506 03:23:12.426 | INFO | Analyzing chunk frames = [8557568,8558080) | 20260506 03:23:12.426 | INFO | MemInfo: { | 20260506 03:23:12.426 | INFO | "gpu" : " 17 GB", | 20260506 03:23:12.426 | INFO | "huge" : "", | 20260506 03:23:12.426 | INFO | "huge_shared" : " 27 GB", | 20260506 03:23:12.426 | INFO | "malloc" : " 14 GB", | 20260506 03:23:12.426 | INFO | "pinned" : " 29 GB" | 20260506 03:23:12.426 | INFO | } | 20260506 03:23:12.426 | INFO | Perf report for the last 5.1317 seconds: | 20260506 03:23:12.426 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:23:12.426 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:12.426 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:12.426 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:12.426 | INFO | ANALYSIS : 60.5%, 3.0, 94.0ms, 0 | 20260506 03:23:12.426 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:12.426 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:12.426 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 03:23:12.426 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:12.426 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:12.426 | INFO | Uploaded 36GiB to GPU in 1.9955s (18.0406GiB/s) | 20260506 03:23:12.426 | INFO | Downloaded 34GiB to GPU in 1.99182s (17.0698GiB/s) | 20260506 03:23:17.563 | INFO | Analyzing chunk frames = [8558080,8558592) | 20260506 03:23:17.563 | INFO | MemInfo: { | 20260506 03:23:17.563 | INFO | "gpu" : " 17 GB", | 20260506 03:23:17.563 | INFO | "huge" : "", | 20260506 03:23:17.563 | INFO | "huge_shared" : " 27 GB", | 20260506 03:23:17.563 | INFO | "malloc" : " 14 GB", | 20260506 03:23:17.563 | INFO | "pinned" : " 29 GB" | 20260506 03:23:17.563 | INFO | } | 20260506 03:23:17.563 | INFO | Perf report for the last 5.13708 seconds: | 20260506 03:23:17.563 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:23:17.563 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:17.563 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:17.563 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:17.563 | INFO | ANALYSIS : 61.2%, 3.0, 94.9ms, 0 | 20260506 03:23:17.563 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:17.563 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:17.563 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 03:23:17.563 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:17.563 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:17.563 | INFO | Uploaded 36GiB to GPU in 2.09977s (17.1447GiB/s) | 20260506 03:23:17.563 | INFO | Downloaded 34GiB to GPU in 2.02285s (16.808GiB/s) | 20260506 03:23:22.704 | INFO | Analyzing chunk frames = [8558592,8559104) | 20260506 03:23:22.704 | INFO | MemInfo: { | 20260506 03:23:22.704 | INFO | "gpu" : " 17 GB", | 20260506 03:23:22.704 | INFO | "huge" : "", | 20260506 03:23:22.704 | INFO | "huge_shared" : " 27 GB", | 20260506 03:23:22.704 | INFO | "malloc" : " 14 GB", | 20260506 03:23:22.704 | INFO | "pinned" : " 29 GB" | 20260506 03:23:22.704 | INFO | } | 20260506 03:23:22.704 | INFO | Perf report for the last 5.14085 seconds: | 20260506 03:23:22.704 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:23:22.704 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:22.704 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:22.704 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:22.704 | INFO | ANALYSIS : 60.7%, 3.0, 94.8ms, 0 | 20260506 03:23:22.704 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:22.704 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:22.704 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:23:22.704 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:22.704 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:22.704 | INFO | Uploaded 36GiB to GPU in 2.07228s (17.3722GiB/s) | 20260506 03:23:22.704 | INFO | Downloaded 34GiB to GPU in 2.03141s (16.7371GiB/s) | 20260506 03:23:27.850 | INFO | Analyzing chunk frames = [8559104,8559616) | 20260506 03:23:27.850 | INFO | MemInfo: { | 20260506 03:23:27.850 | INFO | "gpu" : " 17 GB", | 20260506 03:23:27.850 | INFO | "huge" : "", | 20260506 03:23:27.850 | INFO | "huge_shared" : " 27 GB", | 20260506 03:23:27.850 | INFO | "malloc" : " 14 GB", | 20260506 03:23:27.851 | INFO | "pinned" : " 29 GB" | 20260506 03:23:27.851 | INFO | } | 20260506 03:23:27.851 | INFO | Perf report for the last 5.14668 seconds: | 20260506 03:23:27.851 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:23:27.851 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:27.851 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:27.851 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:27.851 | INFO | ANALYSIS : 60.6%, 3.0, 94.7ms, 0 | 20260506 03:23:27.851 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:27.851 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:27.851 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 03:23:27.851 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:27.851 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:27.851 | INFO | Uploaded 36GiB to GPU in 2.05599s (17.5098GiB/s) | 20260506 03:23:27.851 | INFO | Downloaded 34GiB to GPU in 2.02693s (16.7741GiB/s) | 20260506 03:23:32.991 | INFO | Analyzing chunk frames = [8559616,8560128) | 20260506 03:23:32.991 | INFO | MemInfo: { | 20260506 03:23:32.991 | INFO | "gpu" : " 17 GB", | 20260506 03:23:32.991 | INFO | "huge" : "", | 20260506 03:23:32.991 | INFO | "huge_shared" : " 27 GB", | 20260506 03:23:32.991 | INFO | "malloc" : " 14 GB", | 20260506 03:23:32.991 | INFO | "pinned" : " 29 GB" | 20260506 03:23:32.991 | INFO | } | 20260506 03:23:32.991 | INFO | Perf report for the last 5.14029 seconds: | 20260506 03:23:32.991 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:23:32.991 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:32.991 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:32.991 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:32.991 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260506 03:23:32.991 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:32.991 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:32.991 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 03:23:32.991 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:32.991 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:32.991 | INFO | Uploaded 36GiB to GPU in 2.07958s (17.3112GiB/s) | 20260506 03:23:32.991 | INFO | Downloaded 34GiB to GPU in 2.05807s (16.5204GiB/s) | 20260506 03:23:38.128 | INFO | Analyzing chunk frames = [8560128,8560640) | 20260506 03:23:38.128 | INFO | MemInfo: { | 20260506 03:23:38.128 | INFO | "gpu" : " 17 GB", | 20260506 03:23:38.128 | INFO | "huge" : "", | 20260506 03:23:38.128 | INFO | "huge_shared" : " 27 GB", | 20260506 03:23:38.128 | INFO | "malloc" : " 14 GB", | 20260506 03:23:38.128 | INFO | "pinned" : " 29 GB" | 20260506 03:23:38.128 | INFO | } | 20260506 03:23:38.128 | INFO | Perf report for the last 5.13701 seconds: | 20260506 03:23:38.128 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:23:38.128 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:38.128 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:38.128 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:38.128 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260506 03:23:38.128 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:38.128 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:38.128 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 03:23:38.128 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:38.128 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:38.128 | INFO | Uploaded 36GiB to GPU in 2.06034s (17.4729GiB/s) | 20260506 03:23:38.128 | INFO | Downloaded 34GiB to GPU in 2.02986s (16.7499GiB/s) | 20260506 03:23:38.128 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 03:23:38.128 | INFO | Total mem usage: 15.4 GiB | 20260506 03:23:38.128 | INFO | All Time High Water Marks sum: 16.3 GiB (Savings: 5.2%) | 20260506 03:23:38.128 | INFO | Recent High Water Marks Sum: 14.7 GiB (Savings: -5.18%) | 20260506 03:23:38.128 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 03:23:38.129 | INFO | High Water Breakdown: | 20260506 03:23:38.129 | INFO | 61.5% : 9.03 GiB ( 93.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 03:23:38.129 | INFO | 38.3% : 5.62 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 03:23:38.129 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 03:23:38.129 | INFO | ------------------------------------------ | 20260506 03:23:38.129 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:23:38.129 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 03:23:38.129 | INFO | Total mem usage: 26.5 GiB | 20260506 03:23:38.129 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 03:23:38.129 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 03:23:38.129 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 03:23:38.129 | INFO | High Water Breakdown: | 20260506 03:23:38.129 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 03:23:38.129 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 03:23:38.129 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 03:23:38.129 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 03:23:38.129 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 03:23:38.129 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 03:23:38.129 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 03:23:38.129 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 03:23:38.129 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 03:23:38.129 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 03:23:38.129 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 03:23:38.129 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 03:23:38.129 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 03:23:38.129 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 03:23:38.129 | INFO | ------------------------------------------ | 20260506 03:23:38.129 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:23:38.129 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 03:23:38.129 | INFO | Total mem usage: 25.6 GiB | 20260506 03:23:38.129 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 03:23:38.129 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 03:23:38.129 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 03:23:38.129 | INFO | High Water Breakdown: | 20260506 03:23:38.129 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 03:23:38.129 | INFO | ------------------------------------------ | 20260506 03:23:38.129 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:23:38.129 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 03:23:38.129 | INFO | Total mem usage: 15.7 GiB | 20260506 03:23:38.129 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 03:23:38.129 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 03:23:38.129 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 03:23:38.129 | INFO | High Water Breakdown: | 20260506 03:23:38.129 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 03:23:38.129 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 03:23:38.129 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 03:23:38.129 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 03:23:38.129 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 03:23:38.129 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 03:23:38.129 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 03:23:38.129 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 03:23:38.129 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 03:23:38.129 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 03:23:38.129 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 03:23:38.129 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 03:23:38.129 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 03:23:38.129 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 03:23:38.129 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 03:23:38.129 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 03:23:38.129 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 03:23:38.129 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 03:23:38.129 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 03:23:38.129 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 03:23:38.129 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 03:23:38.129 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 03:23:38.129 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 03:23:38.129 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 03:23:38.129 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 03:23:38.129 | INFO | ------------------------------------------ | 20260506 03:23:38.129 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:23:38.129 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 03:23:38.129 | INFO | PerfReport: Gathered 1536 samples. | 20260506 03:23:38.129 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 03:23:38.129 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.39e+01, 1.34e+01 / 7.90e-01 | 20260506 03:23:38.129 | INFO | PerfReport: 0.29%, AggregateStats -- 9.16e-02 / 1.05e-01, 9.87e-03 / 3.60e-03 | 20260506 03:23:38.129 | INFO | PerfReport: 11.88%, Baseline -- 3.73e+00 / 4.51e+00, 5.92e-01 / 2.54e-01 | 20260506 03:23:38.129 | INFO | PerfReport: 3.32%, Histogram -- 1.04e+00 / 1.52e+00, 2.00e-01 / 9.79e-02 | 20260506 03:23:38.129 | INFO | PerfReport: 16.00%, DME -- 4.02e+01 / 4.43e+01, 3.81e+00 / 9.48e-01 | 20260506 03:23:38.129 | INFO | PerfReport: 57.60%, FrameLabeling -- 1.81e+01 / 1.92e+01, 1.64e+00 / 9.91e-01 | 20260506 03:23:38.129 | INFO | PerfReport: 6.92%, PulseAccumulating -- 2.17e+00 / 2.74e+00, 4.05e-01 / 1.78e-01 | 20260506 03:23:38.129 | INFO | PerfReport: 2.96%, Metrics -- 9.28e-01 / 1.20e+00, 2.50e-01 / 1.26e-01 | 20260506 03:23:38.129 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.21e-01 / 3.66e-01, 5.38e-02 / 2.27e-02 | 20260506 03:23:38.129 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 03:23:38.129 | INFO | PerfReport: Gathered 1536 samples. | 20260506 03:23:38.129 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 03:23:38.129 | INFO | PerfReport: Full Execution -- 9.73e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 03:23:38.129 | INFO | PerfReport: 22.04%, Upload -- 2.14e+01 / 0.00e+00, 6.82e+00 / 0.00e+00 | 20260506 03:23:38.129 | INFO | PerfReport: 21.94%, Download -- 2.13e+01 / 0.00e+00, 7.49e+00 / 0.00e+00 | 20260506 03:23:38.129 | INFO | PerfReport: 32.26%, Compute -- 3.14e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 03:23:38.129 | INFO | PerfReport: 1.94%, LockUpload -- 1.89e+00 / 0.00e+00, 7.72e+00 / 0.00e+00 | 20260506 03:23:38.129 | INFO | PerfReport: 1.27%, LockDownload -- 1.23e+00 / 0.00e+00, 2.95e+00 / 0.00e+00 | 20260506 03:23:38.129 | INFO | PerfReport: 20.54%, LockCompute -- 2.00e+01 / 0.00e+00, 1.74e+01 / 0.00e+00 | 20260506 03:23:38.129 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 2.07e-02 / 0.00e+00 | 20260506 03:23:38.253 | INFO | Starting a round of Baz IO | 20260506 03:23:41.373 | INFO | Populating RTMetrics report | 20260506 03:23:41.373 | INFO | RTMetrics report populated | 20260506 03:23:41.373 | INFO | Opening tmp RTMetrics json file | 20260506 03:23:41.374 | INFO | Serializing RTMetrics to json | 20260506 03:23:41.376 | INFO | Serialization done, swapping tmp file now... | 20260506 03:23:41.378 | INFO | RTMetrics done | 20260506 03:23:41.459 | INFO | Finished a round of Baz IO: Wrote 1.763 GiB in 3.206 seconds | 20260506 03:23:41.459 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.23% / 0.00% / 78.77% | 20260506 03:23:41.459 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0613689% / 0% / 0.000107288% | 20260506 03:23:41.459 | INFO | Baz IO overall: | 20260506 03:23:41.459 | INFO | BytesWritten : 4764451671510 | 20260506 03:23:41.460 | INFO | NumEvents : 2234376098545 | 20260506 03:23:43.265 | INFO | Analyzing chunk frames = [8560640,8561152) | 20260506 03:23:43.265 | INFO | MemInfo: { | 20260506 03:23:43.265 | INFO | "gpu" : " 17 GB", | 20260506 03:23:43.265 | INFO | "huge" : "", | 20260506 03:23:43.265 | INFO | "huge_shared" : " 27 GB", | 20260506 03:23:43.265 | INFO | "malloc" : " 14 GB", | 20260506 03:23:43.265 | INFO | "pinned" : " 29 GB" | 20260506 03:23:43.265 | INFO | } | 20260506 03:23:43.265 | INFO | Perf report for the last 5.13659 seconds: | 20260506 03:23:43.265 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:23:43.265 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:43.265 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:43.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:43.265 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 03:23:43.265 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:43.265 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260506 03:23:43.265 | INFO | PRE_HQ : 59.0%, 1.7, 53.2ms, 0 | 20260506 03:23:43.265 | INFO | BAZWRITER : 61.6%, 3.5, 109.8ms, 0 | 20260506 03:23:43.265 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260506 03:23:43.265 | INFO | Uploaded 36GiB to GPU in 2.08856s (17.2368GiB/s) | 20260506 03:23:43.265 | INFO | Downloaded 38GiB to GPU in 2.22544s (17.0753GiB/s) | 20260506 03:23:48.407 | INFO | Analyzing chunk frames = [8561152,8561664) | 20260506 03:23:48.407 | INFO | MemInfo: { | 20260506 03:23:48.407 | INFO | "gpu" : " 17 GB", | 20260506 03:23:48.407 | INFO | "huge" : "", | 20260506 03:23:48.407 | INFO | "huge_shared" : " 27 GB", | 20260506 03:23:48.407 | INFO | "malloc" : " 14 GB", | 20260506 03:23:48.407 | INFO | "pinned" : " 29 GB" | 20260506 03:23:48.407 | INFO | } | 20260506 03:23:48.407 | INFO | Perf report for the last 5.14239 seconds: | 20260506 03:23:48.407 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:23:48.407 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:48.407 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:48.407 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:48.407 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260506 03:23:48.407 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:48.407 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:48.407 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 03:23:48.407 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:48.407 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:48.407 | INFO | Uploaded 36GiB to GPU in 1.95141s (18.4482GiB/s) | 20260506 03:23:48.407 | INFO | Downloaded 34GiB to GPU in 1.98409s (17.1364GiB/s) | 20260506 03:23:53.549 | INFO | Analyzing chunk frames = [8561664,8562176) | 20260506 03:23:53.549 | INFO | MemInfo: { | 20260506 03:23:53.549 | INFO | "gpu" : " 17 GB", | 20260506 03:23:53.549 | INFO | "huge" : "", | 20260506 03:23:53.549 | INFO | "huge_shared" : " 27 GB", | 20260506 03:23:53.549 | INFO | "malloc" : " 14 GB", | 20260506 03:23:53.549 | INFO | "pinned" : " 29 GB" | 20260506 03:23:53.549 | INFO | } | 20260506 03:23:53.549 | INFO | Perf report for the last 5.14193 seconds: | 20260506 03:23:53.549 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:23:53.549 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:53.549 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:53.549 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:53.549 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260506 03:23:53.549 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:53.549 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:53.549 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 03:23:53.549 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:53.549 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:53.549 | INFO | Uploaded 36GiB to GPU in 1.99921s (18.0071GiB/s) | 20260506 03:23:53.549 | INFO | Downloaded 34GiB to GPU in 1.98958s (17.0891GiB/s) | 20260506 03:23:58.682 | INFO | Analyzing chunk frames = [8562176,8562688) | 20260506 03:23:58.682 | INFO | MemInfo: { | 20260506 03:23:58.682 | INFO | "gpu" : " 17 GB", | 20260506 03:23:58.682 | INFO | "huge" : "", | 20260506 03:23:58.683 | INFO | "huge_shared" : " 27 GB", | 20260506 03:23:58.683 | INFO | "malloc" : " 14 GB", | 20260506 03:23:58.683 | INFO | "pinned" : " 29 GB" | 20260506 03:23:58.683 | INFO | } | 20260506 03:23:58.683 | INFO | Perf report for the last 5.13365 seconds: | 20260506 03:23:58.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:23:58.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:58.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:58.683 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:58.683 | INFO | ANALYSIS : 61.0%, 3.0, 94.7ms, 0 | 20260506 03:23:58.683 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:58.683 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:58.683 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 03:23:58.683 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:58.683 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:23:58.683 | INFO | Uploaded 36GiB to GPU in 2.0939s (17.1928GiB/s) | 20260506 03:23:58.683 | INFO | Downloaded 34GiB to GPU in 2.01085s (16.9083GiB/s) | 20260506 03:24:03.824 | INFO | Analyzing chunk frames = [8562688,8563200) | 20260506 03:24:03.824 | INFO | MemInfo: { | 20260506 03:24:03.824 | INFO | "gpu" : " 17 GB", | 20260506 03:24:03.824 | INFO | "huge" : "", | 20260506 03:24:03.824 | INFO | "huge_shared" : " 27 GB", | 20260506 03:24:03.824 | INFO | "malloc" : " 14 GB", | 20260506 03:24:03.824 | INFO | "pinned" : " 29 GB" | 20260506 03:24:03.824 | INFO | } | 20260506 03:24:03.824 | INFO | Perf report for the last 5.1413 seconds: | 20260506 03:24:03.824 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:24:03.824 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:03.824 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:03.824 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:03.824 | INFO | ANALYSIS : 60.7%, 3.0, 94.8ms, 0 | 20260506 03:24:03.824 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:03.824 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:03.824 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 03:24:03.824 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:03.824 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:03.824 | INFO | Uploaded 36GiB to GPU in 2.11612s (17.0123GiB/s) | 20260506 03:24:03.824 | INFO | Downloaded 34GiB to GPU in 2.03313s (16.723GiB/s) | 20260506 03:24:08.960 | INFO | Analyzing chunk frames = [8563200,8563712) | 20260506 03:24:08.960 | INFO | MemInfo: { | 20260506 03:24:08.960 | INFO | "gpu" : " 17 GB", | 20260506 03:24:08.960 | INFO | "huge" : "", | 20260506 03:24:08.960 | INFO | "huge_shared" : " 27 GB", | 20260506 03:24:08.960 | INFO | "malloc" : " 14 GB", | 20260506 03:24:08.960 | INFO | "pinned" : " 29 GB" | 20260506 03:24:08.960 | INFO | } | 20260506 03:24:08.961 | INFO | Perf report for the last 5.13653 seconds: | 20260506 03:24:08.961 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:24:08.961 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:08.961 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:08.961 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:08.961 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 03:24:08.961 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:08.961 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:08.961 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:24:08.961 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:08.961 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:08.961 | INFO | Uploaded 36GiB to GPU in 2.11908s (16.9885GiB/s) | 20260506 03:24:08.961 | INFO | Downloaded 34GiB to GPU in 2.04844s (16.598GiB/s) | 20260506 03:24:14.102 | INFO | Analyzing chunk frames = [8563712,8564224) | 20260506 03:24:14.102 | INFO | MemInfo: { | 20260506 03:24:14.102 | INFO | "gpu" : " 17 GB", | 20260506 03:24:14.102 | INFO | "huge" : "", | 20260506 03:24:14.102 | INFO | "huge_shared" : " 27 GB", | 20260506 03:24:14.102 | INFO | "malloc" : " 14 GB", | 20260506 03:24:14.102 | INFO | "pinned" : " 29 GB" | 20260506 03:24:14.102 | INFO | } | 20260506 03:24:14.102 | INFO | Perf report for the last 5.14154 seconds: | 20260506 03:24:14.102 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:24:14.102 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:14.102 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:14.102 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:14.102 | INFO | ANALYSIS : 61.8%, 3.0, 96.6ms, 0 | 20260506 03:24:14.102 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:14.102 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:14.102 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 03:24:14.102 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:14.102 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:14.102 | INFO | Uploaded 36GiB to GPU in 2.08879s (17.2348GiB/s) | 20260506 03:24:14.102 | INFO | Downloaded 34GiB to GPU in 2.05827s (16.5187GiB/s) | 20260506 03:24:19.237 | INFO | Analyzing chunk frames = [8564224,8564736) | 20260506 03:24:19.237 | INFO | MemInfo: { | 20260506 03:24:19.237 | INFO | "gpu" : " 17 GB", | 20260506 03:24:19.237 | INFO | "huge" : "", | 20260506 03:24:19.237 | INFO | "huge_shared" : " 27 GB", | 20260506 03:24:19.237 | INFO | "malloc" : " 14 GB", | 20260506 03:24:19.237 | INFO | "pinned" : " 29 GB" | 20260506 03:24:19.237 | INFO | } | 20260506 03:24:19.237 | INFO | Perf report for the last 5.13515 seconds: | 20260506 03:24:19.237 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:24:19.237 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:19.237 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:19.237 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:19.237 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 03:24:19.237 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:19.237 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:19.237 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 03:24:19.237 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:19.237 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:19.237 | INFO | Uploaded 36GiB to GPU in 2.05356s (17.5306GiB/s) | 20260506 03:24:19.238 | INFO | Downloaded 34GiB to GPU in 2.02403s (16.7981GiB/s) | 20260506 03:24:19.364 | INFO | Starting a round of Baz IO | 20260506 03:24:22.490 | INFO | Populating RTMetrics report | 20260506 03:24:22.491 | INFO | RTMetrics report populated | 20260506 03:24:22.491 | INFO | Opening tmp RTMetrics json file | 20260506 03:24:22.491 | INFO | Serializing RTMetrics to json | 20260506 03:24:22.492 | INFO | Serialization done, swapping tmp file now... | 20260506 03:24:22.493 | INFO | RTMetrics done | 20260506 03:24:22.563 | INFO | Finished a round of Baz IO: Wrote 1.762 GiB in 3.199 seconds | 20260506 03:24:22.563 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.23% / 0.00% / 78.77% | 20260506 03:24:22.563 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0617067% / 0% / 8.74201e-05% | 20260506 03:24:22.563 | INFO | Baz IO overall: | 20260506 03:24:22.563 | INFO | BytesWritten : 4766343662932 | 20260506 03:24:22.563 | INFO | NumEvents : 2235233918684 | 20260506 03:24:24.371 | INFO | Analyzing chunk frames = [8564736,8565248) | 20260506 03:24:24.371 | INFO | MemInfo: { | 20260506 03:24:24.371 | INFO | "gpu" : " 17 GB", | 20260506 03:24:24.371 | INFO | "huge" : "", | 20260506 03:24:24.371 | INFO | "huge_shared" : " 27 GB", | 20260506 03:24:24.371 | INFO | "malloc" : " 14 GB", | 20260506 03:24:24.371 | INFO | "pinned" : " 29 GB" | 20260506 03:24:24.371 | INFO | } | 20260506 03:24:24.371 | INFO | Perf report for the last 5.13411 seconds: | 20260506 03:24:24.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:24:24.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:24.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:24.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:24.371 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260506 03:24:24.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:24.371 | INFO | RT_METRICS : 42.6%, 1.0, 22.3ms, 0 | 20260506 03:24:24.371 | INFO | PRE_HQ : 59.1%, 1.7, 53.2ms, 0 | 20260506 03:24:24.371 | INFO | BAZWRITER : 61.5%, 3.5, 109.7ms, 0 | 20260506 03:24:24.371 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 03:24:24.372 | INFO | Uploaded 36GiB to GPU in 2.1232s (16.9555GiB/s) | 20260506 03:24:24.372 | INFO | Downloaded 38GiB to GPU in 2.23541s (16.9991GiB/s) | 20260506 03:24:29.517 | INFO | Analyzing chunk frames = [8565248,8565760) | 20260506 03:24:29.517 | INFO | MemInfo: { | 20260506 03:24:29.517 | INFO | "gpu" : " 17 GB", | 20260506 03:24:29.517 | INFO | "huge" : "", | 20260506 03:24:29.517 | INFO | "huge_shared" : " 27 GB", | 20260506 03:24:29.517 | INFO | "malloc" : " 14 GB", | 20260506 03:24:29.517 | INFO | "pinned" : " 29 GB" | 20260506 03:24:29.517 | INFO | } | 20260506 03:24:29.517 | INFO | Perf report for the last 5.14593 seconds: | 20260506 03:24:29.517 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:24:29.517 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:29.517 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:29.517 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:29.517 | INFO | ANALYSIS : 60.8%, 3.0, 95.0ms, 0 | 20260506 03:24:29.517 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:29.517 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:29.517 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 03:24:29.517 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:29.517 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:29.518 | INFO | Uploaded 36GiB to GPU in 1.99343s (18.0593GiB/s) | 20260506 03:24:29.518 | INFO | Downloaded 34GiB to GPU in 1.99607s (17.0334GiB/s) | 20260506 03:24:34.657 | INFO | Analyzing chunk frames = [8565760,8566272) | 20260506 03:24:34.657 | INFO | MemInfo: { | 20260506 03:24:34.657 | INFO | "gpu" : " 17 GB", | 20260506 03:24:34.657 | INFO | "huge" : "", | 20260506 03:24:34.658 | INFO | "huge_shared" : " 27 GB", | 20260506 03:24:34.658 | INFO | "malloc" : " 14 GB", | 20260506 03:24:34.658 | INFO | "pinned" : " 29 GB" | 20260506 03:24:34.658 | INFO | } | 20260506 03:24:34.658 | INFO | Perf report for the last 5.14034 seconds: | 20260506 03:24:34.658 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:24:34.658 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:34.658 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:34.658 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:34.658 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 03:24:34.658 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:34.658 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:34.658 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 03:24:34.658 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:34.658 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:34.658 | INFO | Uploaded 36GiB to GPU in 2.10564s (17.0969GiB/s) | 20260506 03:24:34.658 | INFO | Downloaded 34GiB to GPU in 2.0415s (16.6544GiB/s) | 20260506 03:24:39.800 | INFO | Analyzing chunk frames = [8566272,8566784) | 20260506 03:24:39.800 | INFO | MemInfo: { | 20260506 03:24:39.800 | INFO | "gpu" : " 17 GB", | 20260506 03:24:39.800 | INFO | "huge" : "", | 20260506 03:24:39.800 | INFO | "huge_shared" : " 27 GB", | 20260506 03:24:39.801 | INFO | "malloc" : " 14 GB", | 20260506 03:24:39.801 | INFO | "pinned" : " 29 GB" | 20260506 03:24:39.801 | INFO | } | 20260506 03:24:39.801 | INFO | Perf report for the last 5.14294 seconds: | 20260506 03:24:39.801 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:24:39.801 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:39.801 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:39.801 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:39.801 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260506 03:24:39.801 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:39.801 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:39.801 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 03:24:39.801 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:39.801 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:39.801 | INFO | Uploaded 36GiB to GPU in 2.08964s (17.2279GiB/s) | 20260506 03:24:39.801 | INFO | Downloaded 34GiB to GPU in 2.01381s (16.8834GiB/s) | 20260506 03:24:44.936 | INFO | Analyzing chunk frames = [8566784,8567296) | 20260506 03:24:44.936 | INFO | MemInfo: { | 20260506 03:24:44.936 | INFO | "gpu" : " 17 GB", | 20260506 03:24:44.936 | INFO | "huge" : "", | 20260506 03:24:44.936 | INFO | "huge_shared" : " 27 GB", | 20260506 03:24:44.936 | INFO | "malloc" : " 14 GB", | 20260506 03:24:44.936 | INFO | "pinned" : " 29 GB" | 20260506 03:24:44.936 | INFO | } | 20260506 03:24:44.936 | INFO | Perf report for the last 5.13531 seconds: | 20260506 03:24:44.936 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:24:44.936 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:44.936 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:44.936 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:44.936 | INFO | ANALYSIS : 60.9%, 3.0, 95.0ms, 0 | 20260506 03:24:44.936 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:44.936 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:44.936 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:24:44.936 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:44.936 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:44.936 | INFO | Uploaded 36GiB to GPU in 2.1187s (16.9915GiB/s) | 20260506 03:24:44.936 | INFO | Downloaded 34GiB to GPU in 2.03872s (16.6771GiB/s) | 20260506 03:24:50.073 | INFO | Analyzing chunk frames = [8567296,8567808) | 20260506 03:24:50.073 | INFO | MemInfo: { | 20260506 03:24:50.073 | INFO | "gpu" : " 17 GB", | 20260506 03:24:50.073 | INFO | "huge" : "", | 20260506 03:24:50.073 | INFO | "huge_shared" : " 27 GB", | 20260506 03:24:50.073 | INFO | "malloc" : " 14 GB", | 20260506 03:24:50.073 | INFO | "pinned" : " 29 GB" | 20260506 03:24:50.073 | INFO | } | 20260506 03:24:50.073 | INFO | Perf report for the last 5.13682 seconds: | 20260506 03:24:50.073 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:24:50.073 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:50.073 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:50.073 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:50.073 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 03:24:50.073 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:50.073 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:50.073 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 03:24:50.073 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:50.073 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:50.073 | INFO | Uploaded 36GiB to GPU in 2.11375s (17.0313GiB/s) | 20260506 03:24:50.073 | INFO | Downloaded 34GiB to GPU in 2.04447s (16.6302GiB/s) | 20260506 03:24:55.214 | INFO | Analyzing chunk frames = [8567808,8568320) | 20260506 03:24:55.214 | INFO | MemInfo: { | 20260506 03:24:55.214 | INFO | "gpu" : " 17 GB", | 20260506 03:24:55.214 | INFO | "huge" : "", | 20260506 03:24:55.214 | INFO | "huge_shared" : " 27 GB", | 20260506 03:24:55.214 | INFO | "malloc" : " 14 GB", | 20260506 03:24:55.214 | INFO | "pinned" : " 29 GB" | 20260506 03:24:55.214 | INFO | } | 20260506 03:24:55.214 | INFO | Perf report for the last 5.14126 seconds: | 20260506 03:24:55.214 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:24:55.214 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:55.214 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:55.214 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:55.214 | INFO | ANALYSIS : 61.9%, 3.0, 96.7ms, 0 | 20260506 03:24:55.214 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:55.214 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:55.214 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 03:24:55.214 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:55.214 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:24:55.214 | INFO | Uploaded 36GiB to GPU in 2.05976s (17.4778GiB/s) | 20260506 03:24:55.214 | INFO | Downloaded 34GiB to GPU in 2.06868s (16.4356GiB/s) | 20260506 03:25:00.354 | INFO | Analyzing chunk frames = [8568320,8568832) | 20260506 03:25:00.354 | INFO | MemInfo: { | 20260506 03:25:00.354 | INFO | "gpu" : " 17 GB", | 20260506 03:25:00.354 | INFO | "huge" : "", | 20260506 03:25:00.354 | INFO | "huge_shared" : " 27 GB", | 20260506 03:25:00.354 | INFO | "malloc" : " 14 GB", | 20260506 03:25:00.354 | INFO | "pinned" : " 29 GB" | 20260506 03:25:00.354 | INFO | } | 20260506 03:25:00.354 | INFO | Perf report for the last 5.13998 seconds: | 20260506 03:25:00.354 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:25:00.354 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:00.354 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:00.354 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:00.354 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260506 03:25:00.354 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:00.354 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:00.354 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:25:00.354 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:00.354 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:00.354 | INFO | Uploaded 36GiB to GPU in 2.04651s (17.5909GiB/s) | 20260506 03:25:00.354 | INFO | Downloaded 34GiB to GPU in 2.01898s (16.8402GiB/s) | 20260506 03:25:00.354 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 03:25:00.354 | INFO | Total mem usage: 15.4 GiB | 20260506 03:25:00.354 | INFO | All Time High Water Marks sum: 16.3 GiB (Savings: 5.2%) | 20260506 03:25:00.354 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -10%) | 20260506 03:25:00.354 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 03:25:00.355 | INFO | High Water Breakdown: | 20260506 03:25:00.355 | INFO | 60.9% : 8.54 GiB ( 88.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 03:25:00.355 | INFO | 39% : 5.47 GiB ( 82.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 03:25:00.355 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 03:25:00.355 | INFO | ------------------------------------------ | 20260506 03:25:00.355 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:25:00.355 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 03:25:00.355 | INFO | Total mem usage: 26.5 GiB | 20260506 03:25:00.355 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 03:25:00.355 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 03:25:00.355 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 03:25:00.355 | INFO | High Water Breakdown: | 20260506 03:25:00.355 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 03:25:00.355 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 03:25:00.355 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 03:25:00.355 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 03:25:00.355 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 03:25:00.355 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 03:25:00.355 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 03:25:00.355 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 03:25:00.355 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 03:25:00.355 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 03:25:00.355 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 03:25:00.355 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 03:25:00.355 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 03:25:00.355 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 03:25:00.355 | INFO | ------------------------------------------ | 20260506 03:25:00.355 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:25:00.355 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 03:25:00.355 | INFO | Total mem usage: 25.6 GiB | 20260506 03:25:00.355 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 03:25:00.355 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 03:25:00.355 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 03:25:00.355 | INFO | High Water Breakdown: | 20260506 03:25:00.355 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 03:25:00.355 | INFO | ------------------------------------------ | 20260506 03:25:00.355 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:25:00.355 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 03:25:00.355 | INFO | Total mem usage: 15.7 GiB | 20260506 03:25:00.355 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 03:25:00.355 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 03:25:00.355 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 03:25:00.355 | INFO | High Water Breakdown: | 20260506 03:25:00.355 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 03:25:00.355 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 03:25:00.355 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 03:25:00.355 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 03:25:00.355 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 03:25:00.355 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 03:25:00.355 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 03:25:00.355 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 03:25:00.355 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 03:25:00.355 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 03:25:00.355 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 03:25:00.355 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 03:25:00.355 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 03:25:00.355 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 03:25:00.355 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 03:25:00.355 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 03:25:00.355 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 03:25:00.356 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 03:25:00.356 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 03:25:00.356 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 03:25:00.356 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 03:25:00.356 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 03:25:00.356 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 03:25:00.356 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 03:25:00.356 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 03:25:00.356 | INFO | ------------------------------------------ | 20260506 03:25:00.356 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:25:00.356 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 03:25:00.356 | INFO | PerfReport: Gathered 1536 samples. | 20260506 03:25:00.356 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 03:25:00.356 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.36e+01, 1.35e+01 / 7.37e-01 | 20260506 03:25:00.356 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.03e-01, 1.19e-02 / 3.58e-03 | 20260506 03:25:00.356 | INFO | PerfReport: 11.89%, Baseline -- 3.72e+00 / 4.55e+00, 5.88e-01 / 2.07e-01 | 20260506 03:25:00.356 | INFO | PerfReport: 3.30%, Histogram -- 1.03e+00 / 1.54e+00, 1.94e-01 / 7.38e-02 | 20260506 03:25:00.356 | INFO | PerfReport: 15.99%, DME -- 4.00e+01 / 4.39e+01, 3.85e+00 / 7.58e-01 | 20260506 03:25:00.356 | INFO | PerfReport: 57.54%, FrameLabeling -- 1.80e+01 / 1.92e+01, 1.52e+00 / 8.62e-01 | 20260506 03:25:00.356 | INFO | PerfReport: 6.99%, PulseAccumulating -- 2.18e+00 / 2.76e+00, 4.14e-01 / 1.49e-01 | 20260506 03:25:00.356 | INFO | PerfReport: 2.98%, Metrics -- 9.30e-01 / 1.21e+00, 2.49e-01 / 1.14e-01 | 20260506 03:25:00.356 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.20e-01 / 3.68e-01, 5.40e-02 / 2.96e-02 | 20260506 03:25:00.356 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 03:25:00.356 | INFO | PerfReport: Gathered 1536 samples. | 20260506 03:25:00.356 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 03:25:00.356 | INFO | PerfReport: Full Execution -- 9.74e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260506 03:25:00.356 | INFO | PerfReport: 22.16%, Upload -- 2.16e+01 / 0.00e+00, 7.08e+00 / 0.00e+00 | 20260506 03:25:00.356 | INFO | PerfReport: 21.94%, Download -- 2.14e+01 / 0.00e+00, 7.52e+00 / 0.00e+00 | 20260506 03:25:00.356 | INFO | PerfReport: 32.09%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 03:25:00.356 | INFO | PerfReport: 2.00%, LockUpload -- 1.95e+00 / 0.00e+00, 7.73e+00 / 0.00e+00 | 20260506 03:25:00.356 | INFO | PerfReport: 1.30%, LockDownload -- 1.27e+00 / 0.00e+00, 2.95e+00 / 0.00e+00 | 20260506 03:25:00.356 | INFO | PerfReport: 20.51%, LockCompute -- 2.00e+01 / 0.00e+00, 1.75e+01 / 0.00e+00 | 20260506 03:25:00.356 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.17e-02 / 0.00e+00 | 20260506 03:25:00.490 | INFO | Starting a round of Baz IO | 20260506 03:25:03.602 | INFO | Populating RTMetrics report | 20260506 03:25:03.602 | INFO | RTMetrics report populated | 20260506 03:25:03.602 | INFO | Opening tmp RTMetrics json file | 20260506 03:25:03.603 | INFO | Serializing RTMetrics to json | 20260506 03:25:03.604 | INFO | Serialization done, swapping tmp file now... | 20260506 03:25:03.605 | INFO | RTMetrics done | 20260506 03:25:03.676 | INFO | Finished a round of Baz IO: Wrote 1.759 GiB in 3.185 seconds | 20260506 03:25:03.676 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.23% / 0.00% / 78.77% | 20260506 03:25:03.676 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0614603% / 0% / 7.15256e-05% | 20260506 03:25:03.676 | INFO | Baz IO overall: | 20260506 03:25:03.676 | INFO | BytesWritten : 4768231991201 | 20260506 03:25:03.676 | INFO | NumEvents : 2236089244384 | 20260506 03:25:05.501 | INFO | Analyzing chunk frames = [8568832,8569344) | 20260506 03:25:05.501 | INFO | MemInfo: { | 20260506 03:25:05.501 | INFO | "gpu" : " 17 GB", | 20260506 03:25:05.501 | INFO | "huge" : "", | 20260506 03:25:05.501 | INFO | "huge_shared" : " 27 GB", | 20260506 03:25:05.501 | INFO | "malloc" : " 14 GB", | 20260506 03:25:05.501 | INFO | "pinned" : " 29 GB" | 20260506 03:25:05.501 | INFO | } | 20260506 03:25:05.501 | INFO | Perf report for the last 5.14727 seconds: | 20260506 03:25:05.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:25:05.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:05.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:05.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:05.501 | INFO | ANALYSIS : 62.1%, 3.0, 97.1ms, 0 | 20260506 03:25:05.502 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:05.502 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260506 03:25:05.502 | INFO | PRE_HQ : 59.2%, 1.7, 53.1ms, 0 | 20260506 03:25:05.502 | INFO | BAZWRITER : 61.7%, 3.5, 110.1ms, 0 | 20260506 03:25:05.502 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 03:25:05.502 | INFO | Uploaded 36GiB to GPU in 2.09982s (17.1443GiB/s) | 20260506 03:25:05.502 | INFO | Downloaded 38GiB to GPU in 2.23613s (16.9936GiB/s) | 20260506 03:25:10.634 | INFO | Analyzing chunk frames = [8569344,8569856) | 20260506 03:25:10.634 | INFO | MemInfo: { | 20260506 03:25:10.634 | INFO | "gpu" : " 17 GB", | 20260506 03:25:10.634 | INFO | "huge" : "", | 20260506 03:25:10.634 | INFO | "huge_shared" : " 27 GB", | 20260506 03:25:10.634 | INFO | "malloc" : " 14 GB", | 20260506 03:25:10.634 | INFO | "pinned" : " 29 GB" | 20260506 03:25:10.634 | INFO | } | 20260506 03:25:10.634 | INFO | Perf report for the last 5.13277 seconds: | 20260506 03:25:10.634 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:25:10.634 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:10.634 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:10.634 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:10.634 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260506 03:25:10.634 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:10.634 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:10.634 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 03:25:10.634 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:10.634 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:10.635 | INFO | Uploaded 36GiB to GPU in 2.02822s (17.7496GiB/s) | 20260506 03:25:10.635 | INFO | Downloaded 34GiB to GPU in 1.99733s (17.0227GiB/s) | 20260506 03:25:15.773 | INFO | Analyzing chunk frames = [8569856,8570368) | 20260506 03:25:15.773 | INFO | MemInfo: { | 20260506 03:25:15.773 | INFO | "gpu" : " 17 GB", | 20260506 03:25:15.773 | INFO | "huge" : "", | 20260506 03:25:15.773 | INFO | "huge_shared" : " 27 GB", | 20260506 03:25:15.773 | INFO | "malloc" : " 14 GB", | 20260506 03:25:15.773 | INFO | "pinned" : " 29 GB" | 20260506 03:25:15.773 | INFO | } | 20260506 03:25:15.773 | INFO | Perf report for the last 5.13933 seconds: | 20260506 03:25:15.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:25:15.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:15.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:15.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:15.773 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 03:25:15.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:15.773 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:15.774 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:25:15.774 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:15.774 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:15.774 | INFO | Uploaded 36GiB to GPU in 2.10423s (17.1084GiB/s) | 20260506 03:25:15.774 | INFO | Downloaded 34GiB to GPU in 2.04331s (16.6396GiB/s) | 20260506 03:25:20.909 | INFO | Analyzing chunk frames = [8570368,8570880) | 20260506 03:25:20.909 | INFO | MemInfo: { | 20260506 03:25:20.909 | INFO | "gpu" : " 17 GB", | 20260506 03:25:20.909 | INFO | "huge" : "", | 20260506 03:25:20.909 | INFO | "huge_shared" : " 27 GB", | 20260506 03:25:20.909 | INFO | "malloc" : " 14 GB", | 20260506 03:25:20.909 | INFO | "pinned" : " 29 GB" | 20260506 03:25:20.909 | INFO | } | 20260506 03:25:20.909 | INFO | Perf report for the last 5.13564 seconds: | 20260506 03:25:20.909 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:25:20.909 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:20.909 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:20.909 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:20.909 | INFO | ANALYSIS : 61.1%, 3.0, 94.8ms, 0 | 20260506 03:25:20.909 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:20.909 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:20.909 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:25:20.909 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:20.909 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:20.909 | INFO | Uploaded 36GiB to GPU in 2.09111s (17.2157GiB/s) | 20260506 03:25:20.909 | INFO | Downloaded 34GiB to GPU in 2.01407s (16.8813GiB/s) | 20260506 03:25:26.049 | INFO | Analyzing chunk frames = [8570880,8571392) | 20260506 03:25:26.049 | INFO | MemInfo: { | 20260506 03:25:26.049 | INFO | "gpu" : " 17 GB", | 20260506 03:25:26.049 | INFO | "huge" : "", | 20260506 03:25:26.049 | INFO | "huge_shared" : " 27 GB", | 20260506 03:25:26.049 | INFO | "malloc" : " 14 GB", | 20260506 03:25:26.049 | INFO | "pinned" : " 29 GB" | 20260506 03:25:26.049 | INFO | } | 20260506 03:25:26.049 | INFO | Perf report for the last 5.14038 seconds: | 20260506 03:25:26.049 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:25:26.049 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:26.049 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:26.049 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:26.049 | INFO | ANALYSIS : 60.8%, 3.0, 94.9ms, 0 | 20260506 03:25:26.049 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:26.049 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:26.049 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:25:26.049 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:26.049 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:26.050 | INFO | Uploaded 36GiB to GPU in 2.12468s (16.9437GiB/s) | 20260506 03:25:26.050 | INFO | Downloaded 34GiB to GPU in 2.04692s (16.6103GiB/s) | 20260506 03:25:31.186 | INFO | Analyzing chunk frames = [8571392,8571904) | 20260506 03:25:31.186 | INFO | MemInfo: { | 20260506 03:25:31.186 | INFO | "gpu" : " 17 GB", | 20260506 03:25:31.186 | INFO | "huge" : "", | 20260506 03:25:31.186 | INFO | "huge_shared" : " 27 GB", | 20260506 03:25:31.186 | INFO | "malloc" : " 14 GB", | 20260506 03:25:31.186 | INFO | "pinned" : " 29 GB" | 20260506 03:25:31.186 | INFO | } | 20260506 03:25:31.186 | INFO | Perf report for the last 5.13642 seconds: | 20260506 03:25:31.186 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:25:31.186 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:31.186 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:31.186 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:31.186 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 03:25:31.186 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:31.186 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:31.186 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 03:25:31.186 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:31.186 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:31.186 | INFO | Uploaded 36GiB to GPU in 2.08975s (17.227GiB/s) | 20260506 03:25:31.186 | INFO | Downloaded 34GiB to GPU in 2.04033s (16.664GiB/s) | 20260506 03:25:36.328 | INFO | Analyzing chunk frames = [8571904,8572416) | 20260506 03:25:36.328 | INFO | MemInfo: { | 20260506 03:25:36.328 | INFO | "gpu" : " 17 GB", | 20260506 03:25:36.328 | INFO | "huge" : "", | 20260506 03:25:36.328 | INFO | "huge_shared" : " 27 GB", | 20260506 03:25:36.328 | INFO | "malloc" : " 14 GB", | 20260506 03:25:36.328 | INFO | "pinned" : " 29 GB" | 20260506 03:25:36.328 | INFO | } | 20260506 03:25:36.328 | INFO | Perf report for the last 5.14239 seconds: | 20260506 03:25:36.328 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:25:36.328 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:36.328 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:36.328 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:36.328 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260506 03:25:36.328 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:36.328 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:36.328 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 03:25:36.328 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:36.328 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:36.329 | INFO | Uploaded 36GiB to GPU in 2.06114s (17.466GiB/s) | 20260506 03:25:36.329 | INFO | Downloaded 34GiB to GPU in 2.07737s (16.3668GiB/s) | 20260506 03:25:41.471 | INFO | Analyzing chunk frames = [8572416,8572928) | 20260506 03:25:41.471 | INFO | MemInfo: { | 20260506 03:25:41.471 | INFO | "gpu" : " 17 GB", | 20260506 03:25:41.471 | INFO | "huge" : "", | 20260506 03:25:41.471 | INFO | "huge_shared" : " 27 GB", | 20260506 03:25:41.471 | INFO | "malloc" : " 14 GB", | 20260506 03:25:41.471 | INFO | "pinned" : " 29 GB" | 20260506 03:25:41.471 | INFO | } | 20260506 03:25:41.471 | INFO | Perf report for the last 5.1425 seconds: | 20260506 03:25:41.471 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:25:41.471 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:41.471 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:41.471 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:41.471 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260506 03:25:41.471 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:41.471 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:41.471 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:25:41.471 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:41.471 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:41.471 | INFO | Uploaded 36GiB to GPU in 2.05972s (17.4781GiB/s) | 20260506 03:25:41.471 | INFO | Downloaded 34GiB to GPU in 2.01841s (16.8449GiB/s) | 20260506 03:25:41.592 | INFO | Starting a round of Baz IO | 20260506 03:25:44.712 | INFO | Populating RTMetrics report | 20260506 03:25:44.712 | INFO | RTMetrics report populated | 20260506 03:25:44.712 | INFO | Opening tmp RTMetrics json file | 20260506 03:25:44.713 | INFO | Serializing RTMetrics to json | 20260506 03:25:44.716 | INFO | Serialization done, swapping tmp file now... | 20260506 03:25:44.716 | INFO | RTMetrics done | 20260506 03:25:44.792 | INFO | Finished a round of Baz IO: Wrote 1.76 GiB in 3.2 seconds | 20260506 03:25:44.792 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.22% / 0.00% / 78.77% | 20260506 03:25:44.792 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0612775% / 3.97364e-06% / 8.34465e-05% | 20260506 03:25:44.792 | INFO | Baz IO overall: | 20260506 03:25:44.792 | INFO | BytesWritten : 4770121924152 | 20260506 03:25:44.793 | INFO | NumEvents : 2236945750864 | 20260506 03:25:46.612 | INFO | Analyzing chunk frames = [8572928,8573440) | 20260506 03:25:46.612 | INFO | MemInfo: { | 20260506 03:25:46.612 | INFO | "gpu" : " 17 GB", | 20260506 03:25:46.612 | INFO | "huge" : "", | 20260506 03:25:46.612 | INFO | "huge_shared" : " 27 GB", | 20260506 03:25:46.612 | INFO | "malloc" : " 14 GB", | 20260506 03:25:46.612 | INFO | "pinned" : " 29 GB" | 20260506 03:25:46.612 | INFO | } | 20260506 03:25:46.612 | INFO | Perf report for the last 5.14132 seconds: | 20260506 03:25:46.612 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:25:46.612 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:46.612 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:46.612 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:46.612 | INFO | ANALYSIS : 62.0%, 3.0, 96.9ms, 0 | 20260506 03:25:46.612 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:46.612 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260506 03:25:46.612 | INFO | PRE_HQ : 59.3%, 1.7, 53.4ms, 0 | 20260506 03:25:46.612 | INFO | BAZWRITER : 61.4%, 3.5, 110.5ms, 0 | 20260506 03:25:46.612 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 03:25:46.612 | INFO | Uploaded 36GiB to GPU in 2.09118s (17.2152GiB/s) | 20260506 03:25:46.612 | INFO | Downloaded 38GiB to GPU in 2.22371s (17.0886GiB/s) | 20260506 03:25:51.746 | INFO | Analyzing chunk frames = [8573440,8573952) | 20260506 03:25:51.746 | INFO | MemInfo: { | 20260506 03:25:51.746 | INFO | "gpu" : " 17 GB", | 20260506 03:25:51.746 | INFO | "huge" : "", | 20260506 03:25:51.746 | INFO | "huge_shared" : " 27 GB", | 20260506 03:25:51.746 | INFO | "malloc" : " 14 GB", | 20260506 03:25:51.746 | INFO | "pinned" : " 29 GB" | 20260506 03:25:51.746 | INFO | } | 20260506 03:25:51.746 | INFO | Perf report for the last 5.13412 seconds: | 20260506 03:25:51.746 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:25:51.746 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:51.746 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:51.746 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:51.746 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260506 03:25:51.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:51.746 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:51.746 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 03:25:51.746 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:51.746 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:51.747 | INFO | Uploaded 36GiB to GPU in 2.00464s (17.9583GiB/s) | 20260506 03:25:51.747 | INFO | Downloaded 34GiB to GPU in 2.00344s (16.9708GiB/s) | 20260506 03:25:56.883 | INFO | Analyzing chunk frames = [8573952,8574464) | 20260506 03:25:56.883 | INFO | MemInfo: { | 20260506 03:25:56.883 | INFO | "gpu" : " 17 GB", | 20260506 03:25:56.883 | INFO | "huge" : "", | 20260506 03:25:56.883 | INFO | "huge_shared" : " 27 GB", | 20260506 03:25:56.883 | INFO | "malloc" : " 14 GB", | 20260506 03:25:56.883 | INFO | "pinned" : " 29 GB" | 20260506 03:25:56.883 | INFO | } | 20260506 03:25:56.884 | INFO | Perf report for the last 5.13726 seconds: | 20260506 03:25:56.884 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:25:56.884 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:56.884 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:56.884 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:56.884 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 03:25:56.884 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:56.884 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:56.884 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 03:25:56.884 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:56.884 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:25:56.884 | INFO | Uploaded 36GiB to GPU in 2.1213s (16.9707GiB/s) | 20260506 03:25:56.884 | INFO | Downloaded 34GiB to GPU in 2.04782s (16.603GiB/s) | 20260506 03:26:02.024 | INFO | Analyzing chunk frames = [8574464,8574976) | 20260506 03:26:02.024 | INFO | MemInfo: { | 20260506 03:26:02.024 | INFO | "gpu" : " 17 GB", | 20260506 03:26:02.024 | INFO | "huge" : "", | 20260506 03:26:02.024 | INFO | "huge_shared" : " 27 GB", | 20260506 03:26:02.024 | INFO | "malloc" : " 14 GB", | 20260506 03:26:02.024 | INFO | "pinned" : " 29 GB" | 20260506 03:26:02.024 | INFO | } | 20260506 03:26:02.024 | INFO | Perf report for the last 5.14016 seconds: | 20260506 03:26:02.024 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:26:02.024 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:02.024 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:02.024 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:02.024 | INFO | ANALYSIS : 61.5%, 3.0, 95.5ms, 0 | 20260506 03:26:02.024 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:02.024 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:02.024 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 03:26:02.024 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:02.024 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:02.024 | INFO | Uploaded 36GiB to GPU in 2.14367s (16.7936GiB/s) | 20260506 03:26:02.024 | INFO | Downloaded 34GiB to GPU in 2.05774s (16.523GiB/s) | 20260506 03:26:07.165 | INFO | Analyzing chunk frames = [8574976,8575488) | 20260506 03:26:07.165 | INFO | MemInfo: { | 20260506 03:26:07.165 | INFO | "gpu" : " 17 GB", | 20260506 03:26:07.165 | INFO | "huge" : "", | 20260506 03:26:07.165 | INFO | "huge_shared" : " 27 GB", | 20260506 03:26:07.165 | INFO | "malloc" : " 14 GB", | 20260506 03:26:07.165 | INFO | "pinned" : " 29 GB" | 20260506 03:26:07.165 | INFO | } | 20260506 03:26:07.165 | INFO | Perf report for the last 5.14116 seconds: | 20260506 03:26:07.165 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:26:07.165 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:07.165 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:07.165 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:07.165 | INFO | ANALYSIS : 60.5%, 3.0, 94.5ms, 0 | 20260506 03:26:07.165 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:07.165 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:07.165 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:26:07.165 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:07.165 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:07.165 | INFO | Uploaded 36GiB to GPU in 2.07639s (17.3377GiB/s) | 20260506 03:26:07.165 | INFO | Downloaded 34GiB to GPU in 2.0299s (16.7496GiB/s) | 20260506 03:26:12.308 | INFO | Analyzing chunk frames = [8575488,8576000) | 20260506 03:26:12.308 | INFO | MemInfo: { | 20260506 03:26:12.308 | INFO | "gpu" : " 17 GB", | 20260506 03:26:12.308 | INFO | "huge" : "", | 20260506 03:26:12.308 | INFO | "huge_shared" : " 27 GB", | 20260506 03:26:12.308 | INFO | "malloc" : " 14 GB", | 20260506 03:26:12.308 | INFO | "pinned" : " 29 GB" | 20260506 03:26:12.308 | INFO | } | 20260506 03:26:12.308 | INFO | Perf report for the last 5.14335 seconds: | 20260506 03:26:12.308 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:26:12.308 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:12.308 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:12.308 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:12.308 | INFO | ANALYSIS : 60.9%, 3.0, 95.2ms, 0 | 20260506 03:26:12.308 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:12.309 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:12.309 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 03:26:12.309 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:12.309 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:12.309 | INFO | Uploaded 36GiB to GPU in 2.0829s (17.2836GiB/s) | 20260506 03:26:12.309 | INFO | Downloaded 34GiB to GPU in 2.04233s (16.6476GiB/s) | 20260506 03:26:17.453 | INFO | Analyzing chunk frames = [8576000,8576512) | 20260506 03:26:17.453 | INFO | MemInfo: { | 20260506 03:26:17.453 | INFO | "gpu" : " 17 GB", | 20260506 03:26:17.453 | INFO | "huge" : "", | 20260506 03:26:17.453 | INFO | "huge_shared" : " 27 GB", | 20260506 03:26:17.453 | INFO | "malloc" : " 14 GB", | 20260506 03:26:17.453 | INFO | "pinned" : " 29 GB" | 20260506 03:26:17.453 | INFO | } | 20260506 03:26:17.453 | INFO | Perf report for the last 5.14469 seconds: | 20260506 03:26:17.453 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:26:17.453 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:17.453 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:17.453 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:17.453 | INFO | ANALYSIS : 61.8%, 3.0, 96.6ms, 0 | 20260506 03:26:17.453 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:17.453 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:17.453 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 03:26:17.453 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:17.453 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:17.453 | INFO | Uploaded 36GiB to GPU in 2.10173s (17.1287GiB/s) | 20260506 03:26:17.453 | INFO | Downloaded 34GiB to GPU in 2.06856s (16.4366GiB/s) | 20260506 03:26:22.586 | INFO | Analyzing chunk frames = [8576512,8577024) | 20260506 03:26:22.586 | INFO | MemInfo: { | 20260506 03:26:22.586 | INFO | "gpu" : " 17 GB", | 20260506 03:26:22.586 | INFO | "huge" : "", | 20260506 03:26:22.586 | INFO | "huge_shared" : " 27 GB", | 20260506 03:26:22.586 | INFO | "malloc" : " 14 GB", | 20260506 03:26:22.586 | INFO | "pinned" : " 29 GB" | 20260506 03:26:22.586 | INFO | } | 20260506 03:26:22.586 | INFO | Perf report for the last 5.13351 seconds: | 20260506 03:26:22.586 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:26:22.586 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:22.587 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:22.587 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:22.587 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 03:26:22.587 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:22.587 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:22.587 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 03:26:22.587 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:22.587 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:22.587 | INFO | Uploaded 36GiB to GPU in 2.05056s (17.5562GiB/s) | 20260506 03:26:22.587 | INFO | Downloaded 34GiB to GPU in 2.03375s (16.7179GiB/s) | 20260506 03:26:22.587 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 03:26:22.587 | INFO | Total mem usage: 15.4 GiB | 20260506 03:26:22.587 | INFO | All Time High Water Marks sum: 16.3 GiB (Savings: 5.2%) | 20260506 03:26:22.587 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -10%) | 20260506 03:26:22.587 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 03:26:22.587 | INFO | High Water Breakdown: | 20260506 03:26:22.587 | INFO | 60.9% : 8.54 GiB ( 88.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 03:26:22.587 | INFO | 39% : 5.47 GiB ( 82.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 03:26:22.587 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 03:26:22.587 | INFO | ------------------------------------------ | 20260506 03:26:22.587 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:26:22.587 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 03:26:22.587 | INFO | Total mem usage: 26.5 GiB | 20260506 03:26:22.587 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 03:26:22.587 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 03:26:22.587 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 03:26:22.587 | INFO | High Water Breakdown: | 20260506 03:26:22.587 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 03:26:22.587 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 03:26:22.587 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 03:26:22.587 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 03:26:22.587 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 03:26:22.587 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 03:26:22.587 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 03:26:22.587 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 03:26:22.587 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 03:26:22.587 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 03:26:22.587 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 03:26:22.587 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 03:26:22.587 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 03:26:22.587 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 03:26:22.587 | INFO | ------------------------------------------ | 20260506 03:26:22.587 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:26:22.587 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 03:26:22.587 | INFO | Total mem usage: 25.6 GiB | 20260506 03:26:22.587 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 03:26:22.587 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 03:26:22.587 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 03:26:22.587 | INFO | High Water Breakdown: | 20260506 03:26:22.587 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 03:26:22.587 | INFO | ------------------------------------------ | 20260506 03:26:22.587 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:26:22.587 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 03:26:22.587 | INFO | Total mem usage: 15.7 GiB | 20260506 03:26:22.587 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 03:26:22.588 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 03:26:22.588 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 03:26:22.588 | INFO | High Water Breakdown: | 20260506 03:26:22.588 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 03:26:22.588 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 03:26:22.588 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 03:26:22.588 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 03:26:22.588 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 03:26:22.588 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 03:26:22.588 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 03:26:22.588 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 03:26:22.588 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 03:26:22.588 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 03:26:22.588 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 03:26:22.588 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 03:26:22.588 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 03:26:22.588 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 03:26:22.588 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 03:26:22.588 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 03:26:22.588 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 03:26:22.588 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 03:26:22.588 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 03:26:22.588 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 03:26:22.588 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 03:26:22.588 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 03:26:22.588 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 03:26:22.588 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 03:26:22.588 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 03:26:22.588 | INFO | ------------------------------------------ | 20260506 03:26:22.588 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:26:22.588 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 03:26:22.588 | INFO | PerfReport: Gathered 1536 samples. | 20260506 03:26:22.588 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 03:26:22.588 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.36e+01, 1.34e+01 / 7.31e-01 | 20260506 03:26:22.588 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.05e-01, 1.03e-02 / 4.54e-03 | 20260506 03:26:22.588 | INFO | PerfReport: 11.85%, Baseline -- 3.70e+00 / 4.46e+00, 5.75e-01 / 2.65e-01 | 20260506 03:26:22.588 | INFO | PerfReport: 3.29%, Histogram -- 1.03e+00 / 1.51e+00, 1.90e-01 / 8.19e-02 | 20260506 03:26:22.588 | INFO | PerfReport: 16.00%, DME -- 4.00e+01 / 4.39e+01, 3.79e+00 / 8.62e-01 | 20260506 03:26:22.588 | INFO | PerfReport: 57.55%, FrameLabeling -- 1.80e+01 / 1.93e+01, 1.52e+00 / 9.67e-01 | 20260506 03:26:22.588 | INFO | PerfReport: 7.00%, PulseAccumulating -- 2.19e+00 / 2.72e+00, 4.15e-01 / 1.85e-01 | 20260506 03:26:22.588 | INFO | PerfReport: 2.98%, Metrics -- 9.30e-01 / 1.21e+00, 2.51e-01 / 1.24e-01 | 20260506 03:26:22.588 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.21e-01 / 3.69e-01, 5.43e-02 / 2.40e-02 | 20260506 03:26:22.588 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 03:26:22.588 | INFO | PerfReport: Gathered 1536 samples. | 20260506 03:26:22.588 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 03:26:22.588 | INFO | PerfReport: Full Execution -- 9.75e+01 / -nan, 2.41e+01 / 0.00e+00 | 20260506 03:26:22.588 | INFO | PerfReport: 22.25%, Upload -- 2.17e+01 / 0.00e+00, 7.25e+00 / 0.00e+00 | 20260506 03:26:22.588 | INFO | PerfReport: 22.01%, Download -- 2.15e+01 / 0.00e+00, 7.62e+00 / 0.00e+00 | 20260506 03:26:22.588 | INFO | PerfReport: 32.02%, Compute -- 3.12e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 03:26:22.588 | INFO | PerfReport: 1.95%, LockUpload -- 1.91e+00 / 0.00e+00, 7.74e+00 / 0.00e+00 | 20260506 03:26:22.588 | INFO | PerfReport: 1.35%, LockDownload -- 1.32e+00 / 0.00e+00, 3.05e+00 / 0.00e+00 | 20260506 03:26:22.588 | INFO | PerfReport: 20.40%, LockCompute -- 1.99e+01 / 0.00e+00, 1.76e+01 / 0.00e+00 | 20260506 03:26:22.588 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.92e-02 / 0.00e+00 | 20260506 03:26:22.711 | INFO | Starting a round of Baz IO | 20260506 03:26:25.831 | INFO | Populating RTMetrics report | 20260506 03:26:25.831 | INFO | RTMetrics report populated | 20260506 03:26:25.831 | INFO | Opening tmp RTMetrics json file | 20260506 03:26:25.832 | INFO | Serializing RTMetrics to json | 20260506 03:26:25.833 | INFO | Serialization done, swapping tmp file now... | 20260506 03:26:25.834 | INFO | RTMetrics done | 20260506 03:26:25.909 | INFO | Finished a round of Baz IO: Wrote 1.761 GiB in 3.198 seconds | 20260506 03:26:25.909 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.22% / 0.00% / 78.78% | 20260506 03:26:25.909 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0611862% / 3.97364e-06% / 6.35783e-05% | 20260506 03:26:25.909 | INFO | Baz IO overall: | 20260506 03:26:25.909 | INFO | BytesWritten : 4772013087210 | 20260506 03:26:25.909 | INFO | NumEvents : 2237803207386 | 20260506 03:26:27.726 | INFO | Analyzing chunk frames = [8577024,8577536) | 20260506 03:26:27.726 | INFO | MemInfo: { | 20260506 03:26:27.726 | INFO | "gpu" : " 17 GB", | 20260506 03:26:27.726 | INFO | "huge" : "", | 20260506 03:26:27.726 | INFO | "huge_shared" : " 27 GB", | 20260506 03:26:27.726 | INFO | "malloc" : " 14 GB", | 20260506 03:26:27.726 | INFO | "pinned" : " 29 GB" | 20260506 03:26:27.726 | INFO | } | 20260506 03:26:27.726 | INFO | Perf report for the last 5.13949 seconds: | 20260506 03:26:27.726 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:26:27.726 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:27.726 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:27.726 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:27.726 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260506 03:26:27.726 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:27.726 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260506 03:26:27.726 | INFO | PRE_HQ : 59.4%, 1.7, 53.7ms, 0 | 20260506 03:26:27.726 | INFO | BAZWRITER : 61.4%, 3.5, 109.3ms, 0 | 20260506 03:26:27.726 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 03:26:27.727 | INFO | Uploaded 36GiB to GPU in 2.10195s (17.1269GiB/s) | 20260506 03:26:27.727 | INFO | Downloaded 38GiB to GPU in 2.21699s (17.1404GiB/s) | 20260506 03:26:32.864 | INFO | Analyzing chunk frames = [8577536,8578048) | 20260506 03:26:32.864 | INFO | MemInfo: { | 20260506 03:26:32.864 | INFO | "gpu" : " 17 GB", | 20260506 03:26:32.864 | INFO | "huge" : "", | 20260506 03:26:32.864 | INFO | "huge_shared" : " 27 GB", | 20260506 03:26:32.864 | INFO | "malloc" : " 14 GB", | 20260506 03:26:32.864 | INFO | "pinned" : " 29 GB" | 20260506 03:26:32.864 | INFO | } | 20260506 03:26:32.864 | INFO | Perf report for the last 5.13808 seconds: | 20260506 03:26:32.864 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:26:32.864 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:32.864 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:32.864 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:32.864 | INFO | ANALYSIS : 60.8%, 3.0, 94.9ms, 0 | 20260506 03:26:32.864 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:32.864 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:32.864 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 03:26:32.864 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:32.864 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:32.864 | INFO | Uploaded 36GiB to GPU in 2.00356s (17.9681GiB/s) | 20260506 03:26:32.864 | INFO | Downloaded 34GiB to GPU in 2.00691s (16.9414GiB/s) | 20260506 03:26:37.996 | INFO | Analyzing chunk frames = [8578048,8578560) | 20260506 03:26:37.996 | INFO | MemInfo: { | 20260506 03:26:37.996 | INFO | "gpu" : " 17 GB", | 20260506 03:26:37.996 | INFO | "huge" : "", | 20260506 03:26:37.996 | INFO | "huge_shared" : " 27 GB", | 20260506 03:26:37.996 | INFO | "malloc" : " 14 GB", | 20260506 03:26:37.996 | INFO | "pinned" : " 29 GB" | 20260506 03:26:37.996 | INFO | } | 20260506 03:26:37.996 | INFO | Perf report for the last 5.13217 seconds: | 20260506 03:26:37.996 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:26:37.996 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:37.996 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:37.996 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:37.996 | INFO | ANALYSIS : 60.5%, 3.0, 94.1ms, 0 | 20260506 03:26:37.996 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:37.996 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:37.996 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 03:26:37.996 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:37.996 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:37.997 | INFO | Uploaded 36GiB to GPU in 1.99385s (18.0555GiB/s) | 20260506 03:26:37.997 | INFO | Downloaded 34GiB to GPU in 1.98759s (17.1062GiB/s) | 20260506 03:26:43.138 | INFO | Analyzing chunk frames = [8578560,8579072) | 20260506 03:26:43.138 | INFO | MemInfo: { | 20260506 03:26:43.138 | INFO | "gpu" : " 17 GB", | 20260506 03:26:43.138 | INFO | "huge" : "", | 20260506 03:26:43.138 | INFO | "huge_shared" : " 27 GB", | 20260506 03:26:43.138 | INFO | "malloc" : " 14 GB", | 20260506 03:26:43.139 | INFO | "pinned" : " 29 GB" | 20260506 03:26:43.139 | INFO | } | 20260506 03:26:43.139 | INFO | Perf report for the last 5.14231 seconds: | 20260506 03:26:43.139 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:26:43.139 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:43.139 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:43.139 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:43.139 | INFO | ANALYSIS : 61.0%, 3.0, 94.8ms, 0 | 20260506 03:26:43.139 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:43.139 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:43.139 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 03:26:43.139 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:43.139 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:43.139 | INFO | Uploaded 36GiB to GPU in 2.09304s (17.1998GiB/s) | 20260506 03:26:43.139 | INFO | Downloaded 34GiB to GPU in 2.02326s (16.8046GiB/s) | 20260506 03:26:48.283 | INFO | Analyzing chunk frames = [8579072,8579584) | 20260506 03:26:48.283 | INFO | MemInfo: { | 20260506 03:26:48.283 | INFO | "gpu" : " 17 GB", | 20260506 03:26:48.283 | INFO | "huge" : "", | 20260506 03:26:48.283 | INFO | "huge_shared" : " 27 GB", | 20260506 03:26:48.283 | INFO | "malloc" : " 14 GB", | 20260506 03:26:48.283 | INFO | "pinned" : " 29 GB" | 20260506 03:26:48.283 | INFO | } | 20260506 03:26:48.283 | INFO | Perf report for the last 5.1446 seconds: | 20260506 03:26:48.283 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:26:48.283 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:48.283 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:48.283 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:48.283 | INFO | ANALYSIS : 60.7%, 3.0, 94.9ms, 0 | 20260506 03:26:48.283 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:48.283 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:48.283 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 03:26:48.283 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:48.283 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:48.284 | INFO | Uploaded 36GiB to GPU in 2.09457s (17.1873GiB/s) | 20260506 03:26:48.284 | INFO | Downloaded 34GiB to GPU in 2.03953s (16.6705GiB/s) | 20260506 03:26:53.425 | INFO | Analyzing chunk frames = [8579584,8580096) | 20260506 03:26:53.425 | INFO | MemInfo: { | 20260506 03:26:53.425 | INFO | "gpu" : " 17 GB", | 20260506 03:26:53.425 | INFO | "huge" : "", | 20260506 03:26:53.425 | INFO | "huge_shared" : " 27 GB", | 20260506 03:26:53.425 | INFO | "malloc" : " 14 GB", | 20260506 03:26:53.425 | INFO | "pinned" : " 29 GB" | 20260506 03:26:53.426 | INFO | } | 20260506 03:26:53.426 | INFO | Perf report for the last 5.14242 seconds: | 20260506 03:26:53.426 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:26:53.426 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:53.426 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:53.426 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:53.426 | INFO | ANALYSIS : 60.7%, 3.0, 94.8ms, 0 | 20260506 03:26:53.426 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:53.426 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:53.426 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 03:26:53.426 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:53.426 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:53.426 | INFO | Uploaded 36GiB to GPU in 2.04874s (17.5717GiB/s) | 20260506 03:26:53.426 | INFO | Downloaded 34GiB to GPU in 2.03562s (16.7026GiB/s) | 20260506 03:26:58.562 | INFO | Analyzing chunk frames = [8580096,8580608) | 20260506 03:26:58.562 | INFO | MemInfo: { | 20260506 03:26:58.562 | INFO | "gpu" : " 17 GB", | 20260506 03:26:58.562 | INFO | "huge" : "", | 20260506 03:26:58.562 | INFO | "huge_shared" : " 27 GB", | 20260506 03:26:58.562 | INFO | "malloc" : " 14 GB", | 20260506 03:26:58.562 | INFO | "pinned" : " 29 GB" | 20260506 03:26:58.562 | INFO | } | 20260506 03:26:58.562 | INFO | Perf report for the last 5.13648 seconds: | 20260506 03:26:58.562 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:26:58.562 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:58.562 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:58.562 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:58.562 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 03:26:58.562 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:58.562 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:58.562 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 03:26:58.562 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:58.562 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:26:58.563 | INFO | Uploaded 36GiB to GPU in 2.08095s (17.2998GiB/s) | 20260506 03:26:58.563 | INFO | Downloaded 34GiB to GPU in 2.05396s (16.5534GiB/s) | 20260506 03:27:03.696 | INFO | Analyzing chunk frames = [8580608,8581120) | 20260506 03:27:03.696 | INFO | MemInfo: { | 20260506 03:27:03.696 | INFO | "gpu" : " 17 GB", | 20260506 03:27:03.696 | INFO | "huge" : "", | 20260506 03:27:03.696 | INFO | "huge_shared" : " 27 GB", | 20260506 03:27:03.696 | INFO | "malloc" : " 14 GB", | 20260506 03:27:03.696 | INFO | "pinned" : " 29 GB" | 20260506 03:27:03.696 | INFO | } | 20260506 03:27:03.696 | INFO | Perf report for the last 5.134 seconds: | 20260506 03:27:03.696 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:27:03.696 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:03.696 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:03.696 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:03.696 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260506 03:27:03.696 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:03.696 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:03.696 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 03:27:03.696 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:03.696 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:03.696 | INFO | Uploaded 36GiB to GPU in 2.06582s (17.4265GiB/s) | 20260506 03:27:03.696 | INFO | Downloaded 34GiB to GPU in 2.02688s (16.7746GiB/s) | 20260506 03:27:03.822 | INFO | Starting a round of Baz IO | 20260506 03:27:06.943 | INFO | Populating RTMetrics report | 20260506 03:27:06.944 | INFO | RTMetrics report populated | 20260506 03:27:06.944 | INFO | Opening tmp RTMetrics json file | 20260506 03:27:06.944 | INFO | Serializing RTMetrics to json | 20260506 03:27:06.946 | INFO | Serialization done, swapping tmp file now... | 20260506 03:27:06.946 | INFO | RTMetrics done | 20260506 03:27:07.025 | INFO | Finished a round of Baz IO: Wrote 1.76 GiB in 3.203 seconds | 20260506 03:27:07.025 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.22% / 0.00% / 78.78% | 20260506 03:27:07.025 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0616868% / 0% / 8.34465e-05% | 20260506 03:27:07.025 | INFO | Baz IO overall: | 20260506 03:27:07.025 | INFO | BytesWritten : 4773902345875 | 20260506 03:27:07.025 | INFO | NumEvents : 2238659358086 | 20260506 03:27:08.832 | INFO | Analyzing chunk frames = [8581120,8581632) | 20260506 03:27:08.832 | INFO | MemInfo: { | 20260506 03:27:08.832 | INFO | "gpu" : " 17 GB", | 20260506 03:27:08.832 | INFO | "huge" : "", | 20260506 03:27:08.832 | INFO | "huge_shared" : " 27 GB", | 20260506 03:27:08.832 | INFO | "malloc" : " 14 GB", | 20260506 03:27:08.832 | INFO | "pinned" : " 29 GB" | 20260506 03:27:08.832 | INFO | } | 20260506 03:27:08.832 | INFO | Perf report for the last 5.136 seconds: | 20260506 03:27:08.832 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:27:08.832 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:08.832 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:08.832 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:08.832 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260506 03:27:08.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:08.832 | INFO | RT_METRICS : 42.8%, 1.0, 22.4ms, 0 | 20260506 03:27:08.832 | INFO | PRE_HQ : 59.3%, 1.7, 53.5ms, 0 | 20260506 03:27:08.832 | INFO | BAZWRITER : 61.3%, 3.5, 110.8ms, 0 | 20260506 03:27:08.832 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 03:27:08.832 | INFO | Uploaded 36GiB to GPU in 2.09216s (17.2071GiB/s) | 20260506 03:27:08.832 | INFO | Downloaded 38GiB to GPU in 2.21927s (17.1227GiB/s) | 20260506 03:27:13.973 | INFO | Analyzing chunk frames = [8581632,8582144) | 20260506 03:27:13.973 | INFO | MemInfo: { | 20260506 03:27:13.973 | INFO | "gpu" : " 17 GB", | 20260506 03:27:13.973 | INFO | "huge" : "", | 20260506 03:27:13.973 | INFO | "huge_shared" : " 27 GB", | 20260506 03:27:13.973 | INFO | "malloc" : " 14 GB", | 20260506 03:27:13.973 | INFO | "pinned" : " 29 GB" | 20260506 03:27:13.973 | INFO | } | 20260506 03:27:13.973 | INFO | Perf report for the last 5.14093 seconds: | 20260506 03:27:13.973 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:27:13.973 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:13.973 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:13.973 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:13.973 | INFO | ANALYSIS : 60.7%, 3.0, 94.8ms, 0 | 20260506 03:27:13.973 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:13.973 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:13.973 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 03:27:13.973 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:13.973 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:13.973 | INFO | Uploaded 36GiB to GPU in 1.97182s (18.2573GiB/s) | 20260506 03:27:13.973 | INFO | Downloaded 34GiB to GPU in 1.98066s (17.166GiB/s) | 20260506 03:27:19.117 | INFO | Analyzing chunk frames = [8582144,8582656) | 20260506 03:27:19.117 | INFO | MemInfo: { | 20260506 03:27:19.117 | INFO | "gpu" : " 17 GB", | 20260506 03:27:19.117 | INFO | "huge" : "", | 20260506 03:27:19.117 | INFO | "huge_shared" : " 27 GB", | 20260506 03:27:19.117 | INFO | "malloc" : " 14 GB", | 20260506 03:27:19.117 | INFO | "pinned" : " 29 GB" | 20260506 03:27:19.117 | INFO | } | 20260506 03:27:19.117 | INFO | Perf report for the last 5.14436 seconds: | 20260506 03:27:19.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:27:19.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:19.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:19.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:19.117 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 03:27:19.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:19.117 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:19.117 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 03:27:19.117 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:19.118 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:19.118 | INFO | Uploaded 36GiB to GPU in 2.10156s (17.1302GiB/s) | 20260506 03:27:19.118 | INFO | Downloaded 34GiB to GPU in 2.0392s (16.6732GiB/s) | 20260506 03:27:24.256 | INFO | Analyzing chunk frames = [8582656,8583168) | 20260506 03:27:24.256 | INFO | MemInfo: { | 20260506 03:27:24.256 | INFO | "gpu" : " 17 GB", | 20260506 03:27:24.256 | INFO | "huge" : "", | 20260506 03:27:24.257 | INFO | "huge_shared" : " 27 GB", | 20260506 03:27:24.257 | INFO | "malloc" : " 14 GB", | 20260506 03:27:24.257 | INFO | "pinned" : " 29 GB" | 20260506 03:27:24.257 | INFO | } | 20260506 03:27:24.257 | INFO | Perf report for the last 5.13909 seconds: | 20260506 03:27:24.257 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:27:24.257 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:24.257 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:24.257 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:24.257 | INFO | ANALYSIS : 61.0%, 3.0, 94.7ms, 0 | 20260506 03:27:24.257 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:24.257 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:24.257 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 03:27:24.257 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:24.257 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:24.257 | INFO | Uploaded 36GiB to GPU in 2.10133s (17.132GiB/s) | 20260506 03:27:24.257 | INFO | Downloaded 34GiB to GPU in 2.01071s (16.9095GiB/s) | 20260506 03:27:29.398 | INFO | Analyzing chunk frames = [8583168,8583680) | 20260506 03:27:29.398 | INFO | MemInfo: { | 20260506 03:27:29.398 | INFO | "gpu" : " 17 GB", | 20260506 03:27:29.398 | INFO | "huge" : "", | 20260506 03:27:29.399 | INFO | "huge_shared" : " 27 GB", | 20260506 03:27:29.399 | INFO | "malloc" : " 14 GB", | 20260506 03:27:29.399 | INFO | "pinned" : " 29 GB" | 20260506 03:27:29.399 | INFO | } | 20260506 03:27:29.399 | INFO | Perf report for the last 5.14201 seconds: | 20260506 03:27:29.399 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:27:29.399 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:29.399 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:29.399 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:29.399 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260506 03:27:29.399 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:29.399 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:29.399 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 03:27:29.399 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:29.399 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:29.399 | INFO | Uploaded 36GiB to GPU in 2.09822s (17.1574GiB/s) | 20260506 03:27:29.399 | INFO | Downloaded 34GiB to GPU in 2.03289s (16.7249GiB/s) | 20260506 03:27:34.537 | INFO | Analyzing chunk frames = [8583680,8584192) | 20260506 03:27:34.537 | INFO | MemInfo: { | 20260506 03:27:34.537 | INFO | "gpu" : " 17 GB", | 20260506 03:27:34.537 | INFO | "huge" : "", | 20260506 03:27:34.537 | INFO | "huge_shared" : " 27 GB", | 20260506 03:27:34.537 | INFO | "malloc" : " 14 GB", | 20260506 03:27:34.537 | INFO | "pinned" : " 29 GB" | 20260506 03:27:34.537 | INFO | } | 20260506 03:27:34.537 | INFO | Perf report for the last 5.13869 seconds: | 20260506 03:27:34.537 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:27:34.537 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:34.537 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:34.537 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:34.537 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 03:27:34.537 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:34.537 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:34.537 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:27:34.537 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:34.537 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:34.537 | INFO | Uploaded 36GiB to GPU in 2.10143s (17.1312GiB/s) | 20260506 03:27:34.538 | INFO | Downloaded 34GiB to GPU in 2.04231s (16.6478GiB/s) | 20260506 03:27:39.673 | INFO | Analyzing chunk frames = [8584192,8584704) | 20260506 03:27:39.673 | INFO | MemInfo: { | 20260506 03:27:39.673 | INFO | "gpu" : " 17 GB", | 20260506 03:27:39.673 | INFO | "huge" : "", | 20260506 03:27:39.673 | INFO | "huge_shared" : " 27 GB", | 20260506 03:27:39.673 | INFO | "malloc" : " 14 GB", | 20260506 03:27:39.673 | INFO | "pinned" : " 29 GB" | 20260506 03:27:39.673 | INFO | } | 20260506 03:27:39.674 | INFO | Perf report for the last 5.13619 seconds: | 20260506 03:27:39.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:27:39.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:39.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:39.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:39.674 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260506 03:27:39.674 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:39.674 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:39.674 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 03:27:39.674 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:39.674 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:39.674 | INFO | Uploaded 36GiB to GPU in 2.09089s (17.2175GiB/s) | 20260506 03:27:39.674 | INFO | Downloaded 34GiB to GPU in 2.06135s (16.4941GiB/s) | 20260506 03:27:44.818 | INFO | Analyzing chunk frames = [8584704,8585216) | 20260506 03:27:44.818 | INFO | MemInfo: { | 20260506 03:27:44.818 | INFO | "gpu" : " 17 GB", | 20260506 03:27:44.818 | INFO | "huge" : "", | 20260506 03:27:44.818 | INFO | "huge_shared" : " 27 GB", | 20260506 03:27:44.818 | INFO | "malloc" : " 14 GB", | 20260506 03:27:44.818 | INFO | "pinned" : " 29 GB" | 20260506 03:27:44.818 | INFO | } | 20260506 03:27:44.818 | INFO | Perf report for the last 5.14447 seconds: | 20260506 03:27:44.818 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:27:44.818 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:44.818 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:44.818 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:44.818 | INFO | ANALYSIS : 61.2%, 3.0, 95.6ms, 0 | 20260506 03:27:44.818 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:44.818 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:44.818 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 03:27:44.818 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:44.818 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:44.818 | INFO | Uploaded 36GiB to GPU in 2.04043s (17.6433GiB/s) | 20260506 03:27:44.818 | INFO | Downloaded 34GiB to GPU in 2.02801s (16.7652GiB/s) | 20260506 03:27:44.818 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 03:27:44.818 | INFO | Total mem usage: 15.4 GiB | 20260506 03:27:44.818 | INFO | All Time High Water Marks sum: 16.3 GiB (Savings: 5.2%) | 20260506 03:27:44.818 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -10%) | 20260506 03:27:44.818 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 03:27:44.818 | INFO | High Water Breakdown: | 20260506 03:27:44.818 | INFO | 60.9% : 8.54 GiB ( 88.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 03:27:44.818 | INFO | 39% : 5.47 GiB ( 82.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 03:27:44.818 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 03:27:44.818 | INFO | ------------------------------------------ | 20260506 03:27:44.818 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:27:44.818 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 03:27:44.818 | INFO | Total mem usage: 26.5 GiB | 20260506 03:27:44.818 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 03:27:44.818 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 03:27:44.818 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 03:27:44.818 | INFO | High Water Breakdown: | 20260506 03:27:44.818 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 03:27:44.818 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 03:27:44.818 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 03:27:44.818 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 03:27:44.818 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 03:27:44.818 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 03:27:44.818 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 03:27:44.819 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 03:27:44.819 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 03:27:44.819 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 03:27:44.819 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 03:27:44.819 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 03:27:44.819 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 03:27:44.819 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 03:27:44.819 | INFO | ------------------------------------------ | 20260506 03:27:44.819 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:27:44.819 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 03:27:44.819 | INFO | Total mem usage: 25.6 GiB | 20260506 03:27:44.819 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 03:27:44.819 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 03:27:44.819 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 03:27:44.819 | INFO | High Water Breakdown: | 20260506 03:27:44.819 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 03:27:44.819 | INFO | ------------------------------------------ | 20260506 03:27:44.819 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:27:44.819 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 03:27:44.819 | INFO | Total mem usage: 15.7 GiB | 20260506 03:27:44.819 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 03:27:44.819 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 03:27:44.819 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 03:27:44.819 | INFO | High Water Breakdown: | 20260506 03:27:44.819 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 03:27:44.819 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 03:27:44.819 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 03:27:44.819 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 03:27:44.819 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 03:27:44.819 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 03:27:44.819 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 03:27:44.819 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 03:27:44.819 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 03:27:44.819 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 03:27:44.819 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 03:27:44.819 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 03:27:44.819 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 03:27:44.819 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 03:27:44.819 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 03:27:44.819 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 03:27:44.819 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 03:27:44.819 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 03:27:44.819 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 03:27:44.819 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 03:27:44.819 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 03:27:44.819 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 03:27:44.819 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 03:27:44.819 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 03:27:44.819 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 03:27:44.819 | INFO | ------------------------------------------ | 20260506 03:27:44.819 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:27:44.819 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 03:27:44.819 | INFO | PerfReport: Gathered 1536 samples. | 20260506 03:27:44.819 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 03:27:44.819 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.36e+01, 1.35e+01 / 8.05e-01 | 20260506 03:27:44.819 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.05e-01, 1.16e-02 / 5.56e-03 | 20260506 03:27:44.819 | INFO | PerfReport: 11.87%, Baseline -- 3.72e+00 / 4.52e+00, 5.86e-01 / 2.86e-01 | 20260506 03:27:44.819 | INFO | PerfReport: 3.31%, Histogram -- 1.04e+00 / 1.52e+00, 1.97e-01 / 1.01e-01 | 20260506 03:27:44.819 | INFO | PerfReport: 15.98%, DME -- 4.00e+01 / 4.38e+01, 3.69e+00 / 7.54e-01 | 20260506 03:27:44.819 | INFO | PerfReport: 57.61%, FrameLabeling -- 1.80e+01 / 1.93e+01, 1.61e+00 / 1.22e+00 | 20260506 03:27:44.819 | INFO | PerfReport: 6.94%, PulseAccumulating -- 2.17e+00 / 2.71e+00, 4.04e-01 / 2.33e-01 | 20260506 03:27:44.819 | INFO | PerfReport: 2.97%, Metrics -- 9.29e-01 / 1.20e+00, 2.48e-01 / 1.21e-01 | 20260506 03:27:44.819 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.21e-01 / 3.68e-01, 5.48e-02 / 2.65e-02 | 20260506 03:27:44.819 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 03:27:44.819 | INFO | PerfReport: Gathered 1536 samples. | 20260506 03:27:44.819 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 03:27:44.819 | INFO | PerfReport: Full Execution -- 9.74e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260506 03:27:44.819 | INFO | PerfReport: 22.12%, Upload -- 2.15e+01 / 0.00e+00, 7.02e+00 / 0.00e+00 | 20260506 03:27:44.819 | INFO | PerfReport: 21.93%, Download -- 2.14e+01 / 0.00e+00, 7.55e+00 / 0.00e+00 | 20260506 03:27:44.819 | INFO | PerfReport: 32.16%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 03:27:44.819 | INFO | PerfReport: 1.94%, LockUpload -- 1.89e+00 / 0.00e+00, 7.63e+00 / 0.00e+00 | 20260506 03:27:44.819 | INFO | PerfReport: 1.27%, LockDownload -- 1.24e+00 / 0.00e+00, 2.92e+00 / 0.00e+00 | 20260506 03:27:44.819 | INFO | PerfReport: 20.56%, LockCompute -- 2.00e+01 / 0.00e+00, 1.76e+01 / 0.00e+00 | 20260506 03:27:44.819 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.63e-02 / 0.00e+00 | 20260506 03:27:44.940 | INFO | Starting a round of Baz IO | 20260506 03:27:48.058 | INFO | Populating RTMetrics report | 20260506 03:27:48.059 | INFO | RTMetrics report populated | 20260506 03:27:48.059 | INFO | Opening tmp RTMetrics json file | 20260506 03:27:48.060 | INFO | Serializing RTMetrics to json | 20260506 03:27:48.062 | INFO | Serialization done, swapping tmp file now... | 20260506 03:27:48.062 | INFO | RTMetrics done | 20260506 03:27:48.133 | INFO | Finished a round of Baz IO: Wrote 1.757 GiB in 3.192 seconds | 20260506 03:27:48.133 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.22% / 0.00% / 78.78% | 20260506 03:27:48.133 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0607888% / 3.97364e-06% / 7.15256e-05% | 20260506 03:27:48.133 | INFO | Baz IO overall: | 20260506 03:27:48.133 | INFO | BytesWritten : 4775789057610 | 20260506 03:27:48.133 | INFO | NumEvents : 2239513578972 | 20260506 03:27:49.948 | INFO | Analyzing chunk frames = [8585216,8585728) | 20260506 03:27:49.948 | INFO | MemInfo: { | 20260506 03:27:49.948 | INFO | "gpu" : " 17 GB", | 20260506 03:27:49.948 | INFO | "huge" : "", | 20260506 03:27:49.948 | INFO | "huge_shared" : " 27 GB", | 20260506 03:27:49.948 | INFO | "malloc" : " 14 GB", | 20260506 03:27:49.948 | INFO | "pinned" : " 29 GB" | 20260506 03:27:49.949 | INFO | } | 20260506 03:27:49.949 | INFO | Perf report for the last 5.13056 seconds: | 20260506 03:27:49.949 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:27:49.949 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:49.949 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:49.949 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:49.949 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 03:27:49.949 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:49.949 | INFO | RT_METRICS : 42.6%, 1.0, 22.3ms, 0 | 20260506 03:27:49.949 | INFO | PRE_HQ : 59.2%, 1.7, 53.2ms, 0 | 20260506 03:27:49.949 | INFO | BAZWRITER : 61.4%, 3.4, 107.5ms, 0 | 20260506 03:27:49.949 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 03:27:49.949 | INFO | Uploaded 36GiB to GPU in 2.08651s (17.2537GiB/s) | 20260506 03:27:49.949 | INFO | Downloaded 38GiB to GPU in 2.22368s (17.0888GiB/s) | 20260506 03:27:55.090 | INFO | Analyzing chunk frames = [8585728,8586240) | 20260506 03:27:55.090 | INFO | MemInfo: { | 20260506 03:27:55.090 | INFO | "gpu" : " 17 GB", | 20260506 03:27:55.090 | INFO | "huge" : "", | 20260506 03:27:55.090 | INFO | "huge_shared" : " 27 GB", | 20260506 03:27:55.090 | INFO | "malloc" : " 14 GB", | 20260506 03:27:55.090 | INFO | "pinned" : " 29 GB" | 20260506 03:27:55.090 | INFO | } | 20260506 03:27:55.090 | INFO | Perf report for the last 5.14126 seconds: | 20260506 03:27:55.090 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:27:55.090 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:55.090 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:55.090 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:55.090 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 03:27:55.090 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:55.090 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:55.090 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 03:27:55.090 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:55.090 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:27:55.090 | INFO | Uploaded 36GiB to GPU in 1.94985s (18.4629GiB/s) | 20260506 03:27:55.090 | INFO | Downloaded 34GiB to GPU in 1.97423s (17.2219GiB/s) | 20260506 03:28:00.231 | INFO | Analyzing chunk frames = [8586240,8586752) | 20260506 03:28:00.231 | INFO | MemInfo: { | 20260506 03:28:00.231 | INFO | "gpu" : " 17 GB", | 20260506 03:28:00.231 | INFO | "huge" : "", | 20260506 03:28:00.232 | INFO | "huge_shared" : " 27 GB", | 20260506 03:28:00.232 | INFO | "malloc" : " 14 GB", | 20260506 03:28:00.232 | INFO | "pinned" : " 29 GB" | 20260506 03:28:00.232 | INFO | } | 20260506 03:28:00.232 | INFO | Perf report for the last 5.14187 seconds: | 20260506 03:28:00.232 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:28:00.232 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:00.232 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:00.232 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:00.232 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 03:28:00.232 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:00.232 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:00.232 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 03:28:00.232 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:00.232 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:00.232 | INFO | Uploaded 36GiB to GPU in 2.10146s (17.1309GiB/s) | 20260506 03:28:00.232 | INFO | Downloaded 34GiB to GPU in 2.04107s (16.6579GiB/s) | 20260506 03:28:05.374 | INFO | Analyzing chunk frames = [8586752,8587264) | 20260506 03:28:05.374 | INFO | MemInfo: { | 20260506 03:28:05.374 | INFO | "gpu" : " 17 GB", | 20260506 03:28:05.374 | INFO | "huge" : "", | 20260506 03:28:05.374 | INFO | "huge_shared" : " 27 GB", | 20260506 03:28:05.374 | INFO | "malloc" : " 14 GB", | 20260506 03:28:05.374 | INFO | "pinned" : " 29 GB" | 20260506 03:28:05.374 | INFO | } | 20260506 03:28:05.374 | INFO | Perf report for the last 5.14262 seconds: | 20260506 03:28:05.374 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:28:05.374 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:05.374 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:05.374 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:05.374 | INFO | ANALYSIS : 60.9%, 3.0, 94.6ms, 0 | 20260506 03:28:05.374 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:05.374 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:05.374 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:28:05.374 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:05.375 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:05.375 | INFO | Uploaded 36GiB to GPU in 2.09887s (17.1521GiB/s) | 20260506 03:28:05.375 | INFO | Downloaded 34GiB to GPU in 2.01149s (16.9029GiB/s) | 20260506 03:28:10.511 | INFO | Analyzing chunk frames = [8587264,8587776) | 20260506 03:28:10.511 | INFO | MemInfo: { | 20260506 03:28:10.511 | INFO | "gpu" : " 17 GB", | 20260506 03:28:10.511 | INFO | "huge" : "", | 20260506 03:28:10.511 | INFO | "huge_shared" : " 27 GB", | 20260506 03:28:10.511 | INFO | "malloc" : " 14 GB", | 20260506 03:28:10.511 | INFO | "pinned" : " 29 GB" | 20260506 03:28:10.511 | INFO | } | 20260506 03:28:10.511 | INFO | Perf report for the last 5.13666 seconds: | 20260506 03:28:10.511 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:28:10.511 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:10.511 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:10.511 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:10.511 | INFO | ANALYSIS : 60.9%, 3.0, 95.0ms, 0 | 20260506 03:28:10.511 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:10.511 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:10.511 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 03:28:10.511 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:10.511 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:10.511 | INFO | Uploaded 36GiB to GPU in 2.11955s (16.9848GiB/s) | 20260506 03:28:10.511 | INFO | Downloaded 34GiB to GPU in 2.03594s (16.6999GiB/s) | 20260506 03:28:15.643 | INFO | Analyzing chunk frames = [8587776,8588288) | 20260506 03:28:15.643 | INFO | MemInfo: { | 20260506 03:28:15.643 | INFO | "gpu" : " 17 GB", | 20260506 03:28:15.643 | INFO | "huge" : "", | 20260506 03:28:15.643 | INFO | "huge_shared" : " 27 GB", | 20260506 03:28:15.643 | INFO | "malloc" : " 14 GB", | 20260506 03:28:15.643 | INFO | "pinned" : " 29 GB" | 20260506 03:28:15.643 | INFO | } | 20260506 03:28:15.643 | INFO | Perf report for the last 5.13239 seconds: | 20260506 03:28:15.643 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:28:15.643 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:15.643 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:15.643 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:15.644 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260506 03:28:15.644 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:15.644 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:15.644 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 03:28:15.644 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:15.644 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:15.644 | INFO | Uploaded 36GiB to GPU in 2.09445s (17.1883GiB/s) | 20260506 03:28:15.644 | INFO | Downloaded 34GiB to GPU in 2.04096s (16.6588GiB/s) | 20260506 03:28:20.785 | INFO | Analyzing chunk frames = [8588288,8588800) | 20260506 03:28:20.785 | INFO | MemInfo: { | 20260506 03:28:20.785 | INFO | "gpu" : " 17 GB", | 20260506 03:28:20.785 | INFO | "huge" : "", | 20260506 03:28:20.785 | INFO | "huge_shared" : " 27 GB", | 20260506 03:28:20.785 | INFO | "malloc" : " 14 GB", | 20260506 03:28:20.785 | INFO | "pinned" : " 29 GB" | 20260506 03:28:20.785 | INFO | } | 20260506 03:28:20.785 | INFO | Perf report for the last 5.14169 seconds: | 20260506 03:28:20.785 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:28:20.785 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:20.785 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:20.785 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:20.785 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260506 03:28:20.785 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:20.785 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:20.785 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 03:28:20.785 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:20.785 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:20.785 | INFO | Uploaded 36GiB to GPU in 2.04689s (17.5876GiB/s) | 20260506 03:28:20.785 | INFO | Downloaded 34GiB to GPU in 2.06837s (16.438GiB/s) | 20260506 03:28:25.928 | INFO | Analyzing chunk frames = [8588800,8589312) | 20260506 03:28:25.928 | INFO | MemInfo: { | 20260506 03:28:25.928 | INFO | "gpu" : " 17 GB", | 20260506 03:28:25.928 | INFO | "huge" : "", | 20260506 03:28:25.928 | INFO | "huge_shared" : " 27 GB", | 20260506 03:28:25.928 | INFO | "malloc" : " 14 GB", | 20260506 03:28:25.928 | INFO | "pinned" : " 29 GB" | 20260506 03:28:25.928 | INFO | } | 20260506 03:28:25.928 | INFO | Perf report for the last 5.14296 seconds: | 20260506 03:28:25.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:28:25.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:25.928 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:25.928 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:25.928 | INFO | ANALYSIS : 61.2%, 3.0, 95.6ms, 0 | 20260506 03:28:25.928 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:25.928 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:25.928 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:28:25.928 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:25.928 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:25.928 | INFO | Uploaded 36GiB to GPU in 2.03425s (17.6969GiB/s) | 20260506 03:28:25.928 | INFO | Downloaded 34GiB to GPU in 2.0202s (16.83GiB/s) | 20260506 03:28:26.055 | INFO | Starting a round of Baz IO | 20260506 03:28:29.180 | INFO | Populating RTMetrics report | 20260506 03:28:29.180 | INFO | RTMetrics report populated | 20260506 03:28:29.180 | INFO | Opening tmp RTMetrics json file | 20260506 03:28:29.180 | INFO | Serializing RTMetrics to json | 20260506 03:28:29.182 | INFO | Serialization done, swapping tmp file now... | 20260506 03:28:29.183 | INFO | RTMetrics done | 20260506 03:28:29.268 | INFO | Finished a round of Baz IO: Wrote 1.757 GiB in 3.213 seconds | 20260506 03:28:29.268 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.22% / 0.00% / 78.78% | 20260506 03:28:29.268 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0606577% / 0% / 0.000103315% | 20260506 03:28:29.268 | INFO | Baz IO overall: | 20260506 03:28:29.268 | INFO | BytesWritten : 4777675530576 | 20260506 03:28:29.268 | INFO | NumEvents : 2240367691523 | 20260506 03:28:31.067 | INFO | Analyzing chunk frames = [8589312,8589824) | 20260506 03:28:31.067 | INFO | MemInfo: { | 20260506 03:28:31.067 | INFO | "gpu" : " 17 GB", | 20260506 03:28:31.067 | INFO | "huge" : "", | 20260506 03:28:31.067 | INFO | "huge_shared" : " 27 GB", | 20260506 03:28:31.067 | INFO | "malloc" : " 14 GB", | 20260506 03:28:31.067 | INFO | "pinned" : " 29 GB" | 20260506 03:28:31.067 | INFO | } | 20260506 03:28:31.067 | INFO | Perf report for the last 5.13926 seconds: | 20260506 03:28:31.067 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:28:31.067 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:31.067 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:31.067 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:31.067 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260506 03:28:31.067 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:31.067 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260506 03:28:31.068 | INFO | PRE_HQ : 59.0%, 1.7, 53.2ms, 0 | 20260506 03:28:31.068 | INFO | BAZWRITER : 61.8%, 3.5, 112.0ms, 0 | 20260506 03:28:31.068 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 03:28:31.068 | INFO | Uploaded 36GiB to GPU in 2.10479s (17.1038GiB/s) | 20260506 03:28:31.068 | INFO | Downloaded 38GiB to GPU in 2.2311s (17.032GiB/s) | 20260506 03:28:36.207 | INFO | Analyzing chunk frames = [8589824,8590336) | 20260506 03:28:36.207 | INFO | MemInfo: { | 20260506 03:28:36.207 | INFO | "gpu" : " 17 GB", | 20260506 03:28:36.207 | INFO | "huge" : "", | 20260506 03:28:36.207 | INFO | "huge_shared" : " 27 GB", | 20260506 03:28:36.207 | INFO | "malloc" : " 14 GB", | 20260506 03:28:36.207 | INFO | "pinned" : " 29 GB" | 20260506 03:28:36.207 | INFO | } | 20260506 03:28:36.207 | INFO | Perf report for the last 5.13991 seconds: | 20260506 03:28:36.207 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:28:36.207 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:36.207 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:36.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:36.207 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 03:28:36.207 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:36.207 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:36.207 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 03:28:36.207 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:36.207 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:36.208 | INFO | Uploaded 36GiB to GPU in 2.01157s (17.8964GiB/s) | 20260506 03:28:36.208 | INFO | Downloaded 34GiB to GPU in 1.99108s (17.0762GiB/s) | 20260506 03:28:41.350 | INFO | Analyzing chunk frames = [8590336,8590848) | 20260506 03:28:41.350 | INFO | MemInfo: { | 20260506 03:28:41.350 | INFO | "gpu" : " 17 GB", | 20260506 03:28:41.350 | INFO | "huge" : "", | 20260506 03:28:41.350 | INFO | "huge_shared" : " 27 GB", | 20260506 03:28:41.350 | INFO | "malloc" : " 14 GB", | 20260506 03:28:41.350 | INFO | "pinned" : " 29 GB" | 20260506 03:28:41.350 | INFO | } | 20260506 03:28:41.350 | INFO | Perf report for the last 5.14278 seconds: | 20260506 03:28:41.350 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:28:41.350 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:41.350 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:41.350 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:41.350 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 03:28:41.350 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:41.350 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:41.350 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:28:41.350 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:41.350 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:41.350 | INFO | Uploaded 36GiB to GPU in 2.08936s (17.2302GiB/s) | 20260506 03:28:41.350 | INFO | Downloaded 34GiB to GPU in 2.02892s (16.7577GiB/s) | 20260506 03:28:46.486 | INFO | Analyzing chunk frames = [8590848,8591360) | 20260506 03:28:46.487 | INFO | MemInfo: { | 20260506 03:28:46.487 | INFO | "gpu" : " 17 GB", | 20260506 03:28:46.487 | INFO | "huge" : "", | 20260506 03:28:46.487 | INFO | "huge_shared" : " 27 GB", | 20260506 03:28:46.487 | INFO | "malloc" : " 14 GB", | 20260506 03:28:46.487 | INFO | "pinned" : " 29 GB" | 20260506 03:28:46.487 | INFO | } | 20260506 03:28:46.487 | INFO | Perf report for the last 5.13667 seconds: | 20260506 03:28:46.487 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:28:46.487 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:46.487 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:46.487 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:46.487 | INFO | ANALYSIS : 61.0%, 3.0, 94.7ms, 0 | 20260506 03:28:46.487 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:46.487 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:46.487 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 03:28:46.487 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:46.487 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:46.487 | INFO | Uploaded 36GiB to GPU in 2.08653s (17.2535GiB/s) | 20260506 03:28:46.487 | INFO | Downloaded 34GiB to GPU in 2.01568s (16.8678GiB/s) | 20260506 03:28:51.625 | INFO | Analyzing chunk frames = [8591360,8591872) | 20260506 03:28:51.625 | INFO | MemInfo: { | 20260506 03:28:51.625 | INFO | "gpu" : " 17 GB", | 20260506 03:28:51.625 | INFO | "huge" : "", | 20260506 03:28:51.625 | INFO | "huge_shared" : " 27 GB", | 20260506 03:28:51.625 | INFO | "malloc" : " 14 GB", | 20260506 03:28:51.625 | INFO | "pinned" : " 29 GB" | 20260506 03:28:51.625 | INFO | } | 20260506 03:28:51.625 | INFO | Perf report for the last 5.1383 seconds: | 20260506 03:28:51.625 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:28:51.625 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:51.625 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:51.625 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:51.625 | INFO | ANALYSIS : 60.8%, 3.0, 94.9ms, 0 | 20260506 03:28:51.625 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:51.625 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:51.625 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 03:28:51.625 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:51.625 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:51.626 | INFO | Uploaded 36GiB to GPU in 2.11426s (17.0273GiB/s) | 20260506 03:28:51.626 | INFO | Downloaded 34GiB to GPU in 2.0408s (16.6601GiB/s) | 20260506 03:28:56.769 | INFO | Analyzing chunk frames = [8591872,8592384) | 20260506 03:28:56.769 | INFO | MemInfo: { | 20260506 03:28:56.769 | INFO | "gpu" : " 17 GB", | 20260506 03:28:56.769 | INFO | "huge" : "", | 20260506 03:28:56.769 | INFO | "huge_shared" : " 27 GB", | 20260506 03:28:56.769 | INFO | "malloc" : " 14 GB", | 20260506 03:28:56.769 | INFO | "pinned" : " 29 GB" | 20260506 03:28:56.769 | INFO | } | 20260506 03:28:56.769 | INFO | Perf report for the last 5.14408 seconds: | 20260506 03:28:56.769 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:28:56.769 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:56.769 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:56.769 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:56.769 | INFO | ANALYSIS : 60.8%, 3.0, 94.9ms, 0 | 20260506 03:28:56.769 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:56.769 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:56.769 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 03:28:56.769 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:56.769 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:28:56.769 | INFO | Uploaded 36GiB to GPU in 2.03832s (17.6616GiB/s) | 20260506 03:28:56.770 | INFO | Downloaded 34GiB to GPU in 2.0445s (16.63GiB/s) | 20260506 03:29:01.903 | INFO | Analyzing chunk frames = [8592384,8592896) | 20260506 03:29:01.903 | INFO | MemInfo: { | 20260506 03:29:01.903 | INFO | "gpu" : " 17 GB", | 20260506 03:29:01.903 | INFO | "huge" : "", | 20260506 03:29:01.904 | INFO | "huge_shared" : " 27 GB", | 20260506 03:29:01.904 | INFO | "malloc" : " 14 GB", | 20260506 03:29:01.904 | INFO | "pinned" : " 29 GB" | 20260506 03:29:01.904 | INFO | } | 20260506 03:29:01.904 | INFO | Perf report for the last 5.13452 seconds: | 20260506 03:29:01.904 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:29:01.904 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:01.904 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:01.904 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:01.904 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 03:29:01.904 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:01.904 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:01.904 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260506 03:29:01.904 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:01.904 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:01.904 | INFO | Uploaded 36GiB to GPU in 2.06387s (17.443GiB/s) | 20260506 03:29:01.904 | INFO | Downloaded 34GiB to GPU in 2.06593s (16.4575GiB/s) | 20260506 03:29:07.043 | INFO | Analyzing chunk frames = [8592896,8593408) | 20260506 03:29:07.043 | INFO | MemInfo: { | 20260506 03:29:07.043 | INFO | "gpu" : " 17 GB", | 20260506 03:29:07.043 | INFO | "huge" : "", | 20260506 03:29:07.043 | INFO | "huge_shared" : " 27 GB", | 20260506 03:29:07.043 | INFO | "malloc" : " 14 GB", | 20260506 03:29:07.044 | INFO | "pinned" : " 29 GB" | 20260506 03:29:07.044 | INFO | } | 20260506 03:29:07.044 | INFO | Perf report for the last 5.13984 seconds: | 20260506 03:29:07.044 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:29:07.044 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:07.044 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:07.044 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:07.044 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260506 03:29:07.044 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:07.044 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:07.044 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:29:07.044 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:07.044 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:07.044 | INFO | Uploaded 36GiB to GPU in 2.04801s (17.578GiB/s) | 20260506 03:29:07.044 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 03:29:07.044 | INFO | Total mem usage: 15.4 GiB | 20260506 03:29:07.044 | INFO | All Time High Water Marks sum: 16.3 GiB (Savings: 5.2%) | 20260506 03:29:07.044 | INFO | Downloaded 34GiB to GPU in 2.02116s (16.822GiB/s) | 20260506 03:29:07.044 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.99%) | 20260506 03:29:07.044 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 03:29:07.044 | INFO | High Water Breakdown: | 20260506 03:29:07.044 | INFO | 60.9% : 8.54 GiB ( 88.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 03:29:07.044 | INFO | 39% : 5.47 GiB ( 82.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 03:29:07.044 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 03:29:07.044 | INFO | ------------------------------------------ | 20260506 03:29:07.044 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:29:07.044 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 03:29:07.044 | INFO | Total mem usage: 26.5 GiB | 20260506 03:29:07.044 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 03:29:07.044 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 03:29:07.044 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 03:29:07.044 | INFO | High Water Breakdown: | 20260506 03:29:07.044 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 03:29:07.044 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 03:29:07.044 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 03:29:07.044 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 03:29:07.044 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 03:29:07.044 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 03:29:07.044 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 03:29:07.044 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 03:29:07.044 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 03:29:07.044 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 03:29:07.044 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 03:29:07.044 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 03:29:07.044 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 03:29:07.044 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 03:29:07.044 | INFO | ------------------------------------------ | 20260506 03:29:07.044 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:29:07.044 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 03:29:07.044 | INFO | Total mem usage: 25.6 GiB | 20260506 03:29:07.044 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 03:29:07.044 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 03:29:07.044 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 03:29:07.044 | INFO | High Water Breakdown: | 20260506 03:29:07.044 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 03:29:07.044 | INFO | ------------------------------------------ | 20260506 03:29:07.044 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:29:07.044 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 03:29:07.044 | INFO | Total mem usage: 15.7 GiB | 20260506 03:29:07.044 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 03:29:07.044 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 03:29:07.044 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 03:29:07.044 | INFO | High Water Breakdown: | 20260506 03:29:07.044 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 03:29:07.044 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 03:29:07.044 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 03:29:07.044 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 03:29:07.044 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 03:29:07.044 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 03:29:07.044 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 03:29:07.044 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 03:29:07.045 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 03:29:07.045 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 03:29:07.045 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 03:29:07.045 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 03:29:07.045 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 03:29:07.045 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 03:29:07.045 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 03:29:07.045 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 03:29:07.045 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 03:29:07.045 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 03:29:07.045 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 03:29:07.045 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 03:29:07.045 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 03:29:07.045 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 03:29:07.045 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 03:29:07.045 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 03:29:07.045 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 03:29:07.045 | INFO | ------------------------------------------ | 20260506 03:29:07.045 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:29:07.045 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 03:29:07.045 | INFO | PerfReport: Gathered 1536 samples. | 20260506 03:29:07.045 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 03:29:07.045 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.35e+01, 1.32e+01 / 7.05e-01 | 20260506 03:29:07.045 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.05e-01, 1.05e-02 / 6.22e-03 | 20260506 03:29:07.045 | INFO | PerfReport: 11.89%, Baseline -- 3.71e+00 / 4.53e+00, 5.83e-01 / 2.59e-01 | 20260506 03:29:07.045 | INFO | PerfReport: 3.30%, Histogram -- 1.03e+00 / 1.53e+00, 1.96e-01 / 8.84e-02 | 20260506 03:29:07.045 | INFO | PerfReport: 16.00%, DME -- 4.00e+01 / 4.38e+01, 3.60e+00 / 7.33e-01 | 20260506 03:29:07.045 | INFO | PerfReport: 57.51%, FrameLabeling -- 1.80e+01 / 1.92e+01, 1.47e+00 / 1.12e+00 | 20260506 03:29:07.045 | INFO | PerfReport: 7.00%, PulseAccumulating -- 2.18e+00 / 2.72e+00, 4.12e-01 / 2.10e-01 | 20260506 03:29:07.045 | INFO | PerfReport: 2.98%, Metrics -- 9.30e-01 / 1.20e+00, 2.50e-01 / 1.30e-01 | 20260506 03:29:07.045 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.21e-01 / 3.70e-01, 5.43e-02 / 2.55e-02 | 20260506 03:29:07.045 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 03:29:07.045 | INFO | PerfReport: Gathered 1536 samples. | 20260506 03:29:07.045 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 03:29:07.045 | INFO | PerfReport: Full Execution -- 9.75e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260506 03:29:07.045 | INFO | PerfReport: 22.10%, Upload -- 2.15e+01 / 0.00e+00, 7.01e+00 / 0.00e+00 | 20260506 03:29:07.045 | INFO | PerfReport: 21.95%, Download -- 2.14e+01 / 0.00e+00, 7.53e+00 / 0.00e+00 | 20260506 03:29:07.045 | INFO | PerfReport: 32.05%, Compute -- 3.12e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 03:29:07.045 | INFO | PerfReport: 1.99%, LockUpload -- 1.94e+00 / 0.00e+00, 7.89e+00 / 0.00e+00 | 20260506 03:29:07.045 | INFO | PerfReport: 1.37%, LockDownload -- 1.34e+00 / 0.00e+00, 3.11e+00 / 0.00e+00 | 20260506 03:29:07.045 | INFO | PerfReport: 20.52%, LockCompute -- 2.00e+01 / 0.00e+00, 1.76e+01 / 0.00e+00 | 20260506 03:29:07.045 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 2.08e-02 / 0.00e+00 | 20260506 03:29:07.169 | INFO | Starting a round of Baz IO | 20260506 03:29:10.284 | INFO | Populating RTMetrics report | 20260506 03:29:10.284 | INFO | RTMetrics report populated | 20260506 03:29:10.284 | INFO | Opening tmp RTMetrics json file | 20260506 03:29:10.285 | INFO | Serializing RTMetrics to json | 20260506 03:29:10.287 | INFO | Serialization done, swapping tmp file now... | 20260506 03:29:10.288 | INFO | RTMetrics done | 20260506 03:29:10.356 | INFO | Finished a round of Baz IO: Wrote 1.759 GiB in 3.186 seconds | 20260506 03:29:10.357 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.22% / 0.00% / 78.78% | 20260506 03:29:10.357 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0605623% / 0% / 7.54992e-05% | 20260506 03:29:10.357 | INFO | Baz IO overall: | 20260506 03:29:10.357 | INFO | BytesWritten : 4779563952337 | 20260506 03:29:10.357 | INFO | NumEvents : 2241223441401 | 20260506 03:29:12.183 | INFO | Analyzing chunk frames = [8593408,8593920) | 20260506 03:29:12.183 | INFO | MemInfo: { | 20260506 03:29:12.183 | INFO | "gpu" : " 17 GB", | 20260506 03:29:12.183 | INFO | "huge" : "", | 20260506 03:29:12.183 | INFO | "huge_shared" : " 27 GB", | 20260506 03:29:12.183 | INFO | "malloc" : " 14 GB", | 20260506 03:29:12.183 | INFO | "pinned" : " 29 GB" | 20260506 03:29:12.183 | INFO | } | 20260506 03:29:12.183 | INFO | Perf report for the last 5.13919 seconds: | 20260506 03:29:12.183 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:29:12.183 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:12.183 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:12.183 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:12.183 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 03:29:12.183 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:12.183 | INFO | RT_METRICS : 42.6%, 1.0, 22.3ms, 0 | 20260506 03:29:12.183 | INFO | PRE_HQ : 58.9%, 1.7, 53.1ms, 0 | 20260506 03:29:12.183 | INFO | BAZWRITER : 61.3%, 3.4, 108.2ms, 0 | 20260506 03:29:12.183 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 03:29:12.183 | INFO | Uploaded 36GiB to GPU in 2.11769s (16.9997GiB/s) | 20260506 03:29:12.183 | INFO | Downloaded 38GiB to GPU in 2.23s (17.0404GiB/s) | 20260506 03:29:17.320 | INFO | Analyzing chunk frames = [8593920,8594432) | 20260506 03:29:17.320 | INFO | MemInfo: { | 20260506 03:29:17.320 | INFO | "gpu" : " 17 GB", | 20260506 03:29:17.320 | INFO | "huge" : "", | 20260506 03:29:17.320 | INFO | "huge_shared" : " 27 GB", | 20260506 03:29:17.320 | INFO | "malloc" : " 14 GB", | 20260506 03:29:17.320 | INFO | "pinned" : " 29 GB" | 20260506 03:29:17.320 | INFO | } | 20260506 03:29:17.320 | INFO | Perf report for the last 5.13721 seconds: | 20260506 03:29:17.320 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:29:17.320 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:17.320 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:17.320 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:17.320 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260506 03:29:17.320 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:17.320 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:17.320 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 03:29:17.320 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:17.320 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:17.320 | INFO | Uploaded 36GiB to GPU in 1.96839s (18.289GiB/s) | 20260506 03:29:17.320 | INFO | Downloaded 34GiB to GPU in 1.99025s (17.0833GiB/s) | 20260506 03:29:22.457 | INFO | Analyzing chunk frames = [8594432,8594944) | 20260506 03:29:22.457 | INFO | MemInfo: { | 20260506 03:29:22.457 | INFO | "gpu" : " 17 GB", | 20260506 03:29:22.457 | INFO | "huge" : "", | 20260506 03:29:22.457 | INFO | "huge_shared" : " 27 GB", | 20260506 03:29:22.457 | INFO | "malloc" : " 14 GB", | 20260506 03:29:22.457 | INFO | "pinned" : " 29 GB" | 20260506 03:29:22.457 | INFO | } | 20260506 03:29:22.457 | INFO | Perf report for the last 5.13741 seconds: | 20260506 03:29:22.457 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:29:22.457 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:22.457 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:22.457 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:22.457 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260506 03:29:22.457 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:22.457 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:22.458 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 03:29:22.458 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:22.458 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:22.458 | INFO | Uploaded 36GiB to GPU in 2.09841s (17.1558GiB/s) | 20260506 03:29:22.458 | INFO | Downloaded 34GiB to GPU in 2.04022s (16.6649GiB/s) | 20260506 03:29:27.596 | INFO | Analyzing chunk frames = [8594944,8595456) | 20260506 03:29:27.596 | INFO | MemInfo: { | 20260506 03:29:27.597 | INFO | "gpu" : " 17 GB", | 20260506 03:29:27.597 | INFO | "huge" : "", | 20260506 03:29:27.597 | INFO | "huge_shared" : " 27 GB", | 20260506 03:29:27.597 | INFO | "malloc" : " 14 GB", | 20260506 03:29:27.597 | INFO | "pinned" : " 29 GB" | 20260506 03:29:27.597 | INFO | } | 20260506 03:29:27.597 | INFO | Perf report for the last 5.13928 seconds: | 20260506 03:29:27.597 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:29:27.597 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:27.597 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:27.597 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:27.597 | INFO | ANALYSIS : 61.5%, 3.0, 95.5ms, 0 | 20260506 03:29:27.597 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:27.597 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:27.597 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 03:29:27.597 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:27.597 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:27.597 | INFO | Uploaded 36GiB to GPU in 2.14953s (16.7478GiB/s) | 20260506 03:29:27.597 | INFO | Downloaded 34GiB to GPU in 2.04846s (16.5978GiB/s) | 20260506 03:29:32.742 | INFO | Analyzing chunk frames = [8595456,8595968) | 20260506 03:29:32.742 | INFO | MemInfo: { | 20260506 03:29:32.742 | INFO | "gpu" : " 17 GB", | 20260506 03:29:32.743 | INFO | "huge" : "", | 20260506 03:29:32.743 | INFO | "huge_shared" : " 27 GB", | 20260506 03:29:32.743 | INFO | "malloc" : " 14 GB", | 20260506 03:29:32.743 | INFO | "pinned" : " 29 GB" | 20260506 03:29:32.743 | INFO | } | 20260506 03:29:32.743 | INFO | Perf report for the last 5.14592 seconds: | 20260506 03:29:32.743 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:29:32.743 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:32.743 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:32.743 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:32.743 | INFO | ANALYSIS : 60.5%, 3.0, 94.5ms, 0 | 20260506 03:29:32.743 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:32.743 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:32.743 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 03:29:32.743 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:32.743 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:32.743 | INFO | Uploaded 36GiB to GPU in 2.09129s (17.2143GiB/s) | 20260506 03:29:32.743 | INFO | Downloaded 34GiB to GPU in 2.03456s (16.7112GiB/s) | 20260506 03:29:37.884 | INFO | Analyzing chunk frames = [8595968,8596480) | 20260506 03:29:37.884 | INFO | MemInfo: { | 20260506 03:29:37.884 | INFO | "gpu" : " 17 GB", | 20260506 03:29:37.884 | INFO | "huge" : "", | 20260506 03:29:37.885 | INFO | "huge_shared" : " 27 GB", | 20260506 03:29:37.885 | INFO | "malloc" : " 14 GB", | 20260506 03:29:37.885 | INFO | "pinned" : " 29 GB" | 20260506 03:29:37.885 | INFO | } | 20260506 03:29:37.885 | INFO | Perf report for the last 5.14203 seconds: | 20260506 03:29:37.885 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:29:37.885 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:37.885 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:37.885 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:37.885 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 03:29:37.885 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:37.885 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:37.885 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 03:29:37.885 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:37.885 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:37.885 | INFO | Uploaded 36GiB to GPU in 2.0787s (17.3185GiB/s) | 20260506 03:29:37.885 | INFO | Downloaded 34GiB to GPU in 2.03061s (16.7437GiB/s) | 20260506 03:29:43.017 | INFO | Analyzing chunk frames = [8596480,8596992) | 20260506 03:29:43.017 | INFO | MemInfo: { | 20260506 03:29:43.017 | INFO | "gpu" : " 17 GB", | 20260506 03:29:43.017 | INFO | "huge" : "", | 20260506 03:29:43.017 | INFO | "huge_shared" : " 27 GB", | 20260506 03:29:43.017 | INFO | "malloc" : " 14 GB", | 20260506 03:29:43.017 | INFO | "pinned" : " 29 GB" | 20260506 03:29:43.017 | INFO | } | 20260506 03:29:43.017 | INFO | Perf report for the last 5.13218 seconds: | 20260506 03:29:43.017 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:29:43.017 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:43.017 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:43.017 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:43.017 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 03:29:43.017 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:43.017 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:43.017 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 03:29:43.017 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:43.017 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:43.017 | INFO | Uploaded 36GiB to GPU in 2.07307s (17.3655GiB/s) | 20260506 03:29:43.017 | INFO | Downloaded 34GiB to GPU in 2.05833s (16.5182GiB/s) | 20260506 03:29:48.154 | INFO | Analyzing chunk frames = [8596992,8597504) | 20260506 03:29:48.154 | INFO | MemInfo: { | 20260506 03:29:48.154 | INFO | "gpu" : " 17 GB", | 20260506 03:29:48.154 | INFO | "huge" : "", | 20260506 03:29:48.154 | INFO | "huge_shared" : " 27 GB", | 20260506 03:29:48.154 | INFO | "malloc" : " 14 GB", | 20260506 03:29:48.154 | INFO | "pinned" : " 29 GB" | 20260506 03:29:48.154 | INFO | } | 20260506 03:29:48.154 | INFO | Perf report for the last 5.13673 seconds: | 20260506 03:29:48.154 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:29:48.154 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:48.154 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:48.154 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:48.154 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260506 03:29:48.154 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:48.154 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:48.154 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 03:29:48.154 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:48.154 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:48.154 | INFO | Uploaded 36GiB to GPU in 2.04447s (17.6085GiB/s) | 20260506 03:29:48.154 | INFO | Downloaded 34GiB to GPU in 2.03095s (16.7409GiB/s) | 20260506 03:29:48.279 | INFO | Starting a round of Baz IO | 20260506 03:29:51.405 | INFO | Populating RTMetrics report | 20260506 03:29:51.406 | INFO | RTMetrics report populated | 20260506 03:29:51.406 | INFO | Opening tmp RTMetrics json file | 20260506 03:29:51.406 | INFO | Serializing RTMetrics to json | 20260506 03:29:51.408 | INFO | Serialization done, swapping tmp file now... | 20260506 03:29:51.409 | INFO | RTMetrics done | 20260506 03:29:51.482 | INFO | Finished a round of Baz IO: Wrote 1.758 GiB in 3.202 seconds | 20260506 03:29:51.482 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.22% / 0.00% / 78.78% | 20260506 03:29:51.482 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0602404% / 3.97364e-06% / 8.74201e-05% | 20260506 03:29:51.482 | INFO | Baz IO overall: | 20260506 03:29:51.482 | INFO | BytesWritten : 4781451646235 | 20260506 03:29:51.482 | INFO | NumEvents : 2242079043720 | 20260506 03:29:53.296 | INFO | Analyzing chunk frames = [8597504,8598016) | 20260506 03:29:53.296 | INFO | MemInfo: { | 20260506 03:29:53.296 | INFO | "gpu" : " 17 GB", | 20260506 03:29:53.296 | INFO | "huge" : "", | 20260506 03:29:53.296 | INFO | "huge_shared" : " 27 GB", | 20260506 03:29:53.296 | INFO | "malloc" : " 14 GB", | 20260506 03:29:53.296 | INFO | "pinned" : " 29 GB" | 20260506 03:29:53.296 | INFO | } | 20260506 03:29:53.296 | INFO | Perf report for the last 5.14269 seconds: | 20260506 03:29:53.296 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:29:53.296 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:53.296 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:53.296 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:53.296 | INFO | ANALYSIS : 62.2%, 3.0, 97.2ms, 0 | 20260506 03:29:53.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:53.296 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260506 03:29:53.296 | INFO | PRE_HQ : 59.0%, 1.7, 53.1ms, 0 | 20260506 03:29:53.296 | INFO | BAZWRITER : 61.4%, 3.5, 109.6ms, 0 | 20260506 03:29:53.296 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 03:29:53.297 | INFO | Uploaded 36GiB to GPU in 2.10455s (17.1058GiB/s) | 20260506 03:29:53.297 | INFO | Downloaded 38GiB to GPU in 2.2151s (17.155GiB/s) | 20260506 03:29:58.437 | INFO | Analyzing chunk frames = [8598016,8598528) | 20260506 03:29:58.437 | INFO | MemInfo: { | 20260506 03:29:58.437 | INFO | "gpu" : " 17 GB", | 20260506 03:29:58.437 | INFO | "huge" : "", | 20260506 03:29:58.437 | INFO | "huge_shared" : " 27 GB", | 20260506 03:29:58.437 | INFO | "malloc" : " 14 GB", | 20260506 03:29:58.437 | INFO | "pinned" : " 29 GB" | 20260506 03:29:58.437 | INFO | } | 20260506 03:29:58.437 | INFO | Perf report for the last 5.14086 seconds: | 20260506 03:29:58.437 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:29:58.437 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:58.437 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:58.437 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:58.437 | INFO | ANALYSIS : 60.6%, 3.0, 94.7ms, 0 | 20260506 03:29:58.437 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:58.437 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:58.437 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 03:29:58.437 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:58.437 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:29:58.438 | INFO | Uploaded 36GiB to GPU in 1.97546s (18.2236GiB/s) | 20260506 03:29:58.438 | INFO | Downloaded 34GiB to GPU in 1.98976s (17.0875GiB/s) | 20260506 03:30:03.577 | INFO | Analyzing chunk frames = [8598528,8599040) | 20260506 03:30:03.577 | INFO | MemInfo: { | 20260506 03:30:03.577 | INFO | "gpu" : " 17 GB", | 20260506 03:30:03.577 | INFO | "huge" : "", | 20260506 03:30:03.577 | INFO | "huge_shared" : " 27 GB", | 20260506 03:30:03.577 | INFO | "malloc" : " 14 GB", | 20260506 03:30:03.577 | INFO | "pinned" : " 29 GB" | 20260506 03:30:03.577 | INFO | } | 20260506 03:30:03.577 | INFO | Perf report for the last 5.13991 seconds: | 20260506 03:30:03.577 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:30:03.577 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:03.577 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:03.577 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:03.577 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260506 03:30:03.577 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:03.577 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:03.577 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 03:30:03.577 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:03.577 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:03.577 | INFO | Uploaded 36GiB to GPU in 2.10255s (17.1221GiB/s) | 20260506 03:30:03.577 | INFO | Downloaded 34GiB to GPU in 2.04481s (16.6275GiB/s) | 20260506 03:30:08.721 | INFO | Analyzing chunk frames = [8599040,8599552) | 20260506 03:30:08.721 | INFO | MemInfo: { | 20260506 03:30:08.721 | INFO | "gpu" : " 17 GB", | 20260506 03:30:08.721 | INFO | "huge" : "", | 20260506 03:30:08.721 | INFO | "huge_shared" : " 27 GB", | 20260506 03:30:08.721 | INFO | "malloc" : " 14 GB", | 20260506 03:30:08.721 | INFO | "pinned" : " 29 GB" | 20260506 03:30:08.721 | INFO | } | 20260506 03:30:08.722 | INFO | Perf report for the last 5.14443 seconds: | 20260506 03:30:08.722 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:30:08.722 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:08.722 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:08.722 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:08.722 | INFO | ANALYSIS : 60.9%, 3.0, 94.6ms, 0 | 20260506 03:30:08.722 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:08.722 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:08.722 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 03:30:08.722 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:08.722 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:08.722 | INFO | Uploaded 36GiB to GPU in 2.07784s (17.3257GiB/s) | 20260506 03:30:08.722 | INFO | Downloaded 34GiB to GPU in 2.00397s (16.9663GiB/s) | 20260506 03:30:13.852 | INFO | Analyzing chunk frames = [8599552,8600064) | 20260506 03:30:13.852 | INFO | MemInfo: { | 20260506 03:30:13.852 | INFO | "gpu" : " 17 GB", | 20260506 03:30:13.852 | INFO | "huge" : "", | 20260506 03:30:13.852 | INFO | "huge_shared" : " 27 GB", | 20260506 03:30:13.852 | INFO | "malloc" : " 14 GB", | 20260506 03:30:13.852 | INFO | "pinned" : " 29 GB" | 20260506 03:30:13.852 | INFO | } | 20260506 03:30:13.852 | INFO | Perf report for the last 5.13086 seconds: | 20260506 03:30:13.852 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:30:13.852 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:13.852 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:13.852 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:13.853 | INFO | ANALYSIS : 60.9%, 3.0, 95.0ms, 0 | 20260506 03:30:13.853 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:13.853 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:13.853 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 03:30:13.853 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:13.853 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:13.853 | INFO | Uploaded 36GiB to GPU in 2.11684s (17.0065GiB/s) | 20260506 03:30:13.853 | INFO | Downloaded 34GiB to GPU in 2.04271s (16.6446GiB/s) | 20260506 03:30:18.996 | INFO | Analyzing chunk frames = [8600064,8600576) | 20260506 03:30:18.996 | INFO | MemInfo: { | 20260506 03:30:18.996 | INFO | "gpu" : " 17 GB", | 20260506 03:30:18.996 | INFO | "huge" : "", | 20260506 03:30:18.996 | INFO | "huge_shared" : " 27 GB", | 20260506 03:30:18.996 | INFO | "malloc" : " 14 GB", | 20260506 03:30:18.996 | INFO | "pinned" : " 29 GB" | 20260506 03:30:18.996 | INFO | } | 20260506 03:30:18.996 | INFO | Perf report for the last 5.14375 seconds: | 20260506 03:30:18.996 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:30:18.996 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:18.996 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:18.996 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:18.996 | INFO | ANALYSIS : 60.7%, 3.0, 94.9ms, 0 | 20260506 03:30:18.996 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:18.996 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:18.996 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 03:30:18.996 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:18.996 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:18.997 | INFO | Uploaded 36GiB to GPU in 2.04084s (17.6398GiB/s) | 20260506 03:30:18.997 | INFO | Downloaded 34GiB to GPU in 2.02128s (16.821GiB/s) | 20260506 03:30:24.131 | INFO | Analyzing chunk frames = [8600576,8601088) | 20260506 03:30:24.131 | INFO | MemInfo: { | 20260506 03:30:24.131 | INFO | "gpu" : " 17 GB", | 20260506 03:30:24.131 | INFO | "huge" : "", | 20260506 03:30:24.131 | INFO | "huge_shared" : " 27 GB", | 20260506 03:30:24.131 | INFO | "malloc" : " 14 GB", | 20260506 03:30:24.131 | INFO | "pinned" : " 29 GB" | 20260506 03:30:24.131 | INFO | } | 20260506 03:30:24.131 | INFO | Perf report for the last 5.13461 seconds: | 20260506 03:30:24.131 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:30:24.131 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:24.131 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:24.131 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:24.131 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 03:30:24.131 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:24.131 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:24.131 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 03:30:24.131 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:24.131 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:24.131 | INFO | Uploaded 36GiB to GPU in 2.07915s (17.3148GiB/s) | 20260506 03:30:24.131 | INFO | Downloaded 34GiB to GPU in 2.05736s (16.526GiB/s) | 20260506 03:30:29.269 | INFO | Analyzing chunk frames = [8601088,8601600) | 20260506 03:30:29.269 | INFO | MemInfo: { | 20260506 03:30:29.269 | INFO | "gpu" : " 17 GB", | 20260506 03:30:29.269 | INFO | "huge" : "", | 20260506 03:30:29.269 | INFO | "huge_shared" : " 27 GB", | 20260506 03:30:29.269 | INFO | "malloc" : " 14 GB", | 20260506 03:30:29.269 | INFO | "pinned" : " 29 GB" | 20260506 03:30:29.269 | INFO | } | 20260506 03:30:29.269 | INFO | Perf report for the last 5.13843 seconds: | 20260506 03:30:29.269 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:30:29.269 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:29.269 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:29.269 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:29.269 | INFO | ANALYSIS : 61.3%, 3.0, 95.8ms, 0 | 20260506 03:30:29.269 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:29.269 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:29.269 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 03:30:29.269 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:29.269 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:29.270 | INFO | Uploaded 36GiB to GPU in 2.07193s (17.3751GiB/s) | 20260506 03:30:29.270 | INFO | Downloaded 34GiB to GPU in 2.03563s (16.7024GiB/s) | 20260506 03:30:29.270 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 03:30:29.270 | INFO | Total mem usage: 15.4 GiB | 20260506 03:30:29.270 | INFO | All Time High Water Marks sum: 16.3 GiB (Savings: 5.2%) | 20260506 03:30:29.270 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -9.99%) | 20260506 03:30:29.270 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 03:30:29.271 | INFO | High Water Breakdown: | 20260506 03:30:29.271 | INFO | 60.9% : 8.54 GiB ( 88.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 03:30:29.271 | INFO | 39% : 5.47 GiB ( 82.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 03:30:29.271 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 03:30:29.271 | INFO | ------------------------------------------ | 20260506 03:30:29.271 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:30:29.271 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 03:30:29.271 | INFO | Total mem usage: 26.5 GiB | 20260506 03:30:29.271 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 03:30:29.271 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 03:30:29.271 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 03:30:29.271 | INFO | High Water Breakdown: | 20260506 03:30:29.271 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 03:30:29.271 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 03:30:29.271 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 03:30:29.271 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 03:30:29.271 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 03:30:29.271 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 03:30:29.271 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 03:30:29.271 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 03:30:29.271 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 03:30:29.271 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 03:30:29.271 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 03:30:29.271 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 03:30:29.271 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 03:30:29.271 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 03:30:29.271 | INFO | ------------------------------------------ | 20260506 03:30:29.271 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:30:29.271 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 03:30:29.271 | INFO | Total mem usage: 25.6 GiB | 20260506 03:30:29.271 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 03:30:29.271 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260506 03:30:29.271 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260506 03:30:29.271 | INFO | High Water Breakdown: | 20260506 03:30:29.271 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260506 03:30:29.271 | INFO | ------------------------------------------ | 20260506 03:30:29.271 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:30:29.271 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 03:30:29.271 | INFO | Total mem usage: 15.7 GiB | 20260506 03:30:29.271 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 03:30:29.271 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 03:30:29.271 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 03:30:29.271 | INFO | High Water Breakdown: | 20260506 03:30:29.271 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 03:30:29.271 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 03:30:29.271 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 03:30:29.271 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 03:30:29.271 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 03:30:29.271 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 03:30:29.271 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 03:30:29.272 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 03:30:29.272 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 03:30:29.272 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 03:30:29.272 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 03:30:29.272 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 03:30:29.272 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 03:30:29.272 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 03:30:29.272 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 03:30:29.272 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 03:30:29.272 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 03:30:29.272 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 03:30:29.272 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 03:30:29.272 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 03:30:29.272 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 03:30:29.272 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 03:30:29.272 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 03:30:29.272 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 03:30:29.272 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 03:30:29.272 | INFO | ------------------------------------------ | 20260506 03:30:29.272 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:30:29.272 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 03:30:29.272 | INFO | PerfReport: Gathered 1536 samples. | 20260506 03:30:29.272 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 03:30:29.272 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.35e+01, 1.35e+01 / 6.66e-01 | 20260506 03:30:29.272 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.05e-01, 1.13e-02 / 4.64e-03 | 20260506 03:30:29.272 | INFO | PerfReport: 11.87%, Baseline -- 3.71e+00 / 4.54e+00, 5.90e-01 / 2.90e-01 | 20260506 03:30:29.272 | INFO | PerfReport: 3.30%, Histogram -- 1.03e+00 / 1.54e+00, 1.95e-01 / 9.39e-02 | 20260506 03:30:29.272 | INFO | PerfReport: 16.00%, DME -- 4.00e+01 / 4.38e+01, 3.74e+00 / 7.42e-01 | 20260506 03:30:29.272 | INFO | PerfReport: 57.53%, FrameLabeling -- 1.80e+01 / 1.92e+01, 1.51e+00 / 1.11e+00 | 20260506 03:30:29.272 | INFO | PerfReport: 7.00%, PulseAccumulating -- 2.19e+00 / 2.73e+00, 4.14e-01 / 2.05e-01 | 20260506 03:30:29.272 | INFO | PerfReport: 2.98%, Metrics -- 9.30e-01 / 1.20e+00, 2.48e-01 / 1.18e-01 | 20260506 03:30:29.272 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.21e-01 / 3.69e-01, 5.38e-02 / 2.65e-02 | 20260506 03:30:29.272 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 03:30:29.272 | INFO | PerfReport: Gathered 1536 samples. | 20260506 03:30:29.272 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 03:30:29.272 | INFO | PerfReport: Full Execution -- 9.74e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260506 03:30:29.272 | INFO | PerfReport: 22.19%, Upload -- 2.16e+01 / 0.00e+00, 7.12e+00 / 0.00e+00 | 20260506 03:30:29.272 | INFO | PerfReport: 21.98%, Download -- 2.14e+01 / 0.00e+00, 7.62e+00 / 0.00e+00 | 20260506 03:30:29.272 | INFO | PerfReport: 32.07%, Compute -- 3.12e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 03:30:29.272 | INFO | PerfReport: 1.97%, LockUpload -- 1.92e+00 / 0.00e+00, 7.78e+00 / 0.00e+00 | 20260506 03:30:29.272 | INFO | PerfReport: 1.29%, LockDownload -- 1.25e+00 / 0.00e+00, 2.92e+00 / 0.00e+00 | 20260506 03:30:29.272 | INFO | PerfReport: 20.50%, LockCompute -- 2.00e+01 / 0.00e+00, 1.74e+01 / 0.00e+00 | 20260506 03:30:29.272 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 2.22e-02 / 0.00e+00 | 20260506 03:30:29.395 | INFO | Starting a round of Baz IO | 20260506 03:30:32.519 | INFO | Populating RTMetrics report | 20260506 03:30:32.520 | INFO | RTMetrics report populated | 20260506 03:30:32.520 | INFO | Opening tmp RTMetrics json file | 20260506 03:30:32.521 | INFO | Serializing RTMetrics to json | 20260506 03:30:32.523 | INFO | Serialization done, swapping tmp file now... | 20260506 03:30:32.524 | INFO | RTMetrics done | 20260506 03:30:32.595 | INFO | Finished a round of Baz IO: Wrote 1.756 GiB in 3.2 seconds | 20260506 03:30:32.595 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.22% / 0.00% / 78.78% | 20260506 03:30:32.595 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0600219% / 3.97364e-06% / 9.93411e-05% | 20260506 03:30:32.595 | INFO | Baz IO overall: | 20260506 03:30:32.595 | INFO | BytesWritten : 4783336912906 | 20260506 03:30:32.595 | INFO | NumEvents : 2242932848172 | 20260506 03:30:34.408 | INFO | Analyzing chunk frames = [8601600,8602112) | 20260506 03:30:34.408 | INFO | MemInfo: { | 20260506 03:30:34.408 | INFO | "gpu" : " 17 GB", | 20260506 03:30:34.408 | INFO | "huge" : "", | 20260506 03:30:34.408 | INFO | "huge_shared" : " 27 GB", | 20260506 03:30:34.408 | INFO | "malloc" : " 14 GB", | 20260506 03:30:34.408 | INFO | "pinned" : " 29 GB" | 20260506 03:30:34.408 | INFO | } | 20260506 03:30:34.408 | INFO | Perf report for the last 5.13835 seconds: | 20260506 03:30:34.408 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:30:34.408 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:34.408 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:34.408 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:34.408 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260506 03:30:34.408 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:34.408 | INFO | RT_METRICS : 42.6%, 1.0, 22.3ms, 0 | 20260506 03:30:34.408 | INFO | PRE_HQ : 58.9%, 1.8, 53.6ms, 0 | 20260506 03:30:34.408 | INFO | BAZWRITER : 61.3%, 3.4, 106.7ms, 0 | 20260506 03:30:34.408 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 03:30:34.408 | INFO | Uploaded 36GiB to GPU in 2.11575s (17.0153GiB/s) | 20260506 03:30:34.408 | INFO | Downloaded 38GiB to GPU in 2.21803s (17.1323GiB/s) | 20260506 03:30:39.550 | INFO | Analyzing chunk frames = [8602112,8602624) | 20260506 03:30:39.550 | INFO | MemInfo: { | 20260506 03:30:39.550 | INFO | "gpu" : " 17 GB", | 20260506 03:30:39.550 | INFO | "huge" : "", | 20260506 03:30:39.550 | INFO | "huge_shared" : " 27 GB", | 20260506 03:30:39.550 | INFO | "malloc" : " 14 GB", | 20260506 03:30:39.550 | INFO | "pinned" : " 29 GB" | 20260506 03:30:39.550 | INFO | } | 20260506 03:30:39.550 | INFO | Perf report for the last 5.14254 seconds: | 20260506 03:30:39.550 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:30:39.550 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:39.550 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:39.550 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:39.550 | INFO | ANALYSIS : 60.7%, 3.0, 94.8ms, 0 | 20260506 03:30:39.550 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:39.550 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:39.550 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 03:30:39.550 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:39.550 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:39.551 | INFO | Uploaded 36GiB to GPU in 1.95479s (18.4163GiB/s) | 20260506 03:30:39.551 | INFO | Downloaded 34GiB to GPU in 1.98003s (17.1715GiB/s) | 20260506 03:30:44.693 | INFO | Analyzing chunk frames = [8602624,8603136) | 20260506 03:30:44.693 | INFO | MemInfo: { | 20260506 03:30:44.693 | INFO | "gpu" : " 17 GB", | 20260506 03:30:44.693 | INFO | "huge" : "", | 20260506 03:30:44.693 | INFO | "huge_shared" : " 27 GB", | 20260506 03:30:44.693 | INFO | "malloc" : " 14 GB", | 20260506 03:30:44.693 | INFO | "pinned" : " 29 GB" | 20260506 03:30:44.693 | INFO | } | 20260506 03:30:44.693 | INFO | Perf report for the last 5.14242 seconds: | 20260506 03:30:44.693 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:30:44.693 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:44.693 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:44.693 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:44.693 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 03:30:44.693 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:44.693 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:44.693 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 03:30:44.693 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:44.693 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:44.693 | INFO | Uploaded 36GiB to GPU in 2.1173s (17.0028GiB/s) | 20260506 03:30:44.693 | INFO | Downloaded 34GiB to GPU in 2.04101s (16.6584GiB/s) | 20260506 03:30:49.835 | INFO | Analyzing chunk frames = [8603136,8603648) | 20260506 03:30:49.835 | INFO | MemInfo: { | 20260506 03:30:49.835 | INFO | "gpu" : " 17 GB", | 20260506 03:30:49.835 | INFO | "huge" : "", | 20260506 03:30:49.835 | INFO | "huge_shared" : " 27 GB", | 20260506 03:30:49.835 | INFO | "malloc" : " 14 GB", | 20260506 03:30:49.835 | INFO | "pinned" : " 29 GB" | 20260506 03:30:49.835 | INFO | } | 20260506 03:30:49.835 | INFO | Perf report for the last 5.14232 seconds: | 20260506 03:30:49.835 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:30:49.835 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:49.835 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:49.835 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:49.835 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260506 03:30:49.835 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:49.835 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:49.835 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 03:30:49.835 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:49.835 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:49.835 | INFO | Uploaded 36GiB to GPU in 2.09299s (17.2003GiB/s) | 20260506 03:30:49.835 | INFO | Downloaded 34GiB to GPU in 2.0153s (16.8709GiB/s) | 20260506 03:30:54.967 | INFO | Analyzing chunk frames = [8603648,8604160) | 20260506 03:30:54.967 | INFO | MemInfo: { | 20260506 03:30:54.967 | INFO | "gpu" : " 17 GB", | 20260506 03:30:54.967 | INFO | "huge" : "", | 20260506 03:30:54.967 | INFO | "huge_shared" : " 27 GB", | 20260506 03:30:54.967 | INFO | "malloc" : " 14 GB", | 20260506 03:30:54.967 | INFO | "pinned" : " 29 GB" | 20260506 03:30:54.967 | INFO | } | 20260506 03:30:54.967 | INFO | Perf report for the last 5.13196 seconds: | 20260506 03:30:54.967 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:30:54.967 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:54.967 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:54.967 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:54.967 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260506 03:30:54.967 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:54.967 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:54.967 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 03:30:54.967 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:54.967 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:30:54.967 | INFO | Uploaded 36GiB to GPU in 2.12899s (16.9094GiB/s) | 20260506 03:30:54.967 | INFO | Downloaded 34GiB to GPU in 2.04169s (16.6528GiB/s) | 20260506 03:31:00.106 | INFO | Analyzing chunk frames = [8604160,8604672) | 20260506 03:31:00.106 | INFO | MemInfo: { | 20260506 03:31:00.106 | INFO | "gpu" : " 17 GB", | 20260506 03:31:00.106 | INFO | "huge" : "", | 20260506 03:31:00.106 | INFO | "huge_shared" : " 27 GB", | 20260506 03:31:00.106 | INFO | "malloc" : " 14 GB", | 20260506 03:31:00.107 | INFO | "pinned" : " 29 GB" | 20260506 03:31:00.107 | INFO | } | 20260506 03:31:00.107 | INFO | Perf report for the last 5.13963 seconds: | 20260506 03:31:00.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:31:00.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:00.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:00.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:00.107 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 03:31:00.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:00.107 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:00.107 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 03:31:00.107 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:00.107 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:00.107 | INFO | Uploaded 36GiB to GPU in 2.10905s (17.0693GiB/s) | 20260506 03:31:00.107 | INFO | Downloaded 34GiB to GPU in 2.04223s (16.6485GiB/s) | 20260506 03:31:05.245 | INFO | Analyzing chunk frames = [8604672,8605184) | 20260506 03:31:05.245 | INFO | MemInfo: { | 20260506 03:31:05.245 | INFO | "gpu" : " 17 GB", | 20260506 03:31:05.245 | INFO | "huge" : "", | 20260506 03:31:05.245 | INFO | "huge_shared" : " 27 GB", | 20260506 03:31:05.245 | INFO | "malloc" : " 14 GB", | 20260506 03:31:05.245 | INFO | "pinned" : " 29 GB" | 20260506 03:31:05.246 | INFO | } | 20260506 03:31:05.246 | INFO | Perf report for the last 5.13913 seconds: | 20260506 03:31:05.246 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:31:05.246 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:05.246 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:05.246 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:05.246 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260506 03:31:05.246 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:05.246 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:05.246 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 03:31:05.246 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:05.246 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:05.246 | INFO | Uploaded 36GiB to GPU in 2.0452s (17.6022GiB/s) | 20260506 03:31:05.246 | INFO | Downloaded 34GiB to GPU in 2.06226s (16.4868GiB/s) | 20260506 03:31:10.392 | INFO | Analyzing chunk frames = [8605184,8605696) | 20260506 03:31:10.392 | INFO | MemInfo: { | 20260506 03:31:10.392 | INFO | "gpu" : " 17 GB", | 20260506 03:31:10.392 | INFO | "huge" : "", | 20260506 03:31:10.392 | INFO | "huge_shared" : " 27 GB", | 20260506 03:31:10.392 | INFO | "malloc" : " 14 GB", | 20260506 03:31:10.392 | INFO | "pinned" : " 29 GB" | 20260506 03:31:10.392 | INFO | } | 20260506 03:31:10.392 | INFO | Perf report for the last 5.14651 seconds: | 20260506 03:31:10.392 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:31:10.392 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:10.392 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:10.392 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:10.392 | INFO | ANALYSIS : 61.2%, 3.0, 95.6ms, 0 | 20260506 03:31:10.392 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:10.392 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:10.392 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 03:31:10.392 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:10.392 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:10.393 | INFO | Uploaded 36GiB to GPU in 2.05951s (17.4799GiB/s) | 20260506 03:31:10.393 | INFO | Downloaded 34GiB to GPU in 2.03272s (16.7264GiB/s) | 20260506 03:31:10.515 | INFO | Starting a round of Baz IO | 20260506 03:31:13.637 | INFO | Populating RTMetrics report | 20260506 03:31:13.637 | INFO | RTMetrics report populated | 20260506 03:31:13.637 | INFO | Opening tmp RTMetrics json file | 20260506 03:31:13.638 | INFO | Serializing RTMetrics to json | 20260506 03:31:13.639 | INFO | Serialization done, swapping tmp file now... | 20260506 03:31:13.640 | INFO | RTMetrics done | 20260506 03:31:13.714 | INFO | Finished a round of Baz IO: Wrote 1.754 GiB in 3.199 seconds | 20260506 03:31:13.714 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.22% / 0.00% / 78.78% | 20260506 03:31:13.714 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0602245% / 0% / 0.000107288% | 20260506 03:31:13.714 | INFO | Baz IO overall: | 20260506 03:31:13.714 | INFO | BytesWritten : 4785220786721 | 20260506 03:31:13.714 | INFO | NumEvents : 2243785596287 | 20260506 03:31:15.525 | INFO | Analyzing chunk frames = [8605696,8606208) | 20260506 03:31:15.525 | INFO | MemInfo: { | 20260506 03:31:15.525 | INFO | "gpu" : " 17 GB", | 20260506 03:31:15.525 | INFO | "huge" : "", | 20260506 03:31:15.525 | INFO | "huge_shared" : " 27 GB", | 20260506 03:31:15.525 | INFO | "malloc" : " 14 GB", | 20260506 03:31:15.525 | INFO | "pinned" : " 29 GB" | 20260506 03:31:15.525 | INFO | } | 20260506 03:31:15.525 | INFO | Perf report for the last 5.13262 seconds: | 20260506 03:31:15.525 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:31:15.525 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:15.525 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:15.525 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:15.525 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260506 03:31:15.525 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:15.525 | INFO | RT_METRICS : 42.6%, 1.0, 22.3ms, 0 | 20260506 03:31:15.525 | INFO | PRE_HQ : 59.1%, 1.7, 53.3ms, 0 | 20260506 03:31:15.525 | INFO | BAZWRITER : 61.5%, 3.5, 109.6ms, 0 | 20260506 03:31:15.525 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 03:31:15.525 | INFO | Uploaded 36GiB to GPU in 2.10514s (17.101GiB/s) | 20260506 03:31:15.525 | INFO | Downloaded 38GiB to GPU in 2.22277s (17.0958GiB/s) | 20260506 03:31:20.667 | INFO | Analyzing chunk frames = [8606208,8606720) | 20260506 03:31:20.667 | INFO | MemInfo: { | 20260506 03:31:20.667 | INFO | "gpu" : " 17 GB", | 20260506 03:31:20.667 | INFO | "huge" : "", | 20260506 03:31:20.667 | INFO | "huge_shared" : " 27 GB", | 20260506 03:31:20.667 | INFO | "malloc" : " 14 GB", | 20260506 03:31:20.667 | INFO | "pinned" : " 29 GB" | 20260506 03:31:20.667 | INFO | } | 20260506 03:31:20.667 | INFO | Perf report for the last 5.1422 seconds: | 20260506 03:31:20.667 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:31:20.667 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:20.667 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:20.667 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:20.667 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260506 03:31:20.667 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:20.667 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:20.667 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 03:31:20.667 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:20.667 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:20.667 | INFO | Uploaded 36GiB to GPU in 1.94325s (18.5256GiB/s) | 20260506 03:31:20.668 | INFO | Downloaded 34GiB to GPU in 1.98075s (17.1652GiB/s) | 20260506 03:31:25.807 | INFO | Analyzing chunk frames = [8606720,8607232) | 20260506 03:31:25.807 | INFO | MemInfo: { | 20260506 03:31:25.807 | INFO | "gpu" : " 17 GB", | 20260506 03:31:25.807 | INFO | "huge" : "", | 20260506 03:31:25.807 | INFO | "huge_shared" : " 27 GB", | 20260506 03:31:25.807 | INFO | "malloc" : " 14 GB", | 20260506 03:31:25.807 | INFO | "pinned" : " 29 GB" | 20260506 03:31:25.807 | INFO | } | 20260506 03:31:25.807 | INFO | Perf report for the last 5.13977 seconds: | 20260506 03:31:25.807 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:31:25.807 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:25.807 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:25.807 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:25.807 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 03:31:25.807 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:25.807 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:25.807 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 03:31:25.807 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:25.807 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:25.807 | INFO | Uploaded 36GiB to GPU in 2.09398s (17.1922GiB/s) | 20260506 03:31:25.807 | INFO | Downloaded 34GiB to GPU in 2.02729s (16.7712GiB/s) | 20260506 03:31:30.947 | INFO | Analyzing chunk frames = [8607232,8607744) | 20260506 03:31:30.947 | INFO | MemInfo: { | 20260506 03:31:30.947 | INFO | "gpu" : " 17 GB", | 20260506 03:31:30.947 | INFO | "huge" : "", | 20260506 03:31:30.947 | INFO | "huge_shared" : " 27 GB", | 20260506 03:31:30.947 | INFO | "malloc" : " 14 GB", | 20260506 03:31:30.947 | INFO | "pinned" : " 29 GB" | 20260506 03:31:30.947 | INFO | } | 20260506 03:31:30.947 | INFO | Perf report for the last 5.14048 seconds: | 20260506 03:31:30.948 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:31:30.948 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:30.948 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:30.948 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:30.948 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260506 03:31:30.948 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:30.948 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:30.948 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 03:31:30.948 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:30.948 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:30.948 | INFO | Uploaded 36GiB to GPU in 2.08564s (17.2609GiB/s) | 20260506 03:31:30.948 | INFO | Downloaded 34GiB to GPU in 2.02138s (16.8202GiB/s) | 20260506 03:31:36.084 | INFO | Analyzing chunk frames = [8607744,8608256) | 20260506 03:31:36.084 | INFO | MemInfo: { | 20260506 03:31:36.084 | INFO | "gpu" : " 17 GB", | 20260506 03:31:36.084 | INFO | "huge" : "", | 20260506 03:31:36.084 | INFO | "huge_shared" : " 27 GB", | 20260506 03:31:36.084 | INFO | "malloc" : " 14 GB", | 20260506 03:31:36.084 | INFO | "pinned" : " 29 GB" | 20260506 03:31:36.084 | INFO | } | 20260506 03:31:36.084 | INFO | Perf report for the last 5.13659 seconds: | 20260506 03:31:36.084 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:31:36.084 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:36.084 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:36.084 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:36.084 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260506 03:31:36.084 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:36.084 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:36.084 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 03:31:36.084 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:36.084 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:36.084 | INFO | Uploaded 36GiB to GPU in 2.11875s (16.9912GiB/s) | 20260506 03:31:36.084 | INFO | Downloaded 34GiB to GPU in 2.03966s (16.6695GiB/s) | 20260506 03:31:41.217 | INFO | Analyzing chunk frames = [8608256,8608768) | 20260506 03:31:41.217 | INFO | MemInfo: { | 20260506 03:31:41.217 | INFO | "gpu" : " 17 GB", | 20260506 03:31:41.217 | INFO | "huge" : "", | 20260506 03:31:41.217 | INFO | "huge_shared" : " 27 GB", | 20260506 03:31:41.217 | INFO | "malloc" : " 14 GB", | 20260506 03:31:41.218 | INFO | "pinned" : " 29 GB" | 20260506 03:31:41.218 | INFO | } | 20260506 03:31:41.218 | INFO | Perf report for the last 5.13371 seconds: | 20260506 03:31:41.218 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:31:41.218 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:41.218 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:41.218 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:41.218 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 03:31:41.218 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:41.218 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:41.218 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 03:31:41.218 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:41.218 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:41.218 | INFO | Uploaded 36GiB to GPU in 2.04901s (17.5694GiB/s) | 20260506 03:31:41.218 | INFO | Downloaded 34GiB to GPU in 2.03161s (16.7355GiB/s) | 20260506 03:31:46.361 | INFO | Analyzing chunk frames = [8608768,8609280) | 20260506 03:31:46.361 | INFO | MemInfo: { | 20260506 03:31:46.361 | INFO | "gpu" : " 17 GB", | 20260506 03:31:46.361 | INFO | "huge" : "", | 20260506 03:31:46.361 | INFO | "huge_shared" : " 27 GB", | 20260506 03:31:46.361 | INFO | "malloc" : " 14 GB", | 20260506 03:31:46.361 | INFO | "pinned" : " 29 GB" | 20260506 03:31:46.361 | INFO | } | 20260506 03:31:46.361 | INFO | Perf report for the last 5.14377 seconds: | 20260506 03:31:46.361 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:31:46.362 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:46.362 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:46.362 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:46.362 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260506 03:31:46.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:46.362 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:46.362 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 03:31:46.362 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:46.362 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:46.362 | INFO | Uploaded 36GiB to GPU in 2.11184s (17.0467GiB/s) | 20260506 03:31:46.362 | INFO | Downloaded 34GiB to GPU in 2.09355s (16.2404GiB/s) | 20260506 03:31:51.509 | INFO | Analyzing chunk frames = [8609280,8609792) | 20260506 03:31:51.509 | INFO | MemInfo: { | 20260506 03:31:51.509 | INFO | "gpu" : " 17 GB", | 20260506 03:31:51.509 | INFO | "huge" : "", | 20260506 03:31:51.509 | INFO | "huge_shared" : " 27 GB", | 20260506 03:31:51.509 | INFO | "malloc" : " 14 GB", | 20260506 03:31:51.510 | INFO | "pinned" : " 29 GB" | 20260506 03:31:51.510 | INFO | } | 20260506 03:31:51.510 | INFO | Perf report for the last 5.14817 seconds: | 20260506 03:31:51.510 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:31:51.510 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:51.510 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:51.510 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:51.510 | INFO | ANALYSIS : 61.2%, 3.0, 95.6ms, 0 | 20260506 03:31:51.510 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:51.510 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:51.510 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 03:31:51.510 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:51.510 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:51.510 | INFO | Uploaded 36GiB to GPU in 2.02496s (17.7781GiB/s) | 20260506 03:31:51.510 | INFO | Downloaded 34GiB to GPU in 2.01241s (16.8952GiB/s) | 20260506 03:31:51.510 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 03:31:51.510 | INFO | Total mem usage: 15.4 GiB | 20260506 03:31:51.510 | INFO | All Time High Water Marks sum: 16.3 GiB (Savings: 5.2%) | 20260506 03:31:51.510 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -10%) | 20260506 03:31:51.510 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 03:31:51.510 | INFO | High Water Breakdown: | 20260506 03:31:51.510 | INFO | 60.9% : 8.54 GiB ( 88.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 03:31:51.510 | INFO | 39% : 5.47 GiB ( 82.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 03:31:51.510 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 03:31:51.510 | INFO | ------------------------------------------ | 20260506 03:31:51.510 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:31:51.510 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 03:31:51.510 | INFO | Total mem usage: 26.5 GiB | 20260506 03:31:51.510 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 03:31:51.510 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 03:31:51.510 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 03:31:51.510 | INFO | High Water Breakdown: | 20260506 03:31:51.510 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 03:31:51.510 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 03:31:51.510 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 03:31:51.510 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 03:31:51.510 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 03:31:51.510 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 03:31:51.510 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 03:31:51.510 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 03:31:51.510 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 03:31:51.510 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 03:31:51.510 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 03:31:51.510 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 03:31:51.511 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 03:31:51.511 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 03:31:51.511 | INFO | ------------------------------------------ | 20260506 03:31:51.511 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:31:51.511 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 03:31:51.511 | INFO | Total mem usage: 25.6 GiB | 20260506 03:31:51.511 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 03:31:51.511 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 03:31:51.511 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 03:31:51.511 | INFO | High Water Breakdown: | 20260506 03:31:51.511 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 03:31:51.511 | INFO | ------------------------------------------ | 20260506 03:31:51.511 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:31:51.511 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 03:31:51.511 | INFO | Total mem usage: 15.7 GiB | 20260506 03:31:51.511 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 03:31:51.511 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 03:31:51.511 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 03:31:51.511 | INFO | High Water Breakdown: | 20260506 03:31:51.511 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 03:31:51.511 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 03:31:51.511 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 03:31:51.511 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 03:31:51.511 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 03:31:51.511 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 03:31:51.511 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 03:31:51.511 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 03:31:51.511 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 03:31:51.511 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 03:31:51.511 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 03:31:51.511 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 03:31:51.511 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 03:31:51.511 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 03:31:51.511 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 03:31:51.511 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 03:31:51.511 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 03:31:51.511 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 03:31:51.511 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 03:31:51.511 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 03:31:51.511 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 03:31:51.511 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 03:31:51.511 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 03:31:51.511 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 03:31:51.511 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 03:31:51.511 | INFO | ------------------------------------------ | 20260506 03:31:51.511 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:31:51.511 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 03:31:51.511 | INFO | PerfReport: Gathered 1536 samples. | 20260506 03:31:51.511 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 03:31:51.511 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.38e+01, 1.33e+01 / 8.43e-01 | 20260506 03:31:51.511 | INFO | PerfReport: 0.29%, AggregateStats -- 9.17e-02 / 1.04e-01, 9.98e-03 / 4.83e-03 | 20260506 03:31:51.511 | INFO | PerfReport: 11.87%, Baseline -- 3.71e+00 / 4.48e+00, 5.82e-01 / 2.64e-01 | 20260506 03:31:51.511 | INFO | PerfReport: 3.29%, Histogram -- 1.03e+00 / 1.51e+00, 1.93e-01 / 9.59e-02 | 20260506 03:31:51.511 | INFO | PerfReport: 16.02%, DME -- 4.00e+01 / 4.41e+01, 3.82e+00 / 1.05e+00 | 20260506 03:31:51.511 | INFO | PerfReport: 57.53%, FrameLabeling -- 1.80e+01 / 1.93e+01, 1.53e+00 / 9.47e-01 | 20260506 03:31:51.511 | INFO | PerfReport: 6.98%, PulseAccumulating -- 2.18e+00 / 2.72e+00, 4.13e-01 / 1.93e-01 | 20260506 03:31:51.511 | INFO | PerfReport: 2.97%, Metrics -- 9.29e-01 / 1.20e+00, 2.49e-01 / 1.05e-01 | 20260506 03:31:51.511 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.21e-01 / 3.74e-01, 5.53e-02 / 2.54e-02 | 20260506 03:31:51.511 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 03:31:51.511 | INFO | PerfReport: Gathered 1536 samples. | 20260506 03:31:51.511 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 03:31:51.511 | INFO | PerfReport: Full Execution -- 9.74e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260506 03:31:51.511 | INFO | PerfReport: 22.15%, Upload -- 2.16e+01 / 0.00e+00, 7.14e+00 / 0.00e+00 | 20260506 03:31:51.511 | INFO | PerfReport: 21.96%, Download -- 2.14e+01 / 0.00e+00, 7.58e+00 / 0.00e+00 | 20260506 03:31:51.511 | INFO | PerfReport: 32.06%, Compute -- 3.12e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 03:31:51.511 | INFO | PerfReport: 2.00%, LockUpload -- 1.95e+00 / 0.00e+00, 7.84e+00 / 0.00e+00 | 20260506 03:31:51.511 | INFO | PerfReport: 1.31%, LockDownload -- 1.28e+00 / 0.00e+00, 2.95e+00 / 0.00e+00 | 20260506 03:31:51.511 | INFO | PerfReport: 20.50%, LockCompute -- 2.00e+01 / 0.00e+00, 1.75e+01 / 0.00e+00 | 20260506 03:31:51.511 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.43e-02 / 0.00e+00 | 20260506 03:31:51.656 | INFO | Starting a round of Baz IO | 20260506 03:31:54.749 | INFO | Populating RTMetrics report | 20260506 03:31:54.749 | INFO | RTMetrics report populated | 20260506 03:31:54.749 | INFO | Opening tmp RTMetrics json file | 20260506 03:31:54.750 | INFO | Serializing RTMetrics to json | 20260506 03:31:54.751 | INFO | Serialization done, swapping tmp file now... | 20260506 03:31:54.752 | INFO | RTMetrics done | 20260506 03:31:54.826 | INFO | Finished a round of Baz IO: Wrote 1.756 GiB in 3.17 seconds | 20260506 03:31:54.826 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.22% / 0.00% / 78.78% | 20260506 03:31:54.826 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0602563% / 0% / 9.93411e-05% | 20260506 03:31:54.826 | INFO | Baz IO overall: | 20260506 03:31:54.826 | INFO | BytesWritten : 4787106553301 | 20260506 03:31:54.826 | INFO | NumEvents : 2244639769187 | 20260506 03:31:56.644 | INFO | Analyzing chunk frames = [8609792,8610304) | 20260506 03:31:56.644 | INFO | MemInfo: { | 20260506 03:31:56.644 | INFO | "gpu" : " 17 GB", | 20260506 03:31:56.644 | INFO | "huge" : "", | 20260506 03:31:56.644 | INFO | "huge_shared" : " 27 GB", | 20260506 03:31:56.644 | INFO | "malloc" : " 14 GB", | 20260506 03:31:56.644 | INFO | "pinned" : " 29 GB" | 20260506 03:31:56.644 | INFO | } | 20260506 03:31:56.644 | INFO | Perf report for the last 5.13474 seconds: | 20260506 03:31:56.644 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:31:56.645 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:56.645 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:56.645 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:56.645 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 03:31:56.645 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:31:56.645 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260506 03:31:56.645 | INFO | PRE_HQ : 59.0%, 1.7, 53.5ms, 0 | 20260506 03:31:56.645 | INFO | BAZWRITER : 61.7%, 3.5, 109.3ms, 0 | 20260506 03:31:56.645 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 03:31:56.645 | INFO | Uploaded 36GiB to GPU in 2.09199s (17.2085GiB/s) | 20260506 03:31:56.645 | INFO | Downloaded 38GiB to GPU in 2.22185s (17.1028GiB/s) | 20260506 03:32:01.780 | INFO | Analyzing chunk frames = [8610304,8610816) | 20260506 03:32:01.780 | INFO | MemInfo: { | 20260506 03:32:01.780 | INFO | "gpu" : " 17 GB", | 20260506 03:32:01.780 | INFO | "huge" : "", | 20260506 03:32:01.780 | INFO | "huge_shared" : " 27 GB", | 20260506 03:32:01.780 | INFO | "malloc" : " 14 GB", | 20260506 03:32:01.780 | INFO | "pinned" : " 29 GB" | 20260506 03:32:01.780 | INFO | } | 20260506 03:32:01.780 | INFO | Perf report for the last 5.13568 seconds: | 20260506 03:32:01.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:32:01.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:01.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:01.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:01.780 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 03:32:01.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:01.780 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:01.780 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 03:32:01.780 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:01.780 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:01.780 | INFO | Uploaded 36GiB to GPU in 2.03431s (17.6965GiB/s) | 20260506 03:32:01.780 | INFO | Downloaded 34GiB to GPU in 1.99339s (17.0564GiB/s) | 20260506 03:32:06.922 | INFO | Analyzing chunk frames = [8610816,8611328) | 20260506 03:32:06.922 | INFO | MemInfo: { | 20260506 03:32:06.922 | INFO | "gpu" : " 17 GB", | 20260506 03:32:06.922 | INFO | "huge" : "", | 20260506 03:32:06.922 | INFO | "huge_shared" : " 27 GB", | 20260506 03:32:06.922 | INFO | "malloc" : " 14 GB", | 20260506 03:32:06.922 | INFO | "pinned" : " 29 GB" | 20260506 03:32:06.922 | INFO | } | 20260506 03:32:06.922 | INFO | Perf report for the last 5.14237 seconds: | 20260506 03:32:06.923 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:32:06.923 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:06.923 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:06.923 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:06.923 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260506 03:32:06.923 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:06.923 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:06.923 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:32:06.923 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:06.923 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:06.923 | INFO | Uploaded 36GiB to GPU in 2.10938s (17.0666GiB/s) | 20260506 03:32:06.923 | INFO | Downloaded 34GiB to GPU in 2.04215s (16.6491GiB/s) | 20260506 03:32:12.058 | INFO | Analyzing chunk frames = [8611328,8611840) | 20260506 03:32:12.058 | INFO | MemInfo: { | 20260506 03:32:12.058 | INFO | "gpu" : " 17 GB", | 20260506 03:32:12.058 | INFO | "huge" : "", | 20260506 03:32:12.058 | INFO | "huge_shared" : " 27 GB", | 20260506 03:32:12.058 | INFO | "malloc" : " 14 GB", | 20260506 03:32:12.058 | INFO | "pinned" : " 29 GB" | 20260506 03:32:12.058 | INFO | } | 20260506 03:32:12.058 | INFO | Perf report for the last 5.13574 seconds: | 20260506 03:32:12.058 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:32:12.058 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:12.058 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:12.058 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:12.058 | INFO | ANALYSIS : 61.0%, 3.0, 94.7ms, 0 | 20260506 03:32:12.058 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:12.058 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:12.058 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 03:32:12.058 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:12.058 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:12.058 | INFO | Uploaded 36GiB to GPU in 2.09147s (17.2128GiB/s) | 20260506 03:32:12.059 | INFO | Downloaded 34GiB to GPU in 2.01075s (16.9091GiB/s) | 20260506 03:32:17.195 | INFO | Analyzing chunk frames = [8611840,8612352) | 20260506 03:32:17.195 | INFO | MemInfo: { | 20260506 03:32:17.195 | INFO | "gpu" : " 17 GB", | 20260506 03:32:17.195 | INFO | "huge" : "", | 20260506 03:32:17.195 | INFO | "huge_shared" : " 27 GB", | 20260506 03:32:17.195 | INFO | "malloc" : " 14 GB", | 20260506 03:32:17.195 | INFO | "pinned" : " 29 GB" | 20260506 03:32:17.195 | INFO | } | 20260506 03:32:17.195 | INFO | Perf report for the last 5.1369 seconds: | 20260506 03:32:17.195 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:32:17.195 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:17.195 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:17.195 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:17.195 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260506 03:32:17.195 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:17.195 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:17.195 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 03:32:17.195 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:17.195 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:17.195 | INFO | Uploaded 36GiB to GPU in 2.11993s (16.9817GiB/s) | 20260506 03:32:17.195 | INFO | Downloaded 34GiB to GPU in 2.03959s (16.67GiB/s) | 20260506 03:32:22.340 | INFO | Analyzing chunk frames = [8612352,8612864) | 20260506 03:32:22.340 | INFO | MemInfo: { | 20260506 03:32:22.340 | INFO | "gpu" : " 17 GB", | 20260506 03:32:22.340 | INFO | "huge" : "", | 20260506 03:32:22.340 | INFO | "huge_shared" : " 27 GB", | 20260506 03:32:22.340 | INFO | "malloc" : " 14 GB", | 20260506 03:32:22.340 | INFO | "pinned" : " 29 GB" | 20260506 03:32:22.340 | INFO | } | 20260506 03:32:22.340 | INFO | Perf report for the last 5.1446 seconds: | 20260506 03:32:22.340 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:32:22.340 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:22.340 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:22.340 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:22.340 | INFO | ANALYSIS : 61.0%, 3.0, 95.3ms, 0 | 20260506 03:32:22.340 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:22.340 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:22.340 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 03:32:22.340 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:22.340 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:22.340 | INFO | Uploaded 36GiB to GPU in 2.11041s (17.0583GiB/s) | 20260506 03:32:22.340 | INFO | Downloaded 34GiB to GPU in 2.04179s (16.6521GiB/s) | 20260506 03:32:27.479 | INFO | Analyzing chunk frames = [8612864,8613376) | 20260506 03:32:27.479 | INFO | MemInfo: { | 20260506 03:32:27.479 | INFO | "gpu" : " 17 GB", | 20260506 03:32:27.479 | INFO | "huge" : "", | 20260506 03:32:27.479 | INFO | "huge_shared" : " 27 GB", | 20260506 03:32:27.479 | INFO | "malloc" : " 14 GB", | 20260506 03:32:27.479 | INFO | "pinned" : " 29 GB" | 20260506 03:32:27.479 | INFO | } | 20260506 03:32:27.479 | INFO | Perf report for the last 5.13929 seconds: | 20260506 03:32:27.479 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:32:27.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:27.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:27.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:27.479 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 03:32:27.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:27.479 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:27.479 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 03:32:27.479 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:27.479 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:27.479 | INFO | Uploaded 36GiB to GPU in 2.02888s (17.7438GiB/s) | 20260506 03:32:27.479 | INFO | Downloaded 34GiB to GPU in 2.07272s (16.4036GiB/s) | 20260506 03:32:32.615 | INFO | Analyzing chunk frames = [8613376,8613888) | 20260506 03:32:32.615 | INFO | MemInfo: { | 20260506 03:32:32.615 | INFO | "gpu" : " 17 GB", | 20260506 03:32:32.615 | INFO | "huge" : "", | 20260506 03:32:32.615 | INFO | "huge_shared" : " 27 GB", | 20260506 03:32:32.615 | INFO | "malloc" : " 14 GB", | 20260506 03:32:32.615 | INFO | "pinned" : " 29 GB" | 20260506 03:32:32.615 | INFO | } | 20260506 03:32:32.615 | INFO | Perf report for the last 5.13581 seconds: | 20260506 03:32:32.615 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:32:32.615 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:32.615 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:32.615 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:32.615 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 03:32:32.615 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:32.615 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:32.615 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 03:32:32.615 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:32.615 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:32.615 | INFO | Uploaded 36GiB to GPU in 2.03669s (17.6757GiB/s) | 20260506 03:32:32.615 | INFO | Downloaded 34GiB to GPU in 2.01941s (16.8366GiB/s) | 20260506 03:32:32.741 | INFO | Starting a round of Baz IO | 20260506 03:32:35.859 | INFO | Populating RTMetrics report | 20260506 03:32:35.859 | INFO | RTMetrics report populated | 20260506 03:32:35.859 | INFO | Opening tmp RTMetrics json file | 20260506 03:32:35.861 | INFO | Serializing RTMetrics to json | 20260506 03:32:35.862 | INFO | Serialization done, swapping tmp file now... | 20260506 03:32:35.863 | INFO | RTMetrics done | 20260506 03:32:35.932 | INFO | Finished a round of Baz IO: Wrote 1.757 GiB in 3.191 seconds | 20260506 03:32:35.932 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.22% / 0.00% / 78.78% | 20260506 03:32:35.932 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0599504% / 0% / 7.54992e-05% | 20260506 03:32:35.932 | INFO | Baz IO overall: | 20260506 03:32:35.932 | INFO | BytesWritten : 4788993575583 | 20260506 03:32:35.932 | INFO | NumEvents : 2245495024106 | 20260506 03:32:37.759 | INFO | Analyzing chunk frames = [8613888,8614400) | 20260506 03:32:37.759 | INFO | MemInfo: { | 20260506 03:32:37.759 | INFO | "gpu" : " 17 GB", | 20260506 03:32:37.760 | INFO | "huge" : "", | 20260506 03:32:37.760 | INFO | "huge_shared" : " 27 GB", | 20260506 03:32:37.760 | INFO | "malloc" : " 14 GB", | 20260506 03:32:37.760 | INFO | "pinned" : " 29 GB" | 20260506 03:32:37.760 | INFO | } | 20260506 03:32:37.760 | INFO | Perf report for the last 5.14473 seconds: | 20260506 03:32:37.760 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:32:37.760 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:37.760 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:37.760 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:37.760 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260506 03:32:37.760 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:37.760 | INFO | RT_METRICS : 42.6%, 1.0, 22.3ms, 0 | 20260506 03:32:37.760 | INFO | PRE_HQ : 59.0%, 1.7, 53.1ms, 0 | 20260506 03:32:37.760 | INFO | BAZWRITER : 61.2%, 3.4, 107.7ms, 0 | 20260506 03:32:37.760 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 03:32:37.760 | INFO | Uploaded 36GiB to GPU in 2.14763s (16.7627GiB/s) | 20260506 03:32:37.760 | INFO | Downloaded 38GiB to GPU in 2.22313s (17.093GiB/s) | 20260506 03:32:42.900 | INFO | Analyzing chunk frames = [8614400,8614912) | 20260506 03:32:42.900 | INFO | MemInfo: { | 20260506 03:32:42.900 | INFO | "gpu" : " 17 GB", | 20260506 03:32:42.900 | INFO | "huge" : "", | 20260506 03:32:42.900 | INFO | "huge_shared" : " 27 GB", | 20260506 03:32:42.900 | INFO | "malloc" : " 14 GB", | 20260506 03:32:42.900 | INFO | "pinned" : " 29 GB" | 20260506 03:32:42.900 | INFO | } | 20260506 03:32:42.900 | INFO | Perf report for the last 5.1406 seconds: | 20260506 03:32:42.900 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:32:42.900 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:42.900 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:42.900 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:42.900 | INFO | ANALYSIS : 60.8%, 3.0, 94.9ms, 0 | 20260506 03:32:42.900 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:42.900 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:42.900 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:32:42.900 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:42.900 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:42.900 | INFO | Uploaded 36GiB to GPU in 2.01526s (17.8637GiB/s) | 20260506 03:32:42.900 | INFO | Downloaded 34GiB to GPU in 2.00466s (16.9605GiB/s) | 20260506 03:32:48.031 | INFO | Analyzing chunk frames = [8614912,8615424) | 20260506 03:32:48.031 | INFO | MemInfo: { | 20260506 03:32:48.031 | INFO | "gpu" : " 17 GB", | 20260506 03:32:48.031 | INFO | "huge" : "", | 20260506 03:32:48.031 | INFO | "huge_shared" : " 27 GB", | 20260506 03:32:48.031 | INFO | "malloc" : " 14 GB", | 20260506 03:32:48.031 | INFO | "pinned" : " 29 GB" | 20260506 03:32:48.031 | INFO | } | 20260506 03:32:48.032 | INFO | Perf report for the last 5.1313 seconds: | 20260506 03:32:48.032 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:32:48.032 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:48.032 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:48.032 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:48.032 | INFO | ANALYSIS : 60.2%, 3.0, 93.5ms, 0 | 20260506 03:32:48.032 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:48.032 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:48.032 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 03:32:48.032 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:48.032 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:48.032 | INFO | Uploaded 36GiB to GPU in 1.96431s (18.327GiB/s) | 20260506 03:32:48.032 | INFO | Downloaded 34GiB to GPU in 1.98478s (17.1304GiB/s) | 20260506 03:32:53.168 | INFO | Analyzing chunk frames = [8615424,8615936) | 20260506 03:32:53.168 | INFO | MemInfo: { | 20260506 03:32:53.168 | INFO | "gpu" : " 17 GB", | 20260506 03:32:53.168 | INFO | "huge" : "", | 20260506 03:32:53.168 | INFO | "huge_shared" : " 27 GB", | 20260506 03:32:53.168 | INFO | "malloc" : " 14 GB", | 20260506 03:32:53.168 | INFO | "pinned" : " 29 GB" | 20260506 03:32:53.168 | INFO | } | 20260506 03:32:53.168 | INFO | Perf report for the last 5.13683 seconds: | 20260506 03:32:53.168 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:32:53.168 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:53.168 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:53.168 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:53.168 | INFO | ANALYSIS : 61.0%, 3.0, 94.6ms, 0 | 20260506 03:32:53.168 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:53.168 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:53.169 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 03:32:53.169 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:53.169 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:53.169 | INFO | Uploaded 36GiB to GPU in 2.08784s (17.2427GiB/s) | 20260506 03:32:53.169 | INFO | Downloaded 34GiB to GPU in 2.01205s (16.8982GiB/s) | 20260506 03:32:58.313 | INFO | Analyzing chunk frames = [8615936,8616448) | 20260506 03:32:58.313 | INFO | MemInfo: { | 20260506 03:32:58.313 | INFO | "gpu" : " 17 GB", | 20260506 03:32:58.313 | INFO | "huge" : "", | 20260506 03:32:58.313 | INFO | "huge_shared" : " 27 GB", | 20260506 03:32:58.313 | INFO | "malloc" : " 14 GB", | 20260506 03:32:58.313 | INFO | "pinned" : " 29 GB" | 20260506 03:32:58.314 | INFO | } | 20260506 03:32:58.314 | INFO | Perf report for the last 5.1452 seconds: | 20260506 03:32:58.314 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:32:58.314 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:58.314 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:58.314 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:58.314 | INFO | ANALYSIS : 60.5%, 3.0, 94.6ms, 0 | 20260506 03:32:58.314 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:58.314 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:58.314 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 03:32:58.314 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:58.314 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:32:58.314 | INFO | Uploaded 36GiB to GPU in 2.07179s (17.3763GiB/s) | 20260506 03:32:58.314 | INFO | Downloaded 34GiB to GPU in 2.03802s (16.6829GiB/s) | 20260506 03:33:03.462 | INFO | Analyzing chunk frames = [8616448,8616960) | 20260506 03:33:03.462 | INFO | MemInfo: { | 20260506 03:33:03.462 | INFO | "gpu" : " 17 GB", | 20260506 03:33:03.462 | INFO | "huge" : "", | 20260506 03:33:03.462 | INFO | "huge_shared" : " 27 GB", | 20260506 03:33:03.462 | INFO | "malloc" : " 14 GB", | 20260506 03:33:03.462 | INFO | "pinned" : " 29 GB" | 20260506 03:33:03.462 | INFO | } | 20260506 03:33:03.463 | INFO | Perf report for the last 5.14888 seconds: | 20260506 03:33:03.463 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:33:03.463 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:03.463 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:03.463 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:03.463 | INFO | ANALYSIS : 60.6%, 3.0, 94.7ms, 0 | 20260506 03:33:03.463 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:03.463 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:03.463 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 03:33:03.463 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:03.463 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:03.463 | INFO | Uploaded 36GiB to GPU in 2.0572s (17.4995GiB/s) | 20260506 03:33:03.463 | INFO | Downloaded 34GiB to GPU in 2.03101s (16.7405GiB/s) | 20260506 03:33:08.594 | INFO | Analyzing chunk frames = [8616960,8617472) | 20260506 03:33:08.594 | INFO | MemInfo: { | 20260506 03:33:08.594 | INFO | "gpu" : " 17 GB", | 20260506 03:33:08.594 | INFO | "huge" : "", | 20260506 03:33:08.594 | INFO | "huge_shared" : " 27 GB", | 20260506 03:33:08.594 | INFO | "malloc" : " 14 GB", | 20260506 03:33:08.594 | INFO | "pinned" : " 29 GB" | 20260506 03:33:08.594 | INFO | } | 20260506 03:33:08.594 | INFO | Perf report for the last 5.13165 seconds: | 20260506 03:33:08.594 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:33:08.594 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:08.594 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:08.594 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:08.594 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 03:33:08.594 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:08.594 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:08.594 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 03:33:08.594 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:08.594 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:08.595 | INFO | Uploaded 36GiB to GPU in 2.07046s (17.3875GiB/s) | 20260506 03:33:08.595 | INFO | Downloaded 34GiB to GPU in 2.05739s (16.5258GiB/s) | 20260506 03:33:13.732 | INFO | Analyzing chunk frames = [8617472,8617984) | 20260506 03:33:13.732 | INFO | MemInfo: { | 20260506 03:33:13.732 | INFO | "gpu" : " 17 GB", | 20260506 03:33:13.732 | INFO | "huge" : "", | 20260506 03:33:13.732 | INFO | "huge_shared" : " 27 GB", | 20260506 03:33:13.732 | INFO | "malloc" : " 14 GB", | 20260506 03:33:13.732 | INFO | "pinned" : " 29 GB" | 20260506 03:33:13.732 | INFO | } | 20260506 03:33:13.732 | INFO | Perf report for the last 5.13741 seconds: | 20260506 03:33:13.732 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:33:13.732 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:13.732 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:13.732 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:13.732 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 03:33:13.732 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:13.732 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:13.732 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:33:13.732 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:13.732 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:13.732 | INFO | Uploaded 36GiB to GPU in 2.02435s (17.7834GiB/s) | 20260506 03:33:13.732 | INFO | Downloaded 34GiB to GPU in 2.01817s (16.847GiB/s) | 20260506 03:33:13.732 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 03:33:13.732 | INFO | Total mem usage: 15.4 GiB | 20260506 03:33:13.732 | INFO | All Time High Water Marks sum: 16.3 GiB (Savings: 5.2%) | 20260506 03:33:13.732 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -10%) | 20260506 03:33:13.732 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 03:33:13.732 | INFO | High Water Breakdown: | 20260506 03:33:13.732 | INFO | 60.9% : 8.54 GiB ( 88.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 03:33:13.732 | INFO | 39% : 5.47 GiB ( 82.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 03:33:13.732 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 03:33:13.732 | INFO | ------------------------------------------ | 20260506 03:33:13.732 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:33:13.732 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 03:33:13.732 | INFO | Total mem usage: 26.5 GiB | 20260506 03:33:13.732 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 03:33:13.732 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 03:33:13.732 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 03:33:13.732 | INFO | High Water Breakdown: | 20260506 03:33:13.732 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 03:33:13.733 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 03:33:13.733 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 03:33:13.733 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 03:33:13.733 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 03:33:13.733 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 03:33:13.733 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 03:33:13.733 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 03:33:13.733 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 03:33:13.733 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 03:33:13.733 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 03:33:13.733 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 03:33:13.733 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 03:33:13.733 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 03:33:13.733 | INFO | ------------------------------------------ | 20260506 03:33:13.733 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:33:13.733 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 03:33:13.733 | INFO | Total mem usage: 25.6 GiB | 20260506 03:33:13.733 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 03:33:13.733 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 03:33:13.733 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 03:33:13.733 | INFO | High Water Breakdown: | 20260506 03:33:13.733 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 03:33:13.733 | INFO | ------------------------------------------ | 20260506 03:33:13.733 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:33:13.733 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 03:33:13.733 | INFO | Total mem usage: 15.7 GiB | 20260506 03:33:13.733 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 03:33:13.733 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 03:33:13.733 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 03:33:13.733 | INFO | High Water Breakdown: | 20260506 03:33:13.733 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 03:33:13.733 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 03:33:13.733 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 03:33:13.733 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 03:33:13.733 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 03:33:13.733 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 03:33:13.733 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 03:33:13.733 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 03:33:13.733 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 03:33:13.733 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 03:33:13.733 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 03:33:13.733 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 03:33:13.733 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 03:33:13.733 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 03:33:13.733 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 03:33:13.733 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 03:33:13.733 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 03:33:13.733 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 03:33:13.733 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 03:33:13.733 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 03:33:13.733 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 03:33:13.733 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 03:33:13.733 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 03:33:13.733 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 03:33:13.733 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 03:33:13.733 | INFO | ------------------------------------------ | 20260506 03:33:13.733 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:33:13.733 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 03:33:13.733 | INFO | PerfReport: Gathered 1536 samples. | 20260506 03:33:13.733 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 03:33:13.733 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.35e+01, 1.35e+01 / 7.38e-01 | 20260506 03:33:13.733 | INFO | PerfReport: 0.29%, AggregateStats -- 9.18e-02 / 1.03e-01, 1.02e-02 / 4.23e-03 | 20260506 03:33:13.733 | INFO | PerfReport: 11.89%, Baseline -- 3.72e+00 / 4.49e+00, 5.92e-01 / 2.98e-01 | 20260506 03:33:13.733 | INFO | PerfReport: 3.30%, Histogram -- 1.03e+00 / 1.52e+00, 1.97e-01 / 9.60e-02 | 20260506 03:33:13.733 | INFO | PerfReport: 16.01%, DME -- 4.00e+01 / 4.40e+01, 3.87e+00 / 9.15e-01 | 20260506 03:33:13.733 | INFO | PerfReport: 57.52%, FrameLabeling -- 1.80e+01 / 1.91e+01, 1.52e+00 / 1.05e+00 | 20260506 03:33:13.733 | INFO | PerfReport: 6.98%, PulseAccumulating -- 2.18e+00 / 2.73e+00, 4.17e-01 / 2.11e-01 | 20260506 03:33:13.733 | INFO | PerfReport: 2.97%, Metrics -- 9.28e-01 / 1.19e+00, 2.50e-01 / 8.91e-02 | 20260506 03:33:13.733 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.20e-01 / 3.70e-01, 5.40e-02 / 2.40e-02 | 20260506 03:33:13.733 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 03:33:13.733 | INFO | PerfReport: Gathered 1536 samples. | 20260506 03:33:13.733 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 03:33:13.733 | INFO | PerfReport: Full Execution -- 9.73e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260506 03:33:13.733 | INFO | PerfReport: 22.13%, Upload -- 2.15e+01 / 0.00e+00, 7.07e+00 / 0.00e+00 | 20260506 03:33:13.733 | INFO | PerfReport: 21.96%, Download -- 2.14e+01 / 0.00e+00, 7.59e+00 / 0.00e+00 | 20260506 03:33:13.733 | INFO | PerfReport: 32.13%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 03:33:13.734 | INFO | PerfReport: 1.94%, LockUpload -- 1.88e+00 / 0.00e+00, 7.72e+00 / 0.00e+00 | 20260506 03:33:13.734 | INFO | PerfReport: 1.31%, LockDownload -- 1.28e+00 / 0.00e+00, 3.00e+00 / 0.00e+00 | 20260506 03:33:13.734 | INFO | PerfReport: 20.51%, LockCompute -- 1.99e+01 / 0.00e+00, 1.75e+01 / 0.00e+00 | 20260506 03:33:13.734 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.94e-02 / 0.00e+00 | 20260506 03:33:13.862 | INFO | Starting a round of Baz IO | 20260506 03:33:16.977 | INFO | Populating RTMetrics report | 20260506 03:33:16.978 | INFO | RTMetrics report populated | 20260506 03:33:16.978 | INFO | Opening tmp RTMetrics json file | 20260506 03:33:16.979 | INFO | Serializing RTMetrics to json | 20260506 03:33:16.980 | INFO | Serialization done, swapping tmp file now... | 20260506 03:33:16.980 | INFO | RTMetrics done | 20260506 03:33:17.051 | INFO | Finished a round of Baz IO: Wrote 1.756 GiB in 3.189 seconds | 20260506 03:33:17.051 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.22% / 0.00% / 78.78% | 20260506 03:33:17.051 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0603159% / 3.97364e-06% / 9.13938e-05% | 20260506 03:33:17.051 | INFO | Baz IO overall: | 20260506 03:33:17.051 | INFO | BytesWritten : 4790879472508 | 20260506 03:33:17.051 | INFO | NumEvents : 2246349651944 | 20260506 03:33:18.867 | INFO | Analyzing chunk frames = [8617984,8618496) | 20260506 03:33:18.867 | INFO | MemInfo: { | 20260506 03:33:18.867 | INFO | "gpu" : " 17 GB", | 20260506 03:33:18.867 | INFO | "huge" : "", | 20260506 03:33:18.867 | INFO | "huge_shared" : " 27 GB", | 20260506 03:33:18.867 | INFO | "malloc" : " 14 GB", | 20260506 03:33:18.867 | INFO | "pinned" : " 29 GB" | 20260506 03:33:18.867 | INFO | } | 20260506 03:33:18.867 | INFO | Perf report for the last 5.13587 seconds: | 20260506 03:33:18.868 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:33:18.868 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:18.868 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:18.868 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:18.868 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260506 03:33:18.868 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:18.868 | INFO | RT_METRICS : 42.6%, 1.0, 22.3ms, 0 | 20260506 03:33:18.868 | INFO | PRE_HQ : 59.2%, 1.7, 53.2ms, 0 | 20260506 03:33:18.868 | INFO | BAZWRITER : 61.3%, 3.4, 108.1ms, 0 | 20260506 03:33:18.868 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 03:33:18.868 | INFO | Uploaded 36GiB to GPU in 2.11141s (17.0502GiB/s) | 20260506 03:33:18.868 | INFO | Downloaded 38GiB to GPU in 2.23289s (17.0183GiB/s) | 20260506 03:33:24.008 | INFO | Analyzing chunk frames = [8618496,8619008) | 20260506 03:33:24.008 | INFO | MemInfo: { | 20260506 03:33:24.008 | INFO | "gpu" : " 17 GB", | 20260506 03:33:24.008 | INFO | "huge" : "", | 20260506 03:33:24.008 | INFO | "huge_shared" : " 27 GB", | 20260506 03:33:24.008 | INFO | "malloc" : " 14 GB", | 20260506 03:33:24.008 | INFO | "pinned" : " 29 GB" | 20260506 03:33:24.008 | INFO | } | 20260506 03:33:24.008 | INFO | Perf report for the last 5.14096 seconds: | 20260506 03:33:24.008 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:33:24.008 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:24.008 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:24.009 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:24.009 | INFO | ANALYSIS : 60.6%, 3.0, 94.7ms, 0 | 20260506 03:33:24.009 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:24.009 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:24.009 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 03:33:24.009 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:24.009 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:24.009 | INFO | Uploaded 36GiB to GPU in 1.93561s (18.5988GiB/s) | 20260506 03:33:24.009 | INFO | Downloaded 34GiB to GPU in 1.97321s (17.2308GiB/s) | 20260506 03:33:29.147 | INFO | Analyzing chunk frames = [8619008,8619520) | 20260506 03:33:29.147 | INFO | MemInfo: { | 20260506 03:33:29.147 | INFO | "gpu" : " 17 GB", | 20260506 03:33:29.147 | INFO | "huge" : "", | 20260506 03:33:29.147 | INFO | "huge_shared" : " 27 GB", | 20260506 03:33:29.147 | INFO | "malloc" : " 14 GB", | 20260506 03:33:29.147 | INFO | "pinned" : " 29 GB" | 20260506 03:33:29.147 | INFO | } | 20260506 03:33:29.147 | INFO | Perf report for the last 5.13868 seconds: | 20260506 03:33:29.147 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:33:29.147 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:29.147 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:29.147 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:29.147 | INFO | ANALYSIS : 60.5%, 3.0, 94.1ms, 0 | 20260506 03:33:29.147 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:29.147 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:29.147 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 03:33:29.147 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:29.147 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:29.148 | INFO | Uploaded 36GiB to GPU in 2.00055s (17.9951GiB/s) | 20260506 03:33:29.148 | INFO | Downloaded 34GiB to GPU in 1.9874s (17.1078GiB/s) | 20260506 03:33:34.284 | INFO | Analyzing chunk frames = [8619520,8620032) | 20260506 03:33:34.284 | INFO | MemInfo: { | 20260506 03:33:34.284 | INFO | "gpu" : " 17 GB", | 20260506 03:33:34.284 | INFO | "huge" : "", | 20260506 03:33:34.284 | INFO | "huge_shared" : " 27 GB", | 20260506 03:33:34.284 | INFO | "malloc" : " 14 GB", | 20260506 03:33:34.284 | INFO | "pinned" : " 29 GB" | 20260506 03:33:34.284 | INFO | } | 20260506 03:33:34.284 | INFO | Perf report for the last 5.1369 seconds: | 20260506 03:33:34.284 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:33:34.284 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:34.284 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:34.284 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:34.284 | INFO | ANALYSIS : 61.0%, 3.0, 94.7ms, 0 | 20260506 03:33:34.284 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:34.284 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:34.284 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 03:33:34.284 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:34.284 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:34.284 | INFO | Uploaded 36GiB to GPU in 2.08984s (17.2262GiB/s) | 20260506 03:33:34.284 | INFO | Downloaded 34GiB to GPU in 2.00098s (16.9917GiB/s) | 20260506 03:33:39.429 | INFO | Analyzing chunk frames = [8620032,8620544) | 20260506 03:33:39.429 | INFO | MemInfo: { | 20260506 03:33:39.429 | INFO | "gpu" : " 17 GB", | 20260506 03:33:39.429 | INFO | "huge" : "", | 20260506 03:33:39.429 | INFO | "huge_shared" : " 27 GB", | 20260506 03:33:39.429 | INFO | "malloc" : " 14 GB", | 20260506 03:33:39.429 | INFO | "pinned" : " 29 GB" | 20260506 03:33:39.429 | INFO | } | 20260506 03:33:39.429 | INFO | Perf report for the last 5.14519 seconds: | 20260506 03:33:39.429 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:33:39.429 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:39.429 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:39.429 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:39.429 | INFO | ANALYSIS : 60.7%, 3.0, 94.8ms, 0 | 20260506 03:33:39.429 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:39.429 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:39.429 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 03:33:39.429 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:39.429 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:39.430 | INFO | Uploaded 36GiB to GPU in 2.08609s (17.2572GiB/s) | 20260506 03:33:39.430 | INFO | Downloaded 34GiB to GPU in 2.03707s (16.6907GiB/s) | 20260506 03:33:44.576 | INFO | Analyzing chunk frames = [8620544,8621056) | 20260506 03:33:44.576 | INFO | MemInfo: { | 20260506 03:33:44.576 | INFO | "gpu" : " 17 GB", | 20260506 03:33:44.576 | INFO | "huge" : "", | 20260506 03:33:44.576 | INFO | "huge_shared" : " 27 GB", | 20260506 03:33:44.576 | INFO | "malloc" : " 14 GB", | 20260506 03:33:44.576 | INFO | "pinned" : " 29 GB" | 20260506 03:33:44.576 | INFO | } | 20260506 03:33:44.576 | INFO | Perf report for the last 5.14669 seconds: | 20260506 03:33:44.576 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:33:44.576 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:44.576 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:44.576 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:44.576 | INFO | ANALYSIS : 60.9%, 3.0, 95.3ms, 0 | 20260506 03:33:44.576 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:44.576 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:44.576 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 03:33:44.576 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:44.576 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:44.576 | INFO | Uploaded 36GiB to GPU in 2.11505s (17.0209GiB/s) | 20260506 03:33:44.576 | INFO | Downloaded 34GiB to GPU in 2.04278s (16.644GiB/s) | 20260506 03:33:49.713 | INFO | Analyzing chunk frames = [8621056,8621568) | 20260506 03:33:49.713 | INFO | MemInfo: { | 20260506 03:33:49.713 | INFO | "gpu" : " 17 GB", | 20260506 03:33:49.713 | INFO | "huge" : "", | 20260506 03:33:49.713 | INFO | "huge_shared" : " 27 GB", | 20260506 03:33:49.713 | INFO | "malloc" : " 14 GB", | 20260506 03:33:49.713 | INFO | "pinned" : " 29 GB" | 20260506 03:33:49.713 | INFO | } | 20260506 03:33:49.713 | INFO | Perf report for the last 5.13676 seconds: | 20260506 03:33:49.713 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:33:49.713 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:49.713 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:49.713 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:49.713 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260506 03:33:49.713 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:49.713 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:49.713 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 03:33:49.713 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:49.713 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:49.713 | INFO | Uploaded 36GiB to GPU in 2.07387s (17.3588GiB/s) | 20260506 03:33:49.713 | INFO | Downloaded 34GiB to GPU in 2.06987s (16.4262GiB/s) | 20260506 03:33:54.846 | INFO | Analyzing chunk frames = [8621568,8622080) | 20260506 03:33:54.846 | INFO | MemInfo: { | 20260506 03:33:54.846 | INFO | "gpu" : " 17 GB", | 20260506 03:33:54.846 | INFO | "huge" : "", | 20260506 03:33:54.846 | INFO | "huge_shared" : " 27 GB", | 20260506 03:33:54.846 | INFO | "malloc" : " 14 GB", | 20260506 03:33:54.846 | INFO | "pinned" : " 29 GB" | 20260506 03:33:54.846 | INFO | } | 20260506 03:33:54.846 | INFO | Perf report for the last 5.13336 seconds: | 20260506 03:33:54.846 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:33:54.846 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:54.846 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:54.846 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:54.846 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 03:33:54.846 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:54.846 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:54.846 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 03:33:54.846 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:54.846 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:54.846 | INFO | Uploaded 36GiB to GPU in 2.06133s (17.4645GiB/s) | 20260506 03:33:54.846 | INFO | Downloaded 34GiB to GPU in 2.0375s (16.6871GiB/s) | 20260506 03:33:54.971 | INFO | Starting a round of Baz IO | 20260506 03:33:58.095 | INFO | Populating RTMetrics report | 20260506 03:33:58.095 | INFO | RTMetrics report populated | 20260506 03:33:58.095 | INFO | Opening tmp RTMetrics json file | 20260506 03:33:58.096 | INFO | Serializing RTMetrics to json | 20260506 03:33:58.097 | INFO | Serialization done, swapping tmp file now... | 20260506 03:33:58.097 | INFO | RTMetrics done | 20260506 03:33:58.169 | INFO | Finished a round of Baz IO: Wrote 1.755 GiB in 3.198 seconds | 20260506 03:33:58.170 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.22% / 0.00% / 78.78% | 20260506 03:33:58.170 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0600497% / 0% / 0.000115236% | 20260506 03:33:58.170 | INFO | Baz IO overall: | 20260506 03:33:58.170 | INFO | BytesWritten : 4792764395508 | 20260506 03:33:58.170 | INFO | NumEvents : 2247203426272 | 20260506 03:33:59.980 | INFO | Analyzing chunk frames = [8622080,8622592) | 20260506 03:33:59.980 | INFO | MemInfo: { | 20260506 03:33:59.980 | INFO | "gpu" : " 17 GB", | 20260506 03:33:59.980 | INFO | "huge" : "", | 20260506 03:33:59.980 | INFO | "huge_shared" : " 27 GB", | 20260506 03:33:59.980 | INFO | "malloc" : " 14 GB", | 20260506 03:33:59.980 | INFO | "pinned" : " 29 GB" | 20260506 03:33:59.980 | INFO | } | 20260506 03:33:59.980 | INFO | Perf report for the last 5.13393 seconds: | 20260506 03:33:59.980 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:33:59.980 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:59.980 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:59.980 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:59.980 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260506 03:33:59.980 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:33:59.980 | INFO | RT_METRICS : 42.6%, 1.0, 22.3ms, 0 | 20260506 03:33:59.980 | INFO | PRE_HQ : 59.2%, 1.7, 53.1ms, 0 | 20260506 03:33:59.980 | INFO | BAZWRITER : 61.4%, 3.4, 107.8ms, 0 | 20260506 03:33:59.980 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 03:33:59.980 | INFO | Uploaded 36GiB to GPU in 2.10769s (17.0803GiB/s) | 20260506 03:33:59.981 | INFO | Downloaded 38GiB to GPU in 2.22405s (17.0859GiB/s) | 20260506 03:34:05.120 | INFO | Analyzing chunk frames = [8622592,8623104) | 20260506 03:34:05.120 | INFO | MemInfo: { | 20260506 03:34:05.120 | INFO | "gpu" : " 17 GB", | 20260506 03:34:05.121 | INFO | "huge" : "", | 20260506 03:34:05.121 | INFO | "huge_shared" : " 27 GB", | 20260506 03:34:05.121 | INFO | "malloc" : " 14 GB", | 20260506 03:34:05.121 | INFO | "pinned" : " 29 GB" | 20260506 03:34:05.121 | INFO | } | 20260506 03:34:05.121 | INFO | Perf report for the last 5.14049 seconds: | 20260506 03:34:05.121 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:34:05.121 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:05.121 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:05.121 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:05.121 | INFO | ANALYSIS : 60.7%, 3.0, 94.8ms, 0 | 20260506 03:34:05.121 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:05.121 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:05.121 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 03:34:05.121 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:05.121 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:05.121 | INFO | Uploaded 36GiB to GPU in 2.0096s (17.914GiB/s) | 20260506 03:34:05.121 | INFO | Downloaded 34GiB to GPU in 1.98744s (17.1074GiB/s) | 20260506 03:34:10.261 | INFO | Analyzing chunk frames = [8623104,8623616) | 20260506 03:34:10.261 | INFO | MemInfo: { | 20260506 03:34:10.261 | INFO | "gpu" : " 17 GB", | 20260506 03:34:10.261 | INFO | "huge" : "", | 20260506 03:34:10.261 | INFO | "huge_shared" : " 27 GB", | 20260506 03:34:10.261 | INFO | "malloc" : " 14 GB", | 20260506 03:34:10.261 | INFO | "pinned" : " 29 GB" | 20260506 03:34:10.261 | INFO | } | 20260506 03:34:10.261 | INFO | Perf report for the last 5.14049 seconds: | 20260506 03:34:10.261 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:34:10.261 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:10.261 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:10.261 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:10.261 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260506 03:34:10.261 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:10.261 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:10.261 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 03:34:10.261 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:10.261 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:10.261 | INFO | Uploaded 36GiB to GPU in 2.10567s (17.0967GiB/s) | 20260506 03:34:10.261 | INFO | Downloaded 34GiB to GPU in 2.04574s (16.6199GiB/s) | 20260506 03:34:15.409 | INFO | Analyzing chunk frames = [8623616,8624128) | 20260506 03:34:15.409 | INFO | MemInfo: { | 20260506 03:34:15.409 | INFO | "gpu" : " 17 GB", | 20260506 03:34:15.409 | INFO | "huge" : "", | 20260506 03:34:15.409 | INFO | "huge_shared" : " 27 GB", | 20260506 03:34:15.409 | INFO | "malloc" : " 14 GB", | 20260506 03:34:15.409 | INFO | "pinned" : " 29 GB" | 20260506 03:34:15.409 | INFO | } | 20260506 03:34:15.409 | INFO | Perf report for the last 5.14813 seconds: | 20260506 03:34:15.409 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:34:15.409 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:15.409 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:15.409 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:15.409 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260506 03:34:15.409 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:15.409 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:15.409 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 03:34:15.409 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:15.409 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:15.409 | INFO | Uploaded 36GiB to GPU in 2.08612s (17.2569GiB/s) | 20260506 03:34:15.409 | INFO | Downloaded 34GiB to GPU in 2.01497s (16.8737GiB/s) | 20260506 03:34:20.546 | INFO | Analyzing chunk frames = [8624128,8624640) | 20260506 03:34:20.546 | INFO | MemInfo: { | 20260506 03:34:20.546 | INFO | "gpu" : " 17 GB", | 20260506 03:34:20.546 | INFO | "huge" : "", | 20260506 03:34:20.546 | INFO | "huge_shared" : " 27 GB", | 20260506 03:34:20.546 | INFO | "malloc" : " 14 GB", | 20260506 03:34:20.546 | INFO | "pinned" : " 29 GB" | 20260506 03:34:20.546 | INFO | } | 20260506 03:34:20.546 | INFO | Perf report for the last 5.13711 seconds: | 20260506 03:34:20.546 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:34:20.546 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:20.546 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:20.546 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:20.546 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 03:34:20.546 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:20.546 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:20.546 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:34:20.546 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:20.547 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:20.547 | INFO | Uploaded 36GiB to GPU in 2.10999s (17.0617GiB/s) | 20260506 03:34:20.547 | INFO | Downloaded 34GiB to GPU in 2.03672s (16.6935GiB/s) | 20260506 03:34:25.683 | INFO | Analyzing chunk frames = [8624640,8625152) | 20260506 03:34:25.683 | INFO | MemInfo: { | 20260506 03:34:25.683 | INFO | "gpu" : " 17 GB", | 20260506 03:34:25.683 | INFO | "huge" : "", | 20260506 03:34:25.683 | INFO | "huge_shared" : " 27 GB", | 20260506 03:34:25.683 | INFO | "malloc" : " 14 GB", | 20260506 03:34:25.683 | INFO | "pinned" : " 29 GB" | 20260506 03:34:25.683 | INFO | } | 20260506 03:34:25.683 | INFO | Perf report for the last 5.13632 seconds: | 20260506 03:34:25.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:34:25.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:25.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:25.683 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:25.683 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260506 03:34:25.683 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:25.683 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:25.683 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 03:34:25.683 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:25.683 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:25.683 | INFO | Uploaded 36GiB to GPU in 2.03745s (17.6691GiB/s) | 20260506 03:34:25.683 | INFO | Downloaded 34GiB to GPU in 2.02853s (16.7609GiB/s) | 20260506 03:34:30.819 | INFO | Analyzing chunk frames = [8625152,8625664) | 20260506 03:34:30.819 | INFO | MemInfo: { | 20260506 03:34:30.819 | INFO | "gpu" : " 17 GB", | 20260506 03:34:30.819 | INFO | "huge" : "", | 20260506 03:34:30.819 | INFO | "huge_shared" : " 27 GB", | 20260506 03:34:30.819 | INFO | "malloc" : " 14 GB", | 20260506 03:34:30.819 | INFO | "pinned" : " 29 GB" | 20260506 03:34:30.819 | INFO | } | 20260506 03:34:30.819 | INFO | Perf report for the last 5.13649 seconds: | 20260506 03:34:30.819 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:34:30.819 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:30.819 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:30.819 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:30.820 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260506 03:34:30.820 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:30.820 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:30.820 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 03:34:30.820 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:30.820 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:30.820 | INFO | Uploaded 36GiB to GPU in 2.08909s (17.2324GiB/s) | 20260506 03:34:30.820 | INFO | Downloaded 34GiB to GPU in 2.08864s (16.2785GiB/s) | 20260506 03:34:35.956 | INFO | Analyzing chunk frames = [8625664,8626176) | 20260506 03:34:35.956 | INFO | MemInfo: { | 20260506 03:34:35.956 | INFO | "gpu" : " 17 GB", | 20260506 03:34:35.956 | INFO | "huge" : "", | 20260506 03:34:35.956 | INFO | "huge_shared" : " 27 GB", | 20260506 03:34:35.956 | INFO | "malloc" : " 14 GB", | 20260506 03:34:35.956 | INFO | "pinned" : " 29 GB" | 20260506 03:34:35.956 | INFO | } | 20260506 03:34:35.956 | INFO | Perf report for the last 5.13661 seconds: | 20260506 03:34:35.956 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:34:35.956 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:35.956 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:35.956 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:35.956 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260506 03:34:35.956 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:35.956 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:35.956 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 03:34:35.956 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:35.956 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:35.956 | INFO | Uploaded 36GiB to GPU in 2.05184s (17.5452GiB/s) | 20260506 03:34:35.956 | INFO | Downloaded 34GiB to GPU in 2.03248s (16.7283GiB/s) | 20260506 03:34:35.956 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 03:34:35.956 | INFO | Total mem usage: 15.4 GiB | 20260506 03:34:35.956 | INFO | All Time High Water Marks sum: 16.3 GiB (Savings: 5.2%) | 20260506 03:34:35.956 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -10%) | 20260506 03:34:35.956 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 03:34:35.956 | INFO | High Water Breakdown: | 20260506 03:34:35.956 | INFO | 60.9% : 8.54 GiB ( 88.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 03:34:35.956 | INFO | 39% : 5.47 GiB ( 82.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 03:34:35.956 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 03:34:35.956 | INFO | ------------------------------------------ | 20260506 03:34:35.956 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:34:35.957 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 03:34:35.957 | INFO | Total mem usage: 26.5 GiB | 20260506 03:34:35.957 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 03:34:35.957 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 03:34:35.957 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 03:34:35.957 | INFO | High Water Breakdown: | 20260506 03:34:35.957 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 03:34:35.957 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 03:34:35.957 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 03:34:35.957 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 03:34:35.957 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 03:34:35.957 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 03:34:35.957 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 03:34:35.957 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 03:34:35.957 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 03:34:35.957 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 03:34:35.957 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 03:34:35.957 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 03:34:35.957 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 03:34:35.957 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 03:34:35.957 | INFO | ------------------------------------------ | 20260506 03:34:35.957 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:34:35.957 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 03:34:35.957 | INFO | Total mem usage: 25.6 GiB | 20260506 03:34:35.957 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 03:34:35.957 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 03:34:35.957 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 03:34:35.957 | INFO | High Water Breakdown: | 20260506 03:34:35.957 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 03:34:35.957 | INFO | ------------------------------------------ | 20260506 03:34:35.957 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:34:35.957 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 03:34:35.957 | INFO | Total mem usage: 15.7 GiB | 20260506 03:34:35.957 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 03:34:35.957 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 03:34:35.957 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 03:34:35.957 | INFO | High Water Breakdown: | 20260506 03:34:35.957 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 03:34:35.957 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 03:34:35.957 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 03:34:35.957 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 03:34:35.957 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 03:34:35.957 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 03:34:35.957 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 03:34:35.957 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 03:34:35.957 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 03:34:35.957 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 03:34:35.957 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 03:34:35.957 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 03:34:35.957 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 03:34:35.957 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 03:34:35.957 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 03:34:35.957 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 03:34:35.957 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 03:34:35.957 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 03:34:35.957 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 03:34:35.957 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 03:34:35.957 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 03:34:35.957 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 03:34:35.957 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 03:34:35.957 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 03:34:35.957 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 03:34:35.957 | INFO | ------------------------------------------ | 20260506 03:34:35.957 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:34:35.957 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 03:34:35.957 | INFO | PerfReport: Gathered 1536 samples. | 20260506 03:34:35.957 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 03:34:35.957 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.36e+01, 1.33e+01 / 7.40e-01 | 20260506 03:34:35.957 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.04e-01, 1.10e-02 / 2.92e-03 | 20260506 03:34:35.958 | INFO | PerfReport: 11.90%, Baseline -- 3.72e+00 / 4.55e+00, 5.94e-01 / 2.46e-01 | 20260506 03:34:35.958 | INFO | PerfReport: 3.32%, Histogram -- 1.04e+00 / 1.55e+00, 2.00e-01 / 7.55e-02 | 20260506 03:34:35.958 | INFO | PerfReport: 16.00%, DME -- 4.00e+01 / 4.39e+01, 3.93e+00 / 9.53e-01 | 20260506 03:34:35.958 | INFO | PerfReport: 57.51%, FrameLabeling -- 1.80e+01 / 1.91e+01, 1.52e+00 / 1.04e+00 | 20260506 03:34:35.958 | INFO | PerfReport: 6.98%, PulseAccumulating -- 2.18e+00 / 2.76e+00, 4.15e-01 / 1.73e-01 | 20260506 03:34:35.958 | INFO | PerfReport: 2.97%, Metrics -- 9.28e-01 / 1.20e+00, 2.48e-01 / 1.11e-01 | 20260506 03:34:35.958 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.20e-01 / 3.69e-01, 5.37e-02 / 2.55e-02 | 20260506 03:34:35.958 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 03:34:35.958 | INFO | PerfReport: Gathered 1536 samples. | 20260506 03:34:35.958 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 03:34:35.958 | INFO | PerfReport: Full Execution -- 9.74e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260506 03:34:35.958 | INFO | PerfReport: 22.11%, Upload -- 2.15e+01 / 0.00e+00, 7.06e+00 / 0.00e+00 | 20260506 03:34:35.958 | INFO | PerfReport: 21.96%, Download -- 2.14e+01 / 0.00e+00, 7.57e+00 / 0.00e+00 | 20260506 03:34:35.958 | INFO | PerfReport: 32.10%, Compute -- 3.12e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 03:34:35.958 | INFO | PerfReport: 1.98%, LockUpload -- 1.93e+00 / 0.00e+00, 7.76e+00 / 0.00e+00 | 20260506 03:34:35.958 | INFO | PerfReport: 1.31%, LockDownload -- 1.28e+00 / 0.00e+00, 2.96e+00 / 0.00e+00 | 20260506 03:34:35.958 | INFO | PerfReport: 20.52%, LockCompute -- 2.00e+01 / 0.00e+00, 1.74e+01 / 0.00e+00 | 20260506 03:34:35.958 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 2.22e-02 / 0.00e+00 | 20260506 03:34:36.082 | INFO | Starting a round of Baz IO | 20260506 03:34:39.204 | INFO | Populating RTMetrics report | 20260506 03:34:39.204 | INFO | RTMetrics report populated | 20260506 03:34:39.204 | INFO | Opening tmp RTMetrics json file | 20260506 03:34:39.205 | INFO | Serializing RTMetrics to json | 20260506 03:34:39.207 | INFO | Serialization done, swapping tmp file now... | 20260506 03:34:39.208 | INFO | RTMetrics done | 20260506 03:34:39.273 | INFO | Finished a round of Baz IO: Wrote 1.755 GiB in 3.19 seconds | 20260506 03:34:39.273 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.22% / 0.00% / 78.78% | 20260506 03:34:39.273 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0598828% / 3.97364e-06% / 6.75519e-05% | 20260506 03:34:39.273 | INFO | Baz IO overall: | 20260506 03:34:39.273 | INFO | BytesWritten : 4794649131615 | 20260506 03:34:39.273 | INFO | NumEvents : 2248056955608 | 20260506 03:34:41.094 | INFO | Analyzing chunk frames = [8626176,8626688) | 20260506 03:34:41.094 | INFO | MemInfo: { | 20260506 03:34:41.094 | INFO | "gpu" : " 17 GB", | 20260506 03:34:41.094 | INFO | "huge" : "", | 20260506 03:34:41.094 | INFO | "huge_shared" : " 27 GB", | 20260506 03:34:41.094 | INFO | "malloc" : " 14 GB", | 20260506 03:34:41.094 | INFO | "pinned" : " 29 GB" | 20260506 03:34:41.094 | INFO | } | 20260506 03:34:41.094 | INFO | Perf report for the last 5.13846 seconds: | 20260506 03:34:41.094 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:34:41.094 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:41.094 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:41.094 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:41.094 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260506 03:34:41.094 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:41.094 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260506 03:34:41.094 | INFO | PRE_HQ : 58.9%, 1.7, 53.0ms, 0 | 20260506 03:34:41.094 | INFO | BAZWRITER : 61.2%, 3.4, 106.8ms, 0 | 20260506 03:34:41.095 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 03:34:41.095 | INFO | Uploaded 36GiB to GPU in 2.1109s (17.0543GiB/s) | 20260506 03:34:41.095 | INFO | Downloaded 38GiB to GPU in 2.2248s (17.0802GiB/s) | 20260506 03:34:46.235 | INFO | Analyzing chunk frames = [8626688,8627200) | 20260506 03:34:46.235 | INFO | MemInfo: { | 20260506 03:34:46.235 | INFO | "gpu" : " 17 GB", | 20260506 03:34:46.235 | INFO | "huge" : "", | 20260506 03:34:46.235 | INFO | "huge_shared" : " 27 GB", | 20260506 03:34:46.235 | INFO | "malloc" : " 14 GB", | 20260506 03:34:46.235 | INFO | "pinned" : " 29 GB" | 20260506 03:34:46.235 | INFO | } | 20260506 03:34:46.235 | INFO | Perf report for the last 5.14107 seconds: | 20260506 03:34:46.235 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:34:46.235 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:46.235 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:46.235 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:46.236 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260506 03:34:46.236 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:46.236 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:46.236 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 03:34:46.236 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:46.236 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:46.236 | INFO | Uploaded 36GiB to GPU in 1.96316s (18.3378GiB/s) | 20260506 03:34:46.236 | INFO | Downloaded 34GiB to GPU in 1.98185s (17.1557GiB/s) | 20260506 03:34:51.378 | INFO | Analyzing chunk frames = [8627200,8627712) | 20260506 03:34:51.378 | INFO | MemInfo: { | 20260506 03:34:51.378 | INFO | "gpu" : " 17 GB", | 20260506 03:34:51.378 | INFO | "huge" : "", | 20260506 03:34:51.378 | INFO | "huge_shared" : " 27 GB", | 20260506 03:34:51.378 | INFO | "malloc" : " 14 GB", | 20260506 03:34:51.378 | INFO | "pinned" : " 29 GB" | 20260506 03:34:51.378 | INFO | } | 20260506 03:34:51.378 | INFO | Perf report for the last 5.14224 seconds: | 20260506 03:34:51.378 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:34:51.378 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:51.378 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:51.378 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:51.378 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260506 03:34:51.378 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:51.378 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:51.378 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 03:34:51.378 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:51.378 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:51.378 | INFO | Uploaded 36GiB to GPU in 2.08693s (17.2503GiB/s) | 20260506 03:34:51.378 | INFO | Downloaded 34GiB to GPU in 2.02494s (16.7906GiB/s) | 20260506 03:34:56.523 | INFO | Analyzing chunk frames = [8627712,8628224) | 20260506 03:34:56.523 | INFO | MemInfo: { | 20260506 03:34:56.523 | INFO | "gpu" : " 17 GB", | 20260506 03:34:56.523 | INFO | "huge" : "", | 20260506 03:34:56.523 | INFO | "huge_shared" : " 27 GB", | 20260506 03:34:56.523 | INFO | "malloc" : " 14 GB", | 20260506 03:34:56.523 | INFO | "pinned" : " 29 GB" | 20260506 03:34:56.523 | INFO | } | 20260506 03:34:56.523 | INFO | Perf report for the last 5.14516 seconds: | 20260506 03:34:56.523 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:34:56.523 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:56.523 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:56.523 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:56.523 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260506 03:34:56.523 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:56.523 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:56.523 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 03:34:56.523 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:56.523 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:34:56.523 | INFO | Uploaded 36GiB to GPU in 2.08316s (17.2814GiB/s) | 20260506 03:34:56.523 | INFO | Downloaded 34GiB to GPU in 2.00767s (16.9351GiB/s) | 20260506 03:35:01.668 | INFO | Analyzing chunk frames = [8628224,8628736) | 20260506 03:35:01.668 | INFO | MemInfo: { | 20260506 03:35:01.668 | INFO | "gpu" : " 17 GB", | 20260506 03:35:01.668 | INFO | "huge" : "", | 20260506 03:35:01.668 | INFO | "huge_shared" : " 27 GB", | 20260506 03:35:01.668 | INFO | "malloc" : " 14 GB", | 20260506 03:35:01.668 | INFO | "pinned" : " 29 GB" | 20260506 03:35:01.668 | INFO | } | 20260506 03:35:01.668 | INFO | Perf report for the last 5.145 seconds: | 20260506 03:35:01.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:35:01.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:01.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:01.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:01.668 | INFO | ANALYSIS : 60.7%, 3.0, 94.9ms, 0 | 20260506 03:35:01.668 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:01.668 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:01.668 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 03:35:01.668 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:01.668 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:01.668 | INFO | Uploaded 36GiB to GPU in 2.12182s (16.9666GiB/s) | 20260506 03:35:01.668 | INFO | Downloaded 34GiB to GPU in 2.04369s (16.6366GiB/s) | 20260506 03:35:06.798 | INFO | Analyzing chunk frames = [8628736,8629248) | 20260506 03:35:06.798 | INFO | MemInfo: { | 20260506 03:35:06.798 | INFO | "gpu" : " 17 GB", | 20260506 03:35:06.798 | INFO | "huge" : "", | 20260506 03:35:06.798 | INFO | "huge_shared" : " 27 GB", | 20260506 03:35:06.798 | INFO | "malloc" : " 14 GB", | 20260506 03:35:06.798 | INFO | "pinned" : " 29 GB" | 20260506 03:35:06.798 | INFO | } | 20260506 03:35:06.798 | INFO | Perf report for the last 5.13007 seconds: | 20260506 03:35:06.798 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:35:06.798 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:06.798 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:06.798 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:06.798 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260506 03:35:06.798 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:06.798 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:06.798 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:35:06.798 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:06.798 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:06.798 | INFO | Uploaded 36GiB to GPU in 2.09459s (17.1871GiB/s) | 20260506 03:35:06.798 | INFO | Downloaded 34GiB to GPU in 2.03818s (16.6816GiB/s) | 20260506 03:35:11.928 | INFO | Analyzing chunk frames = [8629248,8629760) | 20260506 03:35:11.928 | INFO | MemInfo: { | 20260506 03:35:11.928 | INFO | "gpu" : " 17 GB", | 20260506 03:35:11.928 | INFO | "huge" : "", | 20260506 03:35:11.928 | INFO | "huge_shared" : " 27 GB", | 20260506 03:35:11.928 | INFO | "malloc" : " 14 GB", | 20260506 03:35:11.928 | INFO | "pinned" : " 29 GB" | 20260506 03:35:11.928 | INFO | } | 20260506 03:35:11.928 | INFO | Perf report for the last 5.1306 seconds: | 20260506 03:35:11.929 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:35:11.929 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:11.929 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:11.929 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:11.929 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260506 03:35:11.929 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:11.929 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:11.929 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 03:35:11.929 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:11.929 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:11.929 | INFO | Uploaded 36GiB to GPU in 2.12838s (16.9142GiB/s) | 20260506 03:35:11.929 | INFO | Downloaded 34GiB to GPU in 2.09246s (16.2488GiB/s) | 20260506 03:35:17.073 | INFO | Analyzing chunk frames = [8629760,8630272) | 20260506 03:35:17.073 | INFO | MemInfo: { | 20260506 03:35:17.073 | INFO | "gpu" : " 17 GB", | 20260506 03:35:17.073 | INFO | "huge" : "", | 20260506 03:35:17.073 | INFO | "huge_shared" : " 27 GB", | 20260506 03:35:17.074 | INFO | "malloc" : " 14 GB", | 20260506 03:35:17.074 | INFO | "pinned" : " 29 GB" | 20260506 03:35:17.074 | INFO | } | 20260506 03:35:17.074 | INFO | Perf report for the last 5.14529 seconds: | 20260506 03:35:17.074 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:35:17.074 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:17.074 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:17.074 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:17.074 | INFO | ANALYSIS : 61.3%, 3.0, 95.8ms, 0 | 20260506 03:35:17.074 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:17.074 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:17.074 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 03:35:17.074 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:17.074 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:17.074 | INFO | Uploaded 36GiB to GPU in 2.06537s (17.4303GiB/s) | 20260506 03:35:17.074 | INFO | Downloaded 34GiB to GPU in 2.02772s (16.7676GiB/s) | 20260506 03:35:17.200 | INFO | Starting a round of Baz IO | 20260506 03:35:20.326 | INFO | Populating RTMetrics report | 20260506 03:35:20.326 | INFO | RTMetrics report populated | 20260506 03:35:20.326 | INFO | Opening tmp RTMetrics json file | 20260506 03:35:20.327 | INFO | Serializing RTMetrics to json | 20260506 03:35:20.328 | INFO | Serialization done, swapping tmp file now... | 20260506 03:35:20.329 | INFO | RTMetrics done | 20260506 03:35:20.403 | INFO | Finished a round of Baz IO: Wrote 1.756 GiB in 3.203 seconds | 20260506 03:35:20.403 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.22% / 0.00% / 78.78% | 20260506 03:35:20.403 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0599941% / 0% / 7.54992e-05% | 20260506 03:35:20.403 | INFO | Baz IO overall: | 20260506 03:35:20.403 | INFO | BytesWritten : 4796535147331 | 20260506 03:35:20.403 | INFO | NumEvents : 2248911681234 | 20260506 03:35:22.215 | INFO | Analyzing chunk frames = [8630272,8630784) | 20260506 03:35:22.215 | INFO | MemInfo: { | 20260506 03:35:22.215 | INFO | "gpu" : " 17 GB", | 20260506 03:35:22.215 | INFO | "huge" : "", | 20260506 03:35:22.215 | INFO | "huge_shared" : " 27 GB", | 20260506 03:35:22.215 | INFO | "malloc" : " 14 GB", | 20260506 03:35:22.215 | INFO | "pinned" : " 29 GB" | 20260506 03:35:22.215 | INFO | } | 20260506 03:35:22.215 | INFO | Perf report for the last 5.14114 seconds: | 20260506 03:35:22.215 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:35:22.215 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:22.215 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:22.215 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:22.215 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260506 03:35:22.215 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:22.215 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260506 03:35:22.215 | INFO | PRE_HQ : 59.1%, 1.7, 53.1ms, 0 | 20260506 03:35:22.215 | INFO | BAZWRITER : 61.5%, 3.4, 107.7ms, 0 | 20260506 03:35:22.215 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 03:35:22.215 | INFO | Uploaded 36GiB to GPU in 2.10784s (17.0791GiB/s) | 20260506 03:35:22.215 | INFO | Downloaded 38GiB to GPU in 2.2275s (17.0595GiB/s) | 20260506 03:35:27.354 | INFO | Analyzing chunk frames = [8630784,8631296) | 20260506 03:35:27.354 | INFO | MemInfo: { | 20260506 03:35:27.354 | INFO | "gpu" : " 17 GB", | 20260506 03:35:27.354 | INFO | "huge" : "", | 20260506 03:35:27.354 | INFO | "huge_shared" : " 27 GB", | 20260506 03:35:27.354 | INFO | "malloc" : " 14 GB", | 20260506 03:35:27.354 | INFO | "pinned" : " 29 GB" | 20260506 03:35:27.354 | INFO | } | 20260506 03:35:27.354 | INFO | Perf report for the last 5.13919 seconds: | 20260506 03:35:27.354 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:35:27.354 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:27.354 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:27.354 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:27.354 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260506 03:35:27.354 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:27.354 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:27.354 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 03:35:27.354 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:27.354 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:27.355 | INFO | Uploaded 36GiB to GPU in 2.00752s (17.9326GiB/s) | 20260506 03:35:27.355 | INFO | Downloaded 34GiB to GPU in 1.98942s (17.0904GiB/s) | 20260506 03:35:32.494 | INFO | Analyzing chunk frames = [8631296,8631808) | 20260506 03:35:32.494 | INFO | MemInfo: { | 20260506 03:35:32.494 | INFO | "gpu" : " 17 GB", | 20260506 03:35:32.494 | INFO | "huge" : "", | 20260506 03:35:32.494 | INFO | "huge_shared" : " 27 GB", | 20260506 03:35:32.494 | INFO | "malloc" : " 14 GB", | 20260506 03:35:32.494 | INFO | "pinned" : " 29 GB" | 20260506 03:35:32.494 | INFO | } | 20260506 03:35:32.494 | INFO | Perf report for the last 5.14023 seconds: | 20260506 03:35:32.494 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:35:32.494 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:32.494 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:32.494 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:32.494 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260506 03:35:32.494 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:32.494 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:32.494 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 03:35:32.494 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:32.494 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:32.495 | INFO | Uploaded 36GiB to GPU in 2.09089s (17.2176GiB/s) | 20260506 03:35:32.495 | INFO | Downloaded 34GiB to GPU in 2.04414s (16.6329GiB/s) | 20260506 03:35:37.634 | INFO | Analyzing chunk frames = [8631808,8632320) | 20260506 03:35:37.634 | INFO | MemInfo: { | 20260506 03:35:37.634 | INFO | "gpu" : " 17 GB", | 20260506 03:35:37.634 | INFO | "huge" : "", | 20260506 03:35:37.634 | INFO | "huge_shared" : " 27 GB", | 20260506 03:35:37.634 | INFO | "malloc" : " 14 GB", | 20260506 03:35:37.634 | INFO | "pinned" : " 29 GB" | 20260506 03:35:37.634 | INFO | } | 20260506 03:35:37.634 | INFO | Perf report for the last 5.13959 seconds: | 20260506 03:35:37.634 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:35:37.634 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:37.634 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:37.634 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:37.634 | INFO | ANALYSIS : 61.1%, 3.0, 94.9ms, 0 | 20260506 03:35:37.634 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:37.634 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:37.634 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 03:35:37.634 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:37.634 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:37.634 | INFO | Uploaded 36GiB to GPU in 2.09511s (17.1829GiB/s) | 20260506 03:35:37.634 | INFO | Downloaded 34GiB to GPU in 2.02612s (16.7808GiB/s) | 20260506 03:35:42.770 | INFO | Analyzing chunk frames = [8632320,8632832) | 20260506 03:35:42.770 | INFO | MemInfo: { | 20260506 03:35:42.770 | INFO | "gpu" : " 17 GB", | 20260506 03:35:42.770 | INFO | "huge" : "", | 20260506 03:35:42.770 | INFO | "huge_shared" : " 27 GB", | 20260506 03:35:42.770 | INFO | "malloc" : " 14 GB", | 20260506 03:35:42.770 | INFO | "pinned" : " 29 GB" | 20260506 03:35:42.770 | INFO | } | 20260506 03:35:42.770 | INFO | Perf report for the last 5.13606 seconds: | 20260506 03:35:42.770 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:35:42.770 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:42.770 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:42.770 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:42.770 | INFO | ANALYSIS : 60.8%, 3.0, 94.9ms, 0 | 20260506 03:35:42.770 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:42.770 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:42.770 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 03:35:42.770 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:42.770 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:42.770 | INFO | Uploaded 36GiB to GPU in 2.12712s (16.9243GiB/s) | 20260506 03:35:42.771 | INFO | Downloaded 34GiB to GPU in 2.04235s (16.6475GiB/s) | 20260506 03:35:47.909 | INFO | Analyzing chunk frames = [8632832,8633344) | 20260506 03:35:47.909 | INFO | MemInfo: { | 20260506 03:35:47.909 | INFO | "gpu" : " 17 GB", | 20260506 03:35:47.909 | INFO | "huge" : "", | 20260506 03:35:47.909 | INFO | "huge_shared" : " 27 GB", | 20260506 03:35:47.909 | INFO | "malloc" : " 14 GB", | 20260506 03:35:47.909 | INFO | "pinned" : " 29 GB" | 20260506 03:35:47.909 | INFO | } | 20260506 03:35:47.909 | INFO | Perf report for the last 5.13857 seconds: | 20260506 03:35:47.909 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:35:47.909 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:47.909 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:47.909 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:47.909 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260506 03:35:47.909 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:47.909 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:47.909 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:35:47.909 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:47.909 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:47.909 | INFO | Uploaded 36GiB to GPU in 2.05207s (17.5433GiB/s) | 20260506 03:35:47.909 | INFO | Downloaded 34GiB to GPU in 2.02621s (16.7801GiB/s) | 20260506 03:35:53.046 | INFO | Analyzing chunk frames = [8633344,8633856) | 20260506 03:35:53.046 | INFO | MemInfo: { | 20260506 03:35:53.046 | INFO | "gpu" : " 17 GB", | 20260506 03:35:53.046 | INFO | "huge" : "", | 20260506 03:35:53.046 | INFO | "huge_shared" : " 27 GB", | 20260506 03:35:53.046 | INFO | "malloc" : " 14 GB", | 20260506 03:35:53.046 | INFO | "pinned" : " 29 GB" | 20260506 03:35:53.046 | INFO | } | 20260506 03:35:53.046 | INFO | Perf report for the last 5.13751 seconds: | 20260506 03:35:53.046 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:35:53.046 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:53.046 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:53.046 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:53.046 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260506 03:35:53.046 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:53.046 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:53.046 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 03:35:53.046 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:53.047 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:53.047 | INFO | Uploaded 36GiB to GPU in 2.06818s (17.4066GiB/s) | 20260506 03:35:53.047 | INFO | Downloaded 34GiB to GPU in 2.07631s (16.3752GiB/s) | 20260506 03:35:58.184 | INFO | Analyzing chunk frames = [8633856,8634368) | 20260506 03:35:58.184 | INFO | MemInfo: { | 20260506 03:35:58.184 | INFO | "gpu" : " 17 GB", | 20260506 03:35:58.184 | INFO | "huge" : "", | 20260506 03:35:58.184 | INFO | "huge_shared" : " 27 GB", | 20260506 03:35:58.184 | INFO | "malloc" : " 14 GB", | 20260506 03:35:58.184 | INFO | "pinned" : " 29 GB" | 20260506 03:35:58.184 | INFO | } | 20260506 03:35:58.184 | INFO | Perf report for the last 5.13815 seconds: | 20260506 03:35:58.184 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:35:58.184 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:58.184 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:58.185 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:58.185 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260506 03:35:58.185 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:58.185 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:58.185 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 03:35:58.185 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:58.185 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:35:58.185 | INFO | Uploaded 36GiB to GPU in 2.04171s (17.6323GiB/s) | 20260506 03:35:58.185 | INFO | Downloaded 34GiB to GPU in 2.01822s (16.8465GiB/s) | 20260506 03:35:58.185 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 03:35:58.185 | INFO | Total mem usage: 15.4 GiB | 20260506 03:35:58.185 | INFO | All Time High Water Marks sum: 16.3 GiB (Savings: 5.2%) | 20260506 03:35:58.185 | INFO | Recent High Water Marks Sum: 14 GiB (Savings: -10%) | 20260506 03:35:58.185 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 03:35:58.185 | INFO | High Water Breakdown: | 20260506 03:35:58.185 | INFO | 60.9% : 8.54 GiB ( 88.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 03:35:58.185 | INFO | 39% : 5.47 GiB ( 82.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 03:35:58.185 | INFO | 0.142% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 03:35:58.185 | INFO | ------------------------------------------ | 20260506 03:35:58.185 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:35:58.185 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 03:35:58.185 | INFO | Total mem usage: 26.5 GiB | 20260506 03:35:58.185 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 03:35:58.185 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 03:35:58.185 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 03:35:58.185 | INFO | High Water Breakdown: | 20260506 03:35:58.185 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 03:35:58.185 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 03:35:58.185 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 03:35:58.185 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 03:35:58.185 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 03:35:58.185 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 03:35:58.185 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 03:35:58.185 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 03:35:58.185 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 03:35:58.185 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 03:35:58.185 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 03:35:58.185 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 03:35:58.185 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 03:35:58.185 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 03:35:58.185 | INFO | ------------------------------------------ | 20260506 03:35:58.185 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:35:58.185 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 03:35:58.185 | INFO | Total mem usage: 25.6 GiB | 20260506 03:35:58.185 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 03:35:58.185 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 03:35:58.185 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 03:35:58.185 | INFO | High Water Breakdown: | 20260506 03:35:58.186 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 03:35:58.186 | INFO | ------------------------------------------ | 20260506 03:35:58.186 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:35:58.186 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 03:35:58.186 | INFO | Total mem usage: 15.7 GiB | 20260506 03:35:58.186 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 03:35:58.186 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 03:35:58.186 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 03:35:58.186 | INFO | High Water Breakdown: | 20260506 03:35:58.186 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 03:35:58.186 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 03:35:58.186 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 03:35:58.186 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 03:35:58.186 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 03:35:58.186 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 03:35:58.186 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 03:35:58.186 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 03:35:58.186 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 03:35:58.186 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 03:35:58.186 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 03:35:58.186 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 03:35:58.186 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 03:35:58.186 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 03:35:58.186 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 03:35:58.186 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 03:35:58.186 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 03:35:58.186 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 03:35:58.186 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 03:35:58.186 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 03:35:58.186 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 03:35:58.186 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 03:35:58.186 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 03:35:58.186 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 03:35:58.186 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 03:35:58.186 | INFO | ------------------------------------------ | 20260506 03:35:58.186 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:35:58.186 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 03:35:58.186 | INFO | PerfReport: Gathered 1536 samples. | 20260506 03:35:58.186 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 03:35:58.186 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.35e+01, 1.34e+01 / 7.80e-01 | 20260506 03:35:58.186 | INFO | PerfReport: 0.29%, AggregateStats -- 9.18e-02 / 1.05e-01, 1.04e-02 / 3.99e-03 | 20260506 03:35:58.186 | INFO | PerfReport: 11.88%, Baseline -- 3.71e+00 / 4.48e+00, 5.85e-01 / 2.66e-01 | 20260506 03:35:58.186 | INFO | PerfReport: 3.31%, Histogram -- 1.03e+00 / 1.53e+00, 1.96e-01 / 7.83e-02 | 20260506 03:35:58.186 | INFO | PerfReport: 15.99%, DME -- 4.00e+01 / 4.38e+01, 3.83e+00 / 8.61e-01 | 20260506 03:35:58.186 | INFO | PerfReport: 57.58%, FrameLabeling -- 1.80e+01 / 1.93e+01, 1.56e+00 / 1.00e+00 | 20260506 03:35:58.186 | INFO | PerfReport: 6.96%, PulseAccumulating -- 2.18e+00 / 2.70e+00, 4.12e-01 / 2.17e-01 | 20260506 03:35:58.186 | INFO | PerfReport: 2.97%, Metrics -- 9.27e-01 / 1.19e+00, 2.51e-01 / 1.16e-01 | 20260506 03:35:58.186 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.20e-01 / 3.67e-01, 5.44e-02 / 2.08e-02 | 20260506 03:35:58.186 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 03:35:58.186 | INFO | PerfReport: Gathered 1536 samples. | 20260506 03:35:58.186 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 03:35:58.186 | INFO | PerfReport: Full Execution -- 9.75e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260506 03:35:58.186 | INFO | PerfReport: 22.21%, Upload -- 2.16e+01 / 0.00e+00, 7.15e+00 / 0.00e+00 | 20260506 03:35:58.186 | INFO | PerfReport: 21.97%, Download -- 2.14e+01 / 0.00e+00, 7.61e+00 / 0.00e+00 | 20260506 03:35:58.186 | INFO | PerfReport: 32.06%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 03:35:58.186 | INFO | PerfReport: 2.02%, LockUpload -- 1.96e+00 / 0.00e+00, 7.94e+00 / 0.00e+00 | 20260506 03:35:58.186 | INFO | PerfReport: 1.35%, LockDownload -- 1.32e+00 / 0.00e+00, 3.06e+00 / 0.00e+00 | 20260506 03:35:58.186 | INFO | PerfReport: 20.38%, LockCompute -- 1.99e+01 / 0.00e+00, 1.75e+01 / 0.00e+00 | 20260506 03:35:58.186 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.60e-02 / 0.00e+00 | 20260506 03:35:58.310 | INFO | Starting a round of Baz IO | 20260506 03:36:01.429 | INFO | Populating RTMetrics report | 20260506 03:36:01.430 | INFO | RTMetrics report populated | 20260506 03:36:01.430 | INFO | Opening tmp RTMetrics json file | 20260506 03:36:01.431 | INFO | Serializing RTMetrics to json | 20260506 03:36:01.432 | INFO | Serialization done, swapping tmp file now... | 20260506 03:36:01.433 | INFO | RTMetrics done | 20260506 03:36:01.505 | INFO | Finished a round of Baz IO: Wrote 1.755 GiB in 3.195 seconds | 20260506 03:36:01.505 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.22% / 0.00% / 78.78% | 20260506 03:36:01.505 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.059545% / 0% / 9.93411e-05% | 20260506 03:36:01.505 | INFO | Baz IO overall: | 20260506 03:36:01.505 | INFO | BytesWritten : 4798419050234 | 20260506 03:36:01.505 | INFO | NumEvents : 2249764759625 | 20260506 03:36:03.332 | INFO | Analyzing chunk frames = [8634368,8634880) | 20260506 03:36:03.332 | INFO | MemInfo: { | 20260506 03:36:03.332 | INFO | "gpu" : " 17 GB", | 20260506 03:36:03.332 | INFO | "huge" : "", | 20260506 03:36:03.332 | INFO | "huge_shared" : " 27 GB", | 20260506 03:36:03.332 | INFO | "malloc" : " 14 GB", | 20260506 03:36:03.332 | INFO | "pinned" : " 29 GB" | 20260506 03:36:03.332 | INFO | } | 20260506 03:36:03.332 | INFO | Perf report for the last 5.14817 seconds: | 20260506 03:36:03.333 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:36:03.333 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:03.333 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:03.333 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:03.333 | INFO | ANALYSIS : 62.0%, 3.0, 97.0ms, 0 | 20260506 03:36:03.333 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:03.333 | INFO | RT_METRICS : 42.6%, 1.0, 22.4ms, 0 | 20260506 03:36:03.333 | INFO | PRE_HQ : 58.8%, 1.7, 53.2ms, 0 | 20260506 03:36:03.333 | INFO | BAZWRITER : 61.2%, 3.4, 108.6ms, 0 | 20260506 03:36:03.333 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 03:36:03.333 | INFO | Uploaded 36GiB to GPU in 2.10446s (17.1065GiB/s) | 20260506 03:36:03.333 | INFO | Downloaded 38GiB to GPU in 2.21589s (17.1489GiB/s) | 20260506 03:36:08.475 | INFO | Analyzing chunk frames = [8634880,8635392) | 20260506 03:36:08.475 | INFO | MemInfo: { | 20260506 03:36:08.475 | INFO | "gpu" : " 17 GB", | 20260506 03:36:08.475 | INFO | "huge" : "", | 20260506 03:36:08.475 | INFO | "huge_shared" : " 27 GB", | 20260506 03:36:08.475 | INFO | "malloc" : " 14 GB", | 20260506 03:36:08.475 | INFO | "pinned" : " 29 GB" | 20260506 03:36:08.475 | INFO | } | 20260506 03:36:08.475 | INFO | Perf report for the last 5.1422 seconds: | 20260506 03:36:08.475 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:36:08.475 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:08.475 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:08.475 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:08.475 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260506 03:36:08.475 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:08.475 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:08.475 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 03:36:08.475 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:08.475 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:08.475 | INFO | Uploaded 36GiB to GPU in 1.99704s (18.0267GiB/s) | 20260506 03:36:08.475 | INFO | Downloaded 34GiB to GPU in 1.98918s (17.0924GiB/s) | 20260506 03:36:13.610 | INFO | Analyzing chunk frames = [8635392,8635904) | 20260506 03:36:13.610 | INFO | MemInfo: { | 20260506 03:36:13.610 | INFO | "gpu" : " 17 GB", | 20260506 03:36:13.610 | INFO | "huge" : "", | 20260506 03:36:13.610 | INFO | "huge_shared" : " 27 GB", | 20260506 03:36:13.610 | INFO | "malloc" : " 14 GB", | 20260506 03:36:13.610 | INFO | "pinned" : " 29 GB" | 20260506 03:36:13.610 | INFO | } | 20260506 03:36:13.610 | INFO | Perf report for the last 5.13527 seconds: | 20260506 03:36:13.610 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:36:13.610 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:13.610 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:13.610 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:13.610 | INFO | ANALYSIS : 60.3%, 3.0, 93.8ms, 0 | 20260506 03:36:13.610 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:13.610 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:13.610 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 03:36:13.610 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:13.610 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:13.610 | INFO | Uploaded 36GiB to GPU in 1.99696s (18.0274GiB/s) | 20260506 03:36:13.610 | INFO | Downloaded 34GiB to GPU in 1.98044s (17.1679GiB/s) | 20260506 03:36:18.748 | INFO | Analyzing chunk frames = [8635904,8636416) | 20260506 03:36:18.748 | INFO | MemInfo: { | 20260506 03:36:18.748 | INFO | "gpu" : " 17 GB", | 20260506 03:36:18.748 | INFO | "huge" : "", | 20260506 03:36:18.748 | INFO | "huge_shared" : " 27 GB", | 20260506 03:36:18.748 | INFO | "malloc" : " 14 GB", | 20260506 03:36:18.748 | INFO | "pinned" : " 29 GB" | 20260506 03:36:18.748 | INFO | } | 20260506 03:36:18.748 | INFO | Perf report for the last 5.1378 seconds: | 20260506 03:36:18.748 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:36:18.748 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:18.748 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:18.748 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:18.748 | INFO | ANALYSIS : 61.0%, 3.0, 94.7ms, 0 | 20260506 03:36:18.748 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:18.748 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:18.748 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:36:18.748 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:18.748 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:18.748 | INFO | Uploaded 36GiB to GPU in 2.1031s (17.1176GiB/s) | 20260506 03:36:18.748 | INFO | Downloaded 34GiB to GPU in 2.00953s (16.9193GiB/s) | 20260506 03:36:23.883 | INFO | Analyzing chunk frames = [8636416,8636928) | 20260506 03:36:23.883 | INFO | MemInfo: { | 20260506 03:36:23.883 | INFO | "gpu" : " 17 GB", | 20260506 03:36:23.883 | INFO | "huge" : "", | 20260506 03:36:23.883 | INFO | "huge_shared" : " 27 GB", | 20260506 03:36:23.883 | INFO | "malloc" : " 14 GB", | 20260506 03:36:23.883 | INFO | "pinned" : " 29 GB" | 20260506 03:36:23.883 | INFO | } | 20260506 03:36:23.883 | INFO | Perf report for the last 5.13507 seconds: | 20260506 03:36:23.883 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:36:23.883 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:23.883 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:23.883 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:23.883 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 03:36:23.883 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:23.883 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:23.883 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 03:36:23.883 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:23.883 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:23.883 | INFO | Uploaded 36GiB to GPU in 2.10584s (17.0953GiB/s) | 20260506 03:36:23.883 | INFO | Downloaded 34GiB to GPU in 2.03124s (16.7385GiB/s) | 20260506 03:36:29.023 | INFO | Analyzing chunk frames = [8636928,8637440) | 20260506 03:36:29.023 | INFO | MemInfo: { | 20260506 03:36:29.023 | INFO | "gpu" : " 17 GB", | 20260506 03:36:29.023 | INFO | "huge" : "", | 20260506 03:36:29.023 | INFO | "huge_shared" : " 27 GB", | 20260506 03:36:29.023 | INFO | "malloc" : " 14 GB", | 20260506 03:36:29.023 | INFO | "pinned" : " 29 GB" | 20260506 03:36:29.023 | INFO | } | 20260506 03:36:29.023 | INFO | Perf report for the last 5.1406 seconds: | 20260506 03:36:29.023 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:36:29.023 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:29.024 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:29.024 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:29.024 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 03:36:29.024 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:29.024 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:29.024 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 03:36:29.024 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:29.024 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:29.024 | INFO | Uploaded 36GiB to GPU in 2.11144s (17.05GiB/s) | 20260506 03:36:29.024 | INFO | Downloaded 34GiB to GPU in 2.04988s (16.5863GiB/s) | 20260506 03:36:34.164 | INFO | Analyzing chunk frames = [8637440,8637952) | 20260506 03:36:34.164 | INFO | MemInfo: { | 20260506 03:36:34.164 | INFO | "gpu" : " 17 GB", | 20260506 03:36:34.164 | INFO | "huge" : "", | 20260506 03:36:34.164 | INFO | "huge_shared" : " 27 GB", | 20260506 03:36:34.164 | INFO | "malloc" : " 14 GB", | 20260506 03:36:34.164 | INFO | "pinned" : " 29 GB" | 20260506 03:36:34.164 | INFO | } | 20260506 03:36:34.164 | INFO | Perf report for the last 5.14057 seconds: | 20260506 03:36:34.164 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:36:34.164 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:34.164 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:34.164 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:34.164 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 03:36:34.164 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:34.164 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:34.164 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 03:36:34.164 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:34.164 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:34.164 | INFO | Uploaded 36GiB to GPU in 2.08273s (17.285GiB/s) | 20260506 03:36:34.165 | INFO | Downloaded 34GiB to GPU in 2.06199s (16.489GiB/s) | 20260506 03:36:39.303 | INFO | Analyzing chunk frames = [8637952,8638464) | 20260506 03:36:39.303 | INFO | MemInfo: { | 20260506 03:36:39.303 | INFO | "gpu" : " 17 GB", | 20260506 03:36:39.303 | INFO | "huge" : "", | 20260506 03:36:39.303 | INFO | "huge_shared" : " 27 GB", | 20260506 03:36:39.303 | INFO | "malloc" : " 14 GB", | 20260506 03:36:39.303 | INFO | "pinned" : " 29 GB" | 20260506 03:36:39.303 | INFO | } | 20260506 03:36:39.303 | INFO | Perf report for the last 5.13872 seconds: | 20260506 03:36:39.303 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:36:39.303 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:39.303 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:39.303 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:39.303 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260506 03:36:39.303 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:39.303 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:39.303 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 03:36:39.303 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:39.303 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:39.303 | INFO | Uploaded 36GiB to GPU in 2.06579s (17.4268GiB/s) | 20260506 03:36:39.303 | INFO | Downloaded 34GiB to GPU in 2.03015s (16.7475GiB/s) | 20260506 03:36:39.428 | INFO | Starting a round of Baz IO | 20260506 03:36:42.554 | INFO | Populating RTMetrics report | 20260506 03:36:42.555 | INFO | RTMetrics report populated | 20260506 03:36:42.555 | INFO | Opening tmp RTMetrics json file | 20260506 03:36:42.555 | INFO | Serializing RTMetrics to json | 20260506 03:36:42.556 | INFO | Serialization done, swapping tmp file now... | 20260506 03:36:42.557 | INFO | RTMetrics done | 20260506 03:36:42.627 | INFO | Finished a round of Baz IO: Wrote 1.755 GiB in 3.199 seconds | 20260506 03:36:42.627 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.22% / 0.00% / 78.78% | 20260506 03:36:42.627 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0595291% / 3.97364e-06% / 9.53674e-05% | 20260506 03:36:42.627 | INFO | Baz IO overall: | 20260506 03:36:42.627 | INFO | BytesWritten : 4800303905794 | 20260506 03:36:42.627 | INFO | NumEvents : 2250618584158 | 20260506 03:36:44.442 | INFO | Analyzing chunk frames = [8638464,8638976) | 20260506 03:36:44.442 | INFO | MemInfo: { | 20260506 03:36:44.443 | INFO | "gpu" : " 17 GB", | 20260506 03:36:44.443 | INFO | "huge" : "", | 20260506 03:36:44.443 | INFO | "huge_shared" : " 27 GB", | 20260506 03:36:44.443 | INFO | "malloc" : " 14 GB", | 20260506 03:36:44.443 | INFO | "pinned" : " 29 GB" | 20260506 03:36:44.443 | INFO | } | 20260506 03:36:44.443 | INFO | Perf report for the last 5.13986 seconds: | 20260506 03:36:44.443 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:36:44.443 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:44.443 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:44.443 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:44.443 | INFO | ANALYSIS : 62.3%, 3.0, 97.3ms, 0 | 20260506 03:36:44.443 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:44.443 | INFO | RT_METRICS : 42.6%, 1.0, 22.3ms, 0 | 20260506 03:36:44.443 | INFO | PRE_HQ : 58.8%, 1.7, 53.1ms, 0 | 20260506 03:36:44.443 | INFO | BAZWRITER : 61.4%, 3.4, 108.1ms, 0 | 20260506 03:36:44.443 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 03:36:44.443 | INFO | Uploaded 36GiB to GPU in 2.09816s (17.1579GiB/s) | 20260506 03:36:44.443 | INFO | Downloaded 38GiB to GPU in 2.22058s (17.1126GiB/s) | 20260506 03:36:49.581 | INFO | Analyzing chunk frames = [8638976,8639488) | 20260506 03:36:49.581 | INFO | MemInfo: { | 20260506 03:36:49.581 | INFO | "gpu" : " 17 GB", | 20260506 03:36:49.581 | INFO | "huge" : "", | 20260506 03:36:49.581 | INFO | "huge_shared" : " 27 GB", | 20260506 03:36:49.581 | INFO | "malloc" : " 14 GB", | 20260506 03:36:49.581 | INFO | "pinned" : " 29 GB" | 20260506 03:36:49.581 | INFO | } | 20260506 03:36:49.581 | INFO | Perf report for the last 5.13833 seconds: | 20260506 03:36:49.581 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:36:49.581 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:49.581 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:49.581 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:49.581 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260506 03:36:49.581 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:49.581 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:49.581 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 03:36:49.581 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:49.581 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:49.581 | INFO | Uploaded 36GiB to GPU in 1.94099s (18.5473GiB/s) | 20260506 03:36:49.581 | INFO | Downloaded 34GiB to GPU in 1.97683s (17.1992GiB/s) | 20260506 03:36:54.725 | INFO | Reached end of movie, endFrame:8640000 expectedEndFrame:8640000 | 20260506 03:36:54.725 | INFO | DataSourceRunner thread exiting | 20260506 03:36:54.725 | INFO | Analyzing chunk frames = [8639488,8640000) | 20260506 03:36:54.725 | INFO | MemInfo: { | 20260506 03:36:54.725 | INFO | "gpu" : " 17 GB", | 20260506 03:36:54.725 | INFO | "huge" : "", | 20260506 03:36:54.725 | INFO | "huge_shared" : " 27 GB", | 20260506 03:36:54.726 | INFO | "malloc" : " 14 GB", | 20260506 03:36:54.726 | INFO | "pinned" : " 29 GB" | 20260506 03:36:54.726 | INFO | } | 20260506 03:36:54.726 | INFO | Perf report for the last 5.14461 seconds: | 20260506 03:36:54.726 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 03:36:54.726 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:54.726 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:54.726 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:54.726 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260506 03:36:54.726 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:54.726 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:54.726 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 03:36:54.726 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:54.726 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 03:36:54.726 | INFO | Uploaded 36GiB to GPU in 2.10468s (17.1047GiB/s) | 20260506 03:36:54.726 | INFO | Downloaded 34GiB to GPU in 2.03725s (16.6892GiB/s) | 20260506 03:36:54.727 | INFO | Flushing graph node REPACKER with 0 jobs. | 20260506 03:36:57.863 | INFO | Flushing graph node PREP_TRACE with 0 jobs. | 20260506 03:36:57.863 | INFO | Flushing graph node SAVE_TRACE with 0 jobs. | 20260506 03:36:57.863 | INFO | Flushing graph node SAVE_TRACE done. | 20260506 03:36:57.863 | INFO | Flushing graph node PREP_TRACE done. | 20260506 03:36:57.863 | INFO | Flushing graph node ANALYSIS with 0 jobs. | 20260506 03:36:57.863 | INFO | Flushing graph node PREP_RT_METRICS with 0 jobs. | 20260506 03:36:57.863 | INFO | Flushing graph node RT_METRICS with 0 jobs. | 20260506 03:36:57.863 | INFO | Flushing graph node RT_METRICS done. | 20260506 03:36:57.863 | INFO | Flushing graph node PREP_RT_METRICS done. | 20260506 03:36:57.863 | INFO | Flushing graph node PRE_HQ with 96 jobs. | 20260506 03:36:57.874 | INFO | Starting a round of Baz IO | 20260506 03:37:02.007 | INFO | Finished a round of Baz IO: Wrote 2.011 GiB in 4.133 seconds | 20260506 03:37:02.008 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 21.22% / 0.00% / 78.78% | 20260506 03:37:02.008 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.54992e-05% / 0% / 3.57628e-05% | 20260506 03:37:02.008 | INFO | Baz IO overall: | 20260506 03:37:02.008 | INFO | BytesWritten : 4802463091209 | 20260506 03:37:02.008 | INFO | NumEvents : 2250938477777 | 20260506 03:37:02.008 | INFO | Flushing graph node BAZWRITER with 0 jobs. | 20260506 03:37:02.008 | INFO | Flushing graph node BAZWRITER done. | 20260506 03:37:02.008 | INFO | Flushing graph node BAZ_SAMPLING with 0 jobs. | 20260506 03:37:02.008 | INFO | Flushing graph node BAZ_SAMPLING done. | 20260506 03:37:02.008 | INFO | Flushing graph node PRE_HQ done. | 20260506 03:37:02.008 | INFO | Flushing graph node ANALYSIS done. | 20260506 03:37:02.008 | INFO | Flushing graph node REPACKER done. | 20260506 03:37:02.008 | INFO | Exited chunk analysis loop. | 20260506 03:37:02.008 | INFO | framesAnalyzed:8640000 framesRequested:8640000 | 20260506 03:37:02.008 | INFO | All frames analyzed = 8640000 | 20260506 03:37:02.008 | INFO | Analyzed 16875 chunks at 0.194558 chunks/sec (4.89621e+06 zmws/sec) | 20260506 03:37:02.023 | INFO | CpuSchedulerCanary: done | 20260506 03:37:02.177 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 03:37:02.177 | INFO | PerfReport: Gathered 1152 samples. | 20260506 03:37:02.177 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 03:37:02.177 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.35e+01, 1.31e+01 / 6.60e-01 | 20260506 03:37:02.177 | INFO | PerfReport: 0.29%, AggregateStats -- 9.17e-02 / 1.04e-01, 1.03e-02 / 4.29e-03 | 20260506 03:37:02.177 | INFO | PerfReport: 11.87%, Baseline -- 3.71e+00 / 4.53e+00, 5.83e-01 / 1.82e-01 | 20260506 03:37:02.177 | INFO | PerfReport: 3.31%, Histogram -- 1.03e+00 / 1.53e+00, 1.96e-01 / 7.26e-02 | 20260506 03:37:02.177 | INFO | PerfReport: 15.96%, DME -- 3.99e+01 / 4.38e+01, 3.73e+00 / 6.73e-01 | 20260506 03:37:02.177 | INFO | PerfReport: 57.53%, FrameLabeling -- 1.80e+01 / 1.93e+01, 1.56e+00 / 6.56e-01 | 20260506 03:37:02.177 | INFO | PerfReport: 6.98%, PulseAccumulating -- 2.18e+00 / 2.70e+00, 4.10e-01 / 1.28e-01 | 20260506 03:37:02.177 | INFO | PerfReport: 3.05%, Metrics -- 9.52e-01 / 1.22e+00, 2.55e-01 / 5.34e-02 | 20260506 03:37:02.177 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.20e-01 / 3.62e-01, 5.35e-02 / 2.79e-02 | 20260506 03:37:02.177 | INFO | Final Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 03:37:02.177 | INFO | PerfReport: Gathered 1615776 samples. | 20260506 03:37:02.177 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 03:37:02.177 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.53e+01, 1.35e+01 / 1.68e+00 | 20260506 03:37:02.177 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.03e-01, 1.09e-02 / 1.59e-02 | 20260506 03:37:02.177 | INFO | PerfReport: 11.67%, Baseline -- 3.70e+00 / 4.49e+00, 5.62e-01 / 2.13e-01 | 20260506 03:37:02.177 | INFO | PerfReport: 3.27%, Histogram -- 1.04e+00 / 1.51e+00, 1.90e-01 / 9.72e-02 | 20260506 03:37:02.177 | INFO | PerfReport: 15.88%, DME -- 4.03e+01 / 4.47e+01, 4.09e+00 / 1.70e+00 | 20260506 03:37:02.177 | INFO | PerfReport: 56.77%, FrameLabeling -- 1.80e+01 / 1.97e+01, 1.53e+00 / 1.44e+00 | 20260506 03:37:02.177 | INFO | PerfReport: 7.35%, PulseAccumulating -- 2.33e+00 / 2.90e+00, 4.17e-01 / 2.16e-01 | 20260506 03:37:02.177 | INFO | PerfReport: 3.51%, Metrics -- 1.11e+00 / 1.43e+00, 3.01e-01 / 1.56e-01 | 20260506 03:37:02.177 | INFO | PerfReport: 1.26%, PulseSerializing -- 3.98e-01 / 4.48e-01, 7.19e-02 / 3.36e-02 | 20260506 03:37:02.177 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 03:37:02.177 | INFO | PerfReport: Gathered 1152 samples. | 20260506 03:37:02.177 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 03:37:02.177 | INFO | PerfReport: Full Execution -- 9.73e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 03:37:02.177 | INFO | PerfReport: 22.13%, Upload -- 2.15e+01 / 0.00e+00, 7.01e+00 / 0.00e+00 | 20260506 03:37:02.177 | INFO | PerfReport: 21.96%, Download -- 2.14e+01 / 0.00e+00, 7.50e+00 / 0.00e+00 | 20260506 03:37:02.177 | INFO | PerfReport: 32.13%, Compute -- 3.13e+01 / 0.00e+00, 1.31e+01 / 0.00e+00 | 20260506 03:37:02.177 | INFO | PerfReport: 2.05%, LockUpload -- 2.00e+00 / 0.00e+00, 7.94e+00 / 0.00e+00 | 20260506 03:37:02.177 | INFO | PerfReport: 1.30%, LockDownload -- 1.27e+00 / 0.00e+00, 2.97e+00 / 0.00e+00 | 20260506 03:37:02.177 | INFO | PerfReport: 20.41%, LockCompute -- 1.99e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260506 03:37:02.177 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.69e-02 / 0.00e+00 | 20260506 03:37:02.177 | INFO | Final Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 03:37:02.177 | INFO | PerfReport: Gathered 1620000 samples. | 20260506 03:37:02.177 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 03:37:02.177 | INFO | PerfReport: Full Execution -- 9.92e+01 / -nan, 2.42e+01 / 0.00e+00 | 20260506 03:37:02.177 | INFO | PerfReport: 22.08%, Upload -- 2.19e+01 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260506 03:37:02.177 | INFO | PerfReport: 21.46%, Download -- 2.13e+01 / 0.00e+00, 7.49e+00 / 0.00e+00 | 20260506 03:37:02.177 | INFO | PerfReport: 31.98%, Compute -- 3.17e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 03:37:02.177 | INFO | PerfReport: 1.95%, LockUpload -- 1.93e+00 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260506 03:37:02.177 | INFO | PerfReport: 1.15%, LockDownload -- 1.14e+00 / 0.00e+00, 2.68e+00 / 0.00e+00 | 20260506 03:37:02.177 | INFO | PerfReport: 21.36%, LockCompute -- 2.12e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 03:37:02.177 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 2.10e-02 / 0.00e+00 | 20260506 03:37:02.250 | INFO | Closing BAZ file: /data/nrta/2/m84031_260504_231011_s1/pb_internal/baz/m84031_260504_231011_s1.bazfilelist.txt | 20260506 03:37:02.286 | INFO | 90 baz files left to close | 20260506 03:37:02.313 | INFO | 80 baz files left to close | 20260506 03:37:02.317 | INFO | 70 baz files left to close | 20260506 03:37:02.318 | INFO | 60 baz files left to close | 20260506 03:37:02.319 | INFO | 50 baz files left to close | 20260506 03:37:02.320 | INFO | 40 baz files left to close | 20260506 03:37:02.320 | INFO | 30 baz files left to close | 20260506 03:37:02.321 | INFO | 20 baz files left to close | 20260506 03:37:02.322 | INFO | 10 baz files left to close | 20260506 03:37:02.322 | INFO | 0 baz files left to close | 20260506 03:37:02.323 | INFO | Header/Footer Bytes : 2.395 GiB | 20260506 03:37:02.323 | INFO | OverheadBytes : 128.9 GiB | 20260506 03:37:02.323 | INFO | EventBytes : 4.008 TiB | 20260506 03:37:02.323 | INFO | MetricsBytes : 236.9 GiB | 20260506 03:37:02.323 | INFO | NumEvents : 2250938477777 | 20260506 03:37:02.324 | INFO | NumMetrics : 3071850950 | 20260506 03:37:02.324 | INFO | ------------------------------------------ | 20260506 03:37:02.324 | INFO | Total BytesWritten : 4.368 TiB | 20260506 03:37:02.324 | INFO | ------------------------------------------ | 20260506 03:37:02.324 | INFO | UncompressedEventBytes : 4.539 TiB | 20260506 03:37:02.324 | INFO | UncompressedMetricsBytes : 268.9 GiB | 20260506 03:37:02.324 | INFO | Data compression ratio : 1.13 | 20260506 03:37:02.324 | INFO | ------------------------------------------ | 20260506 03:37:02.324 | INFO | 0.000299% of recorded slices had data loss, with 0.000194% from the start of a preliminary HQ regions | 20260506 03:37:02.324 | INFO | Closing baz file "/data/nrta/2/m84031_260504_231011_s1/pb_internal/baz/m84031_260504_231011_s1_basic_preview.baz": | 20260506 03:37:02.324 | INFO | Header/Footer Bytes : 2.464 MiB | 20260506 03:37:02.324 | INFO | OverheadBytes : 131.3 MiB | 20260506 03:37:02.324 | INFO | EventBytes : 4.081 GiB | 20260506 03:37:02.324 | INFO | MetricsBytes : 237.8 MiB | 20260506 03:37:02.324 | INFO | NumEvents : 2237759489 | 20260506 03:37:02.324 | INFO | NumMetrics : 3010547 | 20260506 03:37:02.324 | INFO | ------------------------------------------ | 20260506 03:37:02.324 | INFO | Total BytesWritten : 4.444 GiB | 20260506 03:37:02.324 | INFO | ------------------------------------------ | 20260506 03:37:02.324 | INFO | UncompressedEventBytes : 4.621 GiB | 20260506 03:37:02.324 | INFO | UncompressedMetricsBytes : 269.9 MiB | 20260506 03:37:02.324 | INFO | Data compression ratio : 1.13 | 20260506 03:37:02.324 | INFO | ------------------------------------------ | 20260506 03:37:02.324 | INFO | 0.000193% of recorded slices had data loss, with 0% from the start of a preliminary HQ regions | 20260506 03:37:02.324 | INFO | Closing baz file "/data/nrta/2/m84031_260504_231011_s1/pb_internal/baz/m84031_260504_231011_s1_full_preview.baz": | 20260506 03:37:02.324 | INFO | Header/Footer Bytes : 305.9 KiB | 20260506 03:37:02.324 | INFO | OverheadBytes : 14.79 MiB | 20260506 03:37:02.324 | INFO | EventBytes : 468.9 MiB | 20260506 03:37:02.324 | INFO | MetricsBytes : 27.61 MiB | 20260506 03:37:02.324 | INFO | NumEvents : 250859526 | 20260506 03:37:02.324 | INFO | NumMetrics : 349195 | 20260506 03:37:02.324 | INFO | ------------------------------------------ | 20260506 03:37:02.324 | INFO | Total BytesWritten : 511.6 MiB | 20260506 03:37:02.324 | INFO | ------------------------------------------ | 20260506 03:37:02.324 | INFO | UncompressedEventBytes : 530.6 MiB | 20260506 03:37:02.324 | INFO | UncompressedMetricsBytes : 31.3 MiB | 20260506 03:37:02.324 | INFO | Data compression ratio : 1.13 | 20260506 03:37:02.324 | INFO | ------------------------------------------ | 20260506 03:37:02.324 | INFO | 0.000148% of recorded slices had data loss, with 0% from the start of a preliminary HQ regions | 20260506 03:37:02.326 | NOTICE | ~WXIPCDataSource | 20260506 03:37:02.328 | NOTICE | Packet History start: now:115383.611 1970-01-02T08:03:03.611Z | 20260506 03:37:02.328 | NOTICE | packetHistory[0] 115364.384 2026-05-06T03:36:43.101Z 0xb10000000 66771411297 free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[1] 115364.384 2026-05-06T03:36:43.101Z BURST 1 pointers, free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[2] 115364.426 2026-05-06T03:36:43.143Z 0xae8000000 66905629025 free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[3] 115364.426 2026-05-06T03:36:43.143Z BURST 1 pointers, free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[4] 115364.470 2026-05-06T03:36:43.187Z 0x6b0000000 67039846753 free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[5] 115364.470 2026-05-06T03:36:43.187Z BURST 1 pointers, free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[6] 115364.512 2026-05-06T03:36:43.229Z 0x818000000 67174064481 free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[7] 115364.512 2026-05-06T03:36:43.229Z BURST 1 pointers, free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[8] 115364.555 2026-05-06T03:36:43.272Z 0x8b8000000 67308282209 free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[9] 115364.555 2026-05-06T03:36:43.272Z BURST 1 pointers, free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[10] 115364.598 2026-05-06T03:36:43.315Z 0xad8000000 67442499937 free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[11] 115364.598 2026-05-06T03:36:43.315Z BURST 1 pointers, free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[12] 115364.640 2026-05-06T03:36:43.357Z 0x9f0000000 67576717665 free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[13] 115364.640 2026-05-06T03:36:43.357Z BURST 1 pointers, free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[14] 115364.682 2026-05-06T03:36:43.399Z 0x658000000 67710935393 free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[15] 115364.682 2026-05-06T03:36:43.399Z BURST 1 pointers, free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[16] 115365.769 2026-05-06T03:36:44.486Z 0x968000000 67845153121 free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[17] 115365.769 2026-05-06T03:36:44.486Z BURST 1 pointers, free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[18] 115365.812 2026-05-06T03:36:44.529Z 0xa58000000 67845153121 free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[19] 115365.812 2026-05-06T03:36:44.529Z BURST 1 pointers, free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[20] 115365.855 2026-05-06T03:36:44.572Z 0xab0000000 67845153121 free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[21] 115365.855 2026-05-06T03:36:44.572Z BURST 1 pointers, free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[22] 115365.898 2026-05-06T03:36:44.615Z 0x8a8000000 68065354089 free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[23] 115365.898 2026-05-06T03:36:44.615Z BURST 1 pointers, free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[24] 115365.940 2026-05-06T03:36:44.657Z 0xb50000000 67955253629 free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[25] 115365.940 2026-05-06T03:36:44.657Z BURST 1 pointers, free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[26] 115365.983 2026-05-06T03:36:44.700Z 0x890000000 67845153121 free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[27] 115365.983 2026-05-06T03:36:44.700Z BURST 1 pointers, free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[28] 115366.025 2026-05-06T03:36:44.742Z 0x8d0000000 68063256933 free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[29] 115366.025 2026-05-06T03:36:44.742Z BURST 1 pointers, free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[30] 115366.068 2026-05-06T03:36:44.785Z 0x6a8000000 68065354089 free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[31] 115366.068 2026-05-06T03:36:44.785Z BURST 1 pointers, free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[32] 115366.111 2026-05-06T03:36:44.828Z 0x820000000 67955253605 free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[33] 115366.111 2026-05-06T03:36:44.828Z BURST 1 pointers, free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[34] 115366.153 2026-05-06T03:36:44.870Z 0x5d0000000 67818938721 free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[35] 115366.153 2026-05-06T03:36:44.870Z BURST 1 pointers, free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[36] 115366.196 2026-05-06T03:36:44.913Z 0x780000000 67821035877 free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[37] 115366.196 2026-05-06T03:36:44.913Z BURST 1 pointers, free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[38] 115366.239 2026-05-06T03:36:44.956Z 0x5f8000000 67821035901 free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[39] 115366.239 2026-05-06T03:36:44.956Z BURST 1 pointers, free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[40] 115366.281 2026-05-06T03:36:44.998Z 0x530000000 67710935393 free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[41] 115366.281 2026-05-06T03:36:44.998Z BURST 1 pointers, free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[42] 115366.324 2026-05-06T03:36:45.041Z 0x5d8000000 67821035877 free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[43] 115366.324 2026-05-06T03:36:45.041Z BURST 1 pointers, free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[44] 115366.366 2026-05-06T03:36:45.083Z 0xae0000000 67821035877 free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[45] 115366.366 2026-05-06T03:36:45.083Z BURST 1 pointers, free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[46] 115366.410 2026-05-06T03:36:45.127Z 0xb08000000 67794821477 free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[47] 115366.410 2026-05-06T03:36:45.127Z BURST 1 pointers, free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[48] 115366.452 2026-05-06T03:36:45.169Z 0x5a0000000 67686818149 free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[49] 115366.452 2026-05-06T03:36:45.169Z BURST 1 pointers, free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[50] 115366.495 2026-05-06T03:36:45.212Z 0x538000000 67686818149 free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[51] 115366.495 2026-05-06T03:36:45.212Z BURST 1 pointers, free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[52] 115366.537 2026-05-06T03:36:45.254Z 0x6c8000000 67686818173 free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[53] 115366.537 2026-05-06T03:36:45.254Z BURST 1 pointers, free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[54] 115366.579 2026-05-06T03:36:45.296Z 0x588000000 67686818149 free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[55] 115366.579 2026-05-06T03:36:45.296Z BURST 1 pointers, free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[56] 115366.623 2026-05-06T03:36:45.340Z 0x908000000 67686818149 free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[57] 115366.623 2026-05-06T03:36:45.340Z BURST 1 pointers, free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[58] 115366.665 2026-05-06T03:36:45.382Z 0x508000000 67686818149 free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[59] 115366.665 2026-05-06T03:36:45.382Z BURST 1 pointers, free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[60] 115366.708 2026-05-06T03:36:45.425Z 0xa68000000 67662700905 free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[61] 115366.708 2026-05-06T03:36:45.425Z BURST 1 pointers, free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[62] 115366.750 2026-05-06T03:36:45.467Z 0x888000000 67552600421 free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[63] 115366.750 2026-05-06T03:36:45.467Z BURST 1 pointers, free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[64] 115366.794 2026-05-06T03:36:45.511Z 0x520000000 67552600421 free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[65] 115366.794 2026-05-06T03:36:45.511Z BURST 1 pointers, free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[66] 115366.836 2026-05-06T03:36:45.553Z 0x6e0000000 67418382717 free:0 | 20260506 03:37:02.328 | NOTICE | packetHistory[67] 115366.836 2026-05-06T03:36:45.553Z BURST 1 pointers, free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[68] 115366.878 2026-05-06T03:36:45.595Z 0x6b8000000 67552600421 free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[69] 115366.878 2026-05-06T03:36:45.595Z BURST 1 pointers, free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[70] 115366.921 2026-05-06T03:36:45.638Z 0x688000000 67552600421 free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[71] 115366.921 2026-05-06T03:36:45.638Z BURST 1 pointers, free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[72] 115366.963 2026-05-06T03:36:45.680Z 0x868000000 67552600421 free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[73] 115366.963 2026-05-06T03:36:45.680Z BURST 1 pointers, free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[74] 115367.007 2026-05-06T03:36:45.724Z 0x720000000 67418382693 free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[75] 115367.007 2026-05-06T03:36:45.724Z BURST 1 pointers, free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[76] 115367.049 2026-05-06T03:36:45.766Z 0xa48000000 67394265449 free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[77] 115367.049 2026-05-06T03:36:45.766Z BURST 1 pointers, free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[78] 115367.092 2026-05-06T03:36:45.809Z 0xa28000000 67149947261 free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[79] 115367.092 2026-05-06T03:36:45.809Z BURST 1 pointers, free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[80] 115367.135 2026-05-06T03:36:45.852Z 0x918000000 67284164965 free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[81] 115367.135 2026-05-06T03:36:45.852Z BURST 1 pointers, free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[82] 115367.177 2026-05-06T03:36:45.894Z 0x808000000 67149947237 free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[83] 115367.177 2026-05-06T03:36:45.894Z BURST 1 pointers, free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[84] 115367.219 2026-05-06T03:36:45.936Z 0x8f0000000 67149947237 free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[85] 115367.219 2026-05-06T03:36:45.936Z BURST 1 pointers, free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[86] 115367.263 2026-05-06T03:36:45.980Z 0xb60000000 67015729509 free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[87] 115367.263 2026-05-06T03:36:45.980Z BURST 1 pointers, free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[88] 115367.305 2026-05-06T03:36:46.022Z 0x858000000 66989515109 free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[89] 115367.305 2026-05-06T03:36:46.022Z BURST 1 pointers, free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[90] 115367.347 2026-05-06T03:36:46.064Z 0x9e0000000 66771411297 free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[91] 115367.347 2026-05-06T03:36:46.064Z BURST 1 pointers, free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[92] 115367.391 2026-05-06T03:36:46.108Z 0x510000000 66989515109 free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[93] 115367.391 2026-05-06T03:36:46.108Z BURST 1 pointers, free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[94] 115367.433 2026-05-06T03:36:46.150Z 0x740000000 66747294053 free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[95] 115367.433 2026-05-06T03:36:46.150Z BURST 1 pointers, free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[96] 115367.476 2026-05-06T03:36:46.193Z 0x850000000 66747294053 free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[97] 115367.476 2026-05-06T03:36:46.193Z BURST 1 pointers, free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[98] 115367.519 2026-05-06T03:36:46.236Z 0xac8000000 66613076325 free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[99] 115367.519 2026-05-06T03:36:46.236Z BURST 1 pointers, free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[100] 115367.561 2026-05-06T03:36:46.278Z 0x8d8000000 66478858621 free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[101] 115367.561 2026-05-06T03:36:46.278Z BURST 1 pointers, free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[102] 115367.603 2026-05-06T03:36:46.320Z 0x548000000 66613076325 free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[103] 115367.603 2026-05-06T03:36:46.320Z BURST 1 pointers, free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[104] 115367.647 2026-05-06T03:36:46.364Z 0x8f8000000 66586861925 free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[105] 115367.647 2026-05-06T03:36:46.364Z BURST 1 pointers, free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[106] 115367.689 2026-05-06T03:36:46.406Z 0x938000000 66344640869 free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[107] 115367.689 2026-05-06T03:36:46.406Z BURST 1 pointers, free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[108] 115367.732 2026-05-06T03:36:46.449Z 0x650000000 66344640869 free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[109] 115367.732 2026-05-06T03:36:46.449Z BURST 1 pointers, free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[110] 115367.775 2026-05-06T03:36:46.492Z 0x8b0000000 66320523649 free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[111] 115367.775 2026-05-06T03:36:46.492Z BURST 1 pointers, free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[112] 115367.817 2026-05-06T03:36:46.534Z 0xa50000000 66100322657 free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[113] 115367.817 2026-05-06T03:36:46.534Z BURST 1 pointers, free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[114] 115367.860 2026-05-06T03:36:46.577Z 0xa90000000 66318426469 free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[115] 115367.860 2026-05-06T03:36:46.577Z BURST 1 pointers, free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[116] 115367.903 2026-05-06T03:36:46.620Z 0xa98000000 66076205413 free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[117] 115367.903 2026-05-06T03:36:46.620Z BURST 1 pointers, free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[118] 115367.945 2026-05-06T03:36:46.662Z 0x6e8000000 66076205413 free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[119] 115367.945 2026-05-06T03:36:46.662Z BURST 1 pointers, free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[120] 115367.988 2026-05-06T03:36:46.705Z 0x518000000 65941987685 free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[121] 115367.988 2026-05-06T03:36:46.705Z BURST 1 pointers, free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[122] 115368.031 2026-05-06T03:36:46.748Z 0xa10000000 65807769981 free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[123] 115368.031 2026-05-06T03:36:46.748Z BURST 1 pointers, free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[124] 115368.074 2026-05-06T03:36:46.791Z 0x840000000 65939890529 free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[125] 115368.074 2026-05-06T03:36:46.791Z BURST 1 pointers, free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[126] 115368.116 2026-05-06T03:36:46.833Z 0x5b0000000 65807769957 free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[127] 115368.116 2026-05-06T03:36:46.833Z BURST 1 pointers, free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[128] 115368.158 2026-05-06T03:36:46.875Z 0x880000000 65807769957 free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[129] 115368.158 2026-05-06T03:36:46.875Z BURST 1 pointers, free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[130] 115368.202 2026-05-06T03:36:46.919Z 0x998000000 65673552229 free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[131] 115368.202 2026-05-06T03:36:46.919Z BURST 1 pointers, free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[132] 115368.244 2026-05-06T03:36:46.961Z 0x628000000 65647337829 free:0 | 20260506 03:37:02.329 | NOTICE | packetHistory[133] 115368.244 2026-05-06T03:36:46.961Z BURST 1 pointers, free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[134] 115368.287 2026-05-06T03:36:47.004Z 0x5e0000000 65429234017 free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[135] 115368.287 2026-05-06T03:36:47.004Z BURST 1 pointers, free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[136] 115368.329 2026-05-06T03:36:47.046Z 0x9a0000000 65539334501 free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[137] 115368.329 2026-05-06T03:36:47.046Z BURST 1 pointers, free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[138] 115368.371 2026-05-06T03:36:47.088Z 0x8a0000000 65539334501 free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[139] 115368.371 2026-05-06T03:36:47.088Z BURST 1 pointers, free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[140] 115368.415 2026-05-06T03:36:47.132Z 0x598000000 65405116773 free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[141] 115368.415 2026-05-06T03:36:47.132Z BURST 1 pointers, free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[142] 115368.457 2026-05-06T03:36:47.174Z 0x810000000 65378902373 free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[143] 115368.457 2026-05-06T03:36:47.174Z BURST 1 pointers, free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[144] 115368.500 2026-05-06T03:36:47.217Z 0x9b8000000 65136681341 free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[145] 115368.500 2026-05-06T03:36:47.217Z BURST 1 pointers, free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[146] 115368.543 2026-05-06T03:36:47.260Z 0x9d8000000 65160798561 free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[147] 115368.543 2026-05-06T03:36:47.260Z BURST 1 pointers, free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[148] 115368.585 2026-05-06T03:36:47.302Z 0xb20000000 65270899045 free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[149] 115368.585 2026-05-06T03:36:47.302Z BURST 1 pointers, free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[150] 115368.628 2026-05-06T03:36:47.345Z 0x760000000 65136681317 free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[151] 115368.628 2026-05-06T03:36:47.345Z BURST 1 pointers, free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[152] 115368.671 2026-05-06T03:36:47.388Z 0x638000000 65110466917 free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[153] 115368.671 2026-05-06T03:36:47.388Z BURST 1 pointers, free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[154] 115368.714 2026-05-06T03:36:47.431Z 0xac0000000 64868245861 free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[155] 115368.714 2026-05-06T03:36:47.431Z BURST 1 pointers, free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[156] 115368.756 2026-05-06T03:36:47.473Z 0x6f0000000 64758145401 free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[157] 115368.756 2026-05-06T03:36:47.473Z BURST 1 pointers, free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[158] 115368.798 2026-05-06T03:36:47.515Z 0x698000000 64868245861 free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[159] 115368.798 2026-05-06T03:36:47.515Z BURST 1 pointers, free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[160] 115368.841 2026-05-06T03:36:47.558Z 0xaa0000000 64868245861 free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[161] 115368.841 2026-05-06T03:36:47.558Z BURST 1 pointers, free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[162] 115368.884 2026-05-06T03:36:47.601Z 0xb38000000 64758145377 free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[163] 115368.884 2026-05-06T03:36:47.601Z BURST 1 pointers, free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[164] 115368.927 2026-05-06T03:36:47.644Z 0x790000000 64892363105 free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[165] 115368.927 2026-05-06T03:36:47.644Z BURST 1 pointers, free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[166] 115368.970 2026-05-06T03:36:47.687Z 0x630000000 65026580833 free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[167] 115368.970 2026-05-06T03:36:47.687Z BURST 1 pointers, free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[168] 115369.012 2026-05-06T03:36:47.729Z 0x7d8000000 65160798561 free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[169] 115369.012 2026-05-06T03:36:47.729Z BURST 1 pointers, free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[170] 115369.054 2026-05-06T03:36:47.771Z 0x8c8000000 65295016289 free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[171] 115369.054 2026-05-06T03:36:47.771Z BURST 1 pointers, free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[172] 115369.098 2026-05-06T03:36:47.815Z 0x9e8000000 65429234017 free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[173] 115369.098 2026-05-06T03:36:47.815Z BURST 1 pointers, free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[174] 115369.140 2026-05-06T03:36:47.857Z 0x860000000 65563451745 free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[175] 115369.140 2026-05-06T03:36:47.857Z BURST 1 pointers, free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[176] 115369.183 2026-05-06T03:36:47.900Z 0x648000000 65697669473 free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[177] 115369.183 2026-05-06T03:36:47.900Z BURST 1 pointers, free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[178] 115369.225 2026-05-06T03:36:47.942Z 0x570000000 65831887201 free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[179] 115369.225 2026-05-06T03:36:47.942Z BURST 1 pointers, free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[180] 115369.267 2026-05-06T03:36:47.984Z 0xa18000000 65966104929 free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[181] 115369.267 2026-05-06T03:36:47.984Z BURST 1 pointers, free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[182] 115369.311 2026-05-06T03:36:48.028Z 0x838000000 66100322657 free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[183] 115369.311 2026-05-06T03:36:48.028Z BURST 1 pointers, free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[184] 115369.353 2026-05-06T03:36:48.070Z 0x610000000 66234540385 free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[185] 115369.353 2026-05-06T03:36:48.070Z BURST 1 pointers, free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[186] 115369.395 2026-05-06T03:36:48.113Z 0xa88000000 66368758113 free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[187] 115369.396 2026-05-06T03:36:48.113Z BURST 1 pointers, free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[188] 115369.439 2026-05-06T03:36:48.156Z 0x670000000 66502975841 free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[189] 115369.439 2026-05-06T03:36:48.156Z BURST 1 pointers, free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[190] 115369.481 2026-05-06T03:36:48.198Z 0xa20000000 66637193569 free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[191] 115369.481 2026-05-06T03:36:48.198Z BURST 1 pointers, free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[192] 115369.524 2026-05-06T03:36:48.241Z 0x900000000 66771411297 free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[193] 115369.524 2026-05-06T03:36:48.241Z BURST 1 pointers, free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[194] 115369.567 2026-05-06T03:36:48.284Z 0x6a0000000 66905629025 free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[195] 115369.567 2026-05-06T03:36:48.284Z BURST 1 pointers, free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[196] 115369.609 2026-05-06T03:36:48.326Z 0x960000000 67039846753 free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[197] 115369.609 2026-05-06T03:36:48.326Z BURST 1 pointers, free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[198] 115369.652 2026-05-06T03:36:48.369Z 0xa38000000 67174064481 free:0 | 20260506 03:37:02.330 | NOTICE | packetHistory[199] 115369.652 2026-05-06T03:36:48.369Z BURST 1 pointers, free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[200] 115369.695 2026-05-06T03:36:48.412Z 0x958000000 67308282209 free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[201] 115369.695 2026-05-06T03:36:48.412Z BURST 1 pointers, free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[202] 115369.737 2026-05-06T03:36:48.454Z 0x988000000 67442499937 free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[203] 115369.737 2026-05-06T03:36:48.454Z BURST 1 pointers, free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[204] 115369.780 2026-05-06T03:36:48.497Z 0x948000000 67576717665 free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[205] 115369.780 2026-05-06T03:36:48.497Z BURST 1 pointers, free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[206] 115369.823 2026-05-06T03:36:48.540Z 0x898000000 67710935393 free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[207] 115369.823 2026-05-06T03:36:48.540Z BURST 1 pointers, free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[208] 115370.908 2026-05-06T03:36:49.625Z 0xb00000000 67845153121 free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[209] 115370.908 2026-05-06T03:36:49.625Z BURST 1 pointers, free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[210] 115370.950 2026-05-06T03:36:49.667Z 0x878000000 68089471333 free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[211] 115370.950 2026-05-06T03:36:49.667Z BURST 1 pointers, free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[212] 115370.993 2026-05-06T03:36:49.710Z 0x9b0000000 67955253605 free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[213] 115370.993 2026-05-06T03:36:49.710Z BURST 1 pointers, free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[214] 115371.036 2026-05-06T03:36:49.753Z 0xb58000000 67953156449 free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[215] 115371.036 2026-05-06T03:36:49.753Z BURST 1 pointers, free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[216] 115371.079 2026-05-06T03:36:49.796Z 0x7c0000000 67955253605 free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[217] 115371.079 2026-05-06T03:36:49.796Z BURST 1 pointers, free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[218] 115371.121 2026-05-06T03:36:49.838Z 0x660000000 67955253629 free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[219] 115371.121 2026-05-06T03:36:49.838Z BURST 1 pointers, free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[220] 115371.164 2026-05-06T03:36:49.881Z 0x690000000 67845153121 free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[221] 115371.164 2026-05-06T03:36:49.881Z BURST 1 pointers, free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[222] 115371.207 2026-05-06T03:36:49.924Z 0x668000000 68065354089 free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[223] 115371.207 2026-05-06T03:36:49.924Z BURST 1 pointers, free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[224] 115371.249 2026-05-06T03:36:49.966Z 0x5f0000000 67955253605 free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[225] 115371.249 2026-05-06T03:36:49.966Z BURST 1 pointers, free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[226] 115371.292 2026-05-06T03:36:50.009Z 0x500000000 67955253605 free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[227] 115371.292 2026-05-06T03:36:50.009Z BURST 1 pointers, free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[228] 115371.335 2026-05-06T03:36:50.052Z 0x640000000 67821035877 free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[229] 115371.335 2026-05-06T03:36:50.052Z BURST 1 pointers, free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[230] 115371.377 2026-05-06T03:36:50.094Z 0x700000000 67931136361 free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[231] 115371.377 2026-05-06T03:36:50.094Z BURST 1 pointers, free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[232] 115371.420 2026-05-06T03:36:50.137Z 0x750000000 67821035901 free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[233] 115371.420 2026-05-06T03:36:50.137Z BURST 1 pointers, free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[234] 115371.463 2026-05-06T03:36:50.180Z 0x8c0000000 67710935393 free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[235] 115371.463 2026-05-06T03:36:50.180Z BURST 1 pointers, free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[236] 115371.505 2026-05-06T03:36:50.222Z 0xa00000000 67929039205 free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[237] 115371.505 2026-05-06T03:36:50.222Z BURST 1 pointers, free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[238] 115371.548 2026-05-06T03:36:50.265Z 0xb28000000 67931136361 free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[239] 115371.548 2026-05-06T03:36:50.265Z BURST 1 pointers, free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[240] 115371.591 2026-05-06T03:36:50.308Z 0x710000000 67821035877 free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[241] 115371.591 2026-05-06T03:36:50.308Z BURST 1 pointers, free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[242] 115371.633 2026-05-06T03:36:50.350Z 0x620000000 67686818149 free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[243] 115371.633 2026-05-06T03:36:50.350Z BURST 1 pointers, free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[244] 115371.675 2026-05-06T03:36:50.392Z 0x940000000 67794821477 free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[245] 115371.676 2026-05-06T03:36:50.393Z BURST 1 pointers, free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[246] 115371.719 2026-05-06T03:36:50.436Z 0x618000000 67796918657 free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[247] 115371.719 2026-05-06T03:36:50.436Z BURST 1 pointers, free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[248] 115371.761 2026-05-06T03:36:50.478Z 0xb48000000 67576717665 free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[249] 115371.761 2026-05-06T03:36:50.478Z BURST 1 pointers, free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[250] 115371.804 2026-05-06T03:36:50.521Z 0x528000000 67821035877 free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[251] 115371.804 2026-05-06T03:36:50.521Z BURST 1 pointers, free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[252] 115371.847 2026-05-06T03:36:50.564Z 0xa40000000 67796918633 free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[253] 115371.847 2026-05-06T03:36:50.564Z BURST 1 pointers, free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[254] 115371.889 2026-05-06T03:36:50.606Z 0x608000000 67686818149 free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[255] 115371.889 2026-05-06T03:36:50.606Z BURST 1 pointers, free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[256] 115371.932 2026-05-06T03:36:50.649Z 0x9c0000000 67552600421 free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[257] 115371.932 2026-05-06T03:36:50.649Z BURST 1 pointers, free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[258] 115371.975 2026-05-06T03:36:50.692Z 0x7f0000000 67552600421 free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[259] 115371.975 2026-05-06T03:36:50.692Z BURST 1 pointers, free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[260] 115372.017 2026-05-06T03:36:50.734Z 0xb30000000 67662700929 free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[261] 115372.017 2026-05-06T03:36:50.734Z BURST 1 pointers, free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[262] 115372.059 2026-05-06T03:36:50.776Z 0x7a0000000 67442499937 free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[263] 115372.059 2026-05-06T03:36:50.776Z BURST 1 pointers, free:0 | 20260506 03:37:02.331 | NOTICE | packetHistory[264] 115372.103 2026-05-06T03:36:50.820Z 0xb40000000 67686818149 free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[265] 115372.103 2026-05-06T03:36:50.820Z BURST 1 pointers, free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[266] 115372.145 2026-05-06T03:36:50.862Z 0xa70000000 67660603749 free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[267] 115372.145 2026-05-06T03:36:50.862Z BURST 1 pointers, free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[268] 115372.188 2026-05-06T03:36:50.905Z 0x830000000 67662700905 free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[269] 115372.188 2026-05-06T03:36:50.905Z BURST 1 pointers, free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[270] 115372.231 2026-05-06T03:36:50.948Z 0xa30000000 67526386021 free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[271] 115372.231 2026-05-06T03:36:50.948Z BURST 1 pointers, free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[272] 115372.273 2026-05-06T03:36:50.990Z 0x558000000 67284164965 free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[273] 115372.273 2026-05-06T03:36:50.990Z BURST 1 pointers, free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[274] 115372.316 2026-05-06T03:36:51.033Z 0x680000000 67174064505 free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[275] 115372.316 2026-05-06T03:36:51.033Z BURST 1 pointers, free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[276] 115372.358 2026-05-06T03:36:51.075Z 0x930000000 67284164965 free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[277] 115372.358 2026-05-06T03:36:51.075Z BURST 1 pointers, free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[278] 115372.402 2026-05-06T03:36:51.119Z 0x718000000 67284164965 free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[279] 115372.402 2026-05-06T03:36:51.119Z BURST 1 pointers, free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[280] 115372.444 2026-05-06T03:36:51.161Z 0x578000000 67260047721 free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[281] 115372.444 2026-05-06T03:36:51.161Z BURST 1 pointers, free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[282] 115372.487 2026-05-06T03:36:51.204Z 0x6b0000000 67015729509 free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[283] 115372.487 2026-05-06T03:36:51.204Z BURST 1 pointers, free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[284] 115372.529 2026-05-06T03:36:51.246Z 0xad8000000 66881511805 free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[285] 115372.529 2026-05-06T03:36:51.246Z BURST 1 pointers, free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[286] 115372.572 2026-05-06T03:36:51.289Z 0x8b8000000 67013632353 free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[287] 115372.572 2026-05-06T03:36:51.289Z BURST 1 pointers, free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[288] 115372.614 2026-05-06T03:36:51.331Z 0x658000000 66881511781 free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[289] 115372.614 2026-05-06T03:36:51.331Z BURST 1 pointers, free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[290] 115372.657 2026-05-06T03:36:51.374Z 0xaa8000000 66747294053 free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[291] 115372.657 2026-05-06T03:36:51.374Z BURST 1 pointers, free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[292] 115372.700 2026-05-06T03:36:51.417Z 0x600000000 66747294053 free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[293] 115372.700 2026-05-06T03:36:51.417Z BURST 1 pointers, free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[294] 115372.742 2026-05-06T03:36:51.459Z 0x7f8000000 66613076349 free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[295] 115372.742 2026-05-06T03:36:51.459Z BURST 1 pointers, free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[296] 115372.786 2026-05-06T03:36:51.503Z 0x920000000 66502975841 free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[297] 115372.786 2026-05-06T03:36:51.503Z BURST 1 pointers, free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[298] 115372.828 2026-05-06T03:36:51.545Z 0x870000000 66613076325 free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[299] 115372.828 2026-05-06T03:36:51.545Z BURST 1 pointers, free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[300] 115372.871 2026-05-06T03:36:51.588Z 0xb18000000 66478858597 free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[301] 115372.871 2026-05-06T03:36:51.588Z BURST 1 pointers, free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[302] 115372.913 2026-05-06T03:36:51.630Z 0x980000000 66344640869 free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[303] 115372.913 2026-05-06T03:36:51.630Z BURST 1 pointers, free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[304] 115372.956 2026-05-06T03:36:51.673Z 0x7b0000000 66344640869 free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[305] 115372.956 2026-05-06T03:36:51.673Z BURST 1 pointers, free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[306] 115372.999 2026-05-06T03:36:51.716Z 0xaf8000000 66210423165 free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[307] 115372.999 2026-05-06T03:36:51.716Z BURST 1 pointers, free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[308] 115373.041 2026-05-06T03:36:51.758Z 0x910000000 66344640869 free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[309] 115373.041 2026-05-06T03:36:51.758Z BURST 1 pointers, free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[310] 115373.084 2026-05-06T03:36:51.801Z 0x928000000 66318426469 free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[311] 115373.084 2026-05-06T03:36:51.801Z BURST 1 pointers, free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[312] 115373.127 2026-05-06T03:36:51.844Z 0xa60000000 66076205413 free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[313] 115373.127 2026-05-06T03:36:51.844Z BURST 1 pointers, free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[314] 115373.169 2026-05-06T03:36:51.886Z 0x580000000 66076205413 free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[315] 115373.169 2026-05-06T03:36:51.886Z BURST 1 pointers, free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[316] 115373.212 2026-05-06T03:36:51.929Z 0x5e8000000 65941987709 free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[317] 115373.212 2026-05-06T03:36:51.929Z BURST 1 pointers, free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[318] 115373.255 2026-05-06T03:36:51.972Z 0x560000000 65831887201 free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[319] 115373.255 2026-05-06T03:36:51.972Z BURST 1 pointers, free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[320] 115373.297 2026-05-06T03:36:52.014Z 0x7d0000000 65941987685 free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[321] 115373.297 2026-05-06T03:36:52.014Z BURST 1 pointers, free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[322] 115373.339 2026-05-06T03:36:52.056Z 0x6d8000000 65807769957 free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[323] 115373.339 2026-05-06T03:36:52.056Z BURST 1 pointers, free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[324] 115373.383 2026-05-06T03:36:52.100Z 0x678000000 65673552229 free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[325] 115373.383 2026-05-06T03:36:52.100Z BURST 1 pointers, free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[326] 115373.425 2026-05-06T03:36:52.142Z 0x950000000 65673552229 free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[327] 115373.425 2026-05-06T03:36:52.142Z BURST 1 pointers, free:0 | 20260506 03:37:02.332 | NOTICE | packetHistory[328] 115373.468 2026-05-06T03:36:52.185Z 0xa80000000 65539334525 free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[329] 115373.468 2026-05-06T03:36:52.185Z BURST 1 pointers, free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[330] 115373.510 2026-05-06T03:36:52.227Z 0x550000000 65673552229 free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[331] 115373.510 2026-05-06T03:36:52.227Z BURST 1 pointers, free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[332] 115373.553 2026-05-06T03:36:52.270Z 0x5c8000000 65647337829 free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[333] 115373.553 2026-05-06T03:36:52.270Z BURST 1 pointers, free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[334] 115373.596 2026-05-06T03:36:52.313Z 0x6c0000000 65405116773 free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[335] 115373.596 2026-05-06T03:36:52.313Z BURST 1 pointers, free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[336] 115373.638 2026-05-06T03:36:52.355Z 0x730000000 65405116773 free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[337] 115373.638 2026-05-06T03:36:52.355Z BURST 1 pointers, free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[338] 115373.682 2026-05-06T03:36:52.399Z 0x828000000 65270899069 free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[339] 115373.682 2026-05-06T03:36:52.399Z BURST 1 pointers, free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[340] 115373.724 2026-05-06T03:36:52.441Z 0x978000000 65160798561 free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[341] 115373.724 2026-05-06T03:36:52.441Z BURST 1 pointers, free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[342] 115373.766 2026-05-06T03:36:52.483Z 0x5b8000000 65378902373 free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[343] 115373.766 2026-05-06T03:36:52.483Z BURST 1 pointers, free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[344] 115373.809 2026-05-06T03:36:52.526Z 0x5a8000000 65136681317 free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[345] 115373.809 2026-05-06T03:36:52.526Z BURST 1 pointers, free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[346] 115373.852 2026-05-06T03:36:52.569Z 0x708000000 65002463589 free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[347] 115373.852 2026-05-06T03:36:52.569Z BURST 1 pointers, free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[348] 115373.894 2026-05-06T03:36:52.611Z 0x590000000 65002463589 free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[349] 115373.894 2026-05-06T03:36:52.611Z BURST 1 pointers, free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[350] 115373.938 2026-05-06T03:36:52.655Z 0xb10000000 64868245885 free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[351] 115373.938 2026-05-06T03:36:52.655Z BURST 1 pointers, free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[352] 115373.980 2026-05-06T03:36:52.697Z 0xae8000000 64868245861 free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[353] 115373.980 2026-05-06T03:36:52.697Z BURST 1 pointers, free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[354] 115374.022 2026-05-06T03:36:52.739Z 0x818000000 64758145377 free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[355] 115374.022 2026-05-06T03:36:52.739Z BURST 1 pointers, free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[356] 115374.065 2026-05-06T03:36:52.782Z 0x800000000 64892363105 free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[357] 115374.065 2026-05-06T03:36:52.782Z BURST 1 pointers, free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[358] 115374.108 2026-05-06T03:36:52.825Z 0x8e0000000 65026580833 free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[359] 115374.108 2026-05-06T03:36:52.825Z BURST 1 pointers, free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[360] 115374.151 2026-05-06T03:36:52.868Z 0xa08000000 65160798561 free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[361] 115374.151 2026-05-06T03:36:52.868Z BURST 1 pointers, free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[362] 115374.193 2026-05-06T03:36:52.910Z 0x6d0000000 65295016289 free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[363] 115374.193 2026-05-06T03:36:52.910Z BURST 1 pointers, free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[364] 115374.235 2026-05-06T03:36:52.952Z 0x5c0000000 65429234017 free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[365] 115374.235 2026-05-06T03:36:52.952Z BURST 1 pointers, free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[366] 115374.279 2026-05-06T03:36:52.996Z 0x6f8000000 65563451745 free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[367] 115374.279 2026-05-06T03:36:52.996Z BURST 1 pointers, free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[368] 115374.321 2026-05-06T03:36:53.038Z 0x9d0000000 65697669473 free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[369] 115374.321 2026-05-06T03:36:53.038Z BURST 1 pointers, free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[370] 115374.364 2026-05-06T03:36:53.081Z 0xaf0000000 65831887201 free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[371] 115374.364 2026-05-06T03:36:53.081Z BURST 1 pointers, free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[372] 115374.407 2026-05-06T03:36:53.124Z 0x7e8000000 65966104929 free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[373] 115374.407 2026-05-06T03:36:53.124Z BURST 1 pointers, free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[374] 115374.449 2026-05-06T03:36:53.166Z 0x9a8000000 66100322657 free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[375] 115374.449 2026-05-06T03:36:53.166Z BURST 1 pointers, free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[376] 115374.492 2026-05-06T03:36:53.209Z 0x9f8000000 66234540385 free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[377] 115374.492 2026-05-06T03:36:53.209Z BURST 1 pointers, free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[378] 115374.535 2026-05-06T03:36:53.252Z 0xad0000000 66368758113 free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[379] 115374.535 2026-05-06T03:36:53.252Z BURST 1 pointers, free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[380] 115374.577 2026-05-06T03:36:53.294Z 0x568000000 66502975841 free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[381] 115374.577 2026-05-06T03:36:53.294Z BURST 1 pointers, free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[382] 115374.620 2026-05-06T03:36:53.337Z 0xab8000000 66637193569 free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[383] 115374.620 2026-05-06T03:36:53.337Z BURST 1 pointers, free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[384] 115374.663 2026-05-06T03:36:53.380Z 0x770000000 66771411297 free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[385] 115374.663 2026-05-06T03:36:53.380Z BURST 1 pointers, free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[386] 115374.705 2026-05-06T03:36:53.422Z 0x9f0000000 66905629025 free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[387] 115374.705 2026-05-06T03:36:53.422Z BURST 1 pointers, free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[388] 115374.747 2026-05-06T03:36:53.464Z 0x8e8000000 67039846753 free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[389] 115374.747 2026-05-06T03:36:53.464Z BURST 1 pointers, free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[390] 115374.791 2026-05-06T03:36:53.508Z 0x970000000 67174064481 free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[391] 115374.791 2026-05-06T03:36:53.508Z BURST 1 pointers, free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[392] 115374.833 2026-05-06T03:36:53.550Z 0x990000000 67308282209 free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[393] 115374.833 2026-05-06T03:36:53.550Z BURST 1 pointers, free:0 | 20260506 03:37:02.333 | NOTICE | packetHistory[394] 115374.875 2026-05-06T03:36:53.593Z 0x7e0000000 67442499937 free:0 | 20260506 03:37:02.334 | NOTICE | packetHistory[395] 115374.876 2026-05-06T03:36:53.593Z BURST 1 pointers, free:0 | 20260506 03:37:02.334 | NOTICE | packetHistory[396] 115374.918 2026-05-06T03:36:53.635Z 0x540000000 67576717665 free:0 | 20260506 03:37:02.334 | NOTICE | packetHistory[397] 115374.918 2026-05-06T03:36:53.635Z BURST 1 pointers, free:0 | 20260506 03:37:02.334 | NOTICE | packetHistory[398] 115374.962 2026-05-06T03:36:53.679Z 0x728000000 67710935393 free:0 | 20260506 03:37:02.334 | NOTICE | packetHistory[399] 115374.962 2026-05-06T03:36:53.679Z BURST 1 pointers, free:0 | 20260506 03:37:02.334 | NOTICE | std::size(inFlight):96 freequeue:0 | 20260506 03:37:02.334 | NOTICE | Packet History end | 20260506 03:37:02.334 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 03:37:02.334 | INFO | Total mem usage: 25.6 GiB | 20260506 03:37:02.334 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 03:37:02.334 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 03:37:02.334 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 03:37:02.334 | INFO | High Water Breakdown: | 20260506 03:37:02.334 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 03:37:02.334 | INFO | ------------------------------------------ | 20260506 03:37:02.334 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:37:05.647 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 03:37:05.647 | INFO | Total mem usage: 15.7 GiB | 20260506 03:37:05.647 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 03:37:05.647 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 03:37:05.647 | INFO | Unused Cache: 856 MiB (Waste: 5.33%) | 20260506 03:37:05.647 | INFO | High Water Breakdown: | 20260506 03:37:05.647 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 03:37:05.647 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 03:37:05.647 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 03:37:05.647 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 03:37:05.647 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 03:37:05.647 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 03:37:05.647 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 03:37:05.647 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 03:37:05.647 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 03:37:05.647 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 03:37:05.647 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 03:37:05.647 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 03:37:05.647 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 03:37:05.647 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 03:37:05.647 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 03:37:05.647 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 03:37:05.647 | INFO | 0.623% : 105 MiB ( 98.1% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 03:37:05.647 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 03:37:05.647 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 03:37:05.647 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 03:37:05.647 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 03:37:05.647 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 03:37:05.647 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 03:37:05.647 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 03:37:05.647 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 03:37:05.647 | INFO | ------------------------------------------ | 20260506 03:37:05.647 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:37:13.337 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 03:37:13.337 | INFO | Total mem usage: 26.5 GiB | 20260506 03:37:13.337 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 03:37:13.337 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 03:37:13.337 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 03:37:13.337 | INFO | High Water Breakdown: | 20260506 03:37:13.337 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 03:37:13.337 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 03:37:13.337 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 03:37:13.337 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 03:37:13.337 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 03:37:13.337 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 03:37:13.337 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 03:37:13.337 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 03:37:13.337 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 03:37:13.337 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 03:37:13.337 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 03:37:13.337 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 03:37:13.337 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 03:37:13.337 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 03:37:13.337 | INFO | ------------------------------------------ | 20260506 03:37:13.337 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 03:37:13.338 | NOTICE | Because ALLOCATION_HISTORY is defined, dumping CUDA allocation history for IA-849 | 20260506 03:37:13.338 | NOTICE | ----- PBCudaRuntime DumpAllocationHistoryToLog ---- (begin) | 20260506 03:37:13.338 | NOTICE | [0]: 1970-01-01T07:56:41.984Z HOST thread:139946219454464 size:10485760 | 20260506 03:37:13.339 | NOTICE | [1]: 1970-01-01T07:56:41.991Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.339 | NOTICE | [2]: 1970-01-01T07:56:41.997Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.339 | NOTICE | [3]: 1970-01-01T07:56:42.004Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.339 | NOTICE | [4]: 1970-01-01T07:56:42.008Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.339 | NOTICE | [5]: 1970-01-01T07:56:42.012Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.339 | NOTICE | [6]: 1970-01-01T07:56:42.015Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.339 | NOTICE | [7]: 1970-01-01T07:56:42.018Z GPU thread:139946219454464 size:60293120 | 20260506 03:37:13.339 | NOTICE | [8]: 1970-01-01T07:56:42.018Z GPU thread:139946219454464 size:15728640 | 20260506 03:37:13.339 | NOTICE | [9]: 1970-01-01T07:56:42.018Z GPU thread:139946219454464 size:6291456 | 20260506 03:37:13.339 | NOTICE | [10]: 1970-01-01T07:56:42.018Z GPU thread:139946219454464 size:1048576 | 20260506 03:37:13.339 | NOTICE | [11]: 1970-01-01T07:56:42.018Z HOST thread:139946219454464 size:160432128 | 20260506 03:37:13.339 | NOTICE | [12]: 1970-01-01T07:56:42.120Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.339 | NOTICE | [13]: 1970-01-01T07:56:42.142Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.339 | NOTICE | [14]: 1970-01-01T07:56:42.163Z HOST thread:139946219454464 size:10485760 | 20260506 03:37:13.339 | NOTICE | [15]: 1970-01-01T07:56:42.170Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.339 | NOTICE | [16]: 1970-01-01T07:56:42.177Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.339 | NOTICE | [17]: 1970-01-01T07:56:42.183Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.339 | NOTICE | [18]: 1970-01-01T07:56:42.188Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.339 | NOTICE | [19]: 1970-01-01T07:56:42.192Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.339 | NOTICE | [20]: 1970-01-01T07:56:42.195Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.339 | NOTICE | [21]: 1970-01-01T07:56:42.198Z GPU thread:139946219454464 size:60293120 | 20260506 03:37:13.339 | NOTICE | [22]: 1970-01-01T07:56:42.198Z GPU thread:139946219454464 size:15728640 | 20260506 03:37:13.339 | NOTICE | [23]: 1970-01-01T07:56:42.198Z GPU thread:139946219454464 size:6291456 | 20260506 03:37:13.339 | NOTICE | [24]: 1970-01-01T07:56:42.198Z GPU thread:139946219454464 size:1048576 | 20260506 03:37:13.339 | NOTICE | [25]: 1970-01-01T07:56:42.198Z HOST thread:139946219454464 size:160432128 | 20260506 03:37:13.339 | NOTICE | [26]: 1970-01-01T07:56:42.300Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.339 | NOTICE | [27]: 1970-01-01T07:56:42.322Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.339 | NOTICE | [28]: 1970-01-01T07:56:42.343Z HOST thread:139946219454464 size:10485760 | 20260506 03:37:13.339 | NOTICE | [29]: 1970-01-01T07:56:42.350Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.339 | NOTICE | [30]: 1970-01-01T07:56:42.357Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.339 | NOTICE | [31]: 1970-01-01T07:56:42.363Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.339 | NOTICE | [32]: 1970-01-01T07:56:42.368Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.339 | NOTICE | [33]: 1970-01-01T07:56:42.372Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.339 | NOTICE | [34]: 1970-01-01T07:56:42.375Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.339 | NOTICE | [35]: 1970-01-01T07:56:42.378Z GPU thread:139946219454464 size:60293120 | 20260506 03:37:13.339 | NOTICE | [36]: 1970-01-01T07:56:42.378Z GPU thread:139946219454464 size:15728640 | 20260506 03:37:13.339 | NOTICE | [37]: 1970-01-01T07:56:42.378Z GPU thread:139946219454464 size:6291456 | 20260506 03:37:13.339 | NOTICE | [38]: 1970-01-01T07:56:42.378Z GPU thread:139946219454464 size:1048576 | 20260506 03:37:13.339 | NOTICE | [39]: 1970-01-01T07:56:42.378Z HOST thread:139946219454464 size:160432128 | 20260506 03:37:13.339 | NOTICE | [40]: 1970-01-01T07:56:42.480Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.339 | NOTICE | [41]: 1970-01-01T07:56:42.502Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.339 | NOTICE | [42]: 1970-01-01T07:56:42.523Z HOST thread:139946219454464 size:10485760 | 20260506 03:37:13.339 | NOTICE | [43]: 1970-01-01T07:56:42.530Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.339 | NOTICE | [44]: 1970-01-01T07:56:42.537Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.339 | NOTICE | [45]: 1970-01-01T07:56:42.543Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.339 | NOTICE | [46]: 1970-01-01T07:56:42.548Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.339 | NOTICE | [47]: 1970-01-01T07:56:42.552Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.339 | NOTICE | [48]: 1970-01-01T07:56:42.555Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.339 | NOTICE | [49]: 1970-01-01T07:56:42.558Z GPU thread:139946219454464 size:60293120 | 20260506 03:37:13.339 | NOTICE | [50]: 1970-01-01T07:56:42.558Z GPU thread:139946219454464 size:15728640 | 20260506 03:37:13.339 | NOTICE | [51]: 1970-01-01T07:56:42.558Z GPU thread:139946219454464 size:6291456 | 20260506 03:37:13.339 | NOTICE | [52]: 1970-01-01T07:56:42.558Z GPU thread:139946219454464 size:1048576 | 20260506 03:37:13.339 | NOTICE | [53]: 1970-01-01T07:56:42.558Z HOST thread:139946219454464 size:160432128 | 20260506 03:37:13.339 | NOTICE | [54]: 1970-01-01T07:56:42.660Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.339 | NOTICE | [55]: 1970-01-01T07:56:42.682Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.339 | NOTICE | [56]: 1970-01-01T07:56:42.704Z HOST thread:139946219454464 size:10485760 | 20260506 03:37:13.339 | NOTICE | [57]: 1970-01-01T07:56:42.710Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.339 | NOTICE | [58]: 1970-01-01T07:56:42.717Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.339 | NOTICE | [59]: 1970-01-01T07:56:42.723Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.340 | NOTICE | [60]: 1970-01-01T07:56:42.728Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.340 | NOTICE | [61]: 1970-01-01T07:56:42.732Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.340 | NOTICE | [62]: 1970-01-01T07:56:42.735Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.340 | NOTICE | [63]: 1970-01-01T07:56:42.738Z GPU thread:139946219454464 size:60293120 | 20260506 03:37:13.340 | NOTICE | [64]: 1970-01-01T07:56:42.738Z GPU thread:139946219454464 size:15728640 | 20260506 03:37:13.340 | NOTICE | [65]: 1970-01-01T07:56:42.738Z GPU thread:139946219454464 size:6291456 | 20260506 03:37:13.340 | NOTICE | [66]: 1970-01-01T07:56:42.738Z GPU thread:139946219454464 size:1048576 | 20260506 03:37:13.340 | NOTICE | [67]: 1970-01-01T07:56:42.738Z HOST thread:139946219454464 size:160432128 | 20260506 03:37:13.340 | NOTICE | [68]: 1970-01-01T07:56:42.840Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.340 | NOTICE | [69]: 1970-01-01T07:56:42.862Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.340 | NOTICE | [70]: 1970-01-01T07:56:42.883Z HOST thread:139946219454464 size:10485760 | 20260506 03:37:13.340 | NOTICE | [71]: 1970-01-01T07:56:42.890Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.340 | NOTICE | [72]: 1970-01-01T07:56:42.897Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.340 | NOTICE | [73]: 1970-01-01T07:56:42.903Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.340 | NOTICE | [74]: 1970-01-01T07:56:42.908Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.340 | NOTICE | [75]: 1970-01-01T07:56:42.912Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.340 | NOTICE | [76]: 1970-01-01T07:56:42.915Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.340 | NOTICE | [77]: 1970-01-01T07:56:42.918Z GPU thread:139946219454464 size:60293120 | 20260506 03:37:13.340 | NOTICE | [78]: 1970-01-01T07:56:42.918Z GPU thread:139946219454464 size:15728640 | 20260506 03:37:13.340 | NOTICE | [79]: 1970-01-01T07:56:42.918Z GPU thread:139946219454464 size:6291456 | 20260506 03:37:13.340 | NOTICE | [80]: 1970-01-01T07:56:42.918Z GPU thread:139946219454464 size:1048576 | 20260506 03:37:13.340 | NOTICE | [81]: 1970-01-01T07:56:42.918Z HOST thread:139946219454464 size:160432128 | 20260506 03:37:13.340 | NOTICE | [82]: 1970-01-01T07:56:43.020Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.340 | NOTICE | [83]: 1970-01-01T07:56:43.042Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.340 | NOTICE | [84]: 1970-01-01T07:56:43.063Z HOST thread:139946219454464 size:10485760 | 20260506 03:37:13.340 | NOTICE | [85]: 1970-01-01T07:56:43.070Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.340 | NOTICE | [86]: 1970-01-01T07:56:43.077Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.340 | NOTICE | [87]: 1970-01-01T07:56:43.083Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.340 | NOTICE | [88]: 1970-01-01T07:56:43.088Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.340 | NOTICE | [89]: 1970-01-01T07:56:43.092Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.340 | NOTICE | [90]: 1970-01-01T07:56:43.095Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.340 | NOTICE | [91]: 1970-01-01T07:56:43.098Z GPU thread:139946219454464 size:60293120 | 20260506 03:37:13.340 | NOTICE | [92]: 1970-01-01T07:56:43.098Z GPU thread:139946219454464 size:15728640 | 20260506 03:37:13.340 | NOTICE | [93]: 1970-01-01T07:56:43.098Z GPU thread:139946219454464 size:6291456 | 20260506 03:37:13.340 | NOTICE | [94]: 1970-01-01T07:56:43.098Z GPU thread:139946219454464 size:1048576 | 20260506 03:37:13.340 | NOTICE | [95]: 1970-01-01T07:56:43.098Z HOST thread:139946219454464 size:160432128 | 20260506 03:37:13.340 | NOTICE | [96]: 1970-01-01T07:56:43.200Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.340 | NOTICE | [97]: 1970-01-01T07:56:43.222Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.340 | NOTICE | [98]: 1970-01-01T07:56:43.243Z HOST thread:139946219454464 size:10485760 | 20260506 03:37:13.340 | NOTICE | [99]: 1970-01-01T07:56:43.250Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.340 | NOTICE | [100]: 1970-01-01T07:56:43.257Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.340 | NOTICE | [101]: 1970-01-01T07:56:43.263Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.340 | NOTICE | [102]: 1970-01-01T07:56:43.268Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.340 | NOTICE | [103]: 1970-01-01T07:56:43.272Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.340 | NOTICE | [104]: 1970-01-01T07:56:43.275Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.340 | NOTICE | [105]: 1970-01-01T07:56:43.278Z GPU thread:139946219454464 size:60293120 | 20260506 03:37:13.340 | NOTICE | [106]: 1970-01-01T07:56:43.278Z GPU thread:139946219454464 size:15728640 | 20260506 03:37:13.340 | NOTICE | [107]: 1970-01-01T07:56:43.278Z GPU thread:139946219454464 size:6291456 | 20260506 03:37:13.340 | NOTICE | [108]: 1970-01-01T07:56:43.278Z GPU thread:139946219454464 size:1048576 | 20260506 03:37:13.340 | NOTICE | [109]: 1970-01-01T07:56:43.278Z HOST thread:139946219454464 size:160432128 | 20260506 03:37:13.340 | NOTICE | [110]: 1970-01-01T07:56:43.380Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.340 | NOTICE | [111]: 1970-01-01T07:56:43.402Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.340 | NOTICE | [112]: 1970-01-01T07:56:43.423Z HOST thread:139946219454464 size:10485760 | 20260506 03:37:13.340 | NOTICE | [113]: 1970-01-01T07:56:43.430Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.340 | NOTICE | [114]: 1970-01-01T07:56:43.437Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.340 | NOTICE | [115]: 1970-01-01T07:56:43.443Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.340 | NOTICE | [116]: 1970-01-01T07:56:43.448Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.340 | NOTICE | [117]: 1970-01-01T07:56:43.452Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.340 | NOTICE | [118]: 1970-01-01T07:56:43.455Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.340 | NOTICE | [119]: 1970-01-01T07:56:43.458Z GPU thread:139946219454464 size:60293120 | 20260506 03:37:13.340 | NOTICE | [120]: 1970-01-01T07:56:43.458Z GPU thread:139946219454464 size:15728640 | 20260506 03:37:13.340 | NOTICE | [121]: 1970-01-01T07:56:43.458Z GPU thread:139946219454464 size:6291456 | 20260506 03:37:13.340 | NOTICE | [122]: 1970-01-01T07:56:43.458Z GPU thread:139946219454464 size:1048576 | 20260506 03:37:13.340 | NOTICE | [123]: 1970-01-01T07:56:43.458Z HOST thread:139946219454464 size:160432128 | 20260506 03:37:13.340 | NOTICE | [124]: 1970-01-01T07:56:43.560Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.340 | NOTICE | [125]: 1970-01-01T07:56:43.582Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.340 | NOTICE | [126]: 1970-01-01T07:56:43.603Z HOST thread:139946219454464 size:10485760 | 20260506 03:37:13.340 | NOTICE | [127]: 1970-01-01T07:56:43.610Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.340 | NOTICE | [128]: 1970-01-01T07:56:43.617Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.340 | NOTICE | [129]: 1970-01-01T07:56:43.623Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.340 | NOTICE | [130]: 1970-01-01T07:56:43.628Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.340 | NOTICE | [131]: 1970-01-01T07:56:43.632Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.340 | NOTICE | [132]: 1970-01-01T07:56:43.635Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.340 | NOTICE | [133]: 1970-01-01T07:56:43.638Z GPU thread:139946219454464 size:60293120 | 20260506 03:37:13.340 | NOTICE | [134]: 1970-01-01T07:56:43.638Z GPU thread:139946219454464 size:15728640 | 20260506 03:37:13.341 | NOTICE | [135]: 1970-01-01T07:56:43.638Z GPU thread:139946219454464 size:6291456 | 20260506 03:37:13.341 | NOTICE | [136]: 1970-01-01T07:56:43.638Z GPU thread:139946219454464 size:1048576 | 20260506 03:37:13.341 | NOTICE | [137]: 1970-01-01T07:56:43.638Z HOST thread:139946219454464 size:160432128 | 20260506 03:37:13.341 | NOTICE | [138]: 1970-01-01T07:56:43.740Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.341 | NOTICE | [139]: 1970-01-01T07:56:43.761Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.341 | NOTICE | [140]: 1970-01-01T07:56:43.783Z HOST thread:139946219454464 size:10485760 | 20260506 03:37:13.341 | NOTICE | [141]: 1970-01-01T07:56:43.790Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.341 | NOTICE | [142]: 1970-01-01T07:56:43.796Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.341 | NOTICE | [143]: 1970-01-01T07:56:43.803Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.341 | NOTICE | [144]: 1970-01-01T07:56:43.807Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.341 | NOTICE | [145]: 1970-01-01T07:56:43.812Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.341 | NOTICE | [146]: 1970-01-01T07:56:43.814Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.341 | NOTICE | [147]: 1970-01-01T07:56:43.817Z GPU thread:139946219454464 size:60293120 | 20260506 03:37:13.341 | NOTICE | [148]: 1970-01-01T07:56:43.817Z GPU thread:139946219454464 size:15728640 | 20260506 03:37:13.341 | NOTICE | [149]: 1970-01-01T07:56:43.817Z GPU thread:139946219454464 size:6291456 | 20260506 03:37:13.341 | NOTICE | [150]: 1970-01-01T07:56:43.817Z GPU thread:139946219454464 size:1048576 | 20260506 03:37:13.341 | NOTICE | [151]: 1970-01-01T07:56:43.817Z HOST thread:139946219454464 size:160432128 | 20260506 03:37:13.341 | NOTICE | [152]: 1970-01-01T07:56:43.919Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.341 | NOTICE | [153]: 1970-01-01T07:56:43.941Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.341 | NOTICE | [154]: 1970-01-01T07:56:43.963Z HOST thread:139946219454464 size:10485760 | 20260506 03:37:13.341 | NOTICE | [155]: 1970-01-01T07:56:43.969Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.341 | NOTICE | [156]: 1970-01-01T07:56:43.976Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.341 | NOTICE | [157]: 1970-01-01T07:56:43.982Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.341 | NOTICE | [158]: 1970-01-01T07:56:43.987Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.341 | NOTICE | [159]: 1970-01-01T07:56:43.991Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.341 | NOTICE | [160]: 1970-01-01T07:56:43.994Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.341 | NOTICE | [161]: 1970-01-01T07:56:43.997Z GPU thread:139946219454464 size:60293120 | 20260506 03:37:13.341 | NOTICE | [162]: 1970-01-01T07:56:43.997Z GPU thread:139946219454464 size:15728640 | 20260506 03:37:13.341 | NOTICE | [163]: 1970-01-01T07:56:43.997Z GPU thread:139946219454464 size:6291456 | 20260506 03:37:13.341 | NOTICE | [164]: 1970-01-01T07:56:43.997Z GPU thread:139946219454464 size:1048576 | 20260506 03:37:13.341 | NOTICE | [165]: 1970-01-01T07:56:43.997Z HOST thread:139946219454464 size:160432128 | 20260506 03:37:13.341 | NOTICE | [166]: 1970-01-01T07:56:44.099Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.341 | NOTICE | [167]: 1970-01-01T07:56:44.121Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.341 | NOTICE | [168]: 1970-01-01T07:56:44.142Z HOST thread:139946219454464 size:10485760 | 20260506 03:37:13.341 | NOTICE | [169]: 1970-01-01T07:56:44.149Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.341 | NOTICE | [170]: 1970-01-01T07:56:44.156Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.341 | NOTICE | [171]: 1970-01-01T07:56:44.162Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.341 | NOTICE | [172]: 1970-01-01T07:56:44.167Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.341 | NOTICE | [173]: 1970-01-01T07:56:44.171Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.341 | NOTICE | [174]: 1970-01-01T07:56:44.174Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.341 | NOTICE | [175]: 1970-01-01T07:56:44.177Z GPU thread:139946219454464 size:60293120 | 20260506 03:37:13.341 | NOTICE | [176]: 1970-01-01T07:56:44.177Z GPU thread:139946219454464 size:15728640 | 20260506 03:37:13.341 | NOTICE | [177]: 1970-01-01T07:56:44.177Z GPU thread:139946219454464 size:6291456 | 20260506 03:37:13.341 | NOTICE | [178]: 1970-01-01T07:56:44.177Z GPU thread:139946219454464 size:1048576 | 20260506 03:37:13.341 | NOTICE | [179]: 1970-01-01T07:56:44.177Z HOST thread:139946219454464 size:160432128 | 20260506 03:37:13.341 | NOTICE | [180]: 1970-01-01T07:56:44.279Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.341 | NOTICE | [181]: 1970-01-01T07:56:44.301Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.341 | NOTICE | [182]: 1970-01-01T07:56:44.322Z HOST thread:139946219454464 size:10485760 | 20260506 03:37:13.341 | NOTICE | [183]: 1970-01-01T07:56:44.329Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.341 | NOTICE | [184]: 1970-01-01T07:56:44.336Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.341 | NOTICE | [185]: 1970-01-01T07:56:44.342Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.341 | NOTICE | [186]: 1970-01-01T07:56:44.347Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.341 | NOTICE | [187]: 1970-01-01T07:56:44.351Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.341 | NOTICE | [188]: 1970-01-01T07:56:44.354Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.341 | NOTICE | [189]: 1970-01-01T07:56:44.357Z GPU thread:139946219454464 size:60293120 | 20260506 03:37:13.341 | NOTICE | [190]: 1970-01-01T07:56:44.357Z GPU thread:139946219454464 size:15728640 | 20260506 03:37:13.341 | NOTICE | [191]: 1970-01-01T07:56:44.357Z GPU thread:139946219454464 size:6291456 | 20260506 03:37:13.341 | NOTICE | [192]: 1970-01-01T07:56:44.357Z GPU thread:139946219454464 size:1048576 | 20260506 03:37:13.341 | NOTICE | [193]: 1970-01-01T07:56:44.357Z HOST thread:139946219454464 size:160432128 | 20260506 03:37:13.341 | NOTICE | [194]: 1970-01-01T07:56:44.459Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.341 | NOTICE | [195]: 1970-01-01T07:56:44.481Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.341 | NOTICE | [196]: 1970-01-01T07:56:44.502Z HOST thread:139946219454464 size:10485760 | 20260506 03:37:13.341 | NOTICE | [197]: 1970-01-01T07:56:44.509Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.341 | NOTICE | [198]: 1970-01-01T07:56:44.516Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.341 | NOTICE | [199]: 1970-01-01T07:56:44.522Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.341 | NOTICE | [200]: 1970-01-01T07:56:44.527Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.341 | NOTICE | [201]: 1970-01-01T07:56:44.531Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.341 | NOTICE | [202]: 1970-01-01T07:56:44.534Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.341 | NOTICE | [203]: 1970-01-01T07:56:44.537Z GPU thread:139946219454464 size:60293120 | 20260506 03:37:13.341 | NOTICE | [204]: 1970-01-01T07:56:44.537Z GPU thread:139946219454464 size:15728640 | 20260506 03:37:13.341 | NOTICE | [205]: 1970-01-01T07:56:44.537Z GPU thread:139946219454464 size:6291456 | 20260506 03:37:13.342 | NOTICE | [206]: 1970-01-01T07:56:44.537Z GPU thread:139946219454464 size:1048576 | 20260506 03:37:13.342 | NOTICE | [207]: 1970-01-01T07:56:44.537Z HOST thread:139946219454464 size:160432128 | 20260506 03:37:13.342 | NOTICE | [208]: 1970-01-01T07:56:44.639Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.342 | NOTICE | [209]: 1970-01-01T07:56:44.661Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.342 | NOTICE | [210]: 1970-01-01T07:56:44.682Z HOST thread:139946219454464 size:10485760 | 20260506 03:37:13.342 | NOTICE | [211]: 1970-01-01T07:56:44.689Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.342 | NOTICE | [212]: 1970-01-01T07:56:44.696Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.342 | NOTICE | [213]: 1970-01-01T07:56:44.702Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.342 | NOTICE | [214]: 1970-01-01T07:56:44.707Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.342 | NOTICE | [215]: 1970-01-01T07:56:44.711Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.342 | NOTICE | [216]: 1970-01-01T07:56:44.714Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.342 | NOTICE | [217]: 1970-01-01T07:56:44.717Z GPU thread:139946219454464 size:60293120 | 20260506 03:37:13.342 | NOTICE | [218]: 1970-01-01T07:56:44.717Z GPU thread:139946219454464 size:15728640 | 20260506 03:37:13.342 | NOTICE | [219]: 1970-01-01T07:56:44.717Z GPU thread:139946219454464 size:6291456 | 20260506 03:37:13.342 | NOTICE | [220]: 1970-01-01T07:56:44.717Z GPU thread:139946219454464 size:1048576 | 20260506 03:37:13.342 | NOTICE | [221]: 1970-01-01T07:56:44.717Z HOST thread:139946219454464 size:160432128 | 20260506 03:37:13.342 | NOTICE | [222]: 1970-01-01T07:56:44.819Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.342 | NOTICE | [223]: 1970-01-01T07:56:44.841Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.342 | NOTICE | [224]: 1970-01-01T07:56:44.862Z HOST thread:139946219454464 size:10485760 | 20260506 03:37:13.342 | NOTICE | [225]: 1970-01-01T07:56:44.869Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.342 | NOTICE | [226]: 1970-01-01T07:56:44.876Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.342 | NOTICE | [227]: 1970-01-01T07:56:44.882Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.342 | NOTICE | [228]: 1970-01-01T07:56:44.887Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.342 | NOTICE | [229]: 1970-01-01T07:56:44.891Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.342 | NOTICE | [230]: 1970-01-01T07:56:44.894Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.342 | NOTICE | [231]: 1970-01-01T07:56:44.897Z GPU thread:139946219454464 size:60293120 | 20260506 03:37:13.342 | NOTICE | [232]: 1970-01-01T07:56:44.897Z GPU thread:139946219454464 size:15728640 | 20260506 03:37:13.342 | NOTICE | [233]: 1970-01-01T07:56:44.897Z GPU thread:139946219454464 size:6291456 | 20260506 03:37:13.342 | NOTICE | [234]: 1970-01-01T07:56:44.897Z GPU thread:139946219454464 size:1048576 | 20260506 03:37:13.342 | NOTICE | [235]: 1970-01-01T07:56:44.897Z HOST thread:139946219454464 size:160432128 | 20260506 03:37:13.342 | NOTICE | [236]: 1970-01-01T07:56:44.999Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.342 | NOTICE | [237]: 1970-01-01T07:56:45.021Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.342 | NOTICE | [238]: 1970-01-01T07:56:45.043Z HOST thread:139946219454464 size:10485760 | 20260506 03:37:13.342 | NOTICE | [239]: 1970-01-01T07:56:45.049Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.342 | NOTICE | [240]: 1970-01-01T07:56:45.056Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.342 | NOTICE | [241]: 1970-01-01T07:56:45.062Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.342 | NOTICE | [242]: 1970-01-01T07:56:45.067Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.342 | NOTICE | [243]: 1970-01-01T07:56:45.071Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.342 | NOTICE | [244]: 1970-01-01T07:56:45.074Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.342 | NOTICE | [245]: 1970-01-01T07:56:45.077Z GPU thread:139946219454464 size:60293120 | 20260506 03:37:13.342 | NOTICE | [246]: 1970-01-01T07:56:45.077Z GPU thread:139946219454464 size:15728640 | 20260506 03:37:13.342 | NOTICE | [247]: 1970-01-01T07:56:45.077Z GPU thread:139946219454464 size:6291456 | 20260506 03:37:13.342 | NOTICE | [248]: 1970-01-01T07:56:45.077Z GPU thread:139946219454464 size:1048576 | 20260506 03:37:13.342 | NOTICE | [249]: 1970-01-01T07:56:45.077Z HOST thread:139946219454464 size:160432128 | 20260506 03:37:13.342 | NOTICE | [250]: 1970-01-01T07:56:45.185Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.342 | NOTICE | [251]: 1970-01-01T07:56:45.207Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.342 | NOTICE | [252]: 1970-01-01T07:56:45.229Z HOST thread:139946219454464 size:10485760 | 20260506 03:37:13.342 | NOTICE | [253]: 1970-01-01T07:56:45.236Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.342 | NOTICE | [254]: 1970-01-01T07:56:45.242Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.342 | NOTICE | [255]: 1970-01-01T07:56:45.249Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.342 | NOTICE | [256]: 1970-01-01T07:56:45.253Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.342 | NOTICE | [257]: 1970-01-01T07:56:45.258Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.342 | NOTICE | [258]: 1970-01-01T07:56:45.261Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.342 | NOTICE | [259]: 1970-01-01T07:56:45.264Z GPU thread:139946219454464 size:60293120 | 20260506 03:37:13.342 | NOTICE | [260]: 1970-01-01T07:56:45.264Z GPU thread:139946219454464 size:15728640 | 20260506 03:37:13.342 | NOTICE | [261]: 1970-01-01T07:56:45.264Z GPU thread:139946219454464 size:6291456 | 20260506 03:37:13.342 | NOTICE | [262]: 1970-01-01T07:56:45.264Z GPU thread:139946219454464 size:1048576 | 20260506 03:37:13.342 | NOTICE | [263]: 1970-01-01T07:56:45.264Z HOST thread:139946219454464 size:160432128 | 20260506 03:37:13.342 | NOTICE | [264]: 1970-01-01T07:56:45.367Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.342 | NOTICE | [265]: 1970-01-01T07:56:45.388Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.342 | NOTICE | [266]: 1970-01-01T07:56:45.410Z HOST thread:139946219454464 size:10485760 | 20260506 03:37:13.342 | NOTICE | [267]: 1970-01-01T07:56:45.416Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.342 | NOTICE | [268]: 1970-01-01T07:56:45.423Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.342 | NOTICE | [269]: 1970-01-01T07:56:45.430Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.342 | NOTICE | [270]: 1970-01-01T07:56:45.434Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.342 | NOTICE | [271]: 1970-01-01T07:56:45.438Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.342 | NOTICE | [272]: 1970-01-01T07:56:45.441Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.342 | NOTICE | [273]: 1970-01-01T07:56:45.444Z GPU thread:139946219454464 size:60293120 | 20260506 03:37:13.342 | NOTICE | [274]: 1970-01-01T07:56:45.444Z GPU thread:139946219454464 size:15728640 | 20260506 03:37:13.342 | NOTICE | [275]: 1970-01-01T07:56:45.444Z GPU thread:139946219454464 size:6291456 | 20260506 03:37:13.342 | NOTICE | [276]: 1970-01-01T07:56:45.444Z GPU thread:139946219454464 size:1048576 | 20260506 03:37:13.343 | NOTICE | [277]: 1970-01-01T07:56:45.444Z HOST thread:139946219454464 size:160432128 | 20260506 03:37:13.343 | NOTICE | [278]: 1970-01-01T07:56:45.547Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.343 | NOTICE | [279]: 1970-01-01T07:56:45.568Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.343 | NOTICE | [280]: 1970-01-01T07:56:45.590Z HOST thread:139946219454464 size:10485760 | 20260506 03:37:13.343 | NOTICE | [281]: 1970-01-01T07:56:45.597Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.343 | NOTICE | [282]: 1970-01-01T07:56:45.603Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.343 | NOTICE | [283]: 1970-01-01T07:56:45.610Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.343 | NOTICE | [284]: 1970-01-01T07:56:45.614Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.343 | NOTICE | [285]: 1970-01-01T07:56:45.619Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.343 | NOTICE | [286]: 1970-01-01T07:56:45.622Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.343 | NOTICE | [287]: 1970-01-01T07:56:45.625Z GPU thread:139946219454464 size:60293120 | 20260506 03:37:13.343 | NOTICE | [288]: 1970-01-01T07:56:45.625Z GPU thread:139946219454464 size:15728640 | 20260506 03:37:13.343 | NOTICE | [289]: 1970-01-01T07:56:45.625Z GPU thread:139946219454464 size:6291456 | 20260506 03:37:13.343 | NOTICE | [290]: 1970-01-01T07:56:45.625Z GPU thread:139946219454464 size:1048576 | 20260506 03:37:13.343 | NOTICE | [291]: 1970-01-01T07:56:45.625Z HOST thread:139946219454464 size:160432128 | 20260506 03:37:13.343 | NOTICE | [292]: 1970-01-01T07:56:45.727Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.343 | NOTICE | [293]: 1970-01-01T07:56:45.749Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.343 | NOTICE | [294]: 1970-01-01T07:56:45.770Z HOST thread:139946219454464 size:10485760 | 20260506 03:37:13.343 | NOTICE | [295]: 1970-01-01T07:56:45.777Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.343 | NOTICE | [296]: 1970-01-01T07:56:45.784Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.343 | NOTICE | [297]: 1970-01-01T07:56:45.790Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.343 | NOTICE | [298]: 1970-01-01T07:56:45.795Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.343 | NOTICE | [299]: 1970-01-01T07:56:45.799Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.343 | NOTICE | [300]: 1970-01-01T07:56:45.802Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.343 | NOTICE | [301]: 1970-01-01T07:56:45.805Z GPU thread:139946219454464 size:60293120 | 20260506 03:37:13.343 | NOTICE | [302]: 1970-01-01T07:56:45.805Z GPU thread:139946219454464 size:15728640 | 20260506 03:37:13.343 | NOTICE | [303]: 1970-01-01T07:56:45.805Z GPU thread:139946219454464 size:6291456 | 20260506 03:37:13.343 | NOTICE | [304]: 1970-01-01T07:56:45.805Z GPU thread:139946219454464 size:1048576 | 20260506 03:37:13.343 | NOTICE | [305]: 1970-01-01T07:56:45.805Z HOST thread:139946219454464 size:160432128 | 20260506 03:37:13.343 | NOTICE | [306]: 1970-01-01T07:56:45.907Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.343 | NOTICE | [307]: 1970-01-01T07:56:45.929Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.343 | NOTICE | [308]: 1970-01-01T07:56:45.950Z HOST thread:139946219454464 size:10485760 | 20260506 03:37:13.343 | NOTICE | [309]: 1970-01-01T07:56:45.957Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.343 | NOTICE | [310]: 1970-01-01T07:56:45.964Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.343 | NOTICE | [311]: 1970-01-01T07:56:45.970Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.343 | NOTICE | [312]: 1970-01-01T07:56:45.975Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.343 | NOTICE | [313]: 1970-01-01T07:56:45.979Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.343 | NOTICE | [314]: 1970-01-01T07:56:45.982Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.343 | NOTICE | [315]: 1970-01-01T07:56:45.985Z GPU thread:139946219454464 size:60293120 | 20260506 03:37:13.343 | NOTICE | [316]: 1970-01-01T07:56:45.985Z GPU thread:139946219454464 size:15728640 | 20260506 03:37:13.343 | NOTICE | [317]: 1970-01-01T07:56:45.985Z GPU thread:139946219454464 size:6291456 | 20260506 03:37:13.343 | NOTICE | [318]: 1970-01-01T07:56:45.985Z GPU thread:139946219454464 size:1048576 | 20260506 03:37:13.343 | NOTICE | [319]: 1970-01-01T07:56:45.985Z HOST thread:139946219454464 size:160432128 | 20260506 03:37:13.343 | NOTICE | [320]: 1970-01-01T07:56:46.087Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.343 | NOTICE | [321]: 1970-01-01T07:56:46.109Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.343 | NOTICE | [322]: 1970-01-01T07:56:46.130Z HOST thread:139946219454464 size:10485760 | 20260506 03:37:13.343 | NOTICE | [323]: 1970-01-01T07:56:46.137Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.343 | NOTICE | [324]: 1970-01-01T07:56:46.144Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.343 | NOTICE | [325]: 1970-01-01T07:56:46.150Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.343 | NOTICE | [326]: 1970-01-01T07:56:46.155Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.343 | NOTICE | [327]: 1970-01-01T07:56:46.159Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.343 | NOTICE | [328]: 1970-01-01T07:56:46.162Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.343 | NOTICE | [329]: 1970-01-01T07:56:46.165Z GPU thread:139946219454464 size:60293120 | 20260506 03:37:13.343 | NOTICE | [330]: 1970-01-01T07:56:46.165Z GPU thread:139946219454464 size:15728640 | 20260506 03:37:13.343 | NOTICE | [331]: 1970-01-01T07:56:46.165Z GPU thread:139946219454464 size:6291456 | 20260506 03:37:13.343 | NOTICE | [332]: 1970-01-01T07:56:46.165Z GPU thread:139946219454464 size:1048576 | 20260506 03:37:13.343 | NOTICE | [333]: 1970-01-01T07:56:46.165Z HOST thread:139946219454464 size:160432128 | 20260506 03:37:13.343 | NOTICE | [334]: 1970-01-01T07:56:46.267Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.343 | NOTICE | [335]: 1970-01-01T07:56:46.288Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.343 | NOTICE | [336]: 1970-01-01T07:56:46.310Z HOST thread:139946219454464 size:10485760 | 20260506 03:37:13.343 | NOTICE | [337]: 1970-01-01T07:56:46.317Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.343 | NOTICE | [338]: 1970-01-01T07:56:46.323Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.343 | NOTICE | [339]: 1970-01-01T07:56:46.330Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.343 | NOTICE | [340]: 1970-01-01T07:56:46.334Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.343 | NOTICE | [341]: 1970-01-01T07:56:46.339Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.343 | NOTICE | [342]: 1970-01-01T07:56:46.341Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.343 | NOTICE | [343]: 1970-01-01T07:56:46.344Z GPU thread:139946219454464 size:60293120 | 20260506 03:37:13.343 | NOTICE | [344]: 1970-01-01T07:56:46.344Z GPU thread:139946219454464 size:15728640 | 20260506 03:37:13.343 | NOTICE | [345]: 1970-01-01T07:56:46.344Z GPU thread:139946219454464 size:6291456 | 20260506 03:37:13.343 | NOTICE | [346]: 1970-01-01T07:56:46.344Z GPU thread:139946219454464 size:1048576 | 20260506 03:37:13.343 | NOTICE | [347]: 1970-01-01T07:56:46.344Z HOST thread:139946219454464 size:160432128 | 20260506 03:37:13.343 | NOTICE | [348]: 1970-01-01T07:56:46.446Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.343 | NOTICE | [349]: 1970-01-01T07:56:46.468Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.343 | NOTICE | [350]: 1970-01-01T07:56:46.489Z HOST thread:139946219454464 size:10485760 | 20260506 03:37:13.344 | NOTICE | [351]: 1970-01-01T07:56:46.496Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.344 | NOTICE | [352]: 1970-01-01T07:56:46.503Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.344 | NOTICE | [353]: 1970-01-01T07:56:46.509Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.344 | NOTICE | [354]: 1970-01-01T07:56:46.514Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.344 | NOTICE | [355]: 1970-01-01T07:56:46.518Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.344 | NOTICE | [356]: 1970-01-01T07:56:46.521Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.344 | NOTICE | [357]: 1970-01-01T07:56:46.524Z GPU thread:139946219454464 size:60293120 | 20260506 03:37:13.344 | NOTICE | [358]: 1970-01-01T07:56:46.524Z GPU thread:139946219454464 size:15728640 | 20260506 03:37:13.344 | NOTICE | [359]: 1970-01-01T07:56:46.524Z GPU thread:139946219454464 size:6291456 | 20260506 03:37:13.344 | NOTICE | [360]: 1970-01-01T07:56:46.524Z GPU thread:139946219454464 size:1048576 | 20260506 03:37:13.344 | NOTICE | [361]: 1970-01-01T07:56:46.524Z HOST thread:139946219454464 size:160432128 | 20260506 03:37:13.344 | NOTICE | [362]: 1970-01-01T07:56:46.626Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.344 | NOTICE | [363]: 1970-01-01T07:56:46.647Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.344 | NOTICE | [364]: 1970-01-01T07:56:46.669Z HOST thread:139946219454464 size:10485760 | 20260506 03:37:13.344 | NOTICE | [365]: 1970-01-01T07:56:46.676Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.344 | NOTICE | [366]: 1970-01-01T07:56:46.682Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.344 | NOTICE | [367]: 1970-01-01T07:56:46.689Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.344 | NOTICE | [368]: 1970-01-01T07:56:46.693Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.344 | NOTICE | [369]: 1970-01-01T07:56:46.698Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.344 | NOTICE | [370]: 1970-01-01T07:56:46.700Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.344 | NOTICE | [371]: 1970-01-01T07:56:46.703Z GPU thread:139946219454464 size:60293120 | 20260506 03:37:13.344 | NOTICE | [372]: 1970-01-01T07:56:46.703Z GPU thread:139946219454464 size:15728640 | 20260506 03:37:13.344 | NOTICE | [373]: 1970-01-01T07:56:46.703Z GPU thread:139946219454464 size:6291456 | 20260506 03:37:13.344 | NOTICE | [374]: 1970-01-01T07:56:46.703Z GPU thread:139946219454464 size:1048576 | 20260506 03:37:13.344 | NOTICE | [375]: 1970-01-01T07:56:46.703Z HOST thread:139946219454464 size:160432128 | 20260506 03:37:13.344 | NOTICE | [376]: 1970-01-01T07:56:46.805Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.344 | NOTICE | [377]: 1970-01-01T07:56:46.827Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.344 | NOTICE | [378]: 1970-01-01T07:56:46.849Z HOST thread:139946219454464 size:10485760 | 20260506 03:37:13.344 | NOTICE | [379]: 1970-01-01T07:56:46.855Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.344 | NOTICE | [380]: 1970-01-01T07:56:46.862Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.344 | NOTICE | [381]: 1970-01-01T07:56:46.868Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.344 | NOTICE | [382]: 1970-01-01T07:56:46.873Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.344 | NOTICE | [383]: 1970-01-01T07:56:46.877Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.344 | NOTICE | [384]: 1970-01-01T07:56:46.880Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.344 | NOTICE | [385]: 1970-01-01T07:56:46.883Z GPU thread:139946219454464 size:60293120 | 20260506 03:37:13.344 | NOTICE | [386]: 1970-01-01T07:56:46.883Z GPU thread:139946219454464 size:15728640 | 20260506 03:37:13.344 | NOTICE | [387]: 1970-01-01T07:56:46.883Z GPU thread:139946219454464 size:6291456 | 20260506 03:37:13.344 | NOTICE | [388]: 1970-01-01T07:56:46.883Z GPU thread:139946219454464 size:1048576 | 20260506 03:37:13.344 | NOTICE | [389]: 1970-01-01T07:56:46.883Z HOST thread:139946219454464 size:160432128 | 20260506 03:37:13.344 | NOTICE | [390]: 1970-01-01T07:56:46.985Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.344 | NOTICE | [391]: 1970-01-01T07:56:47.007Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.344 | NOTICE | [392]: 1970-01-01T07:56:47.029Z HOST thread:139946219454464 size:10485760 | 20260506 03:37:13.344 | NOTICE | [393]: 1970-01-01T07:56:47.035Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.344 | NOTICE | [394]: 1970-01-01T07:56:47.042Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.344 | NOTICE | [395]: 1970-01-01T07:56:47.048Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.344 | NOTICE | [396]: 1970-01-01T07:56:47.053Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.344 | NOTICE | [397]: 1970-01-01T07:56:47.057Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.344 | NOTICE | [398]: 1970-01-01T07:56:47.060Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.344 | NOTICE | [399]: 1970-01-01T07:56:47.063Z GPU thread:139946219454464 size:60293120 | 20260506 03:37:13.344 | NOTICE | [400]: 1970-01-01T07:56:47.063Z GPU thread:139946219454464 size:15728640 | 20260506 03:37:13.344 | NOTICE | [401]: 1970-01-01T07:56:47.063Z GPU thread:139946219454464 size:6291456 | 20260506 03:37:13.344 | NOTICE | [402]: 1970-01-01T07:56:47.063Z GPU thread:139946219454464 size:1048576 | 20260506 03:37:13.344 | NOTICE | [403]: 1970-01-01T07:56:47.063Z HOST thread:139946219454464 size:160432128 | 20260506 03:37:13.344 | NOTICE | [404]: 1970-01-01T07:56:47.165Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.344 | NOTICE | [405]: 1970-01-01T07:56:47.187Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.344 | NOTICE | [406]: 1970-01-01T07:56:47.209Z HOST thread:139946219454464 size:10485760 | 20260506 03:37:13.344 | NOTICE | [407]: 1970-01-01T07:56:47.216Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.344 | NOTICE | [408]: 1970-01-01T07:56:47.222Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.344 | NOTICE | [409]: 1970-01-01T07:56:47.229Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.344 | NOTICE | [410]: 1970-01-01T07:56:47.233Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.344 | NOTICE | [411]: 1970-01-01T07:56:47.238Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.344 | NOTICE | [412]: 1970-01-01T07:56:47.240Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.344 | NOTICE | [413]: 1970-01-01T07:56:47.243Z GPU thread:139946219454464 size:60293120 | 20260506 03:37:13.344 | NOTICE | [414]: 1970-01-01T07:56:47.243Z GPU thread:139946219454464 size:15728640 | 20260506 03:37:13.344 | NOTICE | [415]: 1970-01-01T07:56:47.243Z GPU thread:139946219454464 size:6291456 | 20260506 03:37:13.344 | NOTICE | [416]: 1970-01-01T07:56:47.243Z GPU thread:139946219454464 size:1048576 | 20260506 03:37:13.344 | NOTICE | [417]: 1970-01-01T07:56:47.244Z HOST thread:139946219454464 size:160432128 | 20260506 03:37:13.344 | NOTICE | [418]: 1970-01-01T07:56:47.345Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.344 | NOTICE | [419]: 1970-01-01T07:56:47.367Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.344 | NOTICE | [420]: 1970-01-01T07:56:47.389Z HOST thread:139946219454464 size:10485760 | 20260506 03:37:13.344 | NOTICE | [421]: 1970-01-01T07:56:47.395Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.344 | NOTICE | [422]: 1970-01-01T07:56:47.402Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.345 | NOTICE | [423]: 1970-01-01T07:56:47.409Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.345 | NOTICE | [424]: 1970-01-01T07:56:47.413Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.345 | NOTICE | [425]: 1970-01-01T07:56:47.417Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.345 | NOTICE | [426]: 1970-01-01T07:56:47.420Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.345 | NOTICE | [427]: 1970-01-01T07:56:47.423Z GPU thread:139946219454464 size:60293120 | 20260506 03:37:13.345 | NOTICE | [428]: 1970-01-01T07:56:47.423Z GPU thread:139946219454464 size:15728640 | 20260506 03:37:13.345 | NOTICE | [429]: 1970-01-01T07:56:47.423Z GPU thread:139946219454464 size:6291456 | 20260506 03:37:13.345 | NOTICE | [430]: 1970-01-01T07:56:47.423Z GPU thread:139946219454464 size:1048576 | 20260506 03:37:13.345 | NOTICE | [431]: 1970-01-01T07:56:47.423Z HOST thread:139946219454464 size:160432128 | 20260506 03:37:13.345 | NOTICE | [432]: 1970-01-01T07:56:47.525Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.345 | NOTICE | [433]: 1970-01-01T07:56:47.547Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.345 | NOTICE | [434]: 1970-01-01T07:56:47.569Z HOST thread:139946219454464 size:10485760 | 20260506 03:37:13.345 | NOTICE | [435]: 1970-01-01T07:56:47.575Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.345 | NOTICE | [436]: 1970-01-01T07:56:47.582Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.345 | NOTICE | [437]: 1970-01-01T07:56:47.588Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.345 | NOTICE | [438]: 1970-01-01T07:56:47.593Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.345 | NOTICE | [439]: 1970-01-01T07:56:47.597Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.345 | NOTICE | [440]: 1970-01-01T07:56:47.600Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.345 | NOTICE | [441]: 1970-01-01T07:56:47.603Z GPU thread:139946219454464 size:60293120 | 20260506 03:37:13.345 | NOTICE | [442]: 1970-01-01T07:56:47.603Z GPU thread:139946219454464 size:15728640 | 20260506 03:37:13.345 | NOTICE | [443]: 1970-01-01T07:56:47.603Z GPU thread:139946219454464 size:6291456 | 20260506 03:37:13.345 | NOTICE | [444]: 1970-01-01T07:56:47.603Z GPU thread:139946219454464 size:1048576 | 20260506 03:37:13.345 | NOTICE | [445]: 1970-01-01T07:56:47.603Z HOST thread:139946219454464 size:160432128 | 20260506 03:37:13.345 | NOTICE | [446]: 1970-01-01T07:56:47.705Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.345 | NOTICE | [447]: 1970-01-01T07:56:47.727Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.345 | NOTICE | [448]: 1970-01-01T07:56:47.749Z HOST thread:139946219454464 size:10485760 | 20260506 03:37:13.345 | NOTICE | [449]: 1970-01-01T07:56:47.755Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.345 | NOTICE | [450]: 1970-01-01T07:56:47.762Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.345 | NOTICE | [451]: 1970-01-01T07:56:47.768Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.345 | NOTICE | [452]: 1970-01-01T07:56:47.773Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.345 | NOTICE | [453]: 1970-01-01T07:56:47.777Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.345 | NOTICE | [454]: 1970-01-01T07:56:47.780Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.345 | NOTICE | [455]: 1970-01-01T07:56:47.783Z GPU thread:139946219454464 size:60293120 | 20260506 03:37:13.345 | NOTICE | [456]: 1970-01-01T07:56:47.783Z GPU thread:139946219454464 size:15728640 | 20260506 03:37:13.345 | NOTICE | [457]: 1970-01-01T07:56:47.783Z GPU thread:139946219454464 size:6291456 | 20260506 03:37:13.345 | NOTICE | [458]: 1970-01-01T07:56:47.783Z GPU thread:139946219454464 size:1048576 | 20260506 03:37:13.345 | NOTICE | [459]: 1970-01-01T07:56:47.783Z HOST thread:139946219454464 size:160432128 | 20260506 03:37:13.345 | NOTICE | [460]: 1970-01-01T07:56:47.885Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.345 | NOTICE | [461]: 1970-01-01T07:56:47.906Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.345 | NOTICE | [462]: 1970-01-01T07:56:47.928Z HOST thread:139946219454464 size:10485760 | 20260506 03:37:13.345 | NOTICE | [463]: 1970-01-01T07:56:47.935Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.345 | NOTICE | [464]: 1970-01-01T07:56:47.941Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.345 | NOTICE | [465]: 1970-01-01T07:56:47.948Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.345 | NOTICE | [466]: 1970-01-01T07:56:47.952Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.345 | NOTICE | [467]: 1970-01-01T07:56:47.957Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.345 | NOTICE | [468]: 1970-01-01T07:56:47.959Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.345 | NOTICE | [469]: 1970-01-01T07:56:47.962Z GPU thread:139946219454464 size:60293120 | 20260506 03:37:13.345 | NOTICE | [470]: 1970-01-01T07:56:47.962Z GPU thread:139946219454464 size:15728640 | 20260506 03:37:13.345 | NOTICE | [471]: 1970-01-01T07:56:47.962Z GPU thread:139946219454464 size:6291456 | 20260506 03:37:13.345 | NOTICE | [472]: 1970-01-01T07:56:47.962Z GPU thread:139946219454464 size:1048576 | 20260506 03:37:13.345 | NOTICE | [473]: 1970-01-01T07:56:47.962Z HOST thread:139946219454464 size:160432128 | 20260506 03:37:13.345 | NOTICE | [474]: 1970-01-01T07:56:48.064Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.345 | NOTICE | [475]: 1970-01-01T07:56:48.086Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.345 | NOTICE | [476]: 1970-01-01T07:56:48.108Z HOST thread:139946219454464 size:10485760 | 20260506 03:37:13.345 | NOTICE | [477]: 1970-01-01T07:56:48.114Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.345 | NOTICE | [478]: 1970-01-01T07:56:48.121Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.345 | NOTICE | [479]: 1970-01-01T07:56:48.127Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.345 | NOTICE | [480]: 1970-01-01T07:56:48.132Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.345 | NOTICE | [481]: 1970-01-01T07:56:48.136Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.345 | NOTICE | [482]: 1970-01-01T07:56:48.139Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.345 | NOTICE | [483]: 1970-01-01T07:56:48.142Z GPU thread:139946219454464 size:60293120 | 20260506 03:37:13.345 | NOTICE | [484]: 1970-01-01T07:56:48.142Z GPU thread:139946219454464 size:15728640 | 20260506 03:37:13.345 | NOTICE | [485]: 1970-01-01T07:56:48.142Z GPU thread:139946219454464 size:6291456 | 20260506 03:37:13.345 | NOTICE | [486]: 1970-01-01T07:56:48.142Z GPU thread:139946219454464 size:1048576 | 20260506 03:37:13.345 | NOTICE | [487]: 1970-01-01T07:56:48.142Z HOST thread:139946219454464 size:160432128 | 20260506 03:37:13.345 | NOTICE | [488]: 1970-01-01T07:56:48.244Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.345 | NOTICE | [489]: 1970-01-01T07:56:48.266Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.345 | NOTICE | [490]: 1970-01-01T07:56:48.287Z HOST thread:139946219454464 size:10485760 | 20260506 03:37:13.345 | NOTICE | [491]: 1970-01-01T07:56:48.294Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.345 | NOTICE | [492]: 1970-01-01T07:56:48.301Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.345 | NOTICE | [493]: 1970-01-01T07:56:48.307Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.345 | NOTICE | [494]: 1970-01-01T07:56:48.312Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.345 | NOTICE | [495]: 1970-01-01T07:56:48.316Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.346 | NOTICE | [496]: 1970-01-01T07:56:48.319Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.346 | NOTICE | [497]: 1970-01-01T07:56:48.322Z GPU thread:139946219454464 size:60293120 | 20260506 03:37:13.346 | NOTICE | [498]: 1970-01-01T07:56:48.322Z GPU thread:139946219454464 size:15728640 | 20260506 03:37:13.346 | NOTICE | [499]: 1970-01-01T07:56:48.322Z GPU thread:139946219454464 size:6291456 | 20260506 03:37:13.346 | NOTICE | [500]: 1970-01-01T07:56:48.322Z GPU thread:139946219454464 size:1048576 | 20260506 03:37:13.346 | NOTICE | [501]: 1970-01-01T07:56:48.322Z HOST thread:139946219454464 size:160432128 | 20260506 03:37:13.346 | NOTICE | [502]: 1970-01-01T07:56:48.424Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.346 | NOTICE | [503]: 1970-01-01T07:56:48.445Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.346 | NOTICE | [504]: 1970-01-01T07:56:48.467Z HOST thread:139946219454464 size:10485760 | 20260506 03:37:13.346 | NOTICE | [505]: 1970-01-01T07:56:48.474Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.346 | NOTICE | [506]: 1970-01-01T07:56:48.480Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.346 | NOTICE | [507]: 1970-01-01T07:56:48.487Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.346 | NOTICE | [508]: 1970-01-01T07:56:48.491Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.346 | NOTICE | [509]: 1970-01-01T07:56:48.496Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.346 | NOTICE | [510]: 1970-01-01T07:56:48.499Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.346 | NOTICE | [511]: 1970-01-01T07:56:48.501Z GPU thread:139946219454464 size:60293120 | 20260506 03:37:13.346 | NOTICE | [512]: 1970-01-01T07:56:48.502Z GPU thread:139946219454464 size:15728640 | 20260506 03:37:13.346 | NOTICE | [513]: 1970-01-01T07:56:48.502Z GPU thread:139946219454464 size:6291456 | 20260506 03:37:13.346 | NOTICE | [514]: 1970-01-01T07:56:48.502Z GPU thread:139946219454464 size:1048576 | 20260506 03:37:13.346 | NOTICE | [515]: 1970-01-01T07:56:48.502Z HOST thread:139946219454464 size:160432128 | 20260506 03:37:13.346 | NOTICE | [516]: 1970-01-01T07:56:48.604Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.346 | NOTICE | [517]: 1970-01-01T07:56:48.626Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.346 | NOTICE | [518]: 1970-01-01T07:56:48.648Z HOST thread:139946219454464 size:10485760 | 20260506 03:37:13.346 | NOTICE | [519]: 1970-01-01T07:56:48.654Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.346 | NOTICE | [520]: 1970-01-01T07:56:48.661Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.346 | NOTICE | [521]: 1970-01-01T07:56:48.667Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.346 | NOTICE | [522]: 1970-01-01T07:56:48.672Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.346 | NOTICE | [523]: 1970-01-01T07:56:48.676Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.346 | NOTICE | [524]: 1970-01-01T07:56:48.679Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.346 | NOTICE | [525]: 1970-01-01T07:56:48.682Z GPU thread:139946219454464 size:60293120 | 20260506 03:37:13.346 | NOTICE | [526]: 1970-01-01T07:56:48.682Z GPU thread:139946219454464 size:15728640 | 20260506 03:37:13.346 | NOTICE | [527]: 1970-01-01T07:56:48.682Z GPU thread:139946219454464 size:6291456 | 20260506 03:37:13.346 | NOTICE | [528]: 1970-01-01T07:56:48.682Z GPU thread:139946219454464 size:1048576 | 20260506 03:37:13.346 | NOTICE | [529]: 1970-01-01T07:56:48.682Z HOST thread:139946219454464 size:160432128 | 20260506 03:37:13.346 | NOTICE | [530]: 1970-01-01T07:56:48.784Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.346 | NOTICE | [531]: 1970-01-01T07:56:48.806Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.346 | NOTICE | [532]: 1970-01-01T07:56:48.828Z HOST thread:139946219454464 size:10485760 | 20260506 03:37:13.346 | NOTICE | [533]: 1970-01-01T07:56:48.835Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.346 | NOTICE | [534]: 1970-01-01T07:56:48.841Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.346 | NOTICE | [535]: 1970-01-01T07:56:48.848Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.346 | NOTICE | [536]: 1970-01-01T07:56:48.852Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.346 | NOTICE | [537]: 1970-01-01T07:56:48.857Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.346 | NOTICE | [538]: 1970-01-01T07:56:48.859Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.346 | NOTICE | [539]: 1970-01-01T07:56:48.862Z GPU thread:139946219454464 size:60293120 | 20260506 03:37:13.346 | NOTICE | [540]: 1970-01-01T07:56:48.862Z GPU thread:139946219454464 size:15728640 | 20260506 03:37:13.346 | NOTICE | [541]: 1970-01-01T07:56:48.862Z GPU thread:139946219454464 size:6291456 | 20260506 03:37:13.346 | NOTICE | [542]: 1970-01-01T07:56:48.862Z GPU thread:139946219454464 size:1048576 | 20260506 03:37:13.346 | NOTICE | [543]: 1970-01-01T07:56:48.862Z HOST thread:139946219454464 size:160432128 | 20260506 03:37:13.346 | NOTICE | [544]: 1970-01-01T07:56:48.964Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.346 | NOTICE | [545]: 1970-01-01T07:56:48.986Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.346 | NOTICE | [546]: 1970-01-01T07:56:49.008Z HOST thread:139946219454464 size:10485760 | 20260506 03:37:13.346 | NOTICE | [547]: 1970-01-01T07:56:49.015Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.346 | NOTICE | [548]: 1970-01-01T07:56:49.021Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.346 | NOTICE | [549]: 1970-01-01T07:56:49.028Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.346 | NOTICE | [550]: 1970-01-01T07:56:49.032Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.346 | NOTICE | [551]: 1970-01-01T07:56:49.037Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.346 | NOTICE | [552]: 1970-01-01T07:56:49.039Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.346 | NOTICE | [553]: 1970-01-01T07:56:49.042Z GPU thread:139946219454464 size:60293120 | 20260506 03:37:13.346 | NOTICE | [554]: 1970-01-01T07:56:49.042Z GPU thread:139946219454464 size:15728640 | 20260506 03:37:13.346 | NOTICE | [555]: 1970-01-01T07:56:49.042Z GPU thread:139946219454464 size:6291456 | 20260506 03:37:13.346 | NOTICE | [556]: 1970-01-01T07:56:49.042Z GPU thread:139946219454464 size:1048576 | 20260506 03:37:13.346 | NOTICE | [557]: 1970-01-01T07:56:49.042Z HOST thread:139946219454464 size:160432128 | 20260506 03:37:13.346 | NOTICE | [558]: 1970-01-01T07:56:49.144Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.346 | NOTICE | [559]: 1970-01-01T07:56:49.166Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.346 | NOTICE | [560]: 1970-01-01T07:56:49.188Z HOST thread:139946219454464 size:10485760 | 20260506 03:37:13.346 | NOTICE | [561]: 1970-01-01T07:56:49.194Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.346 | NOTICE | [562]: 1970-01-01T07:56:49.201Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.346 | NOTICE | [563]: 1970-01-01T07:56:49.208Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.346 | NOTICE | [564]: 1970-01-01T07:56:49.212Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.346 | NOTICE | [565]: 1970-01-01T07:56:49.216Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.346 | NOTICE | [566]: 1970-01-01T07:56:49.219Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.346 | NOTICE | [567]: 1970-01-01T07:56:49.222Z GPU thread:139946219454464 size:60293120 | 20260506 03:37:13.347 | NOTICE | [568]: 1970-01-01T07:56:49.222Z GPU thread:139946219454464 size:15728640 | 20260506 03:37:13.347 | NOTICE | [569]: 1970-01-01T07:56:49.222Z GPU thread:139946219454464 size:6291456 | 20260506 03:37:13.347 | NOTICE | [570]: 1970-01-01T07:56:49.222Z GPU thread:139946219454464 size:1048576 | 20260506 03:37:13.347 | NOTICE | [571]: 1970-01-01T07:56:49.222Z HOST thread:139946219454464 size:160432128 | 20260506 03:37:13.347 | NOTICE | [572]: 1970-01-01T07:56:49.325Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.347 | NOTICE | [573]: 1970-01-01T07:56:49.346Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.347 | NOTICE | [574]: 1970-01-01T07:56:49.368Z HOST thread:139946219454464 size:10485760 | 20260506 03:37:13.347 | NOTICE | [575]: 1970-01-01T07:56:49.375Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.347 | NOTICE | [576]: 1970-01-01T07:56:49.381Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.347 | NOTICE | [577]: 1970-01-01T07:56:49.388Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.347 | NOTICE | [578]: 1970-01-01T07:56:49.392Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.347 | NOTICE | [579]: 1970-01-01T07:56:49.397Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.347 | NOTICE | [580]: 1970-01-01T07:56:49.399Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.347 | NOTICE | [581]: 1970-01-01T07:56:49.402Z GPU thread:139946219454464 size:60293120 | 20260506 03:37:13.347 | NOTICE | [582]: 1970-01-01T07:56:49.402Z GPU thread:139946219454464 size:15728640 | 20260506 03:37:13.347 | NOTICE | [583]: 1970-01-01T07:56:49.402Z GPU thread:139946219454464 size:6291456 | 20260506 03:37:13.347 | NOTICE | [584]: 1970-01-01T07:56:49.402Z GPU thread:139946219454464 size:1048576 | 20260506 03:37:13.347 | NOTICE | [585]: 1970-01-01T07:56:49.402Z HOST thread:139946219454464 size:160432128 | 20260506 03:37:13.347 | NOTICE | [586]: 1970-01-01T07:56:49.504Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.347 | NOTICE | [587]: 1970-01-01T07:56:49.526Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.347 | NOTICE | [588]: 1970-01-01T07:56:49.548Z HOST thread:139946219454464 size:10485760 | 20260506 03:37:13.347 | NOTICE | [589]: 1970-01-01T07:56:49.555Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.347 | NOTICE | [590]: 1970-01-01T07:56:49.561Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.347 | NOTICE | [591]: 1970-01-01T07:56:49.568Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.347 | NOTICE | [592]: 1970-01-01T07:56:49.572Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.347 | NOTICE | [593]: 1970-01-01T07:56:49.577Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.347 | NOTICE | [594]: 1970-01-01T07:56:49.579Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.347 | NOTICE | [595]: 1970-01-01T07:56:49.582Z GPU thread:139946219454464 size:60293120 | 20260506 03:37:13.347 | NOTICE | [596]: 1970-01-01T07:56:49.582Z GPU thread:139946219454464 size:15728640 | 20260506 03:37:13.347 | NOTICE | [597]: 1970-01-01T07:56:49.582Z GPU thread:139946219454464 size:6291456 | 20260506 03:37:13.347 | NOTICE | [598]: 1970-01-01T07:56:49.582Z GPU thread:139946219454464 size:1048576 | 20260506 03:37:13.347 | NOTICE | [599]: 1970-01-01T07:56:49.582Z HOST thread:139946219454464 size:160432128 | 20260506 03:37:13.347 | NOTICE | [600]: 1970-01-01T07:56:49.685Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.347 | NOTICE | [601]: 1970-01-01T07:56:49.707Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.347 | NOTICE | [602]: 1970-01-01T07:56:49.728Z HOST thread:139946219454464 size:10485760 | 20260506 03:37:13.347 | NOTICE | [603]: 1970-01-01T07:56:49.735Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.347 | NOTICE | [604]: 1970-01-01T07:56:49.742Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.347 | NOTICE | [605]: 1970-01-01T07:56:49.748Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.347 | NOTICE | [606]: 1970-01-01T07:56:49.753Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.347 | NOTICE | [607]: 1970-01-01T07:56:49.757Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.347 | NOTICE | [608]: 1970-01-01T07:56:49.760Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.347 | NOTICE | [609]: 1970-01-01T07:56:49.763Z GPU thread:139946219454464 size:60293120 | 20260506 03:37:13.347 | NOTICE | [610]: 1970-01-01T07:56:49.763Z GPU thread:139946219454464 size:15728640 | 20260506 03:37:13.347 | NOTICE | [611]: 1970-01-01T07:56:49.763Z GPU thread:139946219454464 size:6291456 | 20260506 03:37:13.347 | NOTICE | [612]: 1970-01-01T07:56:49.763Z GPU thread:139946219454464 size:1048576 | 20260506 03:37:13.347 | NOTICE | [613]: 1970-01-01T07:56:49.763Z HOST thread:139946219454464 size:160432128 | 20260506 03:37:13.347 | NOTICE | [614]: 1970-01-01T07:56:49.865Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.347 | NOTICE | [615]: 1970-01-01T07:56:49.887Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.347 | NOTICE | [616]: 1970-01-01T07:56:49.909Z HOST thread:139946219454464 size:10485760 | 20260506 03:37:13.347 | NOTICE | [617]: 1970-01-01T07:56:49.915Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.347 | NOTICE | [618]: 1970-01-01T07:56:49.922Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.347 | NOTICE | [619]: 1970-01-01T07:56:49.929Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.347 | NOTICE | [620]: 1970-01-01T07:56:49.933Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.347 | NOTICE | [621]: 1970-01-01T07:56:49.938Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.347 | NOTICE | [622]: 1970-01-01T07:56:49.940Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.347 | NOTICE | [623]: 1970-01-01T07:56:49.943Z GPU thread:139946219454464 size:60293120 | 20260506 03:37:13.347 | NOTICE | [624]: 1970-01-01T07:56:49.943Z GPU thread:139946219454464 size:15728640 | 20260506 03:37:13.347 | NOTICE | [625]: 1970-01-01T07:56:49.943Z GPU thread:139946219454464 size:6291456 | 20260506 03:37:13.347 | NOTICE | [626]: 1970-01-01T07:56:49.943Z GPU thread:139946219454464 size:1048576 | 20260506 03:37:13.347 | NOTICE | [627]: 1970-01-01T07:56:49.943Z HOST thread:139946219454464 size:160432128 | 20260506 03:37:13.347 | NOTICE | [628]: 1970-01-01T07:56:50.045Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.347 | NOTICE | [629]: 1970-01-01T07:56:50.067Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.347 | NOTICE | [630]: 1970-01-01T07:56:50.088Z HOST thread:139946219454464 size:10485760 | 20260506 03:37:13.347 | NOTICE | [631]: 1970-01-01T07:56:50.095Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.347 | NOTICE | [632]: 1970-01-01T07:56:50.102Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.347 | NOTICE | [633]: 1970-01-01T07:56:50.108Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.347 | NOTICE | [634]: 1970-01-01T07:56:50.113Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.347 | NOTICE | [635]: 1970-01-01T07:56:50.117Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.347 | NOTICE | [636]: 1970-01-01T07:56:50.120Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.347 | NOTICE | [637]: 1970-01-01T07:56:50.123Z GPU thread:139946219454464 size:60293120 | 20260506 03:37:13.347 | NOTICE | [638]: 1970-01-01T07:56:50.123Z GPU thread:139946219454464 size:15728640 | 20260506 03:37:13.347 | NOTICE | [639]: 1970-01-01T07:56:50.123Z GPU thread:139946219454464 size:6291456 | 20260506 03:37:13.347 | NOTICE | [640]: 1970-01-01T07:56:50.123Z GPU thread:139946219454464 size:1048576 | 20260506 03:37:13.347 | NOTICE | [641]: 1970-01-01T07:56:50.123Z HOST thread:139946219454464 size:160432128 | 20260506 03:37:13.347 | NOTICE | [642]: 1970-01-01T07:56:50.225Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.347 | NOTICE | [643]: 1970-01-01T07:56:50.247Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.348 | NOTICE | [644]: 1970-01-01T07:56:50.269Z HOST thread:139946219454464 size:10485760 | 20260506 03:37:13.348 | NOTICE | [645]: 1970-01-01T07:56:50.275Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.348 | NOTICE | [646]: 1970-01-01T07:56:50.282Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.348 | NOTICE | [647]: 1970-01-01T07:56:50.288Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.348 | NOTICE | [648]: 1970-01-01T07:56:50.293Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.348 | NOTICE | [649]: 1970-01-01T07:56:50.297Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.348 | NOTICE | [650]: 1970-01-01T07:56:50.300Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.348 | NOTICE | [651]: 1970-01-01T07:56:50.303Z GPU thread:139946219454464 size:60293120 | 20260506 03:37:13.348 | NOTICE | [652]: 1970-01-01T07:56:50.303Z GPU thread:139946219454464 size:15728640 | 20260506 03:37:13.348 | NOTICE | [653]: 1970-01-01T07:56:50.303Z GPU thread:139946219454464 size:6291456 | 20260506 03:37:13.348 | NOTICE | [654]: 1970-01-01T07:56:50.303Z GPU thread:139946219454464 size:1048576 | 20260506 03:37:13.348 | NOTICE | [655]: 1970-01-01T07:56:50.303Z HOST thread:139946219454464 size:160432128 | 20260506 03:37:13.348 | NOTICE | [656]: 1970-01-01T07:56:50.405Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.348 | NOTICE | [657]: 1970-01-01T07:56:50.427Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.348 | NOTICE | [658]: 1970-01-01T07:56:50.449Z HOST thread:139946219454464 size:10485760 | 20260506 03:37:13.348 | NOTICE | [659]: 1970-01-01T07:56:50.455Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.348 | NOTICE | [660]: 1970-01-01T07:56:50.462Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.348 | NOTICE | [661]: 1970-01-01T07:56:50.468Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.348 | NOTICE | [662]: 1970-01-01T07:56:50.473Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.348 | NOTICE | [663]: 1970-01-01T07:56:50.477Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.348 | NOTICE | [664]: 1970-01-01T07:56:50.480Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.348 | NOTICE | [665]: 1970-01-01T07:56:50.483Z GPU thread:139946219454464 size:60293120 | 20260506 03:37:13.348 | NOTICE | [666]: 1970-01-01T07:56:50.483Z GPU thread:139946219454464 size:15728640 | 20260506 03:37:13.348 | NOTICE | [667]: 1970-01-01T07:56:50.483Z GPU thread:139946219454464 size:6291456 | 20260506 03:37:13.348 | NOTICE | [668]: 1970-01-01T07:56:50.483Z GPU thread:139946219454464 size:1048576 | 20260506 03:37:13.348 | NOTICE | [669]: 1970-01-01T07:56:50.483Z HOST thread:139946219454464 size:160432128 | 20260506 03:37:13.348 | NOTICE | [670]: 1970-01-01T07:56:50.585Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.348 | NOTICE | [671]: 1970-01-01T07:56:50.607Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.348 | NOTICE | [672]: 1970-01-01T07:56:50.629Z HOST thread:139946219454464 size:10485760 | 20260506 03:37:13.348 | NOTICE | [673]: 1970-01-01T07:56:50.635Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.348 | NOTICE | [674]: 1970-01-01T07:56:50.642Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.348 | NOTICE | [675]: 1970-01-01T07:56:50.649Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.348 | NOTICE | [676]: 1970-01-01T07:56:50.653Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.348 | NOTICE | [677]: 1970-01-01T07:56:50.657Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.348 | NOTICE | [678]: 1970-01-01T07:56:50.660Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.348 | NOTICE | [679]: 1970-01-01T07:56:50.663Z GPU thread:139946219454464 size:60293120 | 20260506 03:37:13.348 | NOTICE | [680]: 1970-01-01T07:56:50.663Z GPU thread:139946219454464 size:15728640 | 20260506 03:37:13.348 | NOTICE | [681]: 1970-01-01T07:56:50.663Z GPU thread:139946219454464 size:6291456 | 20260506 03:37:13.348 | NOTICE | [682]: 1970-01-01T07:56:50.663Z GPU thread:139946219454464 size:1048576 | 20260506 03:37:13.348 | NOTICE | [683]: 1970-01-01T07:56:50.663Z HOST thread:139946219454464 size:160432128 | 20260506 03:37:13.348 | NOTICE | [684]: 1970-01-01T07:56:50.765Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.348 | NOTICE | [685]: 1970-01-01T07:56:50.787Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.348 | NOTICE | [686]: 1970-01-01T07:56:50.809Z HOST thread:139946219454464 size:10485760 | 20260506 03:37:13.348 | NOTICE | [687]: 1970-01-01T07:56:50.816Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.348 | NOTICE | [688]: 1970-01-01T07:56:50.822Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.348 | NOTICE | [689]: 1970-01-01T07:56:50.829Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.348 | NOTICE | [690]: 1970-01-01T07:56:50.833Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.348 | NOTICE | [691]: 1970-01-01T07:56:50.838Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.348 | NOTICE | [692]: 1970-01-01T07:56:50.840Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.348 | NOTICE | [693]: 1970-01-01T07:56:50.843Z GPU thread:139946219454464 size:60293120 | 20260506 03:37:13.348 | NOTICE | [694]: 1970-01-01T07:56:50.843Z GPU thread:139946219454464 size:15728640 | 20260506 03:37:13.348 | NOTICE | [695]: 1970-01-01T07:56:50.843Z GPU thread:139946219454464 size:6291456 | 20260506 03:37:13.348 | NOTICE | [696]: 1970-01-01T07:56:50.843Z GPU thread:139946219454464 size:1048576 | 20260506 03:37:13.348 | NOTICE | [697]: 1970-01-01T07:56:50.843Z HOST thread:139946219454464 size:160432128 | 20260506 03:37:13.348 | NOTICE | [698]: 1970-01-01T07:56:50.946Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.348 | NOTICE | [699]: 1970-01-01T07:56:50.967Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.348 | NOTICE | [700]: 1970-01-01T07:56:50.989Z HOST thread:139946219454464 size:10485760 | 20260506 03:37:13.348 | NOTICE | [701]: 1970-01-01T07:56:50.996Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.348 | NOTICE | [702]: 1970-01-01T07:56:51.002Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.348 | NOTICE | [703]: 1970-01-01T07:56:51.009Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.348 | NOTICE | [704]: 1970-01-01T07:56:51.013Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.348 | NOTICE | [705]: 1970-01-01T07:56:51.018Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.348 | NOTICE | [706]: 1970-01-01T07:56:51.021Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.348 | NOTICE | [707]: 1970-01-01T07:56:51.023Z GPU thread:139946219454464 size:60293120 | 20260506 03:37:13.348 | NOTICE | [708]: 1970-01-01T07:56:51.024Z GPU thread:139946219454464 size:15728640 | 20260506 03:37:13.348 | NOTICE | [709]: 1970-01-01T07:56:51.024Z GPU thread:139946219454464 size:6291456 | 20260506 03:37:13.348 | NOTICE | [710]: 1970-01-01T07:56:51.024Z GPU thread:139946219454464 size:1048576 | 20260506 03:37:13.348 | NOTICE | [711]: 1970-01-01T07:56:51.024Z HOST thread:139946219454464 size:160432128 | 20260506 03:37:13.348 | NOTICE | [712]: 1970-01-01T07:56:51.126Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.348 | NOTICE | [713]: 1970-01-01T07:56:51.147Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.348 | NOTICE | [714]: 1970-01-01T07:56:51.169Z HOST thread:139946219454464 size:10485760 | 20260506 03:37:13.348 | NOTICE | [715]: 1970-01-01T07:56:51.176Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.348 | NOTICE | [716]: 1970-01-01T07:56:51.182Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.348 | NOTICE | [717]: 1970-01-01T07:56:51.189Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.348 | NOTICE | [718]: 1970-01-01T07:56:51.193Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.349 | NOTICE | [719]: 1970-01-01T07:56:51.198Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.349 | NOTICE | [720]: 1970-01-01T07:56:51.200Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.349 | NOTICE | [721]: 1970-01-01T07:56:51.203Z GPU thread:139946219454464 size:60293120 | 20260506 03:37:13.349 | NOTICE | [722]: 1970-01-01T07:56:51.203Z GPU thread:139946219454464 size:15728640 | 20260506 03:37:13.349 | NOTICE | [723]: 1970-01-01T07:56:51.204Z GPU thread:139946219454464 size:6291456 | 20260506 03:37:13.349 | NOTICE | [724]: 1970-01-01T07:56:51.204Z GPU thread:139946219454464 size:1048576 | 20260506 03:37:13.349 | NOTICE | [725]: 1970-01-01T07:56:51.204Z HOST thread:139946219454464 size:160432128 | 20260506 03:37:13.349 | NOTICE | [726]: 1970-01-01T07:56:51.306Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.349 | NOTICE | [727]: 1970-01-01T07:56:51.327Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.349 | NOTICE | [728]: 1970-01-01T07:56:51.349Z HOST thread:139946219454464 size:10485760 | 20260506 03:37:13.349 | NOTICE | [729]: 1970-01-01T07:56:51.356Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.349 | NOTICE | [730]: 1970-01-01T07:56:51.362Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.349 | NOTICE | [731]: 1970-01-01T07:56:51.369Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.349 | NOTICE | [732]: 1970-01-01T07:56:51.373Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.349 | NOTICE | [733]: 1970-01-01T07:56:51.378Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.349 | NOTICE | [734]: 1970-01-01T07:56:51.381Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.349 | NOTICE | [735]: 1970-01-01T07:56:51.383Z GPU thread:139946219454464 size:60293120 | 20260506 03:37:13.349 | NOTICE | [736]: 1970-01-01T07:56:51.384Z GPU thread:139946219454464 size:15728640 | 20260506 03:37:13.349 | NOTICE | [737]: 1970-01-01T07:56:51.384Z GPU thread:139946219454464 size:6291456 | 20260506 03:37:13.349 | NOTICE | [738]: 1970-01-01T07:56:51.384Z GPU thread:139946219454464 size:1048576 | 20260506 03:37:13.349 | NOTICE | [739]: 1970-01-01T07:56:51.384Z HOST thread:139946219454464 size:160432128 | 20260506 03:37:13.349 | NOTICE | [740]: 1970-01-01T07:56:51.486Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.349 | NOTICE | [741]: 1970-01-01T07:56:51.508Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.349 | NOTICE | [742]: 1970-01-01T07:56:51.530Z HOST thread:139946219454464 size:10485760 | 20260506 03:37:13.349 | NOTICE | [743]: 1970-01-01T07:56:51.537Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.349 | NOTICE | [744]: 1970-01-01T07:56:51.543Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.349 | NOTICE | [745]: 1970-01-01T07:56:51.550Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.349 | NOTICE | [746]: 1970-01-01T07:56:51.554Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.349 | NOTICE | [747]: 1970-01-01T07:56:51.559Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.349 | NOTICE | [748]: 1970-01-01T07:56:51.562Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.349 | NOTICE | [749]: 1970-01-01T07:56:51.564Z GPU thread:139946219454464 size:60293120 | 20260506 03:37:13.349 | NOTICE | [750]: 1970-01-01T07:56:51.565Z GPU thread:139946219454464 size:15728640 | 20260506 03:37:13.349 | NOTICE | [751]: 1970-01-01T07:56:51.565Z GPU thread:139946219454464 size:6291456 | 20260506 03:37:13.349 | NOTICE | [752]: 1970-01-01T07:56:51.565Z GPU thread:139946219454464 size:1048576 | 20260506 03:37:13.349 | NOTICE | [753]: 1970-01-01T07:56:51.565Z HOST thread:139946219454464 size:160432128 | 20260506 03:37:13.349 | NOTICE | [754]: 1970-01-01T07:56:51.667Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.349 | NOTICE | [755]: 1970-01-01T07:56:51.688Z HOST thread:139946219454464 size:33030144 | 20260506 03:37:13.349 | NOTICE | [756]: 1970-01-01T07:56:51.710Z HOST thread:139946219454464 size:10485760 | 20260506 03:37:13.349 | NOTICE | [757]: 1970-01-01T07:56:51.717Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.349 | NOTICE | [758]: 1970-01-01T07:56:51.723Z HOST thread:139946219454464 size:9961472 | 20260506 03:37:13.349 | NOTICE | [759]: 1970-01-01T07:56:51.730Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.349 | NOTICE | [760]: 1970-01-01T07:56:51.734Z HOST thread:139946219454464 size:6815744 | 20260506 03:37:13.349 | NOTICE | [761]: 1970-01-01T07:56:51.739Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.349 | NOTICE | [762]: 1970-01-01T07:56:51.742Z HOST thread:139946219454464 size:4194304 | 20260506 03:37:13.349 | NOTICE | [763]: 1970-01-01T07:57:34.594Z GPU thread:139904749768704 size:134217728 | 20260506 03:37:13.349 | NOTICE | [764]: 1970-01-01T07:57:34.611Z HOST thread:139904749768704 size:1 | 20260506 03:37:13.349 | NOTICE | [765]: 1970-01-01T07:57:34.611Z GPU thread:139904745570304 size:134217728 | 20260506 03:37:13.349 | NOTICE | [766]: 1970-01-01T07:57:34.611Z HOST thread:139904749768704 size:256 | 20260506 03:37:13.349 | NOTICE | [767]: 1970-01-01T07:57:34.612Z GPU thread:139904749768704 size:268435456 | 20260506 03:37:13.349 | NOTICE | [768]: 1970-01-01T07:57:34.617Z GPU thread:139904749768704 size:24641536 | 20260506 03:37:13.349 | NOTICE | [769]: 1970-01-01T07:57:34.617Z GPU thread:139904749768704 size:134217728 | 20260506 03:37:13.349 | NOTICE | [770]: 1970-01-01T07:57:34.617Z GPU thread:139904749768704 size:134217728 | 20260506 03:37:13.349 | NOTICE | [771]: 1970-01-01T07:57:34.618Z GPU thread:139904745570304 size:160432128 | 20260506 03:37:13.349 | NOTICE | [772]: 1970-01-01T07:57:34.624Z GPU thread:139904745570304 size:33030144 | 20260506 03:37:13.349 | NOTICE | [773]: 1970-01-01T07:57:34.625Z GPU thread:139904745570304 size:33030144 | 20260506 03:37:13.349 | NOTICE | [774]: 1970-01-01T07:57:34.627Z GPU thread:139904745570304 size:10485760 | 20260506 03:37:13.349 | NOTICE | [775]: 1970-01-01T07:57:34.629Z GPU thread:139904745570304 size:9961472 | 20260506 03:37:13.349 | NOTICE | [776]: 1970-01-01T07:57:34.630Z GPU thread:139904745570304 size:9961472 | 20260506 03:37:13.349 | NOTICE | [777]: 1970-01-01T07:57:34.632Z GPU thread:139904745570304 size:6815744 | 20260506 03:37:13.349 | NOTICE | [778]: 1970-01-01T07:57:34.640Z GPU thread:139904745570304 size:6815744 | 20260506 03:37:13.349 | NOTICE | [779]: 1970-01-01T07:57:34.646Z GPU thread:139904745570304 size:4194304 | 20260506 03:37:13.349 | NOTICE | [780]: 1970-01-01T07:57:34.646Z GPU thread:139904745570304 size:4194304 | 20260506 03:37:13.349 | NOTICE | [781]: 1970-01-01T07:57:34.647Z GPU thread:139904758165504 size:134217728 | 20260506 03:37:13.349 | NOTICE | [782]: 1970-01-01T07:57:34.648Z HOST thread:139904749768704 size:268435456 | 20260506 03:37:13.349 | NOTICE | [783]: 1970-01-01T07:57:34.653Z GPU thread:139904758165504 size:160432128 | 20260506 03:37:13.349 | NOTICE | [784]: 1970-01-01T07:57:34.809Z HOST thread:139904749768704 size:12582912 | 20260506 03:37:13.349 | NOTICE | [785]: 1970-01-01T07:57:34.823Z GPU thread:139904758165504 size:33030144 | 20260506 03:37:13.349 | NOTICE | [786]: 1970-01-01T07:57:34.825Z GPU thread:139904758165504 size:33030144 | 20260506 03:37:13.349 | NOTICE | [787]: 1970-01-01T07:57:34.826Z GPU thread:139904758165504 size:10485760 | 20260506 03:37:13.349 | NOTICE | [788]: 1970-01-01T07:57:34.826Z GPU thread:139904758165504 size:9961472 | 20260506 03:37:13.349 | NOTICE | [789]: 1970-01-01T07:57:34.827Z GPU thread:139904758165504 size:9961472 | 20260506 03:37:13.349 | NOTICE | [790]: 1970-01-01T07:57:34.827Z GPU thread:139904758165504 size:6815744 | 20260506 03:37:13.349 | NOTICE | [791]: 1970-01-01T07:57:34.828Z GPU thread:139904758165504 size:6815744 | 20260506 03:37:13.349 | NOTICE | [792]: 1970-01-01T07:57:34.828Z GPU thread:139904758165504 size:4194304 | 20260506 03:37:13.350 | NOTICE | [793]: 1970-01-01T07:57:34.828Z GPU thread:139904758165504 size:4194304 | 20260506 03:37:13.350 | NOTICE | [794]: 1970-01-01T07:57:34.835Z HOST thread:139904749768704 size:540016640 | 20260506 03:37:13.350 | NOTICE | [795]: 1970-01-01T07:57:35.155Z HOST thread:139904749768704 size:1048576 | 20260506 03:37:13.350 | NOTICE | [796]: 1970-01-01T07:57:35.155Z HOST thread:139904749768704 size:1048576 | 20260506 03:37:13.350 | NOTICE | [797]: 1970-01-01T07:57:35.156Z HOST thread:139904749768704 size:4194304 | 20260506 03:37:13.350 | NOTICE | [798]: 1970-01-01T07:57:35.159Z HOST thread:139904749768704 size:1 | 20260506 03:37:13.350 | NOTICE | [799]: 1970-01-01T07:57:35.159Z HOST thread:139904749768704 size:2097156 | 20260506 03:37:13.350 | NOTICE | [800]: 1970-01-01T07:58:10.629Z HOST thread:139904758165504 size:1 | 20260506 03:37:13.350 | NOTICE | [801]: 1970-01-01T07:58:10.629Z HOST thread:139904758165504 size:2097156 | 20260506 03:37:13.350 | NOTICE | [802]: 1970-01-01T07:59:07.124Z GPU thread:139904753967104 size:8912896 | 20260506 03:37:13.350 | NOTICE | [803]: 1970-01-01T07:59:07.310Z GPU thread:139904753967104 size:8912896 | 20260506 03:37:13.350 | NOTICE | [804]: 1970-01-01T07:59:07.489Z GPU thread:139904753967104 size:8912896 | 20260506 03:37:13.350 | NOTICE | [805]: 1970-01-01T07:59:07.646Z GPU thread:139904737173504 size:8912896 | 20260506 03:37:13.350 | NOTICE | [806]: 1970-01-01T07:59:07.849Z GPU thread:139904762363904 size:8912896 | 20260506 03:37:13.350 | NOTICE | [807]: 1970-01-01T07:59:08.028Z GPU thread:139904732975104 size:8912896 | 20260506 03:37:13.350 | NOTICE | [808]: 1970-01-01T07:59:08.209Z GPU thread:139904732975104 size:8912896 | 20260506 03:37:13.350 | NOTICE | [809]: 1970-01-01T07:59:08.389Z GPU thread:139904732975104 size:8912896 | 20260506 03:37:13.350 | NOTICE | [810]: 1970-01-01T07:59:08.568Z GPU thread:139904732975104 size:8912896 | 20260506 03:37:13.350 | NOTICE | [811]: 1970-01-01T07:59:08.747Z GPU thread:139904732975104 size:8912896 | 20260506 03:37:13.350 | NOTICE | [812]: 1970-01-01T07:59:08.926Z GPU thread:139904732975104 size:8912896 | 20260506 03:37:13.350 | NOTICE | [813]: 1970-01-01T07:59:09.106Z GPU thread:139904732975104 size:8912896 | 20260506 03:37:13.350 | NOTICE | [814]: 1970-01-01T07:59:12.261Z GPU thread:139904741371904 size:8912896 | 20260506 03:37:13.350 | NOTICE | [815]: 1970-01-01T07:59:12.448Z GPU thread:139904741371904 size:8912896 | 20260506 03:37:13.350 | NOTICE | [816]: 1970-01-01T07:59:12.628Z GPU thread:139904741371904 size:8912896 | 20260506 03:37:13.350 | NOTICE | [817]: 1970-01-01T07:59:12.808Z GPU thread:139904732975104 size:8912896 | 20260506 03:37:13.350 | NOTICE | [818]: 1970-01-01T07:59:12.988Z GPU thread:139904732975104 size:8912896 | 20260506 03:37:13.350 | NOTICE | [819]: 1970-01-01T07:59:13.170Z GPU thread:139904732975104 size:8912896 | 20260506 03:37:13.350 | NOTICE | [820]: 1970-01-01T07:59:13.351Z GPU thread:139904732975104 size:8912896 | 20260506 03:37:13.350 | NOTICE | [821]: 1970-01-01T07:59:13.533Z GPU thread:139904732975104 size:8912896 | 20260506 03:37:13.350 | NOTICE | [822]: 1970-01-01T07:59:13.713Z GPU thread:139904732975104 size:8912896 | 20260506 03:37:13.350 | NOTICE | [823]: 1970-01-01T07:59:13.893Z GPU thread:139904732975104 size:8912896 | 20260506 03:37:13.350 | NOTICE | [824]: 1970-01-01T07:59:14.073Z GPU thread:139904732975104 size:8912896 | 20260506 03:37:13.350 | NOTICE | [825]: 1970-01-01T07:59:14.254Z GPU thread:139904732975104 size:8912896 | 20260506 03:37:13.350 | NOTICE | [826]: 1970-01-01T07:59:17.403Z GPU thread:139904737173504 size:8912896 | 20260506 03:37:13.350 | NOTICE | [827]: 1970-01-01T07:59:17.586Z GPU thread:139904737173504 size:8912896 | 20260506 03:37:13.350 | NOTICE | [828]: 1970-01-01T07:59:17.768Z GPU thread:139904737173504 size:8912896 | 20260506 03:37:13.350 | NOTICE | [829]: 1970-01-01T07:59:17.950Z GPU thread:139904737173504 size:8912896 | 20260506 03:37:13.350 | NOTICE | [830]: 1970-01-01T07:59:18.133Z GPU thread:139904737173504 size:8912896 | 20260506 03:37:13.350 | NOTICE | [831]: 1970-01-01T07:59:18.317Z GPU thread:139904753967104 size:8912896 | 20260506 03:37:13.350 | NOTICE | [832]: 1970-01-01T07:59:18.500Z GPU thread:139904753967104 size:8912896 | 20260506 03:37:13.350 | NOTICE | [833]: 1970-01-01T07:59:18.684Z GPU thread:139904753967104 size:8912896 | 20260506 03:37:13.350 | NOTICE | [834]: 1970-01-01T07:59:18.867Z GPU thread:139904753967104 size:8912896 | 20260506 03:37:13.350 | NOTICE | [835]: 1970-01-01T07:59:19.050Z GPU thread:139904753967104 size:8912896 | 20260506 03:37:13.350 | NOTICE | [836]: 1970-01-01T07:59:19.232Z GPU thread:139904753967104 size:8912896 | 20260506 03:37:13.350 | NOTICE | [837]: 1970-01-01T07:59:19.417Z GPU thread:139904753967104 size:8912896 | 20260506 03:37:13.350 | NOTICE | [838]: 1970-01-01T07:59:22.541Z GPU thread:139904732975104 size:8912896 | 20260506 03:37:13.350 | NOTICE | [839]: 1970-01-01T07:59:22.724Z GPU thread:139904732975104 size:8912896 | 20260506 03:37:13.350 | NOTICE | [840]: 1970-01-01T07:59:22.907Z GPU thread:139904732975104 size:8912896 | 20260506 03:37:13.350 | NOTICE | [841]: 1970-01-01T07:59:23.091Z GPU thread:139904732975104 size:8912896 | 20260506 03:37:13.350 | NOTICE | [842]: 1970-01-01T07:59:23.276Z GPU thread:139904732975104 size:8912896 | 20260506 03:37:13.350 | NOTICE | [843]: 1970-01-01T07:59:23.460Z GPU thread:139904732975104 size:8912896 | 20260506 03:37:13.350 | NOTICE | [844]: 1970-01-01T07:59:23.644Z GPU thread:139904732975104 size:8912896 | 20260506 03:37:13.350 | NOTICE | [845]: 1970-01-01T07:59:23.828Z GPU thread:139904766562304 size:8912896 | 20260506 03:37:13.350 | NOTICE | [846]: 1970-01-01T07:59:24.010Z GPU thread:139904766562304 size:8912896 | 20260506 03:37:13.350 | NOTICE | [847]: 1970-01-01T07:59:24.192Z GPU thread:139904766562304 size:8912896 | 20260506 03:37:13.350 | NOTICE | [848]: 1970-01-01T07:59:24.376Z GPU thread:139904766562304 size:8912896 | 20260506 03:37:13.350 | NOTICE | [849]: 1970-01-01T07:59:24.559Z GPU thread:139904766562304 size:8912896 | 20260506 03:37:13.350 | NOTICE | [850]: 1970-01-01T07:59:27.680Z GPU thread:139904728776704 size:8912896 | 20260506 03:37:13.350 | NOTICE | [851]: 1970-01-01T07:59:27.865Z GPU thread:139904728776704 size:8912896 | 20260506 03:37:13.350 | NOTICE | [852]: 1970-01-01T07:59:28.049Z GPU thread:139904728776704 size:8912896 | 20260506 03:37:13.350 | NOTICE | [853]: 1970-01-01T07:59:28.235Z GPU thread:139904728776704 size:8912896 | 20260506 03:37:13.350 | NOTICE | [854]: 1970-01-01T07:59:28.420Z GPU thread:139904728776704 size:8912896 | 20260506 03:37:13.350 | NOTICE | [855]: 1970-01-01T07:59:28.606Z GPU thread:139904728776704 size:8912896 | 20260506 03:37:13.350 | NOTICE | [856]: 1970-01-01T07:59:28.791Z GPU thread:139904728776704 size:8912896 | 20260506 03:37:13.350 | NOTICE | [857]: 1970-01-01T07:59:28.976Z GPU thread:139904728776704 size:8912896 | 20260506 03:37:13.350 | NOTICE | [858]: 1970-01-01T07:59:29.160Z GPU thread:139904728776704 size:8912896 | 20260506 03:37:13.350 | NOTICE | [859]: 1970-01-01T07:59:29.344Z GPU thread:139904737173504 size:8912896 | 20260506 03:37:13.350 | NOTICE | [860]: 1970-01-01T07:59:29.529Z GPU thread:139904737173504 size:8912896 | 20260506 03:37:13.350 | NOTICE | [861]: 1970-01-01T07:59:29.716Z GPU thread:139904737173504 size:8912896 | 20260506 03:37:13.350 | NOTICE | [862]: 1970-01-01T07:59:32.819Z GPU thread:139904766562304 size:8912896 | 20260506 03:37:13.350 | NOTICE | [863]: 1970-01-01T07:59:33.002Z GPU thread:139904766562304 size:8912896 | 20260506 03:37:13.350 | NOTICE | [864]: 1970-01-01T07:59:33.187Z GPU thread:139904728776704 size:8912896 | 20260506 03:37:13.351 | NOTICE | [865]: 1970-01-01T07:59:33.373Z GPU thread:139904758165504 size:8912896 | 20260506 03:37:13.351 | NOTICE | [866]: 1970-01-01T07:59:33.558Z GPU thread:139904766562304 size:8912896 | 20260506 03:37:13.351 | NOTICE | [867]: 1970-01-01T07:59:33.743Z GPU thread:139904766562304 size:8912896 | 20260506 03:37:13.351 | NOTICE | [868]: 1970-01-01T07:59:33.928Z GPU thread:139904758165504 size:8912896 | 20260506 03:37:13.351 | NOTICE | [869]: 1970-01-01T07:59:34.114Z GPU thread:139904741371904 size:8912896 | 20260506 03:37:13.351 | NOTICE | [870]: 1970-01-01T07:59:34.298Z GPU thread:139904758165504 size:8912896 | 20260506 03:37:13.351 | NOTICE | [871]: 1970-01-01T07:59:34.482Z GPU thread:139904770760704 size:8912896 | 20260506 03:37:13.351 | NOTICE | [872]: 1970-01-01T07:59:34.667Z GPU thread:139904758165504 size:8912896 | 20260506 03:37:13.351 | NOTICE | [873]: 1970-01-01T07:59:34.853Z GPU thread:139904745570304 size:8912896 | 20260506 03:37:13.351 | NOTICE | [874]: 1970-01-01T07:59:37.959Z GPU thread:139904737173504 size:8912896 | 20260506 03:37:13.351 | NOTICE | [875]: 1970-01-01T07:59:38.147Z GPU thread:139904737173504 size:8912896 | 20260506 03:37:13.351 | NOTICE | [876]: 1970-01-01T07:59:38.334Z GPU thread:139904737173504 size:8912896 | 20260506 03:37:13.351 | NOTICE | [877]: 1970-01-01T07:59:38.521Z GPU thread:139904737173504 size:8912896 | 20260506 03:37:13.351 | NOTICE | [878]: 1970-01-01T07:59:38.708Z GPU thread:139904737173504 size:8912896 | 20260506 03:37:13.351 | NOTICE | [879]: 1970-01-01T07:59:38.896Z GPU thread:139904737173504 size:8912896 | 20260506 03:37:13.351 | NOTICE | [880]: 1970-01-01T07:59:39.084Z GPU thread:139904737173504 size:8912896 | 20260506 03:37:13.351 | NOTICE | [881]: 1970-01-01T07:59:39.274Z GPU thread:139904737173504 size:8912896 | 20260506 03:37:13.351 | NOTICE | [882]: 1970-01-01T07:59:39.461Z GPU thread:139904737173504 size:8912896 | 20260506 03:37:13.351 | NOTICE | [883]: 1970-01-01T07:59:39.648Z GPU thread:139904737173504 size:8912896 | 20260506 03:37:13.351 | NOTICE | [884]: 1970-01-01T07:59:39.834Z GPU thread:139904732975104 size:8912896 | 20260506 03:37:13.351 | NOTICE | [885]: 1970-01-01T07:59:40.021Z GPU thread:139904732975104 size:8912896 | 20260506 03:37:13.351 | NOTICE | [886]: 1970-01-01T07:59:43.098Z GPU thread:139904762363904 size:8912896 | 20260506 03:37:13.351 | NOTICE | [887]: 1970-01-01T07:59:43.286Z GPU thread:139904762363904 size:8912896 | 20260506 03:37:13.351 | NOTICE | [888]: 1970-01-01T07:59:43.473Z GPU thread:139904762363904 size:8912896 | 20260506 03:37:13.351 | NOTICE | [889]: 1970-01-01T07:59:43.659Z GPU thread:139904762363904 size:8912896 | 20260506 03:37:13.351 | NOTICE | [890]: 1970-01-01T07:59:43.848Z GPU thread:139904762363904 size:8912896 | 20260506 03:37:13.351 | NOTICE | [891]: 1970-01-01T07:59:44.036Z GPU thread:139904762363904 size:8912896 | 20260506 03:37:13.351 | NOTICE | [892]: 1970-01-01T07:59:44.224Z GPU thread:139904762363904 size:8912896 | 20260506 03:37:13.351 | NOTICE | [893]: 1970-01-01T07:59:44.413Z GPU thread:139904762363904 size:8912896 | 20260506 03:37:13.351 | NOTICE | [894]: 1970-01-01T07:59:44.600Z GPU thread:139904762363904 size:8912896 | 20260506 03:37:13.351 | NOTICE | [895]: 1970-01-01T07:59:44.788Z GPU thread:139904762363904 size:8912896 | 20260506 03:37:13.351 | NOTICE | [896]: 1970-01-01T07:59:44.976Z GPU thread:139904762363904 size:8912896 | 20260506 03:37:13.351 | NOTICE | [897]: 1970-01-01T07:59:45.168Z GPU thread:139904762363904 size:8912896 | 20260506 03:37:13.351 | NOTICE | [898]: 1970-01-01T07:59:48.237Z GPU thread:139904732975104 size:8912896 | 20260506 03:37:13.351 | NOTICE | [899]: 1970-01-01T07:59:53.414Z GPU thread:139904741371904 size:268435456 | 20260506 03:37:13.351 | NOTICE | [900]: 1970-01-01T07:59:53.415Z GPU thread:139904741371904 size:12582912 | 20260506 03:37:13.351 | NOTICE | [901]: 1970-01-01T07:59:53.415Z GPU thread:139904741371904 size:1048576 | 20260506 03:37:13.351 | NOTICE | [902]: 1970-01-01T07:59:53.415Z GPU thread:139904741371904 size:1107296256 | 20260506 03:37:13.351 | NOTICE | [903]: 1970-01-01T07:59:53.417Z GPU thread:139904741371904 size:279969792 | 20260506 03:37:13.351 | NOTICE | [904]: 1970-01-01T07:59:53.436Z GPU thread:139904741371904 size:540016640 | 20260506 03:37:13.351 | NOTICE | [905]: 1970-01-01T07:59:53.437Z GPU thread:139904741371904 size:1048576 | 20260506 03:37:13.351 | NOTICE | [906]: 1970-01-01T07:59:53.437Z GPU thread:139904741371904 size:1048576 | 20260506 03:37:13.351 | NOTICE | [907]: 1970-01-01T07:59:53.437Z GPU thread:139904741371904 size:4194304 | 20260506 03:37:13.351 | NOTICE | [908]: 1970-01-01T07:59:53.442Z GPU thread:139904741371904 size:108003328 | 20260506 03:37:13.351 | NOTICE | [909]: 1970-01-01T07:59:53.443Z GPU thread:139904741371904 size:2097156 | 20260506 03:37:13.351 | NOTICE | [910]: 1970-01-01T07:59:53.445Z HOST thread:139904741371904 size:108003328 | 20260506 03:37:13.351 | NOTICE | [911]: 1970-01-01T08:00:03.961Z HOST thread:139904732975104 size:108003328 | 20260506 03:37:13.351 | NOTICE | [912]: 1970-01-01T08:00:13.954Z GPU thread:139904749768704 size:38273024 | 20260506 03:37:13.351 | NOTICE | [913]: 1970-01-01T08:00:13.955Z GPU thread:139904749768704 size:11272192 | 20260506 03:37:13.351 | NOTICE | [914]: 1970-01-01T08:00:13.964Z HOST thread:139904749768704 size:38273024 | 20260506 03:37:13.351 | NOTICE | [915]: 1970-01-01T08:00:13.989Z GPU thread:139904741371904 size:11272192 | 20260506 03:37:13.351 | NOTICE | [916]: 1970-01-01T08:00:14.004Z HOST thread:139904741371904 size:38273024 | 20260506 03:37:13.351 | NOTICE | [917]: 1970-01-01T08:00:14.030Z GPU thread:139904766562304 size:11272192 | 20260506 03:37:13.351 | NOTICE | [918]: 1970-01-01T08:00:14.064Z GPU thread:139904753967104 size:11272192 | 20260506 03:37:13.351 | NOTICE | [919]: 1970-01-01T08:00:14.067Z HOST thread:139904753967104 size:108003328 | 20260506 03:37:13.351 | NOTICE | [920]: 1970-01-01T08:00:14.228Z GPU thread:139904749768704 size:11272192 | 20260506 03:37:13.351 | NOTICE | [921]: 1970-01-01T08:00:14.278Z HOST thread:139904741371904 size:2097156 | 20260506 03:37:13.351 | NOTICE | [922]: 1970-01-01T08:00:14.327Z GPU thread:139904762363904 size:11272192 | 20260506 03:37:13.351 | NOTICE | [923]: 1970-01-01T08:00:14.353Z GPU thread:139904732975104 size:11272192 | 20260506 03:37:13.351 | NOTICE | [924]: 1970-01-01T08:00:14.380Z GPU thread:139904758165504 size:11272192 | 20260506 03:37:13.351 | NOTICE | [925]: 1970-01-01T08:00:14.387Z HOST thread:139904758165504 size:38273024 | 20260506 03:37:13.351 | NOTICE | [926]: 1970-01-01T08:00:14.413Z GPU thread:139904749768704 size:11272192 | 20260506 03:37:13.351 | NOTICE | [927]: 1970-01-01T08:00:14.508Z GPU thread:139904737173504 size:11272192 | 20260506 03:37:13.351 | NOTICE | [928]: 1970-01-01T08:00:14.606Z GPU thread:139904766562304 size:11272192 | 20260506 03:37:13.351 | NOTICE | [929]: 1970-01-01T08:00:14.632Z GPU thread:139904770760704 size:11272192 | 20260506 03:37:13.351 | NOTICE | [930]: 1970-01-01T08:00:14.657Z GPU thread:139904753967104 size:11272192 | 20260506 03:37:13.351 | NOTICE | [931]: 1970-01-01T08:00:14.684Z GPU thread:139904732975104 size:11272192 | 20260506 03:37:13.351 | NOTICE | [932]: 1970-01-01T08:00:14.778Z GPU thread:139904741371904 size:11272192 | 20260506 03:37:13.351 | NOTICE | [933]: 1970-01-01T08:00:14.876Z GPU thread:139904745570304 size:11272192 | 20260506 03:37:13.351 | NOTICE | [934]: 1970-01-01T08:00:14.902Z GPU thread:139904762363904 size:11272192 | 20260506 03:37:13.351 | NOTICE | [935]: 1970-01-01T08:00:14.928Z GPU thread:139904766562304 size:11272192 | 20260506 03:37:13.352 | NOTICE | [936]: 1970-01-01T08:00:14.954Z GPU thread:139904753967104 size:11272192 | 20260506 03:37:13.352 | NOTICE | [937]: 1970-01-01T08:00:15.051Z GPU thread:139904732975104 size:11272192 | 20260506 03:37:13.352 | NOTICE | [938]: 1970-01-01T08:00:15.150Z GPU thread:139904766562304 size:11272192 | 20260506 03:37:13.352 | NOTICE | [939]: 1970-01-01T08:00:15.176Z GPU thread:139904728776704 size:11272192 | 20260506 03:37:13.352 | NOTICE | [940]: 1970-01-01T08:00:15.203Z GPU thread:139904758165504 size:11272192 | 20260506 03:37:13.352 | NOTICE | [941]: 1970-01-01T08:00:15.229Z GPU thread:139904737173504 size:11272192 | 20260506 03:37:13.352 | NOTICE | [942]: 1970-01-01T08:00:15.323Z GPU thread:139904745570304 size:11272192 | 20260506 03:37:13.352 | NOTICE | [943]: 1970-01-01T08:00:15.421Z GPU thread:139904728776704 size:11272192 | 20260506 03:37:13.352 | NOTICE | [944]: 1970-01-01T08:00:15.446Z GPU thread:139904758165504 size:11272192 | 20260506 03:37:13.352 | NOTICE | [945]: 1970-01-01T08:00:15.472Z GPU thread:139904753967104 size:11272192 | 20260506 03:37:13.352 | NOTICE | [946]: 1970-01-01T08:00:15.499Z GPU thread:139904745570304 size:11272192 | 20260506 03:37:13.352 | NOTICE | [947]: 1970-01-01T08:00:15.590Z GPU thread:139904737173504 size:11272192 | 20260506 03:37:13.352 | NOTICE | [948]: 1970-01-01T08:00:15.685Z GPU thread:139904728776704 size:11272192 | 20260506 03:37:13.352 | NOTICE | [949]: 1970-01-01T08:00:15.710Z GPU thread:139904749768704 size:11272192 | 20260506 03:37:13.352 | NOTICE | [950]: 1970-01-01T08:00:15.736Z GPU thread:139904762363904 size:11272192 | 20260506 03:37:13.352 | NOTICE | [951]: 1970-01-01T08:00:15.761Z GPU thread:139904758165504 size:11272192 | 20260506 03:37:13.352 | NOTICE | [952]: 1970-01-01T08:00:15.860Z GPU thread:139904766562304 size:11272192 | 20260506 03:37:13.352 | NOTICE | [953]: 1970-01-01T08:00:15.960Z GPU thread:139904749768704 size:11272192 | 20260506 03:37:13.352 | NOTICE | [954]: 1970-01-01T08:00:15.986Z GPU thread:139904753967104 size:11272192 | 20260506 03:37:13.352 | NOTICE | [955]: 1970-01-01T08:00:16.013Z GPU thread:139904745570304 size:11272192 | 20260506 03:37:13.352 | NOTICE | [956]: 1970-01-01T08:00:16.039Z GPU thread:139904741371904 size:11272192 | 20260506 03:37:13.352 | NOTICE | [957]: 1970-01-01T08:00:16.042Z HOST thread:139904741371904 size:108003328 | 20260506 03:37:13.352 | NOTICE | [958]: 1970-01-01T08:00:16.203Z GPU thread:139904732975104 size:11272192 | 20260506 03:37:13.352 | NOTICE | [959]: 1970-01-01T08:00:16.302Z GPU thread:139904753967104 size:11272192 | 20260506 03:37:13.352 | NOTICE | [960]: 1970-01-01T08:00:16.327Z GPU thread:139904770760704 size:11272192 | 20260506 03:37:13.352 | NOTICE | [961]: 1970-01-01T08:00:16.354Z GPU thread:139904758165504 size:11272192 | 20260506 03:37:13.352 | NOTICE | [962]: 1970-01-01T08:00:16.380Z GPU thread:139904749768704 size:11272192 | 20260506 03:37:13.352 | NOTICE | [963]: 1970-01-01T08:00:16.474Z GPU thread:139904745570304 size:11272192 | 20260506 03:37:13.352 | NOTICE | [964]: 1970-01-01T08:00:16.571Z GPU thread:139904737173504 size:11272192 | 20260506 03:37:13.352 | NOTICE | [965]: 1970-01-01T08:00:16.597Z GPU thread:139904728776704 size:11272192 | 20260506 03:37:13.352 | NOTICE | [966]: 1970-01-01T08:00:16.622Z GPU thread:139904753967104 size:11272192 | 20260506 03:37:13.352 | NOTICE | [967]: 1970-01-01T08:00:16.649Z GPU thread:139904741371904 size:11272192 | 20260506 03:37:13.352 | NOTICE | [968]: 1970-01-01T08:00:16.744Z GPU thread:139904770760704 size:11272192 | 20260506 03:37:13.352 | NOTICE | [969]: 1970-01-01T08:00:16.841Z GPU thread:139904732975104 size:11272192 | 20260506 03:37:13.352 | NOTICE | [970]: 1970-01-01T08:00:16.867Z GPU thread:139904753967104 size:11272192 | 20260506 03:37:13.352 | NOTICE | [971]: 1970-01-01T08:00:16.893Z GPU thread:139904762363904 size:11272192 | 20260506 03:37:13.352 | NOTICE | [972]: 1970-01-01T08:00:16.919Z GPU thread:139904737173504 size:11272192 | 20260506 03:37:13.352 | NOTICE | [973]: 1970-01-01T08:00:17.014Z GPU thread:139904758165504 size:11272192 | 20260506 03:37:13.352 | NOTICE | [974]: 1970-01-01T08:00:17.112Z GPU thread:139904770760704 size:11272192 | 20260506 03:37:13.352 | NOTICE | [975]: 1970-01-01T08:00:17.138Z GPU thread:139904728776704 size:11272192 | 20260506 03:37:13.352 | NOTICE | [976]: 1970-01-01T08:00:17.165Z GPU thread:139904732975104 size:11272192 | 20260506 03:37:13.352 | NOTICE | [977]: 1970-01-01T08:00:17.190Z GPU thread:139904758165504 size:11272192 | 20260506 03:37:13.352 | NOTICE | [978]: 1970-01-01T08:00:17.281Z GPU thread:139904766562304 size:11272192 | 20260506 03:37:13.352 | NOTICE | [979]: 1970-01-01T08:00:34.491Z HOST thread:139904762363904 size:24 | 20260506 03:37:13.352 | NOTICE | [980]: 1970-01-01T08:00:34.491Z GPU thread:139904762363904 size:24 | 20260506 03:37:13.352 | NOTICE | [981]: 1970-01-01T08:00:55.859Z HOST thread:139904741371904 size:2097156 | 20260506 03:37:13.352 | NOTICE | [982]: 1970-01-01T08:00:56.233Z HOST thread:139904741371904 size:38273024 | 20260506 03:37:13.352 | NOTICE | [983]: 1970-01-01T08:00:56.924Z GPU thread:139904732975104 size:11272192 | 20260506 03:37:13.352 | NOTICE | [984]: 1970-01-01T08:00:56.950Z GPU thread:139904758165504 size:11272192 | 20260506 03:37:13.352 | NOTICE | [985]: 1970-01-01T08:00:56.976Z GPU thread:139904737173504 size:11272192 | 20260506 03:37:13.352 | NOTICE | [986]: 1970-01-01T08:00:57.002Z GPU thread:139904741371904 size:11272192 | 20260506 03:37:13.352 | NOTICE | [987]: 1970-01-01T08:00:57.028Z GPU thread:139904766562304 size:11272192 | 20260506 03:37:13.352 | NOTICE | [988]: 1970-01-01T08:00:57.144Z GPU thread:139904728776704 size:11272192 | 20260506 03:37:13.352 | NOTICE | [989]: 1970-01-01T08:00:57.171Z GPU thread:139904770760704 size:11272192 | 20260506 03:37:13.352 | NOTICE | [990]: 1970-01-01T08:00:57.197Z GPU thread:139904758165504 size:11272192 | 20260506 03:37:13.352 | NOTICE | [991]: 1970-01-01T08:00:57.223Z GPU thread:139904737173504 size:11272192 | 20260506 03:37:13.352 | NOTICE | [992]: 1970-01-01T08:00:57.250Z GPU thread:139904741371904 size:11272192 | 20260506 03:37:13.352 | NOTICE | [993]: 1970-01-01T08:00:57.276Z GPU thread:139904762363904 size:11272192 | 20260506 03:37:13.352 | NOTICE | [994]: 1970-01-01T08:00:57.302Z GPU thread:139904766562304 size:11272192 | 20260506 03:37:13.352 | NOTICE | [995]: 1970-01-01T08:00:57.417Z GPU thread:139904758165504 size:11272192 | 20260506 03:37:13.352 | NOTICE | [996]: 1970-01-01T08:00:57.443Z GPU thread:139904732975104 size:11272192 | 20260506 03:37:13.352 | NOTICE | [997]: 1970-01-01T08:00:57.469Z GPU thread:139904741371904 size:11272192 | 20260506 03:37:13.352 | NOTICE | [998]: 1970-01-01T08:00:57.496Z GPU thread:139904766562304 size:11272192 | 20260506 03:37:13.352 | NOTICE | [999]: 1970-01-01T08:00:57.522Z GPU thread:139904762363904 size:11272192 | 20260506 03:37:13.352 | NOTICE | [1000]: 1970-01-01T08:00:57.548Z GPU thread:139904770760704 size:11272192 | 20260506 03:37:13.352 | NOTICE | [1001]: 1970-01-01T08:00:57.575Z GPU thread:139904758165504 size:11272192 | 20260506 03:37:13.352 | NOTICE | [1002]: 1970-01-01T08:00:57.690Z GPU thread:139904753967104 size:11272192 | 20260506 03:37:13.352 | NOTICE | [1003]: 1970-01-01T08:00:57.715Z GPU thread:139904766562304 size:11272192 | 20260506 03:37:13.352 | NOTICE | [1004]: 1970-01-01T08:00:57.742Z GPU thread:139904758165504 size:11272192 | 20260506 03:37:13.352 | NOTICE | [1005]: 1970-01-01T08:00:57.768Z GPU thread:139904749768704 size:11272192 | 20260506 03:37:13.353 | NOTICE | [1006]: 1970-01-01T08:00:57.794Z GPU thread:139904732975104 size:11272192 | 20260506 03:37:13.353 | NOTICE | [1007]: 1970-01-01T08:00:57.820Z GPU thread:139904728776704 size:11272192 | 20260506 03:37:13.353 | NOTICE | [1008]: 1970-01-01T08:00:57.846Z GPU thread:139904770760704 size:11272192 | 20260506 03:37:13.353 | NOTICE | [1009]: 1970-01-01T08:00:57.961Z GPU thread:139904762363904 size:11272192 | 20260506 03:37:13.353 | NOTICE | [1010]: 1970-01-01T08:00:57.988Z GPU thread:139904749768704 size:11272192 | 20260506 03:37:13.353 | NOTICE | [1011]: 1970-01-01T08:00:58.014Z GPU thread:139904766562304 size:11272192 | 20260506 03:37:13.353 | NOTICE | [1012]: 1970-01-01T08:00:58.040Z GPU thread:139904758165504 size:11272192 | 20260506 03:37:13.353 | NOTICE | [1013]: 1970-01-01T08:00:58.066Z GPU thread:139904732975104 size:11272192 | 20260506 03:37:13.353 | NOTICE | [1014]: 1970-01-01T08:00:58.092Z GPU thread:139904737173504 size:11272192 | 20260506 03:37:13.353 | NOTICE | [1015]: 1970-01-01T08:00:58.118Z GPU thread:139904753967104 size:11272192 | 20260506 03:37:13.353 | NOTICE | [1016]: 1970-01-01T08:00:58.230Z GPU thread:139904766562304 size:11272192 | 20260506 03:37:13.353 | NOTICE | [1017]: 1970-01-01T08:00:58.255Z GPU thread:139904770760704 size:11272192 | 20260506 03:37:13.353 | NOTICE | [1018]: 1970-01-01T08:00:58.280Z GPU thread:139904753967104 size:11272192 | 20260506 03:37:13.353 | NOTICE | [1019]: 1970-01-01T09:17:00.838Z HOST thread:139904745570304 size:38273024 | 20260506 03:37:13.353 | NOTICE | [1020]: 1970-01-01T09:17:00.867Z GPU thread:139904732975104 size:38273024 | 20260506 03:37:13.353 | NOTICE | [1021]: 1970-01-01T09:17:01.077Z HOST thread:139904762363904 size:108003328 | 20260506 03:37:13.353 | NOTICE | [1022]: 1970-01-01T09:17:01.344Z HOST thread:139904762363904 size:2097156 | 20260506 03:37:13.353 | NOTICE | [1023]: 1970-01-01T19:58:54.764Z GPU thread:139904749768704 size:2097156 | 20260506 03:37:13.353 | NOTICE | ----- PBCudaRuntime DumpAllocationHistoryToLog ---- (end) | 20260506 03:37:13.353 | INFO | Application done | 20260506 03:37:13.353 | INFO | Run Time : 1d 52s | 20260506 03:37:13.353 | INFO | CPU Time : 2d 2h | 20260506 03:37:13.353 | INFO | Peak RSS : 57.518 GB | 20260506 03:37:13.353 | INFO | Exit code : 0